diff --git a/.bumpversion.cfg b/.bumpversion.cfg index bf67e4798de6..57fbc2cdd463 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.50.21 +current_version = 0.50.44 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-[a-z]+)? diff --git a/.devcontainer/destination-duckdb/devcontainer.json b/.devcontainer/destination-duckdb/devcontainer.json new file mode 100644 index 000000000000..10a9ff76d3e6 --- /dev/null +++ b/.devcontainer/destination-duckdb/devcontainer.json @@ -0,0 +1,80 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/python +{ + "name": "DuckDB Destination Connector DevContainer (Python)", + + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "image": "mcr.microsoft.com/devcontainers/python:0-3.10", + + // Features to add to the dev container. More info: https://containers.dev/features. + "features": { + "ghcr.io/devcontainers-contrib/features/poetry:2": {}, + "ghcr.io/devcontainers/features/docker-in-docker": {} + }, + "overrideFeatureInstallOrder": [ + // Deterministic order maximizes cache reuse + "ghcr.io/devcontainers-contrib/features/poetry", + "ghcr.io/devcontainers/features/docker-in-docker" + ], + + "workspaceFolder": "/workspaces/airbyte/airbyte-integrations/connectors/destination-duckdb", + + // Configure tool-specific properties. + "customizations": { + "vscode": { + "extensions": [ + // Python extensions: + "charliermarsh.ruff", + "matangover.mypy", + "ms-python.black-formatter", + "ms-python.python", + "ms-python.vscode-pylance", + + // Toml support + "tamasfe.even-better-toml", + + // Yaml and JSON Schema support: + "redhat.vscode-yaml", + + // Contributing: + "GitHub.vscode-pull-request-github" + ], + "settings": { + "extensions.ignoreRecommendations": true, + "git.autofetch": true, + "git.openRepositoryInParentFolders": "always", + "python.defaultInterpreterPath": ".venv/bin/python", + "python.interpreter.infoVisibility": "always", + "python.terminal.activateEnvironment": true, + "python.testing.pytestEnabled": true, + "python.testing.cwd": "/workspaces/airbyte/airbyte-integrations/connectors/destination-duckdb", + "python.testing.pytestArgs": [ + "--rootdir=/workspaces/airbyte/airbyte-integrations/connectors/destination-duckdb", + "." + ], + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter" + } + } + } + }, + "containerEnv": { + "POETRY_VIRTUALENVS_IN_PROJECT": "true" + }, + + // Mark the root directory as 'safe' for git. + "initializeCommand": "git config --add safe.directory /workspaces/airbyte", + + // Use 'postCreateCommand' to run commands after the container is created. + // Post-create tasks: + // 1. Create a symlink directory. + // 2. Create symlinks for the devcontainer.json and docs markdown file. + // 3. Install the Python/Poetry dependencies. + "postCreateCommand": "mkdir -p ./.symlinks && echo '*' > ./.symlinks/.gitignore && ln -sf /workspaces/airbyte/.devcontainer/destination-duckdb/devcontainer.json ./.symlinks/devcontainer.json && ln -sf /workspaces/airbyte/docs/integrations/destinations/duckdb.md ./.symlinks/duckdb-docs.md && poetry install" + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "root" +} diff --git a/.devcontainer/java-connectors-generic/devcontainer.json b/.devcontainer/java-connectors-generic/devcontainer.json new file mode 100644 index 000000000000..c35b8502dd77 --- /dev/null +++ b/.devcontainer/java-connectors-generic/devcontainer.json @@ -0,0 +1,58 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +{ + "name": "Java Development DevContainer (Generic)", + + "image": "mcr.microsoft.com/devcontainers/java:0-17", + "features": { + "ghcr.io/devcontainers/features/docker-in-docker": {}, + "ghcr.io/devcontainers/features/java:1": { + "installGradle": true, + "version": "latest", + "jdkDistro": "open", + "gradleVersion": "7.5.1" + }, + // Python needed for `airbyte-ci` CLI + "ghcr.io/devcontainers/features/python:1": { + "installGradle": true, + "version": "3.10", + "installTools": true + }, + "ghcr.io/devcontainers-contrib/features/poetry:2": {} + }, + + // Deterministic order reduces cache busting + "overrideFeatureInstallOrder": [ + "ghcr.io/devcontainers/features/docker-in-docker", + "ghcr.io/devcontainers/features/java", + "ghcr.io/devcontainers/features/python", + "ghcr.io/devcontainers-contrib/features/poetry" + ], + + // Configure tool-specific properties. + "customizations": { + "vscode": { + "extensions": ["vscjava.vscode-gradle", "tamasfe.even-better-toml"], + "settings": { + "extensions.ignoreRecommendations": true, + "git.openRepositoryInParentFolders": "always" + } + } + }, + + // Mark the root directory as 'safe' for git. + "initializeCommand": "git config --add safe.directory /workspaces/airbyte", + + // Install Gradle, `airbyte-ci` CLI, and Dagger (installed via airbyte-ci --help) + "postCreateCommand": "./gradlew --version && pipx install --editable ./airbyte-ci/connectors/pipelines/ || airbyte-ci --help || true", + + "containerEnv": { + // Deterministic Poetry virtual env location: `./.venv` + "POETRY_VIRTUALENVS_IN_PROJECT": "true" + } + + // Override to change the directory that the IDE opens by default: + // "workspaceFolder": "/workspaces/airbyte" + + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "root" +} diff --git a/.devcontainer/python-connectors-generic/devcontainer.json b/.devcontainer/python-connectors-generic/devcontainer.json new file mode 100644 index 000000000000..539a80499800 --- /dev/null +++ b/.devcontainer/python-connectors-generic/devcontainer.json @@ -0,0 +1,65 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +{ + "name": "Python Development DevContainer (Generic)", + + "image": "mcr.microsoft.com/devcontainers/python:0-3.10", + "features": { + "ghcr.io/devcontainers/features/docker-in-docker": {}, + "ghcr.io/devcontainers/features/python:1": { + "installGradle": true, + "version": "3.10", + "installTools": true + }, + "ghcr.io/devcontainers-contrib/features/poetry:2": {} + }, + + // Deterministic order reduces cache busting + "overrideFeatureInstallOrder": [ + "ghcr.io/devcontainers/features/docker-in-docker", + "ghcr.io/devcontainers/features/python", + "ghcr.io/devcontainers-contrib/features/poetry" + ], + + // Configure tool-specific properties. + "customizations": { + "vscode": { + "extensions": [ + // Python extensions: + "charliermarsh.ruff", + "matangover.mypy", + "ms-python.python", + "ms-python.vscode-pylance", + + // Toml support + "tamasfe.even-better-toml", + + // Yaml and JSON Schema support: + "redhat.vscode-yaml", + + // Contributing: + "GitHub.vscode-pull-request-github" + ], + "settings": { + "extensions.ignoreRecommendations": true, + "git.openRepositoryInParentFolders": "always" + } + } + }, + + // Mark the root directory as 'safe' for git. + "initializeCommand": "git config --add safe.directory /workspaces/airbyte", + + // Setup airbyte-ci on the container: + "postCreateCommand": "make tools.airbyte-ci-dev.install", + + "containerEnv": { + // Deterministic Poetry virtual env location: `./.venv` + "POETRY_VIRTUALENVS_IN_PROJECT": "true" + } + + // Override to change the directory that the IDE opens by default: + // "workspaceFolder": "/workspaces/airbyte" + + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "root" +} diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index bd3a2d1b2e00..1c76017e418d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,7 +1,15 @@ +# Vector db connectors +/airbyte-integrations/connectors/destination-pinecone @airbytehq/ai-language-models +/airbyte-integrations/connectors/destination-weaviate @airbytehq/ai-language-models +/airbyte-integrations/connectors/destination-milvus @airbytehq/ai-language-models +/airbyte-integrations/connectors/destination-qdrant @airbytehq/ai-language-models +/airbyte-integrations/connectors/destination-chroma @airbytehq/ai-language-models +/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based @airbytehq/ai-language-models + # CDK and Connector Acceptance Tests /airbyte-cdk/python @airbytehq/connector-extensibility /airbyte-integrations/connector-templates/ @airbytehq/connector-extensibility -/airbyte-integrations/bases/connector-acceptance-tests/ @airbytehq/connector-operations +/airbyte-integrations/bases/connector-acceptance-test/ @airbytehq/connector-operations @lazebnyi @oustynova # Protocol related items /docs/understanding-airbyte/airbyte-protocol.md @airbytehq/protocol-reviewers @@ -30,6 +38,9 @@ /airbyte-integrations/connectors/source-tidb/ @airbytehq/dbsources # Java-based destination connectors +airbyte-cdk/java/airbyte-cdk/db-destinations/ @airbytehq/destinations +airbyte-cdk/java/airbyte-cdk/s3-destinations/ @airbytehq/destinations +airbyte-cdk/java/airbyte-cdk/typing-deduping/ @airbytehq/destinations /airbyte-integrations/bases/standard-destination-test/ @airbytehq/destinations /airbyte-integrations/bases/base-java-s3/ @airbytehq/destinations /airbyte-integrations/bases/bases-destination-jdbc/ @airbytehq/destinations @@ -49,3 +60,9 @@ /airbyte-integrations/connectors/destination-s3/ @airbytehq/destinations /airbyte-integrations/connectors/destination-snowflake/ @airbytehq/destinations /airbyte-integrations/connectors/destination-tidb/ @airbytehq/destinations + +# Build customization file change +/airbyte-integrations/connectors/**/build_customization.py @airbytehq/connector-operations + +# airbyte-ci +/airbyte-ci @airbytehq/connector-operations diff --git a/.github/ISSUE_TEMPLATE/1-issue-connector.yaml b/.github/ISSUE_TEMPLATE/1-issue-connector.yaml index 6eafc59c79bf..30a22868f32b 100644 --- a/.github/ISSUE_TEMPLATE/1-issue-connector.yaml +++ b/.github/ISSUE_TEMPLATE/1-issue-connector.yaml @@ -20,12 +20,12 @@ body: Make sure you update this issue with a concise title and provide all information you have to help us debug the problem together. Some examples of good titles following the convention:
    -
  • Source Name: issue description
  • -
  • Destination Name: issue description
  • -
  • Source Postgres: Add `_ab_cdc_inserted_at` column for CDC syncs
  • -
  • Source Chargebee: cannot sync transaction objects
  • -
  • Source Snowflake: support for case sensitive parameters in connection string
  • -
  • Destination BigQuery: normalization incorrectly processes arrays
  • +
  • [source-name] issue description
  • +
  • [destination-name] issue description
  • +
  • [source-postgres] Add `_ab_cdc_inserted_at` column for CDC syncs
  • +
  • [source-chargebee] cannot sync transaction objects
  • +
  • [source-snowflake] support for case sensitive parameters in connection string
  • +
  • [destination-bigquery] typing incorrectly arrays

Issues not following the template will be closed.

@@ -61,7 +61,7 @@ body: - type: textarea id: description attributes: - label: Revelant information + label: Relevant information description: Please give any additional information you have and steps to reproduce the problem. - type: textarea id: logs diff --git a/.github/ISSUE_TEMPLATE/2-issue-docker.yaml b/.github/ISSUE_TEMPLATE/2-issue-docker.yaml new file mode 100644 index 000000000000..f0efb4d501f8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/2-issue-docker.yaml @@ -0,0 +1,52 @@ +name: 🐛 [docker] Report a platform, infra or deployment bug +description: Use this template when you have a problem operating Airbyte platform on Docker +labels: [type/bug, area/platform, needs-triage, docker] +body: + - type: markdown + attributes: + value: > +

+ + + + octavia-welcome + + +

+ - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report... + Make sure to update this issue with a concise title and provide all information you have to + help us debug the problem together. Issues not following the template will be closed. + - type: input + id: platform-version + attributes: + label: Platform Version + description: "Some examples are: (eg. 0.44.1, 0.30.0), you can find the version in the left bottom in Airbyte UI" + validations: + required: true + - type: dropdown + id: step + attributes: + label: What step the error happened? + multiple: false + options: + - On deploy + - During the Sync + - Upgrading the Platform or Helm Chart + - Other + - type: textarea + id: description + attributes: + label: Revelant information + description: Please give any additional information you have and steps to reproduce the problem. + - type: textarea + id: logs + attributes: + label: Relevant log output + description: | + Please copy and paste any relevant log output. + This will be automatically formatted into code, so no need for backticks. + We strongly recommend to upload the log file for further debugging. + render: shell diff --git a/.github/ISSUE_TEMPLATE/2-issue-helm.yaml b/.github/ISSUE_TEMPLATE/2-issue-helm.yaml new file mode 100644 index 000000000000..2dfde346fd2e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/2-issue-helm.yaml @@ -0,0 +1,52 @@ +name: 🐛 [helm] Report a platform, infra or deployment bug +description: Use this template when you have a problem operating Airbyte platform on Helm/Kubernetes +labels: [type/bug, area/platform, needs-triage] +body: + - type: markdown + attributes: + value: > +

+ + + + octavia-welcome + + +

+ - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report... + Make sure to update this issue with a concise title and provide all information you have to + help us debug the problem together. Issues not following the template will be closed. + - type: input + id: platform-version + attributes: + label: Helm Chart Version + description: "What is the Helm Chart App version you're using" + validations: + required: true + - type: dropdown + id: step + attributes: + label: What step the error happened? + multiple: false + options: + - On deploy + - During the Sync + - Upgrading the Platform or Helm Chart + - Other + - type: textarea + id: description + attributes: + label: Revelant information + description: Please give any additional information you have and steps to reproduce the problem. + - type: textarea + id: logs + attributes: + label: Relevant log output + description: | + Please copy and paste any relevant log output. + This will be automatically formatted into code, so no need for backticks. + We strongly recommend to upload the log file for further debugging. + render: shell diff --git a/.github/ISSUE_TEMPLATE/2-issue-platform.yaml b/.github/ISSUE_TEMPLATE/2-issue-platform.yaml deleted file mode 100644 index 66ef7be94999..000000000000 --- a/.github/ISSUE_TEMPLATE/2-issue-platform.yaml +++ /dev/null @@ -1,62 +0,0 @@ -name: 🐛 Report a platform, infra or deployment bug -description: Use this template when you have a problem operating Airbyte platform -labels: [type/bug, area/platform, needs-triage] -body: - - type: markdown - attributes: - value: > -

- - - - octavia-welcome - - -

- - type: markdown - attributes: - value: | - Thanks for taking the time to fill out this bug report... - Make sure to update this issue with a concise title and provide all information you have to - help us debug the problem together. Issues not following the template will be closed. - - type: dropdown - id: deploy - validations: - required: true - attributes: - label: What method are you using to run Airbyte? - multiple: false - options: - - Docker - - Kubernetes - - type: input - id: platform-version - attributes: - label: Platform Version or Helm Chart Version - description: "Some examples are: (eg. 0.44.1, 0.30.0), you can find the version in the left bottom in Airbyte UI or in the .env / value.yaml file" - validations: - required: true - - type: dropdown - id: step - attributes: - label: What step the error happened? - multiple: false - options: - - On deploy - - During the Sync - - Upgrading the Platform or Helm Chart - - Other - - type: textarea - id: description - attributes: - label: Revelant information - description: Please give any additional information you have and steps to reproduce the problem. - - type: textarea - id: logs - attributes: - label: Relevant log output - description: | - Please copy and paste any relevant log output. - This will be automatically formatted into code, so no need for backticks. - We strongly recommend to upload the log file for further debugging. - render: shell diff --git a/.github/ISSUE_TEMPLATE/3-issue-cli.yaml b/.github/ISSUE_TEMPLATE/3-issue-cli.yaml deleted file mode 100644 index e4b7ee5b5d6a..000000000000 --- a/.github/ISSUE_TEMPLATE/3-issue-cli.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: 🐛 Report a problem while using the Octavia CLI -description: Use this template when you discovered a CLI problem -labels: [type/bug, area/octavia-cli, needs-triage] -body: - - type: markdown - attributes: - value: > -

- - - - octavia-welcome - - -

- - type: markdown - attributes: - value: | - Thanks for taking the time to fill out this bug report... - Make sure you update this issue with a concise title and provide all information you have to - help us debug the problem together. Issues not following the template will be closed. - - type: input - id: cli-version - attributes: - label: Octavia CLI Version - description: Give the Octavia CLI version you're using. - validations: - required: true - - type: textarea - id: description - attributes: - label: Revelant information - description: Please give any additional information you have and steps to reproduce the problem. - - type: textarea - id: logs - attributes: - label: Relevant log output - description: | - Please copy and paste any relevant log output. - This will be automatically formatted into code, so no need for backticks. - We strongly recommend to upload the log file for further debugging. - render: shell diff --git a/.github/ISSUE_TEMPLATE/5-feature-new-connector.yaml b/.github/ISSUE_TEMPLATE/5-feature-new-connector.yaml deleted file mode 100644 index 171cc65a23fb..000000000000 --- a/.github/ISSUE_TEMPLATE/5-feature-new-connector.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: ✨ Request a new connector -description: Request for new sources and destinations -labels: [area/connectors, new-connector] -body: - - type: input - id: connector-name - attributes: - label: Connector Name - description: What is the service or database you want to integrate - validations: - required: true - - type: dropdown - id: type - attributes: - label: What type of integration - multiple: false - options: - - Source - - Destination - - type: textarea - id: description - attributes: - label: Revelant Information - description: >- - Why do you need this integration? How does your team intend to use the data? This helps us understand the use case. - How often do you want to run syncs? - If this is an API source connector, which entities/endpoints do you need supported? - If the connector is for a paid service, can we name you as a mutual user when we subscribe for an account? Which company should we name? - - type: checkboxes - id: submit-pr - attributes: - label: Contribute - description: Are you willing to submit the fix? - options: - - label: Yes, I want to contribute - required: false diff --git a/.github/ISSUE_TEMPLATE/6-feature-request.yaml b/.github/ISSUE_TEMPLATE/6-feature-request.yaml deleted file mode 100644 index b0ba7383c7f1..000000000000 --- a/.github/ISSUE_TEMPLATE/6-feature-request.yaml +++ /dev/null @@ -1,20 +0,0 @@ -name: ✨ Request a new feature -description: Request to any new Airbyte features -labels: [type/enhancement, needs-triage] -body: - - type: dropdown - id: type - attributes: - label: What area the feature impact? - multiple: false - options: - - Connectors - - Airbyte Platform - - Helm Chart - - type: textarea - id: description - attributes: - label: Revelant Information - description: >- - What are you trying to do, and why is it hard? A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] - A clear and concise description of what you want to see happen, or the change you would like to see diff --git a/.github/ISSUE_TEMPLATE/7-low-code-component.yaml b/.github/ISSUE_TEMPLATE/7-low-code-component.yaml deleted file mode 100644 index 737b19b092ca..000000000000 --- a/.github/ISSUE_TEMPLATE/7-low-code-component.yaml +++ /dev/null @@ -1,38 +0,0 @@ -name: ✨ Request a new low-code CDK component -description: Use this to request new low-code CDK -labels: - [ - "team/extensibility", - "area/connector-builder", - "area/low-code/components", - "area/low-code", - ] -body: - - type: textarea - id: description - attributes: - label: Component Description - description: Please describe the component you would like to see added to the Low-code CDK and why it's valuable - validations: - required: true - - type: textarea - id: proposed-schema - attributes: - label: Proposed YAML schema - description: If you can wave a magic wand, what would the YAML schema of the component you are suggesting look like? - value: | - ```yaml - ... - ``` - validations: - required: true - - type: input - id: url - attributes: - label: Connector API Docs URL - description: To help us understand your request, please share a link to the API docs for the API you're trying to build, ideally pointing to the specific section of the docs relevant to your use case - - type: input - id: pr-url - attributes: - label: Pull Request URL - description: If there is a pull request which currently implements a custom component in lieu of having this component in the CDK, please link it diff --git a/.github/ISSUE_TEMPLATE/9-other.yaml b/.github/ISSUE_TEMPLATE/9-other.yaml index 9c2c872d16e4..c5132248a6bd 100644 --- a/.github/ISSUE_TEMPLATE/9-other.yaml +++ b/.github/ISSUE_TEMPLATE/9-other.yaml @@ -12,5 +12,5 @@ body: - type: textarea id: description attributes: - label: Revelant information + label: Relevant information description: Please give any aditional information. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000000..12a69ca2c40e --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,6 @@ +--- +blank_issues_enabled: false +contact_links: + - name: Ask a question, get community support or request new features/connectors + url: https://github.com/airbytehq/airbyte/discussions/ + about: Use Github Discussion to request features/connectors or discuss ideas or issues. diff --git a/.github/actions/airbyte-ci-requirements/action.yml b/.github/actions/airbyte-ci-requirements/action.yml new file mode 100644 index 000000000000..d12a7c1b6b10 --- /dev/null +++ b/.github/actions/airbyte-ci-requirements/action.yml @@ -0,0 +1,104 @@ +name: "Get airbyte-ci runner name" +description: "Runs a given airbyte-ci command with the --ci-requirements flag to get the CI requirements for a given command" +inputs: + runner_type: + description: "Type of runner to get requirements for. One of: format, test, nightly, publish" + required: true + runner_size: + description: "One of: format, test, nightly, publish" + required: true + airbyte_ci_command: + description: "airbyte-ci command to get CI requirements for." + required: true + runner_name_prefix: + description: "Prefix of runner name" + required: false + default: ci-runner-connector + github_token: + description: "GitHub token" + required: true + sentry_dsn: + description: "Sentry DSN" + required: false + airbyte_ci_binary_url: + description: "URL to airbyte-ci binary" + required: false + default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci + +runs: + using: "composite" + steps: + - name: Check if PR is from a fork + if: github.event_name == 'pull_request' + shell: bash + run: | + if [ "${{ github.event.pull_request.head.repo.fork }}" == "true" ]; then + echo "PR is from a fork. Exiting workflow..." + exit 78 + fi + + - name: Get changed files + uses: tj-actions/changed-files@v39 + id: changes + with: + files_yaml: | + pipelines: + - 'airbyte-ci/connectors/pipelines/**' + + - name: Determine how Airbyte CI should be installed + shell: bash + id: determine-install-mode + run: | + if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then + echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source." + echo "install-mode=dev" >> $GITHUB_OUTPUT + else + echo "install-mode=production" >> $GITHUB_OUTPUT + fi + + - name: Install airbyte-ci binary + id: install-airbyte-ci + if: steps.determine-install-mode.outputs.install-mode == 'production' + shell: bash + run: | + curl -sSL ${{ inputs.airbyte_ci_binary_url }} --output airbyte-ci-bin + sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci + sudo chmod +x /usr/local/bin/airbyte-ci + + - name: Install Python 3.10 + uses: actions/setup-python@v4 + if: steps.determine-install-mode.outputs.install-mode == 'dev' + with: + python-version: "3.10" + token: ${{ inputs.github_token }} + + - name: Install ci-connector-ops package + if: steps.determine-install-mode.outputs.install-mode == 'dev' + shell: bash + run: | + pip install pipx + pipx ensurepath + pipx install airbyte-ci/connectors/pipelines/ + + - name: Get dagger version from airbyte-ci + id: get-dagger-version + shell: bash + run: | + dagger_version=$(airbyte-ci ${{ inputs.airbyte_ci_command }} --ci-requirements | tail -n 1 | jq -r '.dagger_version') + echo "dagger_version=${dagger_version}" >> "$GITHUB_OUTPUT" + + - name: Get runner name + id: get-runner-name + shell: bash + run: | + runner_name_prefix=${{ inputs.runner_name_prefix }} + runner_type=${{ inputs.runner_type }} + runner_size=${{ inputs.runner_size }} + dashed_dagger_version=$(echo "${{ steps.get-dagger-version.outputs.dagger_version }}" | tr '.' '-') + runner_name="${runner_name_prefix}-${runner_type}-${runner_size}-dagger-${dashed_dagger_version}" + echo ${runner_name} + echo "runner_name=${runner_name}" >> "$GITHUB_OUTPUT" +outputs: + runner_name: + description: "Name of self hosted CI runner to use" + value: ${{ steps.get-runner-name.outputs.runner_name }} diff --git a/.github/actions/cache-build-artifacts/action.yml b/.github/actions/cache-build-artifacts/action.yml deleted file mode 100644 index abc91420f43d..000000000000 --- a/.github/actions/cache-build-artifacts/action.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: "Cache Build Artifacts" -description: "Cache Java, Javascript and Python build artifacts to reduce build time" -inputs: - cache-key: - description: "Key to use for caching" - required: true - -runs: - using: "composite" - steps: - - name: Pip Caching - if: ${{ inputs.cache_python }} == 'true' - uses: actions/cache@v3 - with: - path: | - ~/.cache/pip - key: ${{ inputs.cache-key }}-pip-${{ runner.os }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - ${{ inputs.cache-key }}-pip-${{ runner.os }}- - - - name: Npm Caching - uses: actions/cache@v3 - with: - path: | - ~/.npm - key: ${{ inputs.cache-key }}-npm-${{ runner.os }}-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ inputs.cache-key }}-npm-${{ runner.os }}- - - - name: pnpm Caching - uses: actions/cache@v3 - with: - path: | - ~/.local/share/pnpm/store - key: ${{ inputs.cache-key }}-pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }} - restore-keys: | - ${{ inputs.cache-key }}-pnpm-${{ runner.os }}- - - # this intentionally does not use restore-keys so we don't mess with gradle caching - - name: Gradle and Python Caching - uses: actions/cache@v3 - with: - path: | - ~/.gradle/caches - ~/.gradle/wrapper - **/.venv - key: ${{ inputs.cache-key }}-${{ runner.os }}-${{ hashFiles('**/*.gradle*') }}-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }} diff --git a/.github/actions/ci-java-tests/action.yml b/.github/actions/ci-java-tests/action.yml deleted file mode 100644 index 7f7b3ea302b7..000000000000 --- a/.github/actions/ci-java-tests/action.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: "Runner CI Java Tests" -description: "Runner CI Java Tests" -inputs: - module-name: - required: true - module-folder: - required: true - -runs: - using: "composite" - steps: - - name: "Build" - shell: bash - run: | - rm -rf ${{ inputs.module-folder }}/.venv ${{ inputs.module-folder }}/build - ROOT_DIR=$(git rev-parse --show-toplevel) - ARG=:$(python -c "import os; print(os.path.relpath('${{ inputs.module-folder }}', start='${ROOT_DIR}').replace('/', ':') )") - echo "./gradlew --no-daemon $ARG:build" - ./gradlew --no-daemon "$ARG:clean" - ./gradlew --no-daemon "$ARG:build" - - - diff --git a/.github/actions/ci-py-tests/action.yml b/.github/actions/ci-py-tests/action.yml deleted file mode 100644 index f70fbbbaa20c..000000000000 --- a/.github/actions/ci-py-tests/action.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: "Runner CI Python Tests" -description: "Runner CI Python Tests" -inputs: - module-name: - required: true - module-folder: - required: true -outputs: - coverage-paths: - description: "Coverage Paths" - value: ${{ steps.build-coverage-reports.outputs.coverage-paths }} - flake8-logs: - description: "Flake8 Logs" - value: ${{ steps.build-linter-reports.outputs.flake8-logs }} - mypy-logs: - description: "MyPy Logs" - value: ${{ steps.build-linter-reports.outputs.mypy-logs }} - black-diff: - description: "Black Diff" - value: ${{ steps.build-linter-reports.outputs.black-diff }} - isort-diff: - description: "Isort Diff" - value: ${{ steps.build-linter-reports.outputs.isort-diff }} -runs: - using: "composite" - steps: - - name: Build Coverage Reports - id: build-coverage-reports - shell: bash - run: | - GRADLE_JOB=$(source ./tools/lib/lib.sh; full_path_to_gradle_path ${{ inputs.module-folder }} "unitTest") - REPORT_FOLDER="${{ inputs.module-folder }}/coverage/" - ./gradlew --no-daemon -Preports_folder=${REPORT_FOLDER} ${GRADLE_JOB} - - echo "coverage-paths=coverage/coverage.xml" >> $GITHUB_OUTPUT - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 - with: - file: ${{ steps.build-coverage-reports.outputs.coverage-paths }} - name: "UnitTests of ${{ inputs.module-name }}" - - - name: Build Linter Reports - id: build-linter-reports - shell: bash - run: | - GRADLE_JOB=$(source ./tools/lib/lib.sh; full_path_to_gradle_path ${{ inputs.module-folder }} "airbytePythonReport") - REPORT_FOLDER="${{ inputs.module-folder }}/reports/" - ./gradlew --no-daemon -Preports_folder=${REPORT_FOLDER} ${GRADLE_JOB} - - echo "mypy-logs=reports/mypy.log" >> $GITHUB_OUTPUT - echo "black-diff=reports/black.diff" >> $GITHUB_OUTPUT - echo "isort-diff=reports/isort.diff" >> $GITHUB_OUTPUT - echo "flake8-logs=reports/flake.txt" >> $GITHUB_OUTPUT diff --git a/.github/actions/run-dagger-pipeline/action.yml b/.github/actions/run-dagger-pipeline/action.yml index 8d413a9c658f..8d28ea9e788c 100644 --- a/.github/actions/run-dagger-pipeline/action.yml +++ b/.github/actions/run-dagger-pipeline/action.yml @@ -10,12 +10,20 @@ inputs: github_token: description: "GitHub token" required: true + dagger_cloud_token: + description: "Dagger Cloud token" + required: true docker_hub_username: description: "Dockerhub username" required: true docker_hub_password: description: "Dockerhub password" required: true + docker_registry_mirror_url: + description: "Docker registry mirror URL (not including http or https)" + required: false + # Do not use http or https here + default: "ci-dockerhub-registry.airbyte.com" options: description: "Options for the subcommand" required: false @@ -63,6 +71,19 @@ inputs: ci_job_key: description: "CI job key" required: false + s3_build_cache_access_key_id: + description: "Gradle S3 Build Cache AWS access key ID" + required: false + s3_build_cache_secret_key: + description: "Gradle S3 Build Cache AWS secret key" + required: false + tailscale_auth_key: + description: "Tailscale auth key" + airbyte_ci_binary_url: + description: "URL to airbyte-ci binary" + required: false + default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci + runs: using: "composite" steps: @@ -74,33 +95,67 @@ runs: echo "PR is from a fork. Exiting workflow..." exit 78 fi + + - name: Get changed files + uses: tj-actions/changed-files@v39 + id: changes + with: + files_yaml: | + pipelines: + - 'airbyte-ci/connectors/pipelines/**' + + - name: Determine how Airbyte CI should be installed + shell: bash + id: determine-install-mode + run: | + if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then + echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source." + echo "install-mode=dev" >> $GITHUB_OUTPUT + else + echo "install-mode=production" >> $GITHUB_OUTPUT + fi + - name: Docker login uses: docker/login-action@v1 with: username: ${{ inputs.docker_hub_username }} password: ${{ inputs.docker_hub_password }} + - name: Get start timestamp id: get-start-timestamp shell: bash run: echo "name=start-timestamp=$(date +%s)" >> $GITHUB_OUTPUT + + - name: Install airbyte-ci binary + id: install-airbyte-ci + if: steps.determine-install-mode.outputs.install-mode == 'production' + shell: bash + run: | + curl -sSL ${{ inputs.airbyte_ci_binary_url }} --output airbyte-ci-bin + sudo mv airbyte-ci-bin /usr/local/bin/airbyte-ci + sudo chmod +x /usr/local/bin/airbyte-ci + - name: Install Python 3.10 uses: actions/setup-python@v4 + if: steps.determine-install-mode.outputs.install-mode == 'dev' with: python-version: "3.10" token: ${{ inputs.github_token }} + - name: Install ci-connector-ops package + if: steps.determine-install-mode.outputs.install-mode == 'dev' shell: bash run: | pip install pipx pipx ensurepath pipx install airbyte-ci/connectors/pipelines/ + - name: Run airbyte-ci shell: bash run: | export _EXPERIMENTAL_DAGGER_RUNNER_HOST="unix:///var/run/buildkit/buildkitd.sock" - airbyte-ci-internal --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }} + airbyte-ci --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }} env: - _EXPERIMENTAL_DAGGER_CLOUD_TOKEN: "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k" CI_CONTEXT: "${{ inputs.context }}" CI_GIT_BRANCH: ${{ inputs.git_branch || github.head_ref }} CI_GIT_REVISION: ${{ inputs.git_revision || github.sha }} @@ -108,6 +163,7 @@ runs: CI_JOB_KEY: ${{ inputs.ci_job_key }} CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }} CI_REPORT_BUCKET_NAME: ${{ inputs.report_bucket_name }} + DAGGER_CLOUD_TOKEN: "${{ inputs.dagger_cloud_token }}" GCP_GSM_CREDENTIALS: ${{ inputs.gcp_gsm_credentials }} GCS_CREDENTIALS: ${{ inputs.gcs_credentials }} METADATA_SERVICE_BUCKET_NAME: ${{ inputs.metadata_service_bucket_name }} @@ -115,9 +171,14 @@ runs: PRODUCTION: ${{ inputs.production }} PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} SENTRY_DSN: ${{ inputs.sentry_dsn }} + SENTRY_ENVIRONMENT: ${{ steps.determine-install-mode.outputs.install-mode }} SLACK_WEBHOOK: ${{ inputs.slack_webhook_url }} SPEC_CACHE_BUCKET_NAME: ${{ inputs.spec_cache_bucket_name }} SPEC_CACHE_GCS_CREDENTIALS: ${{ inputs.spec_cache_gcs_credentials }} DOCKER_HUB_USERNAME: ${{ inputs.docker_hub_username }} DOCKER_HUB_PASSWORD: ${{ inputs.docker_hub_password }} + S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ inputs.s3_build_cache_access_key_id }} + S3_BUILD_CACHE_SECRET_KEY: ${{ inputs.s3_build_cache_secret_key }} CI: "True" + TAILSCALE_AUTH_KEY: ${{ inputs.tailscale_auth_key }} + DOCKER_REGISTRY_MIRROR_URL: ${{ inputs.docker_registry_mirror_url }} diff --git a/.github/actions/runner-prepare-for-build/action.yml b/.github/actions/runner-prepare-for-build/action.yml index cb6df8742f2e..cb5a890a968a 100644 --- a/.github/actions/runner-prepare-for-build/action.yml +++ b/.github/actions/runner-prepare-for-build/action.yml @@ -1,18 +1,18 @@ -name: 'Prepare Runner for Building' -description: 'Prepare Runner for Building project' +name: "Prepare Runner for Building" +description: "Prepare Runner for Building project" inputs: install_java: - description: '' + description: "" required: false - default: 'true' + default: "true" install_node: - description: '' + description: "" required: false - default: 'true' + default: "true" install_python: - description: '' + description: "" required: false - default: 'true' + default: "true" runs: using: "composite" steps: diff --git a/.github/labeler.yml b/.github/labeler.yml index 877ecec49311..09aed0435b97 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,36 +1,8 @@ -# union of frontend, api, server, scheduler, protocol, worker, kubernetes -area/platform: - - airbyte-api/* - - airbyte-api/**/* - - airbyte-persistence/* - - airbyte-persistence/**/* - - airbyte-server/* - - airbyte-server/**/* - - airbyte-workers/* - - airbyte-workers/**/* - - kube/* - - kube/**/* - - charts/* - - charts/**/* - +# Union of api, connectors, documentation, octavia-cli, CDK and normalization. area/api: - airbyte-api/* - airbyte-api/**/* -area/server: - - airbyte-server/* - - airbyte-server/**/* - -area/worker: - - airbyte-workers/* - - airbyte-workers/**/* - -kubernetes: - - kube/* - - kube/**/* - - charts/* - - charts/**/* - area/connectors: - airbyte-integrations/connectors/* - airbyte-integrations/connectors/**/* @@ -39,6 +11,10 @@ area/documentation: - docs/* - docs/**/* +area/octavia-cli: + - octavia-cli/* + - octavia-cli/**/* + CDK: - airbyte-cdk/* - airbyte-cdk/**/* diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 476fd613df7e..85f751847748 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -79,9 +79,31 @@ If this is a community PR, the Airbyte engineer reviewing this PR is responsible
Connector Generator - Issue acceptance criteria met -- PR name follows [PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/issues-and-pull-requests#pull-request-title-convention) +- PR name follows [PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook) - If adding a new generator, add it to the [list of scaffold modules being tested](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connector-templates/generator/build.gradle#L41) -- The generator test modules (all connectors with `-scaffold` in their name) have been updated with the latest scaffold by running `./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates` then checking in your changes +- The generator test modules (all connectors with `-scaffold` in their name) have been updated with the latest scaffold by running `./gradlew :airbyte-integrations:connector-templates:generator:generateScaffolds` then checking in your changes - Documentation which references the generator is updated as needed
+ +
Updating the Python CDK + +### Airbyter + +Before merging: +- Pull Request description explains what problem it is solving +- Code change is unit tested +- Build and my-py check pass +- Smoke test the change on at least one affected connector + - On Github: Run [this workflow](https://github.com/airbytehq/airbyte/actions/workflows/connectors_tests.yml), passing `--use-local-cdk --name=source-` as options + - Locally: `airbyte-ci connectors --use-local-cdk --name=source- test` +- PR is reviewed and approved + +After merging: +- [Publish the CDK](https://github.com/airbytehq/airbyte/actions/workflows/publish-cdk-command-manually.yml) + - The CDK does not follow proper semantic versioning. Choose minor if this the change has significant user impact or is a breaking change. Choose patch otherwise. + - Write a thoughtful changelog message so we know what was updated. +- Merge the platform PR that was auto-created for updating the Connector Builder's CDK version + - This step is optional if the change does not affect the connector builder or declarative connectors. + +
diff --git a/.github/workflows/airbyte-ci-release.yml b/.github/workflows/airbyte-ci-release.yml new file mode 100644 index 000000000000..aad8f7629559 --- /dev/null +++ b/.github/workflows/airbyte-ci-release.yml @@ -0,0 +1,128 @@ +name: Connector Ops CI - Airbyte CI Release + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + paths: + - "airbyte-ci/connectors/pipelines/**" + workflow_dispatch: + +env: + DEV_GCS_BUCKET_NAME: dev-airbyte-cloud-connector-metadata-service + PROD_GCS_BUCKET_NAME: prod-airbyte-cloud-connector-metadata-service + BINARY_FILE_NAME: airbyte-ci + +jobs: + build: + runs-on: ${{ matrix.os }}-latest + strategy: + fail-fast: false + matrix: + os: ["ubuntu", "macos"] + + steps: + - name: Checkout Airbyte + id: checkout_airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.sha }} # This is required to make sure that the same commit is checked out on all runners + + - name: Get short SHA + id: get_short_sha + uses: benjlevesque/short-sha@v2.2 + + - name: Install Python + id: install_python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + + - name: Install Poetry + id: install_poetry + uses: snok/install-poetry@v1 + + - name: Install Dependencies + id: install_dependencies + working-directory: airbyte-ci/connectors/pipelines/ + run: poetry install --with dev + + - name: Build release + id: build_release + working-directory: airbyte-ci/connectors/pipelines/ + run: poetry run poe build-release-binary ${{ env.BINARY_FILE_NAME }} + + - uses: actions/upload-artifact@v2 + with: + name: airbyte-ci-${{ matrix.os }}-${{ steps.get_short_sha.outputs.sha }} + path: airbyte-ci/connectors/pipelines/dist/${{ env.BINARY_FILE_NAME }} + + - name: Authenticate to Google Cloud Dev + id: auth_dev + uses: google-github-actions/auth@v1 + with: + credentials_json: "${{ secrets.METADATA_SERVICE_DEV_GCS_CREDENTIALS }}" + + - name: Upload pre-release to GCS dev bucket + id: upload_pre_release_to_gcs + if: github.ref != 'refs/heads/master' + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: airbyte-ci/connectors/pipelines/dist/${{ env.BINARY_FILE_NAME }} + destination: ${{ env.DEV_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/${{ steps.get_short_sha.outputs.sha }} + headers: |- + cache-control:public, max-age=10 + + - name: Print pre-release public url + id: print_pre_release_public_url + run: | + echo "https://storage.googleapis.com/${{ env.DEV_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/${{ steps.get_short_sha.outputs.sha }}/${{ env.BINARY_FILE_NAME }}" + + # if master, upload per version and latest to prod bucket + + - name: Set version from poetry version --short + id: set_version + if: github.ref == 'refs/heads/master' + working-directory: airbyte-ci/connectors/pipelines/ + run: | + echo "::set-output name=version::$(poetry version --short)" + + - name: Authenticate to Google Cloud Prod + id: auth_prod + uses: google-github-actions/auth@v1 + with: + credentials_json: "${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}" + + - name: Upload version release to GCS prod bucket + id: upload_version_release_to_gcs + if: github.ref == 'refs/heads/master' + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: airbyte-ci/connectors/pipelines/dist/${{ env.BINARY_FILE_NAME }} + destination: ${{ env.PROD_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/${{ steps.set_version.outputs.version }} + headers: |- + cache-control:public, max-age=10 + + - name: Print release version public url + id: print_version_release_public_url + if: github.ref == 'refs/heads/master' + run: | + echo "https://storage.googleapis.com/${{ env.PROD_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/${{ steps.set_version.outputs.version }}/${{ env.BINARY_FILE_NAME }}" + + - name: Upload latest release to GCS prod bucket + id: upload_latest_release_to_gcs + if: github.ref == 'refs/heads/master' + uses: google-github-actions/upload-cloud-storage@v1 + with: + path: airbyte-ci/connectors/pipelines/dist/${{ env.BINARY_FILE_NAME }} + destination: ${{ env.PROD_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/latest + headers: |- + cache-control:public, max-age=10 + + - name: Print latest release public url + id: print_latest_release_public_url + if: github.ref == 'refs/heads/master' + run: | + echo "https://storage.googleapis.com/${{ env.PROD_GCS_BUCKET_NAME }}/airbyte-ci/releases/${{ matrix.os }}/latest/${{ env.BINARY_FILE_NAME }}" diff --git a/.github/workflows/airbyte-ci-tests.yml b/.github/workflows/airbyte-ci-tests.yml index c5e5e6edb12b..77f478c7da79 100644 --- a/.github/workflows/airbyte-ci-tests.yml +++ b/.github/workflows/airbyte-ci-tests.yml @@ -1,4 +1,4 @@ -name: Airbyte CI pipeline tests +name: Connector Ops CI - Pipeline Unit Test concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -11,24 +11,159 @@ on: - opened - reopened - synchronize - paths: - - airbyte-ci/** jobs: + get_ci_runner: + runs-on: ubuntu-latest + name: Get CI runner + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + fetch-depth: 1 + - name: Get CI runner + id: get_ci_runner + uses: ./.github/actions/airbyte-ci-requirements + with: + runner_type: "test" + runner_size: "large" + airbyte_ci_command: "test" + github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + outputs: + runner_name: ${{ steps.get_ci_runner.outputs.runner_name }} run-airbyte-ci-tests: name: Run Airbyte CI tests - runs-on: "conn-prod-xlarge-runner" + needs: get_ci_runner + runs-on: ${{ needs.get_ci_runner.outputs.runner_name }} steps: - name: Checkout Airbyte uses: actions/checkout@v3 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.ref }} + + # IMPORTANT! This is nessesary to make sure that a status is reported on the PR + # even if the workflow is skipped. If we used github actions filters, the workflow + # would not be reported as skipped, but instead would be forever pending. + # + # I KNOW THIS SOUNDS CRAZY, BUT IT IS TRUE. + # + # Also it gets worse + # + # IMPORTANT! DO NOT CHANGE THE QUOTES AROUND THE GLOBS. THEY ARE REQUIRED. + # MAKE SURE TO TEST ANY SYNTAX CHANGES BEFORE MERGING. + - name: Get changed files + uses: tj-actions/changed-files@v39 + id: changes + with: + files_yaml: | + ops: + - 'airbyte-ci/connectors/connector_ops/**' + - '!**/*.md' + base_images: + - 'airbyte-ci/connectors/connector_ops/**' + - 'airbyte-ci/connectors/base_images/**' + - '!**/*.md' + pipelines: + - 'airbyte-ci/connectors/connector_ops/**' + - 'airbyte-ci/connectors/base_images/**' + - 'airbyte-ci/connectors/pipelines/**' + - '!**/*.md' + metadata_lib: + - 'airbyte-ci/connectors/metadata_service/lib/**' + - '!**/*.md' + metadata_orchestrator: + - 'airbyte-ci/connectors/metadata_service/lib/**' + - 'airbyte-ci/connectors/metadata_service/orchestrator/**' + - '!**/*.md' + airbyte_lib: + - 'airbyte_lib/**' + - '!**/*.md' + + - name: Run airbyte-ci/connectors/connector_ops tests + if: steps.changes.outputs.ops_any_changed == 'true' + id: run-airbyte-ci-connectors-connector-ops-tests + uses: ./.github/actions/run-dagger-pipeline + with: + context: "pull_request" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + subcommand: "test airbyte-ci/connectors/connector_ops --poetry-run-command='pytest tests'" + tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} + - name: Run airbyte-ci/connectors/pipelines tests id: run-airbyte-ci-connectors-pipelines-tests + if: steps.changes.outputs.pipelines_any_changed == 'true' uses: ./.github/actions/run-dagger-pipeline with: context: "pull_request" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} - gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - subcommand: "test airbyte-ci/connectors/pipelines" + subcommand: "test airbyte-ci/connectors/pipelines --poetry-run-command='pytest tests' --poetry-run-command='mypy pipelines --disallow-untyped-defs' --poetry-run-command='ruff check pipelines'" + tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} + + - name: Run airbyte-ci/connectors/base_images tests + id: run-airbyte-ci-connectors-base-images-tests + if: steps.changes.outputs.base_images_any_changed == 'true' + uses: ./.github/actions/run-dagger-pipeline + with: + context: "pull_request" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + subcommand: "test airbyte-ci/connectors/base_images --poetry-run-command='pytest tests'" + tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} + + - name: Run test pipeline for the metadata lib + id: metadata_lib-test-pipeline + if: steps.changes.outputs.metadata_lib_any_changed == 'true' + uses: ./.github/actions/run-dagger-pipeline + with: + subcommand: "test airbyte-ci/connectors/metadata_service/lib/ --poetry-run-command='pytest tests'" + context: "pull_request" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} + github_token: ${{ secrets.GITHUB_TOKEN }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} + + - name: Run test for the metadata orchestrator + id: metadata_orchestrator-test-pipeline + if: steps.changes.outputs.metadata_orchestrator_any_changed == 'true' + uses: ./.github/actions/run-dagger-pipeline + with: + subcommand: "test airbyte-ci/connectors/metadata_service/orchestrator/ --poetry-run-command='pytest tests'" + context: "pull_request" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} + github_token: ${{ secrets.GITHUB_TOKEN }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} + + - name: Run airbyte-lib tests + if: steps.changes.outputs.airbyte_lib_any_changed == 'true' + id: run-airbyte-lib-tests + uses: ./.github/actions/run-dagger-pipeline + with: + context: "pull_request" + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + subcommand: "test airbyte-lib --pass-env-var=GCP_GSM_CREDENTIALS --poetry-run-command='pytest'" + tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} diff --git a/.github/workflows/approve-and-merge-dispatch.yml b/.github/workflows/approve-and-merge-dispatch.yml index 06128fd505ed..b723f9337c51 100644 --- a/.github/workflows/approve-and-merge-dispatch.yml +++ b/.github/workflows/approve-and-merge-dispatch.yml @@ -11,9 +11,9 @@ jobs: id: scd with: token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} - permission: write + permission: admin issue-type: pull-request - repository: airbytehq/airbyte-cloud + repository: airbytehq/airbyte-platform-internal dispatch-type: repository commands: | approve-and-merge @@ -24,4 +24,25 @@ jobs: with: comment-id: ${{ github.event.comment.id }} body: | - > Error: ${{ steps.scd.outputs.error-message }} + > Error!: ${{ steps.scd.outputs.error-message }} + + - name: Checkout Airbyte + id: checkout + if: failure() || steps.scd.outputs.error-message + uses: actions/checkout@v2 + + - name: Run get_repo_admins.sh + if: failure() || steps.scd.outputs.error-message + id: repo_admins + run: | + echo "REPO_ADMINS=$(./tools/bin/get_repo_admins.sh ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} airbytehq/airbyte)" >> $GITHUB_ENV + + - name: Edit comment with repo admins + if: failure() || steps.scd.outputs.error-message + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.comment.id }} + body: | + > + > Important: This command can only be run by one of the repository admins: + > ${{ env.REPO_ADMINS }} diff --git a/.github/workflows/cat-tests.yml b/.github/workflows/cat-tests.yml index 78b7ac86c3c4..2f8cda400060 100644 --- a/.github/workflows/cat-tests.yml +++ b/.github/workflows/cat-tests.yml @@ -1,4 +1,4 @@ -name: CAT unit tests +name: Connector Ops CI - CAT Unit Tests concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -14,9 +14,31 @@ on: paths: - airbyte-integrations/bases/connector-acceptance-test/** jobs: + get_ci_runner: + runs-on: ubuntu-latest + name: Get CI runner + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + fetch-depth: 1 + - name: Get CI runner + id: get_ci_runner + uses: ./.github/actions/airbyte-ci-requirements + with: + runner_type: "test" + runner_size: "large" + airbyte_ci_command: "test" + github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + outputs: + runner_name: ${{ steps.get_ci_runner.outputs.runner_name }} run-cat-unit-tests: name: Run CAT unit tests - runs-on: "conn-prod-xlarge-runner" + needs: get_ci_runner + runs-on: ${{ needs.get_ci_runner.outputs.runner_name }} steps: - name: Checkout Airbyte uses: actions/checkout@v3 @@ -25,10 +47,12 @@ jobs: uses: ./.github/actions/run-dagger-pipeline with: context: "pull_request" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - subcommand: "test airbyte-integrations/bases/connector-acceptance-test --test-directory=unit_tests" + subcommand: "test airbyte-integrations/bases/connector-acceptance-test --poetry-run-command='pytest unit_tests'" + tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} diff --git a/.github/workflows/connector-performance-command.yml b/.github/workflows/connector-performance-command.yml index 17216d62f12b..3ed30a4ceb77 100644 --- a/.github/workflows/connector-performance-command.yml +++ b/.github/workflows/connector-performance-command.yml @@ -1,10 +1,54 @@ name: Connector Performance Harness on: + workflow_call: + inputs: + connector: + type: string + required: true + dataset: + type: string + required: true + repo: + description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos." + type: string + required: false + default: "airbytehq/airbyte" + gitref: + description: "The git ref to check out from the specified repository." + type: string + required: false + default: master + uuid: + description: "Custom UUID of workflow run. Used because GitHub dispatches endpoint does not return workflow run id." + type: string + required: false + stream-number: + description: "Number of streams to use for destination performance measurement." + type: string + required: false + default: "1" + sync-mode: + description: "Sync mode to use for destination performance measurement." + required: false + type: string + default: "full_refresh" + report-to-datadog: + description: "Whether to report the performance test results to Datadog." + required: false + type: string + default: "true" workflow_dispatch: inputs: connector: description: "Airbyte Connector" + type: choice required: true + options: + - connectors/source-postgres + - connectors/source-mysql + - connectors/source-mongodb-v2 + - connectors/destination-snowflake + default: "connectors/source-postgres" repo: description: "Repo to check out code from. Defaults to the main airbyte repo. Set this when building connectors from forked repos." required: false @@ -30,14 +74,22 @@ on: sync-mode: description: "Sync mode to use for destination performance measurement." required: false + type: choice + options: + - full_refresh + - incremental default: "full_refresh" + report-to-datadog: + description: "Whether to report the performance test results to Datadog." + required: false + default: "false" jobs: uuid: name: "Custom UUID of workflow run" timeout-minutes: 10 runs-on: ubuntu-latest steps: - - name: UUID ${{ github.event.inputs.uuid }} + - name: UUID ${{ inputs.uuid }} run: true start-test-runner: name: Start Build EC2 Runner @@ -51,8 +103,8 @@ jobs: - name: Checkout Airbyte uses: actions/checkout@v3 with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} + repository: ${{ inputs.repo }} + ref: ${{ inputs.gitref }} - name: Check PAT rate limits run: | ./tools/bin/find_non_rate_limited_PAT \ @@ -71,36 +123,38 @@ jobs: runs-on: ${{ needs.start-test-runner.outputs.label }} steps: - name: Link comment to workflow run - if: github.event.inputs.comment-id + if: inputs.comment-id uses: peter-evans/create-or-update-comment@v1 with: - comment-id: ${{ github.event.inputs.comment-id }} + comment-id: ${{ inputs.comment-id }} body: | #### Note: The following `dataset=` values are supported: `1m`(default), `10m`, `20m`, `bottleneck_stream1`, `bottleneck_stream_randomseed. For destinations only: you can also use `stream-numbers=N` to simulate N number of parallel streams. Additionally, `sync-mode=incremental` is supported for destinations. For example: `dataset=1m stream-numbers=2 sync-mode=incremental` - > :runner: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}. + > :runner: ${{inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}. - name: Search for valid connector name format id: regex uses: AsasInnab/regex-action@v1 with: - regex_pattern: "^((connectors|bases)/)?[a-zA-Z0-9-_]+$" + regex_pattern: "^(connectors/)?[a-zA-Z0-9-_]+$" regex_flags: "i" # required to be set for this plugin - search_string: ${{ github.event.inputs.connector }} + search_string: ${{ inputs.connector }} - name: Validate input workflow format - if: steps.regex.outputs.first_match != github.event.inputs.connector + if: steps.regex.outputs.first_match != inputs.connector run: echo "The connector provided has an invalid format!" && exit 1 - name: Filter supported connectors - if: "${{ github.event.inputs.connector != 'connectors/source-postgres' && - github.event.inputs.connector != 'connectors/source-mysql' && - github.event.inputs.connector != 'connectors/destination-snowflake' }}" - run: echo "Only connectors/source-postgres, source-mysql and destination-snowflake currently supported by harness" && exit 1 + if: "${{ inputs.connector != 'connectors/source-postgres' && + inputs.connector != 'connectors/source-mysql' && + inputs.connector != 'connectors/destination-snowflake' && + inputs.connector != 'connectors/source-mongodb-v2' }}" + run: echo "Only connectors/source-postgres, source-mysql, source-mongodb-v2 and destination-snowflake currently supported by harness" && exit 1 - name: Checkout Airbyte uses: actions/checkout@v3 with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ github.event.inputs.gitref }} + repository: ${{ inputs.repo }} + ref: ${{ inputs.gitref }} + fetch-depth: 0 # This is to fetch the main branch in case we are running on a different branch. - name: Install Java uses: actions/setup-java@v3 with: @@ -119,11 +173,14 @@ jobs: - name: Source or Destination harness id: which-harness run: | - the_harness="$(echo ${{github.event.inputs.connector}} | sed 's/.*\///; s/-.*//')"-harness + the_harness="$(echo ${{inputs.connector}} | sed 's/.*\///; s/-.*//')"-harness echo "harness_type=$the_harness" >> "$GITHUB_OUTPUT" - name: Write harness credentials run: | + export PATH="$PATH:/root/.local/bin" ci_credentials connectors-performance/$HARNESS_TYPE write-to-storage + connector_name=$(echo ${{ inputs.connector }} | sed 's,.*/,,') + ci_credentials connectors-performance/$connector_name write-to-storage env: GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} HARNESS_TYPE: ${{ steps.which-harness.outputs.harness_type }} @@ -138,11 +195,13 @@ jobs: - name: build connector shell: bash run: | - echo "Building... ${{github.event.inputs.connector}}" >> $GITHUB_STEP_SUMMARY + echo "Building... ${{inputs.connector}}" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY # this is a blank line - connector_name=$(echo ${{ github.event.inputs.connector }} | cut -d / -f 2) + connector_name=$(echo ${{ inputs.connector }} | sed 's,.*/,,') echo "Running ./gradlew :airbyte-integrations:connectors:$connector_name:build -x check" ./gradlew :airbyte-integrations:connectors:$connector_name:build -x check + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - name: KIND Kubernetes Cluster Setup uses: helm/kind-action@v1.4.0 with: @@ -151,23 +210,23 @@ jobs: id: run-harness shell: bash env: - CONN: ${{ github.event.inputs.connector }} - DS: ${{ github.event.inputs.dataset }} - STREAM_NUMBER: ${{ github.event.inputs.stream-number }} - SYNC_MODE: ${{ github.event.inputs.sync-mode }} + CONN: ${{ inputs.connector }} + DS: ${{ inputs.dataset }} + STREAM_NUMBER: ${{ inputs.stream-number }} + SYNC_MODE: ${{ inputs.sync-mode }} + REPORT_TO_DATADOG: ${{ inputs.report-to-datadog }} PREFIX: '{"type":"LOG","log":{"level":"INFO","message":"INFO i.a.i.p.PerformanceTest(runTest):165' SUFFIX: '"}}' HARNESS_TYPE: ${{ steps.which-harness.outputs.harness_type }} + DD_API_KEY: ${{ secrets.DD_API_KEY }} run: | kubectl apply -f ./tools/bin/admin-service-account.yaml connector_name=$(echo $CONN | cut -d / -f 2) kind load docker-image airbyte/$connector_name:dev --name chart-testing kind load docker-image airbyte/$HARNESS_TYPE:dev --name chart-testing - # envsubst requires variables to be exported + # envsubst requires variables to be exported or setup in the env field in this step. export CONNECTOR_IMAGE_NAME=${CONN/connectors/airbyte}:dev export DATASET=$DS - export STREAM_NUMBER=$STREAM_NUMBER - export SYNC_MODE=$SYNC_MODE export HARNESS=$HARNESS_TYPE envsubst < ./tools/bin/run-harness-process.yaml | kubectl create -f - echo "harness is ${{ steps.which-harness.outputs.harness_type }}" @@ -179,10 +238,11 @@ jobs: kubectl logs --tail=1 $POD | while read line ; do line=${line#"$PREFIX"}; line=${line%"$SUFFIX"}; echo $line >> $GITHUB_OUTPUT ; done echo "$EOF" >> $GITHUB_OUTPUT - name: Link comment to workflow run + if: inputs.comment-id uses: peter-evans/create-or-update-comment@v2 with: - reactions: '+1' - comment-id: ${{ github.event.inputs.comment-id }} + reactions: "+1" + comment-id: ${{ inputs.comment-id }} body: | ## Performance test Result: ``` diff --git a/.github/workflows/connector-performance-cron.yml b/.github/workflows/connector-performance-cron.yml new file mode 100644 index 000000000000..fc50d6b4ee4f --- /dev/null +++ b/.github/workflows/connector-performance-cron.yml @@ -0,0 +1,42 @@ +name: Connector Performance Harness Cron +on: + schedule: + # * is a special character in YAML so you have to quote this string + - # Twice a week, Monday and Thursday. + - cron: "0 0 * * 1,4" + workflow_dispatch: # for manual triggers + +jobs: + postgres-1m-run: + uses: ./.github/workflows/connector-performance-command.yml + with: + connector: "connectors/source-postgres" + dataset: 1m + secrets: inherit + mysql-1m-run: + uses: ./.github/workflows/connector-performance-command.yml + with: + connector: "connectors/source-mysql" + dataset: 1m + secrets: inherit + postgres-1m-run-incremental: + uses: ./.github/workflows/connector-performance-command.yml + with: + connector: "connectors/source-postgres" + dataset: 1m + sync-mode: "incremental" + secrets: inherit + mysql-1m-run-incremental: + uses: ./.github/workflows/connector-performance-command.yml + with: + connector: "connectors/source-mysql" + dataset: 1m + sync-mode: "incremental" + secrets: inherit + mongodb-1m-run-incremental: + uses: ./.github/workflows/connector-performance-command.yml + with: + connector: "connectors/source-mongodb-v2" + dataset: 1m + sync-mode: "incremental" + secrets: inherit diff --git a/.github/workflows/connector_checklist_require.yml b/.github/workflows/connector_checklist_require.yml index 3e65a3894264..073cc69842c7 100644 --- a/.github/workflows/connector_checklist_require.yml +++ b/.github/workflows/connector_checklist_require.yml @@ -1,7 +1,16 @@ name: Require Connector Checklist on: pull_request: - types: [opened, edited, synchronize, labeled, unlabeled, reopened, ready_for_review] + types: + [ + opened, + edited, + synchronize, + labeled, + unlabeled, + reopened, + ready_for_review, + ] paths: - "airbyte-integrations/connectors/source-**" - "airbyte-integrations/connectors/destination-**" diff --git a/.github/workflows/connector_code_freeze.yml b/.github/workflows/connector_code_freeze.yml new file mode 100644 index 000000000000..a5ac4c8a3661 --- /dev/null +++ b/.github/workflows/connector_code_freeze.yml @@ -0,0 +1,79 @@ +# This workflow is meant to be used to prevent/discourage merging to master during code freeze. +# The code freeze dates are set in the env variables CODE_FREEZE_START_DATE and CODE_FREEZE_END_DATE. +# If any connector code has been changed we display a warning message reminding merging is blocked and who to contact. +# If no connector connector code has been changed we only display a warning message reminding merging is discouraged. +# The Code freeze check job will be set as a required check for PRs in branch protection rules. + +name: Code freeze + +on: + pull_request: + types: + - opened + - synchronize + - ready_for_review + +env: + CODE_FREEZE_START_DATE: "2023-12-21" + CODE_FREEZE_END_DATE: "2024-01-02" +jobs: + code-freeze-check: + runs-on: ubuntu-latest + name: Code freeze check + permissions: + # This is required to be able to comment on PRs and list changed files + pull-requests: write + + steps: + # Check if code freeze is in effect by comparing the current date with the start and end date of the code freeze + - name: Check code freeze in effect + id: check-code-freeze-in-effect + run: | + start_date=$(date -d "$CODE_FREEZE_START_DATE" +%s) + end_date=$(date -d "$CODE_FREEZE_END_DATE" +%s) + current_date=$(date +%s) + + if [ "$current_date" -ge "$start_date" ] && [ "$current_date" -le "$end_date" ]; then + echo "Code freeze is in effect" + echo "::set-output name=is_in_code_freeze::true" + else + echo "Code freeze is not in effect" + echo "::set-output name=is_in_code_freeze::false" + fi + + # Use GitHub PR Api to get the list of changed files + # Filter the list to only keep the connectors files + - name: Get changed files + if: steps.check-code-freeze-in-effect.outputs.is_in_code_freeze == 'true' + id: changed-files + uses: tj-actions/changed-files@v40 + with: + files_yaml: | + connectors: + - 'airbyte-integrations/connectors/**' + - '!**/*.md' + + # If any connector code has been changed we display a warning message reminding merging is blocked and who to contact + - name: Code freeze comment on PR + if: steps.changed-files.outputs.connectors_any_changed == 'true' && steps.check-code-freeze-in-effect.outputs.is_in_code_freeze == 'true' + uses: thollander/actions-comment-pull-request@v2 + with: + comment_tag: code_freeze_warning + message: | + > [!WARNING] + > 🚨 Connector code freeze is in effect until ${{ env.CODE_FREEZE_END_DATE }}. This PR is changing connector code. Please contact the current OC engineers if you want to merge this change to master. + + # If no connector code has been changed we only display a warning message reminding merging is discouraged + - name: Code freeze comment on PR + if: steps.changed-files.outputs.connectors_any_changed == 'false' && steps.check-code-freeze-in-effect.outputs.is_in_code_freeze == 'true' + uses: thollander/actions-comment-pull-request@v2 + with: + comment_tag: code_freeze_warning + message: | + > [!WARNING] + > Soft code freeze is in effect until ${{ env.CODE_FREEZE_END_DATE }}. Please avoid merging to master. #freedom-and-responsibility + + # Fail the workflow if connector code has been changed to prevent merging to master + - name: Fail workflow if connector code has been changed + if: steps.changed-files.outputs.connectors_any_changed == 'true' && steps.check-code-freeze-in-effect.outputs.is_in_code_freeze == 'true' + run: echo "Connector code freeze is in effect. Please contact the current OC engineers if you want to merge this change." && exit 1 diff --git a/.github/workflows/connector_teams_review_requirements.yml b/.github/workflows/connector_teams_review_requirements.yml index f4765eef488e..3964f748955e 100644 --- a/.github/workflows/connector_teams_review_requirements.yml +++ b/.github/workflows/connector_teams_review_requirements.yml @@ -2,6 +2,11 @@ name: Connector Ops CI - Check review requirements on: pull_request: + types: + - opened + - reopened + - ready_for_review + - synchronize paths: - "airbyte-integrations/connectors/source-**" pull_request_review: @@ -12,7 +17,7 @@ jobs: name: "Check if a review is required from Connector teams" runs-on: ubuntu-latest - if: ${{ github.repository == 'airbytehq/airbyte' }} + if: ${{ github.repository == 'airbytehq/airbyte' && github.event.pull_request.draft == false }} steps: - name: Checkout Airbyte uses: actions/checkout@v3 @@ -39,5 +44,5 @@ jobs: with: status: ${{ steps.get-mandatory-reviewers.outputs.MANDATORY_REVIEWERS }} token: ${{ secrets.OCTAVIA_4_ROOT_ACCESS }} + request-reviews: true requirements-file: .github/connector_org_review_requirements.yaml - diff --git a/.github/workflows/connectors_nightly_build.yml b/.github/workflows/connectors_nightly_build.yml index acd7d024e3bb..edea7e253ee3 100644 --- a/.github/workflows/connectors_nightly_build.yml +++ b/.github/workflows/connectors_nightly_build.yml @@ -1,4 +1,4 @@ -name: Connectors nightly build +name: Connector Ops CI - Connectors Nightly Tests on: schedule: @@ -6,21 +6,39 @@ on: - cron: "0 0 * * *" workflow_dispatch: inputs: - runs-on: - type: string - default: conn-nightly-xlarge-runner - required: true test-connectors-options: default: --concurrency=5 --support-level=certified required: true -run-name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}" +run-name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }}" jobs: + get_ci_runner: + runs-on: ubuntu-latest + name: Get CI runner + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + fetch-depth: 1 + - name: Get CI runner + id: get_ci_runner + uses: ./.github/actions/airbyte-ci-requirements + with: + runner_type: "nightly" + runner_size: "xlarge" + airbyte_ci_command: "connectors test" + github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + outputs: + runner_name: ${{ steps.get_ci_runner.outputs.runner_name }} test_connectors: - name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}" + name: "Test connectors: ${{ inputs.test-connectors-options || 'nightly build for Certified connectors' }}" timeout-minutes: 720 # 12 hours - runs-on: ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }} + needs: get_ci_runner + runs-on: ${{ needs.get_ci_runner.outputs.runner_name }} steps: - name: Checkout Airbyte uses: actions/checkout@v3 @@ -34,11 +52,15 @@ jobs: - name: Test connectors uses: ./.github/actions/run-dagger-pipeline with: - context: "nightly_builds" + context: "master" + ci_job_key: "nightly_builds" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} git_branch: ${{ steps.extract_branch.outputs.branch }} github_token: ${{ secrets.GITHUB_TOKEN }} + s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} subcommand: "connectors ${{ inputs.test-connectors-options || '--concurrency=8 --support-level=certified' }} test" diff --git a/.github/workflows/connectors_tests.yml b/.github/workflows/connectors_tests.yml index 56820c787cc8..0b07c3c4bac5 100644 --- a/.github/workflows/connectors_tests.yml +++ b/.github/workflows/connectors_tests.yml @@ -1,7 +1,15 @@ -name: Connectors tests +name: Connector Ops CI - Connectors Acceptance Tests concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + # This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow. + # + # - github.head_ref is only defined on PR runs, it makes sure that the concurrency group is unique for pull requests + # ensuring that only one run per pull request is active at a time. + # + # - github.run_id is defined on all runs, it makes sure that the concurrency group is unique for workflow dispatches. + # This allows us to run multiple workflow dispatches in parallel. + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true on: workflow_dispatch: @@ -9,39 +17,51 @@ on: test-connectors-options: description: "Options to pass to the 'airbyte-ci connectors test' command" default: "--modified" - runner: - description: "The runner to use for this job" - default: "conn-prod-xlarge-runner" pull_request: types: - opened - synchronize - ready_for_review jobs: + get_ci_runner: + runs-on: ubuntu-latest + name: Get CI runner + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + fetch-depth: 1 + - name: Get CI runner + id: get_ci_runner + uses: ./.github/actions/airbyte-ci-requirements + with: + runner_type: "test" + runner_size: "large" + airbyte_ci_command: "connectors test" + github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + outputs: + runner_name: ${{ steps.get_ci_runner.outputs.runner_name }} connectors_ci: name: Connectors CI + needs: get_ci_runner + runs-on: ${{ needs.get_ci_runner.outputs.runner_name }} timeout-minutes: 1440 # 24 hours - runs-on: ${{ inputs.runner || 'conn-prod-xlarge-runner'}} steps: - name: Checkout Airbyte uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ + ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} - name: Extract branch name [WORKFLOW DISPATCH] shell: bash if: github.event_name == 'workflow_dispatch' run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT id: extract_branch - # - name: Format connectors [PULL REQUESTS] - # if: github.event_name == 'pull_request' - # uses: ./.github/actions/run-dagger-pipeline - # with: - # context: "pull_request" - # docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} - # docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} - # gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} - # git_branch: ${{ github.head_ref }} - # git_revision: ${{ github.sha }} - # github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} - # subcommand: "connectors --modified format" - name: Fetch last commit id from remote branch [PULL REQUESTS] if: github.event_name == 'pull_request' id: fetch_last_commit_id_pr @@ -50,35 +70,37 @@ jobs: if: github.event_name == 'workflow_dispatch' id: fetch_last_commit_id_wd run: echo "commit_id=$(git rev-parse origin/${{ steps.extract_branch.outputs.branch }})" >> $GITHUB_OUTPUT - - name: Pull formatting changes [PULL REQUESTS] - if: github.event_name == 'pull_request' - uses: actions/checkout@v3 - with: - repository: ${{ github.event.inputs.repo }} - ref: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }} - name: Test connectors [WORKFLOW DISPATCH] if: github.event_name == 'workflow_dispatch' uses: ./.github/actions/run-dagger-pipeline with: context: "manual" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} git_branch: ${{ steps.extract_branch.outputs.branch }} git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} + github_token: ${{ env.PAT }} + s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} subcommand: "connectors ${{ github.event.inputs.test-connectors-options }} test" + tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} - name: Test connectors [PULL REQUESTS] if: github.event_name == 'pull_request' uses: ./.github/actions/run-dagger-pipeline with: context: "pull_request" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} git_branch: ${{ github.head_ref }} git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} + github_token: ${{ env.PAT }} + s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} subcommand: "connectors --modified test" diff --git a/.github/workflows/connectors_weekly_build.yml b/.github/workflows/connectors_weekly_build.yml index 63ee82dfe646..8ec55657715b 100644 --- a/.github/workflows/connectors_weekly_build.yml +++ b/.github/workflows/connectors_weekly_build.yml @@ -1,4 +1,4 @@ -name: Connectors weekly build +name: Connector Ops CI - Connectors Weekly Tests on: schedule: @@ -6,21 +6,39 @@ on: - cron: "0 12 * * 0" workflow_dispatch: inputs: - runs-on: - type: string - default: conn-nightly-xlarge-runner - required: true test-connectors-options: default: --concurrency=3 --support-level=community required: true -run-name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}" +run-name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }}" jobs: + get_ci_runner: + runs-on: ubuntu-latest + name: Get CI runner + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + fetch-depth: 1 + - name: Get CI runner + id: get_ci_runner + uses: ./.github/actions/airbyte-ci-requirements + with: + runner_type: "test" + runner_size: "large" + airbyte_ci_command: "connectors test" + github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + outputs: + runner_name: ${{ steps.get_ci_runner.outputs.runner_name }} test_connectors: - name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }} - on ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }}" + name: "Test connectors: ${{ inputs.test-connectors-options || 'weekly build for Community connectors' }}" timeout-minutes: 8640 # 6 days - runs-on: ${{ inputs.runs-on || 'conn-nightly-xlarge-runner' }} + needs: get_ci_runner + runs-on: ${{ needs.get_ci_runner.outputs.runner_name }} steps: - name: Checkout Airbyte uses: actions/checkout@v3 @@ -34,11 +52,13 @@ jobs: - name: Test connectors uses: ./.github/actions/run-dagger-pipeline with: - context: "nightly_builds" + context: "master" ci_job_key: "weekly_alpha_test" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} git_branch: ${{ steps.extract_branch.outputs.branch }} github_token: ${{ secrets.GITHUB_TOKEN }} - subcommand: "connectors ${{ inputs.test-connectors-options || '--concurrency=3 --support-level=community' }} test" + tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} + subcommand: '--show-dagger-logs connectors ${{ inputs.test-connectors-options || ''--concurrency=3 --metadata-query="(data.ab_internal.ql > 100) & (data.ab_internal.sl < 200)"'' }} test' diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml deleted file mode 100644 index 348fe691854a..000000000000 --- a/.github/workflows/create-release.yml +++ /dev/null @@ -1,73 +0,0 @@ -# This is an action that runs when an Airbyte version bump is merged into master. -# It fetches the changelog from the version bump PR and automatically creates a -# Release for the version bump. - -name: Create Airbyte GH Release - -on: - push: - branches: - - master - -jobs: - create-release: - if: startsWith(github.event.head_commit.message, 'Bump Airbyte version') - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: read - steps: - - name: Fetch Version Bump PR Body - id: fetch_pr_body - env: - COMMIT_ID: ${{ github.event.head_commit.id }} - shell: bash - run: |- - set -x - PR=$(curl \ - -H "Accept: application/vnd.github.v3+json" \ - -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ - https://api.github.com/repos/${{ github.repository }}/commits/$COMMIT_ID/pulls) - # The printf helps escape characters so that jq can parse the output. - # The sed removes carriage returns so that the body is easier to parse later, and - # escapes backticks so that they are not executed as commands. - PR_BODY=$(printf '%s' "$PR" | jq '.[0].body' | sed 's/\\r//g' | sed 's/`/\\`/g') - echo "pr_body<> $GITHUB_ENV - echo "$PR_BODY" >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - - name: Extract Changelog - id: extract_changelog - shell: bash - run: |- - set -x - PR_BODY=${{ env.pr_body}} - if [[ $PR_BODY = "null" ]]; then - echo "No PR body exists for this commit, so a release cannot be generated." - exit 1 - fi - # this regex extracts just the changelog contents - if [[ $PR_BODY =~ Changelog:(\\n)*(.*)\\n\\n ]]; then - CHANGELOG="${BASH_REMATCH[2]}" - else - echo "PR body does not match the changelog extraction regex" - exit 1 - fi - # save CHANGELOG into a multiline env var on the action itself, since Github Actions do not support outputting multiline strings well - echo "CHANGELOG<> $GITHUB_ENV - echo -e "$CHANGELOG" >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - - name: Checkout Airbyte - uses: actions/checkout@v3 - - name: Get Version - id: get_version - shell: bash - run: | - VERSION=$(grep -w VERSION .env | cut -d"=" -f2) - echo VERSION=${VERSION} >> $GITHUB_OUTPUT - - name: Create Release - id: create_release - uses: ncipollo/release-action@v1 - with: - body: ${{ env.CHANGELOG }} - token: ${{ secrets.GITHUB_TOKEN }} - tag: v${{ steps.get_version.outputs.VERSION }} diff --git a/.github/workflows/deploy-docs-site.yml b/.github/workflows/deploy-docs-site.yml deleted file mode 100644 index de0a86587ef2..000000000000 --- a/.github/workflows/deploy-docs-site.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: Deploy docs.airbyte.com - -on: - push: - branches: - - master - paths: - - "docs/**" - - "docusaurus/**" - - pull_request: - types: - - closed - - opened - - reopened - - synchronize - paths: - - "docs/**" - - "docusaurus/**" - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -jobs: - build-and-deploy-docs: - name: Build and Deploy Docs Assets - runs-on: ubuntu-latest - steps: - - name: Check out the repository - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - # Node.js is needed for Yarn - - name: Setup Yarn - uses: actions/setup-node@v3 - with: - node-version: "16.14.0" - cache: "yarn" - cache-dependency-path: docusaurus - - - - name: Run Docusaurus - env: - GITHUB_TOKEN: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} - run: |- - export SKIP_DEPLOY="yes" - if [ "${{github.event_name}}" = 'push' -o "${{github.event_name}}" = 'workflow_dispatch' -o "${{github.event.pull_request.merged}}" = 'true' ]; then - export SKIP_DEPLOY="no" - fi - - ./tools/bin/deploy_docusaurus - - - - name: Notify Slack -- deploy failed - if: always() && ( github.event_name == 'push' || github.event_name == 'workflow_dispatch' ) && job.status != 'success' - uses: abinoda/slack-action@master - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} - with: - # 'C02TYDSUM8F' => '#dev-deploys' - # 'C03BEADRPNY' => '#docs' - args: >- - {\"channel\":\"C03BEADRPNY\",\"attachments\":[ - {\"color\":\"#ff0000\",\"blocks\":[ - {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"OSS Docs deploy fails on the latest master :bangbang:\"}}, - {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}*\"}}, - {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"\"}} - ]}]} - - - name: Notify Slack -- deploy succeeded - if: always() && ( github.event_name == 'push' || github.event_name == 'workflow_dispatch' ) && job.status == 'success' - uses: abinoda/slack-action@master - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} - with: - # 'C02TYDSUM8F' => '#dev-deploys' - # 'C03BEADRPNY' => '#docs' - args: >- - {\"channel\":\"C03BEADRPNY\",\"attachments\":[ - {\"color\":\"#00ff00\",\"blocks\":[ - {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"OSS Docs deploy was successful :tada:\"}}, - {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}*\"}}, - {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"\"}} - ]}]} diff --git a/.github/workflows/doc-link-check.yml b/.github/workflows/doc-link-check.yml index 8e4980b5dc4d..5a179bcb220f 100644 --- a/.github/workflows/doc-link-check.yml +++ b/.github/workflows/doc-link-check.yml @@ -6,7 +6,7 @@ name: Check for broken links in docs on: workflow_dispatch: schedule: - - cron: '0 18 * * *' + - cron: "0 18 * * *" jobs: markdown-link-check: @@ -17,8 +17,8 @@ jobs: # check all files on master - uses: gaurav-nelson/github-action-markdown-link-check@v1 with: - use-quiet-mode: 'yes' - check-modified-files-only: 'no' + use-quiet-mode: "yes" + check-modified-files-only: "no" config-file: .github/workflows/doc-link-check.json # posts to #_doc_link_checker diff --git a/.github/workflows/format_check.yml b/.github/workflows/format_check.yml new file mode 100644 index 000000000000..59e38842801b --- /dev/null +++ b/.github/workflows/format_check.yml @@ -0,0 +1,114 @@ +name: Check for formatting errors +run-name: Check for formatting errors on ${{ github.ref }} +on: + workflow_dispatch: + + push: + branches: + - master + pull_request: + +jobs: + get_ci_runner: + runs-on: ubuntu-latest + name: Get CI runner + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + fetch-depth: 1 + - name: Get CI runner + id: get_ci_runner + uses: ./.github/actions/airbyte-ci-requirements + with: + runner_type: "format" + runner_size: "medium" + airbyte_ci_command: "format" + github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + outputs: + runner_name: ${{ steps.get_ci_runner.outputs.runner_name }} + format-check: + # IMPORTANT: This name must match the require check name on the branch protection settings + name: "Check for formatting errors" + needs: get_ci_runner + runs-on: ${{ needs.get_ci_runner.outputs.runner_name }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + fetch-depth: 1 + - name: Run airbyte-ci format check [MASTER] + id: airbyte_ci_format_check_all_master + if: github.ref == 'refs/heads/master' + uses: ./.github/actions/run-dagger-pipeline + continue-on-error: true + with: + context: "master" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} + subcommand: "format check all" + + - name: Run airbyte-ci format check [PULL REQUEST] + id: airbyte_ci_format_check_all_pr + if: github.event_name == 'pull_request' + uses: ./.github/actions/run-dagger-pipeline + continue-on-error: false + with: + context: "pull_request" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} + subcommand: "format check all" + + - name: Run airbyte-ci format check [WORKFLOW DISPATCH] + id: airbyte_ci_format_check_all_manual + if: github.event_name == 'workflow_dispatch' + uses: ./.github/actions/run-dagger-pipeline + continue-on-error: false + with: + context: "manual" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} + subcommand: "format check all" + + - name: Match GitHub User to Slack User [MASTER] + if: github.ref == 'refs/heads/master' + id: match-github-to-slack-user + uses: ./.github/actions/match-github-to-slack-user + env: + AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }} + GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Format Failure on Master Slack Channel [MASTER] + if: steps.airbyte_ci_format_check_all.outcome == 'failure' && github.ref == 'refs/heads/master' + uses: abinoda/slack-action@master + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + with: + args: >- + {\"channel\":\"C03BEADRPNY\", \"blocks\":[ + {\"type\":\"divider\"}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"Formatting is broken on master! :bangbang: \n\n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"<@${{ steps.match-github-to-slack-user.outputs.slack_user_ids }}> \n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: :octavia-shocked: \n\"}}, + {\"type\":\"divider\"}]} diff --git a/.github/workflows/format_fix.yml b/.github/workflows/format_fix.yml new file mode 100644 index 000000000000..35425a30f28b --- /dev/null +++ b/.github/workflows/format_fix.yml @@ -0,0 +1,71 @@ +name: Fix formatting on a branch +run-name: Fix formatting on ${{ github.ref }} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + # Cancel any previous runs on the same branch if they are still in progress + cancel-in-progress: true + +on: + workflow_dispatch: +jobs: + get_ci_runner: + runs-on: ubuntu-latest + name: Get CI runner + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + fetch-depth: 1 + - name: Get CI runner + id: get_ci_runner + uses: ./.github/actions/airbyte-ci-requirements + with: + runner_type: "format" + runner_size: "large" + airbyte_ci_command: "format" + github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + outputs: + runner_name: ${{ steps.get_ci_runner.outputs.runner_name }} + format-fix: + name: "Run airbyte-ci format fix all" + needs: get_ci_runner + runs-on: ${{ needs.get_ci_runner.outputs.runner_name }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.ref }} + # Important that this is set so that CI checks are triggered again + # Without this we would be forever waiting on required checks to pass + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + + - name: Run airbyte-ci format fix all + uses: ./.github/actions/run-dagger-pipeline + continue-on-error: true + with: + context: "manual" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} + docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} + docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} + gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + github_token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} + tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} + subcommand: "format fix all" + + # This is helpful in the case that we change a previously committed generated file to be ignored by git. + - name: Remove any files that have been gitignored + run: git ls-files -i -c --exclude-from=.gitignore | xargs -r git rm --cached + + - name: Commit Formatting Changes (PR) + uses: stefanzweifel/git-auto-commit-action@v5 + # Don't commit if we're on master + if: github.ref != 'refs/heads/master' + with: + commit_message: "chore: format code" + commit_user_name: Octavia Squidington III + commit_user_email: octavia-squidington-iii@users.noreply.github.com diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index aead5f00f06e..c6de5ead2257 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -1,4 +1,4 @@ -name: Airbyte Connectors & Octavia CI +name: Airbyte CI - Repository Health Check concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -9,314 +9,20 @@ env: S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} on: - #ability to start task manually in Web UI workflow_dispatch: - inputs: - debug_mode: - description: "Enable or disable tmate session for debug during helm ac tests" - type: choice - default: "false" - options: - - "true" - - "false" - required: false push: branches: - master pull_request: - -permissions: write-all + types: + - opened + - reopened + - synchronize jobs: - # The output of this job is used to trigger the following builds. - changes: - name: "Detect Modified Files" - # The filtering action does not deal with well scheduled events so skip to avoid errors. - # See https://github.com/dorny/paths-filter/issues/100 for more info. - # This is okay this workflow is only scheduled on master, where we want to build everything - # so filtering is not required. Use always() in each start block to force the start task. - if: github.event_name != 'schedule' - runs-on: ubuntu-latest - outputs: - build: ${{ steps.filter.outputs.build }} - java_cdk: ${{ steps.filter.outputs.java_cdk }} - python_cdk: ${{ steps.filter.outputs.python_cdk }} - cli: ${{ steps.filter.outputs.cli }} - connectors_base: ${{ steps.filter.outputs.connectors_base }} - db: ${{ steps.filter.outputs.db }} - any_change: ${{ steps.filter.outputs.any_change }} - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - - uses: dorny/paths-filter@v2 - id: filter - with: - # Note: The following glob expression within a filters are ORs. - # Note: If no filters match, the steps are all skipped WITHOUT reported their status check back to github. - # This can cause required checks to go unreported blocking PRs from being merged - # and this is why we have the any_change filter. - filters: | - build: - - '.github/**' - - 'buildSrc/**' - - 'tools/**' - - '*.gradle' - - 'deps.toml' - - 'airbyte-config-oss/**' - python_cdk: - - 'airbyte-cdk/python/**' - java_cdk: - - 'airbyte-cdk/java/**' - cli: - - 'airbyte-api/**' - - 'octavia-cli/**' - connectors_base: - - 'airbyte-integrations/bases/**' - - 'airbyte-integrations/connectors-templates/**' - - 'airbyte-connector-test-harnesses/acceptance-test-harness/**' - db: - - 'airbyte-db/**' - any_change: - - '**/*' - - # Uncomment to debug. - # changes-output: - # name: "Debug Change Detection Logic" - # needs: changes - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v3 - # - run: | - # echo '${{ toJSON(needs) }}' - - format: - needs: changes - runs-on: ubuntu-latest - # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master. - if: needs.changes.outputs.any_change == 'true' || (always() && github.ref == 'refs/heads/master') - name: "Apply All Formatting Rules" - timeout-minutes: 20 - steps: - - name: Checkout Airbyte - uses: actions/checkout@v3 - with: - ref: ${{ github.head_ref }} - - - name: Cache Build Artifacts - uses: ./.github/actions/cache-build-artifacts - with: - cache-key: ${{ secrets.CACHE_VERSION }}-format - - - uses: actions/setup-java@v3 - with: - distribution: "zulu" - java-version: "17" - - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - - - name: Set up CI Gradle Properties - run: | - mkdir -p ~/.gradle/ - cat > ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < ~/.gradle/gradle.properties < ~/.gradle/gradle.properties <: ${{ github.event.inputs.changelog-message }}\n\n" + "text": "A new version of Python CDK has been released with : ${{ github.event.inputs.changelog-message }}\n\n" } }, { @@ -353,13 +353,13 @@ jobs: channel-id: C04J1M66D8B payload: | { - "text": ":warning: A new version of Airbyte CDK has been released but Connector Builder hasn't been automatically updated", + "text": ":warning: A new version of Python CDK has been released but Connector Builder hasn't been automatically updated", "blocks": [ { "type": "section", "text": { "type": "mrkdwn", - "text": "A new version of Airbyte CDK has been released with : ${{ github.event.inputs.changelog-message }}\n\n" + "text": "A new version of Python CDK has been released with : ${{ github.event.inputs.changelog-message }}\n\n" } }, { diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 24bbaaf3633e..a99757ba4ebe 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -44,5 +44,5 @@ jobs: > :warning: The publish slash command is now deprecated.
The connector publication happens on merge to the master branch.
Please use /legacy-publish if you need to publish normalization images.
- Please join the #publish-on-merge-updates slack channel to track ongoing publish pipelines.
+ Please join the #connector-publish-updates slack channel to track ongoing publish pipelines.
Please reach out to the @dev-connector-ops team if you need support in publishing a connector. diff --git a/.github/workflows/publish-java-cdk-command.yml b/.github/workflows/publish-java-cdk-command.yml new file mode 100644 index 000000000000..8ee3d25329c2 --- /dev/null +++ b/.github/workflows/publish-java-cdk-command.yml @@ -0,0 +1,242 @@ +# Usage: This workflow can be invoked manually or by a slash command. +# +# To invoke via GitHub UI, go to Actions tab, select the workflow, and click "Run workflow". +# +# To invoke via slash command, use the following syntax in a comment on a PR: +# /publish-java-cdk # Run with the defaults (dry-run=false, force=false) +# /publish-java-cdk dry-run=true # Run in dry-run mode (no-op) +# /publish-java-cdk force=true # Force-publish if needing to replace an already published version +name: Publish Java CDK +on: + # Temporarily run on commits to the 'java-cdk/publish-workflow' branch. + # TODO: Remove this 'push' trigger before merging to master. + push: + branches: + - java-cdk/publish-workflow + + workflow_dispatch: + inputs: + repo: + description: "Repo to check out code from. Defaults to the main airbyte repo." + # TODO: If publishing from forks is needed, we'll need to revert type to `string` of `choice`. + type: choice + required: true + default: airbytehq/airbyte + options: + - airbytehq/airbyte + dry-run: + description: "Dry run (no-op)" + required: true + type: boolean + default: false + force: + description: "Force release (ignore existing)" + required: true + type: boolean + default: false + gitref: + description: "The git ref to check out from the specified repository." + required: true + comment-id: + description: "Optional comment-id of the slash command. Ignore if not applicable." + required: false + # uuid: + # description: "Custom UUID of workflow run. Used because GitHub dispatches endpoint does not return workflow run id." + # required: false + +concurrency: + group: publish-airbyte-cdk + cancel-in-progress: false + +env: + # Use the provided GITREF or default to the branch triggering the workflow. + REPO: ${{ github.event.inputs.repo }} + GITREF: ${{ github.event.inputs.gitref || github.ref }} + FORCE: "${{ github.event.inputs.force == null && 'false' || github.event.inputs.force }}" + DRY_RUN: "${{ github.event.inputs.dry-run == null && 'true' || github.event.inputs.dry-run }}" + CDK_VERSION_FILE_PATH: "./airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties" + +jobs: + # We are using these runners because they are the same as the one for `publish-command.yml` + # One problem we had using `ubuntu-latest` for example is that the user is not root and some commands would fail in + # `manage.sh` (specifically `apt-get`) + start-publish-docker-image-runner-0: + name: Start Build EC2 Runner 0 + runs-on: ubuntu-latest + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + repository: airbytehq/airbyte + ref: master + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ + ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + github-token: ${{ env.PAT }} + label: ${{ github.run_id }}-publisher + + publish-cdk: + name: Publish Java CDK + needs: start-publish-docker-image-runner-0 + runs-on: ubuntu-latest + steps: + - name: Link comment to workflow run + if: github.event.inputs.comment-id + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :clock2: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + repository: ${{ env.REPO }} + ref: ${{ env.GITREF }} + - name: Read Target Java CDK version + id: read-target-java-cdk-version + run: | + cdk_version=$(cat $CDK_VERSION_FILE_PATH | tr -d '\n') + if [[ -z "$cdk_version" ]]; then + echo "Failed to retrieve CDK version from $CDK_VERSION_FILE_PATH" + exit 1 + fi + echo "CDK_VERSION=${cdk_version}" >> $GITHUB_ENV + - name: Setup Java + uses: actions/setup-java@v3 + with: + distribution: "zulu" + java-version: "17" + - name: Check for already-published version (${{ env.CDK_VERSION }}, FORCE=${{ env.FORCE }}) + if: ${{ !(env.FORCE == 'true') }} + run: ./gradlew :airbyte-cdk:java:airbyte-cdk:assertCdkVersionNotPublished + - name: Build Java CDK + run: ./gradlew --no-daemon :airbyte-cdk:java:airbyte-cdk:build + - name: Publish Java Modules to MavenLocal (Dry-Run) + if: ${{ !(env.DRY_RUN == 'false') }} + run: ./gradlew --no-daemon :airbyte-cdk:java:airbyte-cdk:publishToMavenLocal + - name: Upload jars as artifacts + if: ${{ !(env.DRY_RUN == 'false') }} + uses: actions/upload-artifact@v2 + with: + name: mavenlocal-jars + path: ~/.m2/repository/io/airbyte/ + - name: Publish Java Modules to CloudRepo + if: ${{ env.DRY_RUN == 'false' }} + run: ./gradlew --no-daemon :airbyte-cdk:java:airbyte-cdk:publish + env: + CLOUDREPO_USER: ${{ secrets.CLOUDREPO_USER }} + CLOUDREPO_PASSWORD: ${{ secrets.CLOUDREPO_PASSWORD }} + + - name: Add Success Comment + if: github.event.inputs.comment-id && success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + edit-mode: append + body: | + > :white_check_mark: Successfully published Java CDK ${{ env.CDK_VERSION }}! + - name: Add Failure Comment + if: github.event.inputs.comment-id && failure() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + edit-mode: append + body: | + > :x: Publish Java CDK ${{ env.CDK_VERSION }} failed! + - name: "Post failure to Slack channel `#dev-connectors-extensibility-releases`" + if: ${{ env.DRY_RUN == 'false' && failure() }} + uses: slackapi/slack-github-action@v1.23.0 + continue-on-error: true + with: + channel-id: C04J1M66D8B + payload: | + { + "text": "Error during `publish-cdk` while publishing Java CDK!", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Error while publishing Java CDK!" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "See details on \n" + } + } + ] + } + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + - name: "Post success to Slack channel `#dev-connectors-extensibility-releases`" + if: ${{ env.DRY_RUN == 'false' && !failure() }} + uses: slackapi/slack-github-action@v1.23.0 + continue-on-error: true + with: + channel-id: C04J1M66D8B + payload: | + { + "text": "New `${{ env.CDK_VERSION }}` version of Java CDK was successfully published!", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Java CDK ${{ env.CDK_VERSION }} published successfully!" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "See details on \n" + } + } + ] + } + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + + # In case of self-hosted EC2 errors, remove this block. + stop-publish-docker-image-runner-0: + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs + name: Stop Build EC2 Runner + needs: + - start-publish-docker-image-runner-0 # required to get output from the start-runner job + - publish-cdk # required to wait when the main job is done + runs-on: ubuntu-latest + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.GH_PAT_BUILD_RUNNER_OSS }} \ + ${{ secrets.GH_PAT_BUILD_RUNNER_BACKUP }} + - name: Stop EC2 runner + uses: airbytehq/ec2-github-runner@base64v1.1.0 + with: + mode: stop + github-token: ${{ env.PAT }} + label: ${{ needs.start-publish-docker-image-runner-0.outputs.label }} + ec2-instance-id: ${{ needs.start-publish-docker-image-runner-0.outputs.ec2-instance-id }} diff --git a/.github/workflows/publish_connectors.yml b/.github/workflows/publish_connectors.yml index 566fea7f1024..f0d10033f4a2 100644 --- a/.github/workflows/publish_connectors.yml +++ b/.github/workflows/publish_connectors.yml @@ -1,4 +1,4 @@ -name: Publish connectors on merge to master +name: Connector Ops CI - Publish Connectors on: push: @@ -14,14 +14,32 @@ on: publish-options: description: "Options to pass to the 'airbyte-ci connectors publish' command. Use --pre-release or --main-release depending on whether you want to publish a dev image or not. " default: "--pre-release" - runs-on: - type: string - default: conn-prod-xlarge-runner - required: true jobs: + get_ci_runner: + runs-on: ubuntu-latest + name: Get CI runner + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + ref: ${{ github.head_ref }} + token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + fetch-depth: 1 + - name: Get CI runner + id: get_ci_runner + uses: ./.github/actions/airbyte-ci-requirements + with: + runner_type: "publish" + runner_size: "large" + airbyte_ci_command: "connectors publish" + github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }} + sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} + outputs: + runner_name: ${{ steps.get_ci_runner.outputs.runner_name }} publish_connectors: name: Publish connectors - runs-on: ${{ inputs.runs-on || 'conn-prod-xlarge-runner' }} + needs: get_ci_runner + runs-on: ${{ needs.get_ci_runner.outputs.runner_name }} steps: - name: Checkout Airbyte uses: actions/checkout@v3 @@ -31,6 +49,7 @@ jobs: uses: ./.github/actions/run-dagger-pipeline with: context: "master" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} @@ -40,6 +59,9 @@ jobs: sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }} spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }} + s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} subcommand: "connectors --concurrency=1 --execute-timeout=3600 --metadata-changes-only publish --main-release" - name: Publish connectors [manual] @@ -48,6 +70,7 @@ jobs: uses: ./.github/actions/run-dagger-pipeline with: context: "manual" + dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} @@ -57,4 +80,62 @@ jobs: sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }} spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }} + s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} subcommand: "connectors ${{ github.event.inputs.connectors-options }} publish ${{ github.event.inputs.publish-options }}" + + set-instatus-incident-on-failure: + name: Create Instatus Incident on Failure + runs-on: ubuntu-latest + needs: + - publish_connectors + if: ${{ failure() && github.ref == 'refs/heads/master' }} + steps: + - name: Call Instatus Webhook + uses: joelwmale/webhook-action@master + with: + url: ${{ secrets.INSTATUS_CONNECTOR_CI_WEBHOOK_URL }} + body: '{ "trigger": "down", "status": "HASISSUES" }' + + set-instatus-incident-on-success: + name: Create Instatus Incident on Success + runs-on: ubuntu-latest + needs: + - publish_connectors + if: ${{ success() && github.ref == 'refs/heads/master' }} + steps: + - name: Call Instatus Webhook + uses: joelwmale/webhook-action@master + with: + url: ${{ secrets.INSTATUS_CONNECTOR_CI_WEBHOOK_URL }} + body: '{ "trigger": "up" }' + + notify-failure-slack-channel: + name: "Notify Slack Channel on Build Failures" + runs-on: ubuntu-latest + needs: + - publish_connectors + if: ${{ failure() && github.ref == 'refs/heads/master' }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + - name: Match GitHub User to Slack User + id: match-github-to-slack-user + uses: ./.github/actions/match-github-to-slack-user + env: + AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }} + GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Publish to OSS Build Failure Slack Channel + uses: abinoda/slack-action@master + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }} + with: + args: >- + {\"channel\":\"C056HGD1QSW\", \"blocks\":[ + {\"type\":\"divider\"}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" Publish Connector Failed! :bangbang: \n\n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"_merged by_: *${{ github.actor }}* \n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"<@${{ steps.match-github-to-slack-user.outputs.slack_user_ids }}> \n\"}}, + {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\" :octavia-shocked: :octavia-shocked: \n\"}}, + {\"type\":\"divider\"}]} diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index 477db6af6d6a..1d38dcab8017 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -5,12 +5,12 @@ on: inputs: runs-on: type: string - default: conn-prod-xlarge-runner + default: ci-runner-connector-publish-large-dagger-0-9-5 required: true jobs: no-op: name: No-op - runs-on: ${{ inputs.runs-on || 'conn-prod-xlarge-runner' }} + runs-on: ${{ inputs.runs-on || 'ci-runner-connector-publish-large-dagger-0-9-5' }} steps: - run: echo 'hi!' diff --git a/.github/workflows/release-airbyte-os.yml b/.github/workflows/release-airbyte-os.yml index 70b972fae4dd..ea36f6aea9bf 100644 --- a/.github/workflows/release-airbyte-os.yml +++ b/.github/workflows/release-airbyte-os.yml @@ -115,7 +115,7 @@ jobs: steps: - uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.10" - name: Checkout Airbyte uses: actions/checkout@v3 @@ -188,7 +188,7 @@ jobs: - uses: actions/setup-python@v4 with: - python-version: "3.9" + python-version: "3.10" - name: Release Octavia id: release_octavia @@ -203,8 +203,8 @@ jobs: uses: mariamrf/py-package-publish-action@v1.1.0 with: # specify the same version as in ~/.python-version - python_version: "3.9.11" - pip_version: "21.1" + python_version: "3.10" + pip_version: "23.2" subdir: "octavia-cli/" env: TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} @@ -214,8 +214,8 @@ jobs: uses: mariamrf/py-package-publish-action@v1.1.0 with: # specify the same version as in ~/.python-version - python_version: "3.9.11" - pip_version: "21.1" + python_version: "3.10" + pip_version: "23.2" subdir: "octavia-cli/" env: TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }} diff --git a/.github/workflows/run-mypy-on-modified-cdk-files.yml b/.github/workflows/run-mypy-on-modified-cdk-files.yml new file mode 100644 index 000000000000..2c3d96509521 --- /dev/null +++ b/.github/workflows/run-mypy-on-modified-cdk-files.yml @@ -0,0 +1,28 @@ +name: Connector Extensibility - Run mypy on modified cdk files + +on: + pull_request: + paths: + - "airbyte-cdk/python/airbyte_cdk/**/*.py" +jobs: + run-mypy-on-modified-cdk-files: + name: "Run mypy on modified cdk files" + runs-on: ubuntu-latest + steps: + - name: Checkout Airbyte + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - run: pip install mypy==1.6.0 + - name: Get Python changed files + id: changed-py-files + uses: tj-actions/changed-files@v23 + with: + files: "airbyte-cdk/python/airbyte_cdk/**/*.py" + - name: Run if any of the listed files above is changed + if: steps.changed-py-files.outputs.any_changed == 'true' + run: mypy ${{ steps.changed-py-files.outputs.all_changed_files }} --config-file airbyte-cdk/python/mypy.ini --install-types --non-interactive diff --git a/.github/workflows/run-qa-engine.yml b/.github/workflows/run-qa-engine.yml index fba479628190..6b7ae8b3ca37 100644 --- a/.github/workflows/run-qa-engine.yml +++ b/.github/workflows/run-qa-engine.yml @@ -1,4 +1,4 @@ -name: Run QA Engine +name: Connector Ops CI - QA Engine on: workflow_dispatch: diff --git a/.github/workflows/slash-commands.yml b/.github/workflows/slash-commands.yml index 6ac373540ca3..17e1ea683060 100644 --- a/.github/workflows/slash-commands.yml +++ b/.github/workflows/slash-commands.yml @@ -15,23 +15,19 @@ jobs: echo ref="$(echo $pr_info | jq -r '.head.ref')" >> $GITHUB_OUTPUT echo repo="$(echo $pr_info | jq -r '.head.repo.full_name')" >> $GITHUB_OUTPUT - - name: Slash Command Dispatch + - name: Slash Command Dispatch (Workflow) id: scd - uses: peter-evans/slash-command-dispatch@v2 + uses: peter-evans/slash-command-dispatch@v3 with: - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.GH_PAT_MAINTENANCE_OCTAVIA }} permission: write commands: | test legacy-test test-performance - build-connector - publish-connector publish + publish-java-cdk legacy-publish - publish-external - gke-kube-test - run-specific-test connector-performance static-args: | repo=${{ steps.getref.outputs.repo }} diff --git a/.github/workflows/upload-metadata-files.yml b/.github/workflows/upload-metadata-files.yml index 0853f382a750..a3da50247748 100644 --- a/.github/workflows/upload-metadata-files.yml +++ b/.github/workflows/upload-metadata-files.yml @@ -1,4 +1,4 @@ -name: "Upload any Changed Metadata Files [Exceptional Use!]" +name: "Connector Ops CI - Upload Changed Metadata Files [Emergency Use!]" on: workflow_dispatch: diff --git a/.gitignore b/.gitignore index 56790f53bd9a..b97efca12e81 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,9 @@ secrets updated_configurations !airbyte-integrations/connector-templates/**/secrets +# Connector debug configs +airbyte-integrations/connectors/**/src/test/resources/debug_resources + # Test logs acceptance_tests_logs @@ -70,8 +73,8 @@ docs/SUMMARY.md **/specs_secrets_mask.yaml # Files generated when downloading connector registry -airbyte-config-oss/**/seed/oss_registry.json -airbyte-config-oss/**/seed/oss_catalog.json +**/init-oss/src/main/resources/seed/oss_registry.json +**/init-oss/src/main/resources/seed/oss_catalog.json # Output Files generated by scripts lowcode_connector_names.txt @@ -102,3 +105,6 @@ tools/ci_connector_ops/pipeline_reports/ # ignore local build scan uri output scan-journal.log + +# connectors' cache +*.sqlite diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d3b5f389c993..4fd1a68cdc3b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,66 +1,10 @@ -default_language_version: - python: python3 - repos: - - repo: https://github.com/johann-petrak/licenseheaders.git - rev: v0.8.8 - hooks: - - id: licenseheaders - args: - ["--tmpl=LICENSE_SHORT", "--ext=py", "-x=**/models/__init__.py", "-f"] - - repo: https://github.com/psf/black - rev: 22.3.0 - hooks: - - id: black - args: ["--config", "pyproject.toml"] - - repo: https://github.com/timothycrosley/isort - rev: 5.10.1 - hooks: - - id: isort - args: - [ - "--settings-file", - "pyproject.toml", - "--dont-follow-links", - "--jobs=-1", - ] - additional_dependencies: ["colorama"] - - repo: https://github.com/pre-commit/mirrors-prettier - rev: v2.5.0 - hooks: - - id: prettier - types_or: [yaml, json] - exclude: | - (?x)^.*( - .github/| - .gitlab-ci.yml - ).?$ - - - repo: https://github.com/csachs/pyproject-flake8 - rev: v6.0.0 - hooks: - - id: pyproject-flake8 - args: ["--config", "pyproject.toml"] - additional_dependencies: ["mccabe"] - alias: flake8 - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.930 - hooks: - - id: mypy - args: ["--config-file", "pyproject.toml"] - exclude: | - (?x)^.*( - octavia-cli/unit_tests/| - ).?$ - repo: local hooks: - - id: spec-linter - name: validate connectors spec files + - id: format-fix-all-on-push + always_run: true + entry: airbyte-ci --disable-update-check format fix all language: system - entry: python tools/git_hooks/spec_linter.py - files: ^.*/spec.json$ - exclude: | - (?x)^.*( - /connectors/destination-e2e-test| - /connectors/source-e2e-test - ).*$ + name: Run airbyte-ci format fix on git push (~30s) + pass_filenames: false + stages: [push] diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 000000000000..b556b2b63c60 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,11 @@ +{ + "overrides": [ + { + "files": "*.md", + "options": { + "printWidth": 100, + "proseWrap": "always" + } + } + ] +} diff --git a/.python-version b/.python-version index a9f8d1be337f..c8cfe3959183 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.9.11 +3.10 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 7d0ec94e6392..df6d0baa6779 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,2 +1,2 @@ # Code of conduct -View in [docs.airbyte.io](https://docs.airbyte.io/contributing-to-airbyte/code-of-conduct) +View in [docs.airbyte.io](https://docs.airbyte.com/project-overview/code-of-conduct) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 796352ea1bff..fa4e306a7d7a 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -1,138 +1,421 @@ # Contributors * [69mb](https://github.com/69mb) -* [addack](https://github.com/addack) +* [a-honcharenko](https://github.com/a-honcharenko) +* [aadityasinha-dotcom](https://github.com/aadityasinha-dotcom) +* [aaronsteers](https://github.com/aaronsteers) +* [aazam-gh](https://github.com/aazam-gh) +* [abaerptc](https://github.com/abaerptc) +* [aballiet](https://github.com/aballiet) +* [achaussende](https://github.com/achaussende) +* [ad-m](https://github.com/ad-m) +* [adam-bloom](https://github.com/adam-bloom) +* [adamf](https://github.com/adamf) +* [adamschmidt](https://github.com/adamschmidt) * [AetherUnbound](https://github.com/AetherUnbound) * [afranzi](https://github.com/afranzi) +* [agrass](https://github.com/agrass) +* [ahmed-buksh](https://github.com/ahmed-buksh) * [airbyte-jenny](https://github.com/airbyte-jenny) +* [ajmhatch](https://github.com/ajmhatch) +* [ajzo90](https://github.com/ajzo90) +* [akashkulk](https://github.com/akashkulk) +* [akulgoel96](https://github.com/akulgoel96) +* [alafanechere](https://github.com/alafanechere) +* [alallema](https://github.com/alallema) +* [albert-marrero](https://github.com/albert-marrero) +* [alex-danilin](https://github.com/alex-danilin) +* [alex-gron](https://github.com/alex-gron) +* [alexander-marquardt](https://github.com/alexander-marquardt) +* [AlexanderBatoulis](https://github.com/AlexanderBatoulis) +* [alexandertsukanov](https://github.com/alexandertsukanov) * [alexandr-shegeda](https://github.com/alexandr-shegeda) +* [alexchouraki](https://github.com/alexchouraki) +* [AlexJameson](https://github.com/AlexJameson) +* [alexnikitchuk](https://github.com/alexnikitchuk) +* [Alihassanc5](https://github.com/Alihassanc5) +* [Allexik](https://github.com/Allexik) +* [alovew](https://github.com/alovew) +* [AM-I-Human](https://github.com/AM-I-Human) +* [amaliaroye](https://github.com/amaliaroye) +* [ambirdsall](https://github.com/ambirdsall) +* [aminamos](https://github.com/aminamos) * [amitku](https://github.com/amitku) +* [Amruta-Ranade](https://github.com/Amruta-Ranade) +* [anamargaridarl](https://github.com/anamargaridarl) +* [andnig](https://github.com/andnig) +* [andresbravog](https://github.com/andresbravog) +* [andrewlreeve](https://github.com/andrewlreeve) +* [andreyAtBB](https://github.com/andreyAtBB) * [andriikorotkov](https://github.com/andriikorotkov) +* [andrzejdackiewicz](https://github.com/andrzejdackiewicz) +* [andyjih](https://github.com/andyjih) +* [AndyTwiss](https://github.com/AndyTwiss) +* [animer3009](https://github.com/animer3009) +* [anna-geller](https://github.com/anna-geller) * [annalvova05](https://github.com/annalvova05) * [antixar](https://github.com/antixar) +* [antonioneto-hotmart](https://github.com/antonioneto-hotmart) +* [anujgupta0711](https://github.com/anujgupta0711) * [Anurag870](https://github.com/Anurag870) +* [anushree-agrawal](https://github.com/anushree-agrawal) * [apostoltego](https://github.com/apostoltego) +* [archangelic](https://github.com/archangelic) +* [arimbr](https://github.com/arimbr) +* [arnaudjnn](https://github.com/arnaudjnn) +* [ArneZsng](https://github.com/ArneZsng) +* [arsenlosenko](https://github.com/arsenlosenko) +* [artem1205](https://github.com/artem1205) +* [artusiep](https://github.com/artusiep) +* [asafepy](https://github.com/asafepy) +* [asyarif93](https://github.com/asyarif93) +* [augan-rymkhan](https://github.com/augan-rymkhan) +* [Auric-Manteo](https://github.com/Auric-Manteo) * [avaidyanatha](https://github.com/avaidyanatha) * [avida](https://github.com/avida) +* [avirajsingh7](https://github.com/avirajsingh7) +* [axaysagathiya](https://github.com/axaysagathiya) +* [azhard](https://github.com/azhard) +* [b4stien](https://github.com/b4stien) +* [bala-ceg](https://github.com/bala-ceg) * [bazarnov](https://github.com/bazarnov) +* [bbugh](https://github.com/bbugh) +* [bcbeidel](https://github.com/bcbeidel) +* [bdashrad](https://github.com/bdashrad) +* [benmoriceau](https://github.com/benmoriceau) +* [BenoitFayolle](https://github.com/BenoitFayolle) +* [BenoitHugonnard](https://github.com/BenoitHugonnard) +* [bgroff](https://github.com/bgroff) +* [Bhupesh-V](https://github.com/Bhupesh-V) +* [BirdboyBolu](https://github.com/BirdboyBolu) +* [bjgbeelen](https://github.com/bjgbeelen) * [bkrausz](https://github.com/bkrausz) +* [bleonard](https://github.com/bleonard) +* [bnchrch](https://github.com/bnchrch) +* [bobvanluijt](https://github.com/bobvanluijt) +* [brebuanirello-equinix](https://github.com/brebuanirello-equinix) +* [BrentSouza](https://github.com/BrentSouza) +* [brianjlai](https://github.com/brianjlai) +* [brunofaustino](https://github.com/brunofaustino) +* [bstrawson](https://github.com/bstrawson) +* [btkcodedev](https://github.com/btkcodedev) +* [burmecia](https://github.com/burmecia) +* [bzAmin](https://github.com/bzAmin) +* [calebfornari](https://github.com/calebfornari) +* [cameronwtaylor](https://github.com/cameronwtaylor) * [camro](https://github.com/camro) +* [carlkibler](https://github.com/carlkibler) +* [carlonuccio](https://github.com/carlonuccio) +* [catpineapple](https://github.com/catpineapple) * [cgardens](https://github.com/cgardens) +* [chadthman](https://github.com/chadthman) +* [chandrasekharan98](https://github.com/chandrasekharan98) +* [ChristoGrab](https://github.com/ChristoGrab) * [ChristopheDuong](https://github.com/ChristopheDuong) +* [ciancullinan](https://github.com/ciancullinan) +* [cirdes](https://github.com/cirdes) * [cjwooo](https://github.com/cjwooo) +* [clnoll](https://github.com/clnoll) +* [cobobrien](https://github.com/cobobrien) * [coetzeevs](https://github.com/coetzeevs) -* [coeurdestenebres](https://github.com/coeurdestenebres) +* [colesnodgrass](https://github.com/colesnodgrass) +* [collinscangarella](https://github.com/collinscangarella) +* [cpdeethree](https://github.com/cpdeethree) +* [CrafterKolyan](https://github.com/CrafterKolyan) +* [cstruct](https://github.com/cstruct) +* [ct-martin](https://github.com/ct-martin) +* [cuyk](https://github.com/cuyk) +* [cynthiaxyin](https://github.com/cynthiaxyin) +* [CyprienBarbault](https://github.com/CyprienBarbault) +* [czuares](https://github.com/czuares) +* [Daemonxiao](https://github.com/Daemonxiao) +* [dainiussa](https://github.com/dainiussa) +* [dalo390](https://github.com/dalo390) * [damianlegawiec](https://github.com/damianlegawiec) +* [dandpz](https://github.com/dandpz) +* [daniel-cortez-stevenson](https://github.com/daniel-cortez-stevenson) * [danieldiamond](https://github.com/danieldiamond) +* [Danucas](https://github.com/Danucas) +* [danvass](https://github.com/danvass) +* [darian-heede](https://github.com/darian-heede) +* [darynaishchenko](https://github.com/darynaishchenko) +* [DavidSpek](https://github.com/DavidSpek) * [davinchia](https://github.com/davinchia) +* [davydov-d](https://github.com/davydov-d) +* [dbyzero](https://github.com/dbyzero) +* [ddoyediran](https://github.com/ddoyediran) +* [deepansh96](https://github.com/deepansh96) +* [delenamalan](https://github.com/delenamalan) +* [denis-sokolov](https://github.com/denis-sokolov) * [dependabot[bot]](https://github.com/apps/dependabot) +* [dictcp](https://github.com/dictcp) +* [didistars328](https://github.com/didistars328) +* [digambar-t7](https://github.com/digambar-t7) +* [dijonkitchen](https://github.com/dijonkitchen) +* [dizel852](https://github.com/dizel852) * [dmateusp](https://github.com/dmateusp) -* [DominusKelvin](https://github.com/DominusKelvin) +* [domzae](https://github.com/domzae) * [DoNotPanicUA](https://github.com/DoNotPanicUA) +* [Dracyr](https://github.com/Dracyr) +* [drrest](https://github.com/drrest) +* [dtt101](https://github.com/dtt101) * [edbizarro](https://github.com/edbizarro) +* [edgao](https://github.com/edgao) +* [edmundito](https://github.com/edmundito) +* [efimmatytsin](https://github.com/efimmatytsin) * [eliziario](https://github.com/eliziario) +* [elliottrabac](https://github.com/elliottrabac) +* [emmaling27](https://github.com/emmaling27) +* [erica-airbyte](https://github.com/erica-airbyte) +* [erohmensing](https://github.com/erohmensing) * [etsybaev](https://github.com/etsybaev) * [eugene-kulak](https://github.com/eugene-kulak) +* [evantahler](https://github.com/evantahler) * [ffabss](https://github.com/ffabss) +* [flash1293](https://github.com/flash1293) +* [franviera92](https://github.com/franviera92) * [freimer](https://github.com/freimer) * [FUT](https://github.com/FUT) * [gaart](https://github.com/gaart) -* [gasparakos](https://github.com/gasparakos) -* [geekwhocodes](https://github.com/geekwhocodes) -* [gingeard](https://github.com/gingeard) +* [ganpatagarwal](https://github.com/ganpatagarwal) +* [gargatuma](https://github.com/gargatuma) +* [gergelylendvai](https://github.com/gergelylendvai) +* [girarda](https://github.com/girarda) +* [git-phu](https://github.com/git-phu) +* [github-actions[bot]](https://github.com/apps/github-actions) +* [Gitznik](https://github.com/Gitznik) * [gordalina](https://github.com/gordalina) +* [gosusnp](https://github.com/gosusnp) * [grebessi](https://github.com/grebessi) +* [grishick](https://github.com/grishick) +* [grubberr](https://github.com/grubberr) +* [gvillafanetapia](https://github.com/gvillafanetapia) * [h7kanna](https://github.com/h7kanna) -* [haliva-firmbase](https://github.com/haliva-firmbase) +* [haithem-souala](https://github.com/haithem-souala) +* [haoranyu](https://github.com/haoranyu) * [harshithmullapudi](https://github.com/harshithmullapudi) * [heade](https://github.com/heade) +* [hehex9](https://github.com/hehex9) +* [helderco](https://github.com/helderco) +* [henriblancke](https://github.com/henriblancke) +* [Hesperide](https://github.com/Hesperide) * [hillairet](https://github.com/hillairet) +* [himanshuc3](https://github.com/himanshuc3) +* [hntan](https://github.com/hntan) * [htrueman](https://github.com/htrueman) -* [hudsondba](https://github.com/hudsondba) +* [hydrosquall](https://github.com/hydrosquall) +* [iberchid](https://github.com/iberchid) +* [igrankova](https://github.com/igrankova) +* [igsaf2](https://github.com/igsaf2) +* [Imbruced](https://github.com/Imbruced) * [irynakruk](https://github.com/irynakruk) +* [isaacharrisholt](https://github.com/isaacharrisholt) * [isalikov](https://github.com/isalikov) -* [itaiad200](https://github.com/itaiad200) +* [itaseskii](https://github.com/itaseskii) * [jacqueskpoty](https://github.com/jacqueskpoty) -* [jaimefr](https://github.com/jaimefr) -* [Jamakase](https://github.com/Jamakase) -* [Janardhanpoola](https://github.com/Janardhanpoola) -* [jinnig](https://github.com/jinnig) +* [Jagrutiti](https://github.com/Jagrutiti) +* [jamakase](https://github.com/jamakase) +* [jartek](https://github.com/jartek) +* [jbfbell](https://github.com/jbfbell) +* [jcowanpdx](https://github.com/jcowanpdx) +* [jdclarke5](https://github.com/jdclarke5) +* [jdpgrailsdev](https://github.com/jdpgrailsdev) +* [jeremySrgt](https://github.com/jeremySrgt) +* [jhajajaas](https://github.com/jhajajaas) +* [jhammarstedt](https://github.com/jhammarstedt) +* [jnr0790](https://github.com/jnr0790) +* [joelluijmes](https://github.com/joelluijmes) * [johnlafleur](https://github.com/johnlafleur) -* [jonathan-duval](https://github.com/jonathan-duval) +* [JonsSpaghetti](https://github.com/JonsSpaghetti) * [jonstacks](https://github.com/jonstacks) +* [jordan-glitch](https://github.com/jordan-glitch) +* [josephkmh](https://github.com/josephkmh) * [jrhizor](https://github.com/jrhizor) +* [juliachvyrova](https://github.com/juliachvyrova) +* [JulianRommel](https://github.com/JulianRommel) +* [juliatournant](https://github.com/juliatournant) +* [justinbchau](https://github.com/justinbchau) * [juweins](https://github.com/juweins) +* [jzcruiser](https://github.com/jzcruiser) +* [kaklakariada](https://github.com/kaklakariada) +* [karinakuz](https://github.com/karinakuz) +* [kattos-aws](https://github.com/kattos-aws) +* [KayakinKoder](https://github.com/KayakinKoder) * [keu](https://github.com/keu) +* [kgrover](https://github.com/kgrover) +* [kimerinn](https://github.com/kimerinn) +* [koconder](https://github.com/koconder) +* [koji-m](https://github.com/koji-m) +* [krishnaglick](https://github.com/krishnaglick) +* [krisjan-oldekamp](https://github.com/krisjan-oldekamp) +* [ksengers](https://github.com/ksengers) +* [kzzzr](https://github.com/kzzzr) * [lazebnyi](https://github.com/lazebnyi) +* [leo-schick](https://github.com/leo-schick) +* [letiescanciano](https://github.com/letiescanciano) +* [lgomezm](https://github.com/lgomezm) +* [lideke](https://github.com/lideke) * [lizdeika](https://github.com/lizdeika) -* [lmeyerov](https://github.com/lmeyerov) -* [luizgribeiro](https://github.com/luizgribeiro) -* [m-ronchi](https://github.com/m-ronchi) +* [lmossman](https://github.com/lmossman) * [maciej-nedza](https://github.com/maciej-nedza) +* [macmv](https://github.com/macmv) +* [Mainara](https://github.com/Mainara) * [makalaaneesh](https://github.com/makalaaneesh) -* [manavkohli](https://github.com/manavkohli) +* [makyash](https://github.com/makyash) +* [malikdiarra](https://github.com/malikdiarra) +* [marcelopio](https://github.com/marcelopio) * [marcosmarxm](https://github.com/marcosmarxm) +* [mariamthiam](https://github.com/mariamthiam) * [masonwheeler](https://github.com/masonwheeler) -* [MatheusdiPaula](https://github.com/MatheusdiPaula) +* [masyagin1998](https://github.com/masyagin1998) +* [matter-q](https://github.com/matter-q) +* [maxi297](https://github.com/maxi297) * [MaxKrog](https://github.com/MaxKrog) -* [MaxwellJK](https://github.com/MaxwellJK) -* [mbbroberg](https://github.com/mbbroberg) -* [mhamas](https://github.com/mhamas) +* [mdibaiee](https://github.com/mdibaiee) +* [mfsiega-airbyte](https://github.com/mfsiega-airbyte) +* [michaelnguyen26](https://github.com/michaelnguyen26) * [michel-tricot](https://github.com/michel-tricot) +* [mickaelandrieu](https://github.com/mickaelandrieu) * [midavadim](https://github.com/midavadim) -* [mid](https://github.com/mid) * [mildbyte](https://github.com/mildbyte) -* [minimax75](https://github.com/minimax75) -* [mjirv](https://github.com/mjirv) +* [misteryeo](https://github.com/misteryeo) +* [mkhokh-33](https://github.com/mkhokh-33) +* [mlavoie-sm360](https://github.com/mlavoie-sm360) * [mmolimar](https://github.com/mmolimar) -* [MohamadHaziq](https://github.com/MohamadHaziq) -* [mohammad-bolt](https://github.com/mohammad-bolt) -* [moszutij](https://github.com/moszutij) +* [mohamagdy](https://github.com/mohamagdy) +* [mohitreddy1996](https://github.com/mohitreddy1996) +* [monai](https://github.com/monai) +* [mrhallak](https://github.com/mrhallak) * [Muriloo](https://github.com/Muriloo) -* [muutech](https://github.com/muutech) -* [nclsbayona](https://github.com/nclsbayona) -* [nicholasbull](https://github.com/nicholasbull) +* [mustangJaro](https://github.com/mustangJaro) +* [Mykyta-Serbynevskyi](https://github.com/Mykyta-Serbynevskyi) +* [n0rritt](https://github.com/n0rritt) +* [nastra](https://github.com/nastra) +* [nataliekwong](https://github.com/nataliekwong) +* [natalyjazzviolin](https://github.com/natalyjazzviolin) +* [nauxliu](https://github.com/nauxliu) +* [nguyenaiden](https://github.com/nguyenaiden) +* [NipunaPrashan](https://github.com/NipunaPrashan) +* [Nmaxime](https://github.com/Nmaxime) +* [noahkawasaki-airbyte](https://github.com/noahkawasaki-airbyte) +* [noahkawasakigoogle](https://github.com/noahkawasakigoogle) +* [novotl](https://github.com/novotl) * [ntucker](https://github.com/ntucker) -* [numphileo](https://github.com/numphileo) -* [nyergler](https://github.com/nyergler) +* [octavia-squidington-iii](https://github.com/octavia-squidington-iii) * [olivermeyer](https://github.com/olivermeyer) +* [omid](https://github.com/omid) +* [oreopot](https://github.com/oreopot) +* [pabloescoder](https://github.com/pabloescoder) * [panhavad](https://github.com/panhavad) +* [pecalleja](https://github.com/pecalleja) +* [pedroslopez](https://github.com/pedroslopez) +* [perangel](https://github.com/perangel) +* [peter279k](https://github.com/peter279k) +* [PhilipCorr](https://github.com/PhilipCorr) +* [philippeboyd](https://github.com/philippeboyd) * [Phlair](https://github.com/Phlair) +* [pmossman](https://github.com/pmossman) * [po3na4skld](https://github.com/po3na4skld) -* [ppatali](https://github.com/ppatali) +* [PoCTo](https://github.com/PoCTo) +* [postamar](https://github.com/postamar) * [prasrvenkat](https://github.com/prasrvenkat) -* [rclmenezes](https://github.com/rclmenezes) +* [prateekmukhedkar](https://github.com/prateekmukhedkar) +* [proprefenetre](https://github.com/proprefenetre) +* [Pwaldi](https://github.com/Pwaldi) +* [rach-r](https://github.com/rach-r) +* [ramonvermeulen](https://github.com/ramonvermeulen) * [ReptilianBrain](https://github.com/ReptilianBrain) -* [roshan](https://github.com/roshan) +* [rileybrook](https://github.com/rileybrook) +* [RobertoBonnet](https://github.com/RobertoBonnet) +* [robgleason](https://github.com/robgleason) +* [RobLucchi](https://github.com/RobLucchi) +* [rodireich](https://github.com/rodireich) +* [roisinbolt](https://github.com/roisinbolt) +* [roman-romanov-o](https://github.com/roman-romanov-o) +* [roman-yermilov-gl](https://github.com/roman-yermilov-gl) +* [ron-damon](https://github.com/ron-damon) * [rparrapy](https://github.com/rparrapy) -* [sabifranjo](https://github.com/sabifranjo) +* [ryankfu](https://github.com/ryankfu) +* [sajarin](https://github.com/sajarin) +* [samos123](https://github.com/samos123) +* [sarafonseca-123](https://github.com/sarafonseca-123) +* [sashaNeshcheret](https://github.com/sashaNeshcheret) +* [SatishChGit](https://github.com/SatishChGit) +* [sbjorn](https://github.com/sbjorn) +* [schlattk](https://github.com/schlattk) +* [scottleechua](https://github.com/scottleechua) +* [sdairs](https://github.com/sdairs) +* [sergei-solonitcyn](https://github.com/sergei-solonitcyn) +* [sergio-ropero](https://github.com/sergio-ropero) +* [sh4sh](https://github.com/sh4sh) * [shadabshaukat](https://github.com/shadabshaukat) * [sherifnada](https://github.com/sherifnada) -* [subhaklp](https://github.com/subhaklp) +* [Shishir-rmv](https://github.com/Shishir-rmv) +* [shrodingers](https://github.com/shrodingers) +* [shyngysnurzhan](https://github.com/shyngysnurzhan) +* [siddhant3030](https://github.com/siddhant3030) +* [sivankumar86](https://github.com/sivankumar86) +* [snyk-bot](https://github.com/snyk-bot) +* [SofiiaZaitseva](https://github.com/SofiiaZaitseva) +* [sophia-wiley](https://github.com/sophia-wiley) +* [SPTKL](https://github.com/SPTKL) +* [subhamX](https://github.com/subhamX) * [subodh1810](https://github.com/subodh1810) -* [tgiardina](https://github.com/tgiardina) +* [suhomud](https://github.com/suhomud) +* [supertopher](https://github.com/supertopher) +* [swyxio](https://github.com/swyxio) +* [tbcdns](https://github.com/tbcdns) +* [tealjulia](https://github.com/tealjulia) +* [terencecho](https://github.com/terencecho) +* [thanhlmm](https://github.com/thanhlmm) * [thomas-vl](https://github.com/thomas-vl) -* [troyharvey](https://github.com/troyharvey) +* [timroes](https://github.com/timroes) +* [tirth7777777](https://github.com/tirth7777777) +* [tjirab](https://github.com/tjirab) +* [tkorenko](https://github.com/tkorenko) +* [tolik0](https://github.com/tolik0) +* [topefolorunso](https://github.com/topefolorunso) +* [trowacat](https://github.com/trowacat) +* [tryangul](https://github.com/tryangul) +* [TSkrebe](https://github.com/TSkrebe) +* [tuanchris](https://github.com/tuanchris) * [tuliren](https://github.com/tuliren) +* [tyagi-data-wizard](https://github.com/tyagi-data-wizard) +* [tybernstein](https://github.com/tybernstein) * [TymoshokDmytro](https://github.com/TymoshokDmytro) * [tyschroed](https://github.com/tyschroed) -* [varunbpatil](https://github.com/varunbpatil) -* [vinhloc30796](https://github.com/vinhloc30796) +* [ufou](https://github.com/ufou) +* [Upmitt](https://github.com/Upmitt) * [VitaliiMaltsev](https://github.com/VitaliiMaltsev) * [vitaliizazmic](https://github.com/vitaliizazmic) * [vladimir-remar](https://github.com/vladimir-remar) * [vovavovavovavova](https://github.com/vovavovavovavova) -* [vsayer](https://github.com/vsayer) * [wallies](https://github.com/wallies) * [winar-jin](https://github.com/winar-jin) +* [wissevrowl](https://github.com/wissevrowl) +* [Wittiest](https://github.com/Wittiest) +* [wjwatkinson](https://github.com/wjwatkinson) +* [Xabilahu](https://github.com/Xabilahu) +* [xiaohansong](https://github.com/xiaohansong) +* [xpuska513](https://github.com/xpuska513) * [yahu98](https://github.com/yahu98) +* [yannibenoit](https://github.com/yannibenoit) * [yaroslav-dudar](https://github.com/yaroslav-dudar) * [yaroslav-hrytsaienko](https://github.com/yaroslav-hrytsaienko) +* [YatsukBogdan1](https://github.com/YatsukBogdan1) +* [ycherniaiev](https://github.com/ycherniaiev) * [yevhenii-ldv](https://github.com/yevhenii-ldv) +* [YiyangLi](https://github.com/YiyangLi) +* [YowanR](https://github.com/YowanR) +* [yuhuishi-convect](https://github.com/yuhuishi-convect) +* [yurii-bidiuk](https://github.com/yurii-bidiuk) +* [Zawar92](https://github.com/Zawar92) * [zestyping](https://github.com/zestyping) * [Zirochkaa](https://github.com/Zirochkaa) +* [zkid18](https://github.com/zkid18) * [zuc](https://github.com/zuc) * [zzstoatzz](https://github.com/zzstoatzz) +* [zzztimbo](https://github.com/zzztimbo) ```shell p=1; diff --git a/LICENSE b/LICENSE index 814fd88f57f3..0df58b4829be 100644 --- a/LICENSE +++ b/LICENSE @@ -1,14 +1,17 @@ Airbyte monorepo uses multiple licenses. The license for a particular work is defined with following prioritized rules: + 1. License directly present in the file 2. LICENSE file in the same directory as the work -3. First LICENSE found when exploring parent directories up to the project top level directory -4. Defaults to Elastic License 2.0 +3. A `license` property defined in the `metadata.yaml` configuration file found when exploring parent directories (most connectors) +4. First LICENSE found when exploring parent directories up to the project top level directory +5. Defaults to Elastic License 2.0 If you have any question regarding licenses, just visit our [FAQ](https://airbyte.io/license-faq) or [contact us](mailto:license@airbyte.io). ------------------------------------------------------------------------------------- +--- + MIT License Copyright (c) 2020 Airbyte, Inc. @@ -31,7 +34,8 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------------------------------------------------------------------------------------- +--- + Elastic License 2.0 (ELv2) **Acceptance** @@ -65,16 +69,16 @@ If you use the software in violation of these terms, such use is not licensed, a As far as the law allows, the software comes as is, without any warranty or condition, and the licensor will not be liable to you for any damages arising out of these terms or the use or nature of the software, under any kind of legal claim. **Definitions** -The *licensor* is the entity offering these terms, and the *software* is the software the licensor makes available under these terms, including any portion of it. +The _licensor_ is the entity offering these terms, and the _software_ is the software the licensor makes available under these terms, including any portion of it. -*you* refers to the individual or entity agreeing to these terms. +_you_ refers to the individual or entity agreeing to these terms. -*your company* is any legal entity, sole proprietorship, or other kind of organization that you work for, plus all organizations that have control over, are under the control of, or are under common control with that organization. *control* means ownership of substantially all the assets of an entity, or the power to direct its management and policies by vote, contract, or otherwise. Control can be direct or indirect. +_your company_ is any legal entity, sole proprietorship, or other kind of organization that you work for, plus all organizations that have control over, are under the control of, or are under common control with that organization. _control_ means ownership of substantially all the assets of an entity, or the power to direct its management and policies by vote, contract, or otherwise. Control can be direct or indirect. -*your licenses* are all the licenses granted to you for the software under these terms. +_your licenses_ are all the licenses granted to you for the software under these terms. -*use* means anything you do with the software requiring one of your licenses. +_use_ means anything you do with the software requiring one of your licenses. -*trademark* means trademarks, service marks, and similar rights. +_trademark_ means trademarks, service marks, and similar rights. ------------------------------------------------------------------------------------- +--- diff --git a/Makefile b/Makefile new file mode 100644 index 000000000000..6a2dd841c970 --- /dev/null +++ b/Makefile @@ -0,0 +1,49 @@ +##@ Makefile + +##@ Define the default airbyte-ci version +AIRBYTE_CI_VERSION ?= latest + +## Detect the operating system +OS := $(shell uname) + +tools.airbyte-ci.install: tools.airbyte-ci.clean tools.airbyte-ci-binary.install tools.airbyte-ci.check + +tools.airbyte-ci-binary.install: ## Install airbyte-ci binary + @python airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_install.py ${AIRBYTE_CI_VERSION} + +tools.airbyte-ci-dev.install: ## Install the local development version of airbyte-ci + @python airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_dev_install.py + +tools.airbyte-ci.check: ## Check if airbyte-ci is installed correctly + @./airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_check.sh + +tools.airbyte-ci.clean: ## Clean airbyte-ci installations + @./airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_clean.sh + +tools.git-hooks.clean: ## Clean git hooks + @echo "Unset core.hooksPath" + @git config --unset core.hooksPath || true + @echo "Removing pre-commit hooks..." + @pre-commit uninstall + @echo "Removing pre-push hooks..." + @rm -rf .git/hooks + @echo "Git hooks removed." + +tools.pre-commit.install.Linux: + @echo "Installing pre-commit with pip..." + @pip install --user pre-commit + @echo "Pre-commit installation complete." + +tools.pre-commit.install.Darwin: + @echo "Installing pre-commit with brew..." + @brew install pre-commit + @echo "Pre-commit installation complete" + +tools.pre-commit.setup: tools.airbyte-ci.install tools.pre-commit.install.$(OS) tools.git-hooks.clean ## Setup pre-commit hooks + @echo "Installing pre-commit hooks..." + @pre-commit install --hook-type pre-push + @echo "Pre-push hooks installed." + +tools.install: tools.airbyte-ci.install tools.pre-commit.setup + +.PHONY: tools.install tools.pre-commit.setup tools.airbyte-ci.install tools.airbyte-ci-dev.install tools.airbyte-ci.check tools.airbyte-ci.clean diff --git a/airbyte-api/build.gradle b/airbyte-api/build.gradle deleted file mode 100644 index f08c7799436f..000000000000 --- a/airbyte-api/build.gradle +++ /dev/null @@ -1,204 +0,0 @@ -import org.openapitools.generator.gradle.plugin.tasks.GenerateTask - -plugins { - id "org.openapi.generator" version "6.2.1" - id "java-library" -} - -def specFile = "$projectDir/src/main/openapi/config.yaml" - -// Deprecated -- can be removed once airbyte-server is converted to use the per-domain endpoints generated by generateApiServer -task generateApiServerLegacy(type: GenerateTask) { - def serverOutputDir = "$buildDir/generated/api/server" - - inputs.file specFile - outputs.dir serverOutputDir - - generatorName = "jaxrs-spec" - inputSpec = specFile - outputDir = serverOutputDir - - apiPackage = "io.airbyte.api.generated" - invokerPackage = "io.airbyte.api.invoker.generated" - modelPackage = "io.airbyte.api.model.generated" - - schemaMappings = [ - 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'StreamJsonSchema' : 'com.fasterxml.jackson.databind.JsonNode', - 'StateBlob' : 'com.fasterxml.jackson.databind.JsonNode', - 'FieldSchema' : 'com.fasterxml.jackson.databind.JsonNode', - ] - - generateApiDocumentation = false - - configOptions = [ - dateLibrary : "java8", - generatePom : "false", - interfaceOnly: "true", - /* - JAX-RS generator does not respect nullable properties defined in the OpenApi Spec. - It means that if a field is not nullable but not set it is still returning a null value for this field in the serialized json. - The below Jackson annotation is made to only keep non null values in serialized json. - We are not yet using nullable=true properties in our OpenApi so this is a valid workaround at the moment to circumvent the default JAX-RS behavior described above. - Feel free to read the conversation on https://github.com/airbytehq/airbyte/pull/13370 for more details. - */ - additionalModelTypeAnnotations: "\n@com.fasterxml.jackson.annotation.JsonInclude(com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL)", - ] -} - -task generateApiServer(type: GenerateTask) { - def serverOutputDir = "$buildDir/generated/api/server" - - inputs.file specFile - outputs.dir serverOutputDir - - generatorName = "jaxrs-spec" - inputSpec = specFile - outputDir = serverOutputDir - - apiPackage = "io.airbyte.api.generated" - invokerPackage = "io.airbyte.api.invoker.generated" - modelPackage = "io.airbyte.api.model.generated" - - schemaMappings = [ - 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'StreamJsonSchema' : 'com.fasterxml.jackson.databind.JsonNode', - 'StateBlob' : 'com.fasterxml.jackson.databind.JsonNode', - 'FieldSchema' : 'com.fasterxml.jackson.databind.JsonNode', - ] - - generateApiDocumentation = false - - configOptions = [ - dateLibrary : "java8", - generatePom : "false", - interfaceOnly: "true", - /* - JAX-RS generator does not respect nullable properties defined in the OpenApi Spec. - It means that if a field is not nullable but not set it is still returning a null value for this field in the serialized json. - The below Jackson annotation is made to only keep non null values in serialized json. - We are not yet using nullable=true properties in our OpenApi so this is a valid workaround at the moment to circumvent the default JAX-RS behavior described above. - Feel free to read the conversation on https://github.com/airbytehq/airbyte/pull/13370 for more details. - */ - additionalModelTypeAnnotations: "\n@com.fasterxml.jackson.annotation.JsonInclude(com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL)", - - // Generate separate classes for each endpoint "domain" - useTags: "true" - ] -} - -compileJava.dependsOn tasks.generateApiServerLegacy, tasks.generateApiServer - -task generateApiClient(type: GenerateTask) { - def clientOutputDir = "$buildDir/generated/api/client" - - inputs.file specFile - outputs.dir clientOutputDir - - generatorName = "java" - inputSpec = specFile - outputDir = clientOutputDir - - apiPackage = "io.airbyte.api.client.generated" - invokerPackage = "io.airbyte.api.client.invoker.generated" - modelPackage = "io.airbyte.api.client.model.generated" - - schemaMappings = [ - 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'StreamJsonSchema' : 'com.fasterxml.jackson.databind.JsonNode', - 'StateBlob' : 'com.fasterxml.jackson.databind.JsonNode', - 'FieldSchema' : 'com.fasterxml.jackson.databind.JsonNode', - ] - - library = "native" - - generateApiDocumentation = false - - configOptions = [ - dateLibrary : "java8", - generatePom : "false", - interfaceOnly: "true" - ] -} -compileJava.dependsOn tasks.generateApiClient - -task generateApiDocs(type: GenerateTask) { - def docsOutputDir = "$buildDir/generated/api/docs" - - generatorName = "html" - inputSpec = specFile - outputDir = docsOutputDir - - apiPackage = "io.airbyte.api.client.generated" - invokerPackage = "io.airbyte.api.client.invoker.generated" - modelPackage = "io.airbyte.api.client.model.generated" - - schemaMappings = [ - 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', - 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', - 'DestinationConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', - 'StreamJsonSchema' : 'com.fasterxml.jackson.databind.JsonNode', - 'StateBlob' : 'com.fasterxml.jackson.databind.JsonNode', - 'FieldSchema' : 'com.fasterxml.jackson.databind.JsonNode', - ] - - generateApiDocumentation = false - - configOptions = [ - dateLibrary : "java8", - generatePom : "false", - interfaceOnly: "true" - ] - - doLast { - def target = file(rootProject.file("docs/reference/api/generated-api-html")) - delete target - mkdir target - copy { - from outputDir - include "**/*.html" - includeEmptyDirs = false - into target - } - } -} -compileJava.dependsOn tasks.generateApiDocs - -dependencies { - implementation group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310' - - implementation group: 'io.swagger', name: 'swagger-annotations', version: '1.6.2' - - implementation group: 'javax.annotation', name: 'javax.annotation-api', version: '1.3.2' - implementation group: 'javax.ws.rs', name: 'javax.ws.rs-api', version: '2.1.1' - implementation group: 'javax.validation', name: 'validation-api', version: '2.0.1.Final' - - implementation group: 'org.openapitools', name: 'jackson-databind-nullable', version: '0.2.1' -} - -sourceSets { - main { - java { - srcDirs "$buildDir/generated/api/server/src/gen/java", "$buildDir/generated/api/client/src/main/java", "$projectDir/src/main/java" - } - resources { - srcDir "$projectDir/src/main/openapi/" - } - } -} - -Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project) diff --git a/airbyte-cdk/java/airbyte-cdk/README.md b/airbyte-cdk/java/airbyte-cdk/README.md index e8e106c02372..3939f9081a4c 100644 --- a/airbyte-cdk/java/airbyte-cdk/README.md +++ b/airbyte-cdk/java/airbyte-cdk/README.md @@ -2,7 +2,50 @@ This page will walk through the process of developing with the Java CDK. -## Building the CDK +* [Developing with the Java CDK](#developing-with-the-java-cdk) + * [Intro to the Java CDK](#intro-to-the-java-cdk) + * [What is included in the Java CDK?](#what-is-included-in-the-java-cdk) + * [How is the CDK published?](#how-is-the-cdk-published) + * [Using the Java CDK](#using-the-java-cdk) + * [Building the CDK](#building-the-cdk) + * [Bumping the CDK version](#bumping-the-cdk-version) + * [Publishing the CDK](#publishing-the-cdk) + * [Developing Connectors with the Java CDK](#developing-connectors-with-the-java-cdk) + * [Referencing the CDK from Java connectors](#referencing-the-cdk-from-java-connectors) + * [Developing a connector alongside the CDK](#developing-a-connector-alongside-the-cdk) + * [Publishing the CDK and switching to a pinned CDK reference](#publishing-the-cdk-and-switching-to-a-pinned-cdk-reference) + * [Troubleshooting CDK Dependency Caches](#troubleshooting-cdk-dependency-caches) + * [Developing a connector against a pinned CDK version](#developing-a-connector-against-a-pinned-cdk-version) + * [Common Debugging Tips](#common-debugging-tips) + * [Changelog](#changelog) + * [Java CDK](#java-cdk) + +## Intro to the Java CDK + +### What is included in the Java CDK? + +The java CDK is comprised of separate modules: + +- `core` - Shared classes for building connectors of all types. +- `db-sources` - Shared classes for building DB sources. +- `db-destinations` - Shared classes for building DB destinations. + +Each CDK submodule may contain these elements: + +- `src/main` - (Required.) The classes that will ship with the connector, providing capabilities to the connectors. +- `src/test` - (Required.) These are unit tests that run as part of every build of the CDK. They help ensure that CDK `main` code is in a healthy state. +- `src/test-integration` - (Optional.) Integration tests which provide a more extensive test of the code in `src/main`. These are not by the `build` command but are executed as part of the `integrationTest` or `integrationTestJava` Gradle tasks. +- `src/testFixtures` - (Optional.) These shared classes are exported for connectors for use in the connectors' own test implementations. Connectors will have access to these classes within their unit and integration tests, but the classes will not be shipped with connectors when they are published. + +### How is the CDK published? + +The CDK is published as a set of jar files sharing a version number. Every submodule generates one runtime jar for the main classes. If the submodule contains test fixtures, a second jar will be published with the test fixtures classes. + +Note: Connectors do not have to manage which jars they should depend on, as this is handled automatically by the `airbyte-java-connector` plugin. See example below. + +## Using the Java CDK + +### Building the CDK To build and test the Java CDK, execute the following: @@ -10,83 +53,108 @@ To build and test the Java CDK, execute the following: ./gradlew :airbyte-cdk:java:airbyte-cdk:build ``` -## Bumping the declared CDK version +### Bumping the CDK version You will need to bump this version manually whenever you are making changes to code inside the CDK. -While under development, the next version number for the CDK is tracked in the file: `airbyte-cdk/java/airbyte-cdk/src/main/resources/version.properties`. +While under development, the next version number for the CDK is tracked in the file: `airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties`. If the CDK is not being modified, this file will contain the most recently published version number. -## Publishing the CDK to Local Maven +### Publishing the CDK + +_⚠️ These steps should only be performed after all testing and approvals are in place on the PR. ⚠️_ + +The CDK can be published with a GitHub Workflow and a slash command which can be run by Airbyte personnel. + +To invoke via slash command (recommended), use the following syntax in a comment on the PR that contains your changes: + +```bash +/publish-java-cdk # Run with the defaults (dry-run=false, force=false) +/publish-java-cdk dry-run=true # Run in dry-run mode (no-op) +/publish-java-cdk force=true # Force-publish if needing to replace an already published version +``` + +Note: + +- Remember to **document your changes** in the Changelog section below. +- After you publish the CDK, remember to toggle `useLocalCdk` back to `false` in all connectors. +- Unless you specify `force=true`, the pipeline should fail if the version you are trying to publish already exists. +- By running the publish with `dry-run=true`, you can confirm the process is working as expected, without actually publishing the changes. +- In dry-run mode, you can also view and download the jars that are generated. To do so, navigate to the job status in GitHub Actions and navigate to the 'artifacts' section. +- You can also invoke manually in the GitHub Web UI. To do so: go to `Actions` tab, select the `Publish Java CDK` workflow, and click `Run workflow`. +- You can view and administer published CDK versions here: https://admin.cloudrepo.io/repository/airbyte-public-jars/io/airbyte/airbyte-cdk +- The public endpoint for published CDK versions is here: https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/io/airbyte/airbyte-cdk/ -If your connector pins to a work-in-progress `-SNAPSHOT` version of the CDK (e.g. `0.0.1-SNAPSHOT` or `0.2.0-SNAPSHOT`), Gradle will notice this and automatically run the task to build and publish it to your MavenLocal repository before running the connector's own build and test tasks. +## Developing Connectors with the Java CDK -## Referencing the CDK from Java connectors +### Referencing the CDK from Java connectors You can reference the CDK in your connector's `build.gradle` file: ```groovy -dependencies { - implementation 'io.airbyte:airbyte-cdk:0.0.1-SNAPSHOT' +plugins { + id 'application' + id 'airbyte-java-connector' +} + +airbyteJavaConnector { + cdkVersionRequired = '0.1.0' // The CDK version to pin to. + features = ['db-destinations'] // An array of CDK features to depend on. + useLocalCdk = false // Use 'true' to use a live reference to the + // local cdk project. } + +airbyteJavaConnector.addCdkDependencies() ``` -Replace `0.0.1-SNAPSHOT` with the version you are working with. If you're actively developing the CDK and want to use the latest version locally, use the `-SNAPSHOT` suffix to reference a bumped version number. (See below for version bump instructions.) +Replace `0.1.0` with the CDK version you are working with. If you're actively developing the CDK and want to use the latest version locally, use the `useLocalCdk` flag to use the live CDK code during builds and tests. -## Developing a connector alongside the CDK +### Developing a connector alongside the CDK You can iterate on changes in the CDK local and test them in the connector without needing to publish the CDK changes publicly. When modifying the CDK and a connector in the same PR or branch, please use the following steps: -1. Set the version of the CDK in `version.properties` to the next appropriate version number, along with a `-SNAPSHOT` suffix, as explained above. -1. In your connector project, modify the `build.gradle` to use the _new_ local CDK version with the `-SNAPSHOT` suffix, as explained above. -1. Build and test your connector as usual. Gradle will automatically build the snapshot version of the CDK, and it will use this version when building and testing your connector. -1. As you make additional changes to the CDK, Gradle will automatically rebuild and republish the CDK locally in order to incorporate the latest changes. +1. Set the version of the CDK in `version.properties` to the next appropriate version number and add a description in the `Changelog` at the bottom of this readme file. +2. Modify your connector's build.gradle file as follows: + 1. Set `useLocalCdk` to `true` in the connector you are working on. This will ensure the connector always uses the local CDK definitions instead of the published version. + 2. Set `cdkVersionRequired` to use the new _to-be-published_ CDK version. -## Developing a connector against a pinned CDK version +After the above, you can build and test your connector as usual. Gradle will automatically use the local CDK code files while you are working on the connector. -You can always pin your connector to a prior stable version of the CDK, which may not match what is the latest version in the `airbyte` repo. For instance, your connector can be pinned to `0.1.1` while the latest version may be `0.2.0`. +### Publishing the CDK and switching to a pinned CDK reference -Maven and Gradle will automatically reference the correct (pinned) version of the CDK for your connector, and you can use your local IDE to browse the prior version of the codebase that corresponds to that version. +Once you are done developing and testing your CDK changes: - +_Note: You can also use `./gradlew assertNotUsingLocalCdk` or `./gradlew disableLocalCdkRefs` to run these tasks on **all** connectors simultaneously._ -## Publish and release +### Troubleshooting CDK Dependency Caches -_⚠️ These steps should only be performed after all testing and approvals are in place on the PR. ⚠️_ +Note: after switching between a local and a pinned CDK reference, you may need to refresh dependency caches in Gradle and/or your IDE. -1. Remove `-SNAPSHOT` suffix from CDK version. - - e.g. by running `nano airbyte-cdk/java/airbyte-cdk/src/main/resources/ -version.properties`. -2. Publish the CDK to Maven ([mycloudrepo](https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/io/airbyte/airbyte-cdk/)) - - `./gradlew :airbyte-cdk:java:airbyte-cdk:publish` - - Note: you will need to export the env vars `CLOUDREPO_USER` and `CLOUDREPO_PASSWORD` before publishing. -3. Remove the `-SNAPSHOT` suffix from any connector(s) using the latest version. - - E.g. If modifying `source-mysql`, then remove `-SNAPSHOT` from the CDK `implements` declaration in `airbyte-integrations/connectors/source-mysql/build.gradle`. -4. As per the normal process, modified connector(s) will be automatically published after they are merged to the main branch. +In Gradle, you can use the CLI arg `--refresh-dependencies` the next time you build or test your connector, which will ensure that the correct version of the CDK is used after toggling the `useLocalCdk` value. -Note: +### Developing a connector against a pinned CDK version -- This is documented as a manual process, but we will automate it into a CI workflow. -- You can view and administer published CDK versions here: https://admin.cloudrepo.io/repository/airbyte-public-jars/io/airbyte/airbyte-cdk -- The corresponding public endpoint for published CDK versions is here: https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/io/airbyte/airbyte-cdk/ +You can always pin your connector to a prior stable version of the CDK, which may not match what is the latest version in the `airbyte` repo. For instance, your connector can be pinned to `0.1.1` while the latest version may be `0.2.0`. + +Maven and Gradle will automatically reference the correct (pinned) version of the CDK for your connector, and you can use your local IDE to browse the prior version of the codebase that corresponds to that version. -## Debugging +## Common Debugging Tips MavenLocal debugging steps: 1. Confirm local publish status by running: - `ls -la ~/.m2/repository/io/airbyte/airbyte-cdk\*` + `ls -la ~/.m2/repository/io/airbyte/airbyte-cdk/*` 2. Confirm jar contents by running: `jar tf ~/.m2/repository/io/airbyte/airbyte-cdk/0.0.2-SNAPSHOT/airbyte-cdk-0.0.2-SNAPSHOT.jar` 3. Remove CDK artifacts from MavenLocal by running: - `rm -rf ~/.m2/repository/io/airbyte/airbyte-cdk\*` + `rm -rf ~/.m2/repository/io/airbyte/airbyte-cdk/*` 4. Rebuid CDK artifacts by running: `./gradlew :airbyte-cdk:java:airbyte-cdk:build` or @@ -96,7 +164,65 @@ MavenLocal debugging steps: ### Java CDK -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :--------------------------------------------------------- | :------------------------------------ | -| 0.0.2 | 2023-08-21 | [\#28687](https://github.com/airbytehq/airbyte/pull/28687) | Version bump only (no other changes). | -| 0.0.1 | 2023-08-08 | [\#28687](https://github.com/airbytehq/airbyte/pull/28687) | Initial release for testing. | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.13.2 | 2024-01-18 | [\#34364](https://github.com/airbytehq/airbyte/pull/34364) | Better logging in mongo db source connector | +| 0.13.1 | 2024-01-18 | [\#34236](https://github.com/airbytehq/airbyte/pull/34236) | Add postCreateTable hook in destination JdbcSqlGenerator | +| 0.13.0 | 2024-01-16 | [\#34177](https://github.com/airbytehq/airbyte/pull/34177) | Add `useExpensiveSafeCasting` param in JdbcSqlGenerator methods; add JdbcTypingDedupingTest fixture; other DV2-related changes | +| 0.12.1 | 2024-01-11 | [\#34186](https://github.com/airbytehq/airbyte/pull/34186) | Add hook for additional destination specific checks to JDBC destination check method | +| 0.12.0 | 2024-01-10 | [\#33875](https://github.com/airbytehq/airbyte/pull/33875) | Upgrade sshd-mina to 2.11.1 | +| 0.11.5 | 2024-01-10 | [\#34119](https://github.com/airbytehq/airbyte/pull/34119) | Remove wal2json support for postgres+debezium. | +| 0.11.4 | 2024-01-09 | [\#33305](https://github.com/airbytehq/airbyte/pull/33305) | Source stats in incremental syncs | +| 0.11.3 | 2023-01-09 | [\#33658](https://github.com/airbytehq/airbyte/pull/33658) | Always fail when debezium fails, even if it happened during the setup phase. | +| 0.11.2 | 2024-01-09 | [\#33969](https://github.com/airbytehq/airbyte/pull/33969) | Destination state stats implementation | +| 0.11.1 | 2024-01-04 | [\#33727](https://github.com/airbytehq/airbyte/pull/33727) | SSH bastion heartbeats for Destinations | +| 0.11.0 | 2024-01-04 | [\#33730](https://github.com/airbytehq/airbyte/pull/33730) | DV2 T+D uses Sql struct to represent transactions; other T+D-related changes | +| 0.10.4 | 2023-12-20 | [\#33071](https://github.com/airbytehq/airbyte/pull/33071) | Add the ability to parse JDBC parameters with another delimiter than '&' | +| 0.10.3 | 2024-01-03 | [\#33312](https://github.com/airbytehq/airbyte/pull/33312) | Send out count in AirbyteStateMessage | +| 0.10.1 | 2023-12-21 | [\#33723](https://github.com/airbytehq/airbyte/pull/33723) | Make memory-manager log message less scary | +| 0.10.0 | 2023-12-20 | [\#33704](https://github.com/airbytehq/airbyte/pull/33704) | JdbcDestinationHandler now properly implements `getInitialRawTableState`; reenable SqlGenerator test | +| 0.9.0 | 2023-12-18 | [\#33124](https://github.com/airbytehq/airbyte/pull/33124) | Make Schema Creation Separate from Table Creation, exclude the T&D module from the CDK | +| 0.8.0 | 2023-12-18 | [\#33506](https://github.com/airbytehq/airbyte/pull/33506) | Improve async destination shutdown logic; more JDBC async migration work; improve DAT test schema handling | +| 0.7.9 | 2023-12-18 | [\#33549](https://github.com/airbytehq/airbyte/pull/33549) | Improve MongoDB logging. | +| 0.7.8 | 2023-12-18 | [\#33365](https://github.com/airbytehq/airbyte/pull/33365) | Emit stream statuses more consistently | +| 0.7.7 | 2023-12-18 | [\#33434](https://github.com/airbytehq/airbyte/pull/33307) | Remove LEGACY state | +| 0.7.6 | 2023-12-14 | [\#32328](https://github.com/airbytehq/airbyte/pull/33307) | Add schema less mode for mongodb CDC. Fixes for non standard mongodb id type. | +| 0.7.4 | 2023-12-13 | [\#33232](https://github.com/airbytehq/airbyte/pull/33232) | Track stream record count during sync; only run T+D if a stream had nonzero records or the previous sync left unprocessed records. | +| 0.7.3 | 2023-12-13 | [\#33369](https://github.com/airbytehq/airbyte/pull/33369) | Extract shared JDBC T+D code. | +| 0.7.2 | 2023-12-11 | [\#33307](https://github.com/airbytehq/airbyte/pull/33307) | Fix DV2 JDBC type mappings (code changes in [\#33307](https://github.com/airbytehq/airbyte/pull/33307)). | +| 0.7.1 | 2023-12-01 | [\#33027](https://github.com/airbytehq/airbyte/pull/33027) | Add the abstract DB source debugger. | +| 0.7.0 | 2023-12-07 | [\#32326](https://github.com/airbytehq/airbyte/pull/32326) | Destinations V2 changes for JDBC destinations | +| 0.6.4 | 2023-12-06 | [\#33082](https://github.com/airbytehq/airbyte/pull/33082) | Improvements to schema snapshot error handling + schema snapshot history scope (scoped to configured DB). | +| 0.6.2 | 2023-11-30 | [\#32573](https://github.com/airbytehq/airbyte/pull/32573) | Update MSSQLConverter to enforce 6-digit microsecond precision for timestamp fields | +| 0.6.1 | 2023-11-30 | [\#32610](https://github.com/airbytehq/airbyte/pull/32610) | Support DB initial sync using binary as primary key. | +| 0.6.0 | 2023-11-30 | [\#32888](https://github.com/airbytehq/airbyte/pull/32888) | JDBC destinations now use the async framework | +| 0.5.3 | 2023-11-28 | [\#32686](https://github.com/airbytehq/airbyte/pull/32686) | Better attribution of debezium engine shutdown due to heartbeat. | +| 0.5.1 | 2023-11-27 | [\#32662](https://github.com/airbytehq/airbyte/pull/32662) | Debezium initialization wait time will now read from initial setup time. | +| 0.5.0 | 2023-11-22 | [\#32656](https://github.com/airbytehq/airbyte/pull/32656) | Introduce TestDatabase test fixture, refactor database source test base classes. | +| 0.4.11 | 2023-11-14 | [\#32526](https://github.com/airbytehq/airbyte/pull/32526) | Clean up memory manager logs. | +| 0.4.10 | 2023-11-13 | [\#32285](https://github.com/airbytehq/airbyte/pull/32285) | Fix UUID codec ordering for MongoDB connector | +| 0.4.9 | 2023-11-13 | [\#32468](https://github.com/airbytehq/airbyte/pull/32468) | Further error grouping improvements for DV2 connectors | +| 0.4.8 | 2023-11-09 | [\#32377](https://github.com/airbytehq/airbyte/pull/32377) | source-postgres tests: skip dropping database | +| 0.4.7 | 2023-11-08 | [\#31856](https://github.com/airbytehq/airbyte/pull/31856) | source-postgres: support for infinity date and timestamps | +| 0.4.5 | 2023-11-07 | [\#32112](https://github.com/airbytehq/airbyte/pull/32112) | Async destinations framework: Allow configuring the queue flush threshold | +| 0.4.4 | 2023-11-06 | [\#32119](https://github.com/airbytehq/airbyte/pull/32119) | Add STANDARD UUID codec to MongoDB debezium handler | +| 0.4.2 | 2023-11-06 | [\#32190](https://github.com/airbytehq/airbyte/pull/32190) | Improve error deinterpolation | +| 0.4.1 | 2023-11-02 | [\#32192](https://github.com/airbytehq/airbyte/pull/32192) | Add 's3-destinations' CDK module. | +| 0.4.0 | 2023-11-02 | [\#32050](https://github.com/airbytehq/airbyte/pull/32050) | Fix compiler warnings. | +| 0.3.0 | 2023-11-02 | [\#31983](https://github.com/airbytehq/airbyte/pull/31983) | Add deinterpolation feature to AirbyteExceptionHandler. | +| 0.2.4 | 2023-10-31 | [\#31807](https://github.com/airbytehq/airbyte/pull/31807) | Handle case of debezium update and delete of records in mongodb. | +| 0.2.3 | 2023-10-31 | [\#32022](https://github.com/airbytehq/airbyte/pull/32022) | Update Debezium version from 2.20 -> 2.4.0. | +| 0.2.2 | 2023-10-31 | [\#31976](https://github.com/airbytehq/airbyte/pull/31976) | Debezium tweaks to make tests run faster. | +| 0.2.0 | 2023-10-30 | [\#31960](https://github.com/airbytehq/airbyte/pull/31960) | Hoist top-level gradle subprojects into CDK. | +| 0.1.12 | 2023-10-24 | [\#31674](https://github.com/airbytehq/airbyte/pull/31674) | Fail sync when Debezium does not shut down properly. | +| 0.1.11 | 2023-10-18 | [\#31486](https://github.com/airbytehq/airbyte/pull/31486) | Update constants in AdaptiveSourceRunner. | +| 0.1.9 | 2023-10-12 | [\#31309](https://github.com/airbytehq/airbyte/pull/31309) | Use toPlainString() when handling BigDecimals in PostgresConverter | +| 0.1.8 | 2023-10-11 | [\#31322](https://github.com/airbytehq/airbyte/pull/31322) | Cap log line length to 32KB to prevent loss of records | +| 0.1.7 | 2023-10-10 | [\#31194](https://github.com/airbytehq/airbyte/pull/31194) | Deallocate unused per stream buffer memory when empty | +| 0.1.6 | 2023-10-10 | [\#31083](https://github.com/airbytehq/airbyte/pull/31083) | Fix precision of numeric values in async destinations | +| 0.1.5 | 2023-10-09 | [\#31196](https://github.com/airbytehq/airbyte/pull/31196) | Update typo in CDK (CDN_LSN -> CDC_LSN) | +| 0.1.4 | 2023-10-06 | [\#31139](https://github.com/airbytehq/airbyte/pull/31139) | Reduce async buffer | +| 0.1.1 | 2023-09-28 | [\#30835](https://github.com/airbytehq/airbyte/pull/30835) | JDBC destinations now avoid staging area name collisions by using the raw table name as the stage name. (previously we used the stream name as the stage name) | +| 0.1.0 | 2023-09-27 | [\#30445](https://github.com/airbytehq/airbyte/pull/30445) | First launch, including shared classes for all connectors. | +| 0.0.2 | 2023-08-21 | [\#28687](https://github.com/airbytehq/airbyte/pull/28687) | Version bump only (no other changes). | +| 0.0.1 | 2023-08-08 | [\#28687](https://github.com/airbytehq/airbyte/pull/28687) | Initial release for testing. | diff --git a/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/build.gradle b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/build.gradle new file mode 100644 index 000000000000..0a12cf304887 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/build.gradle @@ -0,0 +1,50 @@ +plugins { + id "java-library" +} + +java { + compileJava { + options.compilerArgs += "-Xlint:-try" + } +} + +dependencies { + annotationProcessor platform(libs.micronaut.bom) + annotationProcessor libs.bundles.micronaut.annotation.processor + + implementation platform(libs.micronaut.bom) + implementation libs.bundles.micronaut + + implementation group: 'joda-time', name: 'joda-time', version: '2.12.5' + implementation 'io.fabric8:kubernetes-client:5.12.2' + implementation 'com.auth0:java-jwt:3.19.2' + implementation libs.guava + implementation(libs.temporal.sdk) { + exclude module: 'guava' + } + implementation 'org.apache.ant:ant:1.10.10' + implementation 'org.apache.commons:commons-text:1.10.0' + implementation libs.bundles.datadog + implementation group: 'io.swagger', name: 'swagger-annotations', version: '1.6.2' + + implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-api') + implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-protocol') + implementation project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') + implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') + + testAnnotationProcessor platform(libs.micronaut.bom) + testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor + testAnnotationProcessor libs.jmh.annotations + + testImplementation libs.bundles.micronaut.test + testImplementation 'com.jayway.jsonpath:json-path:2.7.0' + testImplementation 'org.mockito:mockito-inline:4.7.0' + testImplementation libs.postgresql + testImplementation libs.testcontainers + testImplementation libs.testcontainers.postgresql + testImplementation libs.jmh.core + testImplementation libs.jmh.annotations + testImplementation 'com.github.docker-java:docker-java:3.2.8' + testImplementation 'com.github.docker-java:docker-java-transport-httpclient5:3.2.8' +} diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/TestHarness.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/TestHarness.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/TestHarness.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/TestHarness.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/TestHarnessUtils.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/TestHarnessUtils.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/TestHarnessUtils.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/TestHarnessUtils.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/WorkerConstants.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/WorkerConstants.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/WorkerConstants.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/WorkerConstants.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/exception/TestHarnessException.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/exception/TestHarnessException.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/exception/TestHarnessException.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/exception/TestHarnessException.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/general/CheckConnectionTestHarness.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/CheckConnectionTestHarness.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/general/CheckConnectionTestHarness.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/CheckConnectionTestHarness.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DbtTransformationRunner.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DbtTransformationRunner.java similarity index 97% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DbtTransformationRunner.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DbtTransformationRunner.java index 8235f1ab3384..29d72014dd43 100644 --- a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DbtTransformationRunner.java +++ b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DbtTransformationRunner.java @@ -14,7 +14,6 @@ import com.google.common.collect.ImmutableMap; import io.airbyte.commons.io.LineGobbler; import io.airbyte.commons.logging.LoggingHelper.Color; -import io.airbyte.commons.logging.MdcScope; import io.airbyte.commons.logging.MdcScope.Builder; import io.airbyte.commons.resources.MoreResources; import io.airbyte.configoss.OperatorDbt; @@ -37,7 +36,7 @@ public class DbtTransformationRunner implements AutoCloseable { private static final Logger LOGGER = LoggerFactory.getLogger(DbtTransformationRunner.class); private static final String DBT_ENTRYPOINT_SH = "entrypoint.sh"; - private static final MdcScope.Builder CONTAINER_LOG_MDC_BUILDER = new Builder() + private static final Builder CONTAINER_LOG_MDC_BUILDER = new Builder() .setLogPrefix("dbt") .setPrefixColor(Color.PURPLE_BACKGROUND); diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultCheckConnectionTestHarness.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultCheckConnectionTestHarness.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultCheckConnectionTestHarness.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultCheckConnectionTestHarness.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultDiscoverCatalogTestHarness.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultDiscoverCatalogTestHarness.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultDiscoverCatalogTestHarness.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultDiscoverCatalogTestHarness.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultGetSpecTestHarness.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultGetSpecTestHarness.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultGetSpecTestHarness.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DefaultGetSpecTestHarness.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DiscoverCatalogTestHarness.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DiscoverCatalogTestHarness.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DiscoverCatalogTestHarness.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/DiscoverCatalogTestHarness.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/general/GetSpecTestHarness.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/GetSpecTestHarness.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/general/GetSpecTestHarness.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/general/GetSpecTestHarness.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/CatalogClientConverters.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/CatalogClientConverters.java similarity index 87% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/CatalogClientConverters.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/CatalogClientConverters.java index 52cf8c1c2db1..99fad0c29e18 100644 --- a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/CatalogClientConverters.java +++ b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/CatalogClientConverters.java @@ -49,8 +49,8 @@ public static io.airbyte.protocol.models.AirbyteCatalog toAirbyteProtocol(final } @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") - private static io.airbyte.protocol.models.AirbyteStream toConfiguredProtocol(final io.airbyte.api.client.model.generated.AirbyteStream stream, - AirbyteStreamConfiguration config) + private static AirbyteStream toConfiguredProtocol(final io.airbyte.api.client.model.generated.AirbyteStream stream, + AirbyteStreamConfiguration config) throws JsonValidationException { if (config.getFieldSelectionEnabled() != null && config.getFieldSelectionEnabled()) { // Validate the selected field paths. @@ -95,7 +95,7 @@ private static io.airbyte.protocol.models.AirbyteStream toConfiguredProtocol(fin } ((ObjectNode) properties).retain(selectedFieldNames); } - return new io.airbyte.protocol.models.AirbyteStream() + return new AirbyteStream() .withName(stream.getName()) .withJsonSchema(stream.getJsonSchema()) .withSupportedSyncModes(Enums.convertListTo(stream.getSupportedSyncModes(), io.airbyte.protocol.models.SyncMode.class)) @@ -121,20 +121,20 @@ public static io.airbyte.api.client.model.generated.AirbyteCatalog toAirbyteCata .collect(Collectors.toList())); } - private static io.airbyte.api.client.model.generated.AirbyteStreamConfiguration generateDefaultConfiguration( - final io.airbyte.api.client.model.generated.AirbyteStream stream) { - final io.airbyte.api.client.model.generated.AirbyteStreamConfiguration result = - new io.airbyte.api.client.model.generated.AirbyteStreamConfiguration() + private static AirbyteStreamConfiguration generateDefaultConfiguration( + final io.airbyte.api.client.model.generated.AirbyteStream stream) { + final AirbyteStreamConfiguration result = + new AirbyteStreamConfiguration() .aliasName(Names.toAlphanumericAndUnderscore(stream.getName())) .cursorField(stream.getDefaultCursorField()) - .destinationSyncMode(io.airbyte.api.client.model.generated.DestinationSyncMode.APPEND) + .destinationSyncMode(DestinationSyncMode.APPEND) .primaryKey(stream.getSourceDefinedPrimaryKey()) .selected(true); if (stream.getSupportedSyncModes().size() > 0) { result.setSyncMode(Enums.convertTo(stream.getSupportedSyncModes().get(0), - io.airbyte.api.client.model.generated.SyncMode.class)); + SyncMode.class)); } else { - result.setSyncMode(io.airbyte.api.client.model.generated.SyncMode.INCREMENTAL); + result.setSyncMode(SyncMode.INCREMENTAL); } return result; } @@ -145,7 +145,7 @@ private static io.airbyte.api.client.model.generated.AirbyteStream toAirbyteStre .name(stream.getName()) .jsonSchema(stream.getJsonSchema()) .supportedSyncModes(Enums.convertListTo(stream.getSupportedSyncModes(), - io.airbyte.api.client.model.generated.SyncMode.class)) + SyncMode.class)) .sourceDefinedCursor(stream.getSourceDefinedCursor()) .defaultCursorField(stream.getDefaultCursorField()) .sourceDefinedPrimaryKey(stream.getSourceDefinedPrimaryKey()) diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/ConnectorConfigUpdater.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/ConnectorConfigUpdater.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/ConnectorConfigUpdater.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/ConnectorConfigUpdater.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/EntrypointEnvChecker.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/EntrypointEnvChecker.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/EntrypointEnvChecker.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/EntrypointEnvChecker.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/FailureHelper.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/FailureHelper.java similarity index 97% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/FailureHelper.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/FailureHelper.java index f63e4ce141fa..a518a98d4ff5 100644 --- a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/FailureHelper.java +++ b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/helper/FailureHelper.java @@ -64,7 +64,7 @@ public static FailureReason genericFailure(final AirbyteTraceMessage m, final Lo } else { try { final String traceMessageError = m.getError().getFailureType().toString(); - failureType = FailureReason.FailureType.fromValue(traceMessageError); + failureType = FailureType.fromValue(traceMessageError); } catch (final IllegalArgumentException e) { // the trace message error does not exist as a FailureReason failure type, // so set the failure type to null @@ -125,10 +125,10 @@ public static FailureReason destinationFailure(final AirbyteTraceMessage m, fina public static FailureReason checkFailure(final Throwable t, final Long jobId, final Integer attemptNumber, - final FailureReason.FailureOrigin origin) { + final FailureOrigin origin) { return connectorCommandFailure(t, jobId, attemptNumber, ConnectorCommand.CHECK) .withFailureOrigin(origin) - .withFailureType(FailureReason.FailureType.CONFIG_ERROR) + .withFailureType(FailureType.CONFIG_ERROR) .withRetryable(false) .withExternalMessage(String .format("Checking %s connection failed - please review this connection's configuration to prevent future syncs from failing", origin)); diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteDestination.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteDestination.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteDestination.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteDestination.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriter.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriter.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriter.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriter.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriterFactory.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriterFactory.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriterFactory.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteMessageBufferedWriterFactory.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteProtocolPredicate.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteProtocolPredicate.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteProtocolPredicate.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteProtocolPredicate.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteSource.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteSource.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteSource.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteSource.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteStreamFactory.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteStreamFactory.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteStreamFactory.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/AirbyteStreamFactory.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java similarity index 98% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java index ff4c610a805f..4d5b07bd873e 100644 --- a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java +++ b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteDestination.java @@ -10,7 +10,6 @@ import io.airbyte.commons.io.LineGobbler; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.logging.LoggingHelper.Color; -import io.airbyte.commons.logging.MdcScope; import io.airbyte.commons.logging.MdcScope.Builder; import io.airbyte.commons.protocol.DefaultProtocolSerializer; import io.airbyte.commons.protocol.ProtocolSerializer; @@ -38,7 +37,7 @@ public class DefaultAirbyteDestination implements AirbyteDestination { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultAirbyteDestination.class); - public static final MdcScope.Builder CONTAINER_LOG_MDC_BUILDER = new Builder() + public static final Builder CONTAINER_LOG_MDC_BUILDER = new Builder() .setLogPrefix("destination") .setPrefixColor(Color.YELLOW_BACKGROUND); static final Set IGNORED_EXIT_CODES = Set.of( diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriter.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriter.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriter.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriter.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriterFactory.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriterFactory.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriterFactory.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteMessageBufferedWriterFactory.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteSource.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteSource.java similarity index 98% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteSource.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteSource.java index 2cba6ca3a3f4..8052255bfc55 100644 --- a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteSource.java +++ b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteSource.java @@ -11,7 +11,6 @@ import io.airbyte.commons.io.LineGobbler; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.logging.LoggingHelper.Color; -import io.airbyte.commons.logging.MdcScope; import io.airbyte.commons.logging.MdcScope.Builder; import io.airbyte.commons.protocol.DefaultProtocolSerializer; import io.airbyte.commons.protocol.ProtocolSerializer; @@ -44,7 +43,7 @@ public class DefaultAirbyteSource implements AirbyteSource { 143 // SIGTERM ); - public static final MdcScope.Builder CONTAINER_LOG_MDC_BUILDER = new Builder() + public static final Builder CONTAINER_LOG_MDC_BUILDER = new Builder() .setLogPrefix("source") .setPrefixColor(Color.BLUE_BACKGROUND); diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/HeartbeatMonitor.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/HeartbeatMonitor.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/HeartbeatMonitor.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/HeartbeatMonitor.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriter.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriter.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriter.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriter.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriterFactory.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriterFactory.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriterFactory.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/internal/VersionedAirbyteMessageBufferedWriterFactory.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java similarity index 98% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java index 7fa614a66e4e..c85be16f5738 100644 --- a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java +++ b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java @@ -16,7 +16,6 @@ import io.airbyte.commons.io.LineGobbler; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.logging.LoggingHelper.Color; -import io.airbyte.commons.logging.MdcScope; import io.airbyte.commons.logging.MdcScope.Builder; import io.airbyte.configoss.OperatorDbt; import io.airbyte.configoss.ResourceRequirements; @@ -44,7 +43,7 @@ public class DefaultNormalizationRunner implements NormalizationRunner { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultNormalizationRunner.class); - private static final MdcScope.Builder CONTAINER_LOG_MDC_BUILDER = new Builder() + private static final Builder CONTAINER_LOG_MDC_BUILDER = new Builder() .setLogPrefix("normalization") .setPrefixColor(Color.GREEN_BACKGROUND); diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/NormalizationAirbyteStreamFactory.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/NormalizationAirbyteStreamFactory.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/NormalizationAirbyteStreamFactory.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/NormalizationAirbyteStreamFactory.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/NormalizationRunner.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/NormalizationRunner.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/NormalizationRunner.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/normalization/NormalizationRunner.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java similarity index 98% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java index ef176060d43b..b8592a8e6c19 100644 --- a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java +++ b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/AirbyteIntegrationLauncher.java @@ -216,7 +216,6 @@ private Map getWorkerMetadata() { .put("WORKER_CONNECTOR_IMAGE", imageName) .put("WORKER_JOB_ID", jobId) .put("WORKER_JOB_ATTEMPT", String.valueOf(attempt)) - .put(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, String.valueOf(featureFlags.useStreamCapableState())) .put(EnvVariableFeatureFlags.AUTO_DETECT_SCHEMA, String.valueOf(featureFlags.autoDetectSchema())) .put(EnvVariableFeatureFlags.APPLY_FIELD_SELECTION, String.valueOf(featureFlags.applyFieldSelection())) .put(EnvVariableFeatureFlags.FIELD_SELECTION_WORKSPACES, featureFlags.fieldSelectionWorkspaces()) diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/process/DockerProcessFactory.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/DockerProcessFactory.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/process/DockerProcessFactory.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/DockerProcessFactory.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/process/IntegrationLauncher.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/IntegrationLauncher.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/process/IntegrationLauncher.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/IntegrationLauncher.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/process/Metadata.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/Metadata.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/process/Metadata.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/Metadata.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/process/ProcessFactory.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/ProcessFactory.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/process/ProcessFactory.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/process/ProcessFactory.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/test_utils/AirbyteMessageUtils.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/test_utils/AirbyteMessageUtils.java similarity index 95% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/test_utils/AirbyteMessageUtils.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/test_utils/AirbyteMessageUtils.java index 2ae90d7842dd..15c20de236ba 100644 --- a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/test_utils/AirbyteMessageUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/test_utils/AirbyteMessageUtils.java @@ -34,7 +34,7 @@ public static AirbyteMessage createRecordMessage(final String tableName, final Instant timeExtracted) { return new AirbyteMessage() - .withType(AirbyteMessage.Type.RECORD) + .withType(Type.RECORD) .withRecord(new AirbyteRecordMessage() .withData(record) .withStream(tableName) @@ -45,7 +45,7 @@ public static AirbyteMessage createLogMessage(final AirbyteLogMessage.Level leve final String message) { return new AirbyteMessage() - .withType(AirbyteMessage.Type.LOG) + .withType(Type.LOG) .withLog(new AirbyteLogMessage() .withLevel(level) .withMessage(message)); @@ -70,13 +70,13 @@ public static AirbyteMessage createRecordMessage(final String tableName, public static AirbyteMessage createRecordMessage(final String streamName, final int recordData) { return new AirbyteMessage() - .withType(AirbyteMessage.Type.RECORD) + .withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withData(Jsons.jsonNode(recordData))); } public static AirbyteMessage createStateMessage(final int stateData) { return new AirbyteMessage() - .withType(AirbyteMessage.Type.STATE) + .withType(Type.STATE) .withState(new AirbyteStateMessage().withData(Jsons.jsonNode(stateData))); } @@ -139,7 +139,7 @@ public static AirbyteMessage createEstimateMessage(AirbyteEstimateTraceMessage.T public static AirbyteMessage createErrorMessage(final String message, final Double emittedAt) { return new AirbyteMessage() - .withType(AirbyteMessage.Type.TRACE) + .withType(Type.TRACE) .withTrace(createErrorTraceMessage(message, emittedAt)); } @@ -151,7 +151,7 @@ public static AirbyteTraceMessage createErrorTraceMessage(final String message, final Double emittedAt, final AirbyteErrorTraceMessage.FailureType failureType) { final var msg = new AirbyteTraceMessage() - .withType(io.airbyte.protocol.models.AirbyteTraceMessage.Type.ERROR) + .withType(AirbyteTraceMessage.Type.ERROR) .withError(new AirbyteErrorTraceMessage().withMessage(message)) .withEmittedAt(emittedAt); diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/test_utils/TestConfigHelpers.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/test_utils/TestConfigHelpers.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/java/io/airbyte/workers/test_utils/TestConfigHelpers.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/java/io/airbyte/workers/test_utils/TestConfigHelpers.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/resources/dbt_transformation_entrypoint.sh b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/dbt_transformation_entrypoint.sh similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/resources/dbt_transformation_entrypoint.sh rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/dbt_transformation_entrypoint.sh diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/resources/entrypoints/sync/check.sh b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/entrypoints/sync/check.sh similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/resources/entrypoints/sync/check.sh rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/entrypoints/sync/check.sh diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/resources/entrypoints/sync/init.sh b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/entrypoints/sync/init.sh similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/resources/entrypoints/sync/init.sh rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/entrypoints/sync/init.sh diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/resources/entrypoints/sync/main.sh b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/entrypoints/sync/main.sh similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/resources/entrypoints/sync/main.sh rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/entrypoints/sync/main.sh diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/resources/image_exists.sh b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/image_exists.sh similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/resources/image_exists.sh rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/image_exists.sh diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/main/resources/sshtunneling.sh b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/sshtunneling.sh similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/main/resources/sshtunneling.sh rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/main/resources/sshtunneling.sh diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/test/java/io/airbyte/workers/TestHarnessUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/TestHarnessUtilsTest.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/test/java/io/airbyte/workers/TestHarnessUtilsTest.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/TestHarnessUtilsTest.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/CatalogClientConvertersTest.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/CatalogClientConvertersTest.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/CatalogClientConvertersTest.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/CatalogClientConvertersTest.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/ConnectorConfigUpdaterTest.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/ConnectorConfigUpdaterTest.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/ConnectorConfigUpdaterTest.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/ConnectorConfigUpdaterTest.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/helper/FailureHelperTest.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/test/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactoryTest.java b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactoryTest.java similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/test/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactoryTest.java rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactoryTest.java diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/test/resources/version-detection/logs-with-version.jsonl b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/resources/version-detection/logs-with-version.jsonl similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/test/resources/version-detection/logs-with-version.jsonl rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/resources/version-detection/logs-with-version.jsonl diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/test/resources/version-detection/logs-without-spec-message.jsonl b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/resources/version-detection/logs-without-spec-message.jsonl similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/test/resources/version-detection/logs-without-spec-message.jsonl rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/resources/version-detection/logs-without-spec-message.jsonl diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/src/test/resources/version-detection/logs-without-version.jsonl b/airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/resources/version-detection/logs-without-version.jsonl similarity index 100% rename from airbyte-connector-test-harnesses/acceptance-test-harness/src/test/resources/version-detection/logs-without-version.jsonl rename to airbyte-cdk/java/airbyte-cdk/acceptance-test-harness/src/test/resources/version-detection/logs-without-version.jsonl diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-api/build.gradle b/airbyte-cdk/java/airbyte-cdk/airbyte-api/build.gradle new file mode 100644 index 000000000000..2db31d830e9c --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-api/build.gradle @@ -0,0 +1,222 @@ +import org.openapitools.generator.gradle.plugin.tasks.GenerateTask + +plugins { + id "org.openapi.generator" version "6.2.1" + id "java-library" +} + +java { + compileJava { + options.compilerArgs += "-Xlint:-deprecation" + } +} + +def specFile = "$projectDir/src/main/openapi/config.yaml" + +def generate = tasks.register('generate') + +// Deprecated -- can be removed once airbyte-server is converted to use the per-domain endpoints generated by generateApiServer +def generateApiServerLegacy = tasks.register('generateApiServerLegacy', GenerateTask) { + def serverOutputDir = "$buildDir/generated/api/server" + + inputs.file specFile + outputs.dir serverOutputDir + + generatorName = "jaxrs-spec" + inputSpec = specFile + outputDir = serverOutputDir + + apiPackage = "io.airbyte.api.generated" + invokerPackage = "io.airbyte.api.invoker.generated" + modelPackage = "io.airbyte.api.model.generated" + + schemaMappings.set([ + 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', + 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', + 'DestinationConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'StreamJsonSchema' : 'com.fasterxml.jackson.databind.JsonNode', + 'StateBlob' : 'com.fasterxml.jackson.databind.JsonNode', + 'FieldSchema' : 'com.fasterxml.jackson.databind.JsonNode', + ]) + + generateApiDocumentation = false + + configOptions.set([ + dateLibrary : "java8", + generatePom : "false", + interfaceOnly: "true", + /* + JAX-RS generator does not respect nullable properties defined in the OpenApi Spec. + It means that if a field is not nullable but not set it is still returning a null value for this field in the serialized json. + The below Jackson annotation is made to only keep non null values in serialized json. + We are not yet using nullable=true properties in our OpenApi so this is a valid workaround at the moment to circumvent the default JAX-RS behavior described above. + Feel free to read the conversation on https://github.com/airbytehq/airbyte/pull/13370 for more details. + */ + additionalModelTypeAnnotations: "\n@com.fasterxml.jackson.annotation.JsonInclude(com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL)", + ]) +} +generate.configure { + dependsOn generateApiServerLegacy +} + +def generateApiServer = tasks.register('generateApiServer', GenerateTask) { + def serverOutputDir = "$buildDir/generated/api/server" + + inputs.file specFile + outputs.dir serverOutputDir + + generatorName = "jaxrs-spec" + inputSpec = specFile + outputDir = serverOutputDir + + apiPackage = "io.airbyte.api.generated" + invokerPackage = "io.airbyte.api.invoker.generated" + modelPackage = "io.airbyte.api.model.generated" + + schemaMappings.set([ + 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', + 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', + 'DestinationConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'StreamJsonSchema' : 'com.fasterxml.jackson.databind.JsonNode', + 'StateBlob' : 'com.fasterxml.jackson.databind.JsonNode', + 'FieldSchema' : 'com.fasterxml.jackson.databind.JsonNode', + ]) + + generateApiDocumentation = false + + configOptions.set([ + dateLibrary : "java8", + generatePom : "false", + interfaceOnly: "true", + /* + JAX-RS generator does not respect nullable properties defined in the OpenApi Spec. + It means that if a field is not nullable but not set it is still returning a null value for this field in the serialized json. + The below Jackson annotation is made to only keep non null values in serialized json. + We are not yet using nullable=true properties in our OpenApi so this is a valid workaround at the moment to circumvent the default JAX-RS behavior described above. + Feel free to read the conversation on https://github.com/airbytehq/airbyte/pull/13370 for more details. + */ + additionalModelTypeAnnotations: "\n@com.fasterxml.jackson.annotation.JsonInclude(com.fasterxml.jackson.annotation.JsonInclude.Include.NON_NULL)", + + // Generate separate classes for each endpoint "domain" + useTags: "true" + ]) +} +generate.configure { + dependsOn generateApiServer +} + +def generateApiClient = tasks.register('generateApiClient', GenerateTask) { + def clientOutputDir = "$buildDir/generated/api/client" + + inputs.file specFile + outputs.dir clientOutputDir + + generatorName = "java" + inputSpec = specFile + outputDir = clientOutputDir + + apiPackage = "io.airbyte.api.client.generated" + invokerPackage = "io.airbyte.api.client.invoker.generated" + modelPackage = "io.airbyte.api.client.model.generated" + + schemaMappings.set([ + 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', + 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', + 'DestinationConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'StreamJsonSchema' : 'com.fasterxml.jackson.databind.JsonNode', + 'StateBlob' : 'com.fasterxml.jackson.databind.JsonNode', + 'FieldSchema' : 'com.fasterxml.jackson.databind.JsonNode', + ]) + + library = "native" + + generateApiDocumentation = false + + configOptions.set([ + dateLibrary : "java8", + generatePom : "false", + interfaceOnly: "true" + ]) +} +generate.configure { + dependsOn generateApiClient +} + +def generateApiDocs = tasks.register('generateApiDocs', GenerateTask) { + def docsOutputDir = "$buildDir/generated/api/docs" + + generatorName = "html" + inputSpec = specFile + outputDir = docsOutputDir + + apiPackage = "io.airbyte.api.client.generated" + invokerPackage = "io.airbyte.api.client.invoker.generated" + modelPackage = "io.airbyte.api.client.model.generated" + + schemaMappings.set([ + 'OAuthConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'SourceDefinitionSpecification' : 'com.fasterxml.jackson.databind.JsonNode', + 'SourceConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'DestinationDefinitionSpecification': 'com.fasterxml.jackson.databind.JsonNode', + 'DestinationConfiguration' : 'com.fasterxml.jackson.databind.JsonNode', + 'StreamJsonSchema' : 'com.fasterxml.jackson.databind.JsonNode', + 'StateBlob' : 'com.fasterxml.jackson.databind.JsonNode', + 'FieldSchema' : 'com.fasterxml.jackson.databind.JsonNode', + ]) + + generateApiDocumentation = false + + configOptions.set([ + dateLibrary : "java8", + generatePom : "false", + interfaceOnly: "true" + ]) + + doLast { + def target = file(rootProject.file("docs/reference/api/generated-api-html")) + delete target + mkdir target + copy { + from outputDir + include "**/*.html" + includeEmptyDirs = false + into target + } + } +} +generate.configure { + dependsOn generateApiDocs +} + +dependencies { + implementation group: 'com.fasterxml.jackson.datatype', name: 'jackson-datatype-jsr310' + + implementation group: 'io.swagger', name: 'swagger-annotations', version: '1.6.2' + + implementation group: 'javax.annotation', name: 'javax.annotation-api', version: '1.3.2' + implementation group: 'javax.ws.rs', name: 'javax.ws.rs-api', version: '2.1.1' + implementation group: 'javax.validation', name: 'validation-api', version: '2.0.1.Final' + + implementation group: 'org.openapitools', name: 'jackson-databind-nullable', version: '0.2.1' +} + +sourceSets { + main { + java { + srcDirs "$buildDir/generated/api/server/src/gen/java", "$buildDir/generated/api/client/src/main/java", "$projectDir/src/main/java" + } + resources { + srcDir "$projectDir/src/main/openapi/" + } + } +} + +tasks.named('compileJava').configure { + dependsOn generate +} diff --git a/airbyte-api/readme.md b/airbyte-cdk/java/airbyte-cdk/airbyte-api/readme.md similarity index 100% rename from airbyte-api/readme.md rename to airbyte-cdk/java/airbyte-cdk/airbyte-api/readme.md diff --git a/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java b/airbyte-cdk/java/airbyte-cdk/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java similarity index 100% rename from airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-api/src/main/java/io/airbyte/api/client/AirbyteApiClient.java diff --git a/airbyte-api/src/main/java/io/airbyte/api/client/PatchedLogsApi.java b/airbyte-cdk/java/airbyte-cdk/airbyte-api/src/main/java/io/airbyte/api/client/PatchedLogsApi.java similarity index 100% rename from airbyte-api/src/main/java/io/airbyte/api/client/PatchedLogsApi.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-api/src/main/java/io/airbyte/api/client/PatchedLogsApi.java diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-cdk/java/airbyte-cdk/airbyte-api/src/main/openapi/config.yaml similarity index 100% rename from airbyte-api/src/main/openapi/config.yaml rename to airbyte-cdk/java/airbyte-cdk/airbyte-api/src/main/openapi/config.yaml diff --git a/airbyte-api/src/test/java/io/airbyte/api/client/AirbyteApiClientTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-api/src/test/java/io/airbyte/api/client/AirbyteApiClientTest.java similarity index 100% rename from airbyte-api/src/test/java/io/airbyte/api/client/AirbyteApiClientTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-api/src/test/java/io/airbyte/api/client/AirbyteApiClientTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/build.gradle b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/build.gradle new file mode 100644 index 000000000000..3dbb175d2ad9 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/build.gradle @@ -0,0 +1,7 @@ +plugins { + id "java-library" +} + +dependencies { + implementation 'commons-cli:commons-cli:1.4' +} diff --git a/airbyte-commons-cli/readme.md b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/readme.md similarity index 100% rename from airbyte-commons-cli/readme.md rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/readme.md diff --git a/airbyte-commons-cli/src/main/java/io/airbyte/commons/cli/Clis.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/src/main/java/io/airbyte/commons/cli/Clis.java similarity index 100% rename from airbyte-commons-cli/src/main/java/io/airbyte/commons/cli/Clis.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/src/main/java/io/airbyte/commons/cli/Clis.java diff --git a/airbyte-commons-cli/src/test/java/io/airbyte/commons/cli/ClisTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/src/test/java/io/airbyte/commons/cli/ClisTest.java similarity index 100% rename from airbyte-commons-cli/src/test/java/io/airbyte/commons/cli/ClisTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-cli/src/test/java/io/airbyte/commons/cli/ClisTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/build.gradle b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/build.gradle new file mode 100644 index 000000000000..ae8a69d75513 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/build.gradle @@ -0,0 +1,16 @@ +java { + compileJava { + options.compilerArgs += "-Xlint:-unchecked" + } +} + +dependencies { + annotationProcessor libs.bundles.micronaut.annotation.processor + testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor + + implementation libs.bundles.micronaut.annotation + testImplementation libs.bundles.micronaut.test + + implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') +} diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageMigrator.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageMigrator.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageMigrator.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageMigrator.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProvider.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProvider.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProvider.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProvider.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageVersionedMigrator.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageVersionedMigrator.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageVersionedMigrator.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteMessageVersionedMigrator.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteProtocolVersionedMigratorFactory.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteProtocolVersionedMigratorFactory.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteProtocolVersionedMigratorFactory.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/AirbyteProtocolVersionedMigratorFactory.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/ConfiguredAirbyteCatalogMigrator.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/ConfiguredAirbyteCatalogMigrator.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/ConfiguredAirbyteCatalogMigrator.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/ConfiguredAirbyteCatalogMigrator.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/DefaultProtocolSerializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/DefaultProtocolSerializer.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/DefaultProtocolSerializer.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/DefaultProtocolSerializer.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/ProtocolSerializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/ProtocolSerializer.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/ProtocolSerializer.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/ProtocolSerializer.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/VersionedProtocolSerializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/VersionedProtocolSerializer.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/VersionedProtocolSerializer.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/VersionedProtocolSerializer.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/AirbyteMessageMigration.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/AirbyteMessageMigration.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/AirbyteMessageMigration.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/AirbyteMessageMigration.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/ConfiguredAirbyteCatalogMigration.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/ConfiguredAirbyteCatalogMigration.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/ConfiguredAirbyteCatalogMigration.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/ConfiguredAirbyteCatalogMigration.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/Migration.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/Migration.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/Migration.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/Migration.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/MigrationContainer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/MigrationContainer.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/MigrationContainer.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/MigrationContainer.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/util/RecordMigrations.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/util/RecordMigrations.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/util/RecordMigrations.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/util/RecordMigrations.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/util/SchemaMigrations.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/util/SchemaMigrations.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/util/SchemaMigrations.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/util/SchemaMigrations.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/AirbyteMessageMigrationV1.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/AirbyteMessageMigrationV1.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/AirbyteMessageMigrationV1.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/AirbyteMessageMigrationV1.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/CatalogMigrationV1Helper.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/CatalogMigrationV1Helper.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/CatalogMigrationV1Helper.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/CatalogMigrationV1Helper.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/ConfiguredAirbyteCatalogMigrationV1.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/ConfiguredAirbyteCatalogMigrationV1.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/ConfiguredAirbyteCatalogMigrationV1.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/ConfiguredAirbyteCatalogMigrationV1.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/SchemaMigrationV1.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/SchemaMigrationV1.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/SchemaMigrationV1.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/migrations/v1/SchemaMigrationV1.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageDeserializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageDeserializer.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageDeserializer.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageDeserializer.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericDeserializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericDeserializer.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericDeserializer.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericDeserializer.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericSerializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericSerializer.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericSerializer.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageGenericSerializer.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageSerializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageSerializer.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageSerializer.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageSerializer.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Deserializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Deserializer.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Deserializer.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Deserializer.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Serializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Serializer.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Serializer.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0Serializer.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Deserializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Deserializer.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Deserializer.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Deserializer.java diff --git a/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Serializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Serializer.java similarity index 100% rename from airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Serializer.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/main/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1Serializer.java diff --git a/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageMigratorTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageMigratorTest.java similarity index 100% rename from airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageMigratorTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageMigratorTest.java diff --git a/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderMicronautTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderMicronautTest.java similarity index 100% rename from airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderMicronautTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderMicronautTest.java diff --git a/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderTest.java similarity index 100% rename from airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/AirbyteMessageSerDeProviderTest.java diff --git a/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/MigratorsMicronautTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/MigratorsMicronautTest.java similarity index 100% rename from airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/MigratorsMicronautTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/MigratorsMicronautTest.java diff --git a/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/v1/AirbyteMessageMigrationV1Test.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/v1/AirbyteMessageMigrationV1Test.java similarity index 100% rename from airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/v1/AirbyteMessageMigrationV1Test.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/v1/AirbyteMessageMigrationV1Test.java diff --git a/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/v1/ConfiguredAirbyteCatalogMigrationV1Test.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/v1/ConfiguredAirbyteCatalogMigrationV1Test.java similarity index 100% rename from airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/v1/ConfiguredAirbyteCatalogMigrationV1Test.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/migrations/v1/ConfiguredAirbyteCatalogMigrationV1Test.java diff --git a/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0SerDeTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0SerDeTest.java similarity index 100% rename from airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0SerDeTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV0SerDeTest.java diff --git a/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1SerDeTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1SerDeTest.java similarity index 100% rename from airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1SerDeTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/java/io/airbyte/commons/protocol/serde/AirbyteMessageV1SerDeTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/resources/WellKnownTypes.json b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/resources/WellKnownTypes.json new file mode 100644 index 000000000000..9e4d6656deae --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons-protocol/src/test/resources/WellKnownTypes.json @@ -0,0 +1,86 @@ +{ + "definitions": { + "String": { + "type": "string", + "description": "Arbitrary text" + }, + "BinaryData": { + "type": "string", + "description": "Arbitrary binary data. Represented as base64-encoded strings in the JSON transport. In the future, if we support other transports, may be encoded differently.\n", + "pattern": "^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" + }, + "Date": { + "type": "string", + "oneOf": [ + { + "pattern": "^\\d{4}-\\d{2}-\\d{2}( BC)?$" + }, + { + "enum": ["Infinity", "-Infinity"] + } + ], + "description": "RFC 3339\u00a75.6's full-date format, extended with BC era support and (-)Infinity" + }, + "TimestampWithTimezone": { + "type": "string", + "oneOf": [ + { + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?(Z|[+\\-]\\d{1,2}:\\d{2})( BC)?$" + }, + { + "enum": ["Infinity", "-Infinity"] + } + ], + "description": "An instant in time. Frequently simply referred to as just a timestamp, or timestamptz. Uses RFC 3339\u00a75.6's date-time format, requiring a \"T\" separator, and extended with BC era support and (-)Infinity. Note that we do _not_ accept Unix epochs here.\n" + }, + "TimestampWithoutTimezone": { + "type": "string", + "oneOf": [ + { + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?( BC)?$" + }, + { + "enum": ["Infinity", "-Infinity"] + } + ], + "description": "Also known as a localdatetime, or just datetime. Under RFC 3339\u00a75.6, this would be represented as `full-date \"T\" partial-time`, extended with BC era support and (-)Infinity.\n" + }, + "TimeWithTimezone": { + "type": "string", + "pattern": "^\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?(Z|[+\\-]\\d{1,2}:\\d{2})$", + "description": "An RFC 3339\u00a75.6 full-time" + }, + "TimeWithoutTimezone": { + "type": "string", + "pattern": "^\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?$", + "description": "An RFC 3339\u00a75.6 partial-time" + }, + "Number": { + "type": "string", + "oneOf": [ + { + "pattern": "-?(0|[0-9]\\d*)(\\.\\d+)?" + }, + { + "enum": ["Infinity", "-Infinity", "NaN"] + } + ], + "description": "Note the mix of regex validation for normal numbers, and enum validation for special values." + }, + "Integer": { + "type": "string", + "oneOf": [ + { + "pattern": "-?(0|[0-9]\\d*)" + }, + { + "enum": ["Infinity", "-Infinity", "NaN"] + } + ] + }, + "Boolean": { + "type": "boolean", + "description": "Note the direct usage of a primitive boolean rather than string. Unlike Numbers and Integers, we don't expect unusual values here." + } + } +} diff --git a/airbyte-commons/LICENSE b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/LICENSE similarity index 100% rename from airbyte-commons/LICENSE rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/LICENSE diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle new file mode 100644 index 000000000000..9073e823a0fe --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/build.gradle @@ -0,0 +1,27 @@ +plugins { + id 'java-library' + id 'de.undercouch.download' version "5.4.0" +} + +java { + compileJava { + options.compilerArgs += "-Xlint:-varargs,-try,-deprecation" + } + compileTestJava { + options.compilerArgs += "-Xlint:-try" + } +} + +dependencies { + // Dependencies for this module should be specified in the top-level build.gradle. See readme for more explanation. + + // this dependency is an exception to the above rule because it is only used INTERNALLY to the commons library. + implementation 'com.jayway.jsonpath:json-path:2.7.0' +} + +def downloadSpecSecretMask = tasks.register('downloadSpecSecretMask', Download) { + src 'https://connectors.airbyte.com/files/registries/v0/specs_secrets_mask.yaml' + dest new File(projectDir, 'src/main/resources/seed/specs_secrets_mask.yaml') + overwrite true +} +tasks.named('processResources').configure { dependsOn downloadSpecSecretMask } diff --git a/airbyte-commons/readme.md b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/readme.md similarity index 100% rename from airbyte-commons/readme.md rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/readme.md diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/concurrency/VoidCallable.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/concurrency/VoidCallable.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/concurrency/VoidCallable.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/concurrency/VoidCallable.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/concurrency/WaitingUtils.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/concurrency/WaitingUtils.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/concurrency/WaitingUtils.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/concurrency/WaitingUtils.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/constants/AirbyteSecretConstants.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/constants/AirbyteSecretConstants.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/constants/AirbyteSecretConstants.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/constants/AirbyteSecretConstants.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/enums/Enums.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/enums/Enums.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/enums/Enums.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/enums/Enums.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/exceptions/ConfigErrorException.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/exceptions/ConfigErrorException.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/exceptions/ConfigErrorException.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/exceptions/ConfigErrorException.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/exceptions/ConnectionErrorException.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/exceptions/ConnectionErrorException.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/exceptions/ConnectionErrorException.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/exceptions/ConnectionErrorException.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java similarity index 92% rename from airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java index 9b64edc1f99c..e2e86d1c2688 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/EnvVariableFeatureFlags.java @@ -12,7 +12,6 @@ public class EnvVariableFeatureFlags implements FeatureFlags { private static final Logger log = LoggerFactory.getLogger(EnvVariableFeatureFlags.class); - public static final String USE_STREAM_CAPABLE_STATE = "USE_STREAM_CAPABLE_STATE"; public static final String AUTO_DETECT_SCHEMA = "AUTO_DETECT_SCHEMA"; // Set this value to true to see all messages from the source to destination, set to one second // emission @@ -22,11 +21,7 @@ public class EnvVariableFeatureFlags implements FeatureFlags { public static final String CONCURRENT_SOURCE_STREAM_READ = "CONCURRENT_SOURCE_STREAM_READ"; public static final String STRICT_COMPARISON_NORMALIZATION_WORKSPACES = "STRICT_COMPARISON_NORMALIZATION_WORKSPACES"; public static final String STRICT_COMPARISON_NORMALIZATION_TAG = "STRICT_COMPARISON_NORMALIZATION_TAG"; - - @Override - public boolean useStreamCapableState() { - return getEnvOrDefault(USE_STREAM_CAPABLE_STATE, false, Boolean::parseBoolean); - } + public static final String DEPLOYMENT_MODE = "DEPLOYMENT_MODE"; @Override public boolean autoDetectSchema() { @@ -63,6 +58,11 @@ public String strictComparisonNormalizationTag() { return getEnvOrDefault(STRICT_COMPARISON_NORMALIZATION_TAG, "strict_comparison2", (arg) -> arg); } + @Override + public String deploymentMode() { + return getEnvOrDefault(DEPLOYMENT_MODE, "", (arg) -> arg); + } + // TODO: refactor in order to use the same method than the ones in EnvConfigs.java public T getEnvOrDefault(final String key, final T defaultValue, final Function parser) { final String value = System.getenv(key); diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlagHelper.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlagHelper.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlagHelper.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlagHelper.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java similarity index 88% rename from airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java index b3da9ac764bb..fa55fbd9484c 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlags.java @@ -10,8 +10,6 @@ */ public interface FeatureFlags { - boolean useStreamCapableState(); - boolean autoDetectSchema(); boolean logConnectorMessages(); @@ -49,4 +47,11 @@ public interface FeatureFlags { */ String strictComparisonNormalizationTag(); + /** + * Get the deployment mode used to deploy a connector. + * + * @return empty string for the default deployment mode, "CLOUD" for cloud deployment mode. + */ + String deploymentMode(); + } diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlagsWrapper.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlagsWrapper.java new file mode 100644 index 000000000000..17cdfa91dcbf --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/features/FeatureFlagsWrapper.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.commons.features; + +public class FeatureFlagsWrapper implements FeatureFlags { + + /** + * Overrides the {@link FeatureFlags#deploymentMode} method in the feature flags. + */ + static public FeatureFlags overridingDeploymentMode( + final FeatureFlags wrapped, + final String deploymentMode) { + return new FeatureFlagsWrapper(wrapped) { + + @Override + public String deploymentMode() { + return deploymentMode; + } + + }; + } + + private final FeatureFlags wrapped; + + public FeatureFlagsWrapper(FeatureFlags wrapped) { + this.wrapped = wrapped; + } + + @Override + public boolean autoDetectSchema() { + return wrapped.autoDetectSchema(); + } + + @Override + public boolean logConnectorMessages() { + return wrapped.logConnectorMessages(); + } + + @Override + public boolean concurrentSourceStreamRead() { + return wrapped.concurrentSourceStreamRead(); + } + + @Override + public boolean applyFieldSelection() { + return wrapped.applyFieldSelection(); + } + + @Override + public String fieldSelectionWorkspaces() { + return wrapped.fieldSelectionWorkspaces(); + } + + @Override + public String strictComparisonNormalizationWorkspaces() { + return wrapped.strictComparisonNormalizationWorkspaces(); + } + + @Override + public String strictComparisonNormalizationTag() { + return wrapped.strictComparisonNormalizationTag(); + } + + @Override + public String deploymentMode() { + return wrapped.deploymentMode(); + } + +} diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedBiConsumer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedBiConsumer.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedBiConsumer.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedBiConsumer.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedBiFunction.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedBiFunction.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedBiFunction.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedBiFunction.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedConsumer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedConsumer.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedConsumer.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedConsumer.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedFunction.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedFunction.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedFunction.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedFunction.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedSupplier.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedSupplier.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedSupplier.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/functional/CheckedSupplier.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/io/IOs.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/io/IOs.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/io/IOs.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/io/IOs.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/io/LineGobbler.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/io/LineGobbler.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/io/LineGobbler.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/io/LineGobbler.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/jackson/MoreMappers.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/jackson/MoreMappers.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/jackson/MoreMappers.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/jackson/MoreMappers.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonPaths.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonPaths.java similarity index 99% rename from airbyte-commons/src/main/java/io/airbyte/commons/json/JsonPaths.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonPaths.java index 08953b571827..7041f3325479 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonPaths.java +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonPaths.java @@ -6,7 +6,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; -import com.google.api.client.util.Preconditions; +import com.google.common.base.Preconditions; import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.JsonPath; import com.jayway.jsonpath.Option; diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonSchemas.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonSchemas.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/json/JsonSchemas.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/JsonSchemas.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java similarity index 98% rename from airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java index 18b40c7c5489..2d0420bf63f0 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/json/Jsons.java @@ -121,6 +121,14 @@ public static Optional tryDeserialize(final String jsonString, final Clas } } + public static Optional tryDeserializeExact(final String jsonString, final Class klass) { + try { + return Optional.of(OBJECT_MAPPER_EXACT.readValue(jsonString, klass)); + } catch (final Throwable e) { + return Optional.empty(); + } + } + public static Optional tryDeserialize(final String jsonString) { try { return Optional.of(OBJECT_MAPPER.readTree(jsonString)); diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableConsumer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableConsumer.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableConsumer.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableConsumer.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableQueue.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableQueue.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableQueue.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableQueue.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableShutdownHook.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableShutdownHook.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableShutdownHook.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableShutdownHook.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/lang/Exceptions.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/Exceptions.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/lang/Exceptions.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/Exceptions.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/lang/MoreBooleans.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/MoreBooleans.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/lang/MoreBooleans.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/lang/MoreBooleans.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/LoggingHelper.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/logging/MaskedDataInterceptor.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/MaskedDataInterceptor.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/logging/MaskedDataInterceptor.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/MaskedDataInterceptor.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/logging/MdcScope.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/map/MoreMaps.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/map/MoreMaps.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/map/MoreMaps.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/map/MoreMaps.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/resources/MoreResources.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/resources/MoreResources.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/resources/MoreResources.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/resources/MoreResources.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/stream/AirbyteStreamStatusHolder.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/AirbyteStreamStatusHolder.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/stream/AirbyteStreamStatusHolder.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/AirbyteStreamStatusHolder.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/stream/AirbyteStreamUtils.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/AirbyteStreamUtils.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/stream/AirbyteStreamUtils.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/AirbyteStreamUtils.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/stream/MoreStreams.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/MoreStreams.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/stream/MoreStreams.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/MoreStreams.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/stream/StreamStatusUtils.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/StreamStatusUtils.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/stream/StreamStatusUtils.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/stream/StreamStatusUtils.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/string/Strings.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/string/Strings.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/string/Strings.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/string/Strings.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/text/Names.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/text/Names.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/text/Names.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/text/Names.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/text/Sqls.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/text/Sqls.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/text/Sqls.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/text/Sqls.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/util/AirbyteStreamAware.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/AirbyteStreamAware.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/util/AirbyteStreamAware.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/AirbyteStreamAware.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterator.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterator.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterator.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterator.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterators.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterators.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterators.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/AutoCloseableIterators.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/util/CompositeIterator.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/CompositeIterator.java similarity index 84% rename from airbyte-commons/src/main/java/io/airbyte/commons/util/CompositeIterator.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/CompositeIterator.java index 7c5997d344c6..c8a3030bb92d 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/util/CompositeIterator.java +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/CompositeIterator.java @@ -10,9 +10,11 @@ import io.airbyte.commons.stream.StreamStatusUtils; import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; import java.util.ArrayList; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.function.Consumer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,7 +47,7 @@ public final class CompositeIterator extends AbstractIterator implements A private final List> iterators; private int i; - private boolean firstRead; + private final Set> seenIterators; private boolean hasClosed; CompositeIterator(final List> iterators, final Consumer airbyteStreamStatusConsumer) { @@ -54,7 +56,7 @@ public final class CompositeIterator extends AbstractIterator implements A this.airbyteStreamStatusConsumer = Optional.ofNullable(airbyteStreamStatusConsumer); this.iterators = iterators; this.i = 0; - this.firstRead = true; + this.seenIterators = new HashSet>(); this.hasClosed = false; } @@ -72,6 +74,7 @@ protected T computeNext() { while (!currentIterator().hasNext()) { try { currentIterator().close(); + emitStartStreamStatus(currentIterator().getAirbyteStream()); StreamStatusUtils.emitCompleteStreamStatus(getAirbyteStream(), airbyteStreamStatusConsumer); } catch (final Exception e) { StreamStatusUtils.emitIncompleteStreamStatus(getAirbyteStream(), airbyteStreamStatusConsumer); @@ -80,26 +83,21 @@ protected T computeNext() { if (i + 1 < iterators.size()) { i++; - StreamStatusUtils.emitStartStreamStatus(getAirbyteStream(), airbyteStreamStatusConsumer); - firstRead = true; } else { return endOfData(); } } try { - if (isFirstStream()) { - StreamStatusUtils.emitStartStreamStatus(getAirbyteStream(), airbyteStreamStatusConsumer); + final boolean isFirstRun = emitStartStreamStatus(currentIterator().getAirbyteStream()); + final T next = currentIterator().next(); + if (isFirstRun) { + StreamStatusUtils.emitRunningStreamStatus(getAirbyteStream(), airbyteStreamStatusConsumer); } - return currentIterator().next(); + return next; } catch (final RuntimeException e) { StreamStatusUtils.emitIncompleteStreamStatus(getAirbyteStream(), airbyteStreamStatusConsumer); throw e; - } finally { - if (firstRead) { - StreamStatusUtils.emitRunningStreamStatus(getAirbyteStream(), airbyteStreamStatusConsumer); - firstRead = false; - } } } @@ -107,8 +105,13 @@ private AutoCloseableIterator currentIterator() { return iterators.get(i); } - private boolean isFirstStream() { - return i == 0 && firstRead; + private boolean emitStartStreamStatus(final Optional airbyteStream) { + if (airbyteStream.isPresent() && !seenIterators.contains(airbyteStream)) { + seenIterators.add(airbyteStream); + StreamStatusUtils.emitStartStreamStatus(airbyteStream, airbyteStreamStatusConsumer); + return true; + } + return false; } @Override diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/util/DefaultAutoCloseableIterator.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/DefaultAutoCloseableIterator.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/util/DefaultAutoCloseableIterator.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/DefaultAutoCloseableIterator.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/util/LazyAutoCloseableIterator.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/LazyAutoCloseableIterator.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/util/LazyAutoCloseableIterator.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/LazyAutoCloseableIterator.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/util/MoreIterators.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/MoreIterators.java similarity index 99% rename from airbyte-commons/src/main/java/io/airbyte/commons/util/MoreIterators.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/MoreIterators.java index 32a771f6b8a6..b1924e2ecfac 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/util/MoreIterators.java +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/util/MoreIterators.java @@ -22,6 +22,7 @@ public class MoreIterators { * @param type * @return iterator with all elements */ + @SafeVarargs public static Iterator of(final T... elements) { return Arrays.asList(elements).iterator(); } diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteProtocolVersion.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteProtocolVersion.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteProtocolVersion.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteProtocolVersion.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/version/Version.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/Version.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/version/Version.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/Version.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/version/VersionDeserializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/VersionDeserializer.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/version/VersionDeserializer.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/VersionDeserializer.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/version/VersionSerializer.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/VersionSerializer.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/version/VersionSerializer.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/version/VersionSerializer.java diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/yaml/Yamls.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/yaml/Yamls.java similarity index 100% rename from airbyte-commons/src/main/java/io/airbyte/commons/yaml/Yamls.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/java/io/airbyte/commons/yaml/Yamls.java diff --git a/airbyte-commons/src/main/resources/log4j2.xml b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/resources/log4j2-test.xml similarity index 100% rename from airbyte-commons/src/main/resources/log4j2.xml rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/main/resources/log4j2-test.xml diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/concurrency/WaitingUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/concurrency/WaitingUtilsTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/concurrency/WaitingUtilsTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/concurrency/WaitingUtilsTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/enums/EnumsTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/enums/EnumsTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/enums/EnumsTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/enums/EnumsTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/features/FeatureFlagHelperTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/features/FeatureFlagHelperTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/features/FeatureFlagHelperTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/features/FeatureFlagHelperTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/io/IOsTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/io/IOsTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/io/IOsTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/io/IOsTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/io/LineGobblerTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/io/LineGobblerTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/io/LineGobblerTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/io/LineGobblerTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonPathsTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonPathsTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/json/JsonPathsTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonPathsTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonSchemasTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonSchemasTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/json/JsonSchemasTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonSchemasTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/json/JsonsTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/lang/CloseableShutdownHookTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/lang/CloseableShutdownHookTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/lang/CloseableShutdownHookTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/lang/CloseableShutdownHookTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/lang/ExceptionsTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/lang/ExceptionsTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/lang/ExceptionsTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/lang/ExceptionsTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/lang/MoreBooleansTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/lang/MoreBooleansTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/lang/MoreBooleansTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/lang/MoreBooleansTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/logging/Log4j2ConfigTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/logging/MdcScopeTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/map/MoreMapsTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/map/MoreMapsTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/map/MoreMapsTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/map/MoreMapsTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/resources/MoreResourcesTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/resources/MoreResourcesTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/resources/MoreResourcesTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/resources/MoreResourcesTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/stream/AirbyteStreamStatusHolderTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/stream/AirbyteStreamStatusHolderTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/stream/AirbyteStreamStatusHolderTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/stream/AirbyteStreamStatusHolderTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/stream/StreamStatusUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/stream/StreamStatusUtilsTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/stream/StreamStatusUtilsTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/stream/StreamStatusUtilsTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/string/StringsTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/string/StringsTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/string/StringsTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/string/StringsTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/text/NamesTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/text/NamesTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/text/NamesTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/text/NamesTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/text/SqlsTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/text/SqlsTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/text/SqlsTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/text/SqlsTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/util/AutoCloseableIteratorsTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/AutoCloseableIteratorsTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/util/AutoCloseableIteratorsTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/AutoCloseableIteratorsTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/util/CompositeIteratorTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/CompositeIteratorTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/util/CompositeIteratorTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/CompositeIteratorTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/util/DefaultAutoCloseableIteratorTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/DefaultAutoCloseableIteratorTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/util/DefaultAutoCloseableIteratorTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/DefaultAutoCloseableIteratorTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/util/LazyAutoCloseableIteratorTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/LazyAutoCloseableIteratorTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/util/LazyAutoCloseableIteratorTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/util/LazyAutoCloseableIteratorTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/version/AirbyteVersionTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/version/AirbyteVersionTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/version/AirbyteVersionTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/version/AirbyteVersionTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/version/VersionTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/version/VersionTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/version/VersionTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/version/VersionTest.java diff --git a/airbyte-commons/src/test/java/io/airbyte/commons/yaml/YamlsTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/yaml/YamlsTest.java similarity index 100% rename from airbyte-commons/src/test/java/io/airbyte/commons/yaml/YamlsTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/java/io/airbyte/commons/yaml/YamlsTest.java diff --git a/airbyte-commons/src/test/resources/json_schemas/composite_json_schema.json b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/composite_json_schema.json similarity index 100% rename from airbyte-commons/src/test/resources/json_schemas/composite_json_schema.json rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/composite_json_schema.json diff --git a/airbyte-commons/src/test/resources/json_schemas/json_with_all_types.json b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_all_types.json similarity index 100% rename from airbyte-commons/src/test/resources/json_schemas/json_with_all_types.json rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_all_types.json diff --git a/airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields.json b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields.json similarity index 100% rename from airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields.json rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields.json diff --git a/airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields_no_items.json b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields_no_items.json similarity index 100% rename from airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields_no_items.json rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields_no_items.json diff --git a/airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields_with_composites.json b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields_with_composites.json similarity index 100% rename from airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields_with_composites.json rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/json_schemas/json_with_array_type_fields_with_composites.json diff --git a/airbyte-commons/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker similarity index 100% rename from airbyte-commons/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker diff --git a/airbyte-commons/src/test/resources/resource_test b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/resource_test similarity index 100% rename from airbyte-commons/src/test/resources/resource_test rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/resource_test diff --git a/airbyte-commons/src/test/resources/resource_test_a b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/resource_test_a similarity index 100% rename from airbyte-commons/src/test/resources/resource_test_a rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/resource_test_a diff --git a/airbyte-commons/src/test/resources/subdir/resource_test_a b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/subdir/resource_test_a similarity index 100% rename from airbyte-commons/src/test/resources/subdir/resource_test_a rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/subdir/resource_test_a diff --git a/airbyte-commons/src/test/resources/subdir/resource_test_sub b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/subdir/resource_test_sub similarity index 100% rename from airbyte-commons/src/test/resources/subdir/resource_test_sub rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/subdir/resource_test_sub diff --git a/airbyte-commons/src/test/resources/subdir/resource_test_sub_2 b/airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/subdir/resource_test_sub_2 similarity index 100% rename from airbyte-commons/src/test/resources/subdir/resource_test_sub_2 rename to airbyte-cdk/java/airbyte-cdk/airbyte-commons/src/test/resources/subdir/resource_test_sub_2 diff --git a/airbyte-json-validation/LICENSE b/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/LICENSE similarity index 100% rename from airbyte-json-validation/LICENSE rename to airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/LICENSE diff --git a/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/build.gradle b/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/build.gradle new file mode 100644 index 000000000000..84674554e773 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/build.gradle @@ -0,0 +1,10 @@ +plugins { + id "java-library" +} + +dependencies { + implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + implementation 'com.networknt:json-schema-validator:1.0.72' + // needed so that we can follow $ref when parsing json. jackson does not support this natively. + implementation 'me.andrz.jackson:jackson-json-reference-core:0.3.2' +} diff --git a/airbyte-json-validation/readme.md b/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/readme.md similarity index 100% rename from airbyte-json-validation/readme.md rename to airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/readme.md diff --git a/airbyte-json-validation/src/main/java/io/airbyte/validation/json/AbstractSchemaValidator.java b/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/AbstractSchemaValidator.java similarity index 100% rename from airbyte-json-validation/src/main/java/io/airbyte/validation/json/AbstractSchemaValidator.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/AbstractSchemaValidator.java diff --git a/airbyte-json-validation/src/main/java/io/airbyte/validation/json/ConfigSchemaValidator.java b/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/ConfigSchemaValidator.java similarity index 100% rename from airbyte-json-validation/src/main/java/io/airbyte/validation/json/ConfigSchemaValidator.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/ConfigSchemaValidator.java diff --git a/airbyte-json-validation/src/main/java/io/airbyte/validation/json/JsonSchemaValidator.java b/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/JsonSchemaValidator.java similarity index 100% rename from airbyte-json-validation/src/main/java/io/airbyte/validation/json/JsonSchemaValidator.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/JsonSchemaValidator.java diff --git a/airbyte-json-validation/src/main/java/io/airbyte/validation/json/JsonValidationException.java b/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/JsonValidationException.java similarity index 100% rename from airbyte-json-validation/src/main/java/io/airbyte/validation/json/JsonValidationException.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/main/java/io/airbyte/validation/json/JsonValidationException.java diff --git a/airbyte-json-validation/src/test/java/io/airbyte/validation/json/JsonSchemaValidatorTest.java b/airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/test/java/io/airbyte/validation/json/JsonSchemaValidatorTest.java similarity index 100% rename from airbyte-json-validation/src/test/java/io/airbyte/validation/json/JsonSchemaValidatorTest.java rename to airbyte-cdk/java/airbyte-cdk/airbyte-json-validation/src/test/java/io/airbyte/validation/json/JsonSchemaValidatorTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/build.gradle b/airbyte-cdk/java/airbyte-cdk/build.gradle index 8b7800baa7a1..1523d78df648 100644 --- a/airbyte-cdk/java/airbyte-cdk/build.gradle +++ b/airbyte-cdk/java/airbyte-cdk/build.gradle @@ -1,72 +1,120 @@ -plugins { - id 'java-library' - id 'maven-publish' -} - -group 'io.airbyte' - -// Version is dynamically loaded from version.properties file. -def props = new Properties() -file("src/main/resources/version.properties").withInputStream { props.load(it) } -version = props.getProperty('version') -description = "Airbyte Connector Development Kit (CDK) for Java." +allprojects { + apply plugin: 'java-library' + apply plugin: 'maven-publish' + apply plugin: 'airbyte-java-cdk' + apply plugin: 'airbyte-integration-test-java' + apply plugin: 'airbyte-performance-test-java' + apply plugin: 'java-test-fixtures' -dependencies { - testImplementation 'org.junit.jupiter:junit-jupiter:5.7.2' + group 'io.airbyte.cdk' + version = getCdkTargetVersion() } -publishing { - publications { - maven(MavenPublication) { - groupId = 'io.airbyte' - artifactId = 'airbyte-cdk' - from components.java +subprojects { subproject -> + def artifactBaseName = 'airbyte-cdk-' + subproject.name + // E.g. airbyte-cdk-core, airbyte-cdk-db-sources, airbyte-cdk-db-destinations, etc. + + publishing { + publications { + main(MavenPublication) { + groupId = 'io.airbyte.cdk' + artifactId = artifactBaseName + from components.java + } + testFixtures(MavenPublication) { + groupId = 'io.airbyte.cdk' + artifactId = artifactBaseName + '-test-fixtures' + version = project.version + artifact subproject.tasks.testFixturesJar + } } - } - repositories { - maven { - name 'cloudrepo' - url 'https://airbyte.mycloudrepo.io/repositories/airbyte-public-jars' - credentials { - username System.getenv('CLOUDREPO_USER') - password System.getenv('CLOUDREPO_PASSWORD') + // This repository is only defined and used in the context of an artifact publishing + // It's different from the 'airbyte-public-jars' defined in settings.graddle only in its omission + // of the 'public' directory. Any artifacts publish here will be available in the 'airbyte-public-jars' repo + repositories { + maven { + name 'airbyte-repo' + url 'https://airbyte.mycloudrepo.io/repositories/airbyte-public-jars/' + credentials { + username System.getenv('CLOUDREPO_USER') + password System.getenv('CLOUDREPO_PASSWORD') + } } } } -} -// Adds publishToMavenLocal as final command in the list of 'build' tasks. -build.dependsOn(publishToMavenLocal) + project.configurations { + testImplementation.extendsFrom implementation + testFixturesImplementation.extendsFrom implementation + testFixturesRuntimeOnly.extendsFrom runtimeOnly + } +} -publishToMavenLocal { - // Always re-publish the artifact to MavenLocal - outputs.upToDateWhen { false } +description = "Airbyte Connector Development Kit (CDK) for Java." - doFirst { - println "Running CDK publishToMavenLocal..." - } - doLast { - println "Finished CDK publishToMavenLocal." +def recursiveTasks = [ + 'assemble', + 'build', + 'integrationTestJava', + 'publish', + 'publishToMavenLocal', + 'test', +] +recursiveTasks.each { taskName -> + tasks.named(taskName).configure { + dependsOn subprojects.collect { it.tasks.named(taskName) } } } -// This task will be a no-op if CDK version does not end with '-SNAPSHOT'. +// The `publishSnapshotIfNeeded` task will be a no-op if CDK version does not end with '-SNAPSHOT'. task publishSnapshotIfNeeded {} if (version.endsWith("-SNAPSHOT")) { logger.lifecycle("Version ${version} ends with '-SNAPSHOT'. Enqueing 'publishToMavenLocal'...") publishSnapshotIfNeeded.dependsOn publishToMavenLocal } else { - logger.lifecycle("Version ${version} does not end with '-SNAPSHOT'. Skipping task 'publishToMavenLocal'.") + // Uncomment as needed for debugging: + // logger.lifecycle("Version ${version} does not end with '-SNAPSHOT'. Skipping task 'publishToMavenLocal'.") +} + +task assertCdkVersionNotPublished { + doLast { + def checkGroupId = "io.airbyte.cdk" + def checkArtifactId = "airbyte-cdk-core" + def checkVersion = getCdkTargetVersion() + def repoUrl = "https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars" + def groupIdUrl = checkGroupId.replace('.', '/') + def artifactUrl = "${repoUrl}/${groupIdUrl}/${checkArtifactId}/${checkVersion}/${checkArtifactId}-${checkVersion}.pom" + + def connection = artifactUrl.toURL().openConnection() as HttpURLConnection + connection.setRequestMethod("HEAD") + connection.connect() + + def responseCode = connection.getResponseCode() + + if (responseCode == 200) { + throw new GradleException("Assert failed. Java CDK '${checkVersion}' already published at: ${artifactUrl}") + } else if (responseCode == 404) { + logger.lifecycle( + "Assert succeeded. Version ${checkVersion} of ${checkArtifactId} has not been published. " + + "Checked: ${artifactUrl}" + ) + } else { + logger.error("Received unexpected HTTP response code ${responseCode}. Ensure the repository is accessible.") + throw new GradleException("Error during assertion. Received unexpected HTTP response code ${responseCode}.") + } + } } -test { - useJUnitPlatform() - testLogging { - exceptionFormat = 'full' - showExceptions = true - showCauses = true - showStackTraces = false - events = ['passed', 'skipped', 'failed'] +def cleanLocalCache = tasks.register('cleanLocalCache') { + def userHome = System.getProperty("user.home") + doLast { + delete '.gradle' + delete '${userHome}/.m2/repository/io/airbyte/' + delete '${userHome}/.gradle/caches/modules-2/files-2.1/io.airbyte.cdk/' } } +cleanLocalCache.configure { + dependsOn tasks.named('clean') + dependsOn subprojects.collect { it.tasks.named('clean') } +} diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/README.md b/airbyte-cdk/java/airbyte-cdk/config-models-oss/README.md new file mode 100644 index 000000000000..996bcf073ec8 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/config-models-oss/README.md @@ -0,0 +1,20 @@ +# Config Models + +This module uses `jsonschema2pojo` to generate Java config objects from [json schema](https://json-schema.org/) definitions. See [build.gradle](./build.gradle) for details. + +## How to use +- Update json schema under: + ``` + src/main/resources/types/ + ``` +- Run the following command under the project root: + ```sh + ./gradlew airbyte-cdk:java:airbyte-cdk:config-models-oss:generateJsonSchema2Pojo + ``` + The generated file is under: + ``` + build/generated/src/gen/java/io/airbyte/config/ + ``` + +## Reference +- [`jsonschema2pojo` plugin](https://github.com/joelittlejohn/jsonschema2pojo/tree/master/jsonschema2pojo-gradle-plugin). diff --git a/airbyte-cdk/java/airbyte-cdk/config-models-oss/build.gradle b/airbyte-cdk/java/airbyte-cdk/config-models-oss/build.gradle new file mode 100644 index 000000000000..b64c0a8c7b40 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/config-models-oss/build.gradle @@ -0,0 +1,39 @@ +import org.jsonschema2pojo.SourceType + +plugins { + id "java-library" + id "com.github.eirnym.js2p" version "1.0" +} + +java { + compileJava { + options.compilerArgs += "-Xlint:-unchecked" + } +} + +dependencies { + annotationProcessor libs.bundles.micronaut.annotation.processor + api libs.bundles.micronaut.annotation + + implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') +} + +jsonSchema2Pojo { + sourceType = SourceType.YAMLSCHEMA + source = files("${sourceSets.main.output.resourcesDir}/types") + targetDirectory = new File(project.buildDir, 'generated/src/gen/java/') + + targetPackage = 'io.airbyte.configoss' + useLongIntegers = true + + removeOldOutput = true + + generateBuilders = true + includeConstructors = false + includeSetters = true + serializable = true +} +tasks.register('generate').configure { + dependsOn tasks.named('generateJsonSchema2Pojo') +} diff --git a/airbyte-config-oss/config-models-oss/src/main/java/io/airbyte/configoss/AirbyteConfig.java b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/AirbyteConfig.java similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/java/io/airbyte/configoss/AirbyteConfig.java rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/AirbyteConfig.java diff --git a/airbyte-config-oss/config-models-oss/src/main/java/io/airbyte/configoss/AirbyteConfigValidator.java b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/AirbyteConfigValidator.java similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/java/io/airbyte/configoss/AirbyteConfigValidator.java rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/AirbyteConfigValidator.java diff --git a/airbyte-config-oss/config-models-oss/src/main/java/io/airbyte/configoss/CatalogDefinitionsConfig.java b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/CatalogDefinitionsConfig.java similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/java/io/airbyte/configoss/CatalogDefinitionsConfig.java rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/CatalogDefinitionsConfig.java diff --git a/airbyte-config-oss/config-models-oss/src/main/java/io/airbyte/configoss/ConfigSchema.java b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/ConfigSchema.java similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/java/io/airbyte/configoss/ConfigSchema.java rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/ConfigSchema.java diff --git a/airbyte-config-oss/config-models-oss/src/main/java/io/airbyte/configoss/Configs.java b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/Configs.java similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/java/io/airbyte/configoss/Configs.java rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/Configs.java diff --git a/airbyte-config-oss/config-models-oss/src/main/java/io/airbyte/configoss/EnvConfigs.java b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/EnvConfigs.java similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/java/io/airbyte/configoss/EnvConfigs.java rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/EnvConfigs.java diff --git a/airbyte-config-oss/config-models-oss/src/main/java/io/airbyte/configoss/helpers/StateMessageHelper.java b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/helpers/StateMessageHelper.java similarity index 82% rename from airbyte-config-oss/config-models-oss/src/main/java/io/airbyte/configoss/helpers/StateMessageHelper.java rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/helpers/StateMessageHelper.java index 04b54e013eb0..22274b2dadff 100644 --- a/airbyte-config-oss/config-models-oss/src/main/java/io/airbyte/configoss/helpers/StateMessageHelper.java +++ b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/helpers/StateMessageHelper.java @@ -6,7 +6,6 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.Iterables; import io.airbyte.commons.json.Jsons; import io.airbyte.configoss.State; import io.airbyte.configoss.StateType; @@ -29,7 +28,7 @@ public static class AirbyteStateMessageListTypeReference extends TypeReference getTypedState(final JsonNode state, final boolean useStreamCapableState) { + public static Optional getTypedState(final JsonNode state) { if (state == null) { return Optional.empty(); } else { @@ -49,10 +48,10 @@ public static Optional getTypedState(final JsonNode state, final b } else { switch (stateMessages.get(0).getType()) { case GLOBAL -> { - return Optional.of(provideGlobalState(stateMessages.get(0), useStreamCapableState)); + return Optional.of(provideGlobalState(stateMessages.get(0))); } case STREAM -> { - return Optional.of(provideStreamState(stateMessages, useStreamCapableState)); + return Optional.of(provideStreamState(stateMessages)); } case LEGACY -> { return Optional.of(getLegacyStateWrapper(stateMessages.get(0).getData())); @@ -65,7 +64,7 @@ public static Optional getTypedState(final JsonNode state, final b } } else { if (stateMessages.stream().allMatch(stateMessage -> stateMessage.getType() == AirbyteStateType.STREAM)) { - return Optional.of(provideStreamState(stateMessages, useStreamCapableState)); + return Optional.of(provideStreamState(stateMessages)); } if (stateMessages.stream().allMatch(stateMessage -> stateMessage.getType() == null)) { return Optional.of(getLegacyStateWrapper(state)); @@ -104,16 +103,10 @@ public static Boolean isMigration(final StateType currentStateType, final @Nulla return previousStateType == StateType.LEGACY && currentStateType != StateType.LEGACY; } - private static StateWrapper provideGlobalState(final AirbyteStateMessage stateMessages, final boolean useStreamCapableState) { - if (useStreamCapableState) { - return new StateWrapper() - .withStateType(StateType.GLOBAL) - .withGlobal(stateMessages); - } else { - return new StateWrapper() - .withStateType(StateType.LEGACY) - .withLegacyState(stateMessages.getData()); - } + private static StateWrapper provideGlobalState(final AirbyteStateMessage stateMessages) { + return new StateWrapper() + .withStateType(StateType.GLOBAL) + .withGlobal(stateMessages); } /** @@ -123,16 +116,11 @@ private static StateWrapper provideGlobalState(final AirbyteStateMessage stateMe * @param useStreamCapableState - a flag that indicates whether to return the new format * @return a wrapped state */ - private static StateWrapper provideStreamState(final List stateMessages, final boolean useStreamCapableState) { - if (useStreamCapableState) { - return new StateWrapper() - .withStateType(StateType.STREAM) - .withStateMessages(stateMessages); - } else { - return new StateWrapper() - .withStateType(StateType.LEGACY) - .withLegacyState(Iterables.getLast(stateMessages).getData()); - } + private static StateWrapper provideStreamState(final List stateMessages) { + return new StateWrapper() + .withStateType(StateType.STREAM) + .withStateMessages(stateMessages); + } private static StateWrapper getLegacyStateWrapper(final JsonNode state) { diff --git a/airbyte-config-oss/config-models-oss/src/main/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitions.java b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitions.java similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitions.java rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitions.java diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/ActorDefinitionResourceRequirements.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ActorDefinitionResourceRequirements.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/ActorDefinitionResourceRequirements.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ActorDefinitionResourceRequirements.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/ActorType.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ActorType.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/ActorType.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ActorType.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/AllowedHosts.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/AllowedHosts.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/AllowedHosts.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/AllowedHosts.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/CombinedConnectorCatalog.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/CombinedConnectorCatalog.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/CombinedConnectorCatalog.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/CombinedConnectorCatalog.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/ConnectorJobOutput.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ConnectorJobOutput.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/ConnectorJobOutput.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ConnectorJobOutput.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/DataType.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DataType.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/DataType.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DataType.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/DestinationConnection.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DestinationConnection.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/DestinationConnection.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DestinationConnection.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/DestinationOAuthParameter.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DestinationOAuthParameter.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/DestinationOAuthParameter.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DestinationOAuthParameter.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/DestinationSyncMode.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DestinationSyncMode.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/DestinationSyncMode.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DestinationSyncMode.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/DockerImageSpec.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DockerImageSpec.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/DockerImageSpec.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/DockerImageSpec.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/FailureReason.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/FailureReason.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/FailureReason.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/FailureReason.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/JobGetSpecConfig.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/JobGetSpecConfig.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/JobGetSpecConfig.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/JobGetSpecConfig.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/JobSyncConfig.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/JobSyncConfig.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/JobSyncConfig.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/JobSyncConfig.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/JobType.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/JobType.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/JobType.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/JobType.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/NamespaceDefinitionType.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/NamespaceDefinitionType.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/NamespaceDefinitionType.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/NamespaceDefinitionType.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/NormalizationDestinationDefinitionConfig.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/NormalizationDestinationDefinitionConfig.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/NormalizationDestinationDefinitionConfig.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/NormalizationDestinationDefinitionConfig.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/NotificationType.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/NotificationType.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/NotificationType.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/NotificationType.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/OperatorDbt.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorDbt.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/OperatorDbt.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorDbt.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/OperatorNormalization.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorNormalization.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/OperatorNormalization.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorNormalization.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/OperatorType.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorType.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/OperatorType.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorType.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/OperatorWebhook.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorWebhook.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/OperatorWebhook.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/OperatorWebhook.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/ReplicationStatus.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ReplicationStatus.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/ReplicationStatus.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ReplicationStatus.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/ResourceRequirements.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ResourceRequirements.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/ResourceRequirements.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/ResourceRequirements.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/SourceConnection.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SourceConnection.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/SourceConnection.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SourceConnection.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/SourceOAuthParameter.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SourceOAuthParameter.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/SourceOAuthParameter.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SourceOAuthParameter.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/StandardCheckConnectionInput.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardCheckConnectionInput.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/StandardCheckConnectionInput.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardCheckConnectionInput.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/StandardCheckConnectionOutput.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardCheckConnectionOutput.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/StandardCheckConnectionOutput.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardCheckConnectionOutput.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/StandardDestinationDefinition.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardDestinationDefinition.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/StandardDestinationDefinition.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardDestinationDefinition.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/StandardDiscoverCatalogInput.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardDiscoverCatalogInput.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/StandardDiscoverCatalogInput.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardDiscoverCatalogInput.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/StandardSourceDefinition.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardSourceDefinition.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/StandardSourceDefinition.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardSourceDefinition.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/StandardSyncInput.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardSyncInput.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/StandardSyncInput.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardSyncInput.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/StandardSyncOperation.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardSyncOperation.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/StandardSyncOperation.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StandardSyncOperation.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/State.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/State.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/State.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/State.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/StateType.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StateType.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/StateType.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StateType.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/StateWrapper.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StateWrapper.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/StateWrapper.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/StateWrapper.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/SuggestedStreams.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SuggestedStreams.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/SuggestedStreams.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SuggestedStreams.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/SyncMode.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SyncMode.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/SyncMode.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SyncMode.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/SyncStats.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SyncStats.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/SyncStats.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/SyncStats.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/WebhookOperationConfigs.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WebhookOperationConfigs.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/WebhookOperationConfigs.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WebhookOperationConfigs.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/WebhookOperationSummary.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WebhookOperationSummary.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/WebhookOperationSummary.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WebhookOperationSummary.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/WorkerDestinationConfig.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WorkerDestinationConfig.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/WorkerDestinationConfig.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WorkerDestinationConfig.yaml diff --git a/airbyte-config-oss/config-models-oss/src/main/resources/types/WorkerSourceConfig.yaml b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WorkerSourceConfig.yaml similarity index 100% rename from airbyte-config-oss/config-models-oss/src/main/resources/types/WorkerSourceConfig.yaml rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/main/resources/types/WorkerSourceConfig.yaml diff --git a/airbyte-config-oss/config-models-oss/src/test/java/io/airbyte/configoss/ConfigSchemaTest.java b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/test/java/io/airbyte/configoss/ConfigSchemaTest.java similarity index 100% rename from airbyte-config-oss/config-models-oss/src/test/java/io/airbyte/configoss/ConfigSchemaTest.java rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/test/java/io/airbyte/configoss/ConfigSchemaTest.java diff --git a/airbyte-config-oss/config-models-oss/src/test/java/io/airbyte/configoss/DataTypeEnumTest.java b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/test/java/io/airbyte/configoss/DataTypeEnumTest.java similarity index 100% rename from airbyte-config-oss/config-models-oss/src/test/java/io/airbyte/configoss/DataTypeEnumTest.java rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/test/java/io/airbyte/configoss/DataTypeEnumTest.java diff --git a/airbyte-config-oss/config-models-oss/src/test/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitionsTest.java b/airbyte-cdk/java/airbyte-cdk/config-models-oss/src/test/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitionsTest.java similarity index 100% rename from airbyte-config-oss/config-models-oss/src/test/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitionsTest.java rename to airbyte-cdk/java/airbyte-cdk/config-models-oss/src/test/java/io/airbyte/configoss/helpers/YamlListToStandardDefinitionsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/core/build.gradle b/airbyte-cdk/java/airbyte-cdk/core/build.gradle new file mode 100644 index 000000000000..cfef9562cbbe --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/build.gradle @@ -0,0 +1,106 @@ + +java { + compileJava { + options.compilerArgs += "-Xlint:-deprecation,-try,-rawtypes,-overloads,-cast,-unchecked" + } + compileTestJava { + options.compilerArgs += "-Xlint:-try,-divzero,-cast" + } +} + +configurations.all { + resolutionStrategy { + // TODO: Diagnose conflicting dependencies and remove these force overrides: + force 'org.mockito:mockito-core:4.6.1' + } +} + +dependencies { + // Exported dependencies from upstream projects + api libs.airbyte.protocol + api libs.hikaricp + api libs.jooq + api libs.jooq.meta + + compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-api') + compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') + compileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') + compileOnly project(':airbyte-cdk:java:airbyte-cdk:init-oss') + compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') + testCompileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') + + testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') + testImplementation project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') + + // SSH dependencies + implementation 'net.i2p.crypto:eddsa:0.3.0' + + // First party test dependencies + testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + testImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:db-sources')) + + testFixturesImplementation "org.hamcrest:hamcrest-all:1.3" + + testImplementation libs.bundles.junit + testImplementation libs.junit.jupiter.api + testImplementation libs.junit.jupiter.params + testImplementation 'org.junit.platform:junit-platform-launcher:1.7.0' + testImplementation libs.junit.jupiter.engine + implementation libs.jooq + implementation 'net.sourceforge.argparse4j:argparse4j:0.8.1' + implementation "io.aesy:datasize:1.0.0" + implementation libs.apache.commons + implementation libs.apache.commons.lang + testImplementation 'commons-lang:commons-lang:2.6' + implementation 'commons-cli:commons-cli:1.4' + implementation 'org.apache.commons:commons-csv:1.4' + + // Optional dependencies + // TODO: Change these to 'compileOnly' or 'testCompileOnly' + implementation 'com.azure:azure-storage-blob:12.12.0' + implementation('com.google.cloud:google-cloud-bigquery:1.133.1') + implementation 'org.mongodb:mongodb-driver-sync:4.3.0' + implementation libs.postgresql + + // testImplementation libs.junit.jupiter.api + implementation libs.hikaricp + implementation libs.debezium.api + implementation libs.debezium.embedded + implementation libs.debezium.mysql + implementation libs.debezium.postgres + implementation libs.debezium.mongodb + + api libs.bundles.datadog + implementation 'org.apache.sshd:sshd-mina:2.11.0' + + implementation libs.testcontainers + implementation libs.testcontainers.mysql + implementation libs.testcontainers.jdbc + implementation libs.testcontainers.postgresql + testImplementation libs.testcontainers.jdbc + testImplementation libs.testcontainers.mysql + testImplementation libs.testcontainers.postgresql + implementation 'org.codehaus.plexus:plexus-utils:3.4.2' + + // bouncycastle is pinned to version-match the transitive dependency from kubernetes client-java + // because a version conflict causes "parameter object not a ECParameterSpec" on ssh tunnel initiation + implementation 'org.bouncycastle:bcpkix-jdk15on:1.66' + implementation 'org.bouncycastle:bcprov-jdk15on:1.66' + implementation 'org.bouncycastle:bctls-jdk15on:1.66' + + // Lombok + implementation 'org.projectlombok:lombok:1.18.20' + annotationProcessor 'org.projectlombok:lombok:1.18.20' + testFixturesImplementation 'org.projectlombok:lombok:1.18.20' + testFixturesAnnotationProcessor 'org.projectlombok:lombok:1.18.20' + + testImplementation libs.junit.jupiter.system.stubs + + implementation libs.jackson.annotations + implementation group: 'org.apache.logging.log4j', name: 'log4j-layout-template-json', version: '2.17.2' + + testImplementation 'org.apache.commons:commons-lang3:3.11' + testImplementation 'org.xerial.snappy:snappy-java:1.1.8.4' + testImplementation 'org.mockito:mockito-core:4.6.1' +} diff --git a/airbyte-cdk/java/airbyte-cdk/src/main/java/io/airbyte/cdk/CDKConstants.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/CDKConstants.java similarity index 100% rename from airbyte-cdk/java/airbyte-cdk/src/main/java/io/airbyte/cdk/CDKConstants.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/CDKConstants.java diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/AbstractDatabase.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/AbstractDatabase.java similarity index 97% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/AbstractDatabase.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/AbstractDatabase.java index 7b72ec797e76..94c994163c42 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/AbstractDatabase.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/AbstractDatabase.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db; +package io.airbyte.cdk.db; import com.fasterxml.jackson.databind.JsonNode; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/ContextQueryFunction.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/ContextQueryFunction.java similarity index 90% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/ContextQueryFunction.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/ContextQueryFunction.java index 977392c34c2e..c568227960e8 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/ContextQueryFunction.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/ContextQueryFunction.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db; +package io.airbyte.cdk.db; import java.sql.SQLException; import org.jooq.DSLContext; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/DataTypeSupplier.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/DataTypeSupplier.java similarity index 88% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/DataTypeSupplier.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/DataTypeSupplier.java index be7b490022f9..56f6adc64e63 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/DataTypeSupplier.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/DataTypeSupplier.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db; +package io.airbyte.cdk.db; import java.sql.SQLException; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/DataTypeUtils.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/DataTypeUtils.java similarity index 98% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/DataTypeUtils.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/DataTypeUtils.java index 67af66daf972..cf44f8d9a8a4 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/DataTypeUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/DataTypeUtils.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db; +package io.airbyte.cdk.db; import java.sql.Date; import java.sql.SQLException; @@ -18,7 +18,7 @@ /** * TODO : Replace all the DateTime related logic of this class with - * {@link io.airbyte.db.jdbc.DateTimeConverter} + * {@link io.airbyte.cdk.db.jdbc.DateTimeConverter} */ public class DataTypeUtils { diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/Database.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/Database.java similarity index 96% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/Database.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/Database.java index 5f4d809774b7..ba817100fd7a 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/Database.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/Database.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db; +package io.airbyte.cdk.db; import java.sql.SQLException; import org.jooq.DSLContext; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/ExceptionWrappingDatabase.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/ExceptionWrappingDatabase.java similarity index 96% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/ExceptionWrappingDatabase.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/ExceptionWrappingDatabase.java index 42d7e9fb9ee7..b200cfe9e69a 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/ExceptionWrappingDatabase.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/ExceptionWrappingDatabase.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db; +package io.airbyte.cdk.db; import java.io.IOException; import java.sql.SQLException; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/IncrementalUtils.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/IncrementalUtils.java similarity index 99% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/IncrementalUtils.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/IncrementalUtils.java index d1940d9cb989..7436963249c0 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/IncrementalUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/IncrementalUtils.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db; +package io.airbyte.cdk.db; import io.airbyte.protocol.models.JsonSchemaPrimitiveUtil; import io.airbyte.protocol.models.JsonSchemaPrimitiveUtil.JsonSchemaPrimitive; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/JdbcCompatibleSourceOperations.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/JdbcCompatibleSourceOperations.java similarity index 98% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/JdbcCompatibleSourceOperations.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/JdbcCompatibleSourceOperations.java index c13862527cf5..0bdcc8a9ed14 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/JdbcCompatibleSourceOperations.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/JdbcCompatibleSourceOperations.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db; +package io.airbyte.cdk.db; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/PgLsn.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/PgLsn.java similarity index 98% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/PgLsn.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/PgLsn.java index f41212757948..eb7ab7013756 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/PgLsn.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/PgLsn.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db; +package io.airbyte.cdk.db; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/PostgresUtils.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/PostgresUtils.java new file mode 100644 index 000000000000..0b16eb5fed00 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/PostgresUtils.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.db; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.base.Preconditions; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import java.sql.SQLException; +import java.util.List; + +public class PostgresUtils { + + public static PgLsn getLsn(final JdbcDatabase database) throws SQLException { + // pg version >= 10. For versions < 10 use query select * from pg_current_xlog_location() + final List jsonNodes = database + .bufferedResultSetQuery(conn -> conn.createStatement().executeQuery("select * from pg_current_wal_lsn()"), + resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet)); + + Preconditions.checkState(jsonNodes.size() == 1); + return PgLsn.fromPgString(jsonNodes.get(0).get("pg_current_wal_lsn").asText()); + } + +} diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/SourceOperations.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/SourceOperations.java similarity index 95% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/SourceOperations.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/SourceOperations.java index 0e0dd68b4a54..de69ce653a75 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/SourceOperations.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/SourceOperations.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db; +package io.airbyte.cdk.db; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.protocol.models.JsonSchemaType; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/SqlDatabase.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/SqlDatabase.java similarity index 93% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/SqlDatabase.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/SqlDatabase.java index 524271f335f3..6cb912d325a3 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/SqlDatabase.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/SqlDatabase.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db; +package io.airbyte.cdk.db; import com.fasterxml.jackson.databind.JsonNode; import java.util.stream.Stream; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/bigquery/BigQueryDatabase.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryDatabase.java similarity index 99% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/bigquery/BigQueryDatabase.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryDatabase.java index da98a9036ad6..183ede3d9000 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/bigquery/BigQueryDatabase.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryDatabase.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.bigquery; +package io.airbyte.cdk.db.bigquery; import static java.util.Objects.isNull; import static org.apache.commons.lang3.StringUtils.EMPTY; @@ -23,7 +23,7 @@ import com.google.common.base.Charsets; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Streams; -import io.airbyte.db.SqlDatabase; +import io.airbyte.cdk.db.SqlDatabase; import java.io.ByteArrayInputStream; import java.io.IOException; import java.sql.SQLException; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/bigquery/BigQueryResultSet.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryResultSet.java similarity index 94% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/bigquery/BigQueryResultSet.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryResultSet.java index b92dae12c332..cd8275cd578f 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/bigquery/BigQueryResultSet.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/BigQueryResultSet.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.bigquery; +package io.airbyte.cdk.db.bigquery; import com.google.cloud.bigquery.FieldList; import com.google.cloud.bigquery.FieldValueList; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/bigquery/BigQuerySourceOperations.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/BigQuerySourceOperations.java similarity index 96% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/bigquery/BigQuerySourceOperations.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/BigQuerySourceOperations.java index 07c54ac7a131..fae06719b19e 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/bigquery/BigQuerySourceOperations.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/BigQuerySourceOperations.java @@ -2,10 +2,10 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.bigquery; +package io.airbyte.cdk.db.bigquery; -import static io.airbyte.db.DataTypeUtils.returnNullIfInvalid; -import static io.airbyte.db.DataTypeUtils.toISO8601String; +import static io.airbyte.cdk.db.DataTypeUtils.returnNullIfInvalid; +import static io.airbyte.cdk.db.DataTypeUtils.toISO8601String; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; @@ -17,10 +17,10 @@ import com.google.cloud.bigquery.FieldValue.Attribute; import com.google.cloud.bigquery.QueryParameterValue; import com.google.cloud.bigquery.StandardSQLTypeName; +import io.airbyte.cdk.db.DataTypeUtils; +import io.airbyte.cdk.db.SourceOperations; +import io.airbyte.cdk.db.util.JsonUtil; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.DataTypeUtils; -import io.airbyte.db.SourceOperations; -import io.airbyte.db.util.JsonUtil; import io.airbyte.protocol.models.JsonSchemaType; import java.text.DateFormat; import java.text.ParseException; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/bigquery/TempBigQueryJoolDatabaseImpl.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/TempBigQueryJoolDatabaseImpl.java similarity index 93% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/bigquery/TempBigQueryJoolDatabaseImpl.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/TempBigQueryJoolDatabaseImpl.java index 1460fd3a5fa8..4e4c9deb72ee 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/bigquery/TempBigQueryJoolDatabaseImpl.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/bigquery/TempBigQueryJoolDatabaseImpl.java @@ -2,10 +2,10 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.bigquery; +package io.airbyte.cdk.db.bigquery; -import io.airbyte.db.ContextQueryFunction; -import io.airbyte.db.Database; +import io.airbyte.cdk.db.ContextQueryFunction; +import io.airbyte.cdk.db.Database; import java.sql.SQLException; import javax.annotation.Nullable; import org.jooq.Record; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/ConnectionFactory.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/factory/ConnectionFactory.java similarity index 97% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/ConnectionFactory.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/factory/ConnectionFactory.java index cd428f9e8130..ab2eb4d212b0 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/ConnectionFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/factory/ConnectionFactory.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.factory; +package io.airbyte.cdk.db.factory; import java.sql.Connection; import java.sql.DriverManager; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/DSLContextFactory.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/factory/DSLContextFactory.java similarity index 92% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/DSLContextFactory.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/factory/DSLContextFactory.java index 4526681f44da..b70888255e1c 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/DSLContextFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/factory/DSLContextFactory.java @@ -2,8 +2,9 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.factory; +package io.airbyte.cdk.db.factory; +import java.time.Duration; import java.util.Map; import javax.sql.DataSource; import org.jooq.DSLContext; @@ -62,8 +63,10 @@ public static DSLContext create(final String username, final String driverClassName, final String jdbcConnectionString, final SQLDialect dialect, - final Map connectionProperties) { - return DSL.using(DataSourceFactory.create(username, password, driverClassName, jdbcConnectionString, connectionProperties), dialect); + final Map connectionProperties, + final Duration connectionTimeout) { + return DSL.using(DataSourceFactory.create(username, password, driverClassName, jdbcConnectionString, connectionProperties, + connectionTimeout), dialect); } } diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/factory/DataSourceFactory.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/factory/DataSourceFactory.java new file mode 100644 index 000000000000..a4324a30ebf7 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/factory/DataSourceFactory.java @@ -0,0 +1,290 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.db.factory; + +import com.google.common.base.Preconditions; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import java.io.Closeable; +import java.time.Duration; +import java.util.Map; +import javax.sql.DataSource; + +/** + * Temporary factory class that provides convenience methods for creating a {@link DataSource} + * instance. This class will be removed once the project has been converted to leverage an + * application framework to manage the creation and injection of {@link DataSource} objects. + */ +public class DataSourceFactory { + + /** + * Constructs a new {@link DataSource} using the provided configuration. + * + * @param username The username of the database user. + * @param password The password of the database user. + * @param driverClassName The fully qualified name of the JDBC driver class. + * @param jdbcConnectionString The JDBC connection string. + * @return The configured {@link DataSource}. + */ + public static DataSource create(final String username, + final String password, + final String driverClassName, + final String jdbcConnectionString) { + return new DataSourceBuilder(username, password, driverClassName, jdbcConnectionString) + .build(); + } + + /** + * Constructs a new {@link DataSource} using the provided configuration. + * + * @param username The username of the database user. + * @param password The password of the database user. + * @param driverClassName The fully qualified name of the JDBC driver class. + * @param jdbcConnectionString The JDBC connection string. + * @param connectionProperties Additional configuration properties for the underlying driver. + * @return The configured {@link DataSource}. + */ + public static DataSource create(final String username, + final String password, + final String driverClassName, + final String jdbcConnectionString, + final Map connectionProperties, + final Duration connectionTimeout) { + return new DataSourceBuilder(username, password, driverClassName, jdbcConnectionString) + .withConnectionProperties(connectionProperties) + .withConnectionTimeout(connectionTimeout) + .build(); + } + + /** + * Constructs a new {@link DataSource} using the provided configuration. + * + * @param username The username of the database user. + * @param password The password of the database user. + * @param host The host address of the database. + * @param port The port of the database. + * @param database The name of the database. + * @param driverClassName The fully qualified name of the JDBC driver class. + * @return The configured {@link DataSource}. + */ + public static DataSource create(final String username, + final String password, + final String host, + final int port, + final String database, + final String driverClassName) { + return new DataSourceBuilder(username, password, driverClassName, host, port, database) + .build(); + } + + /** + * Constructs a new {@link DataSource} using the provided configuration. + * + * @param username The username of the database user. + * @param password The password of the database user. + * @param host The host address of the database. + * @param port The port of the database. + * @param database The name of the database. + * @param driverClassName The fully qualified name of the JDBC driver class. + * @param connectionProperties Additional configuration properties for the underlying driver. + * @return The configured {@link DataSource}. + */ + public static DataSource create(final String username, + final String password, + final String host, + final int port, + final String database, + final String driverClassName, + final Map connectionProperties) { + return new DataSourceBuilder(username, password, driverClassName, host, port, database) + .withConnectionProperties(connectionProperties) + .build(); + } + + /** + * Convenience method that constructs a new {@link DataSource} for a PostgreSQL database using the + * provided configuration. + * + * @param username The username of the database user. + * @param password The password of the database user. + * @param host The host address of the database. + * @param port The port of the database. + * @param database The name of the database. + * @return The configured {@link DataSource}. + */ + public static DataSource createPostgres(final String username, + final String password, + final String host, + final int port, + final String database) { + return new DataSourceBuilder(username, password, "org.postgresql.Driver", host, port, database) + .build(); + } + + /** + * Utility method that attempts to close the provided {@link DataSource} if it implements + * {@link Closeable}. + * + * @param dataSource The {@link DataSource} to close. + * @throws Exception if unable to close the data source. + */ + public static void close(final DataSource dataSource) throws Exception { + if (dataSource != null) { + if (dataSource instanceof final AutoCloseable closeable) { + closeable.close(); + } + } + } + + /** + * Builder class used to configure and construct {@link DataSource} instances. + */ + public static class DataSourceBuilder { + + private Map connectionProperties = Map.of(); + private String database; + private String driverClassName; + private String host; + private String jdbcUrl; + private int maximumPoolSize = 10; + private int minimumPoolSize = 0; + private Duration connectionTimeout = Duration.ZERO; + private String password; + private int port = 5432; + private String username; + private String connectionInitSql; + + private DataSourceBuilder(final String username, + final String password, + final String driverClassName) { + this.username = username; + this.password = password; + this.driverClassName = driverClassName; + } + + public DataSourceBuilder(final String username, + final String password, + final String driverClassName, + final String jdbcUrl) { + this(username, password, driverClassName); + this.jdbcUrl = jdbcUrl; + } + + public DataSourceBuilder(final String username, + final String password, + final String driverClassName, + final String host, + final int port, + final String database) { + this(username, password, driverClassName); + this.host = host; + this.port = port; + this.database = database; + } + + public DataSourceBuilder withConnectionProperties(final Map connectionProperties) { + if (connectionProperties != null) { + this.connectionProperties = connectionProperties; + } + return this; + } + + public DataSourceBuilder withDatabase(final String database) { + this.database = database; + return this; + } + + public DataSourceBuilder withDriverClassName(final String driverClassName) { + this.driverClassName = driverClassName; + return this; + } + + public DataSourceBuilder withHost(final String host) { + this.host = host; + return this; + } + + public DataSourceBuilder withJdbcUrl(final String jdbcUrl) { + this.jdbcUrl = jdbcUrl; + return this; + } + + public DataSourceBuilder withMaximumPoolSize(final Integer maximumPoolSize) { + if (maximumPoolSize != null) { + this.maximumPoolSize = maximumPoolSize; + } + return this; + } + + public DataSourceBuilder withMinimumPoolSize(final Integer minimumPoolSize) { + if (minimumPoolSize != null) { + this.minimumPoolSize = minimumPoolSize; + } + return this; + } + + public DataSourceBuilder withConnectionTimeout(final Duration connectionTimeout) { + if (connectionTimeout != null) { + this.connectionTimeout = connectionTimeout; + } + return this; + } + + public DataSourceBuilder withPassword(final String password) { + this.password = password; + return this; + } + + public DataSourceBuilder withPort(final Integer port) { + if (port != null) { + this.port = port; + } + return this; + } + + public DataSourceBuilder withUsername(final String username) { + this.username = username; + return this; + } + + public DataSourceBuilder withConnectionInitSql(final String sql) { + this.connectionInitSql = sql; + return this; + } + + public DataSource build() { + final DatabaseDriver databaseDriver = DatabaseDriver.findByDriverClassName(driverClassName); + + Preconditions.checkNotNull(databaseDriver, "Unknown or blank driver class name: '" + driverClassName + "'."); + + final HikariConfig config = new HikariConfig(); + + config.setDriverClassName(databaseDriver.getDriverClassName()); + config.setJdbcUrl(jdbcUrl != null ? jdbcUrl : String.format(databaseDriver.getUrlFormatString(), host, port, database)); + config.setMaximumPoolSize(maximumPoolSize); + config.setMinimumIdle(minimumPoolSize); + // HikariCP uses milliseconds for all time values: + // https://github.com/brettwooldridge/HikariCP#gear-configuration-knobs-baby + config.setConnectionTimeout(connectionTimeout.toMillis()); + config.setPassword(password); + config.setUsername(username); + + /* + * Disable to prevent failing on startup. Applications may start prior to the database container + * being available. To avoid failing to create the connection pool, disable the fail check. This + * will preserve existing behavior that tests for the connection on first use, not on creation. + */ + config.setInitializationFailTimeout(Integer.MIN_VALUE); + + config.setConnectionInitSql(connectionInitSql); + + connectionProperties.forEach(config::addDataSourceProperty); + + return new HikariDataSource(config); + } + + } + +} diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/DatabaseDriver.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/factory/DatabaseDriver.java similarity index 98% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/DatabaseDriver.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/factory/DatabaseDriver.java index 662ed808de29..27e7750e1847 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/DatabaseDriver.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/factory/DatabaseDriver.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.factory; +package io.airbyte.cdk.db.factory; /** * Collection of JDBC driver class names and the associated JDBC URL format string. diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/AbstractJdbcCompatibleSourceOperations.java similarity index 98% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/AbstractJdbcCompatibleSourceOperations.java index 1c91d044d02b..e7b8514cb7f5 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/AbstractJdbcCompatibleSourceOperations.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/AbstractJdbcCompatibleSourceOperations.java @@ -2,17 +2,17 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc; +package io.airbyte.cdk.db.jdbc; -import static io.airbyte.db.DataTypeUtils.TIMESTAMPTZ_FORMATTER; +import static io.airbyte.cdk.db.DataTypeUtils.TIMESTAMPTZ_FORMATTER; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.DataTypeUtils; +import io.airbyte.cdk.db.JdbcCompatibleSourceOperations; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.DataTypeUtils; -import io.airbyte.db.JdbcCompatibleSourceOperations; import java.math.BigDecimal; import java.sql.Date; import java.sql.PreparedStatement; @@ -222,7 +222,7 @@ protected void setString(final PreparedStatement preparedStatement, final int pa } protected void setBinary(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException { - preparedStatement.setBytes(parameterIndex, DatatypeConverter.parseHexBinary(value)); + preparedStatement.setBytes(parameterIndex, DatatypeConverter.parseBase64Binary(value)); } protected ObjectType getObject(final ResultSet resultSet, final int index, final Class clazz) throws SQLException { diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/DateTimeConverter.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/DateTimeConverter.java similarity index 95% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/DateTimeConverter.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/DateTimeConverter.java index 98d08a2ea7c4..215b679fe697 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/DateTimeConverter.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/DateTimeConverter.java @@ -2,14 +2,14 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc; - -import static io.airbyte.db.DataTypeUtils.DATE_FORMATTER; -import static io.airbyte.db.DataTypeUtils.TIMESTAMPTZ_FORMATTER; -import static io.airbyte.db.DataTypeUtils.TIMESTAMP_FORMATTER; -import static io.airbyte.db.DataTypeUtils.TIMETZ_FORMATTER; -import static io.airbyte.db.DataTypeUtils.TIME_FORMATTER; -import static io.airbyte.db.jdbc.AbstractJdbcCompatibleSourceOperations.resolveEra; +package io.airbyte.cdk.db.jdbc; + +import static io.airbyte.cdk.db.DataTypeUtils.DATE_FORMATTER; +import static io.airbyte.cdk.db.DataTypeUtils.TIMESTAMPTZ_FORMATTER; +import static io.airbyte.cdk.db.DataTypeUtils.TIMESTAMP_FORMATTER; +import static io.airbyte.cdk.db.DataTypeUtils.TIMETZ_FORMATTER; +import static io.airbyte.cdk.db.DataTypeUtils.TIME_FORMATTER; +import static io.airbyte.cdk.db.jdbc.AbstractJdbcCompatibleSourceOperations.resolveEra; import static java.time.ZoneOffset.UTC; import com.fasterxml.jackson.databind.node.ObjectNode; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/DefaultJdbcDatabase.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/DefaultJdbcDatabase.java similarity index 98% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/DefaultJdbcDatabase.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/DefaultJdbcDatabase.java index 00e804f16835..9b3affc6dd33 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/DefaultJdbcDatabase.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/DefaultJdbcDatabase.java @@ -2,13 +2,13 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc; +package io.airbyte.cdk.db.jdbc; import com.google.errorprone.annotations.MustBeClosed; +import io.airbyte.cdk.db.JdbcCompatibleSourceOperations; import io.airbyte.commons.exceptions.ConnectionErrorException; import io.airbyte.commons.functional.CheckedConsumer; import io.airbyte.commons.functional.CheckedFunction; -import io.airbyte.db.JdbcCompatibleSourceOperations; import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.PreparedStatement; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcConstants.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/JdbcConstants.java similarity index 98% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcConstants.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/JdbcConstants.java index 592d1b59b62d..790e5ac37a28 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcConstants.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/JdbcConstants.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc; +package io.airbyte.cdk.db.jdbc; public final class JdbcConstants { diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcDatabase.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/JdbcDatabase.java similarity index 90% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcDatabase.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/JdbcDatabase.java index fcc024d5f4ae..8557aaecece6 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcDatabase.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/JdbcDatabase.java @@ -2,14 +2,14 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc; +package io.airbyte.cdk.db.jdbc; import com.fasterxml.jackson.databind.JsonNode; import com.google.errorprone.annotations.MustBeClosed; +import io.airbyte.cdk.db.JdbcCompatibleSourceOperations; +import io.airbyte.cdk.db.SqlDatabase; import io.airbyte.commons.functional.CheckedConsumer; import io.airbyte.commons.functional.CheckedFunction; -import io.airbyte.db.JdbcCompatibleSourceOperations; -import io.airbyte.db.SqlDatabase; import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.PreparedStatement; @@ -174,15 +174,17 @@ public List queryJsons(final CheckedFunction stream = unsafeQuery(c -> { - PreparedStatement statement = c.prepareStatement(sql); - int i = 1; - for (String param : params) { - statement.setString(i, param); - ++i; - } - return statement; - }, rs -> rs.getInt(1))) { + try (final Stream stream = unsafeQuery( + c -> getPreparedStatement(sql, params, c), + rs -> rs.getInt(1))) { + return stream.findFirst().get(); + } + } + + public boolean queryBoolean(final String sql, final String... params) throws SQLException { + try (final Stream stream = unsafeQuery( + c -> getPreparedStatement(sql, params, c), + rs -> rs.getBoolean(1))) { return stream.findFirst().get(); } } @@ -216,15 +218,8 @@ public List queryJsons(final String sql, final String... params) throw } public ResultSetMetaData queryMetadata(final String sql, final String... params) throws SQLException { - try (final Stream q = unsafeQuery(c -> { - PreparedStatement statement = c.prepareStatement(sql); - int i = 1; - for (String param : params) { - statement.setString(i, param); - ++i; - } - return statement; - }, + try (final Stream q = unsafeQuery( + c -> getPreparedStatement(sql, params, c), ResultSet::getMetaData)) { return q.findFirst().orElse(null); } @@ -232,4 +227,14 @@ public ResultSetMetaData queryMetadata(final String sql, final String... params) public abstract DatabaseMetaData getMetaData() throws SQLException; + private static PreparedStatement getPreparedStatement(String sql, String[] params, Connection c) throws SQLException { + PreparedStatement statement = c.prepareStatement(sql); + int i = 1; + for (String param : params) { + statement.setString(i, param); + i++; + } + return statement; + } + } diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcSourceOperations.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/JdbcSourceOperations.java similarity index 93% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcSourceOperations.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/JdbcSourceOperations.java index 529a5daa4e25..eb7dbafdcab3 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcSourceOperations.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/JdbcSourceOperations.java @@ -2,17 +2,17 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc; +package io.airbyte.cdk.db.jdbc; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_SCHEMA_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_TABLE_NAME; -import static io.airbyte.db.jdbc.JdbcUtils.ALLOWED_CURSOR_TYPES; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_COLUMN_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_SCHEMA_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_TABLE_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcUtils.ALLOWED_CURSOR_TYPES; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.db.SourceOperations; +import io.airbyte.cdk.db.SourceOperations; import io.airbyte.protocol.models.JsonSchemaType; import java.sql.JDBCType; import java.sql.PreparedStatement; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/JdbcUtils.java similarity index 99% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/JdbcUtils.java index 819fd4c2061c..a247359a7092 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/JdbcUtils.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc; +package io.airbyte.cdk.db.jdbc; import static java.sql.JDBCType.BIGINT; import static java.sql.JDBCType.DATE; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/StreamingJdbcDatabase.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/StreamingJdbcDatabase.java similarity index 94% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/StreamingJdbcDatabase.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/StreamingJdbcDatabase.java index d525a3e3df1b..50b5a36d03cb 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/StreamingJdbcDatabase.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/StreamingJdbcDatabase.java @@ -2,12 +2,12 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc; +package io.airbyte.cdk.db.jdbc; import com.google.errorprone.annotations.MustBeClosed; +import io.airbyte.cdk.db.JdbcCompatibleSourceOperations; +import io.airbyte.cdk.db.jdbc.streaming.JdbcStreamingQueryConfig; import io.airbyte.commons.functional.CheckedFunction; -import io.airbyte.db.JdbcCompatibleSourceOperations; -import io.airbyte.db.jdbc.streaming.JdbcStreamingQueryConfig; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -69,7 +69,9 @@ public Stream unsafeQuery(final CheckedFunction { try { - connection.setAutoCommit(true); + if (!connection.getAutoCommit()) { + connection.setAutoCommit(true); + } connection.close(); if (isStreamFailed) { throw new RuntimeException(streamException); diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/AdaptiveStreamingQueryConfig.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/AdaptiveStreamingQueryConfig.java similarity index 97% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/AdaptiveStreamingQueryConfig.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/AdaptiveStreamingQueryConfig.java index dc5f857c52dd..f7c933ac3184 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/AdaptiveStreamingQueryConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/AdaptiveStreamingQueryConfig.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc.streaming; +package io.airbyte.cdk.db.jdbc.streaming; import java.sql.Connection; import java.sql.ResultSet; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/BaseSizeEstimator.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/BaseSizeEstimator.java similarity index 98% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/BaseSizeEstimator.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/BaseSizeEstimator.java index 78e80a0a594f..0582f25fc00e 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/BaseSizeEstimator.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/BaseSizeEstimator.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc.streaming; +package io.airbyte.cdk.db.jdbc.streaming; import com.google.common.annotations.VisibleForTesting; import io.airbyte.commons.json.Jsons; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/FetchSizeConstants.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/FetchSizeConstants.java similarity index 95% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/FetchSizeConstants.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/FetchSizeConstants.java index 5b0263723d6d..26e07e8c016d 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/FetchSizeConstants.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/FetchSizeConstants.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc.streaming; +package io.airbyte.cdk.db.jdbc.streaming; public final class FetchSizeConstants { diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/FetchSizeEstimator.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/FetchSizeEstimator.java similarity index 88% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/FetchSizeEstimator.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/FetchSizeEstimator.java index 812ba5744d5f..acbd491c1dbf 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/FetchSizeEstimator.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/FetchSizeEstimator.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc.streaming; +package io.airbyte.cdk.db.jdbc.streaming; import java.util.Optional; import java.util.function.Consumer; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/InitialSizeEstimator.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/InitialSizeEstimator.java similarity index 96% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/InitialSizeEstimator.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/InitialSizeEstimator.java index c4ee0a3a8d36..1972f14c9b39 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/InitialSizeEstimator.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/InitialSizeEstimator.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc.streaming; +package io.airbyte.cdk.db.jdbc.streaming; import java.util.Optional; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/JdbcStreamingQueryConfig.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/JdbcStreamingQueryConfig.java similarity index 95% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/JdbcStreamingQueryConfig.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/JdbcStreamingQueryConfig.java index b7212623665f..b79b40b64ca9 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/JdbcStreamingQueryConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/JdbcStreamingQueryConfig.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc.streaming; +package io.airbyte.cdk.db.jdbc.streaming; import io.airbyte.commons.functional.CheckedBiConsumer; import java.sql.Connection; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/NoOpStreamingQueryConfig.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/NoOpStreamingQueryConfig.java similarity index 92% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/NoOpStreamingQueryConfig.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/NoOpStreamingQueryConfig.java index e27e456f1ac7..253e92e46b16 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/NoOpStreamingQueryConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/NoOpStreamingQueryConfig.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc.streaming; +package io.airbyte.cdk.db.jdbc.streaming; import java.sql.Connection; import java.sql.ResultSet; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/SamplingSizeEstimator.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/SamplingSizeEstimator.java similarity index 97% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/SamplingSizeEstimator.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/SamplingSizeEstimator.java index 299659c1869e..e075c40e5010 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/SamplingSizeEstimator.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/SamplingSizeEstimator.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc.streaming; +package io.airbyte.cdk.db.jdbc.streaming; import java.util.Optional; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/TwoStageSizeEstimator.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/TwoStageSizeEstimator.java similarity index 98% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/TwoStageSizeEstimator.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/TwoStageSizeEstimator.java index b92cfab62011..aceba25813b1 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/TwoStageSizeEstimator.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/jdbc/streaming/TwoStageSizeEstimator.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc.streaming; +package io.airbyte.cdk.db.jdbc.streaming; import com.google.common.annotations.VisibleForTesting; import java.util.Optional; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/util/JsonUtil.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/util/JsonUtil.java similarity index 98% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/util/JsonUtil.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/util/JsonUtil.java index 4d02314c0b57..0be5e4aa6ad5 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/util/JsonUtil.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/util/JsonUtil.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.util; +package io.airbyte.cdk.db.util; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ContainerNode; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/util/SSLCertificateUtils.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/util/SSLCertificateUtils.java similarity index 96% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/util/SSLCertificateUtils.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/util/SSLCertificateUtils.java index 0322432ed67a..7a502d18bfc6 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/util/SSLCertificateUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/db/util/SSLCertificateUtils.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.util; +package io.airbyte.cdk.db.util; import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; @@ -96,6 +96,12 @@ public static URI keyStoreFromCertificate(final String certString, return keyStoreFromCertificate(fromPEMString(certString), keyStorePassword, filesystem, directory); } + public static URI keyStoreFromCertificate(final String certString, + final String keyStorePassword) + throws CertificateException, IOException, KeyStoreException, NoSuchAlgorithmException { + return keyStoreFromCertificate(fromPEMString(certString), keyStorePassword, null, null); + } + public static URI keyStoreFromCertificate(final String certString, final String keyStorePassword, final String directory) throws CertificateException, IOException, KeyStoreException, NoSuchAlgorithmException { return keyStoreFromCertificate(certString, keyStorePassword, FileSystems.getDefault(), directory); diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/BaseConnector.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/BaseConnector.java similarity index 89% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/BaseConnector.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/BaseConnector.java index 0d7233c1aaaa..29a54a5f6822 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/BaseConnector.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/BaseConnector.java @@ -2,11 +2,11 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations; +package io.airbyte.cdk.integrations; +import io.airbyte.cdk.integrations.base.Integration; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.base.Integration; import io.airbyte.protocol.models.v0.ConnectorSpecification; public abstract class BaseConnector implements Integration { diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/JdbcConnector.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/JdbcConnector.java new file mode 100644 index 000000000000..1037c91858ec --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/JdbcConnector.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations; + +import static org.postgresql.PGProperty.CONNECT_TIMEOUT; + +import io.airbyte.cdk.db.factory.DatabaseDriver; +import java.time.Duration; +import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalUnit; +import java.util.Map; +import java.util.Optional; + +public abstract class JdbcConnector extends BaseConnector { + + public static final String CONNECT_TIMEOUT_KEY = "connectTimeout"; + public static final Duration CONNECT_TIMEOUT_DEFAULT = Duration.ofSeconds(60); + + protected final String driverClassName; + + protected JdbcConnector(String driverClassName) { + this.driverClassName = driverClassName; + } + + protected Duration getConnectionTimeout(final Map connectionProperties) { + return getConnectionTimeout(connectionProperties, driverClassName); + } + + /** + * Retrieves connectionTimeout value from connection properties in millis, default minimum timeout + * is 60 seconds since Hikari default of 30 seconds is not enough for acceptance tests. In the case + * the value is 0, pass the value along as Hikari and Postgres use default max value for 0 timeout + * value. + * + * NOTE: Postgres timeout is measured in seconds: + * https://jdbc.postgresql.org/documentation/head/connect.html + * + * @param connectionProperties custom jdbc_url_parameters containing information on connection + * properties + * @param driverClassName name of the JDBC driver + * @return DataSourceBuilder class used to create dynamic fields for DataSource + */ + public static Duration getConnectionTimeout(final Map connectionProperties, String driverClassName) { + final Optional parsedConnectionTimeout = switch (DatabaseDriver.findByDriverClassName(driverClassName)) { + case POSTGRESQL -> maybeParseDuration(connectionProperties.get(CONNECT_TIMEOUT.getName()), ChronoUnit.SECONDS) + .or(() -> maybeParseDuration(CONNECT_TIMEOUT.getDefaultValue(), ChronoUnit.SECONDS)); + case MYSQL -> maybeParseDuration(connectionProperties.get("connectTimeout"), ChronoUnit.MILLIS); + case MSSQLSERVER -> maybeParseDuration(connectionProperties.get("loginTimeout"), ChronoUnit.SECONDS); + default -> maybeParseDuration(connectionProperties.get(CONNECT_TIMEOUT_KEY), ChronoUnit.SECONDS) + // Enforce minimum timeout duration for unspecified data sources. + .filter(d -> d.compareTo(CONNECT_TIMEOUT_DEFAULT) >= 0); + }; + return parsedConnectionTimeout.orElse(CONNECT_TIMEOUT_DEFAULT); + } + + private static Optional maybeParseDuration(final String stringValue, TemporalUnit unit) { + if (stringValue == null) { + return Optional.empty(); + } + final long number; + try { + number = Long.parseLong(stringValue); + } catch (NumberFormatException __) { + return Optional.empty(); + } + if (number < 0) { + return Optional.empty(); + } + return Optional.of(Duration.of(number, unit)); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java new file mode 100644 index 000000000000..64502fb55232 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandler.java @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.base; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import java.util.Comparator; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AirbyteExceptionHandler implements Thread.UncaughtExceptionHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(AirbyteExceptionHandler.class); + public static final String logMessage = "Something went wrong in the connector. See the logs for more details."; + + // Basic deinterpolation helpers to avoid doing _really_ dumb deinterpolation. + // E.g. if "id" is in the list of strings to remove, we don't want to modify the message "Invalid + // identifier". + private static final String REGEX_PREFIX = "(^|[^A-Za-z0-9])"; + private static final String REGEX_SUFFIX = "($|[^A-Za-z0-9])"; + + /** + * If this list is populated, then the exception handler will attempt to deinterpolate the error + * message before emitting a trace message. This is useful for connectors which (a) emit a single + * exception class, and (b) rely on that exception's message to distinguish between error types. + *

+ * If this is active, then the trace message will: + *

    + *
  1. Not contain the stacktrace at all. This causes Sentry to use its fallback grouping (using + * exception class and message)
  2. + *
  3. Contain the original exception message as the external message, and a mangled message as the + * internal message.
  4. + *
+ */ + @VisibleForTesting + static final Set STRINGS_TO_DEINTERPOLATE = new HashSet<>(); + static { + addCommonStringsToDeinterpolate(); + } + + @VisibleForTesting + static final Set> THROWABLES_TO_DEINTERPOLATE = new HashSet<>(); + + @Override + public void uncaughtException(final Thread thread, final Throwable throwable) { + // This is a naive AirbyteTraceMessage emission in order to emit one when any error occurs in a + // connector. + // If a connector implements AirbyteTraceMessage emission itself, this code will result in an + // additional one being emitted. + // this is fine tho because: + // "The earliest AirbyteTraceMessage where type=error will be used to populate the FailureReason for + // the sync." + // from the spec: + // https://docs.google.com/document/d/1ctrj3Yh_GjtQ93aND-WH3ocqGxsmxyC3jfiarrF6NY0/edit# + LOGGER.error(logMessage, throwable); + + // Attempt to deinterpolate the error message before emitting a trace message + final String mangledMessage; + // If any exception in the chain is of a deinterpolatable type, find it and deinterpolate its + // message. + // This assumes that any wrapping exceptions are just noise (e.g. runtime exception). + final Optional deinterpolatableException = ExceptionUtils.getThrowableList(throwable).stream() + .filter(t -> THROWABLES_TO_DEINTERPOLATE.stream().anyMatch(deinterpolatableClass -> deinterpolatableClass.isAssignableFrom(t.getClass()))) + .findFirst(); + if (deinterpolatableException.isPresent()) { + mangledMessage = STRINGS_TO_DEINTERPOLATE.stream() + // Sort the strings longest to shortest, in case any target string is a substring of another + // e.g. "airbyte_internal" should be swapped out before "airbyte" + .sorted(Comparator.comparing(String::length).reversed()) + .reduce(deinterpolatableException.get().getMessage(), AirbyteExceptionHandler::deinterpolate); + } else { + mangledMessage = throwable.getMessage(); + } + + // If we did not modify the message (either not a deinterpolatable class, or we tried to + // deinterpolate + // but made no changes) then emit our default trace message + if (mangledMessage.equals(throwable.getMessage())) { + AirbyteTraceMessageUtility.emitSystemErrorTrace(throwable, logMessage); + } else { + AirbyteTraceMessageUtility.emitCustomErrorTrace(throwable.getMessage(), mangledMessage); + } + + terminate(); + } + + @NotNull + private static String deinterpolate(final String message, final String targetString) { + final String quotedTarget = '(' + Pattern.quote(targetString) + ')'; + final String targetRegex = REGEX_PREFIX + quotedTarget + REGEX_SUFFIX; + final Pattern pattern = Pattern.compile(targetRegex); + final Matcher matcher = pattern.matcher(message); + + // The pattern has three capturing groups: + // 1. The character before the target string (or an empty string, if it matched start-of-string) + // 2. The target string + // 3. The character after the target string (or empty string for end-of-string) + // We want to preserve the characters before and after the target string, so we use $1 and $3 to + // reinsert them + // but the target string is replaced with just '?' + return matcher.replaceAll("$1?$3"); + } + + public static void addThrowableForDeinterpolation(final Class klass) { + THROWABLES_TO_DEINTERPOLATE.add(klass); + } + + public static void addStringForDeinterpolation(final String string) { + if (string != null) { + STRINGS_TO_DEINTERPOLATE.add(string); + } + } + + public static void addAllStringsInConfigForDeinterpolation(final JsonNode node) { + if (node.isTextual()) { + addStringForDeinterpolation(node.asText()); + } else if (node.isContainerNode()) { + for (final JsonNode subNode : node) { + addAllStringsInConfigForDeinterpolation(subNode); + } + } + } + + // by doing this in a separate method we can mock it to avoid closing the jvm and therefore test + // properly + protected void terminate() { + System.exit(1); + } + + @VisibleForTesting + static void addCommonStringsToDeinterpolate() { + // Add some common strings to deinterpolate, regardless of what the connector is doing + STRINGS_TO_DEINTERPOLATE.add("airbyte"); + STRINGS_TO_DEINTERPOLATE.add("config"); + STRINGS_TO_DEINTERPOLATE.add("configuration"); + STRINGS_TO_DEINTERPOLATE.add("description"); + STRINGS_TO_DEINTERPOLATE.add("email"); + STRINGS_TO_DEINTERPOLATE.add("id"); + STRINGS_TO_DEINTERPOLATE.add("location"); + STRINGS_TO_DEINTERPOLATE.add("message"); + STRINGS_TO_DEINTERPOLATE.add("name"); + STRINGS_TO_DEINTERPOLATE.add("state"); + STRINGS_TO_DEINTERPOLATE.add("status"); + STRINGS_TO_DEINTERPOLATE.add("type"); + STRINGS_TO_DEINTERPOLATE.add("userEmail"); + } + +} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/AirbyteMessageConsumer.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteMessageConsumer.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/AirbyteMessageConsumer.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteMessageConsumer.java index 8322ff9ed0f2..85f045a0c215 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/AirbyteMessageConsumer.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteMessageConsumer.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; import io.airbyte.commons.concurrency.VoidCallable; import io.airbyte.commons.functional.CheckedConsumer; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/AirbyteTraceMessageUtility.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteTraceMessageUtility.java similarity index 89% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/AirbyteTraceMessageUtility.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteTraceMessageUtility.java index d7e1e524bd13..943f2cf8ba75 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/AirbyteTraceMessageUtility.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/AirbyteTraceMessageUtility.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; import io.airbyte.commons.stream.AirbyteStreamStatusHolder; import io.airbyte.protocol.models.v0.AirbyteErrorTraceMessage; @@ -26,6 +26,15 @@ public static void emitConfigErrorTrace(final Throwable e, final String displayM emitErrorTrace(e, displayMessage, FailureType.CONFIG_ERROR); } + public static void emitCustomErrorTrace(final String displayMessage, final String internalMessage) { + emitMessage(makeAirbyteMessageFromTraceMessage( + makeAirbyteTraceMessage(AirbyteTraceMessage.Type.ERROR) + .withError(new AirbyteErrorTraceMessage() + .withFailureType(FailureType.SYSTEM_ERROR) + .withMessage(displayMessage) + .withInternalMessage(internalMessage)))); + } + public static void emitEstimateTrace(final long byteEstimate, final AirbyteEstimateTraceMessage.Type type, final long rowEstimate, diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/Command.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/Command.java similarity index 76% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/Command.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/Command.java index e37502894bb8..6e5897db49cc 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/Command.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/Command.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; public enum Command { SPEC, diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/CommitOnStateAirbyteMessageConsumer.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/CommitOnStateAirbyteMessageConsumer.java similarity index 96% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/CommitOnStateAirbyteMessageConsumer.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/CommitOnStateAirbyteMessageConsumer.java index b7fded66b551..47a0c0b04813 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/CommitOnStateAirbyteMessageConsumer.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/CommitOnStateAirbyteMessageConsumer.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/Destination.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/Destination.java similarity index 99% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/Destination.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/Destination.java index d2e7cfc0b344..20a0c2464bb7 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/Destination.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/Destination.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyDescription; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/DestinationConfig.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/DestinationConfig.java similarity index 94% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/DestinationConfig.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/DestinationConfig.java index bd27cb2255f3..280658683f7b 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/DestinationConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/DestinationConfig.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; @@ -44,6 +44,10 @@ public static DestinationConfig getInstance() { return config; } + public static void clearInstance() { + config = null; + } + public JsonNode getNodeValue(final String key) { final JsonNode node = config.root.get(key); if (node == null) { diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/FailureTrackingAirbyteMessageConsumer.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/FailureTrackingAirbyteMessageConsumer.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/FailureTrackingAirbyteMessageConsumer.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/FailureTrackingAirbyteMessageConsumer.java index cc31dc4505fc..39b1d87e6689 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/FailureTrackingAirbyteMessageConsumer.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/FailureTrackingAirbyteMessageConsumer.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; import io.airbyte.protocol.models.v0.AirbyteMessage; import org.slf4j.Logger; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/Integration.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/Integration.java similarity index 95% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/Integration.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/Integration.java index 049cec7ccfaa..ae3b15eb8acf 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/Integration.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/Integration.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationCliParser.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/IntegrationCliParser.java similarity index 99% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationCliParser.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/IntegrationCliParser.java index 87e36d75db9d..12bfd9009c47 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationCliParser.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/IntegrationCliParser.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; import com.google.common.base.Preconditions; import io.airbyte.commons.cli.Clis; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationConfig.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/IntegrationConfig.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationConfig.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/IntegrationConfig.java index 438ecceb9f42..bbce8342419f 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/IntegrationConfig.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; import com.google.common.base.Preconditions; import java.nio.file.Path; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/IntegrationRunner.java similarity index 95% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/IntegrationRunner.java index f35a78407842..5887466c126d 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/IntegrationRunner.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/IntegrationRunner.java @@ -2,13 +2,16 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import datadog.trace.api.Trace; +import io.airbyte.cdk.integrations.util.ApmTraceUtils; +import io.airbyte.cdk.integrations.util.ConnectorExceptionUtil; +import io.airbyte.cdk.integrations.util.concurrent.ConcurrentStreamConsumer; import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.io.IOs; @@ -16,9 +19,6 @@ import io.airbyte.commons.stream.StreamStatusUtils; import io.airbyte.commons.string.Strings; import io.airbyte.commons.util.AutoCloseableIterator; -import io.airbyte.integrations.util.ApmTraceUtils; -import io.airbyte.integrations.util.ConnectorExceptionUtil; -import io.airbyte.integrations.util.concurrent.ConcurrentStreamConsumer; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; @@ -52,6 +52,8 @@ public class IntegrationRunner { private static final Logger LOGGER = LoggerFactory.getLogger(IntegrationRunner.class); + public static final String TYPE_AND_DEDUPE_THREAD_NAME = "type-and-dedupe"; + /** * Filters threads that should not be considered when looking for orphaned threads at shutdown of * the integration runner. @@ -63,7 +65,7 @@ public class IntegrationRunner { */ @VisibleForTesting static final Predicate ORPHANED_THREAD_FILTER = runningThread -> !runningThread.getName().equals(Thread.currentThread().getName()) - && !runningThread.isDaemon(); + && !runningThread.isDaemon() && !TYPE_AND_DEDUPE_THREAD_NAME.equals(runningThread.getName()); public static final int INTERRUPT_THREAD_DELAY_MINUTES = 60; public static final int EXIT_THREAD_DELAY_MINUTES = 70; @@ -137,6 +139,9 @@ private void runInternal(final IntegrationConfig parsed) throws Exception { case SPEC -> outputRecordCollector.accept(new AirbyteMessage().withType(Type.SPEC).withSpec(integration.spec())); case CHECK -> { final JsonNode config = parseConfig(parsed.getConfigPath()); + if (integration instanceof Destination) { + DestinationConfig.initialize(config); + } try { validateConfig(integration.spec().getConnectionSpecification(), config, "CHECK"); } catch (final Exception e) { @@ -232,7 +237,8 @@ private void produceMessages(final AutoCloseableIterator message messageIterator.getAirbyteStream().ifPresent(s -> LOGGER.debug("Finished producing messages for stream {}...")); } - private void readConcurrent(final JsonNode config, ConfiguredAirbyteCatalog catalog, final Optional stateOptional) throws Exception { + private void readConcurrent(final JsonNode config, final ConfiguredAirbyteCatalog catalog, final Optional stateOptional) + throws Exception { final Collection> streams = source.readStreams(config, catalog, stateOptional.orElse(null)); try (final ConcurrentStreamConsumer streamConsumer = new ConcurrentStreamConsumer(this::consumeFromStream, streams.size())) { @@ -265,7 +271,7 @@ private void readConcurrent(final JsonNode config, ConfiguredAirbyteCatalog cata } } - private void readSerial(final JsonNode config, ConfiguredAirbyteCatalog catalog, final Optional stateOptional) throws Exception { + private void readSerial(final JsonNode config, final ConfiguredAirbyteCatalog catalog, final Optional stateOptional) throws Exception { try (final AutoCloseableIterator messageIterator = source.read(config, catalog, stateOptional.orElse(null))) { produceMessages(messageIterator, outputRecordCollector); } finally { diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/JavaBaseConstants.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/JavaBaseConstants.java similarity index 97% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/JavaBaseConstants.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/JavaBaseConstants.java index 0c5672becbda..5001d6119e7a 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/JavaBaseConstants.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/JavaBaseConstants.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; import java.util.List; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/SerializedAirbyteMessageConsumer.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/SerializedAirbyteMessageConsumer.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/SerializedAirbyteMessageConsumer.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/SerializedAirbyteMessageConsumer.java index 69b866252328..60eccd4e449e 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/SerializedAirbyteMessageConsumer.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/SerializedAirbyteMessageConsumer.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; import io.airbyte.commons.concurrency.VoidCallable; import io.airbyte.commons.functional.CheckedBiConsumer; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/Source.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/Source.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/Source.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/Source.java index 424bd780e5b3..1f092a3b16dd 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/Source.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/Source.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.util.AutoCloseableIterator; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/TypingAndDedupingFlag.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/TypingAndDedupingFlag.java similarity index 81% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/TypingAndDedupingFlag.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/TypingAndDedupingFlag.java index aea71ee4006d..b59e757b62c4 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/TypingAndDedupingFlag.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/TypingAndDedupingFlag.java @@ -2,10 +2,9 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; import java.util.Optional; -import org.elasticsearch.common.Strings; public class TypingAndDedupingFlag { @@ -15,7 +14,7 @@ public static boolean isDestinationV2() { public static Optional getRawNamespaceOverride(String option) { String rawOverride = DestinationConfig.getInstance().getTextValue(option); - if (Strings.isEmpty(rawOverride)) { + if (rawOverride == null || rawOverride.isEmpty()) { return Optional.empty(); } else { return Optional.of(rawOverride); diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/adaptive/AdaptiveDestinationRunner.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/adaptive/AdaptiveDestinationRunner.java similarity index 86% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/adaptive/AdaptiveDestinationRunner.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/adaptive/AdaptiveDestinationRunner.java index 72528ce06ce4..878eef089be0 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/adaptive/AdaptiveDestinationRunner.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/adaptive/AdaptiveDestinationRunner.java @@ -2,15 +2,16 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base.adaptive; - +package io.airbyte.cdk.integrations.base.adaptive; + +import io.airbyte.cdk.integrations.base.Command; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.DestinationConfig; +import io.airbyte.cdk.integrations.base.IntegrationCliParser; +import io.airbyte.cdk.integrations.base.IntegrationConfig; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.Command; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.DestinationConfig; -import io.airbyte.integrations.base.IntegrationCliParser; -import io.airbyte.integrations.base.IntegrationConfig; -import io.airbyte.integrations.base.IntegrationRunner; import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,7 +24,7 @@ public class AdaptiveDestinationRunner { private static final Logger LOGGER = LoggerFactory.getLogger(AdaptiveDestinationRunner.class); - private static final String DEPLOYMENT_MODE_KEY = "DEPLOYMENT_MODE"; + private static final String DEPLOYMENT_MODE_KEY = EnvVariableFeatureFlags.DEPLOYMENT_MODE; private static final String CLOUD_MODE = "CLOUD"; public static OssDestinationBuilder baseOnEnv() { diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/adaptive/AdaptiveSourceRunner.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/adaptive/AdaptiveSourceRunner.java similarity index 86% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/adaptive/AdaptiveSourceRunner.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/adaptive/AdaptiveSourceRunner.java index 82f8026791ba..4bb7f021db50 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/adaptive/AdaptiveSourceRunner.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/adaptive/AdaptiveSourceRunner.java @@ -2,10 +2,11 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base.adaptive; +package io.airbyte.cdk.integrations.base.adaptive; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.commons.features.EnvVariableFeatureFlags; import java.util.function.Supplier; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -17,8 +18,8 @@ public class AdaptiveSourceRunner { private static final Logger LOGGER = LoggerFactory.getLogger(AdaptiveSourceRunner.class); - private static final String DEPLOYMENT_MODE_KEY = "DEPLOYMENT_MODE"; - private static final String COULD_MODE = "CLOUD"; + public static final String DEPLOYMENT_MODE_KEY = EnvVariableFeatureFlags.DEPLOYMENT_MODE; + public static final String CLOUD_MODE = "CLOUD"; public static OssSourceBuilder baseOnEnv() { final String mode = System.getenv(DEPLOYMENT_MODE_KEY); @@ -71,7 +72,7 @@ public Runner(final String deploymentMode, private Source getSource() { LOGGER.info("Running source under deployment mode: {}", deploymentMode); - if (deploymentMode != null && deploymentMode.equals(COULD_MODE)) { + if (deploymentMode != null && deploymentMode.equals(CLOUD_MODE)) { return cloudSourceSupplier.get(); } if (deploymentMode == null) { diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/errors/messages/ErrorMessage.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/errors/messages/ErrorMessage.java similarity index 94% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/errors/messages/ErrorMessage.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/errors/messages/ErrorMessage.java index 82c643035d25..43c19ad6281e 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/errors/messages/ErrorMessage.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/errors/messages/ErrorMessage.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base.errors.messages; +package io.airbyte.cdk.integrations.base.errors.messages; import java.util.Objects; diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/spec_modification/SpecModifyingDestination.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/spec_modification/SpecModifyingDestination.java new file mode 100644 index 000000000000..209c98fdf0ae --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/spec_modification/SpecModifyingDestination.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.base.spec_modification; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConnectorSpecification; +import java.util.function.Consumer; + +public abstract class SpecModifyingDestination implements Destination { + + private final Destination destination; + + public SpecModifyingDestination(final Destination destination) { + this.destination = destination; + } + + public abstract ConnectorSpecification modifySpec(ConnectorSpecification originalSpec) throws Exception; + + @Override + public ConnectorSpecification spec() throws Exception { + return modifySpec(destination.spec()); + } + + @Override + public AirbyteConnectionStatus check(final JsonNode config) throws Exception { + return destination.check(config); + } + + @Override + public AirbyteMessageConsumer getConsumer(final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final Consumer outputRecordCollector) + throws Exception { + return destination.getConsumer(config, catalog, outputRecordCollector); + } + + @Override + public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final Consumer outputRecordCollector) + throws Exception { + return destination.getSerializedMessageConsumer(config, catalog, outputRecordCollector); + } + +} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/spec_modification/SpecModifyingSource.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/spec_modification/SpecModifyingSource.java similarity index 94% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/spec_modification/SpecModifyingSource.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/spec_modification/SpecModifyingSource.java index a0e26a5bcc9c..aa7f1b2b2a4e 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/spec_modification/SpecModifyingSource.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/spec_modification/SpecModifyingSource.java @@ -2,11 +2,11 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base.spec_modification; +package io.airbyte.cdk.integrations.base.spec_modification; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.base.Source; import io.airbyte.commons.util.AutoCloseableIterator; -import io.airbyte.integrations.base.Source; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshBastionContainer.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshBastionContainer.java new file mode 100644 index 000000000000..07a1786f60dd --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshBastionContainer.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.base.ssh; + +import static io.airbyte.cdk.integrations.base.ssh.SshHelpers.getInnerContainerAddress; +import static io.airbyte.cdk.integrations.base.ssh.SshHelpers.getOuterContainerAddress; +import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.TunnelMethod.SSH_KEY_AUTH; +import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.util.HostPortResolver; +import io.airbyte.commons.json.Jsons; +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.JdbcDatabaseContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.images.builder.ImageFromDockerfile; + +public class SshBastionContainer implements AutoCloseable { + + private static final String SSH_USER = "sshuser"; + private static final String SSH_PASSWORD = "secret"; + private GenericContainer bastion; + + public void initAndStartBastion(final Network network) { + bastion = new GenericContainer( + new ImageFromDockerfile("bastion-test") + .withFileFromClasspath("Dockerfile", "bastion/Dockerfile")) + .withNetwork(network) + .withExposedPorts(22); + bastion.start(); + } + + public JsonNode getTunnelMethod(final SshTunnel.TunnelMethod tunnelMethod, + final boolean innerAddress) + throws IOException, InterruptedException { + final var containerAddress = innerAddress ? getInnerContainerAddress(bastion) : getOuterContainerAddress(bastion); + return Jsons.jsonNode(ImmutableMap.builder() + .put("tunnel_host", + Objects.requireNonNull(containerAddress.left)) + .put("tunnel_method", tunnelMethod) + .put("tunnel_port", containerAddress.right) + .put("tunnel_user", SSH_USER) + .put("tunnel_user_password", tunnelMethod.equals(SSH_PASSWORD_AUTH) ? SSH_PASSWORD : "") + .put("ssh_key", tunnelMethod.equals(SSH_KEY_AUTH) ? bastion.execInContainer("cat", "var/bastion/id_rsa").getStdout() : "") + .build()); + } + + public JsonNode getTunnelConfig(final SshTunnel.TunnelMethod tunnelMethod, + final ImmutableMap.Builder builderWithSchema, + final boolean innerAddress) + throws IOException, InterruptedException { + return Jsons.jsonNode(builderWithSchema + .put("tunnel_method", getTunnelMethod(tunnelMethod, innerAddress)) + .build()); + } + + public ImmutableMap.Builder getBasicDbConfigBuider(final JdbcDatabaseContainer db) { + return getBasicDbConfigBuider(db, db.getDatabaseName()); + } + + public ImmutableMap.Builder getBasicDbConfigBuider(final JdbcDatabaseContainer db, final List schemas) { + return getBasicDbConfigBuider(db, db.getDatabaseName()).put("schemas", schemas); + } + + public ImmutableMap.Builder getBasicDbConfigBuider(final JdbcDatabaseContainer db, final String schemaName) { + return ImmutableMap.builder() + .put("host", Objects.requireNonNull(HostPortResolver.resolveHost(db))) + .put("username", db.getUsername()) + .put("password", db.getPassword()) + .put("port", HostPortResolver.resolvePort(db)) + .put("database", schemaName) + .put("ssl", false); + } + + public void stopAndCloseContainers(final JdbcDatabaseContainer db) { + bastion.stop(); + bastion.close(); + db.stop(); + db.close(); + } + + public void stopAndClose() { + bastion.close(); + } + + @Override + public void close() { + stopAndClose(); + } + + public GenericContainer getContainer() { + return bastion; + } + +} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshHelpers.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshHelpers.java similarity index 96% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshHelpers.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshHelpers.java index e1446c34c0a1..367d429bdc2a 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshHelpers.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshHelpers.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base.ssh; +package io.airbyte.cdk.integrations.base.ssh; import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.commons.json.Jsons; @@ -61,7 +61,8 @@ public static ImmutablePair getInnerContainerAddress(final Cont * @return a pair of host and port */ public static ImmutablePair getOuterContainerAddress(final Container container) { - return ImmutablePair.of(container.getHost(), + return ImmutablePair.of( + container.getHost(), container.getFirstMappedPort()); } diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshTunnel.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshTunnel.java similarity index 76% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshTunnel.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshTunnel.java index 77e4937df8c5..42dbc0007df4 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshTunnel.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshTunnel.java @@ -2,16 +2,16 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base.ssh; +package io.airbyte.cdk.integrations.base.ssh; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Preconditions; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.functional.CheckedConsumer; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.integrations.base.AirbyteTraceMessageUtility; import java.io.IOException; import java.io.StringReader; import java.net.InetSocketAddress; @@ -23,14 +23,17 @@ import java.util.Arrays; import java.util.List; import java.util.Locale; +import java.util.Optional; import org.apache.sshd.client.SshClient; import org.apache.sshd.client.keyverifier.AcceptAllServerKeyVerifier; import org.apache.sshd.client.session.ClientSession; import org.apache.sshd.common.SshException; +import org.apache.sshd.common.session.SessionHeartbeatController; import org.apache.sshd.common.util.net.SshdSocketAddress; import org.apache.sshd.common.util.security.SecurityUtils; import org.apache.sshd.core.CoreModuleProperties; import org.apache.sshd.server.forward.AcceptAllForwardingFilter; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,12 +49,24 @@ public class SshTunnel implements AutoCloseable { public static final String SSH_TIMEOUT_DISPLAY_MESSAGE = "Timed out while opening a SSH Tunnel. Please double check the given SSH configurations and try again."; + public static final String CONNECTION_OPTIONS_KEY = "ssh_connection_options"; + public static final String SESSION_HEARTBEAT_INTERVAL_KEY = "session_heartbeat_interval"; + public static final long SESSION_HEARTBEAT_INTERVAL_DEFAULT_IN_MILLIS = 1000; + public static final String GLOBAL_HEARTBEAT_INTERVAL_KEY = "global_heartbeat_interval"; + public static final long GLOBAL_HEARTBEAT_INTERVAL_DEFAULT_IN_MILLIS = 2000; + public static final String IDLE_TIMEOUT_KEY = "idle_timeout"; + public static final long IDLE_TIMEOUT_DEFAULT_INFINITE = 0; + public enum TunnelMethod { NO_TUNNEL, SSH_PASSWORD_AUTH, SSH_KEY_AUTH } + public record SshConnectionOptions(Duration sessionHeartbeatInterval, + Duration globalHeartbeatInterval, + Duration idleTimeout) {} + public static final int TIMEOUT_MILLIS = 15000; // 15 seconds private final JsonNode config; @@ -99,6 +114,7 @@ public enum TunnelMethod { * tunnel host). * @param remoteServicePort - the actual port of the remote service (as it is known to the tunnel * host). + * @param connectionOptions - optional connection options for ssh client. */ public SshTunnel(final JsonNode config, final List hostKey, @@ -112,7 +128,8 @@ public SshTunnel(final JsonNode config, final String sshKey, final String tunnelUserPassword, final String remoteServiceHost, - final int remoteServicePort) { + final int remoteServicePort, + final Optional connectionOptions) { this.config = config; this.hostKey = hostKey; this.portKey = portKey; @@ -168,11 +185,42 @@ public SshTunnel(final JsonNode config, this.tunnelUser = tunnelUser; this.sshKey = sshKey; this.tunnelUserPassword = tunnelUserPassword; - this.sshclient = createClient(); + this.sshclient = connectionOptions.map(sshConnectionOptions -> createClient(sshConnectionOptions.sessionHeartbeatInterval(), + sshConnectionOptions.globalHeartbeatInterval(), + sshConnectionOptions.idleTimeout())).orElseGet(this::createClient); this.tunnelSession = openTunnel(sshclient); } } + public SshTunnel(final JsonNode config, + final List hostKey, + final List portKey, + final String endPointKey, + final String remoteServiceUrl, + final TunnelMethod tunnelMethod, + final String tunnelHost, + final int tunnelPort, + final String tunnelUser, + final String sshKey, + final String tunnelUserPassword, + final String remoteServiceHost, + final int remoteServicePort) { + this(config, + hostKey, + portKey, + endPointKey, + remoteServiceUrl, + tunnelMethod, + tunnelHost, + tunnelPort, + tunnelUser, + sshKey, + tunnelUserPassword, + remoteServiceHost, + remoteServicePort, + Optional.empty()); + } + public JsonNode getOriginalConfig() { return config; } @@ -216,7 +264,32 @@ public static SshTunnel getInstance(final JsonNode config, final List ho Strings.safeTrim(Jsons.getStringOrNull(config, "tunnel_method", "ssh_key")), Strings.safeTrim(Jsons.getStringOrNull(config, "tunnel_method", "tunnel_user_password")), Strings.safeTrim(Jsons.getStringOrNull(config, hostKey)), - Jsons.getIntOrZero(config, portKey)); + Jsons.getIntOrZero(config, portKey), + getSshConnectionOptions(config)); + } + + @NotNull + private static Optional getSshConnectionOptions(JsonNode config) { + // piggybacking on JsonNode config to make it configurable at connector level. + Optional connectionOptionConfig = Jsons.getOptional(config, CONNECTION_OPTIONS_KEY); + final Optional connectionOptions; + if (connectionOptionConfig.isPresent()) { + JsonNode connectionOptionsNode = connectionOptionConfig.get(); + Duration sessionHeartbeatInterval = Jsons.getOptional(connectionOptionsNode, SESSION_HEARTBEAT_INTERVAL_KEY) + .map(interval -> Duration.ofMillis(interval.asLong())) + .orElse(Duration.ofSeconds(1)); + Duration globalHeartbeatInterval = Jsons.getOptional(connectionOptionsNode, GLOBAL_HEARTBEAT_INTERVAL_KEY) + .map(interval -> Duration.ofMillis(interval.asLong())) + .orElse(Duration.ofSeconds(2)); + Duration idleTimeout = Jsons.getOptional(connectionOptionsNode, IDLE_TIMEOUT_KEY) + .map(interval -> Duration.ofMillis(interval.asLong())) + .orElse(Duration.ZERO); + connectionOptions = Optional.of( + new SshConnectionOptions(sessionHeartbeatInterval, globalHeartbeatInterval, idleTimeout)); + } else { + connectionOptions = Optional.empty(); + } + return connectionOptions; } public static SshTunnel getInstance(final JsonNode config, final String endPointKey) throws Exception { @@ -237,7 +310,8 @@ public static SshTunnel getInstance(final JsonNode config, final String endPoint Strings.safeTrim(Jsons.getStringOrNull(config, "tunnel_method", "tunnel_user")), Strings.safeTrim(Jsons.getStringOrNull(config, "tunnel_method", "ssh_key")), Strings.safeTrim(Jsons.getStringOrNull(config, "tunnel_method", "tunnel_user_password")), - null, 0); + null, 0, + getSshConnectionOptions(config)); } public static void sshWrap(final JsonNode config, @@ -332,7 +406,22 @@ private SshClient createClient() { final SshClient client = SshClient.setUpDefaultClient(); client.setForwardingFilter(AcceptAllForwardingFilter.INSTANCE); client.setServerKeyVerifier(AcceptAllServerKeyVerifier.INSTANCE); - CoreModuleProperties.IDLE_TIMEOUT.set(client, Duration.ZERO); + return client; + } + + private SshClient createClient(Duration sessionHeartbeatInterval, Duration globalHeartbeatInterval, Duration idleTimeout) { + LOGGER.info("Creating SSH client with Heartbeat and Keepalive enabled"); + final SshClient client = createClient(); + // Session level heartbeat using SSH_MSG_IGNORE every second. + client.setSessionHeartbeat(SessionHeartbeatController.HeartbeatType.IGNORE, sessionHeartbeatInterval); + // idle-timeout zero indicates NoTimeout. + CoreModuleProperties.IDLE_TIMEOUT.set(client, idleTimeout); + // Use tcp keep-alive mechanism. + CoreModuleProperties.SOCKET_KEEPALIVE.set(client, true); + // Additional delay used for ChannelOutputStream to wait for space in the remote socket send buffer. + CoreModuleProperties.WAIT_FOR_SPACE_TIMEOUT.set(client, Duration.ofMinutes(2)); + // Global keepalive message sent every 2 seconds. This precedes the session level heartbeat. + CoreModuleProperties.HEARTBEAT_INTERVAL.set(client, globalHeartbeatInterval); return client; } diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedDestination.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshWrappedDestination.java similarity index 75% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedDestination.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshWrappedDestination.java index 0f8d6a650373..08f4d794d92b 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedDestination.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshWrappedDestination.java @@ -2,22 +2,25 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base.ssh; +package io.airbyte.cdk.integrations.base.ssh; + +import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.*; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.AirbyteTraceMessageUtility; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.SerializedAirbyteMessageConsumer; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConnectorSpecification; import java.util.List; +import java.util.Optional; import java.util.function.Consumer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,8 +67,8 @@ public ConnectorSpecification spec() throws Exception { @Override public AirbyteConnectionStatus check(final JsonNode config) throws Exception { try { - return (endPointKey != null) ? SshTunnel.sshWrap(config, endPointKey, delegate::check) - : SshTunnel.sshWrap(config, hostKey, portKey, delegate::check); + return (endPointKey != null) ? sshWrap(config, endPointKey, delegate::check) + : sshWrap(config, hostKey, portKey, delegate::check); } catch (final RuntimeException e) { final String sshErrorMessage = "Could not connect with provided SSH configuration. Error: " + e.getMessage(); AirbyteTraceMessageUtility.emitConfigErrorTrace(e, sshErrorMessage); @@ -98,7 +101,17 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN final ConfiguredAirbyteCatalog catalog, final Consumer outputRecordCollector) throws Exception { - final SshTunnel tunnel = getTunnelInstance(config); + final JsonNode clone = Jsons.clone(config); + Optional connectionOptionsConfig = Jsons.getOptional(clone, CONNECTION_OPTIONS_KEY); + if (connectionOptionsConfig.isEmpty()) { + LOGGER.info("No SSH connection options found, using defaults"); + if (clone instanceof ObjectNode) { // Defensive check, it will always be object node + ObjectNode connectionOptions = ((ObjectNode) clone).putObject(CONNECTION_OPTIONS_KEY); + connectionOptions.put(SESSION_HEARTBEAT_INTERVAL_KEY, SESSION_HEARTBEAT_INTERVAL_DEFAULT_IN_MILLIS); + connectionOptions.put(GLOBAL_HEARTBEAT_INTERVAL_KEY, GLOBAL_HEARTBEAT_INTERVAL_DEFAULT_IN_MILLIS); + } + } + final SshTunnel tunnel = getTunnelInstance(clone); final SerializedAirbyteMessageConsumer delegateConsumer; try { delegateConsumer = delegate.getSerializedMessageConsumer(tunnel.getConfigInTunnel(), catalog, outputRecordCollector); @@ -112,8 +125,8 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN protected SshTunnel getTunnelInstance(final JsonNode config) throws Exception { return (endPointKey != null) - ? SshTunnel.getInstance(config, endPointKey) - : SshTunnel.getInstance(config, hostKey, portKey); + ? getInstance(config, endPointKey) + : getInstance(config, hostKey, portKey); } } diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedSource.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshWrappedSource.java similarity index 95% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedSource.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshWrappedSource.java index 08971e9ec768..7abc65d277f7 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshWrappedSource.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/SshWrappedSource.java @@ -2,13 +2,13 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base.ssh; +package io.airbyte.cdk.integrations.base.ssh; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.cdk.integrations.base.Source; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.integrations.base.AirbyteTraceMessageUtility; -import io.airbyte.integrations.base.Source; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/readme.md b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/readme.md similarity index 100% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/readme.md rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/base/ssh/readme.md diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/NamingConventionTransformer.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/NamingConventionTransformer.java similarity index 91% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/NamingConventionTransformer.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/NamingConventionTransformer.java index 89c5d7f64dfb..7fadf5c3c8be 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/NamingConventionTransformer.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/NamingConventionTransformer.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination; +package io.airbyte.cdk.integrations.destination; /** * Destination have their own Naming conventions (which characters are valid or rejected in @@ -51,6 +51,10 @@ public interface NamingConventionTransformer { @Deprecated String getTmpTableName(String name); + default String getTmpTableName(final String streamName, final String randomSuffix) { + return getTmpTableName(streamName); + } + String convertStreamName(final String input); String applyDefaultCase(final String input); diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/StandardNameTransformer.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StandardNameTransformer.java similarity index 92% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/StandardNameTransformer.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StandardNameTransformer.java index a2f0b2d0bab6..a0bb39cc5d25 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/StandardNameTransformer.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StandardNameTransformer.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination; +package io.airbyte.cdk.integrations.destination; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; @@ -40,6 +40,11 @@ public String getTmpTableName(final String streamName) { return convertStreamName(Strings.addRandomSuffix("_airbyte_tmp", "_", 3) + "_" + streamName); } + @Override + public String getTmpTableName(final String streamName, final String randomSuffix) { + return convertStreamName("_airbyte_tmp" + "_" + randomSuffix + "_" + streamName); + } + @Override public String convertStreamName(final String input) { return Names.toAlphanumericAndUnderscore(input); diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StreamSyncSummary.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StreamSyncSummary.java new file mode 100644 index 000000000000..d4a76c862ac7 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/StreamSyncSummary.java @@ -0,0 +1,20 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination; + +import java.util.Optional; + +/** + * @param recordsWritten The number of records written to the stream, or empty if the caller does + * not track this information. (this is primarily for backwards-compatibility with the legacy + * destinations framework; new implementations should always provide this information). If + * this value is empty, consumers should assume that the sync wrote nonzero records for this + * stream. + */ +public record StreamSyncSummary(Optional recordsWritten) { + + public static final StreamSyncSummary DEFAULT = new StreamSyncSummary(Optional.empty()); + +} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/BufferedStreamConsumer.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/BufferedStreamConsumer.java similarity index 94% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/BufferedStreamConsumer.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/BufferedStreamConsumer.java index 8c1da914938f..b4cdd9bd73ee 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/BufferedStreamConsumer.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/BufferedStreamConsumer.java @@ -2,20 +2,20 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.buffered_stream_consumer; +package io.airbyte.cdk.integrations.destination.buffered_stream_consumer; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.destination.dest_state_lifecycle_manager.DefaultDestStateLifecycleManager; +import io.airbyte.cdk.integrations.destination.dest_state_lifecycle_manager.DestStateLifecycleManager; +import io.airbyte.cdk.integrations.destination.record_buffer.BufferFlushType; +import io.airbyte.cdk.integrations.destination.record_buffer.BufferingStrategy; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; -import io.airbyte.integrations.destination.dest_state_lifecycle_manager.DefaultDestStateLifecycleManager; -import io.airbyte.integrations.destination.dest_state_lifecycle_manager.DestStateLifecycleManager; -import io.airbyte.integrations.destination.record_buffer.BufferFlushType; -import io.airbyte.integrations.destination.record_buffer.BufferingStrategy; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; @@ -310,14 +310,16 @@ protected void close(final boolean hasFailed) throws Exception { * not bother committing. otherwise attempt to commit */ if (stateManager.listFlushed().isEmpty()) { - onClose.accept(hasFailed); + // Not updating this class to track record count, because we want to kill it in favor of the + // AsyncStreamConsumer + onClose.accept(hasFailed, new HashMap<>()); } else { /* * if any state message was flushed that means we should try to commit what we have. if * hasFailed=false, then it could be full success. if hasFailed=true, then going for partial * success. */ - onClose.accept(false); + onClose.accept(false, null); } stateManager.listCommitted().forEach(outputRecordCollector); diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/CheckAndRemoveRecordWriter.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/CheckAndRemoveRecordWriter.java similarity index 88% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/CheckAndRemoveRecordWriter.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/CheckAndRemoveRecordWriter.java index 55ed3c1a9ca3..4a48ef00a1e6 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/CheckAndRemoveRecordWriter.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/CheckAndRemoveRecordWriter.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.buffered_stream_consumer; +package io.airbyte.cdk.integrations.destination.buffered_stream_consumer; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/OnCloseFunction.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/OnCloseFunction.java new file mode 100644 index 000000000000..39c4da662a88 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/OnCloseFunction.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.buffered_stream_consumer; + +import io.airbyte.cdk.integrations.destination.StreamSyncSummary; +import io.airbyte.commons.functional.CheckedBiConsumer; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.Map; + +/** + * Interface allowing destination to specify clean up logic that must be executed after all + * record-related logic has finished. + *

+ * The map of StreamSyncSummaries MUST be non-null, but MAY be empty. Streams not present in the map + * will be treated as equivalent to {@link StreamSyncSummary#DEFAULT}. + */ +public interface OnCloseFunction extends CheckedBiConsumer, Exception> { + +} diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/OnStartFunction.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/OnStartFunction.java new file mode 100644 index 000000000000..e13b95dcda68 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/OnStartFunction.java @@ -0,0 +1,11 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.buffered_stream_consumer; + +import io.airbyte.commons.concurrency.VoidCallable; + +public interface OnStartFunction extends VoidCallable { + +} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/RecordSizeEstimator.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/RecordSizeEstimator.java similarity index 97% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/RecordSizeEstimator.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/RecordSizeEstimator.java index b27cb1860386..5c7e24423876 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/RecordSizeEstimator.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/RecordSizeEstimator.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.buffered_stream_consumer; +package io.airbyte.cdk.integrations.destination.buffered_stream_consumer; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/RecordWriter.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/RecordWriter.java similarity index 85% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/RecordWriter.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/RecordWriter.java index fb5641d6af6b..e9ed10871822 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/RecordWriter.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/RecordWriter.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.buffered_stream_consumer; +package io.airbyte.cdk.integrations.destination.buffered_stream_consumer; import io.airbyte.commons.functional.CheckedBiConsumer; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/StreamDateFormatter.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/StreamDateFormatter.java similarity index 79% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/StreamDateFormatter.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/StreamDateFormatter.java index c582bd05a9ea..ad486abb991b 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/StreamDateFormatter.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/StreamDateFormatter.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.buffered_stream_consumer; +package io.airbyte.cdk.integrations.destination.buffered_stream_consumer; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DefaultDestStateLifecycleManager.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/dest_state_lifecycle_manager/DefaultDestStateLifecycleManager.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DefaultDestStateLifecycleManager.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/dest_state_lifecycle_manager/DefaultDestStateLifecycleManager.java index cbde8d7a497c..23b2ac33495b 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DefaultDestStateLifecycleManager.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/dest_state_lifecycle_manager/DefaultDestStateLifecycleManager.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.dest_state_lifecycle_manager; +package io.airbyte.cdk.integrations.destination.dest_state_lifecycle_manager; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestSingleStateLifecycleManager.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/dest_state_lifecycle_manager/DestSingleStateLifecycleManager.java similarity index 97% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestSingleStateLifecycleManager.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/dest_state_lifecycle_manager/DestSingleStateLifecycleManager.java index acc64d35a88a..39158a3d3144 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestSingleStateLifecycleManager.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/dest_state_lifecycle_manager/DestSingleStateLifecycleManager.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.dest_state_lifecycle_manager; +package io.airbyte.cdk.integrations.destination.dest_state_lifecycle_manager; import com.google.common.annotations.VisibleForTesting; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestStateLifecycleManager.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/dest_state_lifecycle_manager/DestStateLifecycleManager.java similarity index 97% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestStateLifecycleManager.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/dest_state_lifecycle_manager/DestStateLifecycleManager.java index 503c790f6a43..183b84a1ccf0 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestStateLifecycleManager.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/dest_state_lifecycle_manager/DestStateLifecycleManager.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.dest_state_lifecycle_manager; +package io.airbyte.cdk.integrations.destination.dest_state_lifecycle_manager; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestStreamStateLifecycleManager.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/dest_state_lifecycle_manager/DestStreamStateLifecycleManager.java similarity index 97% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestStreamStateLifecycleManager.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/dest_state_lifecycle_manager/DestStreamStateLifecycleManager.java index 797d5baa6833..68f93d5f44f8 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestStreamStateLifecycleManager.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/dest_state_lifecycle_manager/DestStreamStateLifecycleManager.java @@ -2,9 +2,8 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.dest_state_lifecycle_manager; +package io.airbyte.cdk.integrations.destination.dest_state_lifecycle_manager; -import com.amazonaws.util.StringUtils; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import io.airbyte.protocol.models.v0.AirbyteMessage; @@ -50,7 +49,8 @@ public void addState(final AirbyteMessage message) { Preconditions.checkArgument(message.getState().getType() == AirbyteStateType.STREAM); final StreamDescriptor originalStreamId = message.getState().getStream().getStreamDescriptor(); final StreamDescriptor actualStreamId; - if (StringUtils.isNullOrEmpty(originalStreamId.getNamespace())) { + final String namespace = originalStreamId.getNamespace(); + if (namespace == null || namespace.isEmpty()) { // If the state's namespace is null/empty, we need to be able to find it using the default namespace // (because many destinations actually set records' namespace to the default namespace before // they make it into this class). diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/normalization/NormalizationLogParser.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/normalization/NormalizationLogParser.java similarity index 97% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/normalization/NormalizationLogParser.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/normalization/NormalizationLogParser.java index 73a1411b72ec..698a9b269f22 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/normalization/NormalizationLogParser.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/normalization/NormalizationLogParser.java @@ -2,12 +2,12 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.normalization; +package io.airbyte.cdk.integrations.destination.normalization; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; +import io.airbyte.cdk.integrations.destination.normalization.SentryExceptionHelper.ErrorMapKeys; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.normalization.SentryExceptionHelper.ErrorMapKeys; import io.airbyte.protocol.models.AirbyteErrorTraceMessage; import io.airbyte.protocol.models.AirbyteErrorTraceMessage.FailureType; import io.airbyte.protocol.models.AirbyteLogMessage; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/normalization/SentryExceptionHelper.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/normalization/SentryExceptionHelper.java similarity index 99% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/normalization/SentryExceptionHelper.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/normalization/SentryExceptionHelper.java index 3f604e568e1c..9c875bf011c5 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/normalization/SentryExceptionHelper.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/normalization/SentryExceptionHelper.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.normalization; +package io.airbyte.cdk.integrations.destination.normalization; import java.util.Arrays; import java.util.HashMap; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/BaseSerializedBuffer.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/BaseSerializedBuffer.java similarity index 99% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/BaseSerializedBuffer.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/BaseSerializedBuffer.java index 39b2925f9ee1..38f951030951 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/BaseSerializedBuffer.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/BaseSerializedBuffer.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.record_buffer; +package io.airbyte.cdk.integrations.destination.record_buffer; import com.google.common.io.CountingOutputStream; import io.airbyte.commons.json.Jsons; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/BufferCreateFunction.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/BufferCreateFunction.java similarity index 89% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/BufferCreateFunction.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/BufferCreateFunction.java index bda03460ff0b..27dd99307e86 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/BufferCreateFunction.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/BufferCreateFunction.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.record_buffer; +package io.airbyte.cdk.integrations.destination.record_buffer; import io.airbyte.commons.functional.CheckedBiFunction; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/BufferFlushType.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/BufferFlushType.java new file mode 100644 index 000000000000..05fcd08f1a95 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/BufferFlushType.java @@ -0,0 +1,10 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.record_buffer; + +public enum BufferFlushType { + FLUSH_ALL, + FLUSH_SINGLE_STREAM +} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/BufferStorage.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/BufferStorage.java similarity index 97% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/BufferStorage.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/BufferStorage.java index c77329cf41f4..4deab7d9c364 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/BufferStorage.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/BufferStorage.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.record_buffer; +package io.airbyte.cdk.integrations.destination.record_buffer; import java.io.File; import java.io.IOException; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/BufferingStrategy.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/BufferingStrategy.java similarity index 90% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/BufferingStrategy.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/BufferingStrategy.java index 0763e6f5add1..1f640d2f8ceb 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/BufferingStrategy.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/BufferingStrategy.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.record_buffer; +package io.airbyte.cdk.integrations.destination.record_buffer; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; @@ -10,7 +10,7 @@ /** * High-level interface used by - * {@link io.airbyte.integrations.destination.buffered_stream_consumer.BufferedStreamConsumer} + * {@link io.airbyte.cdk.integrations.destination.buffered_stream_consumer.BufferedStreamConsumer} * * A Record buffering strategy relies on the capacity available of underlying * {@link SerializableBuffer} to determine what to do when consuming a new {@link AirbyteMessage} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/FileBuffer.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/FileBuffer.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/FileBuffer.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/FileBuffer.java index 029877629bef..d26b5bb09814 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/FileBuffer.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/FileBuffer.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.record_buffer; +package io.airbyte.cdk.integrations.destination.record_buffer; import java.io.BufferedOutputStream; import java.io.File; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/FlushBufferFunction.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/FlushBufferFunction.java similarity index 87% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/FlushBufferFunction.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/FlushBufferFunction.java index be43b75c5591..8d4022754d9d 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/FlushBufferFunction.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/FlushBufferFunction.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.record_buffer; +package io.airbyte.cdk.integrations.destination.record_buffer; import io.airbyte.commons.functional.CheckedBiConsumer; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/InMemoryBuffer.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/InMemoryBuffer.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/InMemoryBuffer.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/InMemoryBuffer.java index d94a73dfd07e..7f178d32a79b 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/InMemoryBuffer.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/InMemoryBuffer.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.record_buffer; +package io.airbyte.cdk.integrations.destination.record_buffer; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/InMemoryRecordBufferingStrategy.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/InMemoryRecordBufferingStrategy.java similarity index 92% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/InMemoryRecordBufferingStrategy.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/InMemoryRecordBufferingStrategy.java index d16ef8dca1e1..932ac18f1ced 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/InMemoryRecordBufferingStrategy.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/InMemoryRecordBufferingStrategy.java @@ -2,11 +2,11 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.record_buffer; +package io.airbyte.cdk.integrations.destination.record_buffer; -import io.airbyte.integrations.destination.buffered_stream_consumer.CheckAndRemoveRecordWriter; -import io.airbyte.integrations.destination.buffered_stream_consumer.RecordSizeEstimator; -import io.airbyte.integrations.destination.buffered_stream_consumer.RecordWriter; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.CheckAndRemoveRecordWriter; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.RecordSizeEstimator; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.RecordWriter; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/SerializableBuffer.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/SerializableBuffer.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/SerializableBuffer.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/SerializableBuffer.java index 1870d779b70f..79477ab5cc5b 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/SerializableBuffer.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/SerializableBuffer.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.record_buffer; +package io.airbyte.cdk.integrations.destination.record_buffer; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.io.File; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/SerializedBufferingStrategy.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/SerializedBufferingStrategy.java similarity index 95% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/SerializedBufferingStrategy.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/SerializedBufferingStrategy.java index d69451440e03..39ed0fc14235 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/SerializedBufferingStrategy.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination/record_buffer/SerializedBufferingStrategy.java @@ -2,9 +2,9 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.record_buffer; +package io.airbyte.cdk.integrations.destination.record_buffer; -import io.airbyte.commons.string.Strings; +import io.airbyte.cdk.integrations.util.ConnectorExceptionUtil; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; @@ -60,7 +60,6 @@ public SerializedBufferingStrategy(final BufferCreateFunction onCreateBuffer, * @param stream stream associated with record * @param message {@link AirbyteMessage} to buffer * @return Optional which contains a {@link BufferFlushType} if a flush occurred, otherwise empty) - * @throws Exception */ @Override public Optional addRecord(final AirbyteStreamNameNamespacePair stream, final AirbyteMessage message) throws Exception { @@ -163,9 +162,8 @@ public void close() throws Exception { LOGGER.error("Exception while closing stream buffer", e); } } - if (!exceptionsThrown.isEmpty()) { - throw new RuntimeException(String.format("Exceptions thrown while closing buffers: %s", Strings.join(exceptionsThrown, "\n"))); - } + + ConnectorExceptionUtil.logAllAndThrowFirst("Exceptions thrown while closing buffers: ", exceptionsThrown); } } diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/AirbyteFileUtils.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/AirbyteFileUtils.java similarity index 96% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/AirbyteFileUtils.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/AirbyteFileUtils.java index 23c72f12c884..c5d79569a2c3 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/AirbyteFileUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/AirbyteFileUtils.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async; +package io.airbyte.cdk.integrations.destination_async; import java.text.DecimalFormat; diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/AsyncStreamConsumer.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/AsyncStreamConsumer.java new file mode 100644 index 000000000000..711326fd919b --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/AsyncStreamConsumer.java @@ -0,0 +1,252 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination_async; + +import static java.util.stream.Collectors.toMap; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.destination.StreamSyncSummary; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.OnStartFunction; +import io.airbyte.cdk.integrations.destination_async.buffers.BufferEnqueue; +import io.airbyte.cdk.integrations.destination_async.buffers.BufferManager; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; +import io.airbyte.cdk.integrations.destination_async.state.FlushFailure; +import io.airbyte.commons.json.Jsons; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Async version of the + * {@link io.airbyte.cdk.integrations.destination.buffered_stream_consumer.BufferedStreamConsumer}. + *

+ * With this consumer, a destination is able to continue reading records until hitting the maximum + * memory limit governed by {@link GlobalMemoryManager}. Record writing is decoupled via + * {@link FlushWorkers}. See the other linked class for more detail. + */ +@Slf4j +public class AsyncStreamConsumer implements SerializedAirbyteMessageConsumer { + + private static final Logger LOGGER = LoggerFactory.getLogger(AsyncStreamConsumer.class); + + private final OnStartFunction onStart; + private final OnCloseFunction onClose; + private final ConfiguredAirbyteCatalog catalog; + private final BufferManager bufferManager; + private final BufferEnqueue bufferEnqueue; + private final FlushWorkers flushWorkers; + private final Set streamNames; + private final FlushFailure flushFailure; + private final String defaultNamespace; + // Note that this map will only be populated for streams with nonzero records. + private final ConcurrentMap recordCounts; + + private boolean hasStarted; + private boolean hasClosed; + private boolean hasFailed = false; + // This is to account for the references when deserialization to a PartialAirbyteMessage. The + // calculation is as follows: + // PartialAirbyteMessage (4) + Max( PartialRecordMessage(4), PartialStateMessage(6)) with + // PartialStateMessage being larger with more nested objects within it. Using 8 bytes as we assumed + // a 64 bit JVM. + final int PARTIAL_DESERIALIZE_REF_BYTES = 10 * 8; + + public AsyncStreamConsumer(final Consumer outputRecordCollector, + final OnStartFunction onStart, + final OnCloseFunction onClose, + final DestinationFlushFunction flusher, + final ConfiguredAirbyteCatalog catalog, + final BufferManager bufferManager, + final String defaultNamespace) { + this(outputRecordCollector, onStart, onClose, flusher, catalog, bufferManager, new FlushFailure(), defaultNamespace); + } + + public AsyncStreamConsumer(final Consumer outputRecordCollector, + final OnStartFunction onStart, + final OnCloseFunction onClose, + final DestinationFlushFunction flusher, + final ConfiguredAirbyteCatalog catalog, + final BufferManager bufferManager, + final String defaultNamespace, + final ExecutorService workerPool) { + this(outputRecordCollector, onStart, onClose, flusher, catalog, bufferManager, new FlushFailure(), defaultNamespace, workerPool); + } + + @VisibleForTesting + public AsyncStreamConsumer(final Consumer outputRecordCollector, + final OnStartFunction onStart, + final OnCloseFunction onClose, + final DestinationFlushFunction flusher, + final ConfiguredAirbyteCatalog catalog, + final BufferManager bufferManager, + final FlushFailure flushFailure, + final String defaultNamespace, + final ExecutorService workerPool) { + this.defaultNamespace = defaultNamespace; + hasStarted = false; + hasClosed = false; + + this.onStart = onStart; + this.onClose = onClose; + this.catalog = catalog; + this.bufferManager = bufferManager; + bufferEnqueue = bufferManager.getBufferEnqueue(); + this.flushFailure = flushFailure; + flushWorkers = + new FlushWorkers(bufferManager.getBufferDequeue(), flusher, outputRecordCollector, flushFailure, bufferManager.getStateManager(), workerPool); + streamNames = StreamDescriptorUtils.fromConfiguredCatalog(catalog); + this.recordCounts = new ConcurrentHashMap<>(); + } + + @VisibleForTesting + public AsyncStreamConsumer(final Consumer outputRecordCollector, + final OnStartFunction onStart, + final OnCloseFunction onClose, + final DestinationFlushFunction flusher, + final ConfiguredAirbyteCatalog catalog, + final BufferManager bufferManager, + final FlushFailure flushFailure, + final String defaultNamespace) { + this(outputRecordCollector, onStart, onClose, flusher, catalog, bufferManager, flushFailure, defaultNamespace, Executors.newFixedThreadPool(5)); + } + + @Override + public void start() throws Exception { + Preconditions.checkState(!hasStarted, "Consumer has already been started."); + hasStarted = true; + + flushWorkers.start(); + + LOGGER.info("{} started.", AsyncStreamConsumer.class); + onStart.call(); + } + + @Override + public void accept(final String messageString, final Integer sizeInBytes) throws Exception { + Preconditions.checkState(hasStarted, "Cannot accept records until consumer has started"); + propagateFlushWorkerExceptionIfPresent(); + /* + * intentionally putting extractStream outside the buffer manager so that if in the future we want + * to try to use a thread pool to partially deserialize to get record type and stream name, we can + * do it without touching buffer manager. + */ + final var message = deserializeAirbyteMessage(messageString); + if (Type.RECORD.equals(message.getType())) { + if (Strings.isNullOrEmpty(message.getRecord().getNamespace())) { + message.getRecord().setNamespace(defaultNamespace); + } + validateRecord(message); + + getRecordCounter(message.getRecord().getStreamDescriptor()).incrementAndGet(); + } + bufferEnqueue.addRecord(message, sizeInBytes + PARTIAL_DESERIALIZE_REF_BYTES, defaultNamespace); + } + + /** + * Deserializes to a {@link PartialAirbyteMessage} which can represent both a Record or a State + * Message + * + * PartialAirbyteMessage holds either: + *

  • entire serialized message string when message is a valid State Message + *
  • serialized AirbyteRecordMessage when message is a valid Record Message
  • + * + * @param messageString the string to deserialize + * @return PartialAirbyteMessage if the message is valid, empty otherwise + */ + @VisibleForTesting + public static PartialAirbyteMessage deserializeAirbyteMessage(final String messageString) { + // TODO: (ryankfu) plumb in the serialized AirbyteStateMessage to match AirbyteRecordMessage code + // parity. https://github.com/airbytehq/airbyte/issues/27530 for additional context + final var partial = Jsons.tryDeserializeExact(messageString, PartialAirbyteMessage.class) + .orElseThrow(() -> new RuntimeException("Unable to deserialize PartialAirbyteMessage.")); + + final var msgType = partial.getType(); + if (Type.RECORD.equals(msgType) && partial.getRecord().getData() != null) { + // store serialized json + partial.withSerialized(partial.getRecord().getData().toString()); + // The connector doesn't need to be able to access to the record value. We can serialize it here and + // drop the json + // object. Having this data stored as a string is slightly more optimal for the memory usage. + partial.getRecord().setData(null); + } else if (Type.STATE.equals(msgType)) { + partial.withSerialized(messageString); + } else { + throw new RuntimeException(String.format("Unsupported message type: %s", msgType)); + } + + return partial; + } + + @Override + public void close() throws Exception { + Preconditions.checkState(hasStarted, "Cannot close; has not started."); + Preconditions.checkState(!hasClosed, "Has already closed."); + hasClosed = true; + + // assume closing upload workers will flush all accepted records. + // we need to close the workers before closing the bufferManagers (and underlying buffers) + // or we risk in-memory data. + flushWorkers.close(); + + bufferManager.close(); + + final Map streamSyncSummaries = streamNames.stream().collect(toMap( + streamDescriptor -> streamDescriptor, + streamDescriptor -> new StreamSyncSummary( + Optional.of(getRecordCounter(streamDescriptor).get())))); + onClose.accept(hasFailed, streamSyncSummaries); + + // as this throws an exception, we need to be after all other close functions. + propagateFlushWorkerExceptionIfPresent(); + LOGGER.info("{} closed", AsyncStreamConsumer.class); + } + + private AtomicLong getRecordCounter(final StreamDescriptor streamDescriptor) { + return recordCounts.computeIfAbsent(streamDescriptor, sd -> new AtomicLong()); + } + + private void propagateFlushWorkerExceptionIfPresent() throws Exception { + if (flushFailure.isFailed()) { + hasFailed = true; + if (flushFailure.getException() == null) { + throw new RuntimeException("The Destination failed with a missing exception. This should not happen. Please check logs."); + } + throw flushFailure.getException(); + } + } + + private void validateRecord(final PartialAirbyteMessage message) { + final StreamDescriptor streamDescriptor = new StreamDescriptor() + .withNamespace(message.getRecord().getNamespace()) + .withName(message.getRecord().getStream()); + // if stream is not part of list of streams to sync to then throw invalid stream exception + if (!streamNames.contains(streamDescriptor)) { + throwUnrecognizedStream(catalog, message); + } + } + + private static void throwUnrecognizedStream(final ConfiguredAirbyteCatalog catalog, final PartialAirbyteMessage message) { + throw new IllegalArgumentException( + String.format("Message contained record from a stream that was not in the catalog. \ncatalog: %s , \nmessage: %s", + Jsons.serialize(catalog), Jsons.serialize(message))); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/DestinationFlushFunction.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/DestinationFlushFunction.java new file mode 100644 index 000000000000..6e7ffd379098 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/DestinationFlushFunction.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination_async; + +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.stream.Stream; + +/** + * An interface meant to be used with {@link FlushWorkers}. + *

    + * A destination instructs workers how to write data by specifying + * {@link #flush(StreamDescriptor, Stream)}. This keeps the worker abstraction generic and reusable. + *

    + * e.g. A database destination's flush function likely involves parsing the stream into SQL + * statements. + *

    + * There are 2 different destination types as of this writing: + *

      + *
    • 1. Destinations that upload files. This includes warehouses and databases.
    • + *
    • 2. Destinations that upload data streams. This mostly includes various Cloud storages. This + * will include reverse-ETL in the future
    • + *
    + * In both cases, the simplest way to model the incoming data is as a stream. + */ +public interface DestinationFlushFunction { + + /** + * Flush a batch of data to the destination. + * + * @param decs the Airbyte stream the data stream belongs to + * @param stream a bounded {@link AirbyteMessage} stream ideally of + * {@link #getOptimalBatchSizeBytes()} size + * @throws Exception + */ + void flush(StreamDescriptor decs, Stream stream) throws Exception; + + /** + * When invoking {@link #flush(StreamDescriptor, Stream)}, best effort attempt to invoke flush with + * a batch of this size. Useful for Destinations that have optimal flush batch sizes. + *

    + * If you increase this, make sure that {@link #getQueueFlushThresholdBytes()} is larger than this + * value. Otherwise we may trigger flushes before reaching the optimal batch size. + * + * @return the optimal batch size in bytes + */ + long getOptimalBatchSizeBytes(); + + /** + * This value should be at least as high as {@link #getOptimalBatchSizeBytes()}. It's used by + * {@link DetectStreamToFlush} as part of deciding when a stream needs to be flushed. I'm being + * vague because I don't understand the specifics. + */ + default long getQueueFlushThresholdBytes() { + return 10 * 1024 * 1024; // 10MB + } + +} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/DetectStreamToFlush.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/DetectStreamToFlush.java similarity index 86% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/DetectStreamToFlush.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/DetectStreamToFlush.java index 6d230dec56d7..ccd24736ce21 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/DetectStreamToFlush.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/DetectStreamToFlush.java @@ -2,18 +2,20 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async; +package io.airbyte.cdk.integrations.destination_async; import com.google.common.annotations.VisibleForTesting; -import io.airbyte.integrations.destination_async.buffers.BufferDequeue; +import io.airbyte.cdk.integrations.destination_async.buffers.BufferDequeue; import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.time.Clock; import java.time.Instant; -import java.time.temporal.ChronoUnit; import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -26,21 +28,32 @@ public class DetectStreamToFlush { private static final double EAGER_FLUSH_THRESHOLD = 0.90; - private static final long QUEUE_FLUSH_THRESHOLD_BYTES = 10 * 1024 * 1024; // 10MB - private static final long MAX_TIME_BETWEEN_REC_MIN = 5L; + private static final long MAX_TIME_BETWEEN_FLUSH_MS = 5 * 60 * 1000; private final BufferDequeue bufferDequeue; private final RunningFlushWorkers runningFlushWorkers; private final AtomicBoolean isClosing; private final DestinationFlushFunction flusher; + private final Clock nowProvider; + private final ConcurrentMap latestFlushTimeMsPerStream = new ConcurrentHashMap<>(); public DetectStreamToFlush(final BufferDequeue bufferDequeue, final RunningFlushWorkers runningFlushWorkers, final AtomicBoolean isClosing, final DestinationFlushFunction flusher) { + this(bufferDequeue, runningFlushWorkers, isClosing, flusher, Clock.systemUTC()); + } + + @VisibleForTesting + DetectStreamToFlush(final BufferDequeue bufferDequeue, + final RunningFlushWorkers runningFlushWorkers, + final AtomicBoolean isClosing, + final DestinationFlushFunction flusher, + final Clock nowProvider) { this.bufferDequeue = bufferDequeue; this.runningFlushWorkers = runningFlushWorkers; this.isClosing = isClosing; this.flusher = flusher; + this.nowProvider = nowProvider; } /** @@ -73,7 +86,7 @@ long computeQueueThreshold() { final boolean isBuffer90Full = EAGER_FLUSH_THRESHOLD <= (double) bufferDequeue.getTotalGlobalQueueSizeBytes() / bufferDequeue.getMaxQueueSizeBytes(); // when we are closing or queues are very full, flush regardless of how few items are in the queue. - return isClosing.get() || isBuffer90Full ? 0 : QUEUE_FLUSH_THRESHOLD_BYTES; + return isClosing.get() || isBuffer90Full ? 0 : flusher.getQueueFlushThresholdBytes(); } // todo (cgardens) - improve prioritization by getting a better estimate of how much data running @@ -83,8 +96,8 @@ long computeQueueThreshold() { * Return an empty optional if no streams are ready. *

    * A stream is ready to flush if it either meets a size threshold or a time threshold. See - * {@link #isSizeTriggered(StreamDescriptor, long)} and {@link #isTimeTriggered(StreamDescriptor)} - * for details on these triggers. + * {@link #isSizeTriggered(StreamDescriptor, long)} and {@link #isTimeTriggered(long)} for details + * on these triggers. * * @param queueSizeThresholdBytes - the size threshold to use for determining if a stream is ready * to flush. @@ -93,7 +106,8 @@ long computeQueueThreshold() { @VisibleForTesting Optional getNextStreamToFlush(final long queueSizeThresholdBytes) { for (final StreamDescriptor stream : orderStreamsByPriority(bufferDequeue.getBufferedStreams())) { - final ImmutablePair isTimeTriggeredResult = isTimeTriggered(stream); + final long latestFlushTimeMs = latestFlushTimeMsPerStream.computeIfAbsent(stream, _k -> nowProvider.millis()); + final ImmutablePair isTimeTriggeredResult = isTimeTriggered(latestFlushTimeMs); final ImmutablePair isSizeTriggeredResult = isSizeTriggered(stream, queueSizeThresholdBytes); final String debugString = String.format( @@ -106,7 +120,7 @@ Optional getNextStreamToFlush(final long queueSizeThresholdByt if (isSizeTriggeredResult.getLeft() || isTimeTriggeredResult.getLeft()) { log.info("flushing: {}", debugString); - + latestFlushTimeMsPerStream.put(stream, nowProvider.millis()); return Optional.of(stream); } } @@ -122,15 +136,13 @@ Optional getNextStreamToFlush(final long queueSizeThresholdByt * This method also returns debug string with info that about the computation. We do it this way, so * that the debug info that is printed is exactly what is used in the computation. * - * @param stream stream + * @param latestFlushTimeMs latestFlushTimeMs * @return is time triggered and a debug string */ @VisibleForTesting - ImmutablePair isTimeTriggered(final StreamDescriptor stream) { - final Boolean isTimeTriggered = bufferDequeue.getTimeOfLastRecord(stream) - .map(time -> time.isBefore(Instant.now().minus(MAX_TIME_BETWEEN_REC_MIN, ChronoUnit.MINUTES))) - .orElse(false); - + ImmutablePair isTimeTriggered(final long latestFlushTimeMs) { + final long timeSinceLastFlushMs = nowProvider.millis() - latestFlushTimeMs; + final Boolean isTimeTriggered = timeSinceLastFlushMs >= MAX_TIME_BETWEEN_FLUSH_MS; final String debugString = String.format("time trigger: %s", isTimeTriggered); return ImmutablePair.of(isTimeTriggered, debugString); diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/FlushWorkers.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/FlushWorkers.java similarity index 80% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/FlushWorkers.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/FlushWorkers.java index 94be07f6f485..603fa5054da2 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/FlushWorkers.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/FlushWorkers.java @@ -2,14 +2,14 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async; +package io.airbyte.cdk.integrations.destination_async; +import io.airbyte.cdk.integrations.destination_async.buffers.BufferDequeue; +import io.airbyte.cdk.integrations.destination_async.buffers.StreamAwareQueue.MessageWithMeta; +import io.airbyte.cdk.integrations.destination_async.state.FlushFailure; +import io.airbyte.cdk.integrations.destination_async.state.GlobalAsyncStateManager; +import io.airbyte.cdk.integrations.destination_async.state.PartialStateWithDestinationStats; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination_async.buffers.BufferDequeue; -import io.airbyte.integrations.destination_async.buffers.StreamAwareQueue.MessageWithMeta; -import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteMessage; -import io.airbyte.integrations.destination_async.state.FlushFailure; -import io.airbyte.integrations.destination_async.state.GlobalAsyncStateManager; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.StreamDescriptor; import java.util.List; @@ -51,7 +51,7 @@ public class FlushWorkers implements AutoCloseable { private static final long SUPERVISOR_INITIAL_DELAY_SECS = 0L; private static final long SUPERVISOR_PERIOD_SECS = 1L; private static final long DEBUG_INITIAL_DELAY_SECS = 0L; - private static final long DEBUG_PERIOD_SECS = 10L; + private static final long DEBUG_PERIOD_SECS = 60L; private final ScheduledExecutorService supervisorThread; private final ExecutorService workerPool; @@ -72,14 +72,23 @@ public FlushWorkers(final BufferDequeue bufferDequeue, final Consumer outputRecordCollector, final FlushFailure flushFailure, final GlobalAsyncStateManager stateManager) { + this(bufferDequeue, flushFunction, outputRecordCollector, flushFailure, stateManager, Executors.newFixedThreadPool(5)); + } + + public FlushWorkers(final BufferDequeue bufferDequeue, + final DestinationFlushFunction flushFunction, + final Consumer outputRecordCollector, + final FlushFailure flushFailure, + final GlobalAsyncStateManager stateManager, + final ExecutorService workerPool) { this.bufferDequeue = bufferDequeue; this.outputRecordCollector = outputRecordCollector; this.flushFailure = flushFailure; this.stateManager = stateManager; + this.workerPool = workerPool; flusher = flushFunction; debugLoop = Executors.newSingleThreadScheduledExecutor(); supervisorThread = Executors.newScheduledThreadPool(1); - workerPool = Executors.newFixedThreadPool(5); isClosing = new AtomicBoolean(false); runningFlushWorkers = new RunningFlushWorkers(); detectStreamToFlush = new DetectStreamToFlush(bufferDequeue, runningFlushWorkers, isClosing, flusher); @@ -126,14 +135,14 @@ private void retrieveWork() { } private void printWorkerInfo() { - final var workerInfo = new StringBuilder().append("WORKER INFO").append(System.lineSeparator()); + final var workerInfo = new StringBuilder().append("[ASYNC WORKER INFO] "); final ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) workerPool; final int queueSize = threadPoolExecutor.getQueue().size(); final int activeCount = threadPoolExecutor.getActiveCount(); - workerInfo.append(String.format(" Pool queue size: %d, Active threads: %d", queueSize, activeCount)); + workerInfo.append(String.format("Pool queue size: %d, Active threads: %d", queueSize, activeCount)); log.info(workerInfo.toString()); } @@ -213,22 +222,27 @@ public void close() throws Exception { // before shutting down the supervisor, flush all state. emitStateMessages(stateManager.flushStates()); supervisorThread.shutdown(); - final var supervisorShut = supervisorThread.awaitTermination(5L, TimeUnit.MINUTES); - log.info("Closing flush workers -- Supervisor shutdown status: {}", supervisorShut); + while (!supervisorThread.awaitTermination(5L, TimeUnit.MINUTES)) { + log.info("Waiting for flush worker supervisor to shut down"); + } + log.info("Closing flush workers -- supervisor shut down"); log.info("Closing flush workers -- Starting worker pool shutdown.."); workerPool.shutdown(); - final var workersShut = workerPool.awaitTermination(5L, TimeUnit.MINUTES); - log.info("Closing flush workers -- Workers shutdown status: {}", workersShut); + while (!workerPool.awaitTermination(5L, TimeUnit.MINUTES)) { + log.info("Waiting for flush workers to shut down"); + } + log.info("Closing flush workers -- workers shut down"); debugLoop.shutdownNow(); } - private void emitStateMessages(final List partials) { - partials - .stream() - .map(partial -> Jsons.deserialize(partial.getSerialized(), AirbyteMessage.class)) - .forEach(outputRecordCollector); + private void emitStateMessages(final List partials) { + for (final PartialStateWithDestinationStats partial : partials) { + final AirbyteMessage message = Jsons.deserialize(partial.stateMessage().getSerialized(), AirbyteMessage.class); + message.getState().setDestinationStats(partial.stats()); + outputRecordCollector.accept(message); + } } private static String humanReadableFlushWorkerId(final UUID flushWorkerId) { diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/GlobalMemoryManager.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/GlobalMemoryManager.java new file mode 100644 index 000000000000..ca8aea8fdbcb --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/GlobalMemoryManager.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination_async; + +import java.util.concurrent.atomic.AtomicLong; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; + +/** + * Responsible for managing buffer memory across multiple queues in a thread-safe way. This does not + * allocate or free memory in the traditional sense, but rather manages based off memory estimates + * provided by the callers. + *

    + * The goal is to enable maximum allowed memory bounds for each queue to be dynamically adjusted + * according to the overall available memory. Memory blocks are managed in chunks of + * {@link #BLOCK_SIZE_BYTES}, and the total amount of memory managed is configured at creation time. + *

    + * As a destination has no information about incoming per-stream records, having static queue sizes + * can cause unnecessary backpressure on a per-stream basis. By providing a dynamic, global view of + * buffer memory management, this class allows each queue to release and request memory dynamically, + * enabling effective sharing of global memory resources across all the queues, and avoiding + * accidental stream backpressure. + *

    + * This becomes particularly useful in the following scenarios: + *

      + *
    • 1. When the incoming records belong to a single stream. Dynamic allocation ensures this one + * stream can utilise all memory.
    • + *
    • 2. When the incoming records are from multiple streams, such as with Change Data Capture + * (CDC). Here, dynamic allocation let us create as many queues as possible, allowing all streams to + * be processed in parallel without accidental backpressure from unnecessary eager flushing.
    • + *
    + */ +@Slf4j +public class GlobalMemoryManager { + + // In cases where a queue is rapidly expanding, a larger block size allows less allocation calls. On + // the flip size, a smaller block size allows more granular memory management. Since this overhead + // is minimal for now, err on a smaller block sizes. + public static final long BLOCK_SIZE_BYTES = 10 * 1024 * 1024; // 10MB + private final long maxMemoryBytes; + + private final AtomicLong currentMemoryBytes = new AtomicLong(0); + + public GlobalMemoryManager(final long maxMemoryBytes) { + this.maxMemoryBytes = maxMemoryBytes; + } + + public long getMaxMemoryBytes() { + return maxMemoryBytes; + } + + public long getCurrentMemoryBytes() { + return currentMemoryBytes.get(); + } + + /** + * Requests a block of memory of {@link #BLOCK_SIZE_BYTES}. Return 0 if memory cannot be freed. + * + * @return the size of the allocated block, in bytes + */ + public synchronized long requestMemory() { + // todo(davin): what happens if the incoming record is larger than 30MB? + if (currentMemoryBytes.get() >= maxMemoryBytes) { + return 0L; + } + + final var freeMem = maxMemoryBytes - currentMemoryBytes.get(); + // Never allocate more than free memory size. + final var toAllocateBytes = Math.min(freeMem, BLOCK_SIZE_BYTES); + currentMemoryBytes.addAndGet(toAllocateBytes); + + log.debug("Memory Requested: max: {}, allocated: {}, allocated in this request: {}", + FileUtils.byteCountToDisplaySize(maxMemoryBytes), + FileUtils.byteCountToDisplaySize(currentMemoryBytes.get()), + FileUtils.byteCountToDisplaySize(toAllocateBytes)); + return toAllocateBytes; + } + + /** + * Releases a block of memory of the given size. If the amount of memory released exceeds the + * current memory allocation, a warning will be logged. + * + * @param bytes the size of the block to free, in bytes + */ + public void free(final long bytes) { + log.info("Freeing {} bytes..", bytes); + currentMemoryBytes.addAndGet(-bytes); + + final long currentMemory = currentMemoryBytes.get(); + if (currentMemory < 0) { + log.info("Freed more memory than allocated ({} of {})", bytes, currentMemory + bytes); + } + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/OnCloseFunction.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/OnCloseFunction.java new file mode 100644 index 000000000000..c1bd6f097d8f --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/OnCloseFunction.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination_async; + +import io.airbyte.cdk.integrations.destination.StreamSyncSummary; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.Map; +import java.util.function.BiConsumer; + +/** + * Async version of + * {@link io.airbyte.cdk.integrations.destination.buffered_stream_consumer.OnCloseFunction}. + * Separately out for easier versioning. + */ +public interface OnCloseFunction extends BiConsumer> { + +} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/RunningFlushWorkers.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/RunningFlushWorkers.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/RunningFlushWorkers.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/RunningFlushWorkers.java index 04cb547eae11..dd0d2ed08e0b 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/RunningFlushWorkers.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/RunningFlushWorkers.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async; +package io.airbyte.cdk.integrations.destination_async; import com.google.common.base.Preconditions; import io.airbyte.protocol.models.v0.StreamDescriptor; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/StreamDescriptorUtils.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/StreamDescriptorUtils.java similarity index 96% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/StreamDescriptorUtils.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/StreamDescriptorUtils.java index e2ca3a6a25e3..bd93f55ebd22 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/StreamDescriptorUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/StreamDescriptorUtils.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async; +package io.airbyte.cdk.integrations.destination_async; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStream; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/buffers/BufferDequeue.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferDequeue.java similarity index 79% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/buffers/BufferDequeue.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferDequeue.java index 40645a21b5f2..3650733cfbcf 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/buffers/BufferDequeue.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferDequeue.java @@ -2,12 +2,12 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async.buffers; +package io.airbyte.cdk.integrations.destination_async.buffers; -import io.airbyte.integrations.destination_async.GlobalMemoryManager; -import io.airbyte.integrations.destination_async.buffers.MemoryBoundedLinkedBlockingQueue.MemoryItem; -import io.airbyte.integrations.destination_async.buffers.StreamAwareQueue.MessageWithMeta; -import io.airbyte.integrations.destination_async.state.GlobalAsyncStateManager; +import io.airbyte.cdk.integrations.destination_async.GlobalMemoryManager; +import io.airbyte.cdk.integrations.destination_async.buffers.MemoryBoundedLinkedBlockingQueue.MemoryItem; +import io.airbyte.cdk.integrations.destination_async.buffers.StreamAwareQueue.MessageWithMeta; +import io.airbyte.cdk.integrations.destination_async.state.GlobalAsyncStateManager; import io.airbyte.protocol.models.v0.StreamDescriptor; import java.time.Instant; import java.util.HashSet; @@ -52,13 +52,11 @@ public BufferDequeue(final GlobalMemoryManager memoryManager, * @return autocloseable batch object, that frees memory. */ public MemoryAwareMessageBatch take(final StreamDescriptor streamDescriptor, final long optimalBytesToRead) { - final var queue = buffers.get(streamDescriptor); + final var lock = bufferLocks.computeIfAbsent(streamDescriptor, _k -> new ReentrantLock()); + lock.lock(); - if (!bufferLocks.containsKey(streamDescriptor)) { - bufferLocks.put(streamDescriptor, new ReentrantLock()); - } + final var queue = buffers.get(streamDescriptor); - bufferLocks.get(streamDescriptor).lock(); try { final AtomicLong bytesRead = new AtomicLong(); @@ -76,7 +74,19 @@ public MemoryAwareMessageBatch take(final StreamDescriptor streamDescriptor, fin } } - queue.addMaxMemory(-bytesRead.get()); + if (queue.isEmpty()) { + final var batchSizeBytes = bytesRead.get(); + final var allocatedBytes = queue.getMaxMemoryUsage(); + + // Free unused allocation for the queue. + // When the batch flushes it will flush its allocation. + memoryManager.free(allocatedBytes - batchSizeBytes); + + // Shrink queue to 0 — any new messages will reallocate. + queue.addMaxMemory(-allocatedBytes); + } else { + queue.addMaxMemory(-bytesRead.get()); + } return new MemoryAwareMessageBatch( output, @@ -84,7 +94,7 @@ public MemoryAwareMessageBatch take(final StreamDescriptor streamDescriptor, fin memoryManager, stateManager); } finally { - bufferLocks.get(streamDescriptor).unlock(); + lock.unlock(); } } diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/buffers/BufferEnqueue.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferEnqueue.java similarity index 80% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/buffers/BufferEnqueue.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferEnqueue.java index 3077b47dd447..09f67f62c786 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/buffers/BufferEnqueue.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferEnqueue.java @@ -2,13 +2,13 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async.buffers; +package io.airbyte.cdk.integrations.destination_async.buffers; import static java.lang.Thread.sleep; -import io.airbyte.integrations.destination_async.GlobalMemoryManager; -import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteMessage; -import io.airbyte.integrations.destination_async.state.GlobalAsyncStateManager; +import io.airbyte.cdk.integrations.destination_async.GlobalMemoryManager; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; +import io.airbyte.cdk.integrations.destination_async.state.GlobalAsyncStateManager; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.StreamDescriptor; import java.util.concurrent.ConcurrentMap; @@ -38,22 +38,19 @@ public BufferEnqueue(final GlobalMemoryManager memoryManager, * @param message to buffer * @param sizeInBytes */ - public void addRecord(final PartialAirbyteMessage message, final Integer sizeInBytes) { + public void addRecord(final PartialAirbyteMessage message, final Integer sizeInBytes, final String defaultNamespace) { if (message.getType() == Type.RECORD) { handleRecord(message, sizeInBytes); } else if (message.getType() == Type.STATE) { - stateManager.trackState(message, sizeInBytes); + stateManager.trackState(message, sizeInBytes, defaultNamespace); } } private void handleRecord(final PartialAirbyteMessage message, final Integer sizeInBytes) { final StreamDescriptor streamDescriptor = extractStateFromRecord(message); - if (streamDescriptor != null && !buffers.containsKey(streamDescriptor)) { - buffers.put(streamDescriptor, new StreamAwareQueue(memoryManager.requestMemory())); - } + final var queue = buffers.computeIfAbsent(streamDescriptor, _k -> new StreamAwareQueue(memoryManager.requestMemory())); final long stateId = stateManager.getStateIdAndIncrementCounter(streamDescriptor); - final var queue = buffers.get(streamDescriptor); var addedToQueue = queue.offer(message, sizeInBytes, stateId); int i = 0; diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferManager.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferManager.java new file mode 100644 index 000000000000..1d824a2b14c0 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferManager.java @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination_async.buffers; + +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.cdk.integrations.destination_async.AirbyteFileUtils; +import io.airbyte.cdk.integrations.destination_async.FlushWorkers; +import io.airbyte.cdk.integrations.destination_async.GlobalMemoryManager; +import io.airbyte.cdk.integrations.destination_async.state.GlobalAsyncStateManager; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.ArrayList; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Slf4j +public class BufferManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(BufferManager.class); + + public final long maxMemory; + private final ConcurrentMap buffers; + private final BufferEnqueue bufferEnqueue; + private final BufferDequeue bufferDequeue; + private final GlobalMemoryManager memoryManager; + + private final GlobalAsyncStateManager stateManager; + private final ScheduledExecutorService debugLoop; + private static final long DEBUG_PERIOD_SECS = 60L; + + public static final double MEMORY_LIMIT_RATIO = 0.7; + + public BufferManager() { + this((long) (Runtime.getRuntime().maxMemory() * MEMORY_LIMIT_RATIO)); + } + + /** + * @param memoryLimit the amount of estimated memory we allow for all buffers. The + * GlobalMemoryManager will apply back pressure once this quota is filled. "Memory" can be + * released back once flushing finishes. This number should be large enough we don't block + * reading unnecessarily, but small enough we apply back pressure before OOMing. + */ + public BufferManager(final long memoryLimit) { + maxMemory = memoryLimit; + LOGGER.info("Max 'memory' available for buffer allocation {}", FileUtils.byteCountToDisplaySize(maxMemory)); + memoryManager = new GlobalMemoryManager(maxMemory); + this.stateManager = new GlobalAsyncStateManager(memoryManager); + buffers = new ConcurrentHashMap<>(); + bufferEnqueue = new BufferEnqueue(memoryManager, buffers, stateManager); + bufferDequeue = new BufferDequeue(memoryManager, buffers, stateManager); + debugLoop = Executors.newSingleThreadScheduledExecutor(); + debugLoop.scheduleAtFixedRate(this::printQueueInfo, 0, DEBUG_PERIOD_SECS, TimeUnit.SECONDS); + } + + public GlobalAsyncStateManager getStateManager() { + return stateManager; + } + + @VisibleForTesting + protected GlobalMemoryManager getMemoryManager() { + return memoryManager; + } + + @VisibleForTesting + protected ConcurrentMap getBuffers() { + return buffers; + } + + public BufferEnqueue getBufferEnqueue() { + return bufferEnqueue; + } + + public BufferDequeue getBufferDequeue() { + return bufferDequeue; + } + + /** + * Closing a queue will flush all items from it. For this reason, this method needs to be called + * after {@link FlushWorkers#close()}. This allows the upload workers to make sure all items in the + * queue has been flushed. + */ + public void close() throws Exception { + debugLoop.shutdownNow(); + log.info("Buffers cleared.."); + } + + private void printQueueInfo() { + final var queueInfo = new StringBuilder().append("[ASYNC QUEUE INFO] "); + final ArrayList messages = new ArrayList<>(); + + messages + .add(String.format("Global: max: %s, allocated: %s (%s MB), %% used: %s", + AirbyteFileUtils.byteCountToDisplaySize(memoryManager.getMaxMemoryBytes()), + AirbyteFileUtils.byteCountToDisplaySize(memoryManager.getCurrentMemoryBytes()), + (double) memoryManager.getCurrentMemoryBytes() / 1024 / 1024, + (double) memoryManager.getCurrentMemoryBytes() / memoryManager.getMaxMemoryBytes())); + + for (final var entry : buffers.entrySet()) { + final var queue = entry.getValue(); + messages.add( + String.format("Queue `%s`, num records: %d, num bytes: %s, allocated bytes: %s", + entry.getKey().getName(), queue.size(), AirbyteFileUtils.byteCountToDisplaySize(queue.getCurrentMemoryUsage()), + AirbyteFileUtils.byteCountToDisplaySize(queue.getMaxMemoryUsage()))); + } + + messages.add(stateManager.getMemoryUsageMessage()); + + queueInfo.append(String.join(" | ", messages)); + + log.info(queueInfo.toString()); + } + +} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/buffers/MemoryAwareMessageBatch.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/MemoryAwareMessageBatch.java similarity index 78% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/buffers/MemoryAwareMessageBatch.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/MemoryAwareMessageBatch.java index 78e31517bcc2..591837196c1a 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/buffers/MemoryAwareMessageBatch.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/MemoryAwareMessageBatch.java @@ -2,12 +2,12 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async.buffers; +package io.airbyte.cdk.integrations.destination_async.buffers; -import io.airbyte.integrations.destination_async.GlobalMemoryManager; -import io.airbyte.integrations.destination_async.buffers.StreamAwareQueue.MessageWithMeta; -import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteMessage; -import io.airbyte.integrations.destination_async.state.GlobalAsyncStateManager; +import io.airbyte.cdk.integrations.destination_async.GlobalMemoryManager; +import io.airbyte.cdk.integrations.destination_async.buffers.StreamAwareQueue.MessageWithMeta; +import io.airbyte.cdk.integrations.destination_async.state.GlobalAsyncStateManager; +import io.airbyte.cdk.integrations.destination_async.state.PartialStateWithDestinationStats; import java.util.List; import java.util.Map; import org.slf4j.Logger; @@ -15,7 +15,7 @@ /** * POJO abstraction representing one discrete buffer read. This allows ergonomics dequeues by - * {@link io.airbyte.integrations.destination_async.FlushWorkers}. + * {@link io.airbyte.cdk.integrations.destination_async.FlushWorkers}. *

    * The contained stream **IS EXPECTED to be a BOUNDED** stream. Returning a boundless stream has * undefined behaviour. @@ -64,7 +64,7 @@ public void close() throws Exception { * * @return list of states that can be flushed */ - public List flushStates(final Map stateIdToCount) { + public List flushStates(final Map stateIdToCount) { stateIdToCount.forEach(stateManager::decrement); return stateManager.flushStates(); } diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/buffers/MemoryBoundedLinkedBlockingQueue.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/MemoryBoundedLinkedBlockingQueue.java similarity index 95% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/buffers/MemoryBoundedLinkedBlockingQueue.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/MemoryBoundedLinkedBlockingQueue.java index 8eb57c81ec27..3478a8258a2f 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/buffers/MemoryBoundedLinkedBlockingQueue.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/MemoryBoundedLinkedBlockingQueue.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async.buffers; +package io.airbyte.cdk.integrations.destination_async.buffers; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; @@ -66,6 +66,10 @@ public MemoryBoundedLinkedBlockingQueue.MemoryItem poll(final long timeout, f return hiddenQueue.poll(timeout, unit); } + public long getMaxMemoryUsage() { + return hiddenQueue.getMaxMemoryUsage(); + } + /** * Extends LinkedBlockingQueue so that we can get a LinkedBlockingQueue bounded by memory. Hidden as * an inner class, so it doesn't get misused, see top-level javadoc comment. @@ -82,6 +86,10 @@ public HiddenQueue(final long maxMemoryUsage) { this.maxMemoryUsage = new AtomicLong(maxMemoryUsage); } + public long getMaxMemoryUsage() { + return maxMemoryUsage.get(); + } + public boolean offer(final E e, final long itemSizeInBytes) { final long newMemoryUsage = currentMemoryUsage.addAndGet(itemSizeInBytes); if (newMemoryUsage <= maxMemoryUsage.get()) { diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/buffers/StreamAwareQueue.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/StreamAwareQueue.java similarity index 87% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/buffers/StreamAwareQueue.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/StreamAwareQueue.java index 87064d638c39..bf75b4415b49 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/buffers/StreamAwareQueue.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/buffers/StreamAwareQueue.java @@ -2,9 +2,9 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async.buffers; +package io.airbyte.cdk.integrations.destination_async.buffers; -import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteMessage; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; import java.time.Instant; import java.util.Optional; import java.util.concurrent.TimeUnit; @@ -27,10 +27,18 @@ public long getCurrentMemoryUsage() { return memoryAwareQueue.getCurrentMemoryUsage(); } + public long getMaxMemoryUsage() { + return memoryAwareQueue.getMaxMemoryUsage(); + } + public void addMaxMemory(final long maxMemoryUsage) { memoryAwareQueue.addMaxMemory(maxMemoryUsage); } + public boolean isEmpty() { + return memoryAwareQueue.size() == 0; + } + public Optional getTimeOfLastMessage() { // if the queue is empty, the time of last message is irrelevant if (size() == 0) { diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/partial_messages/PartialAirbyteMessage.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/partial_messages/PartialAirbyteMessage.java similarity index 84% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/partial_messages/PartialAirbyteMessage.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/partial_messages/PartialAirbyteMessage.java index 1f2026b69736..c0d3739b3285 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/partial_messages/PartialAirbyteMessage.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/partial_messages/PartialAirbyteMessage.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async.partial_messages; +package io.airbyte.cdk.integrations.destination_async.partial_messages; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyDescription; @@ -71,6 +71,15 @@ public PartialAirbyteMessage withState(final PartialAirbyteStateMessage state) { return this; } + /** + * For record messages, this stores the serialized data blob (i.e. + * {@code Jsons.serialize(message.getRecord().getData())}). For state messages, this stores the + * _entire_ message (i.e. {@code Jsons.serialize(message)}). + *

    + * See + * {@link io.airbyte.cdk.integrations.destination_async.AsyncStreamConsumer#deserializeAirbyteMessage(String)} + * for the exact logic of how this field is populated. + */ @JsonProperty("serialized") public String getSerialized() { return serialized; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/partial_messages/PartialAirbyteRecordMessage.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/partial_messages/PartialAirbyteRecordMessage.java similarity index 91% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/partial_messages/PartialAirbyteRecordMessage.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/partial_messages/PartialAirbyteRecordMessage.java index b6218de45473..ebd903fcfc87 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/partial_messages/PartialAirbyteRecordMessage.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/partial_messages/PartialAirbyteRecordMessage.java @@ -2,11 +2,12 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async.partial_messages; +package io.airbyte.cdk.integrations.destination_async.partial_messages; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.protocol.models.v0.StreamDescriptor; import java.util.Objects; // TODO: (ryankfu) remove this and test with low memory resources to ensure OOM is still not a @@ -116,4 +117,8 @@ public String toString() { '}'; } + public StreamDescriptor getStreamDescriptor() { + return new StreamDescriptor().withName(stream).withNamespace(namespace); + } + } diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/partial_messages/PartialAirbyteStateMessage.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/partial_messages/PartialAirbyteStateMessage.java similarity index 95% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/partial_messages/PartialAirbyteStateMessage.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/partial_messages/PartialAirbyteStateMessage.java index 52ae7c486a79..d91a4a13c403 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/partial_messages/PartialAirbyteStateMessage.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/partial_messages/PartialAirbyteStateMessage.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async.partial_messages; +package io.airbyte.cdk.integrations.destination_async.partial_messages; import com.fasterxml.jackson.annotation.JsonProperty; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/partial_messages/PartialAirbyteStreamState.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/partial_messages/PartialAirbyteStreamState.java similarity index 95% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/partial_messages/PartialAirbyteStreamState.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/partial_messages/PartialAirbyteStreamState.java index 1e09d69f297c..7076c5fcbc71 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/partial_messages/PartialAirbyteStreamState.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/partial_messages/PartialAirbyteStreamState.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async.partial_messages; +package io.airbyte.cdk.integrations.destination_async.partial_messages; import com.fasterxml.jackson.annotation.JsonProperty; import io.airbyte.protocol.models.v0.StreamDescriptor; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/state/FlushFailure.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/state/FlushFailure.java similarity index 91% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/state/FlushFailure.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/state/FlushFailure.java index 9fcc81835f5e..2188c6c66bc9 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/state/FlushFailure.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/state/FlushFailure.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async.state; +package io.airbyte.cdk.integrations.destination_async.state; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/state/GlobalAsyncStateManager.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/state/GlobalAsyncStateManager.java new file mode 100644 index 000000000000..e0283bdfe767 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/state/GlobalAsyncStateManager.java @@ -0,0 +1,397 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination_async.state; + +import static java.lang.Thread.sleep; + +import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import io.airbyte.cdk.integrations.destination_async.GlobalMemoryManager; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateStats; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicLong; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.mina.util.ConcurrentHashSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Responsible for managing state within the Destination. The general approach is a ref counter + * approach - each state message is associated with a record count. This count represents the number + * of preceding records. For a state to be emitted, all preceding records have to be written to the + * destination i.e. the counter is 0. + *

    + * A per-stream state queue is maintained internally, with each state within the queue having a + * counter. This means we *ALLOW* records succeeding an unemitted state to be written. This + * decouples record writing from state management at the cost of potentially repeating work if an + * upstream state is never written. + *

    + * One important detail here is the difference between how PER-STREAM & NON-PER-STREAM is handled. + * The PER-STREAM case is simple, and is as described above. The NON-PER-STREAM case is slightly + * tricky. Because we don't know the stream type to begin with, we always assume PER_STREAM until + * the first state message arrives. If this state message is a GLOBAL state, we alias all existing + * state ids to a single global state id via a set of alias ids. From then onwards, we use one id - + * {@link #SENTINEL_GLOBAL_DESC} regardless of stream. Read + * {@link #convertToGlobalIfNeeded(AirbyteMessage)} for more detail. + */ +@Slf4j +public class GlobalAsyncStateManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(GlobalAsyncStateManager.class); + + private static final StreamDescriptor SENTINEL_GLOBAL_DESC = new StreamDescriptor().withName(UUID.randomUUID().toString()); + private final GlobalMemoryManager memoryManager; + + /** + * Memory that the manager has allocated to it to use. It can ask for more memory as needed. + */ + private final AtomicLong memoryAllocated; + /** + * Memory that the manager is currently using. + */ + private final AtomicLong memoryUsed; + + boolean preState = true; + private final ConcurrentMap> descToStateIdQ = new ConcurrentHashMap<>(); + /** + * Both {@link stateIdToCounter} and {@link stateIdToCounterForPopulatingDestinationStats} are used + * to maintain a counter for the number of records associated with a give state i.e. before a state + * was received, how many records were seen until that point. As records are received the value for + * both are incremented. The difference is the purpose of the two attributes. + * {@link stateIdToCounter} is used to determine whether a state is safe to emit or not. This is + * done by decrementing the value as records are committed to the destination. If the value hits 0, + * it means all the records associated with a given state have been committed to the destination, it + * is safe to emit the state back to platform. But because of this we can't use it to determine the + * actual number of records that are associated with a state to update the value of + * {@link AirbyteStateMessage#destinationStats} at the time of emitting the state message. That's + * where we need {@link stateIdToCounterForPopulatingDestinationStats}, which is only reset when a + * state message has been emitted. + */ + private final ConcurrentMap stateIdToCounter = new ConcurrentHashMap<>(); + private final ConcurrentMap stateIdToCounterForPopulatingDestinationStats = new ConcurrentHashMap<>(); + private final ConcurrentMap> stateIdToState = new ConcurrentHashMap<>(); + + // Alias-ing only exists in the non-STREAM case where we have to convert existing state ids to one + // single global id. + // This only happens once. + private final Set aliasIds = new ConcurrentHashSet<>(); + private long retroactiveGlobalStateId = 0; + + public GlobalAsyncStateManager(final GlobalMemoryManager memoryManager) { + this.memoryManager = memoryManager; + memoryAllocated = new AtomicLong(memoryManager.requestMemory()); + memoryUsed = new AtomicLong(); + } + + // Always assume STREAM to begin, and convert only if needed. Most state is per stream anyway. + private AirbyteStateMessage.AirbyteStateType stateType = AirbyteStateMessage.AirbyteStateType.STREAM; + + /** + * Main method to process state messages. + *

    + * The first incoming state message tells us the type of state we are dealing with. We then convert + * internal data structures if needed. + *

    + * Because state messages are a watermark, all preceding records need to be flushed before the state + * message can be processed. + */ + public void trackState(final PartialAirbyteMessage message, final long sizeInBytes, final String defaultNamespace) { + if (preState) { + convertToGlobalIfNeeded(message); + preState = false; + } + // stateType should not change after a conversion. + Preconditions.checkArgument(stateType == extractStateType(message)); + + closeState(message, sizeInBytes, defaultNamespace); + } + + /** + * Identical to {@link #getStateId(StreamDescriptor)} except this increments the associated counter + * by 1. Intended to be called whenever a record is ingested. + * + * @param streamDescriptor - stream to get stateId for. + * @return state id + */ + public long getStateIdAndIncrementCounter(final StreamDescriptor streamDescriptor) { + return getStateIdAndIncrement(streamDescriptor, 1); + } + + /** + * Each decrement represent one written record for a state. A zero counter means there are no more + * inflight records associated with a state and the state can be flushed. + * + * @param stateId reference to a state. + * @param count to decrement. + */ + public void decrement(final long stateId, final long count) { + log.trace("decrementing state id: {}, count: {}", stateId, count); + stateIdToCounter.get(getStateAfterAlias(stateId)).addAndGet(-count); + } + + /** + * Returns state messages with no more inflight records i.e. counter = 0 across all streams. + * Intended to be called by {@link io.airbyte.cdk.integrations.destination_async.FlushWorkers} after + * a worker has finished flushing its record batch. + *

    + * The return list of states should be emitted back to the platform. + * + * @return list of state messages with no more inflight records. + */ + public List flushStates() { + final List output = new ArrayList<>(); + Long bytesFlushed = 0L; + synchronized (this) { + for (final Map.Entry> entry : descToStateIdQ.entrySet()) { + // Remove all states with 0 counters. + // Per-stream synchronized is required to make sure the state (at the head of the queue) + // logic is applied to is the state actually removed. + + final LinkedList stateIdQueue = entry.getValue(); + while (true) { + final Long oldestStateId = stateIdQueue.peek(); + // no state to flush for this stream + if (oldestStateId == null) { + break; + } + + // technically possible this map hasn't been updated yet. + final var oldestStateCounter = stateIdToCounter.get(oldestStateId); + Objects.requireNonNull(oldestStateCounter, "Invariant Violation: No record counter found for state message."); + + final var oldestState = stateIdToState.get(oldestStateId); + // no state to flush for this stream + if (oldestState == null) { + break; + } + + final var allRecordsCommitted = oldestStateCounter.get() == 0; + if (allRecordsCommitted) { + final PartialAirbyteMessage stateMessage = oldestState.getLeft(); + final double flushedRecordsAssociatedWithState = stateIdToCounterForPopulatingDestinationStats.get(oldestStateId).doubleValue(); + output.add(new PartialStateWithDestinationStats(stateMessage, + new AirbyteStateStats().withRecordCount(flushedRecordsAssociatedWithState))); + bytesFlushed += oldestState.getRight(); + + // cleanup + entry.getValue().poll(); + stateIdToState.remove(oldestStateId); + stateIdToCounter.remove(oldestStateId); + stateIdToCounterForPopulatingDestinationStats.remove(oldestStateId); + } else { + break; + } + } + } + } + + freeBytes(bytesFlushed); + return output; + } + + private Long getStateIdAndIncrement(final StreamDescriptor streamDescriptor, final long increment) { + final StreamDescriptor resolvedDescriptor = stateType == AirbyteStateMessage.AirbyteStateType.STREAM ? streamDescriptor : SENTINEL_GLOBAL_DESC; + // As concurrent collections do not guarantee data consistency when iterating, use `get` instead of + // `containsKey`. + if (descToStateIdQ.get(resolvedDescriptor) == null) { + registerNewStreamDescriptor(resolvedDescriptor); + } + final Long stateId = descToStateIdQ.get(resolvedDescriptor).peekLast(); + final var update = stateIdToCounter.get(stateId).addAndGet(increment); + if (increment >= 0) { + stateIdToCounterForPopulatingDestinationStats.get(stateId).addAndGet(increment); + } + log.trace("State id: {}, count: {}", stateId, update); + return stateId; + } + + /** + * Return the internal id of a state message. This is the id that should be used to reference a + * state when interacting with all methods in this class. + * + * @param streamDescriptor - stream to get stateId for. + * @return state id + */ + private long getStateId(final StreamDescriptor streamDescriptor) { + return getStateIdAndIncrement(streamDescriptor, 0); + } + + /** + * Pass this the number of bytes that were flushed. It will track those internally and if the + * memoryUsed gets signficantly lower than what is allocated, then it will return it to the memory + * manager. We don't always return to the memory manager to avoid needlessly allocating / + * de-allocating memory rapidly over a few bytes. + * + * @param bytesFlushed bytes that were flushed (and should be removed from memory used). + */ + private void freeBytes(final long bytesFlushed) { + LOGGER.debug("Bytes flushed memory to store state message. Allocated: {}, Used: {}, Flushed: {}, % Used: {}", + FileUtils.byteCountToDisplaySize(memoryAllocated.get()), + FileUtils.byteCountToDisplaySize(memoryUsed.get()), + FileUtils.byteCountToDisplaySize(bytesFlushed), + (double) memoryUsed.get() / memoryAllocated.get()); + + memoryManager.free(bytesFlushed); + memoryAllocated.addAndGet(-bytesFlushed); + memoryUsed.addAndGet(-bytesFlushed); + LOGGER.debug("Returned {} of memory back to the memory manager.", FileUtils.byteCountToDisplaySize(bytesFlushed)); + } + + private void convertToGlobalIfNeeded(final PartialAirbyteMessage message) { + // instead of checking for global or legacy, check for the inverse of stream. + stateType = extractStateType(message); + if (stateType != AirbyteStateMessage.AirbyteStateType.STREAM) {// alias old stream-level state ids to single global state id + // upon conversion, all previous tracking data structures need to be cleared as we move + // into the non-STREAM world for correctness. + + aliasIds.addAll(descToStateIdQ.values().stream().flatMap(Collection::stream).toList()); + descToStateIdQ.clear(); + retroactiveGlobalStateId = StateIdProvider.getNextId(); + + descToStateIdQ.put(SENTINEL_GLOBAL_DESC, new LinkedList<>()); + descToStateIdQ.get(SENTINEL_GLOBAL_DESC).add(retroactiveGlobalStateId); + + final long combinedCounter = stateIdToCounter.values() + .stream() + .mapToLong(AtomicLong::get) + .sum(); + stateIdToCounter.clear(); + stateIdToCounter.put(retroactiveGlobalStateId, new AtomicLong(combinedCounter)); + + stateIdToCounterForPopulatingDestinationStats.clear(); + stateIdToCounterForPopulatingDestinationStats.put(retroactiveGlobalStateId, new AtomicLong(combinedCounter)); + + } + } + + private AirbyteStateMessage.AirbyteStateType extractStateType(final PartialAirbyteMessage message) { + if (message.getState().getType() == null) { + // Treated the same as GLOBAL. + return AirbyteStateMessage.AirbyteStateType.LEGACY; + } else { + return message.getState().getType(); + } + } + + /** + * When a state message is received, 'close' the previous state to associate the existing state id + * to the newly arrived state message. We also increment the state id in preparation for the next + * state message. + */ + private void closeState(final PartialAirbyteMessage message, final long sizeInBytes, final String defaultNamespace) { + final StreamDescriptor resolvedDescriptor = extractStream(message, defaultNamespace).orElse(SENTINEL_GLOBAL_DESC); + stateIdToState.put(getStateId(resolvedDescriptor), ImmutablePair.of(message, sizeInBytes)); + registerNewStateId(resolvedDescriptor); + allocateMemoryToState(sizeInBytes); + } + + /** + * Given the size of a state message, tracks how much memory the manager is using and requests + * additional memory from the memory manager if needed. + * + * @param sizeInBytes size of the state message + */ + @SuppressWarnings("BusyWait") + private void allocateMemoryToState(final long sizeInBytes) { + if (memoryAllocated.get() < memoryUsed.get() + sizeInBytes) { + while (memoryAllocated.get() < memoryUsed.get() + sizeInBytes) { + memoryAllocated.addAndGet(memoryManager.requestMemory()); + try { + LOGGER.debug("Insufficient memory to store state message. Allocated: {}, Used: {}, Size of State Msg: {}, Needed: {}", + FileUtils.byteCountToDisplaySize(memoryAllocated.get()), + FileUtils.byteCountToDisplaySize(memoryUsed.get()), + FileUtils.byteCountToDisplaySize(sizeInBytes), + FileUtils.byteCountToDisplaySize(sizeInBytes - (memoryAllocated.get() - memoryUsed.get()))); + sleep(1000); + } catch (final InterruptedException e) { + throw new RuntimeException(e); + } + } + LOGGER.debug(getMemoryUsageMessage()); + } + } + + public String getMemoryUsageMessage() { + return String.format("State Manager memory usage: Allocated: %s, Used: %s, percentage Used %f", + FileUtils.byteCountToDisplaySize(memoryAllocated.get()), + FileUtils.byteCountToDisplaySize(memoryUsed.get()), + (double) memoryUsed.get() / memoryAllocated.get()); + } + + /** + * If the user has selected the Destination Namespace as the Destination default while setting up + * the connector, the platform sets the namespace as null in the StreamDescriptor in the + * AirbyteMessages (both record and state messages). The destination checks that if the namespace is + * empty or null, if yes then re-populates it with the defaultNamespace. See + * {@link io.airbyte.cdk.integrations.destination_async.AsyncStreamConsumer#accept(String,Integer)} + * But destination only does this for the record messages. So when state messages arrive without a + * namespace and since the destination doesn't repopulate it with the default namespace, there is a + * mismatch between the StreamDescriptor from record messages and state messages. That breaks the + * logic of the state management class as {@link descToStateIdQ} needs to have consistent + * StreamDescriptor. This is why while trying to extract the StreamDescriptor from state messages, + * we check if the namespace is null, if yes then replace it with defaultNamespace to keep it + * consistent with the record messages. + */ + private static Optional extractStream(final PartialAirbyteMessage message, final String defaultNamespace) { + if (message.getState().getType() != null && message.getState().getType() == AirbyteStateMessage.AirbyteStateType.STREAM) { + final StreamDescriptor streamDescriptor = message.getState().getStream().getStreamDescriptor(); + if (Strings.isNullOrEmpty(streamDescriptor.getNamespace())) { + return Optional.of(new StreamDescriptor().withName(streamDescriptor.getName()).withNamespace(defaultNamespace)); + } + return Optional.of(streamDescriptor); + } + return Optional.empty(); + } + + private long getStateAfterAlias(final long stateId) { + if (aliasIds.contains(stateId)) { + return retroactiveGlobalStateId; + } else { + return stateId; + } + } + + private void registerNewStreamDescriptor(final StreamDescriptor resolvedDescriptor) { + descToStateIdQ.put(resolvedDescriptor, new LinkedList<>()); + registerNewStateId(resolvedDescriptor); + } + + private void registerNewStateId(final StreamDescriptor resolvedDescriptor) { + final long stateId = StateIdProvider.getNextId(); + stateIdToCounter.put(stateId, new AtomicLong(0)); + stateIdToCounterForPopulatingDestinationStats.put(stateId, new AtomicLong(0)); + descToStateIdQ.get(resolvedDescriptor).add(stateId); + } + + /** + * Simplify internal tracking by providing a global always increasing counter for state ids. + */ + private static class StateIdProvider { + + private static final AtomicLong pk = new AtomicLong(0); + + public static long getNextId() { + return pk.incrementAndGet(); + } + + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/state/PartialStateWithDestinationStats.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/state/PartialStateWithDestinationStats.java new file mode 100644 index 000000000000..4270fdc00415 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/destination_async/state/PartialStateWithDestinationStats.java @@ -0,0 +1,10 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination_async.state; + +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStateStats; + +public record PartialStateWithDestinationStats(PartialAirbyteMessage stateMessage, AirbyteStateStats stats) {} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/ApmTraceUtils.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ApmTraceUtils.java similarity index 99% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/ApmTraceUtils.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ApmTraceUtils.java index 555c7d4dd6c8..cc1ff5935b38 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/ApmTraceUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ApmTraceUtils.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.util; +package io.airbyte.cdk.integrations.util; import datadog.trace.api.DDTags; import datadog.trace.api.interceptor.MutableSpan; diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ConnectorExceptionUtil.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ConnectorExceptionUtil.java new file mode 100644 index 000000000000..9f4ae86cfe78 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/ConnectorExceptionUtil.java @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.util; + +import static java.util.stream.Collectors.joining; + +import com.google.common.collect.ImmutableList; +import io.airbyte.cdk.integrations.base.errors.messages.ErrorMessage; +import io.airbyte.commons.exceptions.ConfigErrorException; +import io.airbyte.commons.exceptions.ConnectionErrorException; +import java.sql.SQLException; +import java.sql.SQLSyntaxErrorException; +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility class defining methods for handling configuration exceptions in connectors. + */ +public class ConnectorExceptionUtil { + + private static final Logger LOGGER = LoggerFactory.getLogger(ConnectorExceptionUtil.class); + + public static final String COMMON_EXCEPTION_MESSAGE_TEMPLATE = "Could not connect with provided configuration. Error: %s"; + static final String RECOVERY_CONNECTION_ERROR_MESSAGE = + "We're having issues syncing from a Postgres replica that is configured as a hot standby server. " + + "Please see https://go.airbyte.com/pg-hot-standby-error-message for options and workarounds"; + + public static final List HTTP_AUTHENTICATION_ERROR_CODES = ImmutableList.of(401, 403); + + public static boolean isConfigError(final Throwable e) { + return isConfigErrorException(e) || isConnectionError(e) || + isRecoveryConnectionException(e) || isUnknownColumnInFieldListException(e); + } + + public static String getDisplayMessage(final Throwable e) { + if (e instanceof ConfigErrorException) { + return ((ConfigErrorException) e).getDisplayMessage(); + } else if (e instanceof final ConnectionErrorException connEx) { + return ErrorMessage.getErrorMessage(connEx.getStateCode(), connEx.getErrorCode(), connEx.getExceptionMessage(), connEx); + } else if (isRecoveryConnectionException(e)) { + return RECOVERY_CONNECTION_ERROR_MESSAGE; + } else if (isUnknownColumnInFieldListException(e)) { + return e.getMessage(); + } else { + return String.format(COMMON_EXCEPTION_MESSAGE_TEMPLATE, e.getMessage() != null ? e.getMessage() : ""); + } + } + + /** + * Returns the first instance of an exception associated with a configuration error (if it exists). + * Otherwise, the original exception is returned. + */ + public static Throwable getRootConfigError(final Exception e) { + Throwable current = e; + while (current != null) { + if (ConnectorExceptionUtil.isConfigError(current)) { + return current; + } else { + current = current.getCause(); + } + } + return e; + } + + /** + * Log all the exceptions, and rethrow the first. This is useful for e.g. running multiple futures + * and waiting for them to complete/fail. Rather than combining them into a single mega-exception + * (which works poorly in the UI), we just log all of them, and throw the first exception. + *

    + * In most cases, all the exceptions will look very similar, so the user only needs to see the first + * exception anyway. This mimics e.g. a for-loop over multiple tasks, where the loop would break on + * the first exception. + */ + public static void logAllAndThrowFirst(final String initialMessage, final Collection throwables) throws T { + if (!throwables.isEmpty()) { + final String stacktraces = throwables.stream().map(ExceptionUtils::getStackTrace).collect(joining("\n")); + LOGGER.error(initialMessage + stacktraces + "\nRethrowing first exception."); + throw throwables.iterator().next(); + } + } + + private static boolean isConfigErrorException(Throwable e) { + return e instanceof ConfigErrorException; + } + + private static boolean isConnectionError(Throwable e) { + return e instanceof ConnectionErrorException; + } + + private static boolean isRecoveryConnectionException(Throwable e) { + return e instanceof SQLException && e.getMessage() + .toLowerCase(Locale.ROOT) + .contains("due to conflict with recovery"); + } + + private static boolean isUnknownColumnInFieldListException(Throwable e) { + return e instanceof SQLSyntaxErrorException + && e.getMessage() + .toLowerCase(Locale.ROOT) + .contains("unknown column") + && e.getMessage() + .toLowerCase(Locale.ROOT) + .contains("in 'field list'"); + } + +} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/HostPortResolver.java similarity index 96% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/HostPortResolver.java index f0adef160f77..7b8b4f4ccfc1 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/HostPortResolver.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/HostPortResolver.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.util; +package io.airbyte.cdk.integrations.util; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/PostgresSslConnectionUtils.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/PostgresSslConnectionUtils.java similarity index 99% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/PostgresSslConnectionUtils.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/PostgresSslConnectionUtils.java index d5ab7f8c7a70..ad325ab808bd 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/PostgresSslConnectionUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/PostgresSslConnectionUtils.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.util; +package io.airbyte.cdk.integrations.util; import com.fasterxml.jackson.databind.JsonNode; import java.io.BufferedReader; diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/concurrent/ConcurrentStreamConsumer.java b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/concurrent/ConcurrentStreamConsumer.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/concurrent/ConcurrentStreamConsumer.java rename to airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/concurrent/ConcurrentStreamConsumer.java index 7d9bdb15ead7..3830ebf42eac 100644 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/concurrent/ConcurrentStreamConsumer.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/util/concurrent/ConcurrentStreamConsumer.java @@ -2,12 +2,12 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.util.concurrent; +package io.airbyte.cdk.integrations.util.concurrent; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; import io.airbyte.commons.stream.AirbyteStreamStatusHolder; import io.airbyte.commons.stream.StreamStatusUtils; import io.airbyte.commons.util.AutoCloseableIterator; -import io.airbyte.integrations.base.AirbyteTraceMessageUtility; import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.AirbyteMessage; import java.util.ArrayList; diff --git a/airbyte-integrations/bases/base-java/src/main/resources/AirbyteLogMessageTemplate.json b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/AirbyteLogMessageTemplate.json similarity index 100% rename from airbyte-integrations/bases/base-java/src/main/resources/AirbyteLogMessageTemplate.json rename to airbyte-cdk/java/airbyte-cdk/core/src/main/resources/AirbyteLogMessageTemplate.json diff --git a/airbyte-integrations/bases/base-java/src/main/resources/bastion/Dockerfile b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/bastion/Dockerfile similarity index 100% rename from airbyte-integrations/bases/base-java/src/main/resources/bastion/Dockerfile rename to airbyte-cdk/java/airbyte-cdk/core/src/main/resources/bastion/Dockerfile diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/log4j2.xml b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/log4j2.xml new file mode 100644 index 000000000000..696a4af4451d --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/log4j2.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/airbyte-integrations/bases/base-java/src/main/resources/ssh-tunnel-spec.json b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/ssh-tunnel-spec.json similarity index 100% rename from airbyte-integrations/bases/base-java/src/main/resources/ssh-tunnel-spec.json rename to airbyte-cdk/java/airbyte-cdk/core/src/main/resources/ssh-tunnel-spec.json diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties new file mode 100644 index 000000000000..b18dfa7feb69 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties @@ -0,0 +1 @@ +version=0.13.2 diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/IncrementalUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/IncrementalUtilsTest.java similarity index 99% rename from airbyte-db/db-lib/src/test/java/io/airbyte/db/IncrementalUtilsTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/IncrementalUtilsTest.java index c82e22ae7c8a..830fa3743539 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/IncrementalUtilsTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/IncrementalUtilsTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db; +package io.airbyte.cdk.db; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/PgLsnTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/PgLsnTest.java similarity index 98% rename from airbyte-db/db-lib/src/test/java/io/airbyte/db/PgLsnTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/PgLsnTest.java index ec09e2977ff0..76ab1a48ebbb 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/PgLsnTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/PgLsnTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db; +package io.airbyte.cdk.db; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/PostgresUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/PostgresUtilsTest.java similarity index 90% rename from airbyte-db/db-lib/src/test/java/io/airbyte/db/PostgresUtilsTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/PostgresUtilsTest.java index ff471c5db0f0..471cbb8bb9ca 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/PostgresUtilsTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/PostgresUtilsTest.java @@ -2,22 +2,22 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db; +package io.airbyte.cdk.db; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.PostgreSQLContainerHelper; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.sql.SQLException; import javax.sql.DataSource; import org.junit.jupiter.api.BeforeAll; diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/check/impl/CommonDatabaseCheckTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/check/impl/CommonDatabaseCheckTest.java similarity index 88% rename from airbyte-db/db-lib/src/test/java/io/airbyte/db/check/impl/CommonDatabaseCheckTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/check/impl/CommonDatabaseCheckTest.java index 4c04d0a81391..37086e620289 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/check/impl/CommonDatabaseCheckTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/check/impl/CommonDatabaseCheckTest.java @@ -2,10 +2,10 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.check.impl; +package io.airbyte.cdk.db.check.impl; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DataSourceFactory; import javax.sql.DataSource; import org.jooq.DSLContext; import org.jooq.SQLDialect; diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/factory/CommonFactoryTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/factory/CommonFactoryTest.java similarity index 92% rename from airbyte-db/db-lib/src/test/java/io/airbyte/db/factory/CommonFactoryTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/factory/CommonFactoryTest.java index 61f1d75c3b5d..04dae353989d 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/factory/CommonFactoryTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/factory/CommonFactoryTest.java @@ -2,14 +2,14 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.factory; +package io.airbyte.cdk.db.factory; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.testcontainers.containers.PostgreSQLContainer; /** - * Common test suite for the classes found in the {@code io.airbyte.db.factory} package. + * Common test suite for the classes found in the {@code io.airbyte.cdk.db.factory} package. */ class CommonFactoryTest { diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/factory/DSLContextFactoryTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/factory/DSLContextFactoryTest.java similarity index 92% rename from airbyte-db/db-lib/src/test/java/io/airbyte/db/factory/DSLContextFactoryTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/factory/DSLContextFactoryTest.java index 8bcfbd66aa75..d673b71cfa56 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/factory/DSLContextFactoryTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/factory/DSLContextFactoryTest.java @@ -2,11 +2,12 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.factory; +package io.airbyte.cdk.db.factory; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import io.airbyte.cdk.integrations.JdbcConnector; import java.util.Map; import javax.sql.DataSource; import org.jooq.DSLContext; @@ -51,7 +52,8 @@ void testCreatingADslContextWithIndividualConfigurationAndConnectionProperties() container.getDriverClassName(), container.getJdbcUrl(), dialect, - connectionProperties); + connectionProperties, + JdbcConnector.CONNECT_TIMEOUT_DEFAULT); assertNotNull(dslContext); assertEquals(dialect, dslContext.configuration().dialect()); } diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/factory/DataSourceFactoryTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/factory/DataSourceFactoryTest.java similarity index 85% rename from airbyte-db/db-lib/src/test/java/io/airbyte/db/factory/DataSourceFactoryTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/factory/DataSourceFactoryTest.java index 8a7f7cc1cfce..db8850af63a4 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/factory/DataSourceFactoryTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/factory/DataSourceFactoryTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.factory; +package io.airbyte.cdk.db.factory; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -12,6 +12,7 @@ import static org.mockito.Mockito.verify; import com.zaxxer.hikari.HikariDataSource; +import io.airbyte.cdk.integrations.JdbcConnector; import java.util.Map; import javax.sql.DataSource; import org.junit.jupiter.api.Assertions; @@ -54,7 +55,8 @@ void testCreatingDataSourceWithConnectionTimeoutSetAboveDefault() { password, driverClassName, jdbcUrl, - connectionProperties); + connectionProperties, + JdbcConnector.getConnectionTimeout(connectionProperties, driverClassName)); assertNotNull(dataSource); assertEquals(HikariDataSource.class, dataSource.getClass()); assertEquals(61000, ((HikariDataSource) dataSource).getHikariConfigMXBean().getConnectionTimeout()); @@ -69,7 +71,8 @@ void testCreatingPostgresDataSourceWithConnectionTimeoutSetBelowDefault() { password, driverClassName, jdbcUrl, - connectionProperties); + connectionProperties, + JdbcConnector.getConnectionTimeout(connectionProperties, driverClassName)); assertNotNull(dataSource); assertEquals(HikariDataSource.class, dataSource.getClass()); assertEquals(30000, ((HikariDataSource) dataSource).getHikariConfigMXBean().getConnectionTimeout()); @@ -80,16 +83,17 @@ void testCreatingMySQLDataSourceWithConnectionTimeoutSetBelowDefault() { try (MySQLContainer mySQLContainer = new MySQLContainer<>("mysql:8.0")) { mySQLContainer.start(); final Map connectionProperties = Map.of( - CONNECT_TIMEOUT, "30"); + CONNECT_TIMEOUT, "5000"); final DataSource dataSource = DataSourceFactory.create( mySQLContainer.getUsername(), mySQLContainer.getPassword(), mySQLContainer.getDriverClassName(), mySQLContainer.getJdbcUrl(), - connectionProperties); + connectionProperties, + JdbcConnector.getConnectionTimeout(connectionProperties, mySQLContainer.getDriverClassName())); assertNotNull(dataSource); assertEquals(HikariDataSource.class, dataSource.getClass()); - assertEquals(60000, ((HikariDataSource) dataSource).getHikariConfigMXBean().getConnectionTimeout()); + assertEquals(5000, ((HikariDataSource) dataSource).getHikariConfigMXBean().getConnectionTimeout()); } } @@ -102,7 +106,8 @@ void testCreatingDataSourceWithConnectionTimeoutSetWithZero() { password, driverClassName, jdbcUrl, - connectionProperties); + connectionProperties, + JdbcConnector.getConnectionTimeout(connectionProperties, driverClassName)); assertNotNull(dataSource); assertEquals(HikariDataSource.class, dataSource.getClass()); assertEquals(Integer.MAX_VALUE, ((HikariDataSource) dataSource).getHikariConfigMXBean().getConnectionTimeout()); @@ -116,7 +121,8 @@ void testCreatingPostgresDataSourceWithConnectionTimeoutNotSet() { password, driverClassName, jdbcUrl, - connectionProperties); + connectionProperties, + JdbcConnector.getConnectionTimeout(connectionProperties, driverClassName)); assertNotNull(dataSource); assertEquals(HikariDataSource.class, dataSource.getClass()); assertEquals(10000, ((HikariDataSource) dataSource).getHikariConfigMXBean().getConnectionTimeout()); @@ -132,7 +138,8 @@ void testCreatingMySQLDataSourceWithConnectionTimeoutNotSet() { mySQLContainer.getPassword(), mySQLContainer.getDriverClassName(), mySQLContainer.getJdbcUrl(), - connectionProperties); + connectionProperties, + JdbcConnector.getConnectionTimeout(connectionProperties, mySQLContainer.getDriverClassName())); assertNotNull(dataSource); assertEquals(HikariDataSource.class, dataSource.getClass()); assertEquals(60000, ((HikariDataSource) dataSource).getHikariConfigMXBean().getConnectionTimeout()); @@ -152,7 +159,13 @@ void testCreatingADataSourceWithJdbcUrl() { void testCreatingADataSourceWithJdbcUrlAndConnectionProperties() { final Map connectionProperties = Map.of("foo", "bar"); - final DataSource dataSource = DataSourceFactory.create(username, password, driverClassName, jdbcUrl, connectionProperties); + final DataSource dataSource = DataSourceFactory.create( + username, + password, + driverClassName, + jdbcUrl, + connectionProperties, + JdbcConnector.getConnectionTimeout(connectionProperties, driverClassName)); assertNotNull(dataSource); assertEquals(HikariDataSource.class, dataSource.getClass()); assertEquals(10, ((HikariDataSource) dataSource).getHikariConfigMXBean().getMaximumPoolSize()); diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestDefaultJdbcDatabase.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/TestDefaultJdbcDatabase.java similarity index 95% rename from airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestDefaultJdbcDatabase.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/TestDefaultJdbcDatabase.java index c9f0e596e8b9..8d4c2b9a0728 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestDefaultJdbcDatabase.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/TestDefaultJdbcDatabase.java @@ -2,19 +2,19 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc; +package io.airbyte.cdk.db.jdbc; import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.testutils.PostgreSQLContainerHelper; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.sql.SQLException; import java.util.List; import java.util.stream.Stream; diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/TestJdbcUtils.java similarity index 97% rename from airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/TestJdbcUtils.java index 4c2ffd9dec13..317566be30ec 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestJdbcUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/TestJdbcUtils.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc; +package io.airbyte.cdk.db.jdbc; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -11,18 +11,19 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.BinaryNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Charsets; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.testutils.PostgreSQLContainerHelper; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.stream.MoreStreams; import io.airbyte.commons.string.Strings; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.math.BigDecimal; import java.sql.Connection; import java.sql.JDBCType; @@ -35,6 +36,7 @@ import java.util.Map; import java.util.stream.Collectors; import javax.sql.DataSource; +import org.bouncycastle.util.encoders.Base64; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -376,6 +378,8 @@ private ObjectNode jsonFieldExpectedValues() { arrayNode2.add("3"); expected.set("int_array", arrayNode2); + expected.set("binary1", new BinaryNode("aaaa".getBytes(Charsets.UTF_8))); + return expected; } @@ -396,7 +400,7 @@ private ObjectNode expectedValues() { expected.put("date", "2020-11-01"); expected.put("time", "05:00:00.000000"); expected.put("timestamp", "2001-09-29T03:00:00.000000"); - expected.put("binary1", "aaaa".getBytes(Charsets.UTF_8)); + expected.put("binary1", Base64.decode("61616161".getBytes(Charsets.UTF_8))); return expected; } diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestStreamingJdbcDatabase.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/TestStreamingJdbcDatabase.java similarity index 95% rename from airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestStreamingJdbcDatabase.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/TestStreamingJdbcDatabase.java index bf5450ffcc36..c33db8f0d522 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/TestStreamingJdbcDatabase.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/TestStreamingJdbcDatabase.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc; +package io.airbyte.cdk.db.jdbc; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -11,14 +11,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; +import io.airbyte.cdk.db.jdbc.streaming.FetchSizeConstants; +import io.airbyte.cdk.testutils.PostgreSQLContainerHelper; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; -import io.airbyte.db.jdbc.streaming.FetchSizeConstants; -import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/AdaptiveStreamingQueryConfigTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/streaming/AdaptiveStreamingQueryConfigTest.java similarity index 96% rename from airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/AdaptiveStreamingQueryConfigTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/streaming/AdaptiveStreamingQueryConfigTest.java index 418d6b65917b..2123206c8763 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/AdaptiveStreamingQueryConfigTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/streaming/AdaptiveStreamingQueryConfigTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc.streaming; +package io.airbyte.cdk.db.jdbc.streaming; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.mock; diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/BaseSizeEstimatorTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/streaming/BaseSizeEstimatorTest.java similarity index 98% rename from airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/BaseSizeEstimatorTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/streaming/BaseSizeEstimatorTest.java index a6cc69b4a8e7..a2a89f960269 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/BaseSizeEstimatorTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/streaming/BaseSizeEstimatorTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc.streaming; +package io.airbyte.cdk.db.jdbc.streaming; import static org.junit.jupiter.api.Assertions.*; diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/InitialSizeEstimatorTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/streaming/InitialSizeEstimatorTest.java similarity index 97% rename from airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/InitialSizeEstimatorTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/streaming/InitialSizeEstimatorTest.java index ae5555aaa7e2..f4031f085fc3 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/InitialSizeEstimatorTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/streaming/InitialSizeEstimatorTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc.streaming; +package io.airbyte.cdk.db.jdbc.streaming; import static org.junit.jupiter.api.Assertions.*; diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/SamplingSizeEstimatorTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/streaming/SamplingSizeEstimatorTest.java similarity index 98% rename from airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/SamplingSizeEstimatorTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/streaming/SamplingSizeEstimatorTest.java index 1470d5b0d828..75ba6c872318 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/SamplingSizeEstimatorTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/streaming/SamplingSizeEstimatorTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc.streaming; +package io.airbyte.cdk.db.jdbc.streaming; import static org.junit.jupiter.api.Assertions.*; diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/TwoStageSizeEstimatorTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/streaming/TwoStageSizeEstimatorTest.java similarity index 96% rename from airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/TwoStageSizeEstimatorTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/streaming/TwoStageSizeEstimatorTest.java index c6da06117e99..a3314817a310 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/jdbc/streaming/TwoStageSizeEstimatorTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/jdbc/streaming/TwoStageSizeEstimatorTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.jdbc.streaming; +package io.airbyte.cdk.db.jdbc.streaming; import static org.junit.jupiter.api.Assertions.*; diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/util/SSLCertificateUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/util/SSLCertificateUtilsTest.java similarity index 99% rename from airbyte-db/db-lib/src/test/java/io/airbyte/db/util/SSLCertificateUtilsTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/util/SSLCertificateUtilsTest.java index 1e214850890b..b0387077245e 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/util/SSLCertificateUtilsTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/db/util/SSLCertificateUtilsTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.util; +package io.airbyte.cdk.db.util; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandlerTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandlerTest.java new file mode 100644 index 000000000000..25812410a01a --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteExceptionHandlerTest.java @@ -0,0 +1,199 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.base; + +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.spy; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.protocol.models.AirbyteErrorTraceMessage; +import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.AirbyteTraceMessage; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Optional; +import lombok.SneakyThrows; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class AirbyteExceptionHandlerTest { + + PrintStream originalOut = System.out; + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private AirbyteExceptionHandler airbyteExceptionHandler; + + @BeforeEach + public void setup() { + System.setOut(new PrintStream(outContent, true, StandardCharsets.UTF_8)); + + // mocking terminate() method in AirbyteExceptionHandler, so we don't kill the JVM + airbyteExceptionHandler = spy(new AirbyteExceptionHandler()); + doNothing().when(airbyteExceptionHandler).terminate(); + + AirbyteExceptionHandler.addThrowableForDeinterpolation(RuntimeException.class); + } + + @Test + void testTraceMessageEmission() throws Exception { + runTestWithMessage("error"); + + final AirbyteMessage traceMessage = findFirstTraceMessage(); + assertAll( + () -> assertEquals(AirbyteTraceMessage.Type.ERROR, traceMessage.getTrace().getType()), + () -> assertEquals(AirbyteExceptionHandler.logMessage, traceMessage.getTrace().getError().getMessage()), + () -> assertEquals(AirbyteErrorTraceMessage.FailureType.SYSTEM_ERROR, traceMessage.getTrace().getError().getFailureType())); + } + + @Test + void testMessageDeinterpolation() throws Exception { + AirbyteExceptionHandler.addStringForDeinterpolation("foo"); + AirbyteExceptionHandler.addStringForDeinterpolation("bar"); + + // foo and bar are added to the list explicitly + // name and description are added implicitly by the exception handler. + // all of them should be replaced by '?' + runTestWithMessage("Error happened in arst_foo_bar_zxcv (name: description)"); + + final AirbyteMessage traceMessage = findFirstTraceMessage(); + assertAll( + () -> assertEquals(AirbyteTraceMessage.Type.ERROR, traceMessage.getTrace().getType()), + () -> assertEquals("Error happened in arst_foo_bar_zxcv (name: description)", traceMessage.getTrace().getError().getMessage()), + () -> assertEquals("Error happened in arst_?_?_zxcv (?: ?)", traceMessage.getTrace().getError().getInternalMessage()), + () -> assertEquals(AirbyteErrorTraceMessage.FailureType.SYSTEM_ERROR, traceMessage.getTrace().getError().getFailureType()), + () -> Assertions.assertNull(traceMessage.getTrace().getError().getStackTrace(), + "Stacktrace should be null if deinterpolating the error message")); + } + + /** + * We should only deinterpolate whole words, i.e. if the target string is not adjacent to an + * alphanumeric character. + */ + @Test + void testMessageSmartDeinterpolation() throws Exception { + AirbyteExceptionHandler.addStringForDeinterpolation("foo"); + AirbyteExceptionHandler.addStringForDeinterpolation("bar"); + + runTestWithMessage("Error happened in foobar"); + + final AirbyteMessage traceMessage = findFirstTraceMessage(); + // We shouldn't deinterpolate at all in this case, so we will get the default trace message + // behavior. + assertAll( + () -> assertEquals(AirbyteExceptionHandler.logMessage, traceMessage.getTrace().getError().getMessage()), + () -> assertEquals( + "java.lang.RuntimeException: Error happened in foobar", + traceMessage.getTrace().getError().getInternalMessage())); + } + + /** + * When one of the target strings is a substring of another, we should not deinterpolate the + * substring. + */ + @Test + void testMessageSubstringDeinterpolation() throws Exception { + AirbyteExceptionHandler.addStringForDeinterpolation("airbyte"); + AirbyteExceptionHandler.addStringForDeinterpolation("airbyte_internal"); + + runTestWithMessage("Error happened in airbyte_internal.foo"); + + final AirbyteMessage traceMessage = findFirstTraceMessage(); + assertEquals("Error happened in ?.foo", traceMessage.getTrace().getError().getInternalMessage()); + } + + /** + * We should only deinterpolate specific exception classes. + */ + @Test + void testClassDeinterpolation() throws Exception { + AirbyteExceptionHandler.addStringForDeinterpolation("foo"); + + runTestWithMessage(new IOException("Error happened in foo")); + + final AirbyteMessage traceMessage = findFirstTraceMessage(); + // We shouldn't deinterpolate at all in this case, so we will get the default trace message + // behavior. + assertAll( + () -> assertEquals(AirbyteExceptionHandler.logMessage, traceMessage.getTrace().getError().getMessage()), + () -> assertEquals( + "java.io.IOException: Error happened in foo", + traceMessage.getTrace().getError().getInternalMessage())); + } + + /** + * We should check the classes of the entire exception chain, not just the root exception. + */ + @Test + void testNestedThrowableClassDeinterpolation() throws Exception { + AirbyteExceptionHandler.addStringForDeinterpolation("foo"); + + runTestWithMessage(new Exception(new RuntimeException("Error happened in foo"))); + + final AirbyteMessage traceMessage = findFirstTraceMessage(); + // We shouldn't deinterpolate at all in this case, so we will get the default trace message + // behavior. + assertEquals("Error happened in ?", traceMessage.getTrace().getError().getInternalMessage()); + } + + private void runTestWithMessage(final String message) throws InterruptedException { + runTestWithMessage(new RuntimeException(message)); + } + + private void runTestWithMessage(final Throwable throwable) throws InterruptedException { + // have to spawn a new thread to test the uncaught exception handling, + // because junit catches any exceptions in main thread, i.e. they're not 'uncaught' + final Thread thread = new Thread() { + + @SneakyThrows + public void run() { + final IntegrationRunner runner = Mockito.mock(IntegrationRunner.class); + doThrow(throwable).when(runner).run(new String[] {"write"}); + runner.run(new String[] {"write"}); + } + + }; + thread.setUncaughtExceptionHandler(airbyteExceptionHandler); + thread.start(); + thread.join(); + System.out.flush(); + } + + @AfterEach + public void teardown() { + System.setOut(originalOut); + + AirbyteExceptionHandler.STRINGS_TO_DEINTERPOLATE.clear(); + AirbyteExceptionHandler.addCommonStringsToDeinterpolate(); + + AirbyteExceptionHandler.THROWABLES_TO_DEINTERPOLATE.clear(); + } + + private AirbyteMessage findFirstTraceMessage() { + final Optional maybeTraceMessage = Arrays.stream(outContent.toString(StandardCharsets.UTF_8).split("\n")) + .map(line -> { + // these tests sometimes emit non-json stdout (e.g. log4j warnings) + // so we try-catch to handle those malformed lines. + try { + return Jsons.deserialize(line, AirbyteMessage.class); + } catch (final Exception e) { + return null; + } + }) + .filter(message -> message != null && message.getType() == AirbyteMessage.Type.TRACE) + .findFirst(); + assertTrue(maybeTraceMessage.isPresent(), "Expected to find a trace message in stdout"); + return maybeTraceMessage.get(); + } + +} diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/AirbyteLogMessageTemplateTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteLogMessageTemplateTest.java similarity index 77% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/AirbyteLogMessageTemplateTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteLogMessageTemplateTest.java index 6862221f3d8e..39795319dbf7 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/AirbyteLogMessageTemplateTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteLogMessageTemplateTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -20,15 +20,17 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.appender.OutputStreamAppender; import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.Configurator; import org.apache.logging.log4j.core.config.LoggerConfig; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import org.junit.platform.commons.util.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,6 +43,7 @@ public class AirbyteLogMessageTemplateTest { public static final String CONSOLE_JSON_APPENDER = "ConsoleJSONAppender"; private static OutputStreamAppender outputStreamAppender; private static LoggerConfig rootLoggerConfig; + private static LoggerContext loggerContext; @BeforeAll static void init() { @@ -48,7 +51,7 @@ static void init() { // as the console json appender defined in this project's log4j2.xml file. // We then attach this log appender with the LOGGER instance so that we can validate the logs // produced by code and assert that it matches the expected format. - final LoggerContext loggerContext = (LoggerContext) LogManager.getContext(false); + loggerContext = Configurator.initialize(null, "log4j2.xml"); final Configuration configuration = loggerContext.getConfiguration(); rootLoggerConfig = configuration.getLoggerConfig(""); @@ -69,6 +72,7 @@ void setup() { static void cleanUp() { outputStreamAppender.stop(); rootLoggerConfig.removeAppender(OUTPUT_STREAM_APPENDER); + loggerContext.close(); } @Test @@ -107,4 +111,25 @@ private AirbyteLogMessage validateAirbyteMessageIsLog(final AirbyteMessage airby return airbyteMessage.getLog(); } + @ParameterizedTest + @ValueSource(ints = {2, 100, 9000}) + public void testAirbyteLogMessageLength(int stringRepeatitions) throws java.io.IOException { + final StringBuilder sb = new StringBuilder(); + for (int i = 0; i < stringRepeatitions; i++) { + sb.append("abcd"); + } + LOGGER.info(sb.toString(), new RuntimeException("aaaaa bbbbbb ccccccc dddddd")); + outputContent.flush(); + final String logMessage = outputContent.toString(StandardCharsets.UTF_8); + + final AirbyteMessage airbyteMessage = validateLogIsAirbyteMessage(logMessage); + final AirbyteLogMessage airbyteLogMessage = validateAirbyteMessageIsLog(airbyteMessage); + final String connectorLogMessage = airbyteLogMessage.getMessage(); + + // #30781 - message length is capped at 16,000 charcters. + int j = connectorLogMessage.length(); + assertFalse(connectorLogMessage.length() > 16_001); + assertTrue(logMessage.length() < 32768); + } + } diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/AirbyteTraceMessageUtilityTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteTraceMessageUtilityTest.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/AirbyteTraceMessageUtilityTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteTraceMessageUtilityTest.java index c5f7db19131a..f75f7a01ac99 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/AirbyteTraceMessageUtilityTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/AirbyteTraceMessageUtilityTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/DestinationConfigTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/DestinationConfigTest.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/DestinationConfigTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/DestinationConfigTest.java index 00182f989c26..2d06503baf20 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/DestinationConfigTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/DestinationConfigTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/FailureTrackingAirbyteMessageConsumerTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/FailureTrackingAirbyteMessageConsumerTest.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/FailureTrackingAirbyteMessageConsumerTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/FailureTrackingAirbyteMessageConsumerTest.java index dba9eb0483a4..ad3abb170338 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/FailureTrackingAirbyteMessageConsumerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/FailureTrackingAirbyteMessageConsumerTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/IntegrationCliParserTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/IntegrationCliParserTest.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/IntegrationCliParserTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/IntegrationCliParserTest.java index 384e13347fde..2eac0ed6e631 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/IntegrationCliParserTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/IntegrationCliParserTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/IntegrationConfigTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/IntegrationConfigTest.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/IntegrationConfigTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/IntegrationConfigTest.java index 926fca719906..ae76a8b27afb 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/IntegrationConfigTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/IntegrationConfigTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/IntegrationRunnerBackwardsCompatbilityTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/IntegrationRunnerBackwardsCompatbilityTest.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/IntegrationRunnerBackwardsCompatbilityTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/IntegrationRunnerBackwardsCompatbilityTest.java index 7626a7df576a..e0fef28b0792 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/IntegrationRunnerBackwardsCompatbilityTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/IntegrationRunnerBackwardsCompatbilityTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/IntegrationRunnerTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/IntegrationRunnerTest.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/IntegrationRunnerTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/IntegrationRunnerTest.java index 8f2aaf57615c..742c01e32463 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/IntegrationRunnerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/IntegrationRunnerTest.java @@ -2,10 +2,10 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; -import static io.airbyte.integrations.base.IntegrationRunner.ORPHANED_THREAD_FILTER; -import static io.airbyte.integrations.util.ConnectorExceptionUtil.COMMON_EXCEPTION_MESSAGE_TEMPLATE; +import static io.airbyte.cdk.integrations.base.IntegrationRunner.ORPHANED_THREAD_FILTER; +import static io.airbyte.cdk.integrations.util.ConnectorExceptionUtil.COMMON_EXCEPTION_MESSAGE_TEMPLATE; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; import static org.assertj.core.api.AssertionsForClassTypes.catchThrowable; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -23,12 +23,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.base.Destination.ShimToSerializedAirbyteMessageConsumer; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterators; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.integrations.base.Destination.ShimToSerializedAirbyteMessageConsumer; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/NameTransformerTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/NameTransformerTest.java similarity index 95% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/NameTransformerTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/NameTransformerTest.java index 10e779471311..72ee1cfca98a 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/NameTransformerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/NameTransformerTest.java @@ -2,12 +2,12 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base; +package io.airbyte.cdk.integrations.base; import static org.junit.jupiter.api.Assertions.assertEquals; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import org.junit.jupiter.api.Test; class NameTransformerTest { diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/normalization/NormalizationLogParserTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/normalization/NormalizationLogParserTest.java similarity index 96% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/normalization/NormalizationLogParserTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/normalization/NormalizationLogParserTest.java index 44c9dc74f585..7f8a01af1293 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/normalization/NormalizationLogParserTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/normalization/NormalizationLogParserTest.java @@ -2,11 +2,11 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base.normalization; +package io.airbyte.cdk.integrations.base.normalization; import static org.junit.jupiter.api.Assertions.assertEquals; -import io.airbyte.integrations.destination.normalization.NormalizationLogParser; +import io.airbyte.cdk.integrations.destination.normalization.NormalizationLogParser; import io.airbyte.protocol.models.AirbyteErrorTraceMessage; import io.airbyte.protocol.models.AirbyteErrorTraceMessage.FailureType; import io.airbyte.protocol.models.AirbyteLogMessage; diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/ssh/SshTunnelTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/ssh/SshTunnelTest.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/ssh/SshTunnelTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/ssh/SshTunnelTest.java index 8f5f1a003ecc..06c68a50bdfe 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/ssh/SshTunnelTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/base/ssh/SshTunnelTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.base.ssh; +package io.airbyte.cdk.integrations.base.ssh; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -11,8 +11,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel.TunnelMethod; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.ssh.SshTunnel.TunnelMethod; import java.nio.charset.StandardCharsets; import java.security.KeyPair; import java.security.PrivateKey; diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java similarity index 97% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java index eae9d74b83a8..fc5717ab04fa 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/BufferedStreamConsumerTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.buffered_stream_consumer; +package io.airbyte.cdk.integrations.destination.buffered_stream_consumer; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; @@ -17,11 +17,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.destination.record_buffer.BufferFlushType; +import io.airbyte.cdk.integrations.destination.record_buffer.BufferingStrategy; +import io.airbyte.cdk.integrations.destination.record_buffer.InMemoryRecordBufferingStrategy; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.record_buffer.BufferFlushType; -import io.airbyte.integrations.destination.record_buffer.BufferingStrategy; -import io.airbyte.integrations.destination.record_buffer.InMemoryRecordBufferingStrategy; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteGlobalState; @@ -37,6 +37,7 @@ import java.time.Duration; import java.time.Instant; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Optional; import java.util.concurrent.TimeUnit; @@ -237,7 +238,7 @@ void testExceptionAfterNoStateMessages() throws Exception { @Test void testExceptionDuringOnClose() throws Exception { - doThrow(new IllegalStateException("induced exception")).when(onClose).accept(false); + doThrow(new IllegalStateException("induced exception")).when(onClose).accept(false, new HashMap<>()); final List expectedRecordsBatch1 = generateRecords(1_000); final List expectedRecordsBatch2 = generateRecords(1_000); @@ -507,13 +508,13 @@ private BufferedStreamConsumer getConsumerWithFlushFrequency() { private void verifyStartAndClose() throws Exception { verify(onStart).call(); - verify(onClose).accept(false); + verify(onClose).accept(false, new HashMap<>()); } /** Indicates that a failure occurred while consuming AirbyteMessages */ private void verifyStartAndCloseFailure() throws Exception { verify(onStart).call(); - verify(onClose).accept(true); + verify(onClose).accept(true, new HashMap<>()); } private static void consumeRecords(final BufferedStreamConsumer consumer, final Collection records) { diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/buffered_stream_consumer/RecordSizeEstimatorTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/RecordSizeEstimatorTest.java similarity index 97% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/buffered_stream_consumer/RecordSizeEstimatorTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/RecordSizeEstimatorTest.java index 478398d12aa1..5a9d2d4bba13 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/buffered_stream_consumer/RecordSizeEstimatorTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/buffered_stream_consumer/RecordSizeEstimatorTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.buffered_stream_consumer; +package io.airbyte.cdk.integrations.destination.buffered_stream_consumer; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DefaultDestStateLifecycleManagerTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/dest_state_lifecycle_manager/DefaultDestStateLifecycleManagerTest.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DefaultDestStateLifecycleManagerTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/dest_state_lifecycle_manager/DefaultDestStateLifecycleManagerTest.java index afa85a50ae78..66e5c226bb7b 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DefaultDestStateLifecycleManagerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/dest_state_lifecycle_manager/DefaultDestStateLifecycleManagerTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.dest_state_lifecycle_manager; +package io.airbyte.cdk.integrations.destination.dest_state_lifecycle_manager; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestSingleStateLifecycleManagerTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/dest_state_lifecycle_manager/DestSingleStateLifecycleManagerTest.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestSingleStateLifecycleManagerTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/dest_state_lifecycle_manager/DestSingleStateLifecycleManagerTest.java index c70b415cdcc3..c0ed7621c05f 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestSingleStateLifecycleManagerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/dest_state_lifecycle_manager/DestSingleStateLifecycleManagerTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.dest_state_lifecycle_manager; +package io.airbyte.cdk.integrations.destination.dest_state_lifecycle_manager; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestStreamStateLifecycleManagerTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/dest_state_lifecycle_manager/DestStreamStateLifecycleManagerTest.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestStreamStateLifecycleManagerTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/dest_state_lifecycle_manager/DestStreamStateLifecycleManagerTest.java index a8e69fa8fc2e..b36add37561b 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/dest_state_lifecycle_manager/DestStreamStateLifecycleManagerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/dest_state_lifecycle_manager/DestStreamStateLifecycleManagerTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.dest_state_lifecycle_manager; +package io.airbyte.cdk.integrations.destination.dest_state_lifecycle_manager; import static org.junit.jupiter.api.Assertions.*; diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/record_buffer/InMemoryRecordBufferingStrategyTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/record_buffer/InMemoryRecordBufferingStrategyTest.java similarity index 95% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/record_buffer/InMemoryRecordBufferingStrategyTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/record_buffer/InMemoryRecordBufferingStrategyTest.java index 69ede03c8b8a..d76943f52322 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/record_buffer/InMemoryRecordBufferingStrategyTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/record_buffer/InMemoryRecordBufferingStrategyTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.record_buffer; +package io.airbyte.cdk.integrations.destination.record_buffer; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -12,8 +12,8 @@ import static org.mockito.Mockito.verify; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.RecordWriter; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.buffered_stream_consumer.RecordWriter; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/record_buffer/SerializedBufferingStrategyTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/record_buffer/SerializedBufferingStrategyTest.java similarity index 99% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/record_buffer/SerializedBufferingStrategyTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/record_buffer/SerializedBufferingStrategyTest.java index b38953c3c25b..f94f032a46f0 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination/record_buffer/SerializedBufferingStrategyTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination/record_buffer/SerializedBufferingStrategyTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.record_buffer; +package io.airbyte.cdk.integrations.destination.record_buffer; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/AirbyteFileUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/AirbyteFileUtilsTest.java similarity index 92% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/AirbyteFileUtilsTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/AirbyteFileUtilsTest.java index 286b4e5d8ffa..be46f68ee61b 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/AirbyteFileUtilsTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/AirbyteFileUtilsTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async; +package io.airbyte.cdk.integrations.destination_async; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/AsyncStreamConsumerTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/AsyncStreamConsumerTest.java similarity index 80% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/AsyncStreamConsumerTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/AsyncStreamConsumerTest.java index 06e399d28f36..f06b67121d5b 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/AsyncStreamConsumerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/AsyncStreamConsumerTest.java @@ -2,9 +2,10 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async; +package io.airbyte.cdk.integrations.destination_async; import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.mock; @@ -15,13 +16,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.OnStartFunction; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.RecordSizeEstimator; +import io.airbyte.cdk.integrations.destination_async.buffers.BufferManager; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteRecordMessage; +import io.airbyte.cdk.integrations.destination_async.state.FlushFailure; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.buffered_stream_consumer.OnStartFunction; -import io.airbyte.integrations.destination.buffered_stream_consumer.RecordSizeEstimator; -import io.airbyte.integrations.destination_async.buffers.BufferManager; -import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteMessage; -import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteRecordMessage; -import io.airbyte.integrations.destination_async.state.FlushFailure; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteLogMessage; @@ -30,16 +31,17 @@ import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStateStats; import io.airbyte.protocol.models.v0.AirbyteStreamState; import io.airbyte.protocol.models.v0.CatalogHelpers; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.StreamDescriptor; import java.io.IOException; +import java.math.BigDecimal; import java.time.Instant; import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -136,7 +138,14 @@ void test1StreamWith1State() throws Exception { verifyRecords(STREAM_NAME, SCHEMA_NAME, expectedRecords); - verify(outputRecordCollector).accept(STATE_MESSAGE1); + final AirbyteMessage stateMessageWithDestinationStatsUpdated = new AirbyteMessage() + .withType(Type.STATE) + .withState(new AirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState().withStreamDescriptor(STREAM1_DESC).withStreamState(Jsons.jsonNode(1))) + .withDestinationStats(new AirbyteStateStats().withRecordCount((double) expectedRecords.size()))); + + verify(outputRecordCollector).accept(stateMessageWithDestinationStatsUpdated); } @Test @@ -153,7 +162,14 @@ void test1StreamWith2State() throws Exception { verifyRecords(STREAM_NAME, SCHEMA_NAME, expectedRecords); - verify(outputRecordCollector, times(1)).accept(STATE_MESSAGE2); + final AirbyteMessage stateMessageWithDestinationStatsUpdated = new AirbyteMessage() + .withType(Type.STATE) + .withState(new AirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState().withStreamDescriptor(STREAM1_DESC).withStreamState(Jsons.jsonNode(2))) + .withDestinationStats(new AirbyteStateStats().withRecordCount(0.0))); + + verify(outputRecordCollector, times(1)).accept(stateMessageWithDestinationStatsUpdated); } @Test @@ -185,7 +201,7 @@ void testBackPressure() throws Exception { consumer = new AsyncStreamConsumer( m -> {}, () -> {}, - () -> {}, + (hasFailed, recordCounts) -> {}, flushFunction, CATALOG, new BufferManager(1024 * 10), @@ -236,8 +252,24 @@ void deserializeAirbyteMessageWithAirbyteRecord() { .withData(PAYLOAD)); final String serializedAirbyteMessage = Jsons.serialize(airbyteMessage); final String airbyteRecordString = Jsons.serialize(PAYLOAD); - final Optional partial = AsyncStreamConsumer.deserializeAirbyteMessage(serializedAirbyteMessage); - assertEquals(airbyteRecordString, partial.get().getSerialized()); + final PartialAirbyteMessage partial = AsyncStreamConsumer.deserializeAirbyteMessage(serializedAirbyteMessage); + assertEquals(airbyteRecordString, partial.getSerialized()); + } + + @Test + void deserializeAirbyteMessageWithBigDecimalAirbyteRecord() { + final JsonNode payload = Jsons.jsonNode(Map.of( + "foo", new BigDecimal("1234567890.1234567890"))); + final AirbyteMessage airbyteMessage = new AirbyteMessage() + .withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage() + .withStream(STREAM_NAME) + .withNamespace(SCHEMA_NAME) + .withData(payload)); + final String serializedAirbyteMessage = Jsons.serialize(airbyteMessage); + final String airbyteRecordString = Jsons.serialize(payload); + final PartialAirbyteMessage partial = AsyncStreamConsumer.deserializeAirbyteMessage(serializedAirbyteMessage); + assertEquals(airbyteRecordString, partial.getSerialized()); } @Test @@ -250,8 +282,8 @@ void deserializeAirbyteMessageWithEmptyAirbyteRecord() { .withNamespace(SCHEMA_NAME) .withData(Jsons.jsonNode(emptyMap))); final String serializedAirbyteMessage = Jsons.serialize(airbyteMessage); - final Optional partial = AsyncStreamConsumer.deserializeAirbyteMessage(serializedAirbyteMessage); - assertEquals(emptyMap.toString(), partial.get().getSerialized()); + final PartialAirbyteMessage partial = AsyncStreamConsumer.deserializeAirbyteMessage(serializedAirbyteMessage); + assertEquals(emptyMap.toString(), partial.getSerialized()); } @Test @@ -266,8 +298,8 @@ void deserializeAirbyteMessageWithNoStateOrRecord() { @Test void deserializeAirbyteMessageWithAirbyteState() { final String serializedAirbyteMessage = Jsons.serialize(STATE_MESSAGE1); - final Optional partial = AsyncStreamConsumer.deserializeAirbyteMessage(serializedAirbyteMessage); - assertEquals(serializedAirbyteMessage, partial.get().getSerialized()); + final PartialAirbyteMessage partial = AsyncStreamConsumer.deserializeAirbyteMessage(serializedAirbyteMessage); + assertEquals(serializedAirbyteMessage, partial.getSerialized()); } @Test @@ -348,7 +380,7 @@ private static List generateRecords(final long targetSizeInBytes private void verifyStartAndClose() throws Exception { verify(onStart).call(); - verify(onClose).call(); + verify(onClose).accept(any(), any()); } @SuppressWarnings({"unchecked", "SameParameterValue"}) diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/DetectStreamToFlushTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/DetectStreamToFlushTest.java new file mode 100644 index 000000000000..02bcf78f7c76 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/DetectStreamToFlushTest.java @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination_async; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import io.airbyte.cdk.integrations.destination_async.buffers.BufferDequeue; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class DetectStreamToFlushTest { + + public static final Instant NOW = Instant.now(); + public static final Duration FIVE_MIN = Duration.ofMinutes(5); + private static final long SIZE_10MB = 10 * 1024 * 1024; + private static final long SIZE_200MB = 200 * 1024 * 1024; + + private static final StreamDescriptor DESC1 = new StreamDescriptor().withName("test1"); + + private static DestinationFlushFunction flusher; + + @BeforeEach + void setup() { + flusher = mock(DestinationFlushFunction.class); + when(flusher.getOptimalBatchSizeBytes()).thenReturn(SIZE_200MB); + } + + @Test + void testGetNextSkipsEmptyStreams() { + final BufferDequeue bufferDequeue = mock(BufferDequeue.class); + when(bufferDequeue.getBufferedStreams()).thenReturn(Set.of(DESC1)); + when(bufferDequeue.getQueueSizeBytes(DESC1)).thenReturn(Optional.of(0L)); + final RunningFlushWorkers runningFlushWorkers = mock(RunningFlushWorkers.class); + + final DetectStreamToFlush detect = + new DetectStreamToFlush(bufferDequeue, runningFlushWorkers, new AtomicBoolean(false), flusher); + assertEquals(Optional.empty(), detect.getNextStreamToFlush(0)); + } + + @Test + void testGetNextPicksUpOnSizeTrigger() { + final BufferDequeue bufferDequeue = mock(BufferDequeue.class); + when(bufferDequeue.getBufferedStreams()).thenReturn(Set.of(DESC1)); + when(bufferDequeue.getQueueSizeBytes(DESC1)).thenReturn(Optional.of(1L)); + final RunningFlushWorkers runningFlushWorkers = mock(RunningFlushWorkers.class); + final DetectStreamToFlush detect = + new DetectStreamToFlush(bufferDequeue, runningFlushWorkers, new AtomicBoolean(false), flusher); + // if above threshold, triggers + assertEquals(Optional.of(DESC1), detect.getNextStreamToFlush(0)); + // if below threshold, no trigger + assertEquals(Optional.empty(), detect.getNextStreamToFlush(1)); + } + + @Test + void testGetNextAccountsForAlreadyRunningWorkers() { + final BufferDequeue bufferDequeue = mock(BufferDequeue.class); + when(bufferDequeue.getBufferedStreams()).thenReturn(Set.of(DESC1)); + when(bufferDequeue.getQueueSizeBytes(DESC1)).thenReturn(Optional.of(1L)); + final RunningFlushWorkers runningFlushWorkers = mock(RunningFlushWorkers.class); + when(runningFlushWorkers.getSizesOfRunningWorkerBatches(any())).thenReturn(List.of(Optional.of(SIZE_10MB))); + final DetectStreamToFlush detect = + new DetectStreamToFlush(bufferDequeue, runningFlushWorkers, new AtomicBoolean(false), flusher); + assertEquals(Optional.empty(), detect.getNextStreamToFlush(0)); + } + + @Test + void testGetNextPicksUpOnTimeTrigger() { + final BufferDequeue bufferDequeue = mock(BufferDequeue.class); + when(bufferDequeue.getBufferedStreams()).thenReturn(Set.of(DESC1)); + when(bufferDequeue.getQueueSizeBytes(DESC1)).thenReturn(Optional.of(1L)); + final Clock mockedNowProvider = mock(Clock.class); + + final RunningFlushWorkers runningFlushWorkers = mock(RunningFlushWorkers.class); + when(runningFlushWorkers.getSizesOfRunningWorkerBatches(any())).thenReturn(List.of(Optional.of(SIZE_10MB))); + final DetectStreamToFlush detect = + new DetectStreamToFlush(bufferDequeue, runningFlushWorkers, new AtomicBoolean(false), flusher, + mockedNowProvider); + + // initialize flush time + when(mockedNowProvider.millis()) + .thenReturn(NOW.toEpochMilli()); + + assertEquals(Optional.empty(), detect.getNextStreamToFlush(0)); + + // check 5 minutes later + when(mockedNowProvider.millis()) + .thenReturn(NOW.plus(FIVE_MIN).toEpochMilli()); + + assertEquals(Optional.of(DESC1), detect.getNextStreamToFlush(0)); + + // just flush once + assertEquals(Optional.empty(), detect.getNextStreamToFlush(0)); + + // check another 5 minutes later + when(mockedNowProvider.millis()) + .thenReturn(NOW.plus(FIVE_MIN).plus(FIVE_MIN).toEpochMilli()); + assertEquals(Optional.of(DESC1), detect.getNextStreamToFlush(0)); + } + +} diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/FlushThresholdTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/FlushThresholdTest.java similarity index 77% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/FlushThresholdTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/FlushThresholdTest.java index 5fb0c1c2c31e..e8f56d90c777 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/FlushThresholdTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/FlushThresholdTest.java @@ -2,25 +2,34 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async; +package io.airbyte.cdk.integrations.destination_async; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import io.airbyte.integrations.destination_async.buffers.BufferDequeue; +import io.airbyte.cdk.integrations.destination_async.buffers.BufferDequeue; import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; public class FlushThresholdTest { private static final long SIZE_10MB = 10 * 1024 * 1024; + private DestinationFlushFunction flusher; + + @BeforeEach + void setup() { + flusher = mock(DestinationFlushFunction.class); + when(flusher.getQueueFlushThresholdBytes()).thenReturn(SIZE_10MB); + } + @Test void testBaseThreshold() { final AtomicBoolean isClosing = new AtomicBoolean(false); final BufferDequeue bufferDequeue = mock(BufferDequeue.class); - final DetectStreamToFlush detect = new DetectStreamToFlush(bufferDequeue, null, isClosing, null); + final DetectStreamToFlush detect = new DetectStreamToFlush(bufferDequeue, null, isClosing, flusher); assertEquals(SIZE_10MB, detect.computeQueueThreshold()); } @@ -28,7 +37,7 @@ void testBaseThreshold() { void testClosingThreshold() { final AtomicBoolean isClosing = new AtomicBoolean(true); final BufferDequeue bufferDequeue = mock(BufferDequeue.class); - final DetectStreamToFlush detect = new DetectStreamToFlush(bufferDequeue, null, isClosing, null); + final DetectStreamToFlush detect = new DetectStreamToFlush(bufferDequeue, null, isClosing, flusher); assertEquals(0, detect.computeQueueThreshold()); } @@ -38,7 +47,7 @@ void testEagerFlushThresholdBelowThreshold() { final BufferDequeue bufferDequeue = mock(BufferDequeue.class); when(bufferDequeue.getTotalGlobalQueueSizeBytes()).thenReturn(8L); when(bufferDequeue.getMaxQueueSizeBytes()).thenReturn(10L); - final DetectStreamToFlush detect = new DetectStreamToFlush(bufferDequeue, null, isClosing, null); + final DetectStreamToFlush detect = new DetectStreamToFlush(bufferDequeue, null, isClosing, flusher); assertEquals(SIZE_10MB, detect.computeQueueThreshold()); } @@ -48,7 +57,7 @@ void testEagerFlushThresholdAboveThreshold() { final BufferDequeue bufferDequeue = mock(BufferDequeue.class); when(bufferDequeue.getTotalGlobalQueueSizeBytes()).thenReturn(9L); when(bufferDequeue.getMaxQueueSizeBytes()).thenReturn(10L); - final DetectStreamToFlush detect = new DetectStreamToFlush(bufferDequeue, null, isClosing, null); + final DetectStreamToFlush detect = new DetectStreamToFlush(bufferDequeue, null, isClosing, flusher); assertEquals(0, detect.computeQueueThreshold()); } diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/FlushWorkersTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/FlushWorkersTest.java similarity index 81% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/FlushWorkersTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/FlushWorkersTest.java index 1ed035959716..ff45391c9773 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/FlushWorkersTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/FlushWorkersTest.java @@ -2,16 +2,16 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async; +package io.airbyte.cdk.integrations.destination_async; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import io.airbyte.integrations.destination_async.buffers.BufferDequeue; -import io.airbyte.integrations.destination_async.buffers.MemoryAwareMessageBatch; -import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteMessage; -import io.airbyte.integrations.destination_async.state.FlushFailure; -import io.airbyte.integrations.destination_async.state.GlobalAsyncStateManager; +import io.airbyte.cdk.integrations.destination_async.buffers.BufferDequeue; +import io.airbyte.cdk.integrations.destination_async.buffers.MemoryAwareMessageBatch; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; +import io.airbyte.cdk.integrations.destination_async.state.FlushFailure; +import io.airbyte.cdk.integrations.destination_async.state.GlobalAsyncStateManager; import io.airbyte.protocol.models.v0.StreamDescriptor; import java.io.IOException; import java.util.List; diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/GlobalMemoryManagerTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/GlobalMemoryManagerTest.java new file mode 100644 index 000000000000..9f79ed0554f6 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/GlobalMemoryManagerTest.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination_async; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +public class GlobalMemoryManagerTest { + + private static final long BYTES_MB = 1024 * 1024; + + @Test + void test() { + final GlobalMemoryManager mgr = new GlobalMemoryManager(15 * BYTES_MB); + + assertEquals(10 * BYTES_MB, mgr.requestMemory()); + assertEquals(5 * BYTES_MB, mgr.requestMemory()); + assertEquals(0, mgr.requestMemory()); + + mgr.free(10 * BYTES_MB); + assertEquals(10 * BYTES_MB, mgr.requestMemory()); + mgr.free(16 * BYTES_MB); + assertEquals(10 * BYTES_MB, mgr.requestMemory()); + } + +} diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/PartialAirbyteMessageTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/PartialAirbyteMessageTest.java similarity index 94% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/PartialAirbyteMessageTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/PartialAirbyteMessageTest.java index d9d2e3e5edff..2f65a926f544 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/PartialAirbyteMessageTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/PartialAirbyteMessageTest.java @@ -2,10 +2,10 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async; +package io.airbyte.cdk.integrations.destination_async; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteMessage; import io.airbyte.protocol.models.AirbyteStateMessage; import io.airbyte.protocol.models.AirbyteStreamState; import io.airbyte.protocol.models.StreamDescriptor; diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/RunningFlushWorkersTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/RunningFlushWorkersTest.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/RunningFlushWorkersTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/RunningFlushWorkersTest.java index 4889561eec75..8f2f28d3dbc3 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/RunningFlushWorkersTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/RunningFlushWorkersTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async; +package io.airbyte.cdk.integrations.destination_async; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/RunningSizeEstimateTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/RunningSizeEstimateTest.java similarity index 98% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/RunningSizeEstimateTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/RunningSizeEstimateTest.java index f569a1a5777e..57b360068a7a 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/RunningSizeEstimateTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/RunningSizeEstimateTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async; +package io.airbyte.cdk.integrations.destination_async; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/SizeTriggerTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/SizeTriggerTest.java similarity index 83% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/SizeTriggerTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/SizeTriggerTest.java index fbb2154632dd..7f9a9b2a3397 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/SizeTriggerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/SizeTriggerTest.java @@ -2,14 +2,14 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async; +package io.airbyte.cdk.integrations.destination_async; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import io.airbyte.integrations.destination_async.buffers.BufferDequeue; +import io.airbyte.cdk.integrations.destination_async.buffers.BufferDequeue; import io.airbyte.protocol.models.v0.StreamDescriptor; import java.time.Instant; import java.util.Collections; @@ -43,7 +43,8 @@ void testSizeTriggerOnEmptyQueue() { when(bufferDequeue.getBufferedStreams()).thenReturn(Set.of(DESC1)); when(bufferDequeue.getQueueSizeBytes(DESC1)).thenReturn(Optional.of(0L)); final RunningFlushWorkers runningFlushWorkers = mock(RunningFlushWorkers.class); - final DetectStreamToFlush detect = new DetectStreamToFlush(bufferDequeue, runningFlushWorkers, new AtomicBoolean(false), flusher); + final DetectStreamToFlush detect = + new DetectStreamToFlush(bufferDequeue, runningFlushWorkers, new AtomicBoolean(false), flusher); assertEquals(false, detect.isSizeTriggered(DESC1, SIZE_10MB).getLeft()); } @@ -53,7 +54,8 @@ void testSizeTriggerRespectsThreshold() { when(bufferDequeue.getBufferedStreams()).thenReturn(Set.of(DESC1)); when(bufferDequeue.getQueueSizeBytes(DESC1)).thenReturn(Optional.of(1L)); final RunningFlushWorkers runningFlushWorkers = mock(RunningFlushWorkers.class); - final DetectStreamToFlush detect = new DetectStreamToFlush(bufferDequeue, runningFlushWorkers, new AtomicBoolean(false), flusher); + final DetectStreamToFlush detect = + new DetectStreamToFlush(bufferDequeue, runningFlushWorkers, new AtomicBoolean(false), flusher); // if above threshold, triggers assertEquals(true, detect.isSizeTriggered(DESC1, 0).getLeft()); // if below threshold, no trigger @@ -69,7 +71,8 @@ void testSizeTriggerRespectsRunningWorkersEstimate() { when(runningFlushWorkers.getSizesOfRunningWorkerBatches(any())) .thenReturn(Collections.emptyList()) .thenReturn(List.of(Optional.of(SIZE_10MB))); - final DetectStreamToFlush detect = new DetectStreamToFlush(bufferDequeue, runningFlushWorkers, new AtomicBoolean(false), flusher); + final DetectStreamToFlush detect = + new DetectStreamToFlush(bufferDequeue, runningFlushWorkers, new AtomicBoolean(false), flusher); assertEquals(true, detect.isSizeTriggered(DESC1, 0).getLeft()); assertEquals(false, detect.isSizeTriggered(DESC1, 0).getLeft()); } diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/StreamPriorityTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/StreamPriorityTest.java similarity index 95% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/StreamPriorityTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/StreamPriorityTest.java index 92dfa29c88da..add92636f047 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/StreamPriorityTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/StreamPriorityTest.java @@ -2,15 +2,15 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination_async; +package io.airbyte.cdk.integrations.destination_async; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import io.airbyte.cdk.integrations.destination_async.buffers.BufferDequeue; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination_async.buffers.BufferDequeue; import io.airbyte.protocol.models.v0.StreamDescriptor; import java.time.Instant; import java.util.HashSet; diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/TimeTriggerTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/TimeTriggerTest.java new file mode 100644 index 000000000000..95545118e713 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/TimeTriggerTest.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination_async; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import io.airbyte.cdk.integrations.destination_async.buffers.BufferDequeue; +import java.time.Clock; +import org.junit.jupiter.api.Test; + +public class TimeTriggerTest { + + private static final long NOW_MS = System.currentTimeMillis(); + private static final long ONE_SEC = 1000L; + private static final long FIVE_MIN = 5 * 60 * 1000; + + @Test + void testTimeTrigger() { + final BufferDequeue bufferDequeue = mock(BufferDequeue.class); + + final Clock mockedNowProvider = mock(Clock.class); + when(mockedNowProvider.millis()) + .thenReturn(NOW_MS); + + final DetectStreamToFlush detect = new DetectStreamToFlush(bufferDequeue, null, null, null, mockedNowProvider); + assertEquals(false, detect.isTimeTriggered(NOW_MS).getLeft()); + assertEquals(false, detect.isTimeTriggered(NOW_MS - ONE_SEC).getLeft()); + assertEquals(true, detect.isTimeTriggered(NOW_MS - FIVE_MIN).getLeft()); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferDequeueTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferDequeueTest.java new file mode 100644 index 000000000000..eb565b90ec6d --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferDequeueTest.java @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination_async.buffers; + +import static io.airbyte.cdk.integrations.destination_async.GlobalMemoryManager.BLOCK_SIZE_BYTES; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteRecordMessage; +import io.airbyte.commons.json.Jsons; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +public class BufferDequeueTest { + + private static final int RECORD_SIZE_20_BYTES = 20; + private static final String DEFAULT_NAMESPACE = "foo_namespace"; + public static final String RECORD_20_BYTES = "abc"; + private static final String STREAM_NAME = "stream1"; + private static final StreamDescriptor STREAM_DESC = new StreamDescriptor().withName(STREAM_NAME); + private static final PartialAirbyteMessage RECORD_MSG_20_BYTES = new PartialAirbyteMessage() + .withType(Type.RECORD) + .withRecord(new PartialAirbyteRecordMessage() + .withStream(STREAM_NAME)); + + @Nested + class Take { + + @Test + void testTakeShouldBestEffortRead() { + final BufferManager bufferManager = new BufferManager(); + final BufferEnqueue enqueue = bufferManager.getBufferEnqueue(); + final BufferDequeue dequeue = bufferManager.getBufferDequeue(); + + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + + // total size of records is 80, so we expect 50 to get us 2 records (prefer to under-pull records + // than over-pull). + try (final MemoryAwareMessageBatch take = dequeue.take(STREAM_DESC, 50)) { + assertEquals(2, take.getData().size()); + // verify it only took the records from the queue that it actually returned. + assertEquals(2, dequeue.getQueueSizeInRecords(STREAM_DESC).orElseThrow()); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + @Test + void testTakeShouldReturnAllIfPossible() { + final BufferManager bufferManager = new BufferManager(); + final BufferEnqueue enqueue = bufferManager.getBufferEnqueue(); + final BufferDequeue dequeue = bufferManager.getBufferDequeue(); + + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + + try (final MemoryAwareMessageBatch take = dequeue.take(STREAM_DESC, 60)) { + assertEquals(3, take.getData().size()); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + @Test + void testTakeFewerRecordsThanSizeLimitShouldNotError() { + final BufferManager bufferManager = new BufferManager(); + final BufferEnqueue enqueue = bufferManager.getBufferEnqueue(); + final BufferDequeue dequeue = bufferManager.getBufferDequeue(); + + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + + try (final MemoryAwareMessageBatch take = dequeue.take(STREAM_DESC, Long.MAX_VALUE)) { + assertEquals(2, take.getData().size()); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + } + + @Test + void testMetadataOperationsCorrect() { + final BufferManager bufferManager = new BufferManager(); + final BufferEnqueue enqueue = bufferManager.getBufferEnqueue(); + final BufferDequeue dequeue = bufferManager.getBufferDequeue(); + + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + + final var secondStream = new StreamDescriptor().withName("stream_2"); + final PartialAirbyteMessage recordFromSecondStream = Jsons.clone(RECORD_MSG_20_BYTES); + recordFromSecondStream.getRecord().withStream(secondStream.getName()); + enqueue.addRecord(recordFromSecondStream, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + + assertEquals(60, dequeue.getTotalGlobalQueueSizeBytes()); + + assertEquals(2, dequeue.getQueueSizeInRecords(STREAM_DESC).get()); + assertEquals(1, dequeue.getQueueSizeInRecords(secondStream).get()); + + assertEquals(40, dequeue.getQueueSizeBytes(STREAM_DESC).get()); + assertEquals(20, dequeue.getQueueSizeBytes(secondStream).get()); + + // Buffer of 3 sec to deal with test execution variance. + final var lastThreeSec = Instant.now().minus(3, ChronoUnit.SECONDS); + assertTrue(lastThreeSec.isBefore(dequeue.getTimeOfLastRecord(STREAM_DESC).get())); + assertTrue(lastThreeSec.isBefore(dequeue.getTimeOfLastRecord(secondStream).get())); + } + + @Test + void testMetadataOperationsError() { + final BufferManager bufferManager = new BufferManager(); + final BufferDequeue dequeue = bufferManager.getBufferDequeue(); + + final var ghostStream = new StreamDescriptor().withName("ghost stream"); + + assertEquals(0, dequeue.getTotalGlobalQueueSizeBytes()); + + assertTrue(dequeue.getQueueSizeInRecords(ghostStream).isEmpty()); + + assertTrue(dequeue.getQueueSizeBytes(ghostStream).isEmpty()); + + assertTrue(dequeue.getTimeOfLastRecord(ghostStream).isEmpty()); + } + + @Test + void cleansUpMemoryForEmptyQueues() throws Exception { + final var bufferManager = new BufferManager(); + final var enqueue = bufferManager.getBufferEnqueue(); + final var dequeue = bufferManager.getBufferDequeue(); + final var memoryManager = bufferManager.getMemoryManager(); + + // we initialize with a block for state + assertEquals(BLOCK_SIZE_BYTES, memoryManager.getCurrentMemoryBytes()); + + // allocate a block for new stream + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + assertEquals(2 * BLOCK_SIZE_BYTES, memoryManager.getCurrentMemoryBytes()); + + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + + // no re-allocates as we haven't breached block size + assertEquals(2 * BLOCK_SIZE_BYTES, memoryManager.getCurrentMemoryBytes()); + + final var totalBatchSize = RECORD_SIZE_20_BYTES * 4; + + // read the whole queue + try (final var batch = dequeue.take(STREAM_DESC, totalBatchSize)) { + // slop allocation gets cleaned up + assertEquals(BLOCK_SIZE_BYTES + totalBatchSize, memoryManager.getCurrentMemoryBytes()); + batch.close(); + // back to initial state after flush clears the batch + assertEquals(BLOCK_SIZE_BYTES, memoryManager.getCurrentMemoryBytes()); + assertEquals(0, bufferManager.getBuffers().get(STREAM_DESC).getMaxMemoryUsage()); + } + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferEnqueueTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferEnqueueTest.java new file mode 100644 index 000000000000..a555c403e5c0 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/buffers/BufferEnqueueTest.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination_async.buffers; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; + +import io.airbyte.cdk.integrations.destination_async.GlobalMemoryManager; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteRecordMessage; +import io.airbyte.cdk.integrations.destination_async.state.GlobalAsyncStateManager; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.concurrent.ConcurrentHashMap; +import org.junit.jupiter.api.Test; + +public class BufferEnqueueTest { + + private static final int RECORD_SIZE_20_BYTES = 20; + private static final String DEFAULT_NAMESPACE = "foo_namespace"; + + @Test + void testAddRecordShouldAdd() { + final var twoMB = 2 * 1024 * 1024; + final var streamToBuffer = new ConcurrentHashMap(); + final var enqueue = new BufferEnqueue(new GlobalMemoryManager(twoMB), streamToBuffer, mock(GlobalAsyncStateManager.class)); + + final var streamName = "stream"; + final var stream = new StreamDescriptor().withName(streamName); + final var record = new PartialAirbyteMessage() + .withType(AirbyteMessage.Type.RECORD) + .withRecord(new PartialAirbyteRecordMessage() + .withStream(streamName)); + + enqueue.addRecord(record, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + assertEquals(1, streamToBuffer.get(stream).size()); + assertEquals(20L, streamToBuffer.get(stream).getCurrentMemoryUsage()); + + } + + @Test + public void testAddRecordShouldExpand() { + final var oneKb = 1024; + final var streamToBuffer = new ConcurrentHashMap(); + final var enqueue = + new BufferEnqueue(new GlobalMemoryManager(oneKb), streamToBuffer, mock(GlobalAsyncStateManager.class)); + + final var streamName = "stream"; + final var stream = new StreamDescriptor().withName(streamName); + final var record = new PartialAirbyteMessage() + .withType(AirbyteMessage.Type.RECORD) + .withRecord(new PartialAirbyteRecordMessage() + .withStream(streamName)); + + enqueue.addRecord(record, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + enqueue.addRecord(record, RECORD_SIZE_20_BYTES, DEFAULT_NAMESPACE); + assertEquals(2, streamToBuffer.get(stream).size()); + assertEquals(40, streamToBuffer.get(stream).getCurrentMemoryUsage()); + + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/buffers/MemoryBoundedLinkedBlockingQueueTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/buffers/MemoryBoundedLinkedBlockingQueueTest.java new file mode 100644 index 000000000000..bcd57d56e421 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/buffers/MemoryBoundedLinkedBlockingQueueTest.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination_async.buffers; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +public class MemoryBoundedLinkedBlockingQueueTest { + + @Test + void offerAndTakeShouldReturn() throws InterruptedException { + final MemoryBoundedLinkedBlockingQueue queue = new MemoryBoundedLinkedBlockingQueue<>(1024); + + queue.offer("abc", 6); + + final var item = queue.take(); + + assertEquals("abc", item.item()); + } + + @Test + void testBlocksOnFullMemory() throws InterruptedException { + final MemoryBoundedLinkedBlockingQueue queue = new MemoryBoundedLinkedBlockingQueue<>(10); + assertTrue(queue.offer("abc", 6)); + assertFalse(queue.offer("abc", 6)); + + assertNotNull(queue.poll(1, TimeUnit.NANOSECONDS)); + assertNull(queue.poll(1, TimeUnit.NANOSECONDS)); + } + + @ParameterizedTest + @ValueSource(longs = {1024, 100000, 600}) + void getMaxMemoryUsage(final long size) { + final MemoryBoundedLinkedBlockingQueue queue = new MemoryBoundedLinkedBlockingQueue<>(size); + + assertEquals(0, queue.getCurrentMemoryUsage()); + assertEquals(size, queue.getMaxMemoryUsage()); + + queue.addMaxMemory(-100); + + assertEquals(size - 100, queue.getMaxMemoryUsage()); + + queue.addMaxMemory(123); + + assertEquals(size - 100 + 123, queue.getMaxMemoryUsage()); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/buffers/StreamAwareQueueTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/buffers/StreamAwareQueueTest.java new file mode 100644 index 000000000000..39a32d0b5de9 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/buffers/StreamAwareQueueTest.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination_async.buffers; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +public class StreamAwareQueueTest { + + @Test + void test() throws InterruptedException { + final StreamAwareQueue queue = new StreamAwareQueue(1024); + + assertEquals(0, queue.getCurrentMemoryUsage()); + assertNull(queue.getTimeOfLastMessage().orElse(null)); + + queue.offer(new PartialAirbyteMessage(), 6, 1); + queue.offer(new PartialAirbyteMessage(), 6, 2); + queue.offer(new PartialAirbyteMessage(), 6, 3); + + assertEquals(18, queue.getCurrentMemoryUsage()); + assertNotNull(queue.getTimeOfLastMessage().orElse(null)); + + queue.take(); + queue.take(); + queue.take(); + + assertEquals(0, queue.getCurrentMemoryUsage()); + // This should be null because the queue is empty + assertTrue(queue.getTimeOfLastMessage().isEmpty(), "Expected empty optional; got " + queue.getTimeOfLastMessage()); + } + + @ParameterizedTest + @ValueSource(longs = {1024, 100000, 600}) + void getMaxMemoryUsage(final long size) { + final StreamAwareQueue queue = new StreamAwareQueue(size); + + assertEquals(0, queue.getCurrentMemoryUsage()); + assertEquals(size, queue.getMaxMemoryUsage()); + + queue.addMaxMemory(-100); + + assertEquals(size - 100, queue.getMaxMemoryUsage()); + + queue.addMaxMemory(123); + + assertEquals(size - 100 + 123, queue.getMaxMemoryUsage()); + } + + @Test + void isEmpty() { + final StreamAwareQueue queue = new StreamAwareQueue(1024); + + assertTrue(queue.isEmpty()); + + queue.offer(new PartialAirbyteMessage(), 10, 1); + + assertFalse(queue.isEmpty()); + + queue.offer(new PartialAirbyteMessage(), 10, 1); + queue.offer(new PartialAirbyteMessage(), 10, 1); + queue.offer(new PartialAirbyteMessage(), 10, 1); + + assertFalse(queue.isEmpty()); + + queue.poll(); + queue.poll(); + queue.poll(); + queue.poll(); + + assertTrue(queue.isEmpty()); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/state/GlobalAsyncStateManagerTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/state/GlobalAsyncStateManagerTest.java new file mode 100644 index 000000000000..338b131d3cbe --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/destination_async/state/GlobalAsyncStateManagerTest.java @@ -0,0 +1,253 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination_async.state; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import io.airbyte.cdk.integrations.destination_async.GlobalMemoryManager; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteStateMessage; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteStreamState; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStateStats; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +class GlobalAsyncStateManagerTest { + + private static final long TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES = 100 * 1024 * 1024; // 10MB + private static final String DEFAULT_NAMESPACE = "foo_namespace"; + private static final long STATE_MSG_SIZE = 1000; + + private static final String NAMESPACE = "namespace"; + private static final String STREAM_NAME = "id_and_name"; + private static final String STREAM_NAME2 = STREAM_NAME + 2; + private static final String STREAM_NAME3 = STREAM_NAME + 3; + private static final StreamDescriptor STREAM1_DESC = new StreamDescriptor() + .withName(STREAM_NAME).withNamespace(NAMESPACE); + private static final StreamDescriptor STREAM2_DESC = new StreamDescriptor() + .withName(STREAM_NAME2).withNamespace(NAMESPACE); + private static final StreamDescriptor STREAM3_DESC = new StreamDescriptor() + .withName(STREAM_NAME3).withNamespace(NAMESPACE); + + private static final PartialAirbyteMessage GLOBAL_STATE_MESSAGE1 = new PartialAirbyteMessage() + .withType(Type.STATE) + .withState(new PartialAirbyteStateMessage() + .withType(AirbyteStateType.GLOBAL)); + private static final PartialAirbyteMessage GLOBAL_STATE_MESSAGE2 = new PartialAirbyteMessage() + .withType(Type.STATE) + .withState(new PartialAirbyteStateMessage() + .withType(AirbyteStateType.GLOBAL)); + private static final PartialAirbyteMessage STREAM1_STATE_MESSAGE1 = new PartialAirbyteMessage() + .withType(Type.STATE) + .withState(new PartialAirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream(new PartialAirbyteStreamState().withStreamDescriptor(STREAM1_DESC))); + private static final PartialAirbyteMessage STREAM1_STATE_MESSAGE2 = new PartialAirbyteMessage() + .withType(Type.STATE) + .withState(new PartialAirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream(new PartialAirbyteStreamState().withStreamDescriptor(STREAM1_DESC))); + private static final PartialAirbyteMessage STREAM2_STATE_MESSAGE = new PartialAirbyteMessage() + .withType(Type.STATE) + .withState(new PartialAirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream(new PartialAirbyteStreamState().withStreamDescriptor(STREAM2_DESC))); + + @Test + void testBasic() { + final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); + + final var firstStateId = stateManager.getStateIdAndIncrementCounter(STREAM1_DESC); + final var secondStateId = stateManager.getStateIdAndIncrementCounter(STREAM1_DESC); + assertEquals(firstStateId, secondStateId); + + stateManager.decrement(firstStateId, 2); + // because no state message has been tracked, there is nothing to flush yet. + final Map stateWithStats = + stateManager.flushStates().stream() + .collect(Collectors.toMap(PartialStateWithDestinationStats::stateMessage, PartialStateWithDestinationStats::stats)); + assertEquals(0, stateWithStats.size()); + + stateManager.trackState(STREAM1_STATE_MESSAGE1, STATE_MSG_SIZE, DEFAULT_NAMESPACE); + final Map stateWithStats2 = + stateManager.flushStates().stream() + .collect(Collectors.toMap(PartialStateWithDestinationStats::stateMessage, PartialStateWithDestinationStats::stats)); + assertEquals(List.of(STREAM1_STATE_MESSAGE1), stateWithStats2.keySet().stream().toList()); + assertEquals(List.of(new AirbyteStateStats().withRecordCount(2.0)), stateWithStats2.values().stream().toList()); + } + + @Nested + class GlobalState { + + @Test + void testEmptyQueuesGlobalState() { + final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); + + // GLOBAL + stateManager.trackState(GLOBAL_STATE_MESSAGE1, STATE_MSG_SIZE, DEFAULT_NAMESPACE); + final Map stateWithStats = stateManager.flushStates().stream() + .collect(Collectors.toMap(PartialStateWithDestinationStats::stateMessage, PartialStateWithDestinationStats::stats)); + assertEquals(List.of(GLOBAL_STATE_MESSAGE1), stateWithStats.keySet().stream().toList()); + assertEquals(List.of(new AirbyteStateStats().withRecordCount(0.0)), stateWithStats.values().stream().toList()); + + assertThrows(IllegalArgumentException.class, () -> stateManager.trackState(STREAM1_STATE_MESSAGE1, STATE_MSG_SIZE, DEFAULT_NAMESPACE)); + } + + @Test + void testConversion() { + final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); + + final var preConvertId0 = simulateIncomingRecords(STREAM1_DESC, 10, stateManager); + final var preConvertId1 = simulateIncomingRecords(STREAM2_DESC, 10, stateManager); + final var preConvertId2 = simulateIncomingRecords(STREAM3_DESC, 10, stateManager); + assertEquals(3, Set.of(preConvertId0, preConvertId1, preConvertId2).size()); + + stateManager.trackState(GLOBAL_STATE_MESSAGE1, STATE_MSG_SIZE, DEFAULT_NAMESPACE); + + // Since this is actually a global state, we can only flush after all streams are done. + stateManager.decrement(preConvertId0, 10); + assertEquals(List.of(), stateManager.flushStates()); + stateManager.decrement(preConvertId1, 10); + assertEquals(List.of(), stateManager.flushStates()); + stateManager.decrement(preConvertId2, 10); + final Map stateWithStats = stateManager.flushStates().stream() + .collect(Collectors.toMap(PartialStateWithDestinationStats::stateMessage, PartialStateWithDestinationStats::stats)); + assertEquals(List.of(GLOBAL_STATE_MESSAGE1), stateWithStats.keySet().stream().toList()); + assertEquals(List.of(new AirbyteStateStats().withRecordCount(30.0)), stateWithStats.values().stream().toList()); + + } + + @Test + void testCorrectFlushingOneStream() { + final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); + + final var preConvertId0 = simulateIncomingRecords(STREAM1_DESC, 10, stateManager); + stateManager.trackState(GLOBAL_STATE_MESSAGE1, STATE_MSG_SIZE, DEFAULT_NAMESPACE); + stateManager.decrement(preConvertId0, 10); + final Map stateWithStats = stateManager.flushStates().stream() + .collect(Collectors.toMap(PartialStateWithDestinationStats::stateMessage, PartialStateWithDestinationStats::stats)); + assertEquals(List.of(GLOBAL_STATE_MESSAGE1), stateWithStats.keySet().stream().toList()); + assertEquals(List.of(new AirbyteStateStats().withRecordCount(10.0)), stateWithStats.values().stream().toList()); + + final var afterConvertId1 = simulateIncomingRecords(STREAM1_DESC, 10, stateManager); + stateManager.trackState(GLOBAL_STATE_MESSAGE2, STATE_MSG_SIZE, DEFAULT_NAMESPACE); + stateManager.decrement(afterConvertId1, 10); + final Map stateWithStats2 = stateManager.flushStates().stream() + .collect(Collectors.toMap(PartialStateWithDestinationStats::stateMessage, PartialStateWithDestinationStats::stats)); + assertEquals(List.of(GLOBAL_STATE_MESSAGE2), stateWithStats2.keySet().stream().toList()); + assertEquals(List.of(new AirbyteStateStats().withRecordCount(10.0)), stateWithStats2.values().stream().toList()); + } + + @Test + void testCorrectFlushingManyStreams() { + final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); + + final var preConvertId0 = simulateIncomingRecords(STREAM1_DESC, 10, stateManager); + final var preConvertId1 = simulateIncomingRecords(STREAM2_DESC, 10, stateManager); + assertNotEquals(preConvertId0, preConvertId1); + stateManager.trackState(GLOBAL_STATE_MESSAGE1, STATE_MSG_SIZE, DEFAULT_NAMESPACE); + stateManager.decrement(preConvertId0, 10); + stateManager.decrement(preConvertId1, 10); + final Map stateWithStats = stateManager.flushStates().stream() + .collect(Collectors.toMap(PartialStateWithDestinationStats::stateMessage, PartialStateWithDestinationStats::stats)); + assertEquals(List.of(GLOBAL_STATE_MESSAGE1), stateWithStats.keySet().stream().toList()); + assertEquals(List.of(new AirbyteStateStats().withRecordCount(20.0)), stateWithStats.values().stream().toList()); + + final var afterConvertId0 = simulateIncomingRecords(STREAM1_DESC, 10, stateManager); + final var afterConvertId1 = simulateIncomingRecords(STREAM2_DESC, 10, stateManager); + assertEquals(afterConvertId0, afterConvertId1); + stateManager.trackState(GLOBAL_STATE_MESSAGE2, STATE_MSG_SIZE, DEFAULT_NAMESPACE); + stateManager.decrement(afterConvertId0, 20); + final Map stateWithStats2 = stateManager.flushStates().stream() + .collect(Collectors.toMap(PartialStateWithDestinationStats::stateMessage, PartialStateWithDestinationStats::stats)); + assertEquals(List.of(GLOBAL_STATE_MESSAGE2), stateWithStats2.keySet().stream().toList()); + assertEquals(List.of(new AirbyteStateStats().withRecordCount(20.0)), stateWithStats2.values().stream().toList()); + } + + } + + @Nested + class PerStreamState { + + @Test + void testEmptyQueues() { + final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); + + // GLOBAL + stateManager.trackState(STREAM1_STATE_MESSAGE1, STATE_MSG_SIZE, DEFAULT_NAMESPACE); + final Map stateWithStats = stateManager.flushStates().stream() + .collect(Collectors.toMap(PartialStateWithDestinationStats::stateMessage, PartialStateWithDestinationStats::stats)); + assertEquals(List.of(STREAM1_STATE_MESSAGE1), stateWithStats.keySet().stream().toList()); + assertEquals(List.of(new AirbyteStateStats().withRecordCount(0.0)), stateWithStats.values().stream().toList()); + + assertThrows(IllegalArgumentException.class, () -> stateManager.trackState(GLOBAL_STATE_MESSAGE1, STATE_MSG_SIZE, DEFAULT_NAMESPACE)); + } + + @Test + void testCorrectFlushingOneStream() { + final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); + + var stateId = simulateIncomingRecords(STREAM1_DESC, 3, stateManager); + stateManager.trackState(STREAM1_STATE_MESSAGE1, STATE_MSG_SIZE, DEFAULT_NAMESPACE); + stateManager.decrement(stateId, 3); + final Map stateWithStats = stateManager.flushStates().stream() + .collect(Collectors.toMap(PartialStateWithDestinationStats::stateMessage, PartialStateWithDestinationStats::stats)); + assertEquals(List.of(STREAM1_STATE_MESSAGE1), stateWithStats.keySet().stream().toList()); + assertEquals(List.of(new AirbyteStateStats().withRecordCount(3.0)), stateWithStats.values().stream().toList()); + + stateId = simulateIncomingRecords(STREAM1_DESC, 10, stateManager); + stateManager.trackState(STREAM1_STATE_MESSAGE2, STATE_MSG_SIZE, DEFAULT_NAMESPACE); + stateManager.decrement(stateId, 10); + final Map stateWithStats2 = stateManager.flushStates().stream() + .collect(Collectors.toMap(PartialStateWithDestinationStats::stateMessage, PartialStateWithDestinationStats::stats)); + assertEquals(List.of(STREAM1_STATE_MESSAGE2), stateWithStats2.keySet().stream().toList()); + assertEquals(List.of(new AirbyteStateStats().withRecordCount(10.0)), stateWithStats2.values().stream().toList()); + } + + @Test + void testCorrectFlushingManyStream() { + final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); + + final var stream1StateId = simulateIncomingRecords(STREAM1_DESC, 3, stateManager); + final var stream2StateId = simulateIncomingRecords(STREAM2_DESC, 7, stateManager); + + stateManager.trackState(STREAM1_STATE_MESSAGE1, STATE_MSG_SIZE, DEFAULT_NAMESPACE); + stateManager.decrement(stream1StateId, 3); + final Map stateWithStats = stateManager.flushStates().stream() + .collect(Collectors.toMap(PartialStateWithDestinationStats::stateMessage, PartialStateWithDestinationStats::stats)); + assertEquals(List.of(STREAM1_STATE_MESSAGE1), stateWithStats.keySet().stream().toList()); + assertEquals(List.of(new AirbyteStateStats().withRecordCount(3.0)), stateWithStats.values().stream().toList()); + + stateManager.decrement(stream2StateId, 4); + assertEquals(List.of(), stateManager.flushStates()); + stateManager.trackState(STREAM2_STATE_MESSAGE, STATE_MSG_SIZE, DEFAULT_NAMESPACE); + stateManager.decrement(stream2StateId, 3); + // only flush state if counter is 0. + final Map stateWithStats2 = stateManager.flushStates().stream() + .collect(Collectors.toMap(PartialStateWithDestinationStats::stateMessage, PartialStateWithDestinationStats::stats)); + assertEquals(List.of(STREAM2_STATE_MESSAGE), stateWithStats2.keySet().stream().toList()); + assertEquals(List.of(new AirbyteStateStats().withRecordCount(7.0)), stateWithStats2.values().stream().toList()); + } + + } + + private static long simulateIncomingRecords(final StreamDescriptor desc, final long count, final GlobalAsyncStateManager manager) { + var stateId = 0L; + for (int i = 0; i < count; i++) { + stateId = manager.getStateIdAndIncrementCounter(desc); + } + return stateId; + } + +} diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/util/ConnectorExceptionUtilTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/util/ConnectorExceptionUtilTest.java similarity index 95% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/util/ConnectorExceptionUtilTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/util/ConnectorExceptionUtilTest.java index 5371299b4cc6..c36461af0150 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/util/ConnectorExceptionUtilTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/util/ConnectorExceptionUtilTest.java @@ -2,10 +2,10 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.util; +package io.airbyte.cdk.integrations.util; -import static io.airbyte.integrations.util.ConnectorExceptionUtil.COMMON_EXCEPTION_MESSAGE_TEMPLATE; -import static io.airbyte.integrations.util.ConnectorExceptionUtil.RECOVERY_CONNECTION_ERROR_MESSAGE; +import static io.airbyte.cdk.integrations.util.ConnectorExceptionUtil.COMMON_EXCEPTION_MESSAGE_TEMPLATE; +import static io.airbyte.cdk.integrations.util.ConnectorExceptionUtil.RECOVERY_CONNECTION_ERROR_MESSAGE; import static org.junit.jupiter.api.Assertions.*; import io.airbyte.commons.exceptions.ConfigErrorException; diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/util/concurrent/ConcurrentStreamConsumerTest.java b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/util/concurrent/ConcurrentStreamConsumerTest.java similarity index 99% rename from airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/util/concurrent/ConcurrentStreamConsumerTest.java rename to airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/util/concurrent/ConcurrentStreamConsumerTest.java index db9f92492d88..085833dbf339 100644 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/util/concurrent/ConcurrentStreamConsumerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/core/src/test/java/io/airbyte/cdk/integrations/util/concurrent/ConcurrentStreamConsumerTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.util.concurrent; +package io.airbyte.cdk.integrations.util.concurrent; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/airbyte-db/db-lib/src/test/resources/toys_database/pre_migration_schema.txt b/airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/pre_migration_schema.txt similarity index 100% rename from airbyte-db/db-lib/src/test/resources/toys_database/pre_migration_schema.txt rename to airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/pre_migration_schema.txt diff --git a/airbyte-db/db-lib/src/test/resources/toys_database/schema.sql b/airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/schema.sql similarity index 100% rename from airbyte-db/db-lib/src/test/resources/toys_database/schema.sql rename to airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/schema.sql diff --git a/airbyte-db/db-lib/src/test/resources/toys_database/schema_dump.txt b/airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/schema_dump.txt similarity index 100% rename from airbyte-db/db-lib/src/test/resources/toys_database/schema_dump.txt rename to airbyte-cdk/java/airbyte-cdk/core/src/test/resources/toys_database/schema_dump.txt diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle new file mode 100644 index 000000000000..4668be6e7cca --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/build.gradle @@ -0,0 +1,95 @@ + +java { + compileJava { + options.compilerArgs += "-Xlint:-deprecation" + } +} + +dependencies { + // Depends on core CDK classes (OK 👍) + implementation project(':airbyte-cdk:java:airbyte-cdk:core') + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:typing-deduping')) + + compileOnly project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') + testImplementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') + testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:acceptance-test-harness') + + compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-api') + compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') + compileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') + compileOnly project(':airbyte-cdk:java:airbyte-cdk:init-oss') + compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') + + testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + + + testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') + testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:init-oss') + + + implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} + + testFixturesImplementation "org.hamcrest:hamcrest-all:1.3" + + implementation libs.bundles.junit + // implementation libs.junit.jupiter.api + implementation libs.junit.jupiter.params + implementation 'org.junit.platform:junit-platform-launcher:1.7.0' + implementation libs.jooq + testImplementation libs.junit.jupiter.engine + implementation 'net.sourceforge.argparse4j:argparse4j:0.8.1' + implementation "io.aesy:datasize:1.0.0" + implementation libs.apache.commons + implementation libs.apache.commons.lang + testImplementation 'commons-lang:commons-lang:2.6' + implementation 'commons-cli:commons-cli:1.4' + implementation 'org.apache.commons:commons-csv:1.4' + + implementation libs.google.cloud.storage + + // Optional dependencies + // TODO: Change these to 'compileOnly' or 'testCompileOnly' + implementation 'com.azure:azure-storage-blob:12.12.0' + implementation('com.google.cloud:google-cloud-bigquery:1.133.1') + implementation 'org.mongodb:mongodb-driver-sync:4.3.0' + implementation libs.postgresql + implementation ('org.apache.parquet:parquet-avro:1.12.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} + + // testImplementation libs.junit.jupiter.api + implementation libs.hikaricp + implementation libs.debezium.api + implementation libs.debezium.embedded + implementation libs.debezium.sqlserver + implementation libs.debezium.mysql + implementation libs.debezium.postgres + implementation libs.debezium.mongodb + + implementation libs.bundles.datadog + // implementation 'com.datadoghq:dd-trace-api' + implementation 'org.apache.sshd:sshd-mina:2.8.0' + + implementation libs.testcontainers + implementation libs.testcontainers.mysql + implementation libs.testcontainers.jdbc + implementation libs.testcontainers.postgresql + testImplementation libs.testcontainers.jdbc + testImplementation libs.testcontainers.mysql + testImplementation libs.testcontainers.postgresql + implementation 'org.codehaus.plexus:plexus-utils:3.4.2' + + implementation 'org.bouncycastle:bcprov-jdk15on:1.66' + + // Lombok + implementation 'org.projectlombok:lombok:1.18.20' + annotationProcessor 'org.projectlombok:lombok:1.18.20' + testFixturesImplementation 'org.projectlombok:lombok:1.18.20' + testFixturesAnnotationProcessor 'org.projectlombok:lombok:1.18.20' + + implementation ('org.apache.hadoop:hadoop-common:3.3.3') {exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.slf4j', module: 'slf4j-reload4j'} + implementation ('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.3') {exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.slf4j', module: 'slf4j-reload4j'} + + testImplementation libs.junit.jupiter.system.stubs +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java new file mode 100644 index 000000000000..d25b6ecb4296 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestination.java @@ -0,0 +1,329 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.jdbc; + +import static io.airbyte.cdk.integrations.base.errors.messages.ErrorMessage.getErrorMessage; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.JdbcConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.TypingAndDedupingFlag; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcDestinationHandler; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcV1V2Migrator; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteRecordMessage; +import io.airbyte.commons.exceptions.ConnectionErrorException; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.map.MoreMaps; +import io.airbyte.integrations.base.destination.typing_deduping.CatalogParser; +import io.airbyte.integrations.base.destination.typing_deduping.DefaultTyperDeduper; +import io.airbyte.integrations.base.destination.typing_deduping.DestinationHandler; +import io.airbyte.integrations.base.destination.typing_deduping.NoOpTyperDeduperWithV1V2Migrations; +import io.airbyte.integrations.base.destination.typing_deduping.NoopTyperDeduper; +import io.airbyte.integrations.base.destination.typing_deduping.NoopV2TableMigrator; +import io.airbyte.integrations.base.destination.typing_deduping.ParsedCatalog; +import io.airbyte.integrations.base.destination.typing_deduping.TyperDeduper; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; +import java.util.function.Consumer; +import javax.sql.DataSource; +import org.apache.commons.lang3.NotImplementedException; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class AbstractJdbcDestination extends JdbcConnector implements Destination { + + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractJdbcDestination.class); + + public static final String RAW_SCHEMA_OVERRIDE = "raw_data_schema"; + + public static final String DISABLE_TYPE_DEDUPE = "disable_type_dedupe"; + + private final NamingConventionTransformer namingResolver; + private final SqlOperations sqlOperations; + + protected NamingConventionTransformer getNamingResolver() { + return namingResolver; + } + + protected SqlOperations getSqlOperations() { + return sqlOperations; + } + + public AbstractJdbcDestination(final String driverClass, + final NamingConventionTransformer namingResolver, + final SqlOperations sqlOperations) { + super(driverClass); + this.namingResolver = namingResolver; + this.sqlOperations = sqlOperations; + } + + @Override + public AirbyteConnectionStatus check(final JsonNode config) { + final DataSource dataSource = getDataSource(config); + + try { + final JdbcDatabase database = getDatabase(dataSource); + final String outputSchema = namingResolver.getIdentifier(config.get(JdbcUtils.SCHEMA_KEY).asText()); + attemptTableOperations(outputSchema, database, namingResolver, sqlOperations, false); + if (TypingAndDedupingFlag.isDestinationV2()) { + final var v2RawSchema = namingResolver.getIdentifier(TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE) + .orElse(JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE)); + attemptTableOperations(v2RawSchema, database, namingResolver, sqlOperations, false); + destinationSpecificTableOperations(database); + } + return new AirbyteConnectionStatus().withStatus(Status.SUCCEEDED); + } catch (final ConnectionErrorException ex) { + final String message = getErrorMessage(ex.getStateCode(), ex.getErrorCode(), ex.getExceptionMessage(), ex); + AirbyteTraceMessageUtility.emitConfigErrorTrace(ex, message); + return new AirbyteConnectionStatus() + .withStatus(Status.FAILED) + .withMessage(message); + } catch (final Exception e) { + LOGGER.error("Exception while checking connection: ", e); + return new AirbyteConnectionStatus() + .withStatus(Status.FAILED) + .withMessage("Could not connect with provided configuration. \n" + e.getMessage()); + } finally { + try { + DataSourceFactory.close(dataSource); + } catch (final Exception e) { + LOGGER.warn("Unable to close data source.", e); + } + } + } + + /** + * Specific Databases may have additional checks unique to them which they need to perform, override + * this method to add additional checks. + * + * @param database the database to run checks against + * @throws Exception + */ + protected void destinationSpecificTableOperations(final JdbcDatabase database) throws Exception {} + + /** + * This method is deprecated. It verifies table creation, but not insert right to a newly created + * table. Use attemptTableOperations with the attemptInsert argument instead. + */ + @Deprecated + public static void attemptSQLCreateAndDropTableOperations(final String outputSchema, + final JdbcDatabase database, + final NamingConventionTransformer namingResolver, + final SqlOperations sqlOps) + throws Exception { + attemptTableOperations(outputSchema, database, namingResolver, sqlOps, false); + } + + /** + * Verifies if provided creds has enough permissions. Steps are: 1. Create schema if not exists. 2. + * Create test table. 3. Insert dummy record to newly created table if "attemptInsert" set to true. + * 4. Delete table created on step 2. + * + * @param outputSchema - schema to tests against. + * @param database - database to tests against. + * @param namingResolver - naming resolver. + * @param sqlOps - SqlOperations object + * @param attemptInsert - set true if need to make attempt to insert dummy records to newly created + * table. Set false to skip insert step. + */ + public static void attemptTableOperations(final String outputSchema, + final JdbcDatabase database, + final NamingConventionTransformer namingResolver, + final SqlOperations sqlOps, + final boolean attemptInsert) + throws Exception { + // verify we have write permissions on the target schema by creating a table with a random name, + // then dropping that table + try { + // Get metadata from the database to see whether connection is possible + database.bufferedResultSetQuery(conn -> conn.getMetaData().getCatalogs(), JdbcUtils.getDefaultSourceOperations()::rowToJson); + + // verify we have write permissions on the target schema by creating a table with a random name, + // then dropping that table + final String outputTableName = namingResolver.getIdentifier("_airbyte_connection_test_" + UUID.randomUUID().toString().replaceAll("-", "")); + sqlOps.createSchemaIfNotExists(database, outputSchema); + sqlOps.createTableIfNotExists(database, outputSchema, outputTableName); + // verify if user has permission to make SQL INSERT queries + try { + if (attemptInsert) { + sqlOps.insertRecords(database, List.of(getDummyRecord()), outputSchema, outputTableName); + } + } finally { + sqlOps.dropTableIfExists(database, outputSchema, outputTableName); + } + } catch (final SQLException e) { + if (Objects.isNull(e.getCause()) || !(e.getCause() instanceof SQLException)) { + throw new ConnectionErrorException(e.getSQLState(), e.getErrorCode(), e.getMessage(), e); + } else { + final SQLException cause = (SQLException) e.getCause(); + throw new ConnectionErrorException(e.getSQLState(), cause.getErrorCode(), cause.getMessage(), e); + } + } catch (final Exception e) { + throw new Exception(e); + } + } + + /** + * Generates a dummy AirbyteRecordMessage with random values. + * + * @return AirbyteRecordMessage object with dummy values that may be used to test insert permission. + */ + private static PartialAirbyteMessage getDummyRecord() { + final JsonNode dummyDataToInsert = Jsons.deserialize("{ \"field1\": true }"); + return new PartialAirbyteMessage() + .withRecord(new PartialAirbyteRecordMessage() + .withStream("stream1") + .withEmittedAt(1602637589000L)) + .withSerialized(dummyDataToInsert.toString()); + } + + /** + * Subclasses which need to modify the DataSource should override + * {@link #modifyDataSourceBuilder(DataSourceFactory.DataSourceBuilder)} rather than this method. + */ + @VisibleForTesting + public DataSource getDataSource(final JsonNode config) { + final JsonNode jdbcConfig = toJdbcConfig(config); + final Map connectionProperties = getConnectionProperties(config); + final DataSourceFactory.DataSourceBuilder builder = new DataSourceFactory.DataSourceBuilder( + jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText(), + jdbcConfig.has(JdbcUtils.PASSWORD_KEY) ? jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, + driverClassName, + jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText()) + .withConnectionProperties(connectionProperties) + .withConnectionTimeout(getConnectionTimeout(connectionProperties)); + return modifyDataSourceBuilder(builder).build(); + } + + protected DataSourceFactory.DataSourceBuilder modifyDataSourceBuilder(final DataSourceFactory.DataSourceBuilder builder) { + return builder; + } + + @VisibleForTesting + public JdbcDatabase getDatabase(final DataSource dataSource) { + return new DefaultJdbcDatabase(dataSource); + } + + protected Map getConnectionProperties(final JsonNode config) { + final Map customProperties = JdbcUtils.parseJdbcParameters(config, JdbcUtils.JDBC_URL_PARAMS_KEY); + final Map defaultProperties = getDefaultConnectionProperties(config); + assertCustomParametersDontOverwriteDefaultParameters(customProperties, defaultProperties); + return MoreMaps.merge(customProperties, defaultProperties); + } + + private void assertCustomParametersDontOverwriteDefaultParameters(final Map customParameters, + final Map defaultParameters) { + for (final String key : defaultParameters.keySet()) { + if (customParameters.containsKey(key) && !Objects.equals(customParameters.get(key), defaultParameters.get(key))) { + throw new IllegalArgumentException("Cannot overwrite default JDBC parameter " + key); + } + } + } + + protected abstract Map getDefaultConnectionProperties(final JsonNode config); + + public abstract JsonNode toJdbcConfig(JsonNode config); + + protected abstract JdbcSqlGenerator getSqlGenerator(); + + protected JdbcDestinationHandler getDestinationHandler(final String databaseName, final JdbcDatabase database) { + return new JdbcDestinationHandler(databaseName, database); + } + + /** + * "database" key at root of the config json, for any other variants in config, override this + * method. + * + * @param config + * @return + */ + protected String getDatabaseName(final JsonNode config) { + return config.get(JdbcUtils.DATABASE_KEY).asText(); + } + + @Override + public AirbyteMessageConsumer getConsumer(final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final Consumer outputRecordCollector) { + throw new NotImplementedException("Should use the getSerializedMessageConsumer instead"); + } + + @Override + public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final Consumer outputRecordCollector) + throws Exception { + final DataSource dataSource = getDataSource(config); + final JdbcDatabase database = getDatabase(dataSource); + if (TypingAndDedupingFlag.isDestinationV2()) { + // TODO: This logic exists in all V2 destinations. + // This is sad that if we forget to add this, there will be a null pointer during parseCatalog + final String defaultNamespace = config.get("schema").asText(); + for (final ConfiguredAirbyteStream stream : catalog.getStreams()) { + if (StringUtils.isEmpty(stream.getStream().getNamespace())) { + stream.getStream().setNamespace(defaultNamespace); + } + } + final JdbcSqlGenerator sqlGenerator = getSqlGenerator(); + final ParsedCatalog parsedCatalog = TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE) + .map(override -> new CatalogParser(sqlGenerator, override)) + .orElse(new CatalogParser(sqlGenerator)) + .parseCatalog(catalog); + final String databaseName = getDatabaseName(config); + final var migrator = new JdbcV1V2Migrator(namingResolver, database, databaseName); + final NoopV2TableMigrator v2TableMigrator = new NoopV2TableMigrator(); + final DestinationHandler destinationHandler = getDestinationHandler(databaseName, database); + final boolean disableTypeDedupe = config.has(DISABLE_TYPE_DEDUPE) && config.get(DISABLE_TYPE_DEDUPE).asBoolean(false); + final TyperDeduper typerDeduper; + if (disableTypeDedupe) { + typerDeduper = new NoOpTyperDeduperWithV1V2Migrations<>(sqlGenerator, destinationHandler, parsedCatalog, migrator, v2TableMigrator, + 8); + } else { + typerDeduper = + new DefaultTyperDeduper<>(sqlGenerator, destinationHandler, parsedCatalog, migrator, v2TableMigrator, 8); + } + return JdbcBufferedConsumerFactory.createAsync( + outputRecordCollector, + database, + sqlOperations, + namingResolver, + config, + catalog, + defaultNamespace, + typerDeduper); + } + return JdbcBufferedConsumerFactory.createAsync( + outputRecordCollector, + database, + sqlOperations, + namingResolver, + config, + catalog, + null, + new NoopTyperDeduper()); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/ColumnDefinition.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/ColumnDefinition.java new file mode 100644 index 000000000000..68e715d2cbb2 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/ColumnDefinition.java @@ -0,0 +1,11 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.jdbc; + +import java.sql.SQLType; + +public record ColumnDefinition(String name, String type, SQLType sqlType, int columnSize) { + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/CustomSqlType.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/CustomSqlType.java new file mode 100644 index 000000000000..dad853bb8e08 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/CustomSqlType.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.jdbc; + +import java.sql.SQLType; + +/** + * Custom SqlType definition when there is no mapping in {@link java.sql.JDBCType} + * + * @param name + * @param vendor + * @param vendorTypeNumber + */ +public record CustomSqlType(String name, String vendor, Integer vendorTypeNumber) implements SQLType { + + @Override + public String getName() { + return name; + } + + @Override + public String getVendor() { + return vendor; + } + + @Override + public Integer getVendorTypeNumber() { + return vendorTypeNumber; + } + +} diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/DataAdapter.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/DataAdapter.java similarity index 97% rename from airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/DataAdapter.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/DataAdapter.java index c445eeddacdd..826ce2c3cb31 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/DataAdapter.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/DataAdapter.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.jdbc; +package io.airbyte.cdk.integrations.destination.jdbc; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/JdbcBufferedConsumerFactory.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/JdbcBufferedConsumerFactory.java new file mode 100644 index 000000000000..d0d488c71284 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/JdbcBufferedConsumerFactory.java @@ -0,0 +1,232 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.jdbc; + +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE; +import static io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination.RAW_SCHEMA_OVERRIDE; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.base.Preconditions; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.TypingAndDedupingFlag; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.BufferedStreamConsumer; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.OnStartFunction; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.RecordWriter; +import io.airbyte.cdk.integrations.destination_async.AsyncStreamConsumer; +import io.airbyte.cdk.integrations.destination_async.OnCloseFunction; +import io.airbyte.cdk.integrations.destination_async.buffers.BufferManager; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import io.airbyte.integrations.base.destination.typing_deduping.TyperDeduper; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Strategy: + *

    + * 1. Create a final table for each stream + *

    + * 2. Accumulate records in a buffer. One buffer per stream + *

    + * 3. As records accumulate write them in batch to the database. We set a minimum numbers of records + * before writing to avoid wasteful record-wise writes. In the case with slow syncs this will be + * superseded with a periodic record flush from {@link BufferedStreamConsumer#periodicBufferFlush()} + *

    + * 4. Once all records have been written to buffer, flush the buffer and write any remaining records + * to the database (regardless of how few are left) + */ +public class JdbcBufferedConsumerFactory { + + private static final Logger LOGGER = LoggerFactory.getLogger(JdbcBufferedConsumerFactory.class); + + public static SerializedAirbyteMessageConsumer createAsync(final Consumer outputRecordCollector, + final JdbcDatabase database, + final SqlOperations sqlOperations, + final NamingConventionTransformer namingResolver, + final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final String defaultNamespace, + final TyperDeduper typerDeduper) { + final List writeConfigs = createWriteConfigs(namingResolver, config, catalog, sqlOperations.isSchemaRequired()); + return new AsyncStreamConsumer( + outputRecordCollector, + onStartFunction(database, sqlOperations, writeConfigs, typerDeduper), + onCloseFunction(typerDeduper), + new JdbcInsertFlushFunction(recordWriterFunction(database, sqlOperations, writeConfigs, catalog)), + catalog, + new BufferManager((long) (Runtime.getRuntime().maxMemory() * 0.2)), + defaultNamespace, + Executors.newFixedThreadPool(2)); + } + + private static List createWriteConfigs(final NamingConventionTransformer namingResolver, + final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final boolean schemaRequired) { + if (schemaRequired) { + Preconditions.checkState(config.has("schema"), "jdbc destinations must specify a schema."); + } + return catalog.getStreams().stream().map(toWriteConfig(namingResolver, config, schemaRequired)).collect(Collectors.toList()); + } + + private static Function toWriteConfig( + final NamingConventionTransformer namingResolver, + final JsonNode config, + final boolean schemaRequired) { + return stream -> { + Preconditions.checkNotNull(stream.getDestinationSyncMode(), "Undefined destination sync mode"); + final AirbyteStream abStream = stream.getStream(); + + final String defaultSchemaName = schemaRequired ? namingResolver.getIdentifier(config.get("schema").asText()) + : namingResolver.getIdentifier(config.get(JdbcUtils.DATABASE_KEY).asText()); + // Method checks for v2 + final String outputSchema = getOutputSchema(abStream, defaultSchemaName, namingResolver); + final String streamName = abStream.getName(); + final String tableName; + final String tmpTableName; + // TODO: Should this be injected from outside ? + if (TypingAndDedupingFlag.isDestinationV2()) { + final var finalSchema = Optional.ofNullable(abStream.getNamespace()).orElse(defaultSchemaName); + final var rawName = StreamId.concatenateRawTableName(finalSchema, streamName); + tableName = namingResolver.convertStreamName(rawName); + tmpTableName = namingResolver.getTmpTableName(rawName); + } else { + tableName = namingResolver.getRawTableName(streamName); + tmpTableName = namingResolver.getTmpTableName(streamName); + } + final DestinationSyncMode syncMode = stream.getDestinationSyncMode(); + + final WriteConfig writeConfig = new WriteConfig(streamName, abStream.getNamespace(), outputSchema, tmpTableName, tableName, syncMode); + LOGGER.info("Write config: {}", writeConfig); + + return writeConfig; + }; + } + + /** + * Defer to the {@link AirbyteStream}'s namespace. If this is not set, use the destination's default + * schema. This namespace is source-provided, and can be potentially empty. + *

    + * The logic here matches the logic in the catalog_process.py for Normalization. Any modifications + * need to be reflected there and vice versa. + */ + private static String getOutputSchema(final AirbyteStream stream, + final String defaultDestSchema, + final NamingConventionTransformer namingResolver) { + if (TypingAndDedupingFlag.isDestinationV2()) { + return namingResolver + .getNamespace(TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE).orElse(DEFAULT_AIRBYTE_INTERNAL_NAMESPACE)); + } else { + return namingResolver.getNamespace(Optional.ofNullable(stream.getNamespace()).orElse(defaultDestSchema)); + } + } + + /** + * Sets up destination storage through: + *

    + * 1. Creates Schema (if not exists) + *

    + * 2. Creates airybte_raw table (if not exists) + *

    + * 3. Truncates table if sync mode is in OVERWRITE + * + * @param database JDBC database to connect to + * @param sqlOperations interface for execution SQL queries + * @param writeConfigs settings for each stream + */ + private static OnStartFunction onStartFunction(final JdbcDatabase database, + final SqlOperations sqlOperations, + final Collection writeConfigs, + final TyperDeduper typerDeduper) { + return () -> { + typerDeduper.prepareTables(); + LOGGER.info("Preparing raw tables in destination started for {} streams", writeConfigs.size()); + final List queryList = new ArrayList<>(); + for (final WriteConfig writeConfig : writeConfigs) { + final String schemaName = writeConfig.getOutputSchemaName(); + final String dstTableName = writeConfig.getOutputTableName(); + LOGGER.info("Preparing raw table in destination started for stream {}. schema: {}, table name: {}", + writeConfig.getStreamName(), + schemaName, + dstTableName); + sqlOperations.createSchemaIfNotExists(database, schemaName); + sqlOperations.createTableIfNotExists(database, schemaName, dstTableName); + switch (writeConfig.getSyncMode()) { + case OVERWRITE -> queryList.add(sqlOperations.truncateTableQuery(database, schemaName, dstTableName)); + case APPEND, APPEND_DEDUP -> {} + default -> throw new IllegalStateException("Unrecognized sync mode: " + writeConfig.getSyncMode()); + } + } + sqlOperations.executeTransaction(database, queryList); + LOGGER.info("Preparing raw tables in destination completed."); + }; + } + + /** + * Writes {@link AirbyteRecordMessage} to JDBC database's airbyte_raw table + * + * @param database JDBC database to connect to + * @param sqlOperations interface of SQL queries to execute + * @param writeConfigs settings for each stream + * @param catalog catalog of all streams to sync + */ + private static RecordWriter recordWriterFunction(final JdbcDatabase database, + final SqlOperations sqlOperations, + final List writeConfigs, + final ConfiguredAirbyteCatalog catalog) { + final Map pairToWriteConfig = writeConfigs.stream() + .collect(Collectors.toUnmodifiableMap(JdbcBufferedConsumerFactory::toNameNamespacePair, Function.identity())); + + return (pair, records) -> { + if (!pairToWriteConfig.containsKey(pair)) { + throw new IllegalArgumentException( + String.format("Message contained record from a stream that was not in the catalog. \ncatalog: %s", Jsons.serialize(catalog))); + } + + final WriteConfig writeConfig = pairToWriteConfig.get(pair); + sqlOperations.insertRecords(database, records, writeConfig.getOutputSchemaName(), writeConfig.getOutputTableName()); + }; + } + + /** + * Tear down functionality + */ + private static OnCloseFunction onCloseFunction(final TyperDeduper typerDeduper) { + return (hasFailed, streamSyncSummaries) -> { + try { + typerDeduper.typeAndDedupe(streamSyncSummaries); + typerDeduper.commitFinalTables(); + typerDeduper.cleanup(); + } catch (final Exception e) { + throw new RuntimeException(e); + } + }; + } + + private static AirbyteStreamNameNamespacePair toNameNamespacePair(final WriteConfig config) { + return new AirbyteStreamNameNamespacePair(config.getStreamName(), config.getNamespace()); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/JdbcInsertFlushFunction.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/JdbcInsertFlushFunction.java new file mode 100644 index 000000000000..6990b73b3e6c --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/JdbcInsertFlushFunction.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.jdbc; + +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.RecordWriter; +import io.airbyte.cdk.integrations.destination.jdbc.constants.GlobalDataSizeConstants; +import io.airbyte.cdk.integrations.destination_async.DestinationFlushFunction; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.stream.Stream; + +public class JdbcInsertFlushFunction implements DestinationFlushFunction { + + private final RecordWriter recordWriter; + + public JdbcInsertFlushFunction(final RecordWriter recordWriter) { + this.recordWriter = recordWriter; + } + + @Override + public void flush(final StreamDescriptor desc, final Stream stream) throws Exception { + recordWriter.accept( + new AirbyteStreamNameNamespacePair(desc.getName(), desc.getNamespace()), + stream.toList()); + } + + @Override + public long getOptimalBatchSizeBytes() { + // TODO tune this value - currently SqlOperationUtils partitions 10K records per insert statement, + // but we'd like to stop doing that and instead control sql insert statement size via batch size. + return GlobalDataSizeConstants.DEFAULT_MAX_BATCH_SIZE_BYTES; + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/JdbcSqlOperations.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/JdbcSqlOperations.java new file mode 100644 index 000000000000..1ffd5f0c93ae --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/JdbcSqlOperations.java @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.jdbc; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.base.TypingAndDedupingFlag; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; +import io.airbyte.commons.exceptions.ConfigErrorException; +import io.airbyte.commons.json.Jsons; +import java.io.File; +import java.io.PrintWriter; +import java.nio.charset.StandardCharsets; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVPrinter; + +@SuppressWarnings("OptionalUsedAsFieldOrParameterType") +public abstract class JdbcSqlOperations implements SqlOperations { + + protected static final String SHOW_SCHEMAS = "show schemas;"; + protected static final String NAME = "name"; + + // this adapter modifies record message before inserting them to the destination + protected final Optional dataAdapter; + protected final Set schemaSet = new HashSet<>(); + + protected JdbcSqlOperations() { + this.dataAdapter = Optional.empty(); + } + + protected JdbcSqlOperations(final DataAdapter dataAdapter) { + this.dataAdapter = Optional.of(dataAdapter); + } + + @Override + public void createSchemaIfNotExists(final JdbcDatabase database, final String schemaName) throws Exception { + try { + if (!schemaSet.contains(schemaName) && !isSchemaExists(database, schemaName)) { + database.execute(String.format("CREATE SCHEMA IF NOT EXISTS %s;", schemaName)); + schemaSet.add(schemaName); + } + } catch (final Exception e) { + throw checkForKnownConfigExceptions(e).orElseThrow(() -> e); + } + } + + /** + * When an exception occurs, we may recognize it as an issue with the users permissions or other + * configuration options. In these cases, we can wrap the exception in a + * {@link ConfigErrorException} which will exclude the error from our on-call paging/reporting + * + * @param e the exception to check. + * @return A ConfigErrorException with a message with actionable feedback to the user. + */ + protected Optional checkForKnownConfigExceptions(final Exception e) { + return Optional.empty(); + } + + @Override + public void createTableIfNotExists(final JdbcDatabase database, final String schemaName, final String tableName) throws SQLException { + try { + database.execute(createTableQuery(database, schemaName, tableName)); + for (final String postCreateSql : postCreateTableQueries(schemaName, tableName)) { + database.execute(postCreateSql); + } + } catch (final SQLException e) { + throw checkForKnownConfigExceptions(e).orElseThrow(() -> e); + } + } + + @Override + public String createTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { + if (TypingAndDedupingFlag.isDestinationV2()) { + return createTableQueryV2(schemaName, tableName); + } else { + return createTableQueryV1(schemaName, tableName); + } + } + + /** + * Some subclasses may want to execute additional SQL statements after creating the raw table. For + * example, Postgres does not support index definitions within a CREATE TABLE statement, so we need + * to run CREATE INDEX statements after creating the table. + */ + protected List postCreateTableQueries(final String schemaName, final String tableName) { + return List.of(); + } + + protected String createTableQueryV1(final String schemaName, final String tableName) { + return String.format( + """ + CREATE TABLE IF NOT EXISTS %s.%s ( + %s VARCHAR PRIMARY KEY, + %s JSONB, + %s TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP + ); + """, + schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_ID, JavaBaseConstants.COLUMN_NAME_DATA, JavaBaseConstants.COLUMN_NAME_EMITTED_AT); + } + + protected String createTableQueryV2(final String schemaName, final String tableName) { + return String.format( + """ + CREATE TABLE IF NOT EXISTS %s.%s ( + %s VARCHAR PRIMARY KEY, + %s JSONB, + %s TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, + %s TIMESTAMP WITH TIME ZONE DEFAULT NULL + ); + """, + schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, JavaBaseConstants.COLUMN_NAME_DATA, + JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT, JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT); + } + + // TODO: This method seems to be used by Postgres and others while staging to local temp files. + // Should there be a Local staging operations equivalent + protected void writeBatchToFile(final File tmpFile, final List records) throws Exception { + try (final PrintWriter writer = new PrintWriter(tmpFile, StandardCharsets.UTF_8); + final CSVPrinter csvPrinter = new CSVPrinter(writer, CSVFormat.DEFAULT)) { + for (final PartialAirbyteMessage record : records) { + final var uuid = UUID.randomUUID().toString(); + // TODO we only need to do this is formatData is overridden. If not, we can just do jsonData = + // record.getSerialized() + final var jsonData = Jsons.serialize(formatData(Jsons.deserializeExact(record.getSerialized()))); + final var extractedAt = Timestamp.from(Instant.ofEpochMilli(record.getRecord().getEmittedAt())); + if (TypingAndDedupingFlag.isDestinationV2()) { + csvPrinter.printRecord(uuid, jsonData, extractedAt, null); + } else { + csvPrinter.printRecord(uuid, jsonData, extractedAt); + } + } + } + } + + protected JsonNode formatData(final JsonNode data) { + return data; + } + + @Override + public String truncateTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { + return String.format("TRUNCATE TABLE %s.%s;\n", schemaName, tableName); + } + + @Override + public String insertTableQuery(final JdbcDatabase database, final String schemaName, final String srcTableName, final String dstTableName) { + return String.format("INSERT INTO %s.%s SELECT * FROM %s.%s;\n", schemaName, dstTableName, schemaName, srcTableName); + } + + @Override + public void executeTransaction(final JdbcDatabase database, final List queries) throws Exception { + final StringBuilder appendedQueries = new StringBuilder(); + appendedQueries.append("BEGIN;\n"); + for (final String query : queries) { + appendedQueries.append(query); + } + appendedQueries.append("COMMIT;"); + database.execute(appendedQueries.toString()); + } + + @Override + public void dropTableIfExists(final JdbcDatabase database, final String schemaName, final String tableName) throws SQLException { + try { + database.execute(dropTableIfExistsQuery(schemaName, tableName)); + } catch (final SQLException e) { + throw checkForKnownConfigExceptions(e).orElseThrow(() -> e); + } + } + + public String dropTableIfExistsQuery(final String schemaName, final String tableName) { + return String.format("DROP TABLE IF EXISTS %s.%s;\n", schemaName, tableName); + } + + @Override + public boolean isSchemaRequired() { + return true; + } + + @Override + public boolean isValidData(final JsonNode data) { + return true; + } + + @Override + public final void insertRecords(final JdbcDatabase database, + final List records, + final String schemaName, + final String tableName) + throws Exception { + dataAdapter.ifPresent(adapter -> records.forEach(airbyteRecordMessage -> { + final JsonNode data = Jsons.deserializeExact(airbyteRecordMessage.getSerialized()); + adapter.adapt(data); + airbyteRecordMessage.setSerialized(Jsons.serialize(data)); + })); + if (TypingAndDedupingFlag.isDestinationV2()) { + insertRecordsInternalV2(database, records, schemaName, tableName); + } else { + insertRecordsInternal(database, records, schemaName, tableName); + } + } + + protected abstract void insertRecordsInternal(JdbcDatabase database, + List records, + String schemaName, + String tableName) + throws Exception; + + protected abstract void insertRecordsInternalV2(JdbcDatabase database, + List records, + String schemaName, + String tableName) + throws Exception; + +} diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/SqlOperations.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/SqlOperations.java similarity index 91% rename from airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/SqlOperations.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/SqlOperations.java index 510301d66aad..5f4d47e6eb76 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/SqlOperations.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/SqlOperations.java @@ -2,19 +2,16 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.jdbc; +package io.airbyte.cdk.integrations.destination.jdbc; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * todo (cgardens) - is it necessary to expose so much configurability in this interface. review if - * we can narrow the surface area. - * * SQL queries required for successfully syncing to a destination connector. These operations * include the ability to: *

      @@ -98,7 +95,7 @@ default boolean isSchemaExists(final JdbcDatabase database, final String schemaN * @param tableName Name of table * @throws Exception exception */ - void insertRecords(JdbcDatabase database, List records, String schemaName, String tableName) throws Exception; + void insertRecords(JdbcDatabase database, List records, String schemaName, String tableName) throws Exception; /** * Query to insert all records from source table to destination table. Both tables must be in the diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/SqlOperationsUtils.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/SqlOperationsUtils.java similarity index 78% rename from airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/SqlOperationsUtils.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/SqlOperationsUtils.java index bb6625e21979..9032c07c7cbd 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/SqlOperationsUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/SqlOperationsUtils.java @@ -2,13 +2,13 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.jdbc; +package io.airbyte.cdk.integrations.destination.jdbc; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Iterables; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.TypingAndDedupingFlag; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Timestamp; @@ -33,7 +33,7 @@ public class SqlOperationsUtils { public static void insertRawRecordsInSingleQuery(final String insertQueryComponent, final String recordQueryComponent, final JdbcDatabase jdbcDatabase, - final List records) + final List records) throws SQLException { insertRawRecordsInSingleQuery(insertQueryComponent, recordQueryComponent, jdbcDatabase, records, UUID::randomUUID, true); } @@ -54,7 +54,7 @@ public static void insertRawRecordsInSingleQuery(final String insertQueryCompone public static void insertRawRecordsInSingleQueryNoSem(final String insertQueryComponent, final String recordQueryComponent, final JdbcDatabase jdbcDatabase, - final List records) + final List records) throws SQLException { insertRawRecordsInSingleQuery(insertQueryComponent, recordQueryComponent, jdbcDatabase, records, UUID::randomUUID, false); } @@ -63,7 +63,7 @@ public static void insertRawRecordsInSingleQueryNoSem(final String insertQueryCo static void insertRawRecordsInSingleQuery(final String insertQueryComponent, final String recordQueryComponent, final JdbcDatabase jdbcDatabase, - final List records, + final List records, final Supplier uuidSupplier, final boolean sem) throws SQLException { @@ -83,7 +83,7 @@ static void insertRawRecordsInSingleQuery(final String insertQueryComponent, // how many records can be inserted at once // TODO(sherif) this should use a smarter, destination-aware partitioning scheme instead of 10k by // default - for (List partition : Iterables.partition(records, 10_000)) { + for (final List partition : Iterables.partition(records, 10_000)) { final StringBuilder sql = new StringBuilder(insertQueryComponent); partition.forEach(r -> sql.append(recordQueryComponent)); final String s = sql.toString(); @@ -91,13 +91,26 @@ static void insertRawRecordsInSingleQuery(final String insertQueryComponent, try (final PreparedStatement statement = connection.prepareStatement(s1)) { // second loop: bind values to the SQL string. + // 1-indexed int i = 1; - for (final AirbyteRecordMessage message : partition) { - // 1-indexed + for (final PartialAirbyteMessage message : partition) { + // Airbyte Raw ID statement.setString(i, uuidSupplier.get().toString()); - statement.setString(i + 1, Jsons.serialize(message.getData())); - statement.setTimestamp(i + 2, Timestamp.from(Instant.ofEpochMilli(message.getEmittedAt()))); - i += 3; + i++; + + // Message Data + statement.setString(i, message.getSerialized()); + i++; + + // Extracted At + statement.setTimestamp(i, Timestamp.from(Instant.ofEpochMilli(message.getRecord().getEmittedAt()))); + i++; + + if (TypingAndDedupingFlag.isDestinationV2()) { + // Loaded At + statement.setTimestamp(i, null); + i++; + } } statement.execute(); diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/StagingFilenameGenerator.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/StagingFilenameGenerator.java similarity index 92% rename from airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/StagingFilenameGenerator.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/StagingFilenameGenerator.java index e046d7106480..be12c50249b8 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/StagingFilenameGenerator.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/StagingFilenameGenerator.java @@ -2,9 +2,9 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.jdbc; +package io.airbyte.cdk.integrations.destination.jdbc; -import static io.airbyte.integrations.destination.jdbc.constants.GlobalDataSizeConstants.MAX_FILE_SIZE; +import static io.airbyte.cdk.integrations.destination.jdbc.constants.GlobalDataSizeConstants.MAX_FILE_SIZE; /** * The staging file is uploaded to cloud storage in multiple parts. This class keeps track of the diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/TableDefinition.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/TableDefinition.java new file mode 100644 index 000000000000..353d6d03cb44 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/TableDefinition.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.jdbc; + +import java.util.LinkedHashMap; + +/** + * Jdbc destination table definition representation + * + * @param columns + */ +public record TableDefinition(LinkedHashMap columns) { + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/TableSchemaRecordSet.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/TableSchemaRecordSet.java new file mode 100644 index 000000000000..f87d57218c48 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/TableSchemaRecordSet.java @@ -0,0 +1,9 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.jdbc; + +public record TableSchemaRecordSet() { + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/TypeInfoRecordSet.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/TypeInfoRecordSet.java new file mode 100644 index 000000000000..2ef35e795b24 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/TypeInfoRecordSet.java @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.jdbc; + +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.util.LinkedHashMap; + +/** + * A record representing the {@link java.sql.ResultSet} returned by calling + * {@link DatabaseMetaData#getTypeInfo()} + *

      + * See that method for a better description of the parameters to this record + */ +public record TypeInfoRecordSet( + String typeName, + int dataType, + int precision, + String literalPrefix, + String literalSuffix, + String createParams, + short nullable, + boolean caseSensitive, + short searchable, + boolean unsignedAttribute, + boolean fixedPrecScale, + boolean autoIncrement, + String localTypeName, + short minimumScale, + short maximumScale, + + // Unused + int sqlDataType, + + // Unused + int sqlDatetimeSub, + int numPrecRadix) { + + public static LinkedHashMap getTypeInfoList(final DatabaseMetaData databaseMetaData) throws Exception { + final LinkedHashMap types = new LinkedHashMap<>(); + try (final ResultSet rs = databaseMetaData.getTypeInfo()) { + while (rs.next()) { + final var typeName = rs.getString("TYPE_NAME"); + types.put(typeName, + new TypeInfoRecordSet( + typeName, + rs.getInt("DATA_TYPE"), + rs.getInt("PRECISION"), + rs.getString("LITERAL_PREFIX"), + rs.getString("LITERAL_SUFFIX"), + rs.getString("CREATE_PARAMS"), + rs.getShort("NULLABLE"), + rs.getBoolean("CASE_SENSITIVE"), + rs.getShort("SEARCHABLE"), + rs.getBoolean("UNSIGNED_ATTRIBUTE"), + rs.getBoolean("FIXED_PREC_SCALE"), + rs.getBoolean("AUTO_INCREMENT"), + rs.getString("LOCAL_TYPE_NAME"), + rs.getShort("MINIMUM_SCALE"), + rs.getShort("MAXIMUM_SCALE"), + rs.getInt("SQL_DATA_TYPE"), + rs.getInt("SQL_DATETIME_SUB"), + rs.getInt("NUM_PREC_RADIX"))); + } + } + return types; + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/WriteConfig.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/WriteConfig.java new file mode 100644 index 000000000000..fde85f704ec0 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/WriteConfig.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.jdbc; + +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; + +/** + * Write configuration POJO (plain old java object) for all destinations extending + * {@link AbstractJdbcDestination}. + */ +public class WriteConfig { + + private final String streamName; + private final String namespace; + private final String outputSchemaName; + private final String tmpTableName; + private final String outputTableName; + private final DestinationSyncMode syncMode; + private final DateTime writeDatetime; + + public WriteConfig(final String streamName, + final String namespace, + final String outputSchemaName, + final String tmpTableName, + final String outputTableName, + final DestinationSyncMode syncMode) { + this(streamName, namespace, outputSchemaName, tmpTableName, outputTableName, syncMode, DateTime.now(DateTimeZone.UTC)); + } + + public WriteConfig(final String streamName, + final String namespace, + final String outputSchemaName, + final String tmpTableName, + final String outputTableName, + final DestinationSyncMode syncMode, + final DateTime writeDatetime) { + this.streamName = streamName; + this.namespace = namespace; + this.outputSchemaName = outputSchemaName; + this.tmpTableName = tmpTableName; + this.outputTableName = outputTableName; + this.syncMode = syncMode; + this.writeDatetime = writeDatetime; + } + + public String getStreamName() { + return streamName; + } + + /** + * This is used in {@link JdbcBufferedConsumerFactory} to verify that record is from expected + * streams + * + * @return + */ + public String getNamespace() { + return namespace; + } + + public String getTmpTableName() { + return tmpTableName; + } + + public String getOutputSchemaName() { + return outputSchemaName; + } + + public String getOutputTableName() { + return outputTableName; + } + + public DestinationSyncMode getSyncMode() { + return syncMode; + } + + public DateTime getWriteDatetime() { + return writeDatetime; + } + + @Override + public String toString() { + return "WriteConfig{" + + "streamName=" + streamName + + ", namespace=" + namespace + + ", outputSchemaName=" + outputSchemaName + + ", tmpTableName=" + tmpTableName + + ", outputTableName=" + outputTableName + + ", syncMode=" + syncMode + + '}'; + } + +} diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/constants/GlobalDataSizeConstants.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/constants/GlobalDataSizeConstants.java similarity index 90% rename from airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/constants/GlobalDataSizeConstants.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/constants/GlobalDataSizeConstants.java index eb20778ab000..16f83a9a0955 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/constants/GlobalDataSizeConstants.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/constants/GlobalDataSizeConstants.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.jdbc.constants; +package io.airbyte.cdk.integrations.destination.jdbc.constants; import io.aesy.datasize.ByteUnit.IEC; import io.aesy.datasize.DataSize; diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/CopyConsumerFactory.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/CopyConsumerFactory.java similarity index 86% rename from airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/CopyConsumerFactory.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/CopyConsumerFactory.java index f9dfa14e58f1..d553f3bf2696 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/CopyConsumerFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/CopyConsumerFactory.java @@ -2,21 +2,21 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.jdbc.copy; - -import static io.airbyte.integrations.destination.jdbc.constants.GlobalDataSizeConstants.DEFAULT_MAX_BATCH_SIZE_BYTES; - -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.buffered_stream_consumer.BufferedStreamConsumer; -import io.airbyte.integrations.destination.buffered_stream_consumer.CheckAndRemoveRecordWriter; -import io.airbyte.integrations.destination.buffered_stream_consumer.OnCloseFunction; -import io.airbyte.integrations.destination.buffered_stream_consumer.OnStartFunction; -import io.airbyte.integrations.destination.buffered_stream_consumer.RecordWriter; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.record_buffer.InMemoryRecordBufferingStrategy; +package io.airbyte.cdk.integrations.destination.jdbc.copy; + +import static io.airbyte.cdk.integrations.destination.jdbc.constants.GlobalDataSizeConstants.DEFAULT_MAX_BATCH_SIZE_BYTES; + +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.BufferedStreamConsumer; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.CheckAndRemoveRecordWriter; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.OnCloseFunction; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.OnStartFunction; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.RecordWriter; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.record_buffer.InMemoryRecordBufferingStrategy; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; @@ -93,7 +93,7 @@ private static OnStartFunction onStartFunction(final Map recordWriterFunction(final Map pairToCopier, final SqlOperations sqlOperations, final Map pairToIgnoredRecordCount) { - return (AirbyteStreamNameNamespacePair pair, List records) -> { + return (final AirbyteStreamNameNamespacePair pair, final List records) -> { final var fileName = pairToCopier.get(pair).prepareStagingFile(); for (final AirbyteRecordMessage recordMessage : records) { final var id = UUID.randomUUID(); @@ -109,7 +109,7 @@ private static RecordWriter recordWriterFunction(final Map } private static CheckAndRemoveRecordWriter removeStagingFilePrinter(final Map pairToCopier) { - return (AirbyteStreamNameNamespacePair pair, String stagingFileName) -> { + return (final AirbyteStreamNameNamespacePair pair, final String stagingFileName) -> { final String currentFileName = pairToCopier.get(pair).getCurrentFile(); if (stagingFileName != null && currentFileName != null && !stagingFileName.equals(currentFileName)) { pairToCopier.get(pair).closeNonCurrentStagingFileWriters(); @@ -123,7 +123,7 @@ private static OnCloseFunction onCloseFunction(final Map pairToIgnoredRecordCount, final DataSource dataSource) { - return (hasFailed) -> { + return (hasFailed, streamSyncSummaries) -> { pairToIgnoredRecordCount .forEach((pair, count) -> LOGGER.warn("A total of {} record(s) of data from stream {} were invalid and were ignored.", count, pair)); closeAsOneTransaction(pairToCopier, hasFailed, database, sqlOperations, dataSource); diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/CopyDestination.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/CopyDestination.java similarity index 83% rename from airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/CopyDestination.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/CopyDestination.java index 18df4fa5e36b..a5d36f65dcd8 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/CopyDestination.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/CopyDestination.java @@ -2,21 +2,21 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.jdbc.copy; +package io.airbyte.cdk.integrations.destination.jdbc.copy; -import static io.airbyte.integrations.base.errors.messages.ErrorMessage.getErrorMessage; +import static io.airbyte.cdk.integrations.base.errors.messages.ErrorMessage.getErrorMessage; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; import io.airbyte.commons.exceptions.ConnectionErrorException; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteTraceMessageUtility; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; -import io.airbyte.integrations.destination.jdbc.SqlOperations; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import javax.sql.DataSource; import org.slf4j.Logger; diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/StreamCopier.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopier.java similarity index 97% rename from airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/StreamCopier.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopier.java index a885a42f39b9..a00446457a04 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/StreamCopier.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopier.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.jdbc.copy; +package io.airbyte.cdk.integrations.destination.jdbc.copy; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.util.UUID; diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/StreamCopierFactory.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopierFactory.java similarity index 77% rename from airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/StreamCopierFactory.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopierFactory.java index 6b2247ea8e34..adb4ff3c9b60 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/StreamCopierFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/StreamCopierFactory.java @@ -2,11 +2,11 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.jdbc.copy; +package io.airbyte.cdk.integrations.destination.jdbc.copy; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; public interface StreamCopierFactory { diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/SwitchingDestination.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/SwitchingDestination.java similarity index 91% rename from airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/SwitchingDestination.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/SwitchingDestination.java index ddfa0f535c1f..6ba5d77e2962 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/SwitchingDestination.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/SwitchingDestination.java @@ -2,14 +2,14 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.jdbc.copy; +package io.airbyte.cdk.integrations.destination.jdbc.copy; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Preconditions; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.SerializedAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageConfig.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageConfig.java new file mode 100644 index 000000000000..db5a806ea2a6 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageConfig.java @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.jdbc.copy.azure; + +import com.fasterxml.jackson.databind.JsonNode; +import java.util.Locale; + +public class AzureBlobStorageConfig { + + private static final String DEFAULT_STORAGE_ENDPOINT_DOMAIN_NAME = "blob.core.windows.net"; + + private final String endpointDomainName; + private final String accountName; + private final String containerName; + private final String sasToken; + + public AzureBlobStorageConfig( + String endpointDomainName, + String accountName, + String containerName, + String sasToken) { + this.endpointDomainName = endpointDomainName; + this.accountName = accountName; + this.containerName = containerName; + this.sasToken = sasToken; + } + + public String getEndpointDomainName() { + return endpointDomainName == null ? DEFAULT_STORAGE_ENDPOINT_DOMAIN_NAME : endpointDomainName; + } + + public String getAccountName() { + return accountName; + } + + public String getContainerName() { + return containerName; + } + + public String getSasToken() { + return sasToken; + } + + public String getEndpointUrl() { + return String.format(Locale.ROOT, "https://%s.%s", getAccountName(), getEndpointDomainName()); + } + + public static AzureBlobStorageConfig getAzureBlobConfig(JsonNode config) { + + return new AzureBlobStorageConfig( + config.get("azure_blob_storage_endpoint_domain_name") == null ? DEFAULT_STORAGE_ENDPOINT_DOMAIN_NAME + : config.get("azure_blob_storage_endpoint_domain_name").asText(), + config.get("azure_blob_storage_account_name").asText(), + config.get("azure_blob_storage_container_name").asText(), + config.get("azure_blob_storage_sas_token").asText()); + + } + +} diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopier.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopier.java similarity index 95% rename from airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopier.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopier.java index b16245920981..a4fb22120a21 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopier.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopier.java @@ -2,19 +2,19 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.jdbc.copy.azure; +package io.airbyte.cdk.integrations.destination.jdbc.copy.azure; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.specialized.AppendBlobClient; import com.azure.storage.blob.specialized.SpecializedBlobClientBuilder; import com.google.common.annotations.VisibleForTesting; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.StagingFilenameGenerator; +import io.airbyte.cdk.integrations.destination.jdbc.constants.GlobalDataSizeConstants; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopier; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.StagingFilenameGenerator; -import io.airbyte.integrations.destination.jdbc.constants.GlobalDataSizeConstants; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopier; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.DestinationSyncMode; import java.io.BufferedOutputStream; diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopierFactory.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopierFactory.java similarity index 85% rename from airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopierFactory.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopierFactory.java index dbed77a0ea12..043b2435d695 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopierFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/azure/AzureBlobStorageStreamCopierFactory.java @@ -2,14 +2,14 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.jdbc.copy.azure; +package io.airbyte.cdk.integrations.destination.jdbc.copy.azure; import com.azure.storage.blob.specialized.SpecializedBlobClientBuilder; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopier; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopierFactory; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopier; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopierFactory; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/gcs/GcsConfig.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsConfig.java similarity index 93% rename from airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/gcs/GcsConfig.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsConfig.java index 899458b99137..fc5ddfb90801 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/gcs/GcsConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsConfig.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.jdbc.copy.gcs; +package io.airbyte.cdk.integrations.destination.jdbc.copy.gcs; import com.fasterxml.jackson.databind.JsonNode; diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/gcs/GcsStreamCopier.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopier.java similarity index 95% rename from airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/gcs/GcsStreamCopier.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopier.java index 11574abd07b1..0c74a3853b2d 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/gcs/GcsStreamCopier.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopier.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.jdbc.copy.gcs; +package io.airbyte.cdk.integrations.destination.jdbc.copy.gcs; import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.WriteChannel; @@ -11,13 +11,13 @@ import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; import com.google.common.annotations.VisibleForTesting; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.StagingFilenameGenerator; +import io.airbyte.cdk.integrations.destination.jdbc.constants.GlobalDataSizeConstants; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopier; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.StagingFilenameGenerator; -import io.airbyte.integrations.destination.jdbc.constants.GlobalDataSizeConstants; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopier; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.DestinationSyncMode; import java.io.ByteArrayInputStream; diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/gcs/GcsStreamCopierFactory.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopierFactory.java similarity index 87% rename from airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/gcs/GcsStreamCopierFactory.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopierFactory.java index df1cbad060f4..3247a5c8fbf3 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/gcs/GcsStreamCopierFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/gcs/GcsStreamCopierFactory.java @@ -2,16 +2,16 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.jdbc.copy.gcs; +package io.airbyte.cdk.integrations.destination.jdbc.copy.gcs; import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopier; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopierFactory; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopier; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopierFactory; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcDestinationHandler.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcDestinationHandler.java new file mode 100644 index 000000000000..3981c07c5ad2 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcDestinationHandler.java @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.jdbc.typing_deduping; + +import static org.jooq.impl.DSL.exists; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.jooq.impl.DSL.select; +import static org.jooq.impl.DSL.selectOne; + +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.jdbc.ColumnDefinition; +import io.airbyte.cdk.integrations.destination.jdbc.CustomSqlType; +import io.airbyte.cdk.integrations.destination.jdbc.TableDefinition; +import io.airbyte.integrations.base.destination.typing_deduping.DestinationHandler; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import java.sql.DatabaseMetaData; +import java.sql.JDBCType; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLType; +import java.sql.Timestamp; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; +import lombok.extern.slf4j.Slf4j; +import org.jooq.conf.ParamType; +import org.jooq.impl.DSL; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Slf4j +public class JdbcDestinationHandler implements DestinationHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(JdbcDestinationHandler.class); + + protected final String databaseName; + protected final JdbcDatabase jdbcDatabase; + + public JdbcDestinationHandler(final String databaseName, + final JdbcDatabase jdbcDatabase) { + this.databaseName = databaseName; + this.jdbcDatabase = jdbcDatabase; + } + + @Override + public Optional findExistingTable(final StreamId id) throws Exception { + return findExistingTable(jdbcDatabase, databaseName, id.finalNamespace(), id.finalName()); + } + + @Override + public boolean isFinalTableEmpty(final StreamId id) throws Exception { + return !jdbcDatabase.queryBoolean( + select( + field(exists( + selectOne() + .from(name(id.finalNamespace(), id.finalName())) + .limit(1)))) + .getSQL(ParamType.INLINED)); + } + + @Override + public InitialRawTableState getInitialRawTableState(final StreamId id) throws Exception { + final ResultSet tables = jdbcDatabase.getMetaData().getTables( + databaseName, + id.rawNamespace(), + id.rawName(), + null); + if (!tables.next()) { + // There's no raw table at all. Therefore there are no unprocessed raw records, and this sync + // should not filter raw records by timestamp. + return new InitialRawTableState(false, Optional.empty()); + } + // And use two explicit queries because COALESCE might not short-circuit evaluation. + // This first query tries to find the oldest raw record with loaded_at = NULL. + // Unsafe query requires us to explicitly close the Stream, which is inconvenient, + // but it's also the only method in the JdbcDatabase interface to return non-string/int types + try (final Stream timestampStream = jdbcDatabase.unsafeQuery( + conn -> conn.prepareStatement( + select(field("MIN(_airbyte_extracted_at)").as("min_timestamp")) + .from(name(id.rawNamespace(), id.rawName())) + .where(DSL.condition("_airbyte_loaded_at IS NULL")) + .getSQL()), + record -> record.getTimestamp("min_timestamp"))) { + // Filter for nonNull values in case the query returned NULL (i.e. no unloaded records). + final Optional minUnloadedTimestamp = timestampStream.filter(Objects::nonNull).findFirst(); + if (minUnloadedTimestamp.isPresent()) { + // Decrement by 1 second since timestamp precision varies between databases. + final Optional ts = minUnloadedTimestamp + .map(Timestamp::toInstant) + .map(i -> i.minus(1, ChronoUnit.SECONDS)); + return new InitialRawTableState(true, ts); + } + } + // If there are no unloaded raw records, then we can safely skip all existing raw records. + // This second query just finds the newest raw record. + try (final Stream timestampStream = jdbcDatabase.unsafeQuery( + conn -> conn.prepareStatement( + select(field("MAX(_airbyte_extracted_at)").as("min_timestamp")) + .from(name(id.rawNamespace(), id.rawName())) + .getSQL()), + record -> record.getTimestamp("min_timestamp"))) { + // Filter for nonNull values in case the query returned NULL (i.e. no raw records at all). + final Optional minUnloadedTimestamp = timestampStream.filter(Objects::nonNull).findFirst(); + return new InitialRawTableState(false, minUnloadedTimestamp.map(Timestamp::toInstant)); + } + } + + @Override + public void execute(final Sql sql) throws Exception { + final List> transactions = sql.transactions(); + final UUID queryId = UUID.randomUUID(); + for (final List transaction : transactions) { + final UUID transactionId = UUID.randomUUID(); + LOGGER.info("Executing sql {}-{}: {}", queryId, transactionId, String.join("\n", transaction)); + final long startTime = System.currentTimeMillis(); + + try { + jdbcDatabase.executeWithinTransaction(transaction); + } catch (final SQLException e) { + LOGGER.error("Sql {}-{} failed", queryId, transactionId, e); + throw e; + } + + LOGGER.info("Sql {}-{} completed in {} ms", queryId, transactionId, System.currentTimeMillis() - startTime); + } + } + + public static Optional findExistingTable(final JdbcDatabase jdbcDatabase, + final String databaseName, + final String schemaName, + final String tableName) + throws SQLException { + final DatabaseMetaData metaData = jdbcDatabase.getMetaData(); + // TODO: normalize namespace and finalName strings to quoted-lowercase (as needed. Snowflake + // requires uppercase) + final LinkedHashMap columnDefinitions = new LinkedHashMap<>(); + try (final ResultSet columns = metaData.getColumns(databaseName, schemaName, tableName, null)) { + while (columns.next()) { + final String columnName = columns.getString("COLUMN_NAME"); + final String typeName = columns.getString("TYPE_NAME"); + final int columnSize = columns.getInt("COLUMN_SIZE"); + final int datatype = columns.getInt("DATA_TYPE"); + SQLType sqlType; + try { + sqlType = JDBCType.valueOf(datatype); + } catch (final IllegalArgumentException e) { + // Unknown jdbcType convert to customSqlType + LOGGER.warn("Unrecognized JDBCType {}; falling back to UNKNOWN", datatype, e); + sqlType = new CustomSqlType("Unknown", "Unknown", datatype); + } + columnDefinitions.put(columnName, new ColumnDefinition(columnName, typeName, sqlType, columnSize)); + } + } + // Guard to fail fast + if (columnDefinitions.isEmpty()) { + return Optional.empty(); + } + + return Optional.of(new TableDefinition(columnDefinitions)); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcSqlGenerator.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcSqlGenerator.java new file mode 100644 index 000000000000..0feb56ca9a9e --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcSqlGenerator.java @@ -0,0 +1,515 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.jdbc.typing_deduping; + +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_ID; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_META; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_RAW_ID; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_DATA; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_EMITTED_AT; +import static io.airbyte.integrations.base.destination.typing_deduping.Sql.transactionally; +import static java.util.stream.Collectors.toList; +import static org.jooq.impl.DSL.alterTable; +import static org.jooq.impl.DSL.asterisk; +import static org.jooq.impl.DSL.cast; +import static org.jooq.impl.DSL.dropTableIfExists; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.inline; +import static org.jooq.impl.DSL.name; +import static org.jooq.impl.DSL.noCondition; +import static org.jooq.impl.DSL.quotedName; +import static org.jooq.impl.DSL.select; +import static org.jooq.impl.DSL.table; +import static org.jooq.impl.DSL.update; +import static org.jooq.impl.DSL.with; + +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.TableDefinition; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType; +import io.airbyte.integrations.base.destination.typing_deduping.Array; +import io.airbyte.integrations.base.destination.typing_deduping.ColumnId; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; +import io.airbyte.integrations.base.destination.typing_deduping.SqlGenerator; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import io.airbyte.integrations.base.destination.typing_deduping.Struct; +import io.airbyte.integrations.base.destination.typing_deduping.Union; +import io.airbyte.integrations.base.destination.typing_deduping.UnsupportedOneOf; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; +import org.jooq.CommonTableExpression; +import org.jooq.Condition; +import org.jooq.CreateSchemaFinalStep; +import org.jooq.CreateTableColumnStep; +import org.jooq.DSLContext; +import org.jooq.DataType; +import org.jooq.Field; +import org.jooq.InsertValuesStepN; +import org.jooq.Name; +import org.jooq.Record; +import org.jooq.SQLDialect; +import org.jooq.SelectConditionStep; +import org.jooq.conf.ParamType; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; + +public abstract class JdbcSqlGenerator implements SqlGenerator { + + protected static final String ROW_NUMBER_COLUMN_NAME = "row_number"; + private static final String TYPING_CTE_ALIAS = "intermediate_data"; + private static final String NUMBERED_ROWS_CTE_ALIAS = "numbered_rows"; + + private final NamingConventionTransformer namingTransformer; + protected final ColumnId cdcDeletedAtColumn; + + public JdbcSqlGenerator(final NamingConventionTransformer namingTransformer) { + this.namingTransformer = namingTransformer; + this.cdcDeletedAtColumn = buildColumnId("_ab_cdc_deleted_at"); + } + + @Override + public StreamId buildStreamId(final String namespace, final String name, final String rawNamespaceOverride) { + return new StreamId( + namingTransformer.getNamespace(namespace), + namingTransformer.convertStreamName(name), + namingTransformer.getNamespace(rawNamespaceOverride), + namingTransformer.convertStreamName(StreamId.concatenateRawTableName(namespace, name)), + namespace, + name); + } + + @Override + public ColumnId buildColumnId(final String name, final String suffix) { + final String nameWithSuffix = name + suffix; + return new ColumnId( + namingTransformer.getIdentifier(nameWithSuffix), + name, + namingTransformer.getIdentifier(nameWithSuffix)); + } + + protected DataType toDialectType(final AirbyteType type) { + if (type instanceof final AirbyteProtocolType airbyteProtocolType) { + return toDialectType(airbyteProtocolType); + } + return switch (type.getTypeName()) { + case Struct.TYPE, UnsupportedOneOf.TYPE -> getStructType(); + case Array.TYPE -> getArrayType(); + // No nested Unions supported so this will definitely not result in infinite recursion. + case Union.TYPE -> toDialectType(((Union) type).chooseType()); + default -> throw new IllegalArgumentException("Unsupported AirbyteType: " + type); + }; + } + + @VisibleForTesting + public DataType toDialectType(final AirbyteProtocolType airbyteProtocolType) { + return switch (airbyteProtocolType) { + // Many destinations default to a very short length (e.g. Redshift defaults to 256). + // Explicitly set 64KiB here. Subclasses may want to override this value. + case STRING -> SQLDataType.VARCHAR(65535); + // We default to precision=38, scale=9 across destinations. + // This is the default numeric parameters for both redshift and bigquery. + case NUMBER -> SQLDataType.DECIMAL(38, 9); + case INTEGER -> SQLDataType.BIGINT; + case BOOLEAN -> SQLDataType.BOOLEAN; + case TIMESTAMP_WITH_TIMEZONE -> SQLDataType.TIMESTAMPWITHTIMEZONE; + case TIMESTAMP_WITHOUT_TIMEZONE -> SQLDataType.TIMESTAMP; + case TIME_WITH_TIMEZONE -> SQLDataType.TIMEWITHTIMEZONE; + case TIME_WITHOUT_TIMEZONE -> SQLDataType.TIME; + case DATE -> SQLDataType.DATE; + case UNKNOWN -> getWidestType(); + }; + } + + protected abstract DataType getStructType(); + + protected abstract DataType getArrayType(); + + @VisibleForTesting + public DataType getTimestampWithTimeZoneType() { + return toDialectType(AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); + } + + protected abstract DataType getWidestType(); + + protected abstract SQLDialect getDialect(); + + /** + * @param columns from the schema to be extracted from _airbyte_data column. Use the destination + * specific syntax to extract data + * @param useExpensiveSaferCasting + * @return a list of jooq fields for the final table insert statement. + */ + protected abstract List> extractRawDataFields(final LinkedHashMap columns, boolean useExpensiveSaferCasting); + + /** + * + * @param columns from the schema to be used for type casting errors and construct _airbyte_meta + * column + * @return + */ + protected abstract Field buildAirbyteMetaColumn(final LinkedHashMap columns); + + /** + * Get the cdc_deleted_at column condition for append_dedup mode by extracting it from _airbyte_data + * column in raw table. + * + * @return + */ + protected abstract Condition cdcDeletedAtNotNullCondition(); + + /** + * Get the window step function row_number() over (partition by primary_key order by cursor_field) + * as row_number. + * + * @param primaryKey list of primary keys + * @param cursorField cursor field used for ordering + * @return + */ + protected abstract Field getRowNumber(final List primaryKey, final Optional cursorField); + + protected DSLContext getDslContext() { + return DSL.using(getDialect()); + } + + /** + * build jooq fields for final table with customers columns first and then meta columns. + * + * @param columns + * @param metaColumns + * @return + */ + @VisibleForTesting + List> buildFinalTableFields(final LinkedHashMap columns, final Map> metaColumns) { + final List> fields = + metaColumns.entrySet().stream().map(metaColumn -> field(quotedName(metaColumn.getKey()), metaColumn.getValue())).collect(toList()); + final List> dataFields = + columns.entrySet().stream().map(column -> field(quotedName(column.getKey().name()), toDialectType(column.getValue()))).collect( + toList()); + dataFields.addAll(fields); + return dataFields; + } + + /** + * Use this method to get the final table meta columns with or without _airbyte_meta column. + * + * @param includeMetaColumn + * @return + */ + LinkedHashMap> getFinalTableMetaColumns(final boolean includeMetaColumn) { + final LinkedHashMap> metaColumns = new LinkedHashMap<>(); + metaColumns.put(COLUMN_NAME_AB_RAW_ID, SQLDataType.VARCHAR(36).nullable(false)); + metaColumns.put(COLUMN_NAME_AB_EXTRACTED_AT, getTimestampWithTimeZoneType().nullable(false)); + if (includeMetaColumn) + metaColumns.put(COLUMN_NAME_AB_META, getStructType().nullable(false)); + return metaColumns; + } + + /** + * build jooq fields for raw table with type-casted data columns first and then meta columns without + * _airbyte_meta. + * + * @param columns + * @param metaColumns + * @return + */ + @VisibleForTesting + List> buildRawTableSelectFields(final LinkedHashMap columns, + final Map> metaColumns, + final boolean useExpensiveSaferCasting) { + final List> fields = + metaColumns.entrySet().stream().map(metaColumn -> field(quotedName(metaColumn.getKey()), metaColumn.getValue())).collect(toList()); + // Use originalName with non-sanitized characters when extracting data from _airbyte_data + final List> dataFields = extractRawDataFields(columns, useExpensiveSaferCasting); + dataFields.addAll(fields); + return dataFields; + } + + @VisibleForTesting + Condition rawTableCondition(final DestinationSyncMode syncMode, final boolean isCdcDeletedAtPresent, final Optional minRawTimestamp) { + Condition condition = field(name(COLUMN_NAME_AB_LOADED_AT)).isNull(); + if (syncMode == DestinationSyncMode.APPEND_DEDUP) { + if (isCdcDeletedAtPresent) { + condition = condition.or(cdcDeletedAtNotNullCondition()); + } + } + if (minRawTimestamp.isPresent()) { + condition = condition.and(field(name(COLUMN_NAME_AB_EXTRACTED_AT)).gt(minRawTimestamp.get().toString())); + } + return condition; + } + + @Override + public Sql createSchema(final String schema) { + return Sql.of(createSchemaSql(schema)); + } + + @Override + public Sql createTable(final StreamConfig stream, final String suffix, final boolean force) { + // TODO: Use Naming transformer to sanitize these strings with redshift restrictions. + final String finalTableIdentifier = stream.id().finalName() + suffix.toLowerCase(); + if (!force) { + return transactionally(Stream.concat( + Stream.of(createTableSql(stream.id().finalNamespace(), finalTableIdentifier, stream.columns())), + createIndexSql(stream, suffix).stream()).toList()); + } + return transactionally(Stream.concat( + Stream.of( + dropTableIfExists(quotedName(stream.id().finalNamespace(), finalTableIdentifier)).getSQL(ParamType.INLINED), + createTableSql(stream.id().finalNamespace(), finalTableIdentifier, stream.columns())), + createIndexSql(stream, suffix).stream()).toList()); + } + + @Override + public Sql updateTable(final StreamConfig streamConfig, + final String finalSuffix, + final Optional minRawTimestamp, + final boolean useExpensiveSaferCasting) { + + // TODO: Add flag to use merge vs insert/delete + return insertAndDeleteTransaction(streamConfig, finalSuffix, minRawTimestamp, useExpensiveSaferCasting); + + } + + @Override + public Sql overwriteFinalTable(final StreamId stream, final String finalSuffix) { + return transactionally( + dropTableIfExists(name(stream.finalNamespace(), stream.finalName())).getSQL(ParamType.INLINED), + alterTable(name(stream.finalNamespace(), stream.finalName() + finalSuffix)) + .renameTo(name(stream.finalName())) + .getSQL()); + } + + @Override + public Sql migrateFromV1toV2(final StreamId streamId, final String namespace, final String tableName) { + final Name rawTableName = name(streamId.rawNamespace(), streamId.rawName()); + final DSLContext dsl = getDslContext(); + return transactionally( + dsl.createSchemaIfNotExists(streamId.rawNamespace()).getSQL(), + dsl.dropTableIfExists(rawTableName).getSQL(), + DSL.createTable(rawTableName) + .column(COLUMN_NAME_AB_RAW_ID, SQLDataType.VARCHAR(36).nullable(false)) + .column(COLUMN_NAME_AB_EXTRACTED_AT, getTimestampWithTimeZoneType().nullable(false)) + .column(COLUMN_NAME_AB_LOADED_AT, getTimestampWithTimeZoneType().nullable(false)) + .column(COLUMN_NAME_DATA, getStructType().nullable(false)) + .as(select( + field(COLUMN_NAME_AB_ID).as(COLUMN_NAME_AB_RAW_ID), + field(COLUMN_NAME_EMITTED_AT).as(COLUMN_NAME_AB_EXTRACTED_AT), + cast(null, getTimestampWithTimeZoneType()).as(COLUMN_NAME_AB_LOADED_AT), + field(COLUMN_NAME_DATA).as(COLUMN_NAME_DATA)).from(table(name(namespace, tableName)))) + .getSQL(ParamType.INLINED)); + } + + @Override + public Sql clearLoadedAt(final StreamId streamId) { + return Sql.of(update(table(name(streamId.rawNamespace(), streamId.rawName()))) + .set(field(COLUMN_NAME_AB_LOADED_AT), inline((String) null)) + .getSQL()); + } + + @VisibleForTesting + SelectConditionStep selectFromRawTable(final String schemaName, + final String tableName, + final LinkedHashMap columns, + final Map> metaColumns, + final Condition condition, + final boolean useExpensiveSaferCasting) { + final DSLContext dsl = getDslContext(); + return dsl + .select(buildRawTableSelectFields(columns, metaColumns, useExpensiveSaferCasting)) + .select(buildAirbyteMetaColumn(columns)) + .from(table(quotedName(schemaName, tableName))) + .where(condition); + } + + @VisibleForTesting + InsertValuesStepN insertIntoFinalTable(final String schemaName, + final String tableName, + final LinkedHashMap columns, + final Map> metaFields) { + final DSLContext dsl = getDslContext(); + return dsl + .insertInto(table(quotedName(schemaName, tableName))) + .columns(buildFinalTableFields(columns, metaFields)); + } + + private Sql insertAndDeleteTransaction(final StreamConfig streamConfig, + final String finalSuffix, + final Optional minRawTimestamp, + final boolean useExpensiveSaferCasting) { + final String finalSchema = streamConfig.id().finalNamespace(); + final String finalTable = streamConfig.id().finalName() + (finalSuffix != null ? finalSuffix.toLowerCase() : ""); + final String rawSchema = streamConfig.id().rawNamespace(); + final String rawTable = streamConfig.id().rawName(); + + // Poor person's guarantee of ordering of fields by using same source of ordered list of columns to + // generate fields. + final CommonTableExpression rawTableRowsWithCast = name(TYPING_CTE_ALIAS).as( + selectFromRawTable(rawSchema, rawTable, streamConfig.columns(), + getFinalTableMetaColumns(false), + rawTableCondition(streamConfig.destinationSyncMode(), + streamConfig.columns().containsKey(cdcDeletedAtColumn), + minRawTimestamp), + useExpensiveSaferCasting)); + final List> finalTableFields = buildFinalTableFields(streamConfig.columns(), getFinalTableMetaColumns(true)); + final Field rowNumber = getRowNumber(streamConfig.primaryKey(), streamConfig.cursor()); + final CommonTableExpression filteredRows = name(NUMBERED_ROWS_CTE_ALIAS).as( + select(asterisk(), rowNumber).from(rawTableRowsWithCast)); + + // Used for append-dedupe mode. + final String insertStmtWithDedupe = + insertIntoFinalTable(finalSchema, finalTable, streamConfig.columns(), getFinalTableMetaColumns(true)) + .select(with(rawTableRowsWithCast) + .with(filteredRows) + .select(finalTableFields) + .from(filteredRows) + .where(field(name(ROW_NUMBER_COLUMN_NAME), Integer.class).eq(1)) // Can refer by CTE.field but no use since we don't strongly type + // them. + ) + .getSQL(ParamType.INLINED); + + // Used for append and overwrite modes. + final String insertStmt = + insertIntoFinalTable(finalSchema, finalTable, streamConfig.columns(), getFinalTableMetaColumns(true)) + .select(with(rawTableRowsWithCast) + .select(finalTableFields) + .from(rawTableRowsWithCast)) + .getSQL(ParamType.INLINED); + final String deleteStmt = deleteFromFinalTable(finalSchema, finalTable, streamConfig.primaryKey(), streamConfig.cursor()); + final String deleteCdcDeletesStmt = + streamConfig.columns().containsKey(cdcDeletedAtColumn) ? deleteFromFinalTableCdcDeletes(finalSchema, finalTable) : ""; + final String checkpointStmt = checkpointRawTable(rawSchema, rawTable, minRawTimestamp); + + if (streamConfig.destinationSyncMode() != DestinationSyncMode.APPEND_DEDUP) { + return transactionally( + insertStmt, + checkpointStmt); + } + + // For append-dedupe + return transactionally( + insertStmtWithDedupe, + deleteStmt, + deleteCdcDeletesStmt, + checkpointStmt); + } + + private String mergeTransaction(final StreamConfig streamConfig, + final String finalSuffix, + final Optional minRawTimestamp, + final boolean useExpensiveSaferCasting) { + + throw new UnsupportedOperationException("Not implemented yet"); + + } + + protected String createSchemaSql(final String namespace) { + final DSLContext dsl = getDslContext(); + final CreateSchemaFinalStep createSchemaSql = dsl.createSchemaIfNotExists(quotedName(namespace)); + return createSchemaSql.getSQL(); + } + + protected String createTableSql(final String namespace, final String tableName, final LinkedHashMap columns) { + final DSLContext dsl = getDslContext(); + final CreateTableColumnStep createTableSql = dsl + .createTable(quotedName(namespace, tableName)) + .columns(buildFinalTableFields(columns, getFinalTableMetaColumns(true))); + return createTableSql.getSQL(); + } + + /** + * Subclasses may override this method to add additional indexes after their CREATE TABLE statement. + * This is useful if the destination's CREATE TABLE statement does not accept an index definition. + */ + protected List createIndexSql(final StreamConfig stream, final String suffix) { + return Collections.emptyList(); + } + + protected String beginTransaction() { + return "BEGIN"; + } + + protected String commitTransaction() { + return "COMMIT"; + } + + private String commitTransactionInternal() { + return commitTransaction() + ";"; + } + + private String deleteFromFinalTable(final String schemaName, + final String tableName, + final List primaryKeys, + final Optional cursor) { + final DSLContext dsl = getDslContext(); + // Unknown type doesn't play well with where .. in (select..) + final Field airbyteRawId = field(quotedName(COLUMN_NAME_AB_RAW_ID)); + final Field rowNumber = getRowNumber(primaryKeys, cursor); + return dsl.deleteFrom(table(quotedName(schemaName, tableName))) + .where(airbyteRawId.in( + select(airbyteRawId) + .from(select(airbyteRawId, rowNumber) + .from(table(quotedName(schemaName, tableName))).asTable("airbyte_ids")) + .where(field(name(ROW_NUMBER_COLUMN_NAME)).ne(1)))) + .getSQL(ParamType.INLINED); + } + + private String deleteFromFinalTableCdcDeletes(final String schema, final String tableName) { + final DSLContext dsl = getDslContext(); + return dsl.deleteFrom(table(quotedName(schema, tableName))) + .where(field(quotedName(cdcDeletedAtColumn.name())).isNotNull()) + .getSQL(ParamType.INLINED); + } + + private String checkpointRawTable(final String schemaName, final String tableName, final Optional minRawTimestamp) { + final DSLContext dsl = getDslContext(); + Condition extractedAtCondition = noCondition(); + if (minRawTimestamp.isPresent()) { + extractedAtCondition = extractedAtCondition.and(field(name(COLUMN_NAME_AB_EXTRACTED_AT)).gt(minRawTimestamp.get().toString())); + } + return dsl.update(table(quotedName(schemaName, tableName))) + .set(field(quotedName(COLUMN_NAME_AB_LOADED_AT)), currentTimestamp()) + .where(field(quotedName(COLUMN_NAME_AB_LOADED_AT)).isNull()).and(extractedAtCondition) + .getSQL(ParamType.INLINED); + } + + protected Field castedField( + final Field field, + final AirbyteType type, + final String alias, + final boolean useExpensiveSaferCasting) { + if (type instanceof final AirbyteProtocolType airbyteProtocolType) { + return castedField(field, airbyteProtocolType, useExpensiveSaferCasting).as(quotedName(alias)); + } + + // Redshift SUPER can silently cast an array type to struct and vice versa. + return switch (type.getTypeName()) { + case Struct.TYPE, UnsupportedOneOf.TYPE -> cast(field, getStructType()).as(quotedName(alias)); + case Array.TYPE -> cast(field, getArrayType()).as(quotedName(alias)); + // No nested Unions supported so this will definitely not result in infinite recursion. + case Union.TYPE -> castedField(field, ((Union) type).chooseType(), alias, useExpensiveSaferCasting); + default -> throw new IllegalArgumentException("Unsupported AirbyteType: " + type); + }; + } + + protected Field castedField(final Field field, final AirbyteProtocolType type, final boolean useExpensiveSaferCasting) { + return cast(field, toDialectType(type)); + } + + protected Field currentTimestamp() { + return DSL.currentTimestamp(); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcV1V2Migrator.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcV1V2Migrator.java new file mode 100644 index 000000000000..d5374140fc3e --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/typing_deduping/JdbcV1V2Migrator.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.jdbc.typing_deduping; + +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.TableDefinition; +import io.airbyte.integrations.base.destination.typing_deduping.BaseDestinationV1V2Migrator; +import io.airbyte.integrations.base.destination.typing_deduping.NamespacedTableName; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; +import java.sql.ResultSet; +import java.util.Collection; +import java.util.Optional; +import lombok.SneakyThrows; + +/** + * Largely based on + * {@link io.airbyte.integrations.destination.snowflake.typing_deduping.SnowflakeV1V2Migrator}. + */ +public class JdbcV1V2Migrator extends BaseDestinationV1V2Migrator { + + private final NamingConventionTransformer namingConventionTransformer; + private final JdbcDatabase database; + private final String databaseName; + + public JdbcV1V2Migrator(final NamingConventionTransformer namingConventionTransformer, final JdbcDatabase database, final String databaseName) { + this.namingConventionTransformer = namingConventionTransformer; + this.database = database; + this.databaseName = databaseName; + } + + @SneakyThrows + @Override + protected boolean doesAirbyteInternalNamespaceExist(final StreamConfig streamConfig) throws Exception { + String retrievedSchema = ""; + try (ResultSet columns = database.getMetaData().getSchemas(databaseName, streamConfig.id().rawNamespace())) { + while (columns.next()) { + retrievedSchema = columns.getString("TABLE_SCHEM"); + // Catalog can be null, so don't do anything with it. + String catalog = columns.getString("TABLE_CATALOG"); + } + } + return !retrievedSchema.isEmpty(); + } + + @Override + protected boolean schemaMatchesExpectation(final TableDefinition existingTable, final Collection columns) { + return existingTable.columns().keySet().containsAll(columns); + } + + @SneakyThrows + @Override + protected Optional getTableIfExists(final String namespace, final String tableName) throws Exception { + return JdbcDestinationHandler.findExistingTable(database, databaseName, namespace, tableName); + } + + @Override + protected NamespacedTableName convertToV1RawName(final StreamConfig streamConfig) { + @SuppressWarnings("deprecation") + final String tableName = this.namingConventionTransformer.getRawTableName(streamConfig.id().originalName()); + return new NamespacedTableName( + this.namingConventionTransformer.getIdentifier(streamConfig.id().originalNamespace()), + tableName); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/GeneralStagingFunctions.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/GeneralStagingFunctions.java new file mode 100644 index 000000000000..b01962a17bc7 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/GeneralStagingFunctions.java @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.staging; + +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.OnCloseFunction; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.OnStartFunction; +import io.airbyte.cdk.integrations.destination.jdbc.WriteConfig; +import io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeOperationValve; +import io.airbyte.integrations.base.destination.typing_deduping.TyperDeduper; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.locks.Lock; +import lombok.extern.slf4j.Slf4j; + +/** + * Functions and logic common to all flushing strategies. + */ +@Slf4j +public class GeneralStagingFunctions { + + // using a random string here as a placeholder for the moment. + // This would avoid mixing data in the staging area between different syncs (especially if they + // manipulate streams with similar names) + // if we replaced the random connection id by the actual connection_id, we'd gain the opportunity to + // leverage data that was uploaded to stage + // in a previous attempt but failed to load to the warehouse for some reason (interrupted?) instead. + // This would also allow other programs/scripts + // to load (or reload backups?) in the connection's staging area to be loaded at the next sync. + public static final UUID RANDOM_CONNECTION_ID = UUID.randomUUID(); + + public static OnStartFunction onStartFunction(final JdbcDatabase database, + final StagingOperations stagingOperations, + final List writeConfigs, + final TyperDeduper typerDeduper) { + return () -> { + log.info("Preparing raw tables in destination started for {} streams", writeConfigs.size()); + typerDeduper.prepareTables(); + final List queryList = new ArrayList<>(); + for (final WriteConfig writeConfig : writeConfigs) { + final String schema = writeConfig.getOutputSchemaName(); + final String stream = writeConfig.getStreamName(); + final String dstTableName = writeConfig.getOutputTableName(); + final String stagingPath = + stagingOperations.getStagingPath(SerialStagingConsumerFactory.RANDOM_CONNECTION_ID, schema, stream, writeConfig.getOutputTableName(), + writeConfig.getWriteDatetime()); + + log.info("Preparing staging area in destination started for schema {} stream {}: target table: {}, stage: {}", + schema, stream, dstTableName, stagingPath); + + stagingOperations.createSchemaIfNotExists(database, schema); + stagingOperations.createTableIfNotExists(database, schema, dstTableName); + stagingOperations.createStageIfNotExists(); + + /* + * When we're in OVERWRITE, clear out the table at the start of a sync, this is an expected side + * effect of checkpoint and the removal of temporary tables + */ + switch (writeConfig.getSyncMode()) { + case OVERWRITE -> queryList.add(stagingOperations.truncateTableQuery(database, schema, dstTableName)); + case APPEND, APPEND_DEDUP -> {} + default -> throw new IllegalStateException("Unrecognized sync mode: " + writeConfig.getSyncMode()); + } + + log.info("Preparing staging area in destination completed for schema {} stream {}", schema, stream); + } + log.info("Executing finalization of tables."); + stagingOperations.executeTransaction(database, queryList); + }; + } + + /** + * Handles copying data from staging area to destination table and clean up of staged files if + * upload was unsuccessful + */ + public static void copyIntoTableFromStage(final JdbcDatabase database, + final String stagingPath, + final List stagedFiles, + final String tableName, + final String schemaName, + final StagingOperations stagingOperations, + final String streamNamespace, + final String streamName, + final TypeAndDedupeOperationValve typerDeduperValve, + final TyperDeduper typerDeduper) + throws Exception { + try { + final Lock rawTableInsertLock = typerDeduper.getRawTableInsertLock(streamNamespace, streamName); + rawTableInsertLock.lock(); + try { + stagingOperations.copyIntoTableFromStage(database, stagingPath, stagedFiles, + tableName, schemaName); + } finally { + rawTableInsertLock.unlock(); + } + + final AirbyteStreamNameNamespacePair streamId = new AirbyteStreamNameNamespacePair(streamName, streamNamespace); + typerDeduperValve.addStreamIfAbsent(streamId); + if (typerDeduperValve.readyToTypeAndDedupe(streamId)) { + typerDeduper.typeAndDedupe(streamId.getNamespace(), streamId.getName(), false); + typerDeduperValve.updateTimeAndIncreaseInterval(streamId); + } + } catch (final Exception e) { + throw new RuntimeException("Failed to upload data from stage " + stagingPath, e); + } + } + + /** + * Tear down process, will attempt to try to clean out any staging area + * + * @param database database used for syncing + * @param stagingOperations collection of SQL queries necessary for writing data into a staging area + * @param writeConfigs configuration settings for all destination connectors needed to write + * @param purgeStagingData drop staging area if true, keep otherwise + * @return + */ + public static OnCloseFunction onCloseFunction(final JdbcDatabase database, + final StagingOperations stagingOperations, + final List writeConfigs, + final boolean purgeStagingData, + final TyperDeduper typerDeduper) { + return (hasFailed, streamSyncSummaries) -> { + // After moving data from staging area to the target table (airybte_raw) clean up the staging + // area (if user configured) + log.info("Cleaning up destination started for {} streams", writeConfigs.size()); + typerDeduper.typeAndDedupe(streamSyncSummaries); + for (final WriteConfig writeConfig : writeConfigs) { + final String schemaName = writeConfig.getOutputSchemaName(); + if (purgeStagingData) { + final String stagePath = stagingOperations.getStagingPath( + RANDOM_CONNECTION_ID, + schemaName, + writeConfig.getStreamName(), + writeConfig.getOutputTableName(), + writeConfig.getWriteDatetime()); + log.info("Cleaning stage in destination started for stream {}. schema {}, stage: {}", writeConfig.getStreamName(), schemaName, + stagePath); + stagingOperations.dropStageIfExists(database, stagePath); + } + } + typerDeduper.commitFinalTables(); + typerDeduper.cleanup(); + log.info("Cleaning up destination completed."); + }; + } + +} diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/SerialFlush.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/SerialFlush.java similarity index 85% rename from airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/SerialFlush.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/SerialFlush.java index 1757bfbd3c23..a4cb0c5fdaf3 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/SerialFlush.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/SerialFlush.java @@ -2,18 +2,18 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.staging; +package io.airbyte.cdk.integrations.destination.staging; import static java.util.stream.Collectors.joining; import com.google.common.annotations.VisibleForTesting; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.jdbc.WriteConfig; +import io.airbyte.cdk.integrations.destination.record_buffer.FlushBufferFunction; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeOperationValve; import io.airbyte.integrations.base.destination.typing_deduping.TyperDeduper; -import io.airbyte.integrations.destination.jdbc.WriteConfig; -import io.airbyte.integrations.destination.record_buffer.FlushBufferFunction; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import java.util.HashMap; @@ -50,8 +50,8 @@ public static FlushBufferFunction function( final StagingOperations stagingOperations, final List writeConfigs, final ConfiguredAirbyteCatalog catalog, - TypeAndDedupeOperationValve typerDeduperValve, - TyperDeduper typerDeduper) { + final TypeAndDedupeOperationValve typerDeduperValve, + final TyperDeduper typerDeduper) { // TODO: (ryankfu) move this block of code that executes before the lambda to #onStartFunction final Set conflictingStreams = new HashSet<>(); final Map pairToWriteConfig = new HashMap<>(); @@ -81,14 +81,14 @@ public static FlushBufferFunction function( final WriteConfig writeConfig = pairToWriteConfig.get(pair); final String schemaName = writeConfig.getOutputSchemaName(); - final String stageName = stagingOperations.getStageName(schemaName, writeConfig.getStreamName()); final String stagingPath = - stagingOperations.getStagingPath(StagingConsumerFactory.RANDOM_CONNECTION_ID, schemaName, writeConfig.getStreamName(), - writeConfig.getWriteDatetime()); + stagingOperations.getStagingPath( + SerialStagingConsumerFactory.RANDOM_CONNECTION_ID, schemaName, writeConfig.getStreamName(), + writeConfig.getOutputTableName(), writeConfig.getWriteDatetime()); try (writer) { writer.flush(); - final String stagedFile = stagingOperations.uploadRecordsToStage(database, writer, schemaName, stageName, stagingPath); - GeneralStagingFunctions.copyIntoTableFromStage(database, stageName, stagingPath, List.of(stagedFile), writeConfig.getOutputTableName(), + final String stagedFile = stagingOperations.uploadRecordsToStage(database, writer, schemaName, stagingPath); + GeneralStagingFunctions.copyIntoTableFromStage(database, stagingPath, List.of(stagedFile), writeConfig.getOutputTableName(), schemaName, stagingOperations, writeConfig.getNamespace(), diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/SerialStagingConsumerFactory.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/SerialStagingConsumerFactory.java new file mode 100644 index 000000000000..dc37391f8b06 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/SerialStagingConsumerFactory.java @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.staging; + +import static java.util.stream.Collectors.toList; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.base.Preconditions; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.BufferedStreamConsumer; +import io.airbyte.cdk.integrations.destination.jdbc.WriteConfig; +import io.airbyte.cdk.integrations.destination.record_buffer.BufferCreateFunction; +import io.airbyte.cdk.integrations.destination.record_buffer.SerializedBufferingStrategy; +import io.airbyte.integrations.base.destination.typing_deduping.ParsedCatalog; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeOperationValve; +import io.airbyte.integrations.base.destination.typing_deduping.TyperDeduper; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import java.util.List; +import java.util.UUID; +import java.util.function.Consumer; +import java.util.function.Function; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Uses both Factory and Consumer design pattern to create a single point of creation for consuming + * {@link AirbyteMessage} for processing + */ +public class SerialStagingConsumerFactory { + + private static final Logger LOGGER = LoggerFactory.getLogger(SerialStagingConsumerFactory.class); + + // using a random string here as a placeholder for the moment. + // This would avoid mixing data in the staging area between different syncs (especially if they + // manipulate streams with similar names) + // if we replaced the random connection id by the actual connection_id, we'd gain the opportunity to + // leverage data that was uploaded to stage + // in a previous attempt but failed to load to the warehouse for some reason (interrupted?) instead. + // This would also allow other programs/scripts + // to load (or reload backups?) in the connection's staging area to be loaded at the next sync. + private static final DateTime SYNC_DATETIME = DateTime.now(DateTimeZone.UTC); + public static final UUID RANDOM_CONNECTION_ID = UUID.randomUUID(); + + public AirbyteMessageConsumer create(final Consumer outputRecordCollector, + final JdbcDatabase database, + final StagingOperations stagingOperations, + final NamingConventionTransformer namingResolver, + final BufferCreateFunction onCreateBuffer, + final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final boolean purgeStagingData, + final TypeAndDedupeOperationValve typerDeduperValve, + final TyperDeduper typerDeduper, + final ParsedCatalog parsedCatalog, + final String defaultNamespace, + final boolean useDestinationsV2Columns) { + final List writeConfigs = createWriteConfigs(namingResolver, config, catalog, parsedCatalog, useDestinationsV2Columns); + return new BufferedStreamConsumer( + outputRecordCollector, + GeneralStagingFunctions.onStartFunction(database, stagingOperations, writeConfigs, typerDeduper), + new SerializedBufferingStrategy( + onCreateBuffer, + catalog, + SerialFlush.function(database, stagingOperations, writeConfigs, catalog, typerDeduperValve, typerDeduper)), + GeneralStagingFunctions.onCloseFunction(database, stagingOperations, writeConfigs, purgeStagingData, typerDeduper), + catalog, + stagingOperations::isValidData, + defaultNamespace); + } + + /** + * Creates a list of all {@link WriteConfig} for each stream within a + * {@link ConfiguredAirbyteCatalog}. Each write config represents the configuration settings for + * writing to a destination connector + * + * @param namingResolver {@link NamingConventionTransformer} used to transform names that are + * acceptable by each destination connector + * @param config destination connector configuration parameters + * @param catalog {@link ConfiguredAirbyteCatalog} collection of configured + * {@link ConfiguredAirbyteStream} + * @return list of all write configs for each stream in a {@link ConfiguredAirbyteCatalog} + */ + private static List createWriteConfigs(final NamingConventionTransformer namingResolver, + final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final ParsedCatalog parsedCatalog, + final boolean useDestinationsV2Columns) { + + return catalog.getStreams().stream().map(toWriteConfig(namingResolver, config, parsedCatalog, useDestinationsV2Columns)).collect(toList()); + } + + private static Function toWriteConfig(final NamingConventionTransformer namingResolver, + final JsonNode config, + final ParsedCatalog parsedCatalog, + final boolean useDestinationsV2Columns) { + return stream -> { + Preconditions.checkNotNull(stream.getDestinationSyncMode(), "Undefined destination sync mode"); + final AirbyteStream abStream = stream.getStream(); + final String streamName = abStream.getName(); + + final String outputSchema; + final String tableName; + if (useDestinationsV2Columns) { + final StreamId streamId = parsedCatalog.getStream(abStream.getNamespace(), streamName).id(); + outputSchema = streamId.rawNamespace(); + tableName = streamId.rawName(); + } else { + outputSchema = getOutputSchema(abStream, config.get("schema").asText(), namingResolver); + tableName = namingResolver.getRawTableName(streamName); + } + final String tmpTableName = namingResolver.getTmpTableName(streamName); + final DestinationSyncMode syncMode = stream.getDestinationSyncMode(); + + final WriteConfig writeConfig = + new WriteConfig(streamName, abStream.getNamespace(), outputSchema, tmpTableName, tableName, syncMode, SYNC_DATETIME); + LOGGER.info("Write config: {}", writeConfig); + + return writeConfig; + }; + } + + private static String getOutputSchema(final AirbyteStream stream, + final String defaultDestSchema, + final NamingConventionTransformer namingResolver) { + return stream.getNamespace() != null + ? namingResolver.getNamespace(stream.getNamespace()) + : namingResolver.getNamespace(defaultDestSchema); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingOperations.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingOperations.java new file mode 100644 index 000000000000..aac9351b4b7d --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/StagingOperations.java @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.staging; + +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.record_buffer.SerializableBuffer; +import java.util.List; +import java.util.UUID; +import org.joda.time.DateTime; + +/** + * Staging operations focuses on the SQL queries that are needed to success move data into a staging + * environment like GCS or S3. In general, the reference of staging is the usage of an object + * storage for the purposes of efficiently uploading bulk data to destinations + */ +public interface StagingOperations extends SqlOperations { + + /** + * @param outputTableName The name of the table this staging file will be loaded into (typically a + * raw table). Not all destinations use the table name in the staging path (e.g. Snowflake + * simply uses a timestamp + UUID), but e.g. Redshift does rely on this to ensure uniqueness. + */ + String getStagingPath(UUID connectionId, String namespace, String streamName, String outputTableName, DateTime writeDatetime); + + /** + * Create a staging folder where to upload temporary files before loading into the final destination + */ + void createStageIfNotExists() throws Exception; + + /** + * Upload the data file into the stage area. + * + * @param database database used for syncing + * @param recordsData records stored in in-memory buffer + * @param schemaName name of schema + * @param stagingPath path of staging folder to data files + * @return the name of the file that was uploaded. + */ + String uploadRecordsToStage(JdbcDatabase database, SerializableBuffer recordsData, String schemaName, String stagingPath) + throws Exception; + + /** + * Load the data stored in the stage area into a temporary table in the destination + * + * @param database database interface + * @param stagingPath path to staging files + * @param stagedFiles collection of staged files + * @param tableName name of table to write staging files to + * @param schemaName name of schema + */ + void copyIntoTableFromStage(JdbcDatabase database, + String stagingPath, + List stagedFiles, + String tableName, + String schemaName) + throws Exception; + + /** + * Delete the stage area and all staged files that was in it + * + * @param database database used for syncing + * @param stageName Name of the staging area used to store files + */ + void dropStageIfExists(JdbcDatabase database, String stageName) throws Exception; + +} diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/resources/spec.json b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/resources/spec.json similarity index 100% rename from airbyte-integrations/bases/bases-destination-jdbc/src/main/resources/spec.json rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/resources/spec.json diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestinationTest.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestinationTest.java similarity index 92% rename from airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestinationTest.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestinationTest.java index 302aee22529a..f532b8ba8766 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestinationTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/AbstractJdbcDestinationTest.java @@ -2,17 +2,18 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.jdbc; +package io.airbyte.cdk.integrations.destination.jdbc; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.destination.StandardNameTransformer; import java.util.HashMap; import java.util.Map; import org.junit.jupiter.api.Test; @@ -132,6 +133,12 @@ public JsonNode toJdbcConfig(final JsonNode config) { return config; } + @Override + protected JdbcSqlGenerator getSqlGenerator() { + // TODO do we need to populate this? + return null; + } + } } diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/DataAdapterTest.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/DataAdapterTest.java similarity index 97% rename from airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/DataAdapterTest.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/DataAdapterTest.java index 808ce78e267a..6c2f68efe796 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/DataAdapterTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/DataAdapterTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.jdbc; +package io.airbyte.cdk.integrations.destination.jdbc; import static org.junit.jupiter.api.Assertions.*; diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/TestJdbcSqlOperations.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/TestJdbcSqlOperations.java new file mode 100644 index 000000000000..0ed8606d608f --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/TestJdbcSqlOperations.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.jdbc; + +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; +import java.sql.SQLException; +import java.util.List; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class TestJdbcSqlOperations extends JdbcSqlOperations { + + @Override + public void insertRecordsInternal(final JdbcDatabase database, + final List records, + final String schemaName, + final String tableName) + throws Exception { + // Not required for the testing + } + + @Override + protected void insertRecordsInternalV2(final JdbcDatabase database, + final List records, + final String schemaName, + final String tableName) + throws Exception { + // Not required for the testing + } + + @Test + public void testCreateSchemaIfNotExists() { + final JdbcDatabase db = Mockito.mock(JdbcDatabase.class); + final var schemaName = "foo"; + try { + Mockito.doThrow(new SQLException("TEST")).when(db).execute(Mockito.anyString()); + } catch (final Exception e) { + // This would not be expected, but the `execute` method above will flag as an unhandled exception + assert false; + } + final SQLException exception = Assertions.assertThrows(SQLException.class, () -> createSchemaIfNotExists(db, schemaName)); + Assertions.assertEquals(exception.getMessage(), "TEST"); + } + +} diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/copy/SwitchingDestinationTest.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/copy/SwitchingDestinationTest.java similarity index 95% rename from airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/copy/SwitchingDestinationTest.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/copy/SwitchingDestinationTest.java index cd0b03a451a5..e8ea8f8e12c8 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/copy/SwitchingDestinationTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/copy/SwitchingDestinationTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.jdbc.copy; +package io.airbyte.cdk.integrations.destination.jdbc.copy; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.ArgumentMatchers.any; @@ -12,7 +12,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; -import io.airbyte.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.Destination; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import java.util.Map; import java.util.function.Consumer; diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/test/java/io/airbyte/cdk/integrations/destination/staging/SerialStagingConsumerFactoryTest.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/test/java/io/airbyte/cdk/integrations/destination/staging/SerialStagingConsumerFactoryTest.java new file mode 100644 index 000000000000..4255c9be9884 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/test/java/io/airbyte/cdk/integrations/destination/staging/SerialStagingConsumerFactoryTest.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.staging; + +import static org.junit.jupiter.api.Assertions.*; + +import io.airbyte.cdk.integrations.destination.jdbc.WriteConfig; +import io.airbyte.commons.exceptions.ConfigErrorException; +import java.util.List; +import org.junit.jupiter.api.Test; + +class SerialStagingConsumerFactoryTest { + + @Test() + void detectConflictingStreams() { + final ConfigErrorException configErrorException = assertThrows( + ConfigErrorException.class, + () -> SerialFlush.function( + null, + null, + List.of( + new WriteConfig("example_stream", "source_schema", "destination_default_schema", null, null, null), + new WriteConfig("example_stream", "source_schema", "destination_default_schema", null, null, null)), + null, + null, + null)); + + assertEquals( + "You are trying to write multiple streams to the same table. Consider switching to a custom namespace format using ${SOURCE_NAMESPACE}, or moving one of them into a separate connection with a different stream prefix. Affected streams: source_schema.example_stream, source_schema.example_stream", + configErrorException.getMessage()); + } + +} diff --git a/airbyte-integrations/bases/standard-destination-test/src/test/java/io/airbyte/integrations/standardtest/destination/TestingNamespacesTest.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/test/java/io/airbyte/cdk/integrations/standardtest/destination/TestingNamespacesTest.java similarity index 96% rename from airbyte-integrations/bases/standard-destination-test/src/test/java/io/airbyte/integrations/standardtest/destination/TestingNamespacesTest.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/test/java/io/airbyte/cdk/integrations/standardtest/destination/TestingNamespacesTest.java index 1963174c0052..59faa94eefaa 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/test/java/io/airbyte/integrations/standardtest/destination/TestingNamespacesTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/test/java/io/airbyte/cdk/integrations/standardtest/destination/TestingNamespacesTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.destination; +package io.airbyte.cdk.integrations.standardtest.destination; import static org.junit.jupiter.api.Assertions.*; diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.java similarity index 98% rename from airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.java index 5a7425710a03..6aa6f1e6e055 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTest.java @@ -2,13 +2,13 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.destination; +package io.airbyte.cdk.integrations.standardtest.destination; -import static io.airbyte.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider.INFINITY_TYPE_MESSAGE; -import static io.airbyte.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider.INTEGER_TYPE_CATALOG; -import static io.airbyte.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider.NAN_TYPE_MESSAGE; -import static io.airbyte.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider.NUMBER_TYPE_CATALOG; -import static io.airbyte.integrations.standardtest.destination.argproviders.util.ArgumentProviderUtil.prefixFileNameByVersion; +import static io.airbyte.cdk.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider.INFINITY_TYPE_MESSAGE; +import static io.airbyte.cdk.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider.INTEGER_TYPE_CATALOG; +import static io.airbyte.cdk.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider.NAN_TYPE_MESSAGE; +import static io.airbyte.cdk.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider.NUMBER_TYPE_CATALOG; +import static io.airbyte.cdk.integrations.standardtest.destination.argproviders.util.ArgumentProviderUtil.prefixFileNameByVersion; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -19,6 +19,11 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.argproviders.DataArgumentsProvider; +import io.airbyte.cdk.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.BasicTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.features.EnvVariableFeatureFlags; import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; @@ -33,11 +38,6 @@ import io.airbyte.configoss.StandardDestinationDefinition; import io.airbyte.configoss.WorkerDestinationConfig; import io.airbyte.configoss.init.LocalDefinitionsProvider; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.standardtest.destination.argproviders.DataArgumentsProvider; -import io.airbyte.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider; -import io.airbyte.integrations.standardtest.destination.comparator.BasicTestDataComparator; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; @@ -99,7 +99,7 @@ public abstract class DestinationAcceptanceTest { - protected static final HashSet TEST_SCHEMAS = new HashSet<>(); + protected HashSet TEST_SCHEMAS; private static final Random RANDOM = new Random(); private static final String NORMALIZATION_VERSION = "dev"; @@ -357,7 +357,7 @@ void setUpInternal() throws Exception { LOGGER.info("localRoot: {}", localRoot); testEnv = new TestDestinationEnv(localRoot); mConnectorConfigUpdater = Mockito.mock(ConnectorConfigUpdater.class); - + TEST_SCHEMAS = new HashSet<>(); setup(testEnv, TEST_SCHEMAS); processFactory = new DockerProcessFactory( @@ -1268,6 +1268,13 @@ protected void runSyncAndVerifyStateOutput(final JsonNode config, .stream() .filter(m -> m.getType() == Type.STATE) .findFirst() + .map(msg -> { + // Modify state message to remove destination stats. + final AirbyteStateMessage clone = msg.getState(); + clone.setDestinationStats(null); + msg.setState(clone); + return msg; + }) .orElseGet(() -> { fail("Destination failed to output state"); return null; @@ -1629,7 +1636,7 @@ protected static SpecialNumericTypes getSpecialNumericTypesSupportTest() { /** * The method should be overridden if destination connector support newer protocol version otherwise - * {@link io.airbyte.integrations.standardtest.destination.ProtocolVersion#V0} is used + * {@link io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion#V0} is used *

      * NOTE: Method should be public in a sake of java reflection * diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTestUtils.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTestUtils.java similarity index 90% rename from airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTestUtils.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTestUtils.java index d6e2b874aa83..a22bb3d2a2e8 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/DestinationAcceptanceTestUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/DestinationAcceptanceTestUtils.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.destination; +package io.airbyte.cdk.integrations.standardtest.destination; import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.commons.json.Jsons; diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/JdbcDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/JdbcDestinationAcceptanceTest.java new file mode 100644 index 000000000000..f2c0c6b1e750 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/JdbcDestinationAcceptanceTest.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.standardtest.destination; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.util.Arrays; +import java.util.Optional; +import java.util.function.Function; +import org.jooq.Record; + +public abstract class JdbcDestinationAcceptanceTest extends DestinationAcceptanceTest { + + protected final ObjectMapper mapper = new ObjectMapper(); + + protected JsonNode getJsonFromRecord(final Record record) { + return getJsonFromRecord(record, x -> Optional.empty()); + } + + protected JsonNode getJsonFromRecord(final Record record, final Function> valueParser) { + final ObjectNode node = mapper.createObjectNode(); + + Arrays.stream(record.fields()).forEach(field -> { + final var value = record.get(field); + + final Optional parsedValue = valueParser.apply(value); + if (parsedValue.isPresent()) { + node.put(field.getName(), parsedValue.get()); + } else { + switch (field.getDataType().getTypeName()) { + case "varchar", "nvarchar", "jsonb", "json", "other": + final var stringValue = (value != null ? value.toString() : null); + DestinationAcceptanceTestUtils.putStringIntoJson(stringValue, field.getName(), node); + break; + default: + node.put(field.getName(), (value != null ? value.toString() : null)); + } + } + }); + return node; + } + +} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/LocalAirbyteDestination.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/LocalAirbyteDestination.java similarity index 91% rename from airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/LocalAirbyteDestination.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/LocalAirbyteDestination.java index 7145809c3c3a..7e016559bf92 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/LocalAirbyteDestination.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/LocalAirbyteDestination.java @@ -2,12 +2,12 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.destination; +package io.airbyte.cdk.integrations.standardtest.destination; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; import io.airbyte.commons.json.Jsons; import io.airbyte.configoss.WorkerDestinationConfig; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.workers.internal.AirbyteDestination; import java.nio.file.Path; diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/PerStreamStateMessageTest.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/PerStreamStateMessageTest.java similarity index 94% rename from airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/PerStreamStateMessageTest.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/PerStreamStateMessageTest.java index c44ed35e0053..6ddf6876015a 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/PerStreamStateMessageTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/PerStreamStateMessageTest.java @@ -2,10 +2,10 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.destination; +package io.airbyte.cdk.integrations.standardtest.destination; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.AirbyteStateMessage; diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/ProtocolVersion.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/ProtocolVersion.java similarity index 81% rename from airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/ProtocolVersion.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/ProtocolVersion.java index 1a402f152418..d95daa23dc56 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/ProtocolVersion.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/ProtocolVersion.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.destination; +package io.airbyte.cdk.integrations.standardtest.destination; public enum ProtocolVersion { diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/TestingNamespaces.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/TestingNamespaces.java similarity index 98% rename from airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/TestingNamespaces.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/TestingNamespaces.java index a4391db59870..37530ad8fcbe 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/TestingNamespaces.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/TestingNamespaces.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.destination; +package io.airbyte.cdk.integrations.standardtest.destination; import java.time.Instant; import java.time.LocalDate; diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/DataArgumentsProvider.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/argproviders/DataArgumentsProvider.java similarity index 84% rename from airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/DataArgumentsProvider.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/argproviders/DataArgumentsProvider.java index 918c5fea856a..991da1aed63b 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/DataArgumentsProvider.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/argproviders/DataArgumentsProvider.java @@ -2,12 +2,12 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.destination.argproviders; +package io.airbyte.cdk.integrations.standardtest.destination.argproviders; -import static io.airbyte.integrations.standardtest.destination.argproviders.util.ArgumentProviderUtil.getProtocolVersion; -import static io.airbyte.integrations.standardtest.destination.argproviders.util.ArgumentProviderUtil.prefixFileNameByVersion; +import static io.airbyte.cdk.integrations.standardtest.destination.argproviders.util.ArgumentProviderUtil.getProtocolVersion; +import static io.airbyte.cdk.integrations.standardtest.destination.argproviders.util.ArgumentProviderUtil.prefixFileNameByVersion; -import io.airbyte.integrations.standardtest.destination.ProtocolVersion; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; import java.util.stream.Stream; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.params.provider.Arguments; diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/DataTypeTestArgumentProvider.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/argproviders/DataTypeTestArgumentProvider.java similarity index 94% rename from airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/DataTypeTestArgumentProvider.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/argproviders/DataTypeTestArgumentProvider.java index 1921edb76506..bb2bd81c9fb4 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/DataTypeTestArgumentProvider.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/argproviders/DataTypeTestArgumentProvider.java @@ -2,11 +2,11 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.destination.argproviders; +package io.airbyte.cdk.integrations.standardtest.destination.argproviders; -import static io.airbyte.integrations.standardtest.destination.argproviders.util.ArgumentProviderUtil.getProtocolVersion; +import static io.airbyte.cdk.integrations.standardtest.destination.argproviders.util.ArgumentProviderUtil.getProtocolVersion; -import io.airbyte.integrations.standardtest.destination.ProtocolVersion; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; import java.util.stream.Stream; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.params.provider.Arguments; diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/NumberDataTypeTestArgumentProvider.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/argproviders/NumberDataTypeTestArgumentProvider.java similarity index 77% rename from airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/NumberDataTypeTestArgumentProvider.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/argproviders/NumberDataTypeTestArgumentProvider.java index f04d3ca2ae94..41de26d32b19 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/NumberDataTypeTestArgumentProvider.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/argproviders/NumberDataTypeTestArgumentProvider.java @@ -2,12 +2,12 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.destination.argproviders; +package io.airbyte.cdk.integrations.standardtest.destination.argproviders; -import static io.airbyte.integrations.standardtest.destination.argproviders.util.ArgumentProviderUtil.getProtocolVersion; -import static io.airbyte.integrations.standardtest.destination.argproviders.util.ArgumentProviderUtil.prefixFileNameByVersion; +import static io.airbyte.cdk.integrations.standardtest.destination.argproviders.util.ArgumentProviderUtil.getProtocolVersion; +import static io.airbyte.cdk.integrations.standardtest.destination.argproviders.util.ArgumentProviderUtil.prefixFileNameByVersion; -import io.airbyte.integrations.standardtest.destination.ProtocolVersion; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; import java.util.stream.Stream; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.params.provider.Arguments; diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/util/ArgumentProviderUtil.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/argproviders/util/ArgumentProviderUtil.java similarity index 82% rename from airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/util/ArgumentProviderUtil.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/argproviders/util/ArgumentProviderUtil.java index 76452f2c8828..23a8454add98 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/argproviders/util/ArgumentProviderUtil.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/argproviders/util/ArgumentProviderUtil.java @@ -2,9 +2,9 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.destination.argproviders.util; +package io.airbyte.cdk.integrations.standardtest.destination.argproviders.util; -import io.airbyte.integrations.standardtest.destination.ProtocolVersion; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; import java.lang.reflect.Method; import org.junit.jupiter.api.extension.ExtensionContext; @@ -14,8 +14,8 @@ public class ArgumentProviderUtil { /** * This method use - * {@link io.airbyte.integrations.standardtest.destination.ProtocolVersion#getPrefix()} to prefix - * the file name. + * {@link io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion#getPrefix()} to + * prefix the file name. *

      * example: *

      diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/comparator/AdvancedTestDataComparator.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/comparator/AdvancedTestDataComparator.java similarity index 99% rename from airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/comparator/AdvancedTestDataComparator.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/comparator/AdvancedTestDataComparator.java index ed85be2f52e6..d39eeb794cb5 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/comparator/AdvancedTestDataComparator.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/comparator/AdvancedTestDataComparator.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.destination.comparator; +package io.airbyte.cdk.integrations.standardtest.destination.comparator; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/comparator/BasicTestDataComparator.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/comparator/BasicTestDataComparator.java similarity index 96% rename from airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/comparator/BasicTestDataComparator.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/comparator/BasicTestDataComparator.java index 99106a623306..93da63e5aa02 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/comparator/BasicTestDataComparator.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/comparator/BasicTestDataComparator.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.destination.comparator; +package io.airbyte.cdk.integrations.standardtest.destination.comparator; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/comparator/ComparatorUtils.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/comparator/ComparatorUtils.java similarity index 90% rename from airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/comparator/ComparatorUtils.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/comparator/ComparatorUtils.java index 470a26e7b182..0b077c5cf1dc 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/comparator/ComparatorUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/comparator/ComparatorUtils.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.destination.comparator; +package io.airbyte.cdk.integrations.standardtest.destination.comparator; import com.fasterxml.jackson.databind.JsonNode; import java.util.List; diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/comparator/TestDataComparator.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/comparator/TestDataComparator.java similarity index 78% rename from airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/comparator/TestDataComparator.java rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/comparator/TestDataComparator.java index dcf8daf2156f..ca5f4a229469 100644 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/comparator/TestDataComparator.java +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/comparator/TestDataComparator.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.destination.comparator; +package io.airbyte.cdk.integrations.standardtest.destination.comparator; import com.fasterxml.jackson.databind.JsonNode; import java.util.List; diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/typing_deduping/JdbcSqlGeneratorIntegrationTest.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/typing_deduping/JdbcSqlGeneratorIntegrationTest.java new file mode 100644 index 000000000000..a71711cf17c4 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/typing_deduping/JdbcSqlGeneratorIntegrationTest.java @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.standardtest.destination.typing_deduping; + +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_ID; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_META; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_RAW_ID; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_DATA; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_EMITTED_AT; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.LEGACY_RAW_TABLE_COLUMNS; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.quotedName; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.jdbc.TableDefinition; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; +import io.airbyte.integrations.base.destination.typing_deduping.BaseSqlGeneratorIntegrationTest; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; +import org.jooq.DSLContext; +import org.jooq.DataType; +import org.jooq.Field; +import org.jooq.InsertValuesStepN; +import org.jooq.Name; +import org.jooq.Record; +import org.jooq.SQLDialect; +import org.jooq.conf.ParamType; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; + +public abstract class JdbcSqlGeneratorIntegrationTest extends BaseSqlGeneratorIntegrationTest { + + protected abstract JdbcDatabase getDatabase(); + + protected abstract DataType getStructType(); + + // TODO - can we move this class into db_destinations/testFixtures? + // then we could redefine getSqlGenerator() to return a JdbcSqlGenerator + // and this could be a private method getSqlGenerator().getTimestampWithTimeZoneType() + private DataType getTimestampWithTimeZoneType() { + return getSqlGenerator().toDialectType(AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); + } + + @Override + protected abstract JdbcSqlGenerator getSqlGenerator(); + + protected abstract SQLDialect getSqlDialect(); + + private DSLContext getDslContext() { + return DSL.using(getSqlDialect()); + } + + /** + * Many destinations require special handling to create JSON values. For example, redshift requires + * you to invoke JSON_PARSE('{...}'), and postgres requires you to CAST('{...}' AS JSONB). This + * method allows subclasses to implement that logic. + */ + protected abstract Field toJsonValue(String valueAsString); + + private void insertRecords(final Name tableName, final List columnNames, final List records, final String... columnsToParseJson) + throws SQLException { + InsertValuesStepN insert = getDslContext().insertInto( + DSL.table(tableName), + columnNames.stream().map(columnName -> field(quotedName(columnName))).toList()); + for (final JsonNode record : records) { + insert = insert.values( + columnNames.stream() + .map(fieldName -> { + // Convert this field to a string. Pretty naive implementation. + final JsonNode column = record.get(fieldName); + final String columnAsString; + if (column == null) { + columnAsString = null; + } else if (column.isTextual()) { + columnAsString = column.asText(); + } else { + columnAsString = column.toString(); + } + + if (Arrays.asList(columnsToParseJson).contains(fieldName)) { + return toJsonValue(columnAsString); + } else { + return DSL.val(columnAsString); + } + }) + .toList()); + } + getDatabase().execute(insert.getSQL(ParamType.INLINED)); + } + + @Override + protected void createNamespace(final String namespace) throws Exception { + getDatabase().execute(getDslContext().createSchemaIfNotExists(namespace).getSQL(ParamType.INLINED)); + } + + @Override + protected void createRawTable(final StreamId streamId) throws Exception { + getDatabase().execute(getDslContext().createTable(DSL.name(streamId.rawNamespace(), streamId.rawName())) + .column(COLUMN_NAME_AB_RAW_ID, SQLDataType.VARCHAR(36).nullable(false)) + .column(COLUMN_NAME_AB_EXTRACTED_AT, getTimestampWithTimeZoneType().nullable(false)) + .column(COLUMN_NAME_AB_LOADED_AT, getTimestampWithTimeZoneType()) + .column(COLUMN_NAME_DATA, getStructType().nullable(false)) + .getSQL(ParamType.INLINED)); + } + + @Override + protected void createV1RawTable(final StreamId v1RawTable) throws Exception { + getDatabase().execute(getDslContext().createTable(DSL.name(v1RawTable.rawNamespace(), v1RawTable.rawName())) + .column(COLUMN_NAME_AB_ID, SQLDataType.VARCHAR(36).nullable(false)) + .column(COLUMN_NAME_EMITTED_AT, getTimestampWithTimeZoneType().nullable(false)) + .column(COLUMN_NAME_DATA, getStructType().nullable(false)) + .getSQL(ParamType.INLINED)); + } + + @Override + protected void insertRawTableRecords(final StreamId streamId, final List records) throws Exception { + insertRecords( + DSL.name(streamId.rawNamespace(), streamId.rawName()), + JavaBaseConstants.V2_RAW_TABLE_COLUMN_NAMES, + records, + COLUMN_NAME_DATA); + } + + @Override + protected void insertV1RawTableRecords(final StreamId streamId, final List records) throws Exception { + insertRecords( + DSL.name(streamId.rawNamespace(), streamId.rawName()), + LEGACY_RAW_TABLE_COLUMNS, + records, + COLUMN_NAME_DATA); + } + + @Override + protected void insertFinalTableRecords(final boolean includeCdcDeletedAt, + final StreamId streamId, + final String suffix, + final List records) + throws Exception { + final List columnNames = + includeCdcDeletedAt ? BaseSqlGeneratorIntegrationTest.FINAL_TABLE_COLUMN_NAMES_CDC : BaseSqlGeneratorIntegrationTest.FINAL_TABLE_COLUMN_NAMES; + insertRecords( + DSL.name(streamId.finalNamespace(), streamId.finalName() + suffix), + columnNames, + records, + COLUMN_NAME_AB_META, "struct", "array", "unknown"); + } + + @Override + protected List dumpRawTableRecords(final StreamId streamId) throws Exception { + return getDatabase().queryJsons(getDslContext().selectFrom(DSL.name(streamId.rawNamespace(), streamId.rawName())).getSQL(ParamType.INLINED)); + } + + @Override + protected List dumpFinalTableRecords(final StreamId streamId, final String suffix) throws Exception { + return getDatabase() + .queryJsons(getDslContext().selectFrom(DSL.name(streamId.finalNamespace(), streamId.finalName() + suffix)).getSQL(ParamType.INLINED)); + } + + @Override + protected void teardownNamespace(final String namespace) throws Exception { + getDatabase().execute(getDslContext().dropSchema(namespace).cascade().getSQL(ParamType.INLINED)); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/typing_deduping/JdbcTypingDedupingTest.java b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/typing_deduping/JdbcTypingDedupingTest.java new file mode 100644 index 000000000000..f77448d62170 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/destination/typing_deduping/JdbcTypingDedupingTest.java @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.standardtest.destination.typing_deduping; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.JdbcCompatibleSourceOperations; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.integrations.base.destination.typing_deduping.BaseTypingDedupingTest; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import java.util.List; +import javax.sql.DataSource; +import org.jooq.impl.DSL; + +/** + * This class is largely the same as + * {@link io.airbyte.integrations.destination.snowflake.typing_deduping.AbstractSnowflakeTypingDedupingTest}. + * But (a) it uses jooq to construct the sql statements, and (b) it doesn't need to upcase anything. + * At some point we might (?) want to do a refactor to combine them. + */ +public abstract class JdbcTypingDedupingTest extends BaseTypingDedupingTest { + + private JdbcDatabase database; + private DataSource dataSource; + + /** + * Get the config as declared in GSM (or directly from the testcontainer). This class will do + * further modification to the config to ensure test isolation.i + */ + protected abstract ObjectNode getBaseConfig(); + + protected abstract DataSource getDataSource(JsonNode config); + + /** + * Subclasses may need to return a custom source operations if the default one does not handle + * vendor-specific types correctly. For example, you most likely need to override this method to + * deserialize JSON columns to JsonNode. + */ + protected JdbcCompatibleSourceOperations getSourceOperations() { + return JdbcUtils.getDefaultSourceOperations(); + } + + /** + * Subclasses using a config with a nonstandard raw table schema should override this method. + */ + protected String getRawSchema() { + return JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE; + } + + /** + * Subclasses using a config where the default schema is not in the {@code schema} key should + * override this method and {@link #setDefaultSchema(JsonNode, String)}. + */ + protected String getDefaultSchema(final JsonNode config) { + return config.get("schema").asText(); + } + + /** + * Subclasses using a config where the default schema is not in the {@code schema} key should + * override this method and {@link #getDefaultSchema(JsonNode)}. + */ + protected void setDefaultSchema(final JsonNode config, final String schema) { + ((ObjectNode) config).put("schema", schema); + } + + @Override + protected JsonNode generateConfig() { + final JsonNode config = getBaseConfig(); + setDefaultSchema(config, "typing_deduping_default_schema" + getUniqueSuffix()); + dataSource = getDataSource(config); + database = new DefaultJdbcDatabase(dataSource, getSourceOperations()); + return config; + } + + @Override + protected List dumpRawTableRecords(String streamNamespace, final String streamName) throws Exception { + if (streamNamespace == null) { + streamNamespace = getDefaultSchema(getConfig()); + } + final String tableName = StreamId.concatenateRawTableName(streamNamespace, streamName); + final String schema = getRawSchema(); + return database.queryJsons(DSL.selectFrom(DSL.name(schema, tableName)).getSQL()); + } + + @Override + protected List dumpFinalTableRecords(String streamNamespace, final String streamName) throws Exception { + if (streamNamespace == null) { + streamNamespace = getDefaultSchema(getConfig()); + } + return database.queryJsons(DSL.selectFrom(DSL.name(streamNamespace, streamName)).getSQL()); + } + + @Override + protected void teardownStreamAndNamespace(String streamNamespace, final String streamName) throws Exception { + if (streamNamespace == null) { + streamNamespace = getDefaultSchema(getConfig()); + } + database.execute(DSL.dropTableIfExists(DSL.name(getRawSchema(), StreamId.concatenateRawTableName(streamNamespace, streamName))).getSQL()); + database.execute(DSL.dropSchemaIfExists(DSL.name(streamNamespace)).cascade().getSQL()); + } + + @Override + protected void globalTeardown() throws Exception { + DataSourceFactory.close(dataSource); + } + +} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/namespace_test_cases.json b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/namespace_test_cases.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/namespace_test_cases.json rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/namespace_test_cases.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_array_object_test_catalog.json b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/data_type_array_object_test_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_array_object_test_catalog.json rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/data_type_array_object_test_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_array_object_test_messages.txt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/data_type_array_object_test_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_array_object_test_messages.txt rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/data_type_array_object_test_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_array_test_catalog.json b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/data_type_array_test_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_array_test_catalog.json rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/data_type_array_test_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_array_test_messages.txt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/data_type_array_test_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_array_test_messages.txt rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/data_type_array_test_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_basic_test_catalog.json b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/data_type_basic_test_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_basic_test_catalog.json rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/data_type_basic_test_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_basic_test_messages.txt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/data_type_basic_test_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_basic_test_messages.txt rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/data_type_basic_test_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_object_test_catalog.json b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/data_type_object_test_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_object_test_catalog.json rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/data_type_object_test_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_object_test_messages.txt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/data_type_object_test_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/data_type_object_test_messages.txt rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/data_type_object_test_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/edge_case_catalog.json b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/edge_case_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/edge_case_catalog.json rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/edge_case_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/edge_case_messages.txt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/edge_case_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/edge_case_messages.txt rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/edge_case_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/exchange_rate_catalog.json b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/exchange_rate_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/exchange_rate_catalog.json rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/exchange_rate_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/exchange_rate_messages.txt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/exchange_rate_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/exchange_rate_messages.txt rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/exchange_rate_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/namespace_catalog.json b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/namespace_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/namespace_catalog.json rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/namespace_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/namespace_messages.txt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/namespace_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/namespace_messages.txt rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/namespace_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/number_data_type_array_test_catalog.json b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/number_data_type_array_test_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/number_data_type_array_test_catalog.json rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/number_data_type_array_test_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/number_data_type_array_test_messages.txt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/number_data_type_array_test_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/number_data_type_array_test_messages.txt rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/number_data_type_array_test_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/number_data_type_test_catalog.json b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/number_data_type_test_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/number_data_type_test_catalog.json rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/number_data_type_test_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/number_data_type_test_messages.txt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/number_data_type_test_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/number_data_type_test_messages.txt rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/number_data_type_test_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/stripe_messages.txt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/stripe_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v0/stripe_messages.txt rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/stripe_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_array_object_test_catalog.json b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/data_type_array_object_test_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_array_object_test_catalog.json rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/data_type_array_object_test_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_array_object_test_messages.txt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/data_type_array_object_test_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_array_object_test_messages.txt rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/data_type_array_object_test_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_array_test_catalog.json b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/data_type_array_test_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_array_test_catalog.json rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/data_type_array_test_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_array_test_messages.txt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/data_type_array_test_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_array_test_messages.txt rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/data_type_array_test_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_basic_test_catalog.json b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/data_type_basic_test_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_basic_test_catalog.json rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/data_type_basic_test_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_basic_test_messages.txt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/data_type_basic_test_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_basic_test_messages.txt rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/data_type_basic_test_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_integer_type_test_catalog.json b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/data_type_integer_type_test_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_integer_type_test_catalog.json rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/data_type_integer_type_test_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_number_type_test_catalog.json b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/data_type_number_type_test_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_number_type_test_catalog.json rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/data_type_number_type_test_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_object_test_catalog.json b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/data_type_object_test_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_object_test_catalog.json rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/data_type_object_test_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_object_test_messages.txt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/data_type_object_test_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/data_type_object_test_messages.txt rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/data_type_object_test_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/edge_case_catalog.json b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/edge_case_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/edge_case_catalog.json rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/edge_case_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/edge_case_messages.txt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/edge_case_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/edge_case_messages.txt rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/edge_case_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/exchange_rate_catalog.json b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/exchange_rate_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/exchange_rate_catalog.json rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/exchange_rate_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/exchange_rate_messages.txt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/exchange_rate_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/exchange_rate_messages.txt rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/exchange_rate_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/infinity_type_test_message.txt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/infinity_type_test_message.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/infinity_type_test_message.txt rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/infinity_type_test_message.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/namespace_catalog.json b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/namespace_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/namespace_catalog.json rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/namespace_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/namespace_messages.txt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/namespace_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/namespace_messages.txt rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/namespace_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/nan_type_test_message.txt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/nan_type_test_message.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/nan_type_test_message.txt rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/nan_type_test_message.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/number_data_type_array_test_catalog.json b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/number_data_type_array_test_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/number_data_type_array_test_catalog.json rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/number_data_type_array_test_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/number_data_type_array_test_messages.txt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/number_data_type_array_test_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/number_data_type_array_test_messages.txt rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/number_data_type_array_test_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/number_data_type_test_catalog.json b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/number_data_type_test_catalog.json similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/number_data_type_test_catalog.json rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/number_data_type_test_catalog.json diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/number_data_type_test_messages.txt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/number_data_type_test_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/number_data_type_test_messages.txt rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/number_data_type_test_messages.txt diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/stripe_messages.txt b/airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/stripe_messages.txt similarity index 100% rename from airbyte-integrations/bases/standard-destination-test/src/main/resources/v1/stripe_messages.txt rename to airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v1/stripe_messages.txt diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/build.gradle b/airbyte-cdk/java/airbyte-cdk/db-sources/build.gradle new file mode 100644 index 000000000000..a8842db3aa31 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/build.gradle @@ -0,0 +1,214 @@ +import org.jsonschema2pojo.SourceType +import org.jsoup.Jsoup + +buildscript { + dependencies { + // from standard-source-test: + classpath 'org.jsoup:jsoup:1.13.1' // for generateSourceTestDocs + } +} + +plugins { + id "com.github.eirnym.js2p" version "1.0" + + id 'application' + id 'airbyte-integration-test-java' + id "java-library" + id "java-test-fixtures" // https://docs.gradle.org/current/userguide/java_testing.html#sec:java_test_fixtures +} + +java { + compileJava { + options.compilerArgs += "-Xlint:-try,-rawtypes,-unchecked,-removal" + } +} + +project.configurations { + // From `base-debezium`: + testFixturesImplementation.extendsFrom implementation + + // From source-jdbc + testFixturesImplementation.extendsFrom implementation + testFixturesRuntimeOnly.extendsFrom runtimeOnly +} + +configurations.all { + // From airbyte-test-utils + exclude group: 'io.micronaut.jaxrs' + exclude group: 'io.micronaut.sql' +} + +// Convert yaml to java: relationaldb.models +jsonSchema2Pojo { + sourceType = SourceType.YAMLSCHEMA + source = files("${sourceSets.main.output.resourcesDir}/db_models") + targetDirectory = new File(project.buildDir, 'generated/src/gen/java/') + removeOldOutput = true + + targetPackage = 'io.airbyte.cdk.integrations.source.relationaldb.models' + + useLongIntegers = true + generateBuilders = true + includeConstructors = false + includeSetters = true +} + +dependencies { + implementation project(':airbyte-cdk:java:airbyte-cdk:core') + testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:acceptance-test-harness') + + compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') + compileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') + compileOnly project(':airbyte-cdk:java:airbyte-cdk:init-oss') + compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') + + testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + testImplementation project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') + + testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-api') + testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') + testFixturesCompileOnly project(':airbyte-cdk:java:airbyte-cdk:init-oss') + + testFixturesImplementation "org.hamcrest:hamcrest-all:1.3" + + + implementation libs.bundles.junit + // implementation libs.junit.jupiter.api + implementation libs.junit.jupiter.params + implementation 'org.junit.platform:junit-platform-launcher:1.7.0' + implementation libs.jooq + testImplementation libs.junit.jupiter.engine + implementation 'net.sourceforge.argparse4j:argparse4j:0.8.1' + implementation "io.aesy:datasize:1.0.0" + implementation libs.apache.commons + implementation libs.apache.commons.lang + testImplementation 'commons-lang:commons-lang:2.6' + implementation 'commons-cli:commons-cli:1.4' + implementation 'org.apache.commons:commons-csv:1.4' + + // Optional dependencies + // TODO: Change these to 'compileOnly' or 'testCompileOnly' + + implementation libs.hikaricp + implementation libs.debezium.api + implementation libs.debezium.embedded + implementation libs.debezium.mysql + implementation libs.debezium.postgres + implementation libs.debezium.mongodb + + implementation libs.bundles.datadog + // implementation 'com.datadoghq:dd-trace-api' + implementation 'org.apache.sshd:sshd-mina:2.8.0' + + implementation libs.testcontainers + implementation libs.testcontainers.mysql + implementation libs.testcontainers.jdbc + implementation libs.testcontainers.postgresql + testImplementation libs.testcontainers.jdbc + testImplementation libs.testcontainers.mysql + testImplementation libs.testcontainers.postgresql + implementation 'org.codehaus.plexus:plexus-utils:3.4.2' + + implementation 'org.bouncycastle:bcprov-jdk15on:1.66' + + // Lombok + implementation 'org.projectlombok:lombok:1.18.20' + annotationProcessor 'org.projectlombok:lombok:1.18.20' + testFixturesImplementation 'org.projectlombok:lombok:1.18.20' + testFixturesAnnotationProcessor 'org.projectlombok:lombok:1.18.20' + + testImplementation libs.junit.jupiter.system.stubs + + // From `base-debezium`: + // implementation project(':airbyte-db:db-lib') + // testFixturesImplementation project(':airbyte-db:db-lib') + testFixturesImplementation 'org.junit.jupiter:junit-jupiter-engine:5.4.2' + testFixturesImplementation 'org.junit.jupiter:junit-jupiter-api:5.4.2' + testFixturesImplementation 'org.junit.jupiter:junit-jupiter-params:5.4.2' + + // From source-jdbc + implementation 'org.apache.commons:commons-lang3:3.11' + testImplementation libs.postgresql + integrationTestJavaImplementation libs.testcontainers.postgresql + testFixturesImplementation libs.airbyte.protocol + // todo (cgardens) - the java-test-fixtures plugin doesn't by default extend from test. + // we cannot make it depend on the dependencies of source-jdbc:test, because source-jdbc:test + // is going to depend on these fixtures. need to find a way to get fixtures to inherit the + // common test classes without duplicating them. this should be part of whatever solution we + // decide on for a "test-java-lib". the current implementation is leveraging the existing + // plugin, but we can something different if we don't like this tool. + testFixturesRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.4.2' + testFixturesImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: '4.0.0' + + // From `standard-source-test`: + testFixturesImplementation 'org.mockito:mockito-core:4.6.1' + testFixturesRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.4.2' + testFixturesImplementation 'org.junit.jupiter:junit-jupiter-api:5.4.2' + testFixturesImplementation 'org.junit.jupiter:junit-jupiter-params:5.8.1' + + // From `airbyte-test-utils`: + // api project(':airbyte-db:db-lib') + testFixturesImplementation 'io.fabric8:kubernetes-client:5.12.2' + testFixturesImplementation libs.temporal.sdk + testFixturesApi libs.junit.jupiter.api + // Mark as compile only to avoid leaking transitively to connectors + testFixturesCompileOnly libs.testcontainers.jdbc + testFixturesCompileOnly libs.testcontainers.postgresql + testFixturesCompileOnly libs.testcontainers.cockroachdb + testFixturesImplementation libs.testcontainers.cockroachdb +} + +def getFullPath(String className) { + def matchingFiles = project.fileTree("src/testFixtures/java") + .filter { file -> file.getName().equals("${className}.java".toString()) }.asCollection() + if (matchingFiles.size() == 0) { + throw new IllegalArgumentException("Ambiguous class name ${className}: no file found.") + } + if (matchingFiles.size() > 1) { + throw new IllegalArgumentException("Ambiguous class name ${className}: more than one matching file was found. Files found: ${matchingFiles}") + } + def absoluteFilePath = matchingFiles[0].toString() + def pathInPackage = project.relativePath(absoluteFilePath.toString()).replaceAll("src/testFixtures/java/", "").replaceAll("\\.java", "") + return pathInPackage +} + +def generateSourceTestDocs = tasks.register('generateSourceTestDocs', Javadoc) { + def javadocOutputDir = project.file("${project.buildDir}/docs/testFixturesJavadoc") + + options.addStringOption('Xdoclint:none', '-quiet') + classpath = sourceSets.testFixtures.compileClasspath + source = sourceSets.testFixtures.allJava + destinationDir = javadocOutputDir + + doLast { + def className = "SourceAcceptanceTest" + // this can be made into a list once we have multiple standard tests, and can also be used for destinations + def pathInPackage = getFullPath(className) + def stdSrcTest = project.file("${javadocOutputDir}/${pathInPackage}.html").readLines().join("\n") + def methodList = Jsoup.parse(stdSrcTest).body().select("section.methodDetails>ul>li>section") + def md = "" + for (methodInfo in methodList) { + def annotations = methodInfo.select(".memberSignature>.annotations").text() + if (!annotations.contains("@Test")) { + continue + } + def methodName = methodInfo.selectFirst("div>span.memberName").text() + def methodDocstring = methodInfo.selectFirst("div.block") + + md += "## ${methodName}\n\n" + md += "${methodDocstring != null ? methodDocstring.text().replaceAll(/([()])/, '\\\\$1') : 'No method description was provided'}\n\n" + } + def outputDoc = new File("${rootDir}/docs/connector-development/testing-connectors/standard-source-tests.md") + outputDoc.write "# Standard Source Test Suite\n\n" + outputDoc.append "Test methods start with `test`. Other methods are internal helpers in the java class implementing the test suite.\n\n" + outputDoc.append md + } + + outputs.upToDateWhen { false } +} + +tasks.register('generate').configure { + dependsOn generateSourceTestDocs +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/AirbyteDebeziumHandler.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/AirbyteDebeziumHandler.java new file mode 100644 index 000000000000..6e1d86de3b4f --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/AirbyteDebeziumHandler.java @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium; + +import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_DURATION; +import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_DURATION_PROPERTY; +import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_RECORDS; +import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_RECORDS_PROPERTY; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteFileOffsetBackingStore; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteSchemaHistoryStorage; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteSchemaHistoryStorage.SchemaHistory; +import io.airbyte.cdk.integrations.debezium.internals.ChangeEventWithMetadata; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumRecordIterator; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumRecordPublisher; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumShutdownProcedure; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumStateDecoratingIterator; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.commons.util.AutoCloseableIterators; +import io.airbyte.commons.util.MoreIterators; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.SyncMode; +import io.debezium.engine.ChangeEvent; +import io.debezium.engine.DebeziumEngine; +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.Properties; +import java.util.concurrent.LinkedBlockingQueue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class acts as the bridge between Airbyte DB connectors and debezium. If a DB connector wants + * to use debezium for CDC, it should use this class + */ +public class AirbyteDebeziumHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(AirbyteDebeziumHandler.class); + /** + * We use 10000 as capacity cause the default queue size and batch size of debezium is : + * {@link io.debezium.config.CommonConnectorConfig#DEFAULT_MAX_BATCH_SIZE}is 2048 + * {@link io.debezium.config.CommonConnectorConfig#DEFAULT_MAX_QUEUE_SIZE} is 8192 + */ + private static final int QUEUE_CAPACITY = 10000; + + private final JsonNode config; + private final CdcTargetPosition targetPosition; + private final boolean trackSchemaHistory; + private final Duration firstRecordWaitTime, subsequentRecordWaitTime; + private final OptionalInt queueSize; + + public AirbyteDebeziumHandler(final JsonNode config, + final CdcTargetPosition targetPosition, + final boolean trackSchemaHistory, + final Duration firstRecordWaitTime, + final Duration subsequentRecordWaitTime, + final OptionalInt queueSize) { + this.config = config; + this.targetPosition = targetPosition; + this.trackSchemaHistory = trackSchemaHistory; + this.firstRecordWaitTime = firstRecordWaitTime; + this.subsequentRecordWaitTime = subsequentRecordWaitTime; + this.queueSize = queueSize; + } + + public AutoCloseableIterator getSnapshotIterators( + final ConfiguredAirbyteCatalog catalogContainingStreamsToSnapshot, + final CdcMetadataInjector cdcMetadataInjector, + final Properties snapshotProperties, + final CdcStateHandler cdcStateHandler, + final DebeziumPropertiesManager.DebeziumConnectorType debeziumConnectorType, + final Instant emittedAt) { + + LOGGER.info("Running snapshot for " + catalogContainingStreamsToSnapshot.getStreams().size() + " new tables"); + final LinkedBlockingQueue> queue = new LinkedBlockingQueue<>(queueSize.orElse(QUEUE_CAPACITY)); + + final AirbyteFileOffsetBackingStore offsetManager = AirbyteFileOffsetBackingStore.initializeDummyStateForSnapshotPurpose(); + final DebeziumRecordPublisher tableSnapshotPublisher = new DebeziumRecordPublisher(snapshotProperties, + config, + catalogContainingStreamsToSnapshot, + offsetManager, + schemaHistoryManager(new SchemaHistory<>(Optional.empty(), false), cdcStateHandler.compressSchemaHistoryForState()), + debeziumConnectorType); + tableSnapshotPublisher.start(queue); + + final AutoCloseableIterator eventIterator = new DebeziumRecordIterator<>( + queue, + targetPosition, + tableSnapshotPublisher::hasClosed, + new DebeziumShutdownProcedure<>(queue, tableSnapshotPublisher::close, tableSnapshotPublisher::hasClosed), + firstRecordWaitTime, + subsequentRecordWaitTime); + + return AutoCloseableIterators.concatWithEagerClose(AutoCloseableIterators + .transform( + eventIterator, + (event) -> DebeziumEventUtils.toAirbyteMessage(event, cdcMetadataInjector, catalogContainingStreamsToSnapshot, emittedAt, + debeziumConnectorType, config)), + AutoCloseableIterators + .fromIterator(MoreIterators.singletonIteratorFromSupplier(cdcStateHandler::saveStateAfterCompletionOfSnapshotOfNewStreams))); + } + + public AutoCloseableIterator getIncrementalIterators(final ConfiguredAirbyteCatalog catalog, + final CdcSavedInfoFetcher cdcSavedInfoFetcher, + final CdcStateHandler cdcStateHandler, + final CdcMetadataInjector cdcMetadataInjector, + final Properties connectorProperties, + final DebeziumPropertiesManager.DebeziumConnectorType debeziumConnectorType, + final Instant emittedAt, + final boolean addDbNameToState) { + LOGGER.info("Using CDC: {}", true); + LOGGER.info("Using DBZ version: {}", DebeziumEngine.class.getPackage().getImplementationVersion()); + final AirbyteFileOffsetBackingStore offsetManager = AirbyteFileOffsetBackingStore.initializeState( + cdcSavedInfoFetcher.getSavedOffset(), + addDbNameToState ? Optional.ofNullable(config.get(JdbcUtils.DATABASE_KEY).asText()) : Optional.empty()); + final Optional schemaHistoryManager = + trackSchemaHistory ? schemaHistoryManager( + cdcSavedInfoFetcher.getSavedSchemaHistory(), + cdcStateHandler.compressSchemaHistoryForState()) + : Optional.empty(); + + final var publisher = new DebeziumRecordPublisher( + connectorProperties, config, catalog, offsetManager, schemaHistoryManager, debeziumConnectorType); + final var queue = new LinkedBlockingQueue>(queueSize.orElse(QUEUE_CAPACITY)); + publisher.start(queue); + // handle state machine around pub/sub logic. + final AutoCloseableIterator eventIterator = new DebeziumRecordIterator<>( + queue, + targetPosition, + publisher::hasClosed, + new DebeziumShutdownProcedure<>(queue, publisher::close, publisher::hasClosed), + firstRecordWaitTime, + subsequentRecordWaitTime); + + final Duration syncCheckpointDuration = + config.get(SYNC_CHECKPOINT_DURATION_PROPERTY) != null ? Duration.ofSeconds(config.get(SYNC_CHECKPOINT_DURATION_PROPERTY).asLong()) + : SYNC_CHECKPOINT_DURATION; + final Long syncCheckpointRecords = config.get(SYNC_CHECKPOINT_RECORDS_PROPERTY) != null ? config.get(SYNC_CHECKPOINT_RECORDS_PROPERTY).asLong() + : SYNC_CHECKPOINT_RECORDS; + return AutoCloseableIterators.fromIterator(new DebeziumStateDecoratingIterator<>( + eventIterator, + cdcStateHandler, + targetPosition, + cdcMetadataInjector, + emittedAt, + offsetManager, + trackSchemaHistory, + schemaHistoryManager.orElse(null), + syncCheckpointDuration, + syncCheckpointRecords, + catalog, + debeziumConnectorType, + config)); + } + + private Optional schemaHistoryManager(final SchemaHistory> schemaHistory, + final boolean compressSchemaHistoryForState) { + if (trackSchemaHistory) { + return Optional.of(AirbyteSchemaHistoryStorage.initializeDBHistory(schemaHistory, compressSchemaHistoryForState)); + } + + return Optional.empty(); + } + + public static boolean isAnyStreamIncrementalSyncMode(final ConfiguredAirbyteCatalog catalog) { + return catalog.getStreams().stream().map(ConfiguredAirbyteStream::getSyncMode) + .anyMatch(syncMode -> syncMode == SyncMode.INCREMENTAL); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/CdcMetadataInjector.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/CdcMetadataInjector.java new file mode 100644 index 000000000000..729a3074126d --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/CdcMetadataInjector.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +/** + * This interface is used to add metadata to the records fetched from the database. For instance, in + * Postgres we add the lsn to the records. In MySql we add the file name and position to the + * records. + */ +public interface CdcMetadataInjector { + + /** + * A debezium record contains multiple pieces. Ref : + * https://debezium.io/documentation/reference/1.9/connectors/mysql.html#mysql-create-events + * + * @param event is the actual record which contains data and would be written to the destination + * @param source contains the metadata about the record and we need to extract that metadata and add + * it to the event before writing it to destination + */ + void addMetaData(ObjectNode event, JsonNode source); + + default void addMetaDataToRowsFetchedOutsideDebezium(final ObjectNode record, final String transactionTimestamp, final T metadataToAdd) { + throw new RuntimeException("Not Supported"); + } + + /** + * As part of Airbyte record we need to add the namespace (schema name) + * + * @param source part of debezium record and contains the metadata about the record. We need to + * extract namespace out of this metadata and return Ref : + * https://debezium.io/documentation/reference/1.9/connectors/mysql.html#mysql-create-events + * @return the stream namespace extracted from the change event source. + */ + String namespace(JsonNode source); + + /** + * As part of Airbyte record we need to add the name (e.g. table name) + * + * @param source part of debezium record and contains the metadata about the record. We need to + * extract namespace out of this metadata and return Ref : + * https://debezium.io/documentation/reference/1.9/connectors/mysql.html#mysql-create-events + * @return The stream name extracted from the change event source. + */ + String name(JsonNode source); + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/CdcSavedInfoFetcher.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/CdcSavedInfoFetcher.java new file mode 100644 index 000000000000..27030d4a2597 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/CdcSavedInfoFetcher.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteSchemaHistoryStorage.SchemaHistory; +import java.util.Optional; + +/** + * This interface is used to fetch the saved info required for debezium to run incrementally. Each + * connector saves offset and schema history in different manner + */ +public interface CdcSavedInfoFetcher { + + JsonNode getSavedOffset(); + + SchemaHistory> getSavedSchemaHistory(); + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/CdcStateHandler.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/CdcStateHandler.java new file mode 100644 index 000000000000..7d0d64be1027 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/CdcStateHandler.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium; + +import io.airbyte.cdk.integrations.debezium.internals.AirbyteSchemaHistoryStorage.SchemaHistory; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import java.util.Map; + +/** + * This interface is used to allow connectors to save the offset and schema history in the manner + * which suits them. Also, it adds some utils to verify CDC event status. + */ +public interface CdcStateHandler { + + AirbyteMessage saveState(final Map offset, final SchemaHistory dbHistory); + + AirbyteMessage saveStateAfterCompletionOfSnapshotOfNewStreams(); + + default boolean compressSchemaHistoryForState() { + return false; + } + + /** + * This function is used as feature flag for sending state messages as checkpoints in CDC syncs. + * + * @return Returns `true` if checkpoint state messages are enabled for CDC syncs. Otherwise, it + * returns `false` + */ + default boolean isCdcCheckpointEnabled() { + return false; + } + +} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/CdcTargetPosition.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/CdcTargetPosition.java similarity index 95% rename from airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/CdcTargetPosition.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/CdcTargetPosition.java index 63cf1866f8ae..2af71dfc0849 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/CdcTargetPosition.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/CdcTargetPosition.java @@ -2,9 +2,9 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium; +package io.airbyte.cdk.integrations.debezium; -import io.airbyte.integrations.debezium.internals.ChangeEventWithMetadata; +import io.airbyte.cdk.integrations.debezium.internals.ChangeEventWithMetadata; import java.util.Map; /** diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/DebeziumIteratorConstants.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/DebeziumIteratorConstants.java new file mode 100644 index 000000000000..2e31f1ec7293 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/DebeziumIteratorConstants.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium; + +import java.time.Duration; + +public class DebeziumIteratorConstants { + + public static final String SYNC_CHECKPOINT_DURATION_PROPERTY = "sync_checkpoint_seconds"; + public static final String SYNC_CHECKPOINT_RECORDS_PROPERTY = "sync_checkpoint_records"; + + public static final Duration SYNC_CHECKPOINT_DURATION = Duration.ofMinutes(15); + public static final Integer SYNC_CHECKPOINT_RECORDS = 10_000; + +} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/AirbyteFileOffsetBackingStore.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/AirbyteFileOffsetBackingStore.java similarity index 99% rename from airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/AirbyteFileOffsetBackingStore.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/AirbyteFileOffsetBackingStore.java index 723f21f1132a..abab49414b71 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/AirbyteFileOffsetBackingStore.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/AirbyteFileOffsetBackingStore.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium.internals; +package io.airbyte.cdk.integrations.debezium.internals; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Preconditions; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/AirbyteSchemaHistoryStorage.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/AirbyteSchemaHistoryStorage.java new file mode 100644 index 000000000000..3ad851796acb --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/AirbyteSchemaHistoryStorage.java @@ -0,0 +1,227 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.commons.json.Jsons; +import io.debezium.document.Document; +import io.debezium.document.DocumentReader; +import io.debezium.document.DocumentWriter; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Optional; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The purpose of this class is : to , 1. Read the contents of the file {@link #path} which contains + * the schema history at the end of the sync so that it can be saved in state for future syncs. + * Check {@link #read()} 2. Write the saved content back to the file {@link #path} at the beginning + * of the sync so that debezium can function smoothly. Check persist(Optional<JsonNode>). + */ +public class AirbyteSchemaHistoryStorage { + + private static final Logger LOGGER = LoggerFactory.getLogger(AirbyteSchemaHistoryStorage.class); + private static final long SIZE_LIMIT_TO_COMPRESS_MB = 3; + public static final int ONE_MB = 1024 * 1024; + private static final Charset UTF8 = StandardCharsets.UTF_8; + + private final DocumentReader reader = DocumentReader.defaultReader(); + private final DocumentWriter writer = DocumentWriter.defaultWriter(); + private final Path path; + private final boolean compressSchemaHistoryForState; + + public AirbyteSchemaHistoryStorage(final Path path, final boolean compressSchemaHistoryForState) { + this.path = path; + this.compressSchemaHistoryForState = compressSchemaHistoryForState; + } + + public Path getPath() { + return path; + } + + public record SchemaHistory (T schema, boolean isCompressed) {} + + public SchemaHistory read() { + final double fileSizeMB = (double) path.toFile().length() / (ONE_MB); + if ((fileSizeMB > SIZE_LIMIT_TO_COMPRESS_MB) && compressSchemaHistoryForState) { + LOGGER.info("File Size {} MB is greater than the size limit of {} MB, compressing the content of the file.", fileSizeMB, + SIZE_LIMIT_TO_COMPRESS_MB); + final String schemaHistory = readCompressed(); + final double compressedSizeMB = calculateSizeOfStringInMB(schemaHistory); + if (fileSizeMB > compressedSizeMB) { + LOGGER.info("Content Size post compression is {} MB ", compressedSizeMB); + } else { + throw new RuntimeException("Compressing increased the size of the content. Size before compression " + fileSizeMB + ", after compression " + + compressedSizeMB); + } + return new SchemaHistory<>(schemaHistory, true); + } + if (compressSchemaHistoryForState) { + LOGGER.info("File Size {} MB is less than the size limit of {} MB, reading the content of the file without compression.", fileSizeMB, + SIZE_LIMIT_TO_COMPRESS_MB); + } else { + LOGGER.info("File Size {} MB.", fileSizeMB); + } + final String schemaHistory = readUncompressed(); + return new SchemaHistory<>(schemaHistory, false); + } + + @VisibleForTesting + public String readUncompressed() { + final StringBuilder fileAsString = new StringBuilder(); + try { + for (final String line : Files.readAllLines(path, UTF8)) { + if (line != null && !line.isEmpty()) { + final Document record = reader.read(line); + final String recordAsString = writer.write(record); + fileAsString.append(recordAsString); + fileAsString.append(System.lineSeparator()); + } + } + return fileAsString.toString(); + } catch (final IOException e) { + throw new RuntimeException(e); + } + } + + private String readCompressed() { + final String lineSeparator = System.lineSeparator(); + final ByteArrayOutputStream compressedStream = new ByteArrayOutputStream(); + try (final GZIPOutputStream gzipOutputStream = new GZIPOutputStream(compressedStream); + final BufferedReader bufferedReader = Files.newBufferedReader(path, UTF8)) { + for (;;) { + final String line = bufferedReader.readLine(); + if (line == null) { + break; + } + + if (!line.isEmpty()) { + final Document record = reader.read(line); + final String recordAsString = writer.write(record); + gzipOutputStream.write(recordAsString.getBytes(StandardCharsets.UTF_8)); + gzipOutputStream.write(lineSeparator.getBytes(StandardCharsets.UTF_8)); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + return Jsons.serialize(compressedStream.toByteArray()); + } + + private void makeSureFileExists() { + try { + // Make sure the file exists ... + if (!Files.exists(path)) { + // Create parent directories if we have them ... + if (path.getParent() != null) { + Files.createDirectories(path.getParent()); + } + try { + Files.createFile(path); + } catch (final FileAlreadyExistsException e) { + // do nothing + } + } + } catch (final IOException e) { + throw new IllegalStateException( + "Unable to check or create history file at " + path + ": " + e.getMessage(), e); + } + } + + private void persist(final SchemaHistory> schemaHistory) { + if (schemaHistory.schema().isEmpty()) { + return; + } + final String fileAsString = Jsons.object(schemaHistory.schema().get(), String.class); + + if (fileAsString == null || fileAsString.isEmpty()) { + return; + } + + FileUtils.deleteQuietly(path.toFile()); + makeSureFileExists(); + if (schemaHistory.isCompressed()) { + writeCompressedStringToFile(fileAsString); + } else { + writeToFile(fileAsString); + } + } + + /** + * @param fileAsString Represents the contents of the file saved in state from previous syncs + */ + private void writeToFile(final String fileAsString) { + try { + final String[] split = fileAsString.split(System.lineSeparator()); + for (final String element : split) { + final Document read = reader.read(element); + final String line = writer.write(read); + + try (final BufferedWriter historyWriter = Files + .newBufferedWriter(path, StandardOpenOption.APPEND)) { + try { + historyWriter.append(line); + historyWriter.newLine(); + } catch (final IOException e) { + throw new RuntimeException(e); + } + } + } + } catch (final IOException e) { + throw new RuntimeException(e); + } + } + + private void writeCompressedStringToFile(final String compressedString) { + try (final ByteArrayInputStream inputStream = new ByteArrayInputStream(Jsons.deserialize(compressedString, byte[].class)); + final GZIPInputStream gzipInputStream = new GZIPInputStream(inputStream); + final FileOutputStream fileOutputStream = new FileOutputStream(path.toFile())) { + final byte[] buffer = new byte[1024]; + int bytesRead; + while ((bytesRead = gzipInputStream.read(buffer)) != -1) { + fileOutputStream.write(buffer, 0, bytesRead); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @VisibleForTesting + public static double calculateSizeOfStringInMB(final String string) { + return (double) string.getBytes(StandardCharsets.UTF_8).length / (ONE_MB); + } + + public static AirbyteSchemaHistoryStorage initializeDBHistory(final SchemaHistory> schemaHistory, + final boolean compressSchemaHistoryForState) { + final Path dbHistoryWorkingDir; + try { + dbHistoryWorkingDir = Files.createTempDirectory(Path.of("/tmp"), "cdc-db-history"); + } catch (final IOException e) { + throw new RuntimeException(e); + } + final Path dbHistoryFilePath = dbHistoryWorkingDir.resolve("dbhistory.dat"); + + final AirbyteSchemaHistoryStorage schemaHistoryManager = + new AirbyteSchemaHistoryStorage(dbHistoryFilePath, compressSchemaHistoryForState); + schemaHistoryManager.persist(schemaHistory); + return schemaHistoryManager; + } + +} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/ChangeEventWithMetadata.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/ChangeEventWithMetadata.java similarity index 81% rename from airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/ChangeEventWithMetadata.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/ChangeEventWithMetadata.java index 9a9930b8e21b..c6469b864877 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/ChangeEventWithMetadata.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/ChangeEventWithMetadata.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium.internals; +package io.airbyte.cdk.integrations.debezium.internals; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; @@ -11,11 +11,13 @@ public class ChangeEventWithMetadata { private final ChangeEvent event; + private final JsonNode eventKeyAsJson; private final JsonNode eventValueAsJson; private final SnapshotMetadata snapshotMetadata; public ChangeEventWithMetadata(final ChangeEvent event) { this.event = event; + this.eventKeyAsJson = Jsons.deserialize(event.key()); this.eventValueAsJson = Jsons.deserialize(event.value()); this.snapshotMetadata = SnapshotMetadata.fromString(eventValueAsJson.get("source").get("snapshot").asText()); } @@ -24,6 +26,10 @@ public ChangeEvent event() { return event; } + public JsonNode eventKeyAsJson() { + return eventKeyAsJson; + } + public JsonNode eventValueAsJson() { return eventValueAsJson; } diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumConverterUtils.java similarity index 95% rename from airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumConverterUtils.java index 6a4b8da219ae..4bb065476a41 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumConverterUtils.java @@ -2,9 +2,9 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium.internals; +package io.airbyte.cdk.integrations.debezium.internals; -import io.airbyte.db.DataTypeUtils; +import io.airbyte.cdk.db.DataTypeUtils; import io.debezium.spi.converter.RelationalColumn; import java.sql.Date; import java.sql.Timestamp; @@ -24,7 +24,7 @@ private DebeziumConverterUtils() { } /** - * TODO : Replace usage of this method with {@link io.airbyte.db.jdbc.DateTimeConverter} + * TODO : Replace usage of this method with {@link io.airbyte.cdk.db.jdbc.DateTimeConverter} */ public static String convertDate(final Object input) { /** diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumEventUtils.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumEventUtils.java new file mode 100644 index 000000000000..e531ed5ed1f1 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumEventUtils.java @@ -0,0 +1,203 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.cdk.integrations.debezium.CdcMetadataInjector; +import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcEventUtils; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import java.time.Instant; +import java.util.Set; +import java.util.stream.Collectors; + +public class DebeziumEventUtils { + + public static final String CDC_LSN = "_ab_cdc_lsn"; + public static final String CDC_UPDATED_AT = "_ab_cdc_updated_at"; + public static final String CDC_DELETED_AT = "_ab_cdc_deleted_at"; + + @VisibleForTesting + static final String AFTER_EVENT = "after"; + @VisibleForTesting + static final String BEFORE_EVENT = "before"; + @VisibleForTesting + static final String OPERATION_FIELD = "op"; + @VisibleForTesting + static final String SOURCE_EVENT = "source"; + + public static AirbyteMessage toAirbyteMessage(final ChangeEventWithMetadata event, + final CdcMetadataInjector cdcMetadataInjector, + final ConfiguredAirbyteCatalog configuredAirbyteCatalog, + final Instant emittedAt, + final DebeziumPropertiesManager.DebeziumConnectorType debeziumConnectorType, + final JsonNode config) { + return switch (debeziumConnectorType) { + case MONGODB -> formatMongoDbEvent(event, cdcMetadataInjector, configuredAirbyteCatalog, emittedAt, config); + case RELATIONALDB -> formatRelationalDbEvent(event, cdcMetadataInjector, emittedAt); + }; + } + + private static AirbyteMessage buildAirbyteMessage(final JsonNode source, + final CdcMetadataInjector cdcMetadataInjector, + final Instant emittedAt, + final JsonNode data) { + final String streamNamespace = cdcMetadataInjector.namespace(source); + final String streamName = cdcMetadataInjector.name(source); + + final AirbyteRecordMessage airbyteRecordMessage = new AirbyteRecordMessage() + .withStream(streamName) + .withNamespace(streamNamespace) + .withEmittedAt(emittedAt.toEpochMilli()) + .withData(data); + + return new AirbyteMessage() + .withType(AirbyteMessage.Type.RECORD) + .withRecord(airbyteRecordMessage); + } + + private static AirbyteMessage formatMongoDbEvent(final ChangeEventWithMetadata event, + final CdcMetadataInjector cdcMetadataInjector, + final ConfiguredAirbyteCatalog configuredAirbyteCatalog, + final Instant emittedAt, + final JsonNode config) { + final JsonNode debeziumEventKey = event.eventKeyAsJson(); + final JsonNode debeziumEvent = event.eventValueAsJson(); + final JsonNode before = debeziumEvent.get(BEFORE_EVENT); + final JsonNode after = debeziumEvent.get(AFTER_EVENT); + final JsonNode source = debeziumEvent.get(SOURCE_EVENT); + final String operation = debeziumEvent.get(OPERATION_FIELD).asText(); + final boolean isEnforceSchema = MongoDbCdcEventUtils.isEnforceSchema(config); + + final Set configuredFields = isEnforceSchema ? getConfiguredMongoDbCollectionFields(source, configuredAirbyteCatalog, cdcMetadataInjector) + : null; + + /* + * Delete events need to be handled separately from other CrUD events, as depending on the version + * of the MongoDB server, the contents Debezium event data will be different. See + * #formatMongoDbDeleteDebeziumData() for more details. + */ + final JsonNode data = switch (operation) { + case "c", "i", "u" -> formatMongoDbDebeziumData(before, after, source, debeziumEventKey, cdcMetadataInjector, configuredFields, + isEnforceSchema); + case "d" -> formatMongoDbDeleteDebeziumData(before, debeziumEventKey, source, cdcMetadataInjector, configuredFields, isEnforceSchema); + default -> throw new IllegalArgumentException("Unsupported MongoDB change event operation '" + operation + "'."); + }; + + return buildAirbyteMessage(source, cdcMetadataInjector, emittedAt, data); + } + + private static AirbyteMessage formatRelationalDbEvent(final ChangeEventWithMetadata event, + final CdcMetadataInjector cdcMetadataInjector, + final Instant emittedAt) { + final JsonNode debeziumEvent = event.eventValueAsJson(); + final JsonNode before = debeziumEvent.get(BEFORE_EVENT); + final JsonNode after = debeziumEvent.get(AFTER_EVENT); + final JsonNode source = debeziumEvent.get(SOURCE_EVENT); + + final JsonNode data = formatRelationalDbDebeziumData(before, after, source, cdcMetadataInjector); + return buildAirbyteMessage(source, cdcMetadataInjector, emittedAt, data); + } + + private static JsonNode formatMongoDbDebeziumData(final JsonNode before, + final JsonNode after, + final JsonNode source, + final JsonNode debeziumEventKey, + final CdcMetadataInjector cdcMetadataInjector, + final Set configuredFields, + final boolean isEnforceSchema) { + + if ((before == null || before.isNull()) && (after == null || after.isNull())) { + // In case a mongodb document was updated and then deleted, the update change event will not have + // any information ({after: null}) + // We are going to treat it as a delete. + return formatMongoDbDeleteDebeziumData(before, debeziumEventKey, source, cdcMetadataInjector, configuredFields, isEnforceSchema); + } else { + final String eventJson = (after.isNull() ? before : after).asText(); + return addCdcMetadata( + isEnforceSchema + ? MongoDbCdcEventUtils.transformDataTypes(eventJson, configuredFields) + : MongoDbCdcEventUtils.transformDataTypesNoSchema(eventJson), + source, cdcMetadataInjector, false); + } + } + + private static JsonNode formatMongoDbDeleteDebeziumData(final JsonNode before, + final JsonNode debeziumEventKey, + final JsonNode source, + final CdcMetadataInjector cdcMetadataInjector, + final Set configuredFields, + final boolean isEnforceSchema) { + final String eventJson; + + /* + * The change events produced by MongoDB differ based on the server version. For version BEFORE 6.x, + * the event does not contain the before document. Therefore, the only data that can be extracted is + * the object ID of the deleted document, which is stored in the event key. Otherwise, if the server + * is version 6.+ AND the pre-image support has been enabled on the collection, we can use the + * "before" document from the event to represent the deleted document. + * + * See + * https://www.mongodb.com/docs/manual/reference/change-events/delete/#document-pre--and-post-images + * for more details. + */ + if (!before.isNull()) { + eventJson = before.asText(); + } else { + eventJson = MongoDbCdcEventUtils.generateObjectIdDocument(debeziumEventKey); + } + + return addCdcMetadata( + isEnforceSchema ? MongoDbCdcEventUtils.transformDataTypes(eventJson, configuredFields) + : MongoDbCdcEventUtils.transformDataTypesNoSchema(eventJson), + source, cdcMetadataInjector, true); + } + + private static JsonNode formatRelationalDbDebeziumData(final JsonNode before, + final JsonNode after, + final JsonNode source, + final CdcMetadataInjector cdcMetadataInjector) { + final ObjectNode baseNode = (ObjectNode) (after.isNull() ? before : after); + return addCdcMetadata(baseNode, source, cdcMetadataInjector, after.isNull()); + + } + + private static JsonNode addCdcMetadata(final ObjectNode baseNode, + final JsonNode source, + final CdcMetadataInjector cdcMetadataInjector, + final boolean isDelete) { + + final long transactionMillis = source.get("ts_ms").asLong(); + final String transactionTimestamp = Instant.ofEpochMilli(transactionMillis).toString(); + + baseNode.put(CDC_UPDATED_AT, transactionTimestamp); + cdcMetadataInjector.addMetaData(baseNode, source); + + if (isDelete) { + baseNode.put(CDC_DELETED_AT, transactionTimestamp); + } else { + baseNode.put(CDC_DELETED_AT, (String) null); + } + + return baseNode; + } + + private static Set getConfiguredMongoDbCollectionFields(final JsonNode source, + final ConfiguredAirbyteCatalog configuredAirbyteCatalog, + final CdcMetadataInjector cdcMetadataInjector) { + final String streamNamespace = cdcMetadataInjector.namespace(source); + final String streamName = cdcMetadataInjector.name(source); + return configuredAirbyteCatalog.getStreams().stream() + .filter(s -> streamName.equals(s.getStream().getName()) && streamNamespace.equals(s.getStream().getNamespace())) + .map(CatalogHelpers::getTopLevelFieldNames) + .flatMap(Set::stream) + .collect(Collectors.toSet()); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumPropertiesManager.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumPropertiesManager.java new file mode 100644 index 000000000000..c95e65be5de9 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumPropertiesManager.java @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import java.util.Optional; +import java.util.Properties; + +public abstract class DebeziumPropertiesManager { + + private static final String BYTE_VALUE_256_MB = Integer.toString(256 * 1024 * 1024); + + public static final String NAME_KEY = "name"; + public static final String TOPIC_PREFIX_KEY = "topic.prefix"; + + private final JsonNode config; + private final AirbyteFileOffsetBackingStore offsetManager; + private final Optional schemaHistoryManager; + + private final Properties properties; + private final ConfiguredAirbyteCatalog catalog; + + public DebeziumPropertiesManager(final Properties properties, + final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final AirbyteFileOffsetBackingStore offsetManager, + final Optional schemaHistoryManager) { + this.properties = properties; + this.config = config; + this.catalog = catalog; + this.offsetManager = offsetManager; + this.schemaHistoryManager = schemaHistoryManager; + } + + public Properties getDebeziumProperties() { + final Properties props = new Properties(); + props.putAll(properties); + + // debezium engine configuration + // https://debezium.io/documentation/reference/2.2/development/engine.html#engine-properties + props.setProperty("offset.storage", "org.apache.kafka.connect.storage.FileOffsetBackingStore"); + props.setProperty("offset.storage.file.filename", offsetManager.getOffsetFilePath().toString()); + props.setProperty("offset.flush.interval.ms", "1000"); // todo: make this longer + // default values from debezium CommonConnectorConfig + props.setProperty("max.batch.size", "2048"); + props.setProperty("max.queue.size", "8192"); + + // Disabling retries because debezium startup time might exceed our 60-second wait limit + // The maximum number of retries on connection errors before failing (-1 = no limit, 0 = disabled, > + // 0 = num of retries). + props.setProperty("errors.max.retries", "0"); + // This property must be strictly less than errors.retry.delay.max.ms + // (https://github.com/debezium/debezium/blob/bcc7d49519a4f07d123c616cfa45cd6268def0b9/debezium-core/src/main/java/io/debezium/util/DelayStrategy.java#L135) + props.setProperty("errors.retry.delay.initial.ms", "299"); + props.setProperty("errors.retry.delay.max.ms", "300"); + + if (schemaHistoryManager.isPresent()) { + // https://debezium.io/documentation/reference/2.2/operations/debezium-server.html#debezium-source-database-history-class + // https://debezium.io/documentation/reference/development/engine.html#_in_the_code + // As mentioned in the documents above, debezium connector for MySQL needs to track the schema + // changes. If we don't do this, we can't fetch records for the table. + props.setProperty("schema.history.internal", "io.debezium.storage.file.history.FileSchemaHistory"); + props.setProperty("schema.history.internal.file.filename", schemaHistoryManager.get().getPath().toString()); + props.setProperty("schema.history.internal.store.only.captured.databases.ddl", "true"); + } + + // https://debezium.io/documentation/reference/2.2/configuration/avro.html + props.setProperty("key.converter.schemas.enable", "false"); + props.setProperty("value.converter.schemas.enable", "false"); + + // debezium names + props.setProperty(NAME_KEY, getName(config)); + + // connection configuration + props.putAll(getConnectionConfiguration(config)); + + // By default "decimal.handing.mode=precise" which's caused returning this value as a binary. + // The "double" type may cause a loss of precision, so set Debezium's config to store it as a String + // explicitly in its Kafka messages for more details see: + // https://debezium.io/documentation/reference/2.2/connectors/postgresql.html#postgresql-decimal-types + // https://debezium.io/documentation/faq/#how_to_retrieve_decimal_field_from_binary_representation + props.setProperty("decimal.handling.mode", "string"); + + // https://debezium.io/documentation/reference/2.2/connectors/postgresql.html#postgresql-property-max-queue-size-in-bytes + props.setProperty("max.queue.size.in.bytes", BYTE_VALUE_256_MB); + + // WARNING : Never change the value of this otherwise all the connectors would start syncing from + // scratch + props.setProperty(TOPIC_PREFIX_KEY, getName(config)); + + // includes + props.putAll(getIncludeConfiguration(catalog, config)); + + return props; + } + + protected abstract Properties getConnectionConfiguration(final JsonNode config); + + protected abstract String getName(final JsonNode config); + + protected abstract Properties getIncludeConfiguration(final ConfiguredAirbyteCatalog catalog, final JsonNode config); + + public enum DebeziumConnectorType { + RELATIONALDB, + MONGODB; + } + +} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumRecordIterator.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumRecordIterator.java similarity index 91% rename from airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumRecordIterator.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumRecordIterator.java index b9ee9bbbefab..09ccae30c926 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumRecordIterator.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumRecordIterator.java @@ -2,13 +2,13 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium.internals; +package io.airbyte.cdk.integrations.debezium.internals; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.AbstractIterator; +import io.airbyte.cdk.integrations.debezium.CdcTargetPosition; import io.airbyte.commons.lang.MoreBooleans; import io.airbyte.commons.util.AutoCloseableIterator; -import io.airbyte.integrations.debezium.CdcTargetPosition; import io.debezium.engine.ChangeEvent; import java.lang.reflect.Field; import java.time.Duration; @@ -39,13 +39,11 @@ public class DebeziumRecordIterator extends AbstractIterator, Field> heartbeatEventSourceField; private final LinkedBlockingQueue> queue; private final CdcTargetPosition targetPosition; private final Supplier publisherStatusSupplier; - private final Duration firstRecordWaitTime; + private final Duration firstRecordWaitTime, subsequentRecordWaitTime; private final DebeziumShutdownProcedure> debeziumShutdownProcedure; private boolean receivedFirstRecord; @@ -59,12 +57,14 @@ public DebeziumRecordIterator(final LinkedBlockingQueue targetPosition, final Supplier publisherStatusSupplier, final DebeziumShutdownProcedure> debeziumShutdownProcedure, - final Duration firstRecordWaitTime) { + final Duration firstRecordWaitTime, + final Duration subsequentRecordWaitTime) { this.queue = queue; this.targetPosition = targetPosition; this.publisherStatusSupplier = publisherStatusSupplier; this.debeziumShutdownProcedure = debeziumShutdownProcedure; this.firstRecordWaitTime = firstRecordWaitTime; + this.subsequentRecordWaitTime = subsequentRecordWaitTime; this.heartbeatEventSourceField = new HashMap<>(1); this.receivedFirstRecord = false; @@ -90,7 +90,7 @@ protected ChangeEventWithMetadata computeNext() { while (!MoreBooleans.isTruthy(publisherStatusSupplier.get()) || !queue.isEmpty()) { final ChangeEvent next; - final Duration waitTime = receivedFirstRecord ? SUBSEQUENT_RECORD_WAIT_TIME : this.firstRecordWaitTime; + final Duration waitTime = receivedFirstRecord ? this.subsequentRecordWaitTime : this.firstRecordWaitTime; try { next = queue.poll(waitTime.getSeconds(), TimeUnit.SECONDS); } catch (final InterruptedException e) { @@ -117,9 +117,12 @@ protected ChangeEventWithMetadata computeNext() { final T heartbeatPos = getHeartbeatPosition(next); // wrap up sync if heartbeat position crossed the target OR heartbeat position hasn't changed for // too long - if (hasSyncFinished(heartbeatPos)) { - requestClose("Closing: Heartbeat indicates sync is done"); + if (targetPosition.reachedTargetPosition(heartbeatPos)) { + requestClose("Closing: Heartbeat indicates sync is done by reaching the target position"); + } else if (heartbeatPos.equals(this.lastHeartbeatPosition) && heartbeatPosNotChanging()) { + requestClose("Closing: Heartbeat indicates sync is not progressing"); } + if (!heartbeatPos.equals(lastHeartbeatPosition)) { this.tsLastHeartbeat = LocalDateTime.now(); this.lastHeartbeatPosition = heartbeatPos; @@ -149,8 +152,8 @@ protected ChangeEventWithMetadata computeNext() { while (!debeziumShutdownProcedure.getRecordsRemainingAfterShutdown().isEmpty()) { final ChangeEvent event; try { - event = debeziumShutdownProcedure.getRecordsRemainingAfterShutdown().poll(10, TimeUnit.SECONDS); - } catch (InterruptedException e) { + event = debeziumShutdownProcedure.getRecordsRemainingAfterShutdown().poll(100, TimeUnit.MILLISECONDS); + } catch (final InterruptedException e) { throw new RuntimeException(e); } if (event == null || isHeartbeatEvent(event)) { @@ -164,11 +167,6 @@ protected ChangeEventWithMetadata computeNext() { return endOfData(); } - private boolean hasSyncFinished(final T heartbeatPos) { - return targetPosition.reachedTargetPosition(heartbeatPos) - || (heartbeatPos.equals(this.lastHeartbeatPosition) && heartbeatPosNotChanging()); - } - /** * Debezium was built as an ever running process which keeps on listening for new changes on DB and * immediately processing them. Airbyte needs debezium to work as a start stop mechanism. In order @@ -195,8 +193,11 @@ private boolean isHeartbeatEvent(final ChangeEvent event) { } private boolean heartbeatPosNotChanging() { + if (this.tsLastHeartbeat == null) { + return false; + } final Duration timeElapsedSinceLastHeartbeatTs = Duration.between(this.tsLastHeartbeat, LocalDateTime.now()); - LOGGER.debug("Time since last hb_pos change {}s", timeElapsedSinceLastHeartbeatTs.toSeconds()); + LOGGER.info("Time since last hb_pos change {}s", timeElapsedSinceLastHeartbeatTs.toSeconds()); // wait time for no change in heartbeat position is half of initial waitTime return timeElapsedSinceLastHeartbeatTs.compareTo(this.firstRecordWaitTime.dividedBy(2)) > 0; } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumRecordPublisher.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumRecordPublisher.java new file mode 100644 index 000000000000..2f7e76b29532 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumRecordPublisher.java @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.debezium.engine.ChangeEvent; +import io.debezium.engine.DebeziumEngine; +import io.debezium.engine.format.Json; +import io.debezium.engine.spi.OffsetCommitPolicy; +import java.util.Optional; +import java.util.Properties; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The purpose of this class is to initialize and spawn the debezium engine with the right + * properties to fetch records + */ +public class DebeziumRecordPublisher implements AutoCloseable { + + private static final Logger LOGGER = LoggerFactory.getLogger(DebeziumRecordPublisher.class); + private final ExecutorService executor; + private DebeziumEngine> engine; + private final AtomicBoolean hasClosed; + private final AtomicBoolean isClosing; + private final AtomicReference thrownError; + private final CountDownLatch engineLatch; + private final DebeziumPropertiesManager debeziumPropertiesManager; + + public DebeziumRecordPublisher(final Properties properties, + final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final AirbyteFileOffsetBackingStore offsetManager, + final Optional schemaHistoryManager, + final DebeziumPropertiesManager.DebeziumConnectorType debeziumConnectorType) { + this.debeziumPropertiesManager = createDebeziumPropertiesManager(debeziumConnectorType, properties, config, catalog, offsetManager, + schemaHistoryManager); + this.hasClosed = new AtomicBoolean(false); + this.isClosing = new AtomicBoolean(false); + this.thrownError = new AtomicReference<>(); + this.executor = Executors.newSingleThreadExecutor(); + this.engineLatch = new CountDownLatch(1); + } + + private DebeziumPropertiesManager createDebeziumPropertiesManager(final DebeziumPropertiesManager.DebeziumConnectorType debeziumConnectorType, + final Properties properties, + final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final AirbyteFileOffsetBackingStore offsetManager, + final Optional schemaHistoryManager) { + return switch (debeziumConnectorType) { + case MONGODB -> new MongoDbDebeziumPropertiesManager(properties, config, catalog, offsetManager); + default -> new RelationalDbDebeziumPropertiesManager(properties, config, catalog, offsetManager, schemaHistoryManager); + }; + } + + public void start(final BlockingQueue> queue) { + engine = DebeziumEngine.create(Json.class) + .using(debeziumPropertiesManager.getDebeziumProperties()) + .using(new OffsetCommitPolicy.AlwaysCommitOffsetPolicy()) + .notifying(e -> { + // debezium outputs a tombstone event that has a value of null. this is an artifact of how it + // interacts with kafka. we want to ignore it. + // more on the tombstone: + // https://debezium.io/documentation/reference/2.2/transformations/event-flattening.html + if (e.value() != null) { + try { + queue.put(e); + } catch (final InterruptedException ex) { + Thread.currentThread().interrupt(); + throw new RuntimeException(ex); + } + } + }) + .using((success, message, error) -> { + LOGGER.info("Debezium engine shutdown. Engine terminated successfully : {}", success); + LOGGER.info(message); + if (!success) { + if (error != null) { + thrownError.set(error); + } else { + // There are cases where Debezium doesn't succeed but only fills the message field. + // In that case, we still want to fail loud and clear + thrownError.set(new RuntimeException(message)); + } + } + engineLatch.countDown(); + }) + .build(); + + // Run the engine asynchronously ... + executor.execute(engine); + } + + public boolean hasClosed() { + return hasClosed.get(); + } + + public void close() throws Exception { + if (isClosing.compareAndSet(false, true)) { + // consumers should assume records can be produced until engine has closed. + if (engine != null) { + engine.close(); + } + + // wait for closure before shutting down executor service + engineLatch.await(5, TimeUnit.MINUTES); + + // shut down and await for thread to actually go down + executor.shutdown(); + executor.awaitTermination(5, TimeUnit.MINUTES); + + // after the engine is completely off, we can mark this as closed + hasClosed.set(true); + + if (thrownError.get() != null) { + throw new RuntimeException(thrownError.get()); + } + } + } + +} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumShutdownProcedure.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumShutdownProcedure.java similarity index 89% rename from airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumShutdownProcedure.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumShutdownProcedure.java index 5cb92210e177..d0661e0a7cdc 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumShutdownProcedure.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumShutdownProcedure.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium.internals; +package io.airbyte.cdk.integrations.debezium.internals; import io.airbyte.commons.concurrency.VoidCallable; import io.airbyte.commons.lang.MoreBooleans; @@ -48,9 +48,9 @@ public DebeziumShutdownProcedure(final LinkedBlockingQueue sourceQueue, private Runnable transfer() { return () -> { - while (!sourceQueue.isEmpty() || !MoreBooleans.isTruthy(publisherStatusSupplier.get())) { + while (!sourceQueue.isEmpty() || !hasEngineShutDown()) { try { - T event = sourceQueue.poll(10, TimeUnit.SECONDS); + final T event = sourceQueue.poll(100, TimeUnit.MILLISECONDS); if (event != null) { targetQueue.put(event); } @@ -62,13 +62,17 @@ private Runnable transfer() { }; } + private boolean hasEngineShutDown() { + return MoreBooleans.isTruthy(publisherStatusSupplier.get()); + } + private void initiateTransfer() { executorService.execute(transfer()); } public LinkedBlockingQueue getRecordsRemainingAfterShutdown() { if (!hasTransferThreadShutdown) { - LOGGER.warn("Queue transfer thread has not shutdown, some records might be missing"); + LOGGER.warn("Queue transfer thread has not shut down, some records might be missing."); } return targetQueue; } @@ -83,8 +87,8 @@ public LinkedBlockingQueue getRecordsRemainingAfterShutdown() { * complete we just have to read the remaining records from the {@link targetQueue} */ public void initiateShutdownProcedure() { - if (publisherStatusSupplier.get()) { - LOGGER.info("Engine has already shutdown"); + if (hasEngineShutDown()) { + LOGGER.info("Debezium Engine has already shut down."); return; } Exception exceptionDuringEngineClose = null; diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumStateDecoratingIterator.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumStateDecoratingIterator.java similarity index 82% rename from airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumStateDecoratingIterator.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumStateDecoratingIterator.java index aa689edd4be0..29569515e999 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumStateDecoratingIterator.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumStateDecoratingIterator.java @@ -2,14 +2,18 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium.internals; +package io.airbyte.cdk.integrations.debezium.internals; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.AbstractIterator; -import io.airbyte.integrations.debezium.CdcMetadataInjector; -import io.airbyte.integrations.debezium.CdcStateHandler; -import io.airbyte.integrations.debezium.CdcTargetPosition; +import io.airbyte.cdk.integrations.debezium.CdcMetadataInjector; +import io.airbyte.cdk.integrations.debezium.CdcStateHandler; +import io.airbyte.cdk.integrations.debezium.CdcTargetPosition; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager.DebeziumConnectorType; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateStats; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import java.time.Duration; import java.time.Instant; import java.time.OffsetDateTime; @@ -27,9 +31,6 @@ */ public class DebeziumStateDecoratingIterator extends AbstractIterator implements Iterator { - public static final Duration SYNC_CHECKPOINT_DURATION = Duration.ofMinutes(15); - public static final Integer SYNC_CHECKPOINT_RECORDS = 10_000; - private static final Logger LOGGER = LoggerFactory.getLogger(DebeziumStateDecoratingIterator.class); private final Iterator changeEventIterator; @@ -77,6 +78,9 @@ public class DebeziumStateDecoratingIterator extends AbstractIterator previousCheckpointOffset; + private final DebeziumConnectorType debeziumConnectorType; + private final ConfiguredAirbyteCatalog configuredAirbyteCatalog; + private final JsonNode config; /** * @param changeEventIterator Base iterator that we want to enrich with checkpoint messages @@ -87,6 +91,8 @@ public class DebeziumStateDecoratingIterator extends AbstractIterator changeEventIterator, final CdcStateHandler cdcStateHandler, @@ -97,7 +103,10 @@ public DebeziumStateDecoratingIterator(final Iterator c final boolean trackSchemaHistory, final AirbyteSchemaHistoryStorage schemaHistoryManager, final Duration checkpointDuration, - final Long checkpointRecords) { + final Long checkpointRecords, + final ConfiguredAirbyteCatalog configuredAirbyteCatalog, + final DebeziumConnectorType debeziumConnectorType, + final JsonNode config) { this.changeEventIterator = changeEventIterator; this.cdcStateHandler = cdcStateHandler; this.targetPosition = targetPosition; @@ -106,10 +115,13 @@ public DebeziumStateDecoratingIterator(final Iterator c this.offsetManager = offsetManager; this.trackSchemaHistory = trackSchemaHistory; this.schemaHistoryManager = schemaHistoryManager; + this.configuredAirbyteCatalog = configuredAirbyteCatalog; this.syncCheckpointDuration = checkpointDuration; this.syncCheckpointRecords = checkpointRecords; this.previousCheckpointOffset = (HashMap) offsetManager.read(); + this.debeziumConnectorType = debeziumConnectorType; + this.config = config; resetCheckpointValues(); } @@ -133,7 +145,7 @@ protected AirbyteMessage computeNext() { if (cdcStateHandler.isCdcCheckpointEnabled() && sendCheckpointMessage) { LOGGER.info("Sending CDC checkpoint state message."); - final AirbyteMessage stateMessage = createStateMessage(checkpointOffsetToSend); + final AirbyteMessage stateMessage = createStateMessage(checkpointOffsetToSend, recordsLastSync); previousCheckpointOffset.clear(); previousCheckpointOffset.putAll(checkpointOffsetToSend); resetCheckpointValues(); @@ -167,11 +179,11 @@ protected AirbyteMessage computeNext() { } } recordsLastSync++; - return DebeziumEventUtils.toAirbyteMessage(event, cdcMetadataInjector, emittedAt); + return DebeziumEventUtils.toAirbyteMessage(event, cdcMetadataInjector, configuredAirbyteCatalog, emittedAt, debeziumConnectorType, config); } isSyncFinished = true; - return createStateMessage(offsetManager.read()); + return createStateMessage(offsetManager.read(), recordsLastSync); } /** @@ -190,7 +202,7 @@ private void resetCheckpointValues() { * * @return {@link AirbyteStateMessage} which includes offset and schema history if used. */ - private AirbyteMessage createStateMessage(final Map offset) { + private AirbyteMessage createStateMessage(final Map offset, final long recordCount) { if (trackSchemaHistory && schemaHistoryManager == null) { throw new RuntimeException("Schema History Tracking is true but manager is not initialised"); } @@ -198,7 +210,9 @@ private AirbyteMessage createStateMessage(final Map offset) { throw new RuntimeException("Offset can not be null"); } - return cdcStateHandler.saveState(offset, schemaHistoryManager != null ? schemaHistoryManager.read() : null); + final AirbyteMessage message = cdcStateHandler.saveState(offset, schemaHistoryManager != null ? schemaHistoryManager.read() : null); + message.getState().withSourceStats(new AirbyteStateStats().withRecordCount((double) recordCount)); + return message; } } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumStateUtil.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumStateUtil.java new file mode 100644 index 000000000000..243faccb5939 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumStateUtil.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals; + +import io.debezium.config.Configuration; +import io.debezium.embedded.KafkaConnectUtil; +import java.util.Map; +import java.util.Properties; +import org.apache.kafka.connect.json.JsonConverter; +import org.apache.kafka.connect.json.JsonConverterConfig; +import org.apache.kafka.connect.runtime.WorkerConfig; +import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; +import org.apache.kafka.connect.storage.FileOffsetBackingStore; +import org.apache.kafka.connect.storage.OffsetStorageReaderImpl; + +/** + * Represents a utility class that assists with the parsing of Debezium offset state. + */ +public interface DebeziumStateUtil { + + /** + * The name of the Debezium property that contains the unique name for the Debezium connector. + */ + String CONNECTOR_NAME_PROPERTY = "name"; + + /** + * Configuration for offset state key/value converters. + */ + Map INTERNAL_CONVERTER_CONFIG = Map.of(JsonConverterConfig.SCHEMAS_ENABLE_CONFIG, Boolean.FALSE.toString()); + + /** + * Creates and starts a {@link FileOffsetBackingStore} that is used to store the tracked Debezium + * offset state. + * + * @param properties The Debezium configuration properties for the selected Debezium connector. + * @return A configured and started {@link FileOffsetBackingStore} instance. + */ + default FileOffsetBackingStore getFileOffsetBackingStore(final Properties properties) { + final FileOffsetBackingStore fileOffsetBackingStore = KafkaConnectUtil.fileOffsetBackingStore(); + final Map propertiesMap = Configuration.from(properties).asMap(); + propertiesMap.put(WorkerConfig.KEY_CONVERTER_CLASS_CONFIG, JsonConverter.class.getName()); + propertiesMap.put(WorkerConfig.VALUE_CONVERTER_CLASS_CONFIG, JsonConverter.class.getName()); + fileOffsetBackingStore.configure(new StandaloneConfig(propertiesMap)); + fileOffsetBackingStore.start(); + return fileOffsetBackingStore; + } + + /** + * Creates and returns a {@link JsonConverter} that can be used to parse keys in the Debezium offset + * state storage. + * + * @return A {@link JsonConverter} for key conversion. + */ + default JsonConverter getKeyConverter() { + final JsonConverter keyConverter = new JsonConverter(); + keyConverter.configure(INTERNAL_CONVERTER_CONFIG, true); + return keyConverter; + } + + /** + * Creates and returns an {@link OffsetStorageReaderImpl} instance that can be used to load offset + * state from the offset file storage. + * + * @param fileOffsetBackingStore The {@link FileOffsetBackingStore} that contains the offset state + * saved to disk. + * @param properties The Debezium configuration properties for the selected Debezium connector. + * @return An {@link OffsetStorageReaderImpl} instance that can be used to load the offset state + * from the offset file storage. + */ + default OffsetStorageReaderImpl getOffsetStorageReader(final FileOffsetBackingStore fileOffsetBackingStore, final Properties properties) { + return new OffsetStorageReaderImpl(fileOffsetBackingStore, properties.getProperty(CONNECTOR_NAME_PROPERTY), getKeyConverter(), + getValueConverter()); + } + + /** + * Creates and returns a {@link JsonConverter} that can be used to parse values in the Debezium + * offset state storage. + * + * @return A {@link JsonConverter} for value conversion. + */ + default JsonConverter getValueConverter() { + final JsonConverter valueConverter = new JsonConverter(); + valueConverter.configure(INTERNAL_CONVERTER_CONFIG, false); + return valueConverter; + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/RecordWaitTimeUtil.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/RecordWaitTimeUtil.java new file mode 100644 index 000000000000..4bcec783a70b --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/RecordWaitTimeUtil.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals; + +import com.fasterxml.jackson.databind.JsonNode; +import java.time.Duration; +import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class RecordWaitTimeUtil { + + private static final Logger LOGGER = LoggerFactory.getLogger(RecordWaitTimeUtil.class); + + public static final Duration MIN_FIRST_RECORD_WAIT_TIME = Duration.ofMinutes(2); + public static final Duration MAX_FIRST_RECORD_WAIT_TIME = Duration.ofMinutes(20); + public static final Duration DEFAULT_FIRST_RECORD_WAIT_TIME = Duration.ofMinutes(5); + public static final Duration DEFAULT_SUBSEQUENT_RECORD_WAIT_TIME = Duration.ofMinutes(1); + + public static void checkFirstRecordWaitTime(final JsonNode config) { + // we need to skip the check because in tests, we set initial_waiting_seconds + // to 5 seconds for performance reasons, which is shorter than the minimum + // value allowed in production + if (config.has("is_test") && config.get("is_test").asBoolean()) { + return; + } + + final Optional firstRecordWaitSeconds = getFirstRecordWaitSeconds(config); + if (firstRecordWaitSeconds.isPresent()) { + final int seconds = firstRecordWaitSeconds.get(); + if (seconds < MIN_FIRST_RECORD_WAIT_TIME.getSeconds() || seconds > MAX_FIRST_RECORD_WAIT_TIME.getSeconds()) { + throw new IllegalArgumentException( + String.format("initial_waiting_seconds must be between %d and %d seconds", + MIN_FIRST_RECORD_WAIT_TIME.getSeconds(), MAX_FIRST_RECORD_WAIT_TIME.getSeconds())); + } + } + } + + public static Duration getFirstRecordWaitTime(final JsonNode config) { + final boolean isTest = config.has("is_test") && config.get("is_test").asBoolean(); + Duration firstRecordWaitTime = DEFAULT_FIRST_RECORD_WAIT_TIME; + + final Optional firstRecordWaitSeconds = getFirstRecordWaitSeconds(config); + if (firstRecordWaitSeconds.isPresent()) { + firstRecordWaitTime = Duration.ofSeconds(firstRecordWaitSeconds.get()); + if (!isTest && firstRecordWaitTime.compareTo(MIN_FIRST_RECORD_WAIT_TIME) < 0) { + LOGGER.warn("First record waiting time is overridden to {} minutes, which is the min time allowed for safety.", + MIN_FIRST_RECORD_WAIT_TIME.toMinutes()); + firstRecordWaitTime = MIN_FIRST_RECORD_WAIT_TIME; + } else if (!isTest && firstRecordWaitTime.compareTo(MAX_FIRST_RECORD_WAIT_TIME) > 0) { + LOGGER.warn("First record waiting time is overridden to {} minutes, which is the max time allowed for safety.", + MAX_FIRST_RECORD_WAIT_TIME.toMinutes()); + firstRecordWaitTime = MAX_FIRST_RECORD_WAIT_TIME; + } + } + + LOGGER.info("First record waiting time: {} seconds", firstRecordWaitTime.getSeconds()); + return firstRecordWaitTime; + } + + public static Duration getSubsequentRecordWaitTime(final JsonNode config) { + Duration subsequentRecordWaitTime = DEFAULT_SUBSEQUENT_RECORD_WAIT_TIME; + final boolean isTest = config.has("is_test") && config.get("is_test").asBoolean(); + final Optional firstRecordWaitSeconds = getFirstRecordWaitSeconds(config); + if (isTest && firstRecordWaitSeconds.isPresent()) { + // In tests, reuse the initial_waiting_seconds property to speed things up. + subsequentRecordWaitTime = Duration.ofSeconds(firstRecordWaitSeconds.get()); + } + LOGGER.info("Subsequent record waiting time: {} seconds", subsequentRecordWaitTime.getSeconds()); + return subsequentRecordWaitTime; + } + + public static Optional getFirstRecordWaitSeconds(final JsonNode config) { + final JsonNode replicationMethod = config.get("replication_method"); + if (replicationMethod != null && replicationMethod.has("initial_waiting_seconds")) { + final int seconds = config.get("replication_method").get("initial_waiting_seconds").asInt(); + return Optional.of(seconds); + } + return Optional.empty(); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/RelationalDbDebeziumPropertiesManager.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/RelationalDbDebeziumPropertiesManager.java new file mode 100644 index 000000000000..1d4a81376ac2 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/RelationalDbDebeziumPropertiesManager.java @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.SyncMode; +import java.util.Iterator; +import java.util.Optional; +import java.util.Properties; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; +import org.codehaus.plexus.util.StringUtils; + +public class RelationalDbDebeziumPropertiesManager extends DebeziumPropertiesManager { + + public RelationalDbDebeziumPropertiesManager(final Properties properties, + final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final AirbyteFileOffsetBackingStore offsetManager, + final Optional schemaHistoryManager) { + super(properties, config, catalog, offsetManager, schemaHistoryManager); + } + + @Override + protected Properties getConnectionConfiguration(JsonNode config) { + final Properties properties = new Properties(); + + // db connection configuration + properties.setProperty("database.hostname", config.get(JdbcUtils.HOST_KEY).asText()); + properties.setProperty("database.port", config.get(JdbcUtils.PORT_KEY).asText()); + properties.setProperty("database.user", config.get(JdbcUtils.USERNAME_KEY).asText()); + properties.setProperty("database.dbname", config.get(JdbcUtils.DATABASE_KEY).asText()); + + if (config.has(JdbcUtils.PASSWORD_KEY)) { + properties.setProperty("database.password", config.get(JdbcUtils.PASSWORD_KEY).asText()); + } + + return properties; + } + + @Override + protected String getName(JsonNode config) { + return config.get(JdbcUtils.DATABASE_KEY).asText(); + } + + @Override + protected Properties getIncludeConfiguration(ConfiguredAirbyteCatalog catalog, JsonNode config) { + final Properties properties = new Properties(); + + // table selection + properties.setProperty("table.include.list", getTableIncludelist(catalog)); + // column selection + properties.setProperty("column.include.list", getColumnIncludeList(catalog)); + + return properties; + } + + public static String getTableIncludelist(final ConfiguredAirbyteCatalog catalog) { + // Turn "stream": { + // "namespace": "schema1" + // "name": "table1 + // }, + // "stream": { + // "namespace": "schema2" + // "name": "table2 + // } -------> info "schema1.table1, schema2.table2" + + return catalog.getStreams().stream() + .filter(s -> s.getSyncMode() == SyncMode.INCREMENTAL) + .map(ConfiguredAirbyteStream::getStream) + .map(stream -> stream.getNamespace() + "." + stream.getName()) + // debezium needs commas escaped to split properly + .map(x -> StringUtils.escape(Pattern.quote(x), ",".toCharArray(), "\\,")) + .collect(Collectors.joining(",")); + } + + public static String getColumnIncludeList(final ConfiguredAirbyteCatalog catalog) { + // Turn "stream": { + // "namespace": "schema1" + // "name": "table1" + // "jsonSchema": { + // "properties": { + // "column1": { + // }, + // "column2": { + // } + // } + // } + // } -------> info "schema1.table1.(column1 | column2)" + + return catalog.getStreams().stream() + .filter(s -> s.getSyncMode() == SyncMode.INCREMENTAL) + .map(ConfiguredAirbyteStream::getStream) + .map(s -> { + final String fields = parseFields(s.getJsonSchema().get("properties").fieldNames()); + // schema.table.(col1|col2) + return Pattern.quote(s.getNamespace() + "." + s.getName()) + (StringUtils.isNotBlank(fields) ? "\\." + fields : ""); + }) + .map(x -> StringUtils.escape(x, ",".toCharArray(), "\\,")) + .collect(Collectors.joining(",")); + } + + private static String parseFields(final Iterator fieldNames) { + if (fieldNames == null || !fieldNames.hasNext()) { + return ""; + } + final Iterable iter = () -> fieldNames; + return StreamSupport.stream(iter.spliterator(), false) + .map(f -> Pattern.quote(f)) + .collect(Collectors.joining("|", "(", ")")); + } + +} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/SnapshotMetadata.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/SnapshotMetadata.java similarity index 96% rename from airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/SnapshotMetadata.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/SnapshotMetadata.java index 90bbcabcc032..995d9eac6a19 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/SnapshotMetadata.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/SnapshotMetadata.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium.internals; +package io.airbyte.cdk.integrations.debezium.internals; import com.google.common.collect.ImmutableSet; import java.util.HashMap; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcEventUtils.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcEventUtils.java new file mode 100644 index 000000000000..a5234f2d8f83 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcEventUtils.java @@ -0,0 +1,349 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals.mongodb; + +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.SCHEMALESS_MODE_DATA_FIELD; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.SCHEMA_ENFORCED_CONFIGURATION_KEY; +import static java.util.Arrays.asList; +import static org.bson.BsonType.ARRAY; +import static org.bson.BsonType.DOCUMENT; +import static org.bson.codecs.configuration.CodecRegistries.fromProviders; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.mongodb.DBRefCodecProvider; +import io.airbyte.cdk.db.DataTypeUtils; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.util.MoreIterators; +import java.util.Collections; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import org.apache.commons.lang3.StringUtils; +import org.bson.BsonBinary; +import org.bson.BsonDocument; +import org.bson.BsonDocumentReader; +import org.bson.BsonReader; +import org.bson.BsonRegularExpression; +import org.bson.BsonType; +import org.bson.Document; +import org.bson.UuidRepresentation; +import org.bson.codecs.BsonCodecProvider; +import org.bson.codecs.BsonValueCodecProvider; +import org.bson.codecs.DocumentCodecProvider; +import org.bson.codecs.IterableCodecProvider; +import org.bson.codecs.JsonObjectCodecProvider; +import org.bson.codecs.MapCodecProvider; +import org.bson.codecs.UuidCodecProvider; +import org.bson.codecs.ValueCodecProvider; +import org.bson.codecs.configuration.CodecRegistry; +import org.bson.codecs.jsr310.Jsr310CodecProvider; +import org.bson.types.Decimal128; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Collection of utility methods that are used to transform CDC events. + */ +public class MongoDbCdcEventUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbCdcEventUtils.class); + + public static final String AIRBYTE_SUFFIX = "_aibyte_transform"; + public static final String DOCUMENT_OBJECT_ID_FIELD = "_id"; + public static final String ID_FIELD = "id"; + public static final String OBJECT_ID_FIELD = "$oid"; + public static final String OBJECT_ID_FIELD_PATTERN = "\\" + OBJECT_ID_FIELD; + + /** + * Generates a JSON document with only the {@link #DOCUMENT_OBJECT_ID_FIELD} property. The value is + * extracted from the provided Debezium event key. The result is the following JSON document: + *

      + *

      + * + * { "_id" : "<the object ID as a String>" } + * + * + * @param debeziumEventKey The Debezium change event key as a JSON document. + * @return The modified JSON document with the ID value extracted from the Debezium change event + * key. + */ + public static String generateObjectIdDocument(final JsonNode debeziumEventKey) { + final String idField = debeziumEventKey.get(ID_FIELD).asText(); + if (StringUtils.contains(idField, OBJECT_ID_FIELD)) { + return idField.replaceAll(OBJECT_ID_FIELD_PATTERN, DOCUMENT_OBJECT_ID_FIELD); + } else { + return Jsons.serialize(Jsons.jsonNode(Map.of(DOCUMENT_OBJECT_ID_FIELD, idField.replaceAll("^\"|\"$", "")))); + } + } + + /** + * Normalizes the document's object ID value stored in the change event to match the raw data + * produced by the initial snapshot. + *

      + *

      + * We need to unpack the object ID from the event data in order for it to match up with the data + * produced by the initial snapshot. The event contains the object ID in a nested object: + *

      + *

      + * + * {\"_id\": {\"$oid\": \"64f24244f95155351c4185b1\"}, ...} + * + *

      + *

      + * In order to match the data produced by the initial snapshot, this must be translated into: + *

      + *

      + * + * {\"_id\": \"64f24244f95155351c4185b1\", ...} + * + * + * @param data The {@link ObjectNode} that contains the record data extracted from the change event. + * @return The updated record data with the document object ID normalized. + */ + public static ObjectNode normalizeObjectId(final ObjectNode data) { + if (data.has(DOCUMENT_OBJECT_ID_FIELD) && data.get(DOCUMENT_OBJECT_ID_FIELD).has(OBJECT_ID_FIELD)) { + final String objectId = data.get(DOCUMENT_OBJECT_ID_FIELD).get(OBJECT_ID_FIELD).asText(); + data.put(DOCUMENT_OBJECT_ID_FIELD, objectId); + } + return data; + } + + public static ObjectNode normalizeObjectIdNoSchema(final ObjectNode data) { + normalizeObjectId(data); + // normalize _id in "data" if key exists + final Optional maybeDataField = Optional.ofNullable(data.get(SCHEMALESS_MODE_DATA_FIELD)); + maybeDataField.ifPresent(d -> normalizeObjectId((ObjectNode) d)); + return data; + } + + /** + * Transforms the Debezium event data to ensure that all data types are consistent with those in + * documents generated by initial snapshots. + * + * @param json The Debezium event data as JSON. + * @return The transformed Debezium event data as JSON. + */ + public static ObjectNode transformDataTypes(final String json, final Set configuredFields) { + final ObjectNode objectNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); + final Document document = Document.parse(json); + formatDocument(document, objectNode, configuredFields); + return normalizeObjectId(objectNode); + } + + public static ObjectNode transformDataTypesNoSchema(final String json) { + final ObjectNode objectNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); + final Document document = Document.parse(json); + formatDocumentNoSchema(document, objectNode); + return normalizeObjectIdNoSchema(objectNode); + } + + public static JsonNode toJsonNode(final Document document, final Set columnNames) { + final ObjectNode objectNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); + formatDocument(document, objectNode, columnNames); + return normalizeObjectId(objectNode); + } + + public static JsonNode toJsonNodeNoSchema(final Document document) { + final ObjectNode objectNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); + formatDocumentNoSchema(document, objectNode); + return normalizeObjectIdNoSchema(objectNode); + } + + private static void formatDocument(final Document document, final ObjectNode objectNode, final Set columnNames) { + final BsonDocument bsonDocument = toBsonDocument(document); + try (final BsonReader reader = new BsonDocumentReader(bsonDocument)) { + readDocument(reader, objectNode, columnNames, false); + } catch (final Exception e) { + LOGGER.error("Exception while parsing BsonDocument: {}", e.getMessage()); + throw new RuntimeException(e); + } + } + + private static void formatDocumentNoSchema(final Document document, final ObjectNode objectNode) { + objectNode.set(SCHEMALESS_MODE_DATA_FIELD, Jsons.jsonNode(Collections.emptyMap())); + final BsonDocument bsonDocument = toBsonDocument(document); + try (final BsonReader reader = new BsonDocumentReader(bsonDocument)) { + readDocument(reader, (ObjectNode) objectNode.get(SCHEMALESS_MODE_DATA_FIELD), Collections.emptySet(), true); + final Optional maybeId = Optional.ofNullable(objectNode.get(SCHEMALESS_MODE_DATA_FIELD).get(DOCUMENT_OBJECT_ID_FIELD)); + maybeId.ifPresent(id -> objectNode.set(DOCUMENT_OBJECT_ID_FIELD, id)); + } catch (final Exception e) { + LOGGER.error("Exception while parsing BsonDocument: {}", e.getMessage()); + throw new RuntimeException(e); + } + } + + private static ObjectNode readDocument(final BsonReader reader, + final ObjectNode jsonNodes, + final Set includedFields, + final boolean allowAllFields) { + reader.readStartDocument(); + while (reader.readBsonType() != BsonType.END_OF_DOCUMENT) { + final var fieldName = reader.readName(); + final var fieldType = reader.getCurrentBsonType(); + + if (shouldIncludeField(fieldName, includedFields, allowAllFields)) { + if (DOCUMENT.equals(fieldType)) { + /* + * Recursion in used to parse inner documents. Pass the allow all column name so all nested fields + * are processed. + */ + jsonNodes.set(fieldName, readDocument(reader, (ObjectNode) Jsons.jsonNode(Collections.emptyMap()), Set.of(), true)); + } else if (ARRAY.equals(fieldType)) { + jsonNodes.set(fieldName, readArray(reader, includedFields, fieldName)); + } else { + readField(reader, jsonNodes, fieldName, fieldType); + } + transformToStringIfMarked(jsonNodes, includedFields, fieldName); + } else { + reader.skipValue(); + } + } + reader.readEndDocument(); + + return jsonNodes; + } + + private static JsonNode readArray(final BsonReader reader, final Set columnNames, final String fieldName) { + reader.readStartArray(); + final var elements = Lists.newArrayList(); + + while (reader.readBsonType() != BsonType.END_OF_DOCUMENT) { + final var currentBsonType = reader.getCurrentBsonType(); + if (DOCUMENT.equals(currentBsonType)) { + // recursion is used to read inner doc + elements.add(readDocument(reader, (ObjectNode) Jsons.jsonNode(Collections.emptyMap()), columnNames, true)); + } else if (ARRAY.equals(currentBsonType)) { + // recursion is used to read inner array + elements.add(readArray(reader, columnNames, fieldName)); + } else { + final var element = readField(reader, (ObjectNode) Jsons.jsonNode(Collections.emptyMap()), fieldName, currentBsonType); + elements.add(element.get(fieldName)); + } + } + reader.readEndArray(); + return Jsons.jsonNode(MoreIterators.toList(elements.iterator())); + } + + private static ObjectNode readField(final BsonReader reader, + final ObjectNode o, + final String fieldName, + final BsonType fieldType) { + switch (fieldType) { + case BOOLEAN -> o.put(fieldName, reader.readBoolean()); + case INT32 -> o.put(fieldName, reader.readInt32()); + case INT64 -> o.put(fieldName, reader.readInt64()); + case DOUBLE -> o.put(fieldName, reader.readDouble()); + case DECIMAL128 -> o.put(fieldName, toDouble(reader.readDecimal128())); + case TIMESTAMP -> o.put(fieldName, DataTypeUtils.toISO8601StringWithMilliseconds(reader.readTimestamp().getValue())); + case DATE_TIME -> o.put(fieldName, DataTypeUtils.toISO8601StringWithMilliseconds(reader.readDateTime())); + case BINARY -> o.put(fieldName, toByteArray(reader.readBinaryData())); + case SYMBOL -> o.put(fieldName, reader.readSymbol()); + case STRING -> o.put(fieldName, reader.readString()); + case OBJECT_ID -> o.put(fieldName, toString(reader.readObjectId())); + case JAVASCRIPT -> o.put(fieldName, reader.readJavaScript()); + case JAVASCRIPT_WITH_SCOPE -> readJavaScriptWithScope(o, reader, fieldName); + case REGULAR_EXPRESSION -> o.put(fieldName, readRegularExpression(reader.readRegularExpression())); + default -> reader.skipValue(); + } + + return o; + } + + private static BsonDocument toBsonDocument(final Document document) { + try { + final CodecRegistry customCodecRegistry = + fromProviders(asList( + new UuidCodecProvider(UuidRepresentation.STANDARD), + new ValueCodecProvider(), + new BsonValueCodecProvider(), + new DocumentCodecProvider(), + new IterableCodecProvider(), + new MapCodecProvider(), + new Jsr310CodecProvider(), + new JsonObjectCodecProvider(), + new BsonCodecProvider(), + new DBRefCodecProvider())); + + // Override the default codec registry + return document.toBsonDocument(BsonDocument.class, customCodecRegistry); + } catch (final Exception e) { + LOGGER.error("Exception while converting Document to BsonDocument: {}", e.getMessage()); + throw new RuntimeException(e); + } + } + + private static String toString(final Object value) { + return value == null ? null : value.toString(); + } + + private static Double toDouble(final Decimal128 value) { + return value == null ? null : value.doubleValue(); + } + + private static byte[] toByteArray(final BsonBinary value) { + return value == null ? null : value.getData(); + } + + private static void readJavaScriptWithScope(final ObjectNode o, final BsonReader reader, final String fieldName) { + final var code = reader.readJavaScriptWithScope(); + final var scope = readDocument(reader, (ObjectNode) Jsons.jsonNode(Collections.emptyMap()), Set.of("scope"), false); + o.set(fieldName, Jsons.jsonNode(ImmutableMap.of("code", code, "scope", scope))); + } + + private static String readRegularExpression(final BsonRegularExpression regularExpression) { + if (regularExpression != null) { + final String options = regularExpression.getOptions(); + final String pattern = regularExpression.getPattern(); + return (StringUtils.isNotBlank(options)) ? "(" + options + ")" + pattern : pattern; + } else { + return null; + } + } + + public static void transformToStringIfMarked(final ObjectNode jsonNodes, final Set columnNames, final String fieldName) { + if (columnNames.contains(fieldName + AIRBYTE_SUFFIX)) { + final JsonNode data = jsonNodes.get(fieldName); + if (data != null) { + jsonNodes.remove(fieldName); + jsonNodes.put(fieldName + AIRBYTE_SUFFIX, data.isTextual() ? data.asText() : data.toString()); + } else { + LOGGER.debug("WARNING Field list out of sync, Document doesn't contain field: {}", fieldName); + } + } + } + + /** + * Test if the current field that is included in the configured set of discovered fields. In order + * to support the fields of nested document fields that pass the initial filter, the + * {@code allowAll} flag may be included in the as a way to allow the fields of the nested document + * to be processed. + * + * @param fieldName The name of the current field. + * @param includedFields The discovered fields. + * @param allowAll Flag that overrides the field inclusion comparison. + * @return {@code true} if the current field should be included for processing or {@code false} + * otherwise. + */ + private static boolean shouldIncludeField(final String fieldName, final Set includedFields, final boolean allowAll) { + return allowAll || includedFields.contains(fieldName); + } + + /** + * Parses source-mongodbv2 configuration json for the value of schema_enforced. + * + * @param config config json + * @return true unless a schema_enforced configured to false + */ + public static boolean isEnforceSchema(final JsonNode config) { + return config == null || !config.has(SCHEMA_ENFORCED_CONFIGURATION_KEY) + || (config.has(SCHEMA_ENFORCED_CONFIGURATION_KEY) && config.get( + SCHEMA_ENFORCED_CONFIGURATION_KEY).asBoolean(true)); + + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcTargetPosition.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcTargetPosition.java new file mode 100644 index 000000000000..dcf907e887dc --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcTargetPosition.java @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals.mongodb; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.cdk.integrations.debezium.CdcTargetPosition; +import io.airbyte.cdk.integrations.debezium.internals.ChangeEventWithMetadata; +import io.airbyte.cdk.integrations.debezium.internals.SnapshotMetadata; +import io.airbyte.commons.json.Jsons; +import io.debezium.connector.mongodb.ResumeTokens; +import java.util.Map; +import java.util.Objects; +import org.bson.BsonDocument; +import org.bson.BsonTimestamp; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Implementation of the {@link CdcTargetPosition} interface that provides methods for determining + * when a sync has reached the target position of the CDC log for MongoDB. In this case, the target + * position is a resume token value from the MongoDB oplog. This implementation compares the + * timestamp present in the Debezium change event against the timestamp of the resume token recorded + * at the start of a sync. When the event timestamp exceeds the resume token timestamp, the sync + * should stop to prevent it from running forever. + */ +public class MongoDbCdcTargetPosition implements CdcTargetPosition { + + private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbCdcTargetPosition.class); + + private final BsonTimestamp resumeTokenTimestamp; + + public MongoDbCdcTargetPosition(final BsonDocument resumeToken) { + this.resumeTokenTimestamp = ResumeTokens.getTimestamp(resumeToken); + } + + @VisibleForTesting + BsonTimestamp getResumeTokenTimestamp() { + return resumeTokenTimestamp; + } + + @Override + public boolean isHeartbeatSupported() { + return true; + } + + @Override + public boolean reachedTargetPosition(final ChangeEventWithMetadata changeEventWithMetadata) { + if (changeEventWithMetadata.isSnapshotEvent()) { + return false; + } else if (SnapshotMetadata.LAST == changeEventWithMetadata.snapshotMetadata()) { + LOGGER.info("Signalling close because Snapshot is complete"); + return true; + } else { + final BsonTimestamp eventResumeTokenTimestamp = + MongoDbResumeTokenHelper.extractTimestampFromEvent(changeEventWithMetadata.eventValueAsJson()); + final boolean isEventResumeTokenAfter = resumeTokenTimestamp.compareTo(eventResumeTokenTimestamp) <= 0; + if (isEventResumeTokenAfter) { + LOGGER.info("Signalling close because record's event timestamp {} is after target event timestamp {}.", + eventResumeTokenTimestamp, resumeTokenTimestamp); + } + return isEventResumeTokenAfter; + } + } + + @Override + public boolean reachedTargetPosition(final BsonTimestamp positionFromHeartbeat) { + return positionFromHeartbeat != null && positionFromHeartbeat.compareTo(resumeTokenTimestamp) >= 0; + } + + @Override + public BsonTimestamp extractPositionFromHeartbeatOffset(final Map sourceOffset) { + return ResumeTokens.getTimestamp( + ResumeTokens.fromData( + sourceOffset.get(MongoDbDebeziumConstants.ChangeEvent.SOURCE_RESUME_TOKEN).toString())); + } + + @Override + public boolean isEventAheadOffset(final Map offset, final ChangeEventWithMetadata event) { + if (offset.size() != 1) { + return false; + } + + return MongoDbResumeTokenHelper.extractTimestampFromEvent(event.eventValueAsJson()).getValue() >= ResumeTokens + .getTimestamp(ResumeTokens.fromData(getResumeToken(offset))).getValue(); + } + + @Override + public boolean isSameOffset(final Map offsetA, final Map offsetB) { + if (offsetA == null || offsetA.size() != 1) { + return false; + } + if (offsetB == null || offsetB.size() != 1) { + return false; + } + + return getResumeToken(offsetA).equals(getResumeToken(offsetB)); + } + + private static String getResumeToken(final Map offset) { + final JsonNode offsetJson = Jsons.deserialize((String) offset.values().toArray()[0]); + return offsetJson.get(MongoDbDebeziumConstants.OffsetState.VALUE_RESUME_TOKEN).asText(); + } + + @Override + public boolean equals(final Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + final MongoDbCdcTargetPosition that = (MongoDbCdcTargetPosition) o; + return Objects.equals(resumeTokenTimestamp, that.resumeTokenTimestamp); + } + + @Override + public int hashCode() { + return Objects.hash(resumeTokenTimestamp); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCustomLoader.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCustomLoader.java new file mode 100644 index 000000000000..d5d0cb9f219c --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCustomLoader.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals.mongodb; + +import io.airbyte.commons.json.Jsons; +import io.debezium.connector.mongodb.MongoDbConnectorConfig; +import io.debezium.connector.mongodb.MongoDbOffsetContext; +import io.debezium.connector.mongodb.MongoDbOffsetContext.Loader; +import io.debezium.connector.mongodb.ReplicaSets; +import java.util.Collections; +import java.util.Map; + +/** + * Custom Debezium offset loader for MongoDB. + *

      + *

      + * N.B. In order to extract the offset from the {@link MongoDbCustomLoader}, you must first get the + * {@link io.debezium.connector.mongodb.ReplicaSetOffsetContext} from the + * {@link MongoDbOffsetContext} for the replica set for which the offset is requested. From that + * context, you can then request the actual Debezium offset. + */ +public class MongoDbCustomLoader extends Loader { + + private Map, Map> offsets; + + public MongoDbCustomLoader(final MongoDbConnectorConfig connectorConfig, final ReplicaSets replicaSets) { + super(connectorConfig, replicaSets); + } + + @Override + public MongoDbOffsetContext loadOffsets(final Map, Map> offsets) { + this.offsets = Jsons.clone(offsets); + return super.loadOffsets(offsets); + } + + public Map, Map> getRawOffset() { + return Collections.unmodifiableMap(offsets); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumConstants.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumConstants.java new file mode 100644 index 000000000000..df9485f7b4e5 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumConstants.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals.mongodb; + +import io.debezium.connector.mongodb.SourceInfo; + +/** + * A collection of constants for use with the Debezium MongoDB Connector. + */ +public class MongoDbDebeziumConstants { + + /** + * Constants for Debezium Source Event data. + */ + public static class ChangeEvent { + + public static final String SOURCE = "source"; + public static final String SOURCE_COLLECTION = SourceInfo.COLLECTION; + public static final String SOURCE_DB = "db"; + public static final String SOURCE_ORDER = SourceInfo.ORDER; + public static final String SOURCE_RESUME_TOKEN = "resume_token"; + public static final String SOURCE_SECONDS = SourceInfo.TIMESTAMP; + public static final String SOURCE_TIMESTAMP_MS = "ts_ms"; + + } + + /** + * Constants for the configuration of the MongoDB connector. These constants represent the + * configuration values that are to be mapped to the Debezium configuration. + */ + public static class Configuration { + + public static final String AUTH_SOURCE_CONFIGURATION_KEY = "auth_source"; + public static final String CONNECTION_STRING_CONFIGURATION_KEY = "connection_string"; + public static final String CREDENTIALS_PLACEHOLDER = ":@"; + public static final String DATABASE_CONFIGURATION_KEY = "database"; + public static final String DATABASE_CONFIG_CONFIGURATION_KEY = "database_config"; + public static final String PASSWORD_CONFIGURATION_KEY = "password"; + public static final String USERNAME_CONFIGURATION_KEY = "username"; + public static final String SCHEMA_ENFORCED_CONFIGURATION_KEY = "schema_enforced"; + public static final String SCHEMALESS_MODE_DATA_FIELD = "data"; + + } + + /** + * Constants for Debezium Offset State storage. + */ + public static class OffsetState { + + public static final String KEY_REPLICA_SET = SourceInfo.REPLICA_SET_NAME; + public static final String KEY_SERVER_ID = SourceInfo.SERVER_ID_KEY; + public static final String VALUE_INCREMENT = SourceInfo.ORDER; + public static final String VALUE_RESUME_TOKEN = "resume_token"; + public static final String VALUE_SECONDS = SourceInfo.TIMESTAMP; + public static final String VALUE_TRANSACTION_ID = "transaction_id"; + + } + + private MongoDbDebeziumConstants() {} + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumPropertiesManager.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumPropertiesManager.java new file mode 100644 index 000000000000..bd894b2ff708 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumPropertiesManager.java @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals.mongodb; + +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.AUTH_SOURCE_CONFIGURATION_KEY; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.CONNECTION_STRING_CONFIGURATION_KEY; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.CREDENTIALS_PLACEHOLDER; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.DATABASE_CONFIGURATION_KEY; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.PASSWORD_CONFIGURATION_KEY; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.USERNAME_CONFIGURATION_KEY; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteFileOffsetBackingStore; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import java.util.List; +import java.util.Optional; +import java.util.Properties; +import java.util.stream.Collectors; + +/** + * Custom {@link DebeziumPropertiesManager} specific for the configuration of the Debezium MongoDB + * connector. + *

      + * This implementation provides the specific connection properties required for the Debezium MongoDB + * connector. These properties differ from the general relational database connection properties + * used by the other Debezium connectors. + */ +public class MongoDbDebeziumPropertiesManager extends DebeziumPropertiesManager { + + static final String COLLECTION_INCLUDE_LIST_KEY = "collection.include.list"; + static final String DATABASE_INCLUDE_LIST_KEY = "database.include.list"; + static final String DOUBLE_QUOTES_PATTERN = "\""; + static final String MONGODB_AUTHSOURCE_KEY = "mongodb.authsource"; + static final String MONGODB_CONNECTION_MODE_KEY = "mongodb.connection.mode"; + static final String MONGODB_CONNECTION_MODE_VALUE = "replica_set"; + static final String MONGODB_CONNECTION_STRING_KEY = "mongodb.connection.string"; + static final String MONGODB_PASSWORD_KEY = "mongodb.password"; + static final String MONGODB_SSL_ENABLED_KEY = "mongodb.ssl.enabled"; + static final String MONGODB_SSL_ENABLED_VALUE = Boolean.TRUE.toString(); + static final String MONGODB_USER_KEY = "mongodb.user"; + + public MongoDbDebeziumPropertiesManager(final Properties properties, + final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final AirbyteFileOffsetBackingStore offsetManager) { + super(properties, config, catalog, offsetManager, Optional.empty()); + } + + @Override + protected Properties getConnectionConfiguration(final JsonNode config) { + final Properties properties = new Properties(); + + properties.setProperty(MONGODB_CONNECTION_STRING_KEY, buildConnectionString(config, false)); + properties.setProperty(MONGODB_CONNECTION_MODE_KEY, MONGODB_CONNECTION_MODE_VALUE); + + if (config.has(USERNAME_CONFIGURATION_KEY)) { + properties.setProperty(MONGODB_USER_KEY, config.get(USERNAME_CONFIGURATION_KEY).asText()); + } + if (config.has(PASSWORD_CONFIGURATION_KEY)) { + properties.setProperty(MONGODB_PASSWORD_KEY, config.get(PASSWORD_CONFIGURATION_KEY).asText()); + } + if (config.has(AUTH_SOURCE_CONFIGURATION_KEY)) { + properties.setProperty(MONGODB_AUTHSOURCE_KEY, config.get(AUTH_SOURCE_CONFIGURATION_KEY).asText()); + } + properties.setProperty(MONGODB_SSL_ENABLED_KEY, MONGODB_SSL_ENABLED_VALUE); + return properties; + } + + @Override + protected String getName(final JsonNode config) { + return normalizeName(config.get(DATABASE_CONFIGURATION_KEY).asText()); + } + + @Override + protected Properties getIncludeConfiguration(final ConfiguredAirbyteCatalog catalog, final JsonNode config) { + final Properties properties = new Properties(); + + // Database/collection selection + properties.setProperty(COLLECTION_INCLUDE_LIST_KEY, createCollectionIncludeString(catalog.getStreams())); + properties.setProperty(DATABASE_INCLUDE_LIST_KEY, config.get(DATABASE_CONFIGURATION_KEY).asText()); + + return properties; + } + + protected String createCollectionIncludeString(final List streams) { + return streams.stream() + .map(s -> s.getStream().getNamespace() + "\\." + s.getStream().getName()) + .collect(Collectors.joining(",")); + } + + /** + * Ensure that the name property is formatted correctly for use by Debezium. + * + * @param name The name to be associated with the Debezium connector. + * @return The normalized name. + */ + public static String normalizeName(final String name) { + return name != null ? name.replaceAll("_", "-") : null; + } + + /** + * Builds the MongoDB connection string from the provided configuration. This method handles + * removing any values accidentally copied and pasted from the MongoDB Atlas UI. + * + * @param config The connector configuration. + * @param useSecondary Whether to use the secondary for reads. + * @return The connection string. + */ + public static String buildConnectionString(final JsonNode config, final boolean useSecondary) { + final String connectionString = config.get(CONNECTION_STRING_CONFIGURATION_KEY) + .asText() + .trim() + .replaceAll(DOUBLE_QUOTES_PATTERN, "") + .replaceAll(CREDENTIALS_PLACEHOLDER, ""); + final StringBuilder builder = new StringBuilder(); + builder.append(connectionString); + builder.append("?retryWrites=false&provider=airbyte&tls=true"); + if (useSecondary) { + builder.append("&readPreference=secondary"); + } + return builder.toString(); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtil.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtil.java new file mode 100644 index 000000000000..6e646d53e7c0 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtil.java @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals.mongodb; + +import com.fasterxml.jackson.databind.JsonNode; +import com.mongodb.MongoChangeStreamException; +import com.mongodb.MongoCommandException; +import com.mongodb.client.ChangeStreamIterable; +import com.mongodb.client.MongoClient; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteFileOffsetBackingStore; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumStateUtil; +import io.airbyte.commons.json.Jsons; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.debezium.config.Configuration; +import io.debezium.connector.mongodb.MongoDbConnectorConfig; +import io.debezium.connector.mongodb.MongoDbOffsetContext; +import io.debezium.connector.mongodb.MongoDbTaskContext; +import io.debezium.connector.mongodb.MongoUtil; +import io.debezium.connector.mongodb.ReplicaSetDiscovery; +import io.debezium.connector.mongodb.ReplicaSets; +import io.debezium.connector.mongodb.ResumeTokens; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import org.apache.kafka.connect.storage.FileOffsetBackingStore; +import org.apache.kafka.connect.storage.OffsetStorageReaderImpl; +import org.bson.BsonDocument; +import org.bson.BsonString; +import org.bson.BsonTimestamp; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Collection of utility methods related to the Debezium offset state. + */ +public class MongoDbDebeziumStateUtil implements DebeziumStateUtil { + + private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbDebeziumStateUtil.class); + + /** + * Constructs the initial Debezium offset state that will be used by the incremental CDC snapshot + * after an initial snapshot sync. + * + * @param mongoClient The {@link MongoClient} used to query the MongoDB server. + * @param serverId The ID of the target server. + * @return The initial Debezium offset state storage document as a {@link JsonNode}. + * @throws IllegalStateException if unable to determine the replica set. + */ + public JsonNode constructInitialDebeziumState(final BsonDocument resumeToken, final MongoClient mongoClient, final String serverId) { + final String replicaSet = getReplicaSetName(mongoClient); + LOGGER.info("Initial resume token '{}' constructed, corresponding to timestamp (seconds after epoch) {}", + ResumeTokens.getData(resumeToken).asString().getValue(), ResumeTokens.getTimestamp(resumeToken).getTime()); + final JsonNode state = formatState(serverId, replicaSet, ((BsonString) ResumeTokens.getData(resumeToken)).getValue()); + LOGGER.info("Initial Debezium state constructed: {}", state); + return state; + } + + /** + * Formats the Debezium initial state into a format suitable for storage in the offset data file. + * + * @param serverId The ID target MongoDB database. + * @param replicaSet The name of the target MongoDB replica set. + * @param resumeTokenData The MongoDB resume token that represents the offset state. + * @return The offset state as a {@link JsonNode}. + */ + public static JsonNode formatState(final String serverId, final String replicaSet, final String resumeTokenData) { + final BsonTimestamp timestamp = ResumeTokens.getTimestamp(ResumeTokens.fromData(resumeTokenData)); + + final List key = generateOffsetKey(serverId, replicaSet); + + final Map value = new LinkedHashMap<>(); + value.put(MongoDbDebeziumConstants.OffsetState.VALUE_SECONDS, timestamp.getTime()); + value.put(MongoDbDebeziumConstants.OffsetState.VALUE_INCREMENT, timestamp.getInc()); + value.put(MongoDbDebeziumConstants.OffsetState.VALUE_TRANSACTION_ID, null); + value.put(MongoDbDebeziumConstants.OffsetState.VALUE_RESUME_TOKEN, resumeTokenData); + + return Jsons.jsonNode(Map.of(Jsons.serialize(key), Jsons.serialize(value))); + } + + /** + * Retrieves the replica set name for the current connection. + * + * @param mongoClient The {@link MongoClient} used to retrieve the replica set name. + * @return The replica set name. + * @throws IllegalStateException if unable to determine the replica set. + */ + public static String getReplicaSetName(final MongoClient mongoClient) { + final Optional replicaSetName = MongoUtil.replicaSetName(mongoClient.getClusterDescription()); + return replicaSetName.orElseThrow(() -> new IllegalStateException("Unable to determine replica set.")); + } + + /** + * Test whether the retrieved saved offset resume token value is valid. A valid resume token is one + * that can be used to resume a change event stream in MongoDB. + * + * @param savedOffset The resume token from the saved offset. + * @param mongoClient The {@link MongoClient} used to validate the saved offset. + * @return {@code true} if the saved offset value is valid Otherwise, {@code false} is returned to + * indicate that an initial snapshot should be performed. + */ + public boolean isValidResumeToken(final BsonDocument savedOffset, final MongoClient mongoClient) { + if (Objects.isNull(savedOffset) || savedOffset.isEmpty()) { + return true; + } + + final ChangeStreamIterable stream = mongoClient.watch(BsonDocument.class); + stream.resumeAfter(savedOffset); + try (final var ignored = stream.cursor()) { + LOGGER.info("Valid resume token '{}' present, corresponding to timestamp (seconds after epoch) : {}. Incremental sync will be performed for " + + "up-to-date streams.", + ResumeTokens.getData(savedOffset).asString().getValue(), ResumeTokens.getTimestamp(savedOffset).getTime()); + return true; + } catch (final MongoCommandException | MongoChangeStreamException e) { + LOGGER.info("Invalid resume token '{}' present, corresponding to timestamp (seconds after epoch) : {}. Initial snapshot will be performed for " + + "all streams.", + ResumeTokens.getData(savedOffset).asString().getValue(), ResumeTokens.getTimestamp(savedOffset).getTime()); + return false; + } + } + + /** + * Saves and retrieves the Debezium offset data. This method writes the provided CDC state to the + * offset file and then uses Debezium's code to retrieve the state from the offset file in order to + * verify that Debezium will be able to read the offset data itself when invoked. + * + * @param baseProperties The base Debezium properties. + * @param catalog The configured Airbyte catalog. + * @param cdcState The current CDC state that contains the offset data. + * @param config The source configuration. + * @return The offset value (the timestamp extracted from the resume token) retrieved from the CDC + * state/offset data. + */ + public Optional savedOffset(final Properties baseProperties, + final ConfiguredAirbyteCatalog catalog, + final JsonNode cdcState, + final JsonNode config, + final MongoClient mongoClient) { + LOGGER.debug("Initializing file offset backing store with state '{}'...", cdcState); + final DebeziumPropertiesManager debeziumPropertiesManager = new MongoDbDebeziumPropertiesManager(baseProperties, + config, catalog, + AirbyteFileOffsetBackingStore.initializeState(cdcState, Optional.empty())); + final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(); + return parseSavedOffset(debeziumProperties, mongoClient); + } + + /** + * Loads the offset data from the saved Debezium offset file. + * + * @param properties Properties should contain the relevant properties like path to the Debezium + * state file, etc. It's assumed that the state file is already initialised with the saved + * state + * @return Returns the resume token that Airbyte has acknowledged in the source database server. + */ + private Optional parseSavedOffset(final Properties properties, final MongoClient mongoClient) { + FileOffsetBackingStore fileOffsetBackingStore = null; + OffsetStorageReaderImpl offsetStorageReader = null; + + try { + fileOffsetBackingStore = getFileOffsetBackingStore(properties); + offsetStorageReader = getOffsetStorageReader(fileOffsetBackingStore, properties); + + final Configuration config = Configuration.from(properties); + final MongoDbTaskContext taskContext = new MongoDbTaskContext(config); + final MongoDbConnectorConfig mongoDbConnectorConfig = new MongoDbConnectorConfig(config); + final ReplicaSets replicaSets = new ReplicaSetDiscovery(taskContext).getReplicaSets(mongoClient); + + LOGGER.debug("Parsing saved offset state for replica set '{}' and server ID '{}'...", replicaSets.all().get(0), properties.getProperty("name")); + + final MongoDbOffsetContext.Loader loader = new MongoDbCustomLoader(mongoDbConnectorConfig, replicaSets); + final Collection> partitions = loader.getPartitions(); + final Map, Map> offsets = offsetStorageReader.offsets(partitions); + + if (offsets != null && offsets.values().stream().anyMatch(Objects::nonNull)) { + final MongoDbOffsetContext offsetContext = loader.loadOffsets(offsets); + final Map offset = offsetContext.getReplicaSetOffsetContext(replicaSets.all().get(0)).getOffset(); + final Object resumeTokenData = offset.get(MongoDbDebeziumConstants.OffsetState.VALUE_RESUME_TOKEN); + if (resumeTokenData != null) { + final BsonDocument resumeToken = ResumeTokens.fromData(resumeTokenData.toString()); + return Optional.of(resumeToken); + } else { + LOGGER.warn("Offset data does not contain a resume token: {}", offset); + return Optional.empty(); + } + } else { + LOGGER.warn("Loaded offset data is null or empty: {}", offsets); + return Optional.empty(); + } + } finally { + LOGGER.info("Closing offsetStorageReader and fileOffsetBackingStore"); + if (offsetStorageReader != null) { + offsetStorageReader.close(); + } + + if (fileOffsetBackingStore != null) { + fileOffsetBackingStore.stop(); + } + } + } + + private static List generateOffsetKey(final String serverId, final String replicaSet) { + /* + * N.B. The order of the keys in the sourceInfoMap and key list matters! DO NOT CHANGE the order + * unless you have verified that Debezium has changed its order of the key it builds when retrieving + * data from the offset file. See the "partition(String replicaSetName)" method of the + * io.debezium.connector.mongodb.SourceInfo class for the ordering of keys in the list/map. + */ + final Map sourceInfoMap = new LinkedHashMap<>(); + final String normalizedServerId = MongoDbDebeziumPropertiesManager.normalizeName(serverId); + sourceInfoMap.put(MongoDbDebeziumConstants.OffsetState.KEY_REPLICA_SET, replicaSet); + sourceInfoMap.put(MongoDbDebeziumConstants.OffsetState.KEY_SERVER_ID, normalizedServerId); + + final List key = new LinkedList<>(); + key.add(normalizedServerId); + key.add(sourceInfoMap); + return key; + } + +} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelper.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelper.java similarity index 75% rename from airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelper.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelper.java index 74c726beddb1..48dbc5967e9b 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelper.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelper.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium.internals.mongodb; +package io.airbyte.cdk.integrations.debezium.internals.mongodb; import com.fasterxml.jackson.databind.JsonNode; import com.mongodb.client.ChangeStreamIterable; @@ -29,7 +29,7 @@ public class MongoDbResumeTokenHelper { * @param mongoClient The {@link MongoClient} used to query the MongoDB server. * @return The most recent resume token value. */ - public static BsonDocument getResumeToken(final MongoClient mongoClient) { + public static BsonDocument getMostRecentResumeToken(final MongoClient mongoClient) { final ChangeStreamIterable eventStream = mongoClient.watch(BsonDocument.class); try (final MongoChangeStreamCursor> eventStreamCursor = eventStream.cursor()) { /* @@ -48,15 +48,26 @@ public static BsonDocument getResumeToken(final MongoClient mongoClient) { * @return The extracted timestamp * @throws IllegalStateException if the timestamp could not be extracted from the change event. */ - public static BsonTimestamp extractTimestamp(final JsonNode event) { - return Optional.ofNullable(event.get(MongoDbDebeziumConstants.ChangeEvent.SOURCE)) + public static BsonTimestamp extractTimestampFromEvent(final JsonNode event) { + return extractTimestampFromSource(event.get(MongoDbDebeziumConstants.ChangeEvent.SOURCE)); + } + + /** + * Extracts the timestamp from a Debezium MongoDB change event source object. + * + * @param source The Debezium MongoDB change event source object as JSON. + * @return The extracted timestamp + * @throws IllegalStateException if the timestamp could not be extracted from the change event. + */ + public static BsonTimestamp extractTimestampFromSource(final JsonNode source) { + return Optional.ofNullable(source) .flatMap(MongoDbResumeTokenHelper::createTimestampFromSource) .orElseThrow(() -> new IllegalStateException("Could not find timestamp")); } private static Optional createTimestampFromSource(final JsonNode source) { try { - return Optional.ofNullable( + return Optional.of( new BsonTimestamp( Long.valueOf(TimeUnit.MILLISECONDS.toSeconds( source.get(MongoDbDebeziumConstants.ChangeEvent.SOURCE_TIMESTAMP_MS) diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mysql/CustomMySQLTinyIntOneToBooleanConverter.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/CustomMySQLTinyIntOneToBooleanConverter.java similarity index 90% rename from airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mysql/CustomMySQLTinyIntOneToBooleanConverter.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/CustomMySQLTinyIntOneToBooleanConverter.java index fa52892a3357..f353edaebc02 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mysql/CustomMySQLTinyIntOneToBooleanConverter.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/CustomMySQLTinyIntOneToBooleanConverter.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium.internals.mysql; +package io.airbyte.cdk.integrations.debezium.internals.mysql; import io.debezium.connector.mysql.converters.TinyIntOneToBooleanConverter; import io.debezium.spi.converter.RelationalColumn; diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mysql/MySQLDateTimeConverter.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySQLDateTimeConverter.java similarity index 93% rename from airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mysql/MySQLDateTimeConverter.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySQLDateTimeConverter.java index d57d4aed838b..6d8fd94c1f16 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mysql/MySQLDateTimeConverter.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySQLDateTimeConverter.java @@ -2,11 +2,11 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium.internals.mysql; +package io.airbyte.cdk.integrations.debezium.internals.mysql; -import io.airbyte.db.jdbc.DateTimeConverter; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.debezium.internals.DebeziumConverterUtils; +import io.airbyte.cdk.db.jdbc.DateTimeConverter; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumConverterUtils; import io.debezium.spi.converter.CustomConverter; import io.debezium.spi.converter.RelationalColumn; import io.debezium.time.Conversions; diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mysql/MySqlCdcPosition.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySqlCdcPosition.java similarity index 92% rename from airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mysql/MySqlCdcPosition.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySqlCdcPosition.java index 2efcd8dc8989..6047ee695918 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mysql/MySqlCdcPosition.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySqlCdcPosition.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium.internals.mysql; +package io.airbyte.cdk.integrations.debezium.internals.mysql; import java.util.Objects; diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mysql/MySqlCdcTargetPosition.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySqlCdcTargetPosition.java similarity index 94% rename from airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mysql/MySqlCdcTargetPosition.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySqlCdcTargetPosition.java index 3fed0293920b..f36876620144 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mysql/MySqlCdcTargetPosition.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySqlCdcTargetPosition.java @@ -2,14 +2,14 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium.internals.mysql; +package io.airbyte.cdk.integrations.debezium.internals.mysql; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.debezium.CdcTargetPosition; +import io.airbyte.cdk.integrations.debezium.internals.ChangeEventWithMetadata; +import io.airbyte.cdk.integrations.debezium.internals.SnapshotMetadata; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.debezium.CdcTargetPosition; -import io.airbyte.integrations.debezium.internals.ChangeEventWithMetadata; -import io.airbyte.integrations.debezium.internals.SnapshotMetadata; import java.sql.SQLException; import java.util.List; import java.util.Map; diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mysql/MySqlDebeziumStateUtil.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySqlDebeziumStateUtil.java similarity index 80% rename from airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mysql/MySqlDebeziumStateUtil.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySqlDebeziumStateUtil.java index 08b01c187fa7..08444e1044a3 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mysql/MySqlDebeziumStateUtil.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MySqlDebeziumStateUtil.java @@ -2,19 +2,24 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium.internals.mysql; +package io.airbyte.cdk.integrations.debezium.internals.mysql; +import static io.airbyte.cdk.integrations.debezium.internals.mysql.MysqlCdcStateConstants.COMPRESSION_ENABLED; import static io.debezium.relational.RelationalDatabaseConnectorConfig.DATABASE_NAME; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteFileOffsetBackingStore; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteSchemaHistoryStorage; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteSchemaHistoryStorage.SchemaHistory; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumRecordPublisher; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumStateUtil; +import io.airbyte.cdk.integrations.debezium.internals.RecordWaitTimeUtil; +import io.airbyte.cdk.integrations.debezium.internals.RelationalDbDebeziumPropertiesManager; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.debezium.internals.AirbyteFileOffsetBackingStore; -import io.airbyte.integrations.debezium.internals.AirbyteSchemaHistoryStorage; -import io.airbyte.integrations.debezium.internals.DebeziumPropertiesManager; -import io.airbyte.integrations.debezium.internals.DebeziumRecordPublisher; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.debezium.config.Configuration; import io.debezium.connector.common.OffsetReader; @@ -28,6 +33,7 @@ import io.debezium.pipeline.spi.Partition; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.time.Duration; import java.time.Instant; import java.util.Collections; import java.util.HashMap; @@ -40,20 +46,14 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.stream.Stream; -import org.apache.kafka.connect.json.JsonConverter; -import org.apache.kafka.connect.json.JsonConverterConfig; -import org.apache.kafka.connect.runtime.WorkerConfig; -import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; import org.apache.kafka.connect.storage.FileOffsetBackingStore; import org.apache.kafka.connect.storage.OffsetStorageReaderImpl; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class MySqlDebeziumStateUtil { +public class MySqlDebeziumStateUtil implements DebeziumStateUtil { private static final Logger LOGGER = LoggerFactory.getLogger(MySqlDebeziumStateUtil.class); - public static final String MYSQL_CDC_OFFSET = "mysql_cdc_offset"; - public static final String MYSQL_DB_HISTORY = "mysql_db_history"; public boolean savedOffsetStillPresentOnServer(final JdbcDatabase database, final MysqlDebeziumStateAttributes savedState) { if (savedState.gtidSet().isPresent()) { @@ -148,7 +148,7 @@ private Optional purgedGtidSet(final JdbcDatabase database) { } return Optional.empty(); })) { - List> gtidSet = stream.toList(); + final List> gtidSet = stream.toList(); if (gtidSet.isEmpty()) { return Optional.empty(); } else if (gtidSet.size() == 1) { @@ -169,7 +169,7 @@ public Optional savedOffset(final Properties baseP return Optional.empty(); } - final DebeziumPropertiesManager debeziumPropertiesManager = new DebeziumPropertiesManager(baseProperties, config, catalog, + final DebeziumPropertiesManager debeziumPropertiesManager = new RelationalDbDebeziumPropertiesManager(baseProperties, config, catalog, AirbyteFileOffsetBackingStore.initializeState(cdcOffset, Optional.empty()), Optional.empty()); final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(); @@ -177,36 +177,23 @@ public Optional savedOffset(final Properties baseP } private Optional parseSavedOffset(final Properties properties) { - FileOffsetBackingStore fileOffsetBackingStore = null; OffsetStorageReaderImpl offsetStorageReader = null; + try { - fileOffsetBackingStore = new FileOffsetBackingStore(); - final Map propertiesMap = Configuration.from(properties).asMap(); - propertiesMap.put(WorkerConfig.KEY_CONVERTER_CLASS_CONFIG, JsonConverter.class.getName()); - propertiesMap.put(WorkerConfig.VALUE_CONVERTER_CLASS_CONFIG, JsonConverter.class.getName()); - fileOffsetBackingStore.configure(new StandaloneConfig(propertiesMap)); - fileOffsetBackingStore.start(); - - final Map internalConverterConfig = Collections.singletonMap(JsonConverterConfig.SCHEMAS_ENABLE_CONFIG, "false"); - final JsonConverter keyConverter = new JsonConverter(); - keyConverter.configure(internalConverterConfig, true); - final JsonConverter valueConverter = new JsonConverter(); - valueConverter.configure(internalConverterConfig, false); + fileOffsetBackingStore = getFileOffsetBackingStore(properties); + offsetStorageReader = getOffsetStorageReader(fileOffsetBackingStore, properties); final MySqlConnectorConfig connectorConfig = new MySqlConnectorConfig(Configuration.from(properties)); final MySqlOffsetContext.Loader loader = new MySqlOffsetContext.Loader(connectorConfig); final Set partitions = Collections.singleton(new MySqlPartition(connectorConfig.getLogicalName(), properties.getProperty(DATABASE_NAME.name()))); - offsetStorageReader = new OffsetStorageReaderImpl(fileOffsetBackingStore, properties.getProperty("name"), keyConverter, - valueConverter); final OffsetReader offsetReader = new OffsetReader<>(offsetStorageReader, loader); final Map offsets = offsetReader.offsets(partitions); return extractStateAttributes(partitions, offsets); - } finally { LOGGER.info("Closing offsetStorageReader and fileOffsetBackingStore"); if (offsetStorageReader != null) { @@ -254,14 +241,33 @@ public JsonNode constructInitialDebeziumState(final Properties properties, final AirbyteFileOffsetBackingStore offsetManager = AirbyteFileOffsetBackingStore.initializeState( constructBinlogOffset(database, database.getSourceConfig().get(JdbcUtils.DATABASE_KEY).asText()), Optional.empty()); - final AirbyteSchemaHistoryStorage schemaHistoryStorage = AirbyteSchemaHistoryStorage.initializeDBHistory(Optional.empty()); + final AirbyteSchemaHistoryStorage schemaHistoryStorage = + AirbyteSchemaHistoryStorage.initializeDBHistory(new SchemaHistory<>(Optional.empty(), false), COMPRESSION_ENABLED); final LinkedBlockingQueue> queue = new LinkedBlockingQueue<>(); - try (final DebeziumRecordPublisher publisher = new DebeziumRecordPublisher(properties, database.getSourceConfig(), catalog, offsetManager, - Optional.of(schemaHistoryStorage))) { + try (final DebeziumRecordPublisher publisher = new DebeziumRecordPublisher(properties, + database.getSourceConfig(), + catalog, + offsetManager, + Optional.of(schemaHistoryStorage), + DebeziumPropertiesManager.DebeziumConnectorType.RELATIONALDB)) { publisher.start(queue); + final Instant engineStartTime = Instant.now(); while (!publisher.hasClosed()) { final ChangeEvent event = queue.poll(10, TimeUnit.SECONDS); if (event == null) { + Duration initialWaitingDuration = Duration.ofMinutes(5L); + // If initial waiting seconds is configured and it's greater than 5 minutes, use that value instead + // of the default value + final Duration configuredDuration = RecordWaitTimeUtil.getFirstRecordWaitTime(database.getSourceConfig()); + if (configuredDuration.compareTo(initialWaitingDuration) > 0) { + initialWaitingDuration = configuredDuration; + } + if (Duration.between(engineStartTime, Instant.now()).compareTo(initialWaitingDuration) > 0) { + LOGGER.error("No record is returned even after {} seconds of waiting, closing the engine", initialWaitingDuration.getSeconds()); + publisher.close(); + throw new RuntimeException( + "Building schema history has timed out. Please consider increasing the debezium wait time in advanced options."); + } continue; } LOGGER.info("A record is returned, closing the engine since the state is constructed"); @@ -273,21 +279,30 @@ public JsonNode constructInitialDebeziumState(final Properties properties, } final Map offset = offsetManager.read(); - final String dbHistory = schemaHistoryStorage.read(); + final SchemaHistory schemaHistory = schemaHistoryStorage.read(); assert !offset.isEmpty(); - assert Objects.nonNull(dbHistory); - - final Map state = new HashMap<>(); - state.put(MYSQL_CDC_OFFSET, offset); - state.put(MYSQL_DB_HISTORY, dbHistory); + assert Objects.nonNull(schemaHistory); + assert Objects.nonNull(schemaHistory.schema()); - final JsonNode asJson = Jsons.jsonNode(state); + final JsonNode asJson = serialize(offset, schemaHistory); LOGGER.info("Initial Debezium state constructed: {}", asJson); + if (asJson.get(MysqlCdcStateConstants.MYSQL_DB_HISTORY).asText().isBlank()) { + throw new RuntimeException("Schema history snapshot returned empty history."); + } return asJson; } + public static JsonNode serialize(final Map offset, final SchemaHistory dbHistory) { + final Map state = new HashMap<>(); + state.put(MysqlCdcStateConstants.MYSQL_CDC_OFFSET, offset); + state.put(MysqlCdcStateConstants.MYSQL_DB_HISTORY, dbHistory.schema()); + state.put(MysqlCdcStateConstants.IS_COMPRESSED, dbHistory.isCompressed()); + + return Jsons.jsonNode(state); + } + /** * Method to construct initial Debezium state which can be passed onto Debezium engine to make it * process binlogs from a specific file and position and skip snapshot phase diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MysqlCdcStateConstants.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MysqlCdcStateConstants.java new file mode 100644 index 000000000000..68608775fc1f --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/mysql/MysqlCdcStateConstants.java @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals.mysql; + +public class MysqlCdcStateConstants { + + public static final String MYSQL_CDC_OFFSET = "mysql_cdc_offset"; + public static final String MYSQL_DB_HISTORY = "mysql_db_history"; + public static final String IS_COMPRESSED = "is_compressed"; + public static final boolean COMPRESSION_ENABLED = true; + +} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/postgres/PostgresCdcTargetPosition.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresCdcTargetPosition.java similarity index 83% rename from airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/postgres/PostgresCdcTargetPosition.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresCdcTargetPosition.java index d8a672fac1d1..206c870193db 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/postgres/PostgresCdcTargetPosition.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresCdcTargetPosition.java @@ -2,17 +2,17 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium.internals.postgres; +package io.airbyte.cdk.integrations.debezium.internals.postgres; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; +import io.airbyte.cdk.db.PgLsn; +import io.airbyte.cdk.db.PostgresUtils; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.debezium.CdcTargetPosition; +import io.airbyte.cdk.integrations.debezium.internals.ChangeEventWithMetadata; +import io.airbyte.cdk.integrations.debezium.internals.SnapshotMetadata; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.PgLsn; -import io.airbyte.db.PostgresUtils; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.debezium.CdcTargetPosition; -import io.airbyte.integrations.debezium.internals.ChangeEventWithMetadata; -import io.airbyte.integrations.debezium.internals.SnapshotMetadata; import java.sql.SQLException; import java.util.Map; import java.util.Objects; @@ -62,7 +62,7 @@ public boolean reachedTargetPosition(final ChangeEventWithMetadata changeEventWi return true; } else { final PgLsn eventLsn = extractLsn(changeEventWithMetadata.eventValueAsJson()); - boolean isEventLSNAfter = targetLsn.compareTo(eventLsn) <= 0; + final boolean isEventLSNAfter = targetLsn.compareTo(eventLsn) <= 0; if (isEventLSNAfter) { LOGGER.info("Signalling close because record's LSN : " + eventLsn + " is after target LSN : " + targetLsn); } @@ -72,7 +72,11 @@ public boolean reachedTargetPosition(final ChangeEventWithMetadata changeEventWi @Override public boolean reachedTargetPosition(final Long positionFromHeartbeat) { - return positionFromHeartbeat != null && positionFromHeartbeat.compareTo(targetLsn.asLong()) >= 0; + final boolean reachedTargetPosition = positionFromHeartbeat != null && positionFromHeartbeat.compareTo(targetLsn.asLong()) >= 0; + if (reachedTargetPosition) { + LOGGER.info("Signalling close because heartbeat LSN : " + positionFromHeartbeat + " is after target LSN : " + targetLsn); + } + return reachedTargetPosition; } private PgLsn extractLsn(final JsonNode valueAsJson) { diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/postgres/PostgresConverter.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresConverter.java similarity index 90% rename from airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/postgres/PostgresConverter.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresConverter.java index a260f93761ef..1cfdbbf8eda0 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/postgres/PostgresConverter.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresConverter.java @@ -2,19 +2,20 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium.internals.postgres; +package io.airbyte.cdk.integrations.debezium.internals.postgres; -import static io.airbyte.db.jdbc.DateTimeConverter.convertToDate; -import static io.airbyte.db.jdbc.DateTimeConverter.convertToTime; -import static io.airbyte.db.jdbc.DateTimeConverter.convertToTimestamp; -import static io.airbyte.db.jdbc.DateTimeConverter.convertToTimestampWithTimezone; +import static io.airbyte.cdk.db.jdbc.DateTimeConverter.convertToDate; +import static io.airbyte.cdk.db.jdbc.DateTimeConverter.convertToTime; +import static io.airbyte.cdk.db.jdbc.DateTimeConverter.convertToTimestamp; +import static io.airbyte.cdk.db.jdbc.DateTimeConverter.convertToTimestampWithTimezone; import static org.apache.kafka.connect.data.Schema.OPTIONAL_BOOLEAN_SCHEMA; import static org.apache.kafka.connect.data.Schema.OPTIONAL_FLOAT64_SCHEMA; import static org.apache.kafka.connect.data.Schema.OPTIONAL_INT64_SCHEMA; import static org.apache.kafka.connect.data.Schema.OPTIONAL_STRING_SCHEMA; -import io.airbyte.db.jdbc.DateTimeConverter; -import io.airbyte.integrations.debezium.internals.DebeziumConverterUtils; +import io.airbyte.cdk.db.jdbc.DateTimeConverter; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumConverterUtils; +import io.debezium.connector.postgresql.PostgresValueConverter; import io.debezium.spi.converter.CustomConverter; import io.debezium.spi.converter.RelationalColumn; import io.debezium.time.Conversions; @@ -125,7 +126,7 @@ private void registerNumber(final RelationalColumn field, final ConverterRegistr // The code below strips trailing zeros for integer numbers and represents number with exponent // if this number has decimals point. final double doubleValue = Double.parseDouble(x.toString()); - var valueWithTruncatedZero = BigDecimal.valueOf(doubleValue).stripTrailingZeros().toString(); + final String valueWithTruncatedZero = BigDecimal.valueOf(doubleValue).stripTrailingZeros().toPlainString(); return valueWithTruncatedZero.contains(".") ? String.valueOf(doubleValue) : valueWithTruncatedZero; }); } @@ -239,11 +240,13 @@ private int getTimePrecision(final RelationalColumn field) { return field.scale().orElse(-1); } + private final String POSITIVE_INFINITY_VALUE = "Infinity"; + private final String NEGATIVE_INFINITY_VALUE = "-Infinity"; + // Ref : // https://debezium.io/documentation/reference/2.2/connectors/postgresql.html#postgresql-temporal-types private void registerDate(final RelationalColumn field, final ConverterRegistration registration) { final var fieldType = field.typeName(); - registration.register(SchemaBuilder.string().optional(), x -> { if (x == null) { return DebeziumConverterUtils.convertDefaultValue(field); @@ -252,8 +255,20 @@ private void registerDate(final RelationalColumn field, final ConverterRegistrat case "TIMETZ": return DateTimeConverter.convertToTimeWithTimezone(x); case "TIMESTAMPTZ": + if (x.equals(PostgresValueConverter.NEGATIVE_INFINITY_OFFSET_DATE_TIME)) { + return NEGATIVE_INFINITY_VALUE; + } + if (x.equals(PostgresValueConverter.POSITIVE_INFINITY_OFFSET_DATE_TIME)) { + return POSITIVE_INFINITY_VALUE; + } return DateTimeConverter.convertToTimestampWithTimezone(x); case "TIMESTAMP": + if (x.equals(PostgresValueConverter.NEGATIVE_INFINITY_INSTANT)) { + return NEGATIVE_INFINITY_VALUE; + } + if (x.equals(PostgresValueConverter.POSITIVE_INFINITY_INSTANT)) { + return POSITIVE_INFINITY_VALUE; + } if (x instanceof final Long l) { if (getTimePrecision(field) <= 3) { return convertToTimestamp(Conversions.toInstantFromMillis(l)); @@ -264,6 +279,12 @@ private void registerDate(final RelationalColumn field, final ConverterRegistrat } return convertToTimestamp(x); case "DATE": + if (x.equals(PostgresValueConverter.NEGATIVE_INFINITY_LOCAL_DATE)) { + return NEGATIVE_INFINITY_VALUE; + } + if (x.equals(PostgresValueConverter.POSITIVE_INFINITY_LOCAL_DATE)) { + return POSITIVE_INFINITY_VALUE; + } if (x instanceof Integer) { return convertToDate(LocalDate.ofEpochDay((Integer) x)); } diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/postgres/PostgresCustomLoader.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresCustomLoader.java similarity index 92% rename from airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/postgres/PostgresCustomLoader.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresCustomLoader.java index 7c2c65b6f48a..09b4dcb45240 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/postgres/PostgresCustomLoader.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresCustomLoader.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium.internals.postgres; +package io.airbyte.cdk.integrations.debezium.internals.postgres; import io.airbyte.commons.json.Jsons; import io.debezium.connector.postgresql.PostgresConnectorConfig; diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/postgres/PostgresDebeziumStateUtil.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresDebeziumStateUtil.java similarity index 83% rename from airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/postgres/PostgresDebeziumStateUtil.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresDebeziumStateUtil.java index 2121fac0ea48..e94212bcce43 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/postgres/PostgresDebeziumStateUtil.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresDebeziumStateUtil.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium.internals.postgres; +package io.airbyte.cdk.integrations.debezium.internals.postgres; import static io.debezium.connector.postgresql.PostgresOffsetContext.LAST_COMMIT_LSN_KEY; import static io.debezium.connector.postgresql.SourceInfo.LSN_KEY; @@ -11,11 +11,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; +import io.airbyte.cdk.db.PostgresUtils; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteFileOffsetBackingStore; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumStateUtil; +import io.airbyte.cdk.integrations.debezium.internals.RelationalDbDebeziumPropertiesManager; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.PostgresUtils; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.debezium.internals.AirbyteFileOffsetBackingStore; -import io.airbyte.integrations.debezium.internals.DebeziumPropertiesManager; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.debezium.config.Configuration; import io.debezium.connector.common.OffsetReader; @@ -38,10 +40,6 @@ import java.util.OptionalLong; import java.util.Properties; import java.util.Set; -import org.apache.kafka.connect.json.JsonConverter; -import org.apache.kafka.connect.json.JsonConverterConfig; -import org.apache.kafka.connect.runtime.WorkerConfig; -import org.apache.kafka.connect.runtime.standalone.StandaloneConfig; import org.apache.kafka.connect.storage.FileOffsetBackingStore; import org.apache.kafka.connect.storage.OffsetStorageReaderImpl; import org.postgresql.core.BaseConnection; @@ -55,7 +53,7 @@ * This class is inspired by Debezium's Postgres connector internal implementation on how it parses * the state */ -public class PostgresDebeziumStateUtil { +public class PostgresDebeziumStateUtil implements DebeziumStateUtil { private static final Logger LOGGER = LoggerFactory.getLogger(PostgresDebeziumStateUtil.class); @@ -85,7 +83,7 @@ public OptionalLong savedOffset(final Properties baseProperties, final ConfiguredAirbyteCatalog catalog, final JsonNode cdcState, final JsonNode config) { - final DebeziumPropertiesManager debeziumPropertiesManager = new DebeziumPropertiesManager(baseProperties, config, catalog, + final DebeziumPropertiesManager debeziumPropertiesManager = new RelationalDbDebeziumPropertiesManager(baseProperties, config, catalog, AirbyteFileOffsetBackingStore.initializeState(cdcState, Optional.empty()), Optional.empty()); final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(); @@ -137,13 +135,6 @@ private ChainedLogicalStreamBuilder addSlotOption(final String publicationName, if (pgConnection.haveMinimumServerVersion(140000)) { streamBuilder = streamBuilder.withSlotOption("messages", true); } - } else if (plugin.equalsIgnoreCase("wal2json")) { - streamBuilder = streamBuilder - .withSlotOption("pretty-print", 1) - .withSlotOption("write-in-chunks", 1) - .withSlotOption("include-xids", 1) - .withSlotOption("include-timestamp", 1) - .withSlotOption("include-not-null", "true"); } else { throw new RuntimeException("Unknown plugin value : " + plugin); } @@ -151,6 +142,7 @@ private ChainedLogicalStreamBuilder addSlotOption(final String publicationName, } /** + * Loads the offset data from the saved Debezium offset file. * * @param properties Properties should contain the relevant properties like path to the debezium * state file, etc. It's assumed that the state file is already initialised with the saved @@ -158,34 +150,22 @@ private ChainedLogicalStreamBuilder addSlotOption(final String publicationName, * @return Returns the LSN that Airbyte has acknowledged in the source database server */ private OptionalLong parseSavedOffset(final Properties properties) { - FileOffsetBackingStore fileOffsetBackingStore = null; OffsetStorageReaderImpl offsetStorageReader = null; + try { - fileOffsetBackingStore = new FileOffsetBackingStore(); - final Map propertiesMap = Configuration.from(properties).asMap(); - propertiesMap.put(WorkerConfig.KEY_CONVERTER_CLASS_CONFIG, JsonConverter.class.getName()); - propertiesMap.put(WorkerConfig.VALUE_CONVERTER_CLASS_CONFIG, JsonConverter.class.getName()); - fileOffsetBackingStore.configure(new StandaloneConfig(propertiesMap)); - fileOffsetBackingStore.start(); - - final Map internalConverterConfig = Collections.singletonMap(JsonConverterConfig.SCHEMAS_ENABLE_CONFIG, "false"); - final JsonConverter keyConverter = new JsonConverter(); - keyConverter.configure(internalConverterConfig, true); - final JsonConverter valueConverter = new JsonConverter(); - valueConverter.configure(internalConverterConfig, false); + fileOffsetBackingStore = getFileOffsetBackingStore(properties); + offsetStorageReader = getOffsetStorageReader(fileOffsetBackingStore, properties); final PostgresConnectorConfig postgresConnectorConfig = new PostgresConnectorConfig(Configuration.from(properties)); final PostgresCustomLoader loader = new PostgresCustomLoader(postgresConnectorConfig); final Set partitions = Collections.singleton(new PostgresPartition(postgresConnectorConfig.getLogicalName(), properties.getProperty(DATABASE_NAME.name()))); - offsetStorageReader = new OffsetStorageReaderImpl(fileOffsetBackingStore, properties.getProperty("name"), keyConverter, - valueConverter); + final OffsetReader offsetReader = new OffsetReader<>(offsetStorageReader, loader); final Map offsets = offsetReader.offsets(partitions); return extractLsn(partitions, offsets, loader); - } finally { LOGGER.info("Closing offsetStorageReader and fileOffsetBackingStore"); if (offsetStorageReader != null) { diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/postgres/PostgresReplicationConnection.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresReplicationConnection.java similarity index 93% rename from airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/postgres/PostgresReplicationConnection.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresReplicationConnection.java index c7069afaf9de..85f10313db41 100644 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/postgres/PostgresReplicationConnection.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/debezium/internals/postgres/PostgresReplicationConnection.java @@ -2,11 +2,11 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium.internals.postgres; +package io.airbyte.cdk.integrations.debezium.internals.postgres; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.commons.exceptions.ConfigErrorException; -import io.airbyte.db.jdbc.JdbcUtils; import io.debezium.jdbc.JdbcConnection.ResultSetMapper; import io.debezium.jdbc.JdbcConnection.StatementFactory; import java.sql.*; @@ -48,7 +48,7 @@ public static Connection createConnection(final JsonNode jdbcConfig) throws SQLE validateReplicationConnection(connection); return connection; } catch (final PSQLException exception) { - if (exception.getMessage().equals("FATAL: must be superuser or replication role to start walsender")) { + if ("42501".equals(exception.getSQLState())) { // insufficient_privilege throw new ConfigErrorException(String.format(REPLICATION_PRIVILEGE_ERROR_MESSAGE, jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText())); } throw exception; diff --git a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/jdbc/AbstractJdbcSource.java similarity index 88% rename from airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/jdbc/AbstractJdbcSource.java index aa067a77a587..42d480007285 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSource.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/jdbc/AbstractJdbcSource.java @@ -2,30 +2,30 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.jdbc; - -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_SIZE; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_DECIMAL_DIGITS; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_IS_NULLABLE; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_SCHEMA_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_TABLE_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.JDBC_COLUMN_COLUMN_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.JDBC_COLUMN_DATABASE_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.JDBC_COLUMN_DATA_TYPE; -import static io.airbyte.db.jdbc.JdbcConstants.JDBC_COLUMN_SCHEMA_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.JDBC_COLUMN_SIZE; -import static io.airbyte.db.jdbc.JdbcConstants.JDBC_COLUMN_TABLE_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.JDBC_COLUMN_TYPE_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.JDBC_DECIMAL_DIGITS; -import static io.airbyte.db.jdbc.JdbcConstants.JDBC_IS_NULLABLE; -import static io.airbyte.db.jdbc.JdbcConstants.KEY_SEQ; -import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifier; -import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifierList; -import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; -import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.queryTable; +package io.airbyte.cdk.integrations.source.jdbc; + +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_COLUMN_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_COLUMN_SIZE; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_DECIMAL_DIGITS; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_IS_NULLABLE; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_SCHEMA_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_TABLE_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.JDBC_COLUMN_COLUMN_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.JDBC_COLUMN_DATABASE_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.JDBC_COLUMN_DATA_TYPE; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.JDBC_COLUMN_SCHEMA_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.JDBC_COLUMN_SIZE; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.JDBC_COLUMN_TABLE_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.JDBC_COLUMN_TYPE_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.JDBC_DECIMAL_DIGITS; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.JDBC_IS_NULLABLE; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.KEY_SEQ; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifier; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifierList; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.queryTable; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; @@ -33,24 +33,24 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import datadog.trace.api.Trace; +import io.airbyte.cdk.db.JdbcCompatibleSourceOperations; +import io.airbyte.cdk.db.SqlDatabase; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.jdbc.StreamingJdbcDatabase; +import io.airbyte.cdk.db.jdbc.streaming.JdbcStreamingQueryConfig; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.source.jdbc.dto.JdbcPrivilegeDto; +import io.airbyte.cdk.integrations.source.relationaldb.AbstractDbSource; +import io.airbyte.cdk.integrations.source.relationaldb.CursorInfo; +import io.airbyte.cdk.integrations.source.relationaldb.TableInfo; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; import io.airbyte.commons.functional.CheckedConsumer; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.stream.AirbyteStreamUtils; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.db.JdbcCompatibleSourceOperations; -import io.airbyte.db.SqlDatabase; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.StreamingJdbcDatabase; -import io.airbyte.db.jdbc.streaming.JdbcStreamingQueryConfig; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.source.jdbc.dto.JdbcPrivilegeDto; -import io.airbyte.integrations.source.relationaldb.AbstractDbSource; -import io.airbyte.integrations.source.relationaldb.CursorInfo; -import io.airbyte.integrations.source.relationaldb.TableInfo; -import io.airbyte.integrations.source.relationaldb.state.StateManager; import io.airbyte.protocol.models.CommonField; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; @@ -89,7 +89,6 @@ public abstract class AbstractJdbcSource extends AbstractDbSource streamingQueryConfigProvider; protected final JdbcCompatibleSourceOperations sourceOperations; @@ -99,7 +98,7 @@ public abstract class AbstractJdbcSource extends AbstractDbSource streamingQueryConfigProvider, final JdbcCompatibleSourceOperations sourceOperations) { - this.driverClass = driverClass; + super(driverClass); this.streamingQueryConfigProvider = streamingQueryConfigProvider; this.sourceOperations = sourceOperations; } @@ -427,14 +426,20 @@ protected long getActualCursorRecordCount(final Connection connection, @Override public JdbcDatabase createDatabase(final JsonNode sourceConfig) throws SQLException { + return createDatabase(sourceConfig, JdbcDataSourceUtils.DEFAULT_JDBC_PARAMETERS_DELIMITER); + } + + public JdbcDatabase createDatabase(final JsonNode sourceConfig, String delimiter) throws SQLException { final JsonNode jdbcConfig = toDatabaseConfig(sourceConfig); + Map connectionProperties = JdbcDataSourceUtils.getConnectionProperties(sourceConfig, delimiter); // Create the data source final DataSource dataSource = DataSourceFactory.create( jdbcConfig.has(JdbcUtils.USERNAME_KEY) ? jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText() : null, jdbcConfig.has(JdbcUtils.PASSWORD_KEY) ? jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, - driverClass, + driverClassName, jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), - JdbcDataSourceUtils.getConnectionProperties(sourceConfig)); + connectionProperties, + getConnectionTimeout(connectionProperties)); // Record the data source so that it can be closed. dataSources.add(dataSource); diff --git a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/JdbcDataSourceUtils.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/jdbc/JdbcDataSourceUtils.java similarity index 84% rename from airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/JdbcDataSourceUtils.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/jdbc/JdbcDataSourceUtils.java index 80a922a8867b..f11193178ec4 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/JdbcDataSourceUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/jdbc/JdbcDataSourceUtils.java @@ -2,17 +2,17 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.jdbc; +package io.airbyte.cdk.integrations.source.jdbc; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.commons.map.MoreMaps; -import io.airbyte.db.jdbc.JdbcUtils; import java.util.Map; import java.util.Objects; public class JdbcDataSourceUtils { - public static String DEFAULT_JDBC_PARAMETERS_DELIMITER = "&"; + public static final String DEFAULT_JDBC_PARAMETERS_DELIMITER = "&"; /** * Validates for duplication parameters @@ -38,7 +38,11 @@ public static void assertCustomParametersDontOverwriteDefaultParameters(final Ma * @return A mapping of connection properties */ public static Map getConnectionProperties(final JsonNode config) { - final Map customProperties = JdbcUtils.parseJdbcParameters(config, JdbcUtils.JDBC_URL_PARAMS_KEY); + return getConnectionProperties(config, DEFAULT_JDBC_PARAMETERS_DELIMITER); + } + + public static Map getConnectionProperties(final JsonNode config, String parameterDelimiter) { + final Map customProperties = JdbcUtils.parseJdbcParameters(config, JdbcUtils.JDBC_URL_PARAMS_KEY, parameterDelimiter); final Map defaultProperties = JdbcDataSourceUtils.getDefaultConnectionProperties(config); assertCustomParametersDontOverwriteDefaultParameters(customProperties, defaultProperties); return MoreMaps.merge(customProperties, defaultProperties); diff --git a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/JdbcSSLConnectionUtils.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/jdbc/JdbcSSLConnectionUtils.java similarity index 98% rename from airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/JdbcSSLConnectionUtils.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/jdbc/JdbcSSLConnectionUtils.java index 5298c010ae2a..83106c17d6ce 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/JdbcSSLConnectionUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/jdbc/JdbcSSLConnectionUtils.java @@ -2,11 +2,11 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.jdbc; +package io.airbyte.cdk.integrations.source.jdbc; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.util.SSLCertificateUtils; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.util.SSLCertificateUtils; import java.io.IOException; import java.net.MalformedURLException; import java.net.URI; diff --git a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/JdbcSource.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/jdbc/JdbcSource.java similarity index 78% rename from airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/JdbcSource.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/jdbc/JdbcSource.java index e891352a8109..b50fdd36a931 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/JdbcSource.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/jdbc/JdbcSource.java @@ -2,14 +2,14 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.jdbc; +package io.airbyte.cdk.integrations.source.jdbc; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; import java.sql.JDBCType; import java.util.Set; import org.slf4j.Logger; diff --git a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/dto/JdbcPrivilegeDto.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/jdbc/dto/JdbcPrivilegeDto.java similarity index 97% rename from airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/dto/JdbcPrivilegeDto.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/jdbc/dto/JdbcPrivilegeDto.java index afade9b0b961..b598f041dde4 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/main/java/io/airbyte/integrations/source/jdbc/dto/JdbcPrivilegeDto.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/jdbc/dto/JdbcPrivilegeDto.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.jdbc.dto; +package io.airbyte.cdk.integrations.source.jdbc.dto; import com.google.common.base.Objects; diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSource.java similarity index 95% rename from airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSource.java index 204267aa0304..26d04bed4b6b 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSource.java @@ -2,13 +2,26 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb; +package io.airbyte.cdk.integrations.source.relationaldb; -import static io.airbyte.integrations.base.errors.messages.ErrorMessage.getErrorMessage; +import static io.airbyte.cdk.integrations.base.errors.messages.ErrorMessage.getErrorMessage; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import datadog.trace.api.Trace; +import io.airbyte.cdk.db.AbstractDatabase; +import io.airbyte.cdk.db.IncrementalUtils; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.JdbcConnector; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.source.relationaldb.InvalidCursorInfoUtil.InvalidCursorInfo; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateGeneratorUtils; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManagerFactory; +import io.airbyte.cdk.integrations.util.ApmTraceUtils; +import io.airbyte.cdk.integrations.util.ConnectorExceptionUtil; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.exceptions.ConnectionErrorException; import io.airbyte.commons.features.EnvVariableFeatureFlags; @@ -18,18 +31,6 @@ import io.airbyte.commons.stream.AirbyteStreamUtils; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.db.AbstractDatabase; -import io.airbyte.db.IncrementalUtils; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteTraceMessageUtility; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.source.relationaldb.InvalidCursorInfoUtil.InvalidCursorInfo; -import io.airbyte.integrations.source.relationaldb.state.StateGeneratorUtils; -import io.airbyte.integrations.source.relationaldb.state.StateManager; -import io.airbyte.integrations.source.relationaldb.state.StateManagerFactory; -import io.airbyte.integrations.util.ApmTraceUtils; -import io.airbyte.integrations.util.ConnectorExceptionUtil; import io.airbyte.protocol.models.CommonField; import io.airbyte.protocol.models.JsonSchemaPrimitiveUtil.JsonSchemaPrimitive; import io.airbyte.protocol.models.JsonSchemaType; @@ -68,15 +69,25 @@ * source of both non-relational and relational type */ public abstract class AbstractDbSource extends - BaseConnector implements Source, AutoCloseable { + JdbcConnector implements Source, AutoCloseable { public static final String CHECK_TRACE_OPERATION_NAME = "check-operation"; public static final String DISCOVER_TRACE_OPERATION_NAME = "discover-operation"; public static final String READ_TRACE_OPERATION_NAME = "read-operation"; private static final Logger LOGGER = LoggerFactory.getLogger(AbstractDbSource.class); + // TODO: Remove when the flag is not use anymore - private final FeatureFlags featureFlags = new EnvVariableFeatureFlags(); + protected FeatureFlags featureFlags = new EnvVariableFeatureFlags(); + + protected AbstractDbSource(String driverClassName) { + super(driverClassName); + } + + @VisibleForTesting + public void setFeatureFlags(FeatureFlags featureFlags) { + this.featureFlags = featureFlags; + } @Override @Trace(operationName = CHECK_TRACE_OPERATION_NAME) @@ -139,7 +150,7 @@ public AutoCloseableIterator read(final JsonNode config, final AirbyteStateType supportedStateType = getSupportedStateType(config); final StateManager stateManager = StateManagerFactory.createStateManager(supportedStateType, - StateGeneratorUtils.deserializeInitialState(state, featureFlags.useStreamCapableState(), supportedStateType), catalog); + StateGeneratorUtils.deserializeInitialState(state, supportedStateType), catalog); final Instant emittedAt = Instant.now(); final Database database = createDatabase(config); @@ -678,7 +689,7 @@ protected int getStateEmissionFrequency() { * @return A {@link AirbyteStateType} representing the state supported by this connector. */ protected AirbyteStateType getSupportedStateType(final JsonNode config) { - return AirbyteStateType.LEGACY; + return AirbyteStateType.STREAM; } } diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/CdcStateManager.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/CdcStateManager.java similarity index 93% rename from airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/CdcStateManager.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/CdcStateManager.java index 67fd093f20bd..06a1587bbff5 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/CdcStateManager.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/CdcStateManager.java @@ -2,10 +2,10 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb; +package io.airbyte.cdk.integrations.source.relationaldb; +import io.airbyte.cdk.integrations.source.relationaldb.models.CdcState; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.source.relationaldb.models.CdcState; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import java.util.Collections; diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/CursorInfo.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/CursorInfo.java similarity index 98% rename from airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/CursorInfo.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/CursorInfo.java index b8fc9b82b080..cf92ed8668d4 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/CursorInfo.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/CursorInfo.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb; +package io.airbyte.cdk.integrations.source.relationaldb; import java.util.Objects; diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/DbSourceDiscoverUtil.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/DbSourceDiscoverUtil.java similarity index 99% rename from airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/DbSourceDiscoverUtil.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/DbSourceDiscoverUtil.java index da011c75b3dd..9377190b7595 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/DbSourceDiscoverUtil.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/DbSourceDiscoverUtil.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb; +package io.airbyte.cdk.integrations.source.relationaldb; import static io.airbyte.protocol.models.v0.CatalogHelpers.fieldsToJsonSchema; import static java.util.stream.Collectors.toList; diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/InvalidCursorInfoUtil.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/InvalidCursorInfoUtil.java similarity index 94% rename from airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/InvalidCursorInfoUtil.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/InvalidCursorInfoUtil.java index 86a4a9af3252..650b2a60a0ac 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/InvalidCursorInfoUtil.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/InvalidCursorInfoUtil.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb; +package io.airbyte.cdk.integrations.source.relationaldb; import java.util.List; import java.util.stream.Collectors; diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/RelationalDbQueryUtils.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/RelationalDbQueryUtils.java similarity index 97% rename from airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/RelationalDbQueryUtils.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/RelationalDbQueryUtils.java index 39048d75d460..bffed2b6d040 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/RelationalDbQueryUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/RelationalDbQueryUtils.java @@ -2,13 +2,13 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb; +package io.airbyte.cdk.integrations.source.relationaldb; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.SqlDatabase; import io.airbyte.commons.stream.AirbyteStreamUtils; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.db.SqlDatabase; import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; import java.util.List; import java.util.StringJoiner; diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/StateDecoratingIterator.java similarity index 92% rename from airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/StateDecoratingIterator.java index 47b022d612d4..667a0ceb8152 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/StateDecoratingIterator.java @@ -2,15 +2,16 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb; +package io.airbyte.cdk.integrations.source.relationaldb; import com.google.common.collect.AbstractIterator; -import io.airbyte.db.IncrementalUtils; -import io.airbyte.integrations.source.relationaldb.state.StateManager; +import io.airbyte.cdk.db.IncrementalUtils; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; import io.airbyte.protocol.models.JsonSchemaPrimitiveUtil.JsonSchemaPrimitive; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateStats; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import java.util.Iterator; import java.util.Objects; @@ -53,6 +54,8 @@ public class StateDecoratingIterator extends AbstractIterator im */ private final int stateEmissionFrequency; private int totalRecordCount = 0; + // In between each state message, recordCountInStateMessage will be reset to 0. + private int recordCountInStateMessage = 0; private boolean emitIntermediateState = false; private AirbyteMessage intermediateStateMessage = null; private boolean hasCaughtException = false; @@ -128,6 +131,7 @@ protected AirbyteMessage computeNext() { } totalRecordCount++; + recordCountInStateMessage++; // Use try-catch to catch Exception that could occur when connection to the database fails try { final AirbyteMessage message = messageIterator.next(); @@ -139,7 +143,7 @@ protected AirbyteMessage computeNext() { if (stateEmissionFrequency > 0 && !Objects.equals(currentMaxCursor, initialCursor) && messageIterator.hasNext()) { // Only create an intermediate state when it is not the first or last record message. // The last state message will be processed seperately. - intermediateStateMessage = createStateMessage(false, totalRecordCount); + intermediateStateMessage = createStateMessage(false, recordCountInStateMessage); } currentMaxCursor = cursorCandidate; currentMaxCursorRecordCount = 1L; @@ -164,7 +168,7 @@ protected AirbyteMessage computeNext() { return optionalIntermediateMessage.orElse(endOfData()); } } else if (!hasEmittedFinalState) { - return createStateMessage(true, totalRecordCount); + return createStateMessage(true, recordCountInStateMessage); } else { return endOfData(); } @@ -185,6 +189,7 @@ protected final Optional getIntermediateMessage() { if (emitIntermediateState && intermediateStateMessage != null) { final AirbyteMessage message = intermediateStateMessage; intermediateStateMessage = null; + recordCountInStateMessage = 0; emitIntermediateState = false; return Optional.of(message); } @@ -196,14 +201,15 @@ protected final Optional getIntermediateMessage() { * read up so far * * @param isFinalState marker for if the final state of the iterator has been reached - * @param totalRecordCount count of read messages + * @param recordCount count of read messages * @return AirbyteMessage which includes information on state of records read so far */ - public AirbyteMessage createStateMessage(final boolean isFinalState, final int totalRecordCount) { + public AirbyteMessage createStateMessage(final boolean isFinalState, final int recordCount) { final AirbyteStateMessage stateMessage = stateManager.updateAndEmit(pair, currentMaxCursor, currentMaxCursorRecordCount); final Optional cursorInfo = stateManager.getCursorInfo(pair); + // logging once every 100 messages to reduce log verbosity - if (totalRecordCount % 100 == 0) { + if (recordCount % 100 == 0) { LOGGER.info("State report for stream {} - original: {} = {} (count {}) -> latest: {} = {} (count {})", pair, cursorInfo.map(CursorInfo::getOriginalCursorField).orElse(null), @@ -213,6 +219,10 @@ public AirbyteMessage createStateMessage(final boolean isFinalState, final int t cursorInfo.map(CursorInfo::getCursor).orElse(null), cursorInfo.map(CursorInfo::getCursorRecordCount).orElse(null)); } + + if (stateMessage != null) { + stateMessage.withSourceStats(new AirbyteStateStats().withRecordCount((double) recordCount)); + } if (isFinalState) { hasEmittedFinalState = true; if (stateManager.getCursor(pair).isEmpty()) { diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/TableInfo.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/TableInfo.java similarity index 88% rename from airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/TableInfo.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/TableInfo.java index aa09f0e3b914..1d990bdfd46b 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/TableInfo.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/TableInfo.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb; +package io.airbyte.cdk.integrations.source.relationaldb; import java.util.List; import lombok.Builder; diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/AbstractStateManager.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/AbstractStateManager.java similarity index 96% rename from airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/AbstractStateManager.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/AbstractStateManager.java index 2121a16a07ee..ea4214cf30b1 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/AbstractStateManager.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/AbstractStateManager.java @@ -2,9 +2,9 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb.state; +package io.airbyte.cdk.integrations.source.relationaldb.state; -import io.airbyte.integrations.source.relationaldb.CursorInfo; +import io.airbyte.cdk.integrations.source.relationaldb.CursorInfo; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; @@ -22,7 +22,7 @@ * @param The type associated with the state object managed by this manager. * @param The type associated with the state object stored in the state managed by this manager. */ -public abstract class AbstractStateManager implements StateManager { +public abstract class AbstractStateManager implements StateManager { /** * The {@link CursorManager} responsible for keeping track of the current cursor value for each diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/CursorManager.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/CursorManager.java similarity index 99% rename from airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/CursorManager.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/CursorManager.java index 74a24cbdb8ce..2449c7666d55 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/CursorManager.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/CursorManager.java @@ -2,10 +2,10 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb.state; +package io.airbyte.cdk.integrations.source.relationaldb.state; import com.google.common.annotations.VisibleForTesting; -import io.airbyte.integrations.source.relationaldb.CursorInfo; +import io.airbyte.cdk.integrations.source.relationaldb.CursorInfo; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/GlobalStateManager.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/GlobalStateManager.java similarity index 89% rename from airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/GlobalStateManager.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/GlobalStateManager.java index 579a3cda805a..384bd4d0cb8e 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/GlobalStateManager.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/GlobalStateManager.java @@ -2,18 +2,18 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb.state; +package io.airbyte.cdk.integrations.source.relationaldb.state; -import static io.airbyte.integrations.source.relationaldb.state.StateGeneratorUtils.CURSOR_FIELD_FUNCTION; -import static io.airbyte.integrations.source.relationaldb.state.StateGeneratorUtils.CURSOR_FUNCTION; -import static io.airbyte.integrations.source.relationaldb.state.StateGeneratorUtils.CURSOR_RECORD_COUNT_FUNCTION; -import static io.airbyte.integrations.source.relationaldb.state.StateGeneratorUtils.NAME_NAMESPACE_PAIR_FUNCTION; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateGeneratorUtils.CURSOR_FIELD_FUNCTION; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateGeneratorUtils.CURSOR_FUNCTION; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateGeneratorUtils.CURSOR_RECORD_COUNT_FUNCTION; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateGeneratorUtils.NAME_NAMESPACE_PAIR_FUNCTION; +import io.airbyte.cdk.integrations.source.relationaldb.CdcStateManager; +import io.airbyte.cdk.integrations.source.relationaldb.models.CdcState; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbState; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.source.relationaldb.CdcStateManager; -import io.airbyte.integrations.source.relationaldb.models.CdcState; -import io.airbyte.integrations.source.relationaldb.models.DbState; -import io.airbyte.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.protocol.models.v0.AirbyteGlobalState; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/LegacyStateManager.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/LegacyStateManager.java similarity index 94% rename from airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/LegacyStateManager.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/LegacyStateManager.java index 1fd17289f389..c12137e607a7 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/LegacyStateManager.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/LegacyStateManager.java @@ -2,12 +2,12 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb.state; +package io.airbyte.cdk.integrations.source.relationaldb.state; +import io.airbyte.cdk.integrations.source.relationaldb.CdcStateManager; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbState; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.source.relationaldb.CdcStateManager; -import io.airbyte.integrations.source.relationaldb.models.DbState; -import io.airbyte.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/SourceStateIterator.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/SourceStateIterator.java new file mode 100644 index 000000000000..203244800b42 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/SourceStateIterator.java @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.source.relationaldb.state; + +import com.google.common.collect.AbstractIterator; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateStats; +import java.time.Instant; +import java.util.Iterator; +import javax.annotation.CheckForNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SourceStateIterator extends AbstractIterator implements Iterator { + + private static final Logger LOGGER = LoggerFactory.getLogger(SourceStateIterator.class); + private final Iterator messageIterator; + private boolean hasEmittedFinalState = false; + private long recordCount = 0L; + private Instant lastCheckpoint = Instant.now(); + + private final SourceStateIteratorManager sourceStateIteratorManager; + + public SourceStateIterator(final Iterator messageIterator, + final SourceStateIteratorManager sourceStateIteratorManager) { + this.messageIterator = messageIterator; + this.sourceStateIteratorManager = sourceStateIteratorManager; + } + + @CheckForNull + @Override + protected AirbyteMessage computeNext() { + boolean iteratorHasNextValue = false; + try { + iteratorHasNextValue = messageIterator.hasNext(); + } catch (Exception ex) { + LOGGER.info("Caught exception while trying to get the next from message iterator. Treating hasNext to false. ", ex); + } + if (iteratorHasNextValue) { + if (sourceStateIteratorManager.shouldEmitStateMessage(recordCount, lastCheckpoint)) { + AirbyteStateMessage stateMessage = sourceStateIteratorManager.generateStateMessageAtCheckpoint(); + stateMessage.withSourceStats(new AirbyteStateStats().withRecordCount((double) recordCount)); + + recordCount = 0L; + lastCheckpoint = Instant.now(); + return new AirbyteMessage() + .withType(Type.STATE) + .withState(stateMessage); + } + // Use try-catch to catch Exception that could occur when connection to the database fails + try { + final T message = messageIterator.next(); + final AirbyteMessage processedMessage = sourceStateIteratorManager.processRecordMessage(message); + recordCount++; + return processedMessage; + } catch (final Exception e) { + throw new RuntimeException(e); + } + } else if (!hasEmittedFinalState) { + hasEmittedFinalState = true; + final AirbyteStateMessage finalStateMessage = sourceStateIteratorManager.createFinalStateMessage(); + finalStateMessage.withSourceStats(new AirbyteStateStats().withRecordCount((double) recordCount)); + recordCount = 0L; + return new AirbyteMessage() + .withType(Type.STATE) + .withState(finalStateMessage); + } else { + return endOfData(); + } + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/SourceStateIteratorManager.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/SourceStateIteratorManager.java new file mode 100644 index 000000000000..a76b0256be2f --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/SourceStateIteratorManager.java @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.source.relationaldb.state; + +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import java.time.Instant; + +public interface SourceStateIteratorManager { + + /** + * Returns a state message that should be emitted at checkpoint. + */ + AirbyteStateMessage generateStateMessageAtCheckpoint(); + + /** + * For the incoming record message, this method defines how the connector will consume it. + */ + AirbyteMessage processRecordMessage(final T message); + + /** + * At the end of the iteration, this method will be called and it will generate the final state + * message. + * + * @return + */ + AirbyteStateMessage createFinalStateMessage(); + + /** + * Determines if the iterator has reached checkpoint or not, based on the time and number of record + * messages it has been processed since the last checkpoint. + */ + boolean shouldEmitStateMessage(final long recordCount, final Instant lastCheckpoint); + +} diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StateGeneratorUtils.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/StateGeneratorUtils.java similarity index 95% rename from airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StateGeneratorUtils.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/StateGeneratorUtils.java index 8c4a6d3f8e64..4c272190946b 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StateGeneratorUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/StateGeneratorUtils.java @@ -2,17 +2,17 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb.state; +package io.airbyte.cdk.integrations.source.relationaldb.state; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.source.relationaldb.CursorInfo; +import io.airbyte.cdk.integrations.source.relationaldb.models.CdcState; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbState; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.commons.json.Jsons; import io.airbyte.configoss.StateWrapper; import io.airbyte.configoss.helpers.StateMessageHelper; -import io.airbyte.integrations.source.relationaldb.CursorInfo; -import io.airbyte.integrations.source.relationaldb.models.CdcState; -import io.airbyte.integrations.source.relationaldb.models.DbState; -import io.airbyte.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.protocol.models.v0.AirbyteGlobalState; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; @@ -225,10 +225,8 @@ public static AirbyteStateMessage convertStateMessage(final io.airbyte.protocol. * @return The deserialized object representation of the state. */ public static List deserializeInitialState(final JsonNode initialStateJson, - final boolean useStreamCapableState, final AirbyteStateType supportedStateType) { - final Optional typedState = StateMessageHelper.getTypedState(initialStateJson, - useStreamCapableState); + final Optional typedState = StateMessageHelper.getTypedState(initialStateJson); return typedState .map(state -> switch (state.getStateType()) { case GLOBAL -> List.of(StateGeneratorUtils.convertStateMessage(state.getGlobal())); diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StateManager.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/StateManager.java similarity index 96% rename from airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StateManager.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/StateManager.java index b3b01157e6a1..3bfb211ea2aa 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StateManager.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/StateManager.java @@ -2,11 +2,11 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb.state; +package io.airbyte.cdk.integrations.source.relationaldb.state; import com.google.common.base.Preconditions; -import io.airbyte.integrations.source.relationaldb.CdcStateManager; -import io.airbyte.integrations.source.relationaldb.CursorInfo; +import io.airbyte.cdk.integrations.source.relationaldb.CdcStateManager; +import io.airbyte.cdk.integrations.source.relationaldb.CursorInfo; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import java.util.List; @@ -22,7 +22,7 @@ * @param The type of the state maintained by the manager. * @param The type of the stream(s) stored within the state maintained by the manager. */ -public interface StateManager { +public interface StateManager { Logger LOGGER = LoggerFactory.getLogger(StateManager.class); diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StateManagerFactory.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/StateManagerFactory.java similarity index 94% rename from airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StateManagerFactory.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/StateManagerFactory.java index 3c13fddcdd9c..6c6d8b166443 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StateManagerFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/StateManagerFactory.java @@ -2,10 +2,10 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb.state; +package io.airbyte.cdk.integrations.source.relationaldb.state; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbState; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.source.relationaldb.models.DbState; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; @@ -46,7 +46,9 @@ public static StateManager createStateManager(final AirbyteStateType supportedSt switch (supportedStateType) { case LEGACY: LOGGER.info("Legacy state manager selected to manage state object with type {}.", airbyteStateMessage.getType()); - return new LegacyStateManager(Jsons.object(airbyteStateMessage.getData(), DbState.class), catalog); + @SuppressWarnings("deprecation") + StateManager retVal = new LegacyStateManager(Jsons.object(airbyteStateMessage.getData(), DbState.class), catalog); + return retVal; case GLOBAL: LOGGER.info("Global state manager selected to manage state object with type {}.", airbyteStateMessage.getType()); return new GlobalStateManager(generateGlobalState(airbyteStateMessage), catalog); diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StreamStateManager.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/StreamStateManager.java similarity index 84% rename from airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StreamStateManager.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/StreamStateManager.java index 80f77f3dd6ea..efb874b8b034 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StreamStateManager.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/java/io/airbyte/cdk/integrations/source/relationaldb/state/StreamStateManager.java @@ -2,16 +2,16 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb.state; +package io.airbyte.cdk.integrations.source.relationaldb.state; -import static io.airbyte.integrations.source.relationaldb.state.StateGeneratorUtils.CURSOR_FIELD_FUNCTION; -import static io.airbyte.integrations.source.relationaldb.state.StateGeneratorUtils.CURSOR_FUNCTION; -import static io.airbyte.integrations.source.relationaldb.state.StateGeneratorUtils.CURSOR_RECORD_COUNT_FUNCTION; -import static io.airbyte.integrations.source.relationaldb.state.StateGeneratorUtils.NAME_NAMESPACE_PAIR_FUNCTION; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateGeneratorUtils.CURSOR_FIELD_FUNCTION; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateGeneratorUtils.CURSOR_FUNCTION; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateGeneratorUtils.CURSOR_RECORD_COUNT_FUNCTION; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateGeneratorUtils.NAME_NAMESPACE_PAIR_FUNCTION; +import io.airbyte.cdk.integrations.source.relationaldb.CdcStateManager; +import io.airbyte.cdk.integrations.source.relationaldb.CursorInfo; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.source.relationaldb.CdcStateManager; -import io.airbyte.integrations.source.relationaldb.CursorInfo; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/resources/db_models/db_models.yaml b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/resources/db_models/db_models.yaml similarity index 100% rename from airbyte-integrations/connectors/source-relational-db/src/main/resources/db_models/db_models.yaml rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/resources/db_models/db_models.yaml diff --git a/airbyte-integrations/connectors/source-jdbc/src/main/resources/spec.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/main/resources/spec.json similarity index 100% rename from airbyte-integrations/connectors/source-jdbc/src/main/resources/spec.json rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/main/resources/spec.json diff --git a/airbyte-integrations/connectors/source-jdbc/src/test-integration/resources/dummy_config.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test-integration/resources/dummy_config.json similarity index 100% rename from airbyte-integrations/connectors/source-jdbc/src/test-integration/resources/dummy_config.json rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test-integration/resources/dummy_config.json diff --git a/airbyte-integrations/connectors/source-jdbc/src/test-integration/resources/expected_spec.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test-integration/resources/expected_spec.json similarity index 100% rename from airbyte-integrations/connectors/source-jdbc/src/test-integration/resources/expected_spec.json rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test-integration/resources/expected_spec.json diff --git a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/AirbyteDebeziumHandlerTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/AirbyteDebeziumHandlerTest.java similarity index 89% rename from airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/AirbyteDebeziumHandlerTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/AirbyteDebeziumHandlerTest.java index eaf1e219d98a..95b8e5e26d96 100644 --- a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/AirbyteDebeziumHandlerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/AirbyteDebeziumHandlerTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium; +package io.airbyte.cdk.integrations.debezium; import com.google.common.collect.Lists; import io.airbyte.protocol.models.Field; @@ -33,7 +33,7 @@ public void shouldUseCdcTestShouldReturnTrue() { // set all streams to incremental. configuredCatalog.getStreams().forEach(s -> s.setSyncMode(SyncMode.INCREMENTAL)); - Assertions.assertTrue(AirbyteDebeziumHandler.shouldUseCDC(configuredCatalog)); + Assertions.assertTrue(AirbyteDebeziumHandler.isAnyStreamIncrementalSyncMode(configuredCatalog)); } @Test @@ -50,7 +50,7 @@ public void shouldUseCdcTestShouldReturnFalse() { final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers .toDefaultConfiguredCatalog(catalog); - Assertions.assertFalse(AirbyteDebeziumHandler.shouldUseCDC(configuredCatalog)); + Assertions.assertFalse(AirbyteDebeziumHandler.isAnyStreamIncrementalSyncMode(configuredCatalog)); } } diff --git a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/AirbyteFileOffsetBackingStoreTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/AirbyteFileOffsetBackingStoreTest.java similarity index 96% rename from airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/AirbyteFileOffsetBackingStoreTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/AirbyteFileOffsetBackingStoreTest.java index 86613cdb7000..70fdefe0dd9e 100644 --- a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/AirbyteFileOffsetBackingStoreTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/AirbyteFileOffsetBackingStoreTest.java @@ -2,15 +2,15 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium; +package io.airbyte.cdk.integrations.debezium; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteFileOffsetBackingStore; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.debezium.internals.AirbyteFileOffsetBackingStore; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; diff --git a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/DebeziumRecordPublisherTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/DebeziumRecordPublisherTest.java similarity index 87% rename from airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/DebeziumRecordPublisherTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/DebeziumRecordPublisherTest.java index 32778d8e987c..be906557f431 100644 --- a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/DebeziumRecordPublisherTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/DebeziumRecordPublisherTest.java @@ -2,14 +2,14 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium; +package io.airbyte.cdk.integrations.debezium; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.common.collect.ImmutableList; -import io.airbyte.integrations.debezium.internals.DebeziumPropertiesManager; +import io.airbyte.cdk.integrations.debezium.internals.RelationalDbDebeziumPropertiesManager; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.CatalogHelpers; @@ -28,7 +28,7 @@ public void testTableIncludelistCreation() { CatalogHelpers.createConfiguredAirbyteStream("n\"aMéS", "public").withSyncMode(SyncMode.INCREMENTAL))); final String expectedWhitelist = "\\Qpublic.id_and_name\\E,\\Qpublic.id_\\,something\\E,\\Qpublic.n\"aMéS\\E"; - final String actualWhitelist = DebeziumPropertiesManager.getTableIncludelist(catalog); + final String actualWhitelist = RelationalDbDebeziumPropertiesManager.getTableIncludelist(catalog); assertEquals(expectedWhitelist, actualWhitelist); } @@ -40,7 +40,7 @@ public void testTableIncludelistFiltersFullRefresh() { CatalogHelpers.createConfiguredAirbyteStream("id_and_name2", "public").withSyncMode(SyncMode.FULL_REFRESH))); final String expectedWhitelist = "\\Qpublic.id_and_name\\E"; - final String actualWhitelist = DebeziumPropertiesManager.getTableIncludelist(catalog); + final String actualWhitelist = RelationalDbDebeziumPropertiesManager.getTableIncludelist(catalog); assertEquals(expectedWhitelist, actualWhitelist); } @@ -57,7 +57,7 @@ public void testColumnIncludelistFiltersFullRefresh() { CatalogHelpers.createConfiguredAirbyteStream("n\"aMéS", "public").withSyncMode(SyncMode.INCREMENTAL))); final String expectedWhitelist = "\\Qpublic.id_and_name\\E\\.(\\Qfld2\\E|\\Qfld1\\E),\\Qpublic.id_\\,something\\E,\\Qpublic.n\"aMéS\\E"; - final String actualWhitelist = DebeziumPropertiesManager.getColumnIncludeList(catalog); + final String actualWhitelist = RelationalDbDebeziumPropertiesManager.getColumnIncludeList(catalog); assertEquals(expectedWhitelist, actualWhitelist); } @@ -76,7 +76,7 @@ public void testColumnIncludeListEscaping() { "public", Field.of("fld1", JsonSchemaType.NUMBER), Field.of("fld2", JsonSchemaType.STRING)).withSyncMode(SyncMode.INCREMENTAL))); - final String anchored = "^" + DebeziumPropertiesManager.getColumnIncludeList(catalog) + "$"; + final String anchored = "^" + RelationalDbDebeziumPropertiesManager.getColumnIncludeList(catalog) + "$"; final Pattern pattern = Pattern.compile(anchored); assertTrue(pattern.matcher("public.id_and_name.fld1").find()); diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/AirbyteSchemaHistoryStorageTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/AirbyteSchemaHistoryStorageTest.java new file mode 100644 index 000000000000..268c11a6012e --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/AirbyteSchemaHistoryStorageTest.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.airbyte.cdk.integrations.debezium.internals.AirbyteSchemaHistoryStorage.SchemaHistory; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import java.io.IOException; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +public class AirbyteSchemaHistoryStorageTest { + + @Test + public void testForContentBiggerThan3MBLimit() throws IOException { + final String contentReadDirectlyFromFile = MoreResources.readResource("dbhistory_greater_than_3_mb.dat"); + + final AirbyteSchemaHistoryStorage schemaHistoryStorageFromUncompressedContent = AirbyteSchemaHistoryStorage.initializeDBHistory( + new SchemaHistory<>(Optional.of(Jsons.jsonNode(contentReadDirectlyFromFile)), + false), + true); + final SchemaHistory schemaHistoryFromUncompressedContent = schemaHistoryStorageFromUncompressedContent.read(); + + assertTrue(schemaHistoryFromUncompressedContent.isCompressed()); + assertNotNull(schemaHistoryFromUncompressedContent.schema()); + assertEquals(contentReadDirectlyFromFile, schemaHistoryStorageFromUncompressedContent.readUncompressed()); + + final AirbyteSchemaHistoryStorage schemaHistoryStorageFromCompressedContent = AirbyteSchemaHistoryStorage.initializeDBHistory( + new SchemaHistory<>(Optional.of(Jsons.jsonNode(schemaHistoryFromUncompressedContent.schema())), + true), + true); + final SchemaHistory schemaHistoryFromCompressedContent = schemaHistoryStorageFromCompressedContent.read(); + + assertTrue(schemaHistoryFromCompressedContent.isCompressed()); + assertNotNull(schemaHistoryFromCompressedContent.schema()); + assertEquals(schemaHistoryFromUncompressedContent.schema(), schemaHistoryFromCompressedContent.schema()); + } + + @Test + public void sizeTest() throws IOException { + assertEquals(5.881045341491699, + AirbyteSchemaHistoryStorage.calculateSizeOfStringInMB(MoreResources.readResource("dbhistory_greater_than_3_mb.dat"))); + assertEquals(0.0038671493530273438, + AirbyteSchemaHistoryStorage.calculateSizeOfStringInMB(MoreResources.readResource("dbhistory_less_than_3_mb.dat"))); + } + + @Test + public void testForContentLessThan3MBLimit() throws IOException { + final String contentReadDirectlyFromFile = MoreResources.readResource("dbhistory_less_than_3_mb.dat"); + + final AirbyteSchemaHistoryStorage schemaHistoryStorageFromUncompressedContent = AirbyteSchemaHistoryStorage.initializeDBHistory( + new SchemaHistory<>(Optional.of(Jsons.jsonNode(contentReadDirectlyFromFile)), + false), + true); + final SchemaHistory schemaHistoryFromUncompressedContent = schemaHistoryStorageFromUncompressedContent.read(); + + assertFalse(schemaHistoryFromUncompressedContent.isCompressed()); + assertNotNull(schemaHistoryFromUncompressedContent.schema()); + assertEquals(contentReadDirectlyFromFile, schemaHistoryFromUncompressedContent.schema()); + + final AirbyteSchemaHistoryStorage schemaHistoryStorageFromCompressedContent = AirbyteSchemaHistoryStorage.initializeDBHistory( + new SchemaHistory<>(Optional.of(Jsons.jsonNode(schemaHistoryFromUncompressedContent.schema())), + false), + true); + final SchemaHistory schemaHistoryFromCompressedContent = schemaHistoryStorageFromCompressedContent.read(); + + assertFalse(schemaHistoryFromCompressedContent.isCompressed()); + assertNotNull(schemaHistoryFromCompressedContent.schema()); + assertEquals(schemaHistoryFromUncompressedContent.schema(), schemaHistoryFromCompressedContent.schema()); + } + +} diff --git a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumConverterUtilsTest.java similarity index 98% rename from airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtilsTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumConverterUtilsTest.java index aba9d67ec0a3..59312c888703 100644 --- a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/DebeziumConverterUtilsTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumConverterUtilsTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium.internals; +package io.airbyte.cdk.integrations.debezium.internals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumEventUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumEventUtilsTest.java new file mode 100644 index 000000000000..180f56f781bb --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumEventUtilsTest.java @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals; + +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcEventUtils.ID_FIELD; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcEventUtils.OBJECT_ID_FIELD; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.debezium.CdcMetadataInjector; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager.DebeziumConnectorType; +import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.debezium.engine.ChangeEvent; +import java.io.IOException; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; +import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; + +class DebeziumEventUtilsTest { + + @Test + void testConvertRelationalDbChangeEvent() throws IOException { + final String stream = "names"; + final Instant emittedAt = Instant.now(); + final CdcMetadataInjector cdcMetadataInjector = new DummyMetadataInjector(); + final ChangeEventWithMetadata insertChangeEvent = mockChangeEvent("insert_change_event.json", ""); + final ChangeEventWithMetadata updateChangeEvent = mockChangeEvent("update_change_event.json", ""); + final ChangeEventWithMetadata deleteChangeEvent = mockChangeEvent("delete_change_event.json", ""); + final ConfiguredAirbyteCatalog configuredAirbyteCatalog = mock(ConfiguredAirbyteCatalog.class); + + final AirbyteMessage actualInsert = + DebeziumEventUtils.toAirbyteMessage(insertChangeEvent, cdcMetadataInjector, configuredAirbyteCatalog, emittedAt, + DebeziumConnectorType.RELATIONALDB, Jsons.emptyObject()); + final AirbyteMessage actualUpdate = + DebeziumEventUtils.toAirbyteMessage(updateChangeEvent, cdcMetadataInjector, configuredAirbyteCatalog, emittedAt, + DebeziumConnectorType.RELATIONALDB, Jsons.emptyObject()); + final AirbyteMessage actualDelete = + DebeziumEventUtils.toAirbyteMessage(deleteChangeEvent, cdcMetadataInjector, configuredAirbyteCatalog, emittedAt, + DebeziumConnectorType.RELATIONALDB, Jsons.emptyObject()); + + final AirbyteMessage expectedInsert = createAirbyteMessage(stream, emittedAt, "insert_message.json"); + final AirbyteMessage expectedUpdate = createAirbyteMessage(stream, emittedAt, "update_message.json"); + final AirbyteMessage expectedDelete = createAirbyteMessage(stream, emittedAt, "delete_message.json"); + + deepCompare(expectedInsert, actualInsert); + deepCompare(expectedUpdate, actualUpdate); + deepCompare(expectedDelete, actualDelete); + } + + @Test + void testConvertMongoDbChangeEvent() throws IOException { + final String objectId = "64f24244f95155351c4185b1"; + final String stream = "names"; + final Instant emittedAt = Instant.now(); + final CdcMetadataInjector cdcMetadataInjector = new DummyMetadataInjector(); + final ChangeEventWithMetadata insertChangeEvent = mockChangeEvent("mongodb/change_event_insert.json", ""); + final ChangeEventWithMetadata updateChangeEvent = mockChangeEvent("mongodb/change_event_update.json", ""); + final ChangeEventWithMetadata deleteChangeEvent = mockChangeEvent("mongodb/change_event_delete.json", ""); + final ChangeEventWithMetadata deleteChangeEventNoBefore = mockChangeEvent("mongodb/change_event_delete_no_before.json", + "{\\\"" + OBJECT_ID_FIELD + "\\\": \\\"" + objectId + "\\\"}"); + + final AirbyteMessage expectedInsert = createAirbyteMessage(stream, emittedAt, "mongodb/insert_airbyte_message.json"); + final AirbyteMessage expectedUpdate = createAirbyteMessage(stream, emittedAt, "mongodb/update_airbyte_message.json"); + final AirbyteMessage expectedDelete = createAirbyteMessage(stream, emittedAt, "mongodb/delete_airbyte_message.json"); + final AirbyteMessage expectedDeleteNoBefore = createAirbyteMessage(stream, emittedAt, "mongodb/delete_no_before_airbyte_message.json"); + + final ConfiguredAirbyteCatalog insertConfiguredAirbyteCatalog = buildFromAirbyteMessage(expectedInsert); + final ConfiguredAirbyteCatalog updateConfiguredAirbyteCatalog = buildFromAirbyteMessage(expectedUpdate); + final ConfiguredAirbyteCatalog deleteConfiguredAirbyteCatalog = buildFromAirbyteMessage(expectedDelete); + final ConfiguredAirbyteCatalog deleteNoBeforeConfiguredAirbyteCatalog = buildFromAirbyteMessage(expectedDeleteNoBefore); + + final AirbyteMessage actualInsert = + DebeziumEventUtils.toAirbyteMessage(insertChangeEvent, cdcMetadataInjector, insertConfiguredAirbyteCatalog, emittedAt, + DebeziumConnectorType.MONGODB, Jsons.emptyObject()); + final AirbyteMessage actualUpdate = + DebeziumEventUtils.toAirbyteMessage(updateChangeEvent, cdcMetadataInjector, updateConfiguredAirbyteCatalog, emittedAt, + DebeziumConnectorType.MONGODB, Jsons.emptyObject()); + final AirbyteMessage actualDelete = + DebeziumEventUtils.toAirbyteMessage(deleteChangeEvent, cdcMetadataInjector, deleteConfiguredAirbyteCatalog, emittedAt, + DebeziumConnectorType.MONGODB, Jsons.emptyObject()); + final AirbyteMessage actualDeleteNoBefore = + DebeziumEventUtils.toAirbyteMessage(deleteChangeEventNoBefore, cdcMetadataInjector, deleteNoBeforeConfiguredAirbyteCatalog, emittedAt, + DebeziumConnectorType.MONGODB, Jsons.emptyObject()); + + deepCompare(expectedInsert, actualInsert); + deepCompare(expectedUpdate, actualUpdate); + deepCompare(expectedDelete, actualDelete); + deepCompare(expectedDeleteNoBefore, actualDeleteNoBefore); + } + + @Test + void testConvertMongoDbChangeEventNoSchema() throws IOException { + final String objectId = "64f24244f95155351c4185b1"; + final String stream = "names"; + final Instant emittedAt = Instant.now(); + final CdcMetadataInjector cdcMetadataInjector = new DummyMetadataInjector(); + final ChangeEventWithMetadata insertChangeEvent = mockChangeEvent("mongodb/change_event_insert.json", ""); + final ChangeEventWithMetadata updateChangeEvent = mockChangeEvent("mongodb/change_event_update.json", ""); + final ChangeEventWithMetadata deleteChangeEvent = mockChangeEvent("mongodb/change_event_delete.json", ""); + final ChangeEventWithMetadata deleteChangeEventNoBefore = mockChangeEvent("mongodb/change_event_delete_no_before.json", + "{\\\"" + OBJECT_ID_FIELD + "\\\": \\\"" + objectId + "\\\"}"); + + final AirbyteMessage expectedInsert = createAirbyteMessage(stream, emittedAt, "mongodb/insert_airbyte_message_no_schema.json"); + final AirbyteMessage expectedUpdate = createAirbyteMessage(stream, emittedAt, "mongodb/update_airbyte_message_no_schema.json"); + final AirbyteMessage expectedDelete = createAirbyteMessage(stream, emittedAt, "mongodb/delete_airbyte_message_no_schema.json"); + final AirbyteMessage expectedDeleteNoBefore = createAirbyteMessage(stream, emittedAt, "mongodb/delete_no_before_airbyte_message_no_schema.json"); + + final ConfiguredAirbyteCatalog insertConfiguredAirbyteCatalog = buildFromAirbyteMessage(expectedInsert); + final ConfiguredAirbyteCatalog updateConfiguredAirbyteCatalog = buildFromAirbyteMessage(expectedUpdate); + final ConfiguredAirbyteCatalog deleteConfiguredAirbyteCatalog = buildFromAirbyteMessage(expectedDelete); + final ConfiguredAirbyteCatalog deleteNoBeforeConfiguredAirbyteCatalog = buildFromAirbyteMessage(expectedDeleteNoBefore); + + final JsonNode noSchemaConfig = + Jsons.jsonNode(ImmutableMap.builder().put(MongoDbDebeziumConstants.Configuration.SCHEMA_ENFORCED_CONFIGURATION_KEY, false).build()); + final AirbyteMessage actualInsert = + DebeziumEventUtils.toAirbyteMessage(insertChangeEvent, cdcMetadataInjector, insertConfiguredAirbyteCatalog, emittedAt, + DebeziumConnectorType.MONGODB, noSchemaConfig); + final AirbyteMessage actualUpdate = + DebeziumEventUtils.toAirbyteMessage(updateChangeEvent, cdcMetadataInjector, updateConfiguredAirbyteCatalog, emittedAt, + DebeziumConnectorType.MONGODB, noSchemaConfig); + final AirbyteMessage actualDelete = + DebeziumEventUtils.toAirbyteMessage(deleteChangeEvent, cdcMetadataInjector, deleteConfiguredAirbyteCatalog, emittedAt, + DebeziumConnectorType.MONGODB, noSchemaConfig); + final AirbyteMessage actualDeleteNoBefore = + DebeziumEventUtils.toAirbyteMessage(deleteChangeEventNoBefore, cdcMetadataInjector, deleteNoBeforeConfiguredAirbyteCatalog, emittedAt, + DebeziumConnectorType.MONGODB, noSchemaConfig); + + deepCompare(expectedInsert, actualInsert); + deepCompare(expectedUpdate, actualUpdate); + deepCompare(expectedDelete, actualDelete); + deepCompare(expectedDeleteNoBefore, actualDeleteNoBefore); + } + + @Test + void testConvertMongoDbChangeEventUnsupportedOperation() throws IOException { + final Instant emittedAt = Instant.now(); + final CdcMetadataInjector cdcMetadataInjector = new DummyMetadataInjector(); + final ChangeEventWithMetadata unsupportedOperationEvent = mockChangeEvent("mongodb/change_event_unsupported.json", ""); + final ConfiguredAirbyteCatalog configuredAirbyteCatalog = mock(ConfiguredAirbyteCatalog.class); + assertThrows(IllegalArgumentException.class, + () -> DebeziumEventUtils.toAirbyteMessage(unsupportedOperationEvent, cdcMetadataInjector, configuredAirbyteCatalog, emittedAt, + DebeziumConnectorType.MONGODB, Jsons.emptyObject())); + } + + private ConfiguredAirbyteCatalog buildFromAirbyteMessage(final AirbyteMessage airbyteMessage) { + final ConfiguredAirbyteCatalog configuredAirbyteCatalog = new ConfiguredAirbyteCatalog(); + final ConfiguredAirbyteStream configuredAirbyteStream = new ConfiguredAirbyteStream(); + final AirbyteStream airbyteStream = new AirbyteStream(); + airbyteStream.setName(airbyteMessage.getRecord().getStream()); + airbyteStream.setNamespace(airbyteMessage.getRecord().getNamespace()); + airbyteStream.setJsonSchema(Jsons.jsonNode(Map.of("properties", airbyteMessage.getRecord().getData()))); + configuredAirbyteStream.setStream(airbyteStream); + configuredAirbyteCatalog.setStreams(List.of(configuredAirbyteStream)); + return configuredAirbyteCatalog; + } + + private static ChangeEventWithMetadata mockChangeEvent(final String resourceName, final String idValue) throws IOException { + final ChangeEvent mocked = mock(ChangeEvent.class); + final String resource = MoreResources.readResource(resourceName); + final String key = "{\"" + ID_FIELD + "\":\"" + idValue + "\"}"; + when(mocked.key()).thenReturn(key); + when(mocked.value()).thenReturn(resource); + + return new ChangeEventWithMetadata(mocked); + } + + private static AirbyteMessage createAirbyteMessage(final String stream, final Instant emittedAt, final String resourceName) throws IOException { + final String data = MoreResources.readResource(resourceName); + + final AirbyteRecordMessage recordMessage = new AirbyteRecordMessage() + .withStream(stream) + .withNamespace("public") + .withData(Jsons.deserialize(data)) + .withEmittedAt(emittedAt.toEpochMilli()); + + return new AirbyteMessage() + .withType(AirbyteMessage.Type.RECORD) + .withRecord(recordMessage); + } + + private static void deepCompare(final Object expected, final Object actual) { + assertEquals(Jsons.deserialize(Jsons.serialize(expected)), Jsons.deserialize(Jsons.serialize(actual))); + } + + public static class DummyMetadataInjector implements CdcMetadataInjector { + + @Override + public void addMetaData(final ObjectNode event, final JsonNode source) { + if (source.has("lsn")) { + final long lsn = source.get("lsn").asLong(); + event.put("_ab_cdc_lsn", lsn); + } + } + + @Override + public String namespace(final JsonNode source) { + return source.has("schema") ? source.get("schema").asText() : source.get("db").asText(); + } + + @Override + public String name(final JsonNode source) { + return source.has("table") ? source.get("table").asText() : source.get("collection").asText(); + } + + } + +} diff --git a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/DebeziumRecordIteratorTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumRecordIteratorTest.java similarity index 92% rename from airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/DebeziumRecordIteratorTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumRecordIteratorTest.java index f9f2bedf4f3c..e386b100c647 100644 --- a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/DebeziumRecordIteratorTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumRecordIteratorTest.java @@ -2,12 +2,12 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium.internals; +package io.airbyte.cdk.integrations.debezium.internals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; -import io.airbyte.integrations.debezium.CdcTargetPosition; +import io.airbyte.cdk.integrations.debezium.CdcTargetPosition; import io.debezium.engine.ChangeEvent; import java.time.Duration; import java.util.Collections; @@ -36,6 +36,7 @@ public Long extractPositionFromHeartbeatOffset(final Map sourceOffset }, () -> false, mock(DebeziumShutdownProcedure.class), + Duration.ZERO, Duration.ZERO); final Long lsn = debeziumRecordIterator.getHeartbeatPosition(new ChangeEvent() { diff --git a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/DebeziumShutdownProcedureTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumShutdownProcedureTest.java similarity index 96% rename from airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/DebeziumShutdownProcedureTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumShutdownProcedureTest.java index 4dac5061c866..335d157ed271 100644 --- a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/DebeziumShutdownProcedureTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/DebeziumShutdownProcedureTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium.internals; +package io.airbyte.cdk.integrations.debezium.internals; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; diff --git a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/MysqlDebeziumStateUtilTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/MysqlDebeziumStateUtilTest.java similarity index 91% rename from airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/MysqlDebeziumStateUtilTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/MysqlDebeziumStateUtilTest.java index 3cddad443ba6..7ba8a705691e 100644 --- a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/MysqlDebeziumStateUtilTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/MysqlDebeziumStateUtilTest.java @@ -2,22 +2,22 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium.internals; +package io.airbyte.cdk.integrations.debezium.internals; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil; +import io.airbyte.cdk.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil; -import io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; @@ -67,7 +67,9 @@ public void debeziumInitialStateConstructTest() throws SQLException { final JdbcDatabase database = getJdbcDatabase(container); final MySqlDebeziumStateUtil mySqlDebeziumStateUtil = new MySqlDebeziumStateUtil(); final JsonNode debeziumState = mySqlDebeziumStateUtil.constructInitialDebeziumState(MYSQL_PROPERTIES, CONFIGURED_CATALOG, database); - Assertions.assertEquals(2, Jsons.object(debeziumState, Map.class).size()); + Assertions.assertEquals(3, Jsons.object(debeziumState, Map.class).size()); + Assertions.assertTrue(debeziumState.has("is_compressed")); + Assertions.assertFalse(debeziumState.get("is_compressed").asBoolean()); Assertions.assertTrue(debeziumState.has("mysql_db_history")); Assertions.assertNotNull(debeziumState.get("mysql_db_history")); Assertions.assertTrue(debeziumState.has("mysql_cdc_offset")); diff --git a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/PostgresDebeziumStateUtilTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/PostgresDebeziumStateUtilTest.java similarity index 92% rename from airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/PostgresDebeziumStateUtilTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/PostgresDebeziumStateUtilTest.java index ec3b90bde596..280d0ac2709e 100644 --- a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/PostgresDebeziumStateUtilTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/PostgresDebeziumStateUtilTest.java @@ -2,22 +2,22 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium.internals; +package io.airbyte.cdk.integrations.debezium.internals; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.PostgresUtils; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.debezium.internals.postgres.PostgresDebeziumStateUtil; +import io.airbyte.cdk.testutils.PostgreSQLContainerHelper; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.PostgresUtils; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.debezium.internals.postgres.PostgresDebeziumStateUtil; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.test.utils.PostgreSQLContainerHelper; import io.debezium.connector.postgresql.connection.Lsn; import java.sql.SQLException; import java.time.Instant; @@ -145,9 +145,9 @@ public void emptyState() { Assertions.assertTrue(savedOffsetAfterReplicationSlotLSN); } - @ParameterizedTest - @ValueSource(strings = {"pgoutput", "wal2json"}) - public void LsnCommitTest(final String plugin) throws SQLException { + @Test + public void LsnCommitTest() throws SQLException { + final String plugin = "pgoutput"; final DockerImageName myImage = DockerImageName.parse("debezium/postgres:13-alpine").asCompatibleSubstituteFor("postgres"); final String dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); final String fullReplicationSlot = "debezium_slot" + "_" + dbName; @@ -179,9 +179,10 @@ public void LsnCommitTest(final String plugin) throws SQLException { database.execute("CREATE TABLE public.test_table (id int primary key, name varchar(256));"); database.execute("insert into public.test_table values (1, 'foo');"); database.execute("insert into public.test_table values (2, 'bar');"); + database.execute("CHECKPOINT"); - final Lsn lsnAtTheBeginning = Lsn.valueOf( - getReplicationSlot(database, fullReplicationSlot, plugin, dbName).get("confirmed_flush_lsn").asText()); + final var slotStateAtTheBeginning = getReplicationSlot(database, fullReplicationSlot, plugin, dbName); + final Lsn lsnAtTheBeginning = Lsn.valueOf(slotStateAtTheBeginning.get("confirmed_flush_lsn").asText()); final long targetLsn = PostgresUtils.getLsn(database).asLong(); postgresDebeziumStateUtil.commitLSNToPostgresDatabase(Jsons.jsonNode(databaseConfig), @@ -190,11 +191,13 @@ public void LsnCommitTest(final String plugin) throws SQLException { publication, plugin); - final Lsn lsnAfterCommit = Lsn.valueOf( - getReplicationSlot(database, fullReplicationSlot, plugin, dbName).get("confirmed_flush_lsn").asText()); + final var slotStateAfterCommit = getReplicationSlot(database, fullReplicationSlot, plugin, dbName); + final Lsn lsnAfterCommit = Lsn.valueOf(slotStateAfterCommit.get("confirmed_flush_lsn").asText()); Assertions.assertEquals(1, lsnAfterCommit.compareTo(lsnAtTheBeginning)); Assertions.assertEquals(targetLsn, lsnAfterCommit.asLong()); + Assertions.assertNotEquals(slotStateAtTheBeginning, slotStateAfterCommit); + container.stop(); } } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/RecordWaitTimeUtilTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/RecordWaitTimeUtilTest.java new file mode 100644 index 000000000000..64701dd40668 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/RecordWaitTimeUtilTest.java @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals; + +import static io.airbyte.cdk.integrations.debezium.internals.RecordWaitTimeUtil.MAX_FIRST_RECORD_WAIT_TIME; +import static io.airbyte.cdk.integrations.debezium.internals.RecordWaitTimeUtil.MIN_FIRST_RECORD_WAIT_TIME; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import java.time.Duration; +import java.util.Collections; +import java.util.Map; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +public class RecordWaitTimeUtilTest { + + @Test + void testGetFirstRecordWaitTime() { + final JsonNode emptyConfig = Jsons.jsonNode(Collections.emptyMap()); + assertDoesNotThrow(() -> RecordWaitTimeUtil.checkFirstRecordWaitTime(emptyConfig)); + assertEquals(Optional.empty(), RecordWaitTimeUtil.getFirstRecordWaitSeconds(emptyConfig)); + assertEquals(RecordWaitTimeUtil.DEFAULT_FIRST_RECORD_WAIT_TIME, RecordWaitTimeUtil.getFirstRecordWaitTime(emptyConfig)); + + final JsonNode normalConfig = Jsons.jsonNode(Map.of("replication_method", + Map.of("method", "CDC", "initial_waiting_seconds", 500))); + assertDoesNotThrow(() -> RecordWaitTimeUtil.checkFirstRecordWaitTime(normalConfig)); + assertEquals(Optional.of(500), RecordWaitTimeUtil.getFirstRecordWaitSeconds(normalConfig)); + assertEquals(Duration.ofSeconds(500), RecordWaitTimeUtil.getFirstRecordWaitTime(normalConfig)); + + final int tooShortTimeout = (int) MIN_FIRST_RECORD_WAIT_TIME.getSeconds() - 1; + final JsonNode tooShortConfig = Jsons.jsonNode(Map.of("replication_method", + Map.of("method", "CDC", "initial_waiting_seconds", tooShortTimeout))); + assertThrows(IllegalArgumentException.class, () -> RecordWaitTimeUtil.checkFirstRecordWaitTime(tooShortConfig)); + assertEquals(Optional.of(tooShortTimeout), RecordWaitTimeUtil.getFirstRecordWaitSeconds(tooShortConfig)); + assertEquals(MIN_FIRST_RECORD_WAIT_TIME, RecordWaitTimeUtil.getFirstRecordWaitTime(tooShortConfig)); + + final int tooLongTimeout = (int) MAX_FIRST_RECORD_WAIT_TIME.getSeconds() + 1; + final JsonNode tooLongConfig = Jsons.jsonNode(Map.of("replication_method", + Map.of("method", "CDC", "initial_waiting_seconds", tooLongTimeout))); + assertThrows(IllegalArgumentException.class, () -> RecordWaitTimeUtil.checkFirstRecordWaitTime(tooLongConfig)); + assertEquals(Optional.of(tooLongTimeout), RecordWaitTimeUtil.getFirstRecordWaitSeconds(tooLongConfig)); + assertEquals(MAX_FIRST_RECORD_WAIT_TIME, RecordWaitTimeUtil.getFirstRecordWaitTime(tooLongConfig)); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcEventUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcEventUtilsTest.java new file mode 100644 index 000000000000..5a2df27854b2 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcEventUtilsTest.java @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals.mongodb; + +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcEventUtils.DOCUMENT_OBJECT_ID_FIELD; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcEventUtils.ID_FIELD; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcEventUtils.OBJECT_ID_FIELD; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcEventUtils.OBJECT_ID_FIELD_PATTERN; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.SCHEMALESS_MODE_DATA_FIELD; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.DataTypeUtils; +import io.airbyte.commons.json.Jsons; +import java.nio.charset.Charset; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import org.apache.commons.codec.binary.Base64; +import org.bson.BsonBinary; +import org.bson.BsonBoolean; +import org.bson.BsonDateTime; +import org.bson.BsonDecimal128; +import org.bson.BsonDocument; +import org.bson.BsonDouble; +import org.bson.BsonInt32; +import org.bson.BsonInt64; +import org.bson.BsonJavaScript; +import org.bson.BsonJavaScriptWithScope; +import org.bson.BsonNull; +import org.bson.BsonObjectId; +import org.bson.BsonRegularExpression; +import org.bson.BsonString; +import org.bson.BsonSymbol; +import org.bson.BsonTimestamp; +import org.bson.Document; +import org.bson.UuidRepresentation; +import org.bson.types.Decimal128; +import org.bson.types.ObjectId; +import org.junit.jupiter.api.Test; +import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; + +class MongoDbCdcEventUtilsTest { + + private static final String OBJECT_ID = "64f24244f95155351c4185b1"; + + @Test + void testGenerateObjectIdDocument() { + final String key = "{\"" + OBJECT_ID_FIELD + "\": \"" + OBJECT_ID + "\"}"; + JsonNode debeziumEventKey = Jsons.jsonNode(Map.of(ID_FIELD, key)); + + String updated = MongoDbCdcEventUtils.generateObjectIdDocument(debeziumEventKey); + + assertTrue(updated.contains(DOCUMENT_OBJECT_ID_FIELD)); + assertEquals(key.replaceAll(OBJECT_ID_FIELD_PATTERN, DOCUMENT_OBJECT_ID_FIELD), updated); + + debeziumEventKey = Jsons.jsonNode(Map.of(ID_FIELD, "\"" + OBJECT_ID + "\"")); + updated = MongoDbCdcEventUtils.generateObjectIdDocument(debeziumEventKey); + assertTrue(updated.contains(DOCUMENT_OBJECT_ID_FIELD)); + assertEquals(Jsons.serialize(Jsons.jsonNode(Map.of(DOCUMENT_OBJECT_ID_FIELD, OBJECT_ID))), updated); + } + + @Test + void testNormalizeObjectId() { + final JsonNode data = MongoDbCdcEventUtils.normalizeObjectId((ObjectNode) Jsons.jsonNode( + Map.of(DOCUMENT_OBJECT_ID_FIELD, Map.of(OBJECT_ID_FIELD, OBJECT_ID)))); + assertEquals(OBJECT_ID, data.get(DOCUMENT_OBJECT_ID_FIELD).asText()); + + final JsonNode dataWithoutObjectId = MongoDbCdcEventUtils.normalizeObjectId((ObjectNode) Jsons.jsonNode( + Map.of(DOCUMENT_OBJECT_ID_FIELD, Map.of()))); + assertNotEquals(OBJECT_ID, dataWithoutObjectId.get(DOCUMENT_OBJECT_ID_FIELD).asText()); + + final JsonNode dataWithoutId = MongoDbCdcEventUtils.normalizeObjectId((ObjectNode) Jsons.jsonNode(Map.of())); + assertNull(dataWithoutId.get(DOCUMENT_OBJECT_ID_FIELD)); + + final JsonNode stringId = MongoDbCdcEventUtils.normalizeObjectId((ObjectNode) Jsons.jsonNode(Map.of(DOCUMENT_OBJECT_ID_FIELD, "abcd"))); + assertEquals("abcd", stringId.get(DOCUMENT_OBJECT_ID_FIELD).asText()); + } + + @Test + void testNormalizeObjectIdNoSchema() { + var objectNode = (ObjectNode) Jsons.jsonNode(Map.of(DOCUMENT_OBJECT_ID_FIELD, Map.of(OBJECT_ID_FIELD, OBJECT_ID))); + objectNode.set(SCHEMALESS_MODE_DATA_FIELD, + Jsons.jsonNode(ImmutableMap.of(DOCUMENT_OBJECT_ID_FIELD, Map.of(OBJECT_ID_FIELD, OBJECT_ID)))); + + final JsonNode data = MongoDbCdcEventUtils.normalizeObjectIdNoSchema(objectNode); + assertEquals(OBJECT_ID, data.get(DOCUMENT_OBJECT_ID_FIELD).asText()); + assertEquals(OBJECT_ID, data.get(SCHEMALESS_MODE_DATA_FIELD).get(DOCUMENT_OBJECT_ID_FIELD).asText()); + + objectNode = (ObjectNode) Jsons.jsonNode(Map.of(DOCUMENT_OBJECT_ID_FIELD, Map.of())); + objectNode.set(SCHEMALESS_MODE_DATA_FIELD, Jsons.jsonNode(ImmutableMap.of(DOCUMENT_OBJECT_ID_FIELD, Map.of()))); + final JsonNode dataWithoutObjectId = MongoDbCdcEventUtils.normalizeObjectIdNoSchema(objectNode); + assertNotEquals(OBJECT_ID, dataWithoutObjectId.get(DOCUMENT_OBJECT_ID_FIELD).asText()); + assertNotEquals(OBJECT_ID, dataWithoutObjectId.get(SCHEMALESS_MODE_DATA_FIELD).get(DOCUMENT_OBJECT_ID_FIELD).asText()); + + final JsonNode dataWithoutId = MongoDbCdcEventUtils.normalizeObjectIdNoSchema((ObjectNode) Jsons.jsonNode(Map.of())); + assertNull(dataWithoutId.get(DOCUMENT_OBJECT_ID_FIELD)); + } + + @Test + void testTransformDataTypes() { + final BsonTimestamp bsonTimestamp = new BsonTimestamp(394, 1926745562); + final String expectedTimestamp = DataTypeUtils.toISO8601StringWithMilliseconds(bsonTimestamp.getValue()); + final UUID standardUuid = UUID.randomUUID(); + final UUID legacyUuid = UUID.randomUUID(); + + final Document document = new Document("field1", new BsonBoolean(true)) + .append("field2", new BsonInt32(1)) + .append("field3", new BsonInt64(2)) + .append("field4", new BsonDouble(3.0)) + .append("field5", new BsonDecimal128(new Decimal128(4))) + .append("field6", bsonTimestamp) + .append("field7", new BsonDateTime(bsonTimestamp.getValue())) + .append("field8", new BsonBinary("test".getBytes(Charset.defaultCharset()))) + .append("field9", new BsonSymbol("test2")) + .append("field10", new BsonString("test3")) + .append("field11", new BsonObjectId(new ObjectId(OBJECT_ID))) + .append("field12", new BsonJavaScript("code")) + .append("field13", new BsonJavaScriptWithScope("code2", new BsonDocument("scope", new BsonString("scope")))) + .append("field14", new BsonRegularExpression("pattern")) + .append("field15", new BsonNull()) + .append("field16", new Document("key", "value")) + .append("field17", new BsonBinary(standardUuid, UuidRepresentation.STANDARD)) + .append("field18", new BsonBinary(legacyUuid, UuidRepresentation.JAVA_LEGACY)); + + final String documentAsJson = document.toJson(); + final ObjectNode transformed = MongoDbCdcEventUtils.transformDataTypes(documentAsJson, document.keySet()); + + assertNotNull(transformed); + assertNotEquals(documentAsJson, Jsons.serialize(transformed)); + assertEquals(true, transformed.get("field1").asBoolean()); + assertEquals(1, transformed.get("field2").asInt()); + assertEquals(2, transformed.get("field3").asInt()); + assertEquals(3.0, transformed.get("field4").asDouble()); + assertEquals(4.0, transformed.get("field5").asDouble()); + assertEquals(expectedTimestamp, transformed.get("field6").asText()); + assertEquals(expectedTimestamp, transformed.get("field7").asText()); + assertEquals(Base64.encodeBase64String("test".getBytes(Charset.defaultCharset())), transformed.get("field8").asText()); + assertEquals("test2", transformed.get("field9").asText()); + assertEquals("test3", transformed.get("field10").asText()); + assertEquals(OBJECT_ID, transformed.get("field11").asText()); + assertEquals("code", transformed.get("field12").asText()); + assertEquals("code2", transformed.get("field13").get("code").asText()); + assertEquals("scope", transformed.get("field13").get("scope").get("scope").asText()); + assertEquals("pattern", transformed.get("field14").asText()); + assertFalse(transformed.has("field15")); + assertEquals("value", transformed.get("field16").get("key").asText()); + // Assert that UUIDs can be serialized. Currently, they will be represented as base 64 encoded + // strings. Since the original mongo source + // may have these UUIDs written by a variety of sources, each with different encodings - we cannot + // decode these back to the original UUID. + assertTrue(transformed.has("field17")); + assertTrue(transformed.has("field18")); + } + + @Test + void testTransformDataTypesWithFilteredFields() { + final BsonTimestamp bsonTimestamp = new BsonTimestamp(394, 1926745562); + final String expectedTimestamp = DataTypeUtils.toISO8601StringWithMilliseconds(bsonTimestamp.getValue()); + + final Document document = new Document("field1", new BsonBoolean(true)) + .append("field2", new BsonInt32(1)) + .append("field3", new BsonInt64(2)) + .append("field4", new BsonDouble(3.0)) + .append("field5", new BsonDecimal128(new Decimal128(4))) + .append("field6", bsonTimestamp) + .append("field7", new BsonDateTime(bsonTimestamp.getValue())) + .append("field8", new BsonBinary("test".getBytes(Charset.defaultCharset()))) + .append("field9", new BsonSymbol("test2")) + .append("field10", new BsonString("test3")) + .append("field11", new BsonObjectId(new ObjectId(OBJECT_ID))) + .append("field12", new BsonJavaScript("code")) + .append("field13", new BsonJavaScriptWithScope("code2", new BsonDocument("scope", new BsonString("scope")))) + .append("field14", new BsonRegularExpression("pattern")) + .append("field15", new BsonNull()) + .append("field16", new Document("key", "value")); + + final String documentAsJson = document.toJson(); + final ObjectNode transformed = MongoDbCdcEventUtils.transformDataTypes(documentAsJson, Set.of("field1", "field2", "field3")); + + assertNotNull(transformed); + assertNotEquals(documentAsJson, Jsons.serialize(transformed)); + assertEquals(true, transformed.get("field1").asBoolean()); + assertEquals(1, transformed.get("field2").asInt()); + assertEquals(2, transformed.get("field3").asInt()); + assertFalse(transformed.has("field4")); + assertFalse(transformed.has("field5")); + assertFalse(transformed.has("field6")); + assertFalse(transformed.has("field7")); + assertFalse(transformed.has("field8")); + assertFalse(transformed.has("field9")); + assertFalse(transformed.has("field10")); + assertFalse(transformed.has("field11")); + assertFalse(transformed.has("field12")); + assertFalse(transformed.has("field13")); + assertFalse(transformed.has("field14")); + assertFalse(transformed.has("field15")); + assertFalse(transformed.has("field16")); + } + + @Test + void testTransformDataTypesNoSchema() { + final BsonTimestamp bsonTimestamp = new BsonTimestamp(394, 1926745562); + final String expectedTimestamp = DataTypeUtils.toISO8601StringWithMilliseconds(bsonTimestamp.getValue()); + + final Document document = new Document("field1", new BsonBoolean(true)) + .append("field2", new BsonInt32(1)) + .append("field3", new BsonInt64(2)) + .append("field4", new BsonDouble(3.0)) + .append("field5", new BsonDecimal128(new Decimal128(4))) + .append("field6", bsonTimestamp) + .append("field7", new BsonDateTime(bsonTimestamp.getValue())) + .append("field8", new BsonBinary("test".getBytes(Charset.defaultCharset()))) + .append("field9", new BsonSymbol("test2")) + .append("field10", new BsonString("test3")) + .append("field11", new BsonObjectId(new ObjectId(OBJECT_ID))) + .append("field12", new BsonJavaScript("code")) + .append("field13", new BsonJavaScriptWithScope("code2", new BsonDocument("scope", new BsonString("scope")))) + .append("field14", new BsonRegularExpression("pattern")) + .append("field15", new BsonNull()) + .append("field16", new Document("key", "value")); + + final String documentAsJson = document.toJson(); + final ObjectNode transformed = MongoDbCdcEventUtils.transformDataTypesNoSchema(documentAsJson); + + assertNotNull(transformed); + final var abDataNode = transformed.get(SCHEMALESS_MODE_DATA_FIELD); + assertNotEquals(documentAsJson, Jsons.serialize(abDataNode)); + assertEquals(true, abDataNode.get("field1").asBoolean()); + assertEquals(1, abDataNode.get("field2").asInt()); + assertEquals(2, abDataNode.get("field3").asInt()); + assertEquals(3.0, abDataNode.get("field4").asDouble()); + assertEquals(4.0, abDataNode.get("field5").asDouble()); + assertTrue(abDataNode.has("field6")); + assertTrue(abDataNode.has("field7")); + assertTrue(abDataNode.has("field8")); + assertTrue(abDataNode.has("field9")); + assertTrue(abDataNode.has("field10")); + assertTrue(abDataNode.has("field11")); + assertTrue(abDataNode.has("field12")); + assertTrue(abDataNode.has("field13")); + assertTrue(abDataNode.has("field14")); + assertFalse(abDataNode.has("field15")); + assertTrue(abDataNode.has("field16")); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcTargetPositionTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcTargetPositionTest.java new file mode 100644 index 000000000000..91b5c2334aca --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCdcTargetPositionTest.java @@ -0,0 +1,237 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals.mongodb; + +import static com.mongodb.assertions.Assertions.assertNotNull; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcEventUtils.ID_FIELD; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcEventUtils.OBJECT_ID_FIELD; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.mongodb.client.ChangeStreamIterable; +import com.mongodb.client.MongoChangeStreamCursor; +import com.mongodb.client.MongoClient; +import com.mongodb.client.model.changestream.ChangeStreamDocument; +import io.airbyte.cdk.integrations.debezium.internals.ChangeEventWithMetadata; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.protocol.models.Jsons; +import io.debezium.connector.mongodb.ResumeTokens; +import io.debezium.engine.ChangeEvent; +import java.io.IOException; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.bson.BsonDocument; +import org.bson.BsonTimestamp; +import org.junit.jupiter.api.Test; + +class MongoDbCdcTargetPositionTest { + + private static final String OBJECT_ID = "64f24244f95155351c4185b1"; + private static final String RESUME_TOKEN = "8264BEB9F3000000012B0229296E04"; + private static final String OTHER_RESUME_TOKEN = "8264BEB9F3000000012B0229296E05"; + + @Test + void testCreateTargetPosition() { + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + + final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); + assertNotNull(targetPosition); + assertEquals(ResumeTokens.getTimestamp(resumeTokenDocument), targetPosition.getResumeTokenTimestamp()); + } + + @Test + void testReachedTargetPosition() throws IOException { + final String changeEventJson = MoreResources.readResource("mongodb/change_event.json"); + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + final ChangeEvent changeEvent = mock(ChangeEvent.class); + + when(changeEvent.key()).thenReturn("{\"" + ID_FIELD + "\":\"{\\\"" + OBJECT_ID_FIELD + "\\\": \\\"" + OBJECT_ID + "\\\"}\"}"); + when(changeEvent.value()).thenReturn(changeEventJson); + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + + final ChangeEventWithMetadata changeEventWithMetadata = new ChangeEventWithMetadata(changeEvent); + final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); + assertTrue(targetPosition.reachedTargetPosition(changeEventWithMetadata)); + + when(changeEvent.value()).thenReturn(changeEventJson.replaceAll("\"ts_ms\": \\d+,", "\"ts_ms\": 1590221043000,")); + final ChangeEventWithMetadata changeEventWithMetadata2 = new ChangeEventWithMetadata(changeEvent); + assertFalse(targetPosition.reachedTargetPosition(changeEventWithMetadata2)); + } + + @Test + void testReachedTargetPositionSnapshotEvent() throws IOException { + final String changeEventJson = MoreResources.readResource("mongodb/change_event_snapshot.json"); + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + final ChangeEvent changeEvent = mock(ChangeEvent.class); + + when(changeEvent.key()).thenReturn("{\"" + ID_FIELD + "\":\"{\\\"" + OBJECT_ID_FIELD + "\\\": \\\"" + OBJECT_ID + "\\\"}\"}"); + when(changeEvent.value()).thenReturn(changeEventJson); + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + + final ChangeEventWithMetadata changeEventWithMetadata = new ChangeEventWithMetadata(changeEvent); + final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); + assertFalse(targetPosition.reachedTargetPosition(changeEventWithMetadata)); + } + + @Test + void testReachedTargetPositionSnapshotLastEvent() throws IOException { + final String changeEventJson = MoreResources.readResource("mongodb/change_event_snapshot_last.json"); + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + final ChangeEvent changeEvent = mock(ChangeEvent.class); + + when(changeEvent.key()).thenReturn("{\"" + ID_FIELD + "\":\"{\\\"" + OBJECT_ID_FIELD + "\\\": \\\"" + OBJECT_ID + "\\\"}\"}"); + when(changeEvent.value()).thenReturn(changeEventJson); + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + + final ChangeEventWithMetadata changeEventWithMetadata = new ChangeEventWithMetadata(changeEvent); + final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); + assertTrue(targetPosition.reachedTargetPosition(changeEventWithMetadata)); + } + + @Test + void testReachedTargetPositionFromHeartbeat() { + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + + final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); + final BsonTimestamp heartbeatTimestamp = new BsonTimestamp( + Long.valueOf(ResumeTokens.getTimestamp(resumeTokenDocument).getTime() + TimeUnit.HOURS.toSeconds(1)).intValue(), + 0); + + assertTrue(targetPosition.reachedTargetPosition(heartbeatTimestamp)); + assertFalse(targetPosition.reachedTargetPosition((BsonTimestamp) null)); + } + + @Test + void testIsHeartbeatSupported() { + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + + final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); + + assertTrue(targetPosition.isHeartbeatSupported()); + } + + @Test + void testExtractPositionFromHeartbeatOffset() { + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final BsonTimestamp resumeTokenTimestamp = ResumeTokens.getTimestamp(resumeTokenDocument); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + + final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); + + final Map sourceOffset = Map.of(MongoDbDebeziumConstants.ChangeEvent.SOURCE_SECONDS, resumeTokenTimestamp.getTime(), + MongoDbDebeziumConstants.ChangeEvent.SOURCE_ORDER, resumeTokenTimestamp.getInc(), + MongoDbDebeziumConstants.ChangeEvent.SOURCE_RESUME_TOKEN, RESUME_TOKEN); + + final BsonTimestamp timestamp = targetPosition.extractPositionFromHeartbeatOffset(sourceOffset); + assertEquals(resumeTokenTimestamp, timestamp); + } + + @Test + void testIsEventAheadOfOffset() throws IOException { + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + final String changeEventJson = MoreResources.readResource("mongodb/change_event.json"); + final ChangeEvent changeEvent = mock(ChangeEvent.class); + + when(changeEvent.key()).thenReturn("{\"" + ID_FIELD + "\":\"{\\\"" + OBJECT_ID_FIELD + "\\\": \\\"" + OBJECT_ID + "\\\"}\"}"); + when(changeEvent.value()).thenReturn(changeEventJson); + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + + final ChangeEventWithMetadata changeEventWithMetadata = new ChangeEventWithMetadata(changeEvent); + final Map offset = + Jsons.object(MongoDbDebeziumStateUtil.formatState(null, null, RESUME_TOKEN), new TypeReference<>() {}); + + final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); + final boolean result = targetPosition.isEventAheadOffset(offset, changeEventWithMetadata); + assertTrue(result); + } + + @Test + void testIsSameOffset() { + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + + final Map offsetA = + Jsons.object(MongoDbDebeziumStateUtil.formatState(null, null, RESUME_TOKEN), new TypeReference<>() {}); + final Map offsetB = + Jsons.object(MongoDbDebeziumStateUtil.formatState(null, null, RESUME_TOKEN), new TypeReference<>() {}); + final Map offsetC = + Jsons.object(MongoDbDebeziumStateUtil.formatState(null, null, OTHER_RESUME_TOKEN), new TypeReference<>() {}); + + final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); + + assertTrue(targetPosition.isSameOffset(offsetA, offsetA)); + assertTrue(targetPosition.isSameOffset(offsetA, offsetB)); + assertTrue(targetPosition.isSameOffset(offsetB, offsetA)); + assertFalse(targetPosition.isSameOffset(offsetA, offsetC)); + assertFalse(targetPosition.isSameOffset(offsetB, offsetC)); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCustomLoaderTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCustomLoaderTest.java new file mode 100644 index 000000000000..62b9fecf1f07 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbCustomLoaderTest.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals.mongodb; + +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.ChangeEvent.SOURCE_ORDER; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.ChangeEvent.SOURCE_RESUME_TOKEN; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.ChangeEvent.SOURCE_SECONDS; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.OffsetState.KEY_REPLICA_SET; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.OffsetState.VALUE_TRANSACTION_ID; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.Mockito.mock; + +import com.mongodb.ConnectionString; +import io.debezium.connector.mongodb.MongoDbConnectorConfig; +import io.debezium.connector.mongodb.MongoDbOffsetContext; +import io.debezium.connector.mongodb.ReplicaSets; +import io.debezium.connector.mongodb.ResumeTokens; +import io.debezium.connector.mongodb.connection.ReplicaSet; +import java.util.HashMap; +import java.util.Map; +import org.bson.BsonDocument; +import org.bson.BsonTimestamp; +import org.junit.jupiter.api.Test; + +class MongoDbCustomLoaderTest { + + private static final String RESUME_TOKEN = "8264BEB9F3000000012B0229296E04"; + + @Test + void testLoadOffsets() { + final String replicaSet = "replica-set"; + final BsonDocument resumeToken = ResumeTokens.fromData(RESUME_TOKEN); + final BsonTimestamp timestamp = ResumeTokens.getTimestamp(resumeToken); + final Map key = Map.of(KEY_REPLICA_SET, replicaSet); + final Map value = new HashMap<>(); + value.put(SOURCE_SECONDS, timestamp.getTime()); + value.put(SOURCE_ORDER, timestamp.getInc()); + value.put(SOURCE_RESUME_TOKEN, RESUME_TOKEN); + value.put(VALUE_TRANSACTION_ID, null); + final Map, Map> offsets = Map.of(key, value); + final MongoDbConnectorConfig mongoDbConnectorConfig = mock(MongoDbConnectorConfig.class); + final ReplicaSets replicaSets = ReplicaSets.of( + new ReplicaSet(new ConnectionString("mongodb://localhost:1234/?replicaSet=" + replicaSet))); + final MongoDbCustomLoader loader = new MongoDbCustomLoader(mongoDbConnectorConfig, replicaSets); + + final MongoDbOffsetContext context = loader.loadOffsets(offsets); + final Map offset = context.getReplicaSetOffsetContext(replicaSets.all().get(0)).getOffset(); + + assertNotNull(offset); + assertEquals(value, offset); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumPropertiesManagerTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumPropertiesManagerTest.java new file mode 100644 index 000000000000..aaf7805b3695 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumPropertiesManagerTest.java @@ -0,0 +1,261 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals.mongodb; + +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager.NAME_KEY; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager.TOPIC_PREFIX_KEY; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.AUTH_SOURCE_CONFIGURATION_KEY; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.CONNECTION_STRING_CONFIGURATION_KEY; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.CREDENTIALS_PLACEHOLDER; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.DATABASE_CONFIGURATION_KEY; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.PASSWORD_CONFIGURATION_KEY; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration.USERNAME_CONFIGURATION_KEY; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager.COLLECTION_INCLUDE_LIST_KEY; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager.DATABASE_INCLUDE_LIST_KEY; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager.MONGODB_AUTHSOURCE_KEY; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager.MONGODB_CONNECTION_MODE_KEY; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager.MONGODB_CONNECTION_MODE_VALUE; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager.MONGODB_CONNECTION_STRING_KEY; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager.MONGODB_PASSWORD_KEY; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager.MONGODB_SSL_ENABLED_KEY; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager.MONGODB_SSL_ENABLED_VALUE; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager.MONGODB_USER_KEY; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteFileOffsetBackingStore; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteSchemaHistoryStorage; +import io.airbyte.commons.json.Jsons; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import org.junit.jupiter.api.Test; + +class MongoDbDebeziumPropertiesManagerTest { + + private static final String DATABASE_NAME = "test_database"; + private static final Path PATH = Path.of("."); + public static final String EXPECTED_CONNECTION_STRING = "mongodb://localhost:27017/?retryWrites=false&provider=airbyte&tls=true"; + + @Test + void testDebeziumProperties() { + final List streams = createStreams(4); + final AirbyteFileOffsetBackingStore offsetManager = mock(AirbyteFileOffsetBackingStore.class); + final ConfiguredAirbyteCatalog catalog = mock(ConfiguredAirbyteCatalog.class); + final JsonNode config = createConfiguration(Optional.of("username"), Optional.of("password"), Optional.of("admin")); + + when(offsetManager.getOffsetFilePath()).thenReturn(PATH); + when(catalog.getStreams()).thenReturn(streams); + + final Properties cdcProperties = new Properties(); + cdcProperties.put("test", "value"); + + final MongoDbDebeziumPropertiesManager debeziumPropertiesManager = new MongoDbDebeziumPropertiesManager( + cdcProperties, + config, + catalog, + offsetManager); + + final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(); + assertEquals(22 + cdcProperties.size(), debeziumProperties.size()); + assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(NAME_KEY)); + assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(TOPIC_PREFIX_KEY)); + assertEquals(EXPECTED_CONNECTION_STRING, debeziumProperties.get(MONGODB_CONNECTION_STRING_KEY)); + assertEquals(MONGODB_CONNECTION_MODE_VALUE, debeziumProperties.get(MONGODB_CONNECTION_MODE_KEY)); + assertEquals(config.get(USERNAME_CONFIGURATION_KEY).asText(), debeziumProperties.get(MONGODB_USER_KEY)); + assertEquals(config.get(PASSWORD_CONFIGURATION_KEY).asText(), debeziumProperties.get(MONGODB_PASSWORD_KEY)); + assertEquals(config.get(AUTH_SOURCE_CONFIGURATION_KEY).asText(), debeziumProperties.get(MONGODB_AUTHSOURCE_KEY)); + assertEquals(MONGODB_SSL_ENABLED_VALUE, debeziumProperties.get(MONGODB_SSL_ENABLED_KEY)); + assertEquals(debeziumPropertiesManager.createCollectionIncludeString(streams), debeziumProperties.get(COLLECTION_INCLUDE_LIST_KEY)); + assertEquals(DATABASE_NAME, debeziumProperties.get(DATABASE_INCLUDE_LIST_KEY)); + } + + @Test + void testDebeziumPropertiesConnectionStringCredentialsPlaceholder() { + final List streams = createStreams(4); + final AirbyteFileOffsetBackingStore offsetManager = mock(AirbyteFileOffsetBackingStore.class); + final ConfiguredAirbyteCatalog catalog = mock(ConfiguredAirbyteCatalog.class); + final JsonNode config = createConfiguration(Optional.of("username"), Optional.of("password"), Optional.of("admin")); + ((ObjectNode) config).put(CONNECTION_STRING_CONFIGURATION_KEY, config.get(CONNECTION_STRING_CONFIGURATION_KEY).asText() + .replaceAll("mongodb://", "mongodb://" + CREDENTIALS_PLACEHOLDER)); + + when(offsetManager.getOffsetFilePath()).thenReturn(PATH); + when(catalog.getStreams()).thenReturn(streams); + + final Properties cdcProperties = new Properties(); + cdcProperties.put("test", "value"); + + final MongoDbDebeziumPropertiesManager debeziumPropertiesManager = new MongoDbDebeziumPropertiesManager( + cdcProperties, + config, + catalog, + offsetManager); + + final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(); + assertEquals(22 + cdcProperties.size(), debeziumProperties.size()); + assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(NAME_KEY)); + assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(TOPIC_PREFIX_KEY)); + assertEquals(EXPECTED_CONNECTION_STRING, debeziumProperties.get(MONGODB_CONNECTION_STRING_KEY)); + assertEquals(MONGODB_CONNECTION_MODE_VALUE, debeziumProperties.get(MONGODB_CONNECTION_MODE_KEY)); + assertEquals(config.get(USERNAME_CONFIGURATION_KEY).asText(), debeziumProperties.get(MONGODB_USER_KEY)); + assertEquals(config.get(PASSWORD_CONFIGURATION_KEY).asText(), debeziumProperties.get(MONGODB_PASSWORD_KEY)); + assertEquals(config.get(AUTH_SOURCE_CONFIGURATION_KEY).asText(), debeziumProperties.get(MONGODB_AUTHSOURCE_KEY)); + assertEquals(MONGODB_SSL_ENABLED_VALUE, debeziumProperties.get(MONGODB_SSL_ENABLED_KEY)); + assertEquals(debeziumPropertiesManager.createCollectionIncludeString(streams), debeziumProperties.get(COLLECTION_INCLUDE_LIST_KEY)); + assertEquals(DATABASE_NAME, debeziumProperties.get(DATABASE_INCLUDE_LIST_KEY)); + } + + @Test + void testDebeziumPropertiesQuotedConnectionString() { + final List streams = createStreams(4); + final AirbyteFileOffsetBackingStore offsetManager = mock(AirbyteFileOffsetBackingStore.class); + final ConfiguredAirbyteCatalog catalog = mock(ConfiguredAirbyteCatalog.class); + final JsonNode config = createConfiguration(Optional.of("username"), Optional.of("password"), Optional.of("admin")); + ((ObjectNode) config).put(CONNECTION_STRING_CONFIGURATION_KEY, "\"" + config.get(CONNECTION_STRING_CONFIGURATION_KEY) + "\""); + + when(offsetManager.getOffsetFilePath()).thenReturn(PATH); + when(catalog.getStreams()).thenReturn(streams); + + final Properties cdcProperties = new Properties(); + cdcProperties.put("test", "value"); + + final MongoDbDebeziumPropertiesManager debeziumPropertiesManager = new MongoDbDebeziumPropertiesManager( + cdcProperties, + config, + catalog, + offsetManager); + + final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(); + assertEquals(22 + cdcProperties.size(), debeziumProperties.size()); + assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(NAME_KEY)); + assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(TOPIC_PREFIX_KEY)); + assertEquals(EXPECTED_CONNECTION_STRING, debeziumProperties.get(MONGODB_CONNECTION_STRING_KEY)); + assertEquals(MONGODB_CONNECTION_MODE_VALUE, debeziumProperties.get(MONGODB_CONNECTION_MODE_KEY)); + assertEquals(config.get(USERNAME_CONFIGURATION_KEY).asText(), debeziumProperties.get(MONGODB_USER_KEY)); + assertEquals(config.get(PASSWORD_CONFIGURATION_KEY).asText(), debeziumProperties.get(MONGODB_PASSWORD_KEY)); + assertEquals(config.get(AUTH_SOURCE_CONFIGURATION_KEY).asText(), debeziumProperties.get(MONGODB_AUTHSOURCE_KEY)); + assertEquals(MONGODB_SSL_ENABLED_VALUE, debeziumProperties.get(MONGODB_SSL_ENABLED_KEY)); + assertEquals(debeziumPropertiesManager.createCollectionIncludeString(streams), debeziumProperties.get(COLLECTION_INCLUDE_LIST_KEY)); + assertEquals(DATABASE_NAME, debeziumProperties.get(DATABASE_INCLUDE_LIST_KEY)); + } + + @Test + void testDebeziumPropertiesNoCredentials() { + final List streams = createStreams(4); + final AirbyteFileOffsetBackingStore offsetManager = mock(AirbyteFileOffsetBackingStore.class); + final AirbyteSchemaHistoryStorage schemaHistoryManager = mock(AirbyteSchemaHistoryStorage.class); + final ConfiguredAirbyteCatalog catalog = mock(ConfiguredAirbyteCatalog.class); + final JsonNode config = createConfiguration(Optional.empty(), Optional.empty(), Optional.empty()); + + when(offsetManager.getOffsetFilePath()).thenReturn(PATH); + when(catalog.getStreams()).thenReturn(streams); + + final Properties cdcProperties = new Properties(); + cdcProperties.put("test", "value"); + + final MongoDbDebeziumPropertiesManager debeziumPropertiesManager = new MongoDbDebeziumPropertiesManager( + cdcProperties, + config, + catalog, + offsetManager); + + final Properties debeziumProperties = debeziumPropertiesManager.getDebeziumProperties(); + assertEquals(19 + cdcProperties.size(), debeziumProperties.size()); + assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(NAME_KEY)); + assertEquals(MongoDbDebeziumPropertiesManager.normalizeName(DATABASE_NAME), debeziumProperties.get(TOPIC_PREFIX_KEY)); + assertEquals(EXPECTED_CONNECTION_STRING, debeziumProperties.get(MONGODB_CONNECTION_STRING_KEY)); + assertEquals(MONGODB_CONNECTION_MODE_VALUE, debeziumProperties.get(MONGODB_CONNECTION_MODE_KEY)); + assertFalse(debeziumProperties.containsKey(MONGODB_USER_KEY)); + assertFalse(debeziumProperties.containsKey(MONGODB_PASSWORD_KEY)); + assertFalse(debeziumProperties.containsKey(MONGODB_AUTHSOURCE_KEY)); + assertEquals(MONGODB_SSL_ENABLED_VALUE, debeziumProperties.get(MONGODB_SSL_ENABLED_KEY)); + assertEquals(debeziumPropertiesManager.createCollectionIncludeString(streams), debeziumProperties.get(COLLECTION_INCLUDE_LIST_KEY)); + assertEquals(DATABASE_NAME, debeziumProperties.get(DATABASE_INCLUDE_LIST_KEY)); + } + + @Test + void testNormalizeName() { + final String nameWithUnderscore = "name_with_underscore"; + final String nameWithoutUnderscore = "nameWithout-Underscore"; + final String blankName = ""; + final String nullName = null; + + assertEquals("name-with-underscore", MongoDbDebeziumPropertiesManager.normalizeName(nameWithUnderscore)); + assertEquals(nameWithoutUnderscore, MongoDbDebeziumPropertiesManager.normalizeName(nameWithoutUnderscore)); + assertEquals(blankName, MongoDbDebeziumPropertiesManager.normalizeName(blankName)); + assertNull(MongoDbDebeziumPropertiesManager.normalizeName(nullName)); + + } + + @Test + void testCreateConnectionString() { + final JsonNode config = createConfiguration(Optional.of("username"), Optional.of("password"), Optional.of("admin")); + final String connectionString = MongoDbDebeziumPropertiesManager.buildConnectionString(config, false); + assertNotNull(connectionString); + assertEquals(EXPECTED_CONNECTION_STRING, connectionString); + } + + @Test + void testCreateConnectionStringQuotedString() { + final JsonNode config = createConfiguration(Optional.of("username"), Optional.of("password"), Optional.of("admin")); + final String connectionString = MongoDbDebeziumPropertiesManager.buildConnectionString(config, false); + ((ObjectNode) config).put(CONNECTION_STRING_CONFIGURATION_KEY, "\"" + config.get(CONNECTION_STRING_CONFIGURATION_KEY) + "\""); + assertNotNull(connectionString); + assertEquals(EXPECTED_CONNECTION_STRING, connectionString); + } + + @Test + void testCreateConnectionStringUseSecondary() { + final JsonNode config = createConfiguration(Optional.of("username"), Optional.of("password"), Optional.of("admin")); + final String connectionString = MongoDbDebeziumPropertiesManager.buildConnectionString(config, true); + assertNotNull(connectionString); + assertEquals("mongodb://localhost:27017/?retryWrites=false&provider=airbyte&tls=true&readPreference=secondary", connectionString); + } + + @Test + void testCreateConnectionStringPlaceholderCredentials() { + final JsonNode config = createConfiguration(Optional.of("username"), Optional.of("password"), Optional.of("admin")); + ((ObjectNode) config).put(CONNECTION_STRING_CONFIGURATION_KEY, config.get(CONNECTION_STRING_CONFIGURATION_KEY).asText() + .replaceAll("mongodb://", "mongodb://" + CREDENTIALS_PLACEHOLDER)); + final String connectionString = MongoDbDebeziumPropertiesManager.buildConnectionString(config, false); + assertNotNull(connectionString); + assertEquals(EXPECTED_CONNECTION_STRING, connectionString); + } + + private JsonNode createConfiguration(final Optional username, final Optional password, final Optional authMode) { + final Map baseConfig = Map.of( + DATABASE_CONFIGURATION_KEY, DATABASE_NAME, + CONNECTION_STRING_CONFIGURATION_KEY, "mongodb://localhost:27017/"); + + final Map config = new HashMap<>(baseConfig); + authMode.ifPresent(a -> config.put(AUTH_SOURCE_CONFIGURATION_KEY, a)); + username.ifPresent(u -> config.put(USERNAME_CONFIGURATION_KEY, u)); + password.ifPresent(p -> config.put(PASSWORD_CONFIGURATION_KEY, p)); + return Jsons.deserialize(Jsons.serialize(config)); + } + + private List createStreams(final int numberOfStreams) { + final List streams = new ArrayList<>(); + for (int i = 0; i < numberOfStreams; i++) { + final AirbyteStream stream = new AirbyteStream().withNamespace(DATABASE_NAME).withName("collection" + i); + streams.add(new ConfiguredAirbyteStream().withStream(stream)); + } + return streams; + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtilTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtilTest.java new file mode 100644 index 000000000000..bfb89edc34dd --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtilTest.java @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals.mongodb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.JsonNode; +import com.mongodb.MongoCommandException; +import com.mongodb.ServerAddress; +import com.mongodb.client.ChangeStreamIterable; +import com.mongodb.client.MongoChangeStreamCursor; +import com.mongodb.client.MongoClient; +import com.mongodb.client.model.changestream.ChangeStreamDocument; +import com.mongodb.connection.ClusterDescription; +import com.mongodb.connection.ClusterType; +import com.mongodb.connection.ServerDescription; +import io.airbyte.commons.json.Jsons; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteCatalog; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.SyncMode; +import io.debezium.connector.mongodb.ResumeTokens; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import org.bson.BsonDocument; +import org.bson.BsonTimestamp; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class MongoDbDebeziumStateUtilTest { + + private static final String DATABASE = "test-database"; + private static final String REPLICA_SET = "test-replica-set"; + private static final String RESUME_TOKEN = "8264BEB9F3000000012B0229296E04"; + + private static final AirbyteCatalog CATALOG = new AirbyteCatalog().withStreams(List.of( + CatalogHelpers.createAirbyteStream( + "test-collection", + DATABASE, + Field.of("id", JsonSchemaType.INTEGER), + Field.of("string", JsonSchemaType.STRING)) + .withSupportedSyncModes(List.of(SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of("_id"))))); + protected static final ConfiguredAirbyteCatalog CONFIGURED_CATALOG = CatalogHelpers.toDefaultConfiguredCatalog(CATALOG); + + private MongoDbDebeziumStateUtil mongoDbDebeziumStateUtil; + + @BeforeEach + void setup() { + mongoDbDebeziumStateUtil = new MongoDbDebeziumStateUtil(); + } + + @Test + void testConstructInitialDebeziumState() { + final String database = DATABASE; + final String resumeToken = RESUME_TOKEN; + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(resumeToken); + final ServerDescription serverDescription = mock(ServerDescription.class); + final ClusterDescription clusterDescription = mock(ClusterDescription.class); + final MongoClient mongoClient = mock(MongoClient.class); + final Properties baseProperties = new Properties(); + + final JsonNode config = Jsons.jsonNode(Map.of( + MongoDbDebeziumConstants.Configuration.CONNECTION_STRING_CONFIGURATION_KEY, "mongodb://host:12345/", + MongoDbDebeziumConstants.Configuration.DATABASE_CONFIGURATION_KEY, database)); + + when(serverDescription.getSetName()).thenReturn(REPLICA_SET); + when(clusterDescription.getServerDescriptions()).thenReturn(List.of(serverDescription)); + when(clusterDescription.getType()).thenReturn(ClusterType.REPLICA_SET); + when(mongoClient.getClusterDescription()).thenReturn(clusterDescription); + + final JsonNode initialState = mongoDbDebeziumStateUtil.constructInitialDebeziumState(resumeTokenDocument, mongoClient, database); + + assertNotNull(initialState); + assertEquals(1, initialState.size()); + final BsonTimestamp timestamp = ResumeTokens.getTimestamp(resumeTokenDocument); + final JsonNode offsetState = initialState.fields().next().getValue(); + assertEquals(resumeToken, Jsons.deserialize(offsetState.asText()).get(MongoDbDebeziumConstants.OffsetState.VALUE_RESUME_TOKEN).asText()); + assertEquals(timestamp.getTime(), Jsons.deserialize(offsetState.asText()).get(MongoDbDebeziumConstants.OffsetState.VALUE_SECONDS).asInt()); + assertEquals(timestamp.getInc(), Jsons.deserialize(offsetState.asText()).get(MongoDbDebeziumConstants.OffsetState.VALUE_INCREMENT).asInt()); + assertEquals("null", Jsons.deserialize(offsetState.asText()).get(MongoDbDebeziumConstants.OffsetState.VALUE_TRANSACTION_ID).asText()); + + final Optional parsedOffset = + mongoDbDebeziumStateUtil.savedOffset( + baseProperties, + CONFIGURED_CATALOG, + initialState, + config, + mongoClient); + assertTrue(parsedOffset.isPresent()); + assertEquals(resumeToken, parsedOffset.get().get("_data").asString().getValue()); + } + + @Test + void testConstructInitialDebeziumStateMissingReplicaSet() { + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); + final ServerDescription serverDescription = mock(ServerDescription.class); + final ClusterDescription clusterDescription = mock(ClusterDescription.class); + final MongoClient mongoClient = mock(MongoClient.class); + + when(clusterDescription.getServerDescriptions()).thenReturn(List.of(serverDescription)); + when(clusterDescription.getType()).thenReturn(ClusterType.REPLICA_SET); + when(mongoClient.getClusterDescription()).thenReturn(clusterDescription); + + assertThrows(IllegalStateException.class, + () -> mongoDbDebeziumStateUtil.constructInitialDebeziumState(resumeTokenDocument, mongoClient, DATABASE)); + } + + @Test + void testOffsetDataFormat() { + final JsonNode offsetState = MongoDbDebeziumStateUtil.formatState(DATABASE, REPLICA_SET, RESUME_TOKEN); + + assertNotNull(offsetState); + assertEquals("[\"" + DATABASE + "\",{\"" + MongoDbDebeziumConstants.OffsetState.KEY_REPLICA_SET + "\":\"" + REPLICA_SET + "\",\"" + + MongoDbDebeziumConstants.OffsetState.KEY_SERVER_ID + "\":\"" + DATABASE + "\"}]", offsetState.fieldNames().next()); + } + + @Test + void testIsResumeTokenValid() { + final BsonDocument resumeToken = ResumeTokens.fromData(RESUME_TOKEN); + + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeToken); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(changeStreamIterable.resumeAfter(resumeToken)).thenReturn(changeStreamIterable); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + + assertTrue(mongoDbDebeziumStateUtil.isValidResumeToken(resumeToken, mongoClient)); + } + + @Test + void testIsResumeTokenInvalid() { + final BsonDocument resumeToken = ResumeTokens.fromData(RESUME_TOKEN); + + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeToken); + when(changeStreamIterable.cursor()).thenThrow(new MongoCommandException(new BsonDocument(), new ServerAddress())); + when(changeStreamIterable.resumeAfter(resumeToken)).thenReturn(changeStreamIterable); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + + assertFalse(mongoDbDebeziumStateUtil.isValidResumeToken(resumeToken, mongoClient)); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelperTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelperTest.java new file mode 100644 index 000000000000..98cde65bf61d --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelperTest.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debezium.internals.mongodb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.JsonNode; +import com.mongodb.client.ChangeStreamIterable; +import com.mongodb.client.MongoChangeStreamCursor; +import com.mongodb.client.MongoClient; +import com.mongodb.client.model.changestream.ChangeStreamDocument; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.debezium.connector.mongodb.ResumeTokens; +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import org.bson.BsonDocument; +import org.bson.BsonTimestamp; +import org.junit.jupiter.api.Test; + +class MongoDbResumeTokenHelperTest { + + @Test + void testRetrievingResumeToken() { + final String resumeToken = "8264BEB9F3000000012B0229296E04"; + final BsonDocument resumeTokenDocument = ResumeTokens.fromData(resumeToken); + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor> mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + final MongoClient mongoClient = mock(MongoClient.class); + + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + + final BsonDocument actualResumeToken = MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient); + assertEquals(resumeTokenDocument, actualResumeToken); + } + + @Test + void testTimestampExtractionFromEvent() throws IOException { + final int timestampSec = Long.valueOf(TimeUnit.MILLISECONDS.toSeconds(1692651270000L)).intValue(); + final BsonTimestamp expectedTimestamp = new BsonTimestamp(timestampSec, 2); + final String changeEventJson = MoreResources.readResource("mongodb/change_event.json"); + final JsonNode changeEvent = Jsons.deserialize(changeEventJson); + final BsonTimestamp timestamp = MongoDbResumeTokenHelper.extractTimestampFromEvent(changeEvent); + assertNotNull(timestamp); + assertEquals(expectedTimestamp, timestamp); + } + + @Test + void testTimestampExtractionFromEventSource() throws IOException { + final int timestampSec = Long.valueOf(TimeUnit.MILLISECONDS.toSeconds(1692651270000L)).intValue(); + final BsonTimestamp expectedTimestamp = new BsonTimestamp(timestampSec, 2); + final String changeEventJson = MoreResources.readResource("mongodb/change_event.json"); + final JsonNode changeEvent = Jsons.deserialize(changeEventJson); + + final BsonTimestamp timestamp = MongoDbResumeTokenHelper + .extractTimestampFromSource(changeEvent.get(MongoDbDebeziumConstants.ChangeEvent.SOURCE)); + assertNotNull(timestamp); + assertEquals(expectedTimestamp, timestamp); + } + + @Test + void testTimestampExtractionFromEventSourceNotPresent() { + final JsonNode changeEvent = Jsons.deserialize("{}"); + assertThrows(IllegalStateException.class, () -> MongoDbResumeTokenHelper.extractTimestampFromEvent(changeEvent)); + assertThrows(IllegalStateException.class, () -> MongoDbResumeTokenHelper.extractTimestampFromSource(changeEvent)); + } + + @Test + void testTimestampExtractionTimestampNotPresent() { + final JsonNode changeEvent = Jsons.deserialize("{\"source\":{}}"); + assertThrows(IllegalStateException.class, () -> MongoDbResumeTokenHelper.extractTimestampFromEvent(changeEvent)); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/jdbc/DefaultJdbcSourceAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/jdbc/DefaultJdbcSourceAcceptanceTest.java new file mode 100644 index 000000000000..5d4dcb3e68d0 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/jdbc/DefaultJdbcSourceAcceptanceTest.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.source.jdbc; + +import static io.airbyte.cdk.integrations.source.jdbc.JdbcDataSourceUtils.assertCustomParametersDontOverwriteDefaultParameters; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import io.airbyte.cdk.integrations.util.HostPortResolver; +import io.airbyte.cdk.testutils.TestDatabase; +import io.airbyte.commons.json.Jsons; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import java.sql.JDBCType; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; +import org.jooq.SQLDialect; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.PostgreSQLContainer; + +/** + * Runs the acceptance tests in the source-jdbc test module. We want this module to run these tests + * itself as a sanity check. The trade off here is that this class is duplicated from the one used + * in source-postgres. + */ +class DefaultJdbcSourceAcceptanceTest + extends JdbcSourceAcceptanceTest { + + private static PostgreSQLContainer PSQL_CONTAINER; + + @BeforeAll + static void init() { + PSQL_CONTAINER = new PostgreSQLContainer<>("postgres:13-alpine"); + PSQL_CONTAINER.start(); + CREATE_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s BIT(3) NOT NULL);"; + INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES(B'101');"; + } + + @Override + protected JsonNode config() { + return testdb.testConfigBuilder().build(); + } + + @Override + protected PostgresTestSource source() { + return new PostgresTestSource(); + } + + @Override + protected BareBonesTestDatabase createTestDatabase() { + return new BareBonesTestDatabase(PSQL_CONTAINER).initialized(); + } + + @Override + public boolean supportsSchemas() { + return true; + } + + public JsonNode getConfigWithConnectionProperties(final PostgreSQLContainer psqlDb, final String dbName, final String additionalParameters) { + return Jsons.jsonNode(ImmutableMap.builder() + .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(psqlDb)) + .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(psqlDb)) + .put(JdbcUtils.DATABASE_KEY, dbName) + .put(JdbcUtils.SCHEMAS_KEY, List.of(SCHEMA_NAME)) + .put(JdbcUtils.USERNAME_KEY, psqlDb.getUsername()) + .put(JdbcUtils.PASSWORD_KEY, psqlDb.getPassword()) + .put(JdbcUtils.CONNECTION_PROPERTIES_KEY, additionalParameters) + .build()); + } + + @AfterAll + static void cleanUp() { + PSQL_CONTAINER.close(); + } + + public static class PostgresTestSource extends AbstractJdbcSource implements Source { + + private static final Logger LOGGER = LoggerFactory.getLogger(PostgresTestSource.class); + + static final String DRIVER_CLASS = DatabaseDriver.POSTGRESQL.getDriverClassName(); + + public PostgresTestSource() { + super(DRIVER_CLASS, AdaptiveStreamingQueryConfig::new, JdbcUtils.getDefaultSourceOperations()); + } + + @Override + public JsonNode toDatabaseConfig(final JsonNode config) { + final ImmutableMap.Builder configBuilder = ImmutableMap.builder() + .put(JdbcUtils.USERNAME_KEY, config.get(JdbcUtils.USERNAME_KEY).asText()) + .put(JdbcUtils.JDBC_URL_KEY, String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), + config.get(JdbcUtils.HOST_KEY).asText(), + config.get(JdbcUtils.PORT_KEY).asInt(), + config.get(JdbcUtils.DATABASE_KEY).asText())); + + if (config.has(JdbcUtils.PASSWORD_KEY)) { + configBuilder.put(JdbcUtils.PASSWORD_KEY, config.get(JdbcUtils.PASSWORD_KEY).asText()); + } + + return Jsons.jsonNode(configBuilder.build()); + } + + @Override + public Set getExcludedInternalNameSpaces() { + return Set.of("information_schema", "pg_catalog", "pg_internal", "catalog_history"); + } + + @Override + protected AirbyteStateType getSupportedStateType(final JsonNode config) { + return AirbyteStateType.STREAM; + } + + public static void main(final String[] args) throws Exception { + final Source source = new PostgresTestSource(); + LOGGER.info("starting source: {}", PostgresTestSource.class); + new IntegrationRunner(source).run(args); + LOGGER.info("completed source: {}", PostgresTestSource.class); + } + + } + + static protected class BareBonesTestDatabase + extends TestDatabase, BareBonesTestDatabase, BareBonesTestDatabase.BareBonesConfigBuilder> { + + public BareBonesTestDatabase(PostgreSQLContainer container) { + super(container); + } + + @Override + protected Stream> inContainerBootstrapCmd() { + final var sql = Stream.of( + String.format("CREATE DATABASE %s", getDatabaseName()), + String.format("CREATE USER %s PASSWORD '%s'", getUserName(), getPassword()), + String.format("GRANT ALL PRIVILEGES ON DATABASE %s TO %s", getDatabaseName(), getUserName()), + String.format("ALTER USER %s WITH SUPERUSER", getUserName())); + return Stream.of(Stream.concat( + Stream.of("psql", + "-d", getContainer().getDatabaseName(), + "-U", getContainer().getUsername(), + "-v", "ON_ERROR_STOP=1", + "-a"), + sql.flatMap(stmt -> Stream.of("-c", stmt)))); + } + + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return DatabaseDriver.POSTGRESQL; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.POSTGRES; + } + + @Override + public BareBonesConfigBuilder configBuilder() { + return new BareBonesConfigBuilder(this); + } + + static protected class BareBonesConfigBuilder extends TestDatabase.ConfigBuilder { + + private BareBonesConfigBuilder(BareBonesTestDatabase testDatabase) { + super(testDatabase); + } + + } + + } + + @Test + void testCustomParametersOverwriteDefaultParametersExpectException() { + final String connectionPropertiesUrl = "ssl=false"; + final JsonNode config = getConfigWithConnectionProperties(PSQL_CONTAINER, testdb.getDatabaseName(), connectionPropertiesUrl); + final Map customParameters = JdbcUtils.parseJdbcParameters(config, JdbcUtils.CONNECTION_PROPERTIES_KEY, "&"); + final Map defaultParameters = Map.of( + "ssl", "true", + "sslmode", "require"); + assertThrows(IllegalArgumentException.class, () -> { + assertCustomParametersDontOverwriteDefaultParameters(customParameters, defaultParameters); + }); + } + +} diff --git a/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/DefaultJdbcStressTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/jdbc/DefaultJdbcStressTest.java similarity index 90% rename from airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/DefaultJdbcStressTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/jdbc/DefaultJdbcStressTest.java index 634a62b09e0a..b8d6b88f23ef 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/DefaultJdbcStressTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/jdbc/DefaultJdbcStressTest.java @@ -2,20 +2,20 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.jdbc; +package io.airbyte.cdk.integrations.source.jdbc; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.source.jdbc.test.JdbcStressTest; +import io.airbyte.cdk.testutils.PostgreSQLContainerHelper; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.source.jdbc.test.JdbcStressTest; -import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.sql.JDBCType; import java.util.Optional; import java.util.Set; diff --git a/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/JdbcDataSourceUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/jdbc/JdbcDataSourceUtilsTest.java similarity index 97% rename from airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/JdbcDataSourceUtilsTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/jdbc/JdbcDataSourceUtilsTest.java index 1128a4819cba..116d122d7d31 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/JdbcDataSourceUtilsTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/jdbc/JdbcDataSourceUtilsTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.jdbc; +package io.airbyte.cdk.integrations.source.jdbc; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; import static org.junit.Assert.assertTrue; diff --git a/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/JdbcSourceStressTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/jdbc/JdbcSourceStressTest.java similarity index 90% rename from airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/JdbcSourceStressTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/jdbc/JdbcSourceStressTest.java index 1d1be1765e33..001f0b62b9c4 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/JdbcSourceStressTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/jdbc/JdbcSourceStressTest.java @@ -2,20 +2,20 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.jdbc; +package io.airbyte.cdk.integrations.source.jdbc; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.source.jdbc.test.JdbcStressTest; +import io.airbyte.cdk.testutils.PostgreSQLContainerHelper; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.source.jdbc.test.JdbcStressTest; -import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.sql.JDBCType; import java.util.Optional; import java.util.Set; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSourceTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSourceTest.java new file mode 100644 index 000000000000..9e7bab7177f2 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/AbstractDbSourceTest.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.source.relationaldb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.withSettings; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateGeneratorUtils; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import java.io.IOException; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; +import uk.org.webcompere.systemstubs.jupiter.SystemStub; +import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; + +/** + * Test suite for the {@link AbstractDbSource} class. + */ +@ExtendWith(SystemStubsExtension.class) +public class AbstractDbSourceTest { + + @SystemStub + private EnvironmentVariables environmentVariables; + + @Test + void testDeserializationOfLegacyState() throws IOException { + final AbstractDbSource dbSource = mock(AbstractDbSource.class, withSettings().useConstructor("").defaultAnswer(CALLS_REAL_METHODS)); + final JsonNode config = mock(JsonNode.class); + + final String legacyStateJson = MoreResources.readResource("states/legacy.json"); + final JsonNode legacyState = Jsons.deserialize(legacyStateJson); + + final List result = StateGeneratorUtils.deserializeInitialState(legacyState, + dbSource.getSupportedStateType(config)); + assertEquals(1, result.size()); + assertEquals(AirbyteStateType.LEGACY, result.get(0).getType()); + } + + @Test + void testDeserializationOfGlobalState() throws IOException { + final AbstractDbSource dbSource = mock(AbstractDbSource.class, withSettings().useConstructor("").defaultAnswer(CALLS_REAL_METHODS)); + final JsonNode config = mock(JsonNode.class); + + final String globalStateJson = MoreResources.readResource("states/global.json"); + final JsonNode globalState = Jsons.deserialize(globalStateJson); + + final List result = + StateGeneratorUtils.deserializeInitialState(globalState, dbSource.getSupportedStateType(config)); + assertEquals(1, result.size()); + assertEquals(AirbyteStateType.GLOBAL, result.get(0).getType()); + } + + @Test + void testDeserializationOfStreamState() throws IOException { + final AbstractDbSource dbSource = mock(AbstractDbSource.class, withSettings().useConstructor("").defaultAnswer(CALLS_REAL_METHODS)); + final JsonNode config = mock(JsonNode.class); + + final String streamStateJson = MoreResources.readResource("states/per_stream.json"); + final JsonNode streamState = Jsons.deserialize(streamStateJson); + + final List result = + StateGeneratorUtils.deserializeInitialState(streamState, dbSource.getSupportedStateType(config)); + assertEquals(2, result.size()); + assertEquals(AirbyteStateType.STREAM, result.get(0).getType()); + } + + @Test + void testDeserializationOfNullState() throws IOException { + final AbstractDbSource dbSource = mock(AbstractDbSource.class, withSettings().useConstructor("").defaultAnswer(CALLS_REAL_METHODS)); + final JsonNode config = mock(JsonNode.class); + + final List result = StateGeneratorUtils.deserializeInitialState(null, dbSource.getSupportedStateType(config)); + assertEquals(1, result.size()); + assertEquals(dbSource.getSupportedStateType(config), result.get(0).getType()); + } + +} diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/StateDecoratingIteratorTest.java similarity index 98% rename from airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/StateDecoratingIteratorTest.java index d4204fd3ca9b..e2d64f849748 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/StateDecoratingIteratorTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb; +package io.airbyte.cdk.integrations.source.relationaldb; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -11,14 +11,15 @@ import static org.mockito.Mockito.when; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.integrations.source.relationaldb.state.StateManager; import io.airbyte.protocol.models.JsonSchemaPrimitiveUtil.JsonSchemaPrimitive; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateStats; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import java.sql.SQLException; import java.util.Collections; @@ -69,7 +70,8 @@ private static AirbyteMessage createStateMessage(final String recordValue) { return new AirbyteMessage() .withType(Type.STATE) .withState(new AirbyteStateMessage() - .withData(Jsons.jsonNode(ImmutableMap.of("cursor", recordValue)))); + .withData(Jsons.jsonNode(ImmutableMap.of("cursor", recordValue))) + .withSourceStats(new AirbyteStateStats().withRecordCount(1.0))); } private Iterator createExceptionIterator() { diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/CursorManagerTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/CursorManagerTest.java similarity index 84% rename from airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/CursorManagerTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/CursorManagerTest.java index c1e6c9968552..9f5dccbed7fc 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/CursorManagerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/CursorManagerTest.java @@ -2,21 +2,21 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb.state; - -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.CURSOR; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD1; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD2; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.CURSOR_RECORD_COUNT; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.NAME_NAMESPACE_PAIR1; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.NAME_NAMESPACE_PAIR2; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.getCatalog; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.getState; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.getStream; +package io.airbyte.cdk.integrations.source.relationaldb.state; + +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.CURSOR; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD1; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD2; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.CURSOR_RECORD_COUNT; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.NAME_NAMESPACE_PAIR1; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.NAME_NAMESPACE_PAIR2; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.getCatalog; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.getState; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.getStream; import static org.junit.jupiter.api.Assertions.assertEquals; -import io.airbyte.integrations.source.relationaldb.CursorInfo; -import io.airbyte.integrations.source.relationaldb.models.DbStreamState; +import io.airbyte.cdk.integrations.source.relationaldb.CursorInfo; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import java.util.Collections; import java.util.Optional; diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/GlobalStateManagerTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/GlobalStateManagerTest.java similarity index 91% rename from airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/GlobalStateManagerTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/GlobalStateManagerTest.java index eda20bb17b6c..beee9c73aa89 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/GlobalStateManagerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/GlobalStateManagerTest.java @@ -2,24 +2,24 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb.state; - -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD1; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD2; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.NAMESPACE; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.NAME_NAMESPACE_PAIR1; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME1; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME2; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME3; +package io.airbyte.cdk.integrations.source.relationaldb.state; + +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD1; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD2; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.NAMESPACE; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.NAME_NAMESPACE_PAIR1; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME1; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME2; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME3; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; +import io.airbyte.cdk.integrations.source.relationaldb.models.CdcState; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbState; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.source.relationaldb.models.CdcState; -import io.airbyte.integrations.source.relationaldb.models.DbState; -import io.airbyte.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.protocol.models.v0.AirbyteGlobalState; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; @@ -35,6 +35,7 @@ import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; /** @@ -142,6 +143,9 @@ void testToStateFromLegacyState() { assertEquals(expected, actualFirstEmission); } + // Discovered during CDK migration. + // Failure is: Could not find cursor information for stream: public_cars + @Disabled("Failing test.") @Test void testToState() { final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/LegacyStateManagerTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/LegacyStateManagerTest.java similarity index 88% rename from airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/LegacyStateManagerTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/LegacyStateManagerTest.java index c8cc37c12378..25214d1c7701 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/LegacyStateManagerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/LegacyStateManagerTest.java @@ -2,25 +2,25 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb.state; - -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.CURSOR; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD1; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD2; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.NAMESPACE; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.NAME_NAMESPACE_PAIR1; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.NAME_NAMESPACE_PAIR2; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME1; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME2; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME3; +package io.airbyte.cdk.integrations.source.relationaldb.state; + +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.CURSOR; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD1; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD2; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.NAMESPACE; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.NAME_NAMESPACE_PAIR1; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.NAME_NAMESPACE_PAIR2; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME1; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME2; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME3; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.mockito.Mockito.mock; +import io.airbyte.cdk.integrations.source.relationaldb.models.CdcState; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbState; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.source.relationaldb.models.CdcState; -import io.airbyte.integrations.source.relationaldb.models.DbState; -import io.airbyte.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; import io.airbyte.protocol.models.v0.AirbyteStream; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/SourceStateIteratorTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/SourceStateIteratorTest.java new file mode 100644 index 000000000000..7a32b03607b3 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/SourceStateIteratorTest.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.source.relationaldb.state; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateStats; +import java.util.Iterator; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class SourceStateIteratorTest { + + SourceStateIteratorManager mockProcessor; + Iterator messageIterator; + + SourceStateIterator sourceStateIterator; + + @BeforeEach + void setup() { + mockProcessor = mock(SourceStateIteratorManager.class); + messageIterator = mock(Iterator.class); + sourceStateIterator = new SourceStateIterator(messageIterator, mockProcessor); + } + + // Provides a way to generate a record message and will verify corresponding spied functions have + // been called. + void processRecordMessage() { + doReturn(true).when(messageIterator).hasNext(); + doReturn(false).when(mockProcessor).shouldEmitStateMessage(anyLong(), any()); + AirbyteMessage message = new AirbyteMessage().withType(Type.RECORD).withRecord(new AirbyteRecordMessage()); + doReturn(message).when(mockProcessor).processRecordMessage(any()); + doReturn(message).when(messageIterator).next(); + + assertEquals(message, sourceStateIterator.computeNext()); + verify(mockProcessor, atLeastOnce()).processRecordMessage(message); + verify(mockProcessor, atLeastOnce()).shouldEmitStateMessage(eq(0L), any()); + } + + @Test + void testShouldProcessRecordMessage() { + processRecordMessage(); + } + + @Test + void testShouldEmitStateMessage() { + processRecordMessage(); + doReturn(true).when(mockProcessor).shouldEmitStateMessage(anyLong(), any()); + final AirbyteStateMessage stateMessage = new AirbyteStateMessage(); + doReturn(stateMessage).when(mockProcessor).generateStateMessageAtCheckpoint(); + AirbyteMessage expectedMessage = new AirbyteMessage().withType(Type.STATE).withState(stateMessage); + expectedMessage.getState().withSourceStats(new AirbyteStateStats().withRecordCount(1.0)); + assertEquals(expectedMessage, sourceStateIterator.computeNext()); + } + + @Test + void testShouldEmitFinalStateMessage() { + processRecordMessage(); + processRecordMessage(); + doReturn(false).when(messageIterator).hasNext(); + final AirbyteStateMessage stateMessage = new AirbyteStateMessage(); + doReturn(stateMessage).when(mockProcessor).createFinalStateMessage(); + AirbyteMessage expectedMessage = new AirbyteMessage().withType(Type.STATE).withState(stateMessage); + expectedMessage.getState().withSourceStats(new AirbyteStateStats().withRecordCount(2.0)); + assertEquals(expectedMessage, sourceStateIterator.computeNext()); + } + + @Test + void testShouldSendEndOfData() { + processRecordMessage(); + doReturn(false).when(messageIterator).hasNext(); + doReturn(new AirbyteStateMessage()).when(mockProcessor).createFinalStateMessage(); + sourceStateIterator.computeNext(); + + // After sending the final state, if iterator was called again, we will return null. + assertEquals(null, sourceStateIterator.computeNext()); + } + +} diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StateGeneratorUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/StateGeneratorUtilsTest.java similarity index 96% rename from airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StateGeneratorUtilsTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/StateGeneratorUtilsTest.java index 1a0cc9f40d4c..0f65df39d292 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StateGeneratorUtilsTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/StateGeneratorUtilsTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb.state; +package io.airbyte.cdk.integrations.source.relationaldb.state; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StateManagerFactoryTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/StateManagerFactoryTest.java similarity index 97% rename from airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StateManagerFactoryTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/StateManagerFactoryTest.java index ded4290bf3aa..702429adc999 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StateManagerFactoryTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/StateManagerFactoryTest.java @@ -2,15 +2,15 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb.state; +package io.airbyte.cdk.integrations.source.relationaldb.state; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; +import io.airbyte.cdk.integrations.source.relationaldb.models.CdcState; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbState; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.source.relationaldb.models.CdcState; -import io.airbyte.integrations.source.relationaldb.models.DbState; -import io.airbyte.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.protocol.models.v0.AirbyteGlobalState; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StateTestConstants.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/StateTestConstants.java similarity index 94% rename from airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StateTestConstants.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/StateTestConstants.java index 096b0dc866de..0b6d0c4632d4 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StateTestConstants.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/StateTestConstants.java @@ -2,9 +2,9 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb.state; +package io.airbyte.cdk.integrations.source.relationaldb.state; -import io.airbyte.integrations.source.relationaldb.models.DbStreamState; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StreamStateManagerTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/StreamStateManagerTest.java similarity index 92% rename from airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StreamStateManagerTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/StreamStateManagerTest.java index 6a3db7a5d17d..3ed37ec42308 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StreamStateManagerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/integrations/source/relationaldb/state/StreamStateManagerTest.java @@ -2,26 +2,26 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.relationaldb.state; - -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.CURSOR; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD1; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD2; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.NAMESPACE; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.NAME_NAMESPACE_PAIR1; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.NAME_NAMESPACE_PAIR2; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME1; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME2; -import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME3; +package io.airbyte.cdk.integrations.source.relationaldb.state; + +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.CURSOR; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD1; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD2; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.NAMESPACE; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.NAME_NAMESPACE_PAIR1; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.NAME_NAMESPACE_PAIR2; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME1; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME2; +import static io.airbyte.cdk.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME3; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.mockito.Mockito.mock; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbState; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.source.relationaldb.models.DbState; -import io.airbyte.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; import io.airbyte.protocol.models.v0.AirbyteStream; diff --git a/airbyte-test-utils/src/test/java/io/airbyte/test/utils/DatabaseConnectionHelperTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/test/utils/DatabaseConnectionHelperTest.java similarity index 97% rename from airbyte-test-utils/src/test/java/io/airbyte/test/utils/DatabaseConnectionHelperTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/test/utils/DatabaseConnectionHelperTest.java index 605d6ddeafb2..9f7008f5f6c9 100644 --- a/airbyte-test-utils/src/test/java/io/airbyte/test/utils/DatabaseConnectionHelperTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/java/io/airbyte/cdk/test/utils/DatabaseConnectionHelperTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.test.utils; +package io.airbyte.cdk.testutils; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/dbhistory_greater_than_3_mb.dat b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/dbhistory_greater_than_3_mb.dat new file mode 100644 index 000000000000..2e2313ab7e26 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/dbhistory_greater_than_3_mb.dat @@ -0,0 +1,410 @@ +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842665,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666200,"databaseName":"","ddl":"SET character_set_server=utf8mb4, collation_server=utf8mb4_0900_ai_ci","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666219,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`aastjaeyhuxcevpnrqwdjrvlewbgqqrz`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666221,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`adhcfqcfumxvmsokmylceoltlqhjewvw`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666222,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`ageaxrapcifxppadtgfmvnubppgguatn`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666222,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`agqfberalhewuryqncmijexehjvpvhya`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666223,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`aiuvcawgjavfsityazsbykuodftgteoa`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666224,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`arifozcdgamaweuwmigngbztyhjenxgr`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666225,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`avimlxteragkjlgouvjrcmgasysrsxih`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666226,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`bbvzwwmjvuibbndayvwmfsvfvkomvuwq`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666226,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`bekezigfgafwjjfcxlfvmohuwjxylbco`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666227,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`bfilzxoyiqgaczecequheguojdjdqgkw`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666228,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`brkqrwtvwkcngrjfgrxsbyjsgzuuyjxk`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666228,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`brznqjnnrwxnevpxwjrcthnvpxrsawok`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666229,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`cjbsiglppbkiwuokzgspunevqbbejmfp`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666230,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`crfncrfexubbsoqfpqdpbclcqxnclxmt`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666230,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`crgfzudloyryxyhjtwycumowtyrpwsly`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666231,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`cvducdrtmbckboypjfdtljlqxnmjlvqo`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666232,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`cxgskvumnqmwipfupfwozqwmfnftsxvs`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666232,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`daxdopqapywjintkxohdnessozxvwdwx`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666233,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`ddbewigwzztguunwgdgxqetfoxdxqaht`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666234,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`dfoaznlacthdivxcqnznziszzihljtyi`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666235,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`dfzbmvchizqxwzhseritnruikiymswoq`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666235,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`dktjgzfdvuzusxrwcdfekwieeadvscpa`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666236,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`dmpkfkxwkhuajbllpqhimtoiumvecdvq`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666236,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`eckmzheculilvvtgddqoabqhmblwjros`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666237,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`edtpgezoafwesgnsrttenofpfwbgvskp`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666237,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`efltdshxkjahnyuwforbiwbeqptladwm`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666238,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`ekgwnocryenlhifewsirhpralwokvicj`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666238,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`eniuwqlzawrgrhvrzrcpihxgkdvaarew`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666239,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`feporuidwlohgvwbtiusejyfqsrumrac`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666239,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`fhwposxtdpufqjciekvsudvzcdupkqjv`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666240,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`fknkmyhzgzdgbklxmknjahvidhearmkn`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666240,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`fmydwwpviyonaxoxelqmczxonlvjcnxv`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666241,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`fmyszhkkeojxcsveoimbanarqxdpbqkz`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666241,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`fovddzegnkcsydxjfcghvbhggpiujcxa`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666242,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`fqvrwbozmynffznjtosjaimyhkulylum`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666242,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`fughcvpxapxpanhpwimukohmafwcback`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666242,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`fvwhkwjojhrpuruerizhsjyneudltdgf`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666243,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`fwahqjmgxjbvouzmzetprekihxwdeptf`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666243,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`gchydfeczangasnmvfdlykinnhisenui`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666244,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`ghyqwlwuskidustonvfapzjfqxxirouo`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666244,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`giavkelmcovmviellbpxbpmjayywzwvr`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666245,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`gibjkzyrpmoaypiniuiyywvlmaidwzxy`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666245,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`gjwbaxlnvuczlhgbaytrbajtlzxhwmbn`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666246,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`gmhufugcytipzofjpeipcjeyfkrzjgne`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666246,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`gnrwwzzyowsyoqcxntaclfwfmptzfttq`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666246,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`gqrdivdtlwyumyqwlhrxuviktdbmfnew`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666247,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`grjhfhonuudoavlukzcyzstzmfayaezp`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666247,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`grlnyqotcwelxxfihmfytiykiyggwidh`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666248,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`grmuxqmusubjsziwgvwmpopfcnqtgvps`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666248,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`gsrskijzyhcygzvokrbnmkbtnlfyglog`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666249,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`hchcurdohntyovpxuhajvwyugafgeysr`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666249,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`hkcokgnihhdfnawwtqjefmgjhhitamsj`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666249,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`hvfxfpdbsddegbronmrpaixtvsqjqygk`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666250,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`hwpjwafzowgrotlquywyqtrcabzussxr`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666250,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`hyijqphvtvgdgegsnseyzeyooughktwe`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666251,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`ifbeclftxswghnufrctevakgzhdsmlko`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666251,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`ihrmsdgptsjleslopqmanczyqcdtudap`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666251,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`iigengzntldyhysbkebjxfctfqrglsed`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666252,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`itudgkajsibeeiwiqpdermkbhjhcuhez`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666252,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`ivjawdrlpjejctimphddwlktsjeoqpsc`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666252,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`iwnntobuwmkuzncjkkxxbjyvktvzwcla`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666253,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`jklxyfguyfwhbanvhqbtokakczltxncj`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666253,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`jnnbrmmdrkoflyyvppyijnmfxbdveonu`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666253,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`jwaehjgccilncrxqdqdzytzyjdhdrtfi`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666254,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`jwtirnhbsptqymlhedjwtfeddytgzipg`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666254,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`jxtszwleobqkzaxtmowwchvvkrzyootc`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666254,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`jzkrhqtehgofuwbzfrjzkrenxfjfbkjc`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666255,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`kdodfqyudmcpsufuzzqeocbauczaskpt`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666255,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`kepkeqfmkwienbdkjycwcyzjpzrrkrrg`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666255,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`kfixdbsknznmfkubbmksyozpamncervh`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666256,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`kftwodjtxilhqupkceqeadajangtxxfg`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666256,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`kgeadtdnymxzvfhjvvodgeoxonwagwll`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666256,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`kgjyjbtpzshoafctbllepaalknrmhmzy`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666257,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`kmifjrkanbihkzrymcquzyofrbogiygu`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666257,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`korrbwglsldbalqvsswaqstzuxvnlrps`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666257,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`kuykqleuwgycpydwatkgiczdqgleegnu`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666258,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`kvqskdqluyprqlisnljdnaxumzqetjoo`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666258,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`kwcrcymugsbqmlclxhvjycitcrcbqmuq`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666258,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`kxmwllnisximtjilqoifyciirajdkdqe`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666259,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`kybtprhlbzqgbnsogitpfetdycsqqqmu`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666259,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`lgzqspwbxcvbxbxdceywhoesgrnsmpga`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666259,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`lneozjmnjubkgsxmcosrybaevorbgvvn`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666260,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`lqbwjasfleffegotnszounuxojffyehx`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666260,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`lzbcpccmabxinidkpdxbjeyepkhowvqb`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666260,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`models`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666261,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`models_random`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666261,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`mrhwnmuraagswlujmtsibhxramzicgvh`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666261,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`mrorqfumguckiypivdoraipjpawwjdfq`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666262,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`mwyzswmpkowqnbzyvfhkpmvnjfrgnraz`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666262,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`mxslcfjftoitkvwkjafdifwuxfdmfsym`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666262,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`mykafjtkvhgjskiyabehpblifngdkhwu`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666263,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`nfqrzhysvyortaitkelwhsffveslnthx`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666263,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`nmbgkiphylqvhfzedfvrjrurcnseuvvw`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666263,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`nsbiwnxsqajlnasfuoofgvhmvbabaxfp`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666263,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`ntdsamdzxoluyqsidfutsijfjlwpulzz`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666264,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`nxwatrwouueiyynkxlmdmddfodlkdsgt`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666264,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`nyhwscsvzxplglrwnsgvsfidffshzfly`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666264,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`nyrgogpbmkyjitqgmdttnyaxvmfppyld`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666265,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`okabqmjgjiaxwjjcnmwstkkozaoeqiiu`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666265,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`opxnblpgxrgsptkxokfztqjhkvdodwud`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666265,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`orpveknrbipxbinvykmnqwjwotxsjuoq`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666265,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`otmhuzxdzfbyzubxhxxqqnfslkmomovx`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666266,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`ownvozuqbdsuwedyuvzwnksqhgkngezx`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666266,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`pefrclzfkqemnrdgutlyvuoyvtzxmtuv`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666266,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`pfhgocfvrcmabgqdldeflyidqlcdhbmz`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666267,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`pfqjiwqnmbfnwysrzbqigotyuehiiwaz`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666267,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`pghwpbsubektqcppdifryknztxnmsvhf`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666267,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`phfoifsjghtcrrrmlirrscflzgkfcilb`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666267,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`pmufyqauiycsugklnqmjlwtntfykevoj`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666268,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`ptxoeakypxcinxgiwxnegyucbfpderrz`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666268,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`qfgnabrfurvwokonupfzqzkpbgrlevbv`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666268,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`qggnhwqlqjkouhxetdqfuaqozrqqpcle`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666269,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`qiwtlstwkkhmomafokasfiebppralewy`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666269,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`qlasmsrgsfuikzxkdrpxtdwwbaalkjde`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666269,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`qmflonomnffllkxweddtjcrusntdyhwl`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666269,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`qmzbocxykzmpczrsllqbenafxqafrzhk`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666270,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`qnopovgcxspwoeghyuonqnbbnrvjvhow`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666270,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`qsirphfmlpnkopaqiayapzezhyeudrzs`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666270,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`qxqwzbxrbfsmpxxlxepnlllymnkkslug`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666271,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`qyylbziunhtqyqeloiskfjizpimemone`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666271,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`rdcfwajpujmymsqszcnekivknqxdoqil`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666271,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`rgjeprxzblxygbhnumnnjlryntqzuony`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666271,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`rjjpcxwartpnkawkhzhlboqncbbgzzgx`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666272,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`rklsysjbbggcyaheggpsezportemxlmu`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666272,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`rlpjqdqhwzurijhocbkdpovoiiqctrve`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666272,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`rmacnjxgjbzdfqdleytznzmsuzvkdjrq`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666272,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`rmcsuzfuprljobatuqxsqkuaffhpdtag`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666273,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`rmdvstrvymqpjlchedshuysqvdwtaegw`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666273,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`rrxmvbjbfyljiregbwvggjujgpkwekjf`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666273,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`rvwqmdxmybrqorpqethfdugrbikeeqen`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666274,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`sddicdunfmgiuvegaxtmieursdrycsld`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666274,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`seckcqhiaewieeiomssfzbhcvhkqmjnt`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666274,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`siftmoygptajwmczbovtnwqyjdpcmqfw`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666275,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`sktiezunkkczqbwwviueupxhgmwmasju`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666275,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`somkexlrchwjsnfjblemmdlxdrlqxiwe`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666276,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`sudibszqrvrxrwiazvcqqeovwgppgjth`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666276,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`suvqorxzmwvjejxhmiiitcxlimjbkmok`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666276,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`swgxwfoajgckubpfskppncstrxbnrpfl`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666276,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`sxmfnbngjuiquecrlxtfqulfkuqmjfzr`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666277,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`szptihpdgatfskwdlqlvratsglyrqnga`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666277,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`tcrradlvorgzjcciygegyfvojodcfdhl`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666277,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`tfoaqprarfoabbffxqiypuqnjuzesvmg`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666278,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`tgczemkospnlaxgjoudtzsgbgjwujcbh`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666278,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`tgtlonwoccyfoxpnlcdbuzcncumduhrs`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666278,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`tixcpagetyojezbmkaeetpemukblcpsr`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666278,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`tktrwfqkbwagctiwqbgzfdyjkzxqxrnt`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666279,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`tlawbzmwhcjjoudhkxvxmsfrdeygughc`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666279,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`tpksmtlcpcazciywpqiwdqlzxbnoiktq`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666279,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`tpkxfywbmayegsworhdugtdrzkjvveya`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666280,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`tzetkodbjogqjrhxuxwowogrhxsolvar`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666280,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`tzlncpqzqjgadnlmxsvbcgcpiljusbbe`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666280,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`udikvvdsawrybndbmlsacvseeqbudtkz`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666280,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`udizceykeqmjnuuejwuufbzihqpwbzfp`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666281,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`uflnnphheekurdaziorsgpfwdysuutca`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666281,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`uhncixqtyhjsicsgvnamwatzwbpgfqld`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666281,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`urtqvqjqqavkdscrqvcdnwnrtgnmfnbd`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666281,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`utrkrrrttkgoplueqarmsziysncokcgj`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666282,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`uwhvppgdytwyftjjcnihrmfdnhimzykv`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666282,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`uyjsubdlcesowgvppiweyyynybzozzcb`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666282,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`uyllrczkxxxqdijmtoeuaxgipxtztugl`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666283,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`uzfkyntmiqwwfluspuhyvetjysuvdqgr`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666283,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`uzwzfxgsmncskifwztscboirulhiccox`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666283,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`vankewnkqbxjamlctdsdjgswgeyvrjof`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666283,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`vhmohjbaalgeckwlqgznnqcvgjavlbzv`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666283,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`vlatilsnjpcfvjqannnvxnfclsppggel`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666284,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`vldtvucetkkwbkcoweeuxswdsnytbwak`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666284,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`vqxcfvfvkfrjtrlfeabxtnndjgrileqz`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666284,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`vrbvigqupjgsaeqqwingjxtcxidpztuo`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666284,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`vvukznvxvqlrpqvbrbvlexzlwndtibeh`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666285,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`wchuegpxotggojpmappheurpxfrcivka`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666285,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`wcvjylwqhadvqltmecxwsuarjnpxvtmi`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666285,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`wgkwfskbehhqasowhsdegiiatybhbuvq`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666285,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`wgxcetmtjwvmrtdbuchydagtylsvtsga`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666285,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`whclbatdragfegfzxazuownrpaxqfwpq`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666286,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`whemkduelgfkfynwcciloonbigfttakk`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666286,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`wjejvuvpihwabfbueufasjexzxfdfwxr`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666286,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`wjrpbvixffhcxmgwdtqlaaeddrzznowk`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666287,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`wnbrzqlwqabujjsfpielvqcwmrnmuqwu`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666287,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`wouewinkcurfknfxilwfeellhzmeqtjq`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666287,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`wqdandhuzkcuajvglffrvuwqfxsfunjn`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666287,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`wrlrqixavhpautdnbanimexxwqddrvmn`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666288,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`wzjzwsynaunsloezfpugchitwewtoxto`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666288,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`wzskeeofxzgrcdxuehzyhdhmmlzyxqxo`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666288,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`xkjqfzysfnzfwcmrzcgjzonrwhldoesm`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666288,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`xkwxygabgxrruwrhxqbyhhgahhzdbnwg`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666289,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`xlbwxatoglvtzyxuzszybrvskcdgicob`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666289,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`xmcvrffbufzmyibhrgpsdxwjjojyvegg`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666289,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`xrghycskudhhtjoegkkgwoksolissqht`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666289,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`xyfheiynhztwfzchczoqkoazqlkqybll`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666290,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`ydfgmjyjnfakxnzitneuzhmydvouvjes`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666290,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`yfcmxhvqnfmsqbirfafqjuymmgluegpl`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666291,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`ygbhxqnxduzibovvetyxidhqtnuizvhi`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666291,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`ynchpzshvnthrfykxjbnnndevivsthen`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666291,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`zcemsecafryitcsvmgpbhebgzwmycaiu`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666291,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`zfgsdjvymmttvoqeggdyoqbiudfvvjnu`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666292,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`zjynajtjehfzwlyfxrnocalgmomgzioq`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666292,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`zomwihdblysidiflhkqodbripxteqsje`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666292,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`zrjhsxwmexkwyntdmqgzgaxcjzqahncz`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666293,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`zrpahurtunmdbumraxjhiqvfumrdlnjn`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666293,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`ztpttwzzkgyxsjgpkltayosfjkjwfgok`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666293,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`zvizamagjrkoamcwmhgradfhuizonrdx`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666293,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`zxhltxrsmhtsyvtgrlcsbvouuzpmoisc`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666294,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`zxopybvdnhidckkbrsvwijsunafkxxtb`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666303,"databaseName":"models_schema","ddl":"DROP DATABASE IF EXISTS `models_schema`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666310,"databaseName":"models_schema","ddl":"CREATE DATABASE `models_schema` CHARSET utf8mb4 COLLATE utf8mb4_0900_ai_ci","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666311,"databaseName":"models_schema","ddl":"USE `models_schema`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666482,"databaseName":"models_schema","ddl":"CREATE TABLE `aastjaeyhuxcevpnrqwdjrvlewbgqqrz` (\n `ecsdzfgicetweccjwrzexfuehjztsumn` int NOT NULL,\n `ybqytsxeruqgjlxumznjkyqykgtzfdpy` int DEFAULT NULL,\n `rbzvwhbcszrorefhdjqhzukmeopjiila` int DEFAULT NULL,\n `zjkhtrqlvgowmcrrhrzsbnbxqayvcrqo` int DEFAULT NULL,\n `ulvnpfknymyylfahgsywzusmezyntwmc` int DEFAULT NULL,\n `oxmocspitjivttwebqruwtyrfvwtevho` int DEFAULT NULL,\n `uunkmslbtyztrljelopxgqyujccedqvr` int DEFAULT NULL,\n `lxriudwgopondobyungmdappngtkyabi` int DEFAULT NULL,\n `kdhlufzacbgiviimzsqglkjgfalejycw` int DEFAULT NULL,\n `wghzrkpmwvlpgghobatuvzxuuiulggei` int DEFAULT NULL,\n `kgbmyqnebkeebhviqakyytunqqcqcljd` int DEFAULT NULL,\n `dsxqqgbhcvtwkwtzeypxdsblsytaptxc` int DEFAULT NULL,\n `mmgdyehsavknovjdfwhhcawfvlhbujyl` int DEFAULT NULL,\n `thozzgtziknmgrgkvbqzmaumfyfryqwj` int DEFAULT NULL,\n `yiomnvlnqnrytymylzmsutefxwdqotch` int DEFAULT NULL,\n `ddecvmtifskdwxzqmdyawapllzeaqsue` int DEFAULT NULL,\n `pcwcqpnshlecoenlkqhxpvgcnpknlmvr` int DEFAULT NULL,\n `diogsnhbfmgzvztnvtjarmqcdqhkgwcj` int DEFAULT NULL,\n `vujfvliddbqvezbyecniidvwcltfqngl` int DEFAULT NULL,\n `ifmbncaqrqhoitliqdjwsrfqayyyawls` int DEFAULT NULL,\n `rpbtdsapqqdopdbcnuhgsmrlgwtkpvkh` int DEFAULT NULL,\n `qyxiqjalcvfindvkczoahdldozhhbhhr` int DEFAULT NULL,\n `ivgvbjjgpckjglcuhouideukfguiikxj` int DEFAULT NULL,\n `ovrdlctkfjbntahozxqpmlzfazvetsrx` int DEFAULT NULL,\n `cyqtpkxvnqcwpdnhkknlnvrdwnqmotvd` int DEFAULT NULL,\n `rrssdqoglztevdekczkdyywlwlmoqbkd` int DEFAULT NULL,\n `ltxhncwjgzpbolnagaddtbloamhuscrv` int DEFAULT NULL,\n `mkrllaburxvqufuydsguszglzzojadtt` int DEFAULT NULL,\n `pecnocpxofrboyvlccrnsxlmwooycjby` int DEFAULT NULL,\n `jykwvlwyvwgjnmyjxfpxwluujgdnlonk` int DEFAULT NULL,\n `qcajtwaznrrasyaapogxqfhimngqgbjl` int DEFAULT NULL,\n `uxsberecpyzejgfsnheiowtijoavawtc` int DEFAULT NULL,\n `rghswaxekeavttrcrnrxyqsywpolwyij` int DEFAULT NULL,\n `wyfeqtfralvrfrdhpkuyjbkeziwrrmxk` int DEFAULT NULL,\n `skjhlrzymijdbcbvsjgwtloktrxrgedz` int DEFAULT NULL,\n `utgbrththpfmnoobsctouhgszehnbgoh` int DEFAULT NULL,\n `zldacnbgzfrnyrgicbwewygtihlqphul` int DEFAULT NULL,\n `doclpvpwkrdsmrxonwfgzxdvujipfbnj` int DEFAULT NULL,\n `zyjkjvohuqodbflaozbsriemzjwqypwf` int DEFAULT NULL,\n `jylzicmggsemxwmjfelvlwtwrfrdrhzp` int DEFAULT NULL,\n `gbrwgrbmhjavbecouajkxpmmmoxgimlr` int DEFAULT NULL,\n `rtbzbothpktxpknwstfalzlhvvshorii` int DEFAULT NULL,\n `iahexftofhdsftmhgfnalkghmpffqdnv` int DEFAULT NULL,\n `jmxintavenpchhxqhlbcicbhyekapwfd` int DEFAULT NULL,\n `vxqzrjyouosgneqqbstvtbzpxhghsydp` int DEFAULT NULL,\n `xpttymwxaffktwqkzuxjxiunfseijedn` int DEFAULT NULL,\n `adwdylvrcslnbhunzcqocqrvkigizvht` int DEFAULT NULL,\n `fjjrwtpmhkcpsfhytwsxvexifysxyzdu` int DEFAULT NULL,\n `yqllzdkywgzkjnkjbjjtdjhtrpnvgmld` int DEFAULT NULL,\n `lnekpomshfdcgvbzwxybxoqdhpllqywn` int DEFAULT NULL,\n `ocncjtssovhfurwygcdqdtpqfrfpipes` int DEFAULT NULL,\n `kltergbveepksttfskfpwkjoelvrbvef` int DEFAULT NULL,\n `pkitnlmynnusvobksvrafsztnwdnkpum` int DEFAULT NULL,\n `rcylzyyyjbyqlzklkcxzswjvztetkxyk` int DEFAULT NULL,\n `ccibgetracnincbmezjthlzfynnxkwfs` int DEFAULT NULL,\n `dvnlepwzjfhqpeivmzdjujoyzmbzsxne` int DEFAULT NULL,\n `xzjibqmtmhbkedzazvgafmfqcgptpwra` int DEFAULT NULL,\n `omhruoqlsbkyrxygozztrbqkugyxryse` int DEFAULT NULL,\n `ihnefdndqjwmelgeehhxferqgofwpsba` int DEFAULT NULL,\n `agjhljmknmdgihaktoffcacursyypfeq` int DEFAULT NULL,\n `gxvewwywekxhuntsyzrlcyffeutxctsc` int DEFAULT NULL,\n `cuazwkjomewuyhjuxyqfnhnagnafecor` int DEFAULT NULL,\n `mkwpafohpictqkzcbjzlkvjebrpnqdic` int DEFAULT NULL,\n `wvuydyteqkkqdwspliidihtzvvxdoerf` int DEFAULT NULL,\n `hqoqhjigbelwnreqlbzwuyzdiguioubv` int DEFAULT NULL,\n `idcorureimrfijziiefmmhgkppyrdprv` int DEFAULT NULL,\n `yqzatgqzlvtfskypxjsrvtcnzblrbijs` int DEFAULT NULL,\n `hccsyxgovjferdzrahhlpunfjfhusuvb` int DEFAULT NULL,\n `vedfemwhwejqifvkfbgdjnoyradsmbos` int DEFAULT NULL,\n `zswbpubxfswtryfctjavqpsovazbijzx` int DEFAULT NULL,\n `qofuydtmglubtlbwqnceckmhjwbwrlcd` int DEFAULT NULL,\n `seilzggfxicucsuyxryxqljyaagfvoje` int DEFAULT NULL,\n `fxezgqogyavdnffckbfamdxuhzntwyrn` int DEFAULT NULL,\n `fsvvxfakqgyuaqfffkeacwsykcpiufde` int DEFAULT NULL,\n `yidbhankxieuggzbyywdaffbswijgumz` int DEFAULT NULL,\n `tzhdpvgopwqusyknpdruaebchutuqyrd` int DEFAULT NULL,\n `iwzziscwjlsujjbavqrfigwmexxuhtrw` int DEFAULT NULL,\n `kgefdasixhrmhqsfimkuigsxllevyvfj` int DEFAULT NULL,\n `rdzcbxzclutcrgkleaaiiaixoaxslotr` int DEFAULT NULL,\n `cwylfpyusnredbnnftgywpilbmykpppe` int DEFAULT NULL,\n `fcshzglgshgrlsslddaoazfeobqtmegd` int DEFAULT NULL,\n `ehtvwjykajxoyqxctssvdwcwwqmdnkdr` int DEFAULT NULL,\n `hqdujuaflwrynixedrxivgnqystzlder` int DEFAULT NULL,\n `dzwmxlghyekbmhcugejyfovicrouzpwl` int DEFAULT NULL,\n `jkbeqsgekcohttyezkozxkilkkbaagwn` int DEFAULT NULL,\n `fpixjsjdbwiggysqxcgcmqjigsevhdpa` int DEFAULT NULL,\n `rqjzustpcrjvhnezcrajhelcgxgbrvla` int DEFAULT NULL,\n `tokwmdbwrypbrzxrpsukoumdnyhsulxf` int DEFAULT NULL,\n `nhihfnnpvaydvslfxfvczbahbdajmnpj` int DEFAULT NULL,\n `gejjkyqsrlgcgtdujarwcwpvlewfjlek` int DEFAULT NULL,\n `fkmwtodvsczkmppmuopaltzxystcvakb` int DEFAULT NULL,\n `cxfmcoxvjanczkhjezvomtbgjnghqsfx` int DEFAULT NULL,\n `grlnrdgpcecwmaiomngllugylubwvpac` int DEFAULT NULL,\n `ydfgqmycmlhwghkhrokcdqulruwulujf` int DEFAULT NULL,\n `mcwdtkuipybwdkkacqxifcxepxdjovex` int DEFAULT NULL,\n `xtclujndrhraixzfbzytywvumifpzndm` int DEFAULT NULL,\n `ggsjyjvuhnxgpizvzcioqlcmjzgbvxsr` int DEFAULT NULL,\n `mwxgjeypmqzsxknbickhnswopftgjwlk` int DEFAULT NULL,\n `ymdrwhvvruycjuhtdcrsouekqgkmtejm` int DEFAULT NULL,\n `fhzhgbpnooqvvjlrafqdbicfhimhozds` int DEFAULT NULL,\n PRIMARY KEY (`ecsdzfgicetweccjwrzexfuehjztsumn`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"aastjaeyhuxcevpnrqwdjrvlewbgqqrz\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["ecsdzfgicetweccjwrzexfuehjztsumn"],"columns":[{"name":"ecsdzfgicetweccjwrzexfuehjztsumn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ybqytsxeruqgjlxumznjkyqykgtzfdpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbzvwhbcszrorefhdjqhzukmeopjiila","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjkhtrqlvgowmcrrhrzsbnbxqayvcrqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ulvnpfknymyylfahgsywzusmezyntwmc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oxmocspitjivttwebqruwtyrfvwtevho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uunkmslbtyztrljelopxgqyujccedqvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxriudwgopondobyungmdappngtkyabi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kdhlufzacbgiviimzsqglkjgfalejycw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wghzrkpmwvlpgghobatuvzxuuiulggei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kgbmyqnebkeebhviqakyytunqqcqcljd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsxqqgbhcvtwkwtzeypxdsblsytaptxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmgdyehsavknovjdfwhhcawfvlhbujyl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thozzgtziknmgrgkvbqzmaumfyfryqwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yiomnvlnqnrytymylzmsutefxwdqotch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddecvmtifskdwxzqmdyawapllzeaqsue","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pcwcqpnshlecoenlkqhxpvgcnpknlmvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"diogsnhbfmgzvztnvtjarmqcdqhkgwcj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vujfvliddbqvezbyecniidvwcltfqngl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifmbncaqrqhoitliqdjwsrfqayyyawls","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpbtdsapqqdopdbcnuhgsmrlgwtkpvkh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qyxiqjalcvfindvkczoahdldozhhbhhr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivgvbjjgpckjglcuhouideukfguiikxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovrdlctkfjbntahozxqpmlzfazvetsrx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cyqtpkxvnqcwpdnhkknlnvrdwnqmotvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrssdqoglztevdekczkdyywlwlmoqbkd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ltxhncwjgzpbolnagaddtbloamhuscrv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkrllaburxvqufuydsguszglzzojadtt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pecnocpxofrboyvlccrnsxlmwooycjby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jykwvlwyvwgjnmyjxfpxwluujgdnlonk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qcajtwaznrrasyaapogxqfhimngqgbjl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxsberecpyzejgfsnheiowtijoavawtc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rghswaxekeavttrcrnrxyqsywpolwyij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyfeqtfralvrfrdhpkuyjbkeziwrrmxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skjhlrzymijdbcbvsjgwtloktrxrgedz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utgbrththpfmnoobsctouhgszehnbgoh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zldacnbgzfrnyrgicbwewygtihlqphul","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"doclpvpwkrdsmrxonwfgzxdvujipfbnj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyjkjvohuqodbflaozbsriemzjwqypwf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jylzicmggsemxwmjfelvlwtwrfrdrhzp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gbrwgrbmhjavbecouajkxpmmmoxgimlr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtbzbothpktxpknwstfalzlhvvshorii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iahexftofhdsftmhgfnalkghmpffqdnv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmxintavenpchhxqhlbcicbhyekapwfd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxqzrjyouosgneqqbstvtbzpxhghsydp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpttymwxaffktwqkzuxjxiunfseijedn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"adwdylvrcslnbhunzcqocqrvkigizvht","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjjrwtpmhkcpsfhytwsxvexifysxyzdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqllzdkywgzkjnkjbjjtdjhtrpnvgmld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lnekpomshfdcgvbzwxybxoqdhpllqywn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocncjtssovhfurwygcdqdtpqfrfpipes","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kltergbveepksttfskfpwkjoelvrbvef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkitnlmynnusvobksvrafsztnwdnkpum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcylzyyyjbyqlzklkcxzswjvztetkxyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ccibgetracnincbmezjthlzfynnxkwfs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvnlepwzjfhqpeivmzdjujoyzmbzsxne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xzjibqmtmhbkedzazvgafmfqcgptpwra","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omhruoqlsbkyrxygozztrbqkugyxryse","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ihnefdndqjwmelgeehhxferqgofwpsba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agjhljmknmdgihaktoffcacursyypfeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxvewwywekxhuntsyzrlcyffeutxctsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cuazwkjomewuyhjuxyqfnhnagnafecor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkwpafohpictqkzcbjzlkvjebrpnqdic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvuydyteqkkqdwspliidihtzvvxdoerf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqoqhjigbelwnreqlbzwuyzdiguioubv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idcorureimrfijziiefmmhgkppyrdprv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqzatgqzlvtfskypxjsrvtcnzblrbijs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hccsyxgovjferdzrahhlpunfjfhusuvb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vedfemwhwejqifvkfbgdjnoyradsmbos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zswbpubxfswtryfctjavqpsovazbijzx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qofuydtmglubtlbwqnceckmhjwbwrlcd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"seilzggfxicucsuyxryxqljyaagfvoje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxezgqogyavdnffckbfamdxuhzntwyrn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fsvvxfakqgyuaqfffkeacwsykcpiufde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yidbhankxieuggzbyywdaffbswijgumz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tzhdpvgopwqusyknpdruaebchutuqyrd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwzziscwjlsujjbavqrfigwmexxuhtrw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kgefdasixhrmhqsfimkuigsxllevyvfj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdzcbxzclutcrgkleaaiiaixoaxslotr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwylfpyusnredbnnftgywpilbmykpppe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcshzglgshgrlsslddaoazfeobqtmegd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehtvwjykajxoyqxctssvdwcwwqmdnkdr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqdujuaflwrynixedrxivgnqystzlder","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dzwmxlghyekbmhcugejyfovicrouzpwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jkbeqsgekcohttyezkozxkilkkbaagwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpixjsjdbwiggysqxcgcmqjigsevhdpa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqjzustpcrjvhnezcrajhelcgxgbrvla","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tokwmdbwrypbrzxrpsukoumdnyhsulxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhihfnnpvaydvslfxfvczbahbdajmnpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gejjkyqsrlgcgtdujarwcwpvlewfjlek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fkmwtodvsczkmppmuopaltzxystcvakb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cxfmcoxvjanczkhjezvomtbgjnghqsfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grlnrdgpcecwmaiomngllugylubwvpac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydfgqmycmlhwghkhrokcdqulruwulujf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcwdtkuipybwdkkacqxifcxepxdjovex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xtclujndrhraixzfbzytywvumifpzndm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggsjyjvuhnxgpizvzcioqlcmjzgbvxsr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwxgjeypmqzsxknbickhnswopftgjwlk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ymdrwhvvruycjuhtdcrsouekqgkmtejm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhzhgbpnooqvvjlrafqdbicfhimhozds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666552,"databaseName":"models_schema","ddl":"CREATE TABLE `adhcfqcfumxvmsokmylceoltlqhjewvw` (\n `qzoyzniqxgxucwlppvmwidiqhbzwluaw` int NOT NULL,\n `qjflcvzkqdrmgslqpmokgsqxibbaxxin` int DEFAULT NULL,\n `miaevkbjzhybaieljrjfbdnuldjsotik` int DEFAULT NULL,\n `osajqtaxzudfeifxdwbdzglsxsmniiqm` int DEFAULT NULL,\n `mkmfacdltcspqoqseokzfyujqdncnprv` int DEFAULT NULL,\n `orrhcyiqbzuhauuvdluwkikbhbthcqtw` int DEFAULT NULL,\n `wzmjlkycvjcacqqaygcfffsusvroccpw` int DEFAULT NULL,\n `bnzfnjjllhrumjjwqgcmxzybqffwldik` int DEFAULT NULL,\n `lumdqvewlffvfjdrovliwhlwnsrjsirf` int DEFAULT NULL,\n `rdhcqmbwyfwyqewbsiuasjfpxruiumiu` int DEFAULT NULL,\n `akkwvbwtwjteauvqixeymnghmjftpjlq` int DEFAULT NULL,\n `gtzaprddthraxupgdbhgzgwbwbhuluin` int DEFAULT NULL,\n `wxajlbdnqkwotucynpkxnutjykrbluwm` int DEFAULT NULL,\n `uvevocmdrbdzgabjbebdjwockopsvqzu` int DEFAULT NULL,\n `yqsljvjgxysisprnghcwxhnqhodpgusq` int DEFAULT NULL,\n `vipbckzmnzmiehjfoanhoysuqczmylxt` int DEFAULT NULL,\n `madyjspdkxpbdxnaawtzpnavkchinzqa` int DEFAULT NULL,\n `jrednxitbdrozupwgsesttxeshfekdcp` int DEFAULT NULL,\n `uepgrtfclojkomyrppgwcrdhnpcygmvy` int DEFAULT NULL,\n `jeqlgakeznvpuewvvsanwsigngikwvjj` int DEFAULT NULL,\n `ianweezzfguayslautdlasuyqpkdlkak` int DEFAULT NULL,\n `yqzxvnquwiafejzuryszsiejgsgvngud` int DEFAULT NULL,\n `hihbnjosxcpailclmbvsqctsaohuigve` int DEFAULT NULL,\n `mwlqlpeiravrehzunzadqjbknyzzfxix` int DEFAULT NULL,\n `rnduxvchezzptqqisoyetsjjgbdepahu` int DEFAULT NULL,\n `obzredhsoeeygarmsejtguiqjvkvpfqd` int DEFAULT NULL,\n `phdruiredjlqfoorpioxarasjfjdbbzd` int DEFAULT NULL,\n `wcvlhvuedkkprsemeambfjgubmxwwvtr` int DEFAULT NULL,\n `ryypvgpispuamsuwuoymdyaqirjkrytb` int DEFAULT NULL,\n `sbzihronfnuxbhlyfiijeycsitghzlhz` int DEFAULT NULL,\n `xqbgybwowkekauiexwciyhpwjcjbqwfg` int DEFAULT NULL,\n `wucwzydpvgmhurqosdyrhnjlvunrcpnm` int DEFAULT NULL,\n `irwrqtnrxepmtazekgobevtzdekwqcdt` int DEFAULT NULL,\n `hbrfxbfwbytsxrypirqgupikdpongdru` int DEFAULT NULL,\n `plyxpyoxeeluzufkeiycpkdlvudtxcke` int DEFAULT NULL,\n `ktctzhrcpctmfnhfvomfacccuayggjrk` int DEFAULT NULL,\n `phhwmuzkybklrdtftijlzorxqftckysq` int DEFAULT NULL,\n `rmtrpkhtleooqrjwglymvabaehecoezr` int DEFAULT NULL,\n `jhidlhzxkxtttbgrbfprgywbrxnbtwdk` int DEFAULT NULL,\n `mvsoxmajsiabgnbxjwkzzwaohzwrscfu` int DEFAULT NULL,\n `giefvidgsufbhsptdbajweklueomonwl` int DEFAULT NULL,\n `ybtpalvukdukiwokipmqszsapcleggvg` int DEFAULT NULL,\n `pigrylqkoipxjucyzpwvczllxoefzcju` int DEFAULT NULL,\n `wffhdikhbcecwbpafcrfrkuarqzcofnn` int DEFAULT NULL,\n `lnbhoonnearqdypfevgjebzxavfmajwo` int DEFAULT NULL,\n `mmerypukceasusyulywmkjlmqyfkedox` int DEFAULT NULL,\n `wsnlncrzznwoagyhlvcoczezblxkgyfm` int DEFAULT NULL,\n `hppvidbclmsojammofywhsnxvbyycuea` int DEFAULT NULL,\n `zbnpydykvebozntkwawozpfhcakqyjjz` int DEFAULT NULL,\n `nsijpmaetezxopeoqazmfjahkwknmybg` int DEFAULT NULL,\n `lbdueihxjejjtwxfhznfmdimwbuqvxmd` int DEFAULT NULL,\n `uawidvpfcbxkuvdgoemsndqfbdcjucyu` int DEFAULT NULL,\n `tudjeophngkefltgurcmnnjhytnjokal` int DEFAULT NULL,\n `imaghngnhaasyvjpjgswbbptqkdbejfr` int DEFAULT NULL,\n `efhkmdxshqrzycxqwggebmlvxgsxqmse` int DEFAULT NULL,\n `tlhqybvmnueagojlovfaqlqsdlymzjyk` int DEFAULT NULL,\n `pegqljorjmkgxxujobjyaryjqordnhps` int DEFAULT NULL,\n `hitzilteauemrojypigtazteezbzljaw` int DEFAULT NULL,\n `kvfulsrfldtgwqdeailkayxddmqsjndz` int DEFAULT NULL,\n `kobyaixcgsehvoktwftlukhptghqdfmw` int DEFAULT NULL,\n `xnmaovdehitmkqnrlnkrazdhgzytqmpz` int DEFAULT NULL,\n `naekwvqfilsrfprdkgrbpbomehpwrmlh` int DEFAULT NULL,\n `xqztkpxnwrnpergdmbqnadmcmxpovwig` int DEFAULT NULL,\n `rjgthcavoyjqjpwhdiatajoxrfecmffg` int DEFAULT NULL,\n `aqyeojazxwzrhwtnxdifgoiaefukarjh` int DEFAULT NULL,\n `fcrorblbrtczkrxovprzlnicdcrromfa` int DEFAULT NULL,\n `dubxgpytstlrhowzpwhwxcmykaciypwq` int DEFAULT NULL,\n `mcbleuhwonmespgzvlzvnrwdgxfmvjyo` int DEFAULT NULL,\n `lpyslqycmlrysbphevvtbmpdiofojdoc` int DEFAULT NULL,\n `pcqbabsfenmofeutxmeviozephodetuh` int DEFAULT NULL,\n `txckjrznygbcizfpguwtqzwjswvnwhvs` int DEFAULT NULL,\n `muvzvwamzesksctcdratviqxqcmlxbsd` int DEFAULT NULL,\n `rohvdvtednpythqbgsbnkwyumyfoofnn` int DEFAULT NULL,\n `ebgqexmdosfpqpjhqaxgpptewdzskjaq` int DEFAULT NULL,\n `isffnzredpjxzedwfkfapvvlsexseywh` int DEFAULT NULL,\n `yprsvwjlgxjqgmhlttmomychprvuiytz` int DEFAULT NULL,\n `emwvprfzfibfhqsmnafmmkeymexyjgyo` int DEFAULT NULL,\n `tpesebococsxiyhilsxtieisqpnisbbv` int DEFAULT NULL,\n `bijokuctnphsdqfnujbzjigaoalotrrg` int DEFAULT NULL,\n `eoscggorgukquowcygejlcsxpdpbzhdl` int DEFAULT NULL,\n `zsjltqypqkvnlbidviylkhbgkzfryjdu` int DEFAULT NULL,\n `uvflfdbywxcqrnovyjkecvnbqravstfz` int DEFAULT NULL,\n `qylogijeqsnutuimclgirdvtvxozxbrm` int DEFAULT NULL,\n `vzyftuzkwqoobyryywbrtfebfxpcbmmy` int DEFAULT NULL,\n `skrxsmgtommgwjvizlpwlvxbebenlmzh` int DEFAULT NULL,\n `qomstdxmlussdtxmsbbgbtjuxpvpjzhx` int DEFAULT NULL,\n `vcdspgrvodyrykpessvmpkjbjdnshfep` int DEFAULT NULL,\n `qelhapahqjzgeggamindpfduagbwcguq` int DEFAULT NULL,\n `eatldpqpwomrdrpuzwtriwvyzlwfjidr` int DEFAULT NULL,\n `auokotmaedvfqtkczyxbnlufzcrzoqki` int DEFAULT NULL,\n `hcelyynexdxiwiwsappbqzwrsxbdnqyn` int DEFAULT NULL,\n `dtemxlrynvfsheoxjcuzsjdgiqfibxta` int DEFAULT NULL,\n `ujzoqrepqnhhcvliswcofuuheekpixsj` int DEFAULT NULL,\n `oqitiarqkwtjdryipscbuitdqlquphoi` int DEFAULT NULL,\n `cjlwuwglrukhhixjffojxcvzfqwdwbol` int DEFAULT NULL,\n `tnucmqimfamfhtyyxeqrxbnwgoulolyk` int DEFAULT NULL,\n `jducczqxqcqtlegkqahgombdmtnycstu` int DEFAULT NULL,\n `ygddoaxlthpbbkvhugmzlzbpsqelauut` int DEFAULT NULL,\n `opqmvboojoulbwtnenxawpsqbuiwfams` int DEFAULT NULL,\n `hfpgpuefkcxdyacwidrugpslahtxzcsy` int DEFAULT NULL,\n PRIMARY KEY (`qzoyzniqxgxucwlppvmwidiqhbzwluaw`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"adhcfqcfumxvmsokmylceoltlqhjewvw\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["qzoyzniqxgxucwlppvmwidiqhbzwluaw"],"columns":[{"name":"qzoyzniqxgxucwlppvmwidiqhbzwluaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"qjflcvzkqdrmgslqpmokgsqxibbaxxin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"miaevkbjzhybaieljrjfbdnuldjsotik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"osajqtaxzudfeifxdwbdzglsxsmniiqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkmfacdltcspqoqseokzfyujqdncnprv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orrhcyiqbzuhauuvdluwkikbhbthcqtw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzmjlkycvjcacqqaygcfffsusvroccpw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnzfnjjllhrumjjwqgcmxzybqffwldik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lumdqvewlffvfjdrovliwhlwnsrjsirf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdhcqmbwyfwyqewbsiuasjfpxruiumiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akkwvbwtwjteauvqixeymnghmjftpjlq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtzaprddthraxupgdbhgzgwbwbhuluin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxajlbdnqkwotucynpkxnutjykrbluwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvevocmdrbdzgabjbebdjwockopsvqzu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqsljvjgxysisprnghcwxhnqhodpgusq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vipbckzmnzmiehjfoanhoysuqczmylxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"madyjspdkxpbdxnaawtzpnavkchinzqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jrednxitbdrozupwgsesttxeshfekdcp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uepgrtfclojkomyrppgwcrdhnpcygmvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jeqlgakeznvpuewvvsanwsigngikwvjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ianweezzfguayslautdlasuyqpkdlkak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqzxvnquwiafejzuryszsiejgsgvngud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hihbnjosxcpailclmbvsqctsaohuigve","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwlqlpeiravrehzunzadqjbknyzzfxix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnduxvchezzptqqisoyetsjjgbdepahu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obzredhsoeeygarmsejtguiqjvkvpfqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phdruiredjlqfoorpioxarasjfjdbbzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcvlhvuedkkprsemeambfjgubmxwwvtr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ryypvgpispuamsuwuoymdyaqirjkrytb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbzihronfnuxbhlyfiijeycsitghzlhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqbgybwowkekauiexwciyhpwjcjbqwfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wucwzydpvgmhurqosdyrhnjlvunrcpnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"irwrqtnrxepmtazekgobevtzdekwqcdt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbrfxbfwbytsxrypirqgupikdpongdru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"plyxpyoxeeluzufkeiycpkdlvudtxcke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ktctzhrcpctmfnhfvomfacccuayggjrk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phhwmuzkybklrdtftijlzorxqftckysq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rmtrpkhtleooqrjwglymvabaehecoezr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jhidlhzxkxtttbgrbfprgywbrxnbtwdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvsoxmajsiabgnbxjwkzzwaohzwrscfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"giefvidgsufbhsptdbajweklueomonwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybtpalvukdukiwokipmqszsapcleggvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pigrylqkoipxjucyzpwvczllxoefzcju","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wffhdikhbcecwbpafcrfrkuarqzcofnn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lnbhoonnearqdypfevgjebzxavfmajwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmerypukceasusyulywmkjlmqyfkedox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wsnlncrzznwoagyhlvcoczezblxkgyfm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hppvidbclmsojammofywhsnxvbyycuea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbnpydykvebozntkwawozpfhcakqyjjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nsijpmaetezxopeoqazmfjahkwknmybg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lbdueihxjejjtwxfhznfmdimwbuqvxmd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uawidvpfcbxkuvdgoemsndqfbdcjucyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tudjeophngkefltgurcmnnjhytnjokal","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"imaghngnhaasyvjpjgswbbptqkdbejfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"efhkmdxshqrzycxqwggebmlvxgsxqmse","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlhqybvmnueagojlovfaqlqsdlymzjyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pegqljorjmkgxxujobjyaryjqordnhps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hitzilteauemrojypigtazteezbzljaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvfulsrfldtgwqdeailkayxddmqsjndz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kobyaixcgsehvoktwftlukhptghqdfmw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnmaovdehitmkqnrlnkrazdhgzytqmpz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"naekwvqfilsrfprdkgrbpbomehpwrmlh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqztkpxnwrnpergdmbqnadmcmxpovwig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjgthcavoyjqjpwhdiatajoxrfecmffg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqyeojazxwzrhwtnxdifgoiaefukarjh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcrorblbrtczkrxovprzlnicdcrromfa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dubxgpytstlrhowzpwhwxcmykaciypwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcbleuhwonmespgzvlzvnrwdgxfmvjyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lpyslqycmlrysbphevvtbmpdiofojdoc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pcqbabsfenmofeutxmeviozephodetuh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txckjrznygbcizfpguwtqzwjswvnwhvs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"muvzvwamzesksctcdratviqxqcmlxbsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rohvdvtednpythqbgsbnkwyumyfoofnn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebgqexmdosfpqpjhqaxgpptewdzskjaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"isffnzredpjxzedwfkfapvvlsexseywh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yprsvwjlgxjqgmhlttmomychprvuiytz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"emwvprfzfibfhqsmnafmmkeymexyjgyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpesebococsxiyhilsxtieisqpnisbbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bijokuctnphsdqfnujbzjigaoalotrrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eoscggorgukquowcygejlcsxpdpbzhdl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zsjltqypqkvnlbidviylkhbgkzfryjdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvflfdbywxcqrnovyjkecvnbqravstfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qylogijeqsnutuimclgirdvtvxozxbrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzyftuzkwqoobyryywbrtfebfxpcbmmy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skrxsmgtommgwjvizlpwlvxbebenlmzh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qomstdxmlussdtxmsbbgbtjuxpvpjzhx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vcdspgrvodyrykpessvmpkjbjdnshfep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qelhapahqjzgeggamindpfduagbwcguq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eatldpqpwomrdrpuzwtriwvyzlwfjidr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"auokotmaedvfqtkczyxbnlufzcrzoqki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hcelyynexdxiwiwsappbqzwrsxbdnqyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtemxlrynvfsheoxjcuzsjdgiqfibxta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujzoqrepqnhhcvliswcofuuheekpixsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqitiarqkwtjdryipscbuitdqlquphoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjlwuwglrukhhixjffojxcvzfqwdwbol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnucmqimfamfhtyyxeqrxbnwgoulolyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jducczqxqcqtlegkqahgombdmtnycstu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ygddoaxlthpbbkvhugmzlzbpsqelauut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opqmvboojoulbwtnenxawpsqbuiwfams","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfpgpuefkcxdyacwidrugpslahtxzcsy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666618,"databaseName":"models_schema","ddl":"CREATE TABLE `ageaxrapcifxppadtgfmvnubppgguatn` (\n `qyinjfnceqazkknbpstgjzpjujmvebeo` int NOT NULL,\n `qddlbtmrcbyzeezuubpeuwzklfddbcdq` int DEFAULT NULL,\n `hlppttlpirekmywkprgfdkqgropkdktu` int DEFAULT NULL,\n `aokxtpcjptuhgywkroamzxyunlrbtcng` int DEFAULT NULL,\n `nbvdtubjnlpiqkupgaencxpcjlkpqgta` int DEFAULT NULL,\n `tyyyfmyvqrewpxrukfeqelsvfecpiufk` int DEFAULT NULL,\n `oeqzxpqtssbrldxtuoyfeaekijnmryax` int DEFAULT NULL,\n `pkderrzvtiztriufexwxzqgmbbjlxjld` int DEFAULT NULL,\n `iuuahvtfyghhgcjpoqowvjgcxnsnyclo` int DEFAULT NULL,\n `wutknilphonosinwfmnffbtnuqquzlvr` int DEFAULT NULL,\n `byhkgtblirrvkvqmouiiyklvnqrnznbs` int DEFAULT NULL,\n `bhimzirqeexwresfdildjpuixeawagdl` int DEFAULT NULL,\n `gggfpibqfpghtedeviuddruacoitmyna` int DEFAULT NULL,\n `fqhypzxacbgocsyxguedcjxphhrmzlfk` int DEFAULT NULL,\n `jgiywgogcicdtmghtspqdenbeykcgjvj` int DEFAULT NULL,\n `achlydssnztfgnbkzcjtapnxiqooxzez` int DEFAULT NULL,\n `mcvdapmhmipfkedxvfqubjuaajadukjx` int DEFAULT NULL,\n `upgnvavcmfkytcoaujqdrrwqutcglxpg` int DEFAULT NULL,\n `ajncildyfdqzbjubwcgiymyhelagzgsp` int DEFAULT NULL,\n `thhijijxsukeofrlcstiynipniytcnvm` int DEFAULT NULL,\n `tzydpnydzswihzztbddpnrccpxkmrjtd` int DEFAULT NULL,\n `vmwnpcmmjyvqsnrjlbiainqyzlasxvwd` int DEFAULT NULL,\n `mrqbzwdilwmsgyojzkrdotwibosstuco` int DEFAULT NULL,\n `baynojcfuumdkeswbmtkvybjtvagsaty` int DEFAULT NULL,\n `vnbvqsfaencigqwijifzsunpssqivphd` int DEFAULT NULL,\n `ljfuwugvpcdeipuzadyskaingrhsphtd` int DEFAULT NULL,\n `wrfdcvtdmxtcbhxmewonyfmmlucajtxa` int DEFAULT NULL,\n `boxjzekuufidnkjgmzzetfpydhqlhrxh` int DEFAULT NULL,\n `jwwrlakwonazxevbofztvcnquisqpfkm` int DEFAULT NULL,\n `xdtgawfcixpiuqyaobriigvfwxdovxbn` int DEFAULT NULL,\n `rqoinqtulrwpxkcvyiwxbqsvpykbkeht` int DEFAULT NULL,\n `kcmguhbyknqipjjogmxhdvgxdiqlsbkd` int DEFAULT NULL,\n `tdvduknoujmodnikvevgdrwoocgbngsc` int DEFAULT NULL,\n `ebbigomagkopjjarcczcwugfqvpxonsd` int DEFAULT NULL,\n `vuyfrfmsetwpwcvaobgxorjpyfeejgww` int DEFAULT NULL,\n `mrkiuwlqwnankskuzlfvrcvfwkzujsic` int DEFAULT NULL,\n `kuoaaqiuoknsinlphhiloifhaxzdkozd` int DEFAULT NULL,\n `nibvavanuyhdcpvkbwgtearppmvzhsve` int DEFAULT NULL,\n `mmltqrmtqrngyrvvebtpmsfrewqgwzne` int DEFAULT NULL,\n `xuqlhfaifptcxtztjviwzmrvcicabjqx` int DEFAULT NULL,\n `zklexkgeoigtfgipjvtfadordwwoxkrj` int DEFAULT NULL,\n `hxlchjlrkjcxrsibaianeaxzzffoutub` int DEFAULT NULL,\n `ivvxgxtfyaooqpbumfapntcsxezzkrjv` int DEFAULT NULL,\n `rukvffhdcdclzcjaryyvoaveoerbcxjv` int DEFAULT NULL,\n `cztyirbzqersrfuzplmjbtukxwqynadm` int DEFAULT NULL,\n `tpoeijwhzmqvpslfqifqvsetesbzuyou` int DEFAULT NULL,\n `wcjovwqexuouigiyrajkvtrhnqohdsjd` int DEFAULT NULL,\n `cqgpzlrryxrrxgwsbyvckwjihyczmcwj` int DEFAULT NULL,\n `sadugcxdykmomynzrlztmkqguhbmgimv` int DEFAULT NULL,\n `uljjaliagnbzpucgvsucfcoqyvbnuqyg` int DEFAULT NULL,\n `pmwhulqrzqzpqczttkneqfzerqlfjlen` int DEFAULT NULL,\n `gpldnhxsnzhuraoykiykyfwpdmcqycbw` int DEFAULT NULL,\n `jgpqwsmmtqytrlyuptoltpuanknxkzfg` int DEFAULT NULL,\n `kfatxdjwrcdtfqweohszskkrwgiztrsl` int DEFAULT NULL,\n `uzkxrivciydjpcaxjlbkrpskwkvyoptp` int DEFAULT NULL,\n `iirwidvjqkfkjwjzxnrzyuvybcemybdx` int DEFAULT NULL,\n `qlqystwkwgxudgnezcfrworbgwnoqjgf` int DEFAULT NULL,\n `cfcqgjjtrnpsskpgqsqrjjcsqgrpvxtq` int DEFAULT NULL,\n `ckreoebcxtyekeubjeuebxijekyvlcqq` int DEFAULT NULL,\n `mtfdgildqfestoecerssjglthibqcmvn` int DEFAULT NULL,\n `wfhvvagbvelfuihhjgttjwcrskydwhho` int DEFAULT NULL,\n `hnjcgnpvbpjaxpijjgzxlpdpezyidpfs` int DEFAULT NULL,\n `flvqdfwsxvyomuvzmgrpclgaescueljp` int DEFAULT NULL,\n `bcxhexsvaxjsreynpeypylofvxkqdaub` int DEFAULT NULL,\n `rscszxmjfdttuwikyweaefwmhbqcpigz` int DEFAULT NULL,\n `inqzzemtuvlwbyaumarztogdvilsuuhf` int DEFAULT NULL,\n `bxukwkiiwlbvnasbjmfuomlpgftcnhkt` int DEFAULT NULL,\n `dipowrysaochjmzhkhkzbicjwxppwytm` int DEFAULT NULL,\n `heaecwrygumzcbfkdecofsuikamnfefa` int DEFAULT NULL,\n `fjpehkhbysxtlxmmtzadtgybgagxmlcp` int DEFAULT NULL,\n `ahqyfecencbsaxtzjrwgpbsivyzjxcqp` int DEFAULT NULL,\n `ywbvbxaewwjsqllcxzuzwdnmfdznpczh` int DEFAULT NULL,\n `ewsfikvaubavnzbihbylsbjjeknmsqxj` int DEFAULT NULL,\n `benfhuvgsotizfjlwllnzuhlpvkydgom` int DEFAULT NULL,\n `smrzhcdmlbkuwpdneialliwcspezxome` int DEFAULT NULL,\n `tbzxatxqaekiiiailbtepqtmahugngix` int DEFAULT NULL,\n `mtwoonjpmqsdvozfbnmkmmzwqupvcdxl` int DEFAULT NULL,\n `tlovjrmzssatxppjkczbnlbxnuwfhwqf` int DEFAULT NULL,\n `tkgrpqhvtskpefshkhbrbwfgkmcuwojv` int DEFAULT NULL,\n `ydxnjklpxrxoygmatrypxrumejldpqfi` int DEFAULT NULL,\n `zgmshdsboosuwrxgjspzebshcqbxvodp` int DEFAULT NULL,\n `ivqbbcbiqwszaiovkbmildwcexjfyxtd` int DEFAULT NULL,\n `vojwaaciroblayvwytctbdrpxbmjlnuk` int DEFAULT NULL,\n `btjgfziihxzfkpsurcrvfneicvwktdde` int DEFAULT NULL,\n `nidonijmseedeobvsxtvldqcvifndbvt` int DEFAULT NULL,\n `exhzuechereudkuzeptlznnwzubgsqyd` int DEFAULT NULL,\n `ytnvsubernleinrpkdkxhthxdqdggirt` int DEFAULT NULL,\n `iuxdmckheqqaybqzkzxrmvcemcqymjsv` int DEFAULT NULL,\n `mmdpeuqgixkqcirlqfcxpxbnjsajfzhz` int DEFAULT NULL,\n `kzjavqtanpmihxdsuzigamoonhnqytpo` int DEFAULT NULL,\n `rqrnepipocdvuweizovwsijjombijcag` int DEFAULT NULL,\n `ntvllwgzynecpydetefkimzeqifzpfbx` int DEFAULT NULL,\n `lggutlxsxpqfvnfhqdirilrcvvoscszj` int DEFAULT NULL,\n `ezwgawdgxejsabqahrmnfmdkfyjvrhtg` int DEFAULT NULL,\n `kpxazfpwjiagegqsltynlzjepubkziow` int DEFAULT NULL,\n `yshwnjagvkyifcdaguieiopgvxgbqudo` int DEFAULT NULL,\n `solyxcxlmsekwtzeuquxpurlydchzgyd` int DEFAULT NULL,\n `tfzsouyifuntysiauvcedndmfsyfapje` int DEFAULT NULL,\n `znfozjczokdydcsjapoytvoreofsjfvv` int DEFAULT NULL,\n `mgahpgkeamvayaiqmrcjfpsejabxzkgm` int DEFAULT NULL,\n PRIMARY KEY (`qyinjfnceqazkknbpstgjzpjujmvebeo`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"ageaxrapcifxppadtgfmvnubppgguatn\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["qyinjfnceqazkknbpstgjzpjujmvebeo"],"columns":[{"name":"qyinjfnceqazkknbpstgjzpjujmvebeo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"qddlbtmrcbyzeezuubpeuwzklfddbcdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlppttlpirekmywkprgfdkqgropkdktu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aokxtpcjptuhgywkroamzxyunlrbtcng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbvdtubjnlpiqkupgaencxpcjlkpqgta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tyyyfmyvqrewpxrukfeqelsvfecpiufk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oeqzxpqtssbrldxtuoyfeaekijnmryax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkderrzvtiztriufexwxzqgmbbjlxjld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iuuahvtfyghhgcjpoqowvjgcxnsnyclo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wutknilphonosinwfmnffbtnuqquzlvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"byhkgtblirrvkvqmouiiyklvnqrnznbs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bhimzirqeexwresfdildjpuixeawagdl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gggfpibqfpghtedeviuddruacoitmyna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqhypzxacbgocsyxguedcjxphhrmzlfk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgiywgogcicdtmghtspqdenbeykcgjvj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"achlydssnztfgnbkzcjtapnxiqooxzez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcvdapmhmipfkedxvfqubjuaajadukjx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upgnvavcmfkytcoaujqdrrwqutcglxpg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajncildyfdqzbjubwcgiymyhelagzgsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thhijijxsukeofrlcstiynipniytcnvm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tzydpnydzswihzztbddpnrccpxkmrjtd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmwnpcmmjyvqsnrjlbiainqyzlasxvwd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrqbzwdilwmsgyojzkrdotwibosstuco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"baynojcfuumdkeswbmtkvybjtvagsaty","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnbvqsfaencigqwijifzsunpssqivphd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljfuwugvpcdeipuzadyskaingrhsphtd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wrfdcvtdmxtcbhxmewonyfmmlucajtxa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"boxjzekuufidnkjgmzzetfpydhqlhrxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwwrlakwonazxevbofztvcnquisqpfkm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdtgawfcixpiuqyaobriigvfwxdovxbn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqoinqtulrwpxkcvyiwxbqsvpykbkeht","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcmguhbyknqipjjogmxhdvgxdiqlsbkd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdvduknoujmodnikvevgdrwoocgbngsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebbigomagkopjjarcczcwugfqvpxonsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vuyfrfmsetwpwcvaobgxorjpyfeejgww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrkiuwlqwnankskuzlfvrcvfwkzujsic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kuoaaqiuoknsinlphhiloifhaxzdkozd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nibvavanuyhdcpvkbwgtearppmvzhsve","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmltqrmtqrngyrvvebtpmsfrewqgwzne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xuqlhfaifptcxtztjviwzmrvcicabjqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zklexkgeoigtfgipjvtfadordwwoxkrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxlchjlrkjcxrsibaianeaxzzffoutub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivvxgxtfyaooqpbumfapntcsxezzkrjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rukvffhdcdclzcjaryyvoaveoerbcxjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cztyirbzqersrfuzplmjbtukxwqynadm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpoeijwhzmqvpslfqifqvsetesbzuyou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcjovwqexuouigiyrajkvtrhnqohdsjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqgpzlrryxrrxgwsbyvckwjihyczmcwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sadugcxdykmomynzrlztmkqguhbmgimv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uljjaliagnbzpucgvsucfcoqyvbnuqyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmwhulqrzqzpqczttkneqfzerqlfjlen","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gpldnhxsnzhuraoykiykyfwpdmcqycbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgpqwsmmtqytrlyuptoltpuanknxkzfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfatxdjwrcdtfqweohszskkrwgiztrsl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzkxrivciydjpcaxjlbkrpskwkvyoptp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iirwidvjqkfkjwjzxnrzyuvybcemybdx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qlqystwkwgxudgnezcfrworbgwnoqjgf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfcqgjjtrnpsskpgqsqrjjcsqgrpvxtq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckreoebcxtyekeubjeuebxijekyvlcqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtfdgildqfestoecerssjglthibqcmvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfhvvagbvelfuihhjgttjwcrskydwhho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hnjcgnpvbpjaxpijjgzxlpdpezyidpfs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flvqdfwsxvyomuvzmgrpclgaescueljp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bcxhexsvaxjsreynpeypylofvxkqdaub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rscszxmjfdttuwikyweaefwmhbqcpigz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"inqzzemtuvlwbyaumarztogdvilsuuhf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bxukwkiiwlbvnasbjmfuomlpgftcnhkt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dipowrysaochjmzhkhkzbicjwxppwytm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"heaecwrygumzcbfkdecofsuikamnfefa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjpehkhbysxtlxmmtzadtgybgagxmlcp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahqyfecencbsaxtzjrwgpbsivyzjxcqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywbvbxaewwjsqllcxzuzwdnmfdznpczh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ewsfikvaubavnzbihbylsbjjeknmsqxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"benfhuvgsotizfjlwllnzuhlpvkydgom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smrzhcdmlbkuwpdneialliwcspezxome","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbzxatxqaekiiiailbtepqtmahugngix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtwoonjpmqsdvozfbnmkmmzwqupvcdxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlovjrmzssatxppjkczbnlbxnuwfhwqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkgrpqhvtskpefshkhbrbwfgkmcuwojv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydxnjklpxrxoygmatrypxrumejldpqfi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zgmshdsboosuwrxgjspzebshcqbxvodp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivqbbcbiqwszaiovkbmildwcexjfyxtd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vojwaaciroblayvwytctbdrpxbmjlnuk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"btjgfziihxzfkpsurcrvfneicvwktdde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nidonijmseedeobvsxtvldqcvifndbvt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exhzuechereudkuzeptlznnwzubgsqyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytnvsubernleinrpkdkxhthxdqdggirt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iuxdmckheqqaybqzkzxrmvcemcqymjsv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmdpeuqgixkqcirlqfcxpxbnjsajfzhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzjavqtanpmihxdsuzigamoonhnqytpo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqrnepipocdvuweizovwsijjombijcag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntvllwgzynecpydetefkimzeqifzpfbx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lggutlxsxpqfvnfhqdirilrcvvoscszj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezwgawdgxejsabqahrmnfmdkfyjvrhtg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpxazfpwjiagegqsltynlzjepubkziow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yshwnjagvkyifcdaguieiopgvxgbqudo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"solyxcxlmsekwtzeuquxpurlydchzgyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tfzsouyifuntysiauvcedndmfsyfapje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znfozjczokdydcsjapoytvoreofsjfvv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mgahpgkeamvayaiqmrcjfpsejabxzkgm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666681,"databaseName":"models_schema","ddl":"CREATE TABLE `agqfberalhewuryqncmijexehjvpvhya` (\n `ghlaytatnnqczpiucxvavundreobomqn` int NOT NULL,\n `fmmnbptbktdcaohfdhqxngykphdfawlx` int DEFAULT NULL,\n `cngmneopumtuelexwvmnjvfiylvujesm` int DEFAULT NULL,\n `kuhrqtakdtiicfdpbdaussundwhmeayt` int DEFAULT NULL,\n `edvqgvkfzqpqdpredxdakiwovbvpuzhg` int DEFAULT NULL,\n `wznfechuyulgauqzgrjuvagkenfpnuzc` int DEFAULT NULL,\n `bfjairfmwfdvkvgejszcwxnylwnresky` int DEFAULT NULL,\n `vxoljqbwlcpoogsnhyqbsrvfaorzsifs` int DEFAULT NULL,\n `hhxdeubiumizyktmeduofhwzueqwvrgd` int DEFAULT NULL,\n `bjlvrldmhiarttvugsqhlngelowjboqw` int DEFAULT NULL,\n `yrdoamfqtxoaiwvgshupwspizikluehg` int DEFAULT NULL,\n `fxannboqvkugybbrzyiodiwidyuajgcb` int DEFAULT NULL,\n `cqohjkvkykzeqqbituzrenmniourhgcb` int DEFAULT NULL,\n `rvvfcqktdxunpumonkmcaguwckievuvp` int DEFAULT NULL,\n `cyomcousmpqmtakynisfphrfqcuxuubv` int DEFAULT NULL,\n `pkduoggmbsjjfwuepugeaanmhuxnesni` int DEFAULT NULL,\n `znxvjarspjjaiiejxuqwtdtgtzohgkeq` int DEFAULT NULL,\n `ghbgvzcmtnoricennmqiheyjxihdvktk` int DEFAULT NULL,\n `mexatrepvbfejlorweyvdvtczuidtiyt` int DEFAULT NULL,\n `krahsiflbvryimprpiqklrdghfisrrkx` int DEFAULT NULL,\n `qrfzmtzoekqtszylnznhnuizdmgcymlg` int DEFAULT NULL,\n `leqxtynuubrlpvwypdmtkyqkymdncbll` int DEFAULT NULL,\n `jrtgrwjsjpxweqnhpbffgqeuffgacnfb` int DEFAULT NULL,\n `rqnlnwfskrmwjzybjalknwvgnvnuinby` int DEFAULT NULL,\n `eosiiponnfegfktdlgcwouourismmmxs` int DEFAULT NULL,\n `xxemzqyyhmptezfhdrlapknucssaxqkn` int DEFAULT NULL,\n `gbkxabgfsfnrbsurhsvgcdrhoyepuytf` int DEFAULT NULL,\n `ucsrgntehasgjaqrmkcscusrnzosvtyf` int DEFAULT NULL,\n `lisnuuzassadnbkakrleqsncbhrufxho` int DEFAULT NULL,\n `nofypyambhauokumkvbczdqlonnipvir` int DEFAULT NULL,\n `aysryyyzebrgjxlblsvfpiaitmsppslx` int DEFAULT NULL,\n `urnyibtslqdpxqmhpfoqtwqmlyxrfqru` int DEFAULT NULL,\n `scftcupbvylsyvicxkbesahysklqpxpu` int DEFAULT NULL,\n `qngnasxfbxvhaibcmxcxceicoqwvnqie` int DEFAULT NULL,\n `eumvxnjudfgsrylmbcmlrwbfpnodlxnu` int DEFAULT NULL,\n `ecisloknrjzrcsjvvrccawzdqxvqrone` int DEFAULT NULL,\n `gjqvqjqjgayvbbifhrrokkkrazwpdhju` int DEFAULT NULL,\n `osmxlzcbkovvidyuffknelwuhtirzilw` int DEFAULT NULL,\n `chuszipiatvrlfnxqpffgcttfrfayplh` int DEFAULT NULL,\n `qvdqihniejiddgrpvxjfypfuhiwodwbo` int DEFAULT NULL,\n `uxihyomakenwoaqyjnndndvmhyvjsbea` int DEFAULT NULL,\n `nfspylhcncdfdryxuitvbfgibownogfc` int DEFAULT NULL,\n `rmskdgxfazmqgyilggfmrvkikblmiied` int DEFAULT NULL,\n `iwybdimarwlnazsvcjvpwmjqegjopwad` int DEFAULT NULL,\n `ehbozjrvjehhobybfhiegzxsgktbnkxj` int DEFAULT NULL,\n `nawvlizrnjlpmkkyrkyhswmxwzatbhdi` int DEFAULT NULL,\n `mvbyrfavnuoiyrnlubehouviitupzncx` int DEFAULT NULL,\n `qbgqiiogvwjzhnjloegfnfrxvkjiizdm` int DEFAULT NULL,\n `inqibysnqsannwlvhfaulnkyjpqmbbah` int DEFAULT NULL,\n `aqmrjizmmkxxuchvaubrzhewzcciapgw` int DEFAULT NULL,\n `imwqmwxhphrefrwacjwtzduiwbpuzgoq` int DEFAULT NULL,\n `oamxicuwhqhfkhhmszsoawytvaclrjcz` int DEFAULT NULL,\n `odprbxeknghqoqtmqccbscyuweissecx` int DEFAULT NULL,\n `hildfbwbudlshduchntqmoaorvlzkigl` int DEFAULT NULL,\n `dxisnwcbjjpxqgwtydrsffdeurhlwaui` int DEFAULT NULL,\n `ungtlexyeqaetdfkkbnwrjclebsxzdww` int DEFAULT NULL,\n `tohywacjybpujmuuxafhovkuzscbknpe` int DEFAULT NULL,\n `bqgwscxutockpteiyfzbpwygqyovmtzs` int DEFAULT NULL,\n `lnpkhhwnjzgmhbcnkxpciexaqpysirds` int DEFAULT NULL,\n `hmgaoetgdrnqbfeoswsjguohqrwvsrop` int DEFAULT NULL,\n `oplismtambzazchytpparggtphylscaf` int DEFAULT NULL,\n `cvnosujqtnefmxfqiodqfjrdtuewvmxr` int DEFAULT NULL,\n `fetjvgqeaojheztnkiqxbxrsyrorkijo` int DEFAULT NULL,\n `axrnjvwxscxdsxxclibeeivspbmcluvd` int DEFAULT NULL,\n `ghemdwtwtfvwgqgkmuqchtroomsctfke` int DEFAULT NULL,\n `cxkaitwzdeztuwjmqtepqikwiqgopttq` int DEFAULT NULL,\n `breyydhevroyhqpbpenddzylkcraztoe` int DEFAULT NULL,\n `subnvjlvmphmufehyhvrhkkklesnxpaa` int DEFAULT NULL,\n `zahgmgddseslpjqnpmxsedfkjgpisrxy` int DEFAULT NULL,\n `urrapttugmtqxxbazypcscebkqgyrbpq` int DEFAULT NULL,\n `yysavozyhiskdrwmweasiegcdrgplvbb` int DEFAULT NULL,\n `asgdycpihnbyveysvqhogwcvoxxrntop` int DEFAULT NULL,\n `ayqnncqacdqnyrffvobkdoesvdsemlvp` int DEFAULT NULL,\n `dturzvbcbnxcummkwvqeakggrkxjkoec` int DEFAULT NULL,\n `fadxjnftvtnpnubaxgweuhjzirpnwovi` int DEFAULT NULL,\n `ennhyqoqxuttomnzyymmpvibgapallyz` int DEFAULT NULL,\n `lcilfummvvjmosqnhwpijfszfansuedv` int DEFAULT NULL,\n `okshwqmbsusohtxhjkyepkhwxjkpizwd` int DEFAULT NULL,\n `gtojzccijwrejyaugawyrjsekdgwsxfl` int DEFAULT NULL,\n `pkoohavmmvdjbmbuzwospiykzkaemekp` int DEFAULT NULL,\n `kkwggrrtvxwwnjiinjbweytjlszkcrmw` int DEFAULT NULL,\n `vblygbgewvxyuulswbqzdrcvhhyunkky` int DEFAULT NULL,\n `vziqzuamksplfgcryvnbnqspqfwddvla` int DEFAULT NULL,\n `wrtiaxaxsofjosrzopyeptymfjtpjsna` int DEFAULT NULL,\n `sfwpnbefxwfrxyijpybxknfuuvizdvun` int DEFAULT NULL,\n `uytqroswcnxtqkjtvcjbohtkfmuxscqz` int DEFAULT NULL,\n `fsmcwputgnrtqlycqahjvzatqnkspnjf` int DEFAULT NULL,\n `xxpeekuqxvbfkgxjwsljyqepokbpiull` int DEFAULT NULL,\n `vmbutoojjpzwljtuzmqqpifwietfyifd` int DEFAULT NULL,\n `thsfxteorzqrveedfpsdetoblnppmqdz` int DEFAULT NULL,\n `yqomjiywkqzebuxcvowgatjgrrjdwdjw` int DEFAULT NULL,\n `dsrdhnupwuwnsccmlotrsdcmbapiyofw` int DEFAULT NULL,\n `chwgvhyicgykvdycctdbpzkytpcfgntp` int DEFAULT NULL,\n `tycswuwbzjxhchrzuerlonwpghsojeal` int DEFAULT NULL,\n `drwqldfiwwwrhirsaqpsqczpjnviugyk` int DEFAULT NULL,\n `vgprkgkdmnyrvmvkhgqjfodpfmnixrrr` int DEFAULT NULL,\n `ghqjfwhzuztkgzhbdsptaedevyhnyebz` int DEFAULT NULL,\n `yrlacmnfsrvaonmmixhuwobecreieism` int DEFAULT NULL,\n `qtwcnykeygmwkjqbtrywmaydqbdiiqcs` int DEFAULT NULL,\n `sjogagperuupbadbfzmapnjjzpviigjs` int DEFAULT NULL,\n PRIMARY KEY (`ghlaytatnnqczpiucxvavundreobomqn`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"agqfberalhewuryqncmijexehjvpvhya\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["ghlaytatnnqczpiucxvavundreobomqn"],"columns":[{"name":"ghlaytatnnqczpiucxvavundreobomqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"fmmnbptbktdcaohfdhqxngykphdfawlx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cngmneopumtuelexwvmnjvfiylvujesm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kuhrqtakdtiicfdpbdaussundwhmeayt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edvqgvkfzqpqdpredxdakiwovbvpuzhg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wznfechuyulgauqzgrjuvagkenfpnuzc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bfjairfmwfdvkvgejszcwxnylwnresky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxoljqbwlcpoogsnhyqbsrvfaorzsifs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhxdeubiumizyktmeduofhwzueqwvrgd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjlvrldmhiarttvugsqhlngelowjboqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yrdoamfqtxoaiwvgshupwspizikluehg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxannboqvkugybbrzyiodiwidyuajgcb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqohjkvkykzeqqbituzrenmniourhgcb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rvvfcqktdxunpumonkmcaguwckievuvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cyomcousmpqmtakynisfphrfqcuxuubv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkduoggmbsjjfwuepugeaanmhuxnesni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znxvjarspjjaiiejxuqwtdtgtzohgkeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghbgvzcmtnoricennmqiheyjxihdvktk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mexatrepvbfejlorweyvdvtczuidtiyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krahsiflbvryimprpiqklrdghfisrrkx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrfzmtzoekqtszylnznhnuizdmgcymlg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"leqxtynuubrlpvwypdmtkyqkymdncbll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jrtgrwjsjpxweqnhpbffgqeuffgacnfb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqnlnwfskrmwjzybjalknwvgnvnuinby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eosiiponnfegfktdlgcwouourismmmxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxemzqyyhmptezfhdrlapknucssaxqkn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gbkxabgfsfnrbsurhsvgcdrhoyepuytf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucsrgntehasgjaqrmkcscusrnzosvtyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lisnuuzassadnbkakrleqsncbhrufxho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nofypyambhauokumkvbczdqlonnipvir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aysryyyzebrgjxlblsvfpiaitmsppslx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urnyibtslqdpxqmhpfoqtwqmlyxrfqru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scftcupbvylsyvicxkbesahysklqpxpu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qngnasxfbxvhaibcmxcxceicoqwvnqie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eumvxnjudfgsrylmbcmlrwbfpnodlxnu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecisloknrjzrcsjvvrccawzdqxvqrone","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjqvqjqjgayvbbifhrrokkkrazwpdhju","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"osmxlzcbkovvidyuffknelwuhtirzilw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chuszipiatvrlfnxqpffgcttfrfayplh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvdqihniejiddgrpvxjfypfuhiwodwbo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxihyomakenwoaqyjnndndvmhyvjsbea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfspylhcncdfdryxuitvbfgibownogfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rmskdgxfazmqgyilggfmrvkikblmiied","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwybdimarwlnazsvcjvpwmjqegjopwad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehbozjrvjehhobybfhiegzxsgktbnkxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nawvlizrnjlpmkkyrkyhswmxwzatbhdi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvbyrfavnuoiyrnlubehouviitupzncx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qbgqiiogvwjzhnjloegfnfrxvkjiizdm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"inqibysnqsannwlvhfaulnkyjpqmbbah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqmrjizmmkxxuchvaubrzhewzcciapgw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"imwqmwxhphrefrwacjwtzduiwbpuzgoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oamxicuwhqhfkhhmszsoawytvaclrjcz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"odprbxeknghqoqtmqccbscyuweissecx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hildfbwbudlshduchntqmoaorvlzkigl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxisnwcbjjpxqgwtydrsffdeurhlwaui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ungtlexyeqaetdfkkbnwrjclebsxzdww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tohywacjybpujmuuxafhovkuzscbknpe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqgwscxutockpteiyfzbpwygqyovmtzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lnpkhhwnjzgmhbcnkxpciexaqpysirds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmgaoetgdrnqbfeoswsjguohqrwvsrop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oplismtambzazchytpparggtphylscaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvnosujqtnefmxfqiodqfjrdtuewvmxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fetjvgqeaojheztnkiqxbxrsyrorkijo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axrnjvwxscxdsxxclibeeivspbmcluvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghemdwtwtfvwgqgkmuqchtroomsctfke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cxkaitwzdeztuwjmqtepqikwiqgopttq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"breyydhevroyhqpbpenddzylkcraztoe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"subnvjlvmphmufehyhvrhkkklesnxpaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zahgmgddseslpjqnpmxsedfkjgpisrxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urrapttugmtqxxbazypcscebkqgyrbpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yysavozyhiskdrwmweasiegcdrgplvbb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"asgdycpihnbyveysvqhogwcvoxxrntop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ayqnncqacdqnyrffvobkdoesvdsemlvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dturzvbcbnxcummkwvqeakggrkxjkoec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fadxjnftvtnpnubaxgweuhjzirpnwovi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ennhyqoqxuttomnzyymmpvibgapallyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lcilfummvvjmosqnhwpijfszfansuedv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okshwqmbsusohtxhjkyepkhwxjkpizwd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtojzccijwrejyaugawyrjsekdgwsxfl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkoohavmmvdjbmbuzwospiykzkaemekp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kkwggrrtvxwwnjiinjbweytjlszkcrmw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vblygbgewvxyuulswbqzdrcvhhyunkky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vziqzuamksplfgcryvnbnqspqfwddvla","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wrtiaxaxsofjosrzopyeptymfjtpjsna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfwpnbefxwfrxyijpybxknfuuvizdvun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uytqroswcnxtqkjtvcjbohtkfmuxscqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fsmcwputgnrtqlycqahjvzatqnkspnjf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxpeekuqxvbfkgxjwsljyqepokbpiull","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmbutoojjpzwljtuzmqqpifwietfyifd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thsfxteorzqrveedfpsdetoblnppmqdz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqomjiywkqzebuxcvowgatjgrrjdwdjw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsrdhnupwuwnsccmlotrsdcmbapiyofw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chwgvhyicgykvdycctdbpzkytpcfgntp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tycswuwbzjxhchrzuerlonwpghsojeal","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drwqldfiwwwrhirsaqpsqczpjnviugyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgprkgkdmnyrvmvkhgqjfodpfmnixrrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghqjfwhzuztkgzhbdsptaedevyhnyebz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yrlacmnfsrvaonmmixhuwobecreieism","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtwcnykeygmwkjqbtrywmaydqbdiiqcs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjogagperuupbadbfzmapnjjzpviigjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666734,"databaseName":"models_schema","ddl":"CREATE TABLE `aiuvcawgjavfsityazsbykuodftgteoa` (\n `skzkhkpshjeeplfgwuwpxjinuelzhfya` int NOT NULL,\n `euddrzonhywgjzxjhhgvzjymzgburfkc` int DEFAULT NULL,\n `czvgjnoqmlbostnuriisyblwsarhupxy` int DEFAULT NULL,\n `xsfgmpceigpizljrhglxvoqohjnfdmin` int DEFAULT NULL,\n `uqlurpkjbbmjevhpsvenksgwzayrwhfv` int DEFAULT NULL,\n `byusfqjxbmjhxoexxytilgqslopdxlrj` int DEFAULT NULL,\n `errkcsgherrsncaakqummvaocesvfphh` int DEFAULT NULL,\n `vtnznbwglchaojgkdevilhffgzgbbeho` int DEFAULT NULL,\n `mvwuziyldmognpcivtfttzastmpkqrkl` int DEFAULT NULL,\n `olfpzkpocohvfrblomuloqevvgbyjbmh` int DEFAULT NULL,\n `iipancvoknuptoomllrwbiuxeztrluvz` int DEFAULT NULL,\n `dqprntjcdtvnlylxeokaszwrwkujjwsp` int DEFAULT NULL,\n `svepsloyliefusfhcqxdtvjnlvouhyqr` int DEFAULT NULL,\n `kjqshcpdhpnhzgnzpwerecmybytvgoud` int DEFAULT NULL,\n `xexksodtbiypgczpmmmfftstcgeshkkf` int DEFAULT NULL,\n `wtdlqvpeysqszsjfuntgyjdvtadymlac` int DEFAULT NULL,\n `qlwoptiiryhtwccuhajtecuyxwnhwdfn` int DEFAULT NULL,\n `oyqrdxlrfccbebiqpwgyivhjkduigigo` int DEFAULT NULL,\n `coxsdtkhniviosrgtplexjgzvqhhutbt` int DEFAULT NULL,\n `sgluxonjxyjrmgwwpxzpbbvahlftdyjn` int DEFAULT NULL,\n `srminmfagiimkpljbewuibihkrefbdfu` int DEFAULT NULL,\n `noesasolmsithhgkkbuuepgzfclgqzns` int DEFAULT NULL,\n `ytmzciulhrwainhymqanhjbscbpschma` int DEFAULT NULL,\n `vtgyluyygceueumalpdpagmmgowawuno` int DEFAULT NULL,\n `ezfwwbpikvgsdbuyhqcahdmcdcpcjbtf` int DEFAULT NULL,\n `cyjvffhgwqanscdiyxedgumcyvhdncfz` int DEFAULT NULL,\n `ekdehnowujadfmawasmtgwjwmwdhtitj` int DEFAULT NULL,\n `uwvwuabpiqwwnlhuinfuelbwjuqjxydm` int DEFAULT NULL,\n `gupgvdavjzyjlghcghbybwwmxjlnqnvt` int DEFAULT NULL,\n `pszjzedtbrjmrhnfpgmikqwziankerdc` int DEFAULT NULL,\n `iuwxlxhzvetqcuflnmyeaooevnwwbfjx` int DEFAULT NULL,\n `tfpwqhqiznfxbintjtepxeckxehgnuho` int DEFAULT NULL,\n `xeapscbbkgswfjngqbsjsorwamnvezly` int DEFAULT NULL,\n `qdbevxejivmjjtaueaapyhiccdqvffte` int DEFAULT NULL,\n `zgtqfkqspkygaeveyrszbpyutecdmxhd` int DEFAULT NULL,\n `rknjyduvsphxskunfpifciaunignymjn` int DEFAULT NULL,\n `yyvautxccdncemyhnuzijssqyhmcamze` int DEFAULT NULL,\n `fndtuufiajfwtputgzmeevlsaaymgzgc` int DEFAULT NULL,\n `tcvoupkezpkumdjzvingshfpcagkiuii` int DEFAULT NULL,\n `nikgmlwlfmreivhcqjscxbozcvxjecwd` int DEFAULT NULL,\n `hqpyslqwhodlgcvyawhgzyllkksxjuak` int DEFAULT NULL,\n `prqxsekbnnmpfeiroezvgirvodleuozi` int DEFAULT NULL,\n `ewibzmbegxxlykeprbohuwplgjhqasdl` int DEFAULT NULL,\n `qgzkltmytdmbagvbhkrkroshkzomdxpx` int DEFAULT NULL,\n `cyluuwinrzoygkxxsrhozfbkeauxezwz` int DEFAULT NULL,\n `drijimtctbmtpvmyuitoerjggdlclpfy` int DEFAULT NULL,\n `avtikqsaaghulsrobawtffhksartrjsc` int DEFAULT NULL,\n `bliyuagxkksvomjbuqlbsczxdyebwnxe` int DEFAULT NULL,\n `cfrxnodacfoxddbuvrxajqvruulbfwcg` int DEFAULT NULL,\n `imvuimfxlfhaacexakfqgydrckjfhfzk` int DEFAULT NULL,\n `zblgbpjzatrrjrsiirpzcjntmrqnifwd` int DEFAULT NULL,\n `pnexckvcwqcxnfowdcadzbaitivldbis` int DEFAULT NULL,\n `jwihfmtldmeeihodzpawxuysmbvqolol` int DEFAULT NULL,\n `rmuggvsduwxfabykdcczljeuwugeiaej` int DEFAULT NULL,\n `apooxerakjppmpppiuvcfytysideyyvq` int DEFAULT NULL,\n `rkycwpdpgukhquqvjnjgbcupttworoup` int DEFAULT NULL,\n `bcnyoivkqlkpubczkutaxqkttrktbhiq` int DEFAULT NULL,\n `lzargnlljhxpcljlvxhnjxnrqyvrskcg` int DEFAULT NULL,\n `odtcbzwtmyynkcgcelezpwhsmiaqwhaa` int DEFAULT NULL,\n `tbnlrystljqwwyibtqynkvagwbpkbxyr` int DEFAULT NULL,\n `gywivzolnwwrcittrydctynohfhixaen` int DEFAULT NULL,\n `ibkupfyhiuawoagzbbxqrimzeqfmeorr` int DEFAULT NULL,\n `zjmdfizofioblvmqfhxwuytuerpszzkx` int DEFAULT NULL,\n `wkniinpeweocvqulzmwithtuohuvohev` int DEFAULT NULL,\n `qaqnnkgqvspephcjiulndzwprxfgbixq` int DEFAULT NULL,\n `agonhyegkapzerxefnjhrzozbeisadpn` int DEFAULT NULL,\n `uijvsowrryxmzzwmlrdfnkjximwxbdlp` int DEFAULT NULL,\n `idmgvxgqsxhjouaziiuhfhvuwzwypmcf` int DEFAULT NULL,\n `axgpnneekbbtxguqwosfisjtsvkhmmmu` int DEFAULT NULL,\n `wosgsbxljnqkdcekzlrggkprhcihtbil` int DEFAULT NULL,\n `zcsjztaaipjltwjvgqdujixtyrfapcwy` int DEFAULT NULL,\n `ykdewxwjnhexwnbknceyuzgkprqzwsvl` int DEFAULT NULL,\n `qdpidijfearqbcltracaxxkgvsktbqxm` int DEFAULT NULL,\n `rtuqfmrmlbcjcnhxbfghpjyzivtunlgr` int DEFAULT NULL,\n `jiywwzaiqxwdhhrriuytgmqydoffffqs` int DEFAULT NULL,\n `tmpdpdqqjlkeraowzltyqdbngpxlmyjf` int DEFAULT NULL,\n `yqlnzprdmpqwlijyqpeggwlefxljghyr` int DEFAULT NULL,\n `iugwkwazkjipxecnrcglubmidelelmwa` int DEFAULT NULL,\n `eraullaaikjwsxpuflgesimryhzijfsm` int DEFAULT NULL,\n `ybcczerfbxadyxnxqdolbmnspgwajnqv` int DEFAULT NULL,\n `epeqtoxitvruhjjzkilxodggidaeocdn` int DEFAULT NULL,\n `zcaijjzwanuvtqdhygcchxpkrkkcdpjr` int DEFAULT NULL,\n `eunupvhrjltswasnnzvqlrgimdgrkgwo` int DEFAULT NULL,\n `knoomndavtmoohuzumrloeurntqntmnn` int DEFAULT NULL,\n `ijqbpwpjrsjdzqdtvdhmmjxexeuocqyj` int DEFAULT NULL,\n `kbqmeqxypdtqmzexlzekfbktlpgyebos` int DEFAULT NULL,\n `maezpewgoiebngspswxcfrnohgvkcwvg` int DEFAULT NULL,\n `fpygvpqajnrhhbryabopwmnpvjggtati` int DEFAULT NULL,\n `vejhbkzpbatjbvteghrajaewuggrtprd` int DEFAULT NULL,\n `dabfxmqfncxrqyjqlfwhtfgmtwnzrtxd` int DEFAULT NULL,\n `pkmgmqtzbfnxevdagmdvkxlqoeshprpy` int DEFAULT NULL,\n `dylfpitmsqeecczoxpddzkbaklpisspm` int DEFAULT NULL,\n `azvkihawluhrmavhyqznsdudoeqpvycw` int DEFAULT NULL,\n `gsawpavbgkdacyefzawagrotpszxywjf` int DEFAULT NULL,\n `tllmeadkzqaivtpboxstplfinfihtswm` int DEFAULT NULL,\n `djufjizigmzyxxaabrnoauidievivxkh` int DEFAULT NULL,\n `iodujeyxrsmrpeiwdertczlmnolplwpb` int DEFAULT NULL,\n `jnucuoordnblnjfejwfygwmisltwpyzv` int DEFAULT NULL,\n `jkckolswpiyshsryzphzikxxywvqnsyv` int DEFAULT NULL,\n `lsdzmbydvbpsqmvbrlyktzpoocbcmvfz` int DEFAULT NULL,\n PRIMARY KEY (`skzkhkpshjeeplfgwuwpxjinuelzhfya`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"aiuvcawgjavfsityazsbykuodftgteoa\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["skzkhkpshjeeplfgwuwpxjinuelzhfya"],"columns":[{"name":"skzkhkpshjeeplfgwuwpxjinuelzhfya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"euddrzonhywgjzxjhhgvzjymzgburfkc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"czvgjnoqmlbostnuriisyblwsarhupxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsfgmpceigpizljrhglxvoqohjnfdmin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqlurpkjbbmjevhpsvenksgwzayrwhfv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"byusfqjxbmjhxoexxytilgqslopdxlrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"errkcsgherrsncaakqummvaocesvfphh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtnznbwglchaojgkdevilhffgzgbbeho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvwuziyldmognpcivtfttzastmpkqrkl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"olfpzkpocohvfrblomuloqevvgbyjbmh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iipancvoknuptoomllrwbiuxeztrluvz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqprntjcdtvnlylxeokaszwrwkujjwsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"svepsloyliefusfhcqxdtvjnlvouhyqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjqshcpdhpnhzgnzpwerecmybytvgoud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xexksodtbiypgczpmmmfftstcgeshkkf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtdlqvpeysqszsjfuntgyjdvtadymlac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qlwoptiiryhtwccuhajtecuyxwnhwdfn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oyqrdxlrfccbebiqpwgyivhjkduigigo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"coxsdtkhniviosrgtplexjgzvqhhutbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sgluxonjxyjrmgwwpxzpbbvahlftdyjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"srminmfagiimkpljbewuibihkrefbdfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"noesasolmsithhgkkbuuepgzfclgqzns","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytmzciulhrwainhymqanhjbscbpschma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtgyluyygceueumalpdpagmmgowawuno","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezfwwbpikvgsdbuyhqcahdmcdcpcjbtf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cyjvffhgwqanscdiyxedgumcyvhdncfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekdehnowujadfmawasmtgwjwmwdhtitj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwvwuabpiqwwnlhuinfuelbwjuqjxydm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gupgvdavjzyjlghcghbybwwmxjlnqnvt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pszjzedtbrjmrhnfpgmikqwziankerdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iuwxlxhzvetqcuflnmyeaooevnwwbfjx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tfpwqhqiznfxbintjtepxeckxehgnuho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xeapscbbkgswfjngqbsjsorwamnvezly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdbevxejivmjjtaueaapyhiccdqvffte","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zgtqfkqspkygaeveyrszbpyutecdmxhd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rknjyduvsphxskunfpifciaunignymjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yyvautxccdncemyhnuzijssqyhmcamze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fndtuufiajfwtputgzmeevlsaaymgzgc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tcvoupkezpkumdjzvingshfpcagkiuii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nikgmlwlfmreivhcqjscxbozcvxjecwd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqpyslqwhodlgcvyawhgzyllkksxjuak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prqxsekbnnmpfeiroezvgirvodleuozi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ewibzmbegxxlykeprbohuwplgjhqasdl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgzkltmytdmbagvbhkrkroshkzomdxpx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cyluuwinrzoygkxxsrhozfbkeauxezwz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drijimtctbmtpvmyuitoerjggdlclpfy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avtikqsaaghulsrobawtffhksartrjsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bliyuagxkksvomjbuqlbsczxdyebwnxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfrxnodacfoxddbuvrxajqvruulbfwcg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"imvuimfxlfhaacexakfqgydrckjfhfzk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zblgbpjzatrrjrsiirpzcjntmrqnifwd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnexckvcwqcxnfowdcadzbaitivldbis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwihfmtldmeeihodzpawxuysmbvqolol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rmuggvsduwxfabykdcczljeuwugeiaej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"apooxerakjppmpppiuvcfytysideyyvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkycwpdpgukhquqvjnjgbcupttworoup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bcnyoivkqlkpubczkutaxqkttrktbhiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzargnlljhxpcljlvxhnjxnrqyvrskcg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"odtcbzwtmyynkcgcelezpwhsmiaqwhaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbnlrystljqwwyibtqynkvagwbpkbxyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gywivzolnwwrcittrydctynohfhixaen","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibkupfyhiuawoagzbbxqrimzeqfmeorr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjmdfizofioblvmqfhxwuytuerpszzkx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wkniinpeweocvqulzmwithtuohuvohev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qaqnnkgqvspephcjiulndzwprxfgbixq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agonhyegkapzerxefnjhrzozbeisadpn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uijvsowrryxmzzwmlrdfnkjximwxbdlp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idmgvxgqsxhjouaziiuhfhvuwzwypmcf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axgpnneekbbtxguqwosfisjtsvkhmmmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wosgsbxljnqkdcekzlrggkprhcihtbil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zcsjztaaipjltwjvgqdujixtyrfapcwy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykdewxwjnhexwnbknceyuzgkprqzwsvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdpidijfearqbcltracaxxkgvsktbqxm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtuqfmrmlbcjcnhxbfghpjyzivtunlgr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jiywwzaiqxwdhhrriuytgmqydoffffqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmpdpdqqjlkeraowzltyqdbngpxlmyjf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqlnzprdmpqwlijyqpeggwlefxljghyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iugwkwazkjipxecnrcglubmidelelmwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eraullaaikjwsxpuflgesimryhzijfsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybcczerfbxadyxnxqdolbmnspgwajnqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"epeqtoxitvruhjjzkilxodggidaeocdn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zcaijjzwanuvtqdhygcchxpkrkkcdpjr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eunupvhrjltswasnnzvqlrgimdgrkgwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knoomndavtmoohuzumrloeurntqntmnn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijqbpwpjrsjdzqdtvdhmmjxexeuocqyj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kbqmeqxypdtqmzexlzekfbktlpgyebos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"maezpewgoiebngspswxcfrnohgvkcwvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpygvpqajnrhhbryabopwmnpvjggtati","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vejhbkzpbatjbvteghrajaewuggrtprd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dabfxmqfncxrqyjqlfwhtfgmtwnzrtxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkmgmqtzbfnxevdagmdvkxlqoeshprpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dylfpitmsqeecczoxpddzkbaklpisspm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azvkihawluhrmavhyqznsdudoeqpvycw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsawpavbgkdacyefzawagrotpszxywjf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tllmeadkzqaivtpboxstplfinfihtswm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"djufjizigmzyxxaabrnoauidievivxkh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iodujeyxrsmrpeiwdertczlmnolplwpb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jnucuoordnblnjfejwfygwmisltwpyzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jkckolswpiyshsryzphzikxxywvqnsyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lsdzmbydvbpsqmvbrlyktzpoocbcmvfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666780,"databaseName":"models_schema","ddl":"CREATE TABLE `arifozcdgamaweuwmigngbztyhjenxgr` (\n `umzxfzbxbbxezvmohbojbqcmjtiryhva` int NOT NULL,\n `cmsehnznhgiogazqdbmjyrbafoswhynk` int DEFAULT NULL,\n `vkfvurntvqwafudvtjfvcznpoiyrcrgb` int DEFAULT NULL,\n `weaxrjgnmshrjdjglgihaazvvygvswdc` int DEFAULT NULL,\n `jnxcjszordmdigvnqbcavjxnuvsbmrqy` int DEFAULT NULL,\n `knbrlnzexnwnfzxyoqjmvuckvnvkficn` int DEFAULT NULL,\n `wfvtgxxckjvivqvttxiclgaolgccdofh` int DEFAULT NULL,\n `ryoeyffzhywuzixdajaycopbbfsbiqhe` int DEFAULT NULL,\n `rngieegurhpqdlciuasdwdnqnbzmoiqy` int DEFAULT NULL,\n `tvejvibrgkecnlgbhohfnzcixvomjvxu` int DEFAULT NULL,\n `dghzkmayhslknqdacyvyxfotgdhhkvac` int DEFAULT NULL,\n `lxgckqebriutulmtqbxavonmnlbmqrpo` int DEFAULT NULL,\n `spbfvolddumpicawfjmmvpbqnlpngbry` int DEFAULT NULL,\n `whobxvmfhoflaxzcthtxjefrtvxxewqm` int DEFAULT NULL,\n `casbgkngvamkcafpelcbzpwaduailhqo` int DEFAULT NULL,\n `emalbtiaetvfkokjgfwykpidtvehmfdv` int DEFAULT NULL,\n `ohwhppirxnwgrhzfxlzqzpwadpxzpogr` int DEFAULT NULL,\n `pkjbuplgsercftwtykectcgcizqzebev` int DEFAULT NULL,\n `plzqiewnuoselvwyvdmmqejrgwfdrbqt` int DEFAULT NULL,\n `deayudxqosevulbulhqudntetipscsop` int DEFAULT NULL,\n `plxryhsrfdctjkmsxvifawnmwjjzpfds` int DEFAULT NULL,\n `qqkkjkthqhoakbcbxwxsqbmovynnqtgl` int DEFAULT NULL,\n `pzguhppnzpcqszueqxkmsiflddvmgixs` int DEFAULT NULL,\n `ikseujbmiefdecgdjaaqpzkflggnayoz` int DEFAULT NULL,\n `pkyuegbkbguhxasxxutiudlqkokyuxxl` int DEFAULT NULL,\n `kueucrqmnmzqwawxtqyrodicfaviifrb` int DEFAULT NULL,\n `kdbplxtursjwlcpzfaeyztcgxacejfhs` int DEFAULT NULL,\n `fvuzlkmpjdhewbjgnbnnygvolybgjrsm` int DEFAULT NULL,\n `twggqhoafgvilkptrpacrhgfuszisbjr` int DEFAULT NULL,\n `hcqvlqztyekzgtavsxmpuwfvpwblmsxo` int DEFAULT NULL,\n `ojhysysrdvzgvmfxlrclmiohebfxglrl` int DEFAULT NULL,\n `ndbtmpmjesniqrozjgeauihlxyfxmgqb` int DEFAULT NULL,\n `unnrklvrjpszgejygzmdoyhgxlfpnbmd` int DEFAULT NULL,\n `xvtpzwbshxuzvdnmqsfrkoizjciukwkc` int DEFAULT NULL,\n `afqpzhbbmtbwulhwdsflquphrovxvkhj` int DEFAULT NULL,\n `sblltftzrsbvsyldnlytuvkfepfjcrpd` int DEFAULT NULL,\n `lylagcfpbvqxhzglppyvetlapkbxvylf` int DEFAULT NULL,\n `sxfyyhglmjrwbtohyeafmuzwxytepsgj` int DEFAULT NULL,\n `ityypvhkufezvxpdgcrkjobjmwyhlfsh` int DEFAULT NULL,\n `znvntqtsugljwzhnrkyoevipeufuohfw` int DEFAULT NULL,\n `ypswtvevoabuzrwcjknimuusnkspkhcj` int DEFAULT NULL,\n `icejzemckuckakzykmcrnlwpjysegafe` int DEFAULT NULL,\n `zhavagqijnfyysywphwpcxnzjpkunxxg` int DEFAULT NULL,\n `folrlarletetrrsbgawmolpheyowboyd` int DEFAULT NULL,\n `pasyucdybqydfggrpavayjziegisoxtr` int DEFAULT NULL,\n `ulrozivjapgcgcnrhkgyufxjujwckxmk` int DEFAULT NULL,\n `mpjtwutoprredovayicsgzjevrnzhjax` int DEFAULT NULL,\n `kyrkilqwpgumtvrjwflkjvsagfmgykxc` int DEFAULT NULL,\n `tkcfqomfvukstichwsbjmpqgowzsbzhj` int DEFAULT NULL,\n `vabydergmegxqabpfnykpoligotawutu` int DEFAULT NULL,\n `whgiedmtalkceemhsmuwibrndljagqlk` int DEFAULT NULL,\n `kzmgdcjxjnwcbosalkbbawgrihrqbemf` int DEFAULT NULL,\n `lwubozpyreheldxzzbiinoehxvlscfbl` int DEFAULT NULL,\n `ubfgxyywgqkhpwauykxskviyhrmylade` int DEFAULT NULL,\n `sstvrhjvqjdetvvzsuqnxynnkkbagvep` int DEFAULT NULL,\n `rmnnyvbpfljeypemhjdwrdwantbmbbwa` int DEFAULT NULL,\n `nbusrvldgibnegqircglyxilgvckwlhw` int DEFAULT NULL,\n `uwldsqinzujhmvtzdvcaqtdjnkqjudec` int DEFAULT NULL,\n `oyatlssuikqgvgqyxefwmivxgctrrtdb` int DEFAULT NULL,\n `pwhnyrunopokfxpjwcnvwakybokbvswf` int DEFAULT NULL,\n `rnmtebrbezsdvqojiwwwnfotxdhtcyyn` int DEFAULT NULL,\n `tkfwmjidbysqfdbqexloykmtgzkbvivm` int DEFAULT NULL,\n `kmogknrkaefuacbrkxfaapqqfixltdlv` int DEFAULT NULL,\n `zjosixsyqisauktxxnfmwfmmdoysgfus` int DEFAULT NULL,\n `whdugfjowubdjdsclufeipnnvanaxxww` int DEFAULT NULL,\n `kzqvrcngyjtkacakjdbdmfatiuimubtp` int DEFAULT NULL,\n `gjluehfabdiyysvkxznjfyfdvuurdmqi` int DEFAULT NULL,\n `wniveyusyhbihundenkdtxebmfnsypkl` int DEFAULT NULL,\n `xdzpdowcibdoqhqdhrkfyqjoqunxjtqn` int DEFAULT NULL,\n `yourelprihbqeasaswxykyayaavhnzik` int DEFAULT NULL,\n `dhrysegiibchyepyzsvfqiiapbgxstom` int DEFAULT NULL,\n `qczrohnhabrthphjxkropxgqxsjfaggr` int DEFAULT NULL,\n `nrdojvlcwkeymlvhzmaurdicuqiciimi` int DEFAULT NULL,\n `mnnpzfhzibakwdtzhndheaovyicsdhvr` int DEFAULT NULL,\n `yhzqevpotcwtohthkqnwdkmucvbyzjfe` int DEFAULT NULL,\n `kousxycegfkrpvykdcciqwzebsmwduob` int DEFAULT NULL,\n `lxlbufagsnnrjodqdzzdpiledadxyfqc` int DEFAULT NULL,\n `mlldbaxarruanvmutrrszsasfymzakxh` int DEFAULT NULL,\n `wlgxdxlkqhalchzhnngxnvokqvldrpwa` int DEFAULT NULL,\n `yqmxazpctxidaycbxwrxhgdqcqxmjlsr` int DEFAULT NULL,\n `blfyyfqwgwkxwcidfsfnccgymryxnxdg` int DEFAULT NULL,\n `xmzafxabeumomqrdtnzwqdboqlbinrdt` int DEFAULT NULL,\n `ienhbfkggiwtwblcmmwnjslhtogcfdnq` int DEFAULT NULL,\n `jmjinjcfcrkdyytozqrsqsnjvpsqdjox` int DEFAULT NULL,\n `mzwbyitsfrjlwklodjjxphgibzvsaaza` int DEFAULT NULL,\n `episdhhdsemrgupnvrpkhxtdaubiqnty` int DEFAULT NULL,\n `cabluwubxlryugjftqghmpywdacedqle` int DEFAULT NULL,\n `flmgeohdftmwsdtzrwlkaswhlfubonmh` int DEFAULT NULL,\n `skjsfidydnezbmbywohgekgatmpyjokr` int DEFAULT NULL,\n `ldghttmfchnaguombaqhfnbaqwynjdcw` int DEFAULT NULL,\n `egdfukrzldfiorgpfquijibcehwmwsts` int DEFAULT NULL,\n `nccvbivequnfmynnorrusipxkggejepm` int DEFAULT NULL,\n `qvvkpqxvlzkwtetcdmjjwcmvzplrbcwo` int DEFAULT NULL,\n `itntdtxgfntvqjewyqdjlydnamwjwrry` int DEFAULT NULL,\n `lfbczlxxdrkqusurslvjxfdptsttjtdk` int DEFAULT NULL,\n `ehzsvqmjchfyjwsivjseukhizxdiykzz` int DEFAULT NULL,\n `uvqbiwuslmheolqmbpetferodbhubrvu` int DEFAULT NULL,\n `ehacrvmleiytdscfdicfhwlqfteqoakh` int DEFAULT NULL,\n `nyfqobdihpsheaizffackjyrnubgrbco` int DEFAULT NULL,\n `wrjsrqdxiaalaqthvszatqvoqngqbfyk` int DEFAULT NULL,\n PRIMARY KEY (`umzxfzbxbbxezvmohbojbqcmjtiryhva`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"arifozcdgamaweuwmigngbztyhjenxgr\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["umzxfzbxbbxezvmohbojbqcmjtiryhva"],"columns":[{"name":"umzxfzbxbbxezvmohbojbqcmjtiryhva","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"cmsehnznhgiogazqdbmjyrbafoswhynk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkfvurntvqwafudvtjfvcznpoiyrcrgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"weaxrjgnmshrjdjglgihaazvvygvswdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jnxcjszordmdigvnqbcavjxnuvsbmrqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knbrlnzexnwnfzxyoqjmvuckvnvkficn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfvtgxxckjvivqvttxiclgaolgccdofh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ryoeyffzhywuzixdajaycopbbfsbiqhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rngieegurhpqdlciuasdwdnqnbzmoiqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvejvibrgkecnlgbhohfnzcixvomjvxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dghzkmayhslknqdacyvyxfotgdhhkvac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxgckqebriutulmtqbxavonmnlbmqrpo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"spbfvolddumpicawfjmmvpbqnlpngbry","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"whobxvmfhoflaxzcthtxjefrtvxxewqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"casbgkngvamkcafpelcbzpwaduailhqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"emalbtiaetvfkokjgfwykpidtvehmfdv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohwhppirxnwgrhzfxlzqzpwadpxzpogr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkjbuplgsercftwtykectcgcizqzebev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"plzqiewnuoselvwyvdmmqejrgwfdrbqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"deayudxqosevulbulhqudntetipscsop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"plxryhsrfdctjkmsxvifawnmwjjzpfds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqkkjkthqhoakbcbxwxsqbmovynnqtgl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pzguhppnzpcqszueqxkmsiflddvmgixs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikseujbmiefdecgdjaaqpzkflggnayoz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkyuegbkbguhxasxxutiudlqkokyuxxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kueucrqmnmzqwawxtqyrodicfaviifrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kdbplxtursjwlcpzfaeyztcgxacejfhs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fvuzlkmpjdhewbjgnbnnygvolybgjrsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twggqhoafgvilkptrpacrhgfuszisbjr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hcqvlqztyekzgtavsxmpuwfvpwblmsxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojhysysrdvzgvmfxlrclmiohebfxglrl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ndbtmpmjesniqrozjgeauihlxyfxmgqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unnrklvrjpszgejygzmdoyhgxlfpnbmd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvtpzwbshxuzvdnmqsfrkoizjciukwkc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afqpzhbbmtbwulhwdsflquphrovxvkhj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sblltftzrsbvsyldnlytuvkfepfjcrpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lylagcfpbvqxhzglppyvetlapkbxvylf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sxfyyhglmjrwbtohyeafmuzwxytepsgj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ityypvhkufezvxpdgcrkjobjmwyhlfsh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znvntqtsugljwzhnrkyoevipeufuohfw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypswtvevoabuzrwcjknimuusnkspkhcj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icejzemckuckakzykmcrnlwpjysegafe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhavagqijnfyysywphwpcxnzjpkunxxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"folrlarletetrrsbgawmolpheyowboyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pasyucdybqydfggrpavayjziegisoxtr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ulrozivjapgcgcnrhkgyufxjujwckxmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpjtwutoprredovayicsgzjevrnzhjax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kyrkilqwpgumtvrjwflkjvsagfmgykxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkcfqomfvukstichwsbjmpqgowzsbzhj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vabydergmegxqabpfnykpoligotawutu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"whgiedmtalkceemhsmuwibrndljagqlk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzmgdcjxjnwcbosalkbbawgrihrqbemf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwubozpyreheldxzzbiinoehxvlscfbl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubfgxyywgqkhpwauykxskviyhrmylade","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sstvrhjvqjdetvvzsuqnxynnkkbagvep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rmnnyvbpfljeypemhjdwrdwantbmbbwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbusrvldgibnegqircglyxilgvckwlhw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwldsqinzujhmvtzdvcaqtdjnkqjudec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oyatlssuikqgvgqyxefwmivxgctrrtdb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwhnyrunopokfxpjwcnvwakybokbvswf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnmtebrbezsdvqojiwwwnfotxdhtcyyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkfwmjidbysqfdbqexloykmtgzkbvivm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmogknrkaefuacbrkxfaapqqfixltdlv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjosixsyqisauktxxnfmwfmmdoysgfus","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"whdugfjowubdjdsclufeipnnvanaxxww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzqvrcngyjtkacakjdbdmfatiuimubtp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjluehfabdiyysvkxznjfyfdvuurdmqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wniveyusyhbihundenkdtxebmfnsypkl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdzpdowcibdoqhqdhrkfyqjoqunxjtqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yourelprihbqeasaswxykyayaavhnzik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhrysegiibchyepyzsvfqiiapbgxstom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qczrohnhabrthphjxkropxgqxsjfaggr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nrdojvlcwkeymlvhzmaurdicuqiciimi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnnpzfhzibakwdtzhndheaovyicsdhvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhzqevpotcwtohthkqnwdkmucvbyzjfe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kousxycegfkrpvykdcciqwzebsmwduob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxlbufagsnnrjodqdzzdpiledadxyfqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlldbaxarruanvmutrrszsasfymzakxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wlgxdxlkqhalchzhnngxnvokqvldrpwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqmxazpctxidaycbxwrxhgdqcqxmjlsr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blfyyfqwgwkxwcidfsfnccgymryxnxdg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmzafxabeumomqrdtnzwqdboqlbinrdt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ienhbfkggiwtwblcmmwnjslhtogcfdnq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmjinjcfcrkdyytozqrsqsnjvpsqdjox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzwbyitsfrjlwklodjjxphgibzvsaaza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"episdhhdsemrgupnvrpkhxtdaubiqnty","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cabluwubxlryugjftqghmpywdacedqle","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flmgeohdftmwsdtzrwlkaswhlfubonmh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skjsfidydnezbmbywohgekgatmpyjokr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ldghttmfchnaguombaqhfnbaqwynjdcw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"egdfukrzldfiorgpfquijibcehwmwsts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nccvbivequnfmynnorrusipxkggejepm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvvkpqxvlzkwtetcdmjjwcmvzplrbcwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itntdtxgfntvqjewyqdjlydnamwjwrry","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfbczlxxdrkqusurslvjxfdptsttjtdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehzsvqmjchfyjwsivjseukhizxdiykzz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvqbiwuslmheolqmbpetferodbhubrvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehacrvmleiytdscfdicfhwlqfteqoakh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nyfqobdihpsheaizffackjyrnubgrbco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wrjsrqdxiaalaqthvszatqvoqngqbfyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666818,"databaseName":"models_schema","ddl":"CREATE TABLE `avimlxteragkjlgouvjrcmgasysrsxih` (\n `gfywhjxsvkpssgqmhfiwglqpkrbhgwqz` int NOT NULL,\n `livtmihvmmnjdocbcptcqezadzsnxvdl` int DEFAULT NULL,\n `rhmurssvdebodruziawnvsvzbsakqpxv` int DEFAULT NULL,\n `vxoulpthpekkecajtycytmzkgaioakqz` int DEFAULT NULL,\n `vfycnzjjrqqrzsbbrpyfsiujyuuswzej` int DEFAULT NULL,\n `lgvlqzegiqedyjekyjlabhvnhsdprbrg` int DEFAULT NULL,\n `rkbfhelweyhkunqiambhntcbwjtgwyuk` int DEFAULT NULL,\n `rwfadhupwjexndedlgabgoexnzmsgguo` int DEFAULT NULL,\n `evswqddcdmaudgvrvynzxtrffbyylmfo` int DEFAULT NULL,\n `xexntjhpsxxpcgvrlldpdsfrgojcfnpj` int DEFAULT NULL,\n `mhbsrrfplogdhyzfprralkzapntyyetr` int DEFAULT NULL,\n `pcvnawwfkhpmklfftkqqpjaodvmxezmk` int DEFAULT NULL,\n `zaerfipwqhdqklmmihoqfqxwjodvhrnb` int DEFAULT NULL,\n `ufajhcdxemlmdgigjpqxtapwzqlvyhim` int DEFAULT NULL,\n `kgardodypekovpbmlmajwujbpyhtzadr` int DEFAULT NULL,\n `lzvpxsksypzeokemfiljrweigflrczyt` int DEFAULT NULL,\n `rcgwipzsitsakrrosqewyxpdvnrmwisg` int DEFAULT NULL,\n `luohzzxxidwujwfqffyyxlwnxcphrsyf` int DEFAULT NULL,\n `xwymrfezubqcedkfxakamqqhapqhuxpt` int DEFAULT NULL,\n `nhrcqpezgkvcpshjcfgktqzaxrswagsm` int DEFAULT NULL,\n `uaskatuqrcllzwjmezcsqwwkhkfizukk` int DEFAULT NULL,\n `hexemtctbxzinuvjyfftixaaverjchxh` int DEFAULT NULL,\n `stunoucrgfjyqssuyhhledkfaspxcmwd` int DEFAULT NULL,\n `dzortiwpjwqebbuuacdpbihllxkkqzqz` int DEFAULT NULL,\n `uwdeafbmrmvlfwxezcfajmtjfjksimyf` int DEFAULT NULL,\n `bfdgdfamttuxkhdkqnbpasewaqfzjbec` int DEFAULT NULL,\n `zgywetxpafxlrciutvxfwytcuagdcgtl` int DEFAULT NULL,\n `okhcgbddgtsviaheayupauiirrbjgemn` int DEFAULT NULL,\n `jxlfhhexmidepvqykpvbmeuvokpzzvgz` int DEFAULT NULL,\n `hlafjyllmiunasdwoxrseprfvmkeqvzd` int DEFAULT NULL,\n `ohsdqbgwzwqfkkblsiwfczvsypvuigcm` int DEFAULT NULL,\n `cqfoydokuzmthbpsqrjiuyjvwvgyxapr` int DEFAULT NULL,\n `tskomawwltaakplpvcmywqfdgoxbmtyc` int DEFAULT NULL,\n `ybgnxasybyejjyddswnbtfeuenpquhdd` int DEFAULT NULL,\n `wlqcmsiyuwhyfvbipjrmbnxfwretxfzl` int DEFAULT NULL,\n `tmcfsylcbccqtooytqzbdusxxbkxtlgw` int DEFAULT NULL,\n `wddipcofhzophaxypcskkjbwqposrkzy` int DEFAULT NULL,\n `sbgtpakcwcddsqicprzoodyoifopzavp` int DEFAULT NULL,\n `axaucyurtetkhbmtjxtxqkgnyhfuezhd` int DEFAULT NULL,\n `ogrocyzfbzrwvmevantufysqjssvcnhc` int DEFAULT NULL,\n `njhzoettrufyrmdngzgyyjdnprnbahpf` int DEFAULT NULL,\n `dvzcattjxcmosovmssuhxevsccrssyaf` int DEFAULT NULL,\n `xplmwvxkezjuftjzfswblrvsdvwroxxb` int DEFAULT NULL,\n `ffjpuomyiaqkqwzxzmasfzbbxxhdukhe` int DEFAULT NULL,\n `elbxhfmrbjnhhzrlthqchvglgiydnyzf` int DEFAULT NULL,\n `mynuwnwhkiykqihcnyjpqcwssiqjgrki` int DEFAULT NULL,\n `ltxlbthslqczcnbnrkkysbgvxzwgltva` int DEFAULT NULL,\n `jwywvtckzgxzxgyaxiuojosbclovtptg` int DEFAULT NULL,\n `wbqiriiiyhrjdcwltpyhnnkqpgdujunn` int DEFAULT NULL,\n `lppqziobydwmpigfkoswavrsezpwkiwv` int DEFAULT NULL,\n `qwrydreewpvxftrrqkdcynibveuetsqi` int DEFAULT NULL,\n `edeujwhakckkklqqasirttrbdvdrlihl` int DEFAULT NULL,\n `imaftxsikbcqjqdhbqrvcpzlgteudqid` int DEFAULT NULL,\n `rvcdwrgfkiiiqeooynfhaqvohqojbqau` int DEFAULT NULL,\n `nlohccaklqwckposaazjcvrjkzokykcw` int DEFAULT NULL,\n `cweurnhirdzadhinppfzhstpxdhnkqhc` int DEFAULT NULL,\n `guwxxvjptpqtdxpvwjxzoykemukiivcw` int DEFAULT NULL,\n `saqmovpmiejnnecfznjzyfnlqticipnn` int DEFAULT NULL,\n `sdmgnbdlpvrfyguxhmsjsbgpkohbzskf` int DEFAULT NULL,\n `frkmmekuroqbsxeehxwrgbgispkkgkdf` int DEFAULT NULL,\n `wjdrjpidzzbntqftksascqwgkflpbfhc` int DEFAULT NULL,\n `dccorsudowzwlkdxptevozxzrwbceauk` int DEFAULT NULL,\n `mbuazlsbkzwxenwgcqprweqpjtefbxup` int DEFAULT NULL,\n `uozvxprcjnmgufvjfdywapspmofkemnf` int DEFAULT NULL,\n `wudgqcmpalfksleqvozeoaahhvcdwwoe` int DEFAULT NULL,\n `njdolrwaydwhqeoyzynrncbndfcdmngh` int DEFAULT NULL,\n `icbcqvnkbgbjsuttrotvwwbfykqtuoys` int DEFAULT NULL,\n `kqfmgeivcrgermjkklaslywufcmevbqf` int DEFAULT NULL,\n `cgwackmqlrvdmuzhpfltwxvswjcwkjtq` int DEFAULT NULL,\n `myvqpvhzkgossznllosqxzcqrpmjfgyd` int DEFAULT NULL,\n `ngfrpukxucbvhemyotgxzrqvszptogll` int DEFAULT NULL,\n `cqifujimzvpaclpvjcsmzjslpmpswcla` int DEFAULT NULL,\n `dkclwsyegwofnujywsnaatqgtvkothoy` int DEFAULT NULL,\n `hzxxlxwmsfjkvpnmzxuqvidnreqnjfqk` int DEFAULT NULL,\n `pixcvmzcshxattdejphjtqiyahpuucdb` int DEFAULT NULL,\n `udnsyfiuqypdohmjwqruznqsreadznly` int DEFAULT NULL,\n `psctcxxuyzgbgdtsabsmlqzwwcigwmxz` int DEFAULT NULL,\n `cbmvwarqeeenqlveozxdlwddsaziskoa` int DEFAULT NULL,\n `aaagliaaftisnypmrsfzqbfcdygbugjf` int DEFAULT NULL,\n `cnclfaniyuvvsbrejcwcbzowipejklcr` int DEFAULT NULL,\n `cmtpjzmoyfcvptyyyfpihybtyoxxgrxg` int DEFAULT NULL,\n `fnoymyayocoyhtblcgyaroxjjbyfnyrs` int DEFAULT NULL,\n `ohknwjwhasntozmvixikwqzsehrimhht` int DEFAULT NULL,\n `gqurzkpwdbtdljblkmmgztvjxelbicct` int DEFAULT NULL,\n `qqvjwqhkwkfvqzstiwjfghefbbyepust` int DEFAULT NULL,\n `mnmcsadkfisaosdfhspjjlgoqqokkphq` int DEFAULT NULL,\n `olsvqwmftekozwreytffmguspkojgbci` int DEFAULT NULL,\n `rtrqbjfpofftfvrfajvajckpiblhpkbf` int DEFAULT NULL,\n `zjoejaitihxhzzphleemauyvbgithswo` int DEFAULT NULL,\n `mzdvpnksnomsmfcgrnhhfttltkeyzkzm` int DEFAULT NULL,\n `sijxdsfmhpkhmxuqhdwjyhbdschbpiqc` int DEFAULT NULL,\n `shsvkecoddyvcxdadoliajbhuklmgecs` int DEFAULT NULL,\n `oqobcbkttyvldttkayemapntimwxpxuk` int DEFAULT NULL,\n `rmpmhqdjycqvzacxcfdloutrkgynjqtq` int DEFAULT NULL,\n `lwhnoewtrmdbbmvyepjfypvrhdhephut` int DEFAULT NULL,\n `zhnfgjcfjrykpiokoaloyqcizoglbzrr` int DEFAULT NULL,\n `wnjqwdiyjcldgvmdzogswspqkkrixqew` int DEFAULT NULL,\n `pmokgthqjkqfcwmibztcxejmwjlfcfko` int DEFAULT NULL,\n `timxapzqpsgbevnyrgyjvzifonnqbdth` int DEFAULT NULL,\n `xyydgcjkpiaapkcwvzfmhhirnoyahysk` int DEFAULT NULL,\n PRIMARY KEY (`gfywhjxsvkpssgqmhfiwglqpkrbhgwqz`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"avimlxteragkjlgouvjrcmgasysrsxih\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["gfywhjxsvkpssgqmhfiwglqpkrbhgwqz"],"columns":[{"name":"gfywhjxsvkpssgqmhfiwglqpkrbhgwqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"livtmihvmmnjdocbcptcqezadzsnxvdl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhmurssvdebodruziawnvsvzbsakqpxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxoulpthpekkecajtycytmzkgaioakqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vfycnzjjrqqrzsbbrpyfsiujyuuswzej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgvlqzegiqedyjekyjlabhvnhsdprbrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkbfhelweyhkunqiambhntcbwjtgwyuk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rwfadhupwjexndedlgabgoexnzmsgguo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evswqddcdmaudgvrvynzxtrffbyylmfo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xexntjhpsxxpcgvrlldpdsfrgojcfnpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhbsrrfplogdhyzfprralkzapntyyetr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pcvnawwfkhpmklfftkqqpjaodvmxezmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zaerfipwqhdqklmmihoqfqxwjodvhrnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufajhcdxemlmdgigjpqxtapwzqlvyhim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kgardodypekovpbmlmajwujbpyhtzadr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzvpxsksypzeokemfiljrweigflrczyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcgwipzsitsakrrosqewyxpdvnrmwisg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"luohzzxxidwujwfqffyyxlwnxcphrsyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xwymrfezubqcedkfxakamqqhapqhuxpt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhrcqpezgkvcpshjcfgktqzaxrswagsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uaskatuqrcllzwjmezcsqwwkhkfizukk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hexemtctbxzinuvjyfftixaaverjchxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"stunoucrgfjyqssuyhhledkfaspxcmwd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dzortiwpjwqebbuuacdpbihllxkkqzqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwdeafbmrmvlfwxezcfajmtjfjksimyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bfdgdfamttuxkhdkqnbpasewaqfzjbec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zgywetxpafxlrciutvxfwytcuagdcgtl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okhcgbddgtsviaheayupauiirrbjgemn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jxlfhhexmidepvqykpvbmeuvokpzzvgz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlafjyllmiunasdwoxrseprfvmkeqvzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohsdqbgwzwqfkkblsiwfczvsypvuigcm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqfoydokuzmthbpsqrjiuyjvwvgyxapr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tskomawwltaakplpvcmywqfdgoxbmtyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybgnxasybyejjyddswnbtfeuenpquhdd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wlqcmsiyuwhyfvbipjrmbnxfwretxfzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmcfsylcbccqtooytqzbdusxxbkxtlgw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wddipcofhzophaxypcskkjbwqposrkzy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbgtpakcwcddsqicprzoodyoifopzavp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axaucyurtetkhbmtjxtxqkgnyhfuezhd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ogrocyzfbzrwvmevantufysqjssvcnhc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njhzoettrufyrmdngzgyyjdnprnbahpf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvzcattjxcmosovmssuhxevsccrssyaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xplmwvxkezjuftjzfswblrvsdvwroxxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffjpuomyiaqkqwzxzmasfzbbxxhdukhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elbxhfmrbjnhhzrlthqchvglgiydnyzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mynuwnwhkiykqihcnyjpqcwssiqjgrki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ltxlbthslqczcnbnrkkysbgvxzwgltva","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwywvtckzgxzxgyaxiuojosbclovtptg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbqiriiiyhrjdcwltpyhnnkqpgdujunn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lppqziobydwmpigfkoswavrsezpwkiwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwrydreewpvxftrrqkdcynibveuetsqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edeujwhakckkklqqasirttrbdvdrlihl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"imaftxsikbcqjqdhbqrvcpzlgteudqid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rvcdwrgfkiiiqeooynfhaqvohqojbqau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nlohccaklqwckposaazjcvrjkzokykcw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cweurnhirdzadhinppfzhstpxdhnkqhc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"guwxxvjptpqtdxpvwjxzoykemukiivcw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"saqmovpmiejnnecfznjzyfnlqticipnn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdmgnbdlpvrfyguxhmsjsbgpkohbzskf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"frkmmekuroqbsxeehxwrgbgispkkgkdf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjdrjpidzzbntqftksascqwgkflpbfhc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dccorsudowzwlkdxptevozxzrwbceauk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbuazlsbkzwxenwgcqprweqpjtefbxup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uozvxprcjnmgufvjfdywapspmofkemnf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wudgqcmpalfksleqvozeoaahhvcdwwoe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njdolrwaydwhqeoyzynrncbndfcdmngh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icbcqvnkbgbjsuttrotvwwbfykqtuoys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqfmgeivcrgermjkklaslywufcmevbqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgwackmqlrvdmuzhpfltwxvswjcwkjtq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"myvqpvhzkgossznllosqxzcqrpmjfgyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ngfrpukxucbvhemyotgxzrqvszptogll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqifujimzvpaclpvjcsmzjslpmpswcla","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkclwsyegwofnujywsnaatqgtvkothoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hzxxlxwmsfjkvpnmzxuqvidnreqnjfqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pixcvmzcshxattdejphjtqiyahpuucdb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udnsyfiuqypdohmjwqruznqsreadznly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psctcxxuyzgbgdtsabsmlqzwwcigwmxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbmvwarqeeenqlveozxdlwddsaziskoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aaagliaaftisnypmrsfzqbfcdygbugjf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnclfaniyuvvsbrejcwcbzowipejklcr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmtpjzmoyfcvptyyyfpihybtyoxxgrxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnoymyayocoyhtblcgyaroxjjbyfnyrs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohknwjwhasntozmvixikwqzsehrimhht","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqurzkpwdbtdljblkmmgztvjxelbicct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqvjwqhkwkfvqzstiwjfghefbbyepust","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnmcsadkfisaosdfhspjjlgoqqokkphq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"olsvqwmftekozwreytffmguspkojgbci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtrqbjfpofftfvrfajvajckpiblhpkbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjoejaitihxhzzphleemauyvbgithswo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzdvpnksnomsmfcgrnhhfttltkeyzkzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sijxdsfmhpkhmxuqhdwjyhbdschbpiqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shsvkecoddyvcxdadoliajbhuklmgecs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqobcbkttyvldttkayemapntimwxpxuk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rmpmhqdjycqvzacxcfdloutrkgynjqtq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwhnoewtrmdbbmvyepjfypvrhdhephut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhnfgjcfjrykpiokoaloyqcizoglbzrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnjqwdiyjcldgvmdzogswspqkkrixqew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmokgthqjkqfcwmibztcxejmwjlfcfko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"timxapzqpsgbevnyrgyjvzifonnqbdth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyydgcjkpiaapkcwvzfmhhirnoyahysk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666862,"databaseName":"models_schema","ddl":"CREATE TABLE `bbvzwwmjvuibbndayvwmfsvfvkomvuwq` (\n `ggamaxproxnqoxbyaplpkbafrvgdlhmc` int NOT NULL,\n `wzhliobwwpetdmqathbqlrkrnpnnfnem` int DEFAULT NULL,\n `odbagjfyljxmmrmvqrquqsbgxonuhgpg` int DEFAULT NULL,\n `noscnwxigfjtglwgcpnptfybebfuwjng` int DEFAULT NULL,\n `cxpcoitmglxooeefvmiryceqfratqpsp` int DEFAULT NULL,\n `oudihirqgjvhqhbghhezbceuwofqzire` int DEFAULT NULL,\n `rgwnxjfityjnimjgydgzoywwelpqpbzd` int DEFAULT NULL,\n `brrvfemzsbmnzdiojaetqkjdunhlpvtn` int DEFAULT NULL,\n `ciceujbckcuftaipnzzbqpctdbsbraga` int DEFAULT NULL,\n `axbxfaqvgjlkelhjxvwbzmkeshwixoem` int DEFAULT NULL,\n `whuwxcwlzjnkhrgluqfsnctnlrujcgnw` int DEFAULT NULL,\n `kfwuzdiucrvafajmhjnohmuvvqjsshca` int DEFAULT NULL,\n `wcgwhokhskwzsnfdisausrqcfhlqcnnz` int DEFAULT NULL,\n `ubbsekhezcbjwjibnweoyidcibqpzglp` int DEFAULT NULL,\n `qqispbystmjojrtzogwdnnkjqrrnkjsj` int DEFAULT NULL,\n `yecvozhzspyeadoyaflhrbucnipguuhp` int DEFAULT NULL,\n `tmoobaguzyobkxmzamvymyynmwmfwxgr` int DEFAULT NULL,\n `obcewrededritobdzdrpmurezcpgdnzp` int DEFAULT NULL,\n `tjcuhniskchmdpjlvawfkmeapgfvtnqp` int DEFAULT NULL,\n `qduwogfpmgrkrrbcvmkjyteozvvifuxn` int DEFAULT NULL,\n `gqldccqdvxjvleveekztsoqoacwpolgy` int DEFAULT NULL,\n `rtmhprnlepeaoivvtajluaiztetprhir` int DEFAULT NULL,\n `clobjgklbpqewtxfqtumlsoenxhvgzjj` int DEFAULT NULL,\n `lapdrrklutmngiicvulxcxsdujrcgpmm` int DEFAULT NULL,\n `juiboadbvpkymwgeqrubahfocrfqmfah` int DEFAULT NULL,\n `rgftacmlvllsguowcmnwnhuxmiucpskn` int DEFAULT NULL,\n `qwyahhhzfkavmqdmgdqaaoojiajftmne` int DEFAULT NULL,\n `wwgpbpbinxremzuokybhgsnvzcfpljsv` int DEFAULT NULL,\n `mhkgofykznyowsawzhsowokgfhhfblza` int DEFAULT NULL,\n `blhjwxokovgkjfpkvcuqqmnymgfhkqwf` int DEFAULT NULL,\n `yysxrfyhdprxfhxnbymysjuwvrykvxif` int DEFAULT NULL,\n `zakyknkekejeqndpzcidvchmcyzeyanq` int DEFAULT NULL,\n `nyukxznvsuxwjxnertfqsyoqzxnnheij` int DEFAULT NULL,\n `taehrwnkrqfzwoovcvhjszzrrezfqtyx` int DEFAULT NULL,\n `yiwkoyxbxfrcxvteexzwpwdqlmepwiza` int DEFAULT NULL,\n `igdwhjmtutpzyxbsvcelaauortjtkles` int DEFAULT NULL,\n `rzioblrwuaktpkdkslnsywmzcxdsdpih` int DEFAULT NULL,\n `mjrgbtfqtawfzgbpehqzdcpcecnowfxc` int DEFAULT NULL,\n `ozlufnawbiaotmxcsswxomvfmcwnyzbo` int DEFAULT NULL,\n `bedheokoswjikjpgmymigxforsczccxt` int DEFAULT NULL,\n `xcctmapwlkenotpbhzravpmazloiiegp` int DEFAULT NULL,\n `xyppyyatzwlnjmffkcsloydogegfchpi` int DEFAULT NULL,\n `yuemmoywakkbcoezkyamoxgmxgicautz` int DEFAULT NULL,\n `okuckrkzrekoekoypbvqglimksewnuol` int DEFAULT NULL,\n `urtflipnywxmmusecnysnqucgmovnbbs` int DEFAULT NULL,\n `tvqcjixjaaudaywgcdodcuqgbkaupcqf` int DEFAULT NULL,\n `yqgqijqysxgoxihyldkcahmsjamjatzl` int DEFAULT NULL,\n `yomhhrvsozgzarslinvzingoxhvrtaju` int DEFAULT NULL,\n `ijdnkhrcncwvyfvsxjtrkqvhawojbxgs` int DEFAULT NULL,\n `oqpozozyvmsqfppnkenzthacmrmnrmkv` int DEFAULT NULL,\n `zgpvqgzubtcblofogsnqwrbmnofwdzhv` int DEFAULT NULL,\n `drlcscuogilsxtuxvipepcwtgyqqbtbd` int DEFAULT NULL,\n `ifpxbqjsqgaxjfxpywjmijpiqprbmebz` int DEFAULT NULL,\n `zpvelgqkeklguwcvkpiqwbommhdaiqum` int DEFAULT NULL,\n `bsjriqxgdvjheatwudnysiclrfsmtgpc` int DEFAULT NULL,\n `ucvnlkpvanmzrqbnuutxzormeocxkocw` int DEFAULT NULL,\n `copsxhowrzjwlemzveozyrlgdygjutzz` int DEFAULT NULL,\n `tlxqorpebvprgixxtfoqsndvrczgrgzs` int DEFAULT NULL,\n `zavhxwionvrvtrefinxmluwhkvageicu` int DEFAULT NULL,\n `lxmlfehgbmiumzgqjpzzlnyawayozztv` int DEFAULT NULL,\n `yypkfshexnimyeunhlkgwyxeabxyzkbz` int DEFAULT NULL,\n `oiwwasefhdyifpmmfbabcjpraxvfehrb` int DEFAULT NULL,\n `qzyrbsqxvserobahwsedzbhsbakjepfc` int DEFAULT NULL,\n `yczwhhuupsbkqfzwtfnlmchbhxxqdacf` int DEFAULT NULL,\n `xdqwcnrccpceyzdfyhtdkuusuqaolaja` int DEFAULT NULL,\n `cosouuwgmmdwendhbrwfiqjxofvnukzt` int DEFAULT NULL,\n `pldkuguvdnqhlroacbptdpewwmktogum` int DEFAULT NULL,\n `xdphcmidekolxrufcwoyqmaqbqsudxun` int DEFAULT NULL,\n `khueqsvaavrxpgbyyiltrjtuhqzouhhr` int DEFAULT NULL,\n `zjkzolalllkkcgksbgjdzoqcfgsanrso` int DEFAULT NULL,\n `evkulunmrcbkbpzkzhikhjmlcbyzlgoa` int DEFAULT NULL,\n `ahwuvupmhslkxdsmokjkvuipmkkqfkdy` int DEFAULT NULL,\n `eebfyqsjmshduplfumexwvscehldzwde` int DEFAULT NULL,\n `hdkceeucifubtfifsojdtuewbsxmgnbx` int DEFAULT NULL,\n `diboaeljxonzdhzntetvemqjrapbhqkg` int DEFAULT NULL,\n `peaqbsgdpngtdwfezqijkezkfkhlmkuw` int DEFAULT NULL,\n `qzaysxncukyynzqynkvqxdyqfuzwqhon` int DEFAULT NULL,\n `fearrdgxbxuctvmpyfwwrhfiwkowdjjn` int DEFAULT NULL,\n `qxczeqnwgzyimtucixapwsechaildxdj` int DEFAULT NULL,\n `yrztpmipbpbpmkbimyjvtijyljhpgafx` int DEFAULT NULL,\n `penktqmvsvaytabzxvpadixjsprnvciu` int DEFAULT NULL,\n `tphcapxafalewyfgaqaodsqranxdsyni` int DEFAULT NULL,\n `rkdgfeijaiomfcjjlyqntmhazexiqzmr` int DEFAULT NULL,\n `hqerniiaihnavaemshlgisjmndgneluf` int DEFAULT NULL,\n `dwvgeclvvvjanolzbivsiqujpfjwhcic` int DEFAULT NULL,\n `zpxsqvamnssrxaqnocrxmrvurhecfyxs` int DEFAULT NULL,\n `vskwgaeuxgrfqxntcvxrqtqiwzrzaacb` int DEFAULT NULL,\n `kovvgenfspywplgsudlbjxaeysopxclq` int DEFAULT NULL,\n `dvxqdjcbyhbgfzqqccjrflcrcdajvfim` int DEFAULT NULL,\n `ejptzxdxrywzglkurpdhjvswgfwgxqqx` int DEFAULT NULL,\n `ehliocidxugjiocdntyeyyapczkntslh` int DEFAULT NULL,\n `tqmsrymdsitggdfsxmbxlzwzuovwvneu` int DEFAULT NULL,\n `braphcdjvccwbsfaekyddzjsykhhnspw` int DEFAULT NULL,\n `kfydnxufbjdklqjwjwytlrmzajpioyfg` int DEFAULT NULL,\n `byyleqhrzcyfybgzqpuvgjhvsoubfkjv` int DEFAULT NULL,\n `xajpxejnqcbgzmgcaoygvthtavyybfgw` int DEFAULT NULL,\n `opdsetdqwuotjgvdcyrozfiqlngzetjg` int DEFAULT NULL,\n `xuovxpaclgdptcfkrdljyplwxarybbfh` int DEFAULT NULL,\n `wjcuxaqmoiurrekgqiyzomggwkkgaexz` int DEFAULT NULL,\n `qezrbvsicyofkjeyelhtzpdhietgepna` int DEFAULT NULL,\n PRIMARY KEY (`ggamaxproxnqoxbyaplpkbafrvgdlhmc`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"bbvzwwmjvuibbndayvwmfsvfvkomvuwq\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["ggamaxproxnqoxbyaplpkbafrvgdlhmc"],"columns":[{"name":"ggamaxproxnqoxbyaplpkbafrvgdlhmc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"wzhliobwwpetdmqathbqlrkrnpnnfnem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"odbagjfyljxmmrmvqrquqsbgxonuhgpg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"noscnwxigfjtglwgcpnptfybebfuwjng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cxpcoitmglxooeefvmiryceqfratqpsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oudihirqgjvhqhbghhezbceuwofqzire","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgwnxjfityjnimjgydgzoywwelpqpbzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"brrvfemzsbmnzdiojaetqkjdunhlpvtn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ciceujbckcuftaipnzzbqpctdbsbraga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axbxfaqvgjlkelhjxvwbzmkeshwixoem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"whuwxcwlzjnkhrgluqfsnctnlrujcgnw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfwuzdiucrvafajmhjnohmuvvqjsshca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcgwhokhskwzsnfdisausrqcfhlqcnnz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubbsekhezcbjwjibnweoyidcibqpzglp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqispbystmjojrtzogwdnnkjqrrnkjsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yecvozhzspyeadoyaflhrbucnipguuhp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmoobaguzyobkxmzamvymyynmwmfwxgr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obcewrededritobdzdrpmurezcpgdnzp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjcuhniskchmdpjlvawfkmeapgfvtnqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qduwogfpmgrkrrbcvmkjyteozvvifuxn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqldccqdvxjvleveekztsoqoacwpolgy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtmhprnlepeaoivvtajluaiztetprhir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clobjgklbpqewtxfqtumlsoenxhvgzjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lapdrrklutmngiicvulxcxsdujrcgpmm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"juiboadbvpkymwgeqrubahfocrfqmfah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgftacmlvllsguowcmnwnhuxmiucpskn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwyahhhzfkavmqdmgdqaaoojiajftmne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwgpbpbinxremzuokybhgsnvzcfpljsv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhkgofykznyowsawzhsowokgfhhfblza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blhjwxokovgkjfpkvcuqqmnymgfhkqwf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yysxrfyhdprxfhxnbymysjuwvrykvxif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zakyknkekejeqndpzcidvchmcyzeyanq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nyukxznvsuxwjxnertfqsyoqzxnnheij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"taehrwnkrqfzwoovcvhjszzrrezfqtyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yiwkoyxbxfrcxvteexzwpwdqlmepwiza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igdwhjmtutpzyxbsvcelaauortjtkles","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzioblrwuaktpkdkslnsywmzcxdsdpih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjrgbtfqtawfzgbpehqzdcpcecnowfxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozlufnawbiaotmxcsswxomvfmcwnyzbo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bedheokoswjikjpgmymigxforsczccxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xcctmapwlkenotpbhzravpmazloiiegp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyppyyatzwlnjmffkcsloydogegfchpi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yuemmoywakkbcoezkyamoxgmxgicautz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okuckrkzrekoekoypbvqglimksewnuol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urtflipnywxmmusecnysnqucgmovnbbs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvqcjixjaaudaywgcdodcuqgbkaupcqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqgqijqysxgoxihyldkcahmsjamjatzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yomhhrvsozgzarslinvzingoxhvrtaju","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijdnkhrcncwvyfvsxjtrkqvhawojbxgs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqpozozyvmsqfppnkenzthacmrmnrmkv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zgpvqgzubtcblofogsnqwrbmnofwdzhv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drlcscuogilsxtuxvipepcwtgyqqbtbd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifpxbqjsqgaxjfxpywjmijpiqprbmebz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpvelgqkeklguwcvkpiqwbommhdaiqum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bsjriqxgdvjheatwudnysiclrfsmtgpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucvnlkpvanmzrqbnuutxzormeocxkocw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"copsxhowrzjwlemzveozyrlgdygjutzz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlxqorpebvprgixxtfoqsndvrczgrgzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zavhxwionvrvtrefinxmluwhkvageicu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxmlfehgbmiumzgqjpzzlnyawayozztv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yypkfshexnimyeunhlkgwyxeabxyzkbz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oiwwasefhdyifpmmfbabcjpraxvfehrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzyrbsqxvserobahwsedzbhsbakjepfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yczwhhuupsbkqfzwtfnlmchbhxxqdacf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdqwcnrccpceyzdfyhtdkuusuqaolaja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cosouuwgmmdwendhbrwfiqjxofvnukzt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pldkuguvdnqhlroacbptdpewwmktogum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdphcmidekolxrufcwoyqmaqbqsudxun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khueqsvaavrxpgbyyiltrjtuhqzouhhr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjkzolalllkkcgksbgjdzoqcfgsanrso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evkulunmrcbkbpzkzhikhjmlcbyzlgoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahwuvupmhslkxdsmokjkvuipmkkqfkdy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eebfyqsjmshduplfumexwvscehldzwde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdkceeucifubtfifsojdtuewbsxmgnbx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"diboaeljxonzdhzntetvemqjrapbhqkg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"peaqbsgdpngtdwfezqijkezkfkhlmkuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzaysxncukyynzqynkvqxdyqfuzwqhon","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fearrdgxbxuctvmpyfwwrhfiwkowdjjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxczeqnwgzyimtucixapwsechaildxdj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yrztpmipbpbpmkbimyjvtijyljhpgafx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"penktqmvsvaytabzxvpadixjsprnvciu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tphcapxafalewyfgaqaodsqranxdsyni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkdgfeijaiomfcjjlyqntmhazexiqzmr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqerniiaihnavaemshlgisjmndgneluf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwvgeclvvvjanolzbivsiqujpfjwhcic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpxsqvamnssrxaqnocrxmrvurhecfyxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vskwgaeuxgrfqxntcvxrqtqiwzrzaacb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kovvgenfspywplgsudlbjxaeysopxclq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvxqdjcbyhbgfzqqccjrflcrcdajvfim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejptzxdxrywzglkurpdhjvswgfwgxqqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehliocidxugjiocdntyeyyapczkntslh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqmsrymdsitggdfsxmbxlzwzuovwvneu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"braphcdjvccwbsfaekyddzjsykhhnspw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfydnxufbjdklqjwjwytlrmzajpioyfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"byyleqhrzcyfybgzqpuvgjhvsoubfkjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xajpxejnqcbgzmgcaoygvthtavyybfgw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opdsetdqwuotjgvdcyrozfiqlngzetjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xuovxpaclgdptcfkrdljyplwxarybbfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjcuxaqmoiurrekgqiyzomggwkkgaexz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qezrbvsicyofkjeyelhtzpdhietgepna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666907,"databaseName":"models_schema","ddl":"CREATE TABLE `bekezigfgafwjjfcxlfvmohuwjxylbco` (\n `oxigapxxtuqovfenogygoxktlncotahs` int NOT NULL,\n `soqkzsyhtemymdcbzohiaeizhhattoew` int DEFAULT NULL,\n `qdeztdnqmgzydomvevqoiktienahrahi` int DEFAULT NULL,\n `okfrwhwtslecjjfxgvcrfkgqqcxfexvd` int DEFAULT NULL,\n `fbbrfkoucqngjbsaeyafqivpruxekrej` int DEFAULT NULL,\n `jstszxqfuzwxxvyoztskkhdbhfeafati` int DEFAULT NULL,\n `nmvrwubswuvgixdjxlcqjkxwzttpiagk` int DEFAULT NULL,\n `qaeexstvvyqcduzqthhzpztaakemroon` int DEFAULT NULL,\n `ntmhkdktlwjpltdwdntxhamoxtojcdxh` int DEFAULT NULL,\n `ytjxiliijyqknmcgmeayntvfsveakrcr` int DEFAULT NULL,\n `vzptfsotwzlugxrehwsxbiapeiuzbmsi` int DEFAULT NULL,\n `vgbmlzsarryfbxmwvpiolihavgpghosx` int DEFAULT NULL,\n `zaejbsufgucxlwvqbwybihxtsxlpjmzr` int DEFAULT NULL,\n `uwgrpphiqrwgecujhwlvyizezuzazpiq` int DEFAULT NULL,\n `aoatotpasesldurdyplghrguryeclnci` int DEFAULT NULL,\n `nbgnqigzosgdpofzndbngeykfioqegfc` int DEFAULT NULL,\n `uorgbqyreqflxfesiujwtpymkjcttfqf` int DEFAULT NULL,\n `fbvxrmxmrlaflbxugredsmeaefbqmceh` int DEFAULT NULL,\n `jxaipmxjdgggwnsfchhfgatxbpambaza` int DEFAULT NULL,\n `wfcfdzxwlfmeyrkcvhhapbjjrovxrzxg` int DEFAULT NULL,\n `lctdskxoenqpidqtqssbdeabnsouxjsp` int DEFAULT NULL,\n `kuicmvujwnfmbatbdbvteekorxbuawqh` int DEFAULT NULL,\n `shuevxyldlywclowwadurheltfpxqdzi` int DEFAULT NULL,\n `rmvdmmzdmuzpwmhjaobvmptmlrdbdcxy` int DEFAULT NULL,\n `rioozwoyjfxluosshkiffglkyoeybjaa` int DEFAULT NULL,\n `mgqeweddcvwcumojttptqsfagwzntmis` int DEFAULT NULL,\n `zzbzbwnpljhkgcwbjixtwlksbmiozcwu` int DEFAULT NULL,\n `rgqssmexbmggrbybpbetyfphmnsciuje` int DEFAULT NULL,\n `mrfpalrhvpvkfvcvrcuiipswpjillvmg` int DEFAULT NULL,\n `ivdqzzzsotlwvdctoqlfyegqhlwlpgmk` int DEFAULT NULL,\n `gjrugbwtkyzqjlcfjpgfvajeoxrivewi` int DEFAULT NULL,\n `lziutyiprfrzuyixnmfnecneyntitgfi` int DEFAULT NULL,\n `bjpjmuetoepranodkhxlfolvmoysognk` int DEFAULT NULL,\n `nluoeridjjifphwzydkloaxyicioocbu` int DEFAULT NULL,\n `bhkrauxouxhaagfbzvcssxwqdathlxpk` int DEFAULT NULL,\n `wppwswcjcaslekxffsvnwiieobamcldv` int DEFAULT NULL,\n `njeursgwbnvuignhghfbzekmorowmrri` int DEFAULT NULL,\n `hxqgqjqkrorwrlconyqlgpzmxvuwilsw` int DEFAULT NULL,\n `vdqrqldxncxgpdxuucriqaqqrozrceai` int DEFAULT NULL,\n `twnqddsbjgnmadqhcqzcnyyixdkvklbm` int DEFAULT NULL,\n `mjgogmerfmxppnqxassubmisycezmkqv` int DEFAULT NULL,\n `qshkzfhmqiqtzvmhsytkditybkivqrki` int DEFAULT NULL,\n `xiboprdjuhmcvgvnrnafpvyxoxbmwthx` int DEFAULT NULL,\n `roysrxhynilgmhegfpfqxeswpcthtisi` int DEFAULT NULL,\n `uzovgrjlrkkxjdcbxjjhevaffqahaxsk` int DEFAULT NULL,\n `ktuzoyoqwenhfmhxcaubujwhfhcimdqf` int DEFAULT NULL,\n `cvcgjjxeicuanqenvvrefoivjosmvkzl` int DEFAULT NULL,\n `eocckikvryzfashpbqjlzsbqlzxwzhsa` int DEFAULT NULL,\n `nkdrhccdchfepzbrtumpqduldhlsppgg` int DEFAULT NULL,\n `jkyibliurdtgoebdaieqnpkuscmjntis` int DEFAULT NULL,\n `ebvhiekcdzkfigynvvakqhpxbizhpgkb` int DEFAULT NULL,\n `hppfrclfjwidsusuwvrvlpxxztwptkqt` int DEFAULT NULL,\n `bowxkququyqzayrsjplbyntqtnwhpppi` int DEFAULT NULL,\n `cwovntgcpjmeztnkvmzlaczppnhlmwrv` int DEFAULT NULL,\n `cdacmcdfetyyucjgpeslodgwulfmgcqy` int DEFAULT NULL,\n `wfhjctdwimdsxiwrtlglfepcwndyrpmg` int DEFAULT NULL,\n `beffahrdtsmwpuexsfvabcfljdfvnshb` int DEFAULT NULL,\n `mzowldpdlitcebmwzoxzgxcfblxpsvbz` int DEFAULT NULL,\n `rejzxldxtljxriuftiyydxiezrgrluhi` int DEFAULT NULL,\n `fxukfacdejjdbomdvddtzlbkoerkgxqp` int DEFAULT NULL,\n `mmeoaqzuzfvfflujrjkxvgtshyfrnojm` int DEFAULT NULL,\n `zqcgnlcfvpcocmoonqjpzwqofqfrljuj` int DEFAULT NULL,\n `oohfhgnbrkzjfgwxehljwotdbixshfau` int DEFAULT NULL,\n `fefnpaiuatcfntzdpjkqnfsnmlaqgfwo` int DEFAULT NULL,\n `aabbsqwjpnwpmuqdabithlrxnpjfaxhq` int DEFAULT NULL,\n `boicrtrkcanlxzerrtsszxtfmemdznif` int DEFAULT NULL,\n `rklejkhjnvnwvmulcswfecawluxxvtvq` int DEFAULT NULL,\n `xocrmmfekajwimpdbzkawlomrbgicsnb` int DEFAULT NULL,\n `qxbnqnkvhsrcprfmxfiqqcwioydswldt` int DEFAULT NULL,\n `ipwhyyvfdxyuefgxwiqvohtorcpexnfp` int DEFAULT NULL,\n `ysrjfwtcpqfirxvccjhmkaiitdsuitdc` int DEFAULT NULL,\n `vieagsfrdhhjozycgbnfwzrbjdmwfdsg` int DEFAULT NULL,\n `rznwjyiajrrvtplmtfhektwkyjcfcmca` int DEFAULT NULL,\n `uyeaaxjchfbfzdabpwgecldhdvihxbpv` int DEFAULT NULL,\n `shkftlqrhhoecoiptdjedlexcsoqtsuu` int DEFAULT NULL,\n `pdfdnlctfvanbkvwhutzxmpoutxnovrg` int DEFAULT NULL,\n `bhadajwsuyaufzijmjhoxarncuemolli` int DEFAULT NULL,\n `qxcxnilwahvxmvgyozvzdjlfghnjgqna` int DEFAULT NULL,\n `kzysxruzuordnygensnpgltjbredbniy` int DEFAULT NULL,\n `oxunisvhfaizjupsklkbmycyzfhxgccv` int DEFAULT NULL,\n `dtzbvosswgybfxkvwoppsnwjrdgqsjlx` int DEFAULT NULL,\n `ytkynoekjzzxukynqdahqoczkzqlpnuv` int DEFAULT NULL,\n `tezselatadjplvewksyqqyyedkvmtnbw` int DEFAULT NULL,\n `edoiifkexwwuxrneclqvheffsuulryzs` int DEFAULT NULL,\n `lihgpfaypsvlqqeniulmgdrilwyjiwtm` int DEFAULT NULL,\n `kxnpwzhyvdlpakyajczpujhjsgazzjxr` int DEFAULT NULL,\n `cizucmymkbchonfuascnfzyuavdxcptl` int DEFAULT NULL,\n `ngvtkvqyovzdasklobluxizpwseuhdol` int DEFAULT NULL,\n `ceenjwsakueshzidtztukcojqxfgnobg` int DEFAULT NULL,\n `rpmuiogyznhmzczhadgqxpikcyfubynk` int DEFAULT NULL,\n `sikknmyrifkyiqbfiwdemeueirfhkcbo` int DEFAULT NULL,\n `mhxbgkkiqkdewppeeezcpdwesrszsyqi` int DEFAULT NULL,\n `deonrijpyzjnlwawbbevxotfffkkizcl` int DEFAULT NULL,\n `stqjoabwuybzdqyukaqfxwscllxladyu` int DEFAULT NULL,\n `wkbqdrgcuhufczhebqfbikokivssxkhh` int DEFAULT NULL,\n `atdshgyjbjgpgtfzxvlzdptivnkqnpej` int DEFAULT NULL,\n `eqrkdydkzlliriwnihaexxgrogxccxfw` int DEFAULT NULL,\n `hbuihkdgdrnuqynhkvqqvwurdpzwiqvu` int DEFAULT NULL,\n `ezztpqtdultioovwgiaxwvjkyjwxmwsn` int DEFAULT NULL,\n `damptvyuminktozabpwschpilbzsugun` int DEFAULT NULL,\n PRIMARY KEY (`oxigapxxtuqovfenogygoxktlncotahs`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"bekezigfgafwjjfcxlfvmohuwjxylbco\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["oxigapxxtuqovfenogygoxktlncotahs"],"columns":[{"name":"oxigapxxtuqovfenogygoxktlncotahs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"soqkzsyhtemymdcbzohiaeizhhattoew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdeztdnqmgzydomvevqoiktienahrahi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okfrwhwtslecjjfxgvcrfkgqqcxfexvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbbrfkoucqngjbsaeyafqivpruxekrej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jstszxqfuzwxxvyoztskkhdbhfeafati","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmvrwubswuvgixdjxlcqjkxwzttpiagk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qaeexstvvyqcduzqthhzpztaakemroon","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntmhkdktlwjpltdwdntxhamoxtojcdxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytjxiliijyqknmcgmeayntvfsveakrcr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzptfsotwzlugxrehwsxbiapeiuzbmsi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgbmlzsarryfbxmwvpiolihavgpghosx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zaejbsufgucxlwvqbwybihxtsxlpjmzr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwgrpphiqrwgecujhwlvyizezuzazpiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aoatotpasesldurdyplghrguryeclnci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbgnqigzosgdpofzndbngeykfioqegfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uorgbqyreqflxfesiujwtpymkjcttfqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbvxrmxmrlaflbxugredsmeaefbqmceh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jxaipmxjdgggwnsfchhfgatxbpambaza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfcfdzxwlfmeyrkcvhhapbjjrovxrzxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lctdskxoenqpidqtqssbdeabnsouxjsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kuicmvujwnfmbatbdbvteekorxbuawqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shuevxyldlywclowwadurheltfpxqdzi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rmvdmmzdmuzpwmhjaobvmptmlrdbdcxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rioozwoyjfxluosshkiffglkyoeybjaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mgqeweddcvwcumojttptqsfagwzntmis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzbzbwnpljhkgcwbjixtwlksbmiozcwu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgqssmexbmggrbybpbetyfphmnsciuje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrfpalrhvpvkfvcvrcuiipswpjillvmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivdqzzzsotlwvdctoqlfyegqhlwlpgmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjrugbwtkyzqjlcfjpgfvajeoxrivewi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lziutyiprfrzuyixnmfnecneyntitgfi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjpjmuetoepranodkhxlfolvmoysognk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nluoeridjjifphwzydkloaxyicioocbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bhkrauxouxhaagfbzvcssxwqdathlxpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wppwswcjcaslekxffsvnwiieobamcldv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njeursgwbnvuignhghfbzekmorowmrri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxqgqjqkrorwrlconyqlgpzmxvuwilsw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdqrqldxncxgpdxuucriqaqqrozrceai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twnqddsbjgnmadqhcqzcnyyixdkvklbm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjgogmerfmxppnqxassubmisycezmkqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qshkzfhmqiqtzvmhsytkditybkivqrki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xiboprdjuhmcvgvnrnafpvyxoxbmwthx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"roysrxhynilgmhegfpfqxeswpcthtisi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzovgrjlrkkxjdcbxjjhevaffqahaxsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ktuzoyoqwenhfmhxcaubujwhfhcimdqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvcgjjxeicuanqenvvrefoivjosmvkzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eocckikvryzfashpbqjlzsbqlzxwzhsa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkdrhccdchfepzbrtumpqduldhlsppgg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jkyibliurdtgoebdaieqnpkuscmjntis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebvhiekcdzkfigynvvakqhpxbizhpgkb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hppfrclfjwidsusuwvrvlpxxztwptkqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bowxkququyqzayrsjplbyntqtnwhpppi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwovntgcpjmeztnkvmzlaczppnhlmwrv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cdacmcdfetyyucjgpeslodgwulfmgcqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfhjctdwimdsxiwrtlglfepcwndyrpmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"beffahrdtsmwpuexsfvabcfljdfvnshb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzowldpdlitcebmwzoxzgxcfblxpsvbz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rejzxldxtljxriuftiyydxiezrgrluhi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxukfacdejjdbomdvddtzlbkoerkgxqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmeoaqzuzfvfflujrjkxvgtshyfrnojm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqcgnlcfvpcocmoonqjpzwqofqfrljuj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oohfhgnbrkzjfgwxehljwotdbixshfau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fefnpaiuatcfntzdpjkqnfsnmlaqgfwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aabbsqwjpnwpmuqdabithlrxnpjfaxhq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"boicrtrkcanlxzerrtsszxtfmemdznif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rklejkhjnvnwvmulcswfecawluxxvtvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xocrmmfekajwimpdbzkawlomrbgicsnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxbnqnkvhsrcprfmxfiqqcwioydswldt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ipwhyyvfdxyuefgxwiqvohtorcpexnfp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ysrjfwtcpqfirxvccjhmkaiitdsuitdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vieagsfrdhhjozycgbnfwzrbjdmwfdsg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rznwjyiajrrvtplmtfhektwkyjcfcmca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uyeaaxjchfbfzdabpwgecldhdvihxbpv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shkftlqrhhoecoiptdjedlexcsoqtsuu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdfdnlctfvanbkvwhutzxmpoutxnovrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bhadajwsuyaufzijmjhoxarncuemolli","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxcxnilwahvxmvgyozvzdjlfghnjgqna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzysxruzuordnygensnpgltjbredbniy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oxunisvhfaizjupsklkbmycyzfhxgccv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtzbvosswgybfxkvwoppsnwjrdgqsjlx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytkynoekjzzxukynqdahqoczkzqlpnuv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tezselatadjplvewksyqqyyedkvmtnbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edoiifkexwwuxrneclqvheffsuulryzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lihgpfaypsvlqqeniulmgdrilwyjiwtm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxnpwzhyvdlpakyajczpujhjsgazzjxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cizucmymkbchonfuascnfzyuavdxcptl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ngvtkvqyovzdasklobluxizpwseuhdol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ceenjwsakueshzidtztukcojqxfgnobg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpmuiogyznhmzczhadgqxpikcyfubynk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sikknmyrifkyiqbfiwdemeueirfhkcbo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhxbgkkiqkdewppeeezcpdwesrszsyqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"deonrijpyzjnlwawbbevxotfffkkizcl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"stqjoabwuybzdqyukaqfxwscllxladyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wkbqdrgcuhufczhebqfbikokivssxkhh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atdshgyjbjgpgtfzxvlzdptivnkqnpej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqrkdydkzlliriwnihaexxgrogxccxfw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbuihkdgdrnuqynhkvqqvwurdpzwiqvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezztpqtdultioovwgiaxwvjkyjwxmwsn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"damptvyuminktozabpwschpilbzsugun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666955,"databaseName":"models_schema","ddl":"CREATE TABLE `bfilzxoyiqgaczecequheguojdjdqgkw` (\n `uzdtsvwobxruwoxsvcjgnuqqbddhslkx` int NOT NULL,\n `ztwfatkqmddatpytqnniimqzssuwqgpq` int DEFAULT NULL,\n `kanlztgbzyaotzpojoueipeitbawcwgj` int DEFAULT NULL,\n `dkcxclgucjydbfttobfvhrpbedvahsba` int DEFAULT NULL,\n `wvdpipadoqoygshcfeqsmwfhsidixzfx` int DEFAULT NULL,\n `zwapdueckuxmihznikgsgvfruxorzifb` int DEFAULT NULL,\n `cmqavfqhbaxdnfhpkhveqeinnllhdeap` int DEFAULT NULL,\n `yjnithoarquipjbwvfpdrhufagyehzru` int DEFAULT NULL,\n `qtqejfqfqhlxpbaeiixuswhkgutzwvnt` int DEFAULT NULL,\n `xknggzwlkwoyqotrorncfaqymsahxkeg` int DEFAULT NULL,\n `tqwjdptjcaomxmquorqirocbmuxxzigl` int DEFAULT NULL,\n `ayvmxkjpmogwrfvbgnuahqkokucekyoy` int DEFAULT NULL,\n `admbkxxfrykcytiaoqobhnrbnpthjubx` int DEFAULT NULL,\n `rwnpoauqyrsrpbsvannowmeopheguzck` int DEFAULT NULL,\n `vzvticifocenaicmggzrychmeaqgpwyn` int DEFAULT NULL,\n `tvxaoeliaevnzxedcqosggknjmybgdmh` int DEFAULT NULL,\n `eufgqaykopgyxzwoefaamjevdjmdkdqo` int DEFAULT NULL,\n `fhtnqrskxtcgqfnfuiepyyadisbgfkrp` int DEFAULT NULL,\n `rleoelgvuzmijwvmytobiqmxlhrzmarn` int DEFAULT NULL,\n `jgxxcvraoptulgdvpppgytnvtmnszqha` int DEFAULT NULL,\n `pelpmqskhbdtmckitqhswqudbjfebcuy` int DEFAULT NULL,\n `qflkhuvynlncobnorhwcgwkwfftbaqhg` int DEFAULT NULL,\n `vhqormlbuxakflotcjdrrtsjnuhapwlk` int DEFAULT NULL,\n `jbqestvtcgclfbgxytflyjaompexseoi` int DEFAULT NULL,\n `pjutbflblsushbwyakfbfivxqqbkixqn` int DEFAULT NULL,\n `lzrcqniolfkmrinzpmgstzxlvxirkikj` int DEFAULT NULL,\n `tbbgouujvhmbkmacjrlzfrwnccptletd` int DEFAULT NULL,\n `xvahqqetrkvayrastbbkbhbhulhdxhkn` int DEFAULT NULL,\n `ctjefecjjqwsqjkztkpfmfqckfhqchfq` int DEFAULT NULL,\n `kohtzvdnszxxrekkowuujvvpeipdwvos` int DEFAULT NULL,\n `evaudrbwrjunxzaarpjyoxbotuvfvysr` int DEFAULT NULL,\n `ajkxjqvleinbunxxichhmlonspashhng` int DEFAULT NULL,\n `mmahhjixuutwkpzbvoimjqzitaqsqibv` int DEFAULT NULL,\n `sfsgezzstpxnrvpbpoijutradtnxfvuh` int DEFAULT NULL,\n `naybaltmajjoeiyvhyfcvotbyxyilnvr` int DEFAULT NULL,\n `zrymjwppwkkiobbgbjrquoxxhvaaflbh` int DEFAULT NULL,\n `vypwfeltfvfyaxaerzuytqxptjykbwii` int DEFAULT NULL,\n `lcqsqzcmbxhhnkpeuqlvpvnbfvvcvfew` int DEFAULT NULL,\n `cmuvsebmlpysgbcdrtgmysmsiehuobva` int DEFAULT NULL,\n `wibmjrqygvbozyrgjygullvlqhfrvgun` int DEFAULT NULL,\n `gixcmkyizvwnnkjtxjyzeupelcxgaotd` int DEFAULT NULL,\n `nqgtyfxcmhhkawbjrtqmwviljuekiuql` int DEFAULT NULL,\n `nptozglopxtndpouyutdeozyuixdwyld` int DEFAULT NULL,\n `uchlauzobgrriemtykupeipuygmgkuxr` int DEFAULT NULL,\n `lffrydlxttlukvaqsgacxpcwkddsxuvq` int DEFAULT NULL,\n `xrnpgiutmovfkkfhnvzdrnykubrasbat` int DEFAULT NULL,\n `jmxvuoksjuhepszztdrdhpapuihlmyoo` int DEFAULT NULL,\n `nvwecvxzjlbqwzctfqudcahegtoyrxsx` int DEFAULT NULL,\n `vrhoblpyxrukrsubpgazrzacddzrgmww` int DEFAULT NULL,\n `cfucgwndydfkmjyoonkeppxrumbbeulc` int DEFAULT NULL,\n `xzhcfwccuprsbxpqkyncrerlxubbkajs` int DEFAULT NULL,\n `zyryiglittwcqptlepngdqtpfrzcpnji` int DEFAULT NULL,\n `rkfkzgmnupxmtybwamuyciytubdcfjuq` int DEFAULT NULL,\n `wxhqvdadsekvhubvwhjjlkctttlyrthm` int DEFAULT NULL,\n `usdgybqfncbvxcbpbcolisnytkivlveo` int DEFAULT NULL,\n `xxxyezgbmttrkimqvbmdiiqvzgputjog` int DEFAULT NULL,\n `uucofrzbjnbensbftounnedzpyxvoaed` int DEFAULT NULL,\n `csajdxvswumzuxcqnrzlontekcyuskog` int DEFAULT NULL,\n `thhcrnsrynmmdsylkbslfaewmunxwrcq` int DEFAULT NULL,\n `wqjenuzjexdlpcqinfarwoxinhjsauwv` int DEFAULT NULL,\n `kqbbyosioynybdmevtlktiuggzvkfnta` int DEFAULT NULL,\n `hiedyzofmvyxboumdxnazhjaxgfiraqi` int DEFAULT NULL,\n `ylaeyvmsyycmkzblxuynqkzqkfsedjft` int DEFAULT NULL,\n `jwgljiwzmvsxhcdudisujuhmbeqreega` int DEFAULT NULL,\n `niarzfqxrkyniowjjtgbkgbhghirknjf` int DEFAULT NULL,\n `yxquaqvduwepsbhijufajrrwuhygktih` int DEFAULT NULL,\n `vuvhfrkkrinhpwyjdkaennjvkrronhps` int DEFAULT NULL,\n `xashjazrgxdisfevdrtnztubzixspnpg` int DEFAULT NULL,\n `qjnprxoxpyocfacfzrkuskihxzpjbneh` int DEFAULT NULL,\n `qdnjgpsemlpozhkxxgulvnmaltspgmjj` int DEFAULT NULL,\n `yzfmosligfohljzfzuvmsuzxzqpyzgaq` int DEFAULT NULL,\n `xlitvkfxgcisugxfdmtbizicoskuseqq` int DEFAULT NULL,\n `vhinjpxcdqrleaqabqacjzpqtpacxndf` int DEFAULT NULL,\n `nfqivkcimmvqauasebtzwgseheebfmhp` int DEFAULT NULL,\n `sshhrchdoiemeskglkxjkfodyyyhznbp` int DEFAULT NULL,\n `lfhglczvherqxrgxwlyzuvxdcjcycswq` int DEFAULT NULL,\n `gfbvqbgtssczwbnqazemabisvvfvtrxv` int DEFAULT NULL,\n `pkdhugcxgtzwzgnovnmehrjvksetbxuk` int DEFAULT NULL,\n `xwpabojvjyazxvqtwvjnuclmvblcvuys` int DEFAULT NULL,\n `haidxqakiujvbvneiadbjmrozqptssqv` int DEFAULT NULL,\n `jkhirbzlnietnoyxvpkdmmbexqwomhub` int DEFAULT NULL,\n `luxezlxmjrygklgtbkjkxmlhyceyraac` int DEFAULT NULL,\n `zayqyzcvggnmvgkngivekqnaoujzsdop` int DEFAULT NULL,\n `ohexumsffosbdozggbhdpigwmehwjdmi` int DEFAULT NULL,\n `lvsuvaoebpatqlcfgpqznaxecyvmdaup` int DEFAULT NULL,\n `edvobvmqhlbmnyqwwbmewuzhmmditeeh` int DEFAULT NULL,\n `cuwsilsrjhqgpkhpnjwlbsofogbofhvg` int DEFAULT NULL,\n `afitrmtzmyuwhisfaiffrbngcsbgcjjv` int DEFAULT NULL,\n `gdjcedupvnvdodiikjhgjpkuvveurgsp` int DEFAULT NULL,\n `jbscokzcsyokstgnklyapxrohvkjufxx` int DEFAULT NULL,\n `einjddekgsiotkqtxqfefywgehfpordu` int DEFAULT NULL,\n `axwueldcxfwcvnfwvywlymevszhvgrou` int DEFAULT NULL,\n `kvqawszrhptttmnlliykygyssvmcuntd` int DEFAULT NULL,\n `ccekcripejbymgzuajcuynjcvlfvzroo` int DEFAULT NULL,\n `rhqhwukdxwzshpzsyaslndpvczrptwvk` int DEFAULT NULL,\n `endycnsdtggxbarfxcibvkpqgtznnham` int DEFAULT NULL,\n `naefolwcqkzrpdeawvgibsnznvqmrjgf` int DEFAULT NULL,\n `sicigbktsuadjdcbbdhlgxaothxwmvos` int DEFAULT NULL,\n `bcfhqaldzlfteyizmnusdvrjomkegtlr` int DEFAULT NULL,\n `obtwotfdvlvxhzqjwtdvzkcrdcbelfng` int DEFAULT NULL,\n PRIMARY KEY (`uzdtsvwobxruwoxsvcjgnuqqbddhslkx`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"bfilzxoyiqgaczecequheguojdjdqgkw\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["uzdtsvwobxruwoxsvcjgnuqqbddhslkx"],"columns":[{"name":"uzdtsvwobxruwoxsvcjgnuqqbddhslkx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ztwfatkqmddatpytqnniimqzssuwqgpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kanlztgbzyaotzpojoueipeitbawcwgj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkcxclgucjydbfttobfvhrpbedvahsba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvdpipadoqoygshcfeqsmwfhsidixzfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zwapdueckuxmihznikgsgvfruxorzifb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmqavfqhbaxdnfhpkhveqeinnllhdeap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjnithoarquipjbwvfpdrhufagyehzru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtqejfqfqhlxpbaeiixuswhkgutzwvnt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xknggzwlkwoyqotrorncfaqymsahxkeg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqwjdptjcaomxmquorqirocbmuxxzigl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ayvmxkjpmogwrfvbgnuahqkokucekyoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"admbkxxfrykcytiaoqobhnrbnpthjubx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rwnpoauqyrsrpbsvannowmeopheguzck","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzvticifocenaicmggzrychmeaqgpwyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvxaoeliaevnzxedcqosggknjmybgdmh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eufgqaykopgyxzwoefaamjevdjmdkdqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhtnqrskxtcgqfnfuiepyyadisbgfkrp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rleoelgvuzmijwvmytobiqmxlhrzmarn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgxxcvraoptulgdvpppgytnvtmnszqha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pelpmqskhbdtmckitqhswqudbjfebcuy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qflkhuvynlncobnorhwcgwkwfftbaqhg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhqormlbuxakflotcjdrrtsjnuhapwlk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbqestvtcgclfbgxytflyjaompexseoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjutbflblsushbwyakfbfivxqqbkixqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzrcqniolfkmrinzpmgstzxlvxirkikj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbbgouujvhmbkmacjrlzfrwnccptletd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvahqqetrkvayrastbbkbhbhulhdxhkn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctjefecjjqwsqjkztkpfmfqckfhqchfq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kohtzvdnszxxrekkowuujvvpeipdwvos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evaudrbwrjunxzaarpjyoxbotuvfvysr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajkxjqvleinbunxxichhmlonspashhng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmahhjixuutwkpzbvoimjqzitaqsqibv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfsgezzstpxnrvpbpoijutradtnxfvuh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"naybaltmajjoeiyvhyfcvotbyxyilnvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zrymjwppwkkiobbgbjrquoxxhvaaflbh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vypwfeltfvfyaxaerzuytqxptjykbwii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lcqsqzcmbxhhnkpeuqlvpvnbfvvcvfew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmuvsebmlpysgbcdrtgmysmsiehuobva","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wibmjrqygvbozyrgjygullvlqhfrvgun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gixcmkyizvwnnkjtxjyzeupelcxgaotd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqgtyfxcmhhkawbjrtqmwviljuekiuql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nptozglopxtndpouyutdeozyuixdwyld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uchlauzobgrriemtykupeipuygmgkuxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lffrydlxttlukvaqsgacxpcwkddsxuvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrnpgiutmovfkkfhnvzdrnykubrasbat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmxvuoksjuhepszztdrdhpapuihlmyoo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvwecvxzjlbqwzctfqudcahegtoyrxsx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrhoblpyxrukrsubpgazrzacddzrgmww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfucgwndydfkmjyoonkeppxrumbbeulc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xzhcfwccuprsbxpqkyncrerlxubbkajs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyryiglittwcqptlepngdqtpfrzcpnji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkfkzgmnupxmtybwamuyciytubdcfjuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxhqvdadsekvhubvwhjjlkctttlyrthm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"usdgybqfncbvxcbpbcolisnytkivlveo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxxyezgbmttrkimqvbmdiiqvzgputjog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uucofrzbjnbensbftounnedzpyxvoaed","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"csajdxvswumzuxcqnrzlontekcyuskog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thhcrnsrynmmdsylkbslfaewmunxwrcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqjenuzjexdlpcqinfarwoxinhjsauwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqbbyosioynybdmevtlktiuggzvkfnta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hiedyzofmvyxboumdxnazhjaxgfiraqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ylaeyvmsyycmkzblxuynqkzqkfsedjft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwgljiwzmvsxhcdudisujuhmbeqreega","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"niarzfqxrkyniowjjtgbkgbhghirknjf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxquaqvduwepsbhijufajrrwuhygktih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vuvhfrkkrinhpwyjdkaennjvkrronhps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xashjazrgxdisfevdrtnztubzixspnpg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjnprxoxpyocfacfzrkuskihxzpjbneh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdnjgpsemlpozhkxxgulvnmaltspgmjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzfmosligfohljzfzuvmsuzxzqpyzgaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xlitvkfxgcisugxfdmtbizicoskuseqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhinjpxcdqrleaqabqacjzpqtpacxndf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfqivkcimmvqauasebtzwgseheebfmhp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sshhrchdoiemeskglkxjkfodyyyhznbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfhglczvherqxrgxwlyzuvxdcjcycswq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfbvqbgtssczwbnqazemabisvvfvtrxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkdhugcxgtzwzgnovnmehrjvksetbxuk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xwpabojvjyazxvqtwvjnuclmvblcvuys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"haidxqakiujvbvneiadbjmrozqptssqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jkhirbzlnietnoyxvpkdmmbexqwomhub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"luxezlxmjrygklgtbkjkxmlhyceyraac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zayqyzcvggnmvgkngivekqnaoujzsdop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohexumsffosbdozggbhdpigwmehwjdmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvsuvaoebpatqlcfgpqznaxecyvmdaup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edvobvmqhlbmnyqwwbmewuzhmmditeeh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cuwsilsrjhqgpkhpnjwlbsofogbofhvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afitrmtzmyuwhisfaiffrbngcsbgcjjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdjcedupvnvdodiikjhgjpkuvveurgsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbscokzcsyokstgnklyapxrohvkjufxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"einjddekgsiotkqtxqfefywgehfpordu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axwueldcxfwcvnfwvywlymevszhvgrou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvqawszrhptttmnlliykygyssvmcuntd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ccekcripejbymgzuajcuynjcvlfvzroo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhqhwukdxwzshpzsyaslndpvczrptwvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"endycnsdtggxbarfxcibvkpqgtznnham","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"naefolwcqkzrpdeawvgibsnznvqmrjgf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sicigbktsuadjdcbbdhlgxaothxwmvos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bcfhqaldzlfteyizmnusdvrjomkegtlr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obtwotfdvlvxhzqjwtdvzkcrdcbelfng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842666,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842666999,"databaseName":"models_schema","ddl":"CREATE TABLE `brkqrwtvwkcngrjfgrxsbyjsgzuuyjxk` (\n `mfwztykfbojaazktpyrysevvmcyolght` int NOT NULL,\n `ucwomlvriclxbarpsjbtiqxlbnfocigo` int DEFAULT NULL,\n `bymqtjeupotsfwznpgkbatwgmdijugty` int DEFAULT NULL,\n `urlglmfmokjhziyfwybrykkxjbcduquu` int DEFAULT NULL,\n `aqzzecwgkeojcpcmkjikhutveljluuzu` int DEFAULT NULL,\n `kemtqocgsbmmilyedyyfilylpeabewdt` int DEFAULT NULL,\n `qknnnvfouvojssmerhnmmsexcwauubzc` int DEFAULT NULL,\n `yfmxwyqojbfjrvdnnkrjcndxsficulnq` int DEFAULT NULL,\n `lgtglhxfbfokjfvxhgdruckbykkzuxrp` int DEFAULT NULL,\n `nackuijlnjahvmesbrkdusensvaseizi` int DEFAULT NULL,\n `njjqqexgvgscopsveiuqhymzadqtrsor` int DEFAULT NULL,\n `ivgnrioaiuzsndreobutexcbkycjnrqy` int DEFAULT NULL,\n `rdpunacwwxsjkuqzftywaitnsvqehjvl` int DEFAULT NULL,\n `vpfpqnddiwmpkkngjbzjdxknvnondond` int DEFAULT NULL,\n `gobktzxrzfkylgfcccyqsgxnkxrlujho` int DEFAULT NULL,\n `vpcgkpqtxeyqhqdrdlxhbgnayafvhdxv` int DEFAULT NULL,\n `bnkcqfoxxavepjecdybovvuegbutstcx` int DEFAULT NULL,\n `kqtkkmneinzzhjbcvwrfivbtjqilcfcu` int DEFAULT NULL,\n `ykualodfqnahsnxucaafbiwykkuvipgj` int DEFAULT NULL,\n `awmeotzvkatzvmgwmcontkcwmmcpxncm` int DEFAULT NULL,\n `pcjpsblskviciefbizyoprvhapteeeaf` int DEFAULT NULL,\n `ydvnkhaptoyfeshrjcxfkvdktbnkqdge` int DEFAULT NULL,\n `qdofebchijldsepmxcmoudlmrhntgprm` int DEFAULT NULL,\n `vtniwwyqujfssavchoscairhiebwktza` int DEFAULT NULL,\n `fhbpwtvytcxdifimjkqtmfldcvofpuji` int DEFAULT NULL,\n `fnnmpvesnhhjbevvgeolfvxbbztjjayq` int DEFAULT NULL,\n `pltfyzjnezbwwhkvjjzlkaqziafgjllb` int DEFAULT NULL,\n `duoabwotmpivttureneqgjpicyzyvtsj` int DEFAULT NULL,\n `bdjgtwgicaibdcnbhilimcmqudszqkqu` int DEFAULT NULL,\n `acdmxjcuyikmyduuxjsvkahoiqbymcss` int DEFAULT NULL,\n `gtruqieherppvtfolxcudrieywiryjzy` int DEFAULT NULL,\n `vjcfsfmtyjpcduoyhowphyzsgqiijbro` int DEFAULT NULL,\n `guwajnzvcnmmznnhfbeardtshdvicwmm` int DEFAULT NULL,\n `diacrmhgyfegfwzdljrjyxgfyigqhhqj` int DEFAULT NULL,\n `fzsvxrqklzvxbppoflcwvytxvrberkgw` int DEFAULT NULL,\n `vtdjndcgecbaymbnsrxpsrelggvsdusi` int DEFAULT NULL,\n `faxmavheclhrlchwowkilfkbxbfehsyg` int DEFAULT NULL,\n `gmrfzbfwujwaoonxtznpsezurblrxsfw` int DEFAULT NULL,\n `pzgxgzcpzwvyizpgvemtlejzxjoiyilx` int DEFAULT NULL,\n `vsjtspflfyrurafajuwkonttohhckoig` int DEFAULT NULL,\n `owuxkjpkgagpinzujxtujhdloyvobhgi` int DEFAULT NULL,\n `yfhyrjtmeumzjgvzvziljyabwccihcsc` int DEFAULT NULL,\n `sxeyzjkwlzxcznsqvmsxtnhqkifbatwm` int DEFAULT NULL,\n `qselnnwzuuyqpkzrkhzwfopbhrpdzcgc` int DEFAULT NULL,\n `mafdnvgtbvuwwlarzuvfcckeznnqbwxg` int DEFAULT NULL,\n `bohmycpynkvuaaumbedtdzueksqhvnfv` int DEFAULT NULL,\n `ygndqlualmcalkaudlgpfxcpsbmofkvu` int DEFAULT NULL,\n `lvvzesdvoygvotmzokyszdrjrzzkupjb` int DEFAULT NULL,\n `xcdlpvedqfxantqmducxytswwdkcajtz` int DEFAULT NULL,\n `kbpdhzrxalulyymwivyznocdthakpxhn` int DEFAULT NULL,\n `kizmbrumcacujmwdcjaeuettwydcjykn` int DEFAULT NULL,\n `nxcsrbebesppeirrpypbuhewnpbtufnq` int DEFAULT NULL,\n `jdoqcdgqvswgootmkucutdjjjukequwj` int DEFAULT NULL,\n `oirgczzrxdnoqglyhvwrtdzefwgwlqlu` int DEFAULT NULL,\n `jwpmzvmbhdrvmjcfoqrvyqrababylfhp` int DEFAULT NULL,\n `fttstfvwudjauhjwgwbxphcqprvppkvy` int DEFAULT NULL,\n `zpselxqetqpvcibzmvfsjjgchvygvbjd` int DEFAULT NULL,\n `liyoqalwzugedbyurrecwhdegntlcdwt` int DEFAULT NULL,\n `lospvogewywpjwwaiccywbumqiexlxaf` int DEFAULT NULL,\n `xehmyoljiyobluxhobtruuwyzzytnndq` int DEFAULT NULL,\n `vkzwkgqqnplexjetevugilqvgrphmwfj` int DEFAULT NULL,\n `mqvbqnmxprbilbjyvivezgrcvdoqvccp` int DEFAULT NULL,\n `unmrxlmmqtcmwmnwnveshaitavncwiih` int DEFAULT NULL,\n `frwtmnpgfdivtorjdvqhgblvvgvnnyvq` int DEFAULT NULL,\n `knhtgujguqgfjkgkeyzjqgoynrborhck` int DEFAULT NULL,\n `sjczywhxpauuxitjsocjkmkkfnjbwyej` int DEFAULT NULL,\n `drvhgosjnrxxfylglxqixebzoxeouysx` int DEFAULT NULL,\n `fejwazphtablhufphejjzgeeccuhwylk` int DEFAULT NULL,\n `lvdmqcwnubztyiluhdpvlrpufnyjqwvg` int DEFAULT NULL,\n `dglkzyokwymthaanrgmqnwtldwhbbcqy` int DEFAULT NULL,\n `lmcjbfrwuzbggppmnsqkmrgaxccqpetm` int DEFAULT NULL,\n `zdedcnphhizfbtlnkxnczjwnrzjipuka` int DEFAULT NULL,\n `twxbavycvmdqvwgamcytjczkyqogdzjz` int DEFAULT NULL,\n `ctgdzsuzpobvmptsclxkqoeztmsrtfmf` int DEFAULT NULL,\n `lwaqwegbkfssjpkqvlkvmvfjiqrvnwkh` int DEFAULT NULL,\n `nabnqwjkfoloilxrokhiecuennwdupak` int DEFAULT NULL,\n `eskoqejfzykxwmbboobgmpogmmzkjdyz` int DEFAULT NULL,\n `qvwiihzvejmzxewazrwmtecflaoznxel` int DEFAULT NULL,\n `qitehczcbesnsifazjpickvtokyrrson` int DEFAULT NULL,\n `wqcallgzqpfmtafquqkgzdanscnaylsy` int DEFAULT NULL,\n `vyezvozmdubsofsilfxvdojyhmnhampb` int DEFAULT NULL,\n `yhcevbnuahjvyqcglhvtdrorsqrmgdlb` int DEFAULT NULL,\n `yddfobkmjgahyzoccpksidwxrvegnjvw` int DEFAULT NULL,\n `ljindrtgzsjpctajsiufkpbspqryudrj` int DEFAULT NULL,\n `unfpgrkwkcwmjglbpteyiuudswzqdxub` int DEFAULT NULL,\n `ujocfzwuosfxoobucqptdbsrkjidhobu` int DEFAULT NULL,\n `hjrryxtievspokxjwqvgaapsnwnspinq` int DEFAULT NULL,\n `xsdinmvcljhumjjstlavkineejclaych` int DEFAULT NULL,\n `wcuwyndfhxhawklsgogtjkfrutlrcwyx` int DEFAULT NULL,\n `fnjoqamfusoxekskhlzxhjmhkywnbskm` int DEFAULT NULL,\n `scjypnfriogtxmcndeshqdlgnlxdbxqj` int DEFAULT NULL,\n `mrwuzvxibmhqkdinhlnoodjihkgizqnx` int DEFAULT NULL,\n `uixlgglwdufcikgbdnrgxxehqejkhspp` int DEFAULT NULL,\n `gydlfuygcwgqjpqpnabcavxjkdbzjsqz` int DEFAULT NULL,\n `lrkwnjwsyjuziaivdefpqafoipklpkeu` int DEFAULT NULL,\n `qfrhrejksajomzfkzcfwviffrrpunyjt` int DEFAULT NULL,\n `hpohogwctolqfzwmcxntjbljskaijhab` int DEFAULT NULL,\n `snlqdrqbwtctqysnslqjeqihqfxmkpvz` int DEFAULT NULL,\n `jyjiqpqkyrshyxfezwflftzjsmpcdmhg` int DEFAULT NULL,\n `rcuuuddyeclmygcgyttydtxceetykcxr` int DEFAULT NULL,\n PRIMARY KEY (`mfwztykfbojaazktpyrysevvmcyolght`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"brkqrwtvwkcngrjfgrxsbyjsgzuuyjxk\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["mfwztykfbojaazktpyrysevvmcyolght"],"columns":[{"name":"mfwztykfbojaazktpyrysevvmcyolght","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ucwomlvriclxbarpsjbtiqxlbnfocigo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bymqtjeupotsfwznpgkbatwgmdijugty","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urlglmfmokjhziyfwybrykkxjbcduquu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqzzecwgkeojcpcmkjikhutveljluuzu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kemtqocgsbmmilyedyyfilylpeabewdt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qknnnvfouvojssmerhnmmsexcwauubzc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfmxwyqojbfjrvdnnkrjcndxsficulnq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgtglhxfbfokjfvxhgdruckbykkzuxrp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nackuijlnjahvmesbrkdusensvaseizi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njjqqexgvgscopsveiuqhymzadqtrsor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivgnrioaiuzsndreobutexcbkycjnrqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdpunacwwxsjkuqzftywaitnsvqehjvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpfpqnddiwmpkkngjbzjdxknvnondond","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gobktzxrzfkylgfcccyqsgxnkxrlujho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpcgkpqtxeyqhqdrdlxhbgnayafvhdxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnkcqfoxxavepjecdybovvuegbutstcx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqtkkmneinzzhjbcvwrfivbtjqilcfcu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykualodfqnahsnxucaafbiwykkuvipgj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awmeotzvkatzvmgwmcontkcwmmcpxncm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pcjpsblskviciefbizyoprvhapteeeaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydvnkhaptoyfeshrjcxfkvdktbnkqdge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdofebchijldsepmxcmoudlmrhntgprm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtniwwyqujfssavchoscairhiebwktza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhbpwtvytcxdifimjkqtmfldcvofpuji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnnmpvesnhhjbevvgeolfvxbbztjjayq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pltfyzjnezbwwhkvjjzlkaqziafgjllb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"duoabwotmpivttureneqgjpicyzyvtsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdjgtwgicaibdcnbhilimcmqudszqkqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acdmxjcuyikmyduuxjsvkahoiqbymcss","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtruqieherppvtfolxcudrieywiryjzy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjcfsfmtyjpcduoyhowphyzsgqiijbro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"guwajnzvcnmmznnhfbeardtshdvicwmm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"diacrmhgyfegfwzdljrjyxgfyigqhhqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzsvxrqklzvxbppoflcwvytxvrberkgw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtdjndcgecbaymbnsrxpsrelggvsdusi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"faxmavheclhrlchwowkilfkbxbfehsyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmrfzbfwujwaoonxtznpsezurblrxsfw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pzgxgzcpzwvyizpgvemtlejzxjoiyilx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsjtspflfyrurafajuwkonttohhckoig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owuxkjpkgagpinzujxtujhdloyvobhgi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfhyrjtmeumzjgvzvziljyabwccihcsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sxeyzjkwlzxcznsqvmsxtnhqkifbatwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qselnnwzuuyqpkzrkhzwfopbhrpdzcgc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mafdnvgtbvuwwlarzuvfcckeznnqbwxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bohmycpynkvuaaumbedtdzueksqhvnfv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ygndqlualmcalkaudlgpfxcpsbmofkvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvvzesdvoygvotmzokyszdrjrzzkupjb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xcdlpvedqfxantqmducxytswwdkcajtz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kbpdhzrxalulyymwivyznocdthakpxhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kizmbrumcacujmwdcjaeuettwydcjykn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxcsrbebesppeirrpypbuhewnpbtufnq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdoqcdgqvswgootmkucutdjjjukequwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oirgczzrxdnoqglyhvwrtdzefwgwlqlu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwpmzvmbhdrvmjcfoqrvyqrababylfhp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fttstfvwudjauhjwgwbxphcqprvppkvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpselxqetqpvcibzmvfsjjgchvygvbjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"liyoqalwzugedbyurrecwhdegntlcdwt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lospvogewywpjwwaiccywbumqiexlxaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xehmyoljiyobluxhobtruuwyzzytnndq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkzwkgqqnplexjetevugilqvgrphmwfj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqvbqnmxprbilbjyvivezgrcvdoqvccp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unmrxlmmqtcmwmnwnveshaitavncwiih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"frwtmnpgfdivtorjdvqhgblvvgvnnyvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knhtgujguqgfjkgkeyzjqgoynrborhck","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjczywhxpauuxitjsocjkmkkfnjbwyej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drvhgosjnrxxfylglxqixebzoxeouysx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fejwazphtablhufphejjzgeeccuhwylk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvdmqcwnubztyiluhdpvlrpufnyjqwvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dglkzyokwymthaanrgmqnwtldwhbbcqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmcjbfrwuzbggppmnsqkmrgaxccqpetm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdedcnphhizfbtlnkxnczjwnrzjipuka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twxbavycvmdqvwgamcytjczkyqogdzjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctgdzsuzpobvmptsclxkqoeztmsrtfmf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwaqwegbkfssjpkqvlkvmvfjiqrvnwkh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nabnqwjkfoloilxrokhiecuennwdupak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eskoqejfzykxwmbboobgmpogmmzkjdyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvwiihzvejmzxewazrwmtecflaoznxel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qitehczcbesnsifazjpickvtokyrrson","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqcallgzqpfmtafquqkgzdanscnaylsy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vyezvozmdubsofsilfxvdojyhmnhampb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhcevbnuahjvyqcglhvtdrorsqrmgdlb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yddfobkmjgahyzoccpksidwxrvegnjvw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljindrtgzsjpctajsiufkpbspqryudrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unfpgrkwkcwmjglbpteyiuudswzqdxub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujocfzwuosfxoobucqptdbsrkjidhobu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjrryxtievspokxjwqvgaapsnwnspinq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsdinmvcljhumjjstlavkineejclaych","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcuwyndfhxhawklsgogtjkfrutlrcwyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnjoqamfusoxekskhlzxhjmhkywnbskm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scjypnfriogtxmcndeshqdlgnlxdbxqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrwuzvxibmhqkdinhlnoodjihkgizqnx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uixlgglwdufcikgbdnrgxxehqejkhspp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gydlfuygcwgqjpqpnabcavxjkdbzjsqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lrkwnjwsyjuziaivdefpqafoipklpkeu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfrhrejksajomzfkzcfwviffrrpunyjt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hpohogwctolqfzwmcxntjbljskaijhab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"snlqdrqbwtctqysnslqjeqihqfxmkpvz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jyjiqpqkyrshyxfezwflftzjsmpcdmhg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcuuuddyeclmygcgyttydtxceetykcxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667033,"databaseName":"models_schema","ddl":"CREATE TABLE `brznqjnnrwxnevpxwjrcthnvpxrsawok` (\n `sthxctbpbnsfrrhrqgnoggptbofweior` int NOT NULL,\n `pcwbuqijkvvnkjvyibcicirfjlplbsto` int DEFAULT NULL,\n `mzpcubjltjguzvmaebnheijpuybansjf` int DEFAULT NULL,\n `jorhcxrxdehwmyaelkzmiuiyiwougsrm` int DEFAULT NULL,\n `qvdajacjfvsjmptirfttclidjknlpnsi` int DEFAULT NULL,\n `gowtuaypwpttdpnszvjesdbtzotxjdcp` int DEFAULT NULL,\n `igznaqorutncosltvelyefagwytnylxp` int DEFAULT NULL,\n `icsmcxnumrhioomdksoogndgnmsrhkew` int DEFAULT NULL,\n `hdxocmhpkjdkdmuwfadishqbpnbgptqo` int DEFAULT NULL,\n `ajjbafrnuzjfuuxdzhgswrfzhqwolhqe` int DEFAULT NULL,\n `mcorzyttueqegazaabjofjhzwnzafncg` int DEFAULT NULL,\n `bewzqjqotprbxgrdurucrvxtktwdnmwx` int DEFAULT NULL,\n `zyiyvastlokjjnsxajbtrrbczuqvysjy` int DEFAULT NULL,\n `axumbgsmhibwqonkpwbxfnefpujeqesn` int DEFAULT NULL,\n `mlcwkpznkfxynyytrhtoroopwxemqqgz` int DEFAULT NULL,\n `ddmogzonkekzkortmxtsdmrdrrqfauzk` int DEFAULT NULL,\n `jcnfwnqjqvkivkkjhlgivijbwodetjpf` int DEFAULT NULL,\n `voxgxcvxmhiqztavndkrkkpilckozplq` int DEFAULT NULL,\n `jituqvrangwfbsiqwfhbqpyffctkrcil` int DEFAULT NULL,\n `eijlvbviyiawbbostpgemhtnwwotidbm` int DEFAULT NULL,\n `syonomlrybssgeylshjdsswyxyfyyuss` int DEFAULT NULL,\n `nbqxfjdwteherypbhwxvybqicmxczaqs` int DEFAULT NULL,\n `xmqlontcdvtwmkeuwjqfhjmoafprjljl` int DEFAULT NULL,\n `hnahkjtqfptuoknooituqmiihozswyth` int DEFAULT NULL,\n `qnzcwblgwimfsxtanfdwuegxusuweaby` int DEFAULT NULL,\n `ufcnxnsdtssbqcwexjjnjnvatgmdxkei` int DEFAULT NULL,\n `tujvjwgxgzakklvluozipkaqogyxktfs` int DEFAULT NULL,\n `dfjocpbauhyzhasvrijtaztemtbsipdb` int DEFAULT NULL,\n `kyducgxttathzhvaqklbtsoibkdosseu` int DEFAULT NULL,\n `xbatnewvckfnyvdgrpfknsiuapcnorxa` int DEFAULT NULL,\n `cuaukdeietzycnawmupzcowykzltpokl` int DEFAULT NULL,\n `rhjextdxfzoflpdxasgmpummrmdpejju` int DEFAULT NULL,\n `aedjeenkysxpgpxyfgjhgthtlslcffye` int DEFAULT NULL,\n `cewcaplmcgwnllaubnxfumhnscgpchau` int DEFAULT NULL,\n `mgznrtdjuotodtelfvogtemlfpgxicqr` int DEFAULT NULL,\n `jtipidtsnqsivrdzummaolqbvqzltyqs` int DEFAULT NULL,\n `derlulbgjtqmgwudpgcyhgnfxycduagf` int DEFAULT NULL,\n `jufzhtdnghxqbbqlkskjwpnzrifykzqq` int DEFAULT NULL,\n `yutponmyvmnxwfmbjmmohjdhzmswyvxx` int DEFAULT NULL,\n `mbibohsrzvprhdjympoybrdfbosvtcln` int DEFAULT NULL,\n `iqaxvfalyzqqxodysfvmvwzqusbhlmuk` int DEFAULT NULL,\n `frxtlggwcqyzfdgrqunguytvhsouwoeq` int DEFAULT NULL,\n `udamdgdxbljrszxnhynduahcxnjlqzwq` int DEFAULT NULL,\n `loziepfsmtyoaxsaqmxhmwzabrboavxt` int DEFAULT NULL,\n `xcxiuerbucylwzokwvhulmagvetnfgxv` int DEFAULT NULL,\n `rhqfdkrdztfbimdnjdxdenhvxtphljxc` int DEFAULT NULL,\n `uvnoptgaymzdtvqsariyidyprfxptszj` int DEFAULT NULL,\n `wlhfxtfbhslextvprystvwfuatedumcr` int DEFAULT NULL,\n `wpdgbdmjnirmlbrokhypjdzneryipqka` int DEFAULT NULL,\n `rdjyzhugguaasjnmbypkcpmiaqmlihef` int DEFAULT NULL,\n `ifxhbwpfluvsjjllfsuiqofcmzsfnmdc` int DEFAULT NULL,\n `mdvuvucntyomrvxisenmyfacgajqyxdw` int DEFAULT NULL,\n `cgpaogdapfmrzufqxbpzmikzmvkixvrm` int DEFAULT NULL,\n `lqfpaxpitprtbyntwbfjngdahftzyzql` int DEFAULT NULL,\n `wphyagoxbbtziioovfatsxzrogrzhpuf` int DEFAULT NULL,\n `gltlfvgnnztqgptkvaqwyuumqpmxvdoi` int DEFAULT NULL,\n `svtzxynixtobltalhnyrzmvhcqibvnpx` int DEFAULT NULL,\n `agiuvhkysfbhtnemfztmonzpbcwongxo` int DEFAULT NULL,\n `telktpjvkbjebvtwbirzttbfywyhueub` int DEFAULT NULL,\n `rfdksahtbchccmiqzkfhisysearlhgdy` int DEFAULT NULL,\n `pfvewbxvoylwyjjppqkwojrfstyaednv` int DEFAULT NULL,\n `eykbtooosqixzdadcfwzachppvyefhos` int DEFAULT NULL,\n `hivmvrauleyqfwrbcbynkggivuewzpta` int DEFAULT NULL,\n `nxdandksyztilvrosinusukwnxmnvcnd` int DEFAULT NULL,\n `tyjbnfjowrmpakapufaezrybhfpsatmq` int DEFAULT NULL,\n `pzcogjebecadcksvncbmzspbxqqgvdyv` int DEFAULT NULL,\n `emvepbscyrbzamvboyeqgclccnxxaeos` int DEFAULT NULL,\n `pptxtxnvgdxcxgymihktbwblhdnyuzyd` int DEFAULT NULL,\n `looqvrbhanjgmzeanveyyjojsuwdfcrb` int DEFAULT NULL,\n `hhdnfthffzizimulplveouutozwkhyhn` int DEFAULT NULL,\n `gxbkdswiuaokkrizjdngqgrncgergnhf` int DEFAULT NULL,\n `bzaiomzwhnfslwwfmfuljxtqllafhkrr` int DEFAULT NULL,\n `onaloluioogdoaournxbfkihslypblpz` int DEFAULT NULL,\n `vlwolmhfnuduljbclgbxwoscznglfqaf` int DEFAULT NULL,\n `xezwhgiryyismpfrblrzxrrpulzaswce` int DEFAULT NULL,\n `icnygpiiapxqthtfvlaqaiktovdyogou` int DEFAULT NULL,\n `moerzmkqtrqcjujwqhmcnlrfiifyhmoj` int DEFAULT NULL,\n `hixckidlcndpcrvsfireazikuzpkzlbv` int DEFAULT NULL,\n `pinprywlapumxbzabnfgnnfstufxctma` int DEFAULT NULL,\n `cizslxgftueifnbtehrjakiuoqitdvfh` int DEFAULT NULL,\n `sncykqktckoqupnrtznsttanpcqthots` int DEFAULT NULL,\n `pscsolxmnjziognsriorrgsfokynnhwp` int DEFAULT NULL,\n `xicpxglkokpsfxpdswsjbkoxevfxwjey` int DEFAULT NULL,\n `ksshvsqcmefptbtanurlifhogfysrtio` int DEFAULT NULL,\n `ikicnztmlpnathblesgmqicnuyenneur` int DEFAULT NULL,\n `pbmymqffccsvukxcbgeprsotizvaagws` int DEFAULT NULL,\n `yavplzrxamygeftephoffyjlgenxssyy` int DEFAULT NULL,\n `ysdgdssksegdzgvyunrnqymxbwdajfcu` int DEFAULT NULL,\n `fsevjptczjecgabnjvumvzmmwfnyiyqh` int DEFAULT NULL,\n `mmuuyapvxswoybwrnhnfpzizyrhlgdna` int DEFAULT NULL,\n `rbhwwazjoonwttfubouvclhscckdznes` int DEFAULT NULL,\n `jabzzevvpglkpbojftavmfleokcdycnu` int DEFAULT NULL,\n `tkxrzqnuncbegfixxikxrrlvmhlmygyc` int DEFAULT NULL,\n `ngorviiwjiyljrqokdwlgmurgxcqqtin` int DEFAULT NULL,\n `uwcbgverexwscjgvpccdjzfbpflpvbvl` int DEFAULT NULL,\n `zivkzaernpkvtwubqizbwgsytbvyejla` int DEFAULT NULL,\n `daayznabgyblwzkndpcacdvmvqvwcjqw` int DEFAULT NULL,\n `alolxmomztgscjbgnnswyytxqwvauwlv` int DEFAULT NULL,\n `ggwwhxcspvcgiubkkiajigiqeboymzai` int DEFAULT NULL,\n `upwnqagghydynypjnjujmyzsxsviqbrc` int DEFAULT NULL,\n PRIMARY KEY (`sthxctbpbnsfrrhrqgnoggptbofweior`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"brznqjnnrwxnevpxwjrcthnvpxrsawok\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["sthxctbpbnsfrrhrqgnoggptbofweior"],"columns":[{"name":"sthxctbpbnsfrrhrqgnoggptbofweior","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"pcwbuqijkvvnkjvyibcicirfjlplbsto","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzpcubjltjguzvmaebnheijpuybansjf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jorhcxrxdehwmyaelkzmiuiyiwougsrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvdajacjfvsjmptirfttclidjknlpnsi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gowtuaypwpttdpnszvjesdbtzotxjdcp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igznaqorutncosltvelyefagwytnylxp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icsmcxnumrhioomdksoogndgnmsrhkew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdxocmhpkjdkdmuwfadishqbpnbgptqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajjbafrnuzjfuuxdzhgswrfzhqwolhqe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcorzyttueqegazaabjofjhzwnzafncg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bewzqjqotprbxgrdurucrvxtktwdnmwx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyiyvastlokjjnsxajbtrrbczuqvysjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axumbgsmhibwqonkpwbxfnefpujeqesn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlcwkpznkfxynyytrhtoroopwxemqqgz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddmogzonkekzkortmxtsdmrdrrqfauzk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jcnfwnqjqvkivkkjhlgivijbwodetjpf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"voxgxcvxmhiqztavndkrkkpilckozplq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jituqvrangwfbsiqwfhbqpyffctkrcil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eijlvbviyiawbbostpgemhtnwwotidbm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"syonomlrybssgeylshjdsswyxyfyyuss","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbqxfjdwteherypbhwxvybqicmxczaqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmqlontcdvtwmkeuwjqfhjmoafprjljl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hnahkjtqfptuoknooituqmiihozswyth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qnzcwblgwimfsxtanfdwuegxusuweaby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufcnxnsdtssbqcwexjjnjnvatgmdxkei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tujvjwgxgzakklvluozipkaqogyxktfs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dfjocpbauhyzhasvrijtaztemtbsipdb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kyducgxttathzhvaqklbtsoibkdosseu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbatnewvckfnyvdgrpfknsiuapcnorxa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cuaukdeietzycnawmupzcowykzltpokl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhjextdxfzoflpdxasgmpummrmdpejju","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aedjeenkysxpgpxyfgjhgthtlslcffye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cewcaplmcgwnllaubnxfumhnscgpchau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mgznrtdjuotodtelfvogtemlfpgxicqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jtipidtsnqsivrdzummaolqbvqzltyqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"derlulbgjtqmgwudpgcyhgnfxycduagf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jufzhtdnghxqbbqlkskjwpnzrifykzqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yutponmyvmnxwfmbjmmohjdhzmswyvxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbibohsrzvprhdjympoybrdfbosvtcln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqaxvfalyzqqxodysfvmvwzqusbhlmuk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"frxtlggwcqyzfdgrqunguytvhsouwoeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udamdgdxbljrszxnhynduahcxnjlqzwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"loziepfsmtyoaxsaqmxhmwzabrboavxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xcxiuerbucylwzokwvhulmagvetnfgxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhqfdkrdztfbimdnjdxdenhvxtphljxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvnoptgaymzdtvqsariyidyprfxptszj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wlhfxtfbhslextvprystvwfuatedumcr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wpdgbdmjnirmlbrokhypjdzneryipqka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdjyzhugguaasjnmbypkcpmiaqmlihef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifxhbwpfluvsjjllfsuiqofcmzsfnmdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdvuvucntyomrvxisenmyfacgajqyxdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgpaogdapfmrzufqxbpzmikzmvkixvrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqfpaxpitprtbyntwbfjngdahftzyzql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wphyagoxbbtziioovfatsxzrogrzhpuf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gltlfvgnnztqgptkvaqwyuumqpmxvdoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"svtzxynixtobltalhnyrzmvhcqibvnpx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agiuvhkysfbhtnemfztmonzpbcwongxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"telktpjvkbjebvtwbirzttbfywyhueub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfdksahtbchccmiqzkfhisysearlhgdy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pfvewbxvoylwyjjppqkwojrfstyaednv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eykbtooosqixzdadcfwzachppvyefhos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hivmvrauleyqfwrbcbynkggivuewzpta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxdandksyztilvrosinusukwnxmnvcnd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tyjbnfjowrmpakapufaezrybhfpsatmq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pzcogjebecadcksvncbmzspbxqqgvdyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"emvepbscyrbzamvboyeqgclccnxxaeos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pptxtxnvgdxcxgymihktbwblhdnyuzyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"looqvrbhanjgmzeanveyyjojsuwdfcrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhdnfthffzizimulplveouutozwkhyhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxbkdswiuaokkrizjdngqgrncgergnhf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzaiomzwhnfslwwfmfuljxtqllafhkrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"onaloluioogdoaournxbfkihslypblpz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vlwolmhfnuduljbclgbxwoscznglfqaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xezwhgiryyismpfrblrzxrrpulzaswce","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icnygpiiapxqthtfvlaqaiktovdyogou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"moerzmkqtrqcjujwqhmcnlrfiifyhmoj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hixckidlcndpcrvsfireazikuzpkzlbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pinprywlapumxbzabnfgnnfstufxctma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cizslxgftueifnbtehrjakiuoqitdvfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sncykqktckoqupnrtznsttanpcqthots","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pscsolxmnjziognsriorrgsfokynnhwp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xicpxglkokpsfxpdswsjbkoxevfxwjey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ksshvsqcmefptbtanurlifhogfysrtio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikicnztmlpnathblesgmqicnuyenneur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbmymqffccsvukxcbgeprsotizvaagws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yavplzrxamygeftephoffyjlgenxssyy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ysdgdssksegdzgvyunrnqymxbwdajfcu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fsevjptczjecgabnjvumvzmmwfnyiyqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmuuyapvxswoybwrnhnfpzizyrhlgdna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbhwwazjoonwttfubouvclhscckdznes","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jabzzevvpglkpbojftavmfleokcdycnu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkxrzqnuncbegfixxikxrrlvmhlmygyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ngorviiwjiyljrqokdwlgmurgxcqqtin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwcbgverexwscjgvpccdjzfbpflpvbvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zivkzaernpkvtwubqizbwgsytbvyejla","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"daayznabgyblwzkndpcacdvmvqvwcjqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"alolxmomztgscjbgnnswyytxqwvauwlv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggwwhxcspvcgiubkkiajigiqeboymzai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upwnqagghydynypjnjujmyzsxsviqbrc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667067,"databaseName":"models_schema","ddl":"CREATE TABLE `cjbsiglppbkiwuokzgspunevqbbejmfp` (\n `rbgelgermwzasvqtktmleputckwzpdlm` int NOT NULL,\n `hmnvzmcliuvzuoshtqqvhnqdfalkiubx` int DEFAULT NULL,\n `yflhfpafwojxbrentxnnencwqfxaoltn` int DEFAULT NULL,\n `nxovbatoubbthpjgjnkedvtogsoexjcs` int DEFAULT NULL,\n `smwzvfpoulbfcqdlqgnkdvokvjcweeas` int DEFAULT NULL,\n `pyvkeoeuvscthwxjfcjgsdkfdrdzrmbz` int DEFAULT NULL,\n `jiyxoqfjavxivyqcpgmlqasajdccapng` int DEFAULT NULL,\n `ctdffgofydqzrddorydefmdwjrwzsihi` int DEFAULT NULL,\n `upwoflakeoilzosncsmswywfpyjfhxem` int DEFAULT NULL,\n `vhjkqekryxlzzeqydehleyuxitighzrj` int DEFAULT NULL,\n `yxxxoleouymhtntacmibamneuuovkrnr` int DEFAULT NULL,\n `xitakfayxqtkzwzzwnonlmeyxyiwzojb` int DEFAULT NULL,\n `ozfjjkrdrnddimtjanbzazuuxbdxucax` int DEFAULT NULL,\n `nivpjquwsnqvivjldyrzgisufjajhynu` int DEFAULT NULL,\n `acujwyqngvxpwyiocdaddklsdnxwvbmn` int DEFAULT NULL,\n `nqzrzryewssfljylsfeyzwfmriecvtav` int DEFAULT NULL,\n `ryvwflnumxftblhtqrebbdsltijwgdee` int DEFAULT NULL,\n `zkmxtnoxdjpbvswpgrtkaozrsrrszwnb` int DEFAULT NULL,\n `yaxautsickhkxnlfypyvfhjctselfjqs` int DEFAULT NULL,\n `jjaidngcqvrydmrlvekdskyrumhfbjdu` int DEFAULT NULL,\n `fhsjhqlqkopozjtseygosastppwajpxy` int DEFAULT NULL,\n `femcozvpqjkadtmsrekdtypmfvktvuct` int DEFAULT NULL,\n `vbojbbrfkvjidegrulzmrqunngninyaf` int DEFAULT NULL,\n `hvetfmrljmjulgooexdcwljcnspngmhp` int DEFAULT NULL,\n `lyqorvyiorifcwcyuumgcyoikegxqggx` int DEFAULT NULL,\n `wqvdpvqotdamxoigotmkfgowtrixehat` int DEFAULT NULL,\n `gozkqzshtyoschiufjrqetubinyrhuhp` int DEFAULT NULL,\n `mzsawniofdncmicqsksgiyezczehhltm` int DEFAULT NULL,\n `kzkvjbivnlcyfmbiznygforjfyxpcsjf` int DEFAULT NULL,\n `sbkidtmdgidmnvpuuvmjycinraiquejl` int DEFAULT NULL,\n `shbnzdeudqeqgjobnzqifqyokhzpzjzl` int DEFAULT NULL,\n `jiovocsjxvoabzrsdruzcedfhgjxiyyi` int DEFAULT NULL,\n `szioaiovnhskehksoryjsxixkzxjmyoa` int DEFAULT NULL,\n `darilnnkkpnbfarbakwsynpyaywjxdro` int DEFAULT NULL,\n `jqmjrurgxvreawcsksqwqhphhiofbnok` int DEFAULT NULL,\n `tvveasjzpxlhwveymllademmihogbdqp` int DEFAULT NULL,\n `iubyrsassrefqfyukqlpoicssyndmzve` int DEFAULT NULL,\n `wvpbewwsvdnoztcorbnhugbludnlgsco` int DEFAULT NULL,\n `qmnraiveyvsihkzwtfhjytrxxyuwnafi` int DEFAULT NULL,\n `cezwnrgegnlqdqutvzvmfqmjbgbyasml` int DEFAULT NULL,\n `izmnegqcjkvtzelfjsrlyhizenifhnze` int DEFAULT NULL,\n `agujlprsngizjnnplqmnkokxvscjuupw` int DEFAULT NULL,\n `sujafkepypemkcvaphecasnzgfrgbter` int DEFAULT NULL,\n `vewyrevzyrrwbyzpxzcsoxoneovjticn` int DEFAULT NULL,\n `ajiuqrfhddfxpwvmmoakvfvsxsahzsyf` int DEFAULT NULL,\n `ozuqzcebfhrasaujybbyuuzavbnsdvhe` int DEFAULT NULL,\n `wftqpizbttzjfrepzjgyzdnqybjhijvy` int DEFAULT NULL,\n `mvhkhbydokyckjpfsjuqwphtmmplqbyw` int DEFAULT NULL,\n `sqfiyxkfmaomcpgfkfuqpuuyldkiquau` int DEFAULT NULL,\n `hfnypgevhkqyvsaltnckzgfrmipxazzx` int DEFAULT NULL,\n `fuvykapsvpvtcruppzswacnyjpgxavfc` int DEFAULT NULL,\n `mlqaaoxgnefzloxaerzpmpkbqclscrfc` int DEFAULT NULL,\n `xmdhhffzunqhhgqfexwnuoxqutzawofg` int DEFAULT NULL,\n `olxaufkxeyhtbebyybsbkyorgxnvqrgc` int DEFAULT NULL,\n `fxwbkwjzfyxpbjaimcfnzkteoihxtcau` int DEFAULT NULL,\n `vnhcdkjdpnppzwiiowqkoecpuvhlkyyo` int DEFAULT NULL,\n `ckqbqcpumwzxmntvultkyzlgqathetfs` int DEFAULT NULL,\n `fjgpeargrlabxleopupgmdwkmyplaxgu` int DEFAULT NULL,\n `asipqknmmisntasyzvkzclvhmrsmvzni` int DEFAULT NULL,\n `mjmapfmjgwiqfiifemywjiprgagdpuzd` int DEFAULT NULL,\n `dybcbzuhpioqdoowagyqsrfrnxoyxwoz` int DEFAULT NULL,\n `uihuxpvhcooizgxxvczfuzazdvcvjvic` int DEFAULT NULL,\n `qvovueiytzipdnsaddlmmmqjxeqbbjmt` int DEFAULT NULL,\n `snofzoxshglqprpermiygsajryglrtdl` int DEFAULT NULL,\n `jktdyjdvbieccdmyvppisffdyqtebelq` int DEFAULT NULL,\n `jgpohtoncaunlfwpzvgaorypqqzxnfkj` int DEFAULT NULL,\n `mnpfawbniqjkoyqstlldwgjnjfcjtsyl` int DEFAULT NULL,\n `ydlxjuduxuyzwwrlpxtmxscresfkzlkp` int DEFAULT NULL,\n `iuhnyvbnthbshywsarhkxbglecvpjsot` int DEFAULT NULL,\n `jphjcxpueteggsqygfgsbgkwwcrxqyqs` int DEFAULT NULL,\n `orcxsgfowjsoxsdscfjogitxwjvrblno` int DEFAULT NULL,\n `dflnmrmakcvrbnezolppddgvkuesbtea` int DEFAULT NULL,\n `vsxbcjduoeggvaflwayrsfxrvmwyqnyp` int DEFAULT NULL,\n `wwwwilnawybxaprwbkqxhvbqtdvfxshi` int DEFAULT NULL,\n `issovmseujlrpuzpdsgkpmqfntduuyab` int DEFAULT NULL,\n `rwskkpwtjalvvovxvjiutalrgwlzhppl` int DEFAULT NULL,\n `ugozlrxnzzjirfmnteflfttqulhndfmn` int DEFAULT NULL,\n `jutqbggovwfebhavlyslsgstvigprmwd` int DEFAULT NULL,\n `yxcorjaymtsepptedsrkhbnyewmutgrd` int DEFAULT NULL,\n `djvjnxmlizivnujiqikzavzmkpzeyxoy` int DEFAULT NULL,\n `plvtrvwkibtflranedhaqgksnjcdtwgf` int DEFAULT NULL,\n `cqenejeakuumaplsxyylvvulgtkmgxxq` int DEFAULT NULL,\n `gztfjuciknojhbxkoqlpudtinmdrzgdd` int DEFAULT NULL,\n `nntqsronjsqxubjejuscdrenqtvfeyya` int DEFAULT NULL,\n `dickdknfcctdfxmhpuqdlcgiyqcnargc` int DEFAULT NULL,\n `hibbwzwkhdfcvxqluoqmmkuotuofddwd` int DEFAULT NULL,\n `wwxupufghotyhyijnoaflimytcvrlnaz` int DEFAULT NULL,\n `oduvksjjjkyztteyuiayovgkccnlyehr` int DEFAULT NULL,\n `aicfbnzloojzqrjwzccwjmggmfqtswrk` int DEFAULT NULL,\n `iaboxwvdzqoeuyhwdiiytrxiybbntknu` int DEFAULT NULL,\n `ptaprjabomsxtmpcrnumvfnksenxeelc` int DEFAULT NULL,\n `zeyloqdcmesbpvxafwycbqzuhvgnpgeh` int DEFAULT NULL,\n `zpnumqvsluilgihcpjyohgjaztbtcvbx` int DEFAULT NULL,\n `izyoyfsfnkdpxpavckokgftggsddfjac` int DEFAULT NULL,\n `jsgkbeqappnebuvaslgzpczmhvndoyax` int DEFAULT NULL,\n `ibyaidmrchbvxybcerbvowwvuazjojld` int DEFAULT NULL,\n `ztdqgbrywdwekiqxvdbgkbtplczdownu` int DEFAULT NULL,\n `lhmehodykwclxinhrlclzyzizjemonyp` int DEFAULT NULL,\n `ntcrmjoeihtbdhtpxbjrvgkskdihroqn` int DEFAULT NULL,\n `opxmiwkaarlywamiogyfyewckptnwrwu` int DEFAULT NULL,\n PRIMARY KEY (`rbgelgermwzasvqtktmleputckwzpdlm`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"cjbsiglppbkiwuokzgspunevqbbejmfp\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["rbgelgermwzasvqtktmleputckwzpdlm"],"columns":[{"name":"rbgelgermwzasvqtktmleputckwzpdlm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"hmnvzmcliuvzuoshtqqvhnqdfalkiubx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yflhfpafwojxbrentxnnencwqfxaoltn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxovbatoubbthpjgjnkedvtogsoexjcs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smwzvfpoulbfcqdlqgnkdvokvjcweeas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pyvkeoeuvscthwxjfcjgsdkfdrdzrmbz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jiyxoqfjavxivyqcpgmlqasajdccapng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctdffgofydqzrddorydefmdwjrwzsihi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upwoflakeoilzosncsmswywfpyjfhxem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhjkqekryxlzzeqydehleyuxitighzrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxxxoleouymhtntacmibamneuuovkrnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xitakfayxqtkzwzzwnonlmeyxyiwzojb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozfjjkrdrnddimtjanbzazuuxbdxucax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nivpjquwsnqvivjldyrzgisufjajhynu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acujwyqngvxpwyiocdaddklsdnxwvbmn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqzrzryewssfljylsfeyzwfmriecvtav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ryvwflnumxftblhtqrebbdsltijwgdee","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zkmxtnoxdjpbvswpgrtkaozrsrrszwnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yaxautsickhkxnlfypyvfhjctselfjqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjaidngcqvrydmrlvekdskyrumhfbjdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhsjhqlqkopozjtseygosastppwajpxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"femcozvpqjkadtmsrekdtypmfvktvuct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbojbbrfkvjidegrulzmrqunngninyaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hvetfmrljmjulgooexdcwljcnspngmhp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lyqorvyiorifcwcyuumgcyoikegxqggx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqvdpvqotdamxoigotmkfgowtrixehat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gozkqzshtyoschiufjrqetubinyrhuhp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzsawniofdncmicqsksgiyezczehhltm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzkvjbivnlcyfmbiznygforjfyxpcsjf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbkidtmdgidmnvpuuvmjycinraiquejl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shbnzdeudqeqgjobnzqifqyokhzpzjzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jiovocsjxvoabzrsdruzcedfhgjxiyyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"szioaiovnhskehksoryjsxixkzxjmyoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"darilnnkkpnbfarbakwsynpyaywjxdro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jqmjrurgxvreawcsksqwqhphhiofbnok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvveasjzpxlhwveymllademmihogbdqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iubyrsassrefqfyukqlpoicssyndmzve","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvpbewwsvdnoztcorbnhugbludnlgsco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmnraiveyvsihkzwtfhjytrxxyuwnafi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cezwnrgegnlqdqutvzvmfqmjbgbyasml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"izmnegqcjkvtzelfjsrlyhizenifhnze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agujlprsngizjnnplqmnkokxvscjuupw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sujafkepypemkcvaphecasnzgfrgbter","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vewyrevzyrrwbyzpxzcsoxoneovjticn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajiuqrfhddfxpwvmmoakvfvsxsahzsyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozuqzcebfhrasaujybbyuuzavbnsdvhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wftqpizbttzjfrepzjgyzdnqybjhijvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvhkhbydokyckjpfsjuqwphtmmplqbyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sqfiyxkfmaomcpgfkfuqpuuyldkiquau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfnypgevhkqyvsaltnckzgfrmipxazzx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fuvykapsvpvtcruppzswacnyjpgxavfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlqaaoxgnefzloxaerzpmpkbqclscrfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmdhhffzunqhhgqfexwnuoxqutzawofg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"olxaufkxeyhtbebyybsbkyorgxnvqrgc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxwbkwjzfyxpbjaimcfnzkteoihxtcau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnhcdkjdpnppzwiiowqkoecpuvhlkyyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckqbqcpumwzxmntvultkyzlgqathetfs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjgpeargrlabxleopupgmdwkmyplaxgu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"asipqknmmisntasyzvkzclvhmrsmvzni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjmapfmjgwiqfiifemywjiprgagdpuzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dybcbzuhpioqdoowagyqsrfrnxoyxwoz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uihuxpvhcooizgxxvczfuzazdvcvjvic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvovueiytzipdnsaddlmmmqjxeqbbjmt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"snofzoxshglqprpermiygsajryglrtdl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jktdyjdvbieccdmyvppisffdyqtebelq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgpohtoncaunlfwpzvgaorypqqzxnfkj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnpfawbniqjkoyqstlldwgjnjfcjtsyl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydlxjuduxuyzwwrlpxtmxscresfkzlkp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iuhnyvbnthbshywsarhkxbglecvpjsot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jphjcxpueteggsqygfgsbgkwwcrxqyqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orcxsgfowjsoxsdscfjogitxwjvrblno","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dflnmrmakcvrbnezolppddgvkuesbtea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsxbcjduoeggvaflwayrsfxrvmwyqnyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwwwilnawybxaprwbkqxhvbqtdvfxshi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"issovmseujlrpuzpdsgkpmqfntduuyab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rwskkpwtjalvvovxvjiutalrgwlzhppl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugozlrxnzzjirfmnteflfttqulhndfmn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jutqbggovwfebhavlyslsgstvigprmwd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxcorjaymtsepptedsrkhbnyewmutgrd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"djvjnxmlizivnujiqikzavzmkpzeyxoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"plvtrvwkibtflranedhaqgksnjcdtwgf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqenejeakuumaplsxyylvvulgtkmgxxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gztfjuciknojhbxkoqlpudtinmdrzgdd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nntqsronjsqxubjejuscdrenqtvfeyya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dickdknfcctdfxmhpuqdlcgiyqcnargc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hibbwzwkhdfcvxqluoqmmkuotuofddwd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwxupufghotyhyijnoaflimytcvrlnaz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oduvksjjjkyztteyuiayovgkccnlyehr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aicfbnzloojzqrjwzccwjmggmfqtswrk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iaboxwvdzqoeuyhwdiiytrxiybbntknu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ptaprjabomsxtmpcrnumvfnksenxeelc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zeyloqdcmesbpvxafwycbqzuhvgnpgeh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpnumqvsluilgihcpjyohgjaztbtcvbx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"izyoyfsfnkdpxpavckokgftggsddfjac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jsgkbeqappnebuvaslgzpczmhvndoyax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibyaidmrchbvxybcerbvowwvuazjojld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ztdqgbrywdwekiqxvdbgkbtplczdownu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhmehodykwclxinhrlclzyzizjemonyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntcrmjoeihtbdhtpxbjrvgkskdihroqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opxmiwkaarlywamiogyfyewckptnwrwu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667103,"databaseName":"models_schema","ddl":"CREATE TABLE `crfncrfexubbsoqfpqdpbclcqxnclxmt` (\n `muhpxjnrxlewxtihfeojrtdfymulyblt` int NOT NULL,\n `fhfomwsdlnehxzpmidhsecefrxwcmgus` int DEFAULT NULL,\n `muhqwttgqvkdcmqhrtxucmvqunvfmrig` int DEFAULT NULL,\n `gzpmpclqxliodsbwetqefufqhlccsnre` int DEFAULT NULL,\n `rrcpzizkxnhjfxiimonprhatitsluwme` int DEFAULT NULL,\n `qugyqnhzuoytzrmaqdmkrklzhuljqpzn` int DEFAULT NULL,\n `duabbtphvqsmufccozmchfuxumcmithh` int DEFAULT NULL,\n `fxgzgmtxsbvtwpxhdlbyyosnguwjxnar` int DEFAULT NULL,\n `xnziczwbzrcsijpkielrhakarmsniyqg` int DEFAULT NULL,\n `dgvudemkfqdaulfyzaadxdicsjlliiyh` int DEFAULT NULL,\n `fmztgnbrcmiqaqizokoeejqosoflhbgi` int DEFAULT NULL,\n `lkykgesmedxdgmaemqwawniwxpufapdt` int DEFAULT NULL,\n `bnxridqjefvfmxtktxlyzmjpyskkqpnt` int DEFAULT NULL,\n `nkbcolvewkpxhaqfbbjipbvscpkcrkuf` int DEFAULT NULL,\n `kxzlpkmlkazlmzxwsvzodbhwbmyghhvo` int DEFAULT NULL,\n `iijgvdcmumuevthmvyhcyrjmuafbxrxv` int DEFAULT NULL,\n `llgijwbjsflthggsjrpjogxjgkufeupv` int DEFAULT NULL,\n `brnxrdrukstzapuykbiddbafmtvrhovs` int DEFAULT NULL,\n `detcffdwpxbqmwymnzlponhstfdqgnnc` int DEFAULT NULL,\n `grqavvmevzooaykcmvclqgesrrfccadk` int DEFAULT NULL,\n `gdtzdnchxkfzqxjulmrlmlwycyllszgy` int DEFAULT NULL,\n `embavlwwmvmgidqnmdeohunodkajkurf` int DEFAULT NULL,\n `cxebyxexkqyiivabigfmtrdogwzjoeek` int DEFAULT NULL,\n `mmzjfomwesmyjdxfggongszrqlbszygb` int DEFAULT NULL,\n `hzozcivehqvgijfqdnhufvayfvcsrjpd` int DEFAULT NULL,\n `mxvzmgtxxuytxoaxgokhqmlfqfzutpom` int DEFAULT NULL,\n `potfoledzztrfbpafzptrcuczddkyycx` int DEFAULT NULL,\n `epkpzdinxmgzsbitucekxgdyqickbmla` int DEFAULT NULL,\n `fmswgvvmojryxyfnvohgtekabqtvutay` int DEFAULT NULL,\n `prafruewapooqxynugguckelqswvixvo` int DEFAULT NULL,\n `lhwgrnqwvzmzsimygchsqwjpozotxjqg` int DEFAULT NULL,\n `hnfslkowwyhdqucaoiwqabhrhfhrinqu` int DEFAULT NULL,\n `lradqqkzkfupaacmspcuyklopznkeneb` int DEFAULT NULL,\n `ypjnfgynwqlnvsprpfikuuognrsjekns` int DEFAULT NULL,\n `kcdkuukeotzqvqmuwwhgwkltdikgquzz` int DEFAULT NULL,\n `tmeaxdlhslbxqgcnmnzixrceewllfels` int DEFAULT NULL,\n `vcmwmefnatvtxdzqeoaogpudvrktwpor` int DEFAULT NULL,\n `eqbldbsqzsxoqqrfwrgehdygidfteari` int DEFAULT NULL,\n `azojjoxtdyjqzqmecerexmciyabbnklf` int DEFAULT NULL,\n `qbpmlbxgngblsncgwyrcdamsmxoqfwyb` int DEFAULT NULL,\n `sfcohcffnwjkhgydosoxllumfjfngwbx` int DEFAULT NULL,\n `rrcyznhokqxmelblsezxdmkvhlemnidy` int DEFAULT NULL,\n `wpknbisjfavvcxxksjtwynfpvgylrewl` int DEFAULT NULL,\n `wpyqpevzuyvjryrscconcjjzsvdxxqpy` int DEFAULT NULL,\n `unnudeogzpruypbgzzinrbqyglypuxql` int DEFAULT NULL,\n `mptqartanxmfxaohpeyjimrvybyexrjz` int DEFAULT NULL,\n `wwanegraycnfqvphhllsunmmfctbuzit` int DEFAULT NULL,\n `qvciwhffnmcepzikoyvqxpckuzujvdwq` int DEFAULT NULL,\n `rehnswiveupspgfztfqeztidhelkzesd` int DEFAULT NULL,\n `vjfpwscncldzcsqzpogzwdvxotumjmdd` int DEFAULT NULL,\n `ynldactheipcrzwnwmaissixrcctfoya` int DEFAULT NULL,\n `cnxccoyrfwjizovzzkcllenpiceowktb` int DEFAULT NULL,\n `yobqkjpixciqarpbfvfgpgfhyweofson` int DEFAULT NULL,\n `gvclbwthhiwvysmuchunrhggaklmtjyr` int DEFAULT NULL,\n `pdjersvdylhnebmjxcgyqpwijixsrbwp` int DEFAULT NULL,\n `ggsniconqfsidjwjdbikxijtriujnaoz` int DEFAULT NULL,\n `paicrcbniqodptanvmfinpjhnirnvkig` int DEFAULT NULL,\n `xvzdrnzrtvwmdykhrfbtpplukzuvnbyz` int DEFAULT NULL,\n `xtgohwhvmkohpfoapteekmmqdoibpjym` int DEFAULT NULL,\n `upsxvdrqrxehwfdcnvbrnmbibacepfjj` int DEFAULT NULL,\n `horjpsqpjbmumrxdxjtscldroeqtrtba` int DEFAULT NULL,\n `dtpafducrnwvyxlphquslosmevwekeam` int DEFAULT NULL,\n `cctgfpgazrzkykrbziivyshxhvqdvxiw` int DEFAULT NULL,\n `fbxtxcrfleqhvrtqmdsgwxaspzbxtrqf` int DEFAULT NULL,\n `faqbivvzscbhaakznlzkbxuthrevwrlw` int DEFAULT NULL,\n `ozkfefihkqrfhnuwstzpuutellzvfglt` int DEFAULT NULL,\n `bgdgsagwoflmxdlzbbdhllpyzntmvcgb` int DEFAULT NULL,\n `plrwkhmdsmirwbuvarunywhpdcdkwbxt` int DEFAULT NULL,\n `ivhcjjtwpmlwinrdiebjdqnhfinvtvrg` int DEFAULT NULL,\n `rcdshtahrbdzuzrddvassxtalebbneze` int DEFAULT NULL,\n `mhsnuktfdbtbgtyviqptlhuwpjblrfqt` int DEFAULT NULL,\n `ehymojijgzgppfvefruboosuvobcekox` int DEFAULT NULL,\n `nohkmfzlpsjcpnwjypmiiwnoyehwhfgm` int DEFAULT NULL,\n `fbgbtdymhwghseidxdtkfvetsaxmpkxb` int DEFAULT NULL,\n `akymnehbdtsufjgnhtrkyqoascnxjhld` int DEFAULT NULL,\n `igkxzpvgpomgbkblezziwddsiqonnkai` int DEFAULT NULL,\n `febkedbognzryakhctxwbycmgfwgglyb` int DEFAULT NULL,\n `zivrsmekguksbjbiwrvagvbgbgdnzezd` int DEFAULT NULL,\n `ysnqtfodogozjrzyihmwtndyzlloczer` int DEFAULT NULL,\n `imcxztptuvcdqsbidtyiatltfcsctwys` int DEFAULT NULL,\n `xjofnxitllatjxrucswgcnbaymletkfa` int DEFAULT NULL,\n `ildbaymufivfummikovjstzjgehzyrnm` int DEFAULT NULL,\n `xsrqgrjkyppldmlwtaqzalptayvexxxv` int DEFAULT NULL,\n `npptgbxgmphawjishbefuksznsjwktpc` int DEFAULT NULL,\n `cthqmhffrcldgwtwhzflpfxkiusxijvi` int DEFAULT NULL,\n `flsypklgsohkmumtedovtfjjglovibvb` int DEFAULT NULL,\n `jlnvswznslqbetbavtybiizpknqwtksp` int DEFAULT NULL,\n `kqxtsbhseaysfctjcscttwhuordpidya` int DEFAULT NULL,\n `lxgzwdwxpaciorzgyqmmcbqpizqbimxc` int DEFAULT NULL,\n `nzixpfxppdrvlblvudsundaumuvvrzhr` int DEFAULT NULL,\n `omxlkoapislbgavnhbqafduuzopzblmm` int DEFAULT NULL,\n `uaxmsqxzgmksblfsamhsctmawprduada` int DEFAULT NULL,\n `skzyliaqcihfvxrgqqdlugzhebwjgdxc` int DEFAULT NULL,\n `xmrurnsndimbojgjylyqknrcnxtrifsy` int DEFAULT NULL,\n `lrcdrhtafvisomljbmhxlubxnksagcrl` int DEFAULT NULL,\n `othlihdytjlthkycgrvokzylvrmhjiul` int DEFAULT NULL,\n `rrsgfhnhysemmlivjcxuzgrdhebleypc` int DEFAULT NULL,\n `xeqowmqsxztmmjlqdsusdonubiujczxq` int DEFAULT NULL,\n `wtyowobezuvenudyaqaidccrpojgdwqa` int DEFAULT NULL,\n `chpbeaaiwvonkpfceiklkvevojiyitnd` int DEFAULT NULL,\n PRIMARY KEY (`muhpxjnrxlewxtihfeojrtdfymulyblt`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"crfncrfexubbsoqfpqdpbclcqxnclxmt\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["muhpxjnrxlewxtihfeojrtdfymulyblt"],"columns":[{"name":"muhpxjnrxlewxtihfeojrtdfymulyblt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"fhfomwsdlnehxzpmidhsecefrxwcmgus","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"muhqwttgqvkdcmqhrtxucmvqunvfmrig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzpmpclqxliodsbwetqefufqhlccsnre","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrcpzizkxnhjfxiimonprhatitsluwme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qugyqnhzuoytzrmaqdmkrklzhuljqpzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"duabbtphvqsmufccozmchfuxumcmithh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxgzgmtxsbvtwpxhdlbyyosnguwjxnar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnziczwbzrcsijpkielrhakarmsniyqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgvudemkfqdaulfyzaadxdicsjlliiyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmztgnbrcmiqaqizokoeejqosoflhbgi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lkykgesmedxdgmaemqwawniwxpufapdt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnxridqjefvfmxtktxlyzmjpyskkqpnt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkbcolvewkpxhaqfbbjipbvscpkcrkuf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxzlpkmlkazlmzxwsvzodbhwbmyghhvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iijgvdcmumuevthmvyhcyrjmuafbxrxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llgijwbjsflthggsjrpjogxjgkufeupv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"brnxrdrukstzapuykbiddbafmtvrhovs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"detcffdwpxbqmwymnzlponhstfdqgnnc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grqavvmevzooaykcmvclqgesrrfccadk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdtzdnchxkfzqxjulmrlmlwycyllszgy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"embavlwwmvmgidqnmdeohunodkajkurf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cxebyxexkqyiivabigfmtrdogwzjoeek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmzjfomwesmyjdxfggongszrqlbszygb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hzozcivehqvgijfqdnhufvayfvcsrjpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mxvzmgtxxuytxoaxgokhqmlfqfzutpom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"potfoledzztrfbpafzptrcuczddkyycx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"epkpzdinxmgzsbitucekxgdyqickbmla","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmswgvvmojryxyfnvohgtekabqtvutay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prafruewapooqxynugguckelqswvixvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhwgrnqwvzmzsimygchsqwjpozotxjqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hnfslkowwyhdqucaoiwqabhrhfhrinqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lradqqkzkfupaacmspcuyklopznkeneb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypjnfgynwqlnvsprpfikuuognrsjekns","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcdkuukeotzqvqmuwwhgwkltdikgquzz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmeaxdlhslbxqgcnmnzixrceewllfels","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vcmwmefnatvtxdzqeoaogpudvrktwpor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqbldbsqzsxoqqrfwrgehdygidfteari","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azojjoxtdyjqzqmecerexmciyabbnklf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qbpmlbxgngblsncgwyrcdamsmxoqfwyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfcohcffnwjkhgydosoxllumfjfngwbx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrcyznhokqxmelblsezxdmkvhlemnidy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wpknbisjfavvcxxksjtwynfpvgylrewl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wpyqpevzuyvjryrscconcjjzsvdxxqpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unnudeogzpruypbgzzinrbqyglypuxql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mptqartanxmfxaohpeyjimrvybyexrjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwanegraycnfqvphhllsunmmfctbuzit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvciwhffnmcepzikoyvqxpckuzujvdwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rehnswiveupspgfztfqeztidhelkzesd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjfpwscncldzcsqzpogzwdvxotumjmdd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynldactheipcrzwnwmaissixrcctfoya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnxccoyrfwjizovzzkcllenpiceowktb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yobqkjpixciqarpbfvfgpgfhyweofson","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gvclbwthhiwvysmuchunrhggaklmtjyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdjersvdylhnebmjxcgyqpwijixsrbwp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggsniconqfsidjwjdbikxijtriujnaoz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"paicrcbniqodptanvmfinpjhnirnvkig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvzdrnzrtvwmdykhrfbtpplukzuvnbyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xtgohwhvmkohpfoapteekmmqdoibpjym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upsxvdrqrxehwfdcnvbrnmbibacepfjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"horjpsqpjbmumrxdxjtscldroeqtrtba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtpafducrnwvyxlphquslosmevwekeam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cctgfpgazrzkykrbziivyshxhvqdvxiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbxtxcrfleqhvrtqmdsgwxaspzbxtrqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"faqbivvzscbhaakznlzkbxuthrevwrlw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozkfefihkqrfhnuwstzpuutellzvfglt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bgdgsagwoflmxdlzbbdhllpyzntmvcgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"plrwkhmdsmirwbuvarunywhpdcdkwbxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivhcjjtwpmlwinrdiebjdqnhfinvtvrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcdshtahrbdzuzrddvassxtalebbneze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhsnuktfdbtbgtyviqptlhuwpjblrfqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehymojijgzgppfvefruboosuvobcekox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nohkmfzlpsjcpnwjypmiiwnoyehwhfgm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbgbtdymhwghseidxdtkfvetsaxmpkxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akymnehbdtsufjgnhtrkyqoascnxjhld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igkxzpvgpomgbkblezziwddsiqonnkai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"febkedbognzryakhctxwbycmgfwgglyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zivrsmekguksbjbiwrvagvbgbgdnzezd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ysnqtfodogozjrzyihmwtndyzlloczer","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"imcxztptuvcdqsbidtyiatltfcsctwys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjofnxitllatjxrucswgcnbaymletkfa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ildbaymufivfummikovjstzjgehzyrnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsrqgrjkyppldmlwtaqzalptayvexxxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npptgbxgmphawjishbefuksznsjwktpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cthqmhffrcldgwtwhzflpfxkiusxijvi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flsypklgsohkmumtedovtfjjglovibvb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlnvswznslqbetbavtybiizpknqwtksp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqxtsbhseaysfctjcscttwhuordpidya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxgzwdwxpaciorzgyqmmcbqpizqbimxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzixpfxppdrvlblvudsundaumuvvrzhr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omxlkoapislbgavnhbqafduuzopzblmm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uaxmsqxzgmksblfsamhsctmawprduada","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skzyliaqcihfvxrgqqdlugzhebwjgdxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmrurnsndimbojgjylyqknrcnxtrifsy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lrcdrhtafvisomljbmhxlubxnksagcrl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"othlihdytjlthkycgrvokzylvrmhjiul","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrsgfhnhysemmlivjcxuzgrdhebleypc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xeqowmqsxztmmjlqdsusdonubiujczxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtyowobezuvenudyaqaidccrpojgdwqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chpbeaaiwvonkpfceiklkvevojiyitnd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667138,"databaseName":"models_schema","ddl":"CREATE TABLE `crgfzudloyryxyhjtwycumowtyrpwsly` (\n `vtatooubvbwxxhuqsnkdyntvbsgrfafn` int NOT NULL,\n `bdzeaahelfpxazurxndndcchsbxuwctp` int DEFAULT NULL,\n `zjakrteborjwnqplurfvvrdishwfgfib` int DEFAULT NULL,\n `ypmkkhfqbfojohjpcrwouumomfctechl` int DEFAULT NULL,\n `fdkplvqernbfqeohnvasxqrumyqiqupj` int DEFAULT NULL,\n `lqrsjmygxfawjwljcrwtztgxjbrbjiwj` int DEFAULT NULL,\n `eteuskjobfgcxgtugrjakjfgzxpgbvnh` int DEFAULT NULL,\n `vqgdtqzimidekpikzjumfrtrarmefjyd` int DEFAULT NULL,\n `gzwdpslrcpxpivrrazkuygnxhhprbfau` int DEFAULT NULL,\n `bdakgzcahcfwepesqezrgdafkulykrut` int DEFAULT NULL,\n `gartwgzhborzhchnocqgueswcsmllzyd` int DEFAULT NULL,\n `ozzqyjwtbxetptmphqbknegfhxpsyfhm` int DEFAULT NULL,\n `menioxqhfmuxtysocjfekfqstpjeioni` int DEFAULT NULL,\n `pcosyrpokwzcqoldtwfqnxmpmslkjauq` int DEFAULT NULL,\n `upcxdwfjehmeowlvzmgddodbfkbtcgwt` int DEFAULT NULL,\n `wxrynsespjrzdfrertbxjpzlyqacqnvn` int DEFAULT NULL,\n `hvhjokaqcrcrilhqdmujfvqksiddxuvf` int DEFAULT NULL,\n `dxzgtjkegwhajahuhlbjdbltariorfmu` int DEFAULT NULL,\n `fvmxwfqvregjardabjgjlkjgtvkollnc` int DEFAULT NULL,\n `ucpkdqvehywwuwdnkmzpdztajreagvmw` int DEFAULT NULL,\n `nuavukgzmdchnpsjjqxjpsrvxpvlnaqm` int DEFAULT NULL,\n `fbeqppowxbquegqkacggabyjgkvsitfe` int DEFAULT NULL,\n `dpsoigbecmbosjmgrlmovigsgnldajiw` int DEFAULT NULL,\n `fahmgopmevxtbfkheexpaaifdtwkejom` int DEFAULT NULL,\n `rjevlfczqczlggjiikhjgeedgyvalypp` int DEFAULT NULL,\n `inzfdxivkkysbtybkdovdfhxwecluxkq` int DEFAULT NULL,\n `yjmoupvdgtfkxgnmjlnmwpkmdtgcyxoo` int DEFAULT NULL,\n `vkytxhbelpyisntmijwfcovzeseejfom` int DEFAULT NULL,\n `agxfngzcoicuauiavkbyqnzykzrbacrp` int DEFAULT NULL,\n `fylslanbcncnoanomljfkvaipnqcjtup` int DEFAULT NULL,\n `zdxvecawxjozqsnddfbngffknlfesvub` int DEFAULT NULL,\n `gjnwlvwntpklwjwtkdhgxzcalgtybelm` int DEFAULT NULL,\n `bsexnufhckifedefjtjvwortdofuzkco` int DEFAULT NULL,\n `kfermnqkddixaxyumpjfplwdxpamzcmg` int DEFAULT NULL,\n `cwfnnsnkddqveuanhpwlzaqkpvcczblx` int DEFAULT NULL,\n `meedgmgdurvmuqgbbikyvjlzgenahrsi` int DEFAULT NULL,\n `fhuoeburzvtdkbohpxgucqppjnifdgrq` int DEFAULT NULL,\n `nhrfbdtxmngiosqvjjcbysdvywlrjoyf` int DEFAULT NULL,\n `mevcftpizplqbgjbtxuozjmpmzdppzvf` int DEFAULT NULL,\n `nuppxjqwxrzfitkexvlmjcfpyrhrqfrt` int DEFAULT NULL,\n `mbjfrqxvfhmuvmsmiqskzsyebszykeky` int DEFAULT NULL,\n `eetkzjhyxnzfpwrkaujaxngsopwivwux` int DEFAULT NULL,\n `kfxlndfmvhvwsyjpvmdpfuvaqpkrxwqh` int DEFAULT NULL,\n `dzmaozzqedubrxaiaomxrrfvmvrdesaj` int DEFAULT NULL,\n `aklshqrbfburjjpylqzwvniynypljttl` int DEFAULT NULL,\n `xvqamvzguwjvotliicwdteareogrbeod` int DEFAULT NULL,\n `iqwvgjpdpbbrgncexxrjqyumvvziwlou` int DEFAULT NULL,\n `cnhxuglsqbxobyevgonpreznkqtnpokq` int DEFAULT NULL,\n `jzxexyejylitvrdatdhtgtzjbsilmzko` int DEFAULT NULL,\n `yianmkinqnrcvkxwaercvjooiaocivsp` int DEFAULT NULL,\n `vehiractnyqbxyexmhgejpesfqayjbjb` int DEFAULT NULL,\n `zoxkqhlckicpslycsjvcaxafkyeiordw` int DEFAULT NULL,\n `poabcphoeqrfgfoifjwtxdgrmimjzxfp` int DEFAULT NULL,\n `bcayarqxfybzbddxwnlymnrekrkszgcs` int DEFAULT NULL,\n `iparhfnjgbxenajsxwpjsylqdslefthj` int DEFAULT NULL,\n `htlpbmfhgsmgzdhatgdzbsfhsrourihi` int DEFAULT NULL,\n `gdhkurddhlakeldvseojvbrbwvnkuwtr` int DEFAULT NULL,\n `rphuxpnqbanoletxvcesylhqqkbnmnft` int DEFAULT NULL,\n `tehpmrfiqqkxfrqzyjasnumxwznfcdcr` int DEFAULT NULL,\n `rnxascvvcdtxdrbrusxdcdemqhavfctv` int DEFAULT NULL,\n `yeipwlbixgexajexucmnitvoqvwvmxev` int DEFAULT NULL,\n `zysjkaoffshrgttodgsafmhbaytybvvl` int DEFAULT NULL,\n `aakfvxnknkjpmbxciwqeovzjhuhymqfv` int DEFAULT NULL,\n `ifvdvilvvpljbyefcaperslggrtcfnje` int DEFAULT NULL,\n `khgjrurnpoyhqiitdhbovzhlzxafqvkr` int DEFAULT NULL,\n `mmsqdwocnecwawlqtcliqtypzkljepeo` int DEFAULT NULL,\n `jobukfjwenoserzrfwdskbwilwegcobs` int DEFAULT NULL,\n `cpafoerkklriqlnotgbfxnphxanvvaed` int DEFAULT NULL,\n `leakjgdfvjsjkfkxsizqkvggvwgzckvf` int DEFAULT NULL,\n `dyovprqpbqzvmpwnolhvslghxbmppjmq` int DEFAULT NULL,\n `gfmnndljfhhofouvpkmsunckzqytzhlz` int DEFAULT NULL,\n `lztoctvjzonzctlqumcmdebfpjqurvli` int DEFAULT NULL,\n `xpayrvtjznovkykvqzfywevfpqtafedi` int DEFAULT NULL,\n `pouiqxpdkkwfhhpryviqzhtsdcgbakoo` int DEFAULT NULL,\n `udxabjbtxycsodjznclrcrnzysqeikev` int DEFAULT NULL,\n `kydtcrrxivszlzyoteopszjiuoyqxvps` int DEFAULT NULL,\n `hgblwbegbmrgrmwbcsalgjaqywywypah` int DEFAULT NULL,\n `snsrpdxrbcbkterwwetkuljcpabppwnv` int DEFAULT NULL,\n `oyjcmnjlbbfnlkjriyjdflatsfalybpr` int DEFAULT NULL,\n `zftuliizwnhdtonvlwruihnynosirlsa` int DEFAULT NULL,\n `fchvzwzhahwhkiiqblsctwpawdkosjdt` int DEFAULT NULL,\n `ilwzfxurqzhtrshgsrbbucewqklgeywy` int DEFAULT NULL,\n `wkltgobnkitllrgpkbnkovcjverjahkn` int DEFAULT NULL,\n `qjykvtbuawvqmmmjgbkaoggcsqqazfck` int DEFAULT NULL,\n `fymgacpzwnpobslfoqtmjrroynyadebj` int DEFAULT NULL,\n `vsixvxfgakfpddvvjmbwsbccbgcpdaad` int DEFAULT NULL,\n `mmbzrrnphtoucflttirswclekzagazue` int DEFAULT NULL,\n `fjlovcthtxspqvtxdeycxjzvjoycvmpz` int DEFAULT NULL,\n `owhyuklrnoiwmxhmpuhlojtqbudkoltz` int DEFAULT NULL,\n `potzvtxjchgvwawagxcutyvvnohkbkfv` int DEFAULT NULL,\n `vhygqixjivwwkbmqdmssswqmilvhzupw` int DEFAULT NULL,\n `ypumincaenxzayibflypvbxdzmbyayns` int DEFAULT NULL,\n `hkcsrqnnwlkpynfngfeiutzdjtrnkhym` int DEFAULT NULL,\n `dxkzyyazkapmokonvspzcfewwreppeai` int DEFAULT NULL,\n `ybkgaaswefkxfigvlqqtofkjounfgueq` int DEFAULT NULL,\n `gxiunchhcbonmovwqxadufbnjsskvqkc` int DEFAULT NULL,\n `nysfvkeuwlhmoramzxednleggccmqwih` int DEFAULT NULL,\n `stwzyzbfeqzxbgokgrrevxvxdydocjtl` int DEFAULT NULL,\n `csscewmdchaofkvgdkgdnlyaamhksxja` int DEFAULT NULL,\n `zbocykxcyyychrfollzanghbfmreitnn` int DEFAULT NULL,\n PRIMARY KEY (`vtatooubvbwxxhuqsnkdyntvbsgrfafn`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"crgfzudloyryxyhjtwycumowtyrpwsly\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["vtatooubvbwxxhuqsnkdyntvbsgrfafn"],"columns":[{"name":"vtatooubvbwxxhuqsnkdyntvbsgrfafn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"bdzeaahelfpxazurxndndcchsbxuwctp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjakrteborjwnqplurfvvrdishwfgfib","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypmkkhfqbfojohjpcrwouumomfctechl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fdkplvqernbfqeohnvasxqrumyqiqupj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqrsjmygxfawjwljcrwtztgxjbrbjiwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eteuskjobfgcxgtugrjakjfgzxpgbvnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqgdtqzimidekpikzjumfrtrarmefjyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzwdpslrcpxpivrrazkuygnxhhprbfau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdakgzcahcfwepesqezrgdafkulykrut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gartwgzhborzhchnocqgueswcsmllzyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozzqyjwtbxetptmphqbknegfhxpsyfhm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"menioxqhfmuxtysocjfekfqstpjeioni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pcosyrpokwzcqoldtwfqnxmpmslkjauq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upcxdwfjehmeowlvzmgddodbfkbtcgwt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxrynsespjrzdfrertbxjpzlyqacqnvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hvhjokaqcrcrilhqdmujfvqksiddxuvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxzgtjkegwhajahuhlbjdbltariorfmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fvmxwfqvregjardabjgjlkjgtvkollnc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucpkdqvehywwuwdnkmzpdztajreagvmw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nuavukgzmdchnpsjjqxjpsrvxpvlnaqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbeqppowxbquegqkacggabyjgkvsitfe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpsoigbecmbosjmgrlmovigsgnldajiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fahmgopmevxtbfkheexpaaifdtwkejom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjevlfczqczlggjiikhjgeedgyvalypp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"inzfdxivkkysbtybkdovdfhxwecluxkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjmoupvdgtfkxgnmjlnmwpkmdtgcyxoo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkytxhbelpyisntmijwfcovzeseejfom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agxfngzcoicuauiavkbyqnzykzrbacrp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fylslanbcncnoanomljfkvaipnqcjtup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdxvecawxjozqsnddfbngffknlfesvub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjnwlvwntpklwjwtkdhgxzcalgtybelm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bsexnufhckifedefjtjvwortdofuzkco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfermnqkddixaxyumpjfplwdxpamzcmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwfnnsnkddqveuanhpwlzaqkpvcczblx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"meedgmgdurvmuqgbbikyvjlzgenahrsi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhuoeburzvtdkbohpxgucqppjnifdgrq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhrfbdtxmngiosqvjjcbysdvywlrjoyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mevcftpizplqbgjbtxuozjmpmzdppzvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nuppxjqwxrzfitkexvlmjcfpyrhrqfrt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbjfrqxvfhmuvmsmiqskzsyebszykeky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eetkzjhyxnzfpwrkaujaxngsopwivwux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfxlndfmvhvwsyjpvmdpfuvaqpkrxwqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dzmaozzqedubrxaiaomxrrfvmvrdesaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aklshqrbfburjjpylqzwvniynypljttl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvqamvzguwjvotliicwdteareogrbeod","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqwvgjpdpbbrgncexxrjqyumvvziwlou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnhxuglsqbxobyevgonpreznkqtnpokq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzxexyejylitvrdatdhtgtzjbsilmzko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yianmkinqnrcvkxwaercvjooiaocivsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vehiractnyqbxyexmhgejpesfqayjbjb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zoxkqhlckicpslycsjvcaxafkyeiordw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"poabcphoeqrfgfoifjwtxdgrmimjzxfp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bcayarqxfybzbddxwnlymnrekrkszgcs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iparhfnjgbxenajsxwpjsylqdslefthj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htlpbmfhgsmgzdhatgdzbsfhsrourihi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdhkurddhlakeldvseojvbrbwvnkuwtr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rphuxpnqbanoletxvcesylhqqkbnmnft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tehpmrfiqqkxfrqzyjasnumxwznfcdcr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnxascvvcdtxdrbrusxdcdemqhavfctv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yeipwlbixgexajexucmnitvoqvwvmxev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zysjkaoffshrgttodgsafmhbaytybvvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aakfvxnknkjpmbxciwqeovzjhuhymqfv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifvdvilvvpljbyefcaperslggrtcfnje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khgjrurnpoyhqiitdhbovzhlzxafqvkr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmsqdwocnecwawlqtcliqtypzkljepeo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jobukfjwenoserzrfwdskbwilwegcobs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cpafoerkklriqlnotgbfxnphxanvvaed","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"leakjgdfvjsjkfkxsizqkvggvwgzckvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dyovprqpbqzvmpwnolhvslghxbmppjmq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfmnndljfhhofouvpkmsunckzqytzhlz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lztoctvjzonzctlqumcmdebfpjqurvli","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpayrvtjznovkykvqzfywevfpqtafedi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pouiqxpdkkwfhhpryviqzhtsdcgbakoo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udxabjbtxycsodjznclrcrnzysqeikev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kydtcrrxivszlzyoteopszjiuoyqxvps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hgblwbegbmrgrmwbcsalgjaqywywypah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"snsrpdxrbcbkterwwetkuljcpabppwnv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oyjcmnjlbbfnlkjriyjdflatsfalybpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zftuliizwnhdtonvlwruihnynosirlsa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fchvzwzhahwhkiiqblsctwpawdkosjdt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ilwzfxurqzhtrshgsrbbucewqklgeywy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wkltgobnkitllrgpkbnkovcjverjahkn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjykvtbuawvqmmmjgbkaoggcsqqazfck","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fymgacpzwnpobslfoqtmjrroynyadebj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsixvxfgakfpddvvjmbwsbccbgcpdaad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmbzrrnphtoucflttirswclekzagazue","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjlovcthtxspqvtxdeycxjzvjoycvmpz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owhyuklrnoiwmxhmpuhlojtqbudkoltz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"potzvtxjchgvwawagxcutyvvnohkbkfv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhygqixjivwwkbmqdmssswqmilvhzupw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypumincaenxzayibflypvbxdzmbyayns","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkcsrqnnwlkpynfngfeiutzdjtrnkhym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxkzyyazkapmokonvspzcfewwreppeai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybkgaaswefkxfigvlqqtofkjounfgueq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxiunchhcbonmovwqxadufbnjsskvqkc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nysfvkeuwlhmoramzxednleggccmqwih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"stwzyzbfeqzxbgokgrrevxvxdydocjtl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"csscewmdchaofkvgdkgdnlyaamhksxja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbocykxcyyychrfollzanghbfmreitnn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667172,"databaseName":"models_schema","ddl":"CREATE TABLE `cvducdrtmbckboypjfdtljlqxnmjlvqo` (\n `zctqhgksitzmgkrwqgiylsveuxhgpibh` int NOT NULL,\n `swrkewqmuejcohwtmvpdraqzylbrgfay` int DEFAULT NULL,\n `qmaxvucayegolbgosxtnucwkdmwqrhnw` int DEFAULT NULL,\n `zrejxyqmjizawjnyjogwujrebwnakusa` int DEFAULT NULL,\n `pgvjjdewpahsboeyyazvjadkljwhcike` int DEFAULT NULL,\n `jhdhegeltgcxxplzucekpwqbmzgxcblk` int DEFAULT NULL,\n `zujgirelelmgdajlyanhwridkqkbfgvt` int DEFAULT NULL,\n `fsgjzevwysgxlyvrljvhwxbezneuojdq` int DEFAULT NULL,\n `cypmpbruxxcqycwyybocuscikpfzveud` int DEFAULT NULL,\n `oapdkbjzlltybhbhctpopbqsabbcxzla` int DEFAULT NULL,\n `zfgktnxuacpdovvvbipldepxdemyuuyh` int DEFAULT NULL,\n `dmqccvprjhuaszaqroizgiwprlfwwrgb` int DEFAULT NULL,\n `jdsbetasbqmsdbylyziwytrenbwidkge` int DEFAULT NULL,\n `aeokklhczgngjamtqbwuhnhniuhezecx` int DEFAULT NULL,\n `pgrxpaitohgpsddlpadcocsaoglllpuj` int DEFAULT NULL,\n `ffipjgspxeotumrsrjyqzzcckzrivzdu` int DEFAULT NULL,\n `fjcedbtacqhhtxpxyvqwnaknkdfsgsdf` int DEFAULT NULL,\n `jbajviuumjxmyjtuzflglzfpjhtoffjt` int DEFAULT NULL,\n `magprybcghurwtvvdbiogzunkmqnylfk` int DEFAULT NULL,\n `ejojxvtxkmaaufhovxfmoandlywwppuy` int DEFAULT NULL,\n `ifpkdmpztwlbpewgrsspyuzzrqtthfas` int DEFAULT NULL,\n `mabggiudnfkvcrpluxiomlyzafhbvwta` int DEFAULT NULL,\n `hktqboxfzkcydvdpluaolxqunovdsdfr` int DEFAULT NULL,\n `ikzjngbapeixsnkikaqtkjkndmpkeqdz` int DEFAULT NULL,\n `ipxzjslhmijfkxpxapdaeyjqxqtjvirg` int DEFAULT NULL,\n `rrqjtqqronezgeswluvkuaycoxxkzubd` int DEFAULT NULL,\n `eiqgqdiyrjaiqqatpgnebbmkqmimrovz` int DEFAULT NULL,\n `qtpdntolcvgqjrlwmxbbxitjeelapcfp` int DEFAULT NULL,\n `lwaykjuqzwujxhwbpwnbwjyilxhwtesz` int DEFAULT NULL,\n `glefbrtienynvmrploevzsvhfpdjbqjq` int DEFAULT NULL,\n `dtwrdqdiqihbxdlzgqibohczamiovdiy` int DEFAULT NULL,\n `kculhgsojbylmdoovtwtzswotylatpol` int DEFAULT NULL,\n `iiqttkogiqrjoglysiluhjsdjzxpblhp` int DEFAULT NULL,\n `fkznwjwjjpewggcvbwpshxbtlrjevolf` int DEFAULT NULL,\n `ysippiohkxzddjxuxktzekpvwqnmivtg` int DEFAULT NULL,\n `avaldieaaxqmiwmgwtbihlwinqvodomn` int DEFAULT NULL,\n `keowphmmthlbbcndeecnqyeadrhqjfmk` int DEFAULT NULL,\n `owfueeslcqnlnvqvjtmdyvffvbbtylhz` int DEFAULT NULL,\n `fwgzejpzrcqhewqejuwpmqlmwjpynwye` int DEFAULT NULL,\n `qhrqiwgjcdusefbgoxhpcebxshyirjhv` int DEFAULT NULL,\n `hrcsxybvnioxhesypjekegojjopdbipm` int DEFAULT NULL,\n `dgdmztlaoklrovfppamfjnqxltbdorrf` int DEFAULT NULL,\n `mgnuybptcxxfsgueowodvkfnwmzmdkhi` int DEFAULT NULL,\n `kowukkyeiqsnbuibkhlzwibewzppazsf` int DEFAULT NULL,\n `bkzaxpljgeggsgaodblgqsgkiawsncur` int DEFAULT NULL,\n `lznrqmpgwdzociwkwknijskmqcfvvjhk` int DEFAULT NULL,\n `cldlbrsqgazuumgavqdgpwypzaevlcdd` int DEFAULT NULL,\n `ajecqfjwabvnhafhflqbxighrkcrxieu` int DEFAULT NULL,\n `uexztbwxlqlbomxvxtonpkghztucwixf` int DEFAULT NULL,\n `jlujtrzawemimyrbblsxuihxpevrmgyp` int DEFAULT NULL,\n `geoldoygbyuyxcodztefvqxfaojujltj` int DEFAULT NULL,\n `kcociozeyvwpbxejmcyzmvalrbfzofxs` int DEFAULT NULL,\n `bnqevcjjcdhqyavziagxxaqhcrohnadm` int DEFAULT NULL,\n `mwemqzytoxqbmqthmjxpezpxpnaeeclp` int DEFAULT NULL,\n `riqtdjjmvdlpnnqrqyvtpviosvnynrar` int DEFAULT NULL,\n `ljlejyyybwuauytghmgrvhmhruhufrnn` int DEFAULT NULL,\n `ninupqwzuwtytqvstvzhqpzmibbhszqy` int DEFAULT NULL,\n `wffuxbhwwpeethlcdtswftbswjeletrw` int DEFAULT NULL,\n `bvwgxhqbnfriknmokemvfooxlzyvdzzu` int DEFAULT NULL,\n `guvojyjuobkbltlhejmpimonwobdrvmg` int DEFAULT NULL,\n `xbqrxrdbfpizpcdwzkdcytihhydvudlw` int DEFAULT NULL,\n `teskhontfwnsyxahbhvpbuomuzmygerw` int DEFAULT NULL,\n `qdzvaxfbtuqbhziapdrulraxmkhbyffz` int DEFAULT NULL,\n `pknbesdgcjgxkvfzbeaxsnpfghztmcch` int DEFAULT NULL,\n `ekctotrfsctzxqdnsgfzjeqhiiqygmac` int DEFAULT NULL,\n `zitnyzuheptvatyqprktkraryzbtthwa` int DEFAULT NULL,\n `ktoyqqqwpkofkzogpniioxhjgjqumygz` int DEFAULT NULL,\n `viwswsvplxzdyaxltcjyrbbndjrvyrpd` int DEFAULT NULL,\n `wngaiddkdmqcflocutcrmgqwgyjodfbe` int DEFAULT NULL,\n `wmmpzvlzudyvlujpywdyifjgsjmqmugh` int DEFAULT NULL,\n `gdrlbagstilxghuuzptkomntkytxpvyt` int DEFAULT NULL,\n `eetxzjedhohezmfwxkuviemrnwqwsmxd` int DEFAULT NULL,\n `wbmcaedkdjstyaoijcpberofeynwwbtb` int DEFAULT NULL,\n `tmknjcorrjuwlxogapaaplnxdbvvkjpm` int DEFAULT NULL,\n `bixrggyemzvrmpdufpokzsygqdkrwxao` int DEFAULT NULL,\n `wcsrikijhsrqvibwcvsjvqcipolrpmge` int DEFAULT NULL,\n `lizrbzabxryuiaikfwceqgshkfxcqmcc` int DEFAULT NULL,\n `ozujorzrddxeeognpwytgzxjmxcmdqgs` int DEFAULT NULL,\n `sagaisoybxinhwsjiclittwbliczqpix` int DEFAULT NULL,\n `orzeecjymkwnndteoogldcingrjxrucl` int DEFAULT NULL,\n `vzutucpwiqmimuckascudzmglvsquvdu` int DEFAULT NULL,\n `pzjotwdnwcbnmghulgqmxvaqsljihhfu` int DEFAULT NULL,\n `kthzxupmebjgfrekyvvbmvdebceojgwo` int DEFAULT NULL,\n `hktrdywdloqsnfxumvhlvuivdbzxayqd` int DEFAULT NULL,\n `xkppajkrcsuenkrpmngloojsjryzcfje` int DEFAULT NULL,\n `myduhnpamgqafoqwwawucwqcmuyogifc` int DEFAULT NULL,\n `eoqqerdlbankshzjiipbjllvdmetmcbp` int DEFAULT NULL,\n `tkmnicpacpmxvahzcslrnzqyhtssglia` int DEFAULT NULL,\n `ytebcnhjyqqxnpcwxfgddtddrrvxdffr` int DEFAULT NULL,\n `gzctpxuqgjbmgwxwhvzqvrrlutrmrzmw` int DEFAULT NULL,\n `zacmqkqsoguwjznjceygkjiiwnggjgzi` int DEFAULT NULL,\n `ahepdoonpfrahfoembomgwzkuzjnfqie` int DEFAULT NULL,\n `biinhqftiibicjfkvtgxbtghoymuhgbg` int DEFAULT NULL,\n `hmibnmavcjwzxekstdkidbxllwvzjwzg` int DEFAULT NULL,\n `dschfkapbbwbgnvpumloydtuhbyvwohl` int DEFAULT NULL,\n `jgdajjuieotukgiurqjozegbknixdcoa` int DEFAULT NULL,\n `rdfktwtkmfamronbkocgmhnhsddigybu` int DEFAULT NULL,\n `yivmycbbpmahqzsgombgotuzeumvcykn` int DEFAULT NULL,\n `ikilomkgzgeweifbmtelcnilehypvbvi` int DEFAULT NULL,\n `iockgjrjekzthwawyqmtjqmugkgzrjqq` int DEFAULT NULL,\n PRIMARY KEY (`zctqhgksitzmgkrwqgiylsveuxhgpibh`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"cvducdrtmbckboypjfdtljlqxnmjlvqo\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["zctqhgksitzmgkrwqgiylsveuxhgpibh"],"columns":[{"name":"zctqhgksitzmgkrwqgiylsveuxhgpibh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"swrkewqmuejcohwtmvpdraqzylbrgfay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmaxvucayegolbgosxtnucwkdmwqrhnw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zrejxyqmjizawjnyjogwujrebwnakusa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgvjjdewpahsboeyyazvjadkljwhcike","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jhdhegeltgcxxplzucekpwqbmzgxcblk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zujgirelelmgdajlyanhwridkqkbfgvt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fsgjzevwysgxlyvrljvhwxbezneuojdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cypmpbruxxcqycwyybocuscikpfzveud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oapdkbjzlltybhbhctpopbqsabbcxzla","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zfgktnxuacpdovvvbipldepxdemyuuyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dmqccvprjhuaszaqroizgiwprlfwwrgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdsbetasbqmsdbylyziwytrenbwidkge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aeokklhczgngjamtqbwuhnhniuhezecx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgrxpaitohgpsddlpadcocsaoglllpuj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffipjgspxeotumrsrjyqzzcckzrivzdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjcedbtacqhhtxpxyvqwnaknkdfsgsdf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbajviuumjxmyjtuzflglzfpjhtoffjt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"magprybcghurwtvvdbiogzunkmqnylfk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejojxvtxkmaaufhovxfmoandlywwppuy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifpkdmpztwlbpewgrsspyuzzrqtthfas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mabggiudnfkvcrpluxiomlyzafhbvwta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hktqboxfzkcydvdpluaolxqunovdsdfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikzjngbapeixsnkikaqtkjkndmpkeqdz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ipxzjslhmijfkxpxapdaeyjqxqtjvirg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrqjtqqronezgeswluvkuaycoxxkzubd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eiqgqdiyrjaiqqatpgnebbmkqmimrovz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtpdntolcvgqjrlwmxbbxitjeelapcfp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwaykjuqzwujxhwbpwnbwjyilxhwtesz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"glefbrtienynvmrploevzsvhfpdjbqjq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtwrdqdiqihbxdlzgqibohczamiovdiy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kculhgsojbylmdoovtwtzswotylatpol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iiqttkogiqrjoglysiluhjsdjzxpblhp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fkznwjwjjpewggcvbwpshxbtlrjevolf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ysippiohkxzddjxuxktzekpvwqnmivtg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avaldieaaxqmiwmgwtbihlwinqvodomn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"keowphmmthlbbcndeecnqyeadrhqjfmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owfueeslcqnlnvqvjtmdyvffvbbtylhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwgzejpzrcqhewqejuwpmqlmwjpynwye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhrqiwgjcdusefbgoxhpcebxshyirjhv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hrcsxybvnioxhesypjekegojjopdbipm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgdmztlaoklrovfppamfjnqxltbdorrf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mgnuybptcxxfsgueowodvkfnwmzmdkhi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kowukkyeiqsnbuibkhlzwibewzppazsf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkzaxpljgeggsgaodblgqsgkiawsncur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lznrqmpgwdzociwkwknijskmqcfvvjhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cldlbrsqgazuumgavqdgpwypzaevlcdd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajecqfjwabvnhafhflqbxighrkcrxieu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uexztbwxlqlbomxvxtonpkghztucwixf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlujtrzawemimyrbblsxuihxpevrmgyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"geoldoygbyuyxcodztefvqxfaojujltj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcociozeyvwpbxejmcyzmvalrbfzofxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnqevcjjcdhqyavziagxxaqhcrohnadm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwemqzytoxqbmqthmjxpezpxpnaeeclp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"riqtdjjmvdlpnnqrqyvtpviosvnynrar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljlejyyybwuauytghmgrvhmhruhufrnn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ninupqwzuwtytqvstvzhqpzmibbhszqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wffuxbhwwpeethlcdtswftbswjeletrw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvwgxhqbnfriknmokemvfooxlzyvdzzu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"guvojyjuobkbltlhejmpimonwobdrvmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbqrxrdbfpizpcdwzkdcytihhydvudlw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"teskhontfwnsyxahbhvpbuomuzmygerw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdzvaxfbtuqbhziapdrulraxmkhbyffz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pknbesdgcjgxkvfzbeaxsnpfghztmcch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekctotrfsctzxqdnsgfzjeqhiiqygmac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zitnyzuheptvatyqprktkraryzbtthwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ktoyqqqwpkofkzogpniioxhjgjqumygz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"viwswsvplxzdyaxltcjyrbbndjrvyrpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wngaiddkdmqcflocutcrmgqwgyjodfbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmmpzvlzudyvlujpywdyifjgsjmqmugh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdrlbagstilxghuuzptkomntkytxpvyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eetxzjedhohezmfwxkuviemrnwqwsmxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbmcaedkdjstyaoijcpberofeynwwbtb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmknjcorrjuwlxogapaaplnxdbvvkjpm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bixrggyemzvrmpdufpokzsygqdkrwxao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcsrikijhsrqvibwcvsjvqcipolrpmge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lizrbzabxryuiaikfwceqgshkfxcqmcc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozujorzrddxeeognpwytgzxjmxcmdqgs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sagaisoybxinhwsjiclittwbliczqpix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orzeecjymkwnndteoogldcingrjxrucl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzutucpwiqmimuckascudzmglvsquvdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pzjotwdnwcbnmghulgqmxvaqsljihhfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kthzxupmebjgfrekyvvbmvdebceojgwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hktrdywdloqsnfxumvhlvuivdbzxayqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkppajkrcsuenkrpmngloojsjryzcfje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"myduhnpamgqafoqwwawucwqcmuyogifc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eoqqerdlbankshzjiipbjllvdmetmcbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkmnicpacpmxvahzcslrnzqyhtssglia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytebcnhjyqqxnpcwxfgddtddrrvxdffr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzctpxuqgjbmgwxwhvzqvrrlutrmrzmw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zacmqkqsoguwjznjceygkjiiwnggjgzi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahepdoonpfrahfoembomgwzkuzjnfqie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"biinhqftiibicjfkvtgxbtghoymuhgbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmibnmavcjwzxekstdkidbxllwvzjwzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dschfkapbbwbgnvpumloydtuhbyvwohl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgdajjuieotukgiurqjozegbknixdcoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdfktwtkmfamronbkocgmhnhsddigybu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yivmycbbpmahqzsgombgotuzeumvcykn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikilomkgzgeweifbmtelcnilehypvbvi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iockgjrjekzthwawyqmtjqmugkgzrjqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667203,"databaseName":"models_schema","ddl":"CREATE TABLE `cxgskvumnqmwipfupfwozqwmfnftsxvs` (\n `tywhaxyehyyjvkzpxzbknapwvpuqntxb` int NOT NULL,\n `bllqhjtcrdsortiqzanaorhggjdarlky` int DEFAULT NULL,\n `bkiecnanarghhsmujtcvrjmckcjrjewg` int DEFAULT NULL,\n `tvwxwfcxetaaetypjimieszepnhfpahw` int DEFAULT NULL,\n `zawjppjkbziqbijjgfgtabxhiywquovj` int DEFAULT NULL,\n `ikhqnheoifjjllbzvejhtwqpkxnxjzej` int DEFAULT NULL,\n `mqnuzjkbytwzvxzwapaqloghfobgvsvv` int DEFAULT NULL,\n `touwdxrfcgpafmncsuynzzchgnjrhfum` int DEFAULT NULL,\n `ymvtyfvockhhhanaxmllpjxrbpvvbexh` int DEFAULT NULL,\n `rjfagmpswdxnuxhgxcxfzwfclqvjtbdn` int DEFAULT NULL,\n `pbeldklhewbbrzwmknrkjvsiodxrptbc` int DEFAULT NULL,\n `afgcglwrlffxfqbdvixaimswqeomoaym` int DEFAULT NULL,\n `ohitkaacmuorixnvifofgjnlbagmdhpt` int DEFAULT NULL,\n `yxpwcycwjixlybnxgwmacvoalpimruoi` int DEFAULT NULL,\n `jrtrvgsvmblmcqrygqtnwvselmjzvoiu` int DEFAULT NULL,\n `qmuuphrebtsrpsjwltlyuvtipgzqridr` int DEFAULT NULL,\n `yzkwexzbdpfflwhezzeefjkmtqifaaba` int DEFAULT NULL,\n `xzepwiqbykdsodhbkmzebzziywwdhzxz` int DEFAULT NULL,\n `dzdnjscxhiqmdqybkudwlxbjyztidcpr` int DEFAULT NULL,\n `bacnxjfyedyyrkhmmdexgidjispshatw` int DEFAULT NULL,\n `dphyraiomzyusnjeigumkhgmrvcktchh` int DEFAULT NULL,\n `mfejxjefvrmwptyzsraacrwpvvxfbfri` int DEFAULT NULL,\n `iyzxezkmdxuydbnmfrkbwmmawkjichsp` int DEFAULT NULL,\n `ikndojxwvkiwzywrqfpwpuldhrvcjuso` int DEFAULT NULL,\n `bbxmjxhxwtsaczqugfftlfrxppynjqnb` int DEFAULT NULL,\n `gbeehjgoacegtgqmrhbsfriwtuichhjc` int DEFAULT NULL,\n `uttbxbwcnodwrcxrzyiisfwoyjgkusoc` int DEFAULT NULL,\n `mjgdvucpctysovmwpklrcbapvdvuldjp` int DEFAULT NULL,\n `sljmmojgpdbpziqmuylvzlfnrrfpvcll` int DEFAULT NULL,\n `fcgwgreohtnkykixpismnctdhpkcqrxb` int DEFAULT NULL,\n `wubqigryudrjiulvecphouyerhrpgasv` int DEFAULT NULL,\n `kwslfzxlbrbokxcknoecsziqyuzknoqr` int DEFAULT NULL,\n `odvizirpugvfmlcuflvlnuyzxdrsunax` int DEFAULT NULL,\n `woifirbhmfaksnlgbwdbfcxkfqrntdzp` int DEFAULT NULL,\n `bgnanaznanfacfzoxmrfcpwanusalcqe` int DEFAULT NULL,\n `nqvicdlazmynrycgineevbmtomqtacfa` int DEFAULT NULL,\n `dnvoqfoolhvvfobaztcvzizlrbcroksw` int DEFAULT NULL,\n `bjpldujeguxxbtvhsxkocdsfshiaagda` int DEFAULT NULL,\n `oxsqxksfbcsbwpgqmuyndkkaylavklmn` int DEFAULT NULL,\n `xnloafibjyqazhfgvjcqciqgexkevmkl` int DEFAULT NULL,\n `glkjzuggafnavcyzhpnkcguwpgjahbmj` int DEFAULT NULL,\n `pbwciaapjyejlnqaexiqmywfimtqrsrf` int DEFAULT NULL,\n `wyqnbxvxvjuowcfellkyunkyvtvqkqmb` int DEFAULT NULL,\n `audqofwcdyxlxsrrzfmkngwxlwhfekvv` int DEFAULT NULL,\n `ibasukhfiyttmezailfigekaodhnfzeg` int DEFAULT NULL,\n `bffckrywrsgnqfaqutleahvbpixnjlbq` int DEFAULT NULL,\n `bzqusxnnodiiizypudkoeeivwgximpdl` int DEFAULT NULL,\n `ibwxmwlruwftoxhsjwtuwkkwulnfichq` int DEFAULT NULL,\n `vddvyxpzitgjopvimndwgvcwbfparkjw` int DEFAULT NULL,\n `eunrhwojkdpelzdgrnivpiohcdmluqpk` int DEFAULT NULL,\n `ohvreawyktgaqpmcgaxhutcuxmjdrbhf` int DEFAULT NULL,\n `cfhrxrsflghizlwgtwlswrnzgcdhrhtv` int DEFAULT NULL,\n `eoaaxtrctanvvmyfnbgobkebjzdeabkh` int DEFAULT NULL,\n `rhfckujwlscxpcsingpkoxymagntvtij` int DEFAULT NULL,\n `amvkarhemfihchrkjhwxtoeljnkvjnkr` int DEFAULT NULL,\n `hgjwjgjlgiylgcvwubiajzdbvwbcryld` int DEFAULT NULL,\n `lyegxpxxzfvlliqqfvgqxtbewkwscwdk` int DEFAULT NULL,\n `hxtlpsnavxkwnmvdayeimqwzjolorrjy` int DEFAULT NULL,\n `gapcppfoxviliruvesufpeaojunnouci` int DEFAULT NULL,\n `wmlhaenumdtwsivgsvgmywtbprkhzffp` int DEFAULT NULL,\n `xqiwcfvtnsjgouhjxosnhibaomdjlzpk` int DEFAULT NULL,\n `hhwmacdxsmhadzxrdkkyljqylmgxrusz` int DEFAULT NULL,\n `phbuzboslmsljoyfzzkcxhyzpixrwxss` int DEFAULT NULL,\n `tthowighkuafsuccbfhpzzyzqcutjscg` int DEFAULT NULL,\n `ullzukrzgqvluzxoywyqarhgwibbuzdj` int DEFAULT NULL,\n `gnxnglknsrppmolxwgwfnyjqkpkvnspg` int DEFAULT NULL,\n `fcmppxyaaltmtsafhwizekarlljqtexu` int DEFAULT NULL,\n `rkwuftfrookssxqesjtzkkeruaseidvq` int DEFAULT NULL,\n `lnyafipfrhrkybczcqbgkfsrobrukqht` int DEFAULT NULL,\n `bpmackcsvsocmntbllthqkmqpvuovgbp` int DEFAULT NULL,\n `vmxarpdaqtlwtqbvdrtoojzdxogazlem` int DEFAULT NULL,\n `stjqfyrihudtpjwzfwoomhnqhujrjubh` int DEFAULT NULL,\n `zcegsnlnxtcizuxxyohssxavndnzzgkw` int DEFAULT NULL,\n `hvzsrqkpwjatikyiffezgzqsspfgdxfw` int DEFAULT NULL,\n `tgluyhhhlphlzyhmtkmtgpdtxnsuktne` int DEFAULT NULL,\n `uymmheeevemfdztvwvdmcpwvbprmijka` int DEFAULT NULL,\n `pvokwqjcoqvkghkcnboqdgmosfvlkzya` int DEFAULT NULL,\n `ckacdpvkjdmczwlwxaizityxqkuiwbza` int DEFAULT NULL,\n `sapaypvwowzigbhkqncbyuwsbpaxnjsp` int DEFAULT NULL,\n `gewvcrvgzhemwopyesooddtwtvjonabd` int DEFAULT NULL,\n `noepxkhjmdkywpejdjiwocicyobvzmmt` int DEFAULT NULL,\n `ucsetvnkwtmybazkykvaektjbhdcnfsr` int DEFAULT NULL,\n `iltnsbfvmurlbvplswilgbmkzbqvhxay` int DEFAULT NULL,\n `esntyrfiarcxovylvzztraklhumxjxgn` int DEFAULT NULL,\n `gvrxrebgapejhjchhunnzdlvfwkejvmg` int DEFAULT NULL,\n `evfubzqusfvwnawjhapsztvakkxfatwh` int DEFAULT NULL,\n `ncebgayhaxnyouyihmipijytdtotvras` int DEFAULT NULL,\n `xnzwutzmfalldkseujxjtdpygqeqycce` int DEFAULT NULL,\n `dzqqsfgtknyarlnoiisdosgmcbldvpng` int DEFAULT NULL,\n `yyvzodgyqwtbxuvwistwtrsuksjzeccj` int DEFAULT NULL,\n `vxxoiveblkrelrwezqxotsisaziemsqm` int DEFAULT NULL,\n `opwqbpspiwdjgsamndfqhqpzatarwuty` int DEFAULT NULL,\n `rpjekraoazsdsxeotkfuvnhmvofvufaj` int DEFAULT NULL,\n `qnavykbwejpjwfwosxbavqqhpvvwxjsm` int DEFAULT NULL,\n `smucwtmbxfxfbsblxcffurmbmpbtgbqw` int DEFAULT NULL,\n `ynjviteceykmyfobjschpeorwnqlemxb` int DEFAULT NULL,\n `zcsqddxrrlycvbzqmbhpwqltuogbkzfl` int DEFAULT NULL,\n `rpqcihadlkddoqmebflloactcxzibaym` int DEFAULT NULL,\n `hvavdlpzcemgieibmnpvsaxfowowredy` int DEFAULT NULL,\n `wbzsmfrkbbqgnodlgytlknuepfgmtxfm` int DEFAULT NULL,\n PRIMARY KEY (`tywhaxyehyyjvkzpxzbknapwvpuqntxb`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"cxgskvumnqmwipfupfwozqwmfnftsxvs\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["tywhaxyehyyjvkzpxzbknapwvpuqntxb"],"columns":[{"name":"tywhaxyehyyjvkzpxzbknapwvpuqntxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"bllqhjtcrdsortiqzanaorhggjdarlky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkiecnanarghhsmujtcvrjmckcjrjewg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvwxwfcxetaaetypjimieszepnhfpahw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zawjppjkbziqbijjgfgtabxhiywquovj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikhqnheoifjjllbzvejhtwqpkxnxjzej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqnuzjkbytwzvxzwapaqloghfobgvsvv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"touwdxrfcgpafmncsuynzzchgnjrhfum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ymvtyfvockhhhanaxmllpjxrbpvvbexh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjfagmpswdxnuxhgxcxfzwfclqvjtbdn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbeldklhewbbrzwmknrkjvsiodxrptbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afgcglwrlffxfqbdvixaimswqeomoaym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohitkaacmuorixnvifofgjnlbagmdhpt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxpwcycwjixlybnxgwmacvoalpimruoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jrtrvgsvmblmcqrygqtnwvselmjzvoiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmuuphrebtsrpsjwltlyuvtipgzqridr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzkwexzbdpfflwhezzeefjkmtqifaaba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xzepwiqbykdsodhbkmzebzziywwdhzxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dzdnjscxhiqmdqybkudwlxbjyztidcpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bacnxjfyedyyrkhmmdexgidjispshatw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dphyraiomzyusnjeigumkhgmrvcktchh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfejxjefvrmwptyzsraacrwpvvxfbfri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iyzxezkmdxuydbnmfrkbwmmawkjichsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikndojxwvkiwzywrqfpwpuldhrvcjuso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbxmjxhxwtsaczqugfftlfrxppynjqnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gbeehjgoacegtgqmrhbsfriwtuichhjc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uttbxbwcnodwrcxrzyiisfwoyjgkusoc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjgdvucpctysovmwpklrcbapvdvuldjp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sljmmojgpdbpziqmuylvzlfnrrfpvcll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcgwgreohtnkykixpismnctdhpkcqrxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wubqigryudrjiulvecphouyerhrpgasv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwslfzxlbrbokxcknoecsziqyuzknoqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"odvizirpugvfmlcuflvlnuyzxdrsunax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"woifirbhmfaksnlgbwdbfcxkfqrntdzp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bgnanaznanfacfzoxmrfcpwanusalcqe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqvicdlazmynrycgineevbmtomqtacfa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dnvoqfoolhvvfobaztcvzizlrbcroksw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjpldujeguxxbtvhsxkocdsfshiaagda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oxsqxksfbcsbwpgqmuyndkkaylavklmn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnloafibjyqazhfgvjcqciqgexkevmkl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"glkjzuggafnavcyzhpnkcguwpgjahbmj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbwciaapjyejlnqaexiqmywfimtqrsrf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyqnbxvxvjuowcfellkyunkyvtvqkqmb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"audqofwcdyxlxsrrzfmkngwxlwhfekvv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibasukhfiyttmezailfigekaodhnfzeg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bffckrywrsgnqfaqutleahvbpixnjlbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzqusxnnodiiizypudkoeeivwgximpdl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibwxmwlruwftoxhsjwtuwkkwulnfichq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vddvyxpzitgjopvimndwgvcwbfparkjw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eunrhwojkdpelzdgrnivpiohcdmluqpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohvreawyktgaqpmcgaxhutcuxmjdrbhf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfhrxrsflghizlwgtwlswrnzgcdhrhtv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eoaaxtrctanvvmyfnbgobkebjzdeabkh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhfckujwlscxpcsingpkoxymagntvtij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amvkarhemfihchrkjhwxtoeljnkvjnkr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hgjwjgjlgiylgcvwubiajzdbvwbcryld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lyegxpxxzfvlliqqfvgqxtbewkwscwdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxtlpsnavxkwnmvdayeimqwzjolorrjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gapcppfoxviliruvesufpeaojunnouci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmlhaenumdtwsivgsvgmywtbprkhzffp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqiwcfvtnsjgouhjxosnhibaomdjlzpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhwmacdxsmhadzxrdkkyljqylmgxrusz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phbuzboslmsljoyfzzkcxhyzpixrwxss","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tthowighkuafsuccbfhpzzyzqcutjscg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ullzukrzgqvluzxoywyqarhgwibbuzdj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnxnglknsrppmolxwgwfnyjqkpkvnspg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcmppxyaaltmtsafhwizekarlljqtexu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkwuftfrookssxqesjtzkkeruaseidvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lnyafipfrhrkybczcqbgkfsrobrukqht","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bpmackcsvsocmntbllthqkmqpvuovgbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmxarpdaqtlwtqbvdrtoojzdxogazlem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"stjqfyrihudtpjwzfwoomhnqhujrjubh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zcegsnlnxtcizuxxyohssxavndnzzgkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hvzsrqkpwjatikyiffezgzqsspfgdxfw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tgluyhhhlphlzyhmtkmtgpdtxnsuktne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uymmheeevemfdztvwvdmcpwvbprmijka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvokwqjcoqvkghkcnboqdgmosfvlkzya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckacdpvkjdmczwlwxaizityxqkuiwbza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sapaypvwowzigbhkqncbyuwsbpaxnjsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gewvcrvgzhemwopyesooddtwtvjonabd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"noepxkhjmdkywpejdjiwocicyobvzmmt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucsetvnkwtmybazkykvaektjbhdcnfsr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iltnsbfvmurlbvplswilgbmkzbqvhxay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"esntyrfiarcxovylvzztraklhumxjxgn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gvrxrebgapejhjchhunnzdlvfwkejvmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evfubzqusfvwnawjhapsztvakkxfatwh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ncebgayhaxnyouyihmipijytdtotvras","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnzwutzmfalldkseujxjtdpygqeqycce","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dzqqsfgtknyarlnoiisdosgmcbldvpng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yyvzodgyqwtbxuvwistwtrsuksjzeccj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxxoiveblkrelrwezqxotsisaziemsqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opwqbpspiwdjgsamndfqhqpzatarwuty","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpjekraoazsdsxeotkfuvnhmvofvufaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qnavykbwejpjwfwosxbavqqhpvvwxjsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smucwtmbxfxfbsblxcffurmbmpbtgbqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynjviteceykmyfobjschpeorwnqlemxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zcsqddxrrlycvbzqmbhpwqltuogbkzfl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpqcihadlkddoqmebflloactcxzibaym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hvavdlpzcemgieibmnpvsaxfowowredy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbzsmfrkbbqgnodlgytlknuepfgmtxfm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667238,"databaseName":"models_schema","ddl":"CREATE TABLE `daxdopqapywjintkxohdnessozxvwdwx` (\n `wbxbnnutzbwojhzrswrvwsirslfmgbpr` int NOT NULL,\n `duguicjxaiolwccvkqjddwfgbobgkssz` int DEFAULT NULL,\n `qxmwhtkokddtyvjtrxzumdkltoqmxfhk` int DEFAULT NULL,\n `iexxcvhmjrvotxpdmnmjlobqeurendkj` int DEFAULT NULL,\n `dsfwvvdtkfyudenprxytrdmqoevxuaux` int DEFAULT NULL,\n `mjujewpyacvdijwhfztdresgesfpcrdv` int DEFAULT NULL,\n `rckswrlfhnpemznvzcsixrryjvtzjohn` int DEFAULT NULL,\n `liamtsfpmyxttbasrxvecquhrgphfapb` int DEFAULT NULL,\n `gtmqhiizuhjdqleuepmrmummlukzdjlv` int DEFAULT NULL,\n `oahaojnfmrjwhlltyaekmaewbvyxledc` int DEFAULT NULL,\n `yijmxeambyzluxarnysumzngnjhdxncc` int DEFAULT NULL,\n `cmnsceqsqrxdlbmulyljwffvbspsnxmj` int DEFAULT NULL,\n `ogcgeotyczgvjqtqgmqjsesdtduigxtq` int DEFAULT NULL,\n `tojkjzzcmkmxkrfcngmbxmxpxunhmlxl` int DEFAULT NULL,\n `rkabpkklqfswzslldwqwyyndgufvfmhr` int DEFAULT NULL,\n `zskyubolrpvnemyruzaqxpvdcarhjcbf` int DEFAULT NULL,\n `vtbdelgtcxynpoawgxppwqomfhdaonbq` int DEFAULT NULL,\n `drbphzcwvodjjccxdfmsyzhtpblqctzd` int DEFAULT NULL,\n `flkzdevsafvrsxgabfunfjeeobwdnzve` int DEFAULT NULL,\n `ykywzycqzorvvhmmwpyqdvdabuwxjqjp` int DEFAULT NULL,\n `shymcecauzpshreclaifdljgrxbgqqfw` int DEFAULT NULL,\n `aggyoibxbhvbzucnjiuoawxbiongehok` int DEFAULT NULL,\n `dbaxfyydzbryipqyeevrjbmgmtrwhffv` int DEFAULT NULL,\n `bnhswsownloikjgvbuwqvypovbwlaqks` int DEFAULT NULL,\n `murjilclvnheyzguepcvqmugaziitiab` int DEFAULT NULL,\n `sikdnenplmdrubpcigcwtkpovyphgvlw` int DEFAULT NULL,\n `qmwffnepgiymyfgzvamgqoovbzwpqfnm` int DEFAULT NULL,\n `dbecamcpxxgsovvxtwfueptukgkxfemc` int DEFAULT NULL,\n `cdpztoaqwukuvhbzkcdpgpsmovncprct` int DEFAULT NULL,\n `cpexjnlcmpjnhmbxvgggsywcrequqnlo` int DEFAULT NULL,\n `kweizkvoxgzjvljuxmgrqiafqysgfuzb` int DEFAULT NULL,\n `htzyjehypahoysiycqevejcfidfrupkx` int DEFAULT NULL,\n `aconotyeugqiyfrbbbnffjrkveawrvhn` int DEFAULT NULL,\n `scxuyknbzdhxffydulzcqvtqqpkyzokh` int DEFAULT NULL,\n `saohonjtflqlvhyioeekqdyztcmiesqj` int DEFAULT NULL,\n `xuekxdxosbkwzvvztqetbgjcqnflpqzm` int DEFAULT NULL,\n `rxcdmjzhnrmcrofemxjjonxefdxepgby` int DEFAULT NULL,\n `bdlaswujteyxfpebefuphdtggqvqojmp` int DEFAULT NULL,\n `zcqzchborzenrcmyoxsazpsfyydsdulk` int DEFAULT NULL,\n `dchwuxmfalzxmhexpszmlotifsgshemh` int DEFAULT NULL,\n `vtkcunazhjlajipqlirmlhiqwdjfaxam` int DEFAULT NULL,\n `opwsqocdfnayyttchovpmepxpagboenz` int DEFAULT NULL,\n `gtgxlkyshmmveydkjvqywwdkgrmlzayn` int DEFAULT NULL,\n `tjzurqmsbrorwujlbxotpmoabdrwmvhn` int DEFAULT NULL,\n `jyvoqkookxihgxxcvuefpqtpzskfnzan` int DEFAULT NULL,\n `vpguvevphhmmyriwcvxthmjnuenfxpnt` int DEFAULT NULL,\n `noijosesvikjvbnhlrzqmcbtiuzcsoph` int DEFAULT NULL,\n `urldylkzszbtpvyedpxyfbuflcgnkfuk` int DEFAULT NULL,\n `tjapjmgxyliwilottvtjodzbtjbpyhpe` int DEFAULT NULL,\n `bbcfgbxrwesqffweavndovnetuwigoqb` int DEFAULT NULL,\n `ivyqddxcxhidoljyuvvjjmkpmbzlujny` int DEFAULT NULL,\n `cfglmydruflmrnwsgexbldmesfbjuafi` int DEFAULT NULL,\n `ioxvdwlvcrnjocbpkunchhssdvpmogff` int DEFAULT NULL,\n `vsdihykvvsfgcwgvchzyrsintqpbwuft` int DEFAULT NULL,\n `fpsydsznapwalazmmdtlgiauslrfhumc` int DEFAULT NULL,\n `zujrecwdibcbvajpoqfnduehqfhtybpz` int DEFAULT NULL,\n `yftltogzweheuivvxuwdxzljvfgyjmxs` int DEFAULT NULL,\n `glgllexllsmlqiumkahnhuxuncfrxyix` int DEFAULT NULL,\n `gygnranrgrklmpommcncamqteaferuvg` int DEFAULT NULL,\n `yvdlrlxfuwdoxvuiqhkpirjjawxdsjrw` int DEFAULT NULL,\n `jaeopcqopjgwjszmfunzyhzksaaozyji` int DEFAULT NULL,\n `ackmqfqttrikhtlmlielmwzpfzhdbsmm` int DEFAULT NULL,\n `dhdvsowpjnufaifakeluaaqsifkbbsms` int DEFAULT NULL,\n `kfvrzkvwoykmlijrgnvqcmuhkmtizgvl` int DEFAULT NULL,\n `beenoaydpwyjibqokycfcladbficxbzz` int DEFAULT NULL,\n `dyxljdjglchpbhzdrcgqryqcczrgjgnm` int DEFAULT NULL,\n `oevkccmgtfzidqcbxolkercdkfusuysg` int DEFAULT NULL,\n `gxcitzhsfdoufzgmqixvoyvrjxkgbxsn` int DEFAULT NULL,\n `wolcbkqiyapezvsanbpjqsmygdroedga` int DEFAULT NULL,\n `elgmjcylzebwztbofgdhxumxodogvdiw` int DEFAULT NULL,\n `ykgqwsvgajfgnlpvstvycocjzpeitnow` int DEFAULT NULL,\n `nshypdsyzqeznogpcnaijxrsieenwpjz` int DEFAULT NULL,\n `akclrywimburrmqcnwzhxosvbwvcqehy` int DEFAULT NULL,\n `ebdxwrlqzuwksctyjmsmdfrlqrvplmhm` int DEFAULT NULL,\n `hegikndfrjrodcrheuzlswmqyzrgejed` int DEFAULT NULL,\n `ddsimolfnubjdhiuqzlnfyofgkaqfocb` int DEFAULT NULL,\n `ddbkhnymiofipxcidztiyanacbspsvjs` int DEFAULT NULL,\n `hzayctbwszrsrxyjacvdiwbiwsvqkqfu` int DEFAULT NULL,\n `usefjnykrlmjcdzfdplueojczdinyfif` int DEFAULT NULL,\n `xwpcfkjozrzmfgyxdhbezofrwhozowqm` int DEFAULT NULL,\n `lrtfvcicufqfngymwvnzhqkyuybtwfcu` int DEFAULT NULL,\n `dzupxugqeghjjoajzvnwbwhgvlsmdmrj` int DEFAULT NULL,\n `ocvmwhdikyjydhcspbhpzflgwnygblcn` int DEFAULT NULL,\n `baizhqfynmmmriunbmvbrmdpkkpfohkw` int DEFAULT NULL,\n `zsjquhanjrulkwojoodeaxvfdkvxukdg` int DEFAULT NULL,\n `zyeqdhwjjuleyjxhjipbzfzexibuxhza` int DEFAULT NULL,\n `mgduutsuwzgvdhnxnhejmvtarvseibin` int DEFAULT NULL,\n `pvpipiikwiuswxwcfdoowlweqebxwhti` int DEFAULT NULL,\n `xedfalibwxjlgpfawfxojwbzronzrifc` int DEFAULT NULL,\n `rnrdaybdgjdybdyohoddumqrzwuajpew` int DEFAULT NULL,\n `johpgytizhcmjnuibrkmptgcgupqmvwn` int DEFAULT NULL,\n `tfabunvvpqlwqspunvdqxwfktvomgcbe` int DEFAULT NULL,\n `hhqzuugcmqtuliukhmmueajkjyldxdsj` int DEFAULT NULL,\n `abwvghrjeinghdoowlfxuhnfevxeapag` int DEFAULT NULL,\n `sddswfgcpdxxytjmfydalqydrkorzwjx` int DEFAULT NULL,\n `mpecmkzteffxjkyajmlrnaxwiysmsgom` int DEFAULT NULL,\n `dreazecmwdxvgaflpxkadtmqsmaxtcdh` int DEFAULT NULL,\n `mddjxvnnxnxfckuiznvvrqnjbuphojef` int DEFAULT NULL,\n `peisowpjaobipxegoqdwhfgkcbukejdk` int DEFAULT NULL,\n `wmxvbzijlusfpdjhfwxmgvzfgqzpdozz` int DEFAULT NULL,\n PRIMARY KEY (`wbxbnnutzbwojhzrswrvwsirslfmgbpr`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"daxdopqapywjintkxohdnessozxvwdwx\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["wbxbnnutzbwojhzrswrvwsirslfmgbpr"],"columns":[{"name":"wbxbnnutzbwojhzrswrvwsirslfmgbpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"duguicjxaiolwccvkqjddwfgbobgkssz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxmwhtkokddtyvjtrxzumdkltoqmxfhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iexxcvhmjrvotxpdmnmjlobqeurendkj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsfwvvdtkfyudenprxytrdmqoevxuaux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjujewpyacvdijwhfztdresgesfpcrdv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rckswrlfhnpemznvzcsixrryjvtzjohn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"liamtsfpmyxttbasrxvecquhrgphfapb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtmqhiizuhjdqleuepmrmummlukzdjlv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oahaojnfmrjwhlltyaekmaewbvyxledc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yijmxeambyzluxarnysumzngnjhdxncc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmnsceqsqrxdlbmulyljwffvbspsnxmj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ogcgeotyczgvjqtqgmqjsesdtduigxtq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tojkjzzcmkmxkrfcngmbxmxpxunhmlxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkabpkklqfswzslldwqwyyndgufvfmhr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zskyubolrpvnemyruzaqxpvdcarhjcbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtbdelgtcxynpoawgxppwqomfhdaonbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drbphzcwvodjjccxdfmsyzhtpblqctzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flkzdevsafvrsxgabfunfjeeobwdnzve","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykywzycqzorvvhmmwpyqdvdabuwxjqjp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shymcecauzpshreclaifdljgrxbgqqfw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aggyoibxbhvbzucnjiuoawxbiongehok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbaxfyydzbryipqyeevrjbmgmtrwhffv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnhswsownloikjgvbuwqvypovbwlaqks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"murjilclvnheyzguepcvqmugaziitiab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sikdnenplmdrubpcigcwtkpovyphgvlw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmwffnepgiymyfgzvamgqoovbzwpqfnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbecamcpxxgsovvxtwfueptukgkxfemc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cdpztoaqwukuvhbzkcdpgpsmovncprct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cpexjnlcmpjnhmbxvgggsywcrequqnlo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kweizkvoxgzjvljuxmgrqiafqysgfuzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htzyjehypahoysiycqevejcfidfrupkx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aconotyeugqiyfrbbbnffjrkveawrvhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scxuyknbzdhxffydulzcqvtqqpkyzokh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"saohonjtflqlvhyioeekqdyztcmiesqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xuekxdxosbkwzvvztqetbgjcqnflpqzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxcdmjzhnrmcrofemxjjonxefdxepgby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdlaswujteyxfpebefuphdtggqvqojmp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zcqzchborzenrcmyoxsazpsfyydsdulk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dchwuxmfalzxmhexpszmlotifsgshemh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtkcunazhjlajipqlirmlhiqwdjfaxam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opwsqocdfnayyttchovpmepxpagboenz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtgxlkyshmmveydkjvqywwdkgrmlzayn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjzurqmsbrorwujlbxotpmoabdrwmvhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jyvoqkookxihgxxcvuefpqtpzskfnzan","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpguvevphhmmyriwcvxthmjnuenfxpnt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"noijosesvikjvbnhlrzqmcbtiuzcsoph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urldylkzszbtpvyedpxyfbuflcgnkfuk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjapjmgxyliwilottvtjodzbtjbpyhpe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbcfgbxrwesqffweavndovnetuwigoqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivyqddxcxhidoljyuvvjjmkpmbzlujny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfglmydruflmrnwsgexbldmesfbjuafi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ioxvdwlvcrnjocbpkunchhssdvpmogff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsdihykvvsfgcwgvchzyrsintqpbwuft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpsydsznapwalazmmdtlgiauslrfhumc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zujrecwdibcbvajpoqfnduehqfhtybpz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yftltogzweheuivvxuwdxzljvfgyjmxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"glgllexllsmlqiumkahnhuxuncfrxyix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gygnranrgrklmpommcncamqteaferuvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yvdlrlxfuwdoxvuiqhkpirjjawxdsjrw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jaeopcqopjgwjszmfunzyhzksaaozyji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ackmqfqttrikhtlmlielmwzpfzhdbsmm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhdvsowpjnufaifakeluaaqsifkbbsms","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfvrzkvwoykmlijrgnvqcmuhkmtizgvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"beenoaydpwyjibqokycfcladbficxbzz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dyxljdjglchpbhzdrcgqryqcczrgjgnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oevkccmgtfzidqcbxolkercdkfusuysg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxcitzhsfdoufzgmqixvoyvrjxkgbxsn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wolcbkqiyapezvsanbpjqsmygdroedga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elgmjcylzebwztbofgdhxumxodogvdiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykgqwsvgajfgnlpvstvycocjzpeitnow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nshypdsyzqeznogpcnaijxrsieenwpjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akclrywimburrmqcnwzhxosvbwvcqehy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebdxwrlqzuwksctyjmsmdfrlqrvplmhm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hegikndfrjrodcrheuzlswmqyzrgejed","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddsimolfnubjdhiuqzlnfyofgkaqfocb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddbkhnymiofipxcidztiyanacbspsvjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hzayctbwszrsrxyjacvdiwbiwsvqkqfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"usefjnykrlmjcdzfdplueojczdinyfif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xwpcfkjozrzmfgyxdhbezofrwhozowqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lrtfvcicufqfngymwvnzhqkyuybtwfcu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dzupxugqeghjjoajzvnwbwhgvlsmdmrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocvmwhdikyjydhcspbhpzflgwnygblcn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"baizhqfynmmmriunbmvbrmdpkkpfohkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zsjquhanjrulkwojoodeaxvfdkvxukdg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyeqdhwjjuleyjxhjipbzfzexibuxhza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mgduutsuwzgvdhnxnhejmvtarvseibin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvpipiikwiuswxwcfdoowlweqebxwhti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xedfalibwxjlgpfawfxojwbzronzrifc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnrdaybdgjdybdyohoddumqrzwuajpew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"johpgytizhcmjnuibrkmptgcgupqmvwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tfabunvvpqlwqspunvdqxwfktvomgcbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhqzuugcmqtuliukhmmueajkjyldxdsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abwvghrjeinghdoowlfxuhnfevxeapag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sddswfgcpdxxytjmfydalqydrkorzwjx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpecmkzteffxjkyajmlrnaxwiysmsgom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dreazecmwdxvgaflpxkadtmqsmaxtcdh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mddjxvnnxnxfckuiznvvrqnjbuphojef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"peisowpjaobipxegoqdwhfgkcbukejdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmxvbzijlusfpdjhfwxmgvzfgqzpdozz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667279,"databaseName":"models_schema","ddl":"CREATE TABLE `ddbewigwzztguunwgdgxqetfoxdxqaht` (\n `bkraghqrgjspyumddhdaqcgjzthapdwb` int NOT NULL,\n `mjapylmfuzwjxelpvchgahvcecuaekvr` int DEFAULT NULL,\n `vfdcldfalowkvawwongkbmvcullzofll` int DEFAULT NULL,\n `hbbfxlvhsrbalztwggsdaborvxqeqajc` int DEFAULT NULL,\n `xfahxfkljukxeksnfarerkvseqcitwdc` int DEFAULT NULL,\n `yvpoeqwdzfocfqqyflfbbnoerveezkpf` int DEFAULT NULL,\n `rqmqgtbwogicnoaobswmwiknsqyhsiid` int DEFAULT NULL,\n `uqchtflawgdajxtfssswqarsfpmuuhuq` int DEFAULT NULL,\n `nwyrxrvdxglfapgdigvadndbauxaxckg` int DEFAULT NULL,\n `dktwktxpuvcjqwklmkbzlfudwdsizzfk` int DEFAULT NULL,\n `pnxjbebmcluohwkvdgxrduwegbnsjium` int DEFAULT NULL,\n `ghemgoxstkojfribbatlmqdstjtdrkjg` int DEFAULT NULL,\n `aqhalyrukslyisqjrpvixnlbosaoqhuj` int DEFAULT NULL,\n `xdhlzgeyfkbgxbaxgjqhzvwshcqtbnub` int DEFAULT NULL,\n `afvmrsjfhpqipgnsejsjgpnqsklbmxlx` int DEFAULT NULL,\n `llmncygpzjyzznjzdewhvmeywnadxdok` int DEFAULT NULL,\n `xdpncjactbplwsmgxnxhpkufozzmxnef` int DEFAULT NULL,\n `xidwikdjaewpivorbyqqcglnzhorjmlj` int DEFAULT NULL,\n `xvokotrehlnsuccoanjwgnlqnqczzcbt` int DEFAULT NULL,\n `bemgniitogepwnihxoqooazmjfsqzcrh` int DEFAULT NULL,\n `wpyatcaapurvwflivmvvylrfzeqecbyz` int DEFAULT NULL,\n `hrhsvyaqppxoyokbrfhbnkvlqlzmmabm` int DEFAULT NULL,\n `oswdbpwplfzxhikprzaldgttymscyqdu` int DEFAULT NULL,\n `fwfvbgpdgpfrtxcjtqjqkdwwdrtszlaa` int DEFAULT NULL,\n `vwhbofjzejqydvwdlzjelxsnqtzdbeoh` int DEFAULT NULL,\n `bavliutbzkevcvbtoltlloopdyzpipxg` int DEFAULT NULL,\n `ipulwhnjriohdjgjvzflypcjpewxgxcw` int DEFAULT NULL,\n `krvonutmlmhgkaihtokcltrsxxlejuec` int DEFAULT NULL,\n `stfgdfvgwzguypvgkbzueymmhtlksyhn` int DEFAULT NULL,\n `llxejqrpsceksjzbmcjsozyjpaelqnrm` int DEFAULT NULL,\n `bczecoxwbugawwpybtnxxtntorczwkou` int DEFAULT NULL,\n `rvcxmiuloqqnbcimlapvjolrrrtvqjoz` int DEFAULT NULL,\n `ijuruwfbbhoupmohkowgkuqmhpjlcslf` int DEFAULT NULL,\n `ujlqwshkzvsiwhyvlfeklvbrkjoqtfwl` int DEFAULT NULL,\n `xeropvhnnaxoqoiiwdclczgredchendi` int DEFAULT NULL,\n `tciyndzmzjlzeagezpywejtmdpxnquyg` int DEFAULT NULL,\n `eyzynoiyokhfhbstfbudlfyipkiuftex` int DEFAULT NULL,\n `pcbjycngqowszjrkitndsgkclzyvcqva` int DEFAULT NULL,\n `nmkrdxyrxiknycvjlqnmnhiuzmzrxlcq` int DEFAULT NULL,\n `ohhvimftrhnnkcgfvmglytrtlxhvbvel` int DEFAULT NULL,\n `ybrjcbcnrjstxfmybkhmveiejhnwcvwq` int DEFAULT NULL,\n `tbluspgclxtcdlvrrplvuyqplcsrooph` int DEFAULT NULL,\n `yabxrehzytsvegjnmasvpbutkwtlxaxt` int DEFAULT NULL,\n `yfhnmwjfvqjjemfnjzqydfyvjtzcgzcx` int DEFAULT NULL,\n `mxvqvvuhtaoixmcltcueaklfiysekimn` int DEFAULT NULL,\n `cfvyynnvusymceqztmqqnmkckgjvmznt` int DEFAULT NULL,\n `kirvlpfxrwtsygenrbkvsrsmzosbfyea` int DEFAULT NULL,\n `ppuysozbxctjnyjxpmgixjuskyxbivgu` int DEFAULT NULL,\n `eyeptjkstakxuxcaqddzxykwyibnugxl` int DEFAULT NULL,\n `agjcprmyutdberqfzuxjknmgwcumtelg` int DEFAULT NULL,\n `xsebghvxqyvphouezjewdteryghtlnyl` int DEFAULT NULL,\n `pifctovahyefwvdipfzhlacwkfkoczbf` int DEFAULT NULL,\n `hdybmzwlgutxughncdtkzdenhzpiypso` int DEFAULT NULL,\n `dvcfmxpwsefdyenucozpetryirwwpgut` int DEFAULT NULL,\n `padiratkspybpautxdkthtwcxldrntyd` int DEFAULT NULL,\n `oyszuuxqnfrtgzvpmaybdimcsrngzyxd` int DEFAULT NULL,\n `gagampmuyjtjgajyfseecfxwdxxlcime` int DEFAULT NULL,\n `hfcsocrlbqvdeidcibyfjogerylbwpnr` int DEFAULT NULL,\n `smvdeusvtljsrhfzuonkuzthnfqppwkl` int DEFAULT NULL,\n `opdodgrgmbdattmppsmjqvpfbnttihhq` int DEFAULT NULL,\n `yxwjzmfeahfeyvwlknfvsspqezdykili` int DEFAULT NULL,\n `xzixuydbtorhfozipipjejqnwlxulrsk` int DEFAULT NULL,\n `eqizainswxaswextntotykktmbuvrwhz` int DEFAULT NULL,\n `bthubalycpribzrilwltgmiotbkenrbu` int DEFAULT NULL,\n `fobwzcaxiyjltfrpbzympbaivqrzsqxh` int DEFAULT NULL,\n `zqiosdduqwzuvwczfbxpqpbmkdmysiwv` int DEFAULT NULL,\n `pyxixsrbzvupjcwodidsppsvexqrsewp` int DEFAULT NULL,\n `xilmebqkimsjoihxgyqirzuiclzktobe` int DEFAULT NULL,\n `wmqkvtcdrxpkftoymkwsoicjxzgbwlaa` int DEFAULT NULL,\n `xqebbqpwrvookxolryrujlxobycqdmrh` int DEFAULT NULL,\n `bdlioqiyrzqqvihzxpcsjseomebhmpfr` int DEFAULT NULL,\n `uprgyzijllnevynbbzurqajwhcowtbhc` int DEFAULT NULL,\n `jjgvonghjyprzuetsgallivrgefgjzvt` int DEFAULT NULL,\n `djimajnqmcegrgmowcbtfbbtbpwdjqnf` int DEFAULT NULL,\n `ccgcsurlwhsvaxcgowuspbjtokzddflv` int DEFAULT NULL,\n `wnfycwrzcfbdvjwzpukuxninkbcisoyq` int DEFAULT NULL,\n `ieidvvjxmskqkuehmnlbjsnbgrkhhpjd` int DEFAULT NULL,\n `clgziijipautlviratekpwbaytxgfjdy` int DEFAULT NULL,\n `vjfxcwomxfxtiwyknjnocasvmsecvhhw` int DEFAULT NULL,\n `yhshfvjlrbwnmhyzpxyfmppefssidntf` int DEFAULT NULL,\n `urjettqwpkgbghghbaeydcvpiizvltwe` int DEFAULT NULL,\n `vxjbuktobdcxfgmyryrhczdegqfaigin` int DEFAULT NULL,\n `ighrliecllwdxtaelozfutunqwmoniys` int DEFAULT NULL,\n `vhcmfcmvozqefshzpbztukkgozziirzg` int DEFAULT NULL,\n `msesmpbotceiqsptwxehzmrhulkhaapv` int DEFAULT NULL,\n `jcnlbcyzoaoqwnslemceaoqrhvmdfjbo` int DEFAULT NULL,\n `sppgrfpjykulfqrysnidrnyemcntqbzq` int DEFAULT NULL,\n `fbjzodxtsnyfpjmdzvldllowyypzupxo` int DEFAULT NULL,\n `hfaonokmexjgcqukaijtnsflgmnvnngc` int DEFAULT NULL,\n `enzfrczgiznechjnxmjybsadqxkklzlg` int DEFAULT NULL,\n `vluznxsqtnpjbbgzsepyzcrcqghzdfxv` int DEFAULT NULL,\n `hilwmenncvpdmvjoewusjrxzfbwsmbft` int DEFAULT NULL,\n `hlmfhhqzdijfqpakgifyaoykykozwdbe` int DEFAULT NULL,\n `zhascvrajhsjegqpojvzaymtrnoouktk` int DEFAULT NULL,\n `xthukrktvvezcbhzrjqhuhklfovajgop` int DEFAULT NULL,\n `ksxrjpjntekpqyjhjbctccamduxbshsi` int DEFAULT NULL,\n `cdigewnlvwxoraznmgmdsiixgktruoca` int DEFAULT NULL,\n `kretwmznsjeovzsqeyxailfidqeerwef` int DEFAULT NULL,\n `ivxaucuxgwrbbnormdhpvweqbsomdzaj` int DEFAULT NULL,\n `cqptbmkbgdkqaaizyfdgsfdzhjskjcrg` int DEFAULT NULL,\n PRIMARY KEY (`bkraghqrgjspyumddhdaqcgjzthapdwb`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"ddbewigwzztguunwgdgxqetfoxdxqaht\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["bkraghqrgjspyumddhdaqcgjzthapdwb"],"columns":[{"name":"bkraghqrgjspyumddhdaqcgjzthapdwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"mjapylmfuzwjxelpvchgahvcecuaekvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vfdcldfalowkvawwongkbmvcullzofll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbbfxlvhsrbalztwggsdaborvxqeqajc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xfahxfkljukxeksnfarerkvseqcitwdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yvpoeqwdzfocfqqyflfbbnoerveezkpf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqmqgtbwogicnoaobswmwiknsqyhsiid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqchtflawgdajxtfssswqarsfpmuuhuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nwyrxrvdxglfapgdigvadndbauxaxckg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dktwktxpuvcjqwklmkbzlfudwdsizzfk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnxjbebmcluohwkvdgxrduwegbnsjium","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghemgoxstkojfribbatlmqdstjtdrkjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqhalyrukslyisqjrpvixnlbosaoqhuj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdhlzgeyfkbgxbaxgjqhzvwshcqtbnub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afvmrsjfhpqipgnsejsjgpnqsklbmxlx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llmncygpzjyzznjzdewhvmeywnadxdok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdpncjactbplwsmgxnxhpkufozzmxnef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xidwikdjaewpivorbyqqcglnzhorjmlj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvokotrehlnsuccoanjwgnlqnqczzcbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bemgniitogepwnihxoqooazmjfsqzcrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wpyatcaapurvwflivmvvylrfzeqecbyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hrhsvyaqppxoyokbrfhbnkvlqlzmmabm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oswdbpwplfzxhikprzaldgttymscyqdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwfvbgpdgpfrtxcjtqjqkdwwdrtszlaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vwhbofjzejqydvwdlzjelxsnqtzdbeoh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bavliutbzkevcvbtoltlloopdyzpipxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ipulwhnjriohdjgjvzflypcjpewxgxcw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krvonutmlmhgkaihtokcltrsxxlejuec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"stfgdfvgwzguypvgkbzueymmhtlksyhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llxejqrpsceksjzbmcjsozyjpaelqnrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bczecoxwbugawwpybtnxxtntorczwkou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rvcxmiuloqqnbcimlapvjolrrrtvqjoz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijuruwfbbhoupmohkowgkuqmhpjlcslf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujlqwshkzvsiwhyvlfeklvbrkjoqtfwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xeropvhnnaxoqoiiwdclczgredchendi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tciyndzmzjlzeagezpywejtmdpxnquyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eyzynoiyokhfhbstfbudlfyipkiuftex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pcbjycngqowszjrkitndsgkclzyvcqva","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmkrdxyrxiknycvjlqnmnhiuzmzrxlcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohhvimftrhnnkcgfvmglytrtlxhvbvel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybrjcbcnrjstxfmybkhmveiejhnwcvwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbluspgclxtcdlvrrplvuyqplcsrooph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yabxrehzytsvegjnmasvpbutkwtlxaxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfhnmwjfvqjjemfnjzqydfyvjtzcgzcx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mxvqvvuhtaoixmcltcueaklfiysekimn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfvyynnvusymceqztmqqnmkckgjvmznt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kirvlpfxrwtsygenrbkvsrsmzosbfyea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ppuysozbxctjnyjxpmgixjuskyxbivgu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eyeptjkstakxuxcaqddzxykwyibnugxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agjcprmyutdberqfzuxjknmgwcumtelg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsebghvxqyvphouezjewdteryghtlnyl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pifctovahyefwvdipfzhlacwkfkoczbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdybmzwlgutxughncdtkzdenhzpiypso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvcfmxpwsefdyenucozpetryirwwpgut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"padiratkspybpautxdkthtwcxldrntyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oyszuuxqnfrtgzvpmaybdimcsrngzyxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gagampmuyjtjgajyfseecfxwdxxlcime","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfcsocrlbqvdeidcibyfjogerylbwpnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smvdeusvtljsrhfzuonkuzthnfqppwkl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opdodgrgmbdattmppsmjqvpfbnttihhq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxwjzmfeahfeyvwlknfvsspqezdykili","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xzixuydbtorhfozipipjejqnwlxulrsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqizainswxaswextntotykktmbuvrwhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bthubalycpribzrilwltgmiotbkenrbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fobwzcaxiyjltfrpbzympbaivqrzsqxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqiosdduqwzuvwczfbxpqpbmkdmysiwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pyxixsrbzvupjcwodidsppsvexqrsewp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xilmebqkimsjoihxgyqirzuiclzktobe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmqkvtcdrxpkftoymkwsoicjxzgbwlaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqebbqpwrvookxolryrujlxobycqdmrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdlioqiyrzqqvihzxpcsjseomebhmpfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uprgyzijllnevynbbzurqajwhcowtbhc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjgvonghjyprzuetsgallivrgefgjzvt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"djimajnqmcegrgmowcbtfbbtbpwdjqnf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ccgcsurlwhsvaxcgowuspbjtokzddflv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnfycwrzcfbdvjwzpukuxninkbcisoyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ieidvvjxmskqkuehmnlbjsnbgrkhhpjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clgziijipautlviratekpwbaytxgfjdy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjfxcwomxfxtiwyknjnocasvmsecvhhw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhshfvjlrbwnmhyzpxyfmppefssidntf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urjettqwpkgbghghbaeydcvpiizvltwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxjbuktobdcxfgmyryrhczdegqfaigin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ighrliecllwdxtaelozfutunqwmoniys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhcmfcmvozqefshzpbztukkgozziirzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"msesmpbotceiqsptwxehzmrhulkhaapv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jcnlbcyzoaoqwnslemceaoqrhvmdfjbo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sppgrfpjykulfqrysnidrnyemcntqbzq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbjzodxtsnyfpjmdzvldllowyypzupxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfaonokmexjgcqukaijtnsflgmnvnngc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"enzfrczgiznechjnxmjybsadqxkklzlg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vluznxsqtnpjbbgzsepyzcrcqghzdfxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hilwmenncvpdmvjoewusjrxzfbwsmbft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlmfhhqzdijfqpakgifyaoykykozwdbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhascvrajhsjegqpojvzaymtrnoouktk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xthukrktvvezcbhzrjqhuhklfovajgop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ksxrjpjntekpqyjhjbctccamduxbshsi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cdigewnlvwxoraznmgmdsiixgktruoca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kretwmznsjeovzsqeyxailfidqeerwef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivxaucuxgwrbbnormdhpvweqbsomdzaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqptbmkbgdkqaaizyfdgsfdzhjskjcrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667311,"databaseName":"models_schema","ddl":"CREATE TABLE `dfoaznlacthdivxcqnznziszzihljtyi` (\n `evnmhutgnenitueultneqxoinhlvhpsw` int NOT NULL,\n `srdixelquatkskdcihwpmmcefcjdxzqq` int DEFAULT NULL,\n `hcjyadlllynxsdelzjhmnbrgrtvhxtmj` int DEFAULT NULL,\n `xozzpnmqhbscafkkdfmzbdbdjuxqlbvc` int DEFAULT NULL,\n `djgcokkvzutfvlgdugszbafnbncgkwew` int DEFAULT NULL,\n `xavgjvpbekmhsoobvnbycoobgulnuxac` int DEFAULT NULL,\n `dhcgyttygwqsetkbrhqsknltbnrbyopj` int DEFAULT NULL,\n `brzmwuotbluqdcmxfvrmuhlsdpdstgag` int DEFAULT NULL,\n `fiurukbdfrgkvkqmyzbbovlfdqhdvyin` int DEFAULT NULL,\n `hgerspgmucbvgqdvljoyntqdvaybwqab` int DEFAULT NULL,\n `sowsarszgfjpzocwhjceuofpevxqvcvf` int DEFAULT NULL,\n `xoiifbivisvscchyylwirjgzshqzzehy` int DEFAULT NULL,\n `msgfmdukfjjsxqepdbsqdftqwhwcoxls` int DEFAULT NULL,\n `luokynrfldknhhunzyonjarexehgcuzs` int DEFAULT NULL,\n `yxjufvrwmfezspqtubtmmzfezqlzaazf` int DEFAULT NULL,\n `qhvguwvevhlyagkiljsxvkihzcljdbbi` int DEFAULT NULL,\n `ommnogctliyyujvsrlgziahftddwuydk` int DEFAULT NULL,\n `aqvrmnkucygtgilyjrtodcsdyskckghk` int DEFAULT NULL,\n `drtgnwwccrctvbnnulkyssitdtlgpttm` int DEFAULT NULL,\n `nzvmnyicbofpoeqqiygfuloqmyoslrce` int DEFAULT NULL,\n `niydipdhlrlwuiuuwklkoqzeznwxhryf` int DEFAULT NULL,\n `kmsholxrczhhwuqijlamkdagapbkaohk` int DEFAULT NULL,\n `vflxarkqhyslsmgaytqxnxcppbpksxgy` int DEFAULT NULL,\n `wfeedoohnkubrwadrnnkspzemkdihmvk` int DEFAULT NULL,\n `bkrmtmxnwcqpyelwuhigphrvlgxghofx` int DEFAULT NULL,\n `rgxtlziujttrtmubxcvnhzqadjybrytb` int DEFAULT NULL,\n `kofsbwqzmvflrahlikvgrzsmeyhhnwye` int DEFAULT NULL,\n `gnyuodtbwsjqkpsmukapvfkravwqyedp` int DEFAULT NULL,\n `neozaaihlykekwpzprxofwndgrxaupic` int DEFAULT NULL,\n `mbkxwiynlvzyzehltjktzunpgggqiciv` int DEFAULT NULL,\n `qjlbyptfanegomawnpkhzjsrxkjttple` int DEFAULT NULL,\n `birudzyjnojphzedqyuqonwmugpcwuvp` int DEFAULT NULL,\n `kinxarzohxjdshjsdsvuwccxcjuwugrk` int DEFAULT NULL,\n `dssdvxtbghmaboisavoijdsyeqjsrbkw` int DEFAULT NULL,\n `hjautrfbsmumsrovlkzdwgsospasqawa` int DEFAULT NULL,\n `lgypyggzxoetcgjjcyebkjgpwsymitnu` int DEFAULT NULL,\n `ztihawwcmcvlqsrthdvzxijodoicapvn` int DEFAULT NULL,\n `qwestcmgpboiedqlmuopuuudfjizxwvp` int DEFAULT NULL,\n `rhoszcyztdoigofzzqcswnpvbqfutzpk` int DEFAULT NULL,\n `pdtinknczuvavrfhfnkqjfvvmeiubwfb` int DEFAULT NULL,\n `rfnywmhqcrocpevvgnjngxsbnhcxmfuo` int DEFAULT NULL,\n `avargstzaabpyxbmnmbsjimnvcuohjxa` int DEFAULT NULL,\n `eerdtiavlprvejliuuzdzwcfymchyhza` int DEFAULT NULL,\n `khgttydzpidmvjwgwcodugppnlagisbf` int DEFAULT NULL,\n `unfvqlwhexiygmryidohdjbgvyfzvdqp` int DEFAULT NULL,\n `uxvfregvoqqjgnyftswdxqxixrripwwt` int DEFAULT NULL,\n `zrztolkfhlyxybeirhtivyjjcxlghjcj` int DEFAULT NULL,\n `kcdxznqsivdmawkgnpaurwidiwfcbkrj` int DEFAULT NULL,\n `duvkydsyvwjqoqbqbdasucyutrfgxady` int DEFAULT NULL,\n `rslrocfpkkznezkzxmyixtzvxbszarse` int DEFAULT NULL,\n `gnomdoshwwxaiebganzuaevbadgrjqkn` int DEFAULT NULL,\n `ucqtlvazmizsxcdyosnqujzfcpaqxdfc` int DEFAULT NULL,\n `yemgimtchdfkhtfksqzxjixbingithme` int DEFAULT NULL,\n `qzojkpjxkrfdhbuvlwjisszlwxfgzmmw` int DEFAULT NULL,\n `agytdmonqliyjivyxxohcclgcmkidrnv` int DEFAULT NULL,\n `lvdbzqslketzzjbnuxdetpgpbndjcjnx` int DEFAULT NULL,\n `zpuhqqnwmwbcgktlwhsbamdjibchdvev` int DEFAULT NULL,\n `uypcahjhqntmvleibomjceoxjirmftit` int DEFAULT NULL,\n `fsxaxajvqowxluqkjflzskdwmmympuwf` int DEFAULT NULL,\n `cqkitmrgtjnwnssgtuujegmzlrdtyslc` int DEFAULT NULL,\n `drbzazzkerxemnhpxpelpeeqpmefylxd` int DEFAULT NULL,\n `nrfunuukjzdearaatpdzfocjhuhsexvh` int DEFAULT NULL,\n `msmcvwbhlrnvvefikzrtneawinfpeujc` int DEFAULT NULL,\n `dgapzkqopqeqecdukjyrqbmxiideavlk` int DEFAULT NULL,\n `cgahplyyfkjofvfktghgfivkldycmoqq` int DEFAULT NULL,\n `qfueviwtwhjdunpumywgcloefwnckxnc` int DEFAULT NULL,\n `epbrhfjhjvfnkqxvttmkbizijhlwoyxn` int DEFAULT NULL,\n `zxfdznbciuutuxtwcjnamatllrpzmuzv` int DEFAULT NULL,\n `uowmbsqirjedmxufhwiaorlqqlnxeblg` int DEFAULT NULL,\n `hfuyiarpytrtleqapecogfkcwxlikaqc` int DEFAULT NULL,\n `jyqihwqcyesrxskpbxzankwcjhhrbyjd` int DEFAULT NULL,\n `pwpvzbbktyjtvyxddxldipectkkczhws` int DEFAULT NULL,\n `ujcdjfajdfjlepzwpltgvutkezffektr` int DEFAULT NULL,\n `chbbcpahnswkzfsygfuqsoumhjwfccnn` int DEFAULT NULL,\n `xhdjyogkwfshpfjxvzrqprjbrwydtndz` int DEFAULT NULL,\n `avqqhyvybkeqffwdhyyppnellzgpxhwt` int DEFAULT NULL,\n `ohihyhnmmkmxbxwowziwnjmsibborpwd` int DEFAULT NULL,\n `xwtqgagmarxljtwkmjiedtnqqbwueafs` int DEFAULT NULL,\n `lcxejkadjdwxwvjvigcuyktxqovpemic` int DEFAULT NULL,\n `ezfiqvlshixttubssfyknimmsvapkyil` int DEFAULT NULL,\n `txcfptklvyewmwachqzptuhmstftgsct` int DEFAULT NULL,\n `orjbdtgdiunxnptzvvwjtasufzhpqknr` int DEFAULT NULL,\n `jifnsxybjgyeplqyuzbqdpfmudiwmwwe` int DEFAULT NULL,\n `vpfopqirxmwuwsgfsnbkbnajglnjsxao` int DEFAULT NULL,\n `ctnwyqbumhessagujfxryvilaspgdgny` int DEFAULT NULL,\n `squzmawjwljawanahrvjfkymiwtxftpv` int DEFAULT NULL,\n `toawsvqddrcdvphzzzasreihtknnmxmu` int DEFAULT NULL,\n `ntwwtuwrstkwrerxdmnbsjlcagbrduuc` int DEFAULT NULL,\n `giilcmoyilldsjsjpowynfuswnsgdlab` int DEFAULT NULL,\n `jjlswzgfacilnqxfdulteikiqxbzzplg` int DEFAULT NULL,\n `gyadqhncstchvhxisxntcitwzkbcbsyy` int DEFAULT NULL,\n `difrewxfkkpcwthguacrqlwkbtfsvfgp` int DEFAULT NULL,\n `ubziwvzvyoyphchbubwlfwvyhijvudgl` int DEFAULT NULL,\n `hrnaqqqkuzwopuattmwsrqeoffgdwutg` int DEFAULT NULL,\n `dqmantexlrbbjcmfhotloyccdtmohwqe` int DEFAULT NULL,\n `btrfqdsyffnwikdrzclkiyievtojuirp` int DEFAULT NULL,\n `qhrlkkeaziwhillidahdbrckpmmqiogo` int DEFAULT NULL,\n `milxptzulnvstayupqmlhwmatvpuibnf` int DEFAULT NULL,\n `gjvwciahuxtnkliuraycodphxaoqtgho` int DEFAULT NULL,\n `sxzshwhliefqsengydawcxfmeolorjyp` int DEFAULT NULL,\n PRIMARY KEY (`evnmhutgnenitueultneqxoinhlvhpsw`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"dfoaznlacthdivxcqnznziszzihljtyi\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["evnmhutgnenitueultneqxoinhlvhpsw"],"columns":[{"name":"evnmhutgnenitueultneqxoinhlvhpsw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"srdixelquatkskdcihwpmmcefcjdxzqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hcjyadlllynxsdelzjhmnbrgrtvhxtmj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xozzpnmqhbscafkkdfmzbdbdjuxqlbvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"djgcokkvzutfvlgdugszbafnbncgkwew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xavgjvpbekmhsoobvnbycoobgulnuxac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhcgyttygwqsetkbrhqsknltbnrbyopj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"brzmwuotbluqdcmxfvrmuhlsdpdstgag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fiurukbdfrgkvkqmyzbbovlfdqhdvyin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hgerspgmucbvgqdvljoyntqdvaybwqab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sowsarszgfjpzocwhjceuofpevxqvcvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xoiifbivisvscchyylwirjgzshqzzehy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"msgfmdukfjjsxqepdbsqdftqwhwcoxls","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"luokynrfldknhhunzyonjarexehgcuzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxjufvrwmfezspqtubtmmzfezqlzaazf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhvguwvevhlyagkiljsxvkihzcljdbbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ommnogctliyyujvsrlgziahftddwuydk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqvrmnkucygtgilyjrtodcsdyskckghk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drtgnwwccrctvbnnulkyssitdtlgpttm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzvmnyicbofpoeqqiygfuloqmyoslrce","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"niydipdhlrlwuiuuwklkoqzeznwxhryf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmsholxrczhhwuqijlamkdagapbkaohk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vflxarkqhyslsmgaytqxnxcppbpksxgy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfeedoohnkubrwadrnnkspzemkdihmvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkrmtmxnwcqpyelwuhigphrvlgxghofx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgxtlziujttrtmubxcvnhzqadjybrytb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kofsbwqzmvflrahlikvgrzsmeyhhnwye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnyuodtbwsjqkpsmukapvfkravwqyedp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"neozaaihlykekwpzprxofwndgrxaupic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbkxwiynlvzyzehltjktzunpgggqiciv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjlbyptfanegomawnpkhzjsrxkjttple","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"birudzyjnojphzedqyuqonwmugpcwuvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kinxarzohxjdshjsdsvuwccxcjuwugrk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dssdvxtbghmaboisavoijdsyeqjsrbkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjautrfbsmumsrovlkzdwgsospasqawa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgypyggzxoetcgjjcyebkjgpwsymitnu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ztihawwcmcvlqsrthdvzxijodoicapvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwestcmgpboiedqlmuopuuudfjizxwvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhoszcyztdoigofzzqcswnpvbqfutzpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdtinknczuvavrfhfnkqjfvvmeiubwfb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfnywmhqcrocpevvgnjngxsbnhcxmfuo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avargstzaabpyxbmnmbsjimnvcuohjxa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eerdtiavlprvejliuuzdzwcfymchyhza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khgttydzpidmvjwgwcodugppnlagisbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unfvqlwhexiygmryidohdjbgvyfzvdqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxvfregvoqqjgnyftswdxqxixrripwwt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zrztolkfhlyxybeirhtivyjjcxlghjcj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcdxznqsivdmawkgnpaurwidiwfcbkrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"duvkydsyvwjqoqbqbdasucyutrfgxady","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rslrocfpkkznezkzxmyixtzvxbszarse","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnomdoshwwxaiebganzuaevbadgrjqkn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucqtlvazmizsxcdyosnqujzfcpaqxdfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yemgimtchdfkhtfksqzxjixbingithme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzojkpjxkrfdhbuvlwjisszlwxfgzmmw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agytdmonqliyjivyxxohcclgcmkidrnv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvdbzqslketzzjbnuxdetpgpbndjcjnx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpuhqqnwmwbcgktlwhsbamdjibchdvev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uypcahjhqntmvleibomjceoxjirmftit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fsxaxajvqowxluqkjflzskdwmmympuwf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqkitmrgtjnwnssgtuujegmzlrdtyslc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drbzazzkerxemnhpxpelpeeqpmefylxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nrfunuukjzdearaatpdzfocjhuhsexvh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"msmcvwbhlrnvvefikzrtneawinfpeujc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgapzkqopqeqecdukjyrqbmxiideavlk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgahplyyfkjofvfktghgfivkldycmoqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfueviwtwhjdunpumywgcloefwnckxnc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"epbrhfjhjvfnkqxvttmkbizijhlwoyxn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxfdznbciuutuxtwcjnamatllrpzmuzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uowmbsqirjedmxufhwiaorlqqlnxeblg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfuyiarpytrtleqapecogfkcwxlikaqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jyqihwqcyesrxskpbxzankwcjhhrbyjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwpvzbbktyjtvyxddxldipectkkczhws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujcdjfajdfjlepzwpltgvutkezffektr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chbbcpahnswkzfsygfuqsoumhjwfccnn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xhdjyogkwfshpfjxvzrqprjbrwydtndz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avqqhyvybkeqffwdhyyppnellzgpxhwt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohihyhnmmkmxbxwowziwnjmsibborpwd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xwtqgagmarxljtwkmjiedtnqqbwueafs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lcxejkadjdwxwvjvigcuyktxqovpemic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezfiqvlshixttubssfyknimmsvapkyil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txcfptklvyewmwachqzptuhmstftgsct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orjbdtgdiunxnptzvvwjtasufzhpqknr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jifnsxybjgyeplqyuzbqdpfmudiwmwwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpfopqirxmwuwsgfsnbkbnajglnjsxao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctnwyqbumhessagujfxryvilaspgdgny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"squzmawjwljawanahrvjfkymiwtxftpv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"toawsvqddrcdvphzzzasreihtknnmxmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntwwtuwrstkwrerxdmnbsjlcagbrduuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"giilcmoyilldsjsjpowynfuswnsgdlab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjlswzgfacilnqxfdulteikiqxbzzplg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gyadqhncstchvhxisxntcitwzkbcbsyy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"difrewxfkkpcwthguacrqlwkbtfsvfgp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubziwvzvyoyphchbubwlfwvyhijvudgl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hrnaqqqkuzwopuattmwsrqeoffgdwutg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqmantexlrbbjcmfhotloyccdtmohwqe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"btrfqdsyffnwikdrzclkiyievtojuirp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhrlkkeaziwhillidahdbrckpmmqiogo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"milxptzulnvstayupqmlhwmatvpuibnf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjvwciahuxtnkliuraycodphxaoqtgho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sxzshwhliefqsengydawcxfmeolorjyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667343,"databaseName":"models_schema","ddl":"CREATE TABLE `dfzbmvchizqxwzhseritnruikiymswoq` (\n `azuqxcxvubpvslaqdkmjcfvznnciabmr` int NOT NULL,\n `daxxjcubffwlqiuyvrceezkwaljsdudu` int DEFAULT NULL,\n `xeqhruvtlfyafliogfjrftrtjxcqjzye` int DEFAULT NULL,\n `acsbmzbpcrccrafcvcabtfpfxwgkfhjf` int DEFAULT NULL,\n `wwhcpvbeejzxonbivxazhehrtrhjfbxq` int DEFAULT NULL,\n `cnrcuuenbpoaqltcnggsmopeeberhbqd` int DEFAULT NULL,\n `bqynelpbtocjwugfsmevxxpddjljkvba` int DEFAULT NULL,\n `obushnwgontlxcrnrphsguodlmbbeldd` int DEFAULT NULL,\n `ntfvxxgnzlrgctgkmviqtdhamgesvxke` int DEFAULT NULL,\n `memrhirfqzyalseezuakhojcpfnuzjih` int DEFAULT NULL,\n `qicmlsahyccricalisrjgfmrrlimtspy` int DEFAULT NULL,\n `wzqtipubtawswejrremehfxuhyesryfy` int DEFAULT NULL,\n `uuhspsthsltnamfkrqoqymerhxtvsuzq` int DEFAULT NULL,\n `qpkodytqdgbfqwmoalbqzhmjhukrbvkn` int DEFAULT NULL,\n `hmnawznqeqktkboorsdttpwjjigvzwsk` int DEFAULT NULL,\n `dkwjdhozchcipldhveioeoelyskvbyel` int DEFAULT NULL,\n `qowlwklfplhxoupnzhwdhmrdnehzefwj` int DEFAULT NULL,\n `lwcttxsehnhjhwhxyrcyuqvwnitynkie` int DEFAULT NULL,\n `cpzlnqskkwpyfflwaybsabplvwugaevo` int DEFAULT NULL,\n `pqrpsapxkbhzgoftsizkpgxwrujicffk` int DEFAULT NULL,\n `sawhtpokkegepurgsacepiuraucyzwex` int DEFAULT NULL,\n `twrirpwvctvuqavpiesnmophinbrudhr` int DEFAULT NULL,\n `tymnizzzvqrhkopmkwtlxlczkwehfurx` int DEFAULT NULL,\n `vgnsoabgrnwixpwnonboiukaomyxxpon` int DEFAULT NULL,\n `ymjnqxsaavfcoejisgyfvdfrlwplwecv` int DEFAULT NULL,\n `vfttwfqwdmjzjmlxjxxjogdgtzrqllbk` int DEFAULT NULL,\n `oftwxwdfneincqdfnklpelxumdowyqqd` int DEFAULT NULL,\n `pbqfklhqsyltgwvyuolniayzitbeivxp` int DEFAULT NULL,\n `desheyrgivsiymrfwwmijatosxjsphjo` int DEFAULT NULL,\n `wyhgrmxvploootodgqedsbmmkoivervd` int DEFAULT NULL,\n `qrlmiohsxicrmbpjaaihzgmuotyqvftn` int DEFAULT NULL,\n `dbxloxqnvkhigwnwxbwvxxmfzyeiygdl` int DEFAULT NULL,\n `porynvkiudgizasyzyvcuxxgiwjaatqo` int DEFAULT NULL,\n `uufrteqfcawjcnvpqlksdukeocfkyagy` int DEFAULT NULL,\n `tpeicythvbmlxrfekpegqkqmebqsaxsx` int DEFAULT NULL,\n `dfnntwjgqfmninollcxgxpoucjuocjfm` int DEFAULT NULL,\n `domnywiavgokaqcjnynbchedjggatbec` int DEFAULT NULL,\n `nbozdpnemrgxhpgjlyvrxrgwaxynshlg` int DEFAULT NULL,\n `issveyhkzguxlrlnpuekmdfzhtslzdel` int DEFAULT NULL,\n `xrgmmecpnzqqaxucbmbzwyjroyqglryc` int DEFAULT NULL,\n `dkoqdrdzywjyeflgpxqlaaxoycdbbttw` int DEFAULT NULL,\n `umfhpeficxuxwypciqbckhpwvgkjwqhb` int DEFAULT NULL,\n `udekrntozmhejfvfleaikigdbxctfxvm` int DEFAULT NULL,\n `qrnvpswfrsxswadtozxurdonybssyuvi` int DEFAULT NULL,\n `epwshavnuinnxbcclydcdlkzvxydknhu` int DEFAULT NULL,\n `wajgrwpimvwbvfjcdtpechdzrvzpgqdq` int DEFAULT NULL,\n `jyypvchaxoxmeizdmwsemccdjgvfznkq` int DEFAULT NULL,\n `kwvqxcemspbzfdgmuiuslicrgwpfjnjf` int DEFAULT NULL,\n `jjsctkghzbujfxntpnftzjhvowpzsgfv` int DEFAULT NULL,\n `jsjpxwxzycwkjpvkbvjarwawlufrmfyt` int DEFAULT NULL,\n `jwanykkbsigphkinezovsvxgvjqmciod` int DEFAULT NULL,\n `rqhfsdhyimdfmldfqhuonriznkgkcavc` int DEFAULT NULL,\n `jibcrmhurybzplmclatnkiahnlvjcstx` int DEFAULT NULL,\n `pbpkighkualkmpcgcgzhznaxzawxcdpl` int DEFAULT NULL,\n `hajyvcprivreywkrvovlwfyzqjubphld` int DEFAULT NULL,\n `xsfsfxqfvklztfatfwlanzyjpygsildx` int DEFAULT NULL,\n `ocdfnsqzylsqkabroosaubpqprebnbrr` int DEFAULT NULL,\n `pgtcwpqpoglgmjljalncjmabnthqctiu` int DEFAULT NULL,\n `swsavzbvudikrjttxpightxslxbqgyqo` int DEFAULT NULL,\n `hwixuaaqhqijkdscztpdnzyeroxtwmrv` int DEFAULT NULL,\n `mhlxrkitpucivarxdgjdhziunpgmjmex` int DEFAULT NULL,\n `paqxzfwqypadzwokvksmcjtdpkkotqlz` int DEFAULT NULL,\n `zxuylbbwaoegadzukdeutobeabshxogm` int DEFAULT NULL,\n `gergjmyoqmqbimksxjcxovpkbekvcyjk` int DEFAULT NULL,\n `xrzfuzyemyrbivukxojutrovjckqstux` int DEFAULT NULL,\n `upbvpjlxclbtekroigigvbjfxjrdrihf` int DEFAULT NULL,\n `ecoxqpmleyevnlduzktspodqdljwfspf` int DEFAULT NULL,\n `chajkjdvjimctoisdesuqtxvtpxtalde` int DEFAULT NULL,\n `ndbjwsysssppxleglkuqttpxlbjnzruq` int DEFAULT NULL,\n `mjxnvniwzbpnsxtjvqgaeqwnoxlfssup` int DEFAULT NULL,\n `lbjsaudeyimymhqtrhmjtgshmimhwzor` int DEFAULT NULL,\n `bjzhtzeyghktndnxfjrxmnnucnexgetf` int DEFAULT NULL,\n `gmqbkzefbkbltdhmqkugwugamexuiblt` int DEFAULT NULL,\n `dxmnirvhrxqzkoqfsnexybztvahkapha` int DEFAULT NULL,\n `jwdfvdoknuydollqyepytowcwqimwiwv` int DEFAULT NULL,\n `vxlvixwvqwggmlsxxcsdpnoynzzjcrky` int DEFAULT NULL,\n `sdjzzvyihzsjxwaujnafygnwmisabnmm` int DEFAULT NULL,\n `ebmppaywsyqiijryzigxstttundtxvbh` int DEFAULT NULL,\n `gnyquqjhwwagbretqjqdmdkeoimihyek` int DEFAULT NULL,\n `ohulgwmsuakcpolrodaehrvsycblvctn` int DEFAULT NULL,\n `hjfjdhxtftxyotdobgiijxedlcyarspq` int DEFAULT NULL,\n `tfzihkajfeqsscfmkikrjqnardfpfpgs` int DEFAULT NULL,\n `lmnlhmmsjswnhnmtgezcotbqegykawej` int DEFAULT NULL,\n `ecagoxhbohyunmwyymvhbmlxymmdliqv` int DEFAULT NULL,\n `wcbjuquvlacknanaxngegrkulcaxsvyf` int DEFAULT NULL,\n `woujhsibewchxbhkdeluecvxeylmaftg` int DEFAULT NULL,\n `trpwztiwmedhksbpgonmtztpqzravrjy` int DEFAULT NULL,\n `ostyewkafapfizikiqemlcibarlxmtzx` int DEFAULT NULL,\n `rfwpfvdjbylfldkwbnvdxzkftbqmbrgn` int DEFAULT NULL,\n `cenfssirnbfeofrlkedhhzfwcjipyame` int DEFAULT NULL,\n `pizfuikeyoydepbzcvekjntukcbmruox` int DEFAULT NULL,\n `vtjsajaiggjdtlsdwfzqxviojovarjku` int DEFAULT NULL,\n `wmifzxrsoktdtihjazkycigkhibfsrrg` int DEFAULT NULL,\n `nskmfqajfhdsnsthmjmaczoeqjjwquoa` int DEFAULT NULL,\n `fxhtjigqiawqwyorfztoswklchtppqnx` int DEFAULT NULL,\n `daejfhhugncdrfgjnsbdljochmkrwvdx` int DEFAULT NULL,\n `kpmbtansabdukuolihcrimuaknfayucl` int DEFAULT NULL,\n `tehtwgjiscmrcidibsexflibikzlazny` int DEFAULT NULL,\n `sfhebzlptmiggtohmsuynmjpqatdugyz` int DEFAULT NULL,\n `spweagquinsqzxcozzoazngpdueadjcy` int DEFAULT NULL,\n PRIMARY KEY (`azuqxcxvubpvslaqdkmjcfvznnciabmr`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"dfzbmvchizqxwzhseritnruikiymswoq\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["azuqxcxvubpvslaqdkmjcfvznnciabmr"],"columns":[{"name":"azuqxcxvubpvslaqdkmjcfvznnciabmr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"daxxjcubffwlqiuyvrceezkwaljsdudu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xeqhruvtlfyafliogfjrftrtjxcqjzye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acsbmzbpcrccrafcvcabtfpfxwgkfhjf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwhcpvbeejzxonbivxazhehrtrhjfbxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnrcuuenbpoaqltcnggsmopeeberhbqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqynelpbtocjwugfsmevxxpddjljkvba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obushnwgontlxcrnrphsguodlmbbeldd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntfvxxgnzlrgctgkmviqtdhamgesvxke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"memrhirfqzyalseezuakhojcpfnuzjih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qicmlsahyccricalisrjgfmrrlimtspy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzqtipubtawswejrremehfxuhyesryfy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uuhspsthsltnamfkrqoqymerhxtvsuzq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qpkodytqdgbfqwmoalbqzhmjhukrbvkn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmnawznqeqktkboorsdttpwjjigvzwsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkwjdhozchcipldhveioeoelyskvbyel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qowlwklfplhxoupnzhwdhmrdnehzefwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwcttxsehnhjhwhxyrcyuqvwnitynkie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cpzlnqskkwpyfflwaybsabplvwugaevo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqrpsapxkbhzgoftsizkpgxwrujicffk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sawhtpokkegepurgsacepiuraucyzwex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twrirpwvctvuqavpiesnmophinbrudhr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tymnizzzvqrhkopmkwtlxlczkwehfurx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgnsoabgrnwixpwnonboiukaomyxxpon","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ymjnqxsaavfcoejisgyfvdfrlwplwecv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vfttwfqwdmjzjmlxjxxjogdgtzrqllbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oftwxwdfneincqdfnklpelxumdowyqqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbqfklhqsyltgwvyuolniayzitbeivxp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"desheyrgivsiymrfwwmijatosxjsphjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyhgrmxvploootodgqedsbmmkoivervd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrlmiohsxicrmbpjaaihzgmuotyqvftn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbxloxqnvkhigwnwxbwvxxmfzyeiygdl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"porynvkiudgizasyzyvcuxxgiwjaatqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uufrteqfcawjcnvpqlksdukeocfkyagy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpeicythvbmlxrfekpegqkqmebqsaxsx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dfnntwjgqfmninollcxgxpoucjuocjfm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"domnywiavgokaqcjnynbchedjggatbec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbozdpnemrgxhpgjlyvrxrgwaxynshlg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"issveyhkzguxlrlnpuekmdfzhtslzdel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrgmmecpnzqqaxucbmbzwyjroyqglryc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkoqdrdzywjyeflgpxqlaaxoycdbbttw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umfhpeficxuxwypciqbckhpwvgkjwqhb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udekrntozmhejfvfleaikigdbxctfxvm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrnvpswfrsxswadtozxurdonybssyuvi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"epwshavnuinnxbcclydcdlkzvxydknhu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wajgrwpimvwbvfjcdtpechdzrvzpgqdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jyypvchaxoxmeizdmwsemccdjgvfznkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwvqxcemspbzfdgmuiuslicrgwpfjnjf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjsctkghzbujfxntpnftzjhvowpzsgfv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jsjpxwxzycwkjpvkbvjarwawlufrmfyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwanykkbsigphkinezovsvxgvjqmciod","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqhfsdhyimdfmldfqhuonriznkgkcavc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jibcrmhurybzplmclatnkiahnlvjcstx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbpkighkualkmpcgcgzhznaxzawxcdpl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hajyvcprivreywkrvovlwfyzqjubphld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsfsfxqfvklztfatfwlanzyjpygsildx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocdfnsqzylsqkabroosaubpqprebnbrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgtcwpqpoglgmjljalncjmabnthqctiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swsavzbvudikrjttxpightxslxbqgyqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwixuaaqhqijkdscztpdnzyeroxtwmrv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhlxrkitpucivarxdgjdhziunpgmjmex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"paqxzfwqypadzwokvksmcjtdpkkotqlz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxuylbbwaoegadzukdeutobeabshxogm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gergjmyoqmqbimksxjcxovpkbekvcyjk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrzfuzyemyrbivukxojutrovjckqstux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upbvpjlxclbtekroigigvbjfxjrdrihf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecoxqpmleyevnlduzktspodqdljwfspf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chajkjdvjimctoisdesuqtxvtpxtalde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ndbjwsysssppxleglkuqttpxlbjnzruq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjxnvniwzbpnsxtjvqgaeqwnoxlfssup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lbjsaudeyimymhqtrhmjtgshmimhwzor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjzhtzeyghktndnxfjrxmnnucnexgetf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmqbkzefbkbltdhmqkugwugamexuiblt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxmnirvhrxqzkoqfsnexybztvahkapha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwdfvdoknuydollqyepytowcwqimwiwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxlvixwvqwggmlsxxcsdpnoynzzjcrky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdjzzvyihzsjxwaujnafygnwmisabnmm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebmppaywsyqiijryzigxstttundtxvbh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnyquqjhwwagbretqjqdmdkeoimihyek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohulgwmsuakcpolrodaehrvsycblvctn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjfjdhxtftxyotdobgiijxedlcyarspq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tfzihkajfeqsscfmkikrjqnardfpfpgs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmnlhmmsjswnhnmtgezcotbqegykawej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecagoxhbohyunmwyymvhbmlxymmdliqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcbjuquvlacknanaxngegrkulcaxsvyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"woujhsibewchxbhkdeluecvxeylmaftg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"trpwztiwmedhksbpgonmtztpqzravrjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ostyewkafapfizikiqemlcibarlxmtzx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfwpfvdjbylfldkwbnvdxzkftbqmbrgn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cenfssirnbfeofrlkedhhzfwcjipyame","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pizfuikeyoydepbzcvekjntukcbmruox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtjsajaiggjdtlsdwfzqxviojovarjku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmifzxrsoktdtihjazkycigkhibfsrrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nskmfqajfhdsnsthmjmaczoeqjjwquoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxhtjigqiawqwyorfztoswklchtppqnx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"daejfhhugncdrfgjnsbdljochmkrwvdx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpmbtansabdukuolihcrimuaknfayucl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tehtwgjiscmrcidibsexflibikzlazny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfhebzlptmiggtohmsuynmjpqatdugyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"spweagquinsqzxcozzoazngpdueadjcy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667382,"databaseName":"models_schema","ddl":"CREATE TABLE `dktjgzfdvuzusxrwcdfekwieeadvscpa` (\n `pmupojuusdjgtjacsjeerecdzivbvlfo` int NOT NULL,\n `kkleiyhfxoixnctdcphkqucztrevsdjg` int DEFAULT NULL,\n `lanaqxuevngvvmhqfclirhbairniyzhd` int DEFAULT NULL,\n `htwujykuanweoizqvppoqhwdiqqgdvza` int DEFAULT NULL,\n `ujcfixddkkrkhuugpnuthmpdctwnzxsi` int DEFAULT NULL,\n `ozquhkjxathkhcikxydbhmradgiistdh` int DEFAULT NULL,\n `cvvpdrsyfbdpkuydtedssrpcamwmeqyx` int DEFAULT NULL,\n `thyrorhqirmimliwkdupygnabpsgppjb` int DEFAULT NULL,\n `kkwqrimwgnbiplmbhbasbnkgbcpacsej` int DEFAULT NULL,\n `nmdhartdiomdlsaqkqkcbfezmjvsxxdo` int DEFAULT NULL,\n `ubsohnxwblgnyrsxhcwsptjrbqcbpwvd` int DEFAULT NULL,\n `ylkekgbesvquoywsebnkambcrvpfuesa` int DEFAULT NULL,\n `igukbpncdkmtlsgiwqxphbiixcdzbmue` int DEFAULT NULL,\n `bfdfxnkoieupribiyhqymmbufielyuub` int DEFAULT NULL,\n `aimhlgdfgwmggktjksjwaqzptjetybyi` int DEFAULT NULL,\n `atblznskbcycmeqdwgzlbroyyuityumq` int DEFAULT NULL,\n `fkkmlqgpjdcbysmnhatskqaolthlqkyx` int DEFAULT NULL,\n `pzqhuqscgvkupxqrjnlisteqczpwcmfb` int DEFAULT NULL,\n `fufjeeudpowzdtljkcpdhkyltfkyrrpb` int DEFAULT NULL,\n `lecosjgmosswrmgwpcgebgzqtatyjrbr` int DEFAULT NULL,\n `rdglbxzrefouswchrascaqmuaajrhhdt` int DEFAULT NULL,\n `crsbmnwghjelkbrtsgovezvkeryzzxjg` int DEFAULT NULL,\n `lnaoadubhufotqhmbcubvheqsrcowdqh` int DEFAULT NULL,\n `ylhuhhmdizadorrzwtndsxzztzvfjkgi` int DEFAULT NULL,\n `snddclcinmbxhjthzfisntyqpzfogokb` int DEFAULT NULL,\n `guvgjbfhlykctrcipwprzmqzptgtwcmw` int DEFAULT NULL,\n `zccqplghqhbjvzrdcnoeafvdufprpexq` int DEFAULT NULL,\n `lhlpexzaufoxitacbneltjlhwahyfvhr` int DEFAULT NULL,\n `exmxocudkvpfqkrkchbzhktfqetmjdaa` int DEFAULT NULL,\n `iznchclddklbqqjhbyscbfmdvrtozbsn` int DEFAULT NULL,\n `gkeusmkvejksphynkmbdzwbvbvhahinj` int DEFAULT NULL,\n `frlhabqofdibspqbrzgnfqaopzwmjjpw` int DEFAULT NULL,\n `tylsywbmollnxhnbxgllkwimmlsxbjyx` int DEFAULT NULL,\n `ifdnlmmgdnfmrktnfoakmuypcdswdkkj` int DEFAULT NULL,\n `zafryusczmyrcdrpniuhrgtluoxywggs` int DEFAULT NULL,\n `rdqqqrvfgryybghmizndpcobkmbludfj` int DEFAULT NULL,\n `qunnafchrwkkirttvouqtzbipyauwdta` int DEFAULT NULL,\n `ijiopibhnjoaerfjsjxrmyacrkkibwvq` int DEFAULT NULL,\n `hiupngsnqffckqeohxcusuvvunexokqf` int DEFAULT NULL,\n `gezbdxqlfpzizpwwytkgbwagindfnhnx` int DEFAULT NULL,\n `tzkppfaaffqxcdwglvcbietloloykubx` int DEFAULT NULL,\n `kbacphwpacxfujcetcdorscvtaaqhfmh` int DEFAULT NULL,\n `sjfelkjmuvtvnozefliifeurbatmiqvf` int DEFAULT NULL,\n `kwzlgkripafdfkpbqrcamrtmpsxhdjnf` int DEFAULT NULL,\n `iapgkfkzchkdjxpoihvchoemhtgattkr` int DEFAULT NULL,\n `pcmohurhgfbanipmbejbzygcihquhxhi` int DEFAULT NULL,\n `oknwyvsbimqnzsbmpawllqddwpjqoxkx` int DEFAULT NULL,\n `hrwidjcirahdwgfxonkwpdxylajsevol` int DEFAULT NULL,\n `qummcdsnhojzobogcsnpdorypfaakvet` int DEFAULT NULL,\n `cnwigylgoymdtpqdrxtzsuwxfgabyeag` int DEFAULT NULL,\n `czzadbaohocgkkvccjcdvmbqipmljwwn` int DEFAULT NULL,\n `gayldiztmxmgyrmeiwniudsccpfrmxad` int DEFAULT NULL,\n `qiohedslwyahwpmztpeblphwvscdtdac` int DEFAULT NULL,\n `gompwaemqfdxbbngqgfzkffkgdeoheje` int DEFAULT NULL,\n `aszksszwftbusmcuvyxwyqtbqibrwwrc` int DEFAULT NULL,\n `yvbzvieyiadxuprjplxgeiockjqldpyn` int DEFAULT NULL,\n `vuwqbavgdsinghbxnknrkxcggdpmzpnp` int DEFAULT NULL,\n `lihhpokkybiqnltnoyjefsfgoldbcyto` int DEFAULT NULL,\n `yukhsfcrevmbsyktmzwxmpxszcmxpssk` int DEFAULT NULL,\n `cayysvcravfxzwpzrepggodceyvsneys` int DEFAULT NULL,\n `fbqflbfzsfitaazjzmuxnfkacolpgpef` int DEFAULT NULL,\n `qvssxwjvsshawwghwaujddcyjhesldrm` int DEFAULT NULL,\n `wlskgjbkggntptfqccnafzcvfhdnrcoj` int DEFAULT NULL,\n `jmufkzaenmfxksislzjpapvklvrhspio` int DEFAULT NULL,\n `tcuvkfqqzxexikhqvnltgpfwicmnnrfu` int DEFAULT NULL,\n `gyqwpravbkyopfleeeuprjzaibrbkvzn` int DEFAULT NULL,\n `tooetvrcdanzbopyhpvcpayozrcnfgxu` int DEFAULT NULL,\n `jztbcbzuoxvmotsjlrkclgyftutklmia` int DEFAULT NULL,\n `xxyuznzngixlecimoilpkqewtsbglkih` int DEFAULT NULL,\n `yqyvzpoogdjzfodpdimzbplueabfrapr` int DEFAULT NULL,\n `xndjsdvecuvhjcsbguuexlxjklcmfpbx` int DEFAULT NULL,\n `jzcdxwlwsmegwvcamuzmacrrkdizmakb` int DEFAULT NULL,\n `thoyvkjlfqojqnzlbjxmicpbvtfixzky` int DEFAULT NULL,\n `tyjlbddtlvwdknkwbxbfewnswtnzpriy` int DEFAULT NULL,\n `amwuvskasyqfeqspamahityfmbkfnvln` int DEFAULT NULL,\n `ndwkkhbntwnbijwpqoicinsfqmsxnhme` int DEFAULT NULL,\n `lykxarukkzzhxeemdcdtclafqucdyhdy` int DEFAULT NULL,\n `xbuntswxhfkaihrwsapenmsambxddhwy` int DEFAULT NULL,\n `agrsxomtnmkyfslsuzssaxzzdoamiyhq` int DEFAULT NULL,\n `azhuttwmcinzvwlpmusimbewgiltgnxw` int DEFAULT NULL,\n `gaderwgnsaqelksxgmrxbttofubivpao` int DEFAULT NULL,\n `mqdwnknnhzhotyiowcuvepyfahexknwx` int DEFAULT NULL,\n `okgwsuxauvgclnbsmrkfkgfmshohgevg` int DEFAULT NULL,\n `edotkkcgiwcvamfwcryiqcevemygkhzd` int DEFAULT NULL,\n `umacffktyvsilohfuojeazzzdqhhwpwt` int DEFAULT NULL,\n `sjxamekntgpwloucvxxocvghaysbqhde` int DEFAULT NULL,\n `opcatbtdyuvbfiyxkqfwfqjobdemgdya` int DEFAULT NULL,\n `qtdihperobmuoutegxddwdgfssnlubxc` int DEFAULT NULL,\n `jmyxaorknsfyyklzrmtyjmdtrtawgdrz` int DEFAULT NULL,\n `zkkbvgpdctgnnjmlbtoewguirjqrefgw` int DEFAULT NULL,\n `ghkfykszopznopdenxvuwccwrzodxxpd` int DEFAULT NULL,\n `dgvdayuhgbhjickmjbhiuyogxsqbujex` int DEFAULT NULL,\n `oaxohqrkauxnjhsncjyawubuqdrtlyih` int DEFAULT NULL,\n `vdlasfczncbxnpnrkhxckbdozshxgwdk` int DEFAULT NULL,\n `gpssdwoiqjjexvkkposihckkeiqcgfur` int DEFAULT NULL,\n `vujuswthbbynernuotczjuftgzvmhjva` int DEFAULT NULL,\n `dzwyixxrkhtodiggakvarzwevwaeutvk` int DEFAULT NULL,\n `fwlaqapxoohcvqgazeiatcfabgxtrvzi` int DEFAULT NULL,\n `kqevaaqoydutudmcvojrbtcndsmpoawx` int DEFAULT NULL,\n `anesphouizquhfyxscpgpchkrtgwvkcr` int DEFAULT NULL,\n PRIMARY KEY (`pmupojuusdjgtjacsjeerecdzivbvlfo`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"dktjgzfdvuzusxrwcdfekwieeadvscpa\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["pmupojuusdjgtjacsjeerecdzivbvlfo"],"columns":[{"name":"pmupojuusdjgtjacsjeerecdzivbvlfo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"kkleiyhfxoixnctdcphkqucztrevsdjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lanaqxuevngvvmhqfclirhbairniyzhd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htwujykuanweoizqvppoqhwdiqqgdvza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujcfixddkkrkhuugpnuthmpdctwnzxsi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozquhkjxathkhcikxydbhmradgiistdh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvvpdrsyfbdpkuydtedssrpcamwmeqyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thyrorhqirmimliwkdupygnabpsgppjb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kkwqrimwgnbiplmbhbasbnkgbcpacsej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmdhartdiomdlsaqkqkcbfezmjvsxxdo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubsohnxwblgnyrsxhcwsptjrbqcbpwvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ylkekgbesvquoywsebnkambcrvpfuesa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igukbpncdkmtlsgiwqxphbiixcdzbmue","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bfdfxnkoieupribiyhqymmbufielyuub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aimhlgdfgwmggktjksjwaqzptjetybyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atblznskbcycmeqdwgzlbroyyuityumq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fkkmlqgpjdcbysmnhatskqaolthlqkyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pzqhuqscgvkupxqrjnlisteqczpwcmfb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fufjeeudpowzdtljkcpdhkyltfkyrrpb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lecosjgmosswrmgwpcgebgzqtatyjrbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdglbxzrefouswchrascaqmuaajrhhdt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crsbmnwghjelkbrtsgovezvkeryzzxjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lnaoadubhufotqhmbcubvheqsrcowdqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ylhuhhmdizadorrzwtndsxzztzvfjkgi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"snddclcinmbxhjthzfisntyqpzfogokb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"guvgjbfhlykctrcipwprzmqzptgtwcmw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zccqplghqhbjvzrdcnoeafvdufprpexq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhlpexzaufoxitacbneltjlhwahyfvhr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exmxocudkvpfqkrkchbzhktfqetmjdaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iznchclddklbqqjhbyscbfmdvrtozbsn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkeusmkvejksphynkmbdzwbvbvhahinj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"frlhabqofdibspqbrzgnfqaopzwmjjpw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tylsywbmollnxhnbxgllkwimmlsxbjyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifdnlmmgdnfmrktnfoakmuypcdswdkkj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zafryusczmyrcdrpniuhrgtluoxywggs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdqqqrvfgryybghmizndpcobkmbludfj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qunnafchrwkkirttvouqtzbipyauwdta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijiopibhnjoaerfjsjxrmyacrkkibwvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hiupngsnqffckqeohxcusuvvunexokqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gezbdxqlfpzizpwwytkgbwagindfnhnx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tzkppfaaffqxcdwglvcbietloloykubx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kbacphwpacxfujcetcdorscvtaaqhfmh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjfelkjmuvtvnozefliifeurbatmiqvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwzlgkripafdfkpbqrcamrtmpsxhdjnf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iapgkfkzchkdjxpoihvchoemhtgattkr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pcmohurhgfbanipmbejbzygcihquhxhi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oknwyvsbimqnzsbmpawllqddwpjqoxkx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hrwidjcirahdwgfxonkwpdxylajsevol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qummcdsnhojzobogcsnpdorypfaakvet","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnwigylgoymdtpqdrxtzsuwxfgabyeag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"czzadbaohocgkkvccjcdvmbqipmljwwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gayldiztmxmgyrmeiwniudsccpfrmxad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qiohedslwyahwpmztpeblphwvscdtdac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gompwaemqfdxbbngqgfzkffkgdeoheje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aszksszwftbusmcuvyxwyqtbqibrwwrc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yvbzvieyiadxuprjplxgeiockjqldpyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vuwqbavgdsinghbxnknrkxcggdpmzpnp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lihhpokkybiqnltnoyjefsfgoldbcyto","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yukhsfcrevmbsyktmzwxmpxszcmxpssk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cayysvcravfxzwpzrepggodceyvsneys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbqflbfzsfitaazjzmuxnfkacolpgpef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvssxwjvsshawwghwaujddcyjhesldrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wlskgjbkggntptfqccnafzcvfhdnrcoj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmufkzaenmfxksislzjpapvklvrhspio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tcuvkfqqzxexikhqvnltgpfwicmnnrfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gyqwpravbkyopfleeeuprjzaibrbkvzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tooetvrcdanzbopyhpvcpayozrcnfgxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jztbcbzuoxvmotsjlrkclgyftutklmia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxyuznzngixlecimoilpkqewtsbglkih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqyvzpoogdjzfodpdimzbplueabfrapr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xndjsdvecuvhjcsbguuexlxjklcmfpbx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzcdxwlwsmegwvcamuzmacrrkdizmakb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thoyvkjlfqojqnzlbjxmicpbvtfixzky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tyjlbddtlvwdknkwbxbfewnswtnzpriy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amwuvskasyqfeqspamahityfmbkfnvln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ndwkkhbntwnbijwpqoicinsfqmsxnhme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lykxarukkzzhxeemdcdtclafqucdyhdy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbuntswxhfkaihrwsapenmsambxddhwy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agrsxomtnmkyfslsuzssaxzzdoamiyhq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azhuttwmcinzvwlpmusimbewgiltgnxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gaderwgnsaqelksxgmrxbttofubivpao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqdwnknnhzhotyiowcuvepyfahexknwx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okgwsuxauvgclnbsmrkfkgfmshohgevg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edotkkcgiwcvamfwcryiqcevemygkhzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umacffktyvsilohfuojeazzzdqhhwpwt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjxamekntgpwloucvxxocvghaysbqhde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opcatbtdyuvbfiyxkqfwfqjobdemgdya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtdihperobmuoutegxddwdgfssnlubxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmyxaorknsfyyklzrmtyjmdtrtawgdrz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zkkbvgpdctgnnjmlbtoewguirjqrefgw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghkfykszopznopdenxvuwccwrzodxxpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgvdayuhgbhjickmjbhiuyogxsqbujex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oaxohqrkauxnjhsncjyawubuqdrtlyih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdlasfczncbxnpnrkhxckbdozshxgwdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gpssdwoiqjjexvkkposihckkeiqcgfur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vujuswthbbynernuotczjuftgzvmhjva","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dzwyixxrkhtodiggakvarzwevwaeutvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwlaqapxoohcvqgazeiatcfabgxtrvzi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqevaaqoydutudmcvojrbtcndsmpoawx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"anesphouizquhfyxscpgpchkrtgwvkcr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667420,"databaseName":"models_schema","ddl":"CREATE TABLE `dmpkfkxwkhuajbllpqhimtoiumvecdvq` (\n `zjtuehwyypagpzlzwnpphpxldvgylujb` int NOT NULL,\n `gcjheqzkzuppkbzjxxnverpjdwueiynu` int DEFAULT NULL,\n `apybrwrvgzvyflznsvoprjxwbznavkcz` int DEFAULT NULL,\n `hmhjgkdgfspluzwgwuyygzzlpmsznvcq` int DEFAULT NULL,\n `tdxfyxbmdaahmxyowaxlszatbmvalgqj` int DEFAULT NULL,\n `mkkumatjsyjtwaegiwuyorcfhwinrauq` int DEFAULT NULL,\n `evphojdmeamxxyvqtbpdzgfewgmgupdo` int DEFAULT NULL,\n `zvmepioowcundmjjkkshvfmwkcprhxam` int DEFAULT NULL,\n `gprzrfebyaqjntvwsqkadfaovcafsefz` int DEFAULT NULL,\n `vndezvlgefmazsxepjgfhnaslszwyeok` int DEFAULT NULL,\n `hgzrztwkmuaijszxkmwjiiztdyhznucx` int DEFAULT NULL,\n `yllopfcovygaarqypreombodsfcecenj` int DEFAULT NULL,\n `sgghgoshqzgvpsxivzkjsmiofojggzbr` int DEFAULT NULL,\n `xrfxfdhgwvckjkfcawyrgieaykeqcdta` int DEFAULT NULL,\n `kvzwqxxhvxbfttjbyetzfbyfixujbply` int DEFAULT NULL,\n `pgkskkvkkoldwodfaayzllrqlwtkmvww` int DEFAULT NULL,\n `ruzrfdybcnkbpzamhygyonffrxeifsex` int DEFAULT NULL,\n `kxifldvgzcchcceefvgmpcwbkxqhzuxf` int DEFAULT NULL,\n `zhlxrgnqgeliistoupqxuolmcjowfynq` int DEFAULT NULL,\n `ikxvxogbbdjaavlmhvzdczgjyhawoyfh` int DEFAULT NULL,\n `pbrqmxyakgfsuaizgzznoobqfowsiztn` int DEFAULT NULL,\n `ameeyaldlcgvidaqarpinhndlhrqvgqy` int DEFAULT NULL,\n `luznilvyyeftvoyggiqnjlazhwanfeca` int DEFAULT NULL,\n `fuyyjlkglyoyzoqhfdukgpzfzivxmzvf` int DEFAULT NULL,\n `maphviwgcmfmysgnqpvmhykbltbfgfka` int DEFAULT NULL,\n `okghpnnumyxgndjpmlmulmczmlwjzbht` int DEFAULT NULL,\n `tzpaxutpojoayhgtpzubzplgvnxbvxnv` int DEFAULT NULL,\n `qxymegykxwvevuupsbbatnhuhljyrihc` int DEFAULT NULL,\n `ntxstrsrsxjbnmajxahunncvifnasiva` int DEFAULT NULL,\n `qvcnrgiikhttggbgphxwtwxyacmqotte` int DEFAULT NULL,\n `uynpfvxhtyloywulvxcrgzoqxiipxuem` int DEFAULT NULL,\n `jflnucfhorxwmlttmhuzaranzuzxbiji` int DEFAULT NULL,\n `mnxhrmnaluwawitudxflbdyhktwfehkn` int DEFAULT NULL,\n `dwdslzztidcjagrbvfcasqclfxroawzn` int DEFAULT NULL,\n `usvzcgvrlduwtywzxsneortqmtwdqwhr` int DEFAULT NULL,\n `ggsronbzoioegtyzqmjpukpckyvnueia` int DEFAULT NULL,\n `fedecwkaicivgkretnfudmhjsepebdqj` int DEFAULT NULL,\n `bqrpxwcemgfwvemcdhxaarhfjkpmmiqa` int DEFAULT NULL,\n `pcixnjvsmozpayaydgsqfyoqpfgnxftt` int DEFAULT NULL,\n `lvbixeydudtlrmgwqosiksdgdrsvdbor` int DEFAULT NULL,\n `uvbvhbvopcblrlnojhutfjbqrvomogpd` int DEFAULT NULL,\n `roospijbiumlynflzsnyowtoojwtribm` int DEFAULT NULL,\n `bjgocspgrhrhgnmapnmeakvkandepwgu` int DEFAULT NULL,\n `idfakleucgelqgqmgqfrhguqyfagxlcx` int DEFAULT NULL,\n `kjwpohjyfaxukfryycfmxlfsvlhhzovx` int DEFAULT NULL,\n `jcxpiqnsjgiznnnvatmwxpgqobkkpzxu` int DEFAULT NULL,\n `ujfftxxiztquvnxbulksrgptnzegkrqo` int DEFAULT NULL,\n `vbtnhucdgpeqmgheifcjocxjpphahysg` int DEFAULT NULL,\n `cgbtfykggnedxtvvxdknchflgfpmkbdc` int DEFAULT NULL,\n `hngkfwmpnkbjsdqdoiajjysukmsroifm` int DEFAULT NULL,\n `cbkxijwdglxqkxverhumairvzxedlydw` int DEFAULT NULL,\n `yomcxpqeuvyoclsunlbibklktowusjvz` int DEFAULT NULL,\n `pnqseewwhhgktucdoptdkxwbakgvobpb` int DEFAULT NULL,\n `rrbirsoljfqgebunbijvlqgpoacqgsld` int DEFAULT NULL,\n `tqpbrbwoissiohpxtmcysdmywaptufbn` int DEFAULT NULL,\n `odrbpyitsmdlbkosvkfiziorngdhmiqc` int DEFAULT NULL,\n `uuxygcplhznsnosgizseuxqsorthramo` int DEFAULT NULL,\n `lmukvliggjjjmnjciquindromcywozgj` int DEFAULT NULL,\n `irwejibuulpesfpbdzfcaztnpdbjxmrh` int DEFAULT NULL,\n `ehyvafwrldgpidhzzwglqvnxgndwrdhv` int DEFAULT NULL,\n `wabnyhpqwmccemopaolrmogifqhidgpu` int DEFAULT NULL,\n `lfizqjhnwsnbdoesacnjbxkmcqpxnruw` int DEFAULT NULL,\n `sbaxdrhrnxsndcjiylkiixitpabainop` int DEFAULT NULL,\n `jxcfvkfnqfembskukcmshjubvceckkcn` int DEFAULT NULL,\n `kqrmahhdvuvzgeretjxlzritprajfqls` int DEFAULT NULL,\n `zkylpbxwgpikvurjsgotlaauwmjfofqr` int DEFAULT NULL,\n `nnrajsnpgkmrxvsawbteqkulkjtgiafe` int DEFAULT NULL,\n `dkhvfxehyksixdwjequfymcjxrpvnepp` int DEFAULT NULL,\n `mlgffufntnwfebtxppyveawesdlorkfd` int DEFAULT NULL,\n `ahitaljlqvrllvbqhbdxgrtmrayswbie` int DEFAULT NULL,\n `gspcyvkxttjqynaubblmxgwzxydetmbc` int DEFAULT NULL,\n `covepcakghsszeawqkgskxnqisuejsmj` int DEFAULT NULL,\n `zkmcahmbrnazvexiptpkxljqoucgmeym` int DEFAULT NULL,\n `eleqayyodzvbwznfgcwpubqvuhlzddnw` int DEFAULT NULL,\n `rjrgtfwcfchiphgkjrmibkoskdpeqrlr` int DEFAULT NULL,\n `koqjdbkbazvqsksbfoqmzquybkxsgtrq` int DEFAULT NULL,\n `hikhihsdrpyupmwnpguyfwffqlhhmxuo` int DEFAULT NULL,\n `bheuvhtqknzjflddhgxkijramrfeivgp` int DEFAULT NULL,\n `rtifgedlqwihdvwubhnsvebgpiaffcij` int DEFAULT NULL,\n `uaxriidejnrydolsozgraiqlqvuheydi` int DEFAULT NULL,\n `soujecbhcrmplviqsodlqtixefjvjwoi` int DEFAULT NULL,\n `swufxsakegjtasoiiwwnckwkhbetwxcp` int DEFAULT NULL,\n `pgcaonrwfkatyfqjywmukxcxdgvznntt` int DEFAULT NULL,\n `ztzdpmrmuqprrhjwoylxqfzppcnebwhy` int DEFAULT NULL,\n `nmpgczbajxhvwhhgrqbltdswhtadtbpl` int DEFAULT NULL,\n `mebgrudgpkhyavogwlmkvxsmgzvyzsjt` int DEFAULT NULL,\n `wrnwluknecysjfgsrfziuobrmhfitksm` int DEFAULT NULL,\n `axuxbgrlgxgtjnbxmygnwoqpgwednzsk` int DEFAULT NULL,\n `yfcvatzrbbursxhtbhiusnquavjoyqyq` int DEFAULT NULL,\n `mejmdrrzganvkaknsmdqrvegfxkqxqbk` int DEFAULT NULL,\n `celgxjyheaaytlkvboqzosnidlciigqo` int DEFAULT NULL,\n `rpoomjrzanwelwtevgaqnumdcbpstabd` int DEFAULT NULL,\n `uwpztjnhfzhwndrzsanchzimwzlvqnzc` int DEFAULT NULL,\n `fstybjzkgbgazgaipvhqnjpyuhbivyba` int DEFAULT NULL,\n `rzslmsphbkwabcrkaduolsbxdfblermn` int DEFAULT NULL,\n `bmtcgxlhibscricgmcgrmmccuajwcbui` int DEFAULT NULL,\n `vkftuvuenftljezfholjbixqydlkezoi` int DEFAULT NULL,\n `mjnjhdaocoldsbqrlbswtddpqxsroyhd` int DEFAULT NULL,\n `iyanxewcdpincvhptbtjygimoupgqzlp` int DEFAULT NULL,\n `pcyfnvtoakyeiwfgywcilccptxlgdbxo` int DEFAULT NULL,\n PRIMARY KEY (`zjtuehwyypagpzlzwnpphpxldvgylujb`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"dmpkfkxwkhuajbllpqhimtoiumvecdvq\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["zjtuehwyypagpzlzwnpphpxldvgylujb"],"columns":[{"name":"zjtuehwyypagpzlzwnpphpxldvgylujb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"gcjheqzkzuppkbzjxxnverpjdwueiynu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"apybrwrvgzvyflznsvoprjxwbznavkcz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmhjgkdgfspluzwgwuyygzzlpmsznvcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdxfyxbmdaahmxyowaxlszatbmvalgqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkkumatjsyjtwaegiwuyorcfhwinrauq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evphojdmeamxxyvqtbpdzgfewgmgupdo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvmepioowcundmjjkkshvfmwkcprhxam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gprzrfebyaqjntvwsqkadfaovcafsefz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vndezvlgefmazsxepjgfhnaslszwyeok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hgzrztwkmuaijszxkmwjiiztdyhznucx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yllopfcovygaarqypreombodsfcecenj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sgghgoshqzgvpsxivzkjsmiofojggzbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrfxfdhgwvckjkfcawyrgieaykeqcdta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvzwqxxhvxbfttjbyetzfbyfixujbply","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgkskkvkkoldwodfaayzllrqlwtkmvww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ruzrfdybcnkbpzamhygyonffrxeifsex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxifldvgzcchcceefvgmpcwbkxqhzuxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhlxrgnqgeliistoupqxuolmcjowfynq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikxvxogbbdjaavlmhvzdczgjyhawoyfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbrqmxyakgfsuaizgzznoobqfowsiztn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ameeyaldlcgvidaqarpinhndlhrqvgqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"luznilvyyeftvoyggiqnjlazhwanfeca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fuyyjlkglyoyzoqhfdukgpzfzivxmzvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"maphviwgcmfmysgnqpvmhykbltbfgfka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okghpnnumyxgndjpmlmulmczmlwjzbht","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tzpaxutpojoayhgtpzubzplgvnxbvxnv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxymegykxwvevuupsbbatnhuhljyrihc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntxstrsrsxjbnmajxahunncvifnasiva","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvcnrgiikhttggbgphxwtwxyacmqotte","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uynpfvxhtyloywulvxcrgzoqxiipxuem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jflnucfhorxwmlttmhuzaranzuzxbiji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnxhrmnaluwawitudxflbdyhktwfehkn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwdslzztidcjagrbvfcasqclfxroawzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"usvzcgvrlduwtywzxsneortqmtwdqwhr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggsronbzoioegtyzqmjpukpckyvnueia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fedecwkaicivgkretnfudmhjsepebdqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqrpxwcemgfwvemcdhxaarhfjkpmmiqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pcixnjvsmozpayaydgsqfyoqpfgnxftt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvbixeydudtlrmgwqosiksdgdrsvdbor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvbvhbvopcblrlnojhutfjbqrvomogpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"roospijbiumlynflzsnyowtoojwtribm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjgocspgrhrhgnmapnmeakvkandepwgu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idfakleucgelqgqmgqfrhguqyfagxlcx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjwpohjyfaxukfryycfmxlfsvlhhzovx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jcxpiqnsjgiznnnvatmwxpgqobkkpzxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujfftxxiztquvnxbulksrgptnzegkrqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbtnhucdgpeqmgheifcjocxjpphahysg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgbtfykggnedxtvvxdknchflgfpmkbdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hngkfwmpnkbjsdqdoiajjysukmsroifm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbkxijwdglxqkxverhumairvzxedlydw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yomcxpqeuvyoclsunlbibklktowusjvz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnqseewwhhgktucdoptdkxwbakgvobpb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrbirsoljfqgebunbijvlqgpoacqgsld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqpbrbwoissiohpxtmcysdmywaptufbn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"odrbpyitsmdlbkosvkfiziorngdhmiqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uuxygcplhznsnosgizseuxqsorthramo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmukvliggjjjmnjciquindromcywozgj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"irwejibuulpesfpbdzfcaztnpdbjxmrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehyvafwrldgpidhzzwglqvnxgndwrdhv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wabnyhpqwmccemopaolrmogifqhidgpu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfizqjhnwsnbdoesacnjbxkmcqpxnruw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbaxdrhrnxsndcjiylkiixitpabainop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jxcfvkfnqfembskukcmshjubvceckkcn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqrmahhdvuvzgeretjxlzritprajfqls","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zkylpbxwgpikvurjsgotlaauwmjfofqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nnrajsnpgkmrxvsawbteqkulkjtgiafe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkhvfxehyksixdwjequfymcjxrpvnepp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlgffufntnwfebtxppyveawesdlorkfd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahitaljlqvrllvbqhbdxgrtmrayswbie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gspcyvkxttjqynaubblmxgwzxydetmbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"covepcakghsszeawqkgskxnqisuejsmj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zkmcahmbrnazvexiptpkxljqoucgmeym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eleqayyodzvbwznfgcwpubqvuhlzddnw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjrgtfwcfchiphgkjrmibkoskdpeqrlr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"koqjdbkbazvqsksbfoqmzquybkxsgtrq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hikhihsdrpyupmwnpguyfwffqlhhmxuo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bheuvhtqknzjflddhgxkijramrfeivgp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtifgedlqwihdvwubhnsvebgpiaffcij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uaxriidejnrydolsozgraiqlqvuheydi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"soujecbhcrmplviqsodlqtixefjvjwoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swufxsakegjtasoiiwwnckwkhbetwxcp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgcaonrwfkatyfqjywmukxcxdgvznntt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ztzdpmrmuqprrhjwoylxqfzppcnebwhy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmpgczbajxhvwhhgrqbltdswhtadtbpl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mebgrudgpkhyavogwlmkvxsmgzvyzsjt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wrnwluknecysjfgsrfziuobrmhfitksm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axuxbgrlgxgtjnbxmygnwoqpgwednzsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfcvatzrbbursxhtbhiusnquavjoyqyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mejmdrrzganvkaknsmdqrvegfxkqxqbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"celgxjyheaaytlkvboqzosnidlciigqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpoomjrzanwelwtevgaqnumdcbpstabd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwpztjnhfzhwndrzsanchzimwzlvqnzc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fstybjzkgbgazgaipvhqnjpyuhbivyba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzslmsphbkwabcrkaduolsbxdfblermn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bmtcgxlhibscricgmcgrmmccuajwcbui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkftuvuenftljezfholjbixqydlkezoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjnjhdaocoldsbqrlbswtddpqxsroyhd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iyanxewcdpincvhptbtjygimoupgqzlp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pcyfnvtoakyeiwfgywcilccptxlgdbxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667455,"databaseName":"models_schema","ddl":"CREATE TABLE `eckmzheculilvvtgddqoabqhmblwjros` (\n `aobaekfsyueekamdxjxbeiphmibzzgqx` int NOT NULL,\n `dkqtvzvzwuifugpwigcdktysrfqjonzr` int DEFAULT NULL,\n `xsbtmmdtwppelplvehucqoufyyqmpuig` int DEFAULT NULL,\n `bjiiepcdflkptgsrknubhcixyfejaxqw` int DEFAULT NULL,\n `qdqwgfmpsmyfnriywuqydopiwymkzzvs` int DEFAULT NULL,\n `dqlpifghzewwfsjxmfkoiyplhkfstown` int DEFAULT NULL,\n `pfqdkzslcqfpepkdgucrtbxfovnvksrq` int DEFAULT NULL,\n `xvfemiajgmqrhzsiruvpnkqioutmavkw` int DEFAULT NULL,\n `bvzozhbguflmowstjxycfffkrmdkxbgb` int DEFAULT NULL,\n `yjlwwesyfzbkaoopgyxjejetrnxevnes` int DEFAULT NULL,\n `zjfzhtsjvjcytmnogvqwulbozzlwdhmu` int DEFAULT NULL,\n `pvcpvkqqvpxhdcgsksxztuqehnbdqlfx` int DEFAULT NULL,\n `skvmosszncsqzbybvxkmezrntfefwnkt` int DEFAULT NULL,\n `hlkiksauhgiacyqgeuqhzsaqisdlepxs` int DEFAULT NULL,\n `zugdudyyzqwtqmubrzxspyhcqkrwyhar` int DEFAULT NULL,\n `cqztzouepxwnrwhauglaljpywwxjbiom` int DEFAULT NULL,\n `tbrnkudjdtdrjitwbmrdqteabstopfzb` int DEFAULT NULL,\n `mxzskyktmugjfjzytbxcauvitcnivblr` int DEFAULT NULL,\n `eyucbsuybllylujuqddvvzibnzgezkfi` int DEFAULT NULL,\n `pmupnghusgafpmdizmdcohobvkylwixk` int DEFAULT NULL,\n `iaialuzdqwvraytksdfgopgyahrwbxmj` int DEFAULT NULL,\n `wuznltrvzabiikvmfmssvgliwrgxigfu` int DEFAULT NULL,\n `mzpqdgnixaurgvigzznurxxcvsqriolu` int DEFAULT NULL,\n `oitgcoucaazagoaslcxyhjbevckbsxrc` int DEFAULT NULL,\n `pchrnemgkglwvsxhlclsyjzsjyccylki` int DEFAULT NULL,\n `pzbiqwryzkxootxelvyyemqtuqywxdud` int DEFAULT NULL,\n `dbcyzaukwodiueuquzvblhcrfcckiqtj` int DEFAULT NULL,\n `pnudhwzgegfzjulladkkcwqqzgtimdvw` int DEFAULT NULL,\n `wesgszpyyjphshtgsnnlilhkibrcjslb` int DEFAULT NULL,\n `jwapdjhpjvbeoncnnsyglfutrjebagnb` int DEFAULT NULL,\n `oquuzqbcssljhjwjuspgzwojrlovhvan` int DEFAULT NULL,\n `atoggaobeswsvzmlqgvtlrvuszpuhyux` int DEFAULT NULL,\n `qtevpnpmzbjkdltdldwdtsegiitwlucz` int DEFAULT NULL,\n `ovgptouoasxwamiikxatkwwteivoacqy` int DEFAULT NULL,\n `itzipwbbscchugfdrmhqbcrospkavnaq` int DEFAULT NULL,\n `dndwqqrpuassjooknhkhibroywpcutwq` int DEFAULT NULL,\n `hvvwibbxhurzzsqbesbfwhoincohplcp` int DEFAULT NULL,\n `jdadydrfsznzhkdsnrbrkjqieuiyeeiq` int DEFAULT NULL,\n `xphkgziwneegecaghatqpwugbuudholg` int DEFAULT NULL,\n `vlmtmnsqzqhusfzouaghsijxexroqkso` int DEFAULT NULL,\n `iikkxaiwfdltunqzjhdrqzujvmforedv` int DEFAULT NULL,\n `lzpsiormoqvmeqksmasdpljzwydyffyw` int DEFAULT NULL,\n `bnphfsbbkwnobajiurbqlrxztaurazhd` int DEFAULT NULL,\n `fhkttybonmuwiihhkmntkgwpolfizgxx` int DEFAULT NULL,\n `nlmrhtujeoyegbjvuajtotnfjwmjcpfn` int DEFAULT NULL,\n `qrhlfjdornclnagnlecjrpwyqszxtwyn` int DEFAULT NULL,\n `jxefhzmautsieayeymfjfkgqyudnpupy` int DEFAULT NULL,\n `avrcfkilhhxxfsvifrypwqiznozqddvb` int DEFAULT NULL,\n `splbdweqnizjxhyzenhprutolmhdrdes` int DEFAULT NULL,\n `bonkvkrfwdkbyhvtskiejdoijgfvdbwg` int DEFAULT NULL,\n `ovddtmjbdmfthybybcnlntjddvsluyxo` int DEFAULT NULL,\n `genbxwvfsrfemwzslnrkaoyuhgrsywes` int DEFAULT NULL,\n `shqmwaptfyozwytpsuxnwifteafkgunw` int DEFAULT NULL,\n `kuledpqqkldoormvhmcdehwmubjeyihu` int DEFAULT NULL,\n `rxmwonudwdrmzgzliwiagzucdifjezfn` int DEFAULT NULL,\n `apimrixpckwusomrrfxquftknuuekqqc` int DEFAULT NULL,\n `caoueyukivqwpdmjnimvptclmavcmcdu` int DEFAULT NULL,\n `taqiqnczwhgrdqwlfoaabzkipbahoatk` int DEFAULT NULL,\n `nwmzycsuvptwfwtprgnmzredyfmltyre` int DEFAULT NULL,\n `tszuewpulebudsxbjgcvzuokhxrbjzns` int DEFAULT NULL,\n `hldyusmehivscwhngchomomdaliauazk` int DEFAULT NULL,\n `deaynzcbfiqhwoqhawvlkcnnjfubgriw` int DEFAULT NULL,\n `frpdcdgrdfrfzwqpeewmhsmezlyhypui` int DEFAULT NULL,\n `lgqtegedobiqytytsvxzfgjhifymoubq` int DEFAULT NULL,\n `xwgdpzpwpyjigivylvauwvuulbjnifsr` int DEFAULT NULL,\n `ulilvlfszifivwnwfjmyipzvizeyombg` int DEFAULT NULL,\n `ftgfincpcxujzbzhaxmooreptqvugccx` int DEFAULT NULL,\n `itriwpdzndururdvymqwjaqqzqgtaacs` int DEFAULT NULL,\n `udyfsdnllyaqavxywshcgkwtwdoyfelc` int DEFAULT NULL,\n `hqjbfpqeujejbzlajknszohklpjtnqbt` int DEFAULT NULL,\n `rfbvukliirkprjesivwsooowzjesdrhf` int DEFAULT NULL,\n `htehhbupcxtjhfatmuyzmugtaofejgql` int DEFAULT NULL,\n `tlpbbszrflobbzplraxogljezdlbupmq` int DEFAULT NULL,\n `rdgcygbmgyyxpfjjwrmumabpjbehwtbh` int DEFAULT NULL,\n `njcurshoydvzzliquaqsmpbxyaqunlrr` int DEFAULT NULL,\n `zcyxwxlrknqaqwvvhnhzuzkacltunxvo` int DEFAULT NULL,\n `ggjxxnnnwokmkwcgohggxbiaumkoeukp` int DEFAULT NULL,\n `cwxgthtkgnsgvaqxmpgsuphupymiemhg` int DEFAULT NULL,\n `idrhicxrguomttnjpgnpqysvzvzuinqh` int DEFAULT NULL,\n `zpmjucqmsgqcqlappsveiqudsvygvbht` int DEFAULT NULL,\n `fbnxmzjmwwxpvajzsmmpixpbxadqtwim` int DEFAULT NULL,\n `tqcfsmbagnazlsqgvsvaqnmzvxgmuhvo` int DEFAULT NULL,\n `ushkfgtvszuztwuudgypvbwlyfoofgio` int DEFAULT NULL,\n `gfdoxzlqhlsfvtdeegywsddvabnliyzc` int DEFAULT NULL,\n `ufmjhuepkibwaaxptcvyhtcnlfoxyida` int DEFAULT NULL,\n `utkgxlmvcsvyqjzyixayszznjqiaehvo` int DEFAULT NULL,\n `bbxmnpfibuyfrblfqgrzbcipijflewft` int DEFAULT NULL,\n `lgmzvnuluzdtacpoyqpibrbafutppwom` int DEFAULT NULL,\n `tqyniovsdxrnewoitkrkldusdxnumrjz` int DEFAULT NULL,\n `ikrejkwgncnvtvwzexerraajpmfexxqo` int DEFAULT NULL,\n `yymvkqqhpsdwnuxygyvftraidaaifzti` int DEFAULT NULL,\n `vedabqfhwmevreowbgsurkmpxtklbsvl` int DEFAULT NULL,\n `isqcrrhkuraawkkiaddqobgrbdcjwedo` int DEFAULT NULL,\n `wglnonqbgmryjqyjvdcpnosorqtyprwz` int DEFAULT NULL,\n `znlnqcxeogrrxcnisgkzmtmtndkpsarw` int DEFAULT NULL,\n `mmuuljfrjwfbrebytodwlvtaqqzptjfu` int DEFAULT NULL,\n `hyzycrzvwesyedcamfiwbcdngxsjljwz` int DEFAULT NULL,\n `tlgqblolgtobazjlcfcnyjchgfzecvwa` int DEFAULT NULL,\n `wqtprgnbbjqmwjusbbuxafwrlywufnaf` int DEFAULT NULL,\n `martixaktikcukgksjxowztjznkevbur` int DEFAULT NULL,\n PRIMARY KEY (`aobaekfsyueekamdxjxbeiphmibzzgqx`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"eckmzheculilvvtgddqoabqhmblwjros\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["aobaekfsyueekamdxjxbeiphmibzzgqx"],"columns":[{"name":"aobaekfsyueekamdxjxbeiphmibzzgqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"dkqtvzvzwuifugpwigcdktysrfqjonzr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsbtmmdtwppelplvehucqoufyyqmpuig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjiiepcdflkptgsrknubhcixyfejaxqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdqwgfmpsmyfnriywuqydopiwymkzzvs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqlpifghzewwfsjxmfkoiyplhkfstown","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pfqdkzslcqfpepkdgucrtbxfovnvksrq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvfemiajgmqrhzsiruvpnkqioutmavkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvzozhbguflmowstjxycfffkrmdkxbgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjlwwesyfzbkaoopgyxjejetrnxevnes","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjfzhtsjvjcytmnogvqwulbozzlwdhmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvcpvkqqvpxhdcgsksxztuqehnbdqlfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skvmosszncsqzbybvxkmezrntfefwnkt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlkiksauhgiacyqgeuqhzsaqisdlepxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zugdudyyzqwtqmubrzxspyhcqkrwyhar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqztzouepxwnrwhauglaljpywwxjbiom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbrnkudjdtdrjitwbmrdqteabstopfzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mxzskyktmugjfjzytbxcauvitcnivblr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eyucbsuybllylujuqddvvzibnzgezkfi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmupnghusgafpmdizmdcohobvkylwixk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iaialuzdqwvraytksdfgopgyahrwbxmj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wuznltrvzabiikvmfmssvgliwrgxigfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzpqdgnixaurgvigzznurxxcvsqriolu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oitgcoucaazagoaslcxyhjbevckbsxrc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pchrnemgkglwvsxhlclsyjzsjyccylki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pzbiqwryzkxootxelvyyemqtuqywxdud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbcyzaukwodiueuquzvblhcrfcckiqtj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnudhwzgegfzjulladkkcwqqzgtimdvw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wesgszpyyjphshtgsnnlilhkibrcjslb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwapdjhpjvbeoncnnsyglfutrjebagnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oquuzqbcssljhjwjuspgzwojrlovhvan","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atoggaobeswsvzmlqgvtlrvuszpuhyux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtevpnpmzbjkdltdldwdtsegiitwlucz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovgptouoasxwamiikxatkwwteivoacqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itzipwbbscchugfdrmhqbcrospkavnaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dndwqqrpuassjooknhkhibroywpcutwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hvvwibbxhurzzsqbesbfwhoincohplcp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdadydrfsznzhkdsnrbrkjqieuiyeeiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xphkgziwneegecaghatqpwugbuudholg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vlmtmnsqzqhusfzouaghsijxexroqkso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iikkxaiwfdltunqzjhdrqzujvmforedv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzpsiormoqvmeqksmasdpljzwydyffyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnphfsbbkwnobajiurbqlrxztaurazhd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhkttybonmuwiihhkmntkgwpolfizgxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nlmrhtujeoyegbjvuajtotnfjwmjcpfn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrhlfjdornclnagnlecjrpwyqszxtwyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jxefhzmautsieayeymfjfkgqyudnpupy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avrcfkilhhxxfsvifrypwqiznozqddvb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"splbdweqnizjxhyzenhprutolmhdrdes","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bonkvkrfwdkbyhvtskiejdoijgfvdbwg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovddtmjbdmfthybybcnlntjddvsluyxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"genbxwvfsrfemwzslnrkaoyuhgrsywes","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shqmwaptfyozwytpsuxnwifteafkgunw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kuledpqqkldoormvhmcdehwmubjeyihu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxmwonudwdrmzgzliwiagzucdifjezfn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"apimrixpckwusomrrfxquftknuuekqqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"caoueyukivqwpdmjnimvptclmavcmcdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"taqiqnczwhgrdqwlfoaabzkipbahoatk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nwmzycsuvptwfwtprgnmzredyfmltyre","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tszuewpulebudsxbjgcvzuokhxrbjzns","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hldyusmehivscwhngchomomdaliauazk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"deaynzcbfiqhwoqhawvlkcnnjfubgriw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"frpdcdgrdfrfzwqpeewmhsmezlyhypui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgqtegedobiqytytsvxzfgjhifymoubq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xwgdpzpwpyjigivylvauwvuulbjnifsr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ulilvlfszifivwnwfjmyipzvizeyombg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftgfincpcxujzbzhaxmooreptqvugccx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itriwpdzndururdvymqwjaqqzqgtaacs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udyfsdnllyaqavxywshcgkwtwdoyfelc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqjbfpqeujejbzlajknszohklpjtnqbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfbvukliirkprjesivwsooowzjesdrhf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htehhbupcxtjhfatmuyzmugtaofejgql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlpbbszrflobbzplraxogljezdlbupmq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdgcygbmgyyxpfjjwrmumabpjbehwtbh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njcurshoydvzzliquaqsmpbxyaqunlrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zcyxwxlrknqaqwvvhnhzuzkacltunxvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggjxxnnnwokmkwcgohggxbiaumkoeukp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwxgthtkgnsgvaqxmpgsuphupymiemhg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idrhicxrguomttnjpgnpqysvzvzuinqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpmjucqmsgqcqlappsveiqudsvygvbht","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbnxmzjmwwxpvajzsmmpixpbxadqtwim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqcfsmbagnazlsqgvsvaqnmzvxgmuhvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ushkfgtvszuztwuudgypvbwlyfoofgio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfdoxzlqhlsfvtdeegywsddvabnliyzc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufmjhuepkibwaaxptcvyhtcnlfoxyida","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utkgxlmvcsvyqjzyixayszznjqiaehvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbxmnpfibuyfrblfqgrzbcipijflewft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgmzvnuluzdtacpoyqpibrbafutppwom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqyniovsdxrnewoitkrkldusdxnumrjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikrejkwgncnvtvwzexerraajpmfexxqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yymvkqqhpsdwnuxygyvftraidaaifzti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vedabqfhwmevreowbgsurkmpxtklbsvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"isqcrrhkuraawkkiaddqobgrbdcjwedo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wglnonqbgmryjqyjvdcpnosorqtyprwz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znlnqcxeogrrxcnisgkzmtmtndkpsarw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmuuljfrjwfbrebytodwlvtaqqzptjfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hyzycrzvwesyedcamfiwbcdngxsjljwz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlgqblolgtobazjlcfcnyjchgfzecvwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqtprgnbbjqmwjusbbuxafwrlywufnaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"martixaktikcukgksjxowztjznkevbur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667486,"databaseName":"models_schema","ddl":"CREATE TABLE `edtpgezoafwesgnsrttenofpfwbgvskp` (\n `zvpjkhkvjtxztpblvsgazlosvusscllc` int NOT NULL,\n `zdklckultrzwkfvmpugotjyaipsfiptg` int DEFAULT NULL,\n `ivylmpqpyvxchnbmoljcvchlsaiybbyx` int DEFAULT NULL,\n `pdopxrofftqgdvadnuyipioiwrqnxdas` int DEFAULT NULL,\n `qpnwvpywysisoohxlkenvwrahafbamil` int DEFAULT NULL,\n `sgbderncbhezgndbfpuvtqatnmrojlkw` int DEFAULT NULL,\n `dzifaxtmfjatlapmswbjoewfrceqzkek` int DEFAULT NULL,\n `oonmrctqnfqvsuizldbkcsedrntjzvzg` int DEFAULT NULL,\n `vnnpqqorljagslbrgzjuktbifigcqzro` int DEFAULT NULL,\n `cfdqyybbgnbmzlczvsxuzafydabwxvbg` int DEFAULT NULL,\n `qmtryldmcpvoaakdncufwkeldwyuycev` int DEFAULT NULL,\n `pysscugogxxznpafkkghiolvbzlurrfy` int DEFAULT NULL,\n `jqkueodpglnpqyhnxvnnpgkecklitosk` int DEFAULT NULL,\n `lqiuhmwiqlhxvgkwdrhhwaqggdjltyfl` int DEFAULT NULL,\n `xeylmnuydbtqsooifaifmyptjvuakple` int DEFAULT NULL,\n `awjfnjsufajbkkqbhjgqqfsqdfxajlcn` int DEFAULT NULL,\n `dpiuywunxrjvhruxovnkkmkmsjmkkrzc` int DEFAULT NULL,\n `uzuhgyohpfpaumcswddrctrwlunqncke` int DEFAULT NULL,\n `bpusvkzvsrksktdtwusoutslyefiscde` int DEFAULT NULL,\n `azfgmdirttukeibezazlrotxfugzbrki` int DEFAULT NULL,\n `otltgwkbfzsdciqqbfuycmvsosrbplbl` int DEFAULT NULL,\n `pglgclyppdboxvryyswffcsstqdliina` int DEFAULT NULL,\n `zxrrfpokanqubtwqxzewanklkomzaisd` int DEFAULT NULL,\n `sciugrkumvorbpsmyiozqlqwovzcswml` int DEFAULT NULL,\n `fpmsumsoawozgnmkueeawvmlqyfygobv` int DEFAULT NULL,\n `apmxruvpkhslsvgjvkkxfpvbemhjikmc` int DEFAULT NULL,\n `sielvnvxsepaozuzxkarbkitbpgpracn` int DEFAULT NULL,\n `redrdccnjxdazlzaogbdluvixrwngpos` int DEFAULT NULL,\n `kfvhqfamskybejeswqbnxiusgnasqhqk` int DEFAULT NULL,\n `hkqqiccvfwleopezhhybmgdwiseryhcm` int DEFAULT NULL,\n `vqobmyvbhryxfnxthzhfquxtfkxdobue` int DEFAULT NULL,\n `chsmobaqkhageonbzulxnmrmttjxnujd` int DEFAULT NULL,\n `pmgpdynsxmbycijzceqiusorhiztzxwc` int DEFAULT NULL,\n `glfazpgvxwpakplatefspjqmukroedsp` int DEFAULT NULL,\n `ixhogpthnaoaqsjwkcaivzvlbbfrrdac` int DEFAULT NULL,\n `mcjpwefvguvxolbpmnsspjytwetqvhil` int DEFAULT NULL,\n `uxkxscseudvowysjznltuckxjdjkfkyx` int DEFAULT NULL,\n `wypmrkejfxsxxwkpgxtubrhwcbbtcudh` int DEFAULT NULL,\n `xjcpksfcwexyuduqdfghqxglzqhksall` int DEFAULT NULL,\n `qkxowitgozsrjsxkxbxoxrtiupdphttz` int DEFAULT NULL,\n `tpqcezwmoygwssscuhcfhcmtvewzsslm` int DEFAULT NULL,\n `hjfrbrulcnsvusgebsntowvauiymslpe` int DEFAULT NULL,\n `kbjifigutikamlyorwdvhpsccoiwondr` int DEFAULT NULL,\n `ykuenqxzukyvuxibrirvfcvttyfujpmt` int DEFAULT NULL,\n `mnxkaojhuosbljrogspmgkpvrjhdpxvu` int DEFAULT NULL,\n `ptyssdbkcjrwtssfhwoxgjdidicgxder` int DEFAULT NULL,\n `ytyzejpqmoldxvnvjkoievfibnxdvdrv` int DEFAULT NULL,\n `qropcjndusxaelwthwnxgiqtvtabveox` int DEFAULT NULL,\n `sxsyleqgqnzhelbtmipglfyxigooaojs` int DEFAULT NULL,\n `gsudijtaehdxrzubphtckvgzegigcohb` int DEFAULT NULL,\n `prwfgveqjuenyvekksutzbnbnnuhbqga` int DEFAULT NULL,\n `oyardjenltbrdusukdejputywqncrnzd` int DEFAULT NULL,\n `wgajcqqlzhzzkhfdajjqfwxuvctcslvo` int DEFAULT NULL,\n `tmirrakdyoaxzwfpczezlkfdlhovqtjm` int DEFAULT NULL,\n `ebkqcjgwgjonxypkvmzdfsfqskxrrzfx` int DEFAULT NULL,\n `xawnzxgyikuirebxdnoyqjmyqvoiqrxe` int DEFAULT NULL,\n `mumtqsxsaqaxikznbfrwabdoounxxsxh` int DEFAULT NULL,\n `cuyjzutlacjrfluwothqjixxcpblmqds` int DEFAULT NULL,\n `zagfhicjxrtqwiwnymnlnvizihxsvfxv` int DEFAULT NULL,\n `zrnwlwspqifpopshptucvnvafanzwwwx` int DEFAULT NULL,\n `sysvvbnliazmvgsidhoqmreayqjhllpl` int DEFAULT NULL,\n `tpkqpewsotauvnouctqperowiklfpgyg` int DEFAULT NULL,\n `gbcjfqexoerhlepzvdoucjwcofejhtpb` int DEFAULT NULL,\n `yiketlgdvgtjryuwfekbmgqhhnrxeqxx` int DEFAULT NULL,\n `hhqpzplfytarkjpycgxgybkfiwsbimwl` int DEFAULT NULL,\n `vvkhntrwittvwpyphgloflyzlwehdoqo` int DEFAULT NULL,\n `jfcmjtaxvfilnynghkgfqtrkczsyofka` int DEFAULT NULL,\n `bxlefsfzkpbsuvvvkojgxbtzypwkafsm` int DEFAULT NULL,\n `hexbfebvzuzszguzqzupainshidrvnoc` int DEFAULT NULL,\n `cfxivyftlxcwctiaanvxzbjalpqbnswo` int DEFAULT NULL,\n `smxbqrtchtrgyecncfkbieotnprrelgc` int DEFAULT NULL,\n `srltkspmznjaduhplvnpeyathoixaewu` int DEFAULT NULL,\n `yfnmuzeyfzrtjbnvyczyonpyjjjwlmap` int DEFAULT NULL,\n `soeukimzbfsonrbxpdnqwhgezhaoorrg` int DEFAULT NULL,\n `jjauezjddinlhfkbzoqtvqiedripcowq` int DEFAULT NULL,\n `yncnuiygxfdbhtdauwkxzxugdgkumtdv` int DEFAULT NULL,\n `teqqrbrobybepkdrtazcwdzeqaywkqsk` int DEFAULT NULL,\n `divncvwvshwtirbttzcfqoynwzluzkki` int DEFAULT NULL,\n `uyshxhzkjrdlkravqvdrtpjyspwlcgdh` int DEFAULT NULL,\n `aepeiegfwuijihhdljzcmiihaembrcge` int DEFAULT NULL,\n `xltmvrpfgctyzemlbqvfgmaqfxynvngu` int DEFAULT NULL,\n `eginziknrpbfyljyibxvsnrzshmqejyt` int DEFAULT NULL,\n `uujdvzxpebpsjptbdyepujgshefzyibd` int DEFAULT NULL,\n `itfjsalqedxezcggfeueynuzzgrqmhhc` int DEFAULT NULL,\n `nflbtkdnyfmmusdsesilrzntlvvyjpqs` int DEFAULT NULL,\n `wwnefbyurngctwyootfbexazbzfdfksk` int DEFAULT NULL,\n `lovllwzxglfrfeceadczeydxbhjccude` int DEFAULT NULL,\n `ygtsepdcsogipmavtlwsmgkymswefjfv` int DEFAULT NULL,\n `fbbjqfskdnzyvzchtadqenjtdytpkrii` int DEFAULT NULL,\n `venljjypjcpuzrtjgwrjfaxdrprbamjm` int DEFAULT NULL,\n `osuygpuyosslyumklrfzuuooxndivaop` int DEFAULT NULL,\n `qktakxrsgvdqckwzjblwfsebuiqfapec` int DEFAULT NULL,\n `pewrweuuxhgkfvabmkrdhjyhbtndvebg` int DEFAULT NULL,\n `pwbxgbnmpteybgzwvqmpqpiaxpvaxpyk` int DEFAULT NULL,\n `fpepvshltrwhrjslpurorqzmvvlgzeip` int DEFAULT NULL,\n `gqkaeeyinqurgorewsgyjvcgwuwfmygb` int DEFAULT NULL,\n `oprswzamxjyebzgqdajjcgbdqhfcrddt` int DEFAULT NULL,\n `mgxzsxyicfcwhksxfrwepwbfhwsocnpf` int DEFAULT NULL,\n `ftjrjctiaidnetswkgbdtfyhjdhqjdkw` int DEFAULT NULL,\n `znyoqrmzxwridexxmnrfmifcckldljgj` int DEFAULT NULL,\n PRIMARY KEY (`zvpjkhkvjtxztpblvsgazlosvusscllc`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"edtpgezoafwesgnsrttenofpfwbgvskp\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["zvpjkhkvjtxztpblvsgazlosvusscllc"],"columns":[{"name":"zvpjkhkvjtxztpblvsgazlosvusscllc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"zdklckultrzwkfvmpugotjyaipsfiptg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivylmpqpyvxchnbmoljcvchlsaiybbyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdopxrofftqgdvadnuyipioiwrqnxdas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qpnwvpywysisoohxlkenvwrahafbamil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sgbderncbhezgndbfpuvtqatnmrojlkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dzifaxtmfjatlapmswbjoewfrceqzkek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oonmrctqnfqvsuizldbkcsedrntjzvzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnnpqqorljagslbrgzjuktbifigcqzro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfdqyybbgnbmzlczvsxuzafydabwxvbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmtryldmcpvoaakdncufwkeldwyuycev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pysscugogxxznpafkkghiolvbzlurrfy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jqkueodpglnpqyhnxvnnpgkecklitosk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqiuhmwiqlhxvgkwdrhhwaqggdjltyfl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xeylmnuydbtqsooifaifmyptjvuakple","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awjfnjsufajbkkqbhjgqqfsqdfxajlcn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpiuywunxrjvhruxovnkkmkmsjmkkrzc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzuhgyohpfpaumcswddrctrwlunqncke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bpusvkzvsrksktdtwusoutslyefiscde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azfgmdirttukeibezazlrotxfugzbrki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otltgwkbfzsdciqqbfuycmvsosrbplbl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pglgclyppdboxvryyswffcsstqdliina","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxrrfpokanqubtwqxzewanklkomzaisd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sciugrkumvorbpsmyiozqlqwovzcswml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpmsumsoawozgnmkueeawvmlqyfygobv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"apmxruvpkhslsvgjvkkxfpvbemhjikmc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sielvnvxsepaozuzxkarbkitbpgpracn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"redrdccnjxdazlzaogbdluvixrwngpos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfvhqfamskybejeswqbnxiusgnasqhqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkqqiccvfwleopezhhybmgdwiseryhcm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqobmyvbhryxfnxthzhfquxtfkxdobue","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chsmobaqkhageonbzulxnmrmttjxnujd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmgpdynsxmbycijzceqiusorhiztzxwc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"glfazpgvxwpakplatefspjqmukroedsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixhogpthnaoaqsjwkcaivzvlbbfrrdac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcjpwefvguvxolbpmnsspjytwetqvhil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxkxscseudvowysjznltuckxjdjkfkyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wypmrkejfxsxxwkpgxtubrhwcbbtcudh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjcpksfcwexyuduqdfghqxglzqhksall","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qkxowitgozsrjsxkxbxoxrtiupdphttz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpqcezwmoygwssscuhcfhcmtvewzsslm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjfrbrulcnsvusgebsntowvauiymslpe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kbjifigutikamlyorwdvhpsccoiwondr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykuenqxzukyvuxibrirvfcvttyfujpmt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnxkaojhuosbljrogspmgkpvrjhdpxvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ptyssdbkcjrwtssfhwoxgjdidicgxder","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytyzejpqmoldxvnvjkoievfibnxdvdrv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qropcjndusxaelwthwnxgiqtvtabveox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sxsyleqgqnzhelbtmipglfyxigooaojs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsudijtaehdxrzubphtckvgzegigcohb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prwfgveqjuenyvekksutzbnbnnuhbqga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oyardjenltbrdusukdejputywqncrnzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgajcqqlzhzzkhfdajjqfwxuvctcslvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmirrakdyoaxzwfpczezlkfdlhovqtjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebkqcjgwgjonxypkvmzdfsfqskxrrzfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xawnzxgyikuirebxdnoyqjmyqvoiqrxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mumtqsxsaqaxikznbfrwabdoounxxsxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cuyjzutlacjrfluwothqjixxcpblmqds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zagfhicjxrtqwiwnymnlnvizihxsvfxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zrnwlwspqifpopshptucvnvafanzwwwx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sysvvbnliazmvgsidhoqmreayqjhllpl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpkqpewsotauvnouctqperowiklfpgyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gbcjfqexoerhlepzvdoucjwcofejhtpb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yiketlgdvgtjryuwfekbmgqhhnrxeqxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhqpzplfytarkjpycgxgybkfiwsbimwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vvkhntrwittvwpyphgloflyzlwehdoqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jfcmjtaxvfilnynghkgfqtrkczsyofka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bxlefsfzkpbsuvvvkojgxbtzypwkafsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hexbfebvzuzszguzqzupainshidrvnoc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfxivyftlxcwctiaanvxzbjalpqbnswo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smxbqrtchtrgyecncfkbieotnprrelgc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"srltkspmznjaduhplvnpeyathoixaewu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfnmuzeyfzrtjbnvyczyonpyjjjwlmap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"soeukimzbfsonrbxpdnqwhgezhaoorrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjauezjddinlhfkbzoqtvqiedripcowq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yncnuiygxfdbhtdauwkxzxugdgkumtdv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"teqqrbrobybepkdrtazcwdzeqaywkqsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"divncvwvshwtirbttzcfqoynwzluzkki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uyshxhzkjrdlkravqvdrtpjyspwlcgdh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aepeiegfwuijihhdljzcmiihaembrcge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xltmvrpfgctyzemlbqvfgmaqfxynvngu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eginziknrpbfyljyibxvsnrzshmqejyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uujdvzxpebpsjptbdyepujgshefzyibd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itfjsalqedxezcggfeueynuzzgrqmhhc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nflbtkdnyfmmusdsesilrzntlvvyjpqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwnefbyurngctwyootfbexazbzfdfksk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lovllwzxglfrfeceadczeydxbhjccude","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ygtsepdcsogipmavtlwsmgkymswefjfv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbbjqfskdnzyvzchtadqenjtdytpkrii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"venljjypjcpuzrtjgwrjfaxdrprbamjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"osuygpuyosslyumklrfzuuooxndivaop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qktakxrsgvdqckwzjblwfsebuiqfapec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pewrweuuxhgkfvabmkrdhjyhbtndvebg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwbxgbnmpteybgzwvqmpqpiaxpvaxpyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpepvshltrwhrjslpurorqzmvvlgzeip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqkaeeyinqurgorewsgyjvcgwuwfmygb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oprswzamxjyebzgqdajjcgbdqhfcrddt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mgxzsxyicfcwhksxfrwepwbfhwsocnpf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftjrjctiaidnetswkgbdtfyhjdhqjdkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znyoqrmzxwridexxmnrfmifcckldljgj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667519,"databaseName":"models_schema","ddl":"CREATE TABLE `efltdshxkjahnyuwforbiwbeqptladwm` (\n `bauevokddonrtuktzidwkzduaxonfmis` int NOT NULL,\n `mqeqraszudegjwaqimdlmriiohcyboxf` int DEFAULT NULL,\n `crjupxaexvgkgjakvnirfoksqfhjsino` int DEFAULT NULL,\n `lsoaskvcgbsxxtecmdnjmvcthaeejbhe` int DEFAULT NULL,\n `klmrusstapxiullabhufbsehlsymetvk` int DEFAULT NULL,\n `uzuiirdfbznhwkbblfweuqwpsgxgkhzl` int DEFAULT NULL,\n `mfaewdoiogosvwnuhpowfvkihrxljkaf` int DEFAULT NULL,\n `qvfeaaimaptcgmjfbcqpztipkjueorsj` int DEFAULT NULL,\n `owoeemhxngytxgbzlroirdzssjkyvkge` int DEFAULT NULL,\n `uuwledufthobcsanehihnxxfzegwgtpr` int DEFAULT NULL,\n `emjmpxaizvqiyabhynargzfkzusdnnrw` int DEFAULT NULL,\n `iqqjkvgwjqfgltemacepspqjudgxsjba` int DEFAULT NULL,\n `pvbcpnxdebbkfajfsxqmxklzigqhgtxf` int DEFAULT NULL,\n `ieqwblwijkzsxympoqulovvbifmilhyt` int DEFAULT NULL,\n `eyovelhdfmoyyjotfruqaqyhqnlfnhtt` int DEFAULT NULL,\n `cqobooitnufwieaujbnovewdlincfmym` int DEFAULT NULL,\n `tpoqmyhgxracwxonrwpybpfigzbgsrjl` int DEFAULT NULL,\n `odhghbyxiynbffghqrfpbsgfvcmpzarb` int DEFAULT NULL,\n `qtmzoxuybqynnianiresnmzvglvnyvhf` int DEFAULT NULL,\n `rewvymfutkqejbzgwjscpeammibvfycv` int DEFAULT NULL,\n `hkjcjzhcxzciwxesbywkypnuuahdalap` int DEFAULT NULL,\n `cbsghykvrflyhtqkndxrjfyunzzbjkpl` int DEFAULT NULL,\n `fkldkhcbsfzvgppzupsuczrgtarncmfh` int DEFAULT NULL,\n `qpounvbtlgobxsxuahevdakzmrtzzfyw` int DEFAULT NULL,\n `jpnulacgfzldthqgyfwmwrzkhsfryhws` int DEFAULT NULL,\n `nejbnrokcckukogstltgooobpsccxewj` int DEFAULT NULL,\n `vatkwavskhskawjzhhyzkdrwweqgvhjo` int DEFAULT NULL,\n `kvzditgxgpseoifrgtasagybrisngreq` int DEFAULT NULL,\n `kbfkokaudkuptgtdrwmffybctlwnsfca` int DEFAULT NULL,\n `rpcrmloumzybnxnueoupyngldyqeyzkp` int DEFAULT NULL,\n `jvkzaydhugpqytglkovroqvtfllwmovz` int DEFAULT NULL,\n `retwjfrerwtfalufcfqvrvkqrqdpxndh` int DEFAULT NULL,\n `spqbqntlrvdfwreplwzcjyptuyelddvu` int DEFAULT NULL,\n `kczhwxksnonimimjfvvbjlwrqslpyxrq` int DEFAULT NULL,\n `uyhcnsjkyzcpadhhiovcotfloybwtgpk` int DEFAULT NULL,\n `xwjnisuxbcsnefihzpowobgqnaoerrmu` int DEFAULT NULL,\n `kfefcsieqrqelseqluyssyvvthisawcl` int DEFAULT NULL,\n `pkhwujswheretunjlfgbvuubrfcjgitt` int DEFAULT NULL,\n `ftzhmhzseuhejcayphkdhmjbdtwxwamb` int DEFAULT NULL,\n `sajdoiuotfwrwdvgbvfmryfrjjqhuibd` int DEFAULT NULL,\n `lzxgdizlqtdnxqbmzjkzvbfaacqbhoik` int DEFAULT NULL,\n `ryxozitlhgarbhazomihjpjibknibaos` int DEFAULT NULL,\n `lhomzrywuqmndnggwhzkzkhyckzvfjdg` int DEFAULT NULL,\n `mwaunstjccjedjkivclwervaeskmfdwu` int DEFAULT NULL,\n `lxoenxagjxdygbshaccqkcdjbihwmngz` int DEFAULT NULL,\n `acjaxotzuqttdqxxcqgnaetugeapsixd` int DEFAULT NULL,\n `yxzdoxghfqgvryunfiqvjnfdvzorkekv` int DEFAULT NULL,\n `dcaqmjfboxtpohoyifutjgdslqlxuoke` int DEFAULT NULL,\n `geuialnwryloamusqnbghsksoynfyzux` int DEFAULT NULL,\n `nbwtmnnzbuphhchqzpvprugbzqwdbpen` int DEFAULT NULL,\n `qcdxpsemprqtjshcfdaloberhcpkngot` int DEFAULT NULL,\n `qbdkunbjlfotghlakibkbfmswjfkeavh` int DEFAULT NULL,\n `slodfhorokuvyijoyoeqasrsrjtlztuo` int DEFAULT NULL,\n `jrmkbzocssltjtktygjaievqbggcywtl` int DEFAULT NULL,\n `pdxvdsmuhnloeyqsewcaerupxasmvjil` int DEFAULT NULL,\n `haqtolbvdfpczhflhnzsquszmeuflscm` int DEFAULT NULL,\n `newsdyrnczwvqkcsxqpcqeoxrpuhoisi` int DEFAULT NULL,\n `ulndyypczofdttzzwovxmcxkuboelrvv` int DEFAULT NULL,\n `xnifearcdcuicrjyyjelkaujviurkvgo` int DEFAULT NULL,\n `euavhebyfeiilhobtheknholwaeleotv` int DEFAULT NULL,\n `rubrrqepdpgtmrtcezfhpwisxtwyrvef` int DEFAULT NULL,\n `nyxcarpzrwoxvcxzcylvqlqewyzfcdzf` int DEFAULT NULL,\n `paqiwquqtclwsqrrunxedsrbuhktfhju` int DEFAULT NULL,\n `uicmhfmgrvkqfysrlpsfwierhbzpkjne` int DEFAULT NULL,\n `vslrcpbnptyoksxmyqvfxaezixjfbtpe` int DEFAULT NULL,\n `hriewtfcfjqoezjehaxzuanuzyczrdtr` int DEFAULT NULL,\n `pugakdjxydeaqqligpogqowaxsjfblvw` int DEFAULT NULL,\n `menxlsrzuwydcpricnenhlnypvcdddnc` int DEFAULT NULL,\n `afukkyrdosyvgeafjzkctnqjkmnnboye` int DEFAULT NULL,\n `vruduzsjkgmhiqjkfzptyrtfrtdabfxc` int DEFAULT NULL,\n `pgusjpakjiguzjhedhzxewgeieonuakc` int DEFAULT NULL,\n `lfeikphzhicajqdygrqncugupcsqmefk` int DEFAULT NULL,\n `azoagwmfrfrzxmgkrswbwqkfwvewdqdx` int DEFAULT NULL,\n `pxxhynjweighwsqtdplkdojflhfqppze` int DEFAULT NULL,\n `uyocstwkzjiamagqyzjistnmkdplvgdi` int DEFAULT NULL,\n `dtdbyheqpbwxcqudecvgapkxbrqhgkfu` int DEFAULT NULL,\n `pgfkvsmjeysjruwyawegpkiwpzcdravg` int DEFAULT NULL,\n `jvafdiaararjygytojphpplxkxokgvfn` int DEFAULT NULL,\n `dewaxxsllyoowjwcswcoyqqenqrtnfbc` int DEFAULT NULL,\n `zayvmopulslpfnemsheigzeksayypdku` int DEFAULT NULL,\n `ssytjjdalfxlvoqawvryftfeymxhgjph` int DEFAULT NULL,\n `abyugkrhoynxtqdhajgqhbiaznypgnoo` int DEFAULT NULL,\n `xunizurftcgrhaqbwycnbljzuoybuhnx` int DEFAULT NULL,\n `ipxumcpvkfgfhymhokiqmhfsnqlvxmge` int DEFAULT NULL,\n `sganwwsrekfwsbiqvmcdnskzdgyrbvhg` int DEFAULT NULL,\n `ctvcirlmopauxlddtvyuseitqdvhpknk` int DEFAULT NULL,\n `kcrltmebengfthnlrizsrsuiqyfowixs` int DEFAULT NULL,\n `ituoqgnebhjlptrsturraoygqwcgtcgh` int DEFAULT NULL,\n `giqdpejvnfmmckkuaflgotfuxmutkwle` int DEFAULT NULL,\n `zxproqxolpxdubcccmnufsgjpimjfwyk` int DEFAULT NULL,\n `qfldoprycdjrkjqlbydbvwaidipoftap` int DEFAULT NULL,\n `cvkcpbsyuuqzynfsprvuznkuxgdcvsqp` int DEFAULT NULL,\n `ofvueqdgnhghkfsjqzvzmbvylnhjwusl` int DEFAULT NULL,\n `xjopwhwlwfpdikyopmsxjbehnniiwmwq` int DEFAULT NULL,\n `zjuaqdilmmqxcpauhobztnobbdsbcuxf` int DEFAULT NULL,\n `pxrfwtdkuvbsifkwlrcgqensvimnzhip` int DEFAULT NULL,\n `cdgjzklnfpslnitcccmkwhuosuxdsrth` int DEFAULT NULL,\n `byswodurecuqliowvevsuuoixlumbzik` int DEFAULT NULL,\n `arislcjplmidtzlsatzmlgdblccxcklo` int DEFAULT NULL,\n `dpgbfjfxvoixdmyqcvmtxnoqykwnkunp` int DEFAULT NULL,\n PRIMARY KEY (`bauevokddonrtuktzidwkzduaxonfmis`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"efltdshxkjahnyuwforbiwbeqptladwm\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["bauevokddonrtuktzidwkzduaxonfmis"],"columns":[{"name":"bauevokddonrtuktzidwkzduaxonfmis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"mqeqraszudegjwaqimdlmriiohcyboxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crjupxaexvgkgjakvnirfoksqfhjsino","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lsoaskvcgbsxxtecmdnjmvcthaeejbhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"klmrusstapxiullabhufbsehlsymetvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzuiirdfbznhwkbblfweuqwpsgxgkhzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfaewdoiogosvwnuhpowfvkihrxljkaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvfeaaimaptcgmjfbcqpztipkjueorsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owoeemhxngytxgbzlroirdzssjkyvkge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uuwledufthobcsanehihnxxfzegwgtpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"emjmpxaizvqiyabhynargzfkzusdnnrw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqqjkvgwjqfgltemacepspqjudgxsjba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvbcpnxdebbkfajfsxqmxklzigqhgtxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ieqwblwijkzsxympoqulovvbifmilhyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eyovelhdfmoyyjotfruqaqyhqnlfnhtt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqobooitnufwieaujbnovewdlincfmym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpoqmyhgxracwxonrwpybpfigzbgsrjl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"odhghbyxiynbffghqrfpbsgfvcmpzarb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtmzoxuybqynnianiresnmzvglvnyvhf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rewvymfutkqejbzgwjscpeammibvfycv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkjcjzhcxzciwxesbywkypnuuahdalap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbsghykvrflyhtqkndxrjfyunzzbjkpl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fkldkhcbsfzvgppzupsuczrgtarncmfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qpounvbtlgobxsxuahevdakzmrtzzfyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpnulacgfzldthqgyfwmwrzkhsfryhws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nejbnrokcckukogstltgooobpsccxewj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vatkwavskhskawjzhhyzkdrwweqgvhjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvzditgxgpseoifrgtasagybrisngreq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kbfkokaudkuptgtdrwmffybctlwnsfca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpcrmloumzybnxnueoupyngldyqeyzkp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvkzaydhugpqytglkovroqvtfllwmovz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"retwjfrerwtfalufcfqvrvkqrqdpxndh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"spqbqntlrvdfwreplwzcjyptuyelddvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kczhwxksnonimimjfvvbjlwrqslpyxrq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uyhcnsjkyzcpadhhiovcotfloybwtgpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xwjnisuxbcsnefihzpowobgqnaoerrmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfefcsieqrqelseqluyssyvvthisawcl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkhwujswheretunjlfgbvuubrfcjgitt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftzhmhzseuhejcayphkdhmjbdtwxwamb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sajdoiuotfwrwdvgbvfmryfrjjqhuibd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzxgdizlqtdnxqbmzjkzvbfaacqbhoik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ryxozitlhgarbhazomihjpjibknibaos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhomzrywuqmndnggwhzkzkhyckzvfjdg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwaunstjccjedjkivclwervaeskmfdwu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxoenxagjxdygbshaccqkcdjbihwmngz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acjaxotzuqttdqxxcqgnaetugeapsixd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxzdoxghfqgvryunfiqvjnfdvzorkekv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcaqmjfboxtpohoyifutjgdslqlxuoke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"geuialnwryloamusqnbghsksoynfyzux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbwtmnnzbuphhchqzpvprugbzqwdbpen","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qcdxpsemprqtjshcfdaloberhcpkngot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qbdkunbjlfotghlakibkbfmswjfkeavh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"slodfhorokuvyijoyoeqasrsrjtlztuo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jrmkbzocssltjtktygjaievqbggcywtl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdxvdsmuhnloeyqsewcaerupxasmvjil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"haqtolbvdfpczhflhnzsquszmeuflscm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"newsdyrnczwvqkcsxqpcqeoxrpuhoisi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ulndyypczofdttzzwovxmcxkuboelrvv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnifearcdcuicrjyyjelkaujviurkvgo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"euavhebyfeiilhobtheknholwaeleotv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rubrrqepdpgtmrtcezfhpwisxtwyrvef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nyxcarpzrwoxvcxzcylvqlqewyzfcdzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"paqiwquqtclwsqrrunxedsrbuhktfhju","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uicmhfmgrvkqfysrlpsfwierhbzpkjne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vslrcpbnptyoksxmyqvfxaezixjfbtpe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hriewtfcfjqoezjehaxzuanuzyczrdtr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pugakdjxydeaqqligpogqowaxsjfblvw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"menxlsrzuwydcpricnenhlnypvcdddnc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afukkyrdosyvgeafjzkctnqjkmnnboye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vruduzsjkgmhiqjkfzptyrtfrtdabfxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgusjpakjiguzjhedhzxewgeieonuakc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfeikphzhicajqdygrqncugupcsqmefk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azoagwmfrfrzxmgkrswbwqkfwvewdqdx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxxhynjweighwsqtdplkdojflhfqppze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uyocstwkzjiamagqyzjistnmkdplvgdi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtdbyheqpbwxcqudecvgapkxbrqhgkfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgfkvsmjeysjruwyawegpkiwpzcdravg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvafdiaararjygytojphpplxkxokgvfn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dewaxxsllyoowjwcswcoyqqenqrtnfbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zayvmopulslpfnemsheigzeksayypdku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ssytjjdalfxlvoqawvryftfeymxhgjph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abyugkrhoynxtqdhajgqhbiaznypgnoo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xunizurftcgrhaqbwycnbljzuoybuhnx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ipxumcpvkfgfhymhokiqmhfsnqlvxmge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sganwwsrekfwsbiqvmcdnskzdgyrbvhg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctvcirlmopauxlddtvyuseitqdvhpknk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcrltmebengfthnlrizsrsuiqyfowixs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ituoqgnebhjlptrsturraoygqwcgtcgh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"giqdpejvnfmmckkuaflgotfuxmutkwle","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxproqxolpxdubcccmnufsgjpimjfwyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfldoprycdjrkjqlbydbvwaidipoftap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvkcpbsyuuqzynfsprvuznkuxgdcvsqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofvueqdgnhghkfsjqzvzmbvylnhjwusl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjopwhwlwfpdikyopmsxjbehnniiwmwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjuaqdilmmqxcpauhobztnobbdsbcuxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxrfwtdkuvbsifkwlrcgqensvimnzhip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cdgjzklnfpslnitcccmkwhuosuxdsrth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"byswodurecuqliowvevsuuoixlumbzik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"arislcjplmidtzlsatzmlgdblccxcklo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpgbfjfxvoixdmyqcvmtxnoqykwnkunp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667549,"databaseName":"models_schema","ddl":"CREATE TABLE `ekgwnocryenlhifewsirhpralwokvicj` (\n `ezwrjsgurodhdllijvxswjvdwbothnaj` int NOT NULL,\n `zhhgostrzwamoiilsuuoumqnuahxewrj` int DEFAULT NULL,\n `zpgaeuaklflfbbxyeuyzhzsbekekhoob` int DEFAULT NULL,\n `clmgpiozjrhpcrislkulrukqqmpzwete` int DEFAULT NULL,\n `oxldqaactmvbxaaaspvocebgqozadwop` int DEFAULT NULL,\n `htmpwxqrpjwmicwhmvugwlarxlreogun` int DEFAULT NULL,\n `ufwhidmmkdjlukktvyutgcegqcgwkies` int DEFAULT NULL,\n `odmievzbpivbhiokfkdkpofjpnuxhajq` int DEFAULT NULL,\n `jotujjgsfxogqtxtnwgtqbdzggcpneiw` int DEFAULT NULL,\n `rabgdffsmeuhxpypubhtsydbrnxwptjc` int DEFAULT NULL,\n `sfupceuajuuhrqavaeqwrytdjljektiu` int DEFAULT NULL,\n `pvycxpovhqsvjwietkgefimifwpjrrzy` int DEFAULT NULL,\n `gdmdzlizjjzpitztkdlmdkxniqlsmdfl` int DEFAULT NULL,\n `zbyycvhqsqumdkocbliqxnpzzmavlaug` int DEFAULT NULL,\n `pasbyyuiubxeyzlbqyxuqgpkkvbtdatt` int DEFAULT NULL,\n `syymvmhexuxcklrlvsaglpxdugixdgmb` int DEFAULT NULL,\n `jdkalruqyopkqsaqhdqzwihfiqilpkiz` int DEFAULT NULL,\n `qbczcmppfbuzkwtfmxmivhymaviqdvnk` int DEFAULT NULL,\n `qtdyrobmyersutrqtxjaxlicfpcvmuhh` int DEFAULT NULL,\n `cxklxsalhfbmgafnggwbqrscfjdxeptx` int DEFAULT NULL,\n `pklboccvmnmddirmficbjuzthyhyzdzr` int DEFAULT NULL,\n `uotvpyljzzskipertigwxhnyjbsbhwqu` int DEFAULT NULL,\n `xmrxhovdykobwhmlzexsjooobwrdtzzt` int DEFAULT NULL,\n `mlzwovdwapprcifxoyjzjmowdhrwrlvk` int DEFAULT NULL,\n `eoqhtozxemljtsqlteyhmzhievvmwcit` int DEFAULT NULL,\n `ezeaciymptqgpyobfldkgrscooxpvebj` int DEFAULT NULL,\n `hfoigepqfceokquqofafybdqwcnlrwic` int DEFAULT NULL,\n `xfuqliocwpunzzrtmzfvqiirqqvmvoew` int DEFAULT NULL,\n `izorxuwozuaijritpguzaiqkouuuckuo` int DEFAULT NULL,\n `vxagdlaubyhhiedlehrhsktzhmsqywte` int DEFAULT NULL,\n `vkmklbxeyfnirymnzpyuipbpwotsklej` int DEFAULT NULL,\n `vxssexpnzyuwnwmhpuxcvqtlvpoflwuo` int DEFAULT NULL,\n `aftkamufsgxosqdpgjogagtjrodkoypg` int DEFAULT NULL,\n `qfyalbcezavnqumvtunguprgrvbkbywa` int DEFAULT NULL,\n `qrrnuldqxsaqbddgvzfqupyvubgjjece` int DEFAULT NULL,\n `mumkpjlpayyvkizqpztuqghshxaglavq` int DEFAULT NULL,\n `pjhysxohdgxmsshccsszlrvngqquuifn` int DEFAULT NULL,\n `iwrjsqbrpovfpnzifhtmwrtrjhuwtpcj` int DEFAULT NULL,\n `rlhvrkpauvfireesrjitqiyamybtdoyg` int DEFAULT NULL,\n `noiwiftsunlzkmwqyckykxngmdtwyhxc` int DEFAULT NULL,\n `ufrdwfvakmlegwwmsyeiemduttuedtwa` int DEFAULT NULL,\n `blhgblubnetsylfpoziehqmeietijwid` int DEFAULT NULL,\n `coqalzhdzxqbmnqnyjawtrhamjfasdbo` int DEFAULT NULL,\n `zxujqrcxjxfbhlhsivcvadarebyjmvoi` int DEFAULT NULL,\n `dmzcljpnkqkijunhzotvpnelajiroahj` int DEFAULT NULL,\n `pvtucparikvpsqwllpotvtfxqnnsvslf` int DEFAULT NULL,\n `sjlclhndvsxdpzxfkkciorowfvdbvoub` int DEFAULT NULL,\n `njdubaujvlibczmpouapsvsarwznuudt` int DEFAULT NULL,\n `diyzfodmgnntenizyqfklxuiceevrpah` int DEFAULT NULL,\n `llwlbyrxqoxqkjetbztyaxwsugpoaklt` int DEFAULT NULL,\n `lslckqbioodvdflfsalgvchjpqqwxkyv` int DEFAULT NULL,\n `cgajgrtkogcyietjlyyvxqiaqshtllor` int DEFAULT NULL,\n `kwoqzzgvicrrtowwisnxaomvzmfojabj` int DEFAULT NULL,\n `nsvczjpphwqdyihjzrmrdyhdlarymapv` int DEFAULT NULL,\n `cigctwbasibufoudcyouljuovzqyqzrg` int DEFAULT NULL,\n `lnrvdkocuoeaggyhaapkzenjzxmogcgi` int DEFAULT NULL,\n `wowkupbohlpfhaddqlyutnkawsvnxpxs` int DEFAULT NULL,\n `ycgkqtwccghsnqprqmzvismpdgjagajc` int DEFAULT NULL,\n `aeellooiochvqojecxyqpohmjrptrpxc` int DEFAULT NULL,\n `bdaghtsogztoutbqerwbzcyoukyhavzh` int DEFAULT NULL,\n `momvlzzkkapzugugcsiykwvywasfgvms` int DEFAULT NULL,\n `cwbiurzsvxhrtbqrpmnfpztdcyoopsja` int DEFAULT NULL,\n `swkwmbualhnfdzpprywkuaajntfvlnjd` int DEFAULT NULL,\n `ruarzulnjibsinklngzwmyhkdtoyocvo` int DEFAULT NULL,\n `egeschqnfaqkutgknrzfmhyjydwjnjmd` int DEFAULT NULL,\n `wdgzwouldhblwhrrwctumesfkmmsmfku` int DEFAULT NULL,\n `kdkilztjpmtoxxoddenlhvwdmegyqtge` int DEFAULT NULL,\n `itbdkrvxaxccgzcferkimogvysucfipf` int DEFAULT NULL,\n `wgeqmkwpxkgtdypaenlcqvdvbngqrpim` int DEFAULT NULL,\n `inlwsiiktehfyjlqzyupsdsrjtwvrcpq` int DEFAULT NULL,\n `zpxxjquvzcbkdwuoxclffvbprvotqztn` int DEFAULT NULL,\n `iyvydyoaepyxizdnycerlufwuijbxtgh` int DEFAULT NULL,\n `xpzxmaxbkszncwnbwnnpqtjivvbqhhqx` int DEFAULT NULL,\n `rmfhhclsyxjszbjdvtsnqdgsfaxhtplg` int DEFAULT NULL,\n `hlyepaicgkflklepnekuhqciecpinnkb` int DEFAULT NULL,\n `oedkdskfjhzcnaraemtokwzqwymawnxb` int DEFAULT NULL,\n `ucgwtqgxgozkgcgiravdntawunoerpje` int DEFAULT NULL,\n `ecdsgdgeaeitxityowzasrqxpapjopie` int DEFAULT NULL,\n `ejwfjrlfbvlnccwiqzaojgeixbwtpfld` int DEFAULT NULL,\n `zqgeillxbdqrqzislqfnucazpdryyajm` int DEFAULT NULL,\n `bylmwvzruihtkzfzvwtenlxyzgzvslwj` int DEFAULT NULL,\n `suuydgncurtoxvfteqxbpvrtbqkszcgj` int DEFAULT NULL,\n `bljwixidynntcgjxjelkdkphfkakufkm` int DEFAULT NULL,\n `cyexmpyhvhfjevjzmtvvetstspgxiwob` int DEFAULT NULL,\n `iruonyddjptdubrfdhrfwegmrujslxph` int DEFAULT NULL,\n `hhpcbphqjccjfotmoonpocccvdcknncj` int DEFAULT NULL,\n `ohgvtpxrahzophmkwzogrcrukpwdrong` int DEFAULT NULL,\n `ryxtkrkqbpuveeawtzizwvayfeuufnel` int DEFAULT NULL,\n `crkylhaudjscuriplqwnhqpntqbwkbhe` int DEFAULT NULL,\n `mwxkbxxefpyhzhcuvlgvfyvdmqlgvlvu` int DEFAULT NULL,\n `rguilaabzfnveyyvglnbnxezkebijbrf` int DEFAULT NULL,\n `wtzkgxilskgonuanvxzutatmymmrbcmz` int DEFAULT NULL,\n `mokakwfmqngzhudkiowbykmbvpyuevzu` int DEFAULT NULL,\n `vipmpdfhamcjgkwdkhtxzgqyjqjpbhsp` int DEFAULT NULL,\n `jayfpqiieylsfnrldrdkfnsmypumzpxl` int DEFAULT NULL,\n `hozlzuequjheyulveqcoyppfukktbwsh` int DEFAULT NULL,\n `aazyucmxjffqdkgeepvnsioklgctfvgc` int DEFAULT NULL,\n `rzgijglvvffsiinmhumaafjxqmuhmyoz` int DEFAULT NULL,\n `knnixxutxveuuuwqqnsskwabaeqqvbhp` int DEFAULT NULL,\n `edyfhwuihresiwobfmixshztrcgmbbav` int DEFAULT NULL,\n PRIMARY KEY (`ezwrjsgurodhdllijvxswjvdwbothnaj`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"ekgwnocryenlhifewsirhpralwokvicj\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["ezwrjsgurodhdllijvxswjvdwbothnaj"],"columns":[{"name":"ezwrjsgurodhdllijvxswjvdwbothnaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"zhhgostrzwamoiilsuuoumqnuahxewrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpgaeuaklflfbbxyeuyzhzsbekekhoob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clmgpiozjrhpcrislkulrukqqmpzwete","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oxldqaactmvbxaaaspvocebgqozadwop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htmpwxqrpjwmicwhmvugwlarxlreogun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufwhidmmkdjlukktvyutgcegqcgwkies","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"odmievzbpivbhiokfkdkpofjpnuxhajq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jotujjgsfxogqtxtnwgtqbdzggcpneiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rabgdffsmeuhxpypubhtsydbrnxwptjc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfupceuajuuhrqavaeqwrytdjljektiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvycxpovhqsvjwietkgefimifwpjrrzy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdmdzlizjjzpitztkdlmdkxniqlsmdfl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbyycvhqsqumdkocbliqxnpzzmavlaug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pasbyyuiubxeyzlbqyxuqgpkkvbtdatt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"syymvmhexuxcklrlvsaglpxdugixdgmb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdkalruqyopkqsaqhdqzwihfiqilpkiz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qbczcmppfbuzkwtfmxmivhymaviqdvnk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtdyrobmyersutrqtxjaxlicfpcvmuhh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cxklxsalhfbmgafnggwbqrscfjdxeptx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pklboccvmnmddirmficbjuzthyhyzdzr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uotvpyljzzskipertigwxhnyjbsbhwqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmrxhovdykobwhmlzexsjooobwrdtzzt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlzwovdwapprcifxoyjzjmowdhrwrlvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eoqhtozxemljtsqlteyhmzhievvmwcit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezeaciymptqgpyobfldkgrscooxpvebj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfoigepqfceokquqofafybdqwcnlrwic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xfuqliocwpunzzrtmzfvqiirqqvmvoew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"izorxuwozuaijritpguzaiqkouuuckuo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxagdlaubyhhiedlehrhsktzhmsqywte","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkmklbxeyfnirymnzpyuipbpwotsklej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxssexpnzyuwnwmhpuxcvqtlvpoflwuo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aftkamufsgxosqdpgjogagtjrodkoypg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfyalbcezavnqumvtunguprgrvbkbywa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrrnuldqxsaqbddgvzfqupyvubgjjece","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mumkpjlpayyvkizqpztuqghshxaglavq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjhysxohdgxmsshccsszlrvngqquuifn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwrjsqbrpovfpnzifhtmwrtrjhuwtpcj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlhvrkpauvfireesrjitqiyamybtdoyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"noiwiftsunlzkmwqyckykxngmdtwyhxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufrdwfvakmlegwwmsyeiemduttuedtwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blhgblubnetsylfpoziehqmeietijwid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"coqalzhdzxqbmnqnyjawtrhamjfasdbo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxujqrcxjxfbhlhsivcvadarebyjmvoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dmzcljpnkqkijunhzotvpnelajiroahj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvtucparikvpsqwllpotvtfxqnnsvslf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjlclhndvsxdpzxfkkciorowfvdbvoub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njdubaujvlibczmpouapsvsarwznuudt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"diyzfodmgnntenizyqfklxuiceevrpah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llwlbyrxqoxqkjetbztyaxwsugpoaklt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lslckqbioodvdflfsalgvchjpqqwxkyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgajgrtkogcyietjlyyvxqiaqshtllor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwoqzzgvicrrtowwisnxaomvzmfojabj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nsvczjpphwqdyihjzrmrdyhdlarymapv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cigctwbasibufoudcyouljuovzqyqzrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lnrvdkocuoeaggyhaapkzenjzxmogcgi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wowkupbohlpfhaddqlyutnkawsvnxpxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ycgkqtwccghsnqprqmzvismpdgjagajc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aeellooiochvqojecxyqpohmjrptrpxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdaghtsogztoutbqerwbzcyoukyhavzh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"momvlzzkkapzugugcsiykwvywasfgvms","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwbiurzsvxhrtbqrpmnfpztdcyoopsja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swkwmbualhnfdzpprywkuaajntfvlnjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ruarzulnjibsinklngzwmyhkdtoyocvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"egeschqnfaqkutgknrzfmhyjydwjnjmd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wdgzwouldhblwhrrwctumesfkmmsmfku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kdkilztjpmtoxxoddenlhvwdmegyqtge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itbdkrvxaxccgzcferkimogvysucfipf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgeqmkwpxkgtdypaenlcqvdvbngqrpim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"inlwsiiktehfyjlqzyupsdsrjtwvrcpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpxxjquvzcbkdwuoxclffvbprvotqztn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iyvydyoaepyxizdnycerlufwuijbxtgh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpzxmaxbkszncwnbwnnpqtjivvbqhhqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rmfhhclsyxjszbjdvtsnqdgsfaxhtplg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlyepaicgkflklepnekuhqciecpinnkb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oedkdskfjhzcnaraemtokwzqwymawnxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucgwtqgxgozkgcgiravdntawunoerpje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecdsgdgeaeitxityowzasrqxpapjopie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejwfjrlfbvlnccwiqzaojgeixbwtpfld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqgeillxbdqrqzislqfnucazpdryyajm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bylmwvzruihtkzfzvwtenlxyzgzvslwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suuydgncurtoxvfteqxbpvrtbqkszcgj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bljwixidynntcgjxjelkdkphfkakufkm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cyexmpyhvhfjevjzmtvvetstspgxiwob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iruonyddjptdubrfdhrfwegmrujslxph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhpcbphqjccjfotmoonpocccvdcknncj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohgvtpxrahzophmkwzogrcrukpwdrong","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ryxtkrkqbpuveeawtzizwvayfeuufnel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crkylhaudjscuriplqwnhqpntqbwkbhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwxkbxxefpyhzhcuvlgvfyvdmqlgvlvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rguilaabzfnveyyvglnbnxezkebijbrf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtzkgxilskgonuanvxzutatmymmrbcmz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mokakwfmqngzhudkiowbykmbvpyuevzu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vipmpdfhamcjgkwdkhtxzgqyjqjpbhsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jayfpqiieylsfnrldrdkfnsmypumzpxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hozlzuequjheyulveqcoyppfukktbwsh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aazyucmxjffqdkgeepvnsioklgctfvgc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzgijglvvffsiinmhumaafjxqmuhmyoz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knnixxutxveuuuwqqnsskwabaeqqvbhp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edyfhwuihresiwobfmixshztrcgmbbav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667581,"databaseName":"models_schema","ddl":"CREATE TABLE `eniuwqlzawrgrhvrzrcpihxgkdvaarew` (\n `nxktsybuipbzqpcvolrdixlcswoivxxm` int NOT NULL,\n `qacnnjcklvkgjjzgxgxtmutpbttzgafp` int DEFAULT NULL,\n `oovpwbfpfkrbkvchrjkgsdtbudgiiejk` int DEFAULT NULL,\n `agswznblpwarjwisojzzlbudeutkjedw` int DEFAULT NULL,\n `vofcviaosibzaornvlvrapfcdmdutrfo` int DEFAULT NULL,\n `uefqqozvizadozgmbodfiotipgntclgb` int DEFAULT NULL,\n `rfblkvlxedbqcfrcswjmofcoudmcuquu` int DEFAULT NULL,\n `pdohtbhkxnqyssfxjfnqzwysgdiflxki` int DEFAULT NULL,\n `nukbkukijlwghnoimxxyulpqqxxxbknw` int DEFAULT NULL,\n `tbhifzgblwkmjgubqgagbmrohumbmekr` int DEFAULT NULL,\n `tambsgohinpitkncuskcltbrvbxrbfov` int DEFAULT NULL,\n `shyzwmkrikkesenygmqdcstwtzznilpq` int DEFAULT NULL,\n `igxqhenqwiqgkcgwrryywnruzicgrkvo` int DEFAULT NULL,\n `ncwctqaefjebtnlqqrdsrrulupzyoaji` int DEFAULT NULL,\n `atpgiqszbgtsnguivbneqhubeswxijrw` int DEFAULT NULL,\n `gstcppnpfdxplerykngqvyjnhfeqizix` int DEFAULT NULL,\n `gfetwgrllreghauzzcavzphrzgeikfsq` int DEFAULT NULL,\n `icmvdntncpjghqiehidwzpahwangolog` int DEFAULT NULL,\n `gsgzerzqnhokrxvpyrlsyhzznwgsidlp` int DEFAULT NULL,\n `rqdcyjjnbjicubmwjagppzudzutyaytt` int DEFAULT NULL,\n `ojcgfktaicmxbncopqanhxqqvobgwcuv` int DEFAULT NULL,\n `itpqvpcxqkgytvgnfewjjrkvejupchst` int DEFAULT NULL,\n `bfvdcplxzjaaokbdcorqrqqxryqdrzpp` int DEFAULT NULL,\n `lzwprtzjqqxglqqunwusaroacygmmesa` int DEFAULT NULL,\n `rygcksfeahehzovxwfakvlzvnvfrcosd` int DEFAULT NULL,\n `bjjxbihjtkdnkpwmxlfsakxnfglimosz` int DEFAULT NULL,\n `ujltxlvtycptmqmjxoyjxpkrhfvbtuon` int DEFAULT NULL,\n `klrxvrxwmoxhvekivxbcfvubaimlqwry` int DEFAULT NULL,\n `rikwqinbclwuaqegzqisazjelebrkysy` int DEFAULT NULL,\n `asaxjqrvylmbhxqzpfkqwisfqazsfdnu` int DEFAULT NULL,\n `bclxddsnhghicyaoeobakclfitvhvyva` int DEFAULT NULL,\n `akhcbnicqdyzlaagkmwnyyzemidvtelp` int DEFAULT NULL,\n `lgogrnezseheetvfztzecersmyipwlql` int DEFAULT NULL,\n `lmadvtwfpcqdabinegwrkfxgsqjwjzxm` int DEFAULT NULL,\n `hlcmkrbrcqoplzpkmlzqykggcakbgdqq` int DEFAULT NULL,\n `njecinifofmwduxcstkjseytnldrywlr` int DEFAULT NULL,\n `ggnwhzmafwienxjwurqbqvwewzpkauei` int DEFAULT NULL,\n `dmivniqnbwkmvkdngfuvlqqqjmdhtvlc` int DEFAULT NULL,\n `qxrepgjprrsppqcntvnkrjsmsrdpsdak` int DEFAULT NULL,\n `osuhiqfivphjupkzpuatuxlwhdmsknxj` int DEFAULT NULL,\n `tjytxdhobxeuoiryraqbbqxdntepxysf` int DEFAULT NULL,\n `kbdjthlfhogucjefrwujfzhgqclehtpd` int DEFAULT NULL,\n `ynkmbymnakbjigwifakvfwzqlasvpwup` int DEFAULT NULL,\n `rycjubhzkzvbzevxdplqcqoamaocbezj` int DEFAULT NULL,\n `nobqazuyygtmtnvtrlkjpqcwgcmliaab` int DEFAULT NULL,\n `wqniaqpkwdgfhukgxzloyuxelzhxglhm` int DEFAULT NULL,\n `teqdjfwedmwlvnuztughxasishgxxysg` int DEFAULT NULL,\n `jrpeckadcmhmxbiftgmgrwpnnaxpqqnk` int DEFAULT NULL,\n `tuvotgjshpvuwmnrlphqqslfuunmrrqz` int DEFAULT NULL,\n `pxpmfurvvuecejavqiaftbnbnhbxmwqu` int DEFAULT NULL,\n `pdicpqnfuspiuabmmtswlgwccpxltphb` int DEFAULT NULL,\n `zqpqunctbxmfmhxwaxklnhavdjuvnejw` int DEFAULT NULL,\n `cdixttznipkclcyklogiwdobxxvnwptr` int DEFAULT NULL,\n `pwzxhbqjnffwihvdnechtuydrszdfiol` int DEFAULT NULL,\n `sglnnmloxaorerqadnpqppltgtwxqtyw` int DEFAULT NULL,\n `clzypewznwxrullpgxrbmcuuldspkche` int DEFAULT NULL,\n `mteqdxynpnkmxbosfjihbmaacocftgce` int DEFAULT NULL,\n `uqbfbfkjckhuzkiwievxiwlsikbzbswk` int DEFAULT NULL,\n `pottfrgpwgbdwhyejhloygtkcwjykelf` int DEFAULT NULL,\n `acntsnmhnoscddrrpzcjcjqajjporlxg` int DEFAULT NULL,\n `yimdrcxdaxczlntgiqnimklcsepnvauy` int DEFAULT NULL,\n `zujphalviqusqyuwujmvzflemnupslxd` int DEFAULT NULL,\n `psfrsfvfkbwkutjmqgoluvzbxhczcmtl` int DEFAULT NULL,\n `nxsdpsyhpiqvqgxakaxiojwnirentlyr` int DEFAULT NULL,\n `fmcgfpgtspdsrozyutwvzipilcgnvjvp` int DEFAULT NULL,\n `skdqndggzrjzlscbzghjldlhrmqbmwww` int DEFAULT NULL,\n `spncxrwnpkhbpfuqtqwgolmocsgiqipv` int DEFAULT NULL,\n `zvilukdwxjvpdccmmnpnyznkhqcysach` int DEFAULT NULL,\n `kyqxgjvnifbzbgnulcrzdqbmecamlpbu` int DEFAULT NULL,\n `eqfplfvmwlsqirixbjtrjzsgxgkuwhij` int DEFAULT NULL,\n `kkudwwlprmulwblgalpegoqtexhfdhem` int DEFAULT NULL,\n `wnkzkkrzkzzqkibditobzgbulmgikyzf` int DEFAULT NULL,\n `ueqsczqzmpgahdaybjogmjrwxeellfwb` int DEFAULT NULL,\n `gxsbwkdgwdqhfxnawxukicdounnykexh` int DEFAULT NULL,\n `jrmshevieyrpqduoupvwffmjgtfklwaa` int DEFAULT NULL,\n `sjjkddchovpdjhevawtjhsvmecvdbykp` int DEFAULT NULL,\n `dvqdqjgvvrkjermiurcqomxqhlqumlfl` int DEFAULT NULL,\n `gsvzsvranqzlrunhcvbjuqebxajuspke` int DEFAULT NULL,\n `yesbacfzmarwxgnystnpbrnvatvvggpc` int DEFAULT NULL,\n `erxwixdimmmkgiagchcesxffakeogxyz` int DEFAULT NULL,\n `tqtvcxipfxxhinpkvqooztuiazmjurki` int DEFAULT NULL,\n `xtijumrgvatpmumgrbornerozygcupyg` int DEFAULT NULL,\n `locmvechkrfxacxryhjoktlawgwmbuzh` int DEFAULT NULL,\n `oumlpxukeiwzydpcqcnidfwjhzbhegov` int DEFAULT NULL,\n `rkmeouncyslhvmpmefyoimjalhplzqof` int DEFAULT NULL,\n `hnxjfhnrfmxxmulobgwyzhbhaknmbufw` int DEFAULT NULL,\n `oymeqpgocisvhzzmtppobgmpewfypxqp` int DEFAULT NULL,\n `tvenaubfycatuiqvxqvqvqrlbbljhgxg` int DEFAULT NULL,\n `tkfhmguxwbeiuijkhxvmpdjnerinarqm` int DEFAULT NULL,\n `criflldwkdinryfmzyzvdxjjswuqjpqu` int DEFAULT NULL,\n `meoxksgexffqmgiudjmloistkgtsrdzn` int DEFAULT NULL,\n `piqkaqdcluwtqfsjisjvbwizzdfvurca` int DEFAULT NULL,\n `yfqbkotpxskoajbolcmdtaklxhxcawoq` int DEFAULT NULL,\n `nnipsdbuirwifjxwgpefpktiynwspkbg` int DEFAULT NULL,\n `rjekfknikuhtntvdbvllqsmynxwwudft` int DEFAULT NULL,\n `ontfhgcnqbyoesbvrgqiicbqysafpmcc` int DEFAULT NULL,\n `eaylkdtzwcorwnsfjtwuveyanpxpvdat` int DEFAULT NULL,\n `mrktvukikbzzyvmxjfeniwxmzgtaekop` int DEFAULT NULL,\n `yjzkltodkzbsktjzgccuznhxjulgijmb` int DEFAULT NULL,\n `quckftlzianvlytfddgpivhyzarnabfe` int DEFAULT NULL,\n PRIMARY KEY (`nxktsybuipbzqpcvolrdixlcswoivxxm`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"eniuwqlzawrgrhvrzrcpihxgkdvaarew\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["nxktsybuipbzqpcvolrdixlcswoivxxm"],"columns":[{"name":"nxktsybuipbzqpcvolrdixlcswoivxxm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"qacnnjcklvkgjjzgxgxtmutpbttzgafp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oovpwbfpfkrbkvchrjkgsdtbudgiiejk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agswznblpwarjwisojzzlbudeutkjedw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vofcviaosibzaornvlvrapfcdmdutrfo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uefqqozvizadozgmbodfiotipgntclgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfblkvlxedbqcfrcswjmofcoudmcuquu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdohtbhkxnqyssfxjfnqzwysgdiflxki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nukbkukijlwghnoimxxyulpqqxxxbknw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbhifzgblwkmjgubqgagbmrohumbmekr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tambsgohinpitkncuskcltbrvbxrbfov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shyzwmkrikkesenygmqdcstwtzznilpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igxqhenqwiqgkcgwrryywnruzicgrkvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ncwctqaefjebtnlqqrdsrrulupzyoaji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atpgiqszbgtsnguivbneqhubeswxijrw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gstcppnpfdxplerykngqvyjnhfeqizix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfetwgrllreghauzzcavzphrzgeikfsq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icmvdntncpjghqiehidwzpahwangolog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsgzerzqnhokrxvpyrlsyhzznwgsidlp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqdcyjjnbjicubmwjagppzudzutyaytt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojcgfktaicmxbncopqanhxqqvobgwcuv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itpqvpcxqkgytvgnfewjjrkvejupchst","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bfvdcplxzjaaokbdcorqrqqxryqdrzpp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzwprtzjqqxglqqunwusaroacygmmesa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rygcksfeahehzovxwfakvlzvnvfrcosd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjjxbihjtkdnkpwmxlfsakxnfglimosz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujltxlvtycptmqmjxoyjxpkrhfvbtuon","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"klrxvrxwmoxhvekivxbcfvubaimlqwry","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rikwqinbclwuaqegzqisazjelebrkysy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"asaxjqrvylmbhxqzpfkqwisfqazsfdnu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bclxddsnhghicyaoeobakclfitvhvyva","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akhcbnicqdyzlaagkmwnyyzemidvtelp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgogrnezseheetvfztzecersmyipwlql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmadvtwfpcqdabinegwrkfxgsqjwjzxm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlcmkrbrcqoplzpkmlzqykggcakbgdqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njecinifofmwduxcstkjseytnldrywlr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggnwhzmafwienxjwurqbqvwewzpkauei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dmivniqnbwkmvkdngfuvlqqqjmdhtvlc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxrepgjprrsppqcntvnkrjsmsrdpsdak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"osuhiqfivphjupkzpuatuxlwhdmsknxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjytxdhobxeuoiryraqbbqxdntepxysf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kbdjthlfhogucjefrwujfzhgqclehtpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynkmbymnakbjigwifakvfwzqlasvpwup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rycjubhzkzvbzevxdplqcqoamaocbezj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nobqazuyygtmtnvtrlkjpqcwgcmliaab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqniaqpkwdgfhukgxzloyuxelzhxglhm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"teqdjfwedmwlvnuztughxasishgxxysg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jrpeckadcmhmxbiftgmgrwpnnaxpqqnk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tuvotgjshpvuwmnrlphqqslfuunmrrqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxpmfurvvuecejavqiaftbnbnhbxmwqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdicpqnfuspiuabmmtswlgwccpxltphb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqpqunctbxmfmhxwaxklnhavdjuvnejw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cdixttznipkclcyklogiwdobxxvnwptr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwzxhbqjnffwihvdnechtuydrszdfiol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sglnnmloxaorerqadnpqppltgtwxqtyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clzypewznwxrullpgxrbmcuuldspkche","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mteqdxynpnkmxbosfjihbmaacocftgce","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqbfbfkjckhuzkiwievxiwlsikbzbswk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pottfrgpwgbdwhyejhloygtkcwjykelf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acntsnmhnoscddrrpzcjcjqajjporlxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yimdrcxdaxczlntgiqnimklcsepnvauy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zujphalviqusqyuwujmvzflemnupslxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psfrsfvfkbwkutjmqgoluvzbxhczcmtl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxsdpsyhpiqvqgxakaxiojwnirentlyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmcgfpgtspdsrozyutwvzipilcgnvjvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skdqndggzrjzlscbzghjldlhrmqbmwww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"spncxrwnpkhbpfuqtqwgolmocsgiqipv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvilukdwxjvpdccmmnpnyznkhqcysach","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kyqxgjvnifbzbgnulcrzdqbmecamlpbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqfplfvmwlsqirixbjtrjzsgxgkuwhij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kkudwwlprmulwblgalpegoqtexhfdhem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnkzkkrzkzzqkibditobzgbulmgikyzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ueqsczqzmpgahdaybjogmjrwxeellfwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxsbwkdgwdqhfxnawxukicdounnykexh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jrmshevieyrpqduoupvwffmjgtfklwaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjjkddchovpdjhevawtjhsvmecvdbykp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvqdqjgvvrkjermiurcqomxqhlqumlfl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsvzsvranqzlrunhcvbjuqebxajuspke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yesbacfzmarwxgnystnpbrnvatvvggpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"erxwixdimmmkgiagchcesxffakeogxyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqtvcxipfxxhinpkvqooztuiazmjurki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xtijumrgvatpmumgrbornerozygcupyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"locmvechkrfxacxryhjoktlawgwmbuzh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oumlpxukeiwzydpcqcnidfwjhzbhegov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkmeouncyslhvmpmefyoimjalhplzqof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hnxjfhnrfmxxmulobgwyzhbhaknmbufw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oymeqpgocisvhzzmtppobgmpewfypxqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvenaubfycatuiqvxqvqvqrlbbljhgxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkfhmguxwbeiuijkhxvmpdjnerinarqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"criflldwkdinryfmzyzvdxjjswuqjpqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"meoxksgexffqmgiudjmloistkgtsrdzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"piqkaqdcluwtqfsjisjvbwizzdfvurca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfqbkotpxskoajbolcmdtaklxhxcawoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nnipsdbuirwifjxwgpefpktiynwspkbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjekfknikuhtntvdbvllqsmynxwwudft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ontfhgcnqbyoesbvrgqiicbqysafpmcc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eaylkdtzwcorwnsfjtwuveyanpxpvdat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrktvukikbzzyvmxjfeniwxmzgtaekop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjzkltodkzbsktjzgccuznhxjulgijmb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"quckftlzianvlytfddgpivhyzarnabfe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667611,"databaseName":"models_schema","ddl":"CREATE TABLE `feporuidwlohgvwbtiusejyfqsrumrac` (\n `wwxalanbgblyufpmpecqnceqvmwnjznn` int NOT NULL,\n `dwyqdcrlnacmcupfkjdqlzxqwwgbzkxw` int DEFAULT NULL,\n `cwvbfbjyyexsfjylnhpptkipbxmnwqbs` int DEFAULT NULL,\n `obaizzpqylsciakcczfyjuvxgwstzngq` int DEFAULT NULL,\n `rgyspxggcqhkfbczdictoaevjxiluhye` int DEFAULT NULL,\n `aygwkgmobalgsdvkjkgllbjjfjoopkzs` int DEFAULT NULL,\n `jmdsduhgxlxodzzoqkwgiuankznkstrj` int DEFAULT NULL,\n `rielhjrhceqstwraufgsmqxhfopijsop` int DEFAULT NULL,\n `ppgdhjansglalbokyzphcdrjawtiexcb` int DEFAULT NULL,\n `cyhijlodzdqtaivrzqqpxulxnazxiaeq` int DEFAULT NULL,\n `kntldnndjvwpepuiinuwbkobgdayquzi` int DEFAULT NULL,\n `bwvfbvyivlqrwhomivowocgnkgvwdhbm` int DEFAULT NULL,\n `scgzxyfiskpwxrbmkssqghnebivcjjcq` int DEFAULT NULL,\n `lkfkxgfpfqxupaeaxsvlyaxicmdokpzf` int DEFAULT NULL,\n `ztvkqsuyzdazxbqsunfnvrdftbhfituq` int DEFAULT NULL,\n `wejwrwtnllsohouqskzhswlgdnrbgtdv` int DEFAULT NULL,\n `nhwswembngqtszgbvhtnkvzldbwayult` int DEFAULT NULL,\n `fqgxvtznaomxnxtpzgdekfnewytzhykm` int DEFAULT NULL,\n `cvjazysogboyqfeugmfmmecbudmtihmb` int DEFAULT NULL,\n `zrigtrbiukmmbtjqpojoxnntrykkygar` int DEFAULT NULL,\n `jjxflqxqqzlhqntgzwnvasxvbjagjypx` int DEFAULT NULL,\n `cwaxqyzndmshjdifnimjnpokjtlcqwir` int DEFAULT NULL,\n `llgsboptfqhbepjdpfndbmxagytfnfjy` int DEFAULT NULL,\n `lrfjphevxtpffojotutusgxxqoqrwnwy` int DEFAULT NULL,\n `jcqxebsrglkbrwlpjeihvgfahhgqkotn` int DEFAULT NULL,\n `rsmrrbuqouamktgkayszioxncierznxe` int DEFAULT NULL,\n `ghlhcwdmmsmmvruclroszcrbsafzsdag` int DEFAULT NULL,\n `mtfwkdbdzjxywhkegapcetofibztejqm` int DEFAULT NULL,\n `sdgnhwdrxzisygmdqpjtgwubclgngrfa` int DEFAULT NULL,\n `wwsbfaxtreuavixwmwgrdeykadzxsgfg` int DEFAULT NULL,\n `icdzmhnvicoxwvbpuvexyeyepssmynlz` int DEFAULT NULL,\n `igmduqpwsufjhnjymaqlfsithsatxbmf` int DEFAULT NULL,\n `ngzojmhlgpswikmkgzzqxnitrflmmhdt` int DEFAULT NULL,\n `ujwjbmteouajlevqpezpavqowvivzxzt` int DEFAULT NULL,\n `vowtfznasjyyxphaocxszezmkpxmxdka` int DEFAULT NULL,\n `jxpgcmuxnmugoznzsjvfoldohnfjumqs` int DEFAULT NULL,\n `egsfncjpgkrflnjpsmwppuhkfxeaqtvb` int DEFAULT NULL,\n `uyucpswpoznzrqhczvhvmjkdevsuvpoz` int DEFAULT NULL,\n `dhtgzrrmlxwysrywxkwhpyraegosuhad` int DEFAULT NULL,\n `jbzsxtcthweogiwbumtgmiwfxnmojsif` int DEFAULT NULL,\n `ulrgjdhjwkggqakvthfjjrtfqmcizzji` int DEFAULT NULL,\n `xfeosrogwpxuvjssfukhulvrpvrqpvro` int DEFAULT NULL,\n `hiywlyxrumkfhjqrnxuqaismdiylsxqk` int DEFAULT NULL,\n `tpnfpbgdgywbocnjizzrmjorasylgopu` int DEFAULT NULL,\n `apvdcvebvhmgorelpqmtcldbhoahpxfj` int DEFAULT NULL,\n `hsnqzfttgdexpsvagveuoojbuvivibxy` int DEFAULT NULL,\n `tptozvhewwyiesphpgzwqydujpzinsru` int DEFAULT NULL,\n `pbcdlqckzrajrjzbdtqqvfwdwmnksoiq` int DEFAULT NULL,\n `eqzkvnjaykmsknortobzccvbvvqccykg` int DEFAULT NULL,\n `zpqnnhjreqphhxvyownleyhyuimdzftz` int DEFAULT NULL,\n `alnxbskhpivwsumkgepiirmyfbzectur` int DEFAULT NULL,\n `rlvivvngxdrmrrklotjnpgmtduuzljia` int DEFAULT NULL,\n `kfzyecatdavqnvitxtezurpegylgvlba` int DEFAULT NULL,\n `sahiznawetfnvbozraeqnzrxrhxxbwsc` int DEFAULT NULL,\n `huurcsgmihhchwledfaprmljwjjfyhog` int DEFAULT NULL,\n `lvfwqfkzawvtjhtoyrbshzsgmwzvhdck` int DEFAULT NULL,\n `dipljmbrcjbuyqhfergsgxttrmwycqbl` int DEFAULT NULL,\n `tggpdjaqsjvyjnxtipzdcbjjmsouywzq` int DEFAULT NULL,\n `mxoeahhzzelexlxyocezrnxfaxnbtqqt` int DEFAULT NULL,\n `ffwssswzqagpcjunirsrwisscastirlh` int DEFAULT NULL,\n `kjgshocighrhiqpkcwdkpfklercaskfs` int DEFAULT NULL,\n `fttwnelachbohpabuceerecvooishqzd` int DEFAULT NULL,\n `zmggqsoscxmrykxjvegvqgcutiakcvcu` int DEFAULT NULL,\n `jvsvjaksvrekysspioduhhtvsxybixgh` int DEFAULT NULL,\n `zhymyxqcdbfngusqldqbikkrftejhpdg` int DEFAULT NULL,\n `yjrnrehirlqvmwhekmpqbgmhtlylqmvb` int DEFAULT NULL,\n `euzstzwyfkeabsyfnkcstropslmnjqby` int DEFAULT NULL,\n `cggqulrxjdtlpzszvsmbfofrkqclxgrm` int DEFAULT NULL,\n `izcmeamxlvytywavnvhozfsauwtyumfe` int DEFAULT NULL,\n `lleriwualqnguikwifbwxcunrleqwjdc` int DEFAULT NULL,\n `pzddzuwbhtogvukjvfdmkdxoewvccsrp` int DEFAULT NULL,\n `ehvkcdoprjgmzrhymujtqoouvsfrehbx` int DEFAULT NULL,\n `jyujmpuuldlhabukoddvudvatoeomner` int DEFAULT NULL,\n `fivxfhumqctnqvrqkwejyucqskhylmqs` int DEFAULT NULL,\n `chcscjezdckeuwcntfmwwugyqmxpvsqn` int DEFAULT NULL,\n `fdlrxrcdsijujzxhqsqmiqqayjkhbotz` int DEFAULT NULL,\n `oycpqabfkltqxqhyxfvskinzipymeqlg` int DEFAULT NULL,\n `zmshtezdoljcdzndyihaielvpgzlqzqn` int DEFAULT NULL,\n `cbablnxtqjiohylzvtqdkixwnbozclou` int DEFAULT NULL,\n `zzmcyjzvsbvsdkhdsoxiioxpakucgecg` int DEFAULT NULL,\n `gkameqpyaprhvzujyuhmlwhmjdudspfe` int DEFAULT NULL,\n `jlvzvgpoakdlxdicdnfyoevocplwyrgb` int DEFAULT NULL,\n `wplbhpxyggtztkewrrxnmyxzgksoaooy` int DEFAULT NULL,\n `dxrpkeolzewdzpmwbmvlwkbbcfxpeqyq` int DEFAULT NULL,\n `lxohcvkhtdmohgesqvtjtdgyvaajyekt` int DEFAULT NULL,\n `umkkgtbxeewykuzqbpggpvjsmggdjpll` int DEFAULT NULL,\n `mvjanikhsiksdjujkxcnfugzvsmnjuwo` int DEFAULT NULL,\n `ztffzkhqoyypzmgdyhkkzspmaotcuxzt` int DEFAULT NULL,\n `ucvxvotigtzsfjmiuneqzrlulipprpap` int DEFAULT NULL,\n `usizgczsjzgjmuamjkgkijwxppjtajjh` int DEFAULT NULL,\n `llidmlckricmvtwvdlgkcprjvrwihsdq` int DEFAULT NULL,\n `tzsfeokbmkzmlibbtiakbsrerxtsnvon` int DEFAULT NULL,\n `hdrbzpngupobrlmnpujbyyptuclfvjka` int DEFAULT NULL,\n `gohyhyirjndlbqhrjqafbrgnaevnuhie` int DEFAULT NULL,\n `hlhapjvviyeozaumotflrrwtkxvbybrm` int DEFAULT NULL,\n `xrmbftlsgdzjezbytchysluigbhscerl` int DEFAULT NULL,\n `bktcxqftdxhvobejkkcqwyotkmgkwjkg` int DEFAULT NULL,\n `fcqemvbsnalcwhieohzctjzxrfrqxgxu` int DEFAULT NULL,\n `cbknzddtsoflxjzrtkerltjfgvfapsmb` int DEFAULT NULL,\n `dafcesykhuusrlpbcsgqicormkpltfsx` int DEFAULT NULL,\n PRIMARY KEY (`wwxalanbgblyufpmpecqnceqvmwnjznn`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"feporuidwlohgvwbtiusejyfqsrumrac\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["wwxalanbgblyufpmpecqnceqvmwnjznn"],"columns":[{"name":"wwxalanbgblyufpmpecqnceqvmwnjznn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"dwyqdcrlnacmcupfkjdqlzxqwwgbzkxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwvbfbjyyexsfjylnhpptkipbxmnwqbs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obaizzpqylsciakcczfyjuvxgwstzngq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgyspxggcqhkfbczdictoaevjxiluhye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aygwkgmobalgsdvkjkgllbjjfjoopkzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmdsduhgxlxodzzoqkwgiuankznkstrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rielhjrhceqstwraufgsmqxhfopijsop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ppgdhjansglalbokyzphcdrjawtiexcb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cyhijlodzdqtaivrzqqpxulxnazxiaeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kntldnndjvwpepuiinuwbkobgdayquzi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bwvfbvyivlqrwhomivowocgnkgvwdhbm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scgzxyfiskpwxrbmkssqghnebivcjjcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lkfkxgfpfqxupaeaxsvlyaxicmdokpzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ztvkqsuyzdazxbqsunfnvrdftbhfituq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wejwrwtnllsohouqskzhswlgdnrbgtdv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhwswembngqtszgbvhtnkvzldbwayult","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqgxvtznaomxnxtpzgdekfnewytzhykm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvjazysogboyqfeugmfmmecbudmtihmb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zrigtrbiukmmbtjqpojoxnntrykkygar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjxflqxqqzlhqntgzwnvasxvbjagjypx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwaxqyzndmshjdifnimjnpokjtlcqwir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llgsboptfqhbepjdpfndbmxagytfnfjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lrfjphevxtpffojotutusgxxqoqrwnwy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jcqxebsrglkbrwlpjeihvgfahhgqkotn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rsmrrbuqouamktgkayszioxncierznxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghlhcwdmmsmmvruclroszcrbsafzsdag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtfwkdbdzjxywhkegapcetofibztejqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdgnhwdrxzisygmdqpjtgwubclgngrfa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwsbfaxtreuavixwmwgrdeykadzxsgfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icdzmhnvicoxwvbpuvexyeyepssmynlz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igmduqpwsufjhnjymaqlfsithsatxbmf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ngzojmhlgpswikmkgzzqxnitrflmmhdt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujwjbmteouajlevqpezpavqowvivzxzt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vowtfznasjyyxphaocxszezmkpxmxdka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jxpgcmuxnmugoznzsjvfoldohnfjumqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"egsfncjpgkrflnjpsmwppuhkfxeaqtvb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uyucpswpoznzrqhczvhvmjkdevsuvpoz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhtgzrrmlxwysrywxkwhpyraegosuhad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbzsxtcthweogiwbumtgmiwfxnmojsif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ulrgjdhjwkggqakvthfjjrtfqmcizzji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xfeosrogwpxuvjssfukhulvrpvrqpvro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hiywlyxrumkfhjqrnxuqaismdiylsxqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpnfpbgdgywbocnjizzrmjorasylgopu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"apvdcvebvhmgorelpqmtcldbhoahpxfj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hsnqzfttgdexpsvagveuoojbuvivibxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tptozvhewwyiesphpgzwqydujpzinsru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbcdlqckzrajrjzbdtqqvfwdwmnksoiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqzkvnjaykmsknortobzccvbvvqccykg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpqnnhjreqphhxvyownleyhyuimdzftz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"alnxbskhpivwsumkgepiirmyfbzectur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlvivvngxdrmrrklotjnpgmtduuzljia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfzyecatdavqnvitxtezurpegylgvlba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sahiznawetfnvbozraeqnzrxrhxxbwsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"huurcsgmihhchwledfaprmljwjjfyhog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvfwqfkzawvtjhtoyrbshzsgmwzvhdck","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dipljmbrcjbuyqhfergsgxttrmwycqbl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tggpdjaqsjvyjnxtipzdcbjjmsouywzq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mxoeahhzzelexlxyocezrnxfaxnbtqqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffwssswzqagpcjunirsrwisscastirlh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjgshocighrhiqpkcwdkpfklercaskfs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fttwnelachbohpabuceerecvooishqzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmggqsoscxmrykxjvegvqgcutiakcvcu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvsvjaksvrekysspioduhhtvsxybixgh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhymyxqcdbfngusqldqbikkrftejhpdg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjrnrehirlqvmwhekmpqbgmhtlylqmvb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"euzstzwyfkeabsyfnkcstropslmnjqby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cggqulrxjdtlpzszvsmbfofrkqclxgrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"izcmeamxlvytywavnvhozfsauwtyumfe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lleriwualqnguikwifbwxcunrleqwjdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pzddzuwbhtogvukjvfdmkdxoewvccsrp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehvkcdoprjgmzrhymujtqoouvsfrehbx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jyujmpuuldlhabukoddvudvatoeomner","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fivxfhumqctnqvrqkwejyucqskhylmqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chcscjezdckeuwcntfmwwugyqmxpvsqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fdlrxrcdsijujzxhqsqmiqqayjkhbotz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oycpqabfkltqxqhyxfvskinzipymeqlg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmshtezdoljcdzndyihaielvpgzlqzqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbablnxtqjiohylzvtqdkixwnbozclou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzmcyjzvsbvsdkhdsoxiioxpakucgecg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkameqpyaprhvzujyuhmlwhmjdudspfe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlvzvgpoakdlxdicdnfyoevocplwyrgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wplbhpxyggtztkewrrxnmyxzgksoaooy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxrpkeolzewdzpmwbmvlwkbbcfxpeqyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxohcvkhtdmohgesqvtjtdgyvaajyekt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umkkgtbxeewykuzqbpggpvjsmggdjpll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvjanikhsiksdjujkxcnfugzvsmnjuwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ztffzkhqoyypzmgdyhkkzspmaotcuxzt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucvxvotigtzsfjmiuneqzrlulipprpap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"usizgczsjzgjmuamjkgkijwxppjtajjh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llidmlckricmvtwvdlgkcprjvrwihsdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tzsfeokbmkzmlibbtiakbsrerxtsnvon","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdrbzpngupobrlmnpujbyyptuclfvjka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gohyhyirjndlbqhrjqafbrgnaevnuhie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlhapjvviyeozaumotflrrwtkxvbybrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrmbftlsgdzjezbytchysluigbhscerl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bktcxqftdxhvobejkkcqwyotkmgkwjkg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcqemvbsnalcwhieohzctjzxrfrqxgxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbknzddtsoflxjzrtkerltjfgvfapsmb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dafcesykhuusrlpbcsgqicormkpltfsx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667648,"databaseName":"models_schema","ddl":"CREATE TABLE `fhwposxtdpufqjciekvsudvzcdupkqjv` (\n `aqdntyglbblyaymnjoodjknxqvpkfbbl` int NOT NULL,\n `jyjjpgmdllelhnmkmltthxdaauiwfbuj` int DEFAULT NULL,\n `adqelrmtaycrlvpmtvxavlrgrijpupuh` int DEFAULT NULL,\n `dhqfanenlxcdgebwmkjhoisztvibyyxo` int DEFAULT NULL,\n `zvlwowtymkxkvmyavthzymsaawoccyrm` int DEFAULT NULL,\n `xztszflgncbqjmupgghslznsollhctxo` int DEFAULT NULL,\n `jqqusnjelcidlbzorpqeisjymudqwtdb` int DEFAULT NULL,\n `amexctkqvaltmtcgnwlsquburljvdnvz` int DEFAULT NULL,\n `emzijfublqvwuncitbmrijzbqkyrazyt` int DEFAULT NULL,\n `exozeaqytemtcqqwgikkxrewgiwtswru` int DEFAULT NULL,\n `pbdpcnjlvmzsmiaswchljsvyeykmzfjo` int DEFAULT NULL,\n `yyngshochjtsupizsdshyzfzwsxgwuak` int DEFAULT NULL,\n `kjeghsehsqbswyjepbnqcoaayfonzqbz` int DEFAULT NULL,\n `zqvwkgwtcmusqklceavuixokhrmntksu` int DEFAULT NULL,\n `rbspkenigwznlkywaomizzehraucikuf` int DEFAULT NULL,\n `qxprikvdwqmtnqskaqezflbjzucntnil` int DEFAULT NULL,\n `kabjmwelpfhtsqcedldecugvckypclhn` int DEFAULT NULL,\n `yiqrgosbepsmonfbxjybggvywhiqrrgv` int DEFAULT NULL,\n `qkoltrmjzsonhtpdadxkppkscmdmqnta` int DEFAULT NULL,\n `fmveghzlcxajyqbqsvpotdxqirjliooa` int DEFAULT NULL,\n `krpvokamlthuzqdbpogjdhfzippzaezy` int DEFAULT NULL,\n `tffdbaduzxqhkfbpibfuxefjvxdkahez` int DEFAULT NULL,\n `mvbwyqafbotkgwheksprvcewjxmmfmse` int DEFAULT NULL,\n `ufegxzsdywcrojuwawuqwolovetlhozn` int DEFAULT NULL,\n `igkkmflbvsrdytqyxcalftqfagyfiitz` int DEFAULT NULL,\n `skmfjfdbyhmhbqemgqalnlltubriqekl` int DEFAULT NULL,\n `dhwibjbtvoyfpbniutytivljmpekcdgb` int DEFAULT NULL,\n `rkyzwfufhngowfokvnwqbttzgbiodayk` int DEFAULT NULL,\n `gfazecjdsbgffajhkerchguzqqmfjung` int DEFAULT NULL,\n `myjppyfbkexcjlqfhueanmpfbkonfpfu` int DEFAULT NULL,\n `bccbndqanjtnnwudyaebmfmtuyggscrv` int DEFAULT NULL,\n `hgmdiahhxjhvbljdorfszwotwmzprhxk` int DEFAULT NULL,\n `gfsqcidhiirfztewbzwphoafaucnhtlf` int DEFAULT NULL,\n `hstpcnahswsmkhfkgzxoouliflipajxb` int DEFAULT NULL,\n `fxakxtdftuatjapwmgqjnfumxiwwakdm` int DEFAULT NULL,\n `nyicffztxsxugyqvkakrziceiaudmefr` int DEFAULT NULL,\n `ojssuraksgvszonswsysakyibjciqrhk` int DEFAULT NULL,\n `pjudkvtdqoqnlvelkyympkjqlhloachl` int DEFAULT NULL,\n `prfuorftjrvjaajgnonenvtwelipyflp` int DEFAULT NULL,\n `shgggrjgfqfueynbsdtabavdfwjzmxtx` int DEFAULT NULL,\n `npbpehkznfcxpewwrwzmbtwtdwcuqinn` int DEFAULT NULL,\n `zhtyckesfmwpbkeajtfwigedfchzthil` int DEFAULT NULL,\n `qvucbknpqhgpfehaiilpypyitzxkiwbc` int DEFAULT NULL,\n `euejpjhmtiykxeamktszwtenxocdkbvr` int DEFAULT NULL,\n `xjqxxqutyfdarwpgzuybzofrqjsgnncp` int DEFAULT NULL,\n `ihtqergrftpjnwzwpzkcvzsuhskxnmoy` int DEFAULT NULL,\n `ohljwqqanzpltivvuwzgatgcovcnbgma` int DEFAULT NULL,\n `uuvumhutwsnbruuxzdycjbunkgznoipl` int DEFAULT NULL,\n `pyiyqepshlbxmopvtgpdqliqosghppna` int DEFAULT NULL,\n `jvcprbppbjqnskppdulcproqorfnccra` int DEFAULT NULL,\n `flakzipdanapvjfbtecrusradvvbtzss` int DEFAULT NULL,\n `uricxxaavcotbnfttllxkbedxkomxwdu` int DEFAULT NULL,\n `qpnipwtysgwjxmpikyxulddqqnpfvgbp` int DEFAULT NULL,\n `boorrrdkpttmjdyawnowfnhvdahhpkea` int DEFAULT NULL,\n `rrynrwaldigqswuypfxzgsjgxbuhfyqx` int DEFAULT NULL,\n `xxalzpnludszlulucwcjyfbywprokuln` int DEFAULT NULL,\n `sfzawwxiyvhlzyhsvgpkypzmcxketrgy` int DEFAULT NULL,\n `qbscdvcmqonqjwhmdtcczneqncaqtkny` int DEFAULT NULL,\n `emozcalhykodobzwyulyhgaosqukpisg` int DEFAULT NULL,\n `jzlclvfanqcqwoimrqdviblwjaypuloc` int DEFAULT NULL,\n `wuqrfjdczfmdlubpukwzrfufmkpgpkca` int DEFAULT NULL,\n `jpusrynkuijaeoleonbuyofuzjitrtql` int DEFAULT NULL,\n `zntoepntvejzocphyyiiprczctdxisvc` int DEFAULT NULL,\n `iwwknmimrogmghtdurxbstlaqjrdzbij` int DEFAULT NULL,\n `yngzatardlqjkzcppzusgvzjfqmxptrx` int DEFAULT NULL,\n `ygjnpmaslkjpszcbvvgjpljwuouaqutg` int DEFAULT NULL,\n `hudcmapozrvmmdiclazthkbfwduefmgg` int DEFAULT NULL,\n `iikeyffajbqdrdyztparaxfkfzxszwfv` int DEFAULT NULL,\n `jmvvjubwkbvzzodpccbigocgccyhxoar` int DEFAULT NULL,\n `cptypaytdvivgfmbhdljwwawqknbhixr` int DEFAULT NULL,\n `syohvzpmsxqwmfysgrwrfhbhuupcehdj` int DEFAULT NULL,\n `ixxvmvznqlkfktiekiixclljnedehunm` int DEFAULT NULL,\n `mbvfwdotdlfwgbdlraooffopkghypugl` int DEFAULT NULL,\n `fpbxwphonuzfdxkncnwphxafhhmmvhij` int DEFAULT NULL,\n `msciimygusboohzayveztzxroawcfynh` int DEFAULT NULL,\n `wdobojyvpoygrserlgbmxwdaregocvxu` int DEFAULT NULL,\n `wrnuyzbcymguvcrxilhikceydqlabasn` int DEFAULT NULL,\n `uswzvomgckmqtekcxqrmairdiyrvbvdf` int DEFAULT NULL,\n `aoydvkwdsjdrqyrsgkpcmkbdiulgneri` int DEFAULT NULL,\n `uezmnxumlwjzgkpojeotdcgwyczdgaso` int DEFAULT NULL,\n `lgrlgqrrakqogrqnpqvfgcqcyxahivqw` int DEFAULT NULL,\n `kvfvcvilcxmkgmysnjynvdoidjuqcmyr` int DEFAULT NULL,\n `ukurinqutwiyvbgdooxfudxnmquhnvic` int DEFAULT NULL,\n `uzvziuppugwedzzxdpdegxhoflwlxiqc` int DEFAULT NULL,\n `nscuyhpglhazdnfsdfchypmmegpmruso` int DEFAULT NULL,\n `emffclsabwxrmfdavgxpyvqsunwkszod` int DEFAULT NULL,\n `pasvypxsaiyovyztukkotudfiwepgxmn` int DEFAULT NULL,\n `lmscugmcrmiyrrooomdbfrrduhgauwzd` int DEFAULT NULL,\n `adpeojrruuurfomapubetonnmyjxeoxf` int DEFAULT NULL,\n `zuyptxggoxheyzjjvbczswqnzwwyddeu` int DEFAULT NULL,\n `zbfrxoshevdqmofrgzmcfdukkpoenxoa` int DEFAULT NULL,\n `klxrjuungxfvjsivzkwatdunvgouccnq` int DEFAULT NULL,\n `wzbbkxrarhfrejguvtmgptccimuecbdy` int DEFAULT NULL,\n `mcmvrcddgfwtoaqphpitqrtkthmmxmix` int DEFAULT NULL,\n `phgrjyqlcixbzmdqbryoderfzxgctobe` int DEFAULT NULL,\n `hhoutqqzbeynvbwpwlwtbrfhylybqhze` int DEFAULT NULL,\n `cyhgfnbubunvsmbwrxuylsfnmvwbavxs` int DEFAULT NULL,\n `yembdscskvxaplbplhdsdtyymdkmnrcp` int DEFAULT NULL,\n `qccnnuirqlvpugvojjgqwadmmomyjesv` int DEFAULT NULL,\n `vaaziqhtdermubezxnzubtvhgtifsnot` int DEFAULT NULL,\n PRIMARY KEY (`aqdntyglbblyaymnjoodjknxqvpkfbbl`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"fhwposxtdpufqjciekvsudvzcdupkqjv\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["aqdntyglbblyaymnjoodjknxqvpkfbbl"],"columns":[{"name":"aqdntyglbblyaymnjoodjknxqvpkfbbl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"jyjjpgmdllelhnmkmltthxdaauiwfbuj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"adqelrmtaycrlvpmtvxavlrgrijpupuh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhqfanenlxcdgebwmkjhoisztvibyyxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvlwowtymkxkvmyavthzymsaawoccyrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xztszflgncbqjmupgghslznsollhctxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jqqusnjelcidlbzorpqeisjymudqwtdb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amexctkqvaltmtcgnwlsquburljvdnvz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"emzijfublqvwuncitbmrijzbqkyrazyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exozeaqytemtcqqwgikkxrewgiwtswru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbdpcnjlvmzsmiaswchljsvyeykmzfjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yyngshochjtsupizsdshyzfzwsxgwuak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjeghsehsqbswyjepbnqcoaayfonzqbz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqvwkgwtcmusqklceavuixokhrmntksu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbspkenigwznlkywaomizzehraucikuf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxprikvdwqmtnqskaqezflbjzucntnil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kabjmwelpfhtsqcedldecugvckypclhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yiqrgosbepsmonfbxjybggvywhiqrrgv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qkoltrmjzsonhtpdadxkppkscmdmqnta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmveghzlcxajyqbqsvpotdxqirjliooa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krpvokamlthuzqdbpogjdhfzippzaezy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tffdbaduzxqhkfbpibfuxefjvxdkahez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvbwyqafbotkgwheksprvcewjxmmfmse","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufegxzsdywcrojuwawuqwolovetlhozn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igkkmflbvsrdytqyxcalftqfagyfiitz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skmfjfdbyhmhbqemgqalnlltubriqekl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhwibjbtvoyfpbniutytivljmpekcdgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkyzwfufhngowfokvnwqbttzgbiodayk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfazecjdsbgffajhkerchguzqqmfjung","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"myjppyfbkexcjlqfhueanmpfbkonfpfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bccbndqanjtnnwudyaebmfmtuyggscrv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hgmdiahhxjhvbljdorfszwotwmzprhxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfsqcidhiirfztewbzwphoafaucnhtlf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hstpcnahswsmkhfkgzxoouliflipajxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxakxtdftuatjapwmgqjnfumxiwwakdm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nyicffztxsxugyqvkakrziceiaudmefr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojssuraksgvszonswsysakyibjciqrhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjudkvtdqoqnlvelkyympkjqlhloachl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prfuorftjrvjaajgnonenvtwelipyflp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shgggrjgfqfueynbsdtabavdfwjzmxtx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npbpehkznfcxpewwrwzmbtwtdwcuqinn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhtyckesfmwpbkeajtfwigedfchzthil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvucbknpqhgpfehaiilpypyitzxkiwbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"euejpjhmtiykxeamktszwtenxocdkbvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjqxxqutyfdarwpgzuybzofrqjsgnncp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ihtqergrftpjnwzwpzkcvzsuhskxnmoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohljwqqanzpltivvuwzgatgcovcnbgma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uuvumhutwsnbruuxzdycjbunkgznoipl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pyiyqepshlbxmopvtgpdqliqosghppna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvcprbppbjqnskppdulcproqorfnccra","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flakzipdanapvjfbtecrusradvvbtzss","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uricxxaavcotbnfttllxkbedxkomxwdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qpnipwtysgwjxmpikyxulddqqnpfvgbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"boorrrdkpttmjdyawnowfnhvdahhpkea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrynrwaldigqswuypfxzgsjgxbuhfyqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxalzpnludszlulucwcjyfbywprokuln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfzawwxiyvhlzyhsvgpkypzmcxketrgy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qbscdvcmqonqjwhmdtcczneqncaqtkny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"emozcalhykodobzwyulyhgaosqukpisg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzlclvfanqcqwoimrqdviblwjaypuloc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wuqrfjdczfmdlubpukwzrfufmkpgpkca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpusrynkuijaeoleonbuyofuzjitrtql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zntoepntvejzocphyyiiprczctdxisvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwwknmimrogmghtdurxbstlaqjrdzbij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yngzatardlqjkzcppzusgvzjfqmxptrx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ygjnpmaslkjpszcbvvgjpljwuouaqutg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hudcmapozrvmmdiclazthkbfwduefmgg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iikeyffajbqdrdyztparaxfkfzxszwfv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmvvjubwkbvzzodpccbigocgccyhxoar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cptypaytdvivgfmbhdljwwawqknbhixr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"syohvzpmsxqwmfysgrwrfhbhuupcehdj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixxvmvznqlkfktiekiixclljnedehunm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbvfwdotdlfwgbdlraooffopkghypugl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpbxwphonuzfdxkncnwphxafhhmmvhij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"msciimygusboohzayveztzxroawcfynh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wdobojyvpoygrserlgbmxwdaregocvxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wrnuyzbcymguvcrxilhikceydqlabasn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uswzvomgckmqtekcxqrmairdiyrvbvdf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aoydvkwdsjdrqyrsgkpcmkbdiulgneri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uezmnxumlwjzgkpojeotdcgwyczdgaso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgrlgqrrakqogrqnpqvfgcqcyxahivqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvfvcvilcxmkgmysnjynvdoidjuqcmyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ukurinqutwiyvbgdooxfudxnmquhnvic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzvziuppugwedzzxdpdegxhoflwlxiqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nscuyhpglhazdnfsdfchypmmegpmruso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"emffclsabwxrmfdavgxpyvqsunwkszod","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pasvypxsaiyovyztukkotudfiwepgxmn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmscugmcrmiyrrooomdbfrrduhgauwzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"adpeojrruuurfomapubetonnmyjxeoxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zuyptxggoxheyzjjvbczswqnzwwyddeu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbfrxoshevdqmofrgzmcfdukkpoenxoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"klxrjuungxfvjsivzkwatdunvgouccnq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzbbkxrarhfrejguvtmgptccimuecbdy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcmvrcddgfwtoaqphpitqrtkthmmxmix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phgrjyqlcixbzmdqbryoderfzxgctobe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhoutqqzbeynvbwpwlwtbrfhylybqhze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cyhgfnbubunvsmbwrxuylsfnmvwbavxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yembdscskvxaplbplhdsdtyymdkmnrcp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qccnnuirqlvpugvojjgqwadmmomyjesv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vaaziqhtdermubezxnzubtvhgtifsnot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667696,"databaseName":"models_schema","ddl":"CREATE TABLE `fknkmyhzgzdgbklxmknjahvidhearmkn` (\n `ekgcpnfkygusvdjrukozimpyqlrvbxyr` int NOT NULL,\n `qacrwzpcacdgkbgkquqaumdcplgoccdj` int DEFAULT NULL,\n `vommluqcaexjrnhhctpsdzzbnnpetgdz` int DEFAULT NULL,\n `wnoshwvsjbrrmbjrvnaztzmjfmdvshgx` int DEFAULT NULL,\n `zvuxjbxnpnsmagqnfddubifiqsoxzbqs` int DEFAULT NULL,\n `ulxldkddhgfummrsosdglapvwfyfjntr` int DEFAULT NULL,\n `iiuwpwefoydvpvetvaubzrlgaryjoobm` int DEFAULT NULL,\n `hlastibbugpmnbszsfobtwwlhrkuhauk` int DEFAULT NULL,\n `ixokjwumbddbsiggwxovapehwvypqqpv` int DEFAULT NULL,\n `pvqlwtguxtxptgknlwierqmtcutqplnn` int DEFAULT NULL,\n `kgrlxmgofbglrranyanoojwlcykitbla` int DEFAULT NULL,\n `wkzetmlairtyjeafvgsitrbwavtlrahh` int DEFAULT NULL,\n `lemkbosumngjqrytphzuqeiickagnmmv` int DEFAULT NULL,\n `pmbbfvesrsoehcnakejlcphebwaxvvtn` int DEFAULT NULL,\n `pitcvkssqhdhsjzazuckupuwuhrjxpvv` int DEFAULT NULL,\n `sbkounebpieiiamjayzijkommrnmsxot` int DEFAULT NULL,\n `hyxmnbjwleabkpidtkzihrkiimzvzwrm` int DEFAULT NULL,\n `naysplzfdigbmqnyaeffldeypyrpzuyw` int DEFAULT NULL,\n `fefjommkypjyyximknfiuehpekarcwum` int DEFAULT NULL,\n `usjfukccvsbcnrafuwoswejioyflyxlh` int DEFAULT NULL,\n `rfsetdzskekuxwuwpobgxgpsjoqgceoa` int DEFAULT NULL,\n `egkjwwzjkwxcsowxwjbbdankjdxfyifn` int DEFAULT NULL,\n `mvxlowvaivnfpbtjjkwkzgtebztlvdov` int DEFAULT NULL,\n `wmzsudxdnlehoglckvzuwcqxmznkfvdi` int DEFAULT NULL,\n `tnfhjyzufjxgagyrxtzbexykmnrdteqs` int DEFAULT NULL,\n `gocmkvudlaushszgwevbjtwbdxjjsigq` int DEFAULT NULL,\n `aggloyxsdmhuhkotyhuyqwjmsddsbtui` int DEFAULT NULL,\n `pvjmmnbcvlvubqzdlwkklnnjhzzbodrc` int DEFAULT NULL,\n `spmpszoebnmulpoljjozwxdtnbhomimc` int DEFAULT NULL,\n `kgtuohourgvqbucgrjzigkhrcsqulxxh` int DEFAULT NULL,\n `rfcqlndsnytlscdgaipkswnsaoznpyva` int DEFAULT NULL,\n `pxxsuirazvfcxmxirxeocgexrpocktbj` int DEFAULT NULL,\n `vytgkqeydfxiydtkthushhumqwjrqnkv` int DEFAULT NULL,\n `vysjgejbayejtqyiwgglbgsoiolkydzb` int DEFAULT NULL,\n `pflthoeccyopdsyfbyefauxvyziiofsw` int DEFAULT NULL,\n `mrwzvrvrujdhjoxxpcawgveebpngdroi` int DEFAULT NULL,\n `yixjnhthaiheuichkqzitsvbncxcivld` int DEFAULT NULL,\n `corpbqlwgfooceyjwaanwarbhouqomhy` int DEFAULT NULL,\n `ubznykidofjbdppbjuplrukbiguyisee` int DEFAULT NULL,\n `ohvonylduqevmylygovxynjbtvjmwijh` int DEFAULT NULL,\n `koytlmatcsckcoalkldyqopqfqeznlcc` int DEFAULT NULL,\n `ugrnxyvmatcvvjeunbdhnedvcbhtvxui` int DEFAULT NULL,\n `lrdmegibwzbpahltofyvqnspiwgzlmsh` int DEFAULT NULL,\n `qffbggzytfmrbqhiyldejhdwpzswyciv` int DEFAULT NULL,\n `vcymxnygfgcjpszqolckifzqdgwymyux` int DEFAULT NULL,\n `ndqjsehzixarmududqtnynfyxvadfrcd` int DEFAULT NULL,\n `sodykgjfdgixedxwxjapmkqdczphgsgs` int DEFAULT NULL,\n `njxugwgqztjjluslbfqhmsidlpxuoebr` int DEFAULT NULL,\n `mwlaxnslfgpbssvxbspmxjwtazageerz` int DEFAULT NULL,\n `uwbhbmqwzfryybkjsternyvxgwvzkedu` int DEFAULT NULL,\n `fwlwqkmtdvlqlmbgtnfraxlvqbnbunuh` int DEFAULT NULL,\n `taatllbcgiglhpdetyhrhzcgqupcxdnp` int DEFAULT NULL,\n `zffyrvqblezhjmfsfzkqwowzgkqusguy` int DEFAULT NULL,\n `eovjzfuugwhnfkvkioxesletiuhxwkqr` int DEFAULT NULL,\n `wsiqwkdvrbqdtezpscsdimnahbykmbol` int DEFAULT NULL,\n `nmkttymgiksykwuuzdgxyxufuhdhjzcm` int DEFAULT NULL,\n `rqupxvourwttsxxpzirxlchoiooifnps` int DEFAULT NULL,\n `rnumzwwkozfcrndyubnbkotzrfboukoo` int DEFAULT NULL,\n `mglmkqeffmhjvjdjbueiipldbtasooph` int DEFAULT NULL,\n `zztjyxyxmjdochnkdwsshubpjafodpns` int DEFAULT NULL,\n `uounljusacmeuezowjsjvnpyvwankcyu` int DEFAULT NULL,\n `cakrutzjzyitgamrdjyxktesypienzct` int DEFAULT NULL,\n `yqtknnbkrfmhyxzdbkfsewrwikearydv` int DEFAULT NULL,\n `kasqjsgimdkcggiiqbsgpwtbcdeyuile` int DEFAULT NULL,\n `yszspomchhrhjajlralqxanhgpodiuvy` int DEFAULT NULL,\n `erceugixynqbwnbojposaaapapryzhdl` int DEFAULT NULL,\n `shucvwtsxwomxuolnofmhyswjnrvcjsv` int DEFAULT NULL,\n `zhpwcyzksbubbxkkgpkzxaelcbiisxyd` int DEFAULT NULL,\n `djpswwspwdnpwajmlhhlpxhqvngfciri` int DEFAULT NULL,\n `robcykqtvycgbiisicenswyycpnvrkxn` int DEFAULT NULL,\n `mkbtkqzbrzqcpxdfzkqolbtovpmsklew` int DEFAULT NULL,\n `hrqkouwagwmuunfqgkxcwmvfczpoabom` int DEFAULT NULL,\n `wwwagrntgmvlmhrllypdzccircdapggo` int DEFAULT NULL,\n `ffvqwrvgpqtmqtijmybtmjmsfmxmisug` int DEFAULT NULL,\n `mfijnvgqovevutzfpjfrmuwmkxkgtvmo` int DEFAULT NULL,\n `ftpagyvphzgorqvfjvubauosbormcjvk` int DEFAULT NULL,\n `msgrexapcpigvmcrimnopacwmzjweral` int DEFAULT NULL,\n `eoyywmibywdpeyohbokpzknmcjmlmzrh` int DEFAULT NULL,\n `luegpfsgrvgcebzoojjbidatrquvpeoy` int DEFAULT NULL,\n `vkqtveyivkcxrtecskafbpdgaqpvlfup` int DEFAULT NULL,\n `vfijgirkrzcbcpxddyewjtauzphfiyrm` int DEFAULT NULL,\n `pufefdnjakzjjxrkrlhcoftzklynhjqp` int DEFAULT NULL,\n `fmcqdxwazwdypuwavkachmvnqzgpqogl` int DEFAULT NULL,\n `svdbcxwphrdrnjsoestrkaosvxjrflmy` int DEFAULT NULL,\n `gwlxiilpfybtguqiwlbxylnmiqjmmtok` int DEFAULT NULL,\n `oulslalvbapabkhkiarraegtlrtfwakp` int DEFAULT NULL,\n `nzgpwrzsyblkwitmnecppqnhuaephaft` int DEFAULT NULL,\n `kvlaiqltooyafgqhjbuqpmkaaekyvjii` int DEFAULT NULL,\n `wuwzblyxbmgupwdurdezwywoojxsgeop` int DEFAULT NULL,\n `tstzdlxbajiflrsgzscemqyhvlcabter` int DEFAULT NULL,\n `cpehotngzldyivkwaeiqelqzyhspdynj` int DEFAULT NULL,\n `nruygwmrtkwxqufhpvdmcprcyipatqwh` int DEFAULT NULL,\n `nhuzbvmgyzzdcepcwoinhjhghjrwyvge` int DEFAULT NULL,\n `ekpaandjlroxctlaphcdpzfmvycuqwnq` int DEFAULT NULL,\n `wviqrhdlfxjlmgddegpmvheulzqaqoel` int DEFAULT NULL,\n `rgbbdwodsedsmheiwhfgsfpahmdhlseb` int DEFAULT NULL,\n `vrelndaspirbrfnqirunwenqxodncugz` int DEFAULT NULL,\n `hbsuqsehnsopteydeiwwllyiootsxqzr` int DEFAULT NULL,\n `pbfaquodsukkmfxgkgqwihgurwexrzog` int DEFAULT NULL,\n `absruqfrnhzixyysvtevsfuxgzrxigwr` int DEFAULT NULL,\n PRIMARY KEY (`ekgcpnfkygusvdjrukozimpyqlrvbxyr`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"fknkmyhzgzdgbklxmknjahvidhearmkn\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["ekgcpnfkygusvdjrukozimpyqlrvbxyr"],"columns":[{"name":"ekgcpnfkygusvdjrukozimpyqlrvbxyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"qacrwzpcacdgkbgkquqaumdcplgoccdj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vommluqcaexjrnhhctpsdzzbnnpetgdz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnoshwvsjbrrmbjrvnaztzmjfmdvshgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvuxjbxnpnsmagqnfddubifiqsoxzbqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ulxldkddhgfummrsosdglapvwfyfjntr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iiuwpwefoydvpvetvaubzrlgaryjoobm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlastibbugpmnbszsfobtwwlhrkuhauk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixokjwumbddbsiggwxovapehwvypqqpv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvqlwtguxtxptgknlwierqmtcutqplnn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kgrlxmgofbglrranyanoojwlcykitbla","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wkzetmlairtyjeafvgsitrbwavtlrahh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lemkbosumngjqrytphzuqeiickagnmmv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmbbfvesrsoehcnakejlcphebwaxvvtn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pitcvkssqhdhsjzazuckupuwuhrjxpvv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbkounebpieiiamjayzijkommrnmsxot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hyxmnbjwleabkpidtkzihrkiimzvzwrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"naysplzfdigbmqnyaeffldeypyrpzuyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fefjommkypjyyximknfiuehpekarcwum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"usjfukccvsbcnrafuwoswejioyflyxlh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfsetdzskekuxwuwpobgxgpsjoqgceoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"egkjwwzjkwxcsowxwjbbdankjdxfyifn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvxlowvaivnfpbtjjkwkzgtebztlvdov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmzsudxdnlehoglckvzuwcqxmznkfvdi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnfhjyzufjxgagyrxtzbexykmnrdteqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gocmkvudlaushszgwevbjtwbdxjjsigq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aggloyxsdmhuhkotyhuyqwjmsddsbtui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvjmmnbcvlvubqzdlwkklnnjhzzbodrc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"spmpszoebnmulpoljjozwxdtnbhomimc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kgtuohourgvqbucgrjzigkhrcsqulxxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfcqlndsnytlscdgaipkswnsaoznpyva","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxxsuirazvfcxmxirxeocgexrpocktbj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vytgkqeydfxiydtkthushhumqwjrqnkv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vysjgejbayejtqyiwgglbgsoiolkydzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pflthoeccyopdsyfbyefauxvyziiofsw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrwzvrvrujdhjoxxpcawgveebpngdroi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yixjnhthaiheuichkqzitsvbncxcivld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"corpbqlwgfooceyjwaanwarbhouqomhy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubznykidofjbdppbjuplrukbiguyisee","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohvonylduqevmylygovxynjbtvjmwijh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"koytlmatcsckcoalkldyqopqfqeznlcc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugrnxyvmatcvvjeunbdhnedvcbhtvxui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lrdmegibwzbpahltofyvqnspiwgzlmsh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qffbggzytfmrbqhiyldejhdwpzswyciv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vcymxnygfgcjpszqolckifzqdgwymyux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ndqjsehzixarmududqtnynfyxvadfrcd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sodykgjfdgixedxwxjapmkqdczphgsgs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njxugwgqztjjluslbfqhmsidlpxuoebr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwlaxnslfgpbssvxbspmxjwtazageerz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwbhbmqwzfryybkjsternyvxgwvzkedu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwlwqkmtdvlqlmbgtnfraxlvqbnbunuh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"taatllbcgiglhpdetyhrhzcgqupcxdnp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zffyrvqblezhjmfsfzkqwowzgkqusguy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eovjzfuugwhnfkvkioxesletiuhxwkqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wsiqwkdvrbqdtezpscsdimnahbykmbol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmkttymgiksykwuuzdgxyxufuhdhjzcm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqupxvourwttsxxpzirxlchoiooifnps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnumzwwkozfcrndyubnbkotzrfboukoo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mglmkqeffmhjvjdjbueiipldbtasooph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zztjyxyxmjdochnkdwsshubpjafodpns","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uounljusacmeuezowjsjvnpyvwankcyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cakrutzjzyitgamrdjyxktesypienzct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqtknnbkrfmhyxzdbkfsewrwikearydv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kasqjsgimdkcggiiqbsgpwtbcdeyuile","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yszspomchhrhjajlralqxanhgpodiuvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"erceugixynqbwnbojposaaapapryzhdl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shucvwtsxwomxuolnofmhyswjnrvcjsv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhpwcyzksbubbxkkgpkzxaelcbiisxyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"djpswwspwdnpwajmlhhlpxhqvngfciri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"robcykqtvycgbiisicenswyycpnvrkxn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkbtkqzbrzqcpxdfzkqolbtovpmsklew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hrqkouwagwmuunfqgkxcwmvfczpoabom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwwagrntgmvlmhrllypdzccircdapggo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffvqwrvgpqtmqtijmybtmjmsfmxmisug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfijnvgqovevutzfpjfrmuwmkxkgtvmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftpagyvphzgorqvfjvubauosbormcjvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"msgrexapcpigvmcrimnopacwmzjweral","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eoyywmibywdpeyohbokpzknmcjmlmzrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"luegpfsgrvgcebzoojjbidatrquvpeoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkqtveyivkcxrtecskafbpdgaqpvlfup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vfijgirkrzcbcpxddyewjtauzphfiyrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pufefdnjakzjjxrkrlhcoftzklynhjqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmcqdxwazwdypuwavkachmvnqzgpqogl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"svdbcxwphrdrnjsoestrkaosvxjrflmy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gwlxiilpfybtguqiwlbxylnmiqjmmtok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oulslalvbapabkhkiarraegtlrtfwakp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzgpwrzsyblkwitmnecppqnhuaephaft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvlaiqltooyafgqhjbuqpmkaaekyvjii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wuwzblyxbmgupwdurdezwywoojxsgeop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tstzdlxbajiflrsgzscemqyhvlcabter","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cpehotngzldyivkwaeiqelqzyhspdynj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nruygwmrtkwxqufhpvdmcprcyipatqwh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhuzbvmgyzzdcepcwoinhjhghjrwyvge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekpaandjlroxctlaphcdpzfmvycuqwnq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wviqrhdlfxjlmgddegpmvheulzqaqoel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgbbdwodsedsmheiwhfgsfpahmdhlseb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrelndaspirbrfnqirunwenqxodncugz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbsuqsehnsopteydeiwwllyiootsxqzr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbfaquodsukkmfxgkgqwihgurwexrzog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"absruqfrnhzixyysvtevsfuxgzrxigwr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667740,"databaseName":"models_schema","ddl":"CREATE TABLE `fmydwwpviyonaxoxelqmczxonlvjcnxv` (\n `ffuuegmzvjtdbzazquykbvhsngfsytos` int NOT NULL,\n `gnguibrufmsdhiuxecgxsdkveaynkiuq` int DEFAULT NULL,\n `ddkecnzjqmffnbitmwxhwjgxjmnxliin` int DEFAULT NULL,\n `vrapnxxhntwmgeehrmddradfxjwxbtvh` int DEFAULT NULL,\n `vsrzhzssumguoidlxcleepddgyyodadg` int DEFAULT NULL,\n `jyhpmeavugoydhfgpoajeitrtegaeclx` int DEFAULT NULL,\n `dlurvipneoojbijaeknxpnuccsgphnpy` int DEFAULT NULL,\n `dbwgajmqoquhubeblqcvuxhsabnibnnn` int DEFAULT NULL,\n `isspayffujriylinrdjamrkjeazqhobh` int DEFAULT NULL,\n `xafdsfojrzjwesimccvorrgqhiutgjxs` int DEFAULT NULL,\n `bnmwyjjwakthxfbxjmclazxhjghlsoaz` int DEFAULT NULL,\n `ovzwjeudtgwptrlujygpwmvqxhteateq` int DEFAULT NULL,\n `fjxnyhlksxnamdjbgzsqlbawfumghqpf` int DEFAULT NULL,\n `xflloshumbrdiomlehmxcqqoiqokcosf` int DEFAULT NULL,\n `lqkuujsnwafhgqumujvgsroeflrusjwb` int DEFAULT NULL,\n `eemkjwyybcxaxayuqtkjggzoyvljmjag` int DEFAULT NULL,\n `wbpylewghretxlcmsqhtljbndcyjzjsw` int DEFAULT NULL,\n `trnczwgybjglgphfepsxathhzrlmuejr` int DEFAULT NULL,\n `hnacrayjudsmjsnhekgcksagiiwlyhpe` int DEFAULT NULL,\n `oloxrlkektbabgpazhykrrgzbbhqlecj` int DEFAULT NULL,\n `huxbyexmwdhxffwdttdqknsjowllddlu` int DEFAULT NULL,\n `lyppqywpmdcbdltqixqmuaapnmizythd` int DEFAULT NULL,\n `yyjgigaieqindszotcntbyutgowannja` int DEFAULT NULL,\n `ksbgqtyozgjcuzlwrgqbwisxpzcmoarr` int DEFAULT NULL,\n `fnqjuqysisxxejqigqfpkezmefmrqepd` int DEFAULT NULL,\n `nzdrrolqvbkfpxukmjvjxrpircjdjrye` int DEFAULT NULL,\n `bqccdzqykjkafdsjuzsgwyvuvtfeptbd` int DEFAULT NULL,\n `plfqzblwruhyhjkpmbydmtxjsgllrlsj` int DEFAULT NULL,\n `kcrftwcvxocxkhelncgqztiemhmqxcwu` int DEFAULT NULL,\n `tmdhgaimucnahdnjxxnslezjaersnvyb` int DEFAULT NULL,\n `pceymcrtetivdozjunkddvsmbybmhoxj` int DEFAULT NULL,\n `fchonydkxxzimrdykzcimjpxhlmwubaf` int DEFAULT NULL,\n `rxukqourniyudiyxohhangjagvlvzswj` int DEFAULT NULL,\n `lokoxwihlbdymptefbkcwjpiqpitriwr` int DEFAULT NULL,\n `sbiaoxwjomspjpchrteiobfstutrriyg` int DEFAULT NULL,\n `yjpsrospntqzkjnwztlpkbullsnvxrwu` int DEFAULT NULL,\n `wbffqmgtnwrhaudvrkqylwnpvjczpmbf` int DEFAULT NULL,\n `zzyfoxrrpfxdjjbkskphelqxlrithtrg` int DEFAULT NULL,\n `gqsoujrgvymktbbnkawciavtchgifhyd` int DEFAULT NULL,\n `hwuqjpsjcwkbqthdqzetlzlsfiwjgoep` int DEFAULT NULL,\n `ixagmbssgdgdofpnfwjgjfdjcpcqlolv` int DEFAULT NULL,\n `kbmjpgaegubvbebhgxrikyjisrafhbcg` int DEFAULT NULL,\n `kddgxisuuzmaexutjiwrwjthsqzlhvas` int DEFAULT NULL,\n `qclfucacprjaxmtbttmfiqbgluqqvuya` int DEFAULT NULL,\n `fkqawiqsnoekmzxjblkhrxwzxmuftmoh` int DEFAULT NULL,\n `hmxunhkptuepzqlolpyhmaqprfdrugxt` int DEFAULT NULL,\n `nftfhexmhkdxcxaaxwbzltiujkhadggt` int DEFAULT NULL,\n `twhloocgvwtrnikrgglufeaejgudagah` int DEFAULT NULL,\n `bvtgoakwjxqcqilywsmepflmnjxeqfuc` int DEFAULT NULL,\n `gfuuheyzppyueeyftybokasjbynlkxdu` int DEFAULT NULL,\n `ctebdrxxgrffzifofojaoqipfomrczvl` int DEFAULT NULL,\n `dqgipuhlmpnrlmbbiuyftmstotdftxrq` int DEFAULT NULL,\n `gsfuqhkvwutunnighffixeaigjhdxynx` int DEFAULT NULL,\n `xaebcbbevlzcehxzukptcqmvkgsfwupv` int DEFAULT NULL,\n `udpigqrrdygsbefywpnemkolliskwroq` int DEFAULT NULL,\n `khmfxolknixqxbofmvjgsjydbxomplhu` int DEFAULT NULL,\n `wnsjeuqblvgsjlhuicilnfmnymagdddp` int DEFAULT NULL,\n `vlsgzhgfhwubdjjdiggzazxffmjzqaro` int DEFAULT NULL,\n `fxyqcpecwcowoojnhvraybgelogeqhuc` int DEFAULT NULL,\n `aibtzoyncjlvsnzszhruzsesmpazorjt` int DEFAULT NULL,\n `srkmopfaqkqooixsapdiickenvwbodpy` int DEFAULT NULL,\n `wlndalfhcvvepcogyalabbfbyjbrsrlv` int DEFAULT NULL,\n `ibodevjhukigtuhuutjuarjuncfqjjvq` int DEFAULT NULL,\n `jzzcaiuuobjtphwrgtsbxbxzhmkqoffg` int DEFAULT NULL,\n `ewizvkgvpcxcyjdyxfxpvycifixdimpj` int DEFAULT NULL,\n `rmirgtlxngmngeohrphtnqlynkihllmc` int DEFAULT NULL,\n `fplgdpsjkpoarljjpaylvqewtuubyobk` int DEFAULT NULL,\n `bzfrxhrxxnzrbpjjyfbkquagbcnpgkle` int DEFAULT NULL,\n `vxcnhnqicxqojkldlfnfmrhhajrglfwo` int DEFAULT NULL,\n `eathampkltxafmatwwnpsbvztpwgcrzn` int DEFAULT NULL,\n `xcyizhezlbkkbsguuahkdacafhzaeywq` int DEFAULT NULL,\n `xrhztcjcliidtzfcqvnlzzvxsaluabhq` int DEFAULT NULL,\n `jzlqoothhdyayjxcimiqqszckrcbdafq` int DEFAULT NULL,\n `lvennertshhyyzkzbuzsmikdlcrrysxg` int DEFAULT NULL,\n `ujfkaqeobctgubdanobmurzqtmtpbckm` int DEFAULT NULL,\n `rijzhtjkqypzlnicpetieiabotadfbkp` int DEFAULT NULL,\n `jbdgvnnkryunfssyygqyepvftalivxhh` int DEFAULT NULL,\n `xrytszdaksdctgacfkzkqgyktowzetjm` int DEFAULT NULL,\n `wfjhuhnzkudcmgzlazphiuzzgmfrtbfy` int DEFAULT NULL,\n `ewwyqhiecwqjcivbwqqpbnruujnszswk` int DEFAULT NULL,\n `dnhsgkaktkbjrlotfpqqlthjhnqfzoml` int DEFAULT NULL,\n `thkzdvlibpuofaeiijoqajjeakuluexv` int DEFAULT NULL,\n `coxiatzvtudfuprcelafscssabxtmleg` int DEFAULT NULL,\n `yganwtfejjmmxzodqjfjkioobulfxhka` int DEFAULT NULL,\n `gaxmoddwrxsbkixpgedskzwizherbxrn` int DEFAULT NULL,\n `gdwglfxgjxwygygsqgviblyfubaqwjtt` int DEFAULT NULL,\n `uxianqmxwqbqlqxxaecmmmaufhpscgdk` int DEFAULT NULL,\n `kavnehckilbwmkhpxqqzbrtmvcuvwqwn` int DEFAULT NULL,\n `uoleoubgmpxyjjlvdlgrenzdjnttkhha` int DEFAULT NULL,\n `abkztsmdmxqlkxvpqnjagccrpqnhpohb` int DEFAULT NULL,\n `nxendeyhtyfqhwlojtsftpcvbxfwslyv` int DEFAULT NULL,\n `tyfnvyjhwfscpdecixpdiirfbwzidchq` int DEFAULT NULL,\n `axlwaubbbltnewxhrsluhnndrcatjult` int DEFAULT NULL,\n `prrxpniezcafxvajzmmfbdsdxyevverg` int DEFAULT NULL,\n `pyivnghklrrxpflcdvrielwgqikfbrlt` int DEFAULT NULL,\n `plwiudthnnibhxwjtuenywtvwsxtxoad` int DEFAULT NULL,\n `uydtnjlnpuxbvskskxgbsyogfumjroza` int DEFAULT NULL,\n `iwvnmybwobesothmacwvbeinlsqatovw` int DEFAULT NULL,\n `kuagzrcqutepfadlrgnrzcuoyxjkgfss` int DEFAULT NULL,\n `kxvwspeifpsgpauaogtmdapxdavwfxjz` int DEFAULT NULL,\n PRIMARY KEY (`ffuuegmzvjtdbzazquykbvhsngfsytos`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"fmydwwpviyonaxoxelqmczxonlvjcnxv\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["ffuuegmzvjtdbzazquykbvhsngfsytos"],"columns":[{"name":"ffuuegmzvjtdbzazquykbvhsngfsytos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"gnguibrufmsdhiuxecgxsdkveaynkiuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddkecnzjqmffnbitmwxhwjgxjmnxliin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrapnxxhntwmgeehrmddradfxjwxbtvh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsrzhzssumguoidlxcleepddgyyodadg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jyhpmeavugoydhfgpoajeitrtegaeclx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlurvipneoojbijaeknxpnuccsgphnpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbwgajmqoquhubeblqcvuxhsabnibnnn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"isspayffujriylinrdjamrkjeazqhobh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xafdsfojrzjwesimccvorrgqhiutgjxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnmwyjjwakthxfbxjmclazxhjghlsoaz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovzwjeudtgwptrlujygpwmvqxhteateq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjxnyhlksxnamdjbgzsqlbawfumghqpf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xflloshumbrdiomlehmxcqqoiqokcosf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqkuujsnwafhgqumujvgsroeflrusjwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eemkjwyybcxaxayuqtkjggzoyvljmjag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbpylewghretxlcmsqhtljbndcyjzjsw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"trnczwgybjglgphfepsxathhzrlmuejr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hnacrayjudsmjsnhekgcksagiiwlyhpe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oloxrlkektbabgpazhykrrgzbbhqlecj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"huxbyexmwdhxffwdttdqknsjowllddlu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lyppqywpmdcbdltqixqmuaapnmizythd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yyjgigaieqindszotcntbyutgowannja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ksbgqtyozgjcuzlwrgqbwisxpzcmoarr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnqjuqysisxxejqigqfpkezmefmrqepd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzdrrolqvbkfpxukmjvjxrpircjdjrye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqccdzqykjkafdsjuzsgwyvuvtfeptbd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"plfqzblwruhyhjkpmbydmtxjsgllrlsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcrftwcvxocxkhelncgqztiemhmqxcwu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmdhgaimucnahdnjxxnslezjaersnvyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pceymcrtetivdozjunkddvsmbybmhoxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fchonydkxxzimrdykzcimjpxhlmwubaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxukqourniyudiyxohhangjagvlvzswj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lokoxwihlbdymptefbkcwjpiqpitriwr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbiaoxwjomspjpchrteiobfstutrriyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjpsrospntqzkjnwztlpkbullsnvxrwu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbffqmgtnwrhaudvrkqylwnpvjczpmbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzyfoxrrpfxdjjbkskphelqxlrithtrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqsoujrgvymktbbnkawciavtchgifhyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwuqjpsjcwkbqthdqzetlzlsfiwjgoep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixagmbssgdgdofpnfwjgjfdjcpcqlolv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kbmjpgaegubvbebhgxrikyjisrafhbcg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kddgxisuuzmaexutjiwrwjthsqzlhvas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qclfucacprjaxmtbttmfiqbgluqqvuya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fkqawiqsnoekmzxjblkhrxwzxmuftmoh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmxunhkptuepzqlolpyhmaqprfdrugxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nftfhexmhkdxcxaaxwbzltiujkhadggt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twhloocgvwtrnikrgglufeaejgudagah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvtgoakwjxqcqilywsmepflmnjxeqfuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfuuheyzppyueeyftybokasjbynlkxdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctebdrxxgrffzifofojaoqipfomrczvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqgipuhlmpnrlmbbiuyftmstotdftxrq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsfuqhkvwutunnighffixeaigjhdxynx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xaebcbbevlzcehxzukptcqmvkgsfwupv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udpigqrrdygsbefywpnemkolliskwroq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khmfxolknixqxbofmvjgsjydbxomplhu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnsjeuqblvgsjlhuicilnfmnymagdddp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vlsgzhgfhwubdjjdiggzazxffmjzqaro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxyqcpecwcowoojnhvraybgelogeqhuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aibtzoyncjlvsnzszhruzsesmpazorjt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"srkmopfaqkqooixsapdiickenvwbodpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wlndalfhcvvepcogyalabbfbyjbrsrlv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibodevjhukigtuhuutjuarjuncfqjjvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzzcaiuuobjtphwrgtsbxbxzhmkqoffg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ewizvkgvpcxcyjdyxfxpvycifixdimpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rmirgtlxngmngeohrphtnqlynkihllmc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fplgdpsjkpoarljjpaylvqewtuubyobk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzfrxhrxxnzrbpjjyfbkquagbcnpgkle","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxcnhnqicxqojkldlfnfmrhhajrglfwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eathampkltxafmatwwnpsbvztpwgcrzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xcyizhezlbkkbsguuahkdacafhzaeywq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrhztcjcliidtzfcqvnlzzvxsaluabhq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzlqoothhdyayjxcimiqqszckrcbdafq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvennertshhyyzkzbuzsmikdlcrrysxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujfkaqeobctgubdanobmurzqtmtpbckm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rijzhtjkqypzlnicpetieiabotadfbkp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbdgvnnkryunfssyygqyepvftalivxhh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrytszdaksdctgacfkzkqgyktowzetjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfjhuhnzkudcmgzlazphiuzzgmfrtbfy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ewwyqhiecwqjcivbwqqpbnruujnszswk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dnhsgkaktkbjrlotfpqqlthjhnqfzoml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thkzdvlibpuofaeiijoqajjeakuluexv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"coxiatzvtudfuprcelafscssabxtmleg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yganwtfejjmmxzodqjfjkioobulfxhka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gaxmoddwrxsbkixpgedskzwizherbxrn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdwglfxgjxwygygsqgviblyfubaqwjtt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxianqmxwqbqlqxxaecmmmaufhpscgdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kavnehckilbwmkhpxqqzbrtmvcuvwqwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uoleoubgmpxyjjlvdlgrenzdjnttkhha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abkztsmdmxqlkxvpqnjagccrpqnhpohb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxendeyhtyfqhwlojtsftpcvbxfwslyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tyfnvyjhwfscpdecixpdiirfbwzidchq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axlwaubbbltnewxhrsluhnndrcatjult","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prrxpniezcafxvajzmmfbdsdxyevverg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pyivnghklrrxpflcdvrielwgqikfbrlt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"plwiudthnnibhxwjtuenywtvwsxtxoad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uydtnjlnpuxbvskskxgbsyogfumjroza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwvnmybwobesothmacwvbeinlsqatovw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kuagzrcqutepfadlrgnrzcuoyxjkgfss","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxvwspeifpsgpauaogtmdapxdavwfxjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667784,"databaseName":"models_schema","ddl":"CREATE TABLE `fmyszhkkeojxcsveoimbanarqxdpbqkz` (\n `hnplgyojhftpkhiascztvjfgjhesdpix` int NOT NULL,\n `vabumskvplapiternmoircykzsmvzakm` int DEFAULT NULL,\n `wmcvqbmmufcyjdobrhjileqmextqnvnx` int DEFAULT NULL,\n `qmkxitfwdowmycwmufmaykelzzrfxuox` int DEFAULT NULL,\n `hfbcbtgizlvmvgetnivvtaienylwrrnt` int DEFAULT NULL,\n `vckwwetpoiznvqmonfdzfdsvooxowzmr` int DEFAULT NULL,\n `smltzfomoxiyjubdzpporkciupuqhknw` int DEFAULT NULL,\n `hpekgwqfcuthcivnyasplicpjqacbpdu` int DEFAULT NULL,\n `fisgngvsrzqxzlkogpnyqmmrngoocwll` int DEFAULT NULL,\n `hoaqkxgwqvkwoxwjdszfywbixynnrxgx` int DEFAULT NULL,\n `sfrzdnopyqngphalzfzifvhshkgghsuq` int DEFAULT NULL,\n `pwtzytqvthaufoevozhqmafzzqdraxzl` int DEFAULT NULL,\n `ylvjbtxbyjsvrouxulovzcxedyuxseqv` int DEFAULT NULL,\n `ufvilgoajbpkrmdszjbkouqpkxcrczed` int DEFAULT NULL,\n `efyoqmwtjbcygumorkfhaaztlugaytnr` int DEFAULT NULL,\n `paxticaclpfixaozjhsmjcrpruwdazjn` int DEFAULT NULL,\n `mkdkwnkjxctxzyoutaqzrklwzhknjaqq` int DEFAULT NULL,\n `xscynvghkgsfekvbfxrpzphcxcacwrvm` int DEFAULT NULL,\n `ukjggwaevecanzzstoddvrxdtyhocsaj` int DEFAULT NULL,\n `dqtyonoacbbumalghqcftczxxcbphmfh` int DEFAULT NULL,\n `qwlffaiqhdigtjxtrrgpjmacxzzlgbrx` int DEFAULT NULL,\n `dxukwsnhwawgdckbbkdthoorhwrixiuu` int DEFAULT NULL,\n `fwijdpeqjnqskloetevhvckzcgyxfvqh` int DEFAULT NULL,\n `ypwytknhulgfgdxrhlboipchjxdvgwum` int DEFAULT NULL,\n `ozjtylqopfbckjwdnozmbfacfudgzwim` int DEFAULT NULL,\n `jtjbjpjuhkhspncbyaicnyxtwctpcgts` int DEFAULT NULL,\n `jsdpfcvpkgzamlfumdntiqzlorbaxrin` int DEFAULT NULL,\n `kfeyvywexisbpszsujiipzpljnvlqlbu` int DEFAULT NULL,\n `opjjzdnwcswukcnabbbhckmflegjlmid` int DEFAULT NULL,\n `fpbrqkbqpctazodffxdlsrflzcawtaos` int DEFAULT NULL,\n `ljortgnwcjnpcoikmxngumwudseaksby` int DEFAULT NULL,\n `dryctepwndqffghifaiwtosdvbznqcgv` int DEFAULT NULL,\n `jhpwepispfsahynfzlamfwgqwyfxcdor` int DEFAULT NULL,\n `fnjfwyrroxskmjepxnwlrozqflxmwbci` int DEFAULT NULL,\n `lhaspoxmhhwubmulmiwyxhnzxblutoro` int DEFAULT NULL,\n `pgjwqmavmxldhyvhiugaedopxkdqslca` int DEFAULT NULL,\n `ugjxwsmruelnbkythtlppmsckjokqart` int DEFAULT NULL,\n `cbgwnetrqbmatholkjmpphpkfnelvzbg` int DEFAULT NULL,\n `fyiuccxjejlelgnbcbidcnfabqkcvuxx` int DEFAULT NULL,\n `hwajlepvkhhfkwvpzzoudvusvhzepmpt` int DEFAULT NULL,\n `slamtgyubkhumdnlwlswhpvabsttmfzx` int DEFAULT NULL,\n `xikmeusissemrxogbonvncwszxcvbkon` int DEFAULT NULL,\n `zqlnjrcpspwajsvnydvvgvkjmzuvgtej` int DEFAULT NULL,\n `vocxcagshdasbphihfrbcnvmovuvnacg` int DEFAULT NULL,\n `bkwlrxkolfdibshzquqgiocqrwjkqwkb` int DEFAULT NULL,\n `ewchtxkmsyuevuxokksafnveitxumeqg` int DEFAULT NULL,\n `iqcrulcnzivbljhgszclbeaxitkixrab` int DEFAULT NULL,\n `upprlexunvjwdhhmooxenflzpnjpixsg` int DEFAULT NULL,\n `vooyoemsszlmuxcnxbcwusutmefuunoy` int DEFAULT NULL,\n `gfvxhinkxpsbjyndbaehwxyhzrajbzws` int DEFAULT NULL,\n `rkjdrhaeuuourwssazpgfzybtneexpga` int DEFAULT NULL,\n `kflbjzicynxkzppucdpfynnstamctkaw` int DEFAULT NULL,\n `sauswbukqvcazyideqneinxugrzwytvj` int DEFAULT NULL,\n `nltpovbsjtemhedeeszddmkahqpdwyfo` int DEFAULT NULL,\n `kzmahskewnljfaxxeqbmzptleojeggmu` int DEFAULT NULL,\n `spvhoqdtirlpalynftmqsvllpzfzhand` int DEFAULT NULL,\n `csnvgmdsxqirydfntjlcorlivmtuwpda` int DEFAULT NULL,\n `rkdkednpdritxwieouogsckomwrzvpwy` int DEFAULT NULL,\n `kyvhrjycidfptynllojeuymrpvspvoyq` int DEFAULT NULL,\n `kjdczfvapejbmaokgrcelhnvumswxvly` int DEFAULT NULL,\n `vfqdgovazbexscvhadmfqgpuhzglgncf` int DEFAULT NULL,\n `rcjrfzntmjpmdpimfniypxpmaqgzdvei` int DEFAULT NULL,\n `polvnahyyiwjnxvnxyzisclzsjdlapcn` int DEFAULT NULL,\n `suoihncqywmuqqentbpprjqpgqvnjsxg` int DEFAULT NULL,\n `pkajvicawywbjgxhxfaxfxvaluatgptg` int DEFAULT NULL,\n `afjwajfsmsmoymohrjfjajaltyhxkydb` int DEFAULT NULL,\n `nyureerjuqfiuwxqtlregcnoxdjuoeib` int DEFAULT NULL,\n `gylyooawcxbzyljugsmdkbgihchhvmhh` int DEFAULT NULL,\n `wbybblgpyudbpmdoigbzyazmqvswbevh` int DEFAULT NULL,\n `xvghbhoswyweywhtgnwhuuqkrqcridua` int DEFAULT NULL,\n `gejovnunctdmkktcyruudskxneddybay` int DEFAULT NULL,\n `psrvdwjrmljqoaaimfejpcxeqmulbtlp` int DEFAULT NULL,\n `hgfhqjjtuhgqqgrmrajvxclvzywrhnyl` int DEFAULT NULL,\n `fbvfoiyexesudclujxdzdzsuogjhychg` int DEFAULT NULL,\n `axsdlyokosqeqzwyphoayzegmcsqvmro` int DEFAULT NULL,\n `yetyultyhdorzkuolkhmabckdarpvppi` int DEFAULT NULL,\n `sycwtngorcsueozkhgtsqkhppsivavgu` int DEFAULT NULL,\n `yptvujhneiuwocuoixkakhwhwlcnkgdz` int DEFAULT NULL,\n `dwltehmrttxpsztumyalofefuxqsidip` int DEFAULT NULL,\n `ubniryqblqbmevwnzzxgzhajsbrvnpfg` int DEFAULT NULL,\n `faroacqklyhfnqyxemrpkyzxuvvqilbi` int DEFAULT NULL,\n `mavyftmwewolmhnqtvynfjivbwccnydi` int DEFAULT NULL,\n `lxdsrdrxolijovxfhxtxlhywtjnealwj` int DEFAULT NULL,\n `ognnahpwxjyxnxpqxewtqzjjnejrbijc` int DEFAULT NULL,\n `irjfyiurhxqnnfpxojifjjeoplwnizqy` int DEFAULT NULL,\n `vjwkmldyzwldxjxhqxhdnmtosainxnmz` int DEFAULT NULL,\n `bryxbmglebokmmrvtcoyjkjsuydgujmw` int DEFAULT NULL,\n `gfuqktmgipmjhrppjqqhgocpkmdkqsak` int DEFAULT NULL,\n `xwzezuqxslmfsljuqlwjxnmylsawcigp` int DEFAULT NULL,\n `vsuawqdysryhtzixkpwxvubweavjxdjl` int DEFAULT NULL,\n `nyfvnrlfkxpfiauynhmiawapmhhirubo` int DEFAULT NULL,\n `ndysfdrsdcqvxkyywajrqovmabjpiaha` int DEFAULT NULL,\n `phlugdumqzabvfoftipcobxckernduun` int DEFAULT NULL,\n `fenglcpsallektsrgvqepwwzfdftvqpn` int DEFAULT NULL,\n `anmvkmucfbttujoetaqhwnhbqvboccqa` int DEFAULT NULL,\n `vsotvngzgtrepbwhaqiemldhbsompckm` int DEFAULT NULL,\n `ogmksvbhrunbitdprqtqhddyftofzmte` int DEFAULT NULL,\n `twoxwicraqdcmsaivuzjrfaqwqtpacar` int DEFAULT NULL,\n `nypzjiqwpkhwnxdaprutcfoqteoejkjq` int DEFAULT NULL,\n `pwszgolpmiqtfetagsardhshnbksqyho` int DEFAULT NULL,\n PRIMARY KEY (`hnplgyojhftpkhiascztvjfgjhesdpix`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"fmyszhkkeojxcsveoimbanarqxdpbqkz\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["hnplgyojhftpkhiascztvjfgjhesdpix"],"columns":[{"name":"hnplgyojhftpkhiascztvjfgjhesdpix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"vabumskvplapiternmoircykzsmvzakm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmcvqbmmufcyjdobrhjileqmextqnvnx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmkxitfwdowmycwmufmaykelzzrfxuox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfbcbtgizlvmvgetnivvtaienylwrrnt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vckwwetpoiznvqmonfdzfdsvooxowzmr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smltzfomoxiyjubdzpporkciupuqhknw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hpekgwqfcuthcivnyasplicpjqacbpdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fisgngvsrzqxzlkogpnyqmmrngoocwll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hoaqkxgwqvkwoxwjdszfywbixynnrxgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfrzdnopyqngphalzfzifvhshkgghsuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwtzytqvthaufoevozhqmafzzqdraxzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ylvjbtxbyjsvrouxulovzcxedyuxseqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufvilgoajbpkrmdszjbkouqpkxcrczed","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"efyoqmwtjbcygumorkfhaaztlugaytnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"paxticaclpfixaozjhsmjcrpruwdazjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkdkwnkjxctxzyoutaqzrklwzhknjaqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xscynvghkgsfekvbfxrpzphcxcacwrvm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ukjggwaevecanzzstoddvrxdtyhocsaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqtyonoacbbumalghqcftczxxcbphmfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwlffaiqhdigtjxtrrgpjmacxzzlgbrx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxukwsnhwawgdckbbkdthoorhwrixiuu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwijdpeqjnqskloetevhvckzcgyxfvqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypwytknhulgfgdxrhlboipchjxdvgwum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozjtylqopfbckjwdnozmbfacfudgzwim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jtjbjpjuhkhspncbyaicnyxtwctpcgts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jsdpfcvpkgzamlfumdntiqzlorbaxrin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfeyvywexisbpszsujiipzpljnvlqlbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opjjzdnwcswukcnabbbhckmflegjlmid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpbrqkbqpctazodffxdlsrflzcawtaos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljortgnwcjnpcoikmxngumwudseaksby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dryctepwndqffghifaiwtosdvbznqcgv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jhpwepispfsahynfzlamfwgqwyfxcdor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnjfwyrroxskmjepxnwlrozqflxmwbci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhaspoxmhhwubmulmiwyxhnzxblutoro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgjwqmavmxldhyvhiugaedopxkdqslca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugjxwsmruelnbkythtlppmsckjokqart","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbgwnetrqbmatholkjmpphpkfnelvzbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fyiuccxjejlelgnbcbidcnfabqkcvuxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwajlepvkhhfkwvpzzoudvusvhzepmpt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"slamtgyubkhumdnlwlswhpvabsttmfzx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xikmeusissemrxogbonvncwszxcvbkon","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqlnjrcpspwajsvnydvvgvkjmzuvgtej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vocxcagshdasbphihfrbcnvmovuvnacg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkwlrxkolfdibshzquqgiocqrwjkqwkb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ewchtxkmsyuevuxokksafnveitxumeqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqcrulcnzivbljhgszclbeaxitkixrab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upprlexunvjwdhhmooxenflzpnjpixsg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vooyoemsszlmuxcnxbcwusutmefuunoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfvxhinkxpsbjyndbaehwxyhzrajbzws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkjdrhaeuuourwssazpgfzybtneexpga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kflbjzicynxkzppucdpfynnstamctkaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sauswbukqvcazyideqneinxugrzwytvj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nltpovbsjtemhedeeszddmkahqpdwyfo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzmahskewnljfaxxeqbmzptleojeggmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"spvhoqdtirlpalynftmqsvllpzfzhand","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"csnvgmdsxqirydfntjlcorlivmtuwpda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkdkednpdritxwieouogsckomwrzvpwy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kyvhrjycidfptynllojeuymrpvspvoyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjdczfvapejbmaokgrcelhnvumswxvly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vfqdgovazbexscvhadmfqgpuhzglgncf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcjrfzntmjpmdpimfniypxpmaqgzdvei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"polvnahyyiwjnxvnxyzisclzsjdlapcn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suoihncqywmuqqentbpprjqpgqvnjsxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkajvicawywbjgxhxfaxfxvaluatgptg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afjwajfsmsmoymohrjfjajaltyhxkydb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nyureerjuqfiuwxqtlregcnoxdjuoeib","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gylyooawcxbzyljugsmdkbgihchhvmhh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbybblgpyudbpmdoigbzyazmqvswbevh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvghbhoswyweywhtgnwhuuqkrqcridua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gejovnunctdmkktcyruudskxneddybay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psrvdwjrmljqoaaimfejpcxeqmulbtlp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hgfhqjjtuhgqqgrmrajvxclvzywrhnyl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbvfoiyexesudclujxdzdzsuogjhychg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axsdlyokosqeqzwyphoayzegmcsqvmro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yetyultyhdorzkuolkhmabckdarpvppi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sycwtngorcsueozkhgtsqkhppsivavgu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yptvujhneiuwocuoixkakhwhwlcnkgdz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwltehmrttxpsztumyalofefuxqsidip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubniryqblqbmevwnzzxgzhajsbrvnpfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"faroacqklyhfnqyxemrpkyzxuvvqilbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mavyftmwewolmhnqtvynfjivbwccnydi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxdsrdrxolijovxfhxtxlhywtjnealwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ognnahpwxjyxnxpqxewtqzjjnejrbijc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"irjfyiurhxqnnfpxojifjjeoplwnizqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjwkmldyzwldxjxhqxhdnmtosainxnmz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bryxbmglebokmmrvtcoyjkjsuydgujmw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfuqktmgipmjhrppjqqhgocpkmdkqsak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xwzezuqxslmfsljuqlwjxnmylsawcigp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsuawqdysryhtzixkpwxvubweavjxdjl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nyfvnrlfkxpfiauynhmiawapmhhirubo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ndysfdrsdcqvxkyywajrqovmabjpiaha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phlugdumqzabvfoftipcobxckernduun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fenglcpsallektsrgvqepwwzfdftvqpn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"anmvkmucfbttujoetaqhwnhbqvboccqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsotvngzgtrepbwhaqiemldhbsompckm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ogmksvbhrunbitdprqtqhddyftofzmte","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twoxwicraqdcmsaivuzjrfaqwqtpacar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nypzjiqwpkhwnxdaprutcfoqteoejkjq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwszgolpmiqtfetagsardhshnbksqyho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667819,"databaseName":"models_schema","ddl":"CREATE TABLE `fovddzegnkcsydxjfcghvbhggpiujcxa` (\n `mcvzuniotboipfeyirhztcuhcziojbvz` int NOT NULL,\n `ljgkotgzjnqxhldhbnftzpqizkvfenhp` int DEFAULT NULL,\n `abvwosxsefteschfaoqzzhryubwrxzlk` int DEFAULT NULL,\n `kwnfdkihifscuvqndkaxrpksuluwyfgg` int DEFAULT NULL,\n `pqnsulkxyuxmkaekgqbaxqtcodivhvrw` int DEFAULT NULL,\n `iskrjxwkmvrsycibvwwdgqjptkmvwegs` int DEFAULT NULL,\n `ftlvqzlbioybfshvdaefilmfqfguzntm` int DEFAULT NULL,\n `xmcdoozntrxfsgrlfxawwnperphjcuud` int DEFAULT NULL,\n `qumutbcdphtvgofmrglwvwxcmmhdnvhm` int DEFAULT NULL,\n `dikubcizoardkdcckudsthsmrajuippy` int DEFAULT NULL,\n `fmksgemaxtwwfksnzhjzgwfbwjtluehd` int DEFAULT NULL,\n `eehcfmhguhimsqulblxnjmaqzoiruuqh` int DEFAULT NULL,\n `urqnfqgdplfwyxdhnqxlvyvtrrvglqdu` int DEFAULT NULL,\n `ebsmbusmsnojvqyikonjolnbmirctlpw` int DEFAULT NULL,\n `pxroqdzapbawwxcghplxnvynfpviiydk` int DEFAULT NULL,\n `qcxsnnhnzjqwijxujgtdzlttkpljzsrg` int DEFAULT NULL,\n `fwowmwlmeyglbnedrytrsfudxtuebdqd` int DEFAULT NULL,\n `nvazpthncbjfhhblrlhwjlywzrxghkbc` int DEFAULT NULL,\n `zmwrdhttajgehnuqfzqlwwypkjzecxjy` int DEFAULT NULL,\n `pphnlkqeqyalfhwzyeamcecvhnzhzyri` int DEFAULT NULL,\n `duihxbehmcfkadhrlzdzyaqydgalvcpo` int DEFAULT NULL,\n `fibojzbsxcaplxszyxotakfzjochxjcs` int DEFAULT NULL,\n `enwiqyydzwxofrrjpfapohgxwukjisal` int DEFAULT NULL,\n `ihykuatnhiwbarouxmgsccosttjvlwkb` int DEFAULT NULL,\n `itnwawkqsgmvspugkgdhroaguucdkgcv` int DEFAULT NULL,\n `avlxowgnfmmaigzkkpwnoyfwdumzfjtm` int DEFAULT NULL,\n `zaeiiafjfurrgyyewnjxbnkkvtcjvktn` int DEFAULT NULL,\n `tjwhtvzrdsqtxkamyribvvogccwbqctn` int DEFAULT NULL,\n `jvgfykjlymuqrsarttsixryjmtzmyhrp` int DEFAULT NULL,\n `uwpbwctuubpttiuusvboxccfzhylwget` int DEFAULT NULL,\n `brkvlswziopvzrtuqwzvgnpccnsahpmd` int DEFAULT NULL,\n `vdamnvdlvenyhlgbebmidegzsselbihj` int DEFAULT NULL,\n `uwxlznmcnllzmzniqxygihvumlaxordx` int DEFAULT NULL,\n `nypdfwsmswxobvwdhcvpykzmymrvtnsa` int DEFAULT NULL,\n `pozomwvuwblvsokbsckknrjnukfwrbgn` int DEFAULT NULL,\n `mzybavcipawmffjvsamfteaeijwpxoci` int DEFAULT NULL,\n `yadwhriduuxhxpoleztyljytetgevbnu` int DEFAULT NULL,\n `yufynbxqioevrvcsksyjawdalkxqmdjh` int DEFAULT NULL,\n `qofvqrsadosfpqgscklxwewlgwwloabn` int DEFAULT NULL,\n `itztsppdlrfqvcklgqenrkkhrviicxda` int DEFAULT NULL,\n `fuyduzpeqngecxoowzbhbbbphlkvbngz` int DEFAULT NULL,\n `xywbxupqoryrrgpjfyezpgnehflsdxus` int DEFAULT NULL,\n `cchqrusbejjrqiaxvxrdbadwojidkfay` int DEFAULT NULL,\n `nhoauhihklxcnrbsggmstfrbzqirekdw` int DEFAULT NULL,\n `uetypxvqcqsqvqbnyvyqqcbyimyibhte` int DEFAULT NULL,\n `xnovvritxfhwujutqoqtnlidhvehspzl` int DEFAULT NULL,\n `qmzoepbggnkvatpqbhitfmcdyvoxssja` int DEFAULT NULL,\n `pjgwcwwvgjybivixgzzobkjmvcojuwcv` int DEFAULT NULL,\n `xcohicghipiqsgsomfeuabjkoswuvzsg` int DEFAULT NULL,\n `ltdoyjuevwkureyjqtuyxotybmphkouu` int DEFAULT NULL,\n `syxcpzqxhvtcrdaxsqavwztpmghcwdan` int DEFAULT NULL,\n `tfbzvdqdctrhftxmdbtvyicuqqkfjsha` int DEFAULT NULL,\n `zglyszjjfpfkomcdrhprstpvkfgtiifu` int DEFAULT NULL,\n `auuddhhwaevzmntfitcdqlsgibpmsklh` int DEFAULT NULL,\n `aieyheprgefesuuilfcriygcbzfgqanw` int DEFAULT NULL,\n `irvhagxrgginavhybnxnkfsxdboflnko` int DEFAULT NULL,\n `lpwrmswfgrfwsvzavmftqeqquyeajjjm` int DEFAULT NULL,\n `baspacnspccbnibgcjnmsegvaynlsfko` int DEFAULT NULL,\n `dcxbqmubfjjdilckxwzqedzjcvrcnfyx` int DEFAULT NULL,\n `ejljxnnilgudvxfvzwpbhyotslayqgrk` int DEFAULT NULL,\n `zncoainappavzfdyukcpeelnpsxsngwr` int DEFAULT NULL,\n `rtwnwloqjzldynktdhiqxxissltsopaj` int DEFAULT NULL,\n `moascnazsjpjpahxlyfsoiligkxytocy` int DEFAULT NULL,\n `fwihpqchvhkynmoqbcfcdisptenuioax` int DEFAULT NULL,\n `dvhrkxfexihyunzvmcxrjioeszwstxhm` int DEFAULT NULL,\n `ljctsctwkqncyelwrmdyzydkzlximdid` int DEFAULT NULL,\n `ihkjwvaoxznjhsaunkqpyajfrqdbetec` int DEFAULT NULL,\n `jmrexfgjodsblpynzykvykwxdvuyqckm` int DEFAULT NULL,\n `krdowfoltjohzkbcmogzprgwhtkvvkjy` int DEFAULT NULL,\n `bjhgrylgnazcewpojsacawrgdnmcbnrv` int DEFAULT NULL,\n `syywwlzbczdwomtrgsllruqojmaupooi` int DEFAULT NULL,\n `llmrvcuazqssvwseawokhqgdplrhlutd` int DEFAULT NULL,\n `gqtgpxsxhtfbwjcgarywotxyqpdoabtu` int DEFAULT NULL,\n `bguwzfcqsuxjuubdhwnkudbuyeugzmaa` int DEFAULT NULL,\n `ovkbtvdguzcrtjxvqondypgxdmlfodib` int DEFAULT NULL,\n `qbdmltlkkdzhlvpavnxwhkajrjzxclsu` int DEFAULT NULL,\n `miqtlllocjoqcrbxselfouaabxxvqspq` int DEFAULT NULL,\n `ufnmvrdkjaojwdtuhycchuxxsiskfoxi` int DEFAULT NULL,\n `uqvrcllyjkyczijajnhyexfihqojebwq` int DEFAULT NULL,\n `frtipdcdzrxdlqstpucqupaylaaaaydq` int DEFAULT NULL,\n `tjyrlbfuutnrzufvdqgghwtaoxretvut` int DEFAULT NULL,\n `twmpayuatdskvhukcxdssixajomxndvh` int DEFAULT NULL,\n `rkoddugzgpxlhwjjavbpwhapkixqcynn` int DEFAULT NULL,\n `bjwqmjvetapsrfrgqbtzdnzzruewzdld` int DEFAULT NULL,\n `kotytsclvphpfsvggwlcvltprspdicji` int DEFAULT NULL,\n `vudqvqsevxvpifwmzlrczktaltbdvnvg` int DEFAULT NULL,\n `zbdlphwvcfgiodfwhwtnvwmjkohsilsz` int DEFAULT NULL,\n `gexdsgpgulmxlnabjuysscmkybxnrzfk` int DEFAULT NULL,\n `lidjnozwamdjsirnfajskhbwdsweijbr` int DEFAULT NULL,\n `lbarwjpazhguohoymdkwbptkuiicwnjp` int DEFAULT NULL,\n `lagczahmzjetwnvaqeambcgleplatnbt` int DEFAULT NULL,\n `hqziambpzbsmgmvqdvzocwimnqjrhtjl` int DEFAULT NULL,\n `dxblxrqznpcienbnxcouryhrndyxcevy` int DEFAULT NULL,\n `yghmkttttvsvorkrvsgthpahkxdsuvws` int DEFAULT NULL,\n `awfaerlseyzeauglexzwocrmnvdwasia` int DEFAULT NULL,\n `lhrfusulpzfgqwjcfxxatrjjxttdppps` int DEFAULT NULL,\n `kwhbxskvamtzmcbolmiiwubggwhglgfq` int DEFAULT NULL,\n `jvzaldmmrldgtukvxybaysnuysmsoxgf` int DEFAULT NULL,\n `kzvoismgyfmqrqzhqdquzxrladqunwof` int DEFAULT NULL,\n `jtjmgtecugunffegtorimkqetgprogpd` int DEFAULT NULL,\n PRIMARY KEY (`mcvzuniotboipfeyirhztcuhcziojbvz`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"fovddzegnkcsydxjfcghvbhggpiujcxa\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["mcvzuniotboipfeyirhztcuhcziojbvz"],"columns":[{"name":"mcvzuniotboipfeyirhztcuhcziojbvz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ljgkotgzjnqxhldhbnftzpqizkvfenhp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abvwosxsefteschfaoqzzhryubwrxzlk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwnfdkihifscuvqndkaxrpksuluwyfgg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqnsulkxyuxmkaekgqbaxqtcodivhvrw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iskrjxwkmvrsycibvwwdgqjptkmvwegs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftlvqzlbioybfshvdaefilmfqfguzntm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmcdoozntrxfsgrlfxawwnperphjcuud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qumutbcdphtvgofmrglwvwxcmmhdnvhm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dikubcizoardkdcckudsthsmrajuippy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmksgemaxtwwfksnzhjzgwfbwjtluehd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eehcfmhguhimsqulblxnjmaqzoiruuqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urqnfqgdplfwyxdhnqxlvyvtrrvglqdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebsmbusmsnojvqyikonjolnbmirctlpw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxroqdzapbawwxcghplxnvynfpviiydk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qcxsnnhnzjqwijxujgtdzlttkpljzsrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwowmwlmeyglbnedrytrsfudxtuebdqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvazpthncbjfhhblrlhwjlywzrxghkbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmwrdhttajgehnuqfzqlwwypkjzecxjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pphnlkqeqyalfhwzyeamcecvhnzhzyri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"duihxbehmcfkadhrlzdzyaqydgalvcpo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fibojzbsxcaplxszyxotakfzjochxjcs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"enwiqyydzwxofrrjpfapohgxwukjisal","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ihykuatnhiwbarouxmgsccosttjvlwkb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itnwawkqsgmvspugkgdhroaguucdkgcv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avlxowgnfmmaigzkkpwnoyfwdumzfjtm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zaeiiafjfurrgyyewnjxbnkkvtcjvktn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjwhtvzrdsqtxkamyribvvogccwbqctn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvgfykjlymuqrsarttsixryjmtzmyhrp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwpbwctuubpttiuusvboxccfzhylwget","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"brkvlswziopvzrtuqwzvgnpccnsahpmd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdamnvdlvenyhlgbebmidegzsselbihj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwxlznmcnllzmzniqxygihvumlaxordx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nypdfwsmswxobvwdhcvpykzmymrvtnsa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pozomwvuwblvsokbsckknrjnukfwrbgn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzybavcipawmffjvsamfteaeijwpxoci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yadwhriduuxhxpoleztyljytetgevbnu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yufynbxqioevrvcsksyjawdalkxqmdjh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qofvqrsadosfpqgscklxwewlgwwloabn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itztsppdlrfqvcklgqenrkkhrviicxda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fuyduzpeqngecxoowzbhbbbphlkvbngz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xywbxupqoryrrgpjfyezpgnehflsdxus","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cchqrusbejjrqiaxvxrdbadwojidkfay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhoauhihklxcnrbsggmstfrbzqirekdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uetypxvqcqsqvqbnyvyqqcbyimyibhte","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnovvritxfhwujutqoqtnlidhvehspzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmzoepbggnkvatpqbhitfmcdyvoxssja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjgwcwwvgjybivixgzzobkjmvcojuwcv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xcohicghipiqsgsomfeuabjkoswuvzsg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ltdoyjuevwkureyjqtuyxotybmphkouu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"syxcpzqxhvtcrdaxsqavwztpmghcwdan","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tfbzvdqdctrhftxmdbtvyicuqqkfjsha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zglyszjjfpfkomcdrhprstpvkfgtiifu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"auuddhhwaevzmntfitcdqlsgibpmsklh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aieyheprgefesuuilfcriygcbzfgqanw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"irvhagxrgginavhybnxnkfsxdboflnko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lpwrmswfgrfwsvzavmftqeqquyeajjjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"baspacnspccbnibgcjnmsegvaynlsfko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcxbqmubfjjdilckxwzqedzjcvrcnfyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejljxnnilgudvxfvzwpbhyotslayqgrk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zncoainappavzfdyukcpeelnpsxsngwr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtwnwloqjzldynktdhiqxxissltsopaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"moascnazsjpjpahxlyfsoiligkxytocy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwihpqchvhkynmoqbcfcdisptenuioax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvhrkxfexihyunzvmcxrjioeszwstxhm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljctsctwkqncyelwrmdyzydkzlximdid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ihkjwvaoxznjhsaunkqpyajfrqdbetec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmrexfgjodsblpynzykvykwxdvuyqckm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krdowfoltjohzkbcmogzprgwhtkvvkjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjhgrylgnazcewpojsacawrgdnmcbnrv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"syywwlzbczdwomtrgsllruqojmaupooi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llmrvcuazqssvwseawokhqgdplrhlutd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqtgpxsxhtfbwjcgarywotxyqpdoabtu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bguwzfcqsuxjuubdhwnkudbuyeugzmaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovkbtvdguzcrtjxvqondypgxdmlfodib","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qbdmltlkkdzhlvpavnxwhkajrjzxclsu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"miqtlllocjoqcrbxselfouaabxxvqspq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufnmvrdkjaojwdtuhycchuxxsiskfoxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqvrcllyjkyczijajnhyexfihqojebwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"frtipdcdzrxdlqstpucqupaylaaaaydq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjyrlbfuutnrzufvdqgghwtaoxretvut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twmpayuatdskvhukcxdssixajomxndvh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkoddugzgpxlhwjjavbpwhapkixqcynn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjwqmjvetapsrfrgqbtzdnzzruewzdld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kotytsclvphpfsvggwlcvltprspdicji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vudqvqsevxvpifwmzlrczktaltbdvnvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbdlphwvcfgiodfwhwtnvwmjkohsilsz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gexdsgpgulmxlnabjuysscmkybxnrzfk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lidjnozwamdjsirnfajskhbwdsweijbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lbarwjpazhguohoymdkwbptkuiicwnjp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lagczahmzjetwnvaqeambcgleplatnbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqziambpzbsmgmvqdvzocwimnqjrhtjl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxblxrqznpcienbnxcouryhrndyxcevy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yghmkttttvsvorkrvsgthpahkxdsuvws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awfaerlseyzeauglexzwocrmnvdwasia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhrfusulpzfgqwjcfxxatrjjxttdppps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwhbxskvamtzmcbolmiiwubggwhglgfq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvzaldmmrldgtukvxybaysnuysmsoxgf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzvoismgyfmqrqzhqdquzxrladqunwof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jtjmgtecugunffegtorimkqetgprogpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667850,"databaseName":"models_schema","ddl":"CREATE TABLE `fqvrwbozmynffznjtosjaimyhkulylum` (\n `vnhdtastwtiivvlvtzgteygntwhuwgre` int NOT NULL,\n `gctzpwzckvnzwjcsxnixjfnsgkcklbun` int DEFAULT NULL,\n `cixhutlhjndltubbjcmjqqdwbslbenvl` int DEFAULT NULL,\n `leynaltwlvcituqyaluzskpmpslfzuzc` int DEFAULT NULL,\n `aqziucyjuibchxhqviqezvqtrgrwtvpl` int DEFAULT NULL,\n `uophmtkmqygfhdtupmwouajqxoepcmkg` int DEFAULT NULL,\n `nqidhuvpnehzuaimxxasbleafazhlbqf` int DEFAULT NULL,\n `klszmspymgrxjsjxntnkbqphecsivmca` int DEFAULT NULL,\n `ndxygrtwaxulrodaeczdlllezymnzgex` int DEFAULT NULL,\n `kmzzsvghxzhvajjmnyqbxijecztvgslm` int DEFAULT NULL,\n `kjdnznhinizmstvrtjcxgiqrnaradzas` int DEFAULT NULL,\n `ssicatdzjdquchrfwqoglkmavsyccmjr` int DEFAULT NULL,\n `hdhmrgfoswiozcplmexfpygjtttaxyyt` int DEFAULT NULL,\n `bymmsfrmtfzocexctshryfsbrgcyhcsc` int DEFAULT NULL,\n `erbeudhqesbklgggictrysvrjjmykcox` int DEFAULT NULL,\n `tlnjxwnquvnscagjtocudbdhbkutenku` int DEFAULT NULL,\n `xszexffixbtdzgxzadawixqsuqxcwnur` int DEFAULT NULL,\n `mddovxhwmshxmqucitwtnwvjzkipgovu` int DEFAULT NULL,\n `rbuwtyfvpfbltotqrnbdagbhdgltqmar` int DEFAULT NULL,\n `vlwlohswuabxmcxepekhyikfdjdvmnan` int DEFAULT NULL,\n `cqnxxzoghmncpvpemedxhpbqypypkupd` int DEFAULT NULL,\n `rqgdnwwpaumzoxpiohgxcsynbccedsvk` int DEFAULT NULL,\n `hfqgsehryydxfcdjvhomntjulfpfxftc` int DEFAULT NULL,\n `ayfigjtjrntkyqxserlpzetatyupmqgk` int DEFAULT NULL,\n `xlqrqisuobzpdjaotuwilplockonmbxk` int DEFAULT NULL,\n `xfpkpeuyjvovyqcnpkajuiejxfaplrnw` int DEFAULT NULL,\n `hegrjbxqicipfroplvmrvsmeoindukng` int DEFAULT NULL,\n `kwzggafksobjqfzovuzrisotlbfnusxy` int DEFAULT NULL,\n `nwvifhoegyvjqgzglhodrapuozlvrqvz` int DEFAULT NULL,\n `rrmxghgbgxsvtqudumzyobvntrkzzusv` int DEFAULT NULL,\n `unlpranyghpxhqdisfnkbxomruyvccda` int DEFAULT NULL,\n `zqzmotlzgmpfnoqbmublvzqyxzshrxss` int DEFAULT NULL,\n `nhaervxffskqixdulcinxoohgvvpzwne` int DEFAULT NULL,\n `gkvpwxyoruvjrzaeutjnqbxnaicqfaqy` int DEFAULT NULL,\n `wwhbvndgtbqxafcgymemsovzmuhjqggb` int DEFAULT NULL,\n `osmwgyshnwlafisdceiyedazsffmgphy` int DEFAULT NULL,\n `xphthmrdsxsczcnewngjmlqmwixjrhsr` int DEFAULT NULL,\n `tcvmgktoaotjrwtimwwaamipmdhofjog` int DEFAULT NULL,\n `rxvaennjzxtndgssrjrdozemacxiwdyb` int DEFAULT NULL,\n `uyoqkkhdowfhsccjyjfpgljdmgmtbkxt` int DEFAULT NULL,\n `paeattiftrptpymaiwpcdpkgcwovaipf` int DEFAULT NULL,\n `azkykfolyqdqtekwutfbkypkubvpcabm` int DEFAULT NULL,\n `rrizoxkkxjtzkazearwryzprdotrimeq` int DEFAULT NULL,\n `cpunyfzmpseaznrgxqnppwyqdtlynyye` int DEFAULT NULL,\n `tjzmmduxfmzejbqdbpvrtxuhgrkxhsto` int DEFAULT NULL,\n `akpzdyvwstslfvdofwondhkfxmplipwe` int DEFAULT NULL,\n `temkehatwckcppaidwrtdegkcyupleec` int DEFAULT NULL,\n `xhawurbwhchnewfztayfechtvghpvpwm` int DEFAULT NULL,\n `mcwapnvhtffcrwwljdstlmbixzukxrlp` int DEFAULT NULL,\n `pwozufuklfuxrsgqctnyxjxcmnruhniu` int DEFAULT NULL,\n `qoykdgjhffsogmbxqmcxkadiricbgacq` int DEFAULT NULL,\n `asqxegslzglmwkempxinybpsheriadbv` int DEFAULT NULL,\n `gciyuvpgzvjsqztpjejjmpczylsjxmry` int DEFAULT NULL,\n `dsgnzqnymueimbmoebzbfsmliadnndkx` int DEFAULT NULL,\n `omihsgnnudumgepgffjjgmnesuwlgdst` int DEFAULT NULL,\n `zdoxinjpzqorniwjfzayluivtnmqjpfn` int DEFAULT NULL,\n `drbxalavgxtqhtiehvhhandwqvgcamgv` int DEFAULT NULL,\n `xvzcqtsjywkrpxywfvyfyepesicjmqxu` int DEFAULT NULL,\n `zakqpuwliaxdxrprshphcajlwaqavcux` int DEFAULT NULL,\n `hqbrbczrhhsngjredzgbwwspwxlcllkk` int DEFAULT NULL,\n `gutwffyrcpjnylttkmagallhxcpvohmu` int DEFAULT NULL,\n `iqkqtsjcnszfeuxvwqdcisrytgiueqet` int DEFAULT NULL,\n `zdyzjbrxlgyncqkiqihjrlgmybgjqdmu` int DEFAULT NULL,\n `lemthzzofvwsxmpzvozpqunrhyaswnjb` int DEFAULT NULL,\n `wjrpocckhpzbhshavhmasmgrrdymefef` int DEFAULT NULL,\n `bplkmqoqtyxteizxzyossczcnlmkonug` int DEFAULT NULL,\n `ivkwkmvbcjdfxedjljnztlikknzmudwi` int DEFAULT NULL,\n `toqbqringecnkzxugzzlfqfgrnyxmcqt` int DEFAULT NULL,\n `npvqlgkqzupasmodvrwgpdmrdyatjbnb` int DEFAULT NULL,\n `bqxifwcdrbzlksswmavisqrrogvidupa` int DEFAULT NULL,\n `pevexdbvspnzrjrjnjserygtnsqmtsss` int DEFAULT NULL,\n `rdlegbdnxxddkrioaoacpapgscfyikwu` int DEFAULT NULL,\n `ohwldpjjrvofbwymajohozkzmjwtnggu` int DEFAULT NULL,\n `piobyijfxsfryvnsbnbsqvuiayhlwopf` int DEFAULT NULL,\n `rbhidxmtzvkyvycmwwmdpbtzealapnon` int DEFAULT NULL,\n `jthsrcppoeufdzkvodgvnxqgxzzxhdvu` int DEFAULT NULL,\n `fzjhjzvjzfczgnqshjslyxkqysfblktx` int DEFAULT NULL,\n `nliyfpaxmdxjquakkmuimsnztvoyhdvq` int DEFAULT NULL,\n `atrndxrufxkcasimlwyqdmdpasaigxam` int DEFAULT NULL,\n `keezvxcqxwuvlvxmffrgqmhybkxbhtsb` int DEFAULT NULL,\n `vsyqrlwurleqxvkcfdvbgivnfvreagbg` int DEFAULT NULL,\n `gtnihljfnhfcngspugrqftrhbdxbfqya` int DEFAULT NULL,\n `zbteazqrqsnlkijoxxottxhgyfyhfjjy` int DEFAULT NULL,\n `igvbqrovtikvwlqmjhfxnqdcwxjekuhq` int DEFAULT NULL,\n `mudctyqtxsahzcgubczxjmjfuthucntb` int DEFAULT NULL,\n `twuylnjcedykkmllugyteyeqcnwiurcm` int DEFAULT NULL,\n `rmxlwcgsfmavgxfhuxgyagtexsdfzbwl` int DEFAULT NULL,\n `zwxbbuixjghidmckpsrstrndxtixernj` int DEFAULT NULL,\n `fxiioxonotomcxcycceepghkvfyuzrku` int DEFAULT NULL,\n `orncfwenskvyddvtklctcgviuopwdffe` int DEFAULT NULL,\n `chaotidhqlopwidowtkkbmjqtslktwrb` int DEFAULT NULL,\n `zqmjdisjjrvfwsgqizxyofhsuulrngkm` int DEFAULT NULL,\n `hyfzxfnjsexjcsjemujkwwkklsbcxbbt` int DEFAULT NULL,\n `ydbrzrzrjhsjtjhkipivjokupiyhamxh` int DEFAULT NULL,\n `pvdonyemysptwgkithznqpjtoozfmgeh` int DEFAULT NULL,\n `ieqebjittjkwcihrvfgxfapmjzfltilh` int DEFAULT NULL,\n `ckubmtqcarunowzkvrrkfdmqkfuwymyk` int DEFAULT NULL,\n `vcfmluwmtmugipfspgleswzhnvgfsfot` int DEFAULT NULL,\n `vsfrgzkhwyrwfazzhwljrnkbkhusupqx` int DEFAULT NULL,\n `bbczgbxsrxdxpqtcvyvdexdmvlqleroa` int DEFAULT NULL,\n PRIMARY KEY (`vnhdtastwtiivvlvtzgteygntwhuwgre`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"fqvrwbozmynffznjtosjaimyhkulylum\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["vnhdtastwtiivvlvtzgteygntwhuwgre"],"columns":[{"name":"vnhdtastwtiivvlvtzgteygntwhuwgre","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"gctzpwzckvnzwjcsxnixjfnsgkcklbun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cixhutlhjndltubbjcmjqqdwbslbenvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"leynaltwlvcituqyaluzskpmpslfzuzc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqziucyjuibchxhqviqezvqtrgrwtvpl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uophmtkmqygfhdtupmwouajqxoepcmkg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqidhuvpnehzuaimxxasbleafazhlbqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"klszmspymgrxjsjxntnkbqphecsivmca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ndxygrtwaxulrodaeczdlllezymnzgex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmzzsvghxzhvajjmnyqbxijecztvgslm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjdnznhinizmstvrtjcxgiqrnaradzas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ssicatdzjdquchrfwqoglkmavsyccmjr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdhmrgfoswiozcplmexfpygjtttaxyyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bymmsfrmtfzocexctshryfsbrgcyhcsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"erbeudhqesbklgggictrysvrjjmykcox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlnjxwnquvnscagjtocudbdhbkutenku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xszexffixbtdzgxzadawixqsuqxcwnur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mddovxhwmshxmqucitwtnwvjzkipgovu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbuwtyfvpfbltotqrnbdagbhdgltqmar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vlwlohswuabxmcxepekhyikfdjdvmnan","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqnxxzoghmncpvpemedxhpbqypypkupd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqgdnwwpaumzoxpiohgxcsynbccedsvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfqgsehryydxfcdjvhomntjulfpfxftc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ayfigjtjrntkyqxserlpzetatyupmqgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xlqrqisuobzpdjaotuwilplockonmbxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xfpkpeuyjvovyqcnpkajuiejxfaplrnw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hegrjbxqicipfroplvmrvsmeoindukng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwzggafksobjqfzovuzrisotlbfnusxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nwvifhoegyvjqgzglhodrapuozlvrqvz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrmxghgbgxsvtqudumzyobvntrkzzusv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unlpranyghpxhqdisfnkbxomruyvccda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqzmotlzgmpfnoqbmublvzqyxzshrxss","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhaervxffskqixdulcinxoohgvvpzwne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkvpwxyoruvjrzaeutjnqbxnaicqfaqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwhbvndgtbqxafcgymemsovzmuhjqggb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"osmwgyshnwlafisdceiyedazsffmgphy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xphthmrdsxsczcnewngjmlqmwixjrhsr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tcvmgktoaotjrwtimwwaamipmdhofjog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxvaennjzxtndgssrjrdozemacxiwdyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uyoqkkhdowfhsccjyjfpgljdmgmtbkxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"paeattiftrptpymaiwpcdpkgcwovaipf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azkykfolyqdqtekwutfbkypkubvpcabm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrizoxkkxjtzkazearwryzprdotrimeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cpunyfzmpseaznrgxqnppwyqdtlynyye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjzmmduxfmzejbqdbpvrtxuhgrkxhsto","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akpzdyvwstslfvdofwondhkfxmplipwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"temkehatwckcppaidwrtdegkcyupleec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xhawurbwhchnewfztayfechtvghpvpwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcwapnvhtffcrwwljdstlmbixzukxrlp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwozufuklfuxrsgqctnyxjxcmnruhniu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qoykdgjhffsogmbxqmcxkadiricbgacq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"asqxegslzglmwkempxinybpsheriadbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gciyuvpgzvjsqztpjejjmpczylsjxmry","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsgnzqnymueimbmoebzbfsmliadnndkx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omihsgnnudumgepgffjjgmnesuwlgdst","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdoxinjpzqorniwjfzayluivtnmqjpfn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drbxalavgxtqhtiehvhhandwqvgcamgv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvzcqtsjywkrpxywfvyfyepesicjmqxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zakqpuwliaxdxrprshphcajlwaqavcux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqbrbczrhhsngjredzgbwwspwxlcllkk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gutwffyrcpjnylttkmagallhxcpvohmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqkqtsjcnszfeuxvwqdcisrytgiueqet","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdyzjbrxlgyncqkiqihjrlgmybgjqdmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lemthzzofvwsxmpzvozpqunrhyaswnjb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjrpocckhpzbhshavhmasmgrrdymefef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bplkmqoqtyxteizxzyossczcnlmkonug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivkwkmvbcjdfxedjljnztlikknzmudwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"toqbqringecnkzxugzzlfqfgrnyxmcqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npvqlgkqzupasmodvrwgpdmrdyatjbnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqxifwcdrbzlksswmavisqrrogvidupa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pevexdbvspnzrjrjnjserygtnsqmtsss","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdlegbdnxxddkrioaoacpapgscfyikwu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohwldpjjrvofbwymajohozkzmjwtnggu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"piobyijfxsfryvnsbnbsqvuiayhlwopf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbhidxmtzvkyvycmwwmdpbtzealapnon","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jthsrcppoeufdzkvodgvnxqgxzzxhdvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzjhjzvjzfczgnqshjslyxkqysfblktx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nliyfpaxmdxjquakkmuimsnztvoyhdvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atrndxrufxkcasimlwyqdmdpasaigxam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"keezvxcqxwuvlvxmffrgqmhybkxbhtsb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsyqrlwurleqxvkcfdvbgivnfvreagbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtnihljfnhfcngspugrqftrhbdxbfqya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbteazqrqsnlkijoxxottxhgyfyhfjjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igvbqrovtikvwlqmjhfxnqdcwxjekuhq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mudctyqtxsahzcgubczxjmjfuthucntb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twuylnjcedykkmllugyteyeqcnwiurcm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rmxlwcgsfmavgxfhuxgyagtexsdfzbwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zwxbbuixjghidmckpsrstrndxtixernj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxiioxonotomcxcycceepghkvfyuzrku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orncfwenskvyddvtklctcgviuopwdffe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chaotidhqlopwidowtkkbmjqtslktwrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqmjdisjjrvfwsgqizxyofhsuulrngkm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hyfzxfnjsexjcsjemujkwwkklsbcxbbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydbrzrzrjhsjtjhkipivjokupiyhamxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvdonyemysptwgkithznqpjtoozfmgeh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ieqebjittjkwcihrvfgxfapmjzfltilh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckubmtqcarunowzkvrrkfdmqkfuwymyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vcfmluwmtmugipfspgleswzhnvgfsfot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsfrgzkhwyrwfazzhwljrnkbkhusupqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbczgbxsrxdxpqtcvyvdexdmvlqleroa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667881,"databaseName":"models_schema","ddl":"CREATE TABLE `fughcvpxapxpanhpwimukohmafwcback` (\n `filjppxtosneqwdirvbtldirwxnddftc` int NOT NULL,\n `eurjlbxdksjzphyxaubpuaouconhvkix` int DEFAULT NULL,\n `gilixhtmffngztfdueaiggncsqmxzcpz` int DEFAULT NULL,\n `xoboysnwvvfgfdehnukigbiurtcnlsop` int DEFAULT NULL,\n `nwckkctstskkytrseuxqqkqsqnmgeykf` int DEFAULT NULL,\n `vkyjlrvyhbxhuzdqoceuwjlbyhrucazu` int DEFAULT NULL,\n `qofskxghnylylwbjuyctghqepvijgbpf` int DEFAULT NULL,\n `gapxzsfoeqbnjlbehmdfljftzvhlziro` int DEFAULT NULL,\n `erfhhrdthsmvdnteltwampjscybuglij` int DEFAULT NULL,\n `trtlowbjzebdduutwunzwfxsnckeeoev` int DEFAULT NULL,\n `pozggkidlutyodizdaxbzirscpmhjroc` int DEFAULT NULL,\n `whnoczovzxbiksnblwdtvzfanakruysn` int DEFAULT NULL,\n `tywxtbcpprzozidagyzokhlbjaedcyxm` int DEFAULT NULL,\n `npsalkhtehzjhfkbutarzgkwytluhbjm` int DEFAULT NULL,\n `fwcdelaemjcgbbjtvvptwnjwwcoutkjh` int DEFAULT NULL,\n `bhybwxsigzskhvdvidabooljtibvcute` int DEFAULT NULL,\n `ptaaxxhhkdnixpptuviuaixgfvfscufo` int DEFAULT NULL,\n `zcwzqfesrurwcptrzwcyirsbsdmmyfir` int DEFAULT NULL,\n `qnfyzuxgwfkxbrybvhzevzxiwchjsptz` int DEFAULT NULL,\n `juagysjlrzaivconsyaeyaajudcjisgf` int DEFAULT NULL,\n `pqlbnbosrtcrudjkfmhnqgilqomvrgbm` int DEFAULT NULL,\n `exmucgzmvtiysvnsdlwcxxpjnysmpqar` int DEFAULT NULL,\n `ncisopalbogsefnjolzzwvtfcaxnvjbf` int DEFAULT NULL,\n `rnuqzabrqxcdojkwbjnhngeapcwdcazj` int DEFAULT NULL,\n `xlhhuhsxuioxqjjowyfoizhpxczydcpu` int DEFAULT NULL,\n `bdubglbivvusopznxckomardbloaahxl` int DEFAULT NULL,\n `uhhhdynieswersgwmdrzfmddypbxspql` int DEFAULT NULL,\n `zagmflyxawcsmehrzcowoytyqphzyeum` int DEFAULT NULL,\n `ovkshmzvjknyntqqxslxcgokenyljwbr` int DEFAULT NULL,\n `tkdxupebszvgsileykwnnnngogotxfcb` int DEFAULT NULL,\n `wtflrsfxqafncbaphqmoybsiiaaovkaj` int DEFAULT NULL,\n `thojhkghltwqportpihjthiijpibzwzd` int DEFAULT NULL,\n `krksehktozbzrdrovwyacqqnyzcpkcsw` int DEFAULT NULL,\n `idlwlqwjmyjwbibdtnltkwjnxvbmajjm` int DEFAULT NULL,\n `jhgjdrvlmlawlkuhygswpaawyyueryqm` int DEFAULT NULL,\n `fwfqynapputvfmzsipjngchzwxffwlly` int DEFAULT NULL,\n `kfdjhylexjwrxanbdctfjqslalutmtzx` int DEFAULT NULL,\n `nqyljrahmsgrbhhxalswswfyapltaqkp` int DEFAULT NULL,\n `lwtmlpfeanbgcagmdxsvsemgnottjhub` int DEFAULT NULL,\n `herdvmgimhdjlrhbasbxhisiaofdeswp` int DEFAULT NULL,\n `uvxyzyebjsyzeocnougavlsobibppycf` int DEFAULT NULL,\n `pblpigkztodciasxrshybiguabxgkbki` int DEFAULT NULL,\n `uyurrhgpkotbslcfghkkmlervvfkwrrk` int DEFAULT NULL,\n `rfeuwaxjzhcvbmmgosvgfulqqhfcentm` int DEFAULT NULL,\n `flgmlamzgfnwuqojymirxujywtgxqgym` int DEFAULT NULL,\n `lkgdktirlyxhusfhgjamazcoopuodbdp` int DEFAULT NULL,\n `duqngmqhordbzbdtjxzttvwlqmhutrhy` int DEFAULT NULL,\n `uvcedmlxdgfagdvahscojnpuhocplzwf` int DEFAULT NULL,\n `rdjxgfxsqmrndqkqysuinbtnkqlbjzkg` int DEFAULT NULL,\n `fobnxqrmipvdbhhfsrniijghfkrrxsbl` int DEFAULT NULL,\n `sbzuovwndefuenwqknagsqdbuhplsztz` int DEFAULT NULL,\n `goojedooicxuggpezbhblpkwojprpfyd` int DEFAULT NULL,\n `uecgktuvpkfrsfccqvmdlrurgnthvktp` int DEFAULT NULL,\n `goltbodqszzmdewwozfgmcqcofrkcidf` int DEFAULT NULL,\n `hcqxydewcliwsmlofontrnbcvkhsbxbk` int DEFAULT NULL,\n `ubklkapsgvktiqlcoznbcbzbjdxvxvtb` int DEFAULT NULL,\n `mpzbqreltfxnftjbphmfgbawssnlvsll` int DEFAULT NULL,\n `hmyhezvtnpkqjexqidyjylfusgdqklqv` int DEFAULT NULL,\n `wveogtxoxdpzplxwheugziprnuwbgnso` int DEFAULT NULL,\n `obmwgagefcffzghrshyjegsiijlusajl` int DEFAULT NULL,\n `exudcjvdiuijomiovzljehcnossyxieq` int DEFAULT NULL,\n `boidzixgdnvomgzybicamkaicgwbqcfp` int DEFAULT NULL,\n `uifrpdcixxjgxnyvpsqklwiyvlzyaeoe` int DEFAULT NULL,\n `stpokxrfbvyprhafuhwfbsnsskwtqivw` int DEFAULT NULL,\n `vyfszvfqsgibalyeesptieenkcbnavts` int DEFAULT NULL,\n `zdcyyobhebdunffiyejuxtbzdxgoodkb` int DEFAULT NULL,\n `mqtxgwukukddtcnpdhvbyddqzdudikml` int DEFAULT NULL,\n `gkajcxfnoasoqfmzngtanndsdjqdcmlg` int DEFAULT NULL,\n `gzspmnsredrgqcwnnlswobfqtqcugyfs` int DEFAULT NULL,\n `azfqlqzedqmtxqatlqhpkcdllpjirroi` int DEFAULT NULL,\n `doiirzpnqqlvjaxdreijbpmfrgwvnshf` int DEFAULT NULL,\n `kcjhuhdzmmwfzztphrhwfblveozurokf` int DEFAULT NULL,\n `snfkytmwrfwxbjojcqmyjmwshqmfztju` int DEFAULT NULL,\n `ecubrnyekeuktcxugvivdixcjnrbpzri` int DEFAULT NULL,\n `oylfxruyvyfjuiujopbpwqzdxfgcjiwu` int DEFAULT NULL,\n `zxkyivluxzflqjzchkntjrtfdtyzbpou` int DEFAULT NULL,\n `lnptusifrbfepvhrxazwmxqnnsslwfyr` int DEFAULT NULL,\n `xxdceudrmjckbsnbkhuelqieclvdooap` int DEFAULT NULL,\n `fastnbhxhyrrtlimfngfevldaciuekko` int DEFAULT NULL,\n `lyndanntoiwepozfpiwwwzhaaypqokdu` int DEFAULT NULL,\n `nqxaddxxxkbzghzrlixjlpoqmchuzpal` int DEFAULT NULL,\n `hssyutemxvsokmmrwfchtuscwnpiqsoq` int DEFAULT NULL,\n `chbotavwcyzfwibyzodjarrrxptvolvx` int DEFAULT NULL,\n `zgeqjnvvzjygvszbuqjelxqbmcayyfje` int DEFAULT NULL,\n `mvgvebixqyvieumzinhjayjlkjgnotym` int DEFAULT NULL,\n `eguuuijefftkwhapbsyjglpwbsfvxxmm` int DEFAULT NULL,\n `nvzvqpavxyvtzdpvbajdcyrdsbcdwwnc` int DEFAULT NULL,\n `kklzntxmswgeinkyrprkkoglhbzjpqrf` int DEFAULT NULL,\n `mipppzrogjcsuvhbmuuuothcwaiymavm` int DEFAULT NULL,\n `ugflnlvnqrwsfykbywcmdklchxfbvoze` int DEFAULT NULL,\n `mbtggolpxgcztdbawnldpqrbxzyqbpvv` int DEFAULT NULL,\n `pckbugjotilmyynnvuwmebqbeiytyjym` int DEFAULT NULL,\n `uobdebxklhitmnjvztylslgvzvmvujcd` int DEFAULT NULL,\n `nxxtaysteoxsuxkhzvbhafyssujpusdo` int DEFAULT NULL,\n `ddqudmmqflgwduzoqsjqlfqpszsqjctk` int DEFAULT NULL,\n `lknwxbabgbmuzarycvjjzxjqykexhysc` int DEFAULT NULL,\n `ykzjlzfylhujkknjbtghmvlwejkaatrc` int DEFAULT NULL,\n `zyqpzwpjbnhneacqiukxyigjvmjvlino` int DEFAULT NULL,\n `ttpzjcefpvimvgejyvdsusgtwyuqotkf` int DEFAULT NULL,\n `upfxixdfxxemgjgxhlnngykoswhwqxog` int DEFAULT NULL,\n PRIMARY KEY (`filjppxtosneqwdirvbtldirwxnddftc`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"fughcvpxapxpanhpwimukohmafwcback\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["filjppxtosneqwdirvbtldirwxnddftc"],"columns":[{"name":"filjppxtosneqwdirvbtldirwxnddftc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"eurjlbxdksjzphyxaubpuaouconhvkix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gilixhtmffngztfdueaiggncsqmxzcpz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xoboysnwvvfgfdehnukigbiurtcnlsop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nwckkctstskkytrseuxqqkqsqnmgeykf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkyjlrvyhbxhuzdqoceuwjlbyhrucazu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qofskxghnylylwbjuyctghqepvijgbpf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gapxzsfoeqbnjlbehmdfljftzvhlziro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"erfhhrdthsmvdnteltwampjscybuglij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"trtlowbjzebdduutwunzwfxsnckeeoev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pozggkidlutyodizdaxbzirscpmhjroc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"whnoczovzxbiksnblwdtvzfanakruysn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tywxtbcpprzozidagyzokhlbjaedcyxm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npsalkhtehzjhfkbutarzgkwytluhbjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwcdelaemjcgbbjtvvptwnjwwcoutkjh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bhybwxsigzskhvdvidabooljtibvcute","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ptaaxxhhkdnixpptuviuaixgfvfscufo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zcwzqfesrurwcptrzwcyirsbsdmmyfir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qnfyzuxgwfkxbrybvhzevzxiwchjsptz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"juagysjlrzaivconsyaeyaajudcjisgf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqlbnbosrtcrudjkfmhnqgilqomvrgbm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exmucgzmvtiysvnsdlwcxxpjnysmpqar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ncisopalbogsefnjolzzwvtfcaxnvjbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnuqzabrqxcdojkwbjnhngeapcwdcazj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xlhhuhsxuioxqjjowyfoizhpxczydcpu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdubglbivvusopznxckomardbloaahxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhhhdynieswersgwmdrzfmddypbxspql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zagmflyxawcsmehrzcowoytyqphzyeum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovkshmzvjknyntqqxslxcgokenyljwbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkdxupebszvgsileykwnnnngogotxfcb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtflrsfxqafncbaphqmoybsiiaaovkaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thojhkghltwqportpihjthiijpibzwzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krksehktozbzrdrovwyacqqnyzcpkcsw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idlwlqwjmyjwbibdtnltkwjnxvbmajjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jhgjdrvlmlawlkuhygswpaawyyueryqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwfqynapputvfmzsipjngchzwxffwlly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfdjhylexjwrxanbdctfjqslalutmtzx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqyljrahmsgrbhhxalswswfyapltaqkp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwtmlpfeanbgcagmdxsvsemgnottjhub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"herdvmgimhdjlrhbasbxhisiaofdeswp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvxyzyebjsyzeocnougavlsobibppycf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pblpigkztodciasxrshybiguabxgkbki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uyurrhgpkotbslcfghkkmlervvfkwrrk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfeuwaxjzhcvbmmgosvgfulqqhfcentm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flgmlamzgfnwuqojymirxujywtgxqgym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lkgdktirlyxhusfhgjamazcoopuodbdp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"duqngmqhordbzbdtjxzttvwlqmhutrhy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvcedmlxdgfagdvahscojnpuhocplzwf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdjxgfxsqmrndqkqysuinbtnkqlbjzkg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fobnxqrmipvdbhhfsrniijghfkrrxsbl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbzuovwndefuenwqknagsqdbuhplsztz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"goojedooicxuggpezbhblpkwojprpfyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uecgktuvpkfrsfccqvmdlrurgnthvktp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"goltbodqszzmdewwozfgmcqcofrkcidf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hcqxydewcliwsmlofontrnbcvkhsbxbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubklkapsgvktiqlcoznbcbzbjdxvxvtb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpzbqreltfxnftjbphmfgbawssnlvsll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmyhezvtnpkqjexqidyjylfusgdqklqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wveogtxoxdpzplxwheugziprnuwbgnso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obmwgagefcffzghrshyjegsiijlusajl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exudcjvdiuijomiovzljehcnossyxieq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"boidzixgdnvomgzybicamkaicgwbqcfp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uifrpdcixxjgxnyvpsqklwiyvlzyaeoe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"stpokxrfbvyprhafuhwfbsnsskwtqivw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vyfszvfqsgibalyeesptieenkcbnavts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdcyyobhebdunffiyejuxtbzdxgoodkb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqtxgwukukddtcnpdhvbyddqzdudikml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkajcxfnoasoqfmzngtanndsdjqdcmlg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzspmnsredrgqcwnnlswobfqtqcugyfs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azfqlqzedqmtxqatlqhpkcdllpjirroi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"doiirzpnqqlvjaxdreijbpmfrgwvnshf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcjhuhdzmmwfzztphrhwfblveozurokf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"snfkytmwrfwxbjojcqmyjmwshqmfztju","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecubrnyekeuktcxugvivdixcjnrbpzri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oylfxruyvyfjuiujopbpwqzdxfgcjiwu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxkyivluxzflqjzchkntjrtfdtyzbpou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lnptusifrbfepvhrxazwmxqnnsslwfyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxdceudrmjckbsnbkhuelqieclvdooap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fastnbhxhyrrtlimfngfevldaciuekko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lyndanntoiwepozfpiwwwzhaaypqokdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqxaddxxxkbzghzrlixjlpoqmchuzpal","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hssyutemxvsokmmrwfchtuscwnpiqsoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chbotavwcyzfwibyzodjarrrxptvolvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zgeqjnvvzjygvszbuqjelxqbmcayyfje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvgvebixqyvieumzinhjayjlkjgnotym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eguuuijefftkwhapbsyjglpwbsfvxxmm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvzvqpavxyvtzdpvbajdcyrdsbcdwwnc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kklzntxmswgeinkyrprkkoglhbzjpqrf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mipppzrogjcsuvhbmuuuothcwaiymavm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugflnlvnqrwsfykbywcmdklchxfbvoze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbtggolpxgcztdbawnldpqrbxzyqbpvv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pckbugjotilmyynnvuwmebqbeiytyjym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uobdebxklhitmnjvztylslgvzvmvujcd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxxtaysteoxsuxkhzvbhafyssujpusdo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddqudmmqflgwduzoqsjqlfqpszsqjctk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lknwxbabgbmuzarycvjjzxjqykexhysc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykzjlzfylhujkknjbtghmvlwejkaatrc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyqpzwpjbnhneacqiukxyigjvmjvlino","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ttpzjcefpvimvgejyvdsusgtwyuqotkf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upfxixdfxxemgjgxhlnngykoswhwqxog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667922,"databaseName":"models_schema","ddl":"CREATE TABLE `fvwhkwjojhrpuruerizhsjyneudltdgf` (\n `bnltojxizbcpwvlsnbbpgfxkejgtrayn` int NOT NULL,\n `vfjuleitvkfipthedemrpabvkrhgkdcs` int DEFAULT NULL,\n `rpmhstpzgimqtnwpvwpbktgufklkdmyj` int DEFAULT NULL,\n `titrfdbxpblbhirjdjdwyjouehpbqoie` int DEFAULT NULL,\n `izrnfterbdkiaixvjaklikgxalyzenhf` int DEFAULT NULL,\n `huvokdwsxxypxvsqltewlxyzlcceeqwp` int DEFAULT NULL,\n `jnjcpquqhbwmjqshgqxpzwtjdgboaywg` int DEFAULT NULL,\n `ylpsaqxyyazrjmzkduisqmmxqvojvwyp` int DEFAULT NULL,\n `ljlzmztgpthvelqymmrcxgaxqkfhdhpb` int DEFAULT NULL,\n `fgvymmlsqtqpxywidpvrnkyltftpunlv` int DEFAULT NULL,\n `hkgllskvnhereyssdmjgswiluhspoykm` int DEFAULT NULL,\n `eimrcgptecvklukblvgjyiwlfpikvfoy` int DEFAULT NULL,\n `pbsknurjptludbzwgjqllnbkcvjvprwv` int DEFAULT NULL,\n `qzrfxciohwqejqmsgyknpuaxksmaglcg` int DEFAULT NULL,\n `wxfqwtdmndbrzfufoghhigytuccanehs` int DEFAULT NULL,\n `hlkrielfqquewjqozpnrcdhenhlvqpvd` int DEFAULT NULL,\n `xovmuooiwrrxtfwkncoztphscekfbxlk` int DEFAULT NULL,\n `xdzckmyixjchadlutqxmkmyeaxxcrrmh` int DEFAULT NULL,\n `hzktygzvbyfuitucqomevgiqgmobaohy` int DEFAULT NULL,\n `lcltdnwmjptcllwlqdejgopvopgddril` int DEFAULT NULL,\n `koohlnytrxsdafyruehqipubfjsylnzk` int DEFAULT NULL,\n `ydmduvplxlaohihjsnxcpduntodssvrr` int DEFAULT NULL,\n `dzgkvkhtvvuuegceogwrqgpjzptkvqfq` int DEFAULT NULL,\n `iqetvndagrxzwbkausgelcyvmylvmtsz` int DEFAULT NULL,\n `gjebpunywwbfvukerovdvztdrbohrqkp` int DEFAULT NULL,\n `qemryqhrtngpqjzeftlpkavijkmuruaf` int DEFAULT NULL,\n `ctebvlifxheymbdrxhcdivffscubaeme` int DEFAULT NULL,\n `zhknwldjattvdsrjbqsyeayjkeggemrc` int DEFAULT NULL,\n `byaggtswuxpeywyqwzbvxohyomcdowff` int DEFAULT NULL,\n `jqrwaboazyooexjzsemhrwyavwrdlqzb` int DEFAULT NULL,\n `xsgiduesjrjnhbgksgwnecmtxpwlouhs` int DEFAULT NULL,\n `kenifjznftwbexhuqjnscxumtvaoizqy` int DEFAULT NULL,\n `wvynmkbmszqtadlxxolidzjftzvzsyqq` int DEFAULT NULL,\n `rybiazupxcjtqwcnstoupwbulroydoho` int DEFAULT NULL,\n `vyqsbaywwvbtafdfmtvserlhzvclcaib` int DEFAULT NULL,\n `nbpbsewyiyrqkypbkovotihyjsvnsvgb` int DEFAULT NULL,\n `czwcplfsitdmqbburvfcqnuvitrivmzx` int DEFAULT NULL,\n `gzretvetqnhstwfdksypgjbkiixwxhdy` int DEFAULT NULL,\n `vhjeligjopahomqbstsetifudpqfikjj` int DEFAULT NULL,\n `eompsrcbofvdfomnxybqesdfjpndqqlu` int DEFAULT NULL,\n `gaicspldmlnoymwgdmptocifvovtfzxc` int DEFAULT NULL,\n `jpduptgtpcfmigdykugnfqlpeypucnbw` int DEFAULT NULL,\n `ptjbwpqubaotqpknhyjdfpoaqcowlckw` int DEFAULT NULL,\n `pyxaqobuexquewvbwnejcmnzkevoulpn` int DEFAULT NULL,\n `zderosuvjtasfnjbdjwsvrzjotiojkwc` int DEFAULT NULL,\n `zlacwllyhyirmjzorplaplhwhqctzdlw` int DEFAULT NULL,\n `lellbxrczvdnfcxxceikortjqsuzntoo` int DEFAULT NULL,\n `mmwghixmnabepyqfyzkvuvyblycgpksz` int DEFAULT NULL,\n `vpyhkcxxtyeklfukcshkfontngcudhqk` int DEFAULT NULL,\n `qntqlxwkokylsckmweawdunfynctyijg` int DEFAULT NULL,\n `rnyonwofezghbiiehyxtekrlnbmyindc` int DEFAULT NULL,\n `uramdkjfutrlidlvqhehwcxaotjqdlmn` int DEFAULT NULL,\n `bdezpeqrbnnzjmsoejgoiryibravimui` int DEFAULT NULL,\n `cnzcceorjszlidairnpiwttpjteqditg` int DEFAULT NULL,\n `xjshdefmbmcnmggbjstxkkmyfcumqsae` int DEFAULT NULL,\n `dqtihdvcxlfzvfizeqfynaiutveikiyr` int DEFAULT NULL,\n `rxlivcsyvfnbzdqtwwvkkvzkgyalrhhd` int DEFAULT NULL,\n `jmdglyfhrgnpwgswwblredctkrveqtmq` int DEFAULT NULL,\n `eblzsownwntmdatyhwzqxspjlnqopetf` int DEFAULT NULL,\n `vefcdsqgiqdmnzhfutfzstgywnyyovuz` int DEFAULT NULL,\n `srydnlzrscktlaindyhqkpfckpqsazcu` int DEFAULT NULL,\n `tmyrihozdjynnduhxokfaesubkwztyvp` int DEFAULT NULL,\n `atnmxhxorapbdclktvjyyglwtrkwrdbu` int DEFAULT NULL,\n `kccqppwzkositodtppevklxaolzyaldf` int DEFAULT NULL,\n `pcayobeswsqhkoheaalaogzzjhwsghim` int DEFAULT NULL,\n `ajrmrvfjruqbqjjbxmkinlhopxinusgv` int DEFAULT NULL,\n `cgfbqvlssnoubwcpnwhghlgjxsxcmqjh` int DEFAULT NULL,\n `hvyzjggsihfnwmtxmfzxxxzlnxfgmloz` int DEFAULT NULL,\n `dasyxgkhmdykjqqzhgrbnvrabsvcffgo` int DEFAULT NULL,\n `mcizdikegfksfpsfmhjpbpzhncdstuiv` int DEFAULT NULL,\n `twsmczxfdabitbextjfscsiolepxlucu` int DEFAULT NULL,\n `tsuanotapsvpmigubscbrmurdpxoqeck` int DEFAULT NULL,\n `hdkbtqveelgunvhissxfwkxxcxupiumu` int DEFAULT NULL,\n `bfessjxwjgsalnqzyqowijafodcjdfve` int DEFAULT NULL,\n `dhszygroqynzbeymumoipzqrtisoceqf` int DEFAULT NULL,\n `wdbdtlmhkliobbfxgwlbthnddsmhcgvx` int DEFAULT NULL,\n `qzgipnqxtlzyewokchcvjahelmxlitwc` int DEFAULT NULL,\n `svjjeurzjajjqazfugjfsqrfmilbxzao` int DEFAULT NULL,\n `zosowrxuqqfplbcypwlpkebtbncpkgta` int DEFAULT NULL,\n `rqaqimdddhalfyymxexfqyjgkoaulxra` int DEFAULT NULL,\n `tznzhhrdgmtdquqkgoemydbihfjsypal` int DEFAULT NULL,\n `hldmkakjyvryldlatvjcgjvgsjhyziqt` int DEFAULT NULL,\n `epqinwtughexuvegjlxdramkefkafxaf` int DEFAULT NULL,\n `edefrxphglbsxrrnzrmeosqpxppvlufw` int DEFAULT NULL,\n `hesriivxdkkwwpfxdywozngdsihnxxhg` int DEFAULT NULL,\n `tudnzlrikghguywjaftxleaapcsktfpd` int DEFAULT NULL,\n `nocvpnrjkyyxixxobbivinylohnvuqae` int DEFAULT NULL,\n `sfpqvsgdpbpaqzwodbnbiihtxoqunnau` int DEFAULT NULL,\n `lgqnhvkfeydmyjrmwegmwcefqjgewcdh` int DEFAULT NULL,\n `jqsuczjifpzizybckrvqafcjekkrybnu` int DEFAULT NULL,\n `ixavjtklnminrrhurpemzchnzzjadumi` int DEFAULT NULL,\n `pvzbleohvystjvtdogkyjjtcacnymwqy` int DEFAULT NULL,\n `kxtqpbtdupajctrqxxxiuiebtbmohzpp` int DEFAULT NULL,\n `nqcamorvrlodbnfmkwzjklgyjpiawfsx` int DEFAULT NULL,\n `hifclypongmllbpilgfxcrnppmneduyb` int DEFAULT NULL,\n `lfgjnolwlrkgneocvioiogttqulxccld` int DEFAULT NULL,\n `qhergpcccekjipymxaknpkusvufiwyzq` int DEFAULT NULL,\n `zamgdkxxhnlfvzihztoipgxyajuekxti` int DEFAULT NULL,\n `viuhmltwiblupmirjrwiafzwkksbwfbu` int DEFAULT NULL,\n `aienkprtgzlkkyensxjazsxbxupcitgv` int DEFAULT NULL,\n PRIMARY KEY (`bnltojxizbcpwvlsnbbpgfxkejgtrayn`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"fvwhkwjojhrpuruerizhsjyneudltdgf\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["bnltojxizbcpwvlsnbbpgfxkejgtrayn"],"columns":[{"name":"bnltojxizbcpwvlsnbbpgfxkejgtrayn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"vfjuleitvkfipthedemrpabvkrhgkdcs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpmhstpzgimqtnwpvwpbktgufklkdmyj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"titrfdbxpblbhirjdjdwyjouehpbqoie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"izrnfterbdkiaixvjaklikgxalyzenhf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"huvokdwsxxypxvsqltewlxyzlcceeqwp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jnjcpquqhbwmjqshgqxpzwtjdgboaywg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ylpsaqxyyazrjmzkduisqmmxqvojvwyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljlzmztgpthvelqymmrcxgaxqkfhdhpb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fgvymmlsqtqpxywidpvrnkyltftpunlv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkgllskvnhereyssdmjgswiluhspoykm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eimrcgptecvklukblvgjyiwlfpikvfoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbsknurjptludbzwgjqllnbkcvjvprwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzrfxciohwqejqmsgyknpuaxksmaglcg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxfqwtdmndbrzfufoghhigytuccanehs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlkrielfqquewjqozpnrcdhenhlvqpvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xovmuooiwrrxtfwkncoztphscekfbxlk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdzckmyixjchadlutqxmkmyeaxxcrrmh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hzktygzvbyfuitucqomevgiqgmobaohy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lcltdnwmjptcllwlqdejgopvopgddril","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"koohlnytrxsdafyruehqipubfjsylnzk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydmduvplxlaohihjsnxcpduntodssvrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dzgkvkhtvvuuegceogwrqgpjzptkvqfq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqetvndagrxzwbkausgelcyvmylvmtsz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjebpunywwbfvukerovdvztdrbohrqkp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qemryqhrtngpqjzeftlpkavijkmuruaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctebvlifxheymbdrxhcdivffscubaeme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhknwldjattvdsrjbqsyeayjkeggemrc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"byaggtswuxpeywyqwzbvxohyomcdowff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jqrwaboazyooexjzsemhrwyavwrdlqzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsgiduesjrjnhbgksgwnecmtxpwlouhs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kenifjznftwbexhuqjnscxumtvaoizqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvynmkbmszqtadlxxolidzjftzvzsyqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rybiazupxcjtqwcnstoupwbulroydoho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vyqsbaywwvbtafdfmtvserlhzvclcaib","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbpbsewyiyrqkypbkovotihyjsvnsvgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"czwcplfsitdmqbburvfcqnuvitrivmzx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzretvetqnhstwfdksypgjbkiixwxhdy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhjeligjopahomqbstsetifudpqfikjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eompsrcbofvdfomnxybqesdfjpndqqlu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gaicspldmlnoymwgdmptocifvovtfzxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpduptgtpcfmigdykugnfqlpeypucnbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ptjbwpqubaotqpknhyjdfpoaqcowlckw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pyxaqobuexquewvbwnejcmnzkevoulpn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zderosuvjtasfnjbdjwsvrzjotiojkwc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zlacwllyhyirmjzorplaplhwhqctzdlw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lellbxrczvdnfcxxceikortjqsuzntoo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmwghixmnabepyqfyzkvuvyblycgpksz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpyhkcxxtyeklfukcshkfontngcudhqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qntqlxwkokylsckmweawdunfynctyijg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnyonwofezghbiiehyxtekrlnbmyindc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uramdkjfutrlidlvqhehwcxaotjqdlmn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdezpeqrbnnzjmsoejgoiryibravimui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnzcceorjszlidairnpiwttpjteqditg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjshdefmbmcnmggbjstxkkmyfcumqsae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqtihdvcxlfzvfizeqfynaiutveikiyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxlivcsyvfnbzdqtwwvkkvzkgyalrhhd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmdglyfhrgnpwgswwblredctkrveqtmq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eblzsownwntmdatyhwzqxspjlnqopetf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vefcdsqgiqdmnzhfutfzstgywnyyovuz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"srydnlzrscktlaindyhqkpfckpqsazcu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmyrihozdjynnduhxokfaesubkwztyvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atnmxhxorapbdclktvjyyglwtrkwrdbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kccqppwzkositodtppevklxaolzyaldf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pcayobeswsqhkoheaalaogzzjhwsghim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajrmrvfjruqbqjjbxmkinlhopxinusgv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgfbqvlssnoubwcpnwhghlgjxsxcmqjh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hvyzjggsihfnwmtxmfzxxxzlnxfgmloz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dasyxgkhmdykjqqzhgrbnvrabsvcffgo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcizdikegfksfpsfmhjpbpzhncdstuiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twsmczxfdabitbextjfscsiolepxlucu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tsuanotapsvpmigubscbrmurdpxoqeck","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdkbtqveelgunvhissxfwkxxcxupiumu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bfessjxwjgsalnqzyqowijafodcjdfve","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhszygroqynzbeymumoipzqrtisoceqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wdbdtlmhkliobbfxgwlbthnddsmhcgvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzgipnqxtlzyewokchcvjahelmxlitwc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"svjjeurzjajjqazfugjfsqrfmilbxzao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zosowrxuqqfplbcypwlpkebtbncpkgta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqaqimdddhalfyymxexfqyjgkoaulxra","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tznzhhrdgmtdquqkgoemydbihfjsypal","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hldmkakjyvryldlatvjcgjvgsjhyziqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"epqinwtughexuvegjlxdramkefkafxaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edefrxphglbsxrrnzrmeosqpxppvlufw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hesriivxdkkwwpfxdywozngdsihnxxhg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tudnzlrikghguywjaftxleaapcsktfpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nocvpnrjkyyxixxobbivinylohnvuqae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfpqvsgdpbpaqzwodbnbiihtxoqunnau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgqnhvkfeydmyjrmwegmwcefqjgewcdh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jqsuczjifpzizybckrvqafcjekkrybnu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixavjtklnminrrhurpemzchnzzjadumi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvzbleohvystjvtdogkyjjtcacnymwqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxtqpbtdupajctrqxxxiuiebtbmohzpp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqcamorvrlodbnfmkwzjklgyjpiawfsx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hifclypongmllbpilgfxcrnppmneduyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfgjnolwlrkgneocvioiogttqulxccld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhergpcccekjipymxaknpkusvufiwyzq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zamgdkxxhnlfvzihztoipgxyajuekxti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"viuhmltwiblupmirjrwiafzwkksbwfbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aienkprtgzlkkyensxjazsxbxupcitgv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667959,"databaseName":"models_schema","ddl":"CREATE TABLE `fwahqjmgxjbvouzmzetprekihxwdeptf` (\n `yhmnskczqrpzrtvxtajilbxnzxvxxrso` int NOT NULL,\n `alzxjuljprddrljbvsgfwjxikdjqyxwb` int DEFAULT NULL,\n `tvlrburgxcuuqwrtgkzaqekqsitmnnuq` int DEFAULT NULL,\n `vwvhedcpvfznopitywjafpnggoaprswk` int DEFAULT NULL,\n `hcfniddfpjoxcrewsrxzzobnhjeggnjw` int DEFAULT NULL,\n `zgpwoxhjyfeoexpasspyfnfoccimtull` int DEFAULT NULL,\n `jiwoulnrzwbgfapopwgunrezqhmrbecq` int DEFAULT NULL,\n `ecotynshuylvocgengweuzswqimlfifo` int DEFAULT NULL,\n `bibeekfxvfhnfccxfpmqddnauprdosbi` int DEFAULT NULL,\n `ztwivkstvraatuepuybxrolndkrribci` int DEFAULT NULL,\n `hmpudfijuhcgtvvtzorajehkvafkwhin` int DEFAULT NULL,\n `nmwajgpwhyhllofejkkeqzjibhjscxtw` int DEFAULT NULL,\n `icembmghbusjkvimhswzzycecugpogzg` int DEFAULT NULL,\n `esayffwjeblqukficcgwhyvhmrtkfnpu` int DEFAULT NULL,\n `xoinqsfguluxwqupzqpfzqpxenozqzaj` int DEFAULT NULL,\n `wosxiumrlgkhvmlwmmgirygquzajcfkd` int DEFAULT NULL,\n `xnphzdgehgcayniwvsniwmosizdyplvl` int DEFAULT NULL,\n `zzypyvrtqrafvpgkvkrsmzhugkensazt` int DEFAULT NULL,\n `hmwyxzycircapdymcmkvsszxrnhtcujv` int DEFAULT NULL,\n `fkxrffkaruxkmhnrknctulhgvujqixum` int DEFAULT NULL,\n `pfiaqhumtucspzwbtvanpkchlgeazrav` int DEFAULT NULL,\n `fvditnvaasmrrgrekczocjzjnnagojtk` int DEFAULT NULL,\n `qyagnfgmioadqucuurntlpebzdqfhutq` int DEFAULT NULL,\n `jcxyggfsrmfaodhaodzptembarcfiaah` int DEFAULT NULL,\n `hggdfjkskfxhwjlesadqetopylpatfkz` int DEFAULT NULL,\n `rgaglsatkioljiukqfdzysbzzzwdhnid` int DEFAULT NULL,\n `vsqhwmmhdurxceawbecegeuyqacfcpbr` int DEFAULT NULL,\n `hmfghdvkabfvoprfnqenbzcobsiuzndm` int DEFAULT NULL,\n `wxwlfmppkfkgcgxbnorojtwmsxgpwcos` int DEFAULT NULL,\n `zxpnbqfznbeeksfnbdaeopyzxcnzqmmg` int DEFAULT NULL,\n `xfzvppozzzfdyvxjdfigocbbwbpqwltk` int DEFAULT NULL,\n `pyndbjqbiotilzxhpaoeibhzkbxgvuao` int DEFAULT NULL,\n `eepjkwrraiorfpilcqqwcokybqowzdeh` int DEFAULT NULL,\n `fgriqjmsgwyhpyirauseodkfwrgucdbv` int DEFAULT NULL,\n `wgkmtkhralpafqekrcxprnenfzqyalsr` int DEFAULT NULL,\n `yxfxbemgsezykmfqehrpwonnconpmgig` int DEFAULT NULL,\n `tftysdfzwfxmwjzfbpntbbdfwbtampmw` int DEFAULT NULL,\n `iiujhvdgkiwaxqhiycstettwoglnorsk` int DEFAULT NULL,\n `zwwxjdhnxwzgcltbzijkeqrrslbqzchr` int DEFAULT NULL,\n `ietvameiyorfhtggammnenlnkeatueia` int DEFAULT NULL,\n `dhuwcscvqgjzgbiggibtsxzjvwlzqxiz` int DEFAULT NULL,\n `wfnxekqpitmpxswcjdccdlobgsquzfte` int DEFAULT NULL,\n `imqsivrilzeshtbyqfbwkhtkfcqllrni` int DEFAULT NULL,\n `yzopubtdrsxrtfwsafcdywdltwwytwni` int DEFAULT NULL,\n `xbocupgerwmhuhehostkdsjwzfkuveid` int DEFAULT NULL,\n `kwbjatvbvbtefkdwkqbwcvhobpdtbdoa` int DEFAULT NULL,\n `ngeaeqaaxaurntsynkmaltxkpjztdyyh` int DEFAULT NULL,\n `jxfyisktghhdaxmgnpnsebnnmwcknjse` int DEFAULT NULL,\n `kbkywsdvatgrgpobqyjjvsimvhqfmcfr` int DEFAULT NULL,\n `hrmaiknsuttbagtftaqnwhgrwsqmmjcu` int DEFAULT NULL,\n `kzatuefcpzxqrytbemnujrngxndibvut` int DEFAULT NULL,\n `dypqgkrmdoxudnqtrcwvigojunqskizk` int DEFAULT NULL,\n `tnpsoisbxfwazzcnbrhmbvvpskylyiqo` int DEFAULT NULL,\n `ccouamndtxwvsvigvgfangtzydesmjqi` int DEFAULT NULL,\n `hgjrtwnzbhhieafufhncocflehkrswas` int DEFAULT NULL,\n `fjgcaqzwjfaegzylgmdgymxywidqtkxi` int DEFAULT NULL,\n `xdwzoevqahvdkbttazeiowmoxbbcwett` int DEFAULT NULL,\n `luyvbjzltlhqmhucdxqhbgzvtjnqkxcn` int DEFAULT NULL,\n `rbdrmlfxtcysxnltopyncjbfxfaadpjd` int DEFAULT NULL,\n `etwxfuwsvgmntwfhtuucbgnfeqbwjkla` int DEFAULT NULL,\n `ujkhoqjlaoeknymddknyyieyaedvdled` int DEFAULT NULL,\n `sdvpaqekqonqvfzuitdbrnzpjkuqidca` int DEFAULT NULL,\n `uulbmlaxlzedhlgetbrxccmndremzuxk` int DEFAULT NULL,\n `vdppbsoeexmxjxphobyzdilnbzfsxvcl` int DEFAULT NULL,\n `muvtdjsgashqqrgbvutojgcpeabwnoxb` int DEFAULT NULL,\n `rlnypumnqtdqrsynpphsekocyabjwzvz` int DEFAULT NULL,\n `zrtjujvotfjzjohmzdlsnywxsurvivvk` int DEFAULT NULL,\n `mhntplwwxigbunvmebwjbujjvfyrgfdy` int DEFAULT NULL,\n `oqqgrwkxtldqqyxewltnjziqaercqgza` int DEFAULT NULL,\n `qddesisrhawdimblsnfbyjwgqiaekdtt` int DEFAULT NULL,\n `sedijftwvspzijkmwxfnwskvajujsdxu` int DEFAULT NULL,\n `gwljyunchncghejubqeinkwvhyjanzll` int DEFAULT NULL,\n `gdkdexgyhrxtxjwiyvxrhchmvdnlbxpd` int DEFAULT NULL,\n `tkxbmioxrneutvgwmkhajinbahmhcsyf` int DEFAULT NULL,\n `irxsldtwhcbhzhbptourhdfrxrywrurz` int DEFAULT NULL,\n `prgjnwedxhresrlvxxkllbhrbeaeiupl` int DEFAULT NULL,\n `qryduecwlflcqrpufjoelpnxzppqlrvi` int DEFAULT NULL,\n `yxwsrjavlhhuhdenvabwdbwntrvevnio` int DEFAULT NULL,\n `erzgsufcwkqrzhvncasxoibxqtggdiza` int DEFAULT NULL,\n `tjsuvqvytauixxedhthaycvclqycfjwf` int DEFAULT NULL,\n `ibbohcpmibzkxtekfmbluflhrktcpblt` int DEFAULT NULL,\n `babcwxjhvpeigujaabdwrynwdlkwvese` int DEFAULT NULL,\n `tpsaltfofnykrvhjrpbrjnzwvrcsdike` int DEFAULT NULL,\n `wsycawdawznmeznphxapsstjlehcimhh` int DEFAULT NULL,\n `rjvthhyqrirxcikojgddrnzfsbzbpymw` int DEFAULT NULL,\n `hmquchwgptihrllswjlyyzbpwqbzckpb` int DEFAULT NULL,\n `pdjpvkvnmszaxzifjjomopaysueaugut` int DEFAULT NULL,\n `oinvkjdsvifaswxlaqygwynaabiggopb` int DEFAULT NULL,\n `hdzicnuwvhssdveabxsyjomwjcbndudu` int DEFAULT NULL,\n `mkmqdfwnffqtzezpzhbnzzvkjavvdlcl` int DEFAULT NULL,\n `bplqxpfdfbnxifmhfgirubtwwynwikuq` int DEFAULT NULL,\n `vmbsnzqtexoixakvvhojcihdcgxxwpax` int DEFAULT NULL,\n `fetsphqndklbwsomyrardvuwslzxhhnm` int DEFAULT NULL,\n `vrfpdmutnhzynsnqpautbqyfztwqtjwq` int DEFAULT NULL,\n `oloocoaqbtyrmihdhbjbnablinimlnex` int DEFAULT NULL,\n `yfmhlifgcvwsiqcxnbtshnlmaupdctqi` int DEFAULT NULL,\n `urlypnasubcoyorxoqwsdnuxwuyfsoui` int DEFAULT NULL,\n `yxrexwidkakjvncdfvdtfpbpczdjysfo` int DEFAULT NULL,\n `qujxmairowubjkroxhzuwqvlzuxdteqy` int DEFAULT NULL,\n `lscsoxtkzatshmwjdnsuuacwtntdfkbu` int DEFAULT NULL,\n PRIMARY KEY (`yhmnskczqrpzrtvxtajilbxnzxvxxrso`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"fwahqjmgxjbvouzmzetprekihxwdeptf\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["yhmnskczqrpzrtvxtajilbxnzxvxxrso"],"columns":[{"name":"yhmnskczqrpzrtvxtajilbxnzxvxxrso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"alzxjuljprddrljbvsgfwjxikdjqyxwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvlrburgxcuuqwrtgkzaqekqsitmnnuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vwvhedcpvfznopitywjafpnggoaprswk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hcfniddfpjoxcrewsrxzzobnhjeggnjw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zgpwoxhjyfeoexpasspyfnfoccimtull","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jiwoulnrzwbgfapopwgunrezqhmrbecq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecotynshuylvocgengweuzswqimlfifo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bibeekfxvfhnfccxfpmqddnauprdosbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ztwivkstvraatuepuybxrolndkrribci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmpudfijuhcgtvvtzorajehkvafkwhin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmwajgpwhyhllofejkkeqzjibhjscxtw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icembmghbusjkvimhswzzycecugpogzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"esayffwjeblqukficcgwhyvhmrtkfnpu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xoinqsfguluxwqupzqpfzqpxenozqzaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wosxiumrlgkhvmlwmmgirygquzajcfkd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnphzdgehgcayniwvsniwmosizdyplvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzypyvrtqrafvpgkvkrsmzhugkensazt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmwyxzycircapdymcmkvsszxrnhtcujv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fkxrffkaruxkmhnrknctulhgvujqixum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pfiaqhumtucspzwbtvanpkchlgeazrav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fvditnvaasmrrgrekczocjzjnnagojtk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qyagnfgmioadqucuurntlpebzdqfhutq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jcxyggfsrmfaodhaodzptembarcfiaah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hggdfjkskfxhwjlesadqetopylpatfkz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgaglsatkioljiukqfdzysbzzzwdhnid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsqhwmmhdurxceawbecegeuyqacfcpbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmfghdvkabfvoprfnqenbzcobsiuzndm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxwlfmppkfkgcgxbnorojtwmsxgpwcos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxpnbqfznbeeksfnbdaeopyzxcnzqmmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xfzvppozzzfdyvxjdfigocbbwbpqwltk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pyndbjqbiotilzxhpaoeibhzkbxgvuao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eepjkwrraiorfpilcqqwcokybqowzdeh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fgriqjmsgwyhpyirauseodkfwrgucdbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgkmtkhralpafqekrcxprnenfzqyalsr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxfxbemgsezykmfqehrpwonnconpmgig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tftysdfzwfxmwjzfbpntbbdfwbtampmw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iiujhvdgkiwaxqhiycstettwoglnorsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zwwxjdhnxwzgcltbzijkeqrrslbqzchr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ietvameiyorfhtggammnenlnkeatueia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhuwcscvqgjzgbiggibtsxzjvwlzqxiz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfnxekqpitmpxswcjdccdlobgsquzfte","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"imqsivrilzeshtbyqfbwkhtkfcqllrni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzopubtdrsxrtfwsafcdywdltwwytwni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbocupgerwmhuhehostkdsjwzfkuveid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwbjatvbvbtefkdwkqbwcvhobpdtbdoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ngeaeqaaxaurntsynkmaltxkpjztdyyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jxfyisktghhdaxmgnpnsebnnmwcknjse","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kbkywsdvatgrgpobqyjjvsimvhqfmcfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hrmaiknsuttbagtftaqnwhgrwsqmmjcu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzatuefcpzxqrytbemnujrngxndibvut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dypqgkrmdoxudnqtrcwvigojunqskizk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnpsoisbxfwazzcnbrhmbvvpskylyiqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ccouamndtxwvsvigvgfangtzydesmjqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hgjrtwnzbhhieafufhncocflehkrswas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjgcaqzwjfaegzylgmdgymxywidqtkxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdwzoevqahvdkbttazeiowmoxbbcwett","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"luyvbjzltlhqmhucdxqhbgzvtjnqkxcn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbdrmlfxtcysxnltopyncjbfxfaadpjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"etwxfuwsvgmntwfhtuucbgnfeqbwjkla","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujkhoqjlaoeknymddknyyieyaedvdled","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdvpaqekqonqvfzuitdbrnzpjkuqidca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uulbmlaxlzedhlgetbrxccmndremzuxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdppbsoeexmxjxphobyzdilnbzfsxvcl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"muvtdjsgashqqrgbvutojgcpeabwnoxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlnypumnqtdqrsynpphsekocyabjwzvz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zrtjujvotfjzjohmzdlsnywxsurvivvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhntplwwxigbunvmebwjbujjvfyrgfdy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqqgrwkxtldqqyxewltnjziqaercqgza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qddesisrhawdimblsnfbyjwgqiaekdtt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sedijftwvspzijkmwxfnwskvajujsdxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gwljyunchncghejubqeinkwvhyjanzll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdkdexgyhrxtxjwiyvxrhchmvdnlbxpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkxbmioxrneutvgwmkhajinbahmhcsyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"irxsldtwhcbhzhbptourhdfrxrywrurz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prgjnwedxhresrlvxxkllbhrbeaeiupl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qryduecwlflcqrpufjoelpnxzppqlrvi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxwsrjavlhhuhdenvabwdbwntrvevnio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"erzgsufcwkqrzhvncasxoibxqtggdiza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjsuvqvytauixxedhthaycvclqycfjwf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibbohcpmibzkxtekfmbluflhrktcpblt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"babcwxjhvpeigujaabdwrynwdlkwvese","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpsaltfofnykrvhjrpbrjnzwvrcsdike","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wsycawdawznmeznphxapsstjlehcimhh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjvthhyqrirxcikojgddrnzfsbzbpymw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmquchwgptihrllswjlyyzbpwqbzckpb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdjpvkvnmszaxzifjjomopaysueaugut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oinvkjdsvifaswxlaqygwynaabiggopb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdzicnuwvhssdveabxsyjomwjcbndudu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkmqdfwnffqtzezpzhbnzzvkjavvdlcl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bplqxpfdfbnxifmhfgirubtwwynwikuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmbsnzqtexoixakvvhojcihdcgxxwpax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fetsphqndklbwsomyrardvuwslzxhhnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrfpdmutnhzynsnqpautbqyfztwqtjwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oloocoaqbtyrmihdhbjbnablinimlnex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfmhlifgcvwsiqcxnbtshnlmaupdctqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urlypnasubcoyorxoqwsdnuxwuyfsoui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxrexwidkakjvncdfvdtfpbpczdjysfo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qujxmairowubjkroxhzuwqvlzuxdteqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lscsoxtkzatshmwjdnsuuacwtntdfkbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842667,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842667992,"databaseName":"models_schema","ddl":"CREATE TABLE `gchydfeczangasnmvfdlykinnhisenui` (\n `qtpxljkpyugjughgizwavobxtvucyvwf` int NOT NULL,\n `qbmloglibndxridpdplqbwkvppbrxqkv` int DEFAULT NULL,\n `upnsdqmaimdfuynlaolvxnggfofvzqve` int DEFAULT NULL,\n `ysowedpihzxxwqswfajzsvwduzroplgu` int DEFAULT NULL,\n `pshoiyxzrngrhlutvwbfrfyyayeeddtp` int DEFAULT NULL,\n `cbmhfzpejuodsystxmfsjlmvegsbyfuu` int DEFAULT NULL,\n `esifsgllvoahpriipjhuahmjigmcvvbe` int DEFAULT NULL,\n `qlwcraerbvuruotgaymbqhhlglbtqwws` int DEFAULT NULL,\n `birtdxsafrjejfjaotmevkhejxwogetx` int DEFAULT NULL,\n `mhxmjwkfgtholadngwknrhxnyfbgfncu` int DEFAULT NULL,\n `npyhwsukbwlgkekgddlfvqsrssyptrkw` int DEFAULT NULL,\n `whuotlwfbindlicopfzltyxjswmdxbnr` int DEFAULT NULL,\n `erofkfzzhhpdakefqgryenatzcamvrer` int DEFAULT NULL,\n `unliruqbqmczjsdtsqwzsbdryakjwvep` int DEFAULT NULL,\n `jvwujxktpudxmozgswjxmpywlmschnkr` int DEFAULT NULL,\n `zmwxtlcxddcrdvzhsxvhxzqmgkyeniyp` int DEFAULT NULL,\n `fpwuuhfpvrbeqbajvflzmvhzittmhtch` int DEFAULT NULL,\n `fgewsqfetlkpkzoencctdwmgdondsujk` int DEFAULT NULL,\n `pisqowpmzhwnprvdnabjvyqtdxpnxhaf` int DEFAULT NULL,\n `vizbekqkjhknpwxvjgtpvpqizssnprjh` int DEFAULT NULL,\n `lbqvtjkdwjpljuxuauiljiyphzhuwbuh` int DEFAULT NULL,\n `dbvdnvnazgpsiusgfyvnkytppkavdhtq` int DEFAULT NULL,\n `rcchgyvolpnobvkcnuxlqtrusnyxmckt` int DEFAULT NULL,\n `mquesndkykjtrbbmcphcpejnlyejqwlu` int DEFAULT NULL,\n `txpwvfuicwpvyhpyuyjoeozonptkswmo` int DEFAULT NULL,\n `jyuvgbbgclclaghdosiyklfwegmohgqd` int DEFAULT NULL,\n `wzhngcdnkxprbalmmthtwwykezuwovwa` int DEFAULT NULL,\n `xolclsgyluydbsxyjoykrlxhslafwuqb` int DEFAULT NULL,\n `ggefwuxroikoscvqiawbhbxzdxoymcfz` int DEFAULT NULL,\n `vhdvgihyqmmfmglraliilithpycgkpjq` int DEFAULT NULL,\n `scnhgoktworvgetyfacmdeynmaqxlscr` int DEFAULT NULL,\n `yxfgngopiiktzjyhgrglhhuvbortuhih` int DEFAULT NULL,\n `sfysuerkzirtqfayhvvplctbtkfzdung` int DEFAULT NULL,\n `efizvvrsvjgmsgkkjpuvkjtijxcjyzgy` int DEFAULT NULL,\n `fzcyldtjagavtkdxjyoxtxexqrnbfbbz` int DEFAULT NULL,\n `qdxsjiccepygouqcuwotzynbzmypayrn` int DEFAULT NULL,\n `uzeliqybotqkduaecrnourrckarduksi` int DEFAULT NULL,\n `gtuafeodeowdchxqahnrgzkyiefmsngk` int DEFAULT NULL,\n `xadkhlwaespruxwzkqhsibdfjejdezbg` int DEFAULT NULL,\n `ffmigivcxbabvtdsjokmewuuurptkkkx` int DEFAULT NULL,\n `tvyzhswgzdpcqhltliluotbpdjqyvkim` int DEFAULT NULL,\n `sztjlkaiywhlavomdjgfjotgyscmmbll` int DEFAULT NULL,\n `bspzzzaglhpkhvrwwfpwwmrcsbjabmut` int DEFAULT NULL,\n `hundvgddsznbrnikujtwjsbnhyccasax` int DEFAULT NULL,\n `qhmriwvrkitqgktdyujfktilsnsnsntm` int DEFAULT NULL,\n `psotaokeanlbtjtxbnidghkvbaplluyy` int DEFAULT NULL,\n `ijlgubikuhfsiqfcypunefvaoxrglzjk` int DEFAULT NULL,\n `ofazhbmxmcddbjjlsrhitbumoarkfehr` int DEFAULT NULL,\n `doxvpfgwqqtarjwfxqbaysfmubfjrbhg` int DEFAULT NULL,\n `pftomotfjkgykxuwmhmswiadnvbpuoyo` int DEFAULT NULL,\n `fqsmgugaypzqucxkvqkrdxtxkxmrvbnp` int DEFAULT NULL,\n `udskmyolljpngarhbnfibnjwoeldxbbx` int DEFAULT NULL,\n `kvvglcjwjrkjpvnbyxjrhxcitvhzyvtf` int DEFAULT NULL,\n `nmvyrwthxcmojrxslwywuhgczkypfyqu` int DEFAULT NULL,\n `eqeaeywretpwmalrelkydpxdyhaubhet` int DEFAULT NULL,\n `funxeingdynpphgfmujccyhhunqoeoeo` int DEFAULT NULL,\n `fubdmcrjoedsauqpyupleyxmnvinhgqc` int DEFAULT NULL,\n `xjkznqgvnfhvizacvifxrtvbnahlywgp` int DEFAULT NULL,\n `fgswnejbsizdxxpbfrgioslofmyyofwj` int DEFAULT NULL,\n `tztuizkarswkbotmvkjmxmohhagzlzuq` int DEFAULT NULL,\n `fsikrsubhufwmwnmitawaewaacbmnpbq` int DEFAULT NULL,\n `evvhgkkyuaglsxpmsdwludaenqufbgum` int DEFAULT NULL,\n `bfffhnmepfhsuxfnuoaaftktgzcrxfpc` int DEFAULT NULL,\n `siwucwhxmggkvzkfgvqgexphsecourfe` int DEFAULT NULL,\n `thxwucqekisifwsxrlvxgdwaiksfusqp` int DEFAULT NULL,\n `uzkpqzlhvschjsfkqszartenwlmppine` int DEFAULT NULL,\n `hhqxkpzgpupasnqoqlxabvjguezllpdi` int DEFAULT NULL,\n `tylxyzteriesvssignrgfjewbcillgyz` int DEFAULT NULL,\n `xutjlzkkkoqvmysiqecjormkowrcmgib` int DEFAULT NULL,\n `vemhbnveoxvbwghycjoowlftpubzdufx` int DEFAULT NULL,\n `upcleivrxdevhkxxoegdgbzlvckyqrkz` int DEFAULT NULL,\n `fwohfhghqfyedrnwflepfsluhkngdzyn` int DEFAULT NULL,\n `hlafzykeemdjgbxdgcbeuuxvduqxffuy` int DEFAULT NULL,\n `xlsgwhxwdubdpxtgdwipjyccsafjixof` int DEFAULT NULL,\n `tumdiydhbbglmcwsjiflydmkmbgdpjoz` int DEFAULT NULL,\n `cnpzdmjvaipmxjtsydpualjkqdctkmpz` int DEFAULT NULL,\n `rmspuljpsgmyjtuqajorinhynmznystr` int DEFAULT NULL,\n `koliiyjlqzabauvdybasoxmahvcoecjg` int DEFAULT NULL,\n `hnrufwoogxnhhrjaxemznpkzpogytmuy` int DEFAULT NULL,\n `zmtmmyrqdffarnyvvbrwgmcbmrckfftx` int DEFAULT NULL,\n `qeiaowqusdmdbogqqquaprxnxizufvfa` int DEFAULT NULL,\n `psvjdextjvnqujdryrqzxerhiqhxzcoz` int DEFAULT NULL,\n `idnufvvpgkhbpqzenxaorlzgojafmaev` int DEFAULT NULL,\n `ftsmtcnbwbhfeyhpvywpngygdggknost` int DEFAULT NULL,\n `isbrbvdyhqrijuwjyjbhthlwnfxnckof` int DEFAULT NULL,\n `fnyporrfnkyixbhlkmfvvqqqpiklxhtr` int DEFAULT NULL,\n `ckdjpiejpzuygmwqzwvkysufkihekgwg` int DEFAULT NULL,\n `ekskjlyvircmyvlpdazolarrwpnpoypo` int DEFAULT NULL,\n `gosvjvsenkikeuspemhlsduipgwmtrri` int DEFAULT NULL,\n `rvfkuctsepuswpfjsyjwlpnpbnanqctr` int DEFAULT NULL,\n `rhgddiciuodbkfjjgwmzldzfeehpjrff` int DEFAULT NULL,\n `buqphduhmhqkyluglwmliimezfehxdij` int DEFAULT NULL,\n `xnxctiwjcspczvrcdmntecruuadietwn` int DEFAULT NULL,\n `ddalxzszpvsjvhwqzohvopkhdsckrsoj` int DEFAULT NULL,\n `twricpifbqczvmfarxqwezobdeseyros` int DEFAULT NULL,\n `vlibikzqnnpxosmclbosperdvcegceib` int DEFAULT NULL,\n `eytaoamyphjphjvgfmxpvoinjcbpdexf` int DEFAULT NULL,\n `vjbuzcorimiihsugekusvtubrinstedh` int DEFAULT NULL,\n `dkylwqxrylakmkxenjcayxgfvthmrsek` int DEFAULT NULL,\n `tcvxcuedksjlvgqvhobjifvxpkdyalii` int DEFAULT NULL,\n PRIMARY KEY (`qtpxljkpyugjughgizwavobxtvucyvwf`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"gchydfeczangasnmvfdlykinnhisenui\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["qtpxljkpyugjughgizwavobxtvucyvwf"],"columns":[{"name":"qtpxljkpyugjughgizwavobxtvucyvwf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"qbmloglibndxridpdplqbwkvppbrxqkv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upnsdqmaimdfuynlaolvxnggfofvzqve","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ysowedpihzxxwqswfajzsvwduzroplgu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pshoiyxzrngrhlutvwbfrfyyayeeddtp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbmhfzpejuodsystxmfsjlmvegsbyfuu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"esifsgllvoahpriipjhuahmjigmcvvbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qlwcraerbvuruotgaymbqhhlglbtqwws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"birtdxsafrjejfjaotmevkhejxwogetx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhxmjwkfgtholadngwknrhxnyfbgfncu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npyhwsukbwlgkekgddlfvqsrssyptrkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"whuotlwfbindlicopfzltyxjswmdxbnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"erofkfzzhhpdakefqgryenatzcamvrer","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unliruqbqmczjsdtsqwzsbdryakjwvep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvwujxktpudxmozgswjxmpywlmschnkr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmwxtlcxddcrdvzhsxvhxzqmgkyeniyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpwuuhfpvrbeqbajvflzmvhzittmhtch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fgewsqfetlkpkzoencctdwmgdondsujk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pisqowpmzhwnprvdnabjvyqtdxpnxhaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vizbekqkjhknpwxvjgtpvpqizssnprjh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lbqvtjkdwjpljuxuauiljiyphzhuwbuh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbvdnvnazgpsiusgfyvnkytppkavdhtq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcchgyvolpnobvkcnuxlqtrusnyxmckt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mquesndkykjtrbbmcphcpejnlyejqwlu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txpwvfuicwpvyhpyuyjoeozonptkswmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jyuvgbbgclclaghdosiyklfwegmohgqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzhngcdnkxprbalmmthtwwykezuwovwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xolclsgyluydbsxyjoykrlxhslafwuqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggefwuxroikoscvqiawbhbxzdxoymcfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhdvgihyqmmfmglraliilithpycgkpjq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scnhgoktworvgetyfacmdeynmaqxlscr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxfgngopiiktzjyhgrglhhuvbortuhih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfysuerkzirtqfayhvvplctbtkfzdung","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"efizvvrsvjgmsgkkjpuvkjtijxcjyzgy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzcyldtjagavtkdxjyoxtxexqrnbfbbz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdxsjiccepygouqcuwotzynbzmypayrn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzeliqybotqkduaecrnourrckarduksi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtuafeodeowdchxqahnrgzkyiefmsngk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xadkhlwaespruxwzkqhsibdfjejdezbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffmigivcxbabvtdsjokmewuuurptkkkx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvyzhswgzdpcqhltliluotbpdjqyvkim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sztjlkaiywhlavomdjgfjotgyscmmbll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bspzzzaglhpkhvrwwfpwwmrcsbjabmut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hundvgddsznbrnikujtwjsbnhyccasax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhmriwvrkitqgktdyujfktilsnsnsntm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psotaokeanlbtjtxbnidghkvbaplluyy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijlgubikuhfsiqfcypunefvaoxrglzjk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofazhbmxmcddbjjlsrhitbumoarkfehr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"doxvpfgwqqtarjwfxqbaysfmubfjrbhg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pftomotfjkgykxuwmhmswiadnvbpuoyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqsmgugaypzqucxkvqkrdxtxkxmrvbnp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udskmyolljpngarhbnfibnjwoeldxbbx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvvglcjwjrkjpvnbyxjrhxcitvhzyvtf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmvyrwthxcmojrxslwywuhgczkypfyqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqeaeywretpwmalrelkydpxdyhaubhet","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"funxeingdynpphgfmujccyhhunqoeoeo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fubdmcrjoedsauqpyupleyxmnvinhgqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjkznqgvnfhvizacvifxrtvbnahlywgp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fgswnejbsizdxxpbfrgioslofmyyofwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tztuizkarswkbotmvkjmxmohhagzlzuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fsikrsubhufwmwnmitawaewaacbmnpbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evvhgkkyuaglsxpmsdwludaenqufbgum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bfffhnmepfhsuxfnuoaaftktgzcrxfpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"siwucwhxmggkvzkfgvqgexphsecourfe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thxwucqekisifwsxrlvxgdwaiksfusqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzkpqzlhvschjsfkqszartenwlmppine","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhqxkpzgpupasnqoqlxabvjguezllpdi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tylxyzteriesvssignrgfjewbcillgyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xutjlzkkkoqvmysiqecjormkowrcmgib","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vemhbnveoxvbwghycjoowlftpubzdufx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upcleivrxdevhkxxoegdgbzlvckyqrkz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwohfhghqfyedrnwflepfsluhkngdzyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlafzykeemdjgbxdgcbeuuxvduqxffuy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xlsgwhxwdubdpxtgdwipjyccsafjixof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tumdiydhbbglmcwsjiflydmkmbgdpjoz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnpzdmjvaipmxjtsydpualjkqdctkmpz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rmspuljpsgmyjtuqajorinhynmznystr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"koliiyjlqzabauvdybasoxmahvcoecjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hnrufwoogxnhhrjaxemznpkzpogytmuy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmtmmyrqdffarnyvvbrwgmcbmrckfftx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qeiaowqusdmdbogqqquaprxnxizufvfa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psvjdextjvnqujdryrqzxerhiqhxzcoz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idnufvvpgkhbpqzenxaorlzgojafmaev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftsmtcnbwbhfeyhpvywpngygdggknost","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"isbrbvdyhqrijuwjyjbhthlwnfxnckof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnyporrfnkyixbhlkmfvvqqqpiklxhtr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckdjpiejpzuygmwqzwvkysufkihekgwg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekskjlyvircmyvlpdazolarrwpnpoypo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gosvjvsenkikeuspemhlsduipgwmtrri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rvfkuctsepuswpfjsyjwlpnpbnanqctr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhgddiciuodbkfjjgwmzldzfeehpjrff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"buqphduhmhqkyluglwmliimezfehxdij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnxctiwjcspczvrcdmntecruuadietwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddalxzszpvsjvhwqzohvopkhdsckrsoj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twricpifbqczvmfarxqwezobdeseyros","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vlibikzqnnpxosmclbosperdvcegceib","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eytaoamyphjphjvgfmxpvoinjcbpdexf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjbuzcorimiihsugekusvtubrinstedh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkylwqxrylakmkxenjcayxgfvthmrsek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tcvxcuedksjlvgqvhobjifvxpkdyalii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668023,"databaseName":"models_schema","ddl":"CREATE TABLE `ghyqwlwuskidustonvfapzjfqxxirouo` (\n `halrxzvudetvsbrvsdfcswmnfsvkjaeo` int NOT NULL,\n `rpqhtkysyckfnnwohoxcpvkzpzhnbagw` int DEFAULT NULL,\n `yuidriumfjsawxzhxtnfawrazdrdwndg` int DEFAULT NULL,\n `vmgkolwmoraybparktfyrqumltribrci` int DEFAULT NULL,\n `aqljoptkzfkqhyzbdsxkrvbxzjbzgzqt` int DEFAULT NULL,\n `orlpmrroytcmvpkqjxgyrajrimphelna` int DEFAULT NULL,\n `suvbgahqzvvagluqlxxotexgxshclplx` int DEFAULT NULL,\n `nagfrzpqdkhuivoljvsgpethlnnbeitb` int DEFAULT NULL,\n `wnosvjwivxsffzgfnhzvznbtpvsyrrcp` int DEFAULT NULL,\n `juslbyeifyveixbpxluqhkksbiurecam` int DEFAULT NULL,\n `qppdzsqapgviuadapekisjhkrmtweidm` int DEFAULT NULL,\n `gbtowmwiiqknhsnjcyzoxuvjhrfrlcmg` int DEFAULT NULL,\n `npsivvbnnnulguahhobigkesfowkhefd` int DEFAULT NULL,\n `lsffovevyeuahsgaizjmcpqopcprtwrl` int DEFAULT NULL,\n `kswnlcijqhiqfydeakzmrhetkyjetbnl` int DEFAULT NULL,\n `jfgkkgcpockmpdghpmwxbxrfnnotodrf` int DEFAULT NULL,\n `kgrgeesplkjuwvvzahayksmfwwixfigm` int DEFAULT NULL,\n `eacjhlpnzyqhyckzszmjbtomykhdqvsg` int DEFAULT NULL,\n `txpsqfsrhiqvgzughyskswlgfytachsg` int DEFAULT NULL,\n `xqtvrnmeyzmcqtbpjubbtnjaciyimxbh` int DEFAULT NULL,\n `dcluplzagqulbrnnpbzbxmtvsoqzijok` int DEFAULT NULL,\n `wxbqdrahhkhkycmsamqbfemhetaqfahr` int DEFAULT NULL,\n `hjhoqpqkvrdtgemjvjzpocyjlbhonmtu` int DEFAULT NULL,\n `byxsmvbbnqmzuxdznnucmvthoijcxykb` int DEFAULT NULL,\n `cifsknzvldpnbgploubwtwwzycpclqcv` int DEFAULT NULL,\n `uqyagshzvxlcwzhsqzjbgwfficpijlic` int DEFAULT NULL,\n `vcfgdxgysuwflsspqyiuhwcvctougfbf` int DEFAULT NULL,\n `qkvhxdgwfpoxobhjeeolvvtzscsswolg` int DEFAULT NULL,\n `uffwqcarwwjycehgudpiaobedzjkcuhk` int DEFAULT NULL,\n `hpilnjrkuscqyxidlsjdetowvunqqqfo` int DEFAULT NULL,\n `lfuubhrupomkvcixuhepznwgszcrgftp` int DEFAULT NULL,\n `qoliaftkqjotagmkloilzthjoylpoxzx` int DEFAULT NULL,\n `tbiracolklahzhbtufpawkgtuvwzqdyp` int DEFAULT NULL,\n `vclbqmcwplvxktmgdxhyxmuyzwkilsaf` int DEFAULT NULL,\n `vuflmksfgctrunmqxqkclvkokpdeetvv` int DEFAULT NULL,\n `kzwgtircnhhemldiggkmxxdvosxefjus` int DEFAULT NULL,\n `rcjexhxrhhndasjuyfzjxhsajjelfrmw` int DEFAULT NULL,\n `oubuiubnaofawiyxybmoacpjujstrhny` int DEFAULT NULL,\n `pxjysljpxszfmceqjobsdjdpruwmdioy` int DEFAULT NULL,\n `lnuytnzgrriiewvlfszscrvxhxcevpel` int DEFAULT NULL,\n `kqxbecgeqjnwaixjdeglhvpfetjbslai` int DEFAULT NULL,\n `osiykldomcwyxrzlwlvgdnswlkhyugln` int DEFAULT NULL,\n `lnosbuuudnjcqnehsvymytzekyuywoaj` int DEFAULT NULL,\n `grfmuqyarkymgyjhfzlpzhohzipxmhlf` int DEFAULT NULL,\n `ekuogdzihsrzilubytuvxhsbvfwoeawq` int DEFAULT NULL,\n `wplxmtldmuciqcsgnwjfsmjnrjsxyhrp` int DEFAULT NULL,\n `bkvkyvcdysdshnrhqgbznzvnrmfqbkkc` int DEFAULT NULL,\n `zpyjautkpgmewjjsxmozaprcnnhkfiqt` int DEFAULT NULL,\n `ugevyatsnoeilovzrhjqwyppblsdtkxs` int DEFAULT NULL,\n `ulhcegooxywpcnllfmbmhujufrnpazpb` int DEFAULT NULL,\n `ypvarhqcemmeuqtigzcrxwceokhibygu` int DEFAULT NULL,\n `mbnpeursozztzsffjbrhvywlczituwzu` int DEFAULT NULL,\n `kuvjumrotdxrkktlekxfqaswmwwmogzi` int DEFAULT NULL,\n `yxhgffcvujtoxdrlaowfsbrhbmuanfhc` int DEFAULT NULL,\n `txzjfhwdxoeevbvbxyocpssyhwaeqwyi` int DEFAULT NULL,\n `btsejxikjpvvfjxjfedfkgpvaobjcxas` int DEFAULT NULL,\n `lqdjxggjtqvtvqcbywcosfufrjdybpmv` int DEFAULT NULL,\n `kthathmtckdzlcmodyvbzxwhwysheiuu` int DEFAULT NULL,\n `wnjtuhtmoctjeicxupbtmesygxwzidvc` int DEFAULT NULL,\n `gfpedrvgtsfpjoeudqmqddhwulinyewv` int DEFAULT NULL,\n `svjtmitavhaquluhtefdnrnkzprxrgdh` int DEFAULT NULL,\n `qtwysseixvqupwmogthqauamrvdvzgld` int DEFAULT NULL,\n `miasksfqovydfkqlyxboqspucryhdbtg` int DEFAULT NULL,\n `zqswjjixeeffrkpyugyiqxbjcvtsxbtt` int DEFAULT NULL,\n `vuwykuczahwoxmwhticjcskxzalaales` int DEFAULT NULL,\n `uqnkpabucxaogzupvdmjyuhpsjjyjeli` int DEFAULT NULL,\n `poyzefsigpcwqqqxxtnkszehmjfhtpii` int DEFAULT NULL,\n `dwryvpqrtttbcwocgojvrasirrtwgehe` int DEFAULT NULL,\n `umpfuicyhkpfmoyxrwgcjrvqwifiucqq` int DEFAULT NULL,\n `nshhcbtiqnlwzxfogvgdpwlbtoxzmghc` int DEFAULT NULL,\n `tuxbfhpevbajqclujfbpyqoathiitidv` int DEFAULT NULL,\n `eebwaaztzgoilvtboxiagotfpopvcuem` int DEFAULT NULL,\n `xulkvwwijexlsnrxldzidsjxmwlcpotw` int DEFAULT NULL,\n `mkdwkvfhxhfzcwatxaiwhxjjgqojuemb` int DEFAULT NULL,\n `byjrrrjngwagjkfkuaarcjrfewzooaqq` int DEFAULT NULL,\n `snnktpemehpgrkrdcpwzaukgotbywdzh` int DEFAULT NULL,\n `uifpmtrbiipnyxrkjeuztrskojwacdsx` int DEFAULT NULL,\n `sfejgherfvzurnmddjezsodlhosyuhfa` int DEFAULT NULL,\n `upeiubupufmwhtdnrmsijzvcocfjlunv` int DEFAULT NULL,\n `fonpcrodquoejselftrazchbqzuokiwo` int DEFAULT NULL,\n `oytovagmmskulddtyynzpxismqoonrna` int DEFAULT NULL,\n `sdbjbkrowhewsfcykulhitemfreekkek` int DEFAULT NULL,\n `mlbzckbarqjurberdomloadtgdgsgefl` int DEFAULT NULL,\n `jnnedelofosvfqmldlswhjwwzspfkpit` int DEFAULT NULL,\n `qjtyhpmscjlahctbytnquoqirwkmsbqm` int DEFAULT NULL,\n `vxenewoqkgqwydmbbgqqxohobzasbydn` int DEFAULT NULL,\n `wjnwayregjkobphfhrqxhdyqdzwcgafm` int DEFAULT NULL,\n `bxccebapldvapolxlnblogbaxlykqnzj` int DEFAULT NULL,\n `gaeskunxdeeocygahsxttwbcrnncxmdv` int DEFAULT NULL,\n `rgwnrcyzoirkbcfbbldraamttwbnpcgq` int DEFAULT NULL,\n `eanniqmrowgebyhosvubouptjnjjorhv` int DEFAULT NULL,\n `efiieqvxgqsrqasknxgqplpoddcirrex` int DEFAULT NULL,\n `dyttnqnyujvoztqextfyhffvbqxxgiwc` int DEFAULT NULL,\n `wegovmdswvqebmrxxgafgazdzfasgppi` int DEFAULT NULL,\n `xxptiuxnxnpnrtllmsdhmmpaeqkhibdj` int DEFAULT NULL,\n `oagdkyyezgymzlbxjnondrrrdujeqflx` int DEFAULT NULL,\n `ejothtuxxdfeadhwsjemzfumwiuyoell` int DEFAULT NULL,\n `uyztmsmypvhnrxsejejzyjjwlipaoojz` int DEFAULT NULL,\n `fzxiyydukclboixmfidfcwwvowktkcxs` int DEFAULT NULL,\n `xehfronmriisqxunawgihynpktuqskxk` int DEFAULT NULL,\n PRIMARY KEY (`halrxzvudetvsbrvsdfcswmnfsvkjaeo`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"ghyqwlwuskidustonvfapzjfqxxirouo\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["halrxzvudetvsbrvsdfcswmnfsvkjaeo"],"columns":[{"name":"halrxzvudetvsbrvsdfcswmnfsvkjaeo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"rpqhtkysyckfnnwohoxcpvkzpzhnbagw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yuidriumfjsawxzhxtnfawrazdrdwndg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmgkolwmoraybparktfyrqumltribrci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqljoptkzfkqhyzbdsxkrvbxzjbzgzqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orlpmrroytcmvpkqjxgyrajrimphelna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suvbgahqzvvagluqlxxotexgxshclplx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nagfrzpqdkhuivoljvsgpethlnnbeitb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnosvjwivxsffzgfnhzvznbtpvsyrrcp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"juslbyeifyveixbpxluqhkksbiurecam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qppdzsqapgviuadapekisjhkrmtweidm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gbtowmwiiqknhsnjcyzoxuvjhrfrlcmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npsivvbnnnulguahhobigkesfowkhefd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lsffovevyeuahsgaizjmcpqopcprtwrl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kswnlcijqhiqfydeakzmrhetkyjetbnl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jfgkkgcpockmpdghpmwxbxrfnnotodrf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kgrgeesplkjuwvvzahayksmfwwixfigm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eacjhlpnzyqhyckzszmjbtomykhdqvsg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txpsqfsrhiqvgzughyskswlgfytachsg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqtvrnmeyzmcqtbpjubbtnjaciyimxbh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcluplzagqulbrnnpbzbxmtvsoqzijok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxbqdrahhkhkycmsamqbfemhetaqfahr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjhoqpqkvrdtgemjvjzpocyjlbhonmtu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"byxsmvbbnqmzuxdznnucmvthoijcxykb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cifsknzvldpnbgploubwtwwzycpclqcv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqyagshzvxlcwzhsqzjbgwfficpijlic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vcfgdxgysuwflsspqyiuhwcvctougfbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qkvhxdgwfpoxobhjeeolvvtzscsswolg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uffwqcarwwjycehgudpiaobedzjkcuhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hpilnjrkuscqyxidlsjdetowvunqqqfo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfuubhrupomkvcixuhepznwgszcrgftp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qoliaftkqjotagmkloilzthjoylpoxzx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbiracolklahzhbtufpawkgtuvwzqdyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vclbqmcwplvxktmgdxhyxmuyzwkilsaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vuflmksfgctrunmqxqkclvkokpdeetvv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzwgtircnhhemldiggkmxxdvosxefjus","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcjexhxrhhndasjuyfzjxhsajjelfrmw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oubuiubnaofawiyxybmoacpjujstrhny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxjysljpxszfmceqjobsdjdpruwmdioy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lnuytnzgrriiewvlfszscrvxhxcevpel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqxbecgeqjnwaixjdeglhvpfetjbslai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"osiykldomcwyxrzlwlvgdnswlkhyugln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lnosbuuudnjcqnehsvymytzekyuywoaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grfmuqyarkymgyjhfzlpzhohzipxmhlf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekuogdzihsrzilubytuvxhsbvfwoeawq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wplxmtldmuciqcsgnwjfsmjnrjsxyhrp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkvkyvcdysdshnrhqgbznzvnrmfqbkkc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpyjautkpgmewjjsxmozaprcnnhkfiqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugevyatsnoeilovzrhjqwyppblsdtkxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ulhcegooxywpcnllfmbmhujufrnpazpb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypvarhqcemmeuqtigzcrxwceokhibygu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbnpeursozztzsffjbrhvywlczituwzu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kuvjumrotdxrkktlekxfqaswmwwmogzi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxhgffcvujtoxdrlaowfsbrhbmuanfhc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txzjfhwdxoeevbvbxyocpssyhwaeqwyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"btsejxikjpvvfjxjfedfkgpvaobjcxas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqdjxggjtqvtvqcbywcosfufrjdybpmv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kthathmtckdzlcmodyvbzxwhwysheiuu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnjtuhtmoctjeicxupbtmesygxwzidvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfpedrvgtsfpjoeudqmqddhwulinyewv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"svjtmitavhaquluhtefdnrnkzprxrgdh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtwysseixvqupwmogthqauamrvdvzgld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"miasksfqovydfkqlyxboqspucryhdbtg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqswjjixeeffrkpyugyiqxbjcvtsxbtt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vuwykuczahwoxmwhticjcskxzalaales","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqnkpabucxaogzupvdmjyuhpsjjyjeli","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"poyzefsigpcwqqqxxtnkszehmjfhtpii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwryvpqrtttbcwocgojvrasirrtwgehe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umpfuicyhkpfmoyxrwgcjrvqwifiucqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nshhcbtiqnlwzxfogvgdpwlbtoxzmghc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tuxbfhpevbajqclujfbpyqoathiitidv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eebwaaztzgoilvtboxiagotfpopvcuem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xulkvwwijexlsnrxldzidsjxmwlcpotw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkdwkvfhxhfzcwatxaiwhxjjgqojuemb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"byjrrrjngwagjkfkuaarcjrfewzooaqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"snnktpemehpgrkrdcpwzaukgotbywdzh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uifpmtrbiipnyxrkjeuztrskojwacdsx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfejgherfvzurnmddjezsodlhosyuhfa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upeiubupufmwhtdnrmsijzvcocfjlunv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fonpcrodquoejselftrazchbqzuokiwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oytovagmmskulddtyynzpxismqoonrna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdbjbkrowhewsfcykulhitemfreekkek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlbzckbarqjurberdomloadtgdgsgefl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jnnedelofosvfqmldlswhjwwzspfkpit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjtyhpmscjlahctbytnquoqirwkmsbqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxenewoqkgqwydmbbgqqxohobzasbydn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjnwayregjkobphfhrqxhdyqdzwcgafm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bxccebapldvapolxlnblogbaxlykqnzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gaeskunxdeeocygahsxttwbcrnncxmdv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgwnrcyzoirkbcfbbldraamttwbnpcgq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eanniqmrowgebyhosvubouptjnjjorhv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"efiieqvxgqsrqasknxgqplpoddcirrex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dyttnqnyujvoztqextfyhffvbqxxgiwc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wegovmdswvqebmrxxgafgazdzfasgppi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxptiuxnxnpnrtllmsdhmmpaeqkhibdj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oagdkyyezgymzlbxjnondrrrdujeqflx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejothtuxxdfeadhwsjemzfumwiuyoell","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uyztmsmypvhnrxsejejzyjjwlipaoojz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzxiyydukclboixmfidfcwwvowktkcxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xehfronmriisqxunawgihynpktuqskxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668059,"databaseName":"models_schema","ddl":"CREATE TABLE `giavkelmcovmviellbpxbpmjayywzwvr` (\n `kxsljcsyrqihuxpldunpwqyqxmcuzaph` int NOT NULL,\n `ynomjmlrguokrqvtmhficqqgkrcothwk` int DEFAULT NULL,\n `vfvtjahwnnzxwuepztynuvzxrjcyrzac` int DEFAULT NULL,\n `dcuqedmlbjknzbohrmvohetcftbxedjm` int DEFAULT NULL,\n `zfhbcctqfqqmjscatigxzgaylgfxijfr` int DEFAULT NULL,\n `ppypufxtoooeocgxrqptwybqrumzlnuo` int DEFAULT NULL,\n `iplgiupvwdeoivrajyrcqeevyoedcxjg` int DEFAULT NULL,\n `fmxhualushdwiealrviqfnkosslilbng` int DEFAULT NULL,\n `sqbfeiybwyympdnhfbmnvayqbevmpnff` int DEFAULT NULL,\n `vaohrpgiraohxrnommqnxzezhxabheaw` int DEFAULT NULL,\n `qrflvvbvkauiqordbulgaysmeghjtzdx` int DEFAULT NULL,\n `zzqvgnmfpsemwidhljjvuwikurcmsxql` int DEFAULT NULL,\n `pvwrwbnnqgfzmbbbrefmbrjdklywkwxq` int DEFAULT NULL,\n `mivhgowttxpdhpywfmphblxyfilgtqre` int DEFAULT NULL,\n `fdlxjasrnpmgdnnqsnxhaazcstdizfdc` int DEFAULT NULL,\n `bkbdqtptxewydetkfxykxzpkqxvrwuui` int DEFAULT NULL,\n `flbcqqsmdqwzkzuibmuaadklekhnhhxs` int DEFAULT NULL,\n `rrqhdxmvhlbfusvzirgsyyauhrnxrkyl` int DEFAULT NULL,\n `jqxwoxxfmnplqkpfknoslpdzcdtwrjxg` int DEFAULT NULL,\n `kgmmaryhkjgmbalpwvrjgnzxybiluktq` int DEFAULT NULL,\n `hdlweuczblztwmyyfkrjxaajidpxnbjl` int DEFAULT NULL,\n `fbgjqrlyzmtmvohhvgbdrcowcqcuohwz` int DEFAULT NULL,\n `jvaiyjoehcvwmpitxilozxukzjnaeyfd` int DEFAULT NULL,\n `scwmslithkxnihesbcxeqiiotccamcpj` int DEFAULT NULL,\n `seiihlkakfqyvjwygydmgoogyotwvstg` int DEFAULT NULL,\n `rgqqmppgukvbfqzylpgwwfcidrgvfjgj` int DEFAULT NULL,\n `ljlyjoscyyjvjwaoarpoeejwgohvbfai` int DEFAULT NULL,\n `hicsbdjtssxalbpuymwjdhlypuuonnrn` int DEFAULT NULL,\n `oinmkclcwmdkqsatbxlrhgklrnatirsz` int DEFAULT NULL,\n `ygugvytohhqjtqyksfgtqiqvprfqrbth` int DEFAULT NULL,\n `gicxbajxdblibmctbfyjakgittzqhzoa` int DEFAULT NULL,\n `scpsvtaemexnurchfrymdwjytopbkrlu` int DEFAULT NULL,\n `mkgnbhxffakmlyatfinwvavryjqjybhf` int DEFAULT NULL,\n `tbbydkdfumdrtbcftdpiiikshbmeukhm` int DEFAULT NULL,\n `xepnxjwjjwzrtvsnirpptwpykjijmchi` int DEFAULT NULL,\n `ouwzbhhkyikpmgifwbhpvjtdjwbiqbqn` int DEFAULT NULL,\n `nsadkoxkoczcwshpoysoipdkinwcqbot` int DEFAULT NULL,\n `gmuymmaqbqgobhyszattrgjpjwoyjpmx` int DEFAULT NULL,\n `qxsdrsrvscclvbpcfxvmwwwyqkjnnnxs` int DEFAULT NULL,\n `ugdqbjreknosfpgyexmvxasodsmyheae` int DEFAULT NULL,\n `jwhrwblytnvqzxmsvjdyqhgzmmlnmsio` int DEFAULT NULL,\n `aseccamglzooirjajhieabwelxashfvt` int DEFAULT NULL,\n `aykgaksfkesywtuceeetlpwumpqxavys` int DEFAULT NULL,\n `lfhawlzaujesjocvkuegholfytoogcds` int DEFAULT NULL,\n `wutbzdszrfdzhkqnvjffeiclbiuqsjgp` int DEFAULT NULL,\n `xprtvdtjdmujhakrdmqnfzomzvirlrjh` int DEFAULT NULL,\n `mxrvzzigfkvacteqlsygvvplmvczyump` int DEFAULT NULL,\n `xvqwsgfvyddzrnsihjunqpsmwwvfhqgc` int DEFAULT NULL,\n `vxrtxmyiklzdemeahwegjfovbbdyajtm` int DEFAULT NULL,\n `akmbqiacpjiwxxdrricuednfozrdeyos` int DEFAULT NULL,\n `uijpynlnsnkdfowoonpcttaazfeeqsss` int DEFAULT NULL,\n `esqseasyzbkmbpyxgsakgljrdqnqjcxh` int DEFAULT NULL,\n `tqcuxmnhtuuihnvfnropeyqeawjquuto` int DEFAULT NULL,\n `dqjuuaynvraieergqkelvaidcbcupxcm` int DEFAULT NULL,\n `rqzezbdiqihnzchdvqaaraecuszekxkp` int DEFAULT NULL,\n `aqjfwaimzfwyldfsdfmzyakplgnixarg` int DEFAULT NULL,\n `rfxwkkjmhpntrsdaaqwwocxhaafswpjn` int DEFAULT NULL,\n `qnjmmkjarjvoinuycjsjinauvhgvmehc` int DEFAULT NULL,\n `sgntduxgujqcgzkqiumuzexdauwrdeod` int DEFAULT NULL,\n `ngglzksmdfzfugtojuuvxxncrsvrsecm` int DEFAULT NULL,\n `sbhskfdxrhuhooyvrqtgewdneucmvrds` int DEFAULT NULL,\n `txkjgykkanoxlackywwpxhmnazporaur` int DEFAULT NULL,\n `iezcufboaxfnpmxqgdekgdnvjeldpdtx` int DEFAULT NULL,\n `rbsohrqeptlchefgxkjtiwidqijyovei` int DEFAULT NULL,\n `qzcbsxuynebnoehlbptcjtsagocqkktv` int DEFAULT NULL,\n `owgyajqdqmbauskeidhgzwmhrexhbigs` int DEFAULT NULL,\n `rvacssoztolljuqcbvrzgcpafeodyaha` int DEFAULT NULL,\n `fdikosqdrzqpsfkvbigpeartscudzava` int DEFAULT NULL,\n `nagcxezlosdxywzwpcfwvpxoxspwlkrl` int DEFAULT NULL,\n `mjgtgcnkahvkajgebqltodyxulvrrbns` int DEFAULT NULL,\n `eieaggjefwkonuqlyhpilhnmoxuaffza` int DEFAULT NULL,\n `dfpvtcegpixwzskgyovudsqvayfjougx` int DEFAULT NULL,\n `muyhdeuqjtypldvsgbandbqxruxtrntx` int DEFAULT NULL,\n `ewkcfhfrmjilysqpivvnzocgixmpotzy` int DEFAULT NULL,\n `xgllcchcmqlnccyssqyvpppwdqjulkig` int DEFAULT NULL,\n `voqyjqofxsgqjstxytdkmqxguiqwauso` int DEFAULT NULL,\n `gtlbuxmngkpfzpnizbrgklyhkeklfnqy` int DEFAULT NULL,\n `mtbkizmjghvdladnxsdkdousbbenrlch` int DEFAULT NULL,\n `ftehbmcamxmhtdnlfsgwmwzdqhsrwpwl` int DEFAULT NULL,\n `uzlbiagcimhschojshdelolocgfuqoqp` int DEFAULT NULL,\n `ordivuohfhkcixfkyfnjtmjehqklqpkv` int DEFAULT NULL,\n `kmvpnsqsuxtzoxdrpifeociwtmhgmcec` int DEFAULT NULL,\n `pmqfpoezwchzwwhlhngllruiftqfboco` int DEFAULT NULL,\n `ibwtojsucqmbwlnjirhdcmiagplkaqtf` int DEFAULT NULL,\n `yhuvbkzwulqowyemagmqnwczsppurrsu` int DEFAULT NULL,\n `pkbzmodqtyjxiepvkmbemkgxdeyxvhay` int DEFAULT NULL,\n `kqetvdxarqnpscojmrixiacfoepdtovu` int DEFAULT NULL,\n `kokqvbjydgdyszewbqxthgxsnrzygvni` int DEFAULT NULL,\n `jqzqjkxanxdfnvsfjbzauqrwyiyeqlgx` int DEFAULT NULL,\n `oscmclbolivsrgqlqvgcnlzxuifytwet` int DEFAULT NULL,\n `faymkrurqciuupklyehetqcwzyejmxwm` int DEFAULT NULL,\n `vcaydiiwphvqfexzqxiyjodumsxguhbd` int DEFAULT NULL,\n `igdowgarxelhczqfaufgiegerddnhyjl` int DEFAULT NULL,\n `psnrnrjeqcfxcvbppqpkcjhsmfpscapp` int DEFAULT NULL,\n `mknbzaonqdklwbxotmcljdbwvfhxflmx` int DEFAULT NULL,\n `msifutpojvlsfhvdczvsabbtpwaxxaoa` int DEFAULT NULL,\n `lsqjasgjpkwonfogpureuajtkrpsjpne` int DEFAULT NULL,\n `hvjttwdwfnoqbxeubdpajearomnalieg` int DEFAULT NULL,\n `bbicduuyqpcuhgfscdxvinalqgchdfnh` int DEFAULT NULL,\n `uomcstvoxdftwtcowixsaenpdswyazle` int DEFAULT NULL,\n PRIMARY KEY (`kxsljcsyrqihuxpldunpwqyqxmcuzaph`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"giavkelmcovmviellbpxbpmjayywzwvr\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["kxsljcsyrqihuxpldunpwqyqxmcuzaph"],"columns":[{"name":"kxsljcsyrqihuxpldunpwqyqxmcuzaph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ynomjmlrguokrqvtmhficqqgkrcothwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vfvtjahwnnzxwuepztynuvzxrjcyrzac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcuqedmlbjknzbohrmvohetcftbxedjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zfhbcctqfqqmjscatigxzgaylgfxijfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ppypufxtoooeocgxrqptwybqrumzlnuo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iplgiupvwdeoivrajyrcqeevyoedcxjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmxhualushdwiealrviqfnkosslilbng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sqbfeiybwyympdnhfbmnvayqbevmpnff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vaohrpgiraohxrnommqnxzezhxabheaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrflvvbvkauiqordbulgaysmeghjtzdx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzqvgnmfpsemwidhljjvuwikurcmsxql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvwrwbnnqgfzmbbbrefmbrjdklywkwxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mivhgowttxpdhpywfmphblxyfilgtqre","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fdlxjasrnpmgdnnqsnxhaazcstdizfdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkbdqtptxewydetkfxykxzpkqxvrwuui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flbcqqsmdqwzkzuibmuaadklekhnhhxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrqhdxmvhlbfusvzirgsyyauhrnxrkyl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jqxwoxxfmnplqkpfknoslpdzcdtwrjxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kgmmaryhkjgmbalpwvrjgnzxybiluktq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdlweuczblztwmyyfkrjxaajidpxnbjl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbgjqrlyzmtmvohhvgbdrcowcqcuohwz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvaiyjoehcvwmpitxilozxukzjnaeyfd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scwmslithkxnihesbcxeqiiotccamcpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"seiihlkakfqyvjwygydmgoogyotwvstg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgqqmppgukvbfqzylpgwwfcidrgvfjgj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljlyjoscyyjvjwaoarpoeejwgohvbfai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hicsbdjtssxalbpuymwjdhlypuuonnrn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oinmkclcwmdkqsatbxlrhgklrnatirsz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ygugvytohhqjtqyksfgtqiqvprfqrbth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gicxbajxdblibmctbfyjakgittzqhzoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scpsvtaemexnurchfrymdwjytopbkrlu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkgnbhxffakmlyatfinwvavryjqjybhf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbbydkdfumdrtbcftdpiiikshbmeukhm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xepnxjwjjwzrtvsnirpptwpykjijmchi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ouwzbhhkyikpmgifwbhpvjtdjwbiqbqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nsadkoxkoczcwshpoysoipdkinwcqbot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmuymmaqbqgobhyszattrgjpjwoyjpmx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxsdrsrvscclvbpcfxvmwwwyqkjnnnxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugdqbjreknosfpgyexmvxasodsmyheae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwhrwblytnvqzxmsvjdyqhgzmmlnmsio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aseccamglzooirjajhieabwelxashfvt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aykgaksfkesywtuceeetlpwumpqxavys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfhawlzaujesjocvkuegholfytoogcds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wutbzdszrfdzhkqnvjffeiclbiuqsjgp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xprtvdtjdmujhakrdmqnfzomzvirlrjh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mxrvzzigfkvacteqlsygvvplmvczyump","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvqwsgfvyddzrnsihjunqpsmwwvfhqgc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxrtxmyiklzdemeahwegjfovbbdyajtm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akmbqiacpjiwxxdrricuednfozrdeyos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uijpynlnsnkdfowoonpcttaazfeeqsss","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"esqseasyzbkmbpyxgsakgljrdqnqjcxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqcuxmnhtuuihnvfnropeyqeawjquuto","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqjuuaynvraieergqkelvaidcbcupxcm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqzezbdiqihnzchdvqaaraecuszekxkp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqjfwaimzfwyldfsdfmzyakplgnixarg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfxwkkjmhpntrsdaaqwwocxhaafswpjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qnjmmkjarjvoinuycjsjinauvhgvmehc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sgntduxgujqcgzkqiumuzexdauwrdeod","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ngglzksmdfzfugtojuuvxxncrsvrsecm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbhskfdxrhuhooyvrqtgewdneucmvrds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txkjgykkanoxlackywwpxhmnazporaur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iezcufboaxfnpmxqgdekgdnvjeldpdtx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbsohrqeptlchefgxkjtiwidqijyovei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzcbsxuynebnoehlbptcjtsagocqkktv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owgyajqdqmbauskeidhgzwmhrexhbigs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rvacssoztolljuqcbvrzgcpafeodyaha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fdikosqdrzqpsfkvbigpeartscudzava","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nagcxezlosdxywzwpcfwvpxoxspwlkrl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjgtgcnkahvkajgebqltodyxulvrrbns","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eieaggjefwkonuqlyhpilhnmoxuaffza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dfpvtcegpixwzskgyovudsqvayfjougx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"muyhdeuqjtypldvsgbandbqxruxtrntx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ewkcfhfrmjilysqpivvnzocgixmpotzy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xgllcchcmqlnccyssqyvpppwdqjulkig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"voqyjqofxsgqjstxytdkmqxguiqwauso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtlbuxmngkpfzpnizbrgklyhkeklfnqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtbkizmjghvdladnxsdkdousbbenrlch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftehbmcamxmhtdnlfsgwmwzdqhsrwpwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzlbiagcimhschojshdelolocgfuqoqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ordivuohfhkcixfkyfnjtmjehqklqpkv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmvpnsqsuxtzoxdrpifeociwtmhgmcec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmqfpoezwchzwwhlhngllruiftqfboco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibwtojsucqmbwlnjirhdcmiagplkaqtf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhuvbkzwulqowyemagmqnwczsppurrsu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkbzmodqtyjxiepvkmbemkgxdeyxvhay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqetvdxarqnpscojmrixiacfoepdtovu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kokqvbjydgdyszewbqxthgxsnrzygvni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jqzqjkxanxdfnvsfjbzauqrwyiyeqlgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oscmclbolivsrgqlqvgcnlzxuifytwet","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"faymkrurqciuupklyehetqcwzyejmxwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vcaydiiwphvqfexzqxiyjodumsxguhbd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igdowgarxelhczqfaufgiegerddnhyjl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psnrnrjeqcfxcvbppqpkcjhsmfpscapp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mknbzaonqdklwbxotmcljdbwvfhxflmx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"msifutpojvlsfhvdczvsabbtpwaxxaoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lsqjasgjpkwonfogpureuajtkrpsjpne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hvjttwdwfnoqbxeubdpajearomnalieg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbicduuyqpcuhgfscdxvinalqgchdfnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uomcstvoxdftwtcowixsaenpdswyazle","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668090,"databaseName":"models_schema","ddl":"CREATE TABLE `gibjkzyrpmoaypiniuiyywvlmaidwzxy` (\n `mgrhdcrfffjqcixbnceoherkswomkjdk` int NOT NULL,\n `prgzlvmuntyxsxwytdyhyuvvewpwmatn` int DEFAULT NULL,\n `sgvsvarckajuulsjrlnvuofreqxlqsue` int DEFAULT NULL,\n `uupalpxwrxpnkwragpnyndyufxilftwn` int DEFAULT NULL,\n `myelzqpljodzfeaxnpgjoqdixpqhrnki` int DEFAULT NULL,\n `smgfmaeoowzwdzxqtlkyjswuwalpnhek` int DEFAULT NULL,\n `dmtfvdolrizwrkrleotiylnelpxetgki` int DEFAULT NULL,\n `ipahbwzibflpqrfplxwpwqizwvtxpmks` int DEFAULT NULL,\n `xampjkpvwsvbmfrulhwjkflxuqxzmzwi` int DEFAULT NULL,\n `yviebujskbwsmleuwzwsgsghdbkdiqen` int DEFAULT NULL,\n `avourrksvavqavthcqvxollzouhcoxtc` int DEFAULT NULL,\n `znybpebefbequvlgahicqzdcgcrypxoa` int DEFAULT NULL,\n `binvhbpzchmiqbuewfzipkdirqibvnqb` int DEFAULT NULL,\n `gstmtzavmlsbduggazuccpabcyewctbv` int DEFAULT NULL,\n `btnmsyxjjapmguexagaanipedudbjdvt` int DEFAULT NULL,\n `xxsyjcswmbajxnplqchcwdikglhjiyip` int DEFAULT NULL,\n `vkjdhrgfolrzhhzdekeyzxtblejzayrh` int DEFAULT NULL,\n `qsgcmcauoszwpkfzllepvnglkwojusmt` int DEFAULT NULL,\n `bqrlspmsohftamysaruzoijykknmycvg` int DEFAULT NULL,\n `aqieektprgdxwwmvbomnuigfzvhexaqd` int DEFAULT NULL,\n `pzfdgkhivylqkthnzlokgvimnvxoqnun` int DEFAULT NULL,\n `ouedlrlsvhqoagccuvlprszhvjfgzoyw` int DEFAULT NULL,\n `dlugrzcmgcizbnquqlobrixtxwexqovz` int DEFAULT NULL,\n `ufvhnwybremzpvfpapfxgwiyhdkpavsd` int DEFAULT NULL,\n `yptkdgrbofbuxieogyscamokykikfehv` int DEFAULT NULL,\n `lrbcqxyurrqnfhzsywwqkpurpqduzaht` int DEFAULT NULL,\n `kzdjzydzgtvttdhqeuaklraqiajvcpov` int DEFAULT NULL,\n `xpvremsjynsgcnqxatusnnwqhvvyuxzq` int DEFAULT NULL,\n `lgjndqfwwzhyvigfqmzoictdzikqlgjk` int DEFAULT NULL,\n `ltgblduuzavuyhsjrfmxajlevxhphjyi` int DEFAULT NULL,\n `zftkdhwbgokkckfsnzdsskfgletprlfx` int DEFAULT NULL,\n `fmrwrunankerjskicgxfvlqxuikdxexp` int DEFAULT NULL,\n `tabysmythvjmdfipkylsvexthqfpdbkc` int DEFAULT NULL,\n `blypdxedmpwkvnpbpdifgpsmzbwgusnw` int DEFAULT NULL,\n `dfrvnsiyeoltulredvunnakfvkquefpo` int DEFAULT NULL,\n `cqezcjfewpaxrmkhcfcfvwyrqijwxumm` int DEFAULT NULL,\n `dpndjgufxtvygxcozezrmmkhinllpthe` int DEFAULT NULL,\n `zvxltfngvqcmepjfqqrzzivyaevcqbmy` int DEFAULT NULL,\n `wwxoubaoyjhyxaooaxqafwzzcwsanljf` int DEFAULT NULL,\n `bbqrqxldusygfrtjvqokrqgmqgtegdlu` int DEFAULT NULL,\n `oskerkeskhtaijqvqyrvywubhfqglmmw` int DEFAULT NULL,\n `kfxvrjsrdwouqbtfsjibrpkhkviluyej` int DEFAULT NULL,\n `sxuqadayarovdhfljipcbqterusaqsxj` int DEFAULT NULL,\n `bwjlvjcbzqfmpxwgasyxszndaknolqho` int DEFAULT NULL,\n `yqpgeldvjvltyqvqknaumkzspxfmvura` int DEFAULT NULL,\n `dxkmicgedzsoijpmdjohtnrdqehcrcie` int DEFAULT NULL,\n `aoptrkwezkshatfftbyluzhvoismxymc` int DEFAULT NULL,\n `xdgyxskbhdobkxxyvpujbqxktgeqlpuv` int DEFAULT NULL,\n `rgpbdbqtrkyoisorkeiohwpcyclywudt` int DEFAULT NULL,\n `owkqlolaczntkgxfeqesofwirbqaktqo` int DEFAULT NULL,\n `vvusjfxafvyijaaohztkjezdvkoqahij` int DEFAULT NULL,\n `sjpzswtjxskylcracvdifteerdmwzrqp` int DEFAULT NULL,\n `tofevnijgiaxolnrzrbtepepdobsjsrl` int DEFAULT NULL,\n `sdexcndsgafpaglnnpfwjyenivmuesqw` int DEFAULT NULL,\n `rbmbqvtehiscfjfkenbvvudnecaezqwl` int DEFAULT NULL,\n `ehrijankxmmcswrdmktpyjjftppmrnqj` int DEFAULT NULL,\n `zxpkrblpxefyfwcgbqrjmjgkdhkkqeml` int DEFAULT NULL,\n `mxaqjggcncfxryclfzpkbcuqiejihdua` int DEFAULT NULL,\n `ihojohzmdjldpszlrjxiefdlfyrdvxqo` int DEFAULT NULL,\n `cvqyoqtappkgrxnhgfdmugtlnitbiaun` int DEFAULT NULL,\n `nfztlgjdaebjjgbftdcexeilecsohesh` int DEFAULT NULL,\n `trkbxtuwmkwpgzuxzdwxecnnqdmmaipm` int DEFAULT NULL,\n `miuudzqtcdbrhrrpyipoeuxwepehmsoh` int DEFAULT NULL,\n `bubsqweilzlwzufuoiywsnrgxphplfye` int DEFAULT NULL,\n `gpqcnvvmexhpyimvkvarsogyojzxlppg` int DEFAULT NULL,\n `mpmoegwxqecazzdfziibwfetdpdjxckn` int DEFAULT NULL,\n `pwrjnyccjkzwyaawcezitgiirytlvbhu` int DEFAULT NULL,\n `ovntmaapzbrrjsmztlsqetpnfkmfydmy` int DEFAULT NULL,\n `qstnmpgihktmqicsilcgecphwhtyalyd` int DEFAULT NULL,\n `ccuevewvhczdhfjsdooqgshgbxinlwcc` int DEFAULT NULL,\n `hsswwrmvimsqdfyxalewiurjgalurvih` int DEFAULT NULL,\n `zykjjhtyaxedwesahhmrubrqggjqjgti` int DEFAULT NULL,\n `xdszomsacwvskxuwlnqjkrmnoxgfdxkp` int DEFAULT NULL,\n `azaytzqylkjidxldklpkaytuzxmqmtwn` int DEFAULT NULL,\n `idvlkpqgmclwaudsanbtxseihsauxidx` int DEFAULT NULL,\n `ppcsizxvkrazafqopsmbaitjmtlflamn` int DEFAULT NULL,\n `slyvwqhupahwrcbpxuffugjqdvevnnhg` int DEFAULT NULL,\n `hwsyoejqlihiwxhsqkmryxhhqikutwzw` int DEFAULT NULL,\n `dfuntyvaunfbnudxajqgdvikdmwfhint` int DEFAULT NULL,\n `vhftxyqugqeuoduzfoarxblvaoowwbbw` int DEFAULT NULL,\n `dumtoesiyrahfrfashiqpwfxridiigwr` int DEFAULT NULL,\n `uqkrymwaibamverngozqcuixgpfemunl` int DEFAULT NULL,\n `xhkncjkzlzfrgzakioxomihkioucgvnw` int DEFAULT NULL,\n `gftxjabvrpdbcvbsvivttvwqffqxxhqz` int DEFAULT NULL,\n `eixxrpubmiumexeucovpnbkzhopbxeta` int DEFAULT NULL,\n `xisogmaeuiitdbnmernrummlujgidnbw` int DEFAULT NULL,\n `qcokagrnzhfxnegcbslsrncpeiyddxsb` int DEFAULT NULL,\n `iavsosfvswssqzefxmvcfcsjwaxfagqd` int DEFAULT NULL,\n `ibuwbpogxaxenkmhzetoefdkognnxqop` int DEFAULT NULL,\n `iutkvszaasucojdwkwcelarezwvusprr` int DEFAULT NULL,\n `kduijcptdjyigmwkwmojapjobtfwdkqn` int DEFAULT NULL,\n `tktahophyhlvbzdrmzctfsaqezapimbw` int DEFAULT NULL,\n `sewfhkkygidrxeqejymlyaqvjgmxyabc` int DEFAULT NULL,\n `fiaggqbxmmwtezcmhofixilxsodqhpqf` int DEFAULT NULL,\n `wtnogkhzymuswcgrhuysrbtwrsrzzwsp` int DEFAULT NULL,\n `xxqgmdcxoeqzexmryboxfhmmpzawuzln` int DEFAULT NULL,\n `mseqgtqyakfbqrbxbcdgmbgzldjznmdf` int DEFAULT NULL,\n `zhqfkctujprurnuajekkwbwvhkwavnyq` int DEFAULT NULL,\n `rdxafwkipmofxmdilscxlyngxutmtbty` int DEFAULT NULL,\n `nyrbccddvejukwpxtguqiusssiwqnolt` int DEFAULT NULL,\n PRIMARY KEY (`mgrhdcrfffjqcixbnceoherkswomkjdk`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"gibjkzyrpmoaypiniuiyywvlmaidwzxy\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["mgrhdcrfffjqcixbnceoherkswomkjdk"],"columns":[{"name":"mgrhdcrfffjqcixbnceoherkswomkjdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"prgzlvmuntyxsxwytdyhyuvvewpwmatn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sgvsvarckajuulsjrlnvuofreqxlqsue","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uupalpxwrxpnkwragpnyndyufxilftwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"myelzqpljodzfeaxnpgjoqdixpqhrnki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smgfmaeoowzwdzxqtlkyjswuwalpnhek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dmtfvdolrizwrkrleotiylnelpxetgki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ipahbwzibflpqrfplxwpwqizwvtxpmks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xampjkpvwsvbmfrulhwjkflxuqxzmzwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yviebujskbwsmleuwzwsgsghdbkdiqen","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avourrksvavqavthcqvxollzouhcoxtc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znybpebefbequvlgahicqzdcgcrypxoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"binvhbpzchmiqbuewfzipkdirqibvnqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gstmtzavmlsbduggazuccpabcyewctbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"btnmsyxjjapmguexagaanipedudbjdvt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxsyjcswmbajxnplqchcwdikglhjiyip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkjdhrgfolrzhhzdekeyzxtblejzayrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qsgcmcauoszwpkfzllepvnglkwojusmt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqrlspmsohftamysaruzoijykknmycvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqieektprgdxwwmvbomnuigfzvhexaqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pzfdgkhivylqkthnzlokgvimnvxoqnun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ouedlrlsvhqoagccuvlprszhvjfgzoyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlugrzcmgcizbnquqlobrixtxwexqovz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufvhnwybremzpvfpapfxgwiyhdkpavsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yptkdgrbofbuxieogyscamokykikfehv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lrbcqxyurrqnfhzsywwqkpurpqduzaht","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzdjzydzgtvttdhqeuaklraqiajvcpov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpvremsjynsgcnqxatusnnwqhvvyuxzq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgjndqfwwzhyvigfqmzoictdzikqlgjk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ltgblduuzavuyhsjrfmxajlevxhphjyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zftkdhwbgokkckfsnzdsskfgletprlfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmrwrunankerjskicgxfvlqxuikdxexp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tabysmythvjmdfipkylsvexthqfpdbkc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blypdxedmpwkvnpbpdifgpsmzbwgusnw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dfrvnsiyeoltulredvunnakfvkquefpo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqezcjfewpaxrmkhcfcfvwyrqijwxumm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpndjgufxtvygxcozezrmmkhinllpthe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvxltfngvqcmepjfqqrzzivyaevcqbmy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwxoubaoyjhyxaooaxqafwzzcwsanljf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbqrqxldusygfrtjvqokrqgmqgtegdlu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oskerkeskhtaijqvqyrvywubhfqglmmw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfxvrjsrdwouqbtfsjibrpkhkviluyej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sxuqadayarovdhfljipcbqterusaqsxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bwjlvjcbzqfmpxwgasyxszndaknolqho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqpgeldvjvltyqvqknaumkzspxfmvura","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxkmicgedzsoijpmdjohtnrdqehcrcie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aoptrkwezkshatfftbyluzhvoismxymc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdgyxskbhdobkxxyvpujbqxktgeqlpuv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgpbdbqtrkyoisorkeiohwpcyclywudt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owkqlolaczntkgxfeqesofwirbqaktqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vvusjfxafvyijaaohztkjezdvkoqahij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjpzswtjxskylcracvdifteerdmwzrqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tofevnijgiaxolnrzrbtepepdobsjsrl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdexcndsgafpaglnnpfwjyenivmuesqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbmbqvtehiscfjfkenbvvudnecaezqwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehrijankxmmcswrdmktpyjjftppmrnqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxpkrblpxefyfwcgbqrjmjgkdhkkqeml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mxaqjggcncfxryclfzpkbcuqiejihdua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ihojohzmdjldpszlrjxiefdlfyrdvxqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvqyoqtappkgrxnhgfdmugtlnitbiaun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfztlgjdaebjjgbftdcexeilecsohesh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"trkbxtuwmkwpgzuxzdwxecnnqdmmaipm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"miuudzqtcdbrhrrpyipoeuxwepehmsoh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bubsqweilzlwzufuoiywsnrgxphplfye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gpqcnvvmexhpyimvkvarsogyojzxlppg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpmoegwxqecazzdfziibwfetdpdjxckn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwrjnyccjkzwyaawcezitgiirytlvbhu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovntmaapzbrrjsmztlsqetpnfkmfydmy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qstnmpgihktmqicsilcgecphwhtyalyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ccuevewvhczdhfjsdooqgshgbxinlwcc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hsswwrmvimsqdfyxalewiurjgalurvih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zykjjhtyaxedwesahhmrubrqggjqjgti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdszomsacwvskxuwlnqjkrmnoxgfdxkp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azaytzqylkjidxldklpkaytuzxmqmtwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idvlkpqgmclwaudsanbtxseihsauxidx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ppcsizxvkrazafqopsmbaitjmtlflamn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"slyvwqhupahwrcbpxuffugjqdvevnnhg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwsyoejqlihiwxhsqkmryxhhqikutwzw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dfuntyvaunfbnudxajqgdvikdmwfhint","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhftxyqugqeuoduzfoarxblvaoowwbbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dumtoesiyrahfrfashiqpwfxridiigwr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqkrymwaibamverngozqcuixgpfemunl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xhkncjkzlzfrgzakioxomihkioucgvnw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gftxjabvrpdbcvbsvivttvwqffqxxhqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eixxrpubmiumexeucovpnbkzhopbxeta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xisogmaeuiitdbnmernrummlujgidnbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qcokagrnzhfxnegcbslsrncpeiyddxsb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iavsosfvswssqzefxmvcfcsjwaxfagqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibuwbpogxaxenkmhzetoefdkognnxqop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iutkvszaasucojdwkwcelarezwvusprr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kduijcptdjyigmwkwmojapjobtfwdkqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tktahophyhlvbzdrmzctfsaqezapimbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sewfhkkygidrxeqejymlyaqvjgmxyabc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fiaggqbxmmwtezcmhofixilxsodqhpqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtnogkhzymuswcgrhuysrbtwrsrzzwsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxqgmdcxoeqzexmryboxfhmmpzawuzln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mseqgtqyakfbqrbxbcdgmbgzldjznmdf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhqfkctujprurnuajekkwbwvhkwavnyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdxafwkipmofxmdilscxlyngxutmtbty","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nyrbccddvejukwpxtguqiusssiwqnolt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668121,"databaseName":"models_schema","ddl":"CREATE TABLE `gjwbaxlnvuczlhgbaytrbajtlzxhwmbn` (\n `njtcrfwvlaagzfkltfciztovpaxagnmd` int NOT NULL,\n `qppcgzbpocoifqzqjdojhueuutozockf` int DEFAULT NULL,\n `nygtuhahmybntnifvhizjqbitvkvcquz` int DEFAULT NULL,\n `zbtsbdxebwiqmigfrbiqsnnnwtkmcuaz` int DEFAULT NULL,\n `afrzgftfwqhprkvycpgtsysgkxokhjoz` int DEFAULT NULL,\n `phxhdusrjzbcdptissluagixglmedlmc` int DEFAULT NULL,\n `axpojwlqfqisiripnkjobectmhfeqybj` int DEFAULT NULL,\n `knmgllerhkcyhtulvxfafbwwyiitfnyf` int DEFAULT NULL,\n `rpmrdllqbsqqjjttjdkqsrkwmfxclydp` int DEFAULT NULL,\n `ybbqnjvgjdlbjksefadymerajydcwnje` int DEFAULT NULL,\n `yufzhwkzzpfnrdserxxwaafcxwmcnfsx` int DEFAULT NULL,\n `kkwuyfcderitldsxntuehzghwomdbsrc` int DEFAULT NULL,\n `cowocujypqdbweanwmmptuynobjsmweb` int DEFAULT NULL,\n `sgcajjnrajbftczponytgtardmirhzll` int DEFAULT NULL,\n `wrrdkfukphgyxswzektnkejzjhqmrpmy` int DEFAULT NULL,\n `seqwxtaohzigfsqdtcsecyobjvbnyahk` int DEFAULT NULL,\n `vlyqkpwlketwjrdpwdoxaxiqybiuekfd` int DEFAULT NULL,\n `idetokxwmzxyxjrmmnfkkvhadtqiiuqq` int DEFAULT NULL,\n `hdmqnaawprhalazuyahcuhvkhflugovp` int DEFAULT NULL,\n `dehatlfhrhtortidnmfxlpbifbtflwam` int DEFAULT NULL,\n `hczdfcaqkgldjrgjzahxlyifxsckiopq` int DEFAULT NULL,\n `qcdbuemvctivaagcxljkuynqrmigiqlw` int DEFAULT NULL,\n `qdprkahbgtfwcwqwkkmjhyjjowdieknu` int DEFAULT NULL,\n `nuwxtvkoelcfsrywpnghtqgkuydeksqd` int DEFAULT NULL,\n `mjbguzbjpflnuelkjgxrshgukxtrnlym` int DEFAULT NULL,\n `oqaowljgocxugiilbxhszquhtlpntsyu` int DEFAULT NULL,\n `axlgdmfwdfyyyzrwxajxzljvdjenxbzi` int DEFAULT NULL,\n `ramymhrhwynngcpdzppoknhvemremxye` int DEFAULT NULL,\n `kgmfhzzwhydsswvlmyoodrulspxolbgw` int DEFAULT NULL,\n `wrnloiqljnzmzpzmppzpkoeuailcntvq` int DEFAULT NULL,\n `prsvqsmocbanaayxdvlyosbwvksghwzq` int DEFAULT NULL,\n `ejbyjmpmewxezdgefaznlyuzwbtlbjeo` int DEFAULT NULL,\n `hinovgyatrvruhhnngcxlotifqmadoan` int DEFAULT NULL,\n `vpexunwyocesubjjflexpmghfqisgmyr` int DEFAULT NULL,\n `qtxdyowacqdntymvenczlxptfggpbwkl` int DEFAULT NULL,\n `vpyepocyinigckfwfdchakjezaxmuakl` int DEFAULT NULL,\n `ehbhuciuqosfbflkmybiojnidbgzkibt` int DEFAULT NULL,\n `twqvhfiggmwluhqfoapcubnkgwcdqatj` int DEFAULT NULL,\n `ncdnobbptghvtjvqrqboskevjevywdwm` int DEFAULT NULL,\n `ynfsjrphcxwggwngdhwfwtfypfwrfsnh` int DEFAULT NULL,\n `soaehwslcdxzbsjvrkbgsoscfjxytnrh` int DEFAULT NULL,\n `kkaqetjujzdlqnaisadwwwdhrecsgvlo` int DEFAULT NULL,\n `kisuldyuiokwgsvyucichffehuqhfehi` int DEFAULT NULL,\n `coninxmywoumjzcjgrwcqtmvumdxgcdq` int DEFAULT NULL,\n `xpyygzfznoiujuocklslfffrmdypjthh` int DEFAULT NULL,\n `vcvhqhofhnprljfyecnhbgulapquyivh` int DEFAULT NULL,\n `qlwmhgtbhasckldyplycqvwwsrjgzkgt` int DEFAULT NULL,\n `wbuwvdgzvdinfokqkiplydnbrohrqooe` int DEFAULT NULL,\n `ocuygebbounedubprmjtnecqvbsptnzi` int DEFAULT NULL,\n `phjgefnyrmunozglfivrhltytwtpebgx` int DEFAULT NULL,\n `vntzowkvrfbgidoczzjtdwvefmpgnorj` int DEFAULT NULL,\n `eepsrkmcbpaltkhqlxgsllsvyubonbkj` int DEFAULT NULL,\n `tjxxrooqnewxducsajhvjdxjpsbguofb` int DEFAULT NULL,\n `dsdbdynofqroamxadhcxgztaungplyxp` int DEFAULT NULL,\n `xukkiqlxhxthgrxhnhckuvlxjwiklhcq` int DEFAULT NULL,\n `otwxyhtttbfswaomtomtttqhouajnfos` int DEFAULT NULL,\n `okbtngugtdnubhnvkbwvcewzwqysehqg` int DEFAULT NULL,\n `jlhzdaktucmhllckswkcxojhlezmodbj` int DEFAULT NULL,\n `kruovjcflshswtcqcwhtqngpdzllhdei` int DEFAULT NULL,\n `fozymuicmhvqojbindpulfdfdizytiig` int DEFAULT NULL,\n `awgsbzevlchcczmvjeywpmmfahbbxzjl` int DEFAULT NULL,\n `widutzflvafgaeevjlpqheeyficrzdyo` int DEFAULT NULL,\n `oufepvqpguhmwgzxazuylzicrugtccxl` int DEFAULT NULL,\n `lgnjbcthcmhchyxlbitzcszikibqmxlo` int DEFAULT NULL,\n `sitraomntrkkfnxyfydyauxmdtbuljvc` int DEFAULT NULL,\n `vkcemnxivrgheihvwmpuoknqmwwfhyip` int DEFAULT NULL,\n `poqjdbwsgmzxyhchpizxoebcebrunnoq` int DEFAULT NULL,\n `iwwhpztkfjcldzdgfwmxzbdmlrhothia` int DEFAULT NULL,\n `kynwnuhcgbwrkngnmqkdrsiqkxmkpjhb` int DEFAULT NULL,\n `dyjbbkcozzkesyiaojrslmrnixrxoxyr` int DEFAULT NULL,\n `zicdnzmoxqkdpgrdnesqrjqygymxapbq` int DEFAULT NULL,\n `qoffkovgzdrrfeggmbibctqsrcysswbq` int DEFAULT NULL,\n `rtptwhqarfieedoufcvelebkyydlmroo` int DEFAULT NULL,\n `izaotisheoykjksciupuaktcmuonidgd` int DEFAULT NULL,\n `isksyvrswpqdbtbaoqrnylvztjuxlmxo` int DEFAULT NULL,\n `dddfgzzjxdewaqpymtkenahkgmcqbpvy` int DEFAULT NULL,\n `klczjwqkqpnrttusizrmldlqlcrwjwhr` int DEFAULT NULL,\n `hlvhofoifscrgdnolputhegqapsanrox` int DEFAULT NULL,\n `djfsnnsbpcgupchyebdvdspahpvryjfj` int DEFAULT NULL,\n `qxccxotnharczmqrtyvnirpteqihpdzn` int DEFAULT NULL,\n `iwtksyeyrmpjuppilhtbuhnrfnbrcfjh` int DEFAULT NULL,\n `oqsetotiqojrcrdylqvzijannzpsdwwh` int DEFAULT NULL,\n `aioxqhyqgokcvgpksxzpumdkshslqfdr` int DEFAULT NULL,\n `wnhwxnxgecafwjymfgppcapvgdktveez` int DEFAULT NULL,\n `ywtyrlexviqzhbqgyglzpifnavmussox` int DEFAULT NULL,\n `pkpgmwxekvborlrlmbwyasdxgphpbypf` int DEFAULT NULL,\n `efjbxvfcqsrjauerrnjztagvtptzjges` int DEFAULT NULL,\n `leqnwcpaaqcybgedgzdijtjojufxtmlh` int DEFAULT NULL,\n `utfksucmprqfigvftyhcpxevlnxgjqtj` int DEFAULT NULL,\n `enwqrjbaqunmbyzgexjkftiaowsjoiam` int DEFAULT NULL,\n `ddzeddpvfxbpucyorhszjgmhvtsdksnf` int DEFAULT NULL,\n `uefmtelxcdazaqgaxbrmruebqzieclda` int DEFAULT NULL,\n `zyomnvuiefvzwllinwigdyvjraparays` int DEFAULT NULL,\n `wbtujwkzhesdgmgpmeokntqgvwbbnkop` int DEFAULT NULL,\n `yujdklhhqgmfogihezqgxzubfbkotmpl` int DEFAULT NULL,\n `nyufopgptsdilmkspkmqatsbybdzujsr` int DEFAULT NULL,\n `xsrnylpedkgzjltpmscriqdfjsprkvun` int DEFAULT NULL,\n `guyescfgsawbgsodtihcrdavxekufgmb` int DEFAULT NULL,\n `mkbjaakeyegpbwiavjtcegsfaaebhlgl` int DEFAULT NULL,\n `gfcgzhssqspinypaiyfdopdpbuezxacl` int DEFAULT NULL,\n PRIMARY KEY (`njtcrfwvlaagzfkltfciztovpaxagnmd`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"gjwbaxlnvuczlhgbaytrbajtlzxhwmbn\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["njtcrfwvlaagzfkltfciztovpaxagnmd"],"columns":[{"name":"njtcrfwvlaagzfkltfciztovpaxagnmd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"qppcgzbpocoifqzqjdojhueuutozockf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nygtuhahmybntnifvhizjqbitvkvcquz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbtsbdxebwiqmigfrbiqsnnnwtkmcuaz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afrzgftfwqhprkvycpgtsysgkxokhjoz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phxhdusrjzbcdptissluagixglmedlmc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axpojwlqfqisiripnkjobectmhfeqybj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knmgllerhkcyhtulvxfafbwwyiitfnyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpmrdllqbsqqjjttjdkqsrkwmfxclydp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybbqnjvgjdlbjksefadymerajydcwnje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yufzhwkzzpfnrdserxxwaafcxwmcnfsx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kkwuyfcderitldsxntuehzghwomdbsrc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cowocujypqdbweanwmmptuynobjsmweb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sgcajjnrajbftczponytgtardmirhzll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wrrdkfukphgyxswzektnkejzjhqmrpmy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"seqwxtaohzigfsqdtcsecyobjvbnyahk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vlyqkpwlketwjrdpwdoxaxiqybiuekfd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idetokxwmzxyxjrmmnfkkvhadtqiiuqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdmqnaawprhalazuyahcuhvkhflugovp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dehatlfhrhtortidnmfxlpbifbtflwam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hczdfcaqkgldjrgjzahxlyifxsckiopq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qcdbuemvctivaagcxljkuynqrmigiqlw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdprkahbgtfwcwqwkkmjhyjjowdieknu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nuwxtvkoelcfsrywpnghtqgkuydeksqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjbguzbjpflnuelkjgxrshgukxtrnlym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqaowljgocxugiilbxhszquhtlpntsyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axlgdmfwdfyyyzrwxajxzljvdjenxbzi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ramymhrhwynngcpdzppoknhvemremxye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kgmfhzzwhydsswvlmyoodrulspxolbgw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wrnloiqljnzmzpzmppzpkoeuailcntvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prsvqsmocbanaayxdvlyosbwvksghwzq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejbyjmpmewxezdgefaznlyuzwbtlbjeo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hinovgyatrvruhhnngcxlotifqmadoan","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpexunwyocesubjjflexpmghfqisgmyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtxdyowacqdntymvenczlxptfggpbwkl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpyepocyinigckfwfdchakjezaxmuakl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehbhuciuqosfbflkmybiojnidbgzkibt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twqvhfiggmwluhqfoapcubnkgwcdqatj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ncdnobbptghvtjvqrqboskevjevywdwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynfsjrphcxwggwngdhwfwtfypfwrfsnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"soaehwslcdxzbsjvrkbgsoscfjxytnrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kkaqetjujzdlqnaisadwwwdhrecsgvlo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kisuldyuiokwgsvyucichffehuqhfehi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"coninxmywoumjzcjgrwcqtmvumdxgcdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpyygzfznoiujuocklslfffrmdypjthh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vcvhqhofhnprljfyecnhbgulapquyivh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qlwmhgtbhasckldyplycqvwwsrjgzkgt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbuwvdgzvdinfokqkiplydnbrohrqooe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocuygebbounedubprmjtnecqvbsptnzi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phjgefnyrmunozglfivrhltytwtpebgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vntzowkvrfbgidoczzjtdwvefmpgnorj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eepsrkmcbpaltkhqlxgsllsvyubonbkj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjxxrooqnewxducsajhvjdxjpsbguofb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsdbdynofqroamxadhcxgztaungplyxp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xukkiqlxhxthgrxhnhckuvlxjwiklhcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otwxyhtttbfswaomtomtttqhouajnfos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okbtngugtdnubhnvkbwvcewzwqysehqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlhzdaktucmhllckswkcxojhlezmodbj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kruovjcflshswtcqcwhtqngpdzllhdei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fozymuicmhvqojbindpulfdfdizytiig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awgsbzevlchcczmvjeywpmmfahbbxzjl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"widutzflvafgaeevjlpqheeyficrzdyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oufepvqpguhmwgzxazuylzicrugtccxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgnjbcthcmhchyxlbitzcszikibqmxlo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sitraomntrkkfnxyfydyauxmdtbuljvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkcemnxivrgheihvwmpuoknqmwwfhyip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"poqjdbwsgmzxyhchpizxoebcebrunnoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwwhpztkfjcldzdgfwmxzbdmlrhothia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kynwnuhcgbwrkngnmqkdrsiqkxmkpjhb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dyjbbkcozzkesyiaojrslmrnixrxoxyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zicdnzmoxqkdpgrdnesqrjqygymxapbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qoffkovgzdrrfeggmbibctqsrcysswbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtptwhqarfieedoufcvelebkyydlmroo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"izaotisheoykjksciupuaktcmuonidgd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"isksyvrswpqdbtbaoqrnylvztjuxlmxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dddfgzzjxdewaqpymtkenahkgmcqbpvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"klczjwqkqpnrttusizrmldlqlcrwjwhr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlvhofoifscrgdnolputhegqapsanrox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"djfsnnsbpcgupchyebdvdspahpvryjfj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxccxotnharczmqrtyvnirpteqihpdzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwtksyeyrmpjuppilhtbuhnrfnbrcfjh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqsetotiqojrcrdylqvzijannzpsdwwh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aioxqhyqgokcvgpksxzpumdkshslqfdr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnhwxnxgecafwjymfgppcapvgdktveez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywtyrlexviqzhbqgyglzpifnavmussox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkpgmwxekvborlrlmbwyasdxgphpbypf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"efjbxvfcqsrjauerrnjztagvtptzjges","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"leqnwcpaaqcybgedgzdijtjojufxtmlh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utfksucmprqfigvftyhcpxevlnxgjqtj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"enwqrjbaqunmbyzgexjkftiaowsjoiam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddzeddpvfxbpucyorhszjgmhvtsdksnf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uefmtelxcdazaqgaxbrmruebqzieclda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyomnvuiefvzwllinwigdyvjraparays","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbtujwkzhesdgmgpmeokntqgvwbbnkop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yujdklhhqgmfogihezqgxzubfbkotmpl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nyufopgptsdilmkspkmqatsbybdzujsr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsrnylpedkgzjltpmscriqdfjsprkvun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"guyescfgsawbgsodtihcrdavxekufgmb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkbjaakeyegpbwiavjtcegsfaaebhlgl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfcgzhssqspinypaiyfdopdpbuezxacl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668153,"databaseName":"models_schema","ddl":"CREATE TABLE `gmhufugcytipzofjpeipcjeyfkrzjgne` (\n `eihwaovmtdywputrugexisaqirredwsv` int NOT NULL,\n `nngshhpdjzxsipxecbnegechmdaurlkj` int DEFAULT NULL,\n `sqqpbuiobtmplslbzdjswgrhrlsqvaro` int DEFAULT NULL,\n `brlhkedzwepngpbaxocqekomhkdbuevf` int DEFAULT NULL,\n `xcdavngnvatguuepjanntdaxpiadvrou` int DEFAULT NULL,\n `igffclialrtdscmebrdwpayjaarxxezs` int DEFAULT NULL,\n `mlysiirnsndbeiyircqlypmxodznzogl` int DEFAULT NULL,\n `gsqnslzifekhwkrjhgtmnbcfkvcgpryb` int DEFAULT NULL,\n `shvnsanhvxmkpruqgxzvyijwmctqhrsj` int DEFAULT NULL,\n `inpngwizvarkowkqfvfiifrlcnluxjor` int DEFAULT NULL,\n `mnxisxkhlkkvrmhaklhxvgfmenxrpwtk` int DEFAULT NULL,\n `xpmiivvugxymuoulasgyhoamqlxujzye` int DEFAULT NULL,\n `kvvudxxscfmpioqiqujzzzrkkdgqoouy` int DEFAULT NULL,\n `yrpwyrtluzcpmsdqjnuxrzgegqroxlhb` int DEFAULT NULL,\n `xuibnujilwllrfgosyzysdmvdmrmlski` int DEFAULT NULL,\n `nzblzloqjjynbyxpaqcppntgevryxqxa` int DEFAULT NULL,\n `kntdhguhizltymxxfbilpowkmfenexbn` int DEFAULT NULL,\n `ucrfbogfjgxsonrssxxxhcdcpyagcrhn` int DEFAULT NULL,\n `onnckksqvmkxqtyurcfosrtjzwuamrqr` int DEFAULT NULL,\n `byxynzvbvgjizwitgwoafkihvdhpzjfg` int DEFAULT NULL,\n `cbetlchcqmysgttzcdevuxetmhxgxeld` int DEFAULT NULL,\n `mglnhdbrqbdpnlihqobygvlocyfedfvv` int DEFAULT NULL,\n `cnvtxmdscvhlrrlyeypaikdsnabmfird` int DEFAULT NULL,\n `ifdwxiwtjvvscwcfrxyziehcizdlqbuw` int DEFAULT NULL,\n `iuknwpnrdbfjjssfarryujgivpzvbtjn` int DEFAULT NULL,\n `sbnshuvgvtxpdoncegjokipxbzzmwrpt` int DEFAULT NULL,\n `qizqhosbuydybdvcaxckfztcoupysyko` int DEFAULT NULL,\n `clghfsoumosvildbmqmspdgsuxfnyhxy` int DEFAULT NULL,\n `bjjnugirmeteocvrgsrcpkefvjitlcpb` int DEFAULT NULL,\n `ugtvuawtjygmbmjcisykocmqiksgcqxw` int DEFAULT NULL,\n `enbgrwyasnyadutltqafayuwmyoammio` int DEFAULT NULL,\n `vyrrcvzlncodsjcugoctvcagsvsuxhup` int DEFAULT NULL,\n `nvusulhgyvjlarohvzrvtlcnvxhxuvmr` int DEFAULT NULL,\n `zfavqefjdjbepgzxptdspmcgzononwjf` int DEFAULT NULL,\n `rtytmirybqmemddhzhiauusqixuktiup` int DEFAULT NULL,\n `wiueqctoyxymyzohvzgvreuzrzxxgpty` int DEFAULT NULL,\n `fanuayruluihrzxjmwzxhafnbswjfexe` int DEFAULT NULL,\n `mtgzgspkbalwkufpjlaixzybauuunfyb` int DEFAULT NULL,\n `cvlslsxzulwerehttqzunpmfsceclwse` int DEFAULT NULL,\n `rbortylgqpvdayyjmeynypzowqmcmwat` int DEFAULT NULL,\n `datpknibodzqputezoejhufnansulmow` int DEFAULT NULL,\n `tmyabnobwxpelqusvtpqnvgiaupskutf` int DEFAULT NULL,\n `mzxgoehxuzsvyhiobxzpmlzythsoqxqd` int DEFAULT NULL,\n `bwinfgskaiqwvzmlvfnifnstcfbegkoq` int DEFAULT NULL,\n `zyxzgntehtmcrvddcfloapwtkldaczni` int DEFAULT NULL,\n `whiutmhlunibhrlfyjjilwhsgnaqjnfb` int DEFAULT NULL,\n `bgmzozipsecvsdihqfojuwjurnuufqgt` int DEFAULT NULL,\n `djesoqzxupkfgmqnefyhnxoutssjvtxw` int DEFAULT NULL,\n `yohxxamijynigldctdkjjleiojmzaorc` int DEFAULT NULL,\n `qnzrpybzowdxxdrafyxsqnbvxispffrr` int DEFAULT NULL,\n `ziralkatuqbrtnaxrahsmychlautymtd` int DEFAULT NULL,\n `kmbpumjifejjgjgviiklqlmlsufiqual` int DEFAULT NULL,\n `aprxzqaixjevbtwhksrilozlzbboiiae` int DEFAULT NULL,\n `ytahbarnvicxpbtnvpdmwtfoyjytgfqw` int DEFAULT NULL,\n `aavqbxunxkvwcgrixjfbwdzbciytsbwf` int DEFAULT NULL,\n `sdsytnacmontloawffonyzznurhuwlnd` int DEFAULT NULL,\n `suzzoqlihgbabktvmnrvgmplazujneia` int DEFAULT NULL,\n `nbjjoiblsehbhmlqypkbmgqilsnwohec` int DEFAULT NULL,\n `okuzjglpeoruzijyumckrbchghpkzohd` int DEFAULT NULL,\n `txtqstmgrsfgynyguunpfhjcrdrrlpvo` int DEFAULT NULL,\n `pztdvumyivdjnarpvpyinsmmkossbyui` int DEFAULT NULL,\n `oqnzuwvvfiilpilaqhjrzhubmympcqhe` int DEFAULT NULL,\n `xhjdjitlxyegrrlyxjtyxqmjyfrwmzag` int DEFAULT NULL,\n `gghdbrgzkrswqlzrgdpzghqaybqwzxfp` int DEFAULT NULL,\n `iozbimiaurhnhcplodlwxxjvwaokxlze` int DEFAULT NULL,\n `xosxazzjmklqftquwchgwqcylmcrbgex` int DEFAULT NULL,\n `cxxsviyfchbilzkhqbdsvqusxeoqnudm` int DEFAULT NULL,\n `bdagbyzxrwsoyfzxhzekuczqbdmeaslf` int DEFAULT NULL,\n `zbhpkopdqeaqerolpkxvbijulcetebsq` int DEFAULT NULL,\n `ognmtiefduvwehrxhccdyoozvavndnnx` int DEFAULT NULL,\n `npubuiyhantcujoaykrbrbfnizysnsit` int DEFAULT NULL,\n `xrbjqctmapzzicpehnjmbdfinhokzuiw` int DEFAULT NULL,\n `ttirnnameyqqorzxbjqvuaeinbnzepvd` int DEFAULT NULL,\n `eyanngwzevxdqsuurmvejpxclbluqfdn` int DEFAULT NULL,\n `kvfhocwyfmzpflbtboozxuzklemencuy` int DEFAULT NULL,\n `vpwyyxtdbhyqxziwixzdiggjxlytaykt` int DEFAULT NULL,\n `uhlozwdslziicsgzfxrlducesmbsigvo` int DEFAULT NULL,\n `desuuorprdoyqkhbrbaehpwsvmzlhpia` int DEFAULT NULL,\n `nxwkgpyivedebfdqnaxywuhpwmelcfly` int DEFAULT NULL,\n `upyimkbwwhtnpfhktnnnmtupxxwxlnam` int DEFAULT NULL,\n `kzeguyrsghdgirjjokemwidwtwpomvba` int DEFAULT NULL,\n `cenjjdjaqmzyydomnysyxzpvkgvqlsie` int DEFAULT NULL,\n `obqbgfvymlkxbkwxtpqoshneeymvwyad` int DEFAULT NULL,\n `raglcmqiarlxfsxzzwuunwftejefvaxn` int DEFAULT NULL,\n `aovmzwnstfiybtcbzldqggazwlckrtzu` int DEFAULT NULL,\n `qkmroupszacuzcptxfucdrzlramplklz` int DEFAULT NULL,\n `yjtloiogwssiqbvkiljkgdlqmsbtqefw` int DEFAULT NULL,\n `sykfukczbwdkdsvdiwqqddowbbdclneo` int DEFAULT NULL,\n `mokwzrpslhnynhqizrhplslqisxqlsbz` int DEFAULT NULL,\n `oqlaopopollljtcfkvxfwwmtnkyiuchw` int DEFAULT NULL,\n `zzegfhyqumcbtinkfpbqkvkmnrudlcrh` int DEFAULT NULL,\n `txdzcazztlqyexngiwybgvhvddsdjtdx` int DEFAULT NULL,\n `psmaakeymyycjzcxkuffoillepspxzyt` int DEFAULT NULL,\n `szdvyoiwffesswqwiaxmknpwuhgtqfuc` int DEFAULT NULL,\n `dhsvuisrwshmccyokcbhvgejzebvxhuu` int DEFAULT NULL,\n `nfogiiqilmxhbapscdilpwldgyvxnhid` int DEFAULT NULL,\n `tqpagvypcdymsqikzpjhccndfgdfbaqh` int DEFAULT NULL,\n `cctesgplevazqszlgjwrdfwwtyqxmpgu` int DEFAULT NULL,\n `osknkiqnfpseuuqfkbgyfjiavrgzoexx` int DEFAULT NULL,\n `siemickgegyuqpatfevsoydspatbstfo` int DEFAULT NULL,\n PRIMARY KEY (`eihwaovmtdywputrugexisaqirredwsv`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"gmhufugcytipzofjpeipcjeyfkrzjgne\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["eihwaovmtdywputrugexisaqirredwsv"],"columns":[{"name":"eihwaovmtdywputrugexisaqirredwsv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"nngshhpdjzxsipxecbnegechmdaurlkj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sqqpbuiobtmplslbzdjswgrhrlsqvaro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"brlhkedzwepngpbaxocqekomhkdbuevf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xcdavngnvatguuepjanntdaxpiadvrou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igffclialrtdscmebrdwpayjaarxxezs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlysiirnsndbeiyircqlypmxodznzogl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsqnslzifekhwkrjhgtmnbcfkvcgpryb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shvnsanhvxmkpruqgxzvyijwmctqhrsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"inpngwizvarkowkqfvfiifrlcnluxjor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnxisxkhlkkvrmhaklhxvgfmenxrpwtk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpmiivvugxymuoulasgyhoamqlxujzye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvvudxxscfmpioqiqujzzzrkkdgqoouy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yrpwyrtluzcpmsdqjnuxrzgegqroxlhb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xuibnujilwllrfgosyzysdmvdmrmlski","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzblzloqjjynbyxpaqcppntgevryxqxa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kntdhguhizltymxxfbilpowkmfenexbn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucrfbogfjgxsonrssxxxhcdcpyagcrhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"onnckksqvmkxqtyurcfosrtjzwuamrqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"byxynzvbvgjizwitgwoafkihvdhpzjfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbetlchcqmysgttzcdevuxetmhxgxeld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mglnhdbrqbdpnlihqobygvlocyfedfvv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnvtxmdscvhlrrlyeypaikdsnabmfird","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifdwxiwtjvvscwcfrxyziehcizdlqbuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iuknwpnrdbfjjssfarryujgivpzvbtjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbnshuvgvtxpdoncegjokipxbzzmwrpt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qizqhosbuydybdvcaxckfztcoupysyko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clghfsoumosvildbmqmspdgsuxfnyhxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjjnugirmeteocvrgsrcpkefvjitlcpb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugtvuawtjygmbmjcisykocmqiksgcqxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"enbgrwyasnyadutltqafayuwmyoammio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vyrrcvzlncodsjcugoctvcagsvsuxhup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvusulhgyvjlarohvzrvtlcnvxhxuvmr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zfavqefjdjbepgzxptdspmcgzononwjf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtytmirybqmemddhzhiauusqixuktiup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wiueqctoyxymyzohvzgvreuzrzxxgpty","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fanuayruluihrzxjmwzxhafnbswjfexe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtgzgspkbalwkufpjlaixzybauuunfyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvlslsxzulwerehttqzunpmfsceclwse","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbortylgqpvdayyjmeynypzowqmcmwat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"datpknibodzqputezoejhufnansulmow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmyabnobwxpelqusvtpqnvgiaupskutf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzxgoehxuzsvyhiobxzpmlzythsoqxqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bwinfgskaiqwvzmlvfnifnstcfbegkoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyxzgntehtmcrvddcfloapwtkldaczni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"whiutmhlunibhrlfyjjilwhsgnaqjnfb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bgmzozipsecvsdihqfojuwjurnuufqgt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"djesoqzxupkfgmqnefyhnxoutssjvtxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yohxxamijynigldctdkjjleiojmzaorc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qnzrpybzowdxxdrafyxsqnbvxispffrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ziralkatuqbrtnaxrahsmychlautymtd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmbpumjifejjgjgviiklqlmlsufiqual","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aprxzqaixjevbtwhksrilozlzbboiiae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytahbarnvicxpbtnvpdmwtfoyjytgfqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aavqbxunxkvwcgrixjfbwdzbciytsbwf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdsytnacmontloawffonyzznurhuwlnd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suzzoqlihgbabktvmnrvgmplazujneia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbjjoiblsehbhmlqypkbmgqilsnwohec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okuzjglpeoruzijyumckrbchghpkzohd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txtqstmgrsfgynyguunpfhjcrdrrlpvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pztdvumyivdjnarpvpyinsmmkossbyui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqnzuwvvfiilpilaqhjrzhubmympcqhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xhjdjitlxyegrrlyxjtyxqmjyfrwmzag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gghdbrgzkrswqlzrgdpzghqaybqwzxfp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iozbimiaurhnhcplodlwxxjvwaokxlze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xosxazzjmklqftquwchgwqcylmcrbgex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cxxsviyfchbilzkhqbdsvqusxeoqnudm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdagbyzxrwsoyfzxhzekuczqbdmeaslf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbhpkopdqeaqerolpkxvbijulcetebsq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ognmtiefduvwehrxhccdyoozvavndnnx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npubuiyhantcujoaykrbrbfnizysnsit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrbjqctmapzzicpehnjmbdfinhokzuiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ttirnnameyqqorzxbjqvuaeinbnzepvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eyanngwzevxdqsuurmvejpxclbluqfdn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvfhocwyfmzpflbtboozxuzklemencuy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpwyyxtdbhyqxziwixzdiggjxlytaykt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhlozwdslziicsgzfxrlducesmbsigvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"desuuorprdoyqkhbrbaehpwsvmzlhpia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxwkgpyivedebfdqnaxywuhpwmelcfly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upyimkbwwhtnpfhktnnnmtupxxwxlnam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzeguyrsghdgirjjokemwidwtwpomvba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cenjjdjaqmzyydomnysyxzpvkgvqlsie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obqbgfvymlkxbkwxtpqoshneeymvwyad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"raglcmqiarlxfsxzzwuunwftejefvaxn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aovmzwnstfiybtcbzldqggazwlckrtzu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qkmroupszacuzcptxfucdrzlramplklz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjtloiogwssiqbvkiljkgdlqmsbtqefw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sykfukczbwdkdsvdiwqqddowbbdclneo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mokwzrpslhnynhqizrhplslqisxqlsbz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqlaopopollljtcfkvxfwwmtnkyiuchw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzegfhyqumcbtinkfpbqkvkmnrudlcrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txdzcazztlqyexngiwybgvhvddsdjtdx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psmaakeymyycjzcxkuffoillepspxzyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"szdvyoiwffesswqwiaxmknpwuhgtqfuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhsvuisrwshmccyokcbhvgejzebvxhuu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfogiiqilmxhbapscdilpwldgyvxnhid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqpagvypcdymsqikzpjhccndfgdfbaqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cctesgplevazqszlgjwrdfwwtyqxmpgu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"osknkiqnfpseuuqfkbgyfjiavrgzoexx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"siemickgegyuqpatfevsoydspatbstfo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668189,"databaseName":"models_schema","ddl":"CREATE TABLE `gnrwwzzyowsyoqcxntaclfwfmptzfttq` (\n `dcdsydckscaktwwkwekhyqhgqsdxpyve` int NOT NULL,\n `hqxfusvdukeyomlvesyegeqopukqeorl` int DEFAULT NULL,\n `kokcszpcwillhvykknyjlmdlnkgdzwws` int DEFAULT NULL,\n `ekktdjuhnwldigjkkymzixutmmitkipb` int DEFAULT NULL,\n `vgsnfzemeivfmsqshpdebhnmzpkvfzbe` int DEFAULT NULL,\n `tnfiufjcdtnumphfopywchvoynyhwfwd` int DEFAULT NULL,\n `xofdibvhppusqszcqazxlsgdozzplddy` int DEFAULT NULL,\n `gldehwlgmvljoevaiaolfnzbqsjuomor` int DEFAULT NULL,\n `bjdrgpwkduzeuyrsdjqpejtevyuztpcr` int DEFAULT NULL,\n `bryborluekaptctsvaauphzgofmjgyun` int DEFAULT NULL,\n `rbfmhiduzghnueyzmgncfswmbimxsduu` int DEFAULT NULL,\n `vqxknxdtvvpgkufzsmpitoydltpvkttw` int DEFAULT NULL,\n `cbbgzkmofrlivzyqvggzukebkyalwxwz` int DEFAULT NULL,\n `mfwlwpmvwtnzmyhyndldqdgstkxsmmqg` int DEFAULT NULL,\n `cntvrujuuqyykjddypxscyilpakgciva` int DEFAULT NULL,\n `bbtvqucjqjciffqcjlbogjcqdzybjkog` int DEFAULT NULL,\n `pwkgghbzowvcawkrjpamrcymllucpezb` int DEFAULT NULL,\n `kwvylknwxzsbjwdvgolwucgzdkdwehhv` int DEFAULT NULL,\n `pycsqvnjbajdrnrucukzeczeepofyuka` int DEFAULT NULL,\n `kaonulvglgrubvhqkulnabbqdxnzpkkw` int DEFAULT NULL,\n `gwubgciikfbbglieamornpbtdorxavfs` int DEFAULT NULL,\n `ptthjnmwhjtqzitvrfbkutmgurdqfhwv` int DEFAULT NULL,\n `qhksjsxcgbmxkjeuzkxrzkxcohdfkpkq` int DEFAULT NULL,\n `odixylzbdsiihehhozuztlqabokvenri` int DEFAULT NULL,\n `xkfnixnajobdkpiylbgcgzobjxxgouqd` int DEFAULT NULL,\n `jopxdctxgrcpzuvkgovuuxtdtnhbliko` int DEFAULT NULL,\n `ytaqbbkdxyfqnasbzdgdiqxveceulhxy` int DEFAULT NULL,\n `txvgfmugbubphkhqluomqbsnzuuoxndu` int DEFAULT NULL,\n `bcdvirdendwysjagcqppzvxoysbnrcts` int DEFAULT NULL,\n `zbkexxpxvxrhdhjgapetufsvckmckbtq` int DEFAULT NULL,\n `bhiengsqqsgwpbxatydhfagjblmonkip` int DEFAULT NULL,\n `uttxmqmvindmltmnkkuvctyknlovrtkb` int DEFAULT NULL,\n `bzxhveljhzeuxyegdntkpduorjwvyvke` int DEFAULT NULL,\n `loijdgtnfyrbrdlypizucweyunzkjmsg` int DEFAULT NULL,\n `vdpnhljiqhhmsyotvryljwiqayquabdn` int DEFAULT NULL,\n `zfvehwyzheefbqntxmmarysaqgknzfkr` int DEFAULT NULL,\n `qbnejlhoctjwiqtnkystcluwgkwmicso` int DEFAULT NULL,\n `ewrobcanebnhkkyeggbironvnxnmlxiw` int DEFAULT NULL,\n `xceflkwpquhxiazazapjefujscafgzin` int DEFAULT NULL,\n `nzsahgxtzautvpjblvepfbjjnqdqndcl` int DEFAULT NULL,\n `unitmjtlmgqovvgaffnzukrczpharlmv` int DEFAULT NULL,\n `vedonwwpxdfegxzpkjjalvgpufjrcgbg` int DEFAULT NULL,\n `uxmedazjnhdodwqaucmysrrieqfnfkbt` int DEFAULT NULL,\n `jsnmouhajdwmtznmwqgtkojwlzkreibb` int DEFAULT NULL,\n `koeqzhrytzbmjanrxgathqnjxeslgyqe` int DEFAULT NULL,\n `bpqgrohgacxjiqopxmdfgdrpwugobxaj` int DEFAULT NULL,\n `jtsqkqmqpziqlzskiwubfygqhteodgqo` int DEFAULT NULL,\n `zbvpuajhrnjyfnhgqiupukyfymnepspg` int DEFAULT NULL,\n `qlqieqqmfelfxqzbfagtspurcnhoiwbg` int DEFAULT NULL,\n `uvyltqsvkpdptcfplahiaqgeldzqyerr` int DEFAULT NULL,\n `cbcbkxabqrtrfdhkwlrrhgmgqdylblwx` int DEFAULT NULL,\n `hmdklzihgccuyvxwkgaqwihrtsinpbyj` int DEFAULT NULL,\n `rjtyvvqamalcbqefnzptwyghbbywcynv` int DEFAULT NULL,\n `fdabdfbsnktsondimstrnisgxnrvlolr` int DEFAULT NULL,\n `mcxyygfxuptisswoizdpgawqzokknzbv` int DEFAULT NULL,\n `xyjpwtemowsjjvsssntyrhzqlptssufj` int DEFAULT NULL,\n `yosrbzumznvfitbjacqxmhfutsdzxsft` int DEFAULT NULL,\n `hmyzimstslwndqacfrqggvbknzumwcvf` int DEFAULT NULL,\n `pkhcymgyiqcosxtoqjwutxwcperfxspc` int DEFAULT NULL,\n `gspaxwvbvgxcssfcjxptnszzvyyompvj` int DEFAULT NULL,\n `zvmmywloowcydelwfogmhwahgihgbqaj` int DEFAULT NULL,\n `bahlxytesebvuarhzytasrzenyickxrf` int DEFAULT NULL,\n `itdbnobrmzemynefffdglgtsyygbewvh` int DEFAULT NULL,\n `ecaecuuekomupnvxmzjgqsdpdelitmhx` int DEFAULT NULL,\n `hagrvxzhonpnjskzczgktdltjwfpyjwo` int DEFAULT NULL,\n `qsixzcqxnkkzvhxoruifacfkaoubuvhj` int DEFAULT NULL,\n `cggieynybuomkcbsojhqhjbcbyohmvrx` int DEFAULT NULL,\n `hmxizxzywmigazivbtqisbockbhswrvk` int DEFAULT NULL,\n `ielvxyhfymvmyglepnyrezabhrdiaxdk` int DEFAULT NULL,\n `cjjybybwpkxiwfcqyxxicrgbywbhrzoy` int DEFAULT NULL,\n `nswxeutyatznxikmxidpzwdmtjzleeuu` int DEFAULT NULL,\n `qjzjuxmekefzuksoqnlyifiekezluyho` int DEFAULT NULL,\n `saqjhklqjhwrvlnysepwfhtfxfgllbge` int DEFAULT NULL,\n `fbaqgdzwathkuxlfboeualpzyviobklh` int DEFAULT NULL,\n `fqtlchnuqqcdtfgxacctbzpabypklmuz` int DEFAULT NULL,\n `jaqbwdxxidrrrdpkduivrwhphgjovzso` int DEFAULT NULL,\n `zdawfwiqztavjqlzizmwgxgvrsqbupdv` int DEFAULT NULL,\n `omvriarhwybibysltpakiuoelinqjxko` int DEFAULT NULL,\n `vjcjankwrwitcsrvapkvqtncpsznqsyr` int DEFAULT NULL,\n `fuypvgbueqvcpjohqxqlpgdwoxbihfro` int DEFAULT NULL,\n `zencjyvtxbiblzlupvyzytatmbvpdesl` int DEFAULT NULL,\n `ngsvfsvizdpyijicxapurkapxcvvhnty` int DEFAULT NULL,\n `lnxytsgwuankrjsfkwyirilvsdmelcjo` int DEFAULT NULL,\n `hycwimiodrxapofodtazkspxpgprkeiq` int DEFAULT NULL,\n `yxhonowvfxdeudzktvwqbbmxqoyqrnuv` int DEFAULT NULL,\n `objxxikyuvrbonhpahesuweuxemavngm` int DEFAULT NULL,\n `akylbyzkcjmhvyxmcheuywhrzqwavvfq` int DEFAULT NULL,\n `euvvindkagcexmapfvtyimjwstfpzjkk` int DEFAULT NULL,\n `bobldwsygbgheowdcvkdbwqjlhmldcoq` int DEFAULT NULL,\n `uervuuduuenxwgzkykpeftifdzvwwijf` int DEFAULT NULL,\n `fhitdiumkvemaeiqkwsjdqxioltzblqm` int DEFAULT NULL,\n `iyyxgqylibulbkwkcdkvujztyalphmrz` int DEFAULT NULL,\n `kgquvpxsbzjylqsgwkivicernysgqbqw` int DEFAULT NULL,\n `irdgxcopwzmqbohbuxlfpwidyglgorom` int DEFAULT NULL,\n `rtsozqqqruqnvtnfasjeodiarmkctnpk` int DEFAULT NULL,\n `gzfmcijrvmuvokdevwszvgygtzynjqpj` int DEFAULT NULL,\n `qpbuanilxhxcazmmuorpurprjkavwoob` int DEFAULT NULL,\n `azdtqcgdusfgwehppqyjqdbbzpzipoqj` int DEFAULT NULL,\n `tinjplqnrdbhnurfanrnugdxebbyhoip` int DEFAULT NULL,\n `qtsnxodblkbhbaqrupefgqfudxawvmay` int DEFAULT NULL,\n PRIMARY KEY (`dcdsydckscaktwwkwekhyqhgqsdxpyve`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"gnrwwzzyowsyoqcxntaclfwfmptzfttq\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["dcdsydckscaktwwkwekhyqhgqsdxpyve"],"columns":[{"name":"dcdsydckscaktwwkwekhyqhgqsdxpyve","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"hqxfusvdukeyomlvesyegeqopukqeorl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kokcszpcwillhvykknyjlmdlnkgdzwws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekktdjuhnwldigjkkymzixutmmitkipb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgsnfzemeivfmsqshpdebhnmzpkvfzbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnfiufjcdtnumphfopywchvoynyhwfwd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xofdibvhppusqszcqazxlsgdozzplddy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gldehwlgmvljoevaiaolfnzbqsjuomor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjdrgpwkduzeuyrsdjqpejtevyuztpcr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bryborluekaptctsvaauphzgofmjgyun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbfmhiduzghnueyzmgncfswmbimxsduu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqxknxdtvvpgkufzsmpitoydltpvkttw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbbgzkmofrlivzyqvggzukebkyalwxwz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfwlwpmvwtnzmyhyndldqdgstkxsmmqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cntvrujuuqyykjddypxscyilpakgciva","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbtvqucjqjciffqcjlbogjcqdzybjkog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwkgghbzowvcawkrjpamrcymllucpezb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwvylknwxzsbjwdvgolwucgzdkdwehhv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pycsqvnjbajdrnrucukzeczeepofyuka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kaonulvglgrubvhqkulnabbqdxnzpkkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gwubgciikfbbglieamornpbtdorxavfs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ptthjnmwhjtqzitvrfbkutmgurdqfhwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhksjsxcgbmxkjeuzkxrzkxcohdfkpkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"odixylzbdsiihehhozuztlqabokvenri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkfnixnajobdkpiylbgcgzobjxxgouqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jopxdctxgrcpzuvkgovuuxtdtnhbliko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytaqbbkdxyfqnasbzdgdiqxveceulhxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txvgfmugbubphkhqluomqbsnzuuoxndu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bcdvirdendwysjagcqppzvxoysbnrcts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbkexxpxvxrhdhjgapetufsvckmckbtq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bhiengsqqsgwpbxatydhfagjblmonkip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uttxmqmvindmltmnkkuvctyknlovrtkb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzxhveljhzeuxyegdntkpduorjwvyvke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"loijdgtnfyrbrdlypizucweyunzkjmsg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdpnhljiqhhmsyotvryljwiqayquabdn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zfvehwyzheefbqntxmmarysaqgknzfkr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qbnejlhoctjwiqtnkystcluwgkwmicso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ewrobcanebnhkkyeggbironvnxnmlxiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xceflkwpquhxiazazapjefujscafgzin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzsahgxtzautvpjblvepfbjjnqdqndcl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unitmjtlmgqovvgaffnzukrczpharlmv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vedonwwpxdfegxzpkjjalvgpufjrcgbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxmedazjnhdodwqaucmysrrieqfnfkbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jsnmouhajdwmtznmwqgtkojwlzkreibb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"koeqzhrytzbmjanrxgathqnjxeslgyqe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bpqgrohgacxjiqopxmdfgdrpwugobxaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jtsqkqmqpziqlzskiwubfygqhteodgqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbvpuajhrnjyfnhgqiupukyfymnepspg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qlqieqqmfelfxqzbfagtspurcnhoiwbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvyltqsvkpdptcfplahiaqgeldzqyerr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbcbkxabqrtrfdhkwlrrhgmgqdylblwx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmdklzihgccuyvxwkgaqwihrtsinpbyj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjtyvvqamalcbqefnzptwyghbbywcynv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fdabdfbsnktsondimstrnisgxnrvlolr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcxyygfxuptisswoizdpgawqzokknzbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyjpwtemowsjjvsssntyrhzqlptssufj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yosrbzumznvfitbjacqxmhfutsdzxsft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmyzimstslwndqacfrqggvbknzumwcvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkhcymgyiqcosxtoqjwutxwcperfxspc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gspaxwvbvgxcssfcjxptnszzvyyompvj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvmmywloowcydelwfogmhwahgihgbqaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bahlxytesebvuarhzytasrzenyickxrf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itdbnobrmzemynefffdglgtsyygbewvh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecaecuuekomupnvxmzjgqsdpdelitmhx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hagrvxzhonpnjskzczgktdltjwfpyjwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qsixzcqxnkkzvhxoruifacfkaoubuvhj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cggieynybuomkcbsojhqhjbcbyohmvrx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmxizxzywmigazivbtqisbockbhswrvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ielvxyhfymvmyglepnyrezabhrdiaxdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjjybybwpkxiwfcqyxxicrgbywbhrzoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nswxeutyatznxikmxidpzwdmtjzleeuu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjzjuxmekefzuksoqnlyifiekezluyho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"saqjhklqjhwrvlnysepwfhtfxfgllbge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbaqgdzwathkuxlfboeualpzyviobklh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqtlchnuqqcdtfgxacctbzpabypklmuz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jaqbwdxxidrrrdpkduivrwhphgjovzso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdawfwiqztavjqlzizmwgxgvrsqbupdv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omvriarhwybibysltpakiuoelinqjxko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjcjankwrwitcsrvapkvqtncpsznqsyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fuypvgbueqvcpjohqxqlpgdwoxbihfro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zencjyvtxbiblzlupvyzytatmbvpdesl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ngsvfsvizdpyijicxapurkapxcvvhnty","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lnxytsgwuankrjsfkwyirilvsdmelcjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hycwimiodrxapofodtazkspxpgprkeiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxhonowvfxdeudzktvwqbbmxqoyqrnuv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"objxxikyuvrbonhpahesuweuxemavngm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akylbyzkcjmhvyxmcheuywhrzqwavvfq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"euvvindkagcexmapfvtyimjwstfpzjkk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bobldwsygbgheowdcvkdbwqjlhmldcoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uervuuduuenxwgzkykpeftifdzvwwijf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhitdiumkvemaeiqkwsjdqxioltzblqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iyyxgqylibulbkwkcdkvujztyalphmrz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kgquvpxsbzjylqsgwkivicernysgqbqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"irdgxcopwzmqbohbuxlfpwidyglgorom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtsozqqqruqnvtnfasjeodiarmkctnpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzfmcijrvmuvokdevwszvgygtzynjqpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qpbuanilxhxcazmmuorpurprjkavwoob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azdtqcgdusfgwehppqyjqdbbzpzipoqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tinjplqnrdbhnurfanrnugdxebbyhoip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtsnxodblkbhbaqrupefgqfudxawvmay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668220,"databaseName":"models_schema","ddl":"CREATE TABLE `gqrdivdtlwyumyqwlhrxuviktdbmfnew` (\n `jnliwlfnhmcdilsabwixdaruynfiysdn` int NOT NULL,\n `sngyzgprfrinnvsoxbpoyawforkhebtc` int DEFAULT NULL,\n `wlfpquyaoohqtuljndlxqejdvlocanpf` int DEFAULT NULL,\n `tjvtjvcpvcbepsrpnuxfyddvbvwzrels` int DEFAULT NULL,\n `bghffcpvrgpeccpfobqvnoazrlwcsjlw` int DEFAULT NULL,\n `vlckbvdiazeeptpowbfhbadksfvpkhdc` int DEFAULT NULL,\n `ptpxshrgvfnwfhpffawksdlbtlksvplf` int DEFAULT NULL,\n `xkbpfnjcotcwzkpikvtwbgiupryhrirm` int DEFAULT NULL,\n `lkpdjegtvefhyuszvlasmnkthwmrsebd` int DEFAULT NULL,\n `gheqcqiwmlxzinmahmveajvrltborwnv` int DEFAULT NULL,\n `amrvnkijtjidljhieoeyjumumiegkpgp` int DEFAULT NULL,\n `huzucapjdmhufobhplmoplwcbeawnvno` int DEFAULT NULL,\n `hnvmugreiimwnbynqlsxbleppjvlopgq` int DEFAULT NULL,\n `ixhazvsxqlbytxgcxkloyxrxptjpzmtz` int DEFAULT NULL,\n `edcwlwijfukftgirxdnmspzcesdypafw` int DEFAULT NULL,\n `waimjzbdrvdijfrglwqfmimnakhdjlzl` int DEFAULT NULL,\n `rdqepxtoyprjscpuahpcvrbymzxjcvjc` int DEFAULT NULL,\n `ouhkmllitjtkfwcbneihaaunztrqpwqr` int DEFAULT NULL,\n `oclnmaxjorujjcufilqosjgepilbuzor` int DEFAULT NULL,\n `qmrtngbyotrefgkcwtjwsohvodkxwfyl` int DEFAULT NULL,\n `tgxrbpbyiyjhbhbbvmnqxqeomewjribh` int DEFAULT NULL,\n `hexuxgbnxwubunxemcdtpcbndwdxggxy` int DEFAULT NULL,\n `mrauewrvjhsxqdkgukdkqurescuxidgq` int DEFAULT NULL,\n `bqknatitndouebazsvxtgphytotqmxsq` int DEFAULT NULL,\n `swzesqvdxgnngudgbgwzicjzjojdtdua` int DEFAULT NULL,\n `ltvmfbbfbtbhshgrrmxptzwpadtpvdqn` int DEFAULT NULL,\n `lssxszcdtqrhsxhgappdgtbbytkengdp` int DEFAULT NULL,\n `nkeaweqgirslpvdfvmgjeyvnnapirqxs` int DEFAULT NULL,\n `tzwozklyvzsejekyumppqayperefjmai` int DEFAULT NULL,\n `usrjevrlodhmqaxhlhmttcvtndlhcxvf` int DEFAULT NULL,\n `dhbadwcqwxxkxywinznyrhhihxwhuruf` int DEFAULT NULL,\n `mlygzyvpdpqbavsnjcriqndtwuxefoxo` int DEFAULT NULL,\n `bupmzmnsmgcnlhpmdkostxdlhytetman` int DEFAULT NULL,\n `wqhypehpllkxyqvllaqqhywpvqxbhljn` int DEFAULT NULL,\n `krnzsecvqxlilumhuknqagmzittjukop` int DEFAULT NULL,\n `cvlbwjijtefhmppxghbdwnglvgjcpasf` int DEFAULT NULL,\n `yggraeevkhxlypskbqrbrrdeplghscjt` int DEFAULT NULL,\n `bbieyebhzgocajgrhiapzptkbnxjqblc` int DEFAULT NULL,\n `uzwtjfuppednuazncewmbyxviyvubgou` int DEFAULT NULL,\n `pveegafutoqisfxskkrkwzjitrojbryc` int DEFAULT NULL,\n `njfvndxivwebvfjxhrafpccjitivjdof` int DEFAULT NULL,\n `dlcicspbxdixdwyqwdorwqbkumwpbejk` int DEFAULT NULL,\n `uwngjczkazwwvxgkqkduebrahvhzxmtu` int DEFAULT NULL,\n `dtwlgscbbcvrpcdmlaxerthhgbzaedcw` int DEFAULT NULL,\n `uurhmrncrjaduintnkfczevbdpfkqqeo` int DEFAULT NULL,\n `rdcwuzqjvsvhdodooqlatdfuweqnxnep` int DEFAULT NULL,\n `sfccdkrefpcjhmxstahzglkplvqyrjsn` int DEFAULT NULL,\n `yigwfigvwikloqopthqvbzxdzfopujzb` int DEFAULT NULL,\n `lxnpwbmsjpornxtwvjajkbzmiazwttqm` int DEFAULT NULL,\n `tdjfatjcsemzqewzuznplfhpvgxtdwcy` int DEFAULT NULL,\n `pjnwnofyotepygxikhzdfiooizotbwys` int DEFAULT NULL,\n `qiualbbzjkfdtpcsdpysiqozrbxomghy` int DEFAULT NULL,\n `deueauejomzqdvxuxgmyvubmbprpjmvb` int DEFAULT NULL,\n `rfxqqrhdtqvnnzlwmkhmgattyhvyvtbp` int DEFAULT NULL,\n `daudoxerclvzhigictuvchmqcclacklw` int DEFAULT NULL,\n `jjdmemjwqyjaxsviwfvfkawwqlxoqqdq` int DEFAULT NULL,\n `fmugpuajwuoeqsvjpzepoxumtabirurf` int DEFAULT NULL,\n `kawkvtuibyyvypabdyhjuranorzanerx` int DEFAULT NULL,\n `cqnfdvlypdfggatmypvecbtsfnlkatde` int DEFAULT NULL,\n `naifmurypatzrgtrqyqabbvvdfpkphrh` int DEFAULT NULL,\n `vrhoiiwsbkyovmnzdzalxebxhljnwhef` int DEFAULT NULL,\n `iqgfguyqlshehsuxfxycunfjquxihiia` int DEFAULT NULL,\n `qrbyoynoqcpvhjlcrmyhophohmccjqyx` int DEFAULT NULL,\n `aydcaxwssjspuolkqnpuwevpyjjnvjeg` int DEFAULT NULL,\n `ozqjlaaxwhhwwdwdqxroofdnoafinglb` int DEFAULT NULL,\n `huxbupvfavfertqbnrbrkvdvdljommwr` int DEFAULT NULL,\n `beicrzbrnjrndxgagyklcqfszkynoyaw` int DEFAULT NULL,\n `gfgatfagxeqegznhmhkmowoxxhrzlcbi` int DEFAULT NULL,\n `vscnquetnlagzkugvscajtsopngihfjl` int DEFAULT NULL,\n `upzaayfmwmjyoqwykotowjjlphlqvltr` int DEFAULT NULL,\n `ypjnjgtuybuokhxituaoemsvnilenwuk` int DEFAULT NULL,\n `vdlqfzwuciiuichuovijxurtkecpbrfx` int DEFAULT NULL,\n `vpjusgqsglcvmnwftugintvgijwufvld` int DEFAULT NULL,\n `azdwfpknuolrznuugqdsclbfyxkitpkt` int DEFAULT NULL,\n `uhckshfnhqdzgzvswdvboidqlrethbyt` int DEFAULT NULL,\n `ugxlrexbafhousfwvksblrxydktfjpif` int DEFAULT NULL,\n `tuzcsdxzzgyngearhrufvpvfsxdbahvx` int DEFAULT NULL,\n `dbjvygurpzzgpqwmtwuriakpcklntjjy` int DEFAULT NULL,\n `cuiguuwxxrvdquilltsocvykjvrniwwp` int DEFAULT NULL,\n `ghldklesgeqsqudcyyfjifwvtgpplial` int DEFAULT NULL,\n `tyapelmfuzbuzwdnskwgjtmtqtpgduco` int DEFAULT NULL,\n `whcpdjvlolqkvflzrjmkwtwfeilmfzey` int DEFAULT NULL,\n `hptqqqmgrrpqcuvfnwznikoqtjzicnpj` int DEFAULT NULL,\n `qlklxkcthncuswshhlxciolvvbadttaz` int DEFAULT NULL,\n `tgaqbzastjkyiqexbcdhlcqkhxqkgnqf` int DEFAULT NULL,\n `shuwmdstdsppecpfdcmtmgrfbsyfujaa` int DEFAULT NULL,\n `bbubwpsmiiibghufzannrxottaiccogm` int DEFAULT NULL,\n `blxbkbzdhviheasbxcibttjrcvevijtj` int DEFAULT NULL,\n `rsclhwcfmvdlchzyvfrupnstcaxfxdyr` int DEFAULT NULL,\n `hkgkvofytyxpujtmsllcjabngrccxsqg` int DEFAULT NULL,\n `iogbyxvekjdizsoixkmrwewdcqnutnyc` int DEFAULT NULL,\n `ytovhlcvjeimtogxquusmzwkbdzikuvx` int DEFAULT NULL,\n `xecehrlyuqgtlmwprsxlosmvfyzaywxc` int DEFAULT NULL,\n `tnrucbvqfamozpgptsgcfktcvwnwjwas` int DEFAULT NULL,\n `amgrnuvkyatqctgileoubodywgkhxziv` int DEFAULT NULL,\n `hbenojhlxvbdylvdmuoppfpkqkvqrzph` int DEFAULT NULL,\n `nvopxxpbifcsyvaeutvictriqrtsuggf` int DEFAULT NULL,\n `grpmstrlqbwqxaxqqfppatemkoshlbkr` int DEFAULT NULL,\n `hmylccozkasnvceztkslcbniqbzlqqps` int DEFAULT NULL,\n `glpfjqotrtajvziejflobzkonhojbxrf` int DEFAULT NULL,\n PRIMARY KEY (`jnliwlfnhmcdilsabwixdaruynfiysdn`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"gqrdivdtlwyumyqwlhrxuviktdbmfnew\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["jnliwlfnhmcdilsabwixdaruynfiysdn"],"columns":[{"name":"jnliwlfnhmcdilsabwixdaruynfiysdn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"sngyzgprfrinnvsoxbpoyawforkhebtc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wlfpquyaoohqtuljndlxqejdvlocanpf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjvtjvcpvcbepsrpnuxfyddvbvwzrels","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bghffcpvrgpeccpfobqvnoazrlwcsjlw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vlckbvdiazeeptpowbfhbadksfvpkhdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ptpxshrgvfnwfhpffawksdlbtlksvplf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkbpfnjcotcwzkpikvtwbgiupryhrirm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lkpdjegtvefhyuszvlasmnkthwmrsebd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gheqcqiwmlxzinmahmveajvrltborwnv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amrvnkijtjidljhieoeyjumumiegkpgp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"huzucapjdmhufobhplmoplwcbeawnvno","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hnvmugreiimwnbynqlsxbleppjvlopgq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixhazvsxqlbytxgcxkloyxrxptjpzmtz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edcwlwijfukftgirxdnmspzcesdypafw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"waimjzbdrvdijfrglwqfmimnakhdjlzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdqepxtoyprjscpuahpcvrbymzxjcvjc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ouhkmllitjtkfwcbneihaaunztrqpwqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oclnmaxjorujjcufilqosjgepilbuzor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmrtngbyotrefgkcwtjwsohvodkxwfyl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tgxrbpbyiyjhbhbbvmnqxqeomewjribh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hexuxgbnxwubunxemcdtpcbndwdxggxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrauewrvjhsxqdkgukdkqurescuxidgq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqknatitndouebazsvxtgphytotqmxsq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swzesqvdxgnngudgbgwzicjzjojdtdua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ltvmfbbfbtbhshgrrmxptzwpadtpvdqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lssxszcdtqrhsxhgappdgtbbytkengdp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkeaweqgirslpvdfvmgjeyvnnapirqxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tzwozklyvzsejekyumppqayperefjmai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"usrjevrlodhmqaxhlhmttcvtndlhcxvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhbadwcqwxxkxywinznyrhhihxwhuruf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlygzyvpdpqbavsnjcriqndtwuxefoxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bupmzmnsmgcnlhpmdkostxdlhytetman","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqhypehpllkxyqvllaqqhywpvqxbhljn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krnzsecvqxlilumhuknqagmzittjukop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvlbwjijtefhmppxghbdwnglvgjcpasf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yggraeevkhxlypskbqrbrrdeplghscjt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbieyebhzgocajgrhiapzptkbnxjqblc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzwtjfuppednuazncewmbyxviyvubgou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pveegafutoqisfxskkrkwzjitrojbryc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njfvndxivwebvfjxhrafpccjitivjdof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlcicspbxdixdwyqwdorwqbkumwpbejk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwngjczkazwwvxgkqkduebrahvhzxmtu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtwlgscbbcvrpcdmlaxerthhgbzaedcw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uurhmrncrjaduintnkfczevbdpfkqqeo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdcwuzqjvsvhdodooqlatdfuweqnxnep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfccdkrefpcjhmxstahzglkplvqyrjsn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yigwfigvwikloqopthqvbzxdzfopujzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxnpwbmsjpornxtwvjajkbzmiazwttqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdjfatjcsemzqewzuznplfhpvgxtdwcy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjnwnofyotepygxikhzdfiooizotbwys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qiualbbzjkfdtpcsdpysiqozrbxomghy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"deueauejomzqdvxuxgmyvubmbprpjmvb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfxqqrhdtqvnnzlwmkhmgattyhvyvtbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"daudoxerclvzhigictuvchmqcclacklw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjdmemjwqyjaxsviwfvfkawwqlxoqqdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmugpuajwuoeqsvjpzepoxumtabirurf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kawkvtuibyyvypabdyhjuranorzanerx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqnfdvlypdfggatmypvecbtsfnlkatde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"naifmurypatzrgtrqyqabbvvdfpkphrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrhoiiwsbkyovmnzdzalxebxhljnwhef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqgfguyqlshehsuxfxycunfjquxihiia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrbyoynoqcpvhjlcrmyhophohmccjqyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aydcaxwssjspuolkqnpuwevpyjjnvjeg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozqjlaaxwhhwwdwdqxroofdnoafinglb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"huxbupvfavfertqbnrbrkvdvdljommwr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"beicrzbrnjrndxgagyklcqfszkynoyaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfgatfagxeqegznhmhkmowoxxhrzlcbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vscnquetnlagzkugvscajtsopngihfjl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upzaayfmwmjyoqwykotowjjlphlqvltr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypjnjgtuybuokhxituaoemsvnilenwuk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdlqfzwuciiuichuovijxurtkecpbrfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpjusgqsglcvmnwftugintvgijwufvld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azdwfpknuolrznuugqdsclbfyxkitpkt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhckshfnhqdzgzvswdvboidqlrethbyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugxlrexbafhousfwvksblrxydktfjpif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tuzcsdxzzgyngearhrufvpvfsxdbahvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbjvygurpzzgpqwmtwuriakpcklntjjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cuiguuwxxrvdquilltsocvykjvrniwwp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghldklesgeqsqudcyyfjifwvtgpplial","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tyapelmfuzbuzwdnskwgjtmtqtpgduco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"whcpdjvlolqkvflzrjmkwtwfeilmfzey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hptqqqmgrrpqcuvfnwznikoqtjzicnpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qlklxkcthncuswshhlxciolvvbadttaz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tgaqbzastjkyiqexbcdhlcqkhxqkgnqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shuwmdstdsppecpfdcmtmgrfbsyfujaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbubwpsmiiibghufzannrxottaiccogm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blxbkbzdhviheasbxcibttjrcvevijtj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rsclhwcfmvdlchzyvfrupnstcaxfxdyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkgkvofytyxpujtmsllcjabngrccxsqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iogbyxvekjdizsoixkmrwewdcqnutnyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytovhlcvjeimtogxquusmzwkbdzikuvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xecehrlyuqgtlmwprsxlosmvfyzaywxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnrucbvqfamozpgptsgcfktcvwnwjwas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amgrnuvkyatqctgileoubodywgkhxziv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbenojhlxvbdylvdmuoppfpkqkvqrzph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvopxxpbifcsyvaeutvictriqrtsuggf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grpmstrlqbwqxaxqqfppatemkoshlbkr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmylccozkasnvceztkslcbniqbzlqqps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"glpfjqotrtajvziejflobzkonhojbxrf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668251,"databaseName":"models_schema","ddl":"CREATE TABLE `grjhfhonuudoavlukzcyzstzmfayaezp` (\n `omhsdtdmluprfnlxddytmjaaabiuklvt` int NOT NULL,\n `gpeosmbxbhnrvjvtksgppnluwzpfgzxs` int DEFAULT NULL,\n `yjokvikoshxizrqmcdbcdqxozsmfjzoy` int DEFAULT NULL,\n `ckagapyvmreofpwrkmotpaeonqbiotoq` int DEFAULT NULL,\n `qwklpszsrgxifydipnvfqniijidndxhk` int DEFAULT NULL,\n `dpuipajgnhfqskdpqbbvfbeznnxlzccu` int DEFAULT NULL,\n `bpxjplrgyfhracjkkmgdpogomhfapsfi` int DEFAULT NULL,\n `ydenpbnghxfuoedwqwqafxhsqcxxzkha` int DEFAULT NULL,\n `vbfawvipgnldgmwnjdhdiinwloahldjm` int DEFAULT NULL,\n `zeumdhyrktxfbrbsnjxrifwymgfrrcbd` int DEFAULT NULL,\n `krewpvnngjtirwpcrmcvfayxfmpducrt` int DEFAULT NULL,\n `rycgigvqwlipjjwanqyrgrblcvpomgzf` int DEFAULT NULL,\n `vrwjhofsorfnrbcsdrlpgwdyhcrzpepq` int DEFAULT NULL,\n `gnikyohquqvsvmickiioujwpfdiwmoeq` int DEFAULT NULL,\n `githqciibmihvnxqtgngqnypdmitfpem` int DEFAULT NULL,\n `xxwwyijgnujeeqkdjgxsrhkbgopithic` int DEFAULT NULL,\n `ludgithzekhrtaczjcofyyitsdiixpaz` int DEFAULT NULL,\n `cvmejfojwfybitqiqufpayahnzyfxtqn` int DEFAULT NULL,\n `cepegnljiemzcvmnykkgahntdplqaghe` int DEFAULT NULL,\n `llnhmybyzzinzeihynmpacvhqtqfpxsm` int DEFAULT NULL,\n `vfbrlmkwsdlbarfjahdplhfrwcqfamga` int DEFAULT NULL,\n `ltiiddpdupwwonuomkvppdetarkqipjf` int DEFAULT NULL,\n `mrsqzokpguhkgxnitswspgqgmzfeumrp` int DEFAULT NULL,\n `qpmhzxabcvzuyjbbessnhikclikvwfjn` int DEFAULT NULL,\n `uswqifnhjhawhjpzesqvtlomyjflprrg` int DEFAULT NULL,\n `zambxoldbeypypsnpiebyhyqspxixtyd` int DEFAULT NULL,\n `dtzgyujrcxybrhuoqiowfksgeftpfucz` int DEFAULT NULL,\n `mvwkquhuhrmteyqpmiuqusbgtdcslgqu` int DEFAULT NULL,\n `dplryulbhmrlpryxxccrmcksfmbvunxy` int DEFAULT NULL,\n `uciguqdqnssltsdjurbshzfhjqxvgxjl` int DEFAULT NULL,\n `wlmkecxywesuwyawtfchfcsjflixpdme` int DEFAULT NULL,\n `pshsriruaefizefzgqvnswiibvmjbgnx` int DEFAULT NULL,\n `ofznffimxeqgrklvvnaqccjtdvazdbar` int DEFAULT NULL,\n `ktbmfazjiidvfqhzcarbswodmiaehmlr` int DEFAULT NULL,\n `fuqzdkzgrknoiakxvwaepoyiccdqjcqb` int DEFAULT NULL,\n `ybavpokqbniywerefpawymroaspmergj` int DEFAULT NULL,\n `dkxzivxsjqntzwhorlpsthwyxveldqxm` int DEFAULT NULL,\n `scmlevjcrjkxrlyccisemncozveytaxk` int DEFAULT NULL,\n `mjjmbvrtwrcosknyfhvuymjutypnnicb` int DEFAULT NULL,\n `genxbhcdajbahskxontudqfdrarseqgy` int DEFAULT NULL,\n `cwjbpoqcbbefrktccoxylonvcrmkycel` int DEFAULT NULL,\n `dwslhcvcrjzydqonwzunnoharrmicohv` int DEFAULT NULL,\n `bnzttegqaintkighgtvlcayiprniadya` int DEFAULT NULL,\n `uapqvdbkcwuaxgjkdvkywhiuxckwfteb` int DEFAULT NULL,\n `tqeyoprmxxtliwwkxjsgycvvwrjejurg` int DEFAULT NULL,\n `dorfnpylwytyqctofmcmlxzfwvvektge` int DEFAULT NULL,\n `ixnwngaddcrllstasothfrdgcjigbgvg` int DEFAULT NULL,\n `ljeeashucnoizytqzbtzrqhiuuczjllw` int DEFAULT NULL,\n `vbfntjsdsgbydnhzormmvnywmejnloap` int DEFAULT NULL,\n `rdayhwdkdnsfzklyrawsneubvhpzcgqz` int DEFAULT NULL,\n `yjgjdtjkfaifaxvzlodxzwhkdfkmnbba` int DEFAULT NULL,\n `qnhlqsmwtujtfojvcbuiwybynqtychph` int DEFAULT NULL,\n `tzdmrhphysdmgxubfnzdxqubqkcolurz` int DEFAULT NULL,\n `spaboroxezichkvjeosprthzapypyopw` int DEFAULT NULL,\n `hfuwlwkzubrwqeibaiqhtrcukizndvly` int DEFAULT NULL,\n `iblbpmbcqwqxphdnvfkerwmrwxxhmrwh` int DEFAULT NULL,\n `eeyjxpwmylzwzmqnebrglesxkkuolddu` int DEFAULT NULL,\n `xiuwblxhorvzexjjoctspoyhuxoxrcvq` int DEFAULT NULL,\n `juutexaqjmqdelmtraaazxakktfvedng` int DEFAULT NULL,\n `dqpdjfvrubweguancdxyctykdyftrrzt` int DEFAULT NULL,\n `yazxjfmbqzxgcwprkudbdhxsvvkibfvy` int DEFAULT NULL,\n `qhpciyzuybtbsuvtjejbeqzlpgkzdtic` int DEFAULT NULL,\n `lzjwncewwlmkpynxxftoegdrhpcquckn` int DEFAULT NULL,\n `hwurdbnbbdeyloszbgblaswctzyonvsu` int DEFAULT NULL,\n `inlzanpklksuvtfifbxtojbyywlssulo` int DEFAULT NULL,\n `npvyrdeagilvxdfnxgffzfduhpxqyaoe` int DEFAULT NULL,\n `yssgxhmbiujstkekuiuhdcrxcdiaykma` int DEFAULT NULL,\n `rarmibxpcjrftjlowdxwxzxbcnybxkbe` int DEFAULT NULL,\n `qnvpemqwdhytdziyippwvsmjtjmfmmqs` int DEFAULT NULL,\n `nqbohuvupmluewpxwluiumvyxbepmhnr` int DEFAULT NULL,\n `laowpkconrflxtzjrstrkuynicghkvod` int DEFAULT NULL,\n `mnvotwtyweperdjamgrjcsfhaulegovl` int DEFAULT NULL,\n `vjqvtklpdquuylsfepugodnvztkdqeot` int DEFAULT NULL,\n `vmioikpftxktnzhowotxexkihnoukdod` int DEFAULT NULL,\n `fzxfvcriikboppmnquxcqjdkioiviqif` int DEFAULT NULL,\n `zxvifgvwhjbgjtpnevminlcjxyzyttbh` int DEFAULT NULL,\n `deadgpnmcphejkwpxfmuqtneeqavixzg` int DEFAULT NULL,\n `xejphoycaqzkkojamvjxgtimtojwmxbs` int DEFAULT NULL,\n `rngmydsgusicqfarbprfghoimmuqkaqr` int DEFAULT NULL,\n `tevynznusosndctwkgiyemuhpdylpqxi` int DEFAULT NULL,\n `zesplllsdjcuadzquhaxedzziccneuls` int DEFAULT NULL,\n `tsdqkhfmdosultivdaadgrsgmgvzbimk` int DEFAULT NULL,\n `qferzyagemajqlndzrmmmcppjxvdyuhr` int DEFAULT NULL,\n `plsqkmalbpyxwcysbwnwozlwgrokxyrz` int DEFAULT NULL,\n `xnorokiwpsdyavwrgakdtawssyuabpqd` int DEFAULT NULL,\n `ftnlgguovpzfgvaeokpqjkxidqxzpwgy` int DEFAULT NULL,\n `gbbxizpgdfcbipmbtlapckahwdqjncby` int DEFAULT NULL,\n `fbgglpvopzkzeewiyexqulspfptmfpvd` int DEFAULT NULL,\n `odkcoxomjriclaprtumlficreuovsnfs` int DEFAULT NULL,\n `ymuakxufhklvadxqshxmqnyohtvmvrld` int DEFAULT NULL,\n `ftjbdptprpawblmardzlpnflifxxaacy` int DEFAULT NULL,\n `jotmwcnypudtexuplfabqrgfjanccvpm` int DEFAULT NULL,\n `aqoykivvymycxosathbyfrzhkdqhtskj` int DEFAULT NULL,\n `xbxlteygzqdfvsnhvctajeqwijyrwmiw` int DEFAULT NULL,\n `oqhosjdnszbikphtatpuohxyxolbbytv` int DEFAULT NULL,\n `fbgmvhdopduqtilhvjishgogglymufdk` int DEFAULT NULL,\n `blpzlrmgqgmvjjrquqznzrkfiqrwndeu` int DEFAULT NULL,\n `mebcpcqsiasibrjtvqvqtinfabfyylvt` int DEFAULT NULL,\n `amzsrungrfmlyczbnfkdthrfnkjyzrzq` int DEFAULT NULL,\n `podpiauurpspokncpitgcgnqqurwhbin` int DEFAULT NULL,\n PRIMARY KEY (`omhsdtdmluprfnlxddytmjaaabiuklvt`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"grjhfhonuudoavlukzcyzstzmfayaezp\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["omhsdtdmluprfnlxddytmjaaabiuklvt"],"columns":[{"name":"omhsdtdmluprfnlxddytmjaaabiuklvt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"gpeosmbxbhnrvjvtksgppnluwzpfgzxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjokvikoshxizrqmcdbcdqxozsmfjzoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckagapyvmreofpwrkmotpaeonqbiotoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwklpszsrgxifydipnvfqniijidndxhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpuipajgnhfqskdpqbbvfbeznnxlzccu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bpxjplrgyfhracjkkmgdpogomhfapsfi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydenpbnghxfuoedwqwqafxhsqcxxzkha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbfawvipgnldgmwnjdhdiinwloahldjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zeumdhyrktxfbrbsnjxrifwymgfrrcbd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krewpvnngjtirwpcrmcvfayxfmpducrt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rycgigvqwlipjjwanqyrgrblcvpomgzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrwjhofsorfnrbcsdrlpgwdyhcrzpepq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnikyohquqvsvmickiioujwpfdiwmoeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"githqciibmihvnxqtgngqnypdmitfpem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxwwyijgnujeeqkdjgxsrhkbgopithic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ludgithzekhrtaczjcofyyitsdiixpaz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvmejfojwfybitqiqufpayahnzyfxtqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cepegnljiemzcvmnykkgahntdplqaghe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llnhmybyzzinzeihynmpacvhqtqfpxsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vfbrlmkwsdlbarfjahdplhfrwcqfamga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ltiiddpdupwwonuomkvppdetarkqipjf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrsqzokpguhkgxnitswspgqgmzfeumrp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qpmhzxabcvzuyjbbessnhikclikvwfjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uswqifnhjhawhjpzesqvtlomyjflprrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zambxoldbeypypsnpiebyhyqspxixtyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtzgyujrcxybrhuoqiowfksgeftpfucz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvwkquhuhrmteyqpmiuqusbgtdcslgqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dplryulbhmrlpryxxccrmcksfmbvunxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uciguqdqnssltsdjurbshzfhjqxvgxjl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wlmkecxywesuwyawtfchfcsjflixpdme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pshsriruaefizefzgqvnswiibvmjbgnx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofznffimxeqgrklvvnaqccjtdvazdbar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ktbmfazjiidvfqhzcarbswodmiaehmlr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fuqzdkzgrknoiakxvwaepoyiccdqjcqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybavpokqbniywerefpawymroaspmergj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkxzivxsjqntzwhorlpsthwyxveldqxm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scmlevjcrjkxrlyccisemncozveytaxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjjmbvrtwrcosknyfhvuymjutypnnicb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"genxbhcdajbahskxontudqfdrarseqgy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwjbpoqcbbefrktccoxylonvcrmkycel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwslhcvcrjzydqonwzunnoharrmicohv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnzttegqaintkighgtvlcayiprniadya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uapqvdbkcwuaxgjkdvkywhiuxckwfteb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqeyoprmxxtliwwkxjsgycvvwrjejurg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dorfnpylwytyqctofmcmlxzfwvvektge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixnwngaddcrllstasothfrdgcjigbgvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljeeashucnoizytqzbtzrqhiuuczjllw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbfntjsdsgbydnhzormmvnywmejnloap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdayhwdkdnsfzklyrawsneubvhpzcgqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjgjdtjkfaifaxvzlodxzwhkdfkmnbba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qnhlqsmwtujtfojvcbuiwybynqtychph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tzdmrhphysdmgxubfnzdxqubqkcolurz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"spaboroxezichkvjeosprthzapypyopw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfuwlwkzubrwqeibaiqhtrcukizndvly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iblbpmbcqwqxphdnvfkerwmrwxxhmrwh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eeyjxpwmylzwzmqnebrglesxkkuolddu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xiuwblxhorvzexjjoctspoyhuxoxrcvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"juutexaqjmqdelmtraaazxakktfvedng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqpdjfvrubweguancdxyctykdyftrrzt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yazxjfmbqzxgcwprkudbdhxsvvkibfvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhpciyzuybtbsuvtjejbeqzlpgkzdtic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzjwncewwlmkpynxxftoegdrhpcquckn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwurdbnbbdeyloszbgblaswctzyonvsu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"inlzanpklksuvtfifbxtojbyywlssulo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npvyrdeagilvxdfnxgffzfduhpxqyaoe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yssgxhmbiujstkekuiuhdcrxcdiaykma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rarmibxpcjrftjlowdxwxzxbcnybxkbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qnvpemqwdhytdziyippwvsmjtjmfmmqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqbohuvupmluewpxwluiumvyxbepmhnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"laowpkconrflxtzjrstrkuynicghkvod","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnvotwtyweperdjamgrjcsfhaulegovl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjqvtklpdquuylsfepugodnvztkdqeot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmioikpftxktnzhowotxexkihnoukdod","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzxfvcriikboppmnquxcqjdkioiviqif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxvifgvwhjbgjtpnevminlcjxyzyttbh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"deadgpnmcphejkwpxfmuqtneeqavixzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xejphoycaqzkkojamvjxgtimtojwmxbs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rngmydsgusicqfarbprfghoimmuqkaqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tevynznusosndctwkgiyemuhpdylpqxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zesplllsdjcuadzquhaxedzziccneuls","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tsdqkhfmdosultivdaadgrsgmgvzbimk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qferzyagemajqlndzrmmmcppjxvdyuhr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"plsqkmalbpyxwcysbwnwozlwgrokxyrz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnorokiwpsdyavwrgakdtawssyuabpqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftnlgguovpzfgvaeokpqjkxidqxzpwgy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gbbxizpgdfcbipmbtlapckahwdqjncby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbgglpvopzkzeewiyexqulspfptmfpvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"odkcoxomjriclaprtumlficreuovsnfs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ymuakxufhklvadxqshxmqnyohtvmvrld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftjbdptprpawblmardzlpnflifxxaacy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jotmwcnypudtexuplfabqrgfjanccvpm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqoykivvymycxosathbyfrzhkdqhtskj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbxlteygzqdfvsnhvctajeqwijyrwmiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqhosjdnszbikphtatpuohxyxolbbytv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbgmvhdopduqtilhvjishgogglymufdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blpzlrmgqgmvjjrquqznzrkfiqrwndeu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mebcpcqsiasibrjtvqvqtinfabfyylvt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amzsrungrfmlyczbnfkdthrfnkjyzrzq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"podpiauurpspokncpitgcgnqqurwhbin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668282,"databaseName":"models_schema","ddl":"CREATE TABLE `grlnyqotcwelxxfihmfytiykiyggwidh` (\n `xxwxhhlvzpjcqfsvuneeozpafrxrskar` int NOT NULL,\n `mkjrgjdmcunaqdvpfrfrleeuqwtghvxm` int DEFAULT NULL,\n `vxxmerwewqedugfvryafdmdiaojxomme` int DEFAULT NULL,\n `pavgbcnsfleotnrtnysoazethvucpwoc` int DEFAULT NULL,\n `jiyragskbfrysehvqtogigazjlmstxwt` int DEFAULT NULL,\n `pjeoawujytpnazsearywgqseontkgkyg` int DEFAULT NULL,\n `occscknjtzhjwguepxatzngkamgcmwaw` int DEFAULT NULL,\n `uprdvmfcbsyzldvtimunvhqsolpicmgb` int DEFAULT NULL,\n `jvqtgneziscvaqaltsutubdnioxqvlhf` int DEFAULT NULL,\n `eotqlftczhmyghvletpbdqicmptsjiot` int DEFAULT NULL,\n `vwldhnvnaclgdmmcakqmnnulxlyhhcvc` int DEFAULT NULL,\n `kfuwzlzbzczxazzjdtcgdelykfpilyif` int DEFAULT NULL,\n `fijwqfqnqevikxekmmibbkwrbumksdid` int DEFAULT NULL,\n `klecumoiwiwjzeuypwkdufravwroxjka` int DEFAULT NULL,\n `fvrpqlsozotbpnyjmjrqwqqmdkzzatag` int DEFAULT NULL,\n `wbiolmcwxpjqfshmkdoqtnbuvoskfubr` int DEFAULT NULL,\n `rrcxohxgcuifozqcfirwfmwihmcufwzj` int DEFAULT NULL,\n `qzxdbisrqpmesnpefutelmizuwduxnfh` int DEFAULT NULL,\n `zmfjaojhlyrrxkmurcwuqbhqaewyejds` int DEFAULT NULL,\n `fmuytzwmmmgcdlvhvrcoymmflmugcpss` int DEFAULT NULL,\n `zdiajtcxilojnmzqsdnkgcobqalomeae` int DEFAULT NULL,\n `fpraumrpqrvgqpzbjrjkebdqdculpxcr` int DEFAULT NULL,\n `upzotgjfvcvdwbukyrbsxjpamyrhrjxj` int DEFAULT NULL,\n `tkvijkkeffxprubpvgbjfhocdidhwluq` int DEFAULT NULL,\n `efuzlkfcxlegmsilioexoxmauwqiojxb` int DEFAULT NULL,\n `rkkgoudaepgbujfjfffrjegzlsefttii` int DEFAULT NULL,\n `qamaxymzgimnunoflqxxjugetzvgwsbv` int DEFAULT NULL,\n `fbukffytdifefjjtpovkqhdnyzeuqpdj` int DEFAULT NULL,\n `yokwtxcituovtggmzaagxnybpdevqupk` int DEFAULT NULL,\n `xstnvcrpyzyovtfoctupjufkclrtores` int DEFAULT NULL,\n `pwdzljcnhcrrrohvopvdlfhrwcimwsmh` int DEFAULT NULL,\n `cmxkknpxsmhgpbaleuxzaiqmiidxuien` int DEFAULT NULL,\n `rfaazcuvmxzxncqflpxdxnjooczsgpcv` int DEFAULT NULL,\n `wekwzzdisvmxdewiisxnedfnseidtzte` int DEFAULT NULL,\n `bilkbfbpeiqhnweshzioictodrxihljx` int DEFAULT NULL,\n `ypupvvnonboqoyvszsgjewwbqicezbom` int DEFAULT NULL,\n `fcvpegdwemepiqorwjphiqaufvqnuqxq` int DEFAULT NULL,\n `lovgttmxuvftblfuwayfvvaikchmoenb` int DEFAULT NULL,\n `ufikxthylfzrkuxnnroljxgaazubqsfv` int DEFAULT NULL,\n `kvuwrwwbnloycyvhkkduzwltqutjdxcy` int DEFAULT NULL,\n `jwstxalsuxblhmbiyzbpqzhbqgptrvdc` int DEFAULT NULL,\n `wcsgwxftjygcjcidfynxyncwdndgtoqb` int DEFAULT NULL,\n `eajkhbkwmaziskcsrcalrsivqxwqwlsx` int DEFAULT NULL,\n `ymiodrvvygarftwdquemxxegogoibxwm` int DEFAULT NULL,\n `juoeezyazbmcvhgtbncxdhvdlnrblvck` int DEFAULT NULL,\n `rddegxspyaczzlpxzwcdfzvxyxgstkuj` int DEFAULT NULL,\n `wvejjrjllkeqewnmourpsifebmzkhnuh` int DEFAULT NULL,\n `hgupxxrmlhmkacyhpugycbndtcqhyyby` int DEFAULT NULL,\n `snbehkseyewvkzhdvboadbztdpkocxmk` int DEFAULT NULL,\n `rssyfvlncmqsbdamfrfunjsdkpowxjie` int DEFAULT NULL,\n `mofjqbuhxulvfvkosxstdfvzvihrvwoi` int DEFAULT NULL,\n `objnrvffrrhxoihfguhkvirkeaoylamu` int DEFAULT NULL,\n `lzduhntjtbleupwozfhcglezdfmtfuhx` int DEFAULT NULL,\n `wxcmtowtfmikfgqsvmznqrkemimehjzn` int DEFAULT NULL,\n `jynhplchiwzcsshgryosvnrzkjtuwzkq` int DEFAULT NULL,\n `fvvynhtxcbxrhxvnpbqiuobkofysqrsp` int DEFAULT NULL,\n `lovnnzvksktkzgqrjhswfjlepraihkdm` int DEFAULT NULL,\n `oaelqotjqmwacaulnmovlymjrfjcfahx` int DEFAULT NULL,\n `xnezdspoeymrwmmnsaageyqffamxtrhz` int DEFAULT NULL,\n `blicbjvypablawduwslauidcukvbvaxi` int DEFAULT NULL,\n `wgslmsjlouaugzmphrdvwuowhgfqfzzj` int DEFAULT NULL,\n `rhtjnxrtfsbmoxafpnwzmdoedkadwkdg` int DEFAULT NULL,\n `vlxyayowighacqipaqqdvsbwzkpmyudq` int DEFAULT NULL,\n `uwnzwqjcliikljbctofougehacxegpjb` int DEFAULT NULL,\n `hgiwlzpjlpkynupvgmkgvqvkslhexmql` int DEFAULT NULL,\n `jtchlvfogyrrzhionwkuupycvprfgduk` int DEFAULT NULL,\n `nbgjndzzuuhomqtkwiytmfswpqootqow` int DEFAULT NULL,\n `vpghsijifjopumhndlgcqocaytxwlxfc` int DEFAULT NULL,\n `zfczxguaqmmpndqzpwqwcjkhyvvspvrp` int DEFAULT NULL,\n `bmafdhtyvwazsgxfgxpjegohargzbjme` int DEFAULT NULL,\n `mdrfodmysfyusbejsdelclyklvmwljqs` int DEFAULT NULL,\n `yeatdcoopurtwulmhyqmdzsrbkpqpyzv` int DEFAULT NULL,\n `rpjxqcphupbuhwnxygahcbtazkyykxrd` int DEFAULT NULL,\n `qmlvleinxfzfwglmdkspmygemffldqgr` int DEFAULT NULL,\n `eoffdjqudfnvfvsjcxdwmlrakbihweav` int DEFAULT NULL,\n `pyxenbdsyjwuyqidkwgfgvhtbjpmnzvc` int DEFAULT NULL,\n `ubrveovmqqdqecfyyeutptupdvanmpbh` int DEFAULT NULL,\n `zddtczsgoptmkjkcayxrxitlkyzhmbdw` int DEFAULT NULL,\n `noiaasuwigijkcrsbtobxqwegudivrzu` int DEFAULT NULL,\n `afzgcvtynpxjrglgnoeavspickygzxbt` int DEFAULT NULL,\n `rlvltknsjugeranuajxdsikqicivvxwb` int DEFAULT NULL,\n `dqmaefngeugnicefqojgunpoqfaakmxp` int DEFAULT NULL,\n `eyzgnvnufxithfmsoibijpuhvpmzzxau` int DEFAULT NULL,\n `ehzsseiwxhsurestxxkyjyuzgenvpkxd` int DEFAULT NULL,\n `oftqwjwixbcgovkplcuyfukkyzsmftpp` int DEFAULT NULL,\n `pmxofpzjvkztwbyqdrvefmuyvtdvfrtx` int DEFAULT NULL,\n `eqnzyswzpeoddajksvjooxpdnbbnatbh` int DEFAULT NULL,\n `fzgqkxlshkycxbtpvcphshxswdutvdoi` int DEFAULT NULL,\n `dnclaqqugllxukwfzuuafpxeqpoyskpa` int DEFAULT NULL,\n `vctudvpzslnuzfumelripofgcbxgpheu` int DEFAULT NULL,\n `tdlhnivesficnikefnpjhvxsbngkbdol` int DEFAULT NULL,\n `rmellspxahimdrzobauzeegsuisubntl` int DEFAULT NULL,\n `fohajjsjwbxzgbscdwdjgxkckcjpxvrh` int DEFAULT NULL,\n `meloukikbqjfurdktlljknpnvznvwvrp` int DEFAULT NULL,\n `wqpvyhffsstxvlgbstcjhylkqywgffwy` int DEFAULT NULL,\n `ebxpjmirhnhpvnleggxiepybrxicrebz` int DEFAULT NULL,\n `arvumjirsmjxaeaadvsyjnnfoybawtvp` int DEFAULT NULL,\n `mpetijheutlwwtqnkrsowaejdwokajpz` int DEFAULT NULL,\n `clequxqsrdmxsxnokpksvmyvxavmykat` int DEFAULT NULL,\n `xmbtcxqmexqjzkkhblssekjiyfyltity` int DEFAULT NULL,\n PRIMARY KEY (`xxwxhhlvzpjcqfsvuneeozpafrxrskar`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"grlnyqotcwelxxfihmfytiykiyggwidh\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["xxwxhhlvzpjcqfsvuneeozpafrxrskar"],"columns":[{"name":"xxwxhhlvzpjcqfsvuneeozpafrxrskar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"mkjrgjdmcunaqdvpfrfrleeuqwtghvxm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxxmerwewqedugfvryafdmdiaojxomme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pavgbcnsfleotnrtnysoazethvucpwoc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jiyragskbfrysehvqtogigazjlmstxwt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjeoawujytpnazsearywgqseontkgkyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"occscknjtzhjwguepxatzngkamgcmwaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uprdvmfcbsyzldvtimunvhqsolpicmgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvqtgneziscvaqaltsutubdnioxqvlhf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eotqlftczhmyghvletpbdqicmptsjiot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vwldhnvnaclgdmmcakqmnnulxlyhhcvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfuwzlzbzczxazzjdtcgdelykfpilyif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fijwqfqnqevikxekmmibbkwrbumksdid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"klecumoiwiwjzeuypwkdufravwroxjka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fvrpqlsozotbpnyjmjrqwqqmdkzzatag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbiolmcwxpjqfshmkdoqtnbuvoskfubr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrcxohxgcuifozqcfirwfmwihmcufwzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzxdbisrqpmesnpefutelmizuwduxnfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmfjaojhlyrrxkmurcwuqbhqaewyejds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmuytzwmmmgcdlvhvrcoymmflmugcpss","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdiajtcxilojnmzqsdnkgcobqalomeae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpraumrpqrvgqpzbjrjkebdqdculpxcr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upzotgjfvcvdwbukyrbsxjpamyrhrjxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkvijkkeffxprubpvgbjfhocdidhwluq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"efuzlkfcxlegmsilioexoxmauwqiojxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkkgoudaepgbujfjfffrjegzlsefttii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qamaxymzgimnunoflqxxjugetzvgwsbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbukffytdifefjjtpovkqhdnyzeuqpdj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yokwtxcituovtggmzaagxnybpdevqupk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xstnvcrpyzyovtfoctupjufkclrtores","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwdzljcnhcrrrohvopvdlfhrwcimwsmh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmxkknpxsmhgpbaleuxzaiqmiidxuien","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfaazcuvmxzxncqflpxdxnjooczsgpcv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wekwzzdisvmxdewiisxnedfnseidtzte","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bilkbfbpeiqhnweshzioictodrxihljx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypupvvnonboqoyvszsgjewwbqicezbom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcvpegdwemepiqorwjphiqaufvqnuqxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lovgttmxuvftblfuwayfvvaikchmoenb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufikxthylfzrkuxnnroljxgaazubqsfv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvuwrwwbnloycyvhkkduzwltqutjdxcy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwstxalsuxblhmbiyzbpqzhbqgptrvdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcsgwxftjygcjcidfynxyncwdndgtoqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eajkhbkwmaziskcsrcalrsivqxwqwlsx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ymiodrvvygarftwdquemxxegogoibxwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"juoeezyazbmcvhgtbncxdhvdlnrblvck","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rddegxspyaczzlpxzwcdfzvxyxgstkuj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvejjrjllkeqewnmourpsifebmzkhnuh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hgupxxrmlhmkacyhpugycbndtcqhyyby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"snbehkseyewvkzhdvboadbztdpkocxmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rssyfvlncmqsbdamfrfunjsdkpowxjie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mofjqbuhxulvfvkosxstdfvzvihrvwoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"objnrvffrrhxoihfguhkvirkeaoylamu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzduhntjtbleupwozfhcglezdfmtfuhx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxcmtowtfmikfgqsvmznqrkemimehjzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jynhplchiwzcsshgryosvnrzkjtuwzkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fvvynhtxcbxrhxvnpbqiuobkofysqrsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lovnnzvksktkzgqrjhswfjlepraihkdm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oaelqotjqmwacaulnmovlymjrfjcfahx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnezdspoeymrwmmnsaageyqffamxtrhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blicbjvypablawduwslauidcukvbvaxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgslmsjlouaugzmphrdvwuowhgfqfzzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhtjnxrtfsbmoxafpnwzmdoedkadwkdg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vlxyayowighacqipaqqdvsbwzkpmyudq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwnzwqjcliikljbctofougehacxegpjb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hgiwlzpjlpkynupvgmkgvqvkslhexmql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jtchlvfogyrrzhionwkuupycvprfgduk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbgjndzzuuhomqtkwiytmfswpqootqow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpghsijifjopumhndlgcqocaytxwlxfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zfczxguaqmmpndqzpwqwcjkhyvvspvrp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bmafdhtyvwazsgxfgxpjegohargzbjme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdrfodmysfyusbejsdelclyklvmwljqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yeatdcoopurtwulmhyqmdzsrbkpqpyzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpjxqcphupbuhwnxygahcbtazkyykxrd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmlvleinxfzfwglmdkspmygemffldqgr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eoffdjqudfnvfvsjcxdwmlrakbihweav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pyxenbdsyjwuyqidkwgfgvhtbjpmnzvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubrveovmqqdqecfyyeutptupdvanmpbh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zddtczsgoptmkjkcayxrxitlkyzhmbdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"noiaasuwigijkcrsbtobxqwegudivrzu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afzgcvtynpxjrglgnoeavspickygzxbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlvltknsjugeranuajxdsikqicivvxwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqmaefngeugnicefqojgunpoqfaakmxp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eyzgnvnufxithfmsoibijpuhvpmzzxau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehzsseiwxhsurestxxkyjyuzgenvpkxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oftqwjwixbcgovkplcuyfukkyzsmftpp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmxofpzjvkztwbyqdrvefmuyvtdvfrtx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqnzyswzpeoddajksvjooxpdnbbnatbh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzgqkxlshkycxbtpvcphshxswdutvdoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dnclaqqugllxukwfzuuafpxeqpoyskpa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vctudvpzslnuzfumelripofgcbxgpheu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdlhnivesficnikefnpjhvxsbngkbdol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rmellspxahimdrzobauzeegsuisubntl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fohajjsjwbxzgbscdwdjgxkckcjpxvrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"meloukikbqjfurdktlljknpnvznvwvrp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqpvyhffsstxvlgbstcjhylkqywgffwy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebxpjmirhnhpvnleggxiepybrxicrebz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"arvumjirsmjxaeaadvsyjnnfoybawtvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpetijheutlwwtqnkrsowaejdwokajpz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clequxqsrdmxsxnokpksvmyvxavmykat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmbtcxqmexqjzkkhblssekjiyfyltity","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668313,"databaseName":"models_schema","ddl":"CREATE TABLE `grmuxqmusubjsziwgvwmpopfcnqtgvps` (\n `bxptisjslcdommqdzebvfzelygbcpjte` int NOT NULL,\n `xlwgammxbuzfpwrlqnxppauwhmmsomrq` int DEFAULT NULL,\n `alnczgvjcomdknmuclpbfvryvacmxbiv` int DEFAULT NULL,\n `ujykcolzonmurizfqvdlqxacbmzlpwcq` int DEFAULT NULL,\n `gjtaakajczopxpzzcmwpwqmdkhfftnrl` int DEFAULT NULL,\n `girbmxmlobizoexezobupipxqquapibg` int DEFAULT NULL,\n `bqzydesqwkknyhvtppruvoxsjdrjlmte` int DEFAULT NULL,\n `eemxgfejaqokdamxzylifylelvturdqh` int DEFAULT NULL,\n `cnjvpyxnrdemlawhoimxanyispvokczb` int DEFAULT NULL,\n `ompouyoueozqcwskplyxezdmhwpznwwi` int DEFAULT NULL,\n `gxhoeobowtxutqkzmwhwkaescveqrmqz` int DEFAULT NULL,\n `lthhvhyjvdjszwlpsldoutfafaolbbaj` int DEFAULT NULL,\n `uxxfvjkwxgoqfipibwikaiirtxplmhrf` int DEFAULT NULL,\n `lnjfqmcfvstitpysrbrnxgdpzbtapfcg` int DEFAULT NULL,\n `vukaqnrrtqadtjmcoxbkxkzfcnbadzit` int DEFAULT NULL,\n `epreqlmtfybrevaosgdgpibpgdvnmdeq` int DEFAULT NULL,\n `sqlfqlgrcnoplonhvswjtpotylpkmpsc` int DEFAULT NULL,\n `zrfmarhxapjknaxqxeqvrsnqldbgjjcp` int DEFAULT NULL,\n `zztfdqwkasvkpyjcopdkqiwdskmqbwae` int DEFAULT NULL,\n `ufbzzdfghurdmtfllayphwzjxqnmcfsc` int DEFAULT NULL,\n `hubymnoznavqmslostsflkrwcbcofrbz` int DEFAULT NULL,\n `gwbuackqmmzoptwoynidwkhhpikkpzmd` int DEFAULT NULL,\n `amglwgfpuyxlsakbnqchoirtpagdaxnb` int DEFAULT NULL,\n `kumbtxmhgfgmjkcjdwgyhfljahcgbrdq` int DEFAULT NULL,\n `wixdfygmokmkspncrfcbumfnapdqvtyb` int DEFAULT NULL,\n `bnqosjgvracayzovvrqbtddngdarlumy` int DEFAULT NULL,\n `hlrqmanwknubpmiyhzwehchwzlfycqko` int DEFAULT NULL,\n `mnpcfrhoihrawkcwhsmtpueblyqwlfuu` int DEFAULT NULL,\n `lujvpjytvnasixhynggfcxqkizbvlqww` int DEFAULT NULL,\n `iqpqkaogreedowphdjxlwnpazvjhhagv` int DEFAULT NULL,\n `sdkiqoxxhgjikauiabacirrdipevsbcr` int DEFAULT NULL,\n `docmaflxzgwwvkdjxigynfcnsxxcuttq` int DEFAULT NULL,\n `xnpxbbzadbydeuvcsnlhvxulwmgvtlav` int DEFAULT NULL,\n `tafalfustcsubhvlnelshixissmagdwf` int DEFAULT NULL,\n `jflvxqhwuhodqcuhdycgmcbishojmhtt` int DEFAULT NULL,\n `rhafxnwllapwdsemkraqaxhzdgrfrrdd` int DEFAULT NULL,\n `jdlzkfjdfcislsyfkgbkpmwubleozcbu` int DEFAULT NULL,\n `phvgmofnfjnsislipnoetdnvwoubbama` int DEFAULT NULL,\n `iqtkqlabcknohtzehlwmsgomkepzpmde` int DEFAULT NULL,\n `bftcjvnrciqzcemuinhdmmtdgrnzpthg` int DEFAULT NULL,\n `xpmltbkqzmibwodufpdylioxfmpyiseu` int DEFAULT NULL,\n `nzjgdftcgthrldguffannglzovaglzpi` int DEFAULT NULL,\n `zxfveltgoykdjanslsocvzfmudkaiygh` int DEFAULT NULL,\n `uvakehvzodkwihsefviepsrocckffhqz` int DEFAULT NULL,\n `qycpoarhwagednsutasqyxxkvivapqwn` int DEFAULT NULL,\n `lmeavxpddgvjhuvtzpambajjebaqgqoe` int DEFAULT NULL,\n `lvazsfxbnsipvupyzzvawftspwbzbowg` int DEFAULT NULL,\n `lnfefdaiwnzgpidmgwmeufzorajxllma` int DEFAULT NULL,\n `cpwzewzhfsumglxixqefhsahrbjeiaiq` int DEFAULT NULL,\n `hhdoomiftxsmoloycuqcrizpqewemefs` int DEFAULT NULL,\n `sdpfjjjqentmymvujfxfplrwjalwpple` int DEFAULT NULL,\n `gemltltujpklvabmidjxoqmxbksgpvtt` int DEFAULT NULL,\n `itblgpbmmjlbxfbdjjnrgwzpfnahuzsl` int DEFAULT NULL,\n `mdsaedruryufbmnsawtteowpilnoidjp` int DEFAULT NULL,\n `hsvaaeviybxlbqpwojkreagzqfbapyhn` int DEFAULT NULL,\n `uouvmjupzaihzfceycouwungjxqzobbs` int DEFAULT NULL,\n `sfdfgsxmvdsjyupjgszhrfvvufqswbhc` int DEFAULT NULL,\n `lggtrifvijkbeqgwwfefeuqxlqqtqquf` int DEFAULT NULL,\n `sltjsbwqgfujcgiyvdrmevhixvmkdhhr` int DEFAULT NULL,\n `xthoqhlgslkbbeylduoeebqfwjzqdqaw` int DEFAULT NULL,\n `lytlippaydletizcpeflvilitedytmzv` int DEFAULT NULL,\n `yeljpcmhxymwaathfqykgqatfhudkmgc` int DEFAULT NULL,\n `lzrjckinckgkvnwbigtmthqdppgrzvnm` int DEFAULT NULL,\n `rothkyfhncloyppvoposzlebllpeaiaq` int DEFAULT NULL,\n `noxqjxnszgtprponycpnspxvxoahwaoy` int DEFAULT NULL,\n `sebwkcukjquzyhnelqcaczlnxoyfhuda` int DEFAULT NULL,\n `lkvegqytpxzvdlzeurzdgqpenaoqctbf` int DEFAULT NULL,\n `bkdbhwrpkwdntqccvluqsfepyssotfvb` int DEFAULT NULL,\n `vtjojgjbnmconwagyztonelwakvtblkk` int DEFAULT NULL,\n `yobfwcwufipddiogqyzaabfortenwrfk` int DEFAULT NULL,\n `bebshvjehqjzlgpugedverxqtgacbyob` int DEFAULT NULL,\n `cwpflxbwnykuzovirbytlmjvypdkbpks` int DEFAULT NULL,\n `zkhzehsqcuhkucnegwsckkdntsdfopfc` int DEFAULT NULL,\n `hlxbuuyhpuctrqwforwiyduiabkckmmi` int DEFAULT NULL,\n `fbtvugedqvtbtlwucxsgexiopylcsptv` int DEFAULT NULL,\n `xxpjpdpgqligfefxtqtkpjisrovhgmdg` int DEFAULT NULL,\n `ggngcefztejwabajooanimnirtrqixes` int DEFAULT NULL,\n `eaqzzlfkgswntjyzyqhyudphpijnfxqu` int DEFAULT NULL,\n `elavnoimqzrmrdikanmqwetjpbhnumhk` int DEFAULT NULL,\n `vznudohigyfaumriqaooedxvtjttfguw` int DEFAULT NULL,\n `bfpepkfasvlwcdnlsmppwcrjfuwcvpkc` int DEFAULT NULL,\n `bfzbqmxpgqsozalapvhuevsurzyovxjb` int DEFAULT NULL,\n `jjvmugzlswevsetwdnzxtrzxtcunzbfa` int DEFAULT NULL,\n `znqwfnbprrmwlnntvcttmhfmmrlkizgl` int DEFAULT NULL,\n `bjzelccpochtupsxgysetotcqjjcmhmz` int DEFAULT NULL,\n `eogeqwrvhasdjbaxzaeawmsecmjrmqza` int DEFAULT NULL,\n `ovfczxlvvxrbicobbikmcdgssdbjwvaw` int DEFAULT NULL,\n `evnuqjdkqflemvhgdpwrnnhrujmzsawh` int DEFAULT NULL,\n `xtnrwefmdhhazrmtaijgnuremvnuhjzc` int DEFAULT NULL,\n `llbkrshoikhqeumwpimoylcrcpafaiwi` int DEFAULT NULL,\n `sqocepfturohygfugbwyjshxuhiqmstd` int DEFAULT NULL,\n `ctgugpsbmijnmviholhkugjbzpvukeqt` int DEFAULT NULL,\n `rjngaoujkphuaeepflozrazqaouyoylh` int DEFAULT NULL,\n `kmlgygebfgpefcytgdrqepwehjftobxc` int DEFAULT NULL,\n `krektrebaxhksoanaenseegsmphjjnkm` int DEFAULT NULL,\n `qowiecfesyjfioqcwuwfhwiwobsokjzx` int DEFAULT NULL,\n `hnjkeedwruorflkgdhpyhljfhhsupyjn` int DEFAULT NULL,\n `utqrfavntqjdafpwbpycxnaernzgjoph` int DEFAULT NULL,\n `eufphpdyiakhodgrndvlickwkkplbilq` int DEFAULT NULL,\n `nrexdgralhvfsildepopcvrifdaluerx` int DEFAULT NULL,\n PRIMARY KEY (`bxptisjslcdommqdzebvfzelygbcpjte`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"grmuxqmusubjsziwgvwmpopfcnqtgvps\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["bxptisjslcdommqdzebvfzelygbcpjte"],"columns":[{"name":"bxptisjslcdommqdzebvfzelygbcpjte","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"xlwgammxbuzfpwrlqnxppauwhmmsomrq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"alnczgvjcomdknmuclpbfvryvacmxbiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujykcolzonmurizfqvdlqxacbmzlpwcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjtaakajczopxpzzcmwpwqmdkhfftnrl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"girbmxmlobizoexezobupipxqquapibg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqzydesqwkknyhvtppruvoxsjdrjlmte","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eemxgfejaqokdamxzylifylelvturdqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnjvpyxnrdemlawhoimxanyispvokczb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ompouyoueozqcwskplyxezdmhwpznwwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxhoeobowtxutqkzmwhwkaescveqrmqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lthhvhyjvdjszwlpsldoutfafaolbbaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxxfvjkwxgoqfipibwikaiirtxplmhrf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lnjfqmcfvstitpysrbrnxgdpzbtapfcg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vukaqnrrtqadtjmcoxbkxkzfcnbadzit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"epreqlmtfybrevaosgdgpibpgdvnmdeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sqlfqlgrcnoplonhvswjtpotylpkmpsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zrfmarhxapjknaxqxeqvrsnqldbgjjcp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zztfdqwkasvkpyjcopdkqiwdskmqbwae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufbzzdfghurdmtfllayphwzjxqnmcfsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hubymnoznavqmslostsflkrwcbcofrbz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gwbuackqmmzoptwoynidwkhhpikkpzmd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amglwgfpuyxlsakbnqchoirtpagdaxnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kumbtxmhgfgmjkcjdwgyhfljahcgbrdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wixdfygmokmkspncrfcbumfnapdqvtyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnqosjgvracayzovvrqbtddngdarlumy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlrqmanwknubpmiyhzwehchwzlfycqko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnpcfrhoihrawkcwhsmtpueblyqwlfuu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lujvpjytvnasixhynggfcxqkizbvlqww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqpqkaogreedowphdjxlwnpazvjhhagv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdkiqoxxhgjikauiabacirrdipevsbcr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"docmaflxzgwwvkdjxigynfcnsxxcuttq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnpxbbzadbydeuvcsnlhvxulwmgvtlav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tafalfustcsubhvlnelshixissmagdwf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jflvxqhwuhodqcuhdycgmcbishojmhtt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhafxnwllapwdsemkraqaxhzdgrfrrdd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdlzkfjdfcislsyfkgbkpmwubleozcbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phvgmofnfjnsislipnoetdnvwoubbama","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqtkqlabcknohtzehlwmsgomkepzpmde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bftcjvnrciqzcemuinhdmmtdgrnzpthg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpmltbkqzmibwodufpdylioxfmpyiseu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzjgdftcgthrldguffannglzovaglzpi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxfveltgoykdjanslsocvzfmudkaiygh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvakehvzodkwihsefviepsrocckffhqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qycpoarhwagednsutasqyxxkvivapqwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmeavxpddgvjhuvtzpambajjebaqgqoe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvazsfxbnsipvupyzzvawftspwbzbowg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lnfefdaiwnzgpidmgwmeufzorajxllma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cpwzewzhfsumglxixqefhsahrbjeiaiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhdoomiftxsmoloycuqcrizpqewemefs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdpfjjjqentmymvujfxfplrwjalwpple","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gemltltujpklvabmidjxoqmxbksgpvtt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itblgpbmmjlbxfbdjjnrgwzpfnahuzsl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdsaedruryufbmnsawtteowpilnoidjp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hsvaaeviybxlbqpwojkreagzqfbapyhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uouvmjupzaihzfceycouwungjxqzobbs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfdfgsxmvdsjyupjgszhrfvvufqswbhc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lggtrifvijkbeqgwwfefeuqxlqqtqquf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sltjsbwqgfujcgiyvdrmevhixvmkdhhr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xthoqhlgslkbbeylduoeebqfwjzqdqaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lytlippaydletizcpeflvilitedytmzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yeljpcmhxymwaathfqykgqatfhudkmgc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzrjckinckgkvnwbigtmthqdppgrzvnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rothkyfhncloyppvoposzlebllpeaiaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"noxqjxnszgtprponycpnspxvxoahwaoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sebwkcukjquzyhnelqcaczlnxoyfhuda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lkvegqytpxzvdlzeurzdgqpenaoqctbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkdbhwrpkwdntqccvluqsfepyssotfvb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtjojgjbnmconwagyztonelwakvtblkk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yobfwcwufipddiogqyzaabfortenwrfk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bebshvjehqjzlgpugedverxqtgacbyob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwpflxbwnykuzovirbytlmjvypdkbpks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zkhzehsqcuhkucnegwsckkdntsdfopfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlxbuuyhpuctrqwforwiyduiabkckmmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbtvugedqvtbtlwucxsgexiopylcsptv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxpjpdpgqligfefxtqtkpjisrovhgmdg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggngcefztejwabajooanimnirtrqixes","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eaqzzlfkgswntjyzyqhyudphpijnfxqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elavnoimqzrmrdikanmqwetjpbhnumhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vznudohigyfaumriqaooedxvtjttfguw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bfpepkfasvlwcdnlsmppwcrjfuwcvpkc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bfzbqmxpgqsozalapvhuevsurzyovxjb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjvmugzlswevsetwdnzxtrzxtcunzbfa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znqwfnbprrmwlnntvcttmhfmmrlkizgl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjzelccpochtupsxgysetotcqjjcmhmz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eogeqwrvhasdjbaxzaeawmsecmjrmqza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovfczxlvvxrbicobbikmcdgssdbjwvaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evnuqjdkqflemvhgdpwrnnhrujmzsawh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xtnrwefmdhhazrmtaijgnuremvnuhjzc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llbkrshoikhqeumwpimoylcrcpafaiwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sqocepfturohygfugbwyjshxuhiqmstd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctgugpsbmijnmviholhkugjbzpvukeqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjngaoujkphuaeepflozrazqaouyoylh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmlgygebfgpefcytgdrqepwehjftobxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krektrebaxhksoanaenseegsmphjjnkm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qowiecfesyjfioqcwuwfhwiwobsokjzx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hnjkeedwruorflkgdhpyhljfhhsupyjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utqrfavntqjdafpwbpycxnaernzgjoph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eufphpdyiakhodgrndvlickwkkplbilq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nrexdgralhvfsildepopcvrifdaluerx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668344,"databaseName":"models_schema","ddl":"CREATE TABLE `gsrskijzyhcygzvokrbnmkbtnlfyglog` (\n `efraljbcqbnruygmdcmqwxwxqwbzglpd` int NOT NULL,\n `axjktinpuujnwtseafnqwnswueucovxo` int DEFAULT NULL,\n `bxyfjzpfqijqqqdhbusqsopmafnplkec` int DEFAULT NULL,\n `wwyqqvwjbtvqqovkappzcydsgqurgsyu` int DEFAULT NULL,\n `qwwsaysjfygtqpzjqmboolfhejepidwj` int DEFAULT NULL,\n `cgvvvmfafupycbysldeirsvsueoazfiw` int DEFAULT NULL,\n `tlwuilpmzdlcpfifitilvwzdtprxpbhd` int DEFAULT NULL,\n `lglmqwcfrhzvfjfdlrvsctzowgasfbgp` int DEFAULT NULL,\n `yrvkwkhbfubgnvciqwhcbltwlnkabulk` int DEFAULT NULL,\n `efbxamjronudwwlbipzyrqxkjakmabfz` int DEFAULT NULL,\n `unyrrcpwxjedemdsqwexpytlqlzwlhdu` int DEFAULT NULL,\n `fhimfsyvcbremzrdpkmrosrvfdqskilk` int DEFAULT NULL,\n `zeoraantcpwiwvjlrgywaotvkfjzngpw` int DEFAULT NULL,\n `xisftxkllimwccpvyguvpxgxkvvmshos` int DEFAULT NULL,\n `itwyiuaadtgkwcupkwzrlmxdlyvswcyo` int DEFAULT NULL,\n `wwtyudvudbmbnrhzddsfocoxnwbaxnjt` int DEFAULT NULL,\n `rcejrylbkmtbwbxnokrkypaldlbqamqz` int DEFAULT NULL,\n `zbzkvqsieopewgyjanygotecofpicgoe` int DEFAULT NULL,\n `wqummeiksdlewxsyjyoktjabzpnieort` int DEFAULT NULL,\n `oxkrfwpkexafozqvxhlmgymhgrkdkeuq` int DEFAULT NULL,\n `nkgxbgcgycdvqjwhzmbfskrwxblusjdu` int DEFAULT NULL,\n `cphibuvajxhpgccusoaeviqfinviswci` int DEFAULT NULL,\n `nvsnmwaxxzcwswmuqjloeedpvluauwmf` int DEFAULT NULL,\n `lzcjapmelwkxewdjmjrarlvcqomwmxrz` int DEFAULT NULL,\n `hfdldpirjyfgdydjxcnvwvbvtpyfbolc` int DEFAULT NULL,\n `qsslxiodqpsfohveftbvfveztohlnlmo` int DEFAULT NULL,\n `nnojfvfkuznxxogbrwctjcvvhcgbglfe` int DEFAULT NULL,\n `pudkhhktoadfqfqzfokzbpjpvjkuweez` int DEFAULT NULL,\n `kknykayneqypvwwwgiwyphorifubwygp` int DEFAULT NULL,\n `mytrzqlgkqadmchhygtfqdkukxvgvrcy` int DEFAULT NULL,\n `oyufwcjycnhhtyhdknryojyjasguodxj` int DEFAULT NULL,\n `pzygrqixwyjfwblcsfgsfirruawgpgbr` int DEFAULT NULL,\n `uptpwenqhihorzenbmfqilsyujhoroky` int DEFAULT NULL,\n `fcuratiodbohyqgpwemmdaxfyvehjhqv` int DEFAULT NULL,\n `flivbtqjbwrmircjkwubwuzxvhhpnnxi` int DEFAULT NULL,\n `majftjcbtrtjzpmpxrcxatafhqlnxgst` int DEFAULT NULL,\n `qtlthmcnemwgsptaetajjffintnajbzp` int DEFAULT NULL,\n `csntliyldyckkjobohxmzmxicqpuzamn` int DEFAULT NULL,\n `rjslewgzjmteifxmbvhkrbkandgvrdrn` int DEFAULT NULL,\n `drxvjipealwgpqdlctnygbehimmfvdwb` int DEFAULT NULL,\n `zndkllmlacvncswluqshkvhqrbrffutx` int DEFAULT NULL,\n `bdmpefysensrrrjldfqqwpeqgvyuocqn` int DEFAULT NULL,\n `tzegxcuvgdwdfudxgfarbdnzciumeyih` int DEFAULT NULL,\n `rvpvhxxdexdhsbjedvwgxcrtnhwdeidr` int DEFAULT NULL,\n `etlrfyocwqccsqijhveqkoptaiihukjq` int DEFAULT NULL,\n `xrsnbkwhpetdwrazsuebzeaowkymhmlj` int DEFAULT NULL,\n `maqjhctbcgvyzovflwauncpqnaeoubvg` int DEFAULT NULL,\n `gohmjrybjtcfsebqtoyfsgdrwjsyuadn` int DEFAULT NULL,\n `mtvtmqemswpkmvawjfrxfkcekbcupyoy` int DEFAULT NULL,\n `upquomjnplxyykppnqbaocatbjqdqpjv` int DEFAULT NULL,\n `uopdytinaqwvndbugjhqycshzylucwfq` int DEFAULT NULL,\n `uyhxcevnjrnnfykzmlumxlzouefnctkz` int DEFAULT NULL,\n `hliirmcdknbhmtstreilyltyytizzgzg` int DEFAULT NULL,\n `ksfxvrvttlqhlvimqkmhyqmixkvtatvx` int DEFAULT NULL,\n `nafufflmjwhfmmvnivizitujikduyqnx` int DEFAULT NULL,\n `vtnazfmvpzapwrhbjpfvppilizcakxmq` int DEFAULT NULL,\n `ywxhdhxtrcxtyuukrgaqbrxzepcbbhrl` int DEFAULT NULL,\n `mmrioydifbnmuummlzeetzuwspnczytp` int DEFAULT NULL,\n `pwebsyxnfcrtakrvhkftkclxmlkfrvur` int DEFAULT NULL,\n `wkdzzausxpdwnxljpgbiqnmqdyvqnprh` int DEFAULT NULL,\n `qkzrxnpiukcauopyficzrjfntqhphysh` int DEFAULT NULL,\n `gkhhedboqbpusqmltnhznslebbiydpae` int DEFAULT NULL,\n `temnmkbqtjfciuckctmpwqjhtuymmvor` int DEFAULT NULL,\n `audrfqfabjhvyzsnafsysmqtoiavdoxk` int DEFAULT NULL,\n `lgnpqxajvygltccmuuvvngqylcllanfx` int DEFAULT NULL,\n `dsexhcketmwxdgyndjngoprmwpwpzjvn` int DEFAULT NULL,\n `rwyshmhjwuahzhvprmaizvgherayqwgo` int DEFAULT NULL,\n `dxgmrgsrsjxslcopuzellznitqtpqtbe` int DEFAULT NULL,\n `uxsykfqnpgupnjiueujwsngbvyxymzpw` int DEFAULT NULL,\n `gfiolrynkuoophjeifkfzxkzagqboylm` int DEFAULT NULL,\n `gatdpgczxllxfgrpwzavmgcqamwjayiw` int DEFAULT NULL,\n `mywojrecwoymzmkrdylxnceppzofqqhv` int DEFAULT NULL,\n `wztgpdrvtiarlxrgqcbluzawfrueguam` int DEFAULT NULL,\n `yfvewcvxbaapczicwsqcfwkaojflshjq` int DEFAULT NULL,\n `rpknuppgybtbqwjywukewztzeodzvowu` int DEFAULT NULL,\n `lfvkonbgmsybnpjlibpuxgyznsrjxcmc` int DEFAULT NULL,\n `obmiwgfzjoayknyarfcljrtzsgezlumc` int DEFAULT NULL,\n `uahqiktgmvyfkjhyypmhajnxhyzjrjnb` int DEFAULT NULL,\n `tjleggjykbdbvmqnongccmylghzvvdls` int DEFAULT NULL,\n `rnknupbzmhgxoeqdeljaryqwibmbnojg` int DEFAULT NULL,\n `btcncxuorqiknjnzkwuyzzdktorodden` int DEFAULT NULL,\n `joyrwphwollhhdwygqhgxkjphhziuoyg` int DEFAULT NULL,\n `kczihwjfxzhpzpsktqxhiaqyjpytxwuq` int DEFAULT NULL,\n `ucfjjwzdifjegysfcfbzakgmxtwjyuzn` int DEFAULT NULL,\n `mhscfomtkmwwtncdtplabjzbhnndcyqf` int DEFAULT NULL,\n `ytmtfhdluibmztswaysxffcsvflvwfuc` int DEFAULT NULL,\n `memxnjrdfnddsqiphvfjsgphezblyesu` int DEFAULT NULL,\n `bzasvvjexqaqxmhvlceyasvkraqvyasa` int DEFAULT NULL,\n `ohegvpctejseekenuslavxqgifybtjsk` int DEFAULT NULL,\n `vrqpayvhblmlhxrbajasuvgooxhvrxuy` int DEFAULT NULL,\n `duytbfavkxxwbvnoswepjxtpipcbajmh` int DEFAULT NULL,\n `nulseppieegfqsxtofocihealkyhjjrh` int DEFAULT NULL,\n `abrqlroskagxxjclmirwcuxxsoatxjte` int DEFAULT NULL,\n `tnfiwexsftrzpdithbwmmrixsjwlkivj` int DEFAULT NULL,\n `drlzqdewbuuhlnvsllizwduqxykcwoxd` int DEFAULT NULL,\n `ujikecfxfmvydrgnuksxwzblqltzjybb` int DEFAULT NULL,\n `wjjbbpibnvqcfzllardlvlumnmyrjefw` int DEFAULT NULL,\n `jhoueviczmabbajcqxjaeuwyixtxzbnv` int DEFAULT NULL,\n `faujjugwutfcpmmxthwwolfletjewyhu` int DEFAULT NULL,\n `wamuixjdqgtuflhurihwjwrehojhgeri` int DEFAULT NULL,\n PRIMARY KEY (`efraljbcqbnruygmdcmqwxwxqwbzglpd`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"gsrskijzyhcygzvokrbnmkbtnlfyglog\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["efraljbcqbnruygmdcmqwxwxqwbzglpd"],"columns":[{"name":"efraljbcqbnruygmdcmqwxwxqwbzglpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"axjktinpuujnwtseafnqwnswueucovxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bxyfjzpfqijqqqdhbusqsopmafnplkec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwyqqvwjbtvqqovkappzcydsgqurgsyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwwsaysjfygtqpzjqmboolfhejepidwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgvvvmfafupycbysldeirsvsueoazfiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlwuilpmzdlcpfifitilvwzdtprxpbhd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lglmqwcfrhzvfjfdlrvsctzowgasfbgp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yrvkwkhbfubgnvciqwhcbltwlnkabulk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"efbxamjronudwwlbipzyrqxkjakmabfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unyrrcpwxjedemdsqwexpytlqlzwlhdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhimfsyvcbremzrdpkmrosrvfdqskilk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zeoraantcpwiwvjlrgywaotvkfjzngpw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xisftxkllimwccpvyguvpxgxkvvmshos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itwyiuaadtgkwcupkwzrlmxdlyvswcyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwtyudvudbmbnrhzddsfocoxnwbaxnjt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcejrylbkmtbwbxnokrkypaldlbqamqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbzkvqsieopewgyjanygotecofpicgoe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqummeiksdlewxsyjyoktjabzpnieort","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oxkrfwpkexafozqvxhlmgymhgrkdkeuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkgxbgcgycdvqjwhzmbfskrwxblusjdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cphibuvajxhpgccusoaeviqfinviswci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvsnmwaxxzcwswmuqjloeedpvluauwmf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzcjapmelwkxewdjmjrarlvcqomwmxrz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfdldpirjyfgdydjxcnvwvbvtpyfbolc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qsslxiodqpsfohveftbvfveztohlnlmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nnojfvfkuznxxogbrwctjcvvhcgbglfe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pudkhhktoadfqfqzfokzbpjpvjkuweez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kknykayneqypvwwwgiwyphorifubwygp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mytrzqlgkqadmchhygtfqdkukxvgvrcy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oyufwcjycnhhtyhdknryojyjasguodxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pzygrqixwyjfwblcsfgsfirruawgpgbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uptpwenqhihorzenbmfqilsyujhoroky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcuratiodbohyqgpwemmdaxfyvehjhqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flivbtqjbwrmircjkwubwuzxvhhpnnxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"majftjcbtrtjzpmpxrcxatafhqlnxgst","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtlthmcnemwgsptaetajjffintnajbzp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"csntliyldyckkjobohxmzmxicqpuzamn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjslewgzjmteifxmbvhkrbkandgvrdrn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drxvjipealwgpqdlctnygbehimmfvdwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zndkllmlacvncswluqshkvhqrbrffutx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdmpefysensrrrjldfqqwpeqgvyuocqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tzegxcuvgdwdfudxgfarbdnzciumeyih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rvpvhxxdexdhsbjedvwgxcrtnhwdeidr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"etlrfyocwqccsqijhveqkoptaiihukjq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrsnbkwhpetdwrazsuebzeaowkymhmlj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"maqjhctbcgvyzovflwauncpqnaeoubvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gohmjrybjtcfsebqtoyfsgdrwjsyuadn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtvtmqemswpkmvawjfrxfkcekbcupyoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upquomjnplxyykppnqbaocatbjqdqpjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uopdytinaqwvndbugjhqycshzylucwfq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uyhxcevnjrnnfykzmlumxlzouefnctkz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hliirmcdknbhmtstreilyltyytizzgzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ksfxvrvttlqhlvimqkmhyqmixkvtatvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nafufflmjwhfmmvnivizitujikduyqnx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtnazfmvpzapwrhbjpfvppilizcakxmq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywxhdhxtrcxtyuukrgaqbrxzepcbbhrl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmrioydifbnmuummlzeetzuwspnczytp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwebsyxnfcrtakrvhkftkclxmlkfrvur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wkdzzausxpdwnxljpgbiqnmqdyvqnprh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qkzrxnpiukcauopyficzrjfntqhphysh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkhhedboqbpusqmltnhznslebbiydpae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"temnmkbqtjfciuckctmpwqjhtuymmvor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"audrfqfabjhvyzsnafsysmqtoiavdoxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgnpqxajvygltccmuuvvngqylcllanfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsexhcketmwxdgyndjngoprmwpwpzjvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rwyshmhjwuahzhvprmaizvgherayqwgo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxgmrgsrsjxslcopuzellznitqtpqtbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxsykfqnpgupnjiueujwsngbvyxymzpw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfiolrynkuoophjeifkfzxkzagqboylm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gatdpgczxllxfgrpwzavmgcqamwjayiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mywojrecwoymzmkrdylxnceppzofqqhv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wztgpdrvtiarlxrgqcbluzawfrueguam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfvewcvxbaapczicwsqcfwkaojflshjq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpknuppgybtbqwjywukewztzeodzvowu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfvkonbgmsybnpjlibpuxgyznsrjxcmc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obmiwgfzjoayknyarfcljrtzsgezlumc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uahqiktgmvyfkjhyypmhajnxhyzjrjnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjleggjykbdbvmqnongccmylghzvvdls","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnknupbzmhgxoeqdeljaryqwibmbnojg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"btcncxuorqiknjnzkwuyzzdktorodden","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"joyrwphwollhhdwygqhgxkjphhziuoyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kczihwjfxzhpzpsktqxhiaqyjpytxwuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucfjjwzdifjegysfcfbzakgmxtwjyuzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhscfomtkmwwtncdtplabjzbhnndcyqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytmtfhdluibmztswaysxffcsvflvwfuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"memxnjrdfnddsqiphvfjsgphezblyesu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzasvvjexqaqxmhvlceyasvkraqvyasa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohegvpctejseekenuslavxqgifybtjsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrqpayvhblmlhxrbajasuvgooxhvrxuy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"duytbfavkxxwbvnoswepjxtpipcbajmh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nulseppieegfqsxtofocihealkyhjjrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abrqlroskagxxjclmirwcuxxsoatxjte","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnfiwexsftrzpdithbwmmrixsjwlkivj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drlzqdewbuuhlnvsllizwduqxykcwoxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujikecfxfmvydrgnuksxwzblqltzjybb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjjbbpibnvqcfzllardlvlumnmyrjefw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jhoueviczmabbajcqxjaeuwyixtxzbnv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"faujjugwutfcpmmxthwwolfletjewyhu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wamuixjdqgtuflhurihwjwrehojhgeri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668375,"databaseName":"models_schema","ddl":"CREATE TABLE `hchcurdohntyovpxuhajvwyugafgeysr` (\n `ysonfbupeivnjtbrvxbdsvzckqfambws` int NOT NULL,\n `suaeepkdcxxkllzaeoflhgejwsmccrdz` int DEFAULT NULL,\n `ryjanutcwjxneumbdidfwoqdofibeneh` int DEFAULT NULL,\n `skatsoqjgvmuiwfeptxrsghhmcymlhgj` int DEFAULT NULL,\n `hfgadpgsxzrphqoxcnbiacdqpoatakia` int DEFAULT NULL,\n `pjethlvbwahlvdefnfjgqeseudyvhvce` int DEFAULT NULL,\n `iwjvshuctjvvmomxixrqeyvnipowotjq` int DEFAULT NULL,\n `ojzelqgskzarffydcmfrlzgrccthmebv` int DEFAULT NULL,\n `jabjeqeojxwfyxwwyhcuaibpdqdmiina` int DEFAULT NULL,\n `sdkvjaktmwnspxyoeftszgiqnwovhdrb` int DEFAULT NULL,\n `tlkvnvtjqqvzsivyottslegqibjlqayv` int DEFAULT NULL,\n `yyucokjjqggroyzycogqvkrabawzvxxd` int DEFAULT NULL,\n `ikrntfnwclbydypfiqdcecerxyubaulv` int DEFAULT NULL,\n `xmzctnhwqkwvvjjxbpbwixkzbrosvzue` int DEFAULT NULL,\n `hxkilpxiqaeddnolwkvkcsjlsmjrjfwp` int DEFAULT NULL,\n `gugarfvschzuizdybsfhbobbfyjfppzt` int DEFAULT NULL,\n `ohzxasgknhczvpkjomfwfbbcckvnhvaa` int DEFAULT NULL,\n `rpfhcvoteukkhnfvdyerqnedctpftqcx` int DEFAULT NULL,\n `lgggnbtbzwzjwpfcgvevflkmuurxdrzs` int DEFAULT NULL,\n `scfkoudgsilqfvhtbfxvlbapoailngdb` int DEFAULT NULL,\n `tllmlgfxjqvlslwgryoqhfmpbeqcboyy` int DEFAULT NULL,\n `fmrnrpehycltuakzvnuuhnuggkufanrl` int DEFAULT NULL,\n `dcdbhhwnpgxjzvdfnpygzhdxfqbjyvgr` int DEFAULT NULL,\n `vokeqmfuteqaxbfaeoowmuoplvtltvgh` int DEFAULT NULL,\n `hqvisswoqsqocsdhaajditwdsznvxrez` int DEFAULT NULL,\n `nldlxsqsymrfcdvtvqyoviemprijdeua` int DEFAULT NULL,\n `zemhnujoksrvhnvnjrvbrguozvwjvhap` int DEFAULT NULL,\n `prgvunzuejeoumblmkfamfnvxclkextk` int DEFAULT NULL,\n `zbzyownkapxiqbpczoxzhrxtxoesbgty` int DEFAULT NULL,\n `exbfoobszaihtlgvtisrzfroqtjtjhgl` int DEFAULT NULL,\n `wrbossuygsetgcvrekxcruvuiddwvoev` int DEFAULT NULL,\n `atpyrltsnfhofkgoepclsomcpofoqrzy` int DEFAULT NULL,\n `gcorsrnopdtrjbqfuxzyajaxidyedacv` int DEFAULT NULL,\n `kjjvznfcjnprifuztfohpjsualrpssec` int DEFAULT NULL,\n `hvdyticorzwyaadaezwoffhdhmqkqdey` int DEFAULT NULL,\n `vsvfaymsnsgneaxirvqmwbudtolhffto` int DEFAULT NULL,\n `rdouupjibhstozzpfkuhdnhyzdlovfcp` int DEFAULT NULL,\n `tmrqwvisshlkkfszucazqsddstiwhwej` int DEFAULT NULL,\n `uvzmizbsxnyclhfpkjxpdbbofwyjrjuw` int DEFAULT NULL,\n `zpdvgznrxpdgjtkschbanovpvdkqeoac` int DEFAULT NULL,\n `yqshjkbzcopfnwabpkcfahctzmipuqbp` int DEFAULT NULL,\n `jicgigrxjelapesuuthywmjqbjesmrcg` int DEFAULT NULL,\n `kdqwyhdaahnmdagfjvhouemudantyonp` int DEFAULT NULL,\n `tvtvfvrtwcletzmekiiwwbdglzjnkqbs` int DEFAULT NULL,\n `kmhdbgsedmrjusatecwgpdjhmcqzclhr` int DEFAULT NULL,\n `flpdnrvxgccauugzwakxrmmsovjzbbvw` int DEFAULT NULL,\n `yjryanfbgsvlhjmsuivbniyvolwyakkd` int DEFAULT NULL,\n `hmnpiczfxehrkeilipulwsevvfblonaw` int DEFAULT NULL,\n `oqzytydiwmxcorzztkjisshufuyfkwhe` int DEFAULT NULL,\n `rkhiywynrglxfsrabtkfgvlxoysefril` int DEFAULT NULL,\n `nzxjjpjpbfmbumwfsquczknajqkdjhqt` int DEFAULT NULL,\n `vnttvrejptggcsshiiiopazpgtztyrzp` int DEFAULT NULL,\n `ixfejzxlfuvpwtnnlirgjfxdgvsdrqqg` int DEFAULT NULL,\n `wrtzpnnpbejcgapceqtsucinctqksayo` int DEFAULT NULL,\n `zoyuywkrvsmilyrfrgmzddporrqvzjoh` int DEFAULT NULL,\n `ycjsxxizbmwmdepmuqwifjgmckyapgsd` int DEFAULT NULL,\n `mirqbqhrbdqbbeojxxalhuphzmvovzds` int DEFAULT NULL,\n `zwrreargydesrbmjhlzkbilzrrprfkoa` int DEFAULT NULL,\n `nrxfyhwqbviubsduvaufstvfwaqdsbxd` int DEFAULT NULL,\n `jwgmafzzvakguijzuvqqxmkdsqduasoz` int DEFAULT NULL,\n `alrthtlcrelcknrscfbhgfqgbprytyko` int DEFAULT NULL,\n `egjqftwkfryidmmeuzffwnohddtjlgog` int DEFAULT NULL,\n `ciyujvkcohjpduqkegbxikpnwhgiabxp` int DEFAULT NULL,\n `rtvxvldkoiuqlfwfjiyqbumziveygtjf` int DEFAULT NULL,\n `hjwbjxzkmnjosxlfyetdomhvpupzwmnd` int DEFAULT NULL,\n `dlgtdbzfwxdsphdyskkuxdzxiwtbcalb` int DEFAULT NULL,\n `ykwycpxhwqjcjhenzmxqokobbvywxduz` int DEFAULT NULL,\n `vwdcwowvmtuhhsnwfdaxweeewhotegtu` int DEFAULT NULL,\n `mcxxxdlcktmhmqreaztibymcrrcgvllb` int DEFAULT NULL,\n `pvlydxrnabsnilkwgpeixypzrtofvddh` int DEFAULT NULL,\n `gycaxofasvaziszdkbyoorxckdzomqgw` int DEFAULT NULL,\n `kdigvcteabmcwobjszslikhiwtwbsvhn` int DEFAULT NULL,\n `uxlcddrafjkrlfjnbhnnwpotxzhkxrje` int DEFAULT NULL,\n `gtsdbohaosruuinwuougkwnfwihrnzti` int DEFAULT NULL,\n `umoqishlausrjozshuafqvyzyxuckjzz` int DEFAULT NULL,\n `kechfkgzpyyhbiilxldowjjyqqgqqdhk` int DEFAULT NULL,\n `tpanmdtgqxyismsfwfhtuizieqckghoq` int DEFAULT NULL,\n `sggqzgksmolmtdhmbxnxsjuhpabdanku` int DEFAULT NULL,\n `kqolngzdcsctigpzfzlbouaogocsmzne` int DEFAULT NULL,\n `gcxemyzmvpwnjjvfyzbzlsqddswhtaca` int DEFAULT NULL,\n `ymhfnbqhmmauktykiscoqcaogztqehor` int DEFAULT NULL,\n `guolyadrxwuouobrjjrokmpthwaoovea` int DEFAULT NULL,\n `azrqjebsmywxekidnhoqxodnqzozbvqs` int DEFAULT NULL,\n `epdhlbvsxoqycavqwgwnlqrursypafrr` int DEFAULT NULL,\n `dpjvgrafrgmvcrhkbejxojkqnoswnldd` int DEFAULT NULL,\n `xautzhxonfqpbewjzwfgkjwwoenuhdzp` int DEFAULT NULL,\n `bivvmyksahieihktvijqyhfdqugvyqcb` int DEFAULT NULL,\n `hebodmwvpuwvhuwbioncrnumyehymatt` int DEFAULT NULL,\n `lsihairbywdysdjkmhnbwtnewqetrkxa` int DEFAULT NULL,\n `xtcxapqvmnelenjxwoaoryxhryaxepnn` int DEFAULT NULL,\n `hukatwxzrdfammamqrkfqngttmqsvnlu` int DEFAULT NULL,\n `cvhpjwkvrhyhcpbysfprisfsdesuxhkt` int DEFAULT NULL,\n `qhdzrpejuwgaygqrdgyfmotnragqhxdb` int DEFAULT NULL,\n `duqpcrpurucecbiymwpgjrhfvudbfxau` int DEFAULT NULL,\n `hmfktvhjgwfrwjqysforzbirrslkphlm` int DEFAULT NULL,\n `pimpjatqtzlkilmwkwybdcjydwgrgtlp` int DEFAULT NULL,\n `zjjkupsiahzgdejutslqlbcdakqvabio` int DEFAULT NULL,\n `fsprxxjhbhenzxzmbjreozmtuwpiqntp` int DEFAULT NULL,\n `rxbpwhppgoibersqmlzdxzfgdufolwto` int DEFAULT NULL,\n `wejwkfqxvpvndqreuqfdxyluzyaqoecu` int DEFAULT NULL,\n PRIMARY KEY (`ysonfbupeivnjtbrvxbdsvzckqfambws`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"hchcurdohntyovpxuhajvwyugafgeysr\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["ysonfbupeivnjtbrvxbdsvzckqfambws"],"columns":[{"name":"ysonfbupeivnjtbrvxbdsvzckqfambws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"suaeepkdcxxkllzaeoflhgejwsmccrdz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ryjanutcwjxneumbdidfwoqdofibeneh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skatsoqjgvmuiwfeptxrsghhmcymlhgj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfgadpgsxzrphqoxcnbiacdqpoatakia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjethlvbwahlvdefnfjgqeseudyvhvce","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwjvshuctjvvmomxixrqeyvnipowotjq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojzelqgskzarffydcmfrlzgrccthmebv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jabjeqeojxwfyxwwyhcuaibpdqdmiina","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdkvjaktmwnspxyoeftszgiqnwovhdrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlkvnvtjqqvzsivyottslegqibjlqayv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yyucokjjqggroyzycogqvkrabawzvxxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikrntfnwclbydypfiqdcecerxyubaulv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmzctnhwqkwvvjjxbpbwixkzbrosvzue","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxkilpxiqaeddnolwkvkcsjlsmjrjfwp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gugarfvschzuizdybsfhbobbfyjfppzt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohzxasgknhczvpkjomfwfbbcckvnhvaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpfhcvoteukkhnfvdyerqnedctpftqcx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgggnbtbzwzjwpfcgvevflkmuurxdrzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scfkoudgsilqfvhtbfxvlbapoailngdb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tllmlgfxjqvlslwgryoqhfmpbeqcboyy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmrnrpehycltuakzvnuuhnuggkufanrl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcdbhhwnpgxjzvdfnpygzhdxfqbjyvgr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vokeqmfuteqaxbfaeoowmuoplvtltvgh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqvisswoqsqocsdhaajditwdsznvxrez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nldlxsqsymrfcdvtvqyoviemprijdeua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zemhnujoksrvhnvnjrvbrguozvwjvhap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prgvunzuejeoumblmkfamfnvxclkextk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbzyownkapxiqbpczoxzhrxtxoesbgty","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exbfoobszaihtlgvtisrzfroqtjtjhgl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wrbossuygsetgcvrekxcruvuiddwvoev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atpyrltsnfhofkgoepclsomcpofoqrzy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gcorsrnopdtrjbqfuxzyajaxidyedacv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjjvznfcjnprifuztfohpjsualrpssec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hvdyticorzwyaadaezwoffhdhmqkqdey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsvfaymsnsgneaxirvqmwbudtolhffto","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdouupjibhstozzpfkuhdnhyzdlovfcp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmrqwvisshlkkfszucazqsddstiwhwej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvzmizbsxnyclhfpkjxpdbbofwyjrjuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpdvgznrxpdgjtkschbanovpvdkqeoac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqshjkbzcopfnwabpkcfahctzmipuqbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jicgigrxjelapesuuthywmjqbjesmrcg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kdqwyhdaahnmdagfjvhouemudantyonp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvtvfvrtwcletzmekiiwwbdglzjnkqbs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmhdbgsedmrjusatecwgpdjhmcqzclhr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flpdnrvxgccauugzwakxrmmsovjzbbvw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjryanfbgsvlhjmsuivbniyvolwyakkd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmnpiczfxehrkeilipulwsevvfblonaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqzytydiwmxcorzztkjisshufuyfkwhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkhiywynrglxfsrabtkfgvlxoysefril","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzxjjpjpbfmbumwfsquczknajqkdjhqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnttvrejptggcsshiiiopazpgtztyrzp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixfejzxlfuvpwtnnlirgjfxdgvsdrqqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wrtzpnnpbejcgapceqtsucinctqksayo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zoyuywkrvsmilyrfrgmzddporrqvzjoh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ycjsxxizbmwmdepmuqwifjgmckyapgsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mirqbqhrbdqbbeojxxalhuphzmvovzds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zwrreargydesrbmjhlzkbilzrrprfkoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nrxfyhwqbviubsduvaufstvfwaqdsbxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwgmafzzvakguijzuvqqxmkdsqduasoz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"alrthtlcrelcknrscfbhgfqgbprytyko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"egjqftwkfryidmmeuzffwnohddtjlgog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ciyujvkcohjpduqkegbxikpnwhgiabxp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtvxvldkoiuqlfwfjiyqbumziveygtjf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjwbjxzkmnjosxlfyetdomhvpupzwmnd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlgtdbzfwxdsphdyskkuxdzxiwtbcalb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykwycpxhwqjcjhenzmxqokobbvywxduz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vwdcwowvmtuhhsnwfdaxweeewhotegtu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcxxxdlcktmhmqreaztibymcrrcgvllb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvlydxrnabsnilkwgpeixypzrtofvddh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gycaxofasvaziszdkbyoorxckdzomqgw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kdigvcteabmcwobjszslikhiwtwbsvhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxlcddrafjkrlfjnbhnnwpotxzhkxrje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtsdbohaosruuinwuougkwnfwihrnzti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umoqishlausrjozshuafqvyzyxuckjzz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kechfkgzpyyhbiilxldowjjyqqgqqdhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpanmdtgqxyismsfwfhtuizieqckghoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sggqzgksmolmtdhmbxnxsjuhpabdanku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqolngzdcsctigpzfzlbouaogocsmzne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gcxemyzmvpwnjjvfyzbzlsqddswhtaca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ymhfnbqhmmauktykiscoqcaogztqehor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"guolyadrxwuouobrjjrokmpthwaoovea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azrqjebsmywxekidnhoqxodnqzozbvqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"epdhlbvsxoqycavqwgwnlqrursypafrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpjvgrafrgmvcrhkbejxojkqnoswnldd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xautzhxonfqpbewjzwfgkjwwoenuhdzp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bivvmyksahieihktvijqyhfdqugvyqcb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hebodmwvpuwvhuwbioncrnumyehymatt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lsihairbywdysdjkmhnbwtnewqetrkxa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xtcxapqvmnelenjxwoaoryxhryaxepnn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hukatwxzrdfammamqrkfqngttmqsvnlu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvhpjwkvrhyhcpbysfprisfsdesuxhkt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhdzrpejuwgaygqrdgyfmotnragqhxdb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"duqpcrpurucecbiymwpgjrhfvudbfxau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmfktvhjgwfrwjqysforzbirrslkphlm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pimpjatqtzlkilmwkwybdcjydwgrgtlp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjjkupsiahzgdejutslqlbcdakqvabio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fsprxxjhbhenzxzmbjreozmtuwpiqntp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxbpwhppgoibersqmlzdxzfgdufolwto","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wejwkfqxvpvndqreuqfdxyluzyaqoecu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668407,"databaseName":"models_schema","ddl":"CREATE TABLE `hkcokgnihhdfnawwtqjefmgjhhitamsj` (\n `jrjipngpdvqkwolpnalhyfjhkqiwlind` int NOT NULL,\n `frurxiscojimzoawbiwafbforzdzprxc` int DEFAULT NULL,\n `hjilpdjenkyoapmzwggmtgxtmiqvzjin` int DEFAULT NULL,\n `xshrtkmfnvpuwklblgpsomsfkpezxfnz` int DEFAULT NULL,\n `bvkjsqtwmxzxmvwlpbazwdzirzxwyvcq` int DEFAULT NULL,\n `tswxctabknuyzsypfsgfkxikveecifks` int DEFAULT NULL,\n `fyfhnguvjxyisukgvfqcjhtkuodifaiz` int DEFAULT NULL,\n `fgwndfhxdlmeqkezvdghjjvlajdgtdfz` int DEFAULT NULL,\n `xjpgpomelhbtlthkhzzykxssyjkrswmg` int DEFAULT NULL,\n `mwsabnqmsnaazlhybgqctkzlsoqamkvq` int DEFAULT NULL,\n `grtzpnntpbaervljgjtcvrwwghajkpdp` int DEFAULT NULL,\n `ohiqhmgroqfnqlpneczlmopdounmjjqj` int DEFAULT NULL,\n `exnatteukknkoqbbupdeqkzlkkxfotis` int DEFAULT NULL,\n `vuhmxoesexhbewognuawqhmgxpxnzqpm` int DEFAULT NULL,\n `bbbxcoxyhdregrnqsnsukmqjgtuejiuc` int DEFAULT NULL,\n `ihcypocrmvbzxmbcxsaneychctpngdgr` int DEFAULT NULL,\n `cqgibmokdhrufajccsudpaiufkmerlob` int DEFAULT NULL,\n `igdeakbvvxfctswqwrogstwikxhbfsam` int DEFAULT NULL,\n `wcstpkpvbflktsjwdvgqgbsitebkwirg` int DEFAULT NULL,\n `ynnbxzsakavacsocanfluldiqpoxyodq` int DEFAULT NULL,\n `kkpwwxlxjaljfowamvwpmimsheicbhso` int DEFAULT NULL,\n `hxgtcfywdeadaubjitytqvzqvwhzqger` int DEFAULT NULL,\n `agwujufzbwxzlkazvrkniowlqoyeygxy` int DEFAULT NULL,\n `abpzdfenvnmlhybsehqnacmlcaqhtcaw` int DEFAULT NULL,\n `vcqfxxkcrlmwrraefnkwsmpdwuwmelbz` int DEFAULT NULL,\n `ebleysrspthseevwevywaeauepgxbpwf` int DEFAULT NULL,\n `svhqliwlfkzsluurrovhwhkffdxrvxoo` int DEFAULT NULL,\n `viupaajxtzjvxfawhiebzqrhzilsefbs` int DEFAULT NULL,\n `zemqndbmxtzgiqkjknfcyelrutduqmkq` int DEFAULT NULL,\n `fpzcsqmkggolojrkorljxvewuqajxyfw` int DEFAULT NULL,\n `jxknwmxfndqfftdtdidjrypycnrjvbid` int DEFAULT NULL,\n `juyhtfyuyolnybmmxoymrriaeqqyggxd` int DEFAULT NULL,\n `zkddxbqdcvdfblqbijafzwwxzwoubunp` int DEFAULT NULL,\n `lddoaxmsztnjxxdrwypoxrbsvpcvdwqa` int DEFAULT NULL,\n `ttiazrkwmovmvkhudxlquahetkjhqccb` int DEFAULT NULL,\n `fjaflgdlozrfhmusojsrnhaxwfucfwgx` int DEFAULT NULL,\n `zjlksevjcakgxnhomxwjebybhmakftzh` int DEFAULT NULL,\n `cmrxwfoymdkkdwmfudllsnwgfqjwxzbb` int DEFAULT NULL,\n `uvtulgcbflzrdayzbholcujeekjgkcjn` int DEFAULT NULL,\n `exfekbyxxtuhwqdarfgwceppzgevpoaj` int DEFAULT NULL,\n `ubmvbwenqnseulctlxfkucvhngsmtgyn` int DEFAULT NULL,\n `bdouxxtmzdmiybvjlbigeohbvgwwotce` int DEFAULT NULL,\n `kfmjwtuwwcxvljfenhdlgskleyhosrkj` int DEFAULT NULL,\n `fdvmdujciqkqwvvitngeuzkpjtenzdsf` int DEFAULT NULL,\n `jmqornjldmzwfkylahgmckgcxfpnuuix` int DEFAULT NULL,\n `cnovsgbhkfvryifxnrbvhnhwrtussedw` int DEFAULT NULL,\n `dhrshzzpzhuywqwzvudomvkotskvoyxr` int DEFAULT NULL,\n `lhugmoclktidmkpvsueoviwwytkiwqul` int DEFAULT NULL,\n `sxkilmegpkrsfvgfnftajficmfvxmwcz` int DEFAULT NULL,\n `icdfvbtssqjflorufkjzgegflxhqrpvv` int DEFAULT NULL,\n `wwkkyfmrwxaewhszwgfdezxddpvlykco` int DEFAULT NULL,\n `myskaqxnxtikrephltvyopovsyroqbzw` int DEFAULT NULL,\n `fhbwajhrooxgzuhjbhjvyiczxbjtqwji` int DEFAULT NULL,\n `hbkycgxukkowsbpuciabybaenrqftcjg` int DEFAULT NULL,\n `cnkzaavdmwmjvhpziyyswbvicmdmttrg` int DEFAULT NULL,\n `qtconwtiyyggzqhbphcuxzqzzhbagqeg` int DEFAULT NULL,\n `xjhmbuubbywciuvzuwtmcrvtswlqghgm` int DEFAULT NULL,\n `eecitrrahnlsvtluuultaqygxnwjoclb` int DEFAULT NULL,\n `dioywewhcinreyodmlxkimvivwkkuaew` int DEFAULT NULL,\n `gmxembqaoelqlpxnfeqxtzpoawcviwwh` int DEFAULT NULL,\n `fmmuwsakjpgptapeztxridxzpvhvtncw` int DEFAULT NULL,\n `ieqswdtezpbkvyifwjhzafytuopmnoyn` int DEFAULT NULL,\n `vhxnkazcsxznpwbyygkbmzzevkyoxstl` int DEFAULT NULL,\n `fqbmkbbmezmrncxkyanryhclvdbiwxey` int DEFAULT NULL,\n `txaxpshpcgtjtydvnkgavfyvwfcjtdxv` int DEFAULT NULL,\n `javuoipmtaufkrqltkzqajtqvmmpqcjp` int DEFAULT NULL,\n `mndvxwcnyfylvqftcvpiqgbbgfxlqlck` int DEFAULT NULL,\n `fggrvdlerdybetxhthqrvwkhfonewscu` int DEFAULT NULL,\n `qcqliwjkhoxcdtpcsdzhnofjfwmcrpfs` int DEFAULT NULL,\n `bljyioclaxrefsvpbxrxbtctmaykvzsq` int DEFAULT NULL,\n `fbfhsagzbkxsiyzzjvhhnpuefdyhvbrs` int DEFAULT NULL,\n `ojzrccxkgkqssdvpjpmdiidvrblsntwu` int DEFAULT NULL,\n `atteortedrspruqihcnnklyyakrmjskj` int DEFAULT NULL,\n `zpyiedbquybhvhdlomciejmovpasuayq` int DEFAULT NULL,\n `kvmetbqhqpowipttlcxyhomerqkmhzma` int DEFAULT NULL,\n `axqyrtcqfivzlspaucmhzemuotgjkzvg` int DEFAULT NULL,\n `rbyaiegqdlnufojwgpzoqijhxzvpdtyl` int DEFAULT NULL,\n `bylwmiygzxdynyjxoicwblswwzsygdkd` int DEFAULT NULL,\n `vrtgfmwubdgzwwwikyxjzszrubvognpk` int DEFAULT NULL,\n `lagfknyjhruuzasbyrvlsroifsnmlxaj` int DEFAULT NULL,\n `retvqprtjnndeulmiwulctmlmtrkgtze` int DEFAULT NULL,\n `zthwrddolpdxjowxprniggdfkvzlzrnj` int DEFAULT NULL,\n `jiwrfqbpgkqwraoqcggypxennyoxxzoj` int DEFAULT NULL,\n `hyixxbneiecuerjanmwxemfkzizdfymd` int DEFAULT NULL,\n `ygnygtbfthrsjsdmcttlvcupuchgicjs` int DEFAULT NULL,\n `javxqhxbvxgkouvnysbgfprotvkqpehs` int DEFAULT NULL,\n `pbeukestkougfnlkiomvikkqdlhwouao` int DEFAULT NULL,\n `tcrzbwfxwcmtzviiyszourkauilivjru` int DEFAULT NULL,\n `tvczuyvpyftmafcexproeinxooxinzun` int DEFAULT NULL,\n `wwumzvxvseznxaszjwujogulvybgjazt` int DEFAULT NULL,\n `opzplcovlwfidmoslsqwqgzecrszhzww` int DEFAULT NULL,\n `jngcddkndrgpoujonafcoaafdgeaswco` int DEFAULT NULL,\n `xounyrltuydjrowfackiemtzsntakxqc` int DEFAULT NULL,\n `yoobjwchfbkfbyehfvlhnkfvuxoybyiv` int DEFAULT NULL,\n `ibggtkqeydvshixcrbmixpiuppqsahrk` int DEFAULT NULL,\n `uxggicoigupsteezplemvphzxinnrevh` int DEFAULT NULL,\n `qvtzmriuuhhuuibbdeaciaykdczatzqj` int DEFAULT NULL,\n `ouhwprmhkjfvzbedyyrhuqifjgvdwjgv` int DEFAULT NULL,\n `nueqlakgpaaarltojaqyxeyrgwvlpbhb` int DEFAULT NULL,\n `yxaohphhxrtyltvotnwmkjpzrnrxrttk` int DEFAULT NULL,\n PRIMARY KEY (`jrjipngpdvqkwolpnalhyfjhkqiwlind`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"hkcokgnihhdfnawwtqjefmgjhhitamsj\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["jrjipngpdvqkwolpnalhyfjhkqiwlind"],"columns":[{"name":"jrjipngpdvqkwolpnalhyfjhkqiwlind","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"frurxiscojimzoawbiwafbforzdzprxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjilpdjenkyoapmzwggmtgxtmiqvzjin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xshrtkmfnvpuwklblgpsomsfkpezxfnz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvkjsqtwmxzxmvwlpbazwdzirzxwyvcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tswxctabknuyzsypfsgfkxikveecifks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fyfhnguvjxyisukgvfqcjhtkuodifaiz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fgwndfhxdlmeqkezvdghjjvlajdgtdfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjpgpomelhbtlthkhzzykxssyjkrswmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwsabnqmsnaazlhybgqctkzlsoqamkvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grtzpnntpbaervljgjtcvrwwghajkpdp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohiqhmgroqfnqlpneczlmopdounmjjqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exnatteukknkoqbbupdeqkzlkkxfotis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vuhmxoesexhbewognuawqhmgxpxnzqpm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbbxcoxyhdregrnqsnsukmqjgtuejiuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ihcypocrmvbzxmbcxsaneychctpngdgr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqgibmokdhrufajccsudpaiufkmerlob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igdeakbvvxfctswqwrogstwikxhbfsam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcstpkpvbflktsjwdvgqgbsitebkwirg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynnbxzsakavacsocanfluldiqpoxyodq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kkpwwxlxjaljfowamvwpmimsheicbhso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxgtcfywdeadaubjitytqvzqvwhzqger","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agwujufzbwxzlkazvrkniowlqoyeygxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abpzdfenvnmlhybsehqnacmlcaqhtcaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vcqfxxkcrlmwrraefnkwsmpdwuwmelbz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebleysrspthseevwevywaeauepgxbpwf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"svhqliwlfkzsluurrovhwhkffdxrvxoo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"viupaajxtzjvxfawhiebzqrhzilsefbs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zemqndbmxtzgiqkjknfcyelrutduqmkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpzcsqmkggolojrkorljxvewuqajxyfw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jxknwmxfndqfftdtdidjrypycnrjvbid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"juyhtfyuyolnybmmxoymrriaeqqyggxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zkddxbqdcvdfblqbijafzwwxzwoubunp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lddoaxmsztnjxxdrwypoxrbsvpcvdwqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ttiazrkwmovmvkhudxlquahetkjhqccb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjaflgdlozrfhmusojsrnhaxwfucfwgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjlksevjcakgxnhomxwjebybhmakftzh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmrxwfoymdkkdwmfudllsnwgfqjwxzbb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvtulgcbflzrdayzbholcujeekjgkcjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exfekbyxxtuhwqdarfgwceppzgevpoaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubmvbwenqnseulctlxfkucvhngsmtgyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdouxxtmzdmiybvjlbigeohbvgwwotce","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfmjwtuwwcxvljfenhdlgskleyhosrkj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fdvmdujciqkqwvvitngeuzkpjtenzdsf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmqornjldmzwfkylahgmckgcxfpnuuix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnovsgbhkfvryifxnrbvhnhwrtussedw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhrshzzpzhuywqwzvudomvkotskvoyxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhugmoclktidmkpvsueoviwwytkiwqul","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sxkilmegpkrsfvgfnftajficmfvxmwcz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icdfvbtssqjflorufkjzgegflxhqrpvv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwkkyfmrwxaewhszwgfdezxddpvlykco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"myskaqxnxtikrephltvyopovsyroqbzw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhbwajhrooxgzuhjbhjvyiczxbjtqwji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbkycgxukkowsbpuciabybaenrqftcjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnkzaavdmwmjvhpziyyswbvicmdmttrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtconwtiyyggzqhbphcuxzqzzhbagqeg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjhmbuubbywciuvzuwtmcrvtswlqghgm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eecitrrahnlsvtluuultaqygxnwjoclb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dioywewhcinreyodmlxkimvivwkkuaew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmxembqaoelqlpxnfeqxtzpoawcviwwh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmmuwsakjpgptapeztxridxzpvhvtncw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ieqswdtezpbkvyifwjhzafytuopmnoyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhxnkazcsxznpwbyygkbmzzevkyoxstl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqbmkbbmezmrncxkyanryhclvdbiwxey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txaxpshpcgtjtydvnkgavfyvwfcjtdxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"javuoipmtaufkrqltkzqajtqvmmpqcjp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mndvxwcnyfylvqftcvpiqgbbgfxlqlck","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fggrvdlerdybetxhthqrvwkhfonewscu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qcqliwjkhoxcdtpcsdzhnofjfwmcrpfs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bljyioclaxrefsvpbxrxbtctmaykvzsq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbfhsagzbkxsiyzzjvhhnpuefdyhvbrs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojzrccxkgkqssdvpjpmdiidvrblsntwu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atteortedrspruqihcnnklyyakrmjskj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpyiedbquybhvhdlomciejmovpasuayq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvmetbqhqpowipttlcxyhomerqkmhzma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axqyrtcqfivzlspaucmhzemuotgjkzvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbyaiegqdlnufojwgpzoqijhxzvpdtyl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bylwmiygzxdynyjxoicwblswwzsygdkd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrtgfmwubdgzwwwikyxjzszrubvognpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lagfknyjhruuzasbyrvlsroifsnmlxaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"retvqprtjnndeulmiwulctmlmtrkgtze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zthwrddolpdxjowxprniggdfkvzlzrnj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jiwrfqbpgkqwraoqcggypxennyoxxzoj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hyixxbneiecuerjanmwxemfkzizdfymd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ygnygtbfthrsjsdmcttlvcupuchgicjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"javxqhxbvxgkouvnysbgfprotvkqpehs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbeukestkougfnlkiomvikkqdlhwouao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tcrzbwfxwcmtzviiyszourkauilivjru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvczuyvpyftmafcexproeinxooxinzun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwumzvxvseznxaszjwujogulvybgjazt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opzplcovlwfidmoslsqwqgzecrszhzww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jngcddkndrgpoujonafcoaafdgeaswco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xounyrltuydjrowfackiemtzsntakxqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yoobjwchfbkfbyehfvlhnkfvuxoybyiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibggtkqeydvshixcrbmixpiuppqsahrk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxggicoigupsteezplemvphzxinnrevh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvtzmriuuhhuuibbdeaciaykdczatzqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ouhwprmhkjfvzbedyyrhuqifjgvdwjgv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nueqlakgpaaarltojaqyxeyrgwvlpbhb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxaohphhxrtyltvotnwmkjpzrnrxrttk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668443,"databaseName":"models_schema","ddl":"CREATE TABLE `hvfxfpdbsddegbronmrpaixtvsqjqygk` (\n `swjdxbexcydbgwfprzuqbgflyvaapccg` int NOT NULL,\n `xzjvzklhemimrixabmpflxxfxdyxeqhz` int DEFAULT NULL,\n `vqpsdgbgjiuvhapadmaouqvtygoasjxn` int DEFAULT NULL,\n `jfsssrqnrabfhugpikcueivjlczijipb` int DEFAULT NULL,\n `skjuhktlicrnhiumlwizlvrrinvbuvmb` int DEFAULT NULL,\n `ivaxwfbjxzgnndpjwssumqrxtmppccco` int DEFAULT NULL,\n `ncvhbdiajoqcmhqpstgjuyvytgkjwlfw` int DEFAULT NULL,\n `ofhtldaxppulezacepswqooexoeuwdmk` int DEFAULT NULL,\n `nchzectyftsvuigbdfiociofyzyshuqy` int DEFAULT NULL,\n `sijtmqppmnlfycudfgembqwydricbcba` int DEFAULT NULL,\n `cqpqthrayfcdmyutuioembrfoucdfwgl` int DEFAULT NULL,\n `psgybpjoxvugwgtyhdxigkoqnwseqycj` int DEFAULT NULL,\n `aeoinixgyibktvcyqtozevlelahmjuan` int DEFAULT NULL,\n `gvazivyztxhacjkxxtntcmfvcrojnifl` int DEFAULT NULL,\n `kwvgjnqgozlrysmvottalulqpwjkdspf` int DEFAULT NULL,\n `qrkogkhrjdeekafzjlqkwnpnvqhxvvwg` int DEFAULT NULL,\n `iqgbyqzyecyxjxlcpsixcqybtrmucoqe` int DEFAULT NULL,\n `iiqiwiwkjhzwjakxjeqxkgmpgqpuvypu` int DEFAULT NULL,\n `fppgsmrlaqjxdjzekcyfemrefslytdya` int DEFAULT NULL,\n `gtbkidjpfifrizztfsotmsresqcuekmc` int DEFAULT NULL,\n `zvaqdmtuayrxlytjxwlgbnpguqynzrjn` int DEFAULT NULL,\n `gxerbnobeclgxqyejuttstifvtgqycor` int DEFAULT NULL,\n `qvkpgrvazawmhdrpqontjehlozxmmqlm` int DEFAULT NULL,\n `vibfvgwwucgzrquannaiodwuyjontowz` int DEFAULT NULL,\n `dcyzqtfhiqajfwqyyqozpkygkjexdktz` int DEFAULT NULL,\n `wdddekclrsqqllvjlnsasaankuzpkgkq` int DEFAULT NULL,\n `rlfcdnjayxubqralbcrzspcyabmoycix` int DEFAULT NULL,\n `xvaustjfixvofiickumpalpstbiabtba` int DEFAULT NULL,\n `arrluuauazyysjijinavkzccyksaqnqt` int DEFAULT NULL,\n `mlujnpsnxtpyzsosrekdhpuhuyiucnck` int DEFAULT NULL,\n `gqkhnswhlqbtmckqmtollargboffosde` int DEFAULT NULL,\n `qsiprozbglpkpnoumoiqosjsmjbseqzg` int DEFAULT NULL,\n `zvqugxpgdpfxzyabqnupvjeskmgranlf` int DEFAULT NULL,\n `dsixrfiknrpvivbwydopyfdmezrsgpky` int DEFAULT NULL,\n `jmuykrmpligvepzbyptafhcpmsgvezba` int DEFAULT NULL,\n `owctqamhbnijkalbvniumnabwkabkxuk` int DEFAULT NULL,\n `ypyfwxscnotzuszoayjwhtapmjlbmmov` int DEFAULT NULL,\n `yujftkgplwkbdzerdqohuimfwmdxdpor` int DEFAULT NULL,\n `sioqvzwzhztkhrnoyhwkyssgtpnpozdp` int DEFAULT NULL,\n `huorgyrpsritinmoqimcflaqfpqsqxip` int DEFAULT NULL,\n `elvvjkzalxboemfecokqdqycmqusqfsx` int DEFAULT NULL,\n `qjkzcipmkkixzmbrwktoekxwwycmgusz` int DEFAULT NULL,\n `ugfusnpvicvugdkroghivndkbzqjamsf` int DEFAULT NULL,\n `neradnzbdhxcmmxrixcopxlgqttbkhch` int DEFAULT NULL,\n `kjrrnhtmxhvtwanavkwxznbzxfpauzwv` int DEFAULT NULL,\n `vlaseallxxmsspeqfckjypfzwmfsfhwd` int DEFAULT NULL,\n `cfoxbtkevqwdmggszzkosanusmvjtlxt` int DEFAULT NULL,\n `mqndosdonwpulqtgqzncqcbzwygkqtme` int DEFAULT NULL,\n `skxcbrcirfmbnfdfqidkvldmleclvgqi` int DEFAULT NULL,\n `leskfwhtjfhdussmlwrkypcagycgazov` int DEFAULT NULL,\n `viavotsoymhdkcdovouvovqmzlfyttva` int DEFAULT NULL,\n `szxmgsutuggjdltnqhnfeziizjajgxrn` int DEFAULT NULL,\n `slykycaxhnrsiqpikyvxujhlzcucctns` int DEFAULT NULL,\n `ownbauyretuynffmwpldomjdedivdwfr` int DEFAULT NULL,\n `dgapqioawxkdodgcbqinakfsbslrysyk` int DEFAULT NULL,\n `bzgpxjvtoofxnfxhffrizzekpiktbeeg` int DEFAULT NULL,\n `zmlbcthsqrmwuhadkqgelwcxpaeaxufr` int DEFAULT NULL,\n `fwpfrmlwzqhuqktuzklzrvwsncgcxbou` int DEFAULT NULL,\n `tplofdzqlxyvunswrnbyhbybvereamvy` int DEFAULT NULL,\n `silgrrdhqwswksibzfvjsqselhvqmoyv` int DEFAULT NULL,\n `cicbauzpyjarqhtxeboezhggyshkfcyx` int DEFAULT NULL,\n `bvgcolltqhcfpwpnkhfhwprrqfartche` int DEFAULT NULL,\n `azjsjwqgcqgtqaushjlkfajumqlrfizv` int DEFAULT NULL,\n `kyzxvrsxavptdvrvmlltjbvbdjvxeynv` int DEFAULT NULL,\n `ognuihpbehwmgxxygmxuehdwiujfwrhd` int DEFAULT NULL,\n `gooxhvwukkzudnvtlnnufihhdwnzeepl` int DEFAULT NULL,\n `tsvfsqqlfrboiaajdgfmymarvedzmyfw` int DEFAULT NULL,\n `zrjthshihkbrvifrugdtzvkbavypasbv` int DEFAULT NULL,\n `ztxzyjekxemjfbxojnatrvjsxmmuomrs` int DEFAULT NULL,\n `qcmpoxfralwjllwwigsocayzrywxnvkl` int DEFAULT NULL,\n `sczgwakulyvbfbdmfoczkshtrqnffliq` int DEFAULT NULL,\n `qqoovezhqwmuqwlbzuasbhbategnjyhs` int DEFAULT NULL,\n `ybpowikjcrhtuowbgwrehrrrkxkteuha` int DEFAULT NULL,\n `dwaiexbkaacotbuypoaewlriafwcxwsn` int DEFAULT NULL,\n `nbntoubtiwvbjioookkchduaqdyoeyah` int DEFAULT NULL,\n `jwbnuwljvxfxmeqjoopiqagmorwlnstp` int DEFAULT NULL,\n `epzqdcggvbkqujkrwsrwecqyyvkvzcls` int DEFAULT NULL,\n `ezbrdepvvzmwpydeqaciudqvvdrrzhbs` int DEFAULT NULL,\n `jxfsejrmzdfmxaphrcyebvnxjnfbcjnw` int DEFAULT NULL,\n `cnbiwijywftjoqfchnkdxbmbvgtmvsdv` int DEFAULT NULL,\n `ksdhbrimniwscnthivubvkokmmkdjwgv` int DEFAULT NULL,\n `bxhcvctlaxkhwuzcqgbukzgcxuavrquj` int DEFAULT NULL,\n `ztyqpurbqpsxqfldrvuzgitagkgqafea` int DEFAULT NULL,\n `wqqidwjqptpoepluwoqocmdvyjgpekfq` int DEFAULT NULL,\n `icahorfnxtqbbvactlvavylrnqxfyrhc` int DEFAULT NULL,\n `zvjuvmdxomfgnbwsujbdywsvdluecfcj` int DEFAULT NULL,\n `gfjsgblywxucpzhfpwrytlnugbeiwanj` int DEFAULT NULL,\n `vhlghttenmepqiuwraayffzcddzxfpgi` int DEFAULT NULL,\n `casxcdnklzsifldongcgcmtsvokaqlrt` int DEFAULT NULL,\n `qjtzpkcutylzbvorqgtgyfoxelrddfnm` int DEFAULT NULL,\n `tklnkddaegqpuiteskshwvvbtratsqai` int DEFAULT NULL,\n `huwcgwjgqzwblildhijojzxcbfacixpi` int DEFAULT NULL,\n `suyanjxubxkxwayadksgvlhxxxbvfvpt` int DEFAULT NULL,\n `wivwdmezvdxqwwsfcngdqgojorymybre` int DEFAULT NULL,\n `hldxdapzskpjbhadiqbmtdbfckmvywld` int DEFAULT NULL,\n `uishwgosyrzzvgapfgnwlbocrnuoagie` int DEFAULT NULL,\n `vfsmnligxionqhmzlpvbmfijlzgutrqp` int DEFAULT NULL,\n `ezvxejvefezzcuezjiklomidhmagfglp` int DEFAULT NULL,\n `dwljkkyzdjrxserovyquntmaszcogsdx` int DEFAULT NULL,\n `fofjoadlddksuseseyvglzbiwnblelso` int DEFAULT NULL,\n PRIMARY KEY (`swjdxbexcydbgwfprzuqbgflyvaapccg`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"hvfxfpdbsddegbronmrpaixtvsqjqygk\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["swjdxbexcydbgwfprzuqbgflyvaapccg"],"columns":[{"name":"swjdxbexcydbgwfprzuqbgflyvaapccg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"xzjvzklhemimrixabmpflxxfxdyxeqhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqpsdgbgjiuvhapadmaouqvtygoasjxn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jfsssrqnrabfhugpikcueivjlczijipb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skjuhktlicrnhiumlwizlvrrinvbuvmb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivaxwfbjxzgnndpjwssumqrxtmppccco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ncvhbdiajoqcmhqpstgjuyvytgkjwlfw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofhtldaxppulezacepswqooexoeuwdmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nchzectyftsvuigbdfiociofyzyshuqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sijtmqppmnlfycudfgembqwydricbcba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqpqthrayfcdmyutuioembrfoucdfwgl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psgybpjoxvugwgtyhdxigkoqnwseqycj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aeoinixgyibktvcyqtozevlelahmjuan","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gvazivyztxhacjkxxtntcmfvcrojnifl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwvgjnqgozlrysmvottalulqpwjkdspf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrkogkhrjdeekafzjlqkwnpnvqhxvvwg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqgbyqzyecyxjxlcpsixcqybtrmucoqe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iiqiwiwkjhzwjakxjeqxkgmpgqpuvypu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fppgsmrlaqjxdjzekcyfemrefslytdya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtbkidjpfifrizztfsotmsresqcuekmc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvaqdmtuayrxlytjxwlgbnpguqynzrjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxerbnobeclgxqyejuttstifvtgqycor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvkpgrvazawmhdrpqontjehlozxmmqlm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vibfvgwwucgzrquannaiodwuyjontowz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcyzqtfhiqajfwqyyqozpkygkjexdktz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wdddekclrsqqllvjlnsasaankuzpkgkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlfcdnjayxubqralbcrzspcyabmoycix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvaustjfixvofiickumpalpstbiabtba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"arrluuauazyysjijinavkzccyksaqnqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlujnpsnxtpyzsosrekdhpuhuyiucnck","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqkhnswhlqbtmckqmtollargboffosde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qsiprozbglpkpnoumoiqosjsmjbseqzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvqugxpgdpfxzyabqnupvjeskmgranlf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsixrfiknrpvivbwydopyfdmezrsgpky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmuykrmpligvepzbyptafhcpmsgvezba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owctqamhbnijkalbvniumnabwkabkxuk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypyfwxscnotzuszoayjwhtapmjlbmmov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yujftkgplwkbdzerdqohuimfwmdxdpor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sioqvzwzhztkhrnoyhwkyssgtpnpozdp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"huorgyrpsritinmoqimcflaqfpqsqxip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elvvjkzalxboemfecokqdqycmqusqfsx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjkzcipmkkixzmbrwktoekxwwycmgusz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugfusnpvicvugdkroghivndkbzqjamsf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"neradnzbdhxcmmxrixcopxlgqttbkhch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjrrnhtmxhvtwanavkwxznbzxfpauzwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vlaseallxxmsspeqfckjypfzwmfsfhwd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfoxbtkevqwdmggszzkosanusmvjtlxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqndosdonwpulqtgqzncqcbzwygkqtme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skxcbrcirfmbnfdfqidkvldmleclvgqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"leskfwhtjfhdussmlwrkypcagycgazov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"viavotsoymhdkcdovouvovqmzlfyttva","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"szxmgsutuggjdltnqhnfeziizjajgxrn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"slykycaxhnrsiqpikyvxujhlzcucctns","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ownbauyretuynffmwpldomjdedivdwfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgapqioawxkdodgcbqinakfsbslrysyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzgpxjvtoofxnfxhffrizzekpiktbeeg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmlbcthsqrmwuhadkqgelwcxpaeaxufr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwpfrmlwzqhuqktuzklzrvwsncgcxbou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tplofdzqlxyvunswrnbyhbybvereamvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"silgrrdhqwswksibzfvjsqselhvqmoyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cicbauzpyjarqhtxeboezhggyshkfcyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvgcolltqhcfpwpnkhfhwprrqfartche","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azjsjwqgcqgtqaushjlkfajumqlrfizv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kyzxvrsxavptdvrvmlltjbvbdjvxeynv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ognuihpbehwmgxxygmxuehdwiujfwrhd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gooxhvwukkzudnvtlnnufihhdwnzeepl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tsvfsqqlfrboiaajdgfmymarvedzmyfw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zrjthshihkbrvifrugdtzvkbavypasbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ztxzyjekxemjfbxojnatrvjsxmmuomrs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qcmpoxfralwjllwwigsocayzrywxnvkl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sczgwakulyvbfbdmfoczkshtrqnffliq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqoovezhqwmuqwlbzuasbhbategnjyhs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybpowikjcrhtuowbgwrehrrrkxkteuha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwaiexbkaacotbuypoaewlriafwcxwsn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbntoubtiwvbjioookkchduaqdyoeyah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwbnuwljvxfxmeqjoopiqagmorwlnstp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"epzqdcggvbkqujkrwsrwecqyyvkvzcls","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezbrdepvvzmwpydeqaciudqvvdrrzhbs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jxfsejrmzdfmxaphrcyebvnxjnfbcjnw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnbiwijywftjoqfchnkdxbmbvgtmvsdv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ksdhbrimniwscnthivubvkokmmkdjwgv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bxhcvctlaxkhwuzcqgbukzgcxuavrquj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ztyqpurbqpsxqfldrvuzgitagkgqafea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqqidwjqptpoepluwoqocmdvyjgpekfq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icahorfnxtqbbvactlvavylrnqxfyrhc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvjuvmdxomfgnbwsujbdywsvdluecfcj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfjsgblywxucpzhfpwrytlnugbeiwanj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhlghttenmepqiuwraayffzcddzxfpgi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"casxcdnklzsifldongcgcmtsvokaqlrt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjtzpkcutylzbvorqgtgyfoxelrddfnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tklnkddaegqpuiteskshwvvbtratsqai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"huwcgwjgqzwblildhijojzxcbfacixpi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suyanjxubxkxwayadksgvlhxxxbvfvpt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wivwdmezvdxqwwsfcngdqgojorymybre","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hldxdapzskpjbhadiqbmtdbfckmvywld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uishwgosyrzzvgapfgnwlbocrnuoagie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vfsmnligxionqhmzlpvbmfijlzgutrqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezvxejvefezzcuezjiklomidhmagfglp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwljkkyzdjrxserovyquntmaszcogsdx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fofjoadlddksuseseyvglzbiwnblelso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668478,"databaseName":"models_schema","ddl":"CREATE TABLE `hwpjwafzowgrotlquywyqtrcabzussxr` (\n `rztunwpnxtsuerrfthvxaupcnimhewxs` int NOT NULL,\n `jworunjiwlincvcgdpfrmnruhgtadetm` int DEFAULT NULL,\n `vvrubxacbzccsaadqvxwkepitzzsykme` int DEFAULT NULL,\n `mbpooudzgcgokhsxokzwulbampbpppwh` int DEFAULT NULL,\n `klfhrnxkaxpbjthtixrtaavjyndgigxv` int DEFAULT NULL,\n `gzszxvoxpxgnyoxqkewfmmubbfyjtbbt` int DEFAULT NULL,\n `ompubwjarlqcmvbqjnkiycjvzsgmlmou` int DEFAULT NULL,\n `sheblwzvokjzrkfooskzaumsptuutwrq` int DEFAULT NULL,\n `rnpcjtvcybppsizeisbyhxmktudssmul` int DEFAULT NULL,\n `vvvmgmqqitnmcbixizrmtatxbtmnygpq` int DEFAULT NULL,\n `tnjucsjgvyfnickkwjaqenbkhujlrikw` int DEFAULT NULL,\n `txcgjbvwjhsespnozoqtyegrgevkxyri` int DEFAULT NULL,\n `mcuoeeaeqvxbtgqtnkchtkhuqtztlosd` int DEFAULT NULL,\n `zyeodclsorhyyqrzxbbsasmqfzueezox` int DEFAULT NULL,\n `dbjshvihnuxsuiqmdiffvfmwrhebqyoy` int DEFAULT NULL,\n `jjudtfhautgzkkgeiofmxtfzhpbtggzi` int DEFAULT NULL,\n `kdamgrukjglypvueeufhiytgvngastvm` int DEFAULT NULL,\n `hqxhvtrgueiiopnnauafzlkmohuwvytl` int DEFAULT NULL,\n `dpmclprrxhjraytqzzasqdvoxbjupkmz` int DEFAULT NULL,\n `ikzlvuqtqgogejeufkzdixrjswkqzdte` int DEFAULT NULL,\n `tsaklkpraenpburmgbtcybsbtudkqrsd` int DEFAULT NULL,\n `jiimwbiffkullssolrdrhbccvbxisxqg` int DEFAULT NULL,\n `bnvpssaepitymnimnrjzxdcqcqnavnjc` int DEFAULT NULL,\n `phnulxqafpmcfpjsinpmilhehccrzrjv` int DEFAULT NULL,\n `bkjxogckdpdqodoldfcjxsftpjfgewnv` int DEFAULT NULL,\n `wmetzixdhvgrtgyryovbfvlyjsiovdsz` int DEFAULT NULL,\n `rrpxyuoeddfckgjlyjuofqplmhkchkiq` int DEFAULT NULL,\n `dspkyewclbuawlyaycmzcmrscqnblccw` int DEFAULT NULL,\n `rsyrtcbnilucsicwklunjsidxohpvgmt` int DEFAULT NULL,\n `plqqnbvhpmybfduentrgtmmrazcwsdfy` int DEFAULT NULL,\n `ngthmccgsrbejpzefjgdddylemrfinld` int DEFAULT NULL,\n `gmysjynamftrdxokmphwiixksogxmkyp` int DEFAULT NULL,\n `bpdmkaosxznfqvtrvswubzkneepjckmv` int DEFAULT NULL,\n `sfabamasgpnhrmnvhwhdxauxwwafdyrh` int DEFAULT NULL,\n `sbwpzkqrkdhpacfinbjvhiubesgbyvss` int DEFAULT NULL,\n `csxtgpcfowlywpikofxrjmfstjclgmlf` int DEFAULT NULL,\n `hlvtjnjpskgjbtukxwyylopxpofzipsm` int DEFAULT NULL,\n `smvgcfwyrkvccsttyqrgxnkhivfdvfdx` int DEFAULT NULL,\n `neactvwtmywwijiodwjhjmrsqtwgkmck` int DEFAULT NULL,\n `ivyntochlzoqwmbzwkgxpmmgcqjledup` int DEFAULT NULL,\n `dqkewxcukcpxsyhsqbuzfpuhbocgkmgk` int DEFAULT NULL,\n `jmlbwfskshkxrrnfxvqlcrlkfmjghrvo` int DEFAULT NULL,\n `uznamavncgjcaxgzccxfqupbgspufhxy` int DEFAULT NULL,\n `winavcivpfywwmiyxsdmfhgsqzyoqhve` int DEFAULT NULL,\n `aqrbesefoysgcqhgcmkubzthwnvlpsnj` int DEFAULT NULL,\n `tzkhipgauvayeczmhylhbgfelcxlsdfr` int DEFAULT NULL,\n `uzpmxhufxfiwpdkscwytwhebjcmgcfmb` int DEFAULT NULL,\n `dlnsiatlhovhzacsqnmnszyypwbiffgb` int DEFAULT NULL,\n `zofzsgzsnwmdqmxvayucfjbqitokyzbe` int DEFAULT NULL,\n `njggztdzvzdcxtywzvbhbuebqjashnzp` int DEFAULT NULL,\n `epwwpckbdvrlxolhfrfaeuzhjegmgdkq` int DEFAULT NULL,\n `mmowkxcdgnyjambpexfpolvfjtyguijf` int DEFAULT NULL,\n `vyjcxbthqpibfvbtfrvwcyyzdqcgntka` int DEFAULT NULL,\n `oaxaodauaseqhxsiezeqltmukamgfryk` int DEFAULT NULL,\n `iwzssijfttwgiahefmrtdbrufnjrhjko` int DEFAULT NULL,\n `pmnehpfrhnjknbpjdbfdjvwlmbijbapy` int DEFAULT NULL,\n `yrfxchiqlwpmhtdmrkqddtlpxhgjwpxz` int DEFAULT NULL,\n `grikufnkojvmlibleqxvfawocsjdnniw` int DEFAULT NULL,\n `octpndwwqvjitnzwhotunymcjzycafzg` int DEFAULT NULL,\n `ehwzctvrfoecfhjfrtmpxwmthwcfvnen` int DEFAULT NULL,\n `teoxazltbsfdqzljmaxiqokqyphqtdkv` int DEFAULT NULL,\n `jhqjllmeggmsmqkhmqdskytekbzttlmk` int DEFAULT NULL,\n `uasogjlcbdfubfqxxjgbasxbufjzbgzj` int DEFAULT NULL,\n `znjxzqdxykhzyqkqixqpfubtkwuqufad` int DEFAULT NULL,\n `wazdeolbrjoxrypwhhkjtbrdaqtonzfg` int DEFAULT NULL,\n `arnewjklfobodechsskndcswamwzbfgt` int DEFAULT NULL,\n `wpofvmexpbmbfvkobgbsrukouyyaxgxj` int DEFAULT NULL,\n `wipykpmtkmharkjwnzayuqqvjwrdspds` int DEFAULT NULL,\n `yumbqnuyvdhvzeqiddurktcofljdjeik` int DEFAULT NULL,\n `etkyrawjrlrorxrtgzzcnghivzhbrvco` int DEFAULT NULL,\n `dceryjunvhfirtycldkgsqjkikbwrsxq` int DEFAULT NULL,\n `hipkrtjnkkwmnuxpuhvlpsxmmgusumal` int DEFAULT NULL,\n `fcjznsxgtkatkddxkgnumecaunvdxfzn` int DEFAULT NULL,\n `gbtpstchzwjporqccaqarkpifntpyanm` int DEFAULT NULL,\n `piwyajekwoatidwfugnmtvsiczljzufo` int DEFAULT NULL,\n `syrjgbvlcsbolzxhmfnrwbdrxqdzxbev` int DEFAULT NULL,\n `ctaxdqiayravmanczannpzahnhidmknz` int DEFAULT NULL,\n `tgmyeqocpkuqjljqcecppxgpkgokdytx` int DEFAULT NULL,\n `pusomfbjmvpabpyeehovmujnhlkvqthb` int DEFAULT NULL,\n `eqwkaclurhuyvufhdxclfypldagpgqky` int DEFAULT NULL,\n `bfxkwnexeghyhodnbjjumdkftmvdihtg` int DEFAULT NULL,\n `okubxdtelwclfzqydemhxfdtcrcxmreg` int DEFAULT NULL,\n `tszgoyqxdyxnldixzexxkftqsrsfrjvo` int DEFAULT NULL,\n `spldcvclpyeyyjovkikifaogtctqykdj` int DEFAULT NULL,\n `teqbaeckuvskqvrqgfzkawhtedikyqwv` int DEFAULT NULL,\n `zvryzkkhsipnbbkptfrfyospkwipmlyi` int DEFAULT NULL,\n `wwyrgyburhilertdcbvkpvfqddfqzssi` int DEFAULT NULL,\n `dthguqcwdotbckdotktdvbvbniolvxjg` int DEFAULT NULL,\n `bjtjuyxhanxxbetuburwkkcbxsxzazqj` int DEFAULT NULL,\n `jzqnqmlhytiyxiefmokvgvyjodkfyggo` int DEFAULT NULL,\n `grtuujeqeraocbbhzthfggjzzneqkqsz` int DEFAULT NULL,\n `wzdpauhnrnlunzpklnphjdajvmnxeqwo` int DEFAULT NULL,\n `osjtmiczsshdakykagnetsfqfnoeoowj` int DEFAULT NULL,\n `hsfavvtjrywelzmevndlbqdcsflextks` int DEFAULT NULL,\n `wjoxdqvxkaztsxxzaazdipkfyhswpvje` int DEFAULT NULL,\n `mijuypllildwwofttjpkukeickxmdzdv` int DEFAULT NULL,\n `mavzqycermggfjveiixwfugncrgbtnab` int DEFAULT NULL,\n `qeubnjxzsppledkvqjhpmnpeavuduzrb` int DEFAULT NULL,\n `tnyserjltsxykkragukzagvxwikuqwva` int DEFAULT NULL,\n `xxtkeohokwyimmpyhesqgdaxjipeuqty` int DEFAULT NULL,\n PRIMARY KEY (`rztunwpnxtsuerrfthvxaupcnimhewxs`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"hwpjwafzowgrotlquywyqtrcabzussxr\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["rztunwpnxtsuerrfthvxaupcnimhewxs"],"columns":[{"name":"rztunwpnxtsuerrfthvxaupcnimhewxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"jworunjiwlincvcgdpfrmnruhgtadetm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vvrubxacbzccsaadqvxwkepitzzsykme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbpooudzgcgokhsxokzwulbampbpppwh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"klfhrnxkaxpbjthtixrtaavjyndgigxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzszxvoxpxgnyoxqkewfmmubbfyjtbbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ompubwjarlqcmvbqjnkiycjvzsgmlmou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sheblwzvokjzrkfooskzaumsptuutwrq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnpcjtvcybppsizeisbyhxmktudssmul","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vvvmgmqqitnmcbixizrmtatxbtmnygpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnjucsjgvyfnickkwjaqenbkhujlrikw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txcgjbvwjhsespnozoqtyegrgevkxyri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcuoeeaeqvxbtgqtnkchtkhuqtztlosd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyeodclsorhyyqrzxbbsasmqfzueezox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbjshvihnuxsuiqmdiffvfmwrhebqyoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjudtfhautgzkkgeiofmxtfzhpbtggzi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kdamgrukjglypvueeufhiytgvngastvm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqxhvtrgueiiopnnauafzlkmohuwvytl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpmclprrxhjraytqzzasqdvoxbjupkmz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikzlvuqtqgogejeufkzdixrjswkqzdte","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tsaklkpraenpburmgbtcybsbtudkqrsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jiimwbiffkullssolrdrhbccvbxisxqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnvpssaepitymnimnrjzxdcqcqnavnjc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phnulxqafpmcfpjsinpmilhehccrzrjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkjxogckdpdqodoldfcjxsftpjfgewnv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmetzixdhvgrtgyryovbfvlyjsiovdsz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrpxyuoeddfckgjlyjuofqplmhkchkiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dspkyewclbuawlyaycmzcmrscqnblccw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rsyrtcbnilucsicwklunjsidxohpvgmt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"plqqnbvhpmybfduentrgtmmrazcwsdfy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ngthmccgsrbejpzefjgdddylemrfinld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmysjynamftrdxokmphwiixksogxmkyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bpdmkaosxznfqvtrvswubzkneepjckmv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfabamasgpnhrmnvhwhdxauxwwafdyrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbwpzkqrkdhpacfinbjvhiubesgbyvss","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"csxtgpcfowlywpikofxrjmfstjclgmlf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlvtjnjpskgjbtukxwyylopxpofzipsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smvgcfwyrkvccsttyqrgxnkhivfdvfdx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"neactvwtmywwijiodwjhjmrsqtwgkmck","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivyntochlzoqwmbzwkgxpmmgcqjledup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqkewxcukcpxsyhsqbuzfpuhbocgkmgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmlbwfskshkxrrnfxvqlcrlkfmjghrvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uznamavncgjcaxgzccxfqupbgspufhxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"winavcivpfywwmiyxsdmfhgsqzyoqhve","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqrbesefoysgcqhgcmkubzthwnvlpsnj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tzkhipgauvayeczmhylhbgfelcxlsdfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzpmxhufxfiwpdkscwytwhebjcmgcfmb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlnsiatlhovhzacsqnmnszyypwbiffgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zofzsgzsnwmdqmxvayucfjbqitokyzbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njggztdzvzdcxtywzvbhbuebqjashnzp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"epwwpckbdvrlxolhfrfaeuzhjegmgdkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmowkxcdgnyjambpexfpolvfjtyguijf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vyjcxbthqpibfvbtfrvwcyyzdqcgntka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oaxaodauaseqhxsiezeqltmukamgfryk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwzssijfttwgiahefmrtdbrufnjrhjko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmnehpfrhnjknbpjdbfdjvwlmbijbapy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yrfxchiqlwpmhtdmrkqddtlpxhgjwpxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grikufnkojvmlibleqxvfawocsjdnniw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"octpndwwqvjitnzwhotunymcjzycafzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehwzctvrfoecfhjfrtmpxwmthwcfvnen","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"teoxazltbsfdqzljmaxiqokqyphqtdkv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jhqjllmeggmsmqkhmqdskytekbzttlmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uasogjlcbdfubfqxxjgbasxbufjzbgzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znjxzqdxykhzyqkqixqpfubtkwuqufad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wazdeolbrjoxrypwhhkjtbrdaqtonzfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"arnewjklfobodechsskndcswamwzbfgt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wpofvmexpbmbfvkobgbsrukouyyaxgxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wipykpmtkmharkjwnzayuqqvjwrdspds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yumbqnuyvdhvzeqiddurktcofljdjeik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"etkyrawjrlrorxrtgzzcnghivzhbrvco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dceryjunvhfirtycldkgsqjkikbwrsxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hipkrtjnkkwmnuxpuhvlpsxmmgusumal","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcjznsxgtkatkddxkgnumecaunvdxfzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gbtpstchzwjporqccaqarkpifntpyanm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"piwyajekwoatidwfugnmtvsiczljzufo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"syrjgbvlcsbolzxhmfnrwbdrxqdzxbev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctaxdqiayravmanczannpzahnhidmknz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tgmyeqocpkuqjljqcecppxgpkgokdytx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pusomfbjmvpabpyeehovmujnhlkvqthb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqwkaclurhuyvufhdxclfypldagpgqky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bfxkwnexeghyhodnbjjumdkftmvdihtg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okubxdtelwclfzqydemhxfdtcrcxmreg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tszgoyqxdyxnldixzexxkftqsrsfrjvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"spldcvclpyeyyjovkikifaogtctqykdj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"teqbaeckuvskqvrqgfzkawhtedikyqwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvryzkkhsipnbbkptfrfyospkwipmlyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwyrgyburhilertdcbvkpvfqddfqzssi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dthguqcwdotbckdotktdvbvbniolvxjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjtjuyxhanxxbetuburwkkcbxsxzazqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzqnqmlhytiyxiefmokvgvyjodkfyggo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grtuujeqeraocbbhzthfggjzzneqkqsz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzdpauhnrnlunzpklnphjdajvmnxeqwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"osjtmiczsshdakykagnetsfqfnoeoowj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hsfavvtjrywelzmevndlbqdcsflextks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjoxdqvxkaztsxxzaazdipkfyhswpvje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mijuypllildwwofttjpkukeickxmdzdv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mavzqycermggfjveiixwfugncrgbtnab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qeubnjxzsppledkvqjhpmnpeavuduzrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnyserjltsxykkragukzagvxwikuqwva","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxtkeohokwyimmpyhesqgdaxjipeuqty","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668511,"databaseName":"models_schema","ddl":"CREATE TABLE `hyijqphvtvgdgegsnseyzeyooughktwe` (\n `hwrbeiacfojtvaiodicdkvrtloturcqd` int NOT NULL,\n `scdssplhfymiwzrpklklcxcgoknlvfyh` int DEFAULT NULL,\n `qfavcytewpzqzakgnvsyxrvzwcsldsww` int DEFAULT NULL,\n `djyfezprdqjvdfvryxavsqqlzivuzjjw` int DEFAULT NULL,\n `mtdvslbfxspbeerwsgizwdjonrygmilq` int DEFAULT NULL,\n `kerevgffwjflfhwawomyzbxismvzbuao` int DEFAULT NULL,\n `pijcwneopqtqlbpgztlqvxatilxvjalu` int DEFAULT NULL,\n `ljcnsrlagztqquzbeldwwhygpzlujoel` int DEFAULT NULL,\n `tjsbsjyopsizyxjzudnqpwlapmxfinev` int DEFAULT NULL,\n `smpczoojdzgumexgtlxovhtnoofzyubh` int DEFAULT NULL,\n `bkjcnmuymqrbpmrkvhkvucyufidehcpb` int DEFAULT NULL,\n `plasdypxeevfwmuofgpadfjpoyucepvy` int DEFAULT NULL,\n `xanfqqvhethtyolvdmmqeifokavlyjgm` int DEFAULT NULL,\n `dttljbdnhzrtsmdnwsckudhsxybnwxxy` int DEFAULT NULL,\n `bgpyyjhqtptrrocpazfjudfyaunjjpfq` int DEFAULT NULL,\n `ufqbdwtadgihvfyydgsccmpkekzabhyi` int DEFAULT NULL,\n `ljnajpbhwtpppbnrvzksosleaviwstkz` int DEFAULT NULL,\n `frfalhzrctlnrccfolxlmsiwoantpgwh` int DEFAULT NULL,\n `rdleowhpuhdzovrfsywdfwjodggolxwv` int DEFAULT NULL,\n `cywmascrncerefdqrfdnvodjwwozzoxr` int DEFAULT NULL,\n `jtcbqzqvleylfknytmqzsmulxtvvupeh` int DEFAULT NULL,\n `gpeqzrxkomnrfjwtuxvorlasrqygkqqn` int DEFAULT NULL,\n `pylpubcjannrieyvldqlkucuriaxesxb` int DEFAULT NULL,\n `obtnpwwwczdruxksvajvckibfbisdfow` int DEFAULT NULL,\n `namtadsaqrzzkjbkmepqhorxerackqkv` int DEFAULT NULL,\n `uwidehqncazffznlhfagctsoheoodhue` int DEFAULT NULL,\n `ndtmnedddpdcyzvegpdvedyiuefwudfu` int DEFAULT NULL,\n `quyxnclxfvpozwolnvspgskolzjsthaa` int DEFAULT NULL,\n `gjmaviodpuhlnimldasxfawpqfrqbxlu` int DEFAULT NULL,\n `xfqwkrlbpoxxlzsdqlqipgalvfwwxenv` int DEFAULT NULL,\n `jcloakskluxqbxksgbgsnwstaeqgoarx` int DEFAULT NULL,\n `xwljqbnaglkzynxbrwncjloafbyhaifn` int DEFAULT NULL,\n `ochpnsmyidvqriaupdlwahlbmpwgipqm` int DEFAULT NULL,\n `mpozuktvwtfqoivonvufgwzbfrictypp` int DEFAULT NULL,\n `adhhvamofjdfcxlhtkyjgtlhxsqrryiw` int DEFAULT NULL,\n `aubsqmcvqpvyiqhhqgtnatclhtnksczu` int DEFAULT NULL,\n `ueohgxpesawawylllkwryokeqavcedza` int DEFAULT NULL,\n `tbjzqelttxrghvsyifyywdcfxhxxeuqr` int DEFAULT NULL,\n `keoeklmcpflnfukdeuwrdwlbjeqynjcy` int DEFAULT NULL,\n `zpmyzvajkbmagvzyedarqkmnwuwfmlit` int DEFAULT NULL,\n `dljjwpbukhncaxltwiwszwwhihnfqfku` int DEFAULT NULL,\n `cyvdnzchftkjotyezxtprrijyroqbtuc` int DEFAULT NULL,\n `ncekgpcqskzxlmxgrroqwvlncrghabbr` int DEFAULT NULL,\n `pwxbrtfjwkmxclehqpuzikzstmsapbsj` int DEFAULT NULL,\n `itfbulbidchtdqgtasfglswilwshiimg` int DEFAULT NULL,\n `tomvilhduzulgahmkraszymehcvqmurh` int DEFAULT NULL,\n `cercrhnafizvvkznefbgtqrorfvcvuka` int DEFAULT NULL,\n `kspvettrlgntgezzcpkwrcoxfjojwglm` int DEFAULT NULL,\n `dcauzzkjugreddsqtfptpfeehvxoqhec` int DEFAULT NULL,\n `euoangmtociugjxktizapdkhbentwlfr` int DEFAULT NULL,\n `aobomcmxvfpbsqbkvbnlehxjulcgieiu` int DEFAULT NULL,\n `mscccdgyybszwvjbibbbjdzmxzblmnhu` int DEFAULT NULL,\n `wqbbdeqicliytxebjvnizshmnvdyyhtj` int DEFAULT NULL,\n `evodyngoanvtpdgtkccoztcxyrqbndrr` int DEFAULT NULL,\n `dsmxadaerpdmhsqvvvavqyujjeycnljq` int DEFAULT NULL,\n `qhbvrhpgffifqzjxxdmmpijqfassjkkv` int DEFAULT NULL,\n `qvkwzjznxmpzffnxzdxmubdaywycawlv` int DEFAULT NULL,\n `wxhichsfccbpmrnvnagnbnrpenocxcaz` int DEFAULT NULL,\n `vhunikekrethxzsnbmyawstqldmlyeju` int DEFAULT NULL,\n `njjwydimfitnsovriudxdefvucjnorzp` int DEFAULT NULL,\n `xbloocplbsooikdjhnszjqsedzmvakcf` int DEFAULT NULL,\n `nxgeqlcydhmiccigswpbvbpntkgwtydu` int DEFAULT NULL,\n `ifzeeghdohnsbhhgwlhcalyrdwfzbaln` int DEFAULT NULL,\n `uidigvkmnjmgibekgrvrmilovedsaemy` int DEFAULT NULL,\n `piflkkctjanlhkegaxjzcmaxxxmonjvz` int DEFAULT NULL,\n `gnlgicvaprzroughtrvpoblddqponafd` int DEFAULT NULL,\n `pnerfzexsrfaqotqafjienlhuruzhlpr` int DEFAULT NULL,\n `aojzrbfvpfglcevtkkasnojkqutwoumj` int DEFAULT NULL,\n `ahxvrqunrirovxwnzvckauchsbgiybzp` int DEFAULT NULL,\n `ggtyxgnwldpusitkbwdcosfjgazbnzna` int DEFAULT NULL,\n `llboedpahjxxkngrlneryjjajsjbzfey` int DEFAULT NULL,\n `kmmmwvcctuwhacqrmxmktpzllyxxhdxd` int DEFAULT NULL,\n `mvhtogkqyiftutdswotpyavxriqngnui` int DEFAULT NULL,\n `djqjuqmujabxdsjuxjqhvcbxklxtlysb` int DEFAULT NULL,\n `nukqotxzleveygwxilqfcxsvbnocmmdh` int DEFAULT NULL,\n `alroiljqujhjeyrwdxemqdgnjgbzlnzs` int DEFAULT NULL,\n `ojcyhsvxsaukiovrolvowagazkzxapxg` int DEFAULT NULL,\n `nwwqzcpjsidqyougngtjejvrfendgbzs` int DEFAULT NULL,\n `hobnbjvzrsomwxrjbxyvbasthikbpxum` int DEFAULT NULL,\n `ilgdswcyztjnxftsdewffixtyrwarlhv` int DEFAULT NULL,\n `ntyaulharotanleqnooqnnpusgkgslgv` int DEFAULT NULL,\n `nzetnphazksnppwmxdikzcpssmyfpxep` int DEFAULT NULL,\n `dzjsjdscmoieiywuihluzbgsdtmzwnaj` int DEFAULT NULL,\n `fiuptkqnblizymyctahkgecnerenkgou` int DEFAULT NULL,\n `ztthncaqzxneydmfkpxboaggoboxmqqv` int DEFAULT NULL,\n `switwcsjjzpngqwyposniskkgloftzgx` int DEFAULT NULL,\n `hjnnkhcpxsmuisdnpoibgvcxtyoeiqvr` int DEFAULT NULL,\n `uucjphzxwwuanjlnfasogssktsecpexr` int DEFAULT NULL,\n `xxwbckbxpdjroztcbhynxvzrradlvpec` int DEFAULT NULL,\n `xaqfsljivpscfargvlnekantugcjhnyj` int DEFAULT NULL,\n `vrkxchcrvrtcjjqrwyktenpqetddqmcw` int DEFAULT NULL,\n `ipnapxheperwcdggnfysjbsrnkynefgm` int DEFAULT NULL,\n `xpssuhhbopcdiryztbzlnnxiayjdqvbi` int DEFAULT NULL,\n `rwmnubsmeuqeqrjlwrkrshoupxnaemlk` int DEFAULT NULL,\n `jidakwmhsthgsknzjkhvsuscjzhbacew` int DEFAULT NULL,\n `qajsrwipxsyvmzezqtchvgtjtukibzbf` int DEFAULT NULL,\n `wxonyuxarjypgyfhhedwwgxjvldbxcjb` int DEFAULT NULL,\n `oilkijlpfftapxmzcllrnpiflvxoapix` int DEFAULT NULL,\n `knsryamdtcckhkbjsqqmfufdfmbrjnbs` int DEFAULT NULL,\n `nhvlpbknuftocgfrmljajgxaecifqqpq` int DEFAULT NULL,\n PRIMARY KEY (`hwrbeiacfojtvaiodicdkvrtloturcqd`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"hyijqphvtvgdgegsnseyzeyooughktwe\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["hwrbeiacfojtvaiodicdkvrtloturcqd"],"columns":[{"name":"hwrbeiacfojtvaiodicdkvrtloturcqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"scdssplhfymiwzrpklklcxcgoknlvfyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfavcytewpzqzakgnvsyxrvzwcsldsww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"djyfezprdqjvdfvryxavsqqlzivuzjjw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtdvslbfxspbeerwsgizwdjonrygmilq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kerevgffwjflfhwawomyzbxismvzbuao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pijcwneopqtqlbpgztlqvxatilxvjalu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljcnsrlagztqquzbeldwwhygpzlujoel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjsbsjyopsizyxjzudnqpwlapmxfinev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smpczoojdzgumexgtlxovhtnoofzyubh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkjcnmuymqrbpmrkvhkvucyufidehcpb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"plasdypxeevfwmuofgpadfjpoyucepvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xanfqqvhethtyolvdmmqeifokavlyjgm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dttljbdnhzrtsmdnwsckudhsxybnwxxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bgpyyjhqtptrrocpazfjudfyaunjjpfq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufqbdwtadgihvfyydgsccmpkekzabhyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljnajpbhwtpppbnrvzksosleaviwstkz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"frfalhzrctlnrccfolxlmsiwoantpgwh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdleowhpuhdzovrfsywdfwjodggolxwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cywmascrncerefdqrfdnvodjwwozzoxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jtcbqzqvleylfknytmqzsmulxtvvupeh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gpeqzrxkomnrfjwtuxvorlasrqygkqqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pylpubcjannrieyvldqlkucuriaxesxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obtnpwwwczdruxksvajvckibfbisdfow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"namtadsaqrzzkjbkmepqhorxerackqkv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwidehqncazffznlhfagctsoheoodhue","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ndtmnedddpdcyzvegpdvedyiuefwudfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"quyxnclxfvpozwolnvspgskolzjsthaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjmaviodpuhlnimldasxfawpqfrqbxlu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xfqwkrlbpoxxlzsdqlqipgalvfwwxenv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jcloakskluxqbxksgbgsnwstaeqgoarx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xwljqbnaglkzynxbrwncjloafbyhaifn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ochpnsmyidvqriaupdlwahlbmpwgipqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpozuktvwtfqoivonvufgwzbfrictypp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"adhhvamofjdfcxlhtkyjgtlhxsqrryiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aubsqmcvqpvyiqhhqgtnatclhtnksczu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ueohgxpesawawylllkwryokeqavcedza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbjzqelttxrghvsyifyywdcfxhxxeuqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"keoeklmcpflnfukdeuwrdwlbjeqynjcy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpmyzvajkbmagvzyedarqkmnwuwfmlit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dljjwpbukhncaxltwiwszwwhihnfqfku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cyvdnzchftkjotyezxtprrijyroqbtuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ncekgpcqskzxlmxgrroqwvlncrghabbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwxbrtfjwkmxclehqpuzikzstmsapbsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itfbulbidchtdqgtasfglswilwshiimg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tomvilhduzulgahmkraszymehcvqmurh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cercrhnafizvvkznefbgtqrorfvcvuka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kspvettrlgntgezzcpkwrcoxfjojwglm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcauzzkjugreddsqtfptpfeehvxoqhec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"euoangmtociugjxktizapdkhbentwlfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aobomcmxvfpbsqbkvbnlehxjulcgieiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mscccdgyybszwvjbibbbjdzmxzblmnhu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqbbdeqicliytxebjvnizshmnvdyyhtj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evodyngoanvtpdgtkccoztcxyrqbndrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsmxadaerpdmhsqvvvavqyujjeycnljq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhbvrhpgffifqzjxxdmmpijqfassjkkv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvkwzjznxmpzffnxzdxmubdaywycawlv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxhichsfccbpmrnvnagnbnrpenocxcaz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhunikekrethxzsnbmyawstqldmlyeju","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njjwydimfitnsovriudxdefvucjnorzp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbloocplbsooikdjhnszjqsedzmvakcf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxgeqlcydhmiccigswpbvbpntkgwtydu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifzeeghdohnsbhhgwlhcalyrdwfzbaln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uidigvkmnjmgibekgrvrmilovedsaemy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"piflkkctjanlhkegaxjzcmaxxxmonjvz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnlgicvaprzroughtrvpoblddqponafd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnerfzexsrfaqotqafjienlhuruzhlpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aojzrbfvpfglcevtkkasnojkqutwoumj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahxvrqunrirovxwnzvckauchsbgiybzp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggtyxgnwldpusitkbwdcosfjgazbnzna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llboedpahjxxkngrlneryjjajsjbzfey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmmmwvcctuwhacqrmxmktpzllyxxhdxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvhtogkqyiftutdswotpyavxriqngnui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"djqjuqmujabxdsjuxjqhvcbxklxtlysb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nukqotxzleveygwxilqfcxsvbnocmmdh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"alroiljqujhjeyrwdxemqdgnjgbzlnzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojcyhsvxsaukiovrolvowagazkzxapxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nwwqzcpjsidqyougngtjejvrfendgbzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hobnbjvzrsomwxrjbxyvbasthikbpxum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ilgdswcyztjnxftsdewffixtyrwarlhv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntyaulharotanleqnooqnnpusgkgslgv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzetnphazksnppwmxdikzcpssmyfpxep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dzjsjdscmoieiywuihluzbgsdtmzwnaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fiuptkqnblizymyctahkgecnerenkgou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ztthncaqzxneydmfkpxboaggoboxmqqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"switwcsjjzpngqwyposniskkgloftzgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjnnkhcpxsmuisdnpoibgvcxtyoeiqvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uucjphzxwwuanjlnfasogssktsecpexr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxwbckbxpdjroztcbhynxvzrradlvpec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xaqfsljivpscfargvlnekantugcjhnyj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrkxchcrvrtcjjqrwyktenpqetddqmcw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ipnapxheperwcdggnfysjbsrnkynefgm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpssuhhbopcdiryztbzlnnxiayjdqvbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rwmnubsmeuqeqrjlwrkrshoupxnaemlk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jidakwmhsthgsknzjkhvsuscjzhbacew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qajsrwipxsyvmzezqtchvgtjtukibzbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxonyuxarjypgyfhhedwwgxjvldbxcjb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oilkijlpfftapxmzcllrnpiflvxoapix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knsryamdtcckhkbjsqqmfufdfmbrjnbs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhvlpbknuftocgfrmljajgxaecifqqpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668543,"databaseName":"models_schema","ddl":"CREATE TABLE `ifbeclftxswghnufrctevakgzhdsmlko` (\n `hxpmitvzzpzrvxynbckrserhqsqraxne` int NOT NULL,\n `vtxsyqjsvdhpbztvclvpwgtazkujimyz` int DEFAULT NULL,\n `prtrhlfydwhhvpozjpnmtusbzjwlwljl` int DEFAULT NULL,\n `fcwtcscskdftymfprvanpazbjghtzbpw` int DEFAULT NULL,\n `buaxlhyrtmrxmvwwuesmzbvlxiwnwctb` int DEFAULT NULL,\n `xqfogsbvuvlwrdgxyplhrbizmzervpmz` int DEFAULT NULL,\n `keratpwlnumrivlcivrihlvhejaftkqm` int DEFAULT NULL,\n `eeqomzcfogzyephwpdgltjwqsexwjumo` int DEFAULT NULL,\n `sjwerpccdnpykavjkzhmbpwqmoomsale` int DEFAULT NULL,\n `dhcbmcutefioocymsyusyflawbbipkxm` int DEFAULT NULL,\n `sbttpcikdqlyzhcehbbgqdxbeqecbrrf` int DEFAULT NULL,\n `elrvjfsjnaclefakqjwfovcjulumkqtd` int DEFAULT NULL,\n `lprctxfpntjqjcdpigmtaecgefbkzuxt` int DEFAULT NULL,\n `qtshtvpeircwsqdeipgdasxrhofupgoa` int DEFAULT NULL,\n `uljsjbjnilkhcqiqydijawqhvrrckyso` int DEFAULT NULL,\n `gyyewjgryqdponchtyenbbhgsjrweeqh` int DEFAULT NULL,\n `zydkaxrsmybycbewhpcfvccamwjxdugv` int DEFAULT NULL,\n `opztuqjndkhqhgnzhbylliynytgjyhko` int DEFAULT NULL,\n `fbltudlpsrwfkhzfrshqvfarfiyqfyjj` int DEFAULT NULL,\n `qmylyqiejdijnotpxcxkfxjiyvfvsewk` int DEFAULT NULL,\n `qlgwqoszgztcthgmtsqotxjipoqensrk` int DEFAULT NULL,\n `skewjgulsscyglmjcilkndfzkdkchlmc` int DEFAULT NULL,\n `kibdxwoetkumasylgbbkwsucmrmubisr` int DEFAULT NULL,\n `igwgekkaoahvqxhbhvjadnvwuaoewdcd` int DEFAULT NULL,\n `fokratrgntskelomielhnrzhxmmlpgdm` int DEFAULT NULL,\n `ykflkdjzuixrcqrwhkvubylmmjzsuglr` int DEFAULT NULL,\n `oyyokodrdawwmzglxxulhapntqvvmgyu` int DEFAULT NULL,\n `udmymochdfdrlqcjfkwvzonhsybovnbc` int DEFAULT NULL,\n `qyiddxmtfiqqkyowteefugvumivbefco` int DEFAULT NULL,\n `hbslpqjnltkikqyeuldznwfebpipxvbv` int DEFAULT NULL,\n `jkmewjizgquqdrjnufwceyeylbhehngo` int DEFAULT NULL,\n `odvvsgocqshwtmmxvqnmghehjfdlcgeb` int DEFAULT NULL,\n `ekvtlqzumlzejmxibfrruuloxcsjhitb` int DEFAULT NULL,\n `bioluaghgtvnhqxqnyyvcloosmqnxqnu` int DEFAULT NULL,\n `hvslwfbhxswhvnjxeldtasrxpetwrhfo` int DEFAULT NULL,\n `alwgigngksokqdkugoylfjpwzjfkrioj` int DEFAULT NULL,\n `ntemjtavmlomvrslxzmxoxwlpwtrdkua` int DEFAULT NULL,\n `uosewpojkfrvzozytbgtcvvjdxtudtmj` int DEFAULT NULL,\n `ndcdolhxekbnnthanhjksdsghndmddgk` int DEFAULT NULL,\n `xgggbssgjeawaarzqlappavdjfregatg` int DEFAULT NULL,\n `munizthkozgvzqzkjcsjeredgvoowthn` int DEFAULT NULL,\n `cyuxkwubevleqptfoiphkgjuplobivuu` int DEFAULT NULL,\n `xrgnsqcmsyetufwuzcnyzkespeawntyg` int DEFAULT NULL,\n `leouovocancqvkrqetohjtwwgssqxvnj` int DEFAULT NULL,\n `ppdelcsoztddkjdqaekragubyblbxxvn` int DEFAULT NULL,\n `yqylapdjwhxeproqgmxmklzenzcuvqmy` int DEFAULT NULL,\n `zfmmsbnsuyjnjxfjqfznetcyxkjcinuk` int DEFAULT NULL,\n `sxvteykwtotsaypobaztckdnvyhjcymt` int DEFAULT NULL,\n `obpltbosguppfrqkgricosjpmmymbelw` int DEFAULT NULL,\n `xunmyoquonhqvscazwgqxuaqelppblni` int DEFAULT NULL,\n `fqxegiwtufwooexmbrrtxyblconwnazz` int DEFAULT NULL,\n `aldurwtdlrjezzfkxgxytpwerleohsmt` int DEFAULT NULL,\n `eyzaqnylhjpljaytwfdltaqphqhdebgt` int DEFAULT NULL,\n `ksmqsjfohbqlkfjuxuxihperuawhkrpk` int DEFAULT NULL,\n `dvkvvdpzkogvlwleoniwaskfyzybaqtv` int DEFAULT NULL,\n `mgxlwczxpzruvfeadjqmlrfomjlggvnr` int DEFAULT NULL,\n `qdlxaafmlwzinxkpjsbeybgcwhtebmma` int DEFAULT NULL,\n `ueddugwgbcbbsuxwfwstkmctxirtvxyx` int DEFAULT NULL,\n `jlvzwgwgomgdppgqikzldlbrpqqftxbp` int DEFAULT NULL,\n `oavakmdrmhwsmbtumjzgpdhknrydaxsm` int DEFAULT NULL,\n `qkrmycpxdfjlxrhlvfvwkvwnhotypfwv` int DEFAULT NULL,\n `toayveyrmmggussfjilxioldnurkqljq` int DEFAULT NULL,\n `buggdzldklzfmhvmthrpqsgskavxbles` int DEFAULT NULL,\n `frnluvrlezfgvkgrnlvwvddseulghwkf` int DEFAULT NULL,\n `cutiwjofdbacfmzmwgcacficncnvyshm` int DEFAULT NULL,\n `qscimjetrtvqhrhhpboiqgyiarovnzfm` int DEFAULT NULL,\n `sawbzbonpkvkaxcyacgsqtqxjpewuddk` int DEFAULT NULL,\n `tvklmetonywcnuqudzmgcpdfentmdugs` int DEFAULT NULL,\n `mnjqqstbqtxyevyiczmynbcytphxejgf` int DEFAULT NULL,\n `jtsevwkdssfgnlwndkofnbvhlirzhckg` int DEFAULT NULL,\n `jykyjuoelwfkqaehyghrbvnxwtpzhoto` int DEFAULT NULL,\n `zzxpmafxhtckezrustleelphvygzeuwq` int DEFAULT NULL,\n `ezalqitaevsydcszhnofjcmnrzbmawom` int DEFAULT NULL,\n `sjkasfqxhkodmlxwhnghptioiztvofut` int DEFAULT NULL,\n `rfdtktfndnhorjkxvxicoxinjefzsqmy` int DEFAULT NULL,\n `dtkrgriqyxwhivmofkoqvodyibtjxpkg` int DEFAULT NULL,\n `uozjvlhcnyidwnjdfvrjqfdmsutxeppf` int DEFAULT NULL,\n `hqglkuhnhumbwbjafbhzxkcvjyzbrbrw` int DEFAULT NULL,\n `byvsfcykgmapflzsexbatuuknqrtaflf` int DEFAULT NULL,\n `sfyxcalqzhrinfloylbbrhtphbciqrjm` int DEFAULT NULL,\n `wvjplmuhcnriycpjbhseoqtdqxyqmqlz` int DEFAULT NULL,\n `sfnhksjvcdnvjeuvkmsqgvevmynrghgc` int DEFAULT NULL,\n `edfyomubizlgicvkpitgddvwvitrknwq` int DEFAULT NULL,\n `tzxnwmzklqoisvvobexnwyonqxwvggbk` int DEFAULT NULL,\n `gnmoltulkucnicfixfxrakedvugjmcza` int DEFAULT NULL,\n `uqvxrfogzxqpeeaxebdytxtyqkitcnxp` int DEFAULT NULL,\n `oauzokqrwnouylrvtlhwhpirmfqzxbsd` int DEFAULT NULL,\n `xydcsqdhpdxmdamxatquincqzybhafvb` int DEFAULT NULL,\n `vadajpjedltytakpuumejgoyhfnqfvdx` int DEFAULT NULL,\n `erlfbvcwlegpetykpjxqjtsljqogkfui` int DEFAULT NULL,\n `tnulnalrqqnblgsisfeueqavbnnwedmq` int DEFAULT NULL,\n `wiytxpsidtqoffijiedhmzumfnsuplou` int DEFAULT NULL,\n `geypgkivqznvszigmexukagzswykhxev` int DEFAULT NULL,\n `hsrdizuizzqpthcltqcvphuqgamiciyf` int DEFAULT NULL,\n `idltweusmaozfeumrjpteusewdvuitxa` int DEFAULT NULL,\n `ypvermwcpddwrhlhvvimrziaggnajtdu` int DEFAULT NULL,\n `ojwvjcmhrnmsresdxrqxpnbuglzrtgqb` int DEFAULT NULL,\n `otexsgacnrtruzymbqqxcvxfitfeewae` int DEFAULT NULL,\n `dsxgpunfeahzwonvbubmmeedgjaaeyna` int DEFAULT NULL,\n `notxylcffsytyzpcpbfmkabxzpywnzkw` int DEFAULT NULL,\n PRIMARY KEY (`hxpmitvzzpzrvxynbckrserhqsqraxne`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"ifbeclftxswghnufrctevakgzhdsmlko\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["hxpmitvzzpzrvxynbckrserhqsqraxne"],"columns":[{"name":"hxpmitvzzpzrvxynbckrserhqsqraxne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"vtxsyqjsvdhpbztvclvpwgtazkujimyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prtrhlfydwhhvpozjpnmtusbzjwlwljl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcwtcscskdftymfprvanpazbjghtzbpw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"buaxlhyrtmrxmvwwuesmzbvlxiwnwctb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqfogsbvuvlwrdgxyplhrbizmzervpmz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"keratpwlnumrivlcivrihlvhejaftkqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eeqomzcfogzyephwpdgltjwqsexwjumo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjwerpccdnpykavjkzhmbpwqmoomsale","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhcbmcutefioocymsyusyflawbbipkxm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbttpcikdqlyzhcehbbgqdxbeqecbrrf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elrvjfsjnaclefakqjwfovcjulumkqtd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lprctxfpntjqjcdpigmtaecgefbkzuxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtshtvpeircwsqdeipgdasxrhofupgoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uljsjbjnilkhcqiqydijawqhvrrckyso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gyyewjgryqdponchtyenbbhgsjrweeqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zydkaxrsmybycbewhpcfvccamwjxdugv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opztuqjndkhqhgnzhbylliynytgjyhko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbltudlpsrwfkhzfrshqvfarfiyqfyjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmylyqiejdijnotpxcxkfxjiyvfvsewk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qlgwqoszgztcthgmtsqotxjipoqensrk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skewjgulsscyglmjcilkndfzkdkchlmc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kibdxwoetkumasylgbbkwsucmrmubisr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igwgekkaoahvqxhbhvjadnvwuaoewdcd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fokratrgntskelomielhnrzhxmmlpgdm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykflkdjzuixrcqrwhkvubylmmjzsuglr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oyyokodrdawwmzglxxulhapntqvvmgyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udmymochdfdrlqcjfkwvzonhsybovnbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qyiddxmtfiqqkyowteefugvumivbefco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbslpqjnltkikqyeuldznwfebpipxvbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jkmewjizgquqdrjnufwceyeylbhehngo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"odvvsgocqshwtmmxvqnmghehjfdlcgeb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekvtlqzumlzejmxibfrruuloxcsjhitb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bioluaghgtvnhqxqnyyvcloosmqnxqnu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hvslwfbhxswhvnjxeldtasrxpetwrhfo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"alwgigngksokqdkugoylfjpwzjfkrioj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntemjtavmlomvrslxzmxoxwlpwtrdkua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uosewpojkfrvzozytbgtcvvjdxtudtmj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ndcdolhxekbnnthanhjksdsghndmddgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xgggbssgjeawaarzqlappavdjfregatg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"munizthkozgvzqzkjcsjeredgvoowthn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cyuxkwubevleqptfoiphkgjuplobivuu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrgnsqcmsyetufwuzcnyzkespeawntyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"leouovocancqvkrqetohjtwwgssqxvnj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ppdelcsoztddkjdqaekragubyblbxxvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqylapdjwhxeproqgmxmklzenzcuvqmy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zfmmsbnsuyjnjxfjqfznetcyxkjcinuk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sxvteykwtotsaypobaztckdnvyhjcymt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obpltbosguppfrqkgricosjpmmymbelw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xunmyoquonhqvscazwgqxuaqelppblni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqxegiwtufwooexmbrrtxyblconwnazz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aldurwtdlrjezzfkxgxytpwerleohsmt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eyzaqnylhjpljaytwfdltaqphqhdebgt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ksmqsjfohbqlkfjuxuxihperuawhkrpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvkvvdpzkogvlwleoniwaskfyzybaqtv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mgxlwczxpzruvfeadjqmlrfomjlggvnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdlxaafmlwzinxkpjsbeybgcwhtebmma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ueddugwgbcbbsuxwfwstkmctxirtvxyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlvzwgwgomgdppgqikzldlbrpqqftxbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oavakmdrmhwsmbtumjzgpdhknrydaxsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qkrmycpxdfjlxrhlvfvwkvwnhotypfwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"toayveyrmmggussfjilxioldnurkqljq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"buggdzldklzfmhvmthrpqsgskavxbles","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"frnluvrlezfgvkgrnlvwvddseulghwkf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cutiwjofdbacfmzmwgcacficncnvyshm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qscimjetrtvqhrhhpboiqgyiarovnzfm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sawbzbonpkvkaxcyacgsqtqxjpewuddk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvklmetonywcnuqudzmgcpdfentmdugs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnjqqstbqtxyevyiczmynbcytphxejgf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jtsevwkdssfgnlwndkofnbvhlirzhckg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jykyjuoelwfkqaehyghrbvnxwtpzhoto","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzxpmafxhtckezrustleelphvygzeuwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezalqitaevsydcszhnofjcmnrzbmawom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjkasfqxhkodmlxwhnghptioiztvofut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfdtktfndnhorjkxvxicoxinjefzsqmy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtkrgriqyxwhivmofkoqvodyibtjxpkg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uozjvlhcnyidwnjdfvrjqfdmsutxeppf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqglkuhnhumbwbjafbhzxkcvjyzbrbrw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"byvsfcykgmapflzsexbatuuknqrtaflf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfyxcalqzhrinfloylbbrhtphbciqrjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvjplmuhcnriycpjbhseoqtdqxyqmqlz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfnhksjvcdnvjeuvkmsqgvevmynrghgc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edfyomubizlgicvkpitgddvwvitrknwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tzxnwmzklqoisvvobexnwyonqxwvggbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnmoltulkucnicfixfxrakedvugjmcza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqvxrfogzxqpeeaxebdytxtyqkitcnxp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oauzokqrwnouylrvtlhwhpirmfqzxbsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xydcsqdhpdxmdamxatquincqzybhafvb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vadajpjedltytakpuumejgoyhfnqfvdx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"erlfbvcwlegpetykpjxqjtsljqogkfui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnulnalrqqnblgsisfeueqavbnnwedmq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wiytxpsidtqoffijiedhmzumfnsuplou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"geypgkivqznvszigmexukagzswykhxev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hsrdizuizzqpthcltqcvphuqgamiciyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idltweusmaozfeumrjpteusewdvuitxa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypvermwcpddwrhlhvvimrziaggnajtdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojwvjcmhrnmsresdxrqxpnbuglzrtgqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otexsgacnrtruzymbqqxcvxfitfeewae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsxgpunfeahzwonvbubmmeedgjaaeyna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"notxylcffsytyzpcpbfmkabxzpywnzkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668578,"databaseName":"models_schema","ddl":"CREATE TABLE `ihrmsdgptsjleslopqmanczyqcdtudap` (\n `omgscaajrsnhepoxqpxaeduiojjigfbd` int NOT NULL,\n `emnwinnjxykbjuttwwremknorzdhfftu` int DEFAULT NULL,\n `mguoswiiamsmntrngvlhnwjkkaxbehdp` int DEFAULT NULL,\n `nsvkkmwlodnidabhpjveykcrhaxihgiq` int DEFAULT NULL,\n `bhrehocxhhbycgwztbkwvaojbqqsbaqi` int DEFAULT NULL,\n `nhlwgnttwizbbiddnjgatqowboxsoknb` int DEFAULT NULL,\n `xotevviyujojeupaetyazseoefmepcyg` int DEFAULT NULL,\n `mcyzglgrcygphmvbhbumsekdpkdzmdxl` int DEFAULT NULL,\n `cxfgujtvfwldztfcytarbcaznpasbxmp` int DEFAULT NULL,\n `sstgmmdomrkmcnytosgizzyieqgjscrk` int DEFAULT NULL,\n `qugvqllmceagbebbefrihjlihhqccbyx` int DEFAULT NULL,\n `eklnwvjdaxyfnzdjgwflvsnhfggshghk` int DEFAULT NULL,\n `uinxvxnsghofiejljyyrhuqnfrhruypw` int DEFAULT NULL,\n `uilatowssmjvwpgyvjpqvtxiyfibluzs` int DEFAULT NULL,\n `fmbcffimobbiqaupbnmjbwwupsleuukl` int DEFAULT NULL,\n `ewglddxzlygblkyvtcxfugperntyjnkr` int DEFAULT NULL,\n `geggatzdyjvpjyisxyzberjzyrdlzbpr` int DEFAULT NULL,\n `ygroepsswilohrxemyfiwnvhyubzfgdq` int DEFAULT NULL,\n `pwwfddbymewbebeqdrqzdwuxzmgiewbr` int DEFAULT NULL,\n `xfozfpojvrldlxlthpgkfnutnlyvhlrf` int DEFAULT NULL,\n `ykewzbiexokiurkdkluhxjpsibayzuju` int DEFAULT NULL,\n `jioyhiwfpkslpnxjppreiuzvcyjcrsau` int DEFAULT NULL,\n `ayrgqssrgtzpmbrfbxqncsvaknkvkatk` int DEFAULT NULL,\n `tkmqbiaaokhvikyiofhzkodbuxzxvaqc` int DEFAULT NULL,\n `qhquauetyhekfbyjhddqilggeycfaktr` int DEFAULT NULL,\n `zxdypzufbypigkkqetdnbxeariulpzxy` int DEFAULT NULL,\n `xzaznsbdgvjsyksozydatcsvfckalqzq` int DEFAULT NULL,\n `lbidzfkqgsxwykuuyjgrsszljbxudeyt` int DEFAULT NULL,\n `sisjelhcdpkbmiycpasmjiphhgmtpgzm` int DEFAULT NULL,\n `vqbdoiwyiqgrcxuebljgsqizhyltjypo` int DEFAULT NULL,\n `rcsxmzswajlbofnjhhhyyqojhaoeoyde` int DEFAULT NULL,\n `pxvryiguuutphxrqfzbgbssiibawduqu` int DEFAULT NULL,\n `ohrzbhphuodliwqcaqafqvdfjmbpqvjo` int DEFAULT NULL,\n `zryixbhfsksrfynbtoirafruvhnpdtpj` int DEFAULT NULL,\n `jwojcgobvnwxvttrsvkbdwemmluaehkz` int DEFAULT NULL,\n `fsirhtcxrzbcohmbcthicokjpnzezojd` int DEFAULT NULL,\n `cabaubqflfgvxunqlepervedmdrhluxt` int DEFAULT NULL,\n `pfnoijuupucvwcohjhhnmdfwszigyokf` int DEFAULT NULL,\n `drcykzxrkaidxdsbbexkcwhaktqearyj` int DEFAULT NULL,\n `eroinyxqnlhlzmuplfyrplfxynzozlfi` int DEFAULT NULL,\n `mdmdghtihdxiezcjioogsohlxfxqwpuu` int DEFAULT NULL,\n `tmrcgoojkxbxfqyiuutbsvvcepdiembp` int DEFAULT NULL,\n `kkwyrlwvswcpaqzixfqdzajokflvizyb` int DEFAULT NULL,\n `jtyddhoxrsiyyljmgzfyvlnexnvdvlhw` int DEFAULT NULL,\n `pfhzvdmlncvtgylpzqzpvywxopdgnbvl` int DEFAULT NULL,\n `ianmpbxvcwbcdnlxqsvpymomzjohrlbd` int DEFAULT NULL,\n `fpfkerekmgvqvyvupprnaqbilngntedn` int DEFAULT NULL,\n `orrewoxehzleznrxynrsbafmlgmcfynn` int DEFAULT NULL,\n `dervvynfadxnyngyxfazrllchhsdyxjx` int DEFAULT NULL,\n `mluinycondpvlsrpjedbhjgrgjyikkgj` int DEFAULT NULL,\n `qphhgqsllhzfndncvtwdmmygvojhwxrd` int DEFAULT NULL,\n `rapdlwpimlhhlsqvnfemucvdqveephgs` int DEFAULT NULL,\n `ootwtyoufqtjfivyjokuqhwptwmkouza` int DEFAULT NULL,\n `qtkbnuhtnofpxsirzitwgflwhatyyhqz` int DEFAULT NULL,\n `gdntsyhwgnqmkzbhjukifaciuflzwcvj` int DEFAULT NULL,\n `rxmyxgvgxcgkqnabqcygmhmjsyvcnomo` int DEFAULT NULL,\n `gbplpjphcojaobxwygnqmuiqnyfnurrw` int DEFAULT NULL,\n `rsqzgvmuipfabrgyfwwlnotnnhkhddvt` int DEFAULT NULL,\n `pztczqjdxhabfzwxsaoqiqhjxwqomklw` int DEFAULT NULL,\n `gwujrdlfjgysfqqbjbghogphifixbsrt` int DEFAULT NULL,\n `urlutxtcrvoblpyzusrninzjhazdkpub` int DEFAULT NULL,\n `hbccrjjiywjoymxosmiwxngzarfkvfnn` int DEFAULT NULL,\n `mdjarlgipndbyeyvubqhpcjwxtvvhjkm` int DEFAULT NULL,\n `xjjpvreyrekpvyztaxbnvngafedhdcfr` int DEFAULT NULL,\n `sryuiebstqmkymuqzbpdswgsexiwrtui` int DEFAULT NULL,\n `mtfwuptxkcyxpoqjadszbiyfucxwjuri` int DEFAULT NULL,\n `hudkpxiyyeztonxuobsecgaaeotidvfg` int DEFAULT NULL,\n `kjwwszdyoldpcgkqmjnwvaptarqjgwek` int DEFAULT NULL,\n `lvexgrsshjfskxlzcmwkrdmritvyntwl` int DEFAULT NULL,\n `ukhbhvgrohcofizgkvxvwpvtukcpyglp` int DEFAULT NULL,\n `uatkwdnspinifbkaiyekrqokutddbfam` int DEFAULT NULL,\n `ncmngeelnorhljpwhygmgbwuzqfxbqvm` int DEFAULT NULL,\n `wvdhuaqgqiorxmcpjffahsnelocyfflc` int DEFAULT NULL,\n `avhpoifmjziwyaqcdjinyywznjxzsopy` int DEFAULT NULL,\n `gdavmtqvrkucxzksqwkrjqizqzzjjkqe` int DEFAULT NULL,\n `uqirvdryjggevsricmjryiknjihllnmg` int DEFAULT NULL,\n `jshyyozomtyxxuernfsicpbmcqngzmvw` int DEFAULT NULL,\n `zqpictwhsbrzrzmncnqgdxnrcrwcqxek` int DEFAULT NULL,\n `tvfffabgzyzdkwjoylrocujmgzamhuqq` int DEFAULT NULL,\n `ivfkpyhkexbhxcrrniirblkjklbcpydr` int DEFAULT NULL,\n `isvlquopxiftbbtvgarvizleuswbleqj` int DEFAULT NULL,\n `pyfspamidcmgxjiegfalvulcvzvtlofj` int DEFAULT NULL,\n `wexdsyxsfqzjpvckuhxbqpgzxlovafhn` int DEFAULT NULL,\n `jercaxryluciydpilznzshrrkfoqplzh` int DEFAULT NULL,\n `paybwvnsykrjifcmuanereklpmroclai` int DEFAULT NULL,\n `sxkicylndzwjczoklvltqdgpvezrtxom` int DEFAULT NULL,\n `nvhxxckxgvcxbzqakheynvuipxxtopsw` int DEFAULT NULL,\n `rdsgphwggspqhjobnzncuicxamrdnajs` int DEFAULT NULL,\n `ktdxgfneqxxzdljgyfnfbhaaieueorkr` int DEFAULT NULL,\n `tuloaesdsbiqhljlhduuhosdahaiycsg` int DEFAULT NULL,\n `dloobsupmntjcpobjkvkwowpawbdsjyk` int DEFAULT NULL,\n `hczupmmopyqnnsrijvpitdpwfhvnvucr` int DEFAULT NULL,\n `fncabuxrknicvqgvlbatntnyqmswsidl` int DEFAULT NULL,\n `equsrpiweuwyptjbrdtwdcanuyjmmpcs` int DEFAULT NULL,\n `bqdbrjolcsciqriamijoeyndhnwxrwcs` int DEFAULT NULL,\n `xlgoiphprrbmzwidlfchipnsvquejvyx` int DEFAULT NULL,\n `kopkwzeukmpzridmdzdycudartifxguc` int DEFAULT NULL,\n `ioblxvzxzlywenvmmizyqmlupkvmlipu` int DEFAULT NULL,\n `nhneuljiogdsreyhomlbyqahjkapggmr` int DEFAULT NULL,\n `qlwlaqaigdrrbuspizwprbrxkaqqluzq` int DEFAULT NULL,\n PRIMARY KEY (`omgscaajrsnhepoxqpxaeduiojjigfbd`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"ihrmsdgptsjleslopqmanczyqcdtudap\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["omgscaajrsnhepoxqpxaeduiojjigfbd"],"columns":[{"name":"omgscaajrsnhepoxqpxaeduiojjigfbd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"emnwinnjxykbjuttwwremknorzdhfftu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mguoswiiamsmntrngvlhnwjkkaxbehdp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nsvkkmwlodnidabhpjveykcrhaxihgiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bhrehocxhhbycgwztbkwvaojbqqsbaqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhlwgnttwizbbiddnjgatqowboxsoknb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xotevviyujojeupaetyazseoefmepcyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcyzglgrcygphmvbhbumsekdpkdzmdxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cxfgujtvfwldztfcytarbcaznpasbxmp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sstgmmdomrkmcnytosgizzyieqgjscrk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qugvqllmceagbebbefrihjlihhqccbyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eklnwvjdaxyfnzdjgwflvsnhfggshghk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uinxvxnsghofiejljyyrhuqnfrhruypw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uilatowssmjvwpgyvjpqvtxiyfibluzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmbcffimobbiqaupbnmjbwwupsleuukl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ewglddxzlygblkyvtcxfugperntyjnkr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"geggatzdyjvpjyisxyzberjzyrdlzbpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ygroepsswilohrxemyfiwnvhyubzfgdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwwfddbymewbebeqdrqzdwuxzmgiewbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xfozfpojvrldlxlthpgkfnutnlyvhlrf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykewzbiexokiurkdkluhxjpsibayzuju","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jioyhiwfpkslpnxjppreiuzvcyjcrsau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ayrgqssrgtzpmbrfbxqncsvaknkvkatk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkmqbiaaokhvikyiofhzkodbuxzxvaqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhquauetyhekfbyjhddqilggeycfaktr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxdypzufbypigkkqetdnbxeariulpzxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xzaznsbdgvjsyksozydatcsvfckalqzq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lbidzfkqgsxwykuuyjgrsszljbxudeyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sisjelhcdpkbmiycpasmjiphhgmtpgzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqbdoiwyiqgrcxuebljgsqizhyltjypo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcsxmzswajlbofnjhhhyyqojhaoeoyde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxvryiguuutphxrqfzbgbssiibawduqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohrzbhphuodliwqcaqafqvdfjmbpqvjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zryixbhfsksrfynbtoirafruvhnpdtpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwojcgobvnwxvttrsvkbdwemmluaehkz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fsirhtcxrzbcohmbcthicokjpnzezojd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cabaubqflfgvxunqlepervedmdrhluxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pfnoijuupucvwcohjhhnmdfwszigyokf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drcykzxrkaidxdsbbexkcwhaktqearyj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eroinyxqnlhlzmuplfyrplfxynzozlfi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdmdghtihdxiezcjioogsohlxfxqwpuu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmrcgoojkxbxfqyiuutbsvvcepdiembp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kkwyrlwvswcpaqzixfqdzajokflvizyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jtyddhoxrsiyyljmgzfyvlnexnvdvlhw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pfhzvdmlncvtgylpzqzpvywxopdgnbvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ianmpbxvcwbcdnlxqsvpymomzjohrlbd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpfkerekmgvqvyvupprnaqbilngntedn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orrewoxehzleznrxynrsbafmlgmcfynn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dervvynfadxnyngyxfazrllchhsdyxjx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mluinycondpvlsrpjedbhjgrgjyikkgj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qphhgqsllhzfndncvtwdmmygvojhwxrd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rapdlwpimlhhlsqvnfemucvdqveephgs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ootwtyoufqtjfivyjokuqhwptwmkouza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtkbnuhtnofpxsirzitwgflwhatyyhqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdntsyhwgnqmkzbhjukifaciuflzwcvj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxmyxgvgxcgkqnabqcygmhmjsyvcnomo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gbplpjphcojaobxwygnqmuiqnyfnurrw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rsqzgvmuipfabrgyfwwlnotnnhkhddvt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pztczqjdxhabfzwxsaoqiqhjxwqomklw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gwujrdlfjgysfqqbjbghogphifixbsrt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urlutxtcrvoblpyzusrninzjhazdkpub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbccrjjiywjoymxosmiwxngzarfkvfnn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdjarlgipndbyeyvubqhpcjwxtvvhjkm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjjpvreyrekpvyztaxbnvngafedhdcfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sryuiebstqmkymuqzbpdswgsexiwrtui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtfwuptxkcyxpoqjadszbiyfucxwjuri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hudkpxiyyeztonxuobsecgaaeotidvfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjwwszdyoldpcgkqmjnwvaptarqjgwek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvexgrsshjfskxlzcmwkrdmritvyntwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ukhbhvgrohcofizgkvxvwpvtukcpyglp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uatkwdnspinifbkaiyekrqokutddbfam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ncmngeelnorhljpwhygmgbwuzqfxbqvm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvdhuaqgqiorxmcpjffahsnelocyfflc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avhpoifmjziwyaqcdjinyywznjxzsopy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdavmtqvrkucxzksqwkrjqizqzzjjkqe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqirvdryjggevsricmjryiknjihllnmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jshyyozomtyxxuernfsicpbmcqngzmvw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqpictwhsbrzrzmncnqgdxnrcrwcqxek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvfffabgzyzdkwjoylrocujmgzamhuqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivfkpyhkexbhxcrrniirblkjklbcpydr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"isvlquopxiftbbtvgarvizleuswbleqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pyfspamidcmgxjiegfalvulcvzvtlofj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wexdsyxsfqzjpvckuhxbqpgzxlovafhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jercaxryluciydpilznzshrrkfoqplzh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"paybwvnsykrjifcmuanereklpmroclai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sxkicylndzwjczoklvltqdgpvezrtxom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvhxxckxgvcxbzqakheynvuipxxtopsw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdsgphwggspqhjobnzncuicxamrdnajs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ktdxgfneqxxzdljgyfnfbhaaieueorkr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tuloaesdsbiqhljlhduuhosdahaiycsg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dloobsupmntjcpobjkvkwowpawbdsjyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hczupmmopyqnnsrijvpitdpwfhvnvucr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fncabuxrknicvqgvlbatntnyqmswsidl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"equsrpiweuwyptjbrdtwdcanuyjmmpcs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqdbrjolcsciqriamijoeyndhnwxrwcs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xlgoiphprrbmzwidlfchipnsvquejvyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kopkwzeukmpzridmdzdycudartifxguc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ioblxvzxzlywenvmmizyqmlupkvmlipu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhneuljiogdsreyhomlbyqahjkapggmr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qlwlaqaigdrrbuspizwprbrxkaqqluzq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668609,"databaseName":"models_schema","ddl":"CREATE TABLE `iigengzntldyhysbkebjxfctfqrglsed` (\n `vpfkurfqtwucfpklpyxsbxolxvbiycuu` int NOT NULL,\n `kgdmnemywumwpxdbfxaswetxcnhvouwg` int DEFAULT NULL,\n `sbssjrptjpemezwhpafrdvujaqefctiv` int DEFAULT NULL,\n `xhmfdkweewfirrxymxubogengnbfzrjo` int DEFAULT NULL,\n `kcmiigejxevdavodubcaootceawlquha` int DEFAULT NULL,\n `yvrxcbnstfevzkpjmjzkigfttzzndgve` int DEFAULT NULL,\n `vhnkjmkndghiruvkgxmidjoiwhsjbydk` int DEFAULT NULL,\n `fbfryhnbfkqbsqqrfnyzzusxjerqvamz` int DEFAULT NULL,\n `rexjnpwebkmdaowicoemoljxyosqjaus` int DEFAULT NULL,\n `mrniyhkacqvoxyuccpenrlxszzfpfuxh` int DEFAULT NULL,\n `qkwujmeaqnnyeifeuevoumdmlhnbdrsd` int DEFAULT NULL,\n `dejzijyhvjmwqaurbutkvjpnrxlabcxt` int DEFAULT NULL,\n `zldtshblsjgeelqdiyxuauinkhnisxdp` int DEFAULT NULL,\n `momlltqbekmkvqlyyuelsrgpsmkqxpwo` int DEFAULT NULL,\n `pupcfvwtppgxnjjuohxeestqvezakkuz` int DEFAULT NULL,\n `xjauhgdisjpldbpydmiotappxhzoxpwe` int DEFAULT NULL,\n `ydlagfuubustdsexpgagijjccbnmvoop` int DEFAULT NULL,\n `rxssrrsexvnqjxmgdrkzuiaojfztebha` int DEFAULT NULL,\n `tjicuirytsltahoftembmashvtbhmqoe` int DEFAULT NULL,\n `ytvspkfannuxfjynovnfuyavcqsxfpom` int DEFAULT NULL,\n `zyqyjrmwolafebcgsntlvkjzwitkrllt` int DEFAULT NULL,\n `atwqcduoibpapqwinzkomfadgahfprrz` int DEFAULT NULL,\n `zsgvubtgpyxqemakbxncvqdjhidewvru` int DEFAULT NULL,\n `kzeecrgagohwtqzchbtdesoehcjkiciw` int DEFAULT NULL,\n `rdkkyjxfupaybktjlkskiplvlstksykw` int DEFAULT NULL,\n `nyqodkyppbgmlxbqwnflqexyzwksgzld` int DEFAULT NULL,\n `xagwewbxpvyywlagxsktdptnghcofvfq` int DEFAULT NULL,\n `bvuzayzligrfquohpovsazvweivyopwg` int DEFAULT NULL,\n `hftqahpfpvbcduhrbuawmrtaoaktlvov` int DEFAULT NULL,\n `jbzchfrthfwvpxregpfjtzrshlfwwrvu` int DEFAULT NULL,\n `jajiaujcholmijxcjyopqaixmxeacugy` int DEFAULT NULL,\n `xbkilmshzpanjxdopywswevqbpxnsfdz` int DEFAULT NULL,\n `mwpavyxbmsgkpcegjhtfgfblekepijox` int DEFAULT NULL,\n `xuahijzvbdsrbwucohpefypavwgvduwi` int DEFAULT NULL,\n `obkkflonlqbetlqjqqqdcqjnkwkaoyme` int DEFAULT NULL,\n `cateqgfzzytydlnqeiaqmbatezizadre` int DEFAULT NULL,\n `aitpkeqzckltquisiuvvmstcqxiatira` int DEFAULT NULL,\n `ycuwxplcnjnmleltvoyvraxnnaynsayq` int DEFAULT NULL,\n `edsotvghechundzfoejmkfjundorushp` int DEFAULT NULL,\n `hshpxjzaaqjsvpgexypittvwpewaxoub` int DEFAULT NULL,\n `baoltkgbxaxghtyynrwjxxuwauwvgizo` int DEFAULT NULL,\n `rauzxkgaliwzcbitqnuegxdqhyegccak` int DEFAULT NULL,\n `mzpvvkdrxwfsnbnameouilgkvcnelqsm` int DEFAULT NULL,\n `fbvxzvykggcyhdbbjqjrnwtmcwmbasnw` int DEFAULT NULL,\n `tesnbgmapxviuqjmlflnzhgbdnvwbtko` int DEFAULT NULL,\n `xwrrzqbcyrvcicpnjqzhhgdptlxisjkf` int DEFAULT NULL,\n `kvivhelrkvzyxiyfwgbtbvmccgysqcrk` int DEFAULT NULL,\n `irmglkgiqjnvgfczriesdzchdzdfbbeq` int DEFAULT NULL,\n `jzkmnbizhmddywzmpekapxyealyydfzt` int DEFAULT NULL,\n `wwlpffhsqmzlxzahinecpkbwniuvqbzk` int DEFAULT NULL,\n `btpagtjlcjdjgxjucgccykjmqanoidlp` int DEFAULT NULL,\n `jtzunrodghoaelfqokkqelnnovznvrkw` int DEFAULT NULL,\n `agtiiktcrioghzvkqcckmqivksnqhzzb` int DEFAULT NULL,\n `qmwriffdmqshbqzrmuymcpfisdadjooi` int DEFAULT NULL,\n `ruhezrfqeawpcmcgmkcngsirmfargmzs` int DEFAULT NULL,\n `ojfhubiheaqqgevjskhoteaxxupcckyo` int DEFAULT NULL,\n `ovayanlhpgqanpgwdczzrousubkhsthf` int DEFAULT NULL,\n `aloteffqotkkcicmttyqtnbhmggkppnw` int DEFAULT NULL,\n `isrnylcucnideqqkxgktqzmxakoqflsk` int DEFAULT NULL,\n `tmeoubuawbdhlddmqbbwzsfimxkfaeqs` int DEFAULT NULL,\n `mwmczrzhjwvmcbenxclclaphstxrazid` int DEFAULT NULL,\n `jzqhvkqwkxibtuytrhomoncwnmjujfau` int DEFAULT NULL,\n `kevldcxegzikqscgprkoqdsddmzxygij` int DEFAULT NULL,\n `mhfavblcchmjooveeounfcrwukoaoowj` int DEFAULT NULL,\n `qslnsfnxbxgbgxmrvkoxbphurrkohmca` int DEFAULT NULL,\n `apzlkwukiojblntvdlswticveigufgwk` int DEFAULT NULL,\n `jburgievzfwfliycazackfjbbopcvxxf` int DEFAULT NULL,\n `dhtjrsdzxeqmfofyuzjdvmogbaqcbcqa` int DEFAULT NULL,\n `krzkrgtemjrwhbqnyetqurbfenfxkrgq` int DEFAULT NULL,\n `mdtdvksioggmhrwtpvszfwvypikiidmo` int DEFAULT NULL,\n `lmzbtyhmzyhqtghxykmgyctrrgzyicet` int DEFAULT NULL,\n `icrqhoggqyvinuanmzevawnjbasfbjvj` int DEFAULT NULL,\n `cjustzdnmilewicricaogkpkezbyoqtq` int DEFAULT NULL,\n `hiljzbyygcvvtgiobrjdllcqhrgulxpu` int DEFAULT NULL,\n `qihzwqgkknglulacxmjerfdbmiwdrxtq` int DEFAULT NULL,\n `syfauxhoglsepdptgqxjcdpkmrssibij` int DEFAULT NULL,\n `yovbwltgyerrteyxixqezuyflrmymcsb` int DEFAULT NULL,\n `maiplwbjkiyxjilctypastodwmxjjskr` int DEFAULT NULL,\n `zlelysgjvcrtlwmeofrxxzhtwcndfuqy` int DEFAULT NULL,\n `zosjwdufnvbmyjqpyzwmxfwnjxhknuhi` int DEFAULT NULL,\n `odzdnpixspxsdcjbakhwpacbripfkixa` int DEFAULT NULL,\n `otsiwspcofwmcchkxwcazqzgeihzcokl` int DEFAULT NULL,\n `ehnmailmofmeisidlvfksbjovoiavauh` int DEFAULT NULL,\n `nfzmdrxfolsnrkftnujxopshsvriaavt` int DEFAULT NULL,\n `skpzxdttbjqozeuyvsjtqnkjgpumbicb` int DEFAULT NULL,\n `nfpiabesbicbaqvrbntlvniddoujrdct` int DEFAULT NULL,\n `fcexodwajbnedunnqlucbjhweefgczgs` int DEFAULT NULL,\n `ezejthgrghtowhooqjkqawmaeqcolfyx` int DEFAULT NULL,\n `krmcxwigluccbstehkxsbwjhlkirixdb` int DEFAULT NULL,\n `teatvrqiqrajokvsehqkwzgeutapyshw` int DEFAULT NULL,\n `awsgcphfnprxkhcwdvjihpmrcfbuozrv` int DEFAULT NULL,\n `ammeusdqmgyhxdupnqgubybzyvocvgqv` int DEFAULT NULL,\n `toprjuryxxeigsqiohwtjjvihgsorszc` int DEFAULT NULL,\n `daoekyurjmafmifrhrealeiqyvjxoxzn` int DEFAULT NULL,\n `tpmmczurulnkfmyshxejetzzihhprpim` int DEFAULT NULL,\n `uchtsywcqfyxqypjgpspauxjkpkrzfio` int DEFAULT NULL,\n `ghktgfpaypwqsgtgrollgreivbhbnyfx` int DEFAULT NULL,\n `ikpkmqflimdqczpjicwpvebsxlaploau` int DEFAULT NULL,\n `tfbqfsvspxrhpdxoesajfwwqmhnuayit` int DEFAULT NULL,\n `fospzlzsrucmxassbbmxorepmellzjxc` int DEFAULT NULL,\n PRIMARY KEY (`vpfkurfqtwucfpklpyxsbxolxvbiycuu`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"iigengzntldyhysbkebjxfctfqrglsed\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["vpfkurfqtwucfpklpyxsbxolxvbiycuu"],"columns":[{"name":"vpfkurfqtwucfpklpyxsbxolxvbiycuu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"kgdmnemywumwpxdbfxaswetxcnhvouwg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbssjrptjpemezwhpafrdvujaqefctiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xhmfdkweewfirrxymxubogengnbfzrjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcmiigejxevdavodubcaootceawlquha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yvrxcbnstfevzkpjmjzkigfttzzndgve","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhnkjmkndghiruvkgxmidjoiwhsjbydk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbfryhnbfkqbsqqrfnyzzusxjerqvamz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rexjnpwebkmdaowicoemoljxyosqjaus","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrniyhkacqvoxyuccpenrlxszzfpfuxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qkwujmeaqnnyeifeuevoumdmlhnbdrsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dejzijyhvjmwqaurbutkvjpnrxlabcxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zldtshblsjgeelqdiyxuauinkhnisxdp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"momlltqbekmkvqlyyuelsrgpsmkqxpwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pupcfvwtppgxnjjuohxeestqvezakkuz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjauhgdisjpldbpydmiotappxhzoxpwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydlagfuubustdsexpgagijjccbnmvoop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxssrrsexvnqjxmgdrkzuiaojfztebha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjicuirytsltahoftembmashvtbhmqoe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytvspkfannuxfjynovnfuyavcqsxfpom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyqyjrmwolafebcgsntlvkjzwitkrllt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atwqcduoibpapqwinzkomfadgahfprrz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zsgvubtgpyxqemakbxncvqdjhidewvru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzeecrgagohwtqzchbtdesoehcjkiciw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdkkyjxfupaybktjlkskiplvlstksykw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nyqodkyppbgmlxbqwnflqexyzwksgzld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xagwewbxpvyywlagxsktdptnghcofvfq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvuzayzligrfquohpovsazvweivyopwg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hftqahpfpvbcduhrbuawmrtaoaktlvov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbzchfrthfwvpxregpfjtzrshlfwwrvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jajiaujcholmijxcjyopqaixmxeacugy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbkilmshzpanjxdopywswevqbpxnsfdz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwpavyxbmsgkpcegjhtfgfblekepijox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xuahijzvbdsrbwucohpefypavwgvduwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obkkflonlqbetlqjqqqdcqjnkwkaoyme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cateqgfzzytydlnqeiaqmbatezizadre","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aitpkeqzckltquisiuvvmstcqxiatira","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ycuwxplcnjnmleltvoyvraxnnaynsayq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edsotvghechundzfoejmkfjundorushp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hshpxjzaaqjsvpgexypittvwpewaxoub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"baoltkgbxaxghtyynrwjxxuwauwvgizo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rauzxkgaliwzcbitqnuegxdqhyegccak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzpvvkdrxwfsnbnameouilgkvcnelqsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbvxzvykggcyhdbbjqjrnwtmcwmbasnw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tesnbgmapxviuqjmlflnzhgbdnvwbtko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xwrrzqbcyrvcicpnjqzhhgdptlxisjkf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvivhelrkvzyxiyfwgbtbvmccgysqcrk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"irmglkgiqjnvgfczriesdzchdzdfbbeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzkmnbizhmddywzmpekapxyealyydfzt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwlpffhsqmzlxzahinecpkbwniuvqbzk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"btpagtjlcjdjgxjucgccykjmqanoidlp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jtzunrodghoaelfqokkqelnnovznvrkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agtiiktcrioghzvkqcckmqivksnqhzzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmwriffdmqshbqzrmuymcpfisdadjooi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ruhezrfqeawpcmcgmkcngsirmfargmzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojfhubiheaqqgevjskhoteaxxupcckyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovayanlhpgqanpgwdczzrousubkhsthf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aloteffqotkkcicmttyqtnbhmggkppnw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"isrnylcucnideqqkxgktqzmxakoqflsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmeoubuawbdhlddmqbbwzsfimxkfaeqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwmczrzhjwvmcbenxclclaphstxrazid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzqhvkqwkxibtuytrhomoncwnmjujfau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kevldcxegzikqscgprkoqdsddmzxygij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhfavblcchmjooveeounfcrwukoaoowj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qslnsfnxbxgbgxmrvkoxbphurrkohmca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"apzlkwukiojblntvdlswticveigufgwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jburgievzfwfliycazackfjbbopcvxxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhtjrsdzxeqmfofyuzjdvmogbaqcbcqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krzkrgtemjrwhbqnyetqurbfenfxkrgq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdtdvksioggmhrwtpvszfwvypikiidmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmzbtyhmzyhqtghxykmgyctrrgzyicet","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icrqhoggqyvinuanmzevawnjbasfbjvj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjustzdnmilewicricaogkpkezbyoqtq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hiljzbyygcvvtgiobrjdllcqhrgulxpu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qihzwqgkknglulacxmjerfdbmiwdrxtq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"syfauxhoglsepdptgqxjcdpkmrssibij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yovbwltgyerrteyxixqezuyflrmymcsb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"maiplwbjkiyxjilctypastodwmxjjskr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zlelysgjvcrtlwmeofrxxzhtwcndfuqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zosjwdufnvbmyjqpyzwmxfwnjxhknuhi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"odzdnpixspxsdcjbakhwpacbripfkixa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otsiwspcofwmcchkxwcazqzgeihzcokl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehnmailmofmeisidlvfksbjovoiavauh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfzmdrxfolsnrkftnujxopshsvriaavt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skpzxdttbjqozeuyvsjtqnkjgpumbicb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfpiabesbicbaqvrbntlvniddoujrdct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcexodwajbnedunnqlucbjhweefgczgs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezejthgrghtowhooqjkqawmaeqcolfyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krmcxwigluccbstehkxsbwjhlkirixdb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"teatvrqiqrajokvsehqkwzgeutapyshw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awsgcphfnprxkhcwdvjihpmrcfbuozrv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ammeusdqmgyhxdupnqgubybzyvocvgqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"toprjuryxxeigsqiohwtjjvihgsorszc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"daoekyurjmafmifrhrealeiqyvjxoxzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpmmczurulnkfmyshxejetzzihhprpim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uchtsywcqfyxqypjgpspauxjkpkrzfio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghktgfpaypwqsgtgrollgreivbhbnyfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikpkmqflimdqczpjicwpvebsxlaploau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tfbqfsvspxrhpdxoesajfwwqmhnuayit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fospzlzsrucmxassbbmxorepmellzjxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668639,"databaseName":"models_schema","ddl":"CREATE TABLE `itudgkajsibeeiwiqpdermkbhjhcuhez` (\n `kpexqvffwdzbpmyxtqrqamrqisqffiob` int NOT NULL,\n `hlgbxfswmmlnswwkmzcjpapynsuirkwa` int DEFAULT NULL,\n `pwcazidsgpajkaczmmopkkqulhoqnxnw` int DEFAULT NULL,\n `vbsajgmcdzecpqjntbkifeihtelqjnhu` int DEFAULT NULL,\n `fvvxlkjreexhoqymeqtqdhlwjsxnekll` int DEFAULT NULL,\n `zzgfifvtqjuiacwyrasliubcskzysqyu` int DEFAULT NULL,\n `gtxvwitbqpnenekayobwlroqllftwwdx` int DEFAULT NULL,\n `phsfhouscwjboangsonpkmdliwcxdgts` int DEFAULT NULL,\n `frjetsvhbpulagpwhnrmurdulermsosg` int DEFAULT NULL,\n `hszmosxrrplhbxoiczngwcbdpadzpqpq` int DEFAULT NULL,\n `wpxzknntundefuqvdyrbzshjoimtdxdg` int DEFAULT NULL,\n `obwpafgrpxrmvpovycosjwtoewqdkjob` int DEFAULT NULL,\n `vkkcrkinffefafttqzkdabohgsovpawb` int DEFAULT NULL,\n `kevkwesmflxmrghsivsuklfvrlgiawcb` int DEFAULT NULL,\n `ulvqpdxjnfouodplxngielarzamtdrxq` int DEFAULT NULL,\n `irydiguaibsazidgtsvmurpiputcpmqu` int DEFAULT NULL,\n `rdfubimebqgofsvqflcxuvxjfpwzfefw` int DEFAULT NULL,\n `nokqxecadcxktbirbgybztnextpzqdzn` int DEFAULT NULL,\n `xhkcxkdevnwuobbqsutcsjuyifzqhpbq` int DEFAULT NULL,\n `zflvniyoaggyqwkgfkwywnbhwfujzkgm` int DEFAULT NULL,\n `rnsxcrvdbcsmirdzoggtchtgvozuqols` int DEFAULT NULL,\n `xmoqwiyzknfpfjmdzmoetvxsptrpohwx` int DEFAULT NULL,\n `qdxwysqnnotlpeasnszauptgknucitvx` int DEFAULT NULL,\n `iazcraodrqatxobnnsvrjzlwczfyhnfx` int DEFAULT NULL,\n `nxrayaddrxhivqcrnhyxiwzecaaqrnbk` int DEFAULT NULL,\n `hznpiqrngsvwmptqqiabcnboryucouyc` int DEFAULT NULL,\n `nlkrliiqkbogfkvdnsvcmmapcbutzrrg` int DEFAULT NULL,\n `mvwntgunircjlzfcffzvohmvklzoijmb` int DEFAULT NULL,\n `zwziizgdybjqsewpnizqpeudcieftuhl` int DEFAULT NULL,\n `dbnttypyuosjmcokqmzzbtxjfwjqkucp` int DEFAULT NULL,\n `nxwamosrapcaytvdrxrbaslywrbajutv` int DEFAULT NULL,\n `juxhbtpafgjftjukvtdszxptjxqrtdbd` int DEFAULT NULL,\n `bbzquvlzoryogjnflplknmsvmehvycay` int DEFAULT NULL,\n `gomtvqwhqnjluoiusbkzwphwkjnhbmra` int DEFAULT NULL,\n `peswsvlnoxojttlahukngdldcrbplrdg` int DEFAULT NULL,\n `btuhrspjlqqwwctfjnwebjyelbugodgk` int DEFAULT NULL,\n `feueyhxbvgzqxidtpfervrutjtsnytof` int DEFAULT NULL,\n `hcxjnpqwzxargrjpsgeckuuioypvzzym` int DEFAULT NULL,\n `swsdvynbdxauqbyerewgwkvscwgwxprv` int DEFAULT NULL,\n `progzykbcgvfxkhasvjdfpbevkpympvc` int DEFAULT NULL,\n `npxuaqgejuwsflzbsbwvlbppstyblwda` int DEFAULT NULL,\n `ipzqlddybbmytterxftjxcbnkbzrytfl` int DEFAULT NULL,\n `thubhyghamhwzatjmaanafrqmdfftisq` int DEFAULT NULL,\n `qtlduuulyhdfsspwdqmexphrcqiwudcg` int DEFAULT NULL,\n `kxuieucmyfagkjlowageyswcpxlbvuuc` int DEFAULT NULL,\n `xxldablsyticemtnebvfcorhjcmzrhld` int DEFAULT NULL,\n `rlzjpkinbypjtidukvdpxccctyidruog` int DEFAULT NULL,\n `dhaqravbaogwnqhokwvhslupyufuqovt` int DEFAULT NULL,\n `wngojhueydkpdizjuymfmyimcisgarpu` int DEFAULT NULL,\n `nvljxughhivyamdirtojtgwrcgmqmzhj` int DEFAULT NULL,\n `griyhrjmbxbooydqxmrzxdzbkxyducph` int DEFAULT NULL,\n `tqgfvbnreekbvrgzrorextjnvnkxnzfp` int DEFAULT NULL,\n `ocbzainghtxtfspxbfyfolyvaoeqtlgi` int DEFAULT NULL,\n `pzwwvmwvaxwqlzizumxjqgucfyohherc` int DEFAULT NULL,\n `tkhjxwdxshytuzvfuliodntepbypgmpa` int DEFAULT NULL,\n `hykvmqzdyvfkrapzobhewdqvdzbjtolu` int DEFAULT NULL,\n `cenybgeknxkfzlvqgfpxlbnoszwewvab` int DEFAULT NULL,\n `rpngawxuznxbnpiffhnkrjqmuaseibwn` int DEFAULT NULL,\n `uoktaoqqokyaxmbacvbjowdjlpjotjdc` int DEFAULT NULL,\n `loquoctntclybhysqighpdnbinxqvtzz` int DEFAULT NULL,\n `kpduzpzrhwrnpvqyijfpnzbgnlbufwje` int DEFAULT NULL,\n `dgdkpgouwtrurqdvgulgqpdbaabqhbcl` int DEFAULT NULL,\n `vwidlvyhkaebcpdpmcsgbcydratbcuxx` int DEFAULT NULL,\n `sertktnqigbzyrqazgbugzbyzjvitbqx` int DEFAULT NULL,\n `vvulwvdrihtgehdtmaqrjqwevxfsazpp` int DEFAULT NULL,\n `zesrpqtzfejtsojpdwwcgkggqufswhyt` int DEFAULT NULL,\n `ullznizoxpkamxlnlyxtptxgwyuuzqif` int DEFAULT NULL,\n `mdoiphjbtyesckpcgbeiyqnkhkzafixd` int DEFAULT NULL,\n `hpddnbbamtnrjqnbmlgtbgamzggxvvml` int DEFAULT NULL,\n `bppueljvjqgnrgytedraxmmwjvbjtmfe` int DEFAULT NULL,\n `gfvabsueisvgmuwmxrpxbqndbdwophyy` int DEFAULT NULL,\n `lwptvxadnoebtkxwpxmkscduiesaibak` int DEFAULT NULL,\n `oizdjvwuwelpsvwqwornegrpxwkdmyxl` int DEFAULT NULL,\n `wftnzfbnnzappjbqcgvskbtrzbkvlvum` int DEFAULT NULL,\n `txldnmnjbytddwzvthkocsgqlbcswpsd` int DEFAULT NULL,\n `yvwkejcnpyzengzsjarvuuiffakimmxl` int DEFAULT NULL,\n `rxkarotgblcfyfmklzggkvuzvnandjjf` int DEFAULT NULL,\n `bfoaewlzqywytjykolwjmtpgzewivjeb` int DEFAULT NULL,\n `eeegbytahkvtzqpvhtphypcwrshisunp` int DEFAULT NULL,\n `hmofyzdguhwheqnfoswqkdnkwasmjmkj` int DEFAULT NULL,\n `jfgpvvecwjxxgydkqsyhztpcpljosvee` int DEFAULT NULL,\n `pplnpxmsssrgkznovmmgfpfgeilavwzm` int DEFAULT NULL,\n `etqwejwfyvvnjsrqkvhrdnsxjdadzyfl` int DEFAULT NULL,\n `jnglhbjnolhxnlmzphtvxzrnxcfzwahz` int DEFAULT NULL,\n `ffckakjiuougyhcabdpeuxorqkyixhfn` int DEFAULT NULL,\n `uhzsyfiwwssgomlkzhtqbntazbfnenat` int DEFAULT NULL,\n `ekesreudeuobqlmykguhmvxtxxicqdob` int DEFAULT NULL,\n `tkqgpwuqtmxiwrifqvhghvxsftyjtkkq` int DEFAULT NULL,\n `wlzqmehfbzsyvmhprxcdsmwqyvbwrxah` int DEFAULT NULL,\n `lqwglwasmjjjzhwglgeqeapkgxjdokki` int DEFAULT NULL,\n `dgcjpyafsyvwbqfqnkaernilhntybwcw` int DEFAULT NULL,\n `sqssnusfoxsiimevdurgjthssbssanzo` int DEFAULT NULL,\n `gjrysrpqdsvxlybfddmfpeuxdyashyel` int DEFAULT NULL,\n `umfdlakufwilytdkmbdoygmndaiviajt` int DEFAULT NULL,\n `cqtwldfbzzrfopynmojbiyzxatvdpoej` int DEFAULT NULL,\n `cmmdfszdlodgaitihgkghvbhnfxhlyiv` int DEFAULT NULL,\n `xqtjhaoworocjwxhqfcdtzjqsngaokuc` int DEFAULT NULL,\n `xmsakcxnpjwmqfbfbudbfuunwaxphvea` int DEFAULT NULL,\n `maxrktaslbbkaepcwahaauphokisvdja` int DEFAULT NULL,\n `tllvewatktxmxrohmerbzimazmsnexdp` int DEFAULT NULL,\n PRIMARY KEY (`kpexqvffwdzbpmyxtqrqamrqisqffiob`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"itudgkajsibeeiwiqpdermkbhjhcuhez\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["kpexqvffwdzbpmyxtqrqamrqisqffiob"],"columns":[{"name":"kpexqvffwdzbpmyxtqrqamrqisqffiob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"hlgbxfswmmlnswwkmzcjpapynsuirkwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwcazidsgpajkaczmmopkkqulhoqnxnw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbsajgmcdzecpqjntbkifeihtelqjnhu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fvvxlkjreexhoqymeqtqdhlwjsxnekll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzgfifvtqjuiacwyrasliubcskzysqyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtxvwitbqpnenekayobwlroqllftwwdx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phsfhouscwjboangsonpkmdliwcxdgts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"frjetsvhbpulagpwhnrmurdulermsosg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hszmosxrrplhbxoiczngwcbdpadzpqpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wpxzknntundefuqvdyrbzshjoimtdxdg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obwpafgrpxrmvpovycosjwtoewqdkjob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkkcrkinffefafttqzkdabohgsovpawb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kevkwesmflxmrghsivsuklfvrlgiawcb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ulvqpdxjnfouodplxngielarzamtdrxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"irydiguaibsazidgtsvmurpiputcpmqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdfubimebqgofsvqflcxuvxjfpwzfefw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nokqxecadcxktbirbgybztnextpzqdzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xhkcxkdevnwuobbqsutcsjuyifzqhpbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zflvniyoaggyqwkgfkwywnbhwfujzkgm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnsxcrvdbcsmirdzoggtchtgvozuqols","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmoqwiyzknfpfjmdzmoetvxsptrpohwx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdxwysqnnotlpeasnszauptgknucitvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iazcraodrqatxobnnsvrjzlwczfyhnfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxrayaddrxhivqcrnhyxiwzecaaqrnbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hznpiqrngsvwmptqqiabcnboryucouyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nlkrliiqkbogfkvdnsvcmmapcbutzrrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvwntgunircjlzfcffzvohmvklzoijmb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zwziizgdybjqsewpnizqpeudcieftuhl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbnttypyuosjmcokqmzzbtxjfwjqkucp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxwamosrapcaytvdrxrbaslywrbajutv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"juxhbtpafgjftjukvtdszxptjxqrtdbd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbzquvlzoryogjnflplknmsvmehvycay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gomtvqwhqnjluoiusbkzwphwkjnhbmra","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"peswsvlnoxojttlahukngdldcrbplrdg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"btuhrspjlqqwwctfjnwebjyelbugodgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"feueyhxbvgzqxidtpfervrutjtsnytof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hcxjnpqwzxargrjpsgeckuuioypvzzym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swsdvynbdxauqbyerewgwkvscwgwxprv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"progzykbcgvfxkhasvjdfpbevkpympvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npxuaqgejuwsflzbsbwvlbppstyblwda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ipzqlddybbmytterxftjxcbnkbzrytfl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thubhyghamhwzatjmaanafrqmdfftisq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtlduuulyhdfsspwdqmexphrcqiwudcg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxuieucmyfagkjlowageyswcpxlbvuuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxldablsyticemtnebvfcorhjcmzrhld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlzjpkinbypjtidukvdpxccctyidruog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhaqravbaogwnqhokwvhslupyufuqovt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wngojhueydkpdizjuymfmyimcisgarpu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvljxughhivyamdirtojtgwrcgmqmzhj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"griyhrjmbxbooydqxmrzxdzbkxyducph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqgfvbnreekbvrgzrorextjnvnkxnzfp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocbzainghtxtfspxbfyfolyvaoeqtlgi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pzwwvmwvaxwqlzizumxjqgucfyohherc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkhjxwdxshytuzvfuliodntepbypgmpa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hykvmqzdyvfkrapzobhewdqvdzbjtolu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cenybgeknxkfzlvqgfpxlbnoszwewvab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpngawxuznxbnpiffhnkrjqmuaseibwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uoktaoqqokyaxmbacvbjowdjlpjotjdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"loquoctntclybhysqighpdnbinxqvtzz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpduzpzrhwrnpvqyijfpnzbgnlbufwje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgdkpgouwtrurqdvgulgqpdbaabqhbcl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vwidlvyhkaebcpdpmcsgbcydratbcuxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sertktnqigbzyrqazgbugzbyzjvitbqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vvulwvdrihtgehdtmaqrjqwevxfsazpp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zesrpqtzfejtsojpdwwcgkggqufswhyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ullznizoxpkamxlnlyxtptxgwyuuzqif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdoiphjbtyesckpcgbeiyqnkhkzafixd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hpddnbbamtnrjqnbmlgtbgamzggxvvml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bppueljvjqgnrgytedraxmmwjvbjtmfe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfvabsueisvgmuwmxrpxbqndbdwophyy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwptvxadnoebtkxwpxmkscduiesaibak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oizdjvwuwelpsvwqwornegrpxwkdmyxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wftnzfbnnzappjbqcgvskbtrzbkvlvum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txldnmnjbytddwzvthkocsgqlbcswpsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yvwkejcnpyzengzsjarvuuiffakimmxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxkarotgblcfyfmklzggkvuzvnandjjf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bfoaewlzqywytjykolwjmtpgzewivjeb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eeegbytahkvtzqpvhtphypcwrshisunp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmofyzdguhwheqnfoswqkdnkwasmjmkj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jfgpvvecwjxxgydkqsyhztpcpljosvee","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pplnpxmsssrgkznovmmgfpfgeilavwzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"etqwejwfyvvnjsrqkvhrdnsxjdadzyfl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jnglhbjnolhxnlmzphtvxzrnxcfzwahz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffckakjiuougyhcabdpeuxorqkyixhfn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhzsyfiwwssgomlkzhtqbntazbfnenat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekesreudeuobqlmykguhmvxtxxicqdob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkqgpwuqtmxiwrifqvhghvxsftyjtkkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wlzqmehfbzsyvmhprxcdsmwqyvbwrxah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqwglwasmjjjzhwglgeqeapkgxjdokki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgcjpyafsyvwbqfqnkaernilhntybwcw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sqssnusfoxsiimevdurgjthssbssanzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjrysrpqdsvxlybfddmfpeuxdyashyel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umfdlakufwilytdkmbdoygmndaiviajt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqtwldfbzzrfopynmojbiyzxatvdpoej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmmdfszdlodgaitihgkghvbhnfxhlyiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqtjhaoworocjwxhqfcdtzjqsngaokuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmsakcxnpjwmqfbfbudbfuunwaxphvea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"maxrktaslbbkaepcwahaauphokisvdja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tllvewatktxmxrohmerbzimazmsnexdp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668670,"databaseName":"models_schema","ddl":"CREATE TABLE `ivjawdrlpjejctimphddwlktsjeoqpsc` (\n `higmmeccpfhcfftzbwtocduxwamfmiao` int NOT NULL,\n `ekdwqpqpjxdsxjnnxnuhjjedxtqpqdkz` int DEFAULT NULL,\n `cgapirpteirbzkgxfeizftieskcxtvug` int DEFAULT NULL,\n `omlnngxtyfvfrplnaydaudlwpstlaidp` int DEFAULT NULL,\n `brbkktrpzhfwlxvtyljnjjvpbtmftxqw` int DEFAULT NULL,\n `pdklwsingehudzpqfbkvltqowcpyjbol` int DEFAULT NULL,\n `clvppvtehygnicpkrlmsuvdgqlgyikdu` int DEFAULT NULL,\n `jpavcckyijeaostxrlitvqptrmxghcaa` int DEFAULT NULL,\n `ldtrznvsdwvqzexlmspkamxikiwkppzb` int DEFAULT NULL,\n `pomlgrsmqptcttoibeihdeplxtspptbf` int DEFAULT NULL,\n `aiuagompjmgkwawynmdubtuhcaayvzhj` int DEFAULT NULL,\n `sinsjxkfdncnaxqassmuzpunzxkylixu` int DEFAULT NULL,\n `gcdtepzqjqvqmmikkdjpyaxsaebpnmhh` int DEFAULT NULL,\n `ablmfemhwbsssaidbuspwpkwrlczfggq` int DEFAULT NULL,\n `khtaghnbjhukhqgblzekcowlgmbpqsdy` int DEFAULT NULL,\n `hlpmgwtyfzysmangvhscfzygdasrpwgp` int DEFAULT NULL,\n `tbqcvrawlaseyodhibandjadjxpahvhn` int DEFAULT NULL,\n `uccwcivhztiqxzntfljbwitsajmmiyem` int DEFAULT NULL,\n `sggecczsmtojsxuqurppiibmjgmfjkjx` int DEFAULT NULL,\n `psoesjjktrpzdomvchtvlujuyesoxunv` int DEFAULT NULL,\n `xackualxlncazdapmworwkddszwhmxvn` int DEFAULT NULL,\n `iyujlbvydcnvddkbgqapclfbydhvykam` int DEFAULT NULL,\n `hhznhlpsavxwihxhiehohsawcympfpcl` int DEFAULT NULL,\n `yahxpdexqswjodtbklilyjgjjfcidawl` int DEFAULT NULL,\n `gjrzlxgjwxtlhbetwmwdqjansppjxsdk` int DEFAULT NULL,\n `pfqupfwvspojfodptivyaahidroufwsh` int DEFAULT NULL,\n `wiqadgsmkiivsmpumjqwuajghdlnufvf` int DEFAULT NULL,\n `lnycureemlassjhafuyatdueynanesnp` int DEFAULT NULL,\n `oofovzjmwbacjolsoklmnhfjlvhcbstr` int DEFAULT NULL,\n `galtdhkcbakrsuzolqnteupfmwhqvdja` int DEFAULT NULL,\n `adobnlahniqljuvzcfocnxlqkytximht` int DEFAULT NULL,\n `nqufzgntnxjaypwxsqobkkikdwemakco` int DEFAULT NULL,\n `betzagszkphutoedhvvkyzevveyywbjb` int DEFAULT NULL,\n `zcdqkvwkbhgndneimhonaljzouhxrses` int DEFAULT NULL,\n `bikxsbarndjuoencsiheooquixtssdkg` int DEFAULT NULL,\n `ddpphnhpdhcalynwjazuueeurxyonjnr` int DEFAULT NULL,\n `kcztcanymdeeeflsvdjlkqoftslnixmu` int DEFAULT NULL,\n `zhlalgcabokbigokffhctasmefkzgwsa` int DEFAULT NULL,\n `tjzeuieawoodjgwjiwmonrpkeestokwq` int DEFAULT NULL,\n `kudvcdsidbztspvwnubzwjdfgakyxjuf` int DEFAULT NULL,\n `wjxpynuammuhycrcyskryaudxbzikhkz` int DEFAULT NULL,\n `wyykqndjlmwqhajgluuavpkmivnbqhfa` int DEFAULT NULL,\n `iheydhcjxffqydktwdjszsezjulfuckm` int DEFAULT NULL,\n `zdwvvxygjmigpnbbwzznxpncpszsjnck` int DEFAULT NULL,\n `aympoipseqsryfzwsvloeftvvjbnwmxp` int DEFAULT NULL,\n `khztafuhzsxtsjepostrwfqrkupbgrul` int DEFAULT NULL,\n `yfwctbpvovfaodsgkbvcuniexqyqtwkj` int DEFAULT NULL,\n `wnrrbltfbngrihnwydliwubhwwlabyho` int DEFAULT NULL,\n `agwgidppynlchfwfqhlxpfdadkfqsznh` int DEFAULT NULL,\n `jdwcfvtbewuomsuutelpdwahhxglyofc` int DEFAULT NULL,\n `uofnkjsdcdnkqvwgvzrgalluajkuwgol` int DEFAULT NULL,\n `paktmrtriwnykkyemlvftxlvgrmwisbn` int DEFAULT NULL,\n `gqzetfvrbkmoxytfielwpyvjhclyccve` int DEFAULT NULL,\n `drluawpkghvhizuyjtoziuyyxiglcmkq` int DEFAULT NULL,\n `dfejeuyebudmctvuigucmhxtijpreyov` int DEFAULT NULL,\n `clzvmnbqrdryzgcyhlqaustchhjvtenu` int DEFAULT NULL,\n `bynnzevhbjoopciterhrthjuqlkuaqro` int DEFAULT NULL,\n `cthzoacvhucgyabqynnnfpfjioxmhinv` int DEFAULT NULL,\n `lzyvqokomibqwitlhxcurwyvhtvqptgz` int DEFAULT NULL,\n `igznecwadwvvxrloufgmyyvmfxbjmvjr` int DEFAULT NULL,\n `ywayukgeajuowxdasyakcnnizgozyukl` int DEFAULT NULL,\n `wypzafelhpnssodfqocjzstynwlsnxmk` int DEFAULT NULL,\n `kqhrnilkpmrvqayjofzozgagmzswqbki` int DEFAULT NULL,\n `hthvecxwrcfvvwkojowisbhvqimvrvjd` int DEFAULT NULL,\n `hwrfeatgijbrclvpmlzlsreackegaums` int DEFAULT NULL,\n `ydsjowgrvwaorejgyphdikqgxiqtyjom` int DEFAULT NULL,\n `hywgizxxkmeytwdrkihjrbmvdjpsxjoj` int DEFAULT NULL,\n `cvbaiwyscestsaxrprbuzqcibumdzdvy` int DEFAULT NULL,\n `dhrwupxsbodrwrkhpsiutoymrvmnovyl` int DEFAULT NULL,\n `wsjtpxjqoxakjleytouegyhzqgxpdrzw` int DEFAULT NULL,\n `vujagaeajozbhpgxbgzknicaxczzwwwa` int DEFAULT NULL,\n `zrhkmktdvehoqurqzwdsotjhzhqwstvr` int DEFAULT NULL,\n `qsxagbhhcmlqxveptzmllezcnejrlieu` int DEFAULT NULL,\n `wmjcewtxlhhfaeuedhpgsjthrthfrzxa` int DEFAULT NULL,\n `pxlkfunsqtafkyvwkliobiuvayiddlzf` int DEFAULT NULL,\n `gnrqydmygrmbirqivmqgthkruxfeeasa` int DEFAULT NULL,\n `gbnftoequegqotllgufozszdvnpwnhyp` int DEFAULT NULL,\n `qrffcbwtsatdqhmrztvrqayfgypevspp` int DEFAULT NULL,\n `gphdhcljzxvdmtothyqyaqtyjxkkbsbp` int DEFAULT NULL,\n `vmwtpfhgprcmonhmoihakbchmmsretje` int DEFAULT NULL,\n `hxmbhyxfpfviaurtctkljdewwumopzes` int DEFAULT NULL,\n `homnjtvhnxfmskxwujunvxokloffakfy` int DEFAULT NULL,\n `nypqcmvnomtcucwrpryntqsrjmjxorfs` int DEFAULT NULL,\n `tvacktlcypqxnxsbrsapstbvjimpiwjx` int DEFAULT NULL,\n `flkokmuuybvwrnyuagqiwslbbvkwhwrh` int DEFAULT NULL,\n `qecjtlulnzhdedrcqqpunbcyxausdquu` int DEFAULT NULL,\n `avepqedptzjagmwhwlgitouqocvxlosv` int DEFAULT NULL,\n `iiqmrnbmowjgteokvkvkkmjtobhoodei` int DEFAULT NULL,\n `hwvowbuwkscesbzepzesmyggrsmzmmde` int DEFAULT NULL,\n `akofilwxdumsswsrqyhwrdnnvdkvboiy` int DEFAULT NULL,\n `qdtvwcxaqqwkvifcgopswvqxllukjyhl` int DEFAULT NULL,\n `dvcfskrfjnlgjtihyrcyitqkmhlgqoci` int DEFAULT NULL,\n `ngtvrjkgexbvzavdfwojrssrfxykczcn` int DEFAULT NULL,\n `wmvonlfthwzgttvbddiuuldjfwvzbbpi` int DEFAULT NULL,\n `lwkdraifnrmvsaxepsktttotyiiwontl` int DEFAULT NULL,\n `idmzbpgrrhbdkxcofejxhtflezxcdsmk` int DEFAULT NULL,\n `fmqnluogrekaqovemtonibgxwudqutdm` int DEFAULT NULL,\n `mqajikcmgtnnvggxgqjsaoiuouxojkfs` int DEFAULT NULL,\n `qtbwyahjncwzhttrpmctbwfzepmkspmz` int DEFAULT NULL,\n `nmkfeavmgdgqncahfofkedqbzdpycbiu` int DEFAULT NULL,\n PRIMARY KEY (`higmmeccpfhcfftzbwtocduxwamfmiao`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"ivjawdrlpjejctimphddwlktsjeoqpsc\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["higmmeccpfhcfftzbwtocduxwamfmiao"],"columns":[{"name":"higmmeccpfhcfftzbwtocduxwamfmiao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ekdwqpqpjxdsxjnnxnuhjjedxtqpqdkz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgapirpteirbzkgxfeizftieskcxtvug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omlnngxtyfvfrplnaydaudlwpstlaidp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"brbkktrpzhfwlxvtyljnjjvpbtmftxqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdklwsingehudzpqfbkvltqowcpyjbol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clvppvtehygnicpkrlmsuvdgqlgyikdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpavcckyijeaostxrlitvqptrmxghcaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ldtrznvsdwvqzexlmspkamxikiwkppzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pomlgrsmqptcttoibeihdeplxtspptbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aiuagompjmgkwawynmdubtuhcaayvzhj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sinsjxkfdncnaxqassmuzpunzxkylixu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gcdtepzqjqvqmmikkdjpyaxsaebpnmhh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ablmfemhwbsssaidbuspwpkwrlczfggq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khtaghnbjhukhqgblzekcowlgmbpqsdy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlpmgwtyfzysmangvhscfzygdasrpwgp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbqcvrawlaseyodhibandjadjxpahvhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uccwcivhztiqxzntfljbwitsajmmiyem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sggecczsmtojsxuqurppiibmjgmfjkjx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psoesjjktrpzdomvchtvlujuyesoxunv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xackualxlncazdapmworwkddszwhmxvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iyujlbvydcnvddkbgqapclfbydhvykam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhznhlpsavxwihxhiehohsawcympfpcl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yahxpdexqswjodtbklilyjgjjfcidawl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjrzlxgjwxtlhbetwmwdqjansppjxsdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pfqupfwvspojfodptivyaahidroufwsh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wiqadgsmkiivsmpumjqwuajghdlnufvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lnycureemlassjhafuyatdueynanesnp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oofovzjmwbacjolsoklmnhfjlvhcbstr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"galtdhkcbakrsuzolqnteupfmwhqvdja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"adobnlahniqljuvzcfocnxlqkytximht","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqufzgntnxjaypwxsqobkkikdwemakco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"betzagszkphutoedhvvkyzevveyywbjb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zcdqkvwkbhgndneimhonaljzouhxrses","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bikxsbarndjuoencsiheooquixtssdkg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddpphnhpdhcalynwjazuueeurxyonjnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcztcanymdeeeflsvdjlkqoftslnixmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhlalgcabokbigokffhctasmefkzgwsa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjzeuieawoodjgwjiwmonrpkeestokwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kudvcdsidbztspvwnubzwjdfgakyxjuf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjxpynuammuhycrcyskryaudxbzikhkz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyykqndjlmwqhajgluuavpkmivnbqhfa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iheydhcjxffqydktwdjszsezjulfuckm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdwvvxygjmigpnbbwzznxpncpszsjnck","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aympoipseqsryfzwsvloeftvvjbnwmxp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khztafuhzsxtsjepostrwfqrkupbgrul","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfwctbpvovfaodsgkbvcuniexqyqtwkj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnrrbltfbngrihnwydliwubhwwlabyho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agwgidppynlchfwfqhlxpfdadkfqsznh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdwcfvtbewuomsuutelpdwahhxglyofc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uofnkjsdcdnkqvwgvzrgalluajkuwgol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"paktmrtriwnykkyemlvftxlvgrmwisbn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqzetfvrbkmoxytfielwpyvjhclyccve","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drluawpkghvhizuyjtoziuyyxiglcmkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dfejeuyebudmctvuigucmhxtijpreyov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clzvmnbqrdryzgcyhlqaustchhjvtenu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bynnzevhbjoopciterhrthjuqlkuaqro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cthzoacvhucgyabqynnnfpfjioxmhinv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzyvqokomibqwitlhxcurwyvhtvqptgz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igznecwadwvvxrloufgmyyvmfxbjmvjr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywayukgeajuowxdasyakcnnizgozyukl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wypzafelhpnssodfqocjzstynwlsnxmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqhrnilkpmrvqayjofzozgagmzswqbki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hthvecxwrcfvvwkojowisbhvqimvrvjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwrfeatgijbrclvpmlzlsreackegaums","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydsjowgrvwaorejgyphdikqgxiqtyjom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hywgizxxkmeytwdrkihjrbmvdjpsxjoj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvbaiwyscestsaxrprbuzqcibumdzdvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhrwupxsbodrwrkhpsiutoymrvmnovyl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wsjtpxjqoxakjleytouegyhzqgxpdrzw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vujagaeajozbhpgxbgzknicaxczzwwwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zrhkmktdvehoqurqzwdsotjhzhqwstvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qsxagbhhcmlqxveptzmllezcnejrlieu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmjcewtxlhhfaeuedhpgsjthrthfrzxa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxlkfunsqtafkyvwkliobiuvayiddlzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnrqydmygrmbirqivmqgthkruxfeeasa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gbnftoequegqotllgufozszdvnpwnhyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrffcbwtsatdqhmrztvrqayfgypevspp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gphdhcljzxvdmtothyqyaqtyjxkkbsbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmwtpfhgprcmonhmoihakbchmmsretje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxmbhyxfpfviaurtctkljdewwumopzes","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"homnjtvhnxfmskxwujunvxokloffakfy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nypqcmvnomtcucwrpryntqsrjmjxorfs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvacktlcypqxnxsbrsapstbvjimpiwjx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flkokmuuybvwrnyuagqiwslbbvkwhwrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qecjtlulnzhdedrcqqpunbcyxausdquu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avepqedptzjagmwhwlgitouqocvxlosv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iiqmrnbmowjgteokvkvkkmjtobhoodei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwvowbuwkscesbzepzesmyggrsmzmmde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akofilwxdumsswsrqyhwrdnnvdkvboiy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdtvwcxaqqwkvifcgopswvqxllukjyhl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvcfskrfjnlgjtihyrcyitqkmhlgqoci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ngtvrjkgexbvzavdfwojrssrfxykczcn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmvonlfthwzgttvbddiuuldjfwvzbbpi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwkdraifnrmvsaxepsktttotyiiwontl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idmzbpgrrhbdkxcofejxhtflezxcdsmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmqnluogrekaqovemtonibgxwudqutdm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqajikcmgtnnvggxgqjsaoiuouxojkfs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtbwyahjncwzhttrpmctbwfzepmkspmz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmkfeavmgdgqncahfofkedqbzdpycbiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668702,"databaseName":"models_schema","ddl":"CREATE TABLE `iwnntobuwmkuzncjkkxxbjyvktvzwcla` (\n `zcldsswjalnwvntzwpwkelyiwhpnxebs` int NOT NULL,\n `hftiuabvtuplaqdtdzrfuoriyswniodg` int DEFAULT NULL,\n `idgeaszhaufaskbcexzhyuwxdovdizce` int DEFAULT NULL,\n `raqooccyxupdlsthkdrvukcuxflabhih` int DEFAULT NULL,\n `nflbnhrjoaxcsbnczwgiqwqabqfgyikz` int DEFAULT NULL,\n `kfahrzeotguonqvolkrumkclgkjvmzvt` int DEFAULT NULL,\n `nlmdabcrchwqwisbfqqwsndjqcltxlqi` int DEFAULT NULL,\n `utrlgltfehihwzizqckthaanlgqylbyk` int DEFAULT NULL,\n `hdsbgujmgrcjnralhqawqusxkujuhish` int DEFAULT NULL,\n `trrccvhockkyvyofjxkpldlgsmvsapju` int DEFAULT NULL,\n `zadeadyjxozddklnytyklpsfweijflbs` int DEFAULT NULL,\n `lofjszofzabnmfsacygswgayayyiatwr` int DEFAULT NULL,\n `qhfhrxbzluekayattuwpngggypvimumz` int DEFAULT NULL,\n `nhymduqlpaithjwbcmhpnvpdmolfeevd` int DEFAULT NULL,\n `adywbvttclvocfgyaamoowdtqoxcsetx` int DEFAULT NULL,\n `zxwlobytoofwngxtybuitmdtvvgwsayz` int DEFAULT NULL,\n `rfphmtmawzgqeleyfmbwpodkyoqnmcff` int DEFAULT NULL,\n `nhxtserjbhugnheksxeplvyoguhlrneo` int DEFAULT NULL,\n `nzmsybwdluehumzcfgojrdsnzyhwmsua` int DEFAULT NULL,\n `mrdwjuqszpvjlqboaamoccjzzlnrdbbu` int DEFAULT NULL,\n `hgmpoayzlfmpfmzosdfyhakjsdftveya` int DEFAULT NULL,\n `klseeueujhhotnuwqardsuwuanuwcqxr` int DEFAULT NULL,\n `iptpahwdwwuocgcztqntogjdyrknbhzw` int DEFAULT NULL,\n `yjnzaskwwitkwefyglnhrkbqmehraypw` int DEFAULT NULL,\n `kvgmbfplkwahqfbsklcoexlcubfxlecu` int DEFAULT NULL,\n `lellmogbqtcndrvsthfxnjzkapftvlxt` int DEFAULT NULL,\n `blrziancalsuakxivpyhuvsnhtuoxaji` int DEFAULT NULL,\n `gmwzpwhqccyhhuprwjzecjzgxetaclzx` int DEFAULT NULL,\n `zstokfstlsrhkdrvopsdcgyxpkjddtqm` int DEFAULT NULL,\n `mjrxmkkniuspjqimadwrtgrcdurgoqcs` int DEFAULT NULL,\n `ynqfqoxmbzmhxhcpnqbgehvcdsykznpd` int DEFAULT NULL,\n `dhqizyhklmxvzpgaveeitycrjtpjysrc` int DEFAULT NULL,\n `fmfxocmtylyhbihwphmmgbwbamxwpxcq` int DEFAULT NULL,\n `twlqcxgyvqamrqqwwrctzqtdbzhorock` int DEFAULT NULL,\n `swpvgcjddhrtsvueehdchywaagzeeabk` int DEFAULT NULL,\n `kuoluvplitorpzcawhyjnqafqtpzrcgd` int DEFAULT NULL,\n `dabsrxfwlirkzwvtewrreepchgvqaweg` int DEFAULT NULL,\n `ejjtkwqkkvsjixeorsodrybwiqnmxbrh` int DEFAULT NULL,\n `zuzdixkgfhurtzlkgufdtwpjdnnudeip` int DEFAULT NULL,\n `djzihpworqzoyggukwygnbuupdkughlo` int DEFAULT NULL,\n `aoifapuntzrtjipuxfbpngcvtmziuicm` int DEFAULT NULL,\n `znckcmtdxiehocaqvmlkegnvwqhhdfms` int DEFAULT NULL,\n `fqogzvegiwyclplslwptyqlsimptzvzj` int DEFAULT NULL,\n `ticifxbnounmkhnnxetjejqkcstvgldt` int DEFAULT NULL,\n `fcfmmcovlhkfqwfapmpafdgbcnrhpwvl` int DEFAULT NULL,\n `izleasprtdtpwzvhkoujwrgouykskngn` int DEFAULT NULL,\n `fkvwfaeazrmegyouvfjqralslcuthxpq` int DEFAULT NULL,\n `rvtgtrbmjzlymaxxnygufckosigrcrxl` int DEFAULT NULL,\n `zvtdrklazjfdfjvghltzkbwtkzdhalpi` int DEFAULT NULL,\n `mpcnvutnxoadeiyozhuvovthptejtatt` int DEFAULT NULL,\n `rocckmoskolsoblusbvswrudruthnhio` int DEFAULT NULL,\n `xabukithmttjrxhchzlncjnwnvayebpo` int DEFAULT NULL,\n `zxgqnicriluwvqrfobxsrprjoaiefsyx` int DEFAULT NULL,\n `bgkskkoletvvqatwqecxlppchipfipbm` int DEFAULT NULL,\n `jwvrlrjjptwyefzgwyabigepnvwbeyvo` int DEFAULT NULL,\n `dbaxggabrrjeiswwnicipmkxenxjzumu` int DEFAULT NULL,\n `slzvxdmjdulmnhfniepbefvtxcavdhbv` int DEFAULT NULL,\n `ulqpbaqvozxesersyrgjzbxqfrkoaqng` int DEFAULT NULL,\n `niinqlnvfjpzvkrekionxnbmvkqcxknp` int DEFAULT NULL,\n `slsxbjcaetgvonddnibxstvksefegbsg` int DEFAULT NULL,\n `lalpiarcwgmqiccrbhccjjbpglxdsgmf` int DEFAULT NULL,\n `awhjzvhbtpionycdmvzakhuybigdwsot` int DEFAULT NULL,\n `uxnchwrzprnguyzwspmpyjfjmfovaosr` int DEFAULT NULL,\n `orjyeaygqhfauqpzknpirxzfzoqdglag` int DEFAULT NULL,\n `gclkxxelujnxdscymzdbqfhqriysuthk` int DEFAULT NULL,\n `qwongavsgwiamevgxsqtysobpwrewzjs` int DEFAULT NULL,\n `fjrqeweopishsnbghwxvxwprmtzbuvvl` int DEFAULT NULL,\n `ofpkukmndbiannlbtqzafdxceoghrdlh` int DEFAULT NULL,\n `xamemvnqcnjhahxgulncwqehnuaprine` int DEFAULT NULL,\n `rwzznfqxjmpovswcupajcsgsbgpwbcdb` int DEFAULT NULL,\n `zutqxysenxwgvttfecamoculdiidsshy` int DEFAULT NULL,\n `fubafzdimkkfpxljfvpsshwrvfwyfhkw` int DEFAULT NULL,\n `xkltmdfcbmghqqwkbhlbonggsvpnsqrp` int DEFAULT NULL,\n `qztybypvhqgtwqrigfkczeocklikfees` int DEFAULT NULL,\n `elqwaflevvqjwbfryfjajdnbvyuhqkgo` int DEFAULT NULL,\n `cpwgnynyswfihienwqmudlrmsguuizcf` int DEFAULT NULL,\n `kmkgopgiifxovcpmwezvblytdijqzwcm` int DEFAULT NULL,\n `hlsffyaksloqbjtmcogxkrhnvgdfvlws` int DEFAULT NULL,\n `owtbdjboguudsmeljanuxetbvvyhnkry` int DEFAULT NULL,\n `cgwsbkyfvwecsgcstxqazvigrvxxmvmb` int DEFAULT NULL,\n `wrefdmgdmwlrgghsmfuehgfgssltksli` int DEFAULT NULL,\n `kllapahrkhqaxysexumqbanaybsottbq` int DEFAULT NULL,\n `bdbapwsklwbvveqqjcpnzzemcalwsrws` int DEFAULT NULL,\n `yjirtevwtmuacbyvygtwwpccclgygjzm` int DEFAULT NULL,\n `ebwpmhxgldaycrtujyodcflwtphmlcjs` int DEFAULT NULL,\n `awbbbzwlegtyvwgyfypbydafhwpgqhvr` int DEFAULT NULL,\n `sdgueqwjudenqqvrcslrmxbgcdxfftdo` int DEFAULT NULL,\n `dxqzawfaquagarjpnazwniwwstkquque` int DEFAULT NULL,\n `fbkgfqqqtayozsjuzjqxmtzootraydrr` int DEFAULT NULL,\n `jfkwnmjjinjswfubumctspecmxvklqnz` int DEFAULT NULL,\n `ckzjfhdoqjdzpvyxjboiyvldfvrikybq` int DEFAULT NULL,\n `akckzladypuwnwmhcdgjzbcjfgqyykjn` int DEFAULT NULL,\n `mqzrmspaufueokqaussftqpqwgckjbrm` int DEFAULT NULL,\n `yhepgyaeckkzjrahzvnsabapypiilbvy` int DEFAULT NULL,\n `uzzxcyedblccvnfbmpkizrnunzhobpvn` int DEFAULT NULL,\n `mmscainaryzhckxdggnawqkztoesumda` int DEFAULT NULL,\n `yxjaajtaphlqbrqngkamkibxdbbotehv` int DEFAULT NULL,\n `lxyhksipwtyvnfdrrlzrmtpcejchujgw` int DEFAULT NULL,\n `rxfchzjarmwnfulauxcwlscxfvvxikjt` int DEFAULT NULL,\n `btofrezcikkcelppxbkvcrpzcezbaprj` int DEFAULT NULL,\n PRIMARY KEY (`zcldsswjalnwvntzwpwkelyiwhpnxebs`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"iwnntobuwmkuzncjkkxxbjyvktvzwcla\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["zcldsswjalnwvntzwpwkelyiwhpnxebs"],"columns":[{"name":"zcldsswjalnwvntzwpwkelyiwhpnxebs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"hftiuabvtuplaqdtdzrfuoriyswniodg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idgeaszhaufaskbcexzhyuwxdovdizce","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"raqooccyxupdlsthkdrvukcuxflabhih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nflbnhrjoaxcsbnczwgiqwqabqfgyikz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfahrzeotguonqvolkrumkclgkjvmzvt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nlmdabcrchwqwisbfqqwsndjqcltxlqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utrlgltfehihwzizqckthaanlgqylbyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdsbgujmgrcjnralhqawqusxkujuhish","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"trrccvhockkyvyofjxkpldlgsmvsapju","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zadeadyjxozddklnytyklpsfweijflbs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lofjszofzabnmfsacygswgayayyiatwr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhfhrxbzluekayattuwpngggypvimumz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhymduqlpaithjwbcmhpnvpdmolfeevd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"adywbvttclvocfgyaamoowdtqoxcsetx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxwlobytoofwngxtybuitmdtvvgwsayz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfphmtmawzgqeleyfmbwpodkyoqnmcff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhxtserjbhugnheksxeplvyoguhlrneo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzmsybwdluehumzcfgojrdsnzyhwmsua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrdwjuqszpvjlqboaamoccjzzlnrdbbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hgmpoayzlfmpfmzosdfyhakjsdftveya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"klseeueujhhotnuwqardsuwuanuwcqxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iptpahwdwwuocgcztqntogjdyrknbhzw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjnzaskwwitkwefyglnhrkbqmehraypw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvgmbfplkwahqfbsklcoexlcubfxlecu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lellmogbqtcndrvsthfxnjzkapftvlxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blrziancalsuakxivpyhuvsnhtuoxaji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmwzpwhqccyhhuprwjzecjzgxetaclzx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zstokfstlsrhkdrvopsdcgyxpkjddtqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjrxmkkniuspjqimadwrtgrcdurgoqcs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynqfqoxmbzmhxhcpnqbgehvcdsykznpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhqizyhklmxvzpgaveeitycrjtpjysrc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmfxocmtylyhbihwphmmgbwbamxwpxcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twlqcxgyvqamrqqwwrctzqtdbzhorock","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swpvgcjddhrtsvueehdchywaagzeeabk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kuoluvplitorpzcawhyjnqafqtpzrcgd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dabsrxfwlirkzwvtewrreepchgvqaweg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejjtkwqkkvsjixeorsodrybwiqnmxbrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zuzdixkgfhurtzlkgufdtwpjdnnudeip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"djzihpworqzoyggukwygnbuupdkughlo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aoifapuntzrtjipuxfbpngcvtmziuicm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znckcmtdxiehocaqvmlkegnvwqhhdfms","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqogzvegiwyclplslwptyqlsimptzvzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ticifxbnounmkhnnxetjejqkcstvgldt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcfmmcovlhkfqwfapmpafdgbcnrhpwvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"izleasprtdtpwzvhkoujwrgouykskngn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fkvwfaeazrmegyouvfjqralslcuthxpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rvtgtrbmjzlymaxxnygufckosigrcrxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvtdrklazjfdfjvghltzkbwtkzdhalpi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpcnvutnxoadeiyozhuvovthptejtatt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rocckmoskolsoblusbvswrudruthnhio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xabukithmttjrxhchzlncjnwnvayebpo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxgqnicriluwvqrfobxsrprjoaiefsyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bgkskkoletvvqatwqecxlppchipfipbm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwvrlrjjptwyefzgwyabigepnvwbeyvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbaxggabrrjeiswwnicipmkxenxjzumu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"slzvxdmjdulmnhfniepbefvtxcavdhbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ulqpbaqvozxesersyrgjzbxqfrkoaqng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"niinqlnvfjpzvkrekionxnbmvkqcxknp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"slsxbjcaetgvonddnibxstvksefegbsg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lalpiarcwgmqiccrbhccjjbpglxdsgmf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awhjzvhbtpionycdmvzakhuybigdwsot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxnchwrzprnguyzwspmpyjfjmfovaosr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orjyeaygqhfauqpzknpirxzfzoqdglag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gclkxxelujnxdscymzdbqfhqriysuthk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwongavsgwiamevgxsqtysobpwrewzjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjrqeweopishsnbghwxvxwprmtzbuvvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofpkukmndbiannlbtqzafdxceoghrdlh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xamemvnqcnjhahxgulncwqehnuaprine","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rwzznfqxjmpovswcupajcsgsbgpwbcdb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zutqxysenxwgvttfecamoculdiidsshy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fubafzdimkkfpxljfvpsshwrvfwyfhkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkltmdfcbmghqqwkbhlbonggsvpnsqrp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qztybypvhqgtwqrigfkczeocklikfees","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elqwaflevvqjwbfryfjajdnbvyuhqkgo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cpwgnynyswfihienwqmudlrmsguuizcf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmkgopgiifxovcpmwezvblytdijqzwcm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlsffyaksloqbjtmcogxkrhnvgdfvlws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owtbdjboguudsmeljanuxetbvvyhnkry","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgwsbkyfvwecsgcstxqazvigrvxxmvmb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wrefdmgdmwlrgghsmfuehgfgssltksli","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kllapahrkhqaxysexumqbanaybsottbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdbapwsklwbvveqqjcpnzzemcalwsrws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjirtevwtmuacbyvygtwwpccclgygjzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebwpmhxgldaycrtujyodcflwtphmlcjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awbbbzwlegtyvwgyfypbydafhwpgqhvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdgueqwjudenqqvrcslrmxbgcdxfftdo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxqzawfaquagarjpnazwniwwstkquque","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbkgfqqqtayozsjuzjqxmtzootraydrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jfkwnmjjinjswfubumctspecmxvklqnz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckzjfhdoqjdzpvyxjboiyvldfvrikybq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akckzladypuwnwmhcdgjzbcjfgqyykjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqzrmspaufueokqaussftqpqwgckjbrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhepgyaeckkzjrahzvnsabapypiilbvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzzxcyedblccvnfbmpkizrnunzhobpvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmscainaryzhckxdggnawqkztoesumda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxjaajtaphlqbrqngkamkibxdbbotehv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxyhksipwtyvnfdrrlzrmtpcejchujgw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxfchzjarmwnfulauxcwlscxfvvxikjt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"btofrezcikkcelppxbkvcrpzcezbaprj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668731,"databaseName":"models_schema","ddl":"CREATE TABLE `jklxyfguyfwhbanvhqbtokakczltxncj` (\n `ibqxiavsffarnerqiirqpikdqfacblsa` int NOT NULL,\n `dsneaywkivhqbkvkmchljtrciuaqoxff` int DEFAULT NULL,\n `kpiymkkjkxjzrwbhmxwycpmflueeenyk` int DEFAULT NULL,\n `jsrtrtsuidofdhsrfhqobzinihgsanmf` int DEFAULT NULL,\n `thggkahoyejxiuxnbizvmygbaruhhsfk` int DEFAULT NULL,\n `bgtqnqlegtgjrkmijmznhcmgbsgzgvvh` int DEFAULT NULL,\n `doovgumiyqolyztfooxmfdvnkiaovkee` int DEFAULT NULL,\n `plkeqmjwkfkjdepgligujqmwmpbumfcf` int DEFAULT NULL,\n `ghccizgvcwxkyzvuszjzegtibeefvvov` int DEFAULT NULL,\n `bnugehdjafxhlssaanrsdcsbirbannul` int DEFAULT NULL,\n `hzjfxpmfkulodorgecfnprhtktuwsvrf` int DEFAULT NULL,\n `oreeiwzmyjdbflmnifemrvgyvigwctvd` int DEFAULT NULL,\n `jshhfmelsdqhngvyekxkkqulysqtfzky` int DEFAULT NULL,\n `gkutigxwowdqjkogudbaimyepcszolku` int DEFAULT NULL,\n `aiujnqgcpcwklxuqlshepdoyvaxzuazw` int DEFAULT NULL,\n `mejaskpixzdbfcdsggyxbfxuqxxmtzvf` int DEFAULT NULL,\n `cypcstpsxnxbnqbguyeuncphepjqgsle` int DEFAULT NULL,\n `pdydavibuktnbkakltwjxgsyajzxjmvc` int DEFAULT NULL,\n `nqwiyimojuimkhadkycbgduhwzggvdcw` int DEFAULT NULL,\n `cegkvrdayraivnxccxwtrplomjkkxfgd` int DEFAULT NULL,\n `kjnyrcdqzfsblwwoegytqwfywziuicty` int DEFAULT NULL,\n `qfjzrwfvvkxnrxriadrntzprnjihfoyx` int DEFAULT NULL,\n `cmpvwwfkxeopbekxsskyxomtgbvtqoyj` int DEFAULT NULL,\n `cruodiybnbfkzzmldesbqlqkycyfvpkn` int DEFAULT NULL,\n `gohmsxaxbdswryhxcnqneanmfyhwdyga` int DEFAULT NULL,\n `fqkhtljywbhygqnxzwaamdtodevrqdlw` int DEFAULT NULL,\n `lvffhtyxcsqziwnltxkvernetxdlasqx` int DEFAULT NULL,\n `nxisqlwjxwkgpbvwbjuniiulhrzqozjc` int DEFAULT NULL,\n `xdehihkeatdkyidmjfyhmswmxkhdhqdx` int DEFAULT NULL,\n `hmycaudhurhqjpmphygbhciwghrwwdcy` int DEFAULT NULL,\n `yfuvrbxydogtgvccqgirfbdmknkfijbn` int DEFAULT NULL,\n `kyrnxpyxewqaqrwefoqboqpjtrykivet` int DEFAULT NULL,\n `padmlbhpkmghrtxphcntcyfvmitrreri` int DEFAULT NULL,\n `gzghvionrkyvqyhnqwjqysdfwrpgzqok` int DEFAULT NULL,\n `zlemobdncofrjzubxlyupwabpejasbys` int DEFAULT NULL,\n `gaajfowokqyloaodwzdecgqgfylnyvbo` int DEFAULT NULL,\n `inbzjxruqkbvzhudrrlxgbsircfpqezq` int DEFAULT NULL,\n `fbfwewgmroxicocjlbmlxunlltimxhvc` int DEFAULT NULL,\n `fgcifwnkuoghuxmukqtpvgakiixttwle` int DEFAULT NULL,\n `kiexidbeisjwbubobnbgzendqbqyhzoi` int DEFAULT NULL,\n `lanzdrcmfbuneuxbkezvlouudvcpjwzh` int DEFAULT NULL,\n `oplebgshyanzqxoxrzsqkhexyncgqhkp` int DEFAULT NULL,\n `tahvehzswfqnksybjlyqulebhqursrmu` int DEFAULT NULL,\n `hoyrglpjfshortqfdgbltowxepgrbbkf` int DEFAULT NULL,\n `ehubphjyvleletnvvwmpvpyfvbdzxqep` int DEFAULT NULL,\n `wbsxfkadxfrcuxlsiectbanpqeizfvkh` int DEFAULT NULL,\n `lgwphfxhqhwsliwzdmqgrutatdclivdm` int DEFAULT NULL,\n `jsrriezoxomezrrussfslpzfadtjzmhk` int DEFAULT NULL,\n `owlwxyupjtzyofioiercocekihiofpyy` int DEFAULT NULL,\n `onsapkpwfiqvvrivizbnowmsiyxtsijt` int DEFAULT NULL,\n `evpsdwahhubhaaikpplrnfpqreumupno` int DEFAULT NULL,\n `ruqsfqiyhkmfelsrtujckglksswzxnaw` int DEFAULT NULL,\n `djnnqcymfotnipyphxxnvnrsfaeiqdhs` int DEFAULT NULL,\n `gykkvngxggnukvlnsdcpmfjvefjkciwe` int DEFAULT NULL,\n `xekhjlrnfaaljznnuprimazdkwhqkwof` int DEFAULT NULL,\n `ynesmxrewfpreiiiojpyfghturszuuqg` int DEFAULT NULL,\n `mlxyyrwqnypflejotkyexahbpgzwqoar` int DEFAULT NULL,\n `vccohnnhsjkrewcxkkcphyisxjtxzfee` int DEFAULT NULL,\n `ypzolqcnklvgnsvjdupyidltwpiuxfep` int DEFAULT NULL,\n `tlcrocktdtdmoppxifjgdbmsofqwytpz` int DEFAULT NULL,\n `jnixxoafqhgjfwtaiscgeepxrbuohxps` int DEFAULT NULL,\n `lpyxcjabqiqsxmfvknwtaqzsjhaishnu` int DEFAULT NULL,\n `ulqosqztpwfvevttdrabttbdeekgyzbj` int DEFAULT NULL,\n `nfdjvejiiozidwvbviwgijmxdsmqrgvq` int DEFAULT NULL,\n `dwemcjwyqccgcxvoockeeczgjixlyhmm` int DEFAULT NULL,\n `natfftrykrulyowjpshrzcmpvalboxxf` int DEFAULT NULL,\n `vrlylhtkpiiulljfpmtyxpvyqxxjfiqd` int DEFAULT NULL,\n `dvhfptcquytcvafwjmhxwopwpgxrjvhe` int DEFAULT NULL,\n `yokbxmaabaeidgpskfjgzefupbykwxbv` int DEFAULT NULL,\n `mxdnzzthofvycniepsbtijxwcrlqnmzs` int DEFAULT NULL,\n `xkylerpsyuwtcywrnxonkfevrmsggiwq` int DEFAULT NULL,\n `bphbehkkoxrawbcfuncnzmpmycfjzxys` int DEFAULT NULL,\n `sxsbjlobkevbznztruxlmopijtkbjryx` int DEFAULT NULL,\n `smhdplsqonhzufnbcffejiquuqnxpcol` int DEFAULT NULL,\n `wmztamwgbjjxnvszhscbtmqieycuglwc` int DEFAULT NULL,\n `okvymumotiuwidwtcpuhmeguejhppthu` int DEFAULT NULL,\n `lfjadgfawdkjmxkvxinmlohxlokaofzw` int DEFAULT NULL,\n `qvqfnvmcootxetgdepnpjkiubkbwugyy` int DEFAULT NULL,\n `rltyvqowrstzqpnvmbkocxlztxyczzfv` int DEFAULT NULL,\n `vtooxysuiskixlkneyulugjamvpwgdby` int DEFAULT NULL,\n `knovukmirvbxvhxqgjrjyexskdtdzovj` int DEFAULT NULL,\n `ieaogiytgpkeecuwqmggfxvicpfulvzp` int DEFAULT NULL,\n `iydmvpmbespeuuszmyuhmddbiparfmfb` int DEFAULT NULL,\n `cgcammrmmqqeakxrtvmhepwmgeqydfuw` int DEFAULT NULL,\n `etkkccnnhyufqzggcccasnsefzhuhssy` int DEFAULT NULL,\n `pdkbvobkletdprljgifnicjdefnkvsoi` int DEFAULT NULL,\n `aaxqxswnlrokanwigdhctyskgklnluxo` int DEFAULT NULL,\n `crdgoywdmcojlgnfxtgarxhhjattnoff` int DEFAULT NULL,\n `zczoyoegwnzapyclzjgumltanordwkze` int DEFAULT NULL,\n `nscjrddqxgqskaafwegwdjrcdczqkbch` int DEFAULT NULL,\n `busyuqlfvmnmikrjpgirrqbeomxwddod` int DEFAULT NULL,\n `mqszmtiochsraopqvdbdyfznghkonxll` int DEFAULT NULL,\n `cvqupdsmsifwnqgrqfzvmgcwbzprxnol` int DEFAULT NULL,\n `qzvomrkiwqxghllfajshrpdowdvhzaem` int DEFAULT NULL,\n `actweszxaueanhzzcalifnmgywwudokb` int DEFAULT NULL,\n `atrmjzmbpjbipcagqzoomjltytrhvvey` int DEFAULT NULL,\n `elvlnhpyjwvoqlvukltlclemvbdypyaa` int DEFAULT NULL,\n `qncribyfcefdgdtadpbfbjrsiknvmifi` int DEFAULT NULL,\n `owagelluawahbcxpyejpvyhzyuzkdboz` int DEFAULT NULL,\n `jpybjdzksesggjbnezpqwmvducpmiypd` int DEFAULT NULL,\n PRIMARY KEY (`ibqxiavsffarnerqiirqpikdqfacblsa`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"jklxyfguyfwhbanvhqbtokakczltxncj\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["ibqxiavsffarnerqiirqpikdqfacblsa"],"columns":[{"name":"ibqxiavsffarnerqiirqpikdqfacblsa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"dsneaywkivhqbkvkmchljtrciuaqoxff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpiymkkjkxjzrwbhmxwycpmflueeenyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jsrtrtsuidofdhsrfhqobzinihgsanmf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thggkahoyejxiuxnbizvmygbaruhhsfk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bgtqnqlegtgjrkmijmznhcmgbsgzgvvh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"doovgumiyqolyztfooxmfdvnkiaovkee","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"plkeqmjwkfkjdepgligujqmwmpbumfcf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghccizgvcwxkyzvuszjzegtibeefvvov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnugehdjafxhlssaanrsdcsbirbannul","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hzjfxpmfkulodorgecfnprhtktuwsvrf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oreeiwzmyjdbflmnifemrvgyvigwctvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jshhfmelsdqhngvyekxkkqulysqtfzky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkutigxwowdqjkogudbaimyepcszolku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aiujnqgcpcwklxuqlshepdoyvaxzuazw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mejaskpixzdbfcdsggyxbfxuqxxmtzvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cypcstpsxnxbnqbguyeuncphepjqgsle","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdydavibuktnbkakltwjxgsyajzxjmvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqwiyimojuimkhadkycbgduhwzggvdcw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cegkvrdayraivnxccxwtrplomjkkxfgd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjnyrcdqzfsblwwoegytqwfywziuicty","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfjzrwfvvkxnrxriadrntzprnjihfoyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmpvwwfkxeopbekxsskyxomtgbvtqoyj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cruodiybnbfkzzmldesbqlqkycyfvpkn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gohmsxaxbdswryhxcnqneanmfyhwdyga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqkhtljywbhygqnxzwaamdtodevrqdlw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvffhtyxcsqziwnltxkvernetxdlasqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxisqlwjxwkgpbvwbjuniiulhrzqozjc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdehihkeatdkyidmjfyhmswmxkhdhqdx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmycaudhurhqjpmphygbhciwghrwwdcy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfuvrbxydogtgvccqgirfbdmknkfijbn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kyrnxpyxewqaqrwefoqboqpjtrykivet","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"padmlbhpkmghrtxphcntcyfvmitrreri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzghvionrkyvqyhnqwjqysdfwrpgzqok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zlemobdncofrjzubxlyupwabpejasbys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gaajfowokqyloaodwzdecgqgfylnyvbo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"inbzjxruqkbvzhudrrlxgbsircfpqezq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbfwewgmroxicocjlbmlxunlltimxhvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fgcifwnkuoghuxmukqtpvgakiixttwle","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kiexidbeisjwbubobnbgzendqbqyhzoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lanzdrcmfbuneuxbkezvlouudvcpjwzh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oplebgshyanzqxoxrzsqkhexyncgqhkp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tahvehzswfqnksybjlyqulebhqursrmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hoyrglpjfshortqfdgbltowxepgrbbkf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehubphjyvleletnvvwmpvpyfvbdzxqep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbsxfkadxfrcuxlsiectbanpqeizfvkh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgwphfxhqhwsliwzdmqgrutatdclivdm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jsrriezoxomezrrussfslpzfadtjzmhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owlwxyupjtzyofioiercocekihiofpyy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"onsapkpwfiqvvrivizbnowmsiyxtsijt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evpsdwahhubhaaikpplrnfpqreumupno","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ruqsfqiyhkmfelsrtujckglksswzxnaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"djnnqcymfotnipyphxxnvnrsfaeiqdhs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gykkvngxggnukvlnsdcpmfjvefjkciwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xekhjlrnfaaljznnuprimazdkwhqkwof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynesmxrewfpreiiiojpyfghturszuuqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlxyyrwqnypflejotkyexahbpgzwqoar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vccohnnhsjkrewcxkkcphyisxjtxzfee","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypzolqcnklvgnsvjdupyidltwpiuxfep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlcrocktdtdmoppxifjgdbmsofqwytpz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jnixxoafqhgjfwtaiscgeepxrbuohxps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lpyxcjabqiqsxmfvknwtaqzsjhaishnu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ulqosqztpwfvevttdrabttbdeekgyzbj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfdjvejiiozidwvbviwgijmxdsmqrgvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwemcjwyqccgcxvoockeeczgjixlyhmm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"natfftrykrulyowjpshrzcmpvalboxxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrlylhtkpiiulljfpmtyxpvyqxxjfiqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvhfptcquytcvafwjmhxwopwpgxrjvhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yokbxmaabaeidgpskfjgzefupbykwxbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mxdnzzthofvycniepsbtijxwcrlqnmzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkylerpsyuwtcywrnxonkfevrmsggiwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bphbehkkoxrawbcfuncnzmpmycfjzxys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sxsbjlobkevbznztruxlmopijtkbjryx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smhdplsqonhzufnbcffejiquuqnxpcol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmztamwgbjjxnvszhscbtmqieycuglwc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okvymumotiuwidwtcpuhmeguejhppthu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfjadgfawdkjmxkvxinmlohxlokaofzw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvqfnvmcootxetgdepnpjkiubkbwugyy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rltyvqowrstzqpnvmbkocxlztxyczzfv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtooxysuiskixlkneyulugjamvpwgdby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knovukmirvbxvhxqgjrjyexskdtdzovj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ieaogiytgpkeecuwqmggfxvicpfulvzp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iydmvpmbespeuuszmyuhmddbiparfmfb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgcammrmmqqeakxrtvmhepwmgeqydfuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"etkkccnnhyufqzggcccasnsefzhuhssy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdkbvobkletdprljgifnicjdefnkvsoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aaxqxswnlrokanwigdhctyskgklnluxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crdgoywdmcojlgnfxtgarxhhjattnoff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zczoyoegwnzapyclzjgumltanordwkze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nscjrddqxgqskaafwegwdjrcdczqkbch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"busyuqlfvmnmikrjpgirrqbeomxwddod","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqszmtiochsraopqvdbdyfznghkonxll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvqupdsmsifwnqgrqfzvmgcwbzprxnol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzvomrkiwqxghllfajshrpdowdvhzaem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"actweszxaueanhzzcalifnmgywwudokb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atrmjzmbpjbipcagqzoomjltytrhvvey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elvlnhpyjwvoqlvukltlclemvbdypyaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qncribyfcefdgdtadpbfbjrsiknvmifi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owagelluawahbcxpyejpvyhzyuzkdboz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpybjdzksesggjbnezpqwmvducpmiypd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668763,"databaseName":"models_schema","ddl":"CREATE TABLE `jnnbrmmdrkoflyyvppyijnmfxbdveonu` (\n `wunooyehcpcgtitbrmhscnamyjmmpxdh` int NOT NULL,\n `nqqkvixnpxithgffblgktehpcvbhsrqx` int DEFAULT NULL,\n `pnkvnynxchdakslvjjoyqohrfachchdc` int DEFAULT NULL,\n `hgqiodtdakmrinxbxeobvzzndqwvbbpd` int DEFAULT NULL,\n `jebufehymlvmevmrmgstmaqknyozvkov` int DEFAULT NULL,\n `dpdsypxwizowjqnvgmaajupbcagyekoe` int DEFAULT NULL,\n `ekyshyfqdtrahdbmywueeacjhcfezigs` int DEFAULT NULL,\n `foeopclxbsptpegewchxizavjnekwpgd` int DEFAULT NULL,\n `igyrdnsjayilxuucaxoskuwqvhkqaebm` int DEFAULT NULL,\n `cqsezcdqiwkqjovzfzxmghuzckkqaqsj` int DEFAULT NULL,\n `nfogzydrfkzivelyhonqxruexequijhc` int DEFAULT NULL,\n `hxdjkfgtschrsrpbwxcbnuquarbokxnf` int DEFAULT NULL,\n `wkzurmiagdnewjihkemlhjbckyuxslbh` int DEFAULT NULL,\n `ajmclmklovhkfwhjpxmadwrfmhakvqyu` int DEFAULT NULL,\n `bgmkspmcdegkmrmvdscayhygiduwxpkt` int DEFAULT NULL,\n `ozjiazbpzwtlrtonzofurnypwmhuxvwo` int DEFAULT NULL,\n `ytnvvshvqpdimjhujmufuesivkndfzlt` int DEFAULT NULL,\n `cujououyuaupwfkudlzsuxhxrnofvzkz` int DEFAULT NULL,\n `mkpuprysdkwlcvhdfvluqqqknfilkoak` int DEFAULT NULL,\n `gkucljtzuupfrgrbnpjuelmttrcemrxw` int DEFAULT NULL,\n `iwqhhwzmnjzimnqnnreozqotwnaejkpf` int DEFAULT NULL,\n `hobgxyscdkjlekxtbvyfvcokttmexwpm` int DEFAULT NULL,\n `whycdhbtyakikrjjvoklszasifbukgqj` int DEFAULT NULL,\n `vomjinaqocrgcfjnpqnofegqetlakhsr` int DEFAULT NULL,\n `dzpvmxkpvcjhirnjcxgybyjqmfvjtcot` int DEFAULT NULL,\n `umtibnqnsgjgthtaneobtjzyrknrelha` int DEFAULT NULL,\n `tgufgpkyvazxtwysishupxjxbhxacooi` int DEFAULT NULL,\n `ycxojthgcftbkbtmlpaxjwhlhvtbbxdi` int DEFAULT NULL,\n `cnfbohibwclupyfvhwtyceggiknzyfhy` int DEFAULT NULL,\n `aowwjkrihpvkenpbvzzjmybeqlktuezi` int DEFAULT NULL,\n `xjbktvhylqdszuhknxsotcmlwyjrbnhk` int DEFAULT NULL,\n `vzltnzabelrfcdzmjdhxxrumherhogak` int DEFAULT NULL,\n `uelevjmgntnowmcclqrncaqleixlkhzj` int DEFAULT NULL,\n `giiuyrsdjpvswwjumnddosbuswwroeek` int DEFAULT NULL,\n `safqpjqopffqtuicutnzvcjzddzmmekj` int DEFAULT NULL,\n `ynammgmwfkkfhbfxfnbpwmiwyebpfvdl` int DEFAULT NULL,\n `uzlsobpikxzqzrnmxcnmatgpacrvphgn` int DEFAULT NULL,\n `yqvctadcathhaidleplybizjvbfercip` int DEFAULT NULL,\n `yetynuuhlywgtpytzqfrseeecqtyymfo` int DEFAULT NULL,\n `czvbhrwrhigyzxxkcpkikmhlxbexwelh` int DEFAULT NULL,\n `bqzekgbhphoeiaillfrgjienfnfoeasl` int DEFAULT NULL,\n `ofnzgsteeqvejipkrsyeuvusvlcfpjtf` int DEFAULT NULL,\n `cwwvchdhinmykfanxmpdbldwzvdtynht` int DEFAULT NULL,\n `phynnhrktzzpzhepqaqdayavxezelmhk` int DEFAULT NULL,\n `afwnueoeywtnahdtdsptjykrgzkxxvog` int DEFAULT NULL,\n `hwfmillfyfvcnvulzhltriardfwdhmke` int DEFAULT NULL,\n `gvihadnmzsgoytoulyumquictczmibon` int DEFAULT NULL,\n `mjyhlqbuywrmpjgdskpsahiltsbavddg` int DEFAULT NULL,\n `ymlqbmohterdlewqqeqhdtuzfsoxpcgn` int DEFAULT NULL,\n `jjxmbstkzwqfqynkpjdimrbrqrbmodvd` int DEFAULT NULL,\n `kabilbcydfpxpandkkswwseenfzomqhj` int DEFAULT NULL,\n `qxdwizzmjoltvlazfuigyihtauhamwxj` int DEFAULT NULL,\n `ihznzezeivvlaydpmqlolusaylzzgjzf` int DEFAULT NULL,\n `zsgcpwnwwfyuwrzxgbcvkvlzskprkoet` int DEFAULT NULL,\n `vitcneyeidzptdtjxspxyniqlzlfvxqr` int DEFAULT NULL,\n `pzzjifrbocqrmvlqzrueekjfpyjotwzj` int DEFAULT NULL,\n `ractjatswaumpmuxecmjcncjjznigfto` int DEFAULT NULL,\n `rneghpamubgbpgwblahxbydoqlnqhclj` int DEFAULT NULL,\n `kxrokxxgqcefnhzmhmaryqitulyuukqp` int DEFAULT NULL,\n `vfzacrlybsxgorjhsfddwtxodppnpjzo` int DEFAULT NULL,\n `dyrwidtclpmtstgikzknzosldrfipkjd` int DEFAULT NULL,\n `taotwdqasrbvdeawikdjmottjqzhfjtw` int DEFAULT NULL,\n `sxhtcxvysidbdrbkvzdwjghujkenooho` int DEFAULT NULL,\n `kfqpumfyuwvryxzntgdaqdfcrkahzpka` int DEFAULT NULL,\n `ikyujsnwoitueircnfhttlasfkwqtckd` int DEFAULT NULL,\n `zuyakezrlovhwuaxhqgqzitzpckegycd` int DEFAULT NULL,\n `gqabiihpjwlxceldvkdqgjmqllfipvtb` int DEFAULT NULL,\n `cthqmolomhxvpwzusuqmmmluluzyveki` int DEFAULT NULL,\n `nnqhyddlcfvsuphtjepjuyvwbjdzzyng` int DEFAULT NULL,\n `pfxqgfoenuzzikvetlidhjvsvixjirbd` int DEFAULT NULL,\n `ohroopfwdtfkqisnqfkxtdwhafuytvvc` int DEFAULT NULL,\n `pvrttkmapccrslelydjbwwtxflwrnxqy` int DEFAULT NULL,\n `dyfxpflxuqzvsydignhrneiktguscboq` int DEFAULT NULL,\n `ibnbkfxkqhhslcmidfsyhlibrnoviyff` int DEFAULT NULL,\n `wpkwhyonkufuycynkirugotwjiyltnbf` int DEFAULT NULL,\n `harmxxwmjraffadckkggmrwwigpgkmrn` int DEFAULT NULL,\n `orauenxqzbbsvrenmrqruvjvzlcwjcao` int DEFAULT NULL,\n `qyzalasouvpjqpkzczroqilqkcxouddq` int DEFAULT NULL,\n `nmlsjejsmdwanleculclvqhekmxoxkek` int DEFAULT NULL,\n `yyzbdxrnqoyriahprcjakpsnsvwezfcu` int DEFAULT NULL,\n `oghfqhaulpfkgurfquuiccvompadrxij` int DEFAULT NULL,\n `fhkuxbnvguiyegrehbdreywxkqcwqfce` int DEFAULT NULL,\n `ecgrnaoktzeupzsrldqdhwsjomxqcnde` int DEFAULT NULL,\n `ujejvjduinkprchqnuzjfzhbmiyahhym` int DEFAULT NULL,\n `eovwelonrdjcopiggotqejbwakjjpkmw` int DEFAULT NULL,\n `ujvqplllnztinouheufowafmfnlpwwfb` int DEFAULT NULL,\n `tzqofgpefhmnojlofbbkcrudlcldbxzr` int DEFAULT NULL,\n `yadipimayueeqpmtcwvjlkaxgyiipeuw` int DEFAULT NULL,\n `dkfobgyfuzoargqhzvmrlwbmgrxrlogb` int DEFAULT NULL,\n `ojkosdspyyvwtsioubnvgedcstxvuwho` int DEFAULT NULL,\n `cacsrhkmcnieitmvpbuqwhaiawzoftui` int DEFAULT NULL,\n `krltuopwdcgovxvcyihvhxekawkibuyq` int DEFAULT NULL,\n `wucklplmjgfatyaegrdnpkrdjpjzvvtf` int DEFAULT NULL,\n `sdrricpknsueeejomnlnvmnfddixjkam` int DEFAULT NULL,\n `nkyurdnkrgptvohhartqmmuyzzsvtvgp` int DEFAULT NULL,\n `iydglrgrzdkgsaiktuhzrmskldbdndxi` int DEFAULT NULL,\n `qhspnnyfvqxjxhfwicvdqdwxvduokawc` int DEFAULT NULL,\n `nzphiwxcfbjivhyznrfcubkohqbdfsze` int DEFAULT NULL,\n `edayzlzxeueckguucikgfmezhjigazje` int DEFAULT NULL,\n `euxplodzirkoylyjnprvtbigdzzbkiuz` int DEFAULT NULL,\n PRIMARY KEY (`wunooyehcpcgtitbrmhscnamyjmmpxdh`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"jnnbrmmdrkoflyyvppyijnmfxbdveonu\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["wunooyehcpcgtitbrmhscnamyjmmpxdh"],"columns":[{"name":"wunooyehcpcgtitbrmhscnamyjmmpxdh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"nqqkvixnpxithgffblgktehpcvbhsrqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnkvnynxchdakslvjjoyqohrfachchdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hgqiodtdakmrinxbxeobvzzndqwvbbpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jebufehymlvmevmrmgstmaqknyozvkov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpdsypxwizowjqnvgmaajupbcagyekoe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekyshyfqdtrahdbmywueeacjhcfezigs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"foeopclxbsptpegewchxizavjnekwpgd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igyrdnsjayilxuucaxoskuwqvhkqaebm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqsezcdqiwkqjovzfzxmghuzckkqaqsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfogzydrfkzivelyhonqxruexequijhc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxdjkfgtschrsrpbwxcbnuquarbokxnf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wkzurmiagdnewjihkemlhjbckyuxslbh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajmclmklovhkfwhjpxmadwrfmhakvqyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bgmkspmcdegkmrmvdscayhygiduwxpkt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozjiazbpzwtlrtonzofurnypwmhuxvwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytnvvshvqpdimjhujmufuesivkndfzlt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cujououyuaupwfkudlzsuxhxrnofvzkz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkpuprysdkwlcvhdfvluqqqknfilkoak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkucljtzuupfrgrbnpjuelmttrcemrxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwqhhwzmnjzimnqnnreozqotwnaejkpf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hobgxyscdkjlekxtbvyfvcokttmexwpm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"whycdhbtyakikrjjvoklszasifbukgqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vomjinaqocrgcfjnpqnofegqetlakhsr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dzpvmxkpvcjhirnjcxgybyjqmfvjtcot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umtibnqnsgjgthtaneobtjzyrknrelha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tgufgpkyvazxtwysishupxjxbhxacooi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ycxojthgcftbkbtmlpaxjwhlhvtbbxdi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnfbohibwclupyfvhwtyceggiknzyfhy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aowwjkrihpvkenpbvzzjmybeqlktuezi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjbktvhylqdszuhknxsotcmlwyjrbnhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzltnzabelrfcdzmjdhxxrumherhogak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uelevjmgntnowmcclqrncaqleixlkhzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"giiuyrsdjpvswwjumnddosbuswwroeek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"safqpjqopffqtuicutnzvcjzddzmmekj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynammgmwfkkfhbfxfnbpwmiwyebpfvdl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzlsobpikxzqzrnmxcnmatgpacrvphgn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqvctadcathhaidleplybizjvbfercip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yetynuuhlywgtpytzqfrseeecqtyymfo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"czvbhrwrhigyzxxkcpkikmhlxbexwelh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqzekgbhphoeiaillfrgjienfnfoeasl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofnzgsteeqvejipkrsyeuvusvlcfpjtf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwwvchdhinmykfanxmpdbldwzvdtynht","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phynnhrktzzpzhepqaqdayavxezelmhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afwnueoeywtnahdtdsptjykrgzkxxvog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwfmillfyfvcnvulzhltriardfwdhmke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gvihadnmzsgoytoulyumquictczmibon","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjyhlqbuywrmpjgdskpsahiltsbavddg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ymlqbmohterdlewqqeqhdtuzfsoxpcgn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjxmbstkzwqfqynkpjdimrbrqrbmodvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kabilbcydfpxpandkkswwseenfzomqhj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxdwizzmjoltvlazfuigyihtauhamwxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ihznzezeivvlaydpmqlolusaylzzgjzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zsgcpwnwwfyuwrzxgbcvkvlzskprkoet","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vitcneyeidzptdtjxspxyniqlzlfvxqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pzzjifrbocqrmvlqzrueekjfpyjotwzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ractjatswaumpmuxecmjcncjjznigfto","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rneghpamubgbpgwblahxbydoqlnqhclj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxrokxxgqcefnhzmhmaryqitulyuukqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vfzacrlybsxgorjhsfddwtxodppnpjzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dyrwidtclpmtstgikzknzosldrfipkjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"taotwdqasrbvdeawikdjmottjqzhfjtw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sxhtcxvysidbdrbkvzdwjghujkenooho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfqpumfyuwvryxzntgdaqdfcrkahzpka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikyujsnwoitueircnfhttlasfkwqtckd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zuyakezrlovhwuaxhqgqzitzpckegycd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqabiihpjwlxceldvkdqgjmqllfipvtb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cthqmolomhxvpwzusuqmmmluluzyveki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nnqhyddlcfvsuphtjepjuyvwbjdzzyng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pfxqgfoenuzzikvetlidhjvsvixjirbd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohroopfwdtfkqisnqfkxtdwhafuytvvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvrttkmapccrslelydjbwwtxflwrnxqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dyfxpflxuqzvsydignhrneiktguscboq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibnbkfxkqhhslcmidfsyhlibrnoviyff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wpkwhyonkufuycynkirugotwjiyltnbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"harmxxwmjraffadckkggmrwwigpgkmrn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orauenxqzbbsvrenmrqruvjvzlcwjcao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qyzalasouvpjqpkzczroqilqkcxouddq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmlsjejsmdwanleculclvqhekmxoxkek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yyzbdxrnqoyriahprcjakpsnsvwezfcu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oghfqhaulpfkgurfquuiccvompadrxij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhkuxbnvguiyegrehbdreywxkqcwqfce","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecgrnaoktzeupzsrldqdhwsjomxqcnde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujejvjduinkprchqnuzjfzhbmiyahhym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eovwelonrdjcopiggotqejbwakjjpkmw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujvqplllnztinouheufowafmfnlpwwfb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tzqofgpefhmnojlofbbkcrudlcldbxzr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yadipimayueeqpmtcwvjlkaxgyiipeuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkfobgyfuzoargqhzvmrlwbmgrxrlogb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojkosdspyyvwtsioubnvgedcstxvuwho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cacsrhkmcnieitmvpbuqwhaiawzoftui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krltuopwdcgovxvcyihvhxekawkibuyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wucklplmjgfatyaegrdnpkrdjpjzvvtf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdrricpknsueeejomnlnvmnfddixjkam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkyurdnkrgptvohhartqmmuyzzsvtvgp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iydglrgrzdkgsaiktuhzrmskldbdndxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhspnnyfvqxjxhfwicvdqdwxvduokawc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzphiwxcfbjivhyznrfcubkohqbdfsze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edayzlzxeueckguucikgfmezhjigazje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"euxplodzirkoylyjnprvtbigdzzbkiuz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668793,"databaseName":"models_schema","ddl":"CREATE TABLE `jwaehjgccilncrxqdqdzytzyjdhdrtfi` (\n `ikxvuzwufjkmlyqhcshbeyxpwgumppei` int NOT NULL,\n `kyokwitzbxswqfuhufaqqvvgugzxbmzz` int DEFAULT NULL,\n `orjpcycwvupdenjwlhttxulmcarygewx` int DEFAULT NULL,\n `lnkmwjqcfrfixijqsvqdqrfqcfoamxat` int DEFAULT NULL,\n `wfhuowxejjdmzmnwiiiiabstunszpoih` int DEFAULT NULL,\n `ivolysrwhpuybvzdtowbsywcvulpuics` int DEFAULT NULL,\n `jovjddxjaselpttextmeloesipmglmpk` int DEFAULT NULL,\n `zfcurzmfqezzebjbdybwkvvgfebptfqv` int DEFAULT NULL,\n `fatbmodkvohpeevflsojalktntaodvrb` int DEFAULT NULL,\n `aabrctxcertvbasqzryzqhpetzpuwnsc` int DEFAULT NULL,\n `wrztnchvudvagvjveipsjnmjlvcvvwdq` int DEFAULT NULL,\n `oowiclcbfbqftgqpecqmwrzztgiwwjmu` int DEFAULT NULL,\n `obkuhbkkntocvxcapumarwrbzwdwsljz` int DEFAULT NULL,\n `zjfxtjnmqklagficrizecscocnefydzs` int DEFAULT NULL,\n `mtqwgwptplxbryzngvbovtftsqkbbntc` int DEFAULT NULL,\n `ocitnwzcgjucqjjczxoruoshkpdubydc` int DEFAULT NULL,\n `pxxjeywembzyvtoxllzjlgporqitvogf` int DEFAULT NULL,\n `cdsrgoobdtqizmeqfdxszjkgereutpqp` int DEFAULT NULL,\n `eunbfuawnjoqvghlojosztiiohxneunv` int DEFAULT NULL,\n `oyaxzgybfdiwathziqsjmtzfijcnqxni` int DEFAULT NULL,\n `piwusynabdmcwjvpuergeklaphvwnvwk` int DEFAULT NULL,\n `ukvcjjqfgfizfqdzdmbgxtzhhnnebylg` int DEFAULT NULL,\n `sllkbigxcsuyppnaiiyblsujxdevjaau` int DEFAULT NULL,\n `rzgfhnmaknlycqwkfmpnybhbmqjuixyv` int DEFAULT NULL,\n `iepdhlekdbvsrxhkelifkwstrtiekhiy` int DEFAULT NULL,\n `gaslgbsmcnnekohqnflwnrcvebcwirlr` int DEFAULT NULL,\n `eeayvqhpkujjhzwzwfnnxbjdmzbftftx` int DEFAULT NULL,\n `oxsztblquccvmgiqkkgceoztdwqytjav` int DEFAULT NULL,\n `wmglrllgbgpahinmnhwmhknfvoybegxm` int DEFAULT NULL,\n `rmcwzkhfbrgpczacyslsgrrmfozhwzev` int DEFAULT NULL,\n `ekzbxnslanxzejvojzeaomevcqspnhtc` int DEFAULT NULL,\n `yzbimekncwtbdnwxzcqgnvdpqbwqmalg` int DEFAULT NULL,\n `awatssclxdschmjmtwdpiktwgfyyjrbm` int DEFAULT NULL,\n `axzuzeigugwuevxqolbueorrrzeyufzy` int DEFAULT NULL,\n `okolctosffzihacmbvywizatwvuyszal` int DEFAULT NULL,\n `rtouvmpbukjjbjxgvniyoaxwqbckeczw` int DEFAULT NULL,\n `nypwnoqzscbxljwbnkriqywbjqmzachb` int DEFAULT NULL,\n `cewkmtokgjzwyvcregnnezhitworjniq` int DEFAULT NULL,\n `uzplurgvkxbpijragkzqtqfpolpbbmxd` int DEFAULT NULL,\n `zjgkhpzluqallextxsqzedrgffvvrvzf` int DEFAULT NULL,\n `xjujcdxgdwrsaqlphsgxezblxjumicun` int DEFAULT NULL,\n `qxlzxcutkfyhermlvlulsoxbblxkdewm` int DEFAULT NULL,\n `dzyleuuwdwuklfpoouqjpmbytujfdrae` int DEFAULT NULL,\n `axeeeekfvznpucxhypdavlqseatkozgq` int DEFAULT NULL,\n `fyiyezwssqkmzcpmdqugeimuiqqqpfyu` int DEFAULT NULL,\n `ehajdtztmezdcmzwhadpjvmftysbdqab` int DEFAULT NULL,\n `gebezzbbupanmbqvddjsbjjjwduzejsd` int DEFAULT NULL,\n `hfgynwcokqlfijnctruqhgtddnwmdhae` int DEFAULT NULL,\n `tkmqnmvsbgaewjcjlcgwpoxyrhtwqhuw` int DEFAULT NULL,\n `ehnkclesartdpdheatlakfnmfadrujcz` int DEFAULT NULL,\n `gwtezqvjfhijsjjmjajntxrtorparapd` int DEFAULT NULL,\n `ahnypptfezwqzatrhpzxaltbesczszyl` int DEFAULT NULL,\n `ssczprkydxykqrsbpadbrxqzkdhjcthp` int DEFAULT NULL,\n `jdfwskgldeaptwcvatjyyysptuwdomvr` int DEFAULT NULL,\n `xsnioxtgigbxhvdvsiuwlvtmalckeibu` int DEFAULT NULL,\n `alcevksgqlhgkhqqelyjzijxcpogyova` int DEFAULT NULL,\n `edbkwkycagozrvemhlfyrpxkfozwngxi` int DEFAULT NULL,\n `pyfewffgswzrcgkclyahxhfmtuvrhtys` int DEFAULT NULL,\n `xusuitjmyxqmfqgldusvnqyskbxkmcsi` int DEFAULT NULL,\n `zbyojloahyxmshlagnjetlwcknpzjsgo` int DEFAULT NULL,\n `uaumxuxbnbvjsulwgvhlgtqxshwxsnfy` int DEFAULT NULL,\n `amguzjmzslukogpidatmneduutgbfabh` int DEFAULT NULL,\n `ucartsapmfkqxjkclavgyvgkodwweizh` int DEFAULT NULL,\n `xpyyysehszsjactseriweqlqwxkvkirm` int DEFAULT NULL,\n `ctyoerhonmtjhybknzmhpjlyqjlwbscw` int DEFAULT NULL,\n `hntnxyeoqenbidwqeapaufrwomisriod` int DEFAULT NULL,\n `lukkxkbghvtbzhiyeojxyiutqgykklvs` int DEFAULT NULL,\n `weqspyrlmewmoujjqoynemmpbxzjmqwn` int DEFAULT NULL,\n `fajjrcrzcjecqhfgblfeawhxbdvtzchh` int DEFAULT NULL,\n `hhusnrrjrykszplbkzhjjchmgjwqdjgc` int DEFAULT NULL,\n `rivxpyymlsrjqqikemasygdoepbpczly` int DEFAULT NULL,\n `pbaglwnvczdeucuuisofgqvortomqiuz` int DEFAULT NULL,\n `jrliiouokcrkwcuzwuogkigvcyvozjcr` int DEFAULT NULL,\n `ztjfxkgszgiqrtazitfwrikzwlfbsfmr` int DEFAULT NULL,\n `aehmskdahmyuuwiwrvvnofrmewpftwzj` int DEFAULT NULL,\n `knhcafhyqjvxyasyvviscgyvuhitviyi` int DEFAULT NULL,\n `nhkcloftelrfxircftyxddqktaobinem` int DEFAULT NULL,\n `uifnurgbhxroyeyjsxbgfdobkqoxesnb` int DEFAULT NULL,\n `unrffdzdosqzfltxkjmkgtsezcclasbe` int DEFAULT NULL,\n `vhnykbddbgbaiikmlcgaoeyugwkodbwc` int DEFAULT NULL,\n `hmurbjgsckkkhysuwqrsruclwittwauv` int DEFAULT NULL,\n `xqyxyxwxxpndzacsvlnhhaipqukjqbnr` int DEFAULT NULL,\n `vnajzwkdcleqvbsbherzamnenvbdekbg` int DEFAULT NULL,\n `ydkvhwusigvljamutojzilwsecfoztko` int DEFAULT NULL,\n `tmeeubjbmkztclxbfcnlhhocwfuagikv` int DEFAULT NULL,\n `pmvilzzthzbqsdilxiemlafyjhjeefpc` int DEFAULT NULL,\n `kwzqliwlbvycasewmnjrkjkkwsgcyayf` int DEFAULT NULL,\n `zmnuybhyksbjdjoujgtvjcdgvkyfnwgb` int DEFAULT NULL,\n `uowueqqiwfspouspvbbokcvlvlhqxqwr` int DEFAULT NULL,\n `fdkmpilzeoybzvuzvgskieiqzuidqgzi` int DEFAULT NULL,\n `gjryrmwaxfjdcbchwbpyoncfoxseflpy` int DEFAULT NULL,\n `rjypulguhecrfjzvimfabxhqegdakdbo` int DEFAULT NULL,\n `ywgkqxuapsfbaxkiiwtfnjqsxdylimxg` int DEFAULT NULL,\n `umsolajdmxjonxxnrwhaoudwsciblujr` int DEFAULT NULL,\n `jhqmkrhtiosyqjcnkxzsgxhcrehzsdca` int DEFAULT NULL,\n `ythviazuhoomzzlmamtcjznjmvczeamp` int DEFAULT NULL,\n `bsafpxjzwzjwsmkkwfpsujttyevbdzck` int DEFAULT NULL,\n `qulnoodqmxlvsrproyjkxyrlqrvotiqs` int DEFAULT NULL,\n `ceerytjsrsidgxwwpmluanpszilbmwkf` int DEFAULT NULL,\n `dcojftjkxhdphyjoaonfogblvrnyvgxq` int DEFAULT NULL,\n PRIMARY KEY (`ikxvuzwufjkmlyqhcshbeyxpwgumppei`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"jwaehjgccilncrxqdqdzytzyjdhdrtfi\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["ikxvuzwufjkmlyqhcshbeyxpwgumppei"],"columns":[{"name":"ikxvuzwufjkmlyqhcshbeyxpwgumppei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"kyokwitzbxswqfuhufaqqvvgugzxbmzz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orjpcycwvupdenjwlhttxulmcarygewx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lnkmwjqcfrfixijqsvqdqrfqcfoamxat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfhuowxejjdmzmnwiiiiabstunszpoih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivolysrwhpuybvzdtowbsywcvulpuics","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jovjddxjaselpttextmeloesipmglmpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zfcurzmfqezzebjbdybwkvvgfebptfqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fatbmodkvohpeevflsojalktntaodvrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aabrctxcertvbasqzryzqhpetzpuwnsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wrztnchvudvagvjveipsjnmjlvcvvwdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oowiclcbfbqftgqpecqmwrzztgiwwjmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obkuhbkkntocvxcapumarwrbzwdwsljz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjfxtjnmqklagficrizecscocnefydzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtqwgwptplxbryzngvbovtftsqkbbntc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocitnwzcgjucqjjczxoruoshkpdubydc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxxjeywembzyvtoxllzjlgporqitvogf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cdsrgoobdtqizmeqfdxszjkgereutpqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eunbfuawnjoqvghlojosztiiohxneunv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oyaxzgybfdiwathziqsjmtzfijcnqxni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"piwusynabdmcwjvpuergeklaphvwnvwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ukvcjjqfgfizfqdzdmbgxtzhhnnebylg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sllkbigxcsuyppnaiiyblsujxdevjaau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzgfhnmaknlycqwkfmpnybhbmqjuixyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iepdhlekdbvsrxhkelifkwstrtiekhiy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gaslgbsmcnnekohqnflwnrcvebcwirlr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eeayvqhpkujjhzwzwfnnxbjdmzbftftx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oxsztblquccvmgiqkkgceoztdwqytjav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmglrllgbgpahinmnhwmhknfvoybegxm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rmcwzkhfbrgpczacyslsgrrmfozhwzev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekzbxnslanxzejvojzeaomevcqspnhtc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzbimekncwtbdnwxzcqgnvdpqbwqmalg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awatssclxdschmjmtwdpiktwgfyyjrbm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axzuzeigugwuevxqolbueorrrzeyufzy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okolctosffzihacmbvywizatwvuyszal","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtouvmpbukjjbjxgvniyoaxwqbckeczw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nypwnoqzscbxljwbnkriqywbjqmzachb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cewkmtokgjzwyvcregnnezhitworjniq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzplurgvkxbpijragkzqtqfpolpbbmxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjgkhpzluqallextxsqzedrgffvvrvzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjujcdxgdwrsaqlphsgxezblxjumicun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxlzxcutkfyhermlvlulsoxbblxkdewm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dzyleuuwdwuklfpoouqjpmbytujfdrae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axeeeekfvznpucxhypdavlqseatkozgq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fyiyezwssqkmzcpmdqugeimuiqqqpfyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehajdtztmezdcmzwhadpjvmftysbdqab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gebezzbbupanmbqvddjsbjjjwduzejsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfgynwcokqlfijnctruqhgtddnwmdhae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkmqnmvsbgaewjcjlcgwpoxyrhtwqhuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehnkclesartdpdheatlakfnmfadrujcz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gwtezqvjfhijsjjmjajntxrtorparapd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahnypptfezwqzatrhpzxaltbesczszyl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ssczprkydxykqrsbpadbrxqzkdhjcthp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdfwskgldeaptwcvatjyyysptuwdomvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsnioxtgigbxhvdvsiuwlvtmalckeibu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"alcevksgqlhgkhqqelyjzijxcpogyova","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edbkwkycagozrvemhlfyrpxkfozwngxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pyfewffgswzrcgkclyahxhfmtuvrhtys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xusuitjmyxqmfqgldusvnqyskbxkmcsi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbyojloahyxmshlagnjetlwcknpzjsgo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uaumxuxbnbvjsulwgvhlgtqxshwxsnfy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amguzjmzslukogpidatmneduutgbfabh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucartsapmfkqxjkclavgyvgkodwweizh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpyyysehszsjactseriweqlqwxkvkirm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctyoerhonmtjhybknzmhpjlyqjlwbscw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hntnxyeoqenbidwqeapaufrwomisriod","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lukkxkbghvtbzhiyeojxyiutqgykklvs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"weqspyrlmewmoujjqoynemmpbxzjmqwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fajjrcrzcjecqhfgblfeawhxbdvtzchh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhusnrrjrykszplbkzhjjchmgjwqdjgc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rivxpyymlsrjqqikemasygdoepbpczly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbaglwnvczdeucuuisofgqvortomqiuz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jrliiouokcrkwcuzwuogkigvcyvozjcr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ztjfxkgszgiqrtazitfwrikzwlfbsfmr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aehmskdahmyuuwiwrvvnofrmewpftwzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knhcafhyqjvxyasyvviscgyvuhitviyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhkcloftelrfxircftyxddqktaobinem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uifnurgbhxroyeyjsxbgfdobkqoxesnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unrffdzdosqzfltxkjmkgtsezcclasbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhnykbddbgbaiikmlcgaoeyugwkodbwc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmurbjgsckkkhysuwqrsruclwittwauv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqyxyxwxxpndzacsvlnhhaipqukjqbnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnajzwkdcleqvbsbherzamnenvbdekbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydkvhwusigvljamutojzilwsecfoztko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmeeubjbmkztclxbfcnlhhocwfuagikv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmvilzzthzbqsdilxiemlafyjhjeefpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwzqliwlbvycasewmnjrkjkkwsgcyayf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmnuybhyksbjdjoujgtvjcdgvkyfnwgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uowueqqiwfspouspvbbokcvlvlhqxqwr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fdkmpilzeoybzvuzvgskieiqzuidqgzi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjryrmwaxfjdcbchwbpyoncfoxseflpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjypulguhecrfjzvimfabxhqegdakdbo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywgkqxuapsfbaxkiiwtfnjqsxdylimxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umsolajdmxjonxxnrwhaoudwsciblujr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jhqmkrhtiosyqjcnkxzsgxhcrehzsdca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ythviazuhoomzzlmamtcjznjmvczeamp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bsafpxjzwzjwsmkkwfpsujttyevbdzck","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qulnoodqmxlvsrproyjkxyrlqrvotiqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ceerytjsrsidgxwwpmluanpszilbmwkf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcojftjkxhdphyjoaonfogblvrnyvgxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668826,"databaseName":"models_schema","ddl":"CREATE TABLE `jwtirnhbsptqymlhedjwtfeddytgzipg` (\n `atgxbjcvepakurprykfqphzttwkcvslq` int NOT NULL,\n `puddilzhyzcrtdpboyqfixaesqccgdyt` int DEFAULT NULL,\n `dnbdqozjwaxdqrkpuzlbhexeljbthgqx` int DEFAULT NULL,\n `muhfgevnwssdvpgagxexvsztslgwadud` int DEFAULT NULL,\n `ypmdnwhkupntoryesjfrgnrgpaqaorjw` int DEFAULT NULL,\n `eotsboodgjbmuouazwqhqiywtuydxdxl` int DEFAULT NULL,\n `rijhnhgndgwjvnmhwodhgmsplexqtriw` int DEFAULT NULL,\n `qnnyxdkflfssyecakbmoyfxdhxphsmex` int DEFAULT NULL,\n `vutnebhonkimbluhreammztlysnvynhb` int DEFAULT NULL,\n `umoxagguleuluhmeaijbmfeheabgklwq` int DEFAULT NULL,\n `ctvghpixxjycpbhhwumnsbmijeqtmgnf` int DEFAULT NULL,\n `efjpbpsptbotbyhkvmhuhgyrbeivcsnu` int DEFAULT NULL,\n `mimsylvxbgtrlkqlisnuudviuiftrhsh` int DEFAULT NULL,\n `lbnrnudynejecvzjjpomkvdfdsrbyefo` int DEFAULT NULL,\n `mpeisbdxolgaaxxzbgfsywjuussklorj` int DEFAULT NULL,\n `edqtxabvqgmlsuxfguuomuvzdfgptbsv` int DEFAULT NULL,\n `cvrsszsnoczbqockhacpltnmtavywzot` int DEFAULT NULL,\n `ghprqrhjarlzxtillnjbwweasoxaujia` int DEFAULT NULL,\n `bqqtuyywiapydzrdlsbuchzojlirgwmk` int DEFAULT NULL,\n `rlgfrpjocrikaqqrpfqfhlkiwgbspznt` int DEFAULT NULL,\n `sidwluhvwgozriofnfpkdnefqztcbbfj` int DEFAULT NULL,\n `brohbcecgimapjpfqzxawngbjpqitfua` int DEFAULT NULL,\n `idyuzlmgyzkofbawwiqqvivlgpillkxx` int DEFAULT NULL,\n `vdyafivspeegnckoqijkffccvbaaikhq` int DEFAULT NULL,\n `nenxohgbltqbnlmzhuqrjosaqvrxyqfz` int DEFAULT NULL,\n `fcaouxkgemxoadgcywlnytxmipukfmji` int DEFAULT NULL,\n `jmedetyzjlpcvtrklounlzsxicwumtft` int DEFAULT NULL,\n `bmyllciaknmyuqziunolwcfgzbctuybp` int DEFAULT NULL,\n `bwdxhifkucleoyhjvjlsorufkskrlfel` int DEFAULT NULL,\n `cpkiwejhjxjzetyctrbohjyoorqwwzhw` int DEFAULT NULL,\n `srmsgrfeifnvrgeqkmiyzxoypyibwgtt` int DEFAULT NULL,\n `kvzgeyafghxjspcmrsamwnbljwoetgrm` int DEFAULT NULL,\n `sdqprcfatbzvrpkstulmtobhdtprkmam` int DEFAULT NULL,\n `kexttpungojjgkpeszgrbwwadrbadwsk` int DEFAULT NULL,\n `rfetkodovqmefqszlecghnhyelsowunm` int DEFAULT NULL,\n `gvnsbtzprrgzcznzoazdwoqcaigbjarr` int DEFAULT NULL,\n `icqjvpxxboiszntagyrrlsdrcpcawlyc` int DEFAULT NULL,\n `vbeqnutkryuvvnekrzfbvughcdfjwsgi` int DEFAULT NULL,\n `qglpbysmfjyfrghoukbyvoddydciwaex` int DEFAULT NULL,\n `joraqfmccbcxebuwvxvptxfdqsxulvxk` int DEFAULT NULL,\n `rappcjlaqrgagyafqnzovalqvpevtfme` int DEFAULT NULL,\n `chystuccluflulpmbydtkbfnjgkymfql` int DEFAULT NULL,\n `hshqrdgslbeqvodzyfqqjwuxppmzzhhs` int DEFAULT NULL,\n `ucnzadytjvkfuxcudbicsqvzlogfwsib` int DEFAULT NULL,\n `ggeqdfhpihlrqdqmwjuocyamkupupxpo` int DEFAULT NULL,\n `pitdslvnfjugwnxtxukjwjwqmsvatazh` int DEFAULT NULL,\n `ldjsuldbkqurreqhxerukvjhjepucoal` int DEFAULT NULL,\n `ykbfwtksbupdgqylstfxxrefxfsifvjg` int DEFAULT NULL,\n `svhgjfxstkcuhctuiucvhjtmvikpscci` int DEFAULT NULL,\n `jjykqepjoncgttkmyojqyhzakgeulwba` int DEFAULT NULL,\n `mmjfkaawgjoevpbtxhltgomnjrpylawi` int DEFAULT NULL,\n `oyeeveppqygiwcsbdnsmpmckooqfaonf` int DEFAULT NULL,\n `atntikkepjabnnzngadwrfhcrzclcxpv` int DEFAULT NULL,\n `itcktugprzkwvsbnaefuhrwjmgufzgbo` int DEFAULT NULL,\n `taumrlcvpyfrqtgjfniixufxjfbuhyif` int DEFAULT NULL,\n `jvlejcsdqgoijbuezgwvyrmtnapbezox` int DEFAULT NULL,\n `fgmpzqmkynapfqlwieifbjczjgmmxqkn` int DEFAULT NULL,\n `xtscyrzbqlurgfhnbvamaccqvjjqnlsl` int DEFAULT NULL,\n `kxuwqnsppmpzozeceuryyucnvzsjfzeg` int DEFAULT NULL,\n `zvxkrfmhpheptcqewmzzzgfdmgatejoj` int DEFAULT NULL,\n `junvjtlzslnhyoifglknqionzwabrfxc` int DEFAULT NULL,\n `vqjozstljsxjygwzdywrfnsehvstxzwy` int DEFAULT NULL,\n `xluxrmylxihebxcvrugbzftnfgxpwdvd` int DEFAULT NULL,\n `zwusvlzygwnihogdarqzxzipdcuzsjxx` int DEFAULT NULL,\n `vsxoxfcujiysiffgedwptpfkellpmazy` int DEFAULT NULL,\n `mapizremwwnpekbofbfwijkkjpuowboj` int DEFAULT NULL,\n `fltgsscssnpzinpmvrkfndyahvoadmqd` int DEFAULT NULL,\n `amlkstwwihiviegqxvbvtrbqhlbrajjb` int DEFAULT NULL,\n `vglyjzcushniqawvpvonzwmgrerkgeap` int DEFAULT NULL,\n `gliefyxlajlpujacjsntdgiognpxpfan` int DEFAULT NULL,\n `cjmvbrrpovzmodagnuisorlruczaizmu` int DEFAULT NULL,\n `hutgtdilktjuicdkokhpfpulrgjgofyf` int DEFAULT NULL,\n `fqqbsquqhlzymkyzwgagrgilxdzwktla` int DEFAULT NULL,\n `byyndjzmpiasbsaudcvpyfahpsnvymcc` int DEFAULT NULL,\n `oyovkmtwvelvmzbwvtodtignilpxklpf` int DEFAULT NULL,\n `nfqddvsdblnroqwtahrnwakttchiremk` int DEFAULT NULL,\n `exuofvxvwxioijcemmmdhostpmtticlw` int DEFAULT NULL,\n `aqqqxqqgrgzufxdahvtaqgjbtrnkjqpt` int DEFAULT NULL,\n `rzxcqohjsyfuyjsjyutthuxhccsblcyf` int DEFAULT NULL,\n `fwecxoxlilwqrekiqdrlteldoouhvcoi` int DEFAULT NULL,\n `ayndbxpzjgyrvhempesdpzxqrfnoekdp` int DEFAULT NULL,\n `wqdbyqzkfifetfqjvegvfjmrpppepwbp` int DEFAULT NULL,\n `nfsazkfelfgwcuyrrdajlizhokposasf` int DEFAULT NULL,\n `hqaelbzoycrrremmrehewuhubrjcjqlh` int DEFAULT NULL,\n `ijdyqefupjqcwfyjpcwrwswkdlbxrwit` int DEFAULT NULL,\n `nrcogvtydrffopyhfhqarvmcgqjdfwlb` int DEFAULT NULL,\n `uysxouhiduwsuorzgxovdbycxomvcdyv` int DEFAULT NULL,\n `kisinhniwvifgknxyouwexkkiychearl` int DEFAULT NULL,\n `xvmvjxgwizutkvndfnvypgtybisireip` int DEFAULT NULL,\n `smrluthrwmnfgwabkqhaxytnsexsehmb` int DEFAULT NULL,\n `cggdweojakcoiykjjvklzwdjsazuzbab` int DEFAULT NULL,\n `qdxrtgnswcmlsgfeltzthqvwttahtlre` int DEFAULT NULL,\n `wiszkwwrqwwmzfjgiyggnzfnarhhwyoz` int DEFAULT NULL,\n `mizufjgkhitiudhutrozshbtiimmooya` int DEFAULT NULL,\n `hxrzcamxyjmmveisftzofvdinnddxyzm` int DEFAULT NULL,\n `snjqzifessogkggbwqkbjakfgrvfejtl` int DEFAULT NULL,\n `rdlijziaovgkvjrcpprlquajuamzdzjf` int DEFAULT NULL,\n `moqglsmicbitipuxyjocnxmzcsjutbnh` int DEFAULT NULL,\n `vcnlynxavoqrupesaayzqkuuybjztxbf` int DEFAULT NULL,\n `bivwxlhtxvhdpoflttonhfutmkutxtdh` int DEFAULT NULL,\n PRIMARY KEY (`atgxbjcvepakurprykfqphzttwkcvslq`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"jwtirnhbsptqymlhedjwtfeddytgzipg\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["atgxbjcvepakurprykfqphzttwkcvslq"],"columns":[{"name":"atgxbjcvepakurprykfqphzttwkcvslq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"puddilzhyzcrtdpboyqfixaesqccgdyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dnbdqozjwaxdqrkpuzlbhexeljbthgqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"muhfgevnwssdvpgagxexvsztslgwadud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypmdnwhkupntoryesjfrgnrgpaqaorjw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eotsboodgjbmuouazwqhqiywtuydxdxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rijhnhgndgwjvnmhwodhgmsplexqtriw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qnnyxdkflfssyecakbmoyfxdhxphsmex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vutnebhonkimbluhreammztlysnvynhb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umoxagguleuluhmeaijbmfeheabgklwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctvghpixxjycpbhhwumnsbmijeqtmgnf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"efjpbpsptbotbyhkvmhuhgyrbeivcsnu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mimsylvxbgtrlkqlisnuudviuiftrhsh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lbnrnudynejecvzjjpomkvdfdsrbyefo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpeisbdxolgaaxxzbgfsywjuussklorj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edqtxabvqgmlsuxfguuomuvzdfgptbsv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvrsszsnoczbqockhacpltnmtavywzot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghprqrhjarlzxtillnjbwweasoxaujia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqqtuyywiapydzrdlsbuchzojlirgwmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlgfrpjocrikaqqrpfqfhlkiwgbspznt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sidwluhvwgozriofnfpkdnefqztcbbfj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"brohbcecgimapjpfqzxawngbjpqitfua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idyuzlmgyzkofbawwiqqvivlgpillkxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdyafivspeegnckoqijkffccvbaaikhq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nenxohgbltqbnlmzhuqrjosaqvrxyqfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcaouxkgemxoadgcywlnytxmipukfmji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmedetyzjlpcvtrklounlzsxicwumtft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bmyllciaknmyuqziunolwcfgzbctuybp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bwdxhifkucleoyhjvjlsorufkskrlfel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cpkiwejhjxjzetyctrbohjyoorqwwzhw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"srmsgrfeifnvrgeqkmiyzxoypyibwgtt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvzgeyafghxjspcmrsamwnbljwoetgrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdqprcfatbzvrpkstulmtobhdtprkmam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kexttpungojjgkpeszgrbwwadrbadwsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfetkodovqmefqszlecghnhyelsowunm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gvnsbtzprrgzcznzoazdwoqcaigbjarr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icqjvpxxboiszntagyrrlsdrcpcawlyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbeqnutkryuvvnekrzfbvughcdfjwsgi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qglpbysmfjyfrghoukbyvoddydciwaex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"joraqfmccbcxebuwvxvptxfdqsxulvxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rappcjlaqrgagyafqnzovalqvpevtfme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chystuccluflulpmbydtkbfnjgkymfql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hshqrdgslbeqvodzyfqqjwuxppmzzhhs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucnzadytjvkfuxcudbicsqvzlogfwsib","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggeqdfhpihlrqdqmwjuocyamkupupxpo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pitdslvnfjugwnxtxukjwjwqmsvatazh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ldjsuldbkqurreqhxerukvjhjepucoal","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykbfwtksbupdgqylstfxxrefxfsifvjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"svhgjfxstkcuhctuiucvhjtmvikpscci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjykqepjoncgttkmyojqyhzakgeulwba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmjfkaawgjoevpbtxhltgomnjrpylawi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oyeeveppqygiwcsbdnsmpmckooqfaonf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atntikkepjabnnzngadwrfhcrzclcxpv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itcktugprzkwvsbnaefuhrwjmgufzgbo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"taumrlcvpyfrqtgjfniixufxjfbuhyif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvlejcsdqgoijbuezgwvyrmtnapbezox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fgmpzqmkynapfqlwieifbjczjgmmxqkn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xtscyrzbqlurgfhnbvamaccqvjjqnlsl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxuwqnsppmpzozeceuryyucnvzsjfzeg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvxkrfmhpheptcqewmzzzgfdmgatejoj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"junvjtlzslnhyoifglknqionzwabrfxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqjozstljsxjygwzdywrfnsehvstxzwy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xluxrmylxihebxcvrugbzftnfgxpwdvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zwusvlzygwnihogdarqzxzipdcuzsjxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsxoxfcujiysiffgedwptpfkellpmazy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mapizremwwnpekbofbfwijkkjpuowboj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fltgsscssnpzinpmvrkfndyahvoadmqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amlkstwwihiviegqxvbvtrbqhlbrajjb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vglyjzcushniqawvpvonzwmgrerkgeap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gliefyxlajlpujacjsntdgiognpxpfan","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjmvbrrpovzmodagnuisorlruczaizmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hutgtdilktjuicdkokhpfpulrgjgofyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqqbsquqhlzymkyzwgagrgilxdzwktla","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"byyndjzmpiasbsaudcvpyfahpsnvymcc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oyovkmtwvelvmzbwvtodtignilpxklpf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfqddvsdblnroqwtahrnwakttchiremk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exuofvxvwxioijcemmmdhostpmtticlw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqqqxqqgrgzufxdahvtaqgjbtrnkjqpt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzxcqohjsyfuyjsjyutthuxhccsblcyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwecxoxlilwqrekiqdrlteldoouhvcoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ayndbxpzjgyrvhempesdpzxqrfnoekdp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqdbyqzkfifetfqjvegvfjmrpppepwbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfsazkfelfgwcuyrrdajlizhokposasf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqaelbzoycrrremmrehewuhubrjcjqlh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijdyqefupjqcwfyjpcwrwswkdlbxrwit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nrcogvtydrffopyhfhqarvmcgqjdfwlb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uysxouhiduwsuorzgxovdbycxomvcdyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kisinhniwvifgknxyouwexkkiychearl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvmvjxgwizutkvndfnvypgtybisireip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smrluthrwmnfgwabkqhaxytnsexsehmb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cggdweojakcoiykjjvklzwdjsazuzbab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdxrtgnswcmlsgfeltzthqvwttahtlre","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wiszkwwrqwwmzfjgiyggnzfnarhhwyoz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mizufjgkhitiudhutrozshbtiimmooya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxrzcamxyjmmveisftzofvdinnddxyzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"snjqzifessogkggbwqkbjakfgrvfejtl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdlijziaovgkvjrcpprlquajuamzdzjf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"moqglsmicbitipuxyjocnxmzcsjutbnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vcnlynxavoqrupesaayzqkuuybjztxbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bivwxlhtxvhdpoflttonhfutmkutxtdh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668862,"databaseName":"models_schema","ddl":"CREATE TABLE `jxtszwleobqkzaxtmowwchvvkrzyootc` (\n `efmgqhyeivzukzwluhosnhbdvskqacwl` int NOT NULL,\n `hxsxnvdtgwpehxbzcfssccqbwjpggdpk` int DEFAULT NULL,\n `mrapsyzbcqrfkgobetantcdtfjssypji` int DEFAULT NULL,\n `bomkkukholybohglgkvvqbuuytlrjzhl` int DEFAULT NULL,\n `kfrysrvutvxatpmpwwouichlzlfwvqhn` int DEFAULT NULL,\n `ysngmeyxnfqtsuziqybgoaqesuyuphra` int DEFAULT NULL,\n `liptvuucksmrkreerhzciqydibyzhccl` int DEFAULT NULL,\n `bejtxyuzvxotiddpuabchjlubcgapueo` int DEFAULT NULL,\n `vrimbihaayrwpcwdrmnrtveyzeoiaufv` int DEFAULT NULL,\n `zqhttmkurptfnbojwjzrxjyqmnfggllj` int DEFAULT NULL,\n `weupfiuiqpgplktsjyvseuomhtcjswdf` int DEFAULT NULL,\n `btiepvrmgtziunyhnubkrqbasqbwpgyw` int DEFAULT NULL,\n `ubqsfdgohrxlypyrmebwntluqryrjbdd` int DEFAULT NULL,\n `tiximrahkheafjrbuzppofktpjhbekgy` int DEFAULT NULL,\n `sjqorinxycjxnhugzliullnewkfaybqj` int DEFAULT NULL,\n `piwtjqwyudgneegneewqlhkdtqxsykss` int DEFAULT NULL,\n `pclfclrrpnouabmjfywvtnxpsgewdbcp` int DEFAULT NULL,\n `eipuzosgqrszajgiyiqplufvjlfljbdm` int DEFAULT NULL,\n `ybipvdcmnaulpxscwpomqsoaoenzvfqu` int DEFAULT NULL,\n `jelysxihawrbwpsjqrosmhyqtvjluqri` int DEFAULT NULL,\n `funiplxcotgouwbfxqdqxlxgsxctuazw` int DEFAULT NULL,\n `ixqcfnvfndnnnrzkyetiqfvtxwjsnohi` int DEFAULT NULL,\n `fqqhtrcnjzrwtmqrtjlortylterajecm` int DEFAULT NULL,\n `xmqycjupccoyqpyvkbkabahfkroqmmqt` int DEFAULT NULL,\n `dhatverstpscqrypjqobazwakxboumcv` int DEFAULT NULL,\n `khrdymvzszrfblueokwyvttlqpgyyphv` int DEFAULT NULL,\n `fklowuzpdrdszoxfvagzdkadjqtgbioj` int DEFAULT NULL,\n `pzxkxoycdkfwwcwmiqjykibqzpbfntbo` int DEFAULT NULL,\n `itjyttgxvevzgmlmnbfotprbswlnfoez` int DEFAULT NULL,\n `daxqebzltkaedrptajatkxwowkubeohq` int DEFAULT NULL,\n `hwaqwzbmbxnvtxsprcjpwggxvmjnepmz` int DEFAULT NULL,\n `qeedrqwbpggagyyndtujvcpomiyhhbbp` int DEFAULT NULL,\n `jjnmabmcyniprwrqjdqgqfytuzvypgdp` int DEFAULT NULL,\n `bqzfqmvrggiayfxwkpjdhtolytzbbnnx` int DEFAULT NULL,\n `ckwkhqocsdtzuwgyvkzutekeaeffikai` int DEFAULT NULL,\n `ivqlecfetirgfazfloxfocdgvnilfzlm` int DEFAULT NULL,\n `vrccxqinsrdgihjokixtoptxwnxydpyo` int DEFAULT NULL,\n `bbjmkdpyknrtqekslmpgtlsrrkowwpaf` int DEFAULT NULL,\n `kbcnjmxsroushuqrpcbmmgkzegkxyety` int DEFAULT NULL,\n `fmbuwltvdkwexnentrnfrzophluapsxu` int DEFAULT NULL,\n `hpqygdjbydkzfzewzsaxlrgmgunudsdl` int DEFAULT NULL,\n `kvmsatawfikhfgvshwbpodkgcxfjguoh` int DEFAULT NULL,\n `cydqjwyibaandmgqbtzusbakiexiubug` int DEFAULT NULL,\n `myuylpvvyonrzvsltwibtfplbcavzres` int DEFAULT NULL,\n `eyhxfkexlwgrpdufxkcdgamjkgfhhtcv` int DEFAULT NULL,\n `wbkqhxigpnpnyarwdgneqednuujmgjfj` int DEFAULT NULL,\n `nixffivpaldghyxagtpmnrykcbtszhfh` int DEFAULT NULL,\n `gxxvbylbutkiocrylecvyzkigbjtxrjn` int DEFAULT NULL,\n `ciltvlwyhgbgvrjebaraqyhdwvxeuqgp` int DEFAULT NULL,\n `sukyqrrpcqikhclxrkhbizfrqxruusoc` int DEFAULT NULL,\n `ltrolmckirmxsuucjzukcozhryxbixra` int DEFAULT NULL,\n `rutnawtihtqaimctirvibpwyrrpqzmxa` int DEFAULT NULL,\n `lzfvjoxsptedbinthywstanpivdddjsd` int DEFAULT NULL,\n `zknykrekxayzaogipaoobwcfqrcrgjss` int DEFAULT NULL,\n `amcmepbhyqztalsvkjxoaufvoewfbvno` int DEFAULT NULL,\n `logjxdnththbmeeriiggqquatzmagoru` int DEFAULT NULL,\n `olmxswbolqzjhqjzkqnsfsqhmiqhyeyu` int DEFAULT NULL,\n `zrbbrelcwrkuxgbldotugbjmpggghvda` int DEFAULT NULL,\n `rfmmjxvayctgbcivbqhpiirneyblsrzy` int DEFAULT NULL,\n `yvnwiftyhqsrhlspafamtsrumfdxjjev` int DEFAULT NULL,\n `uriljfmvmmihhihbeiiwtqstzwxosrxo` int DEFAULT NULL,\n `akrhkwhykejocznfjosvlscntinyqcoh` int DEFAULT NULL,\n `cqrzutyolymnvypslknwqhbfgxvfzwey` int DEFAULT NULL,\n `knpojkvfjdakqdjsauddsmqqdolfnilf` int DEFAULT NULL,\n `hmubkejcfxkmdwbsbaqbhdvctdgzedct` int DEFAULT NULL,\n `lygnkzgpdyleyjbjukbgjcpalphiuhgk` int DEFAULT NULL,\n `kyypdvvejfoietqrgiswnqroxuxrzxhe` int DEFAULT NULL,\n `kfaggnluavjwktaxruknpbuizyogqqkz` int DEFAULT NULL,\n `qosxrsarvrlolfwrzwpwyqwyibyzzbwn` int DEFAULT NULL,\n `blcxhtezacsotjlduuwgzaoivgfxznxj` int DEFAULT NULL,\n `yxthnkqglqddqaudncbewwbjurgmiguz` int DEFAULT NULL,\n `cukqbttcckoduxpnvufxcvggcigxqzau` int DEFAULT NULL,\n `ggoaakhqtyvtaopvhdcvxdmiicykynhe` int DEFAULT NULL,\n `irjmvthklvithbllflswmoiaqbwkofji` int DEFAULT NULL,\n `dlxbmycehwszywrsldkuqjvzamzbhyum` int DEFAULT NULL,\n `crypccrixuetoyyvpjwcfabtadeitpuk` int DEFAULT NULL,\n `dzqwaotogpyitsfgydrwzkygnhoeqhqg` int DEFAULT NULL,\n `hbztluxhhpayyfeepfpclcetkhjctjmj` int DEFAULT NULL,\n `gpmwgbvojysrgnrjejnxiogokzspyetj` int DEFAULT NULL,\n `qailkojpuxriqikyledpzaqpmhuemhur` int DEFAULT NULL,\n `moqmtkoxqfyuvpcegsceaazhdwydpzsf` int DEFAULT NULL,\n `lxqomjdrbdpdwwprvwfrprdaspiistkc` int DEFAULT NULL,\n `xcmnjfweyqkvbqksljkoskpvbmpczprs` int DEFAULT NULL,\n `otgrqdnyarzqqxtovamamsapqoajxghm` int DEFAULT NULL,\n `xfimwrvtexmcbgvrfaoyurvlpufqoqsc` int DEFAULT NULL,\n `zbcxukrksuyveuiucebhevqidwuxthon` int DEFAULT NULL,\n `hgoziggcodajglmqretrqrgoiotfadch` int DEFAULT NULL,\n `pzwstlllsfyluepazxhbxlubdlgsjkgg` int DEFAULT NULL,\n `tqnwpywyuwlyeudyqxwferhytfklhezv` int DEFAULT NULL,\n `cndjbxdqpinbptsxnybppvdjbpiqdhxx` int DEFAULT NULL,\n `mzsehaifwrkaniiesbsydjwrcaxxosdw` int DEFAULT NULL,\n `ejgtqjmqdfjauzfyofjqrnvkwlotyjjw` int DEFAULT NULL,\n `rkehuryymnqszmsncvsvxpslxawecljt` int DEFAULT NULL,\n `laeafxkimajkzriaeptqyijzxeiejygw` int DEFAULT NULL,\n `qlfshvycgihdmynsswxpayjsoboyhbwn` int DEFAULT NULL,\n `piaytifajfoswovhgndjoldwzbbrrjjj` int DEFAULT NULL,\n `lafvjprszidasstltcazeibycqzvxnui` int DEFAULT NULL,\n `myxhhmlyilaaooazckcbdphjhfqgtwxq` int DEFAULT NULL,\n `pweodshlrcngiydqnclbjpwviynpducu` int DEFAULT NULL,\n `dorpojhmkfpodglpvqaoirshdlhqczzm` int DEFAULT NULL,\n PRIMARY KEY (`efmgqhyeivzukzwluhosnhbdvskqacwl`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"jxtszwleobqkzaxtmowwchvvkrzyootc\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["efmgqhyeivzukzwluhosnhbdvskqacwl"],"columns":[{"name":"efmgqhyeivzukzwluhosnhbdvskqacwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"hxsxnvdtgwpehxbzcfssccqbwjpggdpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrapsyzbcqrfkgobetantcdtfjssypji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bomkkukholybohglgkvvqbuuytlrjzhl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfrysrvutvxatpmpwwouichlzlfwvqhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ysngmeyxnfqtsuziqybgoaqesuyuphra","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"liptvuucksmrkreerhzciqydibyzhccl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bejtxyuzvxotiddpuabchjlubcgapueo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrimbihaayrwpcwdrmnrtveyzeoiaufv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqhttmkurptfnbojwjzrxjyqmnfggllj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"weupfiuiqpgplktsjyvseuomhtcjswdf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"btiepvrmgtziunyhnubkrqbasqbwpgyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubqsfdgohrxlypyrmebwntluqryrjbdd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tiximrahkheafjrbuzppofktpjhbekgy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjqorinxycjxnhugzliullnewkfaybqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"piwtjqwyudgneegneewqlhkdtqxsykss","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pclfclrrpnouabmjfywvtnxpsgewdbcp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eipuzosgqrszajgiyiqplufvjlfljbdm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybipvdcmnaulpxscwpomqsoaoenzvfqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jelysxihawrbwpsjqrosmhyqtvjluqri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"funiplxcotgouwbfxqdqxlxgsxctuazw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixqcfnvfndnnnrzkyetiqfvtxwjsnohi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqqhtrcnjzrwtmqrtjlortylterajecm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmqycjupccoyqpyvkbkabahfkroqmmqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhatverstpscqrypjqobazwakxboumcv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khrdymvzszrfblueokwyvttlqpgyyphv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fklowuzpdrdszoxfvagzdkadjqtgbioj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pzxkxoycdkfwwcwmiqjykibqzpbfntbo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itjyttgxvevzgmlmnbfotprbswlnfoez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"daxqebzltkaedrptajatkxwowkubeohq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwaqwzbmbxnvtxsprcjpwggxvmjnepmz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qeedrqwbpggagyyndtujvcpomiyhhbbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjnmabmcyniprwrqjdqgqfytuzvypgdp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqzfqmvrggiayfxwkpjdhtolytzbbnnx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckwkhqocsdtzuwgyvkzutekeaeffikai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivqlecfetirgfazfloxfocdgvnilfzlm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrccxqinsrdgihjokixtoptxwnxydpyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbjmkdpyknrtqekslmpgtlsrrkowwpaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kbcnjmxsroushuqrpcbmmgkzegkxyety","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmbuwltvdkwexnentrnfrzophluapsxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hpqygdjbydkzfzewzsaxlrgmgunudsdl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvmsatawfikhfgvshwbpodkgcxfjguoh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cydqjwyibaandmgqbtzusbakiexiubug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"myuylpvvyonrzvsltwibtfplbcavzres","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eyhxfkexlwgrpdufxkcdgamjkgfhhtcv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbkqhxigpnpnyarwdgneqednuujmgjfj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nixffivpaldghyxagtpmnrykcbtszhfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxxvbylbutkiocrylecvyzkigbjtxrjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ciltvlwyhgbgvrjebaraqyhdwvxeuqgp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sukyqrrpcqikhclxrkhbizfrqxruusoc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ltrolmckirmxsuucjzukcozhryxbixra","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rutnawtihtqaimctirvibpwyrrpqzmxa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzfvjoxsptedbinthywstanpivdddjsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zknykrekxayzaogipaoobwcfqrcrgjss","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amcmepbhyqztalsvkjxoaufvoewfbvno","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"logjxdnththbmeeriiggqquatzmagoru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"olmxswbolqzjhqjzkqnsfsqhmiqhyeyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zrbbrelcwrkuxgbldotugbjmpggghvda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfmmjxvayctgbcivbqhpiirneyblsrzy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yvnwiftyhqsrhlspafamtsrumfdxjjev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uriljfmvmmihhihbeiiwtqstzwxosrxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akrhkwhykejocznfjosvlscntinyqcoh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqrzutyolymnvypslknwqhbfgxvfzwey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knpojkvfjdakqdjsauddsmqqdolfnilf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmubkejcfxkmdwbsbaqbhdvctdgzedct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lygnkzgpdyleyjbjukbgjcpalphiuhgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kyypdvvejfoietqrgiswnqroxuxrzxhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfaggnluavjwktaxruknpbuizyogqqkz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qosxrsarvrlolfwrzwpwyqwyibyzzbwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blcxhtezacsotjlduuwgzaoivgfxznxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxthnkqglqddqaudncbewwbjurgmiguz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cukqbttcckoduxpnvufxcvggcigxqzau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggoaakhqtyvtaopvhdcvxdmiicykynhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"irjmvthklvithbllflswmoiaqbwkofji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlxbmycehwszywrsldkuqjvzamzbhyum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crypccrixuetoyyvpjwcfabtadeitpuk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dzqwaotogpyitsfgydrwzkygnhoeqhqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbztluxhhpayyfeepfpclcetkhjctjmj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gpmwgbvojysrgnrjejnxiogokzspyetj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qailkojpuxriqikyledpzaqpmhuemhur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"moqmtkoxqfyuvpcegsceaazhdwydpzsf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxqomjdrbdpdwwprvwfrprdaspiistkc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xcmnjfweyqkvbqksljkoskpvbmpczprs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otgrqdnyarzqqxtovamamsapqoajxghm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xfimwrvtexmcbgvrfaoyurvlpufqoqsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbcxukrksuyveuiucebhevqidwuxthon","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hgoziggcodajglmqretrqrgoiotfadch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pzwstlllsfyluepazxhbxlubdlgsjkgg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqnwpywyuwlyeudyqxwferhytfklhezv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cndjbxdqpinbptsxnybppvdjbpiqdhxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzsehaifwrkaniiesbsydjwrcaxxosdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejgtqjmqdfjauzfyofjqrnvkwlotyjjw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkehuryymnqszmsncvsvxpslxawecljt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"laeafxkimajkzriaeptqyijzxeiejygw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qlfshvycgihdmynsswxpayjsoboyhbwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"piaytifajfoswovhgndjoldwzbbrrjjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lafvjprszidasstltcazeibycqzvxnui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"myxhhmlyilaaooazckcbdphjhfqgtwxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pweodshlrcngiydqnclbjpwviynpducu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dorpojhmkfpodglpvqaoirshdlhqczzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668899,"databaseName":"models_schema","ddl":"CREATE TABLE `jzkrhqtehgofuwbzfrjzkrenxfjfbkjc` (\n `sulhojifeflwkrbluskomlouorxnulgc` int NOT NULL,\n `qjutdldlelmtgferovhbhxrgqzzxpyiq` int DEFAULT NULL,\n `dsiyijkgdgsibqhiycwemqqopofktzxu` int DEFAULT NULL,\n `gsnrpcfkvsvrrnmkntqhaqihelsztyfc` int DEFAULT NULL,\n `ibcqycybfoqprqrtukxuvfxvskgpllen` int DEFAULT NULL,\n `eeidrezcdlyrldtssjglvxlftbzyidja` int DEFAULT NULL,\n `mawhkjtnnxhvcdmfuzathdopelrnfswl` int DEFAULT NULL,\n `hrmgopagvyjcwqckjgbtrgsfqxnsdxyt` int DEFAULT NULL,\n `qjsoxvdiicpxbsjcoqcnnoskylyyubxv` int DEFAULT NULL,\n `xbuesasprlthejrdmojnpyjozpyrdklo` int DEFAULT NULL,\n `tczkfvhamcaugtepllgmupqwgdboadxu` int DEFAULT NULL,\n `omqtuchlcbagxagrhshgplpcdczbxamx` int DEFAULT NULL,\n `tutfxajiljuiyibsizbpxwxebvuuhhvd` int DEFAULT NULL,\n `utvzlwutqqrcvjsixvcpudhmduvdupnl` int DEFAULT NULL,\n `phdrvjrtsbzveiwkhphsxluokdgdetnc` int DEFAULT NULL,\n `oocdlmwjciisbxpxxbdomcakygwyrnjd` int DEFAULT NULL,\n `gcfazkwmlufgzkcznifwknesetchkuwi` int DEFAULT NULL,\n `ebbubxduxfsdplupxattckymrpaukvaf` int DEFAULT NULL,\n `zhqedtmmoywsuveepznimqgkpbfkrbbc` int DEFAULT NULL,\n `qithfqdpytmeiqmhkbbkqxeiisdzyulb` int DEFAULT NULL,\n `nktdllnmetdczareokujogqfkfqvtrsw` int DEFAULT NULL,\n `iukeogqadkxwkzpgvcojljjzrdcclslk` int DEFAULT NULL,\n `qxkocqikdxxovcevidhntgscuhkbnlaf` int DEFAULT NULL,\n `mjmjuvfbegdvljkwlebkwpqklzqpakbp` int DEFAULT NULL,\n `tbbbwsnepnyzuwknkewwxbzlxoigkwsx` int DEFAULT NULL,\n `bffulwbbagwrtbmvzqxrtqgsszpclvix` int DEFAULT NULL,\n `ahaalhmnradspztzgfogrojtachoswlo` int DEFAULT NULL,\n `hmuuliuelpdrrtjjfebbiqenieanmdgn` int DEFAULT NULL,\n `krannlgntklnzeuicezbcjllhjslvgeg` int DEFAULT NULL,\n `dewnlenftnnmodilpbtbrgdxdxqsdsji` int DEFAULT NULL,\n `mifdtejvghssbhyipufzgszpknfxceff` int DEFAULT NULL,\n `eatdwsyaajfkcnsbijrmgrunkmfyxpsw` int DEFAULT NULL,\n `ivvqsokkvqgmfewkeuqlwwfmmztdtgpx` int DEFAULT NULL,\n `kutrodusshfmvdaqrdvwncshdlurnvkt` int DEFAULT NULL,\n `zklvkhrafowhztjahdxsdtfyankjkvmi` int DEFAULT NULL,\n `joaospqpqulxgrjciznkyowszztieqhm` int DEFAULT NULL,\n `eyyuejjcwgffgrqkngrlpbovfxyynfgd` int DEFAULT NULL,\n `tszgtjxvkoqtaaipmtnbtathnmizvwqk` int DEFAULT NULL,\n `zxsnxaslnvzxsixvtmrthqbpsyonkhyd` int DEFAULT NULL,\n `xaqeqkgqqaqoqsselqicfmqswctkctyy` int DEFAULT NULL,\n `xodrqchgmgghleppfaglftouphlzdmsk` int DEFAULT NULL,\n `hpcupwvjwunlmznbicqibunqrfhadilq` int DEFAULT NULL,\n `tgqhqajcxootqtdiobvjypcoyqnbvtuz` int DEFAULT NULL,\n `cedpyyngdfomkzbjcgzzzvgtiytrfqvh` int DEFAULT NULL,\n `jpnkqnbteizvmmhzzmswsmjfdgydcabx` int DEFAULT NULL,\n `fjqrujvuwihtkwdpdfdaejnjiqnzhmad` int DEFAULT NULL,\n `fipbkawhlipmzufidmrecydnbyebaslz` int DEFAULT NULL,\n `rofapacntytmsjezhhrpksifmxtzqdtd` int DEFAULT NULL,\n `ceuhmzozyforprwohngafdtsgdlahwwf` int DEFAULT NULL,\n `cuzrbqndcpxatqqmvjirsivuntdpzpox` int DEFAULT NULL,\n `aqiyejtgfwcbhudtlinxpagcewoweeli` int DEFAULT NULL,\n `jdnhnnjkbhbfpgqbnzvelmdzysqqqcpu` int DEFAULT NULL,\n `gdueuiziepakvemnhijqasknlgzxdyko` int DEFAULT NULL,\n `imvfxeduabmlcczhyjwdqovhpalynwlg` int DEFAULT NULL,\n `suwnwcjijvbqakofxoqtfeinacjzjvfx` int DEFAULT NULL,\n `hxkqsbrismeiuwvhslrakvplmmjhwkrp` int DEFAULT NULL,\n `cbqdqxvnykblbseqmjhyhptjbobgandn` int DEFAULT NULL,\n `kjuodjjrwbsdfcpclwfbhkbzpobgmzmi` int DEFAULT NULL,\n `vbuyjegcfxnonigtkqpdytvprrsdzuhw` int DEFAULT NULL,\n `mwovnuiyfmrilmowgrkydfolatqvkdxr` int DEFAULT NULL,\n `krshnsvfamzmkyigrbkgrsmvsxszdslz` int DEFAULT NULL,\n `rhlaoeooqqxnkqmucjhlakniuwidkycr` int DEFAULT NULL,\n `hefcvvsriwzyenyqovqhoddnysnwqief` int DEFAULT NULL,\n `rsmogvmsforkidlnmjgwbfcrfiqqdqjb` int DEFAULT NULL,\n `velgakllhkeaskhtlmgzbabeizcaeafk` int DEFAULT NULL,\n `yflwbruowdsvxijdkwyruusqpxvexwmv` int DEFAULT NULL,\n `bqthjhmcrxehuiuranktxcpubbpkjufy` int DEFAULT NULL,\n `mxhmgevexryepgfqxnljerdbnqxgfyfd` int DEFAULT NULL,\n `lpwacrsrfddiqgqjisbcaemoxcmnvfzu` int DEFAULT NULL,\n `jufgzdgawoqvxtbvpuamwktmyqjoztze` int DEFAULT NULL,\n `lxkhmvrypogbsgvmnncxmddrwswmvbxi` int DEFAULT NULL,\n `kmnyhrieeweevriyehyjfqftjathhiua` int DEFAULT NULL,\n `qqqsokflwcjpavmpazvwesyszdxsxhmh` int DEFAULT NULL,\n `flhdpooovjlkvgcginrhcfwtdxwnkbxt` int DEFAULT NULL,\n `fzwhbgamwmvfnfpgmbxklpqbdnlfcvve` int DEFAULT NULL,\n `wujayhyebfkupbvjczmgtnrxsypehzet` int DEFAULT NULL,\n `fkwztoylpbegxbqopbfsoolrzyljzcge` int DEFAULT NULL,\n `zkbqjlfrhwdfybqqoxyfobiytipvhcyq` int DEFAULT NULL,\n `bcelmassufxmbcoueberrhbxobemhvmi` int DEFAULT NULL,\n `ckclhsznzvrufremkpkghxjnlxjjihao` int DEFAULT NULL,\n `loeysotqllyukirhuqtdhrevypvdqujf` int DEFAULT NULL,\n `tapowdehmkjkqjrfsxryveslppvgwbgi` int DEFAULT NULL,\n `qkhrchptqhccfgpndqzkxmazahgijcjv` int DEFAULT NULL,\n `vpyugyhlcrrpmnggiaygqxxlpxjecyvs` int DEFAULT NULL,\n `cybjurmplunnbjsxzudxaaaeehcratmc` int DEFAULT NULL,\n `ezuldhfujoqgbjujrnjfgxdlnlxxvpxz` int DEFAULT NULL,\n `jsvawhcwyxyrkvzntngvsznrrnayodcc` int DEFAULT NULL,\n `ftnoidengnosoxbtybkwfwgwsonouuxv` int DEFAULT NULL,\n `raltsozfrtpbxotpmxhznugdyfyafzsu` int DEFAULT NULL,\n `mkbmwzfeqokzgsuemgqpchifkozhjgpz` int DEFAULT NULL,\n `anhdkvqerhiiycpdsomzxftwqvsmsnke` int DEFAULT NULL,\n `hqtcrpgcwuknhzndbkyggiraiwderrbw` int DEFAULT NULL,\n `gekegofsgjawwsqlxqzupkrtkhfuxxat` int DEFAULT NULL,\n `dlxqannohukidywpxtojegcmoaerdenv` int DEFAULT NULL,\n `ickcwuzejaajtfdkdbmqyerrtvuiredh` int DEFAULT NULL,\n `qgotjdwqiwrggagcfhjmmwfgmczaghzi` int DEFAULT NULL,\n `vmjprkxaorsyazmodxbfzmqkuevyawxi` int DEFAULT NULL,\n `xnkgirwdnvssxlcdlelnokwlwispcxpj` int DEFAULT NULL,\n `zqnwwcvnbfduxlnkxbfliyjophkdoiyu` int DEFAULT NULL,\n `ejfekdwfgjyafmtzpyplgoltvzqzmxoe` int DEFAULT NULL,\n PRIMARY KEY (`sulhojifeflwkrbluskomlouorxnulgc`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"jzkrhqtehgofuwbzfrjzkrenxfjfbkjc\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["sulhojifeflwkrbluskomlouorxnulgc"],"columns":[{"name":"sulhojifeflwkrbluskomlouorxnulgc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"qjutdldlelmtgferovhbhxrgqzzxpyiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsiyijkgdgsibqhiycwemqqopofktzxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsnrpcfkvsvrrnmkntqhaqihelsztyfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibcqycybfoqprqrtukxuvfxvskgpllen","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eeidrezcdlyrldtssjglvxlftbzyidja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mawhkjtnnxhvcdmfuzathdopelrnfswl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hrmgopagvyjcwqckjgbtrgsfqxnsdxyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjsoxvdiicpxbsjcoqcnnoskylyyubxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbuesasprlthejrdmojnpyjozpyrdklo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tczkfvhamcaugtepllgmupqwgdboadxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omqtuchlcbagxagrhshgplpcdczbxamx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tutfxajiljuiyibsizbpxwxebvuuhhvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utvzlwutqqrcvjsixvcpudhmduvdupnl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phdrvjrtsbzveiwkhphsxluokdgdetnc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oocdlmwjciisbxpxxbdomcakygwyrnjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gcfazkwmlufgzkcznifwknesetchkuwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebbubxduxfsdplupxattckymrpaukvaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhqedtmmoywsuveepznimqgkpbfkrbbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qithfqdpytmeiqmhkbbkqxeiisdzyulb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nktdllnmetdczareokujogqfkfqvtrsw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iukeogqadkxwkzpgvcojljjzrdcclslk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxkocqikdxxovcevidhntgscuhkbnlaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjmjuvfbegdvljkwlebkwpqklzqpakbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbbbwsnepnyzuwknkewwxbzlxoigkwsx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bffulwbbagwrtbmvzqxrtqgsszpclvix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahaalhmnradspztzgfogrojtachoswlo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmuuliuelpdrrtjjfebbiqenieanmdgn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krannlgntklnzeuicezbcjllhjslvgeg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dewnlenftnnmodilpbtbrgdxdxqsdsji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mifdtejvghssbhyipufzgszpknfxceff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eatdwsyaajfkcnsbijrmgrunkmfyxpsw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivvqsokkvqgmfewkeuqlwwfmmztdtgpx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kutrodusshfmvdaqrdvwncshdlurnvkt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zklvkhrafowhztjahdxsdtfyankjkvmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"joaospqpqulxgrjciznkyowszztieqhm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eyyuejjcwgffgrqkngrlpbovfxyynfgd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tszgtjxvkoqtaaipmtnbtathnmizvwqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxsnxaslnvzxsixvtmrthqbpsyonkhyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xaqeqkgqqaqoqsselqicfmqswctkctyy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xodrqchgmgghleppfaglftouphlzdmsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hpcupwvjwunlmznbicqibunqrfhadilq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tgqhqajcxootqtdiobvjypcoyqnbvtuz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cedpyyngdfomkzbjcgzzzvgtiytrfqvh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpnkqnbteizvmmhzzmswsmjfdgydcabx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjqrujvuwihtkwdpdfdaejnjiqnzhmad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fipbkawhlipmzufidmrecydnbyebaslz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rofapacntytmsjezhhrpksifmxtzqdtd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ceuhmzozyforprwohngafdtsgdlahwwf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cuzrbqndcpxatqqmvjirsivuntdpzpox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqiyejtgfwcbhudtlinxpagcewoweeli","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdnhnnjkbhbfpgqbnzvelmdzysqqqcpu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdueuiziepakvemnhijqasknlgzxdyko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"imvfxeduabmlcczhyjwdqovhpalynwlg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suwnwcjijvbqakofxoqtfeinacjzjvfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxkqsbrismeiuwvhslrakvplmmjhwkrp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbqdqxvnykblbseqmjhyhptjbobgandn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjuodjjrwbsdfcpclwfbhkbzpobgmzmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbuyjegcfxnonigtkqpdytvprrsdzuhw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwovnuiyfmrilmowgrkydfolatqvkdxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krshnsvfamzmkyigrbkgrsmvsxszdslz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhlaoeooqqxnkqmucjhlakniuwidkycr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hefcvvsriwzyenyqovqhoddnysnwqief","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rsmogvmsforkidlnmjgwbfcrfiqqdqjb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"velgakllhkeaskhtlmgzbabeizcaeafk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yflwbruowdsvxijdkwyruusqpxvexwmv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqthjhmcrxehuiuranktxcpubbpkjufy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mxhmgevexryepgfqxnljerdbnqxgfyfd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lpwacrsrfddiqgqjisbcaemoxcmnvfzu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jufgzdgawoqvxtbvpuamwktmyqjoztze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxkhmvrypogbsgvmnncxmddrwswmvbxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmnyhrieeweevriyehyjfqftjathhiua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqqsokflwcjpavmpazvwesyszdxsxhmh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flhdpooovjlkvgcginrhcfwtdxwnkbxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzwhbgamwmvfnfpgmbxklpqbdnlfcvve","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wujayhyebfkupbvjczmgtnrxsypehzet","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fkwztoylpbegxbqopbfsoolrzyljzcge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zkbqjlfrhwdfybqqoxyfobiytipvhcyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bcelmassufxmbcoueberrhbxobemhvmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckclhsznzvrufremkpkghxjnlxjjihao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"loeysotqllyukirhuqtdhrevypvdqujf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tapowdehmkjkqjrfsxryveslppvgwbgi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qkhrchptqhccfgpndqzkxmazahgijcjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpyugyhlcrrpmnggiaygqxxlpxjecyvs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cybjurmplunnbjsxzudxaaaeehcratmc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezuldhfujoqgbjujrnjfgxdlnlxxvpxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jsvawhcwyxyrkvzntngvsznrrnayodcc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftnoidengnosoxbtybkwfwgwsonouuxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"raltsozfrtpbxotpmxhznugdyfyafzsu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkbmwzfeqokzgsuemgqpchifkozhjgpz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"anhdkvqerhiiycpdsomzxftwqvsmsnke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqtcrpgcwuknhzndbkyggiraiwderrbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gekegofsgjawwsqlxqzupkrtkhfuxxat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlxqannohukidywpxtojegcmoaerdenv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ickcwuzejaajtfdkdbmqyerrtvuiredh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgotjdwqiwrggagcfhjmmwfgmczaghzi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmjprkxaorsyazmodxbfzmqkuevyawxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnkgirwdnvssxlcdlelnokwlwispcxpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqnwwcvnbfduxlnkxbfliyjophkdoiyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejfekdwfgjyafmtzpyplgoltvzqzmxoe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668932,"databaseName":"models_schema","ddl":"CREATE TABLE `kdodfqyudmcpsufuzzqeocbauczaskpt` (\n `wydbnnsulqpwkajtidqdmlemhtdugdbp` int NOT NULL,\n `lbavrrguphkzxulpwcrwdpshkymfhozm` int DEFAULT NULL,\n `jbvzlknsnaetyywfdnigovrqtljrfugs` int DEFAULT NULL,\n `hzguxnwnygkvlwveubbltmxfxztarjiq` int DEFAULT NULL,\n `lxbphipovcbtezmmpzaeucmdlurgjvmx` int DEFAULT NULL,\n `uzkujxweotdgvgzebhopylzkqmvmcrpx` int DEFAULT NULL,\n `kfyfglalwmngpvaeyldlfhabupmangdh` int DEFAULT NULL,\n `rgtbjnqeibbtcnocuartmtnotaavsvbu` int DEFAULT NULL,\n `yrqpuqyjjvxgwagsugkqafeqpjzgrqlt` int DEFAULT NULL,\n `lqoguqfbitehdcuioeynmxfhailkrjje` int DEFAULT NULL,\n `hmwxedbxpnncivexdblqgrpqblsbydbu` int DEFAULT NULL,\n `juhjzgzwvzyocuglbksfhuryoydlqdhc` int DEFAULT NULL,\n `rdojawfkbncbdyqmjqeaidnlbbpvumvj` int DEFAULT NULL,\n `pbrdrckffrcuvrmzasetnhcoidoexjla` int DEFAULT NULL,\n `mrxxecamoqksxoktopnzvevicjqmkwst` int DEFAULT NULL,\n `sdvkxgouhsnnxgkjuyrgnuieydgzmrwx` int DEFAULT NULL,\n `hbidlzjmymqccaqppeaupnajvgkkrmep` int DEFAULT NULL,\n `ehwpcrpftoiqailonhowjzfkeyesxsjv` int DEFAULT NULL,\n `bdzurxiokmrofhraplfktukbmpoarswz` int DEFAULT NULL,\n `sasuypscglmnpxeuwwwjgxymwbhqouyb` int DEFAULT NULL,\n `ixhgcluabvbfwjbmnbvnludlxxfdmzyl` int DEFAULT NULL,\n `yhrhyqxyrbnkagjwurrcstewlguhhuip` int DEFAULT NULL,\n `zvfjchnbllrvtdfyjuecggvfdczgutmi` int DEFAULT NULL,\n `ggpxjadihpnulsqpqphufacdipygdfpg` int DEFAULT NULL,\n `fbfwyxgbiietvlyoezmqotvrkvzzprqv` int DEFAULT NULL,\n `uprlfceuejgmzxzwjexxrhoxwvkpsmye` int DEFAULT NULL,\n `rjbzehjnkledoftmbggxhyoqymzpmqts` int DEFAULT NULL,\n `ypeuuocgwyfsqdkypfvoovfoopjerxmr` int DEFAULT NULL,\n `rnglhxdzgiivvxqrzjmpvoclxbyngbqu` int DEFAULT NULL,\n `fazxxhyxoingjxksbmgrrsbkklctvizf` int DEFAULT NULL,\n `gwichjzyjifinxwzkmiptvkkdkdjodww` int DEFAULT NULL,\n `xgyzmlsrkpctlfvxgwmazgqmpzoxzohl` int DEFAULT NULL,\n `wzidnfcbpwdoqesktcsvlfmyvxsvndmg` int DEFAULT NULL,\n `kxsnzkeaekzdiccwpvotgnwcozcbydhz` int DEFAULT NULL,\n `ikjzeqogqbkybnhwcabpexxvgnienshm` int DEFAULT NULL,\n `vfiwyxnwtircnegqfzvjcljjdgyndcrc` int DEFAULT NULL,\n `isfazsmpebuvjsbacvwksqublzhrxdnb` int DEFAULT NULL,\n `vzgbqpfknumsryotxdeokcoeeegusita` int DEFAULT NULL,\n `wtedfwtqrbvxtmusuptqlwtiwioynrjz` int DEFAULT NULL,\n `isrstotsfdszkiwloaunxcjhpjvbcqwq` int DEFAULT NULL,\n `brsfptswreqdpntqpddxfrdgabwxslsm` int DEFAULT NULL,\n `cjoeyzvbbnrsalodtkqngfditeuuiwzn` int DEFAULT NULL,\n `hshkcaposzjfmvtqfxvfsufxngalqzgq` int DEFAULT NULL,\n `hwuunjvzamuhrrlhzyheaedammvhypel` int DEFAULT NULL,\n `qoiokrdveijvggjlsmuzolpgdijzykcp` int DEFAULT NULL,\n `puwaytghmsdhjjfaquiwqnyorfzyefeu` int DEFAULT NULL,\n `rdyzkkhhetgsmylygujsznjzrmmadjfg` int DEFAULT NULL,\n `dpjpitwcdrtelysjqmwyqjwsnfukaipt` int DEFAULT NULL,\n `yufhktyoehkowmuufgxlfoajkfnhvplo` int DEFAULT NULL,\n `budmsfhgjsofujgfsephgcanrapfphvp` int DEFAULT NULL,\n `lhfdbsmemcsthnyjrgygwspwxhyesdxu` int DEFAULT NULL,\n `xgacovfpbgkmpexwiixeaqlajrynmgia` int DEFAULT NULL,\n `cecgfcrkizkivqqghlpkriaryiwucbkl` int DEFAULT NULL,\n `rzrxqoatlvadprjzvhlgufkbzdgjxcxs` int DEFAULT NULL,\n `lfszcehvesvzeaazeupjemtoswqchuxi` int DEFAULT NULL,\n `fltkzbdvpsmwcustpfgbanomcbqnitho` int DEFAULT NULL,\n `rliwqcoagvfnpwmepmpvxdwrzetndhpr` int DEFAULT NULL,\n `uyvnpjcobeqwolmtjrqowiknpbcmihzf` int DEFAULT NULL,\n `xypefadzskdoenbtkylliclqoycwxnul` int DEFAULT NULL,\n `jmstlkvgehhuguutquatejkqqgpeplbp` int DEFAULT NULL,\n `bvwhwzomnmdawelyeubxnsocdppthqcz` int DEFAULT NULL,\n `meesbnquqzfmifamyotyrytuqyathqgr` int DEFAULT NULL,\n `xvfinfzefpkerbxqlyrmanplxucvtsle` int DEFAULT NULL,\n `wcdffubpfuqfiihqzwknwpxrhrzegwcw` int DEFAULT NULL,\n `qtrbuucmnakfrbpsbobzaiqmuntbioqx` int DEFAULT NULL,\n `nukdkxvalabpulhzkaakqlmlmoqdpnjk` int DEFAULT NULL,\n `mlwnaxxxjcptrippfljfhrutmawophuc` int DEFAULT NULL,\n `fjaikkbqjewmixcswjgrqsglhbbhqfgu` int DEFAULT NULL,\n `uinszdugrfmwycbsbtuhhunycaznsvrf` int DEFAULT NULL,\n `osbjbkbspwafudglallazzomqfdurkmx` int DEFAULT NULL,\n `ksgpmamdznxgwjgvvylilwnhxhbcbikn` int DEFAULT NULL,\n `anrwjxdqiedmuftayenqhestekepfrcx` int DEFAULT NULL,\n `nbllseyozxacwpeowopqlcmhalcsjsqf` int DEFAULT NULL,\n `vzoewlttgwojotveodcutfyyspgxtrpo` int DEFAULT NULL,\n `pejphrrqlgfkxtwsvxhlcxfakxieqaed` int DEFAULT NULL,\n `gxhhwzukydjrxgrmjuxgmxwzgxhjbgid` int DEFAULT NULL,\n `cfogmhoxvnydkseivlrdonlupipjnmet` int DEFAULT NULL,\n `loqpkbevxmjmyvkjyiunjsfmyalrtcfi` int DEFAULT NULL,\n `rpbmdywwhpemuergpjiqjgptteffyplm` int DEFAULT NULL,\n `aylfeabljkewdidxqnioiujucgnwvnoq` int DEFAULT NULL,\n `scmxisgjnbxrxsbzardwxrfejbefecww` int DEFAULT NULL,\n `lthbwqxgtghxynqwqndzshdxjlkpbcly` int DEFAULT NULL,\n `ntnlvlpbqdvqiptftttfvzckfxygjuzv` int DEFAULT NULL,\n `akoedmvctejdvsdftpugheruxnbztnor` int DEFAULT NULL,\n `cvqartduukqhhyxdeereohchnsdjhgeu` int DEFAULT NULL,\n `auzlmmsywlfrisekjgwyeruxfslnwgnl` int DEFAULT NULL,\n `nrubikcjkgnoajcanuuegbqxhdnbksbb` int DEFAULT NULL,\n `vuelclbqunnaftcsbcbiqmldbobofzpj` int DEFAULT NULL,\n `irzyeiglxevuekrauhdbgupjfseboexr` int DEFAULT NULL,\n `pakpbnxjmopdyhtwtzcdsxgjpolgsvzx` int DEFAULT NULL,\n `izdsqkokkbnjitvqjpwznqaiyrfhktuy` int DEFAULT NULL,\n `zspkxwikgplnlzojyyqrenvdenbnbljp` int DEFAULT NULL,\n `tdimauyoywvijijrujoasiwdbvqyhxct` int DEFAULT NULL,\n `jvmdypeqngqvnwfjmfateqmxmfqxzquq` int DEFAULT NULL,\n `qjuvhcvnukpqnbdpewottyperlxzbzjy` int DEFAULT NULL,\n `sdxwhwpvqseelvoruzaagrxhyazmbsaq` int DEFAULT NULL,\n `faxtrfljjfdedixubiwmnmwvtmpijgha` int DEFAULT NULL,\n `vebxpejvwpvyiyxeisarafunhtsnvckw` int DEFAULT NULL,\n `pntquiujbbuphgpmqiclhkczgrsjabro` int DEFAULT NULL,\n `oncvismnpsmnxzfinbxpxakaamxwuhke` int DEFAULT NULL,\n PRIMARY KEY (`wydbnnsulqpwkajtidqdmlemhtdugdbp`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"kdodfqyudmcpsufuzzqeocbauczaskpt\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["wydbnnsulqpwkajtidqdmlemhtdugdbp"],"columns":[{"name":"wydbnnsulqpwkajtidqdmlemhtdugdbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"lbavrrguphkzxulpwcrwdpshkymfhozm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbvzlknsnaetyywfdnigovrqtljrfugs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hzguxnwnygkvlwveubbltmxfxztarjiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxbphipovcbtezmmpzaeucmdlurgjvmx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzkujxweotdgvgzebhopylzkqmvmcrpx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfyfglalwmngpvaeyldlfhabupmangdh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgtbjnqeibbtcnocuartmtnotaavsvbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yrqpuqyjjvxgwagsugkqafeqpjzgrqlt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqoguqfbitehdcuioeynmxfhailkrjje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmwxedbxpnncivexdblqgrpqblsbydbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"juhjzgzwvzyocuglbksfhuryoydlqdhc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdojawfkbncbdyqmjqeaidnlbbpvumvj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbrdrckffrcuvrmzasetnhcoidoexjla","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrxxecamoqksxoktopnzvevicjqmkwst","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdvkxgouhsnnxgkjuyrgnuieydgzmrwx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbidlzjmymqccaqppeaupnajvgkkrmep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehwpcrpftoiqailonhowjzfkeyesxsjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdzurxiokmrofhraplfktukbmpoarswz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sasuypscglmnpxeuwwwjgxymwbhqouyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixhgcluabvbfwjbmnbvnludlxxfdmzyl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhrhyqxyrbnkagjwurrcstewlguhhuip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvfjchnbllrvtdfyjuecggvfdczgutmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggpxjadihpnulsqpqphufacdipygdfpg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbfwyxgbiietvlyoezmqotvrkvzzprqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uprlfceuejgmzxzwjexxrhoxwvkpsmye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjbzehjnkledoftmbggxhyoqymzpmqts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypeuuocgwyfsqdkypfvoovfoopjerxmr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnglhxdzgiivvxqrzjmpvoclxbyngbqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fazxxhyxoingjxksbmgrrsbkklctvizf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gwichjzyjifinxwzkmiptvkkdkdjodww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xgyzmlsrkpctlfvxgwmazgqmpzoxzohl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzidnfcbpwdoqesktcsvlfmyvxsvndmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxsnzkeaekzdiccwpvotgnwcozcbydhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikjzeqogqbkybnhwcabpexxvgnienshm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vfiwyxnwtircnegqfzvjcljjdgyndcrc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"isfazsmpebuvjsbacvwksqublzhrxdnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzgbqpfknumsryotxdeokcoeeegusita","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtedfwtqrbvxtmusuptqlwtiwioynrjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"isrstotsfdszkiwloaunxcjhpjvbcqwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"brsfptswreqdpntqpddxfrdgabwxslsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjoeyzvbbnrsalodtkqngfditeuuiwzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hshkcaposzjfmvtqfxvfsufxngalqzgq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwuunjvzamuhrrlhzyheaedammvhypel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qoiokrdveijvggjlsmuzolpgdijzykcp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"puwaytghmsdhjjfaquiwqnyorfzyefeu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdyzkkhhetgsmylygujsznjzrmmadjfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpjpitwcdrtelysjqmwyqjwsnfukaipt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yufhktyoehkowmuufgxlfoajkfnhvplo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"budmsfhgjsofujgfsephgcanrapfphvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhfdbsmemcsthnyjrgygwspwxhyesdxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xgacovfpbgkmpexwiixeaqlajrynmgia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cecgfcrkizkivqqghlpkriaryiwucbkl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzrxqoatlvadprjzvhlgufkbzdgjxcxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfszcehvesvzeaazeupjemtoswqchuxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fltkzbdvpsmwcustpfgbanomcbqnitho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rliwqcoagvfnpwmepmpvxdwrzetndhpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uyvnpjcobeqwolmtjrqowiknpbcmihzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xypefadzskdoenbtkylliclqoycwxnul","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmstlkvgehhuguutquatejkqqgpeplbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvwhwzomnmdawelyeubxnsocdppthqcz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"meesbnquqzfmifamyotyrytuqyathqgr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvfinfzefpkerbxqlyrmanplxucvtsle","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcdffubpfuqfiihqzwknwpxrhrzegwcw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtrbuucmnakfrbpsbobzaiqmuntbioqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nukdkxvalabpulhzkaakqlmlmoqdpnjk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlwnaxxxjcptrippfljfhrutmawophuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjaikkbqjewmixcswjgrqsglhbbhqfgu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uinszdugrfmwycbsbtuhhunycaznsvrf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"osbjbkbspwafudglallazzomqfdurkmx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ksgpmamdznxgwjgvvylilwnhxhbcbikn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"anrwjxdqiedmuftayenqhestekepfrcx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbllseyozxacwpeowopqlcmhalcsjsqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzoewlttgwojotveodcutfyyspgxtrpo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pejphrrqlgfkxtwsvxhlcxfakxieqaed","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxhhwzukydjrxgrmjuxgmxwzgxhjbgid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfogmhoxvnydkseivlrdonlupipjnmet","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"loqpkbevxmjmyvkjyiunjsfmyalrtcfi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpbmdywwhpemuergpjiqjgptteffyplm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aylfeabljkewdidxqnioiujucgnwvnoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scmxisgjnbxrxsbzardwxrfejbefecww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lthbwqxgtghxynqwqndzshdxjlkpbcly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntnlvlpbqdvqiptftttfvzckfxygjuzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akoedmvctejdvsdftpugheruxnbztnor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvqartduukqhhyxdeereohchnsdjhgeu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"auzlmmsywlfrisekjgwyeruxfslnwgnl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nrubikcjkgnoajcanuuegbqxhdnbksbb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vuelclbqunnaftcsbcbiqmldbobofzpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"irzyeiglxevuekrauhdbgupjfseboexr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pakpbnxjmopdyhtwtzcdsxgjpolgsvzx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"izdsqkokkbnjitvqjpwznqaiyrfhktuy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zspkxwikgplnlzojyyqrenvdenbnbljp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdimauyoywvijijrujoasiwdbvqyhxct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvmdypeqngqvnwfjmfateqmxmfqxzquq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjuvhcvnukpqnbdpewottyperlxzbzjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdxwhwpvqseelvoruzaagrxhyazmbsaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"faxtrfljjfdedixubiwmnmwvtmpijgha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vebxpejvwpvyiyxeisarafunhtsnvckw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pntquiujbbuphgpmqiclhkczgrsjabro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oncvismnpsmnxzfinbxpxakaamxwuhke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842668970,"databaseName":"models_schema","ddl":"CREATE TABLE `kepkeqfmkwienbdkjycwcyzjpzrrkrrg` (\n `rfdnmkadwictbudjjnaicqpvswpffdnr` int NOT NULL,\n `ulhkffbemrfmpstcrmzmmvykwjxiwfkl` int DEFAULT NULL,\n `ibnsfzqnlcbbynsygvvvlpmbcpvdmwwm` int DEFAULT NULL,\n `vbhvijqalhwqeavcwjzxoqwpketvjida` int DEFAULT NULL,\n `yknpmyfmbzybqhqioowsnzipzfynpdga` int DEFAULT NULL,\n `byxcwzfmsfvnmgppiedttsowbbacfsjq` int DEFAULT NULL,\n `rruubxvchekihooglevengfifwmxzcsl` int DEFAULT NULL,\n `szphoelepnxaudlafjfiiexiwuofzbdn` int DEFAULT NULL,\n `snzwtjxpcddymzzofmwfuethbiraexip` int DEFAULT NULL,\n `iepsvxqkaqdnjoaixnptuvxrjcjniilf` int DEFAULT NULL,\n `nctfrnghtnxidftdealaldvmkawcgxnj` int DEFAULT NULL,\n `uxvqfnwmiabpjlswyfhpuqlulnybmkdw` int DEFAULT NULL,\n `gdmydhjygrigpgfuerwdzgthliwmofke` int DEFAULT NULL,\n `rflinbosngitlqyrbwzzucsfmbowsfwt` int DEFAULT NULL,\n `rkozxygwuaqqoneauuygixayquadbkbg` int DEFAULT NULL,\n `umjunjiudwkbzpknvptbfyetqgtdzcaa` int DEFAULT NULL,\n `shppayroeyvbfuvsoyneehroydcdmugc` int DEFAULT NULL,\n `eeuazdjnztrwflsssloykivjqdijqtoo` int DEFAULT NULL,\n `xudoorfsyjhucfmnqbbhwmrzfyspqxgn` int DEFAULT NULL,\n `wtdgmdqdmyampfkptnetnbafuezqtsls` int DEFAULT NULL,\n `msgjwdcobzkelzzcwvhmexytofnmgtyw` int DEFAULT NULL,\n `otskjjtmftxjwbtrbubjfbbokatlxgfj` int DEFAULT NULL,\n `fsnndqxmafznzlrotmugsqiufofusrra` int DEFAULT NULL,\n `qpxzobofbjsbizcncmtsejkfltulggkh` int DEFAULT NULL,\n `pmnyqvrdyltmtjxqszdrjyemhnusuibk` int DEFAULT NULL,\n `etrvazyrscpikjtcwhxwnfwuvstgmojl` int DEFAULT NULL,\n `sojmftxrgsbtrjolvtgwiimqalgsyofn` int DEFAULT NULL,\n `mwdtnsvgolhyrcjfvcmhjjlemxxswond` int DEFAULT NULL,\n `pmsctxfsskbieylshyhqbtiftmhuomtn` int DEFAULT NULL,\n `kekulrxykubdzkjxsestsiznshmbeokr` int DEFAULT NULL,\n `bktncrylkufpkhjakydhurfddavruygg` int DEFAULT NULL,\n `gylzugakxxdtfdmjxikrlmmemaakkpyc` int DEFAULT NULL,\n `ngkirbgknvjogseobrobglbkbgjocqwm` int DEFAULT NULL,\n `kanarkkfwocgazzpsdqvlysyayvtvfro` int DEFAULT NULL,\n `sjparnbelkinyjuhcaezuudhymxrsbxh` int DEFAULT NULL,\n `nloijyopucpsrsgujfkwzklnulardxhb` int DEFAULT NULL,\n `ymrqtusxfxqwgdzwvesvxecfgkovnbti` int DEFAULT NULL,\n `unqkxkjkmrlagiantebycgfaqvkesuie` int DEFAULT NULL,\n `zhbsifigxfqztgficnqqypvjblyeatng` int DEFAULT NULL,\n `ywnjloifxpvczbpkijcjgwldfwnfcnft` int DEFAULT NULL,\n `whmchirfvoxsgmdnpqakhbpfarvtypbb` int DEFAULT NULL,\n `milqnltkhbqpcaispxqhihewcndgdcbj` int DEFAULT NULL,\n `lrzriqslavlabhdbpsegpmgfppaefhmo` int DEFAULT NULL,\n `lrycaywfirhtvnmkeyrclihrdjwndcnx` int DEFAULT NULL,\n `cdkapzrumbkayigsypbgobgftejvthjg` int DEFAULT NULL,\n `grkyrsxllozjxtldttnvgjvbbokqvmxu` int DEFAULT NULL,\n `denxpyymdsssyazigjqhnovizhgloxmn` int DEFAULT NULL,\n `kzibexrjefylesmvhsgulofnuxiqkktg` int DEFAULT NULL,\n `ssbdnlxwalhotyydivxaybghxvvlagnp` int DEFAULT NULL,\n `vlwyexxvcejdjlkajrdhqvzdzcqtpffa` int DEFAULT NULL,\n `aoddwkzhcdsxhyugigjvptdlbkuyrvpd` int DEFAULT NULL,\n `lykkxbnxvmzmhkctnskdqnmmjkjndfzp` int DEFAULT NULL,\n `vmekbbswldodlljppkhmwosrxejdgjyc` int DEFAULT NULL,\n `wbwplwgfdssnsbyfiggnvtbaduwnetxm` int DEFAULT NULL,\n `rnztkbtkgngvkipvvxcntanrhhfxooqq` int DEFAULT NULL,\n `navyfyktfrcihuytvabkzgsdebbplbjt` int DEFAULT NULL,\n `anihawspdxvsassmmcyflwniiqdhkupp` int DEFAULT NULL,\n `jytegwxercweoxkvshggnoxxrmuvgvvz` int DEFAULT NULL,\n `tfybwgjnncmdwhbgbskztbargmigqjei` int DEFAULT NULL,\n `tviqwkywtnzonyfhektgqavvrxhpykym` int DEFAULT NULL,\n `kyhthzazvptnbvhsafycdxwzigcaeqyb` int DEFAULT NULL,\n `ylifrrpiqzyjexfxnpilntqeklfrrvwk` int DEFAULT NULL,\n `zyfumejnlvtfxeejtfoadffcinbvqwjo` int DEFAULT NULL,\n `lbkablxvlzpijygghnkdosnbyzxitxjk` int DEFAULT NULL,\n `gdtuznpdrrpnywfieybhwlzgpdlcmuyb` int DEFAULT NULL,\n `xeznzjhxezentkkzsirvowpbrjdulkkv` int DEFAULT NULL,\n `xypsyrttgmdjewoutiarefikjkqyaupr` int DEFAULT NULL,\n `nvomnlpclfgcmcjxqspmovfjugpjmlwi` int DEFAULT NULL,\n `vakcmwmuujbyodzbbewynsapppjtrtha` int DEFAULT NULL,\n `xydgkxavlyqtilihavnrkzjbypkswsmt` int DEFAULT NULL,\n `zrexodjlzubbnpnutyyoxottnjgoeigh` int DEFAULT NULL,\n `zzkyrpfsmrsacsuublhhlqlaqpodlwxw` int DEFAULT NULL,\n `bihcbxdimvnrxjairqjgrevjkxjyswmq` int DEFAULT NULL,\n `padtenlnraqecvqevcshwojjdewibopb` int DEFAULT NULL,\n `baudbfojunvywcgmhxpgpbyevrymhakr` int DEFAULT NULL,\n `ijuqepmexyzxhypfuldlqgyebuhhwvtp` int DEFAULT NULL,\n `svxwwuasparnqnnhupxihulyzfruptub` int DEFAULT NULL,\n `voyiwqkgsnmbsfpfleirzgyrlgpzgszz` int DEFAULT NULL,\n `zmvzgqpmfczcmfosehgpbaetvbjsfggd` int DEFAULT NULL,\n `yownwttopuprssgwrzipzgapomutzcps` int DEFAULT NULL,\n `dqrwrdbvfijridakyfqpensoxkbrdowk` int DEFAULT NULL,\n `scpygdtpaahysbsktyuhqbidepgzefem` int DEFAULT NULL,\n `mhakodedgljnhemsfjfcqeekhpbvqvbk` int DEFAULT NULL,\n `dvfwfzwtkzuveqjrkhcddbgvzyrgmcsc` int DEFAULT NULL,\n `pwapoaonbeavjqlvqcbuaoncbrnpnitb` int DEFAULT NULL,\n `hwbbmdmdpklfjrceangxkchomdymuhsg` int DEFAULT NULL,\n `luqmwvdmjqxbavejwojpjohzbkxhhkzf` int DEFAULT NULL,\n `llehxmntfcnizqzwxmtpwofnoiqnhckw` int DEFAULT NULL,\n `amqyjomgxyijyznfanwjpaklephqugki` int DEFAULT NULL,\n `ckxzqmgjohnwrufqdjtcqfuuuvkpikfe` int DEFAULT NULL,\n `grmtuprevezxmsbstyoigdlxksumscxb` int DEFAULT NULL,\n `xtpenvqytkxixvbafzceleeypsmnvgaz` int DEFAULT NULL,\n `auyfjjvsyyktoilrjkvobrhnafedyhqs` int DEFAULT NULL,\n `bveaqysoyjuokejeqatlcvwfixkqohed` int DEFAULT NULL,\n `qjmofpxycewopmstoxlmfzvajqyyvits` int DEFAULT NULL,\n `eznurmcgtzdubfhxcpcskfqmamyfkfkv` int DEFAULT NULL,\n `mbrwthdyrtheabfjyhwrszppimrapdtm` int DEFAULT NULL,\n `txdgvsrdqwlcdpcuavyqqqunspsemuhy` int DEFAULT NULL,\n `fjiciitoncysbvulpskolwtawffeoqnl` int DEFAULT NULL,\n `lovhbyriuwfwjhvemwdtneqgcigtteby` int DEFAULT NULL,\n PRIMARY KEY (`rfdnmkadwictbudjjnaicqpvswpffdnr`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"kepkeqfmkwienbdkjycwcyzjpzrrkrrg\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["rfdnmkadwictbudjjnaicqpvswpffdnr"],"columns":[{"name":"rfdnmkadwictbudjjnaicqpvswpffdnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ulhkffbemrfmpstcrmzmmvykwjxiwfkl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibnsfzqnlcbbynsygvvvlpmbcpvdmwwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbhvijqalhwqeavcwjzxoqwpketvjida","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yknpmyfmbzybqhqioowsnzipzfynpdga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"byxcwzfmsfvnmgppiedttsowbbacfsjq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rruubxvchekihooglevengfifwmxzcsl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"szphoelepnxaudlafjfiiexiwuofzbdn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"snzwtjxpcddymzzofmwfuethbiraexip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iepsvxqkaqdnjoaixnptuvxrjcjniilf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nctfrnghtnxidftdealaldvmkawcgxnj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxvqfnwmiabpjlswyfhpuqlulnybmkdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdmydhjygrigpgfuerwdzgthliwmofke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rflinbosngitlqyrbwzzucsfmbowsfwt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkozxygwuaqqoneauuygixayquadbkbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umjunjiudwkbzpknvptbfyetqgtdzcaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shppayroeyvbfuvsoyneehroydcdmugc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eeuazdjnztrwflsssloykivjqdijqtoo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xudoorfsyjhucfmnqbbhwmrzfyspqxgn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtdgmdqdmyampfkptnetnbafuezqtsls","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"msgjwdcobzkelzzcwvhmexytofnmgtyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otskjjtmftxjwbtrbubjfbbokatlxgfj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fsnndqxmafznzlrotmugsqiufofusrra","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qpxzobofbjsbizcncmtsejkfltulggkh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmnyqvrdyltmtjxqszdrjyemhnusuibk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"etrvazyrscpikjtcwhxwnfwuvstgmojl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sojmftxrgsbtrjolvtgwiimqalgsyofn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwdtnsvgolhyrcjfvcmhjjlemxxswond","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmsctxfsskbieylshyhqbtiftmhuomtn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kekulrxykubdzkjxsestsiznshmbeokr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bktncrylkufpkhjakydhurfddavruygg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gylzugakxxdtfdmjxikrlmmemaakkpyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ngkirbgknvjogseobrobglbkbgjocqwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kanarkkfwocgazzpsdqvlysyayvtvfro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjparnbelkinyjuhcaezuudhymxrsbxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nloijyopucpsrsgujfkwzklnulardxhb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ymrqtusxfxqwgdzwvesvxecfgkovnbti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unqkxkjkmrlagiantebycgfaqvkesuie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhbsifigxfqztgficnqqypvjblyeatng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywnjloifxpvczbpkijcjgwldfwnfcnft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"whmchirfvoxsgmdnpqakhbpfarvtypbb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"milqnltkhbqpcaispxqhihewcndgdcbj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lrzriqslavlabhdbpsegpmgfppaefhmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lrycaywfirhtvnmkeyrclihrdjwndcnx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cdkapzrumbkayigsypbgobgftejvthjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grkyrsxllozjxtldttnvgjvbbokqvmxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"denxpyymdsssyazigjqhnovizhgloxmn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzibexrjefylesmvhsgulofnuxiqkktg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ssbdnlxwalhotyydivxaybghxvvlagnp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vlwyexxvcejdjlkajrdhqvzdzcqtpffa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aoddwkzhcdsxhyugigjvptdlbkuyrvpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lykkxbnxvmzmhkctnskdqnmmjkjndfzp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmekbbswldodlljppkhmwosrxejdgjyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbwplwgfdssnsbyfiggnvtbaduwnetxm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnztkbtkgngvkipvvxcntanrhhfxooqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"navyfyktfrcihuytvabkzgsdebbplbjt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"anihawspdxvsassmmcyflwniiqdhkupp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jytegwxercweoxkvshggnoxxrmuvgvvz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tfybwgjnncmdwhbgbskztbargmigqjei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tviqwkywtnzonyfhektgqavvrxhpykym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kyhthzazvptnbvhsafycdxwzigcaeqyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ylifrrpiqzyjexfxnpilntqeklfrrvwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyfumejnlvtfxeejtfoadffcinbvqwjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lbkablxvlzpijygghnkdosnbyzxitxjk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdtuznpdrrpnywfieybhwlzgpdlcmuyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xeznzjhxezentkkzsirvowpbrjdulkkv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xypsyrttgmdjewoutiarefikjkqyaupr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvomnlpclfgcmcjxqspmovfjugpjmlwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vakcmwmuujbyodzbbewynsapppjtrtha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xydgkxavlyqtilihavnrkzjbypkswsmt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zrexodjlzubbnpnutyyoxottnjgoeigh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzkyrpfsmrsacsuublhhlqlaqpodlwxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bihcbxdimvnrxjairqjgrevjkxjyswmq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"padtenlnraqecvqevcshwojjdewibopb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"baudbfojunvywcgmhxpgpbyevrymhakr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijuqepmexyzxhypfuldlqgyebuhhwvtp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"svxwwuasparnqnnhupxihulyzfruptub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"voyiwqkgsnmbsfpfleirzgyrlgpzgszz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmvzgqpmfczcmfosehgpbaetvbjsfggd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yownwttopuprssgwrzipzgapomutzcps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqrwrdbvfijridakyfqpensoxkbrdowk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scpygdtpaahysbsktyuhqbidepgzefem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhakodedgljnhemsfjfcqeekhpbvqvbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvfwfzwtkzuveqjrkhcddbgvzyrgmcsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwapoaonbeavjqlvqcbuaoncbrnpnitb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwbbmdmdpklfjrceangxkchomdymuhsg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"luqmwvdmjqxbavejwojpjohzbkxhhkzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llehxmntfcnizqzwxmtpwofnoiqnhckw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amqyjomgxyijyznfanwjpaklephqugki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckxzqmgjohnwrufqdjtcqfuuuvkpikfe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grmtuprevezxmsbstyoigdlxksumscxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xtpenvqytkxixvbafzceleeypsmnvgaz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"auyfjjvsyyktoilrjkvobrhnafedyhqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bveaqysoyjuokejeqatlcvwfixkqohed","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjmofpxycewopmstoxlmfzvajqyyvits","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eznurmcgtzdubfhxcpcskfqmamyfkfkv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbrwthdyrtheabfjyhwrszppimrapdtm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txdgvsrdqwlcdpcuavyqqqunspsemuhy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjiciitoncysbvulpskolwtawffeoqnl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lovhbyriuwfwjhvemwdtneqgcigtteby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842668,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669000,"databaseName":"models_schema","ddl":"CREATE TABLE `kfixdbsknznmfkubbmksyozpamncervh` (\n `xrtreixvlialqwqdjxovbnzwnyvakbjj` int NOT NULL,\n `xiewjjmsnysivixlhzpnpgaagjvrpolr` int DEFAULT NULL,\n `jgqxxrmdigbzoisszntpnmlnvodpdpdb` int DEFAULT NULL,\n `vnpxamiotwfefjwsqovbymqnvaptqxgf` int DEFAULT NULL,\n `vqqjfsxkdquxdjjusenjcaccjlzduhwn` int DEFAULT NULL,\n `icgiqxnecxmfinibvsoutnierzzdxcxi` int DEFAULT NULL,\n `fxcxagyskxtdmxiappqnudyaohevzjyx` int DEFAULT NULL,\n `zimejnjrzwzzlmilacsehccywovicwws` int DEFAULT NULL,\n `hjhkqqrmykycfapfafkvpvwjvnfonllz` int DEFAULT NULL,\n `dnfwnpivocksyswszolhppjrfjhmpiot` int DEFAULT NULL,\n `yqcdghsyuehmnlbnpazniihsnohvjazt` int DEFAULT NULL,\n `entslmchtzdbinalfganseqbbffbfwam` int DEFAULT NULL,\n `yskqtxnhnuvlislztpbpiosfosjmkmmr` int DEFAULT NULL,\n `hkefgtisaynlzkmciykljshowgiqleii` int DEFAULT NULL,\n `iqqvftbpkwfwjsabmpvdobpzvmpuamjq` int DEFAULT NULL,\n `drqmxphnknipmzdijqwzxhuemwrbibvn` int DEFAULT NULL,\n `rfbozwwzruzqwcknsibqwmgbetzoaslj` int DEFAULT NULL,\n `gscpwwgggmidqccxynqduqrscehbirkt` int DEFAULT NULL,\n `wtxdpdayvphqwhaqpspjaugericwgobu` int DEFAULT NULL,\n `iwmgadpdtsppzvoetnfggtegdjzabxne` int DEFAULT NULL,\n `bvtqrpopiocnedqqnpcfqgjohqjdcevb` int DEFAULT NULL,\n `xbuqalxveobkjazvbsxefqbqjojzumwk` int DEFAULT NULL,\n `faufgskliatjbsykgwmqfbhkwvrrjebz` int DEFAULT NULL,\n `yoczfxpflhxohapyiuwcorjuxyxxktsn` int DEFAULT NULL,\n `vwlejjyphtxxlactypjcihwewnhhjnwk` int DEFAULT NULL,\n `ojuxbvnzzaabwvusmwrjfaaugxbjxihf` int DEFAULT NULL,\n `coqtiqgxriyxwzwpwxdkhlwfiihsazpp` int DEFAULT NULL,\n `xkaiepsygivzxlrkrodjlirzyurxnmcu` int DEFAULT NULL,\n `jpcppqpplwwqyzykstkkqljapczrolqb` int DEFAULT NULL,\n `flvwtweegfbrswgvlhyrrunrehokpiie` int DEFAULT NULL,\n `ebmyzwgpijpjiejylduzeuwckqjeimdd` int DEFAULT NULL,\n `uarzvtgljhjczcrjkexwxnfxpoptibsc` int DEFAULT NULL,\n `fuaqowrslgbrzqyymjiakwbhteyflbhs` int DEFAULT NULL,\n `izvxsvmwzpuhshrdstayomjrvocmlnfm` int DEFAULT NULL,\n `kajyroviaburchschsorvskwpkfdfnkz` int DEFAULT NULL,\n `fwtjkcumainteshpxbgcxpwcqirgjuhv` int DEFAULT NULL,\n `yfdurrwyhjrfhgyywgmarcxvnpoialwr` int DEFAULT NULL,\n `sccypdrhajqyiztbsfhryqhjtgryruji` int DEFAULT NULL,\n `oopzjdizcdtibhgbzhuxjbhhqykpnrrh` int DEFAULT NULL,\n `xnslifatydxtgfwupztuakjpnmbiqoyv` int DEFAULT NULL,\n `aypnrkzgnkqmxbukjwgddqresrqgszry` int DEFAULT NULL,\n `uhytdxkoztywgeuowxfglbkhcdtozqjb` int DEFAULT NULL,\n `yfuurtiapzgsdtkcqesuatitdzkcpudd` int DEFAULT NULL,\n `rhyotrcfzjubmpjtphoanlzmjphygitg` int DEFAULT NULL,\n `hdbgkumjxfpocrilnligitwnvfohyiei` int DEFAULT NULL,\n `ycgwuuogxpzeqbggwwfgqpinukzjmxma` int DEFAULT NULL,\n `nnefeqbhnngkmyfkmzlgorjhhxikdwcs` int DEFAULT NULL,\n `onzkiswnfsckyqdffqgepyovanamcqak` int DEFAULT NULL,\n `hiuayaqzqfeegztzylncpasfldimrjsj` int DEFAULT NULL,\n `yisgqjqizsajlfwxholyhpergwtmhgex` int DEFAULT NULL,\n `kbmubxpwxphdbblxusirfkmlwmjdgvwy` int DEFAULT NULL,\n `kqylsoqhwcdwqomeceplcdbeahjpinse` int DEFAULT NULL,\n `kskfwjvolabngicetxarhkctsyotrqxs` int DEFAULT NULL,\n `yprdlrxbnrunpiivkpqsnnaiqbiidxui` int DEFAULT NULL,\n `gxnrmuntwffozbxpalgazsgixrzrxiqw` int DEFAULT NULL,\n `lxlpnllvvbzdemxodfkfutdmllvbxfhs` int DEFAULT NULL,\n `wqyygiknxalegptzxdcagzltsewnzafi` int DEFAULT NULL,\n `hcwqcofyrevskhdigsbypuohrydpmywv` int DEFAULT NULL,\n `pfnwgeqynctfrngssenoycoelxhreioe` int DEFAULT NULL,\n `lormcsirytfhwxdqkzersruvmeyowzat` int DEFAULT NULL,\n `hamcbmrlwdmavcgboqohajvywjpyakpk` int DEFAULT NULL,\n `vzlnfqpdvecjvwrpweypbjyiditiulsa` int DEFAULT NULL,\n `cvxzfesipscedyegtjxeexjnzeqnefpu` int DEFAULT NULL,\n `rhhtcellnyknaradhhptrmkehvpxfuah` int DEFAULT NULL,\n `zyolbwlftzwehzhntsraatuoblrcxtxh` int DEFAULT NULL,\n `ixlqbbidvootkryhhhduipbuvuuytkrh` int DEFAULT NULL,\n `awqgqpzngfxsxeueuttcvwwmmhcfcwfg` int DEFAULT NULL,\n `cvmsgkatmtgvpdcoqwpwguiblzaubffl` int DEFAULT NULL,\n `stktprlqlkzjroqqyqlosguflbuakmhm` int DEFAULT NULL,\n `gwphofmcwegqbktpigxtqppmrwifvomb` int DEFAULT NULL,\n `blgwdkkwlnpspqnpfrmcthxfuqofhere` int DEFAULT NULL,\n `oakdlnomkzcvzpvswmlupekgdqxkkamv` int DEFAULT NULL,\n `oqqavkjxryuyudotviqdsvkzkmexkgpt` int DEFAULT NULL,\n `vbiyzsnkjclffygfycigyduiorynxxnv` int DEFAULT NULL,\n `ivtjglqymihivjstxibhywjsasgvkmjy` int DEFAULT NULL,\n `rkawijnghaufysyypvxosqaxhyijjrsr` int DEFAULT NULL,\n `mjmhnvrhefflqtupqlziqdxeibkcfndk` int DEFAULT NULL,\n `ikyphluecwqmsnujfhxceyerafftusqi` int DEFAULT NULL,\n `jdwrjbnxersvoiawpzgncmzwythlslst` int DEFAULT NULL,\n `cowutphtickvyaduudrbjhslcoldbcji` int DEFAULT NULL,\n `lwrwgbzwfxlpvpsmxtwmqqdtbjocwpil` int DEFAULT NULL,\n `svjngwihpckozwcyrkwzxvpzllydvrye` int DEFAULT NULL,\n `vntsfxzptwzunihpsvqgisbkwrpsjdzq` int DEFAULT NULL,\n `bwcspmjdtexolhgixnhcqdnanarrktca` int DEFAULT NULL,\n `mmufsixuhhpanmuejygfwoiygkqzonrv` int DEFAULT NULL,\n `csqfwepucymozmsxelhgzlwsoncmnpok` int DEFAULT NULL,\n `zfdkvxzhwkxhkofqwtvpqolxrwwktnih` int DEFAULT NULL,\n `bxanninndpombseheqwkgcesuvbtuqqa` int DEFAULT NULL,\n `lpmksdnqufnaomngmjdmqbehbvpksoed` int DEFAULT NULL,\n `dgpyufiubxdsczhicjvmkcefdhlqsaho` int DEFAULT NULL,\n `snkoalkarsxdczbcbtydnjpdgffkeulj` int DEFAULT NULL,\n `upzbnxgdoofqfzcnanwtvgnnnararjpx` int DEFAULT NULL,\n `fccwesicnefcykeuyvdukpsqywdbcsyw` int DEFAULT NULL,\n `cfuqpdzptscmlbraykfqpxbcbqnixnbv` int DEFAULT NULL,\n `btlnedqabdlogfsybxrflchcngniydad` int DEFAULT NULL,\n `vkdfpnxthouxwjfxfbunzieomnjmxwde` int DEFAULT NULL,\n `olpyvrtpbiercvgqkfjpmundevocjcqw` int DEFAULT NULL,\n `qytcfsuchltkpllfvwqlczyjkschiusy` int DEFAULT NULL,\n `cydnubzoslrpdpybflvleaucxbwsvfop` int DEFAULT NULL,\n `yexahtcelhgvfxqeqfushzeuilabmazw` int DEFAULT NULL,\n PRIMARY KEY (`xrtreixvlialqwqdjxovbnzwnyvakbjj`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"kfixdbsknznmfkubbmksyozpamncervh\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["xrtreixvlialqwqdjxovbnzwnyvakbjj"],"columns":[{"name":"xrtreixvlialqwqdjxovbnzwnyvakbjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"xiewjjmsnysivixlhzpnpgaagjvrpolr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgqxxrmdigbzoisszntpnmlnvodpdpdb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnpxamiotwfefjwsqovbymqnvaptqxgf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqqjfsxkdquxdjjusenjcaccjlzduhwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icgiqxnecxmfinibvsoutnierzzdxcxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxcxagyskxtdmxiappqnudyaohevzjyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zimejnjrzwzzlmilacsehccywovicwws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjhkqqrmykycfapfafkvpvwjvnfonllz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dnfwnpivocksyswszolhppjrfjhmpiot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqcdghsyuehmnlbnpazniihsnohvjazt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"entslmchtzdbinalfganseqbbffbfwam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yskqtxnhnuvlislztpbpiosfosjmkmmr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkefgtisaynlzkmciykljshowgiqleii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqqvftbpkwfwjsabmpvdobpzvmpuamjq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drqmxphnknipmzdijqwzxhuemwrbibvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfbozwwzruzqwcknsibqwmgbetzoaslj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gscpwwgggmidqccxynqduqrscehbirkt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtxdpdayvphqwhaqpspjaugericwgobu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwmgadpdtsppzvoetnfggtegdjzabxne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvtqrpopiocnedqqnpcfqgjohqjdcevb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbuqalxveobkjazvbsxefqbqjojzumwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"faufgskliatjbsykgwmqfbhkwvrrjebz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yoczfxpflhxohapyiuwcorjuxyxxktsn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vwlejjyphtxxlactypjcihwewnhhjnwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojuxbvnzzaabwvusmwrjfaaugxbjxihf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"coqtiqgxriyxwzwpwxdkhlwfiihsazpp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkaiepsygivzxlrkrodjlirzyurxnmcu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpcppqpplwwqyzykstkkqljapczrolqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flvwtweegfbrswgvlhyrrunrehokpiie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebmyzwgpijpjiejylduzeuwckqjeimdd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uarzvtgljhjczcrjkexwxnfxpoptibsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fuaqowrslgbrzqyymjiakwbhteyflbhs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"izvxsvmwzpuhshrdstayomjrvocmlnfm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kajyroviaburchschsorvskwpkfdfnkz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwtjkcumainteshpxbgcxpwcqirgjuhv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfdurrwyhjrfhgyywgmarcxvnpoialwr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sccypdrhajqyiztbsfhryqhjtgryruji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oopzjdizcdtibhgbzhuxjbhhqykpnrrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnslifatydxtgfwupztuakjpnmbiqoyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aypnrkzgnkqmxbukjwgddqresrqgszry","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhytdxkoztywgeuowxfglbkhcdtozqjb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfuurtiapzgsdtkcqesuatitdzkcpudd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhyotrcfzjubmpjtphoanlzmjphygitg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdbgkumjxfpocrilnligitwnvfohyiei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ycgwuuogxpzeqbggwwfgqpinukzjmxma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nnefeqbhnngkmyfkmzlgorjhhxikdwcs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"onzkiswnfsckyqdffqgepyovanamcqak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hiuayaqzqfeegztzylncpasfldimrjsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yisgqjqizsajlfwxholyhpergwtmhgex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kbmubxpwxphdbblxusirfkmlwmjdgvwy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqylsoqhwcdwqomeceplcdbeahjpinse","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kskfwjvolabngicetxarhkctsyotrqxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yprdlrxbnrunpiivkpqsnnaiqbiidxui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxnrmuntwffozbxpalgazsgixrzrxiqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxlpnllvvbzdemxodfkfutdmllvbxfhs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqyygiknxalegptzxdcagzltsewnzafi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hcwqcofyrevskhdigsbypuohrydpmywv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pfnwgeqynctfrngssenoycoelxhreioe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lormcsirytfhwxdqkzersruvmeyowzat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hamcbmrlwdmavcgboqohajvywjpyakpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzlnfqpdvecjvwrpweypbjyiditiulsa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvxzfesipscedyegtjxeexjnzeqnefpu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhhtcellnyknaradhhptrmkehvpxfuah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyolbwlftzwehzhntsraatuoblrcxtxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixlqbbidvootkryhhhduipbuvuuytkrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awqgqpzngfxsxeueuttcvwwmmhcfcwfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvmsgkatmtgvpdcoqwpwguiblzaubffl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"stktprlqlkzjroqqyqlosguflbuakmhm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gwphofmcwegqbktpigxtqppmrwifvomb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blgwdkkwlnpspqnpfrmcthxfuqofhere","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oakdlnomkzcvzpvswmlupekgdqxkkamv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqqavkjxryuyudotviqdsvkzkmexkgpt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbiyzsnkjclffygfycigyduiorynxxnv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivtjglqymihivjstxibhywjsasgvkmjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkawijnghaufysyypvxosqaxhyijjrsr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjmhnvrhefflqtupqlziqdxeibkcfndk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikyphluecwqmsnujfhxceyerafftusqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdwrjbnxersvoiawpzgncmzwythlslst","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cowutphtickvyaduudrbjhslcoldbcji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwrwgbzwfxlpvpsmxtwmqqdtbjocwpil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"svjngwihpckozwcyrkwzxvpzllydvrye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vntsfxzptwzunihpsvqgisbkwrpsjdzq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bwcspmjdtexolhgixnhcqdnanarrktca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmufsixuhhpanmuejygfwoiygkqzonrv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"csqfwepucymozmsxelhgzlwsoncmnpok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zfdkvxzhwkxhkofqwtvpqolxrwwktnih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bxanninndpombseheqwkgcesuvbtuqqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lpmksdnqufnaomngmjdmqbehbvpksoed","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgpyufiubxdsczhicjvmkcefdhlqsaho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"snkoalkarsxdczbcbtydnjpdgffkeulj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upzbnxgdoofqfzcnanwtvgnnnararjpx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fccwesicnefcykeuyvdukpsqywdbcsyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfuqpdzptscmlbraykfqpxbcbqnixnbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"btlnedqabdlogfsybxrflchcngniydad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkdfpnxthouxwjfxfbunzieomnjmxwde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"olpyvrtpbiercvgqkfjpmundevocjcqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qytcfsuchltkpllfvwqlczyjkschiusy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cydnubzoslrpdpybflvleaucxbwsvfop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yexahtcelhgvfxqeqfushzeuilabmazw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669031,"databaseName":"models_schema","ddl":"CREATE TABLE `kftwodjtxilhqupkceqeadajangtxxfg` (\n `jsivmqiklvkmmwynhncuidcobicwzriw` int NOT NULL,\n `ssfbcvsiiqyukjqedcohrhucfwpjvbvc` int DEFAULT NULL,\n `ebhzjngmocvpqrwhyskkwtvtepfuvsad` int DEFAULT NULL,\n `qowjqscldmpmrtlhyhleohrzejiyhddf` int DEFAULT NULL,\n `ybrkqmpaytvwbhwqeerfzcspoblfhito` int DEFAULT NULL,\n `xikzpdbvybmgrigwvysvpadqhsvrumgb` int DEFAULT NULL,\n `twhdnkgluxqtdbfrzcuesppuckekigez` int DEFAULT NULL,\n `vbanoqwnpbtymhomkcmvxexyecpagvif` int DEFAULT NULL,\n `ibgqwlqcztpbiizfgqqitmhmvpneeziu` int DEFAULT NULL,\n `vfcfywprzumaphnwxkocakvfgbagtxjz` int DEFAULT NULL,\n `xiiaoadxbmroaebwnlbgiknpnprhezkg` int DEFAULT NULL,\n `phlvbtyovxdeobwgvznhtwrgrdmjpenz` int DEFAULT NULL,\n `aptxnjpozqmpufdhabimgatrwrjzbney` int DEFAULT NULL,\n `iacntyrjpxwfsikyzfvsedcphpuxxvsz` int DEFAULT NULL,\n `perrznclmdtfbajwpolpvlzwkreowyco` int DEFAULT NULL,\n `toxgznxhsxngrcgxenyvdwnzarvofazq` int DEFAULT NULL,\n `ibcqzvlrtxlrlfovcxakmlnnwvgorbut` int DEFAULT NULL,\n `kyvpxsrakphseurvmjybtoyfsaueerxo` int DEFAULT NULL,\n `hxfvezlzjcdtptlvpcplgngedwdtznov` int DEFAULT NULL,\n `qfqfnunxrasciyfwriiafzvrkghjbume` int DEFAULT NULL,\n `fekzclvlsqtvqabckqiyjinqadauyath` int DEFAULT NULL,\n `lgouhoezcqahngaeekdvzzdanudjwhxy` int DEFAULT NULL,\n `ldlwdogqexiuxkrvdgviqlvhpmnvekte` int DEFAULT NULL,\n `hssrvwqdhxiujiioxynyaftbobfafjxa` int DEFAULT NULL,\n `nzjfqbvfjkbekugodmduygkupxoookif` int DEFAULT NULL,\n `ljklizgkkwzyqwmuagxlkuccmqonslyz` int DEFAULT NULL,\n `kyjpjtvsvrbjfupmjgpuljhlmgjyykgf` int DEFAULT NULL,\n `ctajswbsoyfjlbimklruuwmnurtmksjk` int DEFAULT NULL,\n `demdpqezhygygwgcnilhbkyfyvrogfsm` int DEFAULT NULL,\n `wzbbwpzvqqmmigtxrdlwhlhuechktpmf` int DEFAULT NULL,\n `sdojvercolnlfaxvtrqpmnoofjcgihzb` int DEFAULT NULL,\n `klhnxkyrulaefudxdbqychuxacwnvvye` int DEFAULT NULL,\n `jmesjytqatwdheglfntqtofalzxzbmuf` int DEFAULT NULL,\n `xbqihualfviazlwgsfgupzetumsfaejk` int DEFAULT NULL,\n `ztzbjqfvmhzhqahnlaslcsnlfjsmckbj` int DEFAULT NULL,\n `orvhdazjbixbjvgcezioancoljqyaywh` int DEFAULT NULL,\n `bnsbqnehvzzvoytaysowgqoyxfyepevd` int DEFAULT NULL,\n `zmpjnwparwhwpedxfnfdvgtpecnhqott` int DEFAULT NULL,\n `dgerzexifdzlckcjuaoledpxpakivolf` int DEFAULT NULL,\n `xcwkmndxmwqbiodoxbpxxbataydcepbk` int DEFAULT NULL,\n `xkoenykbxzthepftqbudiguvocokcmxw` int DEFAULT NULL,\n `xpeaquapawksrkfsxktuwmzxlbwnvhvs` int DEFAULT NULL,\n `dndhsvtbfcedqzimdrvlhjqtgeerqlfz` int DEFAULT NULL,\n `cjazkrgoydjkpmssldametzsjvvzlxaa` int DEFAULT NULL,\n `kpyvrpmamitdyluwdccwfcxenminmhot` int DEFAULT NULL,\n `fnwqarmylikuasdoutbhemshhwsusvwz` int DEFAULT NULL,\n `tlwvzivwmctwvsydledrfqjbxdfvntjj` int DEFAULT NULL,\n `ryomngkynhcmripmsvxkoxsjswfequda` int DEFAULT NULL,\n `acwkmoavlykhwbboyebcxhwuozaafsmw` int DEFAULT NULL,\n `vawpnbzaoxebemnkifkhfoaeqpciqxbs` int DEFAULT NULL,\n `nmltgrovcpazbbuboubznmkcttjgzaqi` int DEFAULT NULL,\n `mtxqojddwvprkbzqsyeyywoqwwrrxarl` int DEFAULT NULL,\n `kpudxywepiyxgkzegrftxfktgpsnteqj` int DEFAULT NULL,\n `bdpvyvysfpnmlcgaizvbspbeknututgu` int DEFAULT NULL,\n `uarchuzkqhgbobgqqyczvwyxccbxswvc` int DEFAULT NULL,\n `graqrhoqbcdsudcdzvywgecjdiopjlkb` int DEFAULT NULL,\n `ysblsrkzfhpxnhevyrqhpuidcxplworj` int DEFAULT NULL,\n `glejlatsajlskomfwrmjosumerfauuqf` int DEFAULT NULL,\n `aupmtmozdfptjqpnpcblaedslpxexews` int DEFAULT NULL,\n `bihecvpxinrtmwmugajyjotjxdprwyxz` int DEFAULT NULL,\n `pwdlbpixjpohseelqbhylcmkmezcqvkf` int DEFAULT NULL,\n `zzapysxwnwepmnwxsshvdbdcgcgleglh` int DEFAULT NULL,\n `nspxrbkkudbvghcmhmcwvsucgmkfczbw` int DEFAULT NULL,\n `trhatqflqdgrsuzsvbmgjmzzyljyoawz` int DEFAULT NULL,\n `zvgosvsncgriwxbpmxpufnxkaeatuuog` int DEFAULT NULL,\n `nlmvnkbunznvukbilwnjolzetfnooier` int DEFAULT NULL,\n `zwaihbdlloyyuitqiqndbgrgyxslxddi` int DEFAULT NULL,\n `mkjrzgztqjdfkakhpydgsyoumlqhfmmj` int DEFAULT NULL,\n `evklnsgrtizlabxcmruyzegljercwwmv` int DEFAULT NULL,\n `eisxyjprpcuyuegsdvklpkzqyongtyoj` int DEFAULT NULL,\n `heqbamubsgzufhknoghpqtdfwbtxykuz` int DEFAULT NULL,\n `rogzerohgxwugqprxsjglxjrsvwsaabt` int DEFAULT NULL,\n `rqbxcqjdjtrhyvkfonswblpthcssxdtp` int DEFAULT NULL,\n `dgktdlgjanpltvjqnyazwirajdqbxctr` int DEFAULT NULL,\n `hdefpeufmazjydjgzxbipweyszazmxrk` int DEFAULT NULL,\n `brumcomhirjtafxqasurandyvexhvcxo` int DEFAULT NULL,\n `cqpxkoqbrkdgftvwsbmuqdaylfpnjaeq` int DEFAULT NULL,\n `orhqbpevgnzvjdnaatytlukocikqfssp` int DEFAULT NULL,\n `rjfzortffhmtgjdxubrqathddujdymna` int DEFAULT NULL,\n `unzfztnwxnnguvpyzgbmfwhmkjkvqvyv` int DEFAULT NULL,\n `drsuipqlcppzvaaxfadjjffuhrnhfdil` int DEFAULT NULL,\n `axlepfvyqqquogbuameghmmiuznvxkoi` int DEFAULT NULL,\n `ijkdppldttarazpeppsyatifucintroz` int DEFAULT NULL,\n `zqltfgacfaunmsqatcisjghgdzjcrcwk` int DEFAULT NULL,\n `ccakrymadhoosoiaiznkdicufgfnyyro` int DEFAULT NULL,\n `wkejiddvotcryzhxzvynpsplcumxwgra` int DEFAULT NULL,\n `odmtllqdzfrruxiypjyipzjvqjvipnpv` int DEFAULT NULL,\n `pamnzlvsuzawrcyicuibmrxecbfomguj` int DEFAULT NULL,\n `wbbpuyvtnfrqunhejnmvnyhyzwuyaaoi` int DEFAULT NULL,\n `llyrstwrkibfcofihvnswbbstwfvrnnk` int DEFAULT NULL,\n `phwfiqdwmngipdcythwhtocclomcyhgr` int DEFAULT NULL,\n `nxmoudiwgfnroquwucbbtbxtrbwelyof` int DEFAULT NULL,\n `lymnfmeedmfhqbeinkoeiahjainkqmpq` int DEFAULT NULL,\n `ddegvvaxjweyujpvpeurcnjqaqvcxlhk` int DEFAULT NULL,\n `osbrryqyxepprlvwkvutpnvhyxrsxoox` int DEFAULT NULL,\n `yckpaofgjzmlpoflelpdpfupnwmcjxei` int DEFAULT NULL,\n `cathqqciqbhedhjqpmosicqstovfemvt` int DEFAULT NULL,\n `nudoavgvcgwlgsfwhlkeglhbfxoobblh` int DEFAULT NULL,\n `pramrukaaxwoahdxywtzqutdzchgnzwg` int DEFAULT NULL,\n `agukaynvovhiahunozcaycsdxujydngd` int DEFAULT NULL,\n PRIMARY KEY (`jsivmqiklvkmmwynhncuidcobicwzriw`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"kftwodjtxilhqupkceqeadajangtxxfg\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["jsivmqiklvkmmwynhncuidcobicwzriw"],"columns":[{"name":"jsivmqiklvkmmwynhncuidcobicwzriw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ssfbcvsiiqyukjqedcohrhucfwpjvbvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebhzjngmocvpqrwhyskkwtvtepfuvsad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qowjqscldmpmrtlhyhleohrzejiyhddf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybrkqmpaytvwbhwqeerfzcspoblfhito","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xikzpdbvybmgrigwvysvpadqhsvrumgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twhdnkgluxqtdbfrzcuesppuckekigez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbanoqwnpbtymhomkcmvxexyecpagvif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibgqwlqcztpbiizfgqqitmhmvpneeziu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vfcfywprzumaphnwxkocakvfgbagtxjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xiiaoadxbmroaebwnlbgiknpnprhezkg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phlvbtyovxdeobwgvznhtwrgrdmjpenz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aptxnjpozqmpufdhabimgatrwrjzbney","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iacntyrjpxwfsikyzfvsedcphpuxxvsz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"perrznclmdtfbajwpolpvlzwkreowyco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"toxgznxhsxngrcgxenyvdwnzarvofazq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibcqzvlrtxlrlfovcxakmlnnwvgorbut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kyvpxsrakphseurvmjybtoyfsaueerxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxfvezlzjcdtptlvpcplgngedwdtznov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfqfnunxrasciyfwriiafzvrkghjbume","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fekzclvlsqtvqabckqiyjinqadauyath","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgouhoezcqahngaeekdvzzdanudjwhxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ldlwdogqexiuxkrvdgviqlvhpmnvekte","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hssrvwqdhxiujiioxynyaftbobfafjxa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzjfqbvfjkbekugodmduygkupxoookif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljklizgkkwzyqwmuagxlkuccmqonslyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kyjpjtvsvrbjfupmjgpuljhlmgjyykgf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctajswbsoyfjlbimklruuwmnurtmksjk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"demdpqezhygygwgcnilhbkyfyvrogfsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzbbwpzvqqmmigtxrdlwhlhuechktpmf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdojvercolnlfaxvtrqpmnoofjcgihzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"klhnxkyrulaefudxdbqychuxacwnvvye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmesjytqatwdheglfntqtofalzxzbmuf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbqihualfviazlwgsfgupzetumsfaejk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ztzbjqfvmhzhqahnlaslcsnlfjsmckbj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orvhdazjbixbjvgcezioancoljqyaywh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnsbqnehvzzvoytaysowgqoyxfyepevd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmpjnwparwhwpedxfnfdvgtpecnhqott","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgerzexifdzlckcjuaoledpxpakivolf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xcwkmndxmwqbiodoxbpxxbataydcepbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkoenykbxzthepftqbudiguvocokcmxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpeaquapawksrkfsxktuwmzxlbwnvhvs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dndhsvtbfcedqzimdrvlhjqtgeerqlfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjazkrgoydjkpmssldametzsjvvzlxaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpyvrpmamitdyluwdccwfcxenminmhot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnwqarmylikuasdoutbhemshhwsusvwz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlwvzivwmctwvsydledrfqjbxdfvntjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ryomngkynhcmripmsvxkoxsjswfequda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acwkmoavlykhwbboyebcxhwuozaafsmw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vawpnbzaoxebemnkifkhfoaeqpciqxbs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmltgrovcpazbbuboubznmkcttjgzaqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtxqojddwvprkbzqsyeyywoqwwrrxarl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpudxywepiyxgkzegrftxfktgpsnteqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdpvyvysfpnmlcgaizvbspbeknututgu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uarchuzkqhgbobgqqyczvwyxccbxswvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"graqrhoqbcdsudcdzvywgecjdiopjlkb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ysblsrkzfhpxnhevyrqhpuidcxplworj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"glejlatsajlskomfwrmjosumerfauuqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aupmtmozdfptjqpnpcblaedslpxexews","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bihecvpxinrtmwmugajyjotjxdprwyxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwdlbpixjpohseelqbhylcmkmezcqvkf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzapysxwnwepmnwxsshvdbdcgcgleglh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nspxrbkkudbvghcmhmcwvsucgmkfczbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"trhatqflqdgrsuzsvbmgjmzzyljyoawz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvgosvsncgriwxbpmxpufnxkaeatuuog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nlmvnkbunznvukbilwnjolzetfnooier","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zwaihbdlloyyuitqiqndbgrgyxslxddi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkjrzgztqjdfkakhpydgsyoumlqhfmmj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evklnsgrtizlabxcmruyzegljercwwmv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eisxyjprpcuyuegsdvklpkzqyongtyoj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"heqbamubsgzufhknoghpqtdfwbtxykuz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rogzerohgxwugqprxsjglxjrsvwsaabt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqbxcqjdjtrhyvkfonswblpthcssxdtp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgktdlgjanpltvjqnyazwirajdqbxctr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdefpeufmazjydjgzxbipweyszazmxrk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"brumcomhirjtafxqasurandyvexhvcxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqpxkoqbrkdgftvwsbmuqdaylfpnjaeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orhqbpevgnzvjdnaatytlukocikqfssp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjfzortffhmtgjdxubrqathddujdymna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unzfztnwxnnguvpyzgbmfwhmkjkvqvyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drsuipqlcppzvaaxfadjjffuhrnhfdil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axlepfvyqqquogbuameghmmiuznvxkoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijkdppldttarazpeppsyatifucintroz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqltfgacfaunmsqatcisjghgdzjcrcwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ccakrymadhoosoiaiznkdicufgfnyyro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wkejiddvotcryzhxzvynpsplcumxwgra","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"odmtllqdzfrruxiypjyipzjvqjvipnpv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pamnzlvsuzawrcyicuibmrxecbfomguj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbbpuyvtnfrqunhejnmvnyhyzwuyaaoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llyrstwrkibfcofihvnswbbstwfvrnnk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phwfiqdwmngipdcythwhtocclomcyhgr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxmoudiwgfnroquwucbbtbxtrbwelyof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lymnfmeedmfhqbeinkoeiahjainkqmpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddegvvaxjweyujpvpeurcnjqaqvcxlhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"osbrryqyxepprlvwkvutpnvhyxrsxoox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yckpaofgjzmlpoflelpdpfupnwmcjxei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cathqqciqbhedhjqpmosicqstovfemvt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nudoavgvcgwlgsfwhlkeglhbfxoobblh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pramrukaaxwoahdxywtzqutdzchgnzwg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agukaynvovhiahunozcaycsdxujydngd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669061,"databaseName":"models_schema","ddl":"CREATE TABLE `kgeadtdnymxzvfhjvvodgeoxonwagwll` (\n `trzmxekaguqekzyoucmzroqaqswbksug` int NOT NULL,\n `kpuorhlwxpghnchaakxiyuwtxidfkmdc` int DEFAULT NULL,\n `jjnsctwfjweqzhhjtkavrnmufexcxigr` int DEFAULT NULL,\n `nwujhvaphahaogjqqurdudpjngnzpmoy` int DEFAULT NULL,\n `kfabrkrifsrfiyxzjrsicpvfgsvuhqim` int DEFAULT NULL,\n `ayeyqvacuknnnvrvrumbstqmjqswknpv` int DEFAULT NULL,\n `mcltpfagrfeiaomicszeujnylkxsjcrf` int DEFAULT NULL,\n `zmdiktcyhcbefzygareinwkaxpemgwrr` int DEFAULT NULL,\n `rsqnzeuqhffaehxhodgpdrdseyeclphv` int DEFAULT NULL,\n `gdqfkyrjdpfzsnfumnaayprxsqnoglli` int DEFAULT NULL,\n `enddmoweeaoohasacvlutyxxycbdghsq` int DEFAULT NULL,\n `sugguztavtmspgkfclahdrearpbxxrev` int DEFAULT NULL,\n `wsfzcqlhawxzmcwafcifkhpqfsxfdqxc` int DEFAULT NULL,\n `vxoguwvvrxbgxvimvakjegadgldkwuqu` int DEFAULT NULL,\n `cgayjvzvczrllyzvubqdenkphysjlllr` int DEFAULT NULL,\n `rgkwqgncaoskknupfjkhkryegrhzsoxs` int DEFAULT NULL,\n `tqbmxylypxomxquoivjuxuzewpgxvout` int DEFAULT NULL,\n `bzgtstpdoessfdtiphoawrwftvpszkjt` int DEFAULT NULL,\n `olyhsdoeihceolzqyfijvbcubgphxjlu` int DEFAULT NULL,\n `gqnntcmeyxckeyfcbbqnvsgbpjlwaanc` int DEFAULT NULL,\n `nlofftuisyuwsegwswuxryhqwhiroqju` int DEFAULT NULL,\n `durkwcsqpgnacksahlqemuazkjdeyztj` int DEFAULT NULL,\n `nvlcpprsbuprixhydtmiwmmtjkksrkhr` int DEFAULT NULL,\n `gazaxewrpooxmsmzluoxsrwhpiqpabts` int DEFAULT NULL,\n `kvhppxibpxbnneiukqbrirwylgqnyyzw` int DEFAULT NULL,\n `wzaeddngayigvjxhuvxnnqgrdiucowxo` int DEFAULT NULL,\n `btszgdjfblcdkhwbjtobohcpqjbwcrbe` int DEFAULT NULL,\n `fngnfinjjkgtqkakszcjryksbtmlhzly` int DEFAULT NULL,\n `lqkxohcwhrjbjqnwoxzqctfshsgcpsdn` int DEFAULT NULL,\n `umnackinazuzgmafqkpcssrkdmgvhfhj` int DEFAULT NULL,\n `nushuxpihapggpxqtfcbhmgmcbnhbeiw` int DEFAULT NULL,\n `dlhqscksmhwojkehsksrcyrehfdjplgl` int DEFAULT NULL,\n `uncemvqhcullqgkvnsghwpqglalnbbob` int DEFAULT NULL,\n `oahzgghwkschsxxaawvtnffwprlqzncr` int DEFAULT NULL,\n `iqjsycnfbtrnfnznygzzxhhvkrsinmma` int DEFAULT NULL,\n `nlmtnrisfddtwcgbuzoinwglyghdtblc` int DEFAULT NULL,\n `lsrkwtabvaavthnxmbleaksiuyaznjsi` int DEFAULT NULL,\n `hpqyfivswtjphhwloetncuehgizztgiw` int DEFAULT NULL,\n `utodjrnbnmzkembmmbufartrjlkuldzm` int DEFAULT NULL,\n `gqyyyvtzmdwaazmaejcfmpbrjgbmvflf` int DEFAULT NULL,\n `evxxytimnykoeqqpjisdgnmjxqhyjrwg` int DEFAULT NULL,\n `oecftaeiofyactmoefbatflhfoefixow` int DEFAULT NULL,\n `jlxlpqxwezwqqvpdnwpstusaejiimuak` int DEFAULT NULL,\n `qalpuawypsyacksxnfsezffbqkmrgunp` int DEFAULT NULL,\n `ibhubtufmifyinkrewepugeqypvauawa` int DEFAULT NULL,\n `sbdhqqwjdzxoetedbzjozlqbxzmbstrr` int DEFAULT NULL,\n `dvqsoeseehglmzslqinhvviuuzvdxwuq` int DEFAULT NULL,\n `bbhivcsrylhxlsjcqjzfgjfcpzsduclj` int DEFAULT NULL,\n `oialkxykyeqapjerrvpaznfymfofmddv` int DEFAULT NULL,\n `tlfadntzojqqoczlnunxcaclkjlmabnb` int DEFAULT NULL,\n `sjhavoxypxgpcatbskbwxcrtheaxuibd` int DEFAULT NULL,\n `sbrpgyodtdzgdfkajencptsvxitaeknn` int DEFAULT NULL,\n `uxnfhonpeifdkcqzxovwmgqdlrugypkr` int DEFAULT NULL,\n `wwjgxyhtmgmjfcwqtmmjegyxzqrdvejn` int DEFAULT NULL,\n `bbriquouoyflefvyuismejhcyylkkkkb` int DEFAULT NULL,\n `giqcnjrshpgdcexsvsnkrwnleclopwsn` int DEFAULT NULL,\n `giippspiftmbebivsnjdqhfpygummcyi` int DEFAULT NULL,\n `zutipgfcmnjoutipsisricgqnemhbsev` int DEFAULT NULL,\n `jfjwehvgnjdlodzmkmgxugknudswasio` int DEFAULT NULL,\n `gprqzwuazeufewowdiarqrsusxmtkkui` int DEFAULT NULL,\n `zpxztfvguxqkjjbnvbjvqqymlvgxkyiu` int DEFAULT NULL,\n `zrrviqedroojcurijcxoawxfbmewtshl` int DEFAULT NULL,\n `fvpsvrmzjkwiaclpvpodtkpthxwxdxvt` int DEFAULT NULL,\n `ednbopgrnlkiwgzfrldnptfreljymwnk` int DEFAULT NULL,\n `yqfdziigzevmyzrwbfyjfnicelypoyde` int DEFAULT NULL,\n `mctrhoshvwjroovzkyixslhaweoyibxo` int DEFAULT NULL,\n `cufutzcaecaegycknqnsrtzwkajaxhma` int DEFAULT NULL,\n `xubgqwpzhxrglupqqxzawsspgbqoxiie` int DEFAULT NULL,\n `gmxkqeddrztoclikyzdqcdbpkwprrklq` int DEFAULT NULL,\n `zcggiynsxmfsgzkcfmuxejstlaagoiiq` int DEFAULT NULL,\n `wqtcrikiltitukigspyhbutnrvfoygbr` int DEFAULT NULL,\n `hackpnwhunxkgnkgaiibkmdctpihjycf` int DEFAULT NULL,\n `qilnlsniktpjacojnpfufcbvilvlhqbn` int DEFAULT NULL,\n `afvzaiaqvxyzkddwmbbpvytpyzlnhcmp` int DEFAULT NULL,\n `xhvjbqkesudrgiueczyyoxdjmyammexp` int DEFAULT NULL,\n `oyxhrsjsqcuudfhtqgnctebffoecaxqs` int DEFAULT NULL,\n `adutvcrijvhzpoxzasmejnriehkmyjor` int DEFAULT NULL,\n `cgoebszzaqmtsxlxseeuogyvcugtqvxu` int DEFAULT NULL,\n `ctuabhrmmamiqaiinudvmyixigkhhpdx` int DEFAULT NULL,\n `fnyrjhdthknixedhomwlysryniwaotjl` int DEFAULT NULL,\n `ajypchqjogauklngkxnrtphgzjqzehsz` int DEFAULT NULL,\n `lxozywekwwyzckwbzixnjhonkkbntaln` int DEFAULT NULL,\n `sowuylgbcfempksynduznbqyhcqvhmdy` int DEFAULT NULL,\n `bvszhepwwghzlyotvjmaxfscesmmjzyh` int DEFAULT NULL,\n `fsosetqtvogwuhimoajmjihtyxotkmws` int DEFAULT NULL,\n `icynbkqhuyfhsuqzifgrqysiwxneswbk` int DEFAULT NULL,\n `aluuuodvdrrlnuwmabethueijmmjbjcq` int DEFAULT NULL,\n `einhmgzkcqhfpgexrbwnzfeidtozqvrm` int DEFAULT NULL,\n `wjusittqsjyuqzihiuquedvoykfqoydr` int DEFAULT NULL,\n `uumqnfqndfifaqrqxrrygkflltmadrkg` int DEFAULT NULL,\n `sqfvbcqleoswpkypisquebnuhvhbjyok` int DEFAULT NULL,\n `gytirviqybirhjpbgctegrdezzhqnqdw` int DEFAULT NULL,\n `ceomvnmcwuodnhldlumnpbytkrxvpznt` int DEFAULT NULL,\n `ndercrflqxtgqsaqoasnukplvhlapiyc` int DEFAULT NULL,\n `dkordqiktsvgseekpubrjveuufyxtkpg` int DEFAULT NULL,\n `ohbjrbprmsackddglyevhzkvkjmwzbda` int DEFAULT NULL,\n `gfdelwcgqhseobxzmyvntvminwdecxyo` int DEFAULT NULL,\n `qccsrtqisvcoqgyhpmrljqfhltspojrk` int DEFAULT NULL,\n `cvbcvbkletosbtsnvsegnnydrzpktpok` int DEFAULT NULL,\n `pqeskqqfcuvmibzkgwpuaqjnctiqfoae` int DEFAULT NULL,\n PRIMARY KEY (`trzmxekaguqekzyoucmzroqaqswbksug`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"kgeadtdnymxzvfhjvvodgeoxonwagwll\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["trzmxekaguqekzyoucmzroqaqswbksug"],"columns":[{"name":"trzmxekaguqekzyoucmzroqaqswbksug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"kpuorhlwxpghnchaakxiyuwtxidfkmdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjnsctwfjweqzhhjtkavrnmufexcxigr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nwujhvaphahaogjqqurdudpjngnzpmoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfabrkrifsrfiyxzjrsicpvfgsvuhqim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ayeyqvacuknnnvrvrumbstqmjqswknpv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcltpfagrfeiaomicszeujnylkxsjcrf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmdiktcyhcbefzygareinwkaxpemgwrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rsqnzeuqhffaehxhodgpdrdseyeclphv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdqfkyrjdpfzsnfumnaayprxsqnoglli","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"enddmoweeaoohasacvlutyxxycbdghsq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sugguztavtmspgkfclahdrearpbxxrev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wsfzcqlhawxzmcwafcifkhpqfsxfdqxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxoguwvvrxbgxvimvakjegadgldkwuqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgayjvzvczrllyzvubqdenkphysjlllr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgkwqgncaoskknupfjkhkryegrhzsoxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqbmxylypxomxquoivjuxuzewpgxvout","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzgtstpdoessfdtiphoawrwftvpszkjt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"olyhsdoeihceolzqyfijvbcubgphxjlu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqnntcmeyxckeyfcbbqnvsgbpjlwaanc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nlofftuisyuwsegwswuxryhqwhiroqju","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"durkwcsqpgnacksahlqemuazkjdeyztj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvlcpprsbuprixhydtmiwmmtjkksrkhr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gazaxewrpooxmsmzluoxsrwhpiqpabts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvhppxibpxbnneiukqbrirwylgqnyyzw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzaeddngayigvjxhuvxnnqgrdiucowxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"btszgdjfblcdkhwbjtobohcpqjbwcrbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fngnfinjjkgtqkakszcjryksbtmlhzly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqkxohcwhrjbjqnwoxzqctfshsgcpsdn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umnackinazuzgmafqkpcssrkdmgvhfhj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nushuxpihapggpxqtfcbhmgmcbnhbeiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlhqscksmhwojkehsksrcyrehfdjplgl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uncemvqhcullqgkvnsghwpqglalnbbob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oahzgghwkschsxxaawvtnffwprlqzncr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqjsycnfbtrnfnznygzzxhhvkrsinmma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nlmtnrisfddtwcgbuzoinwglyghdtblc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lsrkwtabvaavthnxmbleaksiuyaznjsi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hpqyfivswtjphhwloetncuehgizztgiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utodjrnbnmzkembmmbufartrjlkuldzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqyyyvtzmdwaazmaejcfmpbrjgbmvflf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evxxytimnykoeqqpjisdgnmjxqhyjrwg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oecftaeiofyactmoefbatflhfoefixow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlxlpqxwezwqqvpdnwpstusaejiimuak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qalpuawypsyacksxnfsezffbqkmrgunp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibhubtufmifyinkrewepugeqypvauawa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbdhqqwjdzxoetedbzjozlqbxzmbstrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvqsoeseehglmzslqinhvviuuzvdxwuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbhivcsrylhxlsjcqjzfgjfcpzsduclj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oialkxykyeqapjerrvpaznfymfofmddv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlfadntzojqqoczlnunxcaclkjlmabnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjhavoxypxgpcatbskbwxcrtheaxuibd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbrpgyodtdzgdfkajencptsvxitaeknn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxnfhonpeifdkcqzxovwmgqdlrugypkr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwjgxyhtmgmjfcwqtmmjegyxzqrdvejn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbriquouoyflefvyuismejhcyylkkkkb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"giqcnjrshpgdcexsvsnkrwnleclopwsn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"giippspiftmbebivsnjdqhfpygummcyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zutipgfcmnjoutipsisricgqnemhbsev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jfjwehvgnjdlodzmkmgxugknudswasio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gprqzwuazeufewowdiarqrsusxmtkkui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpxztfvguxqkjjbnvbjvqqymlvgxkyiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zrrviqedroojcurijcxoawxfbmewtshl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fvpsvrmzjkwiaclpvpodtkpthxwxdxvt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ednbopgrnlkiwgzfrldnptfreljymwnk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqfdziigzevmyzrwbfyjfnicelypoyde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mctrhoshvwjroovzkyixslhaweoyibxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cufutzcaecaegycknqnsrtzwkajaxhma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xubgqwpzhxrglupqqxzawsspgbqoxiie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmxkqeddrztoclikyzdqcdbpkwprrklq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zcggiynsxmfsgzkcfmuxejstlaagoiiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqtcrikiltitukigspyhbutnrvfoygbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hackpnwhunxkgnkgaiibkmdctpihjycf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qilnlsniktpjacojnpfufcbvilvlhqbn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afvzaiaqvxyzkddwmbbpvytpyzlnhcmp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xhvjbqkesudrgiueczyyoxdjmyammexp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oyxhrsjsqcuudfhtqgnctebffoecaxqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"adutvcrijvhzpoxzasmejnriehkmyjor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgoebszzaqmtsxlxseeuogyvcugtqvxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctuabhrmmamiqaiinudvmyixigkhhpdx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnyrjhdthknixedhomwlysryniwaotjl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajypchqjogauklngkxnrtphgzjqzehsz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxozywekwwyzckwbzixnjhonkkbntaln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sowuylgbcfempksynduznbqyhcqvhmdy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvszhepwwghzlyotvjmaxfscesmmjzyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fsosetqtvogwuhimoajmjihtyxotkmws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icynbkqhuyfhsuqzifgrqysiwxneswbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aluuuodvdrrlnuwmabethueijmmjbjcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"einhmgzkcqhfpgexrbwnzfeidtozqvrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjusittqsjyuqzihiuquedvoykfqoydr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uumqnfqndfifaqrqxrrygkflltmadrkg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sqfvbcqleoswpkypisquebnuhvhbjyok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gytirviqybirhjpbgctegrdezzhqnqdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ceomvnmcwuodnhldlumnpbytkrxvpznt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ndercrflqxtgqsaqoasnukplvhlapiyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkordqiktsvgseekpubrjveuufyxtkpg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohbjrbprmsackddglyevhzkvkjmwzbda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfdelwcgqhseobxzmyvntvminwdecxyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qccsrtqisvcoqgyhpmrljqfhltspojrk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvbcvbkletosbtsnvsegnnydrzpktpok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqeskqqfcuvmibzkgwpuaqjnctiqfoae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669097,"databaseName":"models_schema","ddl":"CREATE TABLE `kgjyjbtpzshoafctbllepaalknrmhmzy` (\n `ykfozsivykbdsxpqbbwnozsfctwdfvtj` int NOT NULL,\n `cdtyvuqpubpgypqygivitrvhvblfdkrq` int DEFAULT NULL,\n `jvuzkrgvtncodotepixnrmbexkexjaqx` int DEFAULT NULL,\n `qxvlxgtgpizbywzebjiehcjiiutaqmly` int DEFAULT NULL,\n `alheysaqhkhzsrtvcwguormjdlpehrhk` int DEFAULT NULL,\n `osdgqifrchkdczyhqzdhjqksdlikoyat` int DEFAULT NULL,\n `wwepaarauoskpdbczoczxlphmpqwdeqf` int DEFAULT NULL,\n `fccjhbigthifomyklkjwjsejwrcnjgvp` int DEFAULT NULL,\n `aehgtlkhczmbbhihyrdovanwclqrjemj` int DEFAULT NULL,\n `uozqtbhwclnsffjhzaqueuchjrzkdkkw` int DEFAULT NULL,\n `tdtncyigogzxbjpkxvfavkcsluwtlkkw` int DEFAULT NULL,\n `ruxdslqdabtzimukltmydwhztxqshkmx` int DEFAULT NULL,\n `hxweqkyfmolugsmmbnijbamhxfyfwkmw` int DEFAULT NULL,\n `jprrxtqainbpyodoefuyylcrcjcfgrmu` int DEFAULT NULL,\n `zgimeyyefppksbxkfzxjwklqrxpfieuz` int DEFAULT NULL,\n `ouyquumrpsktndpdmijnqlzqtputevur` int DEFAULT NULL,\n `yjerqtfbdjsmfdhtmvusgwehqdnwhtks` int DEFAULT NULL,\n `fhulzanzrcxklrmljurqppcbexrxaofi` int DEFAULT NULL,\n `basbkjmqowcapcqsmvqnoyetubapumib` int DEFAULT NULL,\n `utowrndokitjbwlmqxrqlnyyhdfivkew` int DEFAULT NULL,\n `wtswlometccxmlkykmtpbnrtatirdcgi` int DEFAULT NULL,\n `olqbuyfurajtpyliaduzktvkqsfeklel` int DEFAULT NULL,\n `aybpsgzuyzflimucuzimjhtveleubtoc` int DEFAULT NULL,\n `wvybbivfhuwfzsboxcuhzwjzkyytefbg` int DEFAULT NULL,\n `veyuvffxoqqaxvcwwtozymhdgtfykipi` int DEFAULT NULL,\n `lhondfunrbzrxbjvzuftjdycryqxtpnx` int DEFAULT NULL,\n `huxuqgpcavkzybxvzyazfinalxqlpbqu` int DEFAULT NULL,\n `zddiyjuuprksprhocrfumsfwifxvfrhf` int DEFAULT NULL,\n `wibncirunjqstfgioccjtcwxvfblslqf` int DEFAULT NULL,\n `wsemkdruguqendoouyqoazqosmlxkleh` int DEFAULT NULL,\n `ktwwolagxidtlwramygwhuykeejhdacd` int DEFAULT NULL,\n `ltticajqgkfdtfwhdxqdwbbrkigkquvc` int DEFAULT NULL,\n `oxzxplhxklmauetrycfqnqomknnburiz` int DEFAULT NULL,\n `wjsalyxppysmpzxofocuaikmcvkmbziw` int DEFAULT NULL,\n `iptnzzdfweungiwpvwsdcritvqzgiawe` int DEFAULT NULL,\n `toirncfxuucgproaxveqtpsrbtohwklh` int DEFAULT NULL,\n `clojqxlkcpsctqtxydzouodciefqlfhf` int DEFAULT NULL,\n `bmcrqmwkvtbggbrlxmjfvkibupuslddc` int DEFAULT NULL,\n `cbaasuznysvgecrqyrisixoawisubyls` int DEFAULT NULL,\n `tprtftghjoxmvntffsafgzusobpyfvzg` int DEFAULT NULL,\n `fcgagpjkhabujbgvhhfgielaqmtcrhck` int DEFAULT NULL,\n `xzgfgruhqjhifjflorpmffnttlrgyhtv` int DEFAULT NULL,\n `wtqucnfoxjpcvvdbvcqihwlkqepwpgbk` int DEFAULT NULL,\n `rfaywdqwxhicopcmhlwwbmhuyulwedau` int DEFAULT NULL,\n `ouwkhzmyfvuaiisicrzmduxtkiictngv` int DEFAULT NULL,\n `qdfygdclhhcqtrmuatbahcvptyuveqid` int DEFAULT NULL,\n `sbyunaqkzmdkylzssaosxqjxrljvzchz` int DEFAULT NULL,\n `rukfmanumvdaukktomrdibgrwonxaatn` int DEFAULT NULL,\n `yuvnxdiltqdgkgspdrzczkgspytfbcjg` int DEFAULT NULL,\n `mogzlhexhwjpbkwioivfosykadxtngfh` int DEFAULT NULL,\n `rapehblzpqhghwcbgqaoxkottxtvfxts` int DEFAULT NULL,\n `lfkxoomwyjrahvukrsajnouxrcqguxhz` int DEFAULT NULL,\n `qgqeeyovxkjfyidbspkvxjbghbaomuna` int DEFAULT NULL,\n `fwjaisncbljfeiksvozfldyoilclveiy` int DEFAULT NULL,\n `usaegaumtafmnfxhdveilgvmktyrosrp` int DEFAULT NULL,\n `irtpahgsjzeaibrfhavzwzuujsfnkgtu` int DEFAULT NULL,\n `moedisqvbmjuvtlegepxxmdndaoyhwgw` int DEFAULT NULL,\n `tfbespmcybocfvztyxbfrlxbrnsgdivl` int DEFAULT NULL,\n `nxbfsmnnvzhzjrbmvpjxkymruoflpzdq` int DEFAULT NULL,\n `ukzraawvkzowekurpqobxnjaymsktasf` int DEFAULT NULL,\n `lcekrqlfkukzmfdnrqpqjdnygyxyfynl` int DEFAULT NULL,\n `qqulottedyyljalziswozfrrjhbmdqgm` int DEFAULT NULL,\n `pvecvklumiwejsmjdvzshnjhwvxeuaue` int DEFAULT NULL,\n `ncrqjpscmlajzyldjmedosivgbgntuuz` int DEFAULT NULL,\n `mskzxsvtoofzbvribqxalhpshroqvjhh` int DEFAULT NULL,\n `nfmxeabgbyxvhxzqywskyiqkefgrwzvz` int DEFAULT NULL,\n `wrxpepztnczydpcvkiohevlofddkpbwk` int DEFAULT NULL,\n `oefusigixwhqdlximnjhfltmralrpcos` int DEFAULT NULL,\n `onofddlhvyvtrcvjbaeprwziucfkuwrb` int DEFAULT NULL,\n `dcnrkpqsztldfsziwmizjjgehzqlzvno` int DEFAULT NULL,\n `wiuotxauiyjbnahagfngedwghfqpotzg` int DEFAULT NULL,\n `balonewrcysdgaasbrozosvzefzazjvj` int DEFAULT NULL,\n `kriwgaurjjhvzdlhlfqamdxgqywrukjd` int DEFAULT NULL,\n `ylriffqytmnlnmniuqqpxleiynnfdvwj` int DEFAULT NULL,\n `gzfhuzdaldqfuapftyyfgqnvzbzstlge` int DEFAULT NULL,\n `suqojzaqcwqfaylrxkdyjqjcmzbhrucf` int DEFAULT NULL,\n `hbwmddxnzvoyvawsgrqiyspsvbmlwipu` int DEFAULT NULL,\n `xkmfdafyhgzharjmuskwevwuhigvgokk` int DEFAULT NULL,\n `wemhetrjkohbffgvwdfhlwpvhgxqjxqu` int DEFAULT NULL,\n `zrwlpxjoenaddzfeshrggwvcfsmtlesw` int DEFAULT NULL,\n `jkkywrjlmixzvdmhgczlvwkbijvpovrw` int DEFAULT NULL,\n `oahsdejvowtsdwoajfnrwbkzgwczcbqy` int DEFAULT NULL,\n `rudcmqaebgqbuzihgjuyhzmvwzefiuic` int DEFAULT NULL,\n `kjxzywtplwczcmmnlmzwjyxmsjqmpwrh` int DEFAULT NULL,\n `puxgltsmaliuridjmlhguqwkmxeapnas` int DEFAULT NULL,\n `dnknreaumwmpovcvcqzyuadlydttocdu` int DEFAULT NULL,\n `wlwinedjghrlrtfgicmfgmuohfsghqqa` int DEFAULT NULL,\n `ydbbnbkqrhsrgyoyhxebfbtdawrkwufy` int DEFAULT NULL,\n `bfbfxncbuhwzjbmpoargvjghuulpdngi` int DEFAULT NULL,\n `kryzpncupjxxmbsxgonfjghxcmbynzln` int DEFAULT NULL,\n `qbtakpyegbcjvkvbppndltduxpaqdjvq` int DEFAULT NULL,\n `fjpicfvhahvrcmrtqhsxocrdqpywtvcd` int DEFAULT NULL,\n `uquissdxsbxbunkknmwcwsybydajzjvv` int DEFAULT NULL,\n `hfgwvkywcjvojsgstijdmupnodexjoye` int DEFAULT NULL,\n `muoeqmcawqzrbdosjpjywipireaffpuh` int DEFAULT NULL,\n `nlmjxdvmoecgaifvqwtlmrqwxcsdoqte` int DEFAULT NULL,\n `zmdlhiuqufdpmopgfvwcdftjihmkukgd` int DEFAULT NULL,\n `hhejboilivnppuadueozvppzjgpchksv` int DEFAULT NULL,\n `rwvjzhzloyjkwuwygjbxhcysbwrebsse` int DEFAULT NULL,\n `dliqqngavxopxrkeoxojcawvfbxuhirv` int DEFAULT NULL,\n PRIMARY KEY (`ykfozsivykbdsxpqbbwnozsfctwdfvtj`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"kgjyjbtpzshoafctbllepaalknrmhmzy\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["ykfozsivykbdsxpqbbwnozsfctwdfvtj"],"columns":[{"name":"ykfozsivykbdsxpqbbwnozsfctwdfvtj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"cdtyvuqpubpgypqygivitrvhvblfdkrq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvuzkrgvtncodotepixnrmbexkexjaqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxvlxgtgpizbywzebjiehcjiiutaqmly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"alheysaqhkhzsrtvcwguormjdlpehrhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"osdgqifrchkdczyhqzdhjqksdlikoyat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwepaarauoskpdbczoczxlphmpqwdeqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fccjhbigthifomyklkjwjsejwrcnjgvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aehgtlkhczmbbhihyrdovanwclqrjemj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uozqtbhwclnsffjhzaqueuchjrzkdkkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdtncyigogzxbjpkxvfavkcsluwtlkkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ruxdslqdabtzimukltmydwhztxqshkmx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxweqkyfmolugsmmbnijbamhxfyfwkmw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jprrxtqainbpyodoefuyylcrcjcfgrmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zgimeyyefppksbxkfzxjwklqrxpfieuz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ouyquumrpsktndpdmijnqlzqtputevur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjerqtfbdjsmfdhtmvusgwehqdnwhtks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhulzanzrcxklrmljurqppcbexrxaofi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"basbkjmqowcapcqsmvqnoyetubapumib","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utowrndokitjbwlmqxrqlnyyhdfivkew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtswlometccxmlkykmtpbnrtatirdcgi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"olqbuyfurajtpyliaduzktvkqsfeklel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aybpsgzuyzflimucuzimjhtveleubtoc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvybbivfhuwfzsboxcuhzwjzkyytefbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"veyuvffxoqqaxvcwwtozymhdgtfykipi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhondfunrbzrxbjvzuftjdycryqxtpnx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"huxuqgpcavkzybxvzyazfinalxqlpbqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zddiyjuuprksprhocrfumsfwifxvfrhf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wibncirunjqstfgioccjtcwxvfblslqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wsemkdruguqendoouyqoazqosmlxkleh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ktwwolagxidtlwramygwhuykeejhdacd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ltticajqgkfdtfwhdxqdwbbrkigkquvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oxzxplhxklmauetrycfqnqomknnburiz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjsalyxppysmpzxofocuaikmcvkmbziw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iptnzzdfweungiwpvwsdcritvqzgiawe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"toirncfxuucgproaxveqtpsrbtohwklh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clojqxlkcpsctqtxydzouodciefqlfhf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bmcrqmwkvtbggbrlxmjfvkibupuslddc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbaasuznysvgecrqyrisixoawisubyls","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tprtftghjoxmvntffsafgzusobpyfvzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcgagpjkhabujbgvhhfgielaqmtcrhck","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xzgfgruhqjhifjflorpmffnttlrgyhtv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtqucnfoxjpcvvdbvcqihwlkqepwpgbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfaywdqwxhicopcmhlwwbmhuyulwedau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ouwkhzmyfvuaiisicrzmduxtkiictngv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdfygdclhhcqtrmuatbahcvptyuveqid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbyunaqkzmdkylzssaosxqjxrljvzchz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rukfmanumvdaukktomrdibgrwonxaatn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yuvnxdiltqdgkgspdrzczkgspytfbcjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mogzlhexhwjpbkwioivfosykadxtngfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rapehblzpqhghwcbgqaoxkottxtvfxts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfkxoomwyjrahvukrsajnouxrcqguxhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgqeeyovxkjfyidbspkvxjbghbaomuna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwjaisncbljfeiksvozfldyoilclveiy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"usaegaumtafmnfxhdveilgvmktyrosrp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"irtpahgsjzeaibrfhavzwzuujsfnkgtu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"moedisqvbmjuvtlegepxxmdndaoyhwgw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tfbespmcybocfvztyxbfrlxbrnsgdivl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxbfsmnnvzhzjrbmvpjxkymruoflpzdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ukzraawvkzowekurpqobxnjaymsktasf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lcekrqlfkukzmfdnrqpqjdnygyxyfynl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqulottedyyljalziswozfrrjhbmdqgm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvecvklumiwejsmjdvzshnjhwvxeuaue","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ncrqjpscmlajzyldjmedosivgbgntuuz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mskzxsvtoofzbvribqxalhpshroqvjhh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfmxeabgbyxvhxzqywskyiqkefgrwzvz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wrxpepztnczydpcvkiohevlofddkpbwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oefusigixwhqdlximnjhfltmralrpcos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"onofddlhvyvtrcvjbaeprwziucfkuwrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcnrkpqsztldfsziwmizjjgehzqlzvno","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wiuotxauiyjbnahagfngedwghfqpotzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"balonewrcysdgaasbrozosvzefzazjvj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kriwgaurjjhvzdlhlfqamdxgqywrukjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ylriffqytmnlnmniuqqpxleiynnfdvwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzfhuzdaldqfuapftyyfgqnvzbzstlge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suqojzaqcwqfaylrxkdyjqjcmzbhrucf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbwmddxnzvoyvawsgrqiyspsvbmlwipu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkmfdafyhgzharjmuskwevwuhigvgokk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wemhetrjkohbffgvwdfhlwpvhgxqjxqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zrwlpxjoenaddzfeshrggwvcfsmtlesw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jkkywrjlmixzvdmhgczlvwkbijvpovrw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oahsdejvowtsdwoajfnrwbkzgwczcbqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rudcmqaebgqbuzihgjuyhzmvwzefiuic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjxzywtplwczcmmnlmzwjyxmsjqmpwrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"puxgltsmaliuridjmlhguqwkmxeapnas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dnknreaumwmpovcvcqzyuadlydttocdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wlwinedjghrlrtfgicmfgmuohfsghqqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydbbnbkqrhsrgyoyhxebfbtdawrkwufy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bfbfxncbuhwzjbmpoargvjghuulpdngi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kryzpncupjxxmbsxgonfjghxcmbynzln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qbtakpyegbcjvkvbppndltduxpaqdjvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjpicfvhahvrcmrtqhsxocrdqpywtvcd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uquissdxsbxbunkknmwcwsybydajzjvv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfgwvkywcjvojsgstijdmupnodexjoye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"muoeqmcawqzrbdosjpjywipireaffpuh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nlmjxdvmoecgaifvqwtlmrqwxcsdoqte","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmdlhiuqufdpmopgfvwcdftjihmkukgd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhejboilivnppuadueozvppzjgpchksv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rwvjzhzloyjkwuwygjbxhcysbwrebsse","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dliqqngavxopxrkeoxojcawvfbxuhirv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669128,"databaseName":"models_schema","ddl":"CREATE TABLE `kmifjrkanbihkzrymcquzyofrbogiygu` (\n `gpayyeejjhhwykzwiobdevxxrkykqdpc` int NOT NULL,\n `jmqhbxncdfmwadhhabcecifykvtommay` int DEFAULT NULL,\n `udqfuetqronsykpqgjrqdxdocsrfrvbv` int DEFAULT NULL,\n `agzdqkngtmahrcyonaowbiaycmygyglk` int DEFAULT NULL,\n `cceqsyrrmdauhubbmkbwsaehejnzljzv` int DEFAULT NULL,\n `ijxtzffnnmhqpjeahcvvgaynicgkqfyd` int DEFAULT NULL,\n `tqstazewnjtkvvpyieitwoluztgxslvo` int DEFAULT NULL,\n `rtqskdgndnotbamhfifcbtfviukvkoww` int DEFAULT NULL,\n `bqriexnfcvsqwpffsvsfyjqsfnzvyllg` int DEFAULT NULL,\n `bslyxndvyzfphuvwnspybxuyktlhhqkq` int DEFAULT NULL,\n `hbpyachdenbdokmmgkzniilhejiwtdev` int DEFAULT NULL,\n `vimpojwrdsdsoolwqiirhpiygdnwpehj` int DEFAULT NULL,\n `apjgmmjycwxmtdtvxzcworjbwmcvqphn` int DEFAULT NULL,\n `jcmmgwlkuszmtbkghkngciqczwfkqrrm` int DEFAULT NULL,\n `yfyghachhrgjnrzzpfrzrfubaoczgigo` int DEFAULT NULL,\n `yzalafxfbvenldrsaewkxruzelaeakso` int DEFAULT NULL,\n `txccvssaadichcsxwvqwjksgcfmunhyt` int DEFAULT NULL,\n `eviblbrdaortqjgicasjvlewoddkqykm` int DEFAULT NULL,\n `ufqoxclvdxsxbxofvjaocmgymopsvrtx` int DEFAULT NULL,\n `tvgrlherqeujptbzympznrhfkhqlaabp` int DEFAULT NULL,\n `fevelawtldnwfivjenbtlcstcuhzbhey` int DEFAULT NULL,\n `ozdeykjilwvxgpausuudniyicruczpwq` int DEFAULT NULL,\n `rtvohzgapedtrsfjytxmuyekfogvnltq` int DEFAULT NULL,\n `mdhlixtpotjljwicoqfzevnzzbgbfocm` int DEFAULT NULL,\n `qbwyqndibgpvenezeivatfcrkpcelnjz` int DEFAULT NULL,\n `cmmnenvhagtmrzjmmdduddveiipgqacr` int DEFAULT NULL,\n `qweaczfviwxztrhgdxuwlweehzyivofm` int DEFAULT NULL,\n `iayfwqaxqpwiynfqmhoecnzqdfzzjyfu` int DEFAULT NULL,\n `cqinkkbwfbthxszisaqvpjmmzppzlunf` int DEFAULT NULL,\n `paqebxlbuhjhixhypbzvcwgkekpvvzae` int DEFAULT NULL,\n `qxpwyjflombluofwvotdekzsytnnwata` int DEFAULT NULL,\n `vllesssxncbnjexsottizhwbnmpsbuju` int DEFAULT NULL,\n `paonvuynkvpyejmnkyiajtkcvcqgevbw` int DEFAULT NULL,\n `jpkltckoykqikcbzrslopqiloctywxsx` int DEFAULT NULL,\n `bubsglbgqqlipzbvekjutcbjksutphcg` int DEFAULT NULL,\n `yxsoylyhwznwttwqlqmrshvjfmoheyrx` int DEFAULT NULL,\n `sfnvppxkxaxqucjoywjwkkldtzyckhxu` int DEFAULT NULL,\n `cznzfdtfkpskgwtsybkdeuqyegzppfio` int DEFAULT NULL,\n `kqkefghflppuhluonpmqdsozrewgmruj` int DEFAULT NULL,\n `ktvcofobtaxmtootgkgmulmziojknkfn` int DEFAULT NULL,\n `wyiopgoezlwjqvvrqwtvbfgskcobggob` int DEFAULT NULL,\n `zqzhfugepphyumpsagskbmvsaytvamng` int DEFAULT NULL,\n `ipnhjgvjyvqdxnzfhkhexxsphhxltiyd` int DEFAULT NULL,\n `jcugkzpqnfxpeadimkyvpmxpbqyaezfg` int DEFAULT NULL,\n `lrtyohmbmahgyhwhszbeispvbvxgbvwn` int DEFAULT NULL,\n `busekztpomiljexegrsfnyrggfkescln` int DEFAULT NULL,\n `lfqdgdnswnzkdzsrqkieewtsuaiheqmn` int DEFAULT NULL,\n `royukaveojjudawlmynxbxvlvhrdvgyt` int DEFAULT NULL,\n `fextuxictbppurpkwyjmjmoidpnzapty` int DEFAULT NULL,\n `yhasimokfxivqvmsalfwhfardymbgugy` int DEFAULT NULL,\n `ozjrzxyvklqrbqlsjoqcertpkieensei` int DEFAULT NULL,\n `mfrvhhoknfnpkuysrsrapaxtvgwuyreh` int DEFAULT NULL,\n `udjzrcoxmpjaxklylbrqrvbaruiloybe` int DEFAULT NULL,\n `hoqakjysggprpcsdvrziwyaurpcxuphk` int DEFAULT NULL,\n `bcqqumqbvqsufhwzhmogxjmpaplicjyn` int DEFAULT NULL,\n `ohuhxgwulnfjvorpbqzvsbzkxrkcgcom` int DEFAULT NULL,\n `tqdqxusqbovfsemauuwbidceuoxknjmx` int DEFAULT NULL,\n `gkeupfjivkvaddcnsdtkqtrtdpkyfugi` int DEFAULT NULL,\n `abfcskieovfizcbfujhwtunmyeatidms` int DEFAULT NULL,\n `iuvurpkpjksvcbecdbziyhmlkhpoagwx` int DEFAULT NULL,\n `sgfktybkcwxntpiholkjynqxlzchwvhn` int DEFAULT NULL,\n `ccsqhophgyhmusaqrxizaayuxpheqyra` int DEFAULT NULL,\n `wmebpfpxmjovgipnyegytkxcszqkakei` int DEFAULT NULL,\n `axjwisaivzeeabyqehjmkewurskhncyv` int DEFAULT NULL,\n `jwlkzztlctytzwtcwtfeicirgyholkxn` int DEFAULT NULL,\n `ngdvzlbjlfkkuxshttpofclrqgnrtvfk` int DEFAULT NULL,\n `vzabnxillzfbedzkwjidruiyadgqqofy` int DEFAULT NULL,\n `vmfhyfrcwiqehcjhlscgjrdbphszbyot` int DEFAULT NULL,\n `volwofltvyzvhcwczmjomrlxrgflzkpp` int DEFAULT NULL,\n `tgnvafiphhdtmnhopynsradvmyrmdixe` int DEFAULT NULL,\n `qnzyweqfondmpzrklogghfylqsiyentd` int DEFAULT NULL,\n `djnfyysbiwwwxhmwacpwlixjpcmlprbh` int DEFAULT NULL,\n `erdvjguynbaeocyricennyshjtbtkglb` int DEFAULT NULL,\n `gztbuzmhwstjryvbkxyvbmwwzteufiyg` int DEFAULT NULL,\n `nvrlvhehjxxjcixuswxpzdlxdsuurlwe` int DEFAULT NULL,\n `bqveyiitpdwuvetfbknlpyhvnoeatkvp` int DEFAULT NULL,\n `xcsveiuscghnifdvpxzigykphfgtodsv` int DEFAULT NULL,\n `kwqqafsoonokrketzgnrqokuvpubyxkd` int DEFAULT NULL,\n `lcsolawpwnpizembooemrdkqvbpckaxb` int DEFAULT NULL,\n `rkqwazmldtsjrgziznpylrlsuagwqosg` int DEFAULT NULL,\n `evowzxvapnsenwueznbqpuorzwjetgzf` int DEFAULT NULL,\n `ukhanoxbfmtsonffxjcvupwvkkrczbpr` int DEFAULT NULL,\n `fncpqldxbyhqbsniolnyjgnsuwdjvjjk` int DEFAULT NULL,\n `wfcurcsjtgvscqkxnoffhskwmmwtwhuj` int DEFAULT NULL,\n `uvgsrrpdjveagtjwnmfimzqucoeitxhs` int DEFAULT NULL,\n `xdowvvyvixohstuihugivhtcqgaycrqj` int DEFAULT NULL,\n `exujfekcqettldxxywueuzwuelednjkp` int DEFAULT NULL,\n `yvqpdhcvnsbvvqpyrfswaywrrlyoetmu` int DEFAULT NULL,\n `xhivngdxuvfjlrfqdpxzmislgwxvtpml` int DEFAULT NULL,\n `wcqbasuybvcnbskyyigyuxlfabssvnlz` int DEFAULT NULL,\n `qjvigpwjcjpsuhqlocslrqnzbachebbr` int DEFAULT NULL,\n `qalottsxujgqitwvvwxhvzbatqexhcys` int DEFAULT NULL,\n `ohpxfenrgunkyycyzvwsynswpwxfzbia` int DEFAULT NULL,\n `tctyhgpjqsrpwlqykolkwsizoooalxmv` int DEFAULT NULL,\n `lljxtvhskgipxtntirnzmtvsladvbwri` int DEFAULT NULL,\n `sfjbwgepxdapheldfrajhsybcgygmbpw` int DEFAULT NULL,\n `ifdydcetarytigbzruihlpfvpzautzhu` int DEFAULT NULL,\n `njroxflxdrmwdylcxvbwmdyaxykdxstt` int DEFAULT NULL,\n `xghpwtgjtlnjcnuimqpvcefeomozbhxp` int DEFAULT NULL,\n `zizaowdgwhjqgfsievckvgaesdjdktwt` int DEFAULT NULL,\n PRIMARY KEY (`gpayyeejjhhwykzwiobdevxxrkykqdpc`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"kmifjrkanbihkzrymcquzyofrbogiygu\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["gpayyeejjhhwykzwiobdevxxrkykqdpc"],"columns":[{"name":"gpayyeejjhhwykzwiobdevxxrkykqdpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"jmqhbxncdfmwadhhabcecifykvtommay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udqfuetqronsykpqgjrqdxdocsrfrvbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agzdqkngtmahrcyonaowbiaycmygyglk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cceqsyrrmdauhubbmkbwsaehejnzljzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijxtzffnnmhqpjeahcvvgaynicgkqfyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqstazewnjtkvvpyieitwoluztgxslvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtqskdgndnotbamhfifcbtfviukvkoww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqriexnfcvsqwpffsvsfyjqsfnzvyllg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bslyxndvyzfphuvwnspybxuyktlhhqkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbpyachdenbdokmmgkzniilhejiwtdev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vimpojwrdsdsoolwqiirhpiygdnwpehj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"apjgmmjycwxmtdtvxzcworjbwmcvqphn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jcmmgwlkuszmtbkghkngciqczwfkqrrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfyghachhrgjnrzzpfrzrfubaoczgigo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzalafxfbvenldrsaewkxruzelaeakso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txccvssaadichcsxwvqwjksgcfmunhyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eviblbrdaortqjgicasjvlewoddkqykm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufqoxclvdxsxbxofvjaocmgymopsvrtx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvgrlherqeujptbzympznrhfkhqlaabp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fevelawtldnwfivjenbtlcstcuhzbhey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozdeykjilwvxgpausuudniyicruczpwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtvohzgapedtrsfjytxmuyekfogvnltq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdhlixtpotjljwicoqfzevnzzbgbfocm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qbwyqndibgpvenezeivatfcrkpcelnjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmmnenvhagtmrzjmmdduddveiipgqacr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qweaczfviwxztrhgdxuwlweehzyivofm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iayfwqaxqpwiynfqmhoecnzqdfzzjyfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqinkkbwfbthxszisaqvpjmmzppzlunf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"paqebxlbuhjhixhypbzvcwgkekpvvzae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxpwyjflombluofwvotdekzsytnnwata","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vllesssxncbnjexsottizhwbnmpsbuju","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"paonvuynkvpyejmnkyiajtkcvcqgevbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpkltckoykqikcbzrslopqiloctywxsx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bubsglbgqqlipzbvekjutcbjksutphcg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxsoylyhwznwttwqlqmrshvjfmoheyrx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfnvppxkxaxqucjoywjwkkldtzyckhxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cznzfdtfkpskgwtsybkdeuqyegzppfio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqkefghflppuhluonpmqdsozrewgmruj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ktvcofobtaxmtootgkgmulmziojknkfn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyiopgoezlwjqvvrqwtvbfgskcobggob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqzhfugepphyumpsagskbmvsaytvamng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ipnhjgvjyvqdxnzfhkhexxsphhxltiyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jcugkzpqnfxpeadimkyvpmxpbqyaezfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lrtyohmbmahgyhwhszbeispvbvxgbvwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"busekztpomiljexegrsfnyrggfkescln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfqdgdnswnzkdzsrqkieewtsuaiheqmn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"royukaveojjudawlmynxbxvlvhrdvgyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fextuxictbppurpkwyjmjmoidpnzapty","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhasimokfxivqvmsalfwhfardymbgugy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozjrzxyvklqrbqlsjoqcertpkieensei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfrvhhoknfnpkuysrsrapaxtvgwuyreh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udjzrcoxmpjaxklylbrqrvbaruiloybe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hoqakjysggprpcsdvrziwyaurpcxuphk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bcqqumqbvqsufhwzhmogxjmpaplicjyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohuhxgwulnfjvorpbqzvsbzkxrkcgcom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqdqxusqbovfsemauuwbidceuoxknjmx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkeupfjivkvaddcnsdtkqtrtdpkyfugi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abfcskieovfizcbfujhwtunmyeatidms","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iuvurpkpjksvcbecdbziyhmlkhpoagwx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sgfktybkcwxntpiholkjynqxlzchwvhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ccsqhophgyhmusaqrxizaayuxpheqyra","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmebpfpxmjovgipnyegytkxcszqkakei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axjwisaivzeeabyqehjmkewurskhncyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwlkzztlctytzwtcwtfeicirgyholkxn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ngdvzlbjlfkkuxshttpofclrqgnrtvfk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzabnxillzfbedzkwjidruiyadgqqofy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmfhyfrcwiqehcjhlscgjrdbphszbyot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"volwofltvyzvhcwczmjomrlxrgflzkpp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tgnvafiphhdtmnhopynsradvmyrmdixe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qnzyweqfondmpzrklogghfylqsiyentd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"djnfyysbiwwwxhmwacpwlixjpcmlprbh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"erdvjguynbaeocyricennyshjtbtkglb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gztbuzmhwstjryvbkxyvbmwwzteufiyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvrlvhehjxxjcixuswxpzdlxdsuurlwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqveyiitpdwuvetfbknlpyhvnoeatkvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xcsveiuscghnifdvpxzigykphfgtodsv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwqqafsoonokrketzgnrqokuvpubyxkd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lcsolawpwnpizembooemrdkqvbpckaxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkqwazmldtsjrgziznpylrlsuagwqosg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evowzxvapnsenwueznbqpuorzwjetgzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ukhanoxbfmtsonffxjcvupwvkkrczbpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fncpqldxbyhqbsniolnyjgnsuwdjvjjk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfcurcsjtgvscqkxnoffhskwmmwtwhuj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvgsrrpdjveagtjwnmfimzqucoeitxhs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdowvvyvixohstuihugivhtcqgaycrqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exujfekcqettldxxywueuzwuelednjkp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yvqpdhcvnsbvvqpyrfswaywrrlyoetmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xhivngdxuvfjlrfqdpxzmislgwxvtpml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcqbasuybvcnbskyyigyuxlfabssvnlz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjvigpwjcjpsuhqlocslrqnzbachebbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qalottsxujgqitwvvwxhvzbatqexhcys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohpxfenrgunkyycyzvwsynswpwxfzbia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tctyhgpjqsrpwlqykolkwsizoooalxmv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lljxtvhskgipxtntirnzmtvsladvbwri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfjbwgepxdapheldfrajhsybcgygmbpw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifdydcetarytigbzruihlpfvpzautzhu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njroxflxdrmwdylcxvbwmdyaxykdxstt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xghpwtgjtlnjcnuimqpvcefeomozbhxp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zizaowdgwhjqgfsievckvgaesdjdktwt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669159,"databaseName":"models_schema","ddl":"CREATE TABLE `korrbwglsldbalqvsswaqstzuxvnlrps` (\n `vdnjtbaguirntqnoyqivoefwieuumpwf` int NOT NULL,\n `lfiwchvjvflqxecgnpkpwsmqpegnlsvf` int DEFAULT NULL,\n `wbrfjumgyjjmliqfrzsyuqrndbkchcnb` int DEFAULT NULL,\n `jdddgxoqlrscitvpuexducyowlelzdcy` int DEFAULT NULL,\n `radlgasynweomtklggfuydkkekdqqsqw` int DEFAULT NULL,\n `qfangdfwoeykkaqjffttpqkeykxscsni` int DEFAULT NULL,\n `fvoucsmttcslmbyjtfbtqxdnlzgkgwjr` int DEFAULT NULL,\n `didfbzhfqpkiavkvepvesbwgvywvpyde` int DEFAULT NULL,\n `cmsrgymspnkjoqdmlnrvbjkqpjxxderz` int DEFAULT NULL,\n `bvovwnkvocevpsysplsfmgvpzykerreg` int DEFAULT NULL,\n `tvwjcjkaljaknqcddofynhhyfuiaxubh` int DEFAULT NULL,\n `pqspzmukmxlanpnyewhcfskzpsmhaqux` int DEFAULT NULL,\n `vefowtvbzjsosdvuhsfmajlphlgacuqs` int DEFAULT NULL,\n `qaldiexpfxzstyhxlazjekijwfhtnjkt` int DEFAULT NULL,\n `uqxrvgouajzziroujomshnaubvfvqhew` int DEFAULT NULL,\n `fqaheqxmzhroxxyzkynsxeigkipgxdjh` int DEFAULT NULL,\n `ppetjauxdmbznwdnqiqpefoxjnsrqngi` int DEFAULT NULL,\n `cezjyfemijhghzilgxrnwdskkuyqkfsj` int DEFAULT NULL,\n `pdmaafinzkvyasqpuyneyfjrzjcciahi` int DEFAULT NULL,\n `mtagjeaxgiwneesfdouiiirpwmfmiyjw` int DEFAULT NULL,\n `xurbmhytctkibcnowfqgnxpaoyizvtxt` int DEFAULT NULL,\n `mxnyvruqcjoxcnsrohasvzlkzboevmxw` int DEFAULT NULL,\n `gkjtffzcxdrydlhnouxqbvuvikoykcvn` int DEFAULT NULL,\n `regvhfmgnmgrbjspfhutzhlzxjqhfsie` int DEFAULT NULL,\n `ffcpjhphodtsbajklutdottcqhvupzwt` int DEFAULT NULL,\n `jfrooqzqvfziqeogaiymgtljymotydks` int DEFAULT NULL,\n `drwyfhcaujrkwpaioaunfwjflnubgmwu` int DEFAULT NULL,\n `ccxcfihrjxrzcfwoomdwvokuczfwnmup` int DEFAULT NULL,\n `kmbkdkgotrzodqhwpocgytwderidfddc` int DEFAULT NULL,\n `wpkcpsumgaobuqwnlaecciqylawllibw` int DEFAULT NULL,\n `mrqzuvfcejopgpnbokowsrigcztstfxf` int DEFAULT NULL,\n `erbwrmyjuxaqpbyvmxqkvgtsmnpypgns` int DEFAULT NULL,\n `iqckpvcuuxiquoakfzyzthigcpowgffe` int DEFAULT NULL,\n `ukvffzvglyrkjwrvoyqbpeqcftpctjur` int DEFAULT NULL,\n `hwejpkitorkgvzgwrwlvfhrluenyzmol` int DEFAULT NULL,\n `ghynlavpmvzrlszizfhantssktgrajzc` int DEFAULT NULL,\n `lysavxkbatplmofpzbigogevacmbwolf` int DEFAULT NULL,\n `nwieavbehmrpzgnwvbcwhusbhqeggfed` int DEFAULT NULL,\n `hmkwzxrjtoagsjjnnhawmsskwvmwyxib` int DEFAULT NULL,\n `ulkfdufctrbgleclpzsmshmqtujkcqyw` int DEFAULT NULL,\n `ntahejocyqpqmckbidshzwwyaqdpdami` int DEFAULT NULL,\n `otahycyhnqqalsxzonkquelkvpfcxbtx` int DEFAULT NULL,\n `fwfejyvkqorvylrnhqncochlgfayydzy` int DEFAULT NULL,\n `mojqntppkbrfeanvdekqyfaxypmfjiga` int DEFAULT NULL,\n `eskqzwxmlbpaejhuzhgydkzvqkiiobkc` int DEFAULT NULL,\n `rrekpkrkpfdtwccdcyoraxjtnmbxjwof` int DEFAULT NULL,\n `dshkjyucuyubkylbrrobbnjrfbicmxfn` int DEFAULT NULL,\n `tgqqdextuczmlddbedbhehgiwyehlfqo` int DEFAULT NULL,\n `hcwnkfjqjawbjnarpinqwaqekewhewzz` int DEFAULT NULL,\n `pjcrolnjwvhyxgilmzgygedaiskwrtjr` int DEFAULT NULL,\n `awwmxxgnkycxsjqfhhsailunoxbhdhhk` int DEFAULT NULL,\n `isabpojkhoodatmirnvfdqmmdpttdfxx` int DEFAULT NULL,\n `ehhlwlpmuapaxlaurwtecroekwnagqnb` int DEFAULT NULL,\n `omrxywdgyyjmgdnjdyqzlsqpdmfesuil` int DEFAULT NULL,\n `rkjazzsrpcodqmmrxmyorbdbwqqnemku` int DEFAULT NULL,\n `csdbclkbqddqokbzpxbkgcizgouelfas` int DEFAULT NULL,\n `pvhubgyuzohlnsawglddkxdtkphvhoyw` int DEFAULT NULL,\n `yrpjdgltoebeytelinbldwmdkzhvafpn` int DEFAULT NULL,\n `lhzjgyskifgtgsyusbecuugzahqhspcf` int DEFAULT NULL,\n `iuqikzdttzepccxaohicdmsaikrisbrb` int DEFAULT NULL,\n `rytimonptnmkaynarpdouecurovzxbzn` int DEFAULT NULL,\n `xkgxhqfbaixqptkrvtcdvnqawdovancg` int DEFAULT NULL,\n `yguwfykssqbfgtfxqcspnjfzsztmoeez` int DEFAULT NULL,\n `vbvcvlvebwavyqiocziosvppkdpumvye` int DEFAULT NULL,\n `pajrcffqidiknlgjnqoregjkqopehnhp` int DEFAULT NULL,\n `xrmkkshrdudtgeafwtdiytsjjdnyrrov` int DEFAULT NULL,\n `hpcvwrfppvdkqhsgdmddltthyspzziph` int DEFAULT NULL,\n `gcmbsfspbolljaujfeptfbgdshlhveaj` int DEFAULT NULL,\n `jmawqzlzdbyjtbbqkdubzxmlhlxczrjh` int DEFAULT NULL,\n `jzxxpzyfftezgvnbkqfbcuswiuqxxybw` int DEFAULT NULL,\n `uaypnjmwdxcclaxcdxuiazxucljznrud` int DEFAULT NULL,\n `khsexfyxmaiwosxtbtfellsiomkkljqb` int DEFAULT NULL,\n `urwhdboloixjnqlkrgovuetbchivqwfn` int DEFAULT NULL,\n `dlgwyjpvjuddxjazvuckomtqpalrmctp` int DEFAULT NULL,\n `xhvbbsupyhkuuxyagwbceacsgagceerr` int DEFAULT NULL,\n `qfloipzfoihjpieqlmwjqcetkgqgfsaj` int DEFAULT NULL,\n `iwksnhslhsmfmuvcrdhhzzfjqikoustn` int DEFAULT NULL,\n `knsvxvhlzgahanvkiwcgyspkergxngoz` int DEFAULT NULL,\n `rmqeshlhqfivsmrebmwgylijwwrihmgg` int DEFAULT NULL,\n `idgqbuhorxmmyjltjpubzoziqxsuvzyz` int DEFAULT NULL,\n `lbchoyutgrojuxnenjgezggwzyfqmomh` int DEFAULT NULL,\n `daxspeonmkjojhtjexwervssmunyovvd` int DEFAULT NULL,\n `xetctonjanvenueycrunbouvrddwvtqe` int DEFAULT NULL,\n `nosjigacpqqjenkgxdncdkdecqzxxhrg` int DEFAULT NULL,\n `ubgkprzjwfohnykpclxyfqhminrnjtcs` int DEFAULT NULL,\n `xzmwepqtblnlfueonqkfosewjbjiwapf` int DEFAULT NULL,\n `bhiplyqhylkjlifqfuypnqaydkbzlehj` int DEFAULT NULL,\n `ewjvwkdddfiglezltcbfqpbnfmmdgknc` int DEFAULT NULL,\n `ppgbqhhwsbhwcdhzulguqdsidkaxfqos` int DEFAULT NULL,\n `zqxborpxcmxvtfpzjnkctvsenwpypkof` int DEFAULT NULL,\n `ftjujpwvldvjgawrcdnbqvsuufrtiuwy` int DEFAULT NULL,\n `nonoonievcjzbchgrxuytasovyurmjmk` int DEFAULT NULL,\n `mwwginxqxiglloldwrzyddgcncqdrqrf` int DEFAULT NULL,\n `laorucjvptkeimrkankhryxujigtagbm` int DEFAULT NULL,\n `pqetscaepcidulvdvqobfdlngbivjrit` int DEFAULT NULL,\n `fettsztuuvkgrxrrfcpeulfpqkxsrdes` int DEFAULT NULL,\n `sptevwixnzykynvdwxhaewinolhloyuq` int DEFAULT NULL,\n `prhfayghkijgknlinpzvefsjuhjdcdtw` int DEFAULT NULL,\n `bueollqfjwhssdltmyknkmqbrqhjdivu` int DEFAULT NULL,\n `sfftyjxrtnqrbuavynqeuxapmdkvclkq` int DEFAULT NULL,\n PRIMARY KEY (`vdnjtbaguirntqnoyqivoefwieuumpwf`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"korrbwglsldbalqvsswaqstzuxvnlrps\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["vdnjtbaguirntqnoyqivoefwieuumpwf"],"columns":[{"name":"vdnjtbaguirntqnoyqivoefwieuumpwf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"lfiwchvjvflqxecgnpkpwsmqpegnlsvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbrfjumgyjjmliqfrzsyuqrndbkchcnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdddgxoqlrscitvpuexducyowlelzdcy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"radlgasynweomtklggfuydkkekdqqsqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfangdfwoeykkaqjffttpqkeykxscsni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fvoucsmttcslmbyjtfbtqxdnlzgkgwjr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"didfbzhfqpkiavkvepvesbwgvywvpyde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmsrgymspnkjoqdmlnrvbjkqpjxxderz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvovwnkvocevpsysplsfmgvpzykerreg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvwjcjkaljaknqcddofynhhyfuiaxubh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqspzmukmxlanpnyewhcfskzpsmhaqux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vefowtvbzjsosdvuhsfmajlphlgacuqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qaldiexpfxzstyhxlazjekijwfhtnjkt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqxrvgouajzziroujomshnaubvfvqhew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqaheqxmzhroxxyzkynsxeigkipgxdjh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ppetjauxdmbznwdnqiqpefoxjnsrqngi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cezjyfemijhghzilgxrnwdskkuyqkfsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdmaafinzkvyasqpuyneyfjrzjcciahi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtagjeaxgiwneesfdouiiirpwmfmiyjw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xurbmhytctkibcnowfqgnxpaoyizvtxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mxnyvruqcjoxcnsrohasvzlkzboevmxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkjtffzcxdrydlhnouxqbvuvikoykcvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"regvhfmgnmgrbjspfhutzhlzxjqhfsie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffcpjhphodtsbajklutdottcqhvupzwt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jfrooqzqvfziqeogaiymgtljymotydks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drwyfhcaujrkwpaioaunfwjflnubgmwu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ccxcfihrjxrzcfwoomdwvokuczfwnmup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmbkdkgotrzodqhwpocgytwderidfddc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wpkcpsumgaobuqwnlaecciqylawllibw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrqzuvfcejopgpnbokowsrigcztstfxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"erbwrmyjuxaqpbyvmxqkvgtsmnpypgns","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqckpvcuuxiquoakfzyzthigcpowgffe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ukvffzvglyrkjwrvoyqbpeqcftpctjur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwejpkitorkgvzgwrwlvfhrluenyzmol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghynlavpmvzrlszizfhantssktgrajzc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lysavxkbatplmofpzbigogevacmbwolf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nwieavbehmrpzgnwvbcwhusbhqeggfed","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmkwzxrjtoagsjjnnhawmsskwvmwyxib","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ulkfdufctrbgleclpzsmshmqtujkcqyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntahejocyqpqmckbidshzwwyaqdpdami","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otahycyhnqqalsxzonkquelkvpfcxbtx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwfejyvkqorvylrnhqncochlgfayydzy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mojqntppkbrfeanvdekqyfaxypmfjiga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eskqzwxmlbpaejhuzhgydkzvqkiiobkc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrekpkrkpfdtwccdcyoraxjtnmbxjwof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dshkjyucuyubkylbrrobbnjrfbicmxfn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tgqqdextuczmlddbedbhehgiwyehlfqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hcwnkfjqjawbjnarpinqwaqekewhewzz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjcrolnjwvhyxgilmzgygedaiskwrtjr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awwmxxgnkycxsjqfhhsailunoxbhdhhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"isabpojkhoodatmirnvfdqmmdpttdfxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehhlwlpmuapaxlaurwtecroekwnagqnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omrxywdgyyjmgdnjdyqzlsqpdmfesuil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkjazzsrpcodqmmrxmyorbdbwqqnemku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"csdbclkbqddqokbzpxbkgcizgouelfas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvhubgyuzohlnsawglddkxdtkphvhoyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yrpjdgltoebeytelinbldwmdkzhvafpn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhzjgyskifgtgsyusbecuugzahqhspcf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iuqikzdttzepccxaohicdmsaikrisbrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rytimonptnmkaynarpdouecurovzxbzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkgxhqfbaixqptkrvtcdvnqawdovancg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yguwfykssqbfgtfxqcspnjfzsztmoeez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbvcvlvebwavyqiocziosvppkdpumvye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pajrcffqidiknlgjnqoregjkqopehnhp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrmkkshrdudtgeafwtdiytsjjdnyrrov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hpcvwrfppvdkqhsgdmddltthyspzziph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gcmbsfspbolljaujfeptfbgdshlhveaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmawqzlzdbyjtbbqkdubzxmlhlxczrjh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzxxpzyfftezgvnbkqfbcuswiuqxxybw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uaypnjmwdxcclaxcdxuiazxucljznrud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khsexfyxmaiwosxtbtfellsiomkkljqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urwhdboloixjnqlkrgovuetbchivqwfn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlgwyjpvjuddxjazvuckomtqpalrmctp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xhvbbsupyhkuuxyagwbceacsgagceerr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfloipzfoihjpieqlmwjqcetkgqgfsaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwksnhslhsmfmuvcrdhhzzfjqikoustn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knsvxvhlzgahanvkiwcgyspkergxngoz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rmqeshlhqfivsmrebmwgylijwwrihmgg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idgqbuhorxmmyjltjpubzoziqxsuvzyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lbchoyutgrojuxnenjgezggwzyfqmomh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"daxspeonmkjojhtjexwervssmunyovvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xetctonjanvenueycrunbouvrddwvtqe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nosjigacpqqjenkgxdncdkdecqzxxhrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubgkprzjwfohnykpclxyfqhminrnjtcs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xzmwepqtblnlfueonqkfosewjbjiwapf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bhiplyqhylkjlifqfuypnqaydkbzlehj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ewjvwkdddfiglezltcbfqpbnfmmdgknc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ppgbqhhwsbhwcdhzulguqdsidkaxfqos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqxborpxcmxvtfpzjnkctvsenwpypkof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftjujpwvldvjgawrcdnbqvsuufrtiuwy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nonoonievcjzbchgrxuytasovyurmjmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwwginxqxiglloldwrzyddgcncqdrqrf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"laorucjvptkeimrkankhryxujigtagbm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqetscaepcidulvdvqobfdlngbivjrit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fettsztuuvkgrxrrfcpeulfpqkxsrdes","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sptevwixnzykynvdwxhaewinolhloyuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prhfayghkijgknlinpzvefsjuhjdcdtw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bueollqfjwhssdltmyknkmqbrqhjdivu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfftyjxrtnqrbuavynqeuxapmdkvclkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669194,"databaseName":"models_schema","ddl":"CREATE TABLE `kuykqleuwgycpydwatkgiczdqgleegnu` (\n `uogmseavfyjrmovhuzmjuvhtnjivtsnh` int NOT NULL,\n `ofqgaxtpamspgvdjxwdlokeazpsnmhvl` int DEFAULT NULL,\n `euynytnwjznsidqfhqcysmqlgazzhpjj` int DEFAULT NULL,\n `ddkkarjvpbrdhcecnfoqchesnauufxoo` int DEFAULT NULL,\n `rsmezlhmfmifzfuehnsspxkgesqqwldm` int DEFAULT NULL,\n `bgvvwljyedbbcebondnzecydgyohvphk` int DEFAULT NULL,\n `zqtpdlkgpbmyxkechvybgvekrqrjnwpd` int DEFAULT NULL,\n `xyovhcbbwsvzawlyctoofiveksbalyqg` int DEFAULT NULL,\n `eldikjylgschsnabmlqzycekplrxnqba` int DEFAULT NULL,\n `uvyyqufumpyzmbcqdvzhflihezucrdxz` int DEFAULT NULL,\n `xdwrzffbpltpbofeeggwfljuteohoapv` int DEFAULT NULL,\n `ytjxcxkurlcxqvsmtpidobfbttkrdwdi` int DEFAULT NULL,\n `lxzdwocclfhurxjgxpmkeyufbyqnyhiw` int DEFAULT NULL,\n `hmqbgicgggerizjjymvyhzupsrpntaar` int DEFAULT NULL,\n `jqqpizkxqbdwlvsknzdgmvmxxlhkoqza` int DEFAULT NULL,\n `kzqhwmreyhwoforulpcyasgmushtbcum` int DEFAULT NULL,\n `lyxwbzcrbxrhhzwissarzbbllyqswtsf` int DEFAULT NULL,\n `rjgnioyyhuezryjfqzyemhuphrauvxoa` int DEFAULT NULL,\n `mccgvjrylkgetwlxnezafswueeojsfvp` int DEFAULT NULL,\n `dleksyovbxeuqnykjoxzmqtjkasufvsy` int DEFAULT NULL,\n `byrctakaayrvuuaehvbhpncdhornnxnu` int DEFAULT NULL,\n `ohsaolwcqjjlwpbutdxiuoytwzeltobd` int DEFAULT NULL,\n `xyplearpcdefttruqahaeqaroekayurh` int DEFAULT NULL,\n `jeireskqclcguqgwccuuqzaxheziiaxi` int DEFAULT NULL,\n `qgrsapbhccxfquwlriqvkmsvnmxcfkou` int DEFAULT NULL,\n `clsyodjqorvppprzqatkizmlqepyheyv` int DEFAULT NULL,\n `qtwbdqkdsmpvfsbgkdwrzomkolvejycd` int DEFAULT NULL,\n `pjcqujlutiypyhgubxfetziopxtmqtng` int DEFAULT NULL,\n `zwzgjoddwyertngslimjpfzsosmjwhiz` int DEFAULT NULL,\n `zpnrpsmboutjdbkvkmsodctvuvcogved` int DEFAULT NULL,\n `ettvwahqjbburpbgegumesaayencposa` int DEFAULT NULL,\n `qovvlysugmpqssfmxodstozcsaemgqpi` int DEFAULT NULL,\n `kaudxyqdjwydivmjkqeivetoygmktbsq` int DEFAULT NULL,\n `oioshucdmoflwcdxyaizzdzmuarbgrcd` int DEFAULT NULL,\n `rtnhiemmuheypzswdfxnrtevwcmxsniy` int DEFAULT NULL,\n `qckbohsnwordqyklgbsqdvyflwyarohd` int DEFAULT NULL,\n `gjsslphnjouiieyappfiwnhivffrhopp` int DEFAULT NULL,\n `yjncnhvmddbkobadsrxasaklmizmmmjm` int DEFAULT NULL,\n `erdfybuuqgrusnhjhusjupwewmzwiwug` int DEFAULT NULL,\n `xkzdmgsjmlkhrledaqsksiothjutnpyd` int DEFAULT NULL,\n `nbexgobjmewhbvmckhzkjngiswhiegrl` int DEFAULT NULL,\n `ioioeehtqevxzuevvmaojiuuicohjaou` int DEFAULT NULL,\n `ptlfbjfgkjgexjnhrrbdqcvifflqdpyp` int DEFAULT NULL,\n `ghamfjyojsanxwcrftpszlfqazdkvhwk` int DEFAULT NULL,\n `xcmunppfwbyvcmbkgjfllgtuhzlauiex` int DEFAULT NULL,\n `hmnwoampksyrfklhdxfpawijpusvcbcx` int DEFAULT NULL,\n `ajefmjptmssyjshucjxpnneljrprzhew` int DEFAULT NULL,\n `jfsinbftypleflrxoavvnswvurasmvgd` int DEFAULT NULL,\n `zbqysqlkjdcojmnwdmfjalbembjmpmeo` int DEFAULT NULL,\n `hkmsxstvjdiitewotffkxcrwkafwsptc` int DEFAULT NULL,\n `knxkgszbmnlwzlwnntojfxcepfxvlagd` int DEFAULT NULL,\n `umdjtaigdzyekweqbfbiqdzunigyukkg` int DEFAULT NULL,\n `gzzksydazsvmuoaajxgiitiybsyfzedg` int DEFAULT NULL,\n `uljcrpkkkjwxtfubmdcssicyfwdixcfx` int DEFAULT NULL,\n `qjiskfxmiruwxxhalfllwzqewzbdangb` int DEFAULT NULL,\n `grrwcprkbthptqhmtecybalwmowwfflr` int DEFAULT NULL,\n `qvvuyddakebevpuejxjtnpulwedykrcd` int DEFAULT NULL,\n `aqoyytzysahcyavdadcllsvqhyyufrtw` int DEFAULT NULL,\n `cpcxvppkiqerorrlczrvkbwvyijskrzi` int DEFAULT NULL,\n `fhjjnwnxpbdgnftxlkqyfnnadoboxlzg` int DEFAULT NULL,\n `jmsgchjzllffpzkjislcwudqhfankxav` int DEFAULT NULL,\n `ewgitpsiplqzjsksqxxbatxhrqvxwtaf` int DEFAULT NULL,\n `vrldsgnnrxdnevrzgordkrnjhwgnjrlj` int DEFAULT NULL,\n `givglgqplarawbmhwiicaodeaiwndqzv` int DEFAULT NULL,\n `veurfgmkerdqknpgzogjhucheapttpdb` int DEFAULT NULL,\n `wabjasrdnrffykliecqrgcoodkddclqo` int DEFAULT NULL,\n `axizulxlixutwvgxnawwypdzkzgxjrgv` int DEFAULT NULL,\n `nickgpxxopzlbvidrobyoxtuqkghhgcv` int DEFAULT NULL,\n `znvgwnwjsvbjraxxzajioryfgrspwguc` int DEFAULT NULL,\n `ddtuzzasergtpwetibjbhcbnmpdtpsbw` int DEFAULT NULL,\n `ovrutdgsgrmaivqsmrmvrzpnvtjfxeti` int DEFAULT NULL,\n `dusmpajminzvaflqrsflyyknxwxvlsle` int DEFAULT NULL,\n `usdtecncvgjpppzwvtdvjsguyvjmnngv` int DEFAULT NULL,\n `kxxgjwwsotxpewsubqwqyerftkyyoacb` int DEFAULT NULL,\n `vqcxwnrnqrioqxedffchgnsigwbazmxo` int DEFAULT NULL,\n `facnjqzosbtslewphseznokxnauzzxzc` int DEFAULT NULL,\n `nmpfwtxmypkizkpgymgkamglllwyujro` int DEFAULT NULL,\n `ntkcumibillxidhgrlatfgbwsguznxyv` int DEFAULT NULL,\n `exorxduqixiwscgnyghzcaaehldqmfme` int DEFAULT NULL,\n `mczghtwiunuvjfsymkucqlgymxazhikh` int DEFAULT NULL,\n `dotoldqknfilrtlahyujgnzorqppvpis` int DEFAULT NULL,\n `vgfixtxwwyjkyxbbihifrjygsezkzfqs` int DEFAULT NULL,\n `rxytxunfpluqidzmdvmjpvvyaclszjqr` int DEFAULT NULL,\n `rtycbauedewhiomvpgeefsddudqjihwb` int DEFAULT NULL,\n `zdiqcyxnvkkwqsjinvyulthlkxtbvzoc` int DEFAULT NULL,\n `qhwusgzvxcyupebseiqafkfjxenubccw` int DEFAULT NULL,\n `tevkkkdgakdyjydtsxsxbjnrchfitcsz` int DEFAULT NULL,\n `zkcwkzmsosusyccxhqtbawbbfervzgin` int DEFAULT NULL,\n `lyduklhrjshwkutjervsbcdphaphyjpx` int DEFAULT NULL,\n `llggdwrgdxdvknzajmfmugdkqpnsydxb` int DEFAULT NULL,\n `dgjwqrgejlqrfxbnlukeyjqizisarutk` int DEFAULT NULL,\n `ivtjgkrbpyltmnlqdmgddzbuobihnupu` int DEFAULT NULL,\n `mqfpjgceklbqivyqgfmthsdbslhhjcto` int DEFAULT NULL,\n `npekwhoeahfarpiwtskmquywcqaowalp` int DEFAULT NULL,\n `gnerastowdpblibaxqkciyqstfvhckjl` int DEFAULT NULL,\n `otolgwbsbzxlydyfpcykgcqdvepdgdip` int DEFAULT NULL,\n `blgbvwqasdmgplmaqivryzgbfcphgfuq` int DEFAULT NULL,\n `tocluxntoqenomijjmxhsijzqpfkvgpa` int DEFAULT NULL,\n `qzokmancnexmeyppmgeoyaqokopxmznj` int DEFAULT NULL,\n `jncyoszueabejrwlitshdbyndmjoycua` int DEFAULT NULL,\n PRIMARY KEY (`uogmseavfyjrmovhuzmjuvhtnjivtsnh`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"kuykqleuwgycpydwatkgiczdqgleegnu\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["uogmseavfyjrmovhuzmjuvhtnjivtsnh"],"columns":[{"name":"uogmseavfyjrmovhuzmjuvhtnjivtsnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ofqgaxtpamspgvdjxwdlokeazpsnmhvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"euynytnwjznsidqfhqcysmqlgazzhpjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddkkarjvpbrdhcecnfoqchesnauufxoo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rsmezlhmfmifzfuehnsspxkgesqqwldm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bgvvwljyedbbcebondnzecydgyohvphk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqtpdlkgpbmyxkechvybgvekrqrjnwpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyovhcbbwsvzawlyctoofiveksbalyqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eldikjylgschsnabmlqzycekplrxnqba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvyyqufumpyzmbcqdvzhflihezucrdxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdwrzffbpltpbofeeggwfljuteohoapv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytjxcxkurlcxqvsmtpidobfbttkrdwdi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxzdwocclfhurxjgxpmkeyufbyqnyhiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmqbgicgggerizjjymvyhzupsrpntaar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jqqpizkxqbdwlvsknzdgmvmxxlhkoqza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzqhwmreyhwoforulpcyasgmushtbcum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lyxwbzcrbxrhhzwissarzbbllyqswtsf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjgnioyyhuezryjfqzyemhuphrauvxoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mccgvjrylkgetwlxnezafswueeojsfvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dleksyovbxeuqnykjoxzmqtjkasufvsy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"byrctakaayrvuuaehvbhpncdhornnxnu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohsaolwcqjjlwpbutdxiuoytwzeltobd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyplearpcdefttruqahaeqaroekayurh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jeireskqclcguqgwccuuqzaxheziiaxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgrsapbhccxfquwlriqvkmsvnmxcfkou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clsyodjqorvppprzqatkizmlqepyheyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtwbdqkdsmpvfsbgkdwrzomkolvejycd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjcqujlutiypyhgubxfetziopxtmqtng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zwzgjoddwyertngslimjpfzsosmjwhiz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpnrpsmboutjdbkvkmsodctvuvcogved","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ettvwahqjbburpbgegumesaayencposa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qovvlysugmpqssfmxodstozcsaemgqpi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kaudxyqdjwydivmjkqeivetoygmktbsq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oioshucdmoflwcdxyaizzdzmuarbgrcd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtnhiemmuheypzswdfxnrtevwcmxsniy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qckbohsnwordqyklgbsqdvyflwyarohd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjsslphnjouiieyappfiwnhivffrhopp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjncnhvmddbkobadsrxasaklmizmmmjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"erdfybuuqgrusnhjhusjupwewmzwiwug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkzdmgsjmlkhrledaqsksiothjutnpyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbexgobjmewhbvmckhzkjngiswhiegrl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ioioeehtqevxzuevvmaojiuuicohjaou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ptlfbjfgkjgexjnhrrbdqcvifflqdpyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghamfjyojsanxwcrftpszlfqazdkvhwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xcmunppfwbyvcmbkgjfllgtuhzlauiex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmnwoampksyrfklhdxfpawijpusvcbcx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajefmjptmssyjshucjxpnneljrprzhew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jfsinbftypleflrxoavvnswvurasmvgd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbqysqlkjdcojmnwdmfjalbembjmpmeo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkmsxstvjdiitewotffkxcrwkafwsptc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knxkgszbmnlwzlwnntojfxcepfxvlagd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umdjtaigdzyekweqbfbiqdzunigyukkg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzzksydazsvmuoaajxgiitiybsyfzedg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uljcrpkkkjwxtfubmdcssicyfwdixcfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjiskfxmiruwxxhalfllwzqewzbdangb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grrwcprkbthptqhmtecybalwmowwfflr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvvuyddakebevpuejxjtnpulwedykrcd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqoyytzysahcyavdadcllsvqhyyufrtw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cpcxvppkiqerorrlczrvkbwvyijskrzi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhjjnwnxpbdgnftxlkqyfnnadoboxlzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmsgchjzllffpzkjislcwudqhfankxav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ewgitpsiplqzjsksqxxbatxhrqvxwtaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrldsgnnrxdnevrzgordkrnjhwgnjrlj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"givglgqplarawbmhwiicaodeaiwndqzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"veurfgmkerdqknpgzogjhucheapttpdb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wabjasrdnrffykliecqrgcoodkddclqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axizulxlixutwvgxnawwypdzkzgxjrgv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nickgpxxopzlbvidrobyoxtuqkghhgcv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znvgwnwjsvbjraxxzajioryfgrspwguc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddtuzzasergtpwetibjbhcbnmpdtpsbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovrutdgsgrmaivqsmrmvrzpnvtjfxeti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dusmpajminzvaflqrsflyyknxwxvlsle","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"usdtecncvgjpppzwvtdvjsguyvjmnngv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxxgjwwsotxpewsubqwqyerftkyyoacb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqcxwnrnqrioqxedffchgnsigwbazmxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"facnjqzosbtslewphseznokxnauzzxzc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmpfwtxmypkizkpgymgkamglllwyujro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntkcumibillxidhgrlatfgbwsguznxyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exorxduqixiwscgnyghzcaaehldqmfme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mczghtwiunuvjfsymkucqlgymxazhikh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dotoldqknfilrtlahyujgnzorqppvpis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgfixtxwwyjkyxbbihifrjygsezkzfqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxytxunfpluqidzmdvmjpvvyaclszjqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtycbauedewhiomvpgeefsddudqjihwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdiqcyxnvkkwqsjinvyulthlkxtbvzoc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhwusgzvxcyupebseiqafkfjxenubccw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tevkkkdgakdyjydtsxsxbjnrchfitcsz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zkcwkzmsosusyccxhqtbawbbfervzgin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lyduklhrjshwkutjervsbcdphaphyjpx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llggdwrgdxdvknzajmfmugdkqpnsydxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgjwqrgejlqrfxbnlukeyjqizisarutk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivtjgkrbpyltmnlqdmgddzbuobihnupu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqfpjgceklbqivyqgfmthsdbslhhjcto","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npekwhoeahfarpiwtskmquywcqaowalp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnerastowdpblibaxqkciyqstfvhckjl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otolgwbsbzxlydyfpcykgcqdvepdgdip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blgbvwqasdmgplmaqivryzgbfcphgfuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tocluxntoqenomijjmxhsijzqpfkvgpa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzokmancnexmeyppmgeoyaqokopxmznj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jncyoszueabejrwlitshdbyndmjoycua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669229,"databaseName":"models_schema","ddl":"CREATE TABLE `kvqskdqluyprqlisnljdnaxumzqetjoo` (\n `rrlafvjakfgfwbrlowthkshdowfzdkyj` int NOT NULL,\n `ymjbypvbjfopunyrfoiipeqjolumynks` int DEFAULT NULL,\n `fowdjyovsjhmgrrcvzjqbbipiralosee` int DEFAULT NULL,\n `kghdggstefwibndqkfnliabjqpkzlclj` int DEFAULT NULL,\n `awsoktvkzacpdcrtfgtxbllteegifywe` int DEFAULT NULL,\n `qqqjdwankclmobjydofdpmufdzqrbemm` int DEFAULT NULL,\n `elpeygiyolxgsqfqqfrgaqwcsiwywgfw` int DEFAULT NULL,\n `qrolnuochamdyaxzlkfyyaftlfpelyjl` int DEFAULT NULL,\n `ydawbxifcdqrdybkfhwomjmurvrwbmta` int DEFAULT NULL,\n `atbqzpouxjscgqqfyfjsknkrfvmvgplp` int DEFAULT NULL,\n `hbmkgqkknxelaiygqdwuifsgvoezwwno` int DEFAULT NULL,\n `fbjuswsqwuzdmmxicffpjniszvclzlsc` int DEFAULT NULL,\n `ooziocyxfolpwxsqqrcvtqsigkxaxrzd` int DEFAULT NULL,\n `olunngmxnhdztaokjgqsvztufqsweeyk` int DEFAULT NULL,\n `retvefwqupgfuooyngptkeglzdbjpqlx` int DEFAULT NULL,\n `esdqwlhizfmzghahnlvfadgempsuubjf` int DEFAULT NULL,\n `lkbyykdrdtkttnhshfsewfsibiwgardf` int DEFAULT NULL,\n `auhpmejagxecpeabqtzfcguuyirttqds` int DEFAULT NULL,\n `hzamsdcraobhklvdgxhjafzfqeerehpy` int DEFAULT NULL,\n `jxvjkcjzuwgikbmfezyyzpmwcebejndc` int DEFAULT NULL,\n `fdfuakdidilgrjjvtzeazioyzxceznvv` int DEFAULT NULL,\n `hymhvgrvninzfablvccohcxhwntwufob` int DEFAULT NULL,\n `gmbixipwlvogjcxvrkcpuqbmrqakpwdd` int DEFAULT NULL,\n `htnwhowmvrnftgfeotbljrfrtejtsqib` int DEFAULT NULL,\n `jrrwrdvjvecttzloqwblepckdhaeutfj` int DEFAULT NULL,\n `bolskxuebuqhwjpkjpsqmmrlxjfttxep` int DEFAULT NULL,\n `kjfyysovnapkzrnwbzpkmsydfdlzulku` int DEFAULT NULL,\n `aixenziwpwvdwyghlbwtpdahxluwzplp` int DEFAULT NULL,\n `oycwvsubzfoigduglkzzffbyvufrjiag` int DEFAULT NULL,\n `ikwenfsbuvxgdyplnrgqxszzofkpuffz` int DEFAULT NULL,\n `vqjlhrraxwxsvnsimzqihsvcuefcketl` int DEFAULT NULL,\n `japdqanfmrgztenmbjufhhlrzaslgpmr` int DEFAULT NULL,\n `onyzyrxfhwhimypwexifmveamtmnerdj` int DEFAULT NULL,\n `bgzisrxruaoawuzdpptqtenqfbhlzpgm` int DEFAULT NULL,\n `tysprxxtkpakdiaqtakhatgvowidagra` int DEFAULT NULL,\n `nlxrtzjrdrtpfzryfzwgyoihsdhukhky` int DEFAULT NULL,\n `fjhuhotzdloofjccvzedvlypvwbptnfv` int DEFAULT NULL,\n `mzpnshmnmqfnpbingcabmzomnexfcafx` int DEFAULT NULL,\n `faiqdendsnuliexaazbzjjknmudeizna` int DEFAULT NULL,\n `wqjgfpvorbojlkxhhwdoprlvgwenqgmy` int DEFAULT NULL,\n `zbsrkjfzmrhjcuqnkdhbjfvzxoluavqu` int DEFAULT NULL,\n `jjhinjeaknmgxtloomuwlvdvnrxshsqc` int DEFAULT NULL,\n `yzooxzeyttoxdoibmrhniormrhakvptl` int DEFAULT NULL,\n `rgnyouhptozynahgknhycdmhiscleayu` int DEFAULT NULL,\n `oivmnabaznagrqysjqbifywtamgdfoda` int DEFAULT NULL,\n `lmbpvgzvxjzmjpmctufpceyofiuswzmd` int DEFAULT NULL,\n `tqkrayveevouevrxmgxtxchxgxamdnvg` int DEFAULT NULL,\n `xigytzkbeducorfgqvnkoxbcmovjulmn` int DEFAULT NULL,\n `kvkzeuntfbtetkcxrqlmeqockpyjoyuj` int DEFAULT NULL,\n `ysyomzwbbzstepvvhlkyrwnpcimfswqt` int DEFAULT NULL,\n `usvdvrwhfrhxazcxtfefpsyvdjriddcq` int DEFAULT NULL,\n `acfgvlqalernpbdnhtslckcazvzqmdbh` int DEFAULT NULL,\n `qpudkfydbrwpzustjifqvizfgmakmwsv` int DEFAULT NULL,\n `zksmzkxaraihopappzilepxansvivirs` int DEFAULT NULL,\n `wkilmrjvxapadvutsvcqpjnhqszqpahr` int DEFAULT NULL,\n `zoahwwtxwdpfgxgilzgrychjtedrlpzg` int DEFAULT NULL,\n `qqtylovmqbfrlawrxjnljydczipczrwb` int DEFAULT NULL,\n `besuetnhbgvhspvhdmhoctzphcptnudt` int DEFAULT NULL,\n `bldchpxecokmyrmnlftfkfyoyugjcoqu` int DEFAULT NULL,\n `clvgygeymxmownfttjegfatgcvksliyr` int DEFAULT NULL,\n `dmnrrccawlpbbppgeainhkgkpwnohfhq` int DEFAULT NULL,\n `kysgnqtkfmqnhwckyvionlsrucwiyemv` int DEFAULT NULL,\n `datdcwvrdlrwkjmvuwzaffzketimriqt` int DEFAULT NULL,\n `qfiqxhwmlxglpkjoznarlwhfynutxaud` int DEFAULT NULL,\n `ugcaeqegilwevvafkhnzjabjdqjsvalu` int DEFAULT NULL,\n `uqsanuzmvfblcpjnzrtcpkvgwcdldcsj` int DEFAULT NULL,\n `wthoetennmfhetracjfbhajduilipoff` int DEFAULT NULL,\n `hhnqlwotmxhcumpnzadlfcqhawlwvcuh` int DEFAULT NULL,\n `nqmsjyujdcfqerwscwzqtatyrixcwbom` int DEFAULT NULL,\n `tvodxhmujaakyqmckiawdtimzrbecnrq` int DEFAULT NULL,\n `ztazdedwrjukdubfxaxvnnmtsqpolqms` int DEFAULT NULL,\n `bluzwlefintmazctwfvdpujrmxwthwlr` int DEFAULT NULL,\n `mrcynxfxghghnqcecuucvoolkzvfjjtq` int DEFAULT NULL,\n `lwadbuptnxrhipciukrpwnvqviqbzvqo` int DEFAULT NULL,\n `heakcnlcspfzorjsuqyezdrugejxqknm` int DEFAULT NULL,\n `zlcokmlvumnlhzkljcqrtampcardknqy` int DEFAULT NULL,\n `driuwatqywgzgvdbxlsnglffrpwydzxz` int DEFAULT NULL,\n `ecnuaqupwpwkaldqqbhvhugsnmkoygba` int DEFAULT NULL,\n `cmrucqjijlfpmhgzprndylqhgxzosbdo` int DEFAULT NULL,\n `sytqqozsazmuqehvztqlkahsksxntmim` int DEFAULT NULL,\n `oywjbtveuqovxtgdeqnguvrnzyjtxzgi` int DEFAULT NULL,\n `ekinhsgwvoayyvllbxlefbamolmjozzv` int DEFAULT NULL,\n `fselycmqlehjcuhrucdbftzrfmkxlybt` int DEFAULT NULL,\n `ngigphuombfndmzsodltnqawjogpiftl` int DEFAULT NULL,\n `cwcywsofsleoilmtqytmntzgfrieslag` int DEFAULT NULL,\n `rhvtcoeglwvvgahvgikrxbrlfkmbxnks` int DEFAULT NULL,\n `lkjxtbctmraakecxnlwkjzkizszjyqxb` int DEFAULT NULL,\n `rrfbelrxisujsnechxcdvgvdgyfncihp` int DEFAULT NULL,\n `asosfzkwxinustnelnykcjharwbrqvqt` int DEFAULT NULL,\n `bisokqbyjlfhxtaanlulkfyuqxvlljav` int DEFAULT NULL,\n `maxtdilrenuldkkoblpajelbxinwwlxn` int DEFAULT NULL,\n `qbzqorsfhsoluhxniedvlwsnqmdllxkm` int DEFAULT NULL,\n `ghsrxgsupvlggurgtbizotqxubchejfn` int DEFAULT NULL,\n `hmzlthrglwuwnxcgmtkpcaihocgvldjh` int DEFAULT NULL,\n `ebonfkftqucxacssdjxyxsupqarbxgig` int DEFAULT NULL,\n `oanmnqjghkndhrfofcbomjqvvtkmuwpw` int DEFAULT NULL,\n `dnjxznlyxurnoicgkwsalwytjhmnvsld` int DEFAULT NULL,\n `vsafxtbknvwkzmmeamnfkeexaqnaxxlj` int DEFAULT NULL,\n `mzygdjeryjfynfumqzlmaqrvzwhvysro` int DEFAULT NULL,\n `auahfcfmcxafvbpwognxzozjrkasyjyi` int DEFAULT NULL,\n PRIMARY KEY (`rrlafvjakfgfwbrlowthkshdowfzdkyj`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"kvqskdqluyprqlisnljdnaxumzqetjoo\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["rrlafvjakfgfwbrlowthkshdowfzdkyj"],"columns":[{"name":"rrlafvjakfgfwbrlowthkshdowfzdkyj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ymjbypvbjfopunyrfoiipeqjolumynks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fowdjyovsjhmgrrcvzjqbbipiralosee","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kghdggstefwibndqkfnliabjqpkzlclj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awsoktvkzacpdcrtfgtxbllteegifywe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqqjdwankclmobjydofdpmufdzqrbemm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elpeygiyolxgsqfqqfrgaqwcsiwywgfw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrolnuochamdyaxzlkfyyaftlfpelyjl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydawbxifcdqrdybkfhwomjmurvrwbmta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atbqzpouxjscgqqfyfjsknkrfvmvgplp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbmkgqkknxelaiygqdwuifsgvoezwwno","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbjuswsqwuzdmmxicffpjniszvclzlsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ooziocyxfolpwxsqqrcvtqsigkxaxrzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"olunngmxnhdztaokjgqsvztufqsweeyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"retvefwqupgfuooyngptkeglzdbjpqlx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"esdqwlhizfmzghahnlvfadgempsuubjf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lkbyykdrdtkttnhshfsewfsibiwgardf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"auhpmejagxecpeabqtzfcguuyirttqds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hzamsdcraobhklvdgxhjafzfqeerehpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jxvjkcjzuwgikbmfezyyzpmwcebejndc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fdfuakdidilgrjjvtzeazioyzxceznvv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hymhvgrvninzfablvccohcxhwntwufob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmbixipwlvogjcxvrkcpuqbmrqakpwdd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htnwhowmvrnftgfeotbljrfrtejtsqib","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jrrwrdvjvecttzloqwblepckdhaeutfj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bolskxuebuqhwjpkjpsqmmrlxjfttxep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjfyysovnapkzrnwbzpkmsydfdlzulku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aixenziwpwvdwyghlbwtpdahxluwzplp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oycwvsubzfoigduglkzzffbyvufrjiag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikwenfsbuvxgdyplnrgqxszzofkpuffz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqjlhrraxwxsvnsimzqihsvcuefcketl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"japdqanfmrgztenmbjufhhlrzaslgpmr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"onyzyrxfhwhimypwexifmveamtmnerdj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bgzisrxruaoawuzdpptqtenqfbhlzpgm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tysprxxtkpakdiaqtakhatgvowidagra","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nlxrtzjrdrtpfzryfzwgyoihsdhukhky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjhuhotzdloofjccvzedvlypvwbptnfv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzpnshmnmqfnpbingcabmzomnexfcafx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"faiqdendsnuliexaazbzjjknmudeizna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqjgfpvorbojlkxhhwdoprlvgwenqgmy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbsrkjfzmrhjcuqnkdhbjfvzxoluavqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjhinjeaknmgxtloomuwlvdvnrxshsqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzooxzeyttoxdoibmrhniormrhakvptl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgnyouhptozynahgknhycdmhiscleayu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oivmnabaznagrqysjqbifywtamgdfoda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmbpvgzvxjzmjpmctufpceyofiuswzmd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqkrayveevouevrxmgxtxchxgxamdnvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xigytzkbeducorfgqvnkoxbcmovjulmn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvkzeuntfbtetkcxrqlmeqockpyjoyuj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ysyomzwbbzstepvvhlkyrwnpcimfswqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"usvdvrwhfrhxazcxtfefpsyvdjriddcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acfgvlqalernpbdnhtslckcazvzqmdbh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qpudkfydbrwpzustjifqvizfgmakmwsv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zksmzkxaraihopappzilepxansvivirs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wkilmrjvxapadvutsvcqpjnhqszqpahr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zoahwwtxwdpfgxgilzgrychjtedrlpzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqtylovmqbfrlawrxjnljydczipczrwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"besuetnhbgvhspvhdmhoctzphcptnudt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bldchpxecokmyrmnlftfkfyoyugjcoqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clvgygeymxmownfttjegfatgcvksliyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dmnrrccawlpbbppgeainhkgkpwnohfhq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kysgnqtkfmqnhwckyvionlsrucwiyemv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"datdcwvrdlrwkjmvuwzaffzketimriqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfiqxhwmlxglpkjoznarlwhfynutxaud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugcaeqegilwevvafkhnzjabjdqjsvalu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqsanuzmvfblcpjnzrtcpkvgwcdldcsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wthoetennmfhetracjfbhajduilipoff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhnqlwotmxhcumpnzadlfcqhawlwvcuh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqmsjyujdcfqerwscwzqtatyrixcwbom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvodxhmujaakyqmckiawdtimzrbecnrq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ztazdedwrjukdubfxaxvnnmtsqpolqms","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bluzwlefintmazctwfvdpujrmxwthwlr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrcynxfxghghnqcecuucvoolkzvfjjtq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwadbuptnxrhipciukrpwnvqviqbzvqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"heakcnlcspfzorjsuqyezdrugejxqknm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zlcokmlvumnlhzkljcqrtampcardknqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"driuwatqywgzgvdbxlsnglffrpwydzxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecnuaqupwpwkaldqqbhvhugsnmkoygba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmrucqjijlfpmhgzprndylqhgxzosbdo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sytqqozsazmuqehvztqlkahsksxntmim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oywjbtveuqovxtgdeqnguvrnzyjtxzgi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekinhsgwvoayyvllbxlefbamolmjozzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fselycmqlehjcuhrucdbftzrfmkxlybt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ngigphuombfndmzsodltnqawjogpiftl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwcywsofsleoilmtqytmntzgfrieslag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhvtcoeglwvvgahvgikrxbrlfkmbxnks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lkjxtbctmraakecxnlwkjzkizszjyqxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrfbelrxisujsnechxcdvgvdgyfncihp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"asosfzkwxinustnelnykcjharwbrqvqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bisokqbyjlfhxtaanlulkfyuqxvlljav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"maxtdilrenuldkkoblpajelbxinwwlxn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qbzqorsfhsoluhxniedvlwsnqmdllxkm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghsrxgsupvlggurgtbizotqxubchejfn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmzlthrglwuwnxcgmtkpcaihocgvldjh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebonfkftqucxacssdjxyxsupqarbxgig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oanmnqjghkndhrfofcbomjqvvtkmuwpw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dnjxznlyxurnoicgkwsalwytjhmnvsld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsafxtbknvwkzmmeamnfkeexaqnaxxlj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzygdjeryjfynfumqzlmaqrvzwhvysro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"auahfcfmcxafvbpwognxzozjrkasyjyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669263,"databaseName":"models_schema","ddl":"CREATE TABLE `kwcrcymugsbqmlclxhvjycitcrcbqmuq` (\n `fpkgbhuaifrwqtizwvwxrvdmoidfwujy` int NOT NULL,\n `ieewmyqsohkpxsqbcsotetnosbhhfomf` int DEFAULT NULL,\n `asihplnpepskwstcsobwmcvjfbeuwjly` int DEFAULT NULL,\n `kbchlriuuaagptswuujftaoydatvwhrb` int DEFAULT NULL,\n `wwvipxrsegeawmsmupnfquwlohegygta` int DEFAULT NULL,\n `dlacjrrnmadytllozrqrhlhffjvomkiv` int DEFAULT NULL,\n `wxstkdpdicixqwnncsokhflposigtyjz` int DEFAULT NULL,\n `wednanqhqrvdnaoucqymifselcjngseo` int DEFAULT NULL,\n `eqqfoivodrjvmunwayzadfggmdajbwde` int DEFAULT NULL,\n `zqinmcpdjnysqcxwcownejqhqmymamzj` int DEFAULT NULL,\n `zfrwlgfcpwuxwqntcupmfummywxzciwq` int DEFAULT NULL,\n `csxhphabmqqwyowvpsgnvnnnxxyirmcf` int DEFAULT NULL,\n `azttlntlmbalunvpazfulkmtrkjtblya` int DEFAULT NULL,\n `oepzhohxrhxxqvcymnispdixhjytmnqu` int DEFAULT NULL,\n `fjllrgmqifqpdaykgimxpugotmakfxpd` int DEFAULT NULL,\n `fgcutiulagojcpjjnrgsmlnachmicqzf` int DEFAULT NULL,\n `zppsnraricnkwrkwzomulbliwcnrjono` int DEFAULT NULL,\n `jafojzptebuohsztwszkuuvomxmfvvta` int DEFAULT NULL,\n `zljljblffvttskdltfbkrffvdeuqkcsg` int DEFAULT NULL,\n `zdwnhebnufsttbeefqoxsxwsdjuepanr` int DEFAULT NULL,\n `zcvlzbirbottdajyzlocdxjhzuhwqwek` int DEFAULT NULL,\n `mbhoocpptiwsecgvqjpztfxmonqzffgp` int DEFAULT NULL,\n `mzukhalksbddmvsjdpipkhbjxlgrfivk` int DEFAULT NULL,\n `nfpiujqdcokucglqawutibjxxcwozmkp` int DEFAULT NULL,\n `utzphsnjttdblpnjnwynuhnwuzfqkrar` int DEFAULT NULL,\n `hshefqddhcwlvlnvfckanuledktfaock` int DEFAULT NULL,\n `nqhhavypbrpcudfathlveytxnpfkgulp` int DEFAULT NULL,\n `deytyjyfvcowcdqqypfmekxizawhkfkc` int DEFAULT NULL,\n `tljcvejgtgqulaiijbyspahvpgaxknvt` int DEFAULT NULL,\n `lqbrsniesjtkxysbgtoxdcdcvutiftvm` int DEFAULT NULL,\n `amkstvgebbtqxrumtrpxzsnjsqrdtmim` int DEFAULT NULL,\n `vjodsxygwpmzbwemiaeytwramqetglls` int DEFAULT NULL,\n `ulwotnfodvkmwjrnpyhwqphubjxjwzpo` int DEFAULT NULL,\n `srxavgyxploqqmxzixvrzxkslnkpiteo` int DEFAULT NULL,\n `cqormvbbukeqgydjprhaqtjmylzleehd` int DEFAULT NULL,\n `macprluugscckjcdfxnywwsrarrhuywa` int DEFAULT NULL,\n `azimmqcnejapgfmnaegacmhlcgiahnbv` int DEFAULT NULL,\n `nnmdgjbpindfsgzvfxrhxkxwyguoenyj` int DEFAULT NULL,\n `ijatqaeutlroeespreuajprirfusgoee` int DEFAULT NULL,\n `fozgqmqdtbucmmopxppkxzccgncrbqqd` int DEFAULT NULL,\n `naorrlatoscnouvzqrgxyyvxozghkdmx` int DEFAULT NULL,\n `tmuzetlunknddottleuvdzwiwbcevcbb` int DEFAULT NULL,\n `utsygkwlkhifriyqezendpidlznzmqtt` int DEFAULT NULL,\n `fpwvpwdeyxhzrkdchomsfkgfdubeazgx` int DEFAULT NULL,\n `cezfmexzwgevxnoqazueaflbdzikinsr` int DEFAULT NULL,\n `dcqylunzbtprhrchfnucyqfnxmrnkgyp` int DEFAULT NULL,\n `doobfmjmhljliectfrufytnfwdxqjpxn` int DEFAULT NULL,\n `bsztudieeulseddbhhenmaatnndrxntx` int DEFAULT NULL,\n `cvraaqpaklfgqwqktyyzdbfrlwalfmsc` int DEFAULT NULL,\n `giznildgzbgbskuwvvstrmdlhwxkxmne` int DEFAULT NULL,\n `skddusrbykugmseobhbyghsllrprxclf` int DEFAULT NULL,\n `zgqtujnsojcuikbgrncwzarkeqsraxte` int DEFAULT NULL,\n `ypscijyemubhoyhaituiployvxeukktd` int DEFAULT NULL,\n `isxzpbhtjtvqxaboflcjzfcmqktszgkh` int DEFAULT NULL,\n `uvpwoqgnlurmpayqlksqydfvmlkigxof` int DEFAULT NULL,\n `pqanoxfrwfknfiknenduyhuhfadgjefh` int DEFAULT NULL,\n `giumkrcgfsokhpnbdcckslxygrxwavvf` int DEFAULT NULL,\n `qzafvtzkvjnceplhnphyebabkgtukjig` int DEFAULT NULL,\n `oidcktpjncqigmrqpbrvsxocqxjdjjxn` int DEFAULT NULL,\n `jrqzmthkqohcgqlohypyfsreffcnzvvg` int DEFAULT NULL,\n `aprhhwtdvskvqoikwvjzohrocamuhbnz` int DEFAULT NULL,\n `fxhczudffnthgxalzckehzmhdefynski` int DEFAULT NULL,\n `ppikawkyjbhrlsdhakdlvjxtawodycze` int DEFAULT NULL,\n `dvvupewknbssyfkddixxmiuysqqhlhbq` int DEFAULT NULL,\n `vjwhqxzkiobjijctgudtjryxlonvhzzf` int DEFAULT NULL,\n `kxgrulxdtbpvdzuppedqseikskfnoand` int DEFAULT NULL,\n `cjgoehblgurtdggqvtkuqooabbwrlpjo` int DEFAULT NULL,\n `gvqqwoohhyjvzumeccwkvdpyittgdmwq` int DEFAULT NULL,\n `tblboijzqbrnqjpzljnbuameanodcpcy` int DEFAULT NULL,\n `phlwaphfxcsqtlyzgrgjwozfnsoqepmq` int DEFAULT NULL,\n `yffdzgvoykcrjzebspqlsswtxxbltdqk` int DEFAULT NULL,\n `nvaslmglntwmhgfzrnaotyqwhcmubddd` int DEFAULT NULL,\n `csmnmmoquvyfzvuhvrcwuuqxzemqlzex` int DEFAULT NULL,\n `pxjirofdwxjohaioffoljeieohvbikrz` int DEFAULT NULL,\n `vbcvfrzkflzvbmjhfdnswfzgpdntdhrg` int DEFAULT NULL,\n `smrbqklhsqpruktzdvafrebagxcpuwah` int DEFAULT NULL,\n `cpmesatvanrntcamkvpeuohqqghrlsbi` int DEFAULT NULL,\n `hjwmtnijltegvqctutknadcycwcbeloq` int DEFAULT NULL,\n `zailzqxrqsvfctltyulaifyrvrfmywun` int DEFAULT NULL,\n `iwrtlddpeiwrmafksxsqnxxzgiankjnr` int DEFAULT NULL,\n `zkbkhtihikgrmldhndtvclivlozwpsxh` int DEFAULT NULL,\n `igczluiarsaogjergolgbewzcqpmgahk` int DEFAULT NULL,\n `pybjckupciwvrmqjofmrjyuuzjxpbaqy` int DEFAULT NULL,\n `wcrawubeiobiwgszlspmsgbcjcigvoty` int DEFAULT NULL,\n `cdxiiwrlbixnswnyueuxicuwhxxpvoqr` int DEFAULT NULL,\n `yomrwdvobanzpqzbrrgqosmsuvksxxjz` int DEFAULT NULL,\n `nxatzawizislowhvihaddbwgmohurmjz` int DEFAULT NULL,\n `albdlsvtsamtixkqwflhxrgjriudthqo` int DEFAULT NULL,\n `jvnuooerllklqcuwzcihkemcwuvclldx` int DEFAULT NULL,\n `bcdzaftgsnqrbdrgmqwxjmgcyasmbygk` int DEFAULT NULL,\n `cfcustnasuowofrpyddjhziaxetgtrvp` int DEFAULT NULL,\n `pxnawhxdpmuxuczujemxhodegpdkiiif` int DEFAULT NULL,\n `yeiyxgndsuijoxgsjwzhpuivcnusczqy` int DEFAULT NULL,\n `wcpkaplameyaezbqewnjnbrkmzzcxmjj` int DEFAULT NULL,\n `oxuwhocwpmxskpzzvxghbygkutzdtsuc` int DEFAULT NULL,\n `vbsuwczuaceqzldhvtkegheturnponsd` int DEFAULT NULL,\n `yzuqfmpnpsqksrhibipgyyxvbqofkhva` int DEFAULT NULL,\n `kgqxhsqzwoxsixxyhyllzgevtbodlqyy` int DEFAULT NULL,\n `qpmhsnxnowfjnjjdlcheuydeptbkfzxt` int DEFAULT NULL,\n `ksuvqxrnbwtrqxabwdjywqsyqfwbedfu` int DEFAULT NULL,\n PRIMARY KEY (`fpkgbhuaifrwqtizwvwxrvdmoidfwujy`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"kwcrcymugsbqmlclxhvjycitcrcbqmuq\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["fpkgbhuaifrwqtizwvwxrvdmoidfwujy"],"columns":[{"name":"fpkgbhuaifrwqtizwvwxrvdmoidfwujy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ieewmyqsohkpxsqbcsotetnosbhhfomf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"asihplnpepskwstcsobwmcvjfbeuwjly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kbchlriuuaagptswuujftaoydatvwhrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwvipxrsegeawmsmupnfquwlohegygta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlacjrrnmadytllozrqrhlhffjvomkiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxstkdpdicixqwnncsokhflposigtyjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wednanqhqrvdnaoucqymifselcjngseo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqqfoivodrjvmunwayzadfggmdajbwde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqinmcpdjnysqcxwcownejqhqmymamzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zfrwlgfcpwuxwqntcupmfummywxzciwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"csxhphabmqqwyowvpsgnvnnnxxyirmcf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azttlntlmbalunvpazfulkmtrkjtblya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oepzhohxrhxxqvcymnispdixhjytmnqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjllrgmqifqpdaykgimxpugotmakfxpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fgcutiulagojcpjjnrgsmlnachmicqzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zppsnraricnkwrkwzomulbliwcnrjono","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jafojzptebuohsztwszkuuvomxmfvvta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zljljblffvttskdltfbkrffvdeuqkcsg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdwnhebnufsttbeefqoxsxwsdjuepanr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zcvlzbirbottdajyzlocdxjhzuhwqwek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbhoocpptiwsecgvqjpztfxmonqzffgp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzukhalksbddmvsjdpipkhbjxlgrfivk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfpiujqdcokucglqawutibjxxcwozmkp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utzphsnjttdblpnjnwynuhnwuzfqkrar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hshefqddhcwlvlnvfckanuledktfaock","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqhhavypbrpcudfathlveytxnpfkgulp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"deytyjyfvcowcdqqypfmekxizawhkfkc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tljcvejgtgqulaiijbyspahvpgaxknvt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqbrsniesjtkxysbgtoxdcdcvutiftvm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amkstvgebbtqxrumtrpxzsnjsqrdtmim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjodsxygwpmzbwemiaeytwramqetglls","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ulwotnfodvkmwjrnpyhwqphubjxjwzpo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"srxavgyxploqqmxzixvrzxkslnkpiteo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqormvbbukeqgydjprhaqtjmylzleehd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"macprluugscckjcdfxnywwsrarrhuywa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azimmqcnejapgfmnaegacmhlcgiahnbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nnmdgjbpindfsgzvfxrhxkxwyguoenyj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijatqaeutlroeespreuajprirfusgoee","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fozgqmqdtbucmmopxppkxzccgncrbqqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"naorrlatoscnouvzqrgxyyvxozghkdmx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmuzetlunknddottleuvdzwiwbcevcbb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utsygkwlkhifriyqezendpidlznzmqtt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpwvpwdeyxhzrkdchomsfkgfdubeazgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cezfmexzwgevxnoqazueaflbdzikinsr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcqylunzbtprhrchfnucyqfnxmrnkgyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"doobfmjmhljliectfrufytnfwdxqjpxn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bsztudieeulseddbhhenmaatnndrxntx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvraaqpaklfgqwqktyyzdbfrlwalfmsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"giznildgzbgbskuwvvstrmdlhwxkxmne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skddusrbykugmseobhbyghsllrprxclf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zgqtujnsojcuikbgrncwzarkeqsraxte","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypscijyemubhoyhaituiployvxeukktd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"isxzpbhtjtvqxaboflcjzfcmqktszgkh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvpwoqgnlurmpayqlksqydfvmlkigxof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqanoxfrwfknfiknenduyhuhfadgjefh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"giumkrcgfsokhpnbdcckslxygrxwavvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzafvtzkvjnceplhnphyebabkgtukjig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oidcktpjncqigmrqpbrvsxocqxjdjjxn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jrqzmthkqohcgqlohypyfsreffcnzvvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aprhhwtdvskvqoikwvjzohrocamuhbnz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxhczudffnthgxalzckehzmhdefynski","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ppikawkyjbhrlsdhakdlvjxtawodycze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvvupewknbssyfkddixxmiuysqqhlhbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjwhqxzkiobjijctgudtjryxlonvhzzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxgrulxdtbpvdzuppedqseikskfnoand","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjgoehblgurtdggqvtkuqooabbwrlpjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gvqqwoohhyjvzumeccwkvdpyittgdmwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tblboijzqbrnqjpzljnbuameanodcpcy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phlwaphfxcsqtlyzgrgjwozfnsoqepmq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yffdzgvoykcrjzebspqlsswtxxbltdqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvaslmglntwmhgfzrnaotyqwhcmubddd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"csmnmmoquvyfzvuhvrcwuuqxzemqlzex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxjirofdwxjohaioffoljeieohvbikrz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbcvfrzkflzvbmjhfdnswfzgpdntdhrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smrbqklhsqpruktzdvafrebagxcpuwah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cpmesatvanrntcamkvpeuohqqghrlsbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjwmtnijltegvqctutknadcycwcbeloq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zailzqxrqsvfctltyulaifyrvrfmywun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwrtlddpeiwrmafksxsqnxxzgiankjnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zkbkhtihikgrmldhndtvclivlozwpsxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igczluiarsaogjergolgbewzcqpmgahk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pybjckupciwvrmqjofmrjyuuzjxpbaqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcrawubeiobiwgszlspmsgbcjcigvoty","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cdxiiwrlbixnswnyueuxicuwhxxpvoqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yomrwdvobanzpqzbrrgqosmsuvksxxjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxatzawizislowhvihaddbwgmohurmjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"albdlsvtsamtixkqwflhxrgjriudthqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvnuooerllklqcuwzcihkemcwuvclldx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bcdzaftgsnqrbdrgmqwxjmgcyasmbygk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfcustnasuowofrpyddjhziaxetgtrvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxnawhxdpmuxuczujemxhodegpdkiiif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yeiyxgndsuijoxgsjwzhpuivcnusczqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcpkaplameyaezbqewnjnbrkmzzcxmjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oxuwhocwpmxskpzzvxghbygkutzdtsuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbsuwczuaceqzldhvtkegheturnponsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzuqfmpnpsqksrhibipgyyxvbqofkhva","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kgqxhsqzwoxsixxyhyllzgevtbodlqyy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qpmhsnxnowfjnjjdlcheuydeptbkfzxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ksuvqxrnbwtrqxabwdjywqsyqfwbedfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669293,"databaseName":"models_schema","ddl":"CREATE TABLE `kxmwllnisximtjilqoifyciirajdkdqe` (\n `ameelqvcniuvwnvoaicfoifazyoqyqdj` int NOT NULL,\n `sgvhytprvsnywerhedktuivjybngrdjx` int DEFAULT NULL,\n `xyqwmelzbxwbbkggxtmxkkrlysdlzfox` int DEFAULT NULL,\n `ijssffvpztkibtgqjczpvzzzedaaedms` int DEFAULT NULL,\n `zrsauytaijrpkchwtkdexzmxpqcflfey` int DEFAULT NULL,\n `panoyfndwqhnhbwjygdfydlwzmpgnomo` int DEFAULT NULL,\n `ulppwigmcevycntsiujijxmsqjshiemq` int DEFAULT NULL,\n `jiihivlsgaklojftnfbyrvfhqsczsdav` int DEFAULT NULL,\n `vsektuzomntxttpbfgodepyjemkwmrnm` int DEFAULT NULL,\n `sybwoxlmmpqqropibzfasbxdrvdxabqq` int DEFAULT NULL,\n `rqasytnuupyclhdomhjoxkmnvmovfrsr` int DEFAULT NULL,\n `crflryfjxxsxwvkkpqhidxvfgowhujlm` int DEFAULT NULL,\n `hyvzviprtyhzvrgagfscmtojiaaagpeu` int DEFAULT NULL,\n `qystjgdvambgwlyzjmyygkwlrwvjwoym` int DEFAULT NULL,\n `cwgjepybmnxhdmnshipqamihehjdlcga` int DEFAULT NULL,\n `aayhzyyjfvyojvwmenckoqxemnnhpuve` int DEFAULT NULL,\n `rlvnppbxkufzefnmzdjcapkwcylrbyur` int DEFAULT NULL,\n `pdtyuxgomggwntqdvaiinxfmskenxbbk` int DEFAULT NULL,\n `pgrdrudfrrynyqfgyylqriqqhvlpszny` int DEFAULT NULL,\n `eklrcnbfafulbbqcidbjxztflovxerae` int DEFAULT NULL,\n `hdwllitydxrxxafxpmnpdixgwtaanejj` int DEFAULT NULL,\n `vfadmgsmnjkvzkhrjcjotohwplkztyil` int DEFAULT NULL,\n `hkuayvpqhdjjugfsqqrplzsuprayyjwr` int DEFAULT NULL,\n `ftudsbmlxmelwsgwjkpudfqxsypfvpbu` int DEFAULT NULL,\n `rhtuqijdulxclntezatwnuxyhloonrcc` int DEFAULT NULL,\n `xccmiwneldgulfqurywdncanftnwkukm` int DEFAULT NULL,\n `gqyjdcmsztflpyrjleyrcnjqlwvsiujr` int DEFAULT NULL,\n `ndqporktkxkrdxdickmuzsbdsmvhdcwq` int DEFAULT NULL,\n `dbkvjayqqrvrxoqweteccxtheicomtxp` int DEFAULT NULL,\n `sdhlfqinykhbuudfxtcjltaswfudkzpq` int DEFAULT NULL,\n `acpussvueblyikzbrxxzljrmrdmtugne` int DEFAULT NULL,\n `vaunagwjmswaidxwobmeaomuhxezbkej` int DEFAULT NULL,\n `baaupbhljngsnvvkgadrxpourwpmuskw` int DEFAULT NULL,\n `jrfuqwscitnqmxiluhlihyyshieutemg` int DEFAULT NULL,\n `rzmhrsiydgsgfquilljzjnwxumhuvqyn` int DEFAULT NULL,\n `eeqccojbxznjxlxsxlphuwolwvmkglms` int DEFAULT NULL,\n `tmoratqvtoremqudjtnpoispmepwyzeg` int DEFAULT NULL,\n `tyvmqbjemwcrhsbcsgnoujyaiuyouupx` int DEFAULT NULL,\n `hekacutyzahwoswpusorycgqceouvebh` int DEFAULT NULL,\n `uasmepxqeqmqcpscgkkiajzldvfkzzsf` int DEFAULT NULL,\n `txylsflrwdbhwqaqoeyunazojddbrvan` int DEFAULT NULL,\n `mruxbujmoraufmjzhfxgfrpownzfwujl` int DEFAULT NULL,\n `hwuaapzoufgbjvjjdsdmddcmxhmcrgqz` int DEFAULT NULL,\n `acxivmoszamiwgedtaexxyagvrieftxe` int DEFAULT NULL,\n `xvvomrorgjdxnfgnnyvkvuhkcwatltgm` int DEFAULT NULL,\n `tammnlplvqdtmwnakikgbszkjqebfqoq` int DEFAULT NULL,\n `yxoyntwbngsflzhmmcblyexdrbovcaiv` int DEFAULT NULL,\n `btzcfeydlgmdmaofjevkxgljpjxbmohl` int DEFAULT NULL,\n `dxecwprfsfzqfjbggfhonvmhypazzzys` int DEFAULT NULL,\n `pjjwhyhcqfyxvaxubdrfcuydlkecramv` int DEFAULT NULL,\n `cchlakycfnlzxevqermjsnqacrkalrad` int DEFAULT NULL,\n `lpktegkaflkyxoztzofsofyfcublbsfs` int DEFAULT NULL,\n `opcpzwxyytucdeqcloxkbfhuolgqaarp` int DEFAULT NULL,\n `mzduaqpymuqdnqmcdrshapwecrocpugp` int DEFAULT NULL,\n `pmnkrwvcyjtpzlawlaejgmbbskybuwpl` int DEFAULT NULL,\n `ygnzjcwosvmyfwywmjynnzkwnugeyrzy` int DEFAULT NULL,\n `tcpqnkvuujimskkyjhtfraunliwnjhrt` int DEFAULT NULL,\n `cvoojqhnwmywxciuqwudivwyyrbnkxbe` int DEFAULT NULL,\n `itpmxdvzqttcgvkouqwvfxotmgcfzrtn` int DEFAULT NULL,\n `taumlqhosqlxxaeapojzjtdqbgscckgc` int DEFAULT NULL,\n `iaatigwtqefisdfwvcsosxilfqudwncm` int DEFAULT NULL,\n `msaaqvasfsrxaircsolfhirnkailmhmq` int DEFAULT NULL,\n `pgyrrkflbxsnmkxnhqezrayjilkioadr` int DEFAULT NULL,\n `xignvwphcbnysboasiojaplpfpmwzdlp` int DEFAULT NULL,\n `hkwdfhxvyyikqclgrktmzdxluhmgpbqv` int DEFAULT NULL,\n `gkybkjhyxdfdzaxdflhnbkekyuutagca` int DEFAULT NULL,\n `ecihkvqkqxsdgpcacjdkwlowfknzdrze` int DEFAULT NULL,\n `vkxztlvfbzgcldvpqdbmictzyoupasxy` int DEFAULT NULL,\n `pzimjgdfhhutjzzubyyhvpovwtdlhlfl` int DEFAULT NULL,\n `lmwaftjkravrqxnmqrmerojednjxhixb` int DEFAULT NULL,\n `xhspfdsltwsqbrlntoqakqjffmyslblj` int DEFAULT NULL,\n `dialhtxjfputtpqaykdwpcomknyyflmj` int DEFAULT NULL,\n `ezvslrkjxxhxovbomipvacgxwygbxoje` int DEFAULT NULL,\n `fxvkwcokkhgzvujlqmohsbkyatezksdq` int DEFAULT NULL,\n `rhsqzvgcvlfkovchzosbnbgxbuhviqpd` int DEFAULT NULL,\n `rknpzzfwtnurykllcrwexsbxcnivjigg` int DEFAULT NULL,\n `rgaekhethuyiviwmfbpgskmkcezhvrjn` int DEFAULT NULL,\n `ffgvifvwauxxzvagjogntqhfshsslemy` int DEFAULT NULL,\n `eefgjxlwhesixgleaysxkajzftltquvn` int DEFAULT NULL,\n `gasmmczgizzupwxpymgevmedkyceckfv` int DEFAULT NULL,\n `wucfprszsgogqholzhvndpmibzwrbkem` int DEFAULT NULL,\n `duuysswtxazalcmipvxvscvctxbaapsp` int DEFAULT NULL,\n `gykplnehekfozmxsxchofesgkntcvwij` int DEFAULT NULL,\n `tuwzqkcovzommeganbmlvxaloadmaiqi` int DEFAULT NULL,\n `ectygjfueatkayapaxduaelmdddgoaek` int DEFAULT NULL,\n `abentfrjgwwuyxrlrdkyhswlkaewgybe` int DEFAULT NULL,\n `heglcrzqndmtlsvuldmdemgsrjcptdpu` int DEFAULT NULL,\n `cqhuyinqqhourauldrgatsvofitkjbbi` int DEFAULT NULL,\n `vnvvinmuobljddfafuowdgneloiksfuk` int DEFAULT NULL,\n `zxlcnrxyndoadkthbepweaylunmikxjp` int DEFAULT NULL,\n `pryipistcyumtcjuvsynwvjcmbgdfuiy` int DEFAULT NULL,\n `ivvdlhcsapfpugalexcqoigchlctbvwh` int DEFAULT NULL,\n `royqnqcwsppnhprpdcqpbzwuzqcftnsd` int DEFAULT NULL,\n `hyvjdwthlivjuwkoftvlecwhqgcmnqmc` int DEFAULT NULL,\n `aafyacikdhomvnlxknrpecgfvzhxklwp` int DEFAULT NULL,\n `ggfjnqujcqawtdxythhtlslpvpomuzrd` int DEFAULT NULL,\n `dllbsxvxgjdrrocidomyrgwnhuswaate` int DEFAULT NULL,\n `bqyxxlfjijpkcfnwqmwtxfjvgtzbmast` int DEFAULT NULL,\n `ygpuxyggttibntpbcjbzjfcldmqqiivj` int DEFAULT NULL,\n `xjjofdnoxebbuneqrhqkwgfqkntwqksr` int DEFAULT NULL,\n PRIMARY KEY (`ameelqvcniuvwnvoaicfoifazyoqyqdj`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"kxmwllnisximtjilqoifyciirajdkdqe\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["ameelqvcniuvwnvoaicfoifazyoqyqdj"],"columns":[{"name":"ameelqvcniuvwnvoaicfoifazyoqyqdj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"sgvhytprvsnywerhedktuivjybngrdjx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyqwmelzbxwbbkggxtmxkkrlysdlzfox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijssffvpztkibtgqjczpvzzzedaaedms","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zrsauytaijrpkchwtkdexzmxpqcflfey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"panoyfndwqhnhbwjygdfydlwzmpgnomo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ulppwigmcevycntsiujijxmsqjshiemq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jiihivlsgaklojftnfbyrvfhqsczsdav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsektuzomntxttpbfgodepyjemkwmrnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sybwoxlmmpqqropibzfasbxdrvdxabqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqasytnuupyclhdomhjoxkmnvmovfrsr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crflryfjxxsxwvkkpqhidxvfgowhujlm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hyvzviprtyhzvrgagfscmtojiaaagpeu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qystjgdvambgwlyzjmyygkwlrwvjwoym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwgjepybmnxhdmnshipqamihehjdlcga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aayhzyyjfvyojvwmenckoqxemnnhpuve","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlvnppbxkufzefnmzdjcapkwcylrbyur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdtyuxgomggwntqdvaiinxfmskenxbbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgrdrudfrrynyqfgyylqriqqhvlpszny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eklrcnbfafulbbqcidbjxztflovxerae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdwllitydxrxxafxpmnpdixgwtaanejj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vfadmgsmnjkvzkhrjcjotohwplkztyil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkuayvpqhdjjugfsqqrplzsuprayyjwr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftudsbmlxmelwsgwjkpudfqxsypfvpbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhtuqijdulxclntezatwnuxyhloonrcc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xccmiwneldgulfqurywdncanftnwkukm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqyjdcmsztflpyrjleyrcnjqlwvsiujr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ndqporktkxkrdxdickmuzsbdsmvhdcwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbkvjayqqrvrxoqweteccxtheicomtxp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdhlfqinykhbuudfxtcjltaswfudkzpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acpussvueblyikzbrxxzljrmrdmtugne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vaunagwjmswaidxwobmeaomuhxezbkej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"baaupbhljngsnvvkgadrxpourwpmuskw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jrfuqwscitnqmxiluhlihyyshieutemg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzmhrsiydgsgfquilljzjnwxumhuvqyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eeqccojbxznjxlxsxlphuwolwvmkglms","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmoratqvtoremqudjtnpoispmepwyzeg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tyvmqbjemwcrhsbcsgnoujyaiuyouupx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hekacutyzahwoswpusorycgqceouvebh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uasmepxqeqmqcpscgkkiajzldvfkzzsf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txylsflrwdbhwqaqoeyunazojddbrvan","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mruxbujmoraufmjzhfxgfrpownzfwujl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwuaapzoufgbjvjjdsdmddcmxhmcrgqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acxivmoszamiwgedtaexxyagvrieftxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvvomrorgjdxnfgnnyvkvuhkcwatltgm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tammnlplvqdtmwnakikgbszkjqebfqoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxoyntwbngsflzhmmcblyexdrbovcaiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"btzcfeydlgmdmaofjevkxgljpjxbmohl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxecwprfsfzqfjbggfhonvmhypazzzys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjjwhyhcqfyxvaxubdrfcuydlkecramv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cchlakycfnlzxevqermjsnqacrkalrad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lpktegkaflkyxoztzofsofyfcublbsfs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opcpzwxyytucdeqcloxkbfhuolgqaarp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzduaqpymuqdnqmcdrshapwecrocpugp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmnkrwvcyjtpzlawlaejgmbbskybuwpl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ygnzjcwosvmyfwywmjynnzkwnugeyrzy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tcpqnkvuujimskkyjhtfraunliwnjhrt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvoojqhnwmywxciuqwudivwyyrbnkxbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itpmxdvzqttcgvkouqwvfxotmgcfzrtn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"taumlqhosqlxxaeapojzjtdqbgscckgc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iaatigwtqefisdfwvcsosxilfqudwncm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"msaaqvasfsrxaircsolfhirnkailmhmq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgyrrkflbxsnmkxnhqezrayjilkioadr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xignvwphcbnysboasiojaplpfpmwzdlp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkwdfhxvyyikqclgrktmzdxluhmgpbqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkybkjhyxdfdzaxdflhnbkekyuutagca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecihkvqkqxsdgpcacjdkwlowfknzdrze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkxztlvfbzgcldvpqdbmictzyoupasxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pzimjgdfhhutjzzubyyhvpovwtdlhlfl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmwaftjkravrqxnmqrmerojednjxhixb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xhspfdsltwsqbrlntoqakqjffmyslblj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dialhtxjfputtpqaykdwpcomknyyflmj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezvslrkjxxhxovbomipvacgxwygbxoje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxvkwcokkhgzvujlqmohsbkyatezksdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhsqzvgcvlfkovchzosbnbgxbuhviqpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rknpzzfwtnurykllcrwexsbxcnivjigg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgaekhethuyiviwmfbpgskmkcezhvrjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffgvifvwauxxzvagjogntqhfshsslemy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eefgjxlwhesixgleaysxkajzftltquvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gasmmczgizzupwxpymgevmedkyceckfv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wucfprszsgogqholzhvndpmibzwrbkem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"duuysswtxazalcmipvxvscvctxbaapsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gykplnehekfozmxsxchofesgkntcvwij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tuwzqkcovzommeganbmlvxaloadmaiqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ectygjfueatkayapaxduaelmdddgoaek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abentfrjgwwuyxrlrdkyhswlkaewgybe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"heglcrzqndmtlsvuldmdemgsrjcptdpu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqhuyinqqhourauldrgatsvofitkjbbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnvvinmuobljddfafuowdgneloiksfuk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxlcnrxyndoadkthbepweaylunmikxjp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pryipistcyumtcjuvsynwvjcmbgdfuiy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivvdlhcsapfpugalexcqoigchlctbvwh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"royqnqcwsppnhprpdcqpbzwuzqcftnsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hyvjdwthlivjuwkoftvlecwhqgcmnqmc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aafyacikdhomvnlxknrpecgfvzhxklwp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggfjnqujcqawtdxythhtlslpvpomuzrd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dllbsxvxgjdrrocidomyrgwnhuswaate","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqyxxlfjijpkcfnwqmwtxfjvgtzbmast","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ygpuxyggttibntpbcjbzjfcldmqqiivj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjjofdnoxebbuneqrhqkwgfqkntwqksr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669324,"databaseName":"models_schema","ddl":"CREATE TABLE `kybtprhlbzqgbnsogitpfetdycsqqqmu` (\n `vlxgmkqctxskrwgganlvcaoutiwdonnc` int NOT NULL,\n `iknahmawvpowelgsbxvruuxlkpzilnyu` int DEFAULT NULL,\n `aefhbohfkqcqbwbzjoekgsdyyugoarmj` int DEFAULT NULL,\n `mnzblnkkxanzwzbwszstencyhnghodlt` int DEFAULT NULL,\n `nqlmpyvdosoypvgyflftudpyhpjpkyvz` int DEFAULT NULL,\n `ntajulvhaqafxjenowyscnaxrjxldfci` int DEFAULT NULL,\n `lgwjsyxdtcjoctyqbvowqyraklhwksaf` int DEFAULT NULL,\n `mimyoqdzycvzhpdiqtfmhlgpvgwxqrnt` int DEFAULT NULL,\n `qqqirfdudeferlezqslxcdivdaebrnrz` int DEFAULT NULL,\n `kquloxgbewmajoefhplsxociorgmilek` int DEFAULT NULL,\n `hpbxxggczgnygiuryrvnurtcqhyrgfiv` int DEFAULT NULL,\n `zoelbwaavtqovxaghlxtwneyhyshemcq` int DEFAULT NULL,\n `wgtzyfpfnuoepiceqitcgyajxpswmupd` int DEFAULT NULL,\n `bxytvmknziozalbykogzygcprxlvhsyj` int DEFAULT NULL,\n `itojohsdjoovnvrzwkhtcmekppkjlbnv` int DEFAULT NULL,\n `bmsepyzijtnrpfqcllwgpnjttigysqvr` int DEFAULT NULL,\n `qdqxvwdunaldoldfsxwzrpyisjsmgbdv` int DEFAULT NULL,\n `joqkuedjroyqbmgihqcvnoimjcnsrrim` int DEFAULT NULL,\n `nsccgyfnwovexioyptqllinfurdrquey` int DEFAULT NULL,\n `ggwbtokldwrgfeszrzyujticakurccep` int DEFAULT NULL,\n `redvyddncpotumhepoacamozxityfunk` int DEFAULT NULL,\n `psqpwnrnnevpxvbtfxtsmvkwoznqhrgk` int DEFAULT NULL,\n `qtnlrqnbnmgfeeqmzrecueuzmectmgwq` int DEFAULT NULL,\n `fpfnijcnszlinfqvlokezlkrawecuaju` int DEFAULT NULL,\n `ywdfoagntxuhjahibqqjrxmlqflcgjed` int DEFAULT NULL,\n `unxfgpdprvcqmlsreqrrxoibplyndiwn` int DEFAULT NULL,\n `jlxjovqrpdarldyjmyyasinbalblsbbq` int DEFAULT NULL,\n `mhgglmzkukxcthqlxfnizkcmeyfjmsqn` int DEFAULT NULL,\n `kdviigaftxllivpwqqxzdrwenexdnlea` int DEFAULT NULL,\n `udprzndopcjrcjjudbklzrsxgpntjnfr` int DEFAULT NULL,\n `gflnuwrgqafcuxjxjlwkkeualhpscqye` int DEFAULT NULL,\n `msgmxzukslmqgwanimrqoytjdieefvxh` int DEFAULT NULL,\n `yrtmufnwvnfpjyhezxurhnhsemojvccs` int DEFAULT NULL,\n `wawevhhhdcjzcoxokfwowhindiqkkqeu` int DEFAULT NULL,\n `khozjvcmbzrbkscgbrosksecpihsmfbq` int DEFAULT NULL,\n `qfqsmxflbtnfmpsmbbedqtqphypdqcrx` int DEFAULT NULL,\n `wcxxbzdsyqxdjsihpevphzqreondpiyq` int DEFAULT NULL,\n `wxfjcivoccoskbhxwtysddpgmetflfaf` int DEFAULT NULL,\n `iqbirpyimaxghnpjnqnomkzgwoabxnkx` int DEFAULT NULL,\n `hwmstwxdouueuzdmqloehuarhdmzcohk` int DEFAULT NULL,\n `sanuxhjgkrxrkktrziaxjaczqzvccjeh` int DEFAULT NULL,\n `gkrrabxvjvqxuxkogvspqagucdxzbood` int DEFAULT NULL,\n `kmsfmbhalqrjhnihathetrltkeolyidp` int DEFAULT NULL,\n `eygegfwqrennsirvknhvodnuumkrxvpc` int DEFAULT NULL,\n `jvitpuhsdsmgzdrkdxvdxsaqecusrgrd` int DEFAULT NULL,\n `adhcexggddgcygiqjqdblizitdpdpvtg` int DEFAULT NULL,\n `umxevnouvjqgtyawnhbwvvbtdreguzcs` int DEFAULT NULL,\n `mpvcnageoddxcetymphksxdcentcihag` int DEFAULT NULL,\n `guwmfgdpusaurnqvmcfyefmyyiyymlyc` int DEFAULT NULL,\n `bkbfqcnnldfuwceirfypgondpaiawjxh` int DEFAULT NULL,\n `rjmwcurtnxqgjbkybegjwkbyyaadwznk` int DEFAULT NULL,\n `lohpmgzlngomreiirkkkkytlejozbwjb` int DEFAULT NULL,\n `zutllzwjplzprmfblqtrdczgtuzyjyge` int DEFAULT NULL,\n `zwxqsewcnkwseifkttiieuoovdkecvxr` int DEFAULT NULL,\n `yfwytaeqxweueblslaalpcwaymenljqf` int DEFAULT NULL,\n `awhxxxzrlzsjsokyovlnkjwgrhqifxne` int DEFAULT NULL,\n `qumbcvxucgyorfzbybvnhsbxezxcbvia` int DEFAULT NULL,\n `pnprfnklnzeubikmzqrayysewsznewey` int DEFAULT NULL,\n `gqpeqdpmizcuoterfxwwcbifxzggiexm` int DEFAULT NULL,\n `menyilyvcnxvbnjymcqurkmhzhcniywt` int DEFAULT NULL,\n `dsagufekcxcywaeuehbgqddjnhqkogez` int DEFAULT NULL,\n `ojtiymdkbaxlijimojtgruhzlyplflfk` int DEFAULT NULL,\n `ptephhuiojpcwdnsqkhqdgsfdmwcwikj` int DEFAULT NULL,\n `ztczesdtgfffrbarwcpwuuszgsdaihar` int DEFAULT NULL,\n `ygiimfcrwpsqfalhubvwncqsgoryesdj` int DEFAULT NULL,\n `uhabpvlxnnpyfagjpnlrophgpymqyzcs` int DEFAULT NULL,\n `ygwfdnqnkvdzbnerlcdhhuvvthrmekmv` int DEFAULT NULL,\n `temfqrdlchlegfztdiqfexflejxgzcmp` int DEFAULT NULL,\n `nncpjdqflcfpoxcfgrtxslcjtgqpalfq` int DEFAULT NULL,\n `qstlmjaykmdlmuvrjyjsfvmglzzcydwa` int DEFAULT NULL,\n `xjenprcbootbcpsrdbukvhtgrutunjks` int DEFAULT NULL,\n `oacshsugpxjfgxdvibsqcmhzlpwexzzq` int DEFAULT NULL,\n `wnnwvcephmoiouojdvtlgyqrmhrfbsbe` int DEFAULT NULL,\n `nsauruiadfuqajgrsjuyyhhwhemctvya` int DEFAULT NULL,\n `qcxvupcifpcovyrywtjwgxqaontydyhc` int DEFAULT NULL,\n `hnzncaotoykowydrkakfaatauxxxvvvg` int DEFAULT NULL,\n `tvafqmtmmquuzvxwdtguslveynvzuuop` int DEFAULT NULL,\n `autwewxqptvmfgaslrhbqskpgigfkewi` int DEFAULT NULL,\n `lplmrxbmrihlwrhcsblwyazphocrhakf` int DEFAULT NULL,\n `lkxucpyeehffnveumvxwfdkuiudqjfci` int DEFAULT NULL,\n `bhjwbzfgzemscmssfuvljhhtuseighjx` int DEFAULT NULL,\n `othnnsokazaqmtfmcptxmlvucmzwzihd` int DEFAULT NULL,\n `qagynnohqlifomkarrqtznejrkqmmczx` int DEFAULT NULL,\n `rusrbbsjtxiackefpagrktrlgabsvnep` int DEFAULT NULL,\n `wrkfluffnqjdnkwuxmxgtiyoyypaxued` int DEFAULT NULL,\n `swcbliulimmayggnqnboqaxcpasthjka` int DEFAULT NULL,\n `lvloxsvdvajncrugludywnkfcubtobua` int DEFAULT NULL,\n `nguzqwflzsfuvkkmasvzakmrirgnxdos` int DEFAULT NULL,\n `wkqdqxmqndkasloxyuluumwrtrnqjsar` int DEFAULT NULL,\n `vbqboeukbyoobltwgcezfizqlgpjpzua` int DEFAULT NULL,\n `lorfxwpztwcvdytczadzsgeoqrrpttjy` int DEFAULT NULL,\n `emvhzshgmopnfymwboccxhxkzapyjdxr` int DEFAULT NULL,\n `gavkogyzdbvkqoaqihohzbhelcfdulkd` int DEFAULT NULL,\n `mqfzewmnfcvovvdvcsybnvpyrrnrmkxr` int DEFAULT NULL,\n `hyuetroodpufpauvbrfvfhpjtmxnvlhq` int DEFAULT NULL,\n `nnvbshesmodsvkkloejpqxsrgokpbfzb` int DEFAULT NULL,\n `qqtwxvgrlnshwpitzadmfsdaflnmhown` int DEFAULT NULL,\n `yvfitoshqhurxzwgwgyyqgjbvchzyrby` int DEFAULT NULL,\n `watnbvumkaoxhwguwtbhdiciukekrsgg` int DEFAULT NULL,\n `dewnwwucwzkeofugfhxnbryhlvzzcvcd` int DEFAULT NULL,\n PRIMARY KEY (`vlxgmkqctxskrwgganlvcaoutiwdonnc`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"kybtprhlbzqgbnsogitpfetdycsqqqmu\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["vlxgmkqctxskrwgganlvcaoutiwdonnc"],"columns":[{"name":"vlxgmkqctxskrwgganlvcaoutiwdonnc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"iknahmawvpowelgsbxvruuxlkpzilnyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aefhbohfkqcqbwbzjoekgsdyyugoarmj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnzblnkkxanzwzbwszstencyhnghodlt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqlmpyvdosoypvgyflftudpyhpjpkyvz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntajulvhaqafxjenowyscnaxrjxldfci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgwjsyxdtcjoctyqbvowqyraklhwksaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mimyoqdzycvzhpdiqtfmhlgpvgwxqrnt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqqirfdudeferlezqslxcdivdaebrnrz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kquloxgbewmajoefhplsxociorgmilek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hpbxxggczgnygiuryrvnurtcqhyrgfiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zoelbwaavtqovxaghlxtwneyhyshemcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgtzyfpfnuoepiceqitcgyajxpswmupd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bxytvmknziozalbykogzygcprxlvhsyj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itojohsdjoovnvrzwkhtcmekppkjlbnv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bmsepyzijtnrpfqcllwgpnjttigysqvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdqxvwdunaldoldfsxwzrpyisjsmgbdv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"joqkuedjroyqbmgihqcvnoimjcnsrrim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nsccgyfnwovexioyptqllinfurdrquey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggwbtokldwrgfeszrzyujticakurccep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"redvyddncpotumhepoacamozxityfunk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psqpwnrnnevpxvbtfxtsmvkwoznqhrgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtnlrqnbnmgfeeqmzrecueuzmectmgwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpfnijcnszlinfqvlokezlkrawecuaju","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywdfoagntxuhjahibqqjrxmlqflcgjed","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unxfgpdprvcqmlsreqrrxoibplyndiwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlxjovqrpdarldyjmyyasinbalblsbbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhgglmzkukxcthqlxfnizkcmeyfjmsqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kdviigaftxllivpwqqxzdrwenexdnlea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udprzndopcjrcjjudbklzrsxgpntjnfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gflnuwrgqafcuxjxjlwkkeualhpscqye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"msgmxzukslmqgwanimrqoytjdieefvxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yrtmufnwvnfpjyhezxurhnhsemojvccs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wawevhhhdcjzcoxokfwowhindiqkkqeu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khozjvcmbzrbkscgbrosksecpihsmfbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfqsmxflbtnfmpsmbbedqtqphypdqcrx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcxxbzdsyqxdjsihpevphzqreondpiyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxfjcivoccoskbhxwtysddpgmetflfaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqbirpyimaxghnpjnqnomkzgwoabxnkx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwmstwxdouueuzdmqloehuarhdmzcohk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sanuxhjgkrxrkktrziaxjaczqzvccjeh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkrrabxvjvqxuxkogvspqagucdxzbood","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmsfmbhalqrjhnihathetrltkeolyidp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eygegfwqrennsirvknhvodnuumkrxvpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvitpuhsdsmgzdrkdxvdxsaqecusrgrd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"adhcexggddgcygiqjqdblizitdpdpvtg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umxevnouvjqgtyawnhbwvvbtdreguzcs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpvcnageoddxcetymphksxdcentcihag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"guwmfgdpusaurnqvmcfyefmyyiyymlyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkbfqcnnldfuwceirfypgondpaiawjxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjmwcurtnxqgjbkybegjwkbyyaadwznk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lohpmgzlngomreiirkkkkytlejozbwjb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zutllzwjplzprmfblqtrdczgtuzyjyge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zwxqsewcnkwseifkttiieuoovdkecvxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfwytaeqxweueblslaalpcwaymenljqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awhxxxzrlzsjsokyovlnkjwgrhqifxne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qumbcvxucgyorfzbybvnhsbxezxcbvia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnprfnklnzeubikmzqrayysewsznewey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqpeqdpmizcuoterfxwwcbifxzggiexm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"menyilyvcnxvbnjymcqurkmhzhcniywt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsagufekcxcywaeuehbgqddjnhqkogez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojtiymdkbaxlijimojtgruhzlyplflfk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ptephhuiojpcwdnsqkhqdgsfdmwcwikj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ztczesdtgfffrbarwcpwuuszgsdaihar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ygiimfcrwpsqfalhubvwncqsgoryesdj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhabpvlxnnpyfagjpnlrophgpymqyzcs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ygwfdnqnkvdzbnerlcdhhuvvthrmekmv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"temfqrdlchlegfztdiqfexflejxgzcmp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nncpjdqflcfpoxcfgrtxslcjtgqpalfq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qstlmjaykmdlmuvrjyjsfvmglzzcydwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjenprcbootbcpsrdbukvhtgrutunjks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oacshsugpxjfgxdvibsqcmhzlpwexzzq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnnwvcephmoiouojdvtlgyqrmhrfbsbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nsauruiadfuqajgrsjuyyhhwhemctvya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qcxvupcifpcovyrywtjwgxqaontydyhc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hnzncaotoykowydrkakfaatauxxxvvvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvafqmtmmquuzvxwdtguslveynvzuuop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"autwewxqptvmfgaslrhbqskpgigfkewi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lplmrxbmrihlwrhcsblwyazphocrhakf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lkxucpyeehffnveumvxwfdkuiudqjfci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bhjwbzfgzemscmssfuvljhhtuseighjx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"othnnsokazaqmtfmcptxmlvucmzwzihd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qagynnohqlifomkarrqtznejrkqmmczx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rusrbbsjtxiackefpagrktrlgabsvnep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wrkfluffnqjdnkwuxmxgtiyoyypaxued","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swcbliulimmayggnqnboqaxcpasthjka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvloxsvdvajncrugludywnkfcubtobua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nguzqwflzsfuvkkmasvzakmrirgnxdos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wkqdqxmqndkasloxyuluumwrtrnqjsar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbqboeukbyoobltwgcezfizqlgpjpzua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lorfxwpztwcvdytczadzsgeoqrrpttjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"emvhzshgmopnfymwboccxhxkzapyjdxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gavkogyzdbvkqoaqihohzbhelcfdulkd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqfzewmnfcvovvdvcsybnvpyrrnrmkxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hyuetroodpufpauvbrfvfhpjtmxnvlhq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nnvbshesmodsvkkloejpqxsrgokpbfzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqtwxvgrlnshwpitzadmfsdaflnmhown","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yvfitoshqhurxzwgwgyyqgjbvchzyrby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"watnbvumkaoxhwguwtbhdiciukekrsgg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dewnwwucwzkeofugfhxnbryhlvzzcvcd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669357,"databaseName":"models_schema","ddl":"CREATE TABLE `lgzqspwbxcvbxbxdceywhoesgrnsmpga` (\n `wjrvzrothgodipzyxfsguyamuguaodbs` int NOT NULL,\n `rnlnvaumlgqxsyqlbdmhsdxztisagtnt` int DEFAULT NULL,\n `htkofgwknjmlufuremgwuxdsmzfqnzjz` int DEFAULT NULL,\n `prngjnahcrfaqlfrxuehefbhexrsflfb` int DEFAULT NULL,\n `zvifvmvdendnbnpzvjgecyvxuqkyiwfk` int DEFAULT NULL,\n `qnppuhwiynnttmmwzlrprsdhgvkbkprc` int DEFAULT NULL,\n `mmoxvwbyxznpaeaewyypvpzswefzqxdz` int DEFAULT NULL,\n `kwjmsrykavzckmjhorpswowpfjbmogxq` int DEFAULT NULL,\n `cexaeiuutcmunheuwiyzruejejonhean` int DEFAULT NULL,\n `mkjbnpfiknnkhdwdvouqrqdxipphvoro` int DEFAULT NULL,\n `mepzgwrojyopvxbjkvxmgihjpfvtwena` int DEFAULT NULL,\n `tcjxgikxbesclrrmsdaeitnvzhbcdgoq` int DEFAULT NULL,\n `yxehsfmusnawsgaxoczrbouqzzswxvyf` int DEFAULT NULL,\n `eogoqrwownibmgsqemqtfnfusxfphcoe` int DEFAULT NULL,\n `rhmhiesufbehgecstjiuamkzocthphnr` int DEFAULT NULL,\n `adkaxdofainwufhidlrtktmjlmjelaio` int DEFAULT NULL,\n `sydglfhedthbtvjaakuodhzpehudiome` int DEFAULT NULL,\n `kzijansnkrvmqbqbkryylduzhawpcmpk` int DEFAULT NULL,\n `uxjbmnvkaaoramgrtptyzunzpxalplwk` int DEFAULT NULL,\n `xnxtdmxeiwpohajrahteztsxzihhsgjg` int DEFAULT NULL,\n `oqpfbzpppchcsktxetgupfyblmvxdsfv` int DEFAULT NULL,\n `ofbvtcbmpamrsobtddfmrocbkyhuthgt` int DEFAULT NULL,\n `wxhjyznhhvvafdsvtmomcpgcpdojtbao` int DEFAULT NULL,\n `vyfuguxmviovqeewekbjlxrbtqtritjy` int DEFAULT NULL,\n `krocnjlcnsyfninfnyaajinxphcpitcu` int DEFAULT NULL,\n `uzktdtdsmuuronwsbdtmycsazohgenhg` int DEFAULT NULL,\n `nuuagqvxlpdvxcjydzrjojcaupwiwnqo` int DEFAULT NULL,\n `mrtcknblrxqitxzqmultcywugrugerez` int DEFAULT NULL,\n `ljkhgomectjxuobevwecxxsuytfmrzkp` int DEFAULT NULL,\n `zocxbvhfwqzswygjiyzzbztzfdxkscdw` int DEFAULT NULL,\n `mxnqrsfakmorwfdmlnrswhktrvrvpmsl` int DEFAULT NULL,\n `tbefjxwllujenzeboxeziuffjnnkelci` int DEFAULT NULL,\n `yauyrfearmzqtxyoznjxwypzccqasjfz` int DEFAULT NULL,\n `vqxucnywacmmvawzolthydivzdguulet` int DEFAULT NULL,\n `hrohrhdcgklksmytkumbpcssqquorfab` int DEFAULT NULL,\n `lmhkjpclrbudqffxzdfstzuuutmjynta` int DEFAULT NULL,\n `xifxplvfadhpfhuquwswkysivsizihgv` int DEFAULT NULL,\n `vcfzzgixbdaykdutyurribzmwbjkyjve` int DEFAULT NULL,\n `xkgtpyalsbgsoxacmnisaecwsprsapxh` int DEFAULT NULL,\n `mmbjkmhfpzinkiclrwfaidnymtvvnich` int DEFAULT NULL,\n `skfbfcamuiawqcfmkcubiqdgduajbvgz` int DEFAULT NULL,\n `jiyxtujfbieyxbuuwiebunbspzpesfcg` int DEFAULT NULL,\n `wkszirnfmlbdxbnvbzzdysrmzvotzrrz` int DEFAULT NULL,\n `gntefhxleplmbwxlukqosufrxpaoccqr` int DEFAULT NULL,\n `zyosqzarfwxwezxvtojdilmpzjebvonj` int DEFAULT NULL,\n `wsuxytsggadortorouculbftzcrpdwse` int DEFAULT NULL,\n `ninixiwyrjcwvnltzhgljudirsbhlekk` int DEFAULT NULL,\n `qcmqzycoyoqhqvofpwvftrxgsvunpwqr` int DEFAULT NULL,\n `uvmnfujenwszwsphrbcjirvgprodosyh` int DEFAULT NULL,\n `edwtcghfhhjnhwvvtcmrgaoqtnpyvnxs` int DEFAULT NULL,\n `sgeatykmziyenhzkrhooilktppmifatz` int DEFAULT NULL,\n `stwkuaebngcpnnsancswlegdrfrgbldd` int DEFAULT NULL,\n `tjjudqaihkwlnoqacgxampjfeiacmwgo` int DEFAULT NULL,\n `ezmmrdqtwytjtovzafxgqzmzhsnbmfmc` int DEFAULT NULL,\n `nfrtvrmmahelsqwdwlglidbcpbgaekpc` int DEFAULT NULL,\n `otxeexxnrqqfjcntpuqrvbrtfuqmcigv` int DEFAULT NULL,\n `kxrdahdtfueofejlxwzdnakpfalkviom` int DEFAULT NULL,\n `liyyquwtcbznwugjigcwevpyalfovcei` int DEFAULT NULL,\n `aqannosureorxcfpryzawpzaedmqifhz` int DEFAULT NULL,\n `jxcwmpnaceidpfxlerabvwqqmgqfzlld` int DEFAULT NULL,\n `xxmnqjvvgymnmkqqclhmvwrszlhynppt` int DEFAULT NULL,\n `oueqvwkvejjlhxnctrbxchzsurmtltvg` int DEFAULT NULL,\n `edlvzftulevxrhlhrsfdeanjbsazbsct` int DEFAULT NULL,\n `phvhkemkxzzqfembgohpxqlsquzhoguv` int DEFAULT NULL,\n `gmcyjgknqpfyiwxahbewdeacyrkcotow` int DEFAULT NULL,\n `zihyyeaycrwnfnfmureztzoevnsuazjf` int DEFAULT NULL,\n `izsysmuaadldxslxkkyhpzqxlzvlafxs` int DEFAULT NULL,\n `ugbvljqprahylwyibynrmeimeahqyjyp` int DEFAULT NULL,\n `ezjakesmqxumniofcoysgnwkirpgisuz` int DEFAULT NULL,\n `yflxckviyygrtpaxzbtxandppowezdvt` int DEFAULT NULL,\n `rzcfubxwyyynpqdglrknacoemszkobts` int DEFAULT NULL,\n `iayvryykegqyowzccyqjdranlqccwqjo` int DEFAULT NULL,\n `huevyugppedhrisplayiqypsorxacrpw` int DEFAULT NULL,\n `wsftlasyelygcqwzsbfqnyuxtyrwpexe` int DEFAULT NULL,\n `lqzruhcbdhsmouqvrhndgahfqzqeedpw` int DEFAULT NULL,\n `zpetrvucmkoqyttpvjudiocqfxpxwrfv` int DEFAULT NULL,\n `jwhhpphqnfiuwbqzzesbgeedqqosbxrb` int DEFAULT NULL,\n `kakwdxbctfbjlqnkxouqllignrdlvdds` int DEFAULT NULL,\n `xaiymukglhdfwebftzywxdrriqbblejg` int DEFAULT NULL,\n `eacsbwfddnikicrvcfhktplydquihqbn` int DEFAULT NULL,\n `ijmidyhavtaeaejxihlklwpekuatukzd` int DEFAULT NULL,\n `ftbwegnjtugfhlskkjwmrxsyuskxoqpk` int DEFAULT NULL,\n `vkguiynakzonbayfxlhgqhapqltodjbc` int DEFAULT NULL,\n `ehsrpupfqymqwyatnvfejluwxahdnaoz` int DEFAULT NULL,\n `aqzutujiebeyidwxmignxjkbjvksyjvs` int DEFAULT NULL,\n `odxavawzaromdbbirdicnkrzaxlzocll` int DEFAULT NULL,\n `gjjirfzvfiwenxkwvceawwyvptozoiqn` int DEFAULT NULL,\n `hovhgpbanqstuoaublldataocllzawtp` int DEFAULT NULL,\n `xurntoocdpzesckkadlwvaavptrsrasc` int DEFAULT NULL,\n `unutyfffrvzaintifjcyiktbarrrrego` int DEFAULT NULL,\n `lshorsdfyrfhuenshnnvbyiobekchfss` int DEFAULT NULL,\n `ndpuendygbgdjwavdmuimirrsytblvps` int DEFAULT NULL,\n `opmbojppkgruqgpdivkgfqtzhlvuozbg` int DEFAULT NULL,\n `khsjwmnqwmjbqzggqjdrnmlolsrxidus` int DEFAULT NULL,\n `whzxsvuvkvboudmjhqfntwjyzxrxblji` int DEFAULT NULL,\n `ggggaewhwyvcqxqwvjgjtzbysybhkvbs` int DEFAULT NULL,\n `dtvqoyqzzfuajwuukrrviyfskavnqmfi` int DEFAULT NULL,\n `vwnulrzohcwvpeysmnwtaktejpbfqxhi` int DEFAULT NULL,\n `dkzoxttxjlbpfvpakppwaguetwqelwds` int DEFAULT NULL,\n `ixrlcpwoddgjspvrszakzxmvxubpuunc` int DEFAULT NULL,\n PRIMARY KEY (`wjrvzrothgodipzyxfsguyamuguaodbs`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"lgzqspwbxcvbxbxdceywhoesgrnsmpga\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["wjrvzrothgodipzyxfsguyamuguaodbs"],"columns":[{"name":"wjrvzrothgodipzyxfsguyamuguaodbs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"rnlnvaumlgqxsyqlbdmhsdxztisagtnt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htkofgwknjmlufuremgwuxdsmzfqnzjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prngjnahcrfaqlfrxuehefbhexrsflfb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvifvmvdendnbnpzvjgecyvxuqkyiwfk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qnppuhwiynnttmmwzlrprsdhgvkbkprc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmoxvwbyxznpaeaewyypvpzswefzqxdz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwjmsrykavzckmjhorpswowpfjbmogxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cexaeiuutcmunheuwiyzruejejonhean","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkjbnpfiknnkhdwdvouqrqdxipphvoro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mepzgwrojyopvxbjkvxmgihjpfvtwena","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tcjxgikxbesclrrmsdaeitnvzhbcdgoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxehsfmusnawsgaxoczrbouqzzswxvyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eogoqrwownibmgsqemqtfnfusxfphcoe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhmhiesufbehgecstjiuamkzocthphnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"adkaxdofainwufhidlrtktmjlmjelaio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sydglfhedthbtvjaakuodhzpehudiome","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzijansnkrvmqbqbkryylduzhawpcmpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxjbmnvkaaoramgrtptyzunzpxalplwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnxtdmxeiwpohajrahteztsxzihhsgjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqpfbzpppchcsktxetgupfyblmvxdsfv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofbvtcbmpamrsobtddfmrocbkyhuthgt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxhjyznhhvvafdsvtmomcpgcpdojtbao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vyfuguxmviovqeewekbjlxrbtqtritjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krocnjlcnsyfninfnyaajinxphcpitcu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzktdtdsmuuronwsbdtmycsazohgenhg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nuuagqvxlpdvxcjydzrjojcaupwiwnqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrtcknblrxqitxzqmultcywugrugerez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljkhgomectjxuobevwecxxsuytfmrzkp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zocxbvhfwqzswygjiyzzbztzfdxkscdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mxnqrsfakmorwfdmlnrswhktrvrvpmsl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbefjxwllujenzeboxeziuffjnnkelci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yauyrfearmzqtxyoznjxwypzccqasjfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqxucnywacmmvawzolthydivzdguulet","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hrohrhdcgklksmytkumbpcssqquorfab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmhkjpclrbudqffxzdfstzuuutmjynta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xifxplvfadhpfhuquwswkysivsizihgv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vcfzzgixbdaykdutyurribzmwbjkyjve","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkgtpyalsbgsoxacmnisaecwsprsapxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmbjkmhfpzinkiclrwfaidnymtvvnich","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skfbfcamuiawqcfmkcubiqdgduajbvgz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jiyxtujfbieyxbuuwiebunbspzpesfcg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wkszirnfmlbdxbnvbzzdysrmzvotzrrz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gntefhxleplmbwxlukqosufrxpaoccqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyosqzarfwxwezxvtojdilmpzjebvonj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wsuxytsggadortorouculbftzcrpdwse","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ninixiwyrjcwvnltzhgljudirsbhlekk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qcmqzycoyoqhqvofpwvftrxgsvunpwqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvmnfujenwszwsphrbcjirvgprodosyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edwtcghfhhjnhwvvtcmrgaoqtnpyvnxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sgeatykmziyenhzkrhooilktppmifatz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"stwkuaebngcpnnsancswlegdrfrgbldd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjjudqaihkwlnoqacgxampjfeiacmwgo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezmmrdqtwytjtovzafxgqzmzhsnbmfmc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfrtvrmmahelsqwdwlglidbcpbgaekpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otxeexxnrqqfjcntpuqrvbrtfuqmcigv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxrdahdtfueofejlxwzdnakpfalkviom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"liyyquwtcbznwugjigcwevpyalfovcei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqannosureorxcfpryzawpzaedmqifhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jxcwmpnaceidpfxlerabvwqqmgqfzlld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxmnqjvvgymnmkqqclhmvwrszlhynppt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oueqvwkvejjlhxnctrbxchzsurmtltvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edlvzftulevxrhlhrsfdeanjbsazbsct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phvhkemkxzzqfembgohpxqlsquzhoguv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmcyjgknqpfyiwxahbewdeacyrkcotow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zihyyeaycrwnfnfmureztzoevnsuazjf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"izsysmuaadldxslxkkyhpzqxlzvlafxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugbvljqprahylwyibynrmeimeahqyjyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezjakesmqxumniofcoysgnwkirpgisuz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yflxckviyygrtpaxzbtxandppowezdvt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzcfubxwyyynpqdglrknacoemszkobts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iayvryykegqyowzccyqjdranlqccwqjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"huevyugppedhrisplayiqypsorxacrpw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wsftlasyelygcqwzsbfqnyuxtyrwpexe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqzruhcbdhsmouqvrhndgahfqzqeedpw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpetrvucmkoqyttpvjudiocqfxpxwrfv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwhhpphqnfiuwbqzzesbgeedqqosbxrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kakwdxbctfbjlqnkxouqllignrdlvdds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xaiymukglhdfwebftzywxdrriqbblejg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eacsbwfddnikicrvcfhktplydquihqbn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijmidyhavtaeaejxihlklwpekuatukzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftbwegnjtugfhlskkjwmrxsyuskxoqpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkguiynakzonbayfxlhgqhapqltodjbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehsrpupfqymqwyatnvfejluwxahdnaoz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqzutujiebeyidwxmignxjkbjvksyjvs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"odxavawzaromdbbirdicnkrzaxlzocll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjjirfzvfiwenxkwvceawwyvptozoiqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hovhgpbanqstuoaublldataocllzawtp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xurntoocdpzesckkadlwvaavptrsrasc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unutyfffrvzaintifjcyiktbarrrrego","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lshorsdfyrfhuenshnnvbyiobekchfss","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ndpuendygbgdjwavdmuimirrsytblvps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opmbojppkgruqgpdivkgfqtzhlvuozbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khsjwmnqwmjbqzggqjdrnmlolsrxidus","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"whzxsvuvkvboudmjhqfntwjyzxrxblji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggggaewhwyvcqxqwvjgjtzbysybhkvbs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtvqoyqzzfuajwuukrrviyfskavnqmfi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vwnulrzohcwvpeysmnwtaktejpbfqxhi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkzoxttxjlbpfvpakppwaguetwqelwds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixrlcpwoddgjspvrszakzxmvxubpuunc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669387,"databaseName":"models_schema","ddl":"CREATE TABLE `lneozjmnjubkgsxmcosrybaevorbgvvn` (\n `xmsimqsosbruhpqfjxxdvmklvyetndow` int NOT NULL,\n `gpidrotrnlswhuuovpovrdzxctatpdhx` int DEFAULT NULL,\n `wyswqnbgaazykgltivajzibsxsgovrzg` int DEFAULT NULL,\n `nuupludotkhlaeqlcokcthrywlblilij` int DEFAULT NULL,\n `nfzarkfycsnlbidzkkrpfjwgwyjtppin` int DEFAULT NULL,\n `lxzahowxugpgvldydqjpcsfuiixycbch` int DEFAULT NULL,\n `zebgzmwstoinpjqrhqfndxyidxbsejqr` int DEFAULT NULL,\n `quxlviguljzevxghosyfmmlkgsfavdkh` int DEFAULT NULL,\n `addyvsegvghmstkbcykxeqqxqmthkctp` int DEFAULT NULL,\n `drmehmbkeyugaoinmdentosyshlyfgzl` int DEFAULT NULL,\n `ytmitgmpxwipjcjbsmizgekqzqzsxrii` int DEFAULT NULL,\n `rspojyzrvdpalrwnvuuwlcysqjtfnuso` int DEFAULT NULL,\n `lxxyolpmnjmiwxlpwnqnebsrzpjbhjty` int DEFAULT NULL,\n `lrpecjkimpdgnwmrxkcihedoptdcmvtd` int DEFAULT NULL,\n `hnwgmjolhvsqsndnchfjkmghidydogyq` int DEFAULT NULL,\n `auvebpizmbhdprlzponsizwqejscaouu` int DEFAULT NULL,\n `wqhdqcampghlctdfkgtwughmrwuclbjz` int DEFAULT NULL,\n `iditfstcjtfkkmvwsqftzhwfcfqpwiuy` int DEFAULT NULL,\n `tgtaboajhzjlycfhuqsxkylgcbczpnbs` int DEFAULT NULL,\n `vioedwivvwilujqemadtvolhrfmfxffp` int DEFAULT NULL,\n `gngcaelkirfcduczljmemjjgldcqidct` int DEFAULT NULL,\n `nbpevwidwlgoxkfthanyrumlamolproy` int DEFAULT NULL,\n `sbwdnfjaytowuvscuvxrrffbdzudpmok` int DEFAULT NULL,\n `tmjkwnaijnnmvpfpcyrrdpwiyetdtytt` int DEFAULT NULL,\n `hgdnfcrzxxriiqcmipcevkommigytunc` int DEFAULT NULL,\n `amhsynqykxhalfzflumdwxiyfqlcjame` int DEFAULT NULL,\n `gvtprvbisiuevkgbvrtpwgvyftipcevo` int DEFAULT NULL,\n `rngryjtcajhhtcmretdvktdidplnjhrf` int DEFAULT NULL,\n `cuahnbsoaecsxgyjpusvgfpakeliknyw` int DEFAULT NULL,\n `gaghlcthryjjjktbjvwckjkyijgfuosq` int DEFAULT NULL,\n `eueohnqaiaznpugngkgrtewmbwxemguc` int DEFAULT NULL,\n `kpquajlwrkimqoobthokmaglcajthwwu` int DEFAULT NULL,\n `fvqatdxdcizxkslixgnrspmbgdvjgjjq` int DEFAULT NULL,\n `ndpflobsqdxwvrlilcpdcexpfwkswedf` int DEFAULT NULL,\n `btvwqwmgygaoybldkqtocyiredyxgfpd` int DEFAULT NULL,\n `lucsaijjldlzuwmjewwkjvozxubbvuqc` int DEFAULT NULL,\n `nhyrekjbsvlhlmkcdzjilhdphujwczcm` int DEFAULT NULL,\n `bzpfkrprgvbrstiwlhghnthbxmdirkwj` int DEFAULT NULL,\n `bfojakrkvvgqlqquvoehsddhfmabqgey` int DEFAULT NULL,\n `eqkjijerjbogicdnenkowfjfkyotnvlc` int DEFAULT NULL,\n `rhujlxgvzpcmbotihkitizhmojgiqsdv` int DEFAULT NULL,\n `zykyrqkcvmjtzggqbtvmfbboqpskpbhx` int DEFAULT NULL,\n `ezcngicbsimptnjrjxvluiirtzjxewqo` int DEFAULT NULL,\n `lmzfhiottchaahlnddoaaiiwsjnpogdh` int DEFAULT NULL,\n `rlelkwfaypirvtbgfthaxmcutxyyxwbg` int DEFAULT NULL,\n `kkihjrmcthwhrnuyvyjzbrnzpcxkjhys` int DEFAULT NULL,\n `ognryixbzpzfhbnwlumnxoarlnpmsfyi` int DEFAULT NULL,\n `cwusznvqahduvanyvxrhbbmsrttzxybi` int DEFAULT NULL,\n `mpxlbgfutrziwmipdbjnijdfdwihjjqv` int DEFAULT NULL,\n `pejiivjqyxggqfpawbwpnwmggortvbnv` int DEFAULT NULL,\n `wnpxcohxwwtskyzhevefwhxzfyxavity` int DEFAULT NULL,\n `bnkcmhlbhfvcpfhtzqjehvslbkunttun` int DEFAULT NULL,\n `eluhmdeacjlzwfnwyktizeqclnjzwdgs` int DEFAULT NULL,\n `wekbpkmykkuivvckiugbggnydzoxkwiw` int DEFAULT NULL,\n `tfokdopuplxnclhxohizmkqtdafznyjn` int DEFAULT NULL,\n `qjcschfonzaykivsoiwagmwqojejbzrp` int DEFAULT NULL,\n `asszpjgsnfsouyzmzwehuxvujdmjgbul` int DEFAULT NULL,\n `eeyhtisiipxuhotlvzyqcdprrzkltxvg` int DEFAULT NULL,\n `ltmsapillxoqkfvqgfcpdzinwotyjbiq` int DEFAULT NULL,\n `bxybtpkxilfbvddfabqyxexjudfccovs` int DEFAULT NULL,\n `baufraruebvdozvuynlvazccrvhswxyo` int DEFAULT NULL,\n `rxibyzsbwxkvtqleahdmrdodfqylrncz` int DEFAULT NULL,\n `utwcozncmsjvndbnqcnzfczirrtjzkah` int DEFAULT NULL,\n `mqvdqcehpkfgrbzfkygthbvdwqagtqoh` int DEFAULT NULL,\n `qspycwprnwywcusbnngfdkyknajjycly` int DEFAULT NULL,\n `gtnraxrgjhroarjbrdksiakhiteyjyaz` int DEFAULT NULL,\n `elqtlaorxytgbfiirfquhozbcorxacas` int DEFAULT NULL,\n `caffubzxftpgyoyqpyhpyrssbqrwjqwr` int DEFAULT NULL,\n `xyihmkxcpitvahnvjuuaaymfmaspbaey` int DEFAULT NULL,\n `czmakqidancaogjseylewxpnfgcpbrtd` int DEFAULT NULL,\n `ebbeuwmlvujtyksyftpbofbcbosxizgv` int DEFAULT NULL,\n `kxaqtlzjqmglndojpqaztjkqjkulwajk` int DEFAULT NULL,\n `wnibdxqlsjmjcwtysbizsjqpldwtdvvm` int DEFAULT NULL,\n `fhgtednwuyteakppcshljgrlqoycjthr` int DEFAULT NULL,\n `rioswpnxgapwgyludlcugmyjwofidstn` int DEFAULT NULL,\n `yelfnkvwjuvpdkbsunqknbarbhvdqroy` int DEFAULT NULL,\n `jchqjofthmkfqcixsfgydexbiqliivwp` int DEFAULT NULL,\n `ahcmomguqiylajmciwtosenjyvntrbtc` int DEFAULT NULL,\n `suknuimugjyviqmvqqtpaogspiqplaba` int DEFAULT NULL,\n `znondilrnvrzfvsguqjhawgniufwaonc` int DEFAULT NULL,\n `lupraclayvxccsdroxyhcbagdewvihha` int DEFAULT NULL,\n `bfvllafthqomgitxyjorrchjklpojfuu` int DEFAULT NULL,\n `rxrevpjzzaassnzaefhifzkecmnsddfx` int DEFAULT NULL,\n `avaaqhwjqkkomsluuzvdopohcsfivivu` int DEFAULT NULL,\n `lmretcqijhvefqlcfhlswnpyuoawxlir` int DEFAULT NULL,\n `epfevqmnbpiszgvhkpsvuuxdarfufnuk` int DEFAULT NULL,\n `vcctzakfcphnadirvacubdzjclrcuveu` int DEFAULT NULL,\n `qqwjuolajjyhvmyavkmzhymxnrxkuxzc` int DEFAULT NULL,\n `wcjqcljzwsqbuzmsfemsczkahixevbkk` int DEFAULT NULL,\n `vfvtexhjsjruhwotvbpsdprekcbmjwmz` int DEFAULT NULL,\n `xegrontbcfjdaglymkekednbjazvhrxu` int DEFAULT NULL,\n `qmdykjquuxpnfkwlgudmchjodownxsyj` int DEFAULT NULL,\n `yezzcrgsfrkcbwmsxhwqqrfbvfatptcm` int DEFAULT NULL,\n `rqfzzvkqclwasdqmiqzrjldvpofrqxpq` int DEFAULT NULL,\n `cpfwrsybkodkvovwfekqmafhsrgcmean` int DEFAULT NULL,\n `lltvaxlmmwnfxgtfzvafejjnxjpfxnlw` int DEFAULT NULL,\n `yyvlzbjpqgonwispusqixpvvfyusjybs` int DEFAULT NULL,\n `pycihfamepfgakrbesdhebeobncdovdk` int DEFAULT NULL,\n `ddovzgexkcvalipfzmixtesoiajjnnky` int DEFAULT NULL,\n `pybfjegolupzukerystlnbbqnaczwcpt` int DEFAULT NULL,\n PRIMARY KEY (`xmsimqsosbruhpqfjxxdvmklvyetndow`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"lneozjmnjubkgsxmcosrybaevorbgvvn\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["xmsimqsosbruhpqfjxxdvmklvyetndow"],"columns":[{"name":"xmsimqsosbruhpqfjxxdvmklvyetndow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"gpidrotrnlswhuuovpovrdzxctatpdhx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyswqnbgaazykgltivajzibsxsgovrzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nuupludotkhlaeqlcokcthrywlblilij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfzarkfycsnlbidzkkrpfjwgwyjtppin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxzahowxugpgvldydqjpcsfuiixycbch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zebgzmwstoinpjqrhqfndxyidxbsejqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"quxlviguljzevxghosyfmmlkgsfavdkh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"addyvsegvghmstkbcykxeqqxqmthkctp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drmehmbkeyugaoinmdentosyshlyfgzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytmitgmpxwipjcjbsmizgekqzqzsxrii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rspojyzrvdpalrwnvuuwlcysqjtfnuso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxxyolpmnjmiwxlpwnqnebsrzpjbhjty","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lrpecjkimpdgnwmrxkcihedoptdcmvtd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hnwgmjolhvsqsndnchfjkmghidydogyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"auvebpizmbhdprlzponsizwqejscaouu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqhdqcampghlctdfkgtwughmrwuclbjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iditfstcjtfkkmvwsqftzhwfcfqpwiuy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tgtaboajhzjlycfhuqsxkylgcbczpnbs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vioedwivvwilujqemadtvolhrfmfxffp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gngcaelkirfcduczljmemjjgldcqidct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbpevwidwlgoxkfthanyrumlamolproy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbwdnfjaytowuvscuvxrrffbdzudpmok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmjkwnaijnnmvpfpcyrrdpwiyetdtytt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hgdnfcrzxxriiqcmipcevkommigytunc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amhsynqykxhalfzflumdwxiyfqlcjame","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gvtprvbisiuevkgbvrtpwgvyftipcevo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rngryjtcajhhtcmretdvktdidplnjhrf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cuahnbsoaecsxgyjpusvgfpakeliknyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gaghlcthryjjjktbjvwckjkyijgfuosq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eueohnqaiaznpugngkgrtewmbwxemguc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpquajlwrkimqoobthokmaglcajthwwu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fvqatdxdcizxkslixgnrspmbgdvjgjjq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ndpflobsqdxwvrlilcpdcexpfwkswedf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"btvwqwmgygaoybldkqtocyiredyxgfpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lucsaijjldlzuwmjewwkjvozxubbvuqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhyrekjbsvlhlmkcdzjilhdphujwczcm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzpfkrprgvbrstiwlhghnthbxmdirkwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bfojakrkvvgqlqquvoehsddhfmabqgey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqkjijerjbogicdnenkowfjfkyotnvlc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhujlxgvzpcmbotihkitizhmojgiqsdv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zykyrqkcvmjtzggqbtvmfbboqpskpbhx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezcngicbsimptnjrjxvluiirtzjxewqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmzfhiottchaahlnddoaaiiwsjnpogdh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlelkwfaypirvtbgfthaxmcutxyyxwbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kkihjrmcthwhrnuyvyjzbrnzpcxkjhys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ognryixbzpzfhbnwlumnxoarlnpmsfyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwusznvqahduvanyvxrhbbmsrttzxybi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpxlbgfutrziwmipdbjnijdfdwihjjqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pejiivjqyxggqfpawbwpnwmggortvbnv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnpxcohxwwtskyzhevefwhxzfyxavity","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnkcmhlbhfvcpfhtzqjehvslbkunttun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eluhmdeacjlzwfnwyktizeqclnjzwdgs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wekbpkmykkuivvckiugbggnydzoxkwiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tfokdopuplxnclhxohizmkqtdafznyjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjcschfonzaykivsoiwagmwqojejbzrp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"asszpjgsnfsouyzmzwehuxvujdmjgbul","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eeyhtisiipxuhotlvzyqcdprrzkltxvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ltmsapillxoqkfvqgfcpdzinwotyjbiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bxybtpkxilfbvddfabqyxexjudfccovs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"baufraruebvdozvuynlvazccrvhswxyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxibyzsbwxkvtqleahdmrdodfqylrncz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utwcozncmsjvndbnqcnzfczirrtjzkah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqvdqcehpkfgrbzfkygthbvdwqagtqoh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qspycwprnwywcusbnngfdkyknajjycly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtnraxrgjhroarjbrdksiakhiteyjyaz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elqtlaorxytgbfiirfquhozbcorxacas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"caffubzxftpgyoyqpyhpyrssbqrwjqwr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyihmkxcpitvahnvjuuaaymfmaspbaey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"czmakqidancaogjseylewxpnfgcpbrtd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebbeuwmlvujtyksyftpbofbcbosxizgv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxaqtlzjqmglndojpqaztjkqjkulwajk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnibdxqlsjmjcwtysbizsjqpldwtdvvm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhgtednwuyteakppcshljgrlqoycjthr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rioswpnxgapwgyludlcugmyjwofidstn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yelfnkvwjuvpdkbsunqknbarbhvdqroy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jchqjofthmkfqcixsfgydexbiqliivwp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahcmomguqiylajmciwtosenjyvntrbtc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suknuimugjyviqmvqqtpaogspiqplaba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znondilrnvrzfvsguqjhawgniufwaonc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lupraclayvxccsdroxyhcbagdewvihha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bfvllafthqomgitxyjorrchjklpojfuu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxrevpjzzaassnzaefhifzkecmnsddfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avaaqhwjqkkomsluuzvdopohcsfivivu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmretcqijhvefqlcfhlswnpyuoawxlir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"epfevqmnbpiszgvhkpsvuuxdarfufnuk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vcctzakfcphnadirvacubdzjclrcuveu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqwjuolajjyhvmyavkmzhymxnrxkuxzc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcjqcljzwsqbuzmsfemsczkahixevbkk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vfvtexhjsjruhwotvbpsdprekcbmjwmz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xegrontbcfjdaglymkekednbjazvhrxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmdykjquuxpnfkwlgudmchjodownxsyj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yezzcrgsfrkcbwmsxhwqqrfbvfatptcm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqfzzvkqclwasdqmiqzrjldvpofrqxpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cpfwrsybkodkvovwfekqmafhsrgcmean","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lltvaxlmmwnfxgtfzvafejjnxjpfxnlw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yyvlzbjpqgonwispusqixpvvfyusjybs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pycihfamepfgakrbesdhebeobncdovdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddovzgexkcvalipfzmixtesoiajjnnky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pybfjegolupzukerystlnbbqnaczwcpt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669421,"databaseName":"models_schema","ddl":"CREATE TABLE `lqbwjasfleffegotnszounuxojffyehx` (\n `vxljxeklpgfharzeyqsyyislebnodztg` int NOT NULL,\n `ckmnccpzyyzzpzpwxogjzdedqeodttpd` int DEFAULT NULL,\n `vueqtggktklxmqcagaqydajovraxqwsu` int DEFAULT NULL,\n `nmvfpmsmcjznzkxkcbghrfbgdlufeafs` int DEFAULT NULL,\n `ujkksgugcnzbujxlivzvlrigorzwpscz` int DEFAULT NULL,\n `sxgtxmonpaasxgzuzkcbnbrmpvulslqe` int DEFAULT NULL,\n `nkvhdmrobrdnnftigqbwoddugjoafvik` int DEFAULT NULL,\n `dxdyqofkdcpttabnqlwhcsbuwibwzifl` int DEFAULT NULL,\n `sacoarusaypzdyzstfrcvwltuhflgfaj` int DEFAULT NULL,\n `prumadvvinsgfpczqsjlyroxoarfyjpc` int DEFAULT NULL,\n `gqjbkrloippmcwrzguqwhftjyuujzgnu` int DEFAULT NULL,\n `pgsnbzsfhpdzzcyjwcmzlditxzcwlqtc` int DEFAULT NULL,\n `fidypndwowswusousrpdbweztpugmbwz` int DEFAULT NULL,\n `wrdzikgommvgzhcromgvtlqclzvogwxv` int DEFAULT NULL,\n `thoiffngswmlngrulojugvcydjiqnubu` int DEFAULT NULL,\n `odefpxgwhnguuuwuqrzknzalepxctkzb` int DEFAULT NULL,\n `gkadcngvfbjnjkwpkssygknhrrpsgkdv` int DEFAULT NULL,\n `lsjjtzdmsvmstiofcrrvnaifgurbvghb` int DEFAULT NULL,\n `enkcpbqwkuojytkcransvnindkvfhykj` int DEFAULT NULL,\n `ofverbmdcqzfrquhdidvzjqrwtwxsggo` int DEFAULT NULL,\n `jdfmqihamnprmzzcgkcxfwhydlwnfbno` int DEFAULT NULL,\n `qlannfzyrrurwjjdbzhyrcnsodjntbdl` int DEFAULT NULL,\n `srmnhiyvmqmqxjnssyrsoqcwrihvsllc` int DEFAULT NULL,\n `wcqxwjceuhdvxchqyhvimquugdfoqdfz` int DEFAULT NULL,\n `yiyfdmmivvhzwkvotzduzbylomtiuzms` int DEFAULT NULL,\n `pbdvwvfkvxmqisubxvnkzolgxglwplua` int DEFAULT NULL,\n `bqgaqjicsclnyjxordwfqntbmedfioom` int DEFAULT NULL,\n `vrcslcceefzibmcbpvhoffuoiyleogvk` int DEFAULT NULL,\n `lrciufxzsupehbivkazgvlyjxjzoqmbb` int DEFAULT NULL,\n `ponelxzxxamsseslhhgnpyozszgvwazg` int DEFAULT NULL,\n `rmrqgxxkesdrwzhzvjbwgulpygknxtdf` int DEFAULT NULL,\n `govvjzzanaphvnjzhqlnnjyisggqvrvx` int DEFAULT NULL,\n `lvrkgywnhpjsivdwactgihqrujriexbf` int DEFAULT NULL,\n `navyvcqgggzckxqlottcssudpxtgniav` int DEFAULT NULL,\n `jmoxlhgzflcdzziduwwqaokaqklktfzg` int DEFAULT NULL,\n `axdcnebqrturymxdoivmnytlpwgzoobf` int DEFAULT NULL,\n `gyhoqpalbutchgcsalandgjavuebwndc` int DEFAULT NULL,\n `bqowdsfjxyuueihcsvnqfmuaclwzzuog` int DEFAULT NULL,\n `znemdnszterneirgkfphdlerimjxxsqz` int DEFAULT NULL,\n `evikbuwqrdclyuaohfvolygabedlacmy` int DEFAULT NULL,\n `jajzeqtjfmikmfveyullakicydbwjofh` int DEFAULT NULL,\n `igqgfurowjggjvawezuysfrswwmseqfx` int DEFAULT NULL,\n `pdbrjkmfygmfxheuffxczmljqmxhnjvg` int DEFAULT NULL,\n `nbjesvmehzjipluonjykichhhjifvyqo` int DEFAULT NULL,\n `kedybkusilvwbsxsdswrrhwcwwyalpzf` int DEFAULT NULL,\n `cmudkcidxzlftejungqcquqnnmsqsnrj` int DEFAULT NULL,\n `uaqhjhrbpwloqzzueyphttuocysrgusp` int DEFAULT NULL,\n `uibqdkltkxfxtjsceamhcuqfksevelmr` int DEFAULT NULL,\n `mphmowippkpulejwwvbcqarbvevpujgc` int DEFAULT NULL,\n `rrwocnffwiaorwgzcaiafuvtryatwsoo` int DEFAULT NULL,\n `fdjnuxncputfzqfphfkjtyxaffkqohsv` int DEFAULT NULL,\n `qthiiniofsbqfvzakzzjnbntojwqwvsl` int DEFAULT NULL,\n `nbiofpkblmdmtwvvricrtmuwbytbmcdb` int DEFAULT NULL,\n `ahaoczknkhtkqyntcrymjeiczehvgwfg` int DEFAULT NULL,\n `pdvbrqpoltfdhiglvbvzqpwsblycqaiy` int DEFAULT NULL,\n `norqxsgmkfnmktvsotqkddyicowkdpbv` int DEFAULT NULL,\n `txbmcafyoftspcecygpcvfkvhwnkoojs` int DEFAULT NULL,\n `lcqrcrcgpifbtbhcptbztwxscjjdpftk` int DEFAULT NULL,\n `ymaebhtrnovovbxggndogzpdwyrpgsiv` int DEFAULT NULL,\n `lvacurxqomljynltyotqxqzqvxsnbety` int DEFAULT NULL,\n `hjscamrjqpoxaycuinftyldkxosnmiyq` int DEFAULT NULL,\n `llaewtiheqmgradpczfrlmuvnwlrsbsx` int DEFAULT NULL,\n `fpmktburakjdmowthhxjymuamvvafxfb` int DEFAULT NULL,\n `ircrrchstoobclqttqohprjghyhddkol` int DEFAULT NULL,\n `waoasyyxjtaqzjwpttdbgtyswmrtvyog` int DEFAULT NULL,\n `imdqwmkyhmmoyqdzospckwmagazclqix` int DEFAULT NULL,\n `mzzrrkqxuwxfqnjubqkzrxljrstsaqxb` int DEFAULT NULL,\n `xjbyqnxigtguspsvbofojqzhragkldmw` int DEFAULT NULL,\n `htkfvvrubblutkgilnrjoezcfxuhvzkn` int DEFAULT NULL,\n `urqrxekunuikrsnrnhbfbnuiczkkeikl` int DEFAULT NULL,\n `xbsbnexqyzsrosqfptpuiorsnyownnyk` int DEFAULT NULL,\n `cnbgakjhsuiciykxfiuwxtnkzwtbtksl` int DEFAULT NULL,\n `vkzjzurrqtgzbgrglxafznqcitlalccx` int DEFAULT NULL,\n `swhlhvipfwsmqxeojrwwkychqhdpfvma` int DEFAULT NULL,\n `exaazkaqcmxohnxfjlngfgufbbhowoon` int DEFAULT NULL,\n `kjfifittwqarslcwopgvobighlstmatu` int DEFAULT NULL,\n `otkfrmvgjhobnirrqrnfpfjnitwuwswa` int DEFAULT NULL,\n `tmubobghvlunqtrwortkzkhcbqbegrii` int DEFAULT NULL,\n `ptuaqxnwqxhomngwipflnzdicnjkajdy` int DEFAULT NULL,\n `vxmpmrsruqlpclivrnnnuttxkdknegud` int DEFAULT NULL,\n `rdgmfqztrriuxcuatfdhlpifolisvtxm` int DEFAULT NULL,\n `uclwjzfeubpdajgxxcxnxhfnqjkspmdx` int DEFAULT NULL,\n `lixfcalbgayibdimbwcnbhgqshktqfaz` int DEFAULT NULL,\n `qieqfxychjklzqfnmmygahadlmoughca` int DEFAULT NULL,\n `auwfpymogfhofkqmbhmbxloucvldoeid` int DEFAULT NULL,\n `ktviwxducxageectzaquwzuatmbqciwx` int DEFAULT NULL,\n `ppgkxbeuuyfvovtqndursdoglzcyxnyx` int DEFAULT NULL,\n `ioazggfrwhspmabrwcrzypfrlqotlumg` int DEFAULT NULL,\n `ofzaklsnoeclxebybservusufvsddqeo` int DEFAULT NULL,\n `qkgrlwbwsarwpwcccahkbstqcbmfsvng` int DEFAULT NULL,\n `wxdzquuvqdinyrvslfiejncvkrhbfrom` int DEFAULT NULL,\n `dvzculyjswkxdigwxeywmrlfhvntxeqr` int DEFAULT NULL,\n `xyidbhdbcpikuhvlqcgcyarusfdowzcn` int DEFAULT NULL,\n `aaezatzqfmqsvljdaaswqbgymlcqdudb` int DEFAULT NULL,\n `jbqlducztmmnzngstiukhwtqerxtcvdk` int DEFAULT NULL,\n `tmomlawykiiflkviuxnvrkqbwefeokms` int DEFAULT NULL,\n `tfimijuvlusdzvrhlozeopwwlwrwsqrk` int DEFAULT NULL,\n `qjkrgvaltyldxaiypnuwkyfmuiqbtbyg` int DEFAULT NULL,\n `qvpbfwxsigsdbhkrxywuxuzlusvpruxp` int DEFAULT NULL,\n `wlpruxzwknijqbuvkeivtgkthftpsanw` int DEFAULT NULL,\n PRIMARY KEY (`vxljxeklpgfharzeyqsyyislebnodztg`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"lqbwjasfleffegotnszounuxojffyehx\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["vxljxeklpgfharzeyqsyyislebnodztg"],"columns":[{"name":"vxljxeklpgfharzeyqsyyislebnodztg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ckmnccpzyyzzpzpwxogjzdedqeodttpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vueqtggktklxmqcagaqydajovraxqwsu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmvfpmsmcjznzkxkcbghrfbgdlufeafs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujkksgugcnzbujxlivzvlrigorzwpscz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sxgtxmonpaasxgzuzkcbnbrmpvulslqe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkvhdmrobrdnnftigqbwoddugjoafvik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxdyqofkdcpttabnqlwhcsbuwibwzifl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sacoarusaypzdyzstfrcvwltuhflgfaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prumadvvinsgfpczqsjlyroxoarfyjpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqjbkrloippmcwrzguqwhftjyuujzgnu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgsnbzsfhpdzzcyjwcmzlditxzcwlqtc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fidypndwowswusousrpdbweztpugmbwz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wrdzikgommvgzhcromgvtlqclzvogwxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thoiffngswmlngrulojugvcydjiqnubu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"odefpxgwhnguuuwuqrzknzalepxctkzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkadcngvfbjnjkwpkssygknhrrpsgkdv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lsjjtzdmsvmstiofcrrvnaifgurbvghb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"enkcpbqwkuojytkcransvnindkvfhykj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofverbmdcqzfrquhdidvzjqrwtwxsggo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdfmqihamnprmzzcgkcxfwhydlwnfbno","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qlannfzyrrurwjjdbzhyrcnsodjntbdl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"srmnhiyvmqmqxjnssyrsoqcwrihvsllc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcqxwjceuhdvxchqyhvimquugdfoqdfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yiyfdmmivvhzwkvotzduzbylomtiuzms","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbdvwvfkvxmqisubxvnkzolgxglwplua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqgaqjicsclnyjxordwfqntbmedfioom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrcslcceefzibmcbpvhoffuoiyleogvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lrciufxzsupehbivkazgvlyjxjzoqmbb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ponelxzxxamsseslhhgnpyozszgvwazg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rmrqgxxkesdrwzhzvjbwgulpygknxtdf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"govvjzzanaphvnjzhqlnnjyisggqvrvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvrkgywnhpjsivdwactgihqrujriexbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"navyvcqgggzckxqlottcssudpxtgniav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmoxlhgzflcdzziduwwqaokaqklktfzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axdcnebqrturymxdoivmnytlpwgzoobf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gyhoqpalbutchgcsalandgjavuebwndc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqowdsfjxyuueihcsvnqfmuaclwzzuog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znemdnszterneirgkfphdlerimjxxsqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evikbuwqrdclyuaohfvolygabedlacmy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jajzeqtjfmikmfveyullakicydbwjofh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igqgfurowjggjvawezuysfrswwmseqfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdbrjkmfygmfxheuffxczmljqmxhnjvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbjesvmehzjipluonjykichhhjifvyqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kedybkusilvwbsxsdswrrhwcwwyalpzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmudkcidxzlftejungqcquqnnmsqsnrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uaqhjhrbpwloqzzueyphttuocysrgusp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uibqdkltkxfxtjsceamhcuqfksevelmr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mphmowippkpulejwwvbcqarbvevpujgc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrwocnffwiaorwgzcaiafuvtryatwsoo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fdjnuxncputfzqfphfkjtyxaffkqohsv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qthiiniofsbqfvzakzzjnbntojwqwvsl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbiofpkblmdmtwvvricrtmuwbytbmcdb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahaoczknkhtkqyntcrymjeiczehvgwfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdvbrqpoltfdhiglvbvzqpwsblycqaiy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"norqxsgmkfnmktvsotqkddyicowkdpbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txbmcafyoftspcecygpcvfkvhwnkoojs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lcqrcrcgpifbtbhcptbztwxscjjdpftk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ymaebhtrnovovbxggndogzpdwyrpgsiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvacurxqomljynltyotqxqzqvxsnbety","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjscamrjqpoxaycuinftyldkxosnmiyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llaewtiheqmgradpczfrlmuvnwlrsbsx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpmktburakjdmowthhxjymuamvvafxfb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ircrrchstoobclqttqohprjghyhddkol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"waoasyyxjtaqzjwpttdbgtyswmrtvyog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"imdqwmkyhmmoyqdzospckwmagazclqix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzzrrkqxuwxfqnjubqkzrxljrstsaqxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjbyqnxigtguspsvbofojqzhragkldmw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htkfvvrubblutkgilnrjoezcfxuhvzkn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urqrxekunuikrsnrnhbfbnuiczkkeikl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbsbnexqyzsrosqfptpuiorsnyownnyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnbgakjhsuiciykxfiuwxtnkzwtbtksl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkzjzurrqtgzbgrglxafznqcitlalccx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swhlhvipfwsmqxeojrwwkychqhdpfvma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exaazkaqcmxohnxfjlngfgufbbhowoon","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjfifittwqarslcwopgvobighlstmatu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otkfrmvgjhobnirrqrnfpfjnitwuwswa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmubobghvlunqtrwortkzkhcbqbegrii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ptuaqxnwqxhomngwipflnzdicnjkajdy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxmpmrsruqlpclivrnnnuttxkdknegud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdgmfqztrriuxcuatfdhlpifolisvtxm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uclwjzfeubpdajgxxcxnxhfnqjkspmdx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lixfcalbgayibdimbwcnbhgqshktqfaz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qieqfxychjklzqfnmmygahadlmoughca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"auwfpymogfhofkqmbhmbxloucvldoeid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ktviwxducxageectzaquwzuatmbqciwx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ppgkxbeuuyfvovtqndursdoglzcyxnyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ioazggfrwhspmabrwcrzypfrlqotlumg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofzaklsnoeclxebybservusufvsddqeo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qkgrlwbwsarwpwcccahkbstqcbmfsvng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxdzquuvqdinyrvslfiejncvkrhbfrom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvzculyjswkxdigwxeywmrlfhvntxeqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyidbhdbcpikuhvlqcgcyarusfdowzcn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aaezatzqfmqsvljdaaswqbgymlcqdudb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbqlducztmmnzngstiukhwtqerxtcvdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmomlawykiiflkviuxnvrkqbwefeokms","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tfimijuvlusdzvrhlozeopwwlwrwsqrk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjkrgvaltyldxaiypnuwkyfmuiqbtbyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvpbfwxsigsdbhkrxywuxuzlusvpruxp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wlpruxzwknijqbuvkeivtgkthftpsanw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669458,"databaseName":"models_schema","ddl":"CREATE TABLE `lzbcpccmabxinidkpdxbjeyepkhowvqb` (\n `jipdcpzdxqvfoaowglzumftrwgyvucmk` int NOT NULL,\n `enqcejoiudqbfopqtcimwqwuttrkvfkx` int DEFAULT NULL,\n `ecqbpgudtbpciveheckdlzijuthcgygb` int DEFAULT NULL,\n `bztoeyocahuvkszpknuepyjdndjqcuvh` int DEFAULT NULL,\n `cgqprdsgviwbciuocbdtafahwcgenlco` int DEFAULT NULL,\n `dgbzfnzpvefytudioficdqdqpqnrztwr` int DEFAULT NULL,\n `xbdeuhzieddcfeetkywkqobikbijxrmt` int DEFAULT NULL,\n `gcmfbzxcwsshnguzszdzlqalizjehvpe` int DEFAULT NULL,\n `tdcmaatzauywzmmqhtfxybvblbzqhxpp` int DEFAULT NULL,\n `ucdfunxanqimwsgipgkchdmkkxxjzlna` int DEFAULT NULL,\n `ennsltsutmuxxutovgtbxiogajhlbppd` int DEFAULT NULL,\n `vzenhpzeeqxxltajxvucfzkyvzfkqcvm` int DEFAULT NULL,\n `gmpefnsjwivizocvbvsmuyatamtiqthl` int DEFAULT NULL,\n `uazyqcgddtkzegbidtjgqpznhritpqdj` int DEFAULT NULL,\n `elswuuleqaeqqajxorznizrnchubggtp` int DEFAULT NULL,\n `xgbmatezwjfrwmxqpopkdzivpkifoojt` int DEFAULT NULL,\n `nnspybrlqrmpmdjnemxpyndmdygeuive` int DEFAULT NULL,\n `ntplbexpkcgycdgjhjjwwxgkqjbcyrhs` int DEFAULT NULL,\n `ggedqntlishxnkknfiktxlggoeytnnkc` int DEFAULT NULL,\n `qfsndneradadlmaggmnbhysczbjcqkfo` int DEFAULT NULL,\n `opaoremmbertbdlciolhmcrycgqlbubw` int DEFAULT NULL,\n `uwsgovtelqrauvecikwpojulbamfjkjk` int DEFAULT NULL,\n `gefkccdxsabkiaqnatmgiczbcfqjhtvw` int DEFAULT NULL,\n `tfzraylfvwpzbxzaabwxfxckybhqegqv` int DEFAULT NULL,\n `qnujbmsbwqxmjveqqcuwbuhegasnldav` int DEFAULT NULL,\n `dizfuneyfcustlcxhggqpumwseqgwggs` int DEFAULT NULL,\n `extvhkqcwufgmduhjuhhmulmxmasxfkv` int DEFAULT NULL,\n `jhwcldrwygsszjsmdsvsmcnzattcnhyp` int DEFAULT NULL,\n `zyckxikzkxdhjqznduzzoycowtjzwomr` int DEFAULT NULL,\n `shnwsstncbzcmihmqecvjqiwctlcpmso` int DEFAULT NULL,\n `wbooetpjmzqkutsbkhdzashocqibmomc` int DEFAULT NULL,\n `kgjhunuvggecgfsvavgwjyeebdohdzze` int DEFAULT NULL,\n `sasimxewtnxytqqbqvxqqgzutgmpugor` int DEFAULT NULL,\n `glbudmvacwtlzejrvtfiyqiyqabcbvpq` int DEFAULT NULL,\n `yaftopzhdnqqjpehvydapwwtokvclofs` int DEFAULT NULL,\n `yhnbjdpqvibcscogmafiigysnmegbkyn` int DEFAULT NULL,\n `btkwybryxfhxeoybuwjxckhfcxmtfjdt` int DEFAULT NULL,\n `fnofsuyqovlghiaowqhuohahpybxmpdw` int DEFAULT NULL,\n `hwwazgwulualhxexbabdqsbqhjlzdzkp` int DEFAULT NULL,\n `xrferbdtizdrnhzzstqdonmqjcobcpcb` int DEFAULT NULL,\n `rzzbtaohnuqwbiukompffuoibphmmbrl` int DEFAULT NULL,\n `dplqrxppndwxtpvrowwulgyssriwharu` int DEFAULT NULL,\n `cfqfdypivmzvuwpigkolhoksxwydzaga` int DEFAULT NULL,\n `xqadniffnwpvlzzpfvgedmhyvwyydrcb` int DEFAULT NULL,\n `gfpaojaunkzhejdkwyhksmxajxrbhjki` int DEFAULT NULL,\n `lxxbedqwcpzavxvrxvenznekfamywkmm` int DEFAULT NULL,\n `qdnllacehrutegtxmoyglqvbtdbirbjy` int DEFAULT NULL,\n `ugxvjneuhendxkldehssnswvwzdylmuy` int DEFAULT NULL,\n `fhquclmqaqrlypslsrrecerremhkmagv` int DEFAULT NULL,\n `kryrxukpdawicgwvgrhfiytkpxxjhwbz` int DEFAULT NULL,\n `lylprlfrbowqukrtmpftqfdfrobpjtpd` int DEFAULT NULL,\n `ejkkovooglfaarmmbxhsvvdzwtaymbzn` int DEFAULT NULL,\n `adpvljjshhgjtakoowvvblqrnedntxml` int DEFAULT NULL,\n `tdiputnshinybzgcxkxqwvckrgrwfzqi` int DEFAULT NULL,\n `rirycnqnmghrwfkvauyzlbaweasjycgb` int DEFAULT NULL,\n `rxqnugkrapsqkaicuzzrobdmexperhsg` int DEFAULT NULL,\n `prtqohwxhvcdvnzrztiqwcgfvlzspqxi` int DEFAULT NULL,\n `zdsgamvkmjvujdldhyowbdmcwhprlgkh` int DEFAULT NULL,\n `knkazyadoktfwxflzvwsvqulypsatyvq` int DEFAULT NULL,\n `txbilhhywtyhppzzgllpxktadhuvsbtm` int DEFAULT NULL,\n `timhrflnypimhmmtepdoboxkmmjyyjkp` int DEFAULT NULL,\n `cmxbglcjcsrybqabphvjwocfrvvrzhdj` int DEFAULT NULL,\n `ghbcvhpdvmnxtxojffpbrovcgpjczflj` int DEFAULT NULL,\n `tjumrllgcpwgjkzbyoacboouewrkzovi` int DEFAULT NULL,\n `stqrktzxdvshgkwrmyypishfwpednmqb` int DEFAULT NULL,\n `pyvdfxftlxhlxuohzciqmaqcqkzxorzh` int DEFAULT NULL,\n `yqtnajbwxhbtkqowuqkzupszybuiprgz` int DEFAULT NULL,\n `pdocgnnpcfmzanjrhfcesqqaxbncvtoh` int DEFAULT NULL,\n `evdqazorcumqppnmrpwvxrjrmjrchmvn` int DEFAULT NULL,\n `pndzoutlzxglstbumwiakhcuudyklput` int DEFAULT NULL,\n `uuhqbhkevuzaoepjujhlkbhvzqzihcbx` int DEFAULT NULL,\n `tpcrfhmirpzmosyuejhyesxvqhibphqj` int DEFAULT NULL,\n `mtkxsbetxcgqjskrkipqahgyoewypjeo` int DEFAULT NULL,\n `gcwhacgmburwvohkrmqabwvlwokcvofa` int DEFAULT NULL,\n `tdfpcjwxgffydhkoxmklofmfseawqver` int DEFAULT NULL,\n `klbpwvautujadsidxruwwuebikavgsjm` int DEFAULT NULL,\n `rsnjbsjawzdrdbmxqyotavghvecrwuzf` int DEFAULT NULL,\n `munhetkmecwoxcrgqgzmbyfveekrahea` int DEFAULT NULL,\n `swrajwkhbjdjvhlkmotgxzmdnaqkchbv` int DEFAULT NULL,\n `rjpyivcdgpofktkmqjpnddowtiwmbnxa` int DEFAULT NULL,\n `ahqqajddikzwfofnhcmjcurotsmjndxs` int DEFAULT NULL,\n `pirylitdwffjckrclajacrzntnnrkeyc` int DEFAULT NULL,\n `knpzxmwuqhjuzyqrpoviglxcqwggmggs` int DEFAULT NULL,\n `pbnxtfnolwdsczrgydpipjxdcovvfzbw` int DEFAULT NULL,\n `zhlqtsgfhcxrrwwdklsjclbvoummfymc` int DEFAULT NULL,\n `njgscsucvinhbqsjcdpnrhiptzedagob` int DEFAULT NULL,\n `dwhaqnwwpjuuuspcpvixhjiyzoopavph` int DEFAULT NULL,\n `jjuefwtdzzimzmckccvzalitpsbdcwey` int DEFAULT NULL,\n `zjsqhapuggizogicavspgwmgjteacikh` int DEFAULT NULL,\n `rkzenetqhhjsvmkpthjunipnmdszumms` int DEFAULT NULL,\n `ztksyxlahqwwsghfvhhegpfrnnmtdlmo` int DEFAULT NULL,\n `wphgbryzkzwhctwbuonidjikubvtrhef` int DEFAULT NULL,\n `rqlyratkvrohoohldreezklrwturzpmd` int DEFAULT NULL,\n `xpohsgnaqihunkyjtfsleilptensvzot` int DEFAULT NULL,\n `bfhaxsaxtbpdoallxakpsbvpqufrpnil` int DEFAULT NULL,\n `ncukfduamdecozmbhaecixpcifwlhlka` int DEFAULT NULL,\n `wxrsepbhdaweeqmrylsgnngctzpzbmit` int DEFAULT NULL,\n `mulifjdouebgdnlnziyvvomzhqltkkcb` int DEFAULT NULL,\n `zwgqwmxvjmcmtfpiviemnbedtbhrbzaf` int DEFAULT NULL,\n `qnccnaphvfenfvgycmfzciozultiglir` int DEFAULT NULL,\n PRIMARY KEY (`jipdcpzdxqvfoaowglzumftrwgyvucmk`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"lzbcpccmabxinidkpdxbjeyepkhowvqb\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["jipdcpzdxqvfoaowglzumftrwgyvucmk"],"columns":[{"name":"jipdcpzdxqvfoaowglzumftrwgyvucmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"enqcejoiudqbfopqtcimwqwuttrkvfkx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecqbpgudtbpciveheckdlzijuthcgygb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bztoeyocahuvkszpknuepyjdndjqcuvh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgqprdsgviwbciuocbdtafahwcgenlco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgbzfnzpvefytudioficdqdqpqnrztwr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbdeuhzieddcfeetkywkqobikbijxrmt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gcmfbzxcwsshnguzszdzlqalizjehvpe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdcmaatzauywzmmqhtfxybvblbzqhxpp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucdfunxanqimwsgipgkchdmkkxxjzlna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ennsltsutmuxxutovgtbxiogajhlbppd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzenhpzeeqxxltajxvucfzkyvzfkqcvm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmpefnsjwivizocvbvsmuyatamtiqthl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uazyqcgddtkzegbidtjgqpznhritpqdj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elswuuleqaeqqajxorznizrnchubggtp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xgbmatezwjfrwmxqpopkdzivpkifoojt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nnspybrlqrmpmdjnemxpyndmdygeuive","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntplbexpkcgycdgjhjjwwxgkqjbcyrhs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggedqntlishxnkknfiktxlggoeytnnkc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfsndneradadlmaggmnbhysczbjcqkfo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opaoremmbertbdlciolhmcrycgqlbubw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwsgovtelqrauvecikwpojulbamfjkjk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gefkccdxsabkiaqnatmgiczbcfqjhtvw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tfzraylfvwpzbxzaabwxfxckybhqegqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qnujbmsbwqxmjveqqcuwbuhegasnldav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dizfuneyfcustlcxhggqpumwseqgwggs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"extvhkqcwufgmduhjuhhmulmxmasxfkv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jhwcldrwygsszjsmdsvsmcnzattcnhyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyckxikzkxdhjqznduzzoycowtjzwomr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shnwsstncbzcmihmqecvjqiwctlcpmso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbooetpjmzqkutsbkhdzashocqibmomc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kgjhunuvggecgfsvavgwjyeebdohdzze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sasimxewtnxytqqbqvxqqgzutgmpugor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"glbudmvacwtlzejrvtfiyqiyqabcbvpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yaftopzhdnqqjpehvydapwwtokvclofs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhnbjdpqvibcscogmafiigysnmegbkyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"btkwybryxfhxeoybuwjxckhfcxmtfjdt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnofsuyqovlghiaowqhuohahpybxmpdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwwazgwulualhxexbabdqsbqhjlzdzkp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrferbdtizdrnhzzstqdonmqjcobcpcb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzzbtaohnuqwbiukompffuoibphmmbrl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dplqrxppndwxtpvrowwulgyssriwharu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfqfdypivmzvuwpigkolhoksxwydzaga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqadniffnwpvlzzpfvgedmhyvwyydrcb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfpaojaunkzhejdkwyhksmxajxrbhjki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxxbedqwcpzavxvrxvenznekfamywkmm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdnllacehrutegtxmoyglqvbtdbirbjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugxvjneuhendxkldehssnswvwzdylmuy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhquclmqaqrlypslsrrecerremhkmagv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kryrxukpdawicgwvgrhfiytkpxxjhwbz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lylprlfrbowqukrtmpftqfdfrobpjtpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejkkovooglfaarmmbxhsvvdzwtaymbzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"adpvljjshhgjtakoowvvblqrnedntxml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdiputnshinybzgcxkxqwvckrgrwfzqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rirycnqnmghrwfkvauyzlbaweasjycgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxqnugkrapsqkaicuzzrobdmexperhsg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prtqohwxhvcdvnzrztiqwcgfvlzspqxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdsgamvkmjvujdldhyowbdmcwhprlgkh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knkazyadoktfwxflzvwsvqulypsatyvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txbilhhywtyhppzzgllpxktadhuvsbtm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"timhrflnypimhmmtepdoboxkmmjyyjkp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmxbglcjcsrybqabphvjwocfrvvrzhdj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghbcvhpdvmnxtxojffpbrovcgpjczflj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjumrllgcpwgjkzbyoacboouewrkzovi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"stqrktzxdvshgkwrmyypishfwpednmqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pyvdfxftlxhlxuohzciqmaqcqkzxorzh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqtnajbwxhbtkqowuqkzupszybuiprgz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdocgnnpcfmzanjrhfcesqqaxbncvtoh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evdqazorcumqppnmrpwvxrjrmjrchmvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pndzoutlzxglstbumwiakhcuudyklput","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uuhqbhkevuzaoepjujhlkbhvzqzihcbx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpcrfhmirpzmosyuejhyesxvqhibphqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtkxsbetxcgqjskrkipqahgyoewypjeo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gcwhacgmburwvohkrmqabwvlwokcvofa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdfpcjwxgffydhkoxmklofmfseawqver","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"klbpwvautujadsidxruwwuebikavgsjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rsnjbsjawzdrdbmxqyotavghvecrwuzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"munhetkmecwoxcrgqgzmbyfveekrahea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swrajwkhbjdjvhlkmotgxzmdnaqkchbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjpyivcdgpofktkmqjpnddowtiwmbnxa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahqqajddikzwfofnhcmjcurotsmjndxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pirylitdwffjckrclajacrzntnnrkeyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knpzxmwuqhjuzyqrpoviglxcqwggmggs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbnxtfnolwdsczrgydpipjxdcovvfzbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhlqtsgfhcxrrwwdklsjclbvoummfymc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njgscsucvinhbqsjcdpnrhiptzedagob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwhaqnwwpjuuuspcpvixhjiyzoopavph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjuefwtdzzimzmckccvzalitpsbdcwey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjsqhapuggizogicavspgwmgjteacikh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkzenetqhhjsvmkpthjunipnmdszumms","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ztksyxlahqwwsghfvhhegpfrnnmtdlmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wphgbryzkzwhctwbuonidjikubvtrhef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqlyratkvrohoohldreezklrwturzpmd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpohsgnaqihunkyjtfsleilptensvzot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bfhaxsaxtbpdoallxakpsbvpqufrpnil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ncukfduamdecozmbhaecixpcifwlhlka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxrsepbhdaweeqmrylsgnngctzpzbmit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mulifjdouebgdnlnziyvvomzhqltkkcb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zwgqwmxvjmcmtfpiviemnbedtbhrbzaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qnccnaphvfenfvgycmfzciozultiglir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669467,"databaseName":"models_schema","ddl":"CREATE TABLE `models` (\n `id` int NOT NULL,\n `make_id` int DEFAULT NULL,\n `model` varchar(200) DEFAULT NULL,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"models\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["id"],"columns":[{"name":"id","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"make_id","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"model","jdbcType":12,"typeName":"VARCHAR","typeExpression":"VARCHAR","charsetName":"utf8mb4","length":200,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669472,"databaseName":"models_schema","ddl":"CREATE TABLE `models_random` (\n `id_random` int NOT NULL,\n `make_id_random` int DEFAULT NULL,\n `model_random` varchar(200) DEFAULT NULL,\n PRIMARY KEY (`id_random`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"models_random\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["id_random"],"columns":[{"name":"id_random","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"make_id_random","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"model_random","jdbcType":12,"typeName":"VARCHAR","typeExpression":"VARCHAR","charsetName":"utf8mb4","length":200,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669514,"databaseName":"models_schema","ddl":"CREATE TABLE `mrhwnmuraagswlujmtsibhxramzicgvh` (\n `xtluxxwmkmngdovvfrteprzirxzfmbjw` int NOT NULL,\n `piuqxezvukaamolrmetfgkajjzthxgtw` int DEFAULT NULL,\n `emucoeszifxauvhtbkfjujroguumrwsy` int DEFAULT NULL,\n `tnmxnygmtozarnsptdwcwinfnboiclap` int DEFAULT NULL,\n `lxobjtewcmnlxkfzeufbbnwwzfanxlma` int DEFAULT NULL,\n `tirzbbedbcxtrtwnynpjabizrjgotbto` int DEFAULT NULL,\n `yfvblhhkhlcqjfredplqnwgedslmthyc` int DEFAULT NULL,\n `ltiiajflmjmedfijqhspygefyfztzoyv` int DEFAULT NULL,\n `vceyraywvvchqndubkfkghgppavuhkkd` int DEFAULT NULL,\n `gfdlxfqxqibjgorogiktnsqsxfwakado` int DEFAULT NULL,\n `igtxpqtjnvvbltlhzwpaappwitfydngv` int DEFAULT NULL,\n `fhfkisnrilokscwinywjgsgcsfbazcdi` int DEFAULT NULL,\n `fykjptwelvfanwptnnthzdlrzlhtjohv` int DEFAULT NULL,\n `kuxfhpmuetwmwreqgffzocxflfaefkhz` int DEFAULT NULL,\n `hyalwnlbebsyuiemqhpkyiwbvjkyignv` int DEFAULT NULL,\n `fhyxmdxtuncvrzuzdgbspmxezxofahva` int DEFAULT NULL,\n `sjzooegvbuqtljbmbupyvvngshgayqbz` int DEFAULT NULL,\n `otxxdzlvlyqoasihmoehybwdrlbdxwfx` int DEFAULT NULL,\n `mrbendrtceafsdqhtmuaxajjcbglgkly` int DEFAULT NULL,\n `mcsaofckbiozyssgoevmhnugwhyrtlwm` int DEFAULT NULL,\n `zqsevgvnwkzlmziqrdtvsielssyypfml` int DEFAULT NULL,\n `dahgayxuuywflnttxfidzyeiipwyyxth` int DEFAULT NULL,\n `ipwmclxbjkdtukhbmbwntkhpggsbsfdj` int DEFAULT NULL,\n `xrxcbpyoxnzbxtgoxfelxyjbetupxrbf` int DEFAULT NULL,\n `xomqnobsirvpmcbhmsuqodhqqorhwqkl` int DEFAULT NULL,\n `ljbfkdtexvdmvqhftwjnzsffktwttfpo` int DEFAULT NULL,\n `akdahtakmzrmrakjvrafgiqxiocgrcsp` int DEFAULT NULL,\n `wbogjxunnhiqucyqbfxseqdwqzkkswcl` int DEFAULT NULL,\n `crftlavrpwwghbljekcmmtxhytairkqo` int DEFAULT NULL,\n `qyqdfsdsubkngbdorqpsfgpdjguowndn` int DEFAULT NULL,\n `bjywujuobdjdvbyieesqsayjtgsvdoiu` int DEFAULT NULL,\n `auyyfsdpxvctyvtvjufppnoftivvnlgq` int DEFAULT NULL,\n `duulaiybnpupzfwjfrbzubwzxjksvfsw` int DEFAULT NULL,\n `ygmaejopxbwvcbpblpthkfjnijwmomgc` int DEFAULT NULL,\n `covblkxbnmovjfpnkbwgjcjohfajzeon` int DEFAULT NULL,\n `pkjcpcgqrrmssaooanubvtmfumyhwgks` int DEFAULT NULL,\n `cmadbxccadehomqjctwcncdargcftqzw` int DEFAULT NULL,\n `twiuiqggbjsewdskhxygfbzfcbuablmv` int DEFAULT NULL,\n `hwxljbubxnxmuvsylbfrhjppqcyzyhps` int DEFAULT NULL,\n `fzbutfsnmbqrimchguqohwfgetimydam` int DEFAULT NULL,\n `lvaeabxfqequjaybdbjvnvtbjpkihgun` int DEFAULT NULL,\n `mprkbdephqfcmqjxyikqjyljfbhslarl` int DEFAULT NULL,\n `inofkwyegbpyukgbgmpzmfnnzawchkxl` int DEFAULT NULL,\n `wkijonnpjtzvdehgvyyoofrqwfdaineo` int DEFAULT NULL,\n `sqoftvhvhuqvhfayyhzbrqcrehgmumuw` int DEFAULT NULL,\n `qoxicvjfhukkjkpnlupyxdvdymsoovdq` int DEFAULT NULL,\n `gowtofcpgdetkvjudukrrfjgidkpjuro` int DEFAULT NULL,\n `xgsefhqtptyxiqhartzuuqloskcgdmyp` int DEFAULT NULL,\n `vhbxgteyhccdrpbbbulxuybrbnvlsgaz` int DEFAULT NULL,\n `mshtbnnanfjgscwpyourmknyfahwojhj` int DEFAULT NULL,\n `selmnidwdntduucyzqpcxqkeepyomvmk` int DEFAULT NULL,\n `qkmodbvtjciukdeihcuatfadvwuxbndl` int DEFAULT NULL,\n `jaqwunuhjrcivkjcxheshsupiuaabbzh` int DEFAULT NULL,\n `ppzulzlobrpmqdkqqyqxlgxylprhoknh` int DEFAULT NULL,\n `fqmzyjzrdhjmxdxtnjnoaeitakgpjiym` int DEFAULT NULL,\n `wvgompuxssswsuefugsrchxymzlrjepq` int DEFAULT NULL,\n `dlinfalqzxwccxrerfdsvoghnsbjufsl` int DEFAULT NULL,\n `rfikjzggmtkdnfhuxfzimuojamnvzlkv` int DEFAULT NULL,\n `mjjwulgymhtyqthfloskmedgdmgdsorz` int DEFAULT NULL,\n `vxoofwgliycowavbqemsiodgiyiooceu` int DEFAULT NULL,\n `umjxrndksdmxoycrpzcnibglcwwetkxq` int DEFAULT NULL,\n `pipoqnowldibocletbcutcbwmygilakn` int DEFAULT NULL,\n `juwvhnshgvacbwljyfhxeuyexafbvvto` int DEFAULT NULL,\n `vxbgmopnzdbsnopsirxpngdcbclwxtty` int DEFAULT NULL,\n `yghmmepiuedoneudgopdeqtnbggkvhoa` int DEFAULT NULL,\n `ylxtkkpszsbulvumsjhirrkookxpfuam` int DEFAULT NULL,\n `wssawvjohyidwzxfsfdxnjxprjhprzeb` int DEFAULT NULL,\n `lraphgiszbybpxhitjqvwjbpndzbhufb` int DEFAULT NULL,\n `ppkkzwluwtgopzkoumwyzsijwwidtxnm` int DEFAULT NULL,\n `fedudqjcjmvhzsmcaytvafplugqrdjtp` int DEFAULT NULL,\n `xxvtfgylyhcspqmwkxfrqacohlmnpkrh` int DEFAULT NULL,\n `jaeuaswnatuknqklkylomstyveegeoug` int DEFAULT NULL,\n `eqmnrzdqcwmuoxvpbzblftythzbykynm` int DEFAULT NULL,\n `aesccmyelfwrsxkicwvicvgxmggvcqyq` int DEFAULT NULL,\n `eyoptkvcvnztbpcbrrhkggxystffqhmi` int DEFAULT NULL,\n `zhztrlbdmvmqjfwmbfnryuzwotfcjsmb` int DEFAULT NULL,\n `hqrgxxigwcmorwdkbngkbvbvehhlgqee` int DEFAULT NULL,\n `amfedxcqmbgrjhyaglfmqbzhntbthmcl` int DEFAULT NULL,\n `cvmubnsnwfwdjsmehhylziqewzeoskkg` int DEFAULT NULL,\n `dgacqcspcnnzkdmchccxadythcakwtnh` int DEFAULT NULL,\n `pqfwcahhyzvjpiruusoykcfzroljjscp` int DEFAULT NULL,\n `zuopswbiguwscibkpjpumzyjikicbqsa` int DEFAULT NULL,\n `mxafjuazevsdphqymgxgwejbzbhiqztm` int DEFAULT NULL,\n `rqeseyetvhzhnokaqmjvqaektmqsusbr` int DEFAULT NULL,\n `wgvglurdexfpjdevmsasllhkfotrktqh` int DEFAULT NULL,\n `tqdnltggrzedshamebwkxkrnhaejypmo` int DEFAULT NULL,\n `tvysjgkokjpwinwfvfguhtaipibrgqtu` int DEFAULT NULL,\n `qolazdaocdnmhagaqwrzmpotacnplqqu` int DEFAULT NULL,\n `tahxlqxzzhmdclmrqqowbvnxbwsdqyex` int DEFAULT NULL,\n `iqntjaqpqbicozfeziokjhwtnyrszjit` int DEFAULT NULL,\n `zyucttxkqwmmnjvrtfqjdugkzdzfkkey` int DEFAULT NULL,\n `qdhhjdlhmeqhukajtgadsvmxvupyaikw` int DEFAULT NULL,\n `zrqbdndsjtyqllwasdyifewwrvvdtppk` int DEFAULT NULL,\n `hkhqxhprhrpoyvlmomeywiryxjlfsnpc` int DEFAULT NULL,\n `wifdyfyljpmyrydjvidpwljslzrqxwhd` int DEFAULT NULL,\n `ixcrrwjcrnhoprkpkutvxmasqsthomug` int DEFAULT NULL,\n `shsiiektljprdgtlitdxpjdfkxobhikv` int DEFAULT NULL,\n `ohvoqywseikujkqnhokbgrierhcifass` int DEFAULT NULL,\n `mivqbslmiyktekdxpufehespijafexbl` int DEFAULT NULL,\n `zpcklqzgyeaxpdytdqdjdflmfdsfqlhf` int DEFAULT NULL,\n PRIMARY KEY (`xtluxxwmkmngdovvfrteprzirxzfmbjw`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"mrhwnmuraagswlujmtsibhxramzicgvh\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["xtluxxwmkmngdovvfrteprzirxzfmbjw"],"columns":[{"name":"xtluxxwmkmngdovvfrteprzirxzfmbjw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"piuqxezvukaamolrmetfgkajjzthxgtw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"emucoeszifxauvhtbkfjujroguumrwsy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnmxnygmtozarnsptdwcwinfnboiclap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxobjtewcmnlxkfzeufbbnwwzfanxlma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tirzbbedbcxtrtwnynpjabizrjgotbto","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfvblhhkhlcqjfredplqnwgedslmthyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ltiiajflmjmedfijqhspygefyfztzoyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vceyraywvvchqndubkfkghgppavuhkkd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfdlxfqxqibjgorogiktnsqsxfwakado","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igtxpqtjnvvbltlhzwpaappwitfydngv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhfkisnrilokscwinywjgsgcsfbazcdi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fykjptwelvfanwptnnthzdlrzlhtjohv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kuxfhpmuetwmwreqgffzocxflfaefkhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hyalwnlbebsyuiemqhpkyiwbvjkyignv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhyxmdxtuncvrzuzdgbspmxezxofahva","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjzooegvbuqtljbmbupyvvngshgayqbz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otxxdzlvlyqoasihmoehybwdrlbdxwfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrbendrtceafsdqhtmuaxajjcbglgkly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcsaofckbiozyssgoevmhnugwhyrtlwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqsevgvnwkzlmziqrdtvsielssyypfml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dahgayxuuywflnttxfidzyeiipwyyxth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ipwmclxbjkdtukhbmbwntkhpggsbsfdj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrxcbpyoxnzbxtgoxfelxyjbetupxrbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xomqnobsirvpmcbhmsuqodhqqorhwqkl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljbfkdtexvdmvqhftwjnzsffktwttfpo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akdahtakmzrmrakjvrafgiqxiocgrcsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbogjxunnhiqucyqbfxseqdwqzkkswcl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crftlavrpwwghbljekcmmtxhytairkqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qyqdfsdsubkngbdorqpsfgpdjguowndn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjywujuobdjdvbyieesqsayjtgsvdoiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"auyyfsdpxvctyvtvjufppnoftivvnlgq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"duulaiybnpupzfwjfrbzubwzxjksvfsw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ygmaejopxbwvcbpblpthkfjnijwmomgc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"covblkxbnmovjfpnkbwgjcjohfajzeon","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkjcpcgqrrmssaooanubvtmfumyhwgks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmadbxccadehomqjctwcncdargcftqzw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twiuiqggbjsewdskhxygfbzfcbuablmv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwxljbubxnxmuvsylbfrhjppqcyzyhps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzbutfsnmbqrimchguqohwfgetimydam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvaeabxfqequjaybdbjvnvtbjpkihgun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mprkbdephqfcmqjxyikqjyljfbhslarl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"inofkwyegbpyukgbgmpzmfnnzawchkxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wkijonnpjtzvdehgvyyoofrqwfdaineo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sqoftvhvhuqvhfayyhzbrqcrehgmumuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qoxicvjfhukkjkpnlupyxdvdymsoovdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gowtofcpgdetkvjudukrrfjgidkpjuro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xgsefhqtptyxiqhartzuuqloskcgdmyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhbxgteyhccdrpbbbulxuybrbnvlsgaz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mshtbnnanfjgscwpyourmknyfahwojhj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"selmnidwdntduucyzqpcxqkeepyomvmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qkmodbvtjciukdeihcuatfadvwuxbndl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jaqwunuhjrcivkjcxheshsupiuaabbzh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ppzulzlobrpmqdkqqyqxlgxylprhoknh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqmzyjzrdhjmxdxtnjnoaeitakgpjiym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvgompuxssswsuefugsrchxymzlrjepq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlinfalqzxwccxrerfdsvoghnsbjufsl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfikjzggmtkdnfhuxfzimuojamnvzlkv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjjwulgymhtyqthfloskmedgdmgdsorz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxoofwgliycowavbqemsiodgiyiooceu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umjxrndksdmxoycrpzcnibglcwwetkxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pipoqnowldibocletbcutcbwmygilakn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"juwvhnshgvacbwljyfhxeuyexafbvvto","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxbgmopnzdbsnopsirxpngdcbclwxtty","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yghmmepiuedoneudgopdeqtnbggkvhoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ylxtkkpszsbulvumsjhirrkookxpfuam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wssawvjohyidwzxfsfdxnjxprjhprzeb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lraphgiszbybpxhitjqvwjbpndzbhufb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ppkkzwluwtgopzkoumwyzsijwwidtxnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fedudqjcjmvhzsmcaytvafplugqrdjtp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxvtfgylyhcspqmwkxfrqacohlmnpkrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jaeuaswnatuknqklkylomstyveegeoug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqmnrzdqcwmuoxvpbzblftythzbykynm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aesccmyelfwrsxkicwvicvgxmggvcqyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eyoptkvcvnztbpcbrrhkggxystffqhmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhztrlbdmvmqjfwmbfnryuzwotfcjsmb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqrgxxigwcmorwdkbngkbvbvehhlgqee","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amfedxcqmbgrjhyaglfmqbzhntbthmcl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvmubnsnwfwdjsmehhylziqewzeoskkg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgacqcspcnnzkdmchccxadythcakwtnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqfwcahhyzvjpiruusoykcfzroljjscp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zuopswbiguwscibkpjpumzyjikicbqsa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mxafjuazevsdphqymgxgwejbzbhiqztm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqeseyetvhzhnokaqmjvqaektmqsusbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgvglurdexfpjdevmsasllhkfotrktqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqdnltggrzedshamebwkxkrnhaejypmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvysjgkokjpwinwfvfguhtaipibrgqtu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qolazdaocdnmhagaqwrzmpotacnplqqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tahxlqxzzhmdclmrqqowbvnxbwsdqyex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqntjaqpqbicozfeziokjhwtnyrszjit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyucttxkqwmmnjvrtfqjdugkzdzfkkey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdhhjdlhmeqhukajtgadsvmxvupyaikw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zrqbdndsjtyqllwasdyifewwrvvdtppk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkhqxhprhrpoyvlmomeywiryxjlfsnpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wifdyfyljpmyrydjvidpwljslzrqxwhd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixcrrwjcrnhoprkpkutvxmasqsthomug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shsiiektljprdgtlitdxpjdfkxobhikv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohvoqywseikujkqnhokbgrierhcifass","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mivqbslmiyktekdxpufehespijafexbl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpcklqzgyeaxpdytdqdjdflmfdsfqlhf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669552,"databaseName":"models_schema","ddl":"CREATE TABLE `mrorqfumguckiypivdoraipjpawwjdfq` (\n `uupoucvktvbpruanfeivovahvfbwdjog` int NOT NULL,\n `epvrdgfdkaaphlyvmmeavwslbewoeodw` int DEFAULT NULL,\n `gfkgxqlanjodrwbadkgwytxrcrdbctxp` int DEFAULT NULL,\n `oqxmcozhdnrixwxtxdjdvfmxeshaupmm` int DEFAULT NULL,\n `grsovsylykjwfiqpcjyocsdqxjousgbl` int DEFAULT NULL,\n `fnjfqmtfcozqxfgriryzautjxphyumlg` int DEFAULT NULL,\n `worumbaocvzbgfuhhfniuuitpjuauhcv` int DEFAULT NULL,\n `dilqlkgdvwyhxmfzilnazxjnwkdgwgbv` int DEFAULT NULL,\n `bvioudhyqpjtizlrdgjjqkmkrcdushvg` int DEFAULT NULL,\n `fpmcprtehokmrwkitafszwpqxucxvqyb` int DEFAULT NULL,\n `zxdzisaqppjgqflmczgfbmmvhspewvsd` int DEFAULT NULL,\n `cedvkptrgwisayxyffrtsafngvoapgzm` int DEFAULT NULL,\n `vwtjzcifiazcrktetwwsoypifoabcvzn` int DEFAULT NULL,\n `zsqzqpbxbnunyfbzypvdnzdqfawvmooj` int DEFAULT NULL,\n `sbtqporhcesqpdofjrjbtyqabffoxadi` int DEFAULT NULL,\n `nzriukonyqvvchlfjxlvyvnwlwkakdcy` int DEFAULT NULL,\n `omghefiaktnblrroxkxozinuyxhczhcd` int DEFAULT NULL,\n `nvhvefegqdydmdtlmgictinxwswdcxsz` int DEFAULT NULL,\n `hstrnucekhnvppprcwppjoccykvrqesp` int DEFAULT NULL,\n `mspjfldoliixasklyjmvhqhehpdjxbaa` int DEFAULT NULL,\n `rcmxjnsdnhzxbhtueseyxlknhwfzvtex` int DEFAULT NULL,\n `wmfshgyjrtwkxmcwqjsiiknegmzmhyrc` int DEFAULT NULL,\n `tixcvbixtfixnrvvjoatfakxdlpxmezb` int DEFAULT NULL,\n `zmyxagdkfbfjxofcuoejfcidufpppbay` int DEFAULT NULL,\n `adziungjeqjpjapsqdxprgtdmpsvnsto` int DEFAULT NULL,\n `kocdpwvwlujfkpyyvvcllbdyqscpzllj` int DEFAULT NULL,\n `ucpkzhmaujnchawfzhhupycntudskaub` int DEFAULT NULL,\n `ljzfirncrhbykpwjyyhwwlqwzqmgzmwz` int DEFAULT NULL,\n `aozeqgzvpdnnmoonyarzmwzbcimlchck` int DEFAULT NULL,\n `xbzinadnnenkfgjclvqgnrcsylwuyeib` int DEFAULT NULL,\n `sktksldxocjexqzkfodzrkoxucskyuzo` int DEFAULT NULL,\n `hvmblzugvecuxwfubfkfyrrddncnvwic` int DEFAULT NULL,\n `fbwscolrdisxslfvumcekyeebzesdiog` int DEFAULT NULL,\n `tekxtqixmgzcwfidpfwnjxdrwhlrmdps` int DEFAULT NULL,\n `lgybrfslmbetlfbhsmkqmibvtgrhtdao` int DEFAULT NULL,\n `vgdighiegtitzqapqbwwitcwdjkpjelq` int DEFAULT NULL,\n `ypltavawmbuivcnhttochbpodaeskqdm` int DEFAULT NULL,\n `dbeaogcumbxyrjketurvlgkppfeldgah` int DEFAULT NULL,\n `ikyvbcjpzjqoewelkymnblgdihgbdpue` int DEFAULT NULL,\n `kgcjygajpkggoswkvewnlkliicsgjkvx` int DEFAULT NULL,\n `fchrylyxcpionxaurbcrjmqttrjzqjpw` int DEFAULT NULL,\n `rhqosozpkmzvvbeztinkdmlylblgsvgq` int DEFAULT NULL,\n `fjpkrsiwngztqddvuwpkkhmhvwrvhnki` int DEFAULT NULL,\n `sdiifemdavxljgnmcdncwkonmojnduky` int DEFAULT NULL,\n `jqtiyhbwkahktyrcqdpzqmpnlouqhbyr` int DEFAULT NULL,\n `mfvkncgiortmpforpdynazyzmffwdbiq` int DEFAULT NULL,\n `zakngcbmgpmzwowubfygnabkfgqfuajy` int DEFAULT NULL,\n `ubwkwubfqcyokvhtuyslwwxogilvqgjw` int DEFAULT NULL,\n `ywvoigvhximyiyswzundpjekeyxyxzrk` int DEFAULT NULL,\n `ohhhrfhsyzcaflbpmwzxfkahizlfddiw` int DEFAULT NULL,\n `ucrdzlkwmeoslsibgpizofxgbqnlemob` int DEFAULT NULL,\n `ovzmovsadtuidoaunpqaxmbmjptbwnha` int DEFAULT NULL,\n `kjeccgdibeklomivtalbafctboucbozu` int DEFAULT NULL,\n `wvptiivebqimhvnuixwvkqqowmnktjwh` int DEFAULT NULL,\n `yxbagpuomsybmgiyqkjepdfabsjooirj` int DEFAULT NULL,\n `sltytrbbkkouybqulphdtaheqekfrirb` int DEFAULT NULL,\n `uwevnbstiokbejzlirkjaemsmkhejtnv` int DEFAULT NULL,\n `elqxadsqhextfycqqpjxziaoakuourab` int DEFAULT NULL,\n `zhpmyclpcsjwqxiryklrlgnnccmlxcer` int DEFAULT NULL,\n `xjcwqyuezhrxgtsmbqewummecjhhduvc` int DEFAULT NULL,\n `maphwsfslhmjcaarryjyrlrxyuxmwgrc` int DEFAULT NULL,\n `dsmraqtkdjyibyewvehkgswkakhpxrql` int DEFAULT NULL,\n `denivgzkkattquftwwkptsypoplokouq` int DEFAULT NULL,\n `xsuoazqoeironshlgkskspzzwysjtbkk` int DEFAULT NULL,\n `mtxnoqgrefbhbktiftbeyqtfbxgoxsjz` int DEFAULT NULL,\n `aqensxwzbujizwbaikvahpqvmpwdsfgl` int DEFAULT NULL,\n `gpsmzgvbiwqzghoosbqyzrqtjrkianre` int DEFAULT NULL,\n `sgkanjokmmrcinzqzhxxawwtarpfbldu` int DEFAULT NULL,\n `qxekbexrpdnhmrwakpfdqnmbwkcssckd` int DEFAULT NULL,\n `icvyoyjsybbppkacmtplchxllmpttfpq` int DEFAULT NULL,\n `ocozbupsbddtwihqcqgxisepjdiduwwc` int DEFAULT NULL,\n `ghjdgqqrothuaydgobvjbpinkmhqgdfm` int DEFAULT NULL,\n `nujvcdlwjqoroysmvrvxwtcyurfbiwxn` int DEFAULT NULL,\n `hywdebiokwvfoprdxmsgodtkoqafhmlz` int DEFAULT NULL,\n `raahfwlmgwkaehesckmsjwgcbhezpxrh` int DEFAULT NULL,\n `edprqwiwbkpofhpcwjtfxtfbyetsapqx` int DEFAULT NULL,\n `ntsheevngltyctdmdbgnjqmxcahbrrda` int DEFAULT NULL,\n `tvvgvwcavfetdxmrgdslawayvfavttyi` int DEFAULT NULL,\n `vajpcfjphhxtfgvoufzhzqisvlzytvwc` int DEFAULT NULL,\n `ihacqqdnktjfrkrnjqyxnewkifxnuyvn` int DEFAULT NULL,\n `zhndnafvyoqdvnictusqlabnczurwfsl` int DEFAULT NULL,\n `tytdyjcevjqbhciuqfftdnddizauicpv` int DEFAULT NULL,\n `uzmazvwjisetjmkdxsywqmkeowbisnba` int DEFAULT NULL,\n `wxvpiufmcpznjgymgbrvcvypivikogdd` int DEFAULT NULL,\n `dsacqgipzzetgpiohxwmqxjrlbvzzutm` int DEFAULT NULL,\n `kwoozrirxhqjpbswxlpotyexexhojbny` int DEFAULT NULL,\n `pnyprrqtqfjfbnznjlnahlhnuwbjtabf` int DEFAULT NULL,\n `quemscaiqqxhiguiwrhgwosgsihcmgmr` int DEFAULT NULL,\n `wbiqbtrggjjtgrlgggrohquqjihgrcli` int DEFAULT NULL,\n `tbobjkdwiyobesgthwsmthbrwjqrkzsm` int DEFAULT NULL,\n `swazqthgiblrdusbbpegqokfjctohdpi` int DEFAULT NULL,\n `swxkjkfcwwisyygaeqnuuyxvekaiiezw` int DEFAULT NULL,\n `iqkmogfsgxaybmykcfosawgfeimhaoci` int DEFAULT NULL,\n `bvnfrsdhhqaexsevbkvdcwhfqcflmqnn` int DEFAULT NULL,\n `llqwrdheehxpmconsclucspumsmrqgjk` int DEFAULT NULL,\n `yexavsscxbudvuxtbxylmselgfgyixif` int DEFAULT NULL,\n `sjwlknlrdrgkjlvnawoldkkqoftkgzzl` int DEFAULT NULL,\n `rzuberelwkogucnohdmyyxbylkpohrde` int DEFAULT NULL,\n `pflukjzrksugwmxruohhdmcyzkvyidmi` int DEFAULT NULL,\n `icbyxmwcdxgdaxaqbeyyztbatvxbnerc` int DEFAULT NULL,\n PRIMARY KEY (`uupoucvktvbpruanfeivovahvfbwdjog`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"mrorqfumguckiypivdoraipjpawwjdfq\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["uupoucvktvbpruanfeivovahvfbwdjog"],"columns":[{"name":"uupoucvktvbpruanfeivovahvfbwdjog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"epvrdgfdkaaphlyvmmeavwslbewoeodw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfkgxqlanjodrwbadkgwytxrcrdbctxp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqxmcozhdnrixwxtxdjdvfmxeshaupmm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grsovsylykjwfiqpcjyocsdqxjousgbl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnjfqmtfcozqxfgriryzautjxphyumlg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"worumbaocvzbgfuhhfniuuitpjuauhcv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dilqlkgdvwyhxmfzilnazxjnwkdgwgbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvioudhyqpjtizlrdgjjqkmkrcdushvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpmcprtehokmrwkitafszwpqxucxvqyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxdzisaqppjgqflmczgfbmmvhspewvsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cedvkptrgwisayxyffrtsafngvoapgzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vwtjzcifiazcrktetwwsoypifoabcvzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zsqzqpbxbnunyfbzypvdnzdqfawvmooj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbtqporhcesqpdofjrjbtyqabffoxadi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzriukonyqvvchlfjxlvyvnwlwkakdcy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omghefiaktnblrroxkxozinuyxhczhcd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvhvefegqdydmdtlmgictinxwswdcxsz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hstrnucekhnvppprcwppjoccykvrqesp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mspjfldoliixasklyjmvhqhehpdjxbaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcmxjnsdnhzxbhtueseyxlknhwfzvtex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmfshgyjrtwkxmcwqjsiiknegmzmhyrc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tixcvbixtfixnrvvjoatfakxdlpxmezb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmyxagdkfbfjxofcuoejfcidufpppbay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"adziungjeqjpjapsqdxprgtdmpsvnsto","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kocdpwvwlujfkpyyvvcllbdyqscpzllj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucpkzhmaujnchawfzhhupycntudskaub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljzfirncrhbykpwjyyhwwlqwzqmgzmwz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aozeqgzvpdnnmoonyarzmwzbcimlchck","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbzinadnnenkfgjclvqgnrcsylwuyeib","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sktksldxocjexqzkfodzrkoxucskyuzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hvmblzugvecuxwfubfkfyrrddncnvwic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbwscolrdisxslfvumcekyeebzesdiog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tekxtqixmgzcwfidpfwnjxdrwhlrmdps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgybrfslmbetlfbhsmkqmibvtgrhtdao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgdighiegtitzqapqbwwitcwdjkpjelq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypltavawmbuivcnhttochbpodaeskqdm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbeaogcumbxyrjketurvlgkppfeldgah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikyvbcjpzjqoewelkymnblgdihgbdpue","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kgcjygajpkggoswkvewnlkliicsgjkvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fchrylyxcpionxaurbcrjmqttrjzqjpw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhqosozpkmzvvbeztinkdmlylblgsvgq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjpkrsiwngztqddvuwpkkhmhvwrvhnki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdiifemdavxljgnmcdncwkonmojnduky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jqtiyhbwkahktyrcqdpzqmpnlouqhbyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfvkncgiortmpforpdynazyzmffwdbiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zakngcbmgpmzwowubfygnabkfgqfuajy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubwkwubfqcyokvhtuyslwwxogilvqgjw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywvoigvhximyiyswzundpjekeyxyxzrk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohhhrfhsyzcaflbpmwzxfkahizlfddiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucrdzlkwmeoslsibgpizofxgbqnlemob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovzmovsadtuidoaunpqaxmbmjptbwnha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjeccgdibeklomivtalbafctboucbozu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvptiivebqimhvnuixwvkqqowmnktjwh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxbagpuomsybmgiyqkjepdfabsjooirj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sltytrbbkkouybqulphdtaheqekfrirb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwevnbstiokbejzlirkjaemsmkhejtnv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elqxadsqhextfycqqpjxziaoakuourab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhpmyclpcsjwqxiryklrlgnnccmlxcer","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjcwqyuezhrxgtsmbqewummecjhhduvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"maphwsfslhmjcaarryjyrlrxyuxmwgrc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsmraqtkdjyibyewvehkgswkakhpxrql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"denivgzkkattquftwwkptsypoplokouq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsuoazqoeironshlgkskspzzwysjtbkk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtxnoqgrefbhbktiftbeyqtfbxgoxsjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqensxwzbujizwbaikvahpqvmpwdsfgl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gpsmzgvbiwqzghoosbqyzrqtjrkianre","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sgkanjokmmrcinzqzhxxawwtarpfbldu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxekbexrpdnhmrwakpfdqnmbwkcssckd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icvyoyjsybbppkacmtplchxllmpttfpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocozbupsbddtwihqcqgxisepjdiduwwc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghjdgqqrothuaydgobvjbpinkmhqgdfm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nujvcdlwjqoroysmvrvxwtcyurfbiwxn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hywdebiokwvfoprdxmsgodtkoqafhmlz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"raahfwlmgwkaehesckmsjwgcbhezpxrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edprqwiwbkpofhpcwjtfxtfbyetsapqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntsheevngltyctdmdbgnjqmxcahbrrda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvvgvwcavfetdxmrgdslawayvfavttyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vajpcfjphhxtfgvoufzhzqisvlzytvwc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ihacqqdnktjfrkrnjqyxnewkifxnuyvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhndnafvyoqdvnictusqlabnczurwfsl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tytdyjcevjqbhciuqfftdnddizauicpv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzmazvwjisetjmkdxsywqmkeowbisnba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxvpiufmcpznjgymgbrvcvypivikogdd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsacqgipzzetgpiohxwmqxjrlbvzzutm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwoozrirxhqjpbswxlpotyexexhojbny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnyprrqtqfjfbnznjlnahlhnuwbjtabf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"quemscaiqqxhiguiwrhgwosgsihcmgmr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbiqbtrggjjtgrlgggrohquqjihgrcli","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbobjkdwiyobesgthwsmthbrwjqrkzsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swazqthgiblrdusbbpegqokfjctohdpi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swxkjkfcwwisyygaeqnuuyxvekaiiezw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqkmogfsgxaybmykcfosawgfeimhaoci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvnfrsdhhqaexsevbkvdcwhfqcflmqnn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llqwrdheehxpmconsclucspumsmrqgjk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yexavsscxbudvuxtbxylmselgfgyixif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjwlknlrdrgkjlvnawoldkkqoftkgzzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzuberelwkogucnohdmyyxbylkpohrde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pflukjzrksugwmxruohhdmcyzkvyidmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icbyxmwcdxgdaxaqbeyyztbatvxbnerc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669589,"databaseName":"models_schema","ddl":"CREATE TABLE `mwyzswmpkowqnbzyvfhkpmvnjfrgnraz` (\n `hannjaxilomphxnkxgjoeknpykgtmusd` int NOT NULL,\n `wwjuagezguzxdjinfnafisnvxemmravv` int DEFAULT NULL,\n `ibiyxzvpquldszqqifksyvmezyvjsvcj` int DEFAULT NULL,\n `zkbribagonzehpsbsmidwajfbakuvmmg` int DEFAULT NULL,\n `goukmgdilegrlpnyhncguyvdrnegxqvb` int DEFAULT NULL,\n `ftkguokihzlolfvzmwjwqzwwurvhafxs` int DEFAULT NULL,\n `fmdlzbdndgtkmzbwoxygmnrexiizzjay` int DEFAULT NULL,\n `lmjsjxqsycunkfkyggoocgdinauzntks` int DEFAULT NULL,\n `pfbgqdmfbxibxtixufaslqnatfximbpa` int DEFAULT NULL,\n `daeyggxkbfnloaxqpanygkyurzdczqdb` int DEFAULT NULL,\n `zdmelpfzlcwqbniddnijkphlavbeodlt` int DEFAULT NULL,\n `gthpebbtfrwswnwvdvtbqggicwvmkhga` int DEFAULT NULL,\n `orufdnuzuoppxjiwmfflqeksxqicwjqe` int DEFAULT NULL,\n `owcbdmtkdndxemkbnjztcqsepkolmyfc` int DEFAULT NULL,\n `ctpuyxjaasmftyjchbbgidqlpaktpnyg` int DEFAULT NULL,\n `liwinphmuwvgfvbfgfmnkmuirckxvbed` int DEFAULT NULL,\n `ljmzjbzekwjsievgfyeegadsepqcqemq` int DEFAULT NULL,\n `ssqmjsplmldgqhlkhhynaihollnpiuha` int DEFAULT NULL,\n `eyhmyofktudxbdqdtdfzljgyhbfyshhk` int DEFAULT NULL,\n `awzcjmksfdnakssicwyzjbndhnmhbntf` int DEFAULT NULL,\n `ylffdzcoyicmddcmnbmwvykjlcvqqgna` int DEFAULT NULL,\n `djcbxysreqwlzicninavvwqyudliuudb` int DEFAULT NULL,\n `syorgqxbeboxjilfcqinoasgvzpquhdi` int DEFAULT NULL,\n `yufadxypptzhpuvjaitbkdcnvuifyaew` int DEFAULT NULL,\n `omipaquvfripiipsiyiternrpboglqwr` int DEFAULT NULL,\n `cmxfbvienpaapkqreanmwigjujvmlemc` int DEFAULT NULL,\n `dwcxiwoeyqqvlviedegvlqdfrksrnade` int DEFAULT NULL,\n `vpmuineylkgbxozgkewpxzseuqpqeega` int DEFAULT NULL,\n `fkrpdyxeiajwvesklfxlfdrtdkqvdqkg` int DEFAULT NULL,\n `squjndcwirdkpaluvpfthbnnderlqzac` int DEFAULT NULL,\n `oinobdkzuafrrgovgtppvqruxdaurspj` int DEFAULT NULL,\n `diykmipxbmzsmfmsyqkmrteagfkzbmpj` int DEFAULT NULL,\n `kxkrakjhufcgjjohiwtkpgdkupzumapl` int DEFAULT NULL,\n `pndlvoziauwpvrdwwhlzkvdcfajxezvf` int DEFAULT NULL,\n `obzsxryicirekexlelucbaokuyxfwbsb` int DEFAULT NULL,\n `sshotwnrdyejzfnyigwvtiauqoqfslek` int DEFAULT NULL,\n `hzevnvzrfelpydzgnmawbydhbdsjaqje` int DEFAULT NULL,\n `hetuewigyqijdqitjwzccesqldfnwlqj` int DEFAULT NULL,\n `liexbkrjirdmkpccmixabqmvwwubfusv` int DEFAULT NULL,\n `rmoiqitbfvctalyrdzcdfsmtalmkqumq` int DEFAULT NULL,\n `bbkjjawwmumavayghlmggohtezrbmddm` int DEFAULT NULL,\n `glohcmxuinshowwownhledjfqzfvikgd` int DEFAULT NULL,\n `rotwmqmjezohkkzqrxdwypwbljbzkjwq` int DEFAULT NULL,\n `kfuufegzhvwrpwfcqjxbavixpinrqgur` int DEFAULT NULL,\n `jdbynxrpkniwgwhjnwjtggyzonfeartr` int DEFAULT NULL,\n `fxzhhzgsxpbmiwhmjgkghvqdrmughxnf` int DEFAULT NULL,\n `xgusslwjoxdpxijbcmyryfnwovtgviqd` int DEFAULT NULL,\n `okjcjqnimfzdmiymjmsmyerxhnxpgkid` int DEFAULT NULL,\n `mygrjfpvkdyzrptluxxhdnmhadsvkdyn` int DEFAULT NULL,\n `hwlchdbalhcwpmmjtxfwajjoymxvtqcz` int DEFAULT NULL,\n `rblaraoxqtaxkkivdzvdngfoowfvnojp` int DEFAULT NULL,\n `xhrgdfhcvkkcrmlvyfoqwemrwmzqpjce` int DEFAULT NULL,\n `ypacffkqkkzxhlltawlstllyzgsbsmvk` int DEFAULT NULL,\n `vtfcscnwyelzexreobdmjvqfwhccwtdc` int DEFAULT NULL,\n `ytrzbgvgonmrjqozcerritkexgpzudjv` int DEFAULT NULL,\n `ofidncvxrqunyzjbuxwctsmwxqyudzef` int DEFAULT NULL,\n `prwkbwmnhhdybikpveayxczskejstwuh` int DEFAULT NULL,\n `xjotyuuurbconotdvcdkivodaswoahuo` int DEFAULT NULL,\n `tmzcoeohhknurgisxbqfwjxfwemeugbu` int DEFAULT NULL,\n `jgpjghejnlvxeuffejfrkyyldvcidjih` int DEFAULT NULL,\n `dvazxtmwwpdthtiqwleqvueuqrvxydbt` int DEFAULT NULL,\n `qdlpeiwnpwtywvvlabuwgcqmbrnixpcd` int DEFAULT NULL,\n `yjtrneotrlhttmzlsslokqplaskrshpe` int DEFAULT NULL,\n `qktejrumfybgeagprxpoowrecorrgobt` int DEFAULT NULL,\n `eqbflrdhfjcrzbwaxlprsayawngaxorr` int DEFAULT NULL,\n `lvegqtnjhtsuqtjlhkfgxigjydunyite` int DEFAULT NULL,\n `qspwcgymdeibwfhmkflcputwkvrqptiq` int DEFAULT NULL,\n `sgxgkmhnbuywfkeeasccdgdcodiditao` int DEFAULT NULL,\n `swfreirgboddirzcbzmirjtstpfqzkof` int DEFAULT NULL,\n `ncmyjbycjyrvfghfsupkttzfowtoonbu` int DEFAULT NULL,\n `qubmnelqovvluavtgdxphetalxfevwlt` int DEFAULT NULL,\n `gahipimdhhgnxofytqorujsurzefifgv` int DEFAULT NULL,\n `nswqvzzvxnmuixwagdretqfnenkavspf` int DEFAULT NULL,\n `iciqafayzwhgscuajwphwbfcbgqedplc` int DEFAULT NULL,\n `uzrcufpsbanqyihvuswnxsdehaphoiii` int DEFAULT NULL,\n `qvsujoxspetsgmjpxzeggkvrnjepngxd` int DEFAULT NULL,\n `eietridnbpfzwhrnsxbknqglpdkiubxl` int DEFAULT NULL,\n `oqpxowcoqinhwmyrpotbougdxmfctxsc` int DEFAULT NULL,\n `ordtronddhgrndzkmdyxnxkyaqusrzlk` int DEFAULT NULL,\n `ahccvlalajqkydwwuonrwmllgfgxmpra` int DEFAULT NULL,\n `mgqsjkusxluuudyfosfmydkplhchxwhl` int DEFAULT NULL,\n `gzwhputdrdtxbinydezdwpwmosngltuc` int DEFAULT NULL,\n `ehuerrvztkkliwwpconwoefwhhimbeys` int DEFAULT NULL,\n `qpxkeutgsrrjfdjwdzdbpvdsxegmdmau` int DEFAULT NULL,\n `ssckkvlkwfxwowjfmjwhnfglkzcrojuf` int DEFAULT NULL,\n `qndhncgdfggybihbdktqhrchumybtjie` int DEFAULT NULL,\n `vxmrejiwnmdnckqakpgcsvymmtwhtqyv` int DEFAULT NULL,\n `opnnmexlepwxoilyzfphhquzjjllhvuw` int DEFAULT NULL,\n `blayucedfrzuumjbbxuthlmxqyaytprq` int DEFAULT NULL,\n `adyqlvmzpeipyvfbjaivypsgpmjepnhp` int DEFAULT NULL,\n `yiddzszskszugnkcpfmyiwpvpgcvcyvp` int DEFAULT NULL,\n `zkivrrywnxhxlazbwxektajclogtsoyb` int DEFAULT NULL,\n `abtijtqqvnbctncksvgtytomyhxrzxad` int DEFAULT NULL,\n `ilzuzakhksdxdwwmnfyqzxyvwxiwisyf` int DEFAULT NULL,\n `ahmlrbwzcjplhecqvzyilwordwgaeieu` int DEFAULT NULL,\n `eyxjjotnabetvtchzkropvgrwexcdjow` int DEFAULT NULL,\n `zaqfccgukxkhcybxqjyaoeardleavhgx` int DEFAULT NULL,\n `knftffxltijwluhuscjklqnkmsomtlym` int DEFAULT NULL,\n `ynscipmghbsavmkzmhhwcurjjwsjheqk` int DEFAULT NULL,\n `auaspjprfbsuxjpquiwqkbiwrlttwfab` int DEFAULT NULL,\n PRIMARY KEY (`hannjaxilomphxnkxgjoeknpykgtmusd`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"mwyzswmpkowqnbzyvfhkpmvnjfrgnraz\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["hannjaxilomphxnkxgjoeknpykgtmusd"],"columns":[{"name":"hannjaxilomphxnkxgjoeknpykgtmusd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"wwjuagezguzxdjinfnafisnvxemmravv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibiyxzvpquldszqqifksyvmezyvjsvcj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zkbribagonzehpsbsmidwajfbakuvmmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"goukmgdilegrlpnyhncguyvdrnegxqvb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftkguokihzlolfvzmwjwqzwwurvhafxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmdlzbdndgtkmzbwoxygmnrexiizzjay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmjsjxqsycunkfkyggoocgdinauzntks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pfbgqdmfbxibxtixufaslqnatfximbpa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"daeyggxkbfnloaxqpanygkyurzdczqdb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdmelpfzlcwqbniddnijkphlavbeodlt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gthpebbtfrwswnwvdvtbqggicwvmkhga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orufdnuzuoppxjiwmfflqeksxqicwjqe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owcbdmtkdndxemkbnjztcqsepkolmyfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctpuyxjaasmftyjchbbgidqlpaktpnyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"liwinphmuwvgfvbfgfmnkmuirckxvbed","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljmzjbzekwjsievgfyeegadsepqcqemq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ssqmjsplmldgqhlkhhynaihollnpiuha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eyhmyofktudxbdqdtdfzljgyhbfyshhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awzcjmksfdnakssicwyzjbndhnmhbntf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ylffdzcoyicmddcmnbmwvykjlcvqqgna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"djcbxysreqwlzicninavvwqyudliuudb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"syorgqxbeboxjilfcqinoasgvzpquhdi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yufadxypptzhpuvjaitbkdcnvuifyaew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omipaquvfripiipsiyiternrpboglqwr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmxfbvienpaapkqreanmwigjujvmlemc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwcxiwoeyqqvlviedegvlqdfrksrnade","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpmuineylkgbxozgkewpxzseuqpqeega","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fkrpdyxeiajwvesklfxlfdrtdkqvdqkg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"squjndcwirdkpaluvpfthbnnderlqzac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oinobdkzuafrrgovgtppvqruxdaurspj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"diykmipxbmzsmfmsyqkmrteagfkzbmpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxkrakjhufcgjjohiwtkpgdkupzumapl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pndlvoziauwpvrdwwhlzkvdcfajxezvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obzsxryicirekexlelucbaokuyxfwbsb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sshotwnrdyejzfnyigwvtiauqoqfslek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hzevnvzrfelpydzgnmawbydhbdsjaqje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hetuewigyqijdqitjwzccesqldfnwlqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"liexbkrjirdmkpccmixabqmvwwubfusv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rmoiqitbfvctalyrdzcdfsmtalmkqumq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbkjjawwmumavayghlmggohtezrbmddm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"glohcmxuinshowwownhledjfqzfvikgd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rotwmqmjezohkkzqrxdwypwbljbzkjwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfuufegzhvwrpwfcqjxbavixpinrqgur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdbynxrpkniwgwhjnwjtggyzonfeartr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxzhhzgsxpbmiwhmjgkghvqdrmughxnf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xgusslwjoxdpxijbcmyryfnwovtgviqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okjcjqnimfzdmiymjmsmyerxhnxpgkid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mygrjfpvkdyzrptluxxhdnmhadsvkdyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwlchdbalhcwpmmjtxfwajjoymxvtqcz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rblaraoxqtaxkkivdzvdngfoowfvnojp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xhrgdfhcvkkcrmlvyfoqwemrwmzqpjce","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypacffkqkkzxhlltawlstllyzgsbsmvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtfcscnwyelzexreobdmjvqfwhccwtdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytrzbgvgonmrjqozcerritkexgpzudjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofidncvxrqunyzjbuxwctsmwxqyudzef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prwkbwmnhhdybikpveayxczskejstwuh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjotyuuurbconotdvcdkivodaswoahuo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmzcoeohhknurgisxbqfwjxfwemeugbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgpjghejnlvxeuffejfrkyyldvcidjih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvazxtmwwpdthtiqwleqvueuqrvxydbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdlpeiwnpwtywvvlabuwgcqmbrnixpcd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjtrneotrlhttmzlsslokqplaskrshpe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qktejrumfybgeagprxpoowrecorrgobt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqbflrdhfjcrzbwaxlprsayawngaxorr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvegqtnjhtsuqtjlhkfgxigjydunyite","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qspwcgymdeibwfhmkflcputwkvrqptiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sgxgkmhnbuywfkeeasccdgdcodiditao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swfreirgboddirzcbzmirjtstpfqzkof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ncmyjbycjyrvfghfsupkttzfowtoonbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qubmnelqovvluavtgdxphetalxfevwlt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gahipimdhhgnxofytqorujsurzefifgv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nswqvzzvxnmuixwagdretqfnenkavspf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iciqafayzwhgscuajwphwbfcbgqedplc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzrcufpsbanqyihvuswnxsdehaphoiii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvsujoxspetsgmjpxzeggkvrnjepngxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eietridnbpfzwhrnsxbknqglpdkiubxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqpxowcoqinhwmyrpotbougdxmfctxsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ordtronddhgrndzkmdyxnxkyaqusrzlk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahccvlalajqkydwwuonrwmllgfgxmpra","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mgqsjkusxluuudyfosfmydkplhchxwhl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzwhputdrdtxbinydezdwpwmosngltuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehuerrvztkkliwwpconwoefwhhimbeys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qpxkeutgsrrjfdjwdzdbpvdsxegmdmau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ssckkvlkwfxwowjfmjwhnfglkzcrojuf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qndhncgdfggybihbdktqhrchumybtjie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxmrejiwnmdnckqakpgcsvymmtwhtqyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opnnmexlepwxoilyzfphhquzjjllhvuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blayucedfrzuumjbbxuthlmxqyaytprq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"adyqlvmzpeipyvfbjaivypsgpmjepnhp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yiddzszskszugnkcpfmyiwpvpgcvcyvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zkivrrywnxhxlazbwxektajclogtsoyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abtijtqqvnbctncksvgtytomyhxrzxad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ilzuzakhksdxdwwmnfyqzxyvwxiwisyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahmlrbwzcjplhecqvzyilwordwgaeieu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eyxjjotnabetvtchzkropvgrwexcdjow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zaqfccgukxkhcybxqjyaoeardleavhgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knftffxltijwluhuscjklqnkmsomtlym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynscipmghbsavmkzmhhwcurjjwsjheqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"auaspjprfbsuxjpquiwqkbiwrlttwfab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669634,"databaseName":"models_schema","ddl":"CREATE TABLE `mxslcfjftoitkvwkjafdifwuxfdmfsym` (\n `nuvngcwhgaawiilkhytpztjnohcmjjdp` int NOT NULL,\n `dxbsgzcehkmzonyrnqmpxgwrobrbmcge` int DEFAULT NULL,\n `jjfbbrdtcqrvkrebhvtmcdzwpgiuwcwy` int DEFAULT NULL,\n `kghxtoukyzxyollotrookzrahaiejyru` int DEFAULT NULL,\n `uegwqpjbipzjdsaosmgsrtzneewibsme` int DEFAULT NULL,\n `nhwafywenzlzjkgekngmskqkbdzmxncw` int DEFAULT NULL,\n `ywolpfmintbyrqesfumucdqedoalrzrj` int DEFAULT NULL,\n `lchqauuolebdeylpmslyebraojddreki` int DEFAULT NULL,\n `ohnyynhcuzoebvkujplqcxdlpltdlmdk` int DEFAULT NULL,\n `usssvkfycolvmzapqxmmfsfrwofqifxt` int DEFAULT NULL,\n `exezvanzzryqeczrguioqayubgdhtodg` int DEFAULT NULL,\n `qtbeoogcpgihfoimvaywrihlnavwhytj` int DEFAULT NULL,\n `swkvpfqnthlswfyqbkyujdnkbdipxpaa` int DEFAULT NULL,\n `kmahsxfmwsgslzvexbiywkhgsrbaxnnc` int DEFAULT NULL,\n `cqmyibpgyrmmourpekckonrpvzobfjlx` int DEFAULT NULL,\n `kiwbpdvqqfpxzrpfppfqanfmulmsegnc` int DEFAULT NULL,\n `lwxqgjucnzicsitasgyxigcyebonihgm` int DEFAULT NULL,\n `xvsbgnzmgnkjtixmkikrqojpdhuasevl` int DEFAULT NULL,\n `plxvaulgengvyplntcdjcmkcukvdblpf` int DEFAULT NULL,\n `pnlehutiikrowrfrtkqfaobmksfmiyuh` int DEFAULT NULL,\n `yjulyvxipzzduaauoiujvyftsjxgpbks` int DEFAULT NULL,\n `jhroxmfapttykxtlxvnzkbojzsgbwzhx` int DEFAULT NULL,\n `prqdbujyzillfwpwyhtdunpgjcajobjx` int DEFAULT NULL,\n `oofcrjrofyzekyyzusuchxpeyjejgnyn` int DEFAULT NULL,\n `sidjlrtrurmouxadrgpemniklzouwyts` int DEFAULT NULL,\n `tjzesthoridizeqjbxukjjkqwiotgvpa` int DEFAULT NULL,\n `zcqkglawkrhtlxnwwjjpcibdehhpruyt` int DEFAULT NULL,\n `nyywbkhahigbggxmwpwplgztiaawvdtf` int DEFAULT NULL,\n `rwramejoqsxkxqfxltsxytjbfmzmenkx` int DEFAULT NULL,\n `fbpycncnklyhhdyiqknqielqqigsofrt` int DEFAULT NULL,\n `ciehsuvhrbhcewhjomwkfynypaivxqdo` int DEFAULT NULL,\n `zyostfrihrnscvdmhifuyawwtpaadnqw` int DEFAULT NULL,\n `ujcbwzqydirbvpuoynxcfvdkloqmtajh` int DEFAULT NULL,\n `leptcorflpprqkwbwtsodhltoomsothv` int DEFAULT NULL,\n `mzsuvhivbnuiuorgdrlkqtykceltiybi` int DEFAULT NULL,\n `qmkedfrdclsczfsrkbcviedwurpyyfcz` int DEFAULT NULL,\n `lgvyrkgitxdzjthyajxlyelprkilflum` int DEFAULT NULL,\n `zclxbagmqlrrwufqpfnjhutoikxpafin` int DEFAULT NULL,\n `dmsmjyufyegzbxtmifwolouxbozyqffc` int DEFAULT NULL,\n `tpavuychpjdfytzveccssymhslkamchs` int DEFAULT NULL,\n `fcxhikkzovpvkavnvzqalxeyhcfmhhad` int DEFAULT NULL,\n `mydtpmecxtbelwseovujvfrnzqiuhnad` int DEFAULT NULL,\n `rqgauuxhhldceriedcgsjykypmpygnai` int DEFAULT NULL,\n `jppstccuysqepwdjmakshjppqosbduvr` int DEFAULT NULL,\n `uxhktneokjxwetvyubrtpkcuejhfmtqw` int DEFAULT NULL,\n `ogdljpyhqiuswmozmpwuoxeynbecraiw` int DEFAULT NULL,\n `amlltdighnxkdxmvjyjgtthkqhkwfevw` int DEFAULT NULL,\n `cqrqnceyravqdtcerrhivvkeszcirqwu` int DEFAULT NULL,\n `ixutsvimwdychkbttoteymyafykagknb` int DEFAULT NULL,\n `rfelgfeduwoduiseqqgfvbmdsbcanrkr` int DEFAULT NULL,\n `rjehitwwfgklnsxwtqednzhdmrjieexl` int DEFAULT NULL,\n `mqggzvloquyerqhiyrpztrkkpssfrisi` int DEFAULT NULL,\n `ialmtwyjvblrlpzkujoefnepgvmqqoku` int DEFAULT NULL,\n `gwsosxvalmopwgcabmramvsuqvtgxjop` int DEFAULT NULL,\n `mzfqozddqivqfmqphsffltfwcbsbsssd` int DEFAULT NULL,\n `elqshdfovwuzfarrlxiiqymplaloqjvg` int DEFAULT NULL,\n `oshltoihpxzhkdjyfwahonyxgsxsoyvo` int DEFAULT NULL,\n `bykrcgewfrtyaejbkpuhulghaqocyxre` int DEFAULT NULL,\n `iuhfadzkhoixbnyecnzxlycpselqdnhm` int DEFAULT NULL,\n `itaexwpanbcmhjowofplbcszexphoimd` int DEFAULT NULL,\n `aklufponbtqcvircmbwuustxklaoomks` int DEFAULT NULL,\n `ticzjfrrjawwjfhxobbxgwgwjzxzredf` int DEFAULT NULL,\n `njrlnppoouaxzlcfjkcbskbcxryumpyn` int DEFAULT NULL,\n `fhezmwwosbtaisabjvtcvrpquhbmwonk` int DEFAULT NULL,\n `lyslcgkaapnecadvedtslmkbrtnyvzzz` int DEFAULT NULL,\n `qqidayjrsxtxxmcvqamzroswhxgegwkh` int DEFAULT NULL,\n `prqhejsgpwiuemeajsqwihndthqzdrmh` int DEFAULT NULL,\n `tduytsierwazkvirwhbddpxnvsqfwcrr` int DEFAULT NULL,\n `lgoomgfwcmenluhviuhlvtjfhmzulvff` int DEFAULT NULL,\n `akfcaawknlidimeptdieddzlixcavpkw` int DEFAULT NULL,\n `xhibxbkyxzhnkcguinosnedzszrsmxib` int DEFAULT NULL,\n `nrrzbqcsyupvtseqdpzirsznopnwttit` int DEFAULT NULL,\n `gmjjyqcyxntnlsthjmxgcgdxbbhwfusx` int DEFAULT NULL,\n `qhokgjydfoxfbxtmtrzxwgdvqstteaiu` int DEFAULT NULL,\n `ubecjjpruxszmycbyoszkimysejhhrxl` int DEFAULT NULL,\n `kqfgknoyypenidhmkgagzwhlqudsbdzs` int DEFAULT NULL,\n `nwljavssfmeoceqociwxywvobpsvfeei` int DEFAULT NULL,\n `nzdjemxuvcedwuoswxnlnichfdguqjob` int DEFAULT NULL,\n `aznvvkemvysuhrktpctwuafzkoltwrgn` int DEFAULT NULL,\n `rqvvuadwyaajwofghcrslhucomvfkehr` int DEFAULT NULL,\n `ifvmlyihoadzxpwwqpapjhaaeqrefwjk` int DEFAULT NULL,\n `ejvqreiweanaykezgxqjalxhbigxtedm` int DEFAULT NULL,\n `lhtxzwjczckxojtvtkkqobmloahkqoqf` int DEFAULT NULL,\n `pjyjrhwaipcdoibahpgjaltxihtgueue` int DEFAULT NULL,\n `lttagjrzeortqindlgvlyifwkuhmuhrl` int DEFAULT NULL,\n `dwjvjgvzfxihzfkwozbgzhfierevevin` int DEFAULT NULL,\n `ffszdogsbawqauvikcqkhvrvfzcmjstb` int DEFAULT NULL,\n `tqtshojrjkghflacpjixehdnakdcbcbn` int DEFAULT NULL,\n `fzprlrdwoblzrcwxeoimsucdwkhzncnj` int DEFAULT NULL,\n `inxccbdpgrchfocxaqbflmfckercfktj` int DEFAULT NULL,\n `iaefgidtdxacdvkwfgwatqcmicwcdisk` int DEFAULT NULL,\n `vumdwuxbvzqexlktnhgsnfkfdqfryktu` int DEFAULT NULL,\n `nynuzbiaxstwnbwzpcdfajmluqbnenha` int DEFAULT NULL,\n `vsoeycqrjolpccupshfvejpbexcfsfya` int DEFAULT NULL,\n `kphobtxejltqjwaiymctcvrxajwvehtv` int DEFAULT NULL,\n `kfualxqovgwiisuuavawzvcifgrhnmrc` int DEFAULT NULL,\n `proqhnkbkobruwyktmhjappjeqlyqgfu` int DEFAULT NULL,\n `kqkynokqeempqeaxkrtqmmcwbghpqskc` int DEFAULT NULL,\n `wrqyqpupixbgysqfdsjngriqzbbwifzx` int DEFAULT NULL,\n `acnqjeencsrnfzflsekctyblxwsehxmr` int DEFAULT NULL,\n PRIMARY KEY (`nuvngcwhgaawiilkhytpztjnohcmjjdp`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"mxslcfjftoitkvwkjafdifwuxfdmfsym\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["nuvngcwhgaawiilkhytpztjnohcmjjdp"],"columns":[{"name":"nuvngcwhgaawiilkhytpztjnohcmjjdp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"dxbsgzcehkmzonyrnqmpxgwrobrbmcge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjfbbrdtcqrvkrebhvtmcdzwpgiuwcwy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kghxtoukyzxyollotrookzrahaiejyru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uegwqpjbipzjdsaosmgsrtzneewibsme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhwafywenzlzjkgekngmskqkbdzmxncw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywolpfmintbyrqesfumucdqedoalrzrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lchqauuolebdeylpmslyebraojddreki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohnyynhcuzoebvkujplqcxdlpltdlmdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"usssvkfycolvmzapqxmmfsfrwofqifxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exezvanzzryqeczrguioqayubgdhtodg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtbeoogcpgihfoimvaywrihlnavwhytj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swkvpfqnthlswfyqbkyujdnkbdipxpaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmahsxfmwsgslzvexbiywkhgsrbaxnnc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqmyibpgyrmmourpekckonrpvzobfjlx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kiwbpdvqqfpxzrpfppfqanfmulmsegnc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwxqgjucnzicsitasgyxigcyebonihgm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvsbgnzmgnkjtixmkikrqojpdhuasevl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"plxvaulgengvyplntcdjcmkcukvdblpf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnlehutiikrowrfrtkqfaobmksfmiyuh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjulyvxipzzduaauoiujvyftsjxgpbks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jhroxmfapttykxtlxvnzkbojzsgbwzhx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prqdbujyzillfwpwyhtdunpgjcajobjx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oofcrjrofyzekyyzusuchxpeyjejgnyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sidjlrtrurmouxadrgpemniklzouwyts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjzesthoridizeqjbxukjjkqwiotgvpa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zcqkglawkrhtlxnwwjjpcibdehhpruyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nyywbkhahigbggxmwpwplgztiaawvdtf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rwramejoqsxkxqfxltsxytjbfmzmenkx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbpycncnklyhhdyiqknqielqqigsofrt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ciehsuvhrbhcewhjomwkfynypaivxqdo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyostfrihrnscvdmhifuyawwtpaadnqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujcbwzqydirbvpuoynxcfvdkloqmtajh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"leptcorflpprqkwbwtsodhltoomsothv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzsuvhivbnuiuorgdrlkqtykceltiybi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmkedfrdclsczfsrkbcviedwurpyyfcz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgvyrkgitxdzjthyajxlyelprkilflum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zclxbagmqlrrwufqpfnjhutoikxpafin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dmsmjyufyegzbxtmifwolouxbozyqffc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpavuychpjdfytzveccssymhslkamchs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcxhikkzovpvkavnvzqalxeyhcfmhhad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mydtpmecxtbelwseovujvfrnzqiuhnad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqgauuxhhldceriedcgsjykypmpygnai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jppstccuysqepwdjmakshjppqosbduvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxhktneokjxwetvyubrtpkcuejhfmtqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ogdljpyhqiuswmozmpwuoxeynbecraiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amlltdighnxkdxmvjyjgtthkqhkwfevw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqrqnceyravqdtcerrhivvkeszcirqwu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixutsvimwdychkbttoteymyafykagknb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfelgfeduwoduiseqqgfvbmdsbcanrkr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjehitwwfgklnsxwtqednzhdmrjieexl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqggzvloquyerqhiyrpztrkkpssfrisi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ialmtwyjvblrlpzkujoefnepgvmqqoku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gwsosxvalmopwgcabmramvsuqvtgxjop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzfqozddqivqfmqphsffltfwcbsbsssd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elqshdfovwuzfarrlxiiqymplaloqjvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oshltoihpxzhkdjyfwahonyxgsxsoyvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bykrcgewfrtyaejbkpuhulghaqocyxre","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iuhfadzkhoixbnyecnzxlycpselqdnhm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itaexwpanbcmhjowofplbcszexphoimd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aklufponbtqcvircmbwuustxklaoomks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ticzjfrrjawwjfhxobbxgwgwjzxzredf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njrlnppoouaxzlcfjkcbskbcxryumpyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhezmwwosbtaisabjvtcvrpquhbmwonk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lyslcgkaapnecadvedtslmkbrtnyvzzz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqidayjrsxtxxmcvqamzroswhxgegwkh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prqhejsgpwiuemeajsqwihndthqzdrmh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tduytsierwazkvirwhbddpxnvsqfwcrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgoomgfwcmenluhviuhlvtjfhmzulvff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akfcaawknlidimeptdieddzlixcavpkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xhibxbkyxzhnkcguinosnedzszrsmxib","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nrrzbqcsyupvtseqdpzirsznopnwttit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmjjyqcyxntnlsthjmxgcgdxbbhwfusx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhokgjydfoxfbxtmtrzxwgdvqstteaiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubecjjpruxszmycbyoszkimysejhhrxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqfgknoyypenidhmkgagzwhlqudsbdzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nwljavssfmeoceqociwxywvobpsvfeei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzdjemxuvcedwuoswxnlnichfdguqjob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aznvvkemvysuhrktpctwuafzkoltwrgn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqvvuadwyaajwofghcrslhucomvfkehr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifvmlyihoadzxpwwqpapjhaaeqrefwjk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejvqreiweanaykezgxqjalxhbigxtedm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhtxzwjczckxojtvtkkqobmloahkqoqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjyjrhwaipcdoibahpgjaltxihtgueue","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lttagjrzeortqindlgvlyifwkuhmuhrl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwjvjgvzfxihzfkwozbgzhfierevevin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffszdogsbawqauvikcqkhvrvfzcmjstb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqtshojrjkghflacpjixehdnakdcbcbn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzprlrdwoblzrcwxeoimsucdwkhzncnj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"inxccbdpgrchfocxaqbflmfckercfktj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iaefgidtdxacdvkwfgwatqcmicwcdisk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vumdwuxbvzqexlktnhgsnfkfdqfryktu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nynuzbiaxstwnbwzpcdfajmluqbnenha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsoeycqrjolpccupshfvejpbexcfsfya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kphobtxejltqjwaiymctcvrxajwvehtv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfualxqovgwiisuuavawzvcifgrhnmrc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"proqhnkbkobruwyktmhjappjeqlyqgfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqkynokqeempqeaxkrtqmmcwbghpqskc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wrqyqpupixbgysqfdsjngriqzbbwifzx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acnqjeencsrnfzflsekctyblxwsehxmr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669673,"databaseName":"models_schema","ddl":"CREATE TABLE `mykafjtkvhgjskiyabehpblifngdkhwu` (\n `chsmtctijsbiaabuuijhjpjzzjmtmshf` int NOT NULL,\n `fnneojctkmblijneaohxcnsqrmtrsdle` int DEFAULT NULL,\n `ghjhnhlhqhlgquubosagxccmoauvwuxh` int DEFAULT NULL,\n `vqhfahoswbvwuyibytnpvjulsvjzxgpk` int DEFAULT NULL,\n `rucwegavkoybrnzxgobsdiyzasvcazue` int DEFAULT NULL,\n `jaeanqoinfmabdoevzoskqdyvjzmzbow` int DEFAULT NULL,\n `pgdpvbecfgcecfyalfeioavdmzjmrktd` int DEFAULT NULL,\n `geezcmsqxrxdeihubpemyprffsiisxfy` int DEFAULT NULL,\n `gtyxrsitzfaezauxgonfwzjbjtzqpfgt` int DEFAULT NULL,\n `jwklshufpgdiyqwjqybuswwqeujklqez` int DEFAULT NULL,\n `aghdnubfhrdncijwjboaykztuhdaxvxb` int DEFAULT NULL,\n `yjiabykshzpsxjluxppnjqdezldawrxk` int DEFAULT NULL,\n `pdcqnxnpkdqcvseadpervstiiixdhvhb` int DEFAULT NULL,\n `narcxxcbbgqfwuzypmajagvhupcowwwk` int DEFAULT NULL,\n `pgzytfkalkedukptoxrlvtlomtoeycke` int DEFAULT NULL,\n `ofzmramvotqbdccokwmwtxglbhrehzmh` int DEFAULT NULL,\n `qdwqspfvjtuezxbtastssgkfzizfffah` int DEFAULT NULL,\n `fcmqqujwqrnsuxaimhjrucopjqtgvalx` int DEFAULT NULL,\n `cfggsmzdwstcdvpexoszitwhqwnifdii` int DEFAULT NULL,\n `lcheijcxckaapjgfffjrhnewhfinlmxr` int DEFAULT NULL,\n `srjiemtjrpegmquebgtvgxtiskcnhdlo` int DEFAULT NULL,\n `xyizfwwnanxtdppguxmziuvzpyektxub` int DEFAULT NULL,\n `ajywckmbkpkdldarewwykqwnavidcaih` int DEFAULT NULL,\n `khgngtmvmbxzahzcqqrufeaqozqvmxws` int DEFAULT NULL,\n `nmqnfbvdesxdavvbxmmjbqzdolvcijqz` int DEFAULT NULL,\n `aupstvmvdbmijanjeymeqkjsirrxrvhp` int DEFAULT NULL,\n `xcvjuxycmyewwoybpsqqkbbxytbumafy` int DEFAULT NULL,\n `fnoxcskuwqzgocajdrodcjuoevzvmvfi` int DEFAULT NULL,\n `jhddkolmxrmuoosekvzyrbnjaubsitzh` int DEFAULT NULL,\n `yxhfbsykacjrvrnirmgdkzoknintpcjs` int DEFAULT NULL,\n `swroknlaihmjxzdvtzsgbdrfvkoqnljy` int DEFAULT NULL,\n `mpvlbwgweuamuhtoyydjodpoayzgszfo` int DEFAULT NULL,\n `iltjrlmlsuxgpdxkbmmqzkqpqljfnesn` int DEFAULT NULL,\n `falctyqmlovwdebsupxgmlzqpipghhnh` int DEFAULT NULL,\n `laqtehahialrhjpnfolpxoqfeciojtdm` int DEFAULT NULL,\n `uhdutovhdbwjlwruabgdtkkdutypgcug` int DEFAULT NULL,\n `ymfbeyqqufeutfnsptjvygapakzdwhen` int DEFAULT NULL,\n `pqqjphmiyinxrqmzozwxluhkudyluemz` int DEFAULT NULL,\n `qxclwryqqyfbuhyobdnyuaosdfcuyhla` int DEFAULT NULL,\n `xamfjdwcvzkijtyrhqrubtatxcpnflbn` int DEFAULT NULL,\n `tstkjmjcejeetapbpslkwqdmvletrzpa` int DEFAULT NULL,\n `mpnkmcentjddrxtttxjonleaafpnsifr` int DEFAULT NULL,\n `wxyixoteqldlxuggdebvuqhcoyhjuavh` int DEFAULT NULL,\n `msywiwqhlxiovsfgotvocsqzhgsiwvaq` int DEFAULT NULL,\n `ehssrhedhdarivkcbdvgujrbantroxhp` int DEFAULT NULL,\n `bmrtwvqpnjpyiejjkmnfpvwyxsuhkbzx` int DEFAULT NULL,\n `ispietaajjtueryrazxyqboctzfoyreu` int DEFAULT NULL,\n `wucngcmnalafivwbpfiqtzodzetqmrks` int DEFAULT NULL,\n `vogaxkecdoyxmldsjynjwtqojieqgfni` int DEFAULT NULL,\n `ftfvvgflnjznmfcqivknwzlbqwurlizu` int DEFAULT NULL,\n `qbwcgjppvizbrzvbantufqlrhnoalder` int DEFAULT NULL,\n `efpwuffvufbumitazafoegtbrjluxkyg` int DEFAULT NULL,\n `ztzbcpjgephewbzfumrraplyyoflgxus` int DEFAULT NULL,\n `rzmagczmbfzsttlzshofadjzkhswaaol` int DEFAULT NULL,\n `pngnissauayymnodbaamnkfmfpccpuzb` int DEFAULT NULL,\n `lcccaezeoqfzjrzxbwtuizhhxbfixtqr` int DEFAULT NULL,\n `lmpewidfxmwvqoawuixcmwhoahcltesq` int DEFAULT NULL,\n `kscezgrrtibsyrasbolakseiilbafist` int DEFAULT NULL,\n `shjjokpkgocrxhriuylkiadccyzeobkj` int DEFAULT NULL,\n `yyanbjsqblnzeqdxcvjsayvrixdxhcrv` int DEFAULT NULL,\n `ovcxztswpwgkuekvxrsirrqpllbvhhzh` int DEFAULT NULL,\n `gdfluzifenefpxsstwsqstikohpuuond` int DEFAULT NULL,\n `dvkpynjaifygtimrxsxjdfatiddcrnji` int DEFAULT NULL,\n `uqbksusrpttnavrvdztdidwftxvaxvdv` int DEFAULT NULL,\n `qclchekvzzjhwmuneeldqytnovnucmxp` int DEFAULT NULL,\n `pnwgcolepuaksggxleconanrkbudmvya` int DEFAULT NULL,\n `ymdklpdovuttedjebfmjdxpogroicqzo` int DEFAULT NULL,\n `dxqpxinyffnxriojktgzqoifcifrrjir` int DEFAULT NULL,\n `nxercswwmusuhytszjvpkmtlymtdovdq` int DEFAULT NULL,\n `rdsvuaxeuplzivrxbmxnpfbehtgxddgt` int DEFAULT NULL,\n `gnjdrldwiaavoitzrndhhwsenjaecwmc` int DEFAULT NULL,\n `otnoltwjlmhohjdpfurityjzdopgpcbw` int DEFAULT NULL,\n `fcbdqdlgvejiwbnqdrdfjbyodpyejyvs` int DEFAULT NULL,\n `azkrwsliifssupzhmbgotaokssfytvxa` int DEFAULT NULL,\n `vipzczhdtxkhfkkbislvdbbvgnlwhxyy` int DEFAULT NULL,\n `lovtmcunuvdjdexgimnyohgzlqzutbnx` int DEFAULT NULL,\n `cwfgnyazydwdspikyvjafqswagckiplm` int DEFAULT NULL,\n `fmlrnmuboywmmlapbblmbvsfwaidcdty` int DEFAULT NULL,\n `zyxdxikdfdyqedvfakusycrzkmnufcfb` int DEFAULT NULL,\n `kqrwyxfwknwvjxgidfksjgsopmzvqkto` int DEFAULT NULL,\n `unsvandbpweinvxmpyvpprzytxwpfbcv` int DEFAULT NULL,\n `tzhbwwvurjdkpvexgudgsxknpmogrwrs` int DEFAULT NULL,\n `ycitctudgjiyvncuhbmpqrcftfpleahv` int DEFAULT NULL,\n `xevtrfqpwbcjfoiudavxzweycelzwplm` int DEFAULT NULL,\n `zlmtercfdbhbjkiytfvpnpfcjqcgzuze` int DEFAULT NULL,\n `fbtfdugyyrzyoedzuiuwunefabfeugzx` int DEFAULT NULL,\n `ksdvowcfouobbtldpijyfsviyewohaml` int DEFAULT NULL,\n `ypkihbmyobjlaxxlbvcrjwtwzdfdqmlu` int DEFAULT NULL,\n `hscxypwjmlgasdgpidedudkgxzkvbcem` int DEFAULT NULL,\n `djlkymgongsrytflibycwbuidlhsmwwc` int DEFAULT NULL,\n `auadvwugnqorsecmkduhmdcyzttjdtgj` int DEFAULT NULL,\n `jxabpzuviqscwogqdcryvteoxlsilwmk` int DEFAULT NULL,\n `pklqtllmwilffcrwegyhatwbfpsionps` int DEFAULT NULL,\n `zruqqsggrmibfttcxqvcdcgmoswikvux` int DEFAULT NULL,\n `iwdijobquemqnvwmxycvxmynebpjpumz` int DEFAULT NULL,\n `knxghnkkbcbwteqtbficbyquqitharyx` int DEFAULT NULL,\n `rzgbifuneionddchzjivilbeezyvapmk` int DEFAULT NULL,\n `knhtsflpdcfqmaidxsawhzrhtbstesmk` int DEFAULT NULL,\n `uagljziqpssypipnakltqdhghtztkzuq` int DEFAULT NULL,\n `zmxlnnqvcyesifkznnjwxftfmuuaaqdu` int DEFAULT NULL,\n PRIMARY KEY (`chsmtctijsbiaabuuijhjpjzzjmtmshf`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"mykafjtkvhgjskiyabehpblifngdkhwu\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["chsmtctijsbiaabuuijhjpjzzjmtmshf"],"columns":[{"name":"chsmtctijsbiaabuuijhjpjzzjmtmshf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"fnneojctkmblijneaohxcnsqrmtrsdle","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghjhnhlhqhlgquubosagxccmoauvwuxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqhfahoswbvwuyibytnpvjulsvjzxgpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rucwegavkoybrnzxgobsdiyzasvcazue","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jaeanqoinfmabdoevzoskqdyvjzmzbow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgdpvbecfgcecfyalfeioavdmzjmrktd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"geezcmsqxrxdeihubpemyprffsiisxfy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtyxrsitzfaezauxgonfwzjbjtzqpfgt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwklshufpgdiyqwjqybuswwqeujklqez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aghdnubfhrdncijwjboaykztuhdaxvxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjiabykshzpsxjluxppnjqdezldawrxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdcqnxnpkdqcvseadpervstiiixdhvhb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"narcxxcbbgqfwuzypmajagvhupcowwwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgzytfkalkedukptoxrlvtlomtoeycke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofzmramvotqbdccokwmwtxglbhrehzmh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdwqspfvjtuezxbtastssgkfzizfffah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcmqqujwqrnsuxaimhjrucopjqtgvalx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfggsmzdwstcdvpexoszitwhqwnifdii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lcheijcxckaapjgfffjrhnewhfinlmxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"srjiemtjrpegmquebgtvgxtiskcnhdlo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyizfwwnanxtdppguxmziuvzpyektxub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajywckmbkpkdldarewwykqwnavidcaih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khgngtmvmbxzahzcqqrufeaqozqvmxws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmqnfbvdesxdavvbxmmjbqzdolvcijqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aupstvmvdbmijanjeymeqkjsirrxrvhp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xcvjuxycmyewwoybpsqqkbbxytbumafy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnoxcskuwqzgocajdrodcjuoevzvmvfi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jhddkolmxrmuoosekvzyrbnjaubsitzh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxhfbsykacjrvrnirmgdkzoknintpcjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swroknlaihmjxzdvtzsgbdrfvkoqnljy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpvlbwgweuamuhtoyydjodpoayzgszfo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iltjrlmlsuxgpdxkbmmqzkqpqljfnesn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"falctyqmlovwdebsupxgmlzqpipghhnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"laqtehahialrhjpnfolpxoqfeciojtdm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhdutovhdbwjlwruabgdtkkdutypgcug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ymfbeyqqufeutfnsptjvygapakzdwhen","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqqjphmiyinxrqmzozwxluhkudyluemz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxclwryqqyfbuhyobdnyuaosdfcuyhla","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xamfjdwcvzkijtyrhqrubtatxcpnflbn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tstkjmjcejeetapbpslkwqdmvletrzpa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpnkmcentjddrxtttxjonleaafpnsifr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxyixoteqldlxuggdebvuqhcoyhjuavh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"msywiwqhlxiovsfgotvocsqzhgsiwvaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehssrhedhdarivkcbdvgujrbantroxhp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bmrtwvqpnjpyiejjkmnfpvwyxsuhkbzx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ispietaajjtueryrazxyqboctzfoyreu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wucngcmnalafivwbpfiqtzodzetqmrks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vogaxkecdoyxmldsjynjwtqojieqgfni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftfvvgflnjznmfcqivknwzlbqwurlizu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qbwcgjppvizbrzvbantufqlrhnoalder","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"efpwuffvufbumitazafoegtbrjluxkyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ztzbcpjgephewbzfumrraplyyoflgxus","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzmagczmbfzsttlzshofadjzkhswaaol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pngnissauayymnodbaamnkfmfpccpuzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lcccaezeoqfzjrzxbwtuizhhxbfixtqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmpewidfxmwvqoawuixcmwhoahcltesq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kscezgrrtibsyrasbolakseiilbafist","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shjjokpkgocrxhriuylkiadccyzeobkj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yyanbjsqblnzeqdxcvjsayvrixdxhcrv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovcxztswpwgkuekvxrsirrqpllbvhhzh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdfluzifenefpxsstwsqstikohpuuond","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvkpynjaifygtimrxsxjdfatiddcrnji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqbksusrpttnavrvdztdidwftxvaxvdv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qclchekvzzjhwmuneeldqytnovnucmxp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnwgcolepuaksggxleconanrkbudmvya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ymdklpdovuttedjebfmjdxpogroicqzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxqpxinyffnxriojktgzqoifcifrrjir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxercswwmusuhytszjvpkmtlymtdovdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdsvuaxeuplzivrxbmxnpfbehtgxddgt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnjdrldwiaavoitzrndhhwsenjaecwmc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otnoltwjlmhohjdpfurityjzdopgpcbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcbdqdlgvejiwbnqdrdfjbyodpyejyvs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azkrwsliifssupzhmbgotaokssfytvxa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vipzczhdtxkhfkkbislvdbbvgnlwhxyy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lovtmcunuvdjdexgimnyohgzlqzutbnx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwfgnyazydwdspikyvjafqswagckiplm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmlrnmuboywmmlapbblmbvsfwaidcdty","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyxdxikdfdyqedvfakusycrzkmnufcfb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqrwyxfwknwvjxgidfksjgsopmzvqkto","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unsvandbpweinvxmpyvpprzytxwpfbcv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tzhbwwvurjdkpvexgudgsxknpmogrwrs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ycitctudgjiyvncuhbmpqrcftfpleahv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xevtrfqpwbcjfoiudavxzweycelzwplm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zlmtercfdbhbjkiytfvpnpfcjqcgzuze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbtfdugyyrzyoedzuiuwunefabfeugzx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ksdvowcfouobbtldpijyfsviyewohaml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypkihbmyobjlaxxlbvcrjwtwzdfdqmlu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hscxypwjmlgasdgpidedudkgxzkvbcem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"djlkymgongsrytflibycwbuidlhsmwwc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"auadvwugnqorsecmkduhmdcyzttjdtgj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jxabpzuviqscwogqdcryvteoxlsilwmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pklqtllmwilffcrwegyhatwbfpsionps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zruqqsggrmibfttcxqvcdcgmoswikvux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwdijobquemqnvwmxycvxmynebpjpumz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knxghnkkbcbwteqtbficbyquqitharyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzgbifuneionddchzjivilbeezyvapmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knhtsflpdcfqmaidxsawhzrhtbstesmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uagljziqpssypipnakltqdhghtztkzuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmxlnnqvcyesifkznnjwxftfmuuaaqdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669706,"databaseName":"models_schema","ddl":"CREATE TABLE `nfqrzhysvyortaitkelwhsffveslnthx` (\n `dflqskdirhackweyrdlgnbxtxmwnmjxu` int NOT NULL,\n `goncslwvscpoqdxeksxmlvplwbjhhicl` int DEFAULT NULL,\n `havhcaxflzeciaykaezatjjeglvduheo` int DEFAULT NULL,\n `flnursnwcxyvjrrygosbslaitoqqyhxn` int DEFAULT NULL,\n `wgfzrbbxljpmiuntbtfiosqnpzwpjont` int DEFAULT NULL,\n `zodfkotljmpiufduyfkcyjvofgbgjnjb` int DEFAULT NULL,\n `qnbdkwrmlccylwbumxyyymztqwfqfrpa` int DEFAULT NULL,\n `jrilebmuidkywqmdcciqvebsjcbmyqmm` int DEFAULT NULL,\n `cmegkhixgauygayrdnoltdjoopsrubws` int DEFAULT NULL,\n `zayrqdykxvzudtycndvxeodxbvhelvfx` int DEFAULT NULL,\n `acyvkfebtuscjilxnurzsqwjdqzidwqs` int DEFAULT NULL,\n `dcmfdawpjyqpxygiivdcbwkivkyquucv` int DEFAULT NULL,\n `nxenfgtuabnxzhzzygikfnggnqtblfhz` int DEFAULT NULL,\n `tkqefdzrntfoahrqcsonqnavejvnwmmn` int DEFAULT NULL,\n `xfpevsdnecrvjnbptdpttmifhphwyxqo` int DEFAULT NULL,\n `giqlbizclmkyvdweitcdsjcmqlpvrqjr` int DEFAULT NULL,\n `stpsblarituovqefqdyleulfymiojuau` int DEFAULT NULL,\n `dstpoigpctmkzewrampcjkfzricbstxb` int DEFAULT NULL,\n `azwgeokokgthibxmqibegvxxfxpgaaca` int DEFAULT NULL,\n `zualkxqsgjhjsoobcfjtpypjrbkvqrub` int DEFAULT NULL,\n `iqumqigtiissdkhaaslwrylezsenrsep` int DEFAULT NULL,\n `ycesgwfgovjmnrbhufjiyewprrugfbia` int DEFAULT NULL,\n `avwvqfjqvrwafnwpseyjuckjwsvhhrbc` int DEFAULT NULL,\n `qorqvxrdndbvdwjjgixajkefjbzcrpon` int DEFAULT NULL,\n `ryorsemjwecelkfupqvzphuxhdwwppbu` int DEFAULT NULL,\n `zgatjkdijscytxxcrsrcjfvdmqznkyts` int DEFAULT NULL,\n `ctmxpsomxwzqvxshyenxhdtoogfczokg` int DEFAULT NULL,\n `wrvlccdswkkqahhjdvosdozbwfjdnepu` int DEFAULT NULL,\n `ebppcpijgctydczebvhovkyzgxobchmx` int DEFAULT NULL,\n `wvgaeufbdknymttpgoqxyedgibxdntlo` int DEFAULT NULL,\n `zdixvyvjtarwkdgazibsajyqsetvzdeh` int DEFAULT NULL,\n `mvilvscunheavbdjewikmlwuhvzogqct` int DEFAULT NULL,\n `wnenwayogunxyxjmheorwesbsqqhdatp` int DEFAULT NULL,\n `gtwmadjdwiyiwkgjfsyzdhlqtkxwdwkw` int DEFAULT NULL,\n `ehtgxsoulurhvpagosaytypplcwtsscp` int DEFAULT NULL,\n `wvhwjzoexmsyzzvmbdtxpnotuiyqrblw` int DEFAULT NULL,\n `fjfddgwuntvfpngxgbxywtgxlmlgqyec` int DEFAULT NULL,\n `jfycbxtozisvajbhjyxokazmyqcwzvef` int DEFAULT NULL,\n `dcbilgpxszsyptgqrmdwndhqwzyunlaw` int DEFAULT NULL,\n `csvhjmbulfqqkaydhefnbqrjihhbhusi` int DEFAULT NULL,\n `wkmftkprhgvitgcphagppxqvfsrdyvmt` int DEFAULT NULL,\n `yfusghvxyeyyspabkgipbdknvdhhcxqq` int DEFAULT NULL,\n `zumnirjtpibdqrvwrxxhcrpwpjcdylom` int DEFAULT NULL,\n `miqvempnwrxycftuutisjpzsvozjbiho` int DEFAULT NULL,\n `xlqoadegbwvkyrxroxmlzkwjneqnhgcd` int DEFAULT NULL,\n `yjjnmbkwesmgazbspxushmbchklzntos` int DEFAULT NULL,\n `pnjtkbtgpjsayxmnrdfuqqutgzpjyrns` int DEFAULT NULL,\n `zxripoakcukuztszhdxilkxslykwctfk` int DEFAULT NULL,\n `zqjdwwmundeagnwmabmneozvgreuvket` int DEFAULT NULL,\n `szlcvbyhpncldlrcradwdgoswjauctjm` int DEFAULT NULL,\n `jgsgtcwtieevoakaosngfkaywgegxwsk` int DEFAULT NULL,\n `jgzdvcvimxhpktwibzchmqncddezvwdi` int DEFAULT NULL,\n `aumvletdpdrmteorzfimbyiganwkzqed` int DEFAULT NULL,\n `xsklwoajrqupxuzvgqtswipyzbrrnqpb` int DEFAULT NULL,\n `vulnmjgohjwlsrohbvhyrtwzdrjurknt` int DEFAULT NULL,\n `ljyfrbhspexrflekbqfceodypakdfapt` int DEFAULT NULL,\n `mmgrmgluziacimlsvrcxfklgyizhnwvr` int DEFAULT NULL,\n `scvixkahykbritrozjcbsvmzulenvltf` int DEFAULT NULL,\n `zxprajopwdmeokpwffhbaoaeelbuxbso` int DEFAULT NULL,\n `ctyyyvjezlxfhfzytsnchisrykntvcnd` int DEFAULT NULL,\n `zknebzelkzyxonbhkweedbsgmafzzmlu` int DEFAULT NULL,\n `ojvpusdrarttxetwvftwlctpizgfxtxw` int DEFAULT NULL,\n `dlxpgfmmaeqzuaczbdajpohwprljtckw` int DEFAULT NULL,\n `mmtdzlowqgzdntsviemmtshvgkxtfzkt` int DEFAULT NULL,\n `yqopfqqsiuezkqowksarnljfhypxypgh` int DEFAULT NULL,\n `tpsnmqlmpfyephnrplqwbbpcgfosvhbb` int DEFAULT NULL,\n `ziuyccchelvhjgmcgsjokhwjlazgrcsq` int DEFAULT NULL,\n `xsejyrchpubbfndihuiayljdkjsxrctq` int DEFAULT NULL,\n `ziokluuhlmkymmmpwfjqrkzdilbtbony` int DEFAULT NULL,\n `fmusxcgljfghwpisbghvjfpiobjhvwhd` int DEFAULT NULL,\n `upmsyfvqewdzldadzqkvqsntguepyzes` int DEFAULT NULL,\n `ggomeswxcrytdkeuavvulopzpjmcbgnk` int DEFAULT NULL,\n `fqlpwjqheexaualxspeldblhdaccmcvp` int DEFAULT NULL,\n `xiklussrhcvveuzkjfxjsapqgsdueedc` int DEFAULT NULL,\n `cgkwlmfvqlabnvikerpjankbwnplyapk` int DEFAULT NULL,\n `xzyflzxyaminkfbarqlyhnwpzcuslkzd` int DEFAULT NULL,\n `adufgemorrhymsqnhrdabbhtemeggaqk` int DEFAULT NULL,\n `rnujnzctipbviljawmpdoirwlnfbnmrs` int DEFAULT NULL,\n `mdptobghymmvlyrxjkynqlxkpqcoekjd` int DEFAULT NULL,\n `jroovhthkfnzkxpsdpewttsxnftmemxy` int DEFAULT NULL,\n `wgxfaisocdyvxmkocbymoectuxfnsfrx` int DEFAULT NULL,\n `lymzstnahzzvffzpawfjwauewjnempok` int DEFAULT NULL,\n `tsabdodwwwotfmhzwojprfqkcxhixitp` int DEFAULT NULL,\n `dqbvlioxkarciuhrzurjviwxfwrsxskz` int DEFAULT NULL,\n `ytikuotltxwcbjxsvledyhbolrrpvgik` int DEFAULT NULL,\n `cmnevwnrjgpemlillkcvdntfslcvgsez` int DEFAULT NULL,\n `thpjvvvyisftpukgswwsothcdjkdagbp` int DEFAULT NULL,\n `uluhvevnwgcmpletfengbyjcznrlfkcw` int DEFAULT NULL,\n `ghtlulxrecgbdfohqnxdnihqddplikhz` int DEFAULT NULL,\n `srvpdjoapxrnbtbrlgswyqqapsbxeusf` int DEFAULT NULL,\n `caucsgzpeexmpjtjisfpodjmxlmrsakd` int DEFAULT NULL,\n `phzaoydliassypuyieovjzvxnzdobxaf` int DEFAULT NULL,\n `gemfonsffqzdvdvsaccgytjfmguxpvyo` int DEFAULT NULL,\n `zqhjbjsibxkqfmauihvskjdfiqkawoue` int DEFAULT NULL,\n `alyfrjcfszhicvbpqasnzuqahkmdwblb` int DEFAULT NULL,\n `apvfblyyrhcutbvzghurxmurwjzqdwbb` int DEFAULT NULL,\n `ffduzhmkdqnyvkijqjeneiciqlvmmzos` int DEFAULT NULL,\n `vdxkonjklhvnowisqcwbfnxtpzpdwouu` int DEFAULT NULL,\n `ythbwvulklhkletkzotflqvmurjfgsgl` int DEFAULT NULL,\n `qzwofixpyrdjsxqtlhevsrquaadakxyn` int DEFAULT NULL,\n PRIMARY KEY (`dflqskdirhackweyrdlgnbxtxmwnmjxu`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"nfqrzhysvyortaitkelwhsffveslnthx\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["dflqskdirhackweyrdlgnbxtxmwnmjxu"],"columns":[{"name":"dflqskdirhackweyrdlgnbxtxmwnmjxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"goncslwvscpoqdxeksxmlvplwbjhhicl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"havhcaxflzeciaykaezatjjeglvduheo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flnursnwcxyvjrrygosbslaitoqqyhxn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgfzrbbxljpmiuntbtfiosqnpzwpjont","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zodfkotljmpiufduyfkcyjvofgbgjnjb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qnbdkwrmlccylwbumxyyymztqwfqfrpa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jrilebmuidkywqmdcciqvebsjcbmyqmm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmegkhixgauygayrdnoltdjoopsrubws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zayrqdykxvzudtycndvxeodxbvhelvfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acyvkfebtuscjilxnurzsqwjdqzidwqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcmfdawpjyqpxygiivdcbwkivkyquucv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxenfgtuabnxzhzzygikfnggnqtblfhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkqefdzrntfoahrqcsonqnavejvnwmmn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xfpevsdnecrvjnbptdpttmifhphwyxqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"giqlbizclmkyvdweitcdsjcmqlpvrqjr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"stpsblarituovqefqdyleulfymiojuau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dstpoigpctmkzewrampcjkfzricbstxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azwgeokokgthibxmqibegvxxfxpgaaca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zualkxqsgjhjsoobcfjtpypjrbkvqrub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqumqigtiissdkhaaslwrylezsenrsep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ycesgwfgovjmnrbhufjiyewprrugfbia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avwvqfjqvrwafnwpseyjuckjwsvhhrbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qorqvxrdndbvdwjjgixajkefjbzcrpon","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ryorsemjwecelkfupqvzphuxhdwwppbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zgatjkdijscytxxcrsrcjfvdmqznkyts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctmxpsomxwzqvxshyenxhdtoogfczokg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wrvlccdswkkqahhjdvosdozbwfjdnepu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebppcpijgctydczebvhovkyzgxobchmx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvgaeufbdknymttpgoqxyedgibxdntlo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdixvyvjtarwkdgazibsajyqsetvzdeh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvilvscunheavbdjewikmlwuhvzogqct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnenwayogunxyxjmheorwesbsqqhdatp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtwmadjdwiyiwkgjfsyzdhlqtkxwdwkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehtgxsoulurhvpagosaytypplcwtsscp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvhwjzoexmsyzzvmbdtxpnotuiyqrblw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjfddgwuntvfpngxgbxywtgxlmlgqyec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jfycbxtozisvajbhjyxokazmyqcwzvef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcbilgpxszsyptgqrmdwndhqwzyunlaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"csvhjmbulfqqkaydhefnbqrjihhbhusi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wkmftkprhgvitgcphagppxqvfsrdyvmt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfusghvxyeyyspabkgipbdknvdhhcxqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zumnirjtpibdqrvwrxxhcrpwpjcdylom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"miqvempnwrxycftuutisjpzsvozjbiho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xlqoadegbwvkyrxroxmlzkwjneqnhgcd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjjnmbkwesmgazbspxushmbchklzntos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnjtkbtgpjsayxmnrdfuqqutgzpjyrns","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxripoakcukuztszhdxilkxslykwctfk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqjdwwmundeagnwmabmneozvgreuvket","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"szlcvbyhpncldlrcradwdgoswjauctjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgsgtcwtieevoakaosngfkaywgegxwsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgzdvcvimxhpktwibzchmqncddezvwdi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aumvletdpdrmteorzfimbyiganwkzqed","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsklwoajrqupxuzvgqtswipyzbrrnqpb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vulnmjgohjwlsrohbvhyrtwzdrjurknt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljyfrbhspexrflekbqfceodypakdfapt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmgrmgluziacimlsvrcxfklgyizhnwvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scvixkahykbritrozjcbsvmzulenvltf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxprajopwdmeokpwffhbaoaeelbuxbso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctyyyvjezlxfhfzytsnchisrykntvcnd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zknebzelkzyxonbhkweedbsgmafzzmlu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojvpusdrarttxetwvftwlctpizgfxtxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlxpgfmmaeqzuaczbdajpohwprljtckw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmtdzlowqgzdntsviemmtshvgkxtfzkt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqopfqqsiuezkqowksarnljfhypxypgh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpsnmqlmpfyephnrplqwbbpcgfosvhbb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ziuyccchelvhjgmcgsjokhwjlazgrcsq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsejyrchpubbfndihuiayljdkjsxrctq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ziokluuhlmkymmmpwfjqrkzdilbtbony","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmusxcgljfghwpisbghvjfpiobjhvwhd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upmsyfvqewdzldadzqkvqsntguepyzes","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggomeswxcrytdkeuavvulopzpjmcbgnk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqlpwjqheexaualxspeldblhdaccmcvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xiklussrhcvveuzkjfxjsapqgsdueedc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgkwlmfvqlabnvikerpjankbwnplyapk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xzyflzxyaminkfbarqlyhnwpzcuslkzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"adufgemorrhymsqnhrdabbhtemeggaqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnujnzctipbviljawmpdoirwlnfbnmrs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdptobghymmvlyrxjkynqlxkpqcoekjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jroovhthkfnzkxpsdpewttsxnftmemxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgxfaisocdyvxmkocbymoectuxfnsfrx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lymzstnahzzvffzpawfjwauewjnempok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tsabdodwwwotfmhzwojprfqkcxhixitp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqbvlioxkarciuhrzurjviwxfwrsxskz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytikuotltxwcbjxsvledyhbolrrpvgik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmnevwnrjgpemlillkcvdntfslcvgsez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thpjvvvyisftpukgswwsothcdjkdagbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uluhvevnwgcmpletfengbyjcznrlfkcw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghtlulxrecgbdfohqnxdnihqddplikhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"srvpdjoapxrnbtbrlgswyqqapsbxeusf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"caucsgzpeexmpjtjisfpodjmxlmrsakd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phzaoydliassypuyieovjzvxnzdobxaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gemfonsffqzdvdvsaccgytjfmguxpvyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqhjbjsibxkqfmauihvskjdfiqkawoue","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"alyfrjcfszhicvbpqasnzuqahkmdwblb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"apvfblyyrhcutbvzghurxmurwjzqdwbb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffduzhmkdqnyvkijqjeneiciqlvmmzos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdxkonjklhvnowisqcwbfnxtpzpdwouu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ythbwvulklhkletkzotflqvmurjfgsgl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzwofixpyrdjsxqtlhevsrquaadakxyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669740,"databaseName":"models_schema","ddl":"CREATE TABLE `nmbgkiphylqvhfzedfvrjrurcnseuvvw` (\n `rgfwkirientofqggujieqzgezsrwlilq` int NOT NULL,\n `xacymhcdjoegnoziflbeiryulkwpwzeo` int DEFAULT NULL,\n `wgqxdkaymqissjcmjneyxkpodcwfjefu` int DEFAULT NULL,\n `sgbvxtfdfjffhkitkahniudsxstswoku` int DEFAULT NULL,\n `pfwfddoroxoqnmpmfrtychhoegcoxbth` int DEFAULT NULL,\n `bebfrgjvaecollntyuyjmbxxqjpwbmpl` int DEFAULT NULL,\n `tbxertnhzjxtrxhghuayfwieshypstwm` int DEFAULT NULL,\n `mkusqtmkeoolxxpnzvnfvymkgxnbtizy` int DEFAULT NULL,\n `tpnqatcrxglwvjthjpywlyknydpfdplu` int DEFAULT NULL,\n `qkkbzgxyexazmabhbcxraynyrpwlvvqg` int DEFAULT NULL,\n `syxchuovsyvvkcemwuhecxikoffbvwdw` int DEFAULT NULL,\n `nmxhttrepdxgyrdsydfhmwfwhjbnzike` int DEFAULT NULL,\n `jlhtkeewfelpvuppmjpiloxqsxiwjxuo` int DEFAULT NULL,\n `keuddoaitxjrcvwsafokxathndedktcv` int DEFAULT NULL,\n `nhsnkfnltwmsvjvtpknfjhmmnxhfsuyo` int DEFAULT NULL,\n `qvelwyujwxnopnwishmqgewsflcmounm` int DEFAULT NULL,\n `kefoqpobmvlngragpnwxuqypxahuykiq` int DEFAULT NULL,\n `nokcjtfpweixjhcghkneizqukbybctkv` int DEFAULT NULL,\n `pbijoubcuxuyirahgrtbggdiecglvzth` int DEFAULT NULL,\n `owajjufmnfhiaxznkhvdnwkqcaiwsrng` int DEFAULT NULL,\n `keigylursfeaepsiznmgkihwdgpynfmq` int DEFAULT NULL,\n `gktxsxcpizstsihsxehcvyvjayoancxe` int DEFAULT NULL,\n `fnxjwjxtmalzioyjoujnzsdkjmexuxsh` int DEFAULT NULL,\n `fytzknlypvzqxitcwkhbgvtpzkmksgzo` int DEFAULT NULL,\n `galmobqrlnkidxfmxixyvbzzajdowtwj` int DEFAULT NULL,\n `danqadxuxirvwkpdsqraoivayakwxncc` int DEFAULT NULL,\n `xjkoqfgixxpitosehjkfaagkkxplvqem` int DEFAULT NULL,\n `rjzrshwgngsuzqadtjhvfvifysglghxw` int DEFAULT NULL,\n `mxzxtnmuyrxjwjhcjmrrfjrjwfpbjhwo` int DEFAULT NULL,\n `yyezipyzwualyoxuavojeohoxetzwvyr` int DEFAULT NULL,\n `huwmqrfuiaeszmyhvbfxrtdocanrxval` int DEFAULT NULL,\n `rjgwcavopcmsusjkfjfheibboabbrnfc` int DEFAULT NULL,\n `jypbqmzyfanmrfnjhriczresffzsfukk` int DEFAULT NULL,\n `sokaxswwmsveuoeknvotdxcujyvpfxyz` int DEFAULT NULL,\n `zvkoyktybsrdzdldsgaojwtqhwhcnvac` int DEFAULT NULL,\n `vkxcsmoelhecudqvngofruoskesnycla` int DEFAULT NULL,\n `usgjretgdfslxsoxqztlgkzlwmagjbzy` int DEFAULT NULL,\n `ifezodzreabnsqlzowhbhmsdmzqdijzm` int DEFAULT NULL,\n `holzxwmecswlhrygivgmdesyaypcxqls` int DEFAULT NULL,\n `cinunqbkhppkrfnvqrarzijblypryrql` int DEFAULT NULL,\n `xuqjdjbywifbathjebuzpmdtwrcnkqqu` int DEFAULT NULL,\n `zswsumnkfpjdcqmlbiwmyofiwqceddfh` int DEFAULT NULL,\n `cboawggdctfnwmdglpczrmzcnkgqoosl` int DEFAULT NULL,\n `lqjnieiskjriudlvonyvbkaursrjbywm` int DEFAULT NULL,\n `uhriyomfhvnsqqyhezjhnixmqzlmyskb` int DEFAULT NULL,\n `rxqirbotrqiqwshxkedkcinbsaxfmrqj` int DEFAULT NULL,\n `icifnagvnkzgveahpejhkidszacncvfb` int DEFAULT NULL,\n `didapzengjqupbgugbujzncnxljlwxzo` int DEFAULT NULL,\n `nfawiinnhyvcrnzarrbkdgehgzrxhire` int DEFAULT NULL,\n `qvbhbgmabwtiemidhaincxzlttrejjtt` int DEFAULT NULL,\n `hfewhzdnyzdzuqbuuqedveskaqxmmute` int DEFAULT NULL,\n `xnzevkofqsxrbgfyuqxhykiputpbphbq` int DEFAULT NULL,\n `rlddvxekutnjzurftogusbhlljoxngjp` int DEFAULT NULL,\n `fmqcxraddbqcwhbikqrnvblsaiteolxj` int DEFAULT NULL,\n `ilhqmaqsygubddmgvdcpxhmwxehhdfnj` int DEFAULT NULL,\n `iixbuhdpgidczvxhrldwtmbnuoktakxk` int DEFAULT NULL,\n `rvaldjitcpvyidufbnjbyfzmjbxsnypk` int DEFAULT NULL,\n `itllowsokszkzmskeaxjxqmlbewaiele` int DEFAULT NULL,\n `dfebmsrahrhdjnbpecpbmgjjvmssczbv` int DEFAULT NULL,\n `ytcoaisxokcokbzedlbiagjcjnbxajeh` int DEFAULT NULL,\n `gveovojdkhzihwkcxgbokbstumxtmoni` int DEFAULT NULL,\n `xpahgexyknxnnlbbropzwxjpmuiohuxi` int DEFAULT NULL,\n `ndmgrdxemjjfhewytkqggpxrghakzkrx` int DEFAULT NULL,\n `htwrnqpsmehngiqrmujjeesceurofebo` int DEFAULT NULL,\n `scwvwtyvohtbzqkmfpfxolayapaieolu` int DEFAULT NULL,\n `esqppflrikaszvzjwkgcpjcarrqpeqgi` int DEFAULT NULL,\n `mnwxcxjjywvmixgneurltvkxdxnjbknj` int DEFAULT NULL,\n `exbftpduhlaxmwefavesslvjsjwvldgl` int DEFAULT NULL,\n `jkdrycupdinmgeysvscyxrxoxmpboeya` int DEFAULT NULL,\n `gmieymsbphwkxquzjavhlgkrhmzkpqob` int DEFAULT NULL,\n `bhuiqpmttiprzarxhyxhuvibkbhmbfzb` int DEFAULT NULL,\n `qatlnkmymzzwxnwzgyypqygujmmdygqr` int DEFAULT NULL,\n `ilgeldzvqbkrjghwccniuobdukesmwoo` int DEFAULT NULL,\n `fywjpcmfhtoobispeepgguoypzxzbtte` int DEFAULT NULL,\n `fhoexwpancsfpqjwxdfucyravnsghort` int DEFAULT NULL,\n `nodpbbwnfjvncdwlybvylrypdckunjmz` int DEFAULT NULL,\n `edmlyqatyapajmmrsjyglcmilftyqvsx` int DEFAULT NULL,\n `pdyrfjhvvmqqabgscrjfaqzpiddwdvws` int DEFAULT NULL,\n `xnbepmgalacdvmavgbqsektdxoljvbsb` int DEFAULT NULL,\n `oxxgkyspogmdlbyxuwbazoeufaqvixpr` int DEFAULT NULL,\n `xuzpfofhlapmzubwdekodzaryawglzjv` int DEFAULT NULL,\n `kxeghpluvyqgqxprutvcabwnwbeowlhc` int DEFAULT NULL,\n `zhkuaacbmzezfuendxlancnnydfxdjhj` int DEFAULT NULL,\n `mhnvmipblylnfkhnjjlibnrjhuuaycgj` int DEFAULT NULL,\n `gdzkrdkjafwigatiarukmfspwqwkuwbp` int DEFAULT NULL,\n `llirchsasqhkcpxrjwydsuxolwdvnhml` int DEFAULT NULL,\n `rirjdrzagqxuzbjvarklukpbahxydiwa` int DEFAULT NULL,\n `pydhstttmlyaqxqhlvsgaqdldydvtulf` int DEFAULT NULL,\n `gtywfqeexifoyozxbvudfiwbyksjkmom` int DEFAULT NULL,\n `kkplqtijiaiwzfigxcrdpudgmvmrctgg` int DEFAULT NULL,\n `noukthyaknibwtfljnnyzojqgxvwexzg` int DEFAULT NULL,\n `lknyujxtazlripqctcofwofcxbwszlqp` int DEFAULT NULL,\n `ddwcbcnwclwyqbrqrzbqbdugzskglvqv` int DEFAULT NULL,\n `jbvvooplzzwlcnooxdhkzgkrborwonga` int DEFAULT NULL,\n `gtutnymuquorhzuzdplpiansffukbtkb` int DEFAULT NULL,\n `dreiezdllqhsxntphlpponghsgrjvsgc` int DEFAULT NULL,\n `cpcztxotyvxpaudkostomctozpcdwqgf` int DEFAULT NULL,\n `woxkmvzghtfmgydqxbqejmpsxsgkglhe` int DEFAULT NULL,\n `iuqjnjmycgmbhkuxccgttdzaaloyaxgx` int DEFAULT NULL,\n `rqnnkkkqtzybaxtzpmgharqgfnlwzipi` int DEFAULT NULL,\n PRIMARY KEY (`rgfwkirientofqggujieqzgezsrwlilq`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"nmbgkiphylqvhfzedfvrjrurcnseuvvw\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["rgfwkirientofqggujieqzgezsrwlilq"],"columns":[{"name":"rgfwkirientofqggujieqzgezsrwlilq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"xacymhcdjoegnoziflbeiryulkwpwzeo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgqxdkaymqissjcmjneyxkpodcwfjefu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sgbvxtfdfjffhkitkahniudsxstswoku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pfwfddoroxoqnmpmfrtychhoegcoxbth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bebfrgjvaecollntyuyjmbxxqjpwbmpl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbxertnhzjxtrxhghuayfwieshypstwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkusqtmkeoolxxpnzvnfvymkgxnbtizy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpnqatcrxglwvjthjpywlyknydpfdplu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qkkbzgxyexazmabhbcxraynyrpwlvvqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"syxchuovsyvvkcemwuhecxikoffbvwdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmxhttrepdxgyrdsydfhmwfwhjbnzike","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlhtkeewfelpvuppmjpiloxqsxiwjxuo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"keuddoaitxjrcvwsafokxathndedktcv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhsnkfnltwmsvjvtpknfjhmmnxhfsuyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvelwyujwxnopnwishmqgewsflcmounm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kefoqpobmvlngragpnwxuqypxahuykiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nokcjtfpweixjhcghkneizqukbybctkv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbijoubcuxuyirahgrtbggdiecglvzth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owajjufmnfhiaxznkhvdnwkqcaiwsrng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"keigylursfeaepsiznmgkihwdgpynfmq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gktxsxcpizstsihsxehcvyvjayoancxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnxjwjxtmalzioyjoujnzsdkjmexuxsh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fytzknlypvzqxitcwkhbgvtpzkmksgzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"galmobqrlnkidxfmxixyvbzzajdowtwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"danqadxuxirvwkpdsqraoivayakwxncc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjkoqfgixxpitosehjkfaagkkxplvqem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjzrshwgngsuzqadtjhvfvifysglghxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mxzxtnmuyrxjwjhcjmrrfjrjwfpbjhwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yyezipyzwualyoxuavojeohoxetzwvyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"huwmqrfuiaeszmyhvbfxrtdocanrxval","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjgwcavopcmsusjkfjfheibboabbrnfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jypbqmzyfanmrfnjhriczresffzsfukk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sokaxswwmsveuoeknvotdxcujyvpfxyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvkoyktybsrdzdldsgaojwtqhwhcnvac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkxcsmoelhecudqvngofruoskesnycla","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"usgjretgdfslxsoxqztlgkzlwmagjbzy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifezodzreabnsqlzowhbhmsdmzqdijzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"holzxwmecswlhrygivgmdesyaypcxqls","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cinunqbkhppkrfnvqrarzijblypryrql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xuqjdjbywifbathjebuzpmdtwrcnkqqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zswsumnkfpjdcqmlbiwmyofiwqceddfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cboawggdctfnwmdglpczrmzcnkgqoosl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqjnieiskjriudlvonyvbkaursrjbywm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhriyomfhvnsqqyhezjhnixmqzlmyskb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxqirbotrqiqwshxkedkcinbsaxfmrqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icifnagvnkzgveahpejhkidszacncvfb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"didapzengjqupbgugbujzncnxljlwxzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfawiinnhyvcrnzarrbkdgehgzrxhire","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvbhbgmabwtiemidhaincxzlttrejjtt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfewhzdnyzdzuqbuuqedveskaqxmmute","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnzevkofqsxrbgfyuqxhykiputpbphbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlddvxekutnjzurftogusbhlljoxngjp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmqcxraddbqcwhbikqrnvblsaiteolxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ilhqmaqsygubddmgvdcpxhmwxehhdfnj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iixbuhdpgidczvxhrldwtmbnuoktakxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rvaldjitcpvyidufbnjbyfzmjbxsnypk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itllowsokszkzmskeaxjxqmlbewaiele","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dfebmsrahrhdjnbpecpbmgjjvmssczbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytcoaisxokcokbzedlbiagjcjnbxajeh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gveovojdkhzihwkcxgbokbstumxtmoni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpahgexyknxnnlbbropzwxjpmuiohuxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ndmgrdxemjjfhewytkqggpxrghakzkrx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htwrnqpsmehngiqrmujjeesceurofebo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scwvwtyvohtbzqkmfpfxolayapaieolu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"esqppflrikaszvzjwkgcpjcarrqpeqgi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnwxcxjjywvmixgneurltvkxdxnjbknj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exbftpduhlaxmwefavesslvjsjwvldgl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jkdrycupdinmgeysvscyxrxoxmpboeya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmieymsbphwkxquzjavhlgkrhmzkpqob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bhuiqpmttiprzarxhyxhuvibkbhmbfzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qatlnkmymzzwxnwzgyypqygujmmdygqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ilgeldzvqbkrjghwccniuobdukesmwoo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fywjpcmfhtoobispeepgguoypzxzbtte","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhoexwpancsfpqjwxdfucyravnsghort","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nodpbbwnfjvncdwlybvylrypdckunjmz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edmlyqatyapajmmrsjyglcmilftyqvsx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdyrfjhvvmqqabgscrjfaqzpiddwdvws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnbepmgalacdvmavgbqsektdxoljvbsb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oxxgkyspogmdlbyxuwbazoeufaqvixpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xuzpfofhlapmzubwdekodzaryawglzjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxeghpluvyqgqxprutvcabwnwbeowlhc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhkuaacbmzezfuendxlancnnydfxdjhj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhnvmipblylnfkhnjjlibnrjhuuaycgj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdzkrdkjafwigatiarukmfspwqwkuwbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llirchsasqhkcpxrjwydsuxolwdvnhml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rirjdrzagqxuzbjvarklukpbahxydiwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pydhstttmlyaqxqhlvsgaqdldydvtulf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtywfqeexifoyozxbvudfiwbyksjkmom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kkplqtijiaiwzfigxcrdpudgmvmrctgg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"noukthyaknibwtfljnnyzojqgxvwexzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lknyujxtazlripqctcofwofcxbwszlqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddwcbcnwclwyqbrqrzbqbdugzskglvqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbvvooplzzwlcnooxdhkzgkrborwonga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtutnymuquorhzuzdplpiansffukbtkb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dreiezdllqhsxntphlpponghsgrjvsgc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cpcztxotyvxpaudkostomctozpcdwqgf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"woxkmvzghtfmgydqxbqejmpsxsgkglhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iuqjnjmycgmbhkuxccgttdzaaloyaxgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqnnkkkqtzybaxtzpmgharqgfnlwzipi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669775,"databaseName":"models_schema","ddl":"CREATE TABLE `nsbiwnxsqajlnasfuoofgvhmvbabaxfp` (\n `snmhjgihhkqxiaozyigyozjamdaokusk` int NOT NULL,\n `axaxsncgcrqvotzbdymtkkgvtiajqbqg` int DEFAULT NULL,\n `zztpckwvrortexevagqdxhoohosftwcs` int DEFAULT NULL,\n `ahiqmqsssvlfjmptksqnnpwvvnlfoxbc` int DEFAULT NULL,\n `hydywixwcnafzutlgbqapflspyrxdcrv` int DEFAULT NULL,\n `tfupxfveeadrmbggobjpgrngqffmnvaz` int DEFAULT NULL,\n `ilavkqffmfnoidalujsetjvikzmeomer` int DEFAULT NULL,\n `qwobkkjsuaanzpstyqcsvcnwoqsletov` int DEFAULT NULL,\n `jwgrmkspgujofobqtecgcglpynzhqsfw` int DEFAULT NULL,\n `ijttbzjrzdjecoatqksbiukwlvvohvja` int DEFAULT NULL,\n `pbyoaujqyntzyuuiwhwklajbshvvivak` int DEFAULT NULL,\n `kfzfnunndrkzazokigwlrapsyhzfgoqq` int DEFAULT NULL,\n `rnjoevghkyrzqigkozwsrdliyqwczvtc` int DEFAULT NULL,\n `dzfkwnxmqumeumnebjflwzhunduamabo` int DEFAULT NULL,\n `lgnwfjveugqdsmqahwoltpclhofxzhwu` int DEFAULT NULL,\n `fynrjocarqbioxnrxmxyyzfyiroerslh` int DEFAULT NULL,\n `ajlqrstbvjdmbjtmfghysknaksdljgwa` int DEFAULT NULL,\n `xitmaqpmpehpgzcvhexcockqbakgwilg` int DEFAULT NULL,\n `rhhmhuvetpmbpmadbenpszyqpzxhamas` int DEFAULT NULL,\n `uspmlipfoeauvdmzvfqsgpyakkozgkwr` int DEFAULT NULL,\n `abkgobdweoysenekcafjldqdpfmwkzue` int DEFAULT NULL,\n `xjyyichijymasoctjwoevitzwdkjqnke` int DEFAULT NULL,\n `unbwdcwyeibmpzmnjjossilspcdrbtin` int DEFAULT NULL,\n `hqvtispqbizogpmcfpaqajkuemdctuaj` int DEFAULT NULL,\n `mgwgpppdppadtiowdkroxjjhkxbiepps` int DEFAULT NULL,\n `louwgbplldajmchiwmjbfjpkgtcuczfk` int DEFAULT NULL,\n `jctismfgoqxvuvonahblmwibjgdpctxr` int DEFAULT NULL,\n `nddhygzwitevollcmhljbdhwsdvpbltg` int DEFAULT NULL,\n `cqvdgitaaathpjqpekxdduvqsdoqekih` int DEFAULT NULL,\n `hpwbluniqeenkvaklzerycfkinptqkuk` int DEFAULT NULL,\n `ykwyvqwxvihdfdbiwzvzstpqruiyabvw` int DEFAULT NULL,\n `hezeljtrionirnidhkqazqrwjccknomd` int DEFAULT NULL,\n `bjownjzwfryumlghspxxacburftltpfr` int DEFAULT NULL,\n `jsxeegqzdddvhvstqmhpebpajecvpljo` int DEFAULT NULL,\n `eibdqowwisyxbzozjsetfvboeyvhjpcn` int DEFAULT NULL,\n `qqcgqjihrzeamlwmgcbijifkgemdguvr` int DEFAULT NULL,\n `jfbmidbedutufellsmlxamjrncpwnowd` int DEFAULT NULL,\n `ozoluklyrqcrtgzoxbyworutwvznsdza` int DEFAULT NULL,\n `gsuqlwkhfmunpkezzevxueatjmmpygdq` int DEFAULT NULL,\n `goubcrwjekjhwnuokssujvntpmqqbldy` int DEFAULT NULL,\n `dzrmltmgkyynjohlbduhdgcoqyquqarp` int DEFAULT NULL,\n `vkbrfmhaodotqsiqkzycpbrbslyzmrcf` int DEFAULT NULL,\n `kjdixcevyrdxgokapzkaijkfynbkpslt` int DEFAULT NULL,\n `qfmeotsmtupxitzmptazlmrvmriddojf` int DEFAULT NULL,\n `ryqjwsqnjtggpcreuaaykvusaejobyak` int DEFAULT NULL,\n `muwuxatxmxrxcsmicjwtvhnwjfdwxbxo` int DEFAULT NULL,\n `fgxgilmyowlurzvdroclgqbakazzetda` int DEFAULT NULL,\n `kknozmfkdlftoolftdncykafchbfdldl` int DEFAULT NULL,\n `fjwsbihepcyzvxhvbvcdzpsudcsgitdz` int DEFAULT NULL,\n `tlecmpmnqthqkolfkjwntwviicjeadhu` int DEFAULT NULL,\n `lyaqfjbiwrkaonjwlsbqdqxhsyuzxogu` int DEFAULT NULL,\n `xqtfavtqvaomynzsrdaldzazwdobwbnq` int DEFAULT NULL,\n `eooanccxurvvkslcovgdbruxvmqwrjjo` int DEFAULT NULL,\n `awfuycxuvffpyoxacejfdqrmpyoahmij` int DEFAULT NULL,\n `rcbjpfsjtdopcizprypqcewwxhjbfioc` int DEFAULT NULL,\n `dczcqbevhupjvbulrrabmoedcocqpiqh` int DEFAULT NULL,\n `eaiifahtloelyaxquuxggikqbiqxqsor` int DEFAULT NULL,\n `kyiffxdsjeojvohjdmebvfyozrncakeg` int DEFAULT NULL,\n `uusciaxhbhveclyhbepfkbrwzfsaahss` int DEFAULT NULL,\n `juekzwhyxawyrlloiaifzwwgsuquslik` int DEFAULT NULL,\n `fcfoiipyymdvmfgjuhkkzfnadmrrmmuy` int DEFAULT NULL,\n `cfrjlcorwsueqmacwtxemvfouosamehp` int DEFAULT NULL,\n `tybmwrjfqdgayrxwagnijotyiydoxjyf` int DEFAULT NULL,\n `hejyqqoubsrkipihplmqwjzoiwcmetfs` int DEFAULT NULL,\n `omfgrnsopkcwthnkjdywyvwzscgbgdse` int DEFAULT NULL,\n `oruwhxdjovhosbffrfpwoxzdzhyhneco` int DEFAULT NULL,\n `kplbsxopxlpjxtqbudjpucxcqxazcypm` int DEFAULT NULL,\n `zqaswkmyebbtniwdkipfwgwjwjduwtvs` int DEFAULT NULL,\n `kslywwvtakpscdztusccptbblamggfcd` int DEFAULT NULL,\n `ahgjuznhmcjiyjdvefkdxjaujwafbjuo` int DEFAULT NULL,\n `mmrzqzlmqeqxychmaylzwmfrjjfkatnc` int DEFAULT NULL,\n `xdrvvcukikefqjsbpajdpueqvrtmaxwc` int DEFAULT NULL,\n `rvxfqllvbxwxkxwvronghagphjhylwzb` int DEFAULT NULL,\n `cccndkyndciwcabhmzjlayytzkqylcno` int DEFAULT NULL,\n `aeivqqzbqqnorurxvbyxfeeejgjhmogr` int DEFAULT NULL,\n `bujciudodihsebdinyspcmkzubgwsoiw` int DEFAULT NULL,\n `ggoxdyauxvecvsfgcsfvwtcliadxofrj` int DEFAULT NULL,\n `czvvujmdybduxmmxnhimuhynlyteqwmr` int DEFAULT NULL,\n `lmkncfjqpowoytggwbhpkwudpabcumyy` int DEFAULT NULL,\n `efvmdirdubqvsslieqmjjsiyewfplbow` int DEFAULT NULL,\n `ejhmfznickmewhgfsvvqmyfrqggujubt` int DEFAULT NULL,\n `dfezavnswrbghbftifbatwosnlfwbvfb` int DEFAULT NULL,\n `csujgktyegdhzmuwjpnviewjqtwmlypw` int DEFAULT NULL,\n `btjgydhcuqatrhleqyjihvjdldtjzmxq` int DEFAULT NULL,\n `wsrlxakrqdgvfbqulnugkahfdhxzxagl` int DEFAULT NULL,\n `ttpygqvbkpuygatuhvkfbboaedldtcfv` int DEFAULT NULL,\n `tjyavhbzptjmkrmulnqaggebtnldpngj` int DEFAULT NULL,\n `jqpkrnaulzmbulnirblgvbtpgungpobn` int DEFAULT NULL,\n `tyakdidzivrkrgmmtnszvehjvpyqtpvf` int DEFAULT NULL,\n `lvlknmiuyxjnrozdoopicjlhzutckusl` int DEFAULT NULL,\n `bpuvyddbyzxhsadifryaedrrhxcklocm` int DEFAULT NULL,\n `jaktrwvgckxuudvgnukcudpupbkozzpv` int DEFAULT NULL,\n `tfjpiibxgbooygdmsyxykquujdgiukis` int DEFAULT NULL,\n `nqbokickcbpdsfrweemzkzytynjjnhzm` int DEFAULT NULL,\n `zyjidxqkjdysujpkhhhizktwfxukdehb` int DEFAULT NULL,\n `rhmpkxtybyvlyeonwvwfdsdmbjicjiqy` int DEFAULT NULL,\n `bvjhflgzeurkkjiyoqxmvyvhpbzmzwzj` int DEFAULT NULL,\n `syglxaovsnfjobafdzvsjujollgfrokj` int DEFAULT NULL,\n `xsrqcxtmdxqbryjbuapqyfxgpjuxmsgl` int DEFAULT NULL,\n `zvvntvldmuecgnbqilfkkodxkkvbwzcc` int DEFAULT NULL,\n PRIMARY KEY (`snmhjgihhkqxiaozyigyozjamdaokusk`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"nsbiwnxsqajlnasfuoofgvhmvbabaxfp\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["snmhjgihhkqxiaozyigyozjamdaokusk"],"columns":[{"name":"snmhjgihhkqxiaozyigyozjamdaokusk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"axaxsncgcrqvotzbdymtkkgvtiajqbqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zztpckwvrortexevagqdxhoohosftwcs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahiqmqsssvlfjmptksqnnpwvvnlfoxbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hydywixwcnafzutlgbqapflspyrxdcrv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tfupxfveeadrmbggobjpgrngqffmnvaz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ilavkqffmfnoidalujsetjvikzmeomer","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwobkkjsuaanzpstyqcsvcnwoqsletov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwgrmkspgujofobqtecgcglpynzhqsfw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijttbzjrzdjecoatqksbiukwlvvohvja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbyoaujqyntzyuuiwhwklajbshvvivak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfzfnunndrkzazokigwlrapsyhzfgoqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnjoevghkyrzqigkozwsrdliyqwczvtc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dzfkwnxmqumeumnebjflwzhunduamabo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgnwfjveugqdsmqahwoltpclhofxzhwu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fynrjocarqbioxnrxmxyyzfyiroerslh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajlqrstbvjdmbjtmfghysknaksdljgwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xitmaqpmpehpgzcvhexcockqbakgwilg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhhmhuvetpmbpmadbenpszyqpzxhamas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uspmlipfoeauvdmzvfqsgpyakkozgkwr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abkgobdweoysenekcafjldqdpfmwkzue","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjyyichijymasoctjwoevitzwdkjqnke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unbwdcwyeibmpzmnjjossilspcdrbtin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqvtispqbizogpmcfpaqajkuemdctuaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mgwgpppdppadtiowdkroxjjhkxbiepps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"louwgbplldajmchiwmjbfjpkgtcuczfk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jctismfgoqxvuvonahblmwibjgdpctxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nddhygzwitevollcmhljbdhwsdvpbltg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqvdgitaaathpjqpekxdduvqsdoqekih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hpwbluniqeenkvaklzerycfkinptqkuk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykwyvqwxvihdfdbiwzvzstpqruiyabvw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hezeljtrionirnidhkqazqrwjccknomd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjownjzwfryumlghspxxacburftltpfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jsxeegqzdddvhvstqmhpebpajecvpljo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eibdqowwisyxbzozjsetfvboeyvhjpcn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqcgqjihrzeamlwmgcbijifkgemdguvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jfbmidbedutufellsmlxamjrncpwnowd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozoluklyrqcrtgzoxbyworutwvznsdza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsuqlwkhfmunpkezzevxueatjmmpygdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"goubcrwjekjhwnuokssujvntpmqqbldy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dzrmltmgkyynjohlbduhdgcoqyquqarp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkbrfmhaodotqsiqkzycpbrbslyzmrcf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjdixcevyrdxgokapzkaijkfynbkpslt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfmeotsmtupxitzmptazlmrvmriddojf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ryqjwsqnjtggpcreuaaykvusaejobyak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"muwuxatxmxrxcsmicjwtvhnwjfdwxbxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fgxgilmyowlurzvdroclgqbakazzetda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kknozmfkdlftoolftdncykafchbfdldl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjwsbihepcyzvxhvbvcdzpsudcsgitdz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlecmpmnqthqkolfkjwntwviicjeadhu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lyaqfjbiwrkaonjwlsbqdqxhsyuzxogu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqtfavtqvaomynzsrdaldzazwdobwbnq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eooanccxurvvkslcovgdbruxvmqwrjjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awfuycxuvffpyoxacejfdqrmpyoahmij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcbjpfsjtdopcizprypqcewwxhjbfioc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dczcqbevhupjvbulrrabmoedcocqpiqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eaiifahtloelyaxquuxggikqbiqxqsor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kyiffxdsjeojvohjdmebvfyozrncakeg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uusciaxhbhveclyhbepfkbrwzfsaahss","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"juekzwhyxawyrlloiaifzwwgsuquslik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcfoiipyymdvmfgjuhkkzfnadmrrmmuy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfrjlcorwsueqmacwtxemvfouosamehp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tybmwrjfqdgayrxwagnijotyiydoxjyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hejyqqoubsrkipihplmqwjzoiwcmetfs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omfgrnsopkcwthnkjdywyvwzscgbgdse","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oruwhxdjovhosbffrfpwoxzdzhyhneco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kplbsxopxlpjxtqbudjpucxcqxazcypm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqaswkmyebbtniwdkipfwgwjwjduwtvs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kslywwvtakpscdztusccptbblamggfcd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahgjuznhmcjiyjdvefkdxjaujwafbjuo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmrzqzlmqeqxychmaylzwmfrjjfkatnc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdrvvcukikefqjsbpajdpueqvrtmaxwc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rvxfqllvbxwxkxwvronghagphjhylwzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cccndkyndciwcabhmzjlayytzkqylcno","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aeivqqzbqqnorurxvbyxfeeejgjhmogr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bujciudodihsebdinyspcmkzubgwsoiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggoxdyauxvecvsfgcsfvwtcliadxofrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"czvvujmdybduxmmxnhimuhynlyteqwmr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmkncfjqpowoytggwbhpkwudpabcumyy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"efvmdirdubqvsslieqmjjsiyewfplbow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejhmfznickmewhgfsvvqmyfrqggujubt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dfezavnswrbghbftifbatwosnlfwbvfb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"csujgktyegdhzmuwjpnviewjqtwmlypw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"btjgydhcuqatrhleqyjihvjdldtjzmxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wsrlxakrqdgvfbqulnugkahfdhxzxagl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ttpygqvbkpuygatuhvkfbboaedldtcfv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjyavhbzptjmkrmulnqaggebtnldpngj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jqpkrnaulzmbulnirblgvbtpgungpobn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tyakdidzivrkrgmmtnszvehjvpyqtpvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvlknmiuyxjnrozdoopicjlhzutckusl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bpuvyddbyzxhsadifryaedrrhxcklocm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jaktrwvgckxuudvgnukcudpupbkozzpv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tfjpiibxgbooygdmsyxykquujdgiukis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqbokickcbpdsfrweemzkzytynjjnhzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyjidxqkjdysujpkhhhizktwfxukdehb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhmpkxtybyvlyeonwvwfdsdmbjicjiqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvjhflgzeurkkjiyoqxmvyvhpbzmzwzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"syglxaovsnfjobafdzvsjujollgfrokj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsrqcxtmdxqbryjbuapqyfxgpjuxmsgl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvvntvldmuecgnbqilfkkodxkkvbwzcc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669806,"databaseName":"models_schema","ddl":"CREATE TABLE `ntdsamdzxoluyqsidfutsijfjlwpulzz` (\n `dtpfwumwgexswomaynmjzenakmzoxfdn` int NOT NULL,\n `xfctvyrzfhqyffxqnjyqmjbsluexmkar` int DEFAULT NULL,\n `rntimauabkagzsbsvnahisqpvbitstih` int DEFAULT NULL,\n `ilqvsfpqylrrfnslojlyjofgwjqkbwiu` int DEFAULT NULL,\n `aryjvgokmwrxytpgjdeqvpnwjvcuwbgr` int DEFAULT NULL,\n `gxphsedvsjdejrwtwyrkfzdpjrwinsrj` int DEFAULT NULL,\n `xohqbbzwcubeabpkextlrgdgqjgashtx` int DEFAULT NULL,\n `bxrmccbxmvtvbdwtjoxogrimaljpdcuc` int DEFAULT NULL,\n `kvddkynwjnsmdruhjkxnusjhmeyktlgr` int DEFAULT NULL,\n `asritxnynrhddkreuirsztuvjcpgtdvv` int DEFAULT NULL,\n `aqkszbybllvkiemzvxeujkygmfzhfngr` int DEFAULT NULL,\n `puaxdumumiumjuazigvxlruznqifnlds` int DEFAULT NULL,\n `vakrmwejfainumfqjhipgdpecicchiyo` int DEFAULT NULL,\n `aptxzzzeijxvrzgzpsmsrauqdunxbxvq` int DEFAULT NULL,\n `ijrrvzczqhzvbswjwndxksfdlzstjyia` int DEFAULT NULL,\n `bhmmjevajfhbahqrxoqvqbglisbucspo` int DEFAULT NULL,\n `xakccscajvyyudgqzvsqrdpjkdcehloj` int DEFAULT NULL,\n `lxgfxczvzixyzvtwfknbhwqmxydbiklj` int DEFAULT NULL,\n `yrovtkgowkwacmspuqjvnmdzgjthamdz` int DEFAULT NULL,\n `vmemqvqsmdnrmojjjzswxwvoojiuermf` int DEFAULT NULL,\n `qxskwpldyayhmcwkimmnehiyzjdaxdvn` int DEFAULT NULL,\n `wsaysscpkmhpxjkdztldbgkhfzfbgcez` int DEFAULT NULL,\n `wjwwdxxyerpuldznemdkgqywlxscftbc` int DEFAULT NULL,\n `qqmqqgqoupoxvycnuogbipdeybftxqgz` int DEFAULT NULL,\n `pmnirpracnzbtlfwmklolyzgvlisoevd` int DEFAULT NULL,\n `eqrymiioairwrshsqyfcfbinasdmtvyc` int DEFAULT NULL,\n `anlezyaxpkwabssaftqeuocypaxlwjwh` int DEFAULT NULL,\n `curkisshgwyetrmylxvqbpvlysrftsch` int DEFAULT NULL,\n `trvxkecolaekkbcooewkjyynfypsynci` int DEFAULT NULL,\n `tgcvhcizqfpgspkoczxwpjawmrikbfmu` int DEFAULT NULL,\n `rrkgirnpfqvmmrrwmemdmeaaxwmfitiw` int DEFAULT NULL,\n `hkddcsnslhimahhrselmpwpfvlhsnafk` int DEFAULT NULL,\n `amjwtukevkhaoppojwqbzhlkfrsxopfz` int DEFAULT NULL,\n `tbxixgzfvygojijxwaxrrutuzwbmvtsq` int DEFAULT NULL,\n `auhxxtwkgymzcswokfxvdtlptlxcanxc` int DEFAULT NULL,\n `lgmuldqhhptvdwkezryyvqickywkkifs` int DEFAULT NULL,\n `lrqytvjeurlgcjoezapjetmggcpjwjai` int DEFAULT NULL,\n `phtwmrfvwxrhndwybwvnvgogbfmkusjw` int DEFAULT NULL,\n `vwdpwcffmdqfrsfywclbnpvscxnemefs` int DEFAULT NULL,\n `vlhvaujlqlfqkyxryvdzlknfvgdsuuai` int DEFAULT NULL,\n `annrryymefensmrttkqhbgiamaniepps` int DEFAULT NULL,\n `lqzmnvgrqyllibujzcqcoomyarosltko` int DEFAULT NULL,\n `ykzgnuamqdhpcymsdiyqqqdnyyuucwbc` int DEFAULT NULL,\n `nvjjqlrcansimgshxepagqusqdisoytl` int DEFAULT NULL,\n `hdczgslqxeveyjrhmwcmwfwwuxwojsxf` int DEFAULT NULL,\n `rcoabydnbmniuwngnaddhrgrsgrddmqk` int DEFAULT NULL,\n `ierkvdbyynqohfbacmpuaswowowrnntw` int DEFAULT NULL,\n `iexwtycfmymezncbanptuhxcvtbdfdzs` int DEFAULT NULL,\n `pxgqzpoosxoymezgnrqmukhwixmgdqdg` int DEFAULT NULL,\n `bnbddljainngiiuqkaxarpxnkfnsfrws` int DEFAULT NULL,\n `sfdggeylpyptyvydoccwfwcdltunwtkq` int DEFAULT NULL,\n `girhfmepguyqrukhmhkerxcneirnitvh` int DEFAULT NULL,\n `wjobwibafjhkbdqlacmhualsyurnhpsp` int DEFAULT NULL,\n `dsqqgmzfsidlbxlktlqygdcomafhdgff` int DEFAULT NULL,\n `evcyryfkkkcqjgsrhnjepwjgbcnrjopg` int DEFAULT NULL,\n `ubvpcrobtvxjvlikrvakgkgkcpuzzzxw` int DEFAULT NULL,\n `jclmxvbndrcdxjlpzzvvtzdsuesfwint` int DEFAULT NULL,\n `lgfyvzsdcjjlwhpvitlmctvsiikzpfmg` int DEFAULT NULL,\n `iujrkshaebhetdnonsioiiimvrlcwjpw` int DEFAULT NULL,\n `bbukvlrsqjmxaaqudnfqsctsbsxkinvc` int DEFAULT NULL,\n `woejhhrkokoejhaqvgozjbznckhbgumr` int DEFAULT NULL,\n `dgkrxmthdxhayxchaqdugjaqvabjulza` int DEFAULT NULL,\n `cmcyftrafthsshugswfrcsjcmzclsaxj` int DEFAULT NULL,\n `opjtfhienghzqughjdzwaehqpgbctfzr` int DEFAULT NULL,\n `nckmuofqhwdxbmgiqfvdfbixhwfikszu` int DEFAULT NULL,\n `qjvudnosdkxcbuczvbzthidpzluqouyd` int DEFAULT NULL,\n `lrxyzjezbzotizkejfpmhnupwhjagkpl` int DEFAULT NULL,\n `bhxnlychsjifztjfxgjmfybxgnkkbaea` int DEFAULT NULL,\n `vsxmxluojykblewvjzjtmxgsprpcwexw` int DEFAULT NULL,\n `mzsvyhitxforlywjzavcnwbhqoqqsugx` int DEFAULT NULL,\n `xxsitxqjxvjikrnqwwdfclqxxtkfdaqg` int DEFAULT NULL,\n `dkrxkgwsoknxaygcdzpccdcyastzdosk` int DEFAULT NULL,\n `fflnowrntbnmhvaqueqkkezmmxkvzktf` int DEFAULT NULL,\n `ieugebyewpvktnuncatbhjvnopnigflb` int DEFAULT NULL,\n `eosoenvrgpoozieowpcxfrybqdewjxma` int DEFAULT NULL,\n `wcyuwfehghocbcmeostlhphnkisbeafq` int DEFAULT NULL,\n `ikuhlrbffzbrcbqsxkicvarlyforefsr` int DEFAULT NULL,\n `droaysiysozzzmafahxjvxtezloljlhd` int DEFAULT NULL,\n `bisnorlyyywrbuncyboxdkhfboskgzsa` int DEFAULT NULL,\n `gnoaukqoaljxrdjadmdmzqlgkjdfoybd` int DEFAULT NULL,\n `ebklpcxvnlajpiuekiigkvrnwafvqzbl` int DEFAULT NULL,\n `jqovsonwbljczoenmgvhmyhohsmwzhjd` int DEFAULT NULL,\n `fuuqqmjcpdxeniqhwvexkjyvweburjsk` int DEFAULT NULL,\n `dwncjfbbwiovhgqnbisksejbigmpzqzv` int DEFAULT NULL,\n `kfxrdvjjpzqjeylpudlvyjswdfgimsbt` int DEFAULT NULL,\n `ljtfrclwtbtyyapfdjvrzvocqwaiolsa` int DEFAULT NULL,\n `dtwmepikdhwzppubjvkspxdafodcxdjg` int DEFAULT NULL,\n `flipldmbcdktqnijrbffzvedncedjiev` int DEFAULT NULL,\n `cgiiamkgxyjvqaeynbqlmupoyjacyycq` int DEFAULT NULL,\n `uicjthollfjaattjnylqbuwqbjamigww` int DEFAULT NULL,\n `pqbblaabenopumlrfxpbfuekuildrshf` int DEFAULT NULL,\n `omtriwwzvzechciufxnzplmvmycffvex` int DEFAULT NULL,\n `lrsmiqqqcimhzxzrtbqboudnzyxsbien` int DEFAULT NULL,\n `oegxhibycuwvvwmbtwdtbsjjmrwicmam` int DEFAULT NULL,\n `vpovkkyjlyxbxspqlmflyonsfjujxqea` int DEFAULT NULL,\n `pbpuyuqfkptwnlrxhyzrqrnsotrfiytb` int DEFAULT NULL,\n `fcrqcerosqdhfvtlzhdlrqpejdpototn` int DEFAULT NULL,\n `kyuohaqotwxorxyukaiarkrmfxekbbbl` int DEFAULT NULL,\n `hovncavregqeonuzfquqffvkhhfwdvsn` int DEFAULT NULL,\n `jsftvhyjgluhneayxkffzswnphafeqef` int DEFAULT NULL,\n PRIMARY KEY (`dtpfwumwgexswomaynmjzenakmzoxfdn`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"ntdsamdzxoluyqsidfutsijfjlwpulzz\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["dtpfwumwgexswomaynmjzenakmzoxfdn"],"columns":[{"name":"dtpfwumwgexswomaynmjzenakmzoxfdn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"xfctvyrzfhqyffxqnjyqmjbsluexmkar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rntimauabkagzsbsvnahisqpvbitstih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ilqvsfpqylrrfnslojlyjofgwjqkbwiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aryjvgokmwrxytpgjdeqvpnwjvcuwbgr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxphsedvsjdejrwtwyrkfzdpjrwinsrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xohqbbzwcubeabpkextlrgdgqjgashtx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bxrmccbxmvtvbdwtjoxogrimaljpdcuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvddkynwjnsmdruhjkxnusjhmeyktlgr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"asritxnynrhddkreuirsztuvjcpgtdvv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqkszbybllvkiemzvxeujkygmfzhfngr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"puaxdumumiumjuazigvxlruznqifnlds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vakrmwejfainumfqjhipgdpecicchiyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aptxzzzeijxvrzgzpsmsrauqdunxbxvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijrrvzczqhzvbswjwndxksfdlzstjyia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bhmmjevajfhbahqrxoqvqbglisbucspo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xakccscajvyyudgqzvsqrdpjkdcehloj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxgfxczvzixyzvtwfknbhwqmxydbiklj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yrovtkgowkwacmspuqjvnmdzgjthamdz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmemqvqsmdnrmojjjzswxwvoojiuermf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxskwpldyayhmcwkimmnehiyzjdaxdvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wsaysscpkmhpxjkdztldbgkhfzfbgcez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjwwdxxyerpuldznemdkgqywlxscftbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqmqqgqoupoxvycnuogbipdeybftxqgz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmnirpracnzbtlfwmklolyzgvlisoevd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqrymiioairwrshsqyfcfbinasdmtvyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"anlezyaxpkwabssaftqeuocypaxlwjwh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"curkisshgwyetrmylxvqbpvlysrftsch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"trvxkecolaekkbcooewkjyynfypsynci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tgcvhcizqfpgspkoczxwpjawmrikbfmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrkgirnpfqvmmrrwmemdmeaaxwmfitiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkddcsnslhimahhrselmpwpfvlhsnafk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amjwtukevkhaoppojwqbzhlkfrsxopfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbxixgzfvygojijxwaxrrutuzwbmvtsq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"auhxxtwkgymzcswokfxvdtlptlxcanxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgmuldqhhptvdwkezryyvqickywkkifs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lrqytvjeurlgcjoezapjetmggcpjwjai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phtwmrfvwxrhndwybwvnvgogbfmkusjw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vwdpwcffmdqfrsfywclbnpvscxnemefs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vlhvaujlqlfqkyxryvdzlknfvgdsuuai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"annrryymefensmrttkqhbgiamaniepps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqzmnvgrqyllibujzcqcoomyarosltko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykzgnuamqdhpcymsdiyqqqdnyyuucwbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvjjqlrcansimgshxepagqusqdisoytl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdczgslqxeveyjrhmwcmwfwwuxwojsxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcoabydnbmniuwngnaddhrgrsgrddmqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ierkvdbyynqohfbacmpuaswowowrnntw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iexwtycfmymezncbanptuhxcvtbdfdzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxgqzpoosxoymezgnrqmukhwixmgdqdg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnbddljainngiiuqkaxarpxnkfnsfrws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfdggeylpyptyvydoccwfwcdltunwtkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"girhfmepguyqrukhmhkerxcneirnitvh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjobwibafjhkbdqlacmhualsyurnhpsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsqqgmzfsidlbxlktlqygdcomafhdgff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evcyryfkkkcqjgsrhnjepwjgbcnrjopg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubvpcrobtvxjvlikrvakgkgkcpuzzzxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jclmxvbndrcdxjlpzzvvtzdsuesfwint","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgfyvzsdcjjlwhpvitlmctvsiikzpfmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iujrkshaebhetdnonsioiiimvrlcwjpw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbukvlrsqjmxaaqudnfqsctsbsxkinvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"woejhhrkokoejhaqvgozjbznckhbgumr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgkrxmthdxhayxchaqdugjaqvabjulza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmcyftrafthsshugswfrcsjcmzclsaxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opjtfhienghzqughjdzwaehqpgbctfzr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nckmuofqhwdxbmgiqfvdfbixhwfikszu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjvudnosdkxcbuczvbzthidpzluqouyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lrxyzjezbzotizkejfpmhnupwhjagkpl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bhxnlychsjifztjfxgjmfybxgnkkbaea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsxmxluojykblewvjzjtmxgsprpcwexw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzsvyhitxforlywjzavcnwbhqoqqsugx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxsitxqjxvjikrnqwwdfclqxxtkfdaqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkrxkgwsoknxaygcdzpccdcyastzdosk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fflnowrntbnmhvaqueqkkezmmxkvzktf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ieugebyewpvktnuncatbhjvnopnigflb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eosoenvrgpoozieowpcxfrybqdewjxma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcyuwfehghocbcmeostlhphnkisbeafq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikuhlrbffzbrcbqsxkicvarlyforefsr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"droaysiysozzzmafahxjvxtezloljlhd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bisnorlyyywrbuncyboxdkhfboskgzsa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnoaukqoaljxrdjadmdmzqlgkjdfoybd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebklpcxvnlajpiuekiigkvrnwafvqzbl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jqovsonwbljczoenmgvhmyhohsmwzhjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fuuqqmjcpdxeniqhwvexkjyvweburjsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwncjfbbwiovhgqnbisksejbigmpzqzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfxrdvjjpzqjeylpudlvyjswdfgimsbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljtfrclwtbtyyapfdjvrzvocqwaiolsa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtwmepikdhwzppubjvkspxdafodcxdjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flipldmbcdktqnijrbffzvedncedjiev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgiiamkgxyjvqaeynbqlmupoyjacyycq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uicjthollfjaattjnylqbuwqbjamigww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqbblaabenopumlrfxpbfuekuildrshf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omtriwwzvzechciufxnzplmvmycffvex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lrsmiqqqcimhzxzrtbqboudnzyxsbien","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oegxhibycuwvvwmbtwdtbsjjmrwicmam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpovkkyjlyxbxspqlmflyonsfjujxqea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbpuyuqfkptwnlrxhyzrqrnsotrfiytb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcrqcerosqdhfvtlzhdlrqpejdpototn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kyuohaqotwxorxyukaiarkrmfxekbbbl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hovncavregqeonuzfquqffvkhhfwdvsn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jsftvhyjgluhneayxkffzswnphafeqef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669837,"databaseName":"models_schema","ddl":"CREATE TABLE `nxwatrwouueiyynkxlmdmddfodlkdsgt` (\n `mcytxuzbuzxbcycobevnagzmcpdlqpjf` int NOT NULL,\n `hzruefbumsfjxdtvhtrwkljaevbqqtxn` int DEFAULT NULL,\n `yzaiocmvsrbdwqhiskqmuheeyhdylbvx` int DEFAULT NULL,\n `glsxumgzeevbqcnztvnikcrtzfunbhxt` int DEFAULT NULL,\n `hubaikhvrgnxvptwvajcdcofmohfakkr` int DEFAULT NULL,\n `pnvvzzywnstcdzlouyreszxyknxfhqne` int DEFAULT NULL,\n `vjbgjnayyinosluywgbcokyggnutacao` int DEFAULT NULL,\n `wqdagvdlmulfpqhfjyzsflqxfxcmbbnh` int DEFAULT NULL,\n `tjwslfigbhvqdgdnxhtksxbqukynhtpp` int DEFAULT NULL,\n `tdvvwxokqxrzlrmouldpcmyziorltmjv` int DEFAULT NULL,\n `xgnuhzhxzppcsexilxmxmhrokkegkfut` int DEFAULT NULL,\n `nmuhdgtazydtcmohlffkjxtjjkjundss` int DEFAULT NULL,\n `uatwuiggxzeicwfzdscfyiqqywyvpkch` int DEFAULT NULL,\n `ixpauamixsfzapxbqplcohsfwzaullpg` int DEFAULT NULL,\n `sjqjaucyswzlkooiwpzoangbmknsnpht` int DEFAULT NULL,\n `cjjrvckttyswjnezwghsrrtogcjxbhjf` int DEFAULT NULL,\n `rzpqaujqvxyjnmsahrnmbkipqfrinrqv` int DEFAULT NULL,\n `xkxrzgrtxirstnjzuvrmkqsyihaggoxe` int DEFAULT NULL,\n `stpnnuggshnfrdzhmmanigvuwficnqvy` int DEFAULT NULL,\n `yajkehdwlmislmwgpmpwsixhdbrazjdk` int DEFAULT NULL,\n `uvalpmgcwtqpjgykdynolodfhufanhej` int DEFAULT NULL,\n `cnncmfqfbizwlfpghffscmmcvlbrqfts` int DEFAULT NULL,\n `luntuinpjkmgxpdetbbiukmbpsbhgvrq` int DEFAULT NULL,\n `hdibvoedxatknzsllmumzwdlhmijrneh` int DEFAULT NULL,\n `mowfjflvjqbatopqgjjdgsgbtazjauoi` int DEFAULT NULL,\n `xetqsuifvtbckhpcdkmuzzgoozscevso` int DEFAULT NULL,\n `vuvbxrsilaqotzltxlsrwfuivzvaanrr` int DEFAULT NULL,\n `kdtrhgosjwpiqiclqsaixbokkkpvwkjr` int DEFAULT NULL,\n `etpbioedzyuykpytzqttdgxlzmbbtvpu` int DEFAULT NULL,\n `yewfxqwylwmlyghcdafqiypyfageniko` int DEFAULT NULL,\n `oakzdctiizwlbmshvkuigkipilvbwydu` int DEFAULT NULL,\n `vbdbutxoyxftexkuzdimgznynctroojg` int DEFAULT NULL,\n `ivstodmurkyvwxfkhhaioewcivaehfun` int DEFAULT NULL,\n `hotrmehlcpristfytqzmunsbwrdtovgb` int DEFAULT NULL,\n `nrsncamchopfpinzibirirmvbszvkmzb` int DEFAULT NULL,\n `seemvygavpuzancxdljyxvitswujqdxe` int DEFAULT NULL,\n `grcafmnjeoodgnfqfddrhhvrmuwatuiv` int DEFAULT NULL,\n `chagkdxfiuruqdkgcokzdxmpsbcykmrb` int DEFAULT NULL,\n `qgyyipndgkzhjwugihbydiojmjpygqfm` int DEFAULT NULL,\n `gdhbbfhcpeeeplbezftdtgypvkufmbbr` int DEFAULT NULL,\n `eyvxhqbmitcfaeunkmfxtniykmzjfuvg` int DEFAULT NULL,\n `jwlkryidulexyzuoijbxfomhrrrwkkpq` int DEFAULT NULL,\n `iynlkcemklousywaztxhfwkhmldwzpkl` int DEFAULT NULL,\n `kcdsdnmyisxrtalhcdcukeicwvnoxwqw` int DEFAULT NULL,\n `dpsocnymcyvjfhpjbqpxjgsaoquwoxwc` int DEFAULT NULL,\n `evxwtcwbdpmdhdrhkjutbtbsfhlmrtec` int DEFAULT NULL,\n `fiekqpgbbmptuhdzeaqemgcrggcbztbj` int DEFAULT NULL,\n `omecjllaamtjquzwsgmlhtvrjtemiqzu` int DEFAULT NULL,\n `avwthvpvoamsablvzjgotushcwgiekrm` int DEFAULT NULL,\n `ouhgnckdmarckjyokoqsczgjfddvbaan` int DEFAULT NULL,\n `zriwryphrehhnmbljswuncjhdujjsxlo` int DEFAULT NULL,\n `mdfllhslokxkixfpvcpfyzgzisdqgkga` int DEFAULT NULL,\n `jmurocikmabsgwhrktltltnfupoernsp` int DEFAULT NULL,\n `pkizcnwvverpovrstunkbubopmsnenhi` int DEFAULT NULL,\n `ynexkumpjhfqycmgntxyhvlzzcvxcile` int DEFAULT NULL,\n `ioecqmmvhhskfnvgfzfqwwzzwnksiacu` int DEFAULT NULL,\n `ovlxukuyrctknixnxztolbsrbxzpfnfz` int DEFAULT NULL,\n `veavkrcfrqcwgrtjbgwidfnnrydrtomi` int DEFAULT NULL,\n `jvhikpaehwhfzuvaapsvuvrgrtyccuuz` int DEFAULT NULL,\n `zyppjmxkiwqcaiisxspodqemnjvrylov` int DEFAULT NULL,\n `qlhksyfyzbwxxaygdcmeeewkogzvwbgu` int DEFAULT NULL,\n `bcuedprqwuvjhkbysuhzsbcfztquistn` int DEFAULT NULL,\n `geyzanyrlusbuczwtcerkyeggeqfetwk` int DEFAULT NULL,\n `lstzitipqmujxjddfvgrxjosjnyctbgy` int DEFAULT NULL,\n `qgbmefgvwqanfqntfcjjujabmfivvrpp` int DEFAULT NULL,\n `gfxwpsmcstcaszgxdysxgxxhiqlrlhgm` int DEFAULT NULL,\n `abgabfzaggpppbilixjcpxomncyksoex` int DEFAULT NULL,\n `orwkfxayuywolrpuvqzqnzunqiyjdrko` int DEFAULT NULL,\n `jnjawlzfohbxojiirtdzsunzuvqsiddi` int DEFAULT NULL,\n `qqfjkzjrctbvkuappxkhibrnnktqfhph` int DEFAULT NULL,\n `byrqdnrkugavzhxxolkcaxnvvkiseivz` int DEFAULT NULL,\n `zylgcfmmyyylrzdtabprgiicxwrccewe` int DEFAULT NULL,\n `tdedjsedalvfympnimlbaehepikfrtvn` int DEFAULT NULL,\n `ndvxalmiucvzekxvjnsyjcfuqkljlerk` int DEFAULT NULL,\n `twuegskdhavirbwpbrjkbfqhleiaoqpy` int DEFAULT NULL,\n `idrrdqgayidxcxsqnnitbmzqkscuwrpd` int DEFAULT NULL,\n `wuacozbsgmytdvjexioobaqlgqtqtwxj` int DEFAULT NULL,\n `fxlrgryczinzvnzqwwbgpbzumurbckro` int DEFAULT NULL,\n `nnpgwgxdtbaflzpnqwmrvjmrrbgvjwyh` int DEFAULT NULL,\n `yokgvbrpikkhdgwxigogvwacedndkzhn` int DEFAULT NULL,\n `gnhmuynojorxkzklvidobtspifiyjagi` int DEFAULT NULL,\n `uosjdjnurzafzvzavxtjlfboghceifcg` int DEFAULT NULL,\n `idbglkoogsaqfzdzcerbnuhrytmftyqv` int DEFAULT NULL,\n `pwywaiskldshwovdvehyrwmqbwurasne` int DEFAULT NULL,\n `wxaixbswwqdwuxokislxcdbqxbzrlvhb` int DEFAULT NULL,\n `owsjgbxnojblziwdloqjosldqewcmoac` int DEFAULT NULL,\n `bdmoqkvrmfnkfhrgeksapkrghvyyvdgv` int DEFAULT NULL,\n `hgivsdkqzekjydfzkszjgyrurvvcmxue` int DEFAULT NULL,\n `stohgjfyhmkitaxlbxnnvmxuzdmrejur` int DEFAULT NULL,\n `iqzqqbmfnqnqztftugqdgvzatrnxpbeg` int DEFAULT NULL,\n `wjysjxrbzyqeufnebnycongxhbchqmmr` int DEFAULT NULL,\n `ggbtbmankqcsavbhesxdptunridhzyrx` int DEFAULT NULL,\n `ooztgjymoinjibdwqlndjruisuncvrwx` int DEFAULT NULL,\n `fsjeaeyphjqcizmrnvjswlmgvopfurqq` int DEFAULT NULL,\n `vmjtcegruqniksmlwabxpbumebouepuw` int DEFAULT NULL,\n `nngjzltrlruddgamymsjgiaqunorlyfp` int DEFAULT NULL,\n `kfwkhbdiyhilibsnfeauzfohzsqkyzjd` int DEFAULT NULL,\n `qkxsjuvlvythkicqaruwnwygtijwrmvu` int DEFAULT NULL,\n `subqsijyysgfnofdnklfxdppjrhtwjjs` int DEFAULT NULL,\n `ilnvxzeaynobhqpyblxkxdocarkbfkdq` int DEFAULT NULL,\n PRIMARY KEY (`mcytxuzbuzxbcycobevnagzmcpdlqpjf`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"nxwatrwouueiyynkxlmdmddfodlkdsgt\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["mcytxuzbuzxbcycobevnagzmcpdlqpjf"],"columns":[{"name":"mcytxuzbuzxbcycobevnagzmcpdlqpjf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"hzruefbumsfjxdtvhtrwkljaevbqqtxn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzaiocmvsrbdwqhiskqmuheeyhdylbvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"glsxumgzeevbqcnztvnikcrtzfunbhxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hubaikhvrgnxvptwvajcdcofmohfakkr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnvvzzywnstcdzlouyreszxyknxfhqne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjbgjnayyinosluywgbcokyggnutacao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqdagvdlmulfpqhfjyzsflqxfxcmbbnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjwslfigbhvqdgdnxhtksxbqukynhtpp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdvvwxokqxrzlrmouldpcmyziorltmjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xgnuhzhxzppcsexilxmxmhrokkegkfut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmuhdgtazydtcmohlffkjxtjjkjundss","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uatwuiggxzeicwfzdscfyiqqywyvpkch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixpauamixsfzapxbqplcohsfwzaullpg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjqjaucyswzlkooiwpzoangbmknsnpht","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjjrvckttyswjnezwghsrrtogcjxbhjf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzpqaujqvxyjnmsahrnmbkipqfrinrqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkxrzgrtxirstnjzuvrmkqsyihaggoxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"stpnnuggshnfrdzhmmanigvuwficnqvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yajkehdwlmislmwgpmpwsixhdbrazjdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvalpmgcwtqpjgykdynolodfhufanhej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnncmfqfbizwlfpghffscmmcvlbrqfts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"luntuinpjkmgxpdetbbiukmbpsbhgvrq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdibvoedxatknzsllmumzwdlhmijrneh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mowfjflvjqbatopqgjjdgsgbtazjauoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xetqsuifvtbckhpcdkmuzzgoozscevso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vuvbxrsilaqotzltxlsrwfuivzvaanrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kdtrhgosjwpiqiclqsaixbokkkpvwkjr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"etpbioedzyuykpytzqttdgxlzmbbtvpu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yewfxqwylwmlyghcdafqiypyfageniko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oakzdctiizwlbmshvkuigkipilvbwydu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbdbutxoyxftexkuzdimgznynctroojg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivstodmurkyvwxfkhhaioewcivaehfun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hotrmehlcpristfytqzmunsbwrdtovgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nrsncamchopfpinzibirirmvbszvkmzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"seemvygavpuzancxdljyxvitswujqdxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grcafmnjeoodgnfqfddrhhvrmuwatuiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chagkdxfiuruqdkgcokzdxmpsbcykmrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgyyipndgkzhjwugihbydiojmjpygqfm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdhbbfhcpeeeplbezftdtgypvkufmbbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eyvxhqbmitcfaeunkmfxtniykmzjfuvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwlkryidulexyzuoijbxfomhrrrwkkpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iynlkcemklousywaztxhfwkhmldwzpkl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcdsdnmyisxrtalhcdcukeicwvnoxwqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpsocnymcyvjfhpjbqpxjgsaoquwoxwc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evxwtcwbdpmdhdrhkjutbtbsfhlmrtec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fiekqpgbbmptuhdzeaqemgcrggcbztbj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omecjllaamtjquzwsgmlhtvrjtemiqzu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avwthvpvoamsablvzjgotushcwgiekrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ouhgnckdmarckjyokoqsczgjfddvbaan","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zriwryphrehhnmbljswuncjhdujjsxlo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdfllhslokxkixfpvcpfyzgzisdqgkga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmurocikmabsgwhrktltltnfupoernsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkizcnwvverpovrstunkbubopmsnenhi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynexkumpjhfqycmgntxyhvlzzcvxcile","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ioecqmmvhhskfnvgfzfqwwzzwnksiacu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovlxukuyrctknixnxztolbsrbxzpfnfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"veavkrcfrqcwgrtjbgwidfnnrydrtomi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvhikpaehwhfzuvaapsvuvrgrtyccuuz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyppjmxkiwqcaiisxspodqemnjvrylov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qlhksyfyzbwxxaygdcmeeewkogzvwbgu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bcuedprqwuvjhkbysuhzsbcfztquistn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"geyzanyrlusbuczwtcerkyeggeqfetwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lstzitipqmujxjddfvgrxjosjnyctbgy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgbmefgvwqanfqntfcjjujabmfivvrpp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfxwpsmcstcaszgxdysxgxxhiqlrlhgm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abgabfzaggpppbilixjcpxomncyksoex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orwkfxayuywolrpuvqzqnzunqiyjdrko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jnjawlzfohbxojiirtdzsunzuvqsiddi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqfjkzjrctbvkuappxkhibrnnktqfhph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"byrqdnrkugavzhxxolkcaxnvvkiseivz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zylgcfmmyyylrzdtabprgiicxwrccewe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdedjsedalvfympnimlbaehepikfrtvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ndvxalmiucvzekxvjnsyjcfuqkljlerk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twuegskdhavirbwpbrjkbfqhleiaoqpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idrrdqgayidxcxsqnnitbmzqkscuwrpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wuacozbsgmytdvjexioobaqlgqtqtwxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxlrgryczinzvnzqwwbgpbzumurbckro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nnpgwgxdtbaflzpnqwmrvjmrrbgvjwyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yokgvbrpikkhdgwxigogvwacedndkzhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnhmuynojorxkzklvidobtspifiyjagi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uosjdjnurzafzvzavxtjlfboghceifcg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idbglkoogsaqfzdzcerbnuhrytmftyqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwywaiskldshwovdvehyrwmqbwurasne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxaixbswwqdwuxokislxcdbqxbzrlvhb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owsjgbxnojblziwdloqjosldqewcmoac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdmoqkvrmfnkfhrgeksapkrghvyyvdgv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hgivsdkqzekjydfzkszjgyrurvvcmxue","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"stohgjfyhmkitaxlbxnnvmxuzdmrejur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqzqqbmfnqnqztftugqdgvzatrnxpbeg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjysjxrbzyqeufnebnycongxhbchqmmr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggbtbmankqcsavbhesxdptunridhzyrx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ooztgjymoinjibdwqlndjruisuncvrwx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fsjeaeyphjqcizmrnvjswlmgvopfurqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmjtcegruqniksmlwabxpbumebouepuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nngjzltrlruddgamymsjgiaqunorlyfp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfwkhbdiyhilibsnfeauzfohzsqkyzjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qkxsjuvlvythkicqaruwnwygtijwrmvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"subqsijyysgfnofdnklfxdppjrhtwjjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ilnvxzeaynobhqpyblxkxdocarkbfkdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669867,"databaseName":"models_schema","ddl":"CREATE TABLE `nyhwscsvzxplglrwnsgvsfidffshzfly` (\n `llxaxyicxjerzlfurausvewmmcxwtzsu` int NOT NULL,\n `kovcnjsiuasyefkpuzeviinfshdlqdtf` int DEFAULT NULL,\n `bqnbkecithsfhlcamnowwskvmechicat` int DEFAULT NULL,\n `iksetltlzzvoiaifjmwxivxhiwxaqbic` int DEFAULT NULL,\n `tnofctijzewodjewwjkagngshtlolpuh` int DEFAULT NULL,\n `hjhqsjylslinhkqufdxbrhieguexoxsl` int DEFAULT NULL,\n `vukfbyqbbkgqdccnieuowsbsfovfsijx` int DEFAULT NULL,\n `kpnysgwtgcbfozpxsanxcrupoisaepxr` int DEFAULT NULL,\n `ncvfzzytgccepifegeylyhbumyrhpahn` int DEFAULT NULL,\n `znspfofvychkkhchulosckoecqrdarof` int DEFAULT NULL,\n `pxazdfabpuaczpvoeeuddyzikqghwbtr` int DEFAULT NULL,\n `lzgkeppedkesmgndjedabuiggeovrkpb` int DEFAULT NULL,\n `tnipitshgmfjxdaqerppirviujhtaqrg` int DEFAULT NULL,\n `lfooqzajdybosutqbwstsmtscpvttraf` int DEFAULT NULL,\n `dxxsyttbowiyjeyjhnkqiuednbxgzdro` int DEFAULT NULL,\n `xdualsrscaspjecrevhwmtlnvuxttclc` int DEFAULT NULL,\n `wzrbxsdrzjphjgxpazikzexzvbopvbyy` int DEFAULT NULL,\n `coxukloovgqikydmuegaotxlbhcqnvia` int DEFAULT NULL,\n `fpcoqbetegqjzmmccszikmtvsnlaquri` int DEFAULT NULL,\n `luwxakonecsimoykswvyqsizdetnmyig` int DEFAULT NULL,\n `lzyaeallbqwbkigrmprasalvomwjpplu` int DEFAULT NULL,\n `uvbbnpokpfzbbnrigjsezvynykytxeaa` int DEFAULT NULL,\n `tzxsdsyloyxryceefjdjrjysbjnzwfqa` int DEFAULT NULL,\n `xnyuguwaifmyudbiuamhpsnhrkprgiso` int DEFAULT NULL,\n `kvdyumbgfngflxrfckkjwdesqjhrfetw` int DEFAULT NULL,\n `siqzbfbvvvbidyyjgbgefiqjbchdevso` int DEFAULT NULL,\n `cprrqbcwrrfntbuvvwxmplhrkuhngdml` int DEFAULT NULL,\n `mkxcvihblocneoxfjyerizwhmxbexsnb` int DEFAULT NULL,\n `wlakvdrindoblhqmjtvwmfjmrydpzsmp` int DEFAULT NULL,\n `urchksmwueapnbdkupzxavoqljsnqzwo` int DEFAULT NULL,\n `aqwrrxgqsdltfswvajjhrozfszndubsl` int DEFAULT NULL,\n `bnsroqsijzrcmuhvivczctpjzjflrrex` int DEFAULT NULL,\n `ofqoaqaoaddcgkiqmanlszckcykfijtp` int DEFAULT NULL,\n `vmealzsofyoahbcvcyesgsyravoydvpg` int DEFAULT NULL,\n `rnljznsjfgsenjdthowqpeiqhasikkky` int DEFAULT NULL,\n `bmctpaikfcrekbbxxidsjuykhmueliya` int DEFAULT NULL,\n `lpfjqtatrpwhhzupwxkpsbqdbxovinch` int DEFAULT NULL,\n `smbwddbwjcjrubszhfgjzvsjptfgmbxy` int DEFAULT NULL,\n `ejblmetjxilbmgruhizjjhebysubuumo` int DEFAULT NULL,\n `tniighngghzimcrhrqedzgdabcoxpkxx` int DEFAULT NULL,\n `nbfajgeeefruesugxwgdetmkakplisbe` int DEFAULT NULL,\n `ilifwyqqqpybcnycbmsqzpwmyeycyfwi` int DEFAULT NULL,\n `iguotwrkguplvheeefkroketmvicpnwf` int DEFAULT NULL,\n `urtsgciqrtwwlwuclzoqrngaixrylqyt` int DEFAULT NULL,\n `ezyidnehynbxoktobabpkabjzqotaazn` int DEFAULT NULL,\n `hjfaozkgutkfrupaokfkdmbjwjuhxsxo` int DEFAULT NULL,\n `gcuyrbmwticghkvbqggkxrrkljygrfpj` int DEFAULT NULL,\n `ubuljmagnukchxpioowwwkxbvlxnfhun` int DEFAULT NULL,\n `wudmefcwvrrcnwvqwdwbgccauuppgont` int DEFAULT NULL,\n `wzzbivhqwfcpwzkgxbpujhpboumxrgis` int DEFAULT NULL,\n `fqruguvkfwqsfkezqhwbqtswtgfcyptn` int DEFAULT NULL,\n `rtrxrorntxyjaxtfmbrpastpjzlwgejo` int DEFAULT NULL,\n `naqnsnextgecpunrjmkjvvsssjilqezr` int DEFAULT NULL,\n `qzkxhpcclpupdvmgqvugyuvjbhhrvupb` int DEFAULT NULL,\n `oblmdkczirnsusqvyscnrdwhobyjxwia` int DEFAULT NULL,\n `gcvhviablmybrrsaodxsspidglhswbia` int DEFAULT NULL,\n `sevwwicsqnqkbgfjpvtkakpmdipwmqkw` int DEFAULT NULL,\n `axtaucsozacsmuxlbhsixhasnmejtddg` int DEFAULT NULL,\n `clucyogttaegyybdtnulfyldmgpovuuy` int DEFAULT NULL,\n `mfpqeihcpgygqyacpoekakdipgpbeoat` int DEFAULT NULL,\n `hvajytxerskxrxrdemwjdpqhomzgjfbj` int DEFAULT NULL,\n `wbwiandztrtsezzlcnqnprjfqanisyfr` int DEFAULT NULL,\n `ccxberpoimufcdzlrvqfxobfwjpigkcd` int DEFAULT NULL,\n `rbqmxjtyzxchujxnkzkwlksxzdaepcwv` int DEFAULT NULL,\n `fbmtlwafnlejwsvjjaxelxxvxffsftfe` int DEFAULT NULL,\n `lahklhrptgcmesyqudktehqbzlnezemh` int DEFAULT NULL,\n `awgeiiogumuriyxjsurbabugpqnbwrsr` int DEFAULT NULL,\n `pbotnnknmvwzpdjbvoemeupcagdjkafc` int DEFAULT NULL,\n `auhxqpprhbdcwwzelpeacvbzcgmwzypc` int DEFAULT NULL,\n `pkihmcbbohvsphisfmjouzwapybdtkij` int DEFAULT NULL,\n `hbrpwbukqfazgsnbucthgjpiwjvqyget` int DEFAULT NULL,\n `hyslxahhkvtqpgmvaoyaeycfpqdxgocg` int DEFAULT NULL,\n `jlngczzlabwiqbeucnycvmbiofanrwzq` int DEFAULT NULL,\n `jdgjtodviuymkkdgqalwhsysfkxoebts` int DEFAULT NULL,\n `nbjvsawxpdvaqkkhsnkwhrrpipvoqiiq` int DEFAULT NULL,\n `suhqguuggkwijvumjsncsvzcbfgyahwx` int DEFAULT NULL,\n `iwhywrlftgkzyzgpiatrfygfknyieaoi` int DEFAULT NULL,\n `zbfuurgitccrbgddvxbtcpbozdkmdnqo` int DEFAULT NULL,\n `wddfcjqymaglalyakdnbnacaeroonmpk` int DEFAULT NULL,\n `rugcxrvmrrojbuzyvrekgrftfugdipin` int DEFAULT NULL,\n `biyqrrgqyvydtqozbtldsapdihhooafj` int DEFAULT NULL,\n `xkbwereetijrtizvkdnytttsendgmyuc` int DEFAULT NULL,\n `tojomehsosgaejypinfsmykvnhomrssw` int DEFAULT NULL,\n `uunzgolgvfocchptjatxslmbyqouknmz` int DEFAULT NULL,\n `hguononhdcweawbtwogrtpjpvsbaokrl` int DEFAULT NULL,\n `efqmrzhxmhetqzgrzpuhygpwdyievqzf` int DEFAULT NULL,\n `owlgzdggacziskbjkcaorcbxscgvlkis` int DEFAULT NULL,\n `vpfwizdbczxispllxtgnbwsgwkgvzced` int DEFAULT NULL,\n `jhpodrfwgjzqlkhrganfgpwnhsvjzarj` int DEFAULT NULL,\n `hpwpxjoxkbtfcyjnczvruidawoiwdwjz` int DEFAULT NULL,\n `mocgkuvtgpqkktebneljarawajvtdrbk` int DEFAULT NULL,\n `hytbtbhfaxqkbjlwunjgmqlghaqkseku` int DEFAULT NULL,\n `iwrtoctztrqugfnwawhwvavyefynxenm` int DEFAULT NULL,\n `xssitqjguhxisxrolzoeltwjixerutli` int DEFAULT NULL,\n `earezyksfafkivzevwwxtzravbenebnt` int DEFAULT NULL,\n `vtytyejbuijjpccgakabvssszbahjbzq` int DEFAULT NULL,\n `wseftzfwlraucmrqmsewdmcmqwphmbfr` int DEFAULT NULL,\n `nacuekibqxxxjxazlnorvyiohreglbaj` int DEFAULT NULL,\n `kpzotskfmjallyuioyphyszqpmfcdbhu` int DEFAULT NULL,\n `gqsgtpihkpdswovneojnckycgbvijbtm` int DEFAULT NULL,\n PRIMARY KEY (`llxaxyicxjerzlfurausvewmmcxwtzsu`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"nyhwscsvzxplglrwnsgvsfidffshzfly\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["llxaxyicxjerzlfurausvewmmcxwtzsu"],"columns":[{"name":"llxaxyicxjerzlfurausvewmmcxwtzsu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"kovcnjsiuasyefkpuzeviinfshdlqdtf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqnbkecithsfhlcamnowwskvmechicat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iksetltlzzvoiaifjmwxivxhiwxaqbic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnofctijzewodjewwjkagngshtlolpuh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjhqsjylslinhkqufdxbrhieguexoxsl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vukfbyqbbkgqdccnieuowsbsfovfsijx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpnysgwtgcbfozpxsanxcrupoisaepxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ncvfzzytgccepifegeylyhbumyrhpahn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znspfofvychkkhchulosckoecqrdarof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxazdfabpuaczpvoeeuddyzikqghwbtr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzgkeppedkesmgndjedabuiggeovrkpb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnipitshgmfjxdaqerppirviujhtaqrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfooqzajdybosutqbwstsmtscpvttraf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxxsyttbowiyjeyjhnkqiuednbxgzdro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdualsrscaspjecrevhwmtlnvuxttclc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzrbxsdrzjphjgxpazikzexzvbopvbyy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"coxukloovgqikydmuegaotxlbhcqnvia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpcoqbetegqjzmmccszikmtvsnlaquri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"luwxakonecsimoykswvyqsizdetnmyig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzyaeallbqwbkigrmprasalvomwjpplu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvbbnpokpfzbbnrigjsezvynykytxeaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tzxsdsyloyxryceefjdjrjysbjnzwfqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnyuguwaifmyudbiuamhpsnhrkprgiso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvdyumbgfngflxrfckkjwdesqjhrfetw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"siqzbfbvvvbidyyjgbgefiqjbchdevso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cprrqbcwrrfntbuvvwxmplhrkuhngdml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkxcvihblocneoxfjyerizwhmxbexsnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wlakvdrindoblhqmjtvwmfjmrydpzsmp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urchksmwueapnbdkupzxavoqljsnqzwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqwrrxgqsdltfswvajjhrozfszndubsl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnsroqsijzrcmuhvivczctpjzjflrrex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofqoaqaoaddcgkiqmanlszckcykfijtp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmealzsofyoahbcvcyesgsyravoydvpg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnljznsjfgsenjdthowqpeiqhasikkky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bmctpaikfcrekbbxxidsjuykhmueliya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lpfjqtatrpwhhzupwxkpsbqdbxovinch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smbwddbwjcjrubszhfgjzvsjptfgmbxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejblmetjxilbmgruhizjjhebysubuumo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tniighngghzimcrhrqedzgdabcoxpkxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbfajgeeefruesugxwgdetmkakplisbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ilifwyqqqpybcnycbmsqzpwmyeycyfwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iguotwrkguplvheeefkroketmvicpnwf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urtsgciqrtwwlwuclzoqrngaixrylqyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezyidnehynbxoktobabpkabjzqotaazn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjfaozkgutkfrupaokfkdmbjwjuhxsxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gcuyrbmwticghkvbqggkxrrkljygrfpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubuljmagnukchxpioowwwkxbvlxnfhun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wudmefcwvrrcnwvqwdwbgccauuppgont","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzzbivhqwfcpwzkgxbpujhpboumxrgis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqruguvkfwqsfkezqhwbqtswtgfcyptn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtrxrorntxyjaxtfmbrpastpjzlwgejo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"naqnsnextgecpunrjmkjvvsssjilqezr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzkxhpcclpupdvmgqvugyuvjbhhrvupb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oblmdkczirnsusqvyscnrdwhobyjxwia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gcvhviablmybrrsaodxsspidglhswbia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sevwwicsqnqkbgfjpvtkakpmdipwmqkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axtaucsozacsmuxlbhsixhasnmejtddg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clucyogttaegyybdtnulfyldmgpovuuy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfpqeihcpgygqyacpoekakdipgpbeoat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hvajytxerskxrxrdemwjdpqhomzgjfbj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbwiandztrtsezzlcnqnprjfqanisyfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ccxberpoimufcdzlrvqfxobfwjpigkcd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbqmxjtyzxchujxnkzkwlksxzdaepcwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbmtlwafnlejwsvjjaxelxxvxffsftfe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lahklhrptgcmesyqudktehqbzlnezemh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awgeiiogumuriyxjsurbabugpqnbwrsr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbotnnknmvwzpdjbvoemeupcagdjkafc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"auhxqpprhbdcwwzelpeacvbzcgmwzypc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkihmcbbohvsphisfmjouzwapybdtkij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbrpwbukqfazgsnbucthgjpiwjvqyget","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hyslxahhkvtqpgmvaoyaeycfpqdxgocg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlngczzlabwiqbeucnycvmbiofanrwzq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdgjtodviuymkkdgqalwhsysfkxoebts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbjvsawxpdvaqkkhsnkwhrrpipvoqiiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suhqguuggkwijvumjsncsvzcbfgyahwx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwhywrlftgkzyzgpiatrfygfknyieaoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbfuurgitccrbgddvxbtcpbozdkmdnqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wddfcjqymaglalyakdnbnacaeroonmpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rugcxrvmrrojbuzyvrekgrftfugdipin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"biyqrrgqyvydtqozbtldsapdihhooafj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkbwereetijrtizvkdnytttsendgmyuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tojomehsosgaejypinfsmykvnhomrssw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uunzgolgvfocchptjatxslmbyqouknmz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hguononhdcweawbtwogrtpjpvsbaokrl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"efqmrzhxmhetqzgrzpuhygpwdyievqzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owlgzdggacziskbjkcaorcbxscgvlkis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpfwizdbczxispllxtgnbwsgwkgvzced","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jhpodrfwgjzqlkhrganfgpwnhsvjzarj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hpwpxjoxkbtfcyjnczvruidawoiwdwjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mocgkuvtgpqkktebneljarawajvtdrbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hytbtbhfaxqkbjlwunjgmqlghaqkseku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwrtoctztrqugfnwawhwvavyefynxenm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xssitqjguhxisxrolzoeltwjixerutli","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"earezyksfafkivzevwwxtzravbenebnt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtytyejbuijjpccgakabvssszbahjbzq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wseftzfwlraucmrqmsewdmcmqwphmbfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nacuekibqxxxjxazlnorvyiohreglbaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpzotskfmjallyuioyphyszqpmfcdbhu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqsgtpihkpdswovneojnckycgbvijbtm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669901,"databaseName":"models_schema","ddl":"CREATE TABLE `nyrgogpbmkyjitqgmdttnyaxvmfppyld` (\n `foxepsvxtajfkwrgcmxrbimowcgpgwsm` int NOT NULL,\n `peyybeodhvpuyrtgozbwfoyeojuinsuc` int DEFAULT NULL,\n `kdstddnexziadkgykgpturuseyoutigy` int DEFAULT NULL,\n `wvqwnmuryypegjyxbzqgtbxteqjtkjex` int DEFAULT NULL,\n `tudqskpewnpwpxdivgfayatoqnkskhgk` int DEFAULT NULL,\n `ljkesgapzhitijtirlreecrgmojeomsc` int DEFAULT NULL,\n `hkeyxmriyaxekxvyrzitwjtxjobckjkr` int DEFAULT NULL,\n `xgcaueifksyqiwzjqbfvnrxskbfvwihw` int DEFAULT NULL,\n `ahqhucvzazebhbpfsxdwzcqpylxbhajx` int DEFAULT NULL,\n `nhzmlaumzboajqnipyvbebuashtpveui` int DEFAULT NULL,\n `pvavkngssuvkukexiedmmivyinkwgyse` int DEFAULT NULL,\n `evmhkixoeekfgjojmfjgqzjuxxiwimdm` int DEFAULT NULL,\n `ajhhgvotwqljcxenpzwoziuwcedtcjml` int DEFAULT NULL,\n `ousnrlzmsxzqvxehzmxzmxumahhdfjdm` int DEFAULT NULL,\n `rjmqmvuqdccrzgliyoujdrqaocfauwww` int DEFAULT NULL,\n `rwpodptofjblthkqqrhamcbrhhlhmuln` int DEFAULT NULL,\n `oemmxheoifpahjmdvnkvfqxhpwvgxnlz` int DEFAULT NULL,\n `ausegpnopdcwdrpszukbfwxelmicuzgy` int DEFAULT NULL,\n `tifdzcxfptdnytiutlshppkfanniedtl` int DEFAULT NULL,\n `wvybboihpzvpotpftoefdvehvxlcjroa` int DEFAULT NULL,\n `zjhgrgplnaflgtxojszgzppqxmaatcdu` int DEFAULT NULL,\n `kansruxyacrcumzcdzadstyiepudywil` int DEFAULT NULL,\n `ykkpfgwjdjqdpuqjmcmxbryvajblqamc` int DEFAULT NULL,\n `bklyygxjzzryqnfzkywiiytwtnvcstlf` int DEFAULT NULL,\n `prgkglvrmcgsujdzvspzwqknfgdksuha` int DEFAULT NULL,\n `kxyuligzfwhdfhtdkaiuyafrtvfxnqav` int DEFAULT NULL,\n `pmbrzfkdozidfsgctofjhcstjpqpmwuf` int DEFAULT NULL,\n `xelquouudbwhmfjfqqbkkbqzqfqbybha` int DEFAULT NULL,\n `ryyskrmdzmmcexvhszdrkvzhygrtntfh` int DEFAULT NULL,\n `qscizwrdcqkrrdyaatcvzrpfkaferzsa` int DEFAULT NULL,\n `bgcztbcxcsupkzrpeywwdcknrczhskxx` int DEFAULT NULL,\n `zhkfbkcotuoyklohnreuangfuaheyfoe` int DEFAULT NULL,\n `cobpienhsyhxvrvxrjtrrszmvjahnuip` int DEFAULT NULL,\n `ykdhgeghytfbbwggdrjznenjldapmbwj` int DEFAULT NULL,\n `rikoxvwvytdxkrazaqvdwoeijlhiwzmg` int DEFAULT NULL,\n `dqsrbtqslpsvsmwxryczbvsexshsyezx` int DEFAULT NULL,\n `yyzlpdjqbhputxeodfegthkoshlybouh` int DEFAULT NULL,\n `rjexadpynjcdzncdjfnbeyqjjndlndwp` int DEFAULT NULL,\n `bqwxdhavwschkskuzewjkpspwzdtenuq` int DEFAULT NULL,\n `honiamnaexcaygkyrglkrjcgrzjlhwhh` int DEFAULT NULL,\n `offtznutwgwoydbcvolnzhnndoymuxqe` int DEFAULT NULL,\n `bqqogcrxjvzdmdxedjmadszkniybofxv` int DEFAULT NULL,\n `pdxgfpikafltlourvpirscygznbaqssq` int DEFAULT NULL,\n `fnaxbrlijngizcaeyxgbhchruowspmso` int DEFAULT NULL,\n `zmtqrtzszvgwhjdmnpkipoaqexockbmd` int DEFAULT NULL,\n `rxfcxajnigzxagoofagbrqqxwyejkmzj` int DEFAULT NULL,\n `gesebrbjrhhmyoadphyzaocknqaluooc` int DEFAULT NULL,\n `mzachgfywvbiflbexduzhjaojrnkpoix` int DEFAULT NULL,\n `doltzuwoyduqwojdqcrynuhxfirfwvkc` int DEFAULT NULL,\n `acaookgldzddaartuofhpqwpbiobyznu` int DEFAULT NULL,\n `hecwnhvxvrjjrzhnjswdinvspuxlcnzv` int DEFAULT NULL,\n `rzbzlverctrvevudxalusyjeqbdlkzfl` int DEFAULT NULL,\n `ksfgqrcokcdoieyoyodbypltitrnsnct` int DEFAULT NULL,\n `eoxnexhsqeycpeseypiuhzzadsxwvgfs` int DEFAULT NULL,\n `mqopupgqbijvksvhopqjdiqjazdhvooa` int DEFAULT NULL,\n `euqxoxrgciobadjemtoyjdovrcpxuvpj` int DEFAULT NULL,\n `lxjmmwqjyfumgurscanxsioisqsnpsvh` int DEFAULT NULL,\n `geqlqzmnxqmwvqibfozetrgiuvexhync` int DEFAULT NULL,\n `ntvchzxtrgrmbopeysntlindpufcnxau` int DEFAULT NULL,\n `xjlumbctvgnfbpcwozefcdvwkgtyxlox` int DEFAULT NULL,\n `eyynuxgyzzqjgeriydbxumgfnrlxngey` int DEFAULT NULL,\n `hhqklyjdpxiyluougsotimxymvblgwbk` int DEFAULT NULL,\n `rwwbvrmwurdgspltglgxvrryftwswuti` int DEFAULT NULL,\n `qardftkaunobzmhwnojkxzerwfzmyrmt` int DEFAULT NULL,\n `rxthdlfnlgbcmnilmwvrsjelqnigijkw` int DEFAULT NULL,\n `jicckblczdzzzapihcfxoacleufgniqm` int DEFAULT NULL,\n `sejpzvmcyawyuqzmzlpekxhbitckrcip` int DEFAULT NULL,\n `qlvzdnmsmkcxhakrgyboygkmzqmldbwb` int DEFAULT NULL,\n `ptjwtmybpbkjrapwgpkpzttefplrafog` int DEFAULT NULL,\n `zgpiziygxccdfvhtegcxijojaalritpy` int DEFAULT NULL,\n `cwdvavuhwzrwdvwwmoccbcuqjfrrlyak` int DEFAULT NULL,\n `zhjbjlzbjusfwubjjnbbfxqenqptzinx` int DEFAULT NULL,\n `rbroipropllbnghltbphwheoshufkhxv` int DEFAULT NULL,\n `whwcijtzlsiisihwuzyuhiqxybiobudj` int DEFAULT NULL,\n `rtlbzwnwrpoopsatksnimqiqjashwwyw` int DEFAULT NULL,\n `kwuqonkbhofpglamqddgfjdianmaqfdy` int DEFAULT NULL,\n `cibpcmxolrvcyiibfoldytpootwtfjbo` int DEFAULT NULL,\n `hccdrmyguplpkcnatwgjyaeprclfmcwt` int DEFAULT NULL,\n `jgxudbidnzdkuctubbzbrpcbnvqeolol` int DEFAULT NULL,\n `khvfrjcbmdamqoowvdslqkqkfvatghil` int DEFAULT NULL,\n `ewnkjqxxgxysopzcrhzjtjcsuoyyybyf` int DEFAULT NULL,\n `wtewosikdrnxmsemlyzudotxvfgoeipq` int DEFAULT NULL,\n `qbvsydvnetucaponcypirpjowibdpyjo` int DEFAULT NULL,\n `uomgnhpockxnlclowusahhtjsuxsvkzj` int DEFAULT NULL,\n `ubvdyrzqdhxjzxbncwgkhahfgentppae` int DEFAULT NULL,\n `qjmfkdehubjkvkwycqvbknnmzleexhmp` int DEFAULT NULL,\n `kcjpxqaswvuawzjqntyqgkbxpbfljzai` int DEFAULT NULL,\n `eocqiogfdrlufkrcnysmwwxvscaoxfto` int DEFAULT NULL,\n `wlldzmgavdjwcafzkwyskypbliizjgzs` int DEFAULT NULL,\n `ssdkqidhtuuoeqpnxrjwxdjheipohsxx` int DEFAULT NULL,\n `jcuvprnfcudsnsonmaysknpdfgsdlvay` int DEFAULT NULL,\n `prahkhmlgwfedjkmvgkvepadrftedftf` int DEFAULT NULL,\n `xiyytddfrmrwmcpocnwhmrklxbcsntze` int DEFAULT NULL,\n `yjvdbahskcqujcfmmkgsnhaeycovalzk` int DEFAULT NULL,\n `vqqspjzuuugnlwoaciegcxyynfkozyfw` int DEFAULT NULL,\n `btftkuorodusrkcziwkhasskfwnnirvd` int DEFAULT NULL,\n `wqnfpgzxjwggsuvbhidgkwqpccrbfpkb` int DEFAULT NULL,\n `cxksauwpurexwwdaihlxnsyvqobmlcvq` int DEFAULT NULL,\n `fbkccmlffxggquftcckgzpobfomacjjh` int DEFAULT NULL,\n `jqeksfdxtmtsrvhgeqdyyzexeevsciwr` int DEFAULT NULL,\n PRIMARY KEY (`foxepsvxtajfkwrgcmxrbimowcgpgwsm`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"nyrgogpbmkyjitqgmdttnyaxvmfppyld\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["foxepsvxtajfkwrgcmxrbimowcgpgwsm"],"columns":[{"name":"foxepsvxtajfkwrgcmxrbimowcgpgwsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"peyybeodhvpuyrtgozbwfoyeojuinsuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kdstddnexziadkgykgpturuseyoutigy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvqwnmuryypegjyxbzqgtbxteqjtkjex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tudqskpewnpwpxdivgfayatoqnkskhgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljkesgapzhitijtirlreecrgmojeomsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkeyxmriyaxekxvyrzitwjtxjobckjkr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xgcaueifksyqiwzjqbfvnrxskbfvwihw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahqhucvzazebhbpfsxdwzcqpylxbhajx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhzmlaumzboajqnipyvbebuashtpveui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvavkngssuvkukexiedmmivyinkwgyse","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evmhkixoeekfgjojmfjgqzjuxxiwimdm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajhhgvotwqljcxenpzwoziuwcedtcjml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ousnrlzmsxzqvxehzmxzmxumahhdfjdm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjmqmvuqdccrzgliyoujdrqaocfauwww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rwpodptofjblthkqqrhamcbrhhlhmuln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oemmxheoifpahjmdvnkvfqxhpwvgxnlz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ausegpnopdcwdrpszukbfwxelmicuzgy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tifdzcxfptdnytiutlshppkfanniedtl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvybboihpzvpotpftoefdvehvxlcjroa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjhgrgplnaflgtxojszgzppqxmaatcdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kansruxyacrcumzcdzadstyiepudywil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykkpfgwjdjqdpuqjmcmxbryvajblqamc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bklyygxjzzryqnfzkywiiytwtnvcstlf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prgkglvrmcgsujdzvspzwqknfgdksuha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxyuligzfwhdfhtdkaiuyafrtvfxnqav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmbrzfkdozidfsgctofjhcstjpqpmwuf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xelquouudbwhmfjfqqbkkbqzqfqbybha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ryyskrmdzmmcexvhszdrkvzhygrtntfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qscizwrdcqkrrdyaatcvzrpfkaferzsa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bgcztbcxcsupkzrpeywwdcknrczhskxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhkfbkcotuoyklohnreuangfuaheyfoe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cobpienhsyhxvrvxrjtrrszmvjahnuip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykdhgeghytfbbwggdrjznenjldapmbwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rikoxvwvytdxkrazaqvdwoeijlhiwzmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqsrbtqslpsvsmwxryczbvsexshsyezx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yyzlpdjqbhputxeodfegthkoshlybouh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjexadpynjcdzncdjfnbeyqjjndlndwp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqwxdhavwschkskuzewjkpspwzdtenuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"honiamnaexcaygkyrglkrjcgrzjlhwhh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"offtznutwgwoydbcvolnzhnndoymuxqe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqqogcrxjvzdmdxedjmadszkniybofxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdxgfpikafltlourvpirscygznbaqssq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnaxbrlijngizcaeyxgbhchruowspmso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmtqrtzszvgwhjdmnpkipoaqexockbmd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxfcxajnigzxagoofagbrqqxwyejkmzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gesebrbjrhhmyoadphyzaocknqaluooc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzachgfywvbiflbexduzhjaojrnkpoix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"doltzuwoyduqwojdqcrynuhxfirfwvkc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acaookgldzddaartuofhpqwpbiobyznu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hecwnhvxvrjjrzhnjswdinvspuxlcnzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzbzlverctrvevudxalusyjeqbdlkzfl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ksfgqrcokcdoieyoyodbypltitrnsnct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eoxnexhsqeycpeseypiuhzzadsxwvgfs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqopupgqbijvksvhopqjdiqjazdhvooa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"euqxoxrgciobadjemtoyjdovrcpxuvpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxjmmwqjyfumgurscanxsioisqsnpsvh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"geqlqzmnxqmwvqibfozetrgiuvexhync","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntvchzxtrgrmbopeysntlindpufcnxau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjlumbctvgnfbpcwozefcdvwkgtyxlox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eyynuxgyzzqjgeriydbxumgfnrlxngey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhqklyjdpxiyluougsotimxymvblgwbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rwwbvrmwurdgspltglgxvrryftwswuti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qardftkaunobzmhwnojkxzerwfzmyrmt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxthdlfnlgbcmnilmwvrsjelqnigijkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jicckblczdzzzapihcfxoacleufgniqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sejpzvmcyawyuqzmzlpekxhbitckrcip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qlvzdnmsmkcxhakrgyboygkmzqmldbwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ptjwtmybpbkjrapwgpkpzttefplrafog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zgpiziygxccdfvhtegcxijojaalritpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwdvavuhwzrwdvwwmoccbcuqjfrrlyak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhjbjlzbjusfwubjjnbbfxqenqptzinx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbroipropllbnghltbphwheoshufkhxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"whwcijtzlsiisihwuzyuhiqxybiobudj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtlbzwnwrpoopsatksnimqiqjashwwyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwuqonkbhofpglamqddgfjdianmaqfdy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cibpcmxolrvcyiibfoldytpootwtfjbo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hccdrmyguplpkcnatwgjyaeprclfmcwt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgxudbidnzdkuctubbzbrpcbnvqeolol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khvfrjcbmdamqoowvdslqkqkfvatghil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ewnkjqxxgxysopzcrhzjtjcsuoyyybyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtewosikdrnxmsemlyzudotxvfgoeipq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qbvsydvnetucaponcypirpjowibdpyjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uomgnhpockxnlclowusahhtjsuxsvkzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubvdyrzqdhxjzxbncwgkhahfgentppae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjmfkdehubjkvkwycqvbknnmzleexhmp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcjpxqaswvuawzjqntyqgkbxpbfljzai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eocqiogfdrlufkrcnysmwwxvscaoxfto","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wlldzmgavdjwcafzkwyskypbliizjgzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ssdkqidhtuuoeqpnxrjwxdjheipohsxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jcuvprnfcudsnsonmaysknpdfgsdlvay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prahkhmlgwfedjkmvgkvepadrftedftf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xiyytddfrmrwmcpocnwhmrklxbcsntze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjvdbahskcqujcfmmkgsnhaeycovalzk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqqspjzuuugnlwoaciegcxyynfkozyfw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"btftkuorodusrkcziwkhasskfwnnirvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqnfpgzxjwggsuvbhidgkwqpccrbfpkb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cxksauwpurexwwdaihlxnsyvqobmlcvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbkccmlffxggquftcckgzpobfomacjjh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jqeksfdxtmtsrvhgeqdyyzexeevsciwr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669932,"databaseName":"models_schema","ddl":"CREATE TABLE `okabqmjgjiaxwjjcnmwstkkozaoeqiiu` (\n `igrfyzcacewzwicoxkrgotwzcqvhklai` int NOT NULL,\n `hadgmpesnwwkdfhpofcbrqjaohrkupib` int DEFAULT NULL,\n `sggvakyxhmnmsixjqyejcjqkgkwmfqxb` int DEFAULT NULL,\n `jgqwxebapwvyteelrezduckahrrstlek` int DEFAULT NULL,\n `crknxfgciajabtqjywzfejvdspbudzjd` int DEFAULT NULL,\n `fsmweyvegnntvcwkrgmjxlttuusavyoi` int DEFAULT NULL,\n `fxutncihnltwbaqxccufhmpxqtcomttf` int DEFAULT NULL,\n `vpqdfmgiapceiikvpvcdhjkxejtfenws` int DEFAULT NULL,\n `wygzinvzwripfssqkmlrosevwcslfwap` int DEFAULT NULL,\n `aobmbemtzudakrqohpoeiqsrnnuhpwtl` int DEFAULT NULL,\n `jmmrqlmgghtafodwgxxmqslhqrbkugeb` int DEFAULT NULL,\n `mrwhovwrbjdrvzvjdtklyuebsobosxcy` int DEFAULT NULL,\n `suvrgmyxadgkooissvkoaeslipcpnqcm` int DEFAULT NULL,\n `bmywoixxytavknmbsjynfhlazqwtanvl` int DEFAULT NULL,\n `fnaedujcfoykyxcmgskalyrjbpzterja` int DEFAULT NULL,\n `kstujjrmfbfzkhvkfbguqygwnkwpegwo` int DEFAULT NULL,\n `flmyngfiiqonrhjgzamrsutuzppqbfec` int DEFAULT NULL,\n `kiiqpjsolhyleeeyuyjwthxsnpwzejwa` int DEFAULT NULL,\n `owvxfhabrmgbgyembuolcbmaxbrfhtgs` int DEFAULT NULL,\n `cfmxklhgiykvlkkvbxmtnkonxqkacfpq` int DEFAULT NULL,\n `srppcsizklypvyxqtlkekdaosqxydkjm` int DEFAULT NULL,\n `qkigejiyihiuihalonozkpynwyoohnfg` int DEFAULT NULL,\n `nkfqjrrfcmqjfxvipvxjdsexnpbvncjl` int DEFAULT NULL,\n `jlcpzxhvhzhjjgjvaxsixiurttgcczdq` int DEFAULT NULL,\n `ypagfqonjvvgzvybjsaqwyjbpstpkmwc` int DEFAULT NULL,\n `psqztqusqkptxggpemwbozmsyrikbvgk` int DEFAULT NULL,\n `wcgcejtneuolvbaddhbdybbhvuktcpby` int DEFAULT NULL,\n `jmeivrrbdhbzdpqacqbnimgzxrarerdq` int DEFAULT NULL,\n `qiskthvcepteqabvmtkgvhsuxokwhrbt` int DEFAULT NULL,\n `zcbvudrkasnqahyaintbnyznfztudlac` int DEFAULT NULL,\n `uubbwznjbtwcjpgqgmcvhfeaczdyjyda` int DEFAULT NULL,\n `zjlzonwcaljudiehgozwvebljvdzwkdl` int DEFAULT NULL,\n `tcijurkhzzeunxfzthmoqrmckeuhbezw` int DEFAULT NULL,\n `datlrmqjlmhfuifejtmmiyolauaaeiiw` int DEFAULT NULL,\n `ontphmlztaggdklqmxztpbwitiokiuiy` int DEFAULT NULL,\n `jyryuxopgniuvxnyggietmwcnyslyfjs` int DEFAULT NULL,\n `oqcrhbdbqjqbrnciaorzbrdwnmoojzzv` int DEFAULT NULL,\n `vxwjxfflvnlbbfnnacdyatnfhrrztqyv` int DEFAULT NULL,\n `qwlniemwnyrvqbzrcygwtrtzufiteuuq` int DEFAULT NULL,\n `lorttgctiirdyuzcgxsharbvyibzvtvp` int DEFAULT NULL,\n `cjvnnltrocssptyfsjarbkqyegvnuqgo` int DEFAULT NULL,\n `bdkpemptvcxpswvtvufastlriwdwcfhi` int DEFAULT NULL,\n `lkyndulplmmonpnlewxhwchssflbkgbm` int DEFAULT NULL,\n `azjxxzvpcffqqchrormydwjziwdfxzzv` int DEFAULT NULL,\n `zhfluyrpsclloyskjsxeepvktesrkkbg` int DEFAULT NULL,\n `nemdsevpucjfwqlzgqxhqspuuneeacmz` int DEFAULT NULL,\n `ztvrktjyofxqczrpokplotjseqphalmk` int DEFAULT NULL,\n `kvfkrwvfgbdyyjcngkpvttddvzwnxskf` int DEFAULT NULL,\n `dkpdyxbjzllevodmppgevmknnoeytmsz` int DEFAULT NULL,\n `daptbwlrxtqedgiajmqsagqgmeapugbg` int DEFAULT NULL,\n `ucbvvhhsfkonjudamsjgbgyalfjrfwnb` int DEFAULT NULL,\n `qpxsijovqodpqueiyfpemayktigfsuih` int DEFAULT NULL,\n `ismwrbjxxhbzoungypwwogfgqektunky` int DEFAULT NULL,\n `jydfxbczsdqpfooylmtnxkiyuyzvyfbe` int DEFAULT NULL,\n `kbrvddeiixunygqvmvudmwhjwshbheyt` int DEFAULT NULL,\n `hnfxguzlprfdzlpeqyaqukhaffeykrmj` int DEFAULT NULL,\n `gdbhmhwnhdfvtekquwccjdeuniygypwb` int DEFAULT NULL,\n `xufblcihoanjdjevvcapyjhkqjdnaxvo` int DEFAULT NULL,\n `jjpqmufarylouqgtnxqoafwzlbfuzmxh` int DEFAULT NULL,\n `jbxyjbzurbfjhijhbjpcowucijgojmol` int DEFAULT NULL,\n `bkenwipgxrvojbkcucckemhhljvbqvpn` int DEFAULT NULL,\n `ckqotttuielbgnpmdbtybcxmtsxkhnsw` int DEFAULT NULL,\n `hbtozzyqqoglqkqoflqprgwnfxdfhrwp` int DEFAULT NULL,\n `ftsssvfrwuxrxgvhhancczhvveozfwlo` int DEFAULT NULL,\n `apogleonfblvfevbcerjlllzsofazzeb` int DEFAULT NULL,\n `jsexgbywztjpofizpbdfteojpnsfggps` int DEFAULT NULL,\n `mwvgfzxdjpekffcfqpswflcqujbruxda` int DEFAULT NULL,\n `ktxkrwxppocdlxtxgltgpcpeugekcucn` int DEFAULT NULL,\n `gnwoumkvygmconrrnrpsmtojjsiyxmys` int DEFAULT NULL,\n `hhshipdinpgjjepfjluzyzndrgzhpavg` int DEFAULT NULL,\n `thdjvjsezoflycofbpywsgnmjjnvvqoi` int DEFAULT NULL,\n `dgsgimodnbrthxkwnczdsqknodoftdwb` int DEFAULT NULL,\n `ejnupjelhqmikubxsggzezhjganqqejk` int DEFAULT NULL,\n `pahicfhpnwprmcptqwaszdbxrqvfzfpc` int DEFAULT NULL,\n `qidclcanxpmmsahfubetqsjclsugljvb` int DEFAULT NULL,\n `tjzftosqnejwljwtqbwbpbkirsexzjsl` int DEFAULT NULL,\n `hhxwrxfjcgsiicctksqygdzvnsmkjnql` int DEFAULT NULL,\n `dnysnukcyykgrqtezweatrbmmgdengkp` int DEFAULT NULL,\n `ifdnhygtkoiyycpttkkktcvehzdklmqp` int DEFAULT NULL,\n `trlzegjhzqmnwxkdxfwkubdeoarhabuh` int DEFAULT NULL,\n `kolmkrkduapmlfhyicydjkkpmbttuipx` int DEFAULT NULL,\n `lihfsxmzjqvxztsxohzsoceatztqgfft` int DEFAULT NULL,\n `rzzbhpdxwshwnvndvmucjiczsjptpdol` int DEFAULT NULL,\n `ybqroluqjtbfnimmqicfrlbvsfpvqdmi` int DEFAULT NULL,\n `diipcxacjwujdjgrvmpyypfzhgalyebc` int DEFAULT NULL,\n `mdsfvzvbxbmziocgtpcgrwbsagollnap` int DEFAULT NULL,\n `thbsxqkvmimaewajhrarmftzchqrqvkm` int DEFAULT NULL,\n `scvcoldzalvgnkhssnyijqtqbrcyakkt` int DEFAULT NULL,\n `lgxniwzmkepxqapficzzmpaepowknedf` int DEFAULT NULL,\n `rdeuzvtwvjrxmghgkngyaefwqrcnmfnb` int DEFAULT NULL,\n `rjagtetivccazblauzikjayuzntzajwf` int DEFAULT NULL,\n `hmwnaeduyyxitfbyhpguleyokrtlvljt` int DEFAULT NULL,\n `gkiecxeglrpygawmnbnkdzbehfvvhvhw` int DEFAULT NULL,\n `cnnlryrpcmnwsayczdoxtasnkrpcocnd` int DEFAULT NULL,\n `tbttnmyoclrrkbxfyxnepjnhjzszpvvr` int DEFAULT NULL,\n `refkxavejptegmahfaeehpqhxfqxncqi` int DEFAULT NULL,\n `kccgswpmjaslhbzjqrmobbybeuckygts` int DEFAULT NULL,\n `lxwuwfovvdzpmrmszwjlpcunswredbxk` int DEFAULT NULL,\n `gzanlzddjmymageaptlsrrumjrdrygkk` int DEFAULT NULL,\n `rstqvfcegzurlxfovyvlvdvmnjjunapa` int DEFAULT NULL,\n PRIMARY KEY (`igrfyzcacewzwicoxkrgotwzcqvhklai`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"okabqmjgjiaxwjjcnmwstkkozaoeqiiu\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["igrfyzcacewzwicoxkrgotwzcqvhklai"],"columns":[{"name":"igrfyzcacewzwicoxkrgotwzcqvhklai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"hadgmpesnwwkdfhpofcbrqjaohrkupib","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sggvakyxhmnmsixjqyejcjqkgkwmfqxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgqwxebapwvyteelrezduckahrrstlek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crknxfgciajabtqjywzfejvdspbudzjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fsmweyvegnntvcwkrgmjxlttuusavyoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxutncihnltwbaqxccufhmpxqtcomttf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpqdfmgiapceiikvpvcdhjkxejtfenws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wygzinvzwripfssqkmlrosevwcslfwap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aobmbemtzudakrqohpoeiqsrnnuhpwtl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmmrqlmgghtafodwgxxmqslhqrbkugeb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrwhovwrbjdrvzvjdtklyuebsobosxcy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suvrgmyxadgkooissvkoaeslipcpnqcm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bmywoixxytavknmbsjynfhlazqwtanvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnaedujcfoykyxcmgskalyrjbpzterja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kstujjrmfbfzkhvkfbguqygwnkwpegwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flmyngfiiqonrhjgzamrsutuzppqbfec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kiiqpjsolhyleeeyuyjwthxsnpwzejwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owvxfhabrmgbgyembuolcbmaxbrfhtgs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfmxklhgiykvlkkvbxmtnkonxqkacfpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"srppcsizklypvyxqtlkekdaosqxydkjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qkigejiyihiuihalonozkpynwyoohnfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkfqjrrfcmqjfxvipvxjdsexnpbvncjl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlcpzxhvhzhjjgjvaxsixiurttgcczdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypagfqonjvvgzvybjsaqwyjbpstpkmwc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psqztqusqkptxggpemwbozmsyrikbvgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcgcejtneuolvbaddhbdybbhvuktcpby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmeivrrbdhbzdpqacqbnimgzxrarerdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qiskthvcepteqabvmtkgvhsuxokwhrbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zcbvudrkasnqahyaintbnyznfztudlac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uubbwznjbtwcjpgqgmcvhfeaczdyjyda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjlzonwcaljudiehgozwvebljvdzwkdl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tcijurkhzzeunxfzthmoqrmckeuhbezw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"datlrmqjlmhfuifejtmmiyolauaaeiiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ontphmlztaggdklqmxztpbwitiokiuiy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jyryuxopgniuvxnyggietmwcnyslyfjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqcrhbdbqjqbrnciaorzbrdwnmoojzzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxwjxfflvnlbbfnnacdyatnfhrrztqyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwlniemwnyrvqbzrcygwtrtzufiteuuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lorttgctiirdyuzcgxsharbvyibzvtvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjvnnltrocssptyfsjarbkqyegvnuqgo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdkpemptvcxpswvtvufastlriwdwcfhi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lkyndulplmmonpnlewxhwchssflbkgbm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azjxxzvpcffqqchrormydwjziwdfxzzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhfluyrpsclloyskjsxeepvktesrkkbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nemdsevpucjfwqlzgqxhqspuuneeacmz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ztvrktjyofxqczrpokplotjseqphalmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvfkrwvfgbdyyjcngkpvttddvzwnxskf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkpdyxbjzllevodmppgevmknnoeytmsz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"daptbwlrxtqedgiajmqsagqgmeapugbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucbvvhhsfkonjudamsjgbgyalfjrfwnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qpxsijovqodpqueiyfpemayktigfsuih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ismwrbjxxhbzoungypwwogfgqektunky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jydfxbczsdqpfooylmtnxkiyuyzvyfbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kbrvddeiixunygqvmvudmwhjwshbheyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hnfxguzlprfdzlpeqyaqukhaffeykrmj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdbhmhwnhdfvtekquwccjdeuniygypwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xufblcihoanjdjevvcapyjhkqjdnaxvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjpqmufarylouqgtnxqoafwzlbfuzmxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbxyjbzurbfjhijhbjpcowucijgojmol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkenwipgxrvojbkcucckemhhljvbqvpn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckqotttuielbgnpmdbtybcxmtsxkhnsw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbtozzyqqoglqkqoflqprgwnfxdfhrwp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftsssvfrwuxrxgvhhancczhvveozfwlo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"apogleonfblvfevbcerjlllzsofazzeb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jsexgbywztjpofizpbdfteojpnsfggps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwvgfzxdjpekffcfqpswflcqujbruxda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ktxkrwxppocdlxtxgltgpcpeugekcucn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnwoumkvygmconrrnrpsmtojjsiyxmys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhshipdinpgjjepfjluzyzndrgzhpavg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thdjvjsezoflycofbpywsgnmjjnvvqoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgsgimodnbrthxkwnczdsqknodoftdwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejnupjelhqmikubxsggzezhjganqqejk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pahicfhpnwprmcptqwaszdbxrqvfzfpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qidclcanxpmmsahfubetqsjclsugljvb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjzftosqnejwljwtqbwbpbkirsexzjsl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhxwrxfjcgsiicctksqygdzvnsmkjnql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dnysnukcyykgrqtezweatrbmmgdengkp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifdnhygtkoiyycpttkkktcvehzdklmqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"trlzegjhzqmnwxkdxfwkubdeoarhabuh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kolmkrkduapmlfhyicydjkkpmbttuipx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lihfsxmzjqvxztsxohzsoceatztqgfft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzzbhpdxwshwnvndvmucjiczsjptpdol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybqroluqjtbfnimmqicfrlbvsfpvqdmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"diipcxacjwujdjgrvmpyypfzhgalyebc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdsfvzvbxbmziocgtpcgrwbsagollnap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thbsxqkvmimaewajhrarmftzchqrqvkm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scvcoldzalvgnkhssnyijqtqbrcyakkt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgxniwzmkepxqapficzzmpaepowknedf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdeuzvtwvjrxmghgkngyaefwqrcnmfnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjagtetivccazblauzikjayuzntzajwf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmwnaeduyyxitfbyhpguleyokrtlvljt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkiecxeglrpygawmnbnkdzbehfvvhvhw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnnlryrpcmnwsayczdoxtasnkrpcocnd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbttnmyoclrrkbxfyxnepjnhjzszpvvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"refkxavejptegmahfaeehpqhxfqxncqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kccgswpmjaslhbzjqrmobbybeuckygts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxwuwfovvdzpmrmszwjlpcunswredbxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzanlzddjmymageaptlsrrumjrdrygkk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rstqvfcegzurlxfovyvlvdvmnjjunapa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669963,"databaseName":"models_schema","ddl":"CREATE TABLE `opxnblpgxrgsptkxokfztqjhkvdodwud` (\n `bhrlddnxfhupdkzwsfailxrxncpmocpr` int NOT NULL,\n `uspqbssyegpgvqglwbqmgzkupyxceysm` int DEFAULT NULL,\n `xaziscziuxrwvmjcbynvnlnebinnerjx` int DEFAULT NULL,\n `ibqbapyrmpdqytdoasxjggccejasvyig` int DEFAULT NULL,\n `tgjgknsdwdwcayppolejmewrvwzdtygh` int DEFAULT NULL,\n `zzgjtezcgzotkdrrkbzfjmyddaliysbu` int DEFAULT NULL,\n `kazgkasoyacjzkehmycfffaacsoqszgl` int DEFAULT NULL,\n `zbmzibktclyhbiapirdpcejwojihezxl` int DEFAULT NULL,\n `gxufqvemdiuurigjpwculxorwhdpxmwk` int DEFAULT NULL,\n `zqrnwdysrhnspcnvphdthblrabdeipzk` int DEFAULT NULL,\n `uzuuoxnrmqkuojmnasuymswmlcscnjak` int DEFAULT NULL,\n `dmvmqseajbxthjjxdrxwyyhvsoyqpmop` int DEFAULT NULL,\n `ykvtgrfrjszeqqqpktuumwldemmebxle` int DEFAULT NULL,\n `kolbftzdgdiiitvqaihhdtmwtwftejru` int DEFAULT NULL,\n `kuvdjoplgekgqdahdauuvbuxrvtvypjv` int DEFAULT NULL,\n `wptpbbyftloqxzfryjqljwqcejktpuar` int DEFAULT NULL,\n `dzgeuwqzamcbszntwgydeulemeupgmut` int DEFAULT NULL,\n `ouqnxwoclwhwovmabopbpcbdkzrqpnzi` int DEFAULT NULL,\n `zbyomsagdntupopjwknayqrhhxxdmoqy` int DEFAULT NULL,\n `qojlqupvmzkvddumgwuwhmdpevjykjvn` int DEFAULT NULL,\n `thdltsmfxjloohkjixpjgtmnyccdxkhj` int DEFAULT NULL,\n `yzketgjxgwagzrzjzkkgbvonvvcpyvuf` int DEFAULT NULL,\n `wdcqjkdqaejffgelvtcuwhukbwkskawf` int DEFAULT NULL,\n `rnrglaqssbuabnjzearcvoemencqooit` int DEFAULT NULL,\n `fyntwgipruerarvkwzxscspasfxpmidn` int DEFAULT NULL,\n `lwayozkmdpcsbwbqgtjvxqnrzrttwtbm` int DEFAULT NULL,\n `vceiivrrrarkaoxpfmacjlwiphyikxco` int DEFAULT NULL,\n `lrraypypprqdjtaiqeuzhdpduipsmtyi` int DEFAULT NULL,\n `xuhkrucmwrstyqcqrmtobujtoagmrvdk` int DEFAULT NULL,\n `bcfgjdyzxbufhfaswxtuaympwbfrwppn` int DEFAULT NULL,\n `puflzzrkonsiooglxzifiujsczondgyk` int DEFAULT NULL,\n `eslurqtgrlhjtjpvsknbuwrzkrzqbgqe` int DEFAULT NULL,\n `ysbjxmyltzlxsjbamaweegxfkxcresfc` int DEFAULT NULL,\n `cznutspyddkcymgcvnzmakkrmfamoiom` int DEFAULT NULL,\n `peiusvtvzjebbixkupaxwmitugdwuefg` int DEFAULT NULL,\n `bywiwisliflhtbondwqmtdfibnplgpcx` int DEFAULT NULL,\n `gzdotqcgxiqeeqexirrtjlryoaxzdjxf` int DEFAULT NULL,\n `mlvqjgiamapwloxmtzxfmcmuditxnqxt` int DEFAULT NULL,\n `pqnmtmwbkhguqgzytnwpgpwcdiutjjrh` int DEFAULT NULL,\n `ryfoqexlymhwqhliufpyknchxusntwxh` int DEFAULT NULL,\n `psrtbhcsssdafvltdptuxhysdlypjkun` int DEFAULT NULL,\n `ssawacippyjqpnczputubfcrcsnfiwmn` int DEFAULT NULL,\n `ffcemvqexajzpmkwnuwgblgtspqxquxv` int DEFAULT NULL,\n `shawtqptpxskkcmkshnfgyjekeowafle` int DEFAULT NULL,\n `vmvsjurqmfzlxuxlevbkcybslbdlyycw` int DEFAULT NULL,\n `nmeyifcidzhryxzjzfexdlzlgwvcccdv` int DEFAULT NULL,\n `awmwvhmtfrphmlydwarimhixaynibnzp` int DEFAULT NULL,\n `gkwjyejwsxuucwylecuegnbnmzrzmivo` int DEFAULT NULL,\n `bnmkortziznoophnhdutvmxpimuqpief` int DEFAULT NULL,\n `strrlimdqpwdgjmtxsqstdpmcvazfogv` int DEFAULT NULL,\n `dwwhvbxcswwavihukhjxntbowcsexsmn` int DEFAULT NULL,\n `ilgwrydpjfdvmbprjprobzenyfppgutn` int DEFAULT NULL,\n `rzbrgvswwbtwhgchgrgwkilxkgnetgcw` int DEFAULT NULL,\n `rujyjfgafjujijxeabrgebotpwshnowu` int DEFAULT NULL,\n `stxrxdwptibmtjlgebgusrqejwmxeova` int DEFAULT NULL,\n `ssqtzmuvtxnzyhsnqdjeczjzzgbmusvh` int DEFAULT NULL,\n `zcchxfnckzbwghteflhmcjytdsuannjo` int DEFAULT NULL,\n `yrgkzrxjcwhwuklnhamihvwbprxqmamx` int DEFAULT NULL,\n `avhqgivljvafsdyclaadxdxgdifjbgse` int DEFAULT NULL,\n `wxaprfkjrokhrzfgjixkgkdgbbaakacr` int DEFAULT NULL,\n `oyhkfsueyhewdksfybbtmfyvbwmhgowf` int DEFAULT NULL,\n `azpminavraaldspdsdbjjehahovtyzyw` int DEFAULT NULL,\n `ujjwzqmjebslvtyqflexhmmrnyuszsqx` int DEFAULT NULL,\n `hxlbkxttucyrypyteuhlkzvecbylmbqu` int DEFAULT NULL,\n `jwzbydeqkadcvndcrmotztlgxuizbcuf` int DEFAULT NULL,\n `caaecurwlrtlfhkusqvsvgphryjgycsv` int DEFAULT NULL,\n `rgirzpztfxfgcwpbzxgjeacmvymbsxtj` int DEFAULT NULL,\n `suohwobbkwruxhukmnyqdumfwplmikkd` int DEFAULT NULL,\n `ufheocmqrvdevytplwrozjszaewqzkgk` int DEFAULT NULL,\n `zbuacjzripxubghxhdzdkxhfbtqgaasi` int DEFAULT NULL,\n `cbbesrcxrhqihroalcarhveatfbpgdex` int DEFAULT NULL,\n `hgccmucystqluvzrwbbtysqemkymlpqj` int DEFAULT NULL,\n `rfstwltzlvwxqbbgryhpzwxvgtofzdnf` int DEFAULT NULL,\n `oqqkmkxfxlgipffcsxtllfenoluyndry` int DEFAULT NULL,\n `sqmwlygnlqxznofykhutlftfdwnfdeoi` int DEFAULT NULL,\n `mlkqqpxzctjundqteyqsvsyngvfzwhrw` int DEFAULT NULL,\n `pnemdqaaksatewqvkarwrqvdkqexwcpg` int DEFAULT NULL,\n `zmorxysawcwkppyvfmeagrmpprtgzkwf` int DEFAULT NULL,\n `oqvgvacqvptzicwoocrxtijxklmdmvoi` int DEFAULT NULL,\n `pgmakjtuvbzhoxbchxibuqkllgvwmzqv` int DEFAULT NULL,\n `vmqnjwtjccpqtnlnmghmmlhqaimftzea` int DEFAULT NULL,\n `ponmdhlcsjbvzrgsoqvceshuepyfifez` int DEFAULT NULL,\n `xtrnvjtseknvfvxkezfypbzsmwstxqbb` int DEFAULT NULL,\n `wkpctuzcryyqclkydvqsxlnysummslqj` int DEFAULT NULL,\n `ydgencbzhtixmfuqngphnunqbwyjrnqb` int DEFAULT NULL,\n `xserwgajrmozehmpbfdbemyylwgiytay` int DEFAULT NULL,\n `fvysqzzsdjnuvjdjwnjtgghfuqoezoxt` int DEFAULT NULL,\n `nbnsjctixzmyzinpyfxaprqzetvsmxrh` int DEFAULT NULL,\n `hsgmxzlumzxxjrgdhsemvndsedieuxyj` int DEFAULT NULL,\n `uyrofrtfdhzfjcunyojlgechimgfwvui` int DEFAULT NULL,\n `calizkwamrqhsfozzqoxuhqffdqowoiz` int DEFAULT NULL,\n `xinmmxuadfwpqecyrxbqylcxugqsqacd` int DEFAULT NULL,\n `wqvxjgajjlqqogaajoxskuerqubquicg` int DEFAULT NULL,\n `glgsfdigksmfhljmlmziycwublvjdxsa` int DEFAULT NULL,\n `dexvrvzbchnnobovwtsydwcsbgjtsszy` int DEFAULT NULL,\n `lslbuptprxnofphpymezaoxzfvuzjycr` int DEFAULT NULL,\n `mqdwqdluvdfdbyylfgkazskkwrlzgwvz` int DEFAULT NULL,\n `jmvwrzrsltsfppjttwvcnskdjuamquzm` int DEFAULT NULL,\n `hyzbwrgxmylmkhfqkkhublawjqncqqol` int DEFAULT NULL,\n `agavvorkcxicpchhfufiiefjpfeeftvq` int DEFAULT NULL,\n PRIMARY KEY (`bhrlddnxfhupdkzwsfailxrxncpmocpr`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"opxnblpgxrgsptkxokfztqjhkvdodwud\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["bhrlddnxfhupdkzwsfailxrxncpmocpr"],"columns":[{"name":"bhrlddnxfhupdkzwsfailxrxncpmocpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"uspqbssyegpgvqglwbqmgzkupyxceysm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xaziscziuxrwvmjcbynvnlnebinnerjx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibqbapyrmpdqytdoasxjggccejasvyig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tgjgknsdwdwcayppolejmewrvwzdtygh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzgjtezcgzotkdrrkbzfjmyddaliysbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kazgkasoyacjzkehmycfffaacsoqszgl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbmzibktclyhbiapirdpcejwojihezxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxufqvemdiuurigjpwculxorwhdpxmwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqrnwdysrhnspcnvphdthblrabdeipzk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzuuoxnrmqkuojmnasuymswmlcscnjak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dmvmqseajbxthjjxdrxwyyhvsoyqpmop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykvtgrfrjszeqqqpktuumwldemmebxle","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kolbftzdgdiiitvqaihhdtmwtwftejru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kuvdjoplgekgqdahdauuvbuxrvtvypjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wptpbbyftloqxzfryjqljwqcejktpuar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dzgeuwqzamcbszntwgydeulemeupgmut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ouqnxwoclwhwovmabopbpcbdkzrqpnzi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbyomsagdntupopjwknayqrhhxxdmoqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qojlqupvmzkvddumgwuwhmdpevjykjvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thdltsmfxjloohkjixpjgtmnyccdxkhj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzketgjxgwagzrzjzkkgbvonvvcpyvuf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wdcqjkdqaejffgelvtcuwhukbwkskawf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnrglaqssbuabnjzearcvoemencqooit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fyntwgipruerarvkwzxscspasfxpmidn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwayozkmdpcsbwbqgtjvxqnrzrttwtbm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vceiivrrrarkaoxpfmacjlwiphyikxco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lrraypypprqdjtaiqeuzhdpduipsmtyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xuhkrucmwrstyqcqrmtobujtoagmrvdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bcfgjdyzxbufhfaswxtuaympwbfrwppn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"puflzzrkonsiooglxzifiujsczondgyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eslurqtgrlhjtjpvsknbuwrzkrzqbgqe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ysbjxmyltzlxsjbamaweegxfkxcresfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cznutspyddkcymgcvnzmakkrmfamoiom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"peiusvtvzjebbixkupaxwmitugdwuefg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bywiwisliflhtbondwqmtdfibnplgpcx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzdotqcgxiqeeqexirrtjlryoaxzdjxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlvqjgiamapwloxmtzxfmcmuditxnqxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqnmtmwbkhguqgzytnwpgpwcdiutjjrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ryfoqexlymhwqhliufpyknchxusntwxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psrtbhcsssdafvltdptuxhysdlypjkun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ssawacippyjqpnczputubfcrcsnfiwmn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffcemvqexajzpmkwnuwgblgtspqxquxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shawtqptpxskkcmkshnfgyjekeowafle","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmvsjurqmfzlxuxlevbkcybslbdlyycw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmeyifcidzhryxzjzfexdlzlgwvcccdv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awmwvhmtfrphmlydwarimhixaynibnzp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkwjyejwsxuucwylecuegnbnmzrzmivo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnmkortziznoophnhdutvmxpimuqpief","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"strrlimdqpwdgjmtxsqstdpmcvazfogv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwwhvbxcswwavihukhjxntbowcsexsmn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ilgwrydpjfdvmbprjprobzenyfppgutn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzbrgvswwbtwhgchgrgwkilxkgnetgcw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rujyjfgafjujijxeabrgebotpwshnowu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"stxrxdwptibmtjlgebgusrqejwmxeova","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ssqtzmuvtxnzyhsnqdjeczjzzgbmusvh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zcchxfnckzbwghteflhmcjytdsuannjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yrgkzrxjcwhwuklnhamihvwbprxqmamx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avhqgivljvafsdyclaadxdxgdifjbgse","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxaprfkjrokhrzfgjixkgkdgbbaakacr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oyhkfsueyhewdksfybbtmfyvbwmhgowf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azpminavraaldspdsdbjjehahovtyzyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujjwzqmjebslvtyqflexhmmrnyuszsqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxlbkxttucyrypyteuhlkzvecbylmbqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwzbydeqkadcvndcrmotztlgxuizbcuf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"caaecurwlrtlfhkusqvsvgphryjgycsv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgirzpztfxfgcwpbzxgjeacmvymbsxtj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suohwobbkwruxhukmnyqdumfwplmikkd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufheocmqrvdevytplwrozjszaewqzkgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbuacjzripxubghxhdzdkxhfbtqgaasi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbbesrcxrhqihroalcarhveatfbpgdex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hgccmucystqluvzrwbbtysqemkymlpqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfstwltzlvwxqbbgryhpzwxvgtofzdnf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqqkmkxfxlgipffcsxtllfenoluyndry","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sqmwlygnlqxznofykhutlftfdwnfdeoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlkqqpxzctjundqteyqsvsyngvfzwhrw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnemdqaaksatewqvkarwrqvdkqexwcpg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmorxysawcwkppyvfmeagrmpprtgzkwf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqvgvacqvptzicwoocrxtijxklmdmvoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgmakjtuvbzhoxbchxibuqkllgvwmzqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmqnjwtjccpqtnlnmghmmlhqaimftzea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ponmdhlcsjbvzrgsoqvceshuepyfifez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xtrnvjtseknvfvxkezfypbzsmwstxqbb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wkpctuzcryyqclkydvqsxlnysummslqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydgencbzhtixmfuqngphnunqbwyjrnqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xserwgajrmozehmpbfdbemyylwgiytay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fvysqzzsdjnuvjdjwnjtgghfuqoezoxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbnsjctixzmyzinpyfxaprqzetvsmxrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hsgmxzlumzxxjrgdhsemvndsedieuxyj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uyrofrtfdhzfjcunyojlgechimgfwvui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"calizkwamrqhsfozzqoxuhqffdqowoiz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xinmmxuadfwpqecyrxbqylcxugqsqacd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqvxjgajjlqqogaajoxskuerqubquicg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"glgsfdigksmfhljmlmziycwublvjdxsa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dexvrvzbchnnobovwtsydwcsbgjtsszy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lslbuptprxnofphpymezaoxzfvuzjycr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqdwqdluvdfdbyylfgkazskkwrlzgwvz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmvwrzrsltsfppjttwvcnskdjuamquzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hyzbwrgxmylmkhfqkkhublawjqncqqol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agavvorkcxicpchhfufiiefjpfeeftvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842669,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842669993,"databaseName":"models_schema","ddl":"CREATE TABLE `orpveknrbipxbinvykmnqwjwotxsjuoq` (\n `gcmgmciozrplfiqyxqatbbggttrepcga` int NOT NULL,\n `mtzzfbcaldbicvzjzikafalikolmlghe` int DEFAULT NULL,\n `zwbkmjeuqzmtsqnuhtgyjjtriojhwcqq` int DEFAULT NULL,\n `yqhgirplplxjxtwqgqykhbhrxrlbhxoq` int DEFAULT NULL,\n `suhvcpkwiltgdtlocgttvyoatwuksqpm` int DEFAULT NULL,\n `envjfiwylnyejnaslyzfivpftvbobjih` int DEFAULT NULL,\n `cecazghruejvccuvotgjrsrhjoepvetn` int DEFAULT NULL,\n `pamksvzyciihvowqjelmvlkpqiolpahu` int DEFAULT NULL,\n `gsicbqqdbvabtoswrhvldyqeimikrcfn` int DEFAULT NULL,\n `embhisjjsahkhxdpwbvmdvbqdsjkuhcu` int DEFAULT NULL,\n `svtvutauklmhjpoqukqzijawfoyiovog` int DEFAULT NULL,\n `evypmmsfywgaepjxjwqjsjkbvzhtgykh` int DEFAULT NULL,\n `ucqdwjmwstajgonulgplpeirgoygtade` int DEFAULT NULL,\n `qzffnxcmuahnmizazdrwjfkmfyhgbykc` int DEFAULT NULL,\n `sfddaqyxqmxnstfyyxlegfdogdtoixxo` int DEFAULT NULL,\n `knloopvpplfeyabpuwrvzhxgepxcdoro` int DEFAULT NULL,\n `kwetensrumghzjbjitmqkberhhuzvknv` int DEFAULT NULL,\n `hceiaebeevjukuxnvkohznzgvqzemkxi` int DEFAULT NULL,\n `dcxqmshgfnkchsvnftexdfmpqgaespnp` int DEFAULT NULL,\n `vmfmulhaydvowhcptrycvbweqweqhjwy` int DEFAULT NULL,\n `afgymqgolbqytiaccxibeelidbqkcexg` int DEFAULT NULL,\n `ysqwucdyjixqduwldjhgyuxptbaksjxe` int DEFAULT NULL,\n `nzksmitukolhbtrhzmonyubvatnqrqky` int DEFAULT NULL,\n `ermbufdevmydvobilwargfmbhkbbypzw` int DEFAULT NULL,\n `xkoybgnrwzwnzynfhglvoqjixxohfwoe` int DEFAULT NULL,\n `lmhylpysewjzjomjwbzhwrxinfvddsgm` int DEFAULT NULL,\n `xkvngyqvywwtijbjugyegpvnzvzgvqds` int DEFAULT NULL,\n `zttaenjidejumcecbobvfsbjbwnviele` int DEFAULT NULL,\n `ezqphpkqfohajziadpdwetcwdztpgywg` int DEFAULT NULL,\n `feneyfoxeuxtofoznyqlmuzzolahuvjg` int DEFAULT NULL,\n `tlbmawfdepggofczjnmzavtrnedakvig` int DEFAULT NULL,\n `oxelelejdgzdbeaddmjjcsqrhixbmldk` int DEFAULT NULL,\n `vpkkhkfhujddybspxuatdazpeadyopwg` int DEFAULT NULL,\n `jolliecfytykdrkurrquzdmwykaoibgb` int DEFAULT NULL,\n `npwowrermnzchabrdrjxkafbsojmeujg` int DEFAULT NULL,\n `elkxeevwkpypcnjugyvrsqypnlgnihqt` int DEFAULT NULL,\n `sajzdeqbqhmhgddosleelcczzhqzfuch` int DEFAULT NULL,\n `hoqshmsvrpsokqkjxtjucsnpbhylrxqq` int DEFAULT NULL,\n `qhjehledosrrbzgpohtrdgirfichxgvx` int DEFAULT NULL,\n `ztnscixscngxjqmjfmucyltyarectfpc` int DEFAULT NULL,\n `lwgcayklxrwzjupkbingacpjgeoyngwq` int DEFAULT NULL,\n `ccasgklkmlmizxnhqpaaqwiojhocwknx` int DEFAULT NULL,\n `eghqzxeudivpueuevymchxoqgijijtuw` int DEFAULT NULL,\n `yairwvjngtzxrvmkbtyynqgrxcxkcaib` int DEFAULT NULL,\n `gctusyntpzumtefivxqvwdpurkuyetbq` int DEFAULT NULL,\n `hkozexdrscxvqsgbnvvipjwvfjelafdm` int DEFAULT NULL,\n `urinpazraxaedsjkgxuxyyebtwhspnkr` int DEFAULT NULL,\n `ajjstmjysvtzywntxmkwhquxhmtbsztz` int DEFAULT NULL,\n `uhhcetryogyptiihkyjznouodqzkbexs` int DEFAULT NULL,\n `nvmrsbfpyevohkufcqbjfmzrtszxaeaf` int DEFAULT NULL,\n `xymljwpfgrajmsgnatbuylhbpshajvad` int DEFAULT NULL,\n `mdxckuqssspphwwrjykcbjilkijlqmrq` int DEFAULT NULL,\n `aanixoeptvqujtoeakgztsggrnemkjdq` int DEFAULT NULL,\n `jrypddktdwfctffkimiickseoblfbfkf` int DEFAULT NULL,\n `cmcyjytkcqqvikwlvophzflpsiqnvwob` int DEFAULT NULL,\n `hjspcnnvkdnydfnhmmeofceekqdqkefv` int DEFAULT NULL,\n `mrusotcbkythoykqkvinpvjbpjazjcoa` int DEFAULT NULL,\n `apaeemsqtkrcfmkgrydorfvejhjtepme` int DEFAULT NULL,\n `mfonesmydjzzauthdahrxtinybzaxwht` int DEFAULT NULL,\n `sayeseogblqvuwxlswljmjcphxmjbvnr` int DEFAULT NULL,\n `qcdzcdzjhqdeyktjsxdibdzydpnfhynb` int DEFAULT NULL,\n `ylvaorodbtrzhwgymoxbuskrwzdmgulo` int DEFAULT NULL,\n `xxnjjlwwjdwuiggcqxlqlcthxaqtumzh` int DEFAULT NULL,\n `sqhireptqdkylmobmisxaxixtriwrsss` int DEFAULT NULL,\n `abivhomhzinhxvurhczcaydzzadvfhza` int DEFAULT NULL,\n `pxkctkligjxfuyrzyifcctanroqyutsh` int DEFAULT NULL,\n `uvupcymspjvfthkzcjpcgabiyzcpxxnc` int DEFAULT NULL,\n `hndnexbiuntjxxlkeztdewsakdjsclkj` int DEFAULT NULL,\n `ykwgelcrycorgopmouaownlojwwtxtlm` int DEFAULT NULL,\n `hdwxrunfepsvphwmzukkbskvipgxipaq` int DEFAULT NULL,\n `vbjrmkukgwtmuqagqbcbzmzckcngvfdl` int DEFAULT NULL,\n `shahjfcwxvlsqnbvmlblgbrvqhkhgqmo` int DEFAULT NULL,\n `yqvipnavauyiqsfzvvcarifklcnqaiaz` int DEFAULT NULL,\n `wnhuemnawlsybupptdlvckcwazzgdpka` int DEFAULT NULL,\n `brvaqsthyjbalqlnapufanrinwxsqzic` int DEFAULT NULL,\n `yiiopiooonbvznulceaqphxrulivbjlr` int DEFAULT NULL,\n `cqcnxcbgrxdjkbqjldvgnwjglrvtnwqx` int DEFAULT NULL,\n `thaistmhxfzqstzdwbbpmejizgvfssem` int DEFAULT NULL,\n `vimxbcmixttggjllkcwvxmmmcewoqwtc` int DEFAULT NULL,\n `qnujslyztulsawqdzrailuiujubjnmrs` int DEFAULT NULL,\n `cnjiahwdwxtqdkpsrfniazwfsxkigapb` int DEFAULT NULL,\n `cbeemnqlysihykbwrodytoijozsujjlk` int DEFAULT NULL,\n `taopokmsrgvjemhfkwdppkmxvlczyldo` int DEFAULT NULL,\n `iievlwjfgcjcwojuwbdkmeqaiwxisbyc` int DEFAULT NULL,\n `bihaacmqzlfihlrgzfutvplxaektatkt` int DEFAULT NULL,\n `tpyejkeoxmfllmogmajnujjktwnjkwkj` int DEFAULT NULL,\n `lemfstnubajpgxsnwzjjrsouqjwphhit` int DEFAULT NULL,\n `zbsgkednghogxyhmbycmhoeubqvzyjns` int DEFAULT NULL,\n `nwalajeeiuwvpilsnlxccxwgvwjatkjs` int DEFAULT NULL,\n `hagpfnqyqoarowcpznibksrvfmsdobcv` int DEFAULT NULL,\n `mgwjklyeljnwatiabsxwxhphyldcprbm` int DEFAULT NULL,\n `ixxeruewhdyqrefwpnzljohywczcocxu` int DEFAULT NULL,\n `cgdhnweunsmdyjtngoebkksnhqfgrihy` int DEFAULT NULL,\n `dovfzhpzjwmhxyltywapbjoggadowvxi` int DEFAULT NULL,\n `pnuczwguhrgefhpurbnqljmgklvhbbhz` int DEFAULT NULL,\n `aarcasxjyuykmwdjrlrqczrbdcilqvcw` int DEFAULT NULL,\n `wdqotoeonnmkqffnodoyvegmaewsvtfq` int DEFAULT NULL,\n `pdrjnnhzgwetkdivjzzlvqxunsafhrav` int DEFAULT NULL,\n `banzahuotkzrurrrjmffvijcmmfglckf` int DEFAULT NULL,\n `aejagisfxbkdmkxwcwfyyieokoaydvvq` int DEFAULT NULL,\n PRIMARY KEY (`gcmgmciozrplfiqyxqatbbggttrepcga`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"orpveknrbipxbinvykmnqwjwotxsjuoq\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["gcmgmciozrplfiqyxqatbbggttrepcga"],"columns":[{"name":"gcmgmciozrplfiqyxqatbbggttrepcga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"mtzzfbcaldbicvzjzikafalikolmlghe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zwbkmjeuqzmtsqnuhtgyjjtriojhwcqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqhgirplplxjxtwqgqykhbhrxrlbhxoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suhvcpkwiltgdtlocgttvyoatwuksqpm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"envjfiwylnyejnaslyzfivpftvbobjih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cecazghruejvccuvotgjrsrhjoepvetn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pamksvzyciihvowqjelmvlkpqiolpahu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsicbqqdbvabtoswrhvldyqeimikrcfn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"embhisjjsahkhxdpwbvmdvbqdsjkuhcu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"svtvutauklmhjpoqukqzijawfoyiovog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evypmmsfywgaepjxjwqjsjkbvzhtgykh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucqdwjmwstajgonulgplpeirgoygtade","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzffnxcmuahnmizazdrwjfkmfyhgbykc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfddaqyxqmxnstfyyxlegfdogdtoixxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knloopvpplfeyabpuwrvzhxgepxcdoro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwetensrumghzjbjitmqkberhhuzvknv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hceiaebeevjukuxnvkohznzgvqzemkxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcxqmshgfnkchsvnftexdfmpqgaespnp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmfmulhaydvowhcptrycvbweqweqhjwy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afgymqgolbqytiaccxibeelidbqkcexg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ysqwucdyjixqduwldjhgyuxptbaksjxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzksmitukolhbtrhzmonyubvatnqrqky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ermbufdevmydvobilwargfmbhkbbypzw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkoybgnrwzwnzynfhglvoqjixxohfwoe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmhylpysewjzjomjwbzhwrxinfvddsgm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkvngyqvywwtijbjugyegpvnzvzgvqds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zttaenjidejumcecbobvfsbjbwnviele","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezqphpkqfohajziadpdwetcwdztpgywg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"feneyfoxeuxtofoznyqlmuzzolahuvjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlbmawfdepggofczjnmzavtrnedakvig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oxelelejdgzdbeaddmjjcsqrhixbmldk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpkkhkfhujddybspxuatdazpeadyopwg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jolliecfytykdrkurrquzdmwykaoibgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npwowrermnzchabrdrjxkafbsojmeujg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elkxeevwkpypcnjugyvrsqypnlgnihqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sajzdeqbqhmhgddosleelcczzhqzfuch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hoqshmsvrpsokqkjxtjucsnpbhylrxqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhjehledosrrbzgpohtrdgirfichxgvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ztnscixscngxjqmjfmucyltyarectfpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwgcayklxrwzjupkbingacpjgeoyngwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ccasgklkmlmizxnhqpaaqwiojhocwknx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eghqzxeudivpueuevymchxoqgijijtuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yairwvjngtzxrvmkbtyynqgrxcxkcaib","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gctusyntpzumtefivxqvwdpurkuyetbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkozexdrscxvqsgbnvvipjwvfjelafdm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urinpazraxaedsjkgxuxyyebtwhspnkr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajjstmjysvtzywntxmkwhquxhmtbsztz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhhcetryogyptiihkyjznouodqzkbexs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvmrsbfpyevohkufcqbjfmzrtszxaeaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xymljwpfgrajmsgnatbuylhbpshajvad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdxckuqssspphwwrjykcbjilkijlqmrq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aanixoeptvqujtoeakgztsggrnemkjdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jrypddktdwfctffkimiickseoblfbfkf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmcyjytkcqqvikwlvophzflpsiqnvwob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjspcnnvkdnydfnhmmeofceekqdqkefv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrusotcbkythoykqkvinpvjbpjazjcoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"apaeemsqtkrcfmkgrydorfvejhjtepme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfonesmydjzzauthdahrxtinybzaxwht","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sayeseogblqvuwxlswljmjcphxmjbvnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qcdzcdzjhqdeyktjsxdibdzydpnfhynb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ylvaorodbtrzhwgymoxbuskrwzdmgulo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxnjjlwwjdwuiggcqxlqlcthxaqtumzh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sqhireptqdkylmobmisxaxixtriwrsss","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abivhomhzinhxvurhczcaydzzadvfhza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxkctkligjxfuyrzyifcctanroqyutsh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvupcymspjvfthkzcjpcgabiyzcpxxnc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hndnexbiuntjxxlkeztdewsakdjsclkj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykwgelcrycorgopmouaownlojwwtxtlm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdwxrunfepsvphwmzukkbskvipgxipaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbjrmkukgwtmuqagqbcbzmzckcngvfdl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shahjfcwxvlsqnbvmlblgbrvqhkhgqmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqvipnavauyiqsfzvvcarifklcnqaiaz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnhuemnawlsybupptdlvckcwazzgdpka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"brvaqsthyjbalqlnapufanrinwxsqzic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yiiopiooonbvznulceaqphxrulivbjlr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqcnxcbgrxdjkbqjldvgnwjglrvtnwqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thaistmhxfzqstzdwbbpmejizgvfssem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vimxbcmixttggjllkcwvxmmmcewoqwtc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qnujslyztulsawqdzrailuiujubjnmrs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnjiahwdwxtqdkpsrfniazwfsxkigapb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbeemnqlysihykbwrodytoijozsujjlk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"taopokmsrgvjemhfkwdppkmxvlczyldo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iievlwjfgcjcwojuwbdkmeqaiwxisbyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bihaacmqzlfihlrgzfutvplxaektatkt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpyejkeoxmfllmogmajnujjktwnjkwkj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lemfstnubajpgxsnwzjjrsouqjwphhit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbsgkednghogxyhmbycmhoeubqvzyjns","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nwalajeeiuwvpilsnlxccxwgvwjatkjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hagpfnqyqoarowcpznibksrvfmsdobcv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mgwjklyeljnwatiabsxwxhphyldcprbm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixxeruewhdyqrefwpnzljohywczcocxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgdhnweunsmdyjtngoebkksnhqfgrihy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dovfzhpzjwmhxyltywapbjoggadowvxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnuczwguhrgefhpurbnqljmgklvhbbhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aarcasxjyuykmwdjrlrqczrbdcilqvcw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wdqotoeonnmkqffnodoyvegmaewsvtfq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdrjnnhzgwetkdivjzzlvqxunsafhrav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"banzahuotkzrurrrjmffvijcmmfglckf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aejagisfxbkdmkxwcwfyyieokoaydvvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670025,"databaseName":"models_schema","ddl":"CREATE TABLE `otmhuzxdzfbyzubxhxxqqnfslkmomovx` (\n `cwcuujmmwoqnqoqatvthswqslbxapagf` int NOT NULL,\n `rfctjaglbbmucilrhohzwlosmoutmvhb` int DEFAULT NULL,\n `ifautkhnbytvbtvotsbkfjudxjnbmwci` int DEFAULT NULL,\n `icpejdtlwwjkxidlnbngftlcksjdrxny` int DEFAULT NULL,\n `cqjbfxlkgtwhlbibnudmqqujzoocowdw` int DEFAULT NULL,\n `tcnwrfmnwjfrejheolnojhdrrvqeeoei` int DEFAULT NULL,\n `aourlkkbgnphwwtvfirklxewuwxjkgge` int DEFAULT NULL,\n `wyjzpjkryqqffsnykekacqfggdqfpern` int DEFAULT NULL,\n `kqtvrbcsoleclxoxcqtsckgdunfzgqku` int DEFAULT NULL,\n `faijydwaxpgxwuwopdjqfihjyoesmwck` int DEFAULT NULL,\n `zggaktezddubnabhspavymnaynvaanko` int DEFAULT NULL,\n `hrwzgudobbidrxrtlhpjxypvenqivsah` int DEFAULT NULL,\n `tsepocdgecedorntefknyccyynqxadvk` int DEFAULT NULL,\n `utvxoosrivakwxplyycjgvbdlrkxiaqo` int DEFAULT NULL,\n `itqphpgmgjigaxrkelehylprxheopdcs` int DEFAULT NULL,\n `ggketmmdpoozcqbdbwipivqunosbhpxl` int DEFAULT NULL,\n `oslbhrntpqhxpxosxvakveohbrroiowi` int DEFAULT NULL,\n `gevyadaaeqvqykdhrdocxwtralejsbvf` int DEFAULT NULL,\n `gvrhxkifasuwmykiibncerfsekcgkwtv` int DEFAULT NULL,\n `vttuwcorwgygnopxphhkpovlhzrwuxxx` int DEFAULT NULL,\n `cxjskzsaejaibbhsisrtvnubfwljfhih` int DEFAULT NULL,\n `evinoknzduvryplamhcrnxbdmxoicztu` int DEFAULT NULL,\n `quiagncmrfinblpryyiwtijssisxqxfy` int DEFAULT NULL,\n `rrpgdzacwwuaqhtpyoerjzcxqwsctdoh` int DEFAULT NULL,\n `rrzmluqanhcqqkfivsyaeraeubijsghv` int DEFAULT NULL,\n `yiqmttmfasxkffzullfrsiaoaxbtljnc` int DEFAULT NULL,\n `nvtkfbkdtbukjuvwajeenqqtaoypcwfe` int DEFAULT NULL,\n `fsccgrvvxskozkzwheonufuenelsuyjt` int DEFAULT NULL,\n `efyxssnpfsgdbldjyijzclaguqtfzqeh` int DEFAULT NULL,\n `ifucyeohsxrvjeiuibmtplmtrcyoxnce` int DEFAULT NULL,\n `zrjydmtqvyfggqtlabqdmljwwubiuzrw` int DEFAULT NULL,\n `ehurpxcutbuwbeggbyjfkqxcysomzght` int DEFAULT NULL,\n `noznbgsxsswhzugfnltlifbcalvuonlt` int DEFAULT NULL,\n `rchgdypjmqsaohvbiaukvyknvunqndvr` int DEFAULT NULL,\n `rcabsjmyoujfvlllrknmablujpidykzn` int DEFAULT NULL,\n `duwaecnybpdyofxfmqhbkozjtskecufx` int DEFAULT NULL,\n `cjrlrqvdtimrmmplfkwrfmkxaxhrvoyr` int DEFAULT NULL,\n `lukoxlvuxsuhvxsuyzcexehlhkmoipzs` int DEFAULT NULL,\n `yxfgkinnxppcrlnykulrehifxfbgqbio` int DEFAULT NULL,\n `ylokesrlnopvosmpradnusvwpfdpbdop` int DEFAULT NULL,\n `hfaqtlofhijomybqawounkfnulmmceeo` int DEFAULT NULL,\n `qymienpbpbcvspvddpjbmekqzwtjvaoy` int DEFAULT NULL,\n `bcmdcydytgaqzzahfpifhczkpghizjnb` int DEFAULT NULL,\n `jgabcwzllqfwcnlrsnnmlkkbjfxcdzak` int DEFAULT NULL,\n `mwmdmrbvgpoygvweghhzzycrztqxntme` int DEFAULT NULL,\n `qssmggqudxvczhejgxetthbpyojqkdxy` int DEFAULT NULL,\n `gcwkfostrwkijuzvcgcpwvivtdldbrhq` int DEFAULT NULL,\n `suzugnzuicvbkfpvzpimvnehbejkkazd` int DEFAULT NULL,\n `onsnlmvtmyeumfstfhxravcibtozsqmt` int DEFAULT NULL,\n `gnruzissolrslfbrosqurvqaqbeiahtx` int DEFAULT NULL,\n `mpqacdkdkwahsxaejxblsslaenzulbtn` int DEFAULT NULL,\n `bqbzxmsrqevuxpugslbhsbdgpjlhhlpm` int DEFAULT NULL,\n `uyxgsgjncehybfpzienacvlyrutllgkl` int DEFAULT NULL,\n `gyxeakpbipsiqcaebqadmqnqhupjvoip` int DEFAULT NULL,\n `semoylzqjrfciloehijxvwzoplsgtudi` int DEFAULT NULL,\n `qijdnwckfxdvvyjreoibtvnirjbnpkrx` int DEFAULT NULL,\n `zwnyngnyhmhxttqgonjnrbhkfjcoxpkq` int DEFAULT NULL,\n `etktalllsuwuzeacwvptaszbjtcsqqox` int DEFAULT NULL,\n `tuoxyfxxozqxyajtayjrmvfvlwdqiewe` int DEFAULT NULL,\n `rbbchrptmzyprqrbfctnburlufbquqxr` int DEFAULT NULL,\n `ynufjmyieyjucbptfvxlfvvjdrueqshd` int DEFAULT NULL,\n `pmdrlfgxvafikoqkexdqxkkkshrekvoi` int DEFAULT NULL,\n `jutfhaleshphlkwlhxcggevxcbyqhfku` int DEFAULT NULL,\n `grnjfzzsayqkgipqqzeawdynkslykljg` int DEFAULT NULL,\n `plmirucnhqfbwnllarasayncnanfajud` int DEFAULT NULL,\n `fckfwzfhalmhjhvufuzonvwdxqkpkuny` int DEFAULT NULL,\n `ulguptdoyumgibhzewpojbrnwwegvrga` int DEFAULT NULL,\n `zfnegrwllpbmyzjdczucfmvadyowttij` int DEFAULT NULL,\n `upjdyepjcricghkyxbavqoawpfcbhewr` int DEFAULT NULL,\n `kjqrcbpsdspehtjgwdebyhkkeajbkqrr` int DEFAULT NULL,\n `ymkcmgzxkbnwedfrwbtpmlubzpklraqn` int DEFAULT NULL,\n `ogxdjjnkphayiipippqundlahoajdalk` int DEFAULT NULL,\n `hngoohrvlxlbbwaarxkiwjmypaugfbks` int DEFAULT NULL,\n `upspxlhetfhejfhnhxmazkevfmcawspk` int DEFAULT NULL,\n `guzzrdvjgkceimoecypfmaflalihixdm` int DEFAULT NULL,\n `tylgvoqiwuifihyrlisevoqipdarsrst` int DEFAULT NULL,\n `dfcyhlwnbuuzferjvmohocdifnxrmkww` int DEFAULT NULL,\n `jvqwfhaucczckpmhlydynvrqzqgjjngg` int DEFAULT NULL,\n `mwumersnxdurumuxqudckwmjlrbejkxd` int DEFAULT NULL,\n `zxxnqxikucovxojbhqqobdjtvnkzoucl` int DEFAULT NULL,\n `wnapymoodpxtwxemgjkaepyncbhkhxos` int DEFAULT NULL,\n `msaosrulzptvysabhqajwyxzgrygtevd` int DEFAULT NULL,\n `dimrtojpxoncdtnppdtaiuefgepldsqg` int DEFAULT NULL,\n `svnwmehfwtgcsxasvbbxhpjzratuwyov` int DEFAULT NULL,\n `nkefihksziigbqmkfciqkmnrdopqgotf` int DEFAULT NULL,\n `zowmxafjnbcbgxnjqkarbxjconavzasq` int DEFAULT NULL,\n `uajotvqidngwqyscdtmiqjzxqgttvqpo` int DEFAULT NULL,\n `evaebftfyzaboemoeswtvlriyzhfihaa` int DEFAULT NULL,\n `soqkwfncgivrmfbauewlicujnzcyxcku` int DEFAULT NULL,\n `uevfwsvudsvlgxivjsqjctyiqmlrrske` int DEFAULT NULL,\n `impytlmelbgagdgvivuwflvpwjtqhuta` int DEFAULT NULL,\n `wswormgwjacarumwzzbdntwxhihjlbve` int DEFAULT NULL,\n `ercismgecmtrvamjsklhwybykxsguwby` int DEFAULT NULL,\n `ujokouppyibrotefunfkkmabdpybvvio` int DEFAULT NULL,\n `fwmnnmkpllvdlhtusmrliinilegmkiuh` int DEFAULT NULL,\n `uhsucrzdnxhjnsyfvqppnlogixjlhozl` int DEFAULT NULL,\n `hkvcmttkoejaoqgzlbdaeixfhjwegcud` int DEFAULT NULL,\n `kzpagdqsxrezcaxdjprqjtfdndzxtcgb` int DEFAULT NULL,\n `lvjrycalxexkgagjmhvcnpvaywvznpch` int DEFAULT NULL,\n `nulytykvyztqftesiyjwamzuuxfwjbfc` int DEFAULT NULL,\n PRIMARY KEY (`cwcuujmmwoqnqoqatvthswqslbxapagf`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"otmhuzxdzfbyzubxhxxqqnfslkmomovx\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["cwcuujmmwoqnqoqatvthswqslbxapagf"],"columns":[{"name":"cwcuujmmwoqnqoqatvthswqslbxapagf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"rfctjaglbbmucilrhohzwlosmoutmvhb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifautkhnbytvbtvotsbkfjudxjnbmwci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icpejdtlwwjkxidlnbngftlcksjdrxny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqjbfxlkgtwhlbibnudmqqujzoocowdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tcnwrfmnwjfrejheolnojhdrrvqeeoei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aourlkkbgnphwwtvfirklxewuwxjkgge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyjzpjkryqqffsnykekacqfggdqfpern","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqtvrbcsoleclxoxcqtsckgdunfzgqku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"faijydwaxpgxwuwopdjqfihjyoesmwck","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zggaktezddubnabhspavymnaynvaanko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hrwzgudobbidrxrtlhpjxypvenqivsah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tsepocdgecedorntefknyccyynqxadvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utvxoosrivakwxplyycjgvbdlrkxiaqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itqphpgmgjigaxrkelehylprxheopdcs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggketmmdpoozcqbdbwipivqunosbhpxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oslbhrntpqhxpxosxvakveohbrroiowi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gevyadaaeqvqykdhrdocxwtralejsbvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gvrhxkifasuwmykiibncerfsekcgkwtv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vttuwcorwgygnopxphhkpovlhzrwuxxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cxjskzsaejaibbhsisrtvnubfwljfhih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evinoknzduvryplamhcrnxbdmxoicztu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"quiagncmrfinblpryyiwtijssisxqxfy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrpgdzacwwuaqhtpyoerjzcxqwsctdoh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrzmluqanhcqqkfivsyaeraeubijsghv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yiqmttmfasxkffzullfrsiaoaxbtljnc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvtkfbkdtbukjuvwajeenqqtaoypcwfe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fsccgrvvxskozkzwheonufuenelsuyjt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"efyxssnpfsgdbldjyijzclaguqtfzqeh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifucyeohsxrvjeiuibmtplmtrcyoxnce","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zrjydmtqvyfggqtlabqdmljwwubiuzrw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehurpxcutbuwbeggbyjfkqxcysomzght","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"noznbgsxsswhzugfnltlifbcalvuonlt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rchgdypjmqsaohvbiaukvyknvunqndvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcabsjmyoujfvlllrknmablujpidykzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"duwaecnybpdyofxfmqhbkozjtskecufx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjrlrqvdtimrmmplfkwrfmkxaxhrvoyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lukoxlvuxsuhvxsuyzcexehlhkmoipzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxfgkinnxppcrlnykulrehifxfbgqbio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ylokesrlnopvosmpradnusvwpfdpbdop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfaqtlofhijomybqawounkfnulmmceeo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qymienpbpbcvspvddpjbmekqzwtjvaoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bcmdcydytgaqzzahfpifhczkpghizjnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgabcwzllqfwcnlrsnnmlkkbjfxcdzak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwmdmrbvgpoygvweghhzzycrztqxntme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qssmggqudxvczhejgxetthbpyojqkdxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gcwkfostrwkijuzvcgcpwvivtdldbrhq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suzugnzuicvbkfpvzpimvnehbejkkazd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"onsnlmvtmyeumfstfhxravcibtozsqmt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnruzissolrslfbrosqurvqaqbeiahtx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpqacdkdkwahsxaejxblsslaenzulbtn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqbzxmsrqevuxpugslbhsbdgpjlhhlpm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uyxgsgjncehybfpzienacvlyrutllgkl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gyxeakpbipsiqcaebqadmqnqhupjvoip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"semoylzqjrfciloehijxvwzoplsgtudi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qijdnwckfxdvvyjreoibtvnirjbnpkrx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zwnyngnyhmhxttqgonjnrbhkfjcoxpkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"etktalllsuwuzeacwvptaszbjtcsqqox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tuoxyfxxozqxyajtayjrmvfvlwdqiewe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbbchrptmzyprqrbfctnburlufbquqxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynufjmyieyjucbptfvxlfvvjdrueqshd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmdrlfgxvafikoqkexdqxkkkshrekvoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jutfhaleshphlkwlhxcggevxcbyqhfku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grnjfzzsayqkgipqqzeawdynkslykljg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"plmirucnhqfbwnllarasayncnanfajud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fckfwzfhalmhjhvufuzonvwdxqkpkuny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ulguptdoyumgibhzewpojbrnwwegvrga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zfnegrwllpbmyzjdczucfmvadyowttij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upjdyepjcricghkyxbavqoawpfcbhewr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjqrcbpsdspehtjgwdebyhkkeajbkqrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ymkcmgzxkbnwedfrwbtpmlubzpklraqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ogxdjjnkphayiipippqundlahoajdalk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hngoohrvlxlbbwaarxkiwjmypaugfbks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upspxlhetfhejfhnhxmazkevfmcawspk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"guzzrdvjgkceimoecypfmaflalihixdm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tylgvoqiwuifihyrlisevoqipdarsrst","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dfcyhlwnbuuzferjvmohocdifnxrmkww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvqwfhaucczckpmhlydynvrqzqgjjngg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwumersnxdurumuxqudckwmjlrbejkxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxxnqxikucovxojbhqqobdjtvnkzoucl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnapymoodpxtwxemgjkaepyncbhkhxos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"msaosrulzptvysabhqajwyxzgrygtevd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dimrtojpxoncdtnppdtaiuefgepldsqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"svnwmehfwtgcsxasvbbxhpjzratuwyov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkefihksziigbqmkfciqkmnrdopqgotf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zowmxafjnbcbgxnjqkarbxjconavzasq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uajotvqidngwqyscdtmiqjzxqgttvqpo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evaebftfyzaboemoeswtvlriyzhfihaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"soqkwfncgivrmfbauewlicujnzcyxcku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uevfwsvudsvlgxivjsqjctyiqmlrrske","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"impytlmelbgagdgvivuwflvpwjtqhuta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wswormgwjacarumwzzbdntwxhihjlbve","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ercismgecmtrvamjsklhwybykxsguwby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujokouppyibrotefunfkkmabdpybvvio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwmnnmkpllvdlhtusmrliinilegmkiuh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhsucrzdnxhjnsyfvqppnlogixjlhozl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkvcmttkoejaoqgzlbdaeixfhjwegcud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzpagdqsxrezcaxdjprqjtfdndzxtcgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvjrycalxexkgagjmhvcnpvaywvznpch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nulytykvyztqftesiyjwamzuuxfwjbfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670057,"databaseName":"models_schema","ddl":"CREATE TABLE `ownvozuqbdsuwedyuvzwnksqhgkngezx` (\n `bwqfpmcxcgwvxpbsjomhuwzgkyogzort` int NOT NULL,\n `shmvblakkvmeldkaywprtvuyigdozlvr` int DEFAULT NULL,\n `rdtjgiejvxoeiuxeltwudprvtgwttyak` int DEFAULT NULL,\n `fhspumspssfsrnledhuffpaiiabexvol` int DEFAULT NULL,\n `gukgamqrklzzsfctsahomxcupvmztsbl` int DEFAULT NULL,\n `gwktkujjdiovhnupxdgkholiashcyxcw` int DEFAULT NULL,\n `txtzdhmrhyhaozcqjrpucgcuzosmfjny` int DEFAULT NULL,\n `zuremwbdftlcpyfmcnsrlpctljvevijh` int DEFAULT NULL,\n `ydmhqfvppzgrmcqazbdvkdmfnbimoouo` int DEFAULT NULL,\n `obxcjnxtenoaxndaecuezkmklecbiipk` int DEFAULT NULL,\n `hywesunrbjkfqruqwkrdnsybzccdpnxb` int DEFAULT NULL,\n `snuxcommxbdfjdxztcztcpzpgnquehrk` int DEFAULT NULL,\n `bqffwruhfnenyeramtrzkmpkitdgweca` int DEFAULT NULL,\n `suzfcfhtccypxewmbgiodhawxehaqhhe` int DEFAULT NULL,\n `qgdrzqyjjltvqzfatwsxijkrmcncogpm` int DEFAULT NULL,\n `vewfvrpksanylaiccrkrtqdwnhbalfly` int DEFAULT NULL,\n `ntektahxbmpzublgfxvqnyrzfzcqgfuw` int DEFAULT NULL,\n `gboartvagjyejhjlpeptgoscrfjglvks` int DEFAULT NULL,\n `yttntxwxbjumkzbrhhkcvjlnkspgepzh` int DEFAULT NULL,\n `fielhgyialzusjbaqopbjwdpkzuvaysm` int DEFAULT NULL,\n `umxbcrfftxxxoyypklkmilfugkdzmxha` int DEFAULT NULL,\n `lvbrfkjzhkqgcxpmgweajyhtmjcbygyb` int DEFAULT NULL,\n `avxjaefzwouknlpdrlxcynzmbxuvpkmw` int DEFAULT NULL,\n `njgwpvvfkbwopgnfxdqcwdoqjxjuvjay` int DEFAULT NULL,\n `qrknawtwmdgkzkgiyxzkxnbtiprtpxhn` int DEFAULT NULL,\n `qglmzwlwvgfgpihrgxblheymcpbwayat` int DEFAULT NULL,\n `vymdnnxbipeqyfwbaxzoupmypjiaeldh` int DEFAULT NULL,\n `djgtedxtgxqgdbbmoghazmyglkwggacx` int DEFAULT NULL,\n `qvzkvfnnjrlenfgxkfjfgidscnnjijvq` int DEFAULT NULL,\n `yyeyovoodjpjebdivprjahgwykvgnswn` int DEFAULT NULL,\n `karhxjgoaxbobbxeujnzefkrtbhldlkw` int DEFAULT NULL,\n `xlffdvahqjhydawofuwxnpgaowyooywo` int DEFAULT NULL,\n `qttldgpjgnzoiwgsnixpvzaevdpgzrte` int DEFAULT NULL,\n `fyfskcwisvjenxbnltszckkvlutqsfvw` int DEFAULT NULL,\n `jnzruqjudiuepectgqkssiihneuizzzf` int DEFAULT NULL,\n `ybbqppsxpwylmuugocasrfhqodzhzcxj` int DEFAULT NULL,\n `djhcokdzalxwsgmpetlsofbzyjelvlyp` int DEFAULT NULL,\n `fcjndpcdncxmeukeyvugmexxirrdwhnq` int DEFAULT NULL,\n `oksvanssmtlktreaqtcjielnkqejdfqd` int DEFAULT NULL,\n `kotozohopfkbjdiwodpowyohucssrxzw` int DEFAULT NULL,\n `mrvirwjnbkrvapuxlmxgwnwlxeacatox` int DEFAULT NULL,\n `fqgjankclyeauzzzvnmdndhxzzcfejbv` int DEFAULT NULL,\n `qbsgztpdhrqjnzdvqckvfbyesczsczhl` int DEFAULT NULL,\n `dxrfvdxxigdliphyajzufakoxcpztbwq` int DEFAULT NULL,\n `nvoobzoiqivrbhimkqfrcogybvqlvmwe` int DEFAULT NULL,\n `deqqzwygzqtbrsxayxkhcrcyvlxnxjem` int DEFAULT NULL,\n `dsenyhepulorgiedlbdzkgfgyynlxuwi` int DEFAULT NULL,\n `yqtykkzemfktgzpebuehrwvdpolzmgxw` int DEFAULT NULL,\n `kblzysboybyjcyyroehuaoldbkatijzg` int DEFAULT NULL,\n `xjsvohpxhgeaofijnmztpxklwkkukmwe` int DEFAULT NULL,\n `kixludaluodndisrpqsaccpysuvsxzqx` int DEFAULT NULL,\n `vrrurfpylefagxupaqlxaehbgikqsikb` int DEFAULT NULL,\n `trkxzxdfjwxnjeaquaavgxnlvrcxwkse` int DEFAULT NULL,\n `eiylbogroidlgnxkfcmlxpslswpdqjtd` int DEFAULT NULL,\n `xotvekpfhrdvsbrwsvtbvojmunsbxztu` int DEFAULT NULL,\n `zpifiagefnibbrxbzdgdyrstrcnutlmm` int DEFAULT NULL,\n `yjanchttqsawoivinfdsyoxeaqhbqehh` int DEFAULT NULL,\n `teqnwaralbjkzbygpsvkjgicmmbaxbvd` int DEFAULT NULL,\n `zzopcfeddsakhyutaqxmstjddtqrgnsh` int DEFAULT NULL,\n `rieuwxvladbivhherijjokefqaccysrs` int DEFAULT NULL,\n `rdhccyaulgunfszxtvfulmzqxjosbzne` int DEFAULT NULL,\n `iblisasiofguuspvztvwezfdsmcmnhoq` int DEFAULT NULL,\n `uougerovjonypxhsxpkfaqvqcuupzfxw` int DEFAULT NULL,\n `gmrtbzwnfggpgsvyvkuveztdoymnmhni` int DEFAULT NULL,\n `vuirkafbjbssqgvplctprwncuqeiulgi` int DEFAULT NULL,\n `sjhfqxkkdlsjfuaoishmxbqlfcbawkri` int DEFAULT NULL,\n `lxjzqxzefiqxvodcwpdlidnelgeqexca` int DEFAULT NULL,\n `fpltvaqhykpitzagvhwyxbzqdfempdob` int DEFAULT NULL,\n `xrbzhdwlvkkcfomlwictbwogvxgwdonc` int DEFAULT NULL,\n `wnukjsbwjeereicoxomejajgxgpwgcjt` int DEFAULT NULL,\n `teqqkxhjcnipwyfkbkmiczcpkchetmsu` int DEFAULT NULL,\n `fdjwkkfqmmgpprjgqrvtgyduwkexzvdh` int DEFAULT NULL,\n `vjiixrizntdrtaskegyifqyorlgpzjij` int DEFAULT NULL,\n `safivwhwikndhhyrsngmourgwddhbyip` int DEFAULT NULL,\n `ysknfyjutgrbssmhxmmyeetvafgksnbl` int DEFAULT NULL,\n `qeasxwxpovaupcnithuuoqfcncirqupx` int DEFAULT NULL,\n `ogenhkonfovrvqmlulorrsrmcmnciomp` int DEFAULT NULL,\n `gdavlhzsdpekjrnazpopjbybatjfsiig` int DEFAULT NULL,\n `cgukytiyxbdcksouaqpzobzqiqkaoaqv` int DEFAULT NULL,\n `xpcihyawlmucwkjngoxneynjwqivdckh` int DEFAULT NULL,\n `bdzwtcakkxsrivcbxofjypbqlpmhfuel` int DEFAULT NULL,\n `dynjyndutduejpsyewqugqfnwbhwvepv` int DEFAULT NULL,\n `pajyplbalkbfovyjozkvmgulqgbxymvm` int DEFAULT NULL,\n `gaxrddskoghmvmmgipvlvitasygycfdb` int DEFAULT NULL,\n `tdynqctblgdefhmyqjzloubwfdemflkj` int DEFAULT NULL,\n `suyffdytmarmhqfgnistyjkklvavkmel` int DEFAULT NULL,\n `ubegupbjinldxbhjynvwsstzyrbhaucn` int DEFAULT NULL,\n `pehwpmfzkedbditgtdmaxkqnphyvywwb` int DEFAULT NULL,\n `aaegiaqfkakkwcyefbxtpesunsenddom` int DEFAULT NULL,\n `hwfmebtrnwxnxfjhehsrrmmawccwwrnv` int DEFAULT NULL,\n `zztvznuushvsqsovzxwndqafqqrizrcq` int DEFAULT NULL,\n `wchaptrghyajrydasukcmxgnrfolestg` int DEFAULT NULL,\n `uxucdcbjbonyntqqnihzurklagjpwqrn` int DEFAULT NULL,\n `acdaalghpbknwahsjopwfrrqkyilfryv` int DEFAULT NULL,\n `kktqglfjqircgymzmqjmzmccsoeawbaw` int DEFAULT NULL,\n `fgjcmjvrflvaturwsrnqcsinucgbzbxj` int DEFAULT NULL,\n `xaqxsrvhcxbpzchmppuzwujsmgzijcoj` int DEFAULT NULL,\n `iokameedjvqkwndovpfahaikjoorcflk` int DEFAULT NULL,\n `kdzrtyfxgwzqulbyrlppvczfbbpxgkil` int DEFAULT NULL,\n `sxarvaakkhlirdyxebuiuixsgvblbbtw` int DEFAULT NULL,\n PRIMARY KEY (`bwqfpmcxcgwvxpbsjomhuwzgkyogzort`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"ownvozuqbdsuwedyuvzwnksqhgkngezx\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["bwqfpmcxcgwvxpbsjomhuwzgkyogzort"],"columns":[{"name":"bwqfpmcxcgwvxpbsjomhuwzgkyogzort","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"shmvblakkvmeldkaywprtvuyigdozlvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdtjgiejvxoeiuxeltwudprvtgwttyak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhspumspssfsrnledhuffpaiiabexvol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gukgamqrklzzsfctsahomxcupvmztsbl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gwktkujjdiovhnupxdgkholiashcyxcw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txtzdhmrhyhaozcqjrpucgcuzosmfjny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zuremwbdftlcpyfmcnsrlpctljvevijh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydmhqfvppzgrmcqazbdvkdmfnbimoouo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obxcjnxtenoaxndaecuezkmklecbiipk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hywesunrbjkfqruqwkrdnsybzccdpnxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"snuxcommxbdfjdxztcztcpzpgnquehrk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqffwruhfnenyeramtrzkmpkitdgweca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suzfcfhtccypxewmbgiodhawxehaqhhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgdrzqyjjltvqzfatwsxijkrmcncogpm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vewfvrpksanylaiccrkrtqdwnhbalfly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntektahxbmpzublgfxvqnyrzfzcqgfuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gboartvagjyejhjlpeptgoscrfjglvks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yttntxwxbjumkzbrhhkcvjlnkspgepzh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fielhgyialzusjbaqopbjwdpkzuvaysm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umxbcrfftxxxoyypklkmilfugkdzmxha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvbrfkjzhkqgcxpmgweajyhtmjcbygyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avxjaefzwouknlpdrlxcynzmbxuvpkmw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njgwpvvfkbwopgnfxdqcwdoqjxjuvjay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrknawtwmdgkzkgiyxzkxnbtiprtpxhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qglmzwlwvgfgpihrgxblheymcpbwayat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vymdnnxbipeqyfwbaxzoupmypjiaeldh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"djgtedxtgxqgdbbmoghazmyglkwggacx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvzkvfnnjrlenfgxkfjfgidscnnjijvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yyeyovoodjpjebdivprjahgwykvgnswn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"karhxjgoaxbobbxeujnzefkrtbhldlkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xlffdvahqjhydawofuwxnpgaowyooywo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qttldgpjgnzoiwgsnixpvzaevdpgzrte","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fyfskcwisvjenxbnltszckkvlutqsfvw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jnzruqjudiuepectgqkssiihneuizzzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybbqppsxpwylmuugocasrfhqodzhzcxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"djhcokdzalxwsgmpetlsofbzyjelvlyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcjndpcdncxmeukeyvugmexxirrdwhnq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oksvanssmtlktreaqtcjielnkqejdfqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kotozohopfkbjdiwodpowyohucssrxzw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrvirwjnbkrvapuxlmxgwnwlxeacatox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqgjankclyeauzzzvnmdndhxzzcfejbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qbsgztpdhrqjnzdvqckvfbyesczsczhl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxrfvdxxigdliphyajzufakoxcpztbwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvoobzoiqivrbhimkqfrcogybvqlvmwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"deqqzwygzqtbrsxayxkhcrcyvlxnxjem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsenyhepulorgiedlbdzkgfgyynlxuwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqtykkzemfktgzpebuehrwvdpolzmgxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kblzysboybyjcyyroehuaoldbkatijzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjsvohpxhgeaofijnmztpxklwkkukmwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kixludaluodndisrpqsaccpysuvsxzqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrrurfpylefagxupaqlxaehbgikqsikb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"trkxzxdfjwxnjeaquaavgxnlvrcxwkse","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eiylbogroidlgnxkfcmlxpslswpdqjtd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xotvekpfhrdvsbrwsvtbvojmunsbxztu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpifiagefnibbrxbzdgdyrstrcnutlmm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjanchttqsawoivinfdsyoxeaqhbqehh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"teqnwaralbjkzbygpsvkjgicmmbaxbvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzopcfeddsakhyutaqxmstjddtqrgnsh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rieuwxvladbivhherijjokefqaccysrs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdhccyaulgunfszxtvfulmzqxjosbzne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iblisasiofguuspvztvwezfdsmcmnhoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uougerovjonypxhsxpkfaqvqcuupzfxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmrtbzwnfggpgsvyvkuveztdoymnmhni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vuirkafbjbssqgvplctprwncuqeiulgi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjhfqxkkdlsjfuaoishmxbqlfcbawkri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxjzqxzefiqxvodcwpdlidnelgeqexca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpltvaqhykpitzagvhwyxbzqdfempdob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrbzhdwlvkkcfomlwictbwogvxgwdonc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnukjsbwjeereicoxomejajgxgpwgcjt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"teqqkxhjcnipwyfkbkmiczcpkchetmsu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fdjwkkfqmmgpprjgqrvtgyduwkexzvdh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjiixrizntdrtaskegyifqyorlgpzjij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"safivwhwikndhhyrsngmourgwddhbyip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ysknfyjutgrbssmhxmmyeetvafgksnbl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qeasxwxpovaupcnithuuoqfcncirqupx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ogenhkonfovrvqmlulorrsrmcmnciomp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdavlhzsdpekjrnazpopjbybatjfsiig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgukytiyxbdcksouaqpzobzqiqkaoaqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpcihyawlmucwkjngoxneynjwqivdckh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdzwtcakkxsrivcbxofjypbqlpmhfuel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dynjyndutduejpsyewqugqfnwbhwvepv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pajyplbalkbfovyjozkvmgulqgbxymvm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gaxrddskoghmvmmgipvlvitasygycfdb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdynqctblgdefhmyqjzloubwfdemflkj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suyffdytmarmhqfgnistyjkklvavkmel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubegupbjinldxbhjynvwsstzyrbhaucn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pehwpmfzkedbditgtdmaxkqnphyvywwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aaegiaqfkakkwcyefbxtpesunsenddom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwfmebtrnwxnxfjhehsrrmmawccwwrnv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zztvznuushvsqsovzxwndqafqqrizrcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wchaptrghyajrydasukcmxgnrfolestg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxucdcbjbonyntqqnihzurklagjpwqrn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acdaalghpbknwahsjopwfrrqkyilfryv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kktqglfjqircgymzmqjmzmccsoeawbaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fgjcmjvrflvaturwsrnqcsinucgbzbxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xaqxsrvhcxbpzchmppuzwujsmgzijcoj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iokameedjvqkwndovpfahaikjoorcflk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kdzrtyfxgwzqulbyrlppvczfbbpxgkil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sxarvaakkhlirdyxebuiuixsgvblbbtw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670089,"databaseName":"models_schema","ddl":"CREATE TABLE `pefrclzfkqemnrdgutlyvuoyvtzxmtuv` (\n `xrxldpbalpglkelrdbnyvwdttxdwhyss` int NOT NULL,\n `wpoyoeliyabsxidxzgacxlqkxuyyxspq` int DEFAULT NULL,\n `krynnzduasxotqcivvpwyidgkghzbuzz` int DEFAULT NULL,\n `bskkpbdjuygnoxoqdobxauoispoerage` int DEFAULT NULL,\n `jrfyveurpvtedqxqigixocpjvojmlqkl` int DEFAULT NULL,\n `gxniksjvgxjaryobvzxbrcjiikggtbnx` int DEFAULT NULL,\n `pqupwdpbyktuheucrykmsidjngmwnpie` int DEFAULT NULL,\n `mydevxuwuokesojkqdhrwqmybbgsuung` int DEFAULT NULL,\n `uqrtcorguuglilzcresxmlgqcyxjvecf` int DEFAULT NULL,\n `vrpkaonakogmiivnisumsqllqiufwvkt` int DEFAULT NULL,\n `ampcpmqpkpdmdjoncowjebugdcnowjjw` int DEFAULT NULL,\n `xoypyhgclfuvzzikcsgrjdhjaitqosjs` int DEFAULT NULL,\n `ijotavhqfxflvnlwvkrdicssxefaejmd` int DEFAULT NULL,\n `jkbpyyqydvkvnctjxajnbtyfgxunwyeq` int DEFAULT NULL,\n `itqzcpxewzfvlhsjwtfudozeihagfkgu` int DEFAULT NULL,\n `prrurgdilrmihuzzkqdwbuyqpweuxale` int DEFAULT NULL,\n `qoiiuqrvkdlurqjzsveycrdfztzunmju` int DEFAULT NULL,\n `ranmzwlteffjbvyugqkslbnkbbmhflmx` int DEFAULT NULL,\n `kvbbuqeyhcwjtolixlnnsnpuomgcvuay` int DEFAULT NULL,\n `wwzsomhsyhhacnelfdojalvzurjgrrqf` int DEFAULT NULL,\n `agjzbhcqvhjwjkpohmmhpqilaigwtnrn` int DEFAULT NULL,\n `xxeihwwixxvuaomqtohydjmpdxqyzubn` int DEFAULT NULL,\n `vsxsospjccxvxhjqofgqweytaevodbag` int DEFAULT NULL,\n `kxvmfkwiaaedasicdvsfqsrvpanbfkpc` int DEFAULT NULL,\n `snacbyezytvkomrqrezyscgcbjjtjrai` int DEFAULT NULL,\n `ebwyycockueqsbdiazdgvprlwoqponlk` int DEFAULT NULL,\n `tcknvddzbgarzvbmtuqvyeajxlhqlfof` int DEFAULT NULL,\n `fnmgrsyppxhsaotljocqtzqgeemujlhm` int DEFAULT NULL,\n `mxnnsgzrwnvklmpfjppubewxfdyfkgav` int DEFAULT NULL,\n `eequyduirppfmhhqvdnjcftrnlnqhmbd` int DEFAULT NULL,\n `reivggopeswcghjzbojwcrqndbxqmemp` int DEFAULT NULL,\n `dapzrkzulexgrbhwouivtioojlwlxtsh` int DEFAULT NULL,\n `eghtstanjbwyssrndmezmtphdxmpquob` int DEFAULT NULL,\n `edwekyyzuegljygjnoamdfkhuscasnnh` int DEFAULT NULL,\n `nyfwmknxllfbxtbusxnusylsigaahfrw` int DEFAULT NULL,\n `obetekrcpqjmypsnbmalvzlyvogdmffr` int DEFAULT NULL,\n `edydumecicasusfqyjjkmxevqyyyoscz` int DEFAULT NULL,\n `dcekrynjdrshokapeqtcjetujgefyyhb` int DEFAULT NULL,\n `qebhxswxunqwipavbcaqvghblgabhydf` int DEFAULT NULL,\n `oautvpcjliidxtdyaymjtjlenuwaytah` int DEFAULT NULL,\n `uwnsupkgpkpvguopbfywjjhaayspwiqy` int DEFAULT NULL,\n `zxvxqkldxupyintcovfnvdpkdxcniqre` int DEFAULT NULL,\n `jdjsqqdaknvozmbmrrnugzfhbfxdcbnh` int DEFAULT NULL,\n `pxxwsxcxslkmrroodhywjtzamrbzcqka` int DEFAULT NULL,\n `nttsadjtazubybvcuwxcjfovwwgidmak` int DEFAULT NULL,\n `vwymfkysyemouoyopgppyilheshaipsi` int DEFAULT NULL,\n `ymoxfyhrjkhanqylvfbesanvazlxwsaw` int DEFAULT NULL,\n `bsajzzymfdxijnjoxrvynwfgeqkvvbre` int DEFAULT NULL,\n `cffmsuxufvrsueksbbaxdnswirhdtwux` int DEFAULT NULL,\n `vywgdkuiopktiloailannxnxquufqlxg` int DEFAULT NULL,\n `lhudffojotfzdtfkzszvqucllovnkpih` int DEFAULT NULL,\n `nrmhsixmkelwaauygafxdxapauelovdz` int DEFAULT NULL,\n `vdvxqtbyeqogibaaiaijkzfgzjykbfbg` int DEFAULT NULL,\n `htuwpgcdayqchbiqkfcckynncibnhuxx` int DEFAULT NULL,\n `xuppihrgbttxhxktskrpninigwdhciuw` int DEFAULT NULL,\n `utyhhfdovavmbjytqhwopmmgddqmcbuc` int DEFAULT NULL,\n `sjzgyakwtcfhxszjciiyhsffmbwbbpah` int DEFAULT NULL,\n `jgnvfgfxmrpvntdbqdqpavfbefoylcnj` int DEFAULT NULL,\n `wzyvbxdsdnfbiputgurfthtcisfxvocm` int DEFAULT NULL,\n `ujqjlslzrbfwujlsbzlsvzgtgelqvzic` int DEFAULT NULL,\n `lplkisomdnwztlvxqntwiytqoblblztj` int DEFAULT NULL,\n `becmuriwcnrbydcckujgnscwxzdcwged` int DEFAULT NULL,\n `comruwfaujqvntmhflhygvysjjgzveqv` int DEFAULT NULL,\n `uqobcmhboapfeyydwyhsxbudgutceeqn` int DEFAULT NULL,\n `ouxzrsasqksseoxicrjktirjhxyrxdel` int DEFAULT NULL,\n `oehewwhrkgaaskxjvpaiuwknoxlpkaml` int DEFAULT NULL,\n `ylkbldajiyyfdyrbyqmyfkkznclyefhq` int DEFAULT NULL,\n `vmmjzsuyjpgowvnhhsogcwtfspxnnbxu` int DEFAULT NULL,\n `qrdrqtsoiarcrhzgbhpxsnqgpsrbevat` int DEFAULT NULL,\n `gaajyanlqrpfkwlelizwzfcwslkxfapp` int DEFAULT NULL,\n `hbxbkodfwbinhgqdxvkucrnqrvheriej` int DEFAULT NULL,\n `pqkzlwiovkqdaezkpnjpoblspcjvvuop` int DEFAULT NULL,\n `imxszofkawydrwqogyfsibnrwvutyeyg` int DEFAULT NULL,\n `xsklhbvzsatdzimbeptpcylvjxafmwwz` int DEFAULT NULL,\n `ckpmbewbchwnyjgllbednqmvvcpqpwfz` int DEFAULT NULL,\n `nkhepnbcynvarotphluojtxztsjrysmi` int DEFAULT NULL,\n `gwsyecglamiarwtmefwsdngkvoxzmear` int DEFAULT NULL,\n `gdxuoprgoplkvkplqwtrwvndwqeiwbua` int DEFAULT NULL,\n `jvftnxkisadjmbpdiidiazwsuuatejcr` int DEFAULT NULL,\n `zcwpvvgxikfkhswvtoxmvkchblmvglun` int DEFAULT NULL,\n `ftkbqjjfbhhlckpaflemslajgdjxwipn` int DEFAULT NULL,\n `xqqzhdkbmepmoopoqlzgjvzvjpoxcrhb` int DEFAULT NULL,\n `ldzfktrrxvfkxqqgzwgqddqpsliligjl` int DEFAULT NULL,\n `sksjhssrzemvmuhcpkmjgyhuwowoidro` int DEFAULT NULL,\n `tnffjcomwdanyenasyseomlqhkhceqnn` int DEFAULT NULL,\n `svbktnapqmefkvhatxmabjyvclnnocma` int DEFAULT NULL,\n `asqlnwjodsifosdenxpzinvwsczlxzro` int DEFAULT NULL,\n `zmzmyhyohldwhjnvaaklmxgjwonmlhyy` int DEFAULT NULL,\n `unjjlaygzkveiddficfbvqczchlqfjwf` int DEFAULT NULL,\n `pnkvfrbahgoyvrbmfzweyfxqaypwndlu` int DEFAULT NULL,\n `dbwztldcbulpqaineaeflpilixwkcciy` int DEFAULT NULL,\n `cwqdkfyzicnydbmcrjrdfwaaqngtkzvf` int DEFAULT NULL,\n `znpaawcfhyegmticwtboftzpsujjpxxw` int DEFAULT NULL,\n `nlxhhoouahskhzdrdlpkyytvyqqocssj` int DEFAULT NULL,\n `tpsfpbovpophrbkkyumobdfjssytekil` int DEFAULT NULL,\n `mvbinjfrdwtexelttsttcqrrehvmcucw` int DEFAULT NULL,\n `cagppuzsiifhvdklodqcczvwekitlehw` int DEFAULT NULL,\n `iupertoltyspuscmazjtakczhwcavdcd` int DEFAULT NULL,\n `dcmfxoxpjshvncadbfknmfsxdtumbmhv` int DEFAULT NULL,\n `tavvpqriqrzeaqwgcjmiaomjulqwwsby` int DEFAULT NULL,\n PRIMARY KEY (`xrxldpbalpglkelrdbnyvwdttxdwhyss`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"pefrclzfkqemnrdgutlyvuoyvtzxmtuv\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["xrxldpbalpglkelrdbnyvwdttxdwhyss"],"columns":[{"name":"xrxldpbalpglkelrdbnyvwdttxdwhyss","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"wpoyoeliyabsxidxzgacxlqkxuyyxspq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krynnzduasxotqcivvpwyidgkghzbuzz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bskkpbdjuygnoxoqdobxauoispoerage","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jrfyveurpvtedqxqigixocpjvojmlqkl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxniksjvgxjaryobvzxbrcjiikggtbnx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqupwdpbyktuheucrykmsidjngmwnpie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mydevxuwuokesojkqdhrwqmybbgsuung","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqrtcorguuglilzcresxmlgqcyxjvecf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrpkaonakogmiivnisumsqllqiufwvkt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ampcpmqpkpdmdjoncowjebugdcnowjjw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xoypyhgclfuvzzikcsgrjdhjaitqosjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijotavhqfxflvnlwvkrdicssxefaejmd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jkbpyyqydvkvnctjxajnbtyfgxunwyeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itqzcpxewzfvlhsjwtfudozeihagfkgu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prrurgdilrmihuzzkqdwbuyqpweuxale","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qoiiuqrvkdlurqjzsveycrdfztzunmju","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ranmzwlteffjbvyugqkslbnkbbmhflmx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvbbuqeyhcwjtolixlnnsnpuomgcvuay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwzsomhsyhhacnelfdojalvzurjgrrqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agjzbhcqvhjwjkpohmmhpqilaigwtnrn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxeihwwixxvuaomqtohydjmpdxqyzubn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsxsospjccxvxhjqofgqweytaevodbag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxvmfkwiaaedasicdvsfqsrvpanbfkpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"snacbyezytvkomrqrezyscgcbjjtjrai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebwyycockueqsbdiazdgvprlwoqponlk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tcknvddzbgarzvbmtuqvyeajxlhqlfof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnmgrsyppxhsaotljocqtzqgeemujlhm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mxnnsgzrwnvklmpfjppubewxfdyfkgav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eequyduirppfmhhqvdnjcftrnlnqhmbd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"reivggopeswcghjzbojwcrqndbxqmemp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dapzrkzulexgrbhwouivtioojlwlxtsh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eghtstanjbwyssrndmezmtphdxmpquob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edwekyyzuegljygjnoamdfkhuscasnnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nyfwmknxllfbxtbusxnusylsigaahfrw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obetekrcpqjmypsnbmalvzlyvogdmffr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edydumecicasusfqyjjkmxevqyyyoscz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcekrynjdrshokapeqtcjetujgefyyhb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qebhxswxunqwipavbcaqvghblgabhydf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oautvpcjliidxtdyaymjtjlenuwaytah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwnsupkgpkpvguopbfywjjhaayspwiqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxvxqkldxupyintcovfnvdpkdxcniqre","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdjsqqdaknvozmbmrrnugzfhbfxdcbnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxxwsxcxslkmrroodhywjtzamrbzcqka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nttsadjtazubybvcuwxcjfovwwgidmak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vwymfkysyemouoyopgppyilheshaipsi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ymoxfyhrjkhanqylvfbesanvazlxwsaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bsajzzymfdxijnjoxrvynwfgeqkvvbre","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cffmsuxufvrsueksbbaxdnswirhdtwux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vywgdkuiopktiloailannxnxquufqlxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhudffojotfzdtfkzszvqucllovnkpih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nrmhsixmkelwaauygafxdxapauelovdz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdvxqtbyeqogibaaiaijkzfgzjykbfbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htuwpgcdayqchbiqkfcckynncibnhuxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xuppihrgbttxhxktskrpninigwdhciuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utyhhfdovavmbjytqhwopmmgddqmcbuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjzgyakwtcfhxszjciiyhsffmbwbbpah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgnvfgfxmrpvntdbqdqpavfbefoylcnj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzyvbxdsdnfbiputgurfthtcisfxvocm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujqjlslzrbfwujlsbzlsvzgtgelqvzic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lplkisomdnwztlvxqntwiytqoblblztj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"becmuriwcnrbydcckujgnscwxzdcwged","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"comruwfaujqvntmhflhygvysjjgzveqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqobcmhboapfeyydwyhsxbudgutceeqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ouxzrsasqksseoxicrjktirjhxyrxdel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oehewwhrkgaaskxjvpaiuwknoxlpkaml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ylkbldajiyyfdyrbyqmyfkkznclyefhq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmmjzsuyjpgowvnhhsogcwtfspxnnbxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrdrqtsoiarcrhzgbhpxsnqgpsrbevat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gaajyanlqrpfkwlelizwzfcwslkxfapp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbxbkodfwbinhgqdxvkucrnqrvheriej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqkzlwiovkqdaezkpnjpoblspcjvvuop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"imxszofkawydrwqogyfsibnrwvutyeyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsklhbvzsatdzimbeptpcylvjxafmwwz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckpmbewbchwnyjgllbednqmvvcpqpwfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkhepnbcynvarotphluojtxztsjrysmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gwsyecglamiarwtmefwsdngkvoxzmear","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdxuoprgoplkvkplqwtrwvndwqeiwbua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvftnxkisadjmbpdiidiazwsuuatejcr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zcwpvvgxikfkhswvtoxmvkchblmvglun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftkbqjjfbhhlckpaflemslajgdjxwipn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqqzhdkbmepmoopoqlzgjvzvjpoxcrhb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ldzfktrrxvfkxqqgzwgqddqpsliligjl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sksjhssrzemvmuhcpkmjgyhuwowoidro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnffjcomwdanyenasyseomlqhkhceqnn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"svbktnapqmefkvhatxmabjyvclnnocma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"asqlnwjodsifosdenxpzinvwsczlxzro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmzmyhyohldwhjnvaaklmxgjwonmlhyy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unjjlaygzkveiddficfbvqczchlqfjwf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnkvfrbahgoyvrbmfzweyfxqaypwndlu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbwztldcbulpqaineaeflpilixwkcciy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwqdkfyzicnydbmcrjrdfwaaqngtkzvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znpaawcfhyegmticwtboftzpsujjpxxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nlxhhoouahskhzdrdlpkyytvyqqocssj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpsfpbovpophrbkkyumobdfjssytekil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvbinjfrdwtexelttsttcqrrehvmcucw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cagppuzsiifhvdklodqcczvwekitlehw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iupertoltyspuscmazjtakczhwcavdcd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcmfxoxpjshvncadbfknmfsxdtumbmhv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tavvpqriqrzeaqwgcjmiaomjulqwwsby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670120,"databaseName":"models_schema","ddl":"CREATE TABLE `pfhgocfvrcmabgqdldeflyidqlcdhbmz` (\n `xdovsxfkvhqivpjctcauwzmymelzwmnm` int NOT NULL,\n `apdburzkadpaimxoeianexpkaegdsivs` int DEFAULT NULL,\n `rhkklvjetjktaynotqpzigwkprgbjcns` int DEFAULT NULL,\n `lirjrphmuerbvhogbmfycoufcjmntxcl` int DEFAULT NULL,\n `hhahptkfskyyoenulgawyomjwamozndt` int DEFAULT NULL,\n `xvmontuzjpjyvxanjryapwnkdkitzrwu` int DEFAULT NULL,\n `crdzsytvrntypfwybgeirfoepmlikjsb` int DEFAULT NULL,\n `zpmqzuknjuzknupiioammhnqaznycspq` int DEFAULT NULL,\n `zcyjjswnbwyfvjowplhicoorbjanrnko` int DEFAULT NULL,\n `dbghpgnzmdcjfxevhdougjecncqtnfsp` int DEFAULT NULL,\n `sdsrrdbrczpgxaousrwakvwltnlpkesv` int DEFAULT NULL,\n `ohouiejplpcyxdticaqpbwreeadnrfqy` int DEFAULT NULL,\n `mjadsyodjhxmlojbvwgwgecymixblljg` int DEFAULT NULL,\n `knkyeaslgingnhqtzaphkhdzwwkbrdyq` int DEFAULT NULL,\n `tsnmuypkeanlxvncztwnauenixwhzgtc` int DEFAULT NULL,\n `zpqfnjtwtazmaeygjvcdymrxhqwuowus` int DEFAULT NULL,\n `ntyactqxuhgycjjdrlqzyewejjibqjqb` int DEFAULT NULL,\n `pdkbdxeqnwhxvnbhwolahmdvlupclfeb` int DEFAULT NULL,\n `wininhlwqbekmlhsvrceuhggqfqvhxac` int DEFAULT NULL,\n `geflwqcoexwmotpyqypfhblqfvwqyuva` int DEFAULT NULL,\n `nbfkpbdxxhvjefgkpwfwskbuwqokewfw` int DEFAULT NULL,\n `pxcvztqbhhtjyiwhmtnjtvueomssniqv` int DEFAULT NULL,\n `aqtiuebauosgdrfwkxqdrcrfvjzyjqpo` int DEFAULT NULL,\n `pjejjhbqpxiwzssmszurvurfykmanqmo` int DEFAULT NULL,\n `owkxyqnbslqzolwtjdlflbjvgqbvitjm` int DEFAULT NULL,\n `pjsweuhqwhrvgpqfsxwzfcvuefzgewve` int DEFAULT NULL,\n `xenqjeznvzmnqtllsfxfktywwgcavxgl` int DEFAULT NULL,\n `cutrplpliulajfqdjuqgzuuwkvlzdquw` int DEFAULT NULL,\n `rmipfavzetpvcqgmfuviocivjqcdewai` int DEFAULT NULL,\n `rmoonprgbjjavstqrgkpwevwzlvmodzd` int DEFAULT NULL,\n `kxntotymifktkmonfscxzexgsezrwdnl` int DEFAULT NULL,\n `xvdxkpreylzxqpgslxwbltcqszmlyrbt` int DEFAULT NULL,\n `xvjnsvqzlafapzpgiysecugetwmhrsrq` int DEFAULT NULL,\n `qdtwymifesegcwhwwjrzoazodyisyjhd` int DEFAULT NULL,\n `acpyhngrtwzsfdhitzxefoeofxcumtsc` int DEFAULT NULL,\n `elaqrqllkchjynhdbtempyzbfwroroqb` int DEFAULT NULL,\n `xkrjuwstrmgvieqzsmpjfzjtnjznteob` int DEFAULT NULL,\n `nhlmjcbwkcwzpjtjsizviuydglpuuqfi` int DEFAULT NULL,\n `yfjjcseyuinieghotfdhxplzyhgillxv` int DEFAULT NULL,\n `vdrvmyijgnkgtdhhzxgezrmmrkhrdxxj` int DEFAULT NULL,\n `qgwruagffyawdcugbltoqokeaxpxdxmw` int DEFAULT NULL,\n `uvklcaqdqlnowvnllenpyehpmwkwyqde` int DEFAULT NULL,\n `jnsihvwpuqfaodkitmoenozencbldpmu` int DEFAULT NULL,\n `tcacfvgksmbaxkzpyyanwyjjoreuuwrd` int DEFAULT NULL,\n `jkfkmmiyjraomowfvdterbmjlywjalqu` int DEFAULT NULL,\n `cbvoxhtyqmwhqxryyyhizxwdfthzghfc` int DEFAULT NULL,\n `ormemckvfsbcoqdedhaerxfglcatixag` int DEFAULT NULL,\n `fvlemqoieiczqxxqubbdzozqthpeptdp` int DEFAULT NULL,\n `rryjqtlurckuycgzacrheimoufdfledy` int DEFAULT NULL,\n `lldpydpfryrjwjozengjndxworkzywzo` int DEFAULT NULL,\n `xlchumflxdwatwapoksqqepsflgghlea` int DEFAULT NULL,\n `cugltgbqkpagssgldmixgmmkglyraluq` int DEFAULT NULL,\n `iccnbkswnyuyhnijgadopufpppfwsrvx` int DEFAULT NULL,\n `tqxgxingrojobhzerbkyefinsekkxwxg` int DEFAULT NULL,\n `uhgvtdzdxcprrtdljkooklszulthphvi` int DEFAULT NULL,\n `duucxpxwqqpefxrvfusnybxxbpwnbglm` int DEFAULT NULL,\n `cbmihdxqfjzwtwusitbrurwfaoroxcai` int DEFAULT NULL,\n `aiuhnrinawmqnuvgrnuctscpahnjnahb` int DEFAULT NULL,\n `exaoipabyxalhbwqplequaiveralxdcr` int DEFAULT NULL,\n `ntoqzaqilrpefvtlnuerwanssjxzjees` int DEFAULT NULL,\n `hosntopcsfldlpjibokghoxnlxxjycbs` int DEFAULT NULL,\n `egmngqscbmgomalazodoalwgtmapvtek` int DEFAULT NULL,\n `spltbwgtgfwzzhslugzqlajgnijgpsmf` int DEFAULT NULL,\n `yaolvtamyosijsctqxlkbypcojunvuyc` int DEFAULT NULL,\n `kxhskmyiiysfzarycadeshfnzjtzjllo` int DEFAULT NULL,\n `jukyryllpbnaconmaasbyxlnlisavauw` int DEFAULT NULL,\n `erjwtxqghdhobekiapdsamqxqljwsmvi` int DEFAULT NULL,\n `iiyizrvjxoaflwkfhbcgzdhthyuoopyu` int DEFAULT NULL,\n `ufhmdpdwnxekeuvqvqgxhcyspqfdthwg` int DEFAULT NULL,\n `gnhjcryymhgvzvtythhlktvfhuibemcp` int DEFAULT NULL,\n `ocrdfiguhquvhjpfkbjvwlqrekoxeoei` int DEFAULT NULL,\n `pxyccjrfkqqwdvlypttrypeqigsvotvp` int DEFAULT NULL,\n `gjpqygfmgcpuzhupseygoohbbhyazapk` int DEFAULT NULL,\n `lyanypnlmrsjgwoawmrxgbpftxvenmig` int DEFAULT NULL,\n `ympnrlctgcxjztwwhojbkhanxjhdozbp` int DEFAULT NULL,\n `meefsbxkozquzvchhoeitvnmabvcfsar` int DEFAULT NULL,\n `vnmvlqmextrlhfxxxrwofjcmpljptfbv` int DEFAULT NULL,\n `yefbkkzvxwhjxwykovhvmhfcumjmmsac` int DEFAULT NULL,\n `cktidfqqjaskfxtqfmtqcoiqccwofavm` int DEFAULT NULL,\n `rxyrvtkacwhgmwowkpumuehxmuukoipu` int DEFAULT NULL,\n `erqniuclimtrqgifuqffsjlbtabflkxc` int DEFAULT NULL,\n `fpqpteotbbmkbxgnfdimypovfwjvmgno` int DEFAULT NULL,\n `cbjvftowoawyvppfexsgmltioefbwmkr` int DEFAULT NULL,\n `afmyczqlhjzwrvrzcrfthcyzseytiflp` int DEFAULT NULL,\n `gnmxfcvgfaudeguimzsecjrnhfhkokbr` int DEFAULT NULL,\n `deqqeozhweuyoykzdqxdgkzsnnliiphl` int DEFAULT NULL,\n `qtalefvvaeggvwpqmtflhvczhfblpzyg` int DEFAULT NULL,\n `cavvokrbpheuxhmkomkcphuqssdynwti` int DEFAULT NULL,\n `lshlwdljyzldcfqhvivfbynmdnakyanw` int DEFAULT NULL,\n `bfltgnsnhjbofwosgtiwouaheaisfdur` int DEFAULT NULL,\n `ocqdgtseaokuejeldjbyulmthyfuraob` int DEFAULT NULL,\n `aijesrdebfimvfkrscrezkzstbvmonjl` int DEFAULT NULL,\n `nfgvwigjonygmtxyobwhubmdsazlyyjj` int DEFAULT NULL,\n `iqfndvbnphqsnzcxlzfswqptdbebjrxg` int DEFAULT NULL,\n `wykbtoaxxiayucyicubkljuuvbumosnb` int DEFAULT NULL,\n `xbqteiadmouazlokshbmvbrpfkdbueqj` int DEFAULT NULL,\n `kweqbqiwfsfnfimtfvdxsukauifpfemk` int DEFAULT NULL,\n `xbrepilumnagpcgvxghpchjxmoggqofl` int DEFAULT NULL,\n `fxmmwqyvnbxxubllwdbhzdokhrhjqayh` int DEFAULT NULL,\n `orwaalnrzwbjnudksertmliuxjyjvjaj` int DEFAULT NULL,\n PRIMARY KEY (`xdovsxfkvhqivpjctcauwzmymelzwmnm`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"pfhgocfvrcmabgqdldeflyidqlcdhbmz\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["xdovsxfkvhqivpjctcauwzmymelzwmnm"],"columns":[{"name":"xdovsxfkvhqivpjctcauwzmymelzwmnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"apdburzkadpaimxoeianexpkaegdsivs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhkklvjetjktaynotqpzigwkprgbjcns","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lirjrphmuerbvhogbmfycoufcjmntxcl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhahptkfskyyoenulgawyomjwamozndt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvmontuzjpjyvxanjryapwnkdkitzrwu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crdzsytvrntypfwybgeirfoepmlikjsb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpmqzuknjuzknupiioammhnqaznycspq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zcyjjswnbwyfvjowplhicoorbjanrnko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbghpgnzmdcjfxevhdougjecncqtnfsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdsrrdbrczpgxaousrwakvwltnlpkesv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohouiejplpcyxdticaqpbwreeadnrfqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjadsyodjhxmlojbvwgwgecymixblljg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knkyeaslgingnhqtzaphkhdzwwkbrdyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tsnmuypkeanlxvncztwnauenixwhzgtc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpqfnjtwtazmaeygjvcdymrxhqwuowus","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntyactqxuhgycjjdrlqzyewejjibqjqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdkbdxeqnwhxvnbhwolahmdvlupclfeb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wininhlwqbekmlhsvrceuhggqfqvhxac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"geflwqcoexwmotpyqypfhblqfvwqyuva","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbfkpbdxxhvjefgkpwfwskbuwqokewfw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxcvztqbhhtjyiwhmtnjtvueomssniqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqtiuebauosgdrfwkxqdrcrfvjzyjqpo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjejjhbqpxiwzssmszurvurfykmanqmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owkxyqnbslqzolwtjdlflbjvgqbvitjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjsweuhqwhrvgpqfsxwzfcvuefzgewve","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xenqjeznvzmnqtllsfxfktywwgcavxgl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cutrplpliulajfqdjuqgzuuwkvlzdquw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rmipfavzetpvcqgmfuviocivjqcdewai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rmoonprgbjjavstqrgkpwevwzlvmodzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxntotymifktkmonfscxzexgsezrwdnl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvdxkpreylzxqpgslxwbltcqszmlyrbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvjnsvqzlafapzpgiysecugetwmhrsrq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdtwymifesegcwhwwjrzoazodyisyjhd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acpyhngrtwzsfdhitzxefoeofxcumtsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elaqrqllkchjynhdbtempyzbfwroroqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkrjuwstrmgvieqzsmpjfzjtnjznteob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhlmjcbwkcwzpjtjsizviuydglpuuqfi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfjjcseyuinieghotfdhxplzyhgillxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdrvmyijgnkgtdhhzxgezrmmrkhrdxxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgwruagffyawdcugbltoqokeaxpxdxmw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvklcaqdqlnowvnllenpyehpmwkwyqde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jnsihvwpuqfaodkitmoenozencbldpmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tcacfvgksmbaxkzpyyanwyjjoreuuwrd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jkfkmmiyjraomowfvdterbmjlywjalqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbvoxhtyqmwhqxryyyhizxwdfthzghfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ormemckvfsbcoqdedhaerxfglcatixag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fvlemqoieiczqxxqubbdzozqthpeptdp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rryjqtlurckuycgzacrheimoufdfledy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lldpydpfryrjwjozengjndxworkzywzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xlchumflxdwatwapoksqqepsflgghlea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cugltgbqkpagssgldmixgmmkglyraluq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iccnbkswnyuyhnijgadopufpppfwsrvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqxgxingrojobhzerbkyefinsekkxwxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhgvtdzdxcprrtdljkooklszulthphvi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"duucxpxwqqpefxrvfusnybxxbpwnbglm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbmihdxqfjzwtwusitbrurwfaoroxcai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aiuhnrinawmqnuvgrnuctscpahnjnahb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exaoipabyxalhbwqplequaiveralxdcr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntoqzaqilrpefvtlnuerwanssjxzjees","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hosntopcsfldlpjibokghoxnlxxjycbs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"egmngqscbmgomalazodoalwgtmapvtek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"spltbwgtgfwzzhslugzqlajgnijgpsmf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yaolvtamyosijsctqxlkbypcojunvuyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxhskmyiiysfzarycadeshfnzjtzjllo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jukyryllpbnaconmaasbyxlnlisavauw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"erjwtxqghdhobekiapdsamqxqljwsmvi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iiyizrvjxoaflwkfhbcgzdhthyuoopyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufhmdpdwnxekeuvqvqgxhcyspqfdthwg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnhjcryymhgvzvtythhlktvfhuibemcp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocrdfiguhquvhjpfkbjvwlqrekoxeoei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxyccjrfkqqwdvlypttrypeqigsvotvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjpqygfmgcpuzhupseygoohbbhyazapk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lyanypnlmrsjgwoawmrxgbpftxvenmig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ympnrlctgcxjztwwhojbkhanxjhdozbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"meefsbxkozquzvchhoeitvnmabvcfsar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnmvlqmextrlhfxxxrwofjcmpljptfbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yefbkkzvxwhjxwykovhvmhfcumjmmsac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cktidfqqjaskfxtqfmtqcoiqccwofavm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxyrvtkacwhgmwowkpumuehxmuukoipu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"erqniuclimtrqgifuqffsjlbtabflkxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpqpteotbbmkbxgnfdimypovfwjvmgno","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbjvftowoawyvppfexsgmltioefbwmkr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afmyczqlhjzwrvrzcrfthcyzseytiflp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnmxfcvgfaudeguimzsecjrnhfhkokbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"deqqeozhweuyoykzdqxdgkzsnnliiphl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtalefvvaeggvwpqmtflhvczhfblpzyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cavvokrbpheuxhmkomkcphuqssdynwti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lshlwdljyzldcfqhvivfbynmdnakyanw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bfltgnsnhjbofwosgtiwouaheaisfdur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocqdgtseaokuejeldjbyulmthyfuraob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aijesrdebfimvfkrscrezkzstbvmonjl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfgvwigjonygmtxyobwhubmdsazlyyjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqfndvbnphqsnzcxlzfswqptdbebjrxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wykbtoaxxiayucyicubkljuuvbumosnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbqteiadmouazlokshbmvbrpfkdbueqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kweqbqiwfsfnfimtfvdxsukauifpfemk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbrepilumnagpcgvxghpchjxmoggqofl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxmmwqyvnbxxubllwdbhzdokhrhjqayh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orwaalnrzwbjnudksertmliuxjyjvjaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670154,"databaseName":"models_schema","ddl":"CREATE TABLE `pfqjiwqnmbfnwysrzbqigotyuehiiwaz` (\n `blpojeiwfwymwzcrqfszfpngkgcmltkf` int NOT NULL,\n `solnpencsdgfzobjnhboktlgnfvezbiz` int DEFAULT NULL,\n `jgmijnhcjijypeidxrllcajodgasivpf` int DEFAULT NULL,\n `ojxnrwvuhebjhawpawewlzeybyrflvhq` int DEFAULT NULL,\n `vzeppnvsavbgghkerbxdyqgswnmokrgi` int DEFAULT NULL,\n `ytcxkciejxvgoobsbcgclcmbmwecpdqv` int DEFAULT NULL,\n `bsgnssdalntqnnjxzinzzepnktcrrken` int DEFAULT NULL,\n `wfehhybdozqaxsslzhxdlxddrorbozoc` int DEFAULT NULL,\n `hcpbrabehjuunwsptauoetwdhtemvuau` int DEFAULT NULL,\n `tnyoldcwzfsqliudmeqcqpotqhvtwiwn` int DEFAULT NULL,\n `xawmzjhghorlcykftzycutdscwgvgnpc` int DEFAULT NULL,\n `hhnjegpkfqypqpbrvhlsjvagtnrhlakk` int DEFAULT NULL,\n `xnouuuidrfapwxckrirwpqkcnlofvkgo` int DEFAULT NULL,\n `iparxfllblrapgyuvnhtlxwdkmsfcbir` int DEFAULT NULL,\n `mjghxkfijkohnxpvwqbfinsxkfwqefbm` int DEFAULT NULL,\n `bxjwhuypovfczfmwafgpeedwlmxfmwrg` int DEFAULT NULL,\n `lgahqwtrfnkmoxmhflmhinpdxqsqmcnx` int DEFAULT NULL,\n `tqruzbwbmxseuholvitezegnbvyztwfw` int DEFAULT NULL,\n `txgtvxnpgvrzmgxdpddacrvspanwuvnw` int DEFAULT NULL,\n `zhgfbiydtlujprffexkcpwifkhdvnguh` int DEFAULT NULL,\n `fuaiyrtwdhurlrownzmwfcricwmhjvop` int DEFAULT NULL,\n `rniswhptsdvuhxrumkpnyhyoyqkoponr` int DEFAULT NULL,\n `dtayttdipllmssbwdedueiqiahwydbzl` int DEFAULT NULL,\n `sallsdpfvfiargpfhetobamjbxhgahqg` int DEFAULT NULL,\n `ylusscobexbykxlrtfiurcvgattbuezr` int DEFAULT NULL,\n `trkjtwigusczoiouutyvruhluknjpuli` int DEFAULT NULL,\n `oqdjyzfucheuvcnhmhuutmtkhtpzcvmb` int DEFAULT NULL,\n `nuwtgdwgrfzpajyxniisyaoabsejynnh` int DEFAULT NULL,\n `ymsofunpmppnqbarjryywspcutnizhfw` int DEFAULT NULL,\n `kxhgnkwxfmkuswmfloeivxmhrgcgqaby` int DEFAULT NULL,\n `ofwnsobjneunifyjjsiuyboapaakccif` int DEFAULT NULL,\n `dtxnbkipmgdumdxyqzekrjojhmtloufk` int DEFAULT NULL,\n `uyufaivtnjfijiqzauwzwgxqheposyxs` int DEFAULT NULL,\n `ijztayhpbzzzowngxhalhzckncofdfun` int DEFAULT NULL,\n `kvbiueabteyfbbmvmvveaaqhodkvpnod` int DEFAULT NULL,\n `bqwbovapjkquonlwpsrykbumhbpzufhu` int DEFAULT NULL,\n `itturaydmvveuybihzfljovhfeondnoj` int DEFAULT NULL,\n `muldbwunjsnscubqvmqqmdufcxmwrsun` int DEFAULT NULL,\n `smatspsvfewkkvmjyakrdiifdzamjzez` int DEFAULT NULL,\n `hxyvfrxcjynccpkkyyulzzrbrqoexltm` int DEFAULT NULL,\n `huwensejwstzsmshqalaqgnyiptziwwl` int DEFAULT NULL,\n `tyxuxrauhdboipwsbbtslbqovdaueybe` int DEFAULT NULL,\n `rcsncsukshqmogdpgidlsijnjlhpjlim` int DEFAULT NULL,\n `vqdzspzffosldmdtwngjekxokluvqrjo` int DEFAULT NULL,\n `phljbenmcpmqxgvvdodccwwpogynnnoh` int DEFAULT NULL,\n `oofcaxfjkpemrwipihylydafbdnfoyuo` int DEFAULT NULL,\n `lwpqiklbgaezpdvyuceekhgllybjwmnn` int DEFAULT NULL,\n `cvztrgqvbpfzqadzrpoaufqcxbmfsmtg` int DEFAULT NULL,\n `xllavcqoawtixlawnljojxjdnercxwma` int DEFAULT NULL,\n `gqgkppzdpkgdkjtufzbnmhmbpixhtgin` int DEFAULT NULL,\n `jlrrcswujcwnrfsetxfisnularseamab` int DEFAULT NULL,\n `skviegnvpshtmaukuvambbyfxvdjkqrt` int DEFAULT NULL,\n `gkjxypvccaidforyztxiavgvjhopxolb` int DEFAULT NULL,\n `sqaxppfsipxysbjakwavhkudbcdyudat` int DEFAULT NULL,\n `vtzttaitzdzlvjaudivnzprmnuyaqfnj` int DEFAULT NULL,\n `edhqdurdsuxndugibbynjfctsuxuutof` int DEFAULT NULL,\n `laplivsarfmafiejemzliptxtjqxakcq` int DEFAULT NULL,\n `vpmdnakmgyresqpetyzplkbvffvbarxf` int DEFAULT NULL,\n `uqvwqmgiiqfjwkroxdyhrpzvsdtsmkvb` int DEFAULT NULL,\n `oxyseiqadnnegqwmeynilxfwturhzsno` int DEFAULT NULL,\n `pdzowqnxvosqzjpdpklbimkppfhunnbl` int DEFAULT NULL,\n `kflzxoporlcbonhqtjonfpqlrtoulmlq` int DEFAULT NULL,\n `xmkccsndpmrvnqdmvjzwxkknfbvrlmic` int DEFAULT NULL,\n `ekskyxhdrxyfipsxrmljcyhjxmhrcfjc` int DEFAULT NULL,\n `btqlshbdjahcumwefactdgfilkoupowr` int DEFAULT NULL,\n `fbpfivdnzlrhixvdvvkqlqdguralcist` int DEFAULT NULL,\n `jpfvweukzyjbgsrgfhbjlsoyzypacecb` int DEFAULT NULL,\n `jnutjuuatjlwbtgqgxybnbfuhwlggqly` int DEFAULT NULL,\n `zllomzeynmpihkpfaoclachanclekjnh` int DEFAULT NULL,\n `tzzhkhqqjhhxccdcdhijqptjbjynphcz` int DEFAULT NULL,\n `utkdpmuxvtsoapgdbgaevwarbeosoaqy` int DEFAULT NULL,\n `khqihxkoftqrklrhkawoicvedfylrqcd` int DEFAULT NULL,\n `euphkmznfucbdvfjqmmmlwsiifxngvzh` int DEFAULT NULL,\n `lptxjzwcvvvxzrokbzdifstypvowqbyf` int DEFAULT NULL,\n `feahmltxcwxgkibdjicrkqknbszadcvm` int DEFAULT NULL,\n `knfltjykdbgqyiiatcwhihspukshamei` int DEFAULT NULL,\n `anesgcgjggetdvnffacknwoniexoowxv` int DEFAULT NULL,\n `trdxmnytxaeikoprvxenizfwfvfzlaoe` int DEFAULT NULL,\n `qhsdvccqtlywdchjtegdxqiedtedadtl` int DEFAULT NULL,\n `iqyxnpeeysnrszrgiagwzabefmdibaid` int DEFAULT NULL,\n `rfsaatbhjvvpbgycllwcixgeviqkzpai` int DEFAULT NULL,\n `omydzbhjzimzplbfzmcepzdtqxqsyqow` int DEFAULT NULL,\n `fixmxubeteqglcizzovgkxsyasrkdgns` int DEFAULT NULL,\n `diiesxdlvkvbhfuhkpcjjhhtxqyrspcs` int DEFAULT NULL,\n `qylztegkrsmsxohfyowcmioapqrsptot` int DEFAULT NULL,\n `ehduzgotpyxqcutzkcerlozbgkuuzlgy` int DEFAULT NULL,\n `xrtlqbkrpxopqjfczgfkegfxcjnfpcpi` int DEFAULT NULL,\n `kcexwygaioddcejgguuobmyleoinwxgo` int DEFAULT NULL,\n `gvvwlwhrsdedfxezcalevxcbowgcnblv` int DEFAULT NULL,\n `mzairmlkgpozhsueivjxrgbfctfjvkbg` int DEFAULT NULL,\n `vcyvemrqnfvyiifonkaledstfldaghci` int DEFAULT NULL,\n `llsqrfreryfngockgyekhnzwetrmdume` int DEFAULT NULL,\n `iedsrxgwmajeafczjftymvywcnzxqtal` int DEFAULT NULL,\n `nyxucehtwahpshzpcjiojqpmleuahvfc` int DEFAULT NULL,\n `xwblxqfmznfczympdtflpsusyterrmhn` int DEFAULT NULL,\n `ynivunbjqyeyfnojmyqnpezyhluuteeg` int DEFAULT NULL,\n `ooaxevucibnafhrovpvexckhevrdeaql` int DEFAULT NULL,\n `abdoqtpvvdqokikxxgkhnwscfppwmwdu` int DEFAULT NULL,\n `okdmqfkclagkgluoqoyvyagzocecbsle` int DEFAULT NULL,\n `celxhansaudrpkmzjktzlupxxhxllfww` int DEFAULT NULL,\n PRIMARY KEY (`blpojeiwfwymwzcrqfszfpngkgcmltkf`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"pfqjiwqnmbfnwysrzbqigotyuehiiwaz\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["blpojeiwfwymwzcrqfszfpngkgcmltkf"],"columns":[{"name":"blpojeiwfwymwzcrqfszfpngkgcmltkf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"solnpencsdgfzobjnhboktlgnfvezbiz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgmijnhcjijypeidxrllcajodgasivpf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojxnrwvuhebjhawpawewlzeybyrflvhq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzeppnvsavbgghkerbxdyqgswnmokrgi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytcxkciejxvgoobsbcgclcmbmwecpdqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bsgnssdalntqnnjxzinzzepnktcrrken","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfehhybdozqaxsslzhxdlxddrorbozoc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hcpbrabehjuunwsptauoetwdhtemvuau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnyoldcwzfsqliudmeqcqpotqhvtwiwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xawmzjhghorlcykftzycutdscwgvgnpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhnjegpkfqypqpbrvhlsjvagtnrhlakk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnouuuidrfapwxckrirwpqkcnlofvkgo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iparxfllblrapgyuvnhtlxwdkmsfcbir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjghxkfijkohnxpvwqbfinsxkfwqefbm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bxjwhuypovfczfmwafgpeedwlmxfmwrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgahqwtrfnkmoxmhflmhinpdxqsqmcnx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqruzbwbmxseuholvitezegnbvyztwfw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txgtvxnpgvrzmgxdpddacrvspanwuvnw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhgfbiydtlujprffexkcpwifkhdvnguh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fuaiyrtwdhurlrownzmwfcricwmhjvop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rniswhptsdvuhxrumkpnyhyoyqkoponr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtayttdipllmssbwdedueiqiahwydbzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sallsdpfvfiargpfhetobamjbxhgahqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ylusscobexbykxlrtfiurcvgattbuezr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"trkjtwigusczoiouutyvruhluknjpuli","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqdjyzfucheuvcnhmhuutmtkhtpzcvmb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nuwtgdwgrfzpajyxniisyaoabsejynnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ymsofunpmppnqbarjryywspcutnizhfw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxhgnkwxfmkuswmfloeivxmhrgcgqaby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofwnsobjneunifyjjsiuyboapaakccif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtxnbkipmgdumdxyqzekrjojhmtloufk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uyufaivtnjfijiqzauwzwgxqheposyxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijztayhpbzzzowngxhalhzckncofdfun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvbiueabteyfbbmvmvveaaqhodkvpnod","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqwbovapjkquonlwpsrykbumhbpzufhu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itturaydmvveuybihzfljovhfeondnoj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"muldbwunjsnscubqvmqqmdufcxmwrsun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smatspsvfewkkvmjyakrdiifdzamjzez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxyvfrxcjynccpkkyyulzzrbrqoexltm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"huwensejwstzsmshqalaqgnyiptziwwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tyxuxrauhdboipwsbbtslbqovdaueybe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcsncsukshqmogdpgidlsijnjlhpjlim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqdzspzffosldmdtwngjekxokluvqrjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phljbenmcpmqxgvvdodccwwpogynnnoh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oofcaxfjkpemrwipihylydafbdnfoyuo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwpqiklbgaezpdvyuceekhgllybjwmnn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvztrgqvbpfzqadzrpoaufqcxbmfsmtg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xllavcqoawtixlawnljojxjdnercxwma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqgkppzdpkgdkjtufzbnmhmbpixhtgin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlrrcswujcwnrfsetxfisnularseamab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skviegnvpshtmaukuvambbyfxvdjkqrt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkjxypvccaidforyztxiavgvjhopxolb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sqaxppfsipxysbjakwavhkudbcdyudat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtzttaitzdzlvjaudivnzprmnuyaqfnj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edhqdurdsuxndugibbynjfctsuxuutof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"laplivsarfmafiejemzliptxtjqxakcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpmdnakmgyresqpetyzplkbvffvbarxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqvwqmgiiqfjwkroxdyhrpzvsdtsmkvb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oxyseiqadnnegqwmeynilxfwturhzsno","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdzowqnxvosqzjpdpklbimkppfhunnbl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kflzxoporlcbonhqtjonfpqlrtoulmlq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmkccsndpmrvnqdmvjzwxkknfbvrlmic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekskyxhdrxyfipsxrmljcyhjxmhrcfjc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"btqlshbdjahcumwefactdgfilkoupowr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbpfivdnzlrhixvdvvkqlqdguralcist","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpfvweukzyjbgsrgfhbjlsoyzypacecb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jnutjuuatjlwbtgqgxybnbfuhwlggqly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zllomzeynmpihkpfaoclachanclekjnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tzzhkhqqjhhxccdcdhijqptjbjynphcz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utkdpmuxvtsoapgdbgaevwarbeosoaqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khqihxkoftqrklrhkawoicvedfylrqcd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"euphkmznfucbdvfjqmmmlwsiifxngvzh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lptxjzwcvvvxzrokbzdifstypvowqbyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"feahmltxcwxgkibdjicrkqknbszadcvm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knfltjykdbgqyiiatcwhihspukshamei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"anesgcgjggetdvnffacknwoniexoowxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"trdxmnytxaeikoprvxenizfwfvfzlaoe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhsdvccqtlywdchjtegdxqiedtedadtl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqyxnpeeysnrszrgiagwzabefmdibaid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfsaatbhjvvpbgycllwcixgeviqkzpai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omydzbhjzimzplbfzmcepzdtqxqsyqow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fixmxubeteqglcizzovgkxsyasrkdgns","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"diiesxdlvkvbhfuhkpcjjhhtxqyrspcs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qylztegkrsmsxohfyowcmioapqrsptot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehduzgotpyxqcutzkcerlozbgkuuzlgy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrtlqbkrpxopqjfczgfkegfxcjnfpcpi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcexwygaioddcejgguuobmyleoinwxgo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gvvwlwhrsdedfxezcalevxcbowgcnblv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzairmlkgpozhsueivjxrgbfctfjvkbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vcyvemrqnfvyiifonkaledstfldaghci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llsqrfreryfngockgyekhnzwetrmdume","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iedsrxgwmajeafczjftymvywcnzxqtal","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nyxucehtwahpshzpcjiojqpmleuahvfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xwblxqfmznfczympdtflpsusyterrmhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynivunbjqyeyfnojmyqnpezyhluuteeg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ooaxevucibnafhrovpvexckhevrdeaql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abdoqtpvvdqokikxxgkhnwscfppwmwdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okdmqfkclagkgluoqoyvyagzocecbsle","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"celxhansaudrpkmzjktzlupxxhxllfww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670184,"databaseName":"models_schema","ddl":"CREATE TABLE `pghwpbsubektqcppdifryknztxnmsvhf` (\n `fjhjqdkhjkdlfwvawbkzxsjbtzcmgyvc` int NOT NULL,\n `pbuyhacjbmahoemjhqffgdcmeqpnznos` int DEFAULT NULL,\n `hvetxolhftuzyqiujcjzfgwkxurktrza` int DEFAULT NULL,\n `pcxhfofjkfthzgscnkjnfpydgpnzvmfu` int DEFAULT NULL,\n `kjbmzdvwjemgbvmaggzafhfbxavykndp` int DEFAULT NULL,\n `mbjworkjyrndfgqqjccyjibfthoxbxwy` int DEFAULT NULL,\n `qxyenolayckbhionuckibhisymmbjfwk` int DEFAULT NULL,\n `gnnmlgboynpgijnbbymtdccwnuvsmplp` int DEFAULT NULL,\n `nkbgevvbmzbveydukdhyfddtrbvbjlft` int DEFAULT NULL,\n `bpcarrxflqzuwhimhewrszjrpsqqsggm` int DEFAULT NULL,\n `jiqmgazujpxriywcrciqwmdhtzakwkcc` int DEFAULT NULL,\n `pqwnwxiliuuziatlcznbgokffdzjpevv` int DEFAULT NULL,\n `xlyaizfwnlaluibeogwdphmlktgwnnop` int DEFAULT NULL,\n `eidnwzsxvfjhjfypcienlxsdfnhapmhm` int DEFAULT NULL,\n `jgbpdnwyvpjyhqrnckhvjoavabiyypri` int DEFAULT NULL,\n `lftpspqgnuqehyoobcddlkkynddmvtfk` int DEFAULT NULL,\n `mboyyxmrbvhdxshpsjyllttrmjugvimb` int DEFAULT NULL,\n `mqgoerhwjyzdmccvqkgukbszxabnqlfh` int DEFAULT NULL,\n `njlpnynqmifakcxumjufichekbqcjujg` int DEFAULT NULL,\n `jakuxsfcagkugcrnoirxrtpvjizmgiaf` int DEFAULT NULL,\n `rthleyhliramkvurxovwhrlrzarxsuoy` int DEFAULT NULL,\n `punrtvuexqkgraywfkfdvlhhafqgutcx` int DEFAULT NULL,\n `ybynfalogttclgmnhmtzhfsoqupxhcol` int DEFAULT NULL,\n `jwoysusnuvujajerzgancludkgtjsfho` int DEFAULT NULL,\n `mmykdjrbxjpsmfynoqdgqgmptjaqsyvz` int DEFAULT NULL,\n `vtamywbqdpdsewziodmxhjpqnqlaofgx` int DEFAULT NULL,\n `sauafvdzfsppeycosvberwglinhmdudn` int DEFAULT NULL,\n `whycxmbfhlaknuoyxknearojeykfhxvq` int DEFAULT NULL,\n `nmorwipxdawqhcppinitfngxflpyowyu` int DEFAULT NULL,\n `qsbmwcfsmtcpbplreeqbdodyzhdwurbv` int DEFAULT NULL,\n `dhimijxdqyshsfrjyqhkdkgexbfloamg` int DEFAULT NULL,\n `gucfvwjsjzhxhzmecdontmjupilzfshr` int DEFAULT NULL,\n `txaipuxpiuqjoumnanctgcjcradiqbgm` int DEFAULT NULL,\n `ptogkzevrkvpncpjvvzbzegkiporhlmp` int DEFAULT NULL,\n `wjrksgepsfboyxinesjsqohlqpvlpaji` int DEFAULT NULL,\n `vierjwfwpklbimvxfciksupkkwtlxstk` int DEFAULT NULL,\n `prwwjzhgaeedxgheswgnqzbjxnmilqsl` int DEFAULT NULL,\n `lcrcbfauvdbxenrgyjayxplffipycjsc` int DEFAULT NULL,\n `mftcpsdlppxrocbezydnjcgkudvmtsou` int DEFAULT NULL,\n `uffzlrrpcpvprppnxkoaiinxtghxgykj` int DEFAULT NULL,\n `pwbeafyzutxkjjvigthlmjfivbswrejj` int DEFAULT NULL,\n `gtayzmobewgbeybkztnznrkdrnxuwhoh` int DEFAULT NULL,\n `xnlwuqxhjrkabhrnxqbjrpvahezmhahx` int DEFAULT NULL,\n `kxtrpavkjxgplstksonxspjymqqxdxky` int DEFAULT NULL,\n `yowjuoyqzxjudfzuvmznzthcrbcbrjjd` int DEFAULT NULL,\n `jhaqqboxrojmwkuxqzzgqwbcqbukzupz` int DEFAULT NULL,\n `ljibwkdgaurdyjibralnlzeqfmldffff` int DEFAULT NULL,\n `xugetljnhvrkfthouzggmhklcsyqqgap` int DEFAULT NULL,\n `ynbsubptywyvlospwfgjlqibcebmxena` int DEFAULT NULL,\n `qlfszfcecqfblezflnelrytadlxdwkye` int DEFAULT NULL,\n `oqgcmmkjwblldumcilflskzmotmrwoxu` int DEFAULT NULL,\n `khivwetoxxdmaoktxrvvnaxczknyrpbc` int DEFAULT NULL,\n `xqpvvwmwgprwtqltuyjhpvklzorlblkg` int DEFAULT NULL,\n `uocpoovtubhjguphohtepzzyhlmiczbd` int DEFAULT NULL,\n `oeevlxbpnckwbnxptzsdumbacleuwuzg` int DEFAULT NULL,\n `darvqpelrpdpdqulidmtlxkqcovtsxkq` int DEFAULT NULL,\n `figlhuzqslrqccivrbufcfpixmlkabwu` int DEFAULT NULL,\n `qyrrmtwywxwugkvinccddklcjolpcuxi` int DEFAULT NULL,\n `kpgcrzshjvmqmdtjdiurlikpogtzroat` int DEFAULT NULL,\n `ktmbtpltnvqhmdvrficxfvalvuamdifo` int DEFAULT NULL,\n `lyrnxzounqqyqxajsdxafbvhdwywmqmp` int DEFAULT NULL,\n `uyvjszukvsomtsqkdoasoppanocefaxo` int DEFAULT NULL,\n `nhxxscmyaqjfmozxyoyapregsydmmazy` int DEFAULT NULL,\n `zzxlbmqiprjgrbutqxpzbxazixixtodr` int DEFAULT NULL,\n `jhapvgrllzdqdjxvrzakyarcugndjogr` int DEFAULT NULL,\n `cxddlkrxxteenawuuwoqhwqydcpiwwse` int DEFAULT NULL,\n `acmgrdsqvliexvsodkkgvsevcewpmqpw` int DEFAULT NULL,\n `jmiamzhodvxyfyoptzsduwrsffcgsthc` int DEFAULT NULL,\n `xvkfbdtpmhwbmnzmzoileartnzgplpyh` int DEFAULT NULL,\n `gglpwmrxrhfonpxgpaqzqbuffxibowlt` int DEFAULT NULL,\n `xysdhcvfvcbbeukdfbeizyahozbpobds` int DEFAULT NULL,\n `rftdfjussifynramgtwcrtrmamknurjs` int DEFAULT NULL,\n `shggbdlenhfzovojlzzfuniwndlygojf` int DEFAULT NULL,\n `nvoczigqbgacdbtvntnaqvblhseeeqdq` int DEFAULT NULL,\n `xgibjdhdnobyoessykhsnsmrtetambuq` int DEFAULT NULL,\n `svlywalpwdodswvilqjuhoyyfrbtcwzr` int DEFAULT NULL,\n `cqwcsqrnounlwgohxtxemqnjugvtxzot` int DEFAULT NULL,\n `wpjglhzqajebndksjnikstjoquikptub` int DEFAULT NULL,\n `statwzqsbpjumfngifxqcuwnwhromopo` int DEFAULT NULL,\n `ecfzhfjfjlyyaxaxljmqjzfkqzoclzmc` int DEFAULT NULL,\n `eqeuxbnvfcelwpfnnszzfsxmiimquerw` int DEFAULT NULL,\n `bamamiknqgbuhmzlwnqfysapaowehmno` int DEFAULT NULL,\n `osrjjfgqqxxkbxonmaevabndjhcbgqos` int DEFAULT NULL,\n `pnoetjkiaelsauotqicktvinqrpoqwkq` int DEFAULT NULL,\n `dyrxscqebcuzeacgxkvtrdmfyttcvfgw` int DEFAULT NULL,\n `miidwrqptylzjyowibpghocrmapxlhhw` int DEFAULT NULL,\n `betwounxwabinbczdedxubqmefgbjlgo` int DEFAULT NULL,\n `mlotzwfqukczhdrmitpyvowihhzbhlmj` int DEFAULT NULL,\n `gqmzxihgynmjlkttdsbdxgnecfphjslp` int DEFAULT NULL,\n `vcpxytcadzmbtvqlcihylwocpzhqimku` int DEFAULT NULL,\n `otxsmxobozxfgxxbbbejhuzookzrsjjo` int DEFAULT NULL,\n `lenwulpscxzehlkmfnyhcjixwddqcndb` int DEFAULT NULL,\n `wyronnhjjpntgobkopyeiyazwaspegnl` int DEFAULT NULL,\n `yxrkdalesjzytpaohabmsfarbbakrmou` int DEFAULT NULL,\n `wjllnudkjlednxqjmafyjnmnxtfkcgwl` int DEFAULT NULL,\n `yqvykcwbhdgzwmraqowtmfjtzbivpbsc` int DEFAULT NULL,\n `bhegdlprfruubjtrsngfemyaxujmlbti` int DEFAULT NULL,\n `hrfclimanioxzllxtykmvlwnateejoay` int DEFAULT NULL,\n `gbheiursizbmyrvzdvzfvsimfuxzsres` int DEFAULT NULL,\n `uuuxdcyjsmjcaxzuvsegtebamsykukyx` int DEFAULT NULL,\n PRIMARY KEY (`fjhjqdkhjkdlfwvawbkzxsjbtzcmgyvc`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"pghwpbsubektqcppdifryknztxnmsvhf\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["fjhjqdkhjkdlfwvawbkzxsjbtzcmgyvc"],"columns":[{"name":"fjhjqdkhjkdlfwvawbkzxsjbtzcmgyvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"pbuyhacjbmahoemjhqffgdcmeqpnznos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hvetxolhftuzyqiujcjzfgwkxurktrza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pcxhfofjkfthzgscnkjnfpydgpnzvmfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjbmzdvwjemgbvmaggzafhfbxavykndp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbjworkjyrndfgqqjccyjibfthoxbxwy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxyenolayckbhionuckibhisymmbjfwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnnmlgboynpgijnbbymtdccwnuvsmplp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkbgevvbmzbveydukdhyfddtrbvbjlft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bpcarrxflqzuwhimhewrszjrpsqqsggm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jiqmgazujpxriywcrciqwmdhtzakwkcc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqwnwxiliuuziatlcznbgokffdzjpevv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xlyaizfwnlaluibeogwdphmlktgwnnop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eidnwzsxvfjhjfypcienlxsdfnhapmhm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgbpdnwyvpjyhqrnckhvjoavabiyypri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lftpspqgnuqehyoobcddlkkynddmvtfk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mboyyxmrbvhdxshpsjyllttrmjugvimb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqgoerhwjyzdmccvqkgukbszxabnqlfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njlpnynqmifakcxumjufichekbqcjujg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jakuxsfcagkugcrnoirxrtpvjizmgiaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rthleyhliramkvurxovwhrlrzarxsuoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"punrtvuexqkgraywfkfdvlhhafqgutcx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybynfalogttclgmnhmtzhfsoqupxhcol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwoysusnuvujajerzgancludkgtjsfho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmykdjrbxjpsmfynoqdgqgmptjaqsyvz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtamywbqdpdsewziodmxhjpqnqlaofgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sauafvdzfsppeycosvberwglinhmdudn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"whycxmbfhlaknuoyxknearojeykfhxvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmorwipxdawqhcppinitfngxflpyowyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qsbmwcfsmtcpbplreeqbdodyzhdwurbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhimijxdqyshsfrjyqhkdkgexbfloamg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gucfvwjsjzhxhzmecdontmjupilzfshr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txaipuxpiuqjoumnanctgcjcradiqbgm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ptogkzevrkvpncpjvvzbzegkiporhlmp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjrksgepsfboyxinesjsqohlqpvlpaji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vierjwfwpklbimvxfciksupkkwtlxstk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prwwjzhgaeedxgheswgnqzbjxnmilqsl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lcrcbfauvdbxenrgyjayxplffipycjsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mftcpsdlppxrocbezydnjcgkudvmtsou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uffzlrrpcpvprppnxkoaiinxtghxgykj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwbeafyzutxkjjvigthlmjfivbswrejj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtayzmobewgbeybkztnznrkdrnxuwhoh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnlwuqxhjrkabhrnxqbjrpvahezmhahx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxtrpavkjxgplstksonxspjymqqxdxky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yowjuoyqzxjudfzuvmznzthcrbcbrjjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jhaqqboxrojmwkuxqzzgqwbcqbukzupz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljibwkdgaurdyjibralnlzeqfmldffff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xugetljnhvrkfthouzggmhklcsyqqgap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynbsubptywyvlospwfgjlqibcebmxena","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qlfszfcecqfblezflnelrytadlxdwkye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqgcmmkjwblldumcilflskzmotmrwoxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khivwetoxxdmaoktxrvvnaxczknyrpbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqpvvwmwgprwtqltuyjhpvklzorlblkg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uocpoovtubhjguphohtepzzyhlmiczbd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oeevlxbpnckwbnxptzsdumbacleuwuzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"darvqpelrpdpdqulidmtlxkqcovtsxkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"figlhuzqslrqccivrbufcfpixmlkabwu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qyrrmtwywxwugkvinccddklcjolpcuxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpgcrzshjvmqmdtjdiurlikpogtzroat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ktmbtpltnvqhmdvrficxfvalvuamdifo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lyrnxzounqqyqxajsdxafbvhdwywmqmp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uyvjszukvsomtsqkdoasoppanocefaxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhxxscmyaqjfmozxyoyapregsydmmazy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzxlbmqiprjgrbutqxpzbxazixixtodr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jhapvgrllzdqdjxvrzakyarcugndjogr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cxddlkrxxteenawuuwoqhwqydcpiwwse","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acmgrdsqvliexvsodkkgvsevcewpmqpw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmiamzhodvxyfyoptzsduwrsffcgsthc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvkfbdtpmhwbmnzmzoileartnzgplpyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gglpwmrxrhfonpxgpaqzqbuffxibowlt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xysdhcvfvcbbeukdfbeizyahozbpobds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rftdfjussifynramgtwcrtrmamknurjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shggbdlenhfzovojlzzfuniwndlygojf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvoczigqbgacdbtvntnaqvblhseeeqdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xgibjdhdnobyoessykhsnsmrtetambuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"svlywalpwdodswvilqjuhoyyfrbtcwzr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqwcsqrnounlwgohxtxemqnjugvtxzot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wpjglhzqajebndksjnikstjoquikptub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"statwzqsbpjumfngifxqcuwnwhromopo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecfzhfjfjlyyaxaxljmqjzfkqzoclzmc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqeuxbnvfcelwpfnnszzfsxmiimquerw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bamamiknqgbuhmzlwnqfysapaowehmno","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"osrjjfgqqxxkbxonmaevabndjhcbgqos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnoetjkiaelsauotqicktvinqrpoqwkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dyrxscqebcuzeacgxkvtrdmfyttcvfgw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"miidwrqptylzjyowibpghocrmapxlhhw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"betwounxwabinbczdedxubqmefgbjlgo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlotzwfqukczhdrmitpyvowihhzbhlmj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqmzxihgynmjlkttdsbdxgnecfphjslp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vcpxytcadzmbtvqlcihylwocpzhqimku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otxsmxobozxfgxxbbbejhuzookzrsjjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lenwulpscxzehlkmfnyhcjixwddqcndb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyronnhjjpntgobkopyeiyazwaspegnl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxrkdalesjzytpaohabmsfarbbakrmou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjllnudkjlednxqjmafyjnmnxtfkcgwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqvykcwbhdgzwmraqowtmfjtzbivpbsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bhegdlprfruubjtrsngfemyaxujmlbti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hrfclimanioxzllxtykmvlwnateejoay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gbheiursizbmyrvzdvzfvsimfuxzsres","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uuuxdcyjsmjcaxzuvsegtebamsykukyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670215,"databaseName":"models_schema","ddl":"CREATE TABLE `phfoifsjghtcrrrmlirrscflzgkfcilb` (\n `wamjleiidbltggzivpebuwdltwjyosbh` int NOT NULL,\n `ascypoddefikhbynipytigerugovkzhx` int DEFAULT NULL,\n `sxhbkotfmlmjedpdyxfycsziklnxdjqw` int DEFAULT NULL,\n `norezjxmngalzczpryzvwqauaieehhyc` int DEFAULT NULL,\n `llchxogddmnoprzjcdtgugqsxryerquw` int DEFAULT NULL,\n `gkyrpbzgkghfixqvighutfyjypbpgcpa` int DEFAULT NULL,\n `crvliwfrkbakhniopcrrikqurqxfakeu` int DEFAULT NULL,\n `hnyzegacyrscplcetgsjlfevnlokrsnu` int DEFAULT NULL,\n `xntaabuflozghtzizbphrqepxhwxixhq` int DEFAULT NULL,\n `mcqhuxhlbczrchoacwilejqwzgueqvmk` int DEFAULT NULL,\n `lfszupqeeflfwmlhjikwdyikywrfflkw` int DEFAULT NULL,\n `yndipojbdhhiapoosjwfzxtmskcfnzly` int DEFAULT NULL,\n `qtdrolcfbbveylzodjtpuwfrvimbuuhy` int DEFAULT NULL,\n `regnbizcurhsiaytkqsrfzvappzhkyck` int DEFAULT NULL,\n `vanagyoqbkxlnjvnvdqtclsasilrmsdh` int DEFAULT NULL,\n `rlwverxhndydmosnetqzrbykmzdibqpk` int DEFAULT NULL,\n `lveviwzrdlyymwtwebhhinewvtsbgwyw` int DEFAULT NULL,\n `zfuwjmkhviyiygjgxcdasgnoexvqqgcz` int DEFAULT NULL,\n `veorouldvdhrhvqjmlrkutbogxfdcpuc` int DEFAULT NULL,\n `ugolygcfsfthjhaqodehupxddpxkieof` int DEFAULT NULL,\n `dddzqbtwzxibvsitooyctntqeromfmvy` int DEFAULT NULL,\n `upuwxyozyjegoowgzwhktnumwpzznuqv` int DEFAULT NULL,\n `fhdmhrezutrpffofcbwfnsnjfgwzttvd` int DEFAULT NULL,\n `kibiqnjrgkqgpncmtshojrcubbdbwgrr` int DEFAULT NULL,\n `molpsyphluwfjmvymtxnfrmbqiifykls` int DEFAULT NULL,\n `myyighemhzqqlavnwgzxlrvzityqseun` int DEFAULT NULL,\n `donprqgeppmprurukohybmgbkzrzvizv` int DEFAULT NULL,\n `ipcbqkbbdwxmwzfceusfpvquffocncxe` int DEFAULT NULL,\n `pprdsvnliaoxktgbqkttrfflndcukluu` int DEFAULT NULL,\n `zlblqevgoahuwuphykexevjuboylceyd` int DEFAULT NULL,\n `zqjocjmokusyogqgvkkfezludxuoarfx` int DEFAULT NULL,\n `irtcmxbsusuxxbtqtfiimsvyrgyjkaca` int DEFAULT NULL,\n `mogllhsuzctyulyayjxarvakctnfxqzs` int DEFAULT NULL,\n `djboydzqwiuhzwyzuuzwkwogbumxayhr` int DEFAULT NULL,\n `zgjywmabqnzkrrhumluttdtrtcadutwj` int DEFAULT NULL,\n `slwdplnjppfsxmuaofblbibaygtotnro` int DEFAULT NULL,\n `humcffwsgbzzamlnlbzvpldkpfisjvgh` int DEFAULT NULL,\n `ejwurgqtqczbqixxzjayzxenxuiiegdh` int DEFAULT NULL,\n `usyapabjcftobyasfyrfhxqzdrkmklot` int DEFAULT NULL,\n `shtrwfjufixdmupwyqzhieykygyiijug` int DEFAULT NULL,\n `mjssykrsuvkkwjaxymqoercdpvjjecex` int DEFAULT NULL,\n `kcsufeyhduesopxcbjujelxahonduapx` int DEFAULT NULL,\n `hlhsmbifnrdzcwoakvjubdtkcitjtxad` int DEFAULT NULL,\n `eeyyfkehfnxwhctvgguykrrfweibhzpe` int DEFAULT NULL,\n `mhgzqxyekofnwcvdaxfdraynlweotfbv` int DEFAULT NULL,\n `blsnwwsjbalaockgpzhwypjuiirxceou` int DEFAULT NULL,\n `gkqaftsdfxvvdxprldaabheihzcznguj` int DEFAULT NULL,\n `qmdjzgfzdgaykhrfmzvoumrueigmyddu` int DEFAULT NULL,\n `lalqrccyluhztdvvzhanabxjrpvdzbjw` int DEFAULT NULL,\n `fnwidakzaxahghyuyyihqhwwzknxfkwm` int DEFAULT NULL,\n `lerniecytztdidtjctedyaekmoblfhai` int DEFAULT NULL,\n `ocrzthaeitaguijwzptzzcpyxxjvzdys` int DEFAULT NULL,\n `romwvaydjtoodpxkozrtkwxehgymifxl` int DEFAULT NULL,\n `vjjqgkhrnjbkyeupabhnqzenputsrqow` int DEFAULT NULL,\n `bqdpdzovqvplovffotgyaukqtvmrfvuf` int DEFAULT NULL,\n `lqabvptkfdybkbavqdwelawqzxawuwwz` int DEFAULT NULL,\n `affcbgzsgcxwqcqxlasxudffijtzzekn` int DEFAULT NULL,\n `ouefzepojgbygjwqpwtdapolpopaxbdu` int DEFAULT NULL,\n `lwbryqjdbtaaxdyiuxlbkigcnlwtsnpa` int DEFAULT NULL,\n `pmrfphjtcfrpaopqwyguoyssscbpemdn` int DEFAULT NULL,\n `mwwpxvdyzeeqyaxjjmavvefzycsqmeoq` int DEFAULT NULL,\n `uegkbvdlaliqwqzgmnevgviytpfbznsp` int DEFAULT NULL,\n `jnykobflexzcyazyrbnzstwlmautqviv` int DEFAULT NULL,\n `mtaolgcjsdkxsvynwrkmpeedawzoqgfn` int DEFAULT NULL,\n `jpzkeusrrdbkzdgxyjvdpgxyjhqmsyil` int DEFAULT NULL,\n `cgadgvabztidmavqkxbffftawhbwnxcv` int DEFAULT NULL,\n `qodscwwjigyxdghorcsapfkedryattea` int DEFAULT NULL,\n `louxsddyievmpysjqgucxsnwkbsmquvz` int DEFAULT NULL,\n `onkmlhbgnvaqebabumuodevrotywacdl` int DEFAULT NULL,\n `emphauidatdghovpscnbkjyiydlgobae` int DEFAULT NULL,\n `yvwyiiezpcvttlqtiyompqkiciireasr` int DEFAULT NULL,\n `opehjyrdeexmpvxzqgapbtluzpdlejlr` int DEFAULT NULL,\n `lrspsbcqphrcngiynswtzpiytijoldrg` int DEFAULT NULL,\n `ftieafnyjmqanjgmuzeqeartddbpypfg` int DEFAULT NULL,\n `husjnvskafnjqhnsbpxrgxbqmvksmagp` int DEFAULT NULL,\n `stnitgggbhwshlfjqqqvfmqkulexlwic` int DEFAULT NULL,\n `janovkygfrdnqtrbjhdxozvlojqlqrzm` int DEFAULT NULL,\n `yltlferhheedueiuaplzeztqiumqllvh` int DEFAULT NULL,\n `yzvxftjucbzmfwjkscffnxfyblnalnnd` int DEFAULT NULL,\n `urtwynpecbitijvenbroqsojnzmhgpqv` int DEFAULT NULL,\n `gdttknzokcecoeawctneuuyafdoqikyg` int DEFAULT NULL,\n `bbmnsqcbkotqhsampfythuhyvlzcfjwl` int DEFAULT NULL,\n `fxqibbxqvdsyysptqqdswvbxasfbwpwu` int DEFAULT NULL,\n `muzxeidzqrsoxtvfsjzpzljunfnktqwj` int DEFAULT NULL,\n `ntjrroolzdarehtazubqnkyxjyrokbau` int DEFAULT NULL,\n `xnmqqkyxrwmgiyfqsuiyxujrxmxvddwb` int DEFAULT NULL,\n `mfixfzpifpdjyqzdixtqdkcazoodbrhz` int DEFAULT NULL,\n `fykxayhnzasfupmdyjkfwldtwqapraxv` int DEFAULT NULL,\n `wywlziogkdthqvopxjydwjcdcejkaryp` int DEFAULT NULL,\n `gdtobvxcstblsvdsdgsgzecmzbyxxnxe` int DEFAULT NULL,\n `kziqmvdcxxkxyxofjucvsuysgbvonmeg` int DEFAULT NULL,\n `rdzvrziynojerkcfhcpnfolumdgtodzf` int DEFAULT NULL,\n `pojzvhoixqiilmsrfpnhxalvgqaiagaq` int DEFAULT NULL,\n `iwpwhixwtxqroxtcsoraolcyjmnapowq` int DEFAULT NULL,\n `hqlzvcyjxroptzzqvjdijsdsgwvpnkin` int DEFAULT NULL,\n `zktnnnqhzclcprphldrxuvlvsqtllwmt` int DEFAULT NULL,\n `ummsgpiieevqygfrkhspjobtmukpkkxr` int DEFAULT NULL,\n `jolbguyugxkitsjhaqgqlsjuwtvuzvda` int DEFAULT NULL,\n `dpymwbdokmcpghhdbphwasuntxleulxz` int DEFAULT NULL,\n `gvtsxyhqygyupwyyvzsatdakohxzzwdw` int DEFAULT NULL,\n PRIMARY KEY (`wamjleiidbltggzivpebuwdltwjyosbh`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"phfoifsjghtcrrrmlirrscflzgkfcilb\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["wamjleiidbltggzivpebuwdltwjyosbh"],"columns":[{"name":"wamjleiidbltggzivpebuwdltwjyosbh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ascypoddefikhbynipytigerugovkzhx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sxhbkotfmlmjedpdyxfycsziklnxdjqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"norezjxmngalzczpryzvwqauaieehhyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llchxogddmnoprzjcdtgugqsxryerquw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkyrpbzgkghfixqvighutfyjypbpgcpa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crvliwfrkbakhniopcrrikqurqxfakeu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hnyzegacyrscplcetgsjlfevnlokrsnu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xntaabuflozghtzizbphrqepxhwxixhq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcqhuxhlbczrchoacwilejqwzgueqvmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfszupqeeflfwmlhjikwdyikywrfflkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yndipojbdhhiapoosjwfzxtmskcfnzly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtdrolcfbbveylzodjtpuwfrvimbuuhy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"regnbizcurhsiaytkqsrfzvappzhkyck","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vanagyoqbkxlnjvnvdqtclsasilrmsdh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlwverxhndydmosnetqzrbykmzdibqpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lveviwzrdlyymwtwebhhinewvtsbgwyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zfuwjmkhviyiygjgxcdasgnoexvqqgcz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"veorouldvdhrhvqjmlrkutbogxfdcpuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugolygcfsfthjhaqodehupxddpxkieof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dddzqbtwzxibvsitooyctntqeromfmvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upuwxyozyjegoowgzwhktnumwpzznuqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhdmhrezutrpffofcbwfnsnjfgwzttvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kibiqnjrgkqgpncmtshojrcubbdbwgrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"molpsyphluwfjmvymtxnfrmbqiifykls","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"myyighemhzqqlavnwgzxlrvzityqseun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"donprqgeppmprurukohybmgbkzrzvizv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ipcbqkbbdwxmwzfceusfpvquffocncxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pprdsvnliaoxktgbqkttrfflndcukluu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zlblqevgoahuwuphykexevjuboylceyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqjocjmokusyogqgvkkfezludxuoarfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"irtcmxbsusuxxbtqtfiimsvyrgyjkaca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mogllhsuzctyulyayjxarvakctnfxqzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"djboydzqwiuhzwyzuuzwkwogbumxayhr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zgjywmabqnzkrrhumluttdtrtcadutwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"slwdplnjppfsxmuaofblbibaygtotnro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"humcffwsgbzzamlnlbzvpldkpfisjvgh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejwurgqtqczbqixxzjayzxenxuiiegdh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"usyapabjcftobyasfyrfhxqzdrkmklot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shtrwfjufixdmupwyqzhieykygyiijug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjssykrsuvkkwjaxymqoercdpvjjecex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcsufeyhduesopxcbjujelxahonduapx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlhsmbifnrdzcwoakvjubdtkcitjtxad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eeyyfkehfnxwhctvgguykrrfweibhzpe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhgzqxyekofnwcvdaxfdraynlweotfbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blsnwwsjbalaockgpzhwypjuiirxceou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkqaftsdfxvvdxprldaabheihzcznguj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmdjzgfzdgaykhrfmzvoumrueigmyddu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lalqrccyluhztdvvzhanabxjrpvdzbjw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnwidakzaxahghyuyyihqhwwzknxfkwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lerniecytztdidtjctedyaekmoblfhai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocrzthaeitaguijwzptzzcpyxxjvzdys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"romwvaydjtoodpxkozrtkwxehgymifxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjjqgkhrnjbkyeupabhnqzenputsrqow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqdpdzovqvplovffotgyaukqtvmrfvuf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqabvptkfdybkbavqdwelawqzxawuwwz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"affcbgzsgcxwqcqxlasxudffijtzzekn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ouefzepojgbygjwqpwtdapolpopaxbdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwbryqjdbtaaxdyiuxlbkigcnlwtsnpa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmrfphjtcfrpaopqwyguoyssscbpemdn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwwpxvdyzeeqyaxjjmavvefzycsqmeoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uegkbvdlaliqwqzgmnevgviytpfbznsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jnykobflexzcyazyrbnzstwlmautqviv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtaolgcjsdkxsvynwrkmpeedawzoqgfn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpzkeusrrdbkzdgxyjvdpgxyjhqmsyil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgadgvabztidmavqkxbffftawhbwnxcv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qodscwwjigyxdghorcsapfkedryattea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"louxsddyievmpysjqgucxsnwkbsmquvz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"onkmlhbgnvaqebabumuodevrotywacdl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"emphauidatdghovpscnbkjyiydlgobae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yvwyiiezpcvttlqtiyompqkiciireasr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opehjyrdeexmpvxzqgapbtluzpdlejlr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lrspsbcqphrcngiynswtzpiytijoldrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftieafnyjmqanjgmuzeqeartddbpypfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"husjnvskafnjqhnsbpxrgxbqmvksmagp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"stnitgggbhwshlfjqqqvfmqkulexlwic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"janovkygfrdnqtrbjhdxozvlojqlqrzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yltlferhheedueiuaplzeztqiumqllvh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzvxftjucbzmfwjkscffnxfyblnalnnd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urtwynpecbitijvenbroqsojnzmhgpqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdttknzokcecoeawctneuuyafdoqikyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbmnsqcbkotqhsampfythuhyvlzcfjwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxqibbxqvdsyysptqqdswvbxasfbwpwu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"muzxeidzqrsoxtvfsjzpzljunfnktqwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntjrroolzdarehtazubqnkyxjyrokbau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnmqqkyxrwmgiyfqsuiyxujrxmxvddwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfixfzpifpdjyqzdixtqdkcazoodbrhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fykxayhnzasfupmdyjkfwldtwqapraxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wywlziogkdthqvopxjydwjcdcejkaryp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdtobvxcstblsvdsdgsgzecmzbyxxnxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kziqmvdcxxkxyxofjucvsuysgbvonmeg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdzvrziynojerkcfhcpnfolumdgtodzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pojzvhoixqiilmsrfpnhxalvgqaiagaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwpwhixwtxqroxtcsoraolcyjmnapowq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqlzvcyjxroptzzqvjdijsdsgwvpnkin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zktnnnqhzclcprphldrxuvlvsqtllwmt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ummsgpiieevqygfrkhspjobtmukpkkxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jolbguyugxkitsjhaqgqlsjuwtvuzvda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpymwbdokmcpghhdbphwasuntxleulxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gvtsxyhqygyupwyyvzsatdakohxzzwdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670245,"databaseName":"models_schema","ddl":"CREATE TABLE `pmufyqauiycsugklnqmjlwtntfykevoj` (\n `ftwyponodkggfdecemiwsujjrwyqadcx` int NOT NULL,\n `pheqausdsdvlsnnapxhkzulyodmkaeda` int DEFAULT NULL,\n `ujqcwgapnnldeewojaxedqhiuixnhoah` int DEFAULT NULL,\n `rhshyagqmmfwpsyljcwnbsflgwpxdczt` int DEFAULT NULL,\n `whsanbdgwtnonmcvgluaembxjpyocrcc` int DEFAULT NULL,\n `zgwawzgssjvuohdocrubuyjbmjmympym` int DEFAULT NULL,\n `yfhmwpnronzqztxybofxtscsklelgajo` int DEFAULT NULL,\n `yeequcsdagnbwlrnapqxfpfbttfjdtyv` int DEFAULT NULL,\n `xyzvodhpwebcrsgzrpbziobtonfkyakh` int DEFAULT NULL,\n `bdjsjgjuwgvfoaeofbyhwhlshjckloft` int DEFAULT NULL,\n `ajcmeiellsikjarxcotjgcoaigbkgvid` int DEFAULT NULL,\n `lqbzjwiymtscgzqyxuasabxzjpfmuicx` int DEFAULT NULL,\n `ypcrikflgeclhzcsklczdcnrhuauwvec` int DEFAULT NULL,\n `muvulkcglqvsipoejcnodngyxkbmjlce` int DEFAULT NULL,\n `qyjxdxniesrwzrmhsuqzkzrzomnbycyb` int DEFAULT NULL,\n `ojmygfitgzkgtjipckvccqtsloaugqgk` int DEFAULT NULL,\n `gjhsejpanwigdfxdfgbigybaajidwbif` int DEFAULT NULL,\n `qczvljmljpzcgzbruyhuozlbmnnlixnz` int DEFAULT NULL,\n `anffidjrohwdfcrnajtyjxzhvuqyeuho` int DEFAULT NULL,\n `wbfeqzblewzwzaisezxkkmmiftgppspt` int DEFAULT NULL,\n `wdzrbthdpffwoncidfmlhywulmtyzgdw` int DEFAULT NULL,\n `qqhgcnobpazhzmqlgtpzaduurowovgoy` int DEFAULT NULL,\n `ahlcnmdnbjnopaqplaoonrgfbutepdnt` int DEFAULT NULL,\n `pphbjwzniogzsixueydjkeolzqqlmpky` int DEFAULT NULL,\n `ugwghdnpwrcewmtchjkmptfxqhfcgjgz` int DEFAULT NULL,\n `ibddldkkuznnutrgugpuokismeecggbi` int DEFAULT NULL,\n `qjpwrkdvsksvxvytwvwivkxxreprqxzh` int DEFAULT NULL,\n `jxphoftzpbqlmjdwnvaynzhwhpxwcpxf` int DEFAULT NULL,\n `gpycycrlarlqclmhgrkeouiygvuvdtfa` int DEFAULT NULL,\n `ndkvajziftqnhgdhqkkrjnvnohcatary` int DEFAULT NULL,\n `pdipimebgmfeoiusqojflwurhenltpys` int DEFAULT NULL,\n `alchottmczhiffombzgrjucwnzjbvrtk` int DEFAULT NULL,\n `wgqdwmxydgdvjbwenlylvmbkdkctoybk` int DEFAULT NULL,\n `qyqttmrxuxwondnmivzpjuwqtxutdpqx` int DEFAULT NULL,\n `xiqdubfyegqrnhxayohsnnohnoukdcnp` int DEFAULT NULL,\n `encxwmppkzgfecgzclnfksixfawhjsmf` int DEFAULT NULL,\n `fmkzphxxykoqoomhrbakndgtprlbetrr` int DEFAULT NULL,\n `wjyvyxrvxzbqmgzzpadsqfdqcjgvbugt` int DEFAULT NULL,\n `oaplkzpuzbiovseecqrkrhllhuyzxlhv` int DEFAULT NULL,\n `zdeahizpwcjutoeyjdqrrrmjbxjglwda` int DEFAULT NULL,\n `gkmmcszbawmxdiwcogbqopqrpcvepszt` int DEFAULT NULL,\n `hlobqujftnmtjphstsxigjjoqipjhouo` int DEFAULT NULL,\n `ilxsegsujphfzwietpdydqtuphwcmfbf` int DEFAULT NULL,\n `feojhhhitdjmmwxvlsvibjedkbgugfoy` int DEFAULT NULL,\n `rlzijxilggyaoywevebnhndbhljmtikn` int DEFAULT NULL,\n `evlzlhxncqwaaxftjigynrhdwhosxsvp` int DEFAULT NULL,\n `zbvkhxiavxptjoxzwmzjlwqcfnausqzg` int DEFAULT NULL,\n `axjlpwlzqxuvwtygflnneclvycyktyez` int DEFAULT NULL,\n `kipgokykungvjjoezshhlosekwmwzcew` int DEFAULT NULL,\n `kajrjhhqvgwamvpviidnepnzpkiriacq` int DEFAULT NULL,\n `xiszqhtvmaibcgiiijjuojeofkodtgvi` int DEFAULT NULL,\n `axacbscoalvtudijoyvuqbzemqxhrjwx` int DEFAULT NULL,\n `tdxltpnnzfubtsomjmmjicmnpkcihdky` int DEFAULT NULL,\n `ssdxyifqwqnzsedustkgphylaotqdvzs` int DEFAULT NULL,\n `jufccssffgzrmpuafoljqcktuktwzvbx` int DEFAULT NULL,\n `tvqaefeczhqfjlepicpuvytmoauttesh` int DEFAULT NULL,\n `extmdmeaoduwpqvoiecystozqtdomsma` int DEFAULT NULL,\n `eiemgegtdjiidypkpvdbpubbinapqxoi` int DEFAULT NULL,\n `yelvpcdksztczgkaanomncmkydwlbsqa` int DEFAULT NULL,\n `bghqilbxucyjtkcbnbapqwvwuugfshki` int DEFAULT NULL,\n `wzybdkrgnimsqesivvhmrlifnmtcohrd` int DEFAULT NULL,\n `msrvkyihezyolbtrhkcibwopoikirqpm` int DEFAULT NULL,\n `nbgysndqubssajndohbfiqagoiaevonw` int DEFAULT NULL,\n `azaquxnpqwugryzfvpyudntbduzjipib` int DEFAULT NULL,\n `orchaepefaajvqmagbvkmtukvzockbga` int DEFAULT NULL,\n `eldwxgfnhvbvxvkhcqegzviinwzjdyfg` int DEFAULT NULL,\n `mjmjwrguncvxoyirvplfwzidgkbwftxt` int DEFAULT NULL,\n `zbyfvqdeeankfsbgptuflggtauypydkg` int DEFAULT NULL,\n `wcwensthwgneccawzrcxasaezxbmccto` int DEFAULT NULL,\n `pensjftvwwqvrvxrcysputdcfwvjdfww` int DEFAULT NULL,\n `cbwtbmkxxooztinhqbqslvjxspzdyepm` int DEFAULT NULL,\n `eootcovvmuhzdjcteynzijifcqaearic` int DEFAULT NULL,\n `nkoxpefcsbgwhpfzvhbtrrxqstlxbxbb` int DEFAULT NULL,\n `fxuucqaxrcqfakpgsdrpnkftmdlvxljc` int DEFAULT NULL,\n `kvwtapmuxveuxtjeudsuajkfyjwajdil` int DEFAULT NULL,\n `llclcyhhcesxzpzaqxzbeqhrbtbyavkf` int DEFAULT NULL,\n `tcjwmdpojqzlwrhefsxmettfautdxwuc` int DEFAULT NULL,\n `mcaxdandqztzdamjrkrcjsuzijacbtwx` int DEFAULT NULL,\n `cczokabyjsnkfvrhcutaksbpssslnzvi` int DEFAULT NULL,\n `izpbmmmhrqcszypyyrtgemwcdjxkcihq` int DEFAULT NULL,\n `rayekxmnhsoblfrknxewknndqbhygejg` int DEFAULT NULL,\n `czgrehnbivmwkzamiilbklptevkwhihy` int DEFAULT NULL,\n `kwhadiahumacyssxcxyrmmhmhxrhsvee` int DEFAULT NULL,\n `suyzowrvwwioszzdripkvkpxpkbdyxae` int DEFAULT NULL,\n `jwvastosbdreruwkldcscwxzcihvedab` int DEFAULT NULL,\n `bwolgeljgbqcroocenkbmeeenvmhgxfr` int DEFAULT NULL,\n `nkixkzqhoxaaqkogislzqfdswemcldio` int DEFAULT NULL,\n `fhmivndtwpyfvfumjhujdngqkjzotich` int DEFAULT NULL,\n `mhkfajymukazbysihgtucnibczyvfckb` int DEFAULT NULL,\n `jvlqirpvjddakkqxezxffublesjguvgq` int DEFAULT NULL,\n `jlqndsfneenljlwsytcdkhxqtchvlnxh` int DEFAULT NULL,\n `crrntawjzgafdhjoinzwwoqvoabmxeny` int DEFAULT NULL,\n `aeawhkbcjsojgdxosdhpxbyckyjwzcbl` int DEFAULT NULL,\n `rezxzmlycpbfepeqkzbjfpgzimqsfoii` int DEFAULT NULL,\n `zrgnnqnldskwrfzywkotbhupqpxexgrz` int DEFAULT NULL,\n `tcaagudhjrbkznffmybvceawqljzpima` int DEFAULT NULL,\n `hzbouynoaunlymxlmjocbdxpgtguaiol` int DEFAULT NULL,\n `mgracabbaonxmaploaaajodjjmvxqkfx` int DEFAULT NULL,\n `xfdaatbvaytnrqapgsbdanltogrrbktd` int DEFAULT NULL,\n `gyztacuurxmtknqcstjhqyqiypbsdopt` int DEFAULT NULL,\n PRIMARY KEY (`ftwyponodkggfdecemiwsujjrwyqadcx`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"pmufyqauiycsugklnqmjlwtntfykevoj\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["ftwyponodkggfdecemiwsujjrwyqadcx"],"columns":[{"name":"ftwyponodkggfdecemiwsujjrwyqadcx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"pheqausdsdvlsnnapxhkzulyodmkaeda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujqcwgapnnldeewojaxedqhiuixnhoah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhshyagqmmfwpsyljcwnbsflgwpxdczt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"whsanbdgwtnonmcvgluaembxjpyocrcc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zgwawzgssjvuohdocrubuyjbmjmympym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfhmwpnronzqztxybofxtscsklelgajo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yeequcsdagnbwlrnapqxfpfbttfjdtyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyzvodhpwebcrsgzrpbziobtonfkyakh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdjsjgjuwgvfoaeofbyhwhlshjckloft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajcmeiellsikjarxcotjgcoaigbkgvid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqbzjwiymtscgzqyxuasabxzjpfmuicx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypcrikflgeclhzcsklczdcnrhuauwvec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"muvulkcglqvsipoejcnodngyxkbmjlce","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qyjxdxniesrwzrmhsuqzkzrzomnbycyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojmygfitgzkgtjipckvccqtsloaugqgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjhsejpanwigdfxdfgbigybaajidwbif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qczvljmljpzcgzbruyhuozlbmnnlixnz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"anffidjrohwdfcrnajtyjxzhvuqyeuho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbfeqzblewzwzaisezxkkmmiftgppspt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wdzrbthdpffwoncidfmlhywulmtyzgdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqhgcnobpazhzmqlgtpzaduurowovgoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahlcnmdnbjnopaqplaoonrgfbutepdnt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pphbjwzniogzsixueydjkeolzqqlmpky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugwghdnpwrcewmtchjkmptfxqhfcgjgz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibddldkkuznnutrgugpuokismeecggbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjpwrkdvsksvxvytwvwivkxxreprqxzh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jxphoftzpbqlmjdwnvaynzhwhpxwcpxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gpycycrlarlqclmhgrkeouiygvuvdtfa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ndkvajziftqnhgdhqkkrjnvnohcatary","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdipimebgmfeoiusqojflwurhenltpys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"alchottmczhiffombzgrjucwnzjbvrtk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgqdwmxydgdvjbwenlylvmbkdkctoybk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qyqttmrxuxwondnmivzpjuwqtxutdpqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xiqdubfyegqrnhxayohsnnohnoukdcnp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"encxwmppkzgfecgzclnfksixfawhjsmf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmkzphxxykoqoomhrbakndgtprlbetrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjyvyxrvxzbqmgzzpadsqfdqcjgvbugt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oaplkzpuzbiovseecqrkrhllhuyzxlhv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdeahizpwcjutoeyjdqrrrmjbxjglwda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkmmcszbawmxdiwcogbqopqrpcvepszt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlobqujftnmtjphstsxigjjoqipjhouo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ilxsegsujphfzwietpdydqtuphwcmfbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"feojhhhitdjmmwxvlsvibjedkbgugfoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlzijxilggyaoywevebnhndbhljmtikn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evlzlhxncqwaaxftjigynrhdwhosxsvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbvkhxiavxptjoxzwmzjlwqcfnausqzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axjlpwlzqxuvwtygflnneclvycyktyez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kipgokykungvjjoezshhlosekwmwzcew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kajrjhhqvgwamvpviidnepnzpkiriacq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xiszqhtvmaibcgiiijjuojeofkodtgvi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axacbscoalvtudijoyvuqbzemqxhrjwx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdxltpnnzfubtsomjmmjicmnpkcihdky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ssdxyifqwqnzsedustkgphylaotqdvzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jufccssffgzrmpuafoljqcktuktwzvbx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvqaefeczhqfjlepicpuvytmoauttesh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"extmdmeaoduwpqvoiecystozqtdomsma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eiemgegtdjiidypkpvdbpubbinapqxoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yelvpcdksztczgkaanomncmkydwlbsqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bghqilbxucyjtkcbnbapqwvwuugfshki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzybdkrgnimsqesivvhmrlifnmtcohrd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"msrvkyihezyolbtrhkcibwopoikirqpm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbgysndqubssajndohbfiqagoiaevonw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azaquxnpqwugryzfvpyudntbduzjipib","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orchaepefaajvqmagbvkmtukvzockbga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eldwxgfnhvbvxvkhcqegzviinwzjdyfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjmjwrguncvxoyirvplfwzidgkbwftxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbyfvqdeeankfsbgptuflggtauypydkg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcwensthwgneccawzrcxasaezxbmccto","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pensjftvwwqvrvxrcysputdcfwvjdfww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbwtbmkxxooztinhqbqslvjxspzdyepm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eootcovvmuhzdjcteynzijifcqaearic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkoxpefcsbgwhpfzvhbtrrxqstlxbxbb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxuucqaxrcqfakpgsdrpnkftmdlvxljc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvwtapmuxveuxtjeudsuajkfyjwajdil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llclcyhhcesxzpzaqxzbeqhrbtbyavkf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tcjwmdpojqzlwrhefsxmettfautdxwuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcaxdandqztzdamjrkrcjsuzijacbtwx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cczokabyjsnkfvrhcutaksbpssslnzvi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"izpbmmmhrqcszypyyrtgemwcdjxkcihq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rayekxmnhsoblfrknxewknndqbhygejg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"czgrehnbivmwkzamiilbklptevkwhihy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwhadiahumacyssxcxyrmmhmhxrhsvee","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suyzowrvwwioszzdripkvkpxpkbdyxae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwvastosbdreruwkldcscwxzcihvedab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bwolgeljgbqcroocenkbmeeenvmhgxfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkixkzqhoxaaqkogislzqfdswemcldio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhmivndtwpyfvfumjhujdngqkjzotich","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhkfajymukazbysihgtucnibczyvfckb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvlqirpvjddakkqxezxffublesjguvgq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlqndsfneenljlwsytcdkhxqtchvlnxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crrntawjzgafdhjoinzwwoqvoabmxeny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aeawhkbcjsojgdxosdhpxbyckyjwzcbl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rezxzmlycpbfepeqkzbjfpgzimqsfoii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zrgnnqnldskwrfzywkotbhupqpxexgrz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tcaagudhjrbkznffmybvceawqljzpima","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hzbouynoaunlymxlmjocbdxpgtguaiol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mgracabbaonxmaploaaajodjjmvxqkfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xfdaatbvaytnrqapgsbdanltogrrbktd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gyztacuurxmtknqcstjhqyqiypbsdopt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670277,"databaseName":"models_schema","ddl":"CREATE TABLE `ptxoeakypxcinxgiwxnegyucbfpderrz` (\n `gqpeikpphltvtgvvxoqeqkaofjfalwoq` int NOT NULL,\n `mabtbcdnbafwxoexkquoppknrvqipgqb` int DEFAULT NULL,\n `dsgeazdmkcvmykhlkxsfjzgeyplbyzpz` int DEFAULT NULL,\n `seswbqhxofjenovwosokwhzdttwfdacq` int DEFAULT NULL,\n `lshrotucstdedpzwpksazqvdkxzlclea` int DEFAULT NULL,\n `kyqbzbcrwbllvsbrjudseueuqrlloaaw` int DEFAULT NULL,\n `uznztszsqnhrpvzksifquwrkzojklgfx` int DEFAULT NULL,\n `dcaocnqsplyxzwckcalkdbfiuzyzkikb` int DEFAULT NULL,\n `qhyazxhifjfvuwubqumrmpjptxshkesl` int DEFAULT NULL,\n `qsmquirewjodkcmzqywfnhavxnjvbayh` int DEFAULT NULL,\n `vyrjikuaynacieswsmpwpburaxierewo` int DEFAULT NULL,\n `fsuvhrdqxwwzmiwappvxsrhkegjchajd` int DEFAULT NULL,\n `ojcsddldbtnfalgtjsadxkqftpanumfa` int DEFAULT NULL,\n `yovhxncvulteorchjetjinsralckhqzk` int DEFAULT NULL,\n `npxgevbyygtwwdmvpnicrwncfuhkooia` int DEFAULT NULL,\n `fxyhwifncfcgynlzermyplfsldumcuyb` int DEFAULT NULL,\n `javxzinfispwjdhzagvgwvneiugyfrbr` int DEFAULT NULL,\n `wadvuswyvwcxwiuyfheuclpkdynnfqvk` int DEFAULT NULL,\n `dqxsycfprhzxlvxpswucvfkusjhlmrkh` int DEFAULT NULL,\n `myghmhbyfngzryizcawwhoivilkvcrjc` int DEFAULT NULL,\n `epmhjipcskxznoykemytgptejytuzvqk` int DEFAULT NULL,\n `mhbpsdqbvvltlevjhjgcpwrigpopngdb` int DEFAULT NULL,\n `mnbkbennacnolctczvhwvpmmuorfmfvp` int DEFAULT NULL,\n `iztkxmrtkuijjdscmgzsqfpqtlftxxlg` int DEFAULT NULL,\n `gbzlqswmsfltgfymqiquwcxmnjemtrlz` int DEFAULT NULL,\n `lpywhyuqxzgegfqahyhypjakdhinffqm` int DEFAULT NULL,\n `mmriuhbchjzudompvawviwgfbfzsjcor` int DEFAULT NULL,\n `cetglgupnbxwwmhmkqfwamnbnukrzvrk` int DEFAULT NULL,\n `dplqcyujgcxtkklnanqmmsjpagvbrybf` int DEFAULT NULL,\n `ktlklwvezorsiuymrryuooplpqcbvfti` int DEFAULT NULL,\n `cvnfxoffbuuxxlyypkkipubnoczupkqo` int DEFAULT NULL,\n `ezpnraixduayjpgdkpdlmfniceihvgob` int DEFAULT NULL,\n `gmgajktawnvcaiinuqqoblpdlyijjpzw` int DEFAULT NULL,\n `ghtybfwsitqziiemhwxjqxlzjugntwbf` int DEFAULT NULL,\n `ofyqncgvfzffdrwjpmwrmoaqjduovqtv` int DEFAULT NULL,\n `ywftdvkplgrddclydfzblhybdkgzjrzd` int DEFAULT NULL,\n `alxqaucecefpfnpwvwhxxdjrbmocizoz` int DEFAULT NULL,\n `ipwvlkglaluhffwfycqdckpahbbnyitx` int DEFAULT NULL,\n `agbvhecyyloflxzmmonwwfrzjsfefxqn` int DEFAULT NULL,\n `vatcurfaubrpzeumkedgqaduzxdmecea` int DEFAULT NULL,\n `dwmneiwzivqajftfafzqyuetodqolzyq` int DEFAULT NULL,\n `ukdbcztgffoeuvkzwbjjbhvdfyghzaod` int DEFAULT NULL,\n `zqluyinygzfpabwbkelyorpztnaapmvf` int DEFAULT NULL,\n `tqkdshjcgtbopxpjjuziqumxhynbcvku` int DEFAULT NULL,\n `bfzoiauprwmwuzhlvlfowohugmuvkltk` int DEFAULT NULL,\n `fmsmnfdfqfkiifiwikajchvvchuwxrbk` int DEFAULT NULL,\n `mzjgvnghotossmpqqsmnsqtrixaxtxwf` int DEFAULT NULL,\n `liycphoxdemjeqlsnlwmeueqdxgfpycq` int DEFAULT NULL,\n `rpjyfwvinhghdwbdazmanqebztrgcgmi` int DEFAULT NULL,\n `mvzdrdmqomqqtxnzwlemuttbysxodaek` int DEFAULT NULL,\n `smaccnswkqvfncnybneshlicwdvmhrei` int DEFAULT NULL,\n `mbhfgwgcydobyfnmylrgjyttyebzcqfs` int DEFAULT NULL,\n `scvpscfgbnearoyscsfdrybekzhwyoly` int DEFAULT NULL,\n `gqbqpceppownjeillkkecaeaxktrlwav` int DEFAULT NULL,\n `pwgbittzoywvljzvioaucdbsfhxtanvk` int DEFAULT NULL,\n `ypitfubipgmmqpygwelksqcfxsnipjlr` int DEFAULT NULL,\n `tvnhvqmpsygvgtjgnlbjfhufkfmisscn` int DEFAULT NULL,\n `szvgdeyslvgpvqeixkjubslobgeuqttu` int DEFAULT NULL,\n `yakuqjidfdlpukqlfkejworfgjiqpytp` int DEFAULT NULL,\n `rurtcsgmhsdsnizydvhpvpfrgdqleraq` int DEFAULT NULL,\n `fzeqpsuyjcfsbcuxexgdlxopybpedufq` int DEFAULT NULL,\n `vhbvspqjeoubluvucsrfxwjmwsnrbxvf` int DEFAULT NULL,\n `ctzfatfmrxivqigouyrkijtaeddhyfqg` int DEFAULT NULL,\n `xzaypyvwrpyrhzhbhtqoesodztdtvggi` int DEFAULT NULL,\n `tadzkyltqapjgsbhyqrpqthkltrltcze` int DEFAULT NULL,\n `ozhobhihgrlhogpipszqbbwbcndihhuy` int DEFAULT NULL,\n `xqaogifeqiygfwusvbhnaayuxapbspki` int DEFAULT NULL,\n `hqevrtxwhqvoopuflrgolyvvtztdmghr` int DEFAULT NULL,\n `urzljffgaectnuwejunyxrjshhzyauuu` int DEFAULT NULL,\n `gwyjrzrudommvplakvgpsainmcgjjiom` int DEFAULT NULL,\n `nwoardiaxnftzaqvhqdreruguxackaps` int DEFAULT NULL,\n `zxeyxyxizccfuyfpcgibuykoxmxegvuc` int DEFAULT NULL,\n `gerpdgnwyntuliybjiqszfziporjpcvq` int DEFAULT NULL,\n `cqkcrloirknrvtfmiodcqwrpxgcknfjd` int DEFAULT NULL,\n `ofxocsspjdbyyubhohctvrlpqjbekaif` int DEFAULT NULL,\n `guzqhueniesfiuipbhcwhnxjazuwklok` int DEFAULT NULL,\n `ecdgpufluzikjdvvetdrnspypkwffqmy` int DEFAULT NULL,\n `qjaepwuynxagjscmbkgwzszzlmqixeit` int DEFAULT NULL,\n `aunkkbibruqohsfjkznxltpdfqkmituq` int DEFAULT NULL,\n `exmgkewjicdlnpoahpmbimmbpqcvmofg` int DEFAULT NULL,\n `ucutskzuhprhrrxjcjekokqkuiadofzo` int DEFAULT NULL,\n `ovevmglsdrxrchodnzfervwnnwxosanc` int DEFAULT NULL,\n `hucerpagcaegwsmjqgxxzzavnbivyica` int DEFAULT NULL,\n `rdziifaividucdujstowvgvrlupkenpr` int DEFAULT NULL,\n `kwlcaskgsvhngppeexydemnspreftdgi` int DEFAULT NULL,\n `uclmrlqgukajwoewibklozhyykhrkqun` int DEFAULT NULL,\n `wxzounndpavcdhqdplbffdnesbqeyhhu` int DEFAULT NULL,\n `cjxqvlohquqdkkasyraxzkjugscuipui` int DEFAULT NULL,\n `wvzgamtdrhcgxaqcwqtcgdmvytrkrphw` int DEFAULT NULL,\n `wksbjvxyjukfkurxpofkkxbuxroeafag` int DEFAULT NULL,\n `xvgcltlkphbdrxwkpqbvwyoptwpjavup` int DEFAULT NULL,\n `fqvusiwibequlvaqljvvmbdhpidbeojl` int DEFAULT NULL,\n `yckehepjelgydltbmgibfzfrvwtzcpos` int DEFAULT NULL,\n `tkebdeeplfueuuxlwpkenkyavhyicegi` int DEFAULT NULL,\n `wtgwqdettjvtvpdechqvvoytgwzztrth` int DEFAULT NULL,\n `gnormckgfkdzjuxzjztyryswvuhbupdk` int DEFAULT NULL,\n `uqvaasavdbrhklmromcfahtkattznoff` int DEFAULT NULL,\n `sjyecqoedzvdebtbyizzrxbhkzoveexo` int DEFAULT NULL,\n `pribyuqsrzpnengaejcgfudwucdoqajr` int DEFAULT NULL,\n `rwndwqpzncnbddkwsufhfkyckbrlzhee` int DEFAULT NULL,\n PRIMARY KEY (`gqpeikpphltvtgvvxoqeqkaofjfalwoq`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"ptxoeakypxcinxgiwxnegyucbfpderrz\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["gqpeikpphltvtgvvxoqeqkaofjfalwoq"],"columns":[{"name":"gqpeikpphltvtgvvxoqeqkaofjfalwoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"mabtbcdnbafwxoexkquoppknrvqipgqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsgeazdmkcvmykhlkxsfjzgeyplbyzpz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"seswbqhxofjenovwosokwhzdttwfdacq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lshrotucstdedpzwpksazqvdkxzlclea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kyqbzbcrwbllvsbrjudseueuqrlloaaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uznztszsqnhrpvzksifquwrkzojklgfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcaocnqsplyxzwckcalkdbfiuzyzkikb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhyazxhifjfvuwubqumrmpjptxshkesl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qsmquirewjodkcmzqywfnhavxnjvbayh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vyrjikuaynacieswsmpwpburaxierewo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fsuvhrdqxwwzmiwappvxsrhkegjchajd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojcsddldbtnfalgtjsadxkqftpanumfa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yovhxncvulteorchjetjinsralckhqzk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npxgevbyygtwwdmvpnicrwncfuhkooia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxyhwifncfcgynlzermyplfsldumcuyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"javxzinfispwjdhzagvgwvneiugyfrbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wadvuswyvwcxwiuyfheuclpkdynnfqvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqxsycfprhzxlvxpswucvfkusjhlmrkh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"myghmhbyfngzryizcawwhoivilkvcrjc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"epmhjipcskxznoykemytgptejytuzvqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhbpsdqbvvltlevjhjgcpwrigpopngdb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnbkbennacnolctczvhwvpmmuorfmfvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iztkxmrtkuijjdscmgzsqfpqtlftxxlg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gbzlqswmsfltgfymqiquwcxmnjemtrlz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lpywhyuqxzgegfqahyhypjakdhinffqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmriuhbchjzudompvawviwgfbfzsjcor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cetglgupnbxwwmhmkqfwamnbnukrzvrk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dplqcyujgcxtkklnanqmmsjpagvbrybf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ktlklwvezorsiuymrryuooplpqcbvfti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvnfxoffbuuxxlyypkkipubnoczupkqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezpnraixduayjpgdkpdlmfniceihvgob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmgajktawnvcaiinuqqoblpdlyijjpzw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghtybfwsitqziiemhwxjqxlzjugntwbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofyqncgvfzffdrwjpmwrmoaqjduovqtv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywftdvkplgrddclydfzblhybdkgzjrzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"alxqaucecefpfnpwvwhxxdjrbmocizoz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ipwvlkglaluhffwfycqdckpahbbnyitx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agbvhecyyloflxzmmonwwfrzjsfefxqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vatcurfaubrpzeumkedgqaduzxdmecea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwmneiwzivqajftfafzqyuetodqolzyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ukdbcztgffoeuvkzwbjjbhvdfyghzaod","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqluyinygzfpabwbkelyorpztnaapmvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqkdshjcgtbopxpjjuziqumxhynbcvku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bfzoiauprwmwuzhlvlfowohugmuvkltk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmsmnfdfqfkiifiwikajchvvchuwxrbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzjgvnghotossmpqqsmnsqtrixaxtxwf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"liycphoxdemjeqlsnlwmeueqdxgfpycq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpjyfwvinhghdwbdazmanqebztrgcgmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvzdrdmqomqqtxnzwlemuttbysxodaek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smaccnswkqvfncnybneshlicwdvmhrei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbhfgwgcydobyfnmylrgjyttyebzcqfs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scvpscfgbnearoyscsfdrybekzhwyoly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqbqpceppownjeillkkecaeaxktrlwav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwgbittzoywvljzvioaucdbsfhxtanvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypitfubipgmmqpygwelksqcfxsnipjlr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvnhvqmpsygvgtjgnlbjfhufkfmisscn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"szvgdeyslvgpvqeixkjubslobgeuqttu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yakuqjidfdlpukqlfkejworfgjiqpytp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rurtcsgmhsdsnizydvhpvpfrgdqleraq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzeqpsuyjcfsbcuxexgdlxopybpedufq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhbvspqjeoubluvucsrfxwjmwsnrbxvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctzfatfmrxivqigouyrkijtaeddhyfqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xzaypyvwrpyrhzhbhtqoesodztdtvggi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tadzkyltqapjgsbhyqrpqthkltrltcze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozhobhihgrlhogpipszqbbwbcndihhuy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqaogifeqiygfwusvbhnaayuxapbspki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqevrtxwhqvoopuflrgolyvvtztdmghr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urzljffgaectnuwejunyxrjshhzyauuu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gwyjrzrudommvplakvgpsainmcgjjiom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nwoardiaxnftzaqvhqdreruguxackaps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxeyxyxizccfuyfpcgibuykoxmxegvuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gerpdgnwyntuliybjiqszfziporjpcvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqkcrloirknrvtfmiodcqwrpxgcknfjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofxocsspjdbyyubhohctvrlpqjbekaif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"guzqhueniesfiuipbhcwhnxjazuwklok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecdgpufluzikjdvvetdrnspypkwffqmy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjaepwuynxagjscmbkgwzszzlmqixeit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aunkkbibruqohsfjkznxltpdfqkmituq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exmgkewjicdlnpoahpmbimmbpqcvmofg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucutskzuhprhrrxjcjekokqkuiadofzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovevmglsdrxrchodnzfervwnnwxosanc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hucerpagcaegwsmjqgxxzzavnbivyica","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdziifaividucdujstowvgvrlupkenpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwlcaskgsvhngppeexydemnspreftdgi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uclmrlqgukajwoewibklozhyykhrkqun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxzounndpavcdhqdplbffdnesbqeyhhu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjxqvlohquqdkkasyraxzkjugscuipui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvzgamtdrhcgxaqcwqtcgdmvytrkrphw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wksbjvxyjukfkurxpofkkxbuxroeafag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvgcltlkphbdrxwkpqbvwyoptwpjavup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqvusiwibequlvaqljvvmbdhpidbeojl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yckehepjelgydltbmgibfzfrvwtzcpos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkebdeeplfueuuxlwpkenkyavhyicegi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtgwqdettjvtvpdechqvvoytgwzztrth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnormckgfkdzjuxzjztyryswvuhbupdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqvaasavdbrhklmromcfahtkattznoff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjyecqoedzvdebtbyizzrxbhkzoveexo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pribyuqsrzpnengaejcgfudwucdoqajr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rwndwqpzncnbddkwsufhfkyckbrlzhee","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670308,"databaseName":"models_schema","ddl":"CREATE TABLE `qfgnabrfurvwokonupfzqzkpbgrlevbv` (\n `szqnajfjbzjstieajitgdxtikamrynjc` int NOT NULL,\n `heyeftwmmgwarrfqnllbnogfcbklxvpk` int DEFAULT NULL,\n `gxdxiiorupxoufuzsuqheysmdknglnzj` int DEFAULT NULL,\n `zjduolahmqcdppayqzrbnwskkuavocym` int DEFAULT NULL,\n `ctfnsmcnescbtmqrtsfymjlrjmnymaww` int DEFAULT NULL,\n `mwiyzovylotcrcsbicdjhstcjpxhwdsq` int DEFAULT NULL,\n `uboayjixhiwdlmjamfbbotqiktqapghq` int DEFAULT NULL,\n `dddkhsnysccnmmvllrsurwwribuxrzlc` int DEFAULT NULL,\n `jnxxldwhpiejhsdafntzjdlurppmygmz` int DEFAULT NULL,\n `naiwypsntdvvtfgfelpomckwyiuxuwvb` int DEFAULT NULL,\n `ebfyltgxihtlpamkhyzbblaqbguhxaiv` int DEFAULT NULL,\n `dnxefezelyzisopxksljzgskimkujvwz` int DEFAULT NULL,\n `ugyypnmijlvnjkmbeqejstlinweupofu` int DEFAULT NULL,\n `tpyurfdupucbbygxwzjluntwrrlwzfgn` int DEFAULT NULL,\n `ljqaxxxslwkjdzbpkwqutiqcsxswtcch` int DEFAULT NULL,\n `pixlvgfhqkuscvvxczswwocvmjlyxami` int DEFAULT NULL,\n `agbdsfdtuwenpnnxbfyvoieefnrnsloi` int DEFAULT NULL,\n `ujecfuozqomvnlurkpdqtclwoofhupgf` int DEFAULT NULL,\n `ixfqclnytjnexpsyrsyjeqrtaywkdbjw` int DEFAULT NULL,\n `xvalhshdlfdxeknttmfhywojujrvqdgc` int DEFAULT NULL,\n `nitvjlhsteescwywafruwwdtbedtebnu` int DEFAULT NULL,\n `mcjxrmyhjlzrxrkijkgphyhjwaysdfaq` int DEFAULT NULL,\n `gxkqmlcpymkwgghoqtdccgmweanxwknl` int DEFAULT NULL,\n `qtvsrcvdwcczkquftcepzbgdvqcqejph` int DEFAULT NULL,\n `fwhfmlvibudoyiikqtbrbhcptuumpetf` int DEFAULT NULL,\n `uizanslaazhnqpkxxoknirsoihhacnbd` int DEFAULT NULL,\n `obqcfqbbrdsygkzudvxohlydwdvlsdxv` int DEFAULT NULL,\n `xirqgwevuvuetaaihaxxatkaxeogifhx` int DEFAULT NULL,\n `psqttqduoeurfwslvlaxxnmzpllwymya` int DEFAULT NULL,\n `neearwskahyneuufxgirapmiomzhumee` int DEFAULT NULL,\n `nolubjvkbormtiknevbcqhfhjzmfzawq` int DEFAULT NULL,\n `lqdmdvtlvchyimubvzgrkezshmrqvdct` int DEFAULT NULL,\n `rfjhysjsjwmjkecnivyypvryfqokjrcg` int DEFAULT NULL,\n `upxdjlrcvprbiuwemeubvutgdpurtvxs` int DEFAULT NULL,\n `owqqwrnloaverqrxvayqezkdkuedulad` int DEFAULT NULL,\n `qanwlfpjmrbtkxocjrkpkaoxhygmuvry` int DEFAULT NULL,\n `tkanhtnevkopnpnfsexecqtnhgawrxzt` int DEFAULT NULL,\n `avajxjooktpbooobavpfdxsyzkddxhru` int DEFAULT NULL,\n `nrjtfjvorthnjftpowekmhdnwbdmiojf` int DEFAULT NULL,\n `ezzmwlrnbrnlvdvzkktfnxkxgpsbzjbk` int DEFAULT NULL,\n `uzcrwqmgcowmftdmwjgarxigfqqxqsyc` int DEFAULT NULL,\n `buvdcxybalklrfeqqdyqnrjibodzvgqo` int DEFAULT NULL,\n `iycuffuhdnwbgwwmtgbwvfoderzqxcie` int DEFAULT NULL,\n `fdaffuztzolnvyaxabunhgswgxewhycq` int DEFAULT NULL,\n `uailsvhpxyfrmoqspeytndywfsdyoeon` int DEFAULT NULL,\n `hzyjgfpekmwxsyesabydmpgkjqfalzei` int DEFAULT NULL,\n `ityrqaotjctxhbmanhcwrqaejdyzwtqk` int DEFAULT NULL,\n `yfucenjrxuxeroeineoknxaygjprjtdc` int DEFAULT NULL,\n `kdgecmmjpwplvcaarjpllkzbqkqmfnpv` int DEFAULT NULL,\n `rwkwmqjmcxzpkertzqbefocmspazezxh` int DEFAULT NULL,\n `cjmjlbnreqibghmarnjhkoxhdsprfcnw` int DEFAULT NULL,\n `jkfseztexccxwphsvfercsgzbmemhviq` int DEFAULT NULL,\n `myvofbukajtizqsbnhgrcistlcftbsvh` int DEFAULT NULL,\n `pwgrrmuplvwkvdlshobgijtfwznevdsz` int DEFAULT NULL,\n `gzekmuhqgairoymcarndmkiveeexuvyd` int DEFAULT NULL,\n `sorwdyrivpwqpaxwxnriaemgbymchjmz` int DEFAULT NULL,\n `devyrtddjoiscgondjcrpgqfqjwbgwpa` int DEFAULT NULL,\n `kmnrimaombtxpdoifbkcbdjmwobjvula` int DEFAULT NULL,\n `mlegvfaidfnsrcwlfhkxwuoldzneykxn` int DEFAULT NULL,\n `yspkybifhkolpzouipjoaqrunghkhvam` int DEFAULT NULL,\n `fmsyczppymfgkgpyvfgcieypwmnsbifv` int DEFAULT NULL,\n `vdenodhxxewifjrjdkhbvtugvfbjamam` int DEFAULT NULL,\n `qvcklrtpnticwpuknthzqhybamvkmekm` int DEFAULT NULL,\n `shwgexxxpzliikfwhmzvhwjwckbircra` int DEFAULT NULL,\n `rbzfqebuqtedygfnfkelrpeszpydkqxh` int DEFAULT NULL,\n `fmzmbmofwbtqkdxniqoqhagwwxceeogd` int DEFAULT NULL,\n `zxyasejgzjhnojianhisaatoinzgpmww` int DEFAULT NULL,\n `izyawzpzfcjqlctbkxjxiedeuymuiduu` int DEFAULT NULL,\n `itmvoupbqyvfesqlehkpqjwaljhqrbef` int DEFAULT NULL,\n `twsdzqqkcvirtwnfztoucyxkcdxzgbyz` int DEFAULT NULL,\n `swhpmqblehhtqrhxksuibnxhuxabggei` int DEFAULT NULL,\n `srefyjchycxzwcpzshfkcrqzjxpqgxoi` int DEFAULT NULL,\n `wocakbgiiwibxjhhqhfvsprsjyepveuw` int DEFAULT NULL,\n `aybvsgqbbxyzkkpwlfzvarhvgtaehdgx` int DEFAULT NULL,\n `zllqepxufjuppmytovncysptyacrzqqb` int DEFAULT NULL,\n `zbtwpbhjxpjsjpjrvmcvupoopwluuqnh` int DEFAULT NULL,\n `yftrxscrowgzjnwjbtdrwzdvkhbngrqc` int DEFAULT NULL,\n `ddykzicbdiqviscxnuxbclkyoyvfpxaq` int DEFAULT NULL,\n `jcifkotqykrgdykmhudykgzprqdukbrm` int DEFAULT NULL,\n `rsusraoicqqwdqpsrppjnwwntalbnxfh` int DEFAULT NULL,\n `aepetxppimmipzssjaznhseauauhyjxa` int DEFAULT NULL,\n `otbzudrbszclpiaazdeqplhpbecxllsc` int DEFAULT NULL,\n `fngofjqddjptgramimxbyzrqxmwmfodh` int DEFAULT NULL,\n `yhqvzvkbmtybyxjccywzbdmxadvrkboa` int DEFAULT NULL,\n `ywjcaibndimywaqjqrprzvopendcvvhv` int DEFAULT NULL,\n `lhkroeygpcuyznwvjzpjoqebaotvrjpy` int DEFAULT NULL,\n `oqxkuavglpaxcjdxrclsqtkfzqyerhlw` int DEFAULT NULL,\n `rnwwccelyryhshanyjkchhmmtvchsubo` int DEFAULT NULL,\n `chwjnkycrxbdnbzfpqjvmqbcfslolvly` int DEFAULT NULL,\n `wxnruidikocapkzuywlgcrkhhnqanwbd` int DEFAULT NULL,\n `xutltavwfpdpjmnquyzuimheibsdmklm` int DEFAULT NULL,\n `ttpwulysesnaxucpxarwmfpzfznnzcrt` int DEFAULT NULL,\n `oxvtwpqxthjkeqtjbltrziafuxswxsij` int DEFAULT NULL,\n `ebigmuwmwyjcgahfqygldmscbyotvack` int DEFAULT NULL,\n `itogwxhqsesoofgclaexekwrqxegljjp` int DEFAULT NULL,\n `nhnytylawialxewjwdquwylyodbdihal` int DEFAULT NULL,\n `gjqbaneihavubiegepzmnfuzwaouotmj` int DEFAULT NULL,\n `hdbrnbcluaareecdgguklajkimkcwglb` int DEFAULT NULL,\n `trnnmgfkzidmkefikguezylfouxbocek` int DEFAULT NULL,\n `etffjuemctxbcvgqfaetvynkhrzgafpp` int DEFAULT NULL,\n PRIMARY KEY (`szqnajfjbzjstieajitgdxtikamrynjc`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"qfgnabrfurvwokonupfzqzkpbgrlevbv\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["szqnajfjbzjstieajitgdxtikamrynjc"],"columns":[{"name":"szqnajfjbzjstieajitgdxtikamrynjc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"heyeftwmmgwarrfqnllbnogfcbklxvpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxdxiiorupxoufuzsuqheysmdknglnzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjduolahmqcdppayqzrbnwskkuavocym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctfnsmcnescbtmqrtsfymjlrjmnymaww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwiyzovylotcrcsbicdjhstcjpxhwdsq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uboayjixhiwdlmjamfbbotqiktqapghq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dddkhsnysccnmmvllrsurwwribuxrzlc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jnxxldwhpiejhsdafntzjdlurppmygmz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"naiwypsntdvvtfgfelpomckwyiuxuwvb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebfyltgxihtlpamkhyzbblaqbguhxaiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dnxefezelyzisopxksljzgskimkujvwz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugyypnmijlvnjkmbeqejstlinweupofu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpyurfdupucbbygxwzjluntwrrlwzfgn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljqaxxxslwkjdzbpkwqutiqcsxswtcch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pixlvgfhqkuscvvxczswwocvmjlyxami","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agbdsfdtuwenpnnxbfyvoieefnrnsloi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujecfuozqomvnlurkpdqtclwoofhupgf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixfqclnytjnexpsyrsyjeqrtaywkdbjw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvalhshdlfdxeknttmfhywojujrvqdgc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nitvjlhsteescwywafruwwdtbedtebnu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcjxrmyhjlzrxrkijkgphyhjwaysdfaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxkqmlcpymkwgghoqtdccgmweanxwknl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtvsrcvdwcczkquftcepzbgdvqcqejph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwhfmlvibudoyiikqtbrbhcptuumpetf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uizanslaazhnqpkxxoknirsoihhacnbd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obqcfqbbrdsygkzudvxohlydwdvlsdxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xirqgwevuvuetaaihaxxatkaxeogifhx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psqttqduoeurfwslvlaxxnmzpllwymya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"neearwskahyneuufxgirapmiomzhumee","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nolubjvkbormtiknevbcqhfhjzmfzawq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqdmdvtlvchyimubvzgrkezshmrqvdct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfjhysjsjwmjkecnivyypvryfqokjrcg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upxdjlrcvprbiuwemeubvutgdpurtvxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owqqwrnloaverqrxvayqezkdkuedulad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qanwlfpjmrbtkxocjrkpkaoxhygmuvry","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkanhtnevkopnpnfsexecqtnhgawrxzt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avajxjooktpbooobavpfdxsyzkddxhru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nrjtfjvorthnjftpowekmhdnwbdmiojf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezzmwlrnbrnlvdvzkktfnxkxgpsbzjbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzcrwqmgcowmftdmwjgarxigfqqxqsyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"buvdcxybalklrfeqqdyqnrjibodzvgqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iycuffuhdnwbgwwmtgbwvfoderzqxcie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fdaffuztzolnvyaxabunhgswgxewhycq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uailsvhpxyfrmoqspeytndywfsdyoeon","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hzyjgfpekmwxsyesabydmpgkjqfalzei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ityrqaotjctxhbmanhcwrqaejdyzwtqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfucenjrxuxeroeineoknxaygjprjtdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kdgecmmjpwplvcaarjpllkzbqkqmfnpv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rwkwmqjmcxzpkertzqbefocmspazezxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjmjlbnreqibghmarnjhkoxhdsprfcnw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jkfseztexccxwphsvfercsgzbmemhviq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"myvofbukajtizqsbnhgrcistlcftbsvh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwgrrmuplvwkvdlshobgijtfwznevdsz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzekmuhqgairoymcarndmkiveeexuvyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sorwdyrivpwqpaxwxnriaemgbymchjmz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"devyrtddjoiscgondjcrpgqfqjwbgwpa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmnrimaombtxpdoifbkcbdjmwobjvula","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlegvfaidfnsrcwlfhkxwuoldzneykxn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yspkybifhkolpzouipjoaqrunghkhvam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmsyczppymfgkgpyvfgcieypwmnsbifv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdenodhxxewifjrjdkhbvtugvfbjamam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvcklrtpnticwpuknthzqhybamvkmekm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shwgexxxpzliikfwhmzvhwjwckbircra","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbzfqebuqtedygfnfkelrpeszpydkqxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmzmbmofwbtqkdxniqoqhagwwxceeogd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxyasejgzjhnojianhisaatoinzgpmww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"izyawzpzfcjqlctbkxjxiedeuymuiduu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itmvoupbqyvfesqlehkpqjwaljhqrbef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twsdzqqkcvirtwnfztoucyxkcdxzgbyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swhpmqblehhtqrhxksuibnxhuxabggei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"srefyjchycxzwcpzshfkcrqzjxpqgxoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wocakbgiiwibxjhhqhfvsprsjyepveuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aybvsgqbbxyzkkpwlfzvarhvgtaehdgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zllqepxufjuppmytovncysptyacrzqqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbtwpbhjxpjsjpjrvmcvupoopwluuqnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yftrxscrowgzjnwjbtdrwzdvkhbngrqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddykzicbdiqviscxnuxbclkyoyvfpxaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jcifkotqykrgdykmhudykgzprqdukbrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rsusraoicqqwdqpsrppjnwwntalbnxfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aepetxppimmipzssjaznhseauauhyjxa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otbzudrbszclpiaazdeqplhpbecxllsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fngofjqddjptgramimxbyzrqxmwmfodh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhqvzvkbmtybyxjccywzbdmxadvrkboa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywjcaibndimywaqjqrprzvopendcvvhv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhkroeygpcuyznwvjzpjoqebaotvrjpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqxkuavglpaxcjdxrclsqtkfzqyerhlw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnwwccelyryhshanyjkchhmmtvchsubo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chwjnkycrxbdnbzfpqjvmqbcfslolvly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxnruidikocapkzuywlgcrkhhnqanwbd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xutltavwfpdpjmnquyzuimheibsdmklm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ttpwulysesnaxucpxarwmfpzfznnzcrt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oxvtwpqxthjkeqtjbltrziafuxswxsij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebigmuwmwyjcgahfqygldmscbyotvack","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itogwxhqsesoofgclaexekwrqxegljjp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhnytylawialxewjwdquwylyodbdihal","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjqbaneihavubiegepzmnfuzwaouotmj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdbrnbcluaareecdgguklajkimkcwglb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"trnnmgfkzidmkefikguezylfouxbocek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"etffjuemctxbcvgqfaetvynkhrzgafpp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670341,"databaseName":"models_schema","ddl":"CREATE TABLE `qggnhwqlqjkouhxetdqfuaqozrqqpcle` (\n `nkcdnzebndbyasbnqisxnzspkzhxcesi` int NOT NULL,\n `uiqvevbkevxhawiatnzdpgvljwkgfvhw` int DEFAULT NULL,\n `lfolqnupwcpuubjuqlstcwglnbnawaxg` int DEFAULT NULL,\n `ixousxuvwwhzeexbmnombpwlbboqrcqd` int DEFAULT NULL,\n `lxkregdopjbnovnroytxwbabvctwqcfr` int DEFAULT NULL,\n `abtjljodxeplfeqvvfoinyitbrxqkcsb` int DEFAULT NULL,\n `obhebsguysmhkmlsgoourtsnurqxdmxh` int DEFAULT NULL,\n `irfszabggjuewxzlzfwdudfpfpxbzibu` int DEFAULT NULL,\n `wszaaftmpcvcpmngyusrvmeqjlpshzxy` int DEFAULT NULL,\n `epoilfwakmeoonpctduimdwijciyrbdb` int DEFAULT NULL,\n `zmkoepmobefcdjbpzsmjpdliccgewhig` int DEFAULT NULL,\n `cetcncmwxswktdzbglynztogqafnzdvy` int DEFAULT NULL,\n `ylgwfcznilknwzbtobeizbqrqrctaddr` int DEFAULT NULL,\n `aykiskamzhergximgmxegrqrsoybqgmk` int DEFAULT NULL,\n `qrpzrthvxaxhhhuypmhojxaetxzsijwy` int DEFAULT NULL,\n `ggkxkjprhcpilfhlljaypmhvqmorjlko` int DEFAULT NULL,\n `ikirhvzvvvopqqdytfrrqpdmklhstifa` int DEFAULT NULL,\n `xskizbkmxlggnctpsytsljskfvgcfbuw` int DEFAULT NULL,\n `qlxczkazlglsbkbjoagshaixilszydyl` int DEFAULT NULL,\n `nukjxihcrirmecmxtfqlyvczstpcmfvu` int DEFAULT NULL,\n `huvlnpcqhcbmnflsxaajunoxarcjslcy` int DEFAULT NULL,\n `dvayewcekknoinozvdxcpeqbocmksptq` int DEFAULT NULL,\n `oiiofggandtuklquudlamyoqltwumvbi` int DEFAULT NULL,\n `nelvjweumtuwzzkgxvuzrhehnprzvvhz` int DEFAULT NULL,\n `bmbyvmdlachdixkztesejudtebwbfbyl` int DEFAULT NULL,\n `vbtbcclrawgaikzootmrqbsiyheysuyf` int DEFAULT NULL,\n `xtczabvpiaimsuboractoebikduwinuh` int DEFAULT NULL,\n `gcjlrkwzlwetdqijeiczedujfwpqxdiy` int DEFAULT NULL,\n `vbsgrpwzbvbmqqyrsrkgdplihsvafbrc` int DEFAULT NULL,\n `mstwwgwjjouwavnwocwafamjumbfltwi` int DEFAULT NULL,\n `cijlkywpwykgfffvssczjtpgegoanzfn` int DEFAULT NULL,\n `yihmdywayzsgohjmlltvdfswgjlmqfqh` int DEFAULT NULL,\n `lmwmnouiixdrkorwwqbmvksnuofwqcdo` int DEFAULT NULL,\n `svawbupglkqtffxembfprblcpofkrolf` int DEFAULT NULL,\n `uprcmkndwmjkgcbxmvqnuyupnvuawcjj` int DEFAULT NULL,\n `tlwsknvkkobulcutjndhrfhixgzfbnah` int DEFAULT NULL,\n `lupydelgfjayulzgeqlzjexbircyujkl` int DEFAULT NULL,\n `utrrgbxxhujgugarltjnqbxnmlolrwqd` int DEFAULT NULL,\n `rjzodgzdyxsterqvjpuahsvzhndjtxey` int DEFAULT NULL,\n `yelrkusonrmxbccfaxfaqhtorcrnkcvj` int DEFAULT NULL,\n `yevzolduciyvkamrhfowrykgdfaiweyt` int DEFAULT NULL,\n `gzyypkxaookkvhcilqbcqtvwjtqkkprk` int DEFAULT NULL,\n `zqxksejmjybvfplpdtmltxneltdoyvwl` int DEFAULT NULL,\n `ohpbyvodzenmqaqdimwrabpwumkmhvwq` int DEFAULT NULL,\n `davhhkvnsdzsihudjbruzmjwwkerdbto` int DEFAULT NULL,\n `yarnzbveuxskhvkkggljznqlkvqckelv` int DEFAULT NULL,\n `suxpxqrjgzhirbjvnfszekgrarsrpcur` int DEFAULT NULL,\n `wuphileuwpvvcxtlvmvcoejqskgubyja` int DEFAULT NULL,\n `qbvhcrzrkeccwaqiwehtvuodemwotqpu` int DEFAULT NULL,\n `egdftxtfalkcarturyyiejuctrarjupk` int DEFAULT NULL,\n `aicvrbojrxirtvjqsemiiedegtimdwqs` int DEFAULT NULL,\n `nmjfvxymjhxehbzimpfikgvsifdidoed` int DEFAULT NULL,\n `xjeromaahfboqdncnqwknixptrqicdtl` int DEFAULT NULL,\n `idvrszjmywftltvkjdnlqhyctctdweiy` int DEFAULT NULL,\n `ypjeodahlydllahedlpllrtunlrgqxjp` int DEFAULT NULL,\n `gllzhjbmxfcxhlgytpqzdizyclytgsln` int DEFAULT NULL,\n `jkuflveiryyqtazleonenaukglzfkpzw` int DEFAULT NULL,\n `pikrucouyavuvbmdbvqiurqaemceymqu` int DEFAULT NULL,\n `tzcnvwbtyskqgnstvedjqdlbqbaiunwo` int DEFAULT NULL,\n `vbqslxyxcfjxafrbglmfufyicmwlirnj` int DEFAULT NULL,\n `jbzlsgqabyejgbllmsrxzejkofitlmhc` int DEFAULT NULL,\n `yurkhpfjvhcgccgwylbpmfarrfqxxdmj` int DEFAULT NULL,\n `ivistauytgzmeujqsblvqcpjihgwmsan` int DEFAULT NULL,\n `rsuehzibsivwlheslhkcwkyozgbphmuj` int DEFAULT NULL,\n `xgssmybjifisrhlowoucwsvtdnyqefyc` int DEFAULT NULL,\n `xpjtvjodhqpnkhlffmuczflnqujwadiz` int DEFAULT NULL,\n `gwsuhpdwxajmjcanxrckgnoxxosmzwaa` int DEFAULT NULL,\n `jmmxjoksxsrvzmfhrhvupvmtxxyxszso` int DEFAULT NULL,\n `jarqghggclqboyalnnruftmklwuirieo` int DEFAULT NULL,\n `dinswaklxckhrhakzywdpjvvuhrthczp` int DEFAULT NULL,\n `lxdfykrsgegvbruvprtqpvhwchxnbnpr` int DEFAULT NULL,\n `dlfojduslkceqqblytyhbydvshdaniel` int DEFAULT NULL,\n `fprtqhcjqjzzwdyjkospgquifabzcoel` int DEFAULT NULL,\n `wuoikrmaevhprueufhopnpuxutuzagqf` int DEFAULT NULL,\n `nfjyjwqcglkzsoiarxemxfbqktlyhkwo` int DEFAULT NULL,\n `xzrfqmblyasnsrecrwulzwgqnlsaroiu` int DEFAULT NULL,\n `etcgzwhdamnbackeddckpkavltmomxgt` int DEFAULT NULL,\n `dgksddyqpcjehficbtpiaidncaaskghp` int DEFAULT NULL,\n `slznubfggwlxubtiujahxfzrnkrtlgis` int DEFAULT NULL,\n `cmshzkbdchdblmapvrbuvpnswygmovdm` int DEFAULT NULL,\n `haxwjvqikqztdhsiotnlsncwcsbvksnk` int DEFAULT NULL,\n `cbxprqlxcuhquoupshkbiecnnwmugcsy` int DEFAULT NULL,\n `uyxoxnjkwkgilwjbuhsrhdgfpmlwsnmg` int DEFAULT NULL,\n `pduwcmylgtlucibgnjpceoagninxvfdz` int DEFAULT NULL,\n `pjzicckogruocbiwtetkeijbzpbbfffd` int DEFAULT NULL,\n `ojqwptaytirwjewplloefvktmkctnqqz` int DEFAULT NULL,\n `jroskzdaqpwibtkevxhgzbxwpdrknygu` int DEFAULT NULL,\n `hbiuqlkzkciaxmfzjnbmbzikdpqujoed` int DEFAULT NULL,\n `vdqvikmrgglyyolzeipwytyorbslaoar` int DEFAULT NULL,\n `rttyuhefsswckqdumagfszvlhukudcsn` int DEFAULT NULL,\n `fpvtajekozvzemhuovejqmoyplumamhs` int DEFAULT NULL,\n `rnbgrllwnwokxjbupbvhsrsiprofzulu` int DEFAULT NULL,\n `gnliktpkawtulizyueayfsjcraahhztc` int DEFAULT NULL,\n `qsgjnzmntkhauayqqczrstysfjprllur` int DEFAULT NULL,\n `thgutnwjebdjlmxfiwchknyxyczwjxvx` int DEFAULT NULL,\n `wzdhalhoiocynmpvkdzreyckelyvtbjo` int DEFAULT NULL,\n `wnppirwdkpppssdcprrocxdbksdkwwue` int DEFAULT NULL,\n `veorkdvquezzdazrrfgadnndzalhoyhq` int DEFAULT NULL,\n `ezenpeoysiypqjulongzaclzkxuikdpz` int DEFAULT NULL,\n `ibrrpcedlkaqxfmcgxjxafvifxvwbfdx` int DEFAULT NULL,\n PRIMARY KEY (`nkcdnzebndbyasbnqisxnzspkzhxcesi`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"qggnhwqlqjkouhxetdqfuaqozrqqpcle\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["nkcdnzebndbyasbnqisxnzspkzhxcesi"],"columns":[{"name":"nkcdnzebndbyasbnqisxnzspkzhxcesi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"uiqvevbkevxhawiatnzdpgvljwkgfvhw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfolqnupwcpuubjuqlstcwglnbnawaxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixousxuvwwhzeexbmnombpwlbboqrcqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxkregdopjbnovnroytxwbabvctwqcfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abtjljodxeplfeqvvfoinyitbrxqkcsb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obhebsguysmhkmlsgoourtsnurqxdmxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"irfszabggjuewxzlzfwdudfpfpxbzibu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wszaaftmpcvcpmngyusrvmeqjlpshzxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"epoilfwakmeoonpctduimdwijciyrbdb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmkoepmobefcdjbpzsmjpdliccgewhig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cetcncmwxswktdzbglynztogqafnzdvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ylgwfcznilknwzbtobeizbqrqrctaddr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aykiskamzhergximgmxegrqrsoybqgmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrpzrthvxaxhhhuypmhojxaetxzsijwy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggkxkjprhcpilfhlljaypmhvqmorjlko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikirhvzvvvopqqdytfrrqpdmklhstifa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xskizbkmxlggnctpsytsljskfvgcfbuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qlxczkazlglsbkbjoagshaixilszydyl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nukjxihcrirmecmxtfqlyvczstpcmfvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"huvlnpcqhcbmnflsxaajunoxarcjslcy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvayewcekknoinozvdxcpeqbocmksptq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oiiofggandtuklquudlamyoqltwumvbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nelvjweumtuwzzkgxvuzrhehnprzvvhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bmbyvmdlachdixkztesejudtebwbfbyl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbtbcclrawgaikzootmrqbsiyheysuyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xtczabvpiaimsuboractoebikduwinuh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gcjlrkwzlwetdqijeiczedujfwpqxdiy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbsgrpwzbvbmqqyrsrkgdplihsvafbrc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mstwwgwjjouwavnwocwafamjumbfltwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cijlkywpwykgfffvssczjtpgegoanzfn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yihmdywayzsgohjmlltvdfswgjlmqfqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmwmnouiixdrkorwwqbmvksnuofwqcdo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"svawbupglkqtffxembfprblcpofkrolf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uprcmkndwmjkgcbxmvqnuyupnvuawcjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlwsknvkkobulcutjndhrfhixgzfbnah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lupydelgfjayulzgeqlzjexbircyujkl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utrrgbxxhujgugarltjnqbxnmlolrwqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjzodgzdyxsterqvjpuahsvzhndjtxey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yelrkusonrmxbccfaxfaqhtorcrnkcvj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yevzolduciyvkamrhfowrykgdfaiweyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzyypkxaookkvhcilqbcqtvwjtqkkprk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqxksejmjybvfplpdtmltxneltdoyvwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohpbyvodzenmqaqdimwrabpwumkmhvwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"davhhkvnsdzsihudjbruzmjwwkerdbto","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yarnzbveuxskhvkkggljznqlkvqckelv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suxpxqrjgzhirbjvnfszekgrarsrpcur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wuphileuwpvvcxtlvmvcoejqskgubyja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qbvhcrzrkeccwaqiwehtvuodemwotqpu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"egdftxtfalkcarturyyiejuctrarjupk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aicvrbojrxirtvjqsemiiedegtimdwqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmjfvxymjhxehbzimpfikgvsifdidoed","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjeromaahfboqdncnqwknixptrqicdtl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idvrszjmywftltvkjdnlqhyctctdweiy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypjeodahlydllahedlpllrtunlrgqxjp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gllzhjbmxfcxhlgytpqzdizyclytgsln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jkuflveiryyqtazleonenaukglzfkpzw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pikrucouyavuvbmdbvqiurqaemceymqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tzcnvwbtyskqgnstvedjqdlbqbaiunwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbqslxyxcfjxafrbglmfufyicmwlirnj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbzlsgqabyejgbllmsrxzejkofitlmhc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yurkhpfjvhcgccgwylbpmfarrfqxxdmj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivistauytgzmeujqsblvqcpjihgwmsan","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rsuehzibsivwlheslhkcwkyozgbphmuj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xgssmybjifisrhlowoucwsvtdnyqefyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpjtvjodhqpnkhlffmuczflnqujwadiz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gwsuhpdwxajmjcanxrckgnoxxosmzwaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmmxjoksxsrvzmfhrhvupvmtxxyxszso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jarqghggclqboyalnnruftmklwuirieo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dinswaklxckhrhakzywdpjvvuhrthczp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxdfykrsgegvbruvprtqpvhwchxnbnpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlfojduslkceqqblytyhbydvshdaniel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fprtqhcjqjzzwdyjkospgquifabzcoel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wuoikrmaevhprueufhopnpuxutuzagqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfjyjwqcglkzsoiarxemxfbqktlyhkwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xzrfqmblyasnsrecrwulzwgqnlsaroiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"etcgzwhdamnbackeddckpkavltmomxgt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgksddyqpcjehficbtpiaidncaaskghp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"slznubfggwlxubtiujahxfzrnkrtlgis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmshzkbdchdblmapvrbuvpnswygmovdm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"haxwjvqikqztdhsiotnlsncwcsbvksnk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbxprqlxcuhquoupshkbiecnnwmugcsy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uyxoxnjkwkgilwjbuhsrhdgfpmlwsnmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pduwcmylgtlucibgnjpceoagninxvfdz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjzicckogruocbiwtetkeijbzpbbfffd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojqwptaytirwjewplloefvktmkctnqqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jroskzdaqpwibtkevxhgzbxwpdrknygu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbiuqlkzkciaxmfzjnbmbzikdpqujoed","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdqvikmrgglyyolzeipwytyorbslaoar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rttyuhefsswckqdumagfszvlhukudcsn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpvtajekozvzemhuovejqmoyplumamhs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnbgrllwnwokxjbupbvhsrsiprofzulu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnliktpkawtulizyueayfsjcraahhztc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qsgjnzmntkhauayqqczrstysfjprllur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thgutnwjebdjlmxfiwchknyxyczwjxvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzdhalhoiocynmpvkdzreyckelyvtbjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnppirwdkpppssdcprrocxdbksdkwwue","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"veorkdvquezzdazrrfgadnndzalhoyhq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezenpeoysiypqjulongzaclzkxuikdpz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibrrpcedlkaqxfmcgxjxafvifxvwbfdx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670371,"databaseName":"models_schema","ddl":"CREATE TABLE `qiwtlstwkkhmomafokasfiebppralewy` (\n `yxvugbpvfcahhbyhpxslejdwpjyxlgiy` int NOT NULL,\n `bkcyzzdzxnclmdvhntgdvqhxbtybbqgg` int DEFAULT NULL,\n `qyhesvuwjpnkrumnoucafsjwkuqpwkaj` int DEFAULT NULL,\n `vjtflfqscnuholiiswrncgdarnvadyxu` int DEFAULT NULL,\n `vpcihjidcoatmjfartkzoaywfkqibewi` int DEFAULT NULL,\n `pinqqyfbnigravmzzobzvsdfrtjywqsz` int DEFAULT NULL,\n `ksqahcuuwzpteddvvhyyzhekikvwkkxj` int DEFAULT NULL,\n `qhateemsktxoolydsqxxwthsuiblkhed` int DEFAULT NULL,\n `vyvtodeppeoehiemplmlozsfnnehxjqn` int DEFAULT NULL,\n `nlszyliqpxnzqtpsddhircjwcmlcuvug` int DEFAULT NULL,\n `mgvfvvrlhexsamanhgrgzqencihmefrf` int DEFAULT NULL,\n `fyqujucnefbayvbjulsafogitgezmfql` int DEFAULT NULL,\n `prplhytewrftldzkkihvhawgxolhywbu` int DEFAULT NULL,\n `nzvjddgeijpvbiniqjiffquqanmkzwti` int DEFAULT NULL,\n `ikwbcrioidcypsqsutnndedvtnzgtddg` int DEFAULT NULL,\n `caldtqtohwqqpvqlfpgeeiayxofvbtyi` int DEFAULT NULL,\n `bjqsdzhbdeyyusqozyjfrfwakfifqlrc` int DEFAULT NULL,\n `vklxcxabzsjxtzqisgvulxbzjnvkrpqk` int DEFAULT NULL,\n `zfhnxywqhbtdwhbqbdgspjnxebfsgdhp` int DEFAULT NULL,\n `lfrhvkfqzbhcinaqicjhoakmkmlfkhtz` int DEFAULT NULL,\n `sbvoqfuvjgszralsnxcwprqdtnoenmds` int DEFAULT NULL,\n `xcojaptyrorasdqymaydcnwwxgfsjrub` int DEFAULT NULL,\n `devlhxaffnbtrytzuctnnalfvolyeunw` int DEFAULT NULL,\n `siidjcuerviprkookhbohmzvfqifclht` int DEFAULT NULL,\n `wqhjzgaurxroemeoziabfchvwhmbpneb` int DEFAULT NULL,\n `woyesxopefmzpnzgeuypsnhlicmqmpgt` int DEFAULT NULL,\n `paxnbpipgqvtvwifgyptfobrtfyzcqyg` int DEFAULT NULL,\n `hvgjtsmacdleuffwnjvufdtbzpxqbvbl` int DEFAULT NULL,\n `mdefkizrflljemehwznzjqbcfxdxqjvt` int DEFAULT NULL,\n `rvdonezmrquhlgkfiwfhgszoupfluewm` int DEFAULT NULL,\n `hkssyztpndsvlvxxrbtkvcnncufytvjz` int DEFAULT NULL,\n `nraswgkjgdfyifrqqluiakrfxylabxph` int DEFAULT NULL,\n `ftzcqcuyimkaxarvucqxciedlvjrfyru` int DEFAULT NULL,\n `nckyyiduzfkvovfysmnakryerewnjerv` int DEFAULT NULL,\n `ulaiezkygefdgnqalodxftljukllpnhk` int DEFAULT NULL,\n `mbjqosdubbssldhfaksxxmedpefuxagv` int DEFAULT NULL,\n `knoaqvlefvxpjnwndxavlqbqlqibnjmd` int DEFAULT NULL,\n `xlaevicupefyaoaomerkjvmfocfoqpnr` int DEFAULT NULL,\n `ajucufetnobczqhujhaajwpakayaixxy` int DEFAULT NULL,\n `egqflfryduwihuwckbetnspzxcpapvnk` int DEFAULT NULL,\n `zynsjzhjnmgxlpkchkekcwktdrfpeboq` int DEFAULT NULL,\n `pttndmzcwwwgrsytbvamknmgemnklkmu` int DEFAULT NULL,\n `lsmeoncyqezhoqqrxmjhusyfyoadqowc` int DEFAULT NULL,\n `aunqkeqwttgcbrlecbcscbgitffncpfj` int DEFAULT NULL,\n `ckuercoiupvnyqjqriublggcowpqaomg` int DEFAULT NULL,\n `lqgtxpyqourfkregzmsglfauqsoeamdz` int DEFAULT NULL,\n `circdqnyugnjmwqlyjdjicmspmrgxuhx` int DEFAULT NULL,\n `ygwtdpelgkcfgfjqxcelobcxmywsmwjd` int DEFAULT NULL,\n `exixtouoacwjbpdqzwgvmsutuovnhlxx` int DEFAULT NULL,\n `ekepjjkvrkpbviyimxyjmndozxksmeql` int DEFAULT NULL,\n `elcxzwkhwwplnpqkipgmxzrrbeltlhsq` int DEFAULT NULL,\n `ntzavspzblkfaknsfogidiqapczskoqz` int DEFAULT NULL,\n `ftzoqnimxlkygfgdjgibulcjndtiblpb` int DEFAULT NULL,\n `mrexywcasxdoslhwlluapdzzgjffhrbk` int DEFAULT NULL,\n `idzihmvuttvrykaxcrmmcgeddudwbylh` int DEFAULT NULL,\n `uemqhegsftwtpnvttrxtxbigsofbhlnd` int DEFAULT NULL,\n `mefmloqvgzqqhqritfgfnbbkajrdzfeg` int DEFAULT NULL,\n `ivvhakhsuxtlrgjjhnnxunoihpxfowqj` int DEFAULT NULL,\n `hckswjfttwlyxlkomlbaaztxnzyxjomr` int DEFAULT NULL,\n `lldaczvnphtynqkqpnpiwgttzmvykqye` int DEFAULT NULL,\n `jzuezwkckervulenvwsyjmcgcypmndex` int DEFAULT NULL,\n `pbxjmslaomnpadhwtyhqyoijmpvowifp` int DEFAULT NULL,\n `kkaupvfyyrratblttcqrvecsdkudzsem` int DEFAULT NULL,\n `ydqtgbqmscmflexqbschomqnhimpzdao` int DEFAULT NULL,\n `qtppncsnbixjubgzoavmabsrsvhowxeq` int DEFAULT NULL,\n `cmtjkeknybcjjitwuzqqebbtbmbiioaq` int DEFAULT NULL,\n `rllvhymghaxhslzcnsfsfudtpumcmnky` int DEFAULT NULL,\n `tzqanbjwhebxcshhbayocappfznssvvn` int DEFAULT NULL,\n `azgqdbpdufftpflxgebeellwsqbyessg` int DEFAULT NULL,\n `iqwzzngazmmhshkhvlyocrryddsmfblc` int DEFAULT NULL,\n `fqzzyqvxwtqndhyzzjazsdnoqyfzggrn` int DEFAULT NULL,\n `evfkfwjxxixihpmcfyyggpxaljnxboiw` int DEFAULT NULL,\n `awpvwarjzfcnimtfhzmmreppulkxhxvl` int DEFAULT NULL,\n `tsdvwsibrstzhxixrsfnciqqbtyffudx` int DEFAULT NULL,\n `recadzjrzxerssrnasoddxiyizaawfjj` int DEFAULT NULL,\n `ircuxesztqaevwkhqwuhkmblrikwjhdt` int DEFAULT NULL,\n `thrsemicfckatygnjdguzxoyprkyaynx` int DEFAULT NULL,\n `mmcuqfnhhqevzhlbjrniahaszqmqqayf` int DEFAULT NULL,\n `ytnfwngkfgggkuqhhuhcqyqsvceojzni` int DEFAULT NULL,\n `rycmaltlpyxekwvxhgoyfroqlvggkluu` int DEFAULT NULL,\n `uqioidfabqphcjgulbakqslpodcirblu` int DEFAULT NULL,\n `yptbtmbqbndlvedichnhogixzwhtjite` int DEFAULT NULL,\n `mcvawsiofukegaofxnmyywryedbybdtb` int DEFAULT NULL,\n `mytfxylxlzuvggbrwtovtpcmynupiaeu` int DEFAULT NULL,\n `drbearwfzuesoghnkehidzknamrpchig` int DEFAULT NULL,\n `opfifntwvrnsvzhvkajsxrxszazzfijo` int DEFAULT NULL,\n `tulgqppmetnrjndjidvfrvuyecrfomhu` int DEFAULT NULL,\n `dfmgjwzojfkerwgbotsosiwguxbzzbny` int DEFAULT NULL,\n `hratqkoagrjiejqjusrciryfqcwiepzr` int DEFAULT NULL,\n `yobewiigcjfbcgjxcbmpuvrsuezxgyjv` int DEFAULT NULL,\n `skbnncyxxvpiigxmpqoszatijebfyrba` int DEFAULT NULL,\n `rdybjemulridtvacrhgrruhwwvmzawzs` int DEFAULT NULL,\n `hgruddmswzwboijnztjgtqjjcthwazoi` int DEFAULT NULL,\n `ybnbxfzqjhhnygzejacfbvltxtvsmjif` int DEFAULT NULL,\n `aweumgbnomwphpfrwotzidodznkqbmen` int DEFAULT NULL,\n `mhtjdvhwrleccielimgbsxdpxebytpvl` int DEFAULT NULL,\n `kvobvcxmhkdrryutewwjudsmbwjncmds` int DEFAULT NULL,\n `rgwhbfywkouyxnntscovvmdhtlcrsdnn` int DEFAULT NULL,\n `nhioryslhbwwdmeaalcqcdodvppxlhzv` int DEFAULT NULL,\n `tyeoayuxhkifxhxccaiphhyzuogaquay` int DEFAULT NULL,\n PRIMARY KEY (`yxvugbpvfcahhbyhpxslejdwpjyxlgiy`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"qiwtlstwkkhmomafokasfiebppralewy\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["yxvugbpvfcahhbyhpxslejdwpjyxlgiy"],"columns":[{"name":"yxvugbpvfcahhbyhpxslejdwpjyxlgiy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"bkcyzzdzxnclmdvhntgdvqhxbtybbqgg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qyhesvuwjpnkrumnoucafsjwkuqpwkaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjtflfqscnuholiiswrncgdarnvadyxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpcihjidcoatmjfartkzoaywfkqibewi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pinqqyfbnigravmzzobzvsdfrtjywqsz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ksqahcuuwzpteddvvhyyzhekikvwkkxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhateemsktxoolydsqxxwthsuiblkhed","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vyvtodeppeoehiemplmlozsfnnehxjqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nlszyliqpxnzqtpsddhircjwcmlcuvug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mgvfvvrlhexsamanhgrgzqencihmefrf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fyqujucnefbayvbjulsafogitgezmfql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prplhytewrftldzkkihvhawgxolhywbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzvjddgeijpvbiniqjiffquqanmkzwti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikwbcrioidcypsqsutnndedvtnzgtddg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"caldtqtohwqqpvqlfpgeeiayxofvbtyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjqsdzhbdeyyusqozyjfrfwakfifqlrc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vklxcxabzsjxtzqisgvulxbzjnvkrpqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zfhnxywqhbtdwhbqbdgspjnxebfsgdhp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfrhvkfqzbhcinaqicjhoakmkmlfkhtz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbvoqfuvjgszralsnxcwprqdtnoenmds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xcojaptyrorasdqymaydcnwwxgfsjrub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"devlhxaffnbtrytzuctnnalfvolyeunw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"siidjcuerviprkookhbohmzvfqifclht","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqhjzgaurxroemeoziabfchvwhmbpneb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"woyesxopefmzpnzgeuypsnhlicmqmpgt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"paxnbpipgqvtvwifgyptfobrtfyzcqyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hvgjtsmacdleuffwnjvufdtbzpxqbvbl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdefkizrflljemehwznzjqbcfxdxqjvt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rvdonezmrquhlgkfiwfhgszoupfluewm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkssyztpndsvlvxxrbtkvcnncufytvjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nraswgkjgdfyifrqqluiakrfxylabxph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftzcqcuyimkaxarvucqxciedlvjrfyru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nckyyiduzfkvovfysmnakryerewnjerv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ulaiezkygefdgnqalodxftljukllpnhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbjqosdubbssldhfaksxxmedpefuxagv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knoaqvlefvxpjnwndxavlqbqlqibnjmd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xlaevicupefyaoaomerkjvmfocfoqpnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajucufetnobczqhujhaajwpakayaixxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"egqflfryduwihuwckbetnspzxcpapvnk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zynsjzhjnmgxlpkchkekcwktdrfpeboq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pttndmzcwwwgrsytbvamknmgemnklkmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lsmeoncyqezhoqqrxmjhusyfyoadqowc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aunqkeqwttgcbrlecbcscbgitffncpfj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckuercoiupvnyqjqriublggcowpqaomg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqgtxpyqourfkregzmsglfauqsoeamdz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"circdqnyugnjmwqlyjdjicmspmrgxuhx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ygwtdpelgkcfgfjqxcelobcxmywsmwjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exixtouoacwjbpdqzwgvmsutuovnhlxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekepjjkvrkpbviyimxyjmndozxksmeql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elcxzwkhwwplnpqkipgmxzrrbeltlhsq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntzavspzblkfaknsfogidiqapczskoqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftzoqnimxlkygfgdjgibulcjndtiblpb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrexywcasxdoslhwlluapdzzgjffhrbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idzihmvuttvrykaxcrmmcgeddudwbylh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uemqhegsftwtpnvttrxtxbigsofbhlnd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mefmloqvgzqqhqritfgfnbbkajrdzfeg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivvhakhsuxtlrgjjhnnxunoihpxfowqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hckswjfttwlyxlkomlbaaztxnzyxjomr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lldaczvnphtynqkqpnpiwgttzmvykqye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzuezwkckervulenvwsyjmcgcypmndex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbxjmslaomnpadhwtyhqyoijmpvowifp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kkaupvfyyrratblttcqrvecsdkudzsem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydqtgbqmscmflexqbschomqnhimpzdao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtppncsnbixjubgzoavmabsrsvhowxeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmtjkeknybcjjitwuzqqebbtbmbiioaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rllvhymghaxhslzcnsfsfudtpumcmnky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tzqanbjwhebxcshhbayocappfznssvvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azgqdbpdufftpflxgebeellwsqbyessg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqwzzngazmmhshkhvlyocrryddsmfblc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqzzyqvxwtqndhyzzjazsdnoqyfzggrn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evfkfwjxxixihpmcfyyggpxaljnxboiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awpvwarjzfcnimtfhzmmreppulkxhxvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tsdvwsibrstzhxixrsfnciqqbtyffudx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"recadzjrzxerssrnasoddxiyizaawfjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ircuxesztqaevwkhqwuhkmblrikwjhdt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thrsemicfckatygnjdguzxoyprkyaynx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmcuqfnhhqevzhlbjrniahaszqmqqayf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytnfwngkfgggkuqhhuhcqyqsvceojzni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rycmaltlpyxekwvxhgoyfroqlvggkluu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqioidfabqphcjgulbakqslpodcirblu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yptbtmbqbndlvedichnhogixzwhtjite","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcvawsiofukegaofxnmyywryedbybdtb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mytfxylxlzuvggbrwtovtpcmynupiaeu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drbearwfzuesoghnkehidzknamrpchig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opfifntwvrnsvzhvkajsxrxszazzfijo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tulgqppmetnrjndjidvfrvuyecrfomhu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dfmgjwzojfkerwgbotsosiwguxbzzbny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hratqkoagrjiejqjusrciryfqcwiepzr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yobewiigcjfbcgjxcbmpuvrsuezxgyjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skbnncyxxvpiigxmpqoszatijebfyrba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdybjemulridtvacrhgrruhwwvmzawzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hgruddmswzwboijnztjgtqjjcthwazoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybnbxfzqjhhnygzejacfbvltxtvsmjif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aweumgbnomwphpfrwotzidodznkqbmen","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhtjdvhwrleccielimgbsxdpxebytpvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvobvcxmhkdrryutewwjudsmbwjncmds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgwhbfywkouyxnntscovvmdhtlcrsdnn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhioryslhbwwdmeaalcqcdodvppxlhzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tyeoayuxhkifxhxccaiphhyzuogaquay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670409,"databaseName":"models_schema","ddl":"CREATE TABLE `qlasmsrgsfuikzxkdrpxtdwwbaalkjde` (\n `rzjqvrsqykxqogmcrsmmiohhqfbvoqzw` int NOT NULL,\n `cexovlnrujxonpcnvjllgnhvvxjpjlaf` int DEFAULT NULL,\n `bmwppftkebbzldqpxuntiomszzlvurvv` int DEFAULT NULL,\n `vakaxyntwecenlbsubkjfzwzulygomaq` int DEFAULT NULL,\n `rzagxcpyulnqllykzsruvqijbzscaowk` int DEFAULT NULL,\n `jqpvmoniaiqyztsunblwxvjhgwqjtigj` int DEFAULT NULL,\n `otnvqqnrojogfopasajiggwwotzwrdhz` int DEFAULT NULL,\n `wgwudjnekktkrcjdjmhemfndcuotcarb` int DEFAULT NULL,\n `nxadkltmlhewwrallpcktqvrciewrycl` int DEFAULT NULL,\n `lcjiovnpsajpodtxxinuxpxkrubxqbqb` int DEFAULT NULL,\n `mppkucrgnmoaxizkxkazpujiufkwgzcl` int DEFAULT NULL,\n `rycyvradwnjploazvxautvwfhcorfssu` int DEFAULT NULL,\n `uhxgivziqqqrsukzjqmzsyrpxsimpvrc` int DEFAULT NULL,\n `uggqjjkcgrsgiloivtxspalebiqmkffz` int DEFAULT NULL,\n `ybyeswrrmzoxtkscigtuetwpbqeusrwf` int DEFAULT NULL,\n `dzscfbbbrhcfnhjrtxgqguaedaadcpgn` int DEFAULT NULL,\n `qozexvggtvmklcithxjnlhwjvvxvaoxp` int DEFAULT NULL,\n `hirusxsjmcdfmvcnbmaabkhafxnexzqu` int DEFAULT NULL,\n `temoqmohqaqnwwnypqukrhjhvxhrdvge` int DEFAULT NULL,\n `dppdlfozpeksnighhgsufwlzezragkzm` int DEFAULT NULL,\n `imhbrwicleuymozsavmfjulqyygdrvti` int DEFAULT NULL,\n `okpzvvnxhrijncwrxspmjerjiaxwvbyw` int DEFAULT NULL,\n `zxcrzuyjquiwwfmdqbamhhociuleadbi` int DEFAULT NULL,\n `nqvgbijyxsqiausafhhxipnrvmmakuhe` int DEFAULT NULL,\n `rqrmefuvprvgwsazhiejafbanharjwcr` int DEFAULT NULL,\n `btsecavjscyatrxsdoaxtkifiithgvor` int DEFAULT NULL,\n `drkmsgvkoekifjnckcyuabyqdbmootob` int DEFAULT NULL,\n `lytlnmxakhgjgnwnozamhehpmylowfcb` int DEFAULT NULL,\n `kcsqmfdynofuopmlhjyaesmzvhzmmrrk` int DEFAULT NULL,\n `cccwemvethmxjqvpntqmyjjfwwlvxqje` int DEFAULT NULL,\n `iuqpatafsmhhhpkuzkkcvyopprlvazog` int DEFAULT NULL,\n `mnrqodiczxrtxzgawosvcprvhmhsaehm` int DEFAULT NULL,\n `mfycpxmvytynymxcaxrcopanhxkyjdro` int DEFAULT NULL,\n `qwugodyjvjjkwxqjqjldqyutnokhusbn` int DEFAULT NULL,\n `lfmjnfyyvzblastmoxlpbgwevofhabbn` int DEFAULT NULL,\n `pigymattjgtlygnjjlpduvlnnoeaoumc` int DEFAULT NULL,\n `ywtizdutpxywhkrzbzbtqutqvctpduxm` int DEFAULT NULL,\n `tazkukiayyxieknzwqqmqasmqdjtzfwa` int DEFAULT NULL,\n `qweeokdjqftfrlboqwrgnvsrwacwjrjt` int DEFAULT NULL,\n `rrtoejorporxvrnhqcwohiyloaypeotg` int DEFAULT NULL,\n `ovtmssqhkdhxvxqkfjrfaaawvqxktglx` int DEFAULT NULL,\n `kncxzethwxgtwwkkagdpyiiwyfduyvid` int DEFAULT NULL,\n `phopaxzaqsienarlivegptfjqfmlloft` int DEFAULT NULL,\n `flovarvwwejiihvfodijmvcmrgstnssb` int DEFAULT NULL,\n `bilipvblfipvdedjuutzspceytwxsgal` int DEFAULT NULL,\n `dxjqkeqdvrwanayqbilhunqpugaoqjtx` int DEFAULT NULL,\n `ykveajfwciieebqgfmgarvxfihcztxam` int DEFAULT NULL,\n `xxdfjgztqkowoqtbepjckvmfihbjygep` int DEFAULT NULL,\n `vzmkwkpbmgeeremkhpfjwrhnbhpkxjpx` int DEFAULT NULL,\n `onkvaycibqnbbfbovdeqldiqytpowceg` int DEFAULT NULL,\n `ktzowvssoeecpxjuythbcxxthqcxszjk` int DEFAULT NULL,\n `qldhcuivtgmcjwxuudfigukiblfyggro` int DEFAULT NULL,\n `ghfiqcabbzlprtgfxprvalrmaisnauor` int DEFAULT NULL,\n `ffweqnkzpwrgfqowpcrpoxbxpvkjgwxr` int DEFAULT NULL,\n `rcnylgjmoutxotfjlvlbewmfgnagwxry` int DEFAULT NULL,\n `qllobsuywtcfkcdrwkqwswjnhmbjxcca` int DEFAULT NULL,\n `exvlpigctmzgqclszieokwluxpdnxfhl` int DEFAULT NULL,\n `niynzrungwlroysntapkhducxssagejx` int DEFAULT NULL,\n `jworytmvrocxqwcnpygrgnicxqhkwufs` int DEFAULT NULL,\n `faykdktyimahpdqkciycwjzgnmtvevbc` int DEFAULT NULL,\n `xkhocfsdgfmhqwycikserxqrdekuyynd` int DEFAULT NULL,\n `kfvuymwedmmucrhovvgwhlwwoevckebp` int DEFAULT NULL,\n `xuznyfozmjbdwmstzlrencjhrmjshfut` int DEFAULT NULL,\n `yexrkwqnomekluaphtbcfwhreghqwoyd` int DEFAULT NULL,\n `pjftdpmhumbaaltacszvxbugsbcoxcln` int DEFAULT NULL,\n `oqqzbvjvilxsxyuftwnjnapmiwfpxvjz` int DEFAULT NULL,\n `jacgqnqgvvtstkqvcvoqpqyficcoodfe` int DEFAULT NULL,\n `dvwuzlvdiqtfdvkvhxqxvuynayzizsta` int DEFAULT NULL,\n `wtkzadooqrlpmiyfjmcrmkusygioyawu` int DEFAULT NULL,\n `iqmqqnzawvfzuxhltzcaqxltsouxrwsk` int DEFAULT NULL,\n `npsxyojhflihigdfnexgrknuaqxapewd` int DEFAULT NULL,\n `ffqfljcldaywfdbbveqvhkaefogffdmi` int DEFAULT NULL,\n `jlmokwkfxaozxmmmvzcjcppdgbqtctmt` int DEFAULT NULL,\n `dhiulaawghxjvdbzytemzbucehartuyh` int DEFAULT NULL,\n `dngzfwkdpxialupkbtgcuyxomsninrvo` int DEFAULT NULL,\n `pjjusyfethuxzamnjiblpmonfqhyjqty` int DEFAULT NULL,\n `jpyvizgcqyfuushzpbcbjfftzsrxibqh` int DEFAULT NULL,\n `yusggpjcbhfcyujnpkuwlfwwkpaqihsm` int DEFAULT NULL,\n `hbvchbhembuqykxhgqzxjwrdjjgnxwmz` int DEFAULT NULL,\n `jksjofaskfgmmyykojifqlvxdnishhkf` int DEFAULT NULL,\n `zoubrvfbewxpgfdswexbavbsxreoolme` int DEFAULT NULL,\n `rwakjecdrmfkgvwwbflpfrvbytdobmgb` int DEFAULT NULL,\n `tjdkbxdwzqmfvzzzuwbhlsihsjplkifd` int DEFAULT NULL,\n `iktbiqwslgafzcpuyigspexhzqcqolil` int DEFAULT NULL,\n `ovoqqhrlojcakzflzybeagtqakughcvd` int DEFAULT NULL,\n `deiixxbwyyigyouemdirrasithqbishx` int DEFAULT NULL,\n `gdipprzjhebcocfgzsnenzrfynfaxeku` int DEFAULT NULL,\n `bihmzururrbxxfcsibouvxjtidafkdwc` int DEFAULT NULL,\n `bzvsdzekxvwirqowfcwgiagoyqgylxca` int DEFAULT NULL,\n `wpihavkslpmqplmuuvajuxylysxfpuhj` int DEFAULT NULL,\n `jnotzawcaeatofghamjiskhtynjzwytq` int DEFAULT NULL,\n `culnwghvepuzvoexsxrxybkvopbvlneh` int DEFAULT NULL,\n `xsrhnzdctecgcjjlrxscukoxwdbevebf` int DEFAULT NULL,\n `yihwcsxgabejqpektpridizhbegocqij` int DEFAULT NULL,\n `hadqvtlglynngsdxmlyxwrabwqlgeadp` int DEFAULT NULL,\n `iyfpndydkodelkcimwhhiedokohrfskw` int DEFAULT NULL,\n `kpuczqckdqvitzqicxoalyjgytzfqmzb` int DEFAULT NULL,\n `ievqlwhhyyrclciethyhpldhuyfsnahh` int DEFAULT NULL,\n `llocbfkfzfsizsjszmzgtfoyxhirgdbe` int DEFAULT NULL,\n `vxwmpcyixajvszkpikyedsigllzqgral` int DEFAULT NULL,\n PRIMARY KEY (`rzjqvrsqykxqogmcrsmmiohhqfbvoqzw`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"qlasmsrgsfuikzxkdrpxtdwwbaalkjde\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["rzjqvrsqykxqogmcrsmmiohhqfbvoqzw"],"columns":[{"name":"rzjqvrsqykxqogmcrsmmiohhqfbvoqzw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"cexovlnrujxonpcnvjllgnhvvxjpjlaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bmwppftkebbzldqpxuntiomszzlvurvv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vakaxyntwecenlbsubkjfzwzulygomaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzagxcpyulnqllykzsruvqijbzscaowk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jqpvmoniaiqyztsunblwxvjhgwqjtigj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otnvqqnrojogfopasajiggwwotzwrdhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgwudjnekktkrcjdjmhemfndcuotcarb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxadkltmlhewwrallpcktqvrciewrycl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lcjiovnpsajpodtxxinuxpxkrubxqbqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mppkucrgnmoaxizkxkazpujiufkwgzcl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rycyvradwnjploazvxautvwfhcorfssu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhxgivziqqqrsukzjqmzsyrpxsimpvrc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uggqjjkcgrsgiloivtxspalebiqmkffz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybyeswrrmzoxtkscigtuetwpbqeusrwf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dzscfbbbrhcfnhjrtxgqguaedaadcpgn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qozexvggtvmklcithxjnlhwjvvxvaoxp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hirusxsjmcdfmvcnbmaabkhafxnexzqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"temoqmohqaqnwwnypqukrhjhvxhrdvge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dppdlfozpeksnighhgsufwlzezragkzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"imhbrwicleuymozsavmfjulqyygdrvti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okpzvvnxhrijncwrxspmjerjiaxwvbyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxcrzuyjquiwwfmdqbamhhociuleadbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqvgbijyxsqiausafhhxipnrvmmakuhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqrmefuvprvgwsazhiejafbanharjwcr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"btsecavjscyatrxsdoaxtkifiithgvor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drkmsgvkoekifjnckcyuabyqdbmootob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lytlnmxakhgjgnwnozamhehpmylowfcb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcsqmfdynofuopmlhjyaesmzvhzmmrrk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cccwemvethmxjqvpntqmyjjfwwlvxqje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iuqpatafsmhhhpkuzkkcvyopprlvazog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnrqodiczxrtxzgawosvcprvhmhsaehm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfycpxmvytynymxcaxrcopanhxkyjdro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwugodyjvjjkwxqjqjldqyutnokhusbn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfmjnfyyvzblastmoxlpbgwevofhabbn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pigymattjgtlygnjjlpduvlnnoeaoumc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywtizdutpxywhkrzbzbtqutqvctpduxm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tazkukiayyxieknzwqqmqasmqdjtzfwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qweeokdjqftfrlboqwrgnvsrwacwjrjt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrtoejorporxvrnhqcwohiyloaypeotg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovtmssqhkdhxvxqkfjrfaaawvqxktglx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kncxzethwxgtwwkkagdpyiiwyfduyvid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phopaxzaqsienarlivegptfjqfmlloft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flovarvwwejiihvfodijmvcmrgstnssb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bilipvblfipvdedjuutzspceytwxsgal","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxjqkeqdvrwanayqbilhunqpugaoqjtx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykveajfwciieebqgfmgarvxfihcztxam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxdfjgztqkowoqtbepjckvmfihbjygep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzmkwkpbmgeeremkhpfjwrhnbhpkxjpx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"onkvaycibqnbbfbovdeqldiqytpowceg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ktzowvssoeecpxjuythbcxxthqcxszjk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qldhcuivtgmcjwxuudfigukiblfyggro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghfiqcabbzlprtgfxprvalrmaisnauor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffweqnkzpwrgfqowpcrpoxbxpvkjgwxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcnylgjmoutxotfjlvlbewmfgnagwxry","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qllobsuywtcfkcdrwkqwswjnhmbjxcca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exvlpigctmzgqclszieokwluxpdnxfhl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"niynzrungwlroysntapkhducxssagejx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jworytmvrocxqwcnpygrgnicxqhkwufs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"faykdktyimahpdqkciycwjzgnmtvevbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkhocfsdgfmhqwycikserxqrdekuyynd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfvuymwedmmucrhovvgwhlwwoevckebp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xuznyfozmjbdwmstzlrencjhrmjshfut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yexrkwqnomekluaphtbcfwhreghqwoyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjftdpmhumbaaltacszvxbugsbcoxcln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqqzbvjvilxsxyuftwnjnapmiwfpxvjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jacgqnqgvvtstkqvcvoqpqyficcoodfe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvwuzlvdiqtfdvkvhxqxvuynayzizsta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtkzadooqrlpmiyfjmcrmkusygioyawu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqmqqnzawvfzuxhltzcaqxltsouxrwsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npsxyojhflihigdfnexgrknuaqxapewd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffqfljcldaywfdbbveqvhkaefogffdmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlmokwkfxaozxmmmvzcjcppdgbqtctmt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhiulaawghxjvdbzytemzbucehartuyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dngzfwkdpxialupkbtgcuyxomsninrvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjjusyfethuxzamnjiblpmonfqhyjqty","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpyvizgcqyfuushzpbcbjfftzsrxibqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yusggpjcbhfcyujnpkuwlfwwkpaqihsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbvchbhembuqykxhgqzxjwrdjjgnxwmz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jksjofaskfgmmyykojifqlvxdnishhkf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zoubrvfbewxpgfdswexbavbsxreoolme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rwakjecdrmfkgvwwbflpfrvbytdobmgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjdkbxdwzqmfvzzzuwbhlsihsjplkifd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iktbiqwslgafzcpuyigspexhzqcqolil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovoqqhrlojcakzflzybeagtqakughcvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"deiixxbwyyigyouemdirrasithqbishx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdipprzjhebcocfgzsnenzrfynfaxeku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bihmzururrbxxfcsibouvxjtidafkdwc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzvsdzekxvwirqowfcwgiagoyqgylxca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wpihavkslpmqplmuuvajuxylysxfpuhj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jnotzawcaeatofghamjiskhtynjzwytq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"culnwghvepuzvoexsxrxybkvopbvlneh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsrhnzdctecgcjjlrxscukoxwdbevebf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yihwcsxgabejqpektpridizhbegocqij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hadqvtlglynngsdxmlyxwrabwqlgeadp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iyfpndydkodelkcimwhhiedokohrfskw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpuczqckdqvitzqicxoalyjgytzfqmzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ievqlwhhyyrclciethyhpldhuyfsnahh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llocbfkfzfsizsjszmzgtfoyxhirgdbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxwmpcyixajvszkpikyedsigllzqgral","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670449,"databaseName":"models_schema","ddl":"CREATE TABLE `qmflonomnffllkxweddtjcrusntdyhwl` (\n `gvhsjfytckpbekvlsanodapxvqklvbag` int NOT NULL,\n `zhdpwqivfelmwqkengzrfionmzkyphbg` int DEFAULT NULL,\n `xlymkdssyhnlukkhgptxztqjczxdwnty` int DEFAULT NULL,\n `rfqcncgewoknzgldsmplfmivhycvgrhi` int DEFAULT NULL,\n `rorcdcsbcjsetxagvxxkaioiosgtmkov` int DEFAULT NULL,\n `okceyfmiwwavdlygovczkrcyavsvpkyu` int DEFAULT NULL,\n `ctnrnwfwyuoupuslmqrsevffxakulgtb` int DEFAULT NULL,\n `udfzdjxflrupvpkjazsdkfvggfgvgbxq` int DEFAULT NULL,\n `usbdsjstzxsgocltajkjgdphrqkylqpu` int DEFAULT NULL,\n `ycdyiqfxcduriwnjkzyhpbcshsojsrqe` int DEFAULT NULL,\n `lhxmgfyiglyzviursfyugslvvcxrmvzl` int DEFAULT NULL,\n `xoildhxonzqxmjwckodsehxnkpotdxin` int DEFAULT NULL,\n `qifvrbuktxfpcvzocrljdpjimzdraukw` int DEFAULT NULL,\n `vkvjueopwbwcavgixxwjbifghnnfydeo` int DEFAULT NULL,\n `xovysmilebebrlalcwvzjwlcjsediirz` int DEFAULT NULL,\n `bahixgfsyekhmrqvmrfpfcviucsnblhh` int DEFAULT NULL,\n `staebhhwktrhekdctuicavhdjlxeeldf` int DEFAULT NULL,\n `yaaxhdtpxcvehnyzmajotsmgyyqunwmk` int DEFAULT NULL,\n `snedfurmsbfqxseizjcrdekjadcewwll` int DEFAULT NULL,\n `zclduvlyzlbootsrixzrubjnuclsxyxh` int DEFAULT NULL,\n `troezfywonkwffymwhqqptymnkvuemud` int DEFAULT NULL,\n `myzdbugagvflufzyvuooeifofhlpyrta` int DEFAULT NULL,\n `qugqajnlzlzkiomgmmucpgczbmuwswii` int DEFAULT NULL,\n `mdgxntnwxvwiunofcjvgknosxqqvrahr` int DEFAULT NULL,\n `pyghgvvoamzazbbmhbwdejagbccmtvak` int DEFAULT NULL,\n `coqdtufpyigimiblqqulmoacpzffurss` int DEFAULT NULL,\n `hpwcoxclscyucwcqgwleyjvgkzrgbnej` int DEFAULT NULL,\n `dbhjgunlyrdtdopsqmjgamccjafjedru` int DEFAULT NULL,\n `ocvinqcrbijlhsokacwddlsukloulcym` int DEFAULT NULL,\n `sbraasuojkuwagcalnimrkbudlnulbii` int DEFAULT NULL,\n `ctowjcrkysljjzxgoiquvtqefoznhakn` int DEFAULT NULL,\n `ligpmgjqjsaghxdyivumlimgainnnmub` int DEFAULT NULL,\n `rdoobxtfzyqdtvezhpqnncqwjolcrwtf` int DEFAULT NULL,\n `rqlbjimbeokpnjntlkhmssqfvscvcphc` int DEFAULT NULL,\n `aqldphqaougmpuiustfjljduwbknrpoy` int DEFAULT NULL,\n `mwjfvtvcpiqyaniwzhvkmyxspfphwopa` int DEFAULT NULL,\n `tpzaxgdhgdfftuzbiniskoiapggaijdk` int DEFAULT NULL,\n `qrsghawocakkmwetwaryxzovpowextrg` int DEFAULT NULL,\n `qdiymiwwizuwespitolckadqybimyrkv` int DEFAULT NULL,\n `znaavefarhaiimxjrtwjyjfosxsknxam` int DEFAULT NULL,\n `pnnxrgemkfdusqnwdbowbqfmnhcvduxy` int DEFAULT NULL,\n `ramjaquhhyglbdrbezqjcyxpxpuzcrsr` int DEFAULT NULL,\n `mvvktnwddcbzfpevyavwbhtxmddtyxyc` int DEFAULT NULL,\n `sxuustojyutznlfupbxlxdrhjochzaaw` int DEFAULT NULL,\n `omdbzqybtkcewyfvhvtzqjidalwqeqhm` int DEFAULT NULL,\n `ntwfvrrbczxtzszflfqkhkurppjdrdla` int DEFAULT NULL,\n `nzcpsftmohalzwjkhzzwfkczygyknqvn` int DEFAULT NULL,\n `igfnarpijsfyasjtwokkwssbmjrdkpye` int DEFAULT NULL,\n `pudlrzirmcelacygzcdepczxddgogcuz` int DEFAULT NULL,\n `yxjladgditxnuygimpwxrdzmqnybkxmw` int DEFAULT NULL,\n `fufwcwqzqndqtunleffcrlmhzjpphlix` int DEFAULT NULL,\n `kmlbnpkpjzsgxpfujdvamcxrjtdhgiya` int DEFAULT NULL,\n `lmvysjokktxmmlaoffrgqtwkbaqpbgxm` int DEFAULT NULL,\n `znjdzifwedlxvewivvexbcbanvtaikgl` int DEFAULT NULL,\n `harhaopfgmxmzpkwjjrdplmkeqethrux` int DEFAULT NULL,\n `kahluygjebwqxhjlafjmphjglngmnzba` int DEFAULT NULL,\n `raumihulckqsdzsmlhksvmsenuuxmmeg` int DEFAULT NULL,\n `xjdprmfvglbwiaevnpmubvjvuilvlcef` int DEFAULT NULL,\n `uyqnulrcpwtpiyyulitdwrzohfagrbxg` int DEFAULT NULL,\n `oputqsaxybvtbtjuepvqgimsaczupkdx` int DEFAULT NULL,\n `qohqtraepzxagkovqwtcxlifuuxjwzxk` int DEFAULT NULL,\n `xqxvidrwgainwytazhhxuottcbvschuw` int DEFAULT NULL,\n `tqrvjhhcungkywagmewzhmatajbuylqd` int DEFAULT NULL,\n `sfmxxxjrwfsepfwqqozpzvvoodjindvu` int DEFAULT NULL,\n `eppxqqxrfclowpoltcvdnasgfuhcbcmu` int DEFAULT NULL,\n `ttttgtqaosfzxboxytjvrpqxigcupvum` int DEFAULT NULL,\n `dgtwtybvhbctbilvjebyutrpksaabbir` int DEFAULT NULL,\n `gklflfhtmoxntcaleqwhcnakwbkfpnxk` int DEFAULT NULL,\n `dyjbhfquxwfyjjirhqnzjeueglqhembk` int DEFAULT NULL,\n `cdbflhpeugodmdpheubxnkwtdrkiblsc` int DEFAULT NULL,\n `gskejrgtoiiauttkpujlklloozyweigp` int DEFAULT NULL,\n `ewwholnnbrtkrwzvffzrgrrxpcmwqcmk` int DEFAULT NULL,\n `zidejwxqvfshxsndirshjnpwmvxsnyva` int DEFAULT NULL,\n `jnbbqaejsigirfbifmkbebnliytzzjby` int DEFAULT NULL,\n `vnygsmmneejzxkzvpmhmyccgbflsnyuj` int DEFAULT NULL,\n `sqlugftkbfnuulhajvgbqyznnaocfaeq` int DEFAULT NULL,\n `mzikzgjtjygwqidbcsrgapsbddqfffnk` int DEFAULT NULL,\n `izmoosqzltxfjnujakcnbiuhfxvcpofd` int DEFAULT NULL,\n `pnriymfwlxidauutctimfskpycjbnppw` int DEFAULT NULL,\n `nggbpdmqhnukdbaahvyqekrbaumscnjl` int DEFAULT NULL,\n `twkhskcjjmscfzprggvohdxonclmutrl` int DEFAULT NULL,\n `pamvejlgqfzrpjrtbqwmlxelyqmdyrdr` int DEFAULT NULL,\n `dqubsbtmnoeztkhucnuslgwrefoylsrg` int DEFAULT NULL,\n `fdleqiepttlqvssosheqtqhubiinztxd` int DEFAULT NULL,\n `ynpvhlzwosxjlsiotodbsgtsdtwahcro` int DEFAULT NULL,\n `vkujqcfpmbikgezmtvnhbwhcmgqcmmuv` int DEFAULT NULL,\n `jfncfddoilvdkamqkinnooatunyimgod` int DEFAULT NULL,\n `amjtluqceocsjtopwrhcljgjzjxskgoh` int DEFAULT NULL,\n `omvdhohjzxuucmkomklnesmxpazrquuj` int DEFAULT NULL,\n `abhqjrmvskxkiacbbwpgthjxdlpcckay` int DEFAULT NULL,\n `caxjtgfuixtuwivruxmlzhheiobxzwhq` int DEFAULT NULL,\n `nqfnwqiwwmffhwnwbuorxuccdrtvnxpn` int DEFAULT NULL,\n `anyzslgsupsgxfeilipwiyzauopwlkub` int DEFAULT NULL,\n `kdlbbrbflurvykldlunvfhxpifknigce` int DEFAULT NULL,\n `jgbmgybppvrcxhrttrbthdlvnvjnpwlh` int DEFAULT NULL,\n `xybagyoizyxdrxsfnpklevtcakjftnnj` int DEFAULT NULL,\n `eycybflwzyfekpnrlclkiabgbqwjanhe` int DEFAULT NULL,\n `fnwdyjypqctlirzczsqricimxbmrghbx` int DEFAULT NULL,\n `xbsthdhzpiatvbckrcjexnaxqtuncthf` int DEFAULT NULL,\n `wfooqlhojujgpxokumikyjvljnsculia` int DEFAULT NULL,\n PRIMARY KEY (`gvhsjfytckpbekvlsanodapxvqklvbag`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"qmflonomnffllkxweddtjcrusntdyhwl\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["gvhsjfytckpbekvlsanodapxvqklvbag"],"columns":[{"name":"gvhsjfytckpbekvlsanodapxvqklvbag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"zhdpwqivfelmwqkengzrfionmzkyphbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xlymkdssyhnlukkhgptxztqjczxdwnty","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfqcncgewoknzgldsmplfmivhycvgrhi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rorcdcsbcjsetxagvxxkaioiosgtmkov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okceyfmiwwavdlygovczkrcyavsvpkyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctnrnwfwyuoupuslmqrsevffxakulgtb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udfzdjxflrupvpkjazsdkfvggfgvgbxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"usbdsjstzxsgocltajkjgdphrqkylqpu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ycdyiqfxcduriwnjkzyhpbcshsojsrqe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhxmgfyiglyzviursfyugslvvcxrmvzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xoildhxonzqxmjwckodsehxnkpotdxin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qifvrbuktxfpcvzocrljdpjimzdraukw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkvjueopwbwcavgixxwjbifghnnfydeo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xovysmilebebrlalcwvzjwlcjsediirz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bahixgfsyekhmrqvmrfpfcviucsnblhh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"staebhhwktrhekdctuicavhdjlxeeldf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yaaxhdtpxcvehnyzmajotsmgyyqunwmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"snedfurmsbfqxseizjcrdekjadcewwll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zclduvlyzlbootsrixzrubjnuclsxyxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"troezfywonkwffymwhqqptymnkvuemud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"myzdbugagvflufzyvuooeifofhlpyrta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qugqajnlzlzkiomgmmucpgczbmuwswii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdgxntnwxvwiunofcjvgknosxqqvrahr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pyghgvvoamzazbbmhbwdejagbccmtvak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"coqdtufpyigimiblqqulmoacpzffurss","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hpwcoxclscyucwcqgwleyjvgkzrgbnej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbhjgunlyrdtdopsqmjgamccjafjedru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocvinqcrbijlhsokacwddlsukloulcym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbraasuojkuwagcalnimrkbudlnulbii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctowjcrkysljjzxgoiquvtqefoznhakn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ligpmgjqjsaghxdyivumlimgainnnmub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdoobxtfzyqdtvezhpqnncqwjolcrwtf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqlbjimbeokpnjntlkhmssqfvscvcphc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqldphqaougmpuiustfjljduwbknrpoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwjfvtvcpiqyaniwzhvkmyxspfphwopa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpzaxgdhgdfftuzbiniskoiapggaijdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrsghawocakkmwetwaryxzovpowextrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdiymiwwizuwespitolckadqybimyrkv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znaavefarhaiimxjrtwjyjfosxsknxam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnnxrgemkfdusqnwdbowbqfmnhcvduxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ramjaquhhyglbdrbezqjcyxpxpuzcrsr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvvktnwddcbzfpevyavwbhtxmddtyxyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sxuustojyutznlfupbxlxdrhjochzaaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omdbzqybtkcewyfvhvtzqjidalwqeqhm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntwfvrrbczxtzszflfqkhkurppjdrdla","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzcpsftmohalzwjkhzzwfkczygyknqvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igfnarpijsfyasjtwokkwssbmjrdkpye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pudlrzirmcelacygzcdepczxddgogcuz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxjladgditxnuygimpwxrdzmqnybkxmw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fufwcwqzqndqtunleffcrlmhzjpphlix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmlbnpkpjzsgxpfujdvamcxrjtdhgiya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmvysjokktxmmlaoffrgqtwkbaqpbgxm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znjdzifwedlxvewivvexbcbanvtaikgl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"harhaopfgmxmzpkwjjrdplmkeqethrux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kahluygjebwqxhjlafjmphjglngmnzba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"raumihulckqsdzsmlhksvmsenuuxmmeg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjdprmfvglbwiaevnpmubvjvuilvlcef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uyqnulrcpwtpiyyulitdwrzohfagrbxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oputqsaxybvtbtjuepvqgimsaczupkdx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qohqtraepzxagkovqwtcxlifuuxjwzxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqxvidrwgainwytazhhxuottcbvschuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqrvjhhcungkywagmewzhmatajbuylqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfmxxxjrwfsepfwqqozpzvvoodjindvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eppxqqxrfclowpoltcvdnasgfuhcbcmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ttttgtqaosfzxboxytjvrpqxigcupvum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgtwtybvhbctbilvjebyutrpksaabbir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gklflfhtmoxntcaleqwhcnakwbkfpnxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dyjbhfquxwfyjjirhqnzjeueglqhembk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cdbflhpeugodmdpheubxnkwtdrkiblsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gskejrgtoiiauttkpujlklloozyweigp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ewwholnnbrtkrwzvffzrgrrxpcmwqcmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zidejwxqvfshxsndirshjnpwmvxsnyva","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jnbbqaejsigirfbifmkbebnliytzzjby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnygsmmneejzxkzvpmhmyccgbflsnyuj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sqlugftkbfnuulhajvgbqyznnaocfaeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzikzgjtjygwqidbcsrgapsbddqfffnk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"izmoosqzltxfjnujakcnbiuhfxvcpofd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnriymfwlxidauutctimfskpycjbnppw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nggbpdmqhnukdbaahvyqekrbaumscnjl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twkhskcjjmscfzprggvohdxonclmutrl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pamvejlgqfzrpjrtbqwmlxelyqmdyrdr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqubsbtmnoeztkhucnuslgwrefoylsrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fdleqiepttlqvssosheqtqhubiinztxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynpvhlzwosxjlsiotodbsgtsdtwahcro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkujqcfpmbikgezmtvnhbwhcmgqcmmuv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jfncfddoilvdkamqkinnooatunyimgod","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amjtluqceocsjtopwrhcljgjzjxskgoh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omvdhohjzxuucmkomklnesmxpazrquuj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abhqjrmvskxkiacbbwpgthjxdlpcckay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"caxjtgfuixtuwivruxmlzhheiobxzwhq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqfnwqiwwmffhwnwbuorxuccdrtvnxpn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"anyzslgsupsgxfeilipwiyzauopwlkub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kdlbbrbflurvykldlunvfhxpifknigce","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgbmgybppvrcxhrttrbthdlvnvjnpwlh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xybagyoizyxdrxsfnpklevtcakjftnnj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eycybflwzyfekpnrlclkiabgbqwjanhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnwdyjypqctlirzczsqricimxbmrghbx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbsthdhzpiatvbckrcjexnaxqtuncthf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfooqlhojujgpxokumikyjvljnsculia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670490,"databaseName":"models_schema","ddl":"CREATE TABLE `qmzbocxykzmpczrsllqbenafxqafrzhk` (\n `obyqbmmxzgdksptheqzrnsepjsmjkuub` int NOT NULL,\n `ncacbnyyiemmynnibruuxegttbdymhdk` int DEFAULT NULL,\n `fyuktvqvvbuzwmehoefhqcgmmojhkgyq` int DEFAULT NULL,\n `olgvycoblxdslohxfgvdlwucbqdlrypw` int DEFAULT NULL,\n `bgujrruzrqopnsekezgjhaiyduxtvnvd` int DEFAULT NULL,\n `ostvvwbeljybdhkmcytxllfudkrkxsxc` int DEFAULT NULL,\n `kqmetnxgevvhikpkuxfjvlnmizbcrhxc` int DEFAULT NULL,\n `eefltqqabwhlhjikuzuhmeezknqoskpt` int DEFAULT NULL,\n `eoqnjnxygkicyoodixpmvlxlrmvplpvj` int DEFAULT NULL,\n `ocojkuotdjpdzjdetoecyydshmqricwa` int DEFAULT NULL,\n `awyhvoextbevbcotohrktuxkeqjfnnjp` int DEFAULT NULL,\n `yshyxtsjiempzgoowvoxfxyiwfbvtoom` int DEFAULT NULL,\n `huokdkplrsrzjusdkhjlnsxbrycjlhro` int DEFAULT NULL,\n `egbtqnhputymhjdpamaxowxhygueomdl` int DEFAULT NULL,\n `sursgloqbwyujrwuljglpqkuucndxabi` int DEFAULT NULL,\n `cbtxlkfeawrujgfbowtjuuszbyqbheow` int DEFAULT NULL,\n `mwsijojsqzevyhzelrbcdnkxbcvbsahv` int DEFAULT NULL,\n `leidxwbxtbinnpnawbpkaqxyciqrnosa` int DEFAULT NULL,\n `flepqnmendtwfuptyixmpagtnlzigjgk` int DEFAULT NULL,\n `patchrcxupdiunungdvvogktpbvpirxl` int DEFAULT NULL,\n `ipimwtvbmyruhmwmmlmeheveywtasodj` int DEFAULT NULL,\n `fpbavgavmbpqdzmjtlhwxsswlpvcfzoa` int DEFAULT NULL,\n `sacdhttzscsdzelnmoeixslvuitdngjg` int DEFAULT NULL,\n `cguyfrsmoaquulkpmdquhmbdkfqxiuec` int DEFAULT NULL,\n `agobencxqzmnyhttaktbiykqvwnokepf` int DEFAULT NULL,\n `fujzpqadgqtkxknufqkeuodetokduhtn` int DEFAULT NULL,\n `ftoqkxeqnzxcrkncezitglvwqlguyhgb` int DEFAULT NULL,\n `etchdyqwdjacousrzqpqumzrpzmkqmwc` int DEFAULT NULL,\n `kjmwqrslldgikukxuxsuyhxaybxeifkd` int DEFAULT NULL,\n `nfxsfbxapjdjcgmmgofowkanpmbjothd` int DEFAULT NULL,\n `lhzlrmvuqbdkayjipnienvtjykewcfmc` int DEFAULT NULL,\n `bcictigmnlsusucgwhwzqmnrlyqotsyd` int DEFAULT NULL,\n `gsonciasmsuvteaqmcskvypsxqmxazje` int DEFAULT NULL,\n `mpwwxyvbkwrodlrpxkexdpujakhtmavh` int DEFAULT NULL,\n `xnccjtrqqsaaszzsuoowophveemmlghj` int DEFAULT NULL,\n `dergasstbafilasldwqqzydhcpmrhfsn` int DEFAULT NULL,\n `mktsvbkevxbrtrkoarwdqabvhzioisoz` int DEFAULT NULL,\n `sieuyltogplgumgbeqaqyhtmpiuptsku` int DEFAULT NULL,\n `foeaqddzgglteguuxetbhaqhwbzqypxw` int DEFAULT NULL,\n `ddmdijemytwjfjywkrdikbnumnkuudzf` int DEFAULT NULL,\n `ooxlgntsydjthjctnmstkojlxlbvgoyh` int DEFAULT NULL,\n `rxzfmfwhmzdkihgacwvcmugqurqcauyt` int DEFAULT NULL,\n `qgukdgoseelhccpglwrlxpuljzslotwe` int DEFAULT NULL,\n `vhebubsnyxidydxhcaiyrybvpeqjszxn` int DEFAULT NULL,\n `pyyhioqcrsdptdwuigicgrhqxdajsugr` int DEFAULT NULL,\n `fghonhkvmsowalutwwxwwmihjzkencdh` int DEFAULT NULL,\n `ydozwpdxhasfedzjyxuuqnbsmispoqdk` int DEFAULT NULL,\n `homkhoudgbccjacdvmjbhznipsbehrco` int DEFAULT NULL,\n `vzujadjghbxkuvczsjjkbtksmxnchqri` int DEFAULT NULL,\n `rioxwczrdhxarjraxheqrovacsjkvlfz` int DEFAULT NULL,\n `jnmbdufjhlxzgbiackgfgilbuuzewljd` int DEFAULT NULL,\n `ikbyiipainwbuekrtrxbxbnxvawsgany` int DEFAULT NULL,\n `drboqpgryvjxepqarqvktudgpuzuivxk` int DEFAULT NULL,\n `hqmrgnjuruemkhucvvftyqyqjtqkezcn` int DEFAULT NULL,\n `zushqvfkxjiczfuhfxcougnznziafsfv` int DEFAULT NULL,\n `ajiukktmqxhttkdakcfltbzkfuxsynlv` int DEFAULT NULL,\n `utqgvijpxqxfvsdmbryltshfujjfjfys` int DEFAULT NULL,\n `mmpyhtwipxskgkkemjowssgdzfdgghrm` int DEFAULT NULL,\n `zdqsqztlikbfvadxjchneseggoyjeqiv` int DEFAULT NULL,\n `idajvtupdcosioigmenpjrmjsbnwhbzk` int DEFAULT NULL,\n `veihtwertufswlkeklcdxwrsbpkldkcv` int DEFAULT NULL,\n `cgeeoflihbvyorazmbqladmclipfvwjs` int DEFAULT NULL,\n `ogupwlqlqcanbrqzrvbzwvaxxdisqimj` int DEFAULT NULL,\n `eglaotawposlksstsnxpmgweyyvwtzkz` int DEFAULT NULL,\n `vclbqcwpxnixnitdlhthrodkevwwnibb` int DEFAULT NULL,\n `wgdxxfdeiqsfuwxsbngxhaosfuztbylt` int DEFAULT NULL,\n `vgdsbdqxtudbswnnledisvrouthpttcu` int DEFAULT NULL,\n `znhxfnfqobfxxcpeledntpajwxxxjnyx` int DEFAULT NULL,\n `ryjqbmqalznusflzmbbtueauwbtyefmh` int DEFAULT NULL,\n `iyclsurwuxmqzbfciczsbpsbsdlfqhzb` int DEFAULT NULL,\n `pkhktqqnhxndlcbafepyvojjiubmnxdy` int DEFAULT NULL,\n `pcrxltoxskgefztirpymszsrvowplfkv` int DEFAULT NULL,\n `pepifaxnokxsiqjhhgiyowixkxzfkken` int DEFAULT NULL,\n `tijnugswkakdvnbvwhglxtiavbmafhbi` int DEFAULT NULL,\n `yjjkaswelqeyjgnxyboiztxgowtguivr` int DEFAULT NULL,\n `nfvyyhisvhgzwqemlvgxnbmpxtqfyckq` int DEFAULT NULL,\n `qjqgiydrxlxrqvysivpmmqdllpygfipv` int DEFAULT NULL,\n `cisajuyknaslysjzgppoeuohpslyfmhn` int DEFAULT NULL,\n `ojoryykmdljtbvhfytkfxazakaehkxlk` int DEFAULT NULL,\n `ijoewqoyfiecsczrlnonfnalemngoloh` int DEFAULT NULL,\n `htkijcubtrsgbaxbrktiprtjpvbsyoza` int DEFAULT NULL,\n `ziovhdbpjjjvsjcpjltyyvdixgbjjmiy` int DEFAULT NULL,\n `povmkkabmdzstcikdqunoicumysplpgw` int DEFAULT NULL,\n `qxtwmyzfetfuplhrzuyvopwiccaqvcdu` int DEFAULT NULL,\n `xnqclguebrwijuhyrrsqhahdsqrledtb` int DEFAULT NULL,\n `qwolxpktbfwitlmlwdssmmyhhksfdosw` int DEFAULT NULL,\n `astcqyheutjlnncgxgwxektougnxbupc` int DEFAULT NULL,\n `oabmqssxywlbmzdbrjkgabyvnzgqqmeu` int DEFAULT NULL,\n `ngvjyxgmblphgehykzcjzlusgfaligdq` int DEFAULT NULL,\n `vsxgmmqhyodagzdtjivxmpwgrelknfrx` int DEFAULT NULL,\n `cmofmugxnvnjbvcbwlrntitfhkrxhsjy` int DEFAULT NULL,\n `ggxjzlbcnmwnculcnhcverhvjbqfhfph` int DEFAULT NULL,\n `jvfdwmhcfqpbhhcauaxtmwqlqqeineda` int DEFAULT NULL,\n `bpbesjyvsidsmtcotstjyalaxsbqiftx` int DEFAULT NULL,\n `ilboqidtgqunbnfaitwlmzkmmkdtmkib` int DEFAULT NULL,\n `wwxobkewqnytaibspsaelrvjbmyffhny` int DEFAULT NULL,\n `uzsgrpaymwvsqfhmttrbluadtzaxzcxc` int DEFAULT NULL,\n `hugipnslwyuuijldvktvymgfwtlgfsxl` int DEFAULT NULL,\n `jfjgebxuoayyhvwclqmbsipgzexmylgr` int DEFAULT NULL,\n `jvcwxdoahorlgkhzizefsdkhtcwrncct` int DEFAULT NULL,\n PRIMARY KEY (`obyqbmmxzgdksptheqzrnsepjsmjkuub`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"qmzbocxykzmpczrsllqbenafxqafrzhk\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["obyqbmmxzgdksptheqzrnsepjsmjkuub"],"columns":[{"name":"obyqbmmxzgdksptheqzrnsepjsmjkuub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ncacbnyyiemmynnibruuxegttbdymhdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fyuktvqvvbuzwmehoefhqcgmmojhkgyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"olgvycoblxdslohxfgvdlwucbqdlrypw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bgujrruzrqopnsekezgjhaiyduxtvnvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ostvvwbeljybdhkmcytxllfudkrkxsxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqmetnxgevvhikpkuxfjvlnmizbcrhxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eefltqqabwhlhjikuzuhmeezknqoskpt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eoqnjnxygkicyoodixpmvlxlrmvplpvj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocojkuotdjpdzjdetoecyydshmqricwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awyhvoextbevbcotohrktuxkeqjfnnjp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yshyxtsjiempzgoowvoxfxyiwfbvtoom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"huokdkplrsrzjusdkhjlnsxbrycjlhro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"egbtqnhputymhjdpamaxowxhygueomdl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sursgloqbwyujrwuljglpqkuucndxabi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbtxlkfeawrujgfbowtjuuszbyqbheow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwsijojsqzevyhzelrbcdnkxbcvbsahv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"leidxwbxtbinnpnawbpkaqxyciqrnosa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flepqnmendtwfuptyixmpagtnlzigjgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"patchrcxupdiunungdvvogktpbvpirxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ipimwtvbmyruhmwmmlmeheveywtasodj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpbavgavmbpqdzmjtlhwxsswlpvcfzoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sacdhttzscsdzelnmoeixslvuitdngjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cguyfrsmoaquulkpmdquhmbdkfqxiuec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agobencxqzmnyhttaktbiykqvwnokepf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fujzpqadgqtkxknufqkeuodetokduhtn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftoqkxeqnzxcrkncezitglvwqlguyhgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"etchdyqwdjacousrzqpqumzrpzmkqmwc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjmwqrslldgikukxuxsuyhxaybxeifkd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfxsfbxapjdjcgmmgofowkanpmbjothd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhzlrmvuqbdkayjipnienvtjykewcfmc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bcictigmnlsusucgwhwzqmnrlyqotsyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsonciasmsuvteaqmcskvypsxqmxazje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpwwxyvbkwrodlrpxkexdpujakhtmavh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnccjtrqqsaaszzsuoowophveemmlghj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dergasstbafilasldwqqzydhcpmrhfsn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mktsvbkevxbrtrkoarwdqabvhzioisoz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sieuyltogplgumgbeqaqyhtmpiuptsku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"foeaqddzgglteguuxetbhaqhwbzqypxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddmdijemytwjfjywkrdikbnumnkuudzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ooxlgntsydjthjctnmstkojlxlbvgoyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxzfmfwhmzdkihgacwvcmugqurqcauyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgukdgoseelhccpglwrlxpuljzslotwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhebubsnyxidydxhcaiyrybvpeqjszxn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pyyhioqcrsdptdwuigicgrhqxdajsugr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fghonhkvmsowalutwwxwwmihjzkencdh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydozwpdxhasfedzjyxuuqnbsmispoqdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"homkhoudgbccjacdvmjbhznipsbehrco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzujadjghbxkuvczsjjkbtksmxnchqri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rioxwczrdhxarjraxheqrovacsjkvlfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jnmbdufjhlxzgbiackgfgilbuuzewljd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikbyiipainwbuekrtrxbxbnxvawsgany","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drboqpgryvjxepqarqvktudgpuzuivxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqmrgnjuruemkhucvvftyqyqjtqkezcn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zushqvfkxjiczfuhfxcougnznziafsfv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajiukktmqxhttkdakcfltbzkfuxsynlv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utqgvijpxqxfvsdmbryltshfujjfjfys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmpyhtwipxskgkkemjowssgdzfdgghrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdqsqztlikbfvadxjchneseggoyjeqiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idajvtupdcosioigmenpjrmjsbnwhbzk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"veihtwertufswlkeklcdxwrsbpkldkcv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgeeoflihbvyorazmbqladmclipfvwjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ogupwlqlqcanbrqzrvbzwvaxxdisqimj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eglaotawposlksstsnxpmgweyyvwtzkz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vclbqcwpxnixnitdlhthrodkevwwnibb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgdxxfdeiqsfuwxsbngxhaosfuztbylt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgdsbdqxtudbswnnledisvrouthpttcu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znhxfnfqobfxxcpeledntpajwxxxjnyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ryjqbmqalznusflzmbbtueauwbtyefmh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iyclsurwuxmqzbfciczsbpsbsdlfqhzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkhktqqnhxndlcbafepyvojjiubmnxdy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pcrxltoxskgefztirpymszsrvowplfkv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pepifaxnokxsiqjhhgiyowixkxzfkken","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tijnugswkakdvnbvwhglxtiavbmafhbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjjkaswelqeyjgnxyboiztxgowtguivr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfvyyhisvhgzwqemlvgxnbmpxtqfyckq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjqgiydrxlxrqvysivpmmqdllpygfipv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cisajuyknaslysjzgppoeuohpslyfmhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojoryykmdljtbvhfytkfxazakaehkxlk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijoewqoyfiecsczrlnonfnalemngoloh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htkijcubtrsgbaxbrktiprtjpvbsyoza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ziovhdbpjjjvsjcpjltyyvdixgbjjmiy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"povmkkabmdzstcikdqunoicumysplpgw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxtwmyzfetfuplhrzuyvopwiccaqvcdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnqclguebrwijuhyrrsqhahdsqrledtb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwolxpktbfwitlmlwdssmmyhhksfdosw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"astcqyheutjlnncgxgwxektougnxbupc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oabmqssxywlbmzdbrjkgabyvnzgqqmeu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ngvjyxgmblphgehykzcjzlusgfaligdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsxgmmqhyodagzdtjivxmpwgrelknfrx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmofmugxnvnjbvcbwlrntitfhkrxhsjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggxjzlbcnmwnculcnhcverhvjbqfhfph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvfdwmhcfqpbhhcauaxtmwqlqqeineda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bpbesjyvsidsmtcotstjyalaxsbqiftx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ilboqidtgqunbnfaitwlmzkmmkdtmkib","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwxobkewqnytaibspsaelrvjbmyffhny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzsgrpaymwvsqfhmttrbluadtzaxzcxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hugipnslwyuuijldvktvymgfwtlgfsxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jfjgebxuoayyhvwclqmbsipgzexmylgr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvcwxdoahorlgkhzizefsdkhtcwrncct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670537,"databaseName":"models_schema","ddl":"CREATE TABLE `qnopovgcxspwoeghyuonqnbbnrvjvhow` (\n `jcbvrsbibngtprlgapbmtewsfdpdcfcr` int NOT NULL,\n `ynpkavqwkbrrrenafquwfdwmjaappmje` int DEFAULT NULL,\n `sekgumcvevflgitqqzaedaqanrywvjqh` int DEFAULT NULL,\n `ewjzeztxxtelknryfrnbaoeavxkmwwiu` int DEFAULT NULL,\n `qfrflzwnheqrerbimjvruqgxzhaifcaf` int DEFAULT NULL,\n `zqowksacrqkatpwvlhbgacxkwcdisses` int DEFAULT NULL,\n `htteuzrntgnmicgongnvelrywmobyghu` int DEFAULT NULL,\n `bjhvmlhylbpxpccrsheapvgfeidhjidb` int DEFAULT NULL,\n `ntigqaidleibqoqfveafkjqmwecjepwv` int DEFAULT NULL,\n `cwnradvozubssuugzhgqaibddzhrucgh` int DEFAULT NULL,\n `skbcdixwxoeaxrblkkvrvsalccazmepd` int DEFAULT NULL,\n `zubdhkedzqyoqmvkpvqvxqzvejyooutd` int DEFAULT NULL,\n `gchuvrhvoywchivmortbhsplkaoannxl` int DEFAULT NULL,\n `lixzrwmgyefrcwmztinfqiqadhcqxnjq` int DEFAULT NULL,\n `xvwzlrrpkqihnloysyhcqpnpxbhvpvff` int DEFAULT NULL,\n `kfiubzpaflmdacunkhxxlpznssnrhjao` int DEFAULT NULL,\n `rrzumzulvvnykzoahlexlnywitaoiiqk` int DEFAULT NULL,\n `vzpgjycipcofexoeqfpapvorsxnhsxfq` int DEFAULT NULL,\n `mluwatuanuqsqkzryjajjcivxeuczgde` int DEFAULT NULL,\n `qbjjxoonhreslwsvhlmfllsfnndbrnqx` int DEFAULT NULL,\n `dfscocaypulnxxxspcmwoijyugidwtqw` int DEFAULT NULL,\n `hethjafkaczcazwwqqfhlcfafeutouth` int DEFAULT NULL,\n `xscvmidngfhssnfsuaozhyyzmeqtjyoi` int DEFAULT NULL,\n `ciituryojdsyoeyijjmzvhyvafyermil` int DEFAULT NULL,\n `rfagfhdwdollwsjimedyootugujhxcbc` int DEFAULT NULL,\n `ruqbhulocekfkruntnhgkuwnhvsbgiji` int DEFAULT NULL,\n `pqfyveqzrqstejosbjasvrqlguzfgkhr` int DEFAULT NULL,\n `cmzhyrhvwpkaxduzqrlfdeuolawraogm` int DEFAULT NULL,\n `txnfcvyrkkjneibhivtzheljyunnklue` int DEFAULT NULL,\n `itfbxsshishdippafmraoswteruoancb` int DEFAULT NULL,\n `sbzgkonlpyfzyplinexelorjtaxnpcdt` int DEFAULT NULL,\n `pzmpoywrhyfuawevmorzcflwjgagxcjv` int DEFAULT NULL,\n `grlsbqsfoefuqyrxqcbrxptulcpeqhvf` int DEFAULT NULL,\n `dplvhkntxrpwxemonkiwepepeuvqihov` int DEFAULT NULL,\n `ctahsfnvnqvlwwduqusmtdkxsgacecze` int DEFAULT NULL,\n `thwvtwrogwvsfzhrplnssdfnlweahzvp` int DEFAULT NULL,\n `dhanuqjvmztquvowfoodssadposqkvdf` int DEFAULT NULL,\n `fcdbaysuxtwtrxskauxpjlgfzafuhhpn` int DEFAULT NULL,\n `oirezeljqvrthnvykwwskbgliynotcrp` int DEFAULT NULL,\n `qxwhhzfzlynppyspjdllxlyrrvepbejh` int DEFAULT NULL,\n `guptxdxwuhzkhewxxhzqdgqdllwbfrwe` int DEFAULT NULL,\n `bzevpjrlzisgslqytqwcpemnuuikltlv` int DEFAULT NULL,\n `memqrxaralqvxyuldtuklulshysormtn` int DEFAULT NULL,\n `axjmzfmopudgbluyccsnzfexfejpggbi` int DEFAULT NULL,\n `syuwogdtmnjizwoulgxmbnhqqbhqncer` int DEFAULT NULL,\n `bqsvievqmfyfevorhtnyohvmpscqzdqa` int DEFAULT NULL,\n `ihqztszhcmjdqkvsrmuqgeqoxksazdsl` int DEFAULT NULL,\n `egbrjqnwtksfkrkyzedccjmrabtmrtdd` int DEFAULT NULL,\n `gvrwahrjnjvswysmcsvhygptlfrawtnc` int DEFAULT NULL,\n `pjklhegdmcgmmzjqtpcvqkqilwigxtqc` int DEFAULT NULL,\n `wwhtxbfftxzcbhlajgdijgnfmexebszh` int DEFAULT NULL,\n `gkwdghhgrbtnwxpxpcoqysanvrmjdags` int DEFAULT NULL,\n `iexbvbqpffqmhaifjrqafzmsfybxnjqo` int DEFAULT NULL,\n `ixzrvfdbwehishxotadoxyeiaxptmhgb` int DEFAULT NULL,\n `wjgmpounvymajgyqnblawdijszqgkycp` int DEFAULT NULL,\n `rurdmmtewsakjnqkaneitsrfdrgsjmek` int DEFAULT NULL,\n `gugpabfiykgsxuxxsdlgcwjnijzmbphb` int DEFAULT NULL,\n `vhxygurrsnbjhoxugtpmgjxoikllecqv` int DEFAULT NULL,\n `lfhlseeihskquuathuuzkyepqrvgacwd` int DEFAULT NULL,\n `mjfbpfemowztsgppcjjgocapcvxddtgn` int DEFAULT NULL,\n `dbjvidemtladrkxqjwmoxmhhkybglvza` int DEFAULT NULL,\n `yfzbbqvxqoxvpfqspryidcedlwaoytyh` int DEFAULT NULL,\n `ozooofvwyiiqqmeedsqhdnbqvjpcrmie` int DEFAULT NULL,\n `hrphkzspuygxrkahcoprmfsstrtllyzv` int DEFAULT NULL,\n `txqddmsmethyjaynuzghcgopuoajonxy` int DEFAULT NULL,\n `zhfmmzzuupaxnmjesavvnbduzaifpmwt` int DEFAULT NULL,\n `crpvvvdevhfdrsesqkqeguchsohmjizh` int DEFAULT NULL,\n `mlfukyvinrbthkugvmgubwqhwmuucnth` int DEFAULT NULL,\n `rnqkabuxzpbxsrcxpfwicgjznpsxpipq` int DEFAULT NULL,\n `zktmjhheemajchqjnjehtxiwtnewdfkl` int DEFAULT NULL,\n `wdkzvqqjcbkyueonqwrucchmxayzfvak` int DEFAULT NULL,\n `emmccyzzyrppkkmezaecuvzvwereniun` int DEFAULT NULL,\n `wqqhtycyqacleeuddikdyewazvsqucol` int DEFAULT NULL,\n `nrbtvjbphmzqblngiwwtlfouicqvevdq` int DEFAULT NULL,\n `uelnxhdbduwtzwmnqvdwztcttyhhbejy` int DEFAULT NULL,\n `lanhgfznyavfzvpzaldbwtgwflxfrmhg` int DEFAULT NULL,\n `egyfyiaufxedglgbncskfviqlamqyfkd` int DEFAULT NULL,\n `eleptcdxiukpdevpufwraectfzstgxdm` int DEFAULT NULL,\n `rlnynqniolaildbatmyozmxdnssievks` int DEFAULT NULL,\n `bdunakasjwjadosxzdamhdqlifdikayn` int DEFAULT NULL,\n `mfsrfvwcfccfxykbshwmmtwicsraxmae` int DEFAULT NULL,\n `sowvouuxopfipjvfuvjnaaetiwcqkytd` int DEFAULT NULL,\n `xsintefyuqicrxbihxxgomffdmoqgvuv` int DEFAULT NULL,\n `qhomvmmasxmzwwuuopehxfdlgitilkis` int DEFAULT NULL,\n `exwysgyyjmebifiwsftuaiyrahqewfqh` int DEFAULT NULL,\n `vpmeqkagsnsfcuqopkhpjwbrjqblmbiv` int DEFAULT NULL,\n `khbalapphcxrfceukvlbtrwdgonfdtaw` int DEFAULT NULL,\n `qihcnlbkbclcsdptpwgzgyqlaojoiiko` int DEFAULT NULL,\n `gfqxngkqkisoyxujiugsdglnyhoygbiq` int DEFAULT NULL,\n `mtbmkczsliztgyuyvfrqbnhtblaloofb` int DEFAULT NULL,\n `kqexuptcfijmpgdxagbaxgaegwidhsmo` int DEFAULT NULL,\n `ayaezreycioihcbodxiwiecvybtpyubs` int DEFAULT NULL,\n `qeoxmchcplztnlaeqaqjipmkwfviefki` int DEFAULT NULL,\n `ymxuwsfqecqvltsprdyloxhtlreljafn` int DEFAULT NULL,\n `zuysmjsmbvmzihuybfkorxqbjbnowdhs` int DEFAULT NULL,\n `njxrlwenjxccngpodxercnqjanhbeayo` int DEFAULT NULL,\n `bsryphwscwxbxxosibrzeliyikdnukzk` int DEFAULT NULL,\n `qbladjwzvoqngisrzwfqgteecvlunton` int DEFAULT NULL,\n `qwcjiefxclorcvjlxyvtkgoswkpygecf` int DEFAULT NULL,\n `byokkpadsqvaajvejpvzntgsyyrmkxow` int DEFAULT NULL,\n PRIMARY KEY (`jcbvrsbibngtprlgapbmtewsfdpdcfcr`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"qnopovgcxspwoeghyuonqnbbnrvjvhow\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["jcbvrsbibngtprlgapbmtewsfdpdcfcr"],"columns":[{"name":"jcbvrsbibngtprlgapbmtewsfdpdcfcr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ynpkavqwkbrrrenafquwfdwmjaappmje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sekgumcvevflgitqqzaedaqanrywvjqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ewjzeztxxtelknryfrnbaoeavxkmwwiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfrflzwnheqrerbimjvruqgxzhaifcaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqowksacrqkatpwvlhbgacxkwcdisses","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htteuzrntgnmicgongnvelrywmobyghu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjhvmlhylbpxpccrsheapvgfeidhjidb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntigqaidleibqoqfveafkjqmwecjepwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwnradvozubssuugzhgqaibddzhrucgh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skbcdixwxoeaxrblkkvrvsalccazmepd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zubdhkedzqyoqmvkpvqvxqzvejyooutd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gchuvrhvoywchivmortbhsplkaoannxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lixzrwmgyefrcwmztinfqiqadhcqxnjq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvwzlrrpkqihnloysyhcqpnpxbhvpvff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfiubzpaflmdacunkhxxlpznssnrhjao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrzumzulvvnykzoahlexlnywitaoiiqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzpgjycipcofexoeqfpapvorsxnhsxfq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mluwatuanuqsqkzryjajjcivxeuczgde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qbjjxoonhreslwsvhlmfllsfnndbrnqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dfscocaypulnxxxspcmwoijyugidwtqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hethjafkaczcazwwqqfhlcfafeutouth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xscvmidngfhssnfsuaozhyyzmeqtjyoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ciituryojdsyoeyijjmzvhyvafyermil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfagfhdwdollwsjimedyootugujhxcbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ruqbhulocekfkruntnhgkuwnhvsbgiji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqfyveqzrqstejosbjasvrqlguzfgkhr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmzhyrhvwpkaxduzqrlfdeuolawraogm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txnfcvyrkkjneibhivtzheljyunnklue","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itfbxsshishdippafmraoswteruoancb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbzgkonlpyfzyplinexelorjtaxnpcdt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pzmpoywrhyfuawevmorzcflwjgagxcjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grlsbqsfoefuqyrxqcbrxptulcpeqhvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dplvhkntxrpwxemonkiwepepeuvqihov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctahsfnvnqvlwwduqusmtdkxsgacecze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thwvtwrogwvsfzhrplnssdfnlweahzvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhanuqjvmztquvowfoodssadposqkvdf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcdbaysuxtwtrxskauxpjlgfzafuhhpn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oirezeljqvrthnvykwwskbgliynotcrp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxwhhzfzlynppyspjdllxlyrrvepbejh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"guptxdxwuhzkhewxxhzqdgqdllwbfrwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzevpjrlzisgslqytqwcpemnuuikltlv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"memqrxaralqvxyuldtuklulshysormtn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axjmzfmopudgbluyccsnzfexfejpggbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"syuwogdtmnjizwoulgxmbnhqqbhqncer","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqsvievqmfyfevorhtnyohvmpscqzdqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ihqztszhcmjdqkvsrmuqgeqoxksazdsl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"egbrjqnwtksfkrkyzedccjmrabtmrtdd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gvrwahrjnjvswysmcsvhygptlfrawtnc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjklhegdmcgmmzjqtpcvqkqilwigxtqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwhtxbfftxzcbhlajgdijgnfmexebszh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkwdghhgrbtnwxpxpcoqysanvrmjdags","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iexbvbqpffqmhaifjrqafzmsfybxnjqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixzrvfdbwehishxotadoxyeiaxptmhgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjgmpounvymajgyqnblawdijszqgkycp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rurdmmtewsakjnqkaneitsrfdrgsjmek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gugpabfiykgsxuxxsdlgcwjnijzmbphb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhxygurrsnbjhoxugtpmgjxoikllecqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfhlseeihskquuathuuzkyepqrvgacwd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjfbpfemowztsgppcjjgocapcvxddtgn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbjvidemtladrkxqjwmoxmhhkybglvza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfzbbqvxqoxvpfqspryidcedlwaoytyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozooofvwyiiqqmeedsqhdnbqvjpcrmie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hrphkzspuygxrkahcoprmfsstrtllyzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txqddmsmethyjaynuzghcgopuoajonxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhfmmzzuupaxnmjesavvnbduzaifpmwt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crpvvvdevhfdrsesqkqeguchsohmjizh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlfukyvinrbthkugvmgubwqhwmuucnth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnqkabuxzpbxsrcxpfwicgjznpsxpipq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zktmjhheemajchqjnjehtxiwtnewdfkl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wdkzvqqjcbkyueonqwrucchmxayzfvak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"emmccyzzyrppkkmezaecuvzvwereniun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqqhtycyqacleeuddikdyewazvsqucol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nrbtvjbphmzqblngiwwtlfouicqvevdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uelnxhdbduwtzwmnqvdwztcttyhhbejy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lanhgfznyavfzvpzaldbwtgwflxfrmhg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"egyfyiaufxedglgbncskfviqlamqyfkd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eleptcdxiukpdevpufwraectfzstgxdm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlnynqniolaildbatmyozmxdnssievks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdunakasjwjadosxzdamhdqlifdikayn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfsrfvwcfccfxykbshwmmtwicsraxmae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sowvouuxopfipjvfuvjnaaetiwcqkytd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsintefyuqicrxbihxxgomffdmoqgvuv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhomvmmasxmzwwuuopehxfdlgitilkis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exwysgyyjmebifiwsftuaiyrahqewfqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpmeqkagsnsfcuqopkhpjwbrjqblmbiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khbalapphcxrfceukvlbtrwdgonfdtaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qihcnlbkbclcsdptpwgzgyqlaojoiiko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfqxngkqkisoyxujiugsdglnyhoygbiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtbmkczsliztgyuyvfrqbnhtblaloofb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqexuptcfijmpgdxagbaxgaegwidhsmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ayaezreycioihcbodxiwiecvybtpyubs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qeoxmchcplztnlaeqaqjipmkwfviefki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ymxuwsfqecqvltsprdyloxhtlreljafn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zuysmjsmbvmzihuybfkorxqbjbnowdhs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njxrlwenjxccngpodxercnqjanhbeayo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bsryphwscwxbxxosibrzeliyikdnukzk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qbladjwzvoqngisrzwfqgteecvlunton","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwcjiefxclorcvjlxyvtkgoswkpygecf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"byokkpadsqvaajvejpvzntgsyyrmkxow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670579,"databaseName":"models_schema","ddl":"CREATE TABLE `qsirphfmlpnkopaqiayapzezhyeudrzs` (\n `aqjfiytufyxdwmtirlkezrcrrhmeafcg` int NOT NULL,\n `lusoqpohibanqhaysfqjpymckzqziapg` int DEFAULT NULL,\n `vstvbooumksgqojwvzkjdelbdgiumuus` int DEFAULT NULL,\n `wgzxsecowtvuerwvdrsheunoahrsvdld` int DEFAULT NULL,\n `nozpogduskuldqhhfnpkpnowxtmxcbxk` int DEFAULT NULL,\n `oyeutcnbcbjbosemepdjuyqnihidsvim` int DEFAULT NULL,\n `fsahtemevmcnrwzvpbqtjncteoebbdpa` int DEFAULT NULL,\n `hfcwzcmkqayhwqmetjioehuocjqiizsf` int DEFAULT NULL,\n `lysddankqamybwscqncduruxfhzwmzjy` int DEFAULT NULL,\n `rzsyyijzbmxkztspcthtpmxrthxmtycy` int DEFAULT NULL,\n `cuidakxtxstxxxwgbgxcturzwxoyggbx` int DEFAULT NULL,\n `gffnlgfkutfnztrcipqoivjcsxiedgsw` int DEFAULT NULL,\n `bdilqopofxnsrugyniyzuwcstrnufcxq` int DEFAULT NULL,\n `cdvcgrbehjvoirdetucrnotwpiprlgjv` int DEFAULT NULL,\n `msufdizowdxxegifdigmukahiuxydvry` int DEFAULT NULL,\n `smalpidlmfavjzvnjxoobhbaexhzzzyx` int DEFAULT NULL,\n `etsxnomwgkyeohmrkrxuvqkgimzvroup` int DEFAULT NULL,\n `ljvyoqdtjzlorpwsgpplvpipqwdftyoy` int DEFAULT NULL,\n `qsvzyskcibjrqkeatkwbwurlgyaclumv` int DEFAULT NULL,\n `ajilolgkvzfxdzrjysywggvzfdlqtjle` int DEFAULT NULL,\n `qrxygfsdoevfndpeeikwgillbgoygjnv` int DEFAULT NULL,\n `ybmhacaqhyprwrqsqjwyysiydmxxsadc` int DEFAULT NULL,\n `fjrdeijlxynrqxbeoivioilvfuqsmwls` int DEFAULT NULL,\n `cmrpokbskiczofdzfcwoinsbekpawknx` int DEFAULT NULL,\n `igbmvdrkcbzyjvkrlemwbqeqrwdbtvxj` int DEFAULT NULL,\n `cqxihtkdxesmpeqigumxfociqjixyhwg` int DEFAULT NULL,\n `wbiuvdemtuozfzdaqwkfkkdctdpziykg` int DEFAULT NULL,\n `jjhmixcnbqeeslktgmdxyjywxyrkjzys` int DEFAULT NULL,\n `huxegwkeqwuzvyavhwysbnqzqdfhxvnm` int DEFAULT NULL,\n `uuxitqnhafihbiiokiaptdjuqrtdeyzn` int DEFAULT NULL,\n `kytgbydxhzwzithgngjhxtfbcdvgtozv` int DEFAULT NULL,\n `rpidfcqmzjdnyptwmazjnjsezjfmvcwo` int DEFAULT NULL,\n `hqyohdkktiodpoeslcjixemgaivnweqt` int DEFAULT NULL,\n `oxlhpwutuawffbcrnrxzcpdhyhrtkfsc` int DEFAULT NULL,\n `lzuouqnrxujtkgtjxfjvoirlpewbioza` int DEFAULT NULL,\n `zidboiaqgflzxgzcxzapjvvtqcknjgwh` int DEFAULT NULL,\n `gnqkywwmboqypehozgrfazdhkrggjhsf` int DEFAULT NULL,\n `duljujgqptfvhpshwgmthnzcifnlkccb` int DEFAULT NULL,\n `curcgqhdztfqwcbfpajdwdylvcmfukof` int DEFAULT NULL,\n `jfenivacwzajbpfymqyzgrqntecmwlad` int DEFAULT NULL,\n `zckuyeexmdjaoezojqqniphlcytgvqay` int DEFAULT NULL,\n `yhfrlwncwqtcabigwojwdtlpaomeswcz` int DEFAULT NULL,\n `uxxjcverznhdpdhmdhdixtxgvdzzquzc` int DEFAULT NULL,\n `zhsuytflhrakgobqbbfcjepxsdalymmn` int DEFAULT NULL,\n `fcdrrmupwsishtunirtnllefdozpjadq` int DEFAULT NULL,\n `tljwzwwagqbqigqomggccuntwmlejxno` int DEFAULT NULL,\n `psmgoqbptlnmzhxnkosbubckitinkklm` int DEFAULT NULL,\n `thzxqukszgjkfjjrzetxfbnpowwhsarn` int DEFAULT NULL,\n `ryqxcuqmpbyerwarmvuwlpdsngsnchdl` int DEFAULT NULL,\n `ssibvplvweelrcjatiwivtjrujkhlxat` int DEFAULT NULL,\n `uhjpnfwnhwcdtazusswkwwxtceswudpn` int DEFAULT NULL,\n `eeesfrookzpnxhguxyxsmqnoeutfgeqc` int DEFAULT NULL,\n `uiaolbxtvoggfsmtfqvwgtgdyooepfnh` int DEFAULT NULL,\n `zghrndjtdmjeejpyqqjliuhexhbabfwi` int DEFAULT NULL,\n `mdgyipaszpcyxmpmqbbcbdwxtlazupbu` int DEFAULT NULL,\n `bnfyfyhstamohkkzxozuxsvaguiridgs` int DEFAULT NULL,\n `aqfkoxkwimoicxqbuwjnmvqsvorqyitm` int DEFAULT NULL,\n `lvvbfzhaufqrllcflwcqkoqwkwalbyie` int DEFAULT NULL,\n `xsxgzizzfkemwddfpmkeqzselgsxmnzc` int DEFAULT NULL,\n `ygpzhsxrjaisyorbnkospsifohqgfrwz` int DEFAULT NULL,\n `wrbvjlriueqheuyddkvvzmxipckngvas` int DEFAULT NULL,\n `sdjcsykxyeystidoatzhngdlbcoitqcv` int DEFAULT NULL,\n `rqmsvqcfxeezqucmaqpddjxhvcrjhgof` int DEFAULT NULL,\n `sfzzkvmxojxbnbnpghjedryhhhbhqjpr` int DEFAULT NULL,\n `mfboevuaddafseoenxabsfvojyruyaqe` int DEFAULT NULL,\n `ggnmtgvbrmdxcjbihbefrsvzcaitrlld` int DEFAULT NULL,\n `iqxjzqiadcosylozmhjrcjlhzdopffps` int DEFAULT NULL,\n `dkgbgocgxslldfzcqvhfvogemveuypfo` int DEFAULT NULL,\n `wulgnssgmixeclgxznmalppgduxqtbxb` int DEFAULT NULL,\n `otlfzuhmjjmshyftuxejcsgfvuttvjns` int DEFAULT NULL,\n `zhdawvunyxhrhpbyztpzllkroeeakzuy` int DEFAULT NULL,\n `mvoyhjeaquxfhwcjikaadhjpvpgmmzgs` int DEFAULT NULL,\n `cilrokxeyfsgorwldblvptqxcrtevhqg` int DEFAULT NULL,\n `nchkcvqvoxkbiewznhhgesszxpnofnat` int DEFAULT NULL,\n `mpozdhfphcqmfmuzqanzmmpbbnvxnkek` int DEFAULT NULL,\n `srlsmqocftkkzakzqojdgngueixnavyq` int DEFAULT NULL,\n `ikytbyvqcnzueduaqmuqwenqclngyzml` int DEFAULT NULL,\n `nyfoaehzghigomufhncunpasgbxmsoaj` int DEFAULT NULL,\n `mkukuloelkqvpinzkneicrkmduqoypdi` int DEFAULT NULL,\n `kcidaryggjhzloxkvfvjufzivhekgwkg` int DEFAULT NULL,\n `ooypobabtifgvedoawdqpjwpytwvsvep` int DEFAULT NULL,\n `xvcyhgddnqxyhhbyhvwzbsxahqfzpkfo` int DEFAULT NULL,\n `oyvkcouunkkgxywfcbeuxfgpjqkeeohj` int DEFAULT NULL,\n `zhiemakenhuifafbeeklwkrpaqirkzxu` int DEFAULT NULL,\n `dkgsskyekiyaqisoqsjwojxhjrijgqlr` int DEFAULT NULL,\n `hodnhyvgmmyeuwlknegvmlqnwuejmmcz` int DEFAULT NULL,\n `nwwiitnkrjzlyaswlrguusllyhvwhkxw` int DEFAULT NULL,\n `edswqsesiqrgqeevvxgbszdnmxsgolyt` int DEFAULT NULL,\n `ffdqfercversermasaiaymduovwmnnnk` int DEFAULT NULL,\n `zhrafqlbyutbexpsdhkfhbxarqxtqptq` int DEFAULT NULL,\n `qyofsfxtkhqwebvllsxhuapgglrggwpy` int DEFAULT NULL,\n `turwoloirtzuplwismtyzbptxhzirwbi` int DEFAULT NULL,\n `txllijsqiednfmcjltjtybjbpvgesyum` int DEFAULT NULL,\n `gcoekmxnpidhywujupsbtdgosjabxjqp` int DEFAULT NULL,\n `dbtupfzqxxhdeeqoiscfsmohhiiluqiy` int DEFAULT NULL,\n `jvbnbjyxsnhmkjmbsxpmdynqudummtxk` int DEFAULT NULL,\n `gyhfkuzyaftfyyovtahkrxixjwyslfpt` int DEFAULT NULL,\n `kwdudyovzqocqrlmwnlrzeotuavplqwa` int DEFAULT NULL,\n `ynyzmfcyfxmdeelkamrahmqwgaqwxgkg` int DEFAULT NULL,\n `xrsslkgiupbqjplpmqnkssrptdawdbqi` int DEFAULT NULL,\n PRIMARY KEY (`aqjfiytufyxdwmtirlkezrcrrhmeafcg`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"qsirphfmlpnkopaqiayapzezhyeudrzs\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["aqjfiytufyxdwmtirlkezrcrrhmeafcg"],"columns":[{"name":"aqjfiytufyxdwmtirlkezrcrrhmeafcg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"lusoqpohibanqhaysfqjpymckzqziapg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vstvbooumksgqojwvzkjdelbdgiumuus","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgzxsecowtvuerwvdrsheunoahrsvdld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nozpogduskuldqhhfnpkpnowxtmxcbxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oyeutcnbcbjbosemepdjuyqnihidsvim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fsahtemevmcnrwzvpbqtjncteoebbdpa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfcwzcmkqayhwqmetjioehuocjqiizsf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lysddankqamybwscqncduruxfhzwmzjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzsyyijzbmxkztspcthtpmxrthxmtycy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cuidakxtxstxxxwgbgxcturzwxoyggbx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gffnlgfkutfnztrcipqoivjcsxiedgsw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdilqopofxnsrugyniyzuwcstrnufcxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cdvcgrbehjvoirdetucrnotwpiprlgjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"msufdizowdxxegifdigmukahiuxydvry","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smalpidlmfavjzvnjxoobhbaexhzzzyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"etsxnomwgkyeohmrkrxuvqkgimzvroup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljvyoqdtjzlorpwsgpplvpipqwdftyoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qsvzyskcibjrqkeatkwbwurlgyaclumv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajilolgkvzfxdzrjysywggvzfdlqtjle","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrxygfsdoevfndpeeikwgillbgoygjnv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybmhacaqhyprwrqsqjwyysiydmxxsadc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjrdeijlxynrqxbeoivioilvfuqsmwls","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmrpokbskiczofdzfcwoinsbekpawknx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igbmvdrkcbzyjvkrlemwbqeqrwdbtvxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqxihtkdxesmpeqigumxfociqjixyhwg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbiuvdemtuozfzdaqwkfkkdctdpziykg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjhmixcnbqeeslktgmdxyjywxyrkjzys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"huxegwkeqwuzvyavhwysbnqzqdfhxvnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uuxitqnhafihbiiokiaptdjuqrtdeyzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kytgbydxhzwzithgngjhxtfbcdvgtozv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpidfcqmzjdnyptwmazjnjsezjfmvcwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqyohdkktiodpoeslcjixemgaivnweqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oxlhpwutuawffbcrnrxzcpdhyhrtkfsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzuouqnrxujtkgtjxfjvoirlpewbioza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zidboiaqgflzxgzcxzapjvvtqcknjgwh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnqkywwmboqypehozgrfazdhkrggjhsf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"duljujgqptfvhpshwgmthnzcifnlkccb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"curcgqhdztfqwcbfpajdwdylvcmfukof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jfenivacwzajbpfymqyzgrqntecmwlad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zckuyeexmdjaoezojqqniphlcytgvqay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhfrlwncwqtcabigwojwdtlpaomeswcz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxxjcverznhdpdhmdhdixtxgvdzzquzc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhsuytflhrakgobqbbfcjepxsdalymmn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcdrrmupwsishtunirtnllefdozpjadq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tljwzwwagqbqigqomggccuntwmlejxno","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psmgoqbptlnmzhxnkosbubckitinkklm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thzxqukszgjkfjjrzetxfbnpowwhsarn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ryqxcuqmpbyerwarmvuwlpdsngsnchdl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ssibvplvweelrcjatiwivtjrujkhlxat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhjpnfwnhwcdtazusswkwwxtceswudpn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eeesfrookzpnxhguxyxsmqnoeutfgeqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uiaolbxtvoggfsmtfqvwgtgdyooepfnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zghrndjtdmjeejpyqqjliuhexhbabfwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdgyipaszpcyxmpmqbbcbdwxtlazupbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnfyfyhstamohkkzxozuxsvaguiridgs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqfkoxkwimoicxqbuwjnmvqsvorqyitm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvvbfzhaufqrllcflwcqkoqwkwalbyie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsxgzizzfkemwddfpmkeqzselgsxmnzc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ygpzhsxrjaisyorbnkospsifohqgfrwz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wrbvjlriueqheuyddkvvzmxipckngvas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdjcsykxyeystidoatzhngdlbcoitqcv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqmsvqcfxeezqucmaqpddjxhvcrjhgof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfzzkvmxojxbnbnpghjedryhhhbhqjpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfboevuaddafseoenxabsfvojyruyaqe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggnmtgvbrmdxcjbihbefrsvzcaitrlld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqxjzqiadcosylozmhjrcjlhzdopffps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkgbgocgxslldfzcqvhfvogemveuypfo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wulgnssgmixeclgxznmalppgduxqtbxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otlfzuhmjjmshyftuxejcsgfvuttvjns","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhdawvunyxhrhpbyztpzllkroeeakzuy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvoyhjeaquxfhwcjikaadhjpvpgmmzgs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cilrokxeyfsgorwldblvptqxcrtevhqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nchkcvqvoxkbiewznhhgesszxpnofnat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpozdhfphcqmfmuzqanzmmpbbnvxnkek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"srlsmqocftkkzakzqojdgngueixnavyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikytbyvqcnzueduaqmuqwenqclngyzml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nyfoaehzghigomufhncunpasgbxmsoaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkukuloelkqvpinzkneicrkmduqoypdi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcidaryggjhzloxkvfvjufzivhekgwkg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ooypobabtifgvedoawdqpjwpytwvsvep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvcyhgddnqxyhhbyhvwzbsxahqfzpkfo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oyvkcouunkkgxywfcbeuxfgpjqkeeohj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhiemakenhuifafbeeklwkrpaqirkzxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkgsskyekiyaqisoqsjwojxhjrijgqlr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hodnhyvgmmyeuwlknegvmlqnwuejmmcz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nwwiitnkrjzlyaswlrguusllyhvwhkxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edswqsesiqrgqeevvxgbszdnmxsgolyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffdqfercversermasaiaymduovwmnnnk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhrafqlbyutbexpsdhkfhbxarqxtqptq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qyofsfxtkhqwebvllsxhuapgglrggwpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"turwoloirtzuplwismtyzbptxhzirwbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txllijsqiednfmcjltjtybjbpvgesyum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gcoekmxnpidhywujupsbtdgosjabxjqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbtupfzqxxhdeeqoiscfsmohhiiluqiy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvbnbjyxsnhmkjmbsxpmdynqudummtxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gyhfkuzyaftfyyovtahkrxixjwyslfpt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwdudyovzqocqrlmwnlrzeotuavplqwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynyzmfcyfxmdeelkamrahmqwgaqwxgkg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrsslkgiupbqjplpmqnkssrptdawdbqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670611,"databaseName":"models_schema","ddl":"CREATE TABLE `qxqwzbxrbfsmpxxlxepnlllymnkkslug` (\n `zvgzhakrlszpnvtaclruscugtppigqvj` int NOT NULL,\n `kykzduauydohutqoxzqhqwhpkfmtjfsn` int DEFAULT NULL,\n `rdtehmquycxywsdvppsmycygohagvzzp` int DEFAULT NULL,\n `gwtcoygtzkihvvymmyaiawczloahrcsj` int DEFAULT NULL,\n `cugeeisfvvxuhsidxtyevvzzbjzosjwg` int DEFAULT NULL,\n `xwihfckflxywxslxarmqacyusrwoebxe` int DEFAULT NULL,\n `cxlwyahjmebtaztnsetenatiicmfcvav` int DEFAULT NULL,\n `dbyhipvoxyoptgmwdkdpnnuccatoseoq` int DEFAULT NULL,\n `issigopleabhkqavvbbvdwfqwvipnzey` int DEFAULT NULL,\n `bzikuvfoiiumsbdedlftmhbelbubbdne` int DEFAULT NULL,\n `zqtyhvvbyldxyhwoxmbsbmlkumorsizi` int DEFAULT NULL,\n `phqfwgjrvbzhwdzofkgesuzerlrnyhpe` int DEFAULT NULL,\n `nsuqnhvadrnsmmkanzhmxsqigngkrlbf` int DEFAULT NULL,\n `ozqvehdrqhmujscfxbhhziuhlozrsntq` int DEFAULT NULL,\n `xomkgdoyyxrmysobqffakazpogmfjfum` int DEFAULT NULL,\n `suoqkymsoqpdiibfflfwvdnszobkdrcx` int DEFAULT NULL,\n `txijipjdnzjvadvwwtpvcurvywbaffyr` int DEFAULT NULL,\n `dcpnmbfpzeltivvyfqrukyevkennbcaw` int DEFAULT NULL,\n `kzdhmxnzwrqtgjapivthnhhxavnyuzrm` int DEFAULT NULL,\n `xlnlxwwljfonqdvhvdyktrgwljhatqko` int DEFAULT NULL,\n `olpihiqftxldlarwjtpyrnhgcnqrdiqg` int DEFAULT NULL,\n `iczqxvmmodaftixwgbpleajmhtfhgbox` int DEFAULT NULL,\n `giyfivkeyekqklceoaouemycritcjooh` int DEFAULT NULL,\n `qqsieyyhsmaaxqoervcfglevqikxadff` int DEFAULT NULL,\n `bavwfvyfwofpalrlbwaginndntzwtuyu` int DEFAULT NULL,\n `qrtfjjsslbepcxggeylrxmgwnoxcccdy` int DEFAULT NULL,\n `ufhdjrnyeoahtrugqabkewjrsbrhfkyl` int DEFAULT NULL,\n `awmgcdgevzmsntkautbsiwwvskkpaivh` int DEFAULT NULL,\n `gctdiexwzdwcpnzwusfjixljvwzrwdhf` int DEFAULT NULL,\n `odbpvvdaqtktzbmdhxanefkqhspknvfw` int DEFAULT NULL,\n `hsslgunttgutmxvochufbkzaeetvojyb` int DEFAULT NULL,\n `mfsoiccmvbgjhqoofwnxjanhnajqdivs` int DEFAULT NULL,\n `hevzqpjotsvnbxpajzhydhmmitotkxuc` int DEFAULT NULL,\n `sozkyntnywffgeoxtdgxbjsvppirqvoc` int DEFAULT NULL,\n `iipxxeeapxlqydljferckffxofhhonzx` int DEFAULT NULL,\n `jzbsrdnnubeduoheijjbkivqfzcxqonu` int DEFAULT NULL,\n `uaygyjkqyspamygqifitylrqzjhdhuej` int DEFAULT NULL,\n `zqvjfrbebbegcznrsardblodocblzpza` int DEFAULT NULL,\n `sedspqpcofragwazaolpsxaiwaiecvxd` int DEFAULT NULL,\n `mcwriwlyfqvllrqwjxqlkbpuplvcaaos` int DEFAULT NULL,\n `wyirpdsweyoyvaduygycldnkyjfucuik` int DEFAULT NULL,\n `pqlmowwklgmxphhaziroduxgskkokmyn` int DEFAULT NULL,\n `cotowgobzuzhmkgoydqoztscngwhdiqf` int DEFAULT NULL,\n `afccoskdksfpasrmfmshojzbrvyhnrtx` int DEFAULT NULL,\n `zejiqfdztvawmxcujpllopbwmpmwwbqp` int DEFAULT NULL,\n `mkekdkujoeicidpupbospxvqpmyiorzc` int DEFAULT NULL,\n `toxufazbtxhmrqntojfwolztyradtdmn` int DEFAULT NULL,\n `gevfpoezhxcueaenvfottygeacgctcae` int DEFAULT NULL,\n `aatetrttgtzjaxsoivbocnmnyftqojnr` int DEFAULT NULL,\n `mpxuhlhymrrlgngogldmgboxezcbfzey` int DEFAULT NULL,\n `cakgjfvakgizizvzirzihjzchvcqmgsn` int DEFAULT NULL,\n `tfkovyvyntfyesmehehoacvpreqbzlju` int DEFAULT NULL,\n `xtusyostdqktkwtwwvcllwyyvkzwfqow` int DEFAULT NULL,\n `fgezgyahemkdflkmsfmdkykgtcjkwyet` int DEFAULT NULL,\n `aqwtjkquincvnywiteuzotfhvbpeziwy` int DEFAULT NULL,\n `jthiuofhsmqhdxfyedohlmlsibzdaxmu` int DEFAULT NULL,\n `bctzwbkqoynihiqwhctgahodozdbsmxu` int DEFAULT NULL,\n `zxrsmleqlobpqfxlqjjrkjmvoiifmkpa` int DEFAULT NULL,\n `djqeozokgbpxpzbwxyfgksissblemqqv` int DEFAULT NULL,\n `grvieqqtbmxlsihquqskcizgchhztyxn` int DEFAULT NULL,\n `aurgreqkwmbewhiyefokyeynzhnuuqsc` int DEFAULT NULL,\n `shuioijgngcudqbolcgsxlkaxyqrsdhl` int DEFAULT NULL,\n `tayzhnnykeevkpmluumxizaminjinhri` int DEFAULT NULL,\n `bqantebfsnilvgxfakrezhdtvrkfcuoa` int DEFAULT NULL,\n `bwgpwwllbgvhcbyeffrtsuzsmdsthdoo` int DEFAULT NULL,\n `qimtvmtcuvszjobasrknrewqdklmohsi` int DEFAULT NULL,\n `fjhzdzjphbgqpvkoboawmfzrkrucvsyj` int DEFAULT NULL,\n `fuutkfjdpszuqavacqisqkzpibzicjmk` int DEFAULT NULL,\n `vgmkjstdeqncizldgsjbwqlmltqsfxab` int DEFAULT NULL,\n `oegguoigtssltevadblvxksbimidlvcd` int DEFAULT NULL,\n `siqckzbksmjdwdbjjhnwgsysxazpkleo` int DEFAULT NULL,\n `qskdntldmzunilamnnukvtetrqrxsthl` int DEFAULT NULL,\n `dsmsarqzplhjgyqprlwkkezbgximyazi` int DEFAULT NULL,\n `dvqlfdoxghhtcxenfleiojahlqwvhlib` int DEFAULT NULL,\n `pxqujoeefwhqozmhhkmbgqpsdmzqoetj` int DEFAULT NULL,\n `igcojukulkxnalifueyfodjammimmmjm` int DEFAULT NULL,\n `pswgiurhhefhettujgifcfzmdbxafgod` int DEFAULT NULL,\n `cmmcsqkquskrcwyxscagkzqwbeokcizn` int DEFAULT NULL,\n `bvgxssvmlcgmuospyqkezlwpfqevmhwz` int DEFAULT NULL,\n `jovmytvbepqprwxloaqrmotgqidbxvsp` int DEFAULT NULL,\n `fixtzmhngotonnrwfssiejyjaagffnzr` int DEFAULT NULL,\n `jgvepwwsblipkpasoqamftrnaejjnbpf` int DEFAULT NULL,\n `wsoaumtopikyjvnfsytetyfkwxtdfatn` int DEFAULT NULL,\n `foolgpmugpwscsglvgqwdgdqebmifnsw` int DEFAULT NULL,\n `cwayhxfvuyyuhupmcuejhbbyjeccoyml` int DEFAULT NULL,\n `beorqyxvqkibitnznctbttsefhehyfhw` int DEFAULT NULL,\n `ftlcfegogrbdtfbzhoyllzqgzwrqmaqb` int DEFAULT NULL,\n `fuxxrlyklrackdjehihxredzkmuizdsc` int DEFAULT NULL,\n `qffbcjmmaaklasxwxxqxludsqdvxdfdp` int DEFAULT NULL,\n `itszssgosucgiowxjawxrvphapjjufgz` int DEFAULT NULL,\n `hcsphrgqzjtlfjmkeaqcrfuyqhmjwbmn` int DEFAULT NULL,\n `kqetevnhkzecsjooleouceinosnakmiw` int DEFAULT NULL,\n `xchwuoyttlseqbwesdpvxgrexwnpwzyd` int DEFAULT NULL,\n `tjpgzzepdulgstdlsxmerseimodkdyyd` int DEFAULT NULL,\n `beasfeeiyigoyaqffrzuvpumtrwefmhr` int DEFAULT NULL,\n `lxgzdwvbscacofamyuvafxwfmukhiosy` int DEFAULT NULL,\n `haqciatctpxpimzablnbwpxvrglgeplz` int DEFAULT NULL,\n `gbggljiaoajzswbckhrahvjktsdsuyxv` int DEFAULT NULL,\n `mgeinlbpzhziapfpssvcobtnvcpntquy` int DEFAULT NULL,\n `bqomfdihqvzlxiczzqhforlxzdbtrlym` int DEFAULT NULL,\n PRIMARY KEY (`zvgzhakrlszpnvtaclruscugtppigqvj`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"qxqwzbxrbfsmpxxlxepnlllymnkkslug\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["zvgzhakrlszpnvtaclruscugtppigqvj"],"columns":[{"name":"zvgzhakrlszpnvtaclruscugtppigqvj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"kykzduauydohutqoxzqhqwhpkfmtjfsn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdtehmquycxywsdvppsmycygohagvzzp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gwtcoygtzkihvvymmyaiawczloahrcsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cugeeisfvvxuhsidxtyevvzzbjzosjwg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xwihfckflxywxslxarmqacyusrwoebxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cxlwyahjmebtaztnsetenatiicmfcvav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbyhipvoxyoptgmwdkdpnnuccatoseoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"issigopleabhkqavvbbvdwfqwvipnzey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzikuvfoiiumsbdedlftmhbelbubbdne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqtyhvvbyldxyhwoxmbsbmlkumorsizi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phqfwgjrvbzhwdzofkgesuzerlrnyhpe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nsuqnhvadrnsmmkanzhmxsqigngkrlbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozqvehdrqhmujscfxbhhziuhlozrsntq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xomkgdoyyxrmysobqffakazpogmfjfum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suoqkymsoqpdiibfflfwvdnszobkdrcx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txijipjdnzjvadvwwtpvcurvywbaffyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcpnmbfpzeltivvyfqrukyevkennbcaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzdhmxnzwrqtgjapivthnhhxavnyuzrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xlnlxwwljfonqdvhvdyktrgwljhatqko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"olpihiqftxldlarwjtpyrnhgcnqrdiqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iczqxvmmodaftixwgbpleajmhtfhgbox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"giyfivkeyekqklceoaouemycritcjooh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqsieyyhsmaaxqoervcfglevqikxadff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bavwfvyfwofpalrlbwaginndntzwtuyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrtfjjsslbepcxggeylrxmgwnoxcccdy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufhdjrnyeoahtrugqabkewjrsbrhfkyl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awmgcdgevzmsntkautbsiwwvskkpaivh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gctdiexwzdwcpnzwusfjixljvwzrwdhf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"odbpvvdaqtktzbmdhxanefkqhspknvfw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hsslgunttgutmxvochufbkzaeetvojyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfsoiccmvbgjhqoofwnxjanhnajqdivs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hevzqpjotsvnbxpajzhydhmmitotkxuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sozkyntnywffgeoxtdgxbjsvppirqvoc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iipxxeeapxlqydljferckffxofhhonzx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzbsrdnnubeduoheijjbkivqfzcxqonu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uaygyjkqyspamygqifitylrqzjhdhuej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqvjfrbebbegcznrsardblodocblzpza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sedspqpcofragwazaolpsxaiwaiecvxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcwriwlyfqvllrqwjxqlkbpuplvcaaos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyirpdsweyoyvaduygycldnkyjfucuik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqlmowwklgmxphhaziroduxgskkokmyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cotowgobzuzhmkgoydqoztscngwhdiqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afccoskdksfpasrmfmshojzbrvyhnrtx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zejiqfdztvawmxcujpllopbwmpmwwbqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkekdkujoeicidpupbospxvqpmyiorzc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"toxufazbtxhmrqntojfwolztyradtdmn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gevfpoezhxcueaenvfottygeacgctcae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aatetrttgtzjaxsoivbocnmnyftqojnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpxuhlhymrrlgngogldmgboxezcbfzey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cakgjfvakgizizvzirzihjzchvcqmgsn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tfkovyvyntfyesmehehoacvpreqbzlju","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xtusyostdqktkwtwwvcllwyyvkzwfqow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fgezgyahemkdflkmsfmdkykgtcjkwyet","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqwtjkquincvnywiteuzotfhvbpeziwy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jthiuofhsmqhdxfyedohlmlsibzdaxmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bctzwbkqoynihiqwhctgahodozdbsmxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxrsmleqlobpqfxlqjjrkjmvoiifmkpa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"djqeozokgbpxpzbwxyfgksissblemqqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grvieqqtbmxlsihquqskcizgchhztyxn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aurgreqkwmbewhiyefokyeynzhnuuqsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shuioijgngcudqbolcgsxlkaxyqrsdhl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tayzhnnykeevkpmluumxizaminjinhri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqantebfsnilvgxfakrezhdtvrkfcuoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bwgpwwllbgvhcbyeffrtsuzsmdsthdoo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qimtvmtcuvszjobasrknrewqdklmohsi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjhzdzjphbgqpvkoboawmfzrkrucvsyj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fuutkfjdpszuqavacqisqkzpibzicjmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgmkjstdeqncizldgsjbwqlmltqsfxab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oegguoigtssltevadblvxksbimidlvcd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"siqckzbksmjdwdbjjhnwgsysxazpkleo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qskdntldmzunilamnnukvtetrqrxsthl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsmsarqzplhjgyqprlwkkezbgximyazi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvqlfdoxghhtcxenfleiojahlqwvhlib","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxqujoeefwhqozmhhkmbgqpsdmzqoetj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igcojukulkxnalifueyfodjammimmmjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pswgiurhhefhettujgifcfzmdbxafgod","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmmcsqkquskrcwyxscagkzqwbeokcizn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvgxssvmlcgmuospyqkezlwpfqevmhwz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jovmytvbepqprwxloaqrmotgqidbxvsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fixtzmhngotonnrwfssiejyjaagffnzr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgvepwwsblipkpasoqamftrnaejjnbpf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wsoaumtopikyjvnfsytetyfkwxtdfatn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"foolgpmugpwscsglvgqwdgdqebmifnsw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwayhxfvuyyuhupmcuejhbbyjeccoyml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"beorqyxvqkibitnznctbttsefhehyfhw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftlcfegogrbdtfbzhoyllzqgzwrqmaqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fuxxrlyklrackdjehihxredzkmuizdsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qffbcjmmaaklasxwxxqxludsqdvxdfdp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itszssgosucgiowxjawxrvphapjjufgz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hcsphrgqzjtlfjmkeaqcrfuyqhmjwbmn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqetevnhkzecsjooleouceinosnakmiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xchwuoyttlseqbwesdpvxgrexwnpwzyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjpgzzepdulgstdlsxmerseimodkdyyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"beasfeeiyigoyaqffrzuvpumtrwefmhr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxgzdwvbscacofamyuvafxwfmukhiosy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"haqciatctpxpimzablnbwpxvrglgeplz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gbggljiaoajzswbckhrahvjktsdsuyxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mgeinlbpzhziapfpssvcobtnvcpntquy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqomfdihqvzlxiczzqhforlxzdbtrlym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670642,"databaseName":"models_schema","ddl":"CREATE TABLE `qyylbziunhtqyqeloiskfjizpimemone` (\n `louhygmaxxouzjfuphtnmtjistwvnwyq` int NOT NULL,\n `uiiheihgpclahftojbzrwlmjhabbxwdj` int DEFAULT NULL,\n `xolzclewdpfyzeybyzrwktowsqybamuk` int DEFAULT NULL,\n `nqehknvdnttwydkhhvhoirbyihoyixmm` int DEFAULT NULL,\n `bwrywrmsdyjrpragcpbaohgkijcypftq` int DEFAULT NULL,\n `jwbdvjdcitbduvmknzouvfzpfbunbaty` int DEFAULT NULL,\n `pxwodsherperfbwumwirubqiqkoieozg` int DEFAULT NULL,\n `mtdkmanixyhsonqfbgbbvdjfewkwfgac` int DEFAULT NULL,\n `qhxjjowmsxoysbxkobnxubzeksrorehn` int DEFAULT NULL,\n `sdwogdooqxrwgicsgewrcdrsgdufynqn` int DEFAULT NULL,\n `pteilgotqkvtsbccypafuwyyqskwlade` int DEFAULT NULL,\n `ttcpirpjqculrpwhjjusuvqynmcvudjp` int DEFAULT NULL,\n `ntmqrvxambjjvqflholzcurehynphyzn` int DEFAULT NULL,\n `leletpciqfgpzhlbkzytmhavkxdpbzxs` int DEFAULT NULL,\n `oywnptbicxwpmedbiurwnhtzpglguext` int DEFAULT NULL,\n `qknyeryugkmvicnrxygqxpytxbtummkp` int DEFAULT NULL,\n `doayevikkmovqcqpmeittwhrnrenvjyz` int DEFAULT NULL,\n `kqurivlaevlnjlauptcxgbaeooktywuo` int DEFAULT NULL,\n `kyjuwstjembywywpvuussyxxjzwapxax` int DEFAULT NULL,\n `kdcnvaxvlqjbwlahggrfgppvvrnsudrs` int DEFAULT NULL,\n `dzsbzjfpnccinrgesjcqwibbghmqfuui` int DEFAULT NULL,\n `knzomuyffvmpiafptsoxebrhgmrgjlvf` int DEFAULT NULL,\n `xuxmaftrvuosktoutfkkbvdljeehmsqh` int DEFAULT NULL,\n `sduvgsfgmcnwtkqiiuokrstyzvkrvctx` int DEFAULT NULL,\n `crrsslpqvzavspuyemymlnjuevsoebln` int DEFAULT NULL,\n `yajredckvxxntomdeqkzrlwkkalvkaes` int DEFAULT NULL,\n `phnzjjqqmduoupwwatjcnawsjfgqkjsd` int DEFAULT NULL,\n `rlrmklqntpuaulwnlbqhsacazpwgzqvt` int DEFAULT NULL,\n `msliukhqpeazkypiaftldrekxbbaniua` int DEFAULT NULL,\n `gthtdwojsrqahrwjfusmppebmonszlqb` int DEFAULT NULL,\n `ccntrddlferbnxlqgpllnsbstjwgaegs` int DEFAULT NULL,\n `dxqokylvsryhkodytkwdmxxddcclpprb` int DEFAULT NULL,\n `svwpgybixfajgsfbjwfgmwphxhjjduzf` int DEFAULT NULL,\n `dacerpxwslknssyovfuyvkdvuurhxzcn` int DEFAULT NULL,\n `duiufswyqlvuleqvxqgyamclrrqpernu` int DEFAULT NULL,\n `ivwuhokryesfakcqpgztyhjrlnytsysc` int DEFAULT NULL,\n `otcydnecopzjcmlqwbyzhteboewdfzes` int DEFAULT NULL,\n `lzyawhqphangpdgepstybtegincdszev` int DEFAULT NULL,\n `piqfuhqgojjuohstbcrwfvbhjxntqpys` int DEFAULT NULL,\n `ehlqjyycjaanfcjskmgfzwxpgophndfk` int DEFAULT NULL,\n `pnhbbdqqyexrqoyglxqegshpogkrgbcf` int DEFAULT NULL,\n `oooyjzulkatvtzybpyhkhdavjtcyqgfp` int DEFAULT NULL,\n `yisbfflprdzhficmhjbvkgzvtcnzmzjq` int DEFAULT NULL,\n `khuodhhcqmcgdhiwbugtybzymtteylbc` int DEFAULT NULL,\n `ivkmyemgynobknowhlugtadhfyfesynm` int DEFAULT NULL,\n `yxrswdbzivaqfrayrsuytozzuevcfwoq` int DEFAULT NULL,\n `huplnfmedeffkngobiuatybhapiuixhf` int DEFAULT NULL,\n `rvrfxzsieasftpngggbvpnetwqdadhig` int DEFAULT NULL,\n `ptebwfmgvxsgatgttmlifbhsfzgooshn` int DEFAULT NULL,\n `zsjpslpnasgruqieyfathmdpzqbphibr` int DEFAULT NULL,\n `wjcsipzxhhgfgtzffhrxbmreyxwghuwp` int DEFAULT NULL,\n `qogxpgwvbgwzwpheoyaailmtxhqtqmsq` int DEFAULT NULL,\n `hjwayajsrlkylxhvegfksgvbheditpoi` int DEFAULT NULL,\n `lsfgrqwwnbxxnagsijfvhtgymkkuwbok` int DEFAULT NULL,\n `rkfelnsedrvlbeegfrtupaxadfucmdfi` int DEFAULT NULL,\n `gfbhyftlvpxztlyvknedukctiwxtglch` int DEFAULT NULL,\n `axqicisyreaosnbcegmywtqgyfzemzmg` int DEFAULT NULL,\n `cjrkbcipdcshkqijhkkynjzxpsftvket` int DEFAULT NULL,\n `umqlkcmhadlfqsygijsviymlrkmdrgud` int DEFAULT NULL,\n `nkzunorxmmxjgcmxanrohhuygksiuyrg` int DEFAULT NULL,\n `kyddtfnchetnslrjcdptwvshenunbaax` int DEFAULT NULL,\n `vnhlfxeunnjbiwijjktrlepzzvejjpee` int DEFAULT NULL,\n `majueajriiwdxvldtymbdlsfivdbxjqk` int DEFAULT NULL,\n `qwuoidixaguiksetusztuyyvjqafbsro` int DEFAULT NULL,\n `ltommpnhocilsrheknaudfzekhtmihoj` int DEFAULT NULL,\n `ckarolrsguzspdupmqfhboetsrtvpmpc` int DEFAULT NULL,\n `pfjaysphqmypgonoypedfmicgpxsxycj` int DEFAULT NULL,\n `auqujfbjacvavibplndwkmktobulhuil` int DEFAULT NULL,\n `nkvpgaplbrhdsujdmwmbsepduzrfswku` int DEFAULT NULL,\n `pajvhbpfngaoilnvluphuqivglqueutk` int DEFAULT NULL,\n `mwmxwjiswziyqnxgcuauhbvyhqrrcnjf` int DEFAULT NULL,\n `nfzkdhtbdwrgmhftagvwaanfuhzzrpsg` int DEFAULT NULL,\n `ynohzlnfafnwrnawwhkcgozfjpqwqeit` int DEFAULT NULL,\n `ngogqkxiwgjmlqccqsmmvfvdcqjftbkq` int DEFAULT NULL,\n `kdyjarjxnylefdhvsvispcllppujdlwn` int DEFAULT NULL,\n `qbofsiwfvkqdlboaqguaylpstykkckxe` int DEFAULT NULL,\n `fdvhsfinxwdzhpkoatdafwaholudkwop` int DEFAULT NULL,\n `jxqmprpvkketbexvcawnxppvpoecptkb` int DEFAULT NULL,\n `jbwyumdmjrtkdltzwtnmizlvvplazlyb` int DEFAULT NULL,\n `yoorsvgtiduyclvxqakukvlhimvpkzcw` int DEFAULT NULL,\n `tsuunqpltrdvobfnbubxqaqqsucvwdxj` int DEFAULT NULL,\n `bhxkdoidwkbqnrxhimmdqbcmrnzwrwrb` int DEFAULT NULL,\n `iypyeqqferlvdwsmeyicbjshbylweoxo` int DEFAULT NULL,\n `jvtkttgxbzejmlfeawxiaygouvsharrt` int DEFAULT NULL,\n `ouaixnhlujdirbwhekqgolmlmcgkccaq` int DEFAULT NULL,\n `sgapioghgiqnnzlsuojdqpphabsynwbg` int DEFAULT NULL,\n `ynljoysismutkkkebvscljqdjokgenrd` int DEFAULT NULL,\n `mvahtlheggxvigltkgfwfbedfdbbroll` int DEFAULT NULL,\n `lfvqgvugpkdvqomfasymzpinypqqtvrz` int DEFAULT NULL,\n `foeotozqofpkawseatvmaqcungedcgvp` int DEFAULT NULL,\n `xycxmcpsayemdnlzryhxgngyrrsjrnqx` int DEFAULT NULL,\n `gginumdtmkwgqxelzhjrqkuejajsbtln` int DEFAULT NULL,\n `bjieumrfluekabjvtmwrxkgmxmergtwn` int DEFAULT NULL,\n `knmwkxxfnzfdwrxzzksbqlzezhoqndph` int DEFAULT NULL,\n `nbckdrfccjxlozboufpftieztsppxzsq` int DEFAULT NULL,\n `hytmoxmifxmnglbknbqreeavrkwptgxq` int DEFAULT NULL,\n `nyncgxfwmnlqffiphwbrxhkktmymlxfa` int DEFAULT NULL,\n `ibfmqtbgelqxymuyiyorqoohvizpdbna` int DEFAULT NULL,\n `fhbbsqkavpbcfyahoyhdovspnhwudhds` int DEFAULT NULL,\n `tcgdcavuqctiyslyzsqwknskxptrrdax` int DEFAULT NULL,\n PRIMARY KEY (`louhygmaxxouzjfuphtnmtjistwvnwyq`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"qyylbziunhtqyqeloiskfjizpimemone\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["louhygmaxxouzjfuphtnmtjistwvnwyq"],"columns":[{"name":"louhygmaxxouzjfuphtnmtjistwvnwyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"uiiheihgpclahftojbzrwlmjhabbxwdj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xolzclewdpfyzeybyzrwktowsqybamuk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqehknvdnttwydkhhvhoirbyihoyixmm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bwrywrmsdyjrpragcpbaohgkijcypftq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwbdvjdcitbduvmknzouvfzpfbunbaty","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxwodsherperfbwumwirubqiqkoieozg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtdkmanixyhsonqfbgbbvdjfewkwfgac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhxjjowmsxoysbxkobnxubzeksrorehn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdwogdooqxrwgicsgewrcdrsgdufynqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pteilgotqkvtsbccypafuwyyqskwlade","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ttcpirpjqculrpwhjjusuvqynmcvudjp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntmqrvxambjjvqflholzcurehynphyzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"leletpciqfgpzhlbkzytmhavkxdpbzxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oywnptbicxwpmedbiurwnhtzpglguext","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qknyeryugkmvicnrxygqxpytxbtummkp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"doayevikkmovqcqpmeittwhrnrenvjyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqurivlaevlnjlauptcxgbaeooktywuo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kyjuwstjembywywpvuussyxxjzwapxax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kdcnvaxvlqjbwlahggrfgppvvrnsudrs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dzsbzjfpnccinrgesjcqwibbghmqfuui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knzomuyffvmpiafptsoxebrhgmrgjlvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xuxmaftrvuosktoutfkkbvdljeehmsqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sduvgsfgmcnwtkqiiuokrstyzvkrvctx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crrsslpqvzavspuyemymlnjuevsoebln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yajredckvxxntomdeqkzrlwkkalvkaes","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phnzjjqqmduoupwwatjcnawsjfgqkjsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlrmklqntpuaulwnlbqhsacazpwgzqvt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"msliukhqpeazkypiaftldrekxbbaniua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gthtdwojsrqahrwjfusmppebmonszlqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ccntrddlferbnxlqgpllnsbstjwgaegs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxqokylvsryhkodytkwdmxxddcclpprb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"svwpgybixfajgsfbjwfgmwphxhjjduzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dacerpxwslknssyovfuyvkdvuurhxzcn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"duiufswyqlvuleqvxqgyamclrrqpernu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivwuhokryesfakcqpgztyhjrlnytsysc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otcydnecopzjcmlqwbyzhteboewdfzes","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzyawhqphangpdgepstybtegincdszev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"piqfuhqgojjuohstbcrwfvbhjxntqpys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehlqjyycjaanfcjskmgfzwxpgophndfk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnhbbdqqyexrqoyglxqegshpogkrgbcf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oooyjzulkatvtzybpyhkhdavjtcyqgfp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yisbfflprdzhficmhjbvkgzvtcnzmzjq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khuodhhcqmcgdhiwbugtybzymtteylbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivkmyemgynobknowhlugtadhfyfesynm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxrswdbzivaqfrayrsuytozzuevcfwoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"huplnfmedeffkngobiuatybhapiuixhf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rvrfxzsieasftpngggbvpnetwqdadhig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ptebwfmgvxsgatgttmlifbhsfzgooshn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zsjpslpnasgruqieyfathmdpzqbphibr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjcsipzxhhgfgtzffhrxbmreyxwghuwp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qogxpgwvbgwzwpheoyaailmtxhqtqmsq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjwayajsrlkylxhvegfksgvbheditpoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lsfgrqwwnbxxnagsijfvhtgymkkuwbok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkfelnsedrvlbeegfrtupaxadfucmdfi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfbhyftlvpxztlyvknedukctiwxtglch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axqicisyreaosnbcegmywtqgyfzemzmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjrkbcipdcshkqijhkkynjzxpsftvket","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umqlkcmhadlfqsygijsviymlrkmdrgud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkzunorxmmxjgcmxanrohhuygksiuyrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kyddtfnchetnslrjcdptwvshenunbaax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnhlfxeunnjbiwijjktrlepzzvejjpee","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"majueajriiwdxvldtymbdlsfivdbxjqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwuoidixaguiksetusztuyyvjqafbsro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ltommpnhocilsrheknaudfzekhtmihoj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckarolrsguzspdupmqfhboetsrtvpmpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pfjaysphqmypgonoypedfmicgpxsxycj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"auqujfbjacvavibplndwkmktobulhuil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkvpgaplbrhdsujdmwmbsepduzrfswku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pajvhbpfngaoilnvluphuqivglqueutk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwmxwjiswziyqnxgcuauhbvyhqrrcnjf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfzkdhtbdwrgmhftagvwaanfuhzzrpsg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynohzlnfafnwrnawwhkcgozfjpqwqeit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ngogqkxiwgjmlqccqsmmvfvdcqjftbkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kdyjarjxnylefdhvsvispcllppujdlwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qbofsiwfvkqdlboaqguaylpstykkckxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fdvhsfinxwdzhpkoatdafwaholudkwop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jxqmprpvkketbexvcawnxppvpoecptkb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbwyumdmjrtkdltzwtnmizlvvplazlyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yoorsvgtiduyclvxqakukvlhimvpkzcw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tsuunqpltrdvobfnbubxqaqqsucvwdxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bhxkdoidwkbqnrxhimmdqbcmrnzwrwrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iypyeqqferlvdwsmeyicbjshbylweoxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvtkttgxbzejmlfeawxiaygouvsharrt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ouaixnhlujdirbwhekqgolmlmcgkccaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sgapioghgiqnnzlsuojdqpphabsynwbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynljoysismutkkkebvscljqdjokgenrd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvahtlheggxvigltkgfwfbedfdbbroll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfvqgvugpkdvqomfasymzpinypqqtvrz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"foeotozqofpkawseatvmaqcungedcgvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xycxmcpsayemdnlzryhxgngyrrsjrnqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gginumdtmkwgqxelzhjrqkuejajsbtln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjieumrfluekabjvtmwrxkgmxmergtwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knmwkxxfnzfdwrxzzksbqlzezhoqndph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbckdrfccjxlozboufpftieztsppxzsq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hytmoxmifxmnglbknbqreeavrkwptgxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nyncgxfwmnlqffiphwbrxhkktmymlxfa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibfmqtbgelqxymuyiyorqoohvizpdbna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhbbsqkavpbcfyahoyhdovspnhwudhds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tcgdcavuqctiyslyzsqwknskxptrrdax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670672,"databaseName":"models_schema","ddl":"CREATE TABLE `rdcfwajpujmymsqszcnekivknqxdoqil` (\n `aqvpvqqneggxofjqrspisgguzvhvbzrt` int NOT NULL,\n `rgyqgazfklklfogtrqyruwdfdlzzcegi` int DEFAULT NULL,\n `nqddtfqggyykdgzvilxijekbyvqqchis` int DEFAULT NULL,\n `iyghyqgalnjmttllzcmythgmdgggicgt` int DEFAULT NULL,\n `uvqfjysbaffzuubsupezhccqqvdjrmjn` int DEFAULT NULL,\n `pzonlmgdvsryfdhenttgtstnwgktrykd` int DEFAULT NULL,\n `dupeujtpihnkydmjqpmzynfnxuinlvcy` int DEFAULT NULL,\n `wqzhqttdxbbojyzweilqmvfuvbdjfpwi` int DEFAULT NULL,\n `krsbngaerrfvogglkxolmrrzlznrbagb` int DEFAULT NULL,\n `qsudbjnihsvlsmktkjsxqgmxoulhnsto` int DEFAULT NULL,\n `mugposrkwvpakczqhnvnohvqefhauhwx` int DEFAULT NULL,\n `ftcilfsakxwcxpmtlvrvfafetvhwklly` int DEFAULT NULL,\n `gazqrfdchkbkghkyjgqzvqhtaffvcywc` int DEFAULT NULL,\n `umuwjyatknqnilbvdaiunhmdhjsmdpkr` int DEFAULT NULL,\n `qqfmuqlxfvrhtxooeixujaifigqpuggb` int DEFAULT NULL,\n `chajsiygnaqumqzuycvygncwnynmpjyu` int DEFAULT NULL,\n `deyozguiuoyctqkpvtmixqxlmxkkxhqm` int DEFAULT NULL,\n `lrflszbatymxunomowxrwplxfhzhgkbr` int DEFAULT NULL,\n `bkabtgqytqinvghyqrezmtttogeqewbi` int DEFAULT NULL,\n `khzefnvnjmrqzfsjcgmwpcntgohrmuqd` int DEFAULT NULL,\n `ujlgfofrqjbsrmfrebtycyfaqmkcqcig` int DEFAULT NULL,\n `cscvrrgirncqbvqyfjuaoarryexldhxj` int DEFAULT NULL,\n `cfqawepcspzzhzpgrdotdighruuuhoqx` int DEFAULT NULL,\n `nnxzqwwoyssuqwcwfxeuzlbvtywpsvgk` int DEFAULT NULL,\n `zusnboikzqrzlzwvbmelbgxwfzeocytm` int DEFAULT NULL,\n `eodlhgkpuintldnmbvyqsazpdcdvsssc` int DEFAULT NULL,\n `guejtvayyipdncqapgcjknxhhfhevscn` int DEFAULT NULL,\n `jmvbqxufpsmdelcqplybzachkrknqkrh` int DEFAULT NULL,\n `jzycbtpjtulbmjjdcertviqdkyjhplaw` int DEFAULT NULL,\n `onhfcqmncslwtvtvfhivglrachielbyc` int DEFAULT NULL,\n `sfrypfcabugpdihzejwmsnxdyhsbkdsp` int DEFAULT NULL,\n `gxpndviavqfudwlppwuoilkuodckoiui` int DEFAULT NULL,\n `cfkbudcgzdeevclobryvtdgciiszsrag` int DEFAULT NULL,\n `ngfvukuotrpbgbqkrjnmdossroqadiuf` int DEFAULT NULL,\n `ptofqrjgvwgihntenructrvgbwjafxxy` int DEFAULT NULL,\n `lmenwexdfxdqaffybfxkkjfmevaxwfia` int DEFAULT NULL,\n `zotxxqavxbqsmfdxxfftcxqhuebggkqi` int DEFAULT NULL,\n `yselqlpivbguyneocwhdnwncwxkqjzxm` int DEFAULT NULL,\n `ldvwvsbsnvlgesfbhzcorfpftnddazpk` int DEFAULT NULL,\n `qkdtxuigbgpqjlslskqtwahucqiduuvs` int DEFAULT NULL,\n `ukwohaeeocyfycdxsdmynkwcnkeclroz` int DEFAULT NULL,\n `oylhvdzfjncwdntnhlygrllghajzwhxd` int DEFAULT NULL,\n `ejgesinazqsmkziqsxwzpgnphhpobxbl` int DEFAULT NULL,\n `jaikhnkiazaeiloxsilsgulxcccmcwkk` int DEFAULT NULL,\n `mkgkyirnpgidfyqezngbiqprukdeeuxh` int DEFAULT NULL,\n `govdjksjhknwoplxwcmzldmevgybqzki` int DEFAULT NULL,\n `qergymeebwrmvoqqkwjqxuxfowyfaqlu` int DEFAULT NULL,\n `bkojpcnsswvjofrtuxezwtyhjjxcbrwo` int DEFAULT NULL,\n `faebwsfxbyqjvqakkgvpvymiqeepdhyd` int DEFAULT NULL,\n `odypbpvhleumusvamehjywdqhuzmhvgv` int DEFAULT NULL,\n `gpaivdfwnvswavwjscgililnwtgukwjj` int DEFAULT NULL,\n `rpaxxmnzjrdlsjywwkozevdwybemtodd` int DEFAULT NULL,\n `dauvcaccdnvjkufrzdykmxziwpxosxhh` int DEFAULT NULL,\n `yzzyndctepkfrpmaskcvyrkicxulhnru` int DEFAULT NULL,\n `bbsdoapdyqwcladmkzqpttywygfnvthj` int DEFAULT NULL,\n `vyhkdcxlysnxdgvquqtmtbjioiyktijs` int DEFAULT NULL,\n `hjjvsyuouacbvstfvmnrclqbawukzequ` int DEFAULT NULL,\n `nirokglukqdyscagofhfsoxmvbdushvh` int DEFAULT NULL,\n `xjamtfhyjtvkogitaqplqcpqliyffoek` int DEFAULT NULL,\n `uiigfzvepvhfafoawuqleluoiaceplni` int DEFAULT NULL,\n `ofkkjsznwfanhtzeytqtlalaclsektej` int DEFAULT NULL,\n `hhvppcvwwxwasikpuxlvztngtmwsyjdo` int DEFAULT NULL,\n `fshwxkfhsnfaudweihunkgaelmopnygu` int DEFAULT NULL,\n `njlzzprwqiztdnwoecccmvlsgscarvli` int DEFAULT NULL,\n `lummretmxbfpxgfzaqzvkhiqqbscdhgj` int DEFAULT NULL,\n `ytlfybbsgfcfkukrzsjkjqkdtiemlrkx` int DEFAULT NULL,\n `egogmunyuliqcoodenoqnmbpvneluupx` int DEFAULT NULL,\n `thghlixpsbhwfjpwkowcuwzuiyeqdyui` int DEFAULT NULL,\n `efhblvkuhpudnwtxqfcozciogzzjfqje` int DEFAULT NULL,\n `vzbxbqwprvlsvifsypwdgujvgjdfseoi` int DEFAULT NULL,\n `cawdljvueghintwbfyksynquqarnszvu` int DEFAULT NULL,\n `sjvzgfnbtpapvpdhsupqwgfbjixesksj` int DEFAULT NULL,\n `tygqxjlorekzvknikuypboqqugahvqyu` int DEFAULT NULL,\n `notwrdgeweswvewxyfbtdttathemkgft` int DEFAULT NULL,\n `mtusxrhgtmytexmypwqzcueeovpwibla` int DEFAULT NULL,\n `sirxmogrqrtqjmxrlbedmhzyhrbpumsf` int DEFAULT NULL,\n `kixdhivtzkkjsdqruikhucyxehakfcpt` int DEFAULT NULL,\n `isdukbwdatgjcngxqmdkctgkrimqpbik` int DEFAULT NULL,\n `shqmhwllhnetxrmlqjrwczrbuuodscqc` int DEFAULT NULL,\n `vqkomnzmaszrviaxbfvccnlyjtznehgp` int DEFAULT NULL,\n `mnqnuyuytleontqcfgylrmswsyofjjtd` int DEFAULT NULL,\n `ldarpeilgytpwirxybzscmrsrggwlmlg` int DEFAULT NULL,\n `mjqozzblwugyzvwximjgrdhomcghvhjt` int DEFAULT NULL,\n `yqsrahhyttzxbixrfvpvtekoweazokgs` int DEFAULT NULL,\n `yhtirqldtidcbbmytmnveioivoihypcq` int DEFAULT NULL,\n `aeftysthpaixmrjefqlrsqpgjkxdqzmp` int DEFAULT NULL,\n `sazhchoxdbgjntpjjkugpjkucnfnxfdo` int DEFAULT NULL,\n `mamhppcpuulsyhgfsxqipogofyucuylu` int DEFAULT NULL,\n `uspqeemioqpdlblcgmazetuvldnoggxx` int DEFAULT NULL,\n `dbnylszjapfpgfifgyirrqjdcjlnpcgz` int DEFAULT NULL,\n `pifntpnouozanvcpxljjockrycpzzaml` int DEFAULT NULL,\n `ldoohrakwsidahuxuqnhigymnonzvaig` int DEFAULT NULL,\n `qlcfrbxkgcztrcytshogeezvmmdwrhas` int DEFAULT NULL,\n `pgumwjxpnnvdotxncqsyueyofaqipdll` int DEFAULT NULL,\n `vudzjbicpgturbnckylixhktqcmmeqiu` int DEFAULT NULL,\n `vtpywsqihyiqgtdkaczsgmtdgjxulkdm` int DEFAULT NULL,\n `dgvwoxrqufnkazglyofroidsamnxzvta` int DEFAULT NULL,\n `cbahbbgccmnarcwnsrppycctsccxxden` int DEFAULT NULL,\n `ogabiiqlbazzlhwjxnynhvfwapfcatus` int DEFAULT NULL,\n `wkyoinkdwfziqokhtsgocwokpjivvymj` int DEFAULT NULL,\n PRIMARY KEY (`aqvpvqqneggxofjqrspisgguzvhvbzrt`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"rdcfwajpujmymsqszcnekivknqxdoqil\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["aqvpvqqneggxofjqrspisgguzvhvbzrt"],"columns":[{"name":"aqvpvqqneggxofjqrspisgguzvhvbzrt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"rgyqgazfklklfogtrqyruwdfdlzzcegi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqddtfqggyykdgzvilxijekbyvqqchis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iyghyqgalnjmttllzcmythgmdgggicgt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvqfjysbaffzuubsupezhccqqvdjrmjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pzonlmgdvsryfdhenttgtstnwgktrykd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dupeujtpihnkydmjqpmzynfnxuinlvcy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqzhqttdxbbojyzweilqmvfuvbdjfpwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krsbngaerrfvogglkxolmrrzlznrbagb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qsudbjnihsvlsmktkjsxqgmxoulhnsto","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mugposrkwvpakczqhnvnohvqefhauhwx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftcilfsakxwcxpmtlvrvfafetvhwklly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gazqrfdchkbkghkyjgqzvqhtaffvcywc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umuwjyatknqnilbvdaiunhmdhjsmdpkr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqfmuqlxfvrhtxooeixujaifigqpuggb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chajsiygnaqumqzuycvygncwnynmpjyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"deyozguiuoyctqkpvtmixqxlmxkkxhqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lrflszbatymxunomowxrwplxfhzhgkbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkabtgqytqinvghyqrezmtttogeqewbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khzefnvnjmrqzfsjcgmwpcntgohrmuqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujlgfofrqjbsrmfrebtycyfaqmkcqcig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cscvrrgirncqbvqyfjuaoarryexldhxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfqawepcspzzhzpgrdotdighruuuhoqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nnxzqwwoyssuqwcwfxeuzlbvtywpsvgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zusnboikzqrzlzwvbmelbgxwfzeocytm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eodlhgkpuintldnmbvyqsazpdcdvsssc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"guejtvayyipdncqapgcjknxhhfhevscn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmvbqxufpsmdelcqplybzachkrknqkrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzycbtpjtulbmjjdcertviqdkyjhplaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"onhfcqmncslwtvtvfhivglrachielbyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfrypfcabugpdihzejwmsnxdyhsbkdsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxpndviavqfudwlppwuoilkuodckoiui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfkbudcgzdeevclobryvtdgciiszsrag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ngfvukuotrpbgbqkrjnmdossroqadiuf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ptofqrjgvwgihntenructrvgbwjafxxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmenwexdfxdqaffybfxkkjfmevaxwfia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zotxxqavxbqsmfdxxfftcxqhuebggkqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yselqlpivbguyneocwhdnwncwxkqjzxm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ldvwvsbsnvlgesfbhzcorfpftnddazpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qkdtxuigbgpqjlslskqtwahucqiduuvs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ukwohaeeocyfycdxsdmynkwcnkeclroz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oylhvdzfjncwdntnhlygrllghajzwhxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejgesinazqsmkziqsxwzpgnphhpobxbl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jaikhnkiazaeiloxsilsgulxcccmcwkk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkgkyirnpgidfyqezngbiqprukdeeuxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"govdjksjhknwoplxwcmzldmevgybqzki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qergymeebwrmvoqqkwjqxuxfowyfaqlu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkojpcnsswvjofrtuxezwtyhjjxcbrwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"faebwsfxbyqjvqakkgvpvymiqeepdhyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"odypbpvhleumusvamehjywdqhuzmhvgv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gpaivdfwnvswavwjscgililnwtgukwjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpaxxmnzjrdlsjywwkozevdwybemtodd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dauvcaccdnvjkufrzdykmxziwpxosxhh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzzyndctepkfrpmaskcvyrkicxulhnru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbsdoapdyqwcladmkzqpttywygfnvthj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vyhkdcxlysnxdgvquqtmtbjioiyktijs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjjvsyuouacbvstfvmnrclqbawukzequ","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nirokglukqdyscagofhfsoxmvbdushvh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjamtfhyjtvkogitaqplqcpqliyffoek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uiigfzvepvhfafoawuqleluoiaceplni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofkkjsznwfanhtzeytqtlalaclsektej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhvppcvwwxwasikpuxlvztngtmwsyjdo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fshwxkfhsnfaudweihunkgaelmopnygu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njlzzprwqiztdnwoecccmvlsgscarvli","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lummretmxbfpxgfzaqzvkhiqqbscdhgj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytlfybbsgfcfkukrzsjkjqkdtiemlrkx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"egogmunyuliqcoodenoqnmbpvneluupx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thghlixpsbhwfjpwkowcuwzuiyeqdyui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"efhblvkuhpudnwtxqfcozciogzzjfqje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzbxbqwprvlsvifsypwdgujvgjdfseoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cawdljvueghintwbfyksynquqarnszvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjvzgfnbtpapvpdhsupqwgfbjixesksj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tygqxjlorekzvknikuypboqqugahvqyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"notwrdgeweswvewxyfbtdttathemkgft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtusxrhgtmytexmypwqzcueeovpwibla","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sirxmogrqrtqjmxrlbedmhzyhrbpumsf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kixdhivtzkkjsdqruikhucyxehakfcpt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"isdukbwdatgjcngxqmdkctgkrimqpbik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shqmhwllhnetxrmlqjrwczrbuuodscqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqkomnzmaszrviaxbfvccnlyjtznehgp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnqnuyuytleontqcfgylrmswsyofjjtd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ldarpeilgytpwirxybzscmrsrggwlmlg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjqozzblwugyzvwximjgrdhomcghvhjt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqsrahhyttzxbixrfvpvtekoweazokgs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhtirqldtidcbbmytmnveioivoihypcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aeftysthpaixmrjefqlrsqpgjkxdqzmp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sazhchoxdbgjntpjjkugpjkucnfnxfdo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mamhppcpuulsyhgfsxqipogofyucuylu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uspqeemioqpdlblcgmazetuvldnoggxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbnylszjapfpgfifgyirrqjdcjlnpcgz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pifntpnouozanvcpxljjockrycpzzaml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ldoohrakwsidahuxuqnhigymnonzvaig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qlcfrbxkgcztrcytshogeezvmmdwrhas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgumwjxpnnvdotxncqsyueyofaqipdll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vudzjbicpgturbnckylixhktqcmmeqiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtpywsqihyiqgtdkaczsgmtdgjxulkdm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgvwoxrqufnkazglyofroidsamnxzvta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbahbbgccmnarcwnsrppycctsccxxden","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ogabiiqlbazzlhwjxnynhvfwapfcatus","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wkyoinkdwfziqokhtsgocwokpjivvymj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670704,"databaseName":"models_schema","ddl":"CREATE TABLE `rgjeprxzblxygbhnumnnjlryntqzuony` (\n `vcqwhzuiczusgfbfcizkxxhxvhpwlspc` int NOT NULL,\n `ghglyohfzwyrizmsfgvzbuxzuihhzdag` int DEFAULT NULL,\n `yzvkdpowdpksgqxwdhdmrbqcaqqhmnzy` int DEFAULT NULL,\n `nygrcfsyoplhlsfnmlzmevlbmircdbkd` int DEFAULT NULL,\n `tvxpkilrfyppvmcrakzbmhjxxoiuosiv` int DEFAULT NULL,\n `mzjwbvspdzstzpesjjxrvqwjfszxhivw` int DEFAULT NULL,\n `uodxgciheramqxeziwrpyvfhdwoxbbao` int DEFAULT NULL,\n `zavryoixzlfituqlfsbnjqwffjyadxpk` int DEFAULT NULL,\n `pukgcqrxhqbcchaovgrpymqfdpwhrqje` int DEFAULT NULL,\n `dextbehrfcuieqgkcktyobccsvocrrjx` int DEFAULT NULL,\n `cxnqreoookiogolbxgracrtcireagxca` int DEFAULT NULL,\n `uqdcvnwxykmzuevgvxapskbsafjzrrpv` int DEFAULT NULL,\n `hyrungnhmjzceqroxwiaezsbbwrtvxeq` int DEFAULT NULL,\n `zxcgkpfauxpghobqaumghvxejxsixxzv` int DEFAULT NULL,\n `etwwazumljohombqsztodudllmderpby` int DEFAULT NULL,\n `wnijrnlemoaptewradzovwpbvmldhqei` int DEFAULT NULL,\n `tljinwiccoejiabmbuwioorqrlohjcps` int DEFAULT NULL,\n `gcphefmwdotkilbqvmfmunatuydzgrch` int DEFAULT NULL,\n `bzssxiqjaexfufgaghiwvrteatzbtlyl` int DEFAULT NULL,\n `nzvmvxcbfpnvvqvhmowcsmhqqrybjoao` int DEFAULT NULL,\n `mzkrhkjnvptqplzykaqwwvftumyjlrgy` int DEFAULT NULL,\n `quoxzqihwkkpowldvdpnzdfpdgobnlpk` int DEFAULT NULL,\n `yprfzsunojkmtzhxilibbgcickgajuyh` int DEFAULT NULL,\n `hfyjkbreilyczdvrqyreyevxyqnsmbbr` int DEFAULT NULL,\n `fxmkxenhjfaesktpcykqeqkbkcmnurke` int DEFAULT NULL,\n `bvumrgjsqetkzxjypqsidbxouohlflpr` int DEFAULT NULL,\n `jpsdlqnqeazlmcqrvwifbeoeztomecvy` int DEFAULT NULL,\n `hyxfwreypfzrdrxjwmmjfoojvaibcbms` int DEFAULT NULL,\n `athekmatwollqsmgdhnrcukifkofpnxx` int DEFAULT NULL,\n `hiunbfwtkfpmkmwwnbxcwpvkqvmlvsll` int DEFAULT NULL,\n `grohgesznywvooozfpsozyllotzsfsoa` int DEFAULT NULL,\n `lwzkpihgpdotjoyjphqnxvijfesydlfk` int DEFAULT NULL,\n `hdomkaujeypxysnkzntamuawbgciddxb` int DEFAULT NULL,\n `jwbuxklbeyptoletjlsvnhredzscfhsk` int DEFAULT NULL,\n `lctcgsgnimlhpaeqmejxrythbwsvdyam` int DEFAULT NULL,\n `rxtmmychnmskzlhbqzvvoyyvtehwqcfx` int DEFAULT NULL,\n `ghnnlcpjbewevrznskwexcrlnijayprv` int DEFAULT NULL,\n `xebfomunclewemmqubrwtyqlbhwpobsa` int DEFAULT NULL,\n `vibuybputjgrezbivcsyjeexryaogkqm` int DEFAULT NULL,\n `qqobkyhoaauuxtybyntelgkzjekkkdji` int DEFAULT NULL,\n `iyvhotrfmpmskogetdcontcnphvclcoi` int DEFAULT NULL,\n `csmhkplzenkzaptfptlveqwynlscgutn` int DEFAULT NULL,\n `qvxjemblwmensebpfykapkfcoanshmea` int DEFAULT NULL,\n `vjsrrdswnqfzpixyyrugmzkayntwlyhq` int DEFAULT NULL,\n `mduqagolopxzneslzstblzusacwaurlt` int DEFAULT NULL,\n `aoogiajvlbcoprvwlvdqabsbwmmxnnai` int DEFAULT NULL,\n `epkzowtxplwgxnxowpmoxevgjarnmaep` int DEFAULT NULL,\n `nxpddwzfmcjcxundbkdlahilacqqdlcm` int DEFAULT NULL,\n `wqddhxggakhxnuxbnosoqvddzlexzgcv` int DEFAULT NULL,\n `gffxtylpdfojuzezhkafixlrtlbskuoa` int DEFAULT NULL,\n `bcijuqwcxpddipaxfoekxkdmlmqcfgef` int DEFAULT NULL,\n `apssfhmmyhbkqczfrdrfoxgnrlfmxtmd` int DEFAULT NULL,\n `qeogfywbjhdwvidbispbxndhfrnrdmwd` int DEFAULT NULL,\n `mwvaxxkjwodpfheiephxhxryptcpikbw` int DEFAULT NULL,\n `fetifvonwothikelhhxuphlfgfxzxzyo` int DEFAULT NULL,\n `mtwyqjpmkvhbkhhiisffhixyhjbwxajt` int DEFAULT NULL,\n `nwekiuzuajfskbvbvcqcaytqaqppvchx` int DEFAULT NULL,\n `ftyplrtcjbowipiyazixlevvdvnnwvxr` int DEFAULT NULL,\n `cnprfjaprchfdfbkaxofqcddwxmmfnpy` int DEFAULT NULL,\n `tmbwpdfjteyuknqdypnqerqjucjhetsr` int DEFAULT NULL,\n `opgvxssuicocbtqnbzxgdhyvydnbxddo` int DEFAULT NULL,\n `plmkleqqkmiwaobfmrgizomwcwgjwqnr` int DEFAULT NULL,\n `hydbuvjhqspojtnxgzveticjtntmckws` int DEFAULT NULL,\n `uzmvwhnwksofavmfdfeaxlrsbrzgostc` int DEFAULT NULL,\n `hudiffutipfmvypmupissebfugsayvvl` int DEFAULT NULL,\n `ovtrzoonchiplgxrlgchuvyxhtiklokl` int DEFAULT NULL,\n `wwlkqllwdakrykuvuomoeuqmqylquagf` int DEFAULT NULL,\n `olsudpkiwdcmxtebbwosueryrezaslwo` int DEFAULT NULL,\n `ybxkpernpqxppgoirhticqamdfqsqdbi` int DEFAULT NULL,\n `nrayyffrcluzjlmpwgstyrwggsxeubbb` int DEFAULT NULL,\n `ghaswugnbijybetjdtxoejxxyryhjgbw` int DEFAULT NULL,\n `seeewtdnroqbyhqdiefpxzbosnrhbgmi` int DEFAULT NULL,\n `ptsthfqmuwwaxojgfbyaubbgiaqtwfxw` int DEFAULT NULL,\n `mukhikntbpjitxnsgvfinbaepyupihfu` int DEFAULT NULL,\n `jdmtkmzhgihrxzchkeemehxwdhdvcbwu` int DEFAULT NULL,\n `kgejmgiukaadjxylklyejtinwfhbmydo` int DEFAULT NULL,\n `npfkusvojwdgomzelydvjqxkebxgxlds` int DEFAULT NULL,\n `okiygniwcdtaxivqpfabqenvynsyrhod` int DEFAULT NULL,\n `pdfnpfnmganduiyfscdkwgfzymggsphz` int DEFAULT NULL,\n `qwmtnkdqgfghdzqscncebfidrywntuih` int DEFAULT NULL,\n `rijjyihdieydzioiguoyufmfrgwbrgil` int DEFAULT NULL,\n `zxrahxkwlapnwrrkwqmkwmcxgidudhau` int DEFAULT NULL,\n `mqhnygzsbmxkcsptlngdmqhqlluazeaa` int DEFAULT NULL,\n `sxzovqbvvvkislofeloohpdwhttixvme` int DEFAULT NULL,\n `yscfinxiawvbrkqkqtgjwsriomozskxe` int DEFAULT NULL,\n `cwydixfotsvljqhtesaxkdpevkjgdjgg` int DEFAULT NULL,\n `krwiwfhhszbequtrrccowhsbkqvrxrfb` int DEFAULT NULL,\n `qzbkjfuxyqetvofuqfgnglfshgxedwxl` int DEFAULT NULL,\n `neragabtsembuerjyqxnwcfzsmhpwius` int DEFAULT NULL,\n `orgmxgvcomqchoowvnvzoibwwydmddrn` int DEFAULT NULL,\n `ijteqvcmzitdimiwpwbkiganvaknpyyz` int DEFAULT NULL,\n `wppczijhfwsulhhtdvaovmslymhxhyhq` int DEFAULT NULL,\n `jjmoipfajxpizwbdgvjarcjcsocrslms` int DEFAULT NULL,\n `drmjubwyqumzfxfehnbvqflwdsjixsdh` int DEFAULT NULL,\n `fqcwhybqqlosfmdeynopcwyrjyrxipea` int DEFAULT NULL,\n `rnjxtprgllfkifwvxbzhmbbpncnazbcr` int DEFAULT NULL,\n `vspdccsulyqedasmnqimvnupuulkrayz` int DEFAULT NULL,\n `zzqqkkvknfpntilqnhehvtvpkntmzqag` int DEFAULT NULL,\n `wprxglegvybnpaecrsgmiubgfinefrwh` int DEFAULT NULL,\n `dqgzomyraftwkpdgldsbnnzyfugykyhw` int DEFAULT NULL,\n PRIMARY KEY (`vcqwhzuiczusgfbfcizkxxhxvhpwlspc`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"rgjeprxzblxygbhnumnnjlryntqzuony\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["vcqwhzuiczusgfbfcizkxxhxvhpwlspc"],"columns":[{"name":"vcqwhzuiczusgfbfcizkxxhxvhpwlspc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ghglyohfzwyrizmsfgvzbuxzuihhzdag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzvkdpowdpksgqxwdhdmrbqcaqqhmnzy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nygrcfsyoplhlsfnmlzmevlbmircdbkd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvxpkilrfyppvmcrakzbmhjxxoiuosiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzjwbvspdzstzpesjjxrvqwjfszxhivw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uodxgciheramqxeziwrpyvfhdwoxbbao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zavryoixzlfituqlfsbnjqwffjyadxpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pukgcqrxhqbcchaovgrpymqfdpwhrqje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dextbehrfcuieqgkcktyobccsvocrrjx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cxnqreoookiogolbxgracrtcireagxca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqdcvnwxykmzuevgvxapskbsafjzrrpv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hyrungnhmjzceqroxwiaezsbbwrtvxeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxcgkpfauxpghobqaumghvxejxsixxzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"etwwazumljohombqsztodudllmderpby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnijrnlemoaptewradzovwpbvmldhqei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tljinwiccoejiabmbuwioorqrlohjcps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gcphefmwdotkilbqvmfmunatuydzgrch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzssxiqjaexfufgaghiwvrteatzbtlyl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzvmvxcbfpnvvqvhmowcsmhqqrybjoao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzkrhkjnvptqplzykaqwwvftumyjlrgy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"quoxzqihwkkpowldvdpnzdfpdgobnlpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yprfzsunojkmtzhxilibbgcickgajuyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfyjkbreilyczdvrqyreyevxyqnsmbbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxmkxenhjfaesktpcykqeqkbkcmnurke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvumrgjsqetkzxjypqsidbxouohlflpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpsdlqnqeazlmcqrvwifbeoeztomecvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hyxfwreypfzrdrxjwmmjfoojvaibcbms","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"athekmatwollqsmgdhnrcukifkofpnxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hiunbfwtkfpmkmwwnbxcwpvkqvmlvsll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grohgesznywvooozfpsozyllotzsfsoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwzkpihgpdotjoyjphqnxvijfesydlfk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdomkaujeypxysnkzntamuawbgciddxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwbuxklbeyptoletjlsvnhredzscfhsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lctcgsgnimlhpaeqmejxrythbwsvdyam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxtmmychnmskzlhbqzvvoyyvtehwqcfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghnnlcpjbewevrznskwexcrlnijayprv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xebfomunclewemmqubrwtyqlbhwpobsa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vibuybputjgrezbivcsyjeexryaogkqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqobkyhoaauuxtybyntelgkzjekkkdji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iyvhotrfmpmskogetdcontcnphvclcoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"csmhkplzenkzaptfptlveqwynlscgutn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvxjemblwmensebpfykapkfcoanshmea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjsrrdswnqfzpixyyrugmzkayntwlyhq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mduqagolopxzneslzstblzusacwaurlt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aoogiajvlbcoprvwlvdqabsbwmmxnnai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"epkzowtxplwgxnxowpmoxevgjarnmaep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxpddwzfmcjcxundbkdlahilacqqdlcm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqddhxggakhxnuxbnosoqvddzlexzgcv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gffxtylpdfojuzezhkafixlrtlbskuoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bcijuqwcxpddipaxfoekxkdmlmqcfgef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"apssfhmmyhbkqczfrdrfoxgnrlfmxtmd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qeogfywbjhdwvidbispbxndhfrnrdmwd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwvaxxkjwodpfheiephxhxryptcpikbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fetifvonwothikelhhxuphlfgfxzxzyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtwyqjpmkvhbkhhiisffhixyhjbwxajt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nwekiuzuajfskbvbvcqcaytqaqppvchx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftyplrtcjbowipiyazixlevvdvnnwvxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnprfjaprchfdfbkaxofqcddwxmmfnpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmbwpdfjteyuknqdypnqerqjucjhetsr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opgvxssuicocbtqnbzxgdhyvydnbxddo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"plmkleqqkmiwaobfmrgizomwcwgjwqnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hydbuvjhqspojtnxgzveticjtntmckws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzmvwhnwksofavmfdfeaxlrsbrzgostc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hudiffutipfmvypmupissebfugsayvvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovtrzoonchiplgxrlgchuvyxhtiklokl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwlkqllwdakrykuvuomoeuqmqylquagf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"olsudpkiwdcmxtebbwosueryrezaslwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybxkpernpqxppgoirhticqamdfqsqdbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nrayyffrcluzjlmpwgstyrwggsxeubbb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghaswugnbijybetjdtxoejxxyryhjgbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"seeewtdnroqbyhqdiefpxzbosnrhbgmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ptsthfqmuwwaxojgfbyaubbgiaqtwfxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mukhikntbpjitxnsgvfinbaepyupihfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdmtkmzhgihrxzchkeemehxwdhdvcbwu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kgejmgiukaadjxylklyejtinwfhbmydo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npfkusvojwdgomzelydvjqxkebxgxlds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okiygniwcdtaxivqpfabqenvynsyrhod","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdfnpfnmganduiyfscdkwgfzymggsphz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwmtnkdqgfghdzqscncebfidrywntuih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rijjyihdieydzioiguoyufmfrgwbrgil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxrahxkwlapnwrrkwqmkwmcxgidudhau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqhnygzsbmxkcsptlngdmqhqlluazeaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sxzovqbvvvkislofeloohpdwhttixvme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yscfinxiawvbrkqkqtgjwsriomozskxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwydixfotsvljqhtesaxkdpevkjgdjgg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krwiwfhhszbequtrrccowhsbkqvrxrfb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzbkjfuxyqetvofuqfgnglfshgxedwxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"neragabtsembuerjyqxnwcfzsmhpwius","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orgmxgvcomqchoowvnvzoibwwydmddrn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijteqvcmzitdimiwpwbkiganvaknpyyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wppczijhfwsulhhtdvaovmslymhxhyhq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjmoipfajxpizwbdgvjarcjcsocrslms","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drmjubwyqumzfxfehnbvqflwdsjixsdh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqcwhybqqlosfmdeynopcwyrjyrxipea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnjxtprgllfkifwvxbzhmbbpncnazbcr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vspdccsulyqedasmnqimvnupuulkrayz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzqqkkvknfpntilqnhehvtvpkntmzqag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wprxglegvybnpaecrsgmiubgfinefrwh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqgzomyraftwkpdgldsbnnzyfugykyhw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670734,"databaseName":"models_schema","ddl":"CREATE TABLE `rjjpcxwartpnkawkhzhlboqncbbgzzgx` (\n `hjizdaquosxzmzbjgslgwcrnleintnsq` int NOT NULL,\n `xcenpsvsmpbciosaohftvdfjsfqynypt` int DEFAULT NULL,\n `apmsgsaavmxcfryomwwumtsbiofzvyyo` int DEFAULT NULL,\n `yyfarkmcsyuihmiqwxenuhqoqtymtcmy` int DEFAULT NULL,\n `bnlvhpugjpeufmlvgcltzwwsakvhbxni` int DEFAULT NULL,\n `srowhvkxmoxkslloihoosrvnvqecozhw` int DEFAULT NULL,\n `qftpuhmlpznthzhehcqknlzpxlqvsqgh` int DEFAULT NULL,\n `ytkarogfifqvjlkzuvdjttrrdvtkkmpx` int DEFAULT NULL,\n `asopwampvbbncsqhllsrpqvenrumdyao` int DEFAULT NULL,\n `dnylnmajwcobhavarmjikvdvwnioneea` int DEFAULT NULL,\n `zuuyskkjgvlqvzqmoxohmmurkjcapmgm` int DEFAULT NULL,\n `ufavybhylugwxieenzybhvxtxkiwstym` int DEFAULT NULL,\n `dozzbfkxdaojlfhnwrukfgvjxzhbronn` int DEFAULT NULL,\n `ficrcittgrqlzqtpsdemchdcnlkogjzp` int DEFAULT NULL,\n `gksydyxalkrcbmqjpsrqinszhrdzgssb` int DEFAULT NULL,\n `zbkbcijbsvrotkmcqigxznbvsmrsffbb` int DEFAULT NULL,\n `qggtyjjwkpgmplfuyermabgbttknzrtg` int DEFAULT NULL,\n `xcjczlbgmgibqpauqkjjwisfkcvnqjvc` int DEFAULT NULL,\n `bksakkykiiqsqobhmvlmuuiomskzxklk` int DEFAULT NULL,\n `rlvmfqotqhlzcrkscbcynkcymnxvwhot` int DEFAULT NULL,\n `gkvruxpvlipcpvpvrrtqysijjvdtthpm` int DEFAULT NULL,\n `xeojxevdsjfxzkhogdwmtwhfqbviyqos` int DEFAULT NULL,\n `urxizobzyqhhafzviyhjmdccdjcjqvwh` int DEFAULT NULL,\n `xxdyhewojajujmzzfuzfxbovcxynoyoz` int DEFAULT NULL,\n `ncykivqgabfckeajshbiumxjiskzbbgu` int DEFAULT NULL,\n `vpuvckzkrqjubbqdhgrugmqlivtwcvbr` int DEFAULT NULL,\n `xbotwsarpqlhrhpsdcbpbyhjwdawqfbm` int DEFAULT NULL,\n `fzgjdgdkcfajrepxxgcvenaulsukohbg` int DEFAULT NULL,\n `jdyizqnroghmacjugvjmqdowcwyktiqv` int DEFAULT NULL,\n `hldzeqctkdpajiakvqvxnysnvmmmfgnt` int DEFAULT NULL,\n `boiwjphbmcglamtvyiiqfdwypysafzfh` int DEFAULT NULL,\n `vwzuecmampepmkssqpdomdybfnnjomwf` int DEFAULT NULL,\n `qukulczchzushljnaxzleiqwasibxcft` int DEFAULT NULL,\n `etofhgdimsutvteuedyeefhbfolrrwty` int DEFAULT NULL,\n `fkwyqacvihuvcpbqkervtmkxisbbdbse` int DEFAULT NULL,\n `htefmoaydvmqavlmlkxivaftzkqfoclj` int DEFAULT NULL,\n `gwvwigfjofdasalmwnrwcqzvmxmzoktt` int DEFAULT NULL,\n `gmvlbptqtuehouohnzzzesvgeubpqilz` int DEFAULT NULL,\n `emzuvarsmqcojbndxhnpmfchufoueabs` int DEFAULT NULL,\n `gkuaoopfemauatkujvbwthfyxaoacwth` int DEFAULT NULL,\n `icteaggvtdgxdrllgggphwivggoisyjn` int DEFAULT NULL,\n `aucuprruglxjlixjxgjrbkeheykmerrs` int DEFAULT NULL,\n `avutuzxjkxgwqhuonkcwlhjxfoajaugq` int DEFAULT NULL,\n `rakrlzxuwvrihcmclwimnimpsfnspazj` int DEFAULT NULL,\n `srisaoisxylixvagsitrjgesnpfofflt` int DEFAULT NULL,\n `botflczddeswtzfqcqvgmccgyzkfbpcl` int DEFAULT NULL,\n `rfdhfnmpldnijlkxdyoahfefzveyllib` int DEFAULT NULL,\n `nfgyqmzhvkymaobytlqolhbdsdfdtbkm` int DEFAULT NULL,\n `xdfadrgskhhjucbyiwlwvtuvyvadwpqr` int DEFAULT NULL,\n `scoiwhjrbeenvuvehzigrhgugaykimhz` int DEFAULT NULL,\n `ajzcjwzaczdebwpiowwxwiimcqauzhsy` int DEFAULT NULL,\n `mibnpppjfdkgolmrbcbrkmqspdjvmgmf` int DEFAULT NULL,\n `ujrvocmdtywdgblwgqjrzasifbwtjolm` int DEFAULT NULL,\n `dbvqvcbkcigpfnktlriizuiqettimpke` int DEFAULT NULL,\n `zgrnbpiscpvyfgnfgmhhhvfqrxjpfwpf` int DEFAULT NULL,\n `wvdlirmpyopdettczunjsjlhlzbnixer` int DEFAULT NULL,\n `mpwjckyjylvwmonarlnwqoysnkzbtzab` int DEFAULT NULL,\n `ajwvgjhntjibtxjaheyedihlgdpjjhaz` int DEFAULT NULL,\n `gludyixmghgsbdyvolezdenalhmuenoy` int DEFAULT NULL,\n `fmauotofyakmbxramsympdtctyyjgdwg` int DEFAULT NULL,\n `daapcomnxvjqkjubqryblwkykkwesbpy` int DEFAULT NULL,\n `bkyuaaxohqpmrudqqrtcqcjqtbfubycq` int DEFAULT NULL,\n `yxlnwbfomecetsxzkkonhgfrjvvmlfjc` int DEFAULT NULL,\n `judfmszzcdrluvbizofyjyccecooglhi` int DEFAULT NULL,\n `owckhkelrkvrksciwlkjjzeoxwfqgobm` int DEFAULT NULL,\n `dfqcysxdfcpueiooxcbnbzrwyuflrudr` int DEFAULT NULL,\n `motdcfsdpxnlnvefheyulmljnzrkhhix` int DEFAULT NULL,\n `opvnwwbofrmedvibvfaybeipqklfqdco` int DEFAULT NULL,\n `hrhrcpwginnwpsukzzgqlsaerlncpxpr` int DEFAULT NULL,\n `zdqchhugstmvlhqxdufyqmmrescqgajg` int DEFAULT NULL,\n `nemexsbagdzzzgyitsmshqwilvdocrxd` int DEFAULT NULL,\n `vaihmmcqybyqwaqofizabolsecpflmyj` int DEFAULT NULL,\n `engigqvargxowlcmcdxrjgimhvkkyghq` int DEFAULT NULL,\n `uontujqiqedggcwmagqwbbuxlxiwtakh` int DEFAULT NULL,\n `ampptdolnicgsqmutttbutbpjtqxseka` int DEFAULT NULL,\n `csmioyhwsbglrfdffxyytreatdhhorfr` int DEFAULT NULL,\n `kppquprsoceigecnfhywvninijlhdpvc` int DEFAULT NULL,\n `qyequjezfmzgyahuhdnneotmykqcdtdw` int DEFAULT NULL,\n `qbtpgqfqhzbksnxshvrooutmssctnpfj` int DEFAULT NULL,\n `vckljscrmztuufporrsdxfpezetajdqx` int DEFAULT NULL,\n `huarsntmcfocphkrucvohlwvnrulrpze` int DEFAULT NULL,\n `mrwgyqxcqbkwpydoylrannscixlkkhna` int DEFAULT NULL,\n `eyzbhotgfckguaypzdnwekqosfesyvnf` int DEFAULT NULL,\n `ibfweaihpqwzhvfxbwehdntgxcjsrxat` int DEFAULT NULL,\n `fnhfnnjfnsnkomzgcqwbofwvminbhihe` int DEFAULT NULL,\n `dtnproeqkbajfghfyjnpyuscceexmqux` int DEFAULT NULL,\n `jaakzolwxrormdbmqmvizganklzewurn` int DEFAULT NULL,\n `nogbwasptiyhovdghgvkijawumqbqloo` int DEFAULT NULL,\n `jzewaldzrqzefvqvnoqtmsacyezxbdux` int DEFAULT NULL,\n `qsnepmvzbuqptkdqnrnmbvdehcdkxwum` int DEFAULT NULL,\n `rhzunlngjasuekmfqridqjydtbprusrc` int DEFAULT NULL,\n `zqqrrfosxuvtydcrbgqfqhafmsmifoiq` int DEFAULT NULL,\n `xtdcsbnljgbubtaajgtdydifyierpmbj` int DEFAULT NULL,\n `kolhdgnpatilvavrjbdeumjqnukfbfpd` int DEFAULT NULL,\n `lswckwtwwbmhusyrmmglszuoggzpkodu` int DEFAULT NULL,\n `mhulijnywvlikxgxyoforcvosoxymhib` int DEFAULT NULL,\n `ydsddmxfruiwwrwrvdecwsdtakszgosp` int DEFAULT NULL,\n `zaiqcwnszuygeznhqyityvgmmtuvqevk` int DEFAULT NULL,\n `pwtdzfmwihkakmuhilmfnnoktdmkmekc` int DEFAULT NULL,\n `qeowxhkxlfsmnkmwsvrsgeyhydmvrlts` int DEFAULT NULL,\n PRIMARY KEY (`hjizdaquosxzmzbjgslgwcrnleintnsq`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"rjjpcxwartpnkawkhzhlboqncbbgzzgx\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["hjizdaquosxzmzbjgslgwcrnleintnsq"],"columns":[{"name":"hjizdaquosxzmzbjgslgwcrnleintnsq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"xcenpsvsmpbciosaohftvdfjsfqynypt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"apmsgsaavmxcfryomwwumtsbiofzvyyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yyfarkmcsyuihmiqwxenuhqoqtymtcmy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnlvhpugjpeufmlvgcltzwwsakvhbxni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"srowhvkxmoxkslloihoosrvnvqecozhw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qftpuhmlpznthzhehcqknlzpxlqvsqgh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytkarogfifqvjlkzuvdjttrrdvtkkmpx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"asopwampvbbncsqhllsrpqvenrumdyao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dnylnmajwcobhavarmjikvdvwnioneea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zuuyskkjgvlqvzqmoxohmmurkjcapmgm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufavybhylugwxieenzybhvxtxkiwstym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dozzbfkxdaojlfhnwrukfgvjxzhbronn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ficrcittgrqlzqtpsdemchdcnlkogjzp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gksydyxalkrcbmqjpsrqinszhrdzgssb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbkbcijbsvrotkmcqigxznbvsmrsffbb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qggtyjjwkpgmplfuyermabgbttknzrtg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xcjczlbgmgibqpauqkjjwisfkcvnqjvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bksakkykiiqsqobhmvlmuuiomskzxklk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlvmfqotqhlzcrkscbcynkcymnxvwhot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkvruxpvlipcpvpvrrtqysijjvdtthpm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xeojxevdsjfxzkhogdwmtwhfqbviyqos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urxizobzyqhhafzviyhjmdccdjcjqvwh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxdyhewojajujmzzfuzfxbovcxynoyoz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ncykivqgabfckeajshbiumxjiskzbbgu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpuvckzkrqjubbqdhgrugmqlivtwcvbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbotwsarpqlhrhpsdcbpbyhjwdawqfbm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzgjdgdkcfajrepxxgcvenaulsukohbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdyizqnroghmacjugvjmqdowcwyktiqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hldzeqctkdpajiakvqvxnysnvmmmfgnt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"boiwjphbmcglamtvyiiqfdwypysafzfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vwzuecmampepmkssqpdomdybfnnjomwf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qukulczchzushljnaxzleiqwasibxcft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"etofhgdimsutvteuedyeefhbfolrrwty","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fkwyqacvihuvcpbqkervtmkxisbbdbse","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htefmoaydvmqavlmlkxivaftzkqfoclj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gwvwigfjofdasalmwnrwcqzvmxmzoktt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmvlbptqtuehouohnzzzesvgeubpqilz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"emzuvarsmqcojbndxhnpmfchufoueabs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkuaoopfemauatkujvbwthfyxaoacwth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icteaggvtdgxdrllgggphwivggoisyjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aucuprruglxjlixjxgjrbkeheykmerrs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avutuzxjkxgwqhuonkcwlhjxfoajaugq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rakrlzxuwvrihcmclwimnimpsfnspazj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"srisaoisxylixvagsitrjgesnpfofflt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"botflczddeswtzfqcqvgmccgyzkfbpcl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfdhfnmpldnijlkxdyoahfefzveyllib","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfgyqmzhvkymaobytlqolhbdsdfdtbkm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdfadrgskhhjucbyiwlwvtuvyvadwpqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scoiwhjrbeenvuvehzigrhgugaykimhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajzcjwzaczdebwpiowwxwiimcqauzhsy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mibnpppjfdkgolmrbcbrkmqspdjvmgmf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujrvocmdtywdgblwgqjrzasifbwtjolm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbvqvcbkcigpfnktlriizuiqettimpke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zgrnbpiscpvyfgnfgmhhhvfqrxjpfwpf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvdlirmpyopdettczunjsjlhlzbnixer","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpwjckyjylvwmonarlnwqoysnkzbtzab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajwvgjhntjibtxjaheyedihlgdpjjhaz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gludyixmghgsbdyvolezdenalhmuenoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmauotofyakmbxramsympdtctyyjgdwg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"daapcomnxvjqkjubqryblwkykkwesbpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkyuaaxohqpmrudqqrtcqcjqtbfubycq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxlnwbfomecetsxzkkonhgfrjvvmlfjc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"judfmszzcdrluvbizofyjyccecooglhi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owckhkelrkvrksciwlkjjzeoxwfqgobm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dfqcysxdfcpueiooxcbnbzrwyuflrudr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"motdcfsdpxnlnvefheyulmljnzrkhhix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opvnwwbofrmedvibvfaybeipqklfqdco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hrhrcpwginnwpsukzzgqlsaerlncpxpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdqchhugstmvlhqxdufyqmmrescqgajg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nemexsbagdzzzgyitsmshqwilvdocrxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vaihmmcqybyqwaqofizabolsecpflmyj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"engigqvargxowlcmcdxrjgimhvkkyghq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uontujqiqedggcwmagqwbbuxlxiwtakh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ampptdolnicgsqmutttbutbpjtqxseka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"csmioyhwsbglrfdffxyytreatdhhorfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kppquprsoceigecnfhywvninijlhdpvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qyequjezfmzgyahuhdnneotmykqcdtdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qbtpgqfqhzbksnxshvrooutmssctnpfj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vckljscrmztuufporrsdxfpezetajdqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"huarsntmcfocphkrucvohlwvnrulrpze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrwgyqxcqbkwpydoylrannscixlkkhna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eyzbhotgfckguaypzdnwekqosfesyvnf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibfweaihpqwzhvfxbwehdntgxcjsrxat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnhfnnjfnsnkomzgcqwbofwvminbhihe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtnproeqkbajfghfyjnpyuscceexmqux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jaakzolwxrormdbmqmvizganklzewurn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nogbwasptiyhovdghgvkijawumqbqloo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzewaldzrqzefvqvnoqtmsacyezxbdux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qsnepmvzbuqptkdqnrnmbvdehcdkxwum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhzunlngjasuekmfqridqjydtbprusrc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqqrrfosxuvtydcrbgqfqhafmsmifoiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xtdcsbnljgbubtaajgtdydifyierpmbj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kolhdgnpatilvavrjbdeumjqnukfbfpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lswckwtwwbmhusyrmmglszuoggzpkodu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhulijnywvlikxgxyoforcvosoxymhib","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydsddmxfruiwwrwrvdecwsdtakszgosp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zaiqcwnszuygeznhqyityvgmmtuvqevk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwtdzfmwihkakmuhilmfnnoktdmkmekc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qeowxhkxlfsmnkmwsvrsgeyhydmvrlts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670765,"databaseName":"models_schema","ddl":"CREATE TABLE `rklsysjbbggcyaheggpsezportemxlmu` (\n `torizvmmijgolgkoxkawvfdpjutslegi` int NOT NULL,\n `ypqorfecxnpcbqzxieycahvzspjwwrng` int DEFAULT NULL,\n `srojdrjflerwfdzdjkublsbwaukarhps` int DEFAULT NULL,\n `woavvyjkxqfigdmntewbayhkpalhdncj` int DEFAULT NULL,\n `innorkndjnyusfycebppzwhcprebgfzj` int DEFAULT NULL,\n `agzlnjulqpngwihnscprnvxifvmkjtlk` int DEFAULT NULL,\n `msrziujyxpacuhyhibxkvanhcygamvgc` int DEFAULT NULL,\n `jcdutuwenmythhlzadhirhjonsnszexw` int DEFAULT NULL,\n `ndfalcomzvkwfogjgktorzkvqaqoeahn` int DEFAULT NULL,\n `jiowijxjwwrgttklojgvvdgjjoexpols` int DEFAULT NULL,\n `mtlprfmrmuprqfpybsuemysshaiivlsk` int DEFAULT NULL,\n `kaiyxovaibtkbeqvoyrkcdsuytpfuqrz` int DEFAULT NULL,\n `ydclamijmrkkzgyxipstxughmfsxfiih` int DEFAULT NULL,\n `dyoiximxjizhvegvgnyvyntmsyjwxrcx` int DEFAULT NULL,\n `ihclknrtbvfsjoekhavyvbhygycnurvg` int DEFAULT NULL,\n `bbotdqsnipzvzowfkwysdueiltadddlz` int DEFAULT NULL,\n `uylcosidyvznpdodyvjplbdsketnljop` int DEFAULT NULL,\n `kakyptfaqllswimirqnvlckbygfvsezx` int DEFAULT NULL,\n `lxiivzggqypabdrcpdquacwpufvysvzh` int DEFAULT NULL,\n `ifwzkrvdwcomphhohgqnsszcnnpfrqpj` int DEFAULT NULL,\n `vswemhqykaxsgzrllbdihjcrrasztsgw` int DEFAULT NULL,\n `amkallxzfozjtunnjvfbpctrpvxgakyg` int DEFAULT NULL,\n `xjapufzufdudoxeyfpbicjzhmoyzeujd` int DEFAULT NULL,\n `qltpklsxcnmbmcujnysxucbwfnwpbctu` int DEFAULT NULL,\n `guttznruwdehskfyryaksjvabyadupmr` int DEFAULT NULL,\n `ctokmzeprvjcceckkogbjkcshfmtebtz` int DEFAULT NULL,\n `dndmbeyngldygvemoaytwcsawhlrlckr` int DEFAULT NULL,\n `idhyqeqhkubmqalhkmpvdvhcglvuncru` int DEFAULT NULL,\n `uqudtbxththrghhombwfenshbjiyujgp` int DEFAULT NULL,\n `imgyeryckawptacvajsnfrbifxpspubw` int DEFAULT NULL,\n `oupcmvavnxitcmyhgvtordfpavvhfqqw` int DEFAULT NULL,\n `sappwtybodbyvhberecwdkpvugzmijcr` int DEFAULT NULL,\n `itawnurbcbeoioufepfhweyzeiicybgr` int DEFAULT NULL,\n `rfngumwqpbynzxalusmgdcbirhkfstrq` int DEFAULT NULL,\n `aauutdsrmdezrgsotvcebauymlqtcgev` int DEFAULT NULL,\n `haupjqwanzfieezjklfuggghwqezlzud` int DEFAULT NULL,\n `vgprxmivwvmvyuhuvxiinamkgjntpaqm` int DEFAULT NULL,\n `wukgwcnatxxjfiugncpnmeflywerfwmf` int DEFAULT NULL,\n `tbhtxkobdujzonexfypaejvdzoismrms` int DEFAULT NULL,\n `jkzltlcuqhjnmglndsbwcaliugjcnanm` int DEFAULT NULL,\n `bbeiiahpsvbhyufycacqsogdxjynwvkn` int DEFAULT NULL,\n `bckkrfowlxhlthjrqorghjioybtgrpdg` int DEFAULT NULL,\n `fpdqmkrpststtlvyqmtuvzkaocfusequ` int DEFAULT NULL,\n `aalqmjmdbfmxaskixevhcjrclvuqtaww` int DEFAULT NULL,\n `osfqnzhdiyaswjuuxnucluvjpkqkkozy` int DEFAULT NULL,\n `hldsituasrkdzmprvnrjwpazowakepga` int DEFAULT NULL,\n `giuhgihzjwkovkeltruzywsynevgrpdq` int DEFAULT NULL,\n `oddldypqltwmerckajpynbqkdukysgcq` int DEFAULT NULL,\n `hdwavctejxykwiwfjhiffejtacdwiqen` int DEFAULT NULL,\n `rgpmcxtyezprzallihpvvwngwiuxehlx` int DEFAULT NULL,\n `dguuksdnqkyfkrqfcwtahlhqgsgicwpm` int DEFAULT NULL,\n `nzzkflvmqvjujgglvmmjbnjmifdcwidc` int DEFAULT NULL,\n `brfzazdtasfkgaentnqhfladnhjthhps` int DEFAULT NULL,\n `xoeellukluvcdmhmfnuttdawppsrulry` int DEFAULT NULL,\n `rtapqphyhvovthrbqelriybzlpyfslbg` int DEFAULT NULL,\n `rbegishoabumoinageseualncltpapnz` int DEFAULT NULL,\n `ilklvoemmmzrrgofalgbvgoeyozlzjth` int DEFAULT NULL,\n `trjhdsnwhuixsiniscqixzilrgecbbyf` int DEFAULT NULL,\n `aujejzwldcjcawledbvsfojwwwabbluq` int DEFAULT NULL,\n `bnanilpenofkgxecaqbvgshtdmspntmi` int DEFAULT NULL,\n `qcuxesartaqjfzoolwqkxshdwjgtqntv` int DEFAULT NULL,\n `dgziqbijfdklgjkexygbcqrgsunwvkcq` int DEFAULT NULL,\n `sqhucywdhvdfsuvbvegmadheozusapud` int DEFAULT NULL,\n `utjbfvxipvbzkwyogkozbcvoplgbrxzc` int DEFAULT NULL,\n `tdgbhzefcxdrwqwhwwcxdlwwtonaakpp` int DEFAULT NULL,\n `eqlaicfxcdsxkxsgjeifodlgjkzxgryt` int DEFAULT NULL,\n `eaudtpjpztboamltacqbpfehjlszqprz` int DEFAULT NULL,\n `olkbhyhpeplasoqaxkfcvyqlmllgavdh` int DEFAULT NULL,\n `fprevpwrceeyvgmbnvrpercuahznndhz` int DEFAULT NULL,\n `jmamthqjewbbxgfmgkwaqrdfihbitrpv` int DEFAULT NULL,\n `dlwlqymnhrtdipaaogpqaejvysiemyri` int DEFAULT NULL,\n `bwuryqwbgwgqijbtrywdlxgegpretutf` int DEFAULT NULL,\n `yobohhytgsiacljjoremgxassbcfidlt` int DEFAULT NULL,\n `nlzopojutltlhckswqacozoxcnqxphhg` int DEFAULT NULL,\n `krglxouyjmgzgtbxetokqcsoyrhocyid` int DEFAULT NULL,\n `svgvdvraozjeusnuydmnryydhmadihwp` int DEFAULT NULL,\n `siwjwfxeagwjsbaktupijvjjohzrqtgu` int DEFAULT NULL,\n `ylsjkesiyjufztxdtgkdymsfqxpxvpla` int DEFAULT NULL,\n `qzauxetevaqktgipfcpefmkutdpforov` int DEFAULT NULL,\n `wmnmfmlfjywjzdvxfqyetcsccqpxefgx` int DEFAULT NULL,\n `mfshzklkqjiwtccsxmzcnfbsihmuwmrj` int DEFAULT NULL,\n `ubgfnoypwlxxsgjqnvhtlgodkiwboxqc` int DEFAULT NULL,\n `jxlmeafwemuskpwdttiftaixkacfizan` int DEFAULT NULL,\n `obrccwddpatpsbiugzsegibxobntpcoc` int DEFAULT NULL,\n `mgehzmygjaheynzvykykbgstmraykuob` int DEFAULT NULL,\n `udqnaomzjvkobkutdwagjzrwjuwqedmo` int DEFAULT NULL,\n `jialfsremtflyvhnhzwbqzhhnytrlmpr` int DEFAULT NULL,\n `fzsuxjeugezzutfauyftbsfklgbfmwrc` int DEFAULT NULL,\n `kagdlicoiotpvnljqchlnataptdzjxbt` int DEFAULT NULL,\n `zkiwmdocvklzntucmspohrnwvoygxmbr` int DEFAULT NULL,\n `kriidarsaoscjscfyeaexksconpfrojw` int DEFAULT NULL,\n `weqoabiynnhwcrebetzvdbjuenehkqme` int DEFAULT NULL,\n `zxurnendqswsqauydyqflsckpmxhiypk` int DEFAULT NULL,\n `rbhrmwzsvpuxeijymgqhvuxrruipyqqa` int DEFAULT NULL,\n `ejzfbihppowsherppuydttzitnzrvbuq` int DEFAULT NULL,\n `ftirjchlmcbtvqpaimvfxyesmamksccc` int DEFAULT NULL,\n `dtzzyrjctcyycjwftcjxthxiyfqrtwyu` int DEFAULT NULL,\n `qnhwcnfefdrwwlbwlsbsimensellmofr` int DEFAULT NULL,\n `jfzhzvhdkdwkpvkuvtofrolbkmzptobs` int DEFAULT NULL,\n `lmusvbuvbshhvjlcykzpzxmopefemoct` int DEFAULT NULL,\n PRIMARY KEY (`torizvmmijgolgkoxkawvfdpjutslegi`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"rklsysjbbggcyaheggpsezportemxlmu\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["torizvmmijgolgkoxkawvfdpjutslegi"],"columns":[{"name":"torizvmmijgolgkoxkawvfdpjutslegi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ypqorfecxnpcbqzxieycahvzspjwwrng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"srojdrjflerwfdzdjkublsbwaukarhps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"woavvyjkxqfigdmntewbayhkpalhdncj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"innorkndjnyusfycebppzwhcprebgfzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agzlnjulqpngwihnscprnvxifvmkjtlk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"msrziujyxpacuhyhibxkvanhcygamvgc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jcdutuwenmythhlzadhirhjonsnszexw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ndfalcomzvkwfogjgktorzkvqaqoeahn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jiowijxjwwrgttklojgvvdgjjoexpols","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtlprfmrmuprqfpybsuemysshaiivlsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kaiyxovaibtkbeqvoyrkcdsuytpfuqrz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydclamijmrkkzgyxipstxughmfsxfiih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dyoiximxjizhvegvgnyvyntmsyjwxrcx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ihclknrtbvfsjoekhavyvbhygycnurvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbotdqsnipzvzowfkwysdueiltadddlz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uylcosidyvznpdodyvjplbdsketnljop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kakyptfaqllswimirqnvlckbygfvsezx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxiivzggqypabdrcpdquacwpufvysvzh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifwzkrvdwcomphhohgqnsszcnnpfrqpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vswemhqykaxsgzrllbdihjcrrasztsgw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amkallxzfozjtunnjvfbpctrpvxgakyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjapufzufdudoxeyfpbicjzhmoyzeujd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qltpklsxcnmbmcujnysxucbwfnwpbctu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"guttznruwdehskfyryaksjvabyadupmr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctokmzeprvjcceckkogbjkcshfmtebtz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dndmbeyngldygvemoaytwcsawhlrlckr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idhyqeqhkubmqalhkmpvdvhcglvuncru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqudtbxththrghhombwfenshbjiyujgp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"imgyeryckawptacvajsnfrbifxpspubw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oupcmvavnxitcmyhgvtordfpavvhfqqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sappwtybodbyvhberecwdkpvugzmijcr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itawnurbcbeoioufepfhweyzeiicybgr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfngumwqpbynzxalusmgdcbirhkfstrq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aauutdsrmdezrgsotvcebauymlqtcgev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"haupjqwanzfieezjklfuggghwqezlzud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgprxmivwvmvyuhuvxiinamkgjntpaqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wukgwcnatxxjfiugncpnmeflywerfwmf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbhtxkobdujzonexfypaejvdzoismrms","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jkzltlcuqhjnmglndsbwcaliugjcnanm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbeiiahpsvbhyufycacqsogdxjynwvkn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bckkrfowlxhlthjrqorghjioybtgrpdg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpdqmkrpststtlvyqmtuvzkaocfusequ","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aalqmjmdbfmxaskixevhcjrclvuqtaww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"osfqnzhdiyaswjuuxnucluvjpkqkkozy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hldsituasrkdzmprvnrjwpazowakepga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"giuhgihzjwkovkeltruzywsynevgrpdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oddldypqltwmerckajpynbqkdukysgcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdwavctejxykwiwfjhiffejtacdwiqen","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgpmcxtyezprzallihpvvwngwiuxehlx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dguuksdnqkyfkrqfcwtahlhqgsgicwpm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzzkflvmqvjujgglvmmjbnjmifdcwidc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"brfzazdtasfkgaentnqhfladnhjthhps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xoeellukluvcdmhmfnuttdawppsrulry","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtapqphyhvovthrbqelriybzlpyfslbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbegishoabumoinageseualncltpapnz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ilklvoemmmzrrgofalgbvgoeyozlzjth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"trjhdsnwhuixsiniscqixzilrgecbbyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aujejzwldcjcawledbvsfojwwwabbluq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnanilpenofkgxecaqbvgshtdmspntmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qcuxesartaqjfzoolwqkxshdwjgtqntv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgziqbijfdklgjkexygbcqrgsunwvkcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sqhucywdhvdfsuvbvegmadheozusapud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utjbfvxipvbzkwyogkozbcvoplgbrxzc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdgbhzefcxdrwqwhwwcxdlwwtonaakpp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqlaicfxcdsxkxsgjeifodlgjkzxgryt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eaudtpjpztboamltacqbpfehjlszqprz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"olkbhyhpeplasoqaxkfcvyqlmllgavdh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fprevpwrceeyvgmbnvrpercuahznndhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmamthqjewbbxgfmgkwaqrdfihbitrpv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlwlqymnhrtdipaaogpqaejvysiemyri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bwuryqwbgwgqijbtrywdlxgegpretutf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yobohhytgsiacljjoremgxassbcfidlt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nlzopojutltlhckswqacozoxcnqxphhg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krglxouyjmgzgtbxetokqcsoyrhocyid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"svgvdvraozjeusnuydmnryydhmadihwp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"siwjwfxeagwjsbaktupijvjjohzrqtgu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ylsjkesiyjufztxdtgkdymsfqxpxvpla","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzauxetevaqktgipfcpefmkutdpforov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmnmfmlfjywjzdvxfqyetcsccqpxefgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfshzklkqjiwtccsxmzcnfbsihmuwmrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubgfnoypwlxxsgjqnvhtlgodkiwboxqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jxlmeafwemuskpwdttiftaixkacfizan","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obrccwddpatpsbiugzsegibxobntpcoc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mgehzmygjaheynzvykykbgstmraykuob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udqnaomzjvkobkutdwagjzrwjuwqedmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jialfsremtflyvhnhzwbqzhhnytrlmpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzsuxjeugezzutfauyftbsfklgbfmwrc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kagdlicoiotpvnljqchlnataptdzjxbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zkiwmdocvklzntucmspohrnwvoygxmbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kriidarsaoscjscfyeaexksconpfrojw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"weqoabiynnhwcrebetzvdbjuenehkqme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxurnendqswsqauydyqflsckpmxhiypk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbhrmwzsvpuxeijymgqhvuxrruipyqqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejzfbihppowsherppuydttzitnzrvbuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftirjchlmcbtvqpaimvfxyesmamksccc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtzzyrjctcyycjwftcjxthxiyfqrtwyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qnhwcnfefdrwwlbwlsbsimensellmofr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jfzhzvhdkdwkpvkuvtofrolbkmzptobs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmusvbuvbshhvjlcykzpzxmopefemoct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670795,"databaseName":"models_schema","ddl":"CREATE TABLE `rlpjqdqhwzurijhocbkdpovoiiqctrve` (\n `sjmcbpsrcsuswqxcptvarwcfkiqphyxv` int NOT NULL,\n `edfoszifsfyeznkjzowjwudmcpiwaiaz` int DEFAULT NULL,\n `znaphmhxhkdridihhadohgxugrpbdziq` int DEFAULT NULL,\n `tcffmqcikzaiebjlmmyhsftpzfaveuth` int DEFAULT NULL,\n `eulvfukubzucyxgzyfovjucbhqgxrqxt` int DEFAULT NULL,\n `aahaozognkhsemjtsgzfpzabxtznyqwi` int DEFAULT NULL,\n `xsciyxkqznqaqkonmrfxlhmuomtwbgfj` int DEFAULT NULL,\n `effaiygochrmgeetkyxwiqregrepaguu` int DEFAULT NULL,\n `cuounpmpxfsbbyzhjavsabfqzdbfohhe` int DEFAULT NULL,\n `kwvdwlrejkdxyizfztboggwigktpjxef` int DEFAULT NULL,\n `ohhatdvpziaybeeedukzacgmmigmgmxn` int DEFAULT NULL,\n `mlgumwosjlsdmalftziidoubioawkmvy` int DEFAULT NULL,\n `zrntbilfevzzltbicgzenggwxehgxyhn` int DEFAULT NULL,\n `lssbnffpydpqwohojhqjkwhpfkpmbdup` int DEFAULT NULL,\n `wfhcknmnlysubvhmmjbckoiiazgqxvns` int DEFAULT NULL,\n `ucnnvlwvxgsnwdnttiipgmctvuxkztfs` int DEFAULT NULL,\n `cnpoowwjesbgzwdanwicxrehldfdflas` int DEFAULT NULL,\n `jeycoowquxtmpcutllguhgagjophiyih` int DEFAULT NULL,\n `nzftapaimhaoaxogaqjqxcumeocmdxfl` int DEFAULT NULL,\n `auaajgvlchzscgulztvcmnithenwokjt` int DEFAULT NULL,\n `uzfrnxobdfzexdzzlryxkrkibkkeuwfj` int DEFAULT NULL,\n `viyzlrwnkqeydjzruwqpxggxluhwyxbq` int DEFAULT NULL,\n `wdspkfzlttcihcgbvcgvmfxzjssviiye` int DEFAULT NULL,\n `xnuilmquurqugkykjsucooxnjxubdxkn` int DEFAULT NULL,\n `ihtdrwpeimkqrplcaalpfpiepetjxzuf` int DEFAULT NULL,\n `smhjjxkqhiqsazmupnonpnjifzjgoblb` int DEFAULT NULL,\n `wuydkvzslcbemxqiprvxnijyebyerjym` int DEFAULT NULL,\n `kfujtkwuyjcewptxilizhsyoynqsslck` int DEFAULT NULL,\n `vdjbngwlkdkcxqujauvebbulvuofdhvu` int DEFAULT NULL,\n `xvcapzltldxwdvrsperhhcufpeuoiqcw` int DEFAULT NULL,\n `rgryjqrbkvwryhlyzxqqffpsriweclsv` int DEFAULT NULL,\n `bxphemhmmifogqnobxlnbxgdegkholex` int DEFAULT NULL,\n `yhcrgaxpgcnuxgonjwgvtgnyqlroqthl` int DEFAULT NULL,\n `hjzpvuilwsuvxhgthbkziunxbexcfpii` int DEFAULT NULL,\n `bpgyqriodcyyljottjfblsykdlznfwke` int DEFAULT NULL,\n `qpfbyxuihkdqlpvdpjtgoqjkefcchoxt` int DEFAULT NULL,\n `gjvcifswjndasqskbysnxbhmtwqbtgym` int DEFAULT NULL,\n `jlxhulchvwxqiofpzvfimengorpmuhlc` int DEFAULT NULL,\n `davlmapkuxoxzksnwspecvefrznysvye` int DEFAULT NULL,\n `hbumymbdetilpfaocdercsrtfdolicxj` int DEFAULT NULL,\n `hjpszvlagcbqbmtzbhrpuhszgondktbp` int DEFAULT NULL,\n `rpwnpnnkunyhqpbnwgwupdsfrlwlalgr` int DEFAULT NULL,\n `yzuwztfpdtrnsctqtwzigaoejfiqwebp` int DEFAULT NULL,\n `zjpfgutcrirpfdnfyvedatzireejquyl` int DEFAULT NULL,\n `eahmqkydnsvgekfywfzruqgcwryasbux` int DEFAULT NULL,\n `eguotjobjxgqzkgcwjpgwukvybmbtzfo` int DEFAULT NULL,\n `rtdathkygfhdjshouympwokurnifddvx` int DEFAULT NULL,\n `hvfytcpwjcoakhthdwbbzbnxzbrpawcz` int DEFAULT NULL,\n `riwdbqmesdoylzugqqwlkpkjtsygollo` int DEFAULT NULL,\n `xmchbmohxzlfpmwbbkkhjfvxuwrxtskl` int DEFAULT NULL,\n `njmdfnvtxcxsfmazzuvcseplbrrmhcyg` int DEFAULT NULL,\n `bbfvlgkcvrlzzrormizroeqfflckoxac` int DEFAULT NULL,\n `dbnvnvovekahpjtrjojgyybwbqlscgyb` int DEFAULT NULL,\n `cwdwzjzhhijgcykxxjpsnowgpgabzbwa` int DEFAULT NULL,\n `kjftvdpsaidtukfyhrbnrflzvchucnkb` int DEFAULT NULL,\n `uksiyrgybquickxlxyhkmjfbrjotsqpe` int DEFAULT NULL,\n `jrqpyvypnuihooaetijyzneheqdhzxhu` int DEFAULT NULL,\n `prhjruvdthbgfmlusjlzsctnhsgoanru` int DEFAULT NULL,\n `pjkwkmkergroayfkugrbqriylfzawqga` int DEFAULT NULL,\n `xkpvkmcrvqauvgfttyqeegedqbzuxpoa` int DEFAULT NULL,\n `sfcjmzendlutfvhungyefwafvrazeame` int DEFAULT NULL,\n `enhygatfvkdhpeungcydwtrkwxatgsfd` int DEFAULT NULL,\n `qvdbnorinlqyqakjztgszzvuxtfxmpnv` int DEFAULT NULL,\n `datzuclghhxvcyrlhzxdcihldqwonrew` int DEFAULT NULL,\n `zuxedbcqzyfyqjrnhdcgqmyrtjtejljh` int DEFAULT NULL,\n `xipqxuiovesihnaiuxcjvgfshzdhjalj` int DEFAULT NULL,\n `ovxzpdwtxcykcnhradhaymuwplhlqjvf` int DEFAULT NULL,\n `ypivapwdpmmvylhzsclegkyeexbhimpy` int DEFAULT NULL,\n `qlpulgqsdiufcacsgsigzxtbapaoppaw` int DEFAULT NULL,\n `vlswwjveydizbakdufyzeuitajiollak` int DEFAULT NULL,\n `xibbprryqbcpkovpuajhkhfeagisszbd` int DEFAULT NULL,\n `gqhglnqrsvoawvwggkrsviotxfjdaaaw` int DEFAULT NULL,\n `gnhhdfjkygduobijufgllxmsjlhpqxpn` int DEFAULT NULL,\n `grtieqjimqgrjjhtdoqtavrsbgqtdgbd` int DEFAULT NULL,\n `uveagboutqyseahtdgxcnwsdomwzzznp` int DEFAULT NULL,\n `reodqhezqzhkjibpahbykepzxbsweeqz` int DEFAULT NULL,\n `wurrocugobneielhwgktsxtqnarbdygl` int DEFAULT NULL,\n `xgllkivspzffosgbhjfbsgcmrkydjvet` int DEFAULT NULL,\n `clubthebwmfahwxpftovnqrjmplywyun` int DEFAULT NULL,\n `xccmrksdftjfklpiupaosmodxsckiuyh` int DEFAULT NULL,\n `nyshjpdcdzsswkvnxniottkyshruawwl` int DEFAULT NULL,\n `drxqrcybxtqhaceqqmanclbxmwokqwgr` int DEFAULT NULL,\n `hqumzkrdctkmgmssouxktveujioutuch` int DEFAULT NULL,\n `rnweqeblyatonddwwbhmdtsbtgrqilkm` int DEFAULT NULL,\n `tdabwwyctjpnztsluycjfathfkzongvq` int DEFAULT NULL,\n `mgpqlgpjuvtnjpvmmwygefkdqobqdkjv` int DEFAULT NULL,\n `onnytnbhojgxpxozkpozoyjoolahqjkb` int DEFAULT NULL,\n `ibvzcgmqnbwhqxcyduryoxmjbntggbfx` int DEFAULT NULL,\n `oqcrdidnithmcviddfrwnzhvhtcybofh` int DEFAULT NULL,\n `vsrpabfhtnfolsxydlogvawbrcyuvmkd` int DEFAULT NULL,\n `bifzpukndchovfrzjichbvroyozdqucw` int DEFAULT NULL,\n `gwsdnkmhqizlvphbrxkqncmxfihdlels` int DEFAULT NULL,\n `kmnlnqphcpnjzldhwdktwlxcrylqjech` int DEFAULT NULL,\n `irlttamfnjyobbjvorvksavrohpjnpbp` int DEFAULT NULL,\n `rbbsngdzoswdwyjzefxzcxgalyfijukq` int DEFAULT NULL,\n `vscurpnbahwuvyldsqmexawxmramkxmu` int DEFAULT NULL,\n `xaersnmowuyuqvtrbtzebfsyimkegxfh` int DEFAULT NULL,\n `bouxopaukfsbpdmbuocxnzqssxjrhzvc` int DEFAULT NULL,\n `lsrrjygruuuikmtphiydfwzkzohnkwmh` int DEFAULT NULL,\n `affbzzbfxqlauhfivwbibwhqrwxhgpdp` int DEFAULT NULL,\n PRIMARY KEY (`sjmcbpsrcsuswqxcptvarwcfkiqphyxv`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"rlpjqdqhwzurijhocbkdpovoiiqctrve\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["sjmcbpsrcsuswqxcptvarwcfkiqphyxv"],"columns":[{"name":"sjmcbpsrcsuswqxcptvarwcfkiqphyxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"edfoszifsfyeznkjzowjwudmcpiwaiaz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znaphmhxhkdridihhadohgxugrpbdziq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tcffmqcikzaiebjlmmyhsftpzfaveuth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eulvfukubzucyxgzyfovjucbhqgxrqxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aahaozognkhsemjtsgzfpzabxtznyqwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsciyxkqznqaqkonmrfxlhmuomtwbgfj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"effaiygochrmgeetkyxwiqregrepaguu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cuounpmpxfsbbyzhjavsabfqzdbfohhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwvdwlrejkdxyizfztboggwigktpjxef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohhatdvpziaybeeedukzacgmmigmgmxn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlgumwosjlsdmalftziidoubioawkmvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zrntbilfevzzltbicgzenggwxehgxyhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lssbnffpydpqwohojhqjkwhpfkpmbdup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfhcknmnlysubvhmmjbckoiiazgqxvns","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucnnvlwvxgsnwdnttiipgmctvuxkztfs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnpoowwjesbgzwdanwicxrehldfdflas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jeycoowquxtmpcutllguhgagjophiyih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzftapaimhaoaxogaqjqxcumeocmdxfl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"auaajgvlchzscgulztvcmnithenwokjt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzfrnxobdfzexdzzlryxkrkibkkeuwfj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"viyzlrwnkqeydjzruwqpxggxluhwyxbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wdspkfzlttcihcgbvcgvmfxzjssviiye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnuilmquurqugkykjsucooxnjxubdxkn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ihtdrwpeimkqrplcaalpfpiepetjxzuf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smhjjxkqhiqsazmupnonpnjifzjgoblb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wuydkvzslcbemxqiprvxnijyebyerjym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfujtkwuyjcewptxilizhsyoynqsslck","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdjbngwlkdkcxqujauvebbulvuofdhvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvcapzltldxwdvrsperhhcufpeuoiqcw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgryjqrbkvwryhlyzxqqffpsriweclsv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bxphemhmmifogqnobxlnbxgdegkholex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhcrgaxpgcnuxgonjwgvtgnyqlroqthl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjzpvuilwsuvxhgthbkziunxbexcfpii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bpgyqriodcyyljottjfblsykdlznfwke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qpfbyxuihkdqlpvdpjtgoqjkefcchoxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjvcifswjndasqskbysnxbhmtwqbtgym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlxhulchvwxqiofpzvfimengorpmuhlc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"davlmapkuxoxzksnwspecvefrznysvye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbumymbdetilpfaocdercsrtfdolicxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjpszvlagcbqbmtzbhrpuhszgondktbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpwnpnnkunyhqpbnwgwupdsfrlwlalgr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzuwztfpdtrnsctqtwzigaoejfiqwebp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjpfgutcrirpfdnfyvedatzireejquyl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eahmqkydnsvgekfywfzruqgcwryasbux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eguotjobjxgqzkgcwjpgwukvybmbtzfo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtdathkygfhdjshouympwokurnifddvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hvfytcpwjcoakhthdwbbzbnxzbrpawcz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"riwdbqmesdoylzugqqwlkpkjtsygollo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmchbmohxzlfpmwbbkkhjfvxuwrxtskl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njmdfnvtxcxsfmazzuvcseplbrrmhcyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbfvlgkcvrlzzrormizroeqfflckoxac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbnvnvovekahpjtrjojgyybwbqlscgyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwdwzjzhhijgcykxxjpsnowgpgabzbwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjftvdpsaidtukfyhrbnrflzvchucnkb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uksiyrgybquickxlxyhkmjfbrjotsqpe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jrqpyvypnuihooaetijyzneheqdhzxhu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prhjruvdthbgfmlusjlzsctnhsgoanru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjkwkmkergroayfkugrbqriylfzawqga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkpvkmcrvqauvgfttyqeegedqbzuxpoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfcjmzendlutfvhungyefwafvrazeame","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"enhygatfvkdhpeungcydwtrkwxatgsfd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvdbnorinlqyqakjztgszzvuxtfxmpnv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"datzuclghhxvcyrlhzxdcihldqwonrew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zuxedbcqzyfyqjrnhdcgqmyrtjtejljh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xipqxuiovesihnaiuxcjvgfshzdhjalj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovxzpdwtxcykcnhradhaymuwplhlqjvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypivapwdpmmvylhzsclegkyeexbhimpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qlpulgqsdiufcacsgsigzxtbapaoppaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vlswwjveydizbakdufyzeuitajiollak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xibbprryqbcpkovpuajhkhfeagisszbd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqhglnqrsvoawvwggkrsviotxfjdaaaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnhhdfjkygduobijufgllxmsjlhpqxpn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grtieqjimqgrjjhtdoqtavrsbgqtdgbd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uveagboutqyseahtdgxcnwsdomwzzznp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"reodqhezqzhkjibpahbykepzxbsweeqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wurrocugobneielhwgktsxtqnarbdygl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xgllkivspzffosgbhjfbsgcmrkydjvet","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clubthebwmfahwxpftovnqrjmplywyun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xccmrksdftjfklpiupaosmodxsckiuyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nyshjpdcdzsswkvnxniottkyshruawwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drxqrcybxtqhaceqqmanclbxmwokqwgr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqumzkrdctkmgmssouxktveujioutuch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnweqeblyatonddwwbhmdtsbtgrqilkm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdabwwyctjpnztsluycjfathfkzongvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mgpqlgpjuvtnjpvmmwygefkdqobqdkjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"onnytnbhojgxpxozkpozoyjoolahqjkb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibvzcgmqnbwhqxcyduryoxmjbntggbfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqcrdidnithmcviddfrwnzhvhtcybofh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsrpabfhtnfolsxydlogvawbrcyuvmkd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bifzpukndchovfrzjichbvroyozdqucw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gwsdnkmhqizlvphbrxkqncmxfihdlels","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmnlnqphcpnjzldhwdktwlxcrylqjech","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"irlttamfnjyobbjvorvksavrohpjnpbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbbsngdzoswdwyjzefxzcxgalyfijukq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vscurpnbahwuvyldsqmexawxmramkxmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xaersnmowuyuqvtrbtzebfsyimkegxfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bouxopaukfsbpdmbuocxnzqssxjrhzvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lsrrjygruuuikmtphiydfwzkzohnkwmh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"affbzzbfxqlauhfivwbibwhqrwxhgpdp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670827,"databaseName":"models_schema","ddl":"CREATE TABLE `rmacnjxgjbzdfqdleytznzmsuzvkdjrq` (\n `ndvxyfllpuhaeksiqkjbxatcaoyobzpm` int NOT NULL,\n `ivxvtxaoolrwxcxtlqdcwpjjuukrshie` int DEFAULT NULL,\n `bcbmyzjbdtnnpaiqtutnohjysaedzrvw` int DEFAULT NULL,\n `ovktemtdmqoylrfwklaswzontwghxoxx` int DEFAULT NULL,\n `nbxchtycsrlsuqvnicizdynidhjpwgwb` int DEFAULT NULL,\n `uxqvjaxbkcmuaudjhebdplkxazotwqbw` int DEFAULT NULL,\n `uegoiwtvvkubcbgcqvzejjvrxdlsfkkc` int DEFAULT NULL,\n `jrnwfqabzygivdphtupqyzvzelbgyjwk` int DEFAULT NULL,\n `cenvakxhebjkdyykigyhmsunemmzhqmf` int DEFAULT NULL,\n `uwlkxhsufcvazawyibkodyiygbwdyios` int DEFAULT NULL,\n `lnltfvhidtbxgrsjcfljupjephjmjofd` int DEFAULT NULL,\n `wamsmjsqlwlzqxivopbwlplynafppcew` int DEFAULT NULL,\n `lycqnizasltqtyczfpuiythneogtzvwv` int DEFAULT NULL,\n `ndzdketiyujmprpwpulfhikwmznljdts` int DEFAULT NULL,\n `bsxukgfkegeoicnjhihebqqvvkxczjti` int DEFAULT NULL,\n `otmshctgtwldvsbcflbvbyshrdnebpsz` int DEFAULT NULL,\n `ceovmfvmjskbdrrunydjedhafqpqohrj` int DEFAULT NULL,\n `qrsrqfzkhhsmvhdkbhdjqjmrzkhhtfzj` int DEFAULT NULL,\n `shnadyqfsxpaxmkjiystgyurmibhcfwn` int DEFAULT NULL,\n `mubgupmkfwbompwtsrrctcvxwakceaxv` int DEFAULT NULL,\n `xvulrqvevqjfdsiajbfukdfvxeqwsfpi` int DEFAULT NULL,\n `onaekpxpusvmjjfvynmrfyicooxjtbkx` int DEFAULT NULL,\n `chfwfltqvkffpjpucrnretvtmggroxet` int DEFAULT NULL,\n `epsjphpmqnkydyfhiaxbkpsdtanwttkh` int DEFAULT NULL,\n `ommabzcpmbgzerogocukfakyozbtkxzt` int DEFAULT NULL,\n `ujrtexlueiybiouvhvfqujmrkozbkkgu` int DEFAULT NULL,\n `nicingdruekretrfpyrqxjhscuxoxwtv` int DEFAULT NULL,\n `pmqheddntslnxujqqegepxpgykbfmqop` int DEFAULT NULL,\n `dhociiblmyrnhzspmjiupkbucaqekikn` int DEFAULT NULL,\n `vsbvoucsgcomhpaextejuwnxygrxxrrm` int DEFAULT NULL,\n `wcexcmhqfqurjykuzpmnhgbhjqfhnnyd` int DEFAULT NULL,\n `xmqiwsdrygbemigxujxbgxbvxlyxyuhe` int DEFAULT NULL,\n `ljvlqxbrcjovyhqemngbipgakrbxussm` int DEFAULT NULL,\n `yzfopfioelqikrpulqzayjgihktnfzcx` int DEFAULT NULL,\n `yfzexqpigtnrcoikeraqdehvintaergx` int DEFAULT NULL,\n `sxdkpmcigfzhdqotvwlkcctbyyepyssl` int DEFAULT NULL,\n `zdbbwygcpvckkmhvxunqfrjzitzproyd` int DEFAULT NULL,\n `qopmakfswjscgksbzdadapmyxiiipvzf` int DEFAULT NULL,\n `qxpaakxastuqojyknrxowitptzwujazz` int DEFAULT NULL,\n `lcwzfaccegcjgtxxyjoplptvppepflzj` int DEFAULT NULL,\n `pijmlasbkwufjguglhbklzgeiqorxvme` int DEFAULT NULL,\n `otpdvkuuzwzskadasinypryxegyupcxz` int DEFAULT NULL,\n `kbbmkhlilzcnmfacqccmftteehscgvdt` int DEFAULT NULL,\n `tdbqhlveyymmcqisgwtevookvqdcmlqv` int DEFAULT NULL,\n `rioqrtxbjihxjlnfgfirwbzfratpgphk` int DEFAULT NULL,\n `daxbolqdeolrxdkgeqrqxvwjnvthvvzv` int DEFAULT NULL,\n `vritxeejnsuwuwxylpmuyvxnetjhwdns` int DEFAULT NULL,\n `hhttsbywjrjhbxvhltsdolmaqjjlyvwc` int DEFAULT NULL,\n `oydzsapdkxpyvxjujfkzqvsdmkafiwgg` int DEFAULT NULL,\n `wqsqbzntwfhcxtutblaonesymejixjeg` int DEFAULT NULL,\n `jjbitfkxsehppyligvludvvnaxfxqxvc` int DEFAULT NULL,\n `gmfrraainbehqzehesxqgjnqwttshclh` int DEFAULT NULL,\n `ekyrgvpjdbacjaizjmlrtipedkxrtlfv` int DEFAULT NULL,\n `fvbkalvcmgmkakedsdtmfcfhhsbzcpge` int DEFAULT NULL,\n `ofdxwmjkuwkiqvwcixioxofsvjmuevig` int DEFAULT NULL,\n `crrdgdzvwjdvrtrvdwigcqfzewhzyewn` int DEFAULT NULL,\n `cmvzubgfrvkwqpbvlovvoddatlnilpvg` int DEFAULT NULL,\n `vkcmipidhgammwlsrvfsquzjnqmwmfsq` int DEFAULT NULL,\n `rgxurwxorlvilmydtzzazinpmxcrckcs` int DEFAULT NULL,\n `lxwdlokavpucffdcejoxmkioxzkieagr` int DEFAULT NULL,\n `rxhxpltwnkjerocafksxzohulinmwmcl` int DEFAULT NULL,\n `khguqduexfrrqmvzugdaqefgmesyogha` int DEFAULT NULL,\n `zddcrwyfqoexrmcjflhcpwajmiyoxrln` int DEFAULT NULL,\n `gobnawlpjxucexvcvfkgrnytudhvffaz` int DEFAULT NULL,\n `pkcalbvbwyqgkglckzeendbmhwbsopuf` int DEFAULT NULL,\n `rhzkekeejvrgpmeowrwgvzpjkfrqispd` int DEFAULT NULL,\n `fqeigdgkyeigajyadithufaeekbgaspx` int DEFAULT NULL,\n `ckddmayvqpgizyesockkxcbhgndexpja` int DEFAULT NULL,\n `vkyycvlzgepvdljkdvhemnvwsoueljoe` int DEFAULT NULL,\n `blstiknfwfjrpgibwysavlfewpjzxapb` int DEFAULT NULL,\n `foreaffbhkplwiaedyabjwamqcfiftzx` int DEFAULT NULL,\n `grlihoahqlokosywsxwwytpnuagebxvo` int DEFAULT NULL,\n `zfgeelzutipuuyakgvjyjqlzzjuycvdt` int DEFAULT NULL,\n `ejchitnyhtdoauclpzpscadyharoqdaj` int DEFAULT NULL,\n `cglflepbltdwxswhmcbetmjdnbitjkea` int DEFAULT NULL,\n `kzykcygkikxufrustbzlmzmhjfphvtcn` int DEFAULT NULL,\n `ekkelmpkkfexpfsvkudiuupmaxnxdbsc` int DEFAULT NULL,\n `lbkciphwhllfbwfrhidwszvwtmnzlpna` int DEFAULT NULL,\n `libvxnhaxbjbnpdrxtibxwafftshglqr` int DEFAULT NULL,\n `epvqszecbgcryjeoemqdgwbvkkjeheej` int DEFAULT NULL,\n `qfzyrsvpwajsmspbgqkggoosyzkgpvzp` int DEFAULT NULL,\n `ykmxfbtscldobxpnjalcvqhetesrdcie` int DEFAULT NULL,\n `ppvisfaltfcqzrnofztdgakmoqxusbth` int DEFAULT NULL,\n `nwfvoxllbglmaxzikwonqefharvdlljo` int DEFAULT NULL,\n `gijbepwbdalfziokeemktsurkknxjmir` int DEFAULT NULL,\n `pmudohknarpqgehhamibotvuscjnrwrw` int DEFAULT NULL,\n `lokosurbefsarnpncpwnxhyolenontij` int DEFAULT NULL,\n `vrutlqgomlvefurtvqcvkptpkvxanbkt` int DEFAULT NULL,\n `jpswtkcrjsaxzyqpdqgxoewbyfzratua` int DEFAULT NULL,\n `opehrxbeoddfnxkhdejghjiehqtwwprh` int DEFAULT NULL,\n `kxiqlmxngpuddlngscvvyacjmwqdedrm` int DEFAULT NULL,\n `jovovlxlqgveyhlfuzupypktcddqhwek` int DEFAULT NULL,\n `gnxtzsyvxwwabmcffmklpjzoixrmoryt` int DEFAULT NULL,\n `dfcljxhjkinzhceficbattaoikhgwgas` int DEFAULT NULL,\n `cvththnywxfjocqessyoajugoulswzkr` int DEFAULT NULL,\n `bfnqotcankxgfvpbsootntqylckwtjkz` int DEFAULT NULL,\n `nifymrqatgraqymqindczwmbrrnbhhsr` int DEFAULT NULL,\n `zujemnzoccttzeqefhgtbwgaifqqwvhn` int DEFAULT NULL,\n `tgxcuujinowfqhsvtgjjxgputtphxmct` int DEFAULT NULL,\n `xvhihoyawcejguonukvwpudcoqnkflcs` int DEFAULT NULL,\n PRIMARY KEY (`ndvxyfllpuhaeksiqkjbxatcaoyobzpm`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"rmacnjxgjbzdfqdleytznzmsuzvkdjrq\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["ndvxyfllpuhaeksiqkjbxatcaoyobzpm"],"columns":[{"name":"ndvxyfllpuhaeksiqkjbxatcaoyobzpm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ivxvtxaoolrwxcxtlqdcwpjjuukrshie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bcbmyzjbdtnnpaiqtutnohjysaedzrvw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovktemtdmqoylrfwklaswzontwghxoxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbxchtycsrlsuqvnicizdynidhjpwgwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxqvjaxbkcmuaudjhebdplkxazotwqbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uegoiwtvvkubcbgcqvzejjvrxdlsfkkc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jrnwfqabzygivdphtupqyzvzelbgyjwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cenvakxhebjkdyykigyhmsunemmzhqmf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwlkxhsufcvazawyibkodyiygbwdyios","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lnltfvhidtbxgrsjcfljupjephjmjofd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wamsmjsqlwlzqxivopbwlplynafppcew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lycqnizasltqtyczfpuiythneogtzvwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ndzdketiyujmprpwpulfhikwmznljdts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bsxukgfkegeoicnjhihebqqvvkxczjti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otmshctgtwldvsbcflbvbyshrdnebpsz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ceovmfvmjskbdrrunydjedhafqpqohrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrsrqfzkhhsmvhdkbhdjqjmrzkhhtfzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shnadyqfsxpaxmkjiystgyurmibhcfwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mubgupmkfwbompwtsrrctcvxwakceaxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvulrqvevqjfdsiajbfukdfvxeqwsfpi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"onaekpxpusvmjjfvynmrfyicooxjtbkx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chfwfltqvkffpjpucrnretvtmggroxet","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"epsjphpmqnkydyfhiaxbkpsdtanwttkh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ommabzcpmbgzerogocukfakyozbtkxzt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujrtexlueiybiouvhvfqujmrkozbkkgu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nicingdruekretrfpyrqxjhscuxoxwtv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmqheddntslnxujqqegepxpgykbfmqop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhociiblmyrnhzspmjiupkbucaqekikn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsbvoucsgcomhpaextejuwnxygrxxrrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcexcmhqfqurjykuzpmnhgbhjqfhnnyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmqiwsdrygbemigxujxbgxbvxlyxyuhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljvlqxbrcjovyhqemngbipgakrbxussm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzfopfioelqikrpulqzayjgihktnfzcx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfzexqpigtnrcoikeraqdehvintaergx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sxdkpmcigfzhdqotvwlkcctbyyepyssl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdbbwygcpvckkmhvxunqfrjzitzproyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qopmakfswjscgksbzdadapmyxiiipvzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxpaakxastuqojyknrxowitptzwujazz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lcwzfaccegcjgtxxyjoplptvppepflzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pijmlasbkwufjguglhbklzgeiqorxvme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otpdvkuuzwzskadasinypryxegyupcxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kbbmkhlilzcnmfacqccmftteehscgvdt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdbqhlveyymmcqisgwtevookvqdcmlqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rioqrtxbjihxjlnfgfirwbzfratpgphk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"daxbolqdeolrxdkgeqrqxvwjnvthvvzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vritxeejnsuwuwxylpmuyvxnetjhwdns","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhttsbywjrjhbxvhltsdolmaqjjlyvwc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oydzsapdkxpyvxjujfkzqvsdmkafiwgg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqsqbzntwfhcxtutblaonesymejixjeg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjbitfkxsehppyligvludvvnaxfxqxvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmfrraainbehqzehesxqgjnqwttshclh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekyrgvpjdbacjaizjmlrtipedkxrtlfv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fvbkalvcmgmkakedsdtmfcfhhsbzcpge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofdxwmjkuwkiqvwcixioxofsvjmuevig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crrdgdzvwjdvrtrvdwigcqfzewhzyewn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmvzubgfrvkwqpbvlovvoddatlnilpvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkcmipidhgammwlsrvfsquzjnqmwmfsq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgxurwxorlvilmydtzzazinpmxcrckcs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxwdlokavpucffdcejoxmkioxzkieagr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxhxpltwnkjerocafksxzohulinmwmcl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khguqduexfrrqmvzugdaqefgmesyogha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zddcrwyfqoexrmcjflhcpwajmiyoxrln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gobnawlpjxucexvcvfkgrnytudhvffaz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkcalbvbwyqgkglckzeendbmhwbsopuf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhzkekeejvrgpmeowrwgvzpjkfrqispd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqeigdgkyeigajyadithufaeekbgaspx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckddmayvqpgizyesockkxcbhgndexpja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkyycvlzgepvdljkdvhemnvwsoueljoe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blstiknfwfjrpgibwysavlfewpjzxapb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"foreaffbhkplwiaedyabjwamqcfiftzx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grlihoahqlokosywsxwwytpnuagebxvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zfgeelzutipuuyakgvjyjqlzzjuycvdt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejchitnyhtdoauclpzpscadyharoqdaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cglflepbltdwxswhmcbetmjdnbitjkea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzykcygkikxufrustbzlmzmhjfphvtcn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekkelmpkkfexpfsvkudiuupmaxnxdbsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lbkciphwhllfbwfrhidwszvwtmnzlpna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"libvxnhaxbjbnpdrxtibxwafftshglqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"epvqszecbgcryjeoemqdgwbvkkjeheej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfzyrsvpwajsmspbgqkggoosyzkgpvzp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykmxfbtscldobxpnjalcvqhetesrdcie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ppvisfaltfcqzrnofztdgakmoqxusbth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nwfvoxllbglmaxzikwonqefharvdlljo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gijbepwbdalfziokeemktsurkknxjmir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmudohknarpqgehhamibotvuscjnrwrw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lokosurbefsarnpncpwnxhyolenontij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrutlqgomlvefurtvqcvkptpkvxanbkt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpswtkcrjsaxzyqpdqgxoewbyfzratua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opehrxbeoddfnxkhdejghjiehqtwwprh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxiqlmxngpuddlngscvvyacjmwqdedrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jovovlxlqgveyhlfuzupypktcddqhwek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnxtzsyvxwwabmcffmklpjzoixrmoryt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dfcljxhjkinzhceficbattaoikhgwgas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvththnywxfjocqessyoajugoulswzkr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bfnqotcankxgfvpbsootntqylckwtjkz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nifymrqatgraqymqindczwmbrrnbhhsr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zujemnzoccttzeqefhgtbwgaifqqwvhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tgxcuujinowfqhsvtgjjxgputtphxmct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvhihoyawcejguonukvwpudcoqnkflcs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670857,"databaseName":"models_schema","ddl":"CREATE TABLE `rmcsuzfuprljobatuqxsqkuaffhpdtag` (\n `qkdoamorfeecnspombwpgmtckrrmadjb` int NOT NULL,\n `nrxfyknxorgwagxifhcwyiptrbioffjs` int DEFAULT NULL,\n `fzvtycbqeuzkvlnhzxzieqeyihzfgzgo` int DEFAULT NULL,\n `xexhlhhzzoqoxavnjdficqhammrmwvyl` int DEFAULT NULL,\n `jpthotqmolsvqjhkxnwprsttwlloyzof` int DEFAULT NULL,\n `hegpppgrydbfupsukqxazecolvrfkpqo` int DEFAULT NULL,\n `mpgwljxzazohlifrgobqorkzmphjwmfv` int DEFAULT NULL,\n `qvsotelkaszohgjomfuxbuuruwhddgnv` int DEFAULT NULL,\n `yiayfouidziksjyzaezyczwtltgznwpt` int DEFAULT NULL,\n `kapwdchjvpppucmgpjnfidkdneikpvqg` int DEFAULT NULL,\n `tsbjkzzqxsgdkwvtrlkisasaesbqgcxi` int DEFAULT NULL,\n `roxehwotouhnhvpoenmkzbltxlsvrmdw` int DEFAULT NULL,\n `qclbeaciubyzhqlpsxyaviebvwlqdnxm` int DEFAULT NULL,\n `zejasogulowjyclsevxhdcqirglnuiyg` int DEFAULT NULL,\n `peogpvpzbszwclwfcuercudypdchhhhs` int DEFAULT NULL,\n `acseidaysummsnzhhofhjaknxnklhfae` int DEFAULT NULL,\n `sbzqnrhosrobaiwqbtqsxwqnuawogkck` int DEFAULT NULL,\n `qqzauqpjzblvyogmzdryipdjjxxbxxpq` int DEFAULT NULL,\n `dglyvbqwbdoknpppffbalqllhveogprd` int DEFAULT NULL,\n `sdrzuknhyicweuclsfwnakrkpvlbxhem` int DEFAULT NULL,\n `iinibbuledqxmmwfajewwscrrcmiefem` int DEFAULT NULL,\n `zsuzcsmjqvunngempeejilgmtdryapjm` int DEFAULT NULL,\n `wvafudrxixbtuebbbxzrfbboirrcvdjg` int DEFAULT NULL,\n `ohvjqjcpnsdbhpmhaizjcwkvlzwufzkw` int DEFAULT NULL,\n `dxmcadiqxitwkttzglwjneenxcovrmix` int DEFAULT NULL,\n `klkqdkbpgiuferhnbdcobuejncrrbqle` int DEFAULT NULL,\n `rdlzvwcnqyywcmaaoxzyodgtiuvbyvdq` int DEFAULT NULL,\n `uuprizodcchrqlrxvrigfnjcvwfcqxfj` int DEFAULT NULL,\n `yqwifvqxyicnmgjpvbhtsrsmurqqetxp` int DEFAULT NULL,\n `tinkzyetbgquptxocfthpvfikhtqhzof` int DEFAULT NULL,\n `aatugeswcgbnvnkvpxuovphqppnwmyjt` int DEFAULT NULL,\n `zidfzilbkvsmqemkhwhcxnujygntebtp` int DEFAULT NULL,\n `fnxmwnhndifuubdvyknfxauupzjovddp` int DEFAULT NULL,\n `vjpehlixluvlhbymwcuokxnfrqbuvfur` int DEFAULT NULL,\n `ykmeyscnxhlvwputurzanpayxphmrhah` int DEFAULT NULL,\n `wlixpjjtjallhtcyzyxpwcfyuwzmqtnw` int DEFAULT NULL,\n `czbvzugmfpzmgwhysequwvfjnutmkpeh` int DEFAULT NULL,\n `ajjpqqxcwbxntjfznmwpwhcosnfmplxd` int DEFAULT NULL,\n `khmswicphcbdinyjivwvzyvtzekxchet` int DEFAULT NULL,\n `yzqjwaydepnstznautxjokctyvxydvhb` int DEFAULT NULL,\n `cssqioyivjrwbkdkqtpfdsnqqqletciv` int DEFAULT NULL,\n `dwcnevykjtopdknjxqnmptysusrmznaq` int DEFAULT NULL,\n `eulfrwmzojokvoucmiaullbjzeqxguyb` int DEFAULT NULL,\n `agbqswixigclraugqeuofdwvwrilyusl` int DEFAULT NULL,\n `trwijplsfpodkcrnxcppqgmgclrvealc` int DEFAULT NULL,\n `bnmadyorcliavnjcphenlawwgbmoqiuc` int DEFAULT NULL,\n `fmkysllohgikhoylswqqioyazxfrapwu` int DEFAULT NULL,\n `vjbegiewoqcrwgqtxaedghereoekbthb` int DEFAULT NULL,\n `rfiyrxujmuxxuwkhizwhioxljesdtdcw` int DEFAULT NULL,\n `yzfsjkgtbodoeshrtnmzdpehttijuxfe` int DEFAULT NULL,\n `kplhidvyeliwppgijjsxjkcqtmloaobc` int DEFAULT NULL,\n `lhhocoatqkjuzladyhzfvmyzecqowuab` int DEFAULT NULL,\n `kwjniqcsiptagjfzmwwlhqmxxcfcxqqn` int DEFAULT NULL,\n `eqzztrfnksrapfyfeqybadzwjbfpktcr` int DEFAULT NULL,\n `faebhcoeloouxgxnmdqcqwgkdjjnwytc` int DEFAULT NULL,\n `dmzulxyefbriywyixesknjhstgsbhdqc` int DEFAULT NULL,\n `enbopkaxdxemudxfsszisjkroiywlbru` int DEFAULT NULL,\n `rifsbceuqsxqdmyhsqdzssczsvuacrwl` int DEFAULT NULL,\n `rmfncsngnnlcuxtfqlnkwthygnvkcxns` int DEFAULT NULL,\n `kpzapcolymuhnvkuqurtiklddrployvb` int DEFAULT NULL,\n `itdhyrkpgbyxpycvlwmpgbetqcycevdi` int DEFAULT NULL,\n `asldagfoztipqxkufceelpkiuqwsrddw` int DEFAULT NULL,\n `awqfxoznowhqmymcsufcwrjvlahdnlcv` int DEFAULT NULL,\n `fcsdcdwdvfcwqxmncgfmivfmhtmtwfou` int DEFAULT NULL,\n `vjhcauuvjfbjaxszlxceewksohlqthkk` int DEFAULT NULL,\n `arefkafsycmwlhxtxevhjwflhinohebg` int DEFAULT NULL,\n `cvcoqbccktyjsrxbvkpztntpdcylzzmm` int DEFAULT NULL,\n `fkczvwzmqhkjunfkaltncldnvqwmoxhl` int DEFAULT NULL,\n `tzrywhnhmkvlkgorwfznrguuqtlfusbv` int DEFAULT NULL,\n `mdqmujjpjefdrujpxypsnjofavhjkblo` int DEFAULT NULL,\n `veazpvkbjuegeuycmhkwpdgcnsromdqs` int DEFAULT NULL,\n `typxjwvmennvwagtivcwrjzrctqgwnig` int DEFAULT NULL,\n `qlotimkjojuakgriphcexgiwsnbaxpzj` int DEFAULT NULL,\n `uaaebgwrwxsthbvmokgojqjfpglqiehw` int DEFAULT NULL,\n `gklkercfjjqndwvtgtnjaavhagsukmim` int DEFAULT NULL,\n `qapgsmreblvcpndwpqdscpilmuiukvua` int DEFAULT NULL,\n `nxmpfmbumqozqgmtufdhschzorqlccjx` int DEFAULT NULL,\n `qgkhriyntjjlwgwqjocdkfedmlryodxg` int DEFAULT NULL,\n `aodyzlxtylsxappavwpyqvuyqpjmcohm` int DEFAULT NULL,\n `ygfvzwcvqdoweiwihcjjwblnlnbijmpv` int DEFAULT NULL,\n `sexoazxzmlhfshqsepxzjxmggakmvyto` int DEFAULT NULL,\n `tioxdwhnpkvktxzzjnzcugwfuydrtigc` int DEFAULT NULL,\n `ustpvmtplqexwawcppgalwyhqvaibbhe` int DEFAULT NULL,\n `twblnctlnfunertpdubomgaauwsmowru` int DEFAULT NULL,\n `hjkdxbzxdwzuacweszauvyobtzjbiefh` int DEFAULT NULL,\n `xnghxhbctlqpmdgtaltbfaypmbwmlcdj` int DEFAULT NULL,\n `hdlkmxdqlipvjuwsplntjgywyeevwezb` int DEFAULT NULL,\n `dqwfdlfcdmwxqgpzufpzpvliuuegkplj` int DEFAULT NULL,\n `jzngxedhoxmzfzayeamtvumwepvonbxn` int DEFAULT NULL,\n `kzmnonpfipddnobyoxpottubevfljksi` int DEFAULT NULL,\n `afviottmgyrbopyqnqzruduqqucivoeq` int DEFAULT NULL,\n `dkjnaeobrojkbkqetsllqquqyknkzspm` int DEFAULT NULL,\n `atzxvbacaiteoymfpskttrnzevibwnlm` int DEFAULT NULL,\n `tcwpsrigfnvyabgbokbderzspzbzluvl` int DEFAULT NULL,\n `wezybxikzpvtacszbmsvpgzyzmidfdok` int DEFAULT NULL,\n `fxcseqroisslpnjrpexcgwtjlkquhibi` int DEFAULT NULL,\n `uvygsiretwvybfdfesaztnzgjcsofqca` int DEFAULT NULL,\n `qzkoftdchnuozbyjsymnkwfoobbecdjx` int DEFAULT NULL,\n `miwagmcoryeghkzytotaqprkpuvoihtx` int DEFAULT NULL,\n `ymvakhppqmcrynmudgepgfyffzqfqwjm` int DEFAULT NULL,\n PRIMARY KEY (`qkdoamorfeecnspombwpgmtckrrmadjb`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"rmcsuzfuprljobatuqxsqkuaffhpdtag\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["qkdoamorfeecnspombwpgmtckrrmadjb"],"columns":[{"name":"qkdoamorfeecnspombwpgmtckrrmadjb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"nrxfyknxorgwagxifhcwyiptrbioffjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzvtycbqeuzkvlnhzxzieqeyihzfgzgo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xexhlhhzzoqoxavnjdficqhammrmwvyl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpthotqmolsvqjhkxnwprsttwlloyzof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hegpppgrydbfupsukqxazecolvrfkpqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpgwljxzazohlifrgobqorkzmphjwmfv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvsotelkaszohgjomfuxbuuruwhddgnv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yiayfouidziksjyzaezyczwtltgznwpt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kapwdchjvpppucmgpjnfidkdneikpvqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tsbjkzzqxsgdkwvtrlkisasaesbqgcxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"roxehwotouhnhvpoenmkzbltxlsvrmdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qclbeaciubyzhqlpsxyaviebvwlqdnxm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zejasogulowjyclsevxhdcqirglnuiyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"peogpvpzbszwclwfcuercudypdchhhhs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acseidaysummsnzhhofhjaknxnklhfae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbzqnrhosrobaiwqbtqsxwqnuawogkck","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqzauqpjzblvyogmzdryipdjjxxbxxpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dglyvbqwbdoknpppffbalqllhveogprd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdrzuknhyicweuclsfwnakrkpvlbxhem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iinibbuledqxmmwfajewwscrrcmiefem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zsuzcsmjqvunngempeejilgmtdryapjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvafudrxixbtuebbbxzrfbboirrcvdjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohvjqjcpnsdbhpmhaizjcwkvlzwufzkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxmcadiqxitwkttzglwjneenxcovrmix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"klkqdkbpgiuferhnbdcobuejncrrbqle","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdlzvwcnqyywcmaaoxzyodgtiuvbyvdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uuprizodcchrqlrxvrigfnjcvwfcqxfj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqwifvqxyicnmgjpvbhtsrsmurqqetxp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tinkzyetbgquptxocfthpvfikhtqhzof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aatugeswcgbnvnkvpxuovphqppnwmyjt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zidfzilbkvsmqemkhwhcxnujygntebtp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnxmwnhndifuubdvyknfxauupzjovddp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjpehlixluvlhbymwcuokxnfrqbuvfur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykmeyscnxhlvwputurzanpayxphmrhah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wlixpjjtjallhtcyzyxpwcfyuwzmqtnw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"czbvzugmfpzmgwhysequwvfjnutmkpeh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajjpqqxcwbxntjfznmwpwhcosnfmplxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khmswicphcbdinyjivwvzyvtzekxchet","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzqjwaydepnstznautxjokctyvxydvhb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cssqioyivjrwbkdkqtpfdsnqqqletciv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwcnevykjtopdknjxqnmptysusrmznaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eulfrwmzojokvoucmiaullbjzeqxguyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agbqswixigclraugqeuofdwvwrilyusl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"trwijplsfpodkcrnxcppqgmgclrvealc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnmadyorcliavnjcphenlawwgbmoqiuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmkysllohgikhoylswqqioyazxfrapwu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjbegiewoqcrwgqtxaedghereoekbthb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfiyrxujmuxxuwkhizwhioxljesdtdcw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzfsjkgtbodoeshrtnmzdpehttijuxfe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kplhidvyeliwppgijjsxjkcqtmloaobc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhhocoatqkjuzladyhzfvmyzecqowuab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwjniqcsiptagjfzmwwlhqmxxcfcxqqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqzztrfnksrapfyfeqybadzwjbfpktcr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"faebhcoeloouxgxnmdqcqwgkdjjnwytc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dmzulxyefbriywyixesknjhstgsbhdqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"enbopkaxdxemudxfsszisjkroiywlbru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rifsbceuqsxqdmyhsqdzssczsvuacrwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rmfncsngnnlcuxtfqlnkwthygnvkcxns","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpzapcolymuhnvkuqurtiklddrployvb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itdhyrkpgbyxpycvlwmpgbetqcycevdi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"asldagfoztipqxkufceelpkiuqwsrddw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awqfxoznowhqmymcsufcwrjvlahdnlcv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcsdcdwdvfcwqxmncgfmivfmhtmtwfou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjhcauuvjfbjaxszlxceewksohlqthkk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"arefkafsycmwlhxtxevhjwflhinohebg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvcoqbccktyjsrxbvkpztntpdcylzzmm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fkczvwzmqhkjunfkaltncldnvqwmoxhl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tzrywhnhmkvlkgorwfznrguuqtlfusbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdqmujjpjefdrujpxypsnjofavhjkblo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"veazpvkbjuegeuycmhkwpdgcnsromdqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"typxjwvmennvwagtivcwrjzrctqgwnig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qlotimkjojuakgriphcexgiwsnbaxpzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uaaebgwrwxsthbvmokgojqjfpglqiehw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gklkercfjjqndwvtgtnjaavhagsukmim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qapgsmreblvcpndwpqdscpilmuiukvua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxmpfmbumqozqgmtufdhschzorqlccjx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgkhriyntjjlwgwqjocdkfedmlryodxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aodyzlxtylsxappavwpyqvuyqpjmcohm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ygfvzwcvqdoweiwihcjjwblnlnbijmpv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sexoazxzmlhfshqsepxzjxmggakmvyto","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tioxdwhnpkvktxzzjnzcugwfuydrtigc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ustpvmtplqexwawcppgalwyhqvaibbhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twblnctlnfunertpdubomgaauwsmowru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjkdxbzxdwzuacweszauvyobtzjbiefh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnghxhbctlqpmdgtaltbfaypmbwmlcdj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdlkmxdqlipvjuwsplntjgywyeevwezb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqwfdlfcdmwxqgpzufpzpvliuuegkplj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzngxedhoxmzfzayeamtvumwepvonbxn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzmnonpfipddnobyoxpottubevfljksi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afviottmgyrbopyqnqzruduqqucivoeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkjnaeobrojkbkqetsllqquqyknkzspm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atzxvbacaiteoymfpskttrnzevibwnlm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tcwpsrigfnvyabgbokbderzspzbzluvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wezybxikzpvtacszbmsvpgzyzmidfdok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxcseqroisslpnjrpexcgwtjlkquhibi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvygsiretwvybfdfesaztnzgjcsofqca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzkoftdchnuozbyjsymnkwfoobbecdjx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"miwagmcoryeghkzytotaqprkpuvoihtx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ymvakhppqmcrynmudgepgfyffzqfqwjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670888,"databaseName":"models_schema","ddl":"CREATE TABLE `rmdvstrvymqpjlchedshuysqvdwtaegw` (\n `svwhgelzcoayoptrtahsogllofvxeuij` int NOT NULL,\n `afexvttcuznoxmwzggxzberzsarnnjft` int DEFAULT NULL,\n `loonvhiwldnmmhlizyupryyfptdyrofs` int DEFAULT NULL,\n `afxxudnmfwaqsdmighklloeubevzrlmz` int DEFAULT NULL,\n `wuntfhkxhjpbckusklionseixsudjdds` int DEFAULT NULL,\n `wbcbgencnfeipljcsqmdequcbmaeaajp` int DEFAULT NULL,\n `pqbbmomdrrcgjuhpikjfirndsyuqbkxo` int DEFAULT NULL,\n `xyodtzvmdugvlijvktyhafggyfupabiw` int DEFAULT NULL,\n `kihbqlzntyzxigyqlacqlklcmkoswyni` int DEFAULT NULL,\n `rjrdyaynqdiecagaetdvftizoxqtxoxh` int DEFAULT NULL,\n `kkdtflotjplgznygatuqtiemoplrlljr` int DEFAULT NULL,\n `pbqmpbwwcmiucgdtarqsrjvownwugxaf` int DEFAULT NULL,\n `iwfgbnvmfukwyouklgbkulubufiaujkq` int DEFAULT NULL,\n `wxpytxcgsuhgiaqngxobolrrjfljkpgf` int DEFAULT NULL,\n `erwlrwxasjokqjegepmnyftozxgobumj` int DEFAULT NULL,\n `npsadfummbvgyzxjwoftdjhvzqkdvbwi` int DEFAULT NULL,\n `pkfnqxtpphlrkvykozytotrkbrmrfakp` int DEFAULT NULL,\n `dnnpqvucjwjtjxpvqbxsbuuwtcmahyne` int DEFAULT NULL,\n `miwvunzwxcucgfbrebxskihyqtrdxkmn` int DEFAULT NULL,\n `qdeyejmelpzpqlcowtazmmzsmveuyicv` int DEFAULT NULL,\n `oegedazyrictcjoohnojilpjhsxowvgh` int DEFAULT NULL,\n `ctjceopzssspkukzlslqdlxgzvazjehe` int DEFAULT NULL,\n `prrqjydkdmtgtyyfxtamnbcwfjoewcxi` int DEFAULT NULL,\n `idofysclqhjxbyanfcadnvbfehjhsaak` int DEFAULT NULL,\n `tlppswwpipbonbjzelsqymkfycypttci` int DEFAULT NULL,\n `txrrpcokorxpvvqfzykmzujztqigfjfa` int DEFAULT NULL,\n `moadswwxcwngmkgzsjkcpsmbnqwkvhxb` int DEFAULT NULL,\n `qqxfywbgbtsujjkwasqyagscmdzkncxi` int DEFAULT NULL,\n `vcjzbtjdylohcojjustvsqmqkpdcbamx` int DEFAULT NULL,\n `uypjwyilslewypowmxjnyjhmnxtgaceb` int DEFAULT NULL,\n `wgsuvemwvltfhirnrbudaezosmnmmklj` int DEFAULT NULL,\n `hxjucmkjiwoywrrwsqmkupwedaybbhag` int DEFAULT NULL,\n `vgrjmuaacokgxuuldqodwnaozbiaqjti` int DEFAULT NULL,\n `xntskwxbttzhitepgoyzkzimvvlugiuc` int DEFAULT NULL,\n `appoyoihwisaopnidgwxzgkovroivsrg` int DEFAULT NULL,\n `eiklwjvbsflairjtdjmtrkuwhhrxjppn` int DEFAULT NULL,\n `dbvocitastmjmjrzwtmhyienvswujejd` int DEFAULT NULL,\n `njkgcjkqsdpznghuccmwaefxcybifczp` int DEFAULT NULL,\n `wbjndhjwyyfxeobvvjdkhiweznultiol` int DEFAULT NULL,\n `ergxptznckupevrwenvcccxxsadqxylb` int DEFAULT NULL,\n `veoszqdgdmwicyxjbxlpsicneobqjuhm` int DEFAULT NULL,\n `ujijqyzmxxviurqtbwwsvrbqhzswymqu` int DEFAULT NULL,\n `fmzwhouogoklswhptrwnxtnvljcqmlpn` int DEFAULT NULL,\n `otkoxaziwwldmdxkxohpaajtynwytexb` int DEFAULT NULL,\n `jsgaxooznuzznkylzyrighlycfapvogk` int DEFAULT NULL,\n `jmpbxvhdasgwqtgpqagztigtwiozoohz` int DEFAULT NULL,\n `ubnafofrchintmesyftzbcdgfyzhxebg` int DEFAULT NULL,\n `uqsgxayliiccwzlwtivzgtigzzyakwtt` int DEFAULT NULL,\n `fjwkdomzmutfmheoqxieztrgshlvpdql` int DEFAULT NULL,\n `ugktjrtlwbhrmjkoypcorxasmvbfisrh` int DEFAULT NULL,\n `jkzvyxgzdcqsazypzfwazylknxawfqus` int DEFAULT NULL,\n `uccurfzzazmbdqgurqtcrusyvuwjztyp` int DEFAULT NULL,\n `wpzocvyliggylpvqbvvryzhbllnggdyf` int DEFAULT NULL,\n `fntvyuufhonapldhhlobnztkfuuewxvh` int DEFAULT NULL,\n `bgvzfuothckpfpntxzlpolxwyjbqmndy` int DEFAULT NULL,\n `kdxlrgbhtdtpvbozfhoqspbdfnpenpgj` int DEFAULT NULL,\n `bmpwwpmkxatejhyzfplkekqmuruewhni` int DEFAULT NULL,\n `itbeplqjxtecoinuvklhnszkszcsmkha` int DEFAULT NULL,\n `kcefliketbhlxdujxpqltquagdydjnak` int DEFAULT NULL,\n `jextpluemmixbcxvqsbzqiivbukbtjan` int DEFAULT NULL,\n `dwaigxpncsdbpjhsylfopnyptatmpssu` int DEFAULT NULL,\n `ypsjylglozyeubitcoygbuxensmpmvpd` int DEFAULT NULL,\n `ikzwqgtzokjzvybvcyashdzlcxhwqruk` int DEFAULT NULL,\n `ycjierlibryxbtgxfpuxoyeqfghqhqpr` int DEFAULT NULL,\n `wyxbpsuiishkwbatehscllleulnmztyl` int DEFAULT NULL,\n `kzouurztrqiyrdvvzwpcgpnwsoqhuugi` int DEFAULT NULL,\n `korklsdcwogtembrzanknagrapavwdlh` int DEFAULT NULL,\n `biufiuxrjsgbizgykftwrmeollmzfsdi` int DEFAULT NULL,\n `rlgydzdwcfyqglmxaqwnihzvxafszutj` int DEFAULT NULL,\n `ktjzjjabipwuyuvhrirwpjvshsnxlztk` int DEFAULT NULL,\n `mknzbmsmpmkpclqyciepczmriqdcgmrt` int DEFAULT NULL,\n `ebtwjunjkjwowacufgsvlgsgyuvzjlsp` int DEFAULT NULL,\n `izmxuheczgshvyegelcetnknfnrjshia` int DEFAULT NULL,\n `cizfqjcgaripbhwljalxmqrprgjqyket` int DEFAULT NULL,\n `atkxjqndituxgkhfhhjimzmjfkfdcroo` int DEFAULT NULL,\n `ijarsifhopqfbvffedluvcgvybvxtgbb` int DEFAULT NULL,\n `swlrhmheddjdnugxhfntvupdizzkqcsd` int DEFAULT NULL,\n `wgpehvpszewaaybwdjnhuomcfyzftczv` int DEFAULT NULL,\n `vhkwhrnwsmxnwdntywkiymjsrxrywana` int DEFAULT NULL,\n `cpraroakdgnewpezvjbmtvdcpyokyqyp` int DEFAULT NULL,\n `kwhnrdxockezygsutdjmfdjecesaooor` int DEFAULT NULL,\n `rqkserkxxiutoovbvytlntnxcaxcibbj` int DEFAULT NULL,\n `ntdctkkbhyukmbucuklignkhwsnexlcj` int DEFAULT NULL,\n `husblpksqtxxqmioozugszsunxlhjfwb` int DEFAULT NULL,\n `dvwnehesnvyhqlnntzuznirognudhzby` int DEFAULT NULL,\n `jgfipfspxxfnoymcgvyegbtzhclazvdh` int DEFAULT NULL,\n `flnhqzmetmkijkelvkdybwjjqsejikic` int DEFAULT NULL,\n `nxuypbgugpgqjpzllrithbtbbfghmamn` int DEFAULT NULL,\n `vokpvyciklgfxybzfkhkwwblofwmyguf` int DEFAULT NULL,\n `omjquvtmnywfhcyaymqxbxdbzjspimrs` int DEFAULT NULL,\n `vifeqihdottfrnkmdwjapdatxkyzpyzi` int DEFAULT NULL,\n `alcpmpehcamlmloeoajgtuxyglylrpma` int DEFAULT NULL,\n `cjuixutempdlijdvoyunmsvnlyqsnvqw` int DEFAULT NULL,\n `oseftmlejdnlchubwuwrsdjqamhcjsrd` int DEFAULT NULL,\n `jaqvrnlhnpwwpugdoxyktizaylzrdmed` int DEFAULT NULL,\n `glmrwjqaptmraalijnddpsyedmpttudt` int DEFAULT NULL,\n `ahyyudeqoyldmjunwmqbtpyvegzyrprr` int DEFAULT NULL,\n `ytvstnvuiounkqaysreypdryxmeylpvo` int DEFAULT NULL,\n `rbxiunlkbqmfxdungcdhzuvrgxjhrfdd` int DEFAULT NULL,\n `czmcisthphfwwrxioghafpgemfirtdii` int DEFAULT NULL,\n PRIMARY KEY (`svwhgelzcoayoptrtahsogllofvxeuij`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"rmdvstrvymqpjlchedshuysqvdwtaegw\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["svwhgelzcoayoptrtahsogllofvxeuij"],"columns":[{"name":"svwhgelzcoayoptrtahsogllofvxeuij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"afexvttcuznoxmwzggxzberzsarnnjft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"loonvhiwldnmmhlizyupryyfptdyrofs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afxxudnmfwaqsdmighklloeubevzrlmz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wuntfhkxhjpbckusklionseixsudjdds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbcbgencnfeipljcsqmdequcbmaeaajp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqbbmomdrrcgjuhpikjfirndsyuqbkxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyodtzvmdugvlijvktyhafggyfupabiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kihbqlzntyzxigyqlacqlklcmkoswyni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjrdyaynqdiecagaetdvftizoxqtxoxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kkdtflotjplgznygatuqtiemoplrlljr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbqmpbwwcmiucgdtarqsrjvownwugxaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwfgbnvmfukwyouklgbkulubufiaujkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxpytxcgsuhgiaqngxobolrrjfljkpgf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"erwlrwxasjokqjegepmnyftozxgobumj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npsadfummbvgyzxjwoftdjhvzqkdvbwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkfnqxtpphlrkvykozytotrkbrmrfakp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dnnpqvucjwjtjxpvqbxsbuuwtcmahyne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"miwvunzwxcucgfbrebxskihyqtrdxkmn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdeyejmelpzpqlcowtazmmzsmveuyicv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oegedazyrictcjoohnojilpjhsxowvgh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctjceopzssspkukzlslqdlxgzvazjehe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prrqjydkdmtgtyyfxtamnbcwfjoewcxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idofysclqhjxbyanfcadnvbfehjhsaak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlppswwpipbonbjzelsqymkfycypttci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txrrpcokorxpvvqfzykmzujztqigfjfa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"moadswwxcwngmkgzsjkcpsmbnqwkvhxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqxfywbgbtsujjkwasqyagscmdzkncxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vcjzbtjdylohcojjustvsqmqkpdcbamx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uypjwyilslewypowmxjnyjhmnxtgaceb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgsuvemwvltfhirnrbudaezosmnmmklj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxjucmkjiwoywrrwsqmkupwedaybbhag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgrjmuaacokgxuuldqodwnaozbiaqjti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xntskwxbttzhitepgoyzkzimvvlugiuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"appoyoihwisaopnidgwxzgkovroivsrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eiklwjvbsflairjtdjmtrkuwhhrxjppn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbvocitastmjmjrzwtmhyienvswujejd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njkgcjkqsdpznghuccmwaefxcybifczp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbjndhjwyyfxeobvvjdkhiweznultiol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ergxptznckupevrwenvcccxxsadqxylb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"veoszqdgdmwicyxjbxlpsicneobqjuhm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujijqyzmxxviurqtbwwsvrbqhzswymqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmzwhouogoklswhptrwnxtnvljcqmlpn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otkoxaziwwldmdxkxohpaajtynwytexb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jsgaxooznuzznkylzyrighlycfapvogk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmpbxvhdasgwqtgpqagztigtwiozoohz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubnafofrchintmesyftzbcdgfyzhxebg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqsgxayliiccwzlwtivzgtigzzyakwtt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjwkdomzmutfmheoqxieztrgshlvpdql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugktjrtlwbhrmjkoypcorxasmvbfisrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jkzvyxgzdcqsazypzfwazylknxawfqus","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uccurfzzazmbdqgurqtcrusyvuwjztyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wpzocvyliggylpvqbvvryzhbllnggdyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fntvyuufhonapldhhlobnztkfuuewxvh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bgvzfuothckpfpntxzlpolxwyjbqmndy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kdxlrgbhtdtpvbozfhoqspbdfnpenpgj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bmpwwpmkxatejhyzfplkekqmuruewhni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itbeplqjxtecoinuvklhnszkszcsmkha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcefliketbhlxdujxpqltquagdydjnak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jextpluemmixbcxvqsbzqiivbukbtjan","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwaigxpncsdbpjhsylfopnyptatmpssu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypsjylglozyeubitcoygbuxensmpmvpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikzwqgtzokjzvybvcyashdzlcxhwqruk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ycjierlibryxbtgxfpuxoyeqfghqhqpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyxbpsuiishkwbatehscllleulnmztyl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzouurztrqiyrdvvzwpcgpnwsoqhuugi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"korklsdcwogtembrzanknagrapavwdlh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"biufiuxrjsgbizgykftwrmeollmzfsdi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlgydzdwcfyqglmxaqwnihzvxafszutj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ktjzjjabipwuyuvhrirwpjvshsnxlztk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mknzbmsmpmkpclqyciepczmriqdcgmrt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebtwjunjkjwowacufgsvlgsgyuvzjlsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"izmxuheczgshvyegelcetnknfnrjshia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cizfqjcgaripbhwljalxmqrprgjqyket","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atkxjqndituxgkhfhhjimzmjfkfdcroo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijarsifhopqfbvffedluvcgvybvxtgbb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swlrhmheddjdnugxhfntvupdizzkqcsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgpehvpszewaaybwdjnhuomcfyzftczv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhkwhrnwsmxnwdntywkiymjsrxrywana","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cpraroakdgnewpezvjbmtvdcpyokyqyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwhnrdxockezygsutdjmfdjecesaooor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqkserkxxiutoovbvytlntnxcaxcibbj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntdctkkbhyukmbucuklignkhwsnexlcj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"husblpksqtxxqmioozugszsunxlhjfwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvwnehesnvyhqlnntzuznirognudhzby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgfipfspxxfnoymcgvyegbtzhclazvdh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flnhqzmetmkijkelvkdybwjjqsejikic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxuypbgugpgqjpzllrithbtbbfghmamn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vokpvyciklgfxybzfkhkwwblofwmyguf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omjquvtmnywfhcyaymqxbxdbzjspimrs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vifeqihdottfrnkmdwjapdatxkyzpyzi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"alcpmpehcamlmloeoajgtuxyglylrpma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjuixutempdlijdvoyunmsvnlyqsnvqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oseftmlejdnlchubwuwrsdjqamhcjsrd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jaqvrnlhnpwwpugdoxyktizaylzrdmed","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"glmrwjqaptmraalijnddpsyedmpttudt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahyyudeqoyldmjunwmqbtpyvegzyrprr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytvstnvuiounkqaysreypdryxmeylpvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbxiunlkbqmfxdungcdhzuvrgxjhrfdd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"czmcisthphfwwrxioghafpgemfirtdii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670922,"databaseName":"models_schema","ddl":"CREATE TABLE `rrxmvbjbfyljiregbwvggjujgpkwekjf` (\n `kekqazutbfqfylsnvtflgteoyctgscji` int NOT NULL,\n `mserffeokdbnkxeaalirexaydtlbhoyw` int DEFAULT NULL,\n `dguwlmlftdorxmgfnknqnhqqyazquywe` int DEFAULT NULL,\n `xzmeuxgfcvogeghztimenqrsmcaktcor` int DEFAULT NULL,\n `snwngrqvynxtbiunzbgskvoostibnfka` int DEFAULT NULL,\n `ajjbcaywlovbgslrlljyixtkoczsstpo` int DEFAULT NULL,\n `nlvprsgsxbrejcfebhwoibdcfvgcfuhs` int DEFAULT NULL,\n `mcjyevfgqrkxmgpzhwiymjwzfnonhxhs` int DEFAULT NULL,\n `chinxexsupnmkcwmlvkdlnfobkundzhi` int DEFAULT NULL,\n `xwjfmwlokuuezboeiprmjvqleidljjox` int DEFAULT NULL,\n `haxnoqhxhrqnmdidqztvobnhaeadbrft` int DEFAULT NULL,\n `evkxahnxazsrljjompyjznuxjojjjnam` int DEFAULT NULL,\n `jgoqjxnfcrjigkjzdtzeeybrjkdfbhvp` int DEFAULT NULL,\n `yvrylopztfqmyzltezaxgpneurlaawtk` int DEFAULT NULL,\n `gpdtjhsobqoojvxwmchrzbbxvzplbudc` int DEFAULT NULL,\n `jmoigciodnihbuohxveethsebxkvkexd` int DEFAULT NULL,\n `ifxqiryzlryadpnstbkruvtiwlagubaw` int DEFAULT NULL,\n `qqjdgyfwwjpgxbdessessmqmtnheidwj` int DEFAULT NULL,\n `hocmrecwaueghznnbexelwrdcmtxvdiv` int DEFAULT NULL,\n `posocyikqmyozeltlzskeedyzddqnktm` int DEFAULT NULL,\n `nmobhprmvynkcqrfywcrmgipbjuxnghw` int DEFAULT NULL,\n `ngcjmqkabezkmboeuxjmoxslwdrcylos` int DEFAULT NULL,\n `iopkylvytzcbivqgkfkbktziucbmxnzx` int DEFAULT NULL,\n `gnxpyoeemoafbuqzpdfefajepkmyxnhk` int DEFAULT NULL,\n `atnadqtwqjesqimlerljxwwahoreazwn` int DEFAULT NULL,\n `pykpwivrsibucbvbokswqaydwqaogpic` int DEFAULT NULL,\n `dijnakdiepwvuohqgomxhlmpgdpihfqj` int DEFAULT NULL,\n `kseddcmaledtvcqocmbcfwjpozhvdomy` int DEFAULT NULL,\n `rzxhsuetdgzznjvbvkbpjmclsvittnyo` int DEFAULT NULL,\n `mdrgzfbljrdcpxaiiyremygxwkecukfe` int DEFAULT NULL,\n `rptpnvdqnrzblanvbwcaaigqcelkmblu` int DEFAULT NULL,\n `zfrvgysexscraoqfdqgfxppnknhhgxzk` int DEFAULT NULL,\n `cmdfqvjsmbuusxkfdztbwftmrwybgwbq` int DEFAULT NULL,\n `xdwxxujylrojwoglflvhtijoxugcmqda` int DEFAULT NULL,\n `diohifpautfbnqdqsbukbijldyklzokt` int DEFAULT NULL,\n `grkitjaxzhcdxgiewulrrupmgysmhozd` int DEFAULT NULL,\n `buzzpexmtrzjtzpocvjmqsoescjgfprb` int DEFAULT NULL,\n `xtqrxgludvytftpfwisdazuupzrpjwqm` int DEFAULT NULL,\n `yfcwrgnkhtkgpiwbbviwduocqhiwbdnz` int DEFAULT NULL,\n `nowmgxnargnotrgikfvyqbeoealqryqo` int DEFAULT NULL,\n `wqvlpqgcczdkmspakjgcjmxwbcyafelo` int DEFAULT NULL,\n `jdwdqthzgswmrouspcudeuewrbhxbxkc` int DEFAULT NULL,\n `xwakcpunljpjqteueehaoimkmnzpbxrd` int DEFAULT NULL,\n `kaywifzgrpvufspipbvzxzoeuuwvgcez` int DEFAULT NULL,\n `rzymjukukvdjkjwcmokqjgrxxdfhmnzh` int DEFAULT NULL,\n `kuhtdudhhnemeyqwneqcxyhnsmrrflbh` int DEFAULT NULL,\n `rzqkqtcmluqxhozmsosjyrrnzxolegwk` int DEFAULT NULL,\n `ayegrdocoermaufnvjqjqatcehtoaqbr` int DEFAULT NULL,\n `ljwvaqozwlvaxgjulpukqguqnjzeikcb` int DEFAULT NULL,\n `zadlauatjsgqwyflelimmrhxtuqemeqt` int DEFAULT NULL,\n `zoxpceleidoconnlktkjfktzvsrwyjyz` int DEFAULT NULL,\n `assmiiasqatuvxshicohbzpnkmbpifzu` int DEFAULT NULL,\n `gwvmbrpovisgrcvrvmlngjndfugcpuxz` int DEFAULT NULL,\n `clxknwsxjweclqbbvdeaoqzewiqclvxb` int DEFAULT NULL,\n `fiiyfewkfciqwjzdhpnfgppafbenjlxc` int DEFAULT NULL,\n `dyakhmbtdfgjihhrqsmkgbyayynlleoq` int DEFAULT NULL,\n `hkyisacypnxazthaapfkpsxymvwdjvlt` int DEFAULT NULL,\n `wcouzwzaokoiptbxjcuwoyznctfwjnjr` int DEFAULT NULL,\n `ccsziuxrvzdgmaldxndhgpqbxkizahhm` int DEFAULT NULL,\n `dqsfnzvhnhsfhbqadauqunvjcvxvadft` int DEFAULT NULL,\n `ujbbvkcehdfuerwlnajswbifvibdjxqs` int DEFAULT NULL,\n `monkswgtfvquzjxdtbpfgedgktxsjpkp` int DEFAULT NULL,\n `zhwguvhtfjaarhcqungehpluarzbgsin` int DEFAULT NULL,\n `fkgbgymjvwkucsgbzleftmxpinwfifzo` int DEFAULT NULL,\n `vkyegfpktdcynfaivefzsygijqawtvht` int DEFAULT NULL,\n `ftsabepzyjjqooaezazlhnzufecpzcbs` int DEFAULT NULL,\n `pmtufzusxzkqthsbesaoqejoqmzxwpcx` int DEFAULT NULL,\n `vuxumovwhnkwyirgnisfsrsfhhhsxjvk` int DEFAULT NULL,\n `jcdajcbbapctakajnexfgqeztmxydspi` int DEFAULT NULL,\n `grcmjyyjgnqzgruxefqeyzhzngtscmvd` int DEFAULT NULL,\n `illjlptcprruoutklsawmygipuzcgglu` int DEFAULT NULL,\n `lfspigcgdslhuxddekinxvfwtvaxmdwd` int DEFAULT NULL,\n `kwozwfancmhmqufklcnbuiiuzbgrhqvu` int DEFAULT NULL,\n `hzwychgbxjuzxslzeeagcwnbqsuovzxe` int DEFAULT NULL,\n `yuivxaldfapnnqpidcmhzqrafftljmac` int DEFAULT NULL,\n `bcfukfvjburgditzlelhjlohuujzchlk` int DEFAULT NULL,\n `dkupuwxapuaxjhwzlctacyyskxummgio` int DEFAULT NULL,\n `ldfchnzjjcpctqrrgdwccktysldmbtre` int DEFAULT NULL,\n `dchdthyeshljodqedgmqvoggogmfyzny` int DEFAULT NULL,\n `hmhvmfiucwhzobcnlfzpkpxkxawppubm` int DEFAULT NULL,\n `hbtzwaqayojqgvehbtrkxednqgfqnlov` int DEFAULT NULL,\n `rfqihbjwqarfkdxrfohbkzlgogmfjhvd` int DEFAULT NULL,\n `aqdydhxazpbappermgtulwqfqrbenpfj` int DEFAULT NULL,\n `hpmfgiohqufcgivxrvummxvbjxxmjkmk` int DEFAULT NULL,\n `bzbrswycssjphlwglaaujjvprhhxdpcs` int DEFAULT NULL,\n `cfbzecgcebsdwmwtfacvrokrehephcxf` int DEFAULT NULL,\n `vndhzcuiginpfibnoqwlvejrhfgdlytc` int DEFAULT NULL,\n `xlmwylikzahoiwxcsonhdntrznkvsfey` int DEFAULT NULL,\n `cdtnigovtjcenbxzqcsrmswhgxalusxh` int DEFAULT NULL,\n `abvmzqaerdjligxkutayzfaheoiyfmki` int DEFAULT NULL,\n `xbldvntnhvlkibentgickgsmdbujztlo` int DEFAULT NULL,\n `meibjzxvyhiotlnsyibbtcpomwcadhwd` int DEFAULT NULL,\n `sdrbpylodhqirtlrzjoekbyqslnbopwe` int DEFAULT NULL,\n `qjfpnhotqkbgsjzcssapyhrwtvfgkzqi` int DEFAULT NULL,\n `rgunerutusaontrvhonxdtjeeifymllo` int DEFAULT NULL,\n `nyfiurenlbnjktzhfhspoelrhpfbeubb` int DEFAULT NULL,\n `nihfjgcugwhlznjoupsgopeprdlzjtei` int DEFAULT NULL,\n `tsevojpesdaogckuoogzplouvigkkvkm` int DEFAULT NULL,\n `dukhyopbvfrsdtoglziaasjvfnbahxba` int DEFAULT NULL,\n `odxbwndgrrwfoqcsbicjmrdbkbhenvhc` int DEFAULT NULL,\n PRIMARY KEY (`kekqazutbfqfylsnvtflgteoyctgscji`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"rrxmvbjbfyljiregbwvggjujgpkwekjf\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["kekqazutbfqfylsnvtflgteoyctgscji"],"columns":[{"name":"kekqazutbfqfylsnvtflgteoyctgscji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"mserffeokdbnkxeaalirexaydtlbhoyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dguwlmlftdorxmgfnknqnhqqyazquywe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xzmeuxgfcvogeghztimenqrsmcaktcor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"snwngrqvynxtbiunzbgskvoostibnfka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajjbcaywlovbgslrlljyixtkoczsstpo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nlvprsgsxbrejcfebhwoibdcfvgcfuhs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcjyevfgqrkxmgpzhwiymjwzfnonhxhs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chinxexsupnmkcwmlvkdlnfobkundzhi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xwjfmwlokuuezboeiprmjvqleidljjox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"haxnoqhxhrqnmdidqztvobnhaeadbrft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evkxahnxazsrljjompyjznuxjojjjnam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgoqjxnfcrjigkjzdtzeeybrjkdfbhvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yvrylopztfqmyzltezaxgpneurlaawtk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gpdtjhsobqoojvxwmchrzbbxvzplbudc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmoigciodnihbuohxveethsebxkvkexd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifxqiryzlryadpnstbkruvtiwlagubaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqjdgyfwwjpgxbdessessmqmtnheidwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hocmrecwaueghznnbexelwrdcmtxvdiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"posocyikqmyozeltlzskeedyzddqnktm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmobhprmvynkcqrfywcrmgipbjuxnghw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ngcjmqkabezkmboeuxjmoxslwdrcylos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iopkylvytzcbivqgkfkbktziucbmxnzx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnxpyoeemoafbuqzpdfefajepkmyxnhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atnadqtwqjesqimlerljxwwahoreazwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pykpwivrsibucbvbokswqaydwqaogpic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dijnakdiepwvuohqgomxhlmpgdpihfqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kseddcmaledtvcqocmbcfwjpozhvdomy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzxhsuetdgzznjvbvkbpjmclsvittnyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdrgzfbljrdcpxaiiyremygxwkecukfe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rptpnvdqnrzblanvbwcaaigqcelkmblu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zfrvgysexscraoqfdqgfxppnknhhgxzk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmdfqvjsmbuusxkfdztbwftmrwybgwbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdwxxujylrojwoglflvhtijoxugcmqda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"diohifpautfbnqdqsbukbijldyklzokt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grkitjaxzhcdxgiewulrrupmgysmhozd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"buzzpexmtrzjtzpocvjmqsoescjgfprb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xtqrxgludvytftpfwisdazuupzrpjwqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfcwrgnkhtkgpiwbbviwduocqhiwbdnz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nowmgxnargnotrgikfvyqbeoealqryqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqvlpqgcczdkmspakjgcjmxwbcyafelo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdwdqthzgswmrouspcudeuewrbhxbxkc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xwakcpunljpjqteueehaoimkmnzpbxrd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kaywifzgrpvufspipbvzxzoeuuwvgcez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzymjukukvdjkjwcmokqjgrxxdfhmnzh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kuhtdudhhnemeyqwneqcxyhnsmrrflbh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzqkqtcmluqxhozmsosjyrrnzxolegwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ayegrdocoermaufnvjqjqatcehtoaqbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljwvaqozwlvaxgjulpukqguqnjzeikcb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zadlauatjsgqwyflelimmrhxtuqemeqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zoxpceleidoconnlktkjfktzvsrwyjyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"assmiiasqatuvxshicohbzpnkmbpifzu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gwvmbrpovisgrcvrvmlngjndfugcpuxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clxknwsxjweclqbbvdeaoqzewiqclvxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fiiyfewkfciqwjzdhpnfgppafbenjlxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dyakhmbtdfgjihhrqsmkgbyayynlleoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkyisacypnxazthaapfkpsxymvwdjvlt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcouzwzaokoiptbxjcuwoyznctfwjnjr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ccsziuxrvzdgmaldxndhgpqbxkizahhm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqsfnzvhnhsfhbqadauqunvjcvxvadft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujbbvkcehdfuerwlnajswbifvibdjxqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"monkswgtfvquzjxdtbpfgedgktxsjpkp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhwguvhtfjaarhcqungehpluarzbgsin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fkgbgymjvwkucsgbzleftmxpinwfifzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkyegfpktdcynfaivefzsygijqawtvht","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftsabepzyjjqooaezazlhnzufecpzcbs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmtufzusxzkqthsbesaoqejoqmzxwpcx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vuxumovwhnkwyirgnisfsrsfhhhsxjvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jcdajcbbapctakajnexfgqeztmxydspi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grcmjyyjgnqzgruxefqeyzhzngtscmvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"illjlptcprruoutklsawmygipuzcgglu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfspigcgdslhuxddekinxvfwtvaxmdwd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwozwfancmhmqufklcnbuiiuzbgrhqvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hzwychgbxjuzxslzeeagcwnbqsuovzxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yuivxaldfapnnqpidcmhzqrafftljmac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bcfukfvjburgditzlelhjlohuujzchlk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkupuwxapuaxjhwzlctacyyskxummgio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ldfchnzjjcpctqrrgdwccktysldmbtre","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dchdthyeshljodqedgmqvoggogmfyzny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmhvmfiucwhzobcnlfzpkpxkxawppubm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbtzwaqayojqgvehbtrkxednqgfqnlov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfqihbjwqarfkdxrfohbkzlgogmfjhvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqdydhxazpbappermgtulwqfqrbenpfj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hpmfgiohqufcgivxrvummxvbjxxmjkmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzbrswycssjphlwglaaujjvprhhxdpcs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfbzecgcebsdwmwtfacvrokrehephcxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vndhzcuiginpfibnoqwlvejrhfgdlytc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xlmwylikzahoiwxcsonhdntrznkvsfey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cdtnigovtjcenbxzqcsrmswhgxalusxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abvmzqaerdjligxkutayzfaheoiyfmki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbldvntnhvlkibentgickgsmdbujztlo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"meibjzxvyhiotlnsyibbtcpomwcadhwd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdrbpylodhqirtlrzjoekbyqslnbopwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjfpnhotqkbgsjzcssapyhrwtvfgkzqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgunerutusaontrvhonxdtjeeifymllo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nyfiurenlbnjktzhfhspoelrhpfbeubb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nihfjgcugwhlznjoupsgopeprdlzjtei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tsevojpesdaogckuoogzplouvigkkvkm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dukhyopbvfrsdtoglziaasjvfnbahxba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"odxbwndgrrwfoqcsbicjmrdbkbhenvhc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670961,"databaseName":"models_schema","ddl":"CREATE TABLE `rvwqmdxmybrqorpqethfdugrbikeeqen` (\n `mvovwbsmrpescqzmmhgsnzfzpqqqrxda` int NOT NULL,\n `senwxphrgjivdfluldfoqrhrpbrauukl` int DEFAULT NULL,\n `kchtrgxlbitwatjzijijbssasyzjjkaj` int DEFAULT NULL,\n `lgrokjnoxrcqyavovnrbjumsnjliqmel` int DEFAULT NULL,\n `sjffyfhkltrwhydfdqsgotwpumdurpra` int DEFAULT NULL,\n `wngbkvckykiygjmuhreuplgumfmehcrp` int DEFAULT NULL,\n `fmxcmeozhgfdazwimcadqtagunogwoie` int DEFAULT NULL,\n `hbehizqshgapkmjlmoyfiewtmfhgpgks` int DEFAULT NULL,\n `ocicivqobvamthlrxmjcunculnhjtmni` int DEFAULT NULL,\n `xwoqlzxdxyfoytwrgoiduetyrfovlbhq` int DEFAULT NULL,\n `sukrgsqfoecqgipqefjnvbmvqztwdpmr` int DEFAULT NULL,\n `cjloxxdokedjmbaezwtpqadlsyqyogho` int DEFAULT NULL,\n `awwehdfglitllkcxfzpvwvhjyqfseznx` int DEFAULT NULL,\n `lznkvyrnxcxhfgfbbfzdjhgzmldkhntx` int DEFAULT NULL,\n `wepgukvynvkwsnwdxnhpkuekkdfzfzzn` int DEFAULT NULL,\n `kidugtguuytfetfkhnnxxxxljqkshrfd` int DEFAULT NULL,\n `dsupezsyvkuatusotewacohlnqxmemvh` int DEFAULT NULL,\n `leomwokrqpwxhakqkiwgtwpqexdfygeg` int DEFAULT NULL,\n `usekaawbwavhamjengatfuwydbuphfnd` int DEFAULT NULL,\n `fjwvlokeegntkwcibvuskykxuegvxijp` int DEFAULT NULL,\n `lidhwpjjwpmfhpteopdlhlgpztqabdfm` int DEFAULT NULL,\n `xaxvdsbimubqfrspisoikjmebutzmykf` int DEFAULT NULL,\n `pcieworzoznyultulqiwuzdednvhtimr` int DEFAULT NULL,\n `gadnzgqtrhrcnsonoqjjyvhbuwkjkzkb` int DEFAULT NULL,\n `szjghstsulbzzhoepxskepygpiejippz` int DEFAULT NULL,\n `voqhemmjugolaljbskstaelgdpcayyxw` int DEFAULT NULL,\n `dtiznwsovnblpbrbfqaspfwivpbmammm` int DEFAULT NULL,\n `cejgqncsottrwnhgsyokzgyhtehfrfti` int DEFAULT NULL,\n `ehrsatipdmetlgplivfkqhjcyvxdlqsf` int DEFAULT NULL,\n `rgsakuonlqpkwshirjiujmtaoidsrgul` int DEFAULT NULL,\n `fnmjsydhrnglvvflfnhvbhbzyupawlld` int DEFAULT NULL,\n `mkzikzoitkernrwvpyaznxdqqekokcef` int DEFAULT NULL,\n `lfxiimtlefcbjimnshuknkwvvwbytpcz` int DEFAULT NULL,\n `jevpghzfndotzvljmywqafrzrvtoveqe` int DEFAULT NULL,\n `rmcitzwunrichrgjirhogcuexfkjegws` int DEFAULT NULL,\n `ixxdvselscowejtootctgtnbrgsufbfg` int DEFAULT NULL,\n `uymyvgdembuedxinfwvzktevapcpivbx` int DEFAULT NULL,\n `pxlxjcjwijosrlfllcjutwnzezlkblbp` int DEFAULT NULL,\n `iupputgzwheueyozdeurvwffblxxetji` int DEFAULT NULL,\n `bnkvtbofgqsmiyqtaxjeffcdcmbuvmew` int DEFAULT NULL,\n `btvwwzenldpkykiosgawloynpfdbmred` int DEFAULT NULL,\n `rxlnnrvulqrgpalhaorilinrrhufhqnu` int DEFAULT NULL,\n `djpibrcrlqbxrlqhjwnhwvmrakmkngkb` int DEFAULT NULL,\n `oanqyxezqmstycijlpzrxlrezqbsgsyk` int DEFAULT NULL,\n `orxkqpicekehonnjfxiryyrcwddhopnd` int DEFAULT NULL,\n `uwwneldfrbukunlpxsylvfslxnsqgefp` int DEFAULT NULL,\n `ovibuqhchermpwoqxyuubhxbljhzmtya` int DEFAULT NULL,\n `mxjspzyfclfomhdzwqvaehysnltntlat` int DEFAULT NULL,\n `phzwrjnkbcyoqpljfxvdlvphzmuffmum` int DEFAULT NULL,\n `hfxdolayftzwmozksgbjtrsowkkrhlyq` int DEFAULT NULL,\n `azwpxbjfyjstvqjltsvxeukkxfchyrii` int DEFAULT NULL,\n `vgblzrxfkauveglloridpabeeliowvtw` int DEFAULT NULL,\n `jbkajjdyxenelaoewmbkkvulwrqlacyz` int DEFAULT NULL,\n `nahkzghlaqythwukqdfrmpnaukxyfigc` int DEFAULT NULL,\n `ihsblpxrvstceixjdiijnnihezshjjsc` int DEFAULT NULL,\n `nbykfkmilnjepjjscqyiyewpsziwvbgo` int DEFAULT NULL,\n `oaxhcjxfzgwrrczhwvxailczdawuvheq` int DEFAULT NULL,\n `mpqvrfjopbfehsyiswdtzqrrxzyhksig` int DEFAULT NULL,\n `tathmbqlsikoyllrfdjtbppqhulqaobe` int DEFAULT NULL,\n `haopkbxfplewmlqbijdpztbyaxmggyac` int DEFAULT NULL,\n `nxlpcyvmyyoleancunigmydkfxrzidhc` int DEFAULT NULL,\n `cypxqaebvswotymmhoyeqczytmduasye` int DEFAULT NULL,\n `yxbwtizckjdcfapomxfilqnjxkwzaecj` int DEFAULT NULL,\n `clgdrwgzjggawhewjtskjsauvkqzlhib` int DEFAULT NULL,\n `vgeelrmjckvlviqwzuiggxniqoeglgmn` int DEFAULT NULL,\n `nephjlvzrohegoraftnkmbcpagwnwjep` int DEFAULT NULL,\n `dppvecymkqniekdhlimaxegmpwftylmi` int DEFAULT NULL,\n `xkkuloexgchkmnvmzbmvcybjhzehyrlx` int DEFAULT NULL,\n `sauerpesyrshlleiheysvqpctggllexy` int DEFAULT NULL,\n `psokoxsuzgpxkvexyahucholwrzpdmey` int DEFAULT NULL,\n `xyscyajdggaeeisbhiilpkxrnwetmjad` int DEFAULT NULL,\n `scuipqzxigedycswrzzxxiwbuysiucrb` int DEFAULT NULL,\n `eeubiqhwpjpzqbfvwcufnagsibleropl` int DEFAULT NULL,\n `tlagdwktvzpcqyqzthqacujvacpeyesq` int DEFAULT NULL,\n `igpqttjibmcevspmbtjhsyoddjwygqet` int DEFAULT NULL,\n `budmzdawcdmlksjvuqnloqpbeqegssix` int DEFAULT NULL,\n `uqyduqseivzjzhpxlxlomhzsawywtfer` int DEFAULT NULL,\n `revdscmdyfnbxejuxnnqmwhclatxgpeo` int DEFAULT NULL,\n `kpjnadtxkgzfwcytwcntfjeckobemzno` int DEFAULT NULL,\n `vgloxgvxyejsxnadjmdnwzwnlwodbzme` int DEFAULT NULL,\n `rgclokjzqmonuqmvhcngcrqgimhxhcgv` int DEFAULT NULL,\n `vgqvqnbcelmzrqfxenfcbypxcvsbzrrw` int DEFAULT NULL,\n `spxryppgxazrsbrcrhqpwxeloxdbluaq` int DEFAULT NULL,\n `noudcrlpwosgarmodwodzwdgqcrxvlok` int DEFAULT NULL,\n `ooehufjhmvjbbtjzpsurifiuvvlbnvoq` int DEFAULT NULL,\n `luleeiiljahhksnwgkhgshzrnyodhkwz` int DEFAULT NULL,\n `hwycqocctqxmvsyjkdjlrzzmmzmjvykk` int DEFAULT NULL,\n `meqcubfgjrwebaarleklrsyuydxllozw` int DEFAULT NULL,\n `ykhhhjwifbshxwupzexswmsnxixzjlua` int DEFAULT NULL,\n `syrxeuvuuglfuitpgabqnrzronyckway` int DEFAULT NULL,\n `vagdgceglcndrjmpknjrxakpzjfeusbq` int DEFAULT NULL,\n `msrhpapabnxgjumfgdsfiesiddybmrqo` int DEFAULT NULL,\n `ffypyqicltnnslsamfssvbfcqndhmaio` int DEFAULT NULL,\n `beulorvedxjkwvexjwmjtcplqhhcypji` int DEFAULT NULL,\n `tpkypsuxhsscngthgsdkvafbrvkjfxdc` int DEFAULT NULL,\n `lkvlpmdccesbnxfhdupgwctfcfzzehqx` int DEFAULT NULL,\n `diuvnnsrlhruqhxfxabhzywevcjakpje` int DEFAULT NULL,\n `uvddkszksyozavhtpeucdxcfyqybqiyt` int DEFAULT NULL,\n `akalmajkmdptrrupaguavlsuqdymuvxr` int DEFAULT NULL,\n `hlsiwodzlqsennypetxxbtzvakkpmvax` int DEFAULT NULL,\n PRIMARY KEY (`mvovwbsmrpescqzmmhgsnzfzpqqqrxda`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"rvwqmdxmybrqorpqethfdugrbikeeqen\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["mvovwbsmrpescqzmmhgsnzfzpqqqrxda"],"columns":[{"name":"mvovwbsmrpescqzmmhgsnzfzpqqqrxda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"senwxphrgjivdfluldfoqrhrpbrauukl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kchtrgxlbitwatjzijijbssasyzjjkaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgrokjnoxrcqyavovnrbjumsnjliqmel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjffyfhkltrwhydfdqsgotwpumdurpra","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wngbkvckykiygjmuhreuplgumfmehcrp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmxcmeozhgfdazwimcadqtagunogwoie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbehizqshgapkmjlmoyfiewtmfhgpgks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocicivqobvamthlrxmjcunculnhjtmni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xwoqlzxdxyfoytwrgoiduetyrfovlbhq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sukrgsqfoecqgipqefjnvbmvqztwdpmr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjloxxdokedjmbaezwtpqadlsyqyogho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awwehdfglitllkcxfzpvwvhjyqfseznx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lznkvyrnxcxhfgfbbfzdjhgzmldkhntx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wepgukvynvkwsnwdxnhpkuekkdfzfzzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kidugtguuytfetfkhnnxxxxljqkshrfd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsupezsyvkuatusotewacohlnqxmemvh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"leomwokrqpwxhakqkiwgtwpqexdfygeg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"usekaawbwavhamjengatfuwydbuphfnd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjwvlokeegntkwcibvuskykxuegvxijp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lidhwpjjwpmfhpteopdlhlgpztqabdfm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xaxvdsbimubqfrspisoikjmebutzmykf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pcieworzoznyultulqiwuzdednvhtimr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gadnzgqtrhrcnsonoqjjyvhbuwkjkzkb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"szjghstsulbzzhoepxskepygpiejippz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"voqhemmjugolaljbskstaelgdpcayyxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtiznwsovnblpbrbfqaspfwivpbmammm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cejgqncsottrwnhgsyokzgyhtehfrfti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehrsatipdmetlgplivfkqhjcyvxdlqsf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgsakuonlqpkwshirjiujmtaoidsrgul","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnmjsydhrnglvvflfnhvbhbzyupawlld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkzikzoitkernrwvpyaznxdqqekokcef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfxiimtlefcbjimnshuknkwvvwbytpcz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jevpghzfndotzvljmywqafrzrvtoveqe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rmcitzwunrichrgjirhogcuexfkjegws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixxdvselscowejtootctgtnbrgsufbfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uymyvgdembuedxinfwvzktevapcpivbx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxlxjcjwijosrlfllcjutwnzezlkblbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iupputgzwheueyozdeurvwffblxxetji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnkvtbofgqsmiyqtaxjeffcdcmbuvmew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"btvwwzenldpkykiosgawloynpfdbmred","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxlnnrvulqrgpalhaorilinrrhufhqnu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"djpibrcrlqbxrlqhjwnhwvmrakmkngkb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oanqyxezqmstycijlpzrxlrezqbsgsyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orxkqpicekehonnjfxiryyrcwddhopnd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwwneldfrbukunlpxsylvfslxnsqgefp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovibuqhchermpwoqxyuubhxbljhzmtya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mxjspzyfclfomhdzwqvaehysnltntlat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phzwrjnkbcyoqpljfxvdlvphzmuffmum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfxdolayftzwmozksgbjtrsowkkrhlyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azwpxbjfyjstvqjltsvxeukkxfchyrii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgblzrxfkauveglloridpabeeliowvtw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbkajjdyxenelaoewmbkkvulwrqlacyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nahkzghlaqythwukqdfrmpnaukxyfigc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ihsblpxrvstceixjdiijnnihezshjjsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbykfkmilnjepjjscqyiyewpsziwvbgo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oaxhcjxfzgwrrczhwvxailczdawuvheq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpqvrfjopbfehsyiswdtzqrrxzyhksig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tathmbqlsikoyllrfdjtbppqhulqaobe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"haopkbxfplewmlqbijdpztbyaxmggyac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxlpcyvmyyoleancunigmydkfxrzidhc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cypxqaebvswotymmhoyeqczytmduasye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxbwtizckjdcfapomxfilqnjxkwzaecj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clgdrwgzjggawhewjtskjsauvkqzlhib","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgeelrmjckvlviqwzuiggxniqoeglgmn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nephjlvzrohegoraftnkmbcpagwnwjep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dppvecymkqniekdhlimaxegmpwftylmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkkuloexgchkmnvmzbmvcybjhzehyrlx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sauerpesyrshlleiheysvqpctggllexy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psokoxsuzgpxkvexyahucholwrzpdmey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyscyajdggaeeisbhiilpkxrnwetmjad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scuipqzxigedycswrzzxxiwbuysiucrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eeubiqhwpjpzqbfvwcufnagsibleropl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlagdwktvzpcqyqzthqacujvacpeyesq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igpqttjibmcevspmbtjhsyoddjwygqet","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"budmzdawcdmlksjvuqnloqpbeqegssix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqyduqseivzjzhpxlxlomhzsawywtfer","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"revdscmdyfnbxejuxnnqmwhclatxgpeo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpjnadtxkgzfwcytwcntfjeckobemzno","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgloxgvxyejsxnadjmdnwzwnlwodbzme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgclokjzqmonuqmvhcngcrqgimhxhcgv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgqvqnbcelmzrqfxenfcbypxcvsbzrrw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"spxryppgxazrsbrcrhqpwxeloxdbluaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"noudcrlpwosgarmodwodzwdgqcrxvlok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ooehufjhmvjbbtjzpsurifiuvvlbnvoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"luleeiiljahhksnwgkhgshzrnyodhkwz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwycqocctqxmvsyjkdjlrzzmmzmjvykk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"meqcubfgjrwebaarleklrsyuydxllozw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykhhhjwifbshxwupzexswmsnxixzjlua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"syrxeuvuuglfuitpgabqnrzronyckway","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vagdgceglcndrjmpknjrxakpzjfeusbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"msrhpapabnxgjumfgdsfiesiddybmrqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffypyqicltnnslsamfssvbfcqndhmaio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"beulorvedxjkwvexjwmjtcplqhhcypji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpkypsuxhsscngthgsdkvafbrvkjfxdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lkvlpmdccesbnxfhdupgwctfcfzzehqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"diuvnnsrlhruqhxfxabhzywevcjakpje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvddkszksyozavhtpeucdxcfyqybqiyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akalmajkmdptrrupaguavlsuqdymuvxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlsiwodzlqsennypetxxbtzvakkpmvax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842670,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842670994,"databaseName":"models_schema","ddl":"CREATE TABLE `sddicdunfmgiuvegaxtmieursdrycsld` (\n `lmayayxeczzyswnwokdgtdtezojmhxxh` int NOT NULL,\n `aaiaggipllscvffjwzbyevcbrjgptimx` int DEFAULT NULL,\n `tuwwgithjuimzplkcilocdfoufghdrdg` int DEFAULT NULL,\n `hjqtliiulotixfcjavlyvxswoshthpqd` int DEFAULT NULL,\n `kmulkuxkybimhawsagpgikwuqvhpprmc` int DEFAULT NULL,\n `lxyrjjypgxprdvvencrzsssnburpglkp` int DEFAULT NULL,\n `qlvaumhansxsmeipuzxiaupcbspllqgd` int DEFAULT NULL,\n `rhzxhdektejfovvrhmbxzpflwrirnlxv` int DEFAULT NULL,\n `utzqojnsczfzuuieztatontskccrytio` int DEFAULT NULL,\n `barxdcdfxaathotiheldyimiwbxnxlxa` int DEFAULT NULL,\n `nbbhmmprgxcthdprlduhmdutneczobrp` int DEFAULT NULL,\n `lcdumezscwxqqzrcsxpxniaidduabycy` int DEFAULT NULL,\n `jkgcnhdwsmjndomjfdlmxgovpanshxcz` int DEFAULT NULL,\n `qwakecwcewhmkfqrzorzuoyvtlceprcw` int DEFAULT NULL,\n `bujpeuadbdbkwcbfdxcguwhqjymejlhh` int DEFAULT NULL,\n `fmxjsmsneknkscudhzsqgvylonqfhgvx` int DEFAULT NULL,\n `gijzvnaortyzbitgijtupfjborlresut` int DEFAULT NULL,\n `tfhiwthgytprcibfqiwtcrsyxrksbjrn` int DEFAULT NULL,\n `tqfzpumusmmoucmxmhcnnbgorvqiaymw` int DEFAULT NULL,\n `cqkqcaqdjyvwrngbzmnaxyomudaorsei` int DEFAULT NULL,\n `vsdjcszckhlqaplylbizwqzjqqnpzuad` int DEFAULT NULL,\n `rgnhhlklnfjfvgzaxoolnhjeuqllnqyq` int DEFAULT NULL,\n `daxvlqerwwiuguxwamrtknxivgzjmfex` int DEFAULT NULL,\n `jaahwerurucxzxumamuzfwdghtdsalpz` int DEFAULT NULL,\n `kjfznanuwoxljacuqpbrnddmjbgtjaym` int DEFAULT NULL,\n `barvdpbmnwazsqktwoxsiipnkeaasesz` int DEFAULT NULL,\n `vtvwnotiyrnxgnvkryybcsldffhboxpq` int DEFAULT NULL,\n `uwfbnofnovhnwryvueffvnvmdowlxedw` int DEFAULT NULL,\n `nqcbqkpqtmgfirvaarlxabymespfvlcd` int DEFAULT NULL,\n `hscwkdvatctjvsotbfvevofgylhhetzo` int DEFAULT NULL,\n `zgghklmxyviapwrierhqleoudhmawnfn` int DEFAULT NULL,\n `idmeovotgepjanqjkvlqmudfjdxswamn` int DEFAULT NULL,\n `etzrerwbeytrlnzayocleqhiesfclsmk` int DEFAULT NULL,\n `vvsolhdwfmskaojojwuyydvlckopmnga` int DEFAULT NULL,\n `crbgvzrrmsvtvoxhkzabixygthmtmpjn` int DEFAULT NULL,\n `iwkzkczwteapojhpmyukqqhnbptxdljl` int DEFAULT NULL,\n `yglndmswupsvjfpsxkdrutdeynkxsbln` int DEFAULT NULL,\n `yjwlmvsqsvfkkimyumlfevpjhwcyinwh` int DEFAULT NULL,\n `kxwmhxjcxhfxwfsguaqxuhvpncqtnejt` int DEFAULT NULL,\n `wibscietuqbfpjgdwpnicnytyohvrkrr` int DEFAULT NULL,\n `rzjeztkqotctvcupywauefxpafrpcbxq` int DEFAULT NULL,\n `hirlhaonhrrkfgecbzakdjzvuvbbzopo` int DEFAULT NULL,\n `erncitlkkxudtqdhjthfhhjjmdzzvwof` int DEFAULT NULL,\n `upuiilmlqoobxvrzuvxzvyyzrejosqwm` int DEFAULT NULL,\n `fkcsnqztgklqajjjexxsegxulquyyjet` int DEFAULT NULL,\n `hymbgarexpxmrgflgfkhmatxjztmjufc` int DEFAULT NULL,\n `bmbklowugvislldwcboansaxayrixonc` int DEFAULT NULL,\n `rgrckxzegvdmzzfnyhmswywgdeoaluzc` int DEFAULT NULL,\n `oowyqzcngosotoqyrdotnddpbkmwnbic` int DEFAULT NULL,\n `wsjdcmkaislhfgpvjhjgmpejizldfyit` int DEFAULT NULL,\n `wwoausvlmmvmekvweeyewqrddfdqpihs` int DEFAULT NULL,\n `geuiwapbxecthkwyrbnjblvwzcumgrzf` int DEFAULT NULL,\n `opshdrodlbkdutzfzmkshocqgkbsgwsi` int DEFAULT NULL,\n `taaqnvlxuyktyipuikjltpobiygnovzq` int DEFAULT NULL,\n `dtacxeuoeyydpernuyxgoudmmbzeiimk` int DEFAULT NULL,\n `wioweiwoqugtaosekrlgpjzeoivvrrzy` int DEFAULT NULL,\n `zhvldsaoettruwbameanbxgfzbjjsbbr` int DEFAULT NULL,\n `nsczdunlwgoflxlgawtaagrifwduehhl` int DEFAULT NULL,\n `zxxswtcowvntuiejtojrtxwoekrkgejd` int DEFAULT NULL,\n `nqhkchctrvsawvbfqjgfubqmlkkhbwgj` int DEFAULT NULL,\n `zhcqdfkzusdobccjlxztaeiupiflnccm` int DEFAULT NULL,\n `gzmdygylfanlavwrzixwpvdpabqkfdox` int DEFAULT NULL,\n `qnwlhbpmvbcsihgngnmqrqbxdzojxbay` int DEFAULT NULL,\n `rrlbdnqzvgyzreimeqhhmnkfcnmifibw` int DEFAULT NULL,\n `zkaqdqkwpazwktgafxkchbzwuhytsabf` int DEFAULT NULL,\n `piyvfoneyfmiexbjgcuitjgkrnxhlbck` int DEFAULT NULL,\n `szsbdbaqwfubgdifndfqpszsfwumadmx` int DEFAULT NULL,\n `blizvescwfigelzdujsyrvpgmmtccwnr` int DEFAULT NULL,\n `ajbfefqlxlmixuyffohtmpucpnbvhtyw` int DEFAULT NULL,\n `rpjgnryhevlzyrsycirfokikgprmipme` int DEFAULT NULL,\n `ouseopigfkftwmrmppqosdemnwphnjql` int DEFAULT NULL,\n `hinzrrbgjlmbsskzjauaxchctkgugbys` int DEFAULT NULL,\n `owqlbdpxptwezwwsdsatsbedbjtcxxlu` int DEFAULT NULL,\n `ymqqqmbrdysflopjjxearxxislimfamy` int DEFAULT NULL,\n `dlhxauuevyfxvadkgmzsvejkdamkhlyh` int DEFAULT NULL,\n `dzsxtwlhcbbncxzbiytzwosombmknnke` int DEFAULT NULL,\n `bepavpuqxbpsitpxphatvqpjjconhrcg` int DEFAULT NULL,\n `brlfmyurnvxykmixijscidhzjqtovzxe` int DEFAULT NULL,\n `fajfvszawqcoqenxraytrejprfjzzvsg` int DEFAULT NULL,\n `tpcxfvlyxyscrayftexriktfqujjwzsw` int DEFAULT NULL,\n `ihkbzrfxslwqiwmgcchpwairlhwhlydx` int DEFAULT NULL,\n `seynxgrxgpsgdjyjjrovknnrllzbiski` int DEFAULT NULL,\n `omohmkgmjsvsqdghamrkgnawoxwyofva` int DEFAULT NULL,\n `iuzgvkmlfpuszyxitduqsstusbgwcesg` int DEFAULT NULL,\n `rrgrifymfiqsikvgsuocvcurfjtdgfwx` int DEFAULT NULL,\n `qwmlhkhwjmljjgmtimgcpdjkjejtreeh` int DEFAULT NULL,\n `hehbbvghrviyaqlhuzuofmpslcmygezs` int DEFAULT NULL,\n `nzpsqpcizxxgaxkjfsalgrummvtvoyzm` int DEFAULT NULL,\n `wejeyuvorfjxjtdahyuemwjiljsajxnu` int DEFAULT NULL,\n `jytokbfzvzaukokqjljxhojjrghewohw` int DEFAULT NULL,\n `widuipqknqdhszbncdavfjsdedewsyvs` int DEFAULT NULL,\n `sebtbamrinldmkgtqoftysrsalqyfzka` int DEFAULT NULL,\n `pulsupowtrqleuatxuodtpzkcetfekye` int DEFAULT NULL,\n `ifnzylgvfrchkbnulwdijdwufuvuilii` int DEFAULT NULL,\n `rfyiabobhwujiundkqkpcnnflkelunia` int DEFAULT NULL,\n `ybxzailvvddcmblktovwwrvmjoldcjgu` int DEFAULT NULL,\n `pmdwutycqmclacuukfukcwiqoekchwuv` int DEFAULT NULL,\n `ehvuomldcytwtzfuwsorsxapzfsrvxay` int DEFAULT NULL,\n `mztlwlwkibbhnnesckeslstmbjihdhkc` int DEFAULT NULL,\n `vffrpbisztukffthbiqvjqkwhvlmaamm` int DEFAULT NULL,\n PRIMARY KEY (`lmayayxeczzyswnwokdgtdtezojmhxxh`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"sddicdunfmgiuvegaxtmieursdrycsld\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["lmayayxeczzyswnwokdgtdtezojmhxxh"],"columns":[{"name":"lmayayxeczzyswnwokdgtdtezojmhxxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"aaiaggipllscvffjwzbyevcbrjgptimx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tuwwgithjuimzplkcilocdfoufghdrdg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjqtliiulotixfcjavlyvxswoshthpqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmulkuxkybimhawsagpgikwuqvhpprmc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxyrjjypgxprdvvencrzsssnburpglkp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qlvaumhansxsmeipuzxiaupcbspllqgd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhzxhdektejfovvrhmbxzpflwrirnlxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utzqojnsczfzuuieztatontskccrytio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"barxdcdfxaathotiheldyimiwbxnxlxa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbbhmmprgxcthdprlduhmdutneczobrp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lcdumezscwxqqzrcsxpxniaidduabycy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jkgcnhdwsmjndomjfdlmxgovpanshxcz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwakecwcewhmkfqrzorzuoyvtlceprcw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bujpeuadbdbkwcbfdxcguwhqjymejlhh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmxjsmsneknkscudhzsqgvylonqfhgvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gijzvnaortyzbitgijtupfjborlresut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tfhiwthgytprcibfqiwtcrsyxrksbjrn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqfzpumusmmoucmxmhcnnbgorvqiaymw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqkqcaqdjyvwrngbzmnaxyomudaorsei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsdjcszckhlqaplylbizwqzjqqnpzuad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgnhhlklnfjfvgzaxoolnhjeuqllnqyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"daxvlqerwwiuguxwamrtknxivgzjmfex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jaahwerurucxzxumamuzfwdghtdsalpz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjfznanuwoxljacuqpbrnddmjbgtjaym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"barvdpbmnwazsqktwoxsiipnkeaasesz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtvwnotiyrnxgnvkryybcsldffhboxpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwfbnofnovhnwryvueffvnvmdowlxedw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqcbqkpqtmgfirvaarlxabymespfvlcd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hscwkdvatctjvsotbfvevofgylhhetzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zgghklmxyviapwrierhqleoudhmawnfn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idmeovotgepjanqjkvlqmudfjdxswamn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"etzrerwbeytrlnzayocleqhiesfclsmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vvsolhdwfmskaojojwuyydvlckopmnga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crbgvzrrmsvtvoxhkzabixygthmtmpjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwkzkczwteapojhpmyukqqhnbptxdljl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yglndmswupsvjfpsxkdrutdeynkxsbln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjwlmvsqsvfkkimyumlfevpjhwcyinwh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxwmhxjcxhfxwfsguaqxuhvpncqtnejt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wibscietuqbfpjgdwpnicnytyohvrkrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzjeztkqotctvcupywauefxpafrpcbxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hirlhaonhrrkfgecbzakdjzvuvbbzopo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"erncitlkkxudtqdhjthfhhjjmdzzvwof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upuiilmlqoobxvrzuvxzvyyzrejosqwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fkcsnqztgklqajjjexxsegxulquyyjet","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hymbgarexpxmrgflgfkhmatxjztmjufc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bmbklowugvislldwcboansaxayrixonc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgrckxzegvdmzzfnyhmswywgdeoaluzc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oowyqzcngosotoqyrdotnddpbkmwnbic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wsjdcmkaislhfgpvjhjgmpejizldfyit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwoausvlmmvmekvweeyewqrddfdqpihs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"geuiwapbxecthkwyrbnjblvwzcumgrzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opshdrodlbkdutzfzmkshocqgkbsgwsi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"taaqnvlxuyktyipuikjltpobiygnovzq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtacxeuoeyydpernuyxgoudmmbzeiimk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wioweiwoqugtaosekrlgpjzeoivvrrzy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhvldsaoettruwbameanbxgfzbjjsbbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nsczdunlwgoflxlgawtaagrifwduehhl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxxswtcowvntuiejtojrtxwoekrkgejd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqhkchctrvsawvbfqjgfubqmlkkhbwgj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhcqdfkzusdobccjlxztaeiupiflnccm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzmdygylfanlavwrzixwpvdpabqkfdox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qnwlhbpmvbcsihgngnmqrqbxdzojxbay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrlbdnqzvgyzreimeqhhmnkfcnmifibw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zkaqdqkwpazwktgafxkchbzwuhytsabf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"piyvfoneyfmiexbjgcuitjgkrnxhlbck","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"szsbdbaqwfubgdifndfqpszsfwumadmx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blizvescwfigelzdujsyrvpgmmtccwnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajbfefqlxlmixuyffohtmpucpnbvhtyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpjgnryhevlzyrsycirfokikgprmipme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ouseopigfkftwmrmppqosdemnwphnjql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hinzrrbgjlmbsskzjauaxchctkgugbys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owqlbdpxptwezwwsdsatsbedbjtcxxlu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ymqqqmbrdysflopjjxearxxislimfamy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlhxauuevyfxvadkgmzsvejkdamkhlyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dzsxtwlhcbbncxzbiytzwosombmknnke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bepavpuqxbpsitpxphatvqpjjconhrcg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"brlfmyurnvxykmixijscidhzjqtovzxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fajfvszawqcoqenxraytrejprfjzzvsg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpcxfvlyxyscrayftexriktfqujjwzsw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ihkbzrfxslwqiwmgcchpwairlhwhlydx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"seynxgrxgpsgdjyjjrovknnrllzbiski","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omohmkgmjsvsqdghamrkgnawoxwyofva","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iuzgvkmlfpuszyxitduqsstusbgwcesg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrgrifymfiqsikvgsuocvcurfjtdgfwx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwmlhkhwjmljjgmtimgcpdjkjejtreeh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hehbbvghrviyaqlhuzuofmpslcmygezs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzpsqpcizxxgaxkjfsalgrummvtvoyzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wejeyuvorfjxjtdahyuemwjiljsajxnu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jytokbfzvzaukokqjljxhojjrghewohw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"widuipqknqdhszbncdavfjsdedewsyvs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sebtbamrinldmkgtqoftysrsalqyfzka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pulsupowtrqleuatxuodtpzkcetfekye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifnzylgvfrchkbnulwdijdwufuvuilii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfyiabobhwujiundkqkpcnnflkelunia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybxzailvvddcmblktovwwrvmjoldcjgu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmdwutycqmclacuukfukcwiqoekchwuv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehvuomldcytwtzfuwsorsxapzfsrvxay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mztlwlwkibbhnnesckeslstmbjihdhkc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vffrpbisztukffthbiqvjqkwhvlmaamm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671025,"databaseName":"models_schema","ddl":"CREATE TABLE `seckcqhiaewieeiomssfzbhcvhkqmjnt` (\n `oovfrslhvjczrfstpedrbajuvdsqgkdi` int NOT NULL,\n `reshbefeivgyzcvipyffiralbgwcniwm` int DEFAULT NULL,\n `ebszyvmoitfudfvbxlxawqnczmmahjuq` int DEFAULT NULL,\n `zunfdaaebgofdhlcouwgpaovwldpyjmy` int DEFAULT NULL,\n `jznifskpbwodtorleieoekyfswqeorzq` int DEFAULT NULL,\n `xdbxoncqrkokwzvrmvypzuckxjujjdxl` int DEFAULT NULL,\n `rftwymplokumlpgeueazjzoemnamfgrm` int DEFAULT NULL,\n `xjiutzupgticqmvppagiylocggfdsxme` int DEFAULT NULL,\n `dldmkqnuegxrztoczievaojahhrumnfj` int DEFAULT NULL,\n `fiqwaioorcfhawnzqqreloluupfemkuf` int DEFAULT NULL,\n `egndsdwksuifsiigkanouzndhjtrhhma` int DEFAULT NULL,\n `czhpjibwtzmudelxcxtqoxbznbeewjsc` int DEFAULT NULL,\n `fwziqetwmgvkdvklphpazxdkkefqemii` int DEFAULT NULL,\n `firaliasykqwxuldjmpterrbqiqcpbzw` int DEFAULT NULL,\n `kkqvwheofxlntkuaawbgktgsgwnibcdu` int DEFAULT NULL,\n `mseypvsmnyxschylbfmkkacpejzprlrv` int DEFAULT NULL,\n `fyhdnkzggfkmoacuweancutzagefyjro` int DEFAULT NULL,\n `zjeuwcafakbzmhxmvlznylazxoorjakk` int DEFAULT NULL,\n `qgfanpbxivodnaythctrcnegweuazame` int DEFAULT NULL,\n `ywzrlugxgjjmrulbuevcdsrjofadfiex` int DEFAULT NULL,\n `edslmvlezfdxtrqkeyukjsjamtwxkppg` int DEFAULT NULL,\n `lamoxrjjrtgyjomvnbaapxuekjlszwzd` int DEFAULT NULL,\n `hwetchxahznkhafcbjwfatcngzbzwbie` int DEFAULT NULL,\n `apeirbtzvejpegmonzbpaguhzmjhqyos` int DEFAULT NULL,\n `prmvtubenqnyfreppqefrwhxwtcclexk` int DEFAULT NULL,\n `jraeqcwjthmexugrzqrfqwpfawokjzgi` int DEFAULT NULL,\n `ysoqpbotjoohyyyfmybjgecuiqyhccrk` int DEFAULT NULL,\n `tqohgcpokmyvjwaamtlwmqeyxeupasnn` int DEFAULT NULL,\n `yhtadsupzwybyxnnnchcgeecsdltljtf` int DEFAULT NULL,\n `jyymrvujabpntzvuvfljwbeyqwvlpytr` int DEFAULT NULL,\n `xqodfnpvyivmcuphshrheaqrdmkonfnx` int DEFAULT NULL,\n `kiisxswtdamaorxysexbspaxgygtnxsi` int DEFAULT NULL,\n `lwtuiquuczyfuanzvoepwgbefutxblyx` int DEFAULT NULL,\n `adbxfdscevgfynnxykfaakzlyxycnsld` int DEFAULT NULL,\n `sjjazejdlejskagggmpmtykvdmanzhiy` int DEFAULT NULL,\n `jzctabzglbwwkfctrkjxnbspyiynmsev` int DEFAULT NULL,\n `rxqsxglceuojxiepucxquwgecweaxzcl` int DEFAULT NULL,\n `lrbgyibcwkcrkfrpyijhveexuupsqvey` int DEFAULT NULL,\n `okqgxftvvvuvbwzmfwvsofurwghndcdt` int DEFAULT NULL,\n `jlfbuhxzauildcvknpjbogpgogskhuza` int DEFAULT NULL,\n `voydsijnclkkqoxyngcuatyrhobunrbk` int DEFAULT NULL,\n `lwepcekkvcazclaoyttpogetaxxhtqjr` int DEFAULT NULL,\n `adeimqwucaijhfsstsjawrtcwyunrcaq` int DEFAULT NULL,\n `ncwtguovmkgykhgpvejfkdjdfiprfzrl` int DEFAULT NULL,\n `iowrgyfzilefanqndrghtcbxeutxvsfz` int DEFAULT NULL,\n `kceetimgfujmvnjyftntdxuzptlgvpwt` int DEFAULT NULL,\n `apqbcgmaovdikcrfkbjjvlvguysunqrb` int DEFAULT NULL,\n `dscrxuapcfmayrkxqpphihcurwrhxfyx` int DEFAULT NULL,\n `cnvywfjakkqwzszbitogydwrplmphgzg` int DEFAULT NULL,\n `lsiyawaoemgkrodnsfymorvnjzqixtkx` int DEFAULT NULL,\n `zflejbarwckjjjloexbwgptkphokyoim` int DEFAULT NULL,\n `qwjasricywoyynciltblrmixouupbebr` int DEFAULT NULL,\n `lyksjpzgsccrrxnjwwutyhzsrzzdydaq` int DEFAULT NULL,\n `pkelrozpzmxnicmaczztofkoyoyxcdxy` int DEFAULT NULL,\n `vnlwvbbpxhxbsrvihnifzaqhiamozjbw` int DEFAULT NULL,\n `vyakuwnpkmpurfodutpbmvvbwinjrqym` int DEFAULT NULL,\n `mbrqjmmgcpeyxkkyqkqgpshaepqxvgxs` int DEFAULT NULL,\n `fmbykqallnllhjykmwcfvfraosfbtfet` int DEFAULT NULL,\n `nizrygnndzzrbptdzuslzrlcuxugfyfj` int DEFAULT NULL,\n `ooikfirshlpztfmdojdmhxocuiblwxnc` int DEFAULT NULL,\n `fgqqanfalttvjdqksdpdshcbmyozidhw` int DEFAULT NULL,\n `jktaffqqcfscnmjxopqtzrqxhdaobotr` int DEFAULT NULL,\n `ivtqpqgkidqrenmpjhyfflxesfivijhv` int DEFAULT NULL,\n `zeqtnswoewglaqqttvwltustqmmndpws` int DEFAULT NULL,\n `gmccilgzwwmyazxotszamfbtilwyawmb` int DEFAULT NULL,\n `iyosxytuemravyvbugnfjkhpuyqfimxe` int DEFAULT NULL,\n `uagzurpydbjbxbzbquqyjisxqivmnypr` int DEFAULT NULL,\n `sooodkifpudvxhnsidoxskjekxmahuzn` int DEFAULT NULL,\n `ipjjejhxiwdkeuhmjkfjankpedeshybo` int DEFAULT NULL,\n `bxmsvfjhdgddbvtkomageyhauhjpvvnt` int DEFAULT NULL,\n `jnjymrlgzsnkyszjxjhhjjfhflooqqls` int DEFAULT NULL,\n `jlmlwsiaitlxzyuateyfahfblbfixhnh` int DEFAULT NULL,\n `eufngpwumpauijoaednmopquxltwrzvr` int DEFAULT NULL,\n `sgsvouisbyrkwhcnljedcituikvuaftr` int DEFAULT NULL,\n `vhxwgipinvfzswltnzoxpnwkmeksmnoy` int DEFAULT NULL,\n `bsaqnszdsczrdjolyvbsvnldansqbhos` int DEFAULT NULL,\n `lwoevtgmyhgpatiosfzuvkbjjxmgtbbe` int DEFAULT NULL,\n `iwxmvcljuotzsivcquixoafjiilatfuf` int DEFAULT NULL,\n `zxnmgqhhmrhigjdawodonodiqyicdgxy` int DEFAULT NULL,\n `xsygedvsvfkpkmqxbboruvhwvmxwyqmx` int DEFAULT NULL,\n `qemuwaykztyuoiuzxkuyegaypgscqlra` int DEFAULT NULL,\n `dwpbbgmnetyiuorougweeraxojbxiklg` int DEFAULT NULL,\n `ebenoqdcjpocuiqvhlagshsfhwnkgigp` int DEFAULT NULL,\n `hxqtmydpwvrnvukwcalzobimlranngvc` int DEFAULT NULL,\n `lkrowiuxvnxslgfbnufxkxjaaequbkau` int DEFAULT NULL,\n `demkpbvcuorxjxwsmkokxcyyuzxfnhxy` int DEFAULT NULL,\n `lduthpalozdsezjzxzofkiimaouxychp` int DEFAULT NULL,\n `iypvkunnybmkzdsawljvploiwsmtporq` int DEFAULT NULL,\n `moizvzxxyhhnkfurwgrgvnfyqopedwjq` int DEFAULT NULL,\n `hnppwoopmnohottbfakukzaigocwcsyk` int DEFAULT NULL,\n `hdrlnywltjnfikprzhljnsdxffocxuhu` int DEFAULT NULL,\n `wllnzxihygqwaxvdaqcbwwoivwdhqadb` int DEFAULT NULL,\n `zzjpngaciqmhtzivrjzyilmafqqqjzxg` int DEFAULT NULL,\n `mwtnonivkyjeqkxbrwlxdbjeravzoyum` int DEFAULT NULL,\n `ntkikdfdqstqpjdrtewifnkwjsozdjja` int DEFAULT NULL,\n `eirxbuldsdcyafbzjektjeijduzfuqwg` int DEFAULT NULL,\n `rjnvnytntywusnqzdoyuytkowmterszt` int DEFAULT NULL,\n `vtocfcabnldqnciowgxsymfdqszpdjqd` int DEFAULT NULL,\n `uufitdfbwwxbyqsjkanhefzltdrybwnh` int DEFAULT NULL,\n `fyiyvnhepxfchknkpqtmkybdmrixrbpw` int DEFAULT NULL,\n PRIMARY KEY (`oovfrslhvjczrfstpedrbajuvdsqgkdi`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"seckcqhiaewieeiomssfzbhcvhkqmjnt\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["oovfrslhvjczrfstpedrbajuvdsqgkdi"],"columns":[{"name":"oovfrslhvjczrfstpedrbajuvdsqgkdi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"reshbefeivgyzcvipyffiralbgwcniwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebszyvmoitfudfvbxlxawqnczmmahjuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zunfdaaebgofdhlcouwgpaovwldpyjmy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jznifskpbwodtorleieoekyfswqeorzq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdbxoncqrkokwzvrmvypzuckxjujjdxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rftwymplokumlpgeueazjzoemnamfgrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjiutzupgticqmvppagiylocggfdsxme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dldmkqnuegxrztoczievaojahhrumnfj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fiqwaioorcfhawnzqqreloluupfemkuf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"egndsdwksuifsiigkanouzndhjtrhhma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"czhpjibwtzmudelxcxtqoxbznbeewjsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwziqetwmgvkdvklphpazxdkkefqemii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"firaliasykqwxuldjmpterrbqiqcpbzw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kkqvwheofxlntkuaawbgktgsgwnibcdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mseypvsmnyxschylbfmkkacpejzprlrv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fyhdnkzggfkmoacuweancutzagefyjro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjeuwcafakbzmhxmvlznylazxoorjakk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgfanpbxivodnaythctrcnegweuazame","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywzrlugxgjjmrulbuevcdsrjofadfiex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edslmvlezfdxtrqkeyukjsjamtwxkppg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lamoxrjjrtgyjomvnbaapxuekjlszwzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwetchxahznkhafcbjwfatcngzbzwbie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"apeirbtzvejpegmonzbpaguhzmjhqyos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prmvtubenqnyfreppqefrwhxwtcclexk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jraeqcwjthmexugrzqrfqwpfawokjzgi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ysoqpbotjoohyyyfmybjgecuiqyhccrk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqohgcpokmyvjwaamtlwmqeyxeupasnn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhtadsupzwybyxnnnchcgeecsdltljtf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jyymrvujabpntzvuvfljwbeyqwvlpytr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqodfnpvyivmcuphshrheaqrdmkonfnx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kiisxswtdamaorxysexbspaxgygtnxsi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwtuiquuczyfuanzvoepwgbefutxblyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"adbxfdscevgfynnxykfaakzlyxycnsld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjjazejdlejskagggmpmtykvdmanzhiy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzctabzglbwwkfctrkjxnbspyiynmsev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxqsxglceuojxiepucxquwgecweaxzcl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lrbgyibcwkcrkfrpyijhveexuupsqvey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okqgxftvvvuvbwzmfwvsofurwghndcdt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlfbuhxzauildcvknpjbogpgogskhuza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"voydsijnclkkqoxyngcuatyrhobunrbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwepcekkvcazclaoyttpogetaxxhtqjr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"adeimqwucaijhfsstsjawrtcwyunrcaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ncwtguovmkgykhgpvejfkdjdfiprfzrl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iowrgyfzilefanqndrghtcbxeutxvsfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kceetimgfujmvnjyftntdxuzptlgvpwt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"apqbcgmaovdikcrfkbjjvlvguysunqrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dscrxuapcfmayrkxqpphihcurwrhxfyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnvywfjakkqwzszbitogydwrplmphgzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lsiyawaoemgkrodnsfymorvnjzqixtkx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zflejbarwckjjjloexbwgptkphokyoim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwjasricywoyynciltblrmixouupbebr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lyksjpzgsccrrxnjwwutyhzsrzzdydaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkelrozpzmxnicmaczztofkoyoyxcdxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnlwvbbpxhxbsrvihnifzaqhiamozjbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vyakuwnpkmpurfodutpbmvvbwinjrqym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbrqjmmgcpeyxkkyqkqgpshaepqxvgxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmbykqallnllhjykmwcfvfraosfbtfet","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nizrygnndzzrbptdzuslzrlcuxugfyfj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ooikfirshlpztfmdojdmhxocuiblwxnc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fgqqanfalttvjdqksdpdshcbmyozidhw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jktaffqqcfscnmjxopqtzrqxhdaobotr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivtqpqgkidqrenmpjhyfflxesfivijhv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zeqtnswoewglaqqttvwltustqmmndpws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmccilgzwwmyazxotszamfbtilwyawmb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iyosxytuemravyvbugnfjkhpuyqfimxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uagzurpydbjbxbzbquqyjisxqivmnypr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sooodkifpudvxhnsidoxskjekxmahuzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ipjjejhxiwdkeuhmjkfjankpedeshybo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bxmsvfjhdgddbvtkomageyhauhjpvvnt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jnjymrlgzsnkyszjxjhhjjfhflooqqls","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlmlwsiaitlxzyuateyfahfblbfixhnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eufngpwumpauijoaednmopquxltwrzvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sgsvouisbyrkwhcnljedcituikvuaftr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhxwgipinvfzswltnzoxpnwkmeksmnoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bsaqnszdsczrdjolyvbsvnldansqbhos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwoevtgmyhgpatiosfzuvkbjjxmgtbbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwxmvcljuotzsivcquixoafjiilatfuf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxnmgqhhmrhigjdawodonodiqyicdgxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsygedvsvfkpkmqxbboruvhwvmxwyqmx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qemuwaykztyuoiuzxkuyegaypgscqlra","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwpbbgmnetyiuorougweeraxojbxiklg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebenoqdcjpocuiqvhlagshsfhwnkgigp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxqtmydpwvrnvukwcalzobimlranngvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lkrowiuxvnxslgfbnufxkxjaaequbkau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"demkpbvcuorxjxwsmkokxcyyuzxfnhxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lduthpalozdsezjzxzofkiimaouxychp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iypvkunnybmkzdsawljvploiwsmtporq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"moizvzxxyhhnkfurwgrgvnfyqopedwjq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hnppwoopmnohottbfakukzaigocwcsyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdrlnywltjnfikprzhljnsdxffocxuhu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wllnzxihygqwaxvdaqcbwwoivwdhqadb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzjpngaciqmhtzivrjzyilmafqqqjzxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwtnonivkyjeqkxbrwlxdbjeravzoyum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntkikdfdqstqpjdrtewifnkwjsozdjja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eirxbuldsdcyafbzjektjeijduzfuqwg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjnvnytntywusnqzdoyuytkowmterszt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtocfcabnldqnciowgxsymfdqszpdjqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uufitdfbwwxbyqsjkanhefzltdrybwnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fyiyvnhepxfchknkpqtmkybdmrixrbpw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671055,"databaseName":"models_schema","ddl":"CREATE TABLE `siftmoygptajwmczbovtnwqyjdpcmqfw` (\n `uivlpthlrbnmxdbdqavwzrskepgcbegn` int NOT NULL,\n `pcnxdvwzcfhndagfkfcfrrfukhfepiwl` int DEFAULT NULL,\n `bebhbdmqdnnurtvndltiffxrfgsseugv` int DEFAULT NULL,\n `xzsmpglyqcmknuynawizgfmfjpppmcop` int DEFAULT NULL,\n `awxzgdjtozyfyrzdenrjhytehsnaaprf` int DEFAULT NULL,\n `lphrifnpgqniraprpkodbymhvkwtdzus` int DEFAULT NULL,\n `klhbtxuklakedhklnelkrioayhslrkqi` int DEFAULT NULL,\n `qqtuiqkzvjqhwbmkjsevzzoarmjncvge` int DEFAULT NULL,\n `vsuqlohmuduxiomsrscwagjwnxnojoae` int DEFAULT NULL,\n `yaishqjnletvarkzobmcalvkietcrtqw` int DEFAULT NULL,\n `vlfxfzyxbmixictzwqqnknxcbdfathcl` int DEFAULT NULL,\n `fnqcdufrumetgrebuuexgmckcedwbonb` int DEFAULT NULL,\n `jxjogsehugwrmjdjwshcolsadyzegext` int DEFAULT NULL,\n `lwcmiywmhcgthlavvqcawjxkhtoprtek` int DEFAULT NULL,\n `nsnnjfzivfpwzvbvznzpkbrljayqrwxa` int DEFAULT NULL,\n `vsabfhptvohxhhdsfeydekrbwkkvujfb` int DEFAULT NULL,\n `bkxqdticjpufxekgyxzthgvxmrqoewjv` int DEFAULT NULL,\n `utuikxkmsomatzcytqurhaciwdnqdguk` int DEFAULT NULL,\n `acbooodmjaxgargzptfahjsjqmwdnkjv` int DEFAULT NULL,\n `dddawkrdqyqiceytxkpirjilffozyptc` int DEFAULT NULL,\n `jctrsahixmrvjmqshdpkwpdphypjfuwf` int DEFAULT NULL,\n `bowtrhqyoqknlnkpdskqwqsqjeescghy` int DEFAULT NULL,\n `xniblzipvwotuhnbuhocmsrodcgujdfp` int DEFAULT NULL,\n `bjwrwnmlztnfmonhbnwssfytcutwxnfu` int DEFAULT NULL,\n `hzjwgilnouclppluytcrxubksjhtotga` int DEFAULT NULL,\n `akszsudsiuucalbjusmsnodxnqfhnfxe` int DEFAULT NULL,\n `zwenwdggfzfkndtjvwlqwdiipwosbxct` int DEFAULT NULL,\n `brycbjplxfgqyukcafohxqtoqvbogxvc` int DEFAULT NULL,\n `ovpfsnxuwaedsixntjeayazprdjtbkow` int DEFAULT NULL,\n `yjsgklcezzetunxktrpmnnzldpqobhoa` int DEFAULT NULL,\n `lkocdperbqfndzyuizkzfsdjhxmjsmym` int DEFAULT NULL,\n `sdzctiwmjvvaaflcffmdlhqagfevsxut` int DEFAULT NULL,\n `luziatvhjkqthzoqnmgdyxoxenzosyri` int DEFAULT NULL,\n `ixotwjepztatkeotweciftwzvxybagjk` int DEFAULT NULL,\n `srbxobyrizmjmvkyautefmelgnnkmnnl` int DEFAULT NULL,\n `smlkuzecchztgocppjifsaovhwzkjxrq` int DEFAULT NULL,\n `llmudhexjptisghahamrvdkqxnpadjpl` int DEFAULT NULL,\n `pazctxgnmbmdzjkaamfnkidehvbpwgri` int DEFAULT NULL,\n `turoxqiihpexpbxmzumckomynntgxzxu` int DEFAULT NULL,\n `xvmzigyxlugiimguydjdrwtbugrqlfxm` int DEFAULT NULL,\n `khtmjeuynmxagttktjfojjylkvxcmybj` int DEFAULT NULL,\n `spokzloufobjnpsarbburpvguphuymfi` int DEFAULT NULL,\n `loodgtigqjksabaqmacnlkrhxiwqrlvj` int DEFAULT NULL,\n `wlnaeijmrijhbwcbfqbpasfyfhwvjhse` int DEFAULT NULL,\n `etlnbnlxwspwlecsprufrmsajbekwojk` int DEFAULT NULL,\n `mooxyqnrxqmkiaoekolcncwhteboypiw` int DEFAULT NULL,\n `duinbveduipwzbeemlbmdcumqjexmcth` int DEFAULT NULL,\n `qmqnbzhvbyqmauppubxfrscsxssafdwx` int DEFAULT NULL,\n `rgwuxurhswqbjimuttzzwerpxvggonyu` int DEFAULT NULL,\n `sjuhtoskjalwipmmhllibneltdciaqzc` int DEFAULT NULL,\n `objzxllfsoaymmdfbfiwjyiinltnbjjb` int DEFAULT NULL,\n `ooxodausrcfqsxqbtjmjruwspiacuqog` int DEFAULT NULL,\n `mnvquqtfntoyhpmsizaqkitdaaaneqkw` int DEFAULT NULL,\n `vecpmxqsjfabclayihxrrmpbpsceyitf` int DEFAULT NULL,\n `twlfkdszfrufumayamygpdjymezborzj` int DEFAULT NULL,\n `oaiutjhbwwnqtzqsibnltumisvrktwqi` int DEFAULT NULL,\n `xvqolpnxwinlskrehxdbbvvaqmiuuhlv` int DEFAULT NULL,\n `veeaqmqpcuzqsrvjcjqrielglomwqwcp` int DEFAULT NULL,\n `yhguchpvfkfcdctoadmzizzpzqanbjdv` int DEFAULT NULL,\n `wmzbwueibeogkracamqfihepkytisray` int DEFAULT NULL,\n `jqejgootjilphhwwbsbakewmngqxiuox` int DEFAULT NULL,\n `zjfyfneskufhryezteyoffcksocqnptp` int DEFAULT NULL,\n `tltnfrqnfbdiolgbwpntoosbykwodxfk` int DEFAULT NULL,\n `tbjtmyvtmgjbndfpvokmusfwrmiuyeuc` int DEFAULT NULL,\n `blwzxsdbzouqwlmtjqdnpadtbowpbyqf` int DEFAULT NULL,\n `vcmkwugwysqyoghzyglfqkwzzwgtfnwk` int DEFAULT NULL,\n `zyftlcffmxvkuommutleskuwnyaeocvr` int DEFAULT NULL,\n `aoqvsxbrqmdfijvullghknkkolcaayij` int DEFAULT NULL,\n `yywdoznosfigvjfukfzicfknvoebhubk` int DEFAULT NULL,\n `qhoqggdmkctemmdewgazgreclphsardp` int DEFAULT NULL,\n `xxewbkylnzjmfpznlzzkuvwqzzguungs` int DEFAULT NULL,\n `kgyupjyysbxmyezhamewsnuimskfvvld` int DEFAULT NULL,\n `zguzqrugtrimqhtceopmelgcnyikelzr` int DEFAULT NULL,\n `bhcfundvrmmzridjxvvfwushzkvszlrd` int DEFAULT NULL,\n `hxfdxiobbjvitypmdvctkywkqscjtszx` int DEFAULT NULL,\n `vliveyomikfmbrklliynjnyyqqnpryxz` int DEFAULT NULL,\n `yyptopeqqyvddazekejmlhsyuytpiqil` int DEFAULT NULL,\n `aqexcwvitaaapeqbmetylwhrfijuphpp` int DEFAULT NULL,\n `lqvawyktijsnfarccqvapirjuoybsivj` int DEFAULT NULL,\n `pqaagpevannvpadvdjfftaeptbcybckw` int DEFAULT NULL,\n `bjudmaaevjgkihioktdkdefskhrvbwjj` int DEFAULT NULL,\n `nqhefisbbpefzgqfqvmbtprbnrntidft` int DEFAULT NULL,\n `exmoschfydijitautblekeaqubxxzzej` int DEFAULT NULL,\n `ibunvpqsedvwwdbfdnvlbkqfkvrwoczf` int DEFAULT NULL,\n `ebtpfucqajgamzcubxnamkpdtfpfqxgu` int DEFAULT NULL,\n `emdtxqfjiunqphkjchqtwqzxcfpmdccr` int DEFAULT NULL,\n `zedqgllusxnpansexiqxxzxltxtcfvcp` int DEFAULT NULL,\n `atpxfzqjxfxnlyvasaxktcdxjuwqnlep` int DEFAULT NULL,\n `wsqjvpwunmczplvkpqwndkkerydsaebs` int DEFAULT NULL,\n `efxnfkykpdewgcefdmytualqmpmjgjnh` int DEFAULT NULL,\n `xtszmehlcgxltfvjeqhfjuwyjvqgwhgb` int DEFAULT NULL,\n `mqjacllbvdlnxtzehxejekxeythzllqp` int DEFAULT NULL,\n `bcyzotnsyjxvsshxstanaabhhfejpzie` int DEFAULT NULL,\n `tzdnubstzjmrelewpnnojgjzintqvlgv` int DEFAULT NULL,\n `tptytxekbfhohmuyhjxqbhovjtrlbpuf` int DEFAULT NULL,\n `pacsnapnswcabvscmjgzvdxizbcigzct` int DEFAULT NULL,\n `ifbmnkepabdtatmelymecnmfcwxjbghj` int DEFAULT NULL,\n `cmsantggqazlalgpgeqdrnwqqsqxnytc` int DEFAULT NULL,\n `mpsolxgieibbniykdvpwctnpzvpyvimq` int DEFAULT NULL,\n `dgapahalubckmosgygwxnwmpwyreqcbz` int DEFAULT NULL,\n PRIMARY KEY (`uivlpthlrbnmxdbdqavwzrskepgcbegn`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"siftmoygptajwmczbovtnwqyjdpcmqfw\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["uivlpthlrbnmxdbdqavwzrskepgcbegn"],"columns":[{"name":"uivlpthlrbnmxdbdqavwzrskepgcbegn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"pcnxdvwzcfhndagfkfcfrrfukhfepiwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bebhbdmqdnnurtvndltiffxrfgsseugv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xzsmpglyqcmknuynawizgfmfjpppmcop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awxzgdjtozyfyrzdenrjhytehsnaaprf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lphrifnpgqniraprpkodbymhvkwtdzus","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"klhbtxuklakedhklnelkrioayhslrkqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqtuiqkzvjqhwbmkjsevzzoarmjncvge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsuqlohmuduxiomsrscwagjwnxnojoae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yaishqjnletvarkzobmcalvkietcrtqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vlfxfzyxbmixictzwqqnknxcbdfathcl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnqcdufrumetgrebuuexgmckcedwbonb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jxjogsehugwrmjdjwshcolsadyzegext","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwcmiywmhcgthlavvqcawjxkhtoprtek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nsnnjfzivfpwzvbvznzpkbrljayqrwxa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsabfhptvohxhhdsfeydekrbwkkvujfb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkxqdticjpufxekgyxzthgvxmrqoewjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utuikxkmsomatzcytqurhaciwdnqdguk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acbooodmjaxgargzptfahjsjqmwdnkjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dddawkrdqyqiceytxkpirjilffozyptc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jctrsahixmrvjmqshdpkwpdphypjfuwf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bowtrhqyoqknlnkpdskqwqsqjeescghy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xniblzipvwotuhnbuhocmsrodcgujdfp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjwrwnmlztnfmonhbnwssfytcutwxnfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hzjwgilnouclppluytcrxubksjhtotga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akszsudsiuucalbjusmsnodxnqfhnfxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zwenwdggfzfkndtjvwlqwdiipwosbxct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"brycbjplxfgqyukcafohxqtoqvbogxvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovpfsnxuwaedsixntjeayazprdjtbkow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjsgklcezzetunxktrpmnnzldpqobhoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lkocdperbqfndzyuizkzfsdjhxmjsmym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdzctiwmjvvaaflcffmdlhqagfevsxut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"luziatvhjkqthzoqnmgdyxoxenzosyri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixotwjepztatkeotweciftwzvxybagjk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"srbxobyrizmjmvkyautefmelgnnkmnnl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smlkuzecchztgocppjifsaovhwzkjxrq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llmudhexjptisghahamrvdkqxnpadjpl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pazctxgnmbmdzjkaamfnkidehvbpwgri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"turoxqiihpexpbxmzumckomynntgxzxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvmzigyxlugiimguydjdrwtbugrqlfxm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khtmjeuynmxagttktjfojjylkvxcmybj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"spokzloufobjnpsarbburpvguphuymfi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"loodgtigqjksabaqmacnlkrhxiwqrlvj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wlnaeijmrijhbwcbfqbpasfyfhwvjhse","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"etlnbnlxwspwlecsprufrmsajbekwojk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mooxyqnrxqmkiaoekolcncwhteboypiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"duinbveduipwzbeemlbmdcumqjexmcth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmqnbzhvbyqmauppubxfrscsxssafdwx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgwuxurhswqbjimuttzzwerpxvggonyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjuhtoskjalwipmmhllibneltdciaqzc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"objzxllfsoaymmdfbfiwjyiinltnbjjb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ooxodausrcfqsxqbtjmjruwspiacuqog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnvquqtfntoyhpmsizaqkitdaaaneqkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vecpmxqsjfabclayihxrrmpbpsceyitf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twlfkdszfrufumayamygpdjymezborzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oaiutjhbwwnqtzqsibnltumisvrktwqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvqolpnxwinlskrehxdbbvvaqmiuuhlv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"veeaqmqpcuzqsrvjcjqrielglomwqwcp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhguchpvfkfcdctoadmzizzpzqanbjdv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmzbwueibeogkracamqfihepkytisray","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jqejgootjilphhwwbsbakewmngqxiuox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjfyfneskufhryezteyoffcksocqnptp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tltnfrqnfbdiolgbwpntoosbykwodxfk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbjtmyvtmgjbndfpvokmusfwrmiuyeuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blwzxsdbzouqwlmtjqdnpadtbowpbyqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vcmkwugwysqyoghzyglfqkwzzwgtfnwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyftlcffmxvkuommutleskuwnyaeocvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aoqvsxbrqmdfijvullghknkkolcaayij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yywdoznosfigvjfukfzicfknvoebhubk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhoqggdmkctemmdewgazgreclphsardp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxewbkylnzjmfpznlzzkuvwqzzguungs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kgyupjyysbxmyezhamewsnuimskfvvld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zguzqrugtrimqhtceopmelgcnyikelzr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bhcfundvrmmzridjxvvfwushzkvszlrd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxfdxiobbjvitypmdvctkywkqscjtszx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vliveyomikfmbrklliynjnyyqqnpryxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yyptopeqqyvddazekejmlhsyuytpiqil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqexcwvitaaapeqbmetylwhrfijuphpp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqvawyktijsnfarccqvapirjuoybsivj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqaagpevannvpadvdjfftaeptbcybckw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjudmaaevjgkihioktdkdefskhrvbwjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqhefisbbpefzgqfqvmbtprbnrntidft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exmoschfydijitautblekeaqubxxzzej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibunvpqsedvwwdbfdnvlbkqfkvrwoczf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebtpfucqajgamzcubxnamkpdtfpfqxgu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"emdtxqfjiunqphkjchqtwqzxcfpmdccr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zedqgllusxnpansexiqxxzxltxtcfvcp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atpxfzqjxfxnlyvasaxktcdxjuwqnlep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wsqjvpwunmczplvkpqwndkkerydsaebs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"efxnfkykpdewgcefdmytualqmpmjgjnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xtszmehlcgxltfvjeqhfjuwyjvqgwhgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqjacllbvdlnxtzehxejekxeythzllqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bcyzotnsyjxvsshxstanaabhhfejpzie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tzdnubstzjmrelewpnnojgjzintqvlgv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tptytxekbfhohmuyhjxqbhovjtrlbpuf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pacsnapnswcabvscmjgzvdxizbcigzct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifbmnkepabdtatmelymecnmfcwxjbghj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmsantggqazlalgpgeqdrnwqqsqxnytc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpsolxgieibbniykdvpwctnpzvpyvimq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgapahalubckmosgygwxnwmpwyreqcbz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671091,"databaseName":"models_schema","ddl":"CREATE TABLE `sktiezunkkczqbwwviueupxhgmwmasju` (\n `fylbltbxggysfwfmvfljudjzbvdhcmfr` int NOT NULL,\n `lcgffmaubbohukvqcibusapvkfrrupgp` int DEFAULT NULL,\n `phjwofndwdckutpuuamvgkfqesruzulb` int DEFAULT NULL,\n `bnocdpxatrgubdksuxkmaopsdxoicwtc` int DEFAULT NULL,\n `pmgezjtcawmyyvxobmwjjlzghyeayygr` int DEFAULT NULL,\n `mikrnmobpacqvpaxiepcdvehjxwzzbmb` int DEFAULT NULL,\n `ysotadlyefrizrpcgbzddprssccqnpak` int DEFAULT NULL,\n `exgwhwpwqocqxqmjvhnahunwdqbndcno` int DEFAULT NULL,\n `vuebundptwdgrboeuqqpcpdodewpfjru` int DEFAULT NULL,\n `qcztbusdmjjdzhgyyixwxniynwnedgxr` int DEFAULT NULL,\n `kekgthauvdooreydehvddfltaxuyftii` int DEFAULT NULL,\n `ucxgeufqnvalopgzexreqbufgueakcjz` int DEFAULT NULL,\n `fhdtoboxoedtbjsajjfwubiypazsbarv` int DEFAULT NULL,\n `jfklffccobgmrdbknexzlyvyjhlczixf` int DEFAULT NULL,\n `gqbjbxurbxjjurtddbokbyqkfbebgeko` int DEFAULT NULL,\n `neuvzweulgxfvtwvxbzeutjuxarnyxhl` int DEFAULT NULL,\n `rketwuvrsvowgsfoigaoyjxgdkkxjgwm` int DEFAULT NULL,\n `udymuerdvtiibdjcemwtdobggsdvdpig` int DEFAULT NULL,\n `nkvbnriigmovzxbrkjugbgpaylqbflmd` int DEFAULT NULL,\n `nlyeryddldhrizypnkehvesnkmbrekeb` int DEFAULT NULL,\n `qbfofyvursjhibsfxauzcdnxymnzteoc` int DEFAULT NULL,\n `zvsokpcjbprrvsjzhdcjjeuimcwyjqnr` int DEFAULT NULL,\n `hcdyzismdfhdglwiblbalilkfbzjrjpw` int DEFAULT NULL,\n `rfijcleoihyhklafnpmqljvgaeywrhzs` int DEFAULT NULL,\n `dghuxaadxjuktfjrfslmmvuqqgbdfglo` int DEFAULT NULL,\n `pusbrlbdxxxcqeprvidhjzfdbkftwwsd` int DEFAULT NULL,\n `joqzvaaxiisngozidgbcppdhpbjhiuvu` int DEFAULT NULL,\n `ulrjhwyvcbqzlcddziizwuiawumtslar` int DEFAULT NULL,\n `xdoxmiiaqnsnyelwmquhprmclwhtsxot` int DEFAULT NULL,\n `mcwdftsrvzbhmorhyuouvkcivpjqawuk` int DEFAULT NULL,\n `ocdtuiehrwhjnlmshurnbevhuhlxvftj` int DEFAULT NULL,\n `uvyfgcbkkoglpzyvfqtvwbxscotlizgs` int DEFAULT NULL,\n `mwnamdpucakvpyujhdqlvwjcuqvtghuh` int DEFAULT NULL,\n `ugvviqaklrowqmpmskmpracwclxjthsg` int DEFAULT NULL,\n `slgopobdqmrfrazgufwawjksbepaldqw` int DEFAULT NULL,\n `ixhfhnqpjrehosmhskzhgymjzlqkdpda` int DEFAULT NULL,\n `owcpbtyxsishibpmjqmtcxqncaiygatt` int DEFAULT NULL,\n `vuibumfdchglplsmbaalailgqpcyocdg` int DEFAULT NULL,\n `fqmbeqkmtqtaokhkchxrafejoapmelzz` int DEFAULT NULL,\n `nfputvkhmdhjfbkdcwetvwbmyxekvwvg` int DEFAULT NULL,\n `lwjelmbjhxpdueujotxspjcgbemouifh` int DEFAULT NULL,\n `oidwxhrielyvqemslmagaiuxqtcltjag` int DEFAULT NULL,\n `teltbjcyldtdainkfstqbkhbltrlkxvj` int DEFAULT NULL,\n `wsmiqfcxkpcgehoybkwpkbmyrkmpzitu` int DEFAULT NULL,\n `pxjdpbowwpctsjnenyownflivxhuepvu` int DEFAULT NULL,\n `qclkxwuzohtapustcngwupslzxtzrhfy` int DEFAULT NULL,\n `wpwkmpvvcluhqgjrpxrejbtqutklwich` int DEFAULT NULL,\n `evjokkxedahvlzctpvrznxckpblhhtjs` int DEFAULT NULL,\n `ksiiqvajfxvwrprmdtnnxfgrslrgddgw` int DEFAULT NULL,\n `pjtmkfiiliroihmfxaxnwjntvmuoyvwc` int DEFAULT NULL,\n `tqnaeoubyqzynzvlrmputhwhdzhmvtux` int DEFAULT NULL,\n `gjxjrvyrwnnluggyzbvjbshoiyfznzic` int DEFAULT NULL,\n `bzfijkqqceluemqovdhzxoqognuxkhen` int DEFAULT NULL,\n `iwsytlhvezmnsybfprkjgrpirppniyvi` int DEFAULT NULL,\n `xddkyefuocbkpodgxydntwobqbmeiyxx` int DEFAULT NULL,\n `hsirzmuvvnzhotimdlopsggnmlaweptz` int DEFAULT NULL,\n `frpdwacuexzoadiatwwiwqbkqzhdqlrl` int DEFAULT NULL,\n `wlmcgyhhaabbncodpqdyekmdirqxkavm` int DEFAULT NULL,\n `zftufhxhwksyjavribplhcsoqijudhfz` int DEFAULT NULL,\n `mivxmzujyjrocqyiqmftrvfcgkxgqqme` int DEFAULT NULL,\n `faoxqjqjmmwuxprxujletkqdsjifvtwh` int DEFAULT NULL,\n `wxyblfiqucvtkerwekzqgpvnvlcssxuu` int DEFAULT NULL,\n `tmlrhwmbvbjnjfswrrjnevnixtaaargl` int DEFAULT NULL,\n `nmhuqdeyarggifoueexlcfssqfenyaqb` int DEFAULT NULL,\n `yebmohieorlunlggpfbtrknkyaerpdsw` int DEFAULT NULL,\n `ywaramvadjannnsbjrirqzsdhakfavhs` int DEFAULT NULL,\n `paaipijgfvnhatmaxzwfbzchevlpzjfv` int DEFAULT NULL,\n `pszibimzknvkjaysqwxslhlkiynuuaas` int DEFAULT NULL,\n `xngzjvwrglqxtadmnqvqnpjybiipxruo` int DEFAULT NULL,\n `ytinpxwdtbsqlwttdfgjnryahhjfgcjq` int DEFAULT NULL,\n `kfvdfjhwmfqdctahajpurfqwzdpkbrfu` int DEFAULT NULL,\n `thnismrskwccudhkuojeqrcqhhfbrpob` int DEFAULT NULL,\n `cabnavhifvfemomfmlogqwzxgepyssgr` int DEFAULT NULL,\n `cixjojeilpwfrrxeeamnjnwgblcjfplu` int DEFAULT NULL,\n `ilwaslmrouczdszxgvyitpfxrsgyrths` int DEFAULT NULL,\n `tlfjludumzolkhnbrgxdlbyhpbdewtsn` int DEFAULT NULL,\n `vqrtawdqzhzncaryxjdtggfkleqpyygk` int DEFAULT NULL,\n `jcxyashudquzxwrmpaujpdtsmwxhwsuw` int DEFAULT NULL,\n `amwwyalelxuhmdkiohdnlraimcbftnjs` int DEFAULT NULL,\n `mfuujhltyjnfvwfqyeznlkycesuuuxnd` int DEFAULT NULL,\n `rdqcqkskqocgizqjnomwkubeogbfhqws` int DEFAULT NULL,\n `ptzngjyjvwiauowygdumlxhofvwzfqyr` int DEFAULT NULL,\n `evqjvfskoikraosxwjqpoydqnehzurxv` int DEFAULT NULL,\n `ymgpgckqldbznqoolfratyoewfdxgpsj` int DEFAULT NULL,\n `ubikwixkhtpztgkeixsruymlzerdwagh` int DEFAULT NULL,\n `qhfjnzbldpnzelwjmyumjlbypvhospgk` int DEFAULT NULL,\n `lkrbwjjdxctedhtecdkhslrhqpoxsgtw` int DEFAULT NULL,\n `idaqlvxfymfqpgekfsmitlprqmwswnkp` int DEFAULT NULL,\n `yknyhujwkapgqejgtmyjozziqrpbjrgw` int DEFAULT NULL,\n `ynmxlqrspkdxkyiemzlvdaooykcwvctm` int DEFAULT NULL,\n `yihhuniwabsinabbwdwhorxuknseozmm` int DEFAULT NULL,\n `qezrlifuubluazvxjrnwnfcrmclirbeo` int DEFAULT NULL,\n `xszjxrvaqljlkjuntqftiijvpmglihpj` int DEFAULT NULL,\n `iqenbbigkfujgfdxsoqikfajwdfwyrkv` int DEFAULT NULL,\n `ajtcorghubjfmjrqpqpomegjzmebwtyk` int DEFAULT NULL,\n `abhdfvevzlcahlaorpdqyskdcfuprcfp` int DEFAULT NULL,\n `twogxyemvbcbhidywkdbgwjfwrajslwi` int DEFAULT NULL,\n `vobdvdckbqxehrfzakvlypmolssciobk` int DEFAULT NULL,\n `gtgnwtdnbnjgqnopigvexacztxbzkyft` int DEFAULT NULL,\n `qpqhaziyuvhlwbarwpllmgtrcmuigrvg` int DEFAULT NULL,\n PRIMARY KEY (`fylbltbxggysfwfmvfljudjzbvdhcmfr`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"sktiezunkkczqbwwviueupxhgmwmasju\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["fylbltbxggysfwfmvfljudjzbvdhcmfr"],"columns":[{"name":"fylbltbxggysfwfmvfljudjzbvdhcmfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"lcgffmaubbohukvqcibusapvkfrrupgp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phjwofndwdckutpuuamvgkfqesruzulb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnocdpxatrgubdksuxkmaopsdxoicwtc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmgezjtcawmyyvxobmwjjlzghyeayygr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mikrnmobpacqvpaxiepcdvehjxwzzbmb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ysotadlyefrizrpcgbzddprssccqnpak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exgwhwpwqocqxqmjvhnahunwdqbndcno","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vuebundptwdgrboeuqqpcpdodewpfjru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qcztbusdmjjdzhgyyixwxniynwnedgxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kekgthauvdooreydehvddfltaxuyftii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucxgeufqnvalopgzexreqbufgueakcjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhdtoboxoedtbjsajjfwubiypazsbarv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jfklffccobgmrdbknexzlyvyjhlczixf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqbjbxurbxjjurtddbokbyqkfbebgeko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"neuvzweulgxfvtwvxbzeutjuxarnyxhl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rketwuvrsvowgsfoigaoyjxgdkkxjgwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udymuerdvtiibdjcemwtdobggsdvdpig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkvbnriigmovzxbrkjugbgpaylqbflmd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nlyeryddldhrizypnkehvesnkmbrekeb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qbfofyvursjhibsfxauzcdnxymnzteoc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvsokpcjbprrvsjzhdcjjeuimcwyjqnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hcdyzismdfhdglwiblbalilkfbzjrjpw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfijcleoihyhklafnpmqljvgaeywrhzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dghuxaadxjuktfjrfslmmvuqqgbdfglo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pusbrlbdxxxcqeprvidhjzfdbkftwwsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"joqzvaaxiisngozidgbcppdhpbjhiuvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ulrjhwyvcbqzlcddziizwuiawumtslar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdoxmiiaqnsnyelwmquhprmclwhtsxot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcwdftsrvzbhmorhyuouvkcivpjqawuk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocdtuiehrwhjnlmshurnbevhuhlxvftj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvyfgcbkkoglpzyvfqtvwbxscotlizgs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwnamdpucakvpyujhdqlvwjcuqvtghuh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugvviqaklrowqmpmskmpracwclxjthsg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"slgopobdqmrfrazgufwawjksbepaldqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixhfhnqpjrehosmhskzhgymjzlqkdpda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owcpbtyxsishibpmjqmtcxqncaiygatt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vuibumfdchglplsmbaalailgqpcyocdg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqmbeqkmtqtaokhkchxrafejoapmelzz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfputvkhmdhjfbkdcwetvwbmyxekvwvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwjelmbjhxpdueujotxspjcgbemouifh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oidwxhrielyvqemslmagaiuxqtcltjag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"teltbjcyldtdainkfstqbkhbltrlkxvj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wsmiqfcxkpcgehoybkwpkbmyrkmpzitu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxjdpbowwpctsjnenyownflivxhuepvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qclkxwuzohtapustcngwupslzxtzrhfy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wpwkmpvvcluhqgjrpxrejbtqutklwich","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evjokkxedahvlzctpvrznxckpblhhtjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ksiiqvajfxvwrprmdtnnxfgrslrgddgw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjtmkfiiliroihmfxaxnwjntvmuoyvwc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqnaeoubyqzynzvlrmputhwhdzhmvtux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjxjrvyrwnnluggyzbvjbshoiyfznzic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzfijkqqceluemqovdhzxoqognuxkhen","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwsytlhvezmnsybfprkjgrpirppniyvi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xddkyefuocbkpodgxydntwobqbmeiyxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hsirzmuvvnzhotimdlopsggnmlaweptz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"frpdwacuexzoadiatwwiwqbkqzhdqlrl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wlmcgyhhaabbncodpqdyekmdirqxkavm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zftufhxhwksyjavribplhcsoqijudhfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mivxmzujyjrocqyiqmftrvfcgkxgqqme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"faoxqjqjmmwuxprxujletkqdsjifvtwh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxyblfiqucvtkerwekzqgpvnvlcssxuu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmlrhwmbvbjnjfswrrjnevnixtaaargl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmhuqdeyarggifoueexlcfssqfenyaqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yebmohieorlunlggpfbtrknkyaerpdsw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywaramvadjannnsbjrirqzsdhakfavhs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"paaipijgfvnhatmaxzwfbzchevlpzjfv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pszibimzknvkjaysqwxslhlkiynuuaas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xngzjvwrglqxtadmnqvqnpjybiipxruo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytinpxwdtbsqlwttdfgjnryahhjfgcjq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfvdfjhwmfqdctahajpurfqwzdpkbrfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thnismrskwccudhkuojeqrcqhhfbrpob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cabnavhifvfemomfmlogqwzxgepyssgr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cixjojeilpwfrrxeeamnjnwgblcjfplu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ilwaslmrouczdszxgvyitpfxrsgyrths","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlfjludumzolkhnbrgxdlbyhpbdewtsn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqrtawdqzhzncaryxjdtggfkleqpyygk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jcxyashudquzxwrmpaujpdtsmwxhwsuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amwwyalelxuhmdkiohdnlraimcbftnjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfuujhltyjnfvwfqyeznlkycesuuuxnd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdqcqkskqocgizqjnomwkubeogbfhqws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ptzngjyjvwiauowygdumlxhofvwzfqyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evqjvfskoikraosxwjqpoydqnehzurxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ymgpgckqldbznqoolfratyoewfdxgpsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubikwixkhtpztgkeixsruymlzerdwagh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhfjnzbldpnzelwjmyumjlbypvhospgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lkrbwjjdxctedhtecdkhslrhqpoxsgtw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idaqlvxfymfqpgekfsmitlprqmwswnkp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yknyhujwkapgqejgtmyjozziqrpbjrgw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynmxlqrspkdxkyiemzlvdaooykcwvctm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yihhuniwabsinabbwdwhorxuknseozmm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qezrlifuubluazvxjrnwnfcrmclirbeo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xszjxrvaqljlkjuntqftiijvpmglihpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqenbbigkfujgfdxsoqikfajwdfwyrkv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajtcorghubjfmjrqpqpomegjzmebwtyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abhdfvevzlcahlaorpdqyskdcfuprcfp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twogxyemvbcbhidywkdbgwjfwrajslwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vobdvdckbqxehrfzakvlypmolssciobk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtgnwtdnbnjgqnopigvexacztxbzkyft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qpqhaziyuvhlwbarwpllmgtrcmuigrvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671121,"databaseName":"models_schema","ddl":"CREATE TABLE `somkexlrchwjsnfjblemmdlxdrlqxiwe` (\n `svkwlinegcmrdoiqetusyhushiojbsyl` int NOT NULL,\n `uhfhxqaealjtnifhlsdogqyeaiylgfbk` int DEFAULT NULL,\n `bhyujozpsmaredfmusxqmtogwsuiecnc` int DEFAULT NULL,\n `qjqsvleioczzbqwbikusnfxnlrqcolcc` int DEFAULT NULL,\n `pcwqufcwqqgkwikfvwbfhgojsahrrzah` int DEFAULT NULL,\n `bevbpkusgxsybxrfmwbcfixiwnvhmudg` int DEFAULT NULL,\n `rvrjdnlkiwzbcbvvkzhyyzizntekpxpk` int DEFAULT NULL,\n `iaxafxposijdgphmklqetuupllcjsjts` int DEFAULT NULL,\n `swxnzahiadmtnpzwnmvboatdpzvdmkgn` int DEFAULT NULL,\n `nlvltqjncsqzdhvclvbswkxqvhgeaogc` int DEFAULT NULL,\n `ebflhdygpanrgeydrfnrddwgsczfpktp` int DEFAULT NULL,\n `jlitkntdjtlfjwzqaqwjfeixalnzidos` int DEFAULT NULL,\n `kydlbzwrnujdrsninmimdeovoyjcdiwi` int DEFAULT NULL,\n `vdmojtbvmifzfbpgpluldcaxrpzxymkt` int DEFAULT NULL,\n `oeqglrmnrpuxtekwvhphvgwnfaawnekt` int DEFAULT NULL,\n `dwewlhmmlmslneuaekgibcymnjmwaknf` int DEFAULT NULL,\n `wusxygqspontqafyunchagiillpkpuui` int DEFAULT NULL,\n `hjolmcblqyioqtcquitlcvurpbrbhiiz` int DEFAULT NULL,\n `ksleqjkbvaljkcmqukaloajditfvhykz` int DEFAULT NULL,\n `kxisincxkbrilsgcjuavesohvhtskwxr` int DEFAULT NULL,\n `elewmajsavjhcjgxkqjsdapalztwngkv` int DEFAULT NULL,\n `vnxrgjirtxcnltckfhjtzqeurkhogdua` int DEFAULT NULL,\n `zhsmhjdrftwjvpgklxcjxhnoqrjxdjug` int DEFAULT NULL,\n `lewrzvlhekouloewsdsabqcpcoztdmwj` int DEFAULT NULL,\n `dnqosmoahaebqywhbivgxzwbtgtdenhr` int DEFAULT NULL,\n `cadfjnpwhskcxgalkcrmoroddnhqkgrc` int DEFAULT NULL,\n `hiqssojocnknefwjyqbhcrcvojuaeufy` int DEFAULT NULL,\n `rtbfofwtcplsrkwuhdvzzamnjawdzhqc` int DEFAULT NULL,\n `ezkktrfnpteirrziadswyzergokvuzbp` int DEFAULT NULL,\n `zckpcavfdsbcvsrttebdwaiemfhpobwf` int DEFAULT NULL,\n `zubdicefuwjukrrqxwsnuwrijfhkfjyc` int DEFAULT NULL,\n `iftifsisxugfsfxapdkbkoderxrshhkm` int DEFAULT NULL,\n `shzwisbhhzzhzkveeuyxjdnppqptepwb` int DEFAULT NULL,\n `aqumbxpznftscjhymbidtqtdypfedzrc` int DEFAULT NULL,\n `jbzkdhtkvfikgkzbphkiwpyayqhgwecu` int DEFAULT NULL,\n `gylgtugvjloyhvjppxbnnciatxdompnt` int DEFAULT NULL,\n `loinqnojvjwclwwhwetxajltjdfzdiky` int DEFAULT NULL,\n `xoedimdxxbsxlllchfwbqvwrfjjtzmmp` int DEFAULT NULL,\n `caghincwbqgnpmiopqdiiqxgbuvyxatm` int DEFAULT NULL,\n `zwhsgxvbqgcjnpekidvbkruzhbmzawqr` int DEFAULT NULL,\n `ohjjwrwmkwtrrdexyuqkvzhidoulndip` int DEFAULT NULL,\n `dlkahwzspowggzvrpphlpmtpijawvhph` int DEFAULT NULL,\n `zdaddugnovvdbywpwffvtyjynhiruznj` int DEFAULT NULL,\n `acdqcnqdknovrfjkkfdetucyzbswhxsl` int DEFAULT NULL,\n `txmkgrcfjvkeymfbdjychvjsbqbjsarc` int DEFAULT NULL,\n `kgbfjcbwhvyzvimgngepscezzhlehjdt` int DEFAULT NULL,\n `wzjklnastduaffaoccvjigtdrmfovmvj` int DEFAULT NULL,\n `boqjppedkaoliyngxundpacmfcfkrtpq` int DEFAULT NULL,\n `wlcpinqiafmttllhdorkqgyljdrpmljc` int DEFAULT NULL,\n `vltmjvmptxzafimatnqlvvpmkfpdhecb` int DEFAULT NULL,\n `dpxxtkoimezvhljtwimdkbvmeritwmql` int DEFAULT NULL,\n `qampsmxxfsmmstxmnggoylzoxhmcrlkj` int DEFAULT NULL,\n `sguuehwmlvngyaepllhznibpurqtanou` int DEFAULT NULL,\n `cuguuywbcnhvnhvuwzgflgklggwzeszp` int DEFAULT NULL,\n `vvnpjkfqzfjeitvgdmrwcwfwtajvtxnb` int DEFAULT NULL,\n `euuycsjfqvtyopymzllpwkxhamljqbat` int DEFAULT NULL,\n `mupgdmjrndsgvtkjtkzgzskdiknqxsku` int DEFAULT NULL,\n `itfjyyrwtkczbgaozxzasfnslalgrddw` int DEFAULT NULL,\n `ndyhlsmilehpcqhezfpheanjfxtlmowo` int DEFAULT NULL,\n `wvukwgxdeubtkfrgwsitvrjzarljyvby` int DEFAULT NULL,\n `edwdsbqupyflztzgyffagafqyyxlciun` int DEFAULT NULL,\n `vmgdeweqaihskwlhwphmvcxfajspfyka` int DEFAULT NULL,\n `dsmazjecvojbvopoazkbjmuiqtfpimtv` int DEFAULT NULL,\n `ymnipqomshxqmnexfpsotmdkvtwumjsa` int DEFAULT NULL,\n `agcnmvchdcennuebuhbghzzpzjbhvhen` int DEFAULT NULL,\n `eumaldmkzftstiywbzpvkcquxxgaabij` int DEFAULT NULL,\n `xyxuppotcafafqxtrhtvfdsjwytqyzvv` int DEFAULT NULL,\n `snfomdtslfidhrtuheuaoadaqgodyogc` int DEFAULT NULL,\n `ahyovczuxhmzqlcgafgwgupgxobpxyvi` int DEFAULT NULL,\n `roxvahcisyqpxrnmgcfaokbzmjvnxuyq` int DEFAULT NULL,\n `gmbscrezhtloofhaxuzyjpzxbntwfuqh` int DEFAULT NULL,\n `oufdmxchbmaxixakbddnvgjuijloeeli` int DEFAULT NULL,\n `nvjtqrakyvdhurwhasuxbkbvhgiukegj` int DEFAULT NULL,\n `qfhmyspndreszjvypskiyptxsoqgejws` int DEFAULT NULL,\n `qzofzqbtjvrftqdwvwbjxysbwwxzbqat` int DEFAULT NULL,\n `ekkmwwnvijbwzgjwglbrqnzcjimmvilv` int DEFAULT NULL,\n `qqyeoshsechlxnvdyquipyptatzdbbpz` int DEFAULT NULL,\n `nplhmgeeezazjapkjnuleiebkfintidj` int DEFAULT NULL,\n `gajbbknkrodasacdtpbevznlvnpnvnim` int DEFAULT NULL,\n `knovbeibgblaacmmulipkgdejhbuydal` int DEFAULT NULL,\n `rlfiqwkqwrqykptdqwkbmaspenxqnaeh` int DEFAULT NULL,\n `uryqzuaoamtkldzzsiwfkebvfagnokac` int DEFAULT NULL,\n `qydljvqqkcougtnywreyvautblctmbkm` int DEFAULT NULL,\n `ketburpphkhrbgqmrxemptmuuvgpwuvp` int DEFAULT NULL,\n `evsjuyieyqdjecvzeewystxfjnmchbye` int DEFAULT NULL,\n `ovnfrtxprzluwmhrcjozjzyjbtxmvddi` int DEFAULT NULL,\n `pvaylcsnalpmutmwbcibucattdxgnbul` int DEFAULT NULL,\n `zyemohaxfcevnfdefwfnnifestddsbgx` int DEFAULT NULL,\n `ukvbilphzwnogtdzgcpgstvsuugadpdw` int DEFAULT NULL,\n `oxwmxjdqmjxvvauwtiihfjsdqtzrhivi` int DEFAULT NULL,\n `yugezltskdbkuxoipziusiqmiemstsgc` int DEFAULT NULL,\n `glwgoivqalthpymfxqlymuxeesdikxan` int DEFAULT NULL,\n `awbvhzdxcxjlbcccgpvsgweazrzzqtst` int DEFAULT NULL,\n `xxsaehgxkeityvgfyxplgadncoawyzwj` int DEFAULT NULL,\n `gsgdtegifffdopaefjjpsycnusfgtphj` int DEFAULT NULL,\n `crgfsudctdfjrtsobbyrohsodlgeglbh` int DEFAULT NULL,\n `lvujzovwavrhpgppzwaehklakbjlhozn` int DEFAULT NULL,\n `ozmlphsmyusyehqaofdinnnyrstjtjld` int DEFAULT NULL,\n `sbkpbzdogoknpmccrtjbeejwxpvmwauf` int DEFAULT NULL,\n `ywycrshgdpprwlmxdufurftllxctruju` int DEFAULT NULL,\n PRIMARY KEY (`svkwlinegcmrdoiqetusyhushiojbsyl`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"somkexlrchwjsnfjblemmdlxdrlqxiwe\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["svkwlinegcmrdoiqetusyhushiojbsyl"],"columns":[{"name":"svkwlinegcmrdoiqetusyhushiojbsyl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"uhfhxqaealjtnifhlsdogqyeaiylgfbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bhyujozpsmaredfmusxqmtogwsuiecnc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjqsvleioczzbqwbikusnfxnlrqcolcc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pcwqufcwqqgkwikfvwbfhgojsahrrzah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bevbpkusgxsybxrfmwbcfixiwnvhmudg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rvrjdnlkiwzbcbvvkzhyyzizntekpxpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iaxafxposijdgphmklqetuupllcjsjts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swxnzahiadmtnpzwnmvboatdpzvdmkgn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nlvltqjncsqzdhvclvbswkxqvhgeaogc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebflhdygpanrgeydrfnrddwgsczfpktp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlitkntdjtlfjwzqaqwjfeixalnzidos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kydlbzwrnujdrsninmimdeovoyjcdiwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdmojtbvmifzfbpgpluldcaxrpzxymkt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oeqglrmnrpuxtekwvhphvgwnfaawnekt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwewlhmmlmslneuaekgibcymnjmwaknf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wusxygqspontqafyunchagiillpkpuui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjolmcblqyioqtcquitlcvurpbrbhiiz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ksleqjkbvaljkcmqukaloajditfvhykz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxisincxkbrilsgcjuavesohvhtskwxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elewmajsavjhcjgxkqjsdapalztwngkv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnxrgjirtxcnltckfhjtzqeurkhogdua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhsmhjdrftwjvpgklxcjxhnoqrjxdjug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lewrzvlhekouloewsdsabqcpcoztdmwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dnqosmoahaebqywhbivgxzwbtgtdenhr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cadfjnpwhskcxgalkcrmoroddnhqkgrc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hiqssojocnknefwjyqbhcrcvojuaeufy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtbfofwtcplsrkwuhdvzzamnjawdzhqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezkktrfnpteirrziadswyzergokvuzbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zckpcavfdsbcvsrttebdwaiemfhpobwf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zubdicefuwjukrrqxwsnuwrijfhkfjyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iftifsisxugfsfxapdkbkoderxrshhkm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shzwisbhhzzhzkveeuyxjdnppqptepwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqumbxpznftscjhymbidtqtdypfedzrc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbzkdhtkvfikgkzbphkiwpyayqhgwecu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gylgtugvjloyhvjppxbnnciatxdompnt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"loinqnojvjwclwwhwetxajltjdfzdiky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xoedimdxxbsxlllchfwbqvwrfjjtzmmp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"caghincwbqgnpmiopqdiiqxgbuvyxatm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zwhsgxvbqgcjnpekidvbkruzhbmzawqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohjjwrwmkwtrrdexyuqkvzhidoulndip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlkahwzspowggzvrpphlpmtpijawvhph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdaddugnovvdbywpwffvtyjynhiruznj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acdqcnqdknovrfjkkfdetucyzbswhxsl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txmkgrcfjvkeymfbdjychvjsbqbjsarc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kgbfjcbwhvyzvimgngepscezzhlehjdt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzjklnastduaffaoccvjigtdrmfovmvj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"boqjppedkaoliyngxundpacmfcfkrtpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wlcpinqiafmttllhdorkqgyljdrpmljc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vltmjvmptxzafimatnqlvvpmkfpdhecb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpxxtkoimezvhljtwimdkbvmeritwmql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qampsmxxfsmmstxmnggoylzoxhmcrlkj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sguuehwmlvngyaepllhznibpurqtanou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cuguuywbcnhvnhvuwzgflgklggwzeszp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vvnpjkfqzfjeitvgdmrwcwfwtajvtxnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"euuycsjfqvtyopymzllpwkxhamljqbat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mupgdmjrndsgvtkjtkzgzskdiknqxsku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itfjyyrwtkczbgaozxzasfnslalgrddw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ndyhlsmilehpcqhezfpheanjfxtlmowo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvukwgxdeubtkfrgwsitvrjzarljyvby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edwdsbqupyflztzgyffagafqyyxlciun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmgdeweqaihskwlhwphmvcxfajspfyka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsmazjecvojbvopoazkbjmuiqtfpimtv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ymnipqomshxqmnexfpsotmdkvtwumjsa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agcnmvchdcennuebuhbghzzpzjbhvhen","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eumaldmkzftstiywbzpvkcquxxgaabij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyxuppotcafafqxtrhtvfdsjwytqyzvv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"snfomdtslfidhrtuheuaoadaqgodyogc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahyovczuxhmzqlcgafgwgupgxobpxyvi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"roxvahcisyqpxrnmgcfaokbzmjvnxuyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmbscrezhtloofhaxuzyjpzxbntwfuqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oufdmxchbmaxixakbddnvgjuijloeeli","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvjtqrakyvdhurwhasuxbkbvhgiukegj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfhmyspndreszjvypskiyptxsoqgejws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzofzqbtjvrftqdwvwbjxysbwwxzbqat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekkmwwnvijbwzgjwglbrqnzcjimmvilv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqyeoshsechlxnvdyquipyptatzdbbpz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nplhmgeeezazjapkjnuleiebkfintidj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gajbbknkrodasacdtpbevznlvnpnvnim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knovbeibgblaacmmulipkgdejhbuydal","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlfiqwkqwrqykptdqwkbmaspenxqnaeh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uryqzuaoamtkldzzsiwfkebvfagnokac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qydljvqqkcougtnywreyvautblctmbkm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ketburpphkhrbgqmrxemptmuuvgpwuvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evsjuyieyqdjecvzeewystxfjnmchbye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovnfrtxprzluwmhrcjozjzyjbtxmvddi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvaylcsnalpmutmwbcibucattdxgnbul","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyemohaxfcevnfdefwfnnifestddsbgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ukvbilphzwnogtdzgcpgstvsuugadpdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oxwmxjdqmjxvvauwtiihfjsdqtzrhivi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yugezltskdbkuxoipziusiqmiemstsgc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"glwgoivqalthpymfxqlymuxeesdikxan","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awbvhzdxcxjlbcccgpvsgweazrzzqtst","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxsaehgxkeityvgfyxplgadncoawyzwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsgdtegifffdopaefjjpsycnusfgtphj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crgfsudctdfjrtsobbyrohsodlgeglbh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvujzovwavrhpgppzwaehklakbjlhozn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozmlphsmyusyehqaofdinnnyrstjtjld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbkpbzdogoknpmccrtjbeejwxpvmwauf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywycrshgdpprwlmxdufurftllxctruju","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671154,"databaseName":"models_schema","ddl":"CREATE TABLE `sudibszqrvrxrwiazvcqqeovwgppgjth` (\n `hcrsrshzjtkbmfxdrxugrtpxkucumbfz` int NOT NULL,\n `efsxyqxadwyjpnqwiylrqzdsklsuzspb` int DEFAULT NULL,\n `gmiriofvxsaktfvjwnifvkwsgesnfcrl` int DEFAULT NULL,\n `zfbllcpzvkgsaqyezbwzotnabvorcnme` int DEFAULT NULL,\n `fxfhbcufgzgzjujpfdjudoxykrnmkxgx` int DEFAULT NULL,\n `ftmwdfgpktavkrhkutyttikohxzsozit` int DEFAULT NULL,\n `vdgujuvcsehheoklekqwjbbjfbxnictr` int DEFAULT NULL,\n `hqeljwpcgfcrshqixtcoyrxglbjxpxgw` int DEFAULT NULL,\n `bwssdgejzzoforbicstowxijtqytcanx` int DEFAULT NULL,\n `vijprqydtavxkgdwwpatpcwejydjcqtg` int DEFAULT NULL,\n `svashmyqmkreqwuwygasdmvpeatquqtu` int DEFAULT NULL,\n `xbpuowfnzdrtoxyycokezfijagcpmqcz` int DEFAULT NULL,\n `svbkkyyxpvhkunwvbwgkmuodkwfiigrd` int DEFAULT NULL,\n `sstaxafyepaughfunrthxletdbrkijrt` int DEFAULT NULL,\n `yfwfsjpipdqixhneprjivotusvuumond` int DEFAULT NULL,\n `gfxneviwamuozvyifbzjprdsfkjqyemp` int DEFAULT NULL,\n `teocddjovogpyzpzmzajxwtfebxwkuqw` int DEFAULT NULL,\n `jhdjkaihnsrzudweehexpjlpylmoacpb` int DEFAULT NULL,\n `sldxplupuzsxtponvoplobcbdihheovw` int DEFAULT NULL,\n `yvmxcedqctzmhsuoxovzdngqabdtyymo` int DEFAULT NULL,\n `bsnwbxvrzmwrxvhwcwlpkaiijyfznfzs` int DEFAULT NULL,\n `sxgdqwmucwatjnzgsvgcujbidrmxfflb` int DEFAULT NULL,\n `vmumqpmkauyzvglcxnegbnyurqcpxhnp` int DEFAULT NULL,\n `atdgttwwzrgrfyzqgbuzdvlusfoekhvn` int DEFAULT NULL,\n `vzfkqreorvzouuwyxdmldimbauglosrv` int DEFAULT NULL,\n `iveqrwxoohmuwlrxvwafgpmtdadqdhsr` int DEFAULT NULL,\n `wphhyhnxvtpzqsauglofsbaixzukoqls` int DEFAULT NULL,\n `tkglggbzydbydubcpvzhpjnbumzwpmxm` int DEFAULT NULL,\n `jtoigrjcoksjgqvsyibkgomefbvopgso` int DEFAULT NULL,\n `jtfkyiujzgtiyklikkypqrampjodwlsn` int DEFAULT NULL,\n `xyxyzmgfthzqspersjxqvvciukhsopek` int DEFAULT NULL,\n `tawycsegsiwgdnwigwffmfulwgepvbqm` int DEFAULT NULL,\n `ixpqcoksjncsodwqafpkoiccxiixvmav` int DEFAULT NULL,\n `dbfdqgomjutxvwlvdbsrslktogzzyfca` int DEFAULT NULL,\n `njgcuuosavzlnebitmbasyeghziemyxv` int DEFAULT NULL,\n `avsgesxeauvrlgjdepwagyogpegklgyv` int DEFAULT NULL,\n `xyqcvvytuakdhiyubkymutzrxbtetzgg` int DEFAULT NULL,\n `hslcjjtyncmhghuehcrpseqzwacqszah` int DEFAULT NULL,\n `gsittccsdraopgtnnyiiddbanlwomxij` int DEFAULT NULL,\n `tnmeekzetlhbirdapfgfhpzhujvnzlbx` int DEFAULT NULL,\n `lnkwyaetctlniqfwdyiilebdnvbhortg` int DEFAULT NULL,\n `kyrvomujffxcmnujbmhgkxvzspqkopdb` int DEFAULT NULL,\n `bgfxuzlwetqchmwsbttnijcxtpbuqbxq` int DEFAULT NULL,\n `pnqtajaikbayrictgbmysizwhbrhjqit` int DEFAULT NULL,\n `kljaepgktjsblqvaksbffejazwtwjboe` int DEFAULT NULL,\n `kkhbbyvncigmabrrugwccqxupjiwtdzq` int DEFAULT NULL,\n `yyypoghjzqppkijyjwxlxgdjyxcnfcra` int DEFAULT NULL,\n `dmzqxhsyssuixdnqemyzapkqusiqeuhq` int DEFAULT NULL,\n `pqowtblagmkhvsfshfmhvynmukhlvfqg` int DEFAULT NULL,\n `tmjntzmckfdpwvxzummqvequvivocgdn` int DEFAULT NULL,\n `bmyizqpxnsyqfnwkxeigrkaicwkiglpc` int DEFAULT NULL,\n `rdkdqbabhyanowcdndewiwgynqdplumw` int DEFAULT NULL,\n `evoyqffzjkugebutgcxglvqvgelirwuo` int DEFAULT NULL,\n `eixxnpojafsqefzqowesownpattarirl` int DEFAULT NULL,\n `awbhxoanhsslexzbpqesrhpjycgrnbuv` int DEFAULT NULL,\n `ztwdoihaqbhgosgpxewvvlyddoshokab` int DEFAULT NULL,\n `trbzpuiiuokqbtjihbiezpztwusuyuqo` int DEFAULT NULL,\n `kcbusghcixlgvlpkgcmgyqcutojmooeq` int DEFAULT NULL,\n `ubrsmqacajojwhdidjahocnztznsueze` int DEFAULT NULL,\n `okwcpetxpumgxxpnnpxdqucbxqmihiov` int DEFAULT NULL,\n `wvphybgzltdlrcxrtpugvivyghsjfahb` int DEFAULT NULL,\n `rwhkbvdtroqaetouuzzibzqhlsclnnee` int DEFAULT NULL,\n `txciplvnwwtznnypbtibpbhjybejaxni` int DEFAULT NULL,\n `byrbwfacmpvamgrlwsgdoqwvguspwbqx` int DEFAULT NULL,\n `ujjbgadaddjigkqpwvovxlgsrtttusxk` int DEFAULT NULL,\n `aztkqhxfabewrnglyvgkaqncpugfrqte` int DEFAULT NULL,\n `jlqnmojarxdnnlchnrfvzqflifwrntkq` int DEFAULT NULL,\n `pkfrvyxzabjdnqukqfsxtindpmgmthfh` int DEFAULT NULL,\n `kfyrltxrlkxobhajntnzfyoridhaidoq` int DEFAULT NULL,\n `nnvhewypqiunlnpvpowbgflggxapfojg` int DEFAULT NULL,\n `reesdepppyznsibbobkxjrfgyqfeegqv` int DEFAULT NULL,\n `yxwwrmuzqzchkbplhmrzkuhbbofcuhxo` int DEFAULT NULL,\n `toctrlmvvfxieavzmidtvleduvbafgah` int DEFAULT NULL,\n `dgboqtesdvkiygnllapsxlblcpcmpepc` int DEFAULT NULL,\n `hiktysrizdfffffojtjomnttuaaceqpe` int DEFAULT NULL,\n `vnpewvtqnphwfrewmvflvftrwegzxgco` int DEFAULT NULL,\n `atpxyunntgextqfggztwmvrnxirqhqqc` int DEFAULT NULL,\n `wybcfebthswscmxqvrhrdoqfgfogpoqw` int DEFAULT NULL,\n `zebrfdxncxnjkcehvqbmpizyklpmxtql` int DEFAULT NULL,\n `vfiwsweckbqziassvmhzixuokbfaaxsd` int DEFAULT NULL,\n `gkbofjrtbaekxdzsjtvpwjwctdsoxyrs` int DEFAULT NULL,\n `ifomulgbqzavijdblthzeooedkoacfdp` int DEFAULT NULL,\n `ppbhboudiuuaylpldvtczljlfgewxvbf` int DEFAULT NULL,\n `dpmdlbehklbubfgjnenbzfdaouuyrycz` int DEFAULT NULL,\n `zfvfwrqhenqasvuoivhmwroblfwyfclx` int DEFAULT NULL,\n `fbtjptjvaqwcxjhtwruwomdsgbadepoi` int DEFAULT NULL,\n `cfespsxiumthjkojjmrdfzwkpdffwexi` int DEFAULT NULL,\n `kpbxenlszrflvmtuoistdhxsgvlbidde` int DEFAULT NULL,\n `xwuropzusvblvjmpjesssheobgsyifie` int DEFAULT NULL,\n `fxtuazxswgkpxzlvcbkwbwqbzfvxcrrh` int DEFAULT NULL,\n `xmzsnbdnokjzkzzbclpsrftqvhjpvvmx` int DEFAULT NULL,\n `lopapdheilvvmqzihbucslhzycpjtpiz` int DEFAULT NULL,\n `xwysmwqltvjxgndocabwrdbuqirpupow` int DEFAULT NULL,\n `lejznkjsnhwarxajmvybgvqyzomqtjdg` int DEFAULT NULL,\n `pwxmbiuvaboqtxlonjlymetwsrfvugwp` int DEFAULT NULL,\n `osgmyhwmnofokyxwywfkvgeecxrscvau` int DEFAULT NULL,\n `xjghjeeatutdkdspibjutnncudaeahjt` int DEFAULT NULL,\n `fgysbatshtpktrdwgekitnzlmjjodbzo` int DEFAULT NULL,\n `tptpexfncnbkuvyfhvsjogysjgrkcilr` int DEFAULT NULL,\n `dlanxvfwjawoyujmewatcgmgkyvwnylp` int DEFAULT NULL,\n PRIMARY KEY (`hcrsrshzjtkbmfxdrxugrtpxkucumbfz`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"sudibszqrvrxrwiazvcqqeovwgppgjth\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["hcrsrshzjtkbmfxdrxugrtpxkucumbfz"],"columns":[{"name":"hcrsrshzjtkbmfxdrxugrtpxkucumbfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"efsxyqxadwyjpnqwiylrqzdsklsuzspb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmiriofvxsaktfvjwnifvkwsgesnfcrl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zfbllcpzvkgsaqyezbwzotnabvorcnme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxfhbcufgzgzjujpfdjudoxykrnmkxgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ftmwdfgpktavkrhkutyttikohxzsozit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdgujuvcsehheoklekqwjbbjfbxnictr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqeljwpcgfcrshqixtcoyrxglbjxpxgw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bwssdgejzzoforbicstowxijtqytcanx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vijprqydtavxkgdwwpatpcwejydjcqtg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"svashmyqmkreqwuwygasdmvpeatquqtu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbpuowfnzdrtoxyycokezfijagcpmqcz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"svbkkyyxpvhkunwvbwgkmuodkwfiigrd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sstaxafyepaughfunrthxletdbrkijrt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfwfsjpipdqixhneprjivotusvuumond","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfxneviwamuozvyifbzjprdsfkjqyemp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"teocddjovogpyzpzmzajxwtfebxwkuqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jhdjkaihnsrzudweehexpjlpylmoacpb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sldxplupuzsxtponvoplobcbdihheovw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yvmxcedqctzmhsuoxovzdngqabdtyymo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bsnwbxvrzmwrxvhwcwlpkaiijyfznfzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sxgdqwmucwatjnzgsvgcujbidrmxfflb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmumqpmkauyzvglcxnegbnyurqcpxhnp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atdgttwwzrgrfyzqgbuzdvlusfoekhvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzfkqreorvzouuwyxdmldimbauglosrv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iveqrwxoohmuwlrxvwafgpmtdadqdhsr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wphhyhnxvtpzqsauglofsbaixzukoqls","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkglggbzydbydubcpvzhpjnbumzwpmxm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jtoigrjcoksjgqvsyibkgomefbvopgso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jtfkyiujzgtiyklikkypqrampjodwlsn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyxyzmgfthzqspersjxqvvciukhsopek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tawycsegsiwgdnwigwffmfulwgepvbqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixpqcoksjncsodwqafpkoiccxiixvmav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbfdqgomjutxvwlvdbsrslktogzzyfca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njgcuuosavzlnebitmbasyeghziemyxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avsgesxeauvrlgjdepwagyogpegklgyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyqcvvytuakdhiyubkymutzrxbtetzgg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hslcjjtyncmhghuehcrpseqzwacqszah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsittccsdraopgtnnyiiddbanlwomxij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnmeekzetlhbirdapfgfhpzhujvnzlbx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lnkwyaetctlniqfwdyiilebdnvbhortg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kyrvomujffxcmnujbmhgkxvzspqkopdb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bgfxuzlwetqchmwsbttnijcxtpbuqbxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnqtajaikbayrictgbmysizwhbrhjqit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kljaepgktjsblqvaksbffejazwtwjboe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kkhbbyvncigmabrrugwccqxupjiwtdzq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yyypoghjzqppkijyjwxlxgdjyxcnfcra","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dmzqxhsyssuixdnqemyzapkqusiqeuhq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqowtblagmkhvsfshfmhvynmukhlvfqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmjntzmckfdpwvxzummqvequvivocgdn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bmyizqpxnsyqfnwkxeigrkaicwkiglpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdkdqbabhyanowcdndewiwgynqdplumw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evoyqffzjkugebutgcxglvqvgelirwuo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eixxnpojafsqefzqowesownpattarirl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awbhxoanhsslexzbpqesrhpjycgrnbuv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ztwdoihaqbhgosgpxewvvlyddoshokab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"trbzpuiiuokqbtjihbiezpztwusuyuqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcbusghcixlgvlpkgcmgyqcutojmooeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubrsmqacajojwhdidjahocnztznsueze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okwcpetxpumgxxpnnpxdqucbxqmihiov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvphybgzltdlrcxrtpugvivyghsjfahb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rwhkbvdtroqaetouuzzibzqhlsclnnee","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txciplvnwwtznnypbtibpbhjybejaxni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"byrbwfacmpvamgrlwsgdoqwvguspwbqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujjbgadaddjigkqpwvovxlgsrtttusxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aztkqhxfabewrnglyvgkaqncpugfrqte","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlqnmojarxdnnlchnrfvzqflifwrntkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkfrvyxzabjdnqukqfsxtindpmgmthfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfyrltxrlkxobhajntnzfyoridhaidoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nnvhewypqiunlnpvpowbgflggxapfojg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"reesdepppyznsibbobkxjrfgyqfeegqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxwwrmuzqzchkbplhmrzkuhbbofcuhxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"toctrlmvvfxieavzmidtvleduvbafgah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgboqtesdvkiygnllapsxlblcpcmpepc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hiktysrizdfffffojtjomnttuaaceqpe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnpewvtqnphwfrewmvflvftrwegzxgco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atpxyunntgextqfggztwmvrnxirqhqqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wybcfebthswscmxqvrhrdoqfgfogpoqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zebrfdxncxnjkcehvqbmpizyklpmxtql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vfiwsweckbqziassvmhzixuokbfaaxsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkbofjrtbaekxdzsjtvpwjwctdsoxyrs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifomulgbqzavijdblthzeooedkoacfdp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ppbhboudiuuaylpldvtczljlfgewxvbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpmdlbehklbubfgjnenbzfdaouuyrycz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zfvfwrqhenqasvuoivhmwroblfwyfclx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbtjptjvaqwcxjhtwruwomdsgbadepoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfespsxiumthjkojjmrdfzwkpdffwexi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpbxenlszrflvmtuoistdhxsgvlbidde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xwuropzusvblvjmpjesssheobgsyifie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxtuazxswgkpxzlvcbkwbwqbzfvxcrrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmzsnbdnokjzkzzbclpsrftqvhjpvvmx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lopapdheilvvmqzihbucslhzycpjtpiz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xwysmwqltvjxgndocabwrdbuqirpupow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lejznkjsnhwarxajmvybgvqyzomqtjdg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwxmbiuvaboqtxlonjlymetwsrfvugwp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"osgmyhwmnofokyxwywfkvgeecxrscvau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjghjeeatutdkdspibjutnncudaeahjt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fgysbatshtpktrdwgekitnzlmjjodbzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tptpexfncnbkuvyfhvsjogysjgrkcilr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlanxvfwjawoyujmewatcgmgkyvwnylp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671188,"databaseName":"models_schema","ddl":"CREATE TABLE `suvqorxzmwvjejxhmiiitcxlimjbkmok` (\n `rkvdkhogxztsasbzvenhwqiqpyntpeku` int NOT NULL,\n `pqtdlcheovcwkgolciqgodigisgksqwm` int DEFAULT NULL,\n `cjtumxcjytnjltrvckcvrgitsxeghfxx` int DEFAULT NULL,\n `suuzrfggurmpftteopzpjexttwyyimqw` int DEFAULT NULL,\n `tggrxhkxmmrzotlnrapzturpepgsltzt` int DEFAULT NULL,\n `zwhsckhlcngtfnceayvlnnnwtnrxynyl` int DEFAULT NULL,\n `fwnrtmjyhahvunyilletpuymrauaruak` int DEFAULT NULL,\n `orsifeivjufnedubanlbbfxrdkipcfkt` int DEFAULT NULL,\n `txypmjjdmzeothbeniobnmliwhcrnctr` int DEFAULT NULL,\n `nljwmyvouovkfqmsdrqivgeerpwtlrpj` int DEFAULT NULL,\n `bhljwxidmfjlrihvrizhzjlbrrsbokiw` int DEFAULT NULL,\n `bshydzpbitabpogyierzhsmvsgzsxjum` int DEFAULT NULL,\n `dwsgwxyauqtvxvpnllqcsrgzfeflegct` int DEFAULT NULL,\n `jmjryxnwbmseswgycxlkwjyfqkvatjri` int DEFAULT NULL,\n `jplkfbufopkzuhhkrpiarzlyfttwzjam` int DEFAULT NULL,\n `rijchtcphsctkmjonsjawqminmfthcrj` int DEFAULT NULL,\n `iqbsjvvyewosadgxekdbfamsmderkntw` int DEFAULT NULL,\n `awlxxojwhcdugbjldgtnycxjcogihhbv` int DEFAULT NULL,\n `swwpguvursonddvbdkmdqnlcojjeagnt` int DEFAULT NULL,\n `nbsgkduwvpufjqphlsicvooycjbpsryi` int DEFAULT NULL,\n `nzoqmyoivtxcdrzyhafqhenukxsigiag` int DEFAULT NULL,\n `xyypdnwrldcnygnwsyekpiacfddwrpuu` int DEFAULT NULL,\n `dnhcwdwmehhnvfdqjkiyvvvuvwqmofoy` int DEFAULT NULL,\n `qyqkwrnbyuliickiszwwlhmfiwdeesrz` int DEFAULT NULL,\n `ytpmxbfioefflcmldgpzcjalkbwvlsrd` int DEFAULT NULL,\n `ugnfkqjtbkrdyyfpwznvgkdadibknjtx` int DEFAULT NULL,\n `zpaaagmscgpzjugsetajndknzuvbjszt` int DEFAULT NULL,\n `ifokdqkmoetmldpvoakxisrxtqmvsucq` int DEFAULT NULL,\n `zclmjlmvdpevfecudyzjqdruxinmnibu` int DEFAULT NULL,\n `nxbwauwuvugovxbuxmgnsbrhxwzlgpjs` int DEFAULT NULL,\n `yhcjkuuumaaebfoqhtgfymewsdnhpibe` int DEFAULT NULL,\n `edgqaiexkxmvmmlirbjbhabczmsmximd` int DEFAULT NULL,\n `ztbyfghyepssqqmlkmwewveibnhrlylq` int DEFAULT NULL,\n `fndamouszphbzqamyaivbdgxwchhqtkd` int DEFAULT NULL,\n `yfncydytxpmhxhovbqlqghahaeloxzop` int DEFAULT NULL,\n `vbsoyioxwcstbadyntregehflaxkkkyn` int DEFAULT NULL,\n `soywolnuljgwukixpmqhoumprgslfcaa` int DEFAULT NULL,\n `eellkubscsapntwhuizssbkvypyktksu` int DEFAULT NULL,\n `geuwtqqhmlsassvmrimdbxfpqialpcjn` int DEFAULT NULL,\n `prqmhjtsulszjqfuunsvndtjnsrffpxz` int DEFAULT NULL,\n `kjyrcrtrelhdazxplvhbzkugqipreanh` int DEFAULT NULL,\n `eaparnoqrqanfbboogokmizyljbdgcpo` int DEFAULT NULL,\n `mbqybvpviiimaoiibxyvdtyhaofohmfg` int DEFAULT NULL,\n `evgwqdnlbtctdjlkuxnqbykimaqsalgx` int DEFAULT NULL,\n `dgatiqeqtbjaasixsopgtbcbpiuupkcu` int DEFAULT NULL,\n `fbfmatecjvzfmewbjqcfuysomqscagdt` int DEFAULT NULL,\n `idjzlzqjxdtbzrrfrmivdimxxxssxucf` int DEFAULT NULL,\n `tmrcolqbevmyhyfrekdffngnoqvybnfo` int DEFAULT NULL,\n `mahjfurkvnvexupibuyhkeltvxwnfsnm` int DEFAULT NULL,\n `nurbumbeyqlfujazibiqjjtqvailgisx` int DEFAULT NULL,\n `gqiaxrdcgiicewqqyxvhosrntlkjfyzv` int DEFAULT NULL,\n `ggnfhahblqqhksigjxndyudzkscycusi` int DEFAULT NULL,\n `atlluunnomoqgditlkpohibfisyyxnec` int DEFAULT NULL,\n `iedujpexfbezuswzqhnxeiguzztmrvzo` int DEFAULT NULL,\n `vacwhoqpvxdrbfzooaviezsnqiyrrubs` int DEFAULT NULL,\n `cctyffjmewovreswficqlaxsmrvkdida` int DEFAULT NULL,\n `spszubvrezxyuhqnaxvdiauenazmxhvo` int DEFAULT NULL,\n `ywxuaaevfevtquqimkwohewpvwcighiz` int DEFAULT NULL,\n `ldqyqszqywndrurqnqwnxxtpabvneoxg` int DEFAULT NULL,\n `brttvflqehjqcjqcwsxzlnfiercnhhlj` int DEFAULT NULL,\n `sbboaupvcwkczokkdoszbexelccitgut` int DEFAULT NULL,\n `ayhujutftkiasiyqubpxqrhwpvkdcrzp` int DEFAULT NULL,\n `hpxvbwseqlbcrvmhdjdhdzknangtrxyx` int DEFAULT NULL,\n `okzgidkmkcwpsuhjbampzrordvgavabk` int DEFAULT NULL,\n `gsmwdqfvbavucmugvmopnqrpbnucbiow` int DEFAULT NULL,\n `pvtrbtferfziogggcrbvrapmiboamiiw` int DEFAULT NULL,\n `jtpbocgvzzxnnhynyyqdyhoioxytgadw` int DEFAULT NULL,\n `euczcgpjhcgmcelpzkjkttqtddgmcxqi` int DEFAULT NULL,\n `zxxxtdcpqccxwqqunwlyunyjchayprnp` int DEFAULT NULL,\n `jtozyykliepndbzkmoppmeistkrcjmad` int DEFAULT NULL,\n `ebbxysghmtvbphgpyuldfxznyamlyluc` int DEFAULT NULL,\n `iwmvjloidtuhlfrsnlnkmoxwdgxaqani` int DEFAULT NULL,\n `yjhcswuzwktxwxrffguyzshjljddwzqu` int DEFAULT NULL,\n `iwjxqnbzcqvrmbumfbegtzbbtnnpprfc` int DEFAULT NULL,\n `ogvuflusfhatphgfhlrcmlxudimmjjyo` int DEFAULT NULL,\n `cdqvolvkmhmbygybyvobkdzqnzzzjesr` int DEFAULT NULL,\n `dsinvgajtnmyjvsdapskkpapbnqutmmj` int DEFAULT NULL,\n `aatbqixiilwkrzthlprdgpscxaugfuws` int DEFAULT NULL,\n `rkfvqwtglhwsoclvifbrautwscosicex` int DEFAULT NULL,\n `epoehzroffbzplpuhyociqnapvbdtfxb` int DEFAULT NULL,\n `cbbqzaztstmkhdhhwwginaghqgmmrkpt` int DEFAULT NULL,\n `znguqqrxqukomtigpgaljpdemuiduiyc` int DEFAULT NULL,\n `mqmeblepefjmpqwlinzmiujarbdbbsar` int DEFAULT NULL,\n `vnzqsxmbiemkdzoypoqjsqwiieeoctbo` int DEFAULT NULL,\n `mazpkjcfplikqdubwmscqydyazryexvg` int DEFAULT NULL,\n `ybjvobfkhlqjdoqnwtmmcsryvxilwgbb` int DEFAULT NULL,\n `pprtrpkcritbqztcuqdmgognlvpoekgx` int DEFAULT NULL,\n `rwzklqvehgpujthseeojednbcavcntze` int DEFAULT NULL,\n `oquutovycdolsvlonhwxvxwsbumsichh` int DEFAULT NULL,\n `ecpyvhgjhotglcvefkeyaedhjretdtmi` int DEFAULT NULL,\n `fskntlgyfwuqzsqghsowavnryxhxqtcn` int DEFAULT NULL,\n `oeitpfpizyasxkcfyyrghsnjqzvoellq` int DEFAULT NULL,\n `ikwpvgdbxjupyunvbjqkstgddbvsfgcl` int DEFAULT NULL,\n `kwfysduasdlzwiexnjgpmgwvdmbgbvzm` int DEFAULT NULL,\n `odgssanjnmzvzaaflrrlmnkajopiltep` int DEFAULT NULL,\n `tghrouvjcccagvdxieyzufrcffbmarmu` int DEFAULT NULL,\n `mzrwqauokniqulazysehaximzdkigpzr` int DEFAULT NULL,\n `zepgnuvjxnofnahwweclqbymwuxynmny` int DEFAULT NULL,\n `znogsojxczdrwawrzefdrmdhfsuzzyqz` int DEFAULT NULL,\n `auysxczgnmltfrgcvyqpavmxlcwlnrxp` int DEFAULT NULL,\n PRIMARY KEY (`rkvdkhogxztsasbzvenhwqiqpyntpeku`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"suvqorxzmwvjejxhmiiitcxlimjbkmok\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["rkvdkhogxztsasbzvenhwqiqpyntpeku"],"columns":[{"name":"rkvdkhogxztsasbzvenhwqiqpyntpeku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"pqtdlcheovcwkgolciqgodigisgksqwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjtumxcjytnjltrvckcvrgitsxeghfxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suuzrfggurmpftteopzpjexttwyyimqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tggrxhkxmmrzotlnrapzturpepgsltzt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zwhsckhlcngtfnceayvlnnnwtnrxynyl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwnrtmjyhahvunyilletpuymrauaruak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orsifeivjufnedubanlbbfxrdkipcfkt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txypmjjdmzeothbeniobnmliwhcrnctr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nljwmyvouovkfqmsdrqivgeerpwtlrpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bhljwxidmfjlrihvrizhzjlbrrsbokiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bshydzpbitabpogyierzhsmvsgzsxjum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwsgwxyauqtvxvpnllqcsrgzfeflegct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmjryxnwbmseswgycxlkwjyfqkvatjri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jplkfbufopkzuhhkrpiarzlyfttwzjam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rijchtcphsctkmjonsjawqminmfthcrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqbsjvvyewosadgxekdbfamsmderkntw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awlxxojwhcdugbjldgtnycxjcogihhbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swwpguvursonddvbdkmdqnlcojjeagnt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbsgkduwvpufjqphlsicvooycjbpsryi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzoqmyoivtxcdrzyhafqhenukxsigiag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyypdnwrldcnygnwsyekpiacfddwrpuu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dnhcwdwmehhnvfdqjkiyvvvuvwqmofoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qyqkwrnbyuliickiszwwlhmfiwdeesrz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytpmxbfioefflcmldgpzcjalkbwvlsrd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugnfkqjtbkrdyyfpwznvgkdadibknjtx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpaaagmscgpzjugsetajndknzuvbjszt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifokdqkmoetmldpvoakxisrxtqmvsucq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zclmjlmvdpevfecudyzjqdruxinmnibu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxbwauwuvugovxbuxmgnsbrhxwzlgpjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhcjkuuumaaebfoqhtgfymewsdnhpibe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edgqaiexkxmvmmlirbjbhabczmsmximd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ztbyfghyepssqqmlkmwewveibnhrlylq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fndamouszphbzqamyaivbdgxwchhqtkd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfncydytxpmhxhovbqlqghahaeloxzop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbsoyioxwcstbadyntregehflaxkkkyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"soywolnuljgwukixpmqhoumprgslfcaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eellkubscsapntwhuizssbkvypyktksu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"geuwtqqhmlsassvmrimdbxfpqialpcjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prqmhjtsulszjqfuunsvndtjnsrffpxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjyrcrtrelhdazxplvhbzkugqipreanh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eaparnoqrqanfbboogokmizyljbdgcpo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbqybvpviiimaoiibxyvdtyhaofohmfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evgwqdnlbtctdjlkuxnqbykimaqsalgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgatiqeqtbjaasixsopgtbcbpiuupkcu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbfmatecjvzfmewbjqcfuysomqscagdt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idjzlzqjxdtbzrrfrmivdimxxxssxucf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmrcolqbevmyhyfrekdffngnoqvybnfo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mahjfurkvnvexupibuyhkeltvxwnfsnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nurbumbeyqlfujazibiqjjtqvailgisx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqiaxrdcgiicewqqyxvhosrntlkjfyzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggnfhahblqqhksigjxndyudzkscycusi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atlluunnomoqgditlkpohibfisyyxnec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iedujpexfbezuswzqhnxeiguzztmrvzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vacwhoqpvxdrbfzooaviezsnqiyrrubs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cctyffjmewovreswficqlaxsmrvkdida","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"spszubvrezxyuhqnaxvdiauenazmxhvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywxuaaevfevtquqimkwohewpvwcighiz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ldqyqszqywndrurqnqwnxxtpabvneoxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"brttvflqehjqcjqcwsxzlnfiercnhhlj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbboaupvcwkczokkdoszbexelccitgut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ayhujutftkiasiyqubpxqrhwpvkdcrzp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hpxvbwseqlbcrvmhdjdhdzknangtrxyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okzgidkmkcwpsuhjbampzrordvgavabk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsmwdqfvbavucmugvmopnqrpbnucbiow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvtrbtferfziogggcrbvrapmiboamiiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jtpbocgvzzxnnhynyyqdyhoioxytgadw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"euczcgpjhcgmcelpzkjkttqtddgmcxqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxxxtdcpqccxwqqunwlyunyjchayprnp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jtozyykliepndbzkmoppmeistkrcjmad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebbxysghmtvbphgpyuldfxznyamlyluc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwmvjloidtuhlfrsnlnkmoxwdgxaqani","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjhcswuzwktxwxrffguyzshjljddwzqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwjxqnbzcqvrmbumfbegtzbbtnnpprfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ogvuflusfhatphgfhlrcmlxudimmjjyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cdqvolvkmhmbygybyvobkdzqnzzzjesr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsinvgajtnmyjvsdapskkpapbnqutmmj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aatbqixiilwkrzthlprdgpscxaugfuws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkfvqwtglhwsoclvifbrautwscosicex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"epoehzroffbzplpuhyociqnapvbdtfxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbbqzaztstmkhdhhwwginaghqgmmrkpt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znguqqrxqukomtigpgaljpdemuiduiyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqmeblepefjmpqwlinzmiujarbdbbsar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnzqsxmbiemkdzoypoqjsqwiieeoctbo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mazpkjcfplikqdubwmscqydyazryexvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybjvobfkhlqjdoqnwtmmcsryvxilwgbb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pprtrpkcritbqztcuqdmgognlvpoekgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rwzklqvehgpujthseeojednbcavcntze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oquutovycdolsvlonhwxvxwsbumsichh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecpyvhgjhotglcvefkeyaedhjretdtmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fskntlgyfwuqzsqghsowavnryxhxqtcn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oeitpfpizyasxkcfyyrghsnjqzvoellq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikwpvgdbxjupyunvbjqkstgddbvsfgcl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwfysduasdlzwiexnjgpmgwvdmbgbvzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"odgssanjnmzvzaaflrrlmnkajopiltep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tghrouvjcccagvdxieyzufrcffbmarmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzrwqauokniqulazysehaximzdkigpzr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zepgnuvjxnofnahwweclqbymwuxynmny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znogsojxczdrwawrzefdrmdhfsuzzyqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"auysxczgnmltfrgcvyqpavmxlcwlnrxp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671223,"databaseName":"models_schema","ddl":"CREATE TABLE `swgxwfoajgckubpfskppncstrxbnrpfl` (\n `gihcvacpzfwpnwpnzlxeyeifvictkphz` int NOT NULL,\n `ulbahbqrorobyhaebdiohjwiovcrgzyf` int DEFAULT NULL,\n `cuhotrczeyaltydwlfpzqkjucipwzsok` int DEFAULT NULL,\n `oeczwyqilffqnirjpozghfmxzdexgbai` int DEFAULT NULL,\n `rsnsvkswkwueuqbqglruneterxikboem` int DEFAULT NULL,\n `fzmephxngzydftznfxqmnlvpqoyhdrln` int DEFAULT NULL,\n `gqwqnjfhsfhgdltacyiqwaxpxcwdcabz` int DEFAULT NULL,\n `fkchlgwfgfzifwsyeigwbjiefhieoims` int DEFAULT NULL,\n `gpeusetbwngbzsduoziybipzwbmhvbeq` int DEFAULT NULL,\n `hjlnipmknaxtkwqljfmxydcwekcgfbzp` int DEFAULT NULL,\n `kdmzthncxzomschopqglkiczcgkmhvwi` int DEFAULT NULL,\n `mfyfdtrghlbibrtxpkmptllzlpjlmpri` int DEFAULT NULL,\n `afybxfdtmgwkwuppxarnzneyxmcrwoqr` int DEFAULT NULL,\n `iepfzhqxmlxbrahrjobghfvqwkcmfcfh` int DEFAULT NULL,\n `onzjjlnqxwghcaxyflyvtmltscsujwer` int DEFAULT NULL,\n `voaykofnrpctnzxkawrucgpnddltfdgq` int DEFAULT NULL,\n `jboxmesmqsfjrmeodlzlsftfuxmwiarp` int DEFAULT NULL,\n `zqnhceicwekwwvpbvhhiqyocqogqqfvp` int DEFAULT NULL,\n `rlslkhcmjuqxlnzlulgkbuelpceqygnp` int DEFAULT NULL,\n `dmreephaqvqgkwvoofinzfiawncouwip` int DEFAULT NULL,\n `qhyxpdxcjlmbqvyccrydbgsaiemlsbtt` int DEFAULT NULL,\n `qheigkwtylxljywakzupjvoxvkemnfym` int DEFAULT NULL,\n `okjvilvrxlvygfrwsqypftxgoqtoznvl` int DEFAULT NULL,\n `akuebzgjcgsdpqitzpqmmmtoktuoczid` int DEFAULT NULL,\n `daugbwbicehqmrqxalybywbifyqlzvid` int DEFAULT NULL,\n `sagtjvbhoqvzadjqvaqqrgkmwozqhfof` int DEFAULT NULL,\n `gbgzwzhwtidjygmghwlcgrfyftaagdfh` int DEFAULT NULL,\n `zfwiwsbgobvithlvfstuuwqutuzfgvpc` int DEFAULT NULL,\n `gcctwsuguvqzhktnwknzrodowkkqjcty` int DEFAULT NULL,\n `xvayftdueucvdthlginmsaaeuloylbnh` int DEFAULT NULL,\n `ydqlfissisttlmfniflozyafqvfitfxm` int DEFAULT NULL,\n `gdrrejichncrgceebplwgiqyjvfxqxbs` int DEFAULT NULL,\n `ieybyrxirzuhsbaiaugklphllgzzggad` int DEFAULT NULL,\n `mgqbwosshwaajhapqhcswzzovbufcddx` int DEFAULT NULL,\n `ryhuolfvkrkvgkzernaegegjrhwxwvxj` int DEFAULT NULL,\n `bwikxuxwqjwtqqqerjbeyjkqnzzachsp` int DEFAULT NULL,\n `vghmygeuqdtltqqmavluoeboecytbozo` int DEFAULT NULL,\n `gjtxgwzcddbwmdfbbygyfzzazqukofwx` int DEFAULT NULL,\n `gvvtkcydykevrozkfcbwbunvnrgocxjy` int DEFAULT NULL,\n `yzzpvflfgiglnymplrfnwhomjwbkdmqh` int DEFAULT NULL,\n `vkbuvavalrzanqfmhwparjuufsolrrjc` int DEFAULT NULL,\n `yjbkpegileptcjubcissvsxakeqrhjir` int DEFAULT NULL,\n `yolrayejrboujsmumviemvwiqxrtvxnk` int DEFAULT NULL,\n `yiqimmqhzqchmqmvzgtafkvabqjfzrmc` int DEFAULT NULL,\n `noowmixepsuylhtxkwecexznqtnvxxev` int DEFAULT NULL,\n `lloxjsnawddapmardzcrpifsgmqxmofz` int DEFAULT NULL,\n `ovpconeqovraxkipoqalyqbxsrjcytfq` int DEFAULT NULL,\n `vjdqqoqsjccepeyukfoxilzxfpkaytjg` int DEFAULT NULL,\n `gwtsmvlhibayulfszgelypzpaxxfgoko` int DEFAULT NULL,\n `hmetpjyyfuaicbweskryasjgxkvkawaw` int DEFAULT NULL,\n `cbozotavpqmpojzdeqsuqaiecuhuxkzz` int DEFAULT NULL,\n `scoynecisgalwrwbspmcapjbssdxubft` int DEFAULT NULL,\n `veancbabmtyrgygmltrispvvpnynpbmi` int DEFAULT NULL,\n `hcjjjcgzolvaaxbgjdtpjhrtohrsinpc` int DEFAULT NULL,\n `znbnghgmmqajxgycgieiewzhnlbdpuni` int DEFAULT NULL,\n `wemxlanyywfnjktbfpetsquziiooiluf` int DEFAULT NULL,\n `ixhcjekfonezphbmtfjkjmvogpxjnexu` int DEFAULT NULL,\n `anyktxvialdrvhwqzypjpmisaddxpxvc` int DEFAULT NULL,\n `sththkruwfeklzoqqtytazdnviqigsvz` int DEFAULT NULL,\n `tvdhbhpgdmqhsyadoacyfgqfbuwogfhk` int DEFAULT NULL,\n `cvikcfcwwunxmyurpwvjbrfktpysgbxq` int DEFAULT NULL,\n `ebnfefanqjsdsllashtelriyikxflpwn` int DEFAULT NULL,\n `nowxbbvisxhwjtuplbffghpkzduosrmj` int DEFAULT NULL,\n `hzwcevhqrdcciufmuwnnxeiboikhnmgg` int DEFAULT NULL,\n `fxeikbynysyrpuxaphevxcgubacqvakb` int DEFAULT NULL,\n `qtsqjigwyzzfsjcdckzjwssxzeuuzljn` int DEFAULT NULL,\n `lpnrtztczdvuiriujuzdigsgbgoexxjo` int DEFAULT NULL,\n `fldqpxxlnamzxrtpojkizuxghxgmrvix` int DEFAULT NULL,\n `dcaldeypljvxdrtkgmbybrybqkcigsci` int DEFAULT NULL,\n `kpeoeainurabficnroribqfgusobtylb` int DEFAULT NULL,\n `jtyfdwulqungzbueuvcbfmszimindjcj` int DEFAULT NULL,\n `yjtgtfypywppvbffldeyizxuqfdcmdyx` int DEFAULT NULL,\n `dpapacebgoobmwwshzzblvarpxzqcjgm` int DEFAULT NULL,\n `vtmmmsiveleauvljzsbnrslzmnbhdkcw` int DEFAULT NULL,\n `oyumnadxnecbvklujkirpdochmioqxjf` int DEFAULT NULL,\n `qtyfyfivydmspzoywrdgmcwyjckdeywt` int DEFAULT NULL,\n `fuzdqjuxrfsdfdmslfsszsjeehbyumdv` int DEFAULT NULL,\n `zukupgajckjiatmaukoquqetjjjoqtxh` int DEFAULT NULL,\n `rkuqreaguqrrvxncmyofrxhugahykomx` int DEFAULT NULL,\n `ashvmxwpzwqhiscuifahdbkknwawcsob` int DEFAULT NULL,\n `ggmlymcmimgkczbchlseddfsrvxwkdzb` int DEFAULT NULL,\n `ichdelxbvoqtjrezfenmzmfaybdopqvb` int DEFAULT NULL,\n `bmleyvqawgoguimdahevsrkipstfgxix` int DEFAULT NULL,\n `oanjffciuwuxxrhyospwiitumygicevd` int DEFAULT NULL,\n `fnzuzohgxdmtysdbwcygedtypypzedeu` int DEFAULT NULL,\n `bujydoafhvjsuahtesjwubwmxzlriuln` int DEFAULT NULL,\n `guprxjosageqfsjjltoguxosomwxsgbb` int DEFAULT NULL,\n `uynjklvqfopxbmmoinlfjpgitzmoejek` int DEFAULT NULL,\n `ffjvmhojdvmfmlezotvyylbesodeooik` int DEFAULT NULL,\n `nhmmelbebttbeoeuwnlmdjzlmbwwohvu` int DEFAULT NULL,\n `vxlwdahfdivcrxkvwztyjtxazbhisgxf` int DEFAULT NULL,\n `opykcuqzqgoydktcgjorkerhgnguendm` int DEFAULT NULL,\n `uzvgpuprpqnpytseumlbppgocbgmrwtm` int DEFAULT NULL,\n `zljecqtxktcpffdvfjiusryjupoclmbx` int DEFAULT NULL,\n `xwpoatdpfysnickjjvgbddjfunntazzr` int DEFAULT NULL,\n `tvtabgtqretqndhyxuoxnnratytpwtul` int DEFAULT NULL,\n `smowtnqcrgfqseqhblwxejktjqygfsim` int DEFAULT NULL,\n `herenvefefsavmmsnlswsfmaligxtyry` int DEFAULT NULL,\n `sdcetqpavsmtntcjcfldgqkeggltoirt` int DEFAULT NULL,\n `cezduunyoydfpcuujnyncrwvxniratuj` int DEFAULT NULL,\n PRIMARY KEY (`gihcvacpzfwpnwpnzlxeyeifvictkphz`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"swgxwfoajgckubpfskppncstrxbnrpfl\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["gihcvacpzfwpnwpnzlxeyeifvictkphz"],"columns":[{"name":"gihcvacpzfwpnwpnzlxeyeifvictkphz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ulbahbqrorobyhaebdiohjwiovcrgzyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cuhotrczeyaltydwlfpzqkjucipwzsok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oeczwyqilffqnirjpozghfmxzdexgbai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rsnsvkswkwueuqbqglruneterxikboem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzmephxngzydftznfxqmnlvpqoyhdrln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqwqnjfhsfhgdltacyiqwaxpxcwdcabz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fkchlgwfgfzifwsyeigwbjiefhieoims","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gpeusetbwngbzsduoziybipzwbmhvbeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjlnipmknaxtkwqljfmxydcwekcgfbzp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kdmzthncxzomschopqglkiczcgkmhvwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfyfdtrghlbibrtxpkmptllzlpjlmpri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afybxfdtmgwkwuppxarnzneyxmcrwoqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iepfzhqxmlxbrahrjobghfvqwkcmfcfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"onzjjlnqxwghcaxyflyvtmltscsujwer","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"voaykofnrpctnzxkawrucgpnddltfdgq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jboxmesmqsfjrmeodlzlsftfuxmwiarp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqnhceicwekwwvpbvhhiqyocqogqqfvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlslkhcmjuqxlnzlulgkbuelpceqygnp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dmreephaqvqgkwvoofinzfiawncouwip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhyxpdxcjlmbqvyccrydbgsaiemlsbtt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qheigkwtylxljywakzupjvoxvkemnfym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okjvilvrxlvygfrwsqypftxgoqtoznvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akuebzgjcgsdpqitzpqmmmtoktuoczid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"daugbwbicehqmrqxalybywbifyqlzvid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sagtjvbhoqvzadjqvaqqrgkmwozqhfof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gbgzwzhwtidjygmghwlcgrfyftaagdfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zfwiwsbgobvithlvfstuuwqutuzfgvpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gcctwsuguvqzhktnwknzrodowkkqjcty","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvayftdueucvdthlginmsaaeuloylbnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydqlfissisttlmfniflozyafqvfitfxm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdrrejichncrgceebplwgiqyjvfxqxbs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ieybyrxirzuhsbaiaugklphllgzzggad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mgqbwosshwaajhapqhcswzzovbufcddx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ryhuolfvkrkvgkzernaegegjrhwxwvxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bwikxuxwqjwtqqqerjbeyjkqnzzachsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vghmygeuqdtltqqmavluoeboecytbozo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjtxgwzcddbwmdfbbygyfzzazqukofwx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gvvtkcydykevrozkfcbwbunvnrgocxjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzzpvflfgiglnymplrfnwhomjwbkdmqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkbuvavalrzanqfmhwparjuufsolrrjc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjbkpegileptcjubcissvsxakeqrhjir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yolrayejrboujsmumviemvwiqxrtvxnk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yiqimmqhzqchmqmvzgtafkvabqjfzrmc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"noowmixepsuylhtxkwecexznqtnvxxev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lloxjsnawddapmardzcrpifsgmqxmofz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovpconeqovraxkipoqalyqbxsrjcytfq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjdqqoqsjccepeyukfoxilzxfpkaytjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gwtsmvlhibayulfszgelypzpaxxfgoko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmetpjyyfuaicbweskryasjgxkvkawaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbozotavpqmpojzdeqsuqaiecuhuxkzz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scoynecisgalwrwbspmcapjbssdxubft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"veancbabmtyrgygmltrispvvpnynpbmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hcjjjcgzolvaaxbgjdtpjhrtohrsinpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znbnghgmmqajxgycgieiewzhnlbdpuni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wemxlanyywfnjktbfpetsquziiooiluf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixhcjekfonezphbmtfjkjmvogpxjnexu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"anyktxvialdrvhwqzypjpmisaddxpxvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sththkruwfeklzoqqtytazdnviqigsvz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvdhbhpgdmqhsyadoacyfgqfbuwogfhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvikcfcwwunxmyurpwvjbrfktpysgbxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebnfefanqjsdsllashtelriyikxflpwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nowxbbvisxhwjtuplbffghpkzduosrmj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hzwcevhqrdcciufmuwnnxeiboikhnmgg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxeikbynysyrpuxaphevxcgubacqvakb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtsqjigwyzzfsjcdckzjwssxzeuuzljn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lpnrtztczdvuiriujuzdigsgbgoexxjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fldqpxxlnamzxrtpojkizuxghxgmrvix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcaldeypljvxdrtkgmbybrybqkcigsci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpeoeainurabficnroribqfgusobtylb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jtyfdwulqungzbueuvcbfmszimindjcj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjtgtfypywppvbffldeyizxuqfdcmdyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpapacebgoobmwwshzzblvarpxzqcjgm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtmmmsiveleauvljzsbnrslzmnbhdkcw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oyumnadxnecbvklujkirpdochmioqxjf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtyfyfivydmspzoywrdgmcwyjckdeywt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fuzdqjuxrfsdfdmslfsszsjeehbyumdv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zukupgajckjiatmaukoquqetjjjoqtxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkuqreaguqrrvxncmyofrxhugahykomx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ashvmxwpzwqhiscuifahdbkknwawcsob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggmlymcmimgkczbchlseddfsrvxwkdzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ichdelxbvoqtjrezfenmzmfaybdopqvb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bmleyvqawgoguimdahevsrkipstfgxix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oanjffciuwuxxrhyospwiitumygicevd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnzuzohgxdmtysdbwcygedtypypzedeu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bujydoafhvjsuahtesjwubwmxzlriuln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"guprxjosageqfsjjltoguxosomwxsgbb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uynjklvqfopxbmmoinlfjpgitzmoejek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffjvmhojdvmfmlezotvyylbesodeooik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhmmelbebttbeoeuwnlmdjzlmbwwohvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxlwdahfdivcrxkvwztyjtxazbhisgxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opykcuqzqgoydktcgjorkerhgnguendm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzvgpuprpqnpytseumlbppgocbgmrwtm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zljecqtxktcpffdvfjiusryjupoclmbx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xwpoatdpfysnickjjvgbddjfunntazzr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvtabgtqretqndhyxuoxnnratytpwtul","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smowtnqcrgfqseqhblwxejktjqygfsim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"herenvefefsavmmsnlswsfmaligxtyry","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdcetqpavsmtntcjcfldgqkeggltoirt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cezduunyoydfpcuujnyncrwvxniratuj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671256,"databaseName":"models_schema","ddl":"CREATE TABLE `sxmfnbngjuiquecrlxtfqulfkuqmjfzr` (\n `kzujvppmytdfxtivxjiovnlkyeacxsix` int NOT NULL,\n `tawvvmchpnvxmapdalrmrwsvhypyeeal` int DEFAULT NULL,\n `nxnchjpjrhclfekzrhstfrwbcumfupgv` int DEFAULT NULL,\n `bsmulqfrboxfzkuqaexlinydxxosmrux` int DEFAULT NULL,\n `hffsakpkbjilmurffmacjxuapzdhexap` int DEFAULT NULL,\n `wtpgfecnjedbzqhmiwfxxffdagmselri` int DEFAULT NULL,\n `aibsuzebydsntxtyhphhwtfyhylunihd` int DEFAULT NULL,\n `uhnvozylayxmbyuhqbbzhbkbvlhzotmo` int DEFAULT NULL,\n `tlktnwqayvipyxmfxrhhzsfzjaaurnsf` int DEFAULT NULL,\n `wsbhmdmvjphyetfydvirbrjzlhfpayle` int DEFAULT NULL,\n `sbguwerynxbdonnlquyxljnkcbszmpwe` int DEFAULT NULL,\n `dbwldpswwhbhnrqrghmxyqrjquphebvo` int DEFAULT NULL,\n `jikxluqynwixvgmshksndqwnynookgvz` int DEFAULT NULL,\n `tmyejsaantuplkwjeoysusiaiscuqfjq` int DEFAULT NULL,\n `merjbbeweorryniadkoyxkugklehckpb` int DEFAULT NULL,\n `rioyqmxtouskblmdszgeyjsvpdqytuen` int DEFAULT NULL,\n `vhmhrcroajyooulkaqgwejhojyybusbd` int DEFAULT NULL,\n `evntvomnymggdckgfvqcjtlleqoplowk` int DEFAULT NULL,\n `shduojyvstoohavnzuvdqajfdhwxafmm` int DEFAULT NULL,\n `bqvkpmlcufnofpvmfrffxmoombdcsqpj` int DEFAULT NULL,\n `govbsggcdqfosdpqhcwfgzqzrzyavswe` int DEFAULT NULL,\n `iuyecllwtituygxlutrchlygjyumesmc` int DEFAULT NULL,\n `ongmtcpeipnymvxrhbmwshlhuzaursmk` int DEFAULT NULL,\n `zosnwdadgsmtmcmstepqnsrntfmogkvk` int DEFAULT NULL,\n `ywqpjxbwkqghwpvorcgyqfkkngxbknqt` int DEFAULT NULL,\n `wxxajszmanuqfpxtzhovcbdtzbbormdx` int DEFAULT NULL,\n `pzvhohshlihpmshcktcgwvvphkvcfxfn` int DEFAULT NULL,\n `vnjsmqzqcewuaqtwfgmaqpedfwjqusne` int DEFAULT NULL,\n `vnxfllakvfqzgrbafkyolvtzvyfweuge` int DEFAULT NULL,\n `warqlburlbjdxcowodawvkochoapugak` int DEFAULT NULL,\n `pvgdoudcfgjtirdiksusvctraxzgktnz` int DEFAULT NULL,\n `nvtszvbsdjcicpuzexegwmwirmnuutyk` int DEFAULT NULL,\n `wutxpwqzlymqynvtjxqleihqijvpjjmb` int DEFAULT NULL,\n `yllbtlycrnsrzepgysvbnuuudulbnnha` int DEFAULT NULL,\n `rjrndhivestxfzvpqxprkypyhikcebqe` int DEFAULT NULL,\n `prljvluamvwctfxpqfsjemrqskcsxoss` int DEFAULT NULL,\n `rehwakyqvijnqgbfhtsaicczduwbquyh` int DEFAULT NULL,\n `tthlpnjyybssfleihcdwosilocdpfxwd` int DEFAULT NULL,\n `newjdcnrpvsqughadcarbjqdydkzrdwx` int DEFAULT NULL,\n `lrajuhqkehbauqaagpzhohnpvriutyne` int DEFAULT NULL,\n `swqgmcngdaiucnqmmtfilnbjpdqkyvpu` int DEFAULT NULL,\n `ijydardxguiwdgjjxgqhylxdaptjoowg` int DEFAULT NULL,\n `dxrhqnrdbsealvrzqrchibdtvpcsxzpz` int DEFAULT NULL,\n `eglezbqubnorhbomrlfwwgkmpgaajhqx` int DEFAULT NULL,\n `vlrecbnatisnvovggmlleykvtszpkqyo` int DEFAULT NULL,\n `ghkysrxtefxlfvijqmjkqclkkctiqfob` int DEFAULT NULL,\n `rbgknscolzaupjhqtzurovawhfypdcpq` int DEFAULT NULL,\n `djuiylxxoegbifhqrulkrvtonlcjiolk` int DEFAULT NULL,\n `ldgcczvoksonitjhlzeznnrjhjvxjbmg` int DEFAULT NULL,\n `vnabjprnykboosbtqpgwgyzqgmzacaev` int DEFAULT NULL,\n `zjuicxoeevlujadvrupxsxfuxavoqaxs` int DEFAULT NULL,\n `bfhuecmmxcjlpnvbrvlyvawprnebmtya` int DEFAULT NULL,\n `cmccrunitpooogvyxaxaldtmtvsgrnkd` int DEFAULT NULL,\n `nppbozxywzbyowstlzbnanffccwbkolr` int DEFAULT NULL,\n `zsfiuxaexjleaaqcbjypaowsjhspnwhv` int DEFAULT NULL,\n `wzocybjwwlxyoxurezretdvoaykfjvin` int DEFAULT NULL,\n `cuyhekdfvepqpokxsgqmhcuwuhltnwyo` int DEFAULT NULL,\n `cclvhhcrizsaqksvnvljteocjadbgljx` int DEFAULT NULL,\n `ikrhunipbfeizttivlxfgkiygeykttws` int DEFAULT NULL,\n `qmurrcovisbltkoovpwxkfrgtrosaljj` int DEFAULT NULL,\n `wynooyzmusjvqyamyrliibuqahabrqjz` int DEFAULT NULL,\n `ziczcnhuvqtjbmrbeskkqmmaotlbmrnt` int DEFAULT NULL,\n `zxkdqfrqzdxjrsinvvmpupfemeaoffna` int DEFAULT NULL,\n `xtcspuxxebuvuytzvnruynfmxafclogc` int DEFAULT NULL,\n `gefbxkszgzbmxgrwguxlaobkcpbwcldv` int DEFAULT NULL,\n `onvnnijxeagfxjdtsfltptqvljnyddbb` int DEFAULT NULL,\n `jamhmmulwyoxwzyxhqlvkmfoupqxoltj` int DEFAULT NULL,\n `stwgvmlmlstbdzmqkmtupmrycmwdiaqf` int DEFAULT NULL,\n `yuxerjumimcinklkztigdbygbhyfefzi` int DEFAULT NULL,\n `kkktwctpkbqtutxeaonvqnfiogpcykar` int DEFAULT NULL,\n `apbzbtramohdupzgslecdsvsuohrnsiv` int DEFAULT NULL,\n `fhleojiftsashobkzpmsevvwtieopmzc` int DEFAULT NULL,\n `uguavsogjogvisncehkalveufrscbzlz` int DEFAULT NULL,\n `ehgbrhnvhogayndwskamgmsgcfpkwjdg` int DEFAULT NULL,\n `quqxymugiluefdzlbwhetlfkmnhmfkmp` int DEFAULT NULL,\n `dgpnroeegczwnjznhfjhblnkcaphcilf` int DEFAULT NULL,\n `dafonqjymhpcqhoxsdskjimrtrrjjqll` int DEFAULT NULL,\n `zglnxnlqsizlesxqllrykvnlgbcwkizl` int DEFAULT NULL,\n `mdsdggztlgfjexpchiiifqomijlsptei` int DEFAULT NULL,\n `qvfirvhzlyrncejuuukeuzkxdcqhvrng` int DEFAULT NULL,\n `oxgcspoixmaafqrociibfaeosydkglmz` int DEFAULT NULL,\n `kldneklmlgrvrmamulnmctjhreuhublt` int DEFAULT NULL,\n `wpkoehmwwdgdivmflfxngxsoakjsbubs` int DEFAULT NULL,\n `dolqknaeriqcvxombdrphepctkizvpxe` int DEFAULT NULL,\n `neknltlaifhiyyabhmfpqooppzoxkfuz` int DEFAULT NULL,\n `clxtwifpbndrkuqkgxjcgaolxwzofztl` int DEFAULT NULL,\n `carjimwkunyujbpvpxnrgwqhzreuuhci` int DEFAULT NULL,\n `htedrmxtrrsjgwnwxpxnqdbtmbegtxrb` int DEFAULT NULL,\n `uunfqvmaqzyankqwebnnilstschpodbl` int DEFAULT NULL,\n `mykhoaraqanzkgvdyleyjxkiefpdqksl` int DEFAULT NULL,\n `rfadhudmfpqpxekftbktyaiyreoaorbi` int DEFAULT NULL,\n `hyopngtfrssavjfounzihbybkjwvettw` int DEFAULT NULL,\n `gtimlgcrdtpogoctmxmzogajxxedrlhh` int DEFAULT NULL,\n `jifcxtacmsjitagpisjeeutmykwinfbk` int DEFAULT NULL,\n `dhbqrdqgoscskobfjpnbpoflleyztjhw` int DEFAULT NULL,\n `hkboehitrfvphdbmrjrflsqmccwrnmtb` int DEFAULT NULL,\n `dyxcdrglvbauflogyhrpmnhsrebvcjzw` int DEFAULT NULL,\n `alwjzekbuwjzsshvgsrcohmhcajnowhj` int DEFAULT NULL,\n `svstgbmhqyjnznykghjtjmewhjqgtyag` int DEFAULT NULL,\n `uusdebhpkmmjueybwqgqmjgctritgibj` int DEFAULT NULL,\n PRIMARY KEY (`kzujvppmytdfxtivxjiovnlkyeacxsix`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"sxmfnbngjuiquecrlxtfqulfkuqmjfzr\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["kzujvppmytdfxtivxjiovnlkyeacxsix"],"columns":[{"name":"kzujvppmytdfxtivxjiovnlkyeacxsix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"tawvvmchpnvxmapdalrmrwsvhypyeeal","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxnchjpjrhclfekzrhstfrwbcumfupgv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bsmulqfrboxfzkuqaexlinydxxosmrux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hffsakpkbjilmurffmacjxuapzdhexap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtpgfecnjedbzqhmiwfxxffdagmselri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aibsuzebydsntxtyhphhwtfyhylunihd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhnvozylayxmbyuhqbbzhbkbvlhzotmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlktnwqayvipyxmfxrhhzsfzjaaurnsf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wsbhmdmvjphyetfydvirbrjzlhfpayle","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbguwerynxbdonnlquyxljnkcbszmpwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbwldpswwhbhnrqrghmxyqrjquphebvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jikxluqynwixvgmshksndqwnynookgvz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmyejsaantuplkwjeoysusiaiscuqfjq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"merjbbeweorryniadkoyxkugklehckpb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rioyqmxtouskblmdszgeyjsvpdqytuen","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhmhrcroajyooulkaqgwejhojyybusbd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evntvomnymggdckgfvqcjtlleqoplowk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shduojyvstoohavnzuvdqajfdhwxafmm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqvkpmlcufnofpvmfrffxmoombdcsqpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"govbsggcdqfosdpqhcwfgzqzrzyavswe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iuyecllwtituygxlutrchlygjyumesmc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ongmtcpeipnymvxrhbmwshlhuzaursmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zosnwdadgsmtmcmstepqnsrntfmogkvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywqpjxbwkqghwpvorcgyqfkkngxbknqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxxajszmanuqfpxtzhovcbdtzbbormdx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pzvhohshlihpmshcktcgwvvphkvcfxfn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnjsmqzqcewuaqtwfgmaqpedfwjqusne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnxfllakvfqzgrbafkyolvtzvyfweuge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"warqlburlbjdxcowodawvkochoapugak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvgdoudcfgjtirdiksusvctraxzgktnz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvtszvbsdjcicpuzexegwmwirmnuutyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wutxpwqzlymqynvtjxqleihqijvpjjmb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yllbtlycrnsrzepgysvbnuuudulbnnha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjrndhivestxfzvpqxprkypyhikcebqe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prljvluamvwctfxpqfsjemrqskcsxoss","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rehwakyqvijnqgbfhtsaicczduwbquyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tthlpnjyybssfleihcdwosilocdpfxwd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"newjdcnrpvsqughadcarbjqdydkzrdwx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lrajuhqkehbauqaagpzhohnpvriutyne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swqgmcngdaiucnqmmtfilnbjpdqkyvpu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijydardxguiwdgjjxgqhylxdaptjoowg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxrhqnrdbsealvrzqrchibdtvpcsxzpz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eglezbqubnorhbomrlfwwgkmpgaajhqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vlrecbnatisnvovggmlleykvtszpkqyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghkysrxtefxlfvijqmjkqclkkctiqfob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbgknscolzaupjhqtzurovawhfypdcpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"djuiylxxoegbifhqrulkrvtonlcjiolk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ldgcczvoksonitjhlzeznnrjhjvxjbmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnabjprnykboosbtqpgwgyzqgmzacaev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjuicxoeevlujadvrupxsxfuxavoqaxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bfhuecmmxcjlpnvbrvlyvawprnebmtya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmccrunitpooogvyxaxaldtmtvsgrnkd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nppbozxywzbyowstlzbnanffccwbkolr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zsfiuxaexjleaaqcbjypaowsjhspnwhv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzocybjwwlxyoxurezretdvoaykfjvin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cuyhekdfvepqpokxsgqmhcuwuhltnwyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cclvhhcrizsaqksvnvljteocjadbgljx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikrhunipbfeizttivlxfgkiygeykttws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmurrcovisbltkoovpwxkfrgtrosaljj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wynooyzmusjvqyamyrliibuqahabrqjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ziczcnhuvqtjbmrbeskkqmmaotlbmrnt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxkdqfrqzdxjrsinvvmpupfemeaoffna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xtcspuxxebuvuytzvnruynfmxafclogc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gefbxkszgzbmxgrwguxlaobkcpbwcldv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"onvnnijxeagfxjdtsfltptqvljnyddbb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jamhmmulwyoxwzyxhqlvkmfoupqxoltj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"stwgvmlmlstbdzmqkmtupmrycmwdiaqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yuxerjumimcinklkztigdbygbhyfefzi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kkktwctpkbqtutxeaonvqnfiogpcykar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"apbzbtramohdupzgslecdsvsuohrnsiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhleojiftsashobkzpmsevvwtieopmzc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uguavsogjogvisncehkalveufrscbzlz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehgbrhnvhogayndwskamgmsgcfpkwjdg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"quqxymugiluefdzlbwhetlfkmnhmfkmp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgpnroeegczwnjznhfjhblnkcaphcilf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dafonqjymhpcqhoxsdskjimrtrrjjqll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zglnxnlqsizlesxqllrykvnlgbcwkizl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdsdggztlgfjexpchiiifqomijlsptei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvfirvhzlyrncejuuukeuzkxdcqhvrng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oxgcspoixmaafqrociibfaeosydkglmz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kldneklmlgrvrmamulnmctjhreuhublt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wpkoehmwwdgdivmflfxngxsoakjsbubs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dolqknaeriqcvxombdrphepctkizvpxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"neknltlaifhiyyabhmfpqooppzoxkfuz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clxtwifpbndrkuqkgxjcgaolxwzofztl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"carjimwkunyujbpvpxnrgwqhzreuuhci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htedrmxtrrsjgwnwxpxnqdbtmbegtxrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uunfqvmaqzyankqwebnnilstschpodbl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mykhoaraqanzkgvdyleyjxkiefpdqksl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfadhudmfpqpxekftbktyaiyreoaorbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hyopngtfrssavjfounzihbybkjwvettw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtimlgcrdtpogoctmxmzogajxxedrlhh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jifcxtacmsjitagpisjeeutmykwinfbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhbqrdqgoscskobfjpnbpoflleyztjhw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkboehitrfvphdbmrjrflsqmccwrnmtb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dyxcdrglvbauflogyhrpmnhsrebvcjzw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"alwjzekbuwjzsshvgsrcohmhcajnowhj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"svstgbmhqyjnznykghjtjmewhjqgtyag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uusdebhpkmmjueybwqgqmjgctritgibj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671288,"databaseName":"models_schema","ddl":"CREATE TABLE `szptihpdgatfskwdlqlvratsglyrqnga` (\n `vnywgdcnwgjzalxesopdcvxycjgffzmb` int NOT NULL,\n `ctqsbctwjvspqjmxnyqxpcdiponllvbm` int DEFAULT NULL,\n `agrynwntfwqypoxufzxalfgvytcajesm` int DEFAULT NULL,\n `wggwliqxofzfexcslvswsflntubjorcm` int DEFAULT NULL,\n `rwxgtpygrclalrnsisfxhnzijfhoxrma` int DEFAULT NULL,\n `mbaqrdqrpbkreoqoggrnymnpclxfveja` int DEFAULT NULL,\n `itxdjpmcoeonfgbbxcwoxnzvuteiptvl` int DEFAULT NULL,\n `zuiosnffzlgbkyaqzsjnfiswgmyocjsa` int DEFAULT NULL,\n `rcwtpucqqixxeccmqpeolqyetdvnbxrb` int DEFAULT NULL,\n `xbhmmtgahqpxhyzuqqudvkvtoruynpco` int DEFAULT NULL,\n `xxzwkxxumnigggeajijxhnamdwstazhk` int DEFAULT NULL,\n `ygcjynrrrvjuptzhcdtbovyiiumcjhdz` int DEFAULT NULL,\n `cykqkiiktubhwijteqxkowzhpapvjzxr` int DEFAULT NULL,\n `yvvaesysdduydrmbssajlybxcaxzzuww` int DEFAULT NULL,\n `rjlappukwzgsyxnorxqnrpglwuhirgjo` int DEFAULT NULL,\n `xfvdfuknhvwlfkpuqfkhaldyawahxyfz` int DEFAULT NULL,\n `vztdligjeoiqolccdvbsnfkwcmmyuatw` int DEFAULT NULL,\n `paxpcdoaxnvqzgsdtorfhvefhxotlmpk` int DEFAULT NULL,\n `jxkoqxmgrfabnmzuadxlizucneigyyks` int DEFAULT NULL,\n `obrkeqnzvrxanhueynblxnhdbpooillo` int DEFAULT NULL,\n `kypcivlngcozvazbaxmvnyafbnqwnemn` int DEFAULT NULL,\n `znpeztjxsdqpckpqchdxmwjwjbyvokfy` int DEFAULT NULL,\n `jgyfhxlwzqcpcycfbkhlkdwkartnflxy` int DEFAULT NULL,\n `tlgzmbihwbxehfolwmyrbtckhlnmerne` int DEFAULT NULL,\n `dfwpykczczoniwhukltkkhxbaoxxfdoa` int DEFAULT NULL,\n `qflueemzbryzycnkgggutgpdfmhkohox` int DEFAULT NULL,\n `dxfguplhsqfhcryftleedmavaqwnxvuj` int DEFAULT NULL,\n `uarwdwhwpatomhtpomcbehxjzatrzequ` int DEFAULT NULL,\n `tshpjbzcnicpgiaxfdlafczyuoixueii` int DEFAULT NULL,\n `cvciivqpepbmpiuwvhhgvqfslhnimqyf` int DEFAULT NULL,\n `eaczhxzryckpifwhlgjyfszdrainwqhf` int DEFAULT NULL,\n `gmhxympzmnwebpaafqqrqggcrajwxcwh` int DEFAULT NULL,\n `lcdeiyromtadvmozrlobhfmfmynismuu` int DEFAULT NULL,\n `hftyveldhkteutzvttvwgswtkdvspbfk` int DEFAULT NULL,\n `nldmwetuqrzrmdonhbwisabupzsoppnt` int DEFAULT NULL,\n `glsquyszdkkffwisblcbatayllinuqjg` int DEFAULT NULL,\n `nsjetfrugyuecmcscwmtzbbgdiidxwlo` int DEFAULT NULL,\n `dogvhsmdnnvpbfsauobjklkwlwkvpany` int DEFAULT NULL,\n `ztpihxzeopcqzbvuoxhaijgvtxliqnpd` int DEFAULT NULL,\n `sfkzilogamtlmntcloyjrderadttztdl` int DEFAULT NULL,\n `jvytowpectlddbdhkkkygzroviwbrdst` int DEFAULT NULL,\n `ynkypztbjqfyyltssxxvzfhgadlroqej` int DEFAULT NULL,\n `wfamjedsofbkdsmpxiznlerhybbgdrlx` int DEFAULT NULL,\n `cvvdjcaegilgnjyzugzyeszuoazfwqnk` int DEFAULT NULL,\n `tcppppssglqldtyfuuamcgquojuvydks` int DEFAULT NULL,\n `dkgsfpbqonpxktalytewlfugblomxcon` int DEFAULT NULL,\n `uzkfutoylumhjztbhufnzyiacerhvnvu` int DEFAULT NULL,\n `ekuujqsbvedhiqtnfqzavxefgbcazsrn` int DEFAULT NULL,\n `zppstwwqctqzebjjiqlutltlnaprarez` int DEFAULT NULL,\n `utertmlfniglfbakcinbivueuzknklxs` int DEFAULT NULL,\n `tlhpwfsgcprregbqqpynyouepfyekolm` int DEFAULT NULL,\n `llwnzgmfhmntscwrprzljxqflfhjelfc` int DEFAULT NULL,\n `hutoahlqaiboddijiotmaciezniuzoki` int DEFAULT NULL,\n `pceahsyqqlewryzwfmjfvojeohpukufu` int DEFAULT NULL,\n `claargdjuunjtxbwikgzyvcbpvljokcc` int DEFAULT NULL,\n `boitwmjmrjdabapvemvaforvgsgatlnk` int DEFAULT NULL,\n `skcroxmeyufwjdudiccrneuzslmpxlft` int DEFAULT NULL,\n `xyeqkvokzrvedniwxbovckelqozsqtlk` int DEFAULT NULL,\n `mnmkisxdxpjrjucbumvgzvjcdghzesfs` int DEFAULT NULL,\n `jgbknxdeawclnbrqjvhpmnplthpybbus` int DEFAULT NULL,\n `akvlpwbekxrxwuxixwpqprpkwigngoks` int DEFAULT NULL,\n `orscjathlumywcxlskjhhulbiajphnvi` int DEFAULT NULL,\n `fqvjrbsqghkiygfokurhesygomotlklu` int DEFAULT NULL,\n `szipudkpbpkvxdxgefklqtjjlzxsjyht` int DEFAULT NULL,\n `xyvkioeomkydznixyucalejiwhtyyomw` int DEFAULT NULL,\n `rhtvxeqblmsfisiejdvybwmxuatqfqvg` int DEFAULT NULL,\n `yirmojayuflzcpmgdacrbrxemllehncp` int DEFAULT NULL,\n `rpktgduyylkccezojcplnapyzqfxwhbp` int DEFAULT NULL,\n `jbqjpnfznrnymacjkhtaiybzcwucwaqv` int DEFAULT NULL,\n `ubtdndzmqtxxmqvpgdrbdiihhtadgfbc` int DEFAULT NULL,\n `uwnfmyiqtzmtkyizgutkbpillkjwobej` int DEFAULT NULL,\n `qvkalimzvlsomzhminnyrilqsuooflpq` int DEFAULT NULL,\n `lbxreskfqnxtxrsvrvjcvkxpdozxwxqo` int DEFAULT NULL,\n `cxbfwaotcngveroahzbtbedpollxtrsy` int DEFAULT NULL,\n `hcbwxymmtjlpkxplxftxexfvwhstwrdu` int DEFAULT NULL,\n `dvlscltkehtgymgpvwtdrzblkiryhjip` int DEFAULT NULL,\n `ldtsncvpxglajtakhqeykqrvieatntie` int DEFAULT NULL,\n `lbbhqnqdxcvbzqmpnytyfkqzbyhtgucj` int DEFAULT NULL,\n `mrgrlhfzckawaxvuhwrsluwrwgzepfdp` int DEFAULT NULL,\n `qyryzjzkjsgshskawuyednwissimepcz` int DEFAULT NULL,\n `nbctvsbdlrnlnzbezgrpxvwtvpewlquw` int DEFAULT NULL,\n `nypvrmfbksqiduzfnspuyxacoorzzdia` int DEFAULT NULL,\n `jxrhkcomjyizcdwpsbnscbfcafrkboys` int DEFAULT NULL,\n `akfweskogssenujwfotaipwhhmpcjuda` int DEFAULT NULL,\n `bozvotlaxiwcvfsjgodofzhjtxspprki` int DEFAULT NULL,\n `zbefkipeiztpwvzpmpwkwcjbfyjirqsd` int DEFAULT NULL,\n `bggmvgdpsoshcdkcumuoklmeazxgnoxq` int DEFAULT NULL,\n `eoykipuyolxildggchayqdphmgsygarv` int DEFAULT NULL,\n `pjeuwieeyqvlloofdcrzjcczcznllpnf` int DEFAULT NULL,\n `eqkbxtcjyftsctucnlecxniobichapkm` int DEFAULT NULL,\n `taexulnrrwqvkmnsitpcanwvifnokmoe` int DEFAULT NULL,\n `qsvikoulwdmbibbxvyeevbjhshcyghei` int DEFAULT NULL,\n `agfhmuveaezggjgeizuiycqozelimekq` int DEFAULT NULL,\n `huaojdvrchmvzibuugjvtypnhygybine` int DEFAULT NULL,\n `frcdirxtkysmzyatmcflusybtaheufse` int DEFAULT NULL,\n `iahzijmigvesaerqbfpolbxucrhdyvyw` int DEFAULT NULL,\n `omnhufvuutabyhkhpawiavsdqkjjzdby` int DEFAULT NULL,\n `xzhdofjqhmdnpsitoogxgwfcszoqzygq` int DEFAULT NULL,\n `gzngzozmtfdtpmdbdzseqqcpfgfivegu` int DEFAULT NULL,\n `touwrqkjhgnkicustkpfchrojceovufp` int DEFAULT NULL,\n PRIMARY KEY (`vnywgdcnwgjzalxesopdcvxycjgffzmb`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"szptihpdgatfskwdlqlvratsglyrqnga\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["vnywgdcnwgjzalxesopdcvxycjgffzmb"],"columns":[{"name":"vnywgdcnwgjzalxesopdcvxycjgffzmb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ctqsbctwjvspqjmxnyqxpcdiponllvbm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agrynwntfwqypoxufzxalfgvytcajesm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wggwliqxofzfexcslvswsflntubjorcm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rwxgtpygrclalrnsisfxhnzijfhoxrma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbaqrdqrpbkreoqoggrnymnpclxfveja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itxdjpmcoeonfgbbxcwoxnzvuteiptvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zuiosnffzlgbkyaqzsjnfiswgmyocjsa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcwtpucqqixxeccmqpeolqyetdvnbxrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbhmmtgahqpxhyzuqqudvkvtoruynpco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxzwkxxumnigggeajijxhnamdwstazhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ygcjynrrrvjuptzhcdtbovyiiumcjhdz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cykqkiiktubhwijteqxkowzhpapvjzxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yvvaesysdduydrmbssajlybxcaxzzuww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjlappukwzgsyxnorxqnrpglwuhirgjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xfvdfuknhvwlfkpuqfkhaldyawahxyfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vztdligjeoiqolccdvbsnfkwcmmyuatw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"paxpcdoaxnvqzgsdtorfhvefhxotlmpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jxkoqxmgrfabnmzuadxlizucneigyyks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obrkeqnzvrxanhueynblxnhdbpooillo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kypcivlngcozvazbaxmvnyafbnqwnemn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znpeztjxsdqpckpqchdxmwjwjbyvokfy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgyfhxlwzqcpcycfbkhlkdwkartnflxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlgzmbihwbxehfolwmyrbtckhlnmerne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dfwpykczczoniwhukltkkhxbaoxxfdoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qflueemzbryzycnkgggutgpdfmhkohox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxfguplhsqfhcryftleedmavaqwnxvuj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uarwdwhwpatomhtpomcbehxjzatrzequ","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tshpjbzcnicpgiaxfdlafczyuoixueii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvciivqpepbmpiuwvhhgvqfslhnimqyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eaczhxzryckpifwhlgjyfszdrainwqhf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmhxympzmnwebpaafqqrqggcrajwxcwh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lcdeiyromtadvmozrlobhfmfmynismuu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hftyveldhkteutzvttvwgswtkdvspbfk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nldmwetuqrzrmdonhbwisabupzsoppnt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"glsquyszdkkffwisblcbatayllinuqjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nsjetfrugyuecmcscwmtzbbgdiidxwlo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dogvhsmdnnvpbfsauobjklkwlwkvpany","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ztpihxzeopcqzbvuoxhaijgvtxliqnpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfkzilogamtlmntcloyjrderadttztdl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvytowpectlddbdhkkkygzroviwbrdst","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynkypztbjqfyyltssxxvzfhgadlroqej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfamjedsofbkdsmpxiznlerhybbgdrlx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvvdjcaegilgnjyzugzyeszuoazfwqnk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tcppppssglqldtyfuuamcgquojuvydks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkgsfpbqonpxktalytewlfugblomxcon","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzkfutoylumhjztbhufnzyiacerhvnvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekuujqsbvedhiqtnfqzavxefgbcazsrn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zppstwwqctqzebjjiqlutltlnaprarez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utertmlfniglfbakcinbivueuzknklxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlhpwfsgcprregbqqpynyouepfyekolm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llwnzgmfhmntscwrprzljxqflfhjelfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hutoahlqaiboddijiotmaciezniuzoki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pceahsyqqlewryzwfmjfvojeohpukufu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"claargdjuunjtxbwikgzyvcbpvljokcc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"boitwmjmrjdabapvemvaforvgsgatlnk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skcroxmeyufwjdudiccrneuzslmpxlft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyeqkvokzrvedniwxbovckelqozsqtlk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnmkisxdxpjrjucbumvgzvjcdghzesfs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgbknxdeawclnbrqjvhpmnplthpybbus","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akvlpwbekxrxwuxixwpqprpkwigngoks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orscjathlumywcxlskjhhulbiajphnvi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqvjrbsqghkiygfokurhesygomotlklu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"szipudkpbpkvxdxgefklqtjjlzxsjyht","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyvkioeomkydznixyucalejiwhtyyomw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhtvxeqblmsfisiejdvybwmxuatqfqvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yirmojayuflzcpmgdacrbrxemllehncp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpktgduyylkccezojcplnapyzqfxwhbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbqjpnfznrnymacjkhtaiybzcwucwaqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubtdndzmqtxxmqvpgdrbdiihhtadgfbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwnfmyiqtzmtkyizgutkbpillkjwobej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvkalimzvlsomzhminnyrilqsuooflpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lbxreskfqnxtxrsvrvjcvkxpdozxwxqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cxbfwaotcngveroahzbtbedpollxtrsy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hcbwxymmtjlpkxplxftxexfvwhstwrdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvlscltkehtgymgpvwtdrzblkiryhjip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ldtsncvpxglajtakhqeykqrvieatntie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lbbhqnqdxcvbzqmpnytyfkqzbyhtgucj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrgrlhfzckawaxvuhwrsluwrwgzepfdp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qyryzjzkjsgshskawuyednwissimepcz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbctvsbdlrnlnzbezgrpxvwtvpewlquw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nypvrmfbksqiduzfnspuyxacoorzzdia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jxrhkcomjyizcdwpsbnscbfcafrkboys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akfweskogssenujwfotaipwhhmpcjuda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bozvotlaxiwcvfsjgodofzhjtxspprki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbefkipeiztpwvzpmpwkwcjbfyjirqsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bggmvgdpsoshcdkcumuoklmeazxgnoxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eoykipuyolxildggchayqdphmgsygarv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjeuwieeyqvlloofdcrzjcczcznllpnf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqkbxtcjyftsctucnlecxniobichapkm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"taexulnrrwqvkmnsitpcanwvifnokmoe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qsvikoulwdmbibbxvyeevbjhshcyghei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agfhmuveaezggjgeizuiycqozelimekq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"huaojdvrchmvzibuugjvtypnhygybine","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"frcdirxtkysmzyatmcflusybtaheufse","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iahzijmigvesaerqbfpolbxucrhdyvyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omnhufvuutabyhkhpawiavsdqkjjzdby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xzhdofjqhmdnpsitoogxgwfcszoqzygq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzngzozmtfdtpmdbdzseqqcpfgfivegu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"touwrqkjhgnkicustkpfchrojceovufp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671320,"databaseName":"models_schema","ddl":"CREATE TABLE `tcrradlvorgzjcciygegyfvojodcfdhl` (\n `urambtgjgdcahfjjvowsvgwrwinrshxb` int NOT NULL,\n `fzsniozazdwmmnwtflffiupounbqdjnu` int DEFAULT NULL,\n `wgswzxduxcxxfibirhthoedklalbvgdy` int DEFAULT NULL,\n `oczgubostlndwtpuohrpvnfqknhlbhon` int DEFAULT NULL,\n `uhiisaoyxllwooqogpqbvyqtxlhzsjea` int DEFAULT NULL,\n `osikjddrmbsuyjserzpsbyluopcbkzlp` int DEFAULT NULL,\n `qakupjbftjzuphfalaerjjgemaqgxzly` int DEFAULT NULL,\n `wzoevrprrnutxtikqiefrerydpijnwbn` int DEFAULT NULL,\n `kwgenzcylmzzykylgpjauyzyihgmgmay` int DEFAULT NULL,\n `fysdqtrnxdwawrfailmfbjgcdndargcs` int DEFAULT NULL,\n `jpbzhabjjcyfftrckkviugewhasxknhm` int DEFAULT NULL,\n `spfqbvzjcnyqwvikcwydoxxqfjmjahxg` int DEFAULT NULL,\n `dlhxlnpjueuhvtwmjwkwdkvtxmhxgfwe` int DEFAULT NULL,\n `prywkdddumcmfsdtijfdzjsbvmllecnb` int DEFAULT NULL,\n `kosclooiximyyuiikjcueojjxkzyftha` int DEFAULT NULL,\n `opxkndadjsdzrfdvzzpakesrlegiccyt` int DEFAULT NULL,\n `tywjvzgbljwobenszanbvecznsrtjasn` int DEFAULT NULL,\n `rtzorswcilxerqamhsoyjhtfzvutjfap` int DEFAULT NULL,\n `pjcrevjhrknnxldnlnzymfocldaegvwi` int DEFAULT NULL,\n `ddzeonjaaavwmywndywcxawewomkzyyy` int DEFAULT NULL,\n `jmnmonlltgpwhtuzmlucecbskmhotxay` int DEFAULT NULL,\n `tpzqvsrnbljirmcbhpmuwrbwbnlczwga` int DEFAULT NULL,\n `nrkqddgqidlhwpgrnejrwdthuxpnqnst` int DEFAULT NULL,\n `qgluxjhlfqiobktnwctazarvpwzggwrs` int DEFAULT NULL,\n `upvkjurjcvffklmxxykylmvvjscrpfsd` int DEFAULT NULL,\n `ruijznarjeuortvcjmvrzijexvbkbiks` int DEFAULT NULL,\n `zzrdxsyrdkbyrgewryjthfnzptmpmvzr` int DEFAULT NULL,\n `wefwicuzggtjijpddrkwfebiwtqatoyb` int DEFAULT NULL,\n `folfobqirankyihgicbpayknzbxzpmet` int DEFAULT NULL,\n `yzludzfenlgoeuopjqhnjlufhskwmmks` int DEFAULT NULL,\n `taewkyhavbhxiejezphlixtrlgnogppv` int DEFAULT NULL,\n `uzjdfbxpomdgzxtjiigcyiyyubwwtnfb` int DEFAULT NULL,\n `jhakcmsbuxeqqbjlzcezppkcqrkwjwzl` int DEFAULT NULL,\n `ykesojosllycsybhgeannyjihicgrpdy` int DEFAULT NULL,\n `ysitsdexeamqeexyhkaxmktxognotkcb` int DEFAULT NULL,\n `dwurmhowzytpxuzbqwiwiouqfxpulgcv` int DEFAULT NULL,\n `xuerihapfwnigaipcmbkeasskqhyteox` int DEFAULT NULL,\n `iehaxencrfctenpbhkpnjwfqkuisukkc` int DEFAULT NULL,\n `pucdtmenindgmcavowuiegvgkevwfrki` int DEFAULT NULL,\n `sgndgcmiycrqrszxmskkcbcfnxmkzcpy` int DEFAULT NULL,\n `xmvnnliygljitjtahivrcljkprpynpnm` int DEFAULT NULL,\n `yrrmijgejczteqwywerwtfnqoksbxqqc` int DEFAULT NULL,\n `iagtqeifbkusrcyybvrwimwiyzvdvfqo` int DEFAULT NULL,\n `lzgkvxuixrnseljthstndibvighklqlz` int DEFAULT NULL,\n `qymckdaepejbwxmxbxfmityhsgdkjibv` int DEFAULT NULL,\n `jzpuoaywobfbyxgxifrfneilfcdsqwnw` int DEFAULT NULL,\n `yxnogztfhnkbqdbwllizzddeglxujpeo` int DEFAULT NULL,\n `gzqlyclclsyjdsnwmcphaglzkiqskawi` int DEFAULT NULL,\n `fqlzpzigkjqzoakvuajbsedcbjzkezfa` int DEFAULT NULL,\n `ottqovmdpwkioavfbrrdfhyaigcpuisp` int DEFAULT NULL,\n `lqrgtumdbvfstxelpvzhnekubbbdsjua` int DEFAULT NULL,\n `uclbprzmhenzudbwctnjjchdqmdmsuon` int DEFAULT NULL,\n `tqmcaniidahkiqivpiihgayasudxxyqr` int DEFAULT NULL,\n `sgheeccszsxalkdrqfnquxjtyxcoemaj` int DEFAULT NULL,\n `lqzyxmttaaixzuzfhhpuctixcbjhyugi` int DEFAULT NULL,\n `cxmcddfumxgbvfserevqxdjvejthozeo` int DEFAULT NULL,\n `bczuayxcgscqfoltfilwabeijsngowwe` int DEFAULT NULL,\n `aqrbqbjxitvohnhdimqpynkqvywbpjvs` int DEFAULT NULL,\n `zqwbhzoebpuqtbmwgkvsbbcrrkhlnfox` int DEFAULT NULL,\n `awvicvajtzotdlzyshychxglfyafzmfp` int DEFAULT NULL,\n `wfbklpgzxsnptmzqdnbssazzlldxeteb` int DEFAULT NULL,\n `pspoyjnjtmsqzpyawobzddifmfvuqafn` int DEFAULT NULL,\n `jlwxwymonazqyjrmiqskjtucxyurrhou` int DEFAULT NULL,\n `dziyupmfgitfnbfrfbubfsopzluhxxjt` int DEFAULT NULL,\n `meiknqxbrbuqeahtyhvmrnrubqqedcjp` int DEFAULT NULL,\n `xhybnjxahukmakmkvlfgvcquectdpwpn` int DEFAULT NULL,\n `pubpqsdqlfedrgymcgaukmikxdykgmcy` int DEFAULT NULL,\n `vgnabzopyiwgdtrohzgeofxvlnjixzni` int DEFAULT NULL,\n `gbnqaqkzioikeknhfcczhxpiwaxsajtm` int DEFAULT NULL,\n `bvvdfeymobnuoutlzktwdozxpotxbply` int DEFAULT NULL,\n `xkgzgkuwozatkqxzofqjujfmwjfpckyc` int DEFAULT NULL,\n `ezawspmtbagrargowgefiqjkuqtcroyv` int DEFAULT NULL,\n `ysaffkkixianifgdvihvfcajxhseqzww` int DEFAULT NULL,\n `plmnteftzzzzjrbmwjqljfvvggsdeado` int DEFAULT NULL,\n `wqfkccfbemgeodgvzqbczaoruojbcwwd` int DEFAULT NULL,\n `taleshpcnghgiemyjmmhwdxkcsbaxnpm` int DEFAULT NULL,\n `wuwfvumlzrmtkzpqwgelsalcqktssvbu` int DEFAULT NULL,\n `mtgtlepklpkoslsesgxkdcfsbbpvkemw` int DEFAULT NULL,\n `lsakybhabeuljdkyabyctuuciicqxgfo` int DEFAULT NULL,\n `fssfniwhdlxibqdvridzbdxogevpwfhv` int DEFAULT NULL,\n `bylnbdgcdlpxnsadnhjmuknkmhhyxycv` int DEFAULT NULL,\n `rzyekjcpmwnmebomaxkcqcvmdtczfphx` int DEFAULT NULL,\n `skomlgwakvlhcpkblwwqtthujqzivtfu` int DEFAULT NULL,\n `bgnlsayzlwmgdecedmfefpybufpotqqe` int DEFAULT NULL,\n `sikxydncajtdavttdvglyroggttickee` int DEFAULT NULL,\n `sdnurfphtdrgtgotzgpcbaudydmsngsw` int DEFAULT NULL,\n `ykaaxqpbhwgvimwacapscyxhmxjnntvv` int DEFAULT NULL,\n `rbxtyiwwzomrwqinqhccyepxnrprpapn` int DEFAULT NULL,\n `xnuajzzbkfcjoqzubtvfltqdffaqrmqe` int DEFAULT NULL,\n `zkynelgipsixjtpimccvwlrsoqrsvfxc` int DEFAULT NULL,\n `dynpdbybdiuhtealrptdihdkgoxievwv` int DEFAULT NULL,\n `tpnfxbzwvrcbbsiotziknlgqpcwderku` int DEFAULT NULL,\n `znhawteflcpkybuaozoyebwtnqmagjuu` int DEFAULT NULL,\n `howuimnjosndqihmdzgynpgtoijhfexs` int DEFAULT NULL,\n `seaaxxkxgpeeqlgbtnqoliabcklbsugj` int DEFAULT NULL,\n `etfaqiyfmcgvffktnygdvhtjehhaudpj` int DEFAULT NULL,\n `uxuidfhsvfektudicrlxupcstmhmjcjo` int DEFAULT NULL,\n `onrykpvlgqrvinyexjmncpqxpjytuvns` int DEFAULT NULL,\n `nmtdtlfswhazikjyjgnhsmgxaitgzuxk` int DEFAULT NULL,\n `avsqxiqvaffnwpwfxjsspzurwdmdmzzy` int DEFAULT NULL,\n PRIMARY KEY (`urambtgjgdcahfjjvowsvgwrwinrshxb`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"tcrradlvorgzjcciygegyfvojodcfdhl\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["urambtgjgdcahfjjvowsvgwrwinrshxb"],"columns":[{"name":"urambtgjgdcahfjjvowsvgwrwinrshxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"fzsniozazdwmmnwtflffiupounbqdjnu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgswzxduxcxxfibirhthoedklalbvgdy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oczgubostlndwtpuohrpvnfqknhlbhon","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhiisaoyxllwooqogpqbvyqtxlhzsjea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"osikjddrmbsuyjserzpsbyluopcbkzlp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qakupjbftjzuphfalaerjjgemaqgxzly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzoevrprrnutxtikqiefrerydpijnwbn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwgenzcylmzzykylgpjauyzyihgmgmay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fysdqtrnxdwawrfailmfbjgcdndargcs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpbzhabjjcyfftrckkviugewhasxknhm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"spfqbvzjcnyqwvikcwydoxxqfjmjahxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlhxlnpjueuhvtwmjwkwdkvtxmhxgfwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prywkdddumcmfsdtijfdzjsbvmllecnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kosclooiximyyuiikjcueojjxkzyftha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opxkndadjsdzrfdvzzpakesrlegiccyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tywjvzgbljwobenszanbvecznsrtjasn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtzorswcilxerqamhsoyjhtfzvutjfap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjcrevjhrknnxldnlnzymfocldaegvwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddzeonjaaavwmywndywcxawewomkzyyy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmnmonlltgpwhtuzmlucecbskmhotxay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpzqvsrnbljirmcbhpmuwrbwbnlczwga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nrkqddgqidlhwpgrnejrwdthuxpnqnst","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgluxjhlfqiobktnwctazarvpwzggwrs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upvkjurjcvffklmxxykylmvvjscrpfsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ruijznarjeuortvcjmvrzijexvbkbiks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzrdxsyrdkbyrgewryjthfnzptmpmvzr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wefwicuzggtjijpddrkwfebiwtqatoyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"folfobqirankyihgicbpayknzbxzpmet","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzludzfenlgoeuopjqhnjlufhskwmmks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"taewkyhavbhxiejezphlixtrlgnogppv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzjdfbxpomdgzxtjiigcyiyyubwwtnfb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jhakcmsbuxeqqbjlzcezppkcqrkwjwzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykesojosllycsybhgeannyjihicgrpdy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ysitsdexeamqeexyhkaxmktxognotkcb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwurmhowzytpxuzbqwiwiouqfxpulgcv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xuerihapfwnigaipcmbkeasskqhyteox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iehaxencrfctenpbhkpnjwfqkuisukkc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pucdtmenindgmcavowuiegvgkevwfrki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sgndgcmiycrqrszxmskkcbcfnxmkzcpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmvnnliygljitjtahivrcljkprpynpnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yrrmijgejczteqwywerwtfnqoksbxqqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iagtqeifbkusrcyybvrwimwiyzvdvfqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzgkvxuixrnseljthstndibvighklqlz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qymckdaepejbwxmxbxfmityhsgdkjibv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzpuoaywobfbyxgxifrfneilfcdsqwnw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxnogztfhnkbqdbwllizzddeglxujpeo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzqlyclclsyjdsnwmcphaglzkiqskawi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqlzpzigkjqzoakvuajbsedcbjzkezfa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ottqovmdpwkioavfbrrdfhyaigcpuisp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqrgtumdbvfstxelpvzhnekubbbdsjua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uclbprzmhenzudbwctnjjchdqmdmsuon","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqmcaniidahkiqivpiihgayasudxxyqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sgheeccszsxalkdrqfnquxjtyxcoemaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqzyxmttaaixzuzfhhpuctixcbjhyugi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cxmcddfumxgbvfserevqxdjvejthozeo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bczuayxcgscqfoltfilwabeijsngowwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqrbqbjxitvohnhdimqpynkqvywbpjvs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqwbhzoebpuqtbmwgkvsbbcrrkhlnfox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awvicvajtzotdlzyshychxglfyafzmfp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfbklpgzxsnptmzqdnbssazzlldxeteb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pspoyjnjtmsqzpyawobzddifmfvuqafn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlwxwymonazqyjrmiqskjtucxyurrhou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dziyupmfgitfnbfrfbubfsopzluhxxjt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"meiknqxbrbuqeahtyhvmrnrubqqedcjp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xhybnjxahukmakmkvlfgvcquectdpwpn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pubpqsdqlfedrgymcgaukmikxdykgmcy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgnabzopyiwgdtrohzgeofxvlnjixzni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gbnqaqkzioikeknhfcczhxpiwaxsajtm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvvdfeymobnuoutlzktwdozxpotxbply","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkgzgkuwozatkqxzofqjujfmwjfpckyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezawspmtbagrargowgefiqjkuqtcroyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ysaffkkixianifgdvihvfcajxhseqzww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"plmnteftzzzzjrbmwjqljfvvggsdeado","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqfkccfbemgeodgvzqbczaoruojbcwwd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"taleshpcnghgiemyjmmhwdxkcsbaxnpm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wuwfvumlzrmtkzpqwgelsalcqktssvbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtgtlepklpkoslsesgxkdcfsbbpvkemw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lsakybhabeuljdkyabyctuuciicqxgfo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fssfniwhdlxibqdvridzbdxogevpwfhv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bylnbdgcdlpxnsadnhjmuknkmhhyxycv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzyekjcpmwnmebomaxkcqcvmdtczfphx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skomlgwakvlhcpkblwwqtthujqzivtfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bgnlsayzlwmgdecedmfefpybufpotqqe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sikxydncajtdavttdvglyroggttickee","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdnurfphtdrgtgotzgpcbaudydmsngsw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykaaxqpbhwgvimwacapscyxhmxjnntvv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbxtyiwwzomrwqinqhccyepxnrprpapn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnuajzzbkfcjoqzubtvfltqdffaqrmqe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zkynelgipsixjtpimccvwlrsoqrsvfxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dynpdbybdiuhtealrptdihdkgoxievwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpnfxbzwvrcbbsiotziknlgqpcwderku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znhawteflcpkybuaozoyebwtnqmagjuu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"howuimnjosndqihmdzgynpgtoijhfexs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"seaaxxkxgpeeqlgbtnqoliabcklbsugj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"etfaqiyfmcgvffktnygdvhtjehhaudpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxuidfhsvfektudicrlxupcstmhmjcjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"onrykpvlgqrvinyexjmncpqxpjytuvns","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmtdtlfswhazikjyjgnhsmgxaitgzuxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avsqxiqvaffnwpwfxjsspzurwdmdmzzy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671352,"databaseName":"models_schema","ddl":"CREATE TABLE `tfoaqprarfoabbffxqiypuqnjuzesvmg` (\n `itefojdnavbgbuqpkstlsbwkrposdmwq` int NOT NULL,\n `garzrcvmospocqitmebwjukupcwkkzjp` int DEFAULT NULL,\n `vaxhadgltqvuxnqsyjwbrkwkhmtzhcah` int DEFAULT NULL,\n `igcdeepmsprrngvshawiusblsbyimlbm` int DEFAULT NULL,\n `argnupalshsodhnxmumuhmusrszoxvlh` int DEFAULT NULL,\n `betjmxxktliqftnipmqxtsgjxpbvssab` int DEFAULT NULL,\n `dwsswrvbshqdlvojazjktetsvqllpqhe` int DEFAULT NULL,\n `rxhujgrjennjiydqbbcmrqyebpbuiwbr` int DEFAULT NULL,\n `hwgfnjbxhukjpjgkglzoocryvkbxhkte` int DEFAULT NULL,\n `lgjevubsebxjatzuuggtclgvnntlzswh` int DEFAULT NULL,\n `wyzordnzvslqibqywnpvbctypcgwoveo` int DEFAULT NULL,\n `vybvnerrmgnfbajruupowykvwqwrufbt` int DEFAULT NULL,\n `qwvnuerfyiiiuskvormckmrvjauqhupg` int DEFAULT NULL,\n `jykgyfhrhdbrwabdxtbaekpdmrrztyhj` int DEFAULT NULL,\n `zblrvedmceniszjmhfrohmjeyifqchae` int DEFAULT NULL,\n `ublhfvlispnratgvpvfrugntizheefze` int DEFAULT NULL,\n `kbcyqwvqfswjpthxkywjpwwkpickofsm` int DEFAULT NULL,\n `rwcrabzaejecylphjfaiyatdibufxqrr` int DEFAULT NULL,\n `sjphdbhsbnxnsgdanabhhuisxwsmqlfv` int DEFAULT NULL,\n `uhyzwqjrlxaydgkhdhcxllfssquqcxbq` int DEFAULT NULL,\n `drpunsdvjhilqoeyzpkcpnbrwkrgvcyo` int DEFAULT NULL,\n `egqlveyrubndinrjosadpwdmlbbpqycy` int DEFAULT NULL,\n `ttecplpdgvmsmacbfcpzzaiupvprjtef` int DEFAULT NULL,\n `beihgqzjleorgjtmfjplxoelpuxzzxfa` int DEFAULT NULL,\n `fcqeduzkruvkmjesazdvgwemuoffaiyv` int DEFAULT NULL,\n `nfxoynjnimkwazheykjviguinntubstx` int DEFAULT NULL,\n `bhjhbmroinrswhvjrsplgtizxznbsrds` int DEFAULT NULL,\n `pdmmcthyaunwrohdghumtnxzgdrddsna` int DEFAULT NULL,\n `penhyemmvbjhpjnbmgczpqxgzuoomupk` int DEFAULT NULL,\n `tbbanfiywsxekrtqzbzijuoxuvnmnaim` int DEFAULT NULL,\n `dxdrnuhbsgmgttvluyepdmrdmqyqtrik` int DEFAULT NULL,\n `jlrofifzzcoqsrcdogoanhmuovcivrkf` int DEFAULT NULL,\n `xvcdottcbmebybjypzfqnshoamiwatvc` int DEFAULT NULL,\n `tnandgwndduhwyxfmouqkowjzedtssxw` int DEFAULT NULL,\n `xvdgjesdboytgdnakgdaprtwnzqpqpwa` int DEFAULT NULL,\n `occkpqimeawwdpoxwfephqpzpxrghxll` int DEFAULT NULL,\n `mcvhkkovrzifoweezuopmkccdvapyehs` int DEFAULT NULL,\n `fvrfjuhazohnsmnzjihwwvtlndqargys` int DEFAULT NULL,\n `koviwkxyglozkssqvufbbfokqwgejmyd` int DEFAULT NULL,\n `zewiygkjmfdbkgzqlvhtrzosunakdygd` int DEFAULT NULL,\n `kygsrqsdmpqpspyyrcelifppeiibtesk` int DEFAULT NULL,\n `nzshcbcadroogaxpoudpnndewkszupcb` int DEFAULT NULL,\n `wtvoscproufuasgptmjnbvxgqffkkozo` int DEFAULT NULL,\n `ucltifxlfkbwvmgxmnbqqguixyoyxeuc` int DEFAULT NULL,\n `vxmdkveppvdtiwoweswininmpujvzpob` int DEFAULT NULL,\n `sktqvcuzwyzpcysmukxsuloukdjlxkzm` int DEFAULT NULL,\n `opoabzuuzqpxcovdynlhksxnryapcijo` int DEFAULT NULL,\n `qobyysclucmsqmkpzcjhkrcredzcupwm` int DEFAULT NULL,\n `hmqpzqrydmphbphgoxyojezfgknafbej` int DEFAULT NULL,\n `tuvwevorubckdlkkjohazyscqwxnovvi` int DEFAULT NULL,\n `rzryhxqjydhxfxiucaqdrtcmproaxwza` int DEFAULT NULL,\n `pgvofrqyaahxaiqlvlyidfdnfovxkuyi` int DEFAULT NULL,\n `zygoorizgkzhftbsdrnbpvxgwzlonkpy` int DEFAULT NULL,\n `gtgjgbmnpaigntrejjmjpxrbuigkgokz` int DEFAULT NULL,\n `xjbbstcudsfquwuptpgcierpuqotjxub` int DEFAULT NULL,\n `dleextvxyaxymbkikqrvvcwchxyleaay` int DEFAULT NULL,\n `ayaoplomvfvwnffduwfhjeevlbsbhenq` int DEFAULT NULL,\n `vwujuqmcdfgvtprvwlgvxailizdkpynj` int DEFAULT NULL,\n `bdnwlbmfzfspytjfphjphdphivsvsoyg` int DEFAULT NULL,\n `oxdludidughyclpgtwjsufbbdlcbjaow` int DEFAULT NULL,\n `vwddjtfdwejeqaegkvjzfumsrmrwfyas` int DEFAULT NULL,\n `alcwwehnbmqexvihwvanalamuqebrecs` int DEFAULT NULL,\n `txwdhtdhfukaeuwelctdnpisarzcsrpw` int DEFAULT NULL,\n `ojrgytkqiqrpygagdctfaphyytptirqj` int DEFAULT NULL,\n `aocwqerlzheqjhlonpfqidohsoujrxos` int DEFAULT NULL,\n `eytkuboazvfdrizbceqwbofutfntjazh` int DEFAULT NULL,\n `ytjttxjwcocvjvgoigbeqipvyjpllawx` int DEFAULT NULL,\n `znuwwxprfgltqgvawiqmdftwyprdokim` int DEFAULT NULL,\n `wfdooxuaxxnzdmpnpdokpgrecjvbbxct` int DEFAULT NULL,\n `snhjshzuilepqwkhvcdixuuhbcamdrcp` int DEFAULT NULL,\n `cbhuznyuxgzgffujurgwzvaqevmnxhun` int DEFAULT NULL,\n `ibhazqadbdkujfzvbailsnkgfioiwqll` int DEFAULT NULL,\n `tgtvobcpzofjtfweqdozcelacvaqqzgs` int DEFAULT NULL,\n `lpljxidxwzegheolnovprqyndwyqlykw` int DEFAULT NULL,\n `sfqnczjhjkyrznqxxhdngxkftuidwfke` int DEFAULT NULL,\n `pldtbwgruslxwihuzrfpocjsyqgudjgx` int DEFAULT NULL,\n `hxpxrhaeokzckktkmphkbisyqqhpdydx` int DEFAULT NULL,\n `xkyybfvgddqzalhgcshkbecdxkuphoiz` int DEFAULT NULL,\n `rsjkozhgmpjvcmucmgdxwsmfrylbntcc` int DEFAULT NULL,\n `firmtgypkxmpgntkncoulqmaxxudjzvv` int DEFAULT NULL,\n `csxxtgbskuiljeftsafrkzxuydhgqctu` int DEFAULT NULL,\n `fcfyjybpwtjooxxjepqehaltvufgdxvr` int DEFAULT NULL,\n `jklizouygaywkrmagllucdqnfpmvtnoi` int DEFAULT NULL,\n `zbfijygmivuayaektpfrnjvabugeenbl` int DEFAULT NULL,\n `bvarcbwrbzwxekokeltlupgizpvjurai` int DEFAULT NULL,\n `wyipwfnmesdmlpapnvzdsokrwrkaktxd` int DEFAULT NULL,\n `rvwdcxscdkkwavfqxxpoofgqbqmbyhnp` int DEFAULT NULL,\n `asscsxadypecztbtoypygevzauiwechp` int DEFAULT NULL,\n `bqtqpoiaupidfpxiqwkwvxmlvmuvhcgu` int DEFAULT NULL,\n `uwcodhawytibyunxnplwtdiytmygakxc` int DEFAULT NULL,\n `issqyrbjtqokdqnvrynqgbwnnubwnspd` int DEFAULT NULL,\n `bvarfonzwcaujfodtspzbzkqbrewfrdw` int DEFAULT NULL,\n `flsejhqunkpfpxookvdrhwmkkfmafsbw` int DEFAULT NULL,\n `oghgjvnaldikiowjsoohilyfoxotmeld` int DEFAULT NULL,\n `wiryfszewrevlafmyzcneuobxavbcmbe` int DEFAULT NULL,\n `poxffzcfozbtzbjpguyyvvibvvqcnuwl` int DEFAULT NULL,\n `sqehpvwglvwusdnpnznbmbamccfxpxpn` int DEFAULT NULL,\n `yozekaslmswgaczcjiwmshckzxzrsgjw` int DEFAULT NULL,\n `smaqfacqfhwlwbiznblgvbozkycnqnpo` int DEFAULT NULL,\n `vsligyiaawebmjadxrqvvpmyaaewsuxj` int DEFAULT NULL,\n PRIMARY KEY (`itefojdnavbgbuqpkstlsbwkrposdmwq`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"tfoaqprarfoabbffxqiypuqnjuzesvmg\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["itefojdnavbgbuqpkstlsbwkrposdmwq"],"columns":[{"name":"itefojdnavbgbuqpkstlsbwkrposdmwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"garzrcvmospocqitmebwjukupcwkkzjp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vaxhadgltqvuxnqsyjwbrkwkhmtzhcah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igcdeepmsprrngvshawiusblsbyimlbm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"argnupalshsodhnxmumuhmusrszoxvlh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"betjmxxktliqftnipmqxtsgjxpbvssab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwsswrvbshqdlvojazjktetsvqllpqhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxhujgrjennjiydqbbcmrqyebpbuiwbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwgfnjbxhukjpjgkglzoocryvkbxhkte","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgjevubsebxjatzuuggtclgvnntlzswh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyzordnzvslqibqywnpvbctypcgwoveo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vybvnerrmgnfbajruupowykvwqwrufbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwvnuerfyiiiuskvormckmrvjauqhupg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jykgyfhrhdbrwabdxtbaekpdmrrztyhj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zblrvedmceniszjmhfrohmjeyifqchae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ublhfvlispnratgvpvfrugntizheefze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kbcyqwvqfswjpthxkywjpwwkpickofsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rwcrabzaejecylphjfaiyatdibufxqrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjphdbhsbnxnsgdanabhhuisxwsmqlfv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhyzwqjrlxaydgkhdhcxllfssquqcxbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drpunsdvjhilqoeyzpkcpnbrwkrgvcyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"egqlveyrubndinrjosadpwdmlbbpqycy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ttecplpdgvmsmacbfcpzzaiupvprjtef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"beihgqzjleorgjtmfjplxoelpuxzzxfa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcqeduzkruvkmjesazdvgwemuoffaiyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfxoynjnimkwazheykjviguinntubstx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bhjhbmroinrswhvjrsplgtizxznbsrds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdmmcthyaunwrohdghumtnxzgdrddsna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"penhyemmvbjhpjnbmgczpqxgzuoomupk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbbanfiywsxekrtqzbzijuoxuvnmnaim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxdrnuhbsgmgttvluyepdmrdmqyqtrik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlrofifzzcoqsrcdogoanhmuovcivrkf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvcdottcbmebybjypzfqnshoamiwatvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnandgwndduhwyxfmouqkowjzedtssxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvdgjesdboytgdnakgdaprtwnzqpqpwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"occkpqimeawwdpoxwfephqpzpxrghxll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcvhkkovrzifoweezuopmkccdvapyehs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fvrfjuhazohnsmnzjihwwvtlndqargys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"koviwkxyglozkssqvufbbfokqwgejmyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zewiygkjmfdbkgzqlvhtrzosunakdygd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kygsrqsdmpqpspyyrcelifppeiibtesk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzshcbcadroogaxpoudpnndewkszupcb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtvoscproufuasgptmjnbvxgqffkkozo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucltifxlfkbwvmgxmnbqqguixyoyxeuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxmdkveppvdtiwoweswininmpujvzpob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sktqvcuzwyzpcysmukxsuloukdjlxkzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opoabzuuzqpxcovdynlhksxnryapcijo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qobyysclucmsqmkpzcjhkrcredzcupwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmqpzqrydmphbphgoxyojezfgknafbej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tuvwevorubckdlkkjohazyscqwxnovvi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzryhxqjydhxfxiucaqdrtcmproaxwza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgvofrqyaahxaiqlvlyidfdnfovxkuyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zygoorizgkzhftbsdrnbpvxgwzlonkpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtgjgbmnpaigntrejjmjpxrbuigkgokz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjbbstcudsfquwuptpgcierpuqotjxub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dleextvxyaxymbkikqrvvcwchxyleaay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ayaoplomvfvwnffduwfhjeevlbsbhenq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vwujuqmcdfgvtprvwlgvxailizdkpynj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdnwlbmfzfspytjfphjphdphivsvsoyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oxdludidughyclpgtwjsufbbdlcbjaow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vwddjtfdwejeqaegkvjzfumsrmrwfyas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"alcwwehnbmqexvihwvanalamuqebrecs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txwdhtdhfukaeuwelctdnpisarzcsrpw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojrgytkqiqrpygagdctfaphyytptirqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aocwqerlzheqjhlonpfqidohsoujrxos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eytkuboazvfdrizbceqwbofutfntjazh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytjttxjwcocvjvgoigbeqipvyjpllawx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znuwwxprfgltqgvawiqmdftwyprdokim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfdooxuaxxnzdmpnpdokpgrecjvbbxct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"snhjshzuilepqwkhvcdixuuhbcamdrcp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbhuznyuxgzgffujurgwzvaqevmnxhun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibhazqadbdkujfzvbailsnkgfioiwqll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tgtvobcpzofjtfweqdozcelacvaqqzgs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lpljxidxwzegheolnovprqyndwyqlykw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfqnczjhjkyrznqxxhdngxkftuidwfke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pldtbwgruslxwihuzrfpocjsyqgudjgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxpxrhaeokzckktkmphkbisyqqhpdydx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkyybfvgddqzalhgcshkbecdxkuphoiz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rsjkozhgmpjvcmucmgdxwsmfrylbntcc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"firmtgypkxmpgntkncoulqmaxxudjzvv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"csxxtgbskuiljeftsafrkzxuydhgqctu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcfyjybpwtjooxxjepqehaltvufgdxvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jklizouygaywkrmagllucdqnfpmvtnoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbfijygmivuayaektpfrnjvabugeenbl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvarcbwrbzwxekokeltlupgizpvjurai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyipwfnmesdmlpapnvzdsokrwrkaktxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rvwdcxscdkkwavfqxxpoofgqbqmbyhnp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"asscsxadypecztbtoypygevzauiwechp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqtqpoiaupidfpxiqwkwvxmlvmuvhcgu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwcodhawytibyunxnplwtdiytmygakxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"issqyrbjtqokdqnvrynqgbwnnubwnspd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvarfonzwcaujfodtspzbzkqbrewfrdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flsejhqunkpfpxookvdrhwmkkfmafsbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oghgjvnaldikiowjsoohilyfoxotmeld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wiryfszewrevlafmyzcneuobxavbcmbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"poxffzcfozbtzbjpguyyvvibvvqcnuwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sqehpvwglvwusdnpnznbmbamccfxpxpn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yozekaslmswgaczcjiwmshckzxzrsgjw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smaqfacqfhwlwbiznblgvbozkycnqnpo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsligyiaawebmjadxrqvvpmyaaewsuxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671382,"databaseName":"models_schema","ddl":"CREATE TABLE `tgczemkospnlaxgjoudtzsgbgjwujcbh` (\n `yfkfrnvliwqrsrlqnewainwbjecqxukn` int NOT NULL,\n `rclqqxgmuzbjgkmqyudfwgkzfppmtaat` int DEFAULT NULL,\n `nvuqmhfiluvffkiwoycnyosovrrhoatj` int DEFAULT NULL,\n `jawxvaakbvqzybdlyatiljloejeppwuf` int DEFAULT NULL,\n `qkylbqingygetqrmuuyzbefildzlgafc` int DEFAULT NULL,\n `umzxamcovlkirpgncyooqyqhtbjtdgqg` int DEFAULT NULL,\n `ijzmcnrultugffywurbfgaktoprbrlzo` int DEFAULT NULL,\n `lfdujbzcjemaknfzczuzycrstiayqunw` int DEFAULT NULL,\n `qjyamvqngpddjgajgsexvyudqfgawnun` int DEFAULT NULL,\n `chhbmfeodoaudygddxnhpjncxdkedgea` int DEFAULT NULL,\n `zddhuwghtkrsfqcooitwghcpefmfhtpi` int DEFAULT NULL,\n `uavoaaiveqlosdjbnqcciupfjhucnkzo` int DEFAULT NULL,\n `kjqbaohyjemnkpxbrjmlbbxjmrqwrpbr` int DEFAULT NULL,\n `dwftcwdcykdxmcxactoiwrccfzfmycqe` int DEFAULT NULL,\n `rtxprpqukqmtfavtybhmfbcuhpqegsjn` int DEFAULT NULL,\n `knazyafxuiitvmdmxggifvyaieltvpoi` int DEFAULT NULL,\n `cuzcfpnczxeybhfxzdkgblfpkvavgqae` int DEFAULT NULL,\n `szwwwkqgpdmwssxxsvsyexverdvyimhr` int DEFAULT NULL,\n `lhvbrrgcrzbsobmgpxdqvxiappkaqaph` int DEFAULT NULL,\n `npxauherviusefmqfldkmyjjawscnbmy` int DEFAULT NULL,\n `rnhbrkphdsrzybgjqcldlovxozzqzyku` int DEFAULT NULL,\n `qvtohabygxqudexwxuavvqumonkgioza` int DEFAULT NULL,\n `irrqzslpgqphjvmvnoqsofmzwllwjyiv` int DEFAULT NULL,\n `cvgysyqsxcvfogisomxprikgjblsrekn` int DEFAULT NULL,\n `njpgskcdkrigncqotdkwuplljsdnffwj` int DEFAULT NULL,\n `fiymawnemubavbzfudgrtpdxirafctjs` int DEFAULT NULL,\n `dqexmosuyrnjchxkwmeoltxxpysbjkfv` int DEFAULT NULL,\n `yjwawcignavdnauhuqmqhdzzuhavnmpj` int DEFAULT NULL,\n `xljqcurtkxdxxpziwbdfzonecmcuhwhd` int DEFAULT NULL,\n `rvhoxyyvtmcjzpxqefgppapimvgcyufb` int DEFAULT NULL,\n `wprwmgngwklbbhkzvwqvtqnqthewvhzi` int DEFAULT NULL,\n `ynptacpckrzlngadbijpxukvpivbutre` int DEFAULT NULL,\n `upqnnfkeivhgtunvgumhlomfdihrujyi` int DEFAULT NULL,\n `ubpkioxoxvpaqkmqvazxtjqtlpyolfxd` int DEFAULT NULL,\n `dtroebfzjzzcluhvhwsmwvlkuqonhlav` int DEFAULT NULL,\n `kdqpzfywbfxxmrwhgintvizfyinujvlx` int DEFAULT NULL,\n `itxlqtonisjsghpkygohgccclwiqwcfs` int DEFAULT NULL,\n `wsjnogosvobjvxbqfmdxddkboplzsxwk` int DEFAULT NULL,\n `xfjcbbmjgdumqmixveqwydjdxrfanxiv` int DEFAULT NULL,\n `cbwfsmfqfxiifuefttqpnzchnpmrcwam` int DEFAULT NULL,\n `kmunqjqbkmtmftwawjnixulghosfudzu` int DEFAULT NULL,\n `mvjledqzypuzetjabshxnkslkuddgejs` int DEFAULT NULL,\n `lvppyrtjsgaimkfnyqenhwwfaieavpuk` int DEFAULT NULL,\n `mvlavftcdlwqmearjmftdbertmieeayq` int DEFAULT NULL,\n `ccvwobcfsxgyodfdckxknxfxcdcnpsio` int DEFAULT NULL,\n `whvmcvujfeqwxlmqsvrolkvbxhkkgosu` int DEFAULT NULL,\n `ljqoohgbicljwlhedhkqxdayjhacjpim` int DEFAULT NULL,\n `fafpntglghvesrgvsuwxmwuiqdpeguqj` int DEFAULT NULL,\n `cxebjrulwejgvknmfmfqahbfsqhztjps` int DEFAULT NULL,\n `eqvgruvsgaxmbuofbyqdfixpgsndcvux` int DEFAULT NULL,\n `ydbbqwirvmxusmijwfsmrddbxkmoilgx` int DEFAULT NULL,\n `xdbbzhhyjihdligikdndicqacdaflimz` int DEFAULT NULL,\n `kkuzufpwzigrofwzyksfirhwclibklar` int DEFAULT NULL,\n `bbznqbrduhtwgppcochyqjqwihswmcwe` int DEFAULT NULL,\n `rwsxnetjrjzhlxohvrqsfagrwhihrsnd` int DEFAULT NULL,\n `uxlmfwfnhsvjahhqtbethpllcpbhtbdq` int DEFAULT NULL,\n `etclvguoywygjdebvtfuspgcbnvaetah` int DEFAULT NULL,\n `amyttiyokkayvxsrvgohudnhqgplrqzf` int DEFAULT NULL,\n `mmffsngbekszdtvmrbkdyfywwtotzace` int DEFAULT NULL,\n `lyuktsavdemarxbmwleleohgzbdsqwjy` int DEFAULT NULL,\n `ptgmjzfehnlwdzsiabttwxsgetmpvfmv` int DEFAULT NULL,\n `aatlgrbsgnyfchfpqeozbzsijjbvmzjz` int DEFAULT NULL,\n `vutbfakxjlzfhqcefcaimqpbidkbewsw` int DEFAULT NULL,\n `zrlzylwqmtkiindulefqngkbnfhpiker` int DEFAULT NULL,\n `jyfjkvjcqoxnnzevkssjiqxyacifyjtr` int DEFAULT NULL,\n `wmxesjkpynyewxvbsuqtghpjrsxzmzxe` int DEFAULT NULL,\n `zsyxlbakbmyxnytayqquujmcktrhjkef` int DEFAULT NULL,\n `lxniydlvoqjwdwvskqesrokpeymuuqdp` int DEFAULT NULL,\n `swwcvxtmsmdovdnlizyrrrdrlsklodri` int DEFAULT NULL,\n `ndjmonvqaoqpvldthzignvkrnyhzmiuj` int DEFAULT NULL,\n `ytsptnjjeglqmuprleiuqiujgtvbsdcc` int DEFAULT NULL,\n `jmwnhtenxhftytpgkmkxfkwbozldyirb` int DEFAULT NULL,\n `fnsuuhenyqxyeialstmkvwkoejnzfuwn` int DEFAULT NULL,\n `qxnnilbxbhdidgsbvebhzyxatleudlbo` int DEFAULT NULL,\n `dtorhjakijnhnwwsoyzzrufkifawcwtq` int DEFAULT NULL,\n `nbfavdujxkucowffzanmuvoyxqvmxmdc` int DEFAULT NULL,\n `mrmfsilemnmhdzqtmnejoaryebtorvng` int DEFAULT NULL,\n `relnrfpdwttfzjasyayesfmdwwreljav` int DEFAULT NULL,\n `scdqsdtxmbgxdmdkuffjxbysufbxyvis` int DEFAULT NULL,\n `nzboiygftrhfsrofxwqakcrtfqaovkhp` int DEFAULT NULL,\n `kcwbvydbiutyzjawxpujsljoafloddne` int DEFAULT NULL,\n `qawauictroylkrsgdindzgggwkzklvnb` int DEFAULT NULL,\n `tpcroojyeevnwpddkhwxxtoccwckynnk` int DEFAULT NULL,\n `gzrvoxvoiqxvqftywlivxxxpbonbliua` int DEFAULT NULL,\n `mdmcocsmqzrsicmwwsvopgreyxylvals` int DEFAULT NULL,\n `kawmhjfwdasarbjgelnoptpilppvlvnx` int DEFAULT NULL,\n `cvilvniamgwsgzncpvombkzmuhhvkuhm` int DEFAULT NULL,\n `vlicwdkfuxbbypaudfbsikvcsrfvhtbx` int DEFAULT NULL,\n `xkbdvxutgtqxcikgtgfdsbylidrekvio` int DEFAULT NULL,\n `mrvqrsrljletpvsbbrpftojbwagqkhyb` int DEFAULT NULL,\n `deecsfxuurdsolyyttoguqgmvdxttckn` int DEFAULT NULL,\n `ynhqlahsnfzkyevxarkamovgrmypbjjh` int DEFAULT NULL,\n `lgweizhaxwlvqxmbwankgzsbmsgiiguh` int DEFAULT NULL,\n `efgwxcrvrwukuzxucvbkuteshvbrbpge` int DEFAULT NULL,\n `bsbkpjsehcxbjdfisavvewjogkwtkuyx` int DEFAULT NULL,\n `uhezfntydsvcgnlfbumksalzfjmiudfx` int DEFAULT NULL,\n `fyqfnzxjulsxoyphdzuynrdjjrjoqxsk` int DEFAULT NULL,\n `hanysmwnihvjbjdirueziwfxnwmbwwxk` int DEFAULT NULL,\n `hujnfgsdysttpcqzlmwumszvhekqwwqf` int DEFAULT NULL,\n `xyksuzuasgzkzryqqzupukmoyqqfvopa` int DEFAULT NULL,\n PRIMARY KEY (`yfkfrnvliwqrsrlqnewainwbjecqxukn`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"tgczemkospnlaxgjoudtzsgbgjwujcbh\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["yfkfrnvliwqrsrlqnewainwbjecqxukn"],"columns":[{"name":"yfkfrnvliwqrsrlqnewainwbjecqxukn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"rclqqxgmuzbjgkmqyudfwgkzfppmtaat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvuqmhfiluvffkiwoycnyosovrrhoatj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jawxvaakbvqzybdlyatiljloejeppwuf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qkylbqingygetqrmuuyzbefildzlgafc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umzxamcovlkirpgncyooqyqhtbjtdgqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijzmcnrultugffywurbfgaktoprbrlzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfdujbzcjemaknfzczuzycrstiayqunw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjyamvqngpddjgajgsexvyudqfgawnun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chhbmfeodoaudygddxnhpjncxdkedgea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zddhuwghtkrsfqcooitwghcpefmfhtpi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uavoaaiveqlosdjbnqcciupfjhucnkzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjqbaohyjemnkpxbrjmlbbxjmrqwrpbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwftcwdcykdxmcxactoiwrccfzfmycqe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtxprpqukqmtfavtybhmfbcuhpqegsjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knazyafxuiitvmdmxggifvyaieltvpoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cuzcfpnczxeybhfxzdkgblfpkvavgqae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"szwwwkqgpdmwssxxsvsyexverdvyimhr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhvbrrgcrzbsobmgpxdqvxiappkaqaph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npxauherviusefmqfldkmyjjawscnbmy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnhbrkphdsrzybgjqcldlovxozzqzyku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvtohabygxqudexwxuavvqumonkgioza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"irrqzslpgqphjvmvnoqsofmzwllwjyiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvgysyqsxcvfogisomxprikgjblsrekn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njpgskcdkrigncqotdkwuplljsdnffwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fiymawnemubavbzfudgrtpdxirafctjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqexmosuyrnjchxkwmeoltxxpysbjkfv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjwawcignavdnauhuqmqhdzzuhavnmpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xljqcurtkxdxxpziwbdfzonecmcuhwhd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rvhoxyyvtmcjzpxqefgppapimvgcyufb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wprwmgngwklbbhkzvwqvtqnqthewvhzi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynptacpckrzlngadbijpxukvpivbutre","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upqnnfkeivhgtunvgumhlomfdihrujyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubpkioxoxvpaqkmqvazxtjqtlpyolfxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtroebfzjzzcluhvhwsmwvlkuqonhlav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kdqpzfywbfxxmrwhgintvizfyinujvlx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itxlqtonisjsghpkygohgccclwiqwcfs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wsjnogosvobjvxbqfmdxddkboplzsxwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xfjcbbmjgdumqmixveqwydjdxrfanxiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbwfsmfqfxiifuefttqpnzchnpmrcwam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmunqjqbkmtmftwawjnixulghosfudzu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvjledqzypuzetjabshxnkslkuddgejs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvppyrtjsgaimkfnyqenhwwfaieavpuk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvlavftcdlwqmearjmftdbertmieeayq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ccvwobcfsxgyodfdckxknxfxcdcnpsio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"whvmcvujfeqwxlmqsvrolkvbxhkkgosu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljqoohgbicljwlhedhkqxdayjhacjpim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fafpntglghvesrgvsuwxmwuiqdpeguqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cxebjrulwejgvknmfmfqahbfsqhztjps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqvgruvsgaxmbuofbyqdfixpgsndcvux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydbbqwirvmxusmijwfsmrddbxkmoilgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdbbzhhyjihdligikdndicqacdaflimz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kkuzufpwzigrofwzyksfirhwclibklar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbznqbrduhtwgppcochyqjqwihswmcwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rwsxnetjrjzhlxohvrqsfagrwhihrsnd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxlmfwfnhsvjahhqtbethpllcpbhtbdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"etclvguoywygjdebvtfuspgcbnvaetah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amyttiyokkayvxsrvgohudnhqgplrqzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmffsngbekszdtvmrbkdyfywwtotzace","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lyuktsavdemarxbmwleleohgzbdsqwjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ptgmjzfehnlwdzsiabttwxsgetmpvfmv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aatlgrbsgnyfchfpqeozbzsijjbvmzjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vutbfakxjlzfhqcefcaimqpbidkbewsw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zrlzylwqmtkiindulefqngkbnfhpiker","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jyfjkvjcqoxnnzevkssjiqxyacifyjtr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmxesjkpynyewxvbsuqtghpjrsxzmzxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zsyxlbakbmyxnytayqquujmcktrhjkef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxniydlvoqjwdwvskqesrokpeymuuqdp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swwcvxtmsmdovdnlizyrrrdrlsklodri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ndjmonvqaoqpvldthzignvkrnyhzmiuj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytsptnjjeglqmuprleiuqiujgtvbsdcc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmwnhtenxhftytpgkmkxfkwbozldyirb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnsuuhenyqxyeialstmkvwkoejnzfuwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxnnilbxbhdidgsbvebhzyxatleudlbo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtorhjakijnhnwwsoyzzrufkifawcwtq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbfavdujxkucowffzanmuvoyxqvmxmdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrmfsilemnmhdzqtmnejoaryebtorvng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"relnrfpdwttfzjasyayesfmdwwreljav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scdqsdtxmbgxdmdkuffjxbysufbxyvis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzboiygftrhfsrofxwqakcrtfqaovkhp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcwbvydbiutyzjawxpujsljoafloddne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qawauictroylkrsgdindzgggwkzklvnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpcroojyeevnwpddkhwxxtoccwckynnk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzrvoxvoiqxvqftywlivxxxpbonbliua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdmcocsmqzrsicmwwsvopgreyxylvals","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kawmhjfwdasarbjgelnoptpilppvlvnx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvilvniamgwsgzncpvombkzmuhhvkuhm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vlicwdkfuxbbypaudfbsikvcsrfvhtbx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkbdvxutgtqxcikgtgfdsbylidrekvio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrvqrsrljletpvsbbrpftojbwagqkhyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"deecsfxuurdsolyyttoguqgmvdxttckn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynhqlahsnfzkyevxarkamovgrmypbjjh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgweizhaxwlvqxmbwankgzsbmsgiiguh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"efgwxcrvrwukuzxucvbkuteshvbrbpge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bsbkpjsehcxbjdfisavvewjogkwtkuyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhezfntydsvcgnlfbumksalzfjmiudfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fyqfnzxjulsxoyphdzuynrdjjrjoqxsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hanysmwnihvjbjdirueziwfxnwmbwwxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hujnfgsdysttpcqzlmwumszvhekqwwqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyksuzuasgzkzryqqzupukmoyqqfvopa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671413,"databaseName":"models_schema","ddl":"CREATE TABLE `tgtlonwoccyfoxpnlcdbuzcncumduhrs` (\n `edaspexhikcohkkkfvtodwlwhliqmesg` int NOT NULL,\n `tsruoqutkwwwmufemyfhehdrpphjbdhf` int DEFAULT NULL,\n `cvicqoqndadkctxwjnebyalzboqcvjdr` int DEFAULT NULL,\n `meqoniyedtlgtaozqrauehiqfkvljicj` int DEFAULT NULL,\n `btcctdeqfipeiourtbmtdxmoomuwzayl` int DEFAULT NULL,\n `snjbnpvwbdvebtwetetjruzmmkmdgzsm` int DEFAULT NULL,\n `disaswiwbrxjxnpxxqjaooroxykcxzvh` int DEFAULT NULL,\n `wpouycpgebxwaovwlraorxbzrzxmqnmg` int DEFAULT NULL,\n `nhatcpphsyuzqgsiddcvylseurjddsez` int DEFAULT NULL,\n `iykbqwucczeqsepgowrhcnzcdmjapkmf` int DEFAULT NULL,\n `dcgpbzaalkeqqlaazocvtxvmadwsxqoc` int DEFAULT NULL,\n `rpcpshqagjtivojnjwsncovewvmpwump` int DEFAULT NULL,\n `xctektzlmgbnlszdqkvkjgltrxknwech` int DEFAULT NULL,\n `wlzuwclgujczeuzvliorpifndgkrjlix` int DEFAULT NULL,\n `lcibpaylsaxxodhngxyiuyslacrwhleh` int DEFAULT NULL,\n `iysdzfjnrzhbcgmmophqnhisyrkofslk` int DEFAULT NULL,\n `pccgsevpplygtuhtfyeenicznhmnwoba` int DEFAULT NULL,\n `vbldzcfhmpaopfecuqrxwspfkkwjfnyr` int DEFAULT NULL,\n `ybmynbvnizgvuygybbcajcrlibdeusna` int DEFAULT NULL,\n `zfhbgjnokyqlpuerctjtcafxanxibzwb` int DEFAULT NULL,\n `cktecabsvaxvazofunlyvitomsinleyz` int DEFAULT NULL,\n `ynipfespqoknghwdyydawkjdopebvoio` int DEFAULT NULL,\n `thbsqfdixtjoctdzpgrcfuvxgrkykiwj` int DEFAULT NULL,\n `nsfpgjgkuazptnhltmciijfglhdhiaot` int DEFAULT NULL,\n `ylnlabdnmtptvdxbqihfeyumyvwflknx` int DEFAULT NULL,\n `xvfhsnkiviavqzzifqzzzdpmtvffnoua` int DEFAULT NULL,\n `gcbjozvglrxdabppyqtbijbtxkcnllji` int DEFAULT NULL,\n `gcgmcemeayntkxosdypogqebhpgiivgx` int DEFAULT NULL,\n `wspklmfmzzgznhsvkgvfktcbgevwnklu` int DEFAULT NULL,\n `crbqfqfnirvudlrjzjkqjparcxwcsnup` int DEFAULT NULL,\n `qtxzkmlvniqbqbkqimiaqldzdhpkzboc` int DEFAULT NULL,\n `mubhhzfxemlzhuwwkcqwaahiigexsidg` int DEFAULT NULL,\n `dsuicseukziwgykvpxhxdcgximbguelu` int DEFAULT NULL,\n `ktyqvchzhojmbpmlsttcsnfqfvruetay` int DEFAULT NULL,\n `qghnfwcaqlnwduhhubrsizmunniznaeq` int DEFAULT NULL,\n `ffegvynrntucbjwgxqcwheajkbpnfvjy` int DEFAULT NULL,\n `bdarbcwkpgtdxqiwganensmkkssgecik` int DEFAULT NULL,\n `dhkofpsgilygegwxgiohpucuauxjgktu` int DEFAULT NULL,\n `qhhdmmwtqgcdozgkxjqzjbmkrixfpqop` int DEFAULT NULL,\n `eiwyshkepflmhzjlgklxblsffdwschct` int DEFAULT NULL,\n `rdjvypsxsgqpgzzejhuaewukntyedyqv` int DEFAULT NULL,\n `rliwngmktizdtlzkrjhpvukcarbocgfu` int DEFAULT NULL,\n `metdpfghmrwxmhjtctwkhujouyrujjuv` int DEFAULT NULL,\n `axsreowpkeczdmkmkelzbijojjenrxzk` int DEFAULT NULL,\n `rmdcfkwgxtipbyzqmgqcywoouxjrftvb` int DEFAULT NULL,\n `qbpgtnkagchqmtigudqddfwkiyuvaodv` int DEFAULT NULL,\n `qgwrvntfotjafgczsteuovcuzyeyhoya` int DEFAULT NULL,\n `wcqvucrxftovsgwckwulotkpmtweslkh` int DEFAULT NULL,\n `qxfizmlbnqqumucxqzlgvnsyvxjdhlht` int DEFAULT NULL,\n `dvykaqjtsdgxqivefeuordnzrqwbwfws` int DEFAULT NULL,\n `awqzgssafryzmxwaczltlwlivkfhahjd` int DEFAULT NULL,\n `udtnafqnfumqzltkyjyvzakpnmnpbuyc` int DEFAULT NULL,\n `kltljtyuqghjwpyecnmxnpwgwkaubfwd` int DEFAULT NULL,\n `urdywgbbucuvzpdaxromupaxehzabjuk` int DEFAULT NULL,\n `lhhystdlvswusdebmpoplhiymotojsij` int DEFAULT NULL,\n `hqrxlfavhifwjkwcfmutruslxfdfauej` int DEFAULT NULL,\n `rhthbvkozdacswcgqnrmrdugkghaxzzx` int DEFAULT NULL,\n `molbmldlnmkttwlssqomstgxckrtcwvl` int DEFAULT NULL,\n `qwnssepofbuiqrtjahistcesjvgdbxbv` int DEFAULT NULL,\n `inlytbdciemfjlupjdreqfnlepoysdsc` int DEFAULT NULL,\n `chpykaryjaegnpjidmqnpmqjyyshbskp` int DEFAULT NULL,\n `ocgbmctrhnhnbyrxcbtanntomxmrffns` int DEFAULT NULL,\n `zwvjtoveidgneuvkdlmmwjuqquehkdbg` int DEFAULT NULL,\n `ivyccerdxykyvxdkhcmyehuudjnjwzsz` int DEFAULT NULL,\n `mgaonjimpcdtlsylllipinjaiakdhock` int DEFAULT NULL,\n `ffkhijeidnagnfjrggjaodmkvehexxmo` int DEFAULT NULL,\n `dntrlwvrvavcwylygsvbwnoucabkkevl` int DEFAULT NULL,\n `xfqsoijysygcmrfpuklfvraeuvuepref` int DEFAULT NULL,\n `vyyfrydipbjolcqmgmhywaasgkjyszub` int DEFAULT NULL,\n `ykoobhvfgbibauishrxmzfnkadsgeqjp` int DEFAULT NULL,\n `qjhxiuxupubamddgyduwqyjijlydiybw` int DEFAULT NULL,\n `vhynrikayzfqehxsezvetzbyhwaxbenq` int DEFAULT NULL,\n `fibmnbkzvyhutjvmytqvunuvlgryzlml` int DEFAULT NULL,\n `ieppzhwplceprslglnoynwowzkyahqzo` int DEFAULT NULL,\n `kpugdbjnmofgxomisftiribrskcvqzol` int DEFAULT NULL,\n `crsgmisbviwifkxoizchsdonjeytohrg` int DEFAULT NULL,\n `kttrracxzydjyhwzqurcqxerobfcstmu` int DEFAULT NULL,\n `rksyutwuqtuxbrphzpbjyexicplcqcni` int DEFAULT NULL,\n `vpsgyzeljtwnmaavowcnjxvgkdjzulfn` int DEFAULT NULL,\n `ujtotphrglwjnciburzguegamoqtsgsd` int DEFAULT NULL,\n `vfslwfsvkabqbzxwljxuklygyosetpxf` int DEFAULT NULL,\n `dhapdrmekxlbdjdgtohbwzosnsxicoiu` int DEFAULT NULL,\n `muafdmmmmoiyrxvjorevqfyaggevvywa` int DEFAULT NULL,\n `mlbynxspciaecrawwxdmqulhiecgmqim` int DEFAULT NULL,\n `qgtuybmtndprjpemhfqknlrpylvhnqrx` int DEFAULT NULL,\n `gzdskmhlcqnhupxatihmsuketefeuolp` int DEFAULT NULL,\n `fppxfkextdktgcoprxolvvygumarvdey` int DEFAULT NULL,\n `diltwbmvslzibhnlwvzfmlolgnmxrrqm` int DEFAULT NULL,\n `qntbkbwmlqhinxnruxolttdgcwjybjhn` int DEFAULT NULL,\n `acourfclhsnthsdddhsiziwpjdfbxfpo` int DEFAULT NULL,\n `ojsqknmxmkaoeksqaoydfkkbhgxcbuhz` int DEFAULT NULL,\n `nawelblnyztfmemrxobomgxxjdncntgt` int DEFAULT NULL,\n `fprqvfgizccpzequpsjhsjfvylfehjtt` int DEFAULT NULL,\n `bmvheggyevhbjrnrpaaseghoogfjacug` int DEFAULT NULL,\n `zscepnxfvngazrigxbmdshhaberpyvzt` int DEFAULT NULL,\n `bxiayghkhjmuymxxxgdvnodzjkqpdxbp` int DEFAULT NULL,\n `ivjmpwrgsjwjbfzsuzcdygqkqtwsfald` int DEFAULT NULL,\n `eknujyrxjftcdbnisettnbhgemmtgzdy` int DEFAULT NULL,\n `wusjjuyhfkioqaegtloxkfboigsndifi` int DEFAULT NULL,\n `ddidsgbkheeiryenceibgiolsrtrjgfm` int DEFAULT NULL,\n PRIMARY KEY (`edaspexhikcohkkkfvtodwlwhliqmesg`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"tgtlonwoccyfoxpnlcdbuzcncumduhrs\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["edaspexhikcohkkkfvtodwlwhliqmesg"],"columns":[{"name":"edaspexhikcohkkkfvtodwlwhliqmesg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"tsruoqutkwwwmufemyfhehdrpphjbdhf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvicqoqndadkctxwjnebyalzboqcvjdr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"meqoniyedtlgtaozqrauehiqfkvljicj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"btcctdeqfipeiourtbmtdxmoomuwzayl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"snjbnpvwbdvebtwetetjruzmmkmdgzsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"disaswiwbrxjxnpxxqjaooroxykcxzvh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wpouycpgebxwaovwlraorxbzrzxmqnmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhatcpphsyuzqgsiddcvylseurjddsez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iykbqwucczeqsepgowrhcnzcdmjapkmf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcgpbzaalkeqqlaazocvtxvmadwsxqoc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpcpshqagjtivojnjwsncovewvmpwump","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xctektzlmgbnlszdqkvkjgltrxknwech","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wlzuwclgujczeuzvliorpifndgkrjlix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lcibpaylsaxxodhngxyiuyslacrwhleh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iysdzfjnrzhbcgmmophqnhisyrkofslk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pccgsevpplygtuhtfyeenicznhmnwoba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbldzcfhmpaopfecuqrxwspfkkwjfnyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybmynbvnizgvuygybbcajcrlibdeusna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zfhbgjnokyqlpuerctjtcafxanxibzwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cktecabsvaxvazofunlyvitomsinleyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynipfespqoknghwdyydawkjdopebvoio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thbsqfdixtjoctdzpgrcfuvxgrkykiwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nsfpgjgkuazptnhltmciijfglhdhiaot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ylnlabdnmtptvdxbqihfeyumyvwflknx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvfhsnkiviavqzzifqzzzdpmtvffnoua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gcbjozvglrxdabppyqtbijbtxkcnllji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gcgmcemeayntkxosdypogqebhpgiivgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wspklmfmzzgznhsvkgvfktcbgevwnklu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crbqfqfnirvudlrjzjkqjparcxwcsnup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtxzkmlvniqbqbkqimiaqldzdhpkzboc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mubhhzfxemlzhuwwkcqwaahiigexsidg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsuicseukziwgykvpxhxdcgximbguelu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ktyqvchzhojmbpmlsttcsnfqfvruetay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qghnfwcaqlnwduhhubrsizmunniznaeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffegvynrntucbjwgxqcwheajkbpnfvjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdarbcwkpgtdxqiwganensmkkssgecik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhkofpsgilygegwxgiohpucuauxjgktu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhhdmmwtqgcdozgkxjqzjbmkrixfpqop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eiwyshkepflmhzjlgklxblsffdwschct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdjvypsxsgqpgzzejhuaewukntyedyqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rliwngmktizdtlzkrjhpvukcarbocgfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"metdpfghmrwxmhjtctwkhujouyrujjuv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axsreowpkeczdmkmkelzbijojjenrxzk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rmdcfkwgxtipbyzqmgqcywoouxjrftvb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qbpgtnkagchqmtigudqddfwkiyuvaodv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgwrvntfotjafgczsteuovcuzyeyhoya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcqvucrxftovsgwckwulotkpmtweslkh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxfizmlbnqqumucxqzlgvnsyvxjdhlht","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvykaqjtsdgxqivefeuordnzrqwbwfws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awqzgssafryzmxwaczltlwlivkfhahjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udtnafqnfumqzltkyjyvzakpnmnpbuyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kltljtyuqghjwpyecnmxnpwgwkaubfwd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urdywgbbucuvzpdaxromupaxehzabjuk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhhystdlvswusdebmpoplhiymotojsij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqrxlfavhifwjkwcfmutruslxfdfauej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhthbvkozdacswcgqnrmrdugkghaxzzx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"molbmldlnmkttwlssqomstgxckrtcwvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwnssepofbuiqrtjahistcesjvgdbxbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"inlytbdciemfjlupjdreqfnlepoysdsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chpykaryjaegnpjidmqnpmqjyyshbskp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocgbmctrhnhnbyrxcbtanntomxmrffns","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zwvjtoveidgneuvkdlmmwjuqquehkdbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivyccerdxykyvxdkhcmyehuudjnjwzsz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mgaonjimpcdtlsylllipinjaiakdhock","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffkhijeidnagnfjrggjaodmkvehexxmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dntrlwvrvavcwylygsvbwnoucabkkevl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xfqsoijysygcmrfpuklfvraeuvuepref","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vyyfrydipbjolcqmgmhywaasgkjyszub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykoobhvfgbibauishrxmzfnkadsgeqjp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjhxiuxupubamddgyduwqyjijlydiybw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhynrikayzfqehxsezvetzbyhwaxbenq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fibmnbkzvyhutjvmytqvunuvlgryzlml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ieppzhwplceprslglnoynwowzkyahqzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpugdbjnmofgxomisftiribrskcvqzol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crsgmisbviwifkxoizchsdonjeytohrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kttrracxzydjyhwzqurcqxerobfcstmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rksyutwuqtuxbrphzpbjyexicplcqcni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpsgyzeljtwnmaavowcnjxvgkdjzulfn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujtotphrglwjnciburzguegamoqtsgsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vfslwfsvkabqbzxwljxuklygyosetpxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhapdrmekxlbdjdgtohbwzosnsxicoiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"muafdmmmmoiyrxvjorevqfyaggevvywa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlbynxspciaecrawwxdmqulhiecgmqim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgtuybmtndprjpemhfqknlrpylvhnqrx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzdskmhlcqnhupxatihmsuketefeuolp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fppxfkextdktgcoprxolvvygumarvdey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"diltwbmvslzibhnlwvzfmlolgnmxrrqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qntbkbwmlqhinxnruxolttdgcwjybjhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acourfclhsnthsdddhsiziwpjdfbxfpo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojsqknmxmkaoeksqaoydfkkbhgxcbuhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nawelblnyztfmemrxobomgxxjdncntgt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fprqvfgizccpzequpsjhsjfvylfehjtt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bmvheggyevhbjrnrpaaseghoogfjacug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zscepnxfvngazrigxbmdshhaberpyvzt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bxiayghkhjmuymxxxgdvnodzjkqpdxbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivjmpwrgsjwjbfzsuzcdygqkqtwsfald","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eknujyrxjftcdbnisettnbhgemmtgzdy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wusjjuyhfkioqaegtloxkfboigsndifi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddidsgbkheeiryenceibgiolsrtrjgfm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671446,"databaseName":"models_schema","ddl":"CREATE TABLE `tixcpagetyojezbmkaeetpemukblcpsr` (\n `jqruxhwpoxoxyebdozgwghjudbpldvgy` int NOT NULL,\n `pmwvzxgstpaunqbryridioqamatrifvf` int DEFAULT NULL,\n `hbplbnssdcqjwsiniiyhrvbmgatqkphc` int DEFAULT NULL,\n `nucojgxelmnmnqceqbtcenerjzqmtksr` int DEFAULT NULL,\n `ypqkvbwrqlwrwzllhxyyzeiqdyidpdxa` int DEFAULT NULL,\n `rqkrieebwietonwefqkdluotxvcaolla` int DEFAULT NULL,\n `rkryilvasxsjtdwmnjodgdnldlwlolzn` int DEFAULT NULL,\n `zdybqpsrfxzzyxipiwvtzqphmvzejpbp` int DEFAULT NULL,\n `urrdxeuzlxwuurxsxqbrgdrsxlerncak` int DEFAULT NULL,\n `hecmizwfcicumoylteiokzndmoiimxev` int DEFAULT NULL,\n `oqulqmisdrqurtkcmktvyalfjjivtars` int DEFAULT NULL,\n `dmyrgokvkpuqghfttrntqgtoavzpbdfc` int DEFAULT NULL,\n `xnjjlrtwxpcmaugnrofhngubsoohzhag` int DEFAULT NULL,\n `njxdlxtlnmvnlfitsnnapjgolfmpyghr` int DEFAULT NULL,\n `zrnkawplksducwcjxxwtsjdieeplgoej` int DEFAULT NULL,\n `pgjukpqudcaxfoxtbrugzifpgukqlqmv` int DEFAULT NULL,\n `fvfuadueyhcwtwtgxhldgfrdmguwtiiw` int DEFAULT NULL,\n `jbbsruqssskwbnvddhmecoymsguduidb` int DEFAULT NULL,\n `ttnyputdsqrutjiiuotsobihksocjcsc` int DEFAULT NULL,\n `jzqcnrwgixtkawvuyobuvkrtqybgzrzp` int DEFAULT NULL,\n `mtrgpqsqjepvkebdhssgknqglcnbqjim` int DEFAULT NULL,\n `hindbagiyohudqiexzokkgjneblcnqtn` int DEFAULT NULL,\n `ujtkpduhtpdupesjpqjrhmvikkwkmfrm` int DEFAULT NULL,\n `qoghzrpxdoirbtyqkqwyjuwpdzzunufm` int DEFAULT NULL,\n `dcniqygsysqyaawauytozdwqaoegsnvj` int DEFAULT NULL,\n `otcwbqcspabouxtakaeexanxddtldydz` int DEFAULT NULL,\n `ouunvdjqrjseqaoaycauwndkrzggnlcd` int DEFAULT NULL,\n `vhwiuckxrebambyybitxthtlihbpdgan` int DEFAULT NULL,\n `xaylnymrocjkflotwscixtkjamdbfahy` int DEFAULT NULL,\n `omanfpdyozrbzhofykqxnjrwwidqhqah` int DEFAULT NULL,\n `yalfsvoqcmpylkqzppdviakkhyxmejqo` int DEFAULT NULL,\n `exxygiklogbvqxjmuqycpklhcxdkcdnq` int DEFAULT NULL,\n `xndeijjxvswqlzefdfgfbmpedafnmvbj` int DEFAULT NULL,\n `drnpyoctwhjnlabfempajolgseralzau` int DEFAULT NULL,\n `jpwfttzksuhmfnevveqmpvnlhyobzypv` int DEFAULT NULL,\n `mgovyhgibeeszpwcwvdggqligsjuawww` int DEFAULT NULL,\n `sebqodfcnufpacorlmvoagomfimuxdzq` int DEFAULT NULL,\n `gixwfxittbbswwghsdrnepllxojghabn` int DEFAULT NULL,\n `hlvscqrrtwwiavacwduzmxragwcwepuf` int DEFAULT NULL,\n `xtkobrovvjpvzwvrnrileeeeesqlahyc` int DEFAULT NULL,\n `mgywzfuhlrhifpounupvotmuibrtdgqf` int DEFAULT NULL,\n `ltdwdnyxnotbeyawebghabkqyitizgba` int DEFAULT NULL,\n `gjzpobtbwgmxjfwymmxcfvwtbllhulzi` int DEFAULT NULL,\n `tehigersujrejatwcfdazofuhkpvdyrs` int DEFAULT NULL,\n `knojeyacamyxbhrfrtxdtqvesshjrvzq` int DEFAULT NULL,\n `hspdddyksnpeoynpssaoqcacsqbdbsjf` int DEFAULT NULL,\n `wlbrmehwqjueijiulgaodincewgwuyqy` int DEFAULT NULL,\n `wyltqtobvlerphyzzgghxtsnakbcmave` int DEFAULT NULL,\n `pqqmjwviuqkgkujrlahhjxvkjaybinec` int DEFAULT NULL,\n `wivkflonzioknosggiyjpyosixohmdmi` int DEFAULT NULL,\n `vdjygkrtsbnltckeswmhphqflkhaycff` int DEFAULT NULL,\n `jintepbfqtqosaeaghhdthrksgusgmsq` int DEFAULT NULL,\n `mpcatjutarysvzdqxnqxjubosqmauzdv` int DEFAULT NULL,\n `dkxtcmmvgkfqglhehcaeyltxucymzswu` int DEFAULT NULL,\n `rvmnzdttwyfknznmhnklesxkpkvewvte` int DEFAULT NULL,\n `wgxpkxnxvfdsnicimlpapxwtswvfxqyv` int DEFAULT NULL,\n `hygnyuvzqwnpknnugjsjdqiyqmpvqyfi` int DEFAULT NULL,\n `fqxsgncnnewamepumlmcawxqytbvjzmi` int DEFAULT NULL,\n `mobmhyimfttmxphxdivsojlhrgvsxewu` int DEFAULT NULL,\n `hzhxqhasqzezqggdzvbioukdmkogtcyw` int DEFAULT NULL,\n `daclfspbqvsdwvlhlplhvwelozgpwxof` int DEFAULT NULL,\n `hyrlsgadnmxexunoayddvlwxjdjgkljw` int DEFAULT NULL,\n `iotrbkzmcxlcnivckyctugoqzadezcod` int DEFAULT NULL,\n `cofyutruypmmvgkjvmltckuqiduddwky` int DEFAULT NULL,\n `hfdzanskkqokbwzsxwhsvwzalpwtzzvi` int DEFAULT NULL,\n `mhwcgxklcowtulcnvqoyyntqgivmytqa` int DEFAULT NULL,\n `nezwkkmcpahzkrfezwcwzmbxxhkhbrgw` int DEFAULT NULL,\n `ococwtqdeepsmmijycpqpvvzpdethlxg` int DEFAULT NULL,\n `opxdismgrkqkgdvkfewmutfsehtvsbqj` int DEFAULT NULL,\n `wueldprufzuinstjntsyrqqoogchogfw` int DEFAULT NULL,\n `qhxcjvcuqyoiluqzjsswemubnjytcxwj` int DEFAULT NULL,\n `awudffkygddbcbaaoyolbmhwskskvcan` int DEFAULT NULL,\n `bfaimkeznetbxooyvzlkcwrtkcqdgiaj` int DEFAULT NULL,\n `tbokkxikhsthiwkixiblstqxfjlvayfx` int DEFAULT NULL,\n `icolsefjnlfsdhnfmrsfotufdzeuwsna` int DEFAULT NULL,\n `ietftygslxuaivqnmwoklzvbbqsoqayl` int DEFAULT NULL,\n `chjsmimxdrjnfytvkwftcguiiyaajuwu` int DEFAULT NULL,\n `kuqumajgkdnadglvqltsrkvdubrkfwky` int DEFAULT NULL,\n `zkbwuryacxkrizunwvwsmwgsxjjsjpqu` int DEFAULT NULL,\n `dgzwicguinwguqjrpilwpoilizyaqete` int DEFAULT NULL,\n `rxbsylkorjxsascslqgzjeqbqjfasfrn` int DEFAULT NULL,\n `yeuzoetdpwypeblyrxoaiyahkvppjfib` int DEFAULT NULL,\n `yjyfhlfrqwxzpjbxjxvckumayefsnfwy` int DEFAULT NULL,\n `snhsfxzetypbisswszroizfezgcybbsx` int DEFAULT NULL,\n `ffwqqutmtjtxslfkpaiqeqnunwypyuec` int DEFAULT NULL,\n `vdsfvsoxcuubhxydnhtkzkmhmseeyfqs` int DEFAULT NULL,\n `svjlfmfsshvzczakvebccuzjocbhgiqw` int DEFAULT NULL,\n `boxfjrfkqxcvtbrixyuykuvyffwmvxoq` int DEFAULT NULL,\n `nfdffjqpcgufeeszfslgjuhownxzyfkn` int DEFAULT NULL,\n `hlsmydxoianyejpalntwrzjmchlbqvgl` int DEFAULT NULL,\n `ywgndenoxxtpqwtzazcojtyoqqjegyns` int DEFAULT NULL,\n `utapevqvjuyrftcnmectophvqaykirqp` int DEFAULT NULL,\n `vwhnxaewldatswebjxpvepjobrcylcnl` int DEFAULT NULL,\n `jylbzmaijqugyvjviklffrvzxbmcvkax` int DEFAULT NULL,\n `hgixewdjaphwltxkzbjstaymixenaxml` int DEFAULT NULL,\n `vnhaitlihitzbreithshuyiscyylqkxb` int DEFAULT NULL,\n `ghfrhdkkmenyxmkpsewcattvawxdmsfr` int DEFAULT NULL,\n `imaezkgpsdudeaszayzwqswgxcgxapdi` int DEFAULT NULL,\n `ewmfyfisebfitzscyftywrnfhlmeehpr` int DEFAULT NULL,\n `ufznwlhdjkdawrumiladsyavrjzupxuc` int DEFAULT NULL,\n PRIMARY KEY (`jqruxhwpoxoxyebdozgwghjudbpldvgy`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"tixcpagetyojezbmkaeetpemukblcpsr\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["jqruxhwpoxoxyebdozgwghjudbpldvgy"],"columns":[{"name":"jqruxhwpoxoxyebdozgwghjudbpldvgy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"pmwvzxgstpaunqbryridioqamatrifvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbplbnssdcqjwsiniiyhrvbmgatqkphc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nucojgxelmnmnqceqbtcenerjzqmtksr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypqkvbwrqlwrwzllhxyyzeiqdyidpdxa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqkrieebwietonwefqkdluotxvcaolla","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkryilvasxsjtdwmnjodgdnldlwlolzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdybqpsrfxzzyxipiwvtzqphmvzejpbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urrdxeuzlxwuurxsxqbrgdrsxlerncak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hecmizwfcicumoylteiokzndmoiimxev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqulqmisdrqurtkcmktvyalfjjivtars","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dmyrgokvkpuqghfttrntqgtoavzpbdfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnjjlrtwxpcmaugnrofhngubsoohzhag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njxdlxtlnmvnlfitsnnapjgolfmpyghr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zrnkawplksducwcjxxwtsjdieeplgoej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgjukpqudcaxfoxtbrugzifpgukqlqmv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fvfuadueyhcwtwtgxhldgfrdmguwtiiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbbsruqssskwbnvddhmecoymsguduidb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ttnyputdsqrutjiiuotsobihksocjcsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzqcnrwgixtkawvuyobuvkrtqybgzrzp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtrgpqsqjepvkebdhssgknqglcnbqjim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hindbagiyohudqiexzokkgjneblcnqtn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujtkpduhtpdupesjpqjrhmvikkwkmfrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qoghzrpxdoirbtyqkqwyjuwpdzzunufm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcniqygsysqyaawauytozdwqaoegsnvj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otcwbqcspabouxtakaeexanxddtldydz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ouunvdjqrjseqaoaycauwndkrzggnlcd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhwiuckxrebambyybitxthtlihbpdgan","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xaylnymrocjkflotwscixtkjamdbfahy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omanfpdyozrbzhofykqxnjrwwidqhqah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yalfsvoqcmpylkqzppdviakkhyxmejqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exxygiklogbvqxjmuqycpklhcxdkcdnq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xndeijjxvswqlzefdfgfbmpedafnmvbj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drnpyoctwhjnlabfempajolgseralzau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpwfttzksuhmfnevveqmpvnlhyobzypv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mgovyhgibeeszpwcwvdggqligsjuawww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sebqodfcnufpacorlmvoagomfimuxdzq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gixwfxittbbswwghsdrnepllxojghabn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlvscqrrtwwiavacwduzmxragwcwepuf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xtkobrovvjpvzwvrnrileeeeesqlahyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mgywzfuhlrhifpounupvotmuibrtdgqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ltdwdnyxnotbeyawebghabkqyitizgba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjzpobtbwgmxjfwymmxcfvwtbllhulzi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tehigersujrejatwcfdazofuhkpvdyrs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knojeyacamyxbhrfrtxdtqvesshjrvzq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hspdddyksnpeoynpssaoqcacsqbdbsjf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wlbrmehwqjueijiulgaodincewgwuyqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyltqtobvlerphyzzgghxtsnakbcmave","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqqmjwviuqkgkujrlahhjxvkjaybinec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wivkflonzioknosggiyjpyosixohmdmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdjygkrtsbnltckeswmhphqflkhaycff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jintepbfqtqosaeaghhdthrksgusgmsq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpcatjutarysvzdqxnqxjubosqmauzdv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkxtcmmvgkfqglhehcaeyltxucymzswu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rvmnzdttwyfknznmhnklesxkpkvewvte","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgxpkxnxvfdsnicimlpapxwtswvfxqyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hygnyuvzqwnpknnugjsjdqiyqmpvqyfi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqxsgncnnewamepumlmcawxqytbvjzmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mobmhyimfttmxphxdivsojlhrgvsxewu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hzhxqhasqzezqggdzvbioukdmkogtcyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"daclfspbqvsdwvlhlplhvwelozgpwxof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hyrlsgadnmxexunoayddvlwxjdjgkljw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iotrbkzmcxlcnivckyctugoqzadezcod","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cofyutruypmmvgkjvmltckuqiduddwky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfdzanskkqokbwzsxwhsvwzalpwtzzvi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhwcgxklcowtulcnvqoyyntqgivmytqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nezwkkmcpahzkrfezwcwzmbxxhkhbrgw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ococwtqdeepsmmijycpqpvvzpdethlxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opxdismgrkqkgdvkfewmutfsehtvsbqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wueldprufzuinstjntsyrqqoogchogfw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhxcjvcuqyoiluqzjsswemubnjytcxwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awudffkygddbcbaaoyolbmhwskskvcan","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bfaimkeznetbxooyvzlkcwrtkcqdgiaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbokkxikhsthiwkixiblstqxfjlvayfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icolsefjnlfsdhnfmrsfotufdzeuwsna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ietftygslxuaivqnmwoklzvbbqsoqayl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chjsmimxdrjnfytvkwftcguiiyaajuwu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kuqumajgkdnadglvqltsrkvdubrkfwky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zkbwuryacxkrizunwvwsmwgsxjjsjpqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgzwicguinwguqjrpilwpoilizyaqete","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxbsylkorjxsascslqgzjeqbqjfasfrn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yeuzoetdpwypeblyrxoaiyahkvppjfib","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjyfhlfrqwxzpjbxjxvckumayefsnfwy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"snhsfxzetypbisswszroizfezgcybbsx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffwqqutmtjtxslfkpaiqeqnunwypyuec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdsfvsoxcuubhxydnhtkzkmhmseeyfqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"svjlfmfsshvzczakvebccuzjocbhgiqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"boxfjrfkqxcvtbrixyuykuvyffwmvxoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfdffjqpcgufeeszfslgjuhownxzyfkn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlsmydxoianyejpalntwrzjmchlbqvgl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywgndenoxxtpqwtzazcojtyoqqjegyns","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utapevqvjuyrftcnmectophvqaykirqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vwhnxaewldatswebjxpvepjobrcylcnl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jylbzmaijqugyvjviklffrvzxbmcvkax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hgixewdjaphwltxkzbjstaymixenaxml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnhaitlihitzbreithshuyiscyylqkxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghfrhdkkmenyxmkpsewcattvawxdmsfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"imaezkgpsdudeaszayzwqswgxcgxapdi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ewmfyfisebfitzscyftywrnfhlmeehpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufznwlhdjkdawrumiladsyavrjzupxuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671483,"databaseName":"models_schema","ddl":"CREATE TABLE `tktrwfqkbwagctiwqbgzfdyjkzxqxrnt` (\n `hjzjrtgfehfrnyvwypipnxqjxhdqluim` int NOT NULL,\n `lovcgfqzgabybxfbgibjpvzbdqjozyko` int DEFAULT NULL,\n `senddobufkwdceetfdvpakeiydryhdsl` int DEFAULT NULL,\n `bjkvkpzmokgrmswlcgqrmqehhptfnqll` int DEFAULT NULL,\n `ixgmbowbeuukomzlyoydkwndpmdsyepe` int DEFAULT NULL,\n `wnfjsefmxxjebwywdqfgdpfqicvfrhsn` int DEFAULT NULL,\n `mltesqkxdeejzgksrcjyqubbsttomdak` int DEFAULT NULL,\n `mmzltexxixoyauoqnsqilruemcgmsnua` int DEFAULT NULL,\n `yrfithqlcczwxksazyycsqvhbfyubqly` int DEFAULT NULL,\n `dutwcstyanwuagftunfstaswrorzmqyy` int DEFAULT NULL,\n `wgferknhhpuveyvevorlcfgslgpuofmz` int DEFAULT NULL,\n `xfwfbjbsgvrjpnfaealarkuxlplnjfra` int DEFAULT NULL,\n `blmuixytsgwftqxdoqlishuflaasuwgb` int DEFAULT NULL,\n `epvvekobbjadnecsopxdqgrrxsrdyqsy` int DEFAULT NULL,\n `vzicygscovzpgaxnykiiefcnjydupsbb` int DEFAULT NULL,\n `nykrdfqjrpvdormvcgatwpxtxxzvqdmb` int DEFAULT NULL,\n `vgusixghdexeownyltyvumbykhuxvsnm` int DEFAULT NULL,\n `pyizixlvesaochoxnycbfyofkoikvjfg` int DEFAULT NULL,\n `dlkkvfenbjenqofgmawsmmaglespjqmi` int DEFAULT NULL,\n `poldergqwsxiguhxmyuywklbynwrjstf` int DEFAULT NULL,\n `giqutnagxcwfolrbkrnxzursrzzuahtq` int DEFAULT NULL,\n `igellwnxunsoyhqlysnluluduqbjlbqo` int DEFAULT NULL,\n `omidwlwviaewswdrxausronhysdvexok` int DEFAULT NULL,\n `nnwwbrosoootkiofydwbekfmnfbrytjs` int DEFAULT NULL,\n `exrmqugnoaoiyileudjzkgdsbeeucfpp` int DEFAULT NULL,\n `wbnxdgkbnyxgswfvkfidqsvddejniqxa` int DEFAULT NULL,\n `bqgkxbcihjwjdugbxrhtjjvttgfdnhnm` int DEFAULT NULL,\n `vslckltbpohvnalnpuujgpabiogmxtnf` int DEFAULT NULL,\n `gutgkmcduwjxfrihngqnmmukvfsefvcm` int DEFAULT NULL,\n `liszzffffbgjltyadoknifvmmwemknam` int DEFAULT NULL,\n `uoomefqlzrihuemfmcjsuoetbtfgvgln` int DEFAULT NULL,\n `vhmmivtgjyitgbhirrlipmjivncmgyoo` int DEFAULT NULL,\n `qmghvrnbiphtkfbdqvoditijwrwzbbxq` int DEFAULT NULL,\n `xyvuyyyywjtlqpfkesyhvaummgpqaskd` int DEFAULT NULL,\n `jfthvfbhufqkpdgwcatavggeymqiziqw` int DEFAULT NULL,\n `yxyujccckirkcgymfpxdxgykwkibwxxz` int DEFAULT NULL,\n `pjhwliilpblykmasyeggvmmqgjbivewn` int DEFAULT NULL,\n `swnbsprxsrkjifayarhceyzzsgfuzcwe` int DEFAULT NULL,\n `dwqzdfbdpyrnhxzjpjjrictttxbcotwg` int DEFAULT NULL,\n `foxctpmzlpppcewvvmovviydlpifoohm` int DEFAULT NULL,\n `jqthtvgdiemcpljjxsgjawdxgcttnust` int DEFAULT NULL,\n `iqqcbjzpnysnqjdymqnhrnykmnyhmrwb` int DEFAULT NULL,\n `czfoogolobdemetwkohjfasqiyigzpdx` int DEFAULT NULL,\n `vmbfxwklurifluvmsojqsoqbkoypnxas` int DEFAULT NULL,\n `txdgeegmkmmopqirbuxafbrjhykpekql` int DEFAULT NULL,\n `ouixjaybjtkfeenlawjotfzqxjyvjlwb` int DEFAULT NULL,\n `xtcyureygkqdlizipnudvilyhbziolyb` int DEFAULT NULL,\n `inwivbopasspusykygxubddjfduyuddd` int DEFAULT NULL,\n `cwwztulkgoepxdtiqjuirhzvwztfzblu` int DEFAULT NULL,\n `jauqpkngvijctiwcezfjnfkboelpqwef` int DEFAULT NULL,\n `xoublbvdicddzvhpzulfzlyqzopipsbk` int DEFAULT NULL,\n `rtofqwtgyloetdkndjpyltydmhgfwktx` int DEFAULT NULL,\n `sdidebpedsmgnbzyzxrcexpinclcqgee` int DEFAULT NULL,\n `hgjlkqedacqozvxuhddimicocgcdwbtd` int DEFAULT NULL,\n `nbbynbtgwoaugsyhyttdhlufmpbevvzl` int DEFAULT NULL,\n `vybwrfzebpvlplmbqybopwgihdsmskif` int DEFAULT NULL,\n `ofvigfmpefqowimuayjeajcjbfengxbe` int DEFAULT NULL,\n `qxjskaevpmvzbqyqbduxojntdupzqhqr` int DEFAULT NULL,\n `zzujnetkaudixcndkevauxycedrlyisi` int DEFAULT NULL,\n `ejwmcgnfvqmktlcxtdqozurtsnwtuoyu` int DEFAULT NULL,\n `dfksrdujagtoufotdusuaheyweoqgyny` int DEFAULT NULL,\n `afkkwnsqmjdianwryzejpizxgggombov` int DEFAULT NULL,\n `xbbvxbmqblmmiikzrkuhaiziglfyfmyi` int DEFAULT NULL,\n `yzwfaoykkrhaisfykryovjytwxppbpxi` int DEFAULT NULL,\n `vdkzglsnqubpiqnuappmivqobtbciwqg` int DEFAULT NULL,\n `spsnnadppcmiwtoaaykibqfamrbblrch` int DEFAULT NULL,\n `rfpnfncuimhtbkvnngdatnpiytzsbhxs` int DEFAULT NULL,\n `wamjugldcnuqwoscehpbbmzguzmybqae` int DEFAULT NULL,\n `wdhhyqzxzpiyrbiqrgpwtzghizmltrul` int DEFAULT NULL,\n `rcjqfsfeballpdyambbmiykyzsqshdaw` int DEFAULT NULL,\n `vptjdbevgihvbeloycfrauiknunycmyv` int DEFAULT NULL,\n `ewyzuxqtdxuqkmygxtssidwjacuykdlw` int DEFAULT NULL,\n `bslvjablztsmpsmzruphkenhqrbrctoo` int DEFAULT NULL,\n `viczyizbcturwmfyrxdjzpnmicovtaaw` int DEFAULT NULL,\n `haybzgwneuncssyvifmrkqlwjbumhcbx` int DEFAULT NULL,\n `olvgttwgqsjqmrxuxfsziblelyearhfe` int DEFAULT NULL,\n `wzbzjosvmspozbhwbdhdynfqbjvecyfs` int DEFAULT NULL,\n `ybirfuhshregdidsuowjfpeywgmibjid` int DEFAULT NULL,\n `akbnfbssgxdfilzagtldnbjgqtkwuwza` int DEFAULT NULL,\n `vzchmifuckflabfqksjsbhjswqpohyly` int DEFAULT NULL,\n `tjgeruqtsezlmvvyxnfylbzitggokrax` int DEFAULT NULL,\n `bgznfoyvcsrfufizfjbuyepdzcredvru` int DEFAULT NULL,\n `rgquahuvpcrdnkueutuaumiohgbgyvvl` int DEFAULT NULL,\n `hvxxkdaxgpxkbozyqnyiaamtkastzdxx` int DEFAULT NULL,\n `bgnkygpnazvcghyelynnmueweutzczhs` int DEFAULT NULL,\n `supnzaswdgpwwiboldcswwscqrfvmclz` int DEFAULT NULL,\n `nkhwrxkogrnxluvkwsroocwvhcgtjieo` int DEFAULT NULL,\n `dzzikecbuuxuefsuxtzmqyriphghondt` int DEFAULT NULL,\n `iowslnamyqlobudtugkhsxhmangjfyft` int DEFAULT NULL,\n `nagnpdbaqgdhrwlplextoxuqgowzhfeq` int DEFAULT NULL,\n `xhectesihnycvmlheepdmnosvvhjsjmo` int DEFAULT NULL,\n `yneadkzaskiuottkxhixgkakqudhiyol` int DEFAULT NULL,\n `cujflytzlqustlkmpxennzsfcuyihioo` int DEFAULT NULL,\n `cqyqslkqejuwzftidkqfutkitvatkhei` int DEFAULT NULL,\n `njnhujrlyzeesousekhujzyjacqedyzn` int DEFAULT NULL,\n `eqmbqiaklmcdiczvjmwnauczzgancaji` int DEFAULT NULL,\n `snxqtfifyfrrvbirlgafpkjsnloojvbo` int DEFAULT NULL,\n `tnavxaynknkueaavjmxftuiarmhhxyrr` int DEFAULT NULL,\n `ahvnpaascxazlsvdzvxjfzrvtlvamhzw` int DEFAULT NULL,\n `esfzthsmcreotgofdfrxebysthhwveim` int DEFAULT NULL,\n PRIMARY KEY (`hjzjrtgfehfrnyvwypipnxqjxhdqluim`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"tktrwfqkbwagctiwqbgzfdyjkzxqxrnt\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["hjzjrtgfehfrnyvwypipnxqjxhdqluim"],"columns":[{"name":"hjzjrtgfehfrnyvwypipnxqjxhdqluim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"lovcgfqzgabybxfbgibjpvzbdqjozyko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"senddobufkwdceetfdvpakeiydryhdsl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjkvkpzmokgrmswlcgqrmqehhptfnqll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixgmbowbeuukomzlyoydkwndpmdsyepe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnfjsefmxxjebwywdqfgdpfqicvfrhsn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mltesqkxdeejzgksrcjyqubbsttomdak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmzltexxixoyauoqnsqilruemcgmsnua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yrfithqlcczwxksazyycsqvhbfyubqly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dutwcstyanwuagftunfstaswrorzmqyy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgferknhhpuveyvevorlcfgslgpuofmz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xfwfbjbsgvrjpnfaealarkuxlplnjfra","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blmuixytsgwftqxdoqlishuflaasuwgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"epvvekobbjadnecsopxdqgrrxsrdyqsy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzicygscovzpgaxnykiiefcnjydupsbb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nykrdfqjrpvdormvcgatwpxtxxzvqdmb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgusixghdexeownyltyvumbykhuxvsnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pyizixlvesaochoxnycbfyofkoikvjfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlkkvfenbjenqofgmawsmmaglespjqmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"poldergqwsxiguhxmyuywklbynwrjstf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"giqutnagxcwfolrbkrnxzursrzzuahtq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igellwnxunsoyhqlysnluluduqbjlbqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omidwlwviaewswdrxausronhysdvexok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nnwwbrosoootkiofydwbekfmnfbrytjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exrmqugnoaoiyileudjzkgdsbeeucfpp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbnxdgkbnyxgswfvkfidqsvddejniqxa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqgkxbcihjwjdugbxrhtjjvttgfdnhnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vslckltbpohvnalnpuujgpabiogmxtnf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gutgkmcduwjxfrihngqnmmukvfsefvcm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"liszzffffbgjltyadoknifvmmwemknam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uoomefqlzrihuemfmcjsuoetbtfgvgln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhmmivtgjyitgbhirrlipmjivncmgyoo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmghvrnbiphtkfbdqvoditijwrwzbbxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyvuyyyywjtlqpfkesyhvaummgpqaskd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jfthvfbhufqkpdgwcatavggeymqiziqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxyujccckirkcgymfpxdxgykwkibwxxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjhwliilpblykmasyeggvmmqgjbivewn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swnbsprxsrkjifayarhceyzzsgfuzcwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwqzdfbdpyrnhxzjpjjrictttxbcotwg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"foxctpmzlpppcewvvmovviydlpifoohm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jqthtvgdiemcpljjxsgjawdxgcttnust","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqqcbjzpnysnqjdymqnhrnykmnyhmrwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"czfoogolobdemetwkohjfasqiyigzpdx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmbfxwklurifluvmsojqsoqbkoypnxas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txdgeegmkmmopqirbuxafbrjhykpekql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ouixjaybjtkfeenlawjotfzqxjyvjlwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xtcyureygkqdlizipnudvilyhbziolyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"inwivbopasspusykygxubddjfduyuddd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwwztulkgoepxdtiqjuirhzvwztfzblu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jauqpkngvijctiwcezfjnfkboelpqwef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xoublbvdicddzvhpzulfzlyqzopipsbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtofqwtgyloetdkndjpyltydmhgfwktx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdidebpedsmgnbzyzxrcexpinclcqgee","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hgjlkqedacqozvxuhddimicocgcdwbtd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbbynbtgwoaugsyhyttdhlufmpbevvzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vybwrfzebpvlplmbqybopwgihdsmskif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofvigfmpefqowimuayjeajcjbfengxbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxjskaevpmvzbqyqbduxojntdupzqhqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzujnetkaudixcndkevauxycedrlyisi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejwmcgnfvqmktlcxtdqozurtsnwtuoyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dfksrdujagtoufotdusuaheyweoqgyny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afkkwnsqmjdianwryzejpizxgggombov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbbvxbmqblmmiikzrkuhaiziglfyfmyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzwfaoykkrhaisfykryovjytwxppbpxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdkzglsnqubpiqnuappmivqobtbciwqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"spsnnadppcmiwtoaaykibqfamrbblrch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfpnfncuimhtbkvnngdatnpiytzsbhxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wamjugldcnuqwoscehpbbmzguzmybqae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wdhhyqzxzpiyrbiqrgpwtzghizmltrul","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcjqfsfeballpdyambbmiykyzsqshdaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vptjdbevgihvbeloycfrauiknunycmyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ewyzuxqtdxuqkmygxtssidwjacuykdlw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bslvjablztsmpsmzruphkenhqrbrctoo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"viczyizbcturwmfyrxdjzpnmicovtaaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"haybzgwneuncssyvifmrkqlwjbumhcbx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"olvgttwgqsjqmrxuxfsziblelyearhfe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzbzjosvmspozbhwbdhdynfqbjvecyfs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybirfuhshregdidsuowjfpeywgmibjid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akbnfbssgxdfilzagtldnbjgqtkwuwza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzchmifuckflabfqksjsbhjswqpohyly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjgeruqtsezlmvvyxnfylbzitggokrax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bgznfoyvcsrfufizfjbuyepdzcredvru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgquahuvpcrdnkueutuaumiohgbgyvvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hvxxkdaxgpxkbozyqnyiaamtkastzdxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bgnkygpnazvcghyelynnmueweutzczhs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"supnzaswdgpwwiboldcswwscqrfvmclz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkhwrxkogrnxluvkwsroocwvhcgtjieo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dzzikecbuuxuefsuxtzmqyriphghondt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iowslnamyqlobudtugkhsxhmangjfyft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nagnpdbaqgdhrwlplextoxuqgowzhfeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xhectesihnycvmlheepdmnosvvhjsjmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yneadkzaskiuottkxhixgkakqudhiyol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cujflytzlqustlkmpxennzsfcuyihioo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqyqslkqejuwzftidkqfutkitvatkhei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njnhujrlyzeesousekhujzyjacqedyzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqmbqiaklmcdiczvjmwnauczzgancaji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"snxqtfifyfrrvbirlgafpkjsnloojvbo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnavxaynknkueaavjmxftuiarmhhxyrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahvnpaascxazlsvdzvxjfzrvtlvamhzw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"esfzthsmcreotgofdfrxebysthhwveim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671513,"databaseName":"models_schema","ddl":"CREATE TABLE `tlawbzmwhcjjoudhkxvxmsfrdeygughc` (\n `ztrhbcxcjoedtfrykjakfwewdyewqvlg` int NOT NULL,\n `ozrmxqxvlpzypnpabybibvisjnkalwax` int DEFAULT NULL,\n `letuvyaxylsqbdscflgqgcoqclwvukew` int DEFAULT NULL,\n `qwlquhszxygyrwtiwzamtxxwjjmflzmo` int DEFAULT NULL,\n `goqouxbmjdtpquyrkcsvbqdxmqmysoyg` int DEFAULT NULL,\n `scbkijmewcdpretzzmopydhwruwyekmt` int DEFAULT NULL,\n `sykmfekhrembaoqfjncqazzkjgdipbnw` int DEFAULT NULL,\n `zyuillyijdpprehreijlmqfbbailxsoj` int DEFAULT NULL,\n `gmxbuotbvzfqvqczcnooyqwsvtbhxtfa` int DEFAULT NULL,\n `hmqxnorxrmrqveezenamdxrhgtqkolri` int DEFAULT NULL,\n `wzzthjihcoqalmyvktyasixtqeuzmgcq` int DEFAULT NULL,\n `gqrjetmeygytqzbpfuumkhoboutrrlsz` int DEFAULT NULL,\n `eyqvvjigtbbqkihvwhkydrhsyuwivcyz` int DEFAULT NULL,\n `fnyegexyvgxtxtupxtcrjdnmvabxdktv` int DEFAULT NULL,\n `jkkkwndewuzlmhzcbbjwzbkzuhxqngbu` int DEFAULT NULL,\n `vxmimhytmggkbcdvcfyrxpsvjjshorbb` int DEFAULT NULL,\n `vrsrgadlvodzahhsnoyhvtpkxvivwkdr` int DEFAULT NULL,\n `iydqaimulcossipftawikkepebxouhxf` int DEFAULT NULL,\n `vndbntfbbzrxvqwjxfhbgrbdttwkioca` int DEFAULT NULL,\n `cqoruwahdlzdlnorbflgyjixnsbdqybr` int DEFAULT NULL,\n `kpcwzxdykkmkroigyiheukexiscucucz` int DEFAULT NULL,\n `wyvezxrpqweilaqbiamgklaobnblyaqh` int DEFAULT NULL,\n `zhnvpsugzfqchprisffhdoxkmagwbxud` int DEFAULT NULL,\n `lybgiztujjgcbdqqlwqryzpsaxpgaybb` int DEFAULT NULL,\n `fbdfzxwtqlobozscnjdeatzcmjfchxhi` int DEFAULT NULL,\n `xmlrdygodrfjhlmvnyngpuezmfmplrif` int DEFAULT NULL,\n `hruolmojnwrtgevmmqkhaaqugqesqykr` int DEFAULT NULL,\n `htkrjsogyanlicwivzwtzuqdhjwpfocd` int DEFAULT NULL,\n `jofjcxqkwvdujvhbrrgsgogyijgnbfny` int DEFAULT NULL,\n `zezbojdhfzrlfiqsqvqcxskhavotpjtl` int DEFAULT NULL,\n `nptpqhpsbkgbhgkmovdwejhaiithpqrn` int DEFAULT NULL,\n `ijvnlyfrluyozdwuhaszyozrvvukslih` int DEFAULT NULL,\n `kvmfsxjpzhitkhevtehdpaurikcgrzjd` int DEFAULT NULL,\n `diwzdadsykcvluadktsuumgvngwwujgt` int DEFAULT NULL,\n `pfwlkhtbecufxeodaecoprskowhdmbas` int DEFAULT NULL,\n `kvlwdnqjxaohxhzccszsnkdroxwvbhrr` int DEFAULT NULL,\n `kzezhtbzlpasceronnoyzpyoivlbjjjs` int DEFAULT NULL,\n `kdriuhaeeqtnjyjtgmudsxefniruxjut` int DEFAULT NULL,\n `vpupgushxunugxjhgyuzcydrtrlhvixv` int DEFAULT NULL,\n `lhpntmaqcjkuoabeilgmnicfzvimzapr` int DEFAULT NULL,\n `wwnnaxzkzonhistngemsyghlggmkikco` int DEFAULT NULL,\n `ordrtjgawwpdrqpvdghrptkowueagrgi` int DEFAULT NULL,\n `uofbhbghdtiltrytujbadybrpivnmyec` int DEFAULT NULL,\n `ucpapldecbarrtagdzrbeouzlfgboags` int DEFAULT NULL,\n `wlmdgfduljbanqjxlpfjrajwhykxquxy` int DEFAULT NULL,\n `zvmcixwfgpgfgopexxosfhhkvdjyqfll` int DEFAULT NULL,\n `nzuokkdmqyfcfekmqfmhrxljalxgsxhc` int DEFAULT NULL,\n `abxqaoodcdftvzbecestdprysrvvqnvl` int DEFAULT NULL,\n `vueiaugvhfohhgxoqeylxdvznlesmxtc` int DEFAULT NULL,\n `ykrjcsexsmjiqcksaqtixgwvkygjpuze` int DEFAULT NULL,\n `fcytkwohumyrblrsyjwbbobkiryphelw` int DEFAULT NULL,\n `vqlchcbfmwdlvudocqrmffvywioalkxu` int DEFAULT NULL,\n `atqwqxzdxftelgkktneayqmsautdblcv` int DEFAULT NULL,\n `jzaindfhpwwyvkvgsvsqiufgqjquzqna` int DEFAULT NULL,\n `hkvxhvlsddelxwtbscyanrfwtykdrfrr` int DEFAULT NULL,\n `uikxqwtnmwybaztuncopdkcfkawlpxjk` int DEFAULT NULL,\n `pajxyktdvmjhnaalmvnockahvxlkogvo` int DEFAULT NULL,\n `ocgmtfjsilbngjhgyqcnzkyjgcdxvkgn` int DEFAULT NULL,\n `cjrhztoqzcykhcwqstuggygydudttohf` int DEFAULT NULL,\n `rksnelegfjgwumskgltrnzuklxlbiwqj` int DEFAULT NULL,\n `qcfsyfgrvhwlmiaiwzgscsythbkmbymc` int DEFAULT NULL,\n `isvoxpuvdgvzvkdfqgmtqxyvkcqiukmx` int DEFAULT NULL,\n `brjyvsoqclbykakbyfyftibarmtyhifn` int DEFAULT NULL,\n `ooebgmmllpauukqvglitifjkvpwcxesb` int DEFAULT NULL,\n `ctbsvjnhxuphgssjniyifwoslncduhvz` int DEFAULT NULL,\n `veuvippfanwfueoqqadxorzxpjzkrycw` int DEFAULT NULL,\n `jecallfmukfujbqjhpsmmoxkkcladprc` int DEFAULT NULL,\n `qgjesvfnhmnieohftpredyrokjvxwxby` int DEFAULT NULL,\n `vqlkyjvbmiaqwfvcwmvrazyfuhlswteu` int DEFAULT NULL,\n `sirkynnnngtfjtowpuwvkslsvogyfxsk` int DEFAULT NULL,\n `shxuvtunljvaezplizbpjmwffyqyhiaf` int DEFAULT NULL,\n `xwkwlpzyhbnhjuzslrbiyvbydmxuuszq` int DEFAULT NULL,\n `yhzwatkaskysbmahveyfmcprukrpzrqn` int DEFAULT NULL,\n `zumlskopjriuksobqmdxvzeoeejfikmx` int DEFAULT NULL,\n `ghzyddthmekynrasadukadpepufwlogm` int DEFAULT NULL,\n `snvnsdaaokseavavhfmkiyiemrfvodrr` int DEFAULT NULL,\n `fzautemrqwjhdafpsqnfhpluirdfphfj` int DEFAULT NULL,\n `sigtmzttzmiqeviiqssrxvzcemwkctho` int DEFAULT NULL,\n `udcepbediygjdoyakxpictpydqizwxvr` int DEFAULT NULL,\n `xsfgtgvcjysmtnnnuteqqljmzymuhawi` int DEFAULT NULL,\n `tqrwssvmsbfgrtrlixwgyahhufljnedx` int DEFAULT NULL,\n `xovmdfcvrkgparnyetjjwuuoflkggppk` int DEFAULT NULL,\n `odkiefxlyjxdldgebnanlebfwszodmmi` int DEFAULT NULL,\n `dpymeblgyhvqqarukjzivnpqxhykiyhw` int DEFAULT NULL,\n `wltthgimdrpgdhphxyrmszeemjiwvckw` int DEFAULT NULL,\n `nunrckdktbkwedrtuzkgdwumaojksexr` int DEFAULT NULL,\n `jvqfavfhdefeizxhlgslmyjgcnsalhqp` int DEFAULT NULL,\n `weabtuedvicxzimbfblgwnvhbrcwsxma` int DEFAULT NULL,\n `iofqpcseczwmxgydarhxizcdtkyryawl` int DEFAULT NULL,\n `xctxqnuftebiqelwajhuimuesqiqyqrp` int DEFAULT NULL,\n `mdpxchkiosokepcfzyuhxwylcnujuypd` int DEFAULT NULL,\n `cmofnpsjgyydjfvkfbulrilkxvcwnevt` int DEFAULT NULL,\n `uhxhbgciupsljbqlthbyvcrinpzyhqse` int DEFAULT NULL,\n `vwxkmkgouomclmnzcpeacchgrcuzhrdr` int DEFAULT NULL,\n `ijvjhjnzymwjfjnskmuioitxymaqunhv` int DEFAULT NULL,\n `gjlyqujfobzimdrvztpakkfcgrccgika` int DEFAULT NULL,\n `pojokhmtpwhvdolxzavqjvivfqocfkqs` int DEFAULT NULL,\n `fmbrrufqvypwdufluwxloetpyznsonxa` int DEFAULT NULL,\n `ofqxvkpmzluuspeahrircunumcvrkqwj` int DEFAULT NULL,\n `nvhurvcvqcgcktdlmwhihkykrnaakjhe` int DEFAULT NULL,\n PRIMARY KEY (`ztrhbcxcjoedtfrykjakfwewdyewqvlg`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"tlawbzmwhcjjoudhkxvxmsfrdeygughc\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["ztrhbcxcjoedtfrykjakfwewdyewqvlg"],"columns":[{"name":"ztrhbcxcjoedtfrykjakfwewdyewqvlg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ozrmxqxvlpzypnpabybibvisjnkalwax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"letuvyaxylsqbdscflgqgcoqclwvukew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwlquhszxygyrwtiwzamtxxwjjmflzmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"goqouxbmjdtpquyrkcsvbqdxmqmysoyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scbkijmewcdpretzzmopydhwruwyekmt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sykmfekhrembaoqfjncqazzkjgdipbnw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyuillyijdpprehreijlmqfbbailxsoj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmxbuotbvzfqvqczcnooyqwsvtbhxtfa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmqxnorxrmrqveezenamdxrhgtqkolri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzzthjihcoqalmyvktyasixtqeuzmgcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqrjetmeygytqzbpfuumkhoboutrrlsz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eyqvvjigtbbqkihvwhkydrhsyuwivcyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnyegexyvgxtxtupxtcrjdnmvabxdktv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jkkkwndewuzlmhzcbbjwzbkzuhxqngbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxmimhytmggkbcdvcfyrxpsvjjshorbb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrsrgadlvodzahhsnoyhvtpkxvivwkdr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iydqaimulcossipftawikkepebxouhxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vndbntfbbzrxvqwjxfhbgrbdttwkioca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqoruwahdlzdlnorbflgyjixnsbdqybr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpcwzxdykkmkroigyiheukexiscucucz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyvezxrpqweilaqbiamgklaobnblyaqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhnvpsugzfqchprisffhdoxkmagwbxud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lybgiztujjgcbdqqlwqryzpsaxpgaybb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbdfzxwtqlobozscnjdeatzcmjfchxhi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmlrdygodrfjhlmvnyngpuezmfmplrif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hruolmojnwrtgevmmqkhaaqugqesqykr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htkrjsogyanlicwivzwtzuqdhjwpfocd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jofjcxqkwvdujvhbrrgsgogyijgnbfny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zezbojdhfzrlfiqsqvqcxskhavotpjtl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nptpqhpsbkgbhgkmovdwejhaiithpqrn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijvnlyfrluyozdwuhaszyozrvvukslih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvmfsxjpzhitkhevtehdpaurikcgrzjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"diwzdadsykcvluadktsuumgvngwwujgt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pfwlkhtbecufxeodaecoprskowhdmbas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvlwdnqjxaohxhzccszsnkdroxwvbhrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzezhtbzlpasceronnoyzpyoivlbjjjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kdriuhaeeqtnjyjtgmudsxefniruxjut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpupgushxunugxjhgyuzcydrtrlhvixv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhpntmaqcjkuoabeilgmnicfzvimzapr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwnnaxzkzonhistngemsyghlggmkikco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ordrtjgawwpdrqpvdghrptkowueagrgi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uofbhbghdtiltrytujbadybrpivnmyec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucpapldecbarrtagdzrbeouzlfgboags","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wlmdgfduljbanqjxlpfjrajwhykxquxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvmcixwfgpgfgopexxosfhhkvdjyqfll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzuokkdmqyfcfekmqfmhrxljalxgsxhc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abxqaoodcdftvzbecestdprysrvvqnvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vueiaugvhfohhgxoqeylxdvznlesmxtc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykrjcsexsmjiqcksaqtixgwvkygjpuze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcytkwohumyrblrsyjwbbobkiryphelw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqlchcbfmwdlvudocqrmffvywioalkxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atqwqxzdxftelgkktneayqmsautdblcv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzaindfhpwwyvkvgsvsqiufgqjquzqna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkvxhvlsddelxwtbscyanrfwtykdrfrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uikxqwtnmwybaztuncopdkcfkawlpxjk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pajxyktdvmjhnaalmvnockahvxlkogvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocgmtfjsilbngjhgyqcnzkyjgcdxvkgn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjrhztoqzcykhcwqstuggygydudttohf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rksnelegfjgwumskgltrnzuklxlbiwqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qcfsyfgrvhwlmiaiwzgscsythbkmbymc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"isvoxpuvdgvzvkdfqgmtqxyvkcqiukmx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"brjyvsoqclbykakbyfyftibarmtyhifn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ooebgmmllpauukqvglitifjkvpwcxesb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctbsvjnhxuphgssjniyifwoslncduhvz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"veuvippfanwfueoqqadxorzxpjzkrycw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jecallfmukfujbqjhpsmmoxkkcladprc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgjesvfnhmnieohftpredyrokjvxwxby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqlkyjvbmiaqwfvcwmvrazyfuhlswteu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sirkynnnngtfjtowpuwvkslsvogyfxsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shxuvtunljvaezplizbpjmwffyqyhiaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xwkwlpzyhbnhjuzslrbiyvbydmxuuszq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhzwatkaskysbmahveyfmcprukrpzrqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zumlskopjriuksobqmdxvzeoeejfikmx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghzyddthmekynrasadukadpepufwlogm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"snvnsdaaokseavavhfmkiyiemrfvodrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzautemrqwjhdafpsqnfhpluirdfphfj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sigtmzttzmiqeviiqssrxvzcemwkctho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udcepbediygjdoyakxpictpydqizwxvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsfgtgvcjysmtnnnuteqqljmzymuhawi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqrwssvmsbfgrtrlixwgyahhufljnedx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xovmdfcvrkgparnyetjjwuuoflkggppk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"odkiefxlyjxdldgebnanlebfwszodmmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpymeblgyhvqqarukjzivnpqxhykiyhw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wltthgimdrpgdhphxyrmszeemjiwvckw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nunrckdktbkwedrtuzkgdwumaojksexr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvqfavfhdefeizxhlgslmyjgcnsalhqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"weabtuedvicxzimbfblgwnvhbrcwsxma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iofqpcseczwmxgydarhxizcdtkyryawl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xctxqnuftebiqelwajhuimuesqiqyqrp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdpxchkiosokepcfzyuhxwylcnujuypd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmofnpsjgyydjfvkfbulrilkxvcwnevt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhxhbgciupsljbqlthbyvcrinpzyhqse","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vwxkmkgouomclmnzcpeacchgrcuzhrdr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijvjhjnzymwjfjnskmuioitxymaqunhv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjlyqujfobzimdrvztpakkfcgrccgika","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pojokhmtpwhvdolxzavqjvivfqocfkqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmbrrufqvypwdufluwxloetpyznsonxa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofqxvkpmzluuspeahrircunumcvrkqwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvhurvcvqcgcktdlmwhihkykrnaakjhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671545,"databaseName":"models_schema","ddl":"CREATE TABLE `tpksmtlcpcazciywpqiwdqlzxbnoiktq` (\n `nsnsqgwgdsiwoxzbrtpgkcufmurayjcc` int NOT NULL,\n `xvpnrilojzdvqnmdzglpwqrysryhgaet` int DEFAULT NULL,\n `xlbgqpdzlwncmrhyaaamgbfhgqpiybal` int DEFAULT NULL,\n `dnzxjudmjhxrtbdyvpawzdbkavdsqzsv` int DEFAULT NULL,\n `tskmuothytedlmzztzhqqgibbeiwnvdk` int DEFAULT NULL,\n `ebpklqpcsoetwuyaqzelezyklulfeyst` int DEFAULT NULL,\n `ukmozusspnximcmevzxtwivvmgmfqvst` int DEFAULT NULL,\n `podemnevsbizkqufqyvogoabfpstsdjo` int DEFAULT NULL,\n `jwwnktexipoqiqatrmhyuccgjpnnzgcb` int DEFAULT NULL,\n `eoidmkzommtrlvijsgwdglxubjownfam` int DEFAULT NULL,\n `cpblpzgivohuovxvusxmbzyirgjqycjf` int DEFAULT NULL,\n `jdfrieyaixafiwkjkcbvamgrjpeofcys` int DEFAULT NULL,\n `assnybzeebzezowrpkloueccgucncdhd` int DEFAULT NULL,\n `loxjxcpjmdxdiicupmdpsxjjwlpzkond` int DEFAULT NULL,\n `gtwxtowgbecbeesexxnwiirohspmvxyn` int DEFAULT NULL,\n `tgsoqpipmgtktfguhpptmxtglojiqclh` int DEFAULT NULL,\n `hhxnypzzzpxjrvfnzddtbzmffywkcink` int DEFAULT NULL,\n `nkqlsotyekknzqcyxmytdrqonshdehtr` int DEFAULT NULL,\n `gqexvplxlmnlidreotngxcswasdffsdy` int DEFAULT NULL,\n `hjodlhnmfulfrqvurnmfyttxxkeephip` int DEFAULT NULL,\n `txxtpcwntoxvqdmhocdgsvibilmupafk` int DEFAULT NULL,\n `podwrqlqjmgxwcmpkzbiqhbvtlpzsehg` int DEFAULT NULL,\n `zeoiqghgzrchosjkkuwnckcxynhfelth` int DEFAULT NULL,\n `ryqzpypqoexqusjeohdhxbktgypwvwvw` int DEFAULT NULL,\n `yvqrmbvyfaiwwzxdqspcoibovttwxesf` int DEFAULT NULL,\n `fasfjrxvhtkbwzqhvuhmkuwsmpdupeyh` int DEFAULT NULL,\n `coaetubltslfnjlkzmcqnodluycfoonw` int DEFAULT NULL,\n `jfwzhqgwosqxxzluezegladvgirogsvc` int DEFAULT NULL,\n `eqxyvcazdhapjouqfapmyydqdyajoklk` int DEFAULT NULL,\n `quhjtpbtghyjgwhfbmqfpnzqxcstfqbt` int DEFAULT NULL,\n `gvwjalhgdhvybihvmgeeqdvwcntodoym` int DEFAULT NULL,\n `lyrdtkylrdhbqrvyjiligkukoiglpmmt` int DEFAULT NULL,\n `sjgnrxmzquqpbbgqmiuklavldfjjyrnd` int DEFAULT NULL,\n `cyudpeanfvajuxfyqcrknbxuokqspgje` int DEFAULT NULL,\n `wejucqzvcddydaznwjvthodetzekwasx` int DEFAULT NULL,\n `dzywkuzaeryygmhnudpgqqmkibyanjop` int DEFAULT NULL,\n `lfmskoppuiiuedunixziaaemprqclsev` int DEFAULT NULL,\n `xivbdhzeculdwqjsqycqavxuermxpqgh` int DEFAULT NULL,\n `fxzfbnbkaltfvhsphltzgeacqiynwalz` int DEFAULT NULL,\n `pgulxtolwoefnhoxwdaxocdllfithgit` int DEFAULT NULL,\n `ubyciqfcaupydhryhzxqvupnpohsyskb` int DEFAULT NULL,\n `pwcfkuwwmnimkbhouzbsqgpxgfiqcikt` int DEFAULT NULL,\n `bruaqhcacvbyjwxjfqjnenzwbedaoedq` int DEFAULT NULL,\n `opqzfkdeleypupvsvorrubokfuwwfdjk` int DEFAULT NULL,\n `vlwlbcrliocbuchdsmmadjldfbjfovig` int DEFAULT NULL,\n `bmmegwybkpveiadjbpefjudpmycbwmxo` int DEFAULT NULL,\n `kfsqwywcfxlcxkyhkarjunlpzvesviab` int DEFAULT NULL,\n `tkfcevhchyhdelhgwrkbogvsuuzfsjgv` int DEFAULT NULL,\n `kmoffcixsaoutepaetcpducowxmfzusa` int DEFAULT NULL,\n `vefrklcthqcojogzsctcfktsiqdnfbbk` int DEFAULT NULL,\n `cuckkjccsqgdflxvhyigxuqfbtzxmrio` int DEFAULT NULL,\n `tkbztxzspszlgrcgoqtyxrwmblqmitpr` int DEFAULT NULL,\n `lbjgylbfqnzvadnqqyfyijugfvcncumw` int DEFAULT NULL,\n `inieriunodjwdveoppxlbxyldvmouvau` int DEFAULT NULL,\n `axcdoljhrzwlmrnstocivhrcgptpvbre` int DEFAULT NULL,\n `yjemwinramfjkjwrvwcbtgjcymamohlm` int DEFAULT NULL,\n `ajkdmcqvjhfelbphcmehlwsgwpxxljwa` int DEFAULT NULL,\n `avdxvfzepozcbtotpzoqpaeqhbcvpakd` int DEFAULT NULL,\n `xkhqmrjnywaziqngmskqgulgfktcqduf` int DEFAULT NULL,\n `ortrjebiyxcsqpuboqigwrrdkpogmxic` int DEFAULT NULL,\n `plommnlmauntzgonkjlcsclzxltabxes` int DEFAULT NULL,\n `gfviuzriewwutdkocjfejbkxprhpseaj` int DEFAULT NULL,\n `lwgzgltaqrzwuyevorktzmnocknpmjkg` int DEFAULT NULL,\n `ikhtsaqzlkbglyhpqonigghvbogioich` int DEFAULT NULL,\n `ianznuzvectonsticizyfviswpctnbqw` int DEFAULT NULL,\n `hidkobjrqeovaoqshpmmqgqluonherac` int DEFAULT NULL,\n `aljjmelxinlojotobvfsywxtdudeeiub` int DEFAULT NULL,\n `thfyypgmxaaaklcjvtsdgzdpstozqqpv` int DEFAULT NULL,\n `xtvqcnhnseiwtrhatoaeanrjhlhqbgsd` int DEFAULT NULL,\n `npbmoyjlvvqvfpjbeaddvxpdgnbyxpga` int DEFAULT NULL,\n `utkkkebscvwoeoaykgtteiggwzkgfwfw` int DEFAULT NULL,\n `jgtgopsbbwkhljziqczefmicsyqoltvr` int DEFAULT NULL,\n `zvrlmlccvnmxotxyjtxugpcmqclaptco` int DEFAULT NULL,\n `qjabrnqdldpruazjwepcqttsbwaokvzv` int DEFAULT NULL,\n `upacjrbdlbjwmfkcoehiepwxmkysiapn` int DEFAULT NULL,\n `tkwbeiakreouwycujpcfbtkzjrqjsnam` int DEFAULT NULL,\n `dilbrwsgwpzexgiroxqxsoquirqprehh` int DEFAULT NULL,\n `utizjmqhxhiiazfgfdzufocxwybajayk` int DEFAULT NULL,\n `ynwboaotjwgisjcwyrrjmawxuxuedcde` int DEFAULT NULL,\n `toioratwbhszxcxseifqpqaehjewhmxo` int DEFAULT NULL,\n `qlrjyilyxccspisksjdblgenuldexmsc` int DEFAULT NULL,\n `wwvxcpwlnfitmuoeiudyshocroaodici` int DEFAULT NULL,\n `nlflyoeetidhsrhxwtwtnxzdimytmztf` int DEFAULT NULL,\n `pdpweobwjuidbhvmnvjfzlviydecvtgp` int DEFAULT NULL,\n `syohruqxbshsvyjgfefpwwhbdksnntxt` int DEFAULT NULL,\n `thjpujpyosrluilhhysysmaxjwkjysxs` int DEFAULT NULL,\n `dvhacsrcqrndrxoveyxkygrpnptypiei` int DEFAULT NULL,\n `ossfsehgkwhuvqffykeoobapvuejadtt` int DEFAULT NULL,\n `oikbdoczggryojzibswquzfqhquskjnc` int DEFAULT NULL,\n `kpmozrebgcddjeepltpnffkcvacvstmg` int DEFAULT NULL,\n `nnfjjfkwktbkuspwipbrbhvxdjomrpqt` int DEFAULT NULL,\n `kutvrjkfwaupzxdhpyokuygxeqqqiyxs` int DEFAULT NULL,\n `xcmpnxbodzymjllgqimnnhmdtauzmxgt` int DEFAULT NULL,\n `yovzukgbwlfjmrtpgzzwvcvdyleidspb` int DEFAULT NULL,\n `jlzdfczytjijddlzmazzzutcjpeqwkuf` int DEFAULT NULL,\n `cupumrvfhxousffxesbfolzflshsopdb` int DEFAULT NULL,\n `lqipdgwgfaeppagbhjcnvdhmzyurjfpk` int DEFAULT NULL,\n `uezeqdcckvgxivlbhdgkmduthwvnjaav` int DEFAULT NULL,\n `kmelplwnmtpgqufrseyheiikeshxwhuy` int DEFAULT NULL,\n `ejrcoisanzrhrctbbyrfnhzeutndfeme` int DEFAULT NULL,\n PRIMARY KEY (`nsnsqgwgdsiwoxzbrtpgkcufmurayjcc`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"tpksmtlcpcazciywpqiwdqlzxbnoiktq\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["nsnsqgwgdsiwoxzbrtpgkcufmurayjcc"],"columns":[{"name":"nsnsqgwgdsiwoxzbrtpgkcufmurayjcc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"xvpnrilojzdvqnmdzglpwqrysryhgaet","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xlbgqpdzlwncmrhyaaamgbfhgqpiybal","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dnzxjudmjhxrtbdyvpawzdbkavdsqzsv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tskmuothytedlmzztzhqqgibbeiwnvdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebpklqpcsoetwuyaqzelezyklulfeyst","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ukmozusspnximcmevzxtwivvmgmfqvst","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"podemnevsbizkqufqyvogoabfpstsdjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwwnktexipoqiqatrmhyuccgjpnnzgcb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eoidmkzommtrlvijsgwdglxubjownfam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cpblpzgivohuovxvusxmbzyirgjqycjf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdfrieyaixafiwkjkcbvamgrjpeofcys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"assnybzeebzezowrpkloueccgucncdhd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"loxjxcpjmdxdiicupmdpsxjjwlpzkond","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtwxtowgbecbeesexxnwiirohspmvxyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tgsoqpipmgtktfguhpptmxtglojiqclh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhxnypzzzpxjrvfnzddtbzmffywkcink","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkqlsotyekknzqcyxmytdrqonshdehtr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqexvplxlmnlidreotngxcswasdffsdy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjodlhnmfulfrqvurnmfyttxxkeephip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txxtpcwntoxvqdmhocdgsvibilmupafk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"podwrqlqjmgxwcmpkzbiqhbvtlpzsehg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zeoiqghgzrchosjkkuwnckcxynhfelth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ryqzpypqoexqusjeohdhxbktgypwvwvw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yvqrmbvyfaiwwzxdqspcoibovttwxesf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fasfjrxvhtkbwzqhvuhmkuwsmpdupeyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"coaetubltslfnjlkzmcqnodluycfoonw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jfwzhqgwosqxxzluezegladvgirogsvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqxyvcazdhapjouqfapmyydqdyajoklk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"quhjtpbtghyjgwhfbmqfpnzqxcstfqbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gvwjalhgdhvybihvmgeeqdvwcntodoym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lyrdtkylrdhbqrvyjiligkukoiglpmmt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjgnrxmzquqpbbgqmiuklavldfjjyrnd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cyudpeanfvajuxfyqcrknbxuokqspgje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wejucqzvcddydaznwjvthodetzekwasx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dzywkuzaeryygmhnudpgqqmkibyanjop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfmskoppuiiuedunixziaaemprqclsev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xivbdhzeculdwqjsqycqavxuermxpqgh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxzfbnbkaltfvhsphltzgeacqiynwalz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgulxtolwoefnhoxwdaxocdllfithgit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubyciqfcaupydhryhzxqvupnpohsyskb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwcfkuwwmnimkbhouzbsqgpxgfiqcikt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bruaqhcacvbyjwxjfqjnenzwbedaoedq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opqzfkdeleypupvsvorrubokfuwwfdjk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vlwlbcrliocbuchdsmmadjldfbjfovig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bmmegwybkpveiadjbpefjudpmycbwmxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfsqwywcfxlcxkyhkarjunlpzvesviab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkfcevhchyhdelhgwrkbogvsuuzfsjgv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmoffcixsaoutepaetcpducowxmfzusa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vefrklcthqcojogzsctcfktsiqdnfbbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cuckkjccsqgdflxvhyigxuqfbtzxmrio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkbztxzspszlgrcgoqtyxrwmblqmitpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lbjgylbfqnzvadnqqyfyijugfvcncumw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"inieriunodjwdveoppxlbxyldvmouvau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axcdoljhrzwlmrnstocivhrcgptpvbre","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjemwinramfjkjwrvwcbtgjcymamohlm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajkdmcqvjhfelbphcmehlwsgwpxxljwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avdxvfzepozcbtotpzoqpaeqhbcvpakd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkhqmrjnywaziqngmskqgulgfktcqduf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ortrjebiyxcsqpuboqigwrrdkpogmxic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"plommnlmauntzgonkjlcsclzxltabxes","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfviuzriewwutdkocjfejbkxprhpseaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwgzgltaqrzwuyevorktzmnocknpmjkg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikhtsaqzlkbglyhpqonigghvbogioich","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ianznuzvectonsticizyfviswpctnbqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hidkobjrqeovaoqshpmmqgqluonherac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aljjmelxinlojotobvfsywxtdudeeiub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thfyypgmxaaaklcjvtsdgzdpstozqqpv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xtvqcnhnseiwtrhatoaeanrjhlhqbgsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npbmoyjlvvqvfpjbeaddvxpdgnbyxpga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utkkkebscvwoeoaykgtteiggwzkgfwfw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgtgopsbbwkhljziqczefmicsyqoltvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvrlmlccvnmxotxyjtxugpcmqclaptco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjabrnqdldpruazjwepcqttsbwaokvzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upacjrbdlbjwmfkcoehiepwxmkysiapn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkwbeiakreouwycujpcfbtkzjrqjsnam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dilbrwsgwpzexgiroxqxsoquirqprehh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utizjmqhxhiiazfgfdzufocxwybajayk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynwboaotjwgisjcwyrrjmawxuxuedcde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"toioratwbhszxcxseifqpqaehjewhmxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qlrjyilyxccspisksjdblgenuldexmsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwvxcpwlnfitmuoeiudyshocroaodici","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nlflyoeetidhsrhxwtwtnxzdimytmztf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdpweobwjuidbhvmnvjfzlviydecvtgp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"syohruqxbshsvyjgfefpwwhbdksnntxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thjpujpyosrluilhhysysmaxjwkjysxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvhacsrcqrndrxoveyxkygrpnptypiei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ossfsehgkwhuvqffykeoobapvuejadtt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oikbdoczggryojzibswquzfqhquskjnc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpmozrebgcddjeepltpnffkcvacvstmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nnfjjfkwktbkuspwipbrbhvxdjomrpqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kutvrjkfwaupzxdhpyokuygxeqqqiyxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xcmpnxbodzymjllgqimnnhmdtauzmxgt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yovzukgbwlfjmrtpgzzwvcvdyleidspb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlzdfczytjijddlzmazzzutcjpeqwkuf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cupumrvfhxousffxesbfolzflshsopdb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqipdgwgfaeppagbhjcnvdhmzyurjfpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uezeqdcckvgxivlbhdgkmduthwvnjaav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmelplwnmtpgqufrseyheiikeshxwhuy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejrcoisanzrhrctbbyrfnhzeutndfeme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671575,"databaseName":"models_schema","ddl":"CREATE TABLE `tpkxfywbmayegsworhdugtdrzkjvveya` (\n `pjkcefpcibjkvosywtxgwcbbcvvenjpy` int NOT NULL,\n `lfyphemoycozrwbvaklcnkxnhcesvcxi` int DEFAULT NULL,\n `frlrlahhxnscwydiifkembklgfftzjgr` int DEFAULT NULL,\n `wreuqlbaqftrllkddirgohlxplbnuccw` int DEFAULT NULL,\n `fzbautbcrfchvntxilfquyfmsekxzowk` int DEFAULT NULL,\n `xforkqmpmqypbvbzfznhbjalwhjpydqd` int DEFAULT NULL,\n `suglrgrmlqvwrlcemnbyaeutfnihhiyu` int DEFAULT NULL,\n `drlcudemumtzjdlanofkgsbgtdoinxfp` int DEFAULT NULL,\n `spnxufrqltrmpksehrucqefuhjdlrodc` int DEFAULT NULL,\n `lcqcuakxuxinrmazroqnttlvairqtdyw` int DEFAULT NULL,\n `hphffogojvrncxdrguaelvhrxsvlsrlc` int DEFAULT NULL,\n `tlpououdgtiksutocibrkuycwesvngrt` int DEFAULT NULL,\n `tfyjyjwglcjmwcjpnbxcatejcomwcacn` int DEFAULT NULL,\n `tjkzrlxptekekvfduzcponlrymzbzscy` int DEFAULT NULL,\n `nrtplkbksjgklkwsmqevmztrtpshagbb` int DEFAULT NULL,\n `cdeouauzyqgddyfgmkwpzwqqgsrmkfzy` int DEFAULT NULL,\n `jeprkgklhxdlwxzvkzbszsybcjyusjmw` int DEFAULT NULL,\n `davcyzrdgnxtkpxnuhfsddsjglweddtm` int DEFAULT NULL,\n `fednobbpoouhdzzxnarkrlnpicvjmfsk` int DEFAULT NULL,\n `elmsnqaniikispiqnkapsswcdjbpadfy` int DEFAULT NULL,\n `wijslyzchcunyqwbzrywgggijbsiytbp` int DEFAULT NULL,\n `aubipltydbmypleitfdkabtyuhfqxecj` int DEFAULT NULL,\n `uzlazgtormtyuomxxqmmlndixvduzqge` int DEFAULT NULL,\n `vbkjcgayczwydyyxkilhhtqakfspzyas` int DEFAULT NULL,\n `jzzcmwztyvstoeskkkdxkuxjkbboxteq` int DEFAULT NULL,\n `qypcoourvbhesardyqhkzriedjhloxui` int DEFAULT NULL,\n `insslfbqznydwnkxnkcytvqleqbvblph` int DEFAULT NULL,\n `bkjhkrabfhtfhuemofpvykkchgqupvul` int DEFAULT NULL,\n `seskbflxfilzbseumbgmurikdqrurpdz` int DEFAULT NULL,\n `nritycvrqkrldobsrymbqykzbqexmsua` int DEFAULT NULL,\n `hhtrzeecfgzpzkwvhyptmhzoqazcemxu` int DEFAULT NULL,\n `rrezhddrcdixjslwrumhkmabzgdldkgi` int DEFAULT NULL,\n `ncdezfxoknxvsigyvykjvlkqeymrplts` int DEFAULT NULL,\n `rvghoavpydmjobddkrjgwwfxcyjjbscn` int DEFAULT NULL,\n `gaqwimibpefbinuqolihwyvjsdgmhril` int DEFAULT NULL,\n `cfsynuwpvvhsgyoqmiefkqqxafwgtgog` int DEFAULT NULL,\n `txpxxhtrxbfaheozdxrbjtkmkmejaxau` int DEFAULT NULL,\n `iiczzajgmaieqnpvvltlqiofembnaimk` int DEFAULT NULL,\n `kzbumgvpusifdlhmymegoxepjqzfuqwu` int DEFAULT NULL,\n `ckigfjznvlpzxilkwzaygbanngfqtoba` int DEFAULT NULL,\n `iwndtbmdpvfhjlculqwddeiqvcwsmchu` int DEFAULT NULL,\n `tqyfzuhcyiqgfwwbcyidgzutdlxsorfe` int DEFAULT NULL,\n `laeldcbrercxmtbjdhmbuimdoskxfdjd` int DEFAULT NULL,\n `pgijjmwdfabkklxlzbhapyqwbmnlvxrn` int DEFAULT NULL,\n `wtwkowcdnomycledvfdhklacnaapxrhi` int DEFAULT NULL,\n `deotwdyvwhqmwsuqoizxeldmmmaabtso` int DEFAULT NULL,\n `xktmqptkzuerzntaxbkjuzdddevddczp` int DEFAULT NULL,\n `fveqaghmxbzcsrjckuylousmtajjjwon` int DEFAULT NULL,\n `ajmaywulwumbdvsbkzdphlfhoroiyuxx` int DEFAULT NULL,\n `wiubnhjhzhimlpobpmzxgosfivehfhjr` int DEFAULT NULL,\n `bkttyqwnjvshwzwtsdxxseocpedqozjt` int DEFAULT NULL,\n `hdtlbtguukindzvswiwyrwcwvwzghmea` int DEFAULT NULL,\n `dpukyavphxziehrsystcgacopzegwkgf` int DEFAULT NULL,\n `tuxbfydekepqdshqfrdaxbtvmvgtazeu` int DEFAULT NULL,\n `rvewripnpcqophtehgnohcadywsxpzob` int DEFAULT NULL,\n `ihcofidkwqzbbpdlptwxwmhulwhplsyc` int DEFAULT NULL,\n `serlvfnexmqyoqkocigqdnrcazzgspye` int DEFAULT NULL,\n `ktvstkdqtmqfugrfsqbdotitzvpjlpsh` int DEFAULT NULL,\n `prsyxhdgeurqoypwjtzgcqcwdsnkkknn` int DEFAULT NULL,\n `tduiqdpafixzmazbvrcqzxxlovfliake` int DEFAULT NULL,\n `xelvkjjipjihculepnjhwmbfnodnstdv` int DEFAULT NULL,\n `mpjadkhqhasqjknplbnnsjiqksrrnvzo` int DEFAULT NULL,\n `klqasuzktesjmycotfzubdmaoexvqnsr` int DEFAULT NULL,\n `hzkbempopmzkfsfpsqtiviixyerwfyvn` int DEFAULT NULL,\n `sqndpwqqhnifmzelrrfzhhppcnpkpmqu` int DEFAULT NULL,\n `sozyshbaealuncmitufojslshsbcjcwn` int DEFAULT NULL,\n `jjxshfcasgyqyqpsujxttfwzaiohaexr` int DEFAULT NULL,\n `pceymtktorykqsfqkohgkkxofebigeiw` int DEFAULT NULL,\n `carcryipeysucxddzkupllwoipkbcjaw` int DEFAULT NULL,\n `qkqqzyvqidbiufbulqhgllhceigxsjvw` int DEFAULT NULL,\n `fzumppkeoaysiestfixlrcexojgzyngx` int DEFAULT NULL,\n `vdzkfklxjtizcdgjzddbsqdwcmogysys` int DEFAULT NULL,\n `wkrwizpdeoxgjsiummaaburxzcakxeqc` int DEFAULT NULL,\n `essrgioxvnpygxonztsvpigyeevrzeiv` int DEFAULT NULL,\n `eyangozrbyvyufnqevetdaihgnlrztlz` int DEFAULT NULL,\n `clzgqymvetchaykdmqyufydxyunycpet` int DEFAULT NULL,\n `gbemyqvpmtuhicpemdsuohlxdapgdvmr` int DEFAULT NULL,\n `awmjgxjxexbyjamgkngqiyyoluqclyyj` int DEFAULT NULL,\n `mglqbjwxykavffxugaqckfbzhworlxsp` int DEFAULT NULL,\n `anrybznelxbgcigqiaiypowjfltrofgk` int DEFAULT NULL,\n `dtiokdlgcdbfbfvwelsgqsfwksyvynzo` int DEFAULT NULL,\n `buqybwosfyceggnacfdmnsylzxzmndff` int DEFAULT NULL,\n `bmwbgrkcymsaxhrtiakypgfahmxphpow` int DEFAULT NULL,\n `qattqhaitxusjwluvypukzhmilnwtxea` int DEFAULT NULL,\n `bxdjkwqxhgghkmwkfnsaihhcbhgfhvgp` int DEFAULT NULL,\n `ogaiaosjojtevwfugpmqoyclibdxlsgs` int DEFAULT NULL,\n `inpcujkfxrtotmyttzqvsqagdotzlcjn` int DEFAULT NULL,\n `cfcedvkwyvlsnnrpbdblaqpiejgfjjab` int DEFAULT NULL,\n `tbmtmjdytmxrigfcjyavntsdshocmvum` int DEFAULT NULL,\n `gxhjvmmambeuzdujhwmsypoxbrtykoor` int DEFAULT NULL,\n `agpjxpdgpomgjwsluzkfurntwwjlzyrq` int DEFAULT NULL,\n `bajmfvirojfyasmvjsocilowwqxqlcfc` int DEFAULT NULL,\n `kgrkaqumzamyyglzkooilewqoprbbxry` int DEFAULT NULL,\n `dcflmbkcedabhicfxfkiwitkwhixpqlq` int DEFAULT NULL,\n `xebyytiwbgpwdglsuotzpzeftacadosi` int DEFAULT NULL,\n `izylioayxcubknfjrcpmpjvzcbercjqc` int DEFAULT NULL,\n `ejtrxitxfmcvqefbdwkdbyypkptfvicf` int DEFAULT NULL,\n `pjaenjffbharmphuesnajqbrqzacehnh` int DEFAULT NULL,\n `igyuoqmczhzcngghrjuygorgjehtzljn` int DEFAULT NULL,\n `nytavjkktqjyukqatieuvgxozrtqbrpm` int DEFAULT NULL,\n PRIMARY KEY (`pjkcefpcibjkvosywtxgwcbbcvvenjpy`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"tpkxfywbmayegsworhdugtdrzkjvveya\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["pjkcefpcibjkvosywtxgwcbbcvvenjpy"],"columns":[{"name":"pjkcefpcibjkvosywtxgwcbbcvvenjpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"lfyphemoycozrwbvaklcnkxnhcesvcxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"frlrlahhxnscwydiifkembklgfftzjgr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wreuqlbaqftrllkddirgohlxplbnuccw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzbautbcrfchvntxilfquyfmsekxzowk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xforkqmpmqypbvbzfznhbjalwhjpydqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suglrgrmlqvwrlcemnbyaeutfnihhiyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drlcudemumtzjdlanofkgsbgtdoinxfp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"spnxufrqltrmpksehrucqefuhjdlrodc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lcqcuakxuxinrmazroqnttlvairqtdyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hphffogojvrncxdrguaelvhrxsvlsrlc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlpououdgtiksutocibrkuycwesvngrt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tfyjyjwglcjmwcjpnbxcatejcomwcacn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjkzrlxptekekvfduzcponlrymzbzscy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nrtplkbksjgklkwsmqevmztrtpshagbb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cdeouauzyqgddyfgmkwpzwqqgsrmkfzy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jeprkgklhxdlwxzvkzbszsybcjyusjmw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"davcyzrdgnxtkpxnuhfsddsjglweddtm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fednobbpoouhdzzxnarkrlnpicvjmfsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elmsnqaniikispiqnkapsswcdjbpadfy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wijslyzchcunyqwbzrywgggijbsiytbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aubipltydbmypleitfdkabtyuhfqxecj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzlazgtormtyuomxxqmmlndixvduzqge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbkjcgayczwydyyxkilhhtqakfspzyas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzzcmwztyvstoeskkkdxkuxjkbboxteq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qypcoourvbhesardyqhkzriedjhloxui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"insslfbqznydwnkxnkcytvqleqbvblph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkjhkrabfhtfhuemofpvykkchgqupvul","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"seskbflxfilzbseumbgmurikdqrurpdz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nritycvrqkrldobsrymbqykzbqexmsua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhtrzeecfgzpzkwvhyptmhzoqazcemxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrezhddrcdixjslwrumhkmabzgdldkgi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ncdezfxoknxvsigyvykjvlkqeymrplts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rvghoavpydmjobddkrjgwwfxcyjjbscn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gaqwimibpefbinuqolihwyvjsdgmhril","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfsynuwpvvhsgyoqmiefkqqxafwgtgog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txpxxhtrxbfaheozdxrbjtkmkmejaxau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iiczzajgmaieqnpvvltlqiofembnaimk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzbumgvpusifdlhmymegoxepjqzfuqwu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckigfjznvlpzxilkwzaygbanngfqtoba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwndtbmdpvfhjlculqwddeiqvcwsmchu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqyfzuhcyiqgfwwbcyidgzutdlxsorfe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"laeldcbrercxmtbjdhmbuimdoskxfdjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgijjmwdfabkklxlzbhapyqwbmnlvxrn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtwkowcdnomycledvfdhklacnaapxrhi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"deotwdyvwhqmwsuqoizxeldmmmaabtso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xktmqptkzuerzntaxbkjuzdddevddczp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fveqaghmxbzcsrjckuylousmtajjjwon","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajmaywulwumbdvsbkzdphlfhoroiyuxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wiubnhjhzhimlpobpmzxgosfivehfhjr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkttyqwnjvshwzwtsdxxseocpedqozjt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdtlbtguukindzvswiwyrwcwvwzghmea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpukyavphxziehrsystcgacopzegwkgf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tuxbfydekepqdshqfrdaxbtvmvgtazeu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rvewripnpcqophtehgnohcadywsxpzob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ihcofidkwqzbbpdlptwxwmhulwhplsyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"serlvfnexmqyoqkocigqdnrcazzgspye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ktvstkdqtmqfugrfsqbdotitzvpjlpsh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prsyxhdgeurqoypwjtzgcqcwdsnkkknn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tduiqdpafixzmazbvrcqzxxlovfliake","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xelvkjjipjihculepnjhwmbfnodnstdv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpjadkhqhasqjknplbnnsjiqksrrnvzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"klqasuzktesjmycotfzubdmaoexvqnsr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hzkbempopmzkfsfpsqtiviixyerwfyvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sqndpwqqhnifmzelrrfzhhppcnpkpmqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sozyshbaealuncmitufojslshsbcjcwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjxshfcasgyqyqpsujxttfwzaiohaexr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pceymtktorykqsfqkohgkkxofebigeiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"carcryipeysucxddzkupllwoipkbcjaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qkqqzyvqidbiufbulqhgllhceigxsjvw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzumppkeoaysiestfixlrcexojgzyngx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdzkfklxjtizcdgjzddbsqdwcmogysys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wkrwizpdeoxgjsiummaaburxzcakxeqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"essrgioxvnpygxonztsvpigyeevrzeiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eyangozrbyvyufnqevetdaihgnlrztlz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clzgqymvetchaykdmqyufydxyunycpet","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gbemyqvpmtuhicpemdsuohlxdapgdvmr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awmjgxjxexbyjamgkngqiyyoluqclyyj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mglqbjwxykavffxugaqckfbzhworlxsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"anrybznelxbgcigqiaiypowjfltrofgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtiokdlgcdbfbfvwelsgqsfwksyvynzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"buqybwosfyceggnacfdmnsylzxzmndff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bmwbgrkcymsaxhrtiakypgfahmxphpow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qattqhaitxusjwluvypukzhmilnwtxea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bxdjkwqxhgghkmwkfnsaihhcbhgfhvgp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ogaiaosjojtevwfugpmqoyclibdxlsgs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"inpcujkfxrtotmyttzqvsqagdotzlcjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfcedvkwyvlsnnrpbdblaqpiejgfjjab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbmtmjdytmxrigfcjyavntsdshocmvum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxhjvmmambeuzdujhwmsypoxbrtykoor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agpjxpdgpomgjwsluzkfurntwwjlzyrq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bajmfvirojfyasmvjsocilowwqxqlcfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kgrkaqumzamyyglzkooilewqoprbbxry","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcflmbkcedabhicfxfkiwitkwhixpqlq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xebyytiwbgpwdglsuotzpzeftacadosi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"izylioayxcubknfjrcpmpjvzcbercjqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejtrxitxfmcvqefbdwkdbyypkptfvicf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjaenjffbharmphuesnajqbrqzacehnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igyuoqmczhzcngghrjuygorgjehtzljn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nytavjkktqjyukqatieuvgxozrtqbrpm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671607,"databaseName":"models_schema","ddl":"CREATE TABLE `tzetkodbjogqjrhxuxwowogrhxsolvar` (\n `msrsffbjcrxtgrdomvvwaeqvtkcokzov` int NOT NULL,\n `jqfstbkoyxaibdhpjrnewnjporjssbha` int DEFAULT NULL,\n `bgjklmpabkbwryaxttrvwwblylpbjwnb` int DEFAULT NULL,\n `hmjdplrsqxwwtsitzwpeigkfrgcxhdqt` int DEFAULT NULL,\n `fzyyybcmildnebrnjtldbuugxzfrzaqm` int DEFAULT NULL,\n `wnrslgyzysvczvskisjaemqmakcbcamt` int DEFAULT NULL,\n `vnzfvezqleaznygfmqkghbxidbouirwz` int DEFAULT NULL,\n `kgethotfuepfrstfddvmvesxqyjwhnrn` int DEFAULT NULL,\n `cslmusuvygvbwerbfgbfmphtaoyiomor` int DEFAULT NULL,\n `bkithrfnjitlanjgtlxtxmfzximvdtbw` int DEFAULT NULL,\n `csrugkssbwdxyuktyrmaqtpxzwvbtbbq` int DEFAULT NULL,\n `xxrepueyhplrdbpnenxvoklyqclkyzvh` int DEFAULT NULL,\n `dhvjcktzxbwkhvegptatxnugfwdxmweb` int DEFAULT NULL,\n `dusudimfnlzznpnqeqeidwpjlubdomoi` int DEFAULT NULL,\n `nzgvaaqcpyzomvqryhfuqqobfanmtial` int DEFAULT NULL,\n `hupgdalxkonicxvovpyjpwtdhjrwnmcy` int DEFAULT NULL,\n `iflouasmqswpjupkqqdwoddxrbbdpxxu` int DEFAULT NULL,\n `sgzsrjpjbhtmlekghdyhaojsohwqkbbi` int DEFAULT NULL,\n `adrrbdmylsszzckekdzdzopelkiwqlhp` int DEFAULT NULL,\n `pkwgiynkmyqqiouwiuhdgenspnzegjwe` int DEFAULT NULL,\n `gwmexzovxwbftnwpgyrurifueazbqjir` int DEFAULT NULL,\n `dmfbxrkpxzlaheitvtkrwbxvvvfdekbe` int DEFAULT NULL,\n `ajforhqwfhrapmncrjhjjuynhtchpgth` int DEFAULT NULL,\n `wxomutaxanufvcfeadfeouqozqabbxvr` int DEFAULT NULL,\n `pervmxgmumnvasjkodhwdtjnvhkakenc` int DEFAULT NULL,\n `mtzbpriksoikjqjwhihtjfobcgztwhjl` int DEFAULT NULL,\n `crmipbnzlrnnoxzaowoclqthvwbxcsuc` int DEFAULT NULL,\n `jpqseyhiyvsutknbxhdfmfgqofetodgf` int DEFAULT NULL,\n `qynclvqxbcuatvmdurhtuujidxtuqbcg` int DEFAULT NULL,\n `wyzwepxyxprfymtfevkysnvcwddkxeol` int DEFAULT NULL,\n `manwipmvkqfqepbrhcomchevavkeczhz` int DEFAULT NULL,\n `izythqpzzffaframrldfsyfbnemjenkh` int DEFAULT NULL,\n `zlehrblynysfittwxsxehdhtwoddqjox` int DEFAULT NULL,\n `iwnvctdptmkqmcpcelnnjxulmcmcbruj` int DEFAULT NULL,\n `ghvgatxochimtzhwhommjrhjlwbdeabi` int DEFAULT NULL,\n `eucrdbwlcnnkxdupuktvyzmpkiyyjscq` int DEFAULT NULL,\n `bzvjgregdgjkdqohpghjaeoteremxoiv` int DEFAULT NULL,\n `scvodwhzlemwwuujcnzvkophvtfvjfuq` int DEFAULT NULL,\n `rdudfnzkicdwhtevdddydoywuszyguvn` int DEFAULT NULL,\n `sbyklsheaeygcplqoymrzhkpmlblnsut` int DEFAULT NULL,\n `afpbslimsvzwjkplbqzolwxbbxtxocsl` int DEFAULT NULL,\n `plzlmkqmgudjctxlugyfyvqpedopocyr` int DEFAULT NULL,\n `wrtuhvochowcxwqhzaojnttijulwsghg` int DEFAULT NULL,\n `jloqmjqtjgpjxwtvtdrldeguetypjrgz` int DEFAULT NULL,\n `lqdmxhryydxcwysmttjpgwnibbqavrmx` int DEFAULT NULL,\n `becpcslxlaruozsomifstxjmjtdoogml` int DEFAULT NULL,\n `ajygbvpcqmkdpqpwxgavctcaibthrbkk` int DEFAULT NULL,\n `cpvcixncywmtfnmiznovplhxzwrdbhqh` int DEFAULT NULL,\n `ioxmxaildrzvitrueafdfyfmlnodxkzi` int DEFAULT NULL,\n `drllhgtohcfzazszyuebiinvynwdhspj` int DEFAULT NULL,\n `ylnadxmdudxqrbccfstikhxvbzzpzitd` int DEFAULT NULL,\n `rwwhrymzbkksjjazqsbkfhhnvbdutxri` int DEFAULT NULL,\n `qpujjawamkndlhfydlozxbjsbupivqpg` int DEFAULT NULL,\n `fqbvfbgnqijnqhrffoahsemafzjsqvdh` int DEFAULT NULL,\n `gtkpvsyfdzhxlnnwqsnnuauxyyellvis` int DEFAULT NULL,\n `toftclnosxtgkyypteczvfudjqojumgi` int DEFAULT NULL,\n `vgoprwlcqzfuilwbiypnoifwycbzajwn` int DEFAULT NULL,\n `wukkycmfhctqlvqsyowcdtnithkhfclr` int DEFAULT NULL,\n `czheusmchgodznymzpoldjjucrgoaoah` int DEFAULT NULL,\n `ihjfsyssfgxrohvxirpniedtqydlgduy` int DEFAULT NULL,\n `ixelrigmeqfacfunzcnblrxdmyscuqxw` int DEFAULT NULL,\n `hmslnejiqusybhbyzklaakmwajeuwgqy` int DEFAULT NULL,\n `ctuwuyikthqltoihdhthcparsztzwlxo` int DEFAULT NULL,\n `oryoegawbrftjdzaeczureecxmuepmhs` int DEFAULT NULL,\n `wfhwiwczqwvqalavmbxrdxyobvgihjzg` int DEFAULT NULL,\n `ymhgcceekbudxumynkpflioztpzgpinr` int DEFAULT NULL,\n `mepxccnkmjscxkrueouytuwhvggtspwc` int DEFAULT NULL,\n `nzdxzehyesrcfdonufgzadgrfydswrvu` int DEFAULT NULL,\n `jpfpiargzhtwfiwzmolcjbtccemquvxi` int DEFAULT NULL,\n `lcqrxpyszakljlrhgxfxxcppuakbgjgr` int DEFAULT NULL,\n `zktkszjaicmbnxaiyyrxueiiaxoymqyv` int DEFAULT NULL,\n `xsdqhwnzdcvtkcseaxmvzarvxsjfuxsz` int DEFAULT NULL,\n `udhdgruohkjjdemukbhrdmvkiwmqemmp` int DEFAULT NULL,\n `aoxzkkxqhmpltqxitomciqonwlueseup` int DEFAULT NULL,\n `fjtsvaybvpbzdeymdvtnhtdvokcdpcae` int DEFAULT NULL,\n `cotnvfnlzrjvjkpyszdzvdsldpokmjxl` int DEFAULT NULL,\n `fvsgmvcfafbgzvvmywkdzdfuaarmbsjw` int DEFAULT NULL,\n `xdxdsnlnwjdcbxqdjosgmcccdshawvfj` int DEFAULT NULL,\n `znbqufpalbqjhehriniaclizuwtkbgmo` int DEFAULT NULL,\n `baabvuqumayldkkkgsikyfmeucojrter` int DEFAULT NULL,\n `ofxsprtynpvnvnqaswfgmqfljyjyrunu` int DEFAULT NULL,\n `ehincrgzlaogluorkcnegrqceaqmnrym` int DEFAULT NULL,\n `jbkjgnjzytxfdlfbkdmjscsuqzjcuvyg` int DEFAULT NULL,\n `vvtjfjtonzsyvzuvcdxfxilrusykgnjc` int DEFAULT NULL,\n `pehpndkihkmuwvqlbzwwxilyfnoupyah` int DEFAULT NULL,\n `opeiecqfhvnnrlzutmtcxssmorwppihg` int DEFAULT NULL,\n `kfnlctjxizsqkvclpofyrqvpswcoiyhr` int DEFAULT NULL,\n `nbrqmiedryydahpvyizjnoxwpugnfjxz` int DEFAULT NULL,\n `jxfgzomvreyxrbhcisuclcgjxfsyzshh` int DEFAULT NULL,\n `xpinirslzhjmurgpilvlggxkimttcyxs` int DEFAULT NULL,\n `jvxucfccdpvaoupwtkxwgrtanjaqqcut` int DEFAULT NULL,\n `rbetwdiqlhwyjdeddnpigfgfrpcpjyyk` int DEFAULT NULL,\n `kxbdaerhnnwomuslqiduzdcxdcybunuu` int DEFAULT NULL,\n `tlwbgvhksdataytxafvsijapwziezlli` int DEFAULT NULL,\n `orghxiaqgyslkngbzysrzkcbpytqjxse` int DEFAULT NULL,\n `gymvnlvmwxfppxxrpqetqkdacocycrpy` int DEFAULT NULL,\n `tthmfcgxlzgdwjpcswmmvvpycrdqkrcx` int DEFAULT NULL,\n `ocwtaifuzvvlivftpnlmjfvuhrqtvauf` int DEFAULT NULL,\n `anenoszfdnpegkgniqtjelurwldocdke` int DEFAULT NULL,\n `xmryqcznyoefslyrigkpnkdipbhhucdd` int DEFAULT NULL,\n PRIMARY KEY (`msrsffbjcrxtgrdomvvwaeqvtkcokzov`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"tzetkodbjogqjrhxuxwowogrhxsolvar\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["msrsffbjcrxtgrdomvvwaeqvtkcokzov"],"columns":[{"name":"msrsffbjcrxtgrdomvvwaeqvtkcokzov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"jqfstbkoyxaibdhpjrnewnjporjssbha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bgjklmpabkbwryaxttrvwwblylpbjwnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmjdplrsqxwwtsitzwpeigkfrgcxhdqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzyyybcmildnebrnjtldbuugxzfrzaqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnrslgyzysvczvskisjaemqmakcbcamt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnzfvezqleaznygfmqkghbxidbouirwz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kgethotfuepfrstfddvmvesxqyjwhnrn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cslmusuvygvbwerbfgbfmphtaoyiomor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkithrfnjitlanjgtlxtxmfzximvdtbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"csrugkssbwdxyuktyrmaqtpxzwvbtbbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxrepueyhplrdbpnenxvoklyqclkyzvh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhvjcktzxbwkhvegptatxnugfwdxmweb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dusudimfnlzznpnqeqeidwpjlubdomoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzgvaaqcpyzomvqryhfuqqobfanmtial","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hupgdalxkonicxvovpyjpwtdhjrwnmcy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iflouasmqswpjupkqqdwoddxrbbdpxxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sgzsrjpjbhtmlekghdyhaojsohwqkbbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"adrrbdmylsszzckekdzdzopelkiwqlhp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkwgiynkmyqqiouwiuhdgenspnzegjwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gwmexzovxwbftnwpgyrurifueazbqjir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dmfbxrkpxzlaheitvtkrwbxvvvfdekbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajforhqwfhrapmncrjhjjuynhtchpgth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxomutaxanufvcfeadfeouqozqabbxvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pervmxgmumnvasjkodhwdtjnvhkakenc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtzbpriksoikjqjwhihtjfobcgztwhjl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crmipbnzlrnnoxzaowoclqthvwbxcsuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpqseyhiyvsutknbxhdfmfgqofetodgf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qynclvqxbcuatvmdurhtuujidxtuqbcg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyzwepxyxprfymtfevkysnvcwddkxeol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"manwipmvkqfqepbrhcomchevavkeczhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"izythqpzzffaframrldfsyfbnemjenkh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zlehrblynysfittwxsxehdhtwoddqjox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwnvctdptmkqmcpcelnnjxulmcmcbruj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghvgatxochimtzhwhommjrhjlwbdeabi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eucrdbwlcnnkxdupuktvyzmpkiyyjscq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzvjgregdgjkdqohpghjaeoteremxoiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scvodwhzlemwwuujcnzvkophvtfvjfuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdudfnzkicdwhtevdddydoywuszyguvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbyklsheaeygcplqoymrzhkpmlblnsut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afpbslimsvzwjkplbqzolwxbbxtxocsl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"plzlmkqmgudjctxlugyfyvqpedopocyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wrtuhvochowcxwqhzaojnttijulwsghg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jloqmjqtjgpjxwtvtdrldeguetypjrgz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqdmxhryydxcwysmttjpgwnibbqavrmx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"becpcslxlaruozsomifstxjmjtdoogml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajygbvpcqmkdpqpwxgavctcaibthrbkk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cpvcixncywmtfnmiznovplhxzwrdbhqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ioxmxaildrzvitrueafdfyfmlnodxkzi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drllhgtohcfzazszyuebiinvynwdhspj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ylnadxmdudxqrbccfstikhxvbzzpzitd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rwwhrymzbkksjjazqsbkfhhnvbdutxri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qpujjawamkndlhfydlozxbjsbupivqpg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqbvfbgnqijnqhrffoahsemafzjsqvdh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtkpvsyfdzhxlnnwqsnnuauxyyellvis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"toftclnosxtgkyypteczvfudjqojumgi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgoprwlcqzfuilwbiypnoifwycbzajwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wukkycmfhctqlvqsyowcdtnithkhfclr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"czheusmchgodznymzpoldjjucrgoaoah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ihjfsyssfgxrohvxirpniedtqydlgduy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixelrigmeqfacfunzcnblrxdmyscuqxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmslnejiqusybhbyzklaakmwajeuwgqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctuwuyikthqltoihdhthcparsztzwlxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oryoegawbrftjdzaeczureecxmuepmhs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfhwiwczqwvqalavmbxrdxyobvgihjzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ymhgcceekbudxumynkpflioztpzgpinr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mepxccnkmjscxkrueouytuwhvggtspwc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzdxzehyesrcfdonufgzadgrfydswrvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpfpiargzhtwfiwzmolcjbtccemquvxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lcqrxpyszakljlrhgxfxxcppuakbgjgr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zktkszjaicmbnxaiyyrxueiiaxoymqyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsdqhwnzdcvtkcseaxmvzarvxsjfuxsz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udhdgruohkjjdemukbhrdmvkiwmqemmp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aoxzkkxqhmpltqxitomciqonwlueseup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjtsvaybvpbzdeymdvtnhtdvokcdpcae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cotnvfnlzrjvjkpyszdzvdsldpokmjxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fvsgmvcfafbgzvvmywkdzdfuaarmbsjw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdxdsnlnwjdcbxqdjosgmcccdshawvfj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znbqufpalbqjhehriniaclizuwtkbgmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"baabvuqumayldkkkgsikyfmeucojrter","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofxsprtynpvnvnqaswfgmqfljyjyrunu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehincrgzlaogluorkcnegrqceaqmnrym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbkjgnjzytxfdlfbkdmjscsuqzjcuvyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vvtjfjtonzsyvzuvcdxfxilrusykgnjc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pehpndkihkmuwvqlbzwwxilyfnoupyah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opeiecqfhvnnrlzutmtcxssmorwppihg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfnlctjxizsqkvclpofyrqvpswcoiyhr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbrqmiedryydahpvyizjnoxwpugnfjxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jxfgzomvreyxrbhcisuclcgjxfsyzshh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpinirslzhjmurgpilvlggxkimttcyxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvxucfccdpvaoupwtkxwgrtanjaqqcut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbetwdiqlhwyjdeddnpigfgfrpcpjyyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxbdaerhnnwomuslqiduzdcxdcybunuu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlwbgvhksdataytxafvsijapwziezlli","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orghxiaqgyslkngbzysrzkcbpytqjxse","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gymvnlvmwxfppxxrpqetqkdacocycrpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tthmfcgxlzgdwjpcswmmvvpycrdqkrcx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocwtaifuzvvlivftpnlmjfvuhrqtvauf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"anenoszfdnpegkgniqtjelurwldocdke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmryqcznyoefslyrigkpnkdipbhhucdd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671642,"databaseName":"models_schema","ddl":"CREATE TABLE `tzlncpqzqjgadnlmxsvbcgcpiljusbbe` (\n `wephehgecoqruvgvkndvlqiycpqtbrgt` int NOT NULL,\n `glhydrrklotccgozbskzibkokkfiupzr` int DEFAULT NULL,\n `nvgmnxcjwkpcqcukvwusdseyouqogpmy` int DEFAULT NULL,\n `qmdnyeyuzgsajsfsyomzsnzjrvkvxiip` int DEFAULT NULL,\n `okmohrxgdonhcwuryizqborrgrwnlqwb` int DEFAULT NULL,\n `phfmbbqbwvizneeotizcaslhxvbvimfj` int DEFAULT NULL,\n `svzlvifnmsfvbhwlwcdncvveysvnlupg` int DEFAULT NULL,\n `icopqrqbqvwhrhmvunpjdzwxyepzyyus` int DEFAULT NULL,\n `ocamhqpcivqybascwlpuyfaohdvkexsn` int DEFAULT NULL,\n `hssspjvotdhwvjkpwbsicwnndnbuewud` int DEFAULT NULL,\n `psorrwqnmekzkwpmrapmckvcgohqpltm` int DEFAULT NULL,\n `hylywzykrtextxgkvpuknrenziyqeoen` int DEFAULT NULL,\n `xexgjfqbeymbuowqxkxuspurdvaubexb` int DEFAULT NULL,\n `jvirmwtehjqsqvdjwolzaqyynafilryv` int DEFAULT NULL,\n `kpfhhrzllbdaiyxhbmilsmchsmgvmozo` int DEFAULT NULL,\n `xysdvpgqnvjpphfphpewwnwsxjpadsuv` int DEFAULT NULL,\n `vvohalnbjbgcwboihcryzlzojovmoidd` int DEFAULT NULL,\n `dxnwvkmmicnanymmnhwcqbtnlumisqix` int DEFAULT NULL,\n `wxyegqpwaveuhuvnzerkhxocvecyrcyg` int DEFAULT NULL,\n `xpudocietnfykwgpmqrfleasemzzdeaq` int DEFAULT NULL,\n `bwgsiljfwotayavuuhviofktsonlopro` int DEFAULT NULL,\n `mrzocwteaqlmqkuybiuwmxqueyuwnupu` int DEFAULT NULL,\n `njnfaehlxtaebgoiaznwvxxlwpfslgpw` int DEFAULT NULL,\n `rloyocdgrvazhjarjdpkyjajdkwjjlxx` int DEFAULT NULL,\n `bupexhfnzhebwevrjpzhwnzfqslcngee` int DEFAULT NULL,\n `ewgrwphuaejvghxqeqraquevmxzjxbav` int DEFAULT NULL,\n `dicuihifcxcxbzrvbfcvivnsfsjuzhug` int DEFAULT NULL,\n `irlfbigxoqyihaxpilawygmipcwixdzg` int DEFAULT NULL,\n `fzdlzujojsnxwhfpaqvseiovfnvomsbs` int DEFAULT NULL,\n `fmaetpjkicpfnaqzlzpkjoutnqbkbpat` int DEFAULT NULL,\n `ggygluvbdjbxeawiktncnszkdyhpzcda` int DEFAULT NULL,\n `fvhqxznuqnqqierxqeresvoxkkrvretk` int DEFAULT NULL,\n `moygpdzgxlpuvflshrsfmhrxzsovtowv` int DEFAULT NULL,\n `tddcrrkknhidqremrqgvwadxiusdeyyl` int DEFAULT NULL,\n `zknbirsjqkiffnjxfhttcwgpnzzvywzn` int DEFAULT NULL,\n `jjpgzybtbxxapmoueyykiruzypzddgyp` int DEFAULT NULL,\n `vluwjqkvkturuljbqfohqzwcrogwbxhy` int DEFAULT NULL,\n `efsojjcqeuqxpofjlfelkijjpvuagjqf` int DEFAULT NULL,\n `eysybglhmzobfzgthrihwtrlnsiuraci` int DEFAULT NULL,\n `pfdvmccpxntrsnwmvlkkgwnyuiucblbp` int DEFAULT NULL,\n `pxxyzrpljplkuskxmpdlgregcbwnfabj` int DEFAULT NULL,\n `asucvvrbjzjcihqrjoyebgbqdpzsrnix` int DEFAULT NULL,\n `qtqdveaihvomywehhiutedrznckmdrol` int DEFAULT NULL,\n `mvjysojwxydevtnmhcrpktkyrleashgd` int DEFAULT NULL,\n `zmhqskqkblvudqqdcsdgmoyxxioknaap` int DEFAULT NULL,\n `prfyzfcwejdxiadmawhkllgbmbwfwbcn` int DEFAULT NULL,\n `mumqbzinliptppcnfrtknodfguqavvvf` int DEFAULT NULL,\n `dntthbdbpdotyazlpeshutriqukrojfg` int DEFAULT NULL,\n `awhspuohmmmopzlbhqhoxkkwujqtksbc` int DEFAULT NULL,\n `utlmkwvccvateehvboynokwpkplecdcc` int DEFAULT NULL,\n `vbbtyzmrmtxqcslybnviiiikxpujdcap` int DEFAULT NULL,\n `ldnurxkdgnpzzuodxunwhbmjofnagalc` int DEFAULT NULL,\n `xnvevnkfxsojwtfnflpkxooymyxrsqea` int DEFAULT NULL,\n `zzggnsbtilttrucitxjlcgpfmcqxcqrb` int DEFAULT NULL,\n `qrxbsnbnplrltogwltlzbumgjkzzhoeb` int DEFAULT NULL,\n `vlcekhohxverzopclnumdztckfcqvkem` int DEFAULT NULL,\n `rlwdnmlykerrcvymnqxmzjspifojqvpz` int DEFAULT NULL,\n `ysdkcjocfgioljnyjgbbybglkeozfibe` int DEFAULT NULL,\n `ddnglxysyhltzryofpcssbuqpttuybpn` int DEFAULT NULL,\n `nkkmfklymixinpuzsozczfyjclxeycvn` int DEFAULT NULL,\n `jmkvfqynasuwkeljxzjtdkvrlcgloakh` int DEFAULT NULL,\n `rhgocqqvkikhmovrkcdfilabtcetzeql` int DEFAULT NULL,\n `yovlzdwiwzcwtmfjfxpgtouqedykzudg` int DEFAULT NULL,\n `gllrodxnudunxexgzvpgpfqqyuqygybe` int DEFAULT NULL,\n `ounsibspzrwrypmqaflaynonrsbofjzk` int DEFAULT NULL,\n `rlsahmyomrtpzbdoodqfdxddcloanfqm` int DEFAULT NULL,\n `xjqokgsqvcqomasxmfbmgkmqoxbpqgqh` int DEFAULT NULL,\n `dxtakfregrqqaxmphkozuwiiwhchlyrj` int DEFAULT NULL,\n `lgtycatzuxxflirmqehcoicibkztcgry` int DEFAULT NULL,\n `lbiirfadknlwmvoeiittqspmwbltdwtw` int DEFAULT NULL,\n `tuhrjmiivjxsypbfvimwanrwkbfmhxqh` int DEFAULT NULL,\n `yytztmvczzesglqahrsxrdmpyixapubk` int DEFAULT NULL,\n `swcqfbiyxliczfhpwnyfneuzqnjmhpgz` int DEFAULT NULL,\n `gffmrmrgrlunkquzichdgsekvmrwnffo` int DEFAULT NULL,\n `upfkulyyvfskvtdefnlkjbtinicokovz` int DEFAULT NULL,\n `sewyyxqnmflhqxyoolqqzwewnrvtfygt` int DEFAULT NULL,\n `weohiynvjhtqxlacrakukwxpewqgsqpr` int DEFAULT NULL,\n `kewtscdvsgljmxrdrvvrdgprdicrsmxi` int DEFAULT NULL,\n `qkkuxsauhkqkojijwfmhcrvqunxgavow` int DEFAULT NULL,\n `lqetarwzhlqjdjllgfjwfraqkqeixeoj` int DEFAULT NULL,\n `tgdlxfkavwwndesiiyrbtxivsabqfigu` int DEFAULT NULL,\n `idsorcljirmnfflgttmqqdnppnytfztl` int DEFAULT NULL,\n `oyuogpxiztvqzlbfinubbeptfixzbmff` int DEFAULT NULL,\n `mbyvumtlpgthabrtsebmwkiquthpfnae` int DEFAULT NULL,\n `gzxddhfmitzxasbospeoocwxhavgbjxf` int DEFAULT NULL,\n `jrykipdtximhsqssliefbbxyjyqwcmoh` int DEFAULT NULL,\n `iixepoqvhzurpwjhjohxfmwesgrstedd` int DEFAULT NULL,\n `hjedzhoshnhkmavcujijvjwwolakudhu` int DEFAULT NULL,\n `ugotgorcfxxxtnhfpxstttrrkcjycmhz` int DEFAULT NULL,\n `ajwcsjxygvbemtpaewqzktnnihryolvx` int DEFAULT NULL,\n `fbtyiffhujddpqemogagijvumetlufby` int DEFAULT NULL,\n `tfwpefjhtzkmgdifpvwwromchqvherwr` int DEFAULT NULL,\n `kpgexfckcwvwnprnfwxppdiaiybsovig` int DEFAULT NULL,\n `rboyeiqmcwsoxolwchjdmtfchpbimfcq` int DEFAULT NULL,\n `ypejrnhoqzhorjhsjrifjlabtjxptwbk` int DEFAULT NULL,\n `luzmmwjdcjjuavdgnyzvnqdhrnkrgbhj` int DEFAULT NULL,\n `elgmxvwpwwoolzmuzjbstceslsmiijhk` int DEFAULT NULL,\n `xkayqiajyafeubhztejnswmzfufhzxdc` int DEFAULT NULL,\n `minszlgcyrokwwbtivihcodbdtqegyzy` int DEFAULT NULL,\n `madsqqgvgbtvioonqvkyxkgvgapgswcu` int DEFAULT NULL,\n PRIMARY KEY (`wephehgecoqruvgvkndvlqiycpqtbrgt`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"tzlncpqzqjgadnlmxsvbcgcpiljusbbe\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["wephehgecoqruvgvkndvlqiycpqtbrgt"],"columns":[{"name":"wephehgecoqruvgvkndvlqiycpqtbrgt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"glhydrrklotccgozbskzibkokkfiupzr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvgmnxcjwkpcqcukvwusdseyouqogpmy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmdnyeyuzgsajsfsyomzsnzjrvkvxiip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okmohrxgdonhcwuryizqborrgrwnlqwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phfmbbqbwvizneeotizcaslhxvbvimfj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"svzlvifnmsfvbhwlwcdncvveysvnlupg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icopqrqbqvwhrhmvunpjdzwxyepzyyus","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocamhqpcivqybascwlpuyfaohdvkexsn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hssspjvotdhwvjkpwbsicwnndnbuewud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psorrwqnmekzkwpmrapmckvcgohqpltm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hylywzykrtextxgkvpuknrenziyqeoen","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xexgjfqbeymbuowqxkxuspurdvaubexb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvirmwtehjqsqvdjwolzaqyynafilryv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpfhhrzllbdaiyxhbmilsmchsmgvmozo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xysdvpgqnvjpphfphpewwnwsxjpadsuv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vvohalnbjbgcwboihcryzlzojovmoidd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxnwvkmmicnanymmnhwcqbtnlumisqix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxyegqpwaveuhuvnzerkhxocvecyrcyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpudocietnfykwgpmqrfleasemzzdeaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bwgsiljfwotayavuuhviofktsonlopro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrzocwteaqlmqkuybiuwmxqueyuwnupu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njnfaehlxtaebgoiaznwvxxlwpfslgpw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rloyocdgrvazhjarjdpkyjajdkwjjlxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bupexhfnzhebwevrjpzhwnzfqslcngee","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ewgrwphuaejvghxqeqraquevmxzjxbav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dicuihifcxcxbzrvbfcvivnsfsjuzhug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"irlfbigxoqyihaxpilawygmipcwixdzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzdlzujojsnxwhfpaqvseiovfnvomsbs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmaetpjkicpfnaqzlzpkjoutnqbkbpat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggygluvbdjbxeawiktncnszkdyhpzcda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fvhqxznuqnqqierxqeresvoxkkrvretk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"moygpdzgxlpuvflshrsfmhrxzsovtowv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tddcrrkknhidqremrqgvwadxiusdeyyl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zknbirsjqkiffnjxfhttcwgpnzzvywzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjpgzybtbxxapmoueyykiruzypzddgyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vluwjqkvkturuljbqfohqzwcrogwbxhy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"efsojjcqeuqxpofjlfelkijjpvuagjqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eysybglhmzobfzgthrihwtrlnsiuraci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pfdvmccpxntrsnwmvlkkgwnyuiucblbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxxyzrpljplkuskxmpdlgregcbwnfabj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"asucvvrbjzjcihqrjoyebgbqdpzsrnix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtqdveaihvomywehhiutedrznckmdrol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvjysojwxydevtnmhcrpktkyrleashgd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmhqskqkblvudqqdcsdgmoyxxioknaap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prfyzfcwejdxiadmawhkllgbmbwfwbcn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mumqbzinliptppcnfrtknodfguqavvvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dntthbdbpdotyazlpeshutriqukrojfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awhspuohmmmopzlbhqhoxkkwujqtksbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utlmkwvccvateehvboynokwpkplecdcc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbbtyzmrmtxqcslybnviiiikxpujdcap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ldnurxkdgnpzzuodxunwhbmjofnagalc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnvevnkfxsojwtfnflpkxooymyxrsqea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzggnsbtilttrucitxjlcgpfmcqxcqrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrxbsnbnplrltogwltlzbumgjkzzhoeb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vlcekhohxverzopclnumdztckfcqvkem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlwdnmlykerrcvymnqxmzjspifojqvpz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ysdkcjocfgioljnyjgbbybglkeozfibe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddnglxysyhltzryofpcssbuqpttuybpn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkkmfklymixinpuzsozczfyjclxeycvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmkvfqynasuwkeljxzjtdkvrlcgloakh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhgocqqvkikhmovrkcdfilabtcetzeql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yovlzdwiwzcwtmfjfxpgtouqedykzudg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gllrodxnudunxexgzvpgpfqqyuqygybe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ounsibspzrwrypmqaflaynonrsbofjzk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlsahmyomrtpzbdoodqfdxddcloanfqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjqokgsqvcqomasxmfbmgkmqoxbpqgqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxtakfregrqqaxmphkozuwiiwhchlyrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgtycatzuxxflirmqehcoicibkztcgry","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lbiirfadknlwmvoeiittqspmwbltdwtw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tuhrjmiivjxsypbfvimwanrwkbfmhxqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yytztmvczzesglqahrsxrdmpyixapubk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swcqfbiyxliczfhpwnyfneuzqnjmhpgz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gffmrmrgrlunkquzichdgsekvmrwnffo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upfkulyyvfskvtdefnlkjbtinicokovz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sewyyxqnmflhqxyoolqqzwewnrvtfygt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"weohiynvjhtqxlacrakukwxpewqgsqpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kewtscdvsgljmxrdrvvrdgprdicrsmxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qkkuxsauhkqkojijwfmhcrvqunxgavow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqetarwzhlqjdjllgfjwfraqkqeixeoj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tgdlxfkavwwndesiiyrbtxivsabqfigu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idsorcljirmnfflgttmqqdnppnytfztl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oyuogpxiztvqzlbfinubbeptfixzbmff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbyvumtlpgthabrtsebmwkiquthpfnae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzxddhfmitzxasbospeoocwxhavgbjxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jrykipdtximhsqssliefbbxyjyqwcmoh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iixepoqvhzurpwjhjohxfmwesgrstedd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjedzhoshnhkmavcujijvjwwolakudhu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugotgorcfxxxtnhfpxstttrrkcjycmhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajwcsjxygvbemtpaewqzktnnihryolvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbtyiffhujddpqemogagijvumetlufby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tfwpefjhtzkmgdifpvwwromchqvherwr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpgexfckcwvwnprnfwxppdiaiybsovig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rboyeiqmcwsoxolwchjdmtfchpbimfcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypejrnhoqzhorjhsjrifjlabtjxptwbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"luzmmwjdcjjuavdgnyzvnqdhrnkrgbhj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elgmxvwpwwoolzmuzjbstceslsmiijhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkayqiajyafeubhztejnswmzfufhzxdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"minszlgcyrokwwbtivihcodbdtqegyzy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"madsqqgvgbtvioonqvkyxkgvgapgswcu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671673,"databaseName":"models_schema","ddl":"CREATE TABLE `udikvvdsawrybndbmlsacvseeqbudtkz` (\n `bxzojuplpkznqxnywoaytcoydbrrqbcp` int NOT NULL,\n `hcgcshygkugayyeulfxayozzylltcdvb` int DEFAULT NULL,\n `hmslvfcldiazehimdhdipminyxchshqx` int DEFAULT NULL,\n `tztmdqisfdxulhprqnuzsfsgukbpmtxs` int DEFAULT NULL,\n `kurbyewqwzvcpinxgzplzhmmsmbhsoan` int DEFAULT NULL,\n `rusnsyypdhgxbaordwlgzmwvckzcirho` int DEFAULT NULL,\n `zmuymfvjxpqewmiobrtmiofpitlyrxhq` int DEFAULT NULL,\n `alufvhuzhdujuhjbzqmlhcxbpdakbnbk` int DEFAULT NULL,\n `ghgztxnzdrnpqletakglwojsutcnqewq` int DEFAULT NULL,\n `gupgvfcshzjmpuuachxtjqfvlnpkqwgt` int DEFAULT NULL,\n `lwuyqrdxhbvduppsqsgpuoaocibdcsie` int DEFAULT NULL,\n `qdhipvhforhovknlrxlyauyahlzmilha` int DEFAULT NULL,\n `zsegjcplrfvejilonohkzvpnxbowjxvf` int DEFAULT NULL,\n `ilopfppflwovrasuqyjyvyibfpcrogln` int DEFAULT NULL,\n `adlgtmuvlyyxvjxzwihxqyiyuxuxqwms` int DEFAULT NULL,\n `mqiljbaxphamyqpxapxgbbmtmwarxaqo` int DEFAULT NULL,\n `kojdneyybpciagfohtmatmdqermqyfbu` int DEFAULT NULL,\n `vztkdhhoiymsylllbxsrzfammfipfgzr` int DEFAULT NULL,\n `scqdonewimewskeomtygqyujdybtbkgc` int DEFAULT NULL,\n `liffwddaayvidsayxhdsjaixngazigmj` int DEFAULT NULL,\n `ukzlpfoluynpdvyuudokwdkmutrkcsim` int DEFAULT NULL,\n `gnueoklduniqxsdjjndliczpvrrmjdof` int DEFAULT NULL,\n `oorqdxhufvfycciigauerolvmbfonrbq` int DEFAULT NULL,\n `wenoipqedmqfdfsibupybuzwtxzimzka` int DEFAULT NULL,\n `ccjpidkrlkdzovrbxwoqcbgqwliqdrzq` int DEFAULT NULL,\n `jpplivceyaoskifpamcjhjzmknsfwfop` int DEFAULT NULL,\n `emyqbjdbfnjksbdzicbgobrfxqjvdygg` int DEFAULT NULL,\n `leshrnydpzywclpnhfwrklkwdibuwcku` int DEFAULT NULL,\n `gntkgozkttfjqudxpkjrnqulkvaptcsa` int DEFAULT NULL,\n `vxqazpmzwupyahmuvgiafcdgpcrrtngw` int DEFAULT NULL,\n `ayaugujmzrmronwrbhfdsojyralzafxt` int DEFAULT NULL,\n `czhkcjlgjonrckvfdcacsifxjtwsenzf` int DEFAULT NULL,\n `mediqapiwkqwmxspfzcmfaeiexzrzwlm` int DEFAULT NULL,\n `rokdkiucijeefwvnwyvplviaommtxdkz` int DEFAULT NULL,\n `zvnttfxukqlyidpblihktyxsuqfishdt` int DEFAULT NULL,\n `uhwqxftbqhorhwxmeitcwylpwemwoiiw` int DEFAULT NULL,\n `igolqmfbkfjdlfmczkfiujskjyibujdw` int DEFAULT NULL,\n `lapknwvqukuxwfbojgglnqprcixinzyh` int DEFAULT NULL,\n `tfrbojfwqpxvqeelvjnocbywwmpmibap` int DEFAULT NULL,\n `wjdqpbovdvmodqfmibwqvwoqaptwmlsj` int DEFAULT NULL,\n `fpatrhpkoiuyhxbsfgreyjclcvmmubpk` int DEFAULT NULL,\n `iiqsxhnjsdqkytjhjjazsggwecugatgs` int DEFAULT NULL,\n `unreqispcqfsmqqcdaqkmuopcbgarmga` int DEFAULT NULL,\n `gsnodkwuqixntpvoirucytzdmqmxhrgu` int DEFAULT NULL,\n `oeenaaewutqrqjwmfsrrcxnccgafmgsb` int DEFAULT NULL,\n `cqjbubbguyibgqbzohckllewstokrfrl` int DEFAULT NULL,\n `jdgyevrupfnquvxipdwgszparxpmrisr` int DEFAULT NULL,\n `dwccdcbwxqeptzdqqkdvffomxvqnuhrx` int DEFAULT NULL,\n `nemwsuscsgwyzjzreqewgyuqwkrcfpas` int DEFAULT NULL,\n `lqfhfejmrakadqlpcsrqycpoexlhxdfh` int DEFAULT NULL,\n `uvyyywrgjjhrpdqvhgiuqsklekwpudfq` int DEFAULT NULL,\n `sjguageexrpwusgqfugzlrooeqoykoyb` int DEFAULT NULL,\n `lkrvnrbpvgpqgjoclcbdatutnvlyvrwk` int DEFAULT NULL,\n `iychfcvrigbeuholaxlvuobvtnbnwxjc` int DEFAULT NULL,\n `pvdfudhjbsisbwnimrdkapbpakrisrrg` int DEFAULT NULL,\n `diferszjstgzhgiedfsiozcitxyxfpaz` int DEFAULT NULL,\n `amedomgomesntzbegluvcbimrthdlyel` int DEFAULT NULL,\n `uksqalaydphhdrrrkyhflvlnuudpvulu` int DEFAULT NULL,\n `cbxqbfqkduxedywvvefizlfemkbmhxgy` int DEFAULT NULL,\n `iqrabihhvhnbimyqhrlfbjgzxrioybbh` int DEFAULT NULL,\n `mvwrektkrryfzohmczorsooblfdkuwfe` int DEFAULT NULL,\n `dwrtgjntzyspgesnhappcdrgzgsuhetx` int DEFAULT NULL,\n `vtxtbihyrscjmkmcqvvdhjjrndptahpc` int DEFAULT NULL,\n `sbfjxkutnizouglxjnpjqrmajynetwhu` int DEFAULT NULL,\n `jhdxoumfkrknxrrmglrfgjiefzzqbrjv` int DEFAULT NULL,\n `suryictycqqgcunqcmykvkzeprpcxiqd` int DEFAULT NULL,\n `cdzlbvrloljqgojcsdpinlaiwvijgqlh` int DEFAULT NULL,\n `lrmruflyghmlalktrhsjvacucjdujhre` int DEFAULT NULL,\n `avgbhymfrsvgwxsvrdqcrwtwqwaxdgnl` int DEFAULT NULL,\n `oyeorvwriijhcatgzbcnegnflbojznwk` int DEFAULT NULL,\n `vsenzdsdfbyjeclfpazjiavjtpdvoaph` int DEFAULT NULL,\n `lnqqxhmiitvaowevtjaxbhdakopsnzcf` int DEFAULT NULL,\n `ibxsgqewloavbnggmaqnpnyzwnpajfzk` int DEFAULT NULL,\n `xnydvvdetqxqehxggjjjoyrmnjlglttw` int DEFAULT NULL,\n `pomewxkyjgyypximhrfvninrxcnxulti` int DEFAULT NULL,\n `htsmwiurkpdumlavsrbjxfnriuzhvboy` int DEFAULT NULL,\n `feyqkknclsvcovacyxlauwwswmpqswrl` int DEFAULT NULL,\n `pgqtwjfheynqokaawbvzjbbderqkyijf` int DEFAULT NULL,\n `eplnvprunnrozoilzvvrujqkffjjcfbh` int DEFAULT NULL,\n `bvfeqdlkvwhnnpjimzxnluwufpswqifo` int DEFAULT NULL,\n `bjivkisivgnuzpavgnvvhngopqrgmlzn` int DEFAULT NULL,\n `oetngeeywbdnpekazweyfpxtqaewrbop` int DEFAULT NULL,\n `cfwpfricpwwifucasjkfsdysllblhllb` int DEFAULT NULL,\n `yxirhcclusvkvehdlqtccfdgoeotyhky` int DEFAULT NULL,\n `qcyqadysmxehrtljrhzrltblyftkmpta` int DEFAULT NULL,\n `jvyqkdbxhchymdzpzhchckozplquwdzn` int DEFAULT NULL,\n `ujfjzfkeuzujbovzliwphpgoomzlcpec` int DEFAULT NULL,\n `njowujqggsqctvmokdylrzzfhlweipjb` int DEFAULT NULL,\n `joblcvqsiviclxqxmfsdxhtfohmmcudb` int DEFAULT NULL,\n `jldjrhkaxqflgeevlxsuuedinrdjnaws` int DEFAULT NULL,\n `xaojxvzczolcigstuynjhmpqxqotjmtp` int DEFAULT NULL,\n `omjdnscfzcddogfmjpwifvmvbpscrfix` int DEFAULT NULL,\n `srsneekxpsgivqmugfzuislbynrdcgyl` int DEFAULT NULL,\n `evmwvqmuzxiubfevqufriiytthmuchec` int DEFAULT NULL,\n `ckvalphgvprjqibtaxwnvevaiyxbbmxj` int DEFAULT NULL,\n `yktqczhseeuzigtjsfoxckbfksthznym` int DEFAULT NULL,\n `vgdlhfmvguznruhbgtrwlmgvhxxcptkv` int DEFAULT NULL,\n `ewexwnksmhoiypiwdujqkoknnvimwthy` int DEFAULT NULL,\n `yghktrsanooocnndnaswmaixihldzebo` int DEFAULT NULL,\n `ryeojfuhqrmfyqblhkrclfckiymrwdms` int DEFAULT NULL,\n PRIMARY KEY (`bxzojuplpkznqxnywoaytcoydbrrqbcp`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"udikvvdsawrybndbmlsacvseeqbudtkz\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["bxzojuplpkznqxnywoaytcoydbrrqbcp"],"columns":[{"name":"bxzojuplpkznqxnywoaytcoydbrrqbcp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"hcgcshygkugayyeulfxayozzylltcdvb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmslvfcldiazehimdhdipminyxchshqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tztmdqisfdxulhprqnuzsfsgukbpmtxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kurbyewqwzvcpinxgzplzhmmsmbhsoan","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rusnsyypdhgxbaordwlgzmwvckzcirho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmuymfvjxpqewmiobrtmiofpitlyrxhq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"alufvhuzhdujuhjbzqmlhcxbpdakbnbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghgztxnzdrnpqletakglwojsutcnqewq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gupgvfcshzjmpuuachxtjqfvlnpkqwgt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwuyqrdxhbvduppsqsgpuoaocibdcsie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdhipvhforhovknlrxlyauyahlzmilha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zsegjcplrfvejilonohkzvpnxbowjxvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ilopfppflwovrasuqyjyvyibfpcrogln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"adlgtmuvlyyxvjxzwihxqyiyuxuxqwms","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqiljbaxphamyqpxapxgbbmtmwarxaqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kojdneyybpciagfohtmatmdqermqyfbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vztkdhhoiymsylllbxsrzfammfipfgzr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scqdonewimewskeomtygqyujdybtbkgc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"liffwddaayvidsayxhdsjaixngazigmj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ukzlpfoluynpdvyuudokwdkmutrkcsim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnueoklduniqxsdjjndliczpvrrmjdof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oorqdxhufvfycciigauerolvmbfonrbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wenoipqedmqfdfsibupybuzwtxzimzka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ccjpidkrlkdzovrbxwoqcbgqwliqdrzq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpplivceyaoskifpamcjhjzmknsfwfop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"emyqbjdbfnjksbdzicbgobrfxqjvdygg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"leshrnydpzywclpnhfwrklkwdibuwcku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gntkgozkttfjqudxpkjrnqulkvaptcsa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxqazpmzwupyahmuvgiafcdgpcrrtngw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ayaugujmzrmronwrbhfdsojyralzafxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"czhkcjlgjonrckvfdcacsifxjtwsenzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mediqapiwkqwmxspfzcmfaeiexzrzwlm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rokdkiucijeefwvnwyvplviaommtxdkz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvnttfxukqlyidpblihktyxsuqfishdt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhwqxftbqhorhwxmeitcwylpwemwoiiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igolqmfbkfjdlfmczkfiujskjyibujdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lapknwvqukuxwfbojgglnqprcixinzyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tfrbojfwqpxvqeelvjnocbywwmpmibap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjdqpbovdvmodqfmibwqvwoqaptwmlsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpatrhpkoiuyhxbsfgreyjclcvmmubpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iiqsxhnjsdqkytjhjjazsggwecugatgs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unreqispcqfsmqqcdaqkmuopcbgarmga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsnodkwuqixntpvoirucytzdmqmxhrgu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oeenaaewutqrqjwmfsrrcxnccgafmgsb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqjbubbguyibgqbzohckllewstokrfrl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdgyevrupfnquvxipdwgszparxpmrisr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwccdcbwxqeptzdqqkdvffomxvqnuhrx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nemwsuscsgwyzjzreqewgyuqwkrcfpas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqfhfejmrakadqlpcsrqycpoexlhxdfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvyyywrgjjhrpdqvhgiuqsklekwpudfq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjguageexrpwusgqfugzlrooeqoykoyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lkrvnrbpvgpqgjoclcbdatutnvlyvrwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iychfcvrigbeuholaxlvuobvtnbnwxjc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvdfudhjbsisbwnimrdkapbpakrisrrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"diferszjstgzhgiedfsiozcitxyxfpaz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amedomgomesntzbegluvcbimrthdlyel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uksqalaydphhdrrrkyhflvlnuudpvulu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbxqbfqkduxedywvvefizlfemkbmhxgy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqrabihhvhnbimyqhrlfbjgzxrioybbh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvwrektkrryfzohmczorsooblfdkuwfe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwrtgjntzyspgesnhappcdrgzgsuhetx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtxtbihyrscjmkmcqvvdhjjrndptahpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbfjxkutnizouglxjnpjqrmajynetwhu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jhdxoumfkrknxrrmglrfgjiefzzqbrjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suryictycqqgcunqcmykvkzeprpcxiqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cdzlbvrloljqgojcsdpinlaiwvijgqlh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lrmruflyghmlalktrhsjvacucjdujhre","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avgbhymfrsvgwxsvrdqcrwtwqwaxdgnl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oyeorvwriijhcatgzbcnegnflbojznwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsenzdsdfbyjeclfpazjiavjtpdvoaph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lnqqxhmiitvaowevtjaxbhdakopsnzcf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibxsgqewloavbnggmaqnpnyzwnpajfzk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnydvvdetqxqehxggjjjoyrmnjlglttw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pomewxkyjgyypximhrfvninrxcnxulti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htsmwiurkpdumlavsrbjxfnriuzhvboy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"feyqkknclsvcovacyxlauwwswmpqswrl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgqtwjfheynqokaawbvzjbbderqkyijf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eplnvprunnrozoilzvvrujqkffjjcfbh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvfeqdlkvwhnnpjimzxnluwufpswqifo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjivkisivgnuzpavgnvvhngopqrgmlzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oetngeeywbdnpekazweyfpxtqaewrbop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfwpfricpwwifucasjkfsdysllblhllb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxirhcclusvkvehdlqtccfdgoeotyhky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qcyqadysmxehrtljrhzrltblyftkmpta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvyqkdbxhchymdzpzhchckozplquwdzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujfjzfkeuzujbovzliwphpgoomzlcpec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njowujqggsqctvmokdylrzzfhlweipjb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"joblcvqsiviclxqxmfsdxhtfohmmcudb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jldjrhkaxqflgeevlxsuuedinrdjnaws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xaojxvzczolcigstuynjhmpqxqotjmtp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omjdnscfzcddogfmjpwifvmvbpscrfix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"srsneekxpsgivqmugfzuislbynrdcgyl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evmwvqmuzxiubfevqufriiytthmuchec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckvalphgvprjqibtaxwnvevaiyxbbmxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yktqczhseeuzigtjsfoxckbfksthznym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgdlhfmvguznruhbgtrwlmgvhxxcptkv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ewexwnksmhoiypiwdujqkoknnvimwthy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yghktrsanooocnndnaswmaixihldzebo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ryeojfuhqrmfyqblhkrclfckiymrwdms","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671704,"databaseName":"models_schema","ddl":"CREATE TABLE `udizceykeqmjnuuejwuufbzihqpwbzfp` (\n `bxgnqzsycnlkzseaktegygswkmknbqjz` int NOT NULL,\n `mrbipgatshfwoksdshszgsczsauybett` int DEFAULT NULL,\n `rzqelhfsdjxaygzotmlcegiqshsktvzl` int DEFAULT NULL,\n `oxmbtvspwmofqsvmaewycezwhjzhfhsi` int DEFAULT NULL,\n `iqykdzxhstscuefikrczqjswnxbnjzst` int DEFAULT NULL,\n `oymkzztfqcnznoxnlcgpkzwmkfhbgmwp` int DEFAULT NULL,\n `sguzeeojcsalorraudmqycqidotybbgl` int DEFAULT NULL,\n `tkjjbuijsjqtsjfwcysmlrtgsqdjnyzi` int DEFAULT NULL,\n `mkqiififvajblctbrozhkhyyljefpdwj` int DEFAULT NULL,\n `bodtqylgxuezuemnioouwqfhggnleisu` int DEFAULT NULL,\n `mlxrckmtallrnjmvuclwrrjuanuazwdz` int DEFAULT NULL,\n `rufgfvoksnecacjjbluolyhjcuivbrbq` int DEFAULT NULL,\n `arushphcvbodpmnmtyznhiauvdfuihzz` int DEFAULT NULL,\n `yqyjbgpckhdbmghcvhmzclauhtclnajh` int DEFAULT NULL,\n `wmultmxolzrqjwsogwgbyalawhkoaxae` int DEFAULT NULL,\n `ujnepejwpzuxmyxryaxsjwthraixkobn` int DEFAULT NULL,\n `eqoxeyotklxrfxlztjswfoovjcwejemm` int DEFAULT NULL,\n `byrcmquncpawmoglingnigpzgwlhtijy` int DEFAULT NULL,\n `awqbyqkxwdagvzhqlbmovniootuhiqcs` int DEFAULT NULL,\n `lzpiuxkegspqyoohejxvaybwraucobdd` int DEFAULT NULL,\n `pgfonpgywrelgftabqcwrhrojmttbeeu` int DEFAULT NULL,\n `xiojevbaxhreggxryqjkqvaewlfhrida` int DEFAULT NULL,\n `jrygnxecxliubxecdxxvmqtcpzrczhxz` int DEFAULT NULL,\n `mrpvvegfydaasclbujphtgznvaegxsll` int DEFAULT NULL,\n `gpylfvnbhvkryonvomlbixunijlkenxv` int DEFAULT NULL,\n `wywzawakeazzinqezvpgozuovufksxma` int DEFAULT NULL,\n `bmsunvebnbwnuwpgswwqoxzyvmeugwjg` int DEFAULT NULL,\n `yvrxddyzaigitygabqfxjahbxmtqdiit` int DEFAULT NULL,\n `jyknpqievhmxzulrmnnzltwnnnlkdjnt` int DEFAULT NULL,\n `ovfcurprtwqrpqqzgrbeqppudhkecssq` int DEFAULT NULL,\n `pmxsrgwnusvjxfzlyhddiskiwfxlfvyk` int DEFAULT NULL,\n `zexdcobzrexaovzgnknqioypcamaxexm` int DEFAULT NULL,\n `uqutkcltddvptishnfqnziibvyiswngd` int DEFAULT NULL,\n `rdezyugejmncqbyadpupgxzwjhefyool` int DEFAULT NULL,\n `owlkcmsycahpphxqekiaozvfhoxigurf` int DEFAULT NULL,\n `qyfcjfwmxnbbsdczhgrhjnijuybncqmg` int DEFAULT NULL,\n `vcgdfjpccppjqfljmrgbvezdmlziwoqs` int DEFAULT NULL,\n `xopdqhsgpecudmpfsqyzjcorsmkzuexx` int DEFAULT NULL,\n `breyzfhwlbbpnsprwaqcltgcpolbdqmh` int DEFAULT NULL,\n `zltlmpmusszysdplbzwtcadczfudfnpq` int DEFAULT NULL,\n `ypurqkubksucujghmztzdonryigjrwfl` int DEFAULT NULL,\n `rcvgcoxieigrschuqehvjgvqhzfitfan` int DEFAULT NULL,\n `tgeitmipganqadtirkyugocodbjkgeqs` int DEFAULT NULL,\n `xbtnvifyviwkqqqdzmxjccarudfruzzd` int DEFAULT NULL,\n `zshoscxxbihiafphmfvyesaqvujhgzxi` int DEFAULT NULL,\n `rvzzyvrodaqfwhseizgrtqyezjfmtpet` int DEFAULT NULL,\n `wfpdmomprrobmclsbotexklajqkmerrg` int DEFAULT NULL,\n `cgdkxayaxweeptvcdzsbnlisoofwlgjh` int DEFAULT NULL,\n `zxdwihouoaalxzvldkdjbbbieyfvjfok` int DEFAULT NULL,\n `aqadzkpyyuviyphinyrzscnwersazzwh` int DEFAULT NULL,\n `fecncprgvwgwqvcwmzpwybatqglezrrk` int DEFAULT NULL,\n `kfgpuebiorlsgiskspeanaycpdjnsrad` int DEFAULT NULL,\n `qdnalesncxysmcxzhwalnlwxtjovsazz` int DEFAULT NULL,\n `qsiglreqapdtxnimttwxmntjyapoajwc` int DEFAULT NULL,\n `hvrctdlqpbtyyczlpygehquwgoarcldx` int DEFAULT NULL,\n `lawfxugmkgfycrfhzgbzgxtjtmxnpjtz` int DEFAULT NULL,\n `bnvualmpndefejuwjbnnmqfmuivsutym` int DEFAULT NULL,\n `htnawksmqqoqzzkgrfesxbygwztwybym` int DEFAULT NULL,\n `kctsnsbmjtfngeqyfovlbemuoyvinevt` int DEFAULT NULL,\n `jlkrbbkrpxktruazynuziplpvoazbpup` int DEFAULT NULL,\n `ukqzryepuugqhbwacekhhxcyufeyleom` int DEFAULT NULL,\n `yozdcqglgmsejlybqzgqkcvcsjubwlrd` int DEFAULT NULL,\n `kmttaedxijuvqcxjyjheutwpmcdsfvpm` int DEFAULT NULL,\n `oavlgykvubfkizohidvcgncxiciemhaw` int DEFAULT NULL,\n `raemxfovvppvricoqdysgbvcvtdwvspm` int DEFAULT NULL,\n `rmfyopnnbiaarfsdtpykyemervmmltyl` int DEFAULT NULL,\n `ramrzjxgbwlscyxkmgbagssgepzpfxnr` int DEFAULT NULL,\n `zjkyesednkoiahphqjlmevoztyzddnub` int DEFAULT NULL,\n `jmzalnbngvicvrgmggbgvwsqpqnluksr` int DEFAULT NULL,\n `urelnudbaljusinxtbutcgebwkbuecix` int DEFAULT NULL,\n `icuepuvmvjbwuqpkxfiafegmosbjgqyr` int DEFAULT NULL,\n `vvqffhebdgqhxtylouidxnmnbcvphrbv` int DEFAULT NULL,\n `nsnctexhildsctnnofvncmaipaobhuif` int DEFAULT NULL,\n `nlvcuvklbrextsmtvkipfzsupnoadbyf` int DEFAULT NULL,\n `lwjoexqvtdlaohmtiauegtiwpoppsqni` int DEFAULT NULL,\n `wlsdlkvusmhchbhtpplofqbdovydkzpv` int DEFAULT NULL,\n `amiedjiaazwbiioconnivrbwazseraty` int DEFAULT NULL,\n `parxrcbajyznityxifwphnwattjfyfbg` int DEFAULT NULL,\n `fkewtuqrqgopqjmzngphblyxzeeihnju` int DEFAULT NULL,\n `uzjxxzrjpmnbrtntxyvnvbzquvgxgkva` int DEFAULT NULL,\n `mhqxhynlqhfkptilqamdzkawwtxaydyf` int DEFAULT NULL,\n `annmnuhtnokijcqobexotuzgtdhwezwh` int DEFAULT NULL,\n `eposykyhghmkzevtdxgpwojjisntbcqk` int DEFAULT NULL,\n `ugxkiogueumwfholdhgmyqbpmvohstyi` int DEFAULT NULL,\n `sacrugrverbsismdtwilztesmtrszexs` int DEFAULT NULL,\n `uztjpgubmgwtxzmyzmhmoomnaqzgxseg` int DEFAULT NULL,\n `kbtglzlccxwwyzkdecwptvvxvhcydumn` int DEFAULT NULL,\n `fyjmspdxaymgdrclahqghqfyqahnvnhs` int DEFAULT NULL,\n `bchwticdqfxqidizhqgpugcixdxdyokd` int DEFAULT NULL,\n `xevwzkuxfoyqspenqyngeugscqqbzcpv` int DEFAULT NULL,\n `gubgmpokoodgnfhrahlssmurkjkqjisy` int DEFAULT NULL,\n `ynxajozuxkkydwznvwphbdkatygmxatn` int DEFAULT NULL,\n `cmdimccakbwmkdrcgomoewfgapfobakg` int DEFAULT NULL,\n `tbqnzsbpekgbvczyqmimezgpznfyelyp` int DEFAULT NULL,\n `qjsojnqhzepzykfocoepidygcesqcndt` int DEFAULT NULL,\n `cpepzijacrbpjpgrzvaxfdkbiczrywlq` int DEFAULT NULL,\n `emkrpnyyjwsmtkujtacxcpkuzdcrrgjv` int DEFAULT NULL,\n `erkdrcjtfipbdjwgrmkyvisumokqwgyz` int DEFAULT NULL,\n `izrvgxpnblwxehzqyybkgxqpasqcuptx` int DEFAULT NULL,\n `fyrrxmaruebncjllfyckakqrextifsdw` int DEFAULT NULL,\n PRIMARY KEY (`bxgnqzsycnlkzseaktegygswkmknbqjz`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"udizceykeqmjnuuejwuufbzihqpwbzfp\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["bxgnqzsycnlkzseaktegygswkmknbqjz"],"columns":[{"name":"bxgnqzsycnlkzseaktegygswkmknbqjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"mrbipgatshfwoksdshszgsczsauybett","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzqelhfsdjxaygzotmlcegiqshsktvzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oxmbtvspwmofqsvmaewycezwhjzhfhsi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqykdzxhstscuefikrczqjswnxbnjzst","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oymkzztfqcnznoxnlcgpkzwmkfhbgmwp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sguzeeojcsalorraudmqycqidotybbgl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkjjbuijsjqtsjfwcysmlrtgsqdjnyzi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkqiififvajblctbrozhkhyyljefpdwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bodtqylgxuezuemnioouwqfhggnleisu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlxrckmtallrnjmvuclwrrjuanuazwdz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rufgfvoksnecacjjbluolyhjcuivbrbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"arushphcvbodpmnmtyznhiauvdfuihzz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqyjbgpckhdbmghcvhmzclauhtclnajh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmultmxolzrqjwsogwgbyalawhkoaxae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujnepejwpzuxmyxryaxsjwthraixkobn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqoxeyotklxrfxlztjswfoovjcwejemm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"byrcmquncpawmoglingnigpzgwlhtijy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awqbyqkxwdagvzhqlbmovniootuhiqcs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzpiuxkegspqyoohejxvaybwraucobdd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgfonpgywrelgftabqcwrhrojmttbeeu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xiojevbaxhreggxryqjkqvaewlfhrida","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jrygnxecxliubxecdxxvmqtcpzrczhxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrpvvegfydaasclbujphtgznvaegxsll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gpylfvnbhvkryonvomlbixunijlkenxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wywzawakeazzinqezvpgozuovufksxma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bmsunvebnbwnuwpgswwqoxzyvmeugwjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yvrxddyzaigitygabqfxjahbxmtqdiit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jyknpqievhmxzulrmnnzltwnnnlkdjnt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovfcurprtwqrpqqzgrbeqppudhkecssq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmxsrgwnusvjxfzlyhddiskiwfxlfvyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zexdcobzrexaovzgnknqioypcamaxexm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqutkcltddvptishnfqnziibvyiswngd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdezyugejmncqbyadpupgxzwjhefyool","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owlkcmsycahpphxqekiaozvfhoxigurf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qyfcjfwmxnbbsdczhgrhjnijuybncqmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vcgdfjpccppjqfljmrgbvezdmlziwoqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xopdqhsgpecudmpfsqyzjcorsmkzuexx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"breyzfhwlbbpnsprwaqcltgcpolbdqmh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zltlmpmusszysdplbzwtcadczfudfnpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypurqkubksucujghmztzdonryigjrwfl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcvgcoxieigrschuqehvjgvqhzfitfan","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tgeitmipganqadtirkyugocodbjkgeqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbtnvifyviwkqqqdzmxjccarudfruzzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zshoscxxbihiafphmfvyesaqvujhgzxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rvzzyvrodaqfwhseizgrtqyezjfmtpet","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfpdmomprrobmclsbotexklajqkmerrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgdkxayaxweeptvcdzsbnlisoofwlgjh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxdwihouoaalxzvldkdjbbbieyfvjfok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqadzkpyyuviyphinyrzscnwersazzwh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fecncprgvwgwqvcwmzpwybatqglezrrk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfgpuebiorlsgiskspeanaycpdjnsrad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdnalesncxysmcxzhwalnlwxtjovsazz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qsiglreqapdtxnimttwxmntjyapoajwc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hvrctdlqpbtyyczlpygehquwgoarcldx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lawfxugmkgfycrfhzgbzgxtjtmxnpjtz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnvualmpndefejuwjbnnmqfmuivsutym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htnawksmqqoqzzkgrfesxbygwztwybym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kctsnsbmjtfngeqyfovlbemuoyvinevt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlkrbbkrpxktruazynuziplpvoazbpup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ukqzryepuugqhbwacekhhxcyufeyleom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yozdcqglgmsejlybqzgqkcvcsjubwlrd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmttaedxijuvqcxjyjheutwpmcdsfvpm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oavlgykvubfkizohidvcgncxiciemhaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"raemxfovvppvricoqdysgbvcvtdwvspm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rmfyopnnbiaarfsdtpykyemervmmltyl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ramrzjxgbwlscyxkmgbagssgepzpfxnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjkyesednkoiahphqjlmevoztyzddnub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmzalnbngvicvrgmggbgvwsqpqnluksr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urelnudbaljusinxtbutcgebwkbuecix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icuepuvmvjbwuqpkxfiafegmosbjgqyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vvqffhebdgqhxtylouidxnmnbcvphrbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nsnctexhildsctnnofvncmaipaobhuif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nlvcuvklbrextsmtvkipfzsupnoadbyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwjoexqvtdlaohmtiauegtiwpoppsqni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wlsdlkvusmhchbhtpplofqbdovydkzpv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amiedjiaazwbiioconnivrbwazseraty","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"parxrcbajyznityxifwphnwattjfyfbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fkewtuqrqgopqjmzngphblyxzeeihnju","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzjxxzrjpmnbrtntxyvnvbzquvgxgkva","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhqxhynlqhfkptilqamdzkawwtxaydyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"annmnuhtnokijcqobexotuzgtdhwezwh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eposykyhghmkzevtdxgpwojjisntbcqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugxkiogueumwfholdhgmyqbpmvohstyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sacrugrverbsismdtwilztesmtrszexs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uztjpgubmgwtxzmyzmhmoomnaqzgxseg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kbtglzlccxwwyzkdecwptvvxvhcydumn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fyjmspdxaymgdrclahqghqfyqahnvnhs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bchwticdqfxqidizhqgpugcixdxdyokd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xevwzkuxfoyqspenqyngeugscqqbzcpv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gubgmpokoodgnfhrahlssmurkjkqjisy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynxajozuxkkydwznvwphbdkatygmxatn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmdimccakbwmkdrcgomoewfgapfobakg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbqnzsbpekgbvczyqmimezgpznfyelyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjsojnqhzepzykfocoepidygcesqcndt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cpepzijacrbpjpgrzvaxfdkbiczrywlq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"emkrpnyyjwsmtkujtacxcpkuzdcrrgjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"erkdrcjtfipbdjwgrmkyvisumokqwgyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"izrvgxpnblwxehzqyybkgxqpasqcuptx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fyrrxmaruebncjllfyckakqrextifsdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671736,"databaseName":"models_schema","ddl":"CREATE TABLE `uflnnphheekurdaziorsgpfwdysuutca` (\n `rkzvmigwiktvwsllnmqpxvxqesmulihy` int NOT NULL,\n `gewiefdfxpvjglusquhtbyhkviafgaxp` int DEFAULT NULL,\n `jtdkqcqnbnugrxwkjqkpzetywoibhepi` int DEFAULT NULL,\n `pghhwllnyugoibfmxjhrpodvekydgsbf` int DEFAULT NULL,\n `ikaaqrsiwzpzjmudafxpertymgeorxhk` int DEFAULT NULL,\n `xggybmwtqjqtjgtnfgcfiqkurvrxrxfn` int DEFAULT NULL,\n `pppvghfvcyltxrnbldbcmacsdrpsimpz` int DEFAULT NULL,\n `yepcjngzzfuclplmsepsxzstdffdfvmc` int DEFAULT NULL,\n `fwcposgwpifzwwaautfcdtpkolsdlnwi` int DEFAULT NULL,\n `gwjibgipniskqkfpgunwuzrfyojkxvix` int DEFAULT NULL,\n `rfawatzhyzbtxhcnfwpaspejtdtcmccf` int DEFAULT NULL,\n `urslwizhpghdghgwlktaomtusgdlevmm` int DEFAULT NULL,\n `frzbiofazgfjwmfdbqpokjyfccybyalb` int DEFAULT NULL,\n `dgmqnsgkkctgiyqjnkluirgdzepfzyce` int DEFAULT NULL,\n `obilpuwffbyhlxpmbcdynrquknrxoiqr` int DEFAULT NULL,\n `yzdavrcxkdbbkufymbkjwwjtaubqvtkb` int DEFAULT NULL,\n `suzxhmuckxjrvxsbmcrqlfebjzipppag` int DEFAULT NULL,\n `kplirgiriptwzpzbxxfjbjsblojumdys` int DEFAULT NULL,\n `bfnvcywhbuebqckquhsmcppdgredthpv` int DEFAULT NULL,\n `hlcdufdwmplbhmyjthjcvbnmjbhlfqze` int DEFAULT NULL,\n `bwlhtcmzexvkmepljednnbmwflnzxqth` int DEFAULT NULL,\n `lgybcdjxckkvlkvrgbqbayqsbwiwmjyo` int DEFAULT NULL,\n `kcbdbgwsybefvbhcqpffxbkxbclwdfnz` int DEFAULT NULL,\n `gtltjkgkdrohmrorchvyfljowtzjszyv` int DEFAULT NULL,\n `bnfkjhjbyvolzibhigdtulyqdtzupaxi` int DEFAULT NULL,\n `fmjdibfgdanmhradefufokdjsvfoluct` int DEFAULT NULL,\n `yduxcvjxnfjiyrknzhbxcwusivnibhbq` int DEFAULT NULL,\n `nachjvxmtwiaxjhxakruvpsqhevdqivw` int DEFAULT NULL,\n `ucofqozvjglsslqnmbsozvoischlpeuq` int DEFAULT NULL,\n `ssnwtgwlyhkvxxfjgbylaojdrjwvoaor` int DEFAULT NULL,\n `orcisbvvpaktzdeokwqyzsjsrlfyjvrq` int DEFAULT NULL,\n `ehibnozkqhfckyupvdgtgbdvlzzpgqwc` int DEFAULT NULL,\n `wqwgmturzzrqjjlutjxzjtozgptonbey` int DEFAULT NULL,\n `aksftwsyppppnzefhdolsurobkqriyjm` int DEFAULT NULL,\n `ciqpbwbnillhijpdhybmvzwfkmerutlg` int DEFAULT NULL,\n `unxniuanptxsrbjwblxhimawqcpqrtro` int DEFAULT NULL,\n `pyvyjevudbgxeqgyclrwtjinlounxwvp` int DEFAULT NULL,\n `urljaqzplnykdlpjlasqjnzqhjkdwffr` int DEFAULT NULL,\n `zgguereqwouupghyovxrhlknwihvkjog` int DEFAULT NULL,\n `tpqrnzuvnuwcqgtlozzrbkflkyqgqioo` int DEFAULT NULL,\n `vyktnawlorgaevyexmkgfzgkrmqftckw` int DEFAULT NULL,\n `drnztftgmriqwtkqtujmrvvzaxvroiye` int DEFAULT NULL,\n `mcwberewgdwolzkkwtdhecpacogjqxdn` int DEFAULT NULL,\n `cojdltuomsfxsaqsppapfcuoaxxdycik` int DEFAULT NULL,\n `facaamfnijpznsjcwcszggmlbdsmwmjr` int DEFAULT NULL,\n `vgkxgrtjqsetaagdkmhizczuakmorpzx` int DEFAULT NULL,\n `rvhavvycznbzpibfuhelfxowpeuabekh` int DEFAULT NULL,\n `frhkeiykocvgtssmbkunwiusnhfjsvhp` int DEFAULT NULL,\n `xvrfzxyaagbcipmmyrujeycsvpiciuuf` int DEFAULT NULL,\n `ezozbveqgetsnoddlsjtjssxjnwkagkw` int DEFAULT NULL,\n `vghjlotccqvciysvampilsolzpuvhnvu` int DEFAULT NULL,\n `rwpbbpnugqdmparxccqtxmxijzurqcmb` int DEFAULT NULL,\n `nlsgiocjcohkrucoyypludkvdtysbigj` int DEFAULT NULL,\n `axeurdzbtzcttmynvfgueukhtmexzfaw` int DEFAULT NULL,\n `hhxqwkkftuixkzahjmgjnllqwbzqweue` int DEFAULT NULL,\n `tnpsrktsbumeporxmfavhkezapnyvpem` int DEFAULT NULL,\n `leqvmodjufjsjkqwgumaofjzhfbywlfg` int DEFAULT NULL,\n `hqoynenvndtchuqlkttftipbvuvymilb` int DEFAULT NULL,\n `craarudnkadcdrhykqqjeysliuedszsy` int DEFAULT NULL,\n `kcuhotmctpxvamjkwrzcbzfxdrrfcxas` int DEFAULT NULL,\n `gdbqverxgeagrwmzuuycgmdhhvbsuphx` int DEFAULT NULL,\n `jntmwwmjsyazllswimthjvsvqsarydlq` int DEFAULT NULL,\n `ekkleectwmuttszyerkvkkvzbeltigta` int DEFAULT NULL,\n `stfrdzzrkjmbxjtiofvtmdhcainwdcvl` int DEFAULT NULL,\n `vqpmzmxnlyihqsrsqgswlxmqzhrasbue` int DEFAULT NULL,\n `hcuqdczarrpquegcwfjkgwbyfqqequph` int DEFAULT NULL,\n `anuknyeocupmypecaepkduqughyjwkvx` int DEFAULT NULL,\n `xqbiggndoqnpggiufwfqkmnmcocwfnxm` int DEFAULT NULL,\n `xkexgqoutnyvkyydiblemxhmhqdlmqyu` int DEFAULT NULL,\n `gmbmgukfxjbevwmivqicihmvwnmqansp` int DEFAULT NULL,\n `ldxjxtqjwefwletjdparzlrorbdnesvy` int DEFAULT NULL,\n `grudblylvshwjxsatntyaxskfigovgyq` int DEFAULT NULL,\n `yurmlchnhllagdthneoqcqaposjeyksu` int DEFAULT NULL,\n `xsjvhemwvkebubzdbybghucniptjezmr` int DEFAULT NULL,\n `sfdzynhhwdwdagogkokxjgznswndirot` int DEFAULT NULL,\n `igqourtfeinjkkzlqcnuiuzgdgnzupkd` int DEFAULT NULL,\n `wyofjyiwypqqehpvpafashugzzqkwflq` int DEFAULT NULL,\n `xumvgkowvkcjimpwxttcsnqlfmbmrkyt` int DEFAULT NULL,\n `ypykmrfointmeexpzsbumacuouwnikid` int DEFAULT NULL,\n `rlkfdexzhtizddkpwfoaozopxfifuxou` int DEFAULT NULL,\n `xshffrqhmzwmahglpsnbykkkykppocnm` int DEFAULT NULL,\n `cbcepynqogqhyuupgdzxxrpvphcsvywd` int DEFAULT NULL,\n `vpciftawhpxggrdfivgvscpjdeftaxzy` int DEFAULT NULL,\n `mvvxqnpyaqirkxpfshifnabfgtpfnnck` int DEFAULT NULL,\n `gnrheskprndjxyidjwktxzurvaepebfh` int DEFAULT NULL,\n `ywpkijuxcwacdzuutsdgxyghasnizpcq` int DEFAULT NULL,\n `okvsfchlbhmbajzeqrhqxiutrncqbktj` int DEFAULT NULL,\n `aarfjjenqjehmuqcseqajknbwmddhvcr` int DEFAULT NULL,\n `arbblfmqsmedecafymylchslkbgfszoc` int DEFAULT NULL,\n `jycewjtpclinqtimqpdljxrpemtvtgqt` int DEFAULT NULL,\n `pgopgdpilqzwwsnbfrljihkwmjbwkmys` int DEFAULT NULL,\n `difurhfuraojlqyirblendxcbljljswb` int DEFAULT NULL,\n `pnelubjwphfcoeejdkfirqodkpymratp` int DEFAULT NULL,\n `vogysyotmtrwmwljdlexpyltmoefyudq` int DEFAULT NULL,\n `rgrfzjyhklicevrzaklsrdneweenghgl` int DEFAULT NULL,\n `ldprbyjgtxwlxmkxuwjjkmlooyekjyng` int DEFAULT NULL,\n `xvdbftpiklfebxcrjakvxewmhkczljpm` int DEFAULT NULL,\n `twanqwddbukdjdgoopgmzvcvllpcldbn` int DEFAULT NULL,\n `qwjesazlpmayqyuvpckuftjlozmglmgn` int DEFAULT NULL,\n `axfhsjoxxbagynxrflxotmnztlbkiylz` int DEFAULT NULL,\n PRIMARY KEY (`rkzvmigwiktvwsllnmqpxvxqesmulihy`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"uflnnphheekurdaziorsgpfwdysuutca\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["rkzvmigwiktvwsllnmqpxvxqesmulihy"],"columns":[{"name":"rkzvmigwiktvwsllnmqpxvxqesmulihy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"gewiefdfxpvjglusquhtbyhkviafgaxp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jtdkqcqnbnugrxwkjqkpzetywoibhepi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pghhwllnyugoibfmxjhrpodvekydgsbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikaaqrsiwzpzjmudafxpertymgeorxhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xggybmwtqjqtjgtnfgcfiqkurvrxrxfn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pppvghfvcyltxrnbldbcmacsdrpsimpz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yepcjngzzfuclplmsepsxzstdffdfvmc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwcposgwpifzwwaautfcdtpkolsdlnwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gwjibgipniskqkfpgunwuzrfyojkxvix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfawatzhyzbtxhcnfwpaspejtdtcmccf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urslwizhpghdghgwlktaomtusgdlevmm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"frzbiofazgfjwmfdbqpokjyfccybyalb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgmqnsgkkctgiyqjnkluirgdzepfzyce","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obilpuwffbyhlxpmbcdynrquknrxoiqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzdavrcxkdbbkufymbkjwwjtaubqvtkb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suzxhmuckxjrvxsbmcrqlfebjzipppag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kplirgiriptwzpzbxxfjbjsblojumdys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bfnvcywhbuebqckquhsmcppdgredthpv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlcdufdwmplbhmyjthjcvbnmjbhlfqze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bwlhtcmzexvkmepljednnbmwflnzxqth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgybcdjxckkvlkvrgbqbayqsbwiwmjyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcbdbgwsybefvbhcqpffxbkxbclwdfnz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtltjkgkdrohmrorchvyfljowtzjszyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnfkjhjbyvolzibhigdtulyqdtzupaxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmjdibfgdanmhradefufokdjsvfoluct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yduxcvjxnfjiyrknzhbxcwusivnibhbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nachjvxmtwiaxjhxakruvpsqhevdqivw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucofqozvjglsslqnmbsozvoischlpeuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ssnwtgwlyhkvxxfjgbylaojdrjwvoaor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orcisbvvpaktzdeokwqyzsjsrlfyjvrq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehibnozkqhfckyupvdgtgbdvlzzpgqwc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqwgmturzzrqjjlutjxzjtozgptonbey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aksftwsyppppnzefhdolsurobkqriyjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ciqpbwbnillhijpdhybmvzwfkmerutlg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unxniuanptxsrbjwblxhimawqcpqrtro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pyvyjevudbgxeqgyclrwtjinlounxwvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urljaqzplnykdlpjlasqjnzqhjkdwffr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zgguereqwouupghyovxrhlknwihvkjog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpqrnzuvnuwcqgtlozzrbkflkyqgqioo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vyktnawlorgaevyexmkgfzgkrmqftckw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drnztftgmriqwtkqtujmrvvzaxvroiye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcwberewgdwolzkkwtdhecpacogjqxdn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cojdltuomsfxsaqsppapfcuoaxxdycik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"facaamfnijpznsjcwcszggmlbdsmwmjr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgkxgrtjqsetaagdkmhizczuakmorpzx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rvhavvycznbzpibfuhelfxowpeuabekh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"frhkeiykocvgtssmbkunwiusnhfjsvhp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvrfzxyaagbcipmmyrujeycsvpiciuuf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezozbveqgetsnoddlsjtjssxjnwkagkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vghjlotccqvciysvampilsolzpuvhnvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rwpbbpnugqdmparxccqtxmxijzurqcmb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nlsgiocjcohkrucoyypludkvdtysbigj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axeurdzbtzcttmynvfgueukhtmexzfaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhxqwkkftuixkzahjmgjnllqwbzqweue","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnpsrktsbumeporxmfavhkezapnyvpem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"leqvmodjufjsjkqwgumaofjzhfbywlfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqoynenvndtchuqlkttftipbvuvymilb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"craarudnkadcdrhykqqjeysliuedszsy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcuhotmctpxvamjkwrzcbzfxdrrfcxas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdbqverxgeagrwmzuuycgmdhhvbsuphx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jntmwwmjsyazllswimthjvsvqsarydlq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekkleectwmuttszyerkvkkvzbeltigta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"stfrdzzrkjmbxjtiofvtmdhcainwdcvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqpmzmxnlyihqsrsqgswlxmqzhrasbue","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hcuqdczarrpquegcwfjkgwbyfqqequph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"anuknyeocupmypecaepkduqughyjwkvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqbiggndoqnpggiufwfqkmnmcocwfnxm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkexgqoutnyvkyydiblemxhmhqdlmqyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmbmgukfxjbevwmivqicihmvwnmqansp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ldxjxtqjwefwletjdparzlrorbdnesvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grudblylvshwjxsatntyaxskfigovgyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yurmlchnhllagdthneoqcqaposjeyksu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsjvhemwvkebubzdbybghucniptjezmr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfdzynhhwdwdagogkokxjgznswndirot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igqourtfeinjkkzlqcnuiuzgdgnzupkd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyofjyiwypqqehpvpafashugzzqkwflq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xumvgkowvkcjimpwxttcsnqlfmbmrkyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypykmrfointmeexpzsbumacuouwnikid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlkfdexzhtizddkpwfoaozopxfifuxou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xshffrqhmzwmahglpsnbykkkykppocnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbcepynqogqhyuupgdzxxrpvphcsvywd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpciftawhpxggrdfivgvscpjdeftaxzy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvvxqnpyaqirkxpfshifnabfgtpfnnck","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnrheskprndjxyidjwktxzurvaepebfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywpkijuxcwacdzuutsdgxyghasnizpcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okvsfchlbhmbajzeqrhqxiutrncqbktj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aarfjjenqjehmuqcseqajknbwmddhvcr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"arbblfmqsmedecafymylchslkbgfszoc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jycewjtpclinqtimqpdljxrpemtvtgqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgopgdpilqzwwsnbfrljihkwmjbwkmys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"difurhfuraojlqyirblendxcbljljswb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnelubjwphfcoeejdkfirqodkpymratp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vogysyotmtrwmwljdlexpyltmoefyudq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgrfzjyhklicevrzaklsrdneweenghgl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ldprbyjgtxwlxmkxuwjjkmlooyekjyng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvdbftpiklfebxcrjakvxewmhkczljpm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twanqwddbukdjdgoopgmzvcvllpcldbn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwjesazlpmayqyuvpckuftjlozmglmgn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axfhsjoxxbagynxrflxotmnztlbkiylz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671770,"databaseName":"models_schema","ddl":"CREATE TABLE `uhncixqtyhjsicsgvnamwatzwbpgfqld` (\n `wlgfodccepybwnnlwczvcskqxhiicpng` int NOT NULL,\n `pforffmtqznulzclnijmukrriomndeek` int DEFAULT NULL,\n `aqyllktweptbarhdymrwljihwoadfzhi` int DEFAULT NULL,\n `gqtsodnfrndilwzrwvnthewfyziskocz` int DEFAULT NULL,\n `xzqdrkvysjnmcvxvvabqijmormnvbscb` int DEFAULT NULL,\n `rpnfhsynuilccfgmdnggrlllipsqnndj` int DEFAULT NULL,\n `thopurypvkfihjuwlvwckflpljjhvkrz` int DEFAULT NULL,\n `ytuixlhsfqwdxsjvjixhrlqqicbnduxv` int DEFAULT NULL,\n `lcvaqxxoctlsxfjifpzqewofvnnupixo` int DEFAULT NULL,\n `ixodniefjkdfhnihgbzgaejwmywgqrby` int DEFAULT NULL,\n `acnwtocdqmyqqpzdqenjacbtqasrczhs` int DEFAULT NULL,\n `uxhtdzxxrivxfywqbbxphbcbhynsneor` int DEFAULT NULL,\n `dwtftrzsfldalupgrfcnckpxiiyqedls` int DEFAULT NULL,\n `unotrfiiqkrrrgifcoodupjkcovzukyz` int DEFAULT NULL,\n `tekkkqhxczrsbdwdjtsmebuzeitizgjj` int DEFAULT NULL,\n `oiqstkaelrmiaezpjzppixygygbzjsbh` int DEFAULT NULL,\n `byvxvcusnhpvzpzeqltyggidgberedlt` int DEFAULT NULL,\n `vmkdzzmwcfpjbbavqxogfvogbabamhoz` int DEFAULT NULL,\n `dfbwaodpateouuipnxumvtlcphmprwpa` int DEFAULT NULL,\n `pfjtoclobbqzljevezhgihbajpyanuiv` int DEFAULT NULL,\n `gjneljmbprbljfkxrkdukjbiibuiidus` int DEFAULT NULL,\n `wwhgibejgyyrkmmpsnvtumnjvihzbmwe` int DEFAULT NULL,\n `fsbrdvxvmkvvtricmtvzffjcrqbhtxrm` int DEFAULT NULL,\n `zltujcmkteitkaadhjvfsqlcwuecrwyr` int DEFAULT NULL,\n `esdkbwlpjwczgkyhidhilirqlhxwyqjv` int DEFAULT NULL,\n `zijokmcqplmdubrojrwrwpqmjjfuwggy` int DEFAULT NULL,\n `pdpxrzqeiyriqptkkzplfnguldtaxryb` int DEFAULT NULL,\n `qtzgttkbdvtsvjpjituxhqmskdxnxpvz` int DEFAULT NULL,\n `vzuebzlmsnccaxyqlxtlyxhhxbjrtbsa` int DEFAULT NULL,\n `tdvptngptevljrnhdkprgeswodtmdrni` int DEFAULT NULL,\n `teiitsbskilfsqyljmvjdryockaamcbz` int DEFAULT NULL,\n `hczjghykcqnlrbcmnhuhiedohokynwuq` int DEFAULT NULL,\n `ckhhbfwnywifswkqdylarejpcedsrlrm` int DEFAULT NULL,\n `pdgmwbyloxjdzrdbougkwryqprmabxdb` int DEFAULT NULL,\n `nnfwuqzqbqmvvwenscedfgmzgnyvxwut` int DEFAULT NULL,\n `aflqzygdfbtshgelbrredyudzkdjqwxk` int DEFAULT NULL,\n `trawykqqnucqinzttvhucbmxdjwkztds` int DEFAULT NULL,\n `berbrgqdihwptyzqrvztvqhrfodettwv` int DEFAULT NULL,\n `jgndhfejhvlrpvibznwozatbzbvnqqjn` int DEFAULT NULL,\n `vxwohmoluzguhiqluotgboonvumibjsd` int DEFAULT NULL,\n `mtbxxdmboqxebmyhjdcfrmtlonzkehig` int DEFAULT NULL,\n `qslvjqbblivpbqmxbdrwwahtjloaijjo` int DEFAULT NULL,\n `aclyiwxyvhqrmbxhuoyddxpbbkczayxg` int DEFAULT NULL,\n `qfccuqdxtnxautsssrhgtbyxakybipaa` int DEFAULT NULL,\n `vxmqzurthxzttzschfalsuqvjfvnudva` int DEFAULT NULL,\n `iggdpzuqiewmbijvvseitosspflcdmjq` int DEFAULT NULL,\n `xiebdjdmplmzcvxiaqjycbdrahieraba` int DEFAULT NULL,\n `exnkvsyycvfvfxyprhqaojwtnowiumpe` int DEFAULT NULL,\n `qqhbguolfnrhnmbkujohfhumubhiscjp` int DEFAULT NULL,\n `fswmnwnctkiooduefxnuredqelmlkdmt` int DEFAULT NULL,\n `vsrkjyzkauuonyaharororgemnxdqlle` int DEFAULT NULL,\n `ccpsfblquvolksgxlrhqmggkrssaatch` int DEFAULT NULL,\n `wgcuioejmdptlemdxclkznbfiuktgfcz` int DEFAULT NULL,\n `vejimydriorhvmjuafzqetkprlcnladf` int DEFAULT NULL,\n `ulevxwsipymtechfitsassndfswemfec` int DEFAULT NULL,\n `somgnisbzfszrryticbywouwatmmizvx` int DEFAULT NULL,\n `geqojnbmtprwujzmyaywdafxlwgjwpdz` int DEFAULT NULL,\n `tgsrngpcytsuresletaudbqumzxpzyuq` int DEFAULT NULL,\n `spcgnxwblijdvmgtwhgyvsgpqsqqjvyo` int DEFAULT NULL,\n `eepxlwjtquylgdjconoboefwaasxmucz` int DEFAULT NULL,\n `fomnbbwyogwzlchtfkmftlphtxemeimz` int DEFAULT NULL,\n `ekrxmhuzrorhvcvwttuyibaxfrkytrru` int DEFAULT NULL,\n `vybchklfmkdfwakbdeithwzcqnkxmnuj` int DEFAULT NULL,\n `pxseqvgkdiibifktmvaaivkeopsywaea` int DEFAULT NULL,\n `xipnwlpkrlpewsqzcxoqpibzaspxynwv` int DEFAULT NULL,\n `jvuiplvfuhtohvbzbldminyoikjkpyjs` int DEFAULT NULL,\n `ysbuxlrtechbpoxhifqqsgcvdfjiaeli` int DEFAULT NULL,\n `flotqetpaypwqnwyrjuuzbzlmahabyme` int DEFAULT NULL,\n `jbtbwfopcfqixzbkratfciqpgpbekbfm` int DEFAULT NULL,\n `zsppfnyejpowfwwqdvgdmrycrkpntqtt` int DEFAULT NULL,\n `zsrwtfrejqbbshahgazafnamtgrtkzvy` int DEFAULT NULL,\n `xheziouwqqityorgrvixrnevskbixvel` int DEFAULT NULL,\n `ocbyajvnumctlowkwbohhkeuqctvczjp` int DEFAULT NULL,\n `vwhijuixentcwwfvhfgwlskoeedzuedo` int DEFAULT NULL,\n `fxjabaevtzqlziliktitrdfcpxlbkfyi` int DEFAULT NULL,\n `nuqnjnbxfoiiaplxjvgbmqghbpurhjnl` int DEFAULT NULL,\n `irgvoweuhhfcsyyjexptqbvjzurdagpn` int DEFAULT NULL,\n `ddtpmrueedwhdhegxqzmmmzgdszoqqsw` int DEFAULT NULL,\n `dpcbqjevmhntoyllzfaxnkoogothymou` int DEFAULT NULL,\n `mudqlbtdknjezikkakbmnwixigcqzgyz` int DEFAULT NULL,\n `nixzcfvlrdnqdwkufepajteapyynumjh` int DEFAULT NULL,\n `htqyerjgvuivbbivimwxiqkujwqnarsi` int DEFAULT NULL,\n `burlvubgrnqihgbahsutjrkakgyzhnxm` int DEFAULT NULL,\n `uqinsoydyxefrcgzefdljidomrmjtauv` int DEFAULT NULL,\n `upexhyrcwraeklfoxmljuvbzvoilupph` int DEFAULT NULL,\n `dbqjennslawsddnghpicelfegpptvipj` int DEFAULT NULL,\n `artqavypcfwmhofthcwopprqtquueycs` int DEFAULT NULL,\n `ilfzobfqkzdpbpinzgwhltuikxveyyar` int DEFAULT NULL,\n `rpxhzinbckafxgwduyuxpclwcriyvggi` int DEFAULT NULL,\n `jhlsjajrxblbzzecechhgiexfcqjhuvf` int DEFAULT NULL,\n `vfvfokacnxmcgudrmnzaiokbtpskayef` int DEFAULT NULL,\n `honkqhrbogpxelmxrqckexmryiocwqrr` int DEFAULT NULL,\n `gsbbeibzghmkwaqrrdjanuhqtdgptfpf` int DEFAULT NULL,\n `kxkxfkafdmohybxqwzwowhnsthkuizac` int DEFAULT NULL,\n `ksvdwglkbdwcxrgqmesxydcyfiwnmuao` int DEFAULT NULL,\n `chuiuqtofpsbaqwebojvwhmtacvucnbx` int DEFAULT NULL,\n `wpumeoutowrujuhkvcsbjchulmvceekz` int DEFAULT NULL,\n `mjrsplxaucgrtqpoycvtriyiochvhbve` int DEFAULT NULL,\n `daivvmuvyztcyjdqtvvcwhzlqecoktmy` int DEFAULT NULL,\n `ppkevwhnluyrqmrwhmenoksgztnnzgnj` int DEFAULT NULL,\n PRIMARY KEY (`wlgfodccepybwnnlwczvcskqxhiicpng`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"uhncixqtyhjsicsgvnamwatzwbpgfqld\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["wlgfodccepybwnnlwczvcskqxhiicpng"],"columns":[{"name":"wlgfodccepybwnnlwczvcskqxhiicpng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"pforffmtqznulzclnijmukrriomndeek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqyllktweptbarhdymrwljihwoadfzhi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqtsodnfrndilwzrwvnthewfyziskocz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xzqdrkvysjnmcvxvvabqijmormnvbscb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpnfhsynuilccfgmdnggrlllipsqnndj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thopurypvkfihjuwlvwckflpljjhvkrz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytuixlhsfqwdxsjvjixhrlqqicbnduxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lcvaqxxoctlsxfjifpzqewofvnnupixo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixodniefjkdfhnihgbzgaejwmywgqrby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acnwtocdqmyqqpzdqenjacbtqasrczhs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxhtdzxxrivxfywqbbxphbcbhynsneor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwtftrzsfldalupgrfcnckpxiiyqedls","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unotrfiiqkrrrgifcoodupjkcovzukyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tekkkqhxczrsbdwdjtsmebuzeitizgjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oiqstkaelrmiaezpjzppixygygbzjsbh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"byvxvcusnhpvzpzeqltyggidgberedlt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmkdzzmwcfpjbbavqxogfvogbabamhoz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dfbwaodpateouuipnxumvtlcphmprwpa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pfjtoclobbqzljevezhgihbajpyanuiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjneljmbprbljfkxrkdukjbiibuiidus","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwhgibejgyyrkmmpsnvtumnjvihzbmwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fsbrdvxvmkvvtricmtvzffjcrqbhtxrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zltujcmkteitkaadhjvfsqlcwuecrwyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"esdkbwlpjwczgkyhidhilirqlhxwyqjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zijokmcqplmdubrojrwrwpqmjjfuwggy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdpxrzqeiyriqptkkzplfnguldtaxryb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtzgttkbdvtsvjpjituxhqmskdxnxpvz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzuebzlmsnccaxyqlxtlyxhhxbjrtbsa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdvptngptevljrnhdkprgeswodtmdrni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"teiitsbskilfsqyljmvjdryockaamcbz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hczjghykcqnlrbcmnhuhiedohokynwuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckhhbfwnywifswkqdylarejpcedsrlrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdgmwbyloxjdzrdbougkwryqprmabxdb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nnfwuqzqbqmvvwenscedfgmzgnyvxwut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aflqzygdfbtshgelbrredyudzkdjqwxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"trawykqqnucqinzttvhucbmxdjwkztds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"berbrgqdihwptyzqrvztvqhrfodettwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgndhfejhvlrpvibznwozatbzbvnqqjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxwohmoluzguhiqluotgboonvumibjsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtbxxdmboqxebmyhjdcfrmtlonzkehig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qslvjqbblivpbqmxbdrwwahtjloaijjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aclyiwxyvhqrmbxhuoyddxpbbkczayxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfccuqdxtnxautsssrhgtbyxakybipaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxmqzurthxzttzschfalsuqvjfvnudva","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iggdpzuqiewmbijvvseitosspflcdmjq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xiebdjdmplmzcvxiaqjycbdrahieraba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exnkvsyycvfvfxyprhqaojwtnowiumpe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqhbguolfnrhnmbkujohfhumubhiscjp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fswmnwnctkiooduefxnuredqelmlkdmt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsrkjyzkauuonyaharororgemnxdqlle","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ccpsfblquvolksgxlrhqmggkrssaatch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgcuioejmdptlemdxclkznbfiuktgfcz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vejimydriorhvmjuafzqetkprlcnladf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ulevxwsipymtechfitsassndfswemfec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"somgnisbzfszrryticbywouwatmmizvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"geqojnbmtprwujzmyaywdafxlwgjwpdz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tgsrngpcytsuresletaudbqumzxpzyuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"spcgnxwblijdvmgtwhgyvsgpqsqqjvyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eepxlwjtquylgdjconoboefwaasxmucz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fomnbbwyogwzlchtfkmftlphtxemeimz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekrxmhuzrorhvcvwttuyibaxfrkytrru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vybchklfmkdfwakbdeithwzcqnkxmnuj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxseqvgkdiibifktmvaaivkeopsywaea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xipnwlpkrlpewsqzcxoqpibzaspxynwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvuiplvfuhtohvbzbldminyoikjkpyjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ysbuxlrtechbpoxhifqqsgcvdfjiaeli","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flotqetpaypwqnwyrjuuzbzlmahabyme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbtbwfopcfqixzbkratfciqpgpbekbfm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zsppfnyejpowfwwqdvgdmrycrkpntqtt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zsrwtfrejqbbshahgazafnamtgrtkzvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xheziouwqqityorgrvixrnevskbixvel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocbyajvnumctlowkwbohhkeuqctvczjp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vwhijuixentcwwfvhfgwlskoeedzuedo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxjabaevtzqlziliktitrdfcpxlbkfyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nuqnjnbxfoiiaplxjvgbmqghbpurhjnl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"irgvoweuhhfcsyyjexptqbvjzurdagpn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddtpmrueedwhdhegxqzmmmzgdszoqqsw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpcbqjevmhntoyllzfaxnkoogothymou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mudqlbtdknjezikkakbmnwixigcqzgyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nixzcfvlrdnqdwkufepajteapyynumjh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htqyerjgvuivbbivimwxiqkujwqnarsi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"burlvubgrnqihgbahsutjrkakgyzhnxm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqinsoydyxefrcgzefdljidomrmjtauv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upexhyrcwraeklfoxmljuvbzvoilupph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbqjennslawsddnghpicelfegpptvipj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"artqavypcfwmhofthcwopprqtquueycs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ilfzobfqkzdpbpinzgwhltuikxveyyar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpxhzinbckafxgwduyuxpclwcriyvggi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jhlsjajrxblbzzecechhgiexfcqjhuvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vfvfokacnxmcgudrmnzaiokbtpskayef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"honkqhrbogpxelmxrqckexmryiocwqrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsbbeibzghmkwaqrrdjanuhqtdgptfpf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxkxfkafdmohybxqwzwowhnsthkuizac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ksvdwglkbdwcxrgqmesxydcyfiwnmuao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chuiuqtofpsbaqwebojvwhmtacvucnbx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wpumeoutowrujuhkvcsbjchulmvceekz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjrsplxaucgrtqpoycvtriyiochvhbve","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"daivvmuvyztcyjdqtvvcwhzlqecoktmy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ppkevwhnluyrqmrwhmenoksgztnnzgnj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671801,"databaseName":"models_schema","ddl":"CREATE TABLE `urtqvqjqqavkdscrqvcdnwnrtgnmfnbd` (\n `kwtokworxzrftmcovounpviwljoxbxwm` int NOT NULL,\n `ufzvtjgabtxgaxmqhbxpyevjoiazwson` int DEFAULT NULL,\n `dlsaijtoqgnjhmaelomckygdqlesfwwu` int DEFAULT NULL,\n `klqwhvvfntqzgrriykteruzxubniunee` int DEFAULT NULL,\n `vigawwawlejjczaqpzsoxhgaixlsfomt` int DEFAULT NULL,\n `agakbfdyuwghztxbkqsybaihaiqrtchf` int DEFAULT NULL,\n `iwijiykttrudsygsxulncrxybuntrfpd` int DEFAULT NULL,\n `yjdkubclnbykrjaundhhvffdlothekjl` int DEFAULT NULL,\n `ekttbkpbfgrdbatwnokgkjieikscmxha` int DEFAULT NULL,\n `omnjptywqkdiathgbuqqyaxvwathsiky` int DEFAULT NULL,\n `yhzgvxkbdurwmjgoyejffrafwygztfys` int DEFAULT NULL,\n `sttjcldvukxbprgqofrhrwyclfqmcqea` int DEFAULT NULL,\n `qfbeclittgnrxkphxqupktkdywlaxlky` int DEFAULT NULL,\n `trxyyurxnuwabuyckwhqtymgtnsclwzi` int DEFAULT NULL,\n `iqkuhguwhkbrewnxxdhtljymwqncxegs` int DEFAULT NULL,\n `vbkpjitgnntfhccgygkcxccnzxmjrtpi` int DEFAULT NULL,\n `ijoarqsknskbpjnfxnevicgdsxnsaktj` int DEFAULT NULL,\n `vuywdzjgqvrnhygfxodrkekxngdllfsx` int DEFAULT NULL,\n `pcwpasjpaxwzoamfxprzfhijcujfghpf` int DEFAULT NULL,\n `ooydfrwwpkbgeomprdpxzskoxpqbsqvs` int DEFAULT NULL,\n `qomtdovmencxigqosxkaelssmhzdcpps` int DEFAULT NULL,\n `tyrxbjtuppkyavymtposagemavoivggf` int DEFAULT NULL,\n `otwsvvfrarlfovqsozkhukyrryyldyzo` int DEFAULT NULL,\n `udldegipepjvsffsxhiiusskhhwaesnt` int DEFAULT NULL,\n `vjcwbgtppsjnladbrikmngdrzcrpijzs` int DEFAULT NULL,\n `jldpezithabgkspzepkudfzgcivgozvd` int DEFAULT NULL,\n `imofahztqrmxzgnkainjfqgspedmbqjt` int DEFAULT NULL,\n `rxntiqwtsxrsfpptrssofakpnfqqvfuo` int DEFAULT NULL,\n `aadpjnjbzhdrcnsxausbxuglxlfdlktz` int DEFAULT NULL,\n `iacadhwaodjbizktluswhwkznqlbhyxe` int DEFAULT NULL,\n `tlrdnqyptzgxjlokjidscxshuxvbcmsj` int DEFAULT NULL,\n `kctzkzepuxwzsmbncibcpwonrsysjygc` int DEFAULT NULL,\n `prerwdsnqfowubientkehmtajoahhxfp` int DEFAULT NULL,\n `kkcnocicefjjpoafiiffmufhonqelbxv` int DEFAULT NULL,\n `hobsqdfdodyabroforxnqglbzziqzngy` int DEFAULT NULL,\n `qwikfxcarjqcmgvuuagngmaoqtymsxwq` int DEFAULT NULL,\n `yzrcjuxchlkgetdkoqrznzcujusqyetc` int DEFAULT NULL,\n `hcugtykyvtltpqrawtxyglltcpcbschc` int DEFAULT NULL,\n `ooitbxdlqookvbcuxrjwhgtanzrzmuqh` int DEFAULT NULL,\n `tvqubmkfchexgmoiyvirnjyvpnwyueql` int DEFAULT NULL,\n `ybkbhofcrukcfvhpfvrwzdzeaffywltu` int DEFAULT NULL,\n `inehqahdkppowontwrbhitxdbslotmmw` int DEFAULT NULL,\n `yusszfkhlphrzrxtnjqadtkmdhatdwuh` int DEFAULT NULL,\n `skeqfdtglrvwcfsgcexlwoxycvurqqpj` int DEFAULT NULL,\n `unsasfvbivxgyiptsoxprfefxidjqvag` int DEFAULT NULL,\n `xqzyzodxeyqlvycckadfyjgkgpvzfomy` int DEFAULT NULL,\n `sbnmxtkpyapxjbyvokcwliytleytgihv` int DEFAULT NULL,\n `tjtqpwbybbzovhlfhzatkpeqhpwqbvcy` int DEFAULT NULL,\n `mmaffnttsynfofsqakrfcldvzjoavyes` int DEFAULT NULL,\n `ikzkhgkcpftoozdblgappumijejnrytr` int DEFAULT NULL,\n `yiisapiidiyhphofdqzjnhwvsvaxlypr` int DEFAULT NULL,\n `flwugaieuwsmhqrtwgvtduvpuljayvic` int DEFAULT NULL,\n `caigcvncgtkfgcpzoxxaiubhizenrivk` int DEFAULT NULL,\n `nxbmduxopkbjmojjoaqbgkuekxiwdqvy` int DEFAULT NULL,\n `rglhppvkuwldejwscbejcbahangdokfp` int DEFAULT NULL,\n `wpmwqxzbhhcwseydomwtvbwdecxuvnnh` int DEFAULT NULL,\n `blyxzxyunnqglardchhbynhzwfjtbqiz` int DEFAULT NULL,\n `kmpdjoiupffhbfculqvxysqejufhfplr` int DEFAULT NULL,\n `psmzmhvmqojrmmqqpbxakfmiojekrgdg` int DEFAULT NULL,\n `yjiylkxxfycgmkocxiwssaqiqzlzfdwe` int DEFAULT NULL,\n `ocmalcykmsffwjbdobvmhpfdhofyssmv` int DEFAULT NULL,\n `viqgeggifjadmewjibfmqfskeghvbhzf` int DEFAULT NULL,\n `jmpjgbznbgxuvtmkzlywgaleybrqshuc` int DEFAULT NULL,\n `jrggxpxgpyvzwbdduwonerqdxidnmsep` int DEFAULT NULL,\n `scoqlfaqrxjwvmyadqgbizlldkdjvxvp` int DEFAULT NULL,\n `tnbncmweevwyatmnwynlezkgmglvjpbc` int DEFAULT NULL,\n `gqelhtnptkasclqlxfyeypxgvujhnedf` int DEFAULT NULL,\n `vgilqoscuwxqkuylsqridtlvjjicluvp` int DEFAULT NULL,\n `fadrskjobfxjqqkymedujoehrxvaffcq` int DEFAULT NULL,\n `lydwlkohyruizwchnhmzasloocexirlj` int DEFAULT NULL,\n `qtjsvspsxdhiefmwnjkguxyiexsektir` int DEFAULT NULL,\n `wgnbjstrzakzsjgwzapvgpiauhqrawle` int DEFAULT NULL,\n `fbttlwpibugsvkjlmhuwwgwudtvybqvi` int DEFAULT NULL,\n `qmwveblgaavkhysreiqlsekhubsrerne` int DEFAULT NULL,\n `kfgbisclydnudgjmtovsdxauwnrukquw` int DEFAULT NULL,\n `rcduvxaeolgfcmogmvsuuvgotydoopgn` int DEFAULT NULL,\n `qzmyiijqqkppzfsxlsaghgertwarubpz` int DEFAULT NULL,\n `ssyuewbuntehlqbilhjujmmdfvpzvwye` int DEFAULT NULL,\n `dlztnhnxognlmetpcthimhnogstrieqa` int DEFAULT NULL,\n `cqfdrnsiilijxteadzslnmpdwjyngkqt` int DEFAULT NULL,\n `unuybuqokrpdoyslvfchkmugrcofrzjs` int DEFAULT NULL,\n `icgsicoavaygemsdtywcsjtlcstuymvd` int DEFAULT NULL,\n `xbbxkgdvsptewzvhwhlxgkfwyirydzmf` int DEFAULT NULL,\n `itgqnuuaneujidutvcyfqmyiuelrwunj` int DEFAULT NULL,\n `ztcwrtgmxefpyalzypgohxbedefwgjij` int DEFAULT NULL,\n `aiyhmxwfoaeplbqinavocfdlmizragvo` int DEFAULT NULL,\n `iwzeiyfqymfursyvgvfskxoxtkmuntzc` int DEFAULT NULL,\n `katkyjfuiyjpodisvyicnyunwxjoxbie` int DEFAULT NULL,\n `gipgqewqlrlzworigtikassigdkjqfei` int DEFAULT NULL,\n `rwiuodfjljgbzlqtyfmzeftzocivxhtl` int DEFAULT NULL,\n `kcfyaxdlrlvosnkbgivuulazihphrene` int DEFAULT NULL,\n `xvwomvpaqlfdjxenbrtwivltwqrvtywo` int DEFAULT NULL,\n `pfetwbswlvtaktdsgcqylucafilrytmd` int DEFAULT NULL,\n `ezizjxpkatfuqhyqorfryqnjlyttlgev` int DEFAULT NULL,\n `wdyonogcveylnjtbzzwtwuspwfvgaytn` int DEFAULT NULL,\n `ofwecluwzxxlmfruxqocpevlzzybyhoe` int DEFAULT NULL,\n `pimpqggmcwobgbgynzuxybksxopqxnob` int DEFAULT NULL,\n `uofvankranwufqcpzxpxsvehqrdskbfz` int DEFAULT NULL,\n `tbdedmbvtjycnptafvaywryxszxlyilg` int DEFAULT NULL,\n `uifdjfsxffasijcdoafxwbzwdbkknxiy` int DEFAULT NULL,\n PRIMARY KEY (`kwtokworxzrftmcovounpviwljoxbxwm`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"urtqvqjqqavkdscrqvcdnwnrtgnmfnbd\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["kwtokworxzrftmcovounpviwljoxbxwm"],"columns":[{"name":"kwtokworxzrftmcovounpviwljoxbxwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ufzvtjgabtxgaxmqhbxpyevjoiazwson","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlsaijtoqgnjhmaelomckygdqlesfwwu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"klqwhvvfntqzgrriykteruzxubniunee","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vigawwawlejjczaqpzsoxhgaixlsfomt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agakbfdyuwghztxbkqsybaihaiqrtchf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwijiykttrudsygsxulncrxybuntrfpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjdkubclnbykrjaundhhvffdlothekjl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekttbkpbfgrdbatwnokgkjieikscmxha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omnjptywqkdiathgbuqqyaxvwathsiky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhzgvxkbdurwmjgoyejffrafwygztfys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sttjcldvukxbprgqofrhrwyclfqmcqea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfbeclittgnrxkphxqupktkdywlaxlky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"trxyyurxnuwabuyckwhqtymgtnsclwzi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqkuhguwhkbrewnxxdhtljymwqncxegs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbkpjitgnntfhccgygkcxccnzxmjrtpi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijoarqsknskbpjnfxnevicgdsxnsaktj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vuywdzjgqvrnhygfxodrkekxngdllfsx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pcwpasjpaxwzoamfxprzfhijcujfghpf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ooydfrwwpkbgeomprdpxzskoxpqbsqvs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qomtdovmencxigqosxkaelssmhzdcpps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tyrxbjtuppkyavymtposagemavoivggf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otwsvvfrarlfovqsozkhukyrryyldyzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udldegipepjvsffsxhiiusskhhwaesnt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjcwbgtppsjnladbrikmngdrzcrpijzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jldpezithabgkspzepkudfzgcivgozvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"imofahztqrmxzgnkainjfqgspedmbqjt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxntiqwtsxrsfpptrssofakpnfqqvfuo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aadpjnjbzhdrcnsxausbxuglxlfdlktz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iacadhwaodjbizktluswhwkznqlbhyxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlrdnqyptzgxjlokjidscxshuxvbcmsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kctzkzepuxwzsmbncibcpwonrsysjygc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prerwdsnqfowubientkehmtajoahhxfp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kkcnocicefjjpoafiiffmufhonqelbxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hobsqdfdodyabroforxnqglbzziqzngy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwikfxcarjqcmgvuuagngmaoqtymsxwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzrcjuxchlkgetdkoqrznzcujusqyetc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hcugtykyvtltpqrawtxyglltcpcbschc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ooitbxdlqookvbcuxrjwhgtanzrzmuqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvqubmkfchexgmoiyvirnjyvpnwyueql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybkbhofcrukcfvhpfvrwzdzeaffywltu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"inehqahdkppowontwrbhitxdbslotmmw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yusszfkhlphrzrxtnjqadtkmdhatdwuh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skeqfdtglrvwcfsgcexlwoxycvurqqpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unsasfvbivxgyiptsoxprfefxidjqvag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqzyzodxeyqlvycckadfyjgkgpvzfomy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbnmxtkpyapxjbyvokcwliytleytgihv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjtqpwbybbzovhlfhzatkpeqhpwqbvcy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmaffnttsynfofsqakrfcldvzjoavyes","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikzkhgkcpftoozdblgappumijejnrytr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yiisapiidiyhphofdqzjnhwvsvaxlypr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flwugaieuwsmhqrtwgvtduvpuljayvic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"caigcvncgtkfgcpzoxxaiubhizenrivk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxbmduxopkbjmojjoaqbgkuekxiwdqvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rglhppvkuwldejwscbejcbahangdokfp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wpmwqxzbhhcwseydomwtvbwdecxuvnnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blyxzxyunnqglardchhbynhzwfjtbqiz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmpdjoiupffhbfculqvxysqejufhfplr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psmzmhvmqojrmmqqpbxakfmiojekrgdg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjiylkxxfycgmkocxiwssaqiqzlzfdwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocmalcykmsffwjbdobvmhpfdhofyssmv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"viqgeggifjadmewjibfmqfskeghvbhzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmpjgbznbgxuvtmkzlywgaleybrqshuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jrggxpxgpyvzwbdduwonerqdxidnmsep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scoqlfaqrxjwvmyadqgbizlldkdjvxvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnbncmweevwyatmnwynlezkgmglvjpbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqelhtnptkasclqlxfyeypxgvujhnedf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgilqoscuwxqkuylsqridtlvjjicluvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fadrskjobfxjqqkymedujoehrxvaffcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lydwlkohyruizwchnhmzasloocexirlj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtjsvspsxdhiefmwnjkguxyiexsektir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgnbjstrzakzsjgwzapvgpiauhqrawle","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbttlwpibugsvkjlmhuwwgwudtvybqvi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmwveblgaavkhysreiqlsekhubsrerne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfgbisclydnudgjmtovsdxauwnrukquw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcduvxaeolgfcmogmvsuuvgotydoopgn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzmyiijqqkppzfsxlsaghgertwarubpz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ssyuewbuntehlqbilhjujmmdfvpzvwye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlztnhnxognlmetpcthimhnogstrieqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqfdrnsiilijxteadzslnmpdwjyngkqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unuybuqokrpdoyslvfchkmugrcofrzjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icgsicoavaygemsdtywcsjtlcstuymvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbbxkgdvsptewzvhwhlxgkfwyirydzmf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itgqnuuaneujidutvcyfqmyiuelrwunj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ztcwrtgmxefpyalzypgohxbedefwgjij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aiyhmxwfoaeplbqinavocfdlmizragvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwzeiyfqymfursyvgvfskxoxtkmuntzc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"katkyjfuiyjpodisvyicnyunwxjoxbie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gipgqewqlrlzworigtikassigdkjqfei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rwiuodfjljgbzlqtyfmzeftzocivxhtl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcfyaxdlrlvosnkbgivuulazihphrene","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvwomvpaqlfdjxenbrtwivltwqrvtywo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pfetwbswlvtaktdsgcqylucafilrytmd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezizjxpkatfuqhyqorfryqnjlyttlgev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wdyonogcveylnjtbzzwtwuspwfvgaytn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofwecluwzxxlmfruxqocpevlzzybyhoe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pimpqggmcwobgbgynzuxybksxopqxnob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uofvankranwufqcpzxpxsvehqrdskbfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbdedmbvtjycnptafvaywryxszxlyilg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uifdjfsxffasijcdoafxwbzwdbkknxiy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671833,"databaseName":"models_schema","ddl":"CREATE TABLE `utrkrrrttkgoplueqarmsziysncokcgj` (\n `wcwyrvpbxspwqhpawxiqgsddkggfdgac` int NOT NULL,\n `mphgrlsqfxswfihoewvthpfdcluobkar` int DEFAULT NULL,\n `qyxuyyrnpvncsevddwhunqnapndgotpy` int DEFAULT NULL,\n `jxlkkusddjhqmmbyotjoaeaywpmmbjme` int DEFAULT NULL,\n `diqpsrceyahwbrhbaaqymcdoqzhzktbm` int DEFAULT NULL,\n `jfuemdkjnokdtbrfeewuzfqxgmmtfxif` int DEFAULT NULL,\n `pkqwistgmdkalbtynfrbokgwblznhmbm` int DEFAULT NULL,\n `vwclumjdzjkuamyemnupqjpksftginmb` int DEFAULT NULL,\n `ykqyjtawidmlwktcidnekgplyjeguufc` int DEFAULT NULL,\n `kqwfjwhbyvgtrfkjbaznvpzamnaicgir` int DEFAULT NULL,\n `boowtfapdjkrwuvlisbasriugarsbnhj` int DEFAULT NULL,\n `yhplvetjhrwnvixxhlwujootbknnyzax` int DEFAULT NULL,\n `woelhvgqerfjccrpjnmsgxieahsdkhsm` int DEFAULT NULL,\n `fxedfxlfkncphwyduowsfzqrnftvoplm` int DEFAULT NULL,\n `lgbgomvdaokvedsezjqkmksmnpmrpbgt` int DEFAULT NULL,\n `comxhxqhhfvukqpfjhkifcuwmvaehphd` int DEFAULT NULL,\n `xysbuqwwrymxofoutjvrvziglstpkcfz` int DEFAULT NULL,\n `gnaqndcueewojlzcndmfpjlhxsbwkrku` int DEFAULT NULL,\n `zptnesnebohqqqglztsbmnprfpkyvqze` int DEFAULT NULL,\n `hfdbbebnekdapqacsirrwvpzsyalwztr` int DEFAULT NULL,\n `umdfdsyfyjojiolfwcvksgghelzcramh` int DEFAULT NULL,\n `tznyjwxjhiylyqjydzgwmoertfckidfh` int DEFAULT NULL,\n `pgctksihouwcpteepultbwuxbdmhajza` int DEFAULT NULL,\n `jpqcohdeznpfcjrrnitqykrrqlsnlvpm` int DEFAULT NULL,\n `kwiuddzxpclbhzzywpsfdnyvzeecgiwb` int DEFAULT NULL,\n `owlwtatvetevzeffqkqxmnhcffzvujsi` int DEFAULT NULL,\n `wsxkhfyfotlfynjtkybgvfphngabedww` int DEFAULT NULL,\n `dvmgpvhodhqbvsobmfakcywenfqdovog` int DEFAULT NULL,\n `igeeuevqjfoqdpnezljtyvubulxmgivp` int DEFAULT NULL,\n `rkemumfapvytnwaskytgqmuqmmodgjbo` int DEFAULT NULL,\n `yksgcporyyhboozgsitviwgypizdbbag` int DEFAULT NULL,\n `lyqeynocsrrsbhyvbcmjnnsqcwxgkwky` int DEFAULT NULL,\n `crnptfoeatvyraxpvoocmuybpauoukvm` int DEFAULT NULL,\n `yrcrmkwagypuhsdkarojxcjbdchdpmty` int DEFAULT NULL,\n `upgxfnivicpxkccjgvxqdklxayblkfgc` int DEFAULT NULL,\n `cqdkwxhuuugyoiglbmtcoshyujdxrdnd` int DEFAULT NULL,\n `cgcrbxgfucktkjblkawbapvjxmskafii` int DEFAULT NULL,\n `ajgftxrqzgcftcffeqevflifnclcswno` int DEFAULT NULL,\n `slteramftqhqylohrnhikqzpvpzsqzev` int DEFAULT NULL,\n `fltrqamvzouddclpsbauhpkqrkguzogy` int DEFAULT NULL,\n `topprtixhddvevywqzsnnhboiekqfxth` int DEFAULT NULL,\n `xpsncvfumrrvtlqtphbjafzmquzamzbs` int DEFAULT NULL,\n `lukgvcspalkllhbyuhspipycdpktnhxa` int DEFAULT NULL,\n `dwvlaefvikkypdohgdyjvafcyhrttmer` int DEFAULT NULL,\n `akvjflyuokyxawewfvcasbugyvlwppeb` int DEFAULT NULL,\n `rkjqfmhcmugegsgdjgdvyescgznawtec` int DEFAULT NULL,\n `haujgvffzopsksuifpixrwzhmxsguobt` int DEFAULT NULL,\n `vwygsruhidpnxjpyzcegizkrkyeirxox` int DEFAULT NULL,\n `ioakarqmqkrqhosleafluevevvogered` int DEFAULT NULL,\n `jhesxetrisblpvffzfttlywzzjevxxqk` int DEFAULT NULL,\n `abtrblovsvlsqlpimuhzwolpgrerkunb` int DEFAULT NULL,\n `zlvqasgptioixrsseqalwvgbxukwhfiu` int DEFAULT NULL,\n `heminuueiyugahrpzqserusmrdaqhjph` int DEFAULT NULL,\n `wmhlndpxxrzpujvokwjgstwaatvwuiln` int DEFAULT NULL,\n `phfqwczsrqdrwrhisllrwoiwrmntwsap` int DEFAULT NULL,\n `wtczriehzhxbymiszsfvzgmjmwwtztyf` int DEFAULT NULL,\n `nmemyezmnzkaadrsznvvcldszlicaqlf` int DEFAULT NULL,\n `qhlpdrqteejkuxufwhzjdwpzkgctstqx` int DEFAULT NULL,\n `mxavaootkfxauwqgujcgjwneuwmthkor` int DEFAULT NULL,\n `girvocgwabgrdfjrpxytjoqgdokrdhpq` int DEFAULT NULL,\n `xpqlwktkotouyzlbaiqpcpcrynbbluir` int DEFAULT NULL,\n `mrjzukyhsccjjrigtgdawsjnbhgwuxlu` int DEFAULT NULL,\n `qznixbmtsgxqdomoadvwxayzcdbofjrq` int DEFAULT NULL,\n `tocutiwabtvfvtuoauuvyiqxpxkvesbq` int DEFAULT NULL,\n `vseowlybnjqjtjpqrxgkrrxxlvcxbopl` int DEFAULT NULL,\n `jayetwqwwarcfclvpoinlxtvrcholjvc` int DEFAULT NULL,\n `nilhiljzcogwfuhrvzvqxcrnfaqnblwo` int DEFAULT NULL,\n `gcgypetujzkfkxcigghprczkshwzqcor` int DEFAULT NULL,\n `uyhrfqfoszkffjdyzjxzctunmnupgbbc` int DEFAULT NULL,\n `udmjckeyziqeauaszfoyqzzocrmocbyy` int DEFAULT NULL,\n `uxwobeapcokqvidwiqslcculklvsdptt` int DEFAULT NULL,\n `ucumhvequtkusiukjlsrsuppbqqhosny` int DEFAULT NULL,\n `yprxvddnhtgzlmpjrufckyfhscilktam` int DEFAULT NULL,\n `fzfgrjaknayetlxwuniezlywzrhadefi` int DEFAULT NULL,\n `xrmaytpljlwznlimtsrpupiremmdbfxf` int DEFAULT NULL,\n `pqmhxutgnncpsoydfvspzgyaxvvqquqf` int DEFAULT NULL,\n `dpwpvqaplqghtygambdeappnwvwjkxfx` int DEFAULT NULL,\n `htrutuannknvgyoroshdnybmvrptevym` int DEFAULT NULL,\n `eagpunwirdojvqqxvacakkxadfmysztx` int DEFAULT NULL,\n `sdixfwbuepzcptzizttggchrnavsltxf` int DEFAULT NULL,\n `tvhhhazaytuhrpupcgatenwrsweoqgmx` int DEFAULT NULL,\n `upmvonljjpuogwqqpnrupbeqlqdmlqed` int DEFAULT NULL,\n `ppahkawdzizujxbmbrfbhaosftlmkjhm` int DEFAULT NULL,\n `mahfluzeadkiqvoyvncrbtjdvobabffb` int DEFAULT NULL,\n `ywzmrrriutafhhsyuagmgjosoyevsyoo` int DEFAULT NULL,\n `cgdueqjapwlpoqyuptuzrufqgafwqryw` int DEFAULT NULL,\n `amqossrxoqbjasyrzrfvbynyahulxzso` int DEFAULT NULL,\n `novhlxsoszdfgqkemqhcwsdyfcsjpqyw` int DEFAULT NULL,\n `fsegegrkkjyafbawcfzoipxlhhzeuvaf` int DEFAULT NULL,\n `rwhbrlfkdfxnxguyazpkvzmzkgrygkja` int DEFAULT NULL,\n `boqlozikyoqfcrcltedqsebpdruylfhu` int DEFAULT NULL,\n `ncqnayfucbnifgxonhehmztscsviqdpb` int DEFAULT NULL,\n `dinqcgebjogpoonmgbubpluvzsstypdw` int DEFAULT NULL,\n `sriefykctzcubdxdnfxtsloktyruiita` int DEFAULT NULL,\n `ajcrxnefvkekmoxsgbsumvujsxqhsxwj` int DEFAULT NULL,\n `ydyaczexfkkysofkuxsjcrrlkdvfiwjj` int DEFAULT NULL,\n `glpiduvzlngvnngbpnttluvfqeqscret` int DEFAULT NULL,\n `uimxrylshnhthhximqcxugvhneysimkf` int DEFAULT NULL,\n `ymccquzxmkphokjokbtutgkelioxsqqd` int DEFAULT NULL,\n `wnakvihoswpzxlpqcjvtvyyhikwqtrql` int DEFAULT NULL,\n PRIMARY KEY (`wcwyrvpbxspwqhpawxiqgsddkggfdgac`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"utrkrrrttkgoplueqarmsziysncokcgj\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["wcwyrvpbxspwqhpawxiqgsddkggfdgac"],"columns":[{"name":"wcwyrvpbxspwqhpawxiqgsddkggfdgac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"mphgrlsqfxswfihoewvthpfdcluobkar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qyxuyyrnpvncsevddwhunqnapndgotpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jxlkkusddjhqmmbyotjoaeaywpmmbjme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"diqpsrceyahwbrhbaaqymcdoqzhzktbm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jfuemdkjnokdtbrfeewuzfqxgmmtfxif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkqwistgmdkalbtynfrbokgwblznhmbm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vwclumjdzjkuamyemnupqjpksftginmb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykqyjtawidmlwktcidnekgplyjeguufc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqwfjwhbyvgtrfkjbaznvpzamnaicgir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"boowtfapdjkrwuvlisbasriugarsbnhj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhplvetjhrwnvixxhlwujootbknnyzax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"woelhvgqerfjccrpjnmsgxieahsdkhsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxedfxlfkncphwyduowsfzqrnftvoplm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgbgomvdaokvedsezjqkmksmnpmrpbgt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"comxhxqhhfvukqpfjhkifcuwmvaehphd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xysbuqwwrymxofoutjvrvziglstpkcfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnaqndcueewojlzcndmfpjlhxsbwkrku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zptnesnebohqqqglztsbmnprfpkyvqze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfdbbebnekdapqacsirrwvpzsyalwztr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umdfdsyfyjojiolfwcvksgghelzcramh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tznyjwxjhiylyqjydzgwmoertfckidfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgctksihouwcpteepultbwuxbdmhajza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpqcohdeznpfcjrrnitqykrrqlsnlvpm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kwiuddzxpclbhzzywpsfdnyvzeecgiwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owlwtatvetevzeffqkqxmnhcffzvujsi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wsxkhfyfotlfynjtkybgvfphngabedww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvmgpvhodhqbvsobmfakcywenfqdovog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igeeuevqjfoqdpnezljtyvubulxmgivp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkemumfapvytnwaskytgqmuqmmodgjbo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yksgcporyyhboozgsitviwgypizdbbag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lyqeynocsrrsbhyvbcmjnnsqcwxgkwky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crnptfoeatvyraxpvoocmuybpauoukvm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yrcrmkwagypuhsdkarojxcjbdchdpmty","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upgxfnivicpxkccjgvxqdklxayblkfgc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqdkwxhuuugyoiglbmtcoshyujdxrdnd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgcrbxgfucktkjblkawbapvjxmskafii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajgftxrqzgcftcffeqevflifnclcswno","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"slteramftqhqylohrnhikqzpvpzsqzev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fltrqamvzouddclpsbauhpkqrkguzogy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"topprtixhddvevywqzsnnhboiekqfxth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpsncvfumrrvtlqtphbjafzmquzamzbs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lukgvcspalkllhbyuhspipycdpktnhxa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwvlaefvikkypdohgdyjvafcyhrttmer","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akvjflyuokyxawewfvcasbugyvlwppeb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkjqfmhcmugegsgdjgdvyescgznawtec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"haujgvffzopsksuifpixrwzhmxsguobt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vwygsruhidpnxjpyzcegizkrkyeirxox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ioakarqmqkrqhosleafluevevvogered","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jhesxetrisblpvffzfttlywzzjevxxqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abtrblovsvlsqlpimuhzwolpgrerkunb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zlvqasgptioixrsseqalwvgbxukwhfiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"heminuueiyugahrpzqserusmrdaqhjph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmhlndpxxrzpujvokwjgstwaatvwuiln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phfqwczsrqdrwrhisllrwoiwrmntwsap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtczriehzhxbymiszsfvzgmjmwwtztyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmemyezmnzkaadrsznvvcldszlicaqlf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhlpdrqteejkuxufwhzjdwpzkgctstqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mxavaootkfxauwqgujcgjwneuwmthkor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"girvocgwabgrdfjrpxytjoqgdokrdhpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpqlwktkotouyzlbaiqpcpcrynbbluir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrjzukyhsccjjrigtgdawsjnbhgwuxlu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qznixbmtsgxqdomoadvwxayzcdbofjrq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tocutiwabtvfvtuoauuvyiqxpxkvesbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vseowlybnjqjtjpqrxgkrrxxlvcxbopl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jayetwqwwarcfclvpoinlxtvrcholjvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nilhiljzcogwfuhrvzvqxcrnfaqnblwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gcgypetujzkfkxcigghprczkshwzqcor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uyhrfqfoszkffjdyzjxzctunmnupgbbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udmjckeyziqeauaszfoyqzzocrmocbyy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxwobeapcokqvidwiqslcculklvsdptt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucumhvequtkusiukjlsrsuppbqqhosny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yprxvddnhtgzlmpjrufckyfhscilktam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzfgrjaknayetlxwuniezlywzrhadefi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrmaytpljlwznlimtsrpupiremmdbfxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqmhxutgnncpsoydfvspzgyaxvvqquqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpwpvqaplqghtygambdeappnwvwjkxfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htrutuannknvgyoroshdnybmvrptevym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eagpunwirdojvqqxvacakkxadfmysztx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdixfwbuepzcptzizttggchrnavsltxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvhhhazaytuhrpupcgatenwrsweoqgmx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upmvonljjpuogwqqpnrupbeqlqdmlqed","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ppahkawdzizujxbmbrfbhaosftlmkjhm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mahfluzeadkiqvoyvncrbtjdvobabffb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywzmrrriutafhhsyuagmgjosoyevsyoo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgdueqjapwlpoqyuptuzrufqgafwqryw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amqossrxoqbjasyrzrfvbynyahulxzso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"novhlxsoszdfgqkemqhcwsdyfcsjpqyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fsegegrkkjyafbawcfzoipxlhhzeuvaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rwhbrlfkdfxnxguyazpkvzmzkgrygkja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"boqlozikyoqfcrcltedqsebpdruylfhu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ncqnayfucbnifgxonhehmztscsviqdpb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dinqcgebjogpoonmgbubpluvzsstypdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sriefykctzcubdxdnfxtsloktyruiita","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajcrxnefvkekmoxsgbsumvujsxqhsxwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydyaczexfkkysofkuxsjcrrlkdvfiwjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"glpiduvzlngvnngbpnttluvfqeqscret","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uimxrylshnhthhximqcxugvhneysimkf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ymccquzxmkphokjokbtutgkelioxsqqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnakvihoswpzxlpqcjvtvyyhikwqtrql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671867,"databaseName":"models_schema","ddl":"CREATE TABLE `uwhvppgdytwyftjjcnihrmfdnhimzykv` (\n `obvuojvxffdxrampurhwocxvmxddsvwa` int NOT NULL,\n `nsrqhofkpjvynzkbjwwrhpoxfzkyudjo` int DEFAULT NULL,\n `yvxcmozelmobbzjezvelrloxzyubtjyn` int DEFAULT NULL,\n `vwoxjtbspxeasptcdibtfhuzbesjdxvq` int DEFAULT NULL,\n `izdtrabrhdphlwvjexczqtmghypbefxs` int DEFAULT NULL,\n `yajeafspzozuicybpaveazprefafkkxz` int DEFAULT NULL,\n `ogmgfujexggdcwsjwswbszeuzucfzeaj` int DEFAULT NULL,\n `ybpiuuujnuxojjbvhaxmyymuihrsxfba` int DEFAULT NULL,\n `tdofnepsqgrilcvwnnczreihokqfypgi` int DEFAULT NULL,\n `owymxxpogwynqzslwrkwhffwvdcvazjz` int DEFAULT NULL,\n `zosvfojllfzqopagvavemnqbmtyuhqbd` int DEFAULT NULL,\n `hbzykotfhfarvtrsvulawxoolrtsixgj` int DEFAULT NULL,\n `rajstuensreqxslbeyvalkurdwbvwtxk` int DEFAULT NULL,\n `rdsuxggbvvxizlsituttuqvpditheesi` int DEFAULT NULL,\n `qxjiwgbptkfwxkbdritdftzoqmgqstgi` int DEFAULT NULL,\n `eiurjmiamogihpfnapalofagiskjpswy` int DEFAULT NULL,\n `phxsjusowvemetmqflecdvrpewxdppqr` int DEFAULT NULL,\n `wohogndruuprhcuzflycvqxqomiprbqa` int DEFAULT NULL,\n `qvnlmoumkpcpikvscgjmsahgruverywx` int DEFAULT NULL,\n `rzufinvsmluxcjkultblrnubmlyvsxuh` int DEFAULT NULL,\n `mqbwnrwvcdchzavearpinbbsjzcgdzde` int DEFAULT NULL,\n `lkjjlxuqtcsadclamsbzcvdmdcypzttz` int DEFAULT NULL,\n `gsirdbcltetimowxzgjdgimxabvuybyn` int DEFAULT NULL,\n `hdlnqufiyhdmjxuttdsynjdmlitalpfz` int DEFAULT NULL,\n `rrizfnpkziphpgojhueowbvpgxdtbrqt` int DEFAULT NULL,\n `dunvpdgyyaqfpujuqmkvacmfyjzxzxxt` int DEFAULT NULL,\n `qzooseqegjtuhbmsxamtcnrpkzkbkcpv` int DEFAULT NULL,\n `bdjxnorxusgbnylimdmurpfzdcleccyy` int DEFAULT NULL,\n `tjgronkjqapfmndgfscznkfvqltkzeoa` int DEFAULT NULL,\n `nwwkdvfvtnhejpgjhqcebjepchfqxheh` int DEFAULT NULL,\n `qvfnmnwkksdpzqxtcirsxozecsbtmeiu` int DEFAULT NULL,\n `zjicyvbgbbhulcutfclnwnwndypweerc` int DEFAULT NULL,\n `yglriexdfafwxtvhkjhqkauvmlngtltf` int DEFAULT NULL,\n `xjytydeketwojzmztzkaiddmhmnpjjfl` int DEFAULT NULL,\n `jcmtakhveefktzsqctaxwvmmupiocwmu` int DEFAULT NULL,\n `mjgmptxignhtmqqwppptgjtoocpqmocn` int DEFAULT NULL,\n `ehqffemvoxicanriqkjlndcbpqmqzlrz` int DEFAULT NULL,\n `ngtmreeanqnoszyzthqygsuxumoqcqis` int DEFAULT NULL,\n `pycvhjqdskakaqvtpcrbknvpjwimbbjc` int DEFAULT NULL,\n `fumumkwgyvtjudcmtsgayfqzpzmjkluc` int DEFAULT NULL,\n `ejmzpjecfflbiwzejcspgkjvwviqbzyq` int DEFAULT NULL,\n `ltpnvtaspevtnwrnkmkiqgicewbrgqsu` int DEFAULT NULL,\n `hcvfjbsjuevvuiyajmcvhlyuadoknhjk` int DEFAULT NULL,\n `ldmfraipcfuquvdkewhyngmvtlqewsls` int DEFAULT NULL,\n `wfjnebmpfoegunowyfsaopgwmdfwbuml` int DEFAULT NULL,\n `ypvmwmqldebrqbmcyjbfalgzlazehwig` int DEFAULT NULL,\n `ugnpmfaxhfwntvvhoygdhhgkptuovblp` int DEFAULT NULL,\n `lkrpemchcmgvobclksbcbkzeadesdbgg` int DEFAULT NULL,\n `onrklgwvhnukehgdfxvsfskizeeosfox` int DEFAULT NULL,\n `ylycpbgltowenpevkqripvnurdhvbnlr` int DEFAULT NULL,\n `erlpitbcrlphuivqpspjgwphchhqeomj` int DEFAULT NULL,\n `dauygbizybzhzvysuxloudfmpqajheer` int DEFAULT NULL,\n `vcrtegedrcqltgcldlxemakhlzsvnwdo` int DEFAULT NULL,\n `giicdvgecadjkfbwbgrhworgtduolbzx` int DEFAULT NULL,\n `kihqhefqazvvcgkgkznphsfdddgsbvwi` int DEFAULT NULL,\n `akuawhpyccfgnaqaysthgylniwurklbr` int DEFAULT NULL,\n `oinrvgqidxrgmcsrkccgtdbjugkgtdjc` int DEFAULT NULL,\n `sgxtcxfinypyhmhxotndmqdanyxqqrlj` int DEFAULT NULL,\n `lmhnskdprrlhoahvglmeprnkvhicrxpc` int DEFAULT NULL,\n `leydxlisksotrwccvreseslzpidkbnxq` int DEFAULT NULL,\n `hhlotytjlkxbisajpunsazislnvrjncq` int DEFAULT NULL,\n `xfxapakbkebkwnqvpqqrjoxxtlpusbej` int DEFAULT NULL,\n `evkzslhqqdnpywsigciinmhzyjorvodx` int DEFAULT NULL,\n `vzgyzrmadumurirjllkblxlydkevyild` int DEFAULT NULL,\n `sekeylqvsffwoowzsdslunxpalzsyzte` int DEFAULT NULL,\n `rtoazzxgfrzsgjnxcfdgkfxmzhhvjwae` int DEFAULT NULL,\n `gsxobfgtcvaznixwagivykcjejrbjzmf` int DEFAULT NULL,\n `yauegrsdqjpskgemybgypctfffvpklvq` int DEFAULT NULL,\n `qyjfyqfrlzqgcgqxsolfxamqozngnuao` int DEFAULT NULL,\n `qrfuwgrbhotttxseqnbvezuwpfepurgz` int DEFAULT NULL,\n `bpcdwftkmmtigozjqwvnlmnjiauoxdph` int DEFAULT NULL,\n `utydrramoudnxvseaqoauyxinpuhnndd` int DEFAULT NULL,\n `chgynuxaosygcvzfzccgljhrwjvregdo` int DEFAULT NULL,\n `hwxmnbnzctntpzredmdeygvxlyetkclf` int DEFAULT NULL,\n `lhrnhvhyanbvirbbhrqsfaeujwkysruz` int DEFAULT NULL,\n `tokbgwztqsokuwvzwubqlvgqwvwgzhyq` int DEFAULT NULL,\n `isuygrjbeoxsnaxtdaydluccilttfvlv` int DEFAULT NULL,\n `wornriwwkiaeywagojsdqltjfgpfzopi` int DEFAULT NULL,\n `aaxviakvubidcoerbnaaiucohpffuetl` int DEFAULT NULL,\n `rfzfnfskeeglwgjduuqyakjrlyjnqxzd` int DEFAULT NULL,\n `nqldkrdjeaaorspnzayywxbocivyamfc` int DEFAULT NULL,\n `elarfmgvymlpibbsukvtdaobzehakfyn` int DEFAULT NULL,\n `hshmpodmhutpacjlvppikoybarjdqafg` int DEFAULT NULL,\n `umrgbhahdliqwfgeoaemmcyxagadjodw` int DEFAULT NULL,\n `agkiawnhqqifunaiwvisohuuwlsphqwl` int DEFAULT NULL,\n `ijjfqkwfotkfmrbffovrihuvzkqitnde` int DEFAULT NULL,\n `flpxhzlzbrlmraxbzqhothirqyotqcrt` int DEFAULT NULL,\n `smggtoutabqlfamcxvyicfbjtgdngewl` int DEFAULT NULL,\n `mrsphjcbqgroczyduzpcaaqvptjkeslp` int DEFAULT NULL,\n `efyywmmjkflogjbftxigrkrwkfhraofd` int DEFAULT NULL,\n `hasarturjuubdtonhvhuhaifndcbyqip` int DEFAULT NULL,\n `lapjssfbhesyvkxtprhaxixbfiwershz` int DEFAULT NULL,\n `aaankgfujmcjejwvkvfuyaygrbeffkbf` int DEFAULT NULL,\n `dgfzpjyxioxvgcuglwsztvousfeojxdk` int DEFAULT NULL,\n `zzwxpcxpnduofwukivqqkcmkwovjbgen` int DEFAULT NULL,\n `qpuuyslxyaojwmejhugrnbebrialgdrc` int DEFAULT NULL,\n `uridanjmwkkhcoitgruswyydxfzxwvdq` int DEFAULT NULL,\n `fitvdtrvrkmfrridgywunlnxrwfrjopc` int DEFAULT NULL,\n `xovzuivtkvbvohfryfabqwhwfgovlqqx` int DEFAULT NULL,\n `ofqvkbqihctbhnqljmgzyeahvrhufekk` int DEFAULT NULL,\n PRIMARY KEY (`obvuojvxffdxrampurhwocxvmxddsvwa`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"uwhvppgdytwyftjjcnihrmfdnhimzykv\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["obvuojvxffdxrampurhwocxvmxddsvwa"],"columns":[{"name":"obvuojvxffdxrampurhwocxvmxddsvwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"nsrqhofkpjvynzkbjwwrhpoxfzkyudjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yvxcmozelmobbzjezvelrloxzyubtjyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vwoxjtbspxeasptcdibtfhuzbesjdxvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"izdtrabrhdphlwvjexczqtmghypbefxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yajeafspzozuicybpaveazprefafkkxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ogmgfujexggdcwsjwswbszeuzucfzeaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybpiuuujnuxojjbvhaxmyymuihrsxfba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdofnepsqgrilcvwnnczreihokqfypgi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owymxxpogwynqzslwrkwhffwvdcvazjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zosvfojllfzqopagvavemnqbmtyuhqbd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbzykotfhfarvtrsvulawxoolrtsixgj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rajstuensreqxslbeyvalkurdwbvwtxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdsuxggbvvxizlsituttuqvpditheesi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxjiwgbptkfwxkbdritdftzoqmgqstgi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eiurjmiamogihpfnapalofagiskjpswy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phxsjusowvemetmqflecdvrpewxdppqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wohogndruuprhcuzflycvqxqomiprbqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvnlmoumkpcpikvscgjmsahgruverywx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzufinvsmluxcjkultblrnubmlyvsxuh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqbwnrwvcdchzavearpinbbsjzcgdzde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lkjjlxuqtcsadclamsbzcvdmdcypzttz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsirdbcltetimowxzgjdgimxabvuybyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdlnqufiyhdmjxuttdsynjdmlitalpfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrizfnpkziphpgojhueowbvpgxdtbrqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dunvpdgyyaqfpujuqmkvacmfyjzxzxxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzooseqegjtuhbmsxamtcnrpkzkbkcpv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdjxnorxusgbnylimdmurpfzdcleccyy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjgronkjqapfmndgfscznkfvqltkzeoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nwwkdvfvtnhejpgjhqcebjepchfqxheh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvfnmnwkksdpzqxtcirsxozecsbtmeiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjicyvbgbbhulcutfclnwnwndypweerc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yglriexdfafwxtvhkjhqkauvmlngtltf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjytydeketwojzmztzkaiddmhmnpjjfl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jcmtakhveefktzsqctaxwvmmupiocwmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjgmptxignhtmqqwppptgjtoocpqmocn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehqffemvoxicanriqkjlndcbpqmqzlrz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ngtmreeanqnoszyzthqygsuxumoqcqis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pycvhjqdskakaqvtpcrbknvpjwimbbjc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fumumkwgyvtjudcmtsgayfqzpzmjkluc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejmzpjecfflbiwzejcspgkjvwviqbzyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ltpnvtaspevtnwrnkmkiqgicewbrgqsu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hcvfjbsjuevvuiyajmcvhlyuadoknhjk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ldmfraipcfuquvdkewhyngmvtlqewsls","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfjnebmpfoegunowyfsaopgwmdfwbuml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypvmwmqldebrqbmcyjbfalgzlazehwig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugnpmfaxhfwntvvhoygdhhgkptuovblp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lkrpemchcmgvobclksbcbkzeadesdbgg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"onrklgwvhnukehgdfxvsfskizeeosfox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ylycpbgltowenpevkqripvnurdhvbnlr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"erlpitbcrlphuivqpspjgwphchhqeomj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dauygbizybzhzvysuxloudfmpqajheer","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vcrtegedrcqltgcldlxemakhlzsvnwdo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"giicdvgecadjkfbwbgrhworgtduolbzx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kihqhefqazvvcgkgkznphsfdddgsbvwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akuawhpyccfgnaqaysthgylniwurklbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oinrvgqidxrgmcsrkccgtdbjugkgtdjc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sgxtcxfinypyhmhxotndmqdanyxqqrlj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmhnskdprrlhoahvglmeprnkvhicrxpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"leydxlisksotrwccvreseslzpidkbnxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhlotytjlkxbisajpunsazislnvrjncq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xfxapakbkebkwnqvpqqrjoxxtlpusbej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evkzslhqqdnpywsigciinmhzyjorvodx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzgyzrmadumurirjllkblxlydkevyild","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sekeylqvsffwoowzsdslunxpalzsyzte","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtoazzxgfrzsgjnxcfdgkfxmzhhvjwae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsxobfgtcvaznixwagivykcjejrbjzmf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yauegrsdqjpskgemybgypctfffvpklvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qyjfyqfrlzqgcgqxsolfxamqozngnuao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrfuwgrbhotttxseqnbvezuwpfepurgz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bpcdwftkmmtigozjqwvnlmnjiauoxdph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utydrramoudnxvseaqoauyxinpuhnndd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chgynuxaosygcvzfzccgljhrwjvregdo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwxmnbnzctntpzredmdeygvxlyetkclf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhrnhvhyanbvirbbhrqsfaeujwkysruz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tokbgwztqsokuwvzwubqlvgqwvwgzhyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"isuygrjbeoxsnaxtdaydluccilttfvlv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wornriwwkiaeywagojsdqltjfgpfzopi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aaxviakvubidcoerbnaaiucohpffuetl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfzfnfskeeglwgjduuqyakjrlyjnqxzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqldkrdjeaaorspnzayywxbocivyamfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elarfmgvymlpibbsukvtdaobzehakfyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hshmpodmhutpacjlvppikoybarjdqafg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umrgbhahdliqwfgeoaemmcyxagadjodw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agkiawnhqqifunaiwvisohuuwlsphqwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijjfqkwfotkfmrbffovrihuvzkqitnde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flpxhzlzbrlmraxbzqhothirqyotqcrt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smggtoutabqlfamcxvyicfbjtgdngewl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrsphjcbqgroczyduzpcaaqvptjkeslp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"efyywmmjkflogjbftxigrkrwkfhraofd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hasarturjuubdtonhvhuhaifndcbyqip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lapjssfbhesyvkxtprhaxixbfiwershz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aaankgfujmcjejwvkvfuyaygrbeffkbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgfzpjyxioxvgcuglwsztvousfeojxdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzwxpcxpnduofwukivqqkcmkwovjbgen","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qpuuyslxyaojwmejhugrnbebrialgdrc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uridanjmwkkhcoitgruswyydxfzxwvdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fitvdtrvrkmfrridgywunlnxrwfrjopc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xovzuivtkvbvohfryfabqwhwfgovlqqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofqvkbqihctbhnqljmgzyeahvrhufekk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671896,"databaseName":"models_schema","ddl":"CREATE TABLE `uyjsubdlcesowgvppiweyyynybzozzcb` (\n `hgsixrilgixxhuckbpoeywokzccbmhlv` int NOT NULL,\n `wrzyfhbnnalozkfzeliezifftvoypmfa` int DEFAULT NULL,\n `vnuzhtgskuzubgnxkjvassmkpwrxqxdw` int DEFAULT NULL,\n `juqzfbvcymmrdxweeihgbpxtyogvkxcd` int DEFAULT NULL,\n `jbqytvvaotnpocmkweduawrehrifutfc` int DEFAULT NULL,\n `vxvjeoohctpcleyfpbaiuzdearnxlyjy` int DEFAULT NULL,\n `iezvsfnlhhpmflszlbkqfuleiyajpyyh` int DEFAULT NULL,\n `rqwfclqfnkvpvhjkuoemjuoctjjvchcx` int DEFAULT NULL,\n `pvkllasgnbkgipgqxdiymxlqgszthtmi` int DEFAULT NULL,\n `rldodfnrzdoxhebvzomfhsnbwjyqnudq` int DEFAULT NULL,\n `ithzaxyixmeifgaatqojepqxoojmqmwd` int DEFAULT NULL,\n `cyuazbmztbwaelfjceekiuacagaxylxs` int DEFAULT NULL,\n `oldcufitgudhkvamwtcdgfwesiqxqrtd` int DEFAULT NULL,\n `lcnuzntjhxgolszlffawnqhurayzdgph` int DEFAULT NULL,\n `ecqurhtovbzpvoosdbwonocokkefhwwz` int DEFAULT NULL,\n `fawmnvqviicxkozayiaabxnqgjixcacv` int DEFAULT NULL,\n `vwqdbcomracibotjxkbijcsnbfjyarxo` int DEFAULT NULL,\n `aajgtgqunncvbzexhcrutkvadhptbgkx` int DEFAULT NULL,\n `pkwzdsvuraofdkrcxljkdsiqzcinrznl` int DEFAULT NULL,\n `ixqwemvfzrfnfexyoneodularxczvxro` int DEFAULT NULL,\n `xuvgucilymyarownqswebhwhbconwuza` int DEFAULT NULL,\n `dqoqnlyypaxwgpkvwwsnwddkbmiruvjz` int DEFAULT NULL,\n `hibctiwkfwihoceyqmcnyvfxuvwvsahw` int DEFAULT NULL,\n `woxvunzuxtrygtptfhvqtajfhesqqrbq` int DEFAULT NULL,\n `nqyodnvtrzfxdiladrvdtlhmriljsrbb` int DEFAULT NULL,\n `owzxvrfekmndjuzpmhlbwchpwlfsjqbg` int DEFAULT NULL,\n `dxvagghtzlipkmoauckfcleculqafjis` int DEFAULT NULL,\n `xxkgitlfqpuuuckqaijqmsvnsrbjnhdf` int DEFAULT NULL,\n `tkhucvrqepimqxhmsfazghbeaxuxfymh` int DEFAULT NULL,\n `zxvvilkjlscbepsomzswioozpsykudpt` int DEFAULT NULL,\n `upbitvphzspyivglmreufwaevrqgeemk` int DEFAULT NULL,\n `ekdhajehpdemcjctrwwtyhrrvagjhtoc` int DEFAULT NULL,\n `groxbfbjdvrfdgfjvpeinpbgdbarevei` int DEFAULT NULL,\n `xcbftjvervgeveihguuendxcalycvsok` int DEFAULT NULL,\n `mjtbbnbqgeaiszerpsxhwwozmbkoebmf` int DEFAULT NULL,\n `xswlgizxpuwkxblkdyglzazpaantdcnu` int DEFAULT NULL,\n `tnxdntmtjsfzurylkukjbksjcnuunljr` int DEFAULT NULL,\n `bcqvzfatnwmxxpfpitlydurvzflqqxef` int DEFAULT NULL,\n `raspmfibkycmzbqgdaiqtqyzwsqvyssw` int DEFAULT NULL,\n `vyfqxevrzbghvwhkagnohtmctvhguowr` int DEFAULT NULL,\n `dogvocwiqqivdziyhdetpxjgkzqftdrg` int DEFAULT NULL,\n `jwfmzstzvfqrkuraxbxdcfqgbuvchzwt` int DEFAULT NULL,\n `iyqatxpfwtmujlvnbfamsxwykjuxytiv` int DEFAULT NULL,\n `kufnuappeccsjijzcgcxcoyeaakfspxf` int DEFAULT NULL,\n `hehknlpmppaglihiqgaieuphtlyfkgxd` int DEFAULT NULL,\n `kuyjnrbrxwfikqpcruziifektmqhcvzo` int DEFAULT NULL,\n `yhaltbtromhwzeunmtqrlkumdqwnwzcv` int DEFAULT NULL,\n `xcioznmqcuwrlqqygvpolinasshsjvtk` int DEFAULT NULL,\n `itojskjibfhqynebnciozbxagmverhih` int DEFAULT NULL,\n `mqrjqxpgrkvhghtaooeyssdjinloxpbt` int DEFAULT NULL,\n `ipknwokjwdgxeacvvxmwicxfyckiaoeq` int DEFAULT NULL,\n `nbvnxpiumodlqycamoqkeqlzogjhypxf` int DEFAULT NULL,\n `axdwhkghwjocofugahvcssfxnypxcbgk` int DEFAULT NULL,\n `yfhpgsajtrpfyahllrcqpwqzpjvaoqen` int DEFAULT NULL,\n `dmszncsulxuxmwfsvvenkcmgkozkiymu` int DEFAULT NULL,\n `alnekqalzfkfhgyhynvirvbzeheikxvx` int DEFAULT NULL,\n `yogxsznocevbtyxjqqlgiiiwywdokynm` int DEFAULT NULL,\n `ctqbmcnsuwgbveipidxrwuagdgbkxaoh` int DEFAULT NULL,\n `xvqemipvrouzikzqpecabrxviwqyajbw` int DEFAULT NULL,\n `gvxmkflwcdxqtlqaxpyqatsvmovxdpbi` int DEFAULT NULL,\n `skoxpphhirdrfgmjpslywmiauucvsfxx` int DEFAULT NULL,\n `ipirkzjlzsspcgkqluovxwicgpzajhgh` int DEFAULT NULL,\n `krjuguviilhvcgfirpyunzabtvmkjkyx` int DEFAULT NULL,\n `oyyazcepubtbkuzgutqrpbghbausrple` int DEFAULT NULL,\n `rhfxzyritvuubisvowreuwoujoadaecw` int DEFAULT NULL,\n `uxqsuvtuvnpfejqjkdnqiydwjldsooke` int DEFAULT NULL,\n `gjsdhuleufrnkdowuimpecshpwuqzadg` int DEFAULT NULL,\n `bpkyvmtovarrmgwpbkqbjbfpwbkwzsrm` int DEFAULT NULL,\n `tmzujllptyubhmmdsuhvsgcwtsiuqmwt` int DEFAULT NULL,\n `brtoguyjejewjtxohqdojprkphggonep` int DEFAULT NULL,\n `cyecnhmjvinkihfluyhbznycyrpiblzm` int DEFAULT NULL,\n `vqcwezhgrmlkwklhwmhyryxvwtdvztdx` int DEFAULT NULL,\n `ebrfwzclqanmtwieawwjsxsamsvqjeid` int DEFAULT NULL,\n `xqgvhbyhiloozcuearxwdnktrfhwfuun` int DEFAULT NULL,\n `gdmukkkkpmiydcshcfdooqpxhiaqlofz` int DEFAULT NULL,\n `oisamjoaiijfqfobnohsuhmwvwdtuxyj` int DEFAULT NULL,\n `ckuuyrnplpmwnlpgzyyfftgpyteulell` int DEFAULT NULL,\n `mhdgboaepwdgdlbosfvklimbqxsqktme` int DEFAULT NULL,\n `tawrdkhtvdxtuhrkgtjyxjcarxsmgaxw` int DEFAULT NULL,\n `qljabvgqxhtvxvoagveghskmgcufmdzu` int DEFAULT NULL,\n `hjilijppjosisnpnwpdlwglnwqkztalb` int DEFAULT NULL,\n `nmoyjouvrcklovgavvzspgeadhdagnxe` int DEFAULT NULL,\n `rxccwmrehzvcahrhqsbfdqcdlsmfkpbu` int DEFAULT NULL,\n `qfjsrmotxtblvtgoeohxsaxiffjnfrjs` int DEFAULT NULL,\n `vsjstutfzhgeiiqdqokqsljxcphwirrv` int DEFAULT NULL,\n `ktqejbqphcucuudwguivlqtwfdkksein` int DEFAULT NULL,\n `ehdtqeoiyudgiwdxqdcdypfagysblqed` int DEFAULT NULL,\n `doxqsdycvnvttveqaersltdimurtgjdk` int DEFAULT NULL,\n `bnicxtwvrqoxtdnzzupmuyaideeprdqs` int DEFAULT NULL,\n `zabxxngzkzepcksqdctziretwtjoalek` int DEFAULT NULL,\n `sheawvsbdbqqikqkpfdpfwajjukftnue` int DEFAULT NULL,\n `arnbtuxehsdyqdpkmzzrkyfbeizcjdqn` int DEFAULT NULL,\n `iiyijrixhsrgtpknhsbbhxtjzwpkzyzt` int DEFAULT NULL,\n `xaezeeqcfscowdbngzndplesubzukngg` int DEFAULT NULL,\n `hsbwmfcrawztxvbqjtxynylzrndxxasy` int DEFAULT NULL,\n `vjozdbmdtavffumdzdqbkkxypwpclmeg` int DEFAULT NULL,\n `zuprfmbfdoretufyvsprtulkfqdikvaf` int DEFAULT NULL,\n `qdmcdclzijwirdariluflnbydvcbrvlg` int DEFAULT NULL,\n `vpnuhvmrjozwhtbgqhsatmzcajhasrcd` int DEFAULT NULL,\n `jrzkhpgwscgvoxpevbkgbntpurigraso` int DEFAULT NULL,\n PRIMARY KEY (`hgsixrilgixxhuckbpoeywokzccbmhlv`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"uyjsubdlcesowgvppiweyyynybzozzcb\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["hgsixrilgixxhuckbpoeywokzccbmhlv"],"columns":[{"name":"hgsixrilgixxhuckbpoeywokzccbmhlv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"wrzyfhbnnalozkfzeliezifftvoypmfa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnuzhtgskuzubgnxkjvassmkpwrxqxdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"juqzfbvcymmrdxweeihgbpxtyogvkxcd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbqytvvaotnpocmkweduawrehrifutfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxvjeoohctpcleyfpbaiuzdearnxlyjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iezvsfnlhhpmflszlbkqfuleiyajpyyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqwfclqfnkvpvhjkuoemjuoctjjvchcx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvkllasgnbkgipgqxdiymxlqgszthtmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rldodfnrzdoxhebvzomfhsnbwjyqnudq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ithzaxyixmeifgaatqojepqxoojmqmwd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cyuazbmztbwaelfjceekiuacagaxylxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oldcufitgudhkvamwtcdgfwesiqxqrtd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lcnuzntjhxgolszlffawnqhurayzdgph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecqurhtovbzpvoosdbwonocokkefhwwz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fawmnvqviicxkozayiaabxnqgjixcacv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vwqdbcomracibotjxkbijcsnbfjyarxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aajgtgqunncvbzexhcrutkvadhptbgkx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkwzdsvuraofdkrcxljkdsiqzcinrznl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixqwemvfzrfnfexyoneodularxczvxro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xuvgucilymyarownqswebhwhbconwuza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqoqnlyypaxwgpkvwwsnwddkbmiruvjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hibctiwkfwihoceyqmcnyvfxuvwvsahw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"woxvunzuxtrygtptfhvqtajfhesqqrbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqyodnvtrzfxdiladrvdtlhmriljsrbb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owzxvrfekmndjuzpmhlbwchpwlfsjqbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxvagghtzlipkmoauckfcleculqafjis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxkgitlfqpuuuckqaijqmsvnsrbjnhdf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkhucvrqepimqxhmsfazghbeaxuxfymh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxvvilkjlscbepsomzswioozpsykudpt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upbitvphzspyivglmreufwaevrqgeemk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekdhajehpdemcjctrwwtyhrrvagjhtoc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"groxbfbjdvrfdgfjvpeinpbgdbarevei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xcbftjvervgeveihguuendxcalycvsok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjtbbnbqgeaiszerpsxhwwozmbkoebmf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xswlgizxpuwkxblkdyglzazpaantdcnu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnxdntmtjsfzurylkukjbksjcnuunljr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bcqvzfatnwmxxpfpitlydurvzflqqxef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"raspmfibkycmzbqgdaiqtqyzwsqvyssw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vyfqxevrzbghvwhkagnohtmctvhguowr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dogvocwiqqivdziyhdetpxjgkzqftdrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwfmzstzvfqrkuraxbxdcfqgbuvchzwt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iyqatxpfwtmujlvnbfamsxwykjuxytiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kufnuappeccsjijzcgcxcoyeaakfspxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hehknlpmppaglihiqgaieuphtlyfkgxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kuyjnrbrxwfikqpcruziifektmqhcvzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhaltbtromhwzeunmtqrlkumdqwnwzcv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xcioznmqcuwrlqqygvpolinasshsjvtk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itojskjibfhqynebnciozbxagmverhih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqrjqxpgrkvhghtaooeyssdjinloxpbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ipknwokjwdgxeacvvxmwicxfyckiaoeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbvnxpiumodlqycamoqkeqlzogjhypxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axdwhkghwjocofugahvcssfxnypxcbgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfhpgsajtrpfyahllrcqpwqzpjvaoqen","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dmszncsulxuxmwfsvvenkcmgkozkiymu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"alnekqalzfkfhgyhynvirvbzeheikxvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yogxsznocevbtyxjqqlgiiiwywdokynm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctqbmcnsuwgbveipidxrwuagdgbkxaoh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvqemipvrouzikzqpecabrxviwqyajbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gvxmkflwcdxqtlqaxpyqatsvmovxdpbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skoxpphhirdrfgmjpslywmiauucvsfxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ipirkzjlzsspcgkqluovxwicgpzajhgh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krjuguviilhvcgfirpyunzabtvmkjkyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oyyazcepubtbkuzgutqrpbghbausrple","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhfxzyritvuubisvowreuwoujoadaecw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxqsuvtuvnpfejqjkdnqiydwjldsooke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjsdhuleufrnkdowuimpecshpwuqzadg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bpkyvmtovarrmgwpbkqbjbfpwbkwzsrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmzujllptyubhmmdsuhvsgcwtsiuqmwt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"brtoguyjejewjtxohqdojprkphggonep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cyecnhmjvinkihfluyhbznycyrpiblzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqcwezhgrmlkwklhwmhyryxvwtdvztdx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebrfwzclqanmtwieawwjsxsamsvqjeid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqgvhbyhiloozcuearxwdnktrfhwfuun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdmukkkkpmiydcshcfdooqpxhiaqlofz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oisamjoaiijfqfobnohsuhmwvwdtuxyj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckuuyrnplpmwnlpgzyyfftgpyteulell","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhdgboaepwdgdlbosfvklimbqxsqktme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tawrdkhtvdxtuhrkgtjyxjcarxsmgaxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qljabvgqxhtvxvoagveghskmgcufmdzu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjilijppjosisnpnwpdlwglnwqkztalb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmoyjouvrcklovgavvzspgeadhdagnxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxccwmrehzvcahrhqsbfdqcdlsmfkpbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfjsrmotxtblvtgoeohxsaxiffjnfrjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsjstutfzhgeiiqdqokqsljxcphwirrv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ktqejbqphcucuudwguivlqtwfdkksein","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehdtqeoiyudgiwdxqdcdypfagysblqed","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"doxqsdycvnvttveqaersltdimurtgjdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnicxtwvrqoxtdnzzupmuyaideeprdqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zabxxngzkzepcksqdctziretwtjoalek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sheawvsbdbqqikqkpfdpfwajjukftnue","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"arnbtuxehsdyqdpkmzzrkyfbeizcjdqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iiyijrixhsrgtpknhsbbhxtjzwpkzyzt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xaezeeqcfscowdbngzndplesubzukngg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hsbwmfcrawztxvbqjtxynylzrndxxasy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjozdbmdtavffumdzdqbkkxypwpclmeg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zuprfmbfdoretufyvsprtulkfqdikvaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdmcdclzijwirdariluflnbydvcbrvlg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpnuhvmrjozwhtbgqhsatmzcajhasrcd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jrzkhpgwscgvoxpevbkgbntpurigraso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671930,"databaseName":"models_schema","ddl":"CREATE TABLE `uyllrczkxxxqdijmtoeuaxgipxtztugl` (\n `vruopcjgamceaidwmrlrkdyaqwenwkzo` int NOT NULL,\n `nnsrrthzzyqlxvohoyrwkxbfibuplhvt` int DEFAULT NULL,\n `hmdrouurqfjxzumclwwlfolijyrfzwur` int DEFAULT NULL,\n `ebxaexgmuiodwlodkfmjlaqbvhtsriph` int DEFAULT NULL,\n `jkwctlvmdikacgekkdgegzdepblvnjhf` int DEFAULT NULL,\n `mebaqapqnmysewazkmljccaaskcvpqot` int DEFAULT NULL,\n `xktmcuwszbtutaddyhaegojmhcvezfdw` int DEFAULT NULL,\n `frmqhdmxchleexhqdzpsfkcdututueiu` int DEFAULT NULL,\n `hagiscraubxmterzpitodjajmbqmauly` int DEFAULT NULL,\n `lcjijfdkcslkkcecdeukgscgsvmrwcbp` int DEFAULT NULL,\n `xfvlrqejvawjybzfntdcpocybggvijab` int DEFAULT NULL,\n `xesowoosexseabfaaqcjrbvpofryjwuj` int DEFAULT NULL,\n `yoajwdpahiefdnjwmaxpuihzsdjcydpr` int DEFAULT NULL,\n `wqeahqdcrjdyluscyoqqrkxgbggyfkyz` int DEFAULT NULL,\n `dqiyelgxfoaifsmkdonvrvvfdvqsuzgk` int DEFAULT NULL,\n `svdvnoughgyjycnqdaesfjeepmjkzqnq` int DEFAULT NULL,\n `tqwxvlfuokcutybtwmzpqbpbnmsdtfcg` int DEFAULT NULL,\n `mowqoksvynmkqyzkznabwegrydloboax` int DEFAULT NULL,\n `bqbcgpxgrnympejujagpnfkwwkqyykkt` int DEFAULT NULL,\n `twfbxzdrlfvpdsenlodueqzlmtbgagry` int DEFAULT NULL,\n `lijzngjgtgbyomjglutcqivffuweenxy` int DEFAULT NULL,\n `nltmbjozpjaopsnoxsqrdfkmjtdtlmiv` int DEFAULT NULL,\n `phuvzwrxgsbaputqejsvsotbjgtvaiex` int DEFAULT NULL,\n `hfchssrjtmzcdbafxnoxdzuduwtebqtz` int DEFAULT NULL,\n `byrrnflzvcmemxokyqxfzdlmlacouzrt` int DEFAULT NULL,\n `tddwmvywnhbilijwfydmoqxybuoumers` int DEFAULT NULL,\n `attlmypswzmhucxuxawsahijxqcpydjm` int DEFAULT NULL,\n `ippkaykkaeuvxgxgfgrzxhjamrdvtika` int DEFAULT NULL,\n `dqhlzdftsdyjrpnthrqeynddvrulqwcl` int DEFAULT NULL,\n `dinytqyjtswttrgjburpsxchdeslmdvw` int DEFAULT NULL,\n `mytopzzqbhhadixmdwmzwmkckkcdaolo` int DEFAULT NULL,\n `ggluncvagmrmszhlkvcyhvpmhpnhmall` int DEFAULT NULL,\n `wpycdkzszxqqhtejfktvtaknljntjgdw` int DEFAULT NULL,\n `ocmnrsxvlaxtodhddiwmvuwoycfanmhl` int DEFAULT NULL,\n `wccowwaoawjqefnhrnuzsmbhwffnooxf` int DEFAULT NULL,\n `oircdevbsmmbiflpbheravmbcrqjblhn` int DEFAULT NULL,\n `mbxfgcsogbnvxiokiuvftprdbyprnmna` int DEFAULT NULL,\n `myvmbvzirpbxmzihwtfswaxothdsexhl` int DEFAULT NULL,\n `mqpvfmpjafowafoogkjjappfvguqpofl` int DEFAULT NULL,\n `zcqadbchyzcrmcosgtapqcsrdtugzvrn` int DEFAULT NULL,\n `vcabtnenkiafhsqkvoddxhdzccetngiv` int DEFAULT NULL,\n `onfmwkizlcpvnhaaibwwsgrocadveaae` int DEFAULT NULL,\n `skjcsvcdwhcvazliszjmtfwfwnwlyxzr` int DEFAULT NULL,\n `qmdibtetcqjfebeijcgvsalmnkcpytam` int DEFAULT NULL,\n `bjzepkawwnoqealhmajgpuuxqqohtxvp` int DEFAULT NULL,\n `xkdepjcovazmufdhfmctsistzffcewyv` int DEFAULT NULL,\n `yikdadweymxwpngmgftfdvzihvphnqjc` int DEFAULT NULL,\n `jjiyimfpaosmcacuxwtvxinonlhlyawb` int DEFAULT NULL,\n `tdzvkdgkplxxqscstcwrtymdfbhtosdc` int DEFAULT NULL,\n `rpqzphhmmrkaducbqnkgkrfeodlxxmbr` int DEFAULT NULL,\n `jlaabrqhfcuowcxdomiunrqhdaivzpyx` int DEFAULT NULL,\n `zxoimballsspkjhwkdqrmdrqitgixgdw` int DEFAULT NULL,\n `ckkecgnmanzjpxvqxlelyabjdodmmlby` int DEFAULT NULL,\n `hdkwoqbgpogngocghzjelkvaedhkkijn` int DEFAULT NULL,\n `wjjpddwvvilkksvjxabwgrmikhokiwgo` int DEFAULT NULL,\n `iufdjcflhghspvgavgfbshlwuexfjiiy` int DEFAULT NULL,\n `uzdrnwozygbkiaaakosypxtsywavgmci` int DEFAULT NULL,\n `dnycnewrkdcrkvgvvqkthcfujduqqaqb` int DEFAULT NULL,\n `nsdpkdjbvhdkouwoqjozjiptyaenankm` int DEFAULT NULL,\n `lwamzykmdsuaswbokyylquoacwrihmbb` int DEFAULT NULL,\n `upfewpeqpqzxpgjyjosvvenfonddogwl` int DEFAULT NULL,\n `wasixipmbmcuaknojplpotrdovhzfzhg` int DEFAULT NULL,\n `chyvgtlceksxzysrvfvufhghhxvycdfx` int DEFAULT NULL,\n `reyaqeesnoxgriuhnefaqwxxiyynqhsk` int DEFAULT NULL,\n `eobjcwfugqaymofiwlzmepzauhtshpiw` int DEFAULT NULL,\n `ngpzbrnrvrmbxjaoheqfcrepjcrffwix` int DEFAULT NULL,\n `cjjyxwheztaaruitlodtqjovlnlniblz` int DEFAULT NULL,\n `bjznujthkcpcaffdgrpdogszdtntngcy` int DEFAULT NULL,\n `mpnlpvavgyqiqrnjyypxknjyanifnijy` int DEFAULT NULL,\n `yqywelwcwbsdifxdozlfxdruzjllsjja` int DEFAULT NULL,\n `pmqyvlhwljvgfhhtzstkdxtgszdekuup` int DEFAULT NULL,\n `psjtgqywwbmgawrilnklwiwtegsqeuvt` int DEFAULT NULL,\n `jjraaatqqpgiilphwpctvzbuoiytblgu` int DEFAULT NULL,\n `gatwiomoiymkfgqydhugqycehzilmwlq` int DEFAULT NULL,\n `gtfbdcbxnzcxcaiftunqokamrqcipbub` int DEFAULT NULL,\n `tjzpurowyamoviejvlnxnutjdazgoznd` int DEFAULT NULL,\n `tledpcuhobbizqachkrzdtqnuioevoxu` int DEFAULT NULL,\n `tlcehsusqdkwnookamnzebhlbvsqooie` int DEFAULT NULL,\n `leaklhqrhklrbaqeclcxlqgbmexxpmnd` int DEFAULT NULL,\n `yxtdlgwjzggrecghuulkvjtlpvmyiiob` int DEFAULT NULL,\n `rboxvkejjumosfpxjyccdxpxincamyft` int DEFAULT NULL,\n `esbfuvximnbhchyqmwdtaaqwfyvvisuz` int DEFAULT NULL,\n `fbscgklplpqcdmbrmmcxyxphamfdnhuo` int DEFAULT NULL,\n `tqypvbrogcgxxhqkwqtbskjqgygmrany` int DEFAULT NULL,\n `wqnbgdrwobtwvtabvkzcsfaijrmokkws` int DEFAULT NULL,\n `mpfqoknsiyytusazmyncikbopiqlhfvs` int DEFAULT NULL,\n `vzciinqswreohhbyrbmrnqypupphrxhk` int DEFAULT NULL,\n `idioqhssbulafoewhdnkhpqdbgtuhcmu` int DEFAULT NULL,\n `orqhfxfsjymmtvjiryveaklbgfltsyyc` int DEFAULT NULL,\n `xipiteiskqevtfbactvpoxlzhnwgpuhb` int DEFAULT NULL,\n `mqcpuusqgddpcmitqkangxingdoltpuw` int DEFAULT NULL,\n `ekyupujweztqnrecjjoksoutnpkqktol` int DEFAULT NULL,\n `khomsuxgoludnzrnfcdlfhcqjwkgxskt` int DEFAULT NULL,\n `xszjejedlkomrksovenhqhpsxdubuizt` int DEFAULT NULL,\n `pjphtuheloluicbwhujwwssszlweckbx` int DEFAULT NULL,\n `mwzzfdqotzvpxsphpviorcwnjwelfthi` int DEFAULT NULL,\n `averveantsgkahwbpetlqlcayjhktvju` int DEFAULT NULL,\n `pnsozpazxkynxhhwixeprfxuklzrypgq` int DEFAULT NULL,\n `oppdmwvmiyvppgobnxgawocbntqlhwwy` int DEFAULT NULL,\n `cbnwkrsskrufknszwlyxlxrcnktditau` int DEFAULT NULL,\n PRIMARY KEY (`vruopcjgamceaidwmrlrkdyaqwenwkzo`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"uyllrczkxxxqdijmtoeuaxgipxtztugl\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["vruopcjgamceaidwmrlrkdyaqwenwkzo"],"columns":[{"name":"vruopcjgamceaidwmrlrkdyaqwenwkzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"nnsrrthzzyqlxvohoyrwkxbfibuplhvt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmdrouurqfjxzumclwwlfolijyrfzwur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebxaexgmuiodwlodkfmjlaqbvhtsriph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jkwctlvmdikacgekkdgegzdepblvnjhf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mebaqapqnmysewazkmljccaaskcvpqot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xktmcuwszbtutaddyhaegojmhcvezfdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"frmqhdmxchleexhqdzpsfkcdututueiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hagiscraubxmterzpitodjajmbqmauly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lcjijfdkcslkkcecdeukgscgsvmrwcbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xfvlrqejvawjybzfntdcpocybggvijab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xesowoosexseabfaaqcjrbvpofryjwuj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yoajwdpahiefdnjwmaxpuihzsdjcydpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqeahqdcrjdyluscyoqqrkxgbggyfkyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqiyelgxfoaifsmkdonvrvvfdvqsuzgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"svdvnoughgyjycnqdaesfjeepmjkzqnq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqwxvlfuokcutybtwmzpqbpbnmsdtfcg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mowqoksvynmkqyzkznabwegrydloboax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqbcgpxgrnympejujagpnfkwwkqyykkt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twfbxzdrlfvpdsenlodueqzlmtbgagry","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lijzngjgtgbyomjglutcqivffuweenxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nltmbjozpjaopsnoxsqrdfkmjtdtlmiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phuvzwrxgsbaputqejsvsotbjgtvaiex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfchssrjtmzcdbafxnoxdzuduwtebqtz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"byrrnflzvcmemxokyqxfzdlmlacouzrt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tddwmvywnhbilijwfydmoqxybuoumers","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"attlmypswzmhucxuxawsahijxqcpydjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ippkaykkaeuvxgxgfgrzxhjamrdvtika","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqhlzdftsdyjrpnthrqeynddvrulqwcl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dinytqyjtswttrgjburpsxchdeslmdvw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mytopzzqbhhadixmdwmzwmkckkcdaolo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggluncvagmrmszhlkvcyhvpmhpnhmall","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wpycdkzszxqqhtejfktvtaknljntjgdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocmnrsxvlaxtodhddiwmvuwoycfanmhl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wccowwaoawjqefnhrnuzsmbhwffnooxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oircdevbsmmbiflpbheravmbcrqjblhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbxfgcsogbnvxiokiuvftprdbyprnmna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"myvmbvzirpbxmzihwtfswaxothdsexhl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqpvfmpjafowafoogkjjappfvguqpofl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zcqadbchyzcrmcosgtapqcsrdtugzvrn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vcabtnenkiafhsqkvoddxhdzccetngiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"onfmwkizlcpvnhaaibwwsgrocadveaae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skjcsvcdwhcvazliszjmtfwfwnwlyxzr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmdibtetcqjfebeijcgvsalmnkcpytam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjzepkawwnoqealhmajgpuuxqqohtxvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkdepjcovazmufdhfmctsistzffcewyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yikdadweymxwpngmgftfdvzihvphnqjc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjiyimfpaosmcacuxwtvxinonlhlyawb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdzvkdgkplxxqscstcwrtymdfbhtosdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpqzphhmmrkaducbqnkgkrfeodlxxmbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlaabrqhfcuowcxdomiunrqhdaivzpyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxoimballsspkjhwkdqrmdrqitgixgdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckkecgnmanzjpxvqxlelyabjdodmmlby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdkwoqbgpogngocghzjelkvaedhkkijn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjjpddwvvilkksvjxabwgrmikhokiwgo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iufdjcflhghspvgavgfbshlwuexfjiiy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzdrnwozygbkiaaakosypxtsywavgmci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dnycnewrkdcrkvgvvqkthcfujduqqaqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nsdpkdjbvhdkouwoqjozjiptyaenankm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwamzykmdsuaswbokyylquoacwrihmbb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upfewpeqpqzxpgjyjosvvenfonddogwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wasixipmbmcuaknojplpotrdovhzfzhg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chyvgtlceksxzysrvfvufhghhxvycdfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"reyaqeesnoxgriuhnefaqwxxiyynqhsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eobjcwfugqaymofiwlzmepzauhtshpiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ngpzbrnrvrmbxjaoheqfcrepjcrffwix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjjyxwheztaaruitlodtqjovlnlniblz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjznujthkcpcaffdgrpdogszdtntngcy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpnlpvavgyqiqrnjyypxknjyanifnijy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqywelwcwbsdifxdozlfxdruzjllsjja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmqyvlhwljvgfhhtzstkdxtgszdekuup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psjtgqywwbmgawrilnklwiwtegsqeuvt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjraaatqqpgiilphwpctvzbuoiytblgu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gatwiomoiymkfgqydhugqycehzilmwlq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtfbdcbxnzcxcaiftunqokamrqcipbub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjzpurowyamoviejvlnxnutjdazgoznd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tledpcuhobbizqachkrzdtqnuioevoxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlcehsusqdkwnookamnzebhlbvsqooie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"leaklhqrhklrbaqeclcxlqgbmexxpmnd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxtdlgwjzggrecghuulkvjtlpvmyiiob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rboxvkejjumosfpxjyccdxpxincamyft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"esbfuvximnbhchyqmwdtaaqwfyvvisuz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbscgklplpqcdmbrmmcxyxphamfdnhuo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqypvbrogcgxxhqkwqtbskjqgygmrany","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqnbgdrwobtwvtabvkzcsfaijrmokkws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpfqoknsiyytusazmyncikbopiqlhfvs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzciinqswreohhbyrbmrnqypupphrxhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idioqhssbulafoewhdnkhpqdbgtuhcmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orqhfxfsjymmtvjiryveaklbgfltsyyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xipiteiskqevtfbactvpoxlzhnwgpuhb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqcpuusqgddpcmitqkangxingdoltpuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekyupujweztqnrecjjoksoutnpkqktol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khomsuxgoludnzrnfcdlfhcqjwkgxskt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xszjejedlkomrksovenhqhpsxdubuizt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjphtuheloluicbwhujwwssszlweckbx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwzzfdqotzvpxsphpviorcwnjwelfthi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"averveantsgkahwbpetlqlcayjhktvju","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnsozpazxkynxhhwixeprfxuklzrypgq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oppdmwvmiyvppgobnxgawocbntqlhwwy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbnwkrsskrufknszwlyxlxrcnktditau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842671964,"databaseName":"models_schema","ddl":"CREATE TABLE `uzfkyntmiqwwfluspuhyvetjysuvdqgr` (\n `bkaprbpbvomnlbfrqlbqoslpbvkvhueq` int NOT NULL,\n `zhqwzybapekcrxpgxwcdowawubcoutlt` int DEFAULT NULL,\n `ynjukxzbqojqnbpbwoijbjwtuibmginm` int DEFAULT NULL,\n `fnimeqqirlicfizahngrazzrvypwqnwj` int DEFAULT NULL,\n `kmlkpjuzqttrwicnhlqrmnexjvgglkqz` int DEFAULT NULL,\n `eqgjlvsltqtovrjdnravzlzlmwuemmiw` int DEFAULT NULL,\n `ibcztiabrfdssdhmsqadldzmhreplvyr` int DEFAULT NULL,\n `crkwsnpgfdfemzqgumkzyleocxrryugg` int DEFAULT NULL,\n `mtzbwuigjketbdihtswzryhsgaxcmpqm` int DEFAULT NULL,\n `llvpvnzamncvymdvdceazdufqvmomxqr` int DEFAULT NULL,\n `txpsyjvyaczzvksuzfhvmhtrjqemkvjd` int DEFAULT NULL,\n `msqtrmdwhgxtloldtmpjovghumhxgvgj` int DEFAULT NULL,\n `yqofwddqdguruaueunvectgkkvwuvifb` int DEFAULT NULL,\n `iduvnipmoqymtijmklyiwabtfucjefzu` int DEFAULT NULL,\n `bjtwgrhsyzbzfdfntjmvngyljkkhloru` int DEFAULT NULL,\n `lvwpmptydtlfitdmphgmwaaxkhajkudl` int DEFAULT NULL,\n `yhcvhhvkuonhntovcsqkxwgdmursmccg` int DEFAULT NULL,\n `fcabwrbcnnuodvltovbbqusokxjdzghv` int DEFAULT NULL,\n `azbeyvscwyvmfunfyjeqvhtprwxnzklj` int DEFAULT NULL,\n `flwiabeoplhygjltuhkbvaqylrmakthq` int DEFAULT NULL,\n `bdrfjhbmexovjqculkrevdtmjrervapp` int DEFAULT NULL,\n `gxvuowpwdiknwmwfjxjeukuuzozwnstm` int DEFAULT NULL,\n `dxdzsvkqzljofqkuobovttqbngvypahe` int DEFAULT NULL,\n `favkafubrmkkcxumqpppuruxqzdgwcao` int DEFAULT NULL,\n `lroxpimihzufxfncbzuiyasftrydnqbr` int DEFAULT NULL,\n `ltazmiaggkopazvalvctsidkriesotqm` int DEFAULT NULL,\n `gpwmkbauauynvlgiyhzuqjliowwcqqlx` int DEFAULT NULL,\n `gqrkicunmodspqwtbbldzowuqdtnppqh` int DEFAULT NULL,\n `gvdgsctmlolsxayxknncktnbuztfesjj` int DEFAULT NULL,\n `mlchaejngqlvihuwgfnndhqdcoinqqxe` int DEFAULT NULL,\n `ytrdgukwvrtpnsqrkmwudsbdnwmdkonc` int DEFAULT NULL,\n `veamhkuqrnfdtzxdgbkgwsgjsrocjyqe` int DEFAULT NULL,\n `tmiylhvgbvozctjzjbqbilyfrodtzmwz` int DEFAULT NULL,\n `kdaljphejfiixujuibqfqyqiaxogqrod` int DEFAULT NULL,\n `pflvlxkxywsjijbijgkraqntqkkelskg` int DEFAULT NULL,\n `puygtijkncvlztzruwfkyyrxqqhzltlq` int DEFAULT NULL,\n `xxopwunxjtdnchtyhoiogcfobnjxijxu` int DEFAULT NULL,\n `vaaiucjrxkiolwqwpognpkmsrsfohxcq` int DEFAULT NULL,\n `dvhnvhjfglwgzhdzjkbfnyiyfrrkfqkg` int DEFAULT NULL,\n `teytorsiddfadbccojutfunufmfryrjv` int DEFAULT NULL,\n `wyphiayvscslqwazkvfrnlgagizzjfwz` int DEFAULT NULL,\n `cqxnbktqsqdtfzduzbcsxabzhfajkilb` int DEFAULT NULL,\n `zkbmdloonirnntgoubgvievbbxrdbpsa` int DEFAULT NULL,\n `ysvjawjgfvykvclsxuiwfusppsrkhdlz` int DEFAULT NULL,\n `orcroqzwruirxmgyhiolfqbtoewsshcr` int DEFAULT NULL,\n `ljwjbvudrageyufrwihyzgelzqnzqxpw` int DEFAULT NULL,\n `dxbhtidzvsrletirayveidncmpubwiop` int DEFAULT NULL,\n `ubndnrdtydfnrhkayexbveoragyalfza` int DEFAULT NULL,\n `vuxishkkcmcyaoyzaetdmcsbefzajsso` int DEFAULT NULL,\n `mbovnucnslejhmesncdqdoradmwdvvpy` int DEFAULT NULL,\n `fjubxmckvjxjnvdttxtdwpgtkcccdsug` int DEFAULT NULL,\n `qgytwfzaafhfteredvcbsimzlhfowmsy` int DEFAULT NULL,\n `cgtrupyfnkexrosxiojzrptuzuxdfkcv` int DEFAULT NULL,\n `dofvqcbvmdwqxnxahlelvejasdfjfmbn` int DEFAULT NULL,\n `zcsbevufzggjkdbcwptojxhmgapcqdlo` int DEFAULT NULL,\n `yykhzuaetlhshznfzoeakxjebcyojebt` int DEFAULT NULL,\n `qktlhoyeierngngxrwaweblbhsmynxza` int DEFAULT NULL,\n `ffzdoarqgmghpwgnjykbfzlfzfljffir` int DEFAULT NULL,\n `entwvnzzxzbwrgatzcjyfiititkgpkcy` int DEFAULT NULL,\n `vrpsqcqtxaubtrebaxlilluqtzvsiztc` int DEFAULT NULL,\n `gcumcaxxludgbyqrzgjsthtygpgmwwmh` int DEFAULT NULL,\n `kakxkmftydebgrysyypsxsieviaynipn` int DEFAULT NULL,\n `emmxboyxedisnerwxiyjolniwdnqfyvi` int DEFAULT NULL,\n `hbdsouyxtploerwjtvsuyhelnlotmkyg` int DEFAULT NULL,\n `edyoohtxfaigkfubxvmouxxtuqnbycrd` int DEFAULT NULL,\n `xpcxsaoxycumpascgpoilldjqlooojfd` int DEFAULT NULL,\n `udzfkmgxsnszewqhrltezofcnufmzrre` int DEFAULT NULL,\n `afkzfauwbzupmmtplzgtqitjatocsgxh` int DEFAULT NULL,\n `yutviqrhstrmkapveersvbzmqswjcfwb` int DEFAULT NULL,\n `sjmlylvpvltxsqgtugbljssebodalcoq` int DEFAULT NULL,\n `mfcaupdmwldoycbdaajdxubihzayxvuw` int DEFAULT NULL,\n `hopjokiacktoascbynhjbrdtsmahijvd` int DEFAULT NULL,\n `uurgddmiuxothdojrhhcpbwhmvnhwrdt` int DEFAULT NULL,\n `foxddcephewjtwbaqaahixnjljxuzpjx` int DEFAULT NULL,\n `xkxatvoowudcolvrluwqkhnqsllxmyth` int DEFAULT NULL,\n `nneeicfvhbruoofybbaxflfwlymiqpzq` int DEFAULT NULL,\n `nutcupkhivqqhyrujzzjwifjoncpasyn` int DEFAULT NULL,\n `vxygbzwlpmnrbvdullaiojasqsvfgumc` int DEFAULT NULL,\n `dwagvntigisvtunqmeahynxnrorkxfuo` int DEFAULT NULL,\n `akpyekdbodyninekmptvlqnhbuclmqyu` int DEFAULT NULL,\n `hamujzjrbwjmqvlfnzxbfklpygifnbtq` int DEFAULT NULL,\n `ejpodowciplpcnqwadgpryacippcscdi` int DEFAULT NULL,\n `uwdqmeffhvixgpuxtwtuxmpwziebdzde` int DEFAULT NULL,\n `oenfkovsbabdldrujrbbtuqasfnudhyd` int DEFAULT NULL,\n `yxlhmwtuycxrjpsznbhrpsfmzyfdgiqh` int DEFAULT NULL,\n `twcnexpajadelgihhmjjgfqivdudpqlx` int DEFAULT NULL,\n `stwyenxmytuanchfeinhhhicjicxffbc` int DEFAULT NULL,\n `vztlzufwupslaoycqylxfzxjfedbcbhz` int DEFAULT NULL,\n `ljxfvqqrosiljerpwtfximuylgyrkfng` int DEFAULT NULL,\n `sfzhipnxeridzrqvkcsalbesihqjoqfz` int DEFAULT NULL,\n `aphrjmilmmkmghtedtxudgdwlpihqnum` int DEFAULT NULL,\n `wuvfkklxvyvkuovrvqophovjpydmeiaw` int DEFAULT NULL,\n `nvlpntnqonmatihxhunctuigbnigjoht` int DEFAULT NULL,\n `oktalvrncexfghdklablogwbhpfhyoyz` int DEFAULT NULL,\n `hqhgeiervpkxdyflzdbiztsrobzpqcmf` int DEFAULT NULL,\n `fcyrjncbhylclxoyqznoodagmdliualc` int DEFAULT NULL,\n `udzxwrwmmmagjpdtgjyjtobtbbtffejw` int DEFAULT NULL,\n `nowgfbfntsgkvwvwtwvizluwzrwwlcwz` int DEFAULT NULL,\n `xrrwfnmmibqtdqwfilnrbgghfnjtdmwy` int DEFAULT NULL,\n `ryawfqfnerxkpyaisgxflmneininhlvw` int DEFAULT NULL,\n PRIMARY KEY (`bkaprbpbvomnlbfrqlbqoslpbvkvhueq`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"uzfkyntmiqwwfluspuhyvetjysuvdqgr\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["bkaprbpbvomnlbfrqlbqoslpbvkvhueq"],"columns":[{"name":"bkaprbpbvomnlbfrqlbqoslpbvkvhueq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"zhqwzybapekcrxpgxwcdowawubcoutlt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynjukxzbqojqnbpbwoijbjwtuibmginm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnimeqqirlicfizahngrazzrvypwqnwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmlkpjuzqttrwicnhlqrmnexjvgglkqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqgjlvsltqtovrjdnravzlzlmwuemmiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibcztiabrfdssdhmsqadldzmhreplvyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crkwsnpgfdfemzqgumkzyleocxrryugg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtzbwuigjketbdihtswzryhsgaxcmpqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llvpvnzamncvymdvdceazdufqvmomxqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txpsyjvyaczzvksuzfhvmhtrjqemkvjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"msqtrmdwhgxtloldtmpjovghumhxgvgj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqofwddqdguruaueunvectgkkvwuvifb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iduvnipmoqymtijmklyiwabtfucjefzu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjtwgrhsyzbzfdfntjmvngyljkkhloru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvwpmptydtlfitdmphgmwaaxkhajkudl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhcvhhvkuonhntovcsqkxwgdmursmccg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcabwrbcnnuodvltovbbqusokxjdzghv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azbeyvscwyvmfunfyjeqvhtprwxnzklj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flwiabeoplhygjltuhkbvaqylrmakthq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdrfjhbmexovjqculkrevdtmjrervapp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxvuowpwdiknwmwfjxjeukuuzozwnstm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxdzsvkqzljofqkuobovttqbngvypahe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"favkafubrmkkcxumqpppuruxqzdgwcao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lroxpimihzufxfncbzuiyasftrydnqbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ltazmiaggkopazvalvctsidkriesotqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gpwmkbauauynvlgiyhzuqjliowwcqqlx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqrkicunmodspqwtbbldzowuqdtnppqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gvdgsctmlolsxayxknncktnbuztfesjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlchaejngqlvihuwgfnndhqdcoinqqxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytrdgukwvrtpnsqrkmwudsbdnwmdkonc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"veamhkuqrnfdtzxdgbkgwsgjsrocjyqe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmiylhvgbvozctjzjbqbilyfrodtzmwz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kdaljphejfiixujuibqfqyqiaxogqrod","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pflvlxkxywsjijbijgkraqntqkkelskg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"puygtijkncvlztzruwfkyyrxqqhzltlq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxopwunxjtdnchtyhoiogcfobnjxijxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vaaiucjrxkiolwqwpognpkmsrsfohxcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvhnvhjfglwgzhdzjkbfnyiyfrrkfqkg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"teytorsiddfadbccojutfunufmfryrjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyphiayvscslqwazkvfrnlgagizzjfwz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqxnbktqsqdtfzduzbcsxabzhfajkilb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zkbmdloonirnntgoubgvievbbxrdbpsa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ysvjawjgfvykvclsxuiwfusppsrkhdlz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orcroqzwruirxmgyhiolfqbtoewsshcr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljwjbvudrageyufrwihyzgelzqnzqxpw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxbhtidzvsrletirayveidncmpubwiop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubndnrdtydfnrhkayexbveoragyalfza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vuxishkkcmcyaoyzaetdmcsbefzajsso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbovnucnslejhmesncdqdoradmwdvvpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjubxmckvjxjnvdttxtdwpgtkcccdsug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgytwfzaafhfteredvcbsimzlhfowmsy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgtrupyfnkexrosxiojzrptuzuxdfkcv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dofvqcbvmdwqxnxahlelvejasdfjfmbn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zcsbevufzggjkdbcwptojxhmgapcqdlo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yykhzuaetlhshznfzoeakxjebcyojebt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qktlhoyeierngngxrwaweblbhsmynxza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffzdoarqgmghpwgnjykbfzlfzfljffir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"entwvnzzxzbwrgatzcjyfiititkgpkcy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrpsqcqtxaubtrebaxlilluqtzvsiztc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gcumcaxxludgbyqrzgjsthtygpgmwwmh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kakxkmftydebgrysyypsxsieviaynipn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"emmxboyxedisnerwxiyjolniwdnqfyvi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbdsouyxtploerwjtvsuyhelnlotmkyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edyoohtxfaigkfubxvmouxxtuqnbycrd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpcxsaoxycumpascgpoilldjqlooojfd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udzfkmgxsnszewqhrltezofcnufmzrre","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afkzfauwbzupmmtplzgtqitjatocsgxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yutviqrhstrmkapveersvbzmqswjcfwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjmlylvpvltxsqgtugbljssebodalcoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfcaupdmwldoycbdaajdxubihzayxvuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hopjokiacktoascbynhjbrdtsmahijvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uurgddmiuxothdojrhhcpbwhmvnhwrdt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"foxddcephewjtwbaqaahixnjljxuzpjx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkxatvoowudcolvrluwqkhnqsllxmyth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nneeicfvhbruoofybbaxflfwlymiqpzq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nutcupkhivqqhyrujzzjwifjoncpasyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxygbzwlpmnrbvdullaiojasqsvfgumc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwagvntigisvtunqmeahynxnrorkxfuo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akpyekdbodyninekmptvlqnhbuclmqyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hamujzjrbwjmqvlfnzxbfklpygifnbtq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejpodowciplpcnqwadgpryacippcscdi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwdqmeffhvixgpuxtwtuxmpwziebdzde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oenfkovsbabdldrujrbbtuqasfnudhyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxlhmwtuycxrjpsznbhrpsfmzyfdgiqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twcnexpajadelgihhmjjgfqivdudpqlx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"stwyenxmytuanchfeinhhhicjicxffbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vztlzufwupslaoycqylxfzxjfedbcbhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljxfvqqrosiljerpwtfximuylgyrkfng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfzhipnxeridzrqvkcsalbesihqjoqfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aphrjmilmmkmghtedtxudgdwlpihqnum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wuvfkklxvyvkuovrvqophovjpydmeiaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvlpntnqonmatihxhunctuigbnigjoht","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oktalvrncexfghdklablogwbhpfhyoyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqhgeiervpkxdyflzdbiztsrobzpqcmf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcyrjncbhylclxoyqznoodagmdliualc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udzxwrwmmmagjpdtgjyjtobtbbtffejw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nowgfbfntsgkvwvwtwvizluwzrwwlcwz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrrwfnmmibqtdqwfilnrbgghfnjtdmwy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ryawfqfnerxkpyaisgxflmneininhlvw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842671,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672003,"databaseName":"models_schema","ddl":"CREATE TABLE `uzwzfxgsmncskifwztscboirulhiccox` (\n `jrdhosbhavvgqojdwkhuvnyildycgriw` int NOT NULL,\n `bpdxcxpyvafeageyqwdwomoohmsqkobn` int DEFAULT NULL,\n `seoldpuavfvpmwfvhphqmwouqfvaecvy` int DEFAULT NULL,\n `sxpozibysdkktqstnczezwxfezixjuks` int DEFAULT NULL,\n `bjayfeggbterfunodafgzlplnqjwlfup` int DEFAULT NULL,\n `nulyeczmheuavplpagsgzoykgfkmyese` int DEFAULT NULL,\n `tuauldrywiuimvdhjznzpcwdwapuougf` int DEFAULT NULL,\n `zdlvpdlqoxsunptarnwyaiwydjljpfbn` int DEFAULT NULL,\n `jsmhikdebfcyelfnopjwftvtbhncubiq` int DEFAULT NULL,\n `ubjdxyjsspxnemekahgouzwyraqjuwpn` int DEFAULT NULL,\n `jjlbsvqgjlgqcoxikdvmuokqfkjismpl` int DEFAULT NULL,\n `nzzrxfyujxmdsbseojymebpdqehykqkd` int DEFAULT NULL,\n `rilwnalmhhpgzkodzfbfktimyoarersi` int DEFAULT NULL,\n `mdjtaksmxtvyvrkiidmrydbiinupqtnl` int DEFAULT NULL,\n `hcatbwkjrnnuudbwsqrqiiuihcomxxoi` int DEFAULT NULL,\n `cyeqzpweztvkibdgfiqegrkrypqqjqxo` int DEFAULT NULL,\n `zezkqxzqciouqaourktcomqxhichgzsu` int DEFAULT NULL,\n `ukzwwhvgieovldedqldvrytkejgdygyp` int DEFAULT NULL,\n `airszwztsswvvhhdmjhetoizwgjxabox` int DEFAULT NULL,\n `rimcjyrwpdzxphrvtgvtqatiribyithu` int DEFAULT NULL,\n `itkgtlvblwqlwbnfhvejdlbnhiplugua` int DEFAULT NULL,\n `qeqlfqxknulqtyqxwozbcyabsogzddad` int DEFAULT NULL,\n `xqixjvyxlexzbotmvfywhjvwrlqvlpvb` int DEFAULT NULL,\n `nfzfgdgnowyawijhdluhupudqhggrved` int DEFAULT NULL,\n `jlgavovgntkvtjvbptygsuarvzjnvoga` int DEFAULT NULL,\n `orcsrczltzmhssprwecxcfjqlukxwkgi` int DEFAULT NULL,\n `rubpccmltbnzfheugxmjzppzfrplszps` int DEFAULT NULL,\n `ytcnknfcujgyfegrbobqsikgumxosjqv` int DEFAULT NULL,\n `mhppcmcfibbuxhwszgcucafmsjgzxgri` int DEFAULT NULL,\n `cllovcpfzxosuxicqyteymgssxrqpuzk` int DEFAULT NULL,\n `cldpqxzatfeopoartjtnwomptgbnuwlo` int DEFAULT NULL,\n `tiudjjasxcnhchterwcfnurbfkbdokgu` int DEFAULT NULL,\n `wsekbojuxvnfvqhbjgtlxtattcfmfbbp` int DEFAULT NULL,\n `vnyydrptrpalvkenwaabpmcajwbrsulb` int DEFAULT NULL,\n `ltgipbghufnmmfowxyelpanbfxmvunzg` int DEFAULT NULL,\n `ubyjpsoruaujqwzdesjmyakwozgctska` int DEFAULT NULL,\n `nszyhdzkwoabfqgrmkyvombxblvmcebc` int DEFAULT NULL,\n `tpcjjfdohheomovrihzuscvmkotqzrhy` int DEFAULT NULL,\n `xuknszzxdipodlrbevbixfinayqjohqs` int DEFAULT NULL,\n `pnajnwtgaimnkkschlrgrdtewbjbtfsp` int DEFAULT NULL,\n `cevbxasjthlmbospqgsnqyibsyycmphy` int DEFAULT NULL,\n `kckfnfeuvgwogtjntndmgyxcwrudlfom` int DEFAULT NULL,\n `evukosdohtowjgkmcntwpjnrssmhheea` int DEFAULT NULL,\n `jjqxojhiyuwbqemsoeknxkjbqwtknerc` int DEFAULT NULL,\n `chqbjjkcyanvbbvdpfuafahlzggrtmin` int DEFAULT NULL,\n `zdbydpbyespjndphsqsrkcrrnsncadne` int DEFAULT NULL,\n `uoctkpoqorsaohglvsfmkkilurtnetsk` int DEFAULT NULL,\n `dbuvlbwvlggfdhifkmlwcutkcbfuhjdt` int DEFAULT NULL,\n `zdltfhgiswdzpiwhiwmutpvqghmiqvpy` int DEFAULT NULL,\n `osojfavgftnkmjglvhuoxuznksplslbp` int DEFAULT NULL,\n `nzakhzpwvnkkypcnsiararupozcnrzyk` int DEFAULT NULL,\n `kowclkomzsnvewiwkzuaeugvilipnsag` int DEFAULT NULL,\n `udkejngosstaublohyuaykqcbbnbovag` int DEFAULT NULL,\n `jubzuiegywfqmivwijpofimihoztnrun` int DEFAULT NULL,\n `skbrduzubmxekzzbcgefsfdmwixmqqab` int DEFAULT NULL,\n `ajoqdslekpfncyackbxfkhjibldmssdi` int DEFAULT NULL,\n `uhvvqbwmgkjskkgxouwqdfovjbqlmwcm` int DEFAULT NULL,\n `wmxbjtxmqdulbabsqxgbkmbygqtkkvxl` int DEFAULT NULL,\n `yjjorarxtckibmlngohunpnjuonljggl` int DEFAULT NULL,\n `birfpwbpfqzaejczenfdgtmnfdqycriy` int DEFAULT NULL,\n `qnerhnlpbysrarvcourljqmvmvsyrnax` int DEFAULT NULL,\n `lbrxszbpjsgnepssrptrjtcjieebqvpz` int DEFAULT NULL,\n `vrszdmycgxlqttmlegxtbuazbxpieglv` int DEFAULT NULL,\n `xqlcmbemuikfyageufaiqygaagomzcun` int DEFAULT NULL,\n `wepmyhzvfmkveytsdgrhvfwtdcmxbbgx` int DEFAULT NULL,\n `ydxtumhtqwrogcfczhwgtgwiemzorvmz` int DEFAULT NULL,\n `echnrxpsijyzpzlxmxjfnhfabfwykagg` int DEFAULT NULL,\n `yyjvedtugsptiwfzrwkluzcwscbbouge` int DEFAULT NULL,\n `ykivlgrhbskiplezuhrxpmldwxwpjtct` int DEFAULT NULL,\n `ofrayaizauvliqykiaxirbyesbdoimdc` int DEFAULT NULL,\n `hycrffslcfkppmeijuhpojkmqkraecze` int DEFAULT NULL,\n `mcmvuvvohnnxiezbgzzklqjigbiofqjg` int DEFAULT NULL,\n `qgoclrlqdvslpmegcmvlrmbkixiifoze` int DEFAULT NULL,\n `lvbosltqdhsabvmlvsdtqpwytgwllkkm` int DEFAULT NULL,\n `mcwlsibcqwvslmbfsekpcsvarwsljvks` int DEFAULT NULL,\n `vhmeygmnsgvxvaonwfhsnjgaiyulmxrj` int DEFAULT NULL,\n `ohrncoiygktpxeghabhllaqeuplswupa` int DEFAULT NULL,\n `fztobipnixytjkwrtwrdluurqonurykh` int DEFAULT NULL,\n `qymtscsrmajetaoaevvsarzuwmfrkvny` int DEFAULT NULL,\n `cocsufktbzxywcwrslyckkkhmhlhtwux` int DEFAULT NULL,\n `tsophphjogacszhojuqltvieswjntebs` int DEFAULT NULL,\n `hhrhhomkbcngbbhnzluljvjfrbdffztm` int DEFAULT NULL,\n `gojoyihxygduykunjwokizrzskioxwzi` int DEFAULT NULL,\n `umaindthctrkkgnlpdbjszzxwcvclatj` int DEFAULT NULL,\n `bvgsiyedxntyfidyvvtsrrealmhaemug` int DEFAULT NULL,\n `sjidpnuuabpzqgfxsbtvntdtpjvpxfcq` int DEFAULT NULL,\n `mkqeoqxsxwbbuquozlurhkysumxnydxj` int DEFAULT NULL,\n `auiztrmenzugogofwjvbuwlgqjzwyrre` int DEFAULT NULL,\n `mzmoqdsowrzqnlfiszmuejvmqtlymuuy` int DEFAULT NULL,\n `chkxcedvmhkrvvwgszyxreqhajfbvqyn` int DEFAULT NULL,\n `lncabrsuselrcajkantzesupohozobgw` int DEFAULT NULL,\n `fptbydqdipbkzlamvsdzzsfeoikltnwc` int DEFAULT NULL,\n `jtgnqzhyqixoczppavqjkirassjsrvnx` int DEFAULT NULL,\n `gsynjdiprjpvoxwhkuihjqetrdzzrxei` int DEFAULT NULL,\n `ljujiovgfcuscflrguslwzececytkpjd` int DEFAULT NULL,\n `agjrldkmbxuopfwzahanfbrytytljhva` int DEFAULT NULL,\n `nikqorftswrhtlsyaptyxmkaipaxxudh` int DEFAULT NULL,\n `yhbjwtrfgdxzpvhbfeeqqcdxebamlxbq` int DEFAULT NULL,\n `valaquhbwxtwypukhfogbojtyzmhwdec` int DEFAULT NULL,\n `lmnfisrwsuycncvlxoeziqkyualemafz` int DEFAULT NULL,\n PRIMARY KEY (`jrdhosbhavvgqojdwkhuvnyildycgriw`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"uzwzfxgsmncskifwztscboirulhiccox\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["jrdhosbhavvgqojdwkhuvnyildycgriw"],"columns":[{"name":"jrdhosbhavvgqojdwkhuvnyildycgriw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"bpdxcxpyvafeageyqwdwomoohmsqkobn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"seoldpuavfvpmwfvhphqmwouqfvaecvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sxpozibysdkktqstnczezwxfezixjuks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjayfeggbterfunodafgzlplnqjwlfup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nulyeczmheuavplpagsgzoykgfkmyese","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tuauldrywiuimvdhjznzpcwdwapuougf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdlvpdlqoxsunptarnwyaiwydjljpfbn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jsmhikdebfcyelfnopjwftvtbhncubiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubjdxyjsspxnemekahgouzwyraqjuwpn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjlbsvqgjlgqcoxikdvmuokqfkjismpl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzzrxfyujxmdsbseojymebpdqehykqkd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rilwnalmhhpgzkodzfbfktimyoarersi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdjtaksmxtvyvrkiidmrydbiinupqtnl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hcatbwkjrnnuudbwsqrqiiuihcomxxoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cyeqzpweztvkibdgfiqegrkrypqqjqxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zezkqxzqciouqaourktcomqxhichgzsu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ukzwwhvgieovldedqldvrytkejgdygyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"airszwztsswvvhhdmjhetoizwgjxabox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rimcjyrwpdzxphrvtgvtqatiribyithu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itkgtlvblwqlwbnfhvejdlbnhiplugua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qeqlfqxknulqtyqxwozbcyabsogzddad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqixjvyxlexzbotmvfywhjvwrlqvlpvb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfzfgdgnowyawijhdluhupudqhggrved","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlgavovgntkvtjvbptygsuarvzjnvoga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orcsrczltzmhssprwecxcfjqlukxwkgi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rubpccmltbnzfheugxmjzppzfrplszps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytcnknfcujgyfegrbobqsikgumxosjqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhppcmcfibbuxhwszgcucafmsjgzxgri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cllovcpfzxosuxicqyteymgssxrqpuzk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cldpqxzatfeopoartjtnwomptgbnuwlo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tiudjjasxcnhchterwcfnurbfkbdokgu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wsekbojuxvnfvqhbjgtlxtattcfmfbbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnyydrptrpalvkenwaabpmcajwbrsulb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ltgipbghufnmmfowxyelpanbfxmvunzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubyjpsoruaujqwzdesjmyakwozgctska","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nszyhdzkwoabfqgrmkyvombxblvmcebc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpcjjfdohheomovrihzuscvmkotqzrhy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xuknszzxdipodlrbevbixfinayqjohqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnajnwtgaimnkkschlrgrdtewbjbtfsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cevbxasjthlmbospqgsnqyibsyycmphy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kckfnfeuvgwogtjntndmgyxcwrudlfom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evukosdohtowjgkmcntwpjnrssmhheea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjqxojhiyuwbqemsoeknxkjbqwtknerc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chqbjjkcyanvbbvdpfuafahlzggrtmin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdbydpbyespjndphsqsrkcrrnsncadne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uoctkpoqorsaohglvsfmkkilurtnetsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbuvlbwvlggfdhifkmlwcutkcbfuhjdt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdltfhgiswdzpiwhiwmutpvqghmiqvpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"osojfavgftnkmjglvhuoxuznksplslbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzakhzpwvnkkypcnsiararupozcnrzyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kowclkomzsnvewiwkzuaeugvilipnsag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udkejngosstaublohyuaykqcbbnbovag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jubzuiegywfqmivwijpofimihoztnrun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skbrduzubmxekzzbcgefsfdmwixmqqab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajoqdslekpfncyackbxfkhjibldmssdi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhvvqbwmgkjskkgxouwqdfovjbqlmwcm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmxbjtxmqdulbabsqxgbkmbygqtkkvxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjjorarxtckibmlngohunpnjuonljggl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"birfpwbpfqzaejczenfdgtmnfdqycriy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qnerhnlpbysrarvcourljqmvmvsyrnax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lbrxszbpjsgnepssrptrjtcjieebqvpz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrszdmycgxlqttmlegxtbuazbxpieglv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqlcmbemuikfyageufaiqygaagomzcun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wepmyhzvfmkveytsdgrhvfwtdcmxbbgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydxtumhtqwrogcfczhwgtgwiemzorvmz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"echnrxpsijyzpzlxmxjfnhfabfwykagg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yyjvedtugsptiwfzrwkluzcwscbbouge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykivlgrhbskiplezuhrxpmldwxwpjtct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofrayaizauvliqykiaxirbyesbdoimdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hycrffslcfkppmeijuhpojkmqkraecze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcmvuvvohnnxiezbgzzklqjigbiofqjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgoclrlqdvslpmegcmvlrmbkixiifoze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvbosltqdhsabvmlvsdtqpwytgwllkkm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcwlsibcqwvslmbfsekpcsvarwsljvks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhmeygmnsgvxvaonwfhsnjgaiyulmxrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohrncoiygktpxeghabhllaqeuplswupa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fztobipnixytjkwrtwrdluurqonurykh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qymtscsrmajetaoaevvsarzuwmfrkvny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cocsufktbzxywcwrslyckkkhmhlhtwux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tsophphjogacszhojuqltvieswjntebs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhrhhomkbcngbbhnzluljvjfrbdffztm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gojoyihxygduykunjwokizrzskioxwzi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umaindthctrkkgnlpdbjszzxwcvclatj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvgsiyedxntyfidyvvtsrrealmhaemug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjidpnuuabpzqgfxsbtvntdtpjvpxfcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkqeoqxsxwbbuquozlurhkysumxnydxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"auiztrmenzugogofwjvbuwlgqjzwyrre","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzmoqdsowrzqnlfiszmuejvmqtlymuuy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chkxcedvmhkrvvwgszyxreqhajfbvqyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lncabrsuselrcajkantzesupohozobgw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fptbydqdipbkzlamvsdzzsfeoikltnwc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jtgnqzhyqixoczppavqjkirassjsrvnx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsynjdiprjpvoxwhkuihjqetrdzzrxei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljujiovgfcuscflrguslwzececytkpjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agjrldkmbxuopfwzahanfbrytytljhva","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nikqorftswrhtlsyaptyxmkaipaxxudh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhbjwtrfgdxzpvhbfeeqqcdxebamlxbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"valaquhbwxtwypukhfogbojtyzmhwdec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmnfisrwsuycncvlxoeziqkyualemafz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672040,"databaseName":"models_schema","ddl":"CREATE TABLE `vankewnkqbxjamlctdsdjgswgeyvrjof` (\n `uvuxzbqheqcdqjmesmnceffdedytoccu` int NOT NULL,\n `mvsayisciiugdsgocbtgpozhyilmaxbf` int DEFAULT NULL,\n `hcqptpdzectidrccqxofqgrocgmdonqm` int DEFAULT NULL,\n `nqnaybwbsfshpxupqblnvpeibpbzilvy` int DEFAULT NULL,\n `muayrqxqspbpphxyqikkfuadflvwvimc` int DEFAULT NULL,\n `kfixtcqgzhcbuebxzwvqiuylzwxspmid` int DEFAULT NULL,\n `jppkbbcdbskyoostwwkrjuktjprwlhem` int DEFAULT NULL,\n `khntxwjqivpwkhddzwdivwahmbyyrlbw` int DEFAULT NULL,\n `gtcobfynnbvxlinlyblvhdrmakgditio` int DEFAULT NULL,\n `izhgjqcvshbrqbyjbgxiwselmygnulrv` int DEFAULT NULL,\n `fpujsbrdpvzaytjvxnefhiuqnkygrgpr` int DEFAULT NULL,\n `znhhfjblirdwniaxydaovmxdlwufmjxg` int DEFAULT NULL,\n `zeyvejvxkgpyszcdqqfajfticwhbhtpb` int DEFAULT NULL,\n `afntywsccvjcdentzdbxxuborhjcccry` int DEFAULT NULL,\n `kzschyfonyeebirnusfqfcmeyqrbttsp` int DEFAULT NULL,\n `vegivcbdhknofsegibjzgeulnjrlewkq` int DEFAULT NULL,\n `nczoajapcpdxqjqqwgsmmlrqntmupshs` int DEFAULT NULL,\n `ybkgnhtyufshydsxpklgohdtxcxjlhnt` int DEFAULT NULL,\n `bqvwjrevbikbcyadxgmtdayjaynbbdvv` int DEFAULT NULL,\n `cquojpfrlrdtmervzckxaenndytfygbd` int DEFAULT NULL,\n `fjozicjwvwzwcstyducdiehqyxkwqnov` int DEFAULT NULL,\n `pthuywuprhmxplcckshtjfnonloadicf` int DEFAULT NULL,\n `aakzlrhuasekfnaolgsrzdjvweplucqp` int DEFAULT NULL,\n `vrjkslvchowyuqvtaawuulcpgzuetank` int DEFAULT NULL,\n `tsirqlnhqhyrtcydwzwtbueoyxrlgezn` int DEFAULT NULL,\n `htehohvdbvpbsgwugildwxauvndmcaby` int DEFAULT NULL,\n `zdfqqwnocpzhshkwtdghzgukuvyjdzzl` int DEFAULT NULL,\n `ambhtcufanyuiabwurxplirzsqfdvbgs` int DEFAULT NULL,\n `kfzhoaswnudndmfvrjxmautuliexrvqc` int DEFAULT NULL,\n `atxfcqkrtnimiqrieijieiclntlzxwvy` int DEFAULT NULL,\n `wmimaktzpdqzxctlzcjlcblrjmykmccb` int DEFAULT NULL,\n `hynbpvljqjcngdctkuaaypvwxrzrogrv` int DEFAULT NULL,\n `vsiwuuczdihbumzbaxukylvboocceifm` int DEFAULT NULL,\n `efzbcwzilackehaihlptqnfggrbbroir` int DEFAULT NULL,\n `nzhjcczzyipndnlyeewalrgyftqscznt` int DEFAULT NULL,\n `dadttvskxnxgahoibuhsxyiiflgaejws` int DEFAULT NULL,\n `hsnjbygucuggxwbjqyxumqekfbyxqqmo` int DEFAULT NULL,\n `xoyvqhpkwuvlbakwrmkjzhyxvitsfooz` int DEFAULT NULL,\n `rclhzqqdvgzrparhbwjcycdftsppixui` int DEFAULT NULL,\n `cfbqqwwiwhdumqqaivxjjxzfkmicvpnk` int DEFAULT NULL,\n `qpwnetxmlonfbvzkdraaomjvhsorgggt` int DEFAULT NULL,\n `ofcbpveljgdlthuirqorxunrfwymyvka` int DEFAULT NULL,\n `fcivntjhthsfrtmkpnrtwermybkvbymg` int DEFAULT NULL,\n `jfawxvtzgyictdlgttwdlopgiglhuvcs` int DEFAULT NULL,\n `vxzshyxxgdufbwtpeqphuuqgqykvuorj` int DEFAULT NULL,\n `divgxsjwdiamogpvwhppjaobrcszjwuu` int DEFAULT NULL,\n `viypyzxdnvpgecgcuibtnurzgwljdday` int DEFAULT NULL,\n `ijfbjxsnwlphogsabumkevmndxdloziz` int DEFAULT NULL,\n `dnyykufgwclhjvhrvsnjdnikjwzrsllk` int DEFAULT NULL,\n `khenkiogedabnpteczlncbjcjkciqobf` int DEFAULT NULL,\n `wonopsmmvrgdwzjmflwuctusizicfpbo` int DEFAULT NULL,\n `omirkuuftvwejbprgevknxqktrhnvqfb` int DEFAULT NULL,\n `iyxjplihdjwsenxpvggecxsvnxgvyqvh` int DEFAULT NULL,\n `ilufxgthblamhxdzjqyhboyrjywilgwh` int DEFAULT NULL,\n `rgfwtrvfgitvixyznblmhfoamxoxghuy` int DEFAULT NULL,\n `pljgmohgmbltagdrwntfcmwybstzyoll` int DEFAULT NULL,\n `twvjhpdzwvzjcqhnzjvbtytxestqddzf` int DEFAULT NULL,\n `hddfqewxewbpzowlgqwqlukhubocyaik` int DEFAULT NULL,\n `jubgdrbncyxeejxfqnynyrkrixzstoaq` int DEFAULT NULL,\n `mtlqcbvuzfgnfmvxzzspourqfrrnmhnx` int DEFAULT NULL,\n `zqsjfynxpyqevfkvsllxdjxmvrnvmdmz` int DEFAULT NULL,\n `prifcpwgygvbzgbtvmpcqcupmimpxsob` int DEFAULT NULL,\n `ohgcztunyttmamkopblnfowyqhuuamrp` int DEFAULT NULL,\n `zuuubifhkcgqwjmxwxyffoxwgiocypam` int DEFAULT NULL,\n `qgfeldywhseyvqpnjkmgacinrvbpyazf` int DEFAULT NULL,\n `ulgnqxknmzkjljkyhgqecxhecmtmpgof` int DEFAULT NULL,\n `ngoqbasviuxfnftiklakspontqocirin` int DEFAULT NULL,\n `jyehhtbwpwejjoqsvjtqhvwlijiqxzhe` int DEFAULT NULL,\n `lxcpvsyjhnmqbdehfqpdfnuyztwlkycy` int DEFAULT NULL,\n `lghqukjwyreeujspufruhhnlqbfuomdt` int DEFAULT NULL,\n `apcnjcpwsnrrwvrbrvofgirrvefyndrk` int DEFAULT NULL,\n `epmjrocrbegoxpkctkvaxphqzqgwprgh` int DEFAULT NULL,\n `jqormjkhmtejgurpymisrqivvutpzprb` int DEFAULT NULL,\n `cywsyhhmcarysufrlqzidgwzobhidcdx` int DEFAULT NULL,\n `ghamxuqcfokaidcxjgvptrpehyuywgsg` int DEFAULT NULL,\n `vpmnrvmrrpdkmnkhnzorkhfxctwzvygc` int DEFAULT NULL,\n `fiaymgdfyggaawzkehccbhznmtgkgkcy` int DEFAULT NULL,\n `piikfhykuswnisjcfwrrrxxdnjuebjey` int DEFAULT NULL,\n `jqfgdfqhijzwrtsmijaivapfwyakrpgt` int DEFAULT NULL,\n `viorzfjybjftvpjdbywlyddgoomynrbq` int DEFAULT NULL,\n `zzhfqtcetlwvhlkfjcqcqxlgrnpbshqw` int DEFAULT NULL,\n `luwvupdlwiptgonrkoqyrjqbjhdcjbuz` int DEFAULT NULL,\n `wztavsijsmrjjqicugysmytcmlxhfves` int DEFAULT NULL,\n `fdqvyqqjlvukwwzzxzvjodlevfyeorcj` int DEFAULT NULL,\n `socapfxhgsgxxtgnvznhoearsfdeokch` int DEFAULT NULL,\n `sthyfztpkdlquctjmuraxtpbnrirzdzr` int DEFAULT NULL,\n `xlgkgeqxwsanmuobabpkoblcpfbkenht` int DEFAULT NULL,\n `umcemgrfsssbvwubqgiipvaywrgjcmlm` int DEFAULT NULL,\n `gsrtcjkfhnpyuynzfvgnnzdlqgrjlpxb` int DEFAULT NULL,\n `lvcbtmkesgtegwuzuwrzzajsbyfwzsts` int DEFAULT NULL,\n `cmzrrmdwdnrrkdiqtonmosbiiynedoqw` int DEFAULT NULL,\n `soheraibcqqhliryhgwpccxthguxgvsn` int DEFAULT NULL,\n `xakuxbwajlgxokbxmozxzdvdnuxxjpxq` int DEFAULT NULL,\n `uztxtyzkixopbtkxyudcbnazzibkuhst` int DEFAULT NULL,\n `oblsllmhubdovhykjkrwbfcvujpgyypc` int DEFAULT NULL,\n `exdhhihlerniklnhghpjnomavroemxln` int DEFAULT NULL,\n `fngrdvxdgqsyeusssyxyasfdotnojsqr` int DEFAULT NULL,\n `xdgrygzmhodntpmxpjjjqdobcfpmhuox` int DEFAULT NULL,\n `oiurppndjpusgclzwquazzjkzalxpjmv` int DEFAULT NULL,\n `hbqrlvkcxddozbgfrbprnvhrzanubwox` int DEFAULT NULL,\n PRIMARY KEY (`uvuxzbqheqcdqjmesmnceffdedytoccu`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"vankewnkqbxjamlctdsdjgswgeyvrjof\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["uvuxzbqheqcdqjmesmnceffdedytoccu"],"columns":[{"name":"uvuxzbqheqcdqjmesmnceffdedytoccu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"mvsayisciiugdsgocbtgpozhyilmaxbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hcqptpdzectidrccqxofqgrocgmdonqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqnaybwbsfshpxupqblnvpeibpbzilvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"muayrqxqspbpphxyqikkfuadflvwvimc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfixtcqgzhcbuebxzwvqiuylzwxspmid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jppkbbcdbskyoostwwkrjuktjprwlhem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khntxwjqivpwkhddzwdivwahmbyyrlbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtcobfynnbvxlinlyblvhdrmakgditio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"izhgjqcvshbrqbyjbgxiwselmygnulrv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpujsbrdpvzaytjvxnefhiuqnkygrgpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znhhfjblirdwniaxydaovmxdlwufmjxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zeyvejvxkgpyszcdqqfajfticwhbhtpb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afntywsccvjcdentzdbxxuborhjcccry","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzschyfonyeebirnusfqfcmeyqrbttsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vegivcbdhknofsegibjzgeulnjrlewkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nczoajapcpdxqjqqwgsmmlrqntmupshs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ybkgnhtyufshydsxpklgohdtxcxjlhnt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bqvwjrevbikbcyadxgmtdayjaynbbdvv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cquojpfrlrdtmervzckxaenndytfygbd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjozicjwvwzwcstyducdiehqyxkwqnov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pthuywuprhmxplcckshtjfnonloadicf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aakzlrhuasekfnaolgsrzdjvweplucqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrjkslvchowyuqvtaawuulcpgzuetank","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tsirqlnhqhyrtcydwzwtbueoyxrlgezn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htehohvdbvpbsgwugildwxauvndmcaby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdfqqwnocpzhshkwtdghzgukuvyjdzzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ambhtcufanyuiabwurxplirzsqfdvbgs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfzhoaswnudndmfvrjxmautuliexrvqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atxfcqkrtnimiqrieijieiclntlzxwvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmimaktzpdqzxctlzcjlcblrjmykmccb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hynbpvljqjcngdctkuaaypvwxrzrogrv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsiwuuczdihbumzbaxukylvboocceifm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"efzbcwzilackehaihlptqnfggrbbroir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzhjcczzyipndnlyeewalrgyftqscznt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dadttvskxnxgahoibuhsxyiiflgaejws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hsnjbygucuggxwbjqyxumqekfbyxqqmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xoyvqhpkwuvlbakwrmkjzhyxvitsfooz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rclhzqqdvgzrparhbwjcycdftsppixui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfbqqwwiwhdumqqaivxjjxzfkmicvpnk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qpwnetxmlonfbvzkdraaomjvhsorgggt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofcbpveljgdlthuirqorxunrfwymyvka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcivntjhthsfrtmkpnrtwermybkvbymg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jfawxvtzgyictdlgttwdlopgiglhuvcs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxzshyxxgdufbwtpeqphuuqgqykvuorj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"divgxsjwdiamogpvwhppjaobrcszjwuu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"viypyzxdnvpgecgcuibtnurzgwljdday","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijfbjxsnwlphogsabumkevmndxdloziz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dnyykufgwclhjvhrvsnjdnikjwzrsllk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khenkiogedabnpteczlncbjcjkciqobf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wonopsmmvrgdwzjmflwuctusizicfpbo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omirkuuftvwejbprgevknxqktrhnvqfb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iyxjplihdjwsenxpvggecxsvnxgvyqvh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ilufxgthblamhxdzjqyhboyrjywilgwh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgfwtrvfgitvixyznblmhfoamxoxghuy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pljgmohgmbltagdrwntfcmwybstzyoll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twvjhpdzwvzjcqhnzjvbtytxestqddzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hddfqewxewbpzowlgqwqlukhubocyaik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jubgdrbncyxeejxfqnynyrkrixzstoaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtlqcbvuzfgnfmvxzzspourqfrrnmhnx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqsjfynxpyqevfkvsllxdjxmvrnvmdmz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prifcpwgygvbzgbtvmpcqcupmimpxsob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohgcztunyttmamkopblnfowyqhuuamrp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zuuubifhkcgqwjmxwxyffoxwgiocypam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgfeldywhseyvqpnjkmgacinrvbpyazf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ulgnqxknmzkjljkyhgqecxhecmtmpgof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ngoqbasviuxfnftiklakspontqocirin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jyehhtbwpwejjoqsvjtqhvwlijiqxzhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lxcpvsyjhnmqbdehfqpdfnuyztwlkycy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lghqukjwyreeujspufruhhnlqbfuomdt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"apcnjcpwsnrrwvrbrvofgirrvefyndrk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"epmjrocrbegoxpkctkvaxphqzqgwprgh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jqormjkhmtejgurpymisrqivvutpzprb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cywsyhhmcarysufrlqzidgwzobhidcdx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghamxuqcfokaidcxjgvptrpehyuywgsg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpmnrvmrrpdkmnkhnzorkhfxctwzvygc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fiaymgdfyggaawzkehccbhznmtgkgkcy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"piikfhykuswnisjcfwrrrxxdnjuebjey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jqfgdfqhijzwrtsmijaivapfwyakrpgt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"viorzfjybjftvpjdbywlyddgoomynrbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzhfqtcetlwvhlkfjcqcqxlgrnpbshqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"luwvupdlwiptgonrkoqyrjqbjhdcjbuz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wztavsijsmrjjqicugysmytcmlxhfves","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fdqvyqqjlvukwwzzxzvjodlevfyeorcj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"socapfxhgsgxxtgnvznhoearsfdeokch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sthyfztpkdlquctjmuraxtpbnrirzdzr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xlgkgeqxwsanmuobabpkoblcpfbkenht","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umcemgrfsssbvwubqgiipvaywrgjcmlm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsrtcjkfhnpyuynzfvgnnzdlqgrjlpxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvcbtmkesgtegwuzuwrzzajsbyfwzsts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmzrrmdwdnrrkdiqtonmosbiiynedoqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"soheraibcqqhliryhgwpccxthguxgvsn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xakuxbwajlgxokbxmozxzdvdnuxxjpxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uztxtyzkixopbtkxyudcbnazzibkuhst","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oblsllmhubdovhykjkrwbfcvujpgyypc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exdhhihlerniklnhghpjnomavroemxln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fngrdvxdgqsyeusssyxyasfdotnojsqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdgrygzmhodntpmxpjjjqdobcfpmhuox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oiurppndjpusgclzwquazzjkzalxpjmv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbqrlvkcxddozbgfrbprnvhrzanubwox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672071,"databaseName":"models_schema","ddl":"CREATE TABLE `vhmohjbaalgeckwlqgznnqcvgjavlbzv` (\n `ycqqxdlzqhjijhghfjsidaqoiyrozlgg` int NOT NULL,\n `iknxbcsemjwdvcdkonlrfolzklstnxoj` int DEFAULT NULL,\n `khzkgowyrsespdsiygfvocrpfctyittg` int DEFAULT NULL,\n `sznitnznrivjvnlhrvpslksbvsltzite` int DEFAULT NULL,\n `aezraqqrrfgulwwwjsnbctrcbmzvnduk` int DEFAULT NULL,\n `pztmpfzzpithgdhfytcjeuxdhwsdjjug` int DEFAULT NULL,\n `sytzffpnnutnnaffcogznzbhbbtlfeal` int DEFAULT NULL,\n `stiojllhkqwlkztlqaldqpxsmfzckokb` int DEFAULT NULL,\n `dgzoamxpvdfqujmorofquvfnsxpozzeg` int DEFAULT NULL,\n `brejuxwhgngzfiwvfrqxkukluhtbaiyi` int DEFAULT NULL,\n `zwzdoarhlznwreinykwhmqhbabgdwlpg` int DEFAULT NULL,\n `lahmphgnferjfhzaszxagxtzxyvsrfuc` int DEFAULT NULL,\n `adhvmupufqxkigilntfjizdjplowopwu` int DEFAULT NULL,\n `jghvbchagayhuosvvxwjmtkhdwqdgylb` int DEFAULT NULL,\n `hprvmgavojskskemohpxvtoloraljszd` int DEFAULT NULL,\n `yarzbyprfogdwxctgbhwgqisbdhvndvv` int DEFAULT NULL,\n `mlhoxtbtrumeigcazmhygydjaorqqofs` int DEFAULT NULL,\n `hiszbzvbfpaufqkjxqleacbfckojuojv` int DEFAULT NULL,\n `srcbstifksyrqfzgmbvtlizloappsoiw` int DEFAULT NULL,\n `xmokrngzxdzkkmyoudzbgltqbswizvqw` int DEFAULT NULL,\n `tplpqpagueiraltmurzaumgdqmlgfqie` int DEFAULT NULL,\n `dfacobomtutmtohblcmrdnrorptqaypj` int DEFAULT NULL,\n `klgjbvofzbzqdyfqvmguubtvntwsncwk` int DEFAULT NULL,\n `jqmkaqxidhjzpnnprgtcadqzgnqgguwt` int DEFAULT NULL,\n `ubyerbzvmznzdthfrhnnffusfkqpbhvp` int DEFAULT NULL,\n `ticrcqihlxuhhzrwigrhugyeekgsrxsj` int DEFAULT NULL,\n `ihdwcqwpkrhjugscxruhccvpexgubnwl` int DEFAULT NULL,\n `jthwijcqostuzsjwfphjsyfujfotpujr` int DEFAULT NULL,\n `zhhprqwoyzutppadkzradqrqawnbonln` int DEFAULT NULL,\n `ippqqiphnfneprgaeahcrhocjnlejeof` int DEFAULT NULL,\n `jrfvrinwrkewvlgbagmwbuqwcgqkymvx` int DEFAULT NULL,\n `gngegldiydciqngdcgyocjvkmydysoxb` int DEFAULT NULL,\n `orkfaaejoksepqflymcjblahrpkyogtb` int DEFAULT NULL,\n `iilgjyewgudrbbfnjajloaqncgyihdmt` int DEFAULT NULL,\n `ttjpgdeddktoswxnlfpkwwvygrzcibfi` int DEFAULT NULL,\n `qfwgaaqtihruofeitslddcisjvfkbual` int DEFAULT NULL,\n `kxdwezfocaqpvirwgargnkrfjcsczypg` int DEFAULT NULL,\n `rvdqbbefjhfmvrdjrbtqypcekjcwadkt` int DEFAULT NULL,\n `fatcywocjlafmtgdxhhlaazhhmbyvryx` int DEFAULT NULL,\n `fxmjiqhhaiufiizffjmkifkfbqeigqtv` int DEFAULT NULL,\n `ztgqouocwlfzjmuubxmxxxjcmmzdqjfz` int DEFAULT NULL,\n `ojjkmocskusmxjdmfabqibiqxegzalnm` int DEFAULT NULL,\n `rlrtidnuexatrnallwxjqaucolkebpnn` int DEFAULT NULL,\n `wzntawcvvdzllrmgtppqawkfphlbysnp` int DEFAULT NULL,\n `tvcsynlulubbdgzyxgavjmkwchzraedm` int DEFAULT NULL,\n `kdnbkbmbwogqsygynzcylvmnqgeswntf` int DEFAULT NULL,\n `hihrucbaddtjmfdafdxqmlgomwycfzop` int DEFAULT NULL,\n `jcykpjtpgzkeulmogcnmqjpeyeqcetmi` int DEFAULT NULL,\n `snjhftjzbctayjyzpiagttpgkufegztt` int DEFAULT NULL,\n `qkpmjhujffwgpohpndbnegpwpwrpfsja` int DEFAULT NULL,\n `jlnebggoledmdbvwwevrnruniuqduugs` int DEFAULT NULL,\n `xpvnatgphfbzclrpdjgxudyvenurhhjs` int DEFAULT NULL,\n `urbwcpgbbqtkmrydzfiijgdtflfbhuup` int DEFAULT NULL,\n `ecqtzaedurbgabhxfkjhrknajnpjszyo` int DEFAULT NULL,\n `tikytruentgnehnrbkmcfeocrlfrkzgl` int DEFAULT NULL,\n `jadbsjdrljoisoiwqxmjylccyfnqnaan` int DEFAULT NULL,\n `khrqeatknocwzzerdyyozlygxxrdnmwy` int DEFAULT NULL,\n `qwnmjdzszqlzivbnnttggsucymjiulei` int DEFAULT NULL,\n `nrssbieulmnlluxmusmzbttqftwlgxrl` int DEFAULT NULL,\n `egntfjqwgylfqiaxsjnjphcaujocuzqy` int DEFAULT NULL,\n `dbgsooansolmrqazeziylxfhssestafo` int DEFAULT NULL,\n `ojwtsosisgpykwapqfhttqeyxsshjdcz` int DEFAULT NULL,\n `znrikubmnimpxthewaeepbtcsjdapsrc` int DEFAULT NULL,\n `awghhxzakstpgyfcmwpnobtquhdspngj` int DEFAULT NULL,\n `esazlewyjzsqzfqmizowlvmvfvktrquo` int DEFAULT NULL,\n `gfydwjqvcrkfqdpoykkxrtnutzbansxs` int DEFAULT NULL,\n `nffgepuiivgbraopbwnlwhhltmrisyqc` int DEFAULT NULL,\n `mxdjbetneewlotcuenduvhozvlhadmiu` int DEFAULT NULL,\n `fsdosdmqfaydxvvngdugefhjeowugupe` int DEFAULT NULL,\n `rqhghebeonuskbnidqtobpyakyumqzjj` int DEFAULT NULL,\n `yaueizynvuenzribrikslvnbvaauudxl` int DEFAULT NULL,\n `jnjxieuuenltxrhporrgjlynbnlljmcn` int DEFAULT NULL,\n `aidjhztisajxharycbkicvpqfhcsauyi` int DEFAULT NULL,\n `xmtuvmejjnyorfsgoqfblzzuleuxcwjd` int DEFAULT NULL,\n `lkrwyuzdqkkakkxgfssgrbvhwcvhnuas` int DEFAULT NULL,\n `vvluhfpwfdmnjakktvvjfgqmamdfpsoi` int DEFAULT NULL,\n `tgzymzdrwmzfhsnxwqvkvmzrnbyjewwl` int DEFAULT NULL,\n `lofqebxrlrosztpivvfaehblrpintmsh` int DEFAULT NULL,\n `afmekrpjvssodgjoyrmffqugoqvcqxke` int DEFAULT NULL,\n `inurndzzisqvixhsvfrzzkoelcapfvwi` int DEFAULT NULL,\n `kumcwfzyutfwezggpzkyrjnyshfxdnvb` int DEFAULT NULL,\n `hlefaekyqpqluctaeecngshgldzdbfwp` int DEFAULT NULL,\n `xamunxnkgmechyspjtqgmmupsfvwsqzj` int DEFAULT NULL,\n `clxnbqblmghfvzkozmcdwtwkxabgsdps` int DEFAULT NULL,\n `sjowzblakckimjzxoodyhlbxikeslxak` int DEFAULT NULL,\n `ieiphzuddsruabjkaxyzpgmhdsfzniqd` int DEFAULT NULL,\n `qrvinznjxhczjdidlursjcqrdlmnnerr` int DEFAULT NULL,\n `pvgukpaiimarvcmxzsniyuzlgajkciwk` int DEFAULT NULL,\n `xeujsmxbsgptbjovoyhhxfuujklbypch` int DEFAULT NULL,\n `vvrfxhjepslmcjigzgmumcbhofdyuivn` int DEFAULT NULL,\n `xdbbakiwedmabtappodntyxoepdxrnxt` int DEFAULT NULL,\n `zrlyxotclfbpnatmrzkjsiuvtxnkgiao` int DEFAULT NULL,\n `hwtdtsoqhinopzvyodiceflbxxbinmek` int DEFAULT NULL,\n `zskqfccqnqhanbiynybljrwzuinyjxiq` int DEFAULT NULL,\n `dmfnssuzedqrzsdqbpwxkbtqnitcetko` int DEFAULT NULL,\n `liwpjcasmudkgnzayhxugezxgkvbmgsr` int DEFAULT NULL,\n `ljnndfkkcxcqzfjxkbenetakckemfceg` int DEFAULT NULL,\n `zzptuqdwqsxrgjjdvwqpbvxmrcvcgqqz` int DEFAULT NULL,\n `dzaqrdxneyqpdpskpclimicqqifjjofc` int DEFAULT NULL,\n `lqdyxuztebnfeehpizneumtdqnnyvfiu` int DEFAULT NULL,\n PRIMARY KEY (`ycqqxdlzqhjijhghfjsidaqoiyrozlgg`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"vhmohjbaalgeckwlqgznnqcvgjavlbzv\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["ycqqxdlzqhjijhghfjsidaqoiyrozlgg"],"columns":[{"name":"ycqqxdlzqhjijhghfjsidaqoiyrozlgg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"iknxbcsemjwdvcdkonlrfolzklstnxoj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khzkgowyrsespdsiygfvocrpfctyittg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sznitnznrivjvnlhrvpslksbvsltzite","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aezraqqrrfgulwwwjsnbctrcbmzvnduk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pztmpfzzpithgdhfytcjeuxdhwsdjjug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sytzffpnnutnnaffcogznzbhbbtlfeal","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"stiojllhkqwlkztlqaldqpxsmfzckokb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgzoamxpvdfqujmorofquvfnsxpozzeg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"brejuxwhgngzfiwvfrqxkukluhtbaiyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zwzdoarhlznwreinykwhmqhbabgdwlpg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lahmphgnferjfhzaszxagxtzxyvsrfuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"adhvmupufqxkigilntfjizdjplowopwu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jghvbchagayhuosvvxwjmtkhdwqdgylb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hprvmgavojskskemohpxvtoloraljszd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yarzbyprfogdwxctgbhwgqisbdhvndvv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlhoxtbtrumeigcazmhygydjaorqqofs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hiszbzvbfpaufqkjxqleacbfckojuojv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"srcbstifksyrqfzgmbvtlizloappsoiw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmokrngzxdzkkmyoudzbgltqbswizvqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tplpqpagueiraltmurzaumgdqmlgfqie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dfacobomtutmtohblcmrdnrorptqaypj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"klgjbvofzbzqdyfqvmguubtvntwsncwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jqmkaqxidhjzpnnprgtcadqzgnqgguwt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubyerbzvmznzdthfrhnnffusfkqpbhvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ticrcqihlxuhhzrwigrhugyeekgsrxsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ihdwcqwpkrhjugscxruhccvpexgubnwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jthwijcqostuzsjwfphjsyfujfotpujr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhhprqwoyzutppadkzradqrqawnbonln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ippqqiphnfneprgaeahcrhocjnlejeof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jrfvrinwrkewvlgbagmwbuqwcgqkymvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gngegldiydciqngdcgyocjvkmydysoxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orkfaaejoksepqflymcjblahrpkyogtb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iilgjyewgudrbbfnjajloaqncgyihdmt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ttjpgdeddktoswxnlfpkwwvygrzcibfi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfwgaaqtihruofeitslddcisjvfkbual","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxdwezfocaqpvirwgargnkrfjcsczypg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rvdqbbefjhfmvrdjrbtqypcekjcwadkt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fatcywocjlafmtgdxhhlaazhhmbyvryx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxmjiqhhaiufiizffjmkifkfbqeigqtv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ztgqouocwlfzjmuubxmxxxjcmmzdqjfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojjkmocskusmxjdmfabqibiqxegzalnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlrtidnuexatrnallwxjqaucolkebpnn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzntawcvvdzllrmgtppqawkfphlbysnp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvcsynlulubbdgzyxgavjmkwchzraedm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kdnbkbmbwogqsygynzcylvmnqgeswntf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hihrucbaddtjmfdafdxqmlgomwycfzop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jcykpjtpgzkeulmogcnmqjpeyeqcetmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"snjhftjzbctayjyzpiagttpgkufegztt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qkpmjhujffwgpohpndbnegpwpwrpfsja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlnebggoledmdbvwwevrnruniuqduugs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpvnatgphfbzclrpdjgxudyvenurhhjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urbwcpgbbqtkmrydzfiijgdtflfbhuup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecqtzaedurbgabhxfkjhrknajnpjszyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tikytruentgnehnrbkmcfeocrlfrkzgl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jadbsjdrljoisoiwqxmjylccyfnqnaan","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"khrqeatknocwzzerdyyozlygxxrdnmwy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwnmjdzszqlzivbnnttggsucymjiulei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nrssbieulmnlluxmusmzbttqftwlgxrl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"egntfjqwgylfqiaxsjnjphcaujocuzqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbgsooansolmrqazeziylxfhssestafo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojwtsosisgpykwapqfhttqeyxsshjdcz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znrikubmnimpxthewaeepbtcsjdapsrc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awghhxzakstpgyfcmwpnobtquhdspngj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"esazlewyjzsqzfqmizowlvmvfvktrquo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfydwjqvcrkfqdpoykkxrtnutzbansxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nffgepuiivgbraopbwnlwhhltmrisyqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mxdjbetneewlotcuenduvhozvlhadmiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fsdosdmqfaydxvvngdugefhjeowugupe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqhghebeonuskbnidqtobpyakyumqzjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yaueizynvuenzribrikslvnbvaauudxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jnjxieuuenltxrhporrgjlynbnlljmcn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aidjhztisajxharycbkicvpqfhcsauyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmtuvmejjnyorfsgoqfblzzuleuxcwjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lkrwyuzdqkkakkxgfssgrbvhwcvhnuas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vvluhfpwfdmnjakktvvjfgqmamdfpsoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tgzymzdrwmzfhsnxwqvkvmzrnbyjewwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lofqebxrlrosztpivvfaehblrpintmsh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afmekrpjvssodgjoyrmffqugoqvcqxke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"inurndzzisqvixhsvfrzzkoelcapfvwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kumcwfzyutfwezggpzkyrjnyshfxdnvb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlefaekyqpqluctaeecngshgldzdbfwp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xamunxnkgmechyspjtqgmmupsfvwsqzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clxnbqblmghfvzkozmcdwtwkxabgsdps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjowzblakckimjzxoodyhlbxikeslxak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ieiphzuddsruabjkaxyzpgmhdsfzniqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrvinznjxhczjdidlursjcqrdlmnnerr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvgukpaiimarvcmxzsniyuzlgajkciwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xeujsmxbsgptbjovoyhhxfuujklbypch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vvrfxhjepslmcjigzgmumcbhofdyuivn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdbbakiwedmabtappodntyxoepdxrnxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zrlyxotclfbpnatmrzkjsiuvtxnkgiao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwtdtsoqhinopzvyodiceflbxxbinmek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zskqfccqnqhanbiynybljrwzuinyjxiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dmfnssuzedqrzsdqbpwxkbtqnitcetko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"liwpjcasmudkgnzayhxugezxgkvbmgsr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljnndfkkcxcqzfjxkbenetakckemfceg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzptuqdwqsxrgjjdvwqpbvxmrcvcgqqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dzaqrdxneyqpdpskpclimicqqifjjofc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqdyxuztebnfeehpizneumtdqnnyvfiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672101,"databaseName":"models_schema","ddl":"CREATE TABLE `vlatilsnjpcfvjqannnvxnfclsppggel` (\n `rxbsitczrikwngtubbuaenvyjeggzybc` int NOT NULL,\n `tmhpscxmjlgggznvqzwytosumojmwwph` int DEFAULT NULL,\n `qtccczfkxihivelxeqkbvsqswvjlajrk` int DEFAULT NULL,\n `dsicsltnoahiiczepnybnrrfneghktrv` int DEFAULT NULL,\n `oyncqysawdavufxzgblaqbwuiumatrjg` int DEFAULT NULL,\n `wkgcvnhlyuifxirafdrxxitjsccsnamw` int DEFAULT NULL,\n `qqlceaptwzoxvmvzckmhisbfvjsbaoog` int DEFAULT NULL,\n `cansmkmjiidhvpeiynroejlrcqblfnnh` int DEFAULT NULL,\n `uiehsjfnjcydgqmkcmgtivifgghqrmav` int DEFAULT NULL,\n `foqevvfizxbiwdawzhthirbgpddiqvjq` int DEFAULT NULL,\n `pgfbtbgwhxgzowmweccqtihbloydicxw` int DEFAULT NULL,\n `geikhgpqkcixmeezcbyuqvjnlpylkene` int DEFAULT NULL,\n `jcqolgopnvxdwojgkmgniijnxbpnfrbm` int DEFAULT NULL,\n `ddcisixmjejoqwmmvryaiolnkpncqwef` int DEFAULT NULL,\n `lweuvuqgiueliymmhdfsjsdzxukxlzad` int DEFAULT NULL,\n `vdalcpbuidpfwwecbpvbnafaqtgjfobx` int DEFAULT NULL,\n `jpuktzximzdripzzojfjmvgvwuyqjnpl` int DEFAULT NULL,\n `nrvcgxpimozwtjjckmblcorgsijqhpvn` int DEFAULT NULL,\n `hkkhbjdyqzhjlqdtfilzjvjyrknsxdcf` int DEFAULT NULL,\n `totrhiodeapmxsxzefnlauhgbqiuqcmh` int DEFAULT NULL,\n `tsaujimmqqjykbwjjslqnbcbybyvjpde` int DEFAULT NULL,\n `fnrkvgprusyvnfiklapjqkvfgnvlqgyo` int DEFAULT NULL,\n `dxyyzizguskjxtxplewnhrmaafzbrrex` int DEFAULT NULL,\n `zbblujduxobfhjlgzeodsshabtjcymvk` int DEFAULT NULL,\n `gsyhectprvicbuwiifkxrtmwwukhzhui` int DEFAULT NULL,\n `wrfowlbqmegsotrnjqbduuseusqdtbqu` int DEFAULT NULL,\n `bpgvzjlohvqrehmigvibtqysazzveeyx` int DEFAULT NULL,\n `qoydfvywewiptkzvpoficyglipgznutc` int DEFAULT NULL,\n `djfztszdafrdrhzomnrgmswnqkdmcewo` int DEFAULT NULL,\n `tfdlvzxcecbpkykmrssvlgdusidginjs` int DEFAULT NULL,\n `gvbyveeusfjuthnbqprjjttwkpkgfiji` int DEFAULT NULL,\n `efvksubcmyvcielyihycxsnynncsafsj` int DEFAULT NULL,\n `qgcriztlvogoytutxjobgxevvmgbtgir` int DEFAULT NULL,\n `sacxzlkaqbftzfbflemjsqzmtxykfsed` int DEFAULT NULL,\n `vlyuffikrozxzyofraayuiwfriyhytxo` int DEFAULT NULL,\n `suykfbucfandlsqdosexgcgtbawvynif` int DEFAULT NULL,\n `zjrgrxgbyqakjbhbzxnmgwyshxbhuoyr` int DEFAULT NULL,\n `jrarauxqwcyborvpjbvvtooqxjziofpt` int DEFAULT NULL,\n `bzszxijxxujxqbyvirmucnyigklyxmkp` int DEFAULT NULL,\n `lusncvhqemkbuwgvsozzgrybitzgrrul` int DEFAULT NULL,\n `ljbehbaotpoyikwifwhkabjpsjivwqti` int DEFAULT NULL,\n `rfqbwisfthzzfvxpdngmgprilnfonzyg` int DEFAULT NULL,\n `vsgnthtimunwmlxuqeoeihnykqrbafle` int DEFAULT NULL,\n `dvqewfewmblwsglgjiixpgaqhnhjzica` int DEFAULT NULL,\n `zyvmdqpugmvtklzfdipnhgejxzykpmim` int DEFAULT NULL,\n `datrqsvafkvhvjebplbxvppeghntpstd` int DEFAULT NULL,\n `tulmkdgjobyrklcipotpmsfmmhmpjvpk` int DEFAULT NULL,\n `xvtyzqlfjchpylxmnswamkwvsodbkupb` int DEFAULT NULL,\n `cbunjwdaancuavukmdrketgzjkczsvsj` int DEFAULT NULL,\n `ikumnrmtidbypjgnhbqkyobehmlvjcft` int DEFAULT NULL,\n `uapydjeevnakculpxdbqxbuqsbnwdoil` int DEFAULT NULL,\n `jifkogrfxdxniiyrigwuflhkwicvfrbt` int DEFAULT NULL,\n `taixtgiysdgfdmaqrehpvdvhtdodcueh` int DEFAULT NULL,\n `novvmfafykfjbdiwbwieavynmmzlepag` int DEFAULT NULL,\n `anavcyrnsbzbnzsllapqebasrbgpdjqq` int DEFAULT NULL,\n `wxkqumtfovkkwatsazctxtvhxfyfrzcy` int DEFAULT NULL,\n `yynvkimexdzjsafmzzaaacgmrycdqoyr` int DEFAULT NULL,\n `mcrdmzjtqsbdcisreaulwontxwmikwmo` int DEFAULT NULL,\n `esknnolntqxealdgpudxeodamymfvays` int DEFAULT NULL,\n `xdnckluahrvlziuxqwkdrnqimmatnmwg` int DEFAULT NULL,\n `wujiwydobimftbipttxljknmjiwqsysj` int DEFAULT NULL,\n `ryoathkujemplpdknnfiupkuzvdumdvo` int DEFAULT NULL,\n `vlkbgputkowzqhqvnxhwkxqyyhjmmnqz` int DEFAULT NULL,\n `tgysynnbvbnxczznkmzjahxhdbcfxxqo` int DEFAULT NULL,\n `seiqhinogxpnztjvxokkfihjdwahbpoo` int DEFAULT NULL,\n `wzdjwaxeqiozbtcepiqjejlulkdrrgfl` int DEFAULT NULL,\n `kdulhobstflxopogvvgjgszmcfxazibq` int DEFAULT NULL,\n `gdwlprhiuujltzhlpbxhmeqxhwkaovqa` int DEFAULT NULL,\n `pqzeondqmihjznvwwcaoppmpuukonquj` int DEFAULT NULL,\n `tnoxuqxpzmnuzyubmzfvviinnktgkidc` int DEFAULT NULL,\n `jqflgvyvwwxbjpluybbayyniotiyzupp` int DEFAULT NULL,\n `sokckohyonckhgaeacsfuvjgwshazhds` int DEFAULT NULL,\n `ulfdtasjkqjbypgxbbymphogdwhlqqsi` int DEFAULT NULL,\n `phpzgfbrpfchfgwgwqhqdguarfouzchn` int DEFAULT NULL,\n `qhvxojtgtvpyvitdyckxekkqoynatktt` int DEFAULT NULL,\n `glrpheitggjxqbqrohpioaqoatqwiqvx` int DEFAULT NULL,\n `ovvvptuwpluovroeqchlczieszizwxbr` int DEFAULT NULL,\n `tfjnhmmhfxdjsnxepievylvpxsrvluup` int DEFAULT NULL,\n `izuxktgmraqpfawelmrjypkwmvmpsyyu` int DEFAULT NULL,\n `onnxvjzatneuydhvratfvgmqpippmgiq` int DEFAULT NULL,\n `mkwxldtwpyueocwcniyzbgezqcrdoubp` int DEFAULT NULL,\n `abapjshskwauokfnwyddmowyoogeunmn` int DEFAULT NULL,\n `snboysotpynankkgrblqodwcuiebeaqa` int DEFAULT NULL,\n `rxoavrnelhtyxolvjenxkeviayubqlbp` int DEFAULT NULL,\n `cdcyxemouebnreqimrhelgbnhbgfmnoe` int DEFAULT NULL,\n `tpotyofnoivfoopcfkjfjggbpufybbnk` int DEFAULT NULL,\n `bkovppineltokvimijmayzaimbnzozmc` int DEFAULT NULL,\n `rlftwmyypcgimqchppslxqqzexkwfcbv` int DEFAULT NULL,\n `rumfofflfgopbqistwobujvssdlyytat` int DEFAULT NULL,\n `unfmvuvimkhlktdagebbmlibwzgwfqrk` int DEFAULT NULL,\n `bxcbboqnpsdrwngduuovfyswiawqlcln` int DEFAULT NULL,\n `ikdiltthuonumyeopjpgqrozyegoklsj` int DEFAULT NULL,\n `pmktwypdvojwoybnsouhqkpjgsspzorp` int DEFAULT NULL,\n `lgdgnvzbbabickpmxkjavzrtibystnis` int DEFAULT NULL,\n `mmqdfbtokjowiyxycpwzdjzbqpaiiymq` int DEFAULT NULL,\n `ezcmtxonyukyhvwdtbrupbitbwjfwkdm` int DEFAULT NULL,\n `zaddvqunogkbhbewltjmwmyinhjcpfsm` int DEFAULT NULL,\n `qishhnjgoprywzbqbmtxqkjwfjxsnoxq` int DEFAULT NULL,\n `onnaqkvhamsszksovcwlngtolysazywr` int DEFAULT NULL,\n `jvyqlrigchurhsrqdxwckdyllbfbkqkl` int DEFAULT NULL,\n PRIMARY KEY (`rxbsitczrikwngtubbuaenvyjeggzybc`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"vlatilsnjpcfvjqannnvxnfclsppggel\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["rxbsitczrikwngtubbuaenvyjeggzybc"],"columns":[{"name":"rxbsitczrikwngtubbuaenvyjeggzybc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"tmhpscxmjlgggznvqzwytosumojmwwph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtccczfkxihivelxeqkbvsqswvjlajrk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsicsltnoahiiczepnybnrrfneghktrv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oyncqysawdavufxzgblaqbwuiumatrjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wkgcvnhlyuifxirafdrxxitjsccsnamw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqlceaptwzoxvmvzckmhisbfvjsbaoog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cansmkmjiidhvpeiynroejlrcqblfnnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uiehsjfnjcydgqmkcmgtivifgghqrmav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"foqevvfizxbiwdawzhthirbgpddiqvjq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgfbtbgwhxgzowmweccqtihbloydicxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"geikhgpqkcixmeezcbyuqvjnlpylkene","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jcqolgopnvxdwojgkmgniijnxbpnfrbm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddcisixmjejoqwmmvryaiolnkpncqwef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lweuvuqgiueliymmhdfsjsdzxukxlzad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdalcpbuidpfwwecbpvbnafaqtgjfobx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpuktzximzdripzzojfjmvgvwuyqjnpl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nrvcgxpimozwtjjckmblcorgsijqhpvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkkhbjdyqzhjlqdtfilzjvjyrknsxdcf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"totrhiodeapmxsxzefnlauhgbqiuqcmh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tsaujimmqqjykbwjjslqnbcbybyvjpde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnrkvgprusyvnfiklapjqkvfgnvlqgyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxyyzizguskjxtxplewnhrmaafzbrrex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbblujduxobfhjlgzeodsshabtjcymvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsyhectprvicbuwiifkxrtmwwukhzhui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wrfowlbqmegsotrnjqbduuseusqdtbqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bpgvzjlohvqrehmigvibtqysazzveeyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qoydfvywewiptkzvpoficyglipgznutc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"djfztszdafrdrhzomnrgmswnqkdmcewo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tfdlvzxcecbpkykmrssvlgdusidginjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gvbyveeusfjuthnbqprjjttwkpkgfiji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"efvksubcmyvcielyihycxsnynncsafsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgcriztlvogoytutxjobgxevvmgbtgir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sacxzlkaqbftzfbflemjsqzmtxykfsed","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vlyuffikrozxzyofraayuiwfriyhytxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suykfbucfandlsqdosexgcgtbawvynif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjrgrxgbyqakjbhbzxnmgwyshxbhuoyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jrarauxqwcyborvpjbvvtooqxjziofpt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzszxijxxujxqbyvirmucnyigklyxmkp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lusncvhqemkbuwgvsozzgrybitzgrrul","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljbehbaotpoyikwifwhkabjpsjivwqti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfqbwisfthzzfvxpdngmgprilnfonzyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsgnthtimunwmlxuqeoeihnykqrbafle","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvqewfewmblwsglgjiixpgaqhnhjzica","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyvmdqpugmvtklzfdipnhgejxzykpmim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"datrqsvafkvhvjebplbxvppeghntpstd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tulmkdgjobyrklcipotpmsfmmhmpjvpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvtyzqlfjchpylxmnswamkwvsodbkupb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbunjwdaancuavukmdrketgzjkczsvsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikumnrmtidbypjgnhbqkyobehmlvjcft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uapydjeevnakculpxdbqxbuqsbnwdoil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jifkogrfxdxniiyrigwuflhkwicvfrbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"taixtgiysdgfdmaqrehpvdvhtdodcueh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"novvmfafykfjbdiwbwieavynmmzlepag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"anavcyrnsbzbnzsllapqebasrbgpdjqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxkqumtfovkkwatsazctxtvhxfyfrzcy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yynvkimexdzjsafmzzaaacgmrycdqoyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcrdmzjtqsbdcisreaulwontxwmikwmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"esknnolntqxealdgpudxeodamymfvays","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdnckluahrvlziuxqwkdrnqimmatnmwg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wujiwydobimftbipttxljknmjiwqsysj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ryoathkujemplpdknnfiupkuzvdumdvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vlkbgputkowzqhqvnxhwkxqyyhjmmnqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tgysynnbvbnxczznkmzjahxhdbcfxxqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"seiqhinogxpnztjvxokkfihjdwahbpoo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzdjwaxeqiozbtcepiqjejlulkdrrgfl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kdulhobstflxopogvvgjgszmcfxazibq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdwlprhiuujltzhlpbxhmeqxhwkaovqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqzeondqmihjznvwwcaoppmpuukonquj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnoxuqxpzmnuzyubmzfvviinnktgkidc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jqflgvyvwwxbjpluybbayyniotiyzupp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sokckohyonckhgaeacsfuvjgwshazhds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ulfdtasjkqjbypgxbbymphogdwhlqqsi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phpzgfbrpfchfgwgwqhqdguarfouzchn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhvxojtgtvpyvitdyckxekkqoynatktt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"glrpheitggjxqbqrohpioaqoatqwiqvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovvvptuwpluovroeqchlczieszizwxbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tfjnhmmhfxdjsnxepievylvpxsrvluup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"izuxktgmraqpfawelmrjypkwmvmpsyyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"onnxvjzatneuydhvratfvgmqpippmgiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkwxldtwpyueocwcniyzbgezqcrdoubp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abapjshskwauokfnwyddmowyoogeunmn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"snboysotpynankkgrblqodwcuiebeaqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxoavrnelhtyxolvjenxkeviayubqlbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cdcyxemouebnreqimrhelgbnhbgfmnoe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpotyofnoivfoopcfkjfjggbpufybbnk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkovppineltokvimijmayzaimbnzozmc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlftwmyypcgimqchppslxqqzexkwfcbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rumfofflfgopbqistwobujvssdlyytat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unfmvuvimkhlktdagebbmlibwzgwfqrk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bxcbboqnpsdrwngduuovfyswiawqlcln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikdiltthuonumyeopjpgqrozyegoklsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmktwypdvojwoybnsouhqkpjgsspzorp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgdgnvzbbabickpmxkjavzrtibystnis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmqdfbtokjowiyxycpwzdjzbqpaiiymq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezcmtxonyukyhvwdtbrupbitbwjfwkdm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zaddvqunogkbhbewltjmwmyinhjcpfsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qishhnjgoprywzbqbmtxqkjwfjxsnoxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"onnaqkvhamsszksovcwlngtolysazywr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvyqlrigchurhsrqdxwckdyllbfbkqkl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672142,"databaseName":"models_schema","ddl":"CREATE TABLE `vldtvucetkkwbkcoweeuxswdsnytbwak` (\n `gbregmwxtoqnlnizeokxzpjiygwzhkiy` int NOT NULL,\n `cqpkprrifvsxfzlqmgxwtlpjcrevrvdq` int DEFAULT NULL,\n `icikjgdhzxthxuvsxqlodfrezitpxzgk` int DEFAULT NULL,\n `wfjadtcchpmxtwhdsqpairvjvdkvgahj` int DEFAULT NULL,\n `zzqntsdvafmipwyopldwtpqulotgqnwm` int DEFAULT NULL,\n `bjgiphklukrhbxtwfgqkdgixegqdifur` int DEFAULT NULL,\n `ibyuvtdweffxxhyvdizeukwtormrwbca` int DEFAULT NULL,\n `fzddxfbtxrqqcftotzzrerahwpoigolj` int DEFAULT NULL,\n `pqqsxkiqzapckgyhkwxnfnrtkjflhohk` int DEFAULT NULL,\n `lbquydydypwkypubrglbbmaxlrwmehkr` int DEFAULT NULL,\n `drcfwdpnvmnwxhfzwugxhdttmiwrcbxy` int DEFAULT NULL,\n `xkdizrshrgbfuryphnngizpmpczwvbcl` int DEFAULT NULL,\n `stkulzdyngfwtnevckkgtckcfbsemeoi` int DEFAULT NULL,\n `rikodnndsqvzxhifppymqsvywsuckfag` int DEFAULT NULL,\n `tzznywzgrwglvauqhnpehphxzobpsxen` int DEFAULT NULL,\n `itvwqwdjcmzozhiovgpyrgawjmtccfpc` int DEFAULT NULL,\n `kuwwhjtfrafvwlqwtbivrjwkibsgnlbc` int DEFAULT NULL,\n `pinmlaqovsznbudvgzrdmqcgshlcdxnn` int DEFAULT NULL,\n `fhvehscnbltracthbkmjkzbtkwjlbjon` int DEFAULT NULL,\n `qjkulpojwwwwkapylnbjpagpxxptrouo` int DEFAULT NULL,\n `psropdvejltyrugpkaxobxbibsafubcf` int DEFAULT NULL,\n `jorqxxvoldfqcrtjvvntilamqeomtklq` int DEFAULT NULL,\n `ijigccejqwdiicszluvrvjjejwfqqroy` int DEFAULT NULL,\n `iuppzjaphomohmpgbxgdtiyubgjmtzyp` int DEFAULT NULL,\n `pxsqnrnzoxtjksyctopzmtviafdbwyns` int DEFAULT NULL,\n `spntgvopbpwusvsfsgsvkzfepwazmblc` int DEFAULT NULL,\n `hsbkoaeqynvnpucvyvcnetygssxmvpfl` int DEFAULT NULL,\n `elwonnsosusfmqkeqxznhqendyylccpj` int DEFAULT NULL,\n `wexhuyvkqrezgsxzuovthhjegopxlutu` int DEFAULT NULL,\n `jklfchuirvzaskhkuqqdnxwuopyxvgmj` int DEFAULT NULL,\n `dpvoakrwhusdwhzjczeeypcmwanuiuoo` int DEFAULT NULL,\n `twxpgsyxhwrmllzfikuqoqotbtngkmbn` int DEFAULT NULL,\n `njvsukmxqnneqyyxnpchxoxawymuqabr` int DEFAULT NULL,\n `ciwtbumolinkfxessuxrniprplfvmtsm` int DEFAULT NULL,\n `bffndlwjemtmjwdwzqkgcjpjrejmgfxa` int DEFAULT NULL,\n `grsunkurdryzoulyboqhwxqnsijzvhbi` int DEFAULT NULL,\n `nszwjnipdwibjcoweakwcfiaeukkezqm` int DEFAULT NULL,\n `ouzjsvkmbtkvnjatzgovtavvxtxbaldm` int DEFAULT NULL,\n `otouuynkunorjtzkmnnxpvanuzdnsjoo` int DEFAULT NULL,\n `ioftmvbnrqblsjgicmlxoaqayvavgeep` int DEFAULT NULL,\n `dlftwgwfjuyxeyswxnuvtztpmbsxpmsz` int DEFAULT NULL,\n `fpgohlvlzerxohahzvxkqzhaptjrskvp` int DEFAULT NULL,\n `ioanlivbkbbksycuyeizcpryosisutjk` int DEFAULT NULL,\n `blljjsnwdxqosrmofyhrzderzforccdp` int DEFAULT NULL,\n `nzubxgcuocqyynuoxajdohdquibxfvtg` int DEFAULT NULL,\n `mdkifexaurzxyplohnamemdwakmycmas` int DEFAULT NULL,\n `jzjwleiwslcbwtdpmzeavcexvnnphqwp` int DEFAULT NULL,\n `zajsdisnsitqerbuyshezvaeddyeqkyl` int DEFAULT NULL,\n `flhefltoghuzframlpjorwfvjdrvmxgy` int DEFAULT NULL,\n `uayimflqkhzamyddwtnsptgxvhbpcewi` int DEFAULT NULL,\n `vpgsxtixuhaiseupwrjzkznzsxrggred` int DEFAULT NULL,\n `xqgjjilihdlnpylixbptdowpozmdjcmc` int DEFAULT NULL,\n `zegmtopqlrcjctjiytxgybvljjbruxkp` int DEFAULT NULL,\n `xbyxyyaandcnudkmmlvmxylfvcokoeen` int DEFAULT NULL,\n `tbvldmdogjxwsbtmnvpfqecjgvoawuuo` int DEFAULT NULL,\n `slbgrcneslieivwhyikmgbluuiyagsbj` int DEFAULT NULL,\n `fdocagoiysldveadbavwntvpjlkgenmv` int DEFAULT NULL,\n `bykjjoqkkwxogbxnocequskaoejbbzqj` int DEFAULT NULL,\n `dwwsfhpkljrhdfodapzwrxpleouhhgac` int DEFAULT NULL,\n `daerpwhusmubpngueidwsmoaawtnenik` int DEFAULT NULL,\n `wknenscyyvysyjfxbmbejstzvziencbk` int DEFAULT NULL,\n `bldkajqljzeuylfocgtsblpuvvlxadzs` int DEFAULT NULL,\n `czogxthmermolzhxlcjgxspygbmfufni` int DEFAULT NULL,\n `lehwkjdugficolxegrybhaacyerdpzvj` int DEFAULT NULL,\n `ntrfwknnrpwgjuppteejgkibtdumwjrh` int DEFAULT NULL,\n `gulqgiqmzqalvxcckrpwgcenoptaaegk` int DEFAULT NULL,\n `etwdjxnlupkwrxoevxlhidbyzorvnpta` int DEFAULT NULL,\n `cqujvqjxohxvvdmcmdqfurlvbmmrxtct` int DEFAULT NULL,\n `jxddhsshxswjhjswhivewupixetkjmrw` int DEFAULT NULL,\n `usnxhdjxpdmragigetbdozqybhsmxrvi` int DEFAULT NULL,\n `oojvqwuzhglipzllsmqwzcrjwvzmnuqw` int DEFAULT NULL,\n `yudqwwkrutcyunsqbimjedzrdzxmvpaa` int DEFAULT NULL,\n `dnvcmgedbvwrfuetguvqbjsuaxczfgdj` int DEFAULT NULL,\n `urwvhbhzquzeuhqiweuoutcvcxowwhgo` int DEFAULT NULL,\n `afjcxfkcwdatkbqdjerjolmefqblpfsv` int DEFAULT NULL,\n `kxdprtotpptigrujuazfndzcxtadqunx` int DEFAULT NULL,\n `clqamppcxmlmlxrkoodrijxcnvdugemt` int DEFAULT NULL,\n `lnndkvyzbgttqujvxghrhsofdnzxpzic` int DEFAULT NULL,\n `rnwfolebqfuhfwmtodpfkiiywgqoaktm` int DEFAULT NULL,\n `rvagtwpshjcmgjfuosfsgjqaexjmlnfy` int DEFAULT NULL,\n `rpgdtmypllnvftqrbktnceqwvsqpxkca` int DEFAULT NULL,\n `qutrmrykwoctzxmzpwxigyutnipyguyx` int DEFAULT NULL,\n `esnaeitsdsernsdqumacziaruyenczit` int DEFAULT NULL,\n `jqpbjrmzmwtracotgthklcavgoaaynro` int DEFAULT NULL,\n `vdvhphrongtflkaemfxdcwguctadfugz` int DEFAULT NULL,\n `ywvxctmsrvruroetvfmvtmvdvtzfjbsd` int DEFAULT NULL,\n `lkvvwbnvsflanplajkzjpvdbvdhhruju` int DEFAULT NULL,\n `ufeksqgujoeynhyzldazhbzatiujcbpv` int DEFAULT NULL,\n `dkzbbwzepvhtcekwzqjdirtvbwlkwmok` int DEFAULT NULL,\n `aampbsyjcmzibovqaaplnqkyfmpdhpqp` int DEFAULT NULL,\n `iqecosowrngvacunospfefknbkipeoxw` int DEFAULT NULL,\n `tdszivatknybdqdscpsqwsmbnrvuptxi` int DEFAULT NULL,\n `kydsgzgxqpqfiybsmvuqrxoidfvjhnhh` int DEFAULT NULL,\n `jjquqkpteggzptomdolcauimffmfjjyj` int DEFAULT NULL,\n `xjrishkfehqpheppaylbowvgtygouyga` int DEFAULT NULL,\n `pcmxgscxpvoezqtzapkkpjkwrtvxpvat` int DEFAULT NULL,\n `etxgxrirnjkhzlhoxmboixldxocnagip` int DEFAULT NULL,\n `sblelbslyfeasdyqtbxrdyuyfzvjrxef` int DEFAULT NULL,\n `rdvlizrynuiorqhhjurzapftrvdrkbag` int DEFAULT NULL,\n `bxxjaelpflintslzmcwvszqjvjepwmog` int DEFAULT NULL,\n PRIMARY KEY (`gbregmwxtoqnlnizeokxzpjiygwzhkiy`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"vldtvucetkkwbkcoweeuxswdsnytbwak\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["gbregmwxtoqnlnizeokxzpjiygwzhkiy"],"columns":[{"name":"gbregmwxtoqnlnizeokxzpjiygwzhkiy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"cqpkprrifvsxfzlqmgxwtlpjcrevrvdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icikjgdhzxthxuvsxqlodfrezitpxzgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfjadtcchpmxtwhdsqpairvjvdkvgahj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzqntsdvafmipwyopldwtpqulotgqnwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjgiphklukrhbxtwfgqkdgixegqdifur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibyuvtdweffxxhyvdizeukwtormrwbca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzddxfbtxrqqcftotzzrerahwpoigolj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqqsxkiqzapckgyhkwxnfnrtkjflhohk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lbquydydypwkypubrglbbmaxlrwmehkr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drcfwdpnvmnwxhfzwugxhdttmiwrcbxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkdizrshrgbfuryphnngizpmpczwvbcl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"stkulzdyngfwtnevckkgtckcfbsemeoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rikodnndsqvzxhifppymqsvywsuckfag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tzznywzgrwglvauqhnpehphxzobpsxen","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itvwqwdjcmzozhiovgpyrgawjmtccfpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kuwwhjtfrafvwlqwtbivrjwkibsgnlbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pinmlaqovsznbudvgzrdmqcgshlcdxnn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhvehscnbltracthbkmjkzbtkwjlbjon","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjkulpojwwwwkapylnbjpagpxxptrouo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psropdvejltyrugpkaxobxbibsafubcf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jorqxxvoldfqcrtjvvntilamqeomtklq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijigccejqwdiicszluvrvjjejwfqqroy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iuppzjaphomohmpgbxgdtiyubgjmtzyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxsqnrnzoxtjksyctopzmtviafdbwyns","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"spntgvopbpwusvsfsgsvkzfepwazmblc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hsbkoaeqynvnpucvyvcnetygssxmvpfl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elwonnsosusfmqkeqxznhqendyylccpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wexhuyvkqrezgsxzuovthhjegopxlutu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jklfchuirvzaskhkuqqdnxwuopyxvgmj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpvoakrwhusdwhzjczeeypcmwanuiuoo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twxpgsyxhwrmllzfikuqoqotbtngkmbn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njvsukmxqnneqyyxnpchxoxawymuqabr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ciwtbumolinkfxessuxrniprplfvmtsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bffndlwjemtmjwdwzqkgcjpjrejmgfxa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grsunkurdryzoulyboqhwxqnsijzvhbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nszwjnipdwibjcoweakwcfiaeukkezqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ouzjsvkmbtkvnjatzgovtavvxtxbaldm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otouuynkunorjtzkmnnxpvanuzdnsjoo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ioftmvbnrqblsjgicmlxoaqayvavgeep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlftwgwfjuyxeyswxnuvtztpmbsxpmsz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpgohlvlzerxohahzvxkqzhaptjrskvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ioanlivbkbbksycuyeizcpryosisutjk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blljjsnwdxqosrmofyhrzderzforccdp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzubxgcuocqyynuoxajdohdquibxfvtg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdkifexaurzxyplohnamemdwakmycmas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzjwleiwslcbwtdpmzeavcexvnnphqwp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zajsdisnsitqerbuyshezvaeddyeqkyl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flhefltoghuzframlpjorwfvjdrvmxgy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uayimflqkhzamyddwtnsptgxvhbpcewi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpgsxtixuhaiseupwrjzkznzsxrggred","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqgjjilihdlnpylixbptdowpozmdjcmc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zegmtopqlrcjctjiytxgybvljjbruxkp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbyxyyaandcnudkmmlvmxylfvcokoeen","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbvldmdogjxwsbtmnvpfqecjgvoawuuo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"slbgrcneslieivwhyikmgbluuiyagsbj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fdocagoiysldveadbavwntvpjlkgenmv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bykjjoqkkwxogbxnocequskaoejbbzqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwwsfhpkljrhdfodapzwrxpleouhhgac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"daerpwhusmubpngueidwsmoaawtnenik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wknenscyyvysyjfxbmbejstzvziencbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bldkajqljzeuylfocgtsblpuvvlxadzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"czogxthmermolzhxlcjgxspygbmfufni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lehwkjdugficolxegrybhaacyerdpzvj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntrfwknnrpwgjuppteejgkibtdumwjrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gulqgiqmzqalvxcckrpwgcenoptaaegk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"etwdjxnlupkwrxoevxlhidbyzorvnpta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqujvqjxohxvvdmcmdqfurlvbmmrxtct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jxddhsshxswjhjswhivewupixetkjmrw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"usnxhdjxpdmragigetbdozqybhsmxrvi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oojvqwuzhglipzllsmqwzcrjwvzmnuqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yudqwwkrutcyunsqbimjedzrdzxmvpaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dnvcmgedbvwrfuetguvqbjsuaxczfgdj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urwvhbhzquzeuhqiweuoutcvcxowwhgo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afjcxfkcwdatkbqdjerjolmefqblpfsv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxdprtotpptigrujuazfndzcxtadqunx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clqamppcxmlmlxrkoodrijxcnvdugemt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lnndkvyzbgttqujvxghrhsofdnzxpzic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnwfolebqfuhfwmtodpfkiiywgqoaktm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rvagtwpshjcmgjfuosfsgjqaexjmlnfy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpgdtmypllnvftqrbktnceqwvsqpxkca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qutrmrykwoctzxmzpwxigyutnipyguyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"esnaeitsdsernsdqumacziaruyenczit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jqpbjrmzmwtracotgthklcavgoaaynro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdvhphrongtflkaemfxdcwguctadfugz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywvxctmsrvruroetvfmvtmvdvtzfjbsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lkvvwbnvsflanplajkzjpvdbvdhhruju","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufeksqgujoeynhyzldazhbzatiujcbpv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkzbbwzepvhtcekwzqjdirtvbwlkwmok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aampbsyjcmzibovqaaplnqkyfmpdhpqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqecosowrngvacunospfefknbkipeoxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdszivatknybdqdscpsqwsmbnrvuptxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kydsgzgxqpqfiybsmvuqrxoidfvjhnhh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjquqkpteggzptomdolcauimffmfjjyj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjrishkfehqpheppaylbowvgtygouyga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pcmxgscxpvoezqtzapkkpjkwrtvxpvat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"etxgxrirnjkhzlhoxmboixldxocnagip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sblelbslyfeasdyqtbxrdyuyfzvjrxef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdvlizrynuiorqhhjurzapftrvdrkbag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bxxjaelpflintslzmcwvszqjvjepwmog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672174,"databaseName":"models_schema","ddl":"CREATE TABLE `vqxcfvfvkfrjtrlfeabxtnndjgrileqz` (\n `ihtdicgwfiginzjsvmlofoiaovagtgzo` int NOT NULL,\n `opjkjabmhyozienpwqcfxoruqcqlnbob` int DEFAULT NULL,\n `prxbjlqyuvgxaljfenojaehonmwtbyeu` int DEFAULT NULL,\n `hqomrjajvdosajkcfptwlnyatggauxoe` int DEFAULT NULL,\n `samxnkcxrhknloyfflndbygbrhzpoqye` int DEFAULT NULL,\n `rxrvmszspegvfrwwuxpffejpykfgkeyk` int DEFAULT NULL,\n `tkvslkuqncrxegzekwnqwdplvzxvrwcv` int DEFAULT NULL,\n `lcaiwudzmzjbkksvbzxsehubswtsarmz` int DEFAULT NULL,\n `exqptjlremwaodlwjczgvmorgylkhxrl` int DEFAULT NULL,\n `xidkuqfalhowsvwmjyqpbsdgegyjxlgk` int DEFAULT NULL,\n `csunvntsqzaydobxskyuxuzvxnfiiupe` int DEFAULT NULL,\n `okwzwwgdokeuysjbtrtbzdwoixzwaxem` int DEFAULT NULL,\n `vkxxbornffrazmqgnkpbwgoluqhugzzy` int DEFAULT NULL,\n `tipnypzettcheiozolcqfulwyodccmbo` int DEFAULT NULL,\n `xmqabqyguzgpuseuwikyayzzekpkixzi` int DEFAULT NULL,\n `mrmegkaxjyjpxesmpfbffxbncbhhxseb` int DEFAULT NULL,\n `hudotngmibneskjzkaxkvywjoerxxfic` int DEFAULT NULL,\n `dpvjfhdpxyxlxuclldftrmwdqxrulpnq` int DEFAULT NULL,\n `gnthdinqjmfhoyisykwelvzddnaooygb` int DEFAULT NULL,\n `zmaonqvzrshmaajptleomloyodooaalb` int DEFAULT NULL,\n `xjsxnlvifgtyggtyezhpryibugnwkmfu` int DEFAULT NULL,\n `siuqawpymggbqfwgsdktktoyevxiqpes` int DEFAULT NULL,\n `romxfcxdubyclxtlkxgihjhnxrhzvnan` int DEFAULT NULL,\n `efmoujdsfuxuvpjtawrbqijukysprgax` int DEFAULT NULL,\n `wpetfpfnfasyuiqotbpunubttezldkja` int DEFAULT NULL,\n `yajihgppmzcrdqhakkknqjsqpxxvxnww` int DEFAULT NULL,\n `fczmsoxssnkjvzzeuzxntranxnnpsoak` int DEFAULT NULL,\n `qzpuxpwuabalktjmhzjjgifayvmysgno` int DEFAULT NULL,\n `xpspebfnrhzkzyzsytxznzzxojshfmci` int DEFAULT NULL,\n `duljnumtelgtngkxgktyknxgxceksaie` int DEFAULT NULL,\n `kquawkcgonhmoptowallcxbqpzarrnhf` int DEFAULT NULL,\n `trycfhfvmiajvjrnfoputzxrjyfzcnbr` int DEFAULT NULL,\n `stqgnfplhncirriljpkyqcfgpsptlgkx` int DEFAULT NULL,\n `rxldhqzsdqoqrusnvxbeemwjmbeuqkro` int DEFAULT NULL,\n `ufbkpcfsviaidobrjzuytrmqprfpxqlz` int DEFAULT NULL,\n `ruzlllikhvuixxnwqtjiohlhmfzugrtg` int DEFAULT NULL,\n `hzumdexfwdfkoulqhqjgygvowlfjnbim` int DEFAULT NULL,\n `xtnezsfbgaliylsjygysqpjqxrjnpvgp` int DEFAULT NULL,\n `rpspdmihmlhfysbtiyrbuyholcophhdz` int DEFAULT NULL,\n `icrzuchoogrtyyfavjkgdojunwswxiyj` int DEFAULT NULL,\n `mnjksyxagjixpoucvugtlpdtakspnolr` int DEFAULT NULL,\n `vdanfgchitqzahnoowynaxyadnbrvicf` int DEFAULT NULL,\n `zlxinjdnohhvwurgzirxfmongcvmbisk` int DEFAULT NULL,\n `tjrkpizpbjwkjtobxfjejniaukjgyhba` int DEFAULT NULL,\n `avbskvrhktlvtqqjlxyjrnjhlvujvzbz` int DEFAULT NULL,\n `yybkcldwtqsuuvdgntsclbfrhkegmpgz` int DEFAULT NULL,\n `osbvvalppejlffvppkaavceoaqxczwng` int DEFAULT NULL,\n `flzbitntjtjbhzfvnvswnmyxzxlbjbjm` int DEFAULT NULL,\n `exsuuzzemlejfbmrhuoolwnpdlzfvwkt` int DEFAULT NULL,\n `jmnsvfqeioheppwcgpivpunarlnyyvjk` int DEFAULT NULL,\n `cgwlkcpppskufdxxrzqdoswcewxdklgm` int DEFAULT NULL,\n `fxqnszujzcdrgejeltjghimswbrxguox` int DEFAULT NULL,\n `znjwvrxcwroripqbdfuhseyqgynyznvx` int DEFAULT NULL,\n `imtjomubgbftsdtxclwxatorqtheuvla` int DEFAULT NULL,\n `mpftropqdywcbysqfxamzvcurqwxztuw` int DEFAULT NULL,\n `hlnmeevvhkvvpvtzsdwgrdplnbirlcrd` int DEFAULT NULL,\n `rijqoufpioswfuhdwplhdwwafwjuerxz` int DEFAULT NULL,\n `qdcsgtzcjozsxmkpqdycjmixbrsvpajm` int DEFAULT NULL,\n `jwhrgepuihlpddsvempwanytttuhuikp` int DEFAULT NULL,\n `nwnrndlxvdivsaegkkaecfyzxizxklcg` int DEFAULT NULL,\n `kltrurqsfyogragdtrlzekksghkkehmk` int DEFAULT NULL,\n `daetacpjawcjwxoefijeznmtmsaenkyz` int DEFAULT NULL,\n `ugvnbngesalyhzvofbyzfizydcwnqabx` int DEFAULT NULL,\n `yjdwbaudtxfmrvwfbzgxaljtzzxrznel` int DEFAULT NULL,\n `tbbvcjsswiucjkrzqfzkoglrinnuvurm` int DEFAULT NULL,\n `ibfahoufobblwnvgknwmuytwrdqiqttk` int DEFAULT NULL,\n `jccrnnxpyllcmitorktaxpoewqrtgbep` int DEFAULT NULL,\n `zkhbasfnvoxcwjlwovoblobqihktxpiq` int DEFAULT NULL,\n `hhqzxrmfzfpnikyancdrmfsymitrnhui` int DEFAULT NULL,\n `omvirkwnwnsynbruverkujylqqzlszdr` int DEFAULT NULL,\n `jbmietbafieilgzofxvdnfoifckyqumh` int DEFAULT NULL,\n `vokeocbkftqodvjhhympbrsgngyzuzhw` int DEFAULT NULL,\n `trbmpglbrypuwgbsfsimwpqqhnrsdixa` int DEFAULT NULL,\n `gskuzptpdatfdqyrninelajkarsipsnv` int DEFAULT NULL,\n `ztbkpwcgzmcgpjqaglkhfbgpjzrvjsvw` int DEFAULT NULL,\n `jekvjtvymekbznpkzkaaalojvogfntjm` int DEFAULT NULL,\n `zxapbqshubpwtoasnfqbgzjlmufxpnzg` int DEFAULT NULL,\n `cnrokqfelcgzbmiolxslvciyugaxwean` int DEFAULT NULL,\n `mnchfaxvkmlqegnyhsyycpmsjtiygfqr` int DEFAULT NULL,\n `lwrkaguyznaisszubypetfcgrzueyxtg` int DEFAULT NULL,\n `taqevqjvhnetdfiewzmjcpymlfobfcyr` int DEFAULT NULL,\n `imyluwjdhzibahbisfxzvzlfndvjqwpj` int DEFAULT NULL,\n `wqpdevbxwxyifytwmwtvtuaucwelvuzp` int DEFAULT NULL,\n `rkogprwsjgdalweadogwzkrvihahqesl` int DEFAULT NULL,\n `doejxfydpjxgbozlgfuxliiiwoodoiwn` int DEFAULT NULL,\n `vffcrnbkjuncjmteltzgvqrohwstcpvc` int DEFAULT NULL,\n `hfcbofsywkwooqqrccpfmnepanxmkvwi` int DEFAULT NULL,\n `yhwlmtnzabfcrkmwlkxpztdqfbrfhkam` int DEFAULT NULL,\n `ikanoztushvmjplwwvlyzkmpwxcjjuxh` int DEFAULT NULL,\n `gtywbkifoduxzfeuugydkizhhqeokepo` int DEFAULT NULL,\n `nuxhiubggscidckksgannfusgkxaylfy` int DEFAULT NULL,\n `pxueggelmakfritcobgtwlfnswseooee` int DEFAULT NULL,\n `smgoolxvlgaosfaxqfsnrnvcskersksm` int DEFAULT NULL,\n `fgtyfnlvukwmxypsbmsiudcggjgusfgx` int DEFAULT NULL,\n `divaoutfwkvmenhwqjxbvlsveelonjro` int DEFAULT NULL,\n `zydodueylwkyoanstcxwhfyjlzliazgj` int DEFAULT NULL,\n `fyzflzjkeupyjpijvwtccnjracexicts` int DEFAULT NULL,\n `vzpghedkpsgnkpolbwdcsfmvfhdseeis` int DEFAULT NULL,\n `hnjwvcwqpyixogxqupieukdqzbztlqwl` int DEFAULT NULL,\n `vnfdgpesmxwwkgolwttxshmyxaupirye` int DEFAULT NULL,\n PRIMARY KEY (`ihtdicgwfiginzjsvmlofoiaovagtgzo`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"vqxcfvfvkfrjtrlfeabxtnndjgrileqz\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["ihtdicgwfiginzjsvmlofoiaovagtgzo"],"columns":[{"name":"ihtdicgwfiginzjsvmlofoiaovagtgzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"opjkjabmhyozienpwqcfxoruqcqlnbob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prxbjlqyuvgxaljfenojaehonmwtbyeu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqomrjajvdosajkcfptwlnyatggauxoe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"samxnkcxrhknloyfflndbygbrhzpoqye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxrvmszspegvfrwwuxpffejpykfgkeyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkvslkuqncrxegzekwnqwdplvzxvrwcv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lcaiwudzmzjbkksvbzxsehubswtsarmz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exqptjlremwaodlwjczgvmorgylkhxrl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xidkuqfalhowsvwmjyqpbsdgegyjxlgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"csunvntsqzaydobxskyuxuzvxnfiiupe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okwzwwgdokeuysjbtrtbzdwoixzwaxem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkxxbornffrazmqgnkpbwgoluqhugzzy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tipnypzettcheiozolcqfulwyodccmbo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmqabqyguzgpuseuwikyayzzekpkixzi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrmegkaxjyjpxesmpfbffxbncbhhxseb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hudotngmibneskjzkaxkvywjoerxxfic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpvjfhdpxyxlxuclldftrmwdqxrulpnq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnthdinqjmfhoyisykwelvzddnaooygb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmaonqvzrshmaajptleomloyodooaalb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjsxnlvifgtyggtyezhpryibugnwkmfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"siuqawpymggbqfwgsdktktoyevxiqpes","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"romxfcxdubyclxtlkxgihjhnxrhzvnan","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"efmoujdsfuxuvpjtawrbqijukysprgax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wpetfpfnfasyuiqotbpunubttezldkja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yajihgppmzcrdqhakkknqjsqpxxvxnww","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fczmsoxssnkjvzzeuzxntranxnnpsoak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzpuxpwuabalktjmhzjjgifayvmysgno","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpspebfnrhzkzyzsytxznzzxojshfmci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"duljnumtelgtngkxgktyknxgxceksaie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kquawkcgonhmoptowallcxbqpzarrnhf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"trycfhfvmiajvjrnfoputzxrjyfzcnbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"stqgnfplhncirriljpkyqcfgpsptlgkx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxldhqzsdqoqrusnvxbeemwjmbeuqkro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufbkpcfsviaidobrjzuytrmqprfpxqlz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ruzlllikhvuixxnwqtjiohlhmfzugrtg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hzumdexfwdfkoulqhqjgygvowlfjnbim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xtnezsfbgaliylsjygysqpjqxrjnpvgp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpspdmihmlhfysbtiyrbuyholcophhdz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icrzuchoogrtyyfavjkgdojunwswxiyj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnjksyxagjixpoucvugtlpdtakspnolr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdanfgchitqzahnoowynaxyadnbrvicf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zlxinjdnohhvwurgzirxfmongcvmbisk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjrkpizpbjwkjtobxfjejniaukjgyhba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avbskvrhktlvtqqjlxyjrnjhlvujvzbz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yybkcldwtqsuuvdgntsclbfrhkegmpgz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"osbvvalppejlffvppkaavceoaqxczwng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flzbitntjtjbhzfvnvswnmyxzxlbjbjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exsuuzzemlejfbmrhuoolwnpdlzfvwkt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmnsvfqeioheppwcgpivpunarlnyyvjk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgwlkcpppskufdxxrzqdoswcewxdklgm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxqnszujzcdrgejeltjghimswbrxguox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znjwvrxcwroripqbdfuhseyqgynyznvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"imtjomubgbftsdtxclwxatorqtheuvla","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpftropqdywcbysqfxamzvcurqwxztuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlnmeevvhkvvpvtzsdwgrdplnbirlcrd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rijqoufpioswfuhdwplhdwwafwjuerxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdcsgtzcjozsxmkpqdycjmixbrsvpajm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwhrgepuihlpddsvempwanytttuhuikp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nwnrndlxvdivsaegkkaecfyzxizxklcg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kltrurqsfyogragdtrlzekksghkkehmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"daetacpjawcjwxoefijeznmtmsaenkyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugvnbngesalyhzvofbyzfizydcwnqabx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjdwbaudtxfmrvwfbzgxaljtzzxrznel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbbvcjsswiucjkrzqfzkoglrinnuvurm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibfahoufobblwnvgknwmuytwrdqiqttk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jccrnnxpyllcmitorktaxpoewqrtgbep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zkhbasfnvoxcwjlwovoblobqihktxpiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhqzxrmfzfpnikyancdrmfsymitrnhui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omvirkwnwnsynbruverkujylqqzlszdr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbmietbafieilgzofxvdnfoifckyqumh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vokeocbkftqodvjhhympbrsgngyzuzhw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"trbmpglbrypuwgbsfsimwpqqhnrsdixa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gskuzptpdatfdqyrninelajkarsipsnv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ztbkpwcgzmcgpjqaglkhfbgpjzrvjsvw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jekvjtvymekbznpkzkaaalojvogfntjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxapbqshubpwtoasnfqbgzjlmufxpnzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnrokqfelcgzbmiolxslvciyugaxwean","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnchfaxvkmlqegnyhsyycpmsjtiygfqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwrkaguyznaisszubypetfcgrzueyxtg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"taqevqjvhnetdfiewzmjcpymlfobfcyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"imyluwjdhzibahbisfxzvzlfndvjqwpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqpdevbxwxyifytwmwtvtuaucwelvuzp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkogprwsjgdalweadogwzkrvihahqesl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"doejxfydpjxgbozlgfuxliiiwoodoiwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vffcrnbkjuncjmteltzgvqrohwstcpvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfcbofsywkwooqqrccpfmnepanxmkvwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhwlmtnzabfcrkmwlkxpztdqfbrfhkam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikanoztushvmjplwwvlyzkmpwxcjjuxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtywbkifoduxzfeuugydkizhhqeokepo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nuxhiubggscidckksgannfusgkxaylfy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxueggelmakfritcobgtwlfnswseooee","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smgoolxvlgaosfaxqfsnrnvcskersksm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fgtyfnlvukwmxypsbmsiudcggjgusfgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"divaoutfwkvmenhwqjxbvlsveelonjro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zydodueylwkyoanstcxwhfyjlzliazgj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fyzflzjkeupyjpijvwtccnjracexicts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzpghedkpsgnkpolbwdcsfmvfhdseeis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hnjwvcwqpyixogxqupieukdqzbztlqwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnfdgpesmxwwkgolwttxshmyxaupirye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672204,"databaseName":"models_schema","ddl":"CREATE TABLE `vrbvigqupjgsaeqqwingjxtcxidpztuo` (\n `alekucgiztvxyoaqbankbkjzhlkaswvy` int NOT NULL,\n `bmsdtydntxeyjawmoyhnwwblxwzjdrkf` int DEFAULT NULL,\n `gxejfbpmmuzbqjpejeznjcdrxwcoggbu` int DEFAULT NULL,\n `vwskwxlomywltmlscsrulqifcduijtzd` int DEFAULT NULL,\n `vpsdxrooaysakmxtapadygiufgevgtam` int DEFAULT NULL,\n `vofdmxukrkkugtbengzsjleehdoidtmg` int DEFAULT NULL,\n `rbwwbtjhjzgrxwhhfjpfcubnkgictskf` int DEFAULT NULL,\n `ocbkgmhhclevslrofuentevoavzojrtz` int DEFAULT NULL,\n `adyazvmvfzpslifojlhshlgqaeypszhm` int DEFAULT NULL,\n `cimvqothycqtojfquuvidgoucoyujywy` int DEFAULT NULL,\n `dgyliysfaqnnymrqtpmypvbkxobeztmc` int DEFAULT NULL,\n `dcczyweqhuljksgqnmsqhiuzbwbcjtil` int DEFAULT NULL,\n `bztwbxjnqafosatcmakiosgeyxmqvgln` int DEFAULT NULL,\n `tnovnsxkiiekufhzlhmlenplwmjrebvt` int DEFAULT NULL,\n `tbyhmgfshrrvohphozeqbvffflqsilnr` int DEFAULT NULL,\n `wwqbwrphgdzmyryhuhnaqzgjsraragod` int DEFAULT NULL,\n `cnbbbncukiildlxgnvnbqcuptwjskhjy` int DEFAULT NULL,\n `dwtyhujeafcjnrrkmszqqznzwewgskjc` int DEFAULT NULL,\n `taujlwwtvwrutstpgyvtptehnovjawgk` int DEFAULT NULL,\n `yuafvjivflrevawrdcvnarywbfffkugw` int DEFAULT NULL,\n `luflqjgkzjruzsoplmdfjgwcnqzhbfap` int DEFAULT NULL,\n `odsebqoyadakzshagcvivuooenentuir` int DEFAULT NULL,\n `atvgfwoqkfuizmakxhsfrhzbxmcbaurc` int DEFAULT NULL,\n `ysszanhczeqvrbkqofgtavjdvbrggmab` int DEFAULT NULL,\n `hlibnhicmbkkrmhmeltrgmghmpiigrwp` int DEFAULT NULL,\n `tqklmvvusvsdihlwpapmcngedqitaogl` int DEFAULT NULL,\n `gijotnepaliqkfzplzyogsnjprhweigq` int DEFAULT NULL,\n `krmolrfvpngfpsrbkbnbnordjljmjrjo` int DEFAULT NULL,\n `nzutuwnrgwnrxraohwjjypqmlukkyvla` int DEFAULT NULL,\n `agrymhvbjtemfdveawpqlygdredluxnp` int DEFAULT NULL,\n `eecexeicnfmjujeleygrejprbbbwnokk` int DEFAULT NULL,\n `wtxmkvxgmawysxnmiqiksamvogqqqnnr` int DEFAULT NULL,\n `ulyffyteadohizpzjlxtrmpqypwsrrnb` int DEFAULT NULL,\n `rfybkiklzzdzfsbywdyylykugpnhkgaj` int DEFAULT NULL,\n `qljqmcrdvlkaxlqfoceaozbhuurpaxvg` int DEFAULT NULL,\n `ifcjufjpyrhclmzagaylzumbirsqrgks` int DEFAULT NULL,\n `lexwpxziqiewjbhdhgpbvuadffkhqdby` int DEFAULT NULL,\n `uwezgivelewblxjlrhijzxgovnfzxksr` int DEFAULT NULL,\n `tvrwludsbhbptelekuubdxtbekywadqb` int DEFAULT NULL,\n `lfemensbhzzfhbbqzqgdwqhdrbktrvyo` int DEFAULT NULL,\n `yvafbqdmiiackdqsyxxspptontjxutks` int DEFAULT NULL,\n `jlrjcebxcsuabbynqkkvmizgsucfripr` int DEFAULT NULL,\n `pknkodsqajeapicvosltcexfyqcxmziq` int DEFAULT NULL,\n `slwclvreufbnaddqcjlfgyvxhowkizdo` int DEFAULT NULL,\n `sppugvmioxyvlbngrdhitvusckfrdmvz` int DEFAULT NULL,\n `jqpaymmrlnotjmlbpjpafjkhihxpewch` int DEFAULT NULL,\n `beuwgwuglfhzyoukemiupqhawquenxkk` int DEFAULT NULL,\n `qwbmrewomldrmdjjqygagczkuzcwdzdn` int DEFAULT NULL,\n `qvsgydxguwwjvhfcctbrjxtmxibhigbp` int DEFAULT NULL,\n `rxwggjvfebcspvxkjdwwbnnxtzhpysgk` int DEFAULT NULL,\n `smdkmgwxozxidgxqhpleclmbedyldvin` int DEFAULT NULL,\n `oistfhndwaasbiumsirancreafpasljm` int DEFAULT NULL,\n `fmxeddawffjcdvaacwthptjhbwzxpvpi` int DEFAULT NULL,\n `qxrplupogodgavyzircgipctehrfhtti` int DEFAULT NULL,\n `hkstfsdpsmeevxsawuaxsmxppacktbpj` int DEFAULT NULL,\n `bwjcnocssyrpaerrhfbfupluhotoxhlk` int DEFAULT NULL,\n `kxgwxhqjdumopsogwgyjvunxuqyzprlt` int DEFAULT NULL,\n `vtbyhbywgypvrzynbwwfemfipownakkq` int DEFAULT NULL,\n `rcedbkcjuxiwmrzrtwosbyubjdyzmawu` int DEFAULT NULL,\n `vnthvdsnsnvajssrkpaqymosqhsigpht` int DEFAULT NULL,\n `maadxvdtczvqkfqkqigpmdmxiwxnqmcb` int DEFAULT NULL,\n `ywnlcofxxykblvvmakpakmmvffylyapk` int DEFAULT NULL,\n `vwvwttxzmfkuzsxvdiqjzonsvgqwyqlq` int DEFAULT NULL,\n `vxzftkdqrtujpywzhyjmlxebqsedttft` int DEFAULT NULL,\n `uukuylxjkczfrlyrkqfqctfafhlaisvi` int DEFAULT NULL,\n `rndfsyoiwmizcvtcqwlstizzotrpmbrv` int DEFAULT NULL,\n `viparsmasydsczhbdanwjjqmckitrvku` int DEFAULT NULL,\n `wghihlglahhyqgkfgppuxliplwkrxzst` int DEFAULT NULL,\n `pyykejaiuattzzhjypuyshzbkimxivin` int DEFAULT NULL,\n `xmwgsnvyeyegfbcordcafjrezxyjzxpn` int DEFAULT NULL,\n `mwyljjyayvmegjpffkbqjennwzcrqqfs` int DEFAULT NULL,\n `usfrqbrhlopaovqtlzyfyewjamzqouex` int DEFAULT NULL,\n `ahytajcmwoarecmtfsrgbiunvjikqqgg` int DEFAULT NULL,\n `jnzwlrbudsqmmwccwfxdevhzhlibxxqd` int DEFAULT NULL,\n `zqoyjgenxmsqmmiixqyriyiotlelitni` int DEFAULT NULL,\n `jvuofuxzvlovoytwrlmafeabveyfjwtj` int DEFAULT NULL,\n `nvopaewvdxwvtmshjxwpcxeljvmumxxz` int DEFAULT NULL,\n `uurserznrdupdvbavcwdypxlkdplblrs` int DEFAULT NULL,\n `nyqvqrfycwodsmpfuxdofmuualbccqwy` int DEFAULT NULL,\n `ejptuiblfebkthcguearqwekliddnckz` int DEFAULT NULL,\n `znyefvlhloqjfkppcxboefafnnrmgzub` int DEFAULT NULL,\n `ekmyxyaqighvhdvswaabozuzqhcuszdp` int DEFAULT NULL,\n `ednejtggfbsvmjitlnrvghbsddyamhei` int DEFAULT NULL,\n `ynluinwcwmsqcpnttyvmqnnpdjpiithx` int DEFAULT NULL,\n `ufpdwigzglybcrvhxrlgkbimkfbaajdz` int DEFAULT NULL,\n `plauekzralcalgiubqeuqtmloojzjhfe` int DEFAULT NULL,\n `gefromgwfbwhwhkctyobdcyhnnvwcmvr` int DEFAULT NULL,\n `dgwqdxpzuwlxdtcosxcsxnybcwleqnwg` int DEFAULT NULL,\n `hdqxmgcmjympexdggmkznddvhdjqerkm` int DEFAULT NULL,\n `mldlzokyiuhautdqzobfvivcxzuzgvig` int DEFAULT NULL,\n `zbfugkaqljxlnlyvzyagnznhqmsioglg` int DEFAULT NULL,\n `zhwatwqqlmscngtncpobyfllkjomgphf` int DEFAULT NULL,\n `lkvtaplzwzxcqeqknxthvmfrjtwgmnuv` int DEFAULT NULL,\n `qffvzpoeahmkewnyvxrtuqgpwacxxjli` int DEFAULT NULL,\n `keavxwuxkpneyjwjeqsflqcafielpujy` int DEFAULT NULL,\n `cqsrrzjzupkacfddwzljppzjsrfqgcdt` int DEFAULT NULL,\n `gjfihdcducbipvidbhszsyveelsnvpph` int DEFAULT NULL,\n `tksvzwpbhohpmsmipzejdbfaxseanznb` int DEFAULT NULL,\n `vtfytirxbtjrfyrlwmdffqytzwkrgpds` int DEFAULT NULL,\n `lkyvsivxmoertqqiyaizviesggzomvtl` int DEFAULT NULL,\n PRIMARY KEY (`alekucgiztvxyoaqbankbkjzhlkaswvy`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"vrbvigqupjgsaeqqwingjxtcxidpztuo\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["alekucgiztvxyoaqbankbkjzhlkaswvy"],"columns":[{"name":"alekucgiztvxyoaqbankbkjzhlkaswvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"bmsdtydntxeyjawmoyhnwwblxwzjdrkf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxejfbpmmuzbqjpejeznjcdrxwcoggbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vwskwxlomywltmlscsrulqifcduijtzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpsdxrooaysakmxtapadygiufgevgtam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vofdmxukrkkugtbengzsjleehdoidtmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbwwbtjhjzgrxwhhfjpfcubnkgictskf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocbkgmhhclevslrofuentevoavzojrtz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"adyazvmvfzpslifojlhshlgqaeypszhm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cimvqothycqtojfquuvidgoucoyujywy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgyliysfaqnnymrqtpmypvbkxobeztmc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcczyweqhuljksgqnmsqhiuzbwbcjtil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bztwbxjnqafosatcmakiosgeyxmqvgln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnovnsxkiiekufhzlhmlenplwmjrebvt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbyhmgfshrrvohphozeqbvffflqsilnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwqbwrphgdzmyryhuhnaqzgjsraragod","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnbbbncukiildlxgnvnbqcuptwjskhjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwtyhujeafcjnrrkmszqqznzwewgskjc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"taujlwwtvwrutstpgyvtptehnovjawgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yuafvjivflrevawrdcvnarywbfffkugw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"luflqjgkzjruzsoplmdfjgwcnqzhbfap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"odsebqoyadakzshagcvivuooenentuir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atvgfwoqkfuizmakxhsfrhzbxmcbaurc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ysszanhczeqvrbkqofgtavjdvbrggmab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlibnhicmbkkrmhmeltrgmghmpiigrwp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqklmvvusvsdihlwpapmcngedqitaogl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gijotnepaliqkfzplzyogsnjprhweigq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krmolrfvpngfpsrbkbnbnordjljmjrjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzutuwnrgwnrxraohwjjypqmlukkyvla","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agrymhvbjtemfdveawpqlygdredluxnp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eecexeicnfmjujeleygrejprbbbwnokk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtxmkvxgmawysxnmiqiksamvogqqqnnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ulyffyteadohizpzjlxtrmpqypwsrrnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfybkiklzzdzfsbywdyylykugpnhkgaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qljqmcrdvlkaxlqfoceaozbhuurpaxvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifcjufjpyrhclmzagaylzumbirsqrgks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lexwpxziqiewjbhdhgpbvuadffkhqdby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwezgivelewblxjlrhijzxgovnfzxksr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvrwludsbhbptelekuubdxtbekywadqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfemensbhzzfhbbqzqgdwqhdrbktrvyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yvafbqdmiiackdqsyxxspptontjxutks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlrjcebxcsuabbynqkkvmizgsucfripr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pknkodsqajeapicvosltcexfyqcxmziq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"slwclvreufbnaddqcjlfgyvxhowkizdo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sppugvmioxyvlbngrdhitvusckfrdmvz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jqpaymmrlnotjmlbpjpafjkhihxpewch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"beuwgwuglfhzyoukemiupqhawquenxkk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwbmrewomldrmdjjqygagczkuzcwdzdn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvsgydxguwwjvhfcctbrjxtmxibhigbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxwggjvfebcspvxkjdwwbnnxtzhpysgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smdkmgwxozxidgxqhpleclmbedyldvin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oistfhndwaasbiumsirancreafpasljm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmxeddawffjcdvaacwthptjhbwzxpvpi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxrplupogodgavyzircgipctehrfhtti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkstfsdpsmeevxsawuaxsmxppacktbpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bwjcnocssyrpaerrhfbfupluhotoxhlk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxgwxhqjdumopsogwgyjvunxuqyzprlt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtbyhbywgypvrzynbwwfemfipownakkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcedbkcjuxiwmrzrtwosbyubjdyzmawu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnthvdsnsnvajssrkpaqymosqhsigpht","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"maadxvdtczvqkfqkqigpmdmxiwxnqmcb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywnlcofxxykblvvmakpakmmvffylyapk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vwvwttxzmfkuzsxvdiqjzonsvgqwyqlq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxzftkdqrtujpywzhyjmlxebqsedttft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uukuylxjkczfrlyrkqfqctfafhlaisvi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rndfsyoiwmizcvtcqwlstizzotrpmbrv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"viparsmasydsczhbdanwjjqmckitrvku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wghihlglahhyqgkfgppuxliplwkrxzst","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pyykejaiuattzzhjypuyshzbkimxivin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmwgsnvyeyegfbcordcafjrezxyjzxpn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwyljjyayvmegjpffkbqjennwzcrqqfs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"usfrqbrhlopaovqtlzyfyewjamzqouex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahytajcmwoarecmtfsrgbiunvjikqqgg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jnzwlrbudsqmmwccwfxdevhzhlibxxqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqoyjgenxmsqmmiixqyriyiotlelitni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvuofuxzvlovoytwrlmafeabveyfjwtj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvopaewvdxwvtmshjxwpcxeljvmumxxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uurserznrdupdvbavcwdypxlkdplblrs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nyqvqrfycwodsmpfuxdofmuualbccqwy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejptuiblfebkthcguearqwekliddnckz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znyefvlhloqjfkppcxboefafnnrmgzub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekmyxyaqighvhdvswaabozuzqhcuszdp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ednejtggfbsvmjitlnrvghbsddyamhei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynluinwcwmsqcpnttyvmqnnpdjpiithx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufpdwigzglybcrvhxrlgkbimkfbaajdz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"plauekzralcalgiubqeuqtmloojzjhfe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gefromgwfbwhwhkctyobdcyhnnvwcmvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgwqdxpzuwlxdtcosxcsxnybcwleqnwg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdqxmgcmjympexdggmkznddvhdjqerkm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mldlzokyiuhautdqzobfvivcxzuzgvig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbfugkaqljxlnlyvzyagnznhqmsioglg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhwatwqqlmscngtncpobyfllkjomgphf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lkvtaplzwzxcqeqknxthvmfrjtwgmnuv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qffvzpoeahmkewnyvxrtuqgpwacxxjli","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"keavxwuxkpneyjwjeqsflqcafielpujy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqsrrzjzupkacfddwzljppzjsrfqgcdt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjfihdcducbipvidbhszsyveelsnvpph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tksvzwpbhohpmsmipzejdbfaxseanznb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtfytirxbtjrfyrlwmdffqytzwkrgpds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lkyvsivxmoertqqiyaizviesggzomvtl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672235,"databaseName":"models_schema","ddl":"CREATE TABLE `vvukznvxvqlrpqvbrbvlexzlwndtibeh` (\n `mdctvtvtjqjuthhlvtudzjbqeoguwyzb` int NOT NULL,\n `gcaoyaqjgpjhgfjmldkfbvjtifhxwpeq` int DEFAULT NULL,\n `ytofztdtiqlzfbmqniubffpcyqufzxie` int DEFAULT NULL,\n `zikesmixirncxrwyphhhamgaacdnuoui` int DEFAULT NULL,\n `yhvhwhmcethnrhcjejxyrphtcxzdsvhs` int DEFAULT NULL,\n `eafuylrrpkcaeautmsqhqivmvqienhbv` int DEFAULT NULL,\n `nsoxlglriljqmuwniwhakmegqjzaowfo` int DEFAULT NULL,\n `yslijnqgvdmjeohrrlutqszvvfcbuity` int DEFAULT NULL,\n `zesknbibqewdsqeyftculoslqovhygnb` int DEFAULT NULL,\n `hdxqyznzjwkmsctvgfyjjufvkyclskqq` int DEFAULT NULL,\n `pahbwcaoxgrvunxsbjeixpdwhufngfan` int DEFAULT NULL,\n `hdfuirddmcqirrwlqyixqcsjfxdjemeq` int DEFAULT NULL,\n `smbwgrtbpwlpcbrblixkgcktydsqlekf` int DEFAULT NULL,\n `yjospdlsblyvllctengoqucujpoqpkfx` int DEFAULT NULL,\n `znfzxnpjkwzicntdcthkksoqcltfjjbl` int DEFAULT NULL,\n `innllzgqfmfkqkxwsoofckasyewykqsj` int DEFAULT NULL,\n `evulmrpiyhpnhfufxzdyadjpebtdfmbf` int DEFAULT NULL,\n `oyabfbtgvjvzzvfanbhqpctcjbgjlloi` int DEFAULT NULL,\n `anmgqvjruzlwaesavpjtswlyctzpgfqk` int DEFAULT NULL,\n `lsunupxulancgunrrxdlnhmysvkidogt` int DEFAULT NULL,\n `tnpwvrnxfjlulrhwpmhetjelzfwihefw` int DEFAULT NULL,\n `wekjakucgufnsliygbkekdqvkztybcbu` int DEFAULT NULL,\n `hirfdntpfjlpcdsdottorizgwajyonbe` int DEFAULT NULL,\n `defdbrmtgvnwudtrhzeebvitjocevbit` int DEFAULT NULL,\n `uelcrukrkquprrmtozctltyabpukkzxy` int DEFAULT NULL,\n `pgyqtnjdbhezkhoyrzhgxfjqopveheud` int DEFAULT NULL,\n `nblghyqgsumifchagjrrxdjvtqjhuswn` int DEFAULT NULL,\n `ergqntibvqavtughecnsppcqfzuqenpi` int DEFAULT NULL,\n `ntqjgbznobjtiiqbuipiudvhqrznxusj` int DEFAULT NULL,\n `zehcfsoufhmuhpxppawqsyzrxcgszgra` int DEFAULT NULL,\n `nbqitdoklypbmheoduedrtgtlnhcokga` int DEFAULT NULL,\n `bkuxjpyxuavvxdvlqzjyjxpekanowvhz` int DEFAULT NULL,\n `rcaxtgzafvvrpinyjxcmhtlydshwiucy` int DEFAULT NULL,\n `uoolonlkntaumxeckeioowarxmtyqole` int DEFAULT NULL,\n `regwiqmwmberljaiauihoxxusmcyigef` int DEFAULT NULL,\n `iuxidbhaachychevjioocdajxklinvrr` int DEFAULT NULL,\n `juiufsxyqafenelabaldrkqbbamunzbz` int DEFAULT NULL,\n `zejjwjcxutuvlsnadbecnkvrzkazpvqo` int DEFAULT NULL,\n `mgyjdimludfwkfvgescztymvjperhnjv` int DEFAULT NULL,\n `bbyndjcyeyoqinkkdyqdnrsmvgfokxem` int DEFAULT NULL,\n `nrqpauqwjkapmummllmjykrsiwtgrssh` int DEFAULT NULL,\n `yuxzkzerlnlrtonbnroqqbkexzpexiqr` int DEFAULT NULL,\n `tyrvvbqybeaunokiruhtanmzkbxbkbac` int DEFAULT NULL,\n `eqcamtatukrivzanoohyukvwlalxzbrz` int DEFAULT NULL,\n `ggyyujheihnmfzjdmkkkyldyfcqyiuev` int DEFAULT NULL,\n `quyajwbxamgyrmyqopsxqrnotexugiif` int DEFAULT NULL,\n `jhrcxkzpthcbflarpigqidhsycaczxwr` int DEFAULT NULL,\n `yzqkfixrbwocgyyvkzkvrxhfetdiaeld` int DEFAULT NULL,\n `durhcifjukbscjvpbrkjctvzqohciyib` int DEFAULT NULL,\n `myemuvfjjfndkbryxxafxpjrfgevympl` int DEFAULT NULL,\n `gnbpxboqbfspecukfseabkoueyblguie` int DEFAULT NULL,\n `hgtyrlagayhsuagbwtpzoiyfzprvffta` int DEFAULT NULL,\n `ajsvobeocvlrzissuotwgngdxytajrvu` int DEFAULT NULL,\n `beottpftlqrbmdkmxlbkvwyrnypdfpqb` int DEFAULT NULL,\n `sdzmutzfxzmehtgnnaskznaphalhyqwd` int DEFAULT NULL,\n `rouedbfsptkmybanjadheynhfjhphtej` int DEFAULT NULL,\n `mmxqhacuvdhvhtlrjhiqyybrukubdvqp` int DEFAULT NULL,\n `oozsjsggcuywvsuwhcffwmjptjqpsnpg` int DEFAULT NULL,\n `nunzsxdawafuxnvxbjdpctjwqejjjzdl` int DEFAULT NULL,\n `rihpafyydmjlhpyftfylhrxxnfkkktci` int DEFAULT NULL,\n `jmlygiciszbbwawygkedgppdavchfdsx` int DEFAULT NULL,\n `zaimirmugtkwdkdmlbhvamjazqfgslvc` int DEFAULT NULL,\n `nkmslyeogsvekbqqunbnogwrbvdysohi` int DEFAULT NULL,\n `jfvjeuyubfzyvrjcpeygaymhoimlpkkc` int DEFAULT NULL,\n `bsnlaxkkzdxbqumjdswjtkvboivpzzxb` int DEFAULT NULL,\n `slkkiodenchewkrjqurlilhrxhosecbj` int DEFAULT NULL,\n `eirmfkagzlusiibhbrdcrfrgnlgwxfal` int DEFAULT NULL,\n `dcsrqljfjgcvomhhgwaggbhppgvimjmh` int DEFAULT NULL,\n `aqcjuybfnyrenwfcqxauvdmcjktauzyt` int DEFAULT NULL,\n `wxhipxblpokmbvugzeeyemihjrcbcmgb` int DEFAULT NULL,\n `btaucrufmzvmfuhhbtqlsfdevyxrniei` int DEFAULT NULL,\n `hnyrcgdeuolqqutxndnqhisnhcwttnut` int DEFAULT NULL,\n `aoiqbrnctxcpsowryovibrvbxzavmbao` int DEFAULT NULL,\n `zwwishlujdrkbpiwcuakjmmrllfjvrlw` int DEFAULT NULL,\n `hnbpuyfjkypfzeqshqglotcrlfdypigb` int DEFAULT NULL,\n `uiegbqgbujvypoyhpbhgvuynmcvpxhxt` int DEFAULT NULL,\n `kvxzxquttkhzvfymmigacddpnjxhtvqc` int DEFAULT NULL,\n `mpiasiyepqycyvhtltjbqcpohosrceyu` int DEFAULT NULL,\n `nvnlbzkcaifghrickzpwiwmkcupneblg` int DEFAULT NULL,\n `glmqqzmktmeyitfjsbirauzxcfdxjrll` int DEFAULT NULL,\n `rhwgyyezlhqjyclenrtsfuuejlauttds` int DEFAULT NULL,\n `eajnmzdmeosdrntqidopfukuvavxtcoh` int DEFAULT NULL,\n `mrwpachhwgoqzfbyqudlsxszvxadxwmh` int DEFAULT NULL,\n `fnvzyvhbvxqekqsavpxkyvoithajdkly` int DEFAULT NULL,\n `plktqrrgrhadvdoioknyhkzfbmnrkbqg` int DEFAULT NULL,\n `vgahvkksppwpbumekojskqgfkottdqxs` int DEFAULT NULL,\n `rkwmzijtuqwusemdhloqjtgsepngtmsn` int DEFAULT NULL,\n `enheojilidvztsbrigfroycklekqfsds` int DEFAULT NULL,\n `xqmegyvnxuaricnhnhghgklhyiqfwjqu` int DEFAULT NULL,\n `yqrxmuikptzckvmcupmmlgzifacjmmwi` int DEFAULT NULL,\n `cgtoihsjjzfmwxqammdsmethaqnzbbyy` int DEFAULT NULL,\n `krguwozhmnyjafedmypprgkurnkdqmfx` int DEFAULT NULL,\n `kfsaxlbuswadtnatdhmwsaiaxxxniwml` int DEFAULT NULL,\n `hogqbbsyipeaexbvjrnclelqrcosmcjk` int DEFAULT NULL,\n `ssctlwteizemqwyntawduesoshmhibie` int DEFAULT NULL,\n `edpgdxiyqheqkcagsqoqovcrlfsxjhhm` int DEFAULT NULL,\n `zdhxprrkavvqocmucikodzqlpnmmrypa` int DEFAULT NULL,\n `cvnisurkaaxqwlhpjffjthpdcembkgzf` int DEFAULT NULL,\n `yhcgmhqhtzrcsznchemitlbfhmdyhqcf` int DEFAULT NULL,\n `jmbtgghjhmwcbpktwnbejauyvhzsprwa` int DEFAULT NULL,\n PRIMARY KEY (`mdctvtvtjqjuthhlvtudzjbqeoguwyzb`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"vvukznvxvqlrpqvbrbvlexzlwndtibeh\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["mdctvtvtjqjuthhlvtudzjbqeoguwyzb"],"columns":[{"name":"mdctvtvtjqjuthhlvtudzjbqeoguwyzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"gcaoyaqjgpjhgfjmldkfbvjtifhxwpeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytofztdtiqlzfbmqniubffpcyqufzxie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zikesmixirncxrwyphhhamgaacdnuoui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhvhwhmcethnrhcjejxyrphtcxzdsvhs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eafuylrrpkcaeautmsqhqivmvqienhbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nsoxlglriljqmuwniwhakmegqjzaowfo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yslijnqgvdmjeohrrlutqszvvfcbuity","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zesknbibqewdsqeyftculoslqovhygnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdxqyznzjwkmsctvgfyjjufvkyclskqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pahbwcaoxgrvunxsbjeixpdwhufngfan","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdfuirddmcqirrwlqyixqcsjfxdjemeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smbwgrtbpwlpcbrblixkgcktydsqlekf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjospdlsblyvllctengoqucujpoqpkfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znfzxnpjkwzicntdcthkksoqcltfjjbl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"innllzgqfmfkqkxwsoofckasyewykqsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evulmrpiyhpnhfufxzdyadjpebtdfmbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oyabfbtgvjvzzvfanbhqpctcjbgjlloi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"anmgqvjruzlwaesavpjtswlyctzpgfqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lsunupxulancgunrrxdlnhmysvkidogt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnpwvrnxfjlulrhwpmhetjelzfwihefw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wekjakucgufnsliygbkekdqvkztybcbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hirfdntpfjlpcdsdottorizgwajyonbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"defdbrmtgvnwudtrhzeebvitjocevbit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uelcrukrkquprrmtozctltyabpukkzxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgyqtnjdbhezkhoyrzhgxfjqopveheud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nblghyqgsumifchagjrrxdjvtqjhuswn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ergqntibvqavtughecnsppcqfzuqenpi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntqjgbznobjtiiqbuipiudvhqrznxusj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zehcfsoufhmuhpxppawqsyzrxcgszgra","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbqitdoklypbmheoduedrtgtlnhcokga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkuxjpyxuavvxdvlqzjyjxpekanowvhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcaxtgzafvvrpinyjxcmhtlydshwiucy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uoolonlkntaumxeckeioowarxmtyqole","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"regwiqmwmberljaiauihoxxusmcyigef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iuxidbhaachychevjioocdajxklinvrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"juiufsxyqafenelabaldrkqbbamunzbz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zejjwjcxutuvlsnadbecnkvrzkazpvqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mgyjdimludfwkfvgescztymvjperhnjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbyndjcyeyoqinkkdyqdnrsmvgfokxem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nrqpauqwjkapmummllmjykrsiwtgrssh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yuxzkzerlnlrtonbnroqqbkexzpexiqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tyrvvbqybeaunokiruhtanmzkbxbkbac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqcamtatukrivzanoohyukvwlalxzbrz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggyyujheihnmfzjdmkkkyldyfcqyiuev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"quyajwbxamgyrmyqopsxqrnotexugiif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jhrcxkzpthcbflarpigqidhsycaczxwr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzqkfixrbwocgyyvkzkvrxhfetdiaeld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"durhcifjukbscjvpbrkjctvzqohciyib","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"myemuvfjjfndkbryxxafxpjrfgevympl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnbpxboqbfspecukfseabkoueyblguie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hgtyrlagayhsuagbwtpzoiyfzprvffta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajsvobeocvlrzissuotwgngdxytajrvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"beottpftlqrbmdkmxlbkvwyrnypdfpqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdzmutzfxzmehtgnnaskznaphalhyqwd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rouedbfsptkmybanjadheynhfjhphtej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmxqhacuvdhvhtlrjhiqyybrukubdvqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oozsjsggcuywvsuwhcffwmjptjqpsnpg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nunzsxdawafuxnvxbjdpctjwqejjjzdl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rihpafyydmjlhpyftfylhrxxnfkkktci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmlygiciszbbwawygkedgppdavchfdsx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zaimirmugtkwdkdmlbhvamjazqfgslvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkmslyeogsvekbqqunbnogwrbvdysohi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jfvjeuyubfzyvrjcpeygaymhoimlpkkc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bsnlaxkkzdxbqumjdswjtkvboivpzzxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"slkkiodenchewkrjqurlilhrxhosecbj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eirmfkagzlusiibhbrdcrfrgnlgwxfal","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcsrqljfjgcvomhhgwaggbhppgvimjmh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqcjuybfnyrenwfcqxauvdmcjktauzyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxhipxblpokmbvugzeeyemihjrcbcmgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"btaucrufmzvmfuhhbtqlsfdevyxrniei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hnyrcgdeuolqqutxndnqhisnhcwttnut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aoiqbrnctxcpsowryovibrvbxzavmbao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zwwishlujdrkbpiwcuakjmmrllfjvrlw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hnbpuyfjkypfzeqshqglotcrlfdypigb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uiegbqgbujvypoyhpbhgvuynmcvpxhxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvxzxquttkhzvfymmigacddpnjxhtvqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpiasiyepqycyvhtltjbqcpohosrceyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvnlbzkcaifghrickzpwiwmkcupneblg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"glmqqzmktmeyitfjsbirauzxcfdxjrll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhwgyyezlhqjyclenrtsfuuejlauttds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eajnmzdmeosdrntqidopfukuvavxtcoh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrwpachhwgoqzfbyqudlsxszvxadxwmh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnvzyvhbvxqekqsavpxkyvoithajdkly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"plktqrrgrhadvdoioknyhkzfbmnrkbqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgahvkksppwpbumekojskqgfkottdqxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkwmzijtuqwusemdhloqjtgsepngtmsn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"enheojilidvztsbrigfroycklekqfsds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqmegyvnxuaricnhnhghgklhyiqfwjqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqrxmuikptzckvmcupmmlgzifacjmmwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgtoihsjjzfmwxqammdsmethaqnzbbyy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krguwozhmnyjafedmypprgkurnkdqmfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfsaxlbuswadtnatdhmwsaiaxxxniwml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hogqbbsyipeaexbvjrnclelqrcosmcjk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ssctlwteizemqwyntawduesoshmhibie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edpgdxiyqheqkcagsqoqovcrlfsxjhhm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdhxprrkavvqocmucikodzqlpnmmrypa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvnisurkaaxqwlhpjffjthpdcembkgzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhcgmhqhtzrcsznchemitlbfhmdyhqcf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmbtgghjhmwcbpktwnbejauyvhzsprwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672267,"databaseName":"models_schema","ddl":"CREATE TABLE `wchuegpxotggojpmappheurpxfrcivka` (\n `trqbqtburynwdzamokaftzhytjymdyhw` int NOT NULL,\n `kamunjhnasyejoulaiurvrffbbikwtfc` int DEFAULT NULL,\n `cpyncfjqeqrxwwuboeycclqgcvptryql` int DEFAULT NULL,\n `bisobjttedwpkwylusarfiwodftnlduu` int DEFAULT NULL,\n `wagyeznsqlhnvsikfegrblrpxxfieuas` int DEFAULT NULL,\n `wirgbojhctbzdluuwsgcmwugsxuepbld` int DEFAULT NULL,\n `wqrswkdyrgjpqcyiptvksjwectbhfxbk` int DEFAULT NULL,\n `byzfjcaxhhklsntrahvpkeefqsliiujh` int DEFAULT NULL,\n `bzwbyxsyxxurqhiogavxxszfbkbrtnus` int DEFAULT NULL,\n `vlehyownqkqkqwqsuangcemdnbavmdfu` int DEFAULT NULL,\n `fkahhrhhmrzwaunxbmqkawggnffloyzj` int DEFAULT NULL,\n `jwlwallycumlycvxufndlzoedlhqptup` int DEFAULT NULL,\n `bvfudxhymzhdzfzdkeraxclmpptmqzwh` int DEFAULT NULL,\n `ahocmewiheqeeaqlcnjhhhmfttubjwce` int DEFAULT NULL,\n `sxlslkhfymoyhizxfyqzqrmddakzyguw` int DEFAULT NULL,\n `nmzyzzzfdebeobckemyozewsbpsiaemc` int DEFAULT NULL,\n `qpydqreanxihguylhttnzdxvzsrmbcwy` int DEFAULT NULL,\n `gteqfvqdkwgxzqiknsrlgalzznjoipjk` int DEFAULT NULL,\n `zmgitshqypraqlbgklrsqaeqoatmnvdi` int DEFAULT NULL,\n `ytgeetiyhwcsurmxmifcqbfaoqlvbhfu` int DEFAULT NULL,\n `tpyemxugvjgwkmgiztomwxlixhodltpz` int DEFAULT NULL,\n `dhkhrgdkdgbktomduachzsiykorhiqny` int DEFAULT NULL,\n `yogwuubuxjtkjuzbhxuoaprnfxzzyadw` int DEFAULT NULL,\n `wjmknhinanipqyhagndciaxgrdctqyyb` int DEFAULT NULL,\n `gssbvdenmqomvkdtgdhqbwpmceeewrrr` int DEFAULT NULL,\n `hkmnkgohxzprziewyljlvxipwefxodkx` int DEFAULT NULL,\n `trlyinnzizrgkheqsjkscnhimqrkxwfw` int DEFAULT NULL,\n `reqnljfmjsiqocsfxnvjiavibgryfueq` int DEFAULT NULL,\n `rnnvmrlqlmxxhpchbxejdfzhzgfxgswc` int DEFAULT NULL,\n `ipxbyfczksjznkhwopenypxyrbpatlbv` int DEFAULT NULL,\n `thudxinfgcyaxrfhgoerxtconcfxyfio` int DEFAULT NULL,\n `xkjsaudshzuobawafqstxzdtthbgmplc` int DEFAULT NULL,\n `zjextsggnnrjfgwchsywfnvblvukzqxr` int DEFAULT NULL,\n `qsjmjgnttlqhxdrrktvrazzwztuzqtyq` int DEFAULT NULL,\n `esaeptpummjdiblnkadgblqtapcqeeji` int DEFAULT NULL,\n `cogvwonwqselffpnfnzaajywrmbylmny` int DEFAULT NULL,\n `sovrtwfhysgfqlixvirglwgzrwtjjajw` int DEFAULT NULL,\n `bkjdnymavxgdtgchoolucblnofcvpwvb` int DEFAULT NULL,\n `alyepffdvyvdwrkyjskykscpzhpbjmxi` int DEFAULT NULL,\n `uofyhjblewnxkcamlbaddhdbggzcxqto` int DEFAULT NULL,\n `eihmwyzykdtxgzranthlmdakhffwwjhd` int DEFAULT NULL,\n `vvcighhgeyplnzooyvsdjbbwgccmkvux` int DEFAULT NULL,\n `scfkknxlaowaqhqxfqtudhuygrmixasp` int DEFAULT NULL,\n `ifwqocixudurxeywyncrvnonzptlaalf` int DEFAULT NULL,\n `onnahdhtncmaknddjgnjadpjmqamkmsv` int DEFAULT NULL,\n `xubgbktgfzvbneoxvvviekzvmkgmfdqi` int DEFAULT NULL,\n `nlddyuiancrqpcosqzakutscqjapmzwa` int DEFAULT NULL,\n `xioesqfupnsokmxyaathvrhcvzpglwsx` int DEFAULT NULL,\n `kisfbgpjmoupsziixtaoshfhephuxlgp` int DEFAULT NULL,\n `qolpstnvsbaitgqnklxdrepbhcnoocpl` int DEFAULT NULL,\n `wgtgxdtlbyzeyljnddveawbqlmabydxc` int DEFAULT NULL,\n `lqyxbteiomwdrgskztocuozjdcrypnvq` int DEFAULT NULL,\n `nngdjurtqhnoqqwtpnwxsgqjmpymqivp` int DEFAULT NULL,\n `eijljflxspchwgnnxznsrccwxfmrsvhe` int DEFAULT NULL,\n `mqgoggjvtnrduomuxbicpluomlbxmcid` int DEFAULT NULL,\n `mnpxacgttkbczphqogqawxemkcvxrngo` int DEFAULT NULL,\n `uvktafrqehxgijwibvpuzcfskeoenryw` int DEFAULT NULL,\n `kkldtvcuoqftuzqophoriufnkkacwywm` int DEFAULT NULL,\n `yanmjiwpormyrhbrwqugzgbsbqcgvmfu` int DEFAULT NULL,\n `oeqpwyvfxhtwxawticmxdxlxozuocatq` int DEFAULT NULL,\n `wjeklxcizrsgmntipsqpnhkncylqygyw` int DEFAULT NULL,\n `urgkxgpvuojrarbeuxqbmacxtlwtaxpd` int DEFAULT NULL,\n `ikeckcffkurhooebnagnxvkashltuowg` int DEFAULT NULL,\n `alrwxslpwwgjyvduxgkgmpdjmbmxmmxx` int DEFAULT NULL,\n `jpqylgcqnpsymxmdfoqffuapwzpacngg` int DEFAULT NULL,\n `ozadavxmlxdbmoolrolaslunxzigcyvr` int DEFAULT NULL,\n `vmekeennuttxaajgswxblymjrjqkcuwb` int DEFAULT NULL,\n `xsdgwcxyetzxqnwwdbdwxdtjeevdtrpi` int DEFAULT NULL,\n `zzezxnugykhidheoixozzmtzlbluflie` int DEFAULT NULL,\n `ydrdgupxmxkrxidgwctbtstneupbvgra` int DEFAULT NULL,\n `qfrowoznhcjabuvlzrdaogkgwlqqpokl` int DEFAULT NULL,\n `uilasjnzsnydwfdvhhdlqttonpebpdtf` int DEFAULT NULL,\n `qblzmugxnmmwakezzgmuebendcvpnqwc` int DEFAULT NULL,\n `bffcmkgsnlwvlmvxzugdxirknuiplkdl` int DEFAULT NULL,\n `mfrxamxmgpdptexvdljjhegbifapoutq` int DEFAULT NULL,\n `nnuornqigjjhmipaulekzhdllxtjofqa` int DEFAULT NULL,\n `hqvhoyysafesspobpetmkjdluemxuisf` int DEFAULT NULL,\n `uskrwembkzfazretrspyyppqmdwkoeir` int DEFAULT NULL,\n `xffgwlucmidvruycvrqgyplvelhoadnx` int DEFAULT NULL,\n `kvxzsruonrlbwprofcqjsvtqdrneerql` int DEFAULT NULL,\n `ecefjqvuzfvkuidalicxdtxmdeozjwqz` int DEFAULT NULL,\n `rsdnphglgcmdmufrdwdecswunimwzbqg` int DEFAULT NULL,\n `gsgikkotdwdvambivgzrxhxrfugennco` int DEFAULT NULL,\n `nnpdcunfgouzoclahpcsjhcidgbcbbor` int DEFAULT NULL,\n `sagbyharjoaapjwsplhxebncpsjjmzdr` int DEFAULT NULL,\n `vgtqzzfxnjugladmyvnwcyztfarpltny` int DEFAULT NULL,\n `qgomtfllkaxvzuyganxunasmnumtorpq` int DEFAULT NULL,\n `wwtsjehqdzjlptbrwwjlauxhqbxcviwm` int DEFAULT NULL,\n `fpoiijyvgtuwhlfywbprkjnfgnlcygze` int DEFAULT NULL,\n `prbijrjjpvaoijbknuiwzdqouhsaygnb` int DEFAULT NULL,\n `vkrqtjwebhxlnauuymyqvzvlwfvyhlth` int DEFAULT NULL,\n `raxqcmjsivcqnemwjwnmoyurrhcnfllr` int DEFAULT NULL,\n `pzvugwojewkatfpeazxrpqyemaldoheb` int DEFAULT NULL,\n `dirkyxsbwqwxqyyhjckafxxhdximpnxa` int DEFAULT NULL,\n `pkpoacbxjrjxdfqffwwbzoauiczsxzev` int DEFAULT NULL,\n `fdxoevimdbttyzlqynhovydtmqpucexo` int DEFAULT NULL,\n `zvpgzluzwqejbmrjlacwjcwrqhfiqnex` int DEFAULT NULL,\n `muuypyscnliwadmtzyypatfeqnbbiciz` int DEFAULT NULL,\n `vaskqyugdtisvammbnkjangpuhelhbhq` int DEFAULT NULL,\n `mwbzqbmpuvejpvnaxayyqolfddyempig` int DEFAULT NULL,\n PRIMARY KEY (`trqbqtburynwdzamokaftzhytjymdyhw`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"wchuegpxotggojpmappheurpxfrcivka\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["trqbqtburynwdzamokaftzhytjymdyhw"],"columns":[{"name":"trqbqtburynwdzamokaftzhytjymdyhw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"kamunjhnasyejoulaiurvrffbbikwtfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cpyncfjqeqrxwwuboeycclqgcvptryql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bisobjttedwpkwylusarfiwodftnlduu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wagyeznsqlhnvsikfegrblrpxxfieuas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wirgbojhctbzdluuwsgcmwugsxuepbld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqrswkdyrgjpqcyiptvksjwectbhfxbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"byzfjcaxhhklsntrahvpkeefqsliiujh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzwbyxsyxxurqhiogavxxszfbkbrtnus","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vlehyownqkqkqwqsuangcemdnbavmdfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fkahhrhhmrzwaunxbmqkawggnffloyzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwlwallycumlycvxufndlzoedlhqptup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvfudxhymzhdzfzdkeraxclmpptmqzwh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahocmewiheqeeaqlcnjhhhmfttubjwce","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sxlslkhfymoyhizxfyqzqrmddakzyguw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmzyzzzfdebeobckemyozewsbpsiaemc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qpydqreanxihguylhttnzdxvzsrmbcwy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gteqfvqdkwgxzqiknsrlgalzznjoipjk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmgitshqypraqlbgklrsqaeqoatmnvdi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytgeetiyhwcsurmxmifcqbfaoqlvbhfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpyemxugvjgwkmgiztomwxlixhodltpz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhkhrgdkdgbktomduachzsiykorhiqny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yogwuubuxjtkjuzbhxuoaprnfxzzyadw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjmknhinanipqyhagndciaxgrdctqyyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gssbvdenmqomvkdtgdhqbwpmceeewrrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkmnkgohxzprziewyljlvxipwefxodkx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"trlyinnzizrgkheqsjkscnhimqrkxwfw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"reqnljfmjsiqocsfxnvjiavibgryfueq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnnvmrlqlmxxhpchbxejdfzhzgfxgswc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ipxbyfczksjznkhwopenypxyrbpatlbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thudxinfgcyaxrfhgoerxtconcfxyfio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkjsaudshzuobawafqstxzdtthbgmplc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjextsggnnrjfgwchsywfnvblvukzqxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qsjmjgnttlqhxdrrktvrazzwztuzqtyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"esaeptpummjdiblnkadgblqtapcqeeji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cogvwonwqselffpnfnzaajywrmbylmny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sovrtwfhysgfqlixvirglwgzrwtjjajw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkjdnymavxgdtgchoolucblnofcvpwvb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"alyepffdvyvdwrkyjskykscpzhpbjmxi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uofyhjblewnxkcamlbaddhdbggzcxqto","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eihmwyzykdtxgzranthlmdakhffwwjhd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vvcighhgeyplnzooyvsdjbbwgccmkvux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scfkknxlaowaqhqxfqtudhuygrmixasp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifwqocixudurxeywyncrvnonzptlaalf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"onnahdhtncmaknddjgnjadpjmqamkmsv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xubgbktgfzvbneoxvvviekzvmkgmfdqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nlddyuiancrqpcosqzakutscqjapmzwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xioesqfupnsokmxyaathvrhcvzpglwsx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kisfbgpjmoupsziixtaoshfhephuxlgp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qolpstnvsbaitgqnklxdrepbhcnoocpl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgtgxdtlbyzeyljnddveawbqlmabydxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqyxbteiomwdrgskztocuozjdcrypnvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nngdjurtqhnoqqwtpnwxsgqjmpymqivp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eijljflxspchwgnnxznsrccwxfmrsvhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqgoggjvtnrduomuxbicpluomlbxmcid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnpxacgttkbczphqogqawxemkcvxrngo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvktafrqehxgijwibvpuzcfskeoenryw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kkldtvcuoqftuzqophoriufnkkacwywm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yanmjiwpormyrhbrwqugzgbsbqcgvmfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oeqpwyvfxhtwxawticmxdxlxozuocatq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjeklxcizrsgmntipsqpnhkncylqygyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urgkxgpvuojrarbeuxqbmacxtlwtaxpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikeckcffkurhooebnagnxvkashltuowg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"alrwxslpwwgjyvduxgkgmpdjmbmxmmxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpqylgcqnpsymxmdfoqffuapwzpacngg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozadavxmlxdbmoolrolaslunxzigcyvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmekeennuttxaajgswxblymjrjqkcuwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsdgwcxyetzxqnwwdbdwxdtjeevdtrpi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzezxnugykhidheoixozzmtzlbluflie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydrdgupxmxkrxidgwctbtstneupbvgra","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfrowoznhcjabuvlzrdaogkgwlqqpokl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uilasjnzsnydwfdvhhdlqttonpebpdtf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qblzmugxnmmwakezzgmuebendcvpnqwc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bffcmkgsnlwvlmvxzugdxirknuiplkdl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfrxamxmgpdptexvdljjhegbifapoutq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nnuornqigjjhmipaulekzhdllxtjofqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqvhoyysafesspobpetmkjdluemxuisf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uskrwembkzfazretrspyyppqmdwkoeir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xffgwlucmidvruycvrqgyplvelhoadnx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvxzsruonrlbwprofcqjsvtqdrneerql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecefjqvuzfvkuidalicxdtxmdeozjwqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rsdnphglgcmdmufrdwdecswunimwzbqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsgikkotdwdvambivgzrxhxrfugennco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nnpdcunfgouzoclahpcsjhcidgbcbbor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sagbyharjoaapjwsplhxebncpsjjmzdr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgtqzzfxnjugladmyvnwcyztfarpltny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgomtfllkaxvzuyganxunasmnumtorpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwtsjehqdzjlptbrwwjlauxhqbxcviwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpoiijyvgtuwhlfywbprkjnfgnlcygze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prbijrjjpvaoijbknuiwzdqouhsaygnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkrqtjwebhxlnauuymyqvzvlwfvyhlth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"raxqcmjsivcqnemwjwnmoyurrhcnfllr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pzvugwojewkatfpeazxrpqyemaldoheb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dirkyxsbwqwxqyyhjckafxxhdximpnxa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkpoacbxjrjxdfqffwwbzoauiczsxzev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fdxoevimdbttyzlqynhovydtmqpucexo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvpgzluzwqejbmrjlacwjcwrqhfiqnex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"muuypyscnliwadmtzyypatfeqnbbiciz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vaskqyugdtisvammbnkjangpuhelhbhq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwbzqbmpuvejpvnaxayyqolfddyempig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672301,"databaseName":"models_schema","ddl":"CREATE TABLE `wcvjylwqhadvqltmecxwsuarjnpxvtmi` (\n `hwoioengswxhaousojdmujwdgvnsopyh` int NOT NULL,\n `ygbqztvjtloqokcbjyirnwwbsokopzgo` int DEFAULT NULL,\n `feuwuqeviwyjflxzyqkqehzvzbxgkepi` int DEFAULT NULL,\n `htyrtlqymuijmtqwuhtrdcylstthbdjz` int DEFAULT NULL,\n `kdcyppphiyydjcwdeszamfhxlmyjpagx` int DEFAULT NULL,\n `bzrlkizngsxanpdaeiuygkegkwnpdcud` int DEFAULT NULL,\n `thfachmzoldxuxmvruvejemfycatffea` int DEFAULT NULL,\n `irqxyveshnihmyrfvqgepnmdsfwgcyeh` int DEFAULT NULL,\n `ctexmhdheowplqxglhhuzgexbcmsfzxu` int DEFAULT NULL,\n `fcfvhcggwgxjwubtsqsftwdmdjvuzsvs` int DEFAULT NULL,\n `qhekmvrulfzwhqomtonpqxrjwfecehjj` int DEFAULT NULL,\n `cooghbotyseqoqjxkswtrzubuiketwkv` int DEFAULT NULL,\n `lalojbdcseyuurniymwiladhefvmjmyr` int DEFAULT NULL,\n `nfkxrvbiplyjemyaivmvjtzpzbnwsnbj` int DEFAULT NULL,\n `aceabxehfkmhkzlxqgtjdgrnikjzyexf` int DEFAULT NULL,\n `tjoeapuddyxofdmtjvojtmurohvyzlsq` int DEFAULT NULL,\n `qfllyddgextczsbjcushulttyemnhsor` int DEFAULT NULL,\n `bpnvlypsyeqlrjkjljztkvdeifharapl` int DEFAULT NULL,\n `llaxkrtsikhsipgkgraqmldvxratspac` int DEFAULT NULL,\n `atfatwgkaoioewevtdhpbdcrvpbufzbq` int DEFAULT NULL,\n `tibionnivktavrcmbvdgmqgwlkumnbdw` int DEFAULT NULL,\n `zivtchdhlmmgkknsvfdtvgksripxaxdm` int DEFAULT NULL,\n `dcvxwuurkvbxhhhxrirwooxyqntfpyda` int DEFAULT NULL,\n `vdhszuhuhdkwejejrwnoaumzcemnyyyv` int DEFAULT NULL,\n `cgiiygdpnxkjshmeyigwpdyqarfsdrlm` int DEFAULT NULL,\n `ivwqepkqurbgcvjoclcedrrmunbwzgcc` int DEFAULT NULL,\n `aoydzuijqgdrrkovehbazlhahdylrysb` int DEFAULT NULL,\n `efqwwrfqeudiccbdfyfvqxqbtoquxdfc` int DEFAULT NULL,\n `qymbfbekrirftzxqxwkkhbunwzpgowly` int DEFAULT NULL,\n `ubieewgdbssyyipcfdyronqqtxaszugz` int DEFAULT NULL,\n `tvndjshgrdnsylnqlusrogvgwnuhxfot` int DEFAULT NULL,\n `ekgepsgzmehkwimqhijdpfkguhsuvrju` int DEFAULT NULL,\n `nugfijbcchbapcxviwvtajlxsqninkvv` int DEFAULT NULL,\n `bmisuoqernejjwotpabuptjxgxvmiuaf` int DEFAULT NULL,\n `orstxtgkmjfujkzdfezzzgurxjjnnrsx` int DEFAULT NULL,\n `ldmqrojxwfegpswwjyfhsvmoodlfhysk` int DEFAULT NULL,\n `guueyudwagfpywuqwebhwlwedswipvyz` int DEFAULT NULL,\n `dmynfoeovrampfosgfzbgiijquqrzffj` int DEFAULT NULL,\n `qrlfjniwancjsmmzzllytirlxzyklnhe` int DEFAULT NULL,\n `ciismtfakyogfrgmnopyyvoljnxpcfok` int DEFAULT NULL,\n `ydlvvtscunvlkjcboiyvpxfdufkvicnt` int DEFAULT NULL,\n `cjqojygzceqcxzsjrnlqnbdjppfzieez` int DEFAULT NULL,\n `gigqtzinxcujsfcyilfuilbebugfxfrx` int DEFAULT NULL,\n `arfubkvatkammcgrvmsxpkmufirrnwtn` int DEFAULT NULL,\n `umvvthxbhpwexcveeizfqzdbpymkbmwr` int DEFAULT NULL,\n `rnqqwnstcqenldhbrqxevwlqncqxpcjd` int DEFAULT NULL,\n `oaevbujhmdcfciwjnzqvlvxpwywgkkmh` int DEFAULT NULL,\n `rsdkhcfwevzqrhejwcpvqjvmhhqxwagw` int DEFAULT NULL,\n `xqkcbrjsjarwjqhtiuovrdknhwhxbzzc` int DEFAULT NULL,\n `wxfdvsdqycregiqjcluvwqovcljpdhwm` int DEFAULT NULL,\n `ewdmaueswsnvtamnvyrjpzzsvkusmlhx` int DEFAULT NULL,\n `iglydquyfpwuzzcmtuxsxbgxfjoygslt` int DEFAULT NULL,\n `xemelbrvukgxgotflodumkwbijugkmuj` int DEFAULT NULL,\n `ansvojrejxjxpmxzbqaykeajptwukzzd` int DEFAULT NULL,\n `evvmwxonaeljshtoakhlqxupnmhgfqyi` int DEFAULT NULL,\n `gkbvkgeivyonlqdwpjnfrfjlkpabaitb` int DEFAULT NULL,\n `fcardrwvqxxknxbwgwjmnyptmlrgoefk` int DEFAULT NULL,\n `ptznegdptdmusfgybzxouonmazshnqiu` int DEFAULT NULL,\n `clhgzduwohmaestjvztplikzbkjogyrn` int DEFAULT NULL,\n `vzipwkisxihdbosmizhvwheqfsdsbkzm` int DEFAULT NULL,\n `vupdrleouoksarjennetiwahcbyajvan` int DEFAULT NULL,\n `jmcvzfpwtsmnlznqnemxfwhlikpaepcc` int DEFAULT NULL,\n `ttuczbmjtrsdzuoimwamqmiufbakuctp` int DEFAULT NULL,\n `hkdoqlsvqzuxluaegoftjwlvqtvqsipa` int DEFAULT NULL,\n `ufphkvkpysonzjzcifwzvvkkapjduiiu` int DEFAULT NULL,\n `iwtirasozsqpuhrgyznvoztxwnkqjqee` int DEFAULT NULL,\n `rpvgittpylqbkwjmhgyftaooapiihdyd` int DEFAULT NULL,\n `vmbudvewhuupybaaenxvfdiiosmpnkqx` int DEFAULT NULL,\n `ziydaskwmbmzqzndepahnzhxkcugbpau` int DEFAULT NULL,\n `indslcrvnykjqybhggrmvkaalooecnyr` int DEFAULT NULL,\n `nmqvageavaziigihslzyhupjwdffmhxr` int DEFAULT NULL,\n `mwjbvnzgkjtgbdiuyvtyriylwsqiavji` int DEFAULT NULL,\n `chetikcyrvpzdtrxkhcxzwdikoifkrpy` int DEFAULT NULL,\n `ffdrgxhihlcwcqhbfqkbblrqkxtbuorc` int DEFAULT NULL,\n `ttcczsqeehuqdqvyshztehqcbnwpvywo` int DEFAULT NULL,\n `ekzclyosbktznklbmqakiylllqajkryf` int DEFAULT NULL,\n `wfebxidsjujlxbhricjsldxjhayaounf` int DEFAULT NULL,\n `uascfyeeiiwtltjjkdtokmgehxwrkcdk` int DEFAULT NULL,\n `dtgzsrsxfrtloakyouzxipvtumatebcz` int DEFAULT NULL,\n `rkokbsqvurbfhlqinzkubdwecwdbkloc` int DEFAULT NULL,\n `wftsktxuksnjooupcswrgbkdxsebabdf` int DEFAULT NULL,\n `vitcxfaumbxhmqxolzouflyfjvukqqpu` int DEFAULT NULL,\n `bwfyvozmahoyzktvgbrgcgmcuwjsmvna` int DEFAULT NULL,\n `mfwoslwjydguhakdqszibvmtnwwawxhd` int DEFAULT NULL,\n `lrehbsnttsjikwmghljoavzsblntwgzk` int DEFAULT NULL,\n `nxpwmzxmavtvgvzwmjbmvhnmovyhsnsn` int DEFAULT NULL,\n `otntcgipufmxhnfznoivtaxmzlgefapl` int DEFAULT NULL,\n `zmztthnesyrfspevnbqzwmtdpuyezono` int DEFAULT NULL,\n `xzdtdbpzznetmqfmbltobifsgtwunauv` int DEFAULT NULL,\n `qtesgwthyvurcuivzumkjalozyoexztj` int DEFAULT NULL,\n `ymllbqepbwydpnbhcwfxvgkzmwtegeys` int DEFAULT NULL,\n `tzltastygoiuxdxepxtuikajmvtnxdrz` int DEFAULT NULL,\n `cdvbduswwyjdmaexhdbgsxbgpvtmjasn` int DEFAULT NULL,\n `htfaxdanncjomxzpiolwovrvlpyfjoku` int DEFAULT NULL,\n `ntpaclusbqhreukppwkofpgmjnmhvfyt` int DEFAULT NULL,\n `hlxngrpefhcfnfdiylayduskfopkdwum` int DEFAULT NULL,\n `hxvsqkgskhdyeafzawexyqmlqrzmmlbk` int DEFAULT NULL,\n `bzaaykqpphyiozxfsprqakpcytnohshq` int DEFAULT NULL,\n `zkzkdvzxojxuddueqjsgxwmrwrjygyre` int DEFAULT NULL,\n `vppbxuhfjhgjwaaeprbsdtgsyvyzvosx` int DEFAULT NULL,\n PRIMARY KEY (`hwoioengswxhaousojdmujwdgvnsopyh`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"wcvjylwqhadvqltmecxwsuarjnpxvtmi\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["hwoioengswxhaousojdmujwdgvnsopyh"],"columns":[{"name":"hwoioengswxhaousojdmujwdgvnsopyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ygbqztvjtloqokcbjyirnwwbsokopzgo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"feuwuqeviwyjflxzyqkqehzvzbxgkepi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htyrtlqymuijmtqwuhtrdcylstthbdjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kdcyppphiyydjcwdeszamfhxlmyjpagx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzrlkizngsxanpdaeiuygkegkwnpdcud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thfachmzoldxuxmvruvejemfycatffea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"irqxyveshnihmyrfvqgepnmdsfwgcyeh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctexmhdheowplqxglhhuzgexbcmsfzxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcfvhcggwgxjwubtsqsftwdmdjvuzsvs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhekmvrulfzwhqomtonpqxrjwfecehjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cooghbotyseqoqjxkswtrzubuiketwkv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lalojbdcseyuurniymwiladhefvmjmyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfkxrvbiplyjemyaivmvjtzpzbnwsnbj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aceabxehfkmhkzlxqgtjdgrnikjzyexf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjoeapuddyxofdmtjvojtmurohvyzlsq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfllyddgextczsbjcushulttyemnhsor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bpnvlypsyeqlrjkjljztkvdeifharapl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llaxkrtsikhsipgkgraqmldvxratspac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atfatwgkaoioewevtdhpbdcrvpbufzbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tibionnivktavrcmbvdgmqgwlkumnbdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zivtchdhlmmgkknsvfdtvgksripxaxdm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcvxwuurkvbxhhhxrirwooxyqntfpyda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdhszuhuhdkwejejrwnoaumzcemnyyyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgiiygdpnxkjshmeyigwpdyqarfsdrlm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivwqepkqurbgcvjoclcedrrmunbwzgcc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aoydzuijqgdrrkovehbazlhahdylrysb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"efqwwrfqeudiccbdfyfvqxqbtoquxdfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qymbfbekrirftzxqxwkkhbunwzpgowly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubieewgdbssyyipcfdyronqqtxaszugz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvndjshgrdnsylnqlusrogvgwnuhxfot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekgepsgzmehkwimqhijdpfkguhsuvrju","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nugfijbcchbapcxviwvtajlxsqninkvv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bmisuoqernejjwotpabuptjxgxvmiuaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orstxtgkmjfujkzdfezzzgurxjjnnrsx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ldmqrojxwfegpswwjyfhsvmoodlfhysk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"guueyudwagfpywuqwebhwlwedswipvyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dmynfoeovrampfosgfzbgiijquqrzffj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrlfjniwancjsmmzzllytirlxzyklnhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ciismtfakyogfrgmnopyyvoljnxpcfok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydlvvtscunvlkjcboiyvpxfdufkvicnt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjqojygzceqcxzsjrnlqnbdjppfzieez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gigqtzinxcujsfcyilfuilbebugfxfrx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"arfubkvatkammcgrvmsxpkmufirrnwtn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umvvthxbhpwexcveeizfqzdbpymkbmwr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnqqwnstcqenldhbrqxevwlqncqxpcjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oaevbujhmdcfciwjnzqvlvxpwywgkkmh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rsdkhcfwevzqrhejwcpvqjvmhhqxwagw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqkcbrjsjarwjqhtiuovrdknhwhxbzzc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxfdvsdqycregiqjcluvwqovcljpdhwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ewdmaueswsnvtamnvyrjpzzsvkusmlhx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iglydquyfpwuzzcmtuxsxbgxfjoygslt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xemelbrvukgxgotflodumkwbijugkmuj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ansvojrejxjxpmxzbqaykeajptwukzzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evvmwxonaeljshtoakhlqxupnmhgfqyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkbvkgeivyonlqdwpjnfrfjlkpabaitb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcardrwvqxxknxbwgwjmnyptmlrgoefk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ptznegdptdmusfgybzxouonmazshnqiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clhgzduwohmaestjvztplikzbkjogyrn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzipwkisxihdbosmizhvwheqfsdsbkzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vupdrleouoksarjennetiwahcbyajvan","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmcvzfpwtsmnlznqnemxfwhlikpaepcc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ttuczbmjtrsdzuoimwamqmiufbakuctp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkdoqlsvqzuxluaegoftjwlvqtvqsipa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufphkvkpysonzjzcifwzvvkkapjduiiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwtirasozsqpuhrgyznvoztxwnkqjqee","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpvgittpylqbkwjmhgyftaooapiihdyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmbudvewhuupybaaenxvfdiiosmpnkqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ziydaskwmbmzqzndepahnzhxkcugbpau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"indslcrvnykjqybhggrmvkaalooecnyr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmqvageavaziigihslzyhupjwdffmhxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwjbvnzgkjtgbdiuyvtyriylwsqiavji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chetikcyrvpzdtrxkhcxzwdikoifkrpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffdrgxhihlcwcqhbfqkbblrqkxtbuorc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ttcczsqeehuqdqvyshztehqcbnwpvywo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekzclyosbktznklbmqakiylllqajkryf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfebxidsjujlxbhricjsldxjhayaounf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uascfyeeiiwtltjjkdtokmgehxwrkcdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtgzsrsxfrtloakyouzxipvtumatebcz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkokbsqvurbfhlqinzkubdwecwdbkloc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wftsktxuksnjooupcswrgbkdxsebabdf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vitcxfaumbxhmqxolzouflyfjvukqqpu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bwfyvozmahoyzktvgbrgcgmcuwjsmvna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfwoslwjydguhakdqszibvmtnwwawxhd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lrehbsnttsjikwmghljoavzsblntwgzk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxpwmzxmavtvgvzwmjbmvhnmovyhsnsn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otntcgipufmxhnfznoivtaxmzlgefapl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmztthnesyrfspevnbqzwmtdpuyezono","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xzdtdbpzznetmqfmbltobifsgtwunauv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtesgwthyvurcuivzumkjalozyoexztj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ymllbqepbwydpnbhcwfxvgkzmwtegeys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tzltastygoiuxdxepxtuikajmvtnxdrz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cdvbduswwyjdmaexhdbgsxbgpvtmjasn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htfaxdanncjomxzpiolwovrvlpyfjoku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntpaclusbqhreukppwkofpgmjnmhvfyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlxngrpefhcfnfdiylayduskfopkdwum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxvsqkgskhdyeafzawexyqmlqrzmmlbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzaaykqpphyiozxfsprqakpcytnohshq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zkzkdvzxojxuddueqjsgxwmrwrjygyre","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vppbxuhfjhgjwaaeprbsdtgsyvyzvosx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672331,"databaseName":"models_schema","ddl":"CREATE TABLE `wgkwfskbehhqasowhsdegiiatybhbuvq` (\n `wngtjidkzbxyqfvnbhsplsvmbaakaowi` int NOT NULL,\n `idnbjufpztlxenejintqgwlueduzeimu` int DEFAULT NULL,\n `qhipawwsephogoavtbjjecmdjlxwbigx` int DEFAULT NULL,\n `avncxxgyiemwnwazmmfhzzwvnmymndcf` int DEFAULT NULL,\n `pbxlayfwcwuyieqwewbtmybdwyxemdhg` int DEFAULT NULL,\n `aclzfelsumlqcsttliytaqcpvqwbgmzf` int DEFAULT NULL,\n `ijzuseyymrwvfezdjffmittfvmidncay` int DEFAULT NULL,\n `uctahhaypnbtzsgnxxicloqzczxorhfy` int DEFAULT NULL,\n `fccdyueesgyxidhhytydmuhkcrwevveb` int DEFAULT NULL,\n `ojryabeloinapvyrllobrfyxmjlqttou` int DEFAULT NULL,\n `ykuorlcqrkuihkdpozhlzxxtnzsbpgkd` int DEFAULT NULL,\n `jsvfxivrnyhknotpwafqtponxqnzutwv` int DEFAULT NULL,\n `atykownfaftuhsolxxkferlnvhvbzjpi` int DEFAULT NULL,\n `vtdevtprceyidyvqppjoeylgycluldny` int DEFAULT NULL,\n `ykqavxnfxtiflsxshwkxpzwsjbzgiaqz` int DEFAULT NULL,\n `rgncvtyucrnklacmuqpsamvpdwjenfsu` int DEFAULT NULL,\n `yvaipssaizqzylunsjjnncuyhkqjrlrj` int DEFAULT NULL,\n `ygmrhxftjqzrxssqdjbgbdzkhgsvynno` int DEFAULT NULL,\n `qtcxcodzuudnedkqbclpodlcuqxgwwgx` int DEFAULT NULL,\n `pwmzdxkhpjssxwqocjigrfzgamxbstvn` int DEFAULT NULL,\n `hbqjjrryjwjoncnifnucdeplsxumosaj` int DEFAULT NULL,\n `cozhasgpqxctksyostboogzqarusbnil` int DEFAULT NULL,\n `iugnowxfoqlcmpdtnqwpztrzgbbyimfd` int DEFAULT NULL,\n `npkxxzhfeisfjpyywzslieeuminbclnf` int DEFAULT NULL,\n `iigoubmfwgaxbwoyntptcgscmydoxnot` int DEFAULT NULL,\n `shumctmjlviqbmqrguprcwgffmkbezli` int DEFAULT NULL,\n `kvqdhnduxlmnkpprrxtktehcolntnntj` int DEFAULT NULL,\n `bwnzwxowngzxqlzmcqwkyyndvtfsezeh` int DEFAULT NULL,\n `qgboqplszuqzmvyfhrgeuakefatttygx` int DEFAULT NULL,\n `pjvbnhcxrrgwxkfwimgvykpxaeafcdjg` int DEFAULT NULL,\n `idqexatvemyyjkqaaaxajifagirywric` int DEFAULT NULL,\n `lqmyqkojllwryubabspqshnfqmmssvcq` int DEFAULT NULL,\n `ruioebylapawmynxcobojzvshygudkit` int DEFAULT NULL,\n `bukbelzqqysdblzuuezydbgzjgcynydz` int DEFAULT NULL,\n `nwebqnqwfzvamhdxyihsishtwdjrbrzd` int DEFAULT NULL,\n `xoyrusggrrxkpfuznwnpqmwdfulmqhko` int DEFAULT NULL,\n `tifjvhiauhphbjtcbqhemmburmwdnrai` int DEFAULT NULL,\n `wbnbmkfkhdcjqqbbwlsmtimrcmtgbiyz` int DEFAULT NULL,\n `qmiihnyjrbdwygaplaigwzlcvlapkxcg` int DEFAULT NULL,\n `ojpftajeeuzvewdpvjmpvlzsroscjjuj` int DEFAULT NULL,\n `fxqeopxznmbvbbkdnwuwoujitkbfmchh` int DEFAULT NULL,\n `jbjeskbipmrioslqdxamkmagcxugbxcn` int DEFAULT NULL,\n `yhlfavzfeeurdavchlllggoieglffscn` int DEFAULT NULL,\n `hqjkzkoiyqepbirgosfsvxiovwdumenw` int DEFAULT NULL,\n `unfwniyatrnnxcteygbcunsjwbgvtzik` int DEFAULT NULL,\n `icccxtisvfivgsjgffvsicwgpkprkwtz` int DEFAULT NULL,\n `kfxvhkgzgxopdnnovobylolabnyhkezc` int DEFAULT NULL,\n `iusqtdsmprdjbywddqfzixxczsjcnxqt` int DEFAULT NULL,\n `nddxipbyoxllvlbkwltdhzovlkoybxac` int DEFAULT NULL,\n `fbjyzjfiuamvakbdicdgpealgsvpzavi` int DEFAULT NULL,\n `rhhvsocwkhdoxvrruguykwtxifwyyoub` int DEFAULT NULL,\n `ctcygqmfvletgqrykiakhhxkhxuqwltm` int DEFAULT NULL,\n `xoderbzxkkyzvhetmbpjapsptnaoluox` int DEFAULT NULL,\n `wnuionqtjpnzvrfndkvkoofzelzutqgr` int DEFAULT NULL,\n `muqelelqojexbdqjoyvwwonuufbrllky` int DEFAULT NULL,\n `udunvovzequlajmcqjkfqgbiicfzcthx` int DEFAULT NULL,\n `kjkkjhlfuifwdvezwwvsaorvtpafnmwx` int DEFAULT NULL,\n `hguzoyeewjnhjvbdlfvtahlysuoyeler` int DEFAULT NULL,\n `dihkaujxykcjqlvalychgdyrskqbivje` int DEFAULT NULL,\n `bkyoqxsywszjalfskggsyuqbtruxwgfz` int DEFAULT NULL,\n `uvbxszqnmxhwrfecbjmskbnyacetnzcb` int DEFAULT NULL,\n `flluenxkmqtbfcovwxazadqnktmsmtdr` int DEFAULT NULL,\n `xmzpjxjwlmvoogbxiovjfjlpfggbgvwr` int DEFAULT NULL,\n `itwatstdacgkqsamqoqlkjnqywqezmue` int DEFAULT NULL,\n `esmtaochigskysjbjhhlnxuqmdxemhyk` int DEFAULT NULL,\n `eumahjifujiadmepqakplblwvorhobeu` int DEFAULT NULL,\n `nvvrmjkoiyrhqdgrqoemfyrihmduhmkh` int DEFAULT NULL,\n `gtimunxvilzraeehifbebblvdkhpzuzh` int DEFAULT NULL,\n `mdbuanrlmnmkjnujyjnrnyjidmglhrdy` int DEFAULT NULL,\n `jvhuqwzlzaeorgoxezvhyvvktfdynuit` int DEFAULT NULL,\n `abdxjjlxyuttrjmtpyjlcwufsxntcawe` int DEFAULT NULL,\n `eqycfwvjepbvcadassdapiicrnrbntrc` int DEFAULT NULL,\n `sowbrqfbgrguefspixrifawgmutsussk` int DEFAULT NULL,\n `jenvrwtokefndwzlwhewvtxlyhuatefk` int DEFAULT NULL,\n `wmtmqygrjdbuahhxiftpwykhuhgelgnr` int DEFAULT NULL,\n `hqwujevhtvyxillznibclybhxnqadjxf` int DEFAULT NULL,\n `sqrisnspgcqjdeoanresgdfjitrlwfdh` int DEFAULT NULL,\n `fnyqmmelcvzsbokmvqzzlqhvwruusqvx` int DEFAULT NULL,\n `gnuyvygkwrjqcsvkehdpuatcvbshpviz` int DEFAULT NULL,\n `szegdjazeqewnbisjkvggyoizrggoahg` int DEFAULT NULL,\n `vrkemjbxkwtldlxfqgnsasinuwbeuxga` int DEFAULT NULL,\n `rphivoooubkscppyoigakpoepbkwryrz` int DEFAULT NULL,\n `ijvwoehbifwmczejkwyewjgrypeabyxy` int DEFAULT NULL,\n `jtftlhvuapstsixflyychtgnytcoizuv` int DEFAULT NULL,\n `zqichlqhrnjsduvibbiikpcrnnsaecol` int DEFAULT NULL,\n `ihxtcxkatyslxwfbzrvmlmetyhujgmlg` int DEFAULT NULL,\n `iigwtzrkxqurlxntkerfkbbotxwoadwf` int DEFAULT NULL,\n `hotgsvlvxfizunlgvlwsvesizcsrrtbk` int DEFAULT NULL,\n `pjwzxnwwxcpkvmeikfzhtlqzxbfhoykz` int DEFAULT NULL,\n `bzphjmlneqenwmwkogosdouugfjolzhu` int DEFAULT NULL,\n `bzchskzwyxakgzclqpahmzeyquxtqwfw` int DEFAULT NULL,\n `nmmvwdpotcyrjcxloalbujzclxiudqub` int DEFAULT NULL,\n `csqhfoxfnwovkchocvgfjbynuwrvizai` int DEFAULT NULL,\n `qvklpvlyjhilckmapsvuanizauuzxbxv` int DEFAULT NULL,\n `vsxybpwwaxmglvydhrcesludrkqzypcx` int DEFAULT NULL,\n `zbpobsbwzjdmndpbqrybmfyeztbgvcvd` int DEFAULT NULL,\n `fmdpjcgapfvxrekxlxjqexegxtppuagu` int DEFAULT NULL,\n `qofkfbxjrmadytikokokwrnbtorvkcap` int DEFAULT NULL,\n `tndvfvqotdxowrqzztedevgnzuwgnrwu` int DEFAULT NULL,\n `enwdqwotsozabhrtcjdrnzhmdpjspzfu` int DEFAULT NULL,\n PRIMARY KEY (`wngtjidkzbxyqfvnbhsplsvmbaakaowi`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"wgkwfskbehhqasowhsdegiiatybhbuvq\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["wngtjidkzbxyqfvnbhsplsvmbaakaowi"],"columns":[{"name":"wngtjidkzbxyqfvnbhsplsvmbaakaowi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"idnbjufpztlxenejintqgwlueduzeimu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhipawwsephogoavtbjjecmdjlxwbigx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avncxxgyiemwnwazmmfhzzwvnmymndcf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbxlayfwcwuyieqwewbtmybdwyxemdhg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aclzfelsumlqcsttliytaqcpvqwbgmzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijzuseyymrwvfezdjffmittfvmidncay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uctahhaypnbtzsgnxxicloqzczxorhfy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fccdyueesgyxidhhytydmuhkcrwevveb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojryabeloinapvyrllobrfyxmjlqttou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykuorlcqrkuihkdpozhlzxxtnzsbpgkd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jsvfxivrnyhknotpwafqtponxqnzutwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atykownfaftuhsolxxkferlnvhvbzjpi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtdevtprceyidyvqppjoeylgycluldny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykqavxnfxtiflsxshwkxpzwsjbzgiaqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgncvtyucrnklacmuqpsamvpdwjenfsu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yvaipssaizqzylunsjjnncuyhkqjrlrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ygmrhxftjqzrxssqdjbgbdzkhgsvynno","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtcxcodzuudnedkqbclpodlcuqxgwwgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwmzdxkhpjssxwqocjigrfzgamxbstvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbqjjrryjwjoncnifnucdeplsxumosaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cozhasgpqxctksyostboogzqarusbnil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iugnowxfoqlcmpdtnqwpztrzgbbyimfd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npkxxzhfeisfjpyywzslieeuminbclnf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iigoubmfwgaxbwoyntptcgscmydoxnot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shumctmjlviqbmqrguprcwgffmkbezli","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvqdhnduxlmnkpprrxtktehcolntnntj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bwnzwxowngzxqlzmcqwkyyndvtfsezeh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgboqplszuqzmvyfhrgeuakefatttygx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjvbnhcxrrgwxkfwimgvykpxaeafcdjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idqexatvemyyjkqaaaxajifagirywric","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqmyqkojllwryubabspqshnfqmmssvcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ruioebylapawmynxcobojzvshygudkit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bukbelzqqysdblzuuezydbgzjgcynydz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nwebqnqwfzvamhdxyihsishtwdjrbrzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xoyrusggrrxkpfuznwnpqmwdfulmqhko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tifjvhiauhphbjtcbqhemmburmwdnrai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbnbmkfkhdcjqqbbwlsmtimrcmtgbiyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmiihnyjrbdwygaplaigwzlcvlapkxcg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojpftajeeuzvewdpvjmpvlzsroscjjuj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxqeopxznmbvbbkdnwuwoujitkbfmchh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbjeskbipmrioslqdxamkmagcxugbxcn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhlfavzfeeurdavchlllggoieglffscn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqjkzkoiyqepbirgosfsvxiovwdumenw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unfwniyatrnnxcteygbcunsjwbgvtzik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icccxtisvfivgsjgffvsicwgpkprkwtz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfxvhkgzgxopdnnovobylolabnyhkezc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iusqtdsmprdjbywddqfzixxczsjcnxqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nddxipbyoxllvlbkwltdhzovlkoybxac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbjyzjfiuamvakbdicdgpealgsvpzavi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhhvsocwkhdoxvrruguykwtxifwyyoub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctcygqmfvletgqrykiakhhxkhxuqwltm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xoderbzxkkyzvhetmbpjapsptnaoluox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnuionqtjpnzvrfndkvkoofzelzutqgr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"muqelelqojexbdqjoyvwwonuufbrllky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udunvovzequlajmcqjkfqgbiicfzcthx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjkkjhlfuifwdvezwwvsaorvtpafnmwx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hguzoyeewjnhjvbdlfvtahlysuoyeler","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dihkaujxykcjqlvalychgdyrskqbivje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkyoqxsywszjalfskggsyuqbtruxwgfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvbxszqnmxhwrfecbjmskbnyacetnzcb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flluenxkmqtbfcovwxazadqnktmsmtdr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmzpjxjwlmvoogbxiovjfjlpfggbgvwr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itwatstdacgkqsamqoqlkjnqywqezmue","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"esmtaochigskysjbjhhlnxuqmdxemhyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eumahjifujiadmepqakplblwvorhobeu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvvrmjkoiyrhqdgrqoemfyrihmduhmkh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtimunxvilzraeehifbebblvdkhpzuzh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdbuanrlmnmkjnujyjnrnyjidmglhrdy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvhuqwzlzaeorgoxezvhyvvktfdynuit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abdxjjlxyuttrjmtpyjlcwufsxntcawe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqycfwvjepbvcadassdapiicrnrbntrc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sowbrqfbgrguefspixrifawgmutsussk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jenvrwtokefndwzlwhewvtxlyhuatefk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmtmqygrjdbuahhxiftpwykhuhgelgnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqwujevhtvyxillznibclybhxnqadjxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sqrisnspgcqjdeoanresgdfjitrlwfdh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnyqmmelcvzsbokmvqzzlqhvwruusqvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnuyvygkwrjqcsvkehdpuatcvbshpviz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"szegdjazeqewnbisjkvggyoizrggoahg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrkemjbxkwtldlxfqgnsasinuwbeuxga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rphivoooubkscppyoigakpoepbkwryrz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijvwoehbifwmczejkwyewjgrypeabyxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jtftlhvuapstsixflyychtgnytcoizuv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqichlqhrnjsduvibbiikpcrnnsaecol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ihxtcxkatyslxwfbzrvmlmetyhujgmlg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iigwtzrkxqurlxntkerfkbbotxwoadwf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hotgsvlvxfizunlgvlwsvesizcsrrtbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjwzxnwwxcpkvmeikfzhtlqzxbfhoykz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzphjmlneqenwmwkogosdouugfjolzhu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzchskzwyxakgzclqpahmzeyquxtqwfw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmmvwdpotcyrjcxloalbujzclxiudqub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"csqhfoxfnwovkchocvgfjbynuwrvizai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvklpvlyjhilckmapsvuanizauuzxbxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsxybpwwaxmglvydhrcesludrkqzypcx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbpobsbwzjdmndpbqrybmfyeztbgvcvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmdpjcgapfvxrekxlxjqexegxtppuagu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qofkfbxjrmadytikokokwrnbtorvkcap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tndvfvqotdxowrqzztedevgnzuwgnrwu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"enwdqwotsozabhrtcjdrnzhmdpjspzfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672362,"databaseName":"models_schema","ddl":"CREATE TABLE `wgxcetmtjwvmrtdbuchydagtylsvtsga` (\n `gqzqnjjgsqkiyalyawyyfmekkcnzjvuy` int NOT NULL,\n `inmqwklxqocieqgakzwoisdxdkfrgkws` int DEFAULT NULL,\n `pziazvkohclbfsglwcavbhsgasavareg` int DEFAULT NULL,\n `fjltkcirxbtdwsjqzjpyvrmdisqqzvki` int DEFAULT NULL,\n `xdyaotrrazcvbcptwtomkiwmjwlszlln` int DEFAULT NULL,\n `lgqsolmlscebqadvbopkjmojoevrlxpk` int DEFAULT NULL,\n `bmqpcqxhejcisybzbkgpibtrqlmosdwm` int DEFAULT NULL,\n `fghulyfoucmwrfcfgqeyoxhvtjechywg` int DEFAULT NULL,\n `lijtfygdmtvqcrubdgjozfxreqawagkp` int DEFAULT NULL,\n `kcemdhtcthfvertknwrfdreqwqahwoxd` int DEFAULT NULL,\n `xyqytpzavrhplxxughnpzhromhhhwfgh` int DEFAULT NULL,\n `oudznhuyguuvomjtibvinnjepkilffhx` int DEFAULT NULL,\n `heviqfogruqzockmnrkdjrvgvxnbrjhf` int DEFAULT NULL,\n `qczdrjfscdfgrvczwyilqsswrczyczwx` int DEFAULT NULL,\n `jjrxgbmnvhegaaupqkptiopevicudscw` int DEFAULT NULL,\n `fyudlauckweorkcjgexbxvcjpaejxlnq` int DEFAULT NULL,\n `lleiektpilqxgvifcyinsjfemkecrqhj` int DEFAULT NULL,\n `jivfspszznyhfoqbdzokkbtwhdphivxg` int DEFAULT NULL,\n `nmpvekckqpeeyydstphamlvcqmqcxzmu` int DEFAULT NULL,\n `llpjarrrxnskmzsqszahagexczrqemqq` int DEFAULT NULL,\n `xydblwakmaezllsxhraotadcctitfmdu` int DEFAULT NULL,\n `mubfkjnlkkzwctpknecioltotplylzve` int DEFAULT NULL,\n `irjlklxkqhpizoikbmnusulcirceroxw` int DEFAULT NULL,\n `ydfjsicdlnkiabcawmkedgnnlcfvewxu` int DEFAULT NULL,\n `oczfwydeyhnbpundsdjtpbujlxndhkhm` int DEFAULT NULL,\n `cmlqxruytwqzdskpfsbhorwyhdcqjfna` int DEFAULT NULL,\n `fofwrxsnpysukcnshtcdxqgdtnnrkwbh` int DEFAULT NULL,\n `wsrasyfhxwprpfbfeqsadniqacxoaqxx` int DEFAULT NULL,\n `iylebdnmcpxuzfwmhjsaymhligzqielj` int DEFAULT NULL,\n `czbuexwcgrekffwavimwgiypkazsemxc` int DEFAULT NULL,\n `jzwhuisgyycskpcsfnybgnlqgagnoqjm` int DEFAULT NULL,\n `fzucglaerqyubjhnoxjldrepnyiapazd` int DEFAULT NULL,\n `tkzosqwlnavcnfalutewkgefzuhqcnuw` int DEFAULT NULL,\n `lhdeczcrxuffkfmfuoygeafjwqkmorxw` int DEFAULT NULL,\n `ksgvfpmxxflruularghecmpagiqivebr` int DEFAULT NULL,\n `bnlwuulakxtkxqfajtjgczfdjjvlirlr` int DEFAULT NULL,\n `gffzhewvzhogjawfedirezbeesjysryp` int DEFAULT NULL,\n `yufjdbalqpxtcjttddayqldihxfpkqul` int DEFAULT NULL,\n `wnbruwoitsjnxcxfzglkbinmtyienhwe` int DEFAULT NULL,\n `whpszpidmrjaerabjqijpvhwlliqzijv` int DEFAULT NULL,\n `ruucbaotcphcgfosknukjrlkkqwfwyti` int DEFAULT NULL,\n `xcvandoirxnwlyovhjbfsefocibzdxvy` int DEFAULT NULL,\n `dwwuehghbtsmzecgcdklzterfntxgfcy` int DEFAULT NULL,\n `shfvsyiylwhazlpnmpzdndnmpqfeignd` int DEFAULT NULL,\n `uvhjsarhuyvfyqqsjfohjhxahvshmvnt` int DEFAULT NULL,\n `foruycpxrwirdkkaqnwyplxiegyypyex` int DEFAULT NULL,\n `gbxbgibgtubvqkiktyqpuiyrkfcpbzcz` int DEFAULT NULL,\n `oiwlzyunpdinfjdwodrlembvrqzdmdig` int DEFAULT NULL,\n `rnztrhqlnnruofghmftqqvtpsalwwlis` int DEFAULT NULL,\n `lsbprvfhibbffksylusvxvfidlwavoim` int DEFAULT NULL,\n `qjnaqucyvvnfoaveidxbopekunodbxkn` int DEFAULT NULL,\n `yktrfbsxccbbuyvvboqwoeidfkgspnyd` int DEFAULT NULL,\n `gbwcpvvjjtovpiykkusqehloqqcwkzve` int DEFAULT NULL,\n `xyhzrzmgarjgvkmifehldmxtavvemzfx` int DEFAULT NULL,\n `mzjjflzmmlsbhcwsxsmvqxitdvhqklja` int DEFAULT NULL,\n `uchuvhxbqycgvpltwxbzqnsggeaqcqta` int DEFAULT NULL,\n `vqmgbnpwzoaxmqmcybssapwkzvdooyqc` int DEFAULT NULL,\n `zujjuvrazfvbkpadjpxcpbjyfcgniyak` int DEFAULT NULL,\n `kpgccoadlaupgwtkxtnykovhhgjuegos` int DEFAULT NULL,\n `wwohvgjvqczawxxzkpqvytkpbzuvqcxc` int DEFAULT NULL,\n `kadvprstslnekefqsotrovnsmnyukccg` int DEFAULT NULL,\n `wnhwfwurzrkxmkxtlevvkfvxbxpjbghs` int DEFAULT NULL,\n `ovgsupkalbtbxddxlzphkzbcqkyvrews` int DEFAULT NULL,\n `daxaoubfjdeqtxfquklaajjwmlwnmbpo` int DEFAULT NULL,\n `hmzqiebzeplhcwwjgelcbfsgdqtdqxsk` int DEFAULT NULL,\n `shthpieaisrfvvrpcltqbfkigawsjgsm` int DEFAULT NULL,\n `gzelyobenwyrhklqlxqwzikvjglwlyjt` int DEFAULT NULL,\n `aggnhrogowbstaonkmjjctxrlhzswigb` int DEFAULT NULL,\n `wjeuhkqwcrtdjonzccfczzxzwefwueqm` int DEFAULT NULL,\n `vgalwkhbscwrialcsnfariwbzkayemat` int DEFAULT NULL,\n `rmxmubpvgnxpwgydgshjbuyufxwszzpq` int DEFAULT NULL,\n `qyzrvtgubqjaqvwyfkratrseotlwoufx` int DEFAULT NULL,\n `fdyobnlfgiovhsxigxbkwuesgyabjrus` int DEFAULT NULL,\n `fzuzzhdghfudnpnjyypefnnylylaekae` int DEFAULT NULL,\n `vojftggkcgeszbjmhioobehzbejgliis` int DEFAULT NULL,\n `fquzccoybnyvtmyxmebihupjrpelerrv` int DEFAULT NULL,\n `kbcavvbqajylolfjmxggpfnnfzolebed` int DEFAULT NULL,\n `oirfndhwxwoxgmvgwehmdhlodbgmtoia` int DEFAULT NULL,\n `exxfbtoujnwrwpdilacccctwztqajrdx` int DEFAULT NULL,\n `yptiuovplpjiwplddpexdjnjyjaigvko` int DEFAULT NULL,\n `mtkfxdsnbfkcecinhdhvsudkszguzqmo` int DEFAULT NULL,\n `eblpgqpxynsrjoiptadozbonrfijdnzw` int DEFAULT NULL,\n `jjuymmlkbnunlkgrumqiimdzxvfjisim` int DEFAULT NULL,\n `umfromnuaeidhayjdhyahsbioulysxxd` int DEFAULT NULL,\n `hgcjrpdwqaxjfkgrmjntbdcfhozomebi` int DEFAULT NULL,\n `vtotscolthvywjxklqywnncqsngpmbxu` int DEFAULT NULL,\n `melqehberdqmxwatcyypghrdjsnkbqmf` int DEFAULT NULL,\n `pdshvotjxuyysmlmtujtmkhhczoznwul` int DEFAULT NULL,\n `fqimowuevdcydwgpuzzicgygqbfiaozi` int DEFAULT NULL,\n `acekuxpogvbuguxfikucgfrpzyztnxec` int DEFAULT NULL,\n `ubqjfemeiefhrguspalzunecektwnfai` int DEFAULT NULL,\n `ljbzehftrxduxehjaraicjgilnvyueeo` int DEFAULT NULL,\n `vpnrmjwxjdrswzfxtuwabggfwgnjxofw` int DEFAULT NULL,\n `gxwtdbzvnfwgawuvmdeveypiuhpwkqbi` int DEFAULT NULL,\n `urzkbsabiodtnekjzvkodvedurionchu` int DEFAULT NULL,\n `rolyjighhqbtpkzzvdlsieazfoiyfdbu` int DEFAULT NULL,\n `pccpohfhvyifmuwzubrsgmrarryroqxt` int DEFAULT NULL,\n `miscpeidpwahcioeixryngtciaquytio` int DEFAULT NULL,\n `hlmqayqhbldlirnruchpxubkvfahvozs` int DEFAULT NULL,\n `pqoqruozzjbuxzfaemovefagicrmkand` int DEFAULT NULL,\n PRIMARY KEY (`gqzqnjjgsqkiyalyawyyfmekkcnzjvuy`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"wgxcetmtjwvmrtdbuchydagtylsvtsga\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["gqzqnjjgsqkiyalyawyyfmekkcnzjvuy"],"columns":[{"name":"gqzqnjjgsqkiyalyawyyfmekkcnzjvuy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"inmqwklxqocieqgakzwoisdxdkfrgkws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pziazvkohclbfsglwcavbhsgasavareg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjltkcirxbtdwsjqzjpyvrmdisqqzvki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdyaotrrazcvbcptwtomkiwmjwlszlln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgqsolmlscebqadvbopkjmojoevrlxpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bmqpcqxhejcisybzbkgpibtrqlmosdwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fghulyfoucmwrfcfgqeyoxhvtjechywg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lijtfygdmtvqcrubdgjozfxreqawagkp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcemdhtcthfvertknwrfdreqwqahwoxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyqytpzavrhplxxughnpzhromhhhwfgh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oudznhuyguuvomjtibvinnjepkilffhx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"heviqfogruqzockmnrkdjrvgvxnbrjhf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qczdrjfscdfgrvczwyilqsswrczyczwx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjrxgbmnvhegaaupqkptiopevicudscw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fyudlauckweorkcjgexbxvcjpaejxlnq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lleiektpilqxgvifcyinsjfemkecrqhj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jivfspszznyhfoqbdzokkbtwhdphivxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nmpvekckqpeeyydstphamlvcqmqcxzmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llpjarrrxnskmzsqszahagexczrqemqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xydblwakmaezllsxhraotadcctitfmdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mubfkjnlkkzwctpknecioltotplylzve","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"irjlklxkqhpizoikbmnusulcirceroxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydfjsicdlnkiabcawmkedgnnlcfvewxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oczfwydeyhnbpundsdjtpbujlxndhkhm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmlqxruytwqzdskpfsbhorwyhdcqjfna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fofwrxsnpysukcnshtcdxqgdtnnrkwbh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wsrasyfhxwprpfbfeqsadniqacxoaqxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iylebdnmcpxuzfwmhjsaymhligzqielj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"czbuexwcgrekffwavimwgiypkazsemxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzwhuisgyycskpcsfnybgnlqgagnoqjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzucglaerqyubjhnoxjldrepnyiapazd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkzosqwlnavcnfalutewkgefzuhqcnuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhdeczcrxuffkfmfuoygeafjwqkmorxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ksgvfpmxxflruularghecmpagiqivebr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnlwuulakxtkxqfajtjgczfdjjvlirlr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gffzhewvzhogjawfedirezbeesjysryp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yufjdbalqpxtcjttddayqldihxfpkqul","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnbruwoitsjnxcxfzglkbinmtyienhwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"whpszpidmrjaerabjqijpvhwlliqzijv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ruucbaotcphcgfosknukjrlkkqwfwyti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xcvandoirxnwlyovhjbfsefocibzdxvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwwuehghbtsmzecgcdklzterfntxgfcy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shfvsyiylwhazlpnmpzdndnmpqfeignd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvhjsarhuyvfyqqsjfohjhxahvshmvnt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"foruycpxrwirdkkaqnwyplxiegyypyex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gbxbgibgtubvqkiktyqpuiyrkfcpbzcz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oiwlzyunpdinfjdwodrlembvrqzdmdig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnztrhqlnnruofghmftqqvtpsalwwlis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lsbprvfhibbffksylusvxvfidlwavoim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjnaqucyvvnfoaveidxbopekunodbxkn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yktrfbsxccbbuyvvboqwoeidfkgspnyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gbwcpvvjjtovpiykkusqehloqqcwkzve","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyhzrzmgarjgvkmifehldmxtavvemzfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzjjflzmmlsbhcwsxsmvqxitdvhqklja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uchuvhxbqycgvpltwxbzqnsggeaqcqta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqmgbnpwzoaxmqmcybssapwkzvdooyqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zujjuvrazfvbkpadjpxcpbjyfcgniyak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpgccoadlaupgwtkxtnykovhhgjuegos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwohvgjvqczawxxzkpqvytkpbzuvqcxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kadvprstslnekefqsotrovnsmnyukccg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnhwfwurzrkxmkxtlevvkfvxbxpjbghs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovgsupkalbtbxddxlzphkzbcqkyvrews","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"daxaoubfjdeqtxfquklaajjwmlwnmbpo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmzqiebzeplhcwwjgelcbfsgdqtdqxsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shthpieaisrfvvrpcltqbfkigawsjgsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzelyobenwyrhklqlxqwzikvjglwlyjt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aggnhrogowbstaonkmjjctxrlhzswigb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjeuhkqwcrtdjonzccfczzxzwefwueqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgalwkhbscwrialcsnfariwbzkayemat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rmxmubpvgnxpwgydgshjbuyufxwszzpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qyzrvtgubqjaqvwyfkratrseotlwoufx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fdyobnlfgiovhsxigxbkwuesgyabjrus","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzuzzhdghfudnpnjyypefnnylylaekae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vojftggkcgeszbjmhioobehzbejgliis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fquzccoybnyvtmyxmebihupjrpelerrv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kbcavvbqajylolfjmxggpfnnfzolebed","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oirfndhwxwoxgmvgwehmdhlodbgmtoia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exxfbtoujnwrwpdilacccctwztqajrdx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yptiuovplpjiwplddpexdjnjyjaigvko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtkfxdsnbfkcecinhdhvsudkszguzqmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eblpgqpxynsrjoiptadozbonrfijdnzw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjuymmlkbnunlkgrumqiimdzxvfjisim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umfromnuaeidhayjdhyahsbioulysxxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hgcjrpdwqaxjfkgrmjntbdcfhozomebi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtotscolthvywjxklqywnncqsngpmbxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"melqehberdqmxwatcyypghrdjsnkbqmf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdshvotjxuyysmlmtujtmkhhczoznwul","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqimowuevdcydwgpuzzicgygqbfiaozi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acekuxpogvbuguxfikucgfrpzyztnxec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubqjfemeiefhrguspalzunecektwnfai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljbzehftrxduxehjaraicjgilnvyueeo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpnrmjwxjdrswzfxtuwabggfwgnjxofw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxwtdbzvnfwgawuvmdeveypiuhpwkqbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urzkbsabiodtnekjzvkodvedurionchu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rolyjighhqbtpkzzvdlsieazfoiyfdbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pccpohfhvyifmuwzubrsgmrarryroqxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"miscpeidpwahcioeixryngtciaquytio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlmqayqhbldlirnruchpxubkvfahvozs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqoqruozzjbuxzfaemovefagicrmkand","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672395,"databaseName":"models_schema","ddl":"CREATE TABLE `whclbatdragfegfzxazuownrpaxqfwpq` (\n `aukcupquqkatrkvndesesmsotrpeuldq` int NOT NULL,\n `bkozgtxyculpzgflbzgeubhpchcetzgd` int DEFAULT NULL,\n `udymdrcewuuhexgvbywimxzkksakowbz` int DEFAULT NULL,\n `daicibtbcpjvnwnzjjnmsicrvpjslqak` int DEFAULT NULL,\n `vyosfefzvyczaeecucalakpmnquavtsn` int DEFAULT NULL,\n `eyeztuzqrtcttszibtkfmkzvwmenixpx` int DEFAULT NULL,\n `mxxylpeskuancrhzqaodowghwcomwruu` int DEFAULT NULL,\n `nhxtevsbocdhernhcbqdlisagptguzxh` int DEFAULT NULL,\n `lcdfybueqypycaceojcokbgejumdrqtl` int DEFAULT NULL,\n `hxqglvlbmdjhzvmmuiehqefeoeraapfh` int DEFAULT NULL,\n `xxenrmhsjvdncnwssmgrvyxwijvlvlie` int DEFAULT NULL,\n `ldcahfrgwozfcrllmueyqcbsnttsyurw` int DEFAULT NULL,\n `ujeyryukfpocdppkukbpwqcirzwamokv` int DEFAULT NULL,\n `jkhpdmyxysnnnjueqqfbnankzmjowgmr` int DEFAULT NULL,\n `pperlsjeajckzdtqjdocvziomopnsqhd` int DEFAULT NULL,\n `yeaenborjvuzizvnrlcvvczdmsjcaxcx` int DEFAULT NULL,\n `uvjckfzdkhkvhzumnsmbnarbqzgifjxr` int DEFAULT NULL,\n `mwucscwfqkpzmwkunelajmfsamrcisdn` int DEFAULT NULL,\n `bajqskhomnaupwzqxwyqfmrdpwnmjqer` int DEFAULT NULL,\n `vjzlbpoyfdvfrfflqpjlbhgwatkbvumr` int DEFAULT NULL,\n `aeehsnbxscyhtswnhiuhiwqbvrjxtckn` int DEFAULT NULL,\n `grdudbnwedalboafumeflxiymzvjdkxa` int DEFAULT NULL,\n `xriawbvwkuikxtspwoexjpzikqmosjto` int DEFAULT NULL,\n `fhjwuibbpjmazbodwtlvmmgtflrxcgys` int DEFAULT NULL,\n `vsmqkxdaodofdpuwhyexqqdwreznxhsn` int DEFAULT NULL,\n `tuirlwgvshubnbvmikrznxabdjpsojyb` int DEFAULT NULL,\n `azfjclopfpxbttpdwtopsscbkufgqzwd` int DEFAULT NULL,\n `rmqdukhvzozdylyuottohignmaprxitj` int DEFAULT NULL,\n `dkpcdfsfbhbapgveqfnfxmiizbizseyz` int DEFAULT NULL,\n `bayqzntukfqjznycaunyvavtboohvhlw` int DEFAULT NULL,\n `jdfmpiqlkpktgfgeapawfyuuofhnixwt` int DEFAULT NULL,\n `ndxqpfngqlnzpkcgntfotrmnzbkgwpvu` int DEFAULT NULL,\n `pwyexpdtjrczwigtvaezjkwlkuirqssj` int DEFAULT NULL,\n `pfnrqrmqdpnltateoulbtouybdbzwfjn` int DEFAULT NULL,\n `uzzfxngyfkcjisuvgmatqfamfkacfqzm` int DEFAULT NULL,\n `cetymylhlgpucmqwkiqlmksjfafafrux` int DEFAULT NULL,\n `rqhfkwolhnunimnfdmlrxhewcopdnwnm` int DEFAULT NULL,\n `ssnitaxdqtusfmgwnnjtedmgikmbyrqk` int DEFAULT NULL,\n `wsepahrlgtfbitwtcelowvvkemwzmqoe` int DEFAULT NULL,\n `ucovxfuaiguknvphpnlriqdiqbeiaipk` int DEFAULT NULL,\n `dxdymsatbdafdmigxkfpllkhyxfjhdeo` int DEFAULT NULL,\n `toaiccfcwomtnbznrgesodkmbckdumyh` int DEFAULT NULL,\n `fpbyelvfaclopebajksulripriakjqqk` int DEFAULT NULL,\n `mttgubncnvxhbjttetfmgrlerickhcxq` int DEFAULT NULL,\n `asdcvfirfpcbfperhergtcvbyjzfzybj` int DEFAULT NULL,\n `fjdsjyyxnpwfcfjgvhjmxlsxhljrsglg` int DEFAULT NULL,\n `szhjejizackdsskdrfrpaecwlwqbukwf` int DEFAULT NULL,\n `vgjnfjfaczcmdiyxlagnkctzkxbcppge` int DEFAULT NULL,\n `fsgjlafvojsvxpxpmxuebxnmbqoehpte` int DEFAULT NULL,\n `jmqwrtshnzzxprrtkmdkbnyjfznttujh` int DEFAULT NULL,\n `aopodzyhmparbidbibzwufbnixmyrdsg` int DEFAULT NULL,\n `rnxwuaxvauvxscxejesmtwqmbcqyoweu` int DEFAULT NULL,\n `wodotfhlmmddqbzomfoyoikuzjtdibzg` int DEFAULT NULL,\n `gphhswkqpxovxzprkqdjcxstaauyzlds` int DEFAULT NULL,\n `swznlrghgdchxiduveopgbftgoquqlun` int DEFAULT NULL,\n `zuqwvcbsadlxjwtkhaiynciyziwccycf` int DEFAULT NULL,\n `jylytmzctqridvldopokjkltbjbcoywt` int DEFAULT NULL,\n `fedmfqrdcgxqfqpnggwncutyihcntxxf` int DEFAULT NULL,\n `gbsiskiigujffomhfyzpemttakypsekj` int DEFAULT NULL,\n `criufnjvgqdehuvvlqcpvwlsfgevmgqg` int DEFAULT NULL,\n `xjltipxrogqymektemaqjrwvsafywvvq` int DEFAULT NULL,\n `mpoayifakbdtkzcpgkienhcbbllzmwdj` int DEFAULT NULL,\n `hhpvsljliclednspczaikjhpznhfpkgg` int DEFAULT NULL,\n `ieeajschzsyamhtythjxwsosnfposlms` int DEFAULT NULL,\n `xdocgfuodosarcxacpkxzixomtyeiuho` int DEFAULT NULL,\n `lsbgnfeqgljsihlvxrmtzakxdzotpjwj` int DEFAULT NULL,\n `wuvdcioxdrfgaegebexwcswtcpvaekbr` int DEFAULT NULL,\n `udtxfcayvbnaglmleesgwyuxllejidwj` int DEFAULT NULL,\n `ugcmrxhkvtvkhnkcquoptcitinauxdyd` int DEFAULT NULL,\n `nytyoqqasxblxcnmfpmnhrflffkjkfyf` int DEFAULT NULL,\n `bwbvcgnlrxasioochbwruwohctpophct` int DEFAULT NULL,\n `bnxgnviezzbnbzrmlbqtzhfwidgjwodl` int DEFAULT NULL,\n `fjfzuflgpnmqociagjwcrteidsymggld` int DEFAULT NULL,\n `npjauaydtnyxjjhpetihqjderaozwgbv` int DEFAULT NULL,\n `njtjddibmlzmiafjpfqlurqwlvyinauw` int DEFAULT NULL,\n `exxgefrobgrjrqhbptcbzmgrfcwercyj` int DEFAULT NULL,\n `rlpsgmnrevfzyndwjapmfzifvjvnlxdr` int DEFAULT NULL,\n `gdfjnyzeisjrqtwkmosrmootpshwwcqr` int DEFAULT NULL,\n `rhyppwvoewtwlrmgvvrqkasnaqzcpbat` int DEFAULT NULL,\n `eurdrabevaqvstiebyahjfgposjqqwmv` int DEFAULT NULL,\n `xojzftnrzhcuhbcqpicuqbfpfdaapkrz` int DEFAULT NULL,\n `bgsrgafpcumfvszkqsinxrijhrmomwbi` int DEFAULT NULL,\n `vfereuqsfhrsmuqszndbyhnqechqpdes` int DEFAULT NULL,\n `jbfnecxhltfqssyevkswskjpqtymapaw` int DEFAULT NULL,\n `ppxesugfiraspwqkcuupcbyskdfovpvd` int DEFAULT NULL,\n `kbeqbffhqifmunjeoprtfbpaqyybcevd` int DEFAULT NULL,\n `ghkbxkdlfgpqvtukewyltwmrfilkmsld` int DEFAULT NULL,\n `edqyvhhfbswwvaslsuvpovrdxejwdihl` int DEFAULT NULL,\n `mkevffpayiwpbgeszgfgkrlwlvceedcu` int DEFAULT NULL,\n `amajoyshhfekvzidpubhwgukdpzuzcrv` int DEFAULT NULL,\n `obpqhnwthzzjgwkdcbqtphregfnnonoe` int DEFAULT NULL,\n `otefbioanilpvimianhnioumqrswdiby` int DEFAULT NULL,\n `phpzvashnxgqklbqpazjxpvpybbbroyc` int DEFAULT NULL,\n `quzyexvvnpjdxnndvdccmypticlvukrm` int DEFAULT NULL,\n `prgfihqjjrohziwusiryzfyrprantclk` int DEFAULT NULL,\n `aygjebgmxzpdqozcaboaaxwcuujwfwbl` int DEFAULT NULL,\n `bivzgzbcrlbqhphcxopngjwzkyjuiuls` int DEFAULT NULL,\n `nfdhhicphoplfnksefxfxozxzfmrzegk` int DEFAULT NULL,\n `mlrbyecmvhhkiyfsetcbjvvnpknailii` int DEFAULT NULL,\n `nvjpnnzqkgvmffkvlmiwdlzfhiajqvaf` int DEFAULT NULL,\n PRIMARY KEY (`aukcupquqkatrkvndesesmsotrpeuldq`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"whclbatdragfegfzxazuownrpaxqfwpq\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["aukcupquqkatrkvndesesmsotrpeuldq"],"columns":[{"name":"aukcupquqkatrkvndesesmsotrpeuldq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"bkozgtxyculpzgflbzgeubhpchcetzgd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udymdrcewuuhexgvbywimxzkksakowbz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"daicibtbcpjvnwnzjjnmsicrvpjslqak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vyosfefzvyczaeecucalakpmnquavtsn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eyeztuzqrtcttszibtkfmkzvwmenixpx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mxxylpeskuancrhzqaodowghwcomwruu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhxtevsbocdhernhcbqdlisagptguzxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lcdfybueqypycaceojcokbgejumdrqtl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxqglvlbmdjhzvmmuiehqefeoeraapfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxenrmhsjvdncnwssmgrvyxwijvlvlie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ldcahfrgwozfcrllmueyqcbsnttsyurw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujeyryukfpocdppkukbpwqcirzwamokv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jkhpdmyxysnnnjueqqfbnankzmjowgmr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pperlsjeajckzdtqjdocvziomopnsqhd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yeaenborjvuzizvnrlcvvczdmsjcaxcx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvjckfzdkhkvhzumnsmbnarbqzgifjxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwucscwfqkpzmwkunelajmfsamrcisdn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bajqskhomnaupwzqxwyqfmrdpwnmjqer","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjzlbpoyfdvfrfflqpjlbhgwatkbvumr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aeehsnbxscyhtswnhiuhiwqbvrjxtckn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grdudbnwedalboafumeflxiymzvjdkxa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xriawbvwkuikxtspwoexjpzikqmosjto","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhjwuibbpjmazbodwtlvmmgtflrxcgys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsmqkxdaodofdpuwhyexqqdwreznxhsn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tuirlwgvshubnbvmikrznxabdjpsojyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azfjclopfpxbttpdwtopsscbkufgqzwd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rmqdukhvzozdylyuottohignmaprxitj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkpcdfsfbhbapgveqfnfxmiizbizseyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bayqzntukfqjznycaunyvavtboohvhlw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdfmpiqlkpktgfgeapawfyuuofhnixwt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ndxqpfngqlnzpkcgntfotrmnzbkgwpvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwyexpdtjrczwigtvaezjkwlkuirqssj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pfnrqrmqdpnltateoulbtouybdbzwfjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzzfxngyfkcjisuvgmatqfamfkacfqzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cetymylhlgpucmqwkiqlmksjfafafrux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqhfkwolhnunimnfdmlrxhewcopdnwnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ssnitaxdqtusfmgwnnjtedmgikmbyrqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wsepahrlgtfbitwtcelowvvkemwzmqoe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucovxfuaiguknvphpnlriqdiqbeiaipk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxdymsatbdafdmigxkfpllkhyxfjhdeo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"toaiccfcwomtnbznrgesodkmbckdumyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpbyelvfaclopebajksulripriakjqqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mttgubncnvxhbjttetfmgrlerickhcxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"asdcvfirfpcbfperhergtcvbyjzfzybj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjdsjyyxnpwfcfjgvhjmxlsxhljrsglg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"szhjejizackdsskdrfrpaecwlwqbukwf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgjnfjfaczcmdiyxlagnkctzkxbcppge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fsgjlafvojsvxpxpmxuebxnmbqoehpte","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmqwrtshnzzxprrtkmdkbnyjfznttujh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aopodzyhmparbidbibzwufbnixmyrdsg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rnxwuaxvauvxscxejesmtwqmbcqyoweu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wodotfhlmmddqbzomfoyoikuzjtdibzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gphhswkqpxovxzprkqdjcxstaauyzlds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swznlrghgdchxiduveopgbftgoquqlun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zuqwvcbsadlxjwtkhaiynciyziwccycf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jylytmzctqridvldopokjkltbjbcoywt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fedmfqrdcgxqfqpnggwncutyihcntxxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gbsiskiigujffomhfyzpemttakypsekj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"criufnjvgqdehuvvlqcpvwlsfgevmgqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjltipxrogqymektemaqjrwvsafywvvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpoayifakbdtkzcpgkienhcbbllzmwdj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhpvsljliclednspczaikjhpznhfpkgg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ieeajschzsyamhtythjxwsosnfposlms","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdocgfuodosarcxacpkxzixomtyeiuho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lsbgnfeqgljsihlvxrmtzakxdzotpjwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wuvdcioxdrfgaegebexwcswtcpvaekbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udtxfcayvbnaglmleesgwyuxllejidwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugcmrxhkvtvkhnkcquoptcitinauxdyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nytyoqqasxblxcnmfpmnhrflffkjkfyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bwbvcgnlrxasioochbwruwohctpophct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnxgnviezzbnbzrmlbqtzhfwidgjwodl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjfzuflgpnmqociagjwcrteidsymggld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npjauaydtnyxjjhpetihqjderaozwgbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njtjddibmlzmiafjpfqlurqwlvyinauw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exxgefrobgrjrqhbptcbzmgrfcwercyj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlpsgmnrevfzyndwjapmfzifvjvnlxdr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdfjnyzeisjrqtwkmosrmootpshwwcqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhyppwvoewtwlrmgvvrqkasnaqzcpbat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eurdrabevaqvstiebyahjfgposjqqwmv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xojzftnrzhcuhbcqpicuqbfpfdaapkrz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bgsrgafpcumfvszkqsinxrijhrmomwbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vfereuqsfhrsmuqszndbyhnqechqpdes","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbfnecxhltfqssyevkswskjpqtymapaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ppxesugfiraspwqkcuupcbyskdfovpvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kbeqbffhqifmunjeoprtfbpaqyybcevd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghkbxkdlfgpqvtukewyltwmrfilkmsld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edqyvhhfbswwvaslsuvpovrdxejwdihl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkevffpayiwpbgeszgfgkrlwlvceedcu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amajoyshhfekvzidpubhwgukdpzuzcrv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obpqhnwthzzjgwkdcbqtphregfnnonoe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otefbioanilpvimianhnioumqrswdiby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phpzvashnxgqklbqpazjxpvpybbbroyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"quzyexvvnpjdxnndvdccmypticlvukrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prgfihqjjrohziwusiryzfyrprantclk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aygjebgmxzpdqozcaboaaxwcuujwfwbl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bivzgzbcrlbqhphcxopngjwzkyjuiuls","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfdhhicphoplfnksefxfxozxzfmrzegk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlrbyecmvhhkiyfsetcbjvvnpknailii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvjpnnzqkgvmffkvlmiwdlzfhiajqvaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672428,"databaseName":"models_schema","ddl":"CREATE TABLE `whemkduelgfkfynwcciloonbigfttakk` (\n `dmowheaddeitnezpzaidrcdcjspfhqzv` int NOT NULL,\n `rdwerlbyilwngyodcuyvnzldqjntkrxy` int DEFAULT NULL,\n `bngdrrmguwfmnzephabjvnzkbsokhags` int DEFAULT NULL,\n `iylhssxxenofitlilsixjecugjggfiaq` int DEFAULT NULL,\n `kacumyrbjmydofkzrffrzwejxwcbewvs` int DEFAULT NULL,\n `uhqhadoywjnknyubrrqunmtqjanrmgbu` int DEFAULT NULL,\n `fcwidezpexmpuzpzdeorgpqrbfigruki` int DEFAULT NULL,\n `nopnuplfsxmtoaoxqwpkkjgnacgwhqxt` int DEFAULT NULL,\n `pocdpttszwcprswegslxoezqjktfxtzk` int DEFAULT NULL,\n `dhhazdqehbaffhmcdkpalafjvdcvwqgq` int DEFAULT NULL,\n `zxmzhqfupqxumjzyutxwnziwxdpjrsft` int DEFAULT NULL,\n `hgogwhxbcphedxednayoddansmvysmmn` int DEFAULT NULL,\n `ckhsagzpovbxfpspyyodtjxepuinwutn` int DEFAULT NULL,\n `jsaxlgtywcimwwvvfeukwmibgjxrgizg` int DEFAULT NULL,\n `pddwjgvoihkblzynvpzybcnvmhsokhlq` int DEFAULT NULL,\n `anbxykzskwljlfewgplrikyttaywyfme` int DEFAULT NULL,\n `gpnmmluvwyypoxfrflyjtzdjhvctvgtu` int DEFAULT NULL,\n `sthrytkiugspjrqfkoqtrzixzpdixqpu` int DEFAULT NULL,\n `erkrchejtnhiizbidmuqnfcabvkdzawc` int DEFAULT NULL,\n `taeoodxvqcvuggtbvzvjsebjccrcdfwg` int DEFAULT NULL,\n `zmydyojxrfkhggiyzsnxmnywmhgvnrid` int DEFAULT NULL,\n `csenvdtvfeawnrvdlhmnqcfnxmbavvav` int DEFAULT NULL,\n `ronqkdsoergokzlojfgtujteaglnxlts` int DEFAULT NULL,\n `lolmpazprvfpzbheafpedxerfpeylway` int DEFAULT NULL,\n `ezxhsgcvnubnbwbsknqcnuxieqibzxgx` int DEFAULT NULL,\n `gbocvvpybvixcvqebowfsralreqioctf` int DEFAULT NULL,\n `alvjapudbiseoropjishdifvmfumvppm` int DEFAULT NULL,\n `ljlexhwhpefrsufkhyoeszopbgpsinjy` int DEFAULT NULL,\n `gugercakyowqogopdyqpjoiqtjcbbabz` int DEFAULT NULL,\n `eusucfhfhtlhvrqddwyhhsvfyyqpvqvo` int DEFAULT NULL,\n `xyjhzorqjlezqwlmpjgnxtjepjxgruui` int DEFAULT NULL,\n `nkjelqtbzwdpmdryturbmfbecfwlozdc` int DEFAULT NULL,\n `rhhpvzioaqtjcmebqqkfzsxitjfmgufv` int DEFAULT NULL,\n `zjugjyzbzrfyneqelfzdmlinbrtzubaj` int DEFAULT NULL,\n `ctgecfjqblljqmbnhxcrozmsqifoonos` int DEFAULT NULL,\n `eenhtafhmqztdhzcvxskgooohvqmvlxg` int DEFAULT NULL,\n `wmlpfmocodxetkwpocimrrxkgmyptqap` int DEFAULT NULL,\n `raqmrugcrjghupctnpotflahidbcdcwn` int DEFAULT NULL,\n `ecnpxfonppszwqdzrmwqxwrfqldxgody` int DEFAULT NULL,\n `djczlvqiggwetbeycltcmnoqoqhjwiod` int DEFAULT NULL,\n `hmwyxiketbixspmtgukdppdqyzakklpb` int DEFAULT NULL,\n `jjgwuplexxumiacrkvwlkzzatsvomliz` int DEFAULT NULL,\n `uvwvmsrzpzeslsgrydbrifhydzaqpiop` int DEFAULT NULL,\n `uijtutbvcdwkevozlmwinwynqmwmbdbq` int DEFAULT NULL,\n `hrzemehbeslrnpnqziijhaphqadoqrpp` int DEFAULT NULL,\n `kyauapkubzrporprmteqeywsmmbyqylt` int DEFAULT NULL,\n `dagbanjnlkooadymiibycxhjzhveddih` int DEFAULT NULL,\n `aawslolqpjnsxxgcmydfllazqlvhpmps` int DEFAULT NULL,\n `fyokmbrdzlwujrxazvcvwwdehqzbmeis` int DEFAULT NULL,\n `gactstgxcenclrqykurasfcrkcgkejbv` int DEFAULT NULL,\n `zzxqcmwntqlbitcurnelppynmdbfzbrv` int DEFAULT NULL,\n `dexahwkzqdcrbsuuyvkrsqvxyxdsglab` int DEFAULT NULL,\n `mnglzyqawbnozgatgrnephxyuwajmavx` int DEFAULT NULL,\n `lozidckvkdsgkblvhbgwhohkogqsokhk` int DEFAULT NULL,\n `foimsblrztkzinnxhvyzdnjzsvyfehex` int DEFAULT NULL,\n `shgbrugciztcnhsijjnfdnwsgcgxelsi` int DEFAULT NULL,\n `kejwztievjnyluoggjdmffxcxwobsbri` int DEFAULT NULL,\n `xwmgbcuspvkhjfcrcsxnkhipnwqjkqss` int DEFAULT NULL,\n `txundzcsovmirenoekmqzlzsifnzsqgh` int DEFAULT NULL,\n `dpwzblqibtpuwqlbzqsgcgcpxyelspmj` int DEFAULT NULL,\n `beprgicngpmjzpkxwmqumagzxzcvtxar` int DEFAULT NULL,\n `vehhkprgzlarnusoxtnfnqnwufarkqfj` int DEFAULT NULL,\n `auxtvziksapznibtffedtdzrskwkllxj` int DEFAULT NULL,\n `pbwnupxldrrpsthfkmqbfvxkiuhcjnbl` int DEFAULT NULL,\n `llazqztvzpfnlbnfufuruowepxdkzhxk` int DEFAULT NULL,\n `avrcyudrvcyrkabsdxdybwcmpbzmkglf` int DEFAULT NULL,\n `vfufxyqtwrujdcmhlsmrqcawbryarzbo` int DEFAULT NULL,\n `jcgdwmzqbuankvucuozobryqfyvzsrnm` int DEFAULT NULL,\n `asbwzhsbogdtmikkyzchktthiciuhzfr` int DEFAULT NULL,\n `wrepucrzbuafrxvsvbaodoxkzpainnqa` int DEFAULT NULL,\n `binhrcuvcjggjdbbnfrmupmoipxzpkhl` int DEFAULT NULL,\n `zhdywnonwuywvfsvxrovczjftvyfrahh` int DEFAULT NULL,\n `lsajucybptvcdbvfblgziblyqgomhmxz` int DEFAULT NULL,\n `moimnkecmvbwduuuhokcwnoeoujqkznx` int DEFAULT NULL,\n `uclfwmdsbzlypssqmdjkueipefsbtozo` int DEFAULT NULL,\n `qsrvkzbvzhbwkjcshjlaqmhlbdyiemdl` int DEFAULT NULL,\n `geuntaaieuxjppjwlhvhdompszljojfa` int DEFAULT NULL,\n `gxnadhpyjaltaohdviwkgkdppwtidypj` int DEFAULT NULL,\n `nycrmsepnqyveltfpwelsizlmdcvxvec` int DEFAULT NULL,\n `ciiowoivbftmtlksfuueasimbgkpydnd` int DEFAULT NULL,\n `aediawgxxjszufjyzjaogqtwpqnhpznj` int DEFAULT NULL,\n `dzxeavujaifasyhihwmqgrqbjagxsqjx` int DEFAULT NULL,\n `hhxelqyikoirpxurtafltxoeobvfeqpg` int DEFAULT NULL,\n `gldfnpubzgjiqxasarvtrgosnhiybyzz` int DEFAULT NULL,\n `cdiymvhgjaqxwozigkxteafxjznshmtu` int DEFAULT NULL,\n `wyavnjsvdeylcxtohtsuydublvgkvlzb` int DEFAULT NULL,\n `haqezvfqbbgdxcioesedjlmjrshkaiky` int DEFAULT NULL,\n `xjgecamxmtleunoyenfggacajzkmfwcn` int DEFAULT NULL,\n `yfamabslaqcmukmyoiuraljcbifzzdau` int DEFAULT NULL,\n `wjagfikrakzdouilcdhawbveqggillpc` int DEFAULT NULL,\n `kafgqfblhmwjvbbxydnbzyalcyldgufd` int DEFAULT NULL,\n `ofrxvdpzaedywghkkcfcfinvmsbmtess` int DEFAULT NULL,\n `frmuxvpxwtjwyjvwgdwgtjmpocslzqxw` int DEFAULT NULL,\n `znnceekwvhsmiqhhalwedeoyzkzozmji` int DEFAULT NULL,\n `wqlqpxpjkktbjtttzcvefygurdabkkck` int DEFAULT NULL,\n `uxqpqugytmkmoschelmdpfycfmdwaivt` int DEFAULT NULL,\n `dmpugtvwiembzfftwmddqrhdmwjjifsh` int DEFAULT NULL,\n `gobtvefghfuowrfwftllrhzqsnmbacld` int DEFAULT NULL,\n `rsnfavqjwzfepzptyltxerohqpurfbog` int DEFAULT NULL,\n `mlmmezchvkcnwezduocdewcicvmohbvz` int DEFAULT NULL,\n PRIMARY KEY (`dmowheaddeitnezpzaidrcdcjspfhqzv`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"whemkduelgfkfynwcciloonbigfttakk\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["dmowheaddeitnezpzaidrcdcjspfhqzv"],"columns":[{"name":"dmowheaddeitnezpzaidrcdcjspfhqzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"rdwerlbyilwngyodcuyvnzldqjntkrxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bngdrrmguwfmnzephabjvnzkbsokhags","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iylhssxxenofitlilsixjecugjggfiaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kacumyrbjmydofkzrffrzwejxwcbewvs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhqhadoywjnknyubrrqunmtqjanrmgbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcwidezpexmpuzpzdeorgpqrbfigruki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nopnuplfsxmtoaoxqwpkkjgnacgwhqxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pocdpttszwcprswegslxoezqjktfxtzk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhhazdqehbaffhmcdkpalafjvdcvwqgq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxmzhqfupqxumjzyutxwnziwxdpjrsft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hgogwhxbcphedxednayoddansmvysmmn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckhsagzpovbxfpspyyodtjxepuinwutn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jsaxlgtywcimwwvvfeukwmibgjxrgizg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pddwjgvoihkblzynvpzybcnvmhsokhlq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"anbxykzskwljlfewgplrikyttaywyfme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gpnmmluvwyypoxfrflyjtzdjhvctvgtu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sthrytkiugspjrqfkoqtrzixzpdixqpu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"erkrchejtnhiizbidmuqnfcabvkdzawc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"taeoodxvqcvuggtbvzvjsebjccrcdfwg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmydyojxrfkhggiyzsnxmnywmhgvnrid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"csenvdtvfeawnrvdlhmnqcfnxmbavvav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ronqkdsoergokzlojfgtujteaglnxlts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lolmpazprvfpzbheafpedxerfpeylway","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezxhsgcvnubnbwbsknqcnuxieqibzxgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gbocvvpybvixcvqebowfsralreqioctf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"alvjapudbiseoropjishdifvmfumvppm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljlexhwhpefrsufkhyoeszopbgpsinjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gugercakyowqogopdyqpjoiqtjcbbabz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eusucfhfhtlhvrqddwyhhsvfyyqpvqvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyjhzorqjlezqwlmpjgnxtjepjxgruui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkjelqtbzwdpmdryturbmfbecfwlozdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhhpvzioaqtjcmebqqkfzsxitjfmgufv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjugjyzbzrfyneqelfzdmlinbrtzubaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctgecfjqblljqmbnhxcrozmsqifoonos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eenhtafhmqztdhzcvxskgooohvqmvlxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmlpfmocodxetkwpocimrrxkgmyptqap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"raqmrugcrjghupctnpotflahidbcdcwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecnpxfonppszwqdzrmwqxwrfqldxgody","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"djczlvqiggwetbeycltcmnoqoqhjwiod","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmwyxiketbixspmtgukdppdqyzakklpb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjgwuplexxumiacrkvwlkzzatsvomliz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvwvmsrzpzeslsgrydbrifhydzaqpiop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uijtutbvcdwkevozlmwinwynqmwmbdbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hrzemehbeslrnpnqziijhaphqadoqrpp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kyauapkubzrporprmteqeywsmmbyqylt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dagbanjnlkooadymiibycxhjzhveddih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aawslolqpjnsxxgcmydfllazqlvhpmps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fyokmbrdzlwujrxazvcvwwdehqzbmeis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gactstgxcenclrqykurasfcrkcgkejbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzxqcmwntqlbitcurnelppynmdbfzbrv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dexahwkzqdcrbsuuyvkrsqvxyxdsglab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnglzyqawbnozgatgrnephxyuwajmavx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lozidckvkdsgkblvhbgwhohkogqsokhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"foimsblrztkzinnxhvyzdnjzsvyfehex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shgbrugciztcnhsijjnfdnwsgcgxelsi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kejwztievjnyluoggjdmffxcxwobsbri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xwmgbcuspvkhjfcrcsxnkhipnwqjkqss","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txundzcsovmirenoekmqzlzsifnzsqgh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpwzblqibtpuwqlbzqsgcgcpxyelspmj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"beprgicngpmjzpkxwmqumagzxzcvtxar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vehhkprgzlarnusoxtnfnqnwufarkqfj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"auxtvziksapznibtffedtdzrskwkllxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbwnupxldrrpsthfkmqbfvxkiuhcjnbl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llazqztvzpfnlbnfufuruowepxdkzhxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avrcyudrvcyrkabsdxdybwcmpbzmkglf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vfufxyqtwrujdcmhlsmrqcawbryarzbo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jcgdwmzqbuankvucuozobryqfyvzsrnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"asbwzhsbogdtmikkyzchktthiciuhzfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wrepucrzbuafrxvsvbaodoxkzpainnqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"binhrcuvcjggjdbbnfrmupmoipxzpkhl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhdywnonwuywvfsvxrovczjftvyfrahh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lsajucybptvcdbvfblgziblyqgomhmxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"moimnkecmvbwduuuhokcwnoeoujqkznx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uclfwmdsbzlypssqmdjkueipefsbtozo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qsrvkzbvzhbwkjcshjlaqmhlbdyiemdl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"geuntaaieuxjppjwlhvhdompszljojfa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxnadhpyjaltaohdviwkgkdppwtidypj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nycrmsepnqyveltfpwelsizlmdcvxvec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ciiowoivbftmtlksfuueasimbgkpydnd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aediawgxxjszufjyzjaogqtwpqnhpznj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dzxeavujaifasyhihwmqgrqbjagxsqjx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhxelqyikoirpxurtafltxoeobvfeqpg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gldfnpubzgjiqxasarvtrgosnhiybyzz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cdiymvhgjaqxwozigkxteafxjznshmtu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyavnjsvdeylcxtohtsuydublvgkvlzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"haqezvfqbbgdxcioesedjlmjrshkaiky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjgecamxmtleunoyenfggacajzkmfwcn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfamabslaqcmukmyoiuraljcbifzzdau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjagfikrakzdouilcdhawbveqggillpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kafgqfblhmwjvbbxydnbzyalcyldgufd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofrxvdpzaedywghkkcfcfinvmsbmtess","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"frmuxvpxwtjwyjvwgdwgtjmpocslzqxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znnceekwvhsmiqhhalwedeoyzkzozmji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqlqpxpjkktbjtttzcvefygurdabkkck","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxqpqugytmkmoschelmdpfycfmdwaivt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dmpugtvwiembzfftwmddqrhdmwjjifsh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gobtvefghfuowrfwftllrhzqsnmbacld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rsnfavqjwzfepzptyltxerohqpurfbog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlmmezchvkcnwezduocdewcicvmohbvz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672463,"databaseName":"models_schema","ddl":"CREATE TABLE `wjejvuvpihwabfbueufasjexzxfdfwxr` (\n `pufirulfcfxzxdrpshbulerqeuorzcbe` int NOT NULL,\n `okhqnubaozbefbjnlaetuxtnsppqqsxl` int DEFAULT NULL,\n `sqdtdpqombudlniwhnwxvazfahwvgech` int DEFAULT NULL,\n `qhbibfomcpaduushofskrcmydxyhzhpa` int DEFAULT NULL,\n `lsaqcfbrnoccdcvfmdowgqljflgnqelm` int DEFAULT NULL,\n `hqxolvtyescuepokpilgxmkipgwcuxof` int DEFAULT NULL,\n `dobyvuafrwwriiiotlvstniosxcnynbk` int DEFAULT NULL,\n `cephqjdvpmewczbfrpjstckwufgvssjq` int DEFAULT NULL,\n `zuyayyyvvsskuqvtgynsjqutqxfbtfad` int DEFAULT NULL,\n `igtdmvchqyzjcplulzsslsbbtnjxvibv` int DEFAULT NULL,\n `wbdejtuoruftauzhcppnwyvxczscpjxz` int DEFAULT NULL,\n `cufhklpjzpmuzcrqyhsshffviastcujk` int DEFAULT NULL,\n `fuziarusudzpsazhevnopprckyyvalzl` int DEFAULT NULL,\n `yomqmnuzszqxvoqdboahmwlsqhoaoyjj` int DEFAULT NULL,\n `ovedgtfyxozxasonzdrqcwbzfrykiwbw` int DEFAULT NULL,\n `kbwgvbrewzyzlysrppqhjyijpqweokko` int DEFAULT NULL,\n `tzwulqumhcurpiqvnfgipiajoplywion` int DEFAULT NULL,\n `relagtzgdtoevpahtmvloetqmplzthkz` int DEFAULT NULL,\n `vwxdmzejaqkpeiymrhokbgpdbchtjfaq` int DEFAULT NULL,\n `jwwhalizzumrfpamdskfcmbdanhrzdnw` int DEFAULT NULL,\n `ejxblmqaayhxdhsucvabxtyteshcvbov` int DEFAULT NULL,\n `jdtqgljtpelowfkboraxxbzboghihxse` int DEFAULT NULL,\n `dvjzbipdpwtbjlvvmtlazzmuwavgmpja` int DEFAULT NULL,\n `wyicmbmcsotfwiwxkgiffervapczsgbz` int DEFAULT NULL,\n `ocrnkbecpklnodlhgnknotdqoigyimut` int DEFAULT NULL,\n `umrgkzctokxcmnpbvycuavylshocctxn` int DEFAULT NULL,\n `ozicgscvxlsglwfuiwbxqcgffefjubtf` int DEFAULT NULL,\n `yoqnosteuspozqudidfyhqutumictoeo` int DEFAULT NULL,\n `aqivsjzckgdeltvsogiickpykydwvxyk` int DEFAULT NULL,\n `zfkrkobjtvmtwzpngewfuyommzkgssia` int DEFAULT NULL,\n `lsfzwypeteifxixjzrusphjrbwjimzmi` int DEFAULT NULL,\n `ssgxamymcchelftuenmhbtqcqkokevgy` int DEFAULT NULL,\n `hbujapmbxzrlplyesyvxmmwitfcquuxb` int DEFAULT NULL,\n `mcejgslxkukdwafnoshwvmgkpcitccar` int DEFAULT NULL,\n `yxowmmmpwqvzlwepxmhuqwdveqacrzmo` int DEFAULT NULL,\n `pnjoelmcwqbbkyzyvxlhzpaxqfrkgrjc` int DEFAULT NULL,\n `filyldcukublcsbplrybhkojvmwtgwlv` int DEFAULT NULL,\n `wtpbocuhaigmwjrdsuildvkbqppvdhfa` int DEFAULT NULL,\n `pcrixxwxfqgdlfxcnjnxerzvpdydepwp` int DEFAULT NULL,\n `bxyxiqskapxybbsjrodkiewxpklxygqh` int DEFAULT NULL,\n `aadrlpuddzsveqwerzptzrvlhvrxcqvd` int DEFAULT NULL,\n `ypebckaifetrsvibrrqpdrarwhrhliye` int DEFAULT NULL,\n `wltrpkbelhupijsbrjgbdbgrmcjgeged` int DEFAULT NULL,\n `vgtfwmkjrntgtvadlscmvouhqrffqksd` int DEFAULT NULL,\n `vzhexktslrphxyogyoyldpehntxoalae` int DEFAULT NULL,\n `xatiqpluevainjgtchzhuigbklicmkcu` int DEFAULT NULL,\n `lflcipkbrdehctrijwvzcmueanizdgeh` int DEFAULT NULL,\n `cilnqdnsizjztxvihaoigcpomtbjseix` int DEFAULT NULL,\n `xncuwkcgtmwtxrnsritudwckuaoaunsd` int DEFAULT NULL,\n `zylukygjjmddjonogdfhsheulkpdomcb` int DEFAULT NULL,\n `okqypqeqbbkuqjmnashmorofrxopdyhr` int DEFAULT NULL,\n `yqrlaziteifsxkwwcipkgyvgfcebigxs` int DEFAULT NULL,\n `ozuhwyvyvwjnopfruzupkoicktwixcde` int DEFAULT NULL,\n `bsbmkmiulnhpwxmatnexuzcxoypeoxfg` int DEFAULT NULL,\n `lfelexsgeiglywvzkoxozdtkxdosjcpq` int DEFAULT NULL,\n `xwdsicoxyihlwifldmusqgqhybujaapp` int DEFAULT NULL,\n `yhhfzmyaxrsdpkauxhczkmwkztcntcwj` int DEFAULT NULL,\n `iirexqntqdzkmymwcxujuoywqyngbuuf` int DEFAULT NULL,\n `upbpbystmlunxuxlknnlgkuwvkbmynmj` int DEFAULT NULL,\n `ooiyhslbtcgxsyjqkhemqntggxjyvojr` int DEFAULT NULL,\n `hsssmpilqwyhbfprmfxhnokqxdtzyjlj` int DEFAULT NULL,\n `zvcplpcdzpvgtizdjotzwekdtypygvio` int DEFAULT NULL,\n `nnhnlklgacjfqnrikhvewnmyrlwkrweg` int DEFAULT NULL,\n `bviinxfwsisxvnqxnaqzatqzquuwjced` int DEFAULT NULL,\n `evupbquadhlrenhkyviathwdsvtmabws` int DEFAULT NULL,\n `ohlcwkeebjahobzgtommvtmjbgphokwy` int DEFAULT NULL,\n `kxfzvyqevapekemwwshdzzfepopmpjub` int DEFAULT NULL,\n `wjqgchzyaphecnjgooujgzbburpsathj` int DEFAULT NULL,\n `nihtjbvalfydushdzqhkwehlvvdvfvmh` int DEFAULT NULL,\n `yjjuappwknmqmyvvksksidohqaftwbqx` int DEFAULT NULL,\n `jsgaywcryuncaolrganqspkzbopchlsk` int DEFAULT NULL,\n `omiamvpxfxsthyalxpzzbyiaziorecwo` int DEFAULT NULL,\n `pihvcavvvsnuidzkyeoaywnfqjjcwfwn` int DEFAULT NULL,\n `ksstivywnqaqmqorhxxdvtirhkbbqeaw` int DEFAULT NULL,\n `lbqglnhrinmmznshcygghjkmjlrylqia` int DEFAULT NULL,\n `xxryrixtkegjxbwczaljaybpcfnbqwaa` int DEFAULT NULL,\n `kkamdstfbhpvzjdiliklyivkljfpwmyx` int DEFAULT NULL,\n `xwaqbtnddxrgxomfkoziqbjkcxfkfjsm` int DEFAULT NULL,\n `itoewcbqnwmatbzfmnimrevcytnjmfhv` int DEFAULT NULL,\n `vycndgmvbqhnmuubetyaozsrcmtqnoja` int DEFAULT NULL,\n `sikbffyddymyzqnittkbccyggjzgaamj` int DEFAULT NULL,\n `omtrotetmqydueechtkcpzsdidmtzeio` int DEFAULT NULL,\n `nuwroedfufijsrfkcavdivfxrpqydsje` int DEFAULT NULL,\n `qnqrnorbruftgvefnqskdlfrdmhahnzl` int DEFAULT NULL,\n `pvtyybrliurozizbqxujanoflpcemezz` int DEFAULT NULL,\n `jxhssqbjcxpxnytgpujwgqduquesadqb` int DEFAULT NULL,\n `teiehkjzsoxburqobpkriaosweuctcxr` int DEFAULT NULL,\n `pknlxavplmapdnygyumehnsysflszyzp` int DEFAULT NULL,\n `kurrigyxtrfrbqvpjkbkwfwzqglcydmf` int DEFAULT NULL,\n `rxhwhppynffmnwmtzypvzbthoringubs` int DEFAULT NULL,\n `slepzegrxxgqmhjspnpbklzcprzdtqex` int DEFAULT NULL,\n `yowdwcanbpdevkbnudcjplhkbfkbwjol` int DEFAULT NULL,\n `vuzaglnmbqckvcufnyuyujyvnlxpwpzl` int DEFAULT NULL,\n `otcofkvsucusccdssnpbbevcjqngtwlw` int DEFAULT NULL,\n `dlbtdvvsqqinxggfagbmanmgxqfztmqg` int DEFAULT NULL,\n `nzrpdvvuopvbppdfmhwvyspuljbjayqm` int DEFAULT NULL,\n `wvvnuwkummpnaoagawmbcokthyyncwbr` int DEFAULT NULL,\n `jtlngezsasaxindeuusnpxovywwetspl` int DEFAULT NULL,\n `inkmthtilwikezlzngbenpkygftpvnsz` int DEFAULT NULL,\n `ejtdsivqjbhqgozqgigqphqsvqctyolv` int DEFAULT NULL,\n PRIMARY KEY (`pufirulfcfxzxdrpshbulerqeuorzcbe`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"wjejvuvpihwabfbueufasjexzxfdfwxr\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["pufirulfcfxzxdrpshbulerqeuorzcbe"],"columns":[{"name":"pufirulfcfxzxdrpshbulerqeuorzcbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"okhqnubaozbefbjnlaetuxtnsppqqsxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sqdtdpqombudlniwhnwxvazfahwvgech","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhbibfomcpaduushofskrcmydxyhzhpa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lsaqcfbrnoccdcvfmdowgqljflgnqelm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqxolvtyescuepokpilgxmkipgwcuxof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dobyvuafrwwriiiotlvstniosxcnynbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cephqjdvpmewczbfrpjstckwufgvssjq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zuyayyyvvsskuqvtgynsjqutqxfbtfad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igtdmvchqyzjcplulzsslsbbtnjxvibv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbdejtuoruftauzhcppnwyvxczscpjxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cufhklpjzpmuzcrqyhsshffviastcujk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fuziarusudzpsazhevnopprckyyvalzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yomqmnuzszqxvoqdboahmwlsqhoaoyjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ovedgtfyxozxasonzdrqcwbzfrykiwbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kbwgvbrewzyzlysrppqhjyijpqweokko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tzwulqumhcurpiqvnfgipiajoplywion","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"relagtzgdtoevpahtmvloetqmplzthkz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vwxdmzejaqkpeiymrhokbgpdbchtjfaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwwhalizzumrfpamdskfcmbdanhrzdnw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejxblmqaayhxdhsucvabxtyteshcvbov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdtqgljtpelowfkboraxxbzboghihxse","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvjzbipdpwtbjlvvmtlazzmuwavgmpja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyicmbmcsotfwiwxkgiffervapczsgbz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocrnkbecpklnodlhgnknotdqoigyimut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umrgkzctokxcmnpbvycuavylshocctxn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozicgscvxlsglwfuiwbxqcgffefjubtf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yoqnosteuspozqudidfyhqutumictoeo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqivsjzckgdeltvsogiickpykydwvxyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zfkrkobjtvmtwzpngewfuyommzkgssia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lsfzwypeteifxixjzrusphjrbwjimzmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ssgxamymcchelftuenmhbtqcqkokevgy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbujapmbxzrlplyesyvxmmwitfcquuxb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcejgslxkukdwafnoshwvmgkpcitccar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxowmmmpwqvzlwepxmhuqwdveqacrzmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnjoelmcwqbbkyzyvxlhzpaxqfrkgrjc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"filyldcukublcsbplrybhkojvmwtgwlv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtpbocuhaigmwjrdsuildvkbqppvdhfa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pcrixxwxfqgdlfxcnjnxerzvpdydepwp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bxyxiqskapxybbsjrodkiewxpklxygqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aadrlpuddzsveqwerzptzrvlhvrxcqvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypebckaifetrsvibrrqpdrarwhrhliye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wltrpkbelhupijsbrjgbdbgrmcjgeged","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgtfwmkjrntgtvadlscmvouhqrffqksd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzhexktslrphxyogyoyldpehntxoalae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xatiqpluevainjgtchzhuigbklicmkcu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lflcipkbrdehctrijwvzcmueanizdgeh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cilnqdnsizjztxvihaoigcpomtbjseix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xncuwkcgtmwtxrnsritudwckuaoaunsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zylukygjjmddjonogdfhsheulkpdomcb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okqypqeqbbkuqjmnashmorofrxopdyhr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqrlaziteifsxkwwcipkgyvgfcebigxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozuhwyvyvwjnopfruzupkoicktwixcde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bsbmkmiulnhpwxmatnexuzcxoypeoxfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfelexsgeiglywvzkoxozdtkxdosjcpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xwdsicoxyihlwifldmusqgqhybujaapp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhhfzmyaxrsdpkauxhczkmwkztcntcwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iirexqntqdzkmymwcxujuoywqyngbuuf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upbpbystmlunxuxlknnlgkuwvkbmynmj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ooiyhslbtcgxsyjqkhemqntggxjyvojr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hsssmpilqwyhbfprmfxhnokqxdtzyjlj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvcplpcdzpvgtizdjotzwekdtypygvio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nnhnlklgacjfqnrikhvewnmyrlwkrweg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bviinxfwsisxvnqxnaqzatqzquuwjced","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evupbquadhlrenhkyviathwdsvtmabws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohlcwkeebjahobzgtommvtmjbgphokwy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxfzvyqevapekemwwshdzzfepopmpjub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjqgchzyaphecnjgooujgzbburpsathj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nihtjbvalfydushdzqhkwehlvvdvfvmh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjjuappwknmqmyvvksksidohqaftwbqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jsgaywcryuncaolrganqspkzbopchlsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omiamvpxfxsthyalxpzzbyiaziorecwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pihvcavvvsnuidzkyeoaywnfqjjcwfwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ksstivywnqaqmqorhxxdvtirhkbbqeaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lbqglnhrinmmznshcygghjkmjlrylqia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxryrixtkegjxbwczaljaybpcfnbqwaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kkamdstfbhpvzjdiliklyivkljfpwmyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xwaqbtnddxrgxomfkoziqbjkcxfkfjsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itoewcbqnwmatbzfmnimrevcytnjmfhv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vycndgmvbqhnmuubetyaozsrcmtqnoja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sikbffyddymyzqnittkbccyggjzgaamj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omtrotetmqydueechtkcpzsdidmtzeio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nuwroedfufijsrfkcavdivfxrpqydsje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qnqrnorbruftgvefnqskdlfrdmhahnzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvtyybrliurozizbqxujanoflpcemezz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jxhssqbjcxpxnytgpujwgqduquesadqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"teiehkjzsoxburqobpkriaosweuctcxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pknlxavplmapdnygyumehnsysflszyzp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kurrigyxtrfrbqvpjkbkwfwzqglcydmf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxhwhppynffmnwmtzypvzbthoringubs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"slepzegrxxgqmhjspnpbklzcprzdtqex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yowdwcanbpdevkbnudcjplhkbfkbwjol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vuzaglnmbqckvcufnyuyujyvnlxpwpzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otcofkvsucusccdssnpbbevcjqngtwlw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlbtdvvsqqinxggfagbmanmgxqfztmqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzrpdvvuopvbppdfmhwvyspuljbjayqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvvnuwkummpnaoagawmbcokthyyncwbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jtlngezsasaxindeuusnpxovywwetspl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"inkmthtilwikezlzngbenpkygftpvnsz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejtdsivqjbhqgozqgigqphqsvqctyolv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672496,"databaseName":"models_schema","ddl":"CREATE TABLE `wjrpbvixffhcxmgwdtqlaaeddrzznowk` (\n `sgzzaiiuhndpqaaqesgbskamjobcwcnw` int NOT NULL,\n `haxjwvkzfmrztbapsvbquuaonjcaxwvk` int DEFAULT NULL,\n `vfhuyybyfukxonukgoleeuqqjgjpbwji` int DEFAULT NULL,\n `kbgfeszdigcvtxauqybjzkxvlzmfyqye` int DEFAULT NULL,\n `dwkyfpchjnxduwvclknvefniexfsxgcg` int DEFAULT NULL,\n `myhtqglwuawzrpuqsaloiywziwdtjnmy` int DEFAULT NULL,\n `qqepqdvbfwibejtpmdhxasqcewpwqezu` int DEFAULT NULL,\n `mxkxxthvpbkhmmwdjlrrhjikafsiasbn` int DEFAULT NULL,\n `pdtkqnhlwhajzwavzpsanepcmwxforum` int DEFAULT NULL,\n `upxmujzovsprhvlmwflpxovgvozawpjq` int DEFAULT NULL,\n `zbtyknpguhuvvaqyuhsgoqsinqjkaxjl` int DEFAULT NULL,\n `lugkfgswwyruchqxvgpccuizplzoozpy` int DEFAULT NULL,\n `spvuoqvzpyhohhejhbtipbatctiodfio` int DEFAULT NULL,\n `vphhkbkcofzuparwtgvrzsqqnfxkwhdc` int DEFAULT NULL,\n `ciofgdbbfrcpltdxiuigcxltvgjanapc` int DEFAULT NULL,\n `pmoeulwpwatljkdfhegzkkjjefjnlnrj` int DEFAULT NULL,\n `ppcuvkqsecyngdwsczfaexdbvvbtqabi` int DEFAULT NULL,\n `wwrahfwqnehsmtreyuiqowdrvztqentj` int DEFAULT NULL,\n `fcuktmynfbucpbmzgugewbuhfbtuvtfy` int DEFAULT NULL,\n `fepmqcwscoblfnvlyeswnfytdducwgaa` int DEFAULT NULL,\n `klzpedtlmbjdxooutzvxbieeorekwlhp` int DEFAULT NULL,\n `jytfeuailhgyybejouhhdacwduajprst` int DEFAULT NULL,\n `stmavzmrlkmtarlfhmszywionqalcyvy` int DEFAULT NULL,\n `fivbxxungmdykgqjncqdaepqzijrmfnw` int DEFAULT NULL,\n `wgjbvusyxljcfvaumblqgqzmteknjjuo` int DEFAULT NULL,\n `gjnriuxkbadlsoxgzeasxrknzuxevtag` int DEFAULT NULL,\n `wruzktlmknlumupijitnyxhtibqxiizm` int DEFAULT NULL,\n `dkjlpktmmhauyiiglkdqkjtyvifufjal` int DEFAULT NULL,\n `edjmrmjmskojgauilcwmmbdyzifbbdbr` int DEFAULT NULL,\n `pcxdmrvzfnhptgpfuppgserdqcnjmgre` int DEFAULT NULL,\n `vqkyleehrrlimikxphypymcthievkgdw` int DEFAULT NULL,\n `qrpxulvzqvbiwrtjjbsbmoyzehwdrqvp` int DEFAULT NULL,\n `duvrtwkxnhwlsacthhxwfkhehwgumapc` int DEFAULT NULL,\n `bbqrwymtiihwgzqkjsbndhgwuzmdffdk` int DEFAULT NULL,\n `wmosgsdrcerbyudcmbjwonrfmulfxrvz` int DEFAULT NULL,\n `lludufntstglodqlmokjryaftviyggkq` int DEFAULT NULL,\n `jpwxfljuwjhuhloiaxxanwqnewexmmzk` int DEFAULT NULL,\n `avwvcizvustowclllyfopnwoohrwwlge` int DEFAULT NULL,\n `yajleyxhsuadzgcvvmexapvyecqgezkb` int DEFAULT NULL,\n `learhcycbqkdkgbktprxylhupqdvpaot` int DEFAULT NULL,\n `klgwghcmnsdqbirptyrgewdkwcajlobv` int DEFAULT NULL,\n `kmbsdcmhpxnesjaazxqdlnzjffyfzuga` int DEFAULT NULL,\n `xkgvierufimfgjnnbvtyuqcmxxkozpke` int DEFAULT NULL,\n `mfyzbghhkuslgiagdrfhsusenjmanadn` int DEFAULT NULL,\n `jmtvzoglaovytirpqwgwhxyhbvbgtilj` int DEFAULT NULL,\n `guoqyilcyosvnlfgivxhlzoaoaxwdown` int DEFAULT NULL,\n `vugfeberfpftxazgzaeiaitwkyngccrg` int DEFAULT NULL,\n `swjhjrbrzgqnwfsrpgzmombrnnpuiqaz` int DEFAULT NULL,\n `xworfwnozcezybqnbtstgzetvybxlfki` int DEFAULT NULL,\n `omzwdcgklbxxidmdqufbdexkgctiqbth` int DEFAULT NULL,\n `kefdaehbwhjtmtmslxolxlbdevymtwik` int DEFAULT NULL,\n `ujizfvykuhjgurfwcruyybhbxnbqsrrw` int DEFAULT NULL,\n `xcjjegrqpqotpnhvatucdvgygbkynrbm` int DEFAULT NULL,\n `sggfzmlxbtsfvdtshcgcjfqnavvdhkwk` int DEFAULT NULL,\n `iqzlvscafzwjxvshcgwbgypjeguwecbh` int DEFAULT NULL,\n `sezeaezmddaaftrjfvpwwbcsxbrepjkt` int DEFAULT NULL,\n `vbtinkmvbbytoaacornhphkmowoikksq` int DEFAULT NULL,\n `wejhmaqsnaofzmxhthbgcpyozvstwarx` int DEFAULT NULL,\n `cytrcqamtwenvqcsljblwsgxyqxpaxsd` int DEFAULT NULL,\n `mzhxzvegbcgyhwjjrxixwdmepoheerqr` int DEFAULT NULL,\n `mpixjsmvhzypgszpqtojiyearafeoxdk` int DEFAULT NULL,\n `hojbivvafyvisxfnpmsauhjbkqosmtyw` int DEFAULT NULL,\n `acnjdoryeyajolbehtofsxizfdoqcrry` int DEFAULT NULL,\n `xkegsfpnwmlufplfrwtewgnzqbelvwwe` int DEFAULT NULL,\n `pxiblabxucoiuxmvclgsnwrtqivxvddg` int DEFAULT NULL,\n `cbdqabtafesehvikhhzlnvuetdelqwdj` int DEFAULT NULL,\n `erxkmctvlctuslshrxdipfiiciywnfsr` int DEFAULT NULL,\n `dqbydrjwnniedvmkmkklhzekqungaqwm` int DEFAULT NULL,\n `gdkjmuqfyrfygmdbzeqyzexjlhmmdrxh` int DEFAULT NULL,\n `gekaajxpeggbxjyxcqzezsviwohgglgz` int DEFAULT NULL,\n `vowvsbhqctrjyaarlfuitssttdfmlpzm` int DEFAULT NULL,\n `kzgxozaifezqsrjzsjljakssfqyhktdq` int DEFAULT NULL,\n `vctoxyawoujeroyghvdhiquisfeehwmt` int DEFAULT NULL,\n `giririlhzbrbkdcdtoofnaiettikujyw` int DEFAULT NULL,\n `pfqjucelrujdssenxyyprbkdxmwlgdfv` int DEFAULT NULL,\n `anrptmjmvvipwjzyjkskampqdyqadtsv` int DEFAULT NULL,\n `afxflipopovwawiwfdgyoinxloclevum` int DEFAULT NULL,\n `equgripnuxbokzaauimxjugrbrwpndlu` int DEFAULT NULL,\n `hiiehcgfsflznmeyqqyilprqstsygqeo` int DEFAULT NULL,\n `jkdxxfqacyjilcvrnaqgdaqanapbdgff` int DEFAULT NULL,\n `kqkplnxoraseazpnovwryfouazfbhpfz` int DEFAULT NULL,\n `pozgsboisqipbfxeerbrmusrojmreoev` int DEFAULT NULL,\n `lfhlncjixgzbkdrlkpthipwfjcdzquhb` int DEFAULT NULL,\n `sgnmxbzlwrpjpvobtfbskloametwfdzo` int DEFAULT NULL,\n `gcmnazqqxbreplolieqwjggmfabtgcwz` int DEFAULT NULL,\n `jvsttmoqtwjgvtrqnujmsyiemguzyswu` int DEFAULT NULL,\n `nzwddvkwqqsryjnecljcsejifsvnmzxz` int DEFAULT NULL,\n `vbmragipwpackdbiewnlkzltdkkaaasc` int DEFAULT NULL,\n `kxlkxyyoyjlbvkezrhwvcigyumxiqjgw` int DEFAULT NULL,\n `pmngkxupiwvdfdamunqpzggrmuybfwov` int DEFAULT NULL,\n `mgagegxnrtmhetlsnbtztrripijuvtri` int DEFAULT NULL,\n `daxvmgvvrdfvsadoeypezgfnmxlmifxw` int DEFAULT NULL,\n `eteuxegrcjzpiesqefwvgietrzlgomqm` int DEFAULT NULL,\n `fjfefqmqqbycqfcykjrxheqkdfizoywp` int DEFAULT NULL,\n `ospwpefgeqtdgaqredvlmmqmjcdkdojv` int DEFAULT NULL,\n `ezghjvtpuwrhaloolywltpalzdkeafra` int DEFAULT NULL,\n `hburnuyykbbrwxtkfmkbkzszqpinjoev` int DEFAULT NULL,\n `qadaqaxdkfptzxxqrefqwahqyfrudjrh` int DEFAULT NULL,\n `gecbzwwqqljbdduljadjcgcrwqxmoign` int DEFAULT NULL,\n `nkbiouqzyvmrmdzkykivjpcmstignhsy` int DEFAULT NULL,\n PRIMARY KEY (`sgzzaiiuhndpqaaqesgbskamjobcwcnw`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"wjrpbvixffhcxmgwdtqlaaeddrzznowk\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["sgzzaiiuhndpqaaqesgbskamjobcwcnw"],"columns":[{"name":"sgzzaiiuhndpqaaqesgbskamjobcwcnw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"haxjwvkzfmrztbapsvbquuaonjcaxwvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vfhuyybyfukxonukgoleeuqqjgjpbwji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kbgfeszdigcvtxauqybjzkxvlzmfyqye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwkyfpchjnxduwvclknvefniexfsxgcg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"myhtqglwuawzrpuqsaloiywziwdtjnmy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqepqdvbfwibejtpmdhxasqcewpwqezu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mxkxxthvpbkhmmwdjlrrhjikafsiasbn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdtkqnhlwhajzwavzpsanepcmwxforum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upxmujzovsprhvlmwflpxovgvozawpjq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbtyknpguhuvvaqyuhsgoqsinqjkaxjl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lugkfgswwyruchqxvgpccuizplzoozpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"spvuoqvzpyhohhejhbtipbatctiodfio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vphhkbkcofzuparwtgvrzsqqnfxkwhdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ciofgdbbfrcpltdxiuigcxltvgjanapc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmoeulwpwatljkdfhegzkkjjefjnlnrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ppcuvkqsecyngdwsczfaexdbvvbtqabi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwrahfwqnehsmtreyuiqowdrvztqentj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcuktmynfbucpbmzgugewbuhfbtuvtfy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fepmqcwscoblfnvlyeswnfytdducwgaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"klzpedtlmbjdxooutzvxbieeorekwlhp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jytfeuailhgyybejouhhdacwduajprst","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"stmavzmrlkmtarlfhmszywionqalcyvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fivbxxungmdykgqjncqdaepqzijrmfnw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgjbvusyxljcfvaumblqgqzmteknjjuo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjnriuxkbadlsoxgzeasxrknzuxevtag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wruzktlmknlumupijitnyxhtibqxiizm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkjlpktmmhauyiiglkdqkjtyvifufjal","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edjmrmjmskojgauilcwmmbdyzifbbdbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pcxdmrvzfnhptgpfuppgserdqcnjmgre","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqkyleehrrlimikxphypymcthievkgdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrpxulvzqvbiwrtjjbsbmoyzehwdrqvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"duvrtwkxnhwlsacthhxwfkhehwgumapc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbqrwymtiihwgzqkjsbndhgwuzmdffdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmosgsdrcerbyudcmbjwonrfmulfxrvz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lludufntstglodqlmokjryaftviyggkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpwxfljuwjhuhloiaxxanwqnewexmmzk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avwvcizvustowclllyfopnwoohrwwlge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yajleyxhsuadzgcvvmexapvyecqgezkb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"learhcycbqkdkgbktprxylhupqdvpaot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"klgwghcmnsdqbirptyrgewdkwcajlobv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmbsdcmhpxnesjaazxqdlnzjffyfzuga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkgvierufimfgjnnbvtyuqcmxxkozpke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfyzbghhkuslgiagdrfhsusenjmanadn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmtvzoglaovytirpqwgwhxyhbvbgtilj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"guoqyilcyosvnlfgivxhlzoaoaxwdown","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vugfeberfpftxazgzaeiaitwkyngccrg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swjhjrbrzgqnwfsrpgzmombrnnpuiqaz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xworfwnozcezybqnbtstgzetvybxlfki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omzwdcgklbxxidmdqufbdexkgctiqbth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kefdaehbwhjtmtmslxolxlbdevymtwik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujizfvykuhjgurfwcruyybhbxnbqsrrw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xcjjegrqpqotpnhvatucdvgygbkynrbm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sggfzmlxbtsfvdtshcgcjfqnavvdhkwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqzlvscafzwjxvshcgwbgypjeguwecbh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sezeaezmddaaftrjfvpwwbcsxbrepjkt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbtinkmvbbytoaacornhphkmowoikksq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wejhmaqsnaofzmxhthbgcpyozvstwarx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cytrcqamtwenvqcsljblwsgxyqxpaxsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzhxzvegbcgyhwjjrxixwdmepoheerqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpixjsmvhzypgszpqtojiyearafeoxdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hojbivvafyvisxfnpmsauhjbkqosmtyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acnjdoryeyajolbehtofsxizfdoqcrry","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkegsfpnwmlufplfrwtewgnzqbelvwwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxiblabxucoiuxmvclgsnwrtqivxvddg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbdqabtafesehvikhhzlnvuetdelqwdj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"erxkmctvlctuslshrxdipfiiciywnfsr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqbydrjwnniedvmkmkklhzekqungaqwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdkjmuqfyrfygmdbzeqyzexjlhmmdrxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gekaajxpeggbxjyxcqzezsviwohgglgz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vowvsbhqctrjyaarlfuitssttdfmlpzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzgxozaifezqsrjzsjljakssfqyhktdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vctoxyawoujeroyghvdhiquisfeehwmt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"giririlhzbrbkdcdtoofnaiettikujyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pfqjucelrujdssenxyyprbkdxmwlgdfv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"anrptmjmvvipwjzyjkskampqdyqadtsv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afxflipopovwawiwfdgyoinxloclevum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"equgripnuxbokzaauimxjugrbrwpndlu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hiiehcgfsflznmeyqqyilprqstsygqeo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jkdxxfqacyjilcvrnaqgdaqanapbdgff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqkplnxoraseazpnovwryfouazfbhpfz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pozgsboisqipbfxeerbrmusrojmreoev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfhlncjixgzbkdrlkpthipwfjcdzquhb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sgnmxbzlwrpjpvobtfbskloametwfdzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gcmnazqqxbreplolieqwjggmfabtgcwz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvsttmoqtwjgvtrqnujmsyiemguzyswu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzwddvkwqqsryjnecljcsejifsvnmzxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbmragipwpackdbiewnlkzltdkkaaasc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxlkxyyoyjlbvkezrhwvcigyumxiqjgw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmngkxupiwvdfdamunqpzggrmuybfwov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mgagegxnrtmhetlsnbtztrripijuvtri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"daxvmgvvrdfvsadoeypezgfnmxlmifxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eteuxegrcjzpiesqefwvgietrzlgomqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjfefqmqqbycqfcykjrxheqkdfizoywp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ospwpefgeqtdgaqredvlmmqmjcdkdojv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezghjvtpuwrhaloolywltpalzdkeafra","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hburnuyykbbrwxtkfmkbkzszqpinjoev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qadaqaxdkfptzxxqrefqwahqyfrudjrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gecbzwwqqljbdduljadjcgcrwqxmoign","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkbiouqzyvmrmdzkykivjpcmstignhsy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672531,"databaseName":"models_schema","ddl":"CREATE TABLE `wnbrzqlwqabujjsfpielvqcwmrnmuqwu` (\n `tcgvabuohjlyelwzaatpcdhgjbnnbspv` int NOT NULL,\n `bogcbuytxwntoujfwgfbemrlkqhnglpi` int DEFAULT NULL,\n `bemvmqhnsgpxogtpsevysihudxtunykn` int DEFAULT NULL,\n `tknlzhzijygvlnmxonzpmmkozsqcsshf` int DEFAULT NULL,\n `sjperkfvbtkplmgaaolbmehwsknxccex` int DEFAULT NULL,\n `lfmdtslsoauqqsxnuwpzwcuypgskqliu` int DEFAULT NULL,\n `tkzuizlxzccqhgkasjtgjlexudzwxzyw` int DEFAULT NULL,\n `tvwkfddcjloknskgvttchgfznkkmanut` int DEFAULT NULL,\n `ghsjxdravznotaobbreniiwmzgregobp` int DEFAULT NULL,\n `kohwcmarzzjbtmsbfledknapdieurgbj` int DEFAULT NULL,\n `wvpaxhewpfuphgloclsvndxccvqostnf` int DEFAULT NULL,\n `yqdskazsrfzgvfquxvdjjdwqzrbhklqv` int DEFAULT NULL,\n `cljvtwfxgdnoxgyfrjmtmcpyskmkbmdo` int DEFAULT NULL,\n `asrxnggztnvqxgmyikadiqhdggvhvhpm` int DEFAULT NULL,\n `gyszzdtcfpmzadvdrhcsbnzqkrmierxz` int DEFAULT NULL,\n `mludmoqslmalodtivwsrnsrebqvlxdsv` int DEFAULT NULL,\n `ewwgypvrobdysqvdmsmpqhdtikcgakpu` int DEFAULT NULL,\n `qpfhxjrapguefvtccmtfzyrgaoogdpmp` int DEFAULT NULL,\n `dpcosagzbhltumgxjrigyipwquvcgayc` int DEFAULT NULL,\n `wjtjmfmnysexedxmskpvvyhzyymuymyn` int DEFAULT NULL,\n `pwvvehvtmzwmdhrysnhibxzcbdyiccsw` int DEFAULT NULL,\n `gybpglkeuiwepbduptwzrguvfeqqsgyk` int DEFAULT NULL,\n `smajcyjqabbuhwpdrxxpmuvgvkxvpjnf` int DEFAULT NULL,\n `kjtfwtvyhcpiuoqexnacytjklnddnrwo` int DEFAULT NULL,\n `gdjykmopbylszxcrdpwfqxpfrgtgsdly` int DEFAULT NULL,\n `vqbcwiymtaawzegnwleqvqllmhndpkig` int DEFAULT NULL,\n `tjvivnomgftnlhrlhznlpspgzescuzye` int DEFAULT NULL,\n `rdtzgvwyqvrelseamqpwgunefhxylkag` int DEFAULT NULL,\n `fwzbsbmjcampwkkanpychkthtujnrhiu` int DEFAULT NULL,\n `styopsyrycfsebddaqgiyzcpjwdipdes` int DEFAULT NULL,\n `ktpvtfzuptmjbalplmtvrjharcmirfec` int DEFAULT NULL,\n `lryzrpuomffnqqyirmuwpgrwxlkmgyyi` int DEFAULT NULL,\n `wkecpvkplpstlwosajzobwmatmwdfvuc` int DEFAULT NULL,\n `gqbgkcndqgyugwgoollrmgxrittleflk` int DEFAULT NULL,\n `cfkescndgfynmyfiazcqddrsczsaudqp` int DEFAULT NULL,\n `jsaoznhnffggsneqaoxjtmcwynxvjfuw` int DEFAULT NULL,\n `omtodycvpcobpclpbmnflxlqevlyjlxr` int DEFAULT NULL,\n `lhwodzqmylmmbhglfnuvnaagqxknecrp` int DEFAULT NULL,\n `xdvroczszliwozrvqcicjsbhttfcsedx` int DEFAULT NULL,\n `kfxztyuuazofpgztjitijhdbloomvchj` int DEFAULT NULL,\n `tnvkxpygpovolemftkxplxczfpvwwjhi` int DEFAULT NULL,\n `qfpegdqgqkkvljpowxjfbmlzckhmqmbf` int DEFAULT NULL,\n `xloypeqlfauyhqtyrfkrsghscylvnrkh` int DEFAULT NULL,\n `vgfcyfgpywdznkrhnpavinsfwywwflzd` int DEFAULT NULL,\n `bjtfdosjfaebzfegzmzvihkbmaioyfln` int DEFAULT NULL,\n `evghluwwkbsqxcgmbijekgsisosmbihw` int DEFAULT NULL,\n `wswimufbcmrakycvdggqyklfbfpsfbpz` int DEFAULT NULL,\n `bxdizbqnovkcoshjhxfjvhiwoljjnxth` int DEFAULT NULL,\n `ucwkxryitjcvdxxzdelbohvjyevalwoa` int DEFAULT NULL,\n `tslqceddxgfvmpuywyjgbizgxveveelh` int DEFAULT NULL,\n `zsypjxviqrcmiwaqeruodrnqzbiurmcn` int DEFAULT NULL,\n `ccasqsoopjyajirycxmvhteigoyfkrrn` int DEFAULT NULL,\n `jkyrjbhcodywrqqhwmhzatnpestpiwjd` int DEFAULT NULL,\n `pyqvnaucpxgsnwpnfdmncpsdfelsvuir` int DEFAULT NULL,\n `upnlkmiylqveflcwxhyplrfvodynebxr` int DEFAULT NULL,\n `yizhwnonwwfvliyhaedcfhgqeqqkjyhj` int DEFAULT NULL,\n `bwogczhvkwvvtorllczskjhpwbhwahtz` int DEFAULT NULL,\n `rezcjrpdxcwfiwpleifclrprgebokcqm` int DEFAULT NULL,\n `gzfquhduiizwbzwhihruacfmrdpjrigq` int DEFAULT NULL,\n `ezujmgpqdclbavushkyywybzvjrlljtq` int DEFAULT NULL,\n `skulrcrbckqvmoohnkpvbhmzjkkwnhfg` int DEFAULT NULL,\n `xltqhbpntqvzckdwjxcmempuykkvknmc` int DEFAULT NULL,\n `achlmslvawgulcgzzojdlzpeebsqdvqv` int DEFAULT NULL,\n `anlixkdslqhmeoipbagkhrbkrhvkdcmw` int DEFAULT NULL,\n `cfcwvawexcrquhjmafmgzrwfxollaeta` int DEFAULT NULL,\n `ibnehwpxryemjpdstguwgjactpjrsqfs` int DEFAULT NULL,\n `sszgejwaybqbkevzpyudvlawgdgaupwc` int DEFAULT NULL,\n `ecqknlrwsxbbqzeffqlqjzusprlqtfwo` int DEFAULT NULL,\n `wdcbyccyfaftbrxvluuiyhiutzbnprpw` int DEFAULT NULL,\n `sqqwktaksjoqfwxejwdgpvmmnhqzdcfc` int DEFAULT NULL,\n `woivzqjoxjccawkoyxdejrpmuysngosk` int DEFAULT NULL,\n `udivhdrhnnpkvpfspvbstzffgyzinyom` int DEFAULT NULL,\n `qtnpbmxdtysaprcfllmwrzlttqcvxjcq` int DEFAULT NULL,\n `hizsajzuxjrfksxzojtyqwwdslrwifnp` int DEFAULT NULL,\n `zuvgblycwkywmyvvrvlxgeaqmbuwwzgh` int DEFAULT NULL,\n `kgubynzkgfangucjkfwerqdxrhwzgcrx` int DEFAULT NULL,\n `nebufcyfocxvrkkjgrfqetalxvbupwdx` int DEFAULT NULL,\n `cuovxgxhfzemhiyactialoladnfnooaq` int DEFAULT NULL,\n `iavoglsolmcqyqjcatzjcmljlsjptkyv` int DEFAULT NULL,\n `nnwnacxrjyfoswbdsycszgrqysmfqkgj` int DEFAULT NULL,\n `ccephapsqoboyqkbxewzikhogblmindq` int DEFAULT NULL,\n `lfmfpmxicwgrnmmhsdwbifzehdfbzzej` int DEFAULT NULL,\n `rvksnhytbcvtcopmtpecclnoikzlgwhz` int DEFAULT NULL,\n `easruyyrknwigklottzdbkptsynnqyoh` int DEFAULT NULL,\n `sflbtojiljbfypdhetstjflpeohxiipl` int DEFAULT NULL,\n `uqsehgidnakfnjxvijcyvlraziowxjdp` int DEFAULT NULL,\n `yqnvzffvfuhivczbklaukvvbhzcehbqo` int DEFAULT NULL,\n `tpwkqrrsgfedfdeorftfiobhlrzezdmv` int DEFAULT NULL,\n `wfprkyabdbwwisraeijwuwihzddnlkts` int DEFAULT NULL,\n `spruwrjlagtgbyuxdyayaebwihvnueim` int DEFAULT NULL,\n `doughngvhrpaejzhlffgyvieslgplelt` int DEFAULT NULL,\n `xkpzdaowrksyostrojgrcvptotldppml` int DEFAULT NULL,\n `mwojpeerltqapdljbrzaypjtoiufuzcw` int DEFAULT NULL,\n `qfrsgogudxkmbiqxueponilnzrpbhokw` int DEFAULT NULL,\n `dpxbczfqqwzcpbraswpcipujdtfwrspk` int DEFAULT NULL,\n `lpvjkhuhmagxyprxrdhlyvqdqhrjuddu` int DEFAULT NULL,\n `pnfoyhrobitdemnqubjzoyxqhmefyxwp` int DEFAULT NULL,\n `uttotcsycnlrbipaibkhhufgpribubzm` int DEFAULT NULL,\n `yyvkzmkhbzhcgjcopquqngiocfutolna` int DEFAULT NULL,\n `vqahrylugsvnzbemsqwfrmzddhkfuuwr` int DEFAULT NULL,\n PRIMARY KEY (`tcgvabuohjlyelwzaatpcdhgjbnnbspv`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"wnbrzqlwqabujjsfpielvqcwmrnmuqwu\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["tcgvabuohjlyelwzaatpcdhgjbnnbspv"],"columns":[{"name":"tcgvabuohjlyelwzaatpcdhgjbnnbspv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"bogcbuytxwntoujfwgfbemrlkqhnglpi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bemvmqhnsgpxogtpsevysihudxtunykn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tknlzhzijygvlnmxonzpmmkozsqcsshf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjperkfvbtkplmgaaolbmehwsknxccex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfmdtslsoauqqsxnuwpzwcuypgskqliu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkzuizlxzccqhgkasjtgjlexudzwxzyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvwkfddcjloknskgvttchgfznkkmanut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghsjxdravznotaobbreniiwmzgregobp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kohwcmarzzjbtmsbfledknapdieurgbj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvpaxhewpfuphgloclsvndxccvqostnf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqdskazsrfzgvfquxvdjjdwqzrbhklqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cljvtwfxgdnoxgyfrjmtmcpyskmkbmdo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"asrxnggztnvqxgmyikadiqhdggvhvhpm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gyszzdtcfpmzadvdrhcsbnzqkrmierxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mludmoqslmalodtivwsrnsrebqvlxdsv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ewwgypvrobdysqvdmsmpqhdtikcgakpu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qpfhxjrapguefvtccmtfzyrgaoogdpmp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpcosagzbhltumgxjrigyipwquvcgayc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjtjmfmnysexedxmskpvvyhzyymuymyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwvvehvtmzwmdhrysnhibxzcbdyiccsw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gybpglkeuiwepbduptwzrguvfeqqsgyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smajcyjqabbuhwpdrxxpmuvgvkxvpjnf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjtfwtvyhcpiuoqexnacytjklnddnrwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdjykmopbylszxcrdpwfqxpfrgtgsdly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqbcwiymtaawzegnwleqvqllmhndpkig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjvivnomgftnlhrlhznlpspgzescuzye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdtzgvwyqvrelseamqpwgunefhxylkag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwzbsbmjcampwkkanpychkthtujnrhiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"styopsyrycfsebddaqgiyzcpjwdipdes","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ktpvtfzuptmjbalplmtvrjharcmirfec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lryzrpuomffnqqyirmuwpgrwxlkmgyyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wkecpvkplpstlwosajzobwmatmwdfvuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqbgkcndqgyugwgoollrmgxrittleflk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfkescndgfynmyfiazcqddrsczsaudqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jsaoznhnffggsneqaoxjtmcwynxvjfuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omtodycvpcobpclpbmnflxlqevlyjlxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhwodzqmylmmbhglfnuvnaagqxknecrp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdvroczszliwozrvqcicjsbhttfcsedx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfxztyuuazofpgztjitijhdbloomvchj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnvkxpygpovolemftkxplxczfpvwwjhi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfpegdqgqkkvljpowxjfbmlzckhmqmbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xloypeqlfauyhqtyrfkrsghscylvnrkh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgfcyfgpywdznkrhnpavinsfwywwflzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjtfdosjfaebzfegzmzvihkbmaioyfln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evghluwwkbsqxcgmbijekgsisosmbihw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wswimufbcmrakycvdggqyklfbfpsfbpz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bxdizbqnovkcoshjhxfjvhiwoljjnxth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ucwkxryitjcvdxxzdelbohvjyevalwoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tslqceddxgfvmpuywyjgbizgxveveelh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zsypjxviqrcmiwaqeruodrnqzbiurmcn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ccasqsoopjyajirycxmvhteigoyfkrrn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jkyrjbhcodywrqqhwmhzatnpestpiwjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pyqvnaucpxgsnwpnfdmncpsdfelsvuir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upnlkmiylqveflcwxhyplrfvodynebxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yizhwnonwwfvliyhaedcfhgqeqqkjyhj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bwogczhvkwvvtorllczskjhpwbhwahtz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rezcjrpdxcwfiwpleifclrprgebokcqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzfquhduiizwbzwhihruacfmrdpjrigq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezujmgpqdclbavushkyywybzvjrlljtq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skulrcrbckqvmoohnkpvbhmzjkkwnhfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xltqhbpntqvzckdwjxcmempuykkvknmc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"achlmslvawgulcgzzojdlzpeebsqdvqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"anlixkdslqhmeoipbagkhrbkrhvkdcmw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfcwvawexcrquhjmafmgzrwfxollaeta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibnehwpxryemjpdstguwgjactpjrsqfs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sszgejwaybqbkevzpyudvlawgdgaupwc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecqknlrwsxbbqzeffqlqjzusprlqtfwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wdcbyccyfaftbrxvluuiyhiutzbnprpw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sqqwktaksjoqfwxejwdgpvmmnhqzdcfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"woivzqjoxjccawkoyxdejrpmuysngosk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udivhdrhnnpkvpfspvbstzffgyzinyom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtnpbmxdtysaprcfllmwrzlttqcvxjcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hizsajzuxjrfksxzojtyqwwdslrwifnp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zuvgblycwkywmyvvrvlxgeaqmbuwwzgh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kgubynzkgfangucjkfwerqdxrhwzgcrx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nebufcyfocxvrkkjgrfqetalxvbupwdx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cuovxgxhfzemhiyactialoladnfnooaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iavoglsolmcqyqjcatzjcmljlsjptkyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nnwnacxrjyfoswbdsycszgrqysmfqkgj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ccephapsqoboyqkbxewzikhogblmindq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfmfpmxicwgrnmmhsdwbifzehdfbzzej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rvksnhytbcvtcopmtpecclnoikzlgwhz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"easruyyrknwigklottzdbkptsynnqyoh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sflbtojiljbfypdhetstjflpeohxiipl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqsehgidnakfnjxvijcyvlraziowxjdp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqnvzffvfuhivczbklaukvvbhzcehbqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpwkqrrsgfedfdeorftfiobhlrzezdmv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfprkyabdbwwisraeijwuwihzddnlkts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"spruwrjlagtgbyuxdyayaebwihvnueim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"doughngvhrpaejzhlffgyvieslgplelt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkpzdaowrksyostrojgrcvptotldppml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwojpeerltqapdljbrzaypjtoiufuzcw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfrsgogudxkmbiqxueponilnzrpbhokw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpxbczfqqwzcpbraswpcipujdtfwrspk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lpvjkhuhmagxyprxrdhlyvqdqhrjuddu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnfoyhrobitdemnqubjzoyxqhmefyxwp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uttotcsycnlrbipaibkhhufgpribubzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yyvkzmkhbzhcgjcopquqngiocfutolna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqahrylugsvnzbemsqwfrmzddhkfuuwr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672562,"databaseName":"models_schema","ddl":"CREATE TABLE `wouewinkcurfknfxilwfeellhzmeqtjq` (\n `yjicwzyqezwskgcjloupynhpfbjcykgc` int NOT NULL,\n `jkbubojqlzeutratfmckybrlnchfsbui` int DEFAULT NULL,\n `enfnqvkggpgjvpimkyhkkblodragnzla` int DEFAULT NULL,\n `rlsofoiwrkobytiggpcpmrsnbdtobtjq` int DEFAULT NULL,\n `xqddajwwqzwjdpsncxaxjlrbvtiicuxj` int DEFAULT NULL,\n `zttumzfvmiejemdfonzfbibdfriiquwt` int DEFAULT NULL,\n `irkpyjzxtyurfokloisttjccvvbcbzhe` int DEFAULT NULL,\n `chakfykvdihheflcshkxldzyncslxreb` int DEFAULT NULL,\n `sjzwvkcrwqebbtflcrvjcrvnnszazwsg` int DEFAULT NULL,\n `zqrsnwsfrnejkpjoularayxdxjuohoqq` int DEFAULT NULL,\n `uoiucritqwatpgurlmszyjejpublngoy` int DEFAULT NULL,\n `qmskkaidybpzkeuwgplhikbascsrapyn` int DEFAULT NULL,\n `ecreswbiyttiiacwuyvkanymwshfirkf` int DEFAULT NULL,\n `gkpubsqufjsmlvyzykdjprpusoysyuyp` int DEFAULT NULL,\n `ihjkasvvwjkhpjiyvmmmdgsjfdfzqega` int DEFAULT NULL,\n `yphixgiquelnmfnpkhofpakwrjcabdzb` int DEFAULT NULL,\n `beopodfmssvzlquvdwgqooqnwerxjsmm` int DEFAULT NULL,\n `zjzgsdkjtrxbucocytfasassjuagbpju` int DEFAULT NULL,\n `hoidjsyrmfozaozvtbacqfecoxzopfpu` int DEFAULT NULL,\n `dqawgxyluybzyzslgndmmnakxfullidx` int DEFAULT NULL,\n `lbszgstegibvljuyqorenihbpjhwvaug` int DEFAULT NULL,\n `hsxjlpawsfcnkvsglpzwxjzsntmwldvl` int DEFAULT NULL,\n `vlotbqovzkxryxtfxcvqyfkilaffynfk` int DEFAULT NULL,\n `fjpoyrooyvzepnaaywbqukuqmentgjkf` int DEFAULT NULL,\n `ugdmdppctackdnywxpvhvpzylyryhrcn` int DEFAULT NULL,\n `ebwgpouzriltztxtyfdedrkwhvmebzfi` int DEFAULT NULL,\n `immewtbxqgwspweiqvmtvenxqitvvpsa` int DEFAULT NULL,\n `nligsqaovauhegnjnufytgovyngyooei` int DEFAULT NULL,\n `tmyupfinxbdvlugwvvqghxlhnutiairz` int DEFAULT NULL,\n `zlxoeuqnvwjudnfwkvdzterqnywltlmf` int DEFAULT NULL,\n `ywxrwgjdvsyfbaonfrcxxaqlrbnzydbs` int DEFAULT NULL,\n `kzrvrjzrvyhbrcyqullbsvxuksdyxmnd` int DEFAULT NULL,\n `uthixbrcivxcptpsljaoeclnesjxyebw` int DEFAULT NULL,\n `hkwnipgdwojngphbnmbwjmghcxdmyoom` int DEFAULT NULL,\n `dgteqyxobllcviddhwocjuhrawpwhrlr` int DEFAULT NULL,\n `pnfdmpeikladiyafqpsvracsifnmkqtk` int DEFAULT NULL,\n `swkfxmuazogonhnjhpsbsfmrcceokfom` int DEFAULT NULL,\n `avbotpdmazrgxarfrgrbiwhsjbszncua` int DEFAULT NULL,\n `opkcssiwaddugyqxblesnaampuggmfwm` int DEFAULT NULL,\n `swzkiawnoixucgwohfmhyicjwzajtrvd` int DEFAULT NULL,\n `bsdtzgxzgylhdkloteoxdhrysxzqnikx` int DEFAULT NULL,\n `gtcgbgvjblxnozxrxjvlwqqujmfsdhbd` int DEFAULT NULL,\n `wzaxygxdmlfuvnoujccmwjulgugmeaen` int DEFAULT NULL,\n `wupumkhwyivkchuducaupjgipfkgytlw` int DEFAULT NULL,\n `tnfkoykqmjoywlctsbojvedtdfhxnyzs` int DEFAULT NULL,\n `thjkzivsodnjktbhqqtlofqgzhfngffo` int DEFAULT NULL,\n `qwmffuauclqfqqvdrtsvgirnybswpdoi` int DEFAULT NULL,\n `qfesucdhmruerwvncfzvolbjjsoktefr` int DEFAULT NULL,\n `ixncahmztyfsbqjmsetctiygqahjgwsi` int DEFAULT NULL,\n `tbopeptcsxnmthxkskiolauvyufsmqyz` int DEFAULT NULL,\n `gmrefnegbefhvgqlekevulslkjooqupa` int DEFAULT NULL,\n `pickyrnwnbaezmxklqzyamzfwernsnvx` int DEFAULT NULL,\n `ozxtrarhzovuuiemclitvoxqlukoqrws` int DEFAULT NULL,\n `fszxskpkqirmrnjrsmsmtyknpsnbyyyi` int DEFAULT NULL,\n `zgwqiptrvpsdechugyvhbhnicbiwadda` int DEFAULT NULL,\n `lqfskcpvxzheomupptxbscwsegyxeeuz` int DEFAULT NULL,\n `jgrzdgnrrwfuilsufozfhinwmdgdqjrn` int DEFAULT NULL,\n `etzxedclgmhdqhxgnnoodgegmcanjkzh` int DEFAULT NULL,\n `xntttgpcwxifgbkiprnghfuiuciigtlt` int DEFAULT NULL,\n `suegeortpvmoflpynqmtdencfoqdqqbe` int DEFAULT NULL,\n `msyphgqpcsvgurxbvosodwhaitdlabba` int DEFAULT NULL,\n `hmwloececrjwueokncemhxrpynvdcjpo` int DEFAULT NULL,\n `lhblvkvgusrmeublfnbacthjoewfbtmt` int DEFAULT NULL,\n `wshedvupauauucvymoplfcolelawkmtv` int DEFAULT NULL,\n `wrsvfzgbvheqwnfhtncdajhxiqajqics` int DEFAULT NULL,\n `nwasfouzcxvnfzmhaygclfjacnnbptup` int DEFAULT NULL,\n `bbcbffdzjdqhfajlppjaajhvxcgwhpez` int DEFAULT NULL,\n `gumbfcebfvvsbolzuzpuuiapasrhwvvy` int DEFAULT NULL,\n `ryqxfjpxiiaeuvdbkcedqrmzigthpqaa` int DEFAULT NULL,\n `budxftaetvgpzlbiicuomozcewhelxxq` int DEFAULT NULL,\n `nelchkbhtlpvprpiukiypykdsyczpibt` int DEFAULT NULL,\n `ognnrsdzbzopircysxsqdttnsgxiavii` int DEFAULT NULL,\n `eqobfaonfbumzowidiwzfxdabtvljdhb` int DEFAULT NULL,\n `obosyqkenfptbksfvuekclxpxhuxmezg` int DEFAULT NULL,\n `btaazgplnjxyikfslzuopqjipzvsxmks` int DEFAULT NULL,\n `rgirhkcalevonsqymyxmlubmihjbriav` int DEFAULT NULL,\n `geowsjnsleuvhjizxmsjgzhecruqpthl` int DEFAULT NULL,\n `ljlypdivxjtgqqxjxurepatighmolgun` int DEFAULT NULL,\n `nawowsalsvayfumgynlxoexrxcwlxnbf` int DEFAULT NULL,\n `gfoclbkdyfqzlahpqetppgldlpylqlfy` int DEFAULT NULL,\n `jtvtecqzmugzrempjluetqqqxqjnwlsh` int DEFAULT NULL,\n `jdgmzjcjdmlzyjatckshyeusebcwbaqz` int DEFAULT NULL,\n `lrhtohzxqkftqptaqwpbnouloluohomn` int DEFAULT NULL,\n `vqmnbruhsdycroenzvavqfyqjcyiilnw` int DEFAULT NULL,\n `uuvicghowskpuxqzfvjhmeonvsgvnmzs` int DEFAULT NULL,\n `edjmamghpktmbxfgzlhimwknxjhtuhok` int DEFAULT NULL,\n `mzpfkmoqolgpjwstdkyhaguqvxdcjstv` int DEFAULT NULL,\n `rohpxosazrerjitvhbtyimvqdgrhvdky` int DEFAULT NULL,\n `zqwooxdktjmbspgsnngkkawapnyiqyao` int DEFAULT NULL,\n `uvfpxjglwzrdejnehmcstdefyksfnfwi` int DEFAULT NULL,\n `hfqondpqwkozxlqdnhmjtelbrfccuknq` int DEFAULT NULL,\n `naqzfccggnfqakjjzhcpwgcbgmhqtnlh` int DEFAULT NULL,\n `xvvshguvsciaecwoabeotpmaujypfknm` int DEFAULT NULL,\n `wotqdzmoepvljmyqhqgcppllfxqicapl` int DEFAULT NULL,\n `bnywcgifjgaptuqszjaukyybppnwvrzd` int DEFAULT NULL,\n `jnkzvokkebtczfalozctkdrbppugeiqw` int DEFAULT NULL,\n `tsnhdpqufijylordwyevlntrnswywgbe` int DEFAULT NULL,\n `xcmkydttjhbicbdvmyutblthwpuvaqtv` int DEFAULT NULL,\n `rbflnqekaqmynnxvseelmtbnhgllotfd` int DEFAULT NULL,\n `xwqwxwlkqjpdjrehvxznxtptamxfccik` int DEFAULT NULL,\n PRIMARY KEY (`yjicwzyqezwskgcjloupynhpfbjcykgc`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"wouewinkcurfknfxilwfeellhzmeqtjq\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["yjicwzyqezwskgcjloupynhpfbjcykgc"],"columns":[{"name":"yjicwzyqezwskgcjloupynhpfbjcykgc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"jkbubojqlzeutratfmckybrlnchfsbui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"enfnqvkggpgjvpimkyhkkblodragnzla","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlsofoiwrkobytiggpcpmrsnbdtobtjq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqddajwwqzwjdpsncxaxjlrbvtiicuxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zttumzfvmiejemdfonzfbibdfriiquwt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"irkpyjzxtyurfokloisttjccvvbcbzhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"chakfykvdihheflcshkxldzyncslxreb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjzwvkcrwqebbtflcrvjcrvnnszazwsg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqrsnwsfrnejkpjoularayxdxjuohoqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uoiucritqwatpgurlmszyjejpublngoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmskkaidybpzkeuwgplhikbascsrapyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecreswbiyttiiacwuyvkanymwshfirkf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkpubsqufjsmlvyzykdjprpusoysyuyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ihjkasvvwjkhpjiyvmmmdgsjfdfzqega","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yphixgiquelnmfnpkhofpakwrjcabdzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"beopodfmssvzlquvdwgqooqnwerxjsmm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjzgsdkjtrxbucocytfasassjuagbpju","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hoidjsyrmfozaozvtbacqfecoxzopfpu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqawgxyluybzyzslgndmmnakxfullidx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lbszgstegibvljuyqorenihbpjhwvaug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hsxjlpawsfcnkvsglpzwxjzsntmwldvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vlotbqovzkxryxtfxcvqyfkilaffynfk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjpoyrooyvzepnaaywbqukuqmentgjkf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugdmdppctackdnywxpvhvpzylyryhrcn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebwgpouzriltztxtyfdedrkwhvmebzfi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"immewtbxqgwspweiqvmtvenxqitvvpsa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nligsqaovauhegnjnufytgovyngyooei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmyupfinxbdvlugwvvqghxlhnutiairz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zlxoeuqnvwjudnfwkvdzterqnywltlmf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywxrwgjdvsyfbaonfrcxxaqlrbnzydbs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzrvrjzrvyhbrcyqullbsvxuksdyxmnd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uthixbrcivxcptpsljaoeclnesjxyebw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkwnipgdwojngphbnmbwjmghcxdmyoom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgteqyxobllcviddhwocjuhrawpwhrlr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnfdmpeikladiyafqpsvracsifnmkqtk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swkfxmuazogonhnjhpsbsfmrcceokfom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avbotpdmazrgxarfrgrbiwhsjbszncua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opkcssiwaddugyqxblesnaampuggmfwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swzkiawnoixucgwohfmhyicjwzajtrvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bsdtzgxzgylhdkloteoxdhrysxzqnikx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtcgbgvjblxnozxrxjvlwqqujmfsdhbd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzaxygxdmlfuvnoujccmwjulgugmeaen","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wupumkhwyivkchuducaupjgipfkgytlw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnfkoykqmjoywlctsbojvedtdfhxnyzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thjkzivsodnjktbhqqtlofqgzhfngffo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwmffuauclqfqqvdrtsvgirnybswpdoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfesucdhmruerwvncfzvolbjjsoktefr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixncahmztyfsbqjmsetctiygqahjgwsi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbopeptcsxnmthxkskiolauvyufsmqyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmrefnegbefhvgqlekevulslkjooqupa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pickyrnwnbaezmxklqzyamzfwernsnvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozxtrarhzovuuiemclitvoxqlukoqrws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fszxskpkqirmrnjrsmsmtyknpsnbyyyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zgwqiptrvpsdechugyvhbhnicbiwadda","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lqfskcpvxzheomupptxbscwsegyxeeuz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgrzdgnrrwfuilsufozfhinwmdgdqjrn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"etzxedclgmhdqhxgnnoodgegmcanjkzh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xntttgpcwxifgbkiprnghfuiuciigtlt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suegeortpvmoflpynqmtdencfoqdqqbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"msyphgqpcsvgurxbvosodwhaitdlabba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmwloececrjwueokncemhxrpynvdcjpo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhblvkvgusrmeublfnbacthjoewfbtmt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wshedvupauauucvymoplfcolelawkmtv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wrsvfzgbvheqwnfhtncdajhxiqajqics","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nwasfouzcxvnfzmhaygclfjacnnbptup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbcbffdzjdqhfajlppjaajhvxcgwhpez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gumbfcebfvvsbolzuzpuuiapasrhwvvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ryqxfjpxiiaeuvdbkcedqrmzigthpqaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"budxftaetvgpzlbiicuomozcewhelxxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nelchkbhtlpvprpiukiypykdsyczpibt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ognnrsdzbzopircysxsqdttnsgxiavii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqobfaonfbumzowidiwzfxdabtvljdhb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obosyqkenfptbksfvuekclxpxhuxmezg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"btaazgplnjxyikfslzuopqjipzvsxmks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgirhkcalevonsqymyxmlubmihjbriav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"geowsjnsleuvhjizxmsjgzhecruqpthl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljlypdivxjtgqqxjxurepatighmolgun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nawowsalsvayfumgynlxoexrxcwlxnbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfoclbkdyfqzlahpqetppgldlpylqlfy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jtvtecqzmugzrempjluetqqqxqjnwlsh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdgmzjcjdmlzyjatckshyeusebcwbaqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lrhtohzxqkftqptaqwpbnouloluohomn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqmnbruhsdycroenzvavqfyqjcyiilnw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uuvicghowskpuxqzfvjhmeonvsgvnmzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edjmamghpktmbxfgzlhimwknxjhtuhok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzpfkmoqolgpjwstdkyhaguqvxdcjstv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rohpxosazrerjitvhbtyimvqdgrhvdky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqwooxdktjmbspgsnngkkawapnyiqyao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvfpxjglwzrdejnehmcstdefyksfnfwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfqondpqwkozxlqdnhmjtelbrfccuknq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"naqzfccggnfqakjjzhcpwgcbgmhqtnlh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvvshguvsciaecwoabeotpmaujypfknm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wotqdzmoepvljmyqhqgcppllfxqicapl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnywcgifjgaptuqszjaukyybppnwvrzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jnkzvokkebtczfalozctkdrbppugeiqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tsnhdpqufijylordwyevlntrnswywgbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xcmkydttjhbicbdvmyutblthwpuvaqtv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbflnqekaqmynnxvseelmtbnhgllotfd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xwqwxwlkqjpdjrehvxznxtptamxfccik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672591,"databaseName":"models_schema","ddl":"CREATE TABLE `wqdandhuzkcuajvglffrvuwqfxsfunjn` (\n `uiwtfchkgeetmsrbvopdiasjiyggyrms` int NOT NULL,\n `lxvkxukqoafsppuqggaifzjilendovgw` int DEFAULT NULL,\n `qjtrgjmcuwatdhijachimdhafamikjff` int DEFAULT NULL,\n `wjarvommzwtrbplgyfgcfzinllirynit` int DEFAULT NULL,\n `kxsrdizuojljscijkebyusuqiavytdaf` int DEFAULT NULL,\n `ikagnmjcdmyvoffngolwzbzdjudquatj` int DEFAULT NULL,\n `tethaxeqcilfkfrwqifqtdgitkbyxaxx` int DEFAULT NULL,\n `avkndaalczbqmsnuwmecdqdigeuhmksn` int DEFAULT NULL,\n `owshymzrezmjoaiorajbcfvjcmbubrrv` int DEFAULT NULL,\n `puqpikutypcgumusmfqwqbfgiomijpsy` int DEFAULT NULL,\n `jazpefwvngrmvtawsaplckicdoeeafen` int DEFAULT NULL,\n `meqkmxgfiirpyjuoslwexrercwqorlcr` int DEFAULT NULL,\n `hmihlmsvcwabvkfxqndskzwrrtruwghj` int DEFAULT NULL,\n `bnmnhuwqqoqyevekzluyguhkrghifqqq` int DEFAULT NULL,\n `axcevbjlaivoenkavegxtcnrxumthocn` int DEFAULT NULL,\n `xquephrqxwtfdvblxclitttahiqgsuxr` int DEFAULT NULL,\n `wljchrbgdajnqrcotbqemuhhwctpvzyo` int DEFAULT NULL,\n `ibdybdkdrifkpktmrcpbfjdmvmsepibz` int DEFAULT NULL,\n `gwgvjyqxhzhbnxlgrufnstwihpwkuqrh` int DEFAULT NULL,\n `wftofimyxdbazyitdjutvumoqaxojbsv` int DEFAULT NULL,\n `tgvmudcmjdylggkpfsextnnqqyftksui` int DEFAULT NULL,\n `isakhcevzsyrswecgzvynirwyqwmwpak` int DEFAULT NULL,\n `ilgtkmodphvmphdkigecntyyjcofabvv` int DEFAULT NULL,\n `jfrfwuhkorabqsdppsczpposbjehisbf` int DEFAULT NULL,\n `bffsftgzghfllvczttfosnuzjuukckrt` int DEFAULT NULL,\n `xxzrdnmjrdtlmcuyjggybyhzwplcdkcj` int DEFAULT NULL,\n `pnvjnxxieooyrzfnxhiyqobqzbzgklfp` int DEFAULT NULL,\n `kxterqtmzkswvvdfgazcoozllcidsbbe` int DEFAULT NULL,\n `zzjeoahgzqdzcosjzgojnadmwmgpbbgk` int DEFAULT NULL,\n `mtetwpttviinvmizmzekznwuzcffbjfl` int DEFAULT NULL,\n `uqsnvokmwamnxsmhetmlfudzbphnzkaq` int DEFAULT NULL,\n `fwyrugjxhswzydmqbsxqvxpbfsipdcug` int DEFAULT NULL,\n `vkdejxvdsvgwxqfwqanfzkzleuqlyxwv` int DEFAULT NULL,\n `nfkmftkmxwafxqlibzxmikvnrmzmswqb` int DEFAULT NULL,\n `mqcmuktevnokntxqbzazvcgpskhbzbtc` int DEFAULT NULL,\n `fdxopngpllvhsmepwftkcewgbidrglje` int DEFAULT NULL,\n `xrtwijbasuiapdentieezfdjxylhmypp` int DEFAULT NULL,\n `dnuyyuwvctoqelvxlkopkvlnzftdhygc` int DEFAULT NULL,\n `sxwhdqhxqsgbqesazaeeoilkmviwspay` int DEFAULT NULL,\n `sjsmoqkrlkzsxfihuirdcyccrvwxywvt` int DEFAULT NULL,\n `izlimagrqtnsuwfqnqckiwkelhtqpxzl` int DEFAULT NULL,\n `pwvbsaggcgzhzbblhvjgivtwsrplfryg` int DEFAULT NULL,\n `jyxcuqlxwadjglppblnexcorpicwrueo` int DEFAULT NULL,\n `zssfefkykfbtzzgmoaovfaefmxvdugpk` int DEFAULT NULL,\n `titjxacwkqxnlrirvohrnmwhvmoyucji` int DEFAULT NULL,\n `pqcokuegdfxlswfeeuplktpmjnhhhkrs` int DEFAULT NULL,\n `wotunpxhpibkjvavhnlxsbfuvsuwugao` int DEFAULT NULL,\n `wkojdknhircppykajjswsbrmwfetreyv` int DEFAULT NULL,\n `kiojvzcfuayncjkpygquqjmugmuflfvf` int DEFAULT NULL,\n `cpxpwbgymdzpbizjazdapwjrwghehnvc` int DEFAULT NULL,\n `jcjmyptdlpnesmbksqjylcxsrajdsjrf` int DEFAULT NULL,\n `higmpfvjxyvgxexeuxiuhhmdncdynvoh` int DEFAULT NULL,\n `ugehobueizzeulewqpmwnrrkcshzpdkk` int DEFAULT NULL,\n `elknfgflyqlodwtdpggekowkpwvfsohe` int DEFAULT NULL,\n `wwnccxdyyiruxqlnggkslbmmbulfadps` int DEFAULT NULL,\n `sxidtkgwrpmgpzocfpzjmxuyxfjkblvf` int DEFAULT NULL,\n `stznkdmhzufwwzrcsskzeyxbqegijvwz` int DEFAULT NULL,\n `kzcowebcnavonjzvxblrjskwzbefezvu` int DEFAULT NULL,\n `nlowhbwbyvcyltdyubvnmxzwrpdmsrog` int DEFAULT NULL,\n `jrpnzezfppuskxkgrhffbsvrbwixzgbd` int DEFAULT NULL,\n `fbdeicvymlypqohkmmixlqdmxwkriefw` int DEFAULT NULL,\n `mogkgpdrwnnellunqgbcnlohjdgdeozs` int DEFAULT NULL,\n `kgncahnymghuainmjfwxukseiwrddrma` int DEFAULT NULL,\n `dmdrimpubeuhozcywcadmugpifmpkwne` int DEFAULT NULL,\n `jmihinimicgbyqytbfceurwjltxsxybg` int DEFAULT NULL,\n `iefntsnqjdpzvxqfiwhuanwhnsxjsztg` int DEFAULT NULL,\n `fseklqxhtlrbyuonnbaggnavrfbbomyw` int DEFAULT NULL,\n `irufzwtrwcupkvjmhgcqohgsketmpwzg` int DEFAULT NULL,\n `lficsqibbtiabumkvebeekdyqjlqqgjv` int DEFAULT NULL,\n `zlwgcaxdyxhlyrfpvjitemjoulgwmyhe` int DEFAULT NULL,\n `mfxnekivqnehwndiekinyorngyjhnviw` int DEFAULT NULL,\n `zaqprwazwpgeonchqdffmxcqrqxxhtmv` int DEFAULT NULL,\n `sjkbcocemkeqxunhdewiscojwlfrungd` int DEFAULT NULL,\n `egtvzeuwzohqfycrwejnoihwsjyzluik` int DEFAULT NULL,\n `ruwniizxbughsjynofjctgpzgaczfiyw` int DEFAULT NULL,\n `rvefjtaqompgfhpgwjqwymxxeivezpxj` int DEFAULT NULL,\n `mzigkxryumikuytsfweirwibqlfwztbs` int DEFAULT NULL,\n `ktdilacuqqfllmsrdyptcgzjouywtbob` int DEFAULT NULL,\n `eavxfikyonhydsyuhkgmrvhphkwwuifj` int DEFAULT NULL,\n `glnjbxksmgjvdmmucvcwolpkzpmyckzt` int DEFAULT NULL,\n `lbzlpcalnczkxswnaxuippmrcfvpvomq` int DEFAULT NULL,\n `uyychhefbygwcrlbvabacaptczlhzblc` int DEFAULT NULL,\n `hnhiiinfkcjmtrzusxdhitogugpjwegp` int DEFAULT NULL,\n `moejislfjlettxlsvpiyoccurfvmvnoy` int DEFAULT NULL,\n `sfuhmpprkdgwltrctrqrgyxkvxlfwldj` int DEFAULT NULL,\n `pnnyctufyijtzkaarzypmdmgkruhwlkp` int DEFAULT NULL,\n `tyevqxwruqlrbyazpjsoggbzgbthzvmf` int DEFAULT NULL,\n `buzberjpshtubzyexsnkzowfcxznecof` int DEFAULT NULL,\n `ksqytmoafxmkeagmmlxagwyrqwwrpvqd` int DEFAULT NULL,\n `vjfoizfloksyaznjkztacwiecunseyak` int DEFAULT NULL,\n `tkmxpmfcvadiluwdssmzfzmoyqaunufp` int DEFAULT NULL,\n `pxpqvbkbjhmgxnybqrfgyehwspuqkrru` int DEFAULT NULL,\n `xnlmmheuvexymnybxhghtezioopcybyi` int DEFAULT NULL,\n `foywvxbtvnhqdmchiwfuylduyxrmzzkq` int DEFAULT NULL,\n `eexbmeqypygdtmqjhgrnaezelztkyxoz` int DEFAULT NULL,\n `qujwiehzjuxjzcvthndyvwlffyufboky` int DEFAULT NULL,\n `xldvzvcgptnifzhoeicizrfufvjjkjmi` int DEFAULT NULL,\n `ypphvhmhwgimapqfvwvtukmpkwmetpwe` int DEFAULT NULL,\n `nsqxwvvbtkatchkevwbirfdjraqnmppg` int DEFAULT NULL,\n `fvwdaevfrpwxpdrvmbaqkfmeofbzdhck` int DEFAULT NULL,\n PRIMARY KEY (`uiwtfchkgeetmsrbvopdiasjiyggyrms`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"wqdandhuzkcuajvglffrvuwqfxsfunjn\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["uiwtfchkgeetmsrbvopdiasjiyggyrms"],"columns":[{"name":"uiwtfchkgeetmsrbvopdiasjiyggyrms","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"lxvkxukqoafsppuqggaifzjilendovgw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjtrgjmcuwatdhijachimdhafamikjff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjarvommzwtrbplgyfgcfzinllirynit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxsrdizuojljscijkebyusuqiavytdaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikagnmjcdmyvoffngolwzbzdjudquatj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tethaxeqcilfkfrwqifqtdgitkbyxaxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avkndaalczbqmsnuwmecdqdigeuhmksn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owshymzrezmjoaiorajbcfvjcmbubrrv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"puqpikutypcgumusmfqwqbfgiomijpsy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jazpefwvngrmvtawsaplckicdoeeafen","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"meqkmxgfiirpyjuoslwexrercwqorlcr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmihlmsvcwabvkfxqndskzwrrtruwghj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnmnhuwqqoqyevekzluyguhkrghifqqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axcevbjlaivoenkavegxtcnrxumthocn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xquephrqxwtfdvblxclitttahiqgsuxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wljchrbgdajnqrcotbqemuhhwctpvzyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibdybdkdrifkpktmrcpbfjdmvmsepibz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gwgvjyqxhzhbnxlgrufnstwihpwkuqrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wftofimyxdbazyitdjutvumoqaxojbsv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tgvmudcmjdylggkpfsextnnqqyftksui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"isakhcevzsyrswecgzvynirwyqwmwpak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ilgtkmodphvmphdkigecntyyjcofabvv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jfrfwuhkorabqsdppsczpposbjehisbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bffsftgzghfllvczttfosnuzjuukckrt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxzrdnmjrdtlmcuyjggybyhzwplcdkcj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnvjnxxieooyrzfnxhiyqobqzbzgklfp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxterqtmzkswvvdfgazcoozllcidsbbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzjeoahgzqdzcosjzgojnadmwmgpbbgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtetwpttviinvmizmzekznwuzcffbjfl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqsnvokmwamnxsmhetmlfudzbphnzkaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwyrugjxhswzydmqbsxqvxpbfsipdcug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkdejxvdsvgwxqfwqanfzkzleuqlyxwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfkmftkmxwafxqlibzxmikvnrmzmswqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqcmuktevnokntxqbzazvcgpskhbzbtc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fdxopngpllvhsmepwftkcewgbidrglje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrtwijbasuiapdentieezfdjxylhmypp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dnuyyuwvctoqelvxlkopkvlnzftdhygc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sxwhdqhxqsgbqesazaeeoilkmviwspay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjsmoqkrlkzsxfihuirdcyccrvwxywvt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"izlimagrqtnsuwfqnqckiwkelhtqpxzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwvbsaggcgzhzbblhvjgivtwsrplfryg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jyxcuqlxwadjglppblnexcorpicwrueo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zssfefkykfbtzzgmoaovfaefmxvdugpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"titjxacwkqxnlrirvohrnmwhvmoyucji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqcokuegdfxlswfeeuplktpmjnhhhkrs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wotunpxhpibkjvavhnlxsbfuvsuwugao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wkojdknhircppykajjswsbrmwfetreyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kiojvzcfuayncjkpygquqjmugmuflfvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cpxpwbgymdzpbizjazdapwjrwghehnvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jcjmyptdlpnesmbksqjylcxsrajdsjrf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"higmpfvjxyvgxexeuxiuhhmdncdynvoh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugehobueizzeulewqpmwnrrkcshzpdkk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elknfgflyqlodwtdpggekowkpwvfsohe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwnccxdyyiruxqlnggkslbmmbulfadps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sxidtkgwrpmgpzocfpzjmxuyxfjkblvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"stznkdmhzufwwzrcsskzeyxbqegijvwz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzcowebcnavonjzvxblrjskwzbefezvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nlowhbwbyvcyltdyubvnmxzwrpdmsrog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jrpnzezfppuskxkgrhffbsvrbwixzgbd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbdeicvymlypqohkmmixlqdmxwkriefw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mogkgpdrwnnellunqgbcnlohjdgdeozs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kgncahnymghuainmjfwxukseiwrddrma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dmdrimpubeuhozcywcadmugpifmpkwne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmihinimicgbyqytbfceurwjltxsxybg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iefntsnqjdpzvxqfiwhuanwhnsxjsztg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fseklqxhtlrbyuonnbaggnavrfbbomyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"irufzwtrwcupkvjmhgcqohgsketmpwzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lficsqibbtiabumkvebeekdyqjlqqgjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zlwgcaxdyxhlyrfpvjitemjoulgwmyhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfxnekivqnehwndiekinyorngyjhnviw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zaqprwazwpgeonchqdffmxcqrqxxhtmv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sjkbcocemkeqxunhdewiscojwlfrungd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"egtvzeuwzohqfycrwejnoihwsjyzluik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ruwniizxbughsjynofjctgpzgaczfiyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rvefjtaqompgfhpgwjqwymxxeivezpxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzigkxryumikuytsfweirwibqlfwztbs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ktdilacuqqfllmsrdyptcgzjouywtbob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eavxfikyonhydsyuhkgmrvhphkwwuifj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"glnjbxksmgjvdmmucvcwolpkzpmyckzt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lbzlpcalnczkxswnaxuippmrcfvpvomq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uyychhefbygwcrlbvabacaptczlhzblc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hnhiiinfkcjmtrzusxdhitogugpjwegp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"moejislfjlettxlsvpiyoccurfvmvnoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfuhmpprkdgwltrctrqrgyxkvxlfwldj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnnyctufyijtzkaarzypmdmgkruhwlkp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tyevqxwruqlrbyazpjsoggbzgbthzvmf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"buzberjpshtubzyexsnkzowfcxznecof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ksqytmoafxmkeagmmlxagwyrqwwrpvqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjfoizfloksyaznjkztacwiecunseyak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkmxpmfcvadiluwdssmzfzmoyqaunufp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxpqvbkbjhmgxnybqrfgyehwspuqkrru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnlmmheuvexymnybxhghtezioopcybyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"foywvxbtvnhqdmchiwfuylduyxrmzzkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eexbmeqypygdtmqjhgrnaezelztkyxoz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qujwiehzjuxjzcvthndyvwlffyufboky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xldvzvcgptnifzhoeicizrfufvjjkjmi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypphvhmhwgimapqfvwvtukmpkwmetpwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nsqxwvvbtkatchkevwbirfdjraqnmppg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fvwdaevfrpwxpdrvmbaqkfmeofbzdhck","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672622,"databaseName":"models_schema","ddl":"CREATE TABLE `wrlrqixavhpautdnbanimexxwqddrvmn` (\n `nowyjlvamdozmheealefybkjmbqgvofn` int NOT NULL,\n `uvomucuqrkxangdhtlmesotebiinacjv` int DEFAULT NULL,\n `qijvdzakeoetyedvkjpexzrkniofphrt` int DEFAULT NULL,\n `gjvkhqrxwozbvwbtasachvdjdlzmfltd` int DEFAULT NULL,\n `ffjnjbtnboeoefdgmrvmofglrmwmuhtc` int DEFAULT NULL,\n `zbktcdsgrwdiyigunuuylcfowxiipwob` int DEFAULT NULL,\n `kmnkowbxyuiqbdrhialkbvcacagqpkof` int DEFAULT NULL,\n `qzxabghgcojztmalgazhyygfdhuriwys` int DEFAULT NULL,\n `pxskwplniouiduxtzhliogjpkaruuhzh` int DEFAULT NULL,\n `jpkfzdxbcrhzlrtmdyndqzrltyiblsbr` int DEFAULT NULL,\n `rxhnhkekvvcqeuxjhylrosbqspabdsoc` int DEFAULT NULL,\n `qmseuslerlakzxdixfxnstmxobjcsorl` int DEFAULT NULL,\n `erccupyvpbpmrtxentqyqebxpcrnlivs` int DEFAULT NULL,\n `pmeftyykrsjpnnupsbmtexizpyqwdtri` int DEFAULT NULL,\n `omtkvefagwfubelvfahsgfdcxftuzrwu` int DEFAULT NULL,\n `nzixdndqnftnmoxgpvxiyacsufrdocpz` int DEFAULT NULL,\n `flkltzexewmmhykiddoybpvzfsjkyppf` int DEFAULT NULL,\n `jjgytfcrrokdgrjodxxwwdibdvzfvexb` int DEFAULT NULL,\n `yvrwjnxasebkqkxowggyfcjysyldwwnd` int DEFAULT NULL,\n `ujitbtvzyxcebftgbwsgrjvyghpgeygu` int DEFAULT NULL,\n `zxydtmrmqftktyvgctdcfsphuhpgoyxt` int DEFAULT NULL,\n `gzfvuuxyngavnwuucngcfbvsvvzhgbne` int DEFAULT NULL,\n `wzcjszybydtkwlgdolaclrctjuyzqulm` int DEFAULT NULL,\n `jjzeeoiwabobnmeiinwzpmlgycgjayud` int DEFAULT NULL,\n `pceuzpppdzbapgdawpyicqavprrfhxzl` int DEFAULT NULL,\n `flobkojafgjicflueqqvvvukslvgdcrb` int DEFAULT NULL,\n `txiertrunqjyretbjakbacoanywybvyk` int DEFAULT NULL,\n `alnidutkznjwfgahzetwjcvyznntnsgh` int DEFAULT NULL,\n `bunonfczdcjjmphagqjprlxirauhcpgg` int DEFAULT NULL,\n `uotgttpwozknvjbayomlsgckpoahvjfr` int DEFAULT NULL,\n `euncftoygntxznmhwvkmbqdvrxhsizux` int DEFAULT NULL,\n `yampbowqekvbllkryyyieeuldawzvjft` int DEFAULT NULL,\n `huhdbdsettwzkqmmuczyiwucmzcrmhwd` int DEFAULT NULL,\n `kjvlapcbljxjffopjbttufepmupnlost` int DEFAULT NULL,\n `mhzrmyhzikpijyzhsskztegdhtxtwdpf` int DEFAULT NULL,\n `joeuakdzrvblcnfcxjasdlofwysaohlf` int DEFAULT NULL,\n `xqryfjrhcwfsztnelcyenzsekedagfyi` int DEFAULT NULL,\n `ueugfkwckulhfhgcyitsbnrnopozudih` int DEFAULT NULL,\n `txaxuyquxhjgoxilrdubkkgnaqpkwpxe` int DEFAULT NULL,\n `rxblvlycpziqbakwudhbsrekjqwzkvsx` int DEFAULT NULL,\n `cbedqldibhpebqscfatloirsenuxsznp` int DEFAULT NULL,\n `xvfetioqzysavasthwrwdpwwtdqnnhpx` int DEFAULT NULL,\n `umbheqwdtmyrvwfmqvczirgtrxonqdbl` int DEFAULT NULL,\n `vfscydnaerglmluvddmidmxaiydmbict` int DEFAULT NULL,\n `rcxtxbeqhvmvvvdcgpsusrkzezigilyk` int DEFAULT NULL,\n `zdgchksklrxgrcnlyuvnixbzogqhojum` int DEFAULT NULL,\n `awsruvkvnukwnwbzytcnyqnooojgeayy` int DEFAULT NULL,\n `ssqwgvvqsucvrhcspsuaaphbisyrbqok` int DEFAULT NULL,\n `ujpsjfqcmypqghnaniwzgpsniyibfnfp` int DEFAULT NULL,\n `dpnbnsunflgnmnsqjbxncasrogyeykpu` int DEFAULT NULL,\n `qzereihgqhiyjirsfvyadgejsinhpurs` int DEFAULT NULL,\n `labuvmotxrugwqtomlsrsqbabevzsdgd` int DEFAULT NULL,\n `nnnknkdojyoshtagsnqxcaotjmlosqpo` int DEFAULT NULL,\n `auzgnlquisdpttmguentpkxortxsxhti` int DEFAULT NULL,\n `tvthfwlkpoeryykeatpqxwoftkxibnqq` int DEFAULT NULL,\n `abamtytacsfdqpmzisljavhqhluwtijx` int DEFAULT NULL,\n `sgevkpnywlofaiiauryfwypvecedahcj` int DEFAULT NULL,\n `geyhdoxtnxiemyyaxjzmbpijwqaljkuw` int DEFAULT NULL,\n `lzwobnjvizveunmzdxlxnjsmicjkndqh` int DEFAULT NULL,\n `fpgcwnfgognlfixjijeoynipdxexyfis` int DEFAULT NULL,\n `wscfbmpadlloxhkyfxlervimnouqufcs` int DEFAULT NULL,\n `mqnywaeajqoivqivfhoaysgbeecutgvb` int DEFAULT NULL,\n `wxyxftbzriakzwegpwmwbbqpwuabjyuw` int DEFAULT NULL,\n `bntupxipowaymcjofkwrfwfpgsfjaysw` int DEFAULT NULL,\n `bvlcusruqpqxvphjtgjbtyqtdbxpzfqb` int DEFAULT NULL,\n `vcuwaycyjoxquawmhnbddwgrivqzglzw` int DEFAULT NULL,\n `xpaiychgzuiqmpmqgbafamwcyfbuovlg` int DEFAULT NULL,\n `imwigtmztulqocaoamdgafdlaoxiokbt` int DEFAULT NULL,\n `labaecxmpnusyvozczrsnipkpwuzhufw` int DEFAULT NULL,\n `uloxxgxorfmzvvqderhjrrzwrtucodkt` int DEFAULT NULL,\n `tnkacdzjrpqxldemluhfwfafaushbnvv` int DEFAULT NULL,\n `ejxjqafkucghdsicivabujnjvcrlabfl` int DEFAULT NULL,\n `omxaqffgjxujcrfbnzbjvkuyjphkyjkb` int DEFAULT NULL,\n `pbjdchiadxcobzxdaarqozvcdyohfsop` int DEFAULT NULL,\n `blnybegndpktfzmutjhpjizebzpebhvw` int DEFAULT NULL,\n `fphaslgypqqftcvwyxtwerszvdpklqgy` int DEFAULT NULL,\n `zpgtilnfagywayacygrhbbswdbdyqyzq` int DEFAULT NULL,\n `nnighiuxiukktriiuwnjciykkvgkxoiz` int DEFAULT NULL,\n `fteapvctynkkcwgxzvbyvpfdcbxuvmta` int DEFAULT NULL,\n `mzbxbzjwcfflnulunmylcxeirfwxlnwm` int DEFAULT NULL,\n `sujghakhgsircedtavauxqmxvdbprbbo` int DEFAULT NULL,\n `msupyqxcinsmssnpkltzgzkskfazwqil` int DEFAULT NULL,\n `ifrsjkedjjiwmrpsvmnkoaagfavuhkwz` int DEFAULT NULL,\n `hjugcixbxxnkpaqbjyabuegepbisipjs` int DEFAULT NULL,\n `whqvkzkirjtyevzvjnvzokfjbwoklxxh` int DEFAULT NULL,\n `vtihhfqykzhkggipjawscchblxllwerz` int DEFAULT NULL,\n `uspozkasrgtkmntqxbtxxecjxbsgwrng` int DEFAULT NULL,\n `zkkcujjpwmxnqxxhvdyshbuuwtikhfsk` int DEFAULT NULL,\n `wpaltjzkeuccciltzbawhcsaivvrgded` int DEFAULT NULL,\n `ysimtocvctgjmxhxorroiujyurwwcgrb` int DEFAULT NULL,\n `bxrerorgdbwooceknmkxylcmhdsradwk` int DEFAULT NULL,\n `swxgfyjvcbvzdnocpcvnwydsrecxtibn` int DEFAULT NULL,\n `uciprevzjxasvpmxycagexdbmtpxigub` int DEFAULT NULL,\n `sskmhrfbslzqxujfnljilzdstflqzcst` int DEFAULT NULL,\n `cachoygneuhnppyupprzjksvokkbawuj` int DEFAULT NULL,\n `gfaxnbfjqitsjyegjgbeifladtpbgazc` int DEFAULT NULL,\n `edgywhzuoghbbxexafewxslwipzohaav` int DEFAULT NULL,\n `yvfgnnaoptlndmihlrmygbghxjqlewdq` int DEFAULT NULL,\n `fufhvidfctpgznxwtkzbjdumejmgmmgo` int DEFAULT NULL,\n `ohjzjvvhiqmabmicejnzfjqhaoaknxtn` int DEFAULT NULL,\n PRIMARY KEY (`nowyjlvamdozmheealefybkjmbqgvofn`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"wrlrqixavhpautdnbanimexxwqddrvmn\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["nowyjlvamdozmheealefybkjmbqgvofn"],"columns":[{"name":"nowyjlvamdozmheealefybkjmbqgvofn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"uvomucuqrkxangdhtlmesotebiinacjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qijvdzakeoetyedvkjpexzrkniofphrt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjvkhqrxwozbvwbtasachvdjdlzmfltd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffjnjbtnboeoefdgmrvmofglrmwmuhtc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbktcdsgrwdiyigunuuylcfowxiipwob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kmnkowbxyuiqbdrhialkbvcacagqpkof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzxabghgcojztmalgazhyygfdhuriwys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxskwplniouiduxtzhliogjpkaruuhzh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jpkfzdxbcrhzlrtmdyndqzrltyiblsbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxhnhkekvvcqeuxjhylrosbqspabdsoc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmseuslerlakzxdixfxnstmxobjcsorl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"erccupyvpbpmrtxentqyqebxpcrnlivs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmeftyykrsjpnnupsbmtexizpyqwdtri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omtkvefagwfubelvfahsgfdcxftuzrwu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzixdndqnftnmoxgpvxiyacsufrdocpz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flkltzexewmmhykiddoybpvzfsjkyppf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjgytfcrrokdgrjodxxwwdibdvzfvexb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yvrwjnxasebkqkxowggyfcjysyldwwnd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujitbtvzyxcebftgbwsgrjvyghpgeygu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxydtmrmqftktyvgctdcfsphuhpgoyxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzfvuuxyngavnwuucngcfbvsvvzhgbne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzcjszybydtkwlgdolaclrctjuyzqulm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjzeeoiwabobnmeiinwzpmlgycgjayud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pceuzpppdzbapgdawpyicqavprrfhxzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flobkojafgjicflueqqvvvukslvgdcrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txiertrunqjyretbjakbacoanywybvyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"alnidutkznjwfgahzetwjcvyznntnsgh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bunonfczdcjjmphagqjprlxirauhcpgg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uotgttpwozknvjbayomlsgckpoahvjfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"euncftoygntxznmhwvkmbqdvrxhsizux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yampbowqekvbllkryyyieeuldawzvjft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"huhdbdsettwzkqmmuczyiwucmzcrmhwd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjvlapcbljxjffopjbttufepmupnlost","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhzrmyhzikpijyzhsskztegdhtxtwdpf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"joeuakdzrvblcnfcxjasdlofwysaohlf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqryfjrhcwfsztnelcyenzsekedagfyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ueugfkwckulhfhgcyitsbnrnopozudih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txaxuyquxhjgoxilrdubkkgnaqpkwpxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxblvlycpziqbakwudhbsrekjqwzkvsx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbedqldibhpebqscfatloirsenuxsznp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvfetioqzysavasthwrwdpwwtdqnnhpx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umbheqwdtmyrvwfmqvczirgtrxonqdbl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vfscydnaerglmluvddmidmxaiydmbict","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcxtxbeqhvmvvvdcgpsusrkzezigilyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdgchksklrxgrcnlyuvnixbzogqhojum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awsruvkvnukwnwbzytcnyqnooojgeayy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ssqwgvvqsucvrhcspsuaaphbisyrbqok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujpsjfqcmypqghnaniwzgpsniyibfnfp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpnbnsunflgnmnsqjbxncasrogyeykpu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzereihgqhiyjirsfvyadgejsinhpurs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"labuvmotxrugwqtomlsrsqbabevzsdgd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nnnknkdojyoshtagsnqxcaotjmlosqpo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"auzgnlquisdpttmguentpkxortxsxhti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvthfwlkpoeryykeatpqxwoftkxibnqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abamtytacsfdqpmzisljavhqhluwtijx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sgevkpnywlofaiiauryfwypvecedahcj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"geyhdoxtnxiemyyaxjzmbpijwqaljkuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzwobnjvizveunmzdxlxnjsmicjkndqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpgcwnfgognlfixjijeoynipdxexyfis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wscfbmpadlloxhkyfxlervimnouqufcs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqnywaeajqoivqivfhoaysgbeecutgvb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxyxftbzriakzwegpwmwbbqpwuabjyuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bntupxipowaymcjofkwrfwfpgsfjaysw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvlcusruqpqxvphjtgjbtyqtdbxpzfqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vcuwaycyjoxquawmhnbddwgrivqzglzw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpaiychgzuiqmpmqgbafamwcyfbuovlg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"imwigtmztulqocaoamdgafdlaoxiokbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"labaecxmpnusyvozczrsnipkpwuzhufw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uloxxgxorfmzvvqderhjrrzwrtucodkt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnkacdzjrpqxldemluhfwfafaushbnvv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejxjqafkucghdsicivabujnjvcrlabfl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omxaqffgjxujcrfbnzbjvkuyjphkyjkb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbjdchiadxcobzxdaarqozvcdyohfsop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blnybegndpktfzmutjhpjizebzpebhvw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fphaslgypqqftcvwyxtwerszvdpklqgy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpgtilnfagywayacygrhbbswdbdyqyzq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nnighiuxiukktriiuwnjciykkvgkxoiz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fteapvctynkkcwgxzvbyvpfdcbxuvmta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzbxbzjwcfflnulunmylcxeirfwxlnwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sujghakhgsircedtavauxqmxvdbprbbo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"msupyqxcinsmssnpkltzgzkskfazwqil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifrsjkedjjiwmrpsvmnkoaagfavuhkwz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjugcixbxxnkpaqbjyabuegepbisipjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"whqvkzkirjtyevzvjnvzokfjbwoklxxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtihhfqykzhkggipjawscchblxllwerz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uspozkasrgtkmntqxbtxxecjxbsgwrng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zkkcujjpwmxnqxxhvdyshbuuwtikhfsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wpaltjzkeuccciltzbawhcsaivvrgded","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ysimtocvctgjmxhxorroiujyurwwcgrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bxrerorgdbwooceknmkxylcmhdsradwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swxgfyjvcbvzdnocpcvnwydsrecxtibn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uciprevzjxasvpmxycagexdbmtpxigub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sskmhrfbslzqxujfnljilzdstflqzcst","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cachoygneuhnppyupprzjksvokkbawuj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfaxnbfjqitsjyegjgbeifladtpbgazc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edgywhzuoghbbxexafewxslwipzohaav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yvfgnnaoptlndmihlrmygbghxjqlewdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fufhvidfctpgznxwtkzbjdumejmgmmgo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohjzjvvhiqmabmicejnzfjqhaoaknxtn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672653,"databaseName":"models_schema","ddl":"CREATE TABLE `wzjzwsynaunsloezfpugchitwewtoxto` (\n `mbwyvszuhkhnzdyesplqbrbmaxpcxoml` int NOT NULL,\n `xdtipslwgoxjzapvtmtkwwnowjxxxjtd` int DEFAULT NULL,\n `cwxleldnhqrlseamnzfkcffiayblcaey` int DEFAULT NULL,\n `vgalgkcjxywzfjwephdixwlntjdhfpnd` int DEFAULT NULL,\n `crpuzdedbbxvhqlooakkvitjmmhymhaj` int DEFAULT NULL,\n `lvueblzikqzvwhkrttfjqqboyotanxln` int DEFAULT NULL,\n `ximfqbjzobvogzxfeocidtbweazxcxct` int DEFAULT NULL,\n `quzxnxpbglmdvrarjxsytolpcecrzvjl` int DEFAULT NULL,\n `hnasitepxgpoocanwjniusrgynuglylf` int DEFAULT NULL,\n `rqneyydtjpnyynznxdtfwsfpqgfgiglz` int DEFAULT NULL,\n `qdytmiytkgxmgoagyztupsslzhdgermj` int DEFAULT NULL,\n `gplhkyvslkpzghaqudmgyutuuodhtien` int DEFAULT NULL,\n `qomwpkuovdfmfrrfbvkwwgdfyuwyvusi` int DEFAULT NULL,\n `boemfejdcfwskxmtytlgxmnfyzmmeyqf` int DEFAULT NULL,\n `xbestcqeabtqqstcmbdmoouriuoeucuz` int DEFAULT NULL,\n `xgbtymtcokdieuhtbykypizowkmzizae` int DEFAULT NULL,\n `wnlvtpmfabilxekbptpijxbnkdbvppsq` int DEFAULT NULL,\n `wjrgaicxmowygpmjbnegrxeuaisjijmh` int DEFAULT NULL,\n `dvlnidegslbdfqqhcxfsidclhmwbudgb` int DEFAULT NULL,\n `gpiymstctibsdjlpwalsnhkbtmfxmzer` int DEFAULT NULL,\n `xyvlxwrophrsufvrckjiwugjtztxzxhe` int DEFAULT NULL,\n `kolzsjxsnkdqkwgxiesabwahzejgzrle` int DEFAULT NULL,\n `mzygcocphpyqnwbowizldrvhskvzprrc` int DEFAULT NULL,\n `mnwzicnultbgvhnpowklnbayxohowogz` int DEFAULT NULL,\n `vlahxiepabtlvwuesoepohqehfnpwvnm` int DEFAULT NULL,\n `vjxshzfiutpqdyktzuifttzacznoowtw` int DEFAULT NULL,\n `dlhewonurdfjjvgtltngbbkcmogucrwv` int DEFAULT NULL,\n `kzrzahstdvkkohtnmvegousrclzvhldb` int DEFAULT NULL,\n `shcipahwhaueallwpspciirridttqqrf` int DEFAULT NULL,\n `zhvexugopvoxeubxwvayhgraywonpfln` int DEFAULT NULL,\n `bzzgotcwixgdgtskmmyqprfgqeeywabs` int DEFAULT NULL,\n `nrfyyhggqjctqavjtkldktbnwrcrbywv` int DEFAULT NULL,\n `phlwnmrerivjnkzxpgqopfprpxsbujzi` int DEFAULT NULL,\n `fqmymhlmexcgukfpqgonfoprelzhgkjv` int DEFAULT NULL,\n `rjeqqxwlsbkpyeolatfmnpeomfqkggam` int DEFAULT NULL,\n `axwtddxeshvwrgqovytfsnklzyygxyzg` int DEFAULT NULL,\n `bbymbvdfizgrtjwvggturyyzvppeivos` int DEFAULT NULL,\n `dqyelfiuarbqbdmpjxysoixjylrpdazf` int DEFAULT NULL,\n `yryibazuzvnxkhbamwncvyfjuoraibiu` int DEFAULT NULL,\n `xnspdozkvnkduebmkhoqiqptrjmhzcvx` int DEFAULT NULL,\n `ukcnxcmtqurijtekdqayedssehxrkehv` int DEFAULT NULL,\n `rrgzsssbmddrinkxkqccmyhflbmhpoiz` int DEFAULT NULL,\n `pdbsokngfzceshvvsokcvcvokfimqsna` int DEFAULT NULL,\n `wgrhwvlbpxyczhbmtkfaibvnnkeivgii` int DEFAULT NULL,\n `kvgplryqwdkzwjwwiilfaabdivfpmakt` int DEFAULT NULL,\n `otlneplehijqmovgorayumkirmngvbis` int DEFAULT NULL,\n `zssrvnzhaqdqejsscgeitsidttlmyeuu` int DEFAULT NULL,\n `zzyezhuontzrrpzmxmjbfbsesbjbgzox` int DEFAULT NULL,\n `aaezehgtdixlrqwowrxhhiwuhijuxyor` int DEFAULT NULL,\n `saittkzeqcozqzhhvpejuorrgibbhlqj` int DEFAULT NULL,\n `bgtagtsqnzjjttbbqyfpyafkxloctvoi` int DEFAULT NULL,\n `cnvbpcqabwxuhjmwdyciykcmllzkgrsm` int DEFAULT NULL,\n `ljydvzhkudajlyentpjpboiszwdmwntt` int DEFAULT NULL,\n `hkfhxvmmtaxqctcmkwvrshinlrnuzcem` int DEFAULT NULL,\n `yythkgnqnxoirfejkqfrzlurietdvlip` int DEFAULT NULL,\n `smvktqwjwvkircetrasfzdnjttujpyom` int DEFAULT NULL,\n `qevnzfglvqdzrznjhaujwqqkvajgzaxl` int DEFAULT NULL,\n `fzuuxktcbarmhventomhvpsvvbocwhge` int DEFAULT NULL,\n `fpcxhnoeyndvdtxmiuqaguwntexlmxlj` int DEFAULT NULL,\n `aqqpxyrgydxvqauwmkopdlxeuqdiuxzk` int DEFAULT NULL,\n `yusnjohenfvpwndqxwbxsiusmjaxfpms` int DEFAULT NULL,\n `ypsgzuwrsfnjsurwsrqgmmlfwuuafaqu` int DEFAULT NULL,\n `upgcfqoeqtdnqulwzjmfsxqvbtsdrmqd` int DEFAULT NULL,\n `uijpgjvqrjdzduvapgjmongsfbfxtcbg` int DEFAULT NULL,\n `almxjxybdiwajxmsrwztplvqhycwiruk` int DEFAULT NULL,\n `bswmeeplkqqkfrcxrtteuvswmejoslyz` int DEFAULT NULL,\n `slpgevhdpvrccthpljwptcrhkihrwwlg` int DEFAULT NULL,\n `gtyaldqrigrbhqgovvqdgnyieawhqpsj` int DEFAULT NULL,\n `rbwrqvsngdlacivqyxyxwochwmnydaen` int DEFAULT NULL,\n `psvxjaodtuawydngwyvrgnixhbdcidki` int DEFAULT NULL,\n `xyekpfpijbexaqxjutafkutrajibhwfd` int DEFAULT NULL,\n `mfxgbhzssfojkuzzdtpgslpyxjqgiipb` int DEFAULT NULL,\n `qjrqoqxpymqrrgfinhbjsoyhrtnyfvhd` int DEFAULT NULL,\n `apwurvcaawfmuvdonhsousuhetzszpom` int DEFAULT NULL,\n `atgeffikkbrhscndgvetszkwsxrwxuff` int DEFAULT NULL,\n `pmketeuqmtdhbcoyewjanqcepfvlnltw` int DEFAULT NULL,\n `cpfvskskandvtqhlezvbsgnhguoainsq` int DEFAULT NULL,\n `ictnenuptlzwlpwdmrhfmphrqaaeosey` int DEFAULT NULL,\n `ceyaaiucjlecbvefdbzkowgedhqdmcqk` int DEFAULT NULL,\n `sgpahztpalgraoauhyqzfpqhkopxmazo` int DEFAULT NULL,\n `mvmouglfsngifxwsesyaphhelgjhugqw` int DEFAULT NULL,\n `rsgutchfnxnmxceoefbhjqcbhunnlnyy` int DEFAULT NULL,\n `kbdfxuynsflwqsqwpahlykiilapfawwn` int DEFAULT NULL,\n `jgzmxwudzrmscljkhjmkwwlrfrwzmnst` int DEFAULT NULL,\n `tefkdkaasvyaknsoscvkbgqetosfvzou` int DEFAULT NULL,\n `yymkrqjehuphfatvbqezpqpasghneleq` int DEFAULT NULL,\n `yatjsljqhhtokylndiprdfrpfaroznot` int DEFAULT NULL,\n `cbnyvzucakzaqxowdrzxzcqmpmipxznq` int DEFAULT NULL,\n `mwkppiffggxrpisddkefxezseppndgfh` int DEFAULT NULL,\n `wdswalaorrhvfcqmaykrtrgsmyusckxx` int DEFAULT NULL,\n `dtygnaguenqaatqwnyjpsqmyvmagtfde` int DEFAULT NULL,\n `xsjvfnizhytibupdvjyyxpnxnevfqwoz` int DEFAULT NULL,\n `fyugjnrzrmewkpmgniyxslpjfnsvwnfl` int DEFAULT NULL,\n `fikgdmkvmzfbcitrtinhacvxvwoeefup` int DEFAULT NULL,\n `qwxlkjdmhqkihupaktitbzrqznmcdolb` int DEFAULT NULL,\n `cxzbvhshinxhsnxkqtfbdwplwuhaplvf` int DEFAULT NULL,\n `skdulmzelyriwpubutwzktpmoivswpbr` int DEFAULT NULL,\n `uzivhetwtjvsgprtbvmzrahavssgkigc` int DEFAULT NULL,\n `ekkzkuxolaoyqwbkunjcbtojxfhswejt` int DEFAULT NULL,\n `bjsqtcjeydtyeohymwbkigcimausrwpw` int DEFAULT NULL,\n PRIMARY KEY (`mbwyvszuhkhnzdyesplqbrbmaxpcxoml`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"wzjzwsynaunsloezfpugchitwewtoxto\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["mbwyvszuhkhnzdyesplqbrbmaxpcxoml"],"columns":[{"name":"mbwyvszuhkhnzdyesplqbrbmaxpcxoml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"xdtipslwgoxjzapvtmtkwwnowjxxxjtd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwxleldnhqrlseamnzfkcffiayblcaey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgalgkcjxywzfjwephdixwlntjdhfpnd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crpuzdedbbxvhqlooakkvitjmmhymhaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvueblzikqzvwhkrttfjqqboyotanxln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ximfqbjzobvogzxfeocidtbweazxcxct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"quzxnxpbglmdvrarjxsytolpcecrzvjl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hnasitepxgpoocanwjniusrgynuglylf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqneyydtjpnyynznxdtfwsfpqgfgiglz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdytmiytkgxmgoagyztupsslzhdgermj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gplhkyvslkpzghaqudmgyutuuodhtien","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qomwpkuovdfmfrrfbvkwwgdfyuwyvusi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"boemfejdcfwskxmtytlgxmnfyzmmeyqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbestcqeabtqqstcmbdmoouriuoeucuz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xgbtymtcokdieuhtbykypizowkmzizae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnlvtpmfabilxekbptpijxbnkdbvppsq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjrgaicxmowygpmjbnegrxeuaisjijmh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvlnidegslbdfqqhcxfsidclhmwbudgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gpiymstctibsdjlpwalsnhkbtmfxmzer","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyvlxwrophrsufvrckjiwugjtztxzxhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kolzsjxsnkdqkwgxiesabwahzejgzrle","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzygcocphpyqnwbowizldrvhskvzprrc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnwzicnultbgvhnpowklnbayxohowogz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vlahxiepabtlvwuesoepohqehfnpwvnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjxshzfiutpqdyktzuifttzacznoowtw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlhewonurdfjjvgtltngbbkcmogucrwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzrzahstdvkkohtnmvegousrclzvhldb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shcipahwhaueallwpspciirridttqqrf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zhvexugopvoxeubxwvayhgraywonpfln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzzgotcwixgdgtskmmyqprfgqeeywabs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nrfyyhggqjctqavjtkldktbnwrcrbywv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phlwnmrerivjnkzxpgqopfprpxsbujzi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqmymhlmexcgukfpqgonfoprelzhgkjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjeqqxwlsbkpyeolatfmnpeomfqkggam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axwtddxeshvwrgqovytfsnklzyygxyzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbymbvdfizgrtjwvggturyyzvppeivos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqyelfiuarbqbdmpjxysoixjylrpdazf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yryibazuzvnxkhbamwncvyfjuoraibiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnspdozkvnkduebmkhoqiqptrjmhzcvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ukcnxcmtqurijtekdqayedssehxrkehv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrgzsssbmddrinkxkqccmyhflbmhpoiz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdbsokngfzceshvvsokcvcvokfimqsna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgrhwvlbpxyczhbmtkfaibvnnkeivgii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvgplryqwdkzwjwwiilfaabdivfpmakt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otlneplehijqmovgorayumkirmngvbis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zssrvnzhaqdqejsscgeitsidttlmyeuu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzyezhuontzrrpzmxmjbfbsesbjbgzox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aaezehgtdixlrqwowrxhhiwuhijuxyor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"saittkzeqcozqzhhvpejuorrgibbhlqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bgtagtsqnzjjttbbqyfpyafkxloctvoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnvbpcqabwxuhjmwdyciykcmllzkgrsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljydvzhkudajlyentpjpboiszwdmwntt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkfhxvmmtaxqctcmkwvrshinlrnuzcem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yythkgnqnxoirfejkqfrzlurietdvlip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smvktqwjwvkircetrasfzdnjttujpyom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qevnzfglvqdzrznjhaujwqqkvajgzaxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzuuxktcbarmhventomhvpsvvbocwhge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpcxhnoeyndvdtxmiuqaguwntexlmxlj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqqpxyrgydxvqauwmkopdlxeuqdiuxzk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yusnjohenfvpwndqxwbxsiusmjaxfpms","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypsgzuwrsfnjsurwsrqgmmlfwuuafaqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upgcfqoeqtdnqulwzjmfsxqvbtsdrmqd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uijpgjvqrjdzduvapgjmongsfbfxtcbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"almxjxybdiwajxmsrwztplvqhycwiruk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bswmeeplkqqkfrcxrtteuvswmejoslyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"slpgevhdpvrccthpljwptcrhkihrwwlg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtyaldqrigrbhqgovvqdgnyieawhqpsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbwrqvsngdlacivqyxyxwochwmnydaen","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psvxjaodtuawydngwyvrgnixhbdcidki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyekpfpijbexaqxjutafkutrajibhwfd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfxgbhzssfojkuzzdtpgslpyxjqgiipb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjrqoqxpymqrrgfinhbjsoyhrtnyfvhd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"apwurvcaawfmuvdonhsousuhetzszpom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atgeffikkbrhscndgvetszkwsxrwxuff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmketeuqmtdhbcoyewjanqcepfvlnltw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cpfvskskandvtqhlezvbsgnhguoainsq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ictnenuptlzwlpwdmrhfmphrqaaeosey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ceyaaiucjlecbvefdbzkowgedhqdmcqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sgpahztpalgraoauhyqzfpqhkopxmazo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvmouglfsngifxwsesyaphhelgjhugqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rsgutchfnxnmxceoefbhjqcbhunnlnyy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kbdfxuynsflwqsqwpahlykiilapfawwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgzmxwudzrmscljkhjmkwwlrfrwzmnst","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tefkdkaasvyaknsoscvkbgqetosfvzou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yymkrqjehuphfatvbqezpqpasghneleq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yatjsljqhhtokylndiprdfrpfaroznot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbnyvzucakzaqxowdrzxzcqmpmipxznq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwkppiffggxrpisddkefxezseppndgfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wdswalaorrhvfcqmaykrtrgsmyusckxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtygnaguenqaatqwnyjpsqmyvmagtfde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsjvfnizhytibupdvjyyxpnxnevfqwoz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fyugjnrzrmewkpmgniyxslpjfnsvwnfl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fikgdmkvmzfbcitrtinhacvxvwoeefup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwxlkjdmhqkihupaktitbzrqznmcdolb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cxzbvhshinxhsnxkqtfbdwplwuhaplvf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skdulmzelyriwpubutwzktpmoivswpbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzivhetwtjvsgprtbvmzrahavssgkigc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekkzkuxolaoyqwbkunjcbtojxfhswejt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjsqtcjeydtyeohymwbkigcimausrwpw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672683,"databaseName":"models_schema","ddl":"CREATE TABLE `wzskeeofxzgrcdxuehzyhdhmmlzyxqxo` (\n `cdyaahsfoywvvnifdjqsqndwjkbrlgme` int NOT NULL,\n `cseixoliiylnhxwoaqibefxhakoejhxp` int DEFAULT NULL,\n `vmypacziptmmacthmfmanepvkzxhyawu` int DEFAULT NULL,\n `uohajgvtewwvmfhgpgjoybssyhacoyko` int DEFAULT NULL,\n `nneoxltoyvnqwzlirssrdjppfozgouzy` int DEFAULT NULL,\n `mwvnlmivqsomjhnjbgilicwubszjlakd` int DEFAULT NULL,\n `cvxstehieavrdiazgbyyozcjpskllrcw` int DEFAULT NULL,\n `qdljnvwrcqaropvaitsipjcwunmwiicz` int DEFAULT NULL,\n `dfcqcecqtnplhqhtpsbomonmcnpqeurt` int DEFAULT NULL,\n `noyhpgcuhvwpnunbaqkegypcmatgnkua` int DEFAULT NULL,\n `mzivpbvfixaciwluvmxsjoggdvwbdelc` int DEFAULT NULL,\n `tmqecjrfbenugmankxfwsvpzpgbraeni` int DEFAULT NULL,\n `gwcrryjmzxlzdzlayrugwdqsnufkiwnm` int DEFAULT NULL,\n `oidguynrodrstqynyygkdlferxcheqns` int DEFAULT NULL,\n `padivievkkjpasixykqglefeivaaioyq` int DEFAULT NULL,\n `blhcvbdfpvkajrwydzfzjqenzcrbrnmt` int DEFAULT NULL,\n `wcnfwiwhvinofnhdxhzykqymubgkbbvn` int DEFAULT NULL,\n `blmteijniyzdtxmvpydlhpwveisgvkzb` int DEFAULT NULL,\n `iplgmxoutngugzijlltwsrwdaudatgbb` int DEFAULT NULL,\n `lvuianfheuszfdupjkeplcdwwmgjxgyw` int DEFAULT NULL,\n `qcnjpvmzrkxvuodtrtaffwcdvpiyqwnt` int DEFAULT NULL,\n `qizkgphdlvengelyqtngsuobinhybtlw` int DEFAULT NULL,\n `rpkeawkcuysasfqprzjitdfhqpztfzrt` int DEFAULT NULL,\n `ejxxozjghbuhwkjvhuriwrwhlwwlmcuk` int DEFAULT NULL,\n `bccyenlrpkukedcghzjbhqxmhkdvumnt` int DEFAULT NULL,\n `vkhodioqjcffowpbxflbstxlzlqhcrpr` int DEFAULT NULL,\n `ahrykedtoiihaxlpuwkqmjelkdzebrhl` int DEFAULT NULL,\n `kchfhdsomyzemngwoladimvywzyjsglc` int DEFAULT NULL,\n `bmlyaiitrfdbypdamjnurmiqvvhrluxp` int DEFAULT NULL,\n `hpbdzwzfsurzpqrnfzwoxocvkioqxkud` int DEFAULT NULL,\n `swhspdgrhjdkyygyrazlvjdocifmtjyz` int DEFAULT NULL,\n `bxlsudodjplvhhxqzyqqoxfltoxcmyxc` int DEFAULT NULL,\n `fspmdwyzmlvuanmzxqebucqkdrfnuyub` int DEFAULT NULL,\n `cfipfkfgpernduxrxdmcfyzanlnouwan` int DEFAULT NULL,\n `lwdosrfktqyfdbsizfbtnhxkapowvmvm` int DEFAULT NULL,\n `zoggzudhmzoqrbjhwgxejfyguwxjhbns` int DEFAULT NULL,\n `snkqrqyejwrwyrphgfcqbvfctondyjlb` int DEFAULT NULL,\n `mrhetdummkcqmnseevizikqwpusqlxbh` int DEFAULT NULL,\n `jbtlhbmgjeykfoqmtszinbgsrajuqkaa` int DEFAULT NULL,\n `clmjxtthqpuaqomaeuqfmbpinkxidvtr` int DEFAULT NULL,\n `ojxlcrutvipcnugknhevdjxirctgdxqc` int DEFAULT NULL,\n `lkaffnuwvbpvwpmchrhebxutggokecro` int DEFAULT NULL,\n `wvrrcgclfupwjqjxjesfdxtvddleyqrd` int DEFAULT NULL,\n `vbkkrfpvioekpvxoujajsaakwvmklljz` int DEFAULT NULL,\n `ecxlfyxcotmeffbmzdpshxnmeuccmtaz` int DEFAULT NULL,\n `dkqgidznvqeghqtnuvhkcoqnwwiyvroh` int DEFAULT NULL,\n `xttpfsykumncztilvupbwbhbnzyzteew` int DEFAULT NULL,\n `iqrsdcilohnrgcoqglhfqvlamlnaetcg` int DEFAULT NULL,\n `xromuwzxnlfgscdauvykjaqphazkeoys` int DEFAULT NULL,\n `azjaesfjfzudkemfuwraprhninslkfjo` int DEFAULT NULL,\n `cmxswmzuzdprldhnefeyzanjjnqnmkbr` int DEFAULT NULL,\n `mbhwnhipewoxoqkauhevjmbqeyyoebgg` int DEFAULT NULL,\n `arfvqmeyfcohkelbuoarpqfgkvgcsvmq` int DEFAULT NULL,\n `knlalkpxatljabxbqaohjuigybotvhwt` int DEFAULT NULL,\n `wffexsbkvxggcdpqbzwqgxugakdpvwji` int DEFAULT NULL,\n `ktozzobjzvbdptceowbkodiwjoporplb` int DEFAULT NULL,\n `gbjrfmlzwywpcimqxykphhrckqaezczl` int DEFAULT NULL,\n `rbdbtckiqhqtyporztxhcolzhhmkobvk` int DEFAULT NULL,\n `vqxlofbofwhhshdwmclgeygwcuaaqijq` int DEFAULT NULL,\n `oxeutjrskwfzoddebcoeleifibxirwlz` int DEFAULT NULL,\n `bbyleocgakcxoehddwwzwmgzzhgrnfdl` int DEFAULT NULL,\n `vrentalhkjfimdgydtqzbwhwmanztyns` int DEFAULT NULL,\n `exdodowheqrmrdpnprozzbbuslapviuq` int DEFAULT NULL,\n `irtqtjbmvhqntisdhdoptwbbbubzukez` int DEFAULT NULL,\n `hhwjxmwmpxqgrouyprotiixcczmsueng` int DEFAULT NULL,\n `dhknpkqrpgzdkzegtghhdbwgdmigxfdn` int DEFAULT NULL,\n `idsbcgpttroclsybyvnlqdfajfswhkut` int DEFAULT NULL,\n `mjxhexdssjzvwnksblxmefuanbgtxbwm` int DEFAULT NULL,\n `gneyiyfxptztybceopfqiuzlvtoepkdv` int DEFAULT NULL,\n `jvwcwntxtxratbzuqqbwfmwqaacedqxm` int DEFAULT NULL,\n `jzrjnvcnarefjyvuuwuzijtsabpkibsq` int DEFAULT NULL,\n `qntreppgevjmbmrpdkbdwdvzoebhsjdq` int DEFAULT NULL,\n `bgmkxkbszdhaxtklrxlddxsxkuqbodkl` int DEFAULT NULL,\n `mpvijrdkoxbcfnsobljwrkvmdtvdxswn` int DEFAULT NULL,\n `xdepjcsbgcsckzchxrnvkkvjvhnrnxnk` int DEFAULT NULL,\n `kpxutcqymgqlyqourlytqtissztfmnhn` int DEFAULT NULL,\n `mrirvmbzqbqgcopojsnvehvdkfokqzuv` int DEFAULT NULL,\n `aombcmltukxczxcghaokflgsewphvnac` int DEFAULT NULL,\n `tehxaunxvowsbchdfrghbejvztaaxwdz` int DEFAULT NULL,\n `wpmbjvgqcddurstwbpluvztstlutifws` int DEFAULT NULL,\n `lmkfoalfytrzcafvktukswpzojvicvuf` int DEFAULT NULL,\n `hrimempxstttgfcvozvvnjzsgkpthzhn` int DEFAULT NULL,\n `piymxgtejyqxjpvosqvzvlsyhjejiprb` int DEFAULT NULL,\n `dfcelgjkuamtteaapdsarwydroslvlab` int DEFAULT NULL,\n `kihfmoemutbazqwbnpybuftahzzpudcr` int DEFAULT NULL,\n `tveclkejyqygpyxfolvxkuuswpdvceij` int DEFAULT NULL,\n `ibohjgiljpntdlrhrvamjdnafxnmenko` int DEFAULT NULL,\n `gcummniwcgkjyclxpbvunetzjuwdrgbu` int DEFAULT NULL,\n `sqcvhdkplmkcchqmqfxztinqrjzpcxxl` int DEFAULT NULL,\n `osskfdmhjglpyqhafzbbwvtxxhlcqvkn` int DEFAULT NULL,\n `ukyqykgriqprmmhcfouhlswpcymnedpk` int DEFAULT NULL,\n `cbceoxjygbuyflerfqbsgzjpyvskvtgb` int DEFAULT NULL,\n `lpbhbjevmadnhnjhckdmryzvxyxebtko` int DEFAULT NULL,\n `ydpfmgppnbsnqxmkvttolzqkxkbfpvkk` int DEFAULT NULL,\n `cwtofpbyclkqzyprxyttyteiviqaliuq` int DEFAULT NULL,\n `vpuaiuxptuyyrxhgaqaoqlmgunnfcdfu` int DEFAULT NULL,\n `oogakwzihdcsinujtdailnzdyncplbva` int DEFAULT NULL,\n `wmrshshmjjpxfvyevwgclnwodpxzfyat` int DEFAULT NULL,\n `takarauhjcbawngwxftddntrorfuvuwv` int DEFAULT NULL,\n `iquyqlosbtpfmwckdlhyapeonjltyujf` int DEFAULT NULL,\n PRIMARY KEY (`cdyaahsfoywvvnifdjqsqndwjkbrlgme`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"wzskeeofxzgrcdxuehzyhdhmmlzyxqxo\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["cdyaahsfoywvvnifdjqsqndwjkbrlgme"],"columns":[{"name":"cdyaahsfoywvvnifdjqsqndwjkbrlgme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"cseixoliiylnhxwoaqibefxhakoejhxp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vmypacziptmmacthmfmanepvkzxhyawu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uohajgvtewwvmfhgpgjoybssyhacoyko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nneoxltoyvnqwzlirssrdjppfozgouzy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwvnlmivqsomjhnjbgilicwubszjlakd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvxstehieavrdiazgbyyozcjpskllrcw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdljnvwrcqaropvaitsipjcwunmwiicz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dfcqcecqtnplhqhtpsbomonmcnpqeurt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"noyhpgcuhvwpnunbaqkegypcmatgnkua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzivpbvfixaciwluvmxsjoggdvwbdelc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmqecjrfbenugmankxfwsvpzpgbraeni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gwcrryjmzxlzdzlayrugwdqsnufkiwnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oidguynrodrstqynyygkdlferxcheqns","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"padivievkkjpasixykqglefeivaaioyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blhcvbdfpvkajrwydzfzjqenzcrbrnmt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcnfwiwhvinofnhdxhzykqymubgkbbvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blmteijniyzdtxmvpydlhpwveisgvkzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iplgmxoutngugzijlltwsrwdaudatgbb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvuianfheuszfdupjkeplcdwwmgjxgyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qcnjpvmzrkxvuodtrtaffwcdvpiyqwnt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qizkgphdlvengelyqtngsuobinhybtlw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpkeawkcuysasfqprzjitdfhqpztfzrt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejxxozjghbuhwkjvhuriwrwhlwwlmcuk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bccyenlrpkukedcghzjbhqxmhkdvumnt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkhodioqjcffowpbxflbstxlzlqhcrpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahrykedtoiihaxlpuwkqmjelkdzebrhl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kchfhdsomyzemngwoladimvywzyjsglc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bmlyaiitrfdbypdamjnurmiqvvhrluxp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hpbdzwzfsurzpqrnfzwoxocvkioqxkud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swhspdgrhjdkyygyrazlvjdocifmtjyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bxlsudodjplvhhxqzyqqoxfltoxcmyxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fspmdwyzmlvuanmzxqebucqkdrfnuyub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cfipfkfgpernduxrxdmcfyzanlnouwan","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwdosrfktqyfdbsizfbtnhxkapowvmvm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zoggzudhmzoqrbjhwgxejfyguwxjhbns","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"snkqrqyejwrwyrphgfcqbvfctondyjlb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrhetdummkcqmnseevizikqwpusqlxbh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbtlhbmgjeykfoqmtszinbgsrajuqkaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clmjxtthqpuaqomaeuqfmbpinkxidvtr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojxlcrutvipcnugknhevdjxirctgdxqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lkaffnuwvbpvwpmchrhebxutggokecro","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvrrcgclfupwjqjxjesfdxtvddleyqrd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbkkrfpvioekpvxoujajsaakwvmklljz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecxlfyxcotmeffbmzdpshxnmeuccmtaz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkqgidznvqeghqtnuvhkcoqnwwiyvroh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xttpfsykumncztilvupbwbhbnzyzteew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqrsdcilohnrgcoqglhfqvlamlnaetcg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xromuwzxnlfgscdauvykjaqphazkeoys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azjaesfjfzudkemfuwraprhninslkfjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmxswmzuzdprldhnefeyzanjjnqnmkbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbhwnhipewoxoqkauhevjmbqeyyoebgg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"arfvqmeyfcohkelbuoarpqfgkvgcsvmq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knlalkpxatljabxbqaohjuigybotvhwt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wffexsbkvxggcdpqbzwqgxugakdpvwji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ktozzobjzvbdptceowbkodiwjoporplb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gbjrfmlzwywpcimqxykphhrckqaezczl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbdbtckiqhqtyporztxhcolzhhmkobvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqxlofbofwhhshdwmclgeygwcuaaqijq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oxeutjrskwfzoddebcoeleifibxirwlz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbyleocgakcxoehddwwzwmgzzhgrnfdl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrentalhkjfimdgydtqzbwhwmanztyns","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exdodowheqrmrdpnprozzbbuslapviuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"irtqtjbmvhqntisdhdoptwbbbubzukez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhwjxmwmpxqgrouyprotiixcczmsueng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhknpkqrpgzdkzegtghhdbwgdmigxfdn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idsbcgpttroclsybyvnlqdfajfswhkut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjxhexdssjzvwnksblxmefuanbgtxbwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gneyiyfxptztybceopfqiuzlvtoepkdv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvwcwntxtxratbzuqqbwfmwqaacedqxm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzrjnvcnarefjyvuuwuzijtsabpkibsq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qntreppgevjmbmrpdkbdwdvzoebhsjdq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bgmkxkbszdhaxtklrxlddxsxkuqbodkl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mpvijrdkoxbcfnsobljwrkvmdtvdxswn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdepjcsbgcsckzchxrnvkkvjvhnrnxnk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpxutcqymgqlyqourlytqtissztfmnhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrirvmbzqbqgcopojsnvehvdkfokqzuv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aombcmltukxczxcghaokflgsewphvnac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tehxaunxvowsbchdfrghbejvztaaxwdz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wpmbjvgqcddurstwbpluvztstlutifws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmkfoalfytrzcafvktukswpzojvicvuf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hrimempxstttgfcvozvvnjzsgkpthzhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"piymxgtejyqxjpvosqvzvlsyhjejiprb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dfcelgjkuamtteaapdsarwydroslvlab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kihfmoemutbazqwbnpybuftahzzpudcr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tveclkejyqygpyxfolvxkuuswpdvceij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibohjgiljpntdlrhrvamjdnafxnmenko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gcummniwcgkjyclxpbvunetzjuwdrgbu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sqcvhdkplmkcchqmqfxztinqrjzpcxxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"osskfdmhjglpyqhafzbbwvtxxhlcqvkn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ukyqykgriqprmmhcfouhlswpcymnedpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbceoxjygbuyflerfqbsgzjpyvskvtgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lpbhbjevmadnhnjhckdmryzvxyxebtko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydpfmgppnbsnqxmkvttolzqkxkbfpvkk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwtofpbyclkqzyprxyttyteiviqaliuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpuaiuxptuyyrxhgaqaoqlmgunnfcdfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oogakwzihdcsinujtdailnzdyncplbva","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmrshshmjjpxfvyevwgclnwodpxzfyat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"takarauhjcbawngwxftddntrorfuvuwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iquyqlosbtpfmwckdlhyapeonjltyujf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672713,"databaseName":"models_schema","ddl":"CREATE TABLE `xkjqfzysfnzfwcmrzcgjzonrwhldoesm` (\n `bzwfhlcqizawzfgqqsafuidzkqsdjjmm` int NOT NULL,\n `mbrrtlftqzavpkcgwrwtkytmpypugmvg` int DEFAULT NULL,\n `lvuwfndktzsctauarmaajbcsgxghstwg` int DEFAULT NULL,\n `lgbabymlefvouriyluxffvuxcravpely` int DEFAULT NULL,\n `qrajestlwfestiwabjdmgoyzrugfzntb` int DEFAULT NULL,\n `pkrgeuvbopmcyimhpktrpcbsjpeocdpe` int DEFAULT NULL,\n `zciowroqqtrimmocyydnchhbyhuhdaow` int DEFAULT NULL,\n `mfdwumnxijygztbaliiuzjcrfpinvpkx` int DEFAULT NULL,\n `opsypevosdyftjazpcssqwsogvblyriw` int DEFAULT NULL,\n `revgzbkdfuvnotdlvbeqkuwtkadkfnwi` int DEFAULT NULL,\n `tuqylowyzvgvbewrukduxpxndxoznjmn` int DEFAULT NULL,\n `cexfsheptbdfwmnzfwrsfgxhofeygjtj` int DEFAULT NULL,\n `nfmejzriztyhxbjwqcozuywequfxeana` int DEFAULT NULL,\n `ckxbnuandktvicrzeoitqyifocetqvnr` int DEFAULT NULL,\n `tlrktldjyiwllcctqieaozzsqjjtmphs` int DEFAULT NULL,\n `juvdeiygynzjntdjmmgvhcsrgcskkzce` int DEFAULT NULL,\n `lhzmnqtesayduskpgcusjwtuimdacftx` int DEFAULT NULL,\n `zsnqibohryazswoosricuauxjpdimdeu` int DEFAULT NULL,\n `ixxjvdtculwjwcvgcvihudfrhshqnlad` int DEFAULT NULL,\n `umtqukdgyqwnoporfkaqqlncjtnmkkly` int DEFAULT NULL,\n `ncwldpwssakpzrhqisxxrlmdrnmgtwup` int DEFAULT NULL,\n `akkwjdjcspdqxxwehqtntfoctkstlsgs` int DEFAULT NULL,\n `tttpqtjhhfrwdutpyaagikmgrphqxqcm` int DEFAULT NULL,\n `tmjyjvvriofdgdzvfrkrtjrrvshssgpo` int DEFAULT NULL,\n `ffrookcgzlwlgooybiscqtlxckuoyrpg` int DEFAULT NULL,\n `aqgiukbwahdaniadyvcihoygxtpjvung` int DEFAULT NULL,\n `xetfbzaefbnflwwuuzqdrjzndleklxwk` int DEFAULT NULL,\n `lwwdxcfrxbxzyoxxyxkqxzgallirainr` int DEFAULT NULL,\n `hwcsssiibhqzxoeqsdqfzdbmjasbgplz` int DEFAULT NULL,\n `nvwhphehcyhsbkpppnjawygcesuouxkx` int DEFAULT NULL,\n `dhwagfdajsqswkovqevqyguyjncxgfmb` int DEFAULT NULL,\n `bzxlkpcsovojvqmjeifzsqevqmoyyjac` int DEFAULT NULL,\n `lgybpkqylkgcrgzcgetsirjypjtfuxme` int DEFAULT NULL,\n `eoeyoqqtiivpscotedurgqlolrubhjpb` int DEFAULT NULL,\n `jsfbtdfbbubhlncuycwpwkebeqxnoanp` int DEFAULT NULL,\n `xeslvbjulvjnjuryskrfeipvxznwrsxy` int DEFAULT NULL,\n `lzoyydiupibaeuyejrrbpoqhhgzzojfo` int DEFAULT NULL,\n `gygwwtjsecfeqxxyehxtlmpqinnofmji` int DEFAULT NULL,\n `fvfluplyjutpjphinlxvuipyjhocqugj` int DEFAULT NULL,\n `sduqsvatkuwrxeqwhsmkgjfuwghzirrj` int DEFAULT NULL,\n `bjyfkoscmsrhiceqheytpaplwoaekhos` int DEFAULT NULL,\n `foayhkddhufwqszmmpycsqvdzqxyiwgr` int DEFAULT NULL,\n `otqbkcnuzervyxxhoidhpuvvkpdaorgt` int DEFAULT NULL,\n `leojhydttycsalptbminmjvgazgvdigp` int DEFAULT NULL,\n `sscoqzcccaomygjcviavvpqmthzhbunv` int DEFAULT NULL,\n `igmnyedjsgrqbxhpustzpiphviqnzgce` int DEFAULT NULL,\n `uoiiulsqzswjxzorhmridtazcijkxeyx` int DEFAULT NULL,\n `wyfkvsavhyqrjznjpwkxjfzfpzbkdvjp` int DEFAULT NULL,\n `naudwczyqecbmveiryedkfuutnebcajr` int DEFAULT NULL,\n `rjoqocxeqlekktwboqnemwhpnufyrsfl` int DEFAULT NULL,\n `rdmhbhwxgxzkoyrgwnyystwtuujjyoej` int DEFAULT NULL,\n `yjwehsgrzpjrapupcnhkrheptjpspjox` int DEFAULT NULL,\n `xrkahgekizmhqzigqpfcowemqsgkcvly` int DEFAULT NULL,\n `eljjtfqbwotcsxewtvivnbgeyaxjuzsm` int DEFAULT NULL,\n `blphosvkketfydummfnuaxpxuyxfpfhu` int DEFAULT NULL,\n `omfrlfemalspvfwgpjubrmcljuwsipzc` int DEFAULT NULL,\n `wzhjukxbtkzrebbjjfbvprnanyfkyzhn` int DEFAULT NULL,\n `dbpesrtdiggxwapebtpthgzuwrnsxpzj` int DEFAULT NULL,\n `jwdaleaggrotiwesvxrsfyiqosiowqti` int DEFAULT NULL,\n `gmexddvcwfubbjqjnbvqfkvvikjsnypy` int DEFAULT NULL,\n `obnhendztorihgreumkruagvxukqxieq` int DEFAULT NULL,\n `fczpzfpmdwdkbtmvwnrmclbsymtbrhth` int DEFAULT NULL,\n `dfifeztqvfiotvwznfkebmnpixnzmlvs` int DEFAULT NULL,\n `hflcyyobczrqpefjnxdvrhenkswylrgs` int DEFAULT NULL,\n `xycvwueetqgihqatsftswqknkakvhfwt` int DEFAULT NULL,\n `bjcygkqhumzxergwktwcxgjpdpkawfjs` int DEFAULT NULL,\n `dpjeqncwqrdpkvxblpfnimzvztzvcovj` int DEFAULT NULL,\n `kpbllsbsyvvanfcjigafohvhaoiowcgq` int DEFAULT NULL,\n `hbxshrjjwqeescfqbvjgsqnjhktilsnh` int DEFAULT NULL,\n `cwytopitmczcxvqlblcxbdedchuenamv` int DEFAULT NULL,\n `dbocfzxewrdpvwlkgfwheydgchulvblp` int DEFAULT NULL,\n `revvysczkjlhuaphzzccnxeyxopwxzkd` int DEFAULT NULL,\n `jlphfbofmjyoleulmttisqigjtpmljaq` int DEFAULT NULL,\n `dkrcuivnggwpcyjfwpqtytvrinswjfhy` int DEFAULT NULL,\n `fcjeqewvgeikbybfuyitsnusqokwunga` int DEFAULT NULL,\n `wrklemapzwffzhhyipkcfymznyvsmcyv` int DEFAULT NULL,\n `oqqyuhgfpvhowekpfdmebeyhgbiaxszl` int DEFAULT NULL,\n `esetzcmeghkmlvupipkiffimhherfvta` int DEFAULT NULL,\n `ooixirbyybdyyjihvycjmiqwzmlvqedi` int DEFAULT NULL,\n `sbtblmtdmlhexyswbbwbailqpatvobqb` int DEFAULT NULL,\n `anrlcotybgggiriyvfwywzcbfskoucnj` int DEFAULT NULL,\n `cvyrxafzwihsiumkqrrtrixtkjhlqtmp` int DEFAULT NULL,\n `zyjhgftpbnkoqyeckeyschgktfzpxlwr` int DEFAULT NULL,\n `uivcticvloikskkjfdagjpuqqqwbxuod` int DEFAULT NULL,\n `hhqqoqmkfopqokasdxzlxgmeesltcfdw` int DEFAULT NULL,\n `jdwcaurupkhnctcnnuedobjeivhwqays` int DEFAULT NULL,\n `bwzzhwimbafczjnhdvfaimyuboajfuaj` int DEFAULT NULL,\n `uaomjhzlabuwprpawzxxfjptewtochfg` int DEFAULT NULL,\n `yxxyhlbvtnajryxsigfedgysblxwtmxj` int DEFAULT NULL,\n `eiyvlkgsofppvbceqjtrriheajzddzmv` int DEFAULT NULL,\n `eyqbzeafppjghcmpqkicnzzjudleakhi` int DEFAULT NULL,\n `aemzmyuwkpcmnocgwkdrqvbluvgmhxef` int DEFAULT NULL,\n `mzzuxcpqylgixusvrilwlulvqjylfkai` int DEFAULT NULL,\n `shaoaupealxkmugisjowpqycqccjfsep` int DEFAULT NULL,\n `awvfsupzyvjslshteylvmaseiuzzmmuc` int DEFAULT NULL,\n `wzuddlhzfcnlrherfzmvcauimifhzzqp` int DEFAULT NULL,\n `meswddfukfunkmgvzssevhezdoppxkyc` int DEFAULT NULL,\n `ckfeabrqgmooegkxgfowwehbryiyyquh` int DEFAULT NULL,\n `qtpovsirrexfvmxmyubjzxpkivagdyfu` int DEFAULT NULL,\n `ypkpunwvhhcqrjayieuxkdixzfwtmovc` int DEFAULT NULL,\n PRIMARY KEY (`bzwfhlcqizawzfgqqsafuidzkqsdjjmm`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"xkjqfzysfnzfwcmrzcgjzonrwhldoesm\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["bzwfhlcqizawzfgqqsafuidzkqsdjjmm"],"columns":[{"name":"bzwfhlcqizawzfgqqsafuidzkqsdjjmm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"mbrrtlftqzavpkcgwrwtkytmpypugmvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvuwfndktzsctauarmaajbcsgxghstwg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgbabymlefvouriyluxffvuxcravpely","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrajestlwfestiwabjdmgoyzrugfzntb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkrgeuvbopmcyimhpktrpcbsjpeocdpe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zciowroqqtrimmocyydnchhbyhuhdaow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfdwumnxijygztbaliiuzjcrfpinvpkx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opsypevosdyftjazpcssqwsogvblyriw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"revgzbkdfuvnotdlvbeqkuwtkadkfnwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tuqylowyzvgvbewrukduxpxndxoznjmn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cexfsheptbdfwmnzfwrsfgxhofeygjtj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfmejzriztyhxbjwqcozuywequfxeana","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckxbnuandktvicrzeoitqyifocetqvnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlrktldjyiwllcctqieaozzsqjjtmphs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"juvdeiygynzjntdjmmgvhcsrgcskkzce","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhzmnqtesayduskpgcusjwtuimdacftx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zsnqibohryazswoosricuauxjpdimdeu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixxjvdtculwjwcvgcvihudfrhshqnlad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umtqukdgyqwnoporfkaqqlncjtnmkkly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ncwldpwssakpzrhqisxxrlmdrnmgtwup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akkwjdjcspdqxxwehqtntfoctkstlsgs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tttpqtjhhfrwdutpyaagikmgrphqxqcm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmjyjvvriofdgdzvfrkrtjrrvshssgpo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffrookcgzlwlgooybiscqtlxckuoyrpg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqgiukbwahdaniadyvcihoygxtpjvung","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xetfbzaefbnflwwuuzqdrjzndleklxwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwwdxcfrxbxzyoxxyxkqxzgallirainr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwcsssiibhqzxoeqsdqfzdbmjasbgplz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvwhphehcyhsbkpppnjawygcesuouxkx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dhwagfdajsqswkovqevqyguyjncxgfmb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzxlkpcsovojvqmjeifzsqevqmoyyjac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgybpkqylkgcrgzcgetsirjypjtfuxme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eoeyoqqtiivpscotedurgqlolrubhjpb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jsfbtdfbbubhlncuycwpwkebeqxnoanp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xeslvbjulvjnjuryskrfeipvxznwrsxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzoyydiupibaeuyejrrbpoqhhgzzojfo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gygwwtjsecfeqxxyehxtlmpqinnofmji","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fvfluplyjutpjphinlxvuipyjhocqugj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sduqsvatkuwrxeqwhsmkgjfuwghzirrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjyfkoscmsrhiceqheytpaplwoaekhos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"foayhkddhufwqszmmpycsqvdzqxyiwgr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otqbkcnuzervyxxhoidhpuvvkpdaorgt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"leojhydttycsalptbminmjvgazgvdigp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sscoqzcccaomygjcviavvpqmthzhbunv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igmnyedjsgrqbxhpustzpiphviqnzgce","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uoiiulsqzswjxzorhmridtazcijkxeyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyfkvsavhyqrjznjpwkxjfzfpzbkdvjp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"naudwczyqecbmveiryedkfuutnebcajr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjoqocxeqlekktwboqnemwhpnufyrsfl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdmhbhwxgxzkoyrgwnyystwtuujjyoej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjwehsgrzpjrapupcnhkrheptjpspjox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrkahgekizmhqzigqpfcowemqsgkcvly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eljjtfqbwotcsxewtvivnbgeyaxjuzsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blphosvkketfydummfnuaxpxuyxfpfhu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omfrlfemalspvfwgpjubrmcljuwsipzc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzhjukxbtkzrebbjjfbvprnanyfkyzhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbpesrtdiggxwapebtpthgzuwrnsxpzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwdaleaggrotiwesvxrsfyiqosiowqti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmexddvcwfubbjqjnbvqfkvvikjsnypy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obnhendztorihgreumkruagvxukqxieq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fczpzfpmdwdkbtmvwnrmclbsymtbrhth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dfifeztqvfiotvwznfkebmnpixnzmlvs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hflcyyobczrqpefjnxdvrhenkswylrgs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xycvwueetqgihqatsftswqknkakvhfwt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjcygkqhumzxergwktwcxgjpdpkawfjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpjeqncwqrdpkvxblpfnimzvztzvcovj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpbllsbsyvvanfcjigafohvhaoiowcgq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbxshrjjwqeescfqbvjgsqnjhktilsnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwytopitmczcxvqlblcxbdedchuenamv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbocfzxewrdpvwlkgfwheydgchulvblp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"revvysczkjlhuaphzzccnxeyxopwxzkd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlphfbofmjyoleulmttisqigjtpmljaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkrcuivnggwpcyjfwpqtytvrinswjfhy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcjeqewvgeikbybfuyitsnusqokwunga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wrklemapzwffzhhyipkcfymznyvsmcyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqqyuhgfpvhowekpfdmebeyhgbiaxszl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"esetzcmeghkmlvupipkiffimhherfvta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ooixirbyybdyyjihvycjmiqwzmlvqedi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbtblmtdmlhexyswbbwbailqpatvobqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"anrlcotybgggiriyvfwywzcbfskoucnj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvyrxafzwihsiumkqrrtrixtkjhlqtmp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyjhgftpbnkoqyeckeyschgktfzpxlwr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uivcticvloikskkjfdagjpuqqqwbxuod","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhqqoqmkfopqokasdxzlxgmeesltcfdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdwcaurupkhnctcnnuedobjeivhwqays","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bwzzhwimbafczjnhdvfaimyuboajfuaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uaomjhzlabuwprpawzxxfjptewtochfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxxyhlbvtnajryxsigfedgysblxwtmxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eiyvlkgsofppvbceqjtrriheajzddzmv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eyqbzeafppjghcmpqkicnzzjudleakhi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aemzmyuwkpcmnocgwkdrqvbluvgmhxef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzzuxcpqylgixusvrilwlulvqjylfkai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shaoaupealxkmugisjowpqycqccjfsep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awvfsupzyvjslshteylvmaseiuzzmmuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzuddlhzfcnlrherfzmvcauimifhzzqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"meswddfukfunkmgvzssevhezdoppxkyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckfeabrqgmooegkxgfowwehbryiyyquh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtpovsirrexfvmxmyubjzxpkivagdyfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypkpunwvhhcqrjayieuxkdixzfwtmovc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672743,"databaseName":"models_schema","ddl":"CREATE TABLE `xkwxygabgxrruwrhxqbyhhgahhzdbnwg` (\n `fxbacykfttbojgbzvukowtihwkutxdjn` int NOT NULL,\n `dnjjflcxvxsstievgxizlotzlscjupmu` int DEFAULT NULL,\n `rgtkljsezjthboyzhagwoxxezzbxrybb` int DEFAULT NULL,\n `zrmoyjzmeqohqrbimkzqjssoqecsfzpq` int DEFAULT NULL,\n `gnlyxpdhegmntyiygrpwmpvfptuuhqmo` int DEFAULT NULL,\n `fnhqjldojjarkckaoftjcihuvwwarbge` int DEFAULT NULL,\n `atcsjjyxdubyinkkeynlgviphtnnhlke` int DEFAULT NULL,\n `yggpnjbxpcerdxrthaogbdjzidccaaio` int DEFAULT NULL,\n `mbxikxqrvwejwhgviwmnirskvfemvfgr` int DEFAULT NULL,\n `vhmdtnatbjnalyodoisairytdzjxzxal` int DEFAULT NULL,\n `sylyihbpyxxxkuccauuzlgfhwovehgve` int DEFAULT NULL,\n `elmtfyjyjjloopugjiuksnsugeitktql` int DEFAULT NULL,\n `microtocuxzsuvwhqewgdbmsqgolownb` int DEFAULT NULL,\n `tjjgxlrrjhfdxljonaqhmbnvstkmmnsl` int DEFAULT NULL,\n `lehwrsbmfzwwppqifulhsnyelcmdfhwv` int DEFAULT NULL,\n `ylvuassxycczlhhznizvfsnwyipwukpx` int DEFAULT NULL,\n `xpacjcnhhkbqxfxojxydrcxsdelzhule` int DEFAULT NULL,\n `llstzmefqmgyqdelirqyhzmycwzekdjo` int DEFAULT NULL,\n `isycxwbdkltkkjswojvhyvcbkkirtike` int DEFAULT NULL,\n `qdvbsdbaxjloqdocosswxpzkqqagxhzy` int DEFAULT NULL,\n `mzdlulkxgxzngsxbxhlrvovsvpyedelp` int DEFAULT NULL,\n `fudtdagzgmwebluapaunineoddzavuwj` int DEFAULT NULL,\n `mloiocjiemybyctrqbqlbtabxffutgne` int DEFAULT NULL,\n `leevhtqchwdzsvkkrooidioqrwzrsykw` int DEFAULT NULL,\n `eqjpkiloclcdtuvjgvahbtgbjrdrcanj` int DEFAULT NULL,\n `tiywomhicznltcaodzovrzoxunlofhsb` int DEFAULT NULL,\n `uhhpjszwmtzzuviumuyykghybrxidhqg` int DEFAULT NULL,\n `ndvojtziknxgnyrkzsknegxztvruxchs` int DEFAULT NULL,\n `vfjnesrtxyanpuvfwgajgkrjhlrfeztj` int DEFAULT NULL,\n `cyhruygoylvbnnwhxdhtkonyrhtehhug` int DEFAULT NULL,\n `vboxxjauhsehvpotmycdhxqxypsbqcnb` int DEFAULT NULL,\n `zmmpqhrqkixgsziejhexosbsebkhfjup` int DEFAULT NULL,\n `njvinwotujrdquwjfvreufzeguzqtocv` int DEFAULT NULL,\n `ksnwepclfsvwlcjjafbbkbyxjfeyybeu` int DEFAULT NULL,\n `jwajlgacsdhvfpadtzzvwiesyimqoibz` int DEFAULT NULL,\n `wqexzprsfwxeucbaxmtjccqdzgbeixkw` int DEFAULT NULL,\n `kipfahduslocfobecgaubyscnpvzeerm` int DEFAULT NULL,\n `lizbvfuebgcunyentzqguuqrjuwxlfcf` int DEFAULT NULL,\n `ynszerouiagouqqimvcmqdruwiamwngd` int DEFAULT NULL,\n `onqtdybshqqqbddvfflfdqaezdrpncen` int DEFAULT NULL,\n `zmoakcuubumneropwmivqulisaoarisn` int DEFAULT NULL,\n `swwzimmkpsnfniolctsqccxvxqfrqsbj` int DEFAULT NULL,\n `rwhtfjwaexepcircyhwcqpaygsrkdfql` int DEFAULT NULL,\n `ujbooputhqtditngxyxnojkzlnvijykb` int DEFAULT NULL,\n `cgfjkabpmnsfrsivreblryaacozgetkw` int DEFAULT NULL,\n `cijarweftopblwcueibbtrvygzausnqh` int DEFAULT NULL,\n `vbwggelxpohptlbcnofevhaxgafkgxhl` int DEFAULT NULL,\n `fwsqimylllrijfqwxruusictfkeedqat` int DEFAULT NULL,\n `jwbztrauekmnavaistoyptkuwvnvmqrk` int DEFAULT NULL,\n `idceouyqcavnetecjhplyglioxthnqgh` int DEFAULT NULL,\n `ywrirsuidgrgonuowavzfmywkmupmnij` int DEFAULT NULL,\n `ffpsypsvcmmvupgkfxmfvtvwynsoqygz` int DEFAULT NULL,\n `ccfcwxbfahkbkmocltkzbvjtyeltefmk` int DEFAULT NULL,\n `gfhhqawytqzzphwwtngolzelnpvjrojn` int DEFAULT NULL,\n `cutrsichcfxicpkriuylhsrpklmvuxvk` int DEFAULT NULL,\n `fmfqbxxggptvqfeqbqnthtwvqqyfgoqk` int DEFAULT NULL,\n `fwcjaiffyetjinpcqljiplfoihpqdwfu` int DEFAULT NULL,\n `cjravxhdeaflcsxkewrhibixwyfcllte` int DEFAULT NULL,\n `ctwfruogbgtvbycyybrrqoegwxoopcvq` int DEFAULT NULL,\n `suilndzkpfjhaimnlomwmtzjxviatezc` int DEFAULT NULL,\n `grryqolqavtmhrnoxnsgaahmpkwvzgaq` int DEFAULT NULL,\n `cmzgpophkzwdiulloaykfujcwivzione` int DEFAULT NULL,\n `yeushopmkchuqbecbfvxzqbdvqswupgz` int DEFAULT NULL,\n `sdzkavfparmhlxgelcssjsnvfadqvpjy` int DEFAULT NULL,\n `yeipxrphrmjpqbtqorqbmhquxczvjeln` int DEFAULT NULL,\n `nxpcvgqzwiqqazylxfajtlswntimpjma` int DEFAULT NULL,\n `lvuaxbczvvrdapvwjnrqbkbjnabrlqzn` int DEFAULT NULL,\n `pdyuywxrmhpsgqjavvnahyyajmrudxtr` int DEFAULT NULL,\n `gtunoyjaevybuangovmzbvunudcxcyno` int DEFAULT NULL,\n `hdubcegnzzjapwoqncesyevolnnqfqyb` int DEFAULT NULL,\n `pnwedrklbkfmbemqhjsosuoewvetxlyh` int DEFAULT NULL,\n `lgvqgvtgsrjafsgbfgojtjhtmdwbzqhd` int DEFAULT NULL,\n `awvikdnoafsrzagctqpgvcskbqkogsud` int DEFAULT NULL,\n `fsyohxdyqlldzwzuilfqrclzipdbcina` int DEFAULT NULL,\n `npmfxeafepmmgribkmchjidkjzuatdpt` int DEFAULT NULL,\n `ixlxntlzgcaglamjvtcuaueupwgaznwx` int DEFAULT NULL,\n `fwixsbatxxvlqohpxvpcuckzsexyizny` int DEFAULT NULL,\n `psgtpiqlgnlnhgamrlpvkjzpppplocbm` int DEFAULT NULL,\n `bpmtqnthxanrgoudgvyfkcvulrxlyhvt` int DEFAULT NULL,\n `eoswwwouimgdvghtasdzqhopghxnjhed` int DEFAULT NULL,\n `bvgjhnrvmwhkfrlvqvwyeyslwnhmmndm` int DEFAULT NULL,\n `clyphvaqkqlkueibnoifuhgfphwnhaxg` int DEFAULT NULL,\n `bsfxatxetxqxzwwenmykmphljhccijth` int DEFAULT NULL,\n `kcnyqueswwmjutwgrnijkiosarbnmxiz` int DEFAULT NULL,\n `jtyozxqryjqqfacitnhbdkrwfdhtnlrx` int DEFAULT NULL,\n `ikvwjscmisqgleumnniaqqupfssjsejw` int DEFAULT NULL,\n `grtaqrljizgolkwepmevqeumksvsavnd` int DEFAULT NULL,\n `pqvpawihroucthbslfcmfmiqqnemskvl` int DEFAULT NULL,\n `ydxviwkcqnhvrvzofazaormaqlzxosqz` int DEFAULT NULL,\n `gfmcvkfwhlufjfucfdrijjatpiozyndl` int DEFAULT NULL,\n `ealrhkieuuprpngqsvugcpeymuzvtnfr` int DEFAULT NULL,\n `ljmblnvxqvjfpogvxkwfvyuoiyykerbt` int DEFAULT NULL,\n `quezzqewyjkvutaulghxlxrhhkchakrq` int DEFAULT NULL,\n `ecihpkzibkltvwpanafelqbtihaoxbqj` int DEFAULT NULL,\n `fzyiuawqpycktsreyxzikqxztkxujolv` int DEFAULT NULL,\n `rcorjpzekfeenczhzedzqrzmuvikbpij` int DEFAULT NULL,\n `bcmmaxagrigktifyctmabqvuibujgewn` int DEFAULT NULL,\n `vharyedsmyzcmkmpszyolzlzheksmcaj` int DEFAULT NULL,\n `nzgkblemwefrgorjmvkxnoyrajdmutqe` int DEFAULT NULL,\n `igcnkoyxzqqvdnewxuiutifjitczodnj` int DEFAULT NULL,\n PRIMARY KEY (`fxbacykfttbojgbzvukowtihwkutxdjn`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"xkwxygabgxrruwrhxqbyhhgahhzdbnwg\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["fxbacykfttbojgbzvukowtihwkutxdjn"],"columns":[{"name":"fxbacykfttbojgbzvukowtihwkutxdjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"dnjjflcxvxsstievgxizlotzlscjupmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgtkljsezjthboyzhagwoxxezzbxrybb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zrmoyjzmeqohqrbimkzqjssoqecsfzpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnlyxpdhegmntyiygrpwmpvfptuuhqmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnhqjldojjarkckaoftjcihuvwwarbge","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atcsjjyxdubyinkkeynlgviphtnnhlke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yggpnjbxpcerdxrthaogbdjzidccaaio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbxikxqrvwejwhgviwmnirskvfemvfgr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhmdtnatbjnalyodoisairytdzjxzxal","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sylyihbpyxxxkuccauuzlgfhwovehgve","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elmtfyjyjjloopugjiuksnsugeitktql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"microtocuxzsuvwhqewgdbmsqgolownb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjjgxlrrjhfdxljonaqhmbnvstkmmnsl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lehwrsbmfzwwppqifulhsnyelcmdfhwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ylvuassxycczlhhznizvfsnwyipwukpx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpacjcnhhkbqxfxojxydrcxsdelzhule","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llstzmefqmgyqdelirqyhzmycwzekdjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"isycxwbdkltkkjswojvhyvcbkkirtike","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdvbsdbaxjloqdocosswxpzkqqagxhzy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzdlulkxgxzngsxbxhlrvovsvpyedelp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fudtdagzgmwebluapaunineoddzavuwj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mloiocjiemybyctrqbqlbtabxffutgne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"leevhtqchwdzsvkkrooidioqrwzrsykw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqjpkiloclcdtuvjgvahbtgbjrdrcanj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tiywomhicznltcaodzovrzoxunlofhsb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhhpjszwmtzzuviumuyykghybrxidhqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ndvojtziknxgnyrkzsknegxztvruxchs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vfjnesrtxyanpuvfwgajgkrjhlrfeztj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cyhruygoylvbnnwhxdhtkonyrhtehhug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vboxxjauhsehvpotmycdhxqxypsbqcnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmmpqhrqkixgsziejhexosbsebkhfjup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njvinwotujrdquwjfvreufzeguzqtocv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ksnwepclfsvwlcjjafbbkbyxjfeyybeu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwajlgacsdhvfpadtzzvwiesyimqoibz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqexzprsfwxeucbaxmtjccqdzgbeixkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kipfahduslocfobecgaubyscnpvzeerm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lizbvfuebgcunyentzqguuqrjuwxlfcf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynszerouiagouqqimvcmqdruwiamwngd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"onqtdybshqqqbddvfflfdqaezdrpncen","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmoakcuubumneropwmivqulisaoarisn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swwzimmkpsnfniolctsqccxvxqfrqsbj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rwhtfjwaexepcircyhwcqpaygsrkdfql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujbooputhqtditngxyxnojkzlnvijykb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgfjkabpmnsfrsivreblryaacozgetkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cijarweftopblwcueibbtrvygzausnqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbwggelxpohptlbcnofevhaxgafkgxhl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwsqimylllrijfqwxruusictfkeedqat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwbztrauekmnavaistoyptkuwvnvmqrk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idceouyqcavnetecjhplyglioxthnqgh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywrirsuidgrgonuowavzfmywkmupmnij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffpsypsvcmmvupgkfxmfvtvwynsoqygz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ccfcwxbfahkbkmocltkzbvjtyeltefmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfhhqawytqzzphwwtngolzelnpvjrojn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cutrsichcfxicpkriuylhsrpklmvuxvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmfqbxxggptvqfeqbqnthtwvqqyfgoqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwcjaiffyetjinpcqljiplfoihpqdwfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjravxhdeaflcsxkewrhibixwyfcllte","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctwfruogbgtvbycyybrrqoegwxoopcvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"suilndzkpfjhaimnlomwmtzjxviatezc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grryqolqavtmhrnoxnsgaahmpkwvzgaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmzgpophkzwdiulloaykfujcwivzione","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yeushopmkchuqbecbfvxzqbdvqswupgz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdzkavfparmhlxgelcssjsnvfadqvpjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yeipxrphrmjpqbtqorqbmhquxczvjeln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxpcvgqzwiqqazylxfajtlswntimpjma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvuaxbczvvrdapvwjnrqbkbjnabrlqzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdyuywxrmhpsgqjavvnahyyajmrudxtr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gtunoyjaevybuangovmzbvunudcxcyno","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdubcegnzzjapwoqncesyevolnnqfqyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnwedrklbkfmbemqhjsosuoewvetxlyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgvqgvtgsrjafsgbfgojtjhtmdwbzqhd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awvikdnoafsrzagctqpgvcskbqkogsud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fsyohxdyqlldzwzuilfqrclzipdbcina","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npmfxeafepmmgribkmchjidkjzuatdpt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixlxntlzgcaglamjvtcuaueupwgaznwx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwixsbatxxvlqohpxvpcuckzsexyizny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psgtpiqlgnlnhgamrlpvkjzpppplocbm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bpmtqnthxanrgoudgvyfkcvulrxlyhvt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eoswwwouimgdvghtasdzqhopghxnjhed","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvgjhnrvmwhkfrlvqvwyeyslwnhmmndm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clyphvaqkqlkueibnoifuhgfphwnhaxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bsfxatxetxqxzwwenmykmphljhccijth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcnyqueswwmjutwgrnijkiosarbnmxiz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jtyozxqryjqqfacitnhbdkrwfdhtnlrx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikvwjscmisqgleumnniaqqupfssjsejw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grtaqrljizgolkwepmevqeumksvsavnd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqvpawihroucthbslfcmfmiqqnemskvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydxviwkcqnhvrvzofazaormaqlzxosqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfmcvkfwhlufjfucfdrijjatpiozyndl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ealrhkieuuprpngqsvugcpeymuzvtnfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ljmblnvxqvjfpogvxkwfvyuoiyykerbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"quezzqewyjkvutaulghxlxrhhkchakrq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecihpkzibkltvwpanafelqbtihaoxbqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzyiuawqpycktsreyxzikqxztkxujolv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcorjpzekfeenczhzedzqrzmuvikbpij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bcmmaxagrigktifyctmabqvuibujgewn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vharyedsmyzcmkmpszyolzlzheksmcaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzgkblemwefrgorjmvkxnoyrajdmutqe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igcnkoyxzqqvdnewxuiutifjitczodnj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672773,"databaseName":"models_schema","ddl":"CREATE TABLE `xlbwxatoglvtzyxuzszybrvskcdgicob` (\n `rhksazdkejqohgdodwvmktywriwtdcwl` int NOT NULL,\n `jcwskcmtbdonxsqhsvbqxpwuazyxvroy` int DEFAULT NULL,\n `cdfwddpzywiqtcbhbqecdjscvgftdgsa` int DEFAULT NULL,\n `wzeijkxhuldtbskwidovzlyrxptilqfu` int DEFAULT NULL,\n `dpxopsmjahlbnlnmqiujfcpsrwdctfth` int DEFAULT NULL,\n `ezzjkngjcajzcqtrkthtrirucmqughoq` int DEFAULT NULL,\n `aipaxzeyqbjspbuucschcybnpxsmixce` int DEFAULT NULL,\n `lewqqqglqyqjpgbzjdzhbshegbxgtqta` int DEFAULT NULL,\n `qbdatyqfecsnwmupmsimzzyluiwdktmm` int DEFAULT NULL,\n `rhwgftkswvjudnmxrmxlxtbyowmwgehm` int DEFAULT NULL,\n `cnagtesajabqxxplqqddewxcfmsplqqh` int DEFAULT NULL,\n `wjypcotvmtfvpzayxwpswwpttcgdfkvq` int DEFAULT NULL,\n `zacwepsbmsfbgfzancnzfvrseveyccyi` int DEFAULT NULL,\n `dylhfcnujqbrkwtinrmphbqripbhzbdz` int DEFAULT NULL,\n `qjdyzgwfazygammakorwfzyvjzulprjj` int DEFAULT NULL,\n `ikmzqikeyrtrwaupkomfyxetwfqnxlwo` int DEFAULT NULL,\n `crcvzyjqzpaaacihzldsvgwmrbvudtpq` int DEFAULT NULL,\n `ssxginhevvjufrcyfltzjzdibtizlzhs` int DEFAULT NULL,\n `sspxrqtugdcszfhnyqatkpibcioakvpq` int DEFAULT NULL,\n `xqktyvntkjyvkorjcktjwpjvbanamkuu` int DEFAULT NULL,\n `sfiumqigatmujkzrurfdizioxulrklyz` int DEFAULT NULL,\n `bbrupdbcyciqmtwmwhsxdfqrvvkmrqnb` int DEFAULT NULL,\n `itzrvqnehpzyuxhhflhrylcjsdwlduec` int DEFAULT NULL,\n `bwblsmcrfqueleicngziezeigbjowgwa` int DEFAULT NULL,\n `hsirbivqdpgaxudxpghqhmfhlhbadiko` int DEFAULT NULL,\n `rxqprstpnvgriovtxinwehnxlfjurrwx` int DEFAULT NULL,\n `bjarcxceopznvuryfntenmlyixhmeyzc` int DEFAULT NULL,\n `txrstlkwvizlgztexbilmffadcmyuvnk` int DEFAULT NULL,\n `vbuztpvtesqasdqkufbvjuvejxdxfedw` int DEFAULT NULL,\n `hogydfqmadxhyntotkolhcpuzwkpfydy` int DEFAULT NULL,\n `jkloiwneqwayemnazahtnbnuadzizbzb` int DEFAULT NULL,\n `kbptgdmyxxjgovduhtpwhloyuocuimwg` int DEFAULT NULL,\n `ghbwoyvsnbjmuoskpyfyscaocswowsxm` int DEFAULT NULL,\n `qdkwcagmuddygkvlqhpvpgexbhvckxav` int DEFAULT NULL,\n `vbtwfaqrbqksiofldhfgubmhfmuglfnd` int DEFAULT NULL,\n `itxbkiabadnvxivriennptdojamjczhh` int DEFAULT NULL,\n `myzpvnwhaolhclaxsfsyryifdqutqnnz` int DEFAULT NULL,\n `icefpqmogceyayxxpekrrwlxpqftwwgd` int DEFAULT NULL,\n `mcoxpebxvbxsnafcivmqruvksfgbnthk` int DEFAULT NULL,\n `ykgxzuywynmruvlvfqnypkoundwactjn` int DEFAULT NULL,\n `bntcykfaiztxxzgscibfovjqqapcfllq` int DEFAULT NULL,\n `nghxxpxgqyggmuwwiptqcblzxfmqbuqz` int DEFAULT NULL,\n `xcjjlmovoeitclqrlkiconblqehgwwzj` int DEFAULT NULL,\n `mnumhlfpwjragzxejyrxsmbqooxscqxq` int DEFAULT NULL,\n `dkbfqmeaagdomzncxnfmwernstbkcplo` int DEFAULT NULL,\n `ecocxxfpdjisuxjspzumfoufmhsmqlxt` int DEFAULT NULL,\n `etcdqsuivgxyicafdzqjyiqgycaumdal` int DEFAULT NULL,\n `zwcxrqnmxzevpqbawgjncqlhcneamfxq` int DEFAULT NULL,\n `pqzfftwwddfdfgthocuxipolezhkisqb` int DEFAULT NULL,\n `wlheegggzfnqrjetptjmhilbokpwdceq` int DEFAULT NULL,\n `qbnchuceprsyyzygxbeiilfzxyhkuwej` int DEFAULT NULL,\n `jtuxwkairmoakvwxvohzkeheyysatlks` int DEFAULT NULL,\n `ivitqsgjyedbnvcsltukkkpktebeuunv` int DEFAULT NULL,\n `pjzlbnbgbqiditxpdfrgdtupcqkadlfd` int DEFAULT NULL,\n `bmodkxkzagmpnqfroccdqtsgsthwfnea` int DEFAULT NULL,\n `unmnvhsucdxjvxjaefiefaqthvgvoczt` int DEFAULT NULL,\n `qfexybxvkovxjeartetsfxdkadknovne` int DEFAULT NULL,\n `qenrxvbrsgzkhhetyukxojlgnbndslkd` int DEFAULT NULL,\n `johcmpxtcjfcxbvbyocznbmdqtormyvi` int DEFAULT NULL,\n `cosrrbxmwvdjdaccawutdzipjpnebugn` int DEFAULT NULL,\n `ejcpotnowbnzycpxlemwyfnlpnaifywe` int DEFAULT NULL,\n `xvcnmyebsvsldfgmfioscduwywxcwlnx` int DEFAULT NULL,\n `iveqestoqjtcdhqjwawjibhnaytpyuum` int DEFAULT NULL,\n `vvqcfjrevvabqaeumbfxrmsopiwcxhfo` int DEFAULT NULL,\n `wgbvasmoqfluvwbhavcbrfzcolmlnuxy` int DEFAULT NULL,\n `mbhbquesajgwaubzwkxixxlftdrbyjcg` int DEFAULT NULL,\n `iljaoyhpoeylbjijpqzsbdwmglybicxv` int DEFAULT NULL,\n `txlhyqrcfystoaeyhtddhojsnegwwhqp` int DEFAULT NULL,\n `uqkjkweohwbwusxecuddmyqumptponkh` int DEFAULT NULL,\n `nkbsfrvblwxwcyqbfbfdkwckgxwnvwzq` int DEFAULT NULL,\n `fnfycgweiwrzfgyjugpbpbrtpbeukpjm` int DEFAULT NULL,\n `veyoxcjaoswwealnqefdwdoclvipmyjo` int DEFAULT NULL,\n `ktwchvyqarxttmsjveliycaaltdiirxo` int DEFAULT NULL,\n `yuesvowovdlipdqpwikoqyzwwotsefay` int DEFAULT NULL,\n `bkppdhbckitpkbimcdopczapazgfsurf` int DEFAULT NULL,\n `enpufnkacrmwesdkfdjdhcdyyuezyqob` int DEFAULT NULL,\n `dsizysovqipnhwcwmlwsdebcieqgfiax` int DEFAULT NULL,\n `mbfzucdmvsxgfiybioudjocqxolyaxpa` int DEFAULT NULL,\n `puycezcprdyyunuabdfuyzftenwyyvnl` int DEFAULT NULL,\n `rbuyyiufkzxqpvxhyqkjilqyefkbqsbq` int DEFAULT NULL,\n `kshkrbkghwlhprmoledgpbkbbnoklexi` int DEFAULT NULL,\n `hsqikgzvshwjcknaqpnayziplbwhxten` int DEFAULT NULL,\n `hfcfmpohswuhllrprpomlxxsyznbcklf` int DEFAULT NULL,\n `ytzjcnrysdnptnkgppthxlxmqtegxbmt` int DEFAULT NULL,\n `mvtdvttiygpjtpizwmwbwyccizbvqnol` int DEFAULT NULL,\n `eegwmtudhwokzmrbvhcuwvgornahetud` int DEFAULT NULL,\n `npsvxcrcwhdpzawtyhdungkveekxnvrk` int DEFAULT NULL,\n `cqezhuoxdaxcijthdareoesxbktabaam` int DEFAULT NULL,\n `iycqkyzuoxkszomkmpuskmoxkssvwmua` int DEFAULT NULL,\n `oqufxgbyqxgyoxzoatissayqyhglbnmy` int DEFAULT NULL,\n `dovnthkdnhsrjnbfxddyxaeugprwadcp` int DEFAULT NULL,\n `txzchlpgmzfyvaatkniorffhvougszal` int DEFAULT NULL,\n `othiwoqycrcearhzctdiixycoxzpawyw` int DEFAULT NULL,\n `vypqozuwzrhjtngxafjbzthstkfliemv` int DEFAULT NULL,\n `ncfgciymkpzsnczzuesbztowhcsvhxbt` int DEFAULT NULL,\n `uchrcohzyoazsjrkofneljvydiyjlsrq` int DEFAULT NULL,\n `eiuefkqwjbnlsjjkyppgjwpnevnicyrq` int DEFAULT NULL,\n `jzmqesyihiwzmslthdwltewuvfybvjfb` int DEFAULT NULL,\n `offkdeqeebpbaqacpndkfcmkjmyukcam` int DEFAULT NULL,\n `tivfqjnzvqdeeomfcxrtyysnabyfphmo` int DEFAULT NULL,\n PRIMARY KEY (`rhksazdkejqohgdodwvmktywriwtdcwl`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"xlbwxatoglvtzyxuzszybrvskcdgicob\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["rhksazdkejqohgdodwvmktywriwtdcwl"],"columns":[{"name":"rhksazdkejqohgdodwvmktywriwtdcwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"jcwskcmtbdonxsqhsvbqxpwuazyxvroy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cdfwddpzywiqtcbhbqecdjscvgftdgsa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzeijkxhuldtbskwidovzlyrxptilqfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpxopsmjahlbnlnmqiujfcpsrwdctfth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ezzjkngjcajzcqtrkthtrirucmqughoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aipaxzeyqbjspbuucschcybnpxsmixce","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lewqqqglqyqjpgbzjdzhbshegbxgtqta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qbdatyqfecsnwmupmsimzzyluiwdktmm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhwgftkswvjudnmxrmxlxtbyowmwgehm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnagtesajabqxxplqqddewxcfmsplqqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjypcotvmtfvpzayxwpswwpttcgdfkvq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zacwepsbmsfbgfzancnzfvrseveyccyi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dylhfcnujqbrkwtinrmphbqripbhzbdz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjdyzgwfazygammakorwfzyvjzulprjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikmzqikeyrtrwaupkomfyxetwfqnxlwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crcvzyjqzpaaacihzldsvgwmrbvudtpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ssxginhevvjufrcyfltzjzdibtizlzhs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sspxrqtugdcszfhnyqatkpibcioakvpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqktyvntkjyvkorjcktjwpjvbanamkuu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfiumqigatmujkzrurfdizioxulrklyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbrupdbcyciqmtwmwhsxdfqrvvkmrqnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itzrvqnehpzyuxhhflhrylcjsdwlduec","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bwblsmcrfqueleicngziezeigbjowgwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hsirbivqdpgaxudxpghqhmfhlhbadiko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxqprstpnvgriovtxinwehnxlfjurrwx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjarcxceopznvuryfntenmlyixhmeyzc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txrstlkwvizlgztexbilmffadcmyuvnk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbuztpvtesqasdqkufbvjuvejxdxfedw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hogydfqmadxhyntotkolhcpuzwkpfydy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jkloiwneqwayemnazahtnbnuadzizbzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kbptgdmyxxjgovduhtpwhloyuocuimwg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghbwoyvsnbjmuoskpyfyscaocswowsxm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdkwcagmuddygkvlqhpvpgexbhvckxav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbtwfaqrbqksiofldhfgubmhfmuglfnd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itxbkiabadnvxivriennptdojamjczhh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"myzpvnwhaolhclaxsfsyryifdqutqnnz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icefpqmogceyayxxpekrrwlxpqftwwgd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcoxpebxvbxsnafcivmqruvksfgbnthk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykgxzuywynmruvlvfqnypkoundwactjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bntcykfaiztxxzgscibfovjqqapcfllq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nghxxpxgqyggmuwwiptqcblzxfmqbuqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xcjjlmovoeitclqrlkiconblqehgwwzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnumhlfpwjragzxejyrxsmbqooxscqxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkbfqmeaagdomzncxnfmwernstbkcplo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecocxxfpdjisuxjspzumfoufmhsmqlxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"etcdqsuivgxyicafdzqjyiqgycaumdal","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zwcxrqnmxzevpqbawgjncqlhcneamfxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqzfftwwddfdfgthocuxipolezhkisqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wlheegggzfnqrjetptjmhilbokpwdceq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qbnchuceprsyyzygxbeiilfzxyhkuwej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jtuxwkairmoakvwxvohzkeheyysatlks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivitqsgjyedbnvcsltukkkpktebeuunv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjzlbnbgbqiditxpdfrgdtupcqkadlfd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bmodkxkzagmpnqfroccdqtsgsthwfnea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unmnvhsucdxjvxjaefiefaqthvgvoczt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfexybxvkovxjeartetsfxdkadknovne","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qenrxvbrsgzkhhetyukxojlgnbndslkd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"johcmpxtcjfcxbvbyocznbmdqtormyvi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cosrrbxmwvdjdaccawutdzipjpnebugn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejcpotnowbnzycpxlemwyfnlpnaifywe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvcnmyebsvsldfgmfioscduwywxcwlnx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iveqestoqjtcdhqjwawjibhnaytpyuum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vvqcfjrevvabqaeumbfxrmsopiwcxhfo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wgbvasmoqfluvwbhavcbrfzcolmlnuxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbhbquesajgwaubzwkxixxlftdrbyjcg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iljaoyhpoeylbjijpqzsbdwmglybicxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txlhyqrcfystoaeyhtddhojsnegwwhqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqkjkweohwbwusxecuddmyqumptponkh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkbsfrvblwxwcyqbfbfdkwckgxwnvwzq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnfycgweiwrzfgyjugpbpbrtpbeukpjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"veyoxcjaoswwealnqefdwdoclvipmyjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ktwchvyqarxttmsjveliycaaltdiirxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yuesvowovdlipdqpwikoqyzwwotsefay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkppdhbckitpkbimcdopczapazgfsurf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"enpufnkacrmwesdkfdjdhcdyyuezyqob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsizysovqipnhwcwmlwsdebcieqgfiax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbfzucdmvsxgfiybioudjocqxolyaxpa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"puycezcprdyyunuabdfuyzftenwyyvnl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbuyyiufkzxqpvxhyqkjilqyefkbqsbq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kshkrbkghwlhprmoledgpbkbbnoklexi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hsqikgzvshwjcknaqpnayziplbwhxten","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfcfmpohswuhllrprpomlxxsyznbcklf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytzjcnrysdnptnkgppthxlxmqtegxbmt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvtdvttiygpjtpizwmwbwyccizbvqnol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eegwmtudhwokzmrbvhcuwvgornahetud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npsvxcrcwhdpzawtyhdungkveekxnvrk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqezhuoxdaxcijthdareoesxbktabaam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iycqkyzuoxkszomkmpuskmoxkssvwmua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqufxgbyqxgyoxzoatissayqyhglbnmy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dovnthkdnhsrjnbfxddyxaeugprwadcp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txzchlpgmzfyvaatkniorffhvougszal","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"othiwoqycrcearhzctdiixycoxzpawyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vypqozuwzrhjtngxafjbzthstkfliemv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ncfgciymkpzsnczzuesbztowhcsvhxbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uchrcohzyoazsjrkofneljvydiyjlsrq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eiuefkqwjbnlsjjkyppgjwpnevnicyrq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzmqesyihiwzmslthdwltewuvfybvjfb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"offkdeqeebpbaqacpndkfcmkjmyukcam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tivfqjnzvqdeeomfcxrtyysnabyfphmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672806,"databaseName":"models_schema","ddl":"CREATE TABLE `xmcvrffbufzmyibhrgpsdxwjjojyvegg` (\n `obakmlsnluiljdrxpabhhltidgvjvpub` int NOT NULL,\n `apmyauzkjurvljxaoklrccjvwjewlvma` int DEFAULT NULL,\n `xjiitydwososqszxtdelpysqrlfzflez` int DEFAULT NULL,\n `cgtiyozdreuhvxxvbbeizmruwdwuyrkb` int DEFAULT NULL,\n `wfpyncjbsxizvueusvneqeqdocwmhylu` int DEFAULT NULL,\n `jypbjcrxyvegvsnypokdnoxljsdnywnf` int DEFAULT NULL,\n `arucmhzujdfhhclqsbnhwipoiuoxcura` int DEFAULT NULL,\n `uaouydujspsbihtsamfvkwxjeengscnf` int DEFAULT NULL,\n `luuuyoldtjrtsgkzctevffuzszseleli` int DEFAULT NULL,\n `dxaznbpowvldvpgidxksdjauglykpygo` int DEFAULT NULL,\n `mnlvsmjzrkcigvdtwpyysedzrpjbizai` int DEFAULT NULL,\n `deilbgteanoknuxfvcbhaivinkfaymyc` int DEFAULT NULL,\n `cauivdhbmhockmbvziqcwdaqnnvkqose` int DEFAULT NULL,\n `ylzllfqesdkdvfeiqricayfahidkzppd` int DEFAULT NULL,\n `crjacpsxkbrsjiqkdkkurwsixnvhpxiv` int DEFAULT NULL,\n `gcswhoabodjewqttcsyqbcpfmecojcoq` int DEFAULT NULL,\n `obxqgxdqoxqeqdbhznfamlvwcblwnmlt` int DEFAULT NULL,\n `baudfiiyhjoncqdgvdqngmsqaizowaae` int DEFAULT NULL,\n `jnlcflsidsskbtalkzidyuzztylyghkg` int DEFAULT NULL,\n `hdmivwiuefxizbhrlagmziscpxrrwdek` int DEFAULT NULL,\n `bayjanxisforsdwiuqhiadkikwdezske` int DEFAULT NULL,\n `shdowsvuyqcsmftwjramexxdrxidvdhk` int DEFAULT NULL,\n `ecwitcvppwqgsvdxpgxivtunemfebxov` int DEFAULT NULL,\n `gzakjpiceyqwjrcudyhtyiuzeenjsiev` int DEFAULT NULL,\n `eaumpuqrubzpxovjcvsfnjdlkxjwwxqa` int DEFAULT NULL,\n `lyfveyzgblaovrcyfyhpivqgaemgwgzz` int DEFAULT NULL,\n `topckeaxpznibsaqhulipaqxjeihngva` int DEFAULT NULL,\n `ynyudpgfilquaqputtnibplnstvenbse` int DEFAULT NULL,\n `zulschiabjvdpzqsijygsknmnyhyfnmr` int DEFAULT NULL,\n `ebyivjyfxgdytcltrkikludknugqkbrx` int DEFAULT NULL,\n `rbsokegedppramwnfgjldbelofpzuctt` int DEFAULT NULL,\n `abbotjmxfqzzhdziwcaquargregujnyu` int DEFAULT NULL,\n `dkyxjbsdaqdasihqytcncfpaofcwguky` int DEFAULT NULL,\n `ktvywfolaanzgmydggatcupevnwdjgrj` int DEFAULT NULL,\n `uovushekhplvvxaeyrmutmznfdjvdbnp` int DEFAULT NULL,\n `pbgsmnurlpynoyoizjuynmblncyksybk` int DEFAULT NULL,\n `nikajceubshwddtwhwdmfwuerzayclrf` int DEFAULT NULL,\n `fxyyzoizgpfrmeanblkhyamqirwrkcct` int DEFAULT NULL,\n `qyfsxvrvdyxqusuxnfyuhvzhdannhfjg` int DEFAULT NULL,\n `ozanehzdxrdikvumgmcqdodvbcidkbpu` int DEFAULT NULL,\n `rehvwscvjxzmwswmbrpvesgbvfzuvjhr` int DEFAULT NULL,\n `jyrxxrjuvtnzhgasjjivgrrnepsqdfru` int DEFAULT NULL,\n `efihzyfhackjxfsevjeidcvufrlpheoi` int DEFAULT NULL,\n `zpujkbpvwsblrqovoybtzluehadeuadh` int DEFAULT NULL,\n `sklfervbmifcdfbsmdhqgjwigxmyxcnw` int DEFAULT NULL,\n `vjggietzjkcnejalyhlkawbbnhkzlsze` int DEFAULT NULL,\n `tmainapuiraqadnfubldlcuefvsoceqw` int DEFAULT NULL,\n `kzafcsjywtxykuapftiexywstnehgyxg` int DEFAULT NULL,\n `adbnbhppsfhxqfuxzzrfjepmrfgqmtbw` int DEFAULT NULL,\n `lgddbffdkdwnypzzktitkgyxagymrydb` int DEFAULT NULL,\n `zflralknyosqsajhkgwvbspvnkrxzgpt` int DEFAULT NULL,\n `wxtspwgvhyyuxbbssrisnslywvenqjqe` int DEFAULT NULL,\n `bpivjovhwagngueydibcjrjwbxsomwrb` int DEFAULT NULL,\n `xddynvdxrfljatxeldwntculwakyrlfs` int DEFAULT NULL,\n `imvimdypmvetcjccuowbgxdpulomzcjb` int DEFAULT NULL,\n `jdrkhaxlhamgxwyclyzdqfirdwhrgdhi` int DEFAULT NULL,\n `zoqgghdxsvpmwlclyfmwfnhaydumzbfe` int DEFAULT NULL,\n `hzskugxsqsykucnsskxqvytsydeosvvg` int DEFAULT NULL,\n `jmyifrephnouczoyaeuprqvjyhvskkyx` int DEFAULT NULL,\n `rfawylshlqdcxdtotvoaqfuzvevipexx` int DEFAULT NULL,\n `ngyaycsgyvblwizlerbzyyrwtjhhhvxk` int DEFAULT NULL,\n `okolqdockjqjvblcmguterhhtmkkngzl` int DEFAULT NULL,\n `coefmkfeznitmhdupqgwptufgkjzxkaq` int DEFAULT NULL,\n `pchcgsoviqnacggnsromictyllghchzy` int DEFAULT NULL,\n `fjwsljhqdevcusssifydgvtqkzjkahwx` int DEFAULT NULL,\n `dcqevydlxgopnnqlmjdkmhprzxtmwdim` int DEFAULT NULL,\n `mqdeguyaavhmficcsuhgysdjlaacaafn` int DEFAULT NULL,\n `tkbsleracexybtyiiwkxojfbdtixcjrx` int DEFAULT NULL,\n `kndeclwlgquhjpovevsrhnoxcqhekuxn` int DEFAULT NULL,\n `gqtftcdgcfudhlkiwwnxblwdsiwagaka` int DEFAULT NULL,\n `mwkjxtchjvtdqexjgfbknjaibylsisjd` int DEFAULT NULL,\n `vexwqdjuznihnfjvgflajsinyouxsdgk` int DEFAULT NULL,\n `euvcddphoyacnwebsfwcmznkgmhmfqta` int DEFAULT NULL,\n `froghlodlrfhnwxlvpgvsgmcswctytqc` int DEFAULT NULL,\n `rcjlqjbmfpauhsvxnoltvfeinjdyahyw` int DEFAULT NULL,\n `aspfbgkcituscjqrldvyycywawmcwoud` int DEFAULT NULL,\n `buerovrhysifetybagkfhdvidyjqvonc` int DEFAULT NULL,\n `zqesgisanotuzjvoybutrhbgbdxfbzrj` int DEFAULT NULL,\n `hgihazitujdakhmdzdwonyqmqwyjpxgg` int DEFAULT NULL,\n `unxtlfudovflrugmtxvfgvrvgxbbjtur` int DEFAULT NULL,\n `pvocdqonyentexaklxsmlmpfdxkjsige` int DEFAULT NULL,\n `jumrmzyudnzqpivqkavswnvxfqdixcoh` int DEFAULT NULL,\n `fohednuuksmtwamdjgtylvdwsurtdsby` int DEFAULT NULL,\n `tbgfhdtdmrlrccrtwqhhlkkludtbdnac` int DEFAULT NULL,\n `esejmnsmpopclgsvfbamkkbsqptcvhyg` int DEFAULT NULL,\n `higvgggikntwnkmbhlgapkhbisbxvebn` int DEFAULT NULL,\n `ingissunhynxhgctcqvitijdjvmriirv` int DEFAULT NULL,\n `iszpttofmlgzaoaffnufejsmhvzmgpfe` int DEFAULT NULL,\n `uphmwzpbfsqvhaucweajbyugcrcjalwm` int DEFAULT NULL,\n `ddyqoffbaadhoslrvtdvnaarmcosxqkh` int DEFAULT NULL,\n `uxfgoroafsfbyoashiwphsxfutosmyhl` int DEFAULT NULL,\n `xzucgbxqgtzvemegesolmnrlfwowoaqv` int DEFAULT NULL,\n `inudmfjuehechxmsybbcwkiqklpkaumy` int DEFAULT NULL,\n `euoycydotikqwxoyuuvyvhmsgpehsyme` int DEFAULT NULL,\n `quxqedswkplykkyzzwcrfacydczuwqpd` int DEFAULT NULL,\n `oueiyfgbswsjxobjbnfjmoyzfoydwsio` int DEFAULT NULL,\n `yhynwsnbpflgnmhotgfurgkrkhatiprg` int DEFAULT NULL,\n `tsrwmarhuqpxoloidvhpigycydsxxqmr` int DEFAULT NULL,\n `nohzzphjaqzugkxeevwiawohrpqjkxsz` int DEFAULT NULL,\n `vdpngqdzlbcgwnyznkezfglnuqfupoip` int DEFAULT NULL,\n PRIMARY KEY (`obakmlsnluiljdrxpabhhltidgvjvpub`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"xmcvrffbufzmyibhrgpsdxwjjojyvegg\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["obakmlsnluiljdrxpabhhltidgvjvpub"],"columns":[{"name":"obakmlsnluiljdrxpabhhltidgvjvpub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"apmyauzkjurvljxaoklrccjvwjewlvma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjiitydwososqszxtdelpysqrlfzflez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgtiyozdreuhvxxvbbeizmruwdwuyrkb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfpyncjbsxizvueusvneqeqdocwmhylu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jypbjcrxyvegvsnypokdnoxljsdnywnf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"arucmhzujdfhhclqsbnhwipoiuoxcura","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uaouydujspsbihtsamfvkwxjeengscnf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"luuuyoldtjrtsgkzctevffuzszseleli","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxaznbpowvldvpgidxksdjauglykpygo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnlvsmjzrkcigvdtwpyysedzrpjbizai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"deilbgteanoknuxfvcbhaivinkfaymyc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cauivdhbmhockmbvziqcwdaqnnvkqose","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ylzllfqesdkdvfeiqricayfahidkzppd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crjacpsxkbrsjiqkdkkurwsixnvhpxiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gcswhoabodjewqttcsyqbcpfmecojcoq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obxqgxdqoxqeqdbhznfamlvwcblwnmlt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"baudfiiyhjoncqdgvdqngmsqaizowaae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jnlcflsidsskbtalkzidyuzztylyghkg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdmivwiuefxizbhrlagmziscpxrrwdek","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bayjanxisforsdwiuqhiadkikwdezske","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shdowsvuyqcsmftwjramexxdrxidvdhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecwitcvppwqgsvdxpgxivtunemfebxov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzakjpiceyqwjrcudyhtyiuzeenjsiev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eaumpuqrubzpxovjcvsfnjdlkxjwwxqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lyfveyzgblaovrcyfyhpivqgaemgwgzz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"topckeaxpznibsaqhulipaqxjeihngva","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ynyudpgfilquaqputtnibplnstvenbse","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zulschiabjvdpzqsijygsknmnyhyfnmr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebyivjyfxgdytcltrkikludknugqkbrx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rbsokegedppramwnfgjldbelofpzuctt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abbotjmxfqzzhdziwcaquargregujnyu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkyxjbsdaqdasihqytcncfpaofcwguky","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ktvywfolaanzgmydggatcupevnwdjgrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uovushekhplvvxaeyrmutmznfdjvdbnp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbgsmnurlpynoyoizjuynmblncyksybk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nikajceubshwddtwhwdmfwuerzayclrf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxyyzoizgpfrmeanblkhyamqirwrkcct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qyfsxvrvdyxqusuxnfyuhvzhdannhfjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozanehzdxrdikvumgmcqdodvbcidkbpu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rehvwscvjxzmwswmbrpvesgbvfzuvjhr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jyrxxrjuvtnzhgasjjivgrrnepsqdfru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"efihzyfhackjxfsevjeidcvufrlpheoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpujkbpvwsblrqovoybtzluehadeuadh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sklfervbmifcdfbsmdhqgjwigxmyxcnw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjggietzjkcnejalyhlkawbbnhkzlsze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmainapuiraqadnfubldlcuefvsoceqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzafcsjywtxykuapftiexywstnehgyxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"adbnbhppsfhxqfuxzzrfjepmrfgqmtbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgddbffdkdwnypzzktitkgyxagymrydb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zflralknyosqsajhkgwvbspvnkrxzgpt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxtspwgvhyyuxbbssrisnslywvenqjqe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bpivjovhwagngueydibcjrjwbxsomwrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xddynvdxrfljatxeldwntculwakyrlfs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"imvimdypmvetcjccuowbgxdpulomzcjb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdrkhaxlhamgxwyclyzdqfirdwhrgdhi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zoqgghdxsvpmwlclyfmwfnhaydumzbfe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hzskugxsqsykucnsskxqvytsydeosvvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmyifrephnouczoyaeuprqvjyhvskkyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfawylshlqdcxdtotvoaqfuzvevipexx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ngyaycsgyvblwizlerbzyyrwtjhhhvxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"okolqdockjqjvblcmguterhhtmkkngzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"coefmkfeznitmhdupqgwptufgkjzxkaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pchcgsoviqnacggnsromictyllghchzy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjwsljhqdevcusssifydgvtqkzjkahwx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcqevydlxgopnnqlmjdkmhprzxtmwdim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqdeguyaavhmficcsuhgysdjlaacaafn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkbsleracexybtyiiwkxojfbdtixcjrx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kndeclwlgquhjpovevsrhnoxcqhekuxn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqtftcdgcfudhlkiwwnxblwdsiwagaka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mwkjxtchjvtdqexjgfbknjaibylsisjd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vexwqdjuznihnfjvgflajsinyouxsdgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"euvcddphoyacnwebsfwcmznkgmhmfqta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"froghlodlrfhnwxlvpgvsgmcswctytqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcjlqjbmfpauhsvxnoltvfeinjdyahyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aspfbgkcituscjqrldvyycywawmcwoud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"buerovrhysifetybagkfhdvidyjqvonc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqesgisanotuzjvoybutrhbgbdxfbzrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hgihazitujdakhmdzdwonyqmqwyjpxgg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unxtlfudovflrugmtxvfgvrvgxbbjtur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvocdqonyentexaklxsmlmpfdxkjsige","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jumrmzyudnzqpivqkavswnvxfqdixcoh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fohednuuksmtwamdjgtylvdwsurtdsby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbgfhdtdmrlrccrtwqhhlkkludtbdnac","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"esejmnsmpopclgsvfbamkkbsqptcvhyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"higvgggikntwnkmbhlgapkhbisbxvebn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ingissunhynxhgctcqvitijdjvmriirv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iszpttofmlgzaoaffnufejsmhvzmgpfe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uphmwzpbfsqvhaucweajbyugcrcjalwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddyqoffbaadhoslrvtdvnaarmcosxqkh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxfgoroafsfbyoashiwphsxfutosmyhl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xzucgbxqgtzvemegesolmnrlfwowoaqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"inudmfjuehechxmsybbcwkiqklpkaumy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"euoycydotikqwxoyuuvyvhmsgpehsyme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"quxqedswkplykkyzzwcrfacydczuwqpd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oueiyfgbswsjxobjbnfjmoyzfoydwsio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhynwsnbpflgnmhotgfurgkrkhatiprg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tsrwmarhuqpxoloidvhpigycydsxxqmr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nohzzphjaqzugkxeevwiawohrpqjkxsz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdpngqdzlbcgwnyznkezfglnuqfupoip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672837,"databaseName":"models_schema","ddl":"CREATE TABLE `xrghycskudhhtjoegkkgwoksolissqht` (\n `zydbyjayktfvhoeyqdbnxbnrbizoaxyq` int NOT NULL,\n `vrnrrkoixzbvhthfjiuotzxmwkjecise` int DEFAULT NULL,\n `pqtphrpwyafifghzedbpkyemizrnzrai` int DEFAULT NULL,\n `hxoiunrpzmubwjzneeafnqsaderhvopv` int DEFAULT NULL,\n `abqwyzszosgwqnitgpeefutefparxzlb` int DEFAULT NULL,\n `kyryvnubuvafainohwutnvqpvjicrnmg` int DEFAULT NULL,\n `oirbivlmhaklxmrgyfneemslnlstujtu` int DEFAULT NULL,\n `kvdppaimmcvsntdhuqsytivuqkqaxcvv` int DEFAULT NULL,\n `xhmhczbqjmnjncbqqlgnxipumqytnnlf` int DEFAULT NULL,\n `jayeooqtwmvawcvldifheqawihaazdgj` int DEFAULT NULL,\n `sgwzuxpinzrwipkybaxbwaaxindaxzbt` int DEFAULT NULL,\n `ihnnzsahkqxbayhtutyaxftxorbnuome` int DEFAULT NULL,\n `axftfsjhwezuxdveslngslnsuhubaqrf` int DEFAULT NULL,\n `essifetzhlkxryhvuftpzlulfzplzfzf` int DEFAULT NULL,\n `ejtivnzvmncznrwbgvnmvqjdbhasyrlw` int DEFAULT NULL,\n `ryednrwuqmcgktxozxobkeplvmvhlhsi` int DEFAULT NULL,\n `ofnryclnhcbsmffoauqyjjmyfwsdffgc` int DEFAULT NULL,\n `hmxkbxadiobmrnfbikcuabmxrunhstqs` int DEFAULT NULL,\n `nvnozcqsasbubuwtqygkexaoxkfzthjj` int DEFAULT NULL,\n `ikaaxboypwqrtpqghzqhuczjmizztibv` int DEFAULT NULL,\n `gdchaezemlbcnyeqjcfnthwuadjrsqgw` int DEFAULT NULL,\n `famqrnbhesaalenvdliesbpexbqtwhxv` int DEFAULT NULL,\n `jjvnenucsmfvkmmlxhphjgnqymuvlbqj` int DEFAULT NULL,\n `msudouvferyaiqzhgddhnubnjsgtuaqo` int DEFAULT NULL,\n `sqmitdslyqqxgmlgtdufwfeztmqnqosf` int DEFAULT NULL,\n `ihoslklwzbfnkwktimavsusmaybxuati` int DEFAULT NULL,\n `udwoduiyakcqxcdxmrtmbsgkoakeuufd` int DEFAULT NULL,\n `jppgjhgpvbnlnlsdugododausocvdezs` int DEFAULT NULL,\n `sosojgvbakalalhlmanpujasazowcbos` int DEFAULT NULL,\n `sfdaoxobqabuhcecngwrffdvhvbtmoue` int DEFAULT NULL,\n `cicsitymgyarrbbhzmgqdmqtxlztyhce` int DEFAULT NULL,\n `xixogkrlflypqtmbxktkavhmogwihdvt` int DEFAULT NULL,\n `ejrulsadnslvqstxwlranbmonewvvdzv` int DEFAULT NULL,\n `wzfvrrbaponovixgerprucpzyojbmxlj` int DEFAULT NULL,\n `rlyhmzozbezcgpzfeopyrwdarhorgond` int DEFAULT NULL,\n `jdrbnzxceqfmmzhckjqylxggikltweqv` int DEFAULT NULL,\n `qjdexgbsqnuulthbtabmeraalbsyccev` int DEFAULT NULL,\n `pzsbbqgmjmujummhfbzrpsqnovoqzsgk` int DEFAULT NULL,\n `bkpqcoqwxcgyvjkiglzqwwogcectvkuu` int DEFAULT NULL,\n `dtmkbjkhtdmlrpkxyqqbdrovtqtogjfb` int DEFAULT NULL,\n `znovwekwrqspzcvmknzfhegsphbfopwt` int DEFAULT NULL,\n `elhxjldfodnebvtpfakpzxbowsrxefqs` int DEFAULT NULL,\n `cnolsfxqhjbplgngrvcncfamhqeexrez` int DEFAULT NULL,\n `gxsvayjbdcgzbjnlgelzqvygksxjpeix` int DEFAULT NULL,\n `uhtgmffhqqqznwemzavarelzugnhymlv` int DEFAULT NULL,\n `ejtavwxcsaobiotatapjdcxvcivxccqw` int DEFAULT NULL,\n `tlwzultwskhojwketysbrotvalsjvnjs` int DEFAULT NULL,\n `ifhtpggkjssqvetahzwyoorsaiivkvzp` int DEFAULT NULL,\n `wtscphjegtavlofdbmziodzypunbzudb` int DEFAULT NULL,\n `dlmzthiguszehcnvnofolkcjlxsupqhx` int DEFAULT NULL,\n `lelpxdivmulolncjnbqexufxggdtmpdu` int DEFAULT NULL,\n `pqcosvdkttpnmoojqymaxmketfsimsqq` int DEFAULT NULL,\n `cnlzjbbslmqwtxtiuehbbwruoigatdch` int DEFAULT NULL,\n `mqloadlxrumcfompzvvnwbbxtccxmerr` int DEFAULT NULL,\n `gjvwyhqklsyxotmxtjwfjtvlepzavlsk` int DEFAULT NULL,\n `tdamdcpsdkuhrzfpwhlkqwdimsseqein` int DEFAULT NULL,\n `eirlvpsiyavzpynogpaarlbgjyewbdkm` int DEFAULT NULL,\n `nfpjdlivlhydfggfwthncxcqocxmjqcu` int DEFAULT NULL,\n `oxmwdneebxxfngfowtrqzmomzsbkqclr` int DEFAULT NULL,\n `yroyctteqmsssihykkcesnixqevynlps` int DEFAULT NULL,\n `kjbsfutbxvqilzwpujexxupdyjoozper` int DEFAULT NULL,\n `mhnvakirgdhkkmapjpotglzhojopbcdw` int DEFAULT NULL,\n `hxlaitsesjfdkdeqqjiqmtufvppfxcnz` int DEFAULT NULL,\n `ujtfqncqaykpubyhmludeeqisybvegmc` int DEFAULT NULL,\n `upsfimluhuccqtuorczonsetreljawfo` int DEFAULT NULL,\n `wikhrbakhnaykprtzzndgrseocrhtvnt` int DEFAULT NULL,\n `zwhyddrqlcooucciolubuwekyscwiozx` int DEFAULT NULL,\n `hanvytmbwktujquxffnkkuzgxfiuhagq` int DEFAULT NULL,\n `bsincmguimodonsibwptwzyakblyccqz` int DEFAULT NULL,\n `ylcpyhpggajawdcwroomskqjvmywrwwv` int DEFAULT NULL,\n `wouleocgmuajclangmxhdkvvmsxicivz` int DEFAULT NULL,\n `rjvisasumywnjvanngvdafehzhddxwja` int DEFAULT NULL,\n `ijjszodknbdvovmfrfjborgldkdgpnlo` int DEFAULT NULL,\n `agxwimitbwbbuuhosoxtldsfticepmzg` int DEFAULT NULL,\n `bvmynflsqvmqkwdbvdzgjhfgwahetknk` int DEFAULT NULL,\n `jlnkbfchavpkwdiiubouemshpwsdzcyh` int DEFAULT NULL,\n `zbetiqsrqaqqsmvkhpjkshavlzbtlswk` int DEFAULT NULL,\n `yapmtjfoetxdglbpmdhkmikssipcusre` int DEFAULT NULL,\n `esgczftmnqmtiqlsqssnrieejeejvhcg` int DEFAULT NULL,\n `xjupzcwrxdkcuwdbvqusbzefozfszhte` int DEFAULT NULL,\n `tkjhnuwcxyvsyrubwhvdtyspsssolycp` int DEFAULT NULL,\n `xklymzdcfujyvtomrgctxyhfezuldtdn` int DEFAULT NULL,\n `weodvwhlmhkorujwgmxaoaduiucskzvh` int DEFAULT NULL,\n `ywzyifvbgctmbfrqrwxbzpsfnalyzipx` int DEFAULT NULL,\n `mddjikmoobosfjqddupsdtdwtsjbohlr` int DEFAULT NULL,\n `uaupuwgtifanghjllrapgkigrehepcds` int DEFAULT NULL,\n `nrsxwwkfjrblzoiymnfjkzhdyxipzvua` int DEFAULT NULL,\n `fukaheerrolywsintsvorcxwylixfgux` int DEFAULT NULL,\n `beizgzqgxrtzkshimhommsugmgzbhvmk` int DEFAULT NULL,\n `pxsqrxwrxbwknwvtyegdddciprrbzpyv` int DEFAULT NULL,\n `xjzjxhxlsfmrdhgvbtwdirkvkxmdzjkb` int DEFAULT NULL,\n `fjunrcmqnznqxasamzqsnryaaysrinvo` int DEFAULT NULL,\n `tvynhzizqcffaklojlaokdqtljwtxsvs` int DEFAULT NULL,\n `zsfduvdaibtvqllzmoghnqvsnetfnlbw` int DEFAULT NULL,\n `egtkdfcbkszuicymquaeavqkxuhjarde` int DEFAULT NULL,\n `dpcqtgihnujvcegitomwhdzgbacndhye` int DEFAULT NULL,\n `hlxzxxpqxvwvecklsxaolhedcqzyzofv` int DEFAULT NULL,\n `gbxrmzelbjbhkxswcaqffnsxojelfieg` int DEFAULT NULL,\n `ttbcfjxtlxgmlxrqhpcddkeecagnyuwh` int DEFAULT NULL,\n `ghtcenmqwxtkwuosprkqvuodjcbhodsc` int DEFAULT NULL,\n PRIMARY KEY (`zydbyjayktfvhoeyqdbnxbnrbizoaxyq`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"xrghycskudhhtjoegkkgwoksolissqht\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["zydbyjayktfvhoeyqdbnxbnrbizoaxyq"],"columns":[{"name":"zydbyjayktfvhoeyqdbnxbnrbizoaxyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"vrnrrkoixzbvhthfjiuotzxmwkjecise","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqtphrpwyafifghzedbpkyemizrnzrai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxoiunrpzmubwjzneeafnqsaderhvopv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abqwyzszosgwqnitgpeefutefparxzlb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kyryvnubuvafainohwutnvqpvjicrnmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oirbivlmhaklxmrgyfneemslnlstujtu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvdppaimmcvsntdhuqsytivuqkqaxcvv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xhmhczbqjmnjncbqqlgnxipumqytnnlf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jayeooqtwmvawcvldifheqawihaazdgj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sgwzuxpinzrwipkybaxbwaaxindaxzbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ihnnzsahkqxbayhtutyaxftxorbnuome","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axftfsjhwezuxdveslngslnsuhubaqrf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"essifetzhlkxryhvuftpzlulfzplzfzf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejtivnzvmncznrwbgvnmvqjdbhasyrlw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ryednrwuqmcgktxozxobkeplvmvhlhsi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofnryclnhcbsmffoauqyjjmyfwsdffgc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmxkbxadiobmrnfbikcuabmxrunhstqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvnozcqsasbubuwtqygkexaoxkfzthjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikaaxboypwqrtpqghzqhuczjmizztibv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdchaezemlbcnyeqjcfnthwuadjrsqgw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"famqrnbhesaalenvdliesbpexbqtwhxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jjvnenucsmfvkmmlxhphjgnqymuvlbqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"msudouvferyaiqzhgddhnubnjsgtuaqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sqmitdslyqqxgmlgtdufwfeztmqnqosf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ihoslklwzbfnkwktimavsusmaybxuati","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udwoduiyakcqxcdxmrtmbsgkoakeuufd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jppgjhgpvbnlnlsdugododausocvdezs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sosojgvbakalalhlmanpujasazowcbos","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfdaoxobqabuhcecngwrffdvhvbtmoue","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cicsitymgyarrbbhzmgqdmqtxlztyhce","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xixogkrlflypqtmbxktkavhmogwihdvt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejrulsadnslvqstxwlranbmonewvvdzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzfvrrbaponovixgerprucpzyojbmxlj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlyhmzozbezcgpzfeopyrwdarhorgond","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdrbnzxceqfmmzhckjqylxggikltweqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjdexgbsqnuulthbtabmeraalbsyccev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pzsbbqgmjmujummhfbzrpsqnovoqzsgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkpqcoqwxcgyvjkiglzqwwogcectvkuu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtmkbjkhtdmlrpkxyqqbdrovtqtogjfb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znovwekwrqspzcvmknzfhegsphbfopwt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elhxjldfodnebvtpfakpzxbowsrxefqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnolsfxqhjbplgngrvcncfamhqeexrez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxsvayjbdcgzbjnlgelzqvygksxjpeix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhtgmffhqqqznwemzavarelzugnhymlv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ejtavwxcsaobiotatapjdcxvcivxccqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlwzultwskhojwketysbrotvalsjvnjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifhtpggkjssqvetahzwyoorsaiivkvzp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtscphjegtavlofdbmziodzypunbzudb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dlmzthiguszehcnvnofolkcjlxsupqhx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lelpxdivmulolncjnbqexufxggdtmpdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqcosvdkttpnmoojqymaxmketfsimsqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnlzjbbslmqwtxtiuehbbwruoigatdch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mqloadlxrumcfompzvvnwbbxtccxmerr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjvwyhqklsyxotmxtjwfjtvlepzavlsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdamdcpsdkuhrzfpwhlkqwdimsseqein","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eirlvpsiyavzpynogpaarlbgjyewbdkm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfpjdlivlhydfggfwthncxcqocxmjqcu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oxmwdneebxxfngfowtrqzmomzsbkqclr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yroyctteqmsssihykkcesnixqevynlps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjbsfutbxvqilzwpujexxupdyjoozper","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhnvakirgdhkkmapjpotglzhojopbcdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxlaitsesjfdkdeqqjiqmtufvppfxcnz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujtfqncqaykpubyhmludeeqisybvegmc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upsfimluhuccqtuorczonsetreljawfo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wikhrbakhnaykprtzzndgrseocrhtvnt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zwhyddrqlcooucciolubuwekyscwiozx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hanvytmbwktujquxffnkkuzgxfiuhagq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bsincmguimodonsibwptwzyakblyccqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ylcpyhpggajawdcwroomskqjvmywrwwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wouleocgmuajclangmxhdkvvmsxicivz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rjvisasumywnjvanngvdafehzhddxwja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijjszodknbdvovmfrfjborgldkdgpnlo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"agxwimitbwbbuuhosoxtldsfticepmzg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvmynflsqvmqkwdbvdzgjhfgwahetknk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlnkbfchavpkwdiiubouemshpwsdzcyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbetiqsrqaqqsmvkhpjkshavlzbtlswk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yapmtjfoetxdglbpmdhkmikssipcusre","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"esgczftmnqmtiqlsqssnrieejeejvhcg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjupzcwrxdkcuwdbvqusbzefozfszhte","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkjhnuwcxyvsyrubwhvdtyspsssolycp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xklymzdcfujyvtomrgctxyhfezuldtdn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"weodvwhlmhkorujwgmxaoaduiucskzvh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywzyifvbgctmbfrqrwxbzpsfnalyzipx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mddjikmoobosfjqddupsdtdwtsjbohlr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uaupuwgtifanghjllrapgkigrehepcds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nrsxwwkfjrblzoiymnfjkzhdyxipzvua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fukaheerrolywsintsvorcxwylixfgux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"beizgzqgxrtzkshimhommsugmgzbhvmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxsqrxwrxbwknwvtyegdddciprrbzpyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjzjxhxlsfmrdhgvbtwdirkvkxmdzjkb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjunrcmqnznqxasamzqsnryaaysrinvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvynhzizqcffaklojlaokdqtljwtxsvs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zsfduvdaibtvqllzmoghnqvsnetfnlbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"egtkdfcbkszuicymquaeavqkxuhjarde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dpcqtgihnujvcegitomwhdzgbacndhye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlxzxxpqxvwvecklsxaolhedcqzyzofv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gbxrmzelbjbhkxswcaqffnsxojelfieg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ttbcfjxtlxgmlxrqhpcddkeecagnyuwh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghtcenmqwxtkwuosprkqvuodjcbhodsc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672869,"databaseName":"models_schema","ddl":"CREATE TABLE `xyfheiynhztwfzchczoqkoazqlkqybll` (\n `twegkxbgcloxlzdjocsxagondlseofhd` int NOT NULL,\n `qlsfytsnaptrqppeetbolgpidfugiuyy` int DEFAULT NULL,\n `vaarwttbgxdgbghqqabzmstdcqygqidf` int DEFAULT NULL,\n `eemiczvkmiyxzbuknqoisbhdomkzrwku` int DEFAULT NULL,\n `cqtgjkzryrseindopavrxihrlvlmqctf` int DEFAULT NULL,\n `qcghahcpkmehoiqiqjhfpnnizhsdxdrz` int DEFAULT NULL,\n `cnjgiawcvnuxbhgfqoomkuyfehvowhfm` int DEFAULT NULL,\n `cmapcvjsjivtsdmrkbcagnnnmpozunzl` int DEFAULT NULL,\n `wtlbesjytrnzacaiphwrboqmwbicyaaw` int DEFAULT NULL,\n `ojtnvhipkuixucyhmmrgrukhsmtaoduj` int DEFAULT NULL,\n `dxqkawrybijqhpktctyqeinhykiqqbau` int DEFAULT NULL,\n `rkwbtmbogjrodvfcufvhpyikgdrmdqyf` int DEFAULT NULL,\n `togwtsxlpomcwyoquvttowvbveywdoau` int DEFAULT NULL,\n `uvtpqciuxygdgsfyumbekazzwcljpbzo` int DEFAULT NULL,\n `mztweaxjkbzphjanrloloijrwvutrtfp` int DEFAULT NULL,\n `aqcvvdqwivvfxyzooblwugqyxfekbxnm` int DEFAULT NULL,\n `nfjcmhjvxeykfxkjbzcvaoduqamcmjna` int DEFAULT NULL,\n `llljtqzrdendfvrfbrihvhnimihyrime` int DEFAULT NULL,\n `nawscootczfhbjcyystljtvrcojbbzuy` int DEFAULT NULL,\n `yaivwkkszehkujzdxckgasqqizpffjic` int DEFAULT NULL,\n `zdlwlflzgehmnlmohqeylsyvtzkusgfk` int DEFAULT NULL,\n `axnlqdtzyepcwkmfbzvyarfkmuaoeoof` int DEFAULT NULL,\n `ahgfyihlhvwudwbohvlkxxjlqgovgdvd` int DEFAULT NULL,\n `vhqlnalmijayitbkvfuwkpeikesmioms` int DEFAULT NULL,\n `ozuqkxlbhymhpahmqamzpmxpojjkezmv` int DEFAULT NULL,\n `wcqzocujrnmbjdtbjhfqaidvhzlksqxs` int DEFAULT NULL,\n `isvqpufzfrsjblfosqtlsnipktkxtghn` int DEFAULT NULL,\n `aolhqvnucrvkegneslufwyayszmrltom` int DEFAULT NULL,\n `drlzhnotlunmqgsuupoqoqdykjbozawg` int DEFAULT NULL,\n `lurrbtkbtgakcgothuxevfgjtgvubows` int DEFAULT NULL,\n `szuqevasfcpwuehmwaldgedvsfubnyon` int DEFAULT NULL,\n `beebqyaiadbrhnpcwwmfgtzzauehfitw` int DEFAULT NULL,\n `mxaskmmujyagyxmqwdfabmrwaidxqlxk` int DEFAULT NULL,\n `bcgwsxrwnweyznlmlvrifpwzunifxxpj` int DEFAULT NULL,\n `mhzijzwpyzvxbtsnqypfvoniukjlzeux` int DEFAULT NULL,\n `aohmppobksnucimfbzncxhgayrjaysxt` int DEFAULT NULL,\n `moxhzkukhzotkxejqvmkawkhbsaqgvpx` int DEFAULT NULL,\n `dvddgocvyjegjwfewaermifklvedmohb` int DEFAULT NULL,\n `ksbfzuiwfcxprpuhkwmwwgfppekzuxqg` int DEFAULT NULL,\n `twyhxxkqpjwvawlxfpvcrysqnmyjssqb` int DEFAULT NULL,\n `cwdyyrujtevsxbxfswjaookkgwtrselw` int DEFAULT NULL,\n `qhkwkjhvqqnidmwojcpnamhjnoiibuss` int DEFAULT NULL,\n `yqkxjvrbbjevozkdwphyelybqvjbjajb` int DEFAULT NULL,\n `fxdotivtbtoriuwuirkkxfqtwzckyrko` int DEFAULT NULL,\n `aegonfxxqaqtmyurpnlqiomhhejrjiak` int DEFAULT NULL,\n `ncumphqiqqykliygfydzokoukbmvpatx` int DEFAULT NULL,\n `xzjzaxjkfjdolmhpmqgqygmcxhshmfqz` int DEFAULT NULL,\n `mukskjhracajlojznivvxemmbtohutay` int DEFAULT NULL,\n `wocxfjsongjtbozwcqronueuslixjmbf` int DEFAULT NULL,\n `gyhizjfeskgnxcowfftsftusxlwhujms` int DEFAULT NULL,\n `zstssxspgjbqllzofkbdbbpicttnrhdn` int DEFAULT NULL,\n `pxbujpsvqmfaanvzghnaopnuugyfvgwl` int DEFAULT NULL,\n `olfmwxhlljgorzgjbuvhopirubtkvekh` int DEFAULT NULL,\n `aoeazuzhlqnnxcqmadcdbhwhelmhricl` int DEFAULT NULL,\n `nnfjmnesnapjctnyoelwnoerinodbush` int DEFAULT NULL,\n `nscpihieyuunitfbfkunvohlrpwqswkq` int DEFAULT NULL,\n `vlkysanmdqsqbjgmfmfccmwczxkagelt` int DEFAULT NULL,\n `mhjvpvjrruebxivvhjicexhrxghnbfyh` int DEFAULT NULL,\n `qcwaxephzulsxipaskadgydgvnbzytqt` int DEFAULT NULL,\n `wxzfhnbmwomptdoaikbwomphcljnzqir` int DEFAULT NULL,\n `udnaebxvuoijlprtlkpywxmkbeddkokb` int DEFAULT NULL,\n `czkqcyevvjpongezuvvpreuacntaklyh` int DEFAULT NULL,\n `gefxhwlbjjdtblpthrtuixtakzrpbnaq` int DEFAULT NULL,\n `xwylmqvcmdawlmxnmuyfkxcihwpbfceu` int DEFAULT NULL,\n `ishmowlpgcwaypbialmyniwpcvyxvvnr` int DEFAULT NULL,\n `gzpcvzhrglzvrndkqtsjdmjpsycvkbja` int DEFAULT NULL,\n `isiunilqlsdcirvzmirfjiggtyoxgxlh` int DEFAULT NULL,\n `tkqvilqeonwioihskiceyzjzxvmsofld` int DEFAULT NULL,\n `ojmjswuworntfviicjtqullopypszfms` int DEFAULT NULL,\n `sysqphkjdkuhqpcmrdpciditjhnrazrn` int DEFAULT NULL,\n `gchyvxvrywbiugazmvanebygnevmifob` int DEFAULT NULL,\n `xxdgjywwmcvkwesnzapzrrrmwpwbjpzo` int DEFAULT NULL,\n `ukmmhsmvdqvgibmkipskghfwfgbixfcz` int DEFAULT NULL,\n `bogrhguthujhtrlnevtyqfnmpbasbjwe` int DEFAULT NULL,\n `tmnukyuvcqupttmuxgxnwrorzpusegln` int DEFAULT NULL,\n `npeotaelllswdyofvyjqyexsiiozkbtm` int DEFAULT NULL,\n `hhrhecxnklgshkmvhbwdvgudxlxjbcyw` int DEFAULT NULL,\n `mbufduhmdkygyaglnudtdobfveevbvoh` int DEFAULT NULL,\n `bpsaohuonttaqajaxqdyykkkxdntzrws` int DEFAULT NULL,\n `wzfrhzodomupaqbrpdcpiudcvkawlknp` int DEFAULT NULL,\n `rslkxuaosyvlistxycmckibzgqxteukm` int DEFAULT NULL,\n `ctjtcallkvnxbgkxfptduguoplkepayj` int DEFAULT NULL,\n `ujfrbgcgcclmgasmjpvzdprrjvwuvbqo` int DEFAULT NULL,\n `dghjaswzdyymhrmxdlcijwaouecssgpp` int DEFAULT NULL,\n `edueboetlxpmyhyozwboridwkawmpxlv` int DEFAULT NULL,\n `egaghufrmudqydtdayadsudwztpkrjzv` int DEFAULT NULL,\n `eplodwrmqqbitpryisdnogndgapgaqzb` int DEFAULT NULL,\n `mrgxfvxojujkvrqidzglgvzguspwhdut` int DEFAULT NULL,\n `uuxyhibcezbcvkwhsqnekfefssrvaoqf` int DEFAULT NULL,\n `nrjbgxymiqmlwijtuotbmzpwytxtfqjm` int DEFAULT NULL,\n `xzxnnmzxxeubrmratnkmlukjmdtoyapd` int DEFAULT NULL,\n `ofulrwwbvhjrgsqcnszcxbnxpgsqmucr` int DEFAULT NULL,\n `wabmcaejovnklxbgsciobzxiqcjgtalm` int DEFAULT NULL,\n `ihmkbjklbrfpokzazoeqflrasxrgbeot` int DEFAULT NULL,\n `pfooyktjyubtmdwcdkvnpygskgvcfzey` int DEFAULT NULL,\n `norepkyxpnvdbcrbedwnrmcjybefbfeb` int DEFAULT NULL,\n `fvohyqowjxowqggfoamuixnzdnazffmr` int DEFAULT NULL,\n `sxczhwgzyrtaksuvwvupehvczxtsefgi` int DEFAULT NULL,\n `xdidxtgxekyhfhifwgmhfdvaxgwahayo` int DEFAULT NULL,\n `otfiddellmwkpqyprmusijfzfrprtmfu` int DEFAULT NULL,\n PRIMARY KEY (`twegkxbgcloxlzdjocsxagondlseofhd`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"xyfheiynhztwfzchczoqkoazqlkqybll\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["twegkxbgcloxlzdjocsxagondlseofhd"],"columns":[{"name":"twegkxbgcloxlzdjocsxagondlseofhd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"qlsfytsnaptrqppeetbolgpidfugiuyy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vaarwttbgxdgbghqqabzmstdcqygqidf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eemiczvkmiyxzbuknqoisbhdomkzrwku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqtgjkzryrseindopavrxihrlvlmqctf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qcghahcpkmehoiqiqjhfpnnizhsdxdrz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnjgiawcvnuxbhgfqoomkuyfehvowhfm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmapcvjsjivtsdmrkbcagnnnmpozunzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtlbesjytrnzacaiphwrboqmwbicyaaw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojtnvhipkuixucyhmmrgrukhsmtaoduj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxqkawrybijqhpktctyqeinhykiqqbau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkwbtmbogjrodvfcufvhpyikgdrmdqyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"togwtsxlpomcwyoquvttowvbveywdoau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvtpqciuxygdgsfyumbekazzwcljpbzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mztweaxjkbzphjanrloloijrwvutrtfp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqcvvdqwivvfxyzooblwugqyxfekbxnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfjcmhjvxeykfxkjbzcvaoduqamcmjna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"llljtqzrdendfvrfbrihvhnimihyrime","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nawscootczfhbjcyystljtvrcojbbzuy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yaivwkkszehkujzdxckgasqqizpffjic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdlwlflzgehmnlmohqeylsyvtzkusgfk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axnlqdtzyepcwkmfbzvyarfkmuaoeoof","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahgfyihlhvwudwbohvlkxxjlqgovgdvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhqlnalmijayitbkvfuwkpeikesmioms","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozuqkxlbhymhpahmqamzpmxpojjkezmv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcqzocujrnmbjdtbjhfqaidvhzlksqxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"isvqpufzfrsjblfosqtlsnipktkxtghn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aolhqvnucrvkegneslufwyayszmrltom","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drlzhnotlunmqgsuupoqoqdykjbozawg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lurrbtkbtgakcgothuxevfgjtgvubows","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"szuqevasfcpwuehmwaldgedvsfubnyon","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"beebqyaiadbrhnpcwwmfgtzzauehfitw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mxaskmmujyagyxmqwdfabmrwaidxqlxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bcgwsxrwnweyznlmlvrifpwzunifxxpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhzijzwpyzvxbtsnqypfvoniukjlzeux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aohmppobksnucimfbzncxhgayrjaysxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"moxhzkukhzotkxejqvmkawkhbsaqgvpx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvddgocvyjegjwfewaermifklvedmohb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ksbfzuiwfcxprpuhkwmwwgfppekzuxqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twyhxxkqpjwvawlxfpvcrysqnmyjssqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwdyyrujtevsxbxfswjaookkgwtrselw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhkwkjhvqqnidmwojcpnamhjnoiibuss","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqkxjvrbbjevozkdwphyelybqvjbjajb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxdotivtbtoriuwuirkkxfqtwzckyrko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aegonfxxqaqtmyurpnlqiomhhejrjiak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ncumphqiqqykliygfydzokoukbmvpatx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xzjzaxjkfjdolmhpmqgqygmcxhshmfqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mukskjhracajlojznivvxemmbtohutay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wocxfjsongjtbozwcqronueuslixjmbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gyhizjfeskgnxcowfftsftusxlwhujms","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zstssxspgjbqllzofkbdbbpicttnrhdn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxbujpsvqmfaanvzghnaopnuugyfvgwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"olfmwxhlljgorzgjbuvhopirubtkvekh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aoeazuzhlqnnxcqmadcdbhwhelmhricl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nnfjmnesnapjctnyoelwnoerinodbush","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nscpihieyuunitfbfkunvohlrpwqswkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vlkysanmdqsqbjgmfmfccmwczxkagelt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhjvpvjrruebxivvhjicexhrxghnbfyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qcwaxephzulsxipaskadgydgvnbzytqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxzfhnbmwomptdoaikbwomphcljnzqir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udnaebxvuoijlprtlkpywxmkbeddkokb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"czkqcyevvjpongezuvvpreuacntaklyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gefxhwlbjjdtblpthrtuixtakzrpbnaq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xwylmqvcmdawlmxnmuyfkxcihwpbfceu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ishmowlpgcwaypbialmyniwpcvyxvvnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzpcvzhrglzvrndkqtsjdmjpsycvkbja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"isiunilqlsdcirvzmirfjiggtyoxgxlh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkqvilqeonwioihskiceyzjzxvmsofld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojmjswuworntfviicjtqullopypszfms","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sysqphkjdkuhqpcmrdpciditjhnrazrn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gchyvxvrywbiugazmvanebygnevmifob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxdgjywwmcvkwesnzapzrrrmwpwbjpzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ukmmhsmvdqvgibmkipskghfwfgbixfcz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bogrhguthujhtrlnevtyqfnmpbasbjwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmnukyuvcqupttmuxgxnwrorzpusegln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npeotaelllswdyofvyjqyexsiiozkbtm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhrhecxnklgshkmvhbwdvgudxlxjbcyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbufduhmdkygyaglnudtdobfveevbvoh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bpsaohuonttaqajaxqdyykkkxdntzrws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzfrhzodomupaqbrpdcpiudcvkawlknp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rslkxuaosyvlistxycmckibzgqxteukm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ctjtcallkvnxbgkxfptduguoplkepayj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujfrbgcgcclmgasmjpvzdprrjvwuvbqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dghjaswzdyymhrmxdlcijwaouecssgpp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edueboetlxpmyhyozwboridwkawmpxlv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"egaghufrmudqydtdayadsudwztpkrjzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eplodwrmqqbitpryisdnogndgapgaqzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrgxfvxojujkvrqidzglgvzguspwhdut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uuxyhibcezbcvkwhsqnekfefssrvaoqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nrjbgxymiqmlwijtuotbmzpwytxtfqjm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xzxnnmzxxeubrmratnkmlukjmdtoyapd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofulrwwbvhjrgsqcnszcxbnxpgsqmucr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wabmcaejovnklxbgsciobzxiqcjgtalm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ihmkbjklbrfpokzazoeqflrasxrgbeot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pfooyktjyubtmdwcdkvnpygskgvcfzey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"norepkyxpnvdbcrbedwnrmcjybefbfeb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fvohyqowjxowqggfoamuixnzdnazffmr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sxczhwgzyrtaksuvwvupehvczxtsefgi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdidxtgxekyhfhifwgmhfdvaxgwahayo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otfiddellmwkpqyprmusijfzfrprtmfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672900,"databaseName":"models_schema","ddl":"CREATE TABLE `ydfgmjyjnfakxnzitneuzhmydvouvjes` (\n `exgfxqkbuolnqcuifozhxmerkeiinkrz` int NOT NULL,\n `dirindxkeqpzrcjuuoydmypoendgeybm` int DEFAULT NULL,\n `wnqahvcwcozdyconlmlmpwwcxtteizzd` int DEFAULT NULL,\n `kkjefnndobcjppdejewpgxacudhtklti` int DEFAULT NULL,\n `fzzvxrapkvihuddhqibpyrujhfpwmwgg` int DEFAULT NULL,\n `owqmsipjzesiwappzmtocurxuanwspsu` int DEFAULT NULL,\n `ircfiwppdkpnevkmeevmuyfoxnekgrob` int DEFAULT NULL,\n `vrkehpdithirnyebygtfvtpqsuzdunlu` int DEFAULT NULL,\n `zuismbcyhliuyjjfjgwwjagxlynwpcrt` int DEFAULT NULL,\n `ufuwassncwmudzyhpvxzyozyzzrtstau` int DEFAULT NULL,\n `dtfbrqopbvxfocwrtowbgmwkbndbfftv` int DEFAULT NULL,\n `thyzsimvolvawvjkxyrzuumcmywsdxhg` int DEFAULT NULL,\n `njtnibxdsotkeetklimekyiarfkiuesc` int DEFAULT NULL,\n `ifoguvljcaurhmhzxxpymqxrwaycrggd` int DEFAULT NULL,\n `usvicamkluzvknpgcfcukbxtufgsaxkf` int DEFAULT NULL,\n `eeoxrlamkwmshdkhkulxfrqvmwiouohu` int DEFAULT NULL,\n `tcnepranvfsrcozqxxoldhzuqwkpewrr` int DEFAULT NULL,\n `kckppxlfmermkzwiivppmfgxatqppfyb` int DEFAULT NULL,\n `xikbfjcdjhommtjhrzyzmnaovqfusokk` int DEFAULT NULL,\n `aimcerdgwfvkffxgzbbljevsxguwayel` int DEFAULT NULL,\n `cpuibhzdrkslpnngkxsmnxkiefkinnkt` int DEFAULT NULL,\n `cquafcfovkqzspvrxlekfpwmijecexde` int DEFAULT NULL,\n `uhhwjnpezerhpvgwodqoskwyvncoexqo` int DEFAULT NULL,\n `jbnmuqwwigncoajnznrrbamirifrvwvc` int DEFAULT NULL,\n `wiijgkvswnhnhtzkljxdxuqefllcassb` int DEFAULT NULL,\n `olobqzfwdhfprryqarrdvofgwtapvyme` int DEFAULT NULL,\n `xliioaxbbenddrcoyxnrncwregiytqef` int DEFAULT NULL,\n `tsnrvctdortjeflcgqcwjbdnlevtrfmk` int DEFAULT NULL,\n `kpnstssgqazltzjjvcbgqfjjibtqvoff` int DEFAULT NULL,\n `erorfswoylfjkntdubhhcklfhvfrrdyp` int DEFAULT NULL,\n `gmpvlkltwdmjnsppsijqmeqcvudqraob` int DEFAULT NULL,\n `qkriwlqpehziftzploypkkdmtksztkdy` int DEFAULT NULL,\n `aiszpjctmrxzanqztdjjgfgwgwusuozw` int DEFAULT NULL,\n `wdlnakaxnsgsitsvwrzbcyedfacefdtl` int DEFAULT NULL,\n `xyjreljrhadicowgrguigvsuioicnsde` int DEFAULT NULL,\n `bprkdyxjsftwrnurltsjchnvdlbrikvz` int DEFAULT NULL,\n `blpsnrckdobqpokdbspfjlruclarpbnk` int DEFAULT NULL,\n `ghsbwbsizdmjgnsqervztvkfdxiivosi` int DEFAULT NULL,\n `qiblwsqnwnkezsxyllttaczqdypsdwjc` int DEFAULT NULL,\n `bhssflokceyisexugslfhklsjmismjfs` int DEFAULT NULL,\n `edfekohemjhmkiikqpjawaochrhznfzm` int DEFAULT NULL,\n `pvtwcihznnsnqyizumbioifgwdxtdjsw` int DEFAULT NULL,\n `usaaryvnxnnaejzzewsmfbdsdcyfumah` int DEFAULT NULL,\n `zskohnlytfypyokmybyfrksfoigqdzis` int DEFAULT NULL,\n `jmebqdzpycbenzyeujvzkngjdmhxoxcw` int DEFAULT NULL,\n `mbpyminwqdihfjztfwzmiylhxgxknygw` int DEFAULT NULL,\n `lgkcozflcfrwyhmkrybviugxmrzgvtdn` int DEFAULT NULL,\n `ocscrpdoehtrgpfkjlslcuxkeuxtdizd` int DEFAULT NULL,\n `peqigpqhxggrvbkfogvglvmhbxbpaynz` int DEFAULT NULL,\n `scxxhcxjmteigvwoaslbyxbjrmeezeld` int DEFAULT NULL,\n `kllllkuwcbbpjteryllbwqemgsazxsvl` int DEFAULT NULL,\n `zncjvtyxvluldzzhgwuhsfrpkcajjmbe` int DEFAULT NULL,\n `cclclqgxcvfjysykhrlpvjkxghazexfl` int DEFAULT NULL,\n `whtbqbjrpqoaajbmlconkjbfkxphzuwo` int DEFAULT NULL,\n `mukzltrrydbuatzfzrvosyfiinwiechc` int DEFAULT NULL,\n `tttnqhettsyrkjzsfqdysklrwzptffqk` int DEFAULT NULL,\n `vepharxnpikgnnjvarreutakakcfbzgc` int DEFAULT NULL,\n `ibookcfkchczjhlwgaewkqvpvzhgyisd` int DEFAULT NULL,\n `yswknmzhfshclprwybbdugdcaeavfblb` int DEFAULT NULL,\n `zjdgmmcyvzhfnwudbudoumxpefwifnfr` int DEFAULT NULL,\n `whpypgtreqranixvjkoobvimpsibjghy` int DEFAULT NULL,\n `qzkluydsuqhzlxvqrstbvvvlhdutmqba` int DEFAULT NULL,\n `xrsuvtuimbliqrabfnvgdcxwbnkhkpqa` int DEFAULT NULL,\n `ulsgcfwqiufxzfxhrudjjwkhpoxrjrna` int DEFAULT NULL,\n `sollrkusvnrqbbiaglbkskdsfmkpljua` int DEFAULT NULL,\n `latdxwwaidpehjnysohfsdusxrzedmye` int DEFAULT NULL,\n `zmrnuraqtrmzjtqqdxfgnhzmwlrbbcpo` int DEFAULT NULL,\n `sokrxcpprymidbyoerpthbycgrsajnnn` int DEFAULT NULL,\n `xyywqtmhhnwzahhctnknrkxvthqspwwi` int DEFAULT NULL,\n `yhlqezjzjfadirtcboxmsrcmviispnox` int DEFAULT NULL,\n `luuhmegjwcpgzwzqnbfjdlvfekjemrtp` int DEFAULT NULL,\n `tswzjfkknzistqjidpqkvwklvozafjmn` int DEFAULT NULL,\n `nqhctjsoejlevpeudebsltwnekagwaqh` int DEFAULT NULL,\n `fenuowhxyvypzekmhwyevigejmvqoxqx` int DEFAULT NULL,\n `wpifbimffasqdqkckghuengxyzejguxm` int DEFAULT NULL,\n `kxlhpmalbunkwyykgtkjsejpuuvvctvi` int DEFAULT NULL,\n `rqqzcxlckywbyyoqjazuxfcatxpkeexb` int DEFAULT NULL,\n `gdahbyaudzjupxxlgefnltvwplkihglh` int DEFAULT NULL,\n `htpehfjeicbynncvobnzkphcvwggtpmp` int DEFAULT NULL,\n `yrhmwzhrkforpaoxftmcfsmftpptrelx` int DEFAULT NULL,\n `xlbqmbjlzsokfelxccvbotzuevujsmzm` int DEFAULT NULL,\n `pffwjzoolkwsztajqhoriiaeovmxsgor` int DEFAULT NULL,\n `bbyrcfciquaenqcwoomfehtfvfcttjgr` int DEFAULT NULL,\n `fcffnvcjzqgcqhcsvdpoahaxswmatqmp` int DEFAULT NULL,\n `bvklssbqutlhxhppxpptigvnzvttthtm` int DEFAULT NULL,\n `xecixfbldsgsvgkajmagasmolyehqbit` int DEFAULT NULL,\n `kjyzhdxduqsreziymiquqxmhumujpchd` int DEFAULT NULL,\n `eoqmwwpjrewcfimpeewcmdebuviuxukl` int DEFAULT NULL,\n `aruwbditvfnaiejsplflppsvaacgkmhg` int DEFAULT NULL,\n `hfxmcagtkrpywwbhdbjdbrqjrzyybclf` int DEFAULT NULL,\n `gpxvayctxyeaetokyjredracvytfkbvi` int DEFAULT NULL,\n `qevfyaubjjqjcjpqzxufjvundrczkhbo` int DEFAULT NULL,\n `pdgennvpdyztqkeagossdqkzqzvethme` int DEFAULT NULL,\n `fqoptdailqdzqjsczgqivkfcbaunbrbv` int DEFAULT NULL,\n `fiveyaqzknfyjayykzzzjvkdpdcsuxid` int DEFAULT NULL,\n `vehaxkaizsxftwnciaopgzpmwbcfpcpb` int DEFAULT NULL,\n `bkciqfsssataeaacizwsnbhblfiotxlc` int DEFAULT NULL,\n `xlayqavehzuwcheagzqldzhnrptsyozm` int DEFAULT NULL,\n `tieuhvdowzqynajndchmrkgbonzvnbkl` int DEFAULT NULL,\n `zkaeusfxgejaecdtnsquvlmitqajkbmn` int DEFAULT NULL,\n PRIMARY KEY (`exgfxqkbuolnqcuifozhxmerkeiinkrz`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"ydfgmjyjnfakxnzitneuzhmydvouvjes\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["exgfxqkbuolnqcuifozhxmerkeiinkrz"],"columns":[{"name":"exgfxqkbuolnqcuifozhxmerkeiinkrz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"dirindxkeqpzrcjuuoydmypoendgeybm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnqahvcwcozdyconlmlmpwwcxtteizzd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kkjefnndobcjppdejewpgxacudhtklti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzzvxrapkvihuddhqibpyrujhfpwmwgg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"owqmsipjzesiwappzmtocurxuanwspsu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ircfiwppdkpnevkmeevmuyfoxnekgrob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrkehpdithirnyebygtfvtpqsuzdunlu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zuismbcyhliuyjjfjgwwjagxlynwpcrt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufuwassncwmudzyhpvxzyozyzzrtstau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtfbrqopbvxfocwrtowbgmwkbndbfftv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thyzsimvolvawvjkxyrzuumcmywsdxhg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njtnibxdsotkeetklimekyiarfkiuesc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ifoguvljcaurhmhzxxpymqxrwaycrggd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"usvicamkluzvknpgcfcukbxtufgsaxkf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eeoxrlamkwmshdkhkulxfrqvmwiouohu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tcnepranvfsrcozqxxoldhzuqwkpewrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kckppxlfmermkzwiivppmfgxatqppfyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xikbfjcdjhommtjhrzyzmnaovqfusokk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aimcerdgwfvkffxgzbbljevsxguwayel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cpuibhzdrkslpnngkxsmnxkiefkinnkt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cquafcfovkqzspvrxlekfpwmijecexde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhhwjnpezerhpvgwodqoskwyvncoexqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbnmuqwwigncoajnznrrbamirifrvwvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wiijgkvswnhnhtzkljxdxuqefllcassb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"olobqzfwdhfprryqarrdvofgwtapvyme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xliioaxbbenddrcoyxnrncwregiytqef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tsnrvctdortjeflcgqcwjbdnlevtrfmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpnstssgqazltzjjvcbgqfjjibtqvoff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"erorfswoylfjkntdubhhcklfhvfrrdyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmpvlkltwdmjnsppsijqmeqcvudqraob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qkriwlqpehziftzploypkkdmtksztkdy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aiszpjctmrxzanqztdjjgfgwgwusuozw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wdlnakaxnsgsitsvwrzbcyedfacefdtl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyjreljrhadicowgrguigvsuioicnsde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bprkdyxjsftwrnurltsjchnvdlbrikvz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blpsnrckdobqpokdbspfjlruclarpbnk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ghsbwbsizdmjgnsqervztvkfdxiivosi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qiblwsqnwnkezsxyllttaczqdypsdwjc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bhssflokceyisexugslfhklsjmismjfs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edfekohemjhmkiikqpjawaochrhznfzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pvtwcihznnsnqyizumbioifgwdxtdjsw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"usaaryvnxnnaejzzewsmfbdsdcyfumah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zskohnlytfypyokmybyfrksfoigqdzis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmebqdzpycbenzyeujvzkngjdmhxoxcw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbpyminwqdihfjztfwzmiylhxgxknygw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgkcozflcfrwyhmkrybviugxmrzgvtdn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocscrpdoehtrgpfkjlslcuxkeuxtdizd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"peqigpqhxggrvbkfogvglvmhbxbpaynz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"scxxhcxjmteigvwoaslbyxbjrmeezeld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kllllkuwcbbpjteryllbwqemgsazxsvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zncjvtyxvluldzzhgwuhsfrpkcajjmbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cclclqgxcvfjysykhrlpvjkxghazexfl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"whtbqbjrpqoaajbmlconkjbfkxphzuwo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mukzltrrydbuatzfzrvosyfiinwiechc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tttnqhettsyrkjzsfqdysklrwzptffqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vepharxnpikgnnjvarreutakakcfbzgc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibookcfkchczjhlwgaewkqvpvzhgyisd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yswknmzhfshclprwybbdugdcaeavfblb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zjdgmmcyvzhfnwudbudoumxpefwifnfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"whpypgtreqranixvjkoobvimpsibjghy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzkluydsuqhzlxvqrstbvvvlhdutmqba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrsuvtuimbliqrabfnvgdcxwbnkhkpqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ulsgcfwqiufxzfxhrudjjwkhpoxrjrna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sollrkusvnrqbbiaglbkskdsfmkpljua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"latdxwwaidpehjnysohfsdusxrzedmye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmrnuraqtrmzjtqqdxfgnhzmwlrbbcpo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sokrxcpprymidbyoerpthbycgrsajnnn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyywqtmhhnwzahhctnknrkxvthqspwwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhlqezjzjfadirtcboxmsrcmviispnox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"luuhmegjwcpgzwzqnbfjdlvfekjemrtp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tswzjfkknzistqjidpqkvwklvozafjmn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqhctjsoejlevpeudebsltwnekagwaqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fenuowhxyvypzekmhwyevigejmvqoxqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wpifbimffasqdqkckghuengxyzejguxm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxlhpmalbunkwyykgtkjsejpuuvvctvi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rqqzcxlckywbyyoqjazuxfcatxpkeexb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdahbyaudzjupxxlgefnltvwplkihglh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htpehfjeicbynncvobnzkphcvwggtpmp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yrhmwzhrkforpaoxftmcfsmftpptrelx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xlbqmbjlzsokfelxccvbotzuevujsmzm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pffwjzoolkwsztajqhoriiaeovmxsgor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbyrcfciquaenqcwoomfehtfvfcttjgr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcffnvcjzqgcqhcsvdpoahaxswmatqmp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvklssbqutlhxhppxpptigvnzvttthtm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xecixfbldsgsvgkajmagasmolyehqbit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kjyzhdxduqsreziymiquqxmhumujpchd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eoqmwwpjrewcfimpeewcmdebuviuxukl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aruwbditvfnaiejsplflppsvaacgkmhg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfxmcagtkrpywwbhdbjdbrqjrzyybclf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gpxvayctxyeaetokyjredracvytfkbvi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qevfyaubjjqjcjpqzxufjvundrczkhbo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdgennvpdyztqkeagossdqkzqzvethme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqoptdailqdzqjsczgqivkfcbaunbrbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fiveyaqzknfyjayykzzzjvkdpdcsuxid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vehaxkaizsxftwnciaopgzpmwbcfpcpb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkciqfsssataeaacizwsnbhblfiotxlc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xlayqavehzuwcheagzqldzhnrptsyozm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tieuhvdowzqynajndchmrkgbonzvnbkl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zkaeusfxgejaecdtnsquvlmitqajkbmn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672931,"databaseName":"models_schema","ddl":"CREATE TABLE `yfcmxhvqnfmsqbirfafqjuymmgluegpl` (\n `epafeffqwmxlbfpwbbfcdhilsmpidwyk` int NOT NULL,\n `apawbsldulqruydqgbykjgpiboxclwpr` int DEFAULT NULL,\n `kubzzymdgnybexahufeagdxnttdsmghw` int DEFAULT NULL,\n `qgnexavtndummbzjqekakpjrxwjjyckh` int DEFAULT NULL,\n `lnysccxhjgdweyfgxpghoszgbwxwwiua` int DEFAULT NULL,\n `wjyvowalfzjujbxmrxpnrlqweiwpgvzb` int DEFAULT NULL,\n `cyzawndjflvxnbqidnenorfdbiglvmvc` int DEFAULT NULL,\n `uzosxiteebkcxqmoiigxtukzcxktpxqp` int DEFAULT NULL,\n `logalkcwhpiaprypcypjppkpuwiuaxol` int DEFAULT NULL,\n `vvtomqytlkvvmrfkdhgmcyimmazcsfvo` int DEFAULT NULL,\n `rxrrikseyuunoagyhmwiyooqdeetcjli` int DEFAULT NULL,\n `bjpudwcafxxzsqvlhwmxpghlrsffjoik` int DEFAULT NULL,\n `kuwvhnxacqtsckfgalocgsiufoasgcch` int DEFAULT NULL,\n `bnqvsgypvxxvcnqvssjvtahhmprdeyjs` int DEFAULT NULL,\n `lmbaujekrmijkkkacqslitjbxhxqkncx` int DEFAULT NULL,\n `uwrmuricmxttasiygvosljzigcwcrwnp` int DEFAULT NULL,\n `sbohpsxnyqyxnfrdwhjveespfezmzldb` int DEFAULT NULL,\n `ufxgbzilaahzwazfacjslerhvwcdxzkq` int DEFAULT NULL,\n `bkkmchsfvwyugntjkrxjypzvckjhtdtb` int DEFAULT NULL,\n `kpuefswqcewkzhrbficwksctjrbuvkiy` int DEFAULT NULL,\n `fcuecckzfwdpqihgdaxpjmeizohafgag` int DEFAULT NULL,\n `xyibkiiggcdekksguljspvllnriqzrti` int DEFAULT NULL,\n `ipjsxvisvuxkxyypdlwsjgkzbhbutrhb` int DEFAULT NULL,\n `xlrmvsagkzywplyhrexzhvcpyatspfsz` int DEFAULT NULL,\n `fvtmpongznzraopjmlosmfojczvifazx` int DEFAULT NULL,\n `pqblvupetrckdwjriimfdxuuarllkuwv` int DEFAULT NULL,\n `ujplugmirbylypoduqlrzrdpttoytqvy` int DEFAULT NULL,\n `ojssnvhckglhtubnkqsfkwjqxivrcnpc` int DEFAULT NULL,\n `evhozwrluhypnuiutyfjwosnmfjxamjv` int DEFAULT NULL,\n `lgtyjebyamtvqrxedmscayseegyxrxjz` int DEFAULT NULL,\n `wmqxysibravmnsrrbpccewgvusyocvnv` int DEFAULT NULL,\n `udcmzilwdxvbrutytbpyoitjmscgknyg` int DEFAULT NULL,\n `qgdnddckomkvwxzeueizzwjhpciwxpry` int DEFAULT NULL,\n `njaxtamihduotldbpqkeftjdbxkemjkx` int DEFAULT NULL,\n `xnfapmhnqzrttexyutrigamdwsttmfln` int DEFAULT NULL,\n `gyvmqcayitgapgenbfjgptsmbacutjiz` int DEFAULT NULL,\n `mtpshdswfehqtcqskuwolybzfbykpnot` int DEFAULT NULL,\n `ubbsohvvqjoloifkuwjzqchesimdipki` int DEFAULT NULL,\n `schfsxfyuhrqmfcwplhcpelfjmzkltaf` int DEFAULT NULL,\n `wbthutenphnkrkoslqayqhmdijqvroqc` int DEFAULT NULL,\n `xgxytevcyyqdsxnaxwiprithkkdknbkf` int DEFAULT NULL,\n `azfgvqrkoudeudedktfrbukbfktaphmr` int DEFAULT NULL,\n `pdfjrxejhiurctkwtllhstfwlrdrpvsq` int DEFAULT NULL,\n `asgauganumjrwrjzfyurjsdouwnlyenb` int DEFAULT NULL,\n `gjmeukokhgtvkagzxmmtdlqfenlvpxxx` int DEFAULT NULL,\n `oaatyjhnedexcswxlzeqiatctplapzvh` int DEFAULT NULL,\n `iqjqwulbmecrgjkyjjcppdwhjhwhnnoa` int DEFAULT NULL,\n `madukajxdudapmxngycocgcwtrpmnjqz` int DEFAULT NULL,\n `lenhpkhsuhuwectikaojjzffvuqejcsb` int DEFAULT NULL,\n `rdpfeqbmvsingiyshbrwpcqkcaixowgn` int DEFAULT NULL,\n `cgbcdkjqkrdeavhdgledmwluygxqtgbp` int DEFAULT NULL,\n `zuqpyjbkjcklskrctfzogvzbietfmghx` int DEFAULT NULL,\n `mzaghjdusgzeysptehgycaazvvibmypl` int DEFAULT NULL,\n `qgnzfpqjglwijullhtvbkrnxemtyocdc` int DEFAULT NULL,\n `jzgnwfdkmbkrxgotwgarsincutogscfb` int DEFAULT NULL,\n `fuhsftemzmyakmusvyvsacsnwoqnktko` int DEFAULT NULL,\n `qxooklpoeclfefzasnfvbjziokzkqgdb` int DEFAULT NULL,\n `wfzcryrpqhkfwtvbbuiykfofbtcvvbun` int DEFAULT NULL,\n `dshagcdfcnhodovfbglbkhzvewkxvsjs` int DEFAULT NULL,\n `npnvresudfjcxgakkjffotitrvovcddn` int DEFAULT NULL,\n `ddeotoaxijhhkfotwymzngujmpzgxqfd` int DEFAULT NULL,\n `vdyyxlqmjkvmncyfzxyosgrekywybgbi` int DEFAULT NULL,\n `nzoisurivjwjodupsseiunxmdiqvvkth` int DEFAULT NULL,\n `mxydhlbvalngpcmijshireopervlwcko` int DEFAULT NULL,\n `bgyxvniiuvkxufnzpflmygiphqwelmew` int DEFAULT NULL,\n `lzibxzitzjoykqnfggzsajwzuedgkcgb` int DEFAULT NULL,\n `elarmnsbfcnuhdzykjgsbxkcvcpoqupw` int DEFAULT NULL,\n `zkujpudjuoztjcnimplxspxxtybviiau` int DEFAULT NULL,\n `faxdlwufnaskebubhgrlltciscvualdh` int DEFAULT NULL,\n `qyrhoxqgzzlrcswwercrqdjqdyibzlke` int DEFAULT NULL,\n `vffsualdbfcwnifislwupmxzboqkkdbk` int DEFAULT NULL,\n `wmsgpwuwzdaqlekhvzscaltpcfgqjrez` int DEFAULT NULL,\n `vpwjhomexplmqfzmikjkdssygdikbpxh` int DEFAULT NULL,\n `qqrwluogirvzgmcwhxrlkzrhtceyqwuu` int DEFAULT NULL,\n `lsgczonkpikkevjdoiewzwockfuexzlx` int DEFAULT NULL,\n `tebwceqpmumfpntgvfcskghwdkyxdxnr` int DEFAULT NULL,\n `murwchbzmlyrhiosunipkimwnoysfeto` int DEFAULT NULL,\n `gpjedpgnbxwgtfvttjipytbpomgqbzse` int DEFAULT NULL,\n `nxcaeqyhjnqoqtzdkfluamlmyzlstitw` int DEFAULT NULL,\n `lvpbnonlzjomdzqavqknnjzeckmajegg` int DEFAULT NULL,\n `umvsakxinisaafycxlwwefcbzymyxurc` int DEFAULT NULL,\n `veoaaospfklfqyidjiwtpfoipchcmmqj` int DEFAULT NULL,\n `leantmwysgnhpdkvaebexeigsgooirwv` int DEFAULT NULL,\n `henqfphbfeifbbxuyyxazugzcjlfgscx` int DEFAULT NULL,\n `dnwgotqosatacwthkmoegykjzobycydu` int DEFAULT NULL,\n `zrgayknmwzjgrjuhyksyldykuowkauxz` int DEFAULT NULL,\n `bssvytlijryinugskditqizgdunrpazg` int DEFAULT NULL,\n `nhcsuxdakoizioriuelauscyrtpazkqi` int DEFAULT NULL,\n `euvqhgxbwtibtqwvnrzoyhzsbnkqjhrq` int DEFAULT NULL,\n `zpnraqptjksqoqzraqfujmkmvfgttsbt` int DEFAULT NULL,\n `fdsrzejomaoaylmrlvgforyvsoqiragl` int DEFAULT NULL,\n `vdtjhbkjembidlmmstzthkwugwwkvyzv` int DEFAULT NULL,\n `rrjrgleutbvpdlvsgedfkffljxzqbtey` int DEFAULT NULL,\n `zaakgxcwwsfxywewbtostjksnceibxyz` int DEFAULT NULL,\n `hfwckzlwbitddvptwntnzozmckelvdqh` int DEFAULT NULL,\n `cbnnfdqisyxagjkoedsfseayjllbbjju` int DEFAULT NULL,\n `tiyknzakqozrkezxwwsjmdyeoxtobpya` int DEFAULT NULL,\n `tnaqnzdenkhtkltfiemuxjtjvirwrety` int DEFAULT NULL,\n `aoihqeflwljatmowcnhimkagacmdrtqf` int DEFAULT NULL,\n `qjvlgwldbmfcdqypklxuuqchyisilsho` int DEFAULT NULL,\n PRIMARY KEY (`epafeffqwmxlbfpwbbfcdhilsmpidwyk`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"yfcmxhvqnfmsqbirfafqjuymmgluegpl\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["epafeffqwmxlbfpwbbfcdhilsmpidwyk"],"columns":[{"name":"epafeffqwmxlbfpwbbfcdhilsmpidwyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"apawbsldulqruydqgbykjgpiboxclwpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kubzzymdgnybexahufeagdxnttdsmghw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgnexavtndummbzjqekakpjrxwjjyckh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lnysccxhjgdweyfgxpghoszgbwxwwiua","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wjyvowalfzjujbxmrxpnrlqweiwpgvzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cyzawndjflvxnbqidnenorfdbiglvmvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzosxiteebkcxqmoiigxtukzcxktpxqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"logalkcwhpiaprypcypjppkpuwiuaxol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vvtomqytlkvvmrfkdhgmcyimmazcsfvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxrrikseyuunoagyhmwiyooqdeetcjli","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjpudwcafxxzsqvlhwmxpghlrsffjoik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kuwvhnxacqtsckfgalocgsiufoasgcch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnqvsgypvxxvcnqvssjvtahhmprdeyjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmbaujekrmijkkkacqslitjbxhxqkncx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwrmuricmxttasiygvosljzigcwcrwnp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbohpsxnyqyxnfrdwhjveespfezmzldb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufxgbzilaahzwazfacjslerhvwcdxzkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bkkmchsfvwyugntjkrxjypzvckjhtdtb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kpuefswqcewkzhrbficwksctjrbuvkiy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcuecckzfwdpqihgdaxpjmeizohafgag","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyibkiiggcdekksguljspvllnriqzrti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ipjsxvisvuxkxyypdlwsjgkzbhbutrhb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xlrmvsagkzywplyhrexzhvcpyatspfsz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fvtmpongznzraopjmlosmfojczvifazx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pqblvupetrckdwjriimfdxuuarllkuwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujplugmirbylypoduqlrzrdpttoytqvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojssnvhckglhtubnkqsfkwjqxivrcnpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evhozwrluhypnuiutyfjwosnmfjxamjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgtyjebyamtvqrxedmscayseegyxrxjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmqxysibravmnsrrbpccewgvusyocvnv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"udcmzilwdxvbrutytbpyoitjmscgknyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgdnddckomkvwxzeueizzwjhpciwxpry","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njaxtamihduotldbpqkeftjdbxkemjkx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnfapmhnqzrttexyutrigamdwsttmfln","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gyvmqcayitgapgenbfjgptsmbacutjiz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mtpshdswfehqtcqskuwolybzfbykpnot","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ubbsohvvqjoloifkuwjzqchesimdipki","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"schfsxfyuhrqmfcwplhcpelfjmzkltaf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbthutenphnkrkoslqayqhmdijqvroqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xgxytevcyyqdsxnaxwiprithkkdknbkf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"azfgvqrkoudeudedktfrbukbfktaphmr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdfjrxejhiurctkwtllhstfwlrdrpvsq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"asgauganumjrwrjzfyurjsdouwnlyenb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjmeukokhgtvkagzxmmtdlqfenlvpxxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oaatyjhnedexcswxlzeqiatctplapzvh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqjqwulbmecrgjkyjjcppdwhjhwhnnoa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"madukajxdudapmxngycocgcwtrpmnjqz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lenhpkhsuhuwectikaojjzffvuqejcsb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdpfeqbmvsingiyshbrwpcqkcaixowgn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgbcdkjqkrdeavhdgledmwluygxqtgbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zuqpyjbkjcklskrctfzogvzbietfmghx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzaghjdusgzeysptehgycaazvvibmypl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qgnzfpqjglwijullhtvbkrnxemtyocdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzgnwfdkmbkrxgotwgarsincutogscfb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fuhsftemzmyakmusvyvsacsnwoqnktko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxooklpoeclfefzasnfvbjziokzkqgdb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfzcryrpqhkfwtvbbuiykfofbtcvvbun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dshagcdfcnhodovfbglbkhzvewkxvsjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npnvresudfjcxgakkjffotitrvovcddn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddeotoaxijhhkfotwymzngujmpzgxqfd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdyyxlqmjkvmncyfzxyosgrekywybgbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzoisurivjwjodupsseiunxmdiqvvkth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mxydhlbvalngpcmijshireopervlwcko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bgyxvniiuvkxufnzpflmygiphqwelmew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzibxzitzjoykqnfggzsajwzuedgkcgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"elarmnsbfcnuhdzykjgsbxkcvcpoqupw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zkujpudjuoztjcnimplxspxxtybviiau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"faxdlwufnaskebubhgrlltciscvualdh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qyrhoxqgzzlrcswwercrqdjqdyibzlke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vffsualdbfcwnifislwupmxzboqkkdbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmsgpwuwzdaqlekhvzscaltpcfgqjrez","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpwjhomexplmqfzmikjkdssygdikbpxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqrwluogirvzgmcwhxrlkzrhtceyqwuu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lsgczonkpikkevjdoiewzwockfuexzlx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tebwceqpmumfpntgvfcskghwdkyxdxnr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"murwchbzmlyrhiosunipkimwnoysfeto","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gpjedpgnbxwgtfvttjipytbpomgqbzse","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxcaeqyhjnqoqtzdkfluamlmyzlstitw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvpbnonlzjomdzqavqknnjzeckmajegg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umvsakxinisaafycxlwwefcbzymyxurc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"veoaaospfklfqyidjiwtpfoipchcmmqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"leantmwysgnhpdkvaebexeigsgooirwv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"henqfphbfeifbbxuyyxazugzcjlfgscx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dnwgotqosatacwthkmoegykjzobycydu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zrgayknmwzjgrjuhyksyldykuowkauxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bssvytlijryinugskditqizgdunrpazg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhcsuxdakoizioriuelauscyrtpazkqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"euvqhgxbwtibtqwvnrzoyhzsbnkqjhrq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zpnraqptjksqoqzraqfujmkmvfgttsbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fdsrzejomaoaylmrlvgforyvsoqiragl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdtjhbkjembidlmmstzthkwugwwkvyzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrjrgleutbvpdlvsgedfkffljxzqbtey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zaakgxcwwsfxywewbtostjksnceibxyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfwckzlwbitddvptwntnzozmckelvdqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbnnfdqisyxagjkoedsfseayjllbbjju","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tiyknzakqozrkezxwwsjmdyeoxtobpya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnaqnzdenkhtkltfiemuxjtjvirwrety","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aoihqeflwljatmowcnhimkagacmdrtqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjvlgwldbmfcdqypklxuuqchyisilsho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672962,"databaseName":"models_schema","ddl":"CREATE TABLE `ygbhxqnxduzibovvetyxidhqtnuizvhi` (\n `isfuznhnzgraumzforeffkggjcoirkwa` int NOT NULL,\n `sydcmumjtnsfcneqleczrbikxktvkxgk` int DEFAULT NULL,\n `wsooasrsvplaxdzybrrpufwzzxrnbzii` int DEFAULT NULL,\n `cudvycxulonhisqfjaybjbbdkjxrcjms` int DEFAULT NULL,\n `prmephzvcriehigzyjdlkcfiouhwrtrq` int DEFAULT NULL,\n `clrzybhoajdurwiamsywedcvebdogeem` int DEFAULT NULL,\n `gkxyhggiwfqyjztjegbimbibjayadrqy` int DEFAULT NULL,\n `akhehmwqtqahcjxepylxelexxsarqvxs` int DEFAULT NULL,\n `lgzzgufuaejlvfijhmyabgxgxktgkfql` int DEFAULT NULL,\n `rcxonegnvplubvgccupgkxfaoomohyrv` int DEFAULT NULL,\n `jmmcustnwsmfckqnhwkdctqnvmszcvnm` int DEFAULT NULL,\n `vuysbnwsgjgonloqizzcsoxjrlaozorv` int DEFAULT NULL,\n `xttewemnojamjkkesdzzulizkixpinfx` int DEFAULT NULL,\n `shvxopabgtrapibjyefxhfjkknwcyhzi` int DEFAULT NULL,\n `lljaipkzearewrbbuoiiklyrewndwwkz` int DEFAULT NULL,\n `vglztvqimrnyqyvoqihslpaiedagxfkm` int DEFAULT NULL,\n `xkmtmegwnqpnhlghgxvhfybwelrtauuf` int DEFAULT NULL,\n `xugenhjqtthkqmczrtrjewwzpejyjiaa` int DEFAULT NULL,\n `isrjlxpdttsstvjxrhwmdzlolezxghvx` int DEFAULT NULL,\n `xrmzsckbgyfrcnmfmjxpxhvhzczhmrga` int DEFAULT NULL,\n `wpnmhiynhmbfewgsycocnuroarzmkzba` int DEFAULT NULL,\n `xsvvigknordzufvfjkdfkzqxgpgawpcj` int DEFAULT NULL,\n `xoomxmvxkwolmpyxclvyuepleqwdttlb` int DEFAULT NULL,\n `xrylsroknsmpopakyicvlzxblfsxreks` int DEFAULT NULL,\n `yjeipsjmehxfeugqkpjddcawsvcgpqsq` int DEFAULT NULL,\n `rkrtkqjffycybssnulgpzieejqzairhg` int DEFAULT NULL,\n `zstlcfvujsfwybbqtpcyozpqfnjzqtrw` int DEFAULT NULL,\n `ykriofhfmgivyqeatxptcwloxtyjokry` int DEFAULT NULL,\n `wkybopxwkkcpviheoheqtuoioooktiuh` int DEFAULT NULL,\n `pbdtkntpnmvrxxphcvqxccxdvrqislbd` int DEFAULT NULL,\n `ukdvkcotbhxmifhiwktkqaivhsqqjsew` int DEFAULT NULL,\n `nvyemfubusuxcygtkwqbydbkszhduift` int DEFAULT NULL,\n `ivqowfcjwbdphvlwowczplpifutbqmyd` int DEFAULT NULL,\n `uhenfxppwxxppndsqqfomhpjxmbmrnad` int DEFAULT NULL,\n `yhgsmdbyejazgtharjdrhdtypjrhosam` int DEFAULT NULL,\n `nfgsezhspnwxsdpsrmmlgiavrpdwzxtm` int DEFAULT NULL,\n `pzdopdmepkritqssuqmgytoofubdjgdk` int DEFAULT NULL,\n `nvaahrexsscnmifoqobqghatpmmqefqn` int DEFAULT NULL,\n `hnxqsrljqojgtreutszuobskqisbcdkn` int DEFAULT NULL,\n `fhimryvmyeglkehyrwwwxwqmdzjdxlhv` int DEFAULT NULL,\n `shnwicbbqjfayhvdbtxqeoruwylhvoqk` int DEFAULT NULL,\n `kgycayoxgjrdcmsixhvjcrsrdmlywjsd` int DEFAULT NULL,\n `nnmqhiabcmnswvpzdtrjzavfgapzzsde` int DEFAULT NULL,\n `ehrkjxtlwecrmicfhgdnkigoegcurbwp` int DEFAULT NULL,\n `uuitwriajhtllaaadqrohvtkibeqmvps` int DEFAULT NULL,\n `tuwnanwlvqzenyfxniedglwfkeayxbga` int DEFAULT NULL,\n `tujqcufoeqksrzumhfcflcotiznvdaiy` int DEFAULT NULL,\n `xrdehlnmvxkipsgspjvjptnndmdinrjl` int DEFAULT NULL,\n `swwyxzuxgawdcinugsefbqbrsvoskdyx` int DEFAULT NULL,\n `bwzvswjsbrmzdwculnzpgthlayegmeih` int DEFAULT NULL,\n `xpwoyostazwfxqugbreezntlkzritqrv` int DEFAULT NULL,\n `oydfwazfwziqjmyybmobwvgivxliwiyo` int DEFAULT NULL,\n `kvquiuyezclypmfpppqsscpktuqhaydr` int DEFAULT NULL,\n `fmouexceohiebjfoaihfvdzvqvwysukw` int DEFAULT NULL,\n `jlailoqeinbuioyonsyetwgyqeqvgoor` int DEFAULT NULL,\n `ppnxwkqampxjyvhymeznklhxfsdxudmv` int DEFAULT NULL,\n `znnznjpotbewdvdcxfvdouvgatygqgvu` int DEFAULT NULL,\n `uzheuvurjxmyrztvikxbkmvojvopgcuj` int DEFAULT NULL,\n `qmhughwniwocsydpibkdikcnfhdwqzsj` int DEFAULT NULL,\n `eshspchslyombvgcfdwekxtolouxmvdm` int DEFAULT NULL,\n `lfvkivgysnnjkwkegcgqfckgontyjidf` int DEFAULT NULL,\n `dodlsvcrbilvcrukgjibmtkhlivlaovw` int DEFAULT NULL,\n `bvlzmadrslizpqwlqifdnoypepyefaba` int DEFAULT NULL,\n `eaqphaedfpxxiekelwfgtwkehesxxgxg` int DEFAULT NULL,\n `wyvmdnxivwoooxsdnmzrazpmwnuieufk` int DEFAULT NULL,\n `swseypbraalvmlyxpccqoulcgjzbkoeq` int DEFAULT NULL,\n `yocanxstzwqfgbsyffsqxgsbggifmjhj` int DEFAULT NULL,\n `ueodfnufohbhjhuxutephkxixbimjxwm` int DEFAULT NULL,\n `facmnnmwjxhpzlapdgrozgvffuiethkn` int DEFAULT NULL,\n `tdphvyaiqzmhiqehxavfeqavcdsmpvie` int DEFAULT NULL,\n `ryxsmuutvlwnecfbdyzwhngmskjgqlrq` int DEFAULT NULL,\n `zqyytizflhigacklhrrmsczjqahxcxxe` int DEFAULT NULL,\n `vjqavsqyhwtvnmtgkjuomwqbaccgmbwq` int DEFAULT NULL,\n `bucqltdssclcjntsszifguitdxlcqegm` int DEFAULT NULL,\n `fzjsjryxjjkgijxnawomlfxifwbvqtef` int DEFAULT NULL,\n `ygtnxjppzmzobpfzmkhpqtigabvkxbla` int DEFAULT NULL,\n `uxeqpobemkpoamtxbtnbkcbevwvvfoqg` int DEFAULT NULL,\n `tlzvcdlrwejrravoppxqzptkmenietem` int DEFAULT NULL,\n `psbgmynmcmrenixrmfxiuljlpcjdxvir` int DEFAULT NULL,\n `ofqlbrhbryunyvhpszljwpbjfkeciwox` int DEFAULT NULL,\n `tjmijsipiczuivvavprsydvqehveqtet` int DEFAULT NULL,\n `gjalwkxspasndtubrvqgxwtgleiqngym` int DEFAULT NULL,\n `tuuewhvsagisryyoxkickmarsksjgwkz` int DEFAULT NULL,\n `uuvdugnqncbivpdgdxjcfiyqcaaehozv` int DEFAULT NULL,\n `pznupfxwpmqtaqhqzrodgpvjfecbsrvm` int DEFAULT NULL,\n `tkbrtkzjszuuqghxalplqbyzoyhtemdo` int DEFAULT NULL,\n `njfckhxxlzhptliiieagancxdukqbglc` int DEFAULT NULL,\n `lywkjujpedjqokjndvzewjrqpnplicyt` int DEFAULT NULL,\n `uiecsxhpocwfizapcyvvrgcgsbewykqh` int DEFAULT NULL,\n `mmzoxtwbrdhxvbpbotjecxvfuvymvmzv` int DEFAULT NULL,\n `kaerragwutsvdkkoylumrwymrpepckik` int DEFAULT NULL,\n `lyepluxztxcgfresjojpuauenicwoqkt` int DEFAULT NULL,\n `hacutabvhcijhglhijfmbuknsnlflpni` int DEFAULT NULL,\n `nsehfjhitazeaehnoiwlqmcpjhqplsnm` int DEFAULT NULL,\n `oomhbvmodgyshncunktjjgknvqkmsyht` int DEFAULT NULL,\n `blwkfscrkutsjludfzxamakvirrbmgel` int DEFAULT NULL,\n `hzmrcfesvlvofrzglzvelbkxrifixsyh` int DEFAULT NULL,\n `aigoxiwklncvjzhfxauuvjqihatynwif` int DEFAULT NULL,\n `iyicwqbhxtzmnobdrwgbpublzsbpiiql` int DEFAULT NULL,\n `zlzehpgvwjdjghtnscwbguiyyatvgirh` int DEFAULT NULL,\n PRIMARY KEY (`isfuznhnzgraumzforeffkggjcoirkwa`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"ygbhxqnxduzibovvetyxidhqtnuizvhi\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["isfuznhnzgraumzforeffkggjcoirkwa"],"columns":[{"name":"isfuznhnzgraumzforeffkggjcoirkwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"sydcmumjtnsfcneqleczrbikxktvkxgk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wsooasrsvplaxdzybrrpufwzzxrnbzii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cudvycxulonhisqfjaybjbbdkjxrcjms","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prmephzvcriehigzyjdlkcfiouhwrtrq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clrzybhoajdurwiamsywedcvebdogeem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkxyhggiwfqyjztjegbimbibjayadrqy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akhehmwqtqahcjxepylxelexxsarqvxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgzzgufuaejlvfijhmyabgxgxktgkfql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcxonegnvplubvgccupgkxfaoomohyrv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmmcustnwsmfckqnhwkdctqnvmszcvnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vuysbnwsgjgonloqizzcsoxjrlaozorv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xttewemnojamjkkesdzzulizkixpinfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shvxopabgtrapibjyefxhfjkknwcyhzi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lljaipkzearewrbbuoiiklyrewndwwkz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vglztvqimrnyqyvoqihslpaiedagxfkm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkmtmegwnqpnhlghgxvhfybwelrtauuf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xugenhjqtthkqmczrtrjewwzpejyjiaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"isrjlxpdttsstvjxrhwmdzlolezxghvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrmzsckbgyfrcnmfmjxpxhvhzczhmrga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wpnmhiynhmbfewgsycocnuroarzmkzba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsvvigknordzufvfjkdfkzqxgpgawpcj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xoomxmvxkwolmpyxclvyuepleqwdttlb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrylsroknsmpopakyicvlzxblfsxreks","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjeipsjmehxfeugqkpjddcawsvcgpqsq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rkrtkqjffycybssnulgpzieejqzairhg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zstlcfvujsfwybbqtpcyozpqfnjzqtrw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykriofhfmgivyqeatxptcwloxtyjokry","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wkybopxwkkcpviheoheqtuoioooktiuh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbdtkntpnmvrxxphcvqxccxdvrqislbd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ukdvkcotbhxmifhiwktkqaivhsqqjsew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvyemfubusuxcygtkwqbydbkszhduift","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivqowfcjwbdphvlwowczplpifutbqmyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhenfxppwxxppndsqqfomhpjxmbmrnad","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhgsmdbyejazgtharjdrhdtypjrhosam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfgsezhspnwxsdpsrmmlgiavrpdwzxtm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pzdopdmepkritqssuqmgytoofubdjgdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvaahrexsscnmifoqobqghatpmmqefqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hnxqsrljqojgtreutszuobskqisbcdkn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhimryvmyeglkehyrwwwxwqmdzjdxlhv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"shnwicbbqjfayhvdbtxqeoruwylhvoqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kgycayoxgjrdcmsixhvjcrsrdmlywjsd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nnmqhiabcmnswvpzdtrjzavfgapzzsde","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehrkjxtlwecrmicfhgdnkigoegcurbwp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uuitwriajhtllaaadqrohvtkibeqmvps","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tuwnanwlvqzenyfxniedglwfkeayxbga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tujqcufoeqksrzumhfcflcotiznvdaiy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xrdehlnmvxkipsgspjvjptnndmdinrjl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swwyxzuxgawdcinugsefbqbrsvoskdyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bwzvswjsbrmzdwculnzpgthlayegmeih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpwoyostazwfxqugbreezntlkzritqrv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oydfwazfwziqjmyybmobwvgivxliwiyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvquiuyezclypmfpppqsscpktuqhaydr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmouexceohiebjfoaihfvdzvqvwysukw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlailoqeinbuioyonsyetwgyqeqvgoor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ppnxwkqampxjyvhymeznklhxfsdxudmv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znnznjpotbewdvdcxfvdouvgatygqgvu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uzheuvurjxmyrztvikxbkmvojvopgcuj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmhughwniwocsydpibkdikcnfhdwqzsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eshspchslyombvgcfdwekxtolouxmvdm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfvkivgysnnjkwkegcgqfckgontyjidf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dodlsvcrbilvcrukgjibmtkhlivlaovw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvlzmadrslizpqwlqifdnoypepyefaba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eaqphaedfpxxiekelwfgtwkehesxxgxg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyvmdnxivwoooxsdnmzrazpmwnuieufk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"swseypbraalvmlyxpccqoulcgjzbkoeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yocanxstzwqfgbsyffsqxgsbggifmjhj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ueodfnufohbhjhuxutephkxixbimjxwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"facmnnmwjxhpzlapdgrozgvffuiethkn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdphvyaiqzmhiqehxavfeqavcdsmpvie","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ryxsmuutvlwnecfbdyzwhngmskjgqlrq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqyytizflhigacklhrrmsczjqahxcxxe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjqavsqyhwtvnmtgkjuomwqbaccgmbwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bucqltdssclcjntsszifguitdxlcqegm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzjsjryxjjkgijxnawomlfxifwbvqtef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ygtnxjppzmzobpfzmkhpqtigabvkxbla","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxeqpobemkpoamtxbtnbkcbevwvvfoqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlzvcdlrwejrravoppxqzptkmenietem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psbgmynmcmrenixrmfxiuljlpcjdxvir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofqlbrhbryunyvhpszljwpbjfkeciwox","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjmijsipiczuivvavprsydvqehveqtet","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gjalwkxspasndtubrvqgxwtgleiqngym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tuuewhvsagisryyoxkickmarsksjgwkz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uuvdugnqncbivpdgdxjcfiyqcaaehozv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pznupfxwpmqtaqhqzrodgpvjfecbsrvm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkbrtkzjszuuqghxalplqbyzoyhtemdo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njfckhxxlzhptliiieagancxdukqbglc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lywkjujpedjqokjndvzewjrqpnplicyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uiecsxhpocwfizapcyvvrgcgsbewykqh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mmzoxtwbrdhxvbpbotjecxvfuvymvmzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kaerragwutsvdkkoylumrwymrpepckik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lyepluxztxcgfresjojpuauenicwoqkt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hacutabvhcijhglhijfmbuknsnlflpni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nsehfjhitazeaehnoiwlqmcpjhqplsnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oomhbvmodgyshncunktjjgknvqkmsyht","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blwkfscrkutsjludfzxamakvirrbmgel","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hzmrcfesvlvofrzglzvelbkxrifixsyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aigoxiwklncvjzhfxauuvjqihatynwif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iyicwqbhxtzmnobdrwgbpublzsbpiiql","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zlzehpgvwjdjghtnscwbguiyyatvgirh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842672,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842672992,"databaseName":"models_schema","ddl":"CREATE TABLE `ynchpzshvnthrfykxjbnnndevivsthen` (\n `hxxggtbbzacuciupgovsksmwjdfubwwd` int NOT NULL,\n `kmxztddbbzoeafzwyhqmzxqgbmgzgkds` int DEFAULT NULL,\n `pfdrnczczbzbsmvglvxygtptrudbmiss` int DEFAULT NULL,\n `gujuygchkqkbkljlsrufnghpxdufyuhl` int DEFAULT NULL,\n `fkgkvfmujgwfvopfkjimthdspkcpfolh` int DEFAULT NULL,\n `dbwsrvkcorgbhzwrbpkmsfrmdtbkopbk` int DEFAULT NULL,\n `glxnpxukmghumggcxwjebcefjqehlerj` int DEFAULT NULL,\n `jxtkrgaohaitkzsrsbuejsqnlqsqhmgu` int DEFAULT NULL,\n `liotkbczqwmtxbpavostuhheajqiqryu` int DEFAULT NULL,\n `wfhfqzcipywkkapwwolnjndbroihwdso` int DEFAULT NULL,\n `oblormbfyexclptkhxxtkjwipmtgiejw` int DEFAULT NULL,\n `mjfxxfffowjamlatliymramesbfajgba` int DEFAULT NULL,\n `mukfoefmtkcemylvpaertjeocvxqnict` int DEFAULT NULL,\n `kxlgochfissvrayaxtgxspjhbakcfnfr` int DEFAULT NULL,\n `zuiiiurnuriptaeyvdiaployuqihlorz` int DEFAULT NULL,\n `kvfoqaqsvzgksktbqpojkqwtorlpozwi` int DEFAULT NULL,\n `kizojvstfsovcnileldsmshqsgedtvvl` int DEFAULT NULL,\n `ojhtgqjbrdesktlqzbxqpxubytcnlkrp` int DEFAULT NULL,\n `povaltsobchuoormbdhctvmeoqeennfc` int DEFAULT NULL,\n `gfsvhacfefdvzyuvhzvfxgotxbgkmdpy` int DEFAULT NULL,\n `dorblhtildxxrwcuycwhlanqonftgxdl` int DEFAULT NULL,\n `tpiulkmyryzkjjuygelnzkwjcogzzrzx` int DEFAULT NULL,\n `zxfcvemtqxchfbhwlftucghfsjmojcpg` int DEFAULT NULL,\n `vepcbaolchjgykbcgffbhcbtcrkaxdpp` int DEFAULT NULL,\n `qotucdpmelqomijolvjpdkgzqmvwwkdp` int DEFAULT NULL,\n `kzasojaypuymvkleqyyhfeogaazyyhih` int DEFAULT NULL,\n `pcsuwdoqlagnsvghjdhxkorhkonxbapw` int DEFAULT NULL,\n `vtambfdoamtfdmvrhubfqhopucfizfvo` int DEFAULT NULL,\n `dqfoewycwnnbkqtweqhllylbtldprgcv` int DEFAULT NULL,\n `uryaccowupwwuospsgaokpbjkasxedct` int DEFAULT NULL,\n `xtcevakozrrucvgzwisykcpuqkirivem` int DEFAULT NULL,\n `pooilytepxhbvmvhqlqxiwzhtbommuvk` int DEFAULT NULL,\n `gvaufzozsugipliceweollmeetrniwxn` int DEFAULT NULL,\n `zlfdzzgkjktiolbhhtpiqsryqjeopmwn` int DEFAULT NULL,\n `jzantvawvkzerodzvsryaywmgvuhllpq` int DEFAULT NULL,\n `ckdjpcoyfmefsgezblvgahmmbkipncls` int DEFAULT NULL,\n `zeixeglzhwzxmrbwbiyjnjkbssdmpxis` int DEFAULT NULL,\n `pjhaokxihfmlqvnhcdhtdkugiomuvgqp` int DEFAULT NULL,\n `boxzlzkcpulhgndwldunqnlpwxlhqgyb` int DEFAULT NULL,\n `tvwipxandtejivpiedexfbtfupdifznq` int DEFAULT NULL,\n `razzvcaidoarvbifyqxiypnqiogsqwmv` int DEFAULT NULL,\n `itoehyljfyczjnryunvmlwhzbyyvgoly` int DEFAULT NULL,\n `prnaqsvokswzdbcwusllynjplmbfuswg` int DEFAULT NULL,\n `tganwabirohkznpgnexkzworeisrxvym` int DEFAULT NULL,\n `hkrkpqcaeroobllazgbiwtcpstkprlij` int DEFAULT NULL,\n `xjnmnsudjczfzyllgsukntcrxjrmvima` int DEFAULT NULL,\n `lvkqdomhxfohppfeqofyfpfmfvdcqllq` int DEFAULT NULL,\n `grsrhhfbhrmqecvtekselhzljbaqixqq` int DEFAULT NULL,\n `biwylmwvyancrtqecqhokhwdbuqdvkbv` int DEFAULT NULL,\n `vqeqrddudnpymujghgxjftgwfnqjhhxr` int DEFAULT NULL,\n `tibikfotaqdhnwssgztodtkunycxodpg` int DEFAULT NULL,\n `cmfsjvqsntktnmcxmlwiwprxmpatzrlv` int DEFAULT NULL,\n `hpbpkovtnllnnkyznuwpgegxxalwpsle` int DEFAULT NULL,\n `wrspqaobdymknnrjpoqqrezhkhfdrlzo` int DEFAULT NULL,\n `ttmskwqsbinellcrjqlswnfzjpicgrak` int DEFAULT NULL,\n `srgkzoasqzlfxnuruyucwxjzshmarocd` int DEFAULT NULL,\n `ijxszfnhwvquewftrysxslvfyxjpohvn` int DEFAULT NULL,\n `znndrdlscwuxwmvhxeqstonsjswgavzl` int DEFAULT NULL,\n `kagehizwzzokjujifzjrasubymktwyiq` int DEFAULT NULL,\n `yxktglbrezjeuvkcaldzsudmduumciid` int DEFAULT NULL,\n `nugcbtmjhwkjxqdlqlvbnetoadckpqxh` int DEFAULT NULL,\n `uqryssmnrmhuzalpaskfbsqvoksmtitg` int DEFAULT NULL,\n `bwigeijlbcerbkycycrcdzqctoenngzp` int DEFAULT NULL,\n `hhvwbwauyfhxfirssmxymrtjqsdblyoy` int DEFAULT NULL,\n `txokmvjmqcbmstwwxoumixjtnujyogby` int DEFAULT NULL,\n `qzvxrfnndkxwubjcmuibihaimayiwfes` int DEFAULT NULL,\n `zzsxpiqsipofecwezxvohbdsefadqnxv` int DEFAULT NULL,\n `jmmlyykiguznimtrdrsxbkpftjzynqrp` int DEFAULT NULL,\n `jmjictkjehssxzldllqegnlhxqfurprj` int DEFAULT NULL,\n `qctivtiukeozmducaymhaprnwhafwlso` int DEFAULT NULL,\n `yrzavniqhtekpikjhyqnaaedtqqfstux` int DEFAULT NULL,\n `mcpwahsrtbdiscccimbtptfaqgqhrhsw` int DEFAULT NULL,\n `cgvutvgjffbfyknmjifplvcznieaifyj` int DEFAULT NULL,\n `hlyrkvrcpfwjelbohjevqoevspyfuzha` int DEFAULT NULL,\n `usutvvynpskslbmvvdfafzbvszwgyxbi` int DEFAULT NULL,\n `fjvmcwytkvrpcgvkqmaxdmazattrhhnk` int DEFAULT NULL,\n `vkxibvhhxjrfqncnfbgayarraxwdmpje` int DEFAULT NULL,\n `uoftgrquclotxpbimpnnnwbkxacbuxml` int DEFAULT NULL,\n `fkqaurifkqtpnqeyywnufsynhwapnlqu` int DEFAULT NULL,\n `emhoacgcgjzkiwmxorebicjexxvtljvw` int DEFAULT NULL,\n `ppahvlgznkzcytkwxrzrxfasfjmqsovb` int DEFAULT NULL,\n `vxptcidxdmqsqnwjzsplcxsyergbnhsv` int DEFAULT NULL,\n `ibaqdheblxjxpouwaaibxhhjpfguvypp` int DEFAULT NULL,\n `qmoefgdtifsiqoirftmfmkehzfqsbvab` int DEFAULT NULL,\n `rhwkajxklollxypmctjnypgqkljiodgs` int DEFAULT NULL,\n `reionryxlzfbyvbxeiusppnwyhgqufkb` int DEFAULT NULL,\n `wkdmpiqqxyfmgevgjbbqkxoyfcnxngbj` int DEFAULT NULL,\n `idmmwiroqyudcmqyzhqloykjtjvrragu` int DEFAULT NULL,\n `pjbrbgmvubhtekaqjntjhfusrubvampo` int DEFAULT NULL,\n `jhluitwpmhghrqalelqqqcqjkmivwpqp` int DEFAULT NULL,\n `venzhcvkisgkidkdsnmhydrfplyctism` int DEFAULT NULL,\n `nveolhwwicqhavvdsozsvzdcqtzbdsgw` int DEFAULT NULL,\n `nofzbddrzefolihncmwpwubvsbvtisar` int DEFAULT NULL,\n `pkpdrtqrljgpkvdowahlpzrxzphmdtbt` int DEFAULT NULL,\n `uibsihoybjezzwnfnmguqayvthetqvcs` int DEFAULT NULL,\n `zuimdfauxfqthgcoftickobsrkbjinin` int DEFAULT NULL,\n `blolfupnxewsststumahbazilojiwdyn` int DEFAULT NULL,\n `hjukidgkwfiokbrqccoypbczewlrbipd` int DEFAULT NULL,\n `afdpuxopymlwshrejbzmkfpucdvofvwe` int DEFAULT NULL,\n `yvlcxxgsxdzkwevhwhxytagwpqfhvyiu` int DEFAULT NULL,\n PRIMARY KEY (`hxxggtbbzacuciupgovsksmwjdfubwwd`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"ynchpzshvnthrfykxjbnnndevivsthen\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["hxxggtbbzacuciupgovsksmwjdfubwwd"],"columns":[{"name":"hxxggtbbzacuciupgovsksmwjdfubwwd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"kmxztddbbzoeafzwyhqmzxqgbmgzgkds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pfdrnczczbzbsmvglvxygtptrudbmiss","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gujuygchkqkbkljlsrufnghpxdufyuhl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fkgkvfmujgwfvopfkjimthdspkcpfolh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbwsrvkcorgbhzwrbpkmsfrmdtbkopbk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"glxnpxukmghumggcxwjebcefjqehlerj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jxtkrgaohaitkzsrsbuejsqnlqsqhmgu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"liotkbczqwmtxbpavostuhheajqiqryu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfhfqzcipywkkapwwolnjndbroihwdso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oblormbfyexclptkhxxtkjwipmtgiejw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mjfxxfffowjamlatliymramesbfajgba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mukfoefmtkcemylvpaertjeocvxqnict","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxlgochfissvrayaxtgxspjhbakcfnfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zuiiiurnuriptaeyvdiaployuqihlorz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvfoqaqsvzgksktbqpojkqwtorlpozwi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kizojvstfsovcnileldsmshqsgedtvvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ojhtgqjbrdesktlqzbxqpxubytcnlkrp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"povaltsobchuoormbdhctvmeoqeennfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfsvhacfefdvzyuvhzvfxgotxbgkmdpy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dorblhtildxxrwcuycwhlanqonftgxdl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tpiulkmyryzkjjuygelnzkwjcogzzrzx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxfcvemtqxchfbhwlftucghfsjmojcpg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vepcbaolchjgykbcgffbhcbtcrkaxdpp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qotucdpmelqomijolvjpdkgzqmvwwkdp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzasojaypuymvkleqyyhfeogaazyyhih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pcsuwdoqlagnsvghjdhxkorhkonxbapw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtambfdoamtfdmvrhubfqhopucfizfvo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqfoewycwnnbkqtweqhllylbtldprgcv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uryaccowupwwuospsgaokpbjkasxedct","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xtcevakozrrucvgzwisykcpuqkirivem","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pooilytepxhbvmvhqlqxiwzhtbommuvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gvaufzozsugipliceweollmeetrniwxn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zlfdzzgkjktiolbhhtpiqsryqjeopmwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jzantvawvkzerodzvsryaywmgvuhllpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ckdjpcoyfmefsgezblvgahmmbkipncls","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zeixeglzhwzxmrbwbiyjnjkbssdmpxis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjhaokxihfmlqvnhcdhtdkugiomuvgqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"boxzlzkcpulhgndwldunqnlpwxlhqgyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvwipxandtejivpiedexfbtfupdifznq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"razzvcaidoarvbifyqxiypnqiogsqwmv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"itoehyljfyczjnryunvmlwhzbyyvgoly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"prnaqsvokswzdbcwusllynjplmbfuswg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tganwabirohkznpgnexkzworeisrxvym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkrkpqcaeroobllazgbiwtcpstkprlij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjnmnsudjczfzyllgsukntcrxjrmvima","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lvkqdomhxfohppfeqofyfpfmfvdcqllq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grsrhhfbhrmqecvtekselhzljbaqixqq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"biwylmwvyancrtqecqhokhwdbuqdvkbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqeqrddudnpymujghgxjftgwfnqjhhxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tibikfotaqdhnwssgztodtkunycxodpg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cmfsjvqsntktnmcxmlwiwprxmpatzrlv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hpbpkovtnllnnkyznuwpgegxxalwpsle","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wrspqaobdymknnrjpoqqrezhkhfdrlzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ttmskwqsbinellcrjqlswnfzjpicgrak","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"srgkzoasqzlfxnuruyucwxjzshmarocd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijxszfnhwvquewftrysxslvfyxjpohvn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znndrdlscwuxwmvhxeqstonsjswgavzl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kagehizwzzokjujifzjrasubymktwyiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxktglbrezjeuvkcaldzsudmduumciid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nugcbtmjhwkjxqdlqlvbnetoadckpqxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqryssmnrmhuzalpaskfbsqvoksmtitg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bwigeijlbcerbkycycrcdzqctoenngzp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhvwbwauyfhxfirssmxymrtjqsdblyoy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txokmvjmqcbmstwwxoumixjtnujyogby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qzvxrfnndkxwubjcmuibihaimayiwfes","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzsxpiqsipofecwezxvohbdsefadqnxv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmmlyykiguznimtrdrsxbkpftjzynqrp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmjictkjehssxzldllqegnlhxqfurprj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qctivtiukeozmducaymhaprnwhafwlso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yrzavniqhtekpikjhyqnaaedtqqfstux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcpwahsrtbdiscccimbtptfaqgqhrhsw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cgvutvgjffbfyknmjifplvcznieaifyj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlyrkvrcpfwjelbohjevqoevspyfuzha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"usutvvynpskslbmvvdfafzbvszwgyxbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjvmcwytkvrpcgvkqmaxdmazattrhhnk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkxibvhhxjrfqncnfbgayarraxwdmpje","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uoftgrquclotxpbimpnnnwbkxacbuxml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fkqaurifkqtpnqeyywnufsynhwapnlqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"emhoacgcgjzkiwmxorebicjexxvtljvw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ppahvlgznkzcytkwxrzrxfasfjmqsovb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxptcidxdmqsqnwjzsplcxsyergbnhsv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibaqdheblxjxpouwaaibxhhjpfguvypp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmoefgdtifsiqoirftmfmkehzfqsbvab","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhwkajxklollxypmctjnypgqkljiodgs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"reionryxlzfbyvbxeiusppnwyhgqufkb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wkdmpiqqxyfmgevgjbbqkxoyfcnxngbj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idmmwiroqyudcmqyzhqloykjtjvrragu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjbrbgmvubhtekaqjntjhfusrubvampo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jhluitwpmhghrqalelqqqcqjkmivwpqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"venzhcvkisgkidkdsnmhydrfplyctism","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nveolhwwicqhavvdsozsvzdcqtzbdsgw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nofzbddrzefolihncmwpwubvsbvtisar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pkpdrtqrljgpkvdowahlpzrxzphmdtbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uibsihoybjezzwnfnmguqayvthetqvcs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zuimdfauxfqthgcoftickobsrkbjinin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blolfupnxewsststumahbazilojiwdyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjukidgkwfiokbrqccoypbczewlrbipd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afdpuxopymlwshrejbzmkfpucdvofvwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yvlcxxgsxdzkwevhwhxytagwpqfhvyiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842673,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842673029,"databaseName":"models_schema","ddl":"CREATE TABLE `zcemsecafryitcsvmgpbhebgzwmycaiu` (\n `elbfdhhmprmbqumexljdeuhxkbssitxu` int NOT NULL,\n `rtiltnwcreecorzuelqgtepasvlpsjqa` int DEFAULT NULL,\n `jdsgsrydttrnoffbikflzuxmnldwzdax` int DEFAULT NULL,\n `jiiluepoklhunsoqlotsysgymoezareb` int DEFAULT NULL,\n `fvbfmgmjynjmrxlbzyowtootbfyuznoi` int DEFAULT NULL,\n `esirwlgzntynmgmwdtrbszyfmodxlzsj` int DEFAULT NULL,\n `ebbifkdmcpkcnzxerguncxjdrsnfrrur` int DEFAULT NULL,\n `afnzdufpxsfrmjrlrmiycuaznphskinc` int DEFAULT NULL,\n `ayrpxmrmixhkfxciwwvwlxdflyokpfnw` int DEFAULT NULL,\n `lfgxkgyvzrzhxiicrwkgnxatqznclgex` int DEFAULT NULL,\n `puyhxgizwrjdznhrieyacyvxlefvyica` int DEFAULT NULL,\n `tmacurbvqvsgwncnrlfjtdvfyrufsvzr` int DEFAULT NULL,\n `lnftfuoklfsjhqshysdhmibdpdowdpmj` int DEFAULT NULL,\n `tdyfhwqdarnjxusjkmsqjskfsqbcqljg` int DEFAULT NULL,\n `efzawrndaurrwslaztiudhmxvelhuehj` int DEFAULT NULL,\n `zcgzdyhawvjnzguhnyftkstnkzmykcdv` int DEFAULT NULL,\n `jfazmshwvzhqytljlulcyanllyigezjj` int DEFAULT NULL,\n `tkfjcxjfhyadiitkwnswurupttkqhwmj` int DEFAULT NULL,\n `gpqgpvrudqawenqnickgelrrbvoqhsiu` int DEFAULT NULL,\n `kctaqlvnvadpofnrjfcfntbhrwfqfxkr` int DEFAULT NULL,\n `sekyhjkfjaybeujnbvqipjkuxrevmvex` int DEFAULT NULL,\n `ldsdsfwspnpuxzidyysjuzxjbfxzsyfh` int DEFAULT NULL,\n `fgdpsdqitarjtziajnvmjzwhfbvlheiq` int DEFAULT NULL,\n `jsivhxudoinneycuytgmuunbcdybdjdd` int DEFAULT NULL,\n `nrzxxwbdcztaunwtkrmtcfimcjiexzug` int DEFAULT NULL,\n `ajajkxiovcfgqjpcncomjxwkyxjpaibx` int DEFAULT NULL,\n `hjirymytqqpwjzflzjivojmshyjjgwzk` int DEFAULT NULL,\n `cqmsjakkhkghsqndrhdhwgdlhjxgzjtd` int DEFAULT NULL,\n `kegvjwrwwgqgrpxqscdlyzavbaofhroo` int DEFAULT NULL,\n `vjccqpbwllxyoutwmiodopstxwyyetgl` int DEFAULT NULL,\n `djunvbbcerqjmypjlllmnjoxmkxbxvif` int DEFAULT NULL,\n `goftswwctxfgirszyorbzryszigzxgya` int DEFAULT NULL,\n `vtqkufyyemqvqqcyrxpkcguvzgjbkaok` int DEFAULT NULL,\n `fspxvhlsofwjgsbubcjbseuzyopwtafy` int DEFAULT NULL,\n `epdhdoytrprzgceyvbrahbzveironpax` int DEFAULT NULL,\n `bzrwxkegkzrzuqlprhrujfxzxekwwiey` int DEFAULT NULL,\n `zitwglzkrutmxykrsklhsucxenzanogc` int DEFAULT NULL,\n `pnyjoaygqutdwhzlkumximkdehnsyufe` int DEFAULT NULL,\n `fsctosoorrpbbdaielcifixfdmebprgl` int DEFAULT NULL,\n `dtwvouenwjyvlnoeivdgrcioowzdnedy` int DEFAULT NULL,\n `nzmdakndzqgorseoxrffvpcjvehsnhtt` int DEFAULT NULL,\n `qdzbvtlpdonfiuozzlhppuyortbrjjcp` int DEFAULT NULL,\n `tfqdxhayhtulcztwamezdqltkdfpaidn` int DEFAULT NULL,\n `rscbkemaiwpuyxuyrhctwghjpvbvmnpq` int DEFAULT NULL,\n `wwmihindaisscockicnwtihvtoccbkpt` int DEFAULT NULL,\n `avlfeduryndixwqdvqqggkjbeuvmmqqw` int DEFAULT NULL,\n `ekmtowgeaefebrgnjtqgvbamqhtjeatp` int DEFAULT NULL,\n `lsvqopruwshqxyikkwloptpteosfurny` int DEFAULT NULL,\n `tqrqtdmxezivfkggjvwuyqyvupfwpljz` int DEFAULT NULL,\n `bpcrpcpqesbgjrrcdipqciwuvuubpjjg` int DEFAULT NULL,\n `tuquhvmvjyoydquqaggaypvpkcjpexlu` int DEFAULT NULL,\n `utlbjolkjuahhuwpkqnzgtqzthwlqwgt` int DEFAULT NULL,\n `kaptyniosaewqxpkybuqqxzkxmhsdazm` int DEFAULT NULL,\n `zuinvljjzjhotkbztjkdimvytxxoblbv` int DEFAULT NULL,\n `vgbleusoeemcndrkpcwdimsmxgnvhobg` int DEFAULT NULL,\n `gftvpjvfwwyqvwdkfsxlhrxwcvlnnygy` int DEFAULT NULL,\n `hlssnvyjimrvxsnuwnrbsxlkthxzvrpj` int DEFAULT NULL,\n `aqmhupoflhdrjgcbbiilfprhoysgcflq` int DEFAULT NULL,\n `vcovsvfbzzzqgrqqipprodzbmraundlg` int DEFAULT NULL,\n `bnocrkquqwnlcvegelwwbvxcgsqirxtq` int DEFAULT NULL,\n `wufaenyvubgpiffxdjoamclowqhvgrdc` int DEFAULT NULL,\n `zaskmepgaermcsmkuxqsdmkjvvpcdhka` int DEFAULT NULL,\n `nocfhqgialfzkyvqmotihfcqllkdxozj` int DEFAULT NULL,\n `fangaffvdyzwjnmptnvsqtyetxzboekk` int DEFAULT NULL,\n `frfiigpqyfzvhpiumkfjcxvkaoryfbnq` int DEFAULT NULL,\n `cjjbktjqdgfefkdnvdojdauqcxdedszv` int DEFAULT NULL,\n `lwjgfukjchycvrqtbbwzffccqmkwzpbr` int DEFAULT NULL,\n `eamdjfzzmjbsjfesuufllwgphtpealpj` int DEFAULT NULL,\n `ofggmbtlwnrtluyzubqlldqcksxvuvzn` int DEFAULT NULL,\n `kuevkmctqskctbnkzlpynfqkjhxnropa` int DEFAULT NULL,\n `sfpyzqpdtpamylxvradfvhazgfaovood` int DEFAULT NULL,\n `xjhtspkkjhypcikbmpkzewpiezjompjr` int DEFAULT NULL,\n `vgljucnubahulznslzlufymusunqjjpf` int DEFAULT NULL,\n `npmgxykntvzjayxzavrtuyhfwkvrnsgn` int DEFAULT NULL,\n `erfroualpwggextomognokcstfaaocnd` int DEFAULT NULL,\n `cqgbyoyebmmoioulmxsqshtziwnxnryi` int DEFAULT NULL,\n `dwjxicjkagsyhngmmrhvgmlklqueqaup` int DEFAULT NULL,\n `rpxzpeozdrebevmyjdtrycwvciucayxu` int DEFAULT NULL,\n `lsqvquogsywewiqiyzevzkyrjbfykscy` int DEFAULT NULL,\n `uxgrctgigslejaeasegpxojabrkdfpho` int DEFAULT NULL,\n `qqxzxmvltbjzswjgxxcosyxzixvgtdlv` int DEFAULT NULL,\n `ilztglnrrukucpdftrwlyhvbcmwqyukd` int DEFAULT NULL,\n `auoyiggfpunhjiacedlupapmnuilphxp` int DEFAULT NULL,\n `fzavazqmligxzixnjgwddukdpyzzosug` int DEFAULT NULL,\n `ttjssnusgvigjxrjbpofncoeuswwqtri` int DEFAULT NULL,\n `qkybxuoferavnvtdptnyprvaorbcpsqi` int DEFAULT NULL,\n `iowpdtljqpwwpppqnfqgpecatdayfwmx` int DEFAULT NULL,\n `ewinijicymuxxiuvnullseldjrkhvrxh` int DEFAULT NULL,\n `zplyqcheypzrirmbiersvwqjoilhtbmk` int DEFAULT NULL,\n `hbsilgitodacmsrguyskaauepnatdasn` int DEFAULT NULL,\n `wsictuwqsawjulkglulzlyzzakenvfzj` int DEFAULT NULL,\n `lutcthykddpeqqmquaexgrpwtazhethg` int DEFAULT NULL,\n `ahzkqljrnhssiphxtypbipzegvbxdyxf` int DEFAULT NULL,\n `cpubiikmlmkqusjdzxfkwxqwqzfrssmu` int DEFAULT NULL,\n `vupfeziersykotvdgezcgpprgmrwvrrh` int DEFAULT NULL,\n `wtxdhgcxezwtycdnjucrnkjzhbidorlh` int DEFAULT NULL,\n `lhpfmgmdltnqjxeszysowofbtgsptxuk` int DEFAULT NULL,\n `lfficlpmpjrukvxcrntwhwitkiyuxxxo` int DEFAULT NULL,\n `bpeohpwrnappvrorojrzstmzitwihoep` int DEFAULT NULL,\n `hfbobhaxejbrwiupljmneowaxcwxrbdw` int DEFAULT NULL,\n PRIMARY KEY (`elbfdhhmprmbqumexljdeuhxkbssitxu`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"zcemsecafryitcsvmgpbhebgzwmycaiu\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["elbfdhhmprmbqumexljdeuhxkbssitxu"],"columns":[{"name":"elbfdhhmprmbqumexljdeuhxkbssitxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"rtiltnwcreecorzuelqgtepasvlpsjqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdsgsrydttrnoffbikflzuxmnldwzdax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jiiluepoklhunsoqlotsysgymoezareb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fvbfmgmjynjmrxlbzyowtootbfyuznoi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"esirwlgzntynmgmwdtrbszyfmodxlzsj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebbifkdmcpkcnzxerguncxjdrsnfrrur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afnzdufpxsfrmjrlrmiycuaznphskinc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ayrpxmrmixhkfxciwwvwlxdflyokpfnw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfgxkgyvzrzhxiicrwkgnxatqznclgex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"puyhxgizwrjdznhrieyacyvxlefvyica","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tmacurbvqvsgwncnrlfjtdvfyrufsvzr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lnftfuoklfsjhqshysdhmibdpdowdpmj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdyfhwqdarnjxusjkmsqjskfsqbcqljg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"efzawrndaurrwslaztiudhmxvelhuehj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zcgzdyhawvjnzguhnyftkstnkzmykcdv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jfazmshwvzhqytljlulcyanllyigezjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tkfjcxjfhyadiitkwnswurupttkqhwmj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gpqgpvrudqawenqnickgelrrbvoqhsiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kctaqlvnvadpofnrjfcfntbhrwfqfxkr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sekyhjkfjaybeujnbvqipjkuxrevmvex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ldsdsfwspnpuxzidyysjuzxjbfxzsyfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fgdpsdqitarjtziajnvmjzwhfbvlheiq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jsivhxudoinneycuytgmuunbcdybdjdd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nrzxxwbdcztaunwtkrmtcfimcjiexzug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajajkxiovcfgqjpcncomjxwkyxjpaibx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjirymytqqpwjzflzjivojmshyjjgwzk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqmsjakkhkghsqndrhdhwgdlhjxgzjtd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kegvjwrwwgqgrpxqscdlyzavbaofhroo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjccqpbwllxyoutwmiodopstxwyyetgl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"djunvbbcerqjmypjlllmnjoxmkxbxvif","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"goftswwctxfgirszyorbzryszigzxgya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtqkufyyemqvqqcyrxpkcguvzgjbkaok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fspxvhlsofwjgsbubcjbseuzyopwtafy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"epdhdoytrprzgceyvbrahbzveironpax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzrwxkegkzrzuqlprhrujfxzxekwwiey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zitwglzkrutmxykrsklhsucxenzanogc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pnyjoaygqutdwhzlkumximkdehnsyufe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fsctosoorrpbbdaielcifixfdmebprgl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtwvouenwjyvlnoeivdgrcioowzdnedy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nzmdakndzqgorseoxrffvpcjvehsnhtt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdzbvtlpdonfiuozzlhppuyortbrjjcp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tfqdxhayhtulcztwamezdqltkdfpaidn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rscbkemaiwpuyxuyrhctwghjpvbvmnpq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwmihindaisscockicnwtihvtoccbkpt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avlfeduryndixwqdvqqggkjbeuvmmqqw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekmtowgeaefebrgnjtqgvbamqhtjeatp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lsvqopruwshqxyikkwloptpteosfurny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tqrqtdmxezivfkggjvwuyqyvupfwpljz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bpcrpcpqesbgjrrcdipqciwuvuubpjjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tuquhvmvjyoydquqaggaypvpkcjpexlu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utlbjolkjuahhuwpkqnzgtqzthwlqwgt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kaptyniosaewqxpkybuqqxzkxmhsdazm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zuinvljjzjhotkbztjkdimvytxxoblbv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgbleusoeemcndrkpcwdimsmxgnvhobg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gftvpjvfwwyqvwdkfsxlhrxwcvlnnygy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlssnvyjimrvxsnuwnrbsxlkthxzvrpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqmhupoflhdrjgcbbiilfprhoysgcflq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vcovsvfbzzzqgrqqipprodzbmraundlg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnocrkquqwnlcvegelwwbvxcgsqirxtq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wufaenyvubgpiffxdjoamclowqhvgrdc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zaskmepgaermcsmkuxqsdmkjvvpcdhka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nocfhqgialfzkyvqmotihfcqllkdxozj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fangaffvdyzwjnmptnvsqtyetxzboekk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"frfiigpqyfzvhpiumkfjcxvkaoryfbnq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjjbktjqdgfefkdnvdojdauqcxdedszv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwjgfukjchycvrqtbbwzffccqmkwzpbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eamdjfzzmjbsjfesuufllwgphtpealpj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofggmbtlwnrtluyzubqlldqcksxvuvzn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kuevkmctqskctbnkzlpynfqkjhxnropa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sfpyzqpdtpamylxvradfvhazgfaovood","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xjhtspkkjhypcikbmpkzewpiezjompjr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgljucnubahulznslzlufymusunqjjpf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"npmgxykntvzjayxzavrtuyhfwkvrnsgn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"erfroualpwggextomognokcstfaaocnd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cqgbyoyebmmoioulmxsqshtziwnxnryi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwjxicjkagsyhngmmrhvgmlklqueqaup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpxzpeozdrebevmyjdtrycwvciucayxu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lsqvquogsywewiqiyzevzkyrjbfykscy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxgrctgigslejaeasegpxojabrkdfpho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqxzxmvltbjzswjgxxcosyxzixvgtdlv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ilztglnrrukucpdftrwlyhvbcmwqyukd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"auoyiggfpunhjiacedlupapmnuilphxp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzavazqmligxzixnjgwddukdpyzzosug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ttjssnusgvigjxrjbpofncoeuswwqtri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qkybxuoferavnvtdptnyprvaorbcpsqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iowpdtljqpwwpppqnfqgpecatdayfwmx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ewinijicymuxxiuvnullseldjrkhvrxh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zplyqcheypzrirmbiersvwqjoilhtbmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hbsilgitodacmsrguyskaauepnatdasn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wsictuwqsawjulkglulzlyzzakenvfzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lutcthykddpeqqmquaexgrpwtazhethg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahzkqljrnhssiphxtypbipzegvbxdyxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cpubiikmlmkqusjdzxfkwxqwqzfrssmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vupfeziersykotvdgezcgpprgmrwvrrh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtxdhgcxezwtycdnjucrnkjzhbidorlh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhpfmgmdltnqjxeszysowofbtgsptxuk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfficlpmpjrukvxcrntwhwitkiyuxxxo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bpeohpwrnappvrorojrzstmzitwihoep","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfbobhaxejbrwiupljmneowaxcwxrbdw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842673,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842673068,"databaseName":"models_schema","ddl":"CREATE TABLE `zfgsdjvymmttvoqeggdyoqbiudfvvjnu` (\n `icggjyszavlzpgpdwtgfjhjvgttkwlyo` int NOT NULL,\n `vkbaambjqrqhnldwgwbourdmiqxhueej` int DEFAULT NULL,\n `ikiontkkchkukfntjqyuuzpujaaatdtw` int DEFAULT NULL,\n `lfnelkbyxkxaubedgvsrzmxapxsddssg` int DEFAULT NULL,\n `jwengeqpczaegqpjrbrrhodnnhkxhjrp` int DEFAULT NULL,\n `paqgbyljydlkoaknrygniqtsnwbgyevj` int DEFAULT NULL,\n `ofbkgrqnpwwjbbqrpoqyrzcqcoiwxtvh` int DEFAULT NULL,\n `oiwdibbbswytvfxrrgjicmgkedgqmfuy` int DEFAULT NULL,\n `fupefddhcnjsgwritqfnvfwejqyyocvl` int DEFAULT NULL,\n `yntikvravudmljvfstafjpjpgehpxpkm` int DEFAULT NULL,\n `ncpnrdgmmhcphzvynexwpkjkzvvdrkhd` int DEFAULT NULL,\n `kvailtcrqffpeebyaeomrfchsxcwvexp` int DEFAULT NULL,\n `yxzgiinikurhnatyzwrhrpdxwjraiigl` int DEFAULT NULL,\n `jlaqxukuneenuiqqosooqqkrkvwtdoih` int DEFAULT NULL,\n `yavidqifxykcbdjycvaqklstjcxlmdgj` int DEFAULT NULL,\n `epwbpssciabmqaxkkrqwdqevphsihkja` int DEFAULT NULL,\n `jumylrokqwwjaczgvoobmfewjzvdsyuw` int DEFAULT NULL,\n `zbhfokboszbjcqrmbpnydhtazlmibgey` int DEFAULT NULL,\n `mkbejqdgoafimgodxuxcrcsglpfdyrar` int DEFAULT NULL,\n `zzctfxxlecvafofewgvqawmobadrouhd` int DEFAULT NULL,\n `moeonccsrzxcrqdelmcugisllcjptoor` int DEFAULT NULL,\n `wcimizpwjeckfwcvhdhphsbqggfcsyhr` int DEFAULT NULL,\n `umrdpmgawqfofqwrmggluuwzkjqdetsm` int DEFAULT NULL,\n `kfvlsrzpchnxsmwtujpuxfeszabfttye` int DEFAULT NULL,\n `pwtickxjgbhnwehsptxcwsjdmcidvhoo` int DEFAULT NULL,\n `lmqqmivetaajrethdemykoydflklbygn` int DEFAULT NULL,\n `zqrplswvfqvsnghwjhaytmhtkcghjdeg` int DEFAULT NULL,\n `fzvlflqtkyettyxxkzrgtfjyavxmctud` int DEFAULT NULL,\n `rgirhbzsqhqorcjosvvxbilzxevnnqtx` int DEFAULT NULL,\n `tbmyyvnaolfmbltkmnsrerwgmudirqhm` int DEFAULT NULL,\n `xyrpmrwgkotjdqdaaiyrzcusyhxubxzu` int DEFAULT NULL,\n `jitloswhbjkfxupeisrzfroyjdgkjheb` int DEFAULT NULL,\n `vhjxigbaxckxgszijundxncpczhmbobf` int DEFAULT NULL,\n `tlmnsquudndbowrryjaorszmaqpaeorr` int DEFAULT NULL,\n `fizzvlcbjwbrcnznhcbwmiotowfdvxwb` int DEFAULT NULL,\n `fvbbubwbrtjccfcngjbflsgnpqjiohya` int DEFAULT NULL,\n `phwjjaqzfvxhjkeczzwhlmphncawqbsp` int DEFAULT NULL,\n `arfqhmvffnjgzdfuvbcpesipitoirnyv` int DEFAULT NULL,\n `tjtfmlcuxvdtcjusijxebdoxgnzpahtu` int DEFAULT NULL,\n `vjuxlbpazqqblhgahhjwflmuynqliumr` int DEFAULT NULL,\n `ozqdxlcvdouycqvplmbytentskxuyhfk` int DEFAULT NULL,\n `wslinguirhyzcwvxcsibxlmircadmruk` int DEFAULT NULL,\n `ekxxubvjlxuntfiuqurjhyufacmxevgf` int DEFAULT NULL,\n `bxlgeixskzklbkwcvzgstudgjbbiquew` int DEFAULT NULL,\n `pwoafbosuzttgdpktssgrcxdbcfghoay` int DEFAULT NULL,\n `opmvxhykfwyjfkmcveeyheiypahkvohq` int DEFAULT NULL,\n `eiksrzctsdgdzwkwldofopjjxrzhhapd` int DEFAULT NULL,\n `hchfzbwsrxjdrgxkphauvgrwbvcsvafu` int DEFAULT NULL,\n `ounmkkvnlekickmlzuisvpvzmnoaruqn` int DEFAULT NULL,\n `wcajahmpqidfwfcsdwwqrargsieayelu` int DEFAULT NULL,\n `arrrdcdijikwoinbufoiltlesqkgaipj` int DEFAULT NULL,\n `fddhpuhhgyvikqsdpgdvjkknthyirvvl` int DEFAULT NULL,\n `clmpxrsdlbwlibnhjkwsfykioabckquu` int DEFAULT NULL,\n `frbydkrfxcjnmgttfaiuhdykwfbzckdz` int DEFAULT NULL,\n `wtatojyvudtycwaztfsisjpqrekqcity` int DEFAULT NULL,\n `ihxeuxdydmxotajaqviqythtcqwsawtq` int DEFAULT NULL,\n `fxarbnolxvexszwlwfwegayacxjvdorj` int DEFAULT NULL,\n `dbnfofssxafzrdageuouawklebrkphbc` int DEFAULT NULL,\n `gfohnrpfegjfygsulfzddatjexskjhyd` int DEFAULT NULL,\n `kilfedygkacvzrpobymgmqrgriilawcq` int DEFAULT NULL,\n `pbixuoiewitzgpuwdpcpvxoxlsmaymft` int DEFAULT NULL,\n `srpkrslwjhgdhqgfzjqdegeydqbypmhg` int DEFAULT NULL,\n `tsvshxdcaguwnlrbjyilvwvkiatnmhiy` int DEFAULT NULL,\n `lslkaeccusxvtypceqyazvvetjzivjeh` int DEFAULT NULL,\n `zmfmcrmbtxcwmlgbcakrnzaswncjnpfr` int DEFAULT NULL,\n `oxktcuxdxdesjzqucktsanecwujzwskd` int DEFAULT NULL,\n `fcfyaprektcgjfqatmihbtmcacaqkbws` int DEFAULT NULL,\n `ptpdrckksirtkkjlkywsmcahrdzrqtah` int DEFAULT NULL,\n `sqqzrkzczskjuudikannbzopsbtybmgb` int DEFAULT NULL,\n `ndnhmnfytutpomlarfyynegqbgmdgafa` int DEFAULT NULL,\n `bnzogejiiebnpduxxoybhpxnpiizhhsz` int DEFAULT NULL,\n `wwasascmwrhivrxaczldjshlwrajkvkl` int DEFAULT NULL,\n `lsltxlanystatjewmljvypyxcakkwgdb` int DEFAULT NULL,\n `pzfniasanrumlyplfhywvojvkpnfqizv` int DEFAULT NULL,\n `hwhhqqngcpoxorpvotmydhmaoozcnsip` int DEFAULT NULL,\n `ikjysgzdeubhcygyoflvcyqjthceueyv` int DEFAULT NULL,\n `lbqzvdoojwpjdeimwvytshsmvsjvqivd` int DEFAULT NULL,\n `hsvnfivgykckedfphopbjsibzbdvzqjo` int DEFAULT NULL,\n `htanmyohpeywayrqonycmmozlcpbmgrl` int DEFAULT NULL,\n `jcwvdgjznnejxzjiiwtceeauxbtymfni` int DEFAULT NULL,\n `utohmmlvtosabaebvhwvplzbcqzttnoc` int DEFAULT NULL,\n `exzzxrcikegcsxivezussvtutmkxjhpt` int DEFAULT NULL,\n `yostpgymizqrajkaxvyfugdvslsddjqf` int DEFAULT NULL,\n `yvpyrulflftjfmuqwcippanegbljyumf` int DEFAULT NULL,\n `tarcwgruawdeguepeneueqsambhvziyx` int DEFAULT NULL,\n `yqgvqedidcjlrqxzlogjzztbdfpvmytv` int DEFAULT NULL,\n `ktdrycudlpvyfbomhqocmswizugtiwys` int DEFAULT NULL,\n `smdrkjugvdbgqoygddmocbnelizuztui` int DEFAULT NULL,\n `ziaysfqrhnemvqiacenyjgycpqizgdhf` int DEFAULT NULL,\n `xqbwtqyylezxartsgmxyeumeivbqkuob` int DEFAULT NULL,\n `twqtermlthxghpcusfgvuxtxzqspiqye` int DEFAULT NULL,\n `wqvhjiskwdgvsglggnrcrxrnzrkmweta` int DEFAULT NULL,\n `edmwnzouwusbfbbzfpudymwstprkjmhk` int DEFAULT NULL,\n `rlehpkfmamcoykqrqnjfbjbtvlmhwqna` int DEFAULT NULL,\n `linojonuecmgrifkmdgoidmevrlvucba` int DEFAULT NULL,\n `hvgkdmxxjjempgpgybewyojrujgrzfhy` int DEFAULT NULL,\n `uscukustpbvlyqncucbsuasruocuwqha` int DEFAULT NULL,\n `wzbkksgxgnsmfzurlbykrqgjcfzwcwvt` int DEFAULT NULL,\n `cklzfmtawijmbhxmaafvzpddsbcwoigg` int DEFAULT NULL,\n `xypdzrvpflvkrgqzabndtyxuhtgccyar` int DEFAULT NULL,\n PRIMARY KEY (`icggjyszavlzpgpdwtgfjhjvgttkwlyo`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"zfgsdjvymmttvoqeggdyoqbiudfvvjnu\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["icggjyszavlzpgpdwtgfjhjvgttkwlyo"],"columns":[{"name":"icggjyszavlzpgpdwtgfjhjvgttkwlyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"vkbaambjqrqhnldwgwbourdmiqxhueej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikiontkkchkukfntjqyuuzpujaaatdtw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfnelkbyxkxaubedgvsrzmxapxsddssg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwengeqpczaegqpjrbrrhodnnhkxhjrp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"paqgbyljydlkoaknrygniqtsnwbgyevj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofbkgrqnpwwjbbqrpoqyrzcqcoiwxtvh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oiwdibbbswytvfxrrgjicmgkedgqmfuy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fupefddhcnjsgwritqfnvfwejqyyocvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yntikvravudmljvfstafjpjpgehpxpkm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ncpnrdgmmhcphzvynexwpkjkzvvdrkhd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvailtcrqffpeebyaeomrfchsxcwvexp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxzgiinikurhnatyzwrhrpdxwjraiigl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlaqxukuneenuiqqosooqqkrkvwtdoih","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yavidqifxykcbdjycvaqklstjcxlmdgj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"epwbpssciabmqaxkkrqwdqevphsihkja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jumylrokqwwjaczgvoobmfewjzvdsyuw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zbhfokboszbjcqrmbpnydhtazlmibgey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkbejqdgoafimgodxuxcrcsglpfdyrar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzctfxxlecvafofewgvqawmobadrouhd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"moeonccsrzxcrqdelmcugisllcjptoor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcimizpwjeckfwcvhdhphsbqggfcsyhr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umrdpmgawqfofqwrmggluuwzkjqdetsm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfvlsrzpchnxsmwtujpuxfeszabfttye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwtickxjgbhnwehsptxcwsjdmcidvhoo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmqqmivetaajrethdemykoydflklbygn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqrplswvfqvsnghwjhaytmhtkcghjdeg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzvlflqtkyettyxxkzrgtfjyavxmctud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rgirhbzsqhqorcjosvvxbilzxevnnqtx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tbmyyvnaolfmbltkmnsrerwgmudirqhm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xyrpmrwgkotjdqdaaiyrzcusyhxubxzu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jitloswhbjkfxupeisrzfroyjdgkjheb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhjxigbaxckxgszijundxncpczhmbobf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tlmnsquudndbowrryjaorszmaqpaeorr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fizzvlcbjwbrcnznhcbwmiotowfdvxwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fvbbubwbrtjccfcngjbflsgnpqjiohya","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"phwjjaqzfvxhjkeczzwhlmphncawqbsp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"arfqhmvffnjgzdfuvbcpesipitoirnyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjtfmlcuxvdtcjusijxebdoxgnzpahtu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjuxlbpazqqblhgahhjwflmuynqliumr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ozqdxlcvdouycqvplmbytentskxuyhfk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wslinguirhyzcwvxcsibxlmircadmruk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ekxxubvjlxuntfiuqurjhyufacmxevgf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bxlgeixskzklbkwcvzgstudgjbbiquew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pwoafbosuzttgdpktssgrcxdbcfghoay","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"opmvxhykfwyjfkmcveeyheiypahkvohq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eiksrzctsdgdzwkwldofopjjxrzhhapd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hchfzbwsrxjdrgxkphauvgrwbvcsvafu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ounmkkvnlekickmlzuisvpvzmnoaruqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcajahmpqidfwfcsdwwqrargsieayelu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"arrrdcdijikwoinbufoiltlesqkgaipj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fddhpuhhgyvikqsdpgdvjkknthyirvvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"clmpxrsdlbwlibnhjkwsfykioabckquu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"frbydkrfxcjnmgttfaiuhdykwfbzckdz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtatojyvudtycwaztfsisjpqrekqcity","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ihxeuxdydmxotajaqviqythtcqwsawtq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fxarbnolxvexszwlwfwegayacxjvdorj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbnfofssxafzrdageuouawklebrkphbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfohnrpfegjfygsulfzddatjexskjhyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kilfedygkacvzrpobymgmqrgriilawcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbixuoiewitzgpuwdpcpvxoxlsmaymft","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"srpkrslwjhgdhqgfzjqdegeydqbypmhg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tsvshxdcaguwnlrbjyilvwvkiatnmhiy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lslkaeccusxvtypceqyazvvetjzivjeh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zmfmcrmbtxcwmlgbcakrnzaswncjnpfr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oxktcuxdxdesjzqucktsanecwujzwskd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcfyaprektcgjfqatmihbtmcacaqkbws","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ptpdrckksirtkkjlkywsmcahrdzrqtah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sqqzrkzczskjuudikannbzopsbtybmgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ndnhmnfytutpomlarfyynegqbgmdgafa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnzogejiiebnpduxxoybhpxnpiizhhsz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwasascmwrhivrxaczldjshlwrajkvkl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lsltxlanystatjewmljvypyxcakkwgdb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pzfniasanrumlyplfhywvojvkpnfqizv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwhhqqngcpoxorpvotmydhmaoozcnsip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ikjysgzdeubhcygyoflvcyqjthceueyv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lbqzvdoojwpjdeimwvytshsmvsjvqivd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hsvnfivgykckedfphopbjsibzbdvzqjo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htanmyohpeywayrqonycmmozlcpbmgrl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jcwvdgjznnejxzjiiwtceeauxbtymfni","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"utohmmlvtosabaebvhwvplzbcqzttnoc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exzzxrcikegcsxivezussvtutmkxjhpt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yostpgymizqrajkaxvyfugdvslsddjqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yvpyrulflftjfmuqwcippanegbljyumf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tarcwgruawdeguepeneueqsambhvziyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqgvqedidcjlrqxzlogjzztbdfpvmytv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ktdrycudlpvyfbomhqocmswizugtiwys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smdrkjugvdbgqoygddmocbnelizuztui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ziaysfqrhnemvqiacenyjgycpqizgdhf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqbwtqyylezxartsgmxyeumeivbqkuob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twqtermlthxghpcusfgvuxtxzqspiqye","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqvhjiskwdgvsglggnrcrxrnzrkmweta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edmwnzouwusbfbbzfpudymwstprkjmhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlehpkfmamcoykqrqnjfbjbtvlmhwqna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"linojonuecmgrifkmdgoidmevrlvucba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hvgkdmxxjjempgpgybewyojrujgrzfhy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uscukustpbvlyqncucbsuasruocuwqha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wzbkksgxgnsmfzurlbykrqgjcfzwcwvt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cklzfmtawijmbhxmaafvzpddsbcwoigg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xypdzrvpflvkrgqzabndtyxuhtgccyar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842673,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842673106,"databaseName":"models_schema","ddl":"CREATE TABLE `zjynajtjehfzwlyfxrnocalgmomgzioq` (\n `mlpksasehuuvzzwcczjkjhrgtagnmpud` int NOT NULL,\n `dbuihytdrnthnfcjaseeddjjqlggerap` int DEFAULT NULL,\n `gdiornogcfaazswymtldfuzzpvvlmkph` int DEFAULT NULL,\n `tsphbpphdkjzhsaxdniuwuikhskihkgd` int DEFAULT NULL,\n `fwdkklmigwgxubmvyszrnjkudsygnvkm` int DEFAULT NULL,\n `romqgfksozbjymcnepucnqxqhjzqhdsq` int DEFAULT NULL,\n `yzjjrdonvgeahjmyevmuytxcpqzbjqdn` int DEFAULT NULL,\n `cvonomolszgwfkhoxcbqjpdydhoctvsl` int DEFAULT NULL,\n `kzipflrwlbrjhhljtsyfcekfhspjeyso` int DEFAULT NULL,\n `xsoowjzltzcewtezjevzfjirewygzfgh` int DEFAULT NULL,\n `gkyqqcatnucjvxtnskcmqfexhapvmtfu` int DEFAULT NULL,\n `kidyqkswcsnqzwnsoxtnwxxscboiavdv` int DEFAULT NULL,\n `ofsxqbhfmegzhesmbtxctfrnvqauypbg` int DEFAULT NULL,\n `wyczvaqqnoyxzyhxlaxqgfjqkxmeqhlp` int DEFAULT NULL,\n `oukdsnzkttwvicuttqjttyzwokxqzfyh` int DEFAULT NULL,\n `gxxynzgieyixcdrhxrgvomxbcjaoyjdi` int DEFAULT NULL,\n `vxijpidjfijfgmyosvjdpnozhbjuuwbw` int DEFAULT NULL,\n `exuewbwrntsnecsvtnpmqqnicwxsuoov` int DEFAULT NULL,\n `ohhdgvcerjrrzzgrszychvsanjqufqfw` int DEFAULT NULL,\n `gxcuanuvzsxopqssnozzaocgkwgnafyq` int DEFAULT NULL,\n `htctwkeiayadlvhjdicovgnjnnlddmyw` int DEFAULT NULL,\n `yxxkonzqrxquffozvrwsfjenargoycjw` int DEFAULT NULL,\n `oaqsgpcwjknbmrkhdfrgatszmohibexf` int DEFAULT NULL,\n `qdcyprztrjqdbqkbkiqyfuivsibhpdnm` int DEFAULT NULL,\n `zxecpuzobyxsziaydccdjyedenhxddxd` int DEFAULT NULL,\n `ijvcxklavjdhwrfzdhhqzzazflkmtnyd` int DEFAULT NULL,\n `fhepmrvwwyzdjalrurpnsgokvfolvqjt` int DEFAULT NULL,\n `boslotshnnqmpepthvyyavpazfskjrko` int DEFAULT NULL,\n `bhcfgwlryihzlgyyxpyxzepdqygkgwue` int DEFAULT NULL,\n `vvqrtbqacczamrgmfjhyjvxhcdyijzuh` int DEFAULT NULL,\n `toysmjmqucslxtowesqozyuhhccqzejd` int DEFAULT NULL,\n `fomyqyjfahsrzafcyzpocmcpzcfqanpr` int DEFAULT NULL,\n `lwfdaufryrfxqardwvkobbazonkljxcn` int DEFAULT NULL,\n `pzksnsvnrpbftbqziiukdnlxofeqsvoz` int DEFAULT NULL,\n `ahwetblvhgscgbaxwvqlcpmattcxgean` int DEFAULT NULL,\n `kzntzrwnyjohbbxrsbitqmfxzyfzijpa` int DEFAULT NULL,\n `uxfxpqhuuhinhxekwvneblsjkfemuanq` int DEFAULT NULL,\n `qpeizcwngdnhtinldijbtjrtleatleeq` int DEFAULT NULL,\n `bbstcudbfjcuahvnougctkmsijkwhzes` int DEFAULT NULL,\n `aaphztekzeeywjpisphtmrnyomcglrtg` int DEFAULT NULL,\n `rcrainmuoopdlkmvcyavodrkvergwkyh` int DEFAULT NULL,\n `qmpbwlikyqjexjsjuenxqnngktdsivkq` int DEFAULT NULL,\n `axlljatvcqljbtmgfzsusnuszmltxccl` int DEFAULT NULL,\n `awgylghjlhigmvvnmhjxmeaobidnegiv` int DEFAULT NULL,\n `yzcvtksmggxlizbhesxgftrgqhjackjn` int DEFAULT NULL,\n `bjmuvjqpecqiqebxczixjqakbfnairwg` int DEFAULT NULL,\n `wbdhlnwbbvwuejmvpmzissixagzyypbx` int DEFAULT NULL,\n `ypklvmbzqxvtlecuzpdakfvhajiegrgo` int DEFAULT NULL,\n `vbhhclpopsfuwyjmccdhqwooaaoegvrm` int DEFAULT NULL,\n `epedxvwivyselmwkjialczjarsrnbcxj` int DEFAULT NULL,\n `hlzbcpcmrlaixrukwxbgymfafgghlbpp` int DEFAULT NULL,\n `qwcpoooreldxxjltqdkwbukbctxunall` int DEFAULT NULL,\n `kgpgzeeladsnvvojwazuzqrnyokxyctw` int DEFAULT NULL,\n `lgkzpviveeagpqpmsqpbparmvweharbw` int DEFAULT NULL,\n `ydlewxxwsnkzaantqrkyypuhqxeaaasb` int DEFAULT NULL,\n `ddfavrblwjvbvbnatlqyzwhncfpxohxf` int DEFAULT NULL,\n `cujlcihyegcuobafzfleutmobeddsgid` int DEFAULT NULL,\n `heprkyqvmipqgdngdrzrzgrnxpaawvlb` int DEFAULT NULL,\n `kkpwnhfhnsngvphksqszljgdchhzajef` int DEFAULT NULL,\n `wawopquazevajblodnfxpputfpvqlfyz` int DEFAULT NULL,\n `jczgrovffmgdwizmltxzntppxbswycuy` int DEFAULT NULL,\n `wtmlrrdbruymmvvlfqlujgtacgktrlre` int DEFAULT NULL,\n `czmgdyzgenllnntjqjfpswtfjtdmkjdt` int DEFAULT NULL,\n `qxwzgebhrebnemisgqalbkumfquzvqcb` int DEFAULT NULL,\n `iytxnhxsmcayycfsqafcvdimuzhfbzxs` int DEFAULT NULL,\n `ykienmsemwrdfpxvvwjniyiuowewdiwq` int DEFAULT NULL,\n `mcekfxzxjfngrmnfaiaicizqcctbyesr` int DEFAULT NULL,\n `tphchxylvdnmjgzqpfkzzfwemarbpkqt` int DEFAULT NULL,\n `dtyhtyyhxorwqixksjtsfmnatmbhbyqj` int DEFAULT NULL,\n `lexjkxyitkqirirjewmbbaadyoctqalr` int DEFAULT NULL,\n `mkgrbvpymrnkhewuuwcktyanampxboaj` int DEFAULT NULL,\n `xlcoafmzpkoijmwwmvgqosvjikpijckl` int DEFAULT NULL,\n `ydhdnpzxxslxocjlzwvlflddwfmlukis` int DEFAULT NULL,\n `oirrjcckchpwgslrskiqcbhyzwnhlxng` int DEFAULT NULL,\n `gshwdimcgfulalgbdguughpzmeiziutl` int DEFAULT NULL,\n `ithlpieqclqyyxkgpddegkzxjlzxytey` int DEFAULT NULL,\n `zdbpgcogklvuqeszaagqwlxnwfrpmwhy` int DEFAULT NULL,\n `dcjltxubpweetkvmrvwetwofmmnvifjl` int DEFAULT NULL,\n `twooxknqokhnmhabtczzeewxwiuhqijp` int DEFAULT NULL,\n `oqucjgzkcsgrzykjslhlaymykivdyugi` int DEFAULT NULL,\n `bsgoxgvqqokgfhjffmlxcfmavsfusoan` int DEFAULT NULL,\n `qrhqedpkrljoqhhqxmqhponirqlhtujr` int DEFAULT NULL,\n `oxkrjzwctmxlvrfznpehystxfnwubmdd` int DEFAULT NULL,\n `twsdwyqmvlbrcjhlpxkbhzeerkkcmeud` int DEFAULT NULL,\n `psyglqxngvybgxojucrvvmqgzgaufajs` int DEFAULT NULL,\n `ukmepyhrsyuvnuefnvazadhjlpvzitjj` int DEFAULT NULL,\n `mhjrdgcosrpxemnbuhyknvckeqrylhgb` int DEFAULT NULL,\n `rewsskogxurocbxteqmirvynffmheahu` int DEFAULT NULL,\n `vhdxqlspseupcpmfcezwiuxmtaorpfrr` int DEFAULT NULL,\n `wqdievdrdirvlmrkyrkjkifytavabnza` int DEFAULT NULL,\n `abvgpvntdeufdsastkeqypqsxfrrqpwn` int DEFAULT NULL,\n `atixhijlaaqxngrwjupgvalycazpriyt` int DEFAULT NULL,\n `fitwgxanqxyshsdllirwwjvnidwqumxc` int DEFAULT NULL,\n `qqnvohygksrytbgyzupgcjhakxfohthi` int DEFAULT NULL,\n `rpslvktjyejbkaxppuiaitdbiuxwlyxj` int DEFAULT NULL,\n `hblvlmmtbwmzcbvdpnvmhicwzsgvshgj` int DEFAULT NULL,\n `kcupcfjusjrwfrdgavrjsnzahovokrlg` int DEFAULT NULL,\n `lmycljalqcsguujoglqmoierxuxuzkzr` int DEFAULT NULL,\n `hkkltkjpglfeqdpwcaibdankxjlkmkvy` int DEFAULT NULL,\n `lcujdzlnnnfbqqoyqayjohbqmyfgxkrj` int DEFAULT NULL,\n PRIMARY KEY (`mlpksasehuuvzzwcczjkjhrgtagnmpud`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"zjynajtjehfzwlyfxrnocalgmomgzioq\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["mlpksasehuuvzzwcczjkjhrgtagnmpud"],"columns":[{"name":"mlpksasehuuvzzwcczjkjhrgtagnmpud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"dbuihytdrnthnfcjaseeddjjqlggerap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdiornogcfaazswymtldfuzzpvvlmkph","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tsphbpphdkjzhsaxdniuwuikhskihkgd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fwdkklmigwgxubmvyszrnjkudsygnvkm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"romqgfksozbjymcnepucnqxqhjzqhdsq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzjjrdonvgeahjmyevmuytxcpqzbjqdn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvonomolszgwfkhoxcbqjpdydhoctvsl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzipflrwlbrjhhljtsyfcekfhspjeyso","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsoowjzltzcewtezjevzfjirewygzfgh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkyqqcatnucjvxtnskcmqfexhapvmtfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kidyqkswcsnqzwnsoxtnwxxscboiavdv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofsxqbhfmegzhesmbtxctfrnvqauypbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyczvaqqnoyxzyhxlaxqgfjqkxmeqhlp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oukdsnzkttwvicuttqjttyzwokxqzfyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxxynzgieyixcdrhxrgvomxbcjaoyjdi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxijpidjfijfgmyosvjdpnozhbjuuwbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"exuewbwrntsnecsvtnpmqqnicwxsuoov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohhdgvcerjrrzzgrszychvsanjqufqfw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxcuanuvzsxopqssnozzaocgkwgnafyq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"htctwkeiayadlvhjdicovgnjnnlddmyw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxxkonzqrxquffozvrwsfjenargoycjw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oaqsgpcwjknbmrkhdfrgatszmohibexf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdcyprztrjqdbqkbkiqyfuivsibhpdnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxecpuzobyxsziaydccdjyedenhxddxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ijvcxklavjdhwrfzdhhqzzazflkmtnyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhepmrvwwyzdjalrurpnsgokvfolvqjt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"boslotshnnqmpepthvyyavpazfskjrko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bhcfgwlryihzlgyyxpyxzepdqygkgwue","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vvqrtbqacczamrgmfjhyjvxhcdyijzuh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"toysmjmqucslxtowesqozyuhhccqzejd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fomyqyjfahsrzafcyzpocmcpzcfqanpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwfdaufryrfxqardwvkobbazonkljxcn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pzksnsvnrpbftbqziiukdnlxofeqsvoz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ahwetblvhgscgbaxwvqlcpmattcxgean","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kzntzrwnyjohbbxrsbitqmfxzyfzijpa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxfxpqhuuhinhxekwvneblsjkfemuanq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qpeizcwngdnhtinldijbtjrtleatleeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbstcudbfjcuahvnougctkmsijkwhzes","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aaphztekzeeywjpisphtmrnyomcglrtg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcrainmuoopdlkmvcyavodrkvergwkyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmpbwlikyqjexjsjuenxqnngktdsivkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axlljatvcqljbtmgfzsusnuszmltxccl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awgylghjlhigmvvnmhjxmeaobidnegiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzcvtksmggxlizbhesxgftrgqhjackjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bjmuvjqpecqiqebxczixjqakbfnairwg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbdhlnwbbvwuejmvpmzissixagzyypbx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypklvmbzqxvtlecuzpdakfvhajiegrgo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vbhhclpopsfuwyjmccdhqwooaaoegvrm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"epedxvwivyselmwkjialczjarsrnbcxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlzbcpcmrlaixrukwxbgymfafgghlbpp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwcpoooreldxxjltqdkwbukbctxunall","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kgpgzeeladsnvvojwazuzqrnyokxyctw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lgkzpviveeagpqpmsqpbparmvweharbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydlewxxwsnkzaantqrkyypuhqxeaaasb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ddfavrblwjvbvbnatlqyzwhncfpxohxf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cujlcihyegcuobafzfleutmobeddsgid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"heprkyqvmipqgdngdrzrzgrnxpaawvlb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kkpwnhfhnsngvphksqszljgdchhzajef","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wawopquazevajblodnfxpputfpvqlfyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jczgrovffmgdwizmltxzntppxbswycuy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wtmlrrdbruymmvvlfqlujgtacgktrlre","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"czmgdyzgenllnntjqjfpswtfjtdmkjdt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxwzgebhrebnemisgqalbkumfquzvqcb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iytxnhxsmcayycfsqafcvdimuzhfbzxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykienmsemwrdfpxvvwjniyiuowewdiwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mcekfxzxjfngrmnfaiaicizqcctbyesr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tphchxylvdnmjgzqpfkzzfwemarbpkqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtyhtyyhxorwqixksjtsfmnatmbhbyqj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lexjkxyitkqirirjewmbbaadyoctqalr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mkgrbvpymrnkhewuuwcktyanampxboaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xlcoafmzpkoijmwwmvgqosvjikpijckl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydhdnpzxxslxocjlzwvlflddwfmlukis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oirrjcckchpwgslrskiqcbhyzwnhlxng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gshwdimcgfulalgbdguughpzmeiziutl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ithlpieqclqyyxkgpddegkzxjlzxytey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zdbpgcogklvuqeszaagqwlxnwfrpmwhy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcjltxubpweetkvmrvwetwofmmnvifjl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twooxknqokhnmhabtczzeewxwiuhqijp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oqucjgzkcsgrzykjslhlaymykivdyugi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bsgoxgvqqokgfhjffmlxcfmavsfusoan","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qrhqedpkrljoqhhqxmqhponirqlhtujr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oxkrjzwctmxlvrfznpehystxfnwubmdd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twsdwyqmvlbrcjhlpxkbhzeerkkcmeud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psyglqxngvybgxojucrvvmqgzgaufajs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ukmepyhrsyuvnuefnvazadhjlpvzitjj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mhjrdgcosrpxemnbuhyknvckeqrylhgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rewsskogxurocbxteqmirvynffmheahu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vhdxqlspseupcpmfcezwiuxmtaorpfrr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wqdievdrdirvlmrkyrkjkifytavabnza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"abvgpvntdeufdsastkeqypqsxfrrqpwn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atixhijlaaqxngrwjupgvalycazpriyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fitwgxanqxyshsdllirwwjvnidwqumxc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qqnvohygksrytbgyzupgcjhakxfohthi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpslvktjyejbkaxppuiaitdbiuxwlyxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hblvlmmtbwmzcbvdpnvmhicwzsgvshgj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcupcfjusjrwfrdgavrjsnzahovokrlg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lmycljalqcsguujoglqmoierxuxuzkzr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hkkltkjpglfeqdpwcaibdankxjlkmkvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lcujdzlnnnfbqqoyqayjohbqmyfgxkrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842673,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842673144,"databaseName":"models_schema","ddl":"CREATE TABLE `zomwihdblysidiflhkqodbripxteqsje` (\n `kjnlgikpovwkqnkzocdzewhdovkkhjcx` int NOT NULL,\n `byhuzlgnwufookhwkmtyrevzgbnyshgo` int DEFAULT NULL,\n `zonwfhxslizwagenrcnxfkydrgbudjfc` int DEFAULT NULL,\n `ytaubiagvjfgqaljdhukitslogfclgia` int DEFAULT NULL,\n `hxxxpbagmscnimutyxjfvgflibphptfu` int DEFAULT NULL,\n `ysrnmultlulyqiclyokxpxrglhmaompa` int DEFAULT NULL,\n `fchfnfhnbuplmqvbubvymzbquttymfha` int DEFAULT NULL,\n `psbezmdfisyznxtfluqyeatatwpuscmo` int DEFAULT NULL,\n `cnspxwyrtcxftobyyzxkjkhnzhbezrkl` int DEFAULT NULL,\n `aiahbuicuprcwkcgjsqjkgbjuaekefdd` int DEFAULT NULL,\n `afllirgoepalikcyjwinzbbftmufjjvm` int DEFAULT NULL,\n `ujzckfhctxqbxkduvfrajqfbdurfwunj` int DEFAULT NULL,\n `dltjrmrtaqioiweyemweylvbjiohtggj` int DEFAULT NULL,\n `tdyvpmenffpbyxysgkcrrptqhgzimion` int DEFAULT NULL,\n `iajqvojcdhuomcozrmzmbxxxoqcptplc` int DEFAULT NULL,\n `pyhohklwkhmaxvmmlihmavsvlmpvzwjt` int DEFAULT NULL,\n `goqqmrpfxmjztlcwtvocoqivobunycpp` int DEFAULT NULL,\n `atlxwuqcghecizirdmilwbuoqoxbnvew` int DEFAULT NULL,\n `yjqtiadxxwhagefiwdlxisdybyjrtyfk` int DEFAULT NULL,\n `tycnsrumbhtclskgyyhntobckexbjmjn` int DEFAULT NULL,\n `hhlhakvjsvlqjwjrkoczenxsneecopxr` int DEFAULT NULL,\n `vovwxennxyuygwolrmgkzdsykpfmwdtd` int DEFAULT NULL,\n `zxzoaeynztastnjailmqjvdjhbcghzwr` int DEFAULT NULL,\n `rxuaqwrejeabkqwlegwnalifktifvnsk` int DEFAULT NULL,\n `pfnebbbfehxpxqrfxhrquklfberhvqts` int DEFAULT NULL,\n `skvykqhslottedonxaqtxesnfdrbbvej` int DEFAULT NULL,\n `epthxsrdphbarucwbsjvvtecakmzlptu` int DEFAULT NULL,\n `hmfxrvhfsqbbiiyeltrlaqkloxzpvjzb` int DEFAULT NULL,\n `kqtytjbpskaifeiwmgjhbdkxhutjufyh` int DEFAULT NULL,\n `vedeozibwnsigdwgizyjfextznmjhruk` int DEFAULT NULL,\n `kkphugjgxvqihvifrlepbmrihsdwqwyk` int DEFAULT NULL,\n `hfymhydfjnzzrinmrgwwyuxufteuxzjg` int DEFAULT NULL,\n `bdwylzmyrpkfikxprjythbbfhkfpzqzo` int DEFAULT NULL,\n `nirntjvitxoyqyblxkgbceodmpaapukx` int DEFAULT NULL,\n `ggbkynwqnvyxtftxhcblszhqqrhygtfn` int DEFAULT NULL,\n `bijcnuwrsrkwvgcdgizscrmkkxfpfzht` int DEFAULT NULL,\n `wvpyyoplobxqkhsmeppqeobcurysmaok` int DEFAULT NULL,\n `gwzqpuwrovxvqimfyyvnxhsjptuscuvk` int DEFAULT NULL,\n `iyfgdjefbxncgfnqndgzonggrisfmtin` int DEFAULT NULL,\n `wxhyxeysgptjiqaatrpqnymidaioiooq` int DEFAULT NULL,\n `jwbemgkfadafylqceqpckcmzchcpsxow` int DEFAULT NULL,\n `fnrolaqmcamoapwfqpjcifdwupaumspj` int DEFAULT NULL,\n `lbbtcccsudyukdvpmiotiewcejpwemvx` int DEFAULT NULL,\n `odkqjnkdosxnjjudgfcjcmnsmedpydnh` int DEFAULT NULL,\n `cifzebfmqtymihomeakkayhyvegzvcnu` int DEFAULT NULL,\n `cestnezmfphmsvsubftsotlvxglkfpqs` int DEFAULT NULL,\n `cbkdgyjipxniyccxgoolshwvwxjpyzaz` int DEFAULT NULL,\n `wbaowyslkkxtbsqafaswgdoniqeoukim` int DEFAULT NULL,\n `xxjqpesoaudukhtyoxlenvqijnxzqvyx` int DEFAULT NULL,\n `iqamkjusfmnncziqfauigsmkocynicdm` int DEFAULT NULL,\n `qasqymyephmsrzpmxpztfiyydvnycqzb` int DEFAULT NULL,\n `xgaocghdkcruzipajbpwelznjftlgnut` int DEFAULT NULL,\n `qwwjqiqiiiqeqiqjlonqexsamigtoddw` int DEFAULT NULL,\n `tozutjtetdbuynrtpsoellnyszgxwyik` int DEFAULT NULL,\n `syzqwamtrjdhgawwiylnjuoefbaffitb` int DEFAULT NULL,\n `zeqmnbudqvfdmbfrukoeqjyomxmgtofi` int DEFAULT NULL,\n `hrgxypwemzruekjhlbykbkpdtkvtjoau` int DEFAULT NULL,\n `bnugvetjgdrusulgivnmsiqmwzpztuzh` int DEFAULT NULL,\n `xeiyawvfelarmtsxulgpffjwxqyvelhx` int DEFAULT NULL,\n `xfvunplkgkmiggyopqtxadlobptokgqs` int DEFAULT NULL,\n `edrvvvxkstrrpynwmbucsvtxfihzcflt` int DEFAULT NULL,\n `yemlljfpybnnlzgoyquccqsgrtqmrfjv` int DEFAULT NULL,\n `skkrktvrukrawhsrdzmvpyniapyqcniw` int DEFAULT NULL,\n `ymfdycuncfwyeshmptrxhyftckntegbg` int DEFAULT NULL,\n `oraffxmnhxvanohrplovcoyumceicywb` int DEFAULT NULL,\n `nhzddhxucjntuhtszeqfqhcxckrjbcxz` int DEFAULT NULL,\n `cpxkkxwzjjdwqotvdriqagnkszkryegi` int DEFAULT NULL,\n `deqtaivxjmomzojsloalxmfdkkesgfqk` int DEFAULT NULL,\n `dkbbqsiwjocrhwjionyjubaiujdxbxdu` int DEFAULT NULL,\n `ntzjqhuwuxstncddwmpzrcmpesbldimd` int DEFAULT NULL,\n `wiwsfzoayoqiaprsjpmcjfpftceoccwa` int DEFAULT NULL,\n `dbhtpcsaclqojciyjhumlnpzvovbfmbd` int DEFAULT NULL,\n `rfhnnopijzssnmqcfzvxrvcoirqafdco` int DEFAULT NULL,\n `fmipjunhocoswopqixkyhnpdyrjigsxp` int DEFAULT NULL,\n `jglcdupkehmpgwvoglkllplnfjrbxrbj` int DEFAULT NULL,\n `fclwlpicjmckuehmbecjhotqwskmrxxy` int DEFAULT NULL,\n `ixjeemtsvwroldefpwtegpsjbfxazrhy` int DEFAULT NULL,\n `xgzgmtwwcawwflvzirsbaicqbkqaoaci` int DEFAULT NULL,\n `aafaprvvxuswjavgpwyhtfcfqugzimbt` int DEFAULT NULL,\n `dvzsxspbrvxcjgwmfkxynjtwuqjvezcp` int DEFAULT NULL,\n `wxjinfdvapcbcjskqmvvezeyxeebuyxk` int DEFAULT NULL,\n `qdesjcbmovujsrqkoglpfnslyiyvqsoz` int DEFAULT NULL,\n `gpcokwsvmfaqmotpiaiairzeabyzzxon` int DEFAULT NULL,\n `gunvcrmdesefozajwzcfwmlbtlhjiqcf` int DEFAULT NULL,\n `fivuemqgbycdufrkxvebhvotfxgrgioa` int DEFAULT NULL,\n `tfjliienlqjlshvocujzhiqxiclgfsuh` int DEFAULT NULL,\n `tsupqdpbqlyunvqjkdnpsnohnujsquon` int DEFAULT NULL,\n `rzgzleynrpehggxhfaoziwhfmjffoycz` int DEFAULT NULL,\n `ykumrdxzbdlihbibxqjfjtywixpthewr` int DEFAULT NULL,\n `teirgoaggkmilqkxnwxnhftyacafgraq` int DEFAULT NULL,\n `ivhpkxursjcuztromvbxkwkgiomxaigi` int DEFAULT NULL,\n `enkjnuoqoywhcxxaayetwlrpkatwabkj` int DEFAULT NULL,\n `viwspnotrfokzptpbmwnvdkcvzkzmwsn` int DEFAULT NULL,\n `aobbmrpsyiedqrdiofrprfisrvcnkdeu` int DEFAULT NULL,\n `whgjdszemfvnisqobqjviibooyptouiv` int DEFAULT NULL,\n `sshktyxudsqpftaktrxpkeqfqsdlxmlq` int DEFAULT NULL,\n `eomezqepjgxhrnjaqvptzxjgcrxzyqja` int DEFAULT NULL,\n `aewlkqkeizgbytizzmttgepnroneques` int DEFAULT NULL,\n `viijhbdnqxqrblableckfahbxdpkxddt` int DEFAULT NULL,\n `hvizzqcewzuotsrclmnvwwftqynamjhw` int DEFAULT NULL,\n PRIMARY KEY (`kjnlgikpovwkqnkzocdzewhdovkkhjcx`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"zomwihdblysidiflhkqodbripxteqsje\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["kjnlgikpovwkqnkzocdzewhdovkkhjcx"],"columns":[{"name":"kjnlgikpovwkqnkzocdzewhdovkkhjcx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"byhuzlgnwufookhwkmtyrevzgbnyshgo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zonwfhxslizwagenrcnxfkydrgbudjfc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ytaubiagvjfgqaljdhukitslogfclgia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxxxpbagmscnimutyxjfvgflibphptfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ysrnmultlulyqiclyokxpxrglhmaompa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fchfnfhnbuplmqvbubvymzbquttymfha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"psbezmdfisyznxtfluqyeatatwpuscmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnspxwyrtcxftobyyzxkjkhnzhbezrkl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aiahbuicuprcwkcgjsqjkgbjuaekefdd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afllirgoepalikcyjwinzbbftmufjjvm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ujzckfhctxqbxkduvfrajqfbdurfwunj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dltjrmrtaqioiweyemweylvbjiohtggj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tdyvpmenffpbyxysgkcrrptqhgzimion","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iajqvojcdhuomcozrmzmbxxxoqcptplc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pyhohklwkhmaxvmmlihmavsvlmpvzwjt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"goqqmrpfxmjztlcwtvocoqivobunycpp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"atlxwuqcghecizirdmilwbuoqoxbnvew","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yjqtiadxxwhagefiwdlxisdybyjrtyfk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tycnsrumbhtclskgyyhntobckexbjmjn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hhlhakvjsvlqjwjrkoczenxsneecopxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vovwxennxyuygwolrmgkzdsykpfmwdtd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zxzoaeynztastnjailmqjvdjhbcghzwr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rxuaqwrejeabkqwlegwnalifktifvnsk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pfnebbbfehxpxqrfxhrquklfberhvqts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skvykqhslottedonxaqtxesnfdrbbvej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"epthxsrdphbarucwbsjvvtecakmzlptu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmfxrvhfsqbbiiyeltrlaqkloxzpvjzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqtytjbpskaifeiwmgjhbdkxhutjufyh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vedeozibwnsigdwgizyjfextznmjhruk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kkphugjgxvqihvifrlepbmrihsdwqwyk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfymhydfjnzzrinmrgwwyuxufteuxzjg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdwylzmyrpkfikxprjythbbfhkfpzqzo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nirntjvitxoyqyblxkgbceodmpaapukx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggbkynwqnvyxtftxhcblszhqqrhygtfn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bijcnuwrsrkwvgcdgizscrmkkxfpfzht","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvpyyoplobxqkhsmeppqeobcurysmaok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gwzqpuwrovxvqimfyyvnxhsjptuscuvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iyfgdjefbxncgfnqndgzonggrisfmtin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxhyxeysgptjiqaatrpqnymidaioiooq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwbemgkfadafylqceqpckcmzchcpsxow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnrolaqmcamoapwfqpjcifdwupaumspj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lbbtcccsudyukdvpmiotiewcejpwemvx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"odkqjnkdosxnjjudgfcjcmnsmedpydnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cifzebfmqtymihomeakkayhyvegzvcnu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cestnezmfphmsvsubftsotlvxglkfpqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cbkdgyjipxniyccxgoolshwvwxjpyzaz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wbaowyslkkxtbsqafaswgdoniqeoukim","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxjqpesoaudukhtyoxlenvqijnxzqvyx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqamkjusfmnncziqfauigsmkocynicdm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qasqymyephmsrzpmxpztfiyydvnycqzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xgaocghdkcruzipajbpwelznjftlgnut","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qwwjqiqiiiqeqiqjlonqexsamigtoddw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tozutjtetdbuynrtpsoellnyszgxwyik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"syzqwamtrjdhgawwiylnjuoefbaffitb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zeqmnbudqvfdmbfrukoeqjyomxmgtofi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hrgxypwemzruekjhlbykbkpdtkvtjoau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bnugvetjgdrusulgivnmsiqmwzpztuzh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xeiyawvfelarmtsxulgpffjwxqyvelhx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xfvunplkgkmiggyopqtxadlobptokgqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edrvvvxkstrrpynwmbucsvtxfihzcflt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yemlljfpybnnlzgoyquccqsgrtqmrfjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skkrktvrukrawhsrdzmvpyniapyqcniw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ymfdycuncfwyeshmptrxhyftckntegbg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oraffxmnhxvanohrplovcoyumceicywb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhzddhxucjntuhtszeqfqhcxckrjbcxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cpxkkxwzjjdwqotvdriqagnkszkryegi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"deqtaivxjmomzojsloalxmfdkkesgfqk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dkbbqsiwjocrhwjionyjubaiujdxbxdu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntzjqhuwuxstncddwmpzrcmpesbldimd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wiwsfzoayoqiaprsjpmcjfpftceoccwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dbhtpcsaclqojciyjhumlnpzvovbfmbd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rfhnnopijzssnmqcfzvxrvcoirqafdco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fmipjunhocoswopqixkyhnpdyrjigsxp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jglcdupkehmpgwvoglkllplnfjrbxrbj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fclwlpicjmckuehmbecjhotqwskmrxxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixjeemtsvwroldefpwtegpsjbfxazrhy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xgzgmtwwcawwflvzirsbaicqbkqaoaci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aafaprvvxuswjavgpwyhtfcfqugzimbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dvzsxspbrvxcjgwmfkxynjtwuqjvezcp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wxjinfdvapcbcjskqmvvezeyxeebuyxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qdesjcbmovujsrqkoglpfnslyiyvqsoz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gpcokwsvmfaqmotpiaiairzeabyzzxon","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gunvcrmdesefozajwzcfwmlbtlhjiqcf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fivuemqgbycdufrkxvebhvotfxgrgioa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tfjliienlqjlshvocujzhiqxiclgfsuh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tsupqdpbqlyunvqjkdnpsnohnujsquon","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rzgzleynrpehggxhfaoziwhfmjffoycz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ykumrdxzbdlihbibxqjfjtywixpthewr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"teirgoaggkmilqkxnwxnhftyacafgraq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ivhpkxursjcuztromvbxkwkgiomxaigi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"enkjnuoqoywhcxxaayetwlrpkatwabkj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"viwspnotrfokzptpbmwnvdkcvzkzmwsn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aobbmrpsyiedqrdiofrprfisrvcnkdeu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"whgjdszemfvnisqobqjviibooyptouiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sshktyxudsqpftaktrxpkeqfqsdlxmlq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eomezqepjgxhrnjaqvptzxjgcrxzyqja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aewlkqkeizgbytizzmttgepnroneques","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"viijhbdnqxqrblableckfahbxdpkxddt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hvizzqcewzuotsrclmnvwwftqynamjhw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842673,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842673182,"databaseName":"models_schema","ddl":"CREATE TABLE `zrjhsxwmexkwyntdmqgzgaxcjzqahncz` (\n `ngjtlhadnjojrphyvxhzifevgnmbavzx` int NOT NULL,\n `dxlmojkgtnpzukcrobbevddplqpmaejo` int DEFAULT NULL,\n `qfsekmflpsgrrrwxmmsbsepayeejsafq` int DEFAULT NULL,\n `gayypdaoqfgbcefpxbludpuijaeeuopq` int DEFAULT NULL,\n `jydkzapndeosyxcknjlsnzxikiaimyhp` int DEFAULT NULL,\n `pfmzolskqnnrmdgbfqdjmgbqcjuncjdb` int DEFAULT NULL,\n `ukvivzzacmzvvamfbomxlputxsqfvobo` int DEFAULT NULL,\n `pttoixqyshoeqfsnjkpwxepntsxwschl` int DEFAULT NULL,\n `lwrfrrqfsaebcybltkvceummlgksffys` int DEFAULT NULL,\n `lbyhizdmgaghmflgchwzrsgrjxabdmkq` int DEFAULT NULL,\n `pdnsrxveagqubxxrcxxcfhrekjbogcum` int DEFAULT NULL,\n `lfjhrzzinqlpbtlhlojqlrsbnfucmjvl` int DEFAULT NULL,\n `qmgqmabfurnwwdqxobhnnjrnawunbuwz` int DEFAULT NULL,\n `lszsytlutraumtmwzacjwsgbxobhrris` int DEFAULT NULL,\n `fqnmlxscvremypzmszrklqcnqhwdgjjh` int DEFAULT NULL,\n `xxdzbmisakatqzqpozzzfatycbhslhtb` int DEFAULT NULL,\n `drkguhvrsrmuqexdwuuogodvciguynqp` int DEFAULT NULL,\n `smzslrdiyrefhkvggqbgwgjtldfwdsnj` int DEFAULT NULL,\n `xpmmfmbkukxjvrnrakjvthvubmcpeylg` int DEFAULT NULL,\n `avlqnhspbnyrcpmmkfptnjxwfhkwnjoj` int DEFAULT NULL,\n `sanxgjqfbrvikfthhzwuzmkvkoeccsvs` int DEFAULT NULL,\n `fskwvpubnntasqjoyhkyaoazogmcwxdl` int DEFAULT NULL,\n `zqtzowpakjlzazdvirvpahsekywwyaai` int DEFAULT NULL,\n `tnembildewywrwvmtivjstrzgifsegka` int DEFAULT NULL,\n `yeplqicogsdiwbtbsigirqakzbjixoiv` int DEFAULT NULL,\n `qtpevrrlfvpymzclbxmllwptoetdicvd` int DEFAULT NULL,\n `uxhaalyaykhcarfplytppihgvxvrlvhk` int DEFAULT NULL,\n `togzwaixlktmpurwalgxfccnolwthwvy` int DEFAULT NULL,\n `gihcjkynutedtorruphzkgroguwoxrov` int DEFAULT NULL,\n `dqxrljixlstkrasvhgsfauxoecmfifia` int DEFAULT NULL,\n `eccbawzucejsdietzhkptnffbqxgdyuy` int DEFAULT NULL,\n `wvnxphbnxkgnfdxihtugetpkeazacbqn` int DEFAULT NULL,\n `vtdokgxmxkvcpxascvhnqlqutdpmxxhr` int DEFAULT NULL,\n `rihgjengisatxexysgscbtcihircavuc` int DEFAULT NULL,\n `yfccnnksaydkdndfnupvtdqeuepagvce` int DEFAULT NULL,\n `pyrgmcjfbfqatjemmobzmnuginyybmym` int DEFAULT NULL,\n `vgpfixifkaaqxryziokrsvbxqedfdlup` int DEFAULT NULL,\n `szzzsyszrtjsirmgbslvtefxrdepzett` int DEFAULT NULL,\n `szrllqkvmhxcsxsdvktdksgxghfxisxj` int DEFAULT NULL,\n `vkdszenimiimktsekphtzmspxlnhgzwa` int DEFAULT NULL,\n `heyubmjhydclottdirdkfkgzmreuoaob` int DEFAULT NULL,\n `sioibictbscoazpmeyvxsfalqgcxvlgw` int DEFAULT NULL,\n `rdsuqejghlycvclbphzmkctypcimqzdv` int DEFAULT NULL,\n `jbexrpoxovrqkgqhxagjvnmpaqmtgfiu` int DEFAULT NULL,\n `msdrvxdcvtjrjyiwwymyupncshoyzfui` int DEFAULT NULL,\n `bxzgjodahohwwaggffevxdhicjpsenlc` int DEFAULT NULL,\n `qvfhajuikhsxoheuuiuviyamhwvnbcyj` int DEFAULT NULL,\n `knjykvvbmxwbkyjqhxyvhxekqeburhxl` int DEFAULT NULL,\n `luefpkorhmercqtqxzumkyxewcvdgros` int DEFAULT NULL,\n `mlpwwkifqbhizhuutwyauserbvrjhciq` int DEFAULT NULL,\n `aayjprmdrrhdwjphzeelohdgpkexiuud` int DEFAULT NULL,\n `mnfvsibogdcxugbadjzcxzqpkoczaofm` int DEFAULT NULL,\n `dknkqfydzaosiqfuuxwewylamxxxhpci` int DEFAULT NULL,\n `amrvbuispugrfxspbnffkcbjtvcwcbvi` int DEFAULT NULL,\n `ntadevpbkmjrgglrijabqajqwuppxlig` int DEFAULT NULL,\n `unabjokjujnqsmvtryjpxivxzgkbppny` int DEFAULT NULL,\n `ffxvgkugrhvkhyyaayzylnxznamurbfg` int DEFAULT NULL,\n `orgwnccmqilksewncqgeozkribqfaumu` int DEFAULT NULL,\n `eyjqhyfmoppijtvlcbbhjphgfnzxhqas` int DEFAULT NULL,\n `vxrpvylbpzigvyiyozafpzscpszsdlsg` int DEFAULT NULL,\n `whyfrvyfazogfajoqxolplzlibvhbbub` int DEFAULT NULL,\n `zvraouqvhygttyfdmxrqdxrrerqmlqxz` int DEFAULT NULL,\n `vxrewugestedajexctlwmdgupebxieoc` int DEFAULT NULL,\n `cuokhvbiexcoorittimgwurmopaxqhpt` int DEFAULT NULL,\n `vcbogezzpgkumorhfuqltuvwatwaekpk` int DEFAULT NULL,\n `vsmtrofjresifmoacuuvnzwltsubkrbe` int DEFAULT NULL,\n `bbtonwlkxjcvcasjptietgphgzmweift` int DEFAULT NULL,\n `idxrwmbmvtnogvjckgrlqcmsmntwlpvk` int DEFAULT NULL,\n `zyuuulcaqxichlsldkfmjybbxoiebdcj` int DEFAULT NULL,\n `txhihuzextbwplifrqlsrhqlezoafmtu` int DEFAULT NULL,\n `vvpywzshgmthwqzqrvhzrermuutjtkip` int DEFAULT NULL,\n `jkhlcaxkqrbwqvdroytjqlvkbfkleuqu` int DEFAULT NULL,\n `gsbkqhrvqugakxbtndqcgzzfgoexozsy` int DEFAULT NULL,\n `ehezlllesnwabczhoxmsjlfivjevuoam` int DEFAULT NULL,\n `wprqdhxtdxcfcmmjavsxtqwnaefochlg` int DEFAULT NULL,\n `hdznfnbsyplymbdpdnrnfhchyqzdmoxn` int DEFAULT NULL,\n `wudqpdsgscqyppzohenqwtbauglppdtp` int DEFAULT NULL,\n `aqsdyazwvzgxunfsrizbuoxurbabeuus` int DEFAULT NULL,\n `gkgphkfuwaebmlzrsvflhdlloidualzw` int DEFAULT NULL,\n `qtbhvhpnogwpodivufpwwabpvcxwyrzp` int DEFAULT NULL,\n `blitubckmikspvqxqjzkdsbbscwwsxvg` int DEFAULT NULL,\n `nvzytwwdjxbhhqtwsaiaokhsfqqxzgng` int DEFAULT NULL,\n `cscbokkyufwzvcjdmtogslutbwyxojdh` int DEFAULT NULL,\n `useogmquznrsoiqbclnlxuzzcbieeokd` int DEFAULT NULL,\n `vdqqqzzwjbucrsdpnrvhotfbqceklvgr` int DEFAULT NULL,\n `gvsewgqrauhoklawbfxfiwyvktrgiayy` int DEFAULT NULL,\n `wwtkujmcmakqapydgkfjujeirmdcyexs` int DEFAULT NULL,\n `pprwbltkfqaruhuqxqshcwlwqkufibok` int DEFAULT NULL,\n `uasyigiifhlxknwdpkfzhebbpqhfuhtk` int DEFAULT NULL,\n `kphmcxdvpssvnneawsllmdplurbbkrkw` int DEFAULT NULL,\n `dtpdutldjoojjhslrvlgnbcmtocqitcf` int DEFAULT NULL,\n `igcivbwfyjlkvxrfvkikjwztuqpjrxdo` int DEFAULT NULL,\n `yxcbjvpabjtezrkgeoxhuowucnfgnzha` int DEFAULT NULL,\n `pgrfkbqmtqingzkiajkcmowxdqsyqens` int DEFAULT NULL,\n `adjymsocronpheynmgznsublckgkjqfx` int DEFAULT NULL,\n `gpnyrkkidyruolobyecneqojymvtmcpp` int DEFAULT NULL,\n `blktuxlhtfclhazgshvulcopajxkrczr` int DEFAULT NULL,\n `zfaiczzaklegaopfsgezcliuvwdoszcm` int DEFAULT NULL,\n `gcdksobzokwjtfzhhkgbjhayvmpqoydo` int DEFAULT NULL,\n `loywlcvcmgzqquzcwpstuyscfvtgkojt` int DEFAULT NULL,\n PRIMARY KEY (`ngjtlhadnjojrphyvxhzifevgnmbavzx`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"zrjhsxwmexkwyntdmqgzgaxcjzqahncz\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["ngjtlhadnjojrphyvxhzifevgnmbavzx"],"columns":[{"name":"ngjtlhadnjojrphyvxhzifevgnmbavzx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"dxlmojkgtnpzukcrobbevddplqpmaejo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfsekmflpsgrrrwxmmsbsepayeejsafq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gayypdaoqfgbcefpxbludpuijaeeuopq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jydkzapndeosyxcknjlsnzxikiaimyhp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pfmzolskqnnrmdgbfqdjmgbqcjuncjdb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ukvivzzacmzvvamfbomxlputxsqfvobo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pttoixqyshoeqfsnjkpwxepntsxwschl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwrfrrqfsaebcybltkvceummlgksffys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lbyhizdmgaghmflgchwzrsgrjxabdmkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pdnsrxveagqubxxrcxxcfhrekjbogcum","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfjhrzzinqlpbtlhlojqlrsbnfucmjvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qmgqmabfurnwwdqxobhnnjrnawunbuwz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lszsytlutraumtmwzacjwsgbxobhrris","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqnmlxscvremypzmszrklqcnqhwdgjjh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xxdzbmisakatqzqpozzzfatycbhslhtb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drkguhvrsrmuqexdwuuogodvciguynqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smzslrdiyrefhkvggqbgwgjtldfwdsnj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xpmmfmbkukxjvrnrakjvthvubmcpeylg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"avlqnhspbnyrcpmmkfptnjxwfhkwnjoj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sanxgjqfbrvikfthhzwuzmkvkoeccsvs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fskwvpubnntasqjoyhkyaoazogmcwxdl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqtzowpakjlzazdvirvpahsekywwyaai","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tnembildewywrwvmtivjstrzgifsegka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yeplqicogsdiwbtbsigirqakzbjixoiv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtpevrrlfvpymzclbxmllwptoetdicvd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxhaalyaykhcarfplytppihgvxvrlvhk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"togzwaixlktmpurwalgxfccnolwthwvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gihcjkynutedtorruphzkgroguwoxrov","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dqxrljixlstkrasvhgsfauxoecmfifia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eccbawzucejsdietzhkptnffbqxgdyuy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wvnxphbnxkgnfdxihtugetpkeazacbqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vtdokgxmxkvcpxascvhnqlqutdpmxxhr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rihgjengisatxexysgscbtcihircavuc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfccnnksaydkdndfnupvtdqeuepagvce","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pyrgmcjfbfqatjemmobzmnuginyybmym","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgpfixifkaaqxryziokrsvbxqedfdlup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"szzzsyszrtjsirmgbslvtefxrdepzett","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"szrllqkvmhxcsxsdvktdksgxghfxisxj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vkdszenimiimktsekphtzmspxlnhgzwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"heyubmjhydclottdirdkfkgzmreuoaob","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sioibictbscoazpmeyvxsfalqgcxvlgw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rdsuqejghlycvclbphzmkctypcimqzdv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jbexrpoxovrqkgqhxagjvnmpaqmtgfiu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"msdrvxdcvtjrjyiwwymyupncshoyzfui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bxzgjodahohwwaggffevxdhicjpsenlc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qvfhajuikhsxoheuuiuviyamhwvnbcyj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knjykvvbmxwbkyjqhxyvhxekqeburhxl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"luefpkorhmercqtqxzumkyxewcvdgros","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mlpwwkifqbhizhuutwyauserbvrjhciq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aayjprmdrrhdwjphzeelohdgpkexiuud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnfvsibogdcxugbadjzcxzqpkoczaofm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dknkqfydzaosiqfuuxwewylamxxxhpci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"amrvbuispugrfxspbnffkcbjtvcwcbvi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntadevpbkmjrgglrijabqajqwuppxlig","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unabjokjujnqsmvtryjpxivxzgkbppny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffxvgkugrhvkhyyaayzylnxznamurbfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"orgwnccmqilksewncqgeozkribqfaumu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eyjqhyfmoppijtvlcbbhjphgfnzxhqas","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxrpvylbpzigvyiyozafpzscpszsdlsg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"whyfrvyfazogfajoqxolplzlibvhbbub","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvraouqvhygttyfdmxrqdxrrerqmlqxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vxrewugestedajexctlwmdgupebxieoc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cuokhvbiexcoorittimgwurmopaxqhpt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vcbogezzpgkumorhfuqltuvwatwaekpk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsmtrofjresifmoacuuvnzwltsubkrbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bbtonwlkxjcvcasjptietgphgzmweift","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"idxrwmbmvtnogvjckgrlqcmsmntwlpvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyuuulcaqxichlsldkfmjybbxoiebdcj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"txhihuzextbwplifrqlsrhqlezoafmtu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vvpywzshgmthwqzqrvhzrermuutjtkip","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jkhlcaxkqrbwqvdroytjqlvkbfkleuqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsbkqhrvqugakxbtndqcgzzfgoexozsy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ehezlllesnwabczhoxmsjlfivjevuoam","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wprqdhxtdxcfcmmjavsxtqwnaefochlg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdznfnbsyplymbdpdnrnfhchyqzdmoxn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wudqpdsgscqyppzohenqwtbauglppdtp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqsdyazwvzgxunfsrizbuoxurbabeuus","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkgphkfuwaebmlzrsvflhdlloidualzw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtbhvhpnogwpodivufpwwabpvcxwyrzp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blitubckmikspvqxqjzkdsbbscwwsxvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nvzytwwdjxbhhqtwsaiaokhsfqqxzgng","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cscbokkyufwzvcjdmtogslutbwyxojdh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"useogmquznrsoiqbclnlxuzzcbieeokd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdqqqzzwjbucrsdpnrvhotfbqceklvgr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gvsewgqrauhoklawbfxfiwyvktrgiayy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wwtkujmcmakqapydgkfjujeirmdcyexs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pprwbltkfqaruhuqxqshcwlwqkufibok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uasyigiifhlxknwdpkfzhebbpqhfuhtk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kphmcxdvpssvnneawsllmdplurbbkrkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dtpdutldjoojjhslrvlgnbcmtocqitcf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igcivbwfyjlkvxrfvkikjwztuqpjrxdo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxcbjvpabjtezrkgeoxhuowucnfgnzha","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgrfkbqmtqingzkiajkcmowxdqsyqens","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"adjymsocronpheynmgznsublckgkjqfx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gpnyrkkidyruolobyecneqojymvtmcpp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"blktuxlhtfclhazgshvulcopajxkrczr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zfaiczzaklegaopfsgezcliuvwdoszcm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gcdksobzokwjtfzhhkgbjhayvmpqoydo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"loywlcvcmgzqquzcwpstuyscfvtgkojt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842673,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842673214,"databaseName":"models_schema","ddl":"CREATE TABLE `zrpahurtunmdbumraxjhiqvfumrdlnjn` (\n `roxanjtnahjmktgfcehelycoiajqqebq` int NOT NULL,\n `mbaiuflvreyyywvwhtpkaktuwtvtlquo` int DEFAULT NULL,\n `uqnbcscnzwghjiqwjdduoupgogrnhdpm` int DEFAULT NULL,\n `nomefwdkivynrxedjkscjquougyoowiy` int DEFAULT NULL,\n `ffeopoaqgmhufoanrktesycmnndootby` int DEFAULT NULL,\n `ypsqveaaqtvojgblvhsjgoiqkzcirgtl` int DEFAULT NULL,\n `vqvzrvrqfeajbcmoycntevavxgfoddpr` int DEFAULT NULL,\n `liutpvmdieywhegwgvxiejqjugtkempp` int DEFAULT NULL,\n `hxtjhkcyfuqtukgggwzspxanfhquczmu` int DEFAULT NULL,\n `seauomwxamzadjlpddzwgphmcrjjxcjp` int DEFAULT NULL,\n `hvjhnyqxdrvtntphqmmayeurqjroyurm` int DEFAULT NULL,\n `veyrmheajdfawiwevmjgcminxynssynl` int DEFAULT NULL,\n `odcyldrdffypbrjqkucsvtukeftrhoax` int DEFAULT NULL,\n `hxyfrgobjcccqhtqwdvfyrvcuvemqckq` int DEFAULT NULL,\n `ebdofjhynfuimlqtgpoaiyeeghemruor` int DEFAULT NULL,\n `ndrmqrhltyspwcobtfguborgceohaqvk` int DEFAULT NULL,\n `hmvwbmpgjjrhlkejeucubfgobhqufpdv` int DEFAULT NULL,\n `ttfutgtunaqmovartqkdagsiooptcxhy` int DEFAULT NULL,\n `ietreqmscwjbhwdtphzboffqhbobuuqp` int DEFAULT NULL,\n `crlfnkjkpkiypzkgwmtrktqplejvwgnc` int DEFAULT NULL,\n `wyntwydpxldiubmjyuzkedlqdsuwhrqm` int DEFAULT NULL,\n `ncmbkjqdpulwkermwubwsixndltvmxhe` int DEFAULT NULL,\n `wpznenicoazndaretrbkhibjdbmyxevs` int DEFAULT NULL,\n `znmnkrxrzhinkzxldvuofoxqlfqmorsv` int DEFAULT NULL,\n `wfokdvzbpcivyjlilrcvykkmwmvsoyaj` int DEFAULT NULL,\n `gzgntsugaxdkqlvprupovfrgklnqtdbo` int DEFAULT NULL,\n `hzlzlkpfxhsjdtfjqbibanqttbhlyhfd` int DEFAULT NULL,\n `uhlvlmzdbsqqrmiqectxpciorbpazald` int DEFAULT NULL,\n `tvyuerpllzgbdyatxilswzdeqgykghqx` int DEFAULT NULL,\n `ocvkwtlpxfudrzwtsjekglcogqwehwqc` int DEFAULT NULL,\n `vdxlvesrxbvfopnbgawlrjjjhweezwed` int DEFAULT NULL,\n `qfnuljcahwkdgvfgyukcgbyfxlwwqsff` int DEFAULT NULL,\n `pgfutylxrjdegildmiyggzgolbwhdpmw` int DEFAULT NULL,\n `gugqwplhspseaoruanjqvojeptbilost` int DEFAULT NULL,\n `czhgtfehhacaikxvqifrrlmyvcldqfxd` int DEFAULT NULL,\n `wavlvkdzxfduvjifatmqxlbzniirbuqo` int DEFAULT NULL,\n `wnbmkstdieaxfrhuehztlqgsluculwod` int DEFAULT NULL,\n `edbtsgrwkwqgcmiivlyhewxkgaiiqrys` int DEFAULT NULL,\n `pmffwmtwfeamgmkyehwhqzxeaqmzfqlo` int DEFAULT NULL,\n `jdrwddlcazcmqcwjxlcvjkjzogwadaun` int DEFAULT NULL,\n `nhkyxpgxjjaihvadbsvumpjeeekvranr` int DEFAULT NULL,\n `zvvvvsftavllkfjujydvszasuncovxdp` int DEFAULT NULL,\n `wfniisuqmxbjscvqkbszqlkzjdovuzir` int DEFAULT NULL,\n `hiibfncaedfrqsrgtaeqwhdhaaiqxqqs` int DEFAULT NULL,\n `ajlnuzkkzzwtavqojhcvfwihtivatzqu` int DEFAULT NULL,\n `hiatmmhatnbgmixabgpyjokznfgwgpqc` int DEFAULT NULL,\n `bfrukcrqyqvbmxipktqkrxfvxoqeausc` int DEFAULT NULL,\n `ueimfjbrknunzuilmyxhaghnljtukcbf` int DEFAULT NULL,\n `ohnffagzscnfqdfitesbyukbrvtbwomp` int DEFAULT NULL,\n `hjzukdixlcnmjuaaxhkszgylrkenxrwb` int DEFAULT NULL,\n `dxfwlqekdphazezwdestzicfqdyeibco` int DEFAULT NULL,\n `nkgojkidjmtrmdwchxrclosviyykigcu` int DEFAULT NULL,\n `ppzlziccdzegtsaeklblhzndtbkatxug` int DEFAULT NULL,\n `qtdzeaenosmghvguqmeyphwelqfkhtud` int DEFAULT NULL,\n `ilgyvhitkgrikxrfytyqmxfemkgnbegt` int DEFAULT NULL,\n `lhhywwtkyqleyyjjyayaqgwsledzgani` int DEFAULT NULL,\n `hwfzyczbhldvrpycuqnotoyoebetbavj` int DEFAULT NULL,\n `nbphhvpocrdmmdvxzjphwwjuvakptldy` int DEFAULT NULL,\n `upoimkyzcdbvpudgpawwzentguuklrjq` int DEFAULT NULL,\n `whtuieqbeibfdgwftpupnojixrbwroyn` int DEFAULT NULL,\n `awsvyluxzojzzvqbhsuoznhkwabofzvr` int DEFAULT NULL,\n `pafjwrisvkaihnxvizqgukodgreqevbs` int DEFAULT NULL,\n `fbkgkemmjcxupktohvewhwxwhiktfmfh` int DEFAULT NULL,\n `ocjzfdveptybfparybsngvupiavfeoga` int DEFAULT NULL,\n `krakignjkcjtacnzzxdlpnmsalzubowe` int DEFAULT NULL,\n `xdqweirhuoapbeytoowlioylztrevbkq` int DEFAULT NULL,\n `pftjqacwwhditfgdzdguulgxszmlgymz` int DEFAULT NULL,\n `sptkbtzndvthkjzglbiiaxzowvduunwk` int DEFAULT NULL,\n `yzxhxbosvvyxdncobzcbervxrlodwzwm` int DEFAULT NULL,\n `yxtarhzlfhfhzyobpoqjjzvpvdgslors` int DEFAULT NULL,\n `yzqtnekkmoueoxpirxughktqtthdzxxq` int DEFAULT NULL,\n `cdnlofwbyefmgfrmhvpkextxsksslvbi` int DEFAULT NULL,\n `kqkraciroaebbrotygmxwmxkcpndrhze` int DEFAULT NULL,\n `riwmcqlufnhtswdztknxlogwckougoqv` int DEFAULT NULL,\n `lwlwluxiiwsixixusrqiqfmibjtpaqbe` int DEFAULT NULL,\n `uvxwupkabnotbrwsgyoaiekhldaabazp` int DEFAULT NULL,\n `cjolnnjifetajrcveeyhxwdoeecavkvb` int DEFAULT NULL,\n `qhicttemkonovhglvxuadcouvyzasett` int DEFAULT NULL,\n `wuicjozijswcrncanvkbxhlheguiphwq` int DEFAULT NULL,\n `corlidbeeneapjbarkupystzalggauri` int DEFAULT NULL,\n `zlnvrbgggjzkmipllyxdnwjsnlhpajug` int DEFAULT NULL,\n `nxemxvsnytuffvjxwtllbokceqvdiler` int DEFAULT NULL,\n `hnapwuarkzeyfwbjxduvvvirgiuqasbe` int DEFAULT NULL,\n `sajkhdwfcjjkelwdazngiclmlovmhcvc` int DEFAULT NULL,\n `zoggxjmnqallzqfykesrpmrwxoilfkow` int DEFAULT NULL,\n `frqjcezhhftysoxiynzvwltwdmzztcpc` int DEFAULT NULL,\n `kiaitasfwwjbxoifrgmbxcpnwzejhwtn` int DEFAULT NULL,\n `zyklwscbeuefmbgfivpcydeqocrhgkvs` int DEFAULT NULL,\n `yyultlzifxrdhufnzvypjdzpzmlwblla` int DEFAULT NULL,\n `rlthasgfmixuyrsqodxpviutejmisrzu` int DEFAULT NULL,\n `qocitaaixjyqhtflgficbmuitfdxewue` int DEFAULT NULL,\n `jfmfknowjepwhehpobjywyzlohomlxcp` int DEFAULT NULL,\n `obtdsbpjlnkximzwsteqgzlehqkdpbeg` int DEFAULT NULL,\n `hjzkrdjlkcjnqerrougmuqaozzxcifgp` int DEFAULT NULL,\n `frpuayvbzrxjgypkorgaztnyjgyvtnhn` int DEFAULT NULL,\n `mnfvhuifklwyasfttakyiwtwgogntaui` int DEFAULT NULL,\n `hoiubrtzoxfkacjvrqvtezycgoevcdko` int DEFAULT NULL,\n `sumffyiqkzjcvleeaiptzpmgmxrfbomf` int DEFAULT NULL,\n `qlwmiinawbhswacwpqqdwfueypxapzna` int DEFAULT NULL,\n `wmypcogelquatnowlvrbktgibcqslpbr` int DEFAULT NULL,\n PRIMARY KEY (`roxanjtnahjmktgfcehelycoiajqqebq`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"zrpahurtunmdbumraxjhiqvfumrdlnjn\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["roxanjtnahjmktgfcehelycoiajqqebq"],"columns":[{"name":"roxanjtnahjmktgfcehelycoiajqqebq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"mbaiuflvreyyywvwhtpkaktuwtvtlquo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uqnbcscnzwghjiqwjdduoupgogrnhdpm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nomefwdkivynrxedjkscjquougyoowiy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffeopoaqgmhufoanrktesycmnndootby","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypsqveaaqtvojgblvhsjgoiqkzcirgtl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqvzrvrqfeajbcmoycntevavxgfoddpr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"liutpvmdieywhegwgvxiejqjugtkempp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxtjhkcyfuqtukgggwzspxanfhquczmu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"seauomwxamzadjlpddzwgphmcrjjxcjp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hvjhnyqxdrvtntphqmmayeurqjroyurm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"veyrmheajdfawiwevmjgcminxynssynl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"odcyldrdffypbrjqkucsvtukeftrhoax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hxyfrgobjcccqhtqwdvfyrvcuvemqckq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ebdofjhynfuimlqtgpoaiyeeghemruor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ndrmqrhltyspwcobtfguborgceohaqvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmvwbmpgjjrhlkejeucubfgobhqufpdv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ttfutgtunaqmovartqkdagsiooptcxhy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ietreqmscwjbhwdtphzboffqhbobuuqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"crlfnkjkpkiypzkgwmtrktqplejvwgnc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyntwydpxldiubmjyuzkedlqdsuwhrqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ncmbkjqdpulwkermwubwsixndltvmxhe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wpznenicoazndaretrbkhibjdbmyxevs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"znmnkrxrzhinkzxldvuofoxqlfqmorsv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfokdvzbpcivyjlilrcvykkmwmvsoyaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gzgntsugaxdkqlvprupovfrgklnqtdbo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hzlzlkpfxhsjdtfjqbibanqttbhlyhfd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhlvlmzdbsqqrmiqectxpciorbpazald","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tvyuerpllzgbdyatxilswzdeqgykghqx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocvkwtlpxfudrzwtsjekglcogqwehwqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdxlvesrxbvfopnbgawlrjjjhweezwed","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qfnuljcahwkdgvfgyukcgbyfxlwwqsff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pgfutylxrjdegildmiyggzgolbwhdpmw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gugqwplhspseaoruanjqvojeptbilost","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"czhgtfehhacaikxvqifrrlmyvcldqfxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wavlvkdzxfduvjifatmqxlbzniirbuqo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wnbmkstdieaxfrhuehztlqgsluculwod","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edbtsgrwkwqgcmiivlyhewxkgaiiqrys","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pmffwmtwfeamgmkyehwhqzxeaqmzfqlo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdrwddlcazcmqcwjxlcvjkjzogwadaun","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nhkyxpgxjjaihvadbsvumpjeeekvranr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvvvvsftavllkfjujydvszasuncovxdp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wfniisuqmxbjscvqkbszqlkzjdovuzir","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hiibfncaedfrqsrgtaeqwhdhaaiqxqqs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ajlnuzkkzzwtavqojhcvfwihtivatzqu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hiatmmhatnbgmixabgpyjokznfgwgpqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bfrukcrqyqvbmxipktqkrxfvxoqeausc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ueimfjbrknunzuilmyxhaghnljtukcbf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohnffagzscnfqdfitesbyukbrvtbwomp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjzukdixlcnmjuaaxhkszgylrkenxrwb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxfwlqekdphazezwdestzicfqdyeibco","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkgojkidjmtrmdwchxrclosviyykigcu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ppzlziccdzegtsaeklblhzndtbkatxug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qtdzeaenosmghvguqmeyphwelqfkhtud","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ilgyvhitkgrikxrfytyqmxfemkgnbegt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lhhywwtkyqleyyjjyayaqgwsledzgani","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hwfzyczbhldvrpycuqnotoyoebetbavj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbphhvpocrdmmdvxzjphwwjuvakptldy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"upoimkyzcdbvpudgpawwzentguuklrjq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"whtuieqbeibfdgwftpupnojixrbwroyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"awsvyluxzojzzvqbhsuoznhkwabofzvr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pafjwrisvkaihnxvizqgukodgreqevbs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fbkgkemmjcxupktohvewhwxwhiktfmfh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ocjzfdveptybfparybsngvupiavfeoga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krakignjkcjtacnzzxdlpnmsalzubowe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xdqweirhuoapbeytoowlioylztrevbkq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pftjqacwwhditfgdzdguulgxszmlgymz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sptkbtzndvthkjzglbiiaxzowvduunwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzxhxbosvvyxdncobzcbervxrlodwzwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxtarhzlfhfhzyobpoqjjzvpvdgslors","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yzqtnekkmoueoxpirxughktqtthdzxxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cdnlofwbyefmgfrmhvpkextxsksslvbi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kqkraciroaebbrotygmxwmxkcpndrhze","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"riwmcqlufnhtswdztknxlogwckougoqv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwlwluxiiwsixixusrqiqfmibjtpaqbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uvxwupkabnotbrwsgyoaiekhldaabazp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjolnnjifetajrcveeyhxwdoeecavkvb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qhicttemkonovhglvxuadcouvyzasett","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wuicjozijswcrncanvkbxhlheguiphwq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"corlidbeeneapjbarkupystzalggauri","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zlnvrbgggjzkmipllyxdnwjsnlhpajug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nxemxvsnytuffvjxwtllbokceqvdiler","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hnapwuarkzeyfwbjxduvvvirgiuqasbe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sajkhdwfcjjkelwdazngiclmlovmhcvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zoggxjmnqallzqfykesrpmrwxoilfkow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"frqjcezhhftysoxiynzvwltwdmzztcpc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kiaitasfwwjbxoifrgmbxcpnwzejhwtn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zyklwscbeuefmbgfivpcydeqocrhgkvs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yyultlzifxrdhufnzvypjdzpzmlwblla","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rlthasgfmixuyrsqodxpviutejmisrzu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qocitaaixjyqhtflgficbmuitfdxewue","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jfmfknowjepwhehpobjywyzlohomlxcp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obtdsbpjlnkximzwsteqgzlehqkdpbeg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjzkrdjlkcjnqerrougmuqaozzxcifgp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"frpuayvbzrxjgypkorgaztnyjgyvtnhn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnfvhuifklwyasfttakyiwtwgogntaui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hoiubrtzoxfkacjvrqvtezycgoevcdko","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sumffyiqkzjcvleeaiptzpmgmxrfbomf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qlwmiinawbhswacwpqqdwfueypxapzna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmypcogelquatnowlvrbktgibcqslpbr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842673,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842673245,"databaseName":"models_schema","ddl":"CREATE TABLE `ztpttwzzkgyxsjgpkltayosfjkjwfgok` (\n `sjijcqukjwqtrlepctptghamkgvqlccm` int NOT NULL,\n `ikgcgjgvbjqnscveqxyhqeikwmtpghaa` int DEFAULT NULL,\n `fpyhjdwcmzuggsabjeckrtscmhddjtvp` int DEFAULT NULL,\n `mvjjezpnswfpqlebjcyxfcziplofbjuh` int DEFAULT NULL,\n `mgktzobdmzxlooiudknqprnuleypyuma` int DEFAULT NULL,\n `cjfoaylbssctwzwsrrnvxnlhgzupvthq` int DEFAULT NULL,\n `akzihcollnevtzwhhdkpcdvwbwolhshq` int DEFAULT NULL,\n `kfpnvumgvuoclgmtyqgonvnhaxyuvqok` int DEFAULT NULL,\n `civtzxsqmkljvbgotzokhsrxzvayetxd` int DEFAULT NULL,\n `gcdrbavehknzogtwqjjakgrzszsxgdme` int DEFAULT NULL,\n `ambjhmpyvkcnzilzsbwtdqrrbrrfzubg` int DEFAULT NULL,\n `hlaaeagbwlvoztgjaenqpmxnlapbqlgl` int DEFAULT NULL,\n `optgeetfopshsbecrpebojdrxfifvval` int DEFAULT NULL,\n `axjtwrtqmvkbjojkskxijrfnynagquza` int DEFAULT NULL,\n `cdvkszwopqliftxziibkhpnqtzxmguvl` int DEFAULT NULL,\n `ltijkxfkxlmntizixtwmhtrqaxcyosae` int DEFAULT NULL,\n `fyixelstaxewnbewiymnhcujrdhgizqm` int DEFAULT NULL,\n `fsouugetrnazvhuhafefsriylguwoegs` int DEFAULT NULL,\n `qencallxftqpzdnocjkjuevltlzkpjmb` int DEFAULT NULL,\n `kaytxeifswtjqcxvhnasnmdosyewwnax` int DEFAULT NULL,\n `vjubulxkuhvclnpolchgdwnwtmeauvin` int DEFAULT NULL,\n `saxsjtvbiosqmncrvurbcdsuyhbkhrvc` int DEFAULT NULL,\n `eulvcejwggwyrmfolieovtvcmpqruutj` int DEFAULT NULL,\n `ncgaxogousrsalbwngodvzdiztxladix` int DEFAULT NULL,\n `dwkncjqitggnoqbcwiunyhisiwfsyizc` int DEFAULT NULL,\n `rodlxiofhhngrxczpqeuudmuiafsujav` int DEFAULT NULL,\n `qnpedpmydzrhipcmxodzpfltthgrnfbp` int DEFAULT NULL,\n `skeldnntuftdqhjkdmkzduuepyffyrzj` int DEFAULT NULL,\n `fhoxznryehjrjjdekmputmujhzrxhtvw` int DEFAULT NULL,\n `spaummsredvcfjabslxapmgwaegruwld` int DEFAULT NULL,\n `rrgfnusbbbxprmeknyeevrfomcamnlxx` int DEFAULT NULL,\n `uiveahqsjeypvmdgwgriimozlqispfil` int DEFAULT NULL,\n `ofbjypvfoelyrqxlesaerqchgggtorid` int DEFAULT NULL,\n `zapogdjppgogzkkuhkmexrmmqiimosti` int DEFAULT NULL,\n `iovdhqclbboesfiwufiniyeqrglzfdka` int DEFAULT NULL,\n `bvownwrjwxkyxkunefqgpebzbindhmru` int DEFAULT NULL,\n `eqfafezkfacfjjdrcrxwxvlpnkyxolau` int DEFAULT NULL,\n `dosedpizxnuaphotykrlmkvjdrzrvjpx` int DEFAULT NULL,\n `ouriduowmaradzpfsuvhawnfppzcqdpm` int DEFAULT NULL,\n `xvkeoxromlfboggugqxheovagrerqylq` int DEFAULT NULL,\n `deawdabhcwxldoraohnoebipzwvispup` int DEFAULT NULL,\n `euftckrumswxoxryrkwhyxqwoqpfbwkp` int DEFAULT NULL,\n `fpmgncknukdkiuyifixnradskwdmonwe` int DEFAULT NULL,\n `aauekcyfxlsprlbpujvpgvpmlootdqoo` int DEFAULT NULL,\n `hociaqkmppboodscxdgnfboozjebatme` int DEFAULT NULL,\n `cwkysagutdylpmvzlnvophdqbxgacqqi` int DEFAULT NULL,\n `ycizeloysnbzbuvadzlhklngpfmkqciu` int DEFAULT NULL,\n `xmcmbfclrkwcqdjvcccjywsgahirjudr` int DEFAULT NULL,\n `jmxnaajpptplsmgkgxrxedsmcghkdqit` int DEFAULT NULL,\n `hdkapdjfhwjmclmafnrpdirwbelhifjs` int DEFAULT NULL,\n `saatddjbvgzcmvnygcxnztkikkqnsaqt` int DEFAULT NULL,\n `dgbikmhfwyisikqhkaueqedmvcofozht` int DEFAULT NULL,\n `csdcazpngkgliqsmwteirqxdspwquibf` int DEFAULT NULL,\n `egielinbeksjfpghpymjsogikokfygor` int DEFAULT NULL,\n `ihqqwqvblsfdbexxflieockqilgspgod` int DEFAULT NULL,\n `eaxoajppthldjqjpmgltpojgslncrbyt` int DEFAULT NULL,\n `xsnbqiayjgkppahxvjkuzzyyrzfimdvg` int DEFAULT NULL,\n `bibzdfekbcofkufytmkcxxccnhjtajpn` int DEFAULT NULL,\n `wyjwvlthesogbvgkejiklqpgejckkkmt` int DEFAULT NULL,\n `oukhrsxwkpsoqzoqvanwdqemjptihrqp` int DEFAULT NULL,\n `mzomrzxlzpihjkxqkcysozkmfbhzikyj` int DEFAULT NULL,\n `sglugvdcrelqzafpjssvxkaqeelfijeq` int DEFAULT NULL,\n `fgmcztvzzdezajplrvqfnktdxzgznske` int DEFAULT NULL,\n `umozqlpumuovckrfslrrjrbtxgqbvybh` int DEFAULT NULL,\n `jlaqqaozvcfvtxbrihfsusutrlpdsltt` int DEFAULT NULL,\n `rrtxvcatknebycxrurhdormotujsbzrl` int DEFAULT NULL,\n `jcmfrwijodqgufmyqgvsotliynjrqxna` int DEFAULT NULL,\n `zieahgnvmnmxczurdbcbfiphndueapqn` int DEFAULT NULL,\n `kblugegsoagcfypxekvrekmsphslvyur` int DEFAULT NULL,\n `epvahmlsraixblgzmnxkaqkanytcebcw` int DEFAULT NULL,\n `hohvihabkztplmqvhstxnqfvgvhpclht` int DEFAULT NULL,\n `ewlumzmaiosytnmyzkezqzstlqtxqzgb` int DEFAULT NULL,\n `yhiwmxcepbocezfyfoxxosjzfplcvfqg` int DEFAULT NULL,\n `eftfmjdwwwxhooafmmqpkjiexsyqfvmg` int DEFAULT NULL,\n `egewgytulumfpuudaunqmhwuoqfufued` int DEFAULT NULL,\n `vjgnmxfzizrjjkeuvtvlypdlcuzpoipr` int DEFAULT NULL,\n `smirnwnsundlvyvacthytmqomtfmilyf` int DEFAULT NULL,\n `cwsuzuarmzpobmqjwnnkqhcrsnrcrgen` int DEFAULT NULL,\n `lkntwprtmcwyenvpakzcudrptqlcuhry` int DEFAULT NULL,\n `gggyogynmjyuefgeobmlubsrodgyefsz` int DEFAULT NULL,\n `iiobqdwjdtfjsfeqlrkeglhgkosxyhbw` int DEFAULT NULL,\n `ieeseangssdqizmxnamgvjcfiozylzca` int DEFAULT NULL,\n `ecdnfyngypvlizdfvwhbwkymbaysvcvh` int DEFAULT NULL,\n `nuefojivfemmtmkfaabmfblebpfngmho` int DEFAULT NULL,\n `ylvfecpkefmvmpdezbxmkrkzuqumgknp` int DEFAULT NULL,\n `vpsjpwnbtccuxulywbsdglbzggruxwzs` int DEFAULT NULL,\n `qjwnhqyldbultquqrgaxeikslooxhbxs` int DEFAULT NULL,\n `dxgvfhzjpskfpucrxfowrzygqffsmioo` int DEFAULT NULL,\n `wyjbdoyjlbmardgnkavtuuzpfhkdmpgs` int DEFAULT NULL,\n `vcvlwxdlncciihyqnnhyvkzewjybuert` int DEFAULT NULL,\n `smsowyazoslcipjvmilfaahxupzelejr` int DEFAULT NULL,\n `xaqplfwhzkxsicusgbwfnpaamdqtiftb` int DEFAULT NULL,\n `jdwalkjxzrwuhdwtnmbrqreyqlauptgt` int DEFAULT NULL,\n `taonfhvstqsevbfjqoroxfqolyjjaqtm` int DEFAULT NULL,\n `talogfzffepmlnsdvacdsmnkfnwwkodm` int DEFAULT NULL,\n `cvtcijensexoloidkkohgcedzoijjuia` int DEFAULT NULL,\n `lzbfzksaqqyzfwszjsopemvlkxwaelcq` int DEFAULT NULL,\n `cexbcbvwiubqjalmnaqchchfubhoisjq` int DEFAULT NULL,\n `wcmyqgtukocitnfvtiomcthbklbpuled` int DEFAULT NULL,\n `msbmxbpfqttdaglvzasswxijclitdscq` int DEFAULT NULL,\n PRIMARY KEY (`sjijcqukjwqtrlepctptghamkgvqlccm`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"ztpttwzzkgyxsjgpkltayosfjkjwfgok\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["sjijcqukjwqtrlepctptghamkgvqlccm"],"columns":[{"name":"sjijcqukjwqtrlepctptghamkgvqlccm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"ikgcgjgvbjqnscveqxyhqeikwmtpghaa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpyhjdwcmzuggsabjeckrtscmhddjtvp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvjjezpnswfpqlebjcyxfcziplofbjuh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mgktzobdmzxlooiudknqprnuleypyuma","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjfoaylbssctwzwsrrnvxnlhgzupvthq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akzihcollnevtzwhhdkpcdvwbwolhshq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kfpnvumgvuoclgmtyqgonvnhaxyuvqok","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"civtzxsqmkljvbgotzokhsrxzvayetxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gcdrbavehknzogtwqjjakgrzszsxgdme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ambjhmpyvkcnzilzsbwtdqrrbrrfzubg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hlaaeagbwlvoztgjaenqpmxnlapbqlgl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"optgeetfopshsbecrpebojdrxfifvval","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axjtwrtqmvkbjojkskxijrfnynagquza","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cdvkszwopqliftxziibkhpnqtzxmguvl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ltijkxfkxlmntizixtwmhtrqaxcyosae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fyixelstaxewnbewiymnhcujrdhgizqm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fsouugetrnazvhuhafefsriylguwoegs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qencallxftqpzdnocjkjuevltlzkpjmb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kaytxeifswtjqcxvhnasnmdosyewwnax","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjubulxkuhvclnpolchgdwnwtmeauvin","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"saxsjtvbiosqmncrvurbcdsuyhbkhrvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eulvcejwggwyrmfolieovtvcmpqruutj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ncgaxogousrsalbwngodvzdiztxladix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwkncjqitggnoqbcwiunyhisiwfsyizc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rodlxiofhhngrxczpqeuudmuiafsujav","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qnpedpmydzrhipcmxodzpfltthgrnfbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"skeldnntuftdqhjkdmkzduuepyffyrzj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhoxznryehjrjjdekmputmujhzrxhtvw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"spaummsredvcfjabslxapmgwaegruwld","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrgfnusbbbxprmeknyeevrfomcamnlxx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uiveahqsjeypvmdgwgriimozlqispfil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ofbjypvfoelyrqxlesaerqchgggtorid","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zapogdjppgogzkkuhkmexrmmqiimosti","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iovdhqclbboesfiwufiniyeqrglzfdka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bvownwrjwxkyxkunefqgpebzbindhmru","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eqfafezkfacfjjdrcrxwxvlpnkyxolau","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dosedpizxnuaphotykrlmkvjdrzrvjpx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ouriduowmaradzpfsuvhawnfppzcqdpm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xvkeoxromlfboggugqxheovagrerqylq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"deawdabhcwxldoraohnoebipzwvispup","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"euftckrumswxoxryrkwhyxqwoqpfbwkp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fpmgncknukdkiuyifixnradskwdmonwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aauekcyfxlsprlbpujvpgvpmlootdqoo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hociaqkmppboodscxdgnfboozjebatme","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwkysagutdylpmvzlnvophdqbxgacqqi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ycizeloysnbzbuvadzlhklngpfmkqciu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xmcmbfclrkwcqdjvcccjywsgahirjudr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jmxnaajpptplsmgkgxrxedsmcghkdqit","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hdkapdjfhwjmclmafnrpdirwbelhifjs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"saatddjbvgzcmvnygcxnztkikkqnsaqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dgbikmhfwyisikqhkaueqedmvcofozht","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"csdcazpngkgliqsmwteirqxdspwquibf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"egielinbeksjfpghpymjsogikokfygor","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ihqqwqvblsfdbexxflieockqilgspgod","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eaxoajppthldjqjpmgltpojgslncrbyt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsnbqiayjgkppahxvjkuzzyyrzfimdvg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bibzdfekbcofkufytmkcxxccnhjtajpn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyjwvlthesogbvgkejiklqpgejckkkmt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oukhrsxwkpsoqzoqvanwdqemjptihrqp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzomrzxlzpihjkxqkcysozkmfbhzikyj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sglugvdcrelqzafpjssvxkaqeelfijeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fgmcztvzzdezajplrvqfnktdxzgznske","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"umozqlpumuovckrfslrrjrbtxgqbvybh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlaqqaozvcfvtxbrihfsusutrlpdsltt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rrtxvcatknebycxrurhdormotujsbzrl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jcmfrwijodqgufmyqgvsotliynjrqxna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zieahgnvmnmxczurdbcbfiphndueapqn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kblugegsoagcfypxekvrekmsphslvyur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"epvahmlsraixblgzmnxkaqkanytcebcw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hohvihabkztplmqvhstxnqfvgvhpclht","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ewlumzmaiosytnmyzkezqzstlqtxqzgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yhiwmxcepbocezfyfoxxosjzfplcvfqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"eftfmjdwwwxhooafmmqpkjiexsyqfvmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"egewgytulumfpuudaunqmhwuoqfufued","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjgnmxfzizrjjkeuvtvlypdlcuzpoipr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smirnwnsundlvyvacthytmqomtfmilyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cwsuzuarmzpobmqjwnnkqhcrsnrcrgen","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lkntwprtmcwyenvpakzcudrptqlcuhry","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gggyogynmjyuefgeobmlubsrodgyefsz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iiobqdwjdtfjsfeqlrkeglhgkosxyhbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ieeseangssdqizmxnamgvjcfiozylzca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecdnfyngypvlizdfvwhbwkymbaysvcvh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nuefojivfemmtmkfaabmfblebpfngmho","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ylvfecpkefmvmpdezbxmkrkzuqumgknp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vpsjpwnbtccuxulywbsdglbzggruxwzs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjwnhqyldbultquqrgaxeikslooxhbxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxgvfhzjpskfpucrxfowrzygqffsmioo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wyjbdoyjlbmardgnkavtuuzpfhkdmpgs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vcvlwxdlncciihyqnnhyvkzewjybuert","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"smsowyazoslcipjvmilfaahxupzelejr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xaqplfwhzkxsicusgbwfnpaamdqtiftb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jdwalkjxzrwuhdwtnmbrqreyqlauptgt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"taonfhvstqsevbfjqoroxfqolyjjaqtm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"talogfzffepmlnsdvacdsmnkfnwwkodm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cvtcijensexoloidkkohgcedzoijjuia","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzbfzksaqqyzfwszjsopemvlkxwaelcq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cexbcbvwiubqjalmnaqchchfubhoisjq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wcmyqgtukocitnfvtiomcthbklbpuled","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"msbmxbpfqttdaglvzasswxijclitdscq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842673,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842673276,"databaseName":"models_schema","ddl":"CREATE TABLE `zvizamagjrkoamcwmhgradfhuizonrdx` (\n `tglxgxetqflgsyzersuqhnftbdccfjgm` int NOT NULL,\n `oguzqiyekzbsbmmonbjcypfrnsvsmldk` int DEFAULT NULL,\n `yobqsanjhpnlubgbqmgciopnmigudfpm` int DEFAULT NULL,\n `rtdjvhvufbterstdvttfmtqxdsmkhjcl` int DEFAULT NULL,\n `kbbgjlytoyhrhpjcaaxcnssiohobjrst` int DEFAULT NULL,\n `nqrwdbbfxmdjoehvkpkpbmupvrqfnhqt` int DEFAULT NULL,\n `lbgarzlmpovlmyygjquiyanqnmruqvoz` int DEFAULT NULL,\n `haprigyvhtekbdbsiywruguslpqnzeeo` int DEFAULT NULL,\n `fzjhjjsrqurawdysrzserxgfdwiuvywe` int DEFAULT NULL,\n `rstjkavwouazikbmfoedtdefcwbxkhtc` int DEFAULT NULL,\n `lfdwyjmdoqbcmzuayruczkicoeevbcii` int DEFAULT NULL,\n `vcligyprasbrovrycqhnysiaxwbsttur` int DEFAULT NULL,\n `kgbjmtyqekofbqinxeegmvysvdgxxrff` int DEFAULT NULL,\n `kbnrqfueyidbjdhnxzjjlxmipbkwwqex` int DEFAULT NULL,\n `ffmthqibliklwtniqkaroswtzzvxvgev` int DEFAULT NULL,\n `aqhoqzminksfvraukiwtqxbubszxwgsz` int DEFAULT NULL,\n `otzvmwnxcikemhemfjmajhgckjtimtjy` int DEFAULT NULL,\n `hnxrrvobyodrkzogqgvyuhycghrsvzrb` int DEFAULT NULL,\n `dwslnycgbgyeoykdbgcanrsqtkohzzyz` int DEFAULT NULL,\n `cipsxodhklmzfoofwdmpflumclhhtvej` int DEFAULT NULL,\n `wkoddepehgdlbzhuiutedoqpgxpndyfu` int DEFAULT NULL,\n `wacpxybkjylshfxfyragskwjrfraakdm` int DEFAULT NULL,\n `vzqwnogjgnsgsysnfsvpjdjnalttzonh` int DEFAULT NULL,\n `gsebaixlkbvdbrvklesfhuogqhuovpbc` int DEFAULT NULL,\n `yqpnymzyllsmcdfaktubhwihmlskudvy` int DEFAULT NULL,\n `tiwjpiigptdetcghfdstvymdjrvdflry` int DEFAULT NULL,\n `bzjzsapzgexhupfjcfrsbfhbvbffzrnc` int DEFAULT NULL,\n `rihtfzzrfkvymafyqagxedwgpqrofgqb` int DEFAULT NULL,\n `sbgxxizocowzcfvcbgmperkhramoqrqb` int DEFAULT NULL,\n `vfnquxstzkgmvrhymaripjkdspwpsxgp` int DEFAULT NULL,\n `ibfxcfgorvzcqsxaqjiteeamwtiscxxz` int DEFAULT NULL,\n `wmiwlzroboyiaeueetghfqufvrjevkrx` int DEFAULT NULL,\n `oazpswjlztyefevehserojtoocqbxoey` int DEFAULT NULL,\n `acxwpbiivtepuczfpdqzuynrffadoypc` int DEFAULT NULL,\n `sdkccppzrhxgtsimtwbynguyyxmendtg` int DEFAULT NULL,\n `hqfktvsgxvlccxtpbuhxuqkchezosgij` int DEFAULT NULL,\n `susdzemtgeegmdipevsawlvdqapcdukf` int DEFAULT NULL,\n `xnsvjcqdtqtprcxpuowvwzmlvbejuxey` int DEFAULT NULL,\n `liyyecipafvaykwgcphfeiizbdutiqrf` int DEFAULT NULL,\n `edryouorzbbcmlhpcpyfynyadtycpqyo` int DEFAULT NULL,\n `yoaobouxlneilknwzytjwdjaidptoizy` int DEFAULT NULL,\n `lzmwzshqswolvbaiwxivgtgepdmejpre` int DEFAULT NULL,\n `xkzynphhqenchemlirzvnowsnivwgknn` int DEFAULT NULL,\n `xtxhdudxgjrlbnrrfijtiubcuqqkzppf` int DEFAULT NULL,\n `axpylqknanveogyribkzmmudgeeebmrb` int DEFAULT NULL,\n `yqkxyveuvrumzvaeijijxfmvnapjewtv` int DEFAULT NULL,\n `vsplelbaqwgfymxgumtomkvxpyjnfzdd` int DEFAULT NULL,\n `qngyhaplvlulvkeelldjzawulamkdtwa` int DEFAULT NULL,\n `vqusnsefqlcbzjfsdcwxnkreqmzehcly` int DEFAULT NULL,\n `vencghbcizjjointyapfhkoqjpwudwxw` int DEFAULT NULL,\n `fjpayjwcwlewxuoajoxhwmvsuygzsqvs` int DEFAULT NULL,\n `jgpkdcgfjxpmfbvzktkcunnjseqiweyy` int DEFAULT NULL,\n `waicmkujnsxnwwzesyrcrhiylgfrsiar` int DEFAULT NULL,\n `rhvjbmjfyxjmlzlozhbfdovaokzviaap` int DEFAULT NULL,\n `rakhtobwrbfudsvdfkibapmohhuiyabg` int DEFAULT NULL,\n `bflzxmifvordfqyvpoufmrcjswpmpuxs` int DEFAULT NULL,\n `wdjzrnflvvkcbwpjcckqmzzbgymlzhoe` int DEFAULT NULL,\n `vqjbbvecksjnfvwgevcrvmbhoccylkmo` int DEFAULT NULL,\n `jycaaessyjuidcovgceisdzwussmtyao` int DEFAULT NULL,\n `vnjmgeyfaduxnakkorezkidqcqyjhevz` int DEFAULT NULL,\n `zunobqzybzcqmyndpdibdjebojsewoke` int DEFAULT NULL,\n `nkbvbghnmyepsxozxefhdjcbznveaamx` int DEFAULT NULL,\n `rubwxuvmeavnepthrgnbdwwfdfmsjmug` int DEFAULT NULL,\n `bdlkyewcmxqufwgbtcmlufttlpqqcnyb` int DEFAULT NULL,\n `zcucpbnrryyngncmpxtlqiuklhfgsepq` int DEFAULT NULL,\n `iscyiyunevdjziwaqbzfqqhikjojwktq` int DEFAULT NULL,\n `jrejywizwmxmaekhtzcxgbqgxyadnceq` int DEFAULT NULL,\n `hufcowvyozftgaywywiocqsgjszwffdf` int DEFAULT NULL,\n `imobkbcdcffgxakmesmrmmqqdxzunqxy` int DEFAULT NULL,\n `nseiqdjlsiolcmamqthbwzisfzbpiyqa` int DEFAULT NULL,\n `xsbwyjqffcgvkrutaqjzkqsbwcmbdeil` int DEFAULT NULL,\n `gyqvuggpwdhddxnvqpgumkyfmlfcsdrn` int DEFAULT NULL,\n `mftqljqrynonasgdyxgpmrvqxpnuimlh` int DEFAULT NULL,\n `cxlmejunfifzubpgzuzpkevtpkwoqvfl` int DEFAULT NULL,\n `oagnhperiqumiyqgtmalfowerausunta` int DEFAULT NULL,\n `roouwnyydlxyhxmmqjskccbtxgxtfsis` int DEFAULT NULL,\n `igeucqpdwequbisshjxwvjzrofsvzqwd` int DEFAULT NULL,\n `qakgmzotuxqonkgnyalyvolrseuqygow` int DEFAULT NULL,\n `astxmbskigabvhyvomkprdpuhjemuwds` int DEFAULT NULL,\n `pfpqfgrxzfsurpmncowfpctkvcedfror` int DEFAULT NULL,\n `hmgdzilscntqcdlpdndzvflvoozktkfg` int DEFAULT NULL,\n `oadjtnuvrdmeyueapekcxurprifxvqaz` int DEFAULT NULL,\n `lrswtrzesisysmbxolumwqivojubsdgb` int DEFAULT NULL,\n `tzsywfucqjuqtmfnjvtenksqozgwibik` int DEFAULT NULL,\n `puwrebgesppratljfxnbxpfghmukjfea` int DEFAULT NULL,\n `jarawgwegxfhrzqbryqmqkanfsxpuolc` int DEFAULT NULL,\n `yquhfnohmlowgxzjcxtwqrxrbigceptn` int DEFAULT NULL,\n `kdgbadzdqbcrhozuleyzkakdbhcilyyf` int DEFAULT NULL,\n `iqltpmwiulcbrsqwukyfdetlliwxdizx` int DEFAULT NULL,\n `gmqxlemhafkbfedilzicparbunobpppt` int DEFAULT NULL,\n `gsdpcdvdtspfbgpvluqgeqwjkfcoueev` int DEFAULT NULL,\n `afrizjxlfbwjfzahnkfqwzthaywfpbqr` int DEFAULT NULL,\n `svbgpypuotlvgvtkzllekhbjvsyybnmm` int DEFAULT NULL,\n `mdugbwmkkiijfegbpfjjemnplkkuljkw` int DEFAULT NULL,\n `thqmoreudkwznnqxbdnkuphtuedwnobp` int DEFAULT NULL,\n `parwbtyahtjjfnoasjkaodvjatuuweci` int DEFAULT NULL,\n `dcanembwbcrvvvzqdaiaullbvcnobjvc` int DEFAULT NULL,\n `femydlakikisxzfujvxnbicdspsbjzja` int DEFAULT NULL,\n `nbhnmrqkyucataarzmsxnhfjcreozzpg` int DEFAULT NULL,\n `ziulnlrssqryojsehpzihitbayumrgpo` int DEFAULT NULL,\n PRIMARY KEY (`tglxgxetqflgsyzersuqhnftbdccfjgm`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"zvizamagjrkoamcwmhgradfhuizonrdx\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["tglxgxetqflgsyzersuqhnftbdccfjgm"],"columns":[{"name":"tglxgxetqflgsyzersuqhnftbdccfjgm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"oguzqiyekzbsbmmonbjcypfrnsvsmldk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yobqsanjhpnlubgbqmgciopnmigudfpm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rtdjvhvufbterstdvttfmtqxdsmkhjcl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kbbgjlytoyhrhpjcaaxcnssiohobjrst","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nqrwdbbfxmdjoehvkpkpbmupvrqfnhqt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lbgarzlmpovlmyygjquiyanqnmruqvoz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"haprigyvhtekbdbsiywruguslpqnzeeo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fzjhjjsrqurawdysrzserxgfdwiuvywe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rstjkavwouazikbmfoedtdefcwbxkhtc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lfdwyjmdoqbcmzuayruczkicoeevbcii","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vcligyprasbrovrycqhnysiaxwbsttur","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kgbjmtyqekofbqinxeegmvysvdgxxrff","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kbnrqfueyidbjdhnxzjjlxmipbkwwqex","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffmthqibliklwtniqkaroswtzzvxvgev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aqhoqzminksfvraukiwtqxbubszxwgsz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"otzvmwnxcikemhemfjmajhgckjtimtjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hnxrrvobyodrkzogqgvyuhycghrsvzrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dwslnycgbgyeoykdbgcanrsqtkohzzyz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cipsxodhklmzfoofwdmpflumclhhtvej","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wkoddepehgdlbzhuiutedoqpgxpndyfu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wacpxybkjylshfxfyragskwjrfraakdm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vzqwnogjgnsgsysnfsvpjdjnalttzonh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsebaixlkbvdbrvklesfhuogqhuovpbc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqpnymzyllsmcdfaktubhwihmlskudvy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tiwjpiigptdetcghfdstvymdjrvdflry","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bzjzsapzgexhupfjcfrsbfhbvbffzrnc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rihtfzzrfkvymafyqagxedwgpqrofgqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sbgxxizocowzcfvcbgmperkhramoqrqb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vfnquxstzkgmvrhymaripjkdspwpsxgp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ibfxcfgorvzcqsxaqjiteeamwtiscxxz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmiwlzroboyiaeueetghfqufvrjevkrx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oazpswjlztyefevehserojtoocqbxoey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acxwpbiivtepuczfpdqzuynrffadoypc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sdkccppzrhxgtsimtwbynguyyxmendtg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hqfktvsgxvlccxtpbuhxuqkchezosgij","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"susdzemtgeegmdipevsawlvdqapcdukf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xnsvjcqdtqtprcxpuowvwzmlvbejuxey","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"liyyecipafvaykwgcphfeiizbdutiqrf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"edryouorzbbcmlhpcpyfynyadtycpqyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yoaobouxlneilknwzytjwdjaidptoizy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzmwzshqswolvbaiwxivgtgepdmejpre","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xkzynphhqenchemlirzvnowsnivwgknn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xtxhdudxgjrlbnrrfijtiubcuqqkzppf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"axpylqknanveogyribkzmmudgeeebmrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yqkxyveuvrumzvaeijijxfmvnapjewtv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vsplelbaqwgfymxgumtomkvxpyjnfzdd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qngyhaplvlulvkeelldjzawulamkdtwa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqusnsefqlcbzjfsdcwxnkreqmzehcly","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vencghbcizjjointyapfhkoqjpwudwxw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fjpayjwcwlewxuoajoxhwmvsuygzsqvs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jgpkdcgfjxpmfbvzktkcunnjseqiweyy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"waicmkujnsxnwwzesyrcrhiylgfrsiar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rhvjbmjfyxjmlzlozhbfdovaokzviaap","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rakhtobwrbfudsvdfkibapmohhuiyabg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bflzxmifvordfqyvpoufmrcjswpmpuxs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wdjzrnflvvkcbwpjcckqmzzbgymlzhoe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vqjbbvecksjnfvwgevcrvmbhoccylkmo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jycaaessyjuidcovgceisdzwussmtyao","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vnjmgeyfaduxnakkorezkidqcqyjhevz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zunobqzybzcqmyndpdibdjebojsewoke","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkbvbghnmyepsxozxefhdjcbznveaamx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rubwxuvmeavnepthrgnbdwwfdfmsjmug","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bdlkyewcmxqufwgbtcmlufttlpqqcnyb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zcucpbnrryyngncmpxtlqiuklhfgsepq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iscyiyunevdjziwaqbzfqqhikjojwktq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jrejywizwmxmaekhtzcxgbqgxyadnceq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hufcowvyozftgaywywiocqsgjszwffdf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"imobkbcdcffgxakmesmrmmqqdxzunqxy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nseiqdjlsiolcmamqthbwzisfzbpiyqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xsbwyjqffcgvkrutaqjzkqsbwcmbdeil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gyqvuggpwdhddxnvqpgumkyfmlfcsdrn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mftqljqrynonasgdyxgpmrvqxpnuimlh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cxlmejunfifzubpgzuzpkevtpkwoqvfl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oagnhperiqumiyqgtmalfowerausunta","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"roouwnyydlxyhxmmqjskccbtxgxtfsis","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"igeucqpdwequbisshjxwvjzrofsvzqwd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qakgmzotuxqonkgnyalyvolrseuqygow","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"astxmbskigabvhyvomkprdpuhjemuwds","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pfpqfgrxzfsurpmncowfpctkvcedfror","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hmgdzilscntqcdlpdndzvflvoozktkfg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oadjtnuvrdmeyueapekcxurprifxvqaz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lrswtrzesisysmbxolumwqivojubsdgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tzsywfucqjuqtmfnjvtenksqozgwibik","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"puwrebgesppratljfxnbxpfghmukjfea","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jarawgwegxfhrzqbryqmqkanfsxpuolc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yquhfnohmlowgxzjcxtwqrxrbigceptn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kdgbadzdqbcrhozuleyzkakdbhcilyyf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iqltpmwiulcbrsqwukyfdetlliwxdizx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gmqxlemhafkbfedilzicparbunobpppt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gsdpcdvdtspfbgpvluqgeqwjkfcoueev","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"afrizjxlfbwjfzahnkfqwzthaywfpbqr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"svbgpypuotlvgvtkzllekhbjvsyybnmm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdugbwmkkiijfegbpfjjemnplkkuljkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"thqmoreudkwznnqxbdnkuphtuedwnobp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"parwbtyahtjjfnoasjkaodvjatuuweci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dcanembwbcrvvvzqdaiaullbvcnobjvc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"femydlakikisxzfujvxnbicdspsbjzja","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbhnmrqkyucataarzmsxnhfjcreozzpg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ziulnlrssqryojsehpzihitbayumrgpo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842673,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842673305,"databaseName":"models_schema","ddl":"CREATE TABLE `zxhltxrsmhtsyvtgrlcsbvouuzpmoisc` (\n `rlpjndrfvfzjhogvdkitpfuotzqqwxrj` int NOT NULL,\n `jfghdusmeijnxlghtmzysjyopbtopwuq` int DEFAULT NULL,\n `yyqmdhapoblcjtqucpouaeqemmnnbjzy` int DEFAULT NULL,\n `iwbponzxaexvthfpvbvqjtiokkjqmmdi` int DEFAULT NULL,\n `ngwckvrwqlpqlpvytrxrflxsflhrtire` int DEFAULT NULL,\n `hffrfddjrzeumkmroetetqmtqicopsnw` int DEFAULT NULL,\n `uwlbqivkeefuebxnbfmoefgsjhitiyuq` int DEFAULT NULL,\n `kptedffnldsyoomyrenkunshknheggcd` int DEFAULT NULL,\n `zvetntckneondtpdhfzmumzalqbvhbnz` int DEFAULT NULL,\n `nlshlbfoojwxmyoyoenkbfnjpnhtgopk` int DEFAULT NULL,\n `yfjxpoxrycggvhddskbrziduquyudldq` int DEFAULT NULL,\n `nbfrjjibupnffugipbdnprypfjymheny` int DEFAULT NULL,\n `flvohflfdyzqssjosuzlnffocnpveoxa` int DEFAULT NULL,\n `yfkbflpzwgjrjhzvwgczkwqucybysroo` int DEFAULT NULL,\n `ffrrykilbmrsxbvclfndgjbdkwggsyhu` int DEFAULT NULL,\n `jqrezzqzyoqrjiqefcwlaafocrsoswgq` int DEFAULT NULL,\n `fvyhmlwqbanhzguokonklpidskmshwku` int DEFAULT NULL,\n `cdlmpuizwqmfjglrwikfieyymkyoygeq` int DEFAULT NULL,\n `lutlbrepsaicgxqnagkuwtnoyqiqpkxp` int DEFAULT NULL,\n `cjrnegxfpaofzlwbpbsrrfnlifitexdd` int DEFAULT NULL,\n `lzfpscxgqnhqnbxwgulsqouisrmvjxre` int DEFAULT NULL,\n `poigvxvwyvrrvzuxluayfffcwpcqcpyn` int DEFAULT NULL,\n `hjlldfzrrzldwuhkyzfbldtzdoizqplh` int DEFAULT NULL,\n `ohrxcmwyvrbjipepnkdmczqfjmjitxle` int DEFAULT NULL,\n `lytghmhkqfbochjdswrfdrxijbqtlhio` int DEFAULT NULL,\n `hfpjxlsblluwtjljtptglhegnahlvbbw` int DEFAULT NULL,\n `dxysatsjpwtqphtclvuichgttivodctn` int DEFAULT NULL,\n `zekwaxnzmtvtzgtfdukdphlfpbiqlshl` int DEFAULT NULL,\n `pctcanbilwtlpxovkshmjlnllgwgbcnb` int DEFAULT NULL,\n `fthgyhpetwfiwcngvnppbjhbkpaamqbl` int DEFAULT NULL,\n `obbyprmjcidhmbbkvpwbbxaaisdmwvmg` int DEFAULT NULL,\n `icspqbgqhrknrhukvjphptnuceinsjgr` int DEFAULT NULL,\n `sluzovsmvuubqonngqnflljrhlyivicr` int DEFAULT NULL,\n `tehfcdwnncitslyiplgtncejstgxqcxn` int DEFAULT NULL,\n `oievillmjaqvhblqxindsrnaxwnkkhjz` int DEFAULT NULL,\n `gaqjelcrpuskzlicffatsakghzngccsx` int DEFAULT NULL,\n `wkvumsuxhdcrevuofdhsbgmnwrexjqlt` int DEFAULT NULL,\n `wmpsphrpveiehtfqcxwpcwdrumevpipl` int DEFAULT NULL,\n `zzofnyfbhhmwhxtnoohthxbitnmbziqc` int DEFAULT NULL,\n `galywoqevqmzddkubreeczztpxroobfi` int DEFAULT NULL,\n `ltqbknsztxvhvcqwkbdwgibmyghpylop` int DEFAULT NULL,\n `mvzqlhlkzvlwpnnfmyvahdhenkfxhevu` int DEFAULT NULL,\n `ioghxftuwhudykuaqkikxxbdasyaurui` int DEFAULT NULL,\n `lwaysakaiuomtcnzwrycojajrrezsnah` int DEFAULT NULL,\n `uxnpqtvshpanwgoisshbhttcqvmkbrtj` int DEFAULT NULL,\n `bfzophttsujirkbogtvpsibzgvzlpchk` int DEFAULT NULL,\n `ntrpfsnftvvdtzthjaoftkyousswisvj` int DEFAULT NULL,\n `klljlmfrpodvhpiuqsgebidnvoapzbyo` int DEFAULT NULL,\n `jlnwdhhpejpkxipuhiywxkfzwgtrbidn` int DEFAULT NULL,\n `uhwkowqybuetrrndpmsdhxuidrqlvrdz` int DEFAULT NULL,\n `mvgoudqqqhxcccqyogykfbgcdosihnla` int DEFAULT NULL,\n `tphljwctsnbfrbfkpbhzwmdmhqgjjxxt` int DEFAULT NULL,\n `omzyyigpkpmmuiqqahffbkcquqaqwdqf` int DEFAULT NULL,\n `ydqrlvixzdaiiogequpehnocjlxtgfmt` int DEFAULT NULL,\n `frssfbjfvrhkshqgsrydoohhllygxmyp` int DEFAULT NULL,\n `piyihvrualzfilzuvkyaeimiginpayic` int DEFAULT NULL,\n `nkuvipvedqgaqgqxdkliciizhuavpufv` int DEFAULT NULL,\n `xtpvpfwissphqgpygsxnsqnvpbbocypf` int DEFAULT NULL,\n `oticuytzjbxnxxqyglavhyimjbfbghcr` int DEFAULT NULL,\n `gndlwmrtiovdbicqvwhhrvvxllivnbkw` int DEFAULT NULL,\n `cxwxejlqtpnwqmrhlskzquegwvdpqklh` int DEFAULT NULL,\n `fhcjunflzocpwrtduaxvbdapxsocydcv` int DEFAULT NULL,\n `oiulvvgfkjpjsqwyesrengpqvmhzuqna` int DEFAULT NULL,\n `zcgttweuhllebxunlubuxtqcapbrpmwh` int DEFAULT NULL,\n `mrdtmjvdhgosmcoypfuypmnbvsmossga` int DEFAULT NULL,\n `gfdizqwocrieszyuuxykumnqzmynhxwm` int DEFAULT NULL,\n `czetfmqcnycqartptqcvorxyuazjdvsy` int DEFAULT NULL,\n `gttfxpqdkogitljoipyyezsthmkqyqlq` int DEFAULT NULL,\n `lpmwayfnjgzwqdzlvduobjmtrczhaiwf` int DEFAULT NULL,\n `xbofqnwnxupyekydccjcdxvikhwqyeoo` int DEFAULT NULL,\n `tjvoesdwbyqtdpgvdgecamgwjrcmwxwe` int DEFAULT NULL,\n `drhjrjryxklcpmydgulexcruvxfxelpe` int DEFAULT NULL,\n `fiuxvddicldozdvmsqmnqfkfaypqxqzc` int DEFAULT NULL,\n `bsbquzjqplcvzenczexijgzgfdawjtqa` int DEFAULT NULL,\n `vrqqkiuwdqdknkxqeqnjsynktwwemxzt` int DEFAULT NULL,\n `nbshuofkkbczyayyvbfwrghvogchzcyo` int DEFAULT NULL,\n `kvyfgxzvegqoxybdleohebhfcvvnqasm` int DEFAULT NULL,\n `hotyaoyqvwfjxmzynsuopdvecduhcncg` int DEFAULT NULL,\n `nacsstkwnzdmysdkyryfrbjdqfsjzheg` int DEFAULT NULL,\n `feuvngcicqudjnybvmrttrvxujeejswc` int DEFAULT NULL,\n `dnrintpajemxquaygujqkofnjifxyumc` int DEFAULT NULL,\n `njscdpddlffrhedzzplwmilpavauytmh` int DEFAULT NULL,\n `mfvilvcuopwowjwytjdwquceutxlqyyg` int DEFAULT NULL,\n `gkdfunuqdnsjyackzdevtxvyrjnetcix` int DEFAULT NULL,\n `xgmncxcinbyxxvvbqbwuolfwrbvtvtol` int DEFAULT NULL,\n `vgnrvzwxzuwpdwwgvtgzwiggvdzigire` int DEFAULT NULL,\n `gxwlbfehhgruudxkiqdxlyixjzijitwl` int DEFAULT NULL,\n `bshcctmbluukqhmefszoygcuxptkddgb` int DEFAULT NULL,\n `inkkeawmgdppeztvrzarkghyzisixaci` int DEFAULT NULL,\n `uuqdosrpicozoynnmxxtropbbqhskmzb` int DEFAULT NULL,\n `pjimsyqfnsuetymetnkmgiipmsthagxk` int DEFAULT NULL,\n `ugriyelirfvxjbydmdtlzldkprthooei` int DEFAULT NULL,\n `vdtpflwqkvlukoalmwxgcwfupyeqekkb` int DEFAULT NULL,\n `niuvmhhxesnlpaqwafyyrxarpqqrvxal` int DEFAULT NULL,\n `onixdcbaysaydhpwydxmhkmmfrukexuj` int DEFAULT NULL,\n `acutfqrhdrwqlylhfcrpwfrdtbvprgmv` int DEFAULT NULL,\n `vfpgqwadlogsxityjabnyalpgxjvzeqg` int DEFAULT NULL,\n `mloqgyoxgksakmakmegvriexydrnygry` int DEFAULT NULL,\n `pecrtaxktcuadibrdfdtdbehwwcgjrzv` int DEFAULT NULL,\n `pxpfepkmgkdlxxvfdxptecshffrykhht` int DEFAULT NULL,\n PRIMARY KEY (`rlpjndrfvfzjhogvdkitpfuotzqqwxrj`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"zxhltxrsmhtsyvtgrlcsbvouuzpmoisc\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["rlpjndrfvfzjhogvdkitpfuotzqqwxrj"],"columns":[{"name":"rlpjndrfvfzjhogvdkitpfuotzqqwxrj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"jfghdusmeijnxlghtmzysjyopbtopwuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yyqmdhapoblcjtqucpouaeqemmnnbjzy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"iwbponzxaexvthfpvbvqjtiokkjqmmdi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ngwckvrwqlpqlpvytrxrflxsflhrtire","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hffrfddjrzeumkmroetetqmtqicopsnw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uwlbqivkeefuebxnbfmoefgsjhitiyuq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kptedffnldsyoomyrenkunshknheggcd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zvetntckneondtpdhfzmumzalqbvhbnz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nlshlbfoojwxmyoyoenkbfnjpnhtgopk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfjxpoxrycggvhddskbrziduquyudldq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbfrjjibupnffugipbdnprypfjymheny","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"flvohflfdyzqssjosuzlnffocnpveoxa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfkbflpzwgjrjhzvwgczkwqucybysroo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ffrrykilbmrsxbvclfndgjbdkwggsyhu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jqrezzqzyoqrjiqefcwlaafocrsoswgq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fvyhmlwqbanhzguokonklpidskmshwku","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cdlmpuizwqmfjglrwikfieyymkyoygeq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lutlbrepsaicgxqnagkuwtnoyqiqpkxp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjrnegxfpaofzlwbpbsrrfnlifitexdd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lzfpscxgqnhqnbxwgulsqouisrmvjxre","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"poigvxvwyvrrvzuxluayfffcwpcqcpyn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hjlldfzrrzldwuhkyzfbldtzdoizqplh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ohrxcmwyvrbjipepnkdmczqfjmjitxle","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lytghmhkqfbochjdswrfdrxijbqtlhio","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hfpjxlsblluwtjljtptglhegnahlvbbw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dxysatsjpwtqphtclvuichgttivodctn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zekwaxnzmtvtzgtfdukdphlfpbiqlshl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pctcanbilwtlpxovkshmjlnllgwgbcnb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fthgyhpetwfiwcngvnppbjhbkpaamqbl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"obbyprmjcidhmbbkvpwbbxaaisdmwvmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"icspqbgqhrknrhukvjphptnuceinsjgr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sluzovsmvuubqonngqnflljrhlyivicr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tehfcdwnncitslyiplgtncejstgxqcxn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oievillmjaqvhblqxindsrnaxwnkkhjz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gaqjelcrpuskzlicffatsakghzngccsx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wkvumsuxhdcrevuofdhsbgmnwrexjqlt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"wmpsphrpveiehtfqcxwpcwdrumevpipl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzofnyfbhhmwhxtnoohthxbitnmbziqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"galywoqevqmzddkubreeczztpxroobfi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ltqbknsztxvhvcqwkbdwgibmyghpylop","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvzqlhlkzvlwpnnfmyvahdhenkfxhevu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ioghxftuwhudykuaqkikxxbdasyaurui","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lwaysakaiuomtcnzwrycojajrrezsnah","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uxnpqtvshpanwgoisshbhttcqvmkbrtj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bfzophttsujirkbogtvpsibzgvzlpchk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ntrpfsnftvvdtzthjaoftkyousswisvj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"klljlmfrpodvhpiuqsgebidnvoapzbyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlnwdhhpejpkxipuhiywxkfzwgtrbidn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uhwkowqybuetrrndpmsdhxuidrqlvrdz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mvgoudqqqhxcccqyogykfbgcdosihnla","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tphljwctsnbfrbfkpbhzwmdmhqgjjxxt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"omzyyigpkpmmuiqqahffbkcquqaqwdqf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ydqrlvixzdaiiogequpehnocjlxtgfmt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"frssfbjfvrhkshqgsrydoohhllygxmyp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"piyihvrualzfilzuvkyaeimiginpayic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nkuvipvedqgaqgqxdkliciizhuavpufv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xtpvpfwissphqgpygsxnsqnvpbbocypf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oticuytzjbxnxxqyglavhyimjbfbghcr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gndlwmrtiovdbicqvwhhrvvxllivnbkw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cxwxejlqtpnwqmrhlskzquegwvdpqklh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fhcjunflzocpwrtduaxvbdapxsocydcv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oiulvvgfkjpjsqwyesrengpqvmhzuqna","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zcgttweuhllebxunlubuxtqcapbrpmwh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mrdtmjvdhgosmcoypfuypmnbvsmossga","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gfdizqwocrieszyuuxykumnqzmynhxwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"czetfmqcnycqartptqcvorxyuazjdvsy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gttfxpqdkogitljoipyyezsthmkqyqlq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lpmwayfnjgzwqdzlvduobjmtrczhaiwf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xbofqnwnxupyekydccjcdxvikhwqyeoo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"tjvoesdwbyqtdpgvdgecamgwjrcmwxwe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"drhjrjryxklcpmydgulexcruvxfxelpe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fiuxvddicldozdvmsqmnqfkfaypqxqzc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bsbquzjqplcvzenczexijgzgfdawjtqa","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vrqqkiuwdqdknkxqeqnjsynktwwemxzt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbshuofkkbczyayyvbfwrghvogchzcyo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kvyfgxzvegqoxybdleohebhfcvvnqasm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hotyaoyqvwfjxmzynsuopdvecduhcncg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nacsstkwnzdmysdkyryfrbjdqfsjzheg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"feuvngcicqudjnybvmrttrvxujeejswc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dnrintpajemxquaygujqkofnjifxyumc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"njscdpddlffrhedzzplwmilpavauytmh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mfvilvcuopwowjwytjdwquceutxlqyyg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkdfunuqdnsjyackzdevtxvyrjnetcix","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xgmncxcinbyxxvvbqbwuolfwrbvtvtol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vgnrvzwxzuwpdwwgvtgzwiggvdzigire","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gxwlbfehhgruudxkiqdxlyixjzijitwl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bshcctmbluukqhmefszoygcuxptkddgb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"inkkeawmgdppeztvrzarkghyzisixaci","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"uuqdosrpicozoynnmxxtropbbqhskmzb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pjimsyqfnsuetymetnkmgiipmsthagxk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ugriyelirfvxjbydmdtlzldkprthooei","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vdtpflwqkvlukoalmwxgcwfupyeqekkb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"niuvmhhxesnlpaqwafyyrxarpqqrvxal","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"onixdcbaysaydhpwydxmhkmmfrukexuj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"acutfqrhdrwqlylhfcrpwfrdtbvprgmv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vfpgqwadlogsxityjabnyalpgxjvzeqg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mloqgyoxgksakmakmegvriexydrnygry","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pecrtaxktcuadibrdfdtdbehwwcgjrzv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pxpfepkmgkdlxxvfdxptecshffrykhht","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1693842673,"file":"binlog.000002","pos":905418,"snapshot":true},"ts_ms":1693842673339,"databaseName":"models_schema","ddl":"CREATE TABLE `zxopybvdnhidckkbrsvwijsunafkxxtb` (\n `jwprwtecglqhhhplkddbnhuoqdhibhjv` int NOT NULL,\n `jgbvbksaxmwjvsfifxgrjcmkfukkezfl` int DEFAULT NULL,\n `mbokourhenorazyrstqdiqekxmikerxq` int DEFAULT NULL,\n `ywivtsgmuvzzqoznrfwhywskknewisqc` int DEFAULT NULL,\n `knblqkozqluovtxxqhcmkpueznkzavgf` int DEFAULT NULL,\n `zutdyxxexvpzxdwptrbcoirwcfnvpurr` int DEFAULT NULL,\n `krwupchfnffvxbkengufazkrknbdroxr` int DEFAULT NULL,\n `zqwrrhenomhrhmxzqnwkjffvqbngcqvi` int DEFAULT NULL,\n `yuchgpfybawpeermnffsjicihylwjztl` int DEFAULT NULL,\n `krchlbdltzbowxedtqwbyqrvlfnzddgd` int DEFAULT NULL,\n `xuthcuxluzvkznztcrgpafnulsmxvkvk` int DEFAULT NULL,\n `kcndevgrchcybanfqwtodgnnkxshecjc` int DEFAULT NULL,\n `urxyjcinyolspxquiuhcetbdkngjtzca` int DEFAULT NULL,\n `gdeeowudnxumkorpxoalxnmcbkopjdlf` int DEFAULT NULL,\n `zuybkispqnpmqjakdgmncrwzgxgdyvog` int DEFAULT NULL,\n `yfjqxeekcefemuvvoighfqmmwcumqtdk` int DEFAULT NULL,\n `qjhcepgdnssxrdqucmsvlvbvwmzeeayl` int DEFAULT NULL,\n `luffxmjgkisdkdzxdadavvjvavxbrmnh` int DEFAULT NULL,\n `fcxmxbjceulsloskqguaimuvogdbokxp` int DEFAULT NULL,\n `mnppslxduupxtgrumujgwwukuncbihvs` int DEFAULT NULL,\n `raprazqjzphjejbsuourfncmrymapwba` int DEFAULT NULL,\n `kovjieuvmxqzrdabvhoyywifrrhwvval` int DEFAULT NULL,\n `zffejwnpaedinlkcieokythepyvzlskw` int DEFAULT NULL,\n `yokmydewpocwnejvvuwdnvezwzdiwfjb` int DEFAULT NULL,\n `irqfayiaqfwnmqyntncwldbvnquyfnca` int DEFAULT NULL,\n `akzlzxpibyjhytdvbezzrunjgggcwvms` int DEFAULT NULL,\n `ufotuobibzvcsayclwbpczqieelgmuka` int DEFAULT NULL,\n `nrvwgbdcvtinfgcehifhurndjypsznbt` int DEFAULT NULL,\n `ggqjcxgwsppqhsnzabhuiijigytuapll` int DEFAULT NULL,\n `qyhlggyryacfrpesarshlftnqdwhhhil` int DEFAULT NULL,\n `meloldpqlbuqfckcvhditnlbytusjioi` int DEFAULT NULL,\n `fnwmpjiaurfqlwgfwpipdrbreaehibus` int DEFAULT NULL,\n `lutfplyaopmzivdwplakcdxmtwujivhs` int DEFAULT NULL,\n `nugfusjriclkgbaqoenajvmjlcziopmk` int DEFAULT NULL,\n `jvglumxyovstkfnhalpdazlebibvrncl` int DEFAULT NULL,\n `pbnxruejfqyvvlvdxhooyoktevkyeaae` int DEFAULT NULL,\n `unzlyqhsichwbsjbguxfmaoromfsobal` int DEFAULT NULL,\n `cnxklxgkehqasjbmfkorfywnusybkpcb` int DEFAULT NULL,\n `arcucpwmcrceytvfbkgsxtcbzrklacnw` int DEFAULT NULL,\n `qinezuknodytfxvqdjukuohczoqfafrb` int DEFAULT NULL,\n `nghfjyyjzoegnxsvwfhrkqpcggpwcbjr` int DEFAULT NULL,\n `zzawaoseybnfsslltsbwyqknepewmojx` int DEFAULT NULL,\n `cutqyqhbfgagszrfnmlugqjmrdyuwplw` int DEFAULT NULL,\n `fqgnbiszgsktdhlpqaidevtdfdthtehq` int DEFAULT NULL,\n `isscgpodnkuxxfgzrwvjqcxwajxfnhwc` int DEFAULT NULL,\n `gnrypkntwvcmztqyqtqfnbzokiivecmw` int DEFAULT NULL,\n `ygwegngcevsxvmhgrrkbbmxgzakgazbp` int DEFAULT NULL,\n `rcxcajwcrfojwptxomsfqlqlysobkjlu` int DEFAULT NULL,\n `aweygqxhsskpsndunhawapqjmmmaufju` int DEFAULT NULL,\n `qxkccgsyjehhsoddnrcbfqcfppdzyrch` int DEFAULT NULL,\n `breorqvqlufdwssuxgvikjdbnfakpkwk` int DEFAULT NULL,\n `ixzuutkbhdhmwrtcuchqystmgsypdnnx` int DEFAULT NULL,\n `gkhplwddgbntktqntsywnecgyuqzalgx` int DEFAULT NULL,\n `ryrmmwkedlzjrdponksxxlsjhqrmjjmg` int DEFAULT NULL,\n `ltbvhzkmaonllajlvmqhtkokwcpaxfat` int DEFAULT NULL,\n `gaguncwuvzelipkjcxcjjwooonlbggyd` int DEFAULT NULL,\n `nfspiqdhnswwtdtyafqglezsicsnnfux` int DEFAULT NULL,\n `ecegocrqiryoccerxqemarxpgmhudbwm` int DEFAULT NULL,\n `ypmdwmvkqzhwkwnwgumfenwwhanaenpf` int DEFAULT NULL,\n `cdowhxycdhiwjcjpjuwztyaguesbhiuj` int DEFAULT NULL,\n `qecwtfkumalncggmkjuobicbfzhoawkh` int DEFAULT NULL,\n `rosbdmissaqxozpadwhvmdhxfytawkaz` int DEFAULT NULL,\n `yxpowxokxjtxxzuqynomfznhofefswnm` int DEFAULT NULL,\n `imablfougeamxpuwelhlyeucjruzuzdh` int DEFAULT NULL,\n `mzfydfukfyadeczskkwivwthjolayuts` int DEFAULT NULL,\n `gqaqexaajwsgxmfaclaondguyntettdd` int DEFAULT NULL,\n `nfbmwvubzuehfspuuukpqhuxxdtrgyar` int DEFAULT NULL,\n `jlrywesoazhiewqfcbtccnltjldqgudt` int DEFAULT NULL,\n `bsfzfmhktyczgnlrszzyzdfeaokrldcm` int DEFAULT NULL,\n `myspbdlglhtawchrotidfxfyyszheoaj` int DEFAULT NULL,\n `qjtonbfypzpeaikhhskpkurizsefuuce` int DEFAULT NULL,\n `gisgjlhdbkozewsoqkdmqnppryuuuazu` int DEFAULT NULL,\n `odevbgrfmjsowcdezqlvifpxgzueexkn` int DEFAULT NULL,\n `xqshwwuubijqjkcmdiedbvnvuxagvkcg` int DEFAULT NULL,\n `grkdvogrjimjbmpyyivtcqxnflvdwehp` int DEFAULT NULL,\n `jwecnphnckgjdmcvopvdysqmrupmjllv` int DEFAULT NULL,\n `mdmefkzdogxhsjqsntlmmgpvsqsunepd` int DEFAULT NULL,\n `bchcusxkqxtceworahazwhjyobxympou` int DEFAULT NULL,\n `nbstzihzpfqufqwfmkcnkjmxmyitwovf` int DEFAULT NULL,\n `vjlhylxkcwxbnhrcuujskmptycntprjy` int DEFAULT NULL,\n `twnhyvxqueieseripmiblzlksfgqbhcr` int DEFAULT NULL,\n `kicvbecvojwvvuulwvomovkpjfwhlmic` int DEFAULT NULL,\n `cdfuggudkgcvdsdkdanuqrebhsyenynd` int DEFAULT NULL,\n `fecaezvyuadergzhtabrpyawikfxxlkx` int DEFAULT NULL,\n `nbiiqezaibnnumitwubpvypgcqghuvsx` int DEFAULT NULL,\n `dsprpfyfmttgrqbftrnbgjpebybvucth` int DEFAULT NULL,\n `cjddwfcnvbigqspoeccijlhtkcngkylz` int DEFAULT NULL,\n `kxukqcrdqpbjlwnvgdwhqkwohptnjwzw` int DEFAULT NULL,\n `coqhcxldfolggermsqkcnmzerenejorh` int DEFAULT NULL,\n `usjwxflcpnnmpynssyspizqcumfsvhil` int DEFAULT NULL,\n `evpymsbbwscqqiwgritluuzroeddknml` int DEFAULT NULL,\n `joovmlezxjzfcfatuuftjxodocqroema` int DEFAULT NULL,\n `hgkidcrnamzieqovttjhbahwypgapspb` int DEFAULT NULL,\n `sgwloayfvtaziermgvbcbkqqmntbbksh` int DEFAULT NULL,\n `oaxxilpdoxiqwtyyfyhxcanpjeigdzsu` int DEFAULT NULL,\n `qkctvqcqgacmfhedpkfyxhyvvgbznboe` int DEFAULT NULL,\n `rpbrzrijxhiyddhdogdyajchvameawxd` int DEFAULT NULL,\n `bxfapfxiqerusrjjxvawucjdirtswkpo` int DEFAULT NULL,\n `fkswmxkjarvxuwrvrphbxoiryuvifsol` int DEFAULT NULL,\n `oxhvzkpidvxmensbwnbfslshyyjmkgwy` int DEFAULT NULL,\n PRIMARY KEY (`jwprwtecglqhhhplkddbnhuoqdhibhjv`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"zxopybvdnhidckkbrsvwijsunafkxxtb\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["jwprwtecglqhhhplkddbnhuoqdhibhjv"],"columns":[{"name":"jwprwtecglqhhhplkddbnhuoqdhibhjv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"jgbvbksaxmwjvsfifxgrjcmkfukkezfl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mbokourhenorazyrstqdiqekxmikerxq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ywivtsgmuvzzqoznrfwhywskknewisqc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":4,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"knblqkozqluovtxxqhcmkpueznkzavgf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":5,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zutdyxxexvpzxdwptrbcoirwcfnvpurr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":6,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krwupchfnffvxbkengufazkrknbdroxr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":7,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zqwrrhenomhrhmxzqnwkjffvqbngcqvi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":8,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yuchgpfybawpeermnffsjicihylwjztl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":9,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"krchlbdltzbowxedtqwbyqrvlfnzddgd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":10,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xuthcuxluzvkznztcrgpafnulsmxvkvk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":11,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kcndevgrchcybanfqwtodgnnkxshecjc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":12,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"urxyjcinyolspxquiuhcetbdkngjtzca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":13,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gdeeowudnxumkorpxoalxnmcbkopjdlf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":14,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zuybkispqnpmqjakdgmncrwzgxgdyvog","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":15,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yfjqxeekcefemuvvoighfqmmwcumqtdk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":16,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjhcepgdnssxrdqucmsvlvbvwmzeeayl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":17,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"luffxmjgkisdkdzxdadavvjvavxbrmnh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":18,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fcxmxbjceulsloskqguaimuvogdbokxp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":19,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mnppslxduupxtgrumujgwwukuncbihvs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":20,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"raprazqjzphjejbsuourfncmrymapwba","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":21,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kovjieuvmxqzrdabvhoyywifrrhwvval","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":22,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zffejwnpaedinlkcieokythepyvzlskw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":23,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yokmydewpocwnejvvuwdnvezwzdiwfjb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":24,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"irqfayiaqfwnmqyntncwldbvnquyfnca","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":25,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"akzlzxpibyjhytdvbezzrunjgggcwvms","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":26,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ufotuobibzvcsayclwbpczqieelgmuka","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":27,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nrvwgbdcvtinfgcehifhurndjypsznbt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":28,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ggqjcxgwsppqhsnzabhuiijigytuapll","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":29,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qyhlggyryacfrpesarshlftnqdwhhhil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":30,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"meloldpqlbuqfckcvhditnlbytusjioi","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":31,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fnwmpjiaurfqlwgfwpipdrbreaehibus","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":32,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"lutfplyaopmzivdwplakcdxmtwujivhs","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":33,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nugfusjriclkgbaqoenajvmjlcziopmk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":34,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jvglumxyovstkfnhalpdazlebibvrncl","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":35,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"pbnxruejfqyvvlvdxhooyoktevkyeaae","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":36,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"unzlyqhsichwbsjbguxfmaoromfsobal","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":37,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cnxklxgkehqasjbmfkorfywnusybkpcb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":38,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"arcucpwmcrceytvfbkgsxtcbzrklacnw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":39,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qinezuknodytfxvqdjukuohczoqfafrb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":40,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nghfjyyjzoegnxsvwfhrkqpcggpwcbjr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":41,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"zzawaoseybnfsslltsbwyqknepewmojx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":42,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cutqyqhbfgagszrfnmlugqjmrdyuwplw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":43,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fqgnbiszgsktdhlpqaidevtdfdthtehq","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":44,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"isscgpodnkuxxfgzrwvjqcxwajxfnhwc","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":45,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gnrypkntwvcmztqyqtqfnbzokiivecmw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":46,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ygwegngcevsxvmhgrrkbbmxgzakgazbp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":47,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rcxcajwcrfojwptxomsfqlqlysobkjlu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":48,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"aweygqxhsskpsndunhawapqjmmmaufju","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":49,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qxkccgsyjehhsoddnrcbfqcfppdzyrch","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":50,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"breorqvqlufdwssuxgvikjdbnfakpkwk","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":51,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ixzuutkbhdhmwrtcuchqystmgsypdnnx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":52,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gkhplwddgbntktqntsywnecgyuqzalgx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":53,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ryrmmwkedlzjrdponksxxlsjhqrmjjmg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":54,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ltbvhzkmaonllajlvmqhtkokwcpaxfat","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":55,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gaguncwuvzelipkjcxcjjwooonlbggyd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":56,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfspiqdhnswwtdtyafqglezsicsnnfux","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":57,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ecegocrqiryoccerxqemarxpgmhudbwm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":58,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"ypmdwmvkqzhwkwnwgumfenwwhanaenpf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":59,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cdowhxycdhiwjcjpjuwztyaguesbhiuj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":60,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qecwtfkumalncggmkjuobicbfzhoawkh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":61,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rosbdmissaqxozpadwhvmdhxfytawkaz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":62,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"yxpowxokxjtxxzuqynomfznhofefswnm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":63,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"imablfougeamxpuwelhlyeucjruzuzdh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":64,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mzfydfukfyadeczskkwivwthjolayuts","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":65,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gqaqexaajwsgxmfaclaondguyntettdd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":66,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nfbmwvubzuehfspuuukpqhuxxdtrgyar","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":67,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jlrywesoazhiewqfcbtccnltjldqgudt","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":68,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bsfzfmhktyczgnlrszzyzdfeaokrldcm","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":69,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"myspbdlglhtawchrotidfxfyyszheoaj","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":70,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qjtonbfypzpeaikhhskpkurizsefuuce","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":71,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"gisgjlhdbkozewsoqkdmqnppryuuuazu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":72,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"odevbgrfmjsowcdezqlvifpxgzueexkn","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":73,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"xqshwwuubijqjkcmdiedbvnvuxagvkcg","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":74,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"grkdvogrjimjbmpyyivtcqxnflvdwehp","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":75,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"jwecnphnckgjdmcvopvdysqmrupmjllv","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":76,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"mdmefkzdogxhsjqsntlmmgpvsqsunepd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":77,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bchcusxkqxtceworahazwhjyobxympou","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":78,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbstzihzpfqufqwfmkcnkjmxmyitwovf","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":79,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"vjlhylxkcwxbnhrcuujskmptycntprjy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":80,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"twnhyvxqueieseripmiblzlksfgqbhcr","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":81,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kicvbecvojwvvuulwvomovkpjfwhlmic","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":82,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cdfuggudkgcvdsdkdanuqrebhsyenynd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":83,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fecaezvyuadergzhtabrpyawikfxxlkx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":84,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"nbiiqezaibnnumitwubpvypgcqghuvsx","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":85,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"dsprpfyfmttgrqbftrnbgjpebybvucth","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":86,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"cjddwfcnvbigqspoeccijlhtkcngkylz","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":87,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"kxukqcrdqpbjlwnvgdwhqkwohptnjwzw","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":88,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"coqhcxldfolggermsqkcnmzerenejorh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":89,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"usjwxflcpnnmpynssyspizqcumfsvhil","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":90,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"evpymsbbwscqqiwgritluuzroeddknml","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":91,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"joovmlezxjzfcfatuuftjxodocqroema","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":92,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"hgkidcrnamzieqovttjhbahwypgapspb","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":93,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"sgwloayfvtaziermgvbcbkqqmntbbksh","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":94,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oaxxilpdoxiqwtyyfyhxcanpjeigdzsu","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":95,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"qkctvqcqgacmfhedpkfyxhyvvgbznboe","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":96,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"rpbrzrijxhiyddhdogdyajchvameawxd","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":97,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"bxfapfxiqerusrjjxvawucjdirtswkpo","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":98,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"fkswmxkjarvxuwrvrphbxoiryuvifsol","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":99,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"oxhvzkpidvxmensbwnbfslshyyjmkgwy","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":100,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/dbhistory_less_than_3_mb.dat b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/dbhistory_less_than_3_mb.dat new file mode 100644 index 000000000000..8299bcc09c4c --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/dbhistory_less_than_3_mb.dat @@ -0,0 +1,8 @@ +{"source":{"server":"models_schema"},"position":{"ts_sec":1694417189,"file":"binlog.000002","pos":11149,"snapshot":true},"ts_ms":1694417190273,"databaseName":"","ddl":"SET character_set_server=utf8mb4, collation_server=utf8mb4_0900_ai_ci","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1694417190,"file":"binlog.000002","pos":11149,"snapshot":true},"ts_ms":1694417190295,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`models`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1694417190,"file":"binlog.000002","pos":11149,"snapshot":true},"ts_ms":1694417190297,"databaseName":"models_schema","ddl":"DROP TABLE IF EXISTS `models_schema`.`models_random`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1694417190,"file":"binlog.000002","pos":11149,"snapshot":true},"ts_ms":1694417190307,"databaseName":"models_schema","ddl":"DROP DATABASE IF EXISTS `models_schema`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1694417190,"file":"binlog.000002","pos":11149,"snapshot":true},"ts_ms":1694417190315,"databaseName":"models_schema","ddl":"CREATE DATABASE `models_schema` CHARSET utf8mb4 COLLATE utf8mb4_0900_ai_ci","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1694417190,"file":"binlog.000002","pos":11149,"snapshot":true},"ts_ms":1694417190317,"databaseName":"models_schema","ddl":"USE `models_schema`","tableChanges":[]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1694417190,"file":"binlog.000002","pos":11149,"snapshot":true},"ts_ms":1694417190394,"databaseName":"models_schema","ddl":"CREATE TABLE `models` (\n `id` int NOT NULL,\n `make_id` int DEFAULT NULL,\n `model` varchar(200) DEFAULT NULL,\n PRIMARY KEY (`id`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"models\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["id"],"columns":[{"name":"id","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"make_id","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"model","jdbcType":12,"typeName":"VARCHAR","typeExpression":"VARCHAR","charsetName":"utf8mb4","length":200,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} +{"source":{"server":"models_schema"},"position":{"ts_sec":1694417190,"file":"binlog.000002","pos":11149,"snapshot":true},"ts_ms":1694417190403,"databaseName":"models_schema","ddl":"CREATE TABLE `models_random` (\n `id_random` int NOT NULL,\n `make_id_random` int DEFAULT NULL,\n `model_random` varchar(200) DEFAULT NULL,\n PRIMARY KEY (`id_random`)\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci","tableChanges":[{"type":"CREATE","id":"\"models_schema\".\"models_random\"","table":{"defaultCharsetName":"utf8mb4","primaryKeyColumnNames":["id_random"],"columns":[{"name":"id_random","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":1,"optional":false,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":false,"enumValues":[]},{"name":"make_id_random","jdbcType":4,"typeName":"INT","typeExpression":"INT","charsetName":null,"position":2,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]},{"name":"model_random","jdbcType":12,"typeName":"VARCHAR","typeExpression":"VARCHAR","charsetName":"utf8mb4","length":200,"position":3,"optional":true,"autoIncremented":false,"generated":false,"comment":null,"hasDefaultValue":true,"enumValues":[]}],"attributes":[]},"comment":null}]} diff --git a/airbyte-integrations/bases/debezium/src/test/resources/delete_change_event.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/delete_change_event.json similarity index 100% rename from airbyte-integrations/bases/debezium/src/test/resources/delete_change_event.json rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/delete_change_event.json diff --git a/airbyte-integrations/bases/debezium/src/test/resources/delete_message.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/delete_message.json similarity index 100% rename from airbyte-integrations/bases/debezium/src/test/resources/delete_message.json rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/delete_message.json diff --git a/airbyte-integrations/bases/debezium/src/test/resources/insert_change_event.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/insert_change_event.json similarity index 100% rename from airbyte-integrations/bases/debezium/src/test/resources/insert_change_event.json rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/insert_change_event.json diff --git a/airbyte-integrations/bases/debezium/src/test/resources/insert_message.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/insert_message.json similarity index 100% rename from airbyte-integrations/bases/debezium/src/test/resources/insert_message.json rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/insert_message.json diff --git a/airbyte-integrations/bases/debezium/src/test/resources/mongodb/change_event.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event.json similarity index 100% rename from airbyte-integrations/bases/debezium/src/test/resources/mongodb/change_event.json rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event.json diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_delete.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_delete.json new file mode 100644 index 000000000000..89466aa511d8 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_delete.json @@ -0,0 +1,23 @@ +{ + "before": "{\"_id\": {\"$oid\": \"64f24244f95155351c4185b1\"},\"name\": \"Document 0\",\"description\": \"This is document #0\",\"doubleField\": 0.0,\"intField\": 0, \"data\": \"some data\",\"objectField\": {\"key\": \"value\"},\"timestamp\": {\"$timestamp\": {\"t\": 394,\"i\": 1381162128}}}\"", + "after": null, + "source": { + "version": "2.2.0.Final", + "connector": "mongodb", + "name": "public", + "ts_ms": 1693598277000, + "snapshot": "false", + "db": "public", + "sequence": null, + "rs": "replica-set", + "collection": "names", + "ord": 1, + "lsid": null, + "txnNumber": null, + "wallTime": null + }, + "op": "d", + "ts_ms": 1693599528047, + "transaction": null, + "updateDescription": null +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_delete_no_before.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_delete_no_before.json new file mode 100644 index 000000000000..b81171e6d3a5 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_delete_no_before.json @@ -0,0 +1,23 @@ +{ + "before": null, + "after": null, + "source": { + "version": "2.2.0.Final", + "connector": "mongodb", + "name": "public", + "ts_ms": 1693598277000, + "snapshot": "false", + "db": "public", + "sequence": null, + "rs": "replica-set", + "collection": "names", + "ord": 1, + "lsid": null, + "txnNumber": null, + "wallTime": null + }, + "op": "d", + "ts_ms": 1693599528047, + "transaction": null, + "updateDescription": null +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_insert.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_insert.json new file mode 100644 index 000000000000..c56115e7a381 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_insert.json @@ -0,0 +1,23 @@ +{ + "before": null, + "after": "{\"_id\": {\"$oid\": \"64f24244f95155351c4185b1\"},\"name\": \"Document 0\",\"description\": \"This is document #0\",\"doubleField\": 0.0,\"intField\": 0, \"data\": \"some data\",\"objectField\": {\"key\": \"value\"},\"timestamp\": {\"$timestamp\": {\"t\": 394,\"i\": 1381162128}}}\"", + "source": { + "version": "2.2.0.Final", + "connector": "mongodb", + "name": "public", + "ts_ms": 1693598277000, + "snapshot": "false", + "db": "public", + "sequence": null, + "rs": "replica-set", + "collection": "names", + "ord": 1, + "lsid": null, + "txnNumber": null, + "wallTime": null + }, + "op": "c", + "ts_ms": 1693599528047, + "transaction": null, + "updateDescription": null +} diff --git a/airbyte-integrations/bases/debezium/src/test/resources/mongodb/change_event_snapshot.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_snapshot.json similarity index 100% rename from airbyte-integrations/bases/debezium/src/test/resources/mongodb/change_event_snapshot.json rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_snapshot.json diff --git a/airbyte-integrations/bases/debezium/src/test/resources/mongodb/change_event_snapshot_last.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_snapshot_last.json similarity index 100% rename from airbyte-integrations/bases/debezium/src/test/resources/mongodb/change_event_snapshot_last.json rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_snapshot_last.json diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_unsupported.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_unsupported.json new file mode 100644 index 000000000000..037729999a8f --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_unsupported.json @@ -0,0 +1,23 @@ +{ + "before": null, + "after": null, + "source": { + "version": "2.2.0.Final", + "connector": "mongodb", + "name": "public", + "ts_ms": 1693598277000, + "snapshot": "false", + "db": "public", + "sequence": null, + "rs": "replica-set", + "collection": "names", + "ord": 1, + "lsid": null, + "txnNumber": null, + "wallTime": null + }, + "op": "t", + "ts_ms": 1693599528047, + "transaction": null, + "updateDescription": null +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_update.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_update.json new file mode 100644 index 000000000000..1f132a350759 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/change_event_update.json @@ -0,0 +1,23 @@ +{ + "before": "{\"_id\": {\"$oid\": \"64f24244f95155351c4185b1\"},\"name\": \"Document 1\",\"description\": \"This is document #1\",\"doubleField\": 1.0,\"intField\": 1,\"objectField\": {\"key\": \"value\"},\"timestamp\": {\"$timestamp\": {\"t\": 394,\"i\": 1381162128}}}\"", + "after": "{\"_id\": {\"$oid\": \"64f24244f95155351c4185b1\"},\"name\": \"Document 0\",\"description\": \"This is document #0\",\"doubleField\": 0.0,\"intField\": 0, \"data\": \"some data\",\"objectField\": {\"key\": \"value\"},\"timestamp\": {\"$timestamp\": {\"t\": 394,\"i\": 1381162128}}}\"", + "source": { + "version": "2.2.0.Final", + "connector": "mongodb", + "name": "public", + "ts_ms": 1693598277000, + "snapshot": "false", + "db": "public", + "sequence": null, + "rs": "replica-set", + "collection": "names", + "ord": 1, + "lsid": null, + "txnNumber": null, + "wallTime": null + }, + "op": "c", + "ts_ms": 1693599528047, + "transaction": null, + "updateDescription": null +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/delete_airbyte_message.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/delete_airbyte_message.json new file mode 100644 index 000000000000..9ec95bdaab8e --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/delete_airbyte_message.json @@ -0,0 +1,13 @@ +{ + "_id": "64f24244f95155351c4185b1", + "name": "Document 0", + "description": "This is document #0", + "doubleField": 0.0, + "intField": 0, + "objectField": { + "key": "value" + }, + "timestamp": "2023-09-01T19:57:56.752Z", + "_ab_cdc_updated_at": "2023-09-01T19:57:57Z", + "_ab_cdc_deleted_at": "2023-09-01T19:57:57Z" +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/delete_airbyte_message_no_schema.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/delete_airbyte_message_no_schema.json new file mode 100644 index 000000000000..c3bc65ae2dd0 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/delete_airbyte_message_no_schema.json @@ -0,0 +1,17 @@ +{ + "_id": "64f24244f95155351c4185b1", + "data": { + "_id": "64f24244f95155351c4185b1", + "name": "Document 0", + "description": "This is document #0", + "doubleField": 0.0, + "intField": 0, + "objectField": { + "key": "value" + }, + "timestamp": "2023-09-01T19:57:56.752Z", + "data": "some data" + }, + "_ab_cdc_updated_at": "2023-09-01T19:57:57Z", + "_ab_cdc_deleted_at": "2023-09-01T19:57:57Z" +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/delete_no_before_airbyte_message.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/delete_no_before_airbyte_message.json new file mode 100644 index 000000000000..a0e40cff602f --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/delete_no_before_airbyte_message.json @@ -0,0 +1,5 @@ +{ + "_id": "64f24244f95155351c4185b1", + "_ab_cdc_updated_at": "2023-09-01T19:57:57Z", + "_ab_cdc_deleted_at": "2023-09-01T19:57:57Z" +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/delete_no_before_airbyte_message_no_schema.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/delete_no_before_airbyte_message_no_schema.json new file mode 100644 index 000000000000..89fe4fd4e9ed --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/delete_no_before_airbyte_message_no_schema.json @@ -0,0 +1,8 @@ +{ + "_id": "64f24244f95155351c4185b1", + "data": { + "_id": "64f24244f95155351c4185b1" + }, + "_ab_cdc_updated_at": "2023-09-01T19:57:57Z", + "_ab_cdc_deleted_at": "2023-09-01T19:57:57Z" +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/insert_airbyte_message.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/insert_airbyte_message.json new file mode 100644 index 000000000000..bf62ce6116da --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/insert_airbyte_message.json @@ -0,0 +1,13 @@ +{ + "_id": "64f24244f95155351c4185b1", + "name": "Document 0", + "description": "This is document #0", + "doubleField": 0.0, + "intField": 0, + "objectField": { + "key": "value" + }, + "timestamp": "2023-09-01T19:57:56.752Z", + "_ab_cdc_updated_at": "2023-09-01T19:57:57Z", + "_ab_cdc_deleted_at": null +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/insert_airbyte_message_no_schema.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/insert_airbyte_message_no_schema.json new file mode 100644 index 000000000000..4242b47f8c55 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/insert_airbyte_message_no_schema.json @@ -0,0 +1,17 @@ +{ + "_id": "64f24244f95155351c4185b1", + "data": { + "_id": "64f24244f95155351c4185b1", + "name": "Document 0", + "description": "This is document #0", + "doubleField": 0.0, + "intField": 0, + "objectField": { + "key": "value" + }, + "timestamp": "2023-09-01T19:57:56.752Z", + "data": "some data" + }, + "_ab_cdc_updated_at": "2023-09-01T19:57:57Z", + "_ab_cdc_deleted_at": null +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/update_airbyte_message.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/update_airbyte_message.json new file mode 100644 index 000000000000..bf62ce6116da --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/update_airbyte_message.json @@ -0,0 +1,13 @@ +{ + "_id": "64f24244f95155351c4185b1", + "name": "Document 0", + "description": "This is document #0", + "doubleField": 0.0, + "intField": 0, + "objectField": { + "key": "value" + }, + "timestamp": "2023-09-01T19:57:56.752Z", + "_ab_cdc_updated_at": "2023-09-01T19:57:57Z", + "_ab_cdc_deleted_at": null +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/update_airbyte_message_no_schema.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/update_airbyte_message_no_schema.json new file mode 100644 index 000000000000..4242b47f8c55 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/mongodb/update_airbyte_message_no_schema.json @@ -0,0 +1,17 @@ +{ + "_id": "64f24244f95155351c4185b1", + "data": { + "_id": "64f24244f95155351c4185b1", + "name": "Document 0", + "description": "This is document #0", + "doubleField": 0.0, + "intField": 0, + "objectField": { + "key": "value" + }, + "timestamp": "2023-09-01T19:57:56.752Z", + "data": "some data" + }, + "_ab_cdc_updated_at": "2023-09-01T19:57:57Z", + "_ab_cdc_deleted_at": null +} diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/resources/states/global.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/states/global.json similarity index 100% rename from airbyte-integrations/connectors/source-relational-db/src/test/resources/states/global.json rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/states/global.json diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/resources/states/legacy.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/states/legacy.json similarity index 100% rename from airbyte-integrations/connectors/source-relational-db/src/test/resources/states/legacy.json rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/states/legacy.json diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/resources/states/per_stream.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/states/per_stream.json similarity index 100% rename from airbyte-integrations/connectors/source-relational-db/src/test/resources/states/per_stream.json rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/states/per_stream.json diff --git a/airbyte-integrations/bases/debezium/src/test/resources/test_debezium_offset.dat b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/test_debezium_offset.dat similarity index 100% rename from airbyte-integrations/bases/debezium/src/test/resources/test_debezium_offset.dat rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/test_debezium_offset.dat diff --git a/airbyte-integrations/bases/debezium/src/test/resources/update_change_event.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/update_change_event.json similarity index 100% rename from airbyte-integrations/bases/debezium/src/test/resources/update_change_event.json rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/update_change_event.json diff --git a/airbyte-integrations/bases/debezium/src/test/resources/update_message.json b/airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/update_message.json similarity index 100% rename from airbyte-integrations/bases/debezium/src/test/resources/update_message.json rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/test/resources/update_message.json diff --git a/airbyte-integrations/bases/debezium/src/testFixtures/java/io/airbyte/integrations/debezium/CdcSourceTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/debezium/CdcSourceTest.java similarity index 75% rename from airbyte-integrations/bases/debezium/src/testFixtures/java/io/airbyte/integrations/debezium/CdcSourceTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/debezium/CdcSourceTest.java index 281aeee5924b..a0ee71a226d0 100644 --- a/airbyte-integrations/bases/debezium/src/testFixtures/java/io/airbyte/integrations/debezium/CdcSourceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/debezium/CdcSourceTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.debezium; +package io.airbyte.cdk.integrations.debezium; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; @@ -16,11 +16,11 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.collect.Streams; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.testutils.TestDatabase; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.db.Database; -import io.airbyte.integrations.base.Source; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; @@ -36,7 +36,6 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.StreamDescriptor; import io.airbyte.protocol.models.v0.SyncMode; -import java.sql.SQLException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -48,66 +47,24 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class CdcSourceTest { +public abstract class CdcSourceTest> { - private static final Logger LOGGER = LoggerFactory.getLogger(CdcSourceTest.class); + static private final Logger LOGGER = LoggerFactory.getLogger(CdcSourceTest.class); - protected static final String MODELS_SCHEMA = "models_schema"; - protected static final String MODELS_STREAM_NAME = "models"; - protected static final Set STREAM_NAMES = Sets - .newHashSet(MODELS_STREAM_NAME); - protected static final String COL_ID = "id"; - protected static final String COL_MAKE_ID = "make_id"; - protected static final String COL_MODEL = "model"; - protected static final int INITIAL_WAITING_SECONDS = 5; + static protected final String MODELS_STREAM_NAME = "models"; + static protected final Set STREAM_NAMES = Set.of(MODELS_STREAM_NAME); + static protected final String COL_ID = "id"; + static protected final String COL_MAKE_ID = "make_id"; + static protected final String COL_MODEL = "model"; - protected final List MODEL_RECORDS_RANDOM = ImmutableList.of( - Jsons - .jsonNode(ImmutableMap - .of(COL_ID + "_random", 11000, COL_MAKE_ID + "_random", 1, COL_MODEL + "_random", - "Fiesta-random")), - Jsons.jsonNode(ImmutableMap - .of(COL_ID + "_random", 12000, COL_MAKE_ID + "_random", 1, COL_MODEL + "_random", - "Focus-random")), - Jsons - .jsonNode(ImmutableMap - .of(COL_ID + "_random", 13000, COL_MAKE_ID + "_random", 1, COL_MODEL + "_random", - "Ranger-random")), - Jsons.jsonNode(ImmutableMap - .of(COL_ID + "_random", 14000, COL_MAKE_ID + "_random", 2, COL_MODEL + "_random", - "GLA-random")), - Jsons.jsonNode(ImmutableMap - .of(COL_ID + "_random", 15000, COL_MAKE_ID + "_random", 2, COL_MODEL + "_random", - "A 220-random")), - Jsons - .jsonNode(ImmutableMap - .of(COL_ID + "_random", 16000, COL_MAKE_ID + "_random", 2, COL_MODEL + "_random", - "E 350-random"))); - - protected static final AirbyteCatalog CATALOG = new AirbyteCatalog().withStreams(List.of( - CatalogHelpers.createAirbyteStream( - MODELS_STREAM_NAME, - MODELS_SCHEMA, - Field.of(COL_ID, JsonSchemaType.INTEGER), - Field.of(COL_MAKE_ID, JsonSchemaType.INTEGER), - Field.of(COL_MODEL, JsonSchemaType.STRING)) - .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) - .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID))))); - protected static final ConfiguredAirbyteCatalog CONFIGURED_CATALOG = CatalogHelpers - .toDefaultConfiguredCatalog(CATALOG); - - // set all streams to incremental. - static { - CONFIGURED_CATALOG.getStreams().forEach(s -> s.setSyncMode(SyncMode.INCREMENTAL)); - } - - protected static final List MODEL_RECORDS = ImmutableList.of( + static protected final List MODEL_RECORDS = ImmutableList.of( Jsons.jsonNode(ImmutableMap.of(COL_ID, 11, COL_MAKE_ID, 1, COL_MODEL, "Fiesta")), Jsons.jsonNode(ImmutableMap.of(COL_ID, 12, COL_MAKE_ID, 1, COL_MODEL, "Focus")), Jsons.jsonNode(ImmutableMap.of(COL_ID, 13, COL_MAKE_ID, 1, COL_MODEL, "Ranger")), @@ -115,26 +72,119 @@ public abstract class CdcSourceTest { Jsons.jsonNode(ImmutableMap.of(COL_ID, 15, COL_MAKE_ID, 2, COL_MODEL, "A 220")), Jsons.jsonNode(ImmutableMap.of(COL_ID, 16, COL_MAKE_ID, 2, COL_MODEL, "E 350"))); - protected void setup() throws SQLException { - createAndPopulateTables(); + static protected final String RANDOM_TABLE_NAME = MODELS_STREAM_NAME + "_random"; + + static protected final List MODEL_RECORDS_RANDOM = MODEL_RECORDS.stream() + .map(r -> Jsons.jsonNode(ImmutableMap.of( + COL_ID + "_random", r.get(COL_ID).asInt() * 1000, + COL_MAKE_ID + "_random", r.get(COL_MAKE_ID), + COL_MODEL + "_random", r.get(COL_MODEL).asText() + "-random"))) + .toList(); + + protected T testdb; + + protected String createTableSqlFmt() { + return "CREATE TABLE %s.%s(%s);"; } - private void createAndPopulateTables() { - createAndPopulateActualTable(); - createAndPopulateRandomTable(); + protected String createSchemaSqlFmt() { + return "CREATE SCHEMA %s;"; + } + + protected String modelsSchema() { + return "models_schema"; + } + + /** + * The schema of a random table which is used as a new table in snapshot test + */ + protected String randomSchema() { + return "models_schema_random"; + } + + protected AirbyteCatalog getCatalog() { + return new AirbyteCatalog().withStreams(List.of( + CatalogHelpers.createAirbyteStream( + MODELS_STREAM_NAME, + modelsSchema(), + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_MAKE_ID, JsonSchemaType.INTEGER), + Field.of(COL_MODEL, JsonSchemaType.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID))))); + } + + protected ConfiguredAirbyteCatalog getConfiguredCatalog() { + final var configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog(getCatalog()); + configuredCatalog.getStreams().forEach(s -> s.setSyncMode(SyncMode.INCREMENTAL)); + return configuredCatalog; + } + + protected abstract T createTestDatabase(); + + protected abstract S source(); + + protected abstract JsonNode config(); + + protected abstract CdcTargetPosition cdcLatestTargetPosition(); + + protected abstract CdcTargetPosition extractPosition(final JsonNode record); + + protected abstract void assertNullCdcMetaData(final JsonNode data); + + protected abstract void assertCdcMetaData(final JsonNode data, final boolean deletedAtNull); + + protected abstract void removeCDCColumns(final ObjectNode data); + + protected abstract void addCdcMetadataColumns(final AirbyteStream stream); + + protected abstract void addCdcDefaultCursorField(final AirbyteStream stream); + + protected abstract void assertExpectedStateMessages(final List stateMessages); + + @BeforeEach + protected void setup() { + testdb = createTestDatabase(); + + // create and populate actual table + final var actualColumns = ImmutableMap.of( + COL_ID, "INTEGER", + COL_MAKE_ID, "INTEGER", + COL_MODEL, "VARCHAR(200)"); + testdb + .with(createSchemaSqlFmt(), modelsSchema()) + .with(createTableSqlFmt(), modelsSchema(), MODELS_STREAM_NAME, columnClause(actualColumns, Optional.of(COL_ID))); + for (final JsonNode recordJson : MODEL_RECORDS) { + writeModelRecord(recordJson); + } + + // Create and populate random table. + // This table is not part of Airbyte sync. It is being created just to make sure the schemas not + // being synced by Airbyte are not causing issues with our debezium logic. + final var randomColumns = ImmutableMap.of( + COL_ID + "_random", "INTEGER", + COL_MAKE_ID + "_random", "INTEGER", + COL_MODEL + "_random", "VARCHAR(200)"); + if (!randomSchema().equals(modelsSchema())) { + testdb.with(createSchemaSqlFmt(), randomSchema()); + } + testdb.with(createTableSqlFmt(), randomSchema(), RANDOM_TABLE_NAME, columnClause(randomColumns, Optional.of(COL_ID + "_random"))); + for (final JsonNode recordJson : MODEL_RECORDS_RANDOM) { + writeRecords(recordJson, randomSchema(), RANDOM_TABLE_NAME, + COL_ID + "_random", COL_MAKE_ID + "_random", COL_MODEL + "_random"); + } } - protected void executeQuery(final String query) { + @AfterEach + protected void tearDown() { try { - getDatabase().query( - ctx -> ctx - .execute(query)); - } catch (final SQLException e) { - throw new RuntimeException(e); + testdb.close(); + } catch (Throwable e) { + LOGGER.error("exception during teardown", e); } } - public String columnClause(final Map columnsWithDataType, final Optional primaryKey) { + protected String columnClause(final Map columnsWithDataType, final Optional primaryKey) { final StringBuilder columnClause = new StringBuilder(); int i = 0; for (final Map.Entry column : columnsWithDataType.entrySet()) { @@ -152,50 +202,8 @@ public String columnClause(final Map columnsWithDataType, final return columnClause.toString(); } - public void createTable(final String schemaName, final String tableName, final String columnClause) { - executeQuery(createTableQuery(schemaName, tableName, columnClause)); - } - - public String createTableQuery(final String schemaName, final String tableName, final String columnClause) { - return String.format("CREATE TABLE %s.%s(%s);", schemaName, tableName, columnClause); - } - - public void createSchema(final String schemaName) { - executeQuery(createSchemaQuery(schemaName)); - } - - public String createSchemaQuery(final String schemaName) { - return "CREATE DATABASE " + schemaName + ";"; - } - - private void createAndPopulateActualTable() { - createSchema(MODELS_SCHEMA); - createTable(MODELS_SCHEMA, MODELS_STREAM_NAME, - columnClause(ImmutableMap.of(COL_ID, "INTEGER", COL_MAKE_ID, "INTEGER", COL_MODEL, "VARCHAR(200)"), Optional.of(COL_ID))); - for (final JsonNode recordJson : MODEL_RECORDS) { - writeModelRecord(recordJson); - } - } - - /** - * This database and table is not part of Airbyte sync. It is being created just to make sure the - * databases not being synced by Airbyte are not causing issues with our debezium logic - */ - private void createAndPopulateRandomTable() { - if (!randomTableSchema().equals(MODELS_SCHEMA)) { - createSchema(randomTableSchema()); - } - createTable(randomTableSchema(), MODELS_STREAM_NAME + "_random", - columnClause(ImmutableMap.of(COL_ID + "_random", "INTEGER", COL_MAKE_ID + "_random", "INTEGER", COL_MODEL + "_random", "VARCHAR(200)"), - Optional.of(COL_ID + "_random"))); - for (final JsonNode recordJson : MODEL_RECORDS_RANDOM) { - writeRecords(recordJson, randomTableSchema(), MODELS_STREAM_NAME + "_random", - COL_ID + "_random", COL_MAKE_ID + "_random", COL_MODEL + "_random"); - } - } - protected void writeModelRecord(final JsonNode recordJson) { - writeRecords(recordJson, MODELS_SCHEMA, MODELS_STREAM_NAME, COL_ID, COL_MAKE_ID, COL_MODEL); + writeRecords(recordJson, modelsSchema(), MODELS_STREAM_NAME, COL_ID, COL_MAKE_ID, COL_MODEL); } protected void writeRecords( @@ -205,14 +213,13 @@ protected void writeRecords( final String idCol, final String makeIdCol, final String modelCol) { - executeQuery( - String.format("INSERT INTO %s.%s (%s, %s, %s) VALUES (%s, %s, '%s');", dbName, streamName, - idCol, makeIdCol, modelCol, - recordJson.get(idCol).asInt(), recordJson.get(makeIdCol).asInt(), - recordJson.get(modelCol).asText())); + testdb.with("INSERT INTO %s.%s (%s, %s, %s) VALUES (%s, %s, '%s');", dbName, streamName, + idCol, makeIdCol, modelCol, + recordJson.get(idCol).asInt(), recordJson.get(makeIdCol).asInt(), + recordJson.get(modelCol).asText()); } - protected static Set removeDuplicates(final Set messages) { + static protected Set removeDuplicates(final Set messages) { final Set existingDataRecordsWithoutUpdated = new HashSet<>(); final Set output = new HashSet<>(); @@ -273,7 +280,7 @@ protected void assertExpectedRecords(final Set expectedRecords, final private void assertExpectedRecords(final Set expectedRecords, final Set actualRecords, final Set cdcStreams) { - assertExpectedRecords(expectedRecords, actualRecords, cdcStreams, STREAM_NAMES, MODELS_SCHEMA); + assertExpectedRecords(expectedRecords, actualRecords, cdcStreams, STREAM_NAMES, modelsSchema()); } protected void assertExpectedRecords(final Set expectedRecords, @@ -306,13 +313,11 @@ protected void assertExpectedRecords(final Set expectedRecords, assertEquals(expectedRecords, actualData); } - // Failing on `source-postgres`, possibly others as well. - @Disabled("The 'testExistingData()' test is flaky. https://github.com/airbytehq/airbyte/issues/29411") @Test @DisplayName("On the first sync, produce returns records that exist in the database.") void testExistingData() throws Exception { final CdcTargetPosition targetPosition = cdcLatestTargetPosition(); - final AutoCloseableIterator read = getSource().read(getConfig(), CONFIGURED_CATALOG, null); + final AutoCloseableIterator read = source().read(config(), getConfiguredCatalog(), null); final List actualRecords = AutoCloseableIterators.toListAndClose(read); final Set recordMessages = extractRecordMessages(actualRecords); @@ -320,29 +325,32 @@ void testExistingData() throws Exception { assertNotNull(targetPosition); recordMessages.forEach(record -> { - assertEquals(extractPosition(record.getData()), targetPosition); + compareTargetPositionFromTheRecordsWithTargetPostionGeneratedBeforeSync(targetPosition, record); }); assertExpectedRecords(new HashSet<>(MODEL_RECORDS), recordMessages); assertExpectedStateMessages(stateMessages); } + protected void compareTargetPositionFromTheRecordsWithTargetPostionGeneratedBeforeSync(final CdcTargetPosition targetPosition, + final AirbyteRecordMessage record) { + assertEquals(extractPosition(record.getData()), targetPosition); + } + @Test @DisplayName("When a record is deleted, produces a deletion record.") void testDelete() throws Exception { - final AutoCloseableIterator read1 = getSource() - .read(getConfig(), CONFIGURED_CATALOG, null); + final AutoCloseableIterator read1 = source() + .read(config(), getConfiguredCatalog(), null); final List actualRecords1 = AutoCloseableIterators.toListAndClose(read1); final List stateMessages1 = extractStateMessages(actualRecords1); assertExpectedStateMessages(stateMessages1); - executeQuery(String - .format("DELETE FROM %s.%s WHERE %s = %s", MODELS_SCHEMA, MODELS_STREAM_NAME, COL_ID, - 11)); + testdb.with("DELETE FROM %s.%s WHERE %s = %s", modelsSchema(), MODELS_STREAM_NAME, COL_ID, 11); final JsonNode state = Jsons.jsonNode(Collections.singletonList(stateMessages1.get(stateMessages1.size() - 1))); - final AutoCloseableIterator read2 = getSource() - .read(getConfig(), CONFIGURED_CATALOG, state); + final AutoCloseableIterator read2 = source() + .read(config(), getConfiguredCatalog(), state); final List actualRecords2 = AutoCloseableIterators.toListAndClose(read2); final List recordMessages2 = new ArrayList<>( extractRecordMessages(actualRecords2)); @@ -361,19 +369,18 @@ protected void assertExpectedStateMessagesFromIncrementalSync(final List read1 = getSource() - .read(getConfig(), CONFIGURED_CATALOG, null); + final AutoCloseableIterator read1 = source() + .read(config(), getConfiguredCatalog(), null); final List actualRecords1 = AutoCloseableIterators.toListAndClose(read1); final List stateMessages1 = extractStateMessages(actualRecords1); assertExpectedStateMessages(stateMessages1); - executeQuery(String - .format("UPDATE %s.%s SET %s = '%s' WHERE %s = %s", MODELS_SCHEMA, MODELS_STREAM_NAME, - COL_MODEL, updatedModel, COL_ID, 11)); + testdb.with("UPDATE %s.%s SET %s = '%s' WHERE %s = %s", modelsSchema(), MODELS_STREAM_NAME, + COL_MODEL, updatedModel, COL_ID, 11); final JsonNode state = Jsons.jsonNode(Collections.singletonList(stateMessages1.get(stateMessages1.size() - 1))); - final AutoCloseableIterator read2 = getSource() - .read(getConfig(), CONFIGURED_CATALOG, state); + final AutoCloseableIterator read2 = source() + .read(config(), getConfiguredCatalog(), state); final List actualRecords2 = AutoCloseableIterators.toListAndClose(read2); final List recordMessages2 = new ArrayList<>( extractRecordMessages(actualRecords2)); @@ -400,8 +407,8 @@ protected void testRecordsProducedDuringAndAfterSync() throws Exception { writeModelRecord(record); } - final AutoCloseableIterator firstBatchIterator = getSource() - .read(getConfig(), CONFIGURED_CATALOG, null); + final AutoCloseableIterator firstBatchIterator = source() + .read(config(), getConfiguredCatalog(), null); final List dataFromFirstBatch = AutoCloseableIterators .toListAndClose(firstBatchIterator); final List stateAfterFirstBatch = extractStateMessages(dataFromFirstBatch); @@ -420,8 +427,8 @@ protected void testRecordsProducedDuringAndAfterSync() throws Exception { } final JsonNode state = Jsons.jsonNode(Collections.singletonList(stateAfterFirstBatch.get(stateAfterFirstBatch.size() - 1))); - final AutoCloseableIterator secondBatchIterator = getSource() - .read(getConfig(), CONFIGURED_CATALOG, state); + final AutoCloseableIterator secondBatchIterator = source() + .read(config(), getConfiguredCatalog(), state); final List dataFromSecondBatch = AutoCloseableIterators .toListAndClose(secondBatchIterator); @@ -455,7 +462,7 @@ protected void assertExpectedStateMessagesForRecordsProducedDuringAndAfterSync(f @Test @DisplayName("When both incremental CDC and full refresh are configured for different streams in a sync, the data is replicated as expected.") void testCdcAndFullRefreshInSameSync() throws Exception { - final ConfiguredAirbyteCatalog configuredCatalog = Jsons.clone(CONFIGURED_CATALOG); + final ConfiguredAirbyteCatalog configuredCatalog = Jsons.clone(getConfiguredCatalog()); final List MODEL_RECORDS_2 = ImmutableList.of( Jsons.jsonNode(ImmutableMap.of(COL_ID, 110, COL_MAKE_ID, 1, COL_MODEL, "Fiesta-2")), @@ -465,18 +472,17 @@ void testCdcAndFullRefreshInSameSync() throws Exception { Jsons.jsonNode(ImmutableMap.of(COL_ID, 150, COL_MAKE_ID, 2, COL_MODEL, "A 220-2")), Jsons.jsonNode(ImmutableMap.of(COL_ID, 160, COL_MAKE_ID, 2, COL_MODEL, "E 350-2"))); - createTable(MODELS_SCHEMA, MODELS_STREAM_NAME + "_2", - columnClause(ImmutableMap.of(COL_ID, "INTEGER", COL_MAKE_ID, "INTEGER", COL_MODEL, "VARCHAR(200)"), Optional.of(COL_ID))); + final var columns = ImmutableMap.of(COL_ID, "INTEGER", COL_MAKE_ID, "INTEGER", COL_MODEL, "VARCHAR(200)"); + testdb.with(createTableSqlFmt(), modelsSchema(), MODELS_STREAM_NAME + "_2", columnClause(columns, Optional.of(COL_ID))); for (final JsonNode recordJson : MODEL_RECORDS_2) { - writeRecords(recordJson, MODELS_SCHEMA, MODELS_STREAM_NAME + "_2", COL_ID, - COL_MAKE_ID, COL_MODEL); + writeRecords(recordJson, modelsSchema(), MODELS_STREAM_NAME + "_2", COL_ID, COL_MAKE_ID, COL_MODEL); } final ConfiguredAirbyteStream airbyteStream = new ConfiguredAirbyteStream() .withStream(CatalogHelpers.createAirbyteStream( MODELS_STREAM_NAME + "_2", - MODELS_SCHEMA, + modelsSchema(), Field.of(COL_ID, JsonSchemaType.INTEGER), Field.of(COL_MAKE_ID, JsonSchemaType.INTEGER), Field.of(COL_MODEL, JsonSchemaType.STRING)) @@ -489,8 +495,8 @@ void testCdcAndFullRefreshInSameSync() throws Exception { streams.add(airbyteStream); configuredCatalog.withStreams(streams); - final AutoCloseableIterator read1 = getSource() - .read(getConfig(), configuredCatalog, null); + final AutoCloseableIterator read1 = source() + .read(config(), configuredCatalog, null); final List actualRecords1 = AutoCloseableIterators.toListAndClose(read1); final Set recordMessages1 = extractRecordMessages(actualRecords1); @@ -503,15 +509,15 @@ void testCdcAndFullRefreshInSameSync() throws Exception { recordMessages1, Collections.singleton(MODELS_STREAM_NAME), names, - MODELS_SCHEMA); + modelsSchema()); final JsonNode puntoRecord = Jsons .jsonNode(ImmutableMap.of(COL_ID, 100, COL_MAKE_ID, 3, COL_MODEL, "Punto")); writeModelRecord(puntoRecord); final JsonNode state = Jsons.jsonNode(Collections.singletonList(stateMessages1.get(stateMessages1.size() - 1))); - final AutoCloseableIterator read2 = getSource() - .read(getConfig(), configuredCatalog, state); + final AutoCloseableIterator read2 = source() + .read(config(), configuredCatalog, state); final List actualRecords2 = AutoCloseableIterators.toListAndClose(read2); final Set recordMessages2 = extractRecordMessages(actualRecords2); @@ -523,17 +529,16 @@ void testCdcAndFullRefreshInSameSync() throws Exception { recordMessages2, Collections.singleton(MODELS_STREAM_NAME), names, - MODELS_SCHEMA); + modelsSchema()); } @Test @DisplayName("When no records exist, no records are returned.") void testNoData() throws Exception { - executeQuery(String.format("DELETE FROM %s.%s", MODELS_SCHEMA, MODELS_STREAM_NAME)); + testdb.with("DELETE FROM %s.%s", modelsSchema(), MODELS_STREAM_NAME); - final AutoCloseableIterator read = getSource() - .read(getConfig(), CONFIGURED_CATALOG, null); + final AutoCloseableIterator read = source().read(config(), getConfiguredCatalog(), null); final List actualRecords = AutoCloseableIterators.toListAndClose(read); final Set recordMessages = extractRecordMessages(actualRecords); @@ -549,14 +554,14 @@ protected void assertExpectedStateMessagesForNoData(final List read1 = getSource() - .read(getConfig(), CONFIGURED_CATALOG, null); + final AutoCloseableIterator read1 = source() + .read(config(), getConfiguredCatalog(), null); final List actualRecords1 = AutoCloseableIterators.toListAndClose(read1); final List stateMessagesFromFirstSync = extractStateMessages(actualRecords1); final JsonNode state = Jsons.jsonNode(Collections.singletonList(stateMessagesFromFirstSync.get(stateMessagesFromFirstSync.size() - 1))); - final AutoCloseableIterator read2 = getSource() - .read(getConfig(), CONFIGURED_CATALOG, state); + final AutoCloseableIterator read2 = source() + .read(config(), getConfiguredCatalog(), state); final List actualRecords2 = AutoCloseableIterators.toListAndClose(read2); final Set recordMessages2 = extractRecordMessages(actualRecords2); @@ -568,14 +573,14 @@ void testNoDataOnSecondSync() throws Exception { @Test void testCheck() throws Exception { - final AirbyteConnectionStatus status = getSource().check(getConfig()); + final AirbyteConnectionStatus status = source().check(config()); assertEquals(status.getStatus(), AirbyteConnectionStatus.Status.SUCCEEDED); } @Test void testDiscover() throws Exception { final AirbyteCatalog expectedCatalog = expectedCatalogForDiscover(); - final AirbyteCatalog actualCatalog = getSource().discover(getConfig()); + final AirbyteCatalog actualCatalog = source().discover(config()); assertEquals( expectedCatalog.getStreams().stream().sorted(Comparator.comparing(AirbyteStream::getName)) @@ -586,8 +591,8 @@ void testDiscover() throws Exception { @Test public void newTableSnapshotTest() throws Exception { - final AutoCloseableIterator firstBatchIterator = getSource() - .read(getConfig(), CONFIGURED_CATALOG, null); + final AutoCloseableIterator firstBatchIterator = source() + .read(config(), getConfiguredCatalog(), null); final List dataFromFirstBatch = AutoCloseableIterators .toListAndClose(firstBatchIterator); final Set recordsFromFirstBatch = extractRecordMessages( @@ -603,7 +608,7 @@ public void newTableSnapshotTest() throws Exception { .map(AirbyteStreamState::getStreamDescriptor) .collect(Collectors.toSet()); assertEquals(1, streamsInStateAfterFirstSyncCompletion.size()); - assertTrue(streamsInStateAfterFirstSyncCompletion.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(MODELS_SCHEMA))); + assertTrue(streamsInStateAfterFirstSyncCompletion.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(modelsSchema()))); assertNotNull(stateMessageEmittedAfterFirstSyncCompletion.getData()); assertEquals((MODEL_RECORDS.size()), recordsFromFirstBatch.size()); @@ -614,8 +619,8 @@ public void newTableSnapshotTest() throws Exception { final ConfiguredAirbyteCatalog newTables = CatalogHelpers .toDefaultConfiguredCatalog(new AirbyteCatalog().withStreams(List.of( CatalogHelpers.createAirbyteStream( - MODELS_STREAM_NAME + "_random", - randomTableSchema(), + RANDOM_TABLE_NAME, + randomSchema(), Field.of(COL_ID + "_random", JsonSchemaType.NUMBER), Field.of(COL_MAKE_ID + "_random", JsonSchemaType.NUMBER), Field.of(COL_MODEL + "_random", JsonSchemaType.STRING)) @@ -624,7 +629,7 @@ public void newTableSnapshotTest() throws Exception { newTables.getStreams().forEach(s -> s.setSyncMode(SyncMode.INCREMENTAL)); final List combinedStreams = new ArrayList<>(); - combinedStreams.addAll(CONFIGURED_CATALOG.getStreams()); + combinedStreams.addAll(getConfiguredCatalog().getStreams()); combinedStreams.addAll(newTables.getStreams()); final ConfiguredAirbyteCatalog updatedCatalog = new ConfiguredAirbyteCatalog().withStreams(combinedStreams); @@ -642,8 +647,8 @@ public void newTableSnapshotTest() throws Exception { writeModelRecord(record); } - final AutoCloseableIterator secondBatchIterator = getSource() - .read(getConfig(), updatedCatalog, state); + final AutoCloseableIterator secondBatchIterator = source() + .read(config(), updatedCatalog, state); final List dataFromSecondBatch = AutoCloseableIterators .toListAndClose(secondBatchIterator); @@ -652,10 +657,10 @@ public void newTableSnapshotTest() throws Exception { final Map> recordsStreamWise = extractRecordMessagesStreamWise(dataFromSecondBatch); assertTrue(recordsStreamWise.containsKey(MODELS_STREAM_NAME)); - assertTrue(recordsStreamWise.containsKey(MODELS_STREAM_NAME + "_random")); + assertTrue(recordsStreamWise.containsKey(RANDOM_TABLE_NAME)); final Set recordsForModelsStreamFromSecondBatch = recordsStreamWise.get(MODELS_STREAM_NAME); - final Set recordsForModelsRandomStreamFromSecondBatch = recordsStreamWise.get(MODELS_STREAM_NAME + "_random"); + final Set recordsForModelsRandomStreamFromSecondBatch = recordsStreamWise.get(RANDOM_TABLE_NAME); assertEquals((MODEL_RECORDS_RANDOM.size()), recordsForModelsRandomStreamFromSecondBatch.size()); assertEquals(20, recordsForModelsStreamFromSecondBatch.size()); @@ -663,8 +668,8 @@ public void newTableSnapshotTest() throws Exception { recordsForModelsRandomStreamFromSecondBatch.stream().map(AirbyteRecordMessage::getStream).collect( Collectors.toSet()), Sets - .newHashSet(MODELS_STREAM_NAME + "_random"), - randomTableSchema()); + .newHashSet(RANDOM_TABLE_NAME), + randomSchema()); assertExpectedRecords(recordsWritten, recordsForModelsStreamFromSecondBatch); /* @@ -684,14 +689,14 @@ public void newTableSnapshotTest() throws Exception { .jsonNode(ImmutableMap .of(COL_ID + "_random", 11000 + recordsCreated, COL_MAKE_ID + "_random", 1 + recordsCreated, COL_MODEL + "_random", "Fiesta-random" + recordsCreated)); - writeRecords(record2, randomTableSchema(), MODELS_STREAM_NAME + "_random", + writeRecords(record2, randomSchema(), RANDOM_TABLE_NAME, COL_ID + "_random", COL_MAKE_ID + "_random", COL_MODEL + "_random"); recordsWrittenInRandomTable.add(record2); } final JsonNode state2 = stateAfterSecondBatch.get(stateAfterSecondBatch.size() - 1).getData(); - final AutoCloseableIterator thirdBatchIterator = getSource() - .read(getConfig(), updatedCatalog, state2); + final AutoCloseableIterator thirdBatchIterator = source() + .read(config(), updatedCatalog, state2); final List dataFromThirdBatch = AutoCloseableIterators .toListAndClose(thirdBatchIterator); @@ -708,16 +713,17 @@ public void newTableSnapshotTest() throws Exception { .collect(Collectors.toSet()); assertTrue( streamsInSyncCompletionStateAfterThirdSync.contains( - new StreamDescriptor().withName(MODELS_STREAM_NAME + "_random").withNamespace(randomTableSchema()))); - assertTrue(streamsInSyncCompletionStateAfterThirdSync.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(MODELS_SCHEMA))); + new StreamDescriptor().withName(RANDOM_TABLE_NAME).withNamespace(randomSchema()))); + assertTrue( + streamsInSyncCompletionStateAfterThirdSync.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(modelsSchema()))); assertNotNull(stateMessageEmittedAfterThirdSyncCompletion.getData()); final Map> recordsStreamWiseFromThirdBatch = extractRecordMessagesStreamWise(dataFromThirdBatch); assertTrue(recordsStreamWiseFromThirdBatch.containsKey(MODELS_STREAM_NAME)); - assertTrue(recordsStreamWiseFromThirdBatch.containsKey(MODELS_STREAM_NAME + "_random")); + assertTrue(recordsStreamWiseFromThirdBatch.containsKey(RANDOM_TABLE_NAME)); final Set recordsForModelsStreamFromThirdBatch = recordsStreamWiseFromThirdBatch.get(MODELS_STREAM_NAME); - final Set recordsForModelsRandomStreamFromThirdBatch = recordsStreamWiseFromThirdBatch.get(MODELS_STREAM_NAME + "_random"); + final Set recordsForModelsRandomStreamFromThirdBatch = recordsStreamWiseFromThirdBatch.get(RANDOM_TABLE_NAME); assertEquals(20, recordsForModelsStreamFromThirdBatch.size()); assertEquals(20, recordsForModelsRandomStreamFromThirdBatch.size()); @@ -726,8 +732,8 @@ public void newTableSnapshotTest() throws Exception { recordsForModelsRandomStreamFromThirdBatch.stream().map(AirbyteRecordMessage::getStream).collect( Collectors.toSet()), Sets - .newHashSet(MODELS_STREAM_NAME + "_random"), - randomTableSchema()); + .newHashSet(RANDOM_TABLE_NAME), + randomSchema()); } protected void assertStateMessagesForNewTableSnapshotTest(final List stateMessages, @@ -743,8 +749,8 @@ protected void assertStateMessagesForNewTableSnapshotTest(final List streams = expectedCatalog.getStreams(); // stream with PK @@ -777,7 +783,7 @@ protected AirbyteCatalog expectedCatalogForDiscover() { final AirbyteStream streamWithoutPK = CatalogHelpers.createAirbyteStream( MODELS_STREAM_NAME + "_2", - MODELS_SCHEMA, + modelsSchema(), Field.of(COL_ID, JsonSchemaType.INTEGER), Field.of(COL_MAKE_ID, JsonSchemaType.INTEGER), Field.of(COL_MODEL, JsonSchemaType.STRING)); @@ -787,8 +793,8 @@ protected AirbyteCatalog expectedCatalogForDiscover() { addCdcMetadataColumns(streamWithoutPK); final AirbyteStream randomStream = CatalogHelpers.createAirbyteStream( - MODELS_STREAM_NAME + "_random", - randomTableSchema(), + RANDOM_TABLE_NAME, + randomSchema(), Field.of(COL_ID + "_random", JsonSchemaType.INTEGER), Field.of(COL_MAKE_ID + "_random", JsonSchemaType.INTEGER), Field.of(COL_MODEL + "_random", JsonSchemaType.STRING)) @@ -805,31 +811,4 @@ protected AirbyteCatalog expectedCatalogForDiscover() { return expectedCatalog; } - /** - * The schema of a random table which is used as a new table in snapshot test - */ - protected abstract String randomTableSchema(); - - protected abstract CdcTargetPosition cdcLatestTargetPosition(); - - protected abstract CdcTargetPosition extractPosition(final JsonNode record); - - protected abstract void assertNullCdcMetaData(final JsonNode data); - - protected abstract void assertCdcMetaData(final JsonNode data, final boolean deletedAtNull); - - protected abstract void removeCDCColumns(final ObjectNode data); - - protected abstract void addCdcMetadataColumns(final AirbyteStream stream); - - protected abstract void addCdcDefaultCursorField(final AirbyteStream stream); - - protected abstract Source getSource(); - - protected abstract JsonNode getConfig(); - - protected abstract Database getDatabase(); - - protected abstract void assertExpectedStateMessages(final List stateMessages); - } diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/debug/DebugUtil.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/debug/DebugUtil.java new file mode 100644 index 000000000000..836f6cf50347 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/debug/DebugUtil.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.debug; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import java.util.Collections; + +/** + * Utility class defined to debug a source. Copy over any relevant configurations, catalogs & state + * in the resources/debug_resources directory. + */ +public class DebugUtil { + + @SuppressWarnings({"unchecked", "deprecation", "resource"}) + public static void debug(final Source debugSource) throws Exception { + final JsonNode debugConfig = DebugUtil.getConfig(); + final ConfiguredAirbyteCatalog configuredAirbyteCatalog = DebugUtil.getCatalog(); + JsonNode state; + try { + state = DebugUtil.getState(); + } catch (final Exception e) { + state = null; + } + + debugSource.check(debugConfig); + debugSource.discover(debugConfig); + + final AutoCloseableIterator messageIterator = debugSource.read(debugConfig, configuredAirbyteCatalog, state); + messageIterator.forEachRemaining(message -> {}); + } + + private static JsonNode getConfig() throws Exception { + final JsonNode originalConfig = new ObjectMapper().readTree(MoreResources.readResource("debug_resources/config.json")); + final JsonNode debugConfig = ((ObjectNode) originalConfig.deepCopy()).put("debug_mode", true); + return debugConfig; + } + + private static ConfiguredAirbyteCatalog getCatalog() throws Exception { + final String catalog = MoreResources.readResource("debug_resources/configured_catalog.json"); + return Jsons.deserialize(catalog, ConfiguredAirbyteCatalog.class); + } + + private static JsonNode getState() throws Exception { + final AirbyteStateMessage message = Jsons.deserialize(MoreResources.readResource("debug_resources/state.json"), AirbyteStateMessage.class); + return Jsons.jsonNode(Collections.singletonList(message)); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java new file mode 100644 index 000000000000..ae358d0f8e8d --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java @@ -0,0 +1,1108 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.source.jdbc.test; + +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifier; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.doCallRealMethod; +import static org.mockito.Mockito.spy; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbState; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; +import io.airbyte.cdk.testutils.TestDatabase; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.commons.util.MoreIterators; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteCatalog; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStateStats; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.AirbyteStreamState; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.ConnectorSpecification; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import io.airbyte.protocol.models.v0.SyncMode; +import java.math.BigDecimal; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.hamcrest.Matchers; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * Tests that should be run on all Sources that extend the AbstractJdbcSource. + */ +@SuppressFBWarnings( + value = {"MS_SHOULD_BE_FINAL"}, + justification = "The static variables are updated in subclasses for convenience, and cannot be final.") +abstract public class JdbcSourceAcceptanceTest> { + + static protected String SCHEMA_NAME = "jdbc_integration_test1"; + static protected String SCHEMA_NAME2 = "jdbc_integration_test2"; + static protected Set TEST_SCHEMAS = Set.of(SCHEMA_NAME, SCHEMA_NAME2); + + static protected String TABLE_NAME = "id_and_name"; + static protected String TABLE_NAME_WITH_SPACES = "id and name"; + static protected String TABLE_NAME_WITHOUT_PK = "id_and_name_without_pk"; + static protected String TABLE_NAME_COMPOSITE_PK = "full_name_composite_pk"; + static protected String TABLE_NAME_WITHOUT_CURSOR_TYPE = "table_without_cursor_type"; + static protected String TABLE_NAME_WITH_NULLABLE_CURSOR_TYPE = "table_with_null_cursor_type"; + // this table is used in testing incremental sync with concurrent insertions + static protected String TABLE_NAME_AND_TIMESTAMP = "name_and_timestamp"; + + static protected String COL_ID = "id"; + static protected String COL_NAME = "name"; + static protected String COL_UPDATED_AT = "updated_at"; + static protected String COL_FIRST_NAME = "first_name"; + static protected String COL_LAST_NAME = "last_name"; + static protected String COL_LAST_NAME_WITH_SPACE = "last name"; + static protected String COL_CURSOR = "cursor_field"; + static protected String COL_TIMESTAMP = "timestamp"; + static protected String COL_TIMESTAMP_TYPE = "TIMESTAMP"; + static protected Number ID_VALUE_1 = 1; + static protected Number ID_VALUE_2 = 2; + static protected Number ID_VALUE_3 = 3; + static protected Number ID_VALUE_4 = 4; + static protected Number ID_VALUE_5 = 5; + + static protected String DROP_SCHEMA_QUERY = "DROP SCHEMA IF EXISTS %s CASCADE"; + static protected String COLUMN_CLAUSE_WITH_PK = "id INTEGER, name VARCHAR(200) NOT NULL, updated_at DATE NOT NULL"; + static protected String COLUMN_CLAUSE_WITHOUT_PK = "id INTEGER, name VARCHAR(200) NOT NULL, updated_at DATE NOT NULL"; + static protected String COLUMN_CLAUSE_WITH_COMPOSITE_PK = + "first_name VARCHAR(200) NOT NULL, last_name VARCHAR(200) NOT NULL, updated_at DATE NOT NULL"; + + static protected String CREATE_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s bit NOT NULL);"; + static protected String INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES(0);"; + static protected String CREATE_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s VARCHAR(20));"; + static protected String INSERT_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES('Hello world :)');"; + static protected String INSERT_TABLE_NAME_AND_TIMESTAMP_QUERY = "INSERT INTO %s (name, timestamp) VALUES ('%s', '%s')"; + + protected T testdb; + + protected String streamName() { + return TABLE_NAME; + } + + /** + * A valid configuration to connect to a test database. + * + * @return config + */ + abstract protected JsonNode config(); + + /** + * An instance of the source that should be tests. + * + * @return abstract jdbc source + */ + abstract protected S source(); + + /** + * Creates a TestDatabase instance to be used in {@link #setup()}. + * + * @return TestDatabase instance to use for test case. + */ + abstract protected T createTestDatabase(); + + /** + * These tests write records without specifying a namespace (schema name). They will be written into + * whatever the default schema is for the database. When they are discovered they will be namespaced + * by the schema name (e.g. .). Thus the source needs to tell the + * tests what that default schema name is. If the database does not support schemas, then database + * name should used instead. + * + * @return name that will be used to namespace the record. + */ + abstract protected boolean supportsSchemas(); + + protected String createTableQuery(final String tableName, final String columnClause, final String primaryKeyClause) { + return String.format("CREATE TABLE %s(%s %s %s)", + tableName, columnClause, primaryKeyClause.equals("") ? "" : ",", primaryKeyClause); + } + + protected String primaryKeyClause(final List columns) { + if (columns.isEmpty()) { + return ""; + } + + final StringBuilder clause = new StringBuilder(); + clause.append("PRIMARY KEY ("); + for (int i = 0; i < columns.size(); i++) { + clause.append(columns.get(i)); + if (i != (columns.size() - 1)) { + clause.append(","); + } + } + clause.append(")"); + return clause.toString(); + } + + @BeforeEach + public void setup() throws Exception { + testdb = createTestDatabase(); + if (supportsSchemas()) { + createSchemas(); + } + if (testdb.getDatabaseDriver().equals(DatabaseDriver.ORACLE)) { + testdb.with("ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD'"); + } + testdb + .with(createTableQuery(getFullyQualifiedTableName(TABLE_NAME), COLUMN_CLAUSE_WITH_PK, primaryKeyClause(Collections.singletonList("id")))) + .with("INSERT INTO %s(id, name, updated_at) VALUES (1, 'picard', '2004-10-19')", getFullyQualifiedTableName(TABLE_NAME)) + .with("INSERT INTO %s(id, name, updated_at) VALUES (2, 'crusher', '2005-10-19')", getFullyQualifiedTableName(TABLE_NAME)) + .with("INSERT INTO %s(id, name, updated_at) VALUES (3, 'vash', '2006-10-19')", getFullyQualifiedTableName(TABLE_NAME)) + .with(createTableQuery(getFullyQualifiedTableName(TABLE_NAME_WITHOUT_PK), COLUMN_CLAUSE_WITHOUT_PK, "")) + .with("INSERT INTO %s(id, name, updated_at) VALUES (1, 'picard', '2004-10-19')", getFullyQualifiedTableName(TABLE_NAME_WITHOUT_PK)) + .with("INSERT INTO %s(id, name, updated_at) VALUES (2, 'crusher', '2005-10-19')", getFullyQualifiedTableName(TABLE_NAME_WITHOUT_PK)) + .with("INSERT INTO %s(id, name, updated_at) VALUES (3, 'vash', '2006-10-19')", getFullyQualifiedTableName(TABLE_NAME_WITHOUT_PK)) + .with(createTableQuery(getFullyQualifiedTableName(TABLE_NAME_COMPOSITE_PK), COLUMN_CLAUSE_WITH_COMPOSITE_PK, + primaryKeyClause(List.of("first_name", "last_name")))) + .with("INSERT INTO %s(first_name, last_name, updated_at) VALUES ('first', 'picard', '2004-10-19')", + getFullyQualifiedTableName(TABLE_NAME_COMPOSITE_PK)) + .with("INSERT INTO %s(first_name, last_name, updated_at) VALUES ('second', 'crusher', '2005-10-19')", + getFullyQualifiedTableName(TABLE_NAME_COMPOSITE_PK)) + .with("INSERT INTO %s(first_name, last_name, updated_at) VALUES ('third', 'vash', '2006-10-19')", + getFullyQualifiedTableName(TABLE_NAME_COMPOSITE_PK)); + } + + protected void maybeSetShorterConnectionTimeout(final JsonNode config) { + // Optionally implement this to speed up test cases which will result in a connection timeout. + } + + @AfterEach + public void tearDown() { + testdb.close(); + } + + @Test + void testSpec() throws Exception { + final ConnectorSpecification actual = source().spec(); + final String resourceString = MoreResources.readResource("spec.json"); + final ConnectorSpecification expected = Jsons.deserialize(resourceString, ConnectorSpecification.class); + + assertEquals(expected, actual); + } + + @Test + void testCheckSuccess() throws Exception { + final AirbyteConnectionStatus actual = source().check(config()); + final AirbyteConnectionStatus expected = new AirbyteConnectionStatus().withStatus(Status.SUCCEEDED); + assertEquals(expected, actual); + } + + @Test + protected void testCheckFailure() throws Exception { + final var config = config(); + maybeSetShorterConnectionTimeout(config); + ((ObjectNode) config).put(JdbcUtils.PASSWORD_KEY, "fake"); + final AirbyteConnectionStatus actual = source().check(config); + assertEquals(Status.FAILED, actual.getStatus()); + } + + @Test + void testDiscover() throws Exception { + final AirbyteCatalog actual = filterOutOtherSchemas(source().discover(config())); + final AirbyteCatalog expected = getCatalog(getDefaultNamespace()); + assertEquals(expected.getStreams().size(), actual.getStreams().size()); + actual.getStreams().forEach(actualStream -> { + final Optional expectedStream = + expected.getStreams().stream() + .filter(stream -> stream.getNamespace().equals(actualStream.getNamespace()) && stream.getName().equals(actualStream.getName())) + .findAny(); + assertTrue(expectedStream.isPresent(), String.format("Unexpected stream %s", actualStream.getName())); + assertEquals(expectedStream.get(), actualStream); + }); + } + + @Test + protected void testDiscoverWithNonCursorFields() throws Exception { + testdb.with(CREATE_TABLE_WITHOUT_CURSOR_TYPE_QUERY, getFullyQualifiedTableName(TABLE_NAME_WITHOUT_CURSOR_TYPE), COL_CURSOR) + .with(INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY, getFullyQualifiedTableName(TABLE_NAME_WITHOUT_CURSOR_TYPE)); + final AirbyteCatalog actual = filterOutOtherSchemas(source().discover(config())); + final AirbyteStream stream = + actual.getStreams().stream().filter(s -> s.getName().equalsIgnoreCase(TABLE_NAME_WITHOUT_CURSOR_TYPE)).findFirst().orElse(null); + assertNotNull(stream); + assertEquals(TABLE_NAME_WITHOUT_CURSOR_TYPE.toLowerCase(), stream.getName().toLowerCase()); + assertEquals(1, stream.getSupportedSyncModes().size()); + assertEquals(SyncMode.FULL_REFRESH, stream.getSupportedSyncModes().get(0)); + } + + @Test + protected void testDiscoverWithNullableCursorFields() throws Exception { + testdb.with(CREATE_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY, getFullyQualifiedTableName(TABLE_NAME_WITH_NULLABLE_CURSOR_TYPE), COL_CURSOR) + .with(INSERT_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY, getFullyQualifiedTableName(TABLE_NAME_WITH_NULLABLE_CURSOR_TYPE)); + final AirbyteCatalog actual = filterOutOtherSchemas(source().discover(config())); + final AirbyteStream stream = + actual.getStreams().stream().filter(s -> s.getName().equalsIgnoreCase(TABLE_NAME_WITH_NULLABLE_CURSOR_TYPE)).findFirst().orElse(null); + assertNotNull(stream); + assertEquals(TABLE_NAME_WITH_NULLABLE_CURSOR_TYPE.toLowerCase(), stream.getName().toLowerCase()); + assertEquals(2, stream.getSupportedSyncModes().size()); + assertTrue(stream.getSupportedSyncModes().contains(SyncMode.FULL_REFRESH)); + assertTrue(stream.getSupportedSyncModes().contains(SyncMode.INCREMENTAL)); + } + + protected AirbyteCatalog filterOutOtherSchemas(final AirbyteCatalog catalog) { + if (supportsSchemas()) { + final AirbyteCatalog filteredCatalog = Jsons.clone(catalog); + filteredCatalog.setStreams(filteredCatalog.getStreams() + .stream() + .filter(stream -> TEST_SCHEMAS.stream().anyMatch(schemaName -> stream.getNamespace().startsWith(schemaName))) + .collect(Collectors.toList())); + return filteredCatalog; + } else { + return catalog; + } + + } + + @Test + protected void testDiscoverWithMultipleSchemas() throws Exception { + // clickhouse and mysql do not have a concept of schemas, so this test does not make sense for them. + switch (testdb.getDatabaseDriver()) { + case MYSQL, CLICKHOUSE, TERADATA: + return; + } + + // add table and data to a separate schema. + testdb.with("CREATE TABLE %s(id VARCHAR(200) NOT NULL, name VARCHAR(200) NOT NULL)", + RelationalDbQueryUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME)) + .with("INSERT INTO %s(id, name) VALUES ('1','picard')", + RelationalDbQueryUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME)) + .with("INSERT INTO %s(id, name) VALUES ('2', 'crusher')", + RelationalDbQueryUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME)) + .with("INSERT INTO %s(id, name) VALUES ('3', 'vash')", + RelationalDbQueryUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME)); + + final AirbyteCatalog actual = source().discover(config()); + + final AirbyteCatalog expected = getCatalog(getDefaultNamespace()); + final List catalogStreams = new ArrayList<>(); + catalogStreams.addAll(expected.getStreams()); + catalogStreams.add(CatalogHelpers + .createAirbyteStream(TABLE_NAME, + SCHEMA_NAME2, + Field.of(COL_ID, JsonSchemaType.STRING), + Field.of(COL_NAME, JsonSchemaType.STRING)) + .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))); + expected.setStreams(catalogStreams); + // sort streams by name so that we are comparing lists with the same order. + final Comparator schemaTableCompare = Comparator.comparing(stream -> stream.getNamespace() + "." + stream.getName()); + expected.getStreams().sort(schemaTableCompare); + actual.getStreams().sort(schemaTableCompare); + assertEquals(expected, filterOutOtherSchemas(actual)); + } + + @Test + void testReadSuccess() throws Exception { + final List actualMessages = + MoreIterators.toList( + source().read(config(), getConfiguredCatalogWithOneStream(getDefaultNamespace()), null)); + + setEmittedAtToNull(actualMessages); + final List expectedMessages = getTestMessages(); + assertThat(expectedMessages, Matchers.containsInAnyOrder(actualMessages.toArray())); + assertThat(actualMessages, Matchers.containsInAnyOrder(expectedMessages.toArray())); + } + + @Test + protected void testReadOneColumn() throws Exception { + final ConfiguredAirbyteCatalog catalog = CatalogHelpers + .createConfiguredAirbyteCatalog(streamName(), getDefaultNamespace(), Field.of(COL_ID, JsonSchemaType.NUMBER)); + final List actualMessages = MoreIterators + .toList(source().read(config(), catalog, null)); + + setEmittedAtToNull(actualMessages); + + final List expectedMessages = getAirbyteMessagesReadOneColumn(); + assertEquals(expectedMessages.size(), actualMessages.size()); + assertTrue(expectedMessages.containsAll(actualMessages)); + assertTrue(actualMessages.containsAll(expectedMessages)); + } + + protected List getAirbyteMessagesReadOneColumn() { + final List expectedMessages = getTestMessages().stream() + .map(Jsons::clone) + .peek(m -> { + ((ObjectNode) m.getRecord().getData()).remove(COL_NAME); + ((ObjectNode) m.getRecord().getData()).remove(COL_UPDATED_AT); + ((ObjectNode) m.getRecord().getData()).replace(COL_ID, + convertIdBasedOnDatabase(m.getRecord().getData().get(COL_ID).asInt())); + }) + .collect(Collectors.toList()); + return expectedMessages; + } + + @Test + protected void testReadMultipleTables() throws Exception { + final ConfiguredAirbyteCatalog catalog = getConfiguredCatalogWithOneStream( + getDefaultNamespace()); + final List expectedMessages = new ArrayList<>(getTestMessages()); + + for (int i = 2; i < 10; i++) { + final String streamName2 = streamName() + i; + final String tableName = getFullyQualifiedTableName(TABLE_NAME + i); + testdb.with(createTableQuery(tableName, "id INTEGER, name VARCHAR(200)", "")) + .with("INSERT INTO %s(id, name) VALUES (1,'picard')", tableName) + .with("INSERT INTO %s(id, name) VALUES (2, 'crusher')", tableName) + .with("INSERT INTO %s(id, name) VALUES (3, 'vash')", tableName); + catalog.getStreams().add(CatalogHelpers.createConfiguredAirbyteStream( + streamName2, + getDefaultNamespace(), + Field.of(COL_ID, JsonSchemaType.NUMBER), + Field.of(COL_NAME, JsonSchemaType.STRING))); + + expectedMessages.addAll(getAirbyteMessagesSecondSync(streamName2)); + } + + final List actualMessages = MoreIterators + .toList(source().read(config(), catalog, null)); + + setEmittedAtToNull(actualMessages); + + assertEquals(expectedMessages.size(), actualMessages.size()); + assertTrue(expectedMessages.containsAll(actualMessages)); + assertTrue(actualMessages.containsAll(expectedMessages)); + } + + protected List getAirbyteMessagesSecondSync(final String streamName) { + return getTestMessages() + .stream() + .map(Jsons::clone) + .peek(m -> { + m.getRecord().setStream(streamName); + m.getRecord().setNamespace(getDefaultNamespace()); + ((ObjectNode) m.getRecord().getData()).remove(COL_UPDATED_AT); + ((ObjectNode) m.getRecord().getData()).replace(COL_ID, + convertIdBasedOnDatabase(m.getRecord().getData().get(COL_ID).asInt())); + }) + .collect(Collectors.toList()); + + } + + @Test + protected void testTablesWithQuoting() throws Exception { + final ConfiguredAirbyteStream streamForTableWithSpaces = createTableWithSpaces(); + + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() + .withStreams(List.of( + getConfiguredCatalogWithOneStream(getDefaultNamespace()).getStreams().get(0), + streamForTableWithSpaces)); + final List actualMessages = MoreIterators + .toList(source().read(config(), catalog, null)); + + setEmittedAtToNull(actualMessages); + + final List expectedMessages = new ArrayList<>(getTestMessages()); + expectedMessages.addAll(getAirbyteMessagesForTablesWithQuoting(streamForTableWithSpaces)); + + assertEquals(expectedMessages.size(), actualMessages.size()); + assertTrue(expectedMessages.containsAll(actualMessages)); + assertTrue(actualMessages.containsAll(expectedMessages)); + } + + protected List getAirbyteMessagesForTablesWithQuoting(final ConfiguredAirbyteStream streamForTableWithSpaces) { + return getTestMessages() + .stream() + .map(Jsons::clone) + .peek(m -> { + m.getRecord().setStream(streamForTableWithSpaces.getStream().getName()); + ((ObjectNode) m.getRecord().getData()).set(COL_LAST_NAME_WITH_SPACE, + ((ObjectNode) m.getRecord().getData()).remove(COL_NAME)); + ((ObjectNode) m.getRecord().getData()).remove(COL_UPDATED_AT); + ((ObjectNode) m.getRecord().getData()).replace(COL_ID, + convertIdBasedOnDatabase(m.getRecord().getData().get(COL_ID).asInt())); + }) + .collect(Collectors.toList()); + } + + @SuppressWarnings("ResultOfMethodCallIgnored") + @Test + void testReadFailure() { + final ConfiguredAirbyteStream spiedAbStream = spy( + getConfiguredCatalogWithOneStream(getDefaultNamespace()).getStreams().get(0)); + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() + .withStreams(List.of(spiedAbStream)); + doCallRealMethod().doThrow(new RuntimeException()).when(spiedAbStream).getStream(); + + assertThrows(RuntimeException.class, () -> source().read(config(), catalog, null)); + } + + @Test + void testIncrementalNoPreviousState() throws Exception { + incrementalCursorCheck( + COL_ID, + null, + "3", + getTestMessages()); + } + + @Test + void testIncrementalIntCheckCursor() throws Exception { + incrementalCursorCheck( + COL_ID, + "2", + "3", + List.of(getTestMessages().get(2))); + } + + @Test + void testIncrementalStringCheckCursor() throws Exception { + incrementalCursorCheck( + COL_NAME, + "patent", + "vash", + List.of(getTestMessages().get(0), getTestMessages().get(2))); + } + + @Test + void testIncrementalStringCheckCursorSpaceInColumnName() throws Exception { + final ConfiguredAirbyteStream streamWithSpaces = createTableWithSpaces(); + + final List expectedRecordMessages = getAirbyteMessagesCheckCursorSpaceInColumnName(streamWithSpaces); + incrementalCursorCheck( + COL_LAST_NAME_WITH_SPACE, + COL_LAST_NAME_WITH_SPACE, + "patent", + "vash", + expectedRecordMessages, + streamWithSpaces); + } + + protected List getAirbyteMessagesCheckCursorSpaceInColumnName(final ConfiguredAirbyteStream streamWithSpaces) { + final AirbyteMessage firstMessage = getTestMessages().get(0); + firstMessage.getRecord().setStream(streamWithSpaces.getStream().getName()); + ((ObjectNode) firstMessage.getRecord().getData()).remove(COL_UPDATED_AT); + ((ObjectNode) firstMessage.getRecord().getData()).set(COL_LAST_NAME_WITH_SPACE, + ((ObjectNode) firstMessage.getRecord().getData()).remove(COL_NAME)); + + final AirbyteMessage secondMessage = getTestMessages().get(2); + secondMessage.getRecord().setStream(streamWithSpaces.getStream().getName()); + ((ObjectNode) secondMessage.getRecord().getData()).remove(COL_UPDATED_AT); + ((ObjectNode) secondMessage.getRecord().getData()).set(COL_LAST_NAME_WITH_SPACE, + ((ObjectNode) secondMessage.getRecord().getData()).remove(COL_NAME)); + + return List.of(firstMessage, secondMessage); + } + + @Test + void testIncrementalDateCheckCursor() throws Exception { + incrementalDateCheck(); + } + + protected void incrementalDateCheck() throws Exception { + incrementalCursorCheck( + COL_UPDATED_AT, + "2005-10-18", + "2006-10-19", + List.of(getTestMessages().get(1), getTestMessages().get(2))); + } + + @Test + void testIncrementalCursorChanges() throws Exception { + incrementalCursorCheck( + COL_ID, + COL_NAME, + // cheesing this value a little bit. in the correct implementation this initial cursor value should + // be ignored because the cursor field changed. setting it to a value that if used, will cause + // records to (incorrectly) be filtered out. + "data", + "vash", + getTestMessages()); + } + + @Test + protected void testReadOneTableIncrementallyTwice() throws Exception { + final var config = config(); + final String namespace = getDefaultNamespace(); + final ConfiguredAirbyteCatalog configuredCatalog = getConfiguredCatalogWithOneStream(namespace); + configuredCatalog.getStreams().forEach(airbyteStream -> { + airbyteStream.setSyncMode(SyncMode.INCREMENTAL); + airbyteStream.setCursorField(List.of(COL_ID)); + airbyteStream.setDestinationSyncMode(DestinationSyncMode.APPEND); + }); + + final List actualMessagesFirstSync = MoreIterators + .toList(source().read(config, configuredCatalog, createEmptyState(streamName(), namespace))); + + final Optional stateAfterFirstSyncOptional = actualMessagesFirstSync.stream() + .filter(r -> r.getType() == Type.STATE).findFirst(); + assertTrue(stateAfterFirstSyncOptional.isPresent()); + + executeStatementReadIncrementallyTwice(); + + final List actualMessagesSecondSync = MoreIterators + .toList(source().read(config, configuredCatalog, extractState(stateAfterFirstSyncOptional.get()))); + + assertEquals(2, + (int) actualMessagesSecondSync.stream().filter(r -> r.getType() == Type.RECORD).count()); + final List expectedMessages = getExpectedAirbyteMessagesSecondSync(namespace); + + setEmittedAtToNull(actualMessagesSecondSync); + + assertEquals(expectedMessages.size(), actualMessagesSecondSync.size()); + assertTrue(expectedMessages.containsAll(actualMessagesSecondSync)); + assertTrue(actualMessagesSecondSync.containsAll(expectedMessages)); + } + + protected void executeStatementReadIncrementallyTwice() { + testdb + .with("INSERT INTO %s(id, name, updated_at) VALUES (4, 'riker', '2006-10-19')", getFullyQualifiedTableName(TABLE_NAME)) + .with("INSERT INTO %s(id, name, updated_at) VALUES (5, 'data', '2006-10-19')", getFullyQualifiedTableName(TABLE_NAME)); + } + + protected List getExpectedAirbyteMessagesSecondSync(final String namespace) { + final List expectedMessages = new ArrayList<>(); + expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName()).withNamespace(namespace) + .withData(Jsons.jsonNode(Map + .of(COL_ID, ID_VALUE_4, + COL_NAME, "riker", + COL_UPDATED_AT, "2006-10-19"))))); + expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName()).withNamespace(namespace) + .withData(Jsons.jsonNode(Map + .of(COL_ID, ID_VALUE_5, + COL_NAME, "data", + COL_UPDATED_AT, "2006-10-19"))))); + final DbStreamState state = new DbStreamState() + .withStreamName(streamName()) + .withStreamNamespace(namespace) + .withCursorField(List.of(COL_ID)) + .withCursor("5") + .withCursorRecordCount(1L); + expectedMessages.addAll(createExpectedTestMessages(List.of(state), 2L)); + return expectedMessages; + } + + @Test + protected void testReadMultipleTablesIncrementally() throws Exception { + final String tableName2 = TABLE_NAME + 2; + final String streamName2 = streamName() + 2; + final String fqTableName2 = getFullyQualifiedTableName(tableName2); + testdb.with(createTableQuery(fqTableName2, "id INTEGER, name VARCHAR(200)", "")) + .with("INSERT INTO %s(id, name) VALUES (1,'picard')", fqTableName2) + .with("INSERT INTO %s(id, name) VALUES (2, 'crusher')", fqTableName2) + .with("INSERT INTO %s(id, name) VALUES (3, 'vash')", fqTableName2); + + final String namespace = getDefaultNamespace(); + final ConfiguredAirbyteCatalog configuredCatalog = getConfiguredCatalogWithOneStream( + namespace); + configuredCatalog.getStreams().add(CatalogHelpers.createConfiguredAirbyteStream( + streamName2, + namespace, + Field.of(COL_ID, JsonSchemaType.NUMBER), + Field.of(COL_NAME, JsonSchemaType.STRING))); + configuredCatalog.getStreams().forEach(airbyteStream -> { + airbyteStream.setSyncMode(SyncMode.INCREMENTAL); + airbyteStream.setCursorField(List.of(COL_ID)); + airbyteStream.setDestinationSyncMode(DestinationSyncMode.APPEND); + }); + + final List actualMessagesFirstSync = MoreIterators + .toList(source().read(config(), configuredCatalog, createEmptyState(streamName(), namespace))); + + // get last state message. + final Optional stateAfterFirstSyncOptional = actualMessagesFirstSync.stream() + .filter(r -> r.getType() == Type.STATE) + .reduce((first, second) -> second); + assertTrue(stateAfterFirstSyncOptional.isPresent()); + + // we know the second streams messages are the same as the first minus the updated at column. so we + // cheat and generate the expected messages off of the first expected messages. + final List secondStreamExpectedMessages = getAirbyteMessagesSecondStreamWithNamespace(streamName2); + + // Represents the state after the first stream has been updated + final List expectedStateStreams1 = List.of( + new DbStreamState() + .withStreamName(streamName()) + .withStreamNamespace(namespace) + .withCursorField(List.of(COL_ID)) + .withCursor("3") + .withCursorRecordCount(1L), + new DbStreamState() + .withStreamName(streamName2) + .withStreamNamespace(namespace) + .withCursorField(List.of(COL_ID))); + + // Represents the state after both streams have been updated + final List expectedStateStreams2 = List.of( + new DbStreamState() + .withStreamName(streamName()) + .withStreamNamespace(namespace) + .withCursorField(List.of(COL_ID)) + .withCursor("3") + .withCursorRecordCount(1L), + new DbStreamState() + .withStreamName(streamName2) + .withStreamNamespace(namespace) + .withCursorField(List.of(COL_ID)) + .withCursor("3") + .withCursorRecordCount(1L)); + + final List expectedMessagesFirstSync = new ArrayList<>(getTestMessages()); + expectedMessagesFirstSync.add(createStateMessage(expectedStateStreams1.get(0), expectedStateStreams1, 3L)); + expectedMessagesFirstSync.addAll(secondStreamExpectedMessages); + expectedMessagesFirstSync.add(createStateMessage(expectedStateStreams2.get(1), expectedStateStreams2, 3L)); + + setEmittedAtToNull(actualMessagesFirstSync); + + assertEquals(expectedMessagesFirstSync.size(), actualMessagesFirstSync.size()); + assertTrue(expectedMessagesFirstSync.containsAll(actualMessagesFirstSync)); + assertTrue(actualMessagesFirstSync.containsAll(expectedMessagesFirstSync)); + } + + protected List getAirbyteMessagesSecondStreamWithNamespace(final String streamName2) { + return getTestMessages() + .stream() + .map(Jsons::clone) + .peek(m -> { + m.getRecord().setStream(streamName2); + ((ObjectNode) m.getRecord().getData()).remove(COL_UPDATED_AT); + ((ObjectNode) m.getRecord().getData()).replace(COL_ID, + convertIdBasedOnDatabase(m.getRecord().getData().get(COL_ID).asInt())); + }) + .collect(Collectors.toList()); + } + + // when initial and final cursor fields are the same. + protected void incrementalCursorCheck( + final String cursorField, + final String initialCursorValue, + final String endCursorValue, + final List expectedRecordMessages) + throws Exception { + incrementalCursorCheck(cursorField, cursorField, initialCursorValue, endCursorValue, + expectedRecordMessages); + } + + // See https://github.com/airbytehq/airbyte/issues/14732 for rationale and details. + @Test + public void testIncrementalWithConcurrentInsertion() throws Exception { + final String namespace = getDefaultNamespace(); + final String fullyQualifiedTableName = getFullyQualifiedTableName(TABLE_NAME_AND_TIMESTAMP); + final String columnDefinition = String.format("name VARCHAR(200) NOT NULL, %s %s NOT NULL", COL_TIMESTAMP, COL_TIMESTAMP_TYPE); + + // 1st sync + testdb.with(createTableQuery(fullyQualifiedTableName, columnDefinition, "")) + .with(INSERT_TABLE_NAME_AND_TIMESTAMP_QUERY, fullyQualifiedTableName, "a", "2021-01-01 00:00:00") + .with(INSERT_TABLE_NAME_AND_TIMESTAMP_QUERY, fullyQualifiedTableName, "b", "2021-01-01 00:00:00"); + + final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog( + new AirbyteCatalog().withStreams(List.of( + CatalogHelpers.createAirbyteStream( + TABLE_NAME_AND_TIMESTAMP, + namespace, + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_TIMESTAMP, JsonSchemaType.STRING_TIMESTAMP_WITHOUT_TIMEZONE))))); + + configuredCatalog.getStreams().forEach(airbyteStream -> { + airbyteStream.setSyncMode(SyncMode.INCREMENTAL); + airbyteStream.setCursorField(List.of(COL_TIMESTAMP)); + airbyteStream.setDestinationSyncMode(DestinationSyncMode.APPEND); + }); + + final List firstSyncActualMessages = MoreIterators.toList( + source().read(config(), configuredCatalog, createEmptyState(TABLE_NAME_AND_TIMESTAMP, namespace))); + + // cursor after 1st sync: 2021-01-01 00:00:00, count 2 + final Optional firstSyncStateOptional = firstSyncActualMessages.stream().filter(r -> r.getType() == Type.STATE).findFirst(); + assertTrue(firstSyncStateOptional.isPresent()); + final JsonNode firstSyncState = getStateData(firstSyncStateOptional.get(), TABLE_NAME_AND_TIMESTAMP); + assertEquals(firstSyncState.get("cursor_field").elements().next().asText(), COL_TIMESTAMP); + assertTrue(firstSyncState.get("cursor").asText().contains("2021-01-01")); + assertTrue(firstSyncState.get("cursor").asText().contains("00:00:00")); + assertEquals(2L, firstSyncState.get("cursor_record_count").asLong()); + + final List firstSyncNames = firstSyncActualMessages.stream() + .filter(r -> r.getType() == Type.RECORD) + .map(r -> r.getRecord().getData().get(COL_NAME).asText()) + .toList(); + // some databases don't make insertion order guarantee when equal ordering value + if (testdb.getDatabaseDriver().equals(DatabaseDriver.TERADATA) || testdb.getDatabaseDriver().equals(DatabaseDriver.ORACLE)) { + assertThat(List.of("a", "b"), Matchers.containsInAnyOrder(firstSyncNames.toArray())); + } else { + assertEquals(List.of("a", "b"), firstSyncNames); + } + + // 2nd sync + testdb.with(INSERT_TABLE_NAME_AND_TIMESTAMP_QUERY, fullyQualifiedTableName, "c", "2021-01-02 00:00:00"); + + final List secondSyncActualMessages = MoreIterators.toList( + source().read(config(), configuredCatalog, createState(TABLE_NAME_AND_TIMESTAMP, namespace, firstSyncState))); + + // cursor after 2nd sync: 2021-01-02 00:00:00, count 1 + final Optional secondSyncStateOptional = secondSyncActualMessages.stream().filter(r -> r.getType() == Type.STATE).findFirst(); + assertTrue(secondSyncStateOptional.isPresent()); + final JsonNode secondSyncState = getStateData(secondSyncStateOptional.get(), TABLE_NAME_AND_TIMESTAMP); + assertEquals(secondSyncState.get("cursor_field").elements().next().asText(), COL_TIMESTAMP); + assertTrue(secondSyncState.get("cursor").asText().contains("2021-01-02")); + assertTrue(secondSyncState.get("cursor").asText().contains("00:00:00")); + assertEquals(1L, secondSyncState.get("cursor_record_count").asLong()); + + final List secondSyncNames = secondSyncActualMessages.stream() + .filter(r -> r.getType() == Type.RECORD) + .map(r -> r.getRecord().getData().get(COL_NAME).asText()) + .toList(); + assertEquals(List.of("c"), secondSyncNames); + + // 3rd sync has records with duplicated cursors + testdb.with(INSERT_TABLE_NAME_AND_TIMESTAMP_QUERY, fullyQualifiedTableName, "d", "2021-01-02 00:00:00") + .with(INSERT_TABLE_NAME_AND_TIMESTAMP_QUERY, fullyQualifiedTableName, "e", "2021-01-02 00:00:00") + .with(INSERT_TABLE_NAME_AND_TIMESTAMP_QUERY, fullyQualifiedTableName, "f", "2021-01-03 00:00:00"); + + final List thirdSyncActualMessages = MoreIterators.toList( + source().read(config(), configuredCatalog, createState(TABLE_NAME_AND_TIMESTAMP, namespace, secondSyncState))); + + // Cursor after 3rd sync is: 2021-01-03 00:00:00, count 1. + final Optional thirdSyncStateOptional = thirdSyncActualMessages.stream().filter(r -> r.getType() == Type.STATE).findFirst(); + assertTrue(thirdSyncStateOptional.isPresent()); + final JsonNode thirdSyncState = getStateData(thirdSyncStateOptional.get(), TABLE_NAME_AND_TIMESTAMP); + assertEquals(thirdSyncState.get("cursor_field").elements().next().asText(), COL_TIMESTAMP); + assertTrue(thirdSyncState.get("cursor").asText().contains("2021-01-03")); + assertTrue(thirdSyncState.get("cursor").asText().contains("00:00:00")); + assertEquals(1L, thirdSyncState.get("cursor_record_count").asLong()); + + // The c, d, e, f are duplicated records from this sync, because the cursor + // record count in the database is different from that in the state. + final List thirdSyncExpectedNames = thirdSyncActualMessages.stream() + .filter(r -> r.getType() == Type.RECORD) + .map(r -> r.getRecord().getData().get(COL_NAME).asText()) + .toList(); + + // teradata doesn't make insertion order guarantee when equal ordering value + if (testdb.getDatabaseDriver().equals(DatabaseDriver.TERADATA)) { + assertThat(List.of("c", "d", "e", "f"), Matchers.containsInAnyOrder(thirdSyncExpectedNames.toArray())); + } else { + assertEquals(List.of("c", "d", "e", "f"), thirdSyncExpectedNames); + } + } + + protected JsonNode getStateData(final AirbyteMessage airbyteMessage, final String streamName) { + for (final JsonNode stream : airbyteMessage.getState().getData().get("streams")) { + if (stream.get("stream_name").asText().equals(streamName)) { + return stream; + } + } + throw new IllegalArgumentException("Stream not found in state message: " + streamName); + } + + private void incrementalCursorCheck( + final String initialCursorField, + final String cursorField, + final String initialCursorValue, + final String endCursorValue, + final List expectedRecordMessages) + throws Exception { + incrementalCursorCheck(initialCursorField, cursorField, initialCursorValue, endCursorValue, + expectedRecordMessages, + getConfiguredCatalogWithOneStream(getDefaultNamespace()).getStreams().get(0)); + } + + protected void incrementalCursorCheck( + final String initialCursorField, + final String cursorField, + final String initialCursorValue, + final String endCursorValue, + final List expectedRecordMessages, + final ConfiguredAirbyteStream airbyteStream) + throws Exception { + airbyteStream.setSyncMode(SyncMode.INCREMENTAL); + airbyteStream.setCursorField(List.of(cursorField)); + airbyteStream.setDestinationSyncMode(DestinationSyncMode.APPEND); + + final ConfiguredAirbyteCatalog configuredCatalog = new ConfiguredAirbyteCatalog() + .withStreams(List.of(airbyteStream)); + + final DbStreamState dbStreamState = buildStreamState(airbyteStream, initialCursorField, initialCursorValue); + + final List actualMessages = MoreIterators + .toList(source().read(config(), configuredCatalog, Jsons.jsonNode(createState(List.of(dbStreamState))))); + + setEmittedAtToNull(actualMessages); + + final List expectedStreams = List.of(buildStreamState(airbyteStream, cursorField, endCursorValue)); + + final List expectedMessages = new ArrayList<>(expectedRecordMessages); + expectedMessages.addAll(createExpectedTestMessages(expectedStreams, expectedRecordMessages.size())); + + assertEquals(expectedMessages.size(), actualMessages.size()); + assertTrue(expectedMessages.containsAll(actualMessages)); + assertTrue(actualMessages.containsAll(expectedMessages)); + } + + protected DbStreamState buildStreamState(final ConfiguredAirbyteStream configuredAirbyteStream, + final String cursorField, + final String cursorValue) { + return new DbStreamState() + .withStreamName(configuredAirbyteStream.getStream().getName()) + .withStreamNamespace(configuredAirbyteStream.getStream().getNamespace()) + .withCursorField(List.of(cursorField)) + .withCursor(cursorValue) + .withCursorRecordCount(1L); + } + + // get catalog and perform a defensive copy. + protected ConfiguredAirbyteCatalog getConfiguredCatalogWithOneStream(final String defaultNamespace) { + final ConfiguredAirbyteCatalog catalog = CatalogHelpers.toDefaultConfiguredCatalog(getCatalog(defaultNamespace)); + // Filter to only keep the main stream name as configured stream + catalog.withStreams( + catalog.getStreams().stream().filter(s -> s.getStream().getName().equals(streamName())) + .collect(Collectors.toList())); + return catalog; + } + + protected AirbyteCatalog getCatalog(final String defaultNamespace) { + return new AirbyteCatalog().withStreams(List.of( + CatalogHelpers.createAirbyteStream( + TABLE_NAME, + defaultNamespace, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING)) + .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID))), + CatalogHelpers.createAirbyteStream( + TABLE_NAME_WITHOUT_PK, + defaultNamespace, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING)) + .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(Collections.emptyList()), + CatalogHelpers.createAirbyteStream( + TABLE_NAME_COMPOSITE_PK, + defaultNamespace, + Field.of(COL_FIRST_NAME, JsonSchemaType.STRING), + Field.of(COL_LAST_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING)) + .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey( + List.of(List.of(COL_FIRST_NAME), List.of(COL_LAST_NAME))))); + } + + protected List getTestMessages() { + return List.of( + new AirbyteMessage().withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName()).withNamespace(getDefaultNamespace()) + .withData(Jsons.jsonNode(Map + .of(COL_ID, ID_VALUE_1, + COL_NAME, "picard", + COL_UPDATED_AT, "2004-10-19")))), + new AirbyteMessage().withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName()).withNamespace(getDefaultNamespace()) + .withData(Jsons.jsonNode(Map + .of(COL_ID, ID_VALUE_2, + COL_NAME, "crusher", + COL_UPDATED_AT, + "2005-10-19")))), + new AirbyteMessage().withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName()).withNamespace(getDefaultNamespace()) + .withData(Jsons.jsonNode(Map + .of(COL_ID, ID_VALUE_3, + COL_NAME, "vash", + COL_UPDATED_AT, "2006-10-19"))))); + } + + protected List createExpectedTestMessages(final List states, final long numRecords) { + return states.stream() + .map(s -> new AirbyteMessage().withType(Type.STATE) + .withState( + new AirbyteStateMessage().withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) + .withStreamState(Jsons.jsonNode(s))) + .withData(Jsons.jsonNode(new DbState().withCdc(false).withStreams(states))) + .withSourceStats(new AirbyteStateStats().withRecordCount((double) numRecords)))) + .collect( + Collectors.toList()); + } + + protected List createState(final List states) { + return states.stream() + .map(s -> new AirbyteStateMessage().withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) + .withStreamState(Jsons.jsonNode(s)))) + .collect( + Collectors.toList()); + } + + protected ConfiguredAirbyteStream createTableWithSpaces() throws SQLException { + final String tableNameWithSpaces = TABLE_NAME_WITH_SPACES + "2"; + final String streamName2 = tableNameWithSpaces; + + try (final var connection = testdb.getDataSource().getConnection()) { + final String identifierQuoteString = connection.getMetaData().getIdentifierQuoteString(); + connection.createStatement() + .execute( + createTableQuery(getFullyQualifiedTableName( + enquoteIdentifier(tableNameWithSpaces, identifierQuoteString)), + "id INTEGER, " + enquoteIdentifier(COL_LAST_NAME_WITH_SPACE, identifierQuoteString) + + " VARCHAR(200)", + "")); + connection.createStatement() + .execute(String.format("INSERT INTO %s(id, %s) VALUES (1,'picard')", + getFullyQualifiedTableName( + enquoteIdentifier(tableNameWithSpaces, identifierQuoteString)), + enquoteIdentifier(COL_LAST_NAME_WITH_SPACE, identifierQuoteString))); + connection.createStatement() + .execute(String.format("INSERT INTO %s(id, %s) VALUES (2, 'crusher')", + getFullyQualifiedTableName( + enquoteIdentifier(tableNameWithSpaces, identifierQuoteString)), + enquoteIdentifier(COL_LAST_NAME_WITH_SPACE, identifierQuoteString))); + connection.createStatement() + .execute(String.format("INSERT INTO %s(id, %s) VALUES (3, 'vash')", + getFullyQualifiedTableName( + enquoteIdentifier(tableNameWithSpaces, identifierQuoteString)), + enquoteIdentifier(COL_LAST_NAME_WITH_SPACE, identifierQuoteString))); + } + + return CatalogHelpers.createConfiguredAirbyteStream( + streamName2, + getDefaultNamespace(), + Field.of(COL_ID, JsonSchemaType.NUMBER), + Field.of(COL_LAST_NAME_WITH_SPACE, JsonSchemaType.STRING)); + } + + public String getFullyQualifiedTableName(final String tableName) { + return RelationalDbQueryUtils.getFullyQualifiedTableName(getDefaultSchemaName(), tableName); + } + + protected void createSchemas() { + if (supportsSchemas()) { + for (final String schemaName : TEST_SCHEMAS) { + testdb.with("CREATE SCHEMA %s;", schemaName); + } + } + } + + private JsonNode convertIdBasedOnDatabase(final int idValue) { + return switch (testdb.getDatabaseDriver()) { + case ORACLE, SNOWFLAKE -> Jsons.jsonNode(BigDecimal.valueOf(idValue)); + default -> Jsons.jsonNode(idValue); + }; + } + + private String getDefaultSchemaName() { + return supportsSchemas() ? SCHEMA_NAME : null; + } + + protected String getDefaultNamespace() { + return switch (testdb.getDatabaseDriver()) { + // mysql does not support schemas, it namespaces using database names instead. + case MYSQL, CLICKHOUSE, TERADATA -> testdb.getDatabaseName(); + default -> SCHEMA_NAME; + }; + } + + protected static void setEmittedAtToNull(final Iterable messages) { + for (final AirbyteMessage actualMessage : messages) { + if (actualMessage.getRecord() != null) { + actualMessage.getRecord().setEmittedAt(null); + } + } + } + + /** + * Creates empty state with the provided stream name and namespace. + * + * @param streamName The stream name. + * @param streamNamespace The stream namespace. + * @return {@link JsonNode} representation of the generated empty state. + */ + protected JsonNode createEmptyState(final String streamName, final String streamNamespace) { + final AirbyteStateMessage airbyteStateMessage = new AirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState().withStreamDescriptor(new StreamDescriptor().withName(streamName).withNamespace(streamNamespace))); + return Jsons.jsonNode(List.of(airbyteStateMessage)); + + } + + protected JsonNode createState(final String streamName, final String streamNamespace, final JsonNode stateData) { + final AirbyteStateMessage airbyteStateMessage = new AirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream( + new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withName(streamName).withNamespace(streamNamespace)) + .withStreamState(stateData)); + return Jsons.jsonNode(List.of(airbyteStateMessage)); + } + + protected JsonNode extractState(final AirbyteMessage airbyteMessage) { + return Jsons.jsonNode(List.of(airbyteMessage.getState())); + } + + protected AirbyteMessage createStateMessage(final DbStreamState dbStreamState, final List legacyStates, final long recordCount) { + return new AirbyteMessage().withType(Type.STATE) + .withState( + new AirbyteStateMessage().withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withNamespace(dbStreamState.getStreamNamespace()) + .withName(dbStreamState.getStreamName())) + .withStreamState(Jsons.jsonNode(dbStreamState))) + .withData(Jsons.jsonNode(new DbState().withCdc(false).withStreams(legacyStates))) + .withSourceStats(new AirbyteStateStats().withRecordCount((double) recordCount))); + } + + protected List extractSpecificFieldFromCombinedMessages(final List messages, + final String streamName, + final String field) { + return extractStateMessage(messages).stream() + .filter(s -> s.getStream().getStreamDescriptor().getName().equals(streamName)) + .map(s -> s.getStream().getStreamState().get(field) != null ? s.getStream().getStreamState().get(field).asText() : "").toList(); + } + + protected List filterRecords(final List messages) { + return messages.stream().filter(r -> r.getType() == Type.RECORD) + .collect(Collectors.toList()); + } + + protected List extractStateMessage(final List messages) { + return messages.stream().filter(r -> r.getType() == Type.STATE).map(AirbyteMessage::getState) + .collect(Collectors.toList()); + } + + protected List extractStateMessage(final List messages, final String streamName) { + return messages.stream().filter(r -> r.getType() == Type.STATE && + r.getState().getStream().getStreamDescriptor().getName().equals(streamName)).map(AirbyteMessage::getState) + .collect(Collectors.toList()); + } + + protected AirbyteMessage createRecord(final String stream, final String namespace, final Map data) { + return new AirbyteMessage().withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage().withData(Jsons.jsonNode(data)).withStream(stream).withNamespace(namespace)); + } + +} diff --git a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcStressTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/source/jdbc/test/JdbcStressTest.java similarity index 96% rename from airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcStressTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/source/jdbc/test/JdbcStressTest.java index 2bec57534e56..9c626a9ac911 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcStressTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/source/jdbc/test/JdbcStressTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.jdbc.test; +package io.airbyte.cdk.integrations.source.jdbc.test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -10,14 +10,14 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.stream.MoreStreams; import io.airbyte.commons.string.Strings; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceConnectorTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/AbstractSourceConnectorTest.java similarity index 95% rename from airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceConnectorTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/AbstractSourceConnectorTest.java index cd065642322b..2393c4dcc595 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceConnectorTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/AbstractSourceConnectorTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.source; +package io.airbyte.cdk.integrations.standardtest.source; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -17,6 +17,7 @@ import io.airbyte.api.client.model.generated.DiscoverCatalogResult; import io.airbyte.api.client.model.generated.SourceDiscoverSchemaWriteRequestBody; import io.airbyte.commons.features.EnvVariableFeatureFlags; +import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.json.Jsons; import io.airbyte.configoss.JobGetSpecConfig; import io.airbyte.configoss.StandardCheckConnectionInput; @@ -45,6 +46,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -142,12 +144,14 @@ public void setUpInternal() throws Exception { when(mSourceApi.writeDiscoverCatalogResult(any())) .thenReturn(new DiscoverCatalogResult().catalogId(CATALOG_ID)); mConnectorConfigUpdater = mock(ConnectorConfigUpdater.class); + var envMap = new HashMap<>(new TestEnvConfigs().getJobDefaultEnvMap()); + envMap.put(EnvVariableFeatureFlags.DEPLOYMENT_MODE, featureFlags().deploymentMode()); processFactory = new DockerProcessFactory( workspaceRoot, workspaceRoot.toString(), localRoot.toString(), "host", - new TestEnvConfigs().getJobDefaultEnvMap()); + envMap); postSetup(); } @@ -163,10 +167,14 @@ public void tearDownInternal() throws Exception { tearDown(environment); } + protected FeatureFlags featureFlags() { + return new EnvVariableFeatureFlags(); + } + protected ConnectorSpecification runSpec() throws TestHarnessException { final io.airbyte.protocol.models.ConnectorSpecification spec = new DefaultGetSpecTestHarness( new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory, null, null, false, - new EnvVariableFeatureFlags())) + featureFlags())) .run(new JobGetSpecConfig().withDockerImage(getImageName()), jobRoot).getSpec(); return convertProtocolObject(spec, ConnectorSpecification.class); } @@ -174,7 +182,7 @@ protected ConnectorSpecification runSpec() throws TestHarnessException { protected StandardCheckConnectionOutput runCheck() throws Exception { return new DefaultCheckConnectionTestHarness( new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory, null, null, false, - new EnvVariableFeatureFlags()), + featureFlags()), mConnectorConfigUpdater) .run(new StandardCheckConnectionInput().withConnectionConfiguration(getConfig()), jobRoot).getCheckConnection(); } @@ -182,7 +190,7 @@ protected StandardCheckConnectionOutput runCheck() throws Exception { protected String runCheckAndGetStatusAsString(final JsonNode config) throws Exception { return new DefaultCheckConnectionTestHarness( new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory, null, null, false, - new EnvVariableFeatureFlags()), + featureFlags()), mConnectorConfigUpdater) .run(new StandardCheckConnectionInput().withConnectionConfiguration(config), jobRoot).getCheckConnection().getStatus().toString(); } @@ -191,7 +199,7 @@ protected UUID runDiscover() throws Exception { final UUID toReturn = new DefaultDiscoverCatalogTestHarness( mAirbyteApiClient, new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory, null, null, false, - new EnvVariableFeatureFlags()), + featureFlags()), mConnectorConfigUpdater) .run(new StandardDiscoverCatalogInput().withSourceId(SOURCE_ID.toString()).withConnectionConfiguration(getConfig()), jobRoot) .getDiscoverCatalogId(); @@ -222,12 +230,10 @@ protected List runRead(final ConfiguredAirbyteCatalog catalog, f .withState(state == null ? null : new State().withState(state)) .withCatalog(convertProtocolObject(catalog, io.airbyte.protocol.models.ConfiguredAirbyteCatalog.class)); - final var featureFlags = new EnvVariableFeatureFlags(); - final AirbyteSource source = new DefaultAirbyteSource( new AirbyteIntegrationLauncher(JOB_ID, JOB_ATTEMPT, getImageName(), processFactory, null, null, false, - featureFlags), - featureFlags); + featureFlags()), + featureFlags()); final List messages = new ArrayList<>(); source.start(sourceConfig, jobRoot); while (!source.isFinished()) { @@ -266,7 +272,6 @@ protected Map runReadVerifyNumberOfReceivedMsgs(final Configure } private AirbyteSource prepareAirbyteSource() { - final var featureFlags = new EnvVariableFeatureFlags(); final var integrationLauncher = new AirbyteIntegrationLauncher( JOB_ID, JOB_ATTEMPT, @@ -275,8 +280,8 @@ private AirbyteSource prepareAirbyteSource() { null, null, false, - featureFlags); - return new DefaultAirbyteSource(integrationLauncher, featureFlags); + featureFlags()); + return new DefaultAirbyteSource(integrationLauncher, featureFlags()); } private static V0 convertProtocolObject(final V1 v1, final Class klass) { diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceDatabaseTypeTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/AbstractSourceDatabaseTypeTest.java similarity index 87% rename from airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceDatabaseTypeTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/AbstractSourceDatabaseTypeTest.java index 89e3b0d8a555..c8a274208662 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceDatabaseTypeTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/AbstractSourceDatabaseTypeTest.java @@ -2,15 +2,15 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.source; +package io.airbyte.cdk.integrations.standardtest.source; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.Database; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteMessage; @@ -130,33 +130,43 @@ class MissedRecords { // Stream that is missing any value public String streamName; - // Type associated to the test - public String dataType; // Which are the values that has not being gathered from the source public List missedValues; - public MissedRecords(String streamName, String dataType, List missedValues) { + public MissedRecords(String streamName, List missedValues) { this.streamName = streamName; - this.dataType = dataType; this.missedValues = missedValues; } } + class UnexpectedRecord { + + public final String streamName; + public final String unexpectedValue; + + public UnexpectedRecord(String streamName, String unexpectedValue) { + this.streamName = streamName; + this.unexpectedValue = unexpectedValue; + } + + } + final ConfiguredAirbyteCatalog catalog = getConfiguredCatalog(); final List allMessages = runRead(catalog); final List recordMessages = allMessages.stream().filter(m -> m.getType() == Type.RECORD).toList(); final Map> expectedValues = new HashMap<>(); - final Map testTypes = new HashMap<>(); final ArrayList missedValues = new ArrayList<>(); + final List unexpectedValues = new ArrayList<>(); + final Map testByName = new HashMap<>(); // If there is no expected value in the test set we don't include it in the list to be asserted // (even if the table contains records) testDataHolders.forEach(testDataHolder -> { if (!testDataHolder.getExpectedValues().isEmpty()) { expectedValues.put(testDataHolder.getNameWithTestPrefix(), testDataHolder.getExpectedValues()); - testTypes.put(testDataHolder.getNameWithTestPrefix(), testDataHolder.getSourceType()); + testByName.put(testDataHolder.getNameWithTestPrefix(), testDataHolder); } else { LOGGER.warn("Missing expected values for type: " + testDataHolder.getSourceType()); } @@ -167,23 +177,31 @@ public MissedRecords(String streamName, String dataType, List missedValu final List expectedValuesForStream = expectedValues.get(streamName); if (expectedValuesForStream != null) { final String value = getValueFromJsonNode(message.getRecord().getData().get(getTestColumnName())); - assertTrue(expectedValuesForStream.contains(value), - String.format("Returned value '%s' from stream %s is not in the expected list: %s", - value, streamName, expectedValuesForStream)); - expectedValuesForStream.remove(value); + if (!expectedValuesForStream.contains(value)) { + unexpectedValues.add(new UnexpectedRecord(streamName, value)); + } else { + expectedValuesForStream.remove(value); + } } } + assertTrue(unexpectedValues.isEmpty(), + unexpectedValues.stream().map((entry) -> // stream each entry, map it to string value + "The stream '" + entry.streamName + "' checking type '" + testByName.get(entry.streamName).getSourceType() + "' initialized at " + + testByName.get(entry.streamName).getDeclarationLocation() + " got unexpected values: " + entry.unexpectedValue) + .collect(Collectors.joining("\n"))); // and join them + // Gather all the missing values, so we don't stop the test in the first missed one expectedValues.forEach((streamName, values) -> { if (!values.isEmpty()) { - missedValues.add(new MissedRecords(streamName, testTypes.get(streamName), values)); + missedValues.add(new MissedRecords(streamName, values)); } }); assertTrue(missedValues.isEmpty(), missedValues.stream().map((entry) -> // stream each entry, map it to string value - "The stream '" + entry.streamName + "' checking type '" + entry.dataType + "' is missing values: " + entry.missedValues) + "The stream '" + entry.streamName + "' checking type '" + testByName.get(entry.streamName).getSourceType() + "' initialized at " + + testByName.get(entry.streamName).getDeclarationLocation() + " is missing values: " + entry.missedValues) .collect(Collectors.joining("\n"))); // and join them } @@ -259,6 +277,7 @@ public void addDataTypeTestData(final TestDataHolder test) { test.setNameSpace(getNameSpace()); test.setIdColumnName(getIdColumnName()); test.setTestColumnName(getTestColumnName()); + test.setDeclarationLocation(Thread.currentThread().getStackTrace()); } private String formatCollection(final Collection collection) { diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/PythonSourceAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/PythonSourceAcceptanceTest.java similarity index 98% rename from airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/PythonSourceAcceptanceTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/PythonSourceAcceptanceTest.java index 7ac0c4bf6524..f5caa2ad9978 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/PythonSourceAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/PythonSourceAcceptanceTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.source; +package io.airbyte.cdk.integrations.standardtest.source; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/SourceAcceptanceTest.java similarity index 96% rename from airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/SourceAcceptanceTest.java index 1ff01947da71..9e77e0037d35 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/SourceAcceptanceTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.source; +package io.airbyte.cdk.integrations.standardtest.source; import static io.airbyte.protocol.models.v0.SyncMode.FULL_REFRESH; import static io.airbyte.protocol.models.v0.SyncMode.INCREMENTAL; @@ -30,6 +30,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -111,18 +112,6 @@ public abstract class SourceAcceptanceTest extends AbstractSourceConnectorTest { */ protected abstract JsonNode getState() throws Exception; - /** - * Tests whether the connector under test supports the per-stream state format or should use the - * legacy format for data generated by this test. - * - * @return {@code true} if the connector supports the per-stream state format or {@code false} if it - * does not support the per-stream state format (e.g. legacy format supported). Default - * value is {@code false}. - */ - protected boolean supportsPerStream() { - return false; - } - /** * Verify that a spec operation issued to the connector returns a valid spec. */ @@ -264,7 +253,20 @@ public void testIncrementalSyncWithState() throws Exception { // when we run incremental sync again there should be no new records. Run a sync with the latest // state message and assert no records were emitted. - final JsonNode latestState = Jsons.jsonNode(supportsPerStream() ? stateMessages : List.of(Iterables.getLast(stateMessages))); + JsonNode latestState = null; + for (final AirbyteStateMessage stateMessage : stateMessages) { + if (stateMessage.getType().equals(AirbyteStateMessage.AirbyteStateType.STREAM)) { + latestState = Jsons.jsonNode(stateMessages); + break; + } else if (stateMessage.getType().equals(AirbyteStateMessage.AirbyteStateType.GLOBAL)) { + latestState = Jsons.jsonNode(List.of(Iterables.getLast(stateMessages))); + break; + } else { + throw new RuntimeException("Unknown state type " + stateMessage.getType()); + } + } + + assert Objects.nonNull(latestState); final List secondSyncRecords = filterRecords(runRead(configuredCatalog, latestState)); assertTrue( secondSyncRecords.isEmpty(), diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDataHolder.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/TestDataHolder.java similarity index 95% rename from airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDataHolder.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/TestDataHolder.java index f1db64fa17fd..a58c2150458c 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDataHolder.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/TestDataHolder.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.source; +package io.airbyte.cdk.integrations.standardtest.source; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; @@ -27,6 +27,8 @@ public class TestDataHolder { private String idColumnName; private String testColumnName; + private StackTraceElement[] declarationLocation; + TestDataHolder(final String sourceType, final JsonSchemaType airbyteType, final List values, @@ -218,6 +220,14 @@ public String getCreateSqlQuery() { fullSourceDataType); } + void setDeclarationLocation(StackTraceElement[] declarationLocation) { + this.declarationLocation = declarationLocation; + } + + public String getDeclarationLocation() { + return Arrays.asList(declarationLocation).subList(2, 3).toString(); + } + public List getInsertSqlQueries() { final List insertSqls = new ArrayList<>(); int rowId = 1; diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDestinationEnv.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/TestDestinationEnv.java similarity index 84% rename from airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDestinationEnv.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/TestDestinationEnv.java index 4717e28def42..451cb4864b8c 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestDestinationEnv.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/TestDestinationEnv.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.source; +package io.airbyte.cdk.integrations.standardtest.source; import java.nio.file.Path; diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestEnvConfigs.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/TestEnvConfigs.java similarity index 98% rename from airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestEnvConfigs.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/TestEnvConfigs.java index 0d223edc29d6..88992d8da6c4 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestEnvConfigs.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/TestEnvConfigs.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.source; +package io.airbyte.cdk.integrations.standardtest.source; import com.google.common.base.Preconditions; import io.airbyte.commons.lang.Exceptions; diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestPythonSourceMain.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/TestPythonSourceMain.java similarity index 96% rename from airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestPythonSourceMain.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/TestPythonSourceMain.java index 1b667c12480a..f00f0f2a7e19 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestPythonSourceMain.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/TestPythonSourceMain.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.source; +package io.airbyte.cdk.integrations.standardtest.source; import net.sourceforge.argparse4j.ArgumentParsers; import net.sourceforge.argparse4j.inf.ArgumentParser; diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestRunner.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/TestRunner.java similarity index 96% rename from airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestRunner.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/TestRunner.java index b61e8408f481..1f27307421fc 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/TestRunner.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/TestRunner.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.source; +package io.airbyte.cdk.integrations.standardtest.source; import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; diff --git a/airbyte-integrations/bases/base-standard-source-test-file/src/main/java/io/airbyte/integrations/standardtest/source/fs/ExecutableTestSource.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/fs/ExecutableTestSource.java similarity index 92% rename from airbyte-integrations/bases/base-standard-source-test-file/src/main/java/io/airbyte/integrations/standardtest/source/fs/ExecutableTestSource.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/fs/ExecutableTestSource.java index 1fd4e665586e..9df6e564d945 100644 --- a/airbyte-integrations/bases/base-standard-source-test-file/src/main/java/io/airbyte/integrations/standardtest/source/fs/ExecutableTestSource.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/fs/ExecutableTestSource.java @@ -2,13 +2,13 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.source.fs; +package io.airbyte.cdk.integrations.standardtest.source.fs; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConnectorSpecification; import java.nio.file.Path; diff --git a/airbyte-integrations/bases/base-standard-source-test-file/src/main/java/io/airbyte/integrations/standardtest/source/fs/TestSourceMain.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/fs/TestSourceMain.java similarity index 94% rename from airbyte-integrations/bases/base-standard-source-test-file/src/main/java/io/airbyte/integrations/standardtest/source/fs/TestSourceMain.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/fs/TestSourceMain.java index 23a74f16b2c3..7eb5958b424e 100644 --- a/airbyte-integrations/bases/base-standard-source-test-file/src/main/java/io/airbyte/integrations/standardtest/source/fs/TestSourceMain.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/fs/TestSourceMain.java @@ -2,9 +2,9 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.source.fs; +package io.airbyte.cdk.integrations.standardtest.source.fs; -import io.airbyte.integrations.standardtest.source.TestRunner; +import io.airbyte.cdk.integrations.standardtest.source.TestRunner; import java.nio.file.Path; import net.sourceforge.argparse4j.ArgumentParsers; import net.sourceforge.argparse4j.inf.ArgumentParser; diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/performancetest/AbstractSourceBasePerformanceTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/performancetest/AbstractSourceBasePerformanceTest.java similarity index 82% rename from airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/performancetest/AbstractSourceBasePerformanceTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/performancetest/AbstractSourceBasePerformanceTest.java index 3fceb152937d..c8a4ddaa52f9 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/performancetest/AbstractSourceBasePerformanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/performancetest/AbstractSourceBasePerformanceTest.java @@ -2,10 +2,10 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.source.performancetest; +package io.airbyte.cdk.integrations.standardtest.source.performancetest; -import io.airbyte.integrations.standardtest.source.AbstractSourceConnectorTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.cdk.integrations.standardtest.source.AbstractSourceConnectorTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; /** * This abstract class contains common methods for both steams - Fill Db scripts and Performance diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/performancetest/AbstractSourceFillDbWithTestData.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/performancetest/AbstractSourceFillDbWithTestData.java similarity index 98% rename from airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/performancetest/AbstractSourceFillDbWithTestData.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/performancetest/AbstractSourceFillDbWithTestData.java index 4fc671f4d555..b8066a7aae8e 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/performancetest/AbstractSourceFillDbWithTestData.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/performancetest/AbstractSourceFillDbWithTestData.java @@ -2,9 +2,9 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.source.performancetest; +package io.airbyte.cdk.integrations.standardtest.source.performancetest; -import io.airbyte.db.Database; +import io.airbyte.cdk.db.Database; import java.util.StringJoiner; import java.util.stream.Stream; import org.junit.jupiter.api.Disabled; diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/performancetest/AbstractSourcePerformanceTest.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/performancetest/AbstractSourcePerformanceTest.java similarity index 97% rename from airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/performancetest/AbstractSourcePerformanceTest.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/performancetest/AbstractSourcePerformanceTest.java index 56c995e41506..c4279364c5ad 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/performancetest/AbstractSourcePerformanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/integrations/standardtest/source/performancetest/AbstractSourcePerformanceTest.java @@ -2,13 +2,13 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.standardtest.source.performancetest; +package io.airbyte.cdk.integrations.standardtest.source.performancetest; import static org.junit.jupiter.api.Assertions.fail; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteStream; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/ContainerFactory.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/ContainerFactory.java new file mode 100644 index 000000000000..4735716dc05e --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/ContainerFactory.java @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.testutils; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.JdbcDatabaseContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.utility.DockerImageName; + +/** + * ContainerFactory is the companion interface to {@link TestDatabase} for providing it with + * suitable testcontainer instances. + */ +public interface ContainerFactory> { + + /** + * Creates a new, unshared testcontainer instance. This usually wraps the default constructor for + * the testcontainer type. + */ + C createNewContainer(DockerImageName imageName); + + /** + * Returns the class object of the testcontainer. + */ + Class getContainerClass(); + + /** + * Returns a shared instance of the testcontainer. + */ + default C shared(String imageName, String... methods) { + final String mapKey = Stream.concat( + Stream.of(imageName, this.getClass().getCanonicalName()), + Stream.of(methods)) + .collect(Collectors.joining("+")); + return Singleton.getOrCreate(mapKey, this); + } + + /** + * This class is exclusively used by {@link #shared(String, String...)}. It wraps a specific shared + * testcontainer instance, which is created exactly once. + */ + class Singleton> { + + static private final Logger LOGGER = LoggerFactory.getLogger(Singleton.class); + static private final ConcurrentHashMap> LAZY = new ConcurrentHashMap<>(); + + @SuppressWarnings("unchecked") + static private > C getOrCreate(String mapKey, ContainerFactory factory) { + final Singleton singleton = LAZY.computeIfAbsent(mapKey, Singleton::new); + return ((Singleton) singleton).getOrCreate(factory); + } + + final private String imageName; + final private List methodNames; + + private C sharedContainer; + private RuntimeException containerCreationError; + + private Singleton(String imageNamePlusMethods) { + final String[] parts = imageNamePlusMethods.split("\\+"); + this.imageName = parts[0]; + this.methodNames = Arrays.stream(parts).skip(2).toList(); + } + + private synchronized C getOrCreate(ContainerFactory factory) { + if (sharedContainer == null && containerCreationError == null) { + try { + create(imageName, factory, methodNames); + } catch (RuntimeException e) { + sharedContainer = null; + containerCreationError = e; + } + } + if (containerCreationError != null) { + throw new RuntimeException( + "Error during container creation for imageName=" + imageName + + ", factory=" + factory.getClass().getName() + + ", methods=" + methodNames, + containerCreationError); + } + return sharedContainer; + } + + private void create(String imageName, ContainerFactory factory, List methodNames) { + LOGGER.info("Creating new shared container based on {} with {}.", imageName, methodNames); + try { + final var parsed = DockerImageName.parse(imageName); + final var methods = new ArrayList(); + for (String methodName : methodNames) { + methods.add(factory.getClass().getMethod(methodName, factory.getContainerClass())); + } + sharedContainer = factory.createNewContainer(parsed); + sharedContainer.withLogConsumer(new Slf4jLogConsumer(LOGGER)); + for (Method method : methods) { + LOGGER.info("Calling {} in {} on new shared container based on {}.", + method.getName(), factory.getClass().getName(), imageName); + method.invoke(factory, sharedContainer); + } + sharedContainer.start(); + } catch (IllegalAccessException | InvocationTargetException | NoSuchMethodException e) { + throw new RuntimeException(e); + } + } + + } + +} diff --git a/airbyte-test-utils/src/main/java/io/airbyte/test/utils/DatabaseConnectionHelper.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/DatabaseConnectionHelper.java similarity index 91% rename from airbyte-test-utils/src/main/java/io/airbyte/test/utils/DatabaseConnectionHelper.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/DatabaseConnectionHelper.java index 8d7307538734..da503eb21dfb 100644 --- a/airbyte-test-utils/src/main/java/io/airbyte/test/utils/DatabaseConnectionHelper.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/DatabaseConnectionHelper.java @@ -2,10 +2,10 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.test.utils; +package io.airbyte.cdk.testutils; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DataSourceFactory; import javax.sql.DataSource; import org.jooq.DSLContext; import org.jooq.SQLDialect; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/NonContainer.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/NonContainer.java new file mode 100644 index 000000000000..badf004d4f99 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/NonContainer.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.testutils; + +import org.testcontainers.containers.JdbcDatabaseContainer; + +/** + * This is used when a source (such as Snowflake) relies on an always-on resource and therefore + * doesn't need an actual container. compatible + */ +public class NonContainer extends JdbcDatabaseContainer { + + private final String username; + private final String password; + private final String jdbcUrl; + + private final String driverClassName; + + public NonContainer(final String userName, + final String password, + final String jdbcUrl, + final String driverClassName, + final String dockerImageName) { + super(dockerImageName); + this.username = userName; + this.password = password; + this.jdbcUrl = jdbcUrl; + this.driverClassName = driverClassName; + } + + @Override + public String getDriverClassName() { + return driverClassName; + } + + @Override + public String getJdbcUrl() { + return jdbcUrl; + } + + @Override + public String getUsername() { + return username; + } + + @Override + public String getPassword() { + return password; + } + + @Override + protected String getTestQueryString() { + return "SELECT 1"; + } + +} diff --git a/airbyte-test-utils/src/main/java/io/airbyte/test/utils/PostgreSQLContainerHelper.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/PostgreSQLContainerHelper.java similarity index 95% rename from airbyte-test-utils/src/main/java/io/airbyte/test/utils/PostgreSQLContainerHelper.java rename to airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/PostgreSQLContainerHelper.java index b9163685c9e0..45eb9ee04bae 100644 --- a/airbyte-test-utils/src/main/java/io/airbyte/test/utils/PostgreSQLContainerHelper.java +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/PostgreSQLContainerHelper.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.test.utils; +package io.airbyte.cdk.testutils; import java.io.IOException; import java.util.UUID; diff --git a/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/TestDatabase.java b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/TestDatabase.java new file mode 100644 index 000000000000..7af235093894 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/db-sources/src/testFixtures/java/io/airbyte/cdk/testutils/TestDatabase.java @@ -0,0 +1,298 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.testutils; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.ContextQueryFunction; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.JdbcConnector; +import io.airbyte.cdk.integrations.util.HostPortResolver; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.string.Strings; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.sql.SQLException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; +import javax.sql.DataSource; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.JdbcDatabaseContainer; + +/** + * TestDatabase provides a convenient pattern for interacting with databases when testing SQL + * database sources. The basic idea is to share the same database testcontainer instance for all + * tests and to use SQL constructs such as DATABASE and USER to isolate each test case's state. + * + * @param the type of the backing testcontainer. + * @param itself + * @param the type of the object returned by {@link #configBuilder()} + */ +abstract public class TestDatabase, T extends TestDatabase, B extends TestDatabase.ConfigBuilder> + implements AutoCloseable { + + static private final Logger LOGGER = LoggerFactory.getLogger(TestDatabase.class); + + final private C container; + final private String suffix; + final private ArrayList cleanupSQL = new ArrayList<>(); + final private Map connectionProperties = new HashMap<>(); + + private DataSource dataSource; + private DSLContext dslContext; + + protected TestDatabase(C container) { + this.container = container; + this.suffix = Strings.addRandomSuffix("", "_", 10); + } + + @SuppressWarnings("unchecked") + protected T self() { + return (T) this; + } + + /** + * Adds a key-value pair to the JDBC URL's query parameters. + */ + public T withConnectionProperty(String key, String value) { + if (isInitialized()) { + throw new RuntimeException("TestDatabase instance is already initialized"); + } + connectionProperties.put(key, value); + return self(); + } + + /** + * Enqueues a SQL statement to be executed when this object is closed. + */ + public T onClose(String fmtSql, Object... fmtArgs) { + cleanupSQL.add(String.format(fmtSql, fmtArgs)); + return self(); + } + + /** + * Executes a SQL statement after calling String.format on the arguments. + */ + public T with(String fmtSql, Object... fmtArgs) { + execSQL(Stream.of(String.format(fmtSql, fmtArgs))); + return self(); + } + + /** + * Executes SQL statements as root to provide the necessary isolation for the lifetime of this + * object. This typically entails at least a CREATE DATABASE and a CREATE USER. Also Initializes the + * {@link DataSource} and {@link DSLContext} owned by this object. + */ + final public T initialized() { + inContainerBootstrapCmd().forEach(this::execInContainer); + this.dataSource = DataSourceFactory.create( + getUserName(), + getPassword(), + getDatabaseDriver().getDriverClassName(), + getJdbcUrl(), + connectionProperties, + JdbcConnector.getConnectionTimeout(connectionProperties, getDatabaseDriver().getDriverClassName())); + this.dslContext = DSLContextFactory.create(dataSource, getSqlDialect()); + return self(); + } + + final public boolean isInitialized() { + return dslContext != null; + } + + abstract protected Stream> inContainerBootstrapCmd(); + + abstract protected Stream inContainerUndoBootstrapCmd(); + + abstract public DatabaseDriver getDatabaseDriver(); + + abstract public SQLDialect getSqlDialect(); + + final public C getContainer() { + return container; + } + + public String withNamespace(String name) { + return name + suffix; + } + + public String getDatabaseName() { + return withNamespace("db"); + } + + public String getUserName() { + return withNamespace("user"); + } + + public String getPassword() { + return "password"; + } + + public DataSource getDataSource() { + if (!isInitialized()) { + throw new RuntimeException("TestDatabase instance is not yet initialized"); + } + return dataSource; + } + + final public DSLContext getDslContext() { + if (!isInitialized()) { + throw new RuntimeException("TestDatabase instance is not yet initialized"); + } + return dslContext; + } + + public String getJdbcUrl() { + return String.format( + getDatabaseDriver().getUrlFormatString(), + getContainer().getHost(), + getContainer().getFirstMappedPort(), + getDatabaseName()); + } + + public Database getDatabase() { + return new Database(getDslContext()); + } + + protected void execSQL(final Stream sql) { + try { + getDatabase().query(ctx -> { + sql.forEach(statement -> { + LOGGER.debug("{}", statement); + ctx.execute(statement); + }); + return null; + }); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + protected void execInContainer(Stream cmds) { + final List cmd = cmds.toList(); + if (cmd.isEmpty()) { + return; + } + try { + LOGGER.debug("executing {}", Strings.join(cmd, " ")); + final var exec = getContainer().execInContainer(cmd.toArray(new String[0])); + if (exec.getExitCode() == 0) { + LOGGER.debug("execution success\nstdout:\n{}\nstderr:\n{}", exec.getStdout(), exec.getStderr()); + } else { + LOGGER.error("execution failure, code {}\nstdout:\n{}\nstderr:\n{}", exec.getExitCode(), exec.getStdout(), exec.getStderr()); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + + public X query(final ContextQueryFunction transform) throws SQLException { + return getDatabase().query(transform); + } + + public X transaction(final ContextQueryFunction transform) throws SQLException { + return getDatabase().transaction(transform); + } + + /** + * Returns a builder for the connector config object. + */ + public B configBuilder() { + return new ConfigBuilder(self()).self(); + } + + public B testConfigBuilder() { + return configBuilder() + .withHostAndPort() + .withCredentials() + .withDatabase(); + } + + public B integrationTestConfigBuilder() { + return configBuilder() + .withResolvedHostAndPort() + .withCredentials() + .withDatabase(); + } + + @Override + public void close() { + execSQL(this.cleanupSQL.stream()); + dslContext.close(); + execInContainer(inContainerUndoBootstrapCmd()); + } + + static public class ConfigBuilder, B extends ConfigBuilder> { + + static public final Duration DEFAULT_CDC_REPLICATION_INITIAL_WAIT = Duration.ofSeconds(5); + + protected final ImmutableMap.Builder builder = ImmutableMap.builder(); + protected final T testDatabase; + + protected ConfigBuilder(T testDatabase) { + this.testDatabase = testDatabase; + } + + public JsonNode build() { + return Jsons.jsonNode(builder.build()); + } + + @SuppressWarnings("unchecked") + final protected B self() { + return (B) this; + } + + public B with(Object key, Object value) { + builder.put(key, value); + return self(); + } + + public B withDatabase() { + return this + .with(JdbcUtils.DATABASE_KEY, testDatabase.getDatabaseName()); + } + + public B withCredentials() { + return this + .with(JdbcUtils.USERNAME_KEY, testDatabase.getUserName()) + .with(JdbcUtils.PASSWORD_KEY, testDatabase.getPassword()); + } + + public B withResolvedHostAndPort() { + return this + .with(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(testDatabase.getContainer())) + .with(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(testDatabase.getContainer())); + } + + public B withHostAndPort() { + return this + .with(JdbcUtils.HOST_KEY, testDatabase.getContainer().getHost()) + .with(JdbcUtils.PORT_KEY, testDatabase.getContainer().getFirstMappedPort()); + } + + public B withoutSsl() { + return with(JdbcUtils.SSL_KEY, false); + } + + public B withSsl(Map sslMode) { + return with(JdbcUtils.SSL_KEY, true).with(JdbcUtils.SSL_MODE_KEY, sslMode); + } + + } + +} diff --git a/airbyte-config-oss/init-oss/bin/main/icons/rss.svg b/airbyte-cdk/java/airbyte-cdk/init-oss/bin/main/icons/rss.svg similarity index 100% rename from airbyte-config-oss/init-oss/bin/main/icons/rss.svg rename to airbyte-cdk/java/airbyte-cdk/init-oss/bin/main/icons/rss.svg diff --git a/airbyte-cdk/java/airbyte-cdk/init-oss/build.gradle b/airbyte-cdk/java/airbyte-cdk/init-oss/build.gradle new file mode 100644 index 000000000000..c76851730e11 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/init-oss/build.gradle @@ -0,0 +1,28 @@ +plugins { + id 'java-library' + id "de.undercouch.download" version "5.4.0" +} + +dependencies { + annotationProcessor libs.bundles.micronaut.annotation.processor + api libs.bundles.micronaut.annotation + + implementation 'commons-cli:commons-cli:1.4' + implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') + implementation project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') + implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') + implementation libs.lombok + implementation libs.micronaut.cache.caffeine + + testImplementation 'com.squareup.okhttp3:mockwebserver:4.9.1' +} + +def downloadConnectorRegistry = tasks.register('downloadConnectorRegistry', Download) { + src 'https://connectors.airbyte.com/files/registries/v0/oss_registry.json' + dest new File(projectDir, 'src/main/resources/seed/oss_registry.json') + overwrite true +} +tasks.named('processResources')configure { + dependsOn downloadConnectorRegistry +} diff --git a/airbyte-config-oss/init-oss/readme.md b/airbyte-cdk/java/airbyte-cdk/init-oss/readme.md similarity index 100% rename from airbyte-config-oss/init-oss/readme.md rename to airbyte-cdk/java/airbyte-cdk/init-oss/readme.md diff --git a/airbyte-config-oss/init-oss/scripts/create_mount_directories.sh b/airbyte-cdk/java/airbyte-cdk/init-oss/scripts/create_mount_directories.sh similarity index 100% rename from airbyte-config-oss/init-oss/scripts/create_mount_directories.sh rename to airbyte-cdk/java/airbyte-cdk/init-oss/scripts/create_mount_directories.sh diff --git a/airbyte-config-oss/init-oss/src/main/java/io/airbyte/configoss/init/ConfigNotFoundException.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/ConfigNotFoundException.java similarity index 100% rename from airbyte-config-oss/init-oss/src/main/java/io/airbyte/configoss/init/ConfigNotFoundException.java rename to airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/ConfigNotFoundException.java diff --git a/airbyte-config-oss/init-oss/src/main/java/io/airbyte/configoss/init/DefinitionsProvider.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/DefinitionsProvider.java similarity index 100% rename from airbyte-config-oss/init-oss/src/main/java/io/airbyte/configoss/init/DefinitionsProvider.java rename to airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/DefinitionsProvider.java diff --git a/airbyte-config-oss/init-oss/src/main/java/io/airbyte/configoss/init/JsonDefinitionsHelper.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/JsonDefinitionsHelper.java similarity index 100% rename from airbyte-config-oss/init-oss/src/main/java/io/airbyte/configoss/init/JsonDefinitionsHelper.java rename to airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/JsonDefinitionsHelper.java diff --git a/airbyte-config-oss/init-oss/src/main/java/io/airbyte/configoss/init/LocalDefinitionsProvider.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/LocalDefinitionsProvider.java similarity index 100% rename from airbyte-config-oss/init-oss/src/main/java/io/airbyte/configoss/init/LocalDefinitionsProvider.java rename to airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/LocalDefinitionsProvider.java diff --git a/airbyte-config-oss/init-oss/src/main/java/io/airbyte/configoss/init/PostLoadExecutor.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/PostLoadExecutor.java similarity index 100% rename from airbyte-config-oss/init-oss/src/main/java/io/airbyte/configoss/init/PostLoadExecutor.java rename to airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/PostLoadExecutor.java diff --git a/airbyte-config-oss/init-oss/src/main/java/io/airbyte/configoss/init/RemoteDefinitionsProvider.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/RemoteDefinitionsProvider.java similarity index 100% rename from airbyte-config-oss/init-oss/src/main/java/io/airbyte/configoss/init/RemoteDefinitionsProvider.java rename to airbyte-cdk/java/airbyte-cdk/init-oss/src/main/java/io/airbyte/configoss/init/RemoteDefinitionsProvider.java diff --git a/airbyte-config-oss/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json similarity index 100% rename from airbyte-config-oss/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json rename to airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/424892c4-daac-4491-b35d-c6688ba547ba.json diff --git a/airbyte-config-oss/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json similarity index 100% rename from airbyte-config-oss/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json rename to airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/4816b78f-1489-44c1-9060-4b19d5fa9362.json diff --git a/airbyte-config-oss/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/f7a7d195-377f-cf5b-70a5-be6b819019dc.json b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/f7a7d195-377f-cf5b-70a5-be6b819019dc.json similarity index 100% rename from airbyte-config-oss/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/f7a7d195-377f-cf5b-70a5-be6b819019dc.json rename to airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_DESTINATION_DEFINITION/f7a7d195-377f-cf5b-70a5-be6b819019dc.json diff --git a/airbyte-config-oss/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2817b3f0-04e4-4c7a-9f32-7a5e8a83db95.json b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2817b3f0-04e4-4c7a-9f32-7a5e8a83db95.json similarity index 100% rename from airbyte-config-oss/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2817b3f0-04e4-4c7a-9f32-7a5e8a83db95.json rename to airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/2817b3f0-04e4-4c7a-9f32-7a5e8a83db95.json diff --git a/airbyte-config-oss/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6fe89830-d04d-401b-aad6-6552ffa5c4af.json b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6fe89830-d04d-401b-aad6-6552ffa5c4af.json similarity index 100% rename from airbyte-config-oss/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6fe89830-d04d-401b-aad6-6552ffa5c4af.json rename to airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6fe89830-d04d-401b-aad6-6552ffa5c4af.json diff --git a/airbyte-config-oss/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/7e20ce3e-d820-4327-ad7a-88f3927fd97a.json b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/7e20ce3e-d820-4327-ad7a-88f3927fd97a.json similarity index 100% rename from airbyte-config-oss/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/7e20ce3e-d820-4327-ad7a-88f3927fd97a.json rename to airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/7e20ce3e-d820-4327-ad7a-88f3927fd97a.json diff --git a/airbyte-config-oss/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c47d6804-8b98-449f-970a-5ddb5cb5d7aa.json b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c47d6804-8b98-449f-970a-5ddb5cb5d7aa.json similarity index 100% rename from airbyte-config-oss/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c47d6804-8b98-449f-970a-5ddb5cb5d7aa.json rename to airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/c47d6804-8b98-449f-970a-5ddb5cb5d7aa.json diff --git a/airbyte-config-oss/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json similarity index 100% rename from airbyte-config-oss/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json rename to airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json diff --git a/airbyte-config-oss/init-oss/src/main/resources/icons/airbyte.svg b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/icons/airbyte.svg similarity index 100% rename from airbyte-config-oss/init-oss/src/main/resources/icons/airbyte.svg rename to airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/icons/airbyte.svg diff --git a/airbyte-config-oss/init-oss/src/main/resources/icons/ringcentral.svg b/airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/icons/ringcentral.svg similarity index 100% rename from airbyte-config-oss/init-oss/src/main/resources/icons/ringcentral.svg rename to airbyte-cdk/java/airbyte-cdk/init-oss/src/main/resources/icons/ringcentral.svg diff --git a/airbyte-config-oss/init-oss/src/test/java/io/airbyte/configoss/init/LocalDefinitionsProviderTest.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/LocalDefinitionsProviderTest.java similarity index 100% rename from airbyte-config-oss/init-oss/src/test/java/io/airbyte/configoss/init/LocalDefinitionsProviderTest.java rename to airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/LocalDefinitionsProviderTest.java diff --git a/airbyte-config-oss/init-oss/src/test/java/io/airbyte/configoss/init/RemoteDefinitionsProviderTest.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/RemoteDefinitionsProviderTest.java similarity index 100% rename from airbyte-config-oss/init-oss/src/test/java/io/airbyte/configoss/init/RemoteDefinitionsProviderTest.java rename to airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/RemoteDefinitionsProviderTest.java diff --git a/airbyte-config-oss/init-oss/src/test/java/io/airbyte/configoss/init/SpecFormatTest.java b/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/SpecFormatTest.java similarity index 100% rename from airbyte-config-oss/init-oss/src/test/java/io/airbyte/configoss/init/SpecFormatTest.java rename to airbyte-cdk/java/airbyte-cdk/init-oss/src/test/java/io/airbyte/configoss/init/SpecFormatTest.java diff --git a/airbyte-config-oss/init-oss/src/test/resources/connector_catalog.json b/airbyte-cdk/java/airbyte-cdk/init-oss/src/test/resources/connector_catalog.json similarity index 100% rename from airbyte-config-oss/init-oss/src/test/resources/connector_catalog.json rename to airbyte-cdk/java/airbyte-cdk/init-oss/src/test/resources/connector_catalog.json diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle b/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle new file mode 100644 index 000000000000..cc3b7e90aaf5 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/build.gradle @@ -0,0 +1,56 @@ + +dependencies { + + implementation project(':airbyte-cdk:java:airbyte-cdk:core') + implementation project(':airbyte-cdk:java:airbyte-cdk:db-destinations') + implementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') + testImplementation project(':airbyte-cdk:java:airbyte-cdk:db-destinations') + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:db-destinations') + testFixturesImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:db-destinations')) + + + compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-api') + compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') + compileOnly project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') + compileOnly project(':airbyte-cdk:java:airbyte-cdk:init-oss') + compileOnly project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') + + testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + + testImplementation 'org.mockito:mockito-core:4.6.1' + + // Lombok + implementation 'org.projectlombok:lombok:1.18.20' + annotationProcessor 'org.projectlombok:lombok:1.18.20' + testFixturesImplementation 'org.projectlombok:lombok:1.18.20' + testFixturesAnnotationProcessor 'org.projectlombok:lombok:1.18.20' + + implementation ('org.apache.hadoop:hadoop-aws:3.3.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} + implementation ('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.3') {exclude group: 'org.slf4j', module: 'slf4j-log4j12' exclude group: 'org.slf4j', module: 'slf4j-reload4j'} + implementation group: 'com.hadoop.gplcompression', name: 'hadoop-lzo', version: '0.4.20' + implementation ('org.apache.hadoop:hadoop-common:3.3.3') { + exclude group: 'org.slf4j', module: 'slf4j-log4j12' + exclude group: 'org.slf4j', module: 'slf4j-reload4j' + } + + implementation 'com.github.alexmojaki:s3-stream-upload:2.2.2' + implementation 'org.apache.commons:commons-csv:1.4' + implementation ('org.apache.parquet:parquet-avro:1.12.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} + implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} + + implementation libs.bundles.junit + testImplementation libs.junit.jupiter.system.stubs + + +} + +java { + compileJava { + options.compilerArgs.remove("-Werror") + } + compileTestJava { + options.compilerArgs += "-Xlint:-try" + } +} diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3CopyConfig.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3CopyConfig.java similarity index 87% rename from airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3CopyConfig.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3CopyConfig.java index 9f782e0c176c..ca25e43f0ca2 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3CopyConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3CopyConfig.java @@ -2,10 +2,10 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.jdbc.copy.s3; +package io.airbyte.cdk.integrations.destination.jdbc.copy.s3; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; /** * S3 copy destinations need an S3DestinationConfig to configure the basic upload behavior. We also diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopier.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3StreamCopier.java similarity index 91% rename from airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopier.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3StreamCopier.java index 10d0da5e880f..bdf669194a91 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopier.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3StreamCopier.java @@ -2,21 +2,21 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.jdbc.copy.s3; +package io.airbyte.cdk.integrations.destination.jdbc.copy.s3; import com.amazonaws.services.s3.AmazonS3; import com.google.common.annotations.VisibleForTesting; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopier; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.S3FormatConfig; -import io.airbyte.integrations.destination.s3.csv.S3CsvFormatConfig; -import io.airbyte.integrations.destination.s3.csv.S3CsvWriter; -import io.airbyte.integrations.destination.s3.csv.StagingDatabaseCsvSheetGenerator; -import io.airbyte.integrations.destination.s3.util.CompressionType; -import io.airbyte.integrations.destination.s3.writer.DestinationFileWriter; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopier; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; +import io.airbyte.cdk.integrations.destination.s3.csv.S3CsvFormatConfig; +import io.airbyte.cdk.integrations.destination.s3.csv.S3CsvWriter; +import io.airbyte.cdk.integrations.destination.s3.csv.StagingDatabaseCsvSheetGenerator; +import io.airbyte.cdk.integrations.destination.s3.util.CompressionType; +import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopierFactory.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3StreamCopierFactory.java similarity index 82% rename from airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopierFactory.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3StreamCopierFactory.java index 0a7c4a90f0a3..b9b94c72c329 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopierFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3StreamCopierFactory.java @@ -2,14 +2,14 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.jdbc.copy.s3; +package io.airbyte.cdk.integrations.destination.jdbc.copy.s3; import com.amazonaws.services.s3.AmazonS3; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopier; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopierFactory; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopier; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopierFactory; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/AesCbcEnvelopeEncryption.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/AesCbcEnvelopeEncryption.java similarity index 97% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/AesCbcEnvelopeEncryption.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/AesCbcEnvelopeEncryption.java index 7f6e167b2a1d..cff03ce93789 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/AesCbcEnvelopeEncryption.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/AesCbcEnvelopeEncryption.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import com.fasterxml.jackson.databind.JsonNode; import java.security.NoSuchAlgorithmException; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/AesCbcEnvelopeEncryptionBlobDecorator.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/AesCbcEnvelopeEncryptionBlobDecorator.java similarity index 99% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/AesCbcEnvelopeEncryptionBlobDecorator.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/AesCbcEnvelopeEncryptionBlobDecorator.java index c8e99d4a994d..d8d9113d220b 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/AesCbcEnvelopeEncryptionBlobDecorator.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/AesCbcEnvelopeEncryptionBlobDecorator.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import com.google.common.annotations.VisibleForTesting; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/BaseS3Destination.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/BaseS3Destination.java similarity index 86% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/BaseS3Destination.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/BaseS3Destination.java index 482a80fe77c7..fab68a7a8d5a 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/BaseS3Destination.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/BaseS3Destination.java @@ -2,16 +2,16 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.record_buffer.FileBuffer; -import io.airbyte.integrations.destination.s3.util.S3NameTransformer; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.record_buffer.FileBuffer; +import io.airbyte.cdk.integrations.destination.s3.util.S3NameTransformer; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/BlobDecorator.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/BlobDecorator.java similarity index 96% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/BlobDecorator.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/BlobDecorator.java index dd8ef4c873ee..61831fd995f2 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/BlobDecorator.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/BlobDecorator.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import com.google.common.annotations.VisibleForTesting; import java.io.OutputStream; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/BlobStorageOperations.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/BlobStorageOperations.java similarity index 90% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/BlobStorageOperations.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/BlobStorageOperations.java index 8090a2f06850..9df281e9e19b 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/BlobStorageOperations.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/BlobStorageOperations.java @@ -2,10 +2,10 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.destination.record_buffer.SerializableBuffer; +import io.airbyte.cdk.integrations.destination.record_buffer.SerializableBuffer; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -31,7 +31,7 @@ protected BlobStorageOperations() { * * @return the name of the file that was uploaded. */ - public abstract String uploadRecordsToBucket(SerializableBuffer recordsData, String namespace, String streamName, String objectPath) + public abstract String uploadRecordsToBucket(SerializableBuffer recordsData, String namespace, String objectPath) throws Exception; /** diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/EncryptionConfig.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/EncryptionConfig.java similarity index 94% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/EncryptionConfig.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/EncryptionConfig.java index 021f67c2a1bf..ecd02c1e6da3 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/EncryptionConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/EncryptionConfig.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import com.fasterxml.jackson.databind.JsonNode; import java.util.Base64; diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/NoEncryption.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/NoEncryption.java new file mode 100644 index 000000000000..7dcbd6669195 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/NoEncryption.java @@ -0,0 +1,9 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.s3; + +public final class NoEncryption implements EncryptionConfig { + +} diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3BaseChecks.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3BaseChecks.java similarity index 97% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3BaseChecks.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3BaseChecks.java index 2f4ddc929f88..81219eb00a50 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3BaseChecks.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3BaseChecks.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import alex.mojaki.s3upload.MultiPartOutputStream; import alex.mojaki.s3upload.StreamTransferManager; @@ -10,7 +10,7 @@ import com.amazonaws.services.s3.model.ListObjectsRequest; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; -import io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory; +import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; import java.io.IOException; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3ConsumerFactory.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3ConsumerFactory.java similarity index 90% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3ConsumerFactory.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3ConsumerFactory.java index 6be10507d527..38068dbf38c1 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3ConsumerFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3ConsumerFactory.java @@ -2,18 +2,18 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import com.google.common.base.Preconditions; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.BufferedStreamConsumer; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.OnCloseFunction; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.OnStartFunction; +import io.airbyte.cdk.integrations.destination.record_buffer.BufferCreateFunction; +import io.airbyte.cdk.integrations.destination.record_buffer.FlushBufferFunction; +import io.airbyte.cdk.integrations.destination.record_buffer.SerializedBufferingStrategy; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.buffered_stream_consumer.BufferedStreamConsumer; -import io.airbyte.integrations.destination.buffered_stream_consumer.OnCloseFunction; -import io.airbyte.integrations.destination.buffered_stream_consumer.OnStartFunction; -import io.airbyte.integrations.destination.record_buffer.BufferCreateFunction; -import io.airbyte.integrations.destination.record_buffer.FlushBufferFunction; -import io.airbyte.integrations.destination.record_buffer.SerializedBufferingStrategy; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; @@ -128,7 +128,6 @@ private FlushBufferFunction flushBufferFunction(final BlobStorageOperations stor writeConfig.addStoredFile(storageOperations.uploadRecordsToBucket( writer, writeConfig.getNamespace(), - writeConfig.getStreamName(), writeConfig.getFullOutputPath())); } catch (final Exception e) { LOGGER.error("Failed to flush and upload buffer to storage:", e); @@ -139,7 +138,7 @@ private FlushBufferFunction flushBufferFunction(final BlobStorageOperations stor private OnCloseFunction onCloseFunction(final BlobStorageOperations storageOperations, final List writeConfigs) { - return (hasFailed) -> { + return (hasFailed, streamSyncSummaries) -> { if (hasFailed) { LOGGER.info("Cleaning up destination started for {} streams", writeConfigs.size()); for (final WriteConfig writeConfig : writeConfigs) { diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3DestinationConfig.java similarity index 90% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3DestinationConfig.java index 333504189367..6694c0484dd3 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3DestinationConfig.java @@ -2,17 +2,17 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; - -import static io.airbyte.integrations.destination.s3.constant.S3Constants.ACCESS_KEY_ID; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.ACCOUNT_ID; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.FILE_NAME_PATTERN; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.SECRET_ACCESS_KEY; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_NAME; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_PATH; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_REGION; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_ENDPOINT; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_PATH_FORMAT; +package io.airbyte.cdk.integrations.destination.s3; + +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.ACCESS_KEY_ID; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.ACCOUNT_ID; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.FILE_NAME_PATTERN; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.SECRET_ACCESS_KEY; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_NAME; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_PATH; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_REGION; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_ENDPOINT; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_PATH_FORMAT; import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; @@ -22,10 +22,10 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.destination.s3.credential.S3AWSDefaultProfileCredentialConfig; -import io.airbyte.integrations.destination.s3.credential.S3AccessKeyCredentialConfig; -import io.airbyte.integrations.destination.s3.credential.S3CredentialConfig; -import io.airbyte.integrations.destination.s3.credential.S3CredentialType; +import io.airbyte.cdk.integrations.destination.s3.credential.S3AWSDefaultProfileCredentialConfig; +import io.airbyte.cdk.integrations.destination.s3.credential.S3AccessKeyCredentialConfig; +import io.airbyte.cdk.integrations.destination.s3.credential.S3CredentialConfig; +import io.airbyte.cdk.integrations.destination.s3.credential.S3CredentialType; import java.util.Objects; import javax.annotation.Nonnull; import javax.annotation.Nullable; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfigFactory.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3DestinationConfigFactory.java similarity index 88% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfigFactory.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3DestinationConfigFactory.java index 69624258cde1..6dbbe4cd9772 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfigFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3DestinationConfigFactory.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import com.fasterxml.jackson.databind.JsonNode; import javax.annotation.Nonnull; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConstants.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3DestinationConstants.java similarity index 80% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConstants.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3DestinationConstants.java index 678cd5f1c800..82585016f7c4 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConstants.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3DestinationConstants.java @@ -2,10 +2,10 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; -import io.airbyte.integrations.destination.s3.util.CompressionType; -import io.airbyte.integrations.destination.s3.util.S3NameTransformer; +import io.airbyte.cdk.integrations.destination.s3.util.CompressionType; +import io.airbyte.cdk.integrations.destination.s3.util.S3NameTransformer; public final class S3DestinationConstants { diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Format.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3Format.java similarity index 87% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Format.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3Format.java index 781004739ac8..319dbd7ef545 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Format.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3Format.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; public enum S3Format { diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfig.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3FormatConfig.java similarity index 94% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfig.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3FormatConfig.java index a8a843684ca9..f4852b09be2a 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3FormatConfig.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import com.fasterxml.jackson.databind.JsonNode; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3FormatConfigs.java similarity index 77% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3FormatConfigs.java index d759059432e3..8a64fad378da 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3FormatConfigs.java @@ -2,14 +2,14 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.s3.avro.S3AvroFormatConfig; +import io.airbyte.cdk.integrations.destination.s3.csv.S3CsvFormatConfig; +import io.airbyte.cdk.integrations.destination.s3.jsonl.S3JsonlFormatConfig; +import io.airbyte.cdk.integrations.destination.s3.parquet.S3ParquetFormatConfig; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.s3.avro.S3AvroFormatConfig; -import io.airbyte.integrations.destination.s3.csv.S3CsvFormatConfig; -import io.airbyte.integrations.destination.s3.jsonl.S3JsonlFormatConfig; -import io.airbyte.integrations.destination.s3.parquet.S3ParquetFormatConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3StorageOperations.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3StorageOperations.java similarity index 86% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3StorageOperations.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3StorageOperations.java index b5c4fa3cde34..9db0d0d4994a 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/S3StorageOperations.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/S3StorageOperations.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import static org.apache.logging.log4j.util.Strings.isNotBlank; @@ -16,14 +16,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.record_buffer.SerializableBuffer; +import io.airbyte.cdk.integrations.destination.s3.template.S3FilenameTemplateManager; +import io.airbyte.cdk.integrations.destination.s3.template.S3FilenameTemplateParameterObject; +import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; +import io.airbyte.cdk.integrations.util.ConnectorExceptionUtil; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.string.Strings; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.record_buffer.SerializableBuffer; -import io.airbyte.integrations.destination.s3.template.S3FilenameTemplateManager; -import io.airbyte.integrations.destination.s3.template.S3FilenameTemplateParameterObject; -import io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory; -import io.airbyte.integrations.util.ConnectorExceptionUtil; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -32,6 +32,9 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.regex.Pattern; import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.StringUtils; @@ -65,6 +68,7 @@ public class S3StorageOperations extends BlobStorageOperations { private static final String FORMAT_VARIABLE_EPOCH = "${EPOCH}"; private static final String FORMAT_VARIABLE_UUID = "${UUID}"; private static final String GZ_FILE_EXTENSION = "gz"; + private final ConcurrentMap partCounts = new ConcurrentHashMap<>(); private final NamingConventionTransformer nameTransformer; protected final S3DestinationConfig s3Config; @@ -116,7 +120,6 @@ protected boolean doesBucketExist(final String bucket) { @Override public String uploadRecordsToBucket(final SerializableBuffer recordsData, final String namespace, - final String streamName, final String objectPath) { final List exceptionsThrown = new ArrayList<>(); while (exceptionsThrown.size() < UPLOAD_RETRY_LIMIT) { @@ -156,7 +159,7 @@ public String uploadRecordsToBucket(final SerializableBuffer recordsData, private String loadDataIntoBucket(final String objectPath, final SerializableBuffer recordsData) throws IOException { final long partSize = DEFAULT_PART_SIZE; final String bucket = s3Config.getBucketName(); - final String partId = UUID.randomUUID().toString(); + final String partId = getPartId(objectPath); final String fileExtension = getExtension(recordsData.getFilename()); final String fullObjectKey; if (StringUtils.isNotBlank(s3Config.getFileNamePattern())) { @@ -215,6 +218,41 @@ private String loadDataIntoBucket(final String objectPath, final SerializableBuf return newFilename; } + /** + * Users want deterministic file names (e.g. the first file part is really foo-0.csv). Using UUIDs + * (previous approach) doesn't allow that. However, using pure integers could lead to a collision + * with an upload from another thread. We also want to be able to continue the same offset between + * attempts. So, we'll count up the existing files in the directory and use that as a lazy-offset, + * assuming airbyte manages the dir and has similar naming conventions. `getPartId` will be + * 0-indexed. + */ + @VisibleForTesting + synchronized String getPartId(String objectPath) { + final AtomicInteger partCount = partCounts.computeIfAbsent(objectPath, k -> new AtomicInteger(0)); + + if (partCount.get() == 0) { + ObjectListing objects; + int objectCount = 0; + + final String bucket = s3Config.getBucketName(); + objects = s3Client.listObjects(bucket, objectPath); + + if (objects != null) { + objectCount = objectCount + objects.getObjectSummaries().size(); + while (objects != null && objects.getNextMarker() != null) { + objects = s3Client.listObjects(new ListObjectsRequest().withBucketName(bucket).withPrefix(objectPath).withMarker(objects.getNextMarker())); + if (objects != null) { + objectCount = objectCount + objects.getObjectSummaries().size(); + } + } + } + + partCount.set(objectCount); + } + + return Integer.toString(partCount.getAndIncrement()); + } + @VisibleForTesting static String getFilename(final String fullPath) { return fullPath.substring(fullPath.lastIndexOf("/") + 1); diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/SerializedBufferFactory.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/SerializedBufferFactory.java new file mode 100644 index 000000000000..a4deb0aa5706 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/SerializedBufferFactory.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.s3; + +import io.airbyte.cdk.integrations.destination.record_buffer.BufferCreateFunction; +import io.airbyte.cdk.integrations.destination.record_buffer.BufferStorage; +import io.airbyte.cdk.integrations.destination.record_buffer.SerializableBuffer; +import io.airbyte.cdk.integrations.destination.s3.avro.AvroSerializedBuffer; +import io.airbyte.cdk.integrations.destination.s3.avro.S3AvroFormatConfig; +import io.airbyte.cdk.integrations.destination.s3.csv.CsvSerializedBuffer; +import io.airbyte.cdk.integrations.destination.s3.csv.S3CsvFormatConfig; +import io.airbyte.cdk.integrations.destination.s3.jsonl.JsonLSerializedBuffer; +import io.airbyte.cdk.integrations.destination.s3.jsonl.S3JsonlFormatConfig; +import io.airbyte.cdk.integrations.destination.s3.parquet.ParquetSerializedBuffer; +import io.airbyte.commons.json.Jsons; +import java.util.concurrent.Callable; +import java.util.function.Function; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SerializedBufferFactory { + + protected static final Logger LOGGER = LoggerFactory.getLogger(SerializedBufferFactory.class); + + /** + * When running a + * {@link io.airbyte.cdk.integrations.destination.record_buffer.SerializedBufferingStrategy}, it + * would usually need to instantiate new buffers when flushing data or when it receives data for a + * brand-new stream. This factory fills this need and @return the function to be called on such + * events. + *

      + * The factory is responsible for choosing the correct constructor function for a new + * {@link SerializableBuffer} that handles the correct serialized format of the data. It is + * configured by composition with another function to create a new {@link BufferStorage} where to + * store it. + *

      + * This factory determines which {@link S3FormatConfig} to use depending on the user provided @param + * config, The @param createStorageFunctionWithoutExtension is the constructor function to call when + * creating a new buffer where to store data. Note that we typically associate which format is being + * stored in the storage object thanks to its file extension. + */ + public static BufferCreateFunction getCreateFunction(final S3DestinationConfig config, + final Function createStorageFunctionWithoutExtension) { + final S3FormatConfig formatConfig = config.getFormatConfig(); + LOGGER.info("S3 format config: {}", formatConfig.toString()); + switch (formatConfig.getFormat()) { + case AVRO -> { + final Callable createStorageFunctionWithExtension = + () -> createStorageFunctionWithoutExtension.apply(formatConfig.getFileExtension()); + return AvroSerializedBuffer.createFunction((S3AvroFormatConfig) formatConfig, createStorageFunctionWithExtension); + } + case CSV -> { + final Callable createStorageFunctionWithExtension = + () -> createStorageFunctionWithoutExtension.apply(formatConfig.getFileExtension()); + return CsvSerializedBuffer.createFunction((S3CsvFormatConfig) formatConfig, createStorageFunctionWithExtension); + } + case JSONL -> { + final Callable createStorageFunctionWithExtension = + () -> createStorageFunctionWithoutExtension.apply(formatConfig.getFileExtension()); + return JsonLSerializedBuffer.createBufferFunction((S3JsonlFormatConfig) formatConfig, createStorageFunctionWithExtension); + } + case PARQUET -> { + // we can't choose the type of buffer storage with parquet because of how the underlying hadoop + // library is imposing file usage. + return ParquetSerializedBuffer.createFunction(config); + } + default -> { + throw new RuntimeException("Unexpected output format: " + Jsons.serialize(config)); + } + } + } + +} diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/StorageProvider.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/StorageProvider.java similarity index 75% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/StorageProvider.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/StorageProvider.java index fe94b4df192e..c21ad66667f0 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/StorageProvider.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/StorageProvider.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; /** * Represents storage provider type diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/WriteConfig.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/WriteConfig.java new file mode 100644 index 000000000000..be4bdc2e0f40 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/WriteConfig.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.s3; + +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import java.util.ArrayList; +import java.util.List; + +/** + * Write configuration POJO for blob storage destinations + */ +public class WriteConfig { + + private final String namespace; + private final String streamName; + private final String outputBucketPath; + private final String pathFormat; + private final String fullOutputPath; + private final DestinationSyncMode syncMode; + private final List storedFiles; + + public WriteConfig(final String namespace, + final String streamName, + final String outputBucketPath, + final String pathFormat, + final String fullOutputPath, + final DestinationSyncMode syncMode) { + this.namespace = namespace; + this.streamName = streamName; + this.outputBucketPath = outputBucketPath; + this.pathFormat = pathFormat; + this.fullOutputPath = fullOutputPath; + this.syncMode = syncMode; + this.storedFiles = new ArrayList<>(); + } + + public String getNamespace() { + return namespace; + } + + public String getStreamName() { + return streamName; + } + + public String getOutputBucketPath() { + return outputBucketPath; + } + + public String getPathFormat() { + return pathFormat; + } + + public String getFullOutputPath() { + return fullOutputPath; + } + + public DestinationSyncMode getSyncMode() { + return syncMode; + } + + public List getStoredFiles() { + return storedFiles; + } + + public void addStoredFile(final String file) { + storedFiles.add(file); + } + + public void clearStoredFiles() { + storedFiles.clear(); + } + + @Override + public String toString() { + return "WriteConfig{" + + "streamName=" + streamName + + ", namespace=" + namespace + + ", outputBucketPath=" + outputBucketPath + + ", pathFormat=" + pathFormat + + ", fullOutputPath=" + fullOutputPath + + ", syncMode=" + syncMode + + '}'; + } + +} diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/AvroConstants.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/AvroConstants.java similarity index 94% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/AvroConstants.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/AvroConstants.java index 7a4d5097df30..46455961059b 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/AvroConstants.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/AvroConstants.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.avro; +package io.airbyte.cdk.integrations.destination.s3.avro; import java.util.Set; import tech.allegro.schema.json2avro.converter.JsonAvroConverter; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/AvroNameTransformer.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/AvroNameTransformer.java similarity index 90% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/AvroNameTransformer.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/AvroNameTransformer.java index b9378ffd0ac0..8521b687e6df 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/AvroNameTransformer.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/AvroNameTransformer.java @@ -2,9 +2,9 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.avro; +package io.airbyte.cdk.integrations.destination.s3.avro; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import java.util.Arrays; /** diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/AvroRecordFactory.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/AvroRecordFactory.java new file mode 100644 index 000000000000..8115d9f98357 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/AvroRecordFactory.java @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.s3.avro; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.commons.jackson.MoreMappers; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import java.util.UUID; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import tech.allegro.schema.json2avro.converter.JsonAvroConverter; + +public class AvroRecordFactory { + + private static final ObjectMapper MAPPER = MoreMappers.initMapper(); + private static final ObjectWriter WRITER = MAPPER.writer(); + + private final Schema schema; + private final JsonAvroConverter converter; + + public AvroRecordFactory(final Schema schema, final JsonAvroConverter converter) { + this.schema = schema; + this.converter = converter; + } + + public GenericData.Record getAvroRecord(final UUID id, final AirbyteRecordMessage recordMessage) throws JsonProcessingException { + final ObjectNode jsonRecord = MAPPER.createObjectNode(); + jsonRecord.put(JavaBaseConstants.COLUMN_NAME_AB_ID, id.toString()); + jsonRecord.put(JavaBaseConstants.COLUMN_NAME_EMITTED_AT, recordMessage.getEmittedAt()); + jsonRecord.setAll((ObjectNode) recordMessage.getData()); + + return converter.convertToGenericDataRecord(WRITER.writeValueAsBytes(jsonRecord), schema); + } + + public GenericData.Record getAvroRecord(final JsonNode formattedData) throws JsonProcessingException { + final var bytes = WRITER.writeValueAsBytes(formattedData); + return converter.convertToGenericDataRecord(bytes, schema); + } + +} diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/AvroSerializedBuffer.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/AvroSerializedBuffer.java similarity index 91% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/AvroSerializedBuffer.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/AvroSerializedBuffer.java index 6bf1d069d2f6..bcfcb9ac9e47 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/AvroSerializedBuffer.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/AvroSerializedBuffer.java @@ -2,11 +2,11 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.avro; +package io.airbyte.cdk.integrations.destination.s3.avro; -import io.airbyte.integrations.destination.record_buffer.BaseSerializedBuffer; -import io.airbyte.integrations.destination.record_buffer.BufferCreateFunction; -import io.airbyte.integrations.destination.record_buffer.BufferStorage; +import io.airbyte.cdk.integrations.destination.record_buffer.BaseSerializedBuffer; +import io.airbyte.cdk.integrations.destination.record_buffer.BufferCreateFunction; +import io.airbyte.cdk.integrations.destination.record_buffer.BufferStorage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/JsonFieldNameUpdater.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/JsonFieldNameUpdater.java similarity index 95% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/JsonFieldNameUpdater.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/JsonFieldNameUpdater.java index cbb3b77499a6..1da76566a71c 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/JsonFieldNameUpdater.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/JsonFieldNameUpdater.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.avro; +package io.airbyte.cdk.integrations.destination.s3.avro; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/JsonSchemaType.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/JsonSchemaType.java new file mode 100644 index 000000000000..14dee754f0f9 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/JsonSchemaType.java @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.s3.avro; + +import java.util.Arrays; +import java.util.List; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import org.apache.avro.Schema; + +/** + * Mapping of JsonSchema types to Avro types. + */ +public enum JsonSchemaType { + + STRING_V1("WellKnownTypes.json#/definitions/String", Schema.Type.STRING), + INTEGER_V1("WellKnownTypes.json#/definitions/Integer", Schema.Type.LONG), + NUMBER_V1("WellKnownTypes.json#/definitions/Number", Schema.Type.DOUBLE), + BOOLEAN_V1("WellKnownTypes.json#/definitions/Boolean", Schema.Type.BOOLEAN), + BINARY_DATA_V1("WellKnownTypes.json#/definitions/BinaryData", Schema.Type.BYTES), + DATE_V1("WellKnownTypes.json#/definitions/Date", Schema.Type.INT), + TIMESTAMP_WITH_TIMEZONE_V1("WellKnownTypes.json#/definitions/TimestampWithTimezone", Schema.Type.LONG), + TIMESTAMP_WITHOUT_TIMEZONE_V1("WellKnownTypes.json#/definitions/TimestampWithoutTimezone", Schema.Type.LONG), + TIME_WITH_TIMEZONE_V1("WellKnownTypes.json#/definitions/TimeWithTimezone", Schema.Type.STRING), + TIME_WITHOUT_TIMEZONE_V1("WellKnownTypes.json#/definitions/TimeWithoutTimezone", Schema.Type.LONG), + OBJECT("object", Schema.Type.RECORD), + ARRAY("array", Schema.Type.ARRAY), + COMBINED("combined", Schema.Type.UNION), + @Deprecated + STRING_V0("string", null, Schema.Type.STRING), + @Deprecated + NUMBER_INT_V0("number", "integer", Schema.Type.LONG), + @Deprecated + NUMBER_BIGINT_V0("string", "big_integer", Schema.Type.STRING), + @Deprecated + NUMBER_FLOAT_V0("number", "float", Schema.Type.FLOAT), + @Deprecated + NUMBER_V0("number", null, Schema.Type.DOUBLE), + @Deprecated + INTEGER_V0("integer", null, Schema.Type.LONG), + @Deprecated + BOOLEAN_V0("boolean", null, Schema.Type.BOOLEAN), + @Deprecated + NULL("null", null, Schema.Type.NULL); + + private final String jsonSchemaType; + private final Schema.Type avroType; + private String jsonSchemaAirbyteType; + + JsonSchemaType(final String jsonSchemaType, final String jsonSchemaAirbyteType, final Schema.Type avroType) { + this.jsonSchemaType = jsonSchemaType; + this.jsonSchemaAirbyteType = jsonSchemaAirbyteType; + this.avroType = avroType; + } + + JsonSchemaType(final String jsonSchemaType, final Schema.Type avroType) { + this.jsonSchemaType = jsonSchemaType; + this.avroType = avroType; + } + + public static JsonSchemaType fromJsonSchemaType(final String jsonSchemaType) { + return fromJsonSchemaType(jsonSchemaType, null); + } + + public static JsonSchemaType fromJsonSchemaType(final @Nonnull String jsonSchemaType, final @Nullable String jsonSchemaAirbyteType) { + List matchSchemaType = null; + // Match by Type + airbyteType + if (jsonSchemaAirbyteType != null) { + matchSchemaType = Arrays.stream(values()) + .filter(type -> jsonSchemaType.equals(type.jsonSchemaType)) + .filter(type -> jsonSchemaAirbyteType.equals(type.jsonSchemaAirbyteType)) + .toList(); + } + + // Match by Type are no results already + if (matchSchemaType == null || matchSchemaType.isEmpty()) { + matchSchemaType = + Arrays.stream(values()).filter(format -> jsonSchemaType.equals(format.jsonSchemaType) && format.jsonSchemaAirbyteType == null).toList(); + } + + if (matchSchemaType.isEmpty()) { + throw new IllegalArgumentException( + String.format("Unexpected jsonSchemaType - %s and jsonSchemaAirbyteType - %s", jsonSchemaType, jsonSchemaAirbyteType)); + } else if (matchSchemaType.size() > 1) { + throw new RuntimeException( + String.format("Match with more than one json type! Matched types : %s, Inputs jsonSchemaType : %s, jsonSchemaAirbyteType : %s", + matchSchemaType, jsonSchemaType, jsonSchemaAirbyteType)); + } else { + return matchSchemaType.get(0); + } + } + + public String getJsonSchemaType() { + return jsonSchemaType; + } + + public Schema.Type getAvroType() { + return avroType; + } + + @Override + public String toString() { + return jsonSchemaType; + } + + public String getJsonSchemaAirbyteType() { + return jsonSchemaAirbyteType; + } + +} diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/JsonToAvroSchemaConverter.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/JsonToAvroSchemaConverter.java similarity index 99% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/JsonToAvroSchemaConverter.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/JsonToAvroSchemaConverter.java index 7ecaf92cadff..d29e3f8476da 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/JsonToAvroSchemaConverter.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/JsonToAvroSchemaConverter.java @@ -2,13 +2,13 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.avro; +package io.airbyte.cdk.integrations.destination.s3.avro; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.google.common.base.Preconditions; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.integrations.base.JavaBaseConstants; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/S3AvroFormatConfig.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/S3AvroFormatConfig.java similarity index 95% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/S3AvroFormatConfig.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/S3AvroFormatConfig.java index 015bf6566f53..abd5d81df6db 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/S3AvroFormatConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/S3AvroFormatConfig.java @@ -2,11 +2,11 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.avro; +package io.airbyte.cdk.integrations.destination.s3.avro; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.destination.s3.S3Format; -import io.airbyte.integrations.destination.s3.S3FormatConfig; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; import org.apache.avro.file.CodecFactory; public class S3AvroFormatConfig implements S3FormatConfig { diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/S3AvroWriter.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/S3AvroWriter.java similarity index 88% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/S3AvroWriter.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/S3AvroWriter.java index 52afa7115cd4..4d4512e67b06 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/S3AvroWriter.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/avro/S3AvroWriter.java @@ -2,18 +2,18 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.avro; +package io.airbyte.cdk.integrations.destination.s3.avro; import alex.mojaki.s3upload.MultiPartOutputStream; import alex.mojaki.s3upload.StreamTransferManager; import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.S3Format; -import io.airbyte.integrations.destination.s3.template.S3FilenameTemplateParameterObject; -import io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory; -import io.airbyte.integrations.destination.s3.writer.BaseS3Writer; -import io.airbyte.integrations.destination.s3.writer.DestinationFileWriter; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.template.S3FilenameTemplateParameterObject; +import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; +import io.airbyte.cdk.integrations.destination.s3.writer.BaseS3Writer; +import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.io.IOException; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/constant/S3Constants.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/constant/S3Constants.java similarity index 93% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/constant/S3Constants.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/constant/S3Constants.java index d8df92788b48..8bb737e24f32 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/constant/S3Constants.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/constant/S3Constants.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.constant; +package io.airbyte.cdk.integrations.destination.s3.constant; public final class S3Constants { diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/credential/BlobStorageCredentialConfig.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/credential/BlobStorageCredentialConfig.java new file mode 100644 index 000000000000..564e85c3dc62 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/credential/BlobStorageCredentialConfig.java @@ -0,0 +1,11 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.s3.credential; + +public interface BlobStorageCredentialConfig { + + CredentialType getCredentialType(); + +} diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/credential/S3AWSDefaultProfileCredentialConfig.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/credential/S3AWSDefaultProfileCredentialConfig.java similarity index 88% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/credential/S3AWSDefaultProfileCredentialConfig.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/credential/S3AWSDefaultProfileCredentialConfig.java index 22b0a861166f..141a12d4cec5 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/credential/S3AWSDefaultProfileCredentialConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/credential/S3AWSDefaultProfileCredentialConfig.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.credential; +package io.airbyte.cdk.integrations.destination.s3.credential; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/credential/S3AccessKeyCredentialConfig.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/credential/S3AccessKeyCredentialConfig.java similarity index 94% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/credential/S3AccessKeyCredentialConfig.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/credential/S3AccessKeyCredentialConfig.java index e01967cbd40d..0f775162ca08 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/credential/S3AccessKeyCredentialConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/credential/S3AccessKeyCredentialConfig.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.credential; +package io.airbyte.cdk.integrations.destination.s3.credential; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/credential/S3CredentialConfig.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/credential/S3CredentialConfig.java similarity index 80% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/credential/S3CredentialConfig.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/credential/S3CredentialConfig.java index 4e684200d978..d85f5fa07faf 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/credential/S3CredentialConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/credential/S3CredentialConfig.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.credential; +package io.airbyte.cdk.integrations.destination.s3.credential; import com.amazonaws.auth.AWSCredentialsProvider; diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/credential/S3CredentialType.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/credential/S3CredentialType.java new file mode 100644 index 000000000000..2f65c1b98d64 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/credential/S3CredentialType.java @@ -0,0 +1,12 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.s3.credential; + +public enum S3CredentialType { + + ACCESS_KEY, + DEFAULT_PROFILE + +} diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/credential/S3InstanceProfileCredentialConfig.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/credential/S3InstanceProfileCredentialConfig.java similarity index 88% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/credential/S3InstanceProfileCredentialConfig.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/credential/S3InstanceProfileCredentialConfig.java index 9e8536918ed0..13af785fc6b5 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/credential/S3InstanceProfileCredentialConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/credential/S3InstanceProfileCredentialConfig.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.credential; +package io.airbyte.cdk.integrations.destination.s3.credential; import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.InstanceProfileCredentialsProvider; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/BaseSheetGenerator.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/BaseSheetGenerator.java similarity index 95% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/BaseSheetGenerator.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/BaseSheetGenerator.java index e813f1ad52ae..89962a61088a 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/BaseSheetGenerator.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/BaseSheetGenerator.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.csv; +package io.airbyte.cdk.integrations.destination.s3.csv; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSerializedBuffer.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/CsvSerializedBuffer.java similarity index 92% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSerializedBuffer.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/CsvSerializedBuffer.java index b535e20660f1..8555dc0d58e4 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSerializedBuffer.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/CsvSerializedBuffer.java @@ -2,12 +2,12 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.csv; +package io.airbyte.cdk.integrations.destination.s3.csv; -import io.airbyte.integrations.destination.record_buffer.BaseSerializedBuffer; -import io.airbyte.integrations.destination.record_buffer.BufferCreateFunction; -import io.airbyte.integrations.destination.record_buffer.BufferStorage; -import io.airbyte.integrations.destination.s3.util.CompressionType; +import io.airbyte.cdk.integrations.destination.record_buffer.BaseSerializedBuffer; +import io.airbyte.cdk.integrations.destination.record_buffer.BufferCreateFunction; +import io.airbyte.cdk.integrations.destination.record_buffer.BufferStorage; +import io.airbyte.cdk.integrations.destination.s3.util.CompressionType; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSheetGenerator.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/CsvSheetGenerator.java similarity index 91% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSheetGenerator.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/CsvSheetGenerator.java index 72512c26a717..3da15a08780c 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSheetGenerator.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/CsvSheetGenerator.java @@ -2,10 +2,10 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.csv; +package io.airbyte.cdk.integrations.destination.s3.csv; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.destination.s3.util.Flattening; +import io.airbyte.cdk.integrations.destination.s3.util.Flattening; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.util.List; import java.util.UUID; diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/CsvSheetGenerators.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/CsvSheetGenerators.java new file mode 100644 index 000000000000..3f65f3875196 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/CsvSheetGenerators.java @@ -0,0 +1,9 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.s3.csv; + +public class CsvSheetGenerators { + +} diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/NoFlatteningSheetGenerator.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/NoFlatteningSheetGenerator.java similarity index 87% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/NoFlatteningSheetGenerator.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/NoFlatteningSheetGenerator.java index 798bf088f65c..e37c26020aba 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/NoFlatteningSheetGenerator.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/NoFlatteningSheetGenerator.java @@ -2,12 +2,12 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.csv; +package io.airbyte.cdk.integrations.destination.s3.csv; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; import java.util.Collections; import java.util.List; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/RootLevelFlatteningSheetGenerator.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/RootLevelFlatteningSheetGenerator.java similarity index 94% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/RootLevelFlatteningSheetGenerator.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/RootLevelFlatteningSheetGenerator.java index 2d86f26d55f3..9be064aad4aa 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/RootLevelFlatteningSheetGenerator.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/RootLevelFlatteningSheetGenerator.java @@ -2,13 +2,13 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.csv; +package io.airbyte.cdk.integrations.destination.s3.csv; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.integrations.base.JavaBaseConstants; import java.util.LinkedList; import java.util.List; import java.util.stream.Collectors; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvFormatConfig.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/S3CsvFormatConfig.java similarity index 75% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvFormatConfig.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/S3CsvFormatConfig.java index 54420f3e3ca3..663e39192ace 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvFormatConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/S3CsvFormatConfig.java @@ -2,17 +2,17 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.csv; +package io.airbyte.cdk.integrations.destination.s3.csv; -import static io.airbyte.integrations.destination.s3.S3DestinationConstants.COMPRESSION_ARG_NAME; -import static io.airbyte.integrations.destination.s3.S3DestinationConstants.DEFAULT_COMPRESSION_TYPE; +import static io.airbyte.cdk.integrations.destination.s3.S3DestinationConstants.COMPRESSION_ARG_NAME; +import static io.airbyte.cdk.integrations.destination.s3.S3DestinationConstants.DEFAULT_COMPRESSION_TYPE; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.destination.s3.S3Format; -import io.airbyte.integrations.destination.s3.S3FormatConfig; -import io.airbyte.integrations.destination.s3.util.CompressionType; -import io.airbyte.integrations.destination.s3.util.CompressionTypeHelper; -import io.airbyte.integrations.destination.s3.util.Flattening; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; +import io.airbyte.cdk.integrations.destination.s3.util.CompressionType; +import io.airbyte.cdk.integrations.destination.s3.util.CompressionTypeHelper; +import io.airbyte.cdk.integrations.destination.s3.util.Flattening; import java.util.Objects; public class S3CsvFormatConfig implements S3FormatConfig { diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvWriter.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/S3CsvWriter.java similarity index 92% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvWriter.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/S3CsvWriter.java index a1d3f1db812e..294d01bbd5c9 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvWriter.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/S3CsvWriter.java @@ -2,18 +2,18 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.csv; +package io.airbyte.cdk.integrations.destination.s3.csv; import alex.mojaki.s3upload.MultiPartOutputStream; import alex.mojaki.s3upload.StreamTransferManager; import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.S3Format; -import io.airbyte.integrations.destination.s3.template.S3FilenameTemplateParameterObject; -import io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory; -import io.airbyte.integrations.destination.s3.writer.BaseS3Writer; -import io.airbyte.integrations.destination.s3.writer.DestinationFileWriter; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.template.S3FilenameTemplateParameterObject; +import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; +import io.airbyte.cdk.integrations.destination.s3.writer.BaseS3Writer; +import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.io.IOException; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/StagingDatabaseCsvSheetGenerator.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/StagingDatabaseCsvSheetGenerator.java similarity index 80% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/StagingDatabaseCsvSheetGenerator.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/StagingDatabaseCsvSheetGenerator.java index 60f34128358d..32d2e977a26d 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/StagingDatabaseCsvSheetGenerator.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/csv/StagingDatabaseCsvSheetGenerator.java @@ -2,12 +2,11 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.csv; +package io.airbyte.cdk.integrations.destination.s3.csv; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.base.TypingAndDedupingFlag; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.sql.Timestamp; import java.time.Instant; @@ -30,12 +29,16 @@ */ public class StagingDatabaseCsvSheetGenerator implements CsvSheetGenerator { - private final boolean use1s1t; + private final boolean useDestinationsV2Columns; private final List header; public StagingDatabaseCsvSheetGenerator() { - use1s1t = TypingAndDedupingFlag.isDestinationV2(); - this.header = use1s1t ? JavaBaseConstants.V2_RAW_TABLE_COLUMN_NAMES : JavaBaseConstants.LEGACY_RAW_TABLE_COLUMNS; + this(false); + } + + public StagingDatabaseCsvSheetGenerator(final boolean useDestinationsV2Columns) { + this.useDestinationsV2Columns = useDestinationsV2Columns; + this.header = this.useDestinationsV2Columns ? JavaBaseConstants.V2_RAW_TABLE_COLUMN_NAMES : JavaBaseConstants.LEGACY_RAW_TABLE_COLUMNS; } // TODO is this even used anywhere? @@ -56,7 +59,7 @@ public List getDataRow(final JsonNode formattedData) { @Override public List getDataRow(final UUID id, final String formattedString, final long emittedAt) { - if (use1s1t) { + if (useDestinationsV2Columns) { return List.of( id, Timestamp.from(Instant.ofEpochMilli(emittedAt)), diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/JsonLSerializedBuffer.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/jsonl/JsonLSerializedBuffer.java similarity index 83% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/JsonLSerializedBuffer.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/jsonl/JsonLSerializedBuffer.java index b2e05d704abd..6e901ce7a19f 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/JsonLSerializedBuffer.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/jsonl/JsonLSerializedBuffer.java @@ -2,21 +2,21 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.jsonl; +package io.airbyte.cdk.integrations.destination.s3.jsonl; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.record_buffer.BaseSerializedBuffer; +import io.airbyte.cdk.integrations.destination.record_buffer.BufferCreateFunction; +import io.airbyte.cdk.integrations.destination.record_buffer.BufferStorage; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConstants; +import io.airbyte.cdk.integrations.destination.s3.util.CompressionType; +import io.airbyte.cdk.integrations.destination.s3.util.Flattening; import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.record_buffer.BaseSerializedBuffer; -import io.airbyte.integrations.destination.record_buffer.BufferCreateFunction; -import io.airbyte.integrations.destination.record_buffer.BufferStorage; -import io.airbyte.integrations.destination.s3.S3DestinationConstants; -import io.airbyte.integrations.destination.s3.util.CompressionType; -import io.airbyte.integrations.destination.s3.util.Flattening; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/jsonl/S3JsonlFormatConfig.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/jsonl/S3JsonlFormatConfig.java new file mode 100644 index 000000000000..bc9cf84aa29f --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/jsonl/S3JsonlFormatConfig.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.s3.jsonl; + +import static io.airbyte.cdk.integrations.destination.s3.S3DestinationConstants.COMPRESSION_ARG_NAME; +import static io.airbyte.cdk.integrations.destination.s3.S3DestinationConstants.DEFAULT_COMPRESSION_TYPE; +import static io.airbyte.cdk.integrations.destination.s3.S3DestinationConstants.FLATTENING_ARG_NAME; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; +import io.airbyte.cdk.integrations.destination.s3.util.CompressionType; +import io.airbyte.cdk.integrations.destination.s3.util.CompressionTypeHelper; +import io.airbyte.cdk.integrations.destination.s3.util.Flattening; +import java.util.Objects; +import lombok.ToString; + +@ToString +public class S3JsonlFormatConfig implements S3FormatConfig { + + public static final String JSONL_SUFFIX = ".jsonl"; + + private final Flattening flattening; + + private final CompressionType compressionType; + + public S3JsonlFormatConfig(final JsonNode formatConfig) { + this( + formatConfig.has(FLATTENING_ARG_NAME) + ? Flattening.fromValue(formatConfig.get(FLATTENING_ARG_NAME).asText()) + : Flattening.NO, + formatConfig.has(COMPRESSION_ARG_NAME) + ? CompressionTypeHelper.parseCompressionType(formatConfig.get(COMPRESSION_ARG_NAME)) + : DEFAULT_COMPRESSION_TYPE); + } + + public S3JsonlFormatConfig(final Flattening flattening, final CompressionType compressionType) { + this.flattening = flattening; + this.compressionType = compressionType; + } + + @Override + public S3Format getFormat() { + return S3Format.JSONL; + } + + @Override + public String getFileExtension() { + return JSONL_SUFFIX + compressionType.getFileExtension(); + } + + public CompressionType getCompressionType() { + return compressionType; + } + + public Flattening getFlatteningType() { + return flattening; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final S3JsonlFormatConfig that = (S3JsonlFormatConfig) o; + return flattening == that.flattening + && Objects.equals(compressionType, that.compressionType); + } + + @Override + public int hashCode() { + return Objects.hash(flattening, compressionType); + } + +} diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/S3JsonlWriter.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/jsonl/S3JsonlWriter.java similarity index 86% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/S3JsonlWriter.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/jsonl/S3JsonlWriter.java index d7c27e0211a5..c4e96533486a 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/S3JsonlWriter.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/jsonl/S3JsonlWriter.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.jsonl; +package io.airbyte.cdk.integrations.destination.s3.jsonl; import alex.mojaki.s3upload.MultiPartOutputStream; import alex.mojaki.s3upload.StreamTransferManager; @@ -10,15 +10,15 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.template.S3FilenameTemplateParameterObject; +import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; +import io.airbyte.cdk.integrations.destination.s3.writer.BaseS3Writer; +import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter; import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.S3Format; -import io.airbyte.integrations.destination.s3.template.S3FilenameTemplateParameterObject; -import io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory; -import io.airbyte.integrations.destination.s3.writer.BaseS3Writer; -import io.airbyte.integrations.destination.s3.writer.DestinationFileWriter; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.io.IOException; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBuffer.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/parquet/ParquetSerializedBuffer.java similarity index 87% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBuffer.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/parquet/ParquetSerializedBuffer.java index 67896c2de35c..f33778d751b7 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBuffer.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/parquet/ParquetSerializedBuffer.java @@ -2,17 +2,17 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.parquet; +package io.airbyte.cdk.integrations.destination.s3.parquet; import static org.apache.parquet.avro.AvroWriteSupport.WRITE_OLD_LIST_STRUCTURE; -import io.airbyte.integrations.destination.record_buffer.BufferCreateFunction; -import io.airbyte.integrations.destination.record_buffer.FileBuffer; -import io.airbyte.integrations.destination.record_buffer.SerializableBuffer; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.avro.AvroConstants; -import io.airbyte.integrations.destination.s3.avro.AvroRecordFactory; -import io.airbyte.integrations.destination.s3.avro.JsonToAvroSchemaConverter; +import io.airbyte.cdk.integrations.destination.record_buffer.BufferCreateFunction; +import io.airbyte.cdk.integrations.destination.record_buffer.FileBuffer; +import io.airbyte.cdk.integrations.destination.record_buffer.SerializableBuffer; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.avro.AvroConstants; +import io.airbyte.cdk.integrations.destination.s3.avro.AvroRecordFactory; +import io.airbyte.cdk.integrations.destination.s3.avro.JsonToAvroSchemaConverter; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; @@ -35,9 +35,9 @@ import org.slf4j.LoggerFactory; /** - * The {@link io.airbyte.integrations.destination.record_buffer.BaseSerializedBuffer} class - * abstracts the {@link io.airbyte.integrations.destination.record_buffer.BufferStorage} from the - * details of the format the data is going to be stored in. + * The {@link io.airbyte.cdk.integrations.destination.record_buffer.BaseSerializedBuffer} class + * abstracts the {@link io.airbyte.cdk.integrations.destination.record_buffer.BufferStorage} from + * the details of the format the data is going to be stored in. *

      * Unfortunately, the Parquet library doesn't allow us to manipulate the output stream and forces us * to go through {@link HadoopOutputFile} instead. So we can't benefit from the abstraction diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetConstants.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/parquet/S3ParquetConstants.java similarity index 90% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetConstants.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/parquet/S3ParquetConstants.java index a65e2047adc7..6a6ccb948603 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetConstants.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/parquet/S3ParquetConstants.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.parquet; +package io.airbyte.cdk.integrations.destination.s3.parquet; import org.apache.parquet.hadoop.metadata.CompressionCodecName; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetFormatConfig.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/parquet/S3ParquetFormatConfig.java similarity index 93% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetFormatConfig.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/parquet/S3ParquetFormatConfig.java index 6be4c9c8328a..e7e14a363434 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetFormatConfig.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/parquet/S3ParquetFormatConfig.java @@ -2,11 +2,11 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.parquet; +package io.airbyte.cdk.integrations.destination.s3.parquet; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.destination.s3.S3Format; -import io.airbyte.integrations.destination.s3.S3FormatConfig; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; import org.apache.parquet.hadoop.metadata.CompressionCodecName; public class S3ParquetFormatConfig implements S3FormatConfig { diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetWriter.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/parquet/S3ParquetWriter.java similarity index 90% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetWriter.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/parquet/S3ParquetWriter.java index 420a959f73ac..91d940f08db8 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetWriter.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/parquet/S3ParquetWriter.java @@ -2,19 +2,19 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.parquet; +package io.airbyte.cdk.integrations.destination.s3.parquet; import static org.apache.parquet.avro.AvroWriteSupport.WRITE_OLD_LIST_STRUCTURE; import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.S3Format; -import io.airbyte.integrations.destination.s3.avro.AvroRecordFactory; -import io.airbyte.integrations.destination.s3.credential.S3AccessKeyCredentialConfig; -import io.airbyte.integrations.destination.s3.template.S3FilenameTemplateParameterObject; -import io.airbyte.integrations.destination.s3.writer.BaseS3Writer; -import io.airbyte.integrations.destination.s3.writer.DestinationFileWriter; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.avro.AvroRecordFactory; +import io.airbyte.cdk.integrations.destination.s3.credential.S3AccessKeyCredentialConfig; +import io.airbyte.cdk.integrations.destination.s3.template.S3FilenameTemplateParameterObject; +import io.airbyte.cdk.integrations.destination.s3.writer.BaseS3Writer; +import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.io.IOException; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/template/S3FilenameTemplateManager.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/template/S3FilenameTemplateManager.java similarity index 97% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/template/S3FilenameTemplateManager.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/template/S3FilenameTemplateManager.java index b9a31b8a51f5..73058c75cf55 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/template/S3FilenameTemplateManager.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/template/S3FilenameTemplateManager.java @@ -2,12 +2,12 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.template; +package io.airbyte.cdk.integrations.destination.s3.template; import static java.util.Optional.ofNullable; import static org.apache.commons.lang3.StringUtils.EMPTY; -import io.airbyte.integrations.destination.s3.S3DestinationConstants; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConstants; import java.io.IOException; import java.text.DateFormat; import java.text.SimpleDateFormat; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/template/S3FilenameTemplateParameterObject.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/template/S3FilenameTemplateParameterObject.java similarity index 96% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/template/S3FilenameTemplateParameterObject.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/template/S3FilenameTemplateParameterObject.java index 817801e50c3d..ca270a9f91b4 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/template/S3FilenameTemplateParameterObject.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/template/S3FilenameTemplateParameterObject.java @@ -2,10 +2,10 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.template; +package io.airbyte.cdk.integrations.destination.s3.template; -import io.airbyte.integrations.destination.record_buffer.SerializableBuffer; -import io.airbyte.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.record_buffer.SerializableBuffer; +import io.airbyte.cdk.integrations.destination.s3.S3Format; import java.sql.Timestamp; import java.util.Objects; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/AvroRecordHelper.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/AvroRecordHelper.java similarity index 85% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/AvroRecordHelper.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/AvroRecordHelper.java index c21a1557dce4..5b24b92ace71 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/AvroRecordHelper.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/AvroRecordHelper.java @@ -2,14 +2,14 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.util; +package io.airbyte.cdk.integrations.destination.s3.util; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.s3.avro.JsonFieldNameUpdater; +import io.airbyte.cdk.integrations.destination.s3.avro.JsonToAvroSchemaConverter; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.s3.avro.JsonFieldNameUpdater; -import io.airbyte.integrations.destination.s3.avro.JsonToAvroSchemaConverter; /** * Helper methods for unit tests. This is needed by multiple modules, so it is in the src directory. diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/CompressionType.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/CompressionType.java similarity index 85% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/CompressionType.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/CompressionType.java index 997a81759e44..8fc79df5a74e 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/CompressionType.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/CompressionType.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.util; +package io.airbyte.cdk.integrations.destination.s3.util; public enum CompressionType { diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/CompressionTypeHelper.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/CompressionTypeHelper.java new file mode 100644 index 000000000000..0963a1ff63a3 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/CompressionTypeHelper.java @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.s3.util; + +import static io.airbyte.cdk.integrations.destination.s3.S3DestinationConstants.COMPRESSION_TYPE_ARG_NAME; +import static io.airbyte.cdk.integrations.destination.s3.S3DestinationConstants.DEFAULT_COMPRESSION_TYPE; + +import com.fasterxml.jackson.databind.JsonNode; + +public class CompressionTypeHelper { + + private CompressionTypeHelper() {} + + /** + * Sample expected input: { "compression_type": "No Compression" } + */ + public static CompressionType parseCompressionType(final JsonNode compressionConfig) { + if (compressionConfig == null || compressionConfig.isNull()) { + return DEFAULT_COMPRESSION_TYPE; + } + final String compressionType = compressionConfig.get(COMPRESSION_TYPE_ARG_NAME).asText(); + if (compressionType.toUpperCase().equals(CompressionType.GZIP.name())) { + return CompressionType.GZIP; + } else { + return CompressionType.NO_COMPRESSION; + } + } + +} diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/Flattening.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/Flattening.java similarity index 91% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/Flattening.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/Flattening.java index ace6c681fd83..57248ef4f1da 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/Flattening.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/Flattening.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.util; +package io.airbyte.cdk.integrations.destination.s3.util; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/JavaProcessRunner.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/JavaProcessRunner.java similarity index 94% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/JavaProcessRunner.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/JavaProcessRunner.java index 200eb4323586..4c6e92fd647e 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/JavaProcessRunner.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/JavaProcessRunner.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.util; +package io.airbyte.cdk.integrations.destination.s3.util; import io.airbyte.commons.io.LineGobbler; import java.io.File; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/S3NameTransformer.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/S3NameTransformer.java similarity index 86% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/S3NameTransformer.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/S3NameTransformer.java index 72c229524709..96d5377a2f45 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/S3NameTransformer.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/S3NameTransformer.java @@ -2,9 +2,9 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.util; +package io.airbyte.cdk.integrations.destination.s3.util; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import java.text.Normalizer; import java.util.regex.Pattern; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/S3OutputPathHelper.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/S3OutputPathHelper.java similarity index 86% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/S3OutputPathHelper.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/S3OutputPathHelper.java index ad3c00266a41..147bcd577b09 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/S3OutputPathHelper.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/S3OutputPathHelper.java @@ -2,9 +2,9 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.util; +package io.airbyte.cdk.integrations.destination.s3.util; -import static io.airbyte.integrations.destination.s3.S3DestinationConstants.NAME_TRANSFORMER; +import static io.airbyte.cdk.integrations.destination.s3.S3DestinationConstants.NAME_TRANSFORMER; import io.airbyte.protocol.models.v0.AirbyteStream; import java.util.LinkedList; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/StreamTransferManagerFactory.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/StreamTransferManagerFactory.java similarity index 98% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/StreamTransferManagerFactory.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/StreamTransferManagerFactory.java index 2ab37bb4334e..3738b0a416aa 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/StreamTransferManagerFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/StreamTransferManagerFactory.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.util; +package io.airbyte.cdk.integrations.destination.s3.util; import alex.mojaki.s3upload.StreamTransferManager; import com.amazonaws.services.s3.AmazonS3; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/StreamTransferManagerWithMetadata.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/StreamTransferManagerWithMetadata.java similarity index 96% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/StreamTransferManagerWithMetadata.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/StreamTransferManagerWithMetadata.java index cbc42c040fda..968a35a3a9e9 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/StreamTransferManagerWithMetadata.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/util/StreamTransferManagerWithMetadata.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.util; +package io.airbyte.cdk.integrations.destination.s3.util; import alex.mojaki.s3upload.StreamTransferManager; import com.amazonaws.services.s3.AmazonS3; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/writer/BaseS3Writer.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/writer/BaseS3Writer.java similarity index 92% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/writer/BaseS3Writer.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/writer/BaseS3Writer.java index be83e5a37310..97426c9043f5 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/writer/BaseS3Writer.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/writer/BaseS3Writer.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.writer; +package io.airbyte.cdk.integrations.destination.s3.writer; import static org.apache.commons.lang3.StringUtils.isNotBlank; @@ -11,11 +11,11 @@ import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; import com.amazonaws.services.s3.model.DeleteObjectsResult; import com.amazonaws.services.s3.model.S3ObjectSummary; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.S3DestinationConstants; -import io.airbyte.integrations.destination.s3.template.S3FilenameTemplateManager; -import io.airbyte.integrations.destination.s3.template.S3FilenameTemplateParameterObject; -import io.airbyte.integrations.destination.s3.util.S3OutputPathHelper; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConstants; +import io.airbyte.cdk.integrations.destination.s3.template.S3FilenameTemplateManager; +import io.airbyte.cdk.integrations.destination.s3.template.S3FilenameTemplateParameterObject; +import io.airbyte.cdk.integrations.destination.s3.util.S3OutputPathHelper; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/writer/DestinationFileWriter.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/writer/DestinationFileWriter.java new file mode 100644 index 000000000000..8006127a21bb --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/writer/DestinationFileWriter.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.s3.writer; + +import io.airbyte.cdk.integrations.destination.s3.S3Format; + +public interface DestinationFileWriter extends DestinationWriter { + + String getFileLocation(); + + S3Format getFileFormat(); + + String getOutputPath(); + +} diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/writer/DestinationWriter.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/writer/DestinationWriter.java new file mode 100644 index 000000000000..20fa6b926c73 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/writer/DestinationWriter.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.s3.writer; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.protocol.models.Jsons; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import java.io.IOException; +import java.util.UUID; + +/** + * {@link DestinationWriter} is responsible for writing Airbyte stream data to an S3 location in a + * specific format. + */ +public interface DestinationWriter { + + /** + * Prepare an S3 writer for the stream. + */ + void initialize() throws IOException; + + /** + * Write an Airbyte record message to an S3 object. + */ + void write(UUID id, AirbyteRecordMessage recordMessage) throws IOException; + + void write(JsonNode formattedData) throws IOException; + + default void write(String formattedData) throws IOException { + write(Jsons.deserialize(formattedData)); + } + + /** + * Close the S3 writer for the stream. + */ + void close(boolean hasFailed) throws IOException; + + default void closeAfterPush() throws IOException { + close(false); + } + +} diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/writer/ProductionWriterFactory.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/writer/ProductionWriterFactory.java similarity index 77% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/writer/ProductionWriterFactory.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/writer/ProductionWriterFactory.java index 13552903116d..afe16c137978 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/writer/ProductionWriterFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/writer/ProductionWriterFactory.java @@ -2,17 +2,17 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.writer; +package io.airbyte.cdk.integrations.destination.s3.writer; import com.amazonaws.services.s3.AmazonS3; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.S3Format; -import io.airbyte.integrations.destination.s3.avro.AvroConstants; -import io.airbyte.integrations.destination.s3.avro.JsonToAvroSchemaConverter; -import io.airbyte.integrations.destination.s3.avro.S3AvroWriter; -import io.airbyte.integrations.destination.s3.csv.S3CsvWriter; -import io.airbyte.integrations.destination.s3.jsonl.S3JsonlWriter; -import io.airbyte.integrations.destination.s3.parquet.S3ParquetWriter; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.avro.AvroConstants; +import io.airbyte.cdk.integrations.destination.s3.avro.JsonToAvroSchemaConverter; +import io.airbyte.cdk.integrations.destination.s3.avro.S3AvroWriter; +import io.airbyte.cdk.integrations.destination.s3.csv.S3CsvWriter; +import io.airbyte.cdk.integrations.destination.s3.jsonl.S3JsonlWriter; +import io.airbyte.cdk.integrations.destination.s3.parquet.S3ParquetWriter; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.sql.Timestamp; diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/writer/S3WriterFactory.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/writer/S3WriterFactory.java similarity index 82% rename from airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/writer/S3WriterFactory.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/writer/S3WriterFactory.java index 09c03fb08e10..1855f1dd0e24 100644 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/writer/S3WriterFactory.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/s3/writer/S3WriterFactory.java @@ -2,10 +2,10 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.writer; +package io.airbyte.cdk.integrations.destination.s3.writer; import com.amazonaws.services.s3.AmazonS3; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.sql.Timestamp; diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/AsyncFlush.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/AsyncFlush.java similarity index 78% rename from airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/AsyncFlush.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/AsyncFlush.java index 645280485b28..564e3d3ade85 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/AsyncFlush.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/java/io/airbyte/cdk/integrations/destination/staging/AsyncFlush.java @@ -2,18 +2,18 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.staging; +package io.airbyte.cdk.integrations.destination.staging; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.jdbc.WriteConfig; +import io.airbyte.cdk.integrations.destination.record_buffer.FileBuffer; +import io.airbyte.cdk.integrations.destination.s3.csv.CsvSerializedBuffer; +import io.airbyte.cdk.integrations.destination.s3.csv.StagingDatabaseCsvSheetGenerator; +import io.airbyte.cdk.integrations.destination_async.DestinationFlushFunction; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeOperationValve; import io.airbyte.integrations.base.destination.typing_deduping.TyperDeduper; -import io.airbyte.integrations.destination.jdbc.WriteConfig; -import io.airbyte.integrations.destination.record_buffer.FileBuffer; -import io.airbyte.integrations.destination.s3.csv.CsvSerializedBuffer; -import io.airbyte.integrations.destination.s3.csv.StagingDatabaseCsvSheetGenerator; -import io.airbyte.integrations.destination_async.DestinationFlushFunction; -import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.StreamDescriptor; import java.util.List; @@ -35,14 +35,16 @@ class AsyncFlush implements DestinationFlushFunction { private final TypeAndDedupeOperationValve typerDeduperValve; private final TyperDeduper typerDeduper; private final long optimalBatchSizeBytes; + private final boolean useDestinationsV2Columns; public AsyncFlush(final Map streamDescToWriteConfig, final StagingOperations stagingOperations, final JdbcDatabase database, final ConfiguredAirbyteCatalog catalog, final TypeAndDedupeOperationValve typerDeduperValve, - final TyperDeduper typerDeduper) { - this(streamDescToWriteConfig, stagingOperations, database, catalog, typerDeduperValve, typerDeduper, 50 * 1024 * 1024); + final TyperDeduper typerDeduper, + final boolean useDestinationsV2Columns) { + this(streamDescToWriteConfig, stagingOperations, database, catalog, typerDeduperValve, typerDeduper, 50 * 1024 * 1024, useDestinationsV2Columns); } public AsyncFlush(final Map streamDescToWriteConfig, @@ -56,7 +58,8 @@ public AsyncFlush(final Map streamDescToWriteConf // resource the connector will usually at most fill up around 150 MB in a single queue. By lowering // the batch size, the AsyncFlusher will flush in smaller batches which allows for memory to be // freed earlier similar to a sliding window effect - long optimalBatchSizeBytes) { + final long optimalBatchSizeBytes, + final boolean useDestinationsV2Columns) { this.streamDescToWriteConfig = streamDescToWriteConfig; this.stagingOperations = stagingOperations; this.database = database; @@ -64,6 +67,7 @@ public AsyncFlush(final Map streamDescToWriteConf this.typerDeduperValve = typerDeduperValve; this.typerDeduper = typerDeduper; this.optimalBatchSizeBytes = optimalBatchSizeBytes; + this.useDestinationsV2Columns = useDestinationsV2Columns; } @Override @@ -72,7 +76,7 @@ public void flush(final StreamDescriptor decs, final Stream outputRecordCollector, + final JdbcDatabase database, + final StagingOperations stagingOperations, + final NamingConventionTransformer namingResolver, + final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final boolean purgeStagingData, + final TypeAndDedupeOperationValve typerDeduperValve, + final TyperDeduper typerDeduper, + final ParsedCatalog parsedCatalog, + final String defaultNamespace, + final boolean useDestinationsV2Columns) { + return createAsync(outputRecordCollector, + database, + stagingOperations, + namingResolver, + config, + catalog, + purgeStagingData, + typerDeduperValve, + typerDeduper, + parsedCatalog, + defaultNamespace, + useDestinationsV2Columns, + Optional.empty()); + } + + public SerializedAirbyteMessageConsumer createAsync(final Consumer outputRecordCollector, + final JdbcDatabase database, + final StagingOperations stagingOperations, + final NamingConventionTransformer namingResolver, + final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final boolean purgeStagingData, + final TypeAndDedupeOperationValve typerDeduperValve, + final TyperDeduper typerDeduper, + final ParsedCatalog parsedCatalog, + final String defaultNamespace, + final boolean useDestinationsV2Columns, + final Optional bufferMemoryLimit) { + final List writeConfigs = createWriteConfigs(namingResolver, config, catalog, parsedCatalog, useDestinationsV2Columns); + final var streamDescToWriteConfig = streamDescToWriteConfig(writeConfigs); + final var flusher = + new AsyncFlush(streamDescToWriteConfig, stagingOperations, database, catalog, typerDeduperValve, typerDeduper, useDestinationsV2Columns); + return new AsyncStreamConsumer( + outputRecordCollector, + GeneralStagingFunctions.onStartFunction(database, stagingOperations, writeConfigs, typerDeduper), + // todo (cgardens) - wrapping the old close function to avoid more code churn. + (hasFailed, streamSyncSummaries) -> { + try { + GeneralStagingFunctions.onCloseFunction( + database, + stagingOperations, + writeConfigs, + purgeStagingData, + typerDeduper).accept(false, streamSyncSummaries); + } catch (final Exception e) { + throw new RuntimeException(e); + } + }, + flusher, + catalog, + new BufferManager(getMemoryLimit(bufferMemoryLimit)), + defaultNamespace); + } + + private static long getMemoryLimit(final Optional bufferMemoryLimit) { + return bufferMemoryLimit.orElse((long) (Runtime.getRuntime().maxMemory() * MEMORY_LIMIT_RATIO)); + } + + private static Map streamDescToWriteConfig(final List writeConfigs) { + final Set conflictingStreams = new HashSet<>(); + final Map streamDescToWriteConfig = new HashMap<>(); + for (final WriteConfig config : writeConfigs) { + final StreamDescriptor streamIdentifier = toStreamDescriptor(config); + if (streamDescToWriteConfig.containsKey(streamIdentifier)) { + conflictingStreams.add(config); + final WriteConfig existingConfig = streamDescToWriteConfig.get(streamIdentifier); + // The first conflicting stream won't have any problems, so we need to explicitly add it here. + conflictingStreams.add(existingConfig); + } else { + streamDescToWriteConfig.put(streamIdentifier, config); + } + } + if (!conflictingStreams.isEmpty()) { + final String message = String.format( + "You are trying to write multiple streams to the same table. Consider switching to a custom namespace format using ${SOURCE_NAMESPACE}, or moving one of them into a separate connection with a different stream prefix. Affected streams: %s", + conflictingStreams.stream().map(config -> config.getNamespace() + "." + config.getStreamName()).collect(joining(", "))); + throw new ConfigErrorException(message); + } + return streamDescToWriteConfig; + } + + private static StreamDescriptor toStreamDescriptor(final WriteConfig config) { + return new StreamDescriptor().withName(config.getStreamName()).withNamespace(config.getNamespace()); + } + + /** + * Creates a list of all {@link WriteConfig} for each stream within a + * {@link ConfiguredAirbyteCatalog}. Each write config represents the configuration settings for + * writing to a destination connector + * + * @param namingResolver {@link NamingConventionTransformer} used to transform names that are + * acceptable by each destination connector + * @param config destination connector configuration parameters + * @param catalog {@link ConfiguredAirbyteCatalog} collection of configured + * {@link ConfiguredAirbyteStream} + * @return list of all write configs for each stream in a {@link ConfiguredAirbyteCatalog} + */ + private static List createWriteConfigs(final NamingConventionTransformer namingResolver, + final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final ParsedCatalog parsedCatalog, + final boolean useDestinationsV2Columns) { + + return catalog.getStreams().stream().map(toWriteConfig(namingResolver, config, parsedCatalog, useDestinationsV2Columns)).collect(toList()); + } + + private static Function toWriteConfig(final NamingConventionTransformer namingResolver, + final JsonNode config, + final ParsedCatalog parsedCatalog, + final boolean useDestinationsV2Columns) { + return stream -> { + Preconditions.checkNotNull(stream.getDestinationSyncMode(), "Undefined destination sync mode"); + final AirbyteStream abStream = stream.getStream(); + final String streamName = abStream.getName(); + + final String outputSchema; + final String tableName; + if (useDestinationsV2Columns) { + final StreamId streamId = parsedCatalog.getStream(abStream.getNamespace(), streamName).id(); + outputSchema = streamId.rawNamespace(); + tableName = streamId.rawName(); + } else { + outputSchema = getOutputSchema(abStream, config.get("schema").asText(), namingResolver); + tableName = namingResolver.getRawTableName(streamName); + } + final String tmpTableName = namingResolver.getTmpTableName(streamName); + final DestinationSyncMode syncMode = stream.getDestinationSyncMode(); + + final WriteConfig writeConfig = + new WriteConfig(streamName, abStream.getNamespace(), outputSchema, tmpTableName, tableName, syncMode, SYNC_DATETIME); + LOGGER.info("Write config: {}", writeConfig); + + return writeConfig; + }; + } + + private static String getOutputSchema(final AirbyteStream stream, + final String defaultDestSchema, + final NamingConventionTransformer namingResolver) { + return stream.getNamespace() != null + ? namingResolver.getNamespace(stream.getNamespace()) + : namingResolver.getNamespace(defaultDestSchema); + } + +} diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3CopyConfigTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3CopyConfigTest.java similarity index 93% rename from airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3CopyConfigTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3CopyConfigTest.java index 6ffbcb9fc050..36efcfdb2656 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3CopyConfigTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3CopyConfigTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.jdbc.copy.s3; +package io.airbyte.cdk.integrations.destination.jdbc.copy.s3; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopierTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3StreamCopierTest.java similarity index 93% rename from airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopierTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3StreamCopierTest.java index 5c375a0bc6d9..770643e875e4 100644 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/copy/s3/S3StreamCopierTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/jdbc/copy/s3/S3StreamCopierTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.jdbc.copy.s3; +package io.airbyte.cdk.integrations.destination.jdbc.copy.s3; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -15,17 +15,17 @@ import com.amazonaws.services.s3.AmazonS3Client; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.DestinationConfig; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.csv.CsvSheetGenerator; +import io.airbyte.cdk.integrations.destination.s3.csv.S3CsvFormatConfig; +import io.airbyte.cdk.integrations.destination.s3.csv.S3CsvWriter; +import io.airbyte.cdk.integrations.destination.s3.csv.StagingDatabaseCsvSheetGenerator; +import io.airbyte.cdk.integrations.destination.s3.util.CompressionType; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.DestinationConfig; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.csv.CsvSheetGenerator; -import io.airbyte.integrations.destination.s3.csv.S3CsvFormatConfig; -import io.airbyte.integrations.destination.s3.csv.S3CsvWriter; -import io.airbyte.integrations.destination.s3.csv.StagingDatabaseCsvSheetGenerator; -import io.airbyte.integrations.destination.s3.util.CompressionType; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/AesCbcEnvelopeEncryptionBlobDecoratorTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/AesCbcEnvelopeEncryptionBlobDecoratorTest.java similarity index 98% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/AesCbcEnvelopeEncryptionBlobDecoratorTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/AesCbcEnvelopeEncryptionBlobDecoratorTest.java index b6271188fdf5..15e30ec92d6e 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/AesCbcEnvelopeEncryptionBlobDecoratorTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/AesCbcEnvelopeEncryptionBlobDecoratorTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import java.io.ByteArrayOutputStream; import java.io.IOException; diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/BlobDecoratorTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/BlobDecoratorTest.java similarity index 96% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/BlobDecoratorTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/BlobDecoratorTest.java index 053612949bdb..54c74060569c 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/BlobDecoratorTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/BlobDecoratorTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/S3BaseChecksTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/S3BaseChecksTest.java similarity index 96% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/S3BaseChecksTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/S3BaseChecksTest.java index 1a7990f493c5..dee0146ed431 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/S3BaseChecksTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/S3BaseChecksTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; @@ -14,7 +14,7 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.ListObjectsRequest; -import io.airbyte.integrations.destination.s3.util.S3NameTransformer; +import io.airbyte.cdk.integrations.destination.s3.util.S3NameTransformer; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentMatchers; diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/S3DestinationConfigTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/S3DestinationConfigTest.java similarity index 97% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/S3DestinationConfigTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/S3DestinationConfigTest.java index ebab6a99b79b..b6166cfbc687 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/S3DestinationConfigTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/S3DestinationConfigTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -10,8 +10,8 @@ import com.amazonaws.auth.AWSCredentials; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.s3.credential.S3AccessKeyCredentialConfig; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.s3.credential.S3AccessKeyCredentialConfig; import org.junit.jupiter.api.Test; class S3DestinationConfigTest { diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/S3FormatConfigsTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/S3FormatConfigsTest.java similarity index 83% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/S3FormatConfigsTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/S3FormatConfigsTest.java index 2bf24acc30b7..0b921efdefb2 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/S3FormatConfigsTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/S3FormatConfigsTest.java @@ -2,16 +2,16 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.s3.csv.S3CsvFormatConfig; +import io.airbyte.cdk.integrations.destination.s3.util.CompressionType; +import io.airbyte.cdk.integrations.destination.s3.util.Flattening; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.s3.csv.S3CsvFormatConfig; -import io.airbyte.integrations.destination.s3.util.CompressionType; -import io.airbyte.integrations.destination.s3.util.Flattening; import java.util.Map; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/S3StorageOperationsTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/S3StorageOperationsTest.java new file mode 100644 index 000000000000..138e9d393ce5 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/S3StorageOperationsTest.java @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.s3; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.DeleteObjectsRequest; +import com.amazonaws.services.s3.model.ListObjectsRequest; +import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.s3.util.S3NameTransformer; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; +import org.joda.time.DateTime; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +public class S3StorageOperationsTest { + + private static final String BUCKET_NAME = "fake-bucket"; + private static final String FAKE_BUCKET_PATH = "fake-bucketPath"; + private static final String NAMESPACE = "namespace"; + private static final String STREAM_NAME = "stream_name1"; + private static final String OBJECT_TO_DELETE = NAMESPACE + "/" + STREAM_NAME + "/2022_04_04_123456789_0.csv.gz"; + private AmazonS3 s3Client; + private S3StorageOperations s3StorageOperations; + + @BeforeEach + public void setup() { + final NamingConventionTransformer nameTransformer = new S3NameTransformer(); + s3Client = mock(AmazonS3.class); + + final S3ObjectSummary objectSummary1 = mock(S3ObjectSummary.class); + final S3ObjectSummary objectSummary2 = mock(S3ObjectSummary.class); + final S3ObjectSummary objectSummary3 = mock(S3ObjectSummary.class); + when(objectSummary1.getKey()).thenReturn(OBJECT_TO_DELETE); + when(objectSummary2.getKey()).thenReturn(NAMESPACE + "/stream_name2/2022_04_04_123456789_0.csv.gz"); + when(objectSummary3.getKey()).thenReturn("other_files.txt"); + + final ObjectListing results = mock(ObjectListing.class); + when(results.isTruncated()).thenReturn(false); + when(results.getObjectSummaries()).thenReturn(List.of(objectSummary1, objectSummary2, objectSummary3)); + when(s3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(results); + + final S3DestinationConfig s3Config = S3DestinationConfig.create(BUCKET_NAME, FAKE_BUCKET_PATH, "fake-region") + .withEndpoint("fake-endpoint") + .withAccessKeyCredential("fake-accessKeyId", "fake-secretAccessKey") + .withS3Client(s3Client) + .get(); + s3StorageOperations = new S3StorageOperations(nameTransformer, s3Client, s3Config); + } + + @Test + void testRegexMatch() { + final Pattern regexFormat = + Pattern.compile(s3StorageOperations.getRegexFormat(NAMESPACE, STREAM_NAME, S3DestinationConstants.DEFAULT_PATH_FORMAT)); + assertTrue(regexFormat.matcher(OBJECT_TO_DELETE).matches()); + assertTrue(regexFormat + .matcher(s3StorageOperations.getBucketObjectPath(NAMESPACE, STREAM_NAME, DateTime.now(), S3DestinationConstants.DEFAULT_PATH_FORMAT)) + .matches()); + assertFalse(regexFormat.matcher(NAMESPACE + "/" + STREAM_NAME + "/some_random_file_0.doc").matches()); + assertFalse(regexFormat.matcher(NAMESPACE + "/stream_name2/2022_04_04_123456789_0.csv.gz").matches()); + } + + @Test + void testCustomRegexMatch() { + final String customFormat = "${NAMESPACE}_${STREAM_NAME}_${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}-${SECOND}-${MILLISECOND}-${EPOCH}-${UUID}"; + assertTrue(Pattern + .compile(s3StorageOperations.getRegexFormat(NAMESPACE, STREAM_NAME, customFormat)) + .matcher(s3StorageOperations.getBucketObjectPath(NAMESPACE, STREAM_NAME, DateTime.now(), customFormat)).matches()); + } + + @Test + void testGetExtension() { + assertEquals(".csv.gz", S3StorageOperations.getExtension("test.csv.gz")); + assertEquals(".gz", S3StorageOperations.getExtension("test.gz")); + assertEquals(".avro", S3StorageOperations.getExtension("test.avro")); + assertEquals("", S3StorageOperations.getExtension("test-file")); + } + + @Test + void testCleanUpBucketObject() { + final String pathFormat = S3DestinationConstants.DEFAULT_PATH_FORMAT; + s3StorageOperations.cleanUpBucketObject(NAMESPACE, STREAM_NAME, FAKE_BUCKET_PATH, pathFormat); + final ArgumentCaptor deleteRequest = ArgumentCaptor.forClass(DeleteObjectsRequest.class); + verify(s3Client).deleteObjects(deleteRequest.capture()); + assertEquals(1, deleteRequest.getValue().getKeys().size()); + assertEquals(OBJECT_TO_DELETE, deleteRequest.getValue().getKeys().get(0).getKey()); + } + + @Test + void testGetFilename() { + assertEquals("filename", S3StorageOperations.getFilename("filename")); + assertEquals("filename", S3StorageOperations.getFilename("/filename")); + assertEquals("filename", S3StorageOperations.getFilename("/p1/p2/filename")); + assertEquals("filename.csv", S3StorageOperations.getFilename("/p1/p2/filename.csv")); + } + + @Test + void getPartId() throws InterruptedException { + + // Multithreaded utility class + class PartIdGetter implements Runnable { + + final List responses = new ArrayList<>(); + final S3StorageOperations s3StorageOperations; + + PartIdGetter(S3StorageOperations instance) { + s3StorageOperations = instance; + } + + public void run() { + responses.add(s3StorageOperations.getPartId(FAKE_BUCKET_PATH)); + } + + List getResponses() { + return responses; + } + + } + + PartIdGetter partIdGetter = new PartIdGetter(s3StorageOperations); + + // single threaded + partIdGetter.run(); // 0 + partIdGetter.run(); // 1 + partIdGetter.run(); // 2 + + // multithreaded + ExecutorService executor = Executors.newFixedThreadPool(3); + for (int i = 0; i < 7; i++) { + executor.execute(partIdGetter); + } + executor.shutdown(); + executor.awaitTermination(5, TimeUnit.SECONDS); + + List responses = partIdGetter.getResponses(); + assertEquals(10, responses.size()); + for (int i = 0; i <= 9; i++) { + assertTrue(responses.contains(Integer.toString(i))); + } + } + + @Test + void getPartIdMultiplePaths() { + assertEquals("0", s3StorageOperations.getPartId(FAKE_BUCKET_PATH)); + assertEquals("1", s3StorageOperations.getPartId(FAKE_BUCKET_PATH)); + + assertEquals("0", s3StorageOperations.getPartId("other_path")); + } + +} diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/AvroNameTransformerTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/avro/AvroNameTransformerTest.java similarity index 96% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/AvroNameTransformerTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/avro/AvroNameTransformerTest.java index f1bc2ebc97b4..fa5d28c36c1d 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/AvroNameTransformerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/avro/AvroNameTransformerTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.avro; +package io.airbyte.cdk.integrations.destination.s3.avro; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/AvroSerializedBufferTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/avro/AvroSerializedBufferTest.java similarity index 88% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/AvroSerializedBufferTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/avro/AvroSerializedBufferTest.java index 334fae868442..33faf55f8369 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/AvroSerializedBufferTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/avro/AvroSerializedBufferTest.java @@ -2,18 +2,18 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.avro; +package io.airbyte.cdk.integrations.destination.s3.avro; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.base.DestinationConfig; +import io.airbyte.cdk.integrations.destination.record_buffer.BufferStorage; +import io.airbyte.cdk.integrations.destination.record_buffer.FileBuffer; +import io.airbyte.cdk.integrations.destination.record_buffer.InMemoryBuffer; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.DestinationConfig; -import io.airbyte.integrations.destination.record_buffer.BufferStorage; -import io.airbyte.integrations.destination.record_buffer.FileBuffer; -import io.airbyte.integrations.destination.record_buffer.InMemoryBuffer; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; @@ -29,6 +29,7 @@ import org.apache.avro.generic.GenericData.Record; import org.apache.avro.generic.GenericDatumReader; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; public class AvroSerializedBufferTest { @@ -57,6 +58,8 @@ public static void setup() { } @Test + @Disabled("Flaky on CI, See run https://github.com/airbytehq/airbyte/actions/runs/7126781640/job/19405426141?pr=33201 " + + "org.opentest4j.AssertionFailedError: Expected size between 964 and 985, but actual size was 991 ==> expected: but was: ") public void testSnappyAvroWriter() throws Exception { final S3AvroFormatConfig config = new S3AvroFormatConfig(Jsons.jsonNode(Map.of("compression_codec", Map.of( "codec", "snappy")))); @@ -69,7 +72,7 @@ public void testGzipAvroFileWriter() throws Exception { "codec", "zstandard", "compression_level", 20, "include_checksum", true)))); - runTest(new FileBuffer(AvroSerializedBuffer.DEFAULT_SUFFIX), 970L, 985L, config, getExpectedString()); + runTest(new FileBuffer(AvroSerializedBuffer.DEFAULT_SUFFIX), 965L, 985L, config, getExpectedString()); } @Test diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/JsonFieldNameUpdaterTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/avro/JsonFieldNameUpdaterTest.java similarity index 95% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/JsonFieldNameUpdaterTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/avro/JsonFieldNameUpdaterTest.java index 4b4e746ddab0..236ab209f5c8 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/JsonFieldNameUpdaterTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/avro/JsonFieldNameUpdaterTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.avro; +package io.airbyte.cdk.integrations.destination.s3.avro; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/avro/JsonSchemaTypeTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/avro/JsonSchemaTypeTest.java new file mode 100644 index 000000000000..b447613a937f --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/avro/JsonSchemaTypeTest.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cdk.integrations.destination.s3.avro; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.stream.Stream; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.ArgumentsProvider; +import org.junit.jupiter.params.provider.ArgumentsSource; + +public class JsonSchemaTypeTest { + + @ParameterizedTest + @ArgumentsSource(JsonSchemaTypeProvider.class) + public void testFromJsonSchemaType(String type, String airbyteType, JsonSchemaType expectedJsonSchemaType) { + assertEquals( + expectedJsonSchemaType, + JsonSchemaType.fromJsonSchemaType(type, airbyteType)); + } + + public static class JsonSchemaTypeProvider implements ArgumentsProvider { + + @Override + public Stream provideArguments(ExtensionContext context) { + return Stream.of( + Arguments.of("WellKnownTypes.json#/definitions/Number", null, JsonSchemaType.NUMBER_V1), + Arguments.of("WellKnownTypes.json#/definitions/String", null, JsonSchemaType.STRING_V1), + Arguments.of("WellKnownTypes.json#/definitions/Integer", null, JsonSchemaType.INTEGER_V1), + Arguments.of("WellKnownTypes.json#/definitions/Boolean", null, JsonSchemaType.BOOLEAN_V1), + Arguments.of("WellKnownTypes.json#/definitions/BinaryData", null, JsonSchemaType.BINARY_DATA_V1), + Arguments.of("WellKnownTypes.json#/definitions/Date", null, JsonSchemaType.DATE_V1), + Arguments.of("WellKnownTypes.json#/definitions/TimestampWithTimezone", null, JsonSchemaType.TIMESTAMP_WITH_TIMEZONE_V1), + Arguments.of("WellKnownTypes.json#/definitions/TimestampWithoutTimezone", null, JsonSchemaType.TIMESTAMP_WITHOUT_TIMEZONE_V1), + Arguments.of("WellKnownTypes.json#/definitions/TimeWithTimezone", null, JsonSchemaType.TIME_WITH_TIMEZONE_V1), + Arguments.of("WellKnownTypes.json#/definitions/TimeWithoutTimezone", null, JsonSchemaType.TIME_WITHOUT_TIMEZONE_V1), + Arguments.of("number", "integer", JsonSchemaType.NUMBER_INT_V0), + Arguments.of("string", "big_integer", JsonSchemaType.NUMBER_BIGINT_V0), + Arguments.of("number", "float", JsonSchemaType.NUMBER_FLOAT_V0), + Arguments.of("number", null, JsonSchemaType.NUMBER_V0), + Arguments.of("string", null, JsonSchemaType.STRING_V0), + Arguments.of("integer", null, JsonSchemaType.INTEGER_V0), + Arguments.of("boolean", null, JsonSchemaType.BOOLEAN_V0), + Arguments.of("null", null, JsonSchemaType.NULL), + Arguments.of("object", null, JsonSchemaType.OBJECT), + Arguments.of("array", null, JsonSchemaType.ARRAY), + Arguments.of("combined", null, JsonSchemaType.COMBINED)); + } + + } + +} diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/JsonToAvroConverterTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/avro/JsonToAvroConverterTest.java similarity index 99% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/JsonToAvroConverterTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/avro/JsonToAvroConverterTest.java index 5958b7471c6f..534112d5a391 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/JsonToAvroConverterTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/avro/JsonToAvroConverterTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.avro; +package io.airbyte.cdk.integrations.destination.s3.avro; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/S3AvroFormatConfigTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/avro/S3AvroFormatConfigTest.java similarity index 91% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/S3AvroFormatConfigTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/avro/S3AvroFormatConfigTest.java index 6cc3966b8b5e..42266df26ef1 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/S3AvroFormatConfigTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/avro/S3AvroFormatConfigTest.java @@ -2,22 +2,22 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.avro; +package io.airbyte.cdk.integrations.destination.s3.avro; import static com.amazonaws.services.s3.internal.Constants.MB; -import static io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; +import static io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; import alex.mojaki.s3upload.StreamTransferManager; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; +import io.airbyte.cdk.integrations.destination.s3.StorageProvider; +import io.airbyte.cdk.integrations.destination.s3.util.ConfigTestUtils; +import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.S3FormatConfig; -import io.airbyte.integrations.destination.s3.StorageProvider; -import io.airbyte.integrations.destination.s3.util.ConfigTestUtils; -import io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory; import java.util.List; import org.apache.avro.file.CodecFactory; import org.apache.avro.file.DataFileConstants; diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/CsvSerializedBufferTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/csv/CsvSerializedBufferTest.java similarity index 93% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/CsvSerializedBufferTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/csv/CsvSerializedBufferTest.java index 3fa5f1c78497..db18a75df87d 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/CsvSerializedBufferTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/csv/CsvSerializedBufferTest.java @@ -2,20 +2,20 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.csv; +package io.airbyte.cdk.integrations.destination.s3.csv; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.base.DestinationConfig; +import io.airbyte.cdk.integrations.destination.record_buffer.BufferStorage; +import io.airbyte.cdk.integrations.destination.record_buffer.FileBuffer; +import io.airbyte.cdk.integrations.destination.record_buffer.InMemoryBuffer; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.util.Flattening; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.DestinationConfig; -import io.airbyte.integrations.destination.record_buffer.BufferStorage; -import io.airbyte.integrations.destination.record_buffer.FileBuffer; -import io.airbyte.integrations.destination.record_buffer.InMemoryBuffer; -import io.airbyte.integrations.destination.s3.S3Format; -import io.airbyte.integrations.destination.s3.util.Flattening; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/NoFlatteningSheetGeneratorTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/csv/NoFlatteningSheetGeneratorTest.java similarity index 92% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/NoFlatteningSheetGeneratorTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/csv/NoFlatteningSheetGeneratorTest.java index 82dce0912f5f..e4163f55c8ea 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/NoFlatteningSheetGeneratorTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/csv/NoFlatteningSheetGeneratorTest.java @@ -2,15 +2,15 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.csv; +package io.airbyte.cdk.integrations.destination.s3.csv; import static org.junit.jupiter.api.Assertions.assertLinesMatch; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.commons.jackson.MoreMappers; -import io.airbyte.integrations.base.JavaBaseConstants; import java.util.Collections; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/RootLevelFlatteningSheetGeneratorTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/csv/RootLevelFlatteningSheetGeneratorTest.java similarity index 94% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/RootLevelFlatteningSheetGeneratorTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/csv/RootLevelFlatteningSheetGeneratorTest.java index d94772849271..f9953ce87de6 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/RootLevelFlatteningSheetGeneratorTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/csv/RootLevelFlatteningSheetGeneratorTest.java @@ -2,15 +2,15 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.csv; +package io.airbyte.cdk.integrations.destination.s3.csv; import static org.junit.jupiter.api.Assertions.assertLinesMatch; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.commons.jackson.MoreMappers; -import io.airbyte.integrations.base.JavaBaseConstants; import java.util.Collections; import java.util.List; import org.junit.jupiter.api.BeforeEach; diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvFormatConfigTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/csv/S3CsvFormatConfigTest.java similarity index 85% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvFormatConfigTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/csv/S3CsvFormatConfigTest.java index a2a72f20d62f..08e4f94ca9d0 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvFormatConfigTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/csv/S3CsvFormatConfigTest.java @@ -2,23 +2,23 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.csv; +package io.airbyte.cdk.integrations.destination.s3.csv; import static com.amazonaws.services.s3.internal.Constants.MB; -import static io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; +import static io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import alex.mojaki.s3upload.StreamTransferManager; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConstants; +import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; +import io.airbyte.cdk.integrations.destination.s3.util.CompressionType; +import io.airbyte.cdk.integrations.destination.s3.util.ConfigTestUtils; +import io.airbyte.cdk.integrations.destination.s3.util.Flattening; +import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.S3DestinationConstants; -import io.airbyte.integrations.destination.s3.S3FormatConfig; -import io.airbyte.integrations.destination.s3.util.CompressionType; -import io.airbyte.integrations.destination.s3.util.ConfigTestUtils; -import io.airbyte.integrations.destination.s3.util.Flattening; -import io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory; import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvWriterTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/csv/S3CsvWriterTest.java similarity index 93% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvWriterTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/csv/S3CsvWriterTest.java index 48a08a6e4feb..5fe69ffa9923 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/csv/S3CsvWriterTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/csv/S3CsvWriterTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.csv; +package io.airbyte.cdk.integrations.destination.s3.csv; import static java.util.Collections.singletonList; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; @@ -22,12 +22,13 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3Client; import com.fasterxml.jackson.databind.ObjectMapper; +import io.airbyte.cdk.integrations.base.DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.csv.S3CsvWriter.Builder; +import io.airbyte.cdk.integrations.destination.s3.util.CompressionType; +import io.airbyte.cdk.integrations.destination.s3.util.Flattening; +import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerWithMetadata; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.DestinationConfig; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.csv.S3CsvWriter.Builder; -import io.airbyte.integrations.destination.s3.util.CompressionType; -import io.airbyte.integrations.destination.s3.util.Flattening; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; @@ -80,7 +81,7 @@ class S3CsvWriterTest { private AmazonS3 s3Client; - private MockedConstruction streamTransferManagerMockedConstruction; + private MockedConstruction streamTransferManagerMockedConstruction; private List streamTransferManagerConstructorArguments; private List outputStreams; @@ -95,7 +96,7 @@ public void setup() { // This is basically RETURNS_SELF, except with getMultiPartOutputStreams configured correctly. // Other non-void methods (e.g. toString()) will return null. streamTransferManagerMockedConstruction = mockConstruction( - StreamTransferManager.class, + StreamTransferManagerWithMetadata.class, (mock, context) -> { // Mockito doesn't seem to provide an easy way to actually retrieve these arguments later on, so // manually store them on construction. @@ -174,7 +175,7 @@ public void closesS3Upload_when_stagingUploaderClosedSuccessfully() throws Excep writer.close(false); - final List managers = streamTransferManagerMockedConstruction.constructed(); + final List managers = streamTransferManagerMockedConstruction.constructed(); final StreamTransferManager manager = managers.get(0); verify(manager).complete(); } @@ -185,7 +186,7 @@ public void closesS3Upload_when_stagingUploaderClosedFailingly() throws Exceptio writer.close(true); - final List managers = streamTransferManagerMockedConstruction.constructed(); + final List managers = streamTransferManagerMockedConstruction.constructed(); final StreamTransferManager manager = managers.get(0); verify(manager).abort(); } diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/jsonl/JsonLSerializedBufferTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/jsonl/JsonLSerializedBufferTest.java similarity index 93% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/jsonl/JsonLSerializedBufferTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/jsonl/JsonLSerializedBufferTest.java index 7e12960f945c..9fabee2c6189 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/jsonl/JsonLSerializedBufferTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/jsonl/JsonLSerializedBufferTest.java @@ -2,17 +2,17 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.jsonl; +package io.airbyte.cdk.integrations.destination.s3.jsonl; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.record_buffer.BufferStorage; +import io.airbyte.cdk.integrations.destination.record_buffer.FileBuffer; +import io.airbyte.cdk.integrations.destination.record_buffer.InMemoryBuffer; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.record_buffer.BufferStorage; -import io.airbyte.integrations.destination.record_buffer.FileBuffer; -import io.airbyte.integrations.destination.record_buffer.InMemoryBuffer; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/jsonl/S3JsonlFormatConfigTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/jsonl/S3JsonlFormatConfigTest.java similarity index 83% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/jsonl/S3JsonlFormatConfigTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/jsonl/S3JsonlFormatConfigTest.java index b9bb580e1787..8f54ece18ce6 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/jsonl/S3JsonlFormatConfigTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/jsonl/S3JsonlFormatConfigTest.java @@ -2,21 +2,21 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.jsonl; +package io.airbyte.cdk.integrations.destination.s3.jsonl; import static com.amazonaws.services.s3.internal.Constants.MB; -import static io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; +import static io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import alex.mojaki.s3upload.StreamTransferManager; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; +import io.airbyte.cdk.integrations.destination.s3.util.ConfigTestUtils; +import io.airbyte.cdk.integrations.destination.s3.util.Flattening; +import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.S3FormatConfig; -import io.airbyte.integrations.destination.s3.util.ConfigTestUtils; -import io.airbyte.integrations.destination.s3.util.Flattening; -import io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory; import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBufferTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/parquet/ParquetSerializedBufferTest.java similarity index 95% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBufferTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/parquet/ParquetSerializedBufferTest.java index a9ca83f66389..c163fab60354 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBufferTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/parquet/ParquetSerializedBufferTest.java @@ -2,18 +2,18 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.parquet; +package io.airbyte.cdk.integrations.destination.s3.parquet; -import static io.airbyte.integrations.destination.s3.util.JavaProcessRunner.runProcess; +import static io.airbyte.cdk.integrations.destination.s3.util.JavaProcessRunner.runProcess; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import com.amazonaws.util.IOUtils; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.base.DestinationConfig; +import io.airbyte.cdk.integrations.destination.record_buffer.SerializableBuffer; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.DestinationConfig; -import io.airbyte.integrations.destination.record_buffer.SerializableBuffer; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitiveUtil.JsonSchemaPrimitive; import io.airbyte.protocol.models.JsonSchemaType; diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetFormatConfigTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/parquet/S3ParquetFormatConfigTest.java similarity index 95% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetFormatConfigTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/parquet/S3ParquetFormatConfigTest.java index a1509c555251..2414355b30e0 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetFormatConfigTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/parquet/S3ParquetFormatConfigTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.parquet; +package io.airbyte.cdk.integrations.destination.s3.parquet; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/tamplate/S3FilenameTemplateManagerTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/tamplate/S3FilenameTemplateManagerTest.java similarity index 90% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/tamplate/S3FilenameTemplateManagerTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/tamplate/S3FilenameTemplateManagerTest.java index 0fae0394e7b3..081f8edea789 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/tamplate/S3FilenameTemplateManagerTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/tamplate/S3FilenameTemplateManagerTest.java @@ -2,14 +2,14 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.tamplate; +package io.airbyte.cdk.integrations.destination.s3.tamplate; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mockStatic; -import io.airbyte.integrations.destination.s3.S3DestinationConstants; -import io.airbyte.integrations.destination.s3.template.S3FilenameTemplateManager; -import io.airbyte.integrations.destination.s3.template.S3FilenameTemplateParameterObject; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConstants; +import io.airbyte.cdk.integrations.destination.s3.template.S3FilenameTemplateManager; +import io.airbyte.cdk.integrations.destination.s3.template.S3FilenameTemplateParameterObject; import java.io.IOException; import java.text.DateFormat; import java.text.SimpleDateFormat; diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/util/CompressionTypeHelperTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/util/CompressionTypeHelperTest.java similarity index 85% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/util/CompressionTypeHelperTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/util/CompressionTypeHelperTest.java index 1221a1bc8198..09b2c056e388 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/util/CompressionTypeHelperTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/util/CompressionTypeHelperTest.java @@ -2,12 +2,12 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.util; +package io.airbyte.cdk.integrations.destination.s3.util; import static org.junit.jupiter.api.Assertions.assertEquals; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConstants; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.s3.S3DestinationConstants; import java.util.Map; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/util/ConfigTestUtils.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/util/ConfigTestUtils.java similarity index 86% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/util/ConfigTestUtils.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/util/ConfigTestUtils.java index 73ee1fd5797b..95d01a2e8ecf 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/util/ConfigTestUtils.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/util/ConfigTestUtils.java @@ -2,14 +2,14 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.util; +package io.airbyte.cdk.integrations.destination.s3.util; import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.credential.S3AccessKeyCredentialConfig; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.credential.S3AccessKeyCredentialConfig; public class ConfigTestUtils { diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/util/S3OutputPathHelperTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/util/S3OutputPathHelperTest.java similarity index 97% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/util/S3OutputPathHelperTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/util/S3OutputPathHelperTest.java index bdf6160fe8c6..f33de9c70954 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/util/S3OutputPathHelperTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/util/S3OutputPathHelperTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.util; +package io.airbyte.cdk.integrations.destination.s3.util; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/writer/BaseS3WriterTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/writer/BaseS3WriterTest.java similarity index 75% rename from airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/writer/BaseS3WriterTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/writer/BaseS3WriterTest.java index 25e0a548130f..15a1b5e141ae 100644 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/writer/BaseS3WriterTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/java/io/airbyte/cdk/integrations/destination/s3/writer/BaseS3WriterTest.java @@ -2,12 +2,12 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3.writer; +package io.airbyte.cdk.integrations.destination.s3.writer; import static org.junit.jupiter.api.Assertions.assertEquals; -import io.airbyte.integrations.destination.s3.S3Format; -import io.airbyte.integrations.destination.s3.template.S3FilenameTemplateParameterObject; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.template.S3FilenameTemplateParameterObject; import java.io.IOException; import java.sql.Timestamp; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/bases/base-java-s3/src/test/resources/parquet/json_field_name_updater/test_case.json b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/resources/parquet/json_field_name_updater/test_case.json similarity index 100% rename from airbyte-integrations/bases/base-java-s3/src/test/resources/parquet/json_field_name_updater/test_case.json rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/resources/parquet/json_field_name_updater/test_case.json diff --git a/airbyte-integrations/bases/base-java-s3/src/test/resources/parquet/json_schema_converter/json_conversion_test_cases_v0.json b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/resources/parquet/json_schema_converter/json_conversion_test_cases_v0.json similarity index 100% rename from airbyte-integrations/bases/base-java-s3/src/test/resources/parquet/json_schema_converter/json_conversion_test_cases_v0.json rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/resources/parquet/json_schema_converter/json_conversion_test_cases_v0.json diff --git a/airbyte-integrations/bases/base-java-s3/src/test/resources/parquet/json_schema_converter/json_conversion_test_cases_v1.json b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/resources/parquet/json_schema_converter/json_conversion_test_cases_v1.json similarity index 100% rename from airbyte-integrations/bases/base-java-s3/src/test/resources/parquet/json_schema_converter/json_conversion_test_cases_v1.json rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/resources/parquet/json_schema_converter/json_conversion_test_cases_v1.json diff --git a/airbyte-integrations/bases/base-java-s3/src/test/resources/parquet/json_schema_converter/type_conversion_test_cases_v0.json b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/resources/parquet/json_schema_converter/type_conversion_test_cases_v0.json similarity index 100% rename from airbyte-integrations/bases/base-java-s3/src/test/resources/parquet/json_schema_converter/type_conversion_test_cases_v0.json rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/resources/parquet/json_schema_converter/type_conversion_test_cases_v0.json diff --git a/airbyte-integrations/bases/base-java-s3/src/test/resources/parquet/json_schema_converter/type_conversion_test_cases_v1.json b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/resources/parquet/json_schema_converter/type_conversion_test_cases_v1.json similarity index 100% rename from airbyte-integrations/bases/base-java-s3/src/test/resources/parquet/json_schema_converter/type_conversion_test_cases_v1.json rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/test/resources/parquet/json_schema_converter/type_conversion_test_cases_v1.json diff --git a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3AvroParquetDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3AvroParquetDestinationAcceptanceTest.java similarity index 96% rename from airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3AvroParquetDestinationAcceptanceTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3AvroParquetDestinationAcceptanceTest.java index 2d5d39bc90c6..42e209811c4b 100644 --- a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3AvroParquetDestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3AvroParquetDestinationAcceptanceTest.java @@ -2,15 +2,15 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.s3.avro.JsonSchemaType; +import io.airbyte.cdk.integrations.standardtest.destination.argproviders.NumberDataTypeTestArgumentProvider; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.destination.s3.avro.JsonSchemaType; -import io.airbyte.integrations.standardtest.destination.argproviders.NumberDataTypeTestArgumentProvider; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStream; diff --git a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3BaseAvroDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroDestinationAcceptanceTest.java similarity index 91% rename from airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3BaseAvroDestinationAcceptanceTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroDestinationAcceptanceTest.java index 85153bad2722..a048fd69dbfb 100644 --- a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3BaseAvroDestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroDestinationAcceptanceTest.java @@ -2,17 +2,17 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectReader; +import io.airbyte.cdk.integrations.destination.s3.avro.AvroConstants; +import io.airbyte.cdk.integrations.destination.s3.avro.JsonFieldNameUpdater; +import io.airbyte.cdk.integrations.destination.s3.util.AvroRecordHelper; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.s3.avro.AvroConstants; -import io.airbyte.integrations.destination.s3.avro.JsonFieldNameUpdater; -import io.airbyte.integrations.destination.s3.util.AvroRecordHelper; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.util.HashMap; import java.util.LinkedList; import java.util.List; diff --git a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3BaseAvroParquetTestDataComparator.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroParquetTestDataComparator.java similarity index 90% rename from airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3BaseAvroParquetTestDataComparator.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroParquetTestDataComparator.java index 5961d2ae552d..4fe9040168eb 100644 --- a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3BaseAvroParquetTestDataComparator.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseAvroParquetTestDataComparator.java @@ -2,9 +2,9 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; diff --git a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3BaseCsvDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseCsvDestinationAcceptanceTest.java similarity index 96% rename from airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3BaseCsvDestinationAcceptanceTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseCsvDestinationAcceptanceTest.java index 931daa3a71d8..55a01e4af982 100644 --- a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3BaseCsvDestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseCsvDestinationAcceptanceTest.java @@ -2,15 +2,15 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.s3.util.Flattening; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.s3.util.Flattening; import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; diff --git a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3BaseCsvGzipDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseCsvGzipDestinationAcceptanceTest.java similarity index 88% rename from airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3BaseCsvGzipDestinationAcceptanceTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseCsvGzipDestinationAcceptanceTest.java index b86fae961bec..05117564a1f3 100644 --- a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3BaseCsvGzipDestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseCsvGzipDestinationAcceptanceTest.java @@ -2,12 +2,12 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import com.amazonaws.services.s3.model.S3Object; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.s3.util.Flattening; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.s3.util.Flattening; import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; diff --git a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3BaseJsonlDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseJsonlDestinationAcceptanceTest.java similarity index 91% rename from airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3BaseJsonlDestinationAcceptanceTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseJsonlDestinationAcceptanceTest.java index 5ccaddf5a7ca..faa374bfedb2 100644 --- a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3BaseJsonlDestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseJsonlDestinationAcceptanceTest.java @@ -2,14 +2,14 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.s3.util.Flattening; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.s3.util.Flattening; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; diff --git a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3BaseJsonlGzipDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseJsonlGzipDestinationAcceptanceTest.java similarity index 88% rename from airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3BaseJsonlGzipDestinationAcceptanceTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseJsonlGzipDestinationAcceptanceTest.java index 84286d98fc53..5a7689bdb69b 100644 --- a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3BaseJsonlGzipDestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseJsonlGzipDestinationAcceptanceTest.java @@ -2,12 +2,12 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import com.amazonaws.services.s3.model.S3Object; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.s3.util.Flattening; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.s3.util.Flattening; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; diff --git a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3BaseParquetDestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseParquetDestinationAcceptanceTest.java similarity index 90% rename from airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3BaseParquetDestinationAcceptanceTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseParquetDestinationAcceptanceTest.java index 22777e70d8d4..44abebe02905 100644 --- a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3BaseParquetDestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3BaseParquetDestinationAcceptanceTest.java @@ -2,18 +2,18 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectReader; +import io.airbyte.cdk.integrations.destination.s3.avro.AvroConstants; +import io.airbyte.cdk.integrations.destination.s3.avro.JsonFieldNameUpdater; +import io.airbyte.cdk.integrations.destination.s3.parquet.S3ParquetWriter; +import io.airbyte.cdk.integrations.destination.s3.util.AvroRecordHelper; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.s3.avro.AvroConstants; -import io.airbyte.integrations.destination.s3.avro.JsonFieldNameUpdater; -import io.airbyte.integrations.destination.s3.parquet.S3ParquetWriter; -import io.airbyte.integrations.destination.s3.util.AvroRecordHelper; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; diff --git a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationAcceptanceTest.java b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3DestinationAcceptanceTest.java similarity index 93% rename from airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationAcceptanceTest.java rename to airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3DestinationAcceptanceTest.java index fe19b22414c4..11cb05dde761 100644 --- a/airbyte-integrations/bases/s3-destination-base-integration-test/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationAcceptanceTest.java +++ b/airbyte-cdk/java/airbyte-cdk/s3-destinations/src/testFixtures/java/io/airbyte/cdk/integrations/destination/s3/S3DestinationAcceptanceTest.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.destination.s3; +package io.airbyte.cdk.integrations.destination.s3; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.DeleteObjectsRequest; @@ -12,14 +12,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.s3.util.S3NameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.io.IOs; import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.s3.util.S3NameTransformer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.nio.file.Path; import java.util.Comparator; import java.util.HashSet; diff --git a/airbyte-cdk/java/airbyte-cdk/settings.gradle b/airbyte-cdk/java/airbyte-cdk/settings.gradle new file mode 100644 index 000000000000..de5c30bca513 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/settings.gradle @@ -0,0 +1,16 @@ +rootProject.name = 'airbyte' + +include ':airbyte-cdk:java:airbyte-cdk:airbyte-commons' +include ':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation' +include ':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli' +include ':airbyte-cdk:java:airbyte-cdk:airbyte-commons-protocol' +include ':airbyte-cdk:java:airbyte-cdk:airbyte-api' +include ':airbyte-cdk:java:airbyte-cdk:config-models-oss' +include ':airbyte-cdk:java:airbyte-cdk:init-oss' +include ':airbyte-cdk:java:airbyte-cdk:acceptance-test-harness' +include ':airbyte-cdk:java:airbyte-cdk:core' +include ':airbyte-cdk:java:airbyte-cdk:db-sources' +include ':airbyte-cdk:java:airbyte-cdk:db-destinations' +include ':airbyte-cdk:java:airbyte-cdk:s3-destinations' +// Leaving this out until we fully commit to moving this to the cdk +//include ':airbyte-cdk:java:airbyte-cdk:typing-deduping' diff --git a/airbyte-cdk/java/airbyte-cdk/src/main/resources/version.properties b/airbyte-cdk/java/airbyte-cdk/src/main/resources/version.properties deleted file mode 100644 index 6360017a8ee7..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/src/main/resources/version.properties +++ /dev/null @@ -1 +0,0 @@ -version=0.0.2 diff --git a/airbyte-cdk/java/airbyte-cdk/src/test/java/io/airbyte/cdk/CDKConstantsTest.java b/airbyte-cdk/java/airbyte-cdk/src/test/java/io/airbyte/cdk/CDKConstantsTest.java deleted file mode 100644 index a4942ebf0080..000000000000 --- a/airbyte-cdk/java/airbyte-cdk/src/test/java/io/airbyte/cdk/CDKConstantsTest.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.cdk; - -import static org.junit.jupiter.api.Assertions.*; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -class CDKConstantsTest { - - /* TODO: Remove these canary tests once real tests are in place. */ - @Test - void getVersion() { - assertEquals("0.0.2", CDKConstants.VERSION.replace("-SNAPSHOT", "")); - } - - @Test - // Comment out this line to force failure: - @Disabled("This is an intentionally failing test (skipped).") - void mustFail() { - fail("This is an intentionally failing test."); - } - -} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle b/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle new file mode 100644 index 000000000000..fc3ba8f061e8 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/build.gradle @@ -0,0 +1,26 @@ +plugins { + id 'java-library' +} + +dependencies { + implementation project(':airbyte-cdk:java:airbyte-cdk:core') + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:acceptance-test-harness') + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') + testFixturesImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + + testImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + implementation group: 'commons-codec', name: 'commons-codec', version: '1.16.0' + implementation libs.jooq + testFixturesImplementation libs.airbyte.protocol + + testFixturesImplementation(platform('org.junit:junit-bom:5.8.2')) + testFixturesImplementation 'org.junit.jupiter:junit-jupiter-api' + testFixturesImplementation 'org.junit.jupiter:junit-jupiter-params' + testFixturesImplementation 'org.mockito:mockito-core:4.6.1' +} + +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteProtocolType.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteProtocolType.java similarity index 97% rename from airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteProtocolType.java rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteProtocolType.java index ab800697b0e1..6d0320ca7b74 100644 --- a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteProtocolType.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteProtocolType.java @@ -75,4 +75,9 @@ protected static AirbyteProtocolType fromJson(final JsonNode node) { return UNKNOWN; } + @Override + public String getTypeName() { + return this.name(); + } + } diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteType.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteType.java similarity index 99% rename from airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteType.java rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteType.java index de59c763ed9c..33ede887c42b 100644 --- a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteType.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteType.java @@ -117,4 +117,6 @@ private static JsonNode getTrimmedJsonSchema(final JsonNode schema, final String return schemaClone; } + String getTypeName(); + } diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AlterTableReport.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AlterTableReport.java similarity index 100% rename from airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AlterTableReport.java rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/AlterTableReport.java diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Array.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Array.java new file mode 100644 index 000000000000..11f6b5287982 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Array.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +public record Array(AirbyteType items) implements AirbyteType { + + public static final String TYPE = "ARRAY"; + + @Override + public String getTypeName() { + return TYPE; + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseDestinationV1V2Migrator.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseDestinationV1V2Migrator.java similarity index 87% rename from airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseDestinationV1V2Migrator.java rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseDestinationV1V2Migrator.java index d7f2e6ab67ed..a33c3b715630 100644 --- a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseDestinationV1V2Migrator.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseDestinationV1V2Migrator.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.base.destination.typing_deduping; -import static io.airbyte.integrations.base.JavaBaseConstants.LEGACY_RAW_TABLE_COLUMNS; -import static io.airbyte.integrations.base.JavaBaseConstants.V2_RAW_TABLE_COLUMN_NAMES; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.LEGACY_RAW_TABLE_COLUMNS; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.V2_RAW_TABLE_COLUMN_NAMES; import io.airbyte.protocol.models.v0.DestinationSyncMode; import java.util.Collection; @@ -13,16 +13,16 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class BaseDestinationV1V2Migrator implements DestinationV1V2Migrator { +public abstract class BaseDestinationV1V2Migrator implements DestinationV1V2Migrator { protected static final Logger LOGGER = LoggerFactory.getLogger(BaseDestinationV1V2Migrator.class); @Override public void migrateIfNecessary( - final SqlGenerator sqlGenerator, - final DestinationHandler destinationHandler, + final SqlGenerator sqlGenerator, + final DestinationHandler destinationHandler, final StreamConfig streamConfig) - throws TableNotMigratedException, UnexpectedSchemaException { + throws Exception { LOGGER.info("Assessing whether migration is necessary for stream {}", streamConfig.id().finalName()); if (shouldMigrate(streamConfig)) { LOGGER.info("Starting v2 Migration for stream {}", streamConfig.id().finalName()); @@ -40,7 +40,7 @@ public void migrateIfNecessary( * @param streamConfig the stream in question * @return whether to migrate the stream */ - protected boolean shouldMigrate(final StreamConfig streamConfig) { + protected boolean shouldMigrate(final StreamConfig streamConfig) throws Exception { final var v1RawTable = convertToV1RawName(streamConfig); LOGGER.info("Checking whether v1 raw table {} in dataset {} exists", v1RawTable.tableName(), v1RawTable.namespace()); final var syncModeNeedsMigration = isMigrationRequiredForSyncMode(streamConfig.destinationSyncMode()); @@ -66,7 +66,7 @@ public void migrate(final SqlGenerator sqlGenerator, final var namespacedTableName = convertToV1RawName(streamConfig); try { destinationHandler.execute(sqlGenerator.migrateFromV1toV2(streamConfig.id(), namespacedTableName.namespace(), namespacedTableName.tableName())); - } catch (Exception e) { + } catch (final Exception e) { final var message = "Attempted and failed to migrate stream %s".formatted(streamConfig.id().finalName()); throw new TableNotMigratedException(message, e); } @@ -78,7 +78,7 @@ public void migrate(final SqlGenerator sqlGenerator, * @param existingV2AirbyteRawTable the v1 raw table * @return whether the schema is as expected */ - private boolean doesV1RawTableMatchExpectedSchema(DialectTableDefinition existingV2AirbyteRawTable) { + private boolean doesV1RawTableMatchExpectedSchema(final DialectTableDefinition existingV2AirbyteRawTable) { return schemaMatchesExpectation(existingV2AirbyteRawTable, LEGACY_RAW_TABLE_COLUMNS); } @@ -88,7 +88,7 @@ private boolean doesV1RawTableMatchExpectedSchema(DialectTableDefinition existin * * @param existingV2AirbyteRawTable the v2 raw table */ - private void validateAirbyteInternalNamespaceRawTableMatchExpectedV2Schema(DialectTableDefinition existingV2AirbyteRawTable) { + private void validateAirbyteInternalNamespaceRawTableMatchExpectedV2Schema(final DialectTableDefinition existingV2AirbyteRawTable) { if (!schemaMatchesExpectation(existingV2AirbyteRawTable, V2_RAW_TABLE_COLUMN_NAMES)) { throw new UnexpectedSchemaException("Destination V2 Raw Table does not match expected Schema"); } @@ -110,7 +110,7 @@ private boolean isMigrationRequiredForSyncMode(final DestinationSyncMode destina * @param streamConfig the raw table to check * @return whether it exists and is in the correct format */ - private boolean doesValidV2RawTableAlreadyExist(final StreamConfig streamConfig) { + private boolean doesValidV2RawTableAlreadyExist(final StreamConfig streamConfig) throws Exception { if (doesAirbyteInternalNamespaceExist(streamConfig)) { final var existingV2Table = getTableIfExists(streamConfig.id().rawNamespace(), streamConfig.id().rawName()); existingV2Table.ifPresent(this::validateAirbyteInternalNamespaceRawTableMatchExpectedV2Schema); @@ -126,7 +126,7 @@ private boolean doesValidV2RawTableAlreadyExist(final StreamConfig streamConfig) * @param tableName * @return whether it exists and is in the correct format */ - protected boolean doesValidV1RawTableExist(final String namespace, final String tableName) { + protected boolean doesValidV1RawTableExist(final String namespace, final String tableName) throws Exception { final var existingV1RawTable = getTableIfExists(namespace, tableName); return existingV1RawTable.isPresent() && doesV1RawTableMatchExpectedSchema(existingV1RawTable.get()); } @@ -137,7 +137,7 @@ protected boolean doesValidV1RawTableExist(final String namespace, final String * @param streamConfig the stream to check * @return whether the schema exists */ - abstract protected boolean doesAirbyteInternalNamespaceExist(StreamConfig streamConfig); + abstract protected boolean doesAirbyteInternalNamespaceExist(StreamConfig streamConfig) throws Exception; /** * Checks a Table's schema and compares it to an expected schema to make sure it matches @@ -155,7 +155,7 @@ protected boolean doesValidV1RawTableExist(final String namespace, final String * @param tableName * @return an optional potentially containing a reference to the table */ - abstract protected Optional getTableIfExists(String namespace, String tableName); + abstract protected Optional getTableIfExists(String namespace, String tableName) throws Exception; /** * We use different naming conventions for raw table names in destinations v2, we need a way to map diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParser.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParser.java new file mode 100644 index 000000000000..925d5037ea28 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParser.java @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE; + +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.cdk.integrations.base.AirbyteExceptionHandler; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map.Entry; +import java.util.Optional; +import org.apache.commons.codec.digest.DigestUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CatalogParser { + + private static final Logger LOGGER = LoggerFactory.getLogger(CatalogParser.class); + + private final SqlGenerator sqlGenerator; + private final String rawNamespace; + + public CatalogParser(final SqlGenerator sqlGenerator) { + this(sqlGenerator, DEFAULT_AIRBYTE_INTERNAL_NAMESPACE); + } + + public CatalogParser(final SqlGenerator sqlGenerator, final String rawNamespace) { + this.sqlGenerator = sqlGenerator; + this.rawNamespace = rawNamespace; + } + + public ParsedCatalog parseCatalog(final ConfiguredAirbyteCatalog catalog) { + // this code is bad and I feel bad + // it's mostly a port of the old normalization logic to prevent tablename collisions. + // tbh I have no idea if it works correctly. + final List streamConfigs = new ArrayList<>(); + for (final ConfiguredAirbyteStream stream : catalog.getStreams()) { + final StreamConfig originalStreamConfig = toStreamConfig(stream); + final StreamConfig actualStreamConfig; + // Use empty string quote because we don't really care + if (streamConfigs.stream().anyMatch(s -> s.id().finalTableId("").equals(originalStreamConfig.id().finalTableId(""))) + || streamConfigs.stream().anyMatch(s -> s.id().rawTableId("").equals(originalStreamConfig.id().rawTableId("")))) { + final String originalNamespace = stream.getStream().getNamespace(); + final String originalName = stream.getStream().getName(); + + LOGGER.info("Detected table name collision for {}.{}", originalNamespace, originalName); + + // ... this logic is ported from legacy normalization, and maybe should change? + // We're taking a hash of the quoted namespace and the unquoted stream name + final String hash = DigestUtils.sha1Hex(originalStreamConfig.id().finalNamespace() + "&airbyte&" + originalName).substring(0, 3); + final String newName = originalName + "_" + hash; + actualStreamConfig = new StreamConfig( + sqlGenerator.buildStreamId(originalNamespace, newName, rawNamespace), + originalStreamConfig.syncMode(), + originalStreamConfig.destinationSyncMode(), + originalStreamConfig.primaryKey(), + originalStreamConfig.cursor(), + originalStreamConfig.columns()); + } else { + actualStreamConfig = originalStreamConfig; + } + streamConfigs.add(actualStreamConfig); + + // Populate some interesting strings into the exception handler string deinterpolator + AirbyteExceptionHandler.addStringForDeinterpolation(actualStreamConfig.id().rawNamespace()); + AirbyteExceptionHandler.addStringForDeinterpolation(actualStreamConfig.id().rawName()); + AirbyteExceptionHandler.addStringForDeinterpolation(actualStreamConfig.id().finalNamespace()); + AirbyteExceptionHandler.addStringForDeinterpolation(actualStreamConfig.id().finalName()); + AirbyteExceptionHandler.addStringForDeinterpolation(actualStreamConfig.id().originalNamespace()); + AirbyteExceptionHandler.addStringForDeinterpolation(actualStreamConfig.id().originalName()); + actualStreamConfig.columns().keySet().forEach(columnId -> { + AirbyteExceptionHandler.addStringForDeinterpolation(columnId.name()); + AirbyteExceptionHandler.addStringForDeinterpolation(columnId.originalName()); + }); + // It's (unfortunately) possible for a cursor/PK to be declared that don't actually exist in the + // schema. + // Add their strings explicitly. + actualStreamConfig.cursor().ifPresent(cursor -> { + AirbyteExceptionHandler.addStringForDeinterpolation(cursor.name()); + AirbyteExceptionHandler.addStringForDeinterpolation(cursor.originalName()); + }); + actualStreamConfig.primaryKey().forEach(pk -> { + AirbyteExceptionHandler.addStringForDeinterpolation(pk.name()); + AirbyteExceptionHandler.addStringForDeinterpolation(pk.originalName()); + }); + } + return new ParsedCatalog(streamConfigs); + } + + // TODO maybe we should extract the column collision stuff to a separate method, since that's the + // interesting bit + @VisibleForTesting + public StreamConfig toStreamConfig(final ConfiguredAirbyteStream stream) { + final AirbyteType schema = AirbyteType.fromJsonSchema(stream.getStream().getJsonSchema()); + final LinkedHashMap airbyteColumns; + if (schema instanceof final Struct o) { + airbyteColumns = o.properties(); + } else if (schema instanceof final Union u) { + airbyteColumns = u.asColumns(); + } else { + throw new IllegalArgumentException("Top-level schema must be an object"); + } + + if (stream.getPrimaryKey().stream().anyMatch(key -> key.size() > 1)) { + throw new IllegalArgumentException("Only top-level primary keys are supported"); + } + final List primaryKey = stream.getPrimaryKey().stream().map(key -> sqlGenerator.buildColumnId(key.get(0))).toList(); + + if (stream.getCursorField().size() > 1) { + throw new IllegalArgumentException("Only top-level cursors are supported"); + } + final Optional cursor; + if (stream.getCursorField().size() > 0) { + cursor = Optional.of(sqlGenerator.buildColumnId(stream.getCursorField().get(0))); + } else { + cursor = Optional.empty(); + } + + // this code is really bad and I'm not convinced we need to preserve this behavior. + // as with the tablename collisions thing above - we're trying to preserve legacy normalization's + // naming conventions here. + final LinkedHashMap columns = new LinkedHashMap<>(); + for (final Entry entry : airbyteColumns.entrySet()) { + final ColumnId originalColumnId = sqlGenerator.buildColumnId(entry.getKey()); + ColumnId columnId; + if (columns.keySet().stream().noneMatch(c -> c.canonicalName().equals(originalColumnId.canonicalName()))) { + // None of the existing columns have the same name. We can add this new column as-is. + columnId = originalColumnId; + } else { + LOGGER.info( + "Detected column name collision for {}.{}.{}", + stream.getStream().getNamespace(), + stream.getStream().getName(), + entry.getKey()); + // One of the existing columns has the same name. We need to handle this collision. + // Append _1, _2, _3, ... to the column name until we find one that doesn't collide. + int i = 1; + while (true) { + columnId = sqlGenerator.buildColumnId(entry.getKey(), "_" + i); + final String canonicalName = columnId.canonicalName(); + if (columns.keySet().stream().noneMatch(c -> c.canonicalName().equals(canonicalName))) { + break; + } else { + i++; + } + } + // But we need to keep the original name so that we can still fetch it out of the JSON records. + columnId = new ColumnId( + columnId.name(), + originalColumnId.originalName(), + columnId.canonicalName()); + } + + columns.put(columnId, entry.getValue()); + } + + return new StreamConfig( + sqlGenerator.buildStreamId(stream.getStream().getNamespace(), stream.getStream().getName(), rawNamespace), + stream.getSyncMode(), + stream.getDestinationSyncMode(), + primaryKey, + cursor, + columns); + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CollectionUtils.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CollectionUtils.java similarity index 100% rename from airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CollectionUtils.java rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CollectionUtils.java diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/ColumnId.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/ColumnId.java similarity index 100% rename from airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/ColumnId.java rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/ColumnId.java diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java new file mode 100644 index 000000000000..d01f47060ba4 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java @@ -0,0 +1,359 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import static io.airbyte.cdk.integrations.base.IntegrationRunner.TYPE_AND_DEDUPE_THREAD_NAME; +import static io.airbyte.integrations.base.destination.typing_deduping.FutureUtils.countOfTypingDedupingThreads; +import static io.airbyte.integrations.base.destination.typing_deduping.FutureUtils.reduceExceptions; +import static java.util.Collections.singleton; + +import com.google.common.collect.Streams; +import io.airbyte.cdk.integrations.destination.StreamSyncSummary; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.concurrent.BasicThreadFactory; +import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * An abstraction over SqlGenerator and DestinationHandler. Destinations will still need to call + * {@code new CatalogParser(new FooSqlGenerator()).parseCatalog()}, but should otherwise avoid + * interacting directly with these classes. + *

      + * In a typical sync, destinations should call the methods: + *

        + *
      1. {@link #prepareTables()} once at the start of the sync
      2. + *
      3. {@link #typeAndDedupe(String, String, boolean)} as needed throughout the sync
      4. + *
      5. {@link #commitFinalTables()} once at the end of the sync
      6. + *
      + * Note that #prepareTables() initializes some internal state. The other methods will throw an + * exception if that method was not called. + */ +public class DefaultTyperDeduper implements TyperDeduper { + + private static final Logger LOGGER = LoggerFactory.getLogger(TyperDeduper.class); + + private static final String NO_SUFFIX = ""; + private static final String TMP_OVERWRITE_TABLE_SUFFIX = "_airbyte_tmp"; + + private final SqlGenerator sqlGenerator; + private final DestinationHandler destinationHandler; + + private final DestinationV1V2Migrator v1V2Migrator; + private final V2TableMigrator v2TableMigrator; + private final ParsedCatalog parsedCatalog; + private Set overwriteStreamsWithTmpTable; + private final Set> streamsWithSuccessfulSetup; + private final Map initialRawTableStateByStream; + // We only want to run a single instance of T+D per stream at a time. These objects are used for + // synchronization per stream. + // Use a read-write lock because we need the same semantics: + // * any number of threads can insert to the raw tables at the same time, as long as T+D isn't + // running (i.e. "read lock") + // * T+D must run in complete isolation (i.e. "write lock") + private final Map tdLocks; + // These locks are used to prevent multiple simultaneous attempts to T+D the same stream. + // We use tryLock with these so that we don't queue up multiple T+D runs for the same stream. + private final Map internalTdLocks; + + private final ExecutorService executorService; + + public DefaultTyperDeduper(final SqlGenerator sqlGenerator, + final DestinationHandler destinationHandler, + final ParsedCatalog parsedCatalog, + final DestinationV1V2Migrator v1V2Migrator, + final V2TableMigrator v2TableMigrator, + final int defaultThreadCount) { + this.sqlGenerator = sqlGenerator; + this.destinationHandler = destinationHandler; + this.parsedCatalog = parsedCatalog; + this.v1V2Migrator = v1V2Migrator; + this.v2TableMigrator = v2TableMigrator; + this.initialRawTableStateByStream = new ConcurrentHashMap<>(); + this.streamsWithSuccessfulSetup = ConcurrentHashMap.newKeySet(parsedCatalog.streams().size()); + this.tdLocks = new ConcurrentHashMap<>(); + this.internalTdLocks = new ConcurrentHashMap<>(); + this.executorService = Executors.newFixedThreadPool(countOfTypingDedupingThreads(defaultThreadCount), + new BasicThreadFactory.Builder().namingPattern(TYPE_AND_DEDUPE_THREAD_NAME).build()); + } + + public DefaultTyperDeduper( + final SqlGenerator sqlGenerator, + final DestinationHandler destinationHandler, + final ParsedCatalog parsedCatalog, + final DestinationV1V2Migrator v1V2Migrator, + final int defaultThreadCount) { + this(sqlGenerator, destinationHandler, parsedCatalog, v1V2Migrator, new NoopV2TableMigrator(), defaultThreadCount); + } + + private void prepareSchemas(final ParsedCatalog parsedCatalog) throws Exception { + final var rawSchema = parsedCatalog.streams().stream().map(stream -> stream.id().rawNamespace()); + final var finalSchema = parsedCatalog.streams().stream().map(stream -> stream.id().finalNamespace()); + final var createAllSchemasSql = Streams.concat(rawSchema, finalSchema) + .filter(Objects::nonNull) + .distinct() + .map(sqlGenerator::createSchema) + .toList(); + destinationHandler.execute(Sql.concat(createAllSchemasSql)); + } + + @Override + public void prepareTables() throws Exception { + if (overwriteStreamsWithTmpTable != null) { + throw new IllegalStateException("Tables were already prepared."); + } + overwriteStreamsWithTmpTable = ConcurrentHashMap.newKeySet(); + LOGGER.info("Preparing tables"); + + prepareSchemas(parsedCatalog); + final Set>> prepareTablesTasks = new HashSet<>(); + for (final StreamConfig stream : parsedCatalog.streams()) { + prepareTablesTasks.add(prepareTablesFuture(stream)); + } + CompletableFuture.allOf(prepareTablesTasks.toArray(CompletableFuture[]::new)).join(); + reduceExceptions(prepareTablesTasks, "The following exceptions were thrown attempting to prepare tables:\n"); + } + + private CompletableFuture> prepareTablesFuture(final StreamConfig stream) { + // For each stream, make sure that its corresponding final table exists. + // Also, for OVERWRITE streams, decide if we're writing directly to the final table, or into an + // _airbyte_tmp table. + return CompletableFuture.supplyAsync(() -> { + try { + // Migrate the Raw Tables if this is the first v2 sync after a v1 sync + v1V2Migrator.migrateIfNecessary(sqlGenerator, destinationHandler, stream); + v2TableMigrator.migrateIfNecessary(stream); + + final Optional existingTable = destinationHandler.findExistingTable(stream.id()); + if (existingTable.isPresent()) { + LOGGER.info("Final Table exists for stream {}", stream.id().finalName()); + // The table already exists. Decide whether we're writing to it directly, or using a tmp table. + if (stream.destinationSyncMode() == DestinationSyncMode.OVERWRITE) { + if (!destinationHandler.isFinalTableEmpty(stream.id()) || !sqlGenerator.existingSchemaMatchesStreamConfig(stream, existingTable.get())) { + // We want to overwrite an existing table. Write into a tmp table. We'll overwrite the table at the + // end of the sync. + overwriteStreamsWithTmpTable.add(stream.id()); + // overwrite an existing tmp table if needed. + destinationHandler.execute(sqlGenerator.createTable(stream, TMP_OVERWRITE_TABLE_SUFFIX, true)); + LOGGER.info("Using temp final table for stream {}, will overwrite existing table at end of sync", stream.id().finalName()); + } else { + LOGGER.info("Final Table for stream {} is empty and matches the expected v2 format, writing to table directly", + stream.id().finalName()); + } + + } else if (!sqlGenerator.existingSchemaMatchesStreamConfig(stream, existingTable.get())) { + // We're loading data directly into the existing table. Make sure it has the right schema. + TypeAndDedupeTransaction.executeSoftReset(sqlGenerator, destinationHandler, stream); + } + } else { + LOGGER.info("Final Table does not exist for stream {}, creating.", stream.id().finalName()); + // The table doesn't exist. Create it. Don't force. + destinationHandler.execute(sqlGenerator.createTable(stream, NO_SUFFIX, false)); + } + final DestinationHandler.InitialRawTableState initialRawTableState = destinationHandler.getInitialRawTableState(stream.id()); + initialRawTableStateByStream.put(stream.id(), initialRawTableState); + + streamsWithSuccessfulSetup.add(Pair.of(stream.id().originalNamespace(), stream.id().originalName())); + + // Use fair locking. This slows down lock operations, but that performance hit is by far dwarfed + // by our IO costs. This lock needs to be fair because the raw table writers are running almost + // constantly, + // and we don't want them to starve T+D. + tdLocks.put(stream.id(), new ReentrantReadWriteLock(true)); + // This lock doesn't need to be fair; any T+D instance is equivalent and we'll skip T+D if we can't + // immediately acquire the lock. + internalTdLocks.put(stream.id(), new ReentrantLock()); + + return Optional.empty(); + } catch (final Exception e) { + LOGGER.error("Exception occurred while preparing tables for stream " + stream.id().originalName(), e); + return Optional.of(e); + } + }, this.executorService); + } + + public void typeAndDedupe(final String originalNamespace, final String originalName, final boolean mustRun) throws Exception { + final var streamConfig = parsedCatalog.getStream(originalNamespace, originalName); + final CompletableFuture> task = typeAndDedupeTask(streamConfig, mustRun); + reduceExceptions( + singleton(task), + String.format( + "The Following Exceptions were thrown while typing and deduping %s.%s:\n", + originalNamespace, + originalName)); + } + + @Override + public Lock getRawTableInsertLock(final String originalNamespace, final String originalName) { + final var streamConfig = parsedCatalog.getStream(originalNamespace, originalName); + return tdLocks.get(streamConfig.id()).readLock(); + } + + private boolean streamSetupSucceeded(final StreamConfig streamConfig) { + final var originalNamespace = streamConfig.id().originalNamespace(); + final var originalName = streamConfig.id().originalName(); + if (!streamsWithSuccessfulSetup.contains(Pair.of(originalNamespace, originalName))) { + // For example, if T+D setup fails, but the consumer tries to run T+D on all streams during close, + // we should skip it. + LOGGER.warn("Skipping typing and deduping for {}.{} because we could not set up the tables for this stream.", originalNamespace, + originalName); + return false; + } + return true; + } + + public CompletableFuture> typeAndDedupeTask(final StreamConfig streamConfig, final boolean mustRun) { + return CompletableFuture.supplyAsync(() -> { + final var originalNamespace = streamConfig.id().originalNamespace(); + final var originalName = streamConfig.id().originalName(); + try { + if (!streamSetupSucceeded(streamConfig)) { + return Optional.empty(); + } + + final boolean run; + final Lock internalLock = internalTdLocks.get(streamConfig.id()); + if (mustRun) { + // If we must run T+D, then wait until we acquire the lock. + internalLock.lock(); + run = true; + } else { + // Otherwise, try and get the lock. If another thread already has it, then we should noop here. + run = internalLock.tryLock(); + } + + if (run) { + LOGGER.info("Waiting for raw table writes to pause for {}.{}", originalNamespace, originalName); + final Lock externalLock = tdLocks.get(streamConfig.id()).writeLock(); + externalLock.lock(); + try { + final DestinationHandler.InitialRawTableState initialRawTableState = initialRawTableStateByStream.get(streamConfig.id()); + TypeAndDedupeTransaction.executeTypeAndDedupe( + sqlGenerator, + destinationHandler, + streamConfig, + initialRawTableState.maxProcessedTimestamp(), + getFinalTableSuffix(streamConfig.id())); + } finally { + LOGGER.info("Allowing other threads to proceed for {}.{}", originalNamespace, originalName); + externalLock.unlock(); + internalLock.unlock(); + } + } else { + LOGGER.info("Another thread is already trying to run typing and deduping for {}.{}. Skipping it here.", originalNamespace, + originalName); + } + return Optional.empty(); + } catch (final Exception e) { + LOGGER.error("Exception occurred while typing and deduping stream " + originalName, e); + return Optional.of(e); + } + }, this.executorService); + } + + @Override + public void typeAndDedupe(final Map streamSyncSummaries) throws Exception { + LOGGER.info("Typing and deduping all tables"); + final Set>> typeAndDedupeTasks = new HashSet<>(); + parsedCatalog.streams().stream() + .filter(streamConfig -> { + // Skip if stream setup failed. + if (!streamSetupSucceeded(streamConfig)) { + return false; + } + // Skip if we don't have any records for this stream. + final StreamSyncSummary streamSyncSummary = streamSyncSummaries.getOrDefault( + streamConfig.id().asStreamDescriptor(), + StreamSyncSummary.DEFAULT); + final boolean nonzeroRecords = streamSyncSummary.recordsWritten() + .map(r -> r > 0) + // If we didn't track record counts during the sync, assume we had nonzero records for this stream + .orElse(true); + final boolean unprocessedRecordsPreexist = initialRawTableStateByStream.get(streamConfig.id()).hasUnprocessedRecords(); + // If this sync emitted records, or the previous sync left behind some unprocessed records, + // then the raw table has some unprocessed records right now. + // Run T+D if either of those conditions are true. + final boolean shouldRunTypingDeduping = nonzeroRecords || unprocessedRecordsPreexist; + if (!shouldRunTypingDeduping) { + LOGGER.info( + "Skipping typing and deduping for stream {}.{} because it had no records during this sync and no unprocessed records from a previous sync.", + streamConfig.id().originalNamespace(), + streamConfig.id().originalName()); + } + return shouldRunTypingDeduping; + }).forEach(streamConfig -> typeAndDedupeTasks.add(typeAndDedupeTask(streamConfig, true))); + CompletableFuture.allOf(typeAndDedupeTasks.toArray(CompletableFuture[]::new)).join(); + reduceExceptions(typeAndDedupeTasks, "The Following Exceptions were thrown while typing and deduping tables:\n"); + } + + /** + * Does any "end of sync" work. For most streams, this is a noop. + *

      + * For OVERWRITE streams where we're writing to a temp table, this is where we swap the temp table + * into the final table. + */ + @Override + public void commitFinalTables() throws Exception { + LOGGER.info("Committing final tables"); + final Set>> tableCommitTasks = new HashSet<>(); + for (final StreamConfig streamConfig : parsedCatalog.streams()) { + if (!streamsWithSuccessfulSetup.contains(Pair.of(streamConfig.id().originalNamespace(), + streamConfig.id().originalName()))) { + LOGGER.warn("Skipping committing final table for for {}.{} because we could not set up the tables for this stream.", + streamConfig.id().originalNamespace(), streamConfig.id().originalName()); + continue; + } + if (DestinationSyncMode.OVERWRITE.equals(streamConfig.destinationSyncMode())) { + tableCommitTasks.add(commitFinalTableTask(streamConfig)); + } + } + CompletableFuture.allOf(tableCommitTasks.toArray(CompletableFuture[]::new)).join(); + reduceExceptions(tableCommitTasks, "The Following Exceptions were thrown while committing final tables:\n"); + } + + private CompletableFuture> commitFinalTableTask(final StreamConfig streamConfig) { + return CompletableFuture.supplyAsync(() -> { + final StreamId streamId = streamConfig.id(); + final String finalSuffix = getFinalTableSuffix(streamId); + if (!StringUtils.isEmpty(finalSuffix)) { + final Sql overwriteFinalTable = sqlGenerator.overwriteFinalTable(streamId, finalSuffix); + LOGGER.info("Overwriting final table with tmp table for stream {}.{}", streamId.originalNamespace(), streamId.originalName()); + try { + destinationHandler.execute(overwriteFinalTable); + } catch (final Exception e) { + LOGGER.error("Exception Occurred while committing final table for stream " + streamId.originalName(), e); + return Optional.of(e); + } + } + return Optional.empty(); + }, this.executorService); + } + + private String getFinalTableSuffix(final StreamId streamId) { + return overwriteStreamsWithTmpTable.contains(streamId) ? TMP_OVERWRITE_TABLE_SUFFIX : NO_SUFFIX; + } + + @Override + public void cleanup() { + LOGGER.info("Cleaning Up type-and-dedupe thread pool"); + this.executorService.shutdown(); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationHandler.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationHandler.java new file mode 100644 index 000000000000..0b20a1cb02d7 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationHandler.java @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import java.time.Instant; +import java.util.Optional; + +public interface DestinationHandler { + + Optional findExistingTable(StreamId id) throws Exception; + + boolean isFinalTableEmpty(StreamId id) throws Exception; + + /** + * Returns the highest timestamp such that all records with _airbyte_extracted equal to or earlier + * than that timestamp have non-null _airbyte_loaded_at. + *

      + * If the raw table is empty or does not exist, return an empty optional. + */ + InitialRawTableState getInitialRawTableState(StreamId id) throws Exception; + + record InitialRawTableState(boolean hasUnprocessedRecords, Optional maxProcessedTimestamp) {} + + void execute(final Sql sql) throws Exception; + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2Migrator.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2Migrator.java similarity index 98% rename from airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2Migrator.java rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2Migrator.java index bfe3973e7d31..7e28906673a6 100644 --- a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2Migrator.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2Migrator.java @@ -20,6 +20,6 @@ void migrateIfNecessary( final SqlGenerator sqlGenerator, final DestinationHandler destinationHandler, final StreamConfig streamConfig) - throws TableNotMigratedException, UnexpectedSchemaException; + throws TableNotMigratedException, UnexpectedSchemaException, Exception; } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/FutureUtils.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/FutureUtils.java new file mode 100644 index 000000000000..349437e4acec --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/FutureUtils.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import io.airbyte.cdk.integrations.util.ConnectorExceptionUtil; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; + +public class FutureUtils { + + /** + * Allow for configuring the number of typing and deduping threads via an enviornment variable in + * the destination container. + * + * @return the number of threads to use in the typing and deduping pool + */ + public static int countOfTypingDedupingThreads(final int defaultThreads) { + return Optional.ofNullable(System.getenv("TD_THREADS")) + .map(Integer::valueOf) + .orElse(defaultThreads); + } + + /** + * Log all exceptions from a list of futures, and rethrow the first exception if there is one. This + * mimics the behavior of running the futures in serial, where the first failure + */ + public static void reduceExceptions(final Collection>> potentialExceptions, final String initialMessage) + throws Exception { + final List exceptions = potentialExceptions.stream() + .map(CompletableFuture::join) + .filter(Optional::isPresent) + .map(Optional::get) + .toList(); + ConnectorExceptionUtil.logAllAndThrowFirst(initialMessage, exceptions); + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NamespacedTableName.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NamespacedTableName.java similarity index 100% rename from airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NamespacedTableName.java rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NamespacedTableName.java diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpDestinationV1V2Migrator.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpDestinationV1V2Migrator.java similarity index 100% rename from airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpDestinationV1V2Migrator.java rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpDestinationV1V2Migrator.java diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java new file mode 100644 index 000000000000..f35d1a92356d --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoOpTyperDeduperWithV1V2Migrations.java @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import static io.airbyte.cdk.integrations.base.IntegrationRunner.TYPE_AND_DEDUPE_THREAD_NAME; +import static io.airbyte.integrations.base.destination.typing_deduping.FutureUtils.countOfTypingDedupingThreads; +import static io.airbyte.integrations.base.destination.typing_deduping.FutureUtils.reduceExceptions; + +import com.google.common.collect.Streams; +import io.airbyte.cdk.integrations.destination.StreamSyncSummary; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.Lock; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.concurrent.BasicThreadFactory; + +/** + * This is a NoOp implementation which skips and Typing and Deduping operations and does not emit + * the final tables. However, this implementation still performs V1->V2 migrations and V2 + * json->string migrations in the raw tables. + */ +@Slf4j +public class NoOpTyperDeduperWithV1V2Migrations implements TyperDeduper { + + private final DestinationV1V2Migrator v1V2Migrator; + private final V2TableMigrator v2TableMigrator; + private final ExecutorService executorService; + private final ParsedCatalog parsedCatalog; + private final SqlGenerator sqlGenerator; + private final DestinationHandler destinationHandler; + + public NoOpTyperDeduperWithV1V2Migrations(final SqlGenerator sqlGenerator, + final DestinationHandler destinationHandler, + final ParsedCatalog parsedCatalog, + final DestinationV1V2Migrator v1V2Migrator, + final V2TableMigrator v2TableMigrator, + final int defaultThreadCount) { + this.sqlGenerator = sqlGenerator; + this.destinationHandler = destinationHandler; + this.parsedCatalog = parsedCatalog; + this.v1V2Migrator = v1V2Migrator; + this.v2TableMigrator = v2TableMigrator; + this.executorService = Executors.newFixedThreadPool(countOfTypingDedupingThreads(defaultThreadCount), + new BasicThreadFactory.Builder().namingPattern(TYPE_AND_DEDUPE_THREAD_NAME).build()); + } + + private void prepareSchemas(final ParsedCatalog parsedCatalog) throws Exception { + final var rawSchema = parsedCatalog.streams().stream().map(stream -> stream.id().rawNamespace()); + final var finalSchema = parsedCatalog.streams().stream().map(stream -> stream.id().finalNamespace()); + final var createAllSchemasSql = Streams.concat(rawSchema, finalSchema) + .filter(Objects::nonNull) + .distinct() + .map(sqlGenerator::createSchema) + .toList(); + destinationHandler.execute(Sql.concat(createAllSchemasSql)); + } + + @Override + public void prepareTables() throws Exception { + log.info("ensuring schemas exist for prepareTables with V1V2 migrations"); + prepareSchemas(parsedCatalog); + final Set>> prepareTablesTasks = new HashSet<>(); + for (final StreamConfig stream : parsedCatalog.streams()) { + prepareTablesTasks.add(CompletableFuture.supplyAsync(() -> { + // Migrate the Raw Tables if this is the first v2 sync after a v1 sync + try { + log.info("Migrating V1->V2 for stream {}", stream.id()); + v1V2Migrator.migrateIfNecessary(sqlGenerator, destinationHandler, stream); + log.info("Migrating V2 legacy for stream {}", stream.id()); + v2TableMigrator.migrateIfNecessary(stream); + return Optional.empty(); + } catch (final Exception e) { + return Optional.of(e); + } + }, executorService)); + } + CompletableFuture.allOf(prepareTablesTasks.toArray(CompletableFuture[]::new)).join(); + reduceExceptions(prepareTablesTasks, "The following exceptions were thrown attempting to prepare tables:\n"); + } + + @Override + public void typeAndDedupe(final String originalNamespace, final String originalName, final boolean mustRun) { + log.info("Skipping TypeAndDedupe"); + } + + @Override + public Lock getRawTableInsertLock(final String originalNamespace, final String originalName) { + return new Lock() { + + @Override + public void lock() { + + } + + @Override + public void lockInterruptibly() { + + } + + @Override + public boolean tryLock() { + // To mimic NoOp behavior always return true that lock is acquired + return true; + } + + @Override + public boolean tryLock(final long time, final TimeUnit unit) { + // To mimic NoOp behavior always return true that lock is acquired + return true; + } + + @Override + public void unlock() { + + } + + @Override + public Condition newCondition() { + // Always throw exception to avoid callers from using this path + throw new UnsupportedOperationException("This lock implementation does not support retrieving a Condition"); + } + + }; + } + + @Override + public void typeAndDedupe(final Map streamSyncSummaries) { + log.info("Skipping TypeAndDedupe final"); + } + + @Override + public void commitFinalTables() { + log.info("Skipping commitFinalTables final"); + } + + @Override + public void cleanup() { + log.info("Cleaning Up type-and-dedupe thread pool"); + this.executorService.shutdown(); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoopTyperDeduper.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoopTyperDeduper.java new file mode 100644 index 000000000000..af8529e3d2b2 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoopTyperDeduper.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import io.airbyte.cdk.integrations.destination.StreamSyncSummary; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.Lock; + +public class NoopTyperDeduper implements TyperDeduper { + + @Override + public void prepareTables() { + + } + + @Override + public void typeAndDedupe(final String originalNamespace, final String originalName, final boolean mustRun) { + + } + + @Override + public Lock getRawTableInsertLock(final String originalNamespace, final String originalName) { + // Return a fake lock that does nothing. + return new Lock() { + + @Override + public void lock() { + + } + + @Override + public void lockInterruptibly() { + + } + + @Override + public boolean tryLock() { + // To mimic NoOp behavior always return true that lock is acquired + return true; + } + + @Override + public boolean tryLock(final long time, final TimeUnit unit) { + // To mimic NoOp behavior always return true that lock is acquired + return true; + } + + @Override + public void unlock() { + + } + + @Override + public Condition newCondition() { + return null; + } + + }; + } + + @Override + public void commitFinalTables() { + + } + + @Override + public void typeAndDedupe(final Map streamSyncSummaries) { + + } + + @Override + public void cleanup() { + + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoopV2TableMigrator.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoopV2TableMigrator.java new file mode 100644 index 000000000000..f2f2a9c41497 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoopV2TableMigrator.java @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +public class NoopV2TableMigrator implements V2TableMigrator { + + @Override + public void migrateIfNecessary(final StreamConfig streamConfig) { + // do nothing + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/ParsedCatalog.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/ParsedCatalog.java similarity index 100% rename from airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/ParsedCatalog.java rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/ParsedCatalog.java diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Sql.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Sql.java new file mode 100644 index 000000000000..f9744e34da96 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Sql.java @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; +import org.elasticsearch.common.Strings; + +/** + * Represents a list of SQL transactions, where each transaction consists of one or more SQL + * statements. Each transaction MUST NOT contain the BEGIN/COMMIT statements. Each inner list is a + * single transaction, and each String is a single statement within that transaction. + *

      + * Most callers likely only need a single transaction, but e.g. BigQuery disallows running DDL + * inside transactions, and so needs to run sequential "CREATE SCHEMA", "CREATE TABLE" as separate + * transactions. + *

      + * Callers are encouraged to use the static factory methods instead of the public constructor. + */ +public record Sql(List> transactions) { + + public Sql { + transactions.forEach(transaction -> { + if (transaction.isEmpty()) { + throw new IllegalArgumentException("Transaction must not be empty"); + } + if (transaction.stream().anyMatch(Strings::isNullOrEmpty)) { + throw new IllegalArgumentException("Transaction must not contain empty statements"); + } + }); + } + + /** + * @param begin The SQL statement to start a transaction, typically "BEGIN" + * @param commit The SQL statement to commit a transaction, typically "COMMIT" + * @return A list of SQL strings, each of which represents a transaction. + */ + public List asSqlStrings(final String begin, final String commit) { + return transactions().stream() + .map(transaction -> { + // If there's only one statement, we don't need to wrap it in a transaction. + if (transaction.size() == 1) { + return transaction.get(0); + } + final StringBuilder builder = new StringBuilder(); + builder.append(begin); + builder.append(";\n"); + transaction.forEach(statement -> { + builder.append(statement); + // No semicolon - statements already end with a semicolon + builder.append("\n"); + }); + builder.append(commit); + builder.append(";\n"); + return builder.toString(); + }).toList(); + } + + /** + * Execute a list of SQL statements in a single transaction. + */ + public static Sql transactionally(final List statements) { + return create(List.of(statements)); + } + + public static Sql transactionally(final String... statements) { + return transactionally(Stream.of(statements).toList()); + } + + /** + * Execute each statement as its own transaction. + */ + public static Sql separately(final List statements) { + return create(statements.stream().map(Collections::singletonList).toList()); + } + + public static Sql separately(final String... statements) { + return separately(Stream.of(statements).toList()); + } + + /** + * Convenience method for indicating intent. Equivalent to calling + * {@link #transactionally(String...)} or {@link #separately(String...)} with the same string. + */ + public static Sql of(final String statement) { + return transactionally(statement); + } + + public static Sql concat(final Sql... sqls) { + return create(Stream.of(sqls).flatMap(sql -> sql.transactions.stream()).toList()); + } + + public static Sql concat(final List sqls) { + return create(sqls.stream().flatMap(sql -> sql.transactions.stream()).toList()); + } + + /** + * Utility method to create a Sql object without empty statements/transactions, and appending + * semicolons when needed. + */ + public static Sql create(final List> transactions) { + return new Sql(transactions.stream() + .map(transaction -> transaction.stream() + .filter(statement -> !Strings.isNullOrEmpty(statement)) + .map(statement -> { + if (!statement.trim().endsWith(";")) { + return statement + ";"; + } + return statement; + }) + .toList()) + .filter(transaction -> !transaction.isEmpty()) + .toList()); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/SqlGenerator.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/SqlGenerator.java new file mode 100644 index 000000000000..568fd688e9bb --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/SqlGenerator.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import static io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeTransaction.SOFT_RESET_SUFFIX; + +import java.time.Instant; +import java.util.Optional; + +public interface SqlGenerator { + + StreamId buildStreamId(String namespace, String name, String rawNamespaceOverride); + + default ColumnId buildColumnId(final String name) { + return buildColumnId(name, ""); + } + + ColumnId buildColumnId(String name, String suffix); + + /** + * Generate a SQL statement to create a fresh table to match the given stream. + *

      + * The generated SQL should throw an exception if the table already exists and {@code force} is + * false. Callers should use + * {@link #existingSchemaMatchesStreamConfig(StreamConfig, java.lang.Object)} if the table is known + * to exist, and potentially softReset + * + * @param suffix A suffix to add to the stream name. Useful for full refresh overwrite syncs, where + * we write the entire sync to a temp table. + * @param force If true, will overwrite an existing table. If false, will throw an exception if the + * table already exists. If you're passing a non-empty prefix, you likely want to set this to + * true. + */ + Sql createTable(final StreamConfig stream, final String suffix, boolean force); + + /** + * Used to create either the airbyte_internal or final schemas if they don't exist + * + * @param schema the schema to create + * @return SQL to create the schema if it does not exist + */ + Sql createSchema(final String schema); + + /** + * Check the final table's schema and compare it to what the stream config would generate. + * + * @param stream the stream/stable in question + * @param existingTable the existing table mapped to the stream + * @return whether the existing table matches the expected schema + */ + boolean existingSchemaMatchesStreamConfig(final StreamConfig stream, final DialectTableDefinition existingTable); + + /** + * Generate a SQL statement to copy new data from the raw table into the final table. + *

      + * Responsible for: + *

        + *
      • Pulling new raw records from a table (i.e. records with null _airbyte_loaded_at)
      • + *
      • Extracting the JSON fields and casting to the appropriate types
      • + *
      • Handling errors in those casts
      • + *
      • Merging those typed records into an existing table
      • + *
      • Updating the raw records with SET _airbyte_loaded_at = now()
      • + *
      + *

      + * Implementing classes are recommended to break this into smaller methods, which can be tested in + * isolation. However, this interface only requires a single mega-method. + * + * @param finalSuffix the suffix of the final table to write to. If empty string, writes to the + * final table directly. Useful for full refresh overwrite syncs, where we write the entire + * sync to a temp table and then swap it into the final table at the end. + * + * @param minRawTimestamp The latest _airbyte_extracted_at for which all raw records with that + * timestamp have already been typed+deduped. Implementations MAY use this value in a + * {@code _airbyte_extracted_at > minRawTimestamp} filter on the raw table to improve query + * performance. + * @param useExpensiveSaferCasting often the data coming from the source can be faithfully + * represented in the destination without issue, and using a "CAST" expression works fine, + * however sometimes we get badly typed data. In these cases we can use a more expensive + * query which handles casting exceptions. + */ + Sql updateTable(final StreamConfig stream, String finalSuffix, Optional minRawTimestamp, final boolean useExpensiveSaferCasting); + + /** + * Drop the previous final table, and rename the new final table to match the old final table. + *

      + * This method may assume that the stream is an OVERWRITE stream, and that the final suffix is + * non-empty. Callers are responsible for verifying those are true. + */ + Sql overwriteFinalTable(StreamId stream, String finalSuffix); + + /** + * Creates a sql query which will create a v2 raw table from the v1 raw table, then performs a soft + * reset. + * + * @param streamId the stream to migrate + * @param namespace + * @param tableName + * @return a string containing the necessary sql to migrate + */ + Sql migrateFromV1toV2(StreamId streamId, String namespace, String tableName); + + /** + * Typically we need to create a soft reset temporary table and clear loaded at values + * + * @return + */ + default Sql prepareTablesForSoftReset(final StreamConfig stream) { + final Sql createTempTable = createTable(stream, SOFT_RESET_SUFFIX, true); + final Sql clearLoadedAt = clearLoadedAt(stream.id()); + return Sql.concat(createTempTable, clearLoadedAt); + } + + Sql clearLoadedAt(final StreamId streamId); + + /** + * Implementation specific if there is no option to retry again with safe casted SQL or the specific + * cause of the exception can be retried or not. + * + * @return true if the exception should be retried with a safer query + */ + default boolean shouldRetry(final Exception e) { + return true; + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/StreamConfig.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/StreamConfig.java similarity index 100% rename from airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/StreamConfig.java rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/StreamConfig.java diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/StreamId.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/StreamId.java similarity index 94% rename from airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/StreamId.java rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/StreamId.java index 9851ee7b7e59..e65cfa72259c 100644 --- a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/StreamId.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/StreamId.java @@ -5,6 +5,7 @@ package io.airbyte.integrations.base.destination.typing_deduping; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.StreamDescriptor; /** * In general, callers should not directly instantiate this class. Use @@ -56,6 +57,10 @@ public AirbyteStreamNameNamespacePair asPair() { return new AirbyteStreamNameNamespacePair(originalName, originalNamespace); } + public StreamDescriptor asStreamDescriptor() { + return new StreamDescriptor().withNamespace(originalNamespace).withName(originalName); + } + /** * Build the raw table name as namespace + (delimiter) + name. For example, given a stream with * namespace "public__ab" and name "abab_users", we will end up with raw table name diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Struct.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Struct.java new file mode 100644 index 000000000000..c28bfe876166 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Struct.java @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import java.util.LinkedHashMap; + +/** + * @param properties Use LinkedHashMap to preserve insertion order. + */ +public record Struct(LinkedHashMap properties) implements AirbyteType { + + public static final String TYPE = "STRUCT"; + + @Override + public String getTypeName() { + return TYPE; + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TableNotMigratedException.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TableNotMigratedException.java similarity index 100% rename from airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TableNotMigratedException.java rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TableNotMigratedException.java diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValve.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValve.java similarity index 81% rename from airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValve.java rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValve.java index 524c052db0a1..d638125371cd 100644 --- a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValve.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValve.java @@ -4,10 +4,13 @@ package io.airbyte.integrations.base.destination.typing_deduping; +import io.airbyte.cdk.integrations.base.DestinationConfig; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import java.util.List; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Supplier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A slightly more complicated way to keep track of when to perform type and dedupe operations per @@ -15,11 +18,10 @@ */ public class TypeAndDedupeOperationValve extends ConcurrentHashMap { + private static final Logger LOGGER = LoggerFactory.getLogger(TypeAndDedupeOperationValve.class); + private static final long NEGATIVE_MILLIS = -1; - private static final long FIFTEEN_MINUTES_MILLIS = 1000 * 60 * 15; - private static final long ONE_HOUR_MILLIS = 1000 * 60 * 60 * 1; - private static final long TWO_HOURS_MILLIS = 1000 * 60 * 60 * 2; - private static final long FOUR_HOURS_MILLIS = 1000 * 60 * 60 * 4; + private static final long SIX_HOURS_MILLIS = 1000 * 60 * 60 * 6; // New users of airbyte likely want to see data flowing into their tables as soon as possible, and // we want to catch new errors which might appear early within an incremental sync. @@ -28,12 +30,11 @@ public class TypeAndDedupeOperationValve extends ConcurrentHashMap typeAndDedupeIncreasingIntervals = - List.of(NEGATIVE_MILLIS, FIFTEEN_MINUTES_MILLIS, ONE_HOUR_MILLIS, TWO_HOURS_MILLIS, FOUR_HOURS_MILLIS); + public static final List typeAndDedupeIncreasingIntervals = List.of(NEGATIVE_MILLIS, SIX_HOURS_MILLIS); private static final Supplier SYSTEM_NOW = () -> System.currentTimeMillis(); - private ConcurrentHashMap incrementalIndex; + private final ConcurrentHashMap incrementalIndex; private final Supplier nowness; @@ -46,7 +47,7 @@ public TypeAndDedupeOperationValve() { * * @param nownessSupplier Supplier which will return a long value representing now */ - public TypeAndDedupeOperationValve(Supplier nownessSupplier) { + public TypeAndDedupeOperationValve(final Supplier nownessSupplier) { super(); incrementalIndex = new ConcurrentHashMap<>(); this.nowness = nownessSupplier; @@ -70,6 +71,11 @@ public void addStream(final AirbyteStreamNameNamespacePair key) { put(key, nowness.get()); } + public void addStreamIfAbsent(final AirbyteStreamNameNamespacePair key) { + putIfAbsent(key, nowness.get()); + incrementalIndex.putIfAbsent(key, 0); + } + /** * Whether we should type and dedupe at this point in time for this particular stream. * @@ -78,6 +84,10 @@ public void addStream(final AirbyteStreamNameNamespacePair key) { * deduping. */ public boolean readyToTypeAndDedupe(final AirbyteStreamNameNamespacePair key) { + if (!DestinationConfig.getInstance().getBooleanValue("enable_incremental_final_table_updates")) { + LOGGER.info("Skipping Incremental Typing and Deduping"); + return false; + } if (!containsKey(key)) { return false; } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeTransaction.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeTransaction.java new file mode 100644 index 000000000000..a1c1f8cc1684 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeTransaction.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import java.time.Instant; +import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TypeAndDedupeTransaction { + + public static final String SOFT_RESET_SUFFIX = "_ab_soft_reset"; + private static final Logger LOGGER = LoggerFactory.getLogger(TypeAndDedupeTransaction.class); + + /** + * It can be expensive to build the errors array in the airbyte_meta column, so we first attempt an + * 'unsafe' transaction which assumes everything is typed correctly. If that fails, we will run a + * more expensive query which handles casting errors + * + * @param sqlGenerator for generating sql for the destination + * @param destinationHandler for executing sql created + * @param streamConfig which stream to operate on + * @param minExtractedAt to reduce the amount of data in the query + * @param suffix table suffix for temporary tables + * @throws Exception if the safe query fails + */ + public static void executeTypeAndDedupe(final SqlGenerator sqlGenerator, + final DestinationHandler destinationHandler, + final StreamConfig streamConfig, + final Optional minExtractedAt, + final String suffix) + throws Exception { + try { + LOGGER.info("Attempting typing and deduping for {}.{} with suffix {}", streamConfig.id().originalNamespace(), streamConfig.id().originalName(), + suffix); + final Sql unsafeSql = sqlGenerator.updateTable(streamConfig, suffix, minExtractedAt, false); + destinationHandler.execute(unsafeSql); + } catch (final Exception e) { + if (sqlGenerator.shouldRetry(e)) { + // TODO Destination specific non-retryable exceptions should be added. + LOGGER.error("Encountered Exception on unsafe SQL for stream {} {} with suffix {}, attempting with error handling", + streamConfig.id().originalNamespace(), streamConfig.id().originalName(), suffix, e); + final Sql saferSql = sqlGenerator.updateTable(streamConfig, suffix, minExtractedAt, true); + destinationHandler.execute(saferSql); + } else { + LOGGER.error("Encountered Exception on unsafe SQL for stream {} {} with suffix {}, Retry is skipped", + streamConfig.id().originalNamespace(), streamConfig.id().originalName(), suffix, e); + throw e; + } + } + } + + /** + * Everything in + * {@link TypeAndDedupeTransaction#executeTypeAndDedupe(SqlGenerator, DestinationHandler, StreamConfig, Optional, String)} + * but with a little extra prep work for the soft reset temp tables + * + * @param sqlGenerator for generating sql for the destination + * @param destinationHandler for executing sql created + * @param streamConfig which stream to operate on + * @throws Exception if the safe query fails + */ + public static void executeSoftReset(final SqlGenerator sqlGenerator, final DestinationHandler destinationHandler, final StreamConfig streamConfig) + throws Exception { + LOGGER.info("Attempting soft reset for stream {} {}", streamConfig.id().originalNamespace(), streamConfig.id().originalName()); + destinationHandler.execute(sqlGenerator.prepareTablesForSoftReset(streamConfig)); + executeTypeAndDedupe(sqlGenerator, destinationHandler, streamConfig, Optional.empty(), SOFT_RESET_SUFFIX); + destinationHandler.execute(sqlGenerator.overwriteFinalTable(streamConfig.id(), SOFT_RESET_SUFFIX)); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduper.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduper.java new file mode 100644 index 000000000000..263c9a11742c --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduper.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import io.airbyte.cdk.integrations.destination.StreamSyncSummary; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.Map; +import java.util.concurrent.locks.Lock; + +public interface TyperDeduper { + + /** + * Create the tables that T+D will write to during the sync. In OVERWRITE mode, these might not be + * the true final tables. Specifically, other than an initial sync (i.e. table does not exist, or is + * empty) we write to a temporary final table, and swap it into the true final table at the end of + * the sync. This is to prevent user downtime during a sync. + */ + void prepareTables() throws Exception; + + /** + * Suggest that we execute typing and deduping for a single stream (i.e. fetch new raw records into + * the final table, etc.). + *

      + * This method is thread-safe; multiple threads can call it concurrently. If T+D is already running + * for the given stream, this method may choose to do nothing. If a caller wishes to force T+D to + * run (for example, at the end of a sync), they may set {@code mustRun} to true. + *

      + * This method relies on callers to prevent concurrent modification to the underlying raw tables. + * This is most easily accomplished using {@link #getRawTableInsertLock(String, String)}, if the + * caller guards all raw table writes using {@code getRawTableInsertLock().lock()} and + * {@code getRawTableInsertLock().unlock()}. While {@code typeAndDedupe} is executing, that lock + * will be unavailable. However, callers are free to enforce this in other ways (for example, + * single- threaded callers do not need to use the lock). + * + * @param originalNamespace The stream's namespace, as declared in the configured catalog + * @param originalName The stream's name, as declared in the configured catalog + */ + void typeAndDedupe(String originalNamespace, String originalName, boolean mustRun) throws Exception; + + /** + * Get the lock that should be used to synchronize inserts to the raw table for a given stream. This + * lock permits any number of threads to hold the lock, but + * {@link #typeAndDedupe(String, String, boolean)} will not proceed while this lock is held. + *

      + * This lock provides fairness guarantees, i.e. typeAndDedupe will not starve while waiting for the + * lock (and similarly, raw table writers will not starve if many typeAndDedupe calls are queued). + */ + Lock getRawTableInsertLock(final String originalNamespace, final String originalName); + + /** + * Does any "end of sync" work. For most streams, this is a noop. + *

      + * For OVERWRITE streams where we're writing to a temp table, this is where we swap the temp table + * into the final table. + * + * @param streamSyncSummaries Information about what happened during the sync. Implementations + * SHOULD use this information to skip T+D when possible (this is not a requirement for + * correctness, but does allow us to save time/money). This parameter MUST NOT be null. + * Streams MAY be omitted, which will be treated as though they were mapped to + * {@link StreamSyncSummary#DEFAULT}. + */ + void typeAndDedupe(Map streamSyncSummaries) throws Exception; + + void commitFinalTables() throws Exception; + + void cleanup(); + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/UnexpectedSchemaException.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/UnexpectedSchemaException.java similarity index 100% rename from airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/UnexpectedSchemaException.java rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/UnexpectedSchemaException.java diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Union.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Union.java similarity index 95% rename from airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Union.java rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Union.java index e8b62dc36eed..c50e22357698 100644 --- a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Union.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Union.java @@ -24,6 +24,8 @@ */ public record Union(List options) implements AirbyteType { + public static final String TYPE = "UNION"; + /** * This is a hack to handle weird schemas like {type: [object, string]}. If a stream's top-level * schema looks like this, we still want to be able to extract the object properties (i.e. treat it @@ -62,4 +64,9 @@ public AirbyteType chooseType() { return options.stream().min(comparator).orElse(AirbyteProtocolType.UNKNOWN); } + @Override + public String getTypeName() { + return TYPE; + } + } diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/UnsupportedOneOf.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/UnsupportedOneOf.java similarity index 75% rename from airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/UnsupportedOneOf.java rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/UnsupportedOneOf.java index 3d3c84636a3c..417e7f5cc5c9 100644 --- a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/UnsupportedOneOf.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/UnsupportedOneOf.java @@ -13,4 +13,11 @@ */ public record UnsupportedOneOf(List options) implements AirbyteType { + public static final String TYPE = "UNSUPPORTED_ONE_OF"; + + @Override + public String getTypeName() { + return TYPE; + } + } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/V2TableMigrator.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/V2TableMigrator.java new file mode 100644 index 000000000000..27b056d5d399 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/V2TableMigrator.java @@ -0,0 +1,11 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +public interface V2TableMigrator { + + void migrateIfNecessary(final StreamConfig streamConfig) throws Exception; + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteTypeTest.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteTypeTest.java similarity index 100% rename from airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteTypeTest.java rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/AirbyteTypeTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParserTest.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParserTest.java new file mode 100644 index 000000000000..3922f8ebe4bf --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParserTest.java @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.mockito.Mockito.*; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import java.util.List; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class CatalogParserTest { + + private SqlGenerator sqlGenerator; + private CatalogParser parser; + + @BeforeEach + public void setup() { + sqlGenerator = mock(SqlGenerator.class); + // noop quoting logic + when(sqlGenerator.buildColumnId(any())).thenAnswer(invocation -> { + final String fieldName = invocation.getArgument(0); + return new ColumnId(fieldName, fieldName, fieldName); + }); + when(sqlGenerator.buildStreamId(any(), any(), any())).thenAnswer(invocation -> { + final String namespace = invocation.getArgument(0); + final String name = invocation.getArgument(1); + final String rawNamespace = invocation.getArgument(1); + return new StreamId(namespace, name, rawNamespace, namespace + "_abab_" + name, namespace, name); + }); + + parser = new CatalogParser(sqlGenerator); + } + + /** + * Both these streams will write to the same final table name ("foofoo"). Verify that they don't + * actually use the same tablename. + */ + @Test + public void finalNameCollision() { + when(sqlGenerator.buildStreamId(any(), any(), any())).thenAnswer(invocation -> { + final String originalNamespace = invocation.getArgument(0); + final String originalName = (invocation.getArgument(1)); + final String originalRawNamespace = (invocation.getArgument(1)); + + // emulate quoting logic that causes a name collision + final String quotedName = originalName.replaceAll("bar", ""); + return new StreamId(originalNamespace, quotedName, originalRawNamespace, originalNamespace + "_abab_" + quotedName, originalNamespace, + originalName); + }); + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + stream("a", "foobarfoo"), + stream("a", "foofoo"))); + + final ParsedCatalog parsedCatalog = parser.parseCatalog(catalog); + + assertNotEquals( + parsedCatalog.streams().get(0).id().finalName(), + parsedCatalog.streams().get(1).id().finalName()); + } + + /** + * The schema contains two fields, which will both end up named "foofoo" after quoting. Verify that + * they don't actually use the same column name. + */ + @Test + public void columnNameCollision() { + when(sqlGenerator.buildColumnId(any(), any())).thenAnswer(invocation -> { + final String originalName = invocation.getArgument(0); + + // emulate quoting logic that causes a name collision + final String quotedName = originalName.replaceAll("bar", ""); + return new ColumnId(quotedName, originalName, quotedName); + }); + final JsonNode schema = Jsons.deserialize(""" + { + "type": "object", + "properties": { + "foobarfoo": {"type": "string"}, + "foofoo": {"type": "string"} + } + } + """); + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of(stream("a", "a", schema))); + + final ParsedCatalog parsedCatalog = parser.parseCatalog(catalog); + + assertEquals(2, parsedCatalog.streams().get(0).columns().size()); + } + + private static ConfiguredAirbyteStream stream(final String namespace, final String name) { + return stream( + namespace, + name, + Jsons.deserialize(""" + { + "type": "object", + "properties": { + "name": {"type": "string"} + } + } + """)); + } + + private static ConfiguredAirbyteStream stream(final String namespace, final String name, final JsonNode schema) { + return new ConfiguredAirbyteStream().withStream( + new AirbyteStream() + .withNamespace(namespace) + .withName(name) + .withJsonSchema(schema)); + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/CollectionUtilsTest.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/CollectionUtilsTest.java similarity index 100% rename from airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/CollectionUtilsTest.java rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/CollectionUtilsTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduperTest.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduperTest.java new file mode 100644 index 000000000000..c81629611501 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduperTest.java @@ -0,0 +1,277 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import static io.airbyte.integrations.base.destination.typing_deduping.Sql.separately; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.clearInvocations; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.ignoreStubs; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; + +import io.airbyte.cdk.integrations.destination.StreamSyncSummary; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class DefaultTyperDeduperTest { + + private MockSqlGenerator sqlGenerator; + private DestinationHandler destinationHandler; + + private DestinationV1V2Migrator migrator; + private TyperDeduper typerDeduper; + + @BeforeEach + void setup() throws Exception { + sqlGenerator = spy(new MockSqlGenerator()); + destinationHandler = mock(DestinationHandler.class); + when(destinationHandler.getInitialRawTableState(any())).thenReturn(new DestinationHandler.InitialRawTableState(true, Optional.empty())); + migrator = new NoOpDestinationV1V2Migrator<>(); + + final ParsedCatalog parsedCatalog = new ParsedCatalog(List.of( + new StreamConfig( + new StreamId("overwrite_ns", "overwrite_stream", null, null, "overwrite_ns", "overwrite_stream"), + null, + DestinationSyncMode.OVERWRITE, + null, + null, + null), + new StreamConfig( + new StreamId("append_ns", "append_stream", null, null, "append_ns", "append_stream"), + null, + DestinationSyncMode.APPEND, + null, + null, + null), + new StreamConfig( + new StreamId("dedup_ns", "dedup_stream", null, null, "dedup_ns", "dedup_stream"), + null, + DestinationSyncMode.APPEND_DEDUP, + null, + null, + null))); + + typerDeduper = new DefaultTyperDeduper<>(sqlGenerator, destinationHandler, parsedCatalog, migrator, 1); + } + + /** + * When there are no existing tables, we should create them and write to them directly. + */ + @Test + void emptyDestination() throws Exception { + when(destinationHandler.findExistingTable(any())).thenReturn(Optional.empty()); + + typerDeduper.prepareTables(); + verify(destinationHandler).execute(separately("CREATE SCHEMA overwrite_ns", "CREATE SCHEMA append_ns", "CREATE SCHEMA dedup_ns")); + verify(destinationHandler).execute(Sql.of("CREATE TABLE overwrite_ns.overwrite_stream")); + verify(destinationHandler).execute(Sql.of("CREATE TABLE append_ns.append_stream")); + verify(destinationHandler).execute(Sql.of("CREATE TABLE dedup_ns.dedup_stream")); + verifyNoMoreInteractions(ignoreStubs(destinationHandler)); + clearInvocations(destinationHandler); + + typerDeduper.typeAndDedupe("overwrite_ns", "overwrite_stream", false); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE overwrite_ns.overwrite_stream WITHOUT SAFER CASTING")); + typerDeduper.typeAndDedupe("append_ns", "append_stream", false); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE append_ns.append_stream WITHOUT SAFER CASTING")); + typerDeduper.typeAndDedupe("dedup_ns", "dedup_stream", false); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE dedup_ns.dedup_stream WITHOUT SAFER CASTING")); + verifyNoMoreInteractions(ignoreStubs(destinationHandler)); + clearInvocations(destinationHandler); + + typerDeduper.commitFinalTables(); + verify(destinationHandler, never()).execute(any()); + } + + /** + * When there's an existing table but it's empty, we should ensure it has the right schema and write + * to it directly. + */ + @Test + void existingEmptyTable() throws Exception { + when(destinationHandler.findExistingTable(any())).thenReturn(Optional.of("foo")); + when(destinationHandler.isFinalTableEmpty(any())).thenReturn(true); + when(sqlGenerator.existingSchemaMatchesStreamConfig(any(), any())).thenReturn(false); + typerDeduper.prepareTables(); + verify(destinationHandler).execute(separately("CREATE SCHEMA overwrite_ns", "CREATE SCHEMA append_ns", "CREATE SCHEMA dedup_ns")); + verify(destinationHandler).execute(Sql.of("CREATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp")); + verify(destinationHandler).execute(Sql.of("PREPARE append_ns.append_stream FOR SOFT RESET")); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE append_ns.append_stream_ab_soft_reset WITHOUT SAFER CASTING")); + verify(destinationHandler).execute(Sql.of("OVERWRITE TABLE append_ns.append_stream FROM append_ns.append_stream_ab_soft_reset")); + verify(destinationHandler).execute(Sql.of("PREPARE dedup_ns.dedup_stream FOR SOFT RESET")); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE dedup_ns.dedup_stream_ab_soft_reset WITHOUT SAFER CASTING")); + verify(destinationHandler).execute(Sql.of("OVERWRITE TABLE dedup_ns.dedup_stream FROM dedup_ns.dedup_stream_ab_soft_reset")); + verifyNoMoreInteractions(ignoreStubs(destinationHandler)); + clearInvocations(destinationHandler); + + typerDeduper.typeAndDedupe("overwrite_ns", "overwrite_stream", false); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp WITHOUT SAFER CASTING")); + typerDeduper.typeAndDedupe("append_ns", "append_stream", false); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE append_ns.append_stream WITHOUT SAFER CASTING")); + typerDeduper.typeAndDedupe("dedup_ns", "dedup_stream", false); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE dedup_ns.dedup_stream WITHOUT SAFER CASTING")); + verifyNoMoreInteractions(ignoreStubs(destinationHandler)); + clearInvocations(destinationHandler); + + typerDeduper.commitFinalTables(); + verify(destinationHandler).execute(Sql.of("OVERWRITE TABLE overwrite_ns.overwrite_stream FROM overwrite_ns.overwrite_stream_airbyte_tmp")); + verifyNoMoreInteractions(ignoreStubs(destinationHandler)); + } + + /** + * When there's an existing empty table with the right schema, we don't need to do anything during + * setup. + */ + @Test + void existingEmptyTableMatchingSchema() throws Exception { + when(destinationHandler.findExistingTable(any())).thenReturn(Optional.of("foo")); + when(destinationHandler.isFinalTableEmpty(any())).thenReturn(true); + when(sqlGenerator.existingSchemaMatchesStreamConfig(any(), any())).thenReturn(true); + + typerDeduper.prepareTables(); + verify(destinationHandler).execute(separately("CREATE SCHEMA overwrite_ns", "CREATE SCHEMA append_ns", "CREATE SCHEMA dedup_ns")); + clearInvocations(destinationHandler); + verify(destinationHandler, never()).execute(any()); + } + + /** + * When there's an existing nonempty table, we should alter it. For the OVERWRITE stream, we also + * need to write to a tmp table, and overwrite the real table at the end of the sync. + */ + @Test + void existingNonemptyTable() throws Exception { + when(destinationHandler.getInitialRawTableState(any())) + .thenReturn(new DestinationHandler.InitialRawTableState(true, Optional.of(Instant.parse("2023-01-01T12:34:56Z")))); + when(destinationHandler.findExistingTable(any())).thenReturn(Optional.of("foo")); + when(destinationHandler.isFinalTableEmpty(any())).thenReturn(false); + + typerDeduper.prepareTables(); + verify(destinationHandler).execute(separately("CREATE SCHEMA overwrite_ns", "CREATE SCHEMA append_ns", "CREATE SCHEMA dedup_ns")); + // NB: We only create a tmp table for the overwrite stream, and do _not_ soft reset the existing + // overwrite stream's table. + + verify(destinationHandler).execute(Sql.of("CREATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp")); + verify(destinationHandler).execute(Sql.of("PREPARE append_ns.append_stream FOR SOFT RESET")); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE append_ns.append_stream_ab_soft_reset WITHOUT SAFER CASTING")); + verify(destinationHandler).execute(Sql.of("OVERWRITE TABLE append_ns.append_stream FROM append_ns.append_stream_ab_soft_reset")); + verify(destinationHandler).execute(Sql.of("PREPARE dedup_ns.dedup_stream FOR SOFT RESET")); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE dedup_ns.dedup_stream_ab_soft_reset WITHOUT SAFER CASTING")); + verify(destinationHandler).execute(Sql.of("OVERWRITE TABLE dedup_ns.dedup_stream FROM dedup_ns.dedup_stream_ab_soft_reset")); + verifyNoMoreInteractions(ignoreStubs(destinationHandler)); + clearInvocations(destinationHandler); + + typerDeduper.typeAndDedupe("overwrite_ns", "overwrite_stream", false); + // NB: no airbyte_tmp suffix on the non-overwrite streams + verify(destinationHandler) + .execute(Sql.of("UPDATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp WITHOUT SAFER CASTING WHERE extracted_at > 2023-01-01T12:34:56Z")); + typerDeduper.typeAndDedupe("append_ns", "append_stream", false); + verify(destinationHandler) + .execute(Sql.of("UPDATE TABLE append_ns.append_stream WITHOUT SAFER CASTING WHERE extracted_at > 2023-01-01T12:34:56Z")); + typerDeduper.typeAndDedupe("dedup_ns", "dedup_stream", false); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE dedup_ns.dedup_stream WITHOUT SAFER CASTING WHERE extracted_at > 2023-01-01T12:34:56Z")); + verifyNoMoreInteractions(ignoreStubs(destinationHandler)); + clearInvocations(destinationHandler); + + typerDeduper.commitFinalTables(); + verify(destinationHandler).execute(Sql.of("OVERWRITE TABLE overwrite_ns.overwrite_stream FROM overwrite_ns.overwrite_stream_airbyte_tmp")); + verifyNoMoreInteractions(ignoreStubs(destinationHandler)); + } + + /** + * When there's an existing nonempty table with the right schema, we don't need to modify it, but + * OVERWRITE streams still need to create a tmp table. + */ + @Test + void existingNonemptyTableMatchingSchema() throws Exception { + when(destinationHandler.getInitialRawTableState(any())).thenReturn(new DestinationHandler.InitialRawTableState(true, Optional.of(Instant.now()))); + when(destinationHandler.findExistingTable(any())).thenReturn(Optional.of("foo")); + when(destinationHandler.isFinalTableEmpty(any())).thenReturn(false); + when(sqlGenerator.existingSchemaMatchesStreamConfig(any(), any())).thenReturn(true); + + typerDeduper.prepareTables(); + // NB: We only create one tmp table here. + // Also, we need to alter the existing _real_ table, not the tmp table! + verify(destinationHandler).execute(separately("CREATE SCHEMA overwrite_ns", "CREATE SCHEMA append_ns", "CREATE SCHEMA dedup_ns")); + verify(destinationHandler).execute(Sql.of("CREATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp")); + verifyNoMoreInteractions(ignoreStubs(destinationHandler)); + } + + @Test + void nonexistentStream() { + assertThrows(IllegalArgumentException.class, + () -> typerDeduper.typeAndDedupe("nonexistent_ns", "nonexistent_stream", false)); + verifyNoInteractions(ignoreStubs(destinationHandler)); + } + + @Test + void failedSetup() throws Exception { + doThrow(new RuntimeException("foo")).when(destinationHandler).execute(any()); + + assertThrows(Exception.class, () -> typerDeduper.prepareTables()); + clearInvocations(destinationHandler); + + typerDeduper.typeAndDedupe("dedup_ns", "dedup_stream", false); + typerDeduper.commitFinalTables(); + + verifyNoInteractions(ignoreStubs(destinationHandler)); + } + + /** + * Test a typical sync, where the previous sync left no unprocessed raw records. If this sync writes + * some records for a stream, we should run T+D for that stream. + */ + @Test + void noUnprocessedRecords() throws Exception { + when(destinationHandler.getInitialRawTableState(any())).thenReturn(new DestinationHandler.InitialRawTableState(false, Optional.empty())); + typerDeduper.prepareTables(); + clearInvocations(destinationHandler); + + typerDeduper.typeAndDedupe(Map.of( + new StreamDescriptor().withName("overwrite_stream").withNamespace("overwrite_ns"), new StreamSyncSummary(Optional.of(0L)), + new StreamDescriptor().withName("append_stream").withNamespace("append_ns"), new StreamSyncSummary(Optional.of(1L)))); + + // append_stream and dedup_stream should be T+D-ed. overwrite_stream has explicitly 0 records, but + // dedup_stream + // is missing from the map, so implicitly has nonzero records. + verify(destinationHandler).execute(Sql.of("UPDATE TABLE append_ns.append_stream WITHOUT SAFER CASTING")); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE dedup_ns.dedup_stream WITHOUT SAFER CASTING")); + verifyNoMoreInteractions(destinationHandler); + } + + /** + * Test a sync where the previous sync failed to run T+D for some stream. Even if this sync writes + * zero records, it should still run T+D. + */ + @Test + void unprocessedRecords() throws Exception { + when(destinationHandler.getInitialRawTableState(any())) + .thenReturn(new DestinationHandler.InitialRawTableState(true, Optional.of(Instant.parse("2023-01-23T12:34:56Z")))); + typerDeduper.prepareTables(); + clearInvocations(destinationHandler); + + typerDeduper.typeAndDedupe(Map.of( + new StreamDescriptor().withName("overwrite_stream").withNamespace("overwrite_ns"), new StreamSyncSummary(Optional.of(0L)), + new StreamDescriptor().withName("append_stream").withNamespace("append_ns"), new StreamSyncSummary(Optional.of(1L)))); + + verify(destinationHandler) + .execute(Sql.of("UPDATE TABLE overwrite_ns.overwrite_stream WITHOUT SAFER CASTING WHERE extracted_at > 2023-01-23T12:34:56Z")); + verify(destinationHandler) + .execute(Sql.of("UPDATE TABLE append_ns.append_stream WITHOUT SAFER CASTING WHERE extracted_at > 2023-01-23T12:34:56Z")); + verify(destinationHandler).execute(Sql.of("UPDATE TABLE dedup_ns.dedup_stream WITHOUT SAFER CASTING WHERE extracted_at > 2023-01-23T12:34:56Z")); + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2MigratorTest.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2MigratorTest.java similarity index 78% rename from airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2MigratorTest.java rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2MigratorTest.java index 8fe695b81ed0..86893c442ef9 100644 --- a/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2MigratorTest.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationV1V2MigratorTest.java @@ -4,8 +4,9 @@ package io.airbyte.integrations.base.destination.typing_deduping; -import static io.airbyte.integrations.base.JavaBaseConstants.LEGACY_RAW_TABLE_COLUMNS; -import static io.airbyte.integrations.base.JavaBaseConstants.V2_RAW_TABLE_COLUMN_NAMES; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.LEGACY_RAW_TABLE_COLUMNS; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.V2_RAW_TABLE_COLUMN_NAMES; +import static org.mockito.ArgumentMatchers.any; import io.airbyte.protocol.models.v0.DestinationSyncMode; import java.util.Optional; @@ -27,7 +28,7 @@ public class DestinationV1V2MigratorTest { public static class ShouldMigrateTestArgumentProvider implements ArgumentsProvider { @Override - public Stream provideArguments(ExtensionContext context) throws Exception { + public Stream provideArguments(final ExtensionContext context) throws Exception { // Don't throw an exception final boolean v2SchemaMatches = true; @@ -52,24 +53,25 @@ public Stream provideArguments(ExtensionContext context) th @ParameterizedTest @ArgumentsSource(ShouldMigrateTestArgumentProvider.class) - public void testShouldMigrate(final DestinationSyncMode destinationSyncMode, final BaseDestinationV1V2Migrator migrator, boolean expected) { + public void testShouldMigrate(final DestinationSyncMode destinationSyncMode, final BaseDestinationV1V2Migrator migrator, final boolean expected) + throws Exception { final StreamConfig config = new StreamConfig(STREAM_ID, null, destinationSyncMode, null, null, null); final var actual = migrator.shouldMigrate(config); Assertions.assertEquals(expected, actual); } @Test - public void testMismatchedSchemaThrowsException() { + public void testMismatchedSchemaThrowsException() throws Exception { final StreamConfig config = new StreamConfig(STREAM_ID, null, DestinationSyncMode.APPEND_DEDUP, null, null, null); final var migrator = makeMockMigrator(true, true, false, false, false); - UnexpectedSchemaException exception = Assertions.assertThrows(UnexpectedSchemaException.class, + final UnexpectedSchemaException exception = Assertions.assertThrows(UnexpectedSchemaException.class, () -> migrator.shouldMigrate(config)); Assertions.assertEquals("Destination V2 Raw Table does not match expected Schema", exception.getMessage()); } @SneakyThrows @Test - public void testMigrate() { + public void testMigrate() throws Exception { final var sqlGenerator = new MockSqlGenerator(); final StreamConfig stream = new StreamConfig(STREAM_ID, null, DestinationSyncMode.APPEND_DEDUP, null, null, null); final DestinationHandler handler = Mockito.mock(DestinationHandler.class); @@ -79,8 +81,8 @@ public void testMigrate() { migrator.migrate(sqlGenerator, handler, stream); Mockito.verify(handler).execute(sql); // Exception thrown when executing sql, TableNotMigratedException thrown - Mockito.doThrow(Exception.class).when(handler).execute(Mockito.anyString()); - TableNotMigratedException exception = Assertions.assertThrows(TableNotMigratedException.class, + Mockito.doThrow(Exception.class).when(handler).execute(any()); + final TableNotMigratedException exception = Assertions.assertThrows(TableNotMigratedException.class, () -> migrator.migrate(sqlGenerator, handler, stream)); Assertions.assertEquals("Attempted and failed to migrate stream final_table", exception.getMessage()); } @@ -88,22 +90,23 @@ public void testMigrate() { public static BaseDestinationV1V2Migrator makeMockMigrator(final boolean v2NamespaceExists, final boolean v2TableExists, final boolean v2RawSchemaMatches, - boolean v1RawTableExists, - boolean v1RawTableSchemaMatches) { + final boolean v1RawTableExists, + final boolean v1RawTableSchemaMatches) + throws Exception { final BaseDestinationV1V2Migrator migrator = Mockito.spy(BaseDestinationV1V2Migrator.class); - Mockito.when(migrator.doesAirbyteInternalNamespaceExist(Mockito.any())).thenReturn(v2NamespaceExists); + Mockito.when(migrator.doesAirbyteInternalNamespaceExist(any())).thenReturn(v2NamespaceExists); final var existingTable = v2TableExists ? Optional.of("v2_raw") : Optional.empty(); Mockito.when(migrator.getTableIfExists("raw", "raw_table")).thenReturn(existingTable); Mockito.when(migrator.schemaMatchesExpectation("v2_raw", V2_RAW_TABLE_COLUMN_NAMES)).thenReturn(v2RawSchemaMatches); - Mockito.when(migrator.convertToV1RawName(Mockito.any())).thenReturn(new NamespacedTableName("v1_raw_namespace", "v1_raw_table")); + Mockito.when(migrator.convertToV1RawName(any())).thenReturn(new NamespacedTableName("v1_raw_namespace", "v1_raw_table")); final var existingV1RawTable = v1RawTableExists ? Optional.of("v1_raw") : Optional.empty(); Mockito.when(migrator.getTableIfExists("v1_raw_namespace", "v1_raw_table")).thenReturn(existingV1RawTable); Mockito.when(migrator.schemaMatchesExpectation("v1_raw", LEGACY_RAW_TABLE_COLUMNS)).thenReturn(v1RawTableSchemaMatches); return migrator; } - public static BaseDestinationV1V2Migrator noIssuesMigrator() { + public static BaseDestinationV1V2Migrator noIssuesMigrator() throws Exception { return makeMockMigrator(true, false, true, true, true); } diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/MockSqlGenerator.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/MockSqlGenerator.java new file mode 100644 index 000000000000..c9fed75526a1 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/MockSqlGenerator.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import java.time.Instant; +import java.util.Optional; + +/** + * Basic SqlGenerator mock. See {@link DefaultTyperDeduperTest} for example usage. + */ +class MockSqlGenerator implements SqlGenerator { + + @Override + public StreamId buildStreamId(final String namespace, final String name, final String rawNamespaceOverride) { + return null; + } + + @Override + public ColumnId buildColumnId(final String name, final String suffix) { + return null; + } + + @Override + public Sql createSchema(final String schema) { + return Sql.of("CREATE SCHEMA " + schema); + } + + @Override + public Sql createTable(final StreamConfig stream, final String suffix, final boolean force) { + return Sql.of("CREATE TABLE " + stream.id().finalTableId("", suffix)); + } + + @Override + public boolean existingSchemaMatchesStreamConfig(final StreamConfig stream, final String existingTable) throws TableNotMigratedException { + return false; + } + + @Override + public Sql updateTable(final StreamConfig stream, + final String finalSuffix, + final Optional minRawTimestamp, + final boolean useExpensiveSaferCasting) { + final String timestampFilter = minRawTimestamp + .map(timestamp -> " WHERE extracted_at > " + timestamp) + .orElse(""); + final String casting = useExpensiveSaferCasting ? " WITH" : " WITHOUT" + " SAFER CASTING"; + return Sql.of("UPDATE TABLE " + stream.id().finalTableId("", finalSuffix) + casting + timestampFilter); + } + + @Override + public Sql overwriteFinalTable(final StreamId stream, final String finalSuffix) { + return Sql.of("OVERWRITE TABLE " + stream.finalTableId("") + " FROM " + stream.finalTableId("", finalSuffix)); + } + + @Override + public Sql migrateFromV1toV2(final StreamId streamId, final String namespace, final String tableName) { + return Sql.of("MIGRATE TABLE " + String.join(".", namespace, tableName) + " TO " + streamId.rawTableId("")); + } + + @Override + public Sql prepareTablesForSoftReset(final StreamConfig stream) { + return Sql.of("PREPARE " + String.join(".", stream.id().originalNamespace(), stream.id().originalName()) + " FOR SOFT RESET"); + } + + @Override + public Sql clearLoadedAt(final StreamId streamId) { + return null; + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/StreamIdTest.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/StreamIdTest.java similarity index 100% rename from airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/StreamIdTest.java rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/StreamIdTest.java diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValveTest.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValveTest.java new file mode 100644 index 000000000000..3ada28f544d4 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValveTest.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.base.DestinationConfig; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; +import java.util.stream.IntStream; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +public class TypeAndDedupeOperationValveTest { + + private static final AirbyteStreamNameNamespacePair STREAM_A = new AirbyteStreamNameNamespacePair("a", "a"); + private static final AirbyteStreamNameNamespacePair STREAM_B = new AirbyteStreamNameNamespacePair("b", "b"); + private static final Supplier ALWAYS_ZERO = () -> 0l; + + private Supplier minuteUpdates; + + @BeforeEach + public void setup() { + AtomicLong start = new AtomicLong(0); + minuteUpdates = () -> start.getAndUpdate(l -> l + (60 * 1000)); + } + + @AfterEach + public void clearDestinationConfig() { + DestinationConfig.clearInstance(); + } + + private void initializeDestinationConfigOption(final boolean enableIncrementalTypingAndDeduping) { + ObjectMapper mapper = new ObjectMapper(); + ObjectNode objectNode = mapper.createObjectNode(); + objectNode.put("enable_incremental_final_table_updates", enableIncrementalTypingAndDeduping); + DestinationConfig.initialize(objectNode); + } + + private void elapseTime(Supplier timing, int iterations) { + IntStream.range(0, iterations).forEach(__ -> { + timing.get(); + }); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testAddStream(final boolean enableIncrementalTypingAndDeduping) { + initializeDestinationConfigOption(enableIncrementalTypingAndDeduping); + final var valve = new TypeAndDedupeOperationValve(ALWAYS_ZERO); + valve.addStream(STREAM_A); + Assertions.assertEquals(-1, valve.getIncrementInterval(STREAM_A)); + Assertions.assertEquals(valve.readyToTypeAndDedupe(STREAM_A), enableIncrementalTypingAndDeduping); + Assertions.assertEquals(valve.get(STREAM_A), 0l); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testReadyToTypeAndDedupe(final boolean enableIncrementalTypingAndDeduping) { + initializeDestinationConfigOption(enableIncrementalTypingAndDeduping); + final var valve = new TypeAndDedupeOperationValve(minuteUpdates); + // method call increments time + valve.addStream(STREAM_A); + elapseTime(minuteUpdates, 1); + // method call increments time + valve.addStream(STREAM_B); + // method call increments time + Assertions.assertEquals(valve.readyToTypeAndDedupe(STREAM_A), enableIncrementalTypingAndDeduping); + elapseTime(minuteUpdates, 1); + Assertions.assertEquals(valve.readyToTypeAndDedupe(STREAM_B), enableIncrementalTypingAndDeduping); + valve.updateTimeAndIncreaseInterval(STREAM_A); + Assertions.assertEquals(1000 * 60 * 60 * 6, + valve.getIncrementInterval(STREAM_A)); + // method call increments time + Assertions.assertFalse(valve.readyToTypeAndDedupe(STREAM_A)); + // More than enough time has passed now + elapseTime(minuteUpdates, 60 * 6); + Assertions.assertEquals(valve.readyToTypeAndDedupe(STREAM_A), enableIncrementalTypingAndDeduping); + } + + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testUpdateTimeAndIncreaseInterval(final boolean enableIncrementalTypingAndDeduping) { + initializeDestinationConfigOption(enableIncrementalTypingAndDeduping); + final var valve = new TypeAndDedupeOperationValve(minuteUpdates); + valve.addStream(STREAM_A); + IntStream.range(0, 1).forEach(__ -> Assertions.assertEquals(valve.readyToTypeAndDedupe(STREAM_A), enableIncrementalTypingAndDeduping)); // start + // ready + // to T&D + Assertions.assertEquals(valve.readyToTypeAndDedupe(STREAM_A), enableIncrementalTypingAndDeduping); + valve.updateTimeAndIncreaseInterval(STREAM_A); + IntStream.range(0, 360).forEach(__ -> Assertions.assertFalse(valve.readyToTypeAndDedupe(STREAM_A))); + Assertions.assertEquals(valve.readyToTypeAndDedupe(STREAM_A), enableIncrementalTypingAndDeduping); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/java/io/airbyte/integrations/base/destination/typing_deduping/BaseSqlGeneratorIntegrationTest.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/java/io/airbyte/integrations/base/destination/typing_deduping/BaseSqlGeneratorIntegrationTest.java new file mode 100644 index 000000000000..cfc7eae3fa8a --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/java/io/airbyte/integrations/base/destination/typing_deduping/BaseSqlGeneratorIntegrationTest.java @@ -0,0 +1,1291 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Streams; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.string.Strings; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.SyncMode; +import java.time.Instant; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class exercises {@link SqlGenerator} implementations. All destinations should extend this + * class for their respective implementation. Subclasses are encouraged to add additional tests with + * destination-specific behavior (for example, verifying that datasets are created in the correct + * BigQuery region). + *

      + * Subclasses should implement a {@link org.junit.jupiter.api.BeforeAll} method to load any secrets + * and connect to the destination. This test expects to be able to run + * {@link #getDestinationHandler()} in a {@link org.junit.jupiter.api.BeforeEach} method. + */ +@Execution(ExecutionMode.CONCURRENT) +public abstract class BaseSqlGeneratorIntegrationTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(BaseSqlGeneratorIntegrationTest.class); + /** + * This, along with {@link #FINAL_TABLE_COLUMN_NAMES_CDC}, is the list of columns that should be in + * the final table. They're useful for generating SQL queries to insert records into the final + * table. + */ + protected static final List FINAL_TABLE_COLUMN_NAMES = List.of( + "_airbyte_raw_id", + "_airbyte_extracted_at", + "_airbyte_meta", + "id1", + "id2", + "updated_at", + "struct", + "array", + "string", + "number", + "integer", + "boolean", + "timestamp_with_timezone", + "timestamp_without_timezone", + "time_with_timezone", + "time_without_timezone", + "date", + "unknown"); + protected static final List FINAL_TABLE_COLUMN_NAMES_CDC; + + static { + FINAL_TABLE_COLUMN_NAMES_CDC = Streams.concat( + FINAL_TABLE_COLUMN_NAMES.stream(), + Stream.of("_ab_cdc_deleted_at")).toList(); + } + + protected RecordDiffer DIFFER; + + /** + * Subclasses may use these four StreamConfigs in their tests. + */ + protected StreamConfig incrementalDedupStream; + /** + * We intentionally don't have full refresh overwrite/append streams. Those actually behave + * identically in the sqlgenerator. Overwrite mode is actually handled in + * {@link DefaultTyperDeduper}. + */ + protected StreamConfig incrementalAppendStream; + protected StreamConfig cdcIncrementalDedupStream; + /** + * This isn't particularly realistic, but it's technically possible. + */ + protected StreamConfig cdcIncrementalAppendStream; + + protected SqlGenerator generator; + protected DestinationHandler destinationHandler; + protected String namespace; + + protected StreamId streamId; + private List primaryKey; + private ColumnId cursor; + private LinkedHashMap COLUMNS; + + protected abstract SqlGenerator getSqlGenerator(); + + protected abstract DestinationHandler getDestinationHandler(); + + /** + * Subclasses should override this method if they need to make changes to the stream ID. For + * example, you could upcase the final table name here. + */ + protected StreamId buildStreamId(final String namespace, final String finalTableName, final String rawTableName) { + return new StreamId(namespace, finalTableName, namespace, rawTableName, namespace, finalTableName); + } + + /** + * Do any setup work to create a namespace for this test run. For example, this might create a + * BigQuery dataset, or a Snowflake schema. + */ + protected abstract void createNamespace(String namespace) throws Exception; + + /** + * Create a raw table using the StreamId's rawTableId. + */ + protected abstract void createRawTable(StreamId streamId) throws Exception; + + /** + * Creates a raw table in the v1 format + */ + protected abstract void createV1RawTable(StreamId v1RawTable) throws Exception; + + protected abstract void insertRawTableRecords(StreamId streamId, List records) throws Exception; + + protected abstract void insertV1RawTableRecords(StreamId streamId, List records) throws Exception; + + protected abstract void insertFinalTableRecords(boolean includeCdcDeletedAt, StreamId streamId, String suffix, List records) + throws Exception; + + /** + * The two dump methods are defined identically as in {@link BaseTypingDedupingTest}, but with + * slightly different method signature. This test expects subclasses to respect the raw/finalTableId + * on the StreamId object, rather than hardcoding e.g. the airbyte_internal dataset. + *

      + * The {@code _airbyte_data} field must be deserialized into an ObjectNode, even if it's stored in + * the destination as a string. + */ + protected abstract List dumpRawTableRecords(StreamId streamId) throws Exception; + + protected abstract List dumpFinalTableRecords(StreamId streamId, String suffix) throws Exception; + + /** + * Clean up all resources in the namespace. For example, this might delete the BigQuery dataset + * created in {@link #createNamespace(String)}. + */ + protected abstract void teardownNamespace(String namespace) throws Exception; + + /** + * Identical to {@link BaseTypingDedupingTest#getRawMetadataColumnNames()}. + */ + protected Map getRawMetadataColumnNames() { + return new HashMap<>(); + } + + /** + * Identical to {@link BaseTypingDedupingTest#getFinalMetadataColumnNames()}. + */ + protected Map getFinalMetadataColumnNames() { + return new HashMap<>(); + } + + /** + * This test implementation is extremely destination-specific, but all destinations must implement + * it. This test should verify that creating a table using {@link #incrementalDedupStream} works as + * expected, including column types, indexing, partitioning, etc. + *

      + * Note that subclasses must also annotate their implementation with @Test. + */ + @Test + public abstract void testCreateTableIncremental() throws Exception; + + @BeforeEach + public void setup() throws Exception { + generator = getSqlGenerator(); + destinationHandler = getDestinationHandler(); + + final ColumnId id1 = generator.buildColumnId("id1"); + final ColumnId id2 = generator.buildColumnId("id2"); + primaryKey = List.of(id1, id2); + cursor = generator.buildColumnId("updated_at"); + + COLUMNS = new LinkedHashMap<>(); + COLUMNS.put(id1, AirbyteProtocolType.INTEGER); + COLUMNS.put(id2, AirbyteProtocolType.INTEGER); + COLUMNS.put(cursor, AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); + COLUMNS.put(generator.buildColumnId("struct"), new Struct(new LinkedHashMap<>())); + COLUMNS.put(generator.buildColumnId("array"), new Array(AirbyteProtocolType.UNKNOWN)); + COLUMNS.put(generator.buildColumnId("string"), AirbyteProtocolType.STRING); + COLUMNS.put(generator.buildColumnId("number"), AirbyteProtocolType.NUMBER); + COLUMNS.put(generator.buildColumnId("integer"), AirbyteProtocolType.INTEGER); + COLUMNS.put(generator.buildColumnId("boolean"), AirbyteProtocolType.BOOLEAN); + COLUMNS.put(generator.buildColumnId("timestamp_with_timezone"), AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); + COLUMNS.put(generator.buildColumnId("timestamp_without_timezone"), AirbyteProtocolType.TIMESTAMP_WITHOUT_TIMEZONE); + COLUMNS.put(generator.buildColumnId("time_with_timezone"), AirbyteProtocolType.TIME_WITH_TIMEZONE); + COLUMNS.put(generator.buildColumnId("time_without_timezone"), AirbyteProtocolType.TIME_WITHOUT_TIMEZONE); + COLUMNS.put(generator.buildColumnId("date"), AirbyteProtocolType.DATE); + COLUMNS.put(generator.buildColumnId("unknown"), AirbyteProtocolType.UNKNOWN); + + final LinkedHashMap cdcColumns = new LinkedHashMap<>(COLUMNS); + cdcColumns.put(generator.buildColumnId("_ab_cdc_deleted_at"), AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); + + DIFFER = new RecordDiffer( + getRawMetadataColumnNames(), + getFinalMetadataColumnNames(), + Pair.of(id1, AirbyteProtocolType.INTEGER), + Pair.of(id2, AirbyteProtocolType.INTEGER), + Pair.of(cursor, AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE)); + + namespace = Strings.addRandomSuffix("sql_generator_test", "_", 10); + // This is not a typical stream ID would look like, but SqlGenerator isn't allowed to make any + // assumptions about StreamId structure. + // In practice, the final table would be testDataset.users, and the raw table would be + // airbyte_internal.testDataset_raw__stream_users. + streamId = buildStreamId(namespace, "users_final", "users_raw"); + + incrementalDedupStream = new StreamConfig( + streamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND_DEDUP, + primaryKey, + Optional.of(cursor), + COLUMNS); + incrementalAppendStream = new StreamConfig( + streamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND, + primaryKey, + Optional.of(cursor), + COLUMNS); + + cdcIncrementalDedupStream = new StreamConfig( + streamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND_DEDUP, + primaryKey, + Optional.of(cursor), + cdcColumns); + cdcIncrementalAppendStream = new StreamConfig( + streamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND, + primaryKey, + Optional.of(cursor), + cdcColumns); + + LOGGER.info("Running with namespace {}", namespace); + createNamespace(namespace); + } + + @AfterEach + public void teardown() throws Exception { + teardownNamespace(namespace); + } + + /** + * Create a table and verify that we correctly recognize it as identical to itself. + */ + @Test + public void detectNoSchemaChange() throws Exception { + final Sql createTable = generator.createTable(incrementalDedupStream, "", false); + destinationHandler.execute(createTable); + + final Optional existingTable = destinationHandler.findExistingTable(streamId); + if (!existingTable.isPresent()) { + fail("Destination handler could not find existing table"); + } + + assertTrue( + generator.existingSchemaMatchesStreamConfig(incrementalDedupStream, existingTable.get()), + "Unchanged schema was incorrectly detected as a schema change."); + } + + /** + * Verify that adding a new column is detected as a schema change. + */ + @Test + public void detectColumnAdded() throws Exception { + final Sql createTable = generator.createTable(incrementalDedupStream, "", false); + destinationHandler.execute(createTable); + + final Optional existingTable = destinationHandler.findExistingTable(streamId); + if (!existingTable.isPresent()) { + fail("Destination handler could not find existing table"); + } + + incrementalDedupStream.columns().put( + generator.buildColumnId("new_column"), + AirbyteProtocolType.STRING); + + assertFalse( + generator.existingSchemaMatchesStreamConfig(incrementalDedupStream, existingTable.get()), + "Adding a new column was not detected as a schema change."); + } + + /** + * Verify that removing a column is detected as a schema change. + */ + @Test + public void detectColumnRemoved() throws Exception { + final Sql createTable = generator.createTable(incrementalDedupStream, "", false); + destinationHandler.execute(createTable); + + final Optional existingTable = destinationHandler.findExistingTable(streamId); + if (!existingTable.isPresent()) { + fail("Destination handler could not find existing table"); + } + + incrementalDedupStream.columns().remove(generator.buildColumnId("string")); + + assertFalse( + generator.existingSchemaMatchesStreamConfig(incrementalDedupStream, existingTable.get()), + "Removing a column was not detected as a schema change."); + } + + /** + * Verify that changing a column's type is detected as a schema change. + */ + @Test + public void detectColumnChanged() throws Exception { + final Sql createTable = generator.createTable(incrementalDedupStream, "", false); + destinationHandler.execute(createTable); + + final Optional existingTable = destinationHandler.findExistingTable(streamId); + if (!existingTable.isPresent()) { + fail("Destination handler could not find existing table"); + } + + incrementalDedupStream.columns().put( + generator.buildColumnId("string"), + AirbyteProtocolType.INTEGER); + + assertFalse( + generator.existingSchemaMatchesStreamConfig(incrementalDedupStream, existingTable.get()), + "Altering a column was not detected as a schema change."); + } + + /** + * Test that T+D supports streams whose name and namespace are the same. + */ + @Test + public void incrementalDedupSameNameNamespace() throws Exception { + final StreamId streamId = buildStreamId(namespace, namespace, namespace + "_raw"); + final StreamConfig stream = new StreamConfig( + streamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND_DEDUP, + incrementalDedupStream.primaryKey(), + incrementalDedupStream.cursor(), + incrementalDedupStream.columns()); + + createRawTable(streamId); + createFinalTable(stream, ""); + insertRawTableRecords( + streamId, + List.of(Jsons.deserialize( + """ + { + "_airbyte_raw_id": "5ce60e70-98aa-4fe3-8159-67207352c4f0", + "_airbyte_extracted_at": "2023-01-01T00:00:00Z", + "_airbyte_data": {"id1": 1, "id2": 100} + } + """))); + + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, stream, Optional.empty(), ""); + + final List rawRecords = dumpRawTableRecords(streamId); + final List finalRecords = dumpFinalTableRecords(streamId, ""); + verifyRecordCounts(1, rawRecords, 1, finalRecords); + } + + /** + * Run a full T+D update for an incremental-dedup stream, writing to a final table with "_foo" + * suffix, with values for all data types. Verifies all behaviors for all types: + *

        + *
      • A valid, nonnull value
      • + *
      • No value (i.e. the column is missing from the record)
      • + *
      • A JSON null value
      • + *
      • An invalid value
      • + *
      + *

      + * In practice, incremental streams never write to a suffixed table, but SqlGenerator isn't allowed + * to make that assumption (and we might as well exercise that code path). + */ + @Test + public void allTypes() throws Exception { + // Add case-sensitive columnName to test json path querying + incrementalDedupStream.columns().put( + generator.buildColumnId("IamACaseSensitiveColumnName"), + AirbyteProtocolType.STRING); + createRawTable(streamId); + createFinalTable(incrementalDedupStream, ""); + insertRawTableRecords( + streamId, + BaseTypingDedupingTest.readRecords("sqlgenerator/alltypes_inputrecords.jsonl")); + + assertTrue(destinationHandler.isFinalTableEmpty(streamId), "Final table should be empty before T+D"); + + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, incrementalDedupStream, Optional.empty(), ""); + + verifyRecords( + "sqlgenerator/alltypes_expectedrecords_raw.jsonl", + dumpRawTableRecords(streamId), + "sqlgenerator/alltypes_expectedrecords_final.jsonl", + dumpFinalTableRecords(streamId, "")); + assertFalse(destinationHandler.isFinalTableEmpty(streamId), "Final table should not be empty after T+D"); + } + + /** + * Run a basic test to verify that we don't throw an exception on basic data values. + */ + @Test + public void allTypesUnsafe() throws Exception { + createRawTable(streamId); + createFinalTable(incrementalDedupStream, ""); + insertRawTableRecords( + streamId, + BaseTypingDedupingTest.readRecords("sqlgenerator/alltypes_unsafe_inputrecords.jsonl")); + + assertTrue(destinationHandler.isFinalTableEmpty(streamId), "Final table should be empty before T+D"); + + // Instead of using the full T+D transaction, explicitly run with useSafeCasting=false. + final Sql unsafeSql = generator.updateTable(incrementalDedupStream, "", Optional.empty(), false); + destinationHandler.execute(unsafeSql); + + assertFalse(destinationHandler.isFinalTableEmpty(streamId), "Final table should not be empty after T+D"); + } + + /** + * Run through some plausible T+D scenarios to verify that we correctly identify the min raw + * timestamp. + */ + @Test + public void minTimestampBehavesCorrectly() throws Exception { + // When the raw table doesn't exist, there are no unprocessed records and no timestamp + assertEquals(new DestinationHandler.InitialRawTableState(false, Optional.empty()), destinationHandler.getInitialRawTableState(streamId)); + + // When the raw table is empty, there are still no unprocessed records and no timestamp + createRawTable(streamId); + assertEquals(new DestinationHandler.InitialRawTableState(false, Optional.empty()), destinationHandler.getInitialRawTableState(streamId)); + + // If we insert some raw records with null loaded_at, we should get the min extracted_at + insertRawTableRecords( + streamId, + List.of( + Jsons.deserialize( + """ + { + "_airbyte_raw_id": "899d3bc3-7921-44f0-8517-c748a28fe338", + "_airbyte_extracted_at": "2023-01-01T00:00:00Z", + "_airbyte_data": {} + } + """), + Jsons.deserialize( + """ + { + "_airbyte_raw_id": "47f46eb6-fcae-469c-a7fc-31d4b9ce7474", + "_airbyte_extracted_at": "2023-01-02T00:00:00Z", + "_airbyte_data": {} + } + """))); + DestinationHandler.InitialRawTableState tableState = destinationHandler.getInitialRawTableState(streamId); + assertTrue(tableState.hasUnprocessedRecords(), + "When all raw records have null loaded_at, we should recognize that there are unprocessed records"); + assertTrue( + tableState.maxProcessedTimestamp().get().isBefore(Instant.parse("2023-01-01T00:00:00Z")), + "When all raw records have null loaded_at, the min timestamp should be earlier than all of their extracted_at values (2023-01-01). Was actually " + + tableState.maxProcessedTimestamp().get()); + + // Execute T+D to set loaded_at on the records + createFinalTable(incrementalAppendStream, ""); + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, incrementalAppendStream, Optional.empty(), ""); + + assertEquals( + destinationHandler.getInitialRawTableState(streamId), + new DestinationHandler.InitialRawTableState(false, Optional.of(Instant.parse("2023-01-02T00:00:00Z"))), + "When all raw records have non-null loaded_at, we should recognize that there are no unprocessed records, and the min timestamp should be equal to the latest extracted_at"); + + // If we insert another raw record with older extracted_at than the typed records, we should fetch a + // timestamp earlier than this new record. + // This emulates a sync inserting some records out of order, running T+D on newer records, inserting + // an older record, and then crashing before it can execute T+D. The next sync should recognize + // that older record as still needing to be processed. + insertRawTableRecords( + streamId, + List.of(Jsons.deserialize( + """ + { + "_airbyte_raw_id": "899d3bc3-7921-44f0-8517-c748a28fe338", + "_airbyte_extracted_at": "2023-01-01T12:00:00Z", + "_airbyte_data": {} + } + """))); + tableState = destinationHandler.getInitialRawTableState(streamId); + // this is a pretty confusing pair of assertions. To explain them in more detail: There are three + // records in the raw table: + // * loaded_at not null, extracted_at = 2023-01-01 00:00Z + // * loaded_at is null, extracted_at = 2023-01-01 12:00Z + // * loaded_at not null, extracted_at = 2023-01-02 00:00Z + // We should have a timestamp which is older than the second record, but newer than or equal to + // (i.e. not before) the first record. This allows us to query the raw table using + // `_airbyte_extracted_at > ?`, which will include the second record and exclude the first record. + assertTrue(tableState.hasUnprocessedRecords(), + "When some raw records have null loaded_at, we should recognize that there are unprocessed records"); + assertTrue( + tableState.maxProcessedTimestamp().get().isBefore(Instant.parse("2023-01-01T12:00:00Z")), + "When some raw records have null loaded_at, the min timestamp should be earlier than the oldest unloaded record (2023-01-01 12:00Z). Was actually " + + tableState); + assertFalse( + tableState.maxProcessedTimestamp().get().isBefore(Instant.parse("2023-01-01T00:00:00Z")), + "When some raw records have null loaded_at, the min timestamp should be later than the newest loaded record older than the oldest unloaded record (2023-01-01 00:00Z). Was actually " + + tableState); + } + + /** + * Identical to {@link #allTypes()}, but queries for the min raw timestamp first. This verifies that + * if a previous sync doesn't fully type-and-dedupe a table, we still get those records on the next + * sync. + */ + @Test + public void handlePreexistingRecords() throws Exception { + // Add case-sensitive columnName to test json path querying + incrementalDedupStream.columns().put( + generator.buildColumnId("IamACaseSensitiveColumnName"), + AirbyteProtocolType.STRING); + createRawTable(streamId); + createFinalTable(incrementalDedupStream, ""); + insertRawTableRecords( + streamId, + BaseTypingDedupingTest.readRecords("sqlgenerator/alltypes_inputrecords.jsonl")); + + final DestinationHandler.InitialRawTableState tableState = destinationHandler.getInitialRawTableState(streamId); + assertAll( + () -> assertTrue(tableState.hasUnprocessedRecords(), + "After writing some raw records, we should recognize that there are unprocessed records"), + () -> assertTrue(tableState.maxProcessedTimestamp().isPresent(), "After writing some raw records, the min timestamp should be present.")); + + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, incrementalDedupStream, tableState.maxProcessedTimestamp(), ""); + + verifyRecords( + "sqlgenerator/alltypes_expectedrecords_raw.jsonl", + dumpRawTableRecords(streamId), + "sqlgenerator/alltypes_expectedrecords_final.jsonl", + dumpFinalTableRecords(streamId, "")); + } + + /** + * Identical to {@link #handlePreexistingRecords()}, but queries for the min timestamp before + * inserting any raw records. This emulates a sync starting with an empty table. + */ + @Test + public void handleNoPreexistingRecords() throws Exception { + // Add case-sensitive columnName to test json path querying + incrementalDedupStream.columns().put( + generator.buildColumnId("IamACaseSensitiveColumnName"), + AirbyteProtocolType.STRING); + createRawTable(streamId); + final DestinationHandler.InitialRawTableState tableState = destinationHandler.getInitialRawTableState(streamId); + assertAll( + () -> assertFalse(tableState.hasUnprocessedRecords(), "With an empty raw table, we should recognize that there are no unprocessed records"), + () -> assertEquals(Optional.empty(), tableState.maxProcessedTimestamp(), "With an empty raw table, the min timestamp should be empty")); + + createFinalTable(incrementalDedupStream, ""); + insertRawTableRecords( + streamId, + BaseTypingDedupingTest.readRecords("sqlgenerator/alltypes_inputrecords.jsonl")); + + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, incrementalDedupStream, tableState.maxProcessedTimestamp(), ""); + + verifyRecords( + "sqlgenerator/alltypes_expectedrecords_raw.jsonl", + dumpRawTableRecords(streamId), + "sqlgenerator/alltypes_expectedrecords_final.jsonl", + dumpFinalTableRecords(streamId, "")); + } + + /** + * Verify that we correctly only process raw records with recent extracted_at. In practice, + * destinations should not do this - but their SQL should work correctly. + *

      + * Create two raw records, one with an old extracted_at. Verify that updatedTable only T+Ds the new + * record, and doesn't set loaded_at on the old record. + */ + @Test + public void ignoreOldRawRecords() throws Exception { + createRawTable(streamId); + createFinalTable(incrementalAppendStream, ""); + insertRawTableRecords( + streamId, + List.of( + Jsons.deserialize( + """ + { + "_airbyte_raw_id": "c5bcae50-962e-4b92-b2eb-1659eae31693", + "_airbyte_extracted_at": "2022-01-01T00:00:00Z", + "_airbyte_data": { + "string": "foo" + } + } + """), + Jsons.deserialize( + """ + { + "_airbyte_raw_id": "93f1bdd8-1916-4e6c-94dc-29a5d9701179", + "_airbyte_extracted_at": "2023-01-01T01:00:00Z", + "_airbyte_data": { + "string": "bar" + } + } + """))); + + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, incrementalAppendStream, + Optional.of(Instant.parse("2023-01-01T00:00:00Z")), ""); + + final List rawRecords = dumpRawTableRecords(streamId); + final List finalRecords = dumpFinalTableRecords(streamId, ""); + assertAll( + () -> assertEquals( + 1, + rawRecords.stream().filter(record -> record.get("_airbyte_loaded_at") == null).count(), + "Raw table should only have non-null loaded_at on the newer record"), + () -> assertEquals(1, finalRecords.size(), "T+D should only execute on the newer record")); + } + + /** + * Test JSON Types encounted for a String Type field. + * + * @throws Exception + */ + @Test + public void jsonStringifyTypes() throws Exception { + createRawTable(streamId); + createFinalTable(incrementalDedupStream, "_foo"); + insertRawTableRecords( + streamId, + BaseTypingDedupingTest.readRecords("sqlgenerator/json_types_in_string_inputrecords.jsonl")); + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, incrementalDedupStream, Optional.empty(), "_foo"); + verifyRecords( + "sqlgenerator/json_types_in_string_expectedrecords_raw.jsonl", + dumpRawTableRecords(streamId), + "sqlgenerator/json_types_in_string_expectedrecords_final.jsonl", + dumpFinalTableRecords(streamId, "_foo")); + } + + @Test + public void timestampFormats() throws Exception { + createRawTable(streamId); + createFinalTable(incrementalAppendStream, ""); + insertRawTableRecords( + streamId, + BaseTypingDedupingTest.readRecords("sqlgenerator/timestampformats_inputrecords.jsonl")); + + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, incrementalAppendStream, Optional.empty(), ""); + + DIFFER.diffFinalTableRecords( + BaseTypingDedupingTest.readRecords("sqlgenerator/timestampformats_expectedrecords_final.jsonl"), + dumpFinalTableRecords(streamId, "")); + } + + @Test + public void incrementalDedup() throws Exception { + createRawTable(streamId); + createFinalTable(incrementalDedupStream, ""); + insertRawTableRecords( + streamId, + BaseTypingDedupingTest.readRecords("sqlgenerator/incrementaldedup_inputrecords.jsonl")); + + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, incrementalDedupStream, Optional.empty(), ""); + + verifyRecords( + "sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl", + dumpRawTableRecords(streamId), + "sqlgenerator/incrementaldedup_expectedrecords_final.jsonl", + dumpFinalTableRecords(streamId, "")); + } + + /** + * We shouldn't crash on a sync with null cursor. Insert two records and verify that we keep the + * record with higher extracted_at. + */ + @Test + public void incrementalDedupNoCursor() throws Exception { + final StreamConfig streamConfig = new StreamConfig( + streamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND_DEDUP, + primaryKey, + Optional.empty(), + COLUMNS); + createRawTable(streamId); + createFinalTable(streamConfig, ""); + insertRawTableRecords( + streamId, + List.of( + Jsons.deserialize( + """ + { + "_airbyte_raw_id": "c5bcae50-962e-4b92-b2eb-1659eae31693", + "_airbyte_extracted_at": "2023-01-01T00:00:00Z", + "_airbyte_data": { + "id1": 1, + "id2": 100, + "string": "foo" + } + } + """), + Jsons.deserialize( + """ + { + "_airbyte_raw_id": "93f1bdd8-1916-4e6c-94dc-29a5d9701179", + "_airbyte_extracted_at": "2023-01-01T01:00:00Z", + "_airbyte_data": { + "id1": 1, + "id2": 100, + "string": "bar" + } + } + """))); + + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, streamConfig, Optional.empty(), ""); + + final List actualRawRecords = dumpRawTableRecords(streamId); + final List actualFinalRecords = dumpFinalTableRecords(streamId, ""); + verifyRecordCounts( + 2, + actualRawRecords, + 1, + actualFinalRecords); + assertEquals("bar", actualFinalRecords.get(0).get(generator.buildColumnId("string").name()).asText()); + } + + @Test + public void incrementalAppend() throws Exception { + createRawTable(streamId); + createFinalTable(incrementalAppendStream, ""); + insertRawTableRecords( + streamId, + BaseTypingDedupingTest.readRecords("sqlgenerator/incrementaldedup_inputrecords.jsonl")); + + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, incrementalAppendStream, Optional.empty(), ""); + + verifyRecordCounts( + 3, + dumpRawTableRecords(streamId), + 3, + dumpFinalTableRecords(streamId, "")); + } + + /** + * Create a nonempty users_final_tmp table. Overwrite users_final from users_final_tmp. Verify that + * users_final now exists and contains nonzero records. + */ + @Test + public void overwriteFinalTable() throws Exception { + createFinalTable(incrementalAppendStream, "_tmp"); + final List records = singletonList(Jsons.deserialize( + """ + { + "_airbyte_raw_id": "4fa4efe2-3097-4464-bd22-11211cc3e15b", + "_airbyte_extracted_at": "2023-01-01T00:00:00Z", + "_airbyte_meta": {} + } + """)); + insertFinalTableRecords( + false, + streamId, + "_tmp", + records); + + final Sql sql = generator.overwriteFinalTable(streamId, "_tmp"); + destinationHandler.execute(sql); + + assertEquals(1, dumpFinalTableRecords(streamId, "").size()); + } + + @Test + public void cdcImmediateDeletion() throws Exception { + createRawTable(streamId); + createFinalTable(cdcIncrementalDedupStream, ""); + insertRawTableRecords( + streamId, + singletonList(Jsons.deserialize( + """ + { + "_airbyte_raw_id": "4fa4efe2-3097-4464-bd22-11211cc3e15b", + "_airbyte_extracted_at": "2023-01-01T00:00:00Z", + "_airbyte_data": { + "id1": 1, + "id2": 100, + "updated_at": "2023-01-01T00:00:00Z", + "_ab_cdc_deleted_at": "2023-01-01T00:01:00Z" + } + } + """))); + + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, cdcIncrementalDedupStream, Optional.empty(), ""); + + verifyRecordCounts( + 1, + dumpRawTableRecords(streamId), + 0, + dumpFinalTableRecords(streamId, "")); + } + + /** + * Verify that running T+D twice is idempotent. Previously there was a bug where non-dedup syncs + * with an _ab_cdc_deleted_at column would duplicate "deleted" records on each run. + */ + @Test + public void cdcIdempotent() throws Exception { + createRawTable(streamId); + createFinalTable(cdcIncrementalAppendStream, ""); + insertRawTableRecords( + streamId, + singletonList(Jsons.deserialize( + """ + { + "_airbyte_raw_id": "4fa4efe2-3097-4464-bd22-11211cc3e15b", + "_airbyte_extracted_at": "2023-01-01T00:00:00Z", + "_airbyte_data": { + "id1": 1, + "id2": 100, + "updated_at": "2023-01-01T00:00:00Z", + "_ab_cdc_deleted_at": "2023-01-01T00:01:00Z" + } + } + """))); + + // Execute T+D twice + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, cdcIncrementalAppendStream, Optional.empty(), ""); + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, cdcIncrementalAppendStream, Optional.empty(), ""); + + verifyRecordCounts( + 1, + dumpRawTableRecords(streamId), + 1, + dumpFinalTableRecords(streamId, "")); + } + + @Test + public void cdcComplexUpdate() throws Exception { + createRawTable(streamId); + createFinalTable(cdcIncrementalDedupStream, ""); + insertRawTableRecords( + streamId, + BaseTypingDedupingTest.readRecords("sqlgenerator/cdcupdate_inputrecords_raw.jsonl")); + insertFinalTableRecords( + true, + streamId, + "", + BaseTypingDedupingTest.readRecords("sqlgenerator/cdcupdate_inputrecords_final.jsonl")); + + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, cdcIncrementalDedupStream, Optional.empty(), ""); + + verifyRecordCounts( + 11, + dumpRawTableRecords(streamId), + 6, + dumpFinalTableRecords(streamId, "")); + } + + /** + * source operations: + *

        + *
      1. insert id=1 (lsn 10000)
      2. + *
      3. delete id=1 (lsn 10001)
      4. + *
      + *

      + * But the destination writes lsn 10001 before 10000. We should still end up with no records in the + * final table. + *

      + * All records have the same emitted_at timestamp. This means that we live or die purely based on + * our ability to use _ab_cdc_lsn. + */ + @Test + public void testCdcOrdering_updateAfterDelete() throws Exception { + createRawTable(streamId); + createFinalTable(cdcIncrementalDedupStream, ""); + insertRawTableRecords( + streamId, + BaseTypingDedupingTest.readRecords("sqlgenerator/cdcordering_updateafterdelete_inputrecords.jsonl")); + + final DestinationHandler.InitialRawTableState tableState = destinationHandler.getInitialRawTableState(cdcIncrementalDedupStream.id()); + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, cdcIncrementalDedupStream, tableState.maxProcessedTimestamp(), ""); + + verifyRecordCounts( + 2, + dumpRawTableRecords(streamId), + 0, + dumpFinalTableRecords(streamId, "")); + } + + /** + * source operations: + *

        + *
      1. arbitrary history...
      2. + *
      3. delete id=1 (lsn 10001)
      4. + *
      5. reinsert id=1 (lsn 10002)
      6. + *
      + *

      + * But the destination receives LSNs 10002 before 10001. In this case, we should keep the reinserted + * record in the final table. + *

      + * All records have the same emitted_at timestamp. This means that we live or die purely based on + * our ability to use _ab_cdc_lsn. + */ + @Test + public void testCdcOrdering_insertAfterDelete() throws Exception { + createRawTable(streamId); + createFinalTable(cdcIncrementalDedupStream, ""); + insertRawTableRecords( + streamId, + BaseTypingDedupingTest.readRecords("sqlgenerator/cdcordering_insertafterdelete_inputrecords_raw.jsonl")); + insertFinalTableRecords( + true, + streamId, + "", + BaseTypingDedupingTest.readRecords("sqlgenerator/cdcordering_insertafterdelete_inputrecords_final.jsonl")); + + final DestinationHandler.InitialRawTableState tableState = destinationHandler.getInitialRawTableState(cdcIncrementalAppendStream.id()); + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, cdcIncrementalDedupStream, tableState.maxProcessedTimestamp(), ""); + verifyRecordCounts( + 2, + dumpRawTableRecords(streamId), + 1, + dumpFinalTableRecords(streamId, "")); + } + + /** + * Create a table which includes the _ab_cdc_deleted_at column, then soft reset it using the non-cdc + * stream config. Verify that the deleted_at column gets dropped. + */ + @Test + public void softReset() throws Exception { + createRawTable(streamId); + createFinalTable(cdcIncrementalAppendStream, ""); + insertRawTableRecords( + streamId, + singletonList(Jsons.deserialize( + """ + { + "_airbyte_raw_id": "arst", + "_airbyte_extracted_at": "2023-01-01T00:00:00Z", + "_airbyte_loaded_at": "2023-01-01T00:00:00Z", + "_airbyte_data": { + "id1": 1, + "id2": 100, + "_ab_cdc_deleted_at": "2023-01-01T00:01:00Z" + } + } + """))); + insertFinalTableRecords( + true, + streamId, + "", + singletonList(Jsons.deserialize( + """ + { + "_airbyte_raw_id": "arst", + "_airbyte_extracted_at": "2023-01-01T00:00:00Z", + "_airbyte_meta": {}, + "id1": 1, + "id2": 100, + "_ab_cdc_deleted_at": "2023-01-01T00:01:00Z" + } + """))); + + TypeAndDedupeTransaction.executeSoftReset(generator, destinationHandler, incrementalAppendStream); + + final List actualRawRecords = dumpRawTableRecords(streamId); + final List actualFinalRecords = dumpFinalTableRecords(streamId, ""); + assertAll( + () -> assertEquals(1, actualRawRecords.size()), + () -> assertEquals(1, actualFinalRecords.size()), + () -> assertTrue( + actualFinalRecords.stream().noneMatch(record -> record.has("_ab_cdc_deleted_at")), + "_ab_cdc_deleted_at column was expected to be dropped. Actual final table had: " + actualFinalRecords)); + } + + @Test + public void weirdColumnNames() throws Exception { + createRawTable(streamId); + insertRawTableRecords( + streamId, + BaseTypingDedupingTest.readRecords("sqlgenerator/weirdcolumnnames_inputrecords_raw.jsonl")); + final StreamConfig stream = new StreamConfig( + streamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND_DEDUP, + primaryKey, + Optional.of(cursor), + new LinkedHashMap<>() { + + { + put(generator.buildColumnId("id1"), AirbyteProtocolType.INTEGER); + put(generator.buildColumnId("id2"), AirbyteProtocolType.INTEGER); + put(generator.buildColumnId("updated_at"), AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); + put(generator.buildColumnId("$starts_with_dollar_sign"), AirbyteProtocolType.STRING); + put(generator.buildColumnId("includes\"doublequote"), AirbyteProtocolType.STRING); + put(generator.buildColumnId("includes'singlequote"), AirbyteProtocolType.STRING); + put(generator.buildColumnId("includes`backtick"), AirbyteProtocolType.STRING); + put(generator.buildColumnId("includes.period"), AirbyteProtocolType.STRING); + put(generator.buildColumnId("includes$$doubledollar"), AirbyteProtocolType.STRING); + put(generator.buildColumnId("endswithbackslash\\"), AirbyteProtocolType.STRING); + } + + }); + + final Sql createTable = generator.createTable(stream, "", false); + destinationHandler.execute(createTable); + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, stream, Optional.empty(), ""); + + verifyRecords( + "sqlgenerator/weirdcolumnnames_expectedrecords_raw.jsonl", + dumpRawTableRecords(streamId), + "sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl", + dumpFinalTableRecords(streamId, "")); + } + + /** + * Verify that we don't crash when there are special characters in the stream namespace, name, + * primary key, or cursor. + */ + @ParameterizedTest + @ValueSource(strings = {"$", "${", "${${", "${foo}", "\"", "'", "`", ".", "$$", "\\", "{", "}"}) + public void noCrashOnSpecialCharacters(final String specialChars) throws Exception { + final String str = specialChars + "_" + namespace + "_" + specialChars; + final StreamId originalStreamId = generator.buildStreamId(str, str, "unused"); + final StreamId modifiedStreamId = buildStreamId( + originalStreamId.finalNamespace(), + originalStreamId.finalName(), + "raw_table"); + final ColumnId columnId = generator.buildColumnId(str); + try { + createNamespace(modifiedStreamId.finalNamespace()); + createRawTable(modifiedStreamId); + insertRawTableRecords( + modifiedStreamId, + List.of(Jsons.jsonNode(Map.of( + "_airbyte_raw_id", "758989f2-b148-4dd3-8754-30d9c17d05fb", + "_airbyte_extracted_at", "2023-01-01T00:00:00Z", + "_airbyte_data", Map.of(str, "bar"))))); + final StreamConfig stream = new StreamConfig( + modifiedStreamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND_DEDUP, + List.of(columnId), + Optional.of(columnId), + new LinkedHashMap<>() { + + { + put(columnId, AirbyteProtocolType.STRING); + } + + }); + + final Sql createTable = generator.createTable(stream, "", false); + destinationHandler.execute(createTable); + // Not verifying anything about the data; let's just make sure we don't crash. + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, stream, Optional.empty(), ""); + } finally { + teardownNamespace(modifiedStreamId.finalNamespace()); + } + } + + /** + * Verify column names that are reserved keywords are handled successfully. Each destination should + * always have at least 1 column in the record data that is a reserved keyword. + */ + @Test + public void testReservedKeywords() throws Exception { + createRawTable(streamId); + insertRawTableRecords( + streamId, + BaseTypingDedupingTest.readRecords("sqlgenerator/reservedkeywords_inputrecords_raw.jsonl")); + final StreamConfig stream = new StreamConfig( + streamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND, + null, + Optional.empty(), + new LinkedHashMap<>() { + + { + put(generator.buildColumnId("current_date"), AirbyteProtocolType.STRING); + put(generator.buildColumnId("join"), AirbyteProtocolType.STRING); + } + + }); + + final Sql createTable = generator.createTable(stream, "", false); + destinationHandler.execute(createTable); + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, stream, Optional.empty(), ""); + + DIFFER.diffFinalTableRecords( + BaseTypingDedupingTest.readRecords("sqlgenerator/reservedkeywords_expectedrecords_final.jsonl"), + dumpFinalTableRecords(streamId, "")); + } + + /** + * A stream with no columns is weird, but we shouldn't treat it specially in any way. It should + * create a final table as usual, and populate it with the relevant metadata columns. + */ + @Test + public void noColumns() throws Exception { + createRawTable(streamId); + insertRawTableRecords( + streamId, + List.of(Jsons.deserialize( + """ + { + "_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", + "_airbyte_extracted_at": "2023-01-01T00:00:00Z", + "_airbyte_data": {} + } + """))); + final StreamConfig stream = new StreamConfig( + streamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND, + emptyList(), + Optional.empty(), + new LinkedHashMap<>()); + + final Sql createTable = generator.createTable(stream, "", false); + destinationHandler.execute(createTable); + TypeAndDedupeTransaction.executeTypeAndDedupe(generator, destinationHandler, stream, Optional.empty(), ""); + + verifyRecords( + "sqlgenerator/nocolumns_expectedrecords_raw.jsonl", + dumpRawTableRecords(streamId), + "sqlgenerator/nocolumns_expectedrecords_final.jsonl", + dumpFinalTableRecords(streamId, "")); + } + + @Test + public void testV1V2migration() throws Exception { + // This is maybe a little hacky, but it avoids having to refactor this entire class and subclasses + // for something that is going away + // Add case-sensitive columnName to test json path querying + incrementalDedupStream.columns().put( + generator.buildColumnId("IamACaseSensitiveColumnName"), + AirbyteProtocolType.STRING); + final StreamId v1RawTableStreamId = new StreamId(null, null, streamId.finalNamespace(), "v1_" + streamId.rawName(), null, null); + createV1RawTable(v1RawTableStreamId); + insertV1RawTableRecords(v1RawTableStreamId, BaseTypingDedupingTest.readRecords( + "sqlgenerator/all_types_v1_inputrecords.jsonl")); + final Sql migration = generator.migrateFromV1toV2(streamId, v1RawTableStreamId.rawNamespace(), v1RawTableStreamId.rawName()); + destinationHandler.execute(migration); + final List v1RawRecords = dumpV1RawTableRecords(v1RawTableStreamId); + final List v2RawRecords = dumpRawTableRecords(streamId); + migrationAssertions(v1RawRecords, v2RawRecords); + + // And then run T+D on the migrated raw data + final Sql createTable = generator.createTable(incrementalDedupStream, "", false); + destinationHandler.execute(createTable); + final Sql updateTable = generator.updateTable(incrementalDedupStream, "", Optional.empty(), true); + destinationHandler.execute(updateTable); + verifyRecords( + "sqlgenerator/alltypes_expectedrecords_raw.jsonl", + dumpRawTableRecords(streamId), + "sqlgenerator/alltypes_expectedrecords_final.jsonl", + dumpFinalTableRecords(streamId, "")); + } + + /** + * Sometimes, a sync doesn't delete its soft reset temp table. (it's not entirely clear why this + * happens.) In these cases, the next sync should not crash. + */ + @Test + public void softResetIgnoresPreexistingTempTable() throws Exception { + createRawTable(incrementalDedupStream.id()); + + // Create a soft reset table. Use incremental append mode, in case the destination connector uses + // different + // indexing/partitioning/etc. + final Sql createOldTempTable = generator.createTable(incrementalAppendStream, TypeAndDedupeTransaction.SOFT_RESET_SUFFIX, false); + destinationHandler.execute(createOldTempTable); + + // Execute a soft reset. This should not crash. + TypeAndDedupeTransaction.executeSoftReset(generator, destinationHandler, incrementalAppendStream); + } + + protected void migrationAssertions(final List v1RawRecords, final List v2RawRecords) { + final var v2RecordMap = v2RawRecords.stream().collect(Collectors.toMap( + record -> record.get("_airbyte_raw_id").asText(), + Function.identity())); + assertAll( + () -> assertEquals(6, v1RawRecords.size()), + () -> assertEquals(6, v2RawRecords.size())); + v1RawRecords.forEach(v1Record -> { + final var v1id = v1Record.get("_airbyte_ab_id").asText(); + assertAll( + () -> assertEquals(v1id, v2RecordMap.get(v1id).get("_airbyte_raw_id").asText()), + () -> assertEquals(v1Record.get("_airbyte_emitted_at").asText(), v2RecordMap.get(v1id).get("_airbyte_extracted_at").asText()), + () -> assertNull(v2RecordMap.get(v1id).get("_airbyte_loaded_at"))); + JsonNode originalData = v1Record.get("_airbyte_data"); + if (originalData.isTextual()) { + originalData = Jsons.deserializeExact(originalData.asText()); + } + JsonNode migratedData = v2RecordMap.get(v1id).get("_airbyte_data"); + if (migratedData.isTextual()) { + migratedData = Jsons.deserializeExact(migratedData.asText()); + } + // hacky thing because we only care about the data contents. + // diffRawTableRecords makes some assumptions about the structure of the blob. + DIFFER.diffFinalTableRecords(List.of(originalData), List.of(migratedData)); + }); + } + + protected List dumpV1RawTableRecords(final StreamId streamId) throws Exception { + return dumpRawTableRecords(streamId); + } + + @Test + public void testCreateTableForce() throws Exception { + final Sql createTableNoForce = generator.createTable(incrementalDedupStream, "", false); + final Sql createTableForce = generator.createTable(incrementalDedupStream, "", true); + + destinationHandler.execute(createTableNoForce); + assertThrows(Exception.class, () -> destinationHandler.execute(createTableNoForce)); + // This should not throw an exception + destinationHandler.execute(createTableForce); + + assertTrue(destinationHandler.findExistingTable(streamId).isPresent()); + } + + protected void createFinalTable(final StreamConfig stream, final String suffix) throws Exception { + final Sql createTable = generator.createTable(stream, suffix, false); + destinationHandler.execute(createTable); + } + + private void verifyRecords(final String expectedRawRecordsFile, + final List actualRawRecords, + final String expectedFinalRecordsFile, + final List actualFinalRecords) { + assertAll( + () -> DIFFER.diffRawTableRecords( + BaseTypingDedupingTest.readRecords(expectedRawRecordsFile), + actualRawRecords), + () -> assertEquals( + 0, + actualRawRecords.stream() + .filter(record -> !record.hasNonNull("_airbyte_loaded_at")) + .count()), + () -> DIFFER.diffFinalTableRecords( + BaseTypingDedupingTest.readRecords(expectedFinalRecordsFile), + actualFinalRecords)); + } + + private void verifyRecordCounts(final int expectedRawRecords, + final List actualRawRecords, + final int expectedFinalRecords, + final List actualFinalRecords) { + assertAll( + () -> assertEquals( + expectedRawRecords, + actualRawRecords.size(), + "Raw record count was incorrect"), + () -> assertEquals( + 0, + actualRawRecords.stream() + .filter(record -> !record.hasNonNull("_airbyte_loaded_at")) + .count()), + () -> assertEquals( + expectedFinalRecords, + actualFinalRecords.size(), + "Final record count was incorrect")); + } + +} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/java/io/airbyte/integrations/base/destination/typing_deduping/BaseTypingDedupingTest.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/java/io/airbyte/integrations/base/destination/typing_deduping/BaseTypingDedupingTest.java new file mode 100644 index 000000000000..d4e46d7cc3d5 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/java/io/airbyte/integrations/base/destination/typing_deduping/BaseTypingDedupingTest.java @@ -0,0 +1,865 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.base.destination.typing_deduping; + +import static org.junit.jupiter.api.Assertions.assertAll; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.features.EnvVariableFeatureFlags; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.lang.Exceptions; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.configoss.WorkerDestinationConfig; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.SyncMode; +import io.airbyte.workers.internal.AirbyteDestination; +import io.airbyte.workers.internal.DefaultAirbyteDestination; +import io.airbyte.workers.process.AirbyteIntegrationLauncher; +import io.airbyte.workers.process.DockerProcessFactory; +import io.airbyte.workers.process.ProcessFactory; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.function.Function; +import java.util.stream.Stream; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This is loosely based on standard-destination-tests's DestinationAcceptanceTest class. The + * sync-running code is copy-pasted from there. + *

      + * All tests use a single stream, whose schema is defined in {@code resources/schema.json}. Each + * test case constructs a ConfiguredAirbyteCatalog dynamically. + *

      + * For sync modes which use a primary key, the stream provides a composite key of (id1, id2). For + * sync modes which use a cursor, the stream provides an updated_at field. The stream also has an + * _ab_cdc_deleted_at field. + */ +// If you're running from inside intellij, you must run your specific subclass to get concurrent +// execution. +@Execution(ExecutionMode.CONCURRENT) +public abstract class BaseTypingDedupingTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(BaseTypingDedupingTest.class); + protected static final JsonNode SCHEMA; + static { + try { + SCHEMA = Jsons.deserialize(MoreResources.readResource("dat/schema.json")); + } catch (final IOException e) { + throw new RuntimeException(e); + } + } + protected RecordDiffer DIFFER; + + private String randomSuffix; + private JsonNode config; + protected String streamNamespace; + protected String streamName; + private List streamsToTearDown; + + /** + * @return the docker image to run, e.g. {@code "airbyte/destination-bigquery:dev"}. + */ + protected abstract String getImageName(); + + /** + * Get the destination connector config. Subclasses may use this method for other setup work, e.g. + * opening a connection to the destination. + *

      + * Subclasses should _not_ start testcontainers in this method; that belongs in a BeforeAll method. + * The tests in this class are intended to be run concurrently on a shared database and will not + * interfere with each other. + *

      + * Sublcasses which need access to the config may use {@link #getConfig()}. + */ + protected abstract JsonNode generateConfig() throws Exception; + + /** + * For a given stream, return the records that exist in the destination's raw table. Each record + * must be in the format {"_airbyte_raw_id": "...", "_airbyte_extracted_at": "...", + * "_airbyte_loaded_at": "...", "_airbyte_data": {fields...}}. + *

      + * The {@code _airbyte_data} column must be an + * {@link com.fasterxml.jackson.databind.node.ObjectNode} (i.e. it cannot be a string value). + *

      + * streamNamespace may be null, in which case you should query from the default namespace. + */ + protected abstract List dumpRawTableRecords(String streamNamespace, String streamName) throws Exception; + + /** + * Utility method for tests to check if table exists + * + * @param streamNamespace + * @param streamName + * @return + * @throws Exception + */ + protected boolean checkTableExists(final String streamNamespace, final String streamName) { + // Implementation is specific to destination's tests. + return true; + } + + /** + * For a given stream, return the records that exist in the destination's final table. Each record + * must be in the format {"_airbyte_raw_id": "...", "_airbyte_extracted_at": "...", "_airbyte_meta": + * {...}, "field1": ..., "field2": ..., ...}. If the destination renames (e.g. upcases) the airbyte + * fields, this method must revert that naming to use the exact strings "_airbyte_raw_id", etc. + *

      + * For JSON-valued columns, there is some nuance: a SQL null should be represented as a missing + * entry, whereas a JSON null should be represented as a + * {@link com.fasterxml.jackson.databind.node.NullNode}. For example, in the JSON blob {"name": + * null}, the `name` field is a JSON null, and the `address` field is a SQL null. + *

      + * The corresponding SQL looks like + * {@code INSERT INTO ... (name, address) VALUES ('null' :: jsonb, NULL)}. + *

      + * streamNamespace may be null, in which case you should query from the default namespace. + */ + protected abstract List dumpFinalTableRecords(String streamNamespace, String streamName) throws Exception; + + /** + * Delete any resources in the destination associated with this stream AND its namespace. We need + * this because we write raw tables to a shared {@code airbyte} namespace, which we can't drop + * wholesale. Must handle the case where the table/namespace doesn't exist (e.g. if the connector + * crashed without writing any data). + *

      + * In general, this should resemble + * {@code DROP TABLE IF EXISTS airbyte._; DROP SCHEMA IF EXISTS }. + */ + protected abstract void teardownStreamAndNamespace(String streamNamespace, String streamName) throws Exception; + + protected abstract SqlGenerator getSqlGenerator(); + + /** + * Destinations which need to clean up resources after an entire test finishes should override this + * method. For example, if you want to gracefully close a database connection, you should do that + * here. + */ + protected void globalTeardown() throws Exception {} + + /** + * Conceptually identical to {@link #getFinalMetadataColumnNames()}, but for the raw table. + */ + protected Map getRawMetadataColumnNames() { + return new HashMap<>(); + } + + /** + * If the destination connector uses a nonstandard schema for the final table, override this method. + * For example, destination-snowflake upcases all column names in the final tables. + *

      + * You only need to add mappings for the airbyte metadata column names (_airbyte_raw_id, + * _airbyte_extracted_at, etc.). The test framework automatically populates mappings for the primary + * key and cursor using the SqlGenerator. + */ + protected Map getFinalMetadataColumnNames() { + return new HashMap<>(); + } + + /** + * @return A suffix which is different for each concurrent test, but stable within a single test. + */ + protected synchronized String getUniqueSuffix() { + if (randomSuffix == null) { + randomSuffix = "_" + RandomStringUtils.randomAlphabetic(10).toLowerCase(); + } + return randomSuffix; + } + + protected JsonNode getConfig() { + return config; + } + + /** + * Override this method only when skipping T&D and only compare raw tables and skip final table + * comparison. For every other case it should always return false. + * + * @return + */ + protected boolean disableFinalTableComparison() { + return false; + } + + @BeforeEach + public void setup() throws Exception { + config = generateConfig(); + streamNamespace = "typing_deduping_test" + getUniqueSuffix(); + streamName = "test_stream" + getUniqueSuffix(); + streamsToTearDown = new ArrayList<>(); + + final SqlGenerator generator = getSqlGenerator(); + DIFFER = new RecordDiffer( + getRawMetadataColumnNames(), + getFinalMetadataColumnNames(), + Pair.of(generator.buildColumnId("id1"), AirbyteProtocolType.INTEGER), + Pair.of(generator.buildColumnId("id2"), AirbyteProtocolType.INTEGER), + Pair.of(generator.buildColumnId("updated_at"), AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE), + Pair.of(generator.buildColumnId("old_cursor"), AirbyteProtocolType.INTEGER)); + + LOGGER.info("Using stream namespace {} and name {}", streamNamespace, streamName); + } + + @AfterEach + public void teardown() throws Exception { + for (final AirbyteStreamNameNamespacePair streamId : streamsToTearDown) { + teardownStreamAndNamespace(streamId.getNamespace(), streamId.getName()); + } + globalTeardown(); + } + + /** + * Starting with an empty destination, execute a full refresh overwrite sync. Verify that the + * records are written to the destination table. Then run a second sync, and verify that the records + * are overwritten. + */ + @Test + public void fullRefreshOverwrite() throws Exception { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.FULL_REFRESH) + .withDestinationSyncMode(DestinationSyncMode.OVERWRITE) + .withStream(new AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(SCHEMA)))); + + // First sync + final List messages1 = readMessages("dat/sync1_messages.jsonl"); + + runSync(catalog, messages1); + + final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_raw.jsonl"); + final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_final.jsonl"); + verifySyncResult(expectedRawRecords1, expectedFinalRecords1, disableFinalTableComparison()); + + // Second sync + final List messages2 = readMessages("dat/sync2_messages.jsonl"); + + runSync(catalog, messages2); + + final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl"); + final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl"); + verifySyncResult(expectedRawRecords2, expectedFinalRecords2, disableFinalTableComparison()); + } + + /** + * Starting with an empty destination, execute a full refresh append sync. Verify that the records + * are written to the destination table. Then run a second sync, and verify that the old and new + * records are all present. + */ + @Test + public void fullRefreshAppend() throws Exception { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.FULL_REFRESH) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(new AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(SCHEMA)))); + + // First sync + final List messages1 = readMessages("dat/sync1_messages.jsonl"); + + runSync(catalog, messages1); + + final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_raw.jsonl"); + final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_final.jsonl"); + verifySyncResult(expectedRawRecords1, expectedFinalRecords1, disableFinalTableComparison()); + + // Second sync + final List messages2 = readMessages("dat/sync2_messages.jsonl"); + + runSync(catalog, messages2); + + final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_raw.jsonl"); + final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_append_final.jsonl"); + verifySyncResult(expectedRawRecords2, expectedFinalRecords2, disableFinalTableComparison()); + } + + /** + * Starting with an empty destination, execute an incremental append sync. + *

      + * This is (not so secretly) identical to {@link #fullRefreshAppend()}, and uses the same set of + * expected records. Incremental as a concept only exists in the source. From the destination's + * perspective, we only care about the destination sync mode. + */ + @Test + public void incrementalAppend() throws Exception { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + // These two lines are literally the only difference between this test and fullRefreshAppend + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(List.of("updated_at")) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(new AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(SCHEMA)))); + + // First sync + final List messages1 = readMessages("dat/sync1_messages.jsonl"); + + runSync(catalog, messages1); + + final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_raw.jsonl"); + final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_final.jsonl"); + verifySyncResult(expectedRawRecords1, expectedFinalRecords1, disableFinalTableComparison()); + + // Second sync + final List messages2 = readMessages("dat/sync2_messages.jsonl"); + + runSync(catalog, messages2); + + final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_raw.jsonl"); + final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_append_final.jsonl"); + verifySyncResult(expectedRawRecords2, expectedFinalRecords2, disableFinalTableComparison()); + } + + /** + * Starting with an empty destination, execute an incremental dedup sync. Verify that the records + * are written to the destination table. Then run a second sync, and verify that the raw/final + * tables contain the correct records. + */ + @Test + public void incrementalDedup() throws Exception { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(List.of("updated_at")) + .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP) + .withPrimaryKey(List.of(List.of("id1"), List.of("id2"))) + .withStream(new AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(SCHEMA)))); + + // First sync + final List messages1 = readMessages("dat/sync1_messages.jsonl"); + + runSync(catalog, messages1); + + final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_raw.jsonl"); + final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_dedup_final.jsonl"); + verifySyncResult(expectedRawRecords1, expectedFinalRecords1, disableFinalTableComparison()); + + // Second sync + final List messages2 = readMessages("dat/sync2_messages.jsonl"); + + runSync(catalog, messages2); + + final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_raw.jsonl"); + final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_incremental_dedup_final.jsonl"); + verifySyncResult(expectedRawRecords2, expectedFinalRecords2, disableFinalTableComparison()); + } + + /** + * Identical to {@link #incrementalDedup()}, except that the stream has no namespace. + */ + @Test + public void incrementalDedupDefaultNamespace() throws Exception { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(List.of("updated_at")) + .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP) + .withPrimaryKey(List.of(List.of("id1"), List.of("id2"))) + .withStream(new AirbyteStream() + // NB: we don't call `withNamespace` here + .withName(streamName) + .withJsonSchema(SCHEMA)))); + + // First sync + final List messages1 = readMessages("dat/sync1_messages.jsonl", null, streamName); + + runSync(catalog, messages1); + + final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_raw.jsonl"); + final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_dedup_final.jsonl"); + verifySyncResult(expectedRawRecords1, expectedFinalRecords1, null, streamName, disableFinalTableComparison()); + + // Second sync + final List messages2 = readMessages("dat/sync2_messages.jsonl", null, streamName); + + runSync(catalog, messages2); + + final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_raw.jsonl"); + final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_incremental_dedup_final.jsonl"); + verifySyncResult(expectedRawRecords2, expectedFinalRecords2, null, streamName, disableFinalTableComparison()); + } + + @Test + @Disabled("Not yet implemented") + public void testLineBreakCharacters() throws Exception { + // TODO verify that we can handle strings with interesting characters + // build an airbyterecordmessage using something like this, and add it to the input messages: + Jsons.jsonNode(ImmutableMap.builder() + .put("id", 1) + .put("currency", "USD\u2028") + .put("date", "2020-03-\n31T00:00:00Z\r") + // TODO(sherifnada) hack: write decimals with sigfigs because Snowflake stores 10.1 as "10" which + // fails destination tests + .put("HKD", 10.1) + .put("NZD", 700.1) + .build()); + } + + /** + * Run a sync, then remove the {@code name} column from the schema and run a second sync. Verify + * that the final table doesn't contain the `name` column after the second sync. + */ + @Test + public void testIncrementalSyncDropOneColumn() throws Exception { + final AirbyteStream stream = new AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(SCHEMA); + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(List.of("updated_at")) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(stream))); + + // First sync + final List messages1 = readMessages("dat/sync1_messages.jsonl"); + + runSync(catalog, messages1); + + final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_raw.jsonl"); + final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_final.jsonl"); + verifySyncResult(expectedRawRecords1, expectedFinalRecords1, disableFinalTableComparison()); + + // Second sync + final List messages2 = readMessages("dat/sync2_messages.jsonl"); + final JsonNode trimmedSchema = SCHEMA.deepCopy(); + ((ObjectNode) trimmedSchema.get("properties")).remove("name"); + stream.setJsonSchema(trimmedSchema); + + runSync(catalog, messages2); + + // The raw data is unaffected by the schema, but the final table should not have a `name` column. + final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_raw.jsonl"); + final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_append_final.jsonl").stream() + .peek(record -> ((ObjectNode) record).remove(getSqlGenerator().buildColumnId("name").name())) + .toList(); + verifySyncResult(expectedRawRecords2, expectedFinalRecords2, disableFinalTableComparison()); + } + + @Test + @Disabled("Not yet implemented") + public void testSyncUsesAirbyteStreamNamespaceIfNotNull() throws Exception { + // TODO duplicate this test for each sync mode. Run 1st+2nd syncs using a stream with null + // namespace: + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.FULL_REFRESH) + .withCursorField(List.of("updated_at")) + .withDestinationSyncMode(DestinationSyncMode.OVERWRITE) + .withPrimaryKey(List.of(List.of("id1"), List.of("id2"))) + .withStream(new AirbyteStream() + .withNamespace(null) + .withName(streamName) + .withJsonSchema(SCHEMA)))); + } + + // TODO duplicate this test for each sync mode. Run 1st+2nd syncs using two streams with the same + // name but different namespace + // TODO maybe we don't even need the single-stream versions... + /** + * Identical to {@link #incrementalDedup()}, except there are two streams with the same name and + * different namespace. + */ + @Test + public void incrementalDedupIdenticalName() throws Exception { + final String namespace1 = streamNamespace + "_1"; + final String namespace2 = streamNamespace + "_2"; + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(List.of("updated_at")) + .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP) + .withPrimaryKey(List.of(List.of("id1"), List.of("id2"))) + .withStream(new AirbyteStream() + .withNamespace(namespace1) + .withName(streamName) + .withJsonSchema(SCHEMA)), + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(List.of("updated_at")) + .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP) + .withPrimaryKey(List.of(List.of("id1"), List.of("id2"))) + .withStream(new AirbyteStream() + .withNamespace(namespace2) + .withName(streamName) + .withJsonSchema(SCHEMA)))); + + // First sync + final List messages1 = Stream.concat( + readMessages("dat/sync1_messages.jsonl", namespace1, streamName).stream(), + readMessages("dat/sync1_messages2.jsonl", namespace2, streamName).stream()).toList(); + + runSync(catalog, messages1); + + verifySyncResult( + readRecords("dat/sync1_expectedrecords_raw.jsonl"), + readRecords("dat/sync1_expectedrecords_dedup_final.jsonl"), + namespace1, + streamName, disableFinalTableComparison()); + verifySyncResult( + readRecords("dat/sync1_expectedrecords_raw2.jsonl"), + readRecords("dat/sync1_expectedrecords_dedup_final2.jsonl"), + namespace2, + streamName, disableFinalTableComparison()); + + // Second sync + final List messages2 = Stream.concat( + readMessages("dat/sync2_messages.jsonl", namespace1, streamName).stream(), + readMessages("dat/sync2_messages2.jsonl", namespace2, streamName).stream()).toList(); + + runSync(catalog, messages2); + + verifySyncResult( + readRecords("dat/sync2_expectedrecords_raw.jsonl"), + readRecords("dat/sync2_expectedrecords_incremental_dedup_final.jsonl"), + namespace1, + streamName, disableFinalTableComparison()); + verifySyncResult( + readRecords("dat/sync2_expectedrecords_raw2.jsonl"), + readRecords("dat/sync2_expectedrecords_incremental_dedup_final2.jsonl"), + namespace2, + streamName, disableFinalTableComparison()); + } + + /** + * Run two syncs at the same time. They each have one stream, which has the same name for both syncs + * but different namespace. This should work fine. This test is similar to + * {@link #incrementalDedupIdenticalName()}, but uses two separate syncs instead of one sync with + * two streams. + *

      + * Note that destination stdout is a bit misleading: The two syncs' stdout _should_ be interleaved, + * but we're just dumping the entire sync1 stdout, and then the entire sync2 stdout. + */ + @Test + public void identicalNameSimultaneousSync() throws Exception { + final String namespace1 = streamNamespace + "_1"; + final ConfiguredAirbyteCatalog catalog1 = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(List.of("updated_at")) + .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP) + .withPrimaryKey(List.of(List.of("id1"), List.of("id2"))) + .withStream(new AirbyteStream() + .withNamespace(namespace1) + .withName(streamName) + .withJsonSchema(SCHEMA)))); + + final String namespace2 = streamNamespace + "_2"; + final ConfiguredAirbyteCatalog catalog2 = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(List.of("updated_at")) + .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP) + .withPrimaryKey(List.of(List.of("id1"), List.of("id2"))) + .withStream(new AirbyteStream() + .withNamespace(namespace2) + .withName(streamName) + .withJsonSchema(SCHEMA)))); + + final List messages1 = readMessages("dat/sync1_messages.jsonl", namespace1, streamName); + final List messages2 = readMessages("dat/sync1_messages2.jsonl", namespace2, streamName); + + // Start two concurrent syncs + final AirbyteDestination sync1 = startSync(catalog1); + final AirbyteDestination sync2 = startSync(catalog2); + // Write some messages to both syncs. Write a lot of data to sync 2 to try and force a flush. + pushMessages(messages1, sync1); + for (int i = 0; i < 100_000; i++) { + pushMessages(messages2, sync2); + } + endSync(sync1); + // Write some more messages to the second sync. It should not be affected by the first sync's + // shutdown. + for (int i = 0; i < 100_000; i++) { + pushMessages(messages2, sync2); + } + endSync(sync2); + + // For simplicity, don't verify the raw table. Assume that if the final table is correct, then + // the raw data is correct. This is generally a safe assumption. + assertAll( + () -> DIFFER.diffFinalTableRecords( + readRecords("dat/sync1_expectedrecords_dedup_final.jsonl"), + dumpFinalTableRecords(namespace1, streamName)), + () -> DIFFER.diffFinalTableRecords( + readRecords("dat/sync1_expectedrecords_dedup_final2.jsonl"), + dumpFinalTableRecords(namespace2, streamName))); + } + + @Test + @Disabled("Not yet implemented") + public void testSyncNotFailsWithNewFields() throws Exception { + // TODO duplicate this test for each sync mode. Run a sync, then add a new field to the schema, then + // run another sync + // We might want to write a test that verifies more general schema evolution (e.g. all valid + // evolutions) + } + + /** + * Change the cursor column in the second sync to a column that doesn't exist in the first sync. + * Verify that we overwrite everything correctly. + *

      + * This essentially verifies that the destination connector correctly recognizes NULL cursors as + * older than non-NULL cursors. + */ + @Test + public void incrementalDedupChangeCursor() throws Exception { + final JsonNode mangledSchema = SCHEMA.deepCopy(); + ((ObjectNode) mangledSchema.get("properties")).remove("updated_at"); + ((ObjectNode) mangledSchema.get("properties")).set( + "old_cursor", + Jsons.deserialize( + """ + {"type": "integer"} + """)); + final ConfiguredAirbyteStream configuredStream = new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(List.of("old_cursor")) + .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP) + .withPrimaryKey(List.of(List.of("id1"), List.of("id2"))) + .withStream(new AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(mangledSchema)); + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of(configuredStream)); + + // First sync + final List messages1 = readMessages("dat/sync1_cursorchange_messages.jsonl"); + + runSync(catalog, messages1); + + final List expectedRawRecords1 = readRecords("dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl"); + final List expectedFinalRecords1 = readRecords("dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl"); + verifySyncResult(expectedRawRecords1, expectedFinalRecords1, disableFinalTableComparison()); + + // Second sync + final List messages2 = readMessages("dat/sync2_messages.jsonl"); + configuredStream.getStream().setJsonSchema(SCHEMA); + configuredStream.setCursorField(List.of("updated_at")); + + runSync(catalog, messages2); + + final List expectedRawRecords2 = readRecords("dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl"); + final List expectedFinalRecords2 = readRecords("dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl"); + verifySyncResult(expectedRawRecords2, expectedFinalRecords2, disableFinalTableComparison()); + } + + @Test + @Disabled("Not yet implemented") + public void testSyncWithLargeRecordBatch() throws Exception { + // TODO duplicate this test for each sync mode. Run a single sync with many records + /* + * copied from DATs: This serves to test MSSQL 2100 limit parameters in a single query. this means + * that for Airbyte insert data need to limit to ~ 700 records (3 columns for the raw tables) = 2100 + * params + * + * this maybe needs configuration per destination to specify that limit? + */ + } + + @Test + @Disabled("Not yet implemented") + public void testDataTypes() throws Exception { + // TODO duplicate this test for each sync mode. See DataTypeTestArgumentProvider for what this test + // does in DAT-land + // we probably don't want to do the exact same thing, but the general spirit of testing a wide range + // of values for every data type is approximately correct + // this test probably needs some configuration per destination to specify what values are supported? + } + + protected void verifySyncResult(final List expectedRawRecords, + final List expectedFinalRecords, + final boolean disableFinalTableComparison) + throws Exception { + verifySyncResult(expectedRawRecords, expectedFinalRecords, streamNamespace, streamName, disableFinalTableComparison); + } + + private void verifySyncResult(final List expectedRawRecords, + final List expectedFinalRecords, + final String streamNamespace, + final String streamName, + final boolean disableFinalTableComparison) + throws Exception { + final List actualRawRecords = dumpRawTableRecords(streamNamespace, streamName); + if (disableFinalTableComparison) { + DIFFER.diffRawTableRecords(expectedRawRecords, actualRawRecords); + } else { + final List actualFinalRecords = dumpFinalTableRecords(streamNamespace, streamName); + DIFFER.verifySyncResult(expectedRawRecords, actualRawRecords, expectedFinalRecords, actualFinalRecords); + } + } + + public static List readRecords(final String filename) throws IOException { + return MoreResources.readResource(filename).lines() + .map(String::trim) + .filter(line -> !line.isEmpty()) + .filter(line -> !line.startsWith("//")) + .map(Jsons::deserializeExact) + .toList(); + } + + protected List readMessages(final String filename) throws IOException { + return readMessages(filename, streamNamespace, streamName); + } + + protected static List readMessages(final String filename, final String streamNamespace, final String streamName) + throws IOException { + return readRecords(filename).stream() + .map(record -> Jsons.convertValue(record, AirbyteMessage.class)) + .peek(message -> { + message.getRecord().setNamespace(streamNamespace); + message.getRecord().setStream(streamName); + }).toList(); + } + + /* + * !!!!!! WARNING !!!!!! The code below was mostly copypasted from DestinationAcceptanceTest. If you + * make edits here, you probably want to also edit there. + */ + + protected void runSync(final ConfiguredAirbyteCatalog catalog, final List messages) throws Exception { + runSync(catalog, messages, getImageName()); + } + + protected void runSync(final ConfiguredAirbyteCatalog catalog, final List messages, final String imageName) throws Exception { + runSync(catalog, messages, imageName, Function.identity()); + } + + protected void runSync(final ConfiguredAirbyteCatalog catalog, + final List messages, + final String imageName, + final Function configTransformer) + throws Exception { + final AirbyteDestination destination = startSync(catalog, imageName, configTransformer); + pushMessages(messages, destination); + endSync(destination); + } + + protected AirbyteDestination startSync(final ConfiguredAirbyteCatalog catalog) throws Exception { + return startSync(catalog, getImageName()); + } + + protected AirbyteDestination startSync(final ConfiguredAirbyteCatalog catalog, final String imageName) throws Exception { + return startSync(catalog, imageName, Function.identity()); + } + + /** + * + * @param catalog + * @param imageName + * @param configTransformer - test specific config overrides or additions can be performed with this + * function + * @return + * @throws Exception + */ + protected AirbyteDestination startSync(final ConfiguredAirbyteCatalog catalog, + final String imageName, + final Function configTransformer) + throws Exception { + synchronized (this) { + catalog.getStreams().forEach(s -> streamsToTearDown.add(AirbyteStreamNameNamespacePair.fromAirbyteStream(s.getStream()))); + } + + final Path testDir = Path.of("/tmp/airbyte_tests/"); + Files.createDirectories(testDir); + final Path workspaceRoot = Files.createTempDirectory(testDir, "test"); + final Path jobRoot = Files.createDirectories(Path.of(workspaceRoot.toString(), "job")); + final Path localRoot = Files.createTempDirectory(testDir, "output"); + final ProcessFactory processFactory = new DockerProcessFactory( + workspaceRoot, + workspaceRoot.toString(), + localRoot.toString(), + "host", + Collections.emptyMap()); + final JsonNode transformedConfig = configTransformer.apply(config); + final WorkerDestinationConfig destinationConfig = new WorkerDestinationConfig() + .withConnectionId(UUID.randomUUID()) + .withCatalog(convertProtocolObject(catalog, io.airbyte.protocol.models.ConfiguredAirbyteCatalog.class)) + .withDestinationConnectionConfiguration(transformedConfig); + + final AirbyteDestination destination = new DefaultAirbyteDestination(new AirbyteIntegrationLauncher( + "0", + 0, + imageName, + processFactory, + null, + null, + false, + new EnvVariableFeatureFlags())); + + destination.start(destinationConfig, jobRoot, Collections.emptyMap()); + + // In the background, read messages from the destination until it terminates. We need to clear + // stdout in real time, to prevent the buffer from filling up and blocking the destination. + // TODO Eventually we'll want to somehow extract the state messages while a sync is running, to + // verify checkpointing. + final ExecutorService messageHandler = Executors.newSingleThreadExecutor( + // run as a daemon thread just in case we run into an exception or something + r -> { + final Thread t = Executors.defaultThreadFactory().newThread(r); + t.setDaemon(true); + return t; + }); + messageHandler.submit(() -> { + while (!destination.isFinished()) { + // attemptRead isn't threadsafe, we read stdout fully here. + // i.e. we shouldn't call attemptRead anywhere else. + destination.attemptRead(); + } + }); + messageHandler.shutdown(); + + return destination; + } + + protected static void pushMessages(final List messages, final AirbyteDestination destination) { + messages.forEach( + message -> Exceptions.toRuntime(() -> destination.accept(convertProtocolObject(message, io.airbyte.protocol.models.AirbyteMessage.class)))); + } + + protected static void endSync(final AirbyteDestination destination) throws Exception { + destination.notifyEndOfInput(); + destination.close(); + } + + private static V0 convertProtocolObject(final V1 v1, final Class klass) { + return Jsons.object(Jsons.jsonNode(v1), klass); + } + +} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/RecordDiffer.java b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/java/io/airbyte/integrations/base/destination/typing_deduping/RecordDiffer.java similarity index 75% rename from airbyte-integrations/bases/base-typing-deduping-test/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/RecordDiffer.java rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/java/io/airbyte/integrations/base/destination/typing_deduping/RecordDiffer.java index 058986346d29..e4cea52498a8 100644 --- a/airbyte-integrations/bases/base-typing-deduping-test/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/RecordDiffer.java +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/java/io/airbyte/integrations/base/destination/typing_deduping/RecordDiffer.java @@ -37,19 +37,49 @@ */ public class RecordDiffer { - private final Comparator recordIdentityComparator; - private final Comparator recordSortComparator; - private final Function recordIdentityExtractor; + private final Comparator rawRecordIdentityComparator; + private final Comparator rawRecordSortComparator; + private final Function rawRecordIdentityExtractor; + private final Map rawRecordColumnNames; + + private final Comparator finalRecordIdentityComparator; + private final Comparator finalRecordSortComparator; + private final Function finalRecordIdentityExtractor; + private final Map finalRecordColumnNames; /** + * @param rawRecordColumnNames + * @param finalRecordColumnNames * @param identifyingColumns Which fields constitute a unique record (typically PK+cursor). Do _not_ * include extracted_at; it is handled automatically. */ @SafeVarargs - public RecordDiffer(final Pair... identifyingColumns) { - this.recordIdentityComparator = buildIdentityComparator(identifyingColumns); - this.recordSortComparator = recordIdentityComparator.thenComparing(record -> asString(record.get("_airbyte_raw_id"))); - this.recordIdentityExtractor = buildIdentityExtractor(identifyingColumns); + public RecordDiffer(final Map rawRecordColumnNames, + final Map finalRecordColumnNames, + final Pair... identifyingColumns) { + this.rawRecordColumnNames = rawRecordColumnNames; + this.finalRecordColumnNames = finalRecordColumnNames; + final Pair[] rawTableIdentifyingColumns = Arrays.stream(identifyingColumns) + .map(p -> Pair.of( + // Raw tables always retain the original column names + p.getLeft().originalName(), + p.getRight())) + .toArray(Pair[]::new); + this.rawRecordIdentityComparator = buildIdentityComparator(rawTableIdentifyingColumns, rawRecordColumnNames); + this.rawRecordSortComparator = rawRecordIdentityComparator + .thenComparing(record -> asString(record.get(getMetadataColumnName(rawRecordColumnNames, "_airbyte_raw_id")))); + this.rawRecordIdentityExtractor = buildIdentityExtractor(rawTableIdentifyingColumns, rawRecordColumnNames); + + final Pair[] finalTableIdentifyingColumns = Arrays.stream(identifyingColumns) + .map(p -> Pair.of( + // Final tables may have modified the column names, so use the final name here. + p.getLeft().name(), + p.getRight())) + .toArray(Pair[]::new); + this.finalRecordIdentityComparator = buildIdentityComparator(finalTableIdentifyingColumns, finalRecordColumnNames); + this.finalRecordSortComparator = finalRecordIdentityComparator + .thenComparing(record -> asString(record.get(getMetadataColumnName(finalRecordColumnNames, "_airbyte_raw_id")))); + this.finalRecordIdentityExtractor = buildIdentityExtractor(finalTableIdentifyingColumns, finalRecordColumnNames); } /** @@ -68,11 +98,12 @@ public void verifySyncResult(final List expectedRawRecords, public void diffRawTableRecords(final List expectedRecords, final List actualRecords) { final String diff = diffRecords( - expectedRecords.stream().map(RecordDiffer::copyWithLiftedData).collect(toList()), - actualRecords.stream().map(RecordDiffer::copyWithLiftedData).collect(toList()), - recordIdentityComparator, - recordSortComparator, - recordIdentityExtractor); + expectedRecords.stream().map(this::copyWithLiftedData).collect(toList()), + actualRecords.stream().map(this::copyWithLiftedData).collect(toList()), + rawRecordIdentityComparator, + rawRecordSortComparator, + rawRecordIdentityExtractor, + rawRecordColumnNames); if (!diff.isEmpty()) { fail("Raw table was incorrect.\n" + diff); @@ -83,9 +114,10 @@ public void diffFinalTableRecords(final List expectedRecords, final Li final String diff = diffRecords( expectedRecords, actualRecords, - recordIdentityComparator, - recordSortComparator, - recordIdentityExtractor); + finalRecordIdentityComparator, + finalRecordSortComparator, + finalRecordIdentityExtractor, + finalRecordColumnNames); if (!diff.isEmpty()) { fail("Final table was incorrect.\n" + diff); @@ -97,10 +129,10 @@ public void diffFinalTableRecords(final List expectedRecords, final Li * * @return A copy of the record, but with all fields in _airbyte_data lifted to the top level. */ - private static JsonNode copyWithLiftedData(final JsonNode record) { + private JsonNode copyWithLiftedData(final JsonNode record) { final ObjectNode copy = record.deepCopy(); - copy.remove("_airbyte_data"); - JsonNode airbyteData = record.get("_airbyte_data"); + copy.remove(getMetadataColumnName(rawRecordColumnNames, "_airbyte_data")); + JsonNode airbyteData = record.get(getMetadataColumnName(rawRecordColumnNames, "_airbyte_data")); if (airbyteData.isTextual()) { airbyteData = Jsons.deserializeExact(airbyteData.asText()); } @@ -120,24 +152,26 @@ private static JsonNode copyWithLiftedData(final JsonNode record) { * Build a Comparator to detect equality between two records. It first compares all the identifying * columns in order, and breaks ties using extracted_at. */ - private Comparator buildIdentityComparator(final Pair[] identifyingColumns) { + private Comparator buildIdentityComparator(final Pair[] identifyingColumns, final Map columnNames) { // Start with a noop comparator for convenience Comparator comp = Comparator.comparing(record -> 0); for (final Pair column : identifyingColumns) { comp = comp.thenComparing(record -> extract(record, column.getKey(), column.getValue())); } - comp = comp.thenComparing(record -> asTimestampWithTimezone(record.get("_airbyte_extracted_at"))); + comp = comp.thenComparing(record -> asTimestampWithTimezone(record.get(getMetadataColumnName(columnNames, "_airbyte_extracted_at")))); return comp; } /** - * See {@link #buildIdentityComparator(Pair[])} for an explanation of dataExtractor. + * See {@link #buildIdentityComparator(Pair[], Map)} for an explanation of + * dataExtractor. */ - private Function buildIdentityExtractor(final Pair[] identifyingColumns) { + private Function buildIdentityExtractor(final Pair[] identifyingColumns, + final Map columnNames) { return record -> Arrays.stream(identifyingColumns) .map(column -> getPrintableFieldIfPresent(record, column.getKey())) .collect(Collectors.joining(", ")) - + getPrintableFieldIfPresent(record, "_airbyte_extracted_at"); + + getPrintableFieldIfPresent(record, getMetadataColumnName(columnNames, "_airbyte_extracted_at")); } private static String getPrintableFieldIfPresent(final JsonNode record, final String field) { @@ -165,11 +199,12 @@ private static String getPrintableFieldIfPresent(final JsonNode record, final St * @param recordIdExtractor Dump the record's PK+cursor+extracted_at into a human-readable string * @return The diff, or empty string if there were no differences */ - private static String diffRecords(final List originalExpectedRecords, - final List originalActualRecords, - final Comparator identityComparator, - final Comparator sortComparator, - final Function recordIdExtractor) { + private String diffRecords(final List originalExpectedRecords, + final List originalActualRecords, + final Comparator identityComparator, + final Comparator sortComparator, + final Function recordIdExtractor, + final Map columnNames) { final List expectedRecords = originalExpectedRecords.stream().sorted(sortComparator).toList(); final List actualRecords = originalActualRecords.stream().sorted(sortComparator).toList(); @@ -185,7 +220,7 @@ private static String diffRecords(final List originalExpectedRecords, if (compare == 0) { // These records should be the same. Find the specific fields that are different and move on // to the next records in both lists. - message += diffSingleRecord(recordIdExtractor, expectedRecord, actualRecord); + message += diffSingleRecord(recordIdExtractor, expectedRecord, actualRecord, columnNames); expectedRecordIndex++; actualRecordIndex++; } else if (compare < 0) { @@ -213,9 +248,10 @@ private static String diffRecords(final List originalExpectedRecords, return message; } - private static String diffSingleRecord(final Function recordIdExtractor, - final JsonNode expectedRecord, - final JsonNode actualRecord) { + private String diffSingleRecord(final Function recordIdExtractor, + final JsonNode expectedRecord, + final JsonNode actualRecord, + final Map columnNames) { boolean foundMismatch = false; String mismatchedRecordMessage = "Row had incorrect data: " + recordIdExtractor.apply(expectedRecord) + "\n"; // Iterate through each column in the expected record and compare it to the actual record's value. @@ -229,7 +265,7 @@ private static String diffSingleRecord(final Function recordId } } // Then check the entire actual record for any columns that we weren't expecting. - final LinkedHashMap extraColumns = checkForExtraOrNonNullFields(expectedRecord, actualRecord); + final LinkedHashMap extraColumns = checkForExtraOrNonNullFields(expectedRecord, actualRecord, columnNames); if (extraColumns.size() > 0) { for (final Map.Entry extraColumn : extraColumns.entrySet()) { mismatchedRecordMessage += generateFieldError("column " + extraColumn.getKey(), null, extraColumn.getValue()); @@ -277,11 +313,16 @@ private static boolean areJsonNodesEquivalent(final JsonNode expectedValue, fina * This has the side benefit of detecting completely unexpected columns, which would be a very weird * bug but is probably still useful to catch. */ - private static LinkedHashMap checkForExtraOrNonNullFields(final JsonNode expectedRecord, final JsonNode actualRecord) { + private LinkedHashMap checkForExtraOrNonNullFields(final JsonNode expectedRecord, + final JsonNode actualRecord, + final Map columnNames) { final LinkedHashMap extraFields = new LinkedHashMap<>(); for (final String column : Streams.stream(actualRecord.fieldNames()).sorted().toList()) { // loaded_at and raw_id are generated dynamically, so we just ignore them. - if (!"_airbyte_loaded_at".equals(column) && !"_airbyte_raw_id".equals(column) && !expectedRecord.has(column)) { + final boolean isLoadedAt = getMetadataColumnName(columnNames, "_airbyte_loaded_at").equals(column); + final boolean isRawId = getMetadataColumnName(columnNames, "_airbyte_raw_id").equals(column); + final boolean isExpected = expectedRecord.has(column); + if (!(isLoadedAt || isRawId || isExpected)) { extraFields.put(column, actualRecord.get(column)); } } @@ -411,4 +452,8 @@ private static Comparable extract(final JsonNode node, final String field, final } } + private String getMetadataColumnName(final Map columnNames, final String columnName) { + return columnNames.getOrDefault(columnName, columnName); + } + } diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/dat/schema.json b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/dat/schema.json similarity index 100% rename from airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/dat/schema.json rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/dat/schema.json diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/dat/sync1_cursorchange_messages.jsonl b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/dat/sync1_cursorchange_messages.jsonl similarity index 100% rename from airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/dat/sync1_cursorchange_messages.jsonl rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/dat/sync1_cursorchange_messages.jsonl diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/dat/sync1_messages.jsonl b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/dat/sync1_messages.jsonl similarity index 100% rename from airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/dat/sync1_messages.jsonl rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/dat/sync1_messages.jsonl diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/dat/sync1_messages2.jsonl b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/dat/sync1_messages2.jsonl new file mode 100644 index 000000000000..3a1d3052f621 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/dat/sync1_messages2.jsonl @@ -0,0 +1 @@ +{"type": "RECORD", "record": {"emitted_at": 1000, "data": {"id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different"}}} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/dat/sync2_messages.jsonl b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/dat/sync2_messages.jsonl similarity index 100% rename from airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/dat/sync2_messages.jsonl rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/dat/sync2_messages.jsonl diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/dat/sync2_messages2.jsonl b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/dat/sync2_messages2.jsonl new file mode 100644 index 000000000000..42dac3ee1174 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/dat/sync2_messages2.jsonl @@ -0,0 +1 @@ +{"type": "RECORD", "record": {"emitted_at": 2000, "data": {"id1": 1, "id2": 200, "updated_at": "2001-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different v2"}}} diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/dat/sync_null_pk.jsonl b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/dat/sync_null_pk.jsonl new file mode 100644 index 000000000000..03cc6d40e21d --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/dat/sync_null_pk.jsonl @@ -0,0 +1,4 @@ +// there is no entry for id2, which is a required PK for this schema +{"type": "RECORD", "record": {"emitted_at": 1000, "data": {"id1": 1, "id2": null, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}}} +{"type": "RECORD", "record": {"emitted_at": 2000, "data": {"id1": 1, "id2": null, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}}} +{"type": "RECORD", "record": {"emitted_at": 3000, "data": {"id1": 1, "id2": null, "updated_at": "2000-01-03T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}}} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/all_types_v1_inputrecords.jsonl b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/all_types_v1_inputrecords.jsonl similarity index 77% rename from airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/all_types_v1_inputrecords.jsonl rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/all_types_v1_inputrecords.jsonl index 71e96f28af46..e2cde49ad980 100644 --- a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/all_types_v1_inputrecords.jsonl +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/all_types_v1_inputrecords.jsonl @@ -2,5 +2,6 @@ {"_airbyte_ab_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_emitted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": null, "struct": null, "string": null, "number": null, "integer": null, "boolean": null, "timestamp_with_timezone": null, "timestamp_without_timezone": null, "time_with_timezone": null, "time_without_timezone": null, "date": null, "unknown": null}} {"_airbyte_ab_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_emitted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z"}} // Note that array and struct have invalid values ({} and [] respectively). -{"_airbyte_ab_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_emitted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": {}, "number": {}, "integer": {}, "boolean": {}, "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": {}, "unknown": null}} +{"_airbyte_ab_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_emitted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": null, "number": "foo", "integer": "bar", "boolean": "fizz", "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": "airbyte", "unknown": null}} {"_airbyte_ab_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_emitted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118}} +{"_airbyte_ab_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fce", "_airbyte_emitted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 6, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "IamACaseSensitiveColumnName": "Case senstive value"}} \ No newline at end of file diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/alltypes_inputrecords.jsonl b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/alltypes_inputrecords.jsonl new file mode 100644 index 000000000000..c21fc0bbb6ab --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/alltypes_inputrecords.jsonl @@ -0,0 +1,7 @@ +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": null, "struct": null, "string": null, "number": null, "integer": null, "boolean": null, "timestamp_with_timezone": null, "timestamp_without_timezone": null, "time_with_timezone": null, "time_without_timezone": null, "date": null, "unknown": null}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z"}} +// Note that array and struct have invalid values ({} and [] respectively). +{"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": null, "number": "foo", "integer": "bar", "boolean": "fizz", "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": "airbyte", "unknown": null}} +{"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fce", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 6, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "IamACaseSensitiveColumnName": "Case senstive value"}} \ No newline at end of file diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/alltypes_unsafe_inputrecords.jsonl b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/alltypes_unsafe_inputrecords.jsonl new file mode 100644 index 000000000000..55a509408d14 --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/alltypes_unsafe_inputrecords.jsonl @@ -0,0 +1,3 @@ +// this is a strict subset of the alltypes_inputrecords file. All these records have valid values, i.e. can be processed with unsafe casting. +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": null, "struct": null, "string": null, "number": null, "integer": null, "boolean": null, "timestamp_with_timezone": null, "timestamp_without_timezone": null, "time_with_timezone": null, "time_without_timezone": null, "date": null, "unknown": null}} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcordering_insertafterdelete_inputrecords_final.jsonl b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/cdcordering_insertafterdelete_inputrecords_final.jsonl similarity index 100% rename from airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcordering_insertafterdelete_inputrecords_final.jsonl rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/cdcordering_insertafterdelete_inputrecords_final.jsonl diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcordering_insertafterdelete_inputrecords_raw.jsonl b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/cdcordering_insertafterdelete_inputrecords_raw.jsonl similarity index 100% rename from airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcordering_insertafterdelete_inputrecords_raw.jsonl rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/cdcordering_insertafterdelete_inputrecords_raw.jsonl diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcordering_updateafterdelete_inputrecords.jsonl b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/cdcordering_updateafterdelete_inputrecords.jsonl similarity index 100% rename from airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcordering_updateafterdelete_inputrecords.jsonl rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/cdcordering_updateafterdelete_inputrecords.jsonl diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcupdate_inputrecords_final.jsonl b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/cdcupdate_inputrecords_final.jsonl similarity index 100% rename from airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcupdate_inputrecords_final.jsonl rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/cdcupdate_inputrecords_final.jsonl diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcupdate_inputrecords_raw.jsonl b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/cdcupdate_inputrecords_raw.jsonl similarity index 94% rename from airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcupdate_inputrecords_raw.jsonl rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/cdcupdate_inputrecords_raw.jsonl index 2327710d6e84..e5752b06c025 100644 --- a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/cdcupdate_inputrecords_raw.jsonl +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/cdcupdate_inputrecords_raw.jsonl @@ -12,5 +12,5 @@ {"_airbyte_raw_id": "4d8674a5-eb6e-41ca-a310-69c64c88d101", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 0, "id2": 100, "updated_at": "2023-01-01T05:00:00Z", "_ab_cdc_deleted_at": null, "string": "zombie_returned"}} // CDC generally outputs an explicit null for deleted_at, but verify that we can also handle the case where deleted_at is unset. {"_airbyte_raw_id": "f0b59e49-8c74-4101-9f14-cb4d1193fd5a", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T06:00:00Z", "string": "charlie"}} -// Verify that we can handle weird values in deleted_at +// Invalid values in _ab_cdc_deleted_at result in the record NOT being deleted. This behavior is up for debate, but it's an extreme edge case so not a high priority. {"_airbyte_raw_id": "d4e1d989-c115-403c-9e68-5d320e6376bb", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T07:00:00Z", "_ab_cdc_deleted_at": {}, "string": "david1"}} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/incrementaldedup_inputrecords.jsonl b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/incrementaldedup_inputrecords.jsonl similarity index 100% rename from airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/incrementaldedup_inputrecords.jsonl rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/incrementaldedup_inputrecords.jsonl diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/json_types_in_string_inputrecords.jsonl b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/json_types_in_string_inputrecords.jsonl new file mode 100644 index 000000000000..96ce4458e7af --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/json_types_in_string_inputrecords.jsonl @@ -0,0 +1,5 @@ +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": ["I", "am", "an", "array"], "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": {"I": "am", "an": "object"}, "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": true, "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": 3.14, "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "I am a valid json string", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} \ No newline at end of file diff --git a/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/reservedkeywords_inputrecords_raw.jsonl b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/reservedkeywords_inputrecords_raw.jsonl new file mode 100644 index 000000000000..a2c5acd3fe7e --- /dev/null +++ b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/reservedkeywords_inputrecords_raw.jsonl @@ -0,0 +1,2 @@ +// A column name that is a reserved keyword should be added for each typing & deduping destination. +{"_airbyte_raw_id": "b2e0efc4-38a8-47ba-970c-8103f09f08d5", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"current_date": "foo", "join": "bar"}} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/timestampformats_inputrecords.jsonl b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/timestampformats_inputrecords.jsonl similarity index 100% rename from airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/timestampformats_inputrecords.jsonl rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/timestampformats_inputrecords.jsonl diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/weirdcolumnnames_inputrecords_raw.jsonl b/airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/weirdcolumnnames_inputrecords_raw.jsonl similarity index 100% rename from airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/weirdcolumnnames_inputrecords_raw.jsonl rename to airbyte-cdk/java/airbyte-cdk/typing-deduping/src/testFixtures/resources/sqlgenerator/weirdcolumnnames_inputrecords_raw.jsonl diff --git a/airbyte-cdk/python/.bumpversion.cfg b/airbyte-cdk/python/.bumpversion.cfg index 4d6fec045246..641822eeb211 100644 --- a/airbyte-cdk/python/.bumpversion.cfg +++ b/airbyte-cdk/python/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.51.6 +current_version = 0.59.0 commit = False [bumpversion:file:setup.py] diff --git a/airbyte-cdk/python/.gitignore b/airbyte-cdk/python/.gitignore index a65b291546c6..9847b9678308 100644 --- a/airbyte-cdk/python/.gitignore +++ b/airbyte-cdk/python/.gitignore @@ -1,6 +1,4 @@ .coverage # TODO: these are tmp files generated by unit tests. They should go to the /tmp directory. -cache_http_stream*.yml ` + +## 0.56.1 +no-op to verify pypi publish flow + +## 0.56.0 +Allow for connectors to continue syncing when a stream fails + +## 0.55.5 +File-based CDK: hide source-defined primary key; users can define primary keys in the connection's configuration + +## 0.55.4 +Source Integration tests: decoupling entrypoint wrapper from pytest + +## 0.55.3 +First iteration of integration tests tooling (http mocker and response builder) + +## 0.55.2 +concurrent-cdk: factory method initializes concurrent source with default number of max tasks + +## 0.55.1 +Vector DB CDK: Add omit_raw_text flag + +## 0.55.0 +concurrent cdk: read multiple streams concurrently + +## 0.54.0 +low-code: fix injection of page token if first request + +## 0.53.9 +Fix of generate the error message using _try_get_error based on list of errors + +## 0.53.8 +Vector DB CDK: Remove CDC records, File CDK: Update unstructured parser + +## 0.53.7 +low-code: fix debug logging when using --debug flag + +## 0.53.6 +Increase maximum_attempts_to_acquire to avoid crashing in acquire_call + +## 0.53.5 +File CDK: Improve stream config appearance + +## 0.53.4 +Concurrent CDK: fix futures pruning + +## 0.53.3 +Fix spec schema generation for File CDK and Vector DB CDK and allow skipping invalid files in document file parser + +## 0.53.2 +Concurrent CDK: Increase connection pool size to allow for 20 max workers + +## 0.53.1 +Concurrent CDK: Improve handling of future to avoid memory leak and improve performances + +## 0.53.0 +Add call rate functionality + +## 0.52.10 +Fix class SessionTokenAuthenticator for CLASS_TYPES_REGISTRY mapper + +## 0.52.9 +File CDK: Improve file type detection in document file type parser + +## 0.52.8 +Concurrent CDK: incremental (missing state conversion). Outside of concurrent specific work, this includes the following changes: +* Checkpointing state was acting on the number of records per slice. This has been changed to consider the number of records per syncs +* `Source.read_state` and `Source._emit_legacy_state_format` are now classmethods to allow for developers to have access to the state before instantiating the source + +## 0.52.7 +File CDK: Add pptx support + +## 0.52.6 +make parameter as not required for default backoff handler + +## 0.52.5 +use in-memory cache if no file path is provided + +## 0.52.4 +File CDK: Add unstructured parser + +## 0.52.3 +Update source-declarative-manifest base image to update Linux alpine and Python + +## 0.52.2 + + +## 0.52.1 +Add max time for backoff handler + +## 0.52.0 +File CDK: Add CustomFileBasedException for custom errors + +## 0.51.44 +low-code: Allow connector developers to specify the type of an added field + +## 0.51.43 +concurrent cdk: fail fast if a partition raises an exception + +## 0.51.42 +File CDK: Avoid listing all files for check command + +## 0.51.41 +Vector DB CDK: Expose stream identifier logic, add field remapping to processing | File CDK: Emit analytics message for used streams + +## 0.51.40 +Add filters for base64 encode and decode in Jinja Interpolation + +## 0.51.39 +Few bug fixes for concurrent cdk + +## 0.51.38 +Add ability to wrap HTTP errors with specific status codes occurred during access token refresh into AirbyteTracedException + +## 0.51.37 +Enable debug logging when running availability check + +## 0.51.36 +Enable debug logging when running availability check + +## 0.51.35 +File CDK: Allow configuring number of tested files for schema inference and parsability check + +## 0.51.34 +Vector DB CDK: Fix OpenAI compatible embedder when used without api key + +## 0.51.33 +Vector DB CDK: Improve batching process + +## 0.51.32 +Introduce experimental ThreadBasedConcurrentStream + +## 0.51.31 +Fix initialize of token_expiry_is_time_of_expiration field + +## 0.51.30 +Add new token_expiry_is_time_of_expiration property for AbstractOauth2Authenticator for indicate that token's expiry_in is a time of expiration + +## 0.51.29 +Coerce read_records to iterable in http availabilty strategy + +## 0.51.28 +Add functionality enabling Page Number/Offset to be set on the first request + +## 0.51.27 +Fix parsing of UUID fields in avro files + +## 0.51.26 +Vector DB CDK: Fix OpenAI embedder batch size + +## 0.51.25 +Add configurable OpenAI embedder to cdk and add cloud environment helper + +## 0.51.24 +Fix previous version of request_cache clearing + +## 0.51.23 +Fix request_cache clearing and move it to tmp folder + +## 0.51.22 +Vector DB CDK: Adjust batch size for Azure embedder to current limits + +## 0.51.21 +Change Error message if Stream is not found + +## 0.51.20 +Vector DB CDK: Add text splitting options to document processing + +## 0.51.19 +Ensuring invalid user-provided urls does not generate sentry issues + +## 0.51.18 +Vector DB CDK adjustments: Prevent failures with big records and OpenAI embedder + +## 0.51.17 +[ISSUE #30353] File-Based CDK: remove file_type from stream config + +## 0.51.16 +Connector Builder: fix datetime format inference for str parsable as int but not isdecimal + +## 0.51.15 +Vector DB CDK: Add Azure OpenAI embedder + +## 0.51.14 +File-based CDK: improve error message for CSV parsing error + +## 0.51.13 +File-based CDK: migrated parsing error to config error to avoid sentry alerts + +## 0.51.12 +Add from-field embedder to vector db CDK + +## 0.51.11 +FIle-based CDK: Update spec and fix autogenerated headers with skip after + +## 0.51.10 +Vector DB CDK adjustments: Fix id generation, improve config spec, add base test case + +## 0.51.9 +[Issue #29660] Support empty keys with record selection + +## 0.51.8 +Add vector db CDK helpers + +## 0.51.7 +File-based CDK: allow user to provided column names for CSV files + ## 0.51.6 File-based CDK: allow for extension mismatch diff --git a/airbyte-cdk/python/Dockerfile b/airbyte-cdk/python/Dockerfile index db173bb548c5..f095029dbace 100644 --- a/airbyte-cdk/python/Dockerfile +++ b/airbyte-cdk/python/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.11-alpine3.15 as base +FROM python:3.9.18-alpine3.18 as base # build and load all requirements FROM base as builder @@ -10,7 +10,7 @@ RUN apk --no-cache upgrade \ && apk --no-cache add tzdata build-base # install airbyte-cdk -RUN pip install --prefix=/install airbyte-cdk==0.51.6 +RUN pip install --prefix=/install airbyte-cdk==0.58.5 # build a clean environment FROM base @@ -32,5 +32,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] # needs to be the same as CDK -LABEL io.airbyte.version=0.51.6 +LABEL io.airbyte.version=0.59.0 LABEL io.airbyte.name=airbyte/source-declarative-manifest diff --git a/airbyte-cdk/python/README.md b/airbyte-cdk/python/README.md index 1efd6f48104a..fd42dd0f9ff7 100644 --- a/airbyte-cdk/python/README.md +++ b/airbyte-cdk/python/README.md @@ -1,10 +1,9 @@ # Connector Development Kit \(Python\) -The Airbyte Python CDK is a framework for rapidly developing production-grade Airbyte connectors. The CDK currently offers helpers specific for creating Airbyte source connectors for: +The Airbyte Python CDK is a framework for rapidly developing production-grade Airbyte connectors.The CDK currently offers helpers specific for creating Airbyte source connectors for: -* HTTP APIs \(REST APIs, GraphQL, etc..\) -* Singer Taps -* Generic Python sources \(anything not covered by the above\) +- HTTP APIs \(REST APIs, GraphQL, etc..\) +- Generic Python sources \(anything not covered by the above\) The CDK provides an improved developer experience by providing basic implementation structure and abstracting away low-level glue boilerplate. @@ -14,14 +13,14 @@ This document is a general introduction to the CDK. Readers should have basic fa Generate an empty connector using the code generator. First clone the Airbyte repository then from the repository root run -```text +```bash cd airbyte-integrations/connector-templates/generator ./generate.sh ``` then follow the interactive prompt. Next, find all `TODO`s in the generated project directory -- they're accompanied by lots of comments explaining what you'll need to do in order to implement your connector. Upon completing all TODOs properly, you should have a functioning connector. -Additionally, you can follow [this tutorial](https://docs.airbyte.io/connector-development/tutorials/cdk-tutorial-python-http) for a complete walkthrough of creating an HTTP connector using the Airbyte CDK. +Additionally, you can follow [this tutorial](https://docs.airbyte.com/connector-development/cdk-python/) for a complete walkthrough of creating an HTTP connector using the Airbyte CDK. ### Concepts & Documentation @@ -31,29 +30,23 @@ See the [concepts docs](docs/concepts/) for a tour through what the API offers. **HTTP Connectors**: -* [Exchangerates API](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/source.py) -* [Stripe](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py) -* [Slack](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-slack/source_slack/source.py) - -**Singer connectors**: - -* [Salesforce](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-salesforce-singer/source_salesforce_singer/source.py) -* [Github](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-github-singer/source_github_singer/source.py) +- [Stripe](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py) +- [Slack](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-slack/source_slack/source.py) **Simple Python connectors using the barebones `Source` abstraction**: -* [Google Sheets](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/google_sheets_source.py) -* [Mailchimp](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py) +- [Google Sheets](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/google_sheets_source.py) +- [Mailchimp](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py) ## Contributing ### First time setup -We assume `python` points to python >=3.8. +We assume `python` points to Python 3.9 or higher. Setup a virtual env: -```text +```bash python -m venv .venv source .venv/bin/activate pip install -e ".[dev]" # [dev] installs development-only dependencies @@ -61,17 +54,20 @@ pip install -e ".[dev]" # [dev] installs development-only dependencies #### Iteration -* Iterate on the code locally -* Run tests via `python -m pytest -s unit_tests` -* Perform static type checks using `mypy airbyte_cdk`. `MyPy` configuration is in `mypy.ini`. - * Run `mypy ` to only check specific files. This is useful as the CDK still contains code that is not compliant. -* The `type_check_and_test.sh` script bundles both type checking and testing in one convenient command. Feel free to use it! +- Iterate on the code locally +- Run tests via `python -m pytest -s unit_tests` +- Perform static type checks using `mypy airbyte_cdk`. `MyPy` configuration is in `mypy.ini`. +- Run `mypy ` to only check specific files. This is useful as the CDK still contains code that is not compliant. +- The `type_check_and_test.sh` script bundles both type checking and testing in one convenient command. Feel free to use it! ##### Autogenerated files + If the iteration you are working on includes changes to the models, you might want to regenerate them. In order to do that, you can run: -```commandline -SUB_BUILD=CDK ./gradlew format + +```bash +./gradlew :airbyte-cdk:python:format ``` + This will generate the files based on the schemas, add the license information and format the code. If you want to only do the former and rely on pre-commit to the others, you can run the appropriate generation command i.e. `./gradlew generateComponentManifestClassFiles`. @@ -82,14 +78,16 @@ All tests are located in the `unit_tests` directory. Run `python -m pytest --cov #### Building and testing a connector with your local CDK When developing a new feature in the CDK, you may find it helpful to run a connector that uses that new feature. You can test this in one of two ways: -* Running a connector locally -* Building and running a source via Docker + +- Running a connector locally +- Building and running a source via Docker ##### Installing your local CDK into a local Python connector In order to get a local Python connector running your local CDK, do the following. First, make sure you have your connector's virtual environment active: + ```bash # from the `airbyte/airbyte-integrations/connectors/` directory source .venv/bin/activate @@ -99,6 +97,7 @@ pip install -e . ``` Then, navigate to the CDK and install it in editable mode: + ```bash cd ../../../airbyte-cdk/python pip install -e . @@ -108,37 +107,54 @@ You should see that `pip` has uninstalled the version of `airbyte-cdk` defined b ##### Building a Python connector in Docker with your local CDK installed +_Pre-requisite: Install the [`airbyte-ci` CLI](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md)_ + You can build your connector image with the local CDK using + ```bash # from the airbytehq/airbyte base directory -CONNECTOR_TAG= CONNECTOR_NAME= sh airbyte-integrations/scripts/build-connector-image-with-local-cdk.sh +airbyte-ci connectors --use-local-cdk --name= build ``` + Note that the local CDK is injected at build time, so if you make changes, you will have to run the build command again to see them reflected. ##### Running Connector Acceptance Tests for a single connector in Docker with your local CDK installed +_Pre-requisite: Install the [`airbyte-ci` CLI](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md)_ + To run acceptance tests for a single connectors using the local CDK, from the connector directory, run + ```bash -LOCAL_CDK=1 sh acceptance-test-docker.sh +airbyte-ci connectors --use-local-cdk --name= test +``` + +#### When you don't have access to the API + +There can be some time where you do not have access to the API (either because you don't have the credentials, network access, etc...) You will probably still want to do end-to-end testing at least once. In order to do so, you can emulate the server you would be reaching using a server stubbing tool. + +For example, using [mockserver](https://www.mock-server.com/), you can set up an expectation file like this: + +```json +{ + "httpRequest": { + "method": "GET", + "path": "/data" + }, + "httpResponse": { + "body": "{\"data\": [{\"record_key\": 1}, {\"record_key\": 2}]}" + } +} ``` -To additionally fetch secrets required by CATs, set the `FETCH_SECRETS` environment variable. This requires you to have a Google Service Account, and the GCP_GSM_CREDENTIALS environment variable to be set, per the instructions [here](https://github.com/airbytehq/airbyte/tree/master/airbyte-ci/connectors/ci_credentials). -##### Running Connector Acceptance Tests for multiple connectors in Docker with your local CDK installed +Assuming this file has been created at `secrets/mock_server_config/expectations.json`, running the following command will allow to match any requests on path `/data` to return the response defined in the expectation file: -To run acceptance tests for multiple connectors using the local CDK, from the root of the `airbyte` repo, run ```bash -./airbyte-cdk/python/bin/run-cats-with-local-cdk.sh -c ,,... +docker run -d --rm -v $(pwd)/secrets/mock_server_config:/config -p 8113:8113 --env MOCKSERVER_LOG_LEVEL=TRACE --env MOCKSERVER_SERVER_PORT=8113 --env MOCKSERVER_WATCH_INITIALIZATION_JSON=true --env MOCKSERVER_PERSISTED_EXPECTATIONS_PATH=/config/expectations.json --env MOCKSERVER_INITIALIZATION_JSON_PATH=/config/expectations.json mockserver/mockserver:5.15.0 ``` +HTTP requests to `localhost:8113/data` should now return the body defined in the expectations file. To test this, the implementer either has to change the code which defines the base URL for Python source or update the `url_base` from low-code. With the Connector Builder running in docker, you will have to use domain `host.docker.internal` instead of `localhost` as the requests are executed within docker. + #### Publishing a new version to PyPi 1. Open a PR 2. Once it is approved and **merged**, an Airbyte member must run the `Publish CDK Manually` workflow from master using `release-type=major|manor|patch` and setting the changelog message. - -## Coming Soon - -* Full OAuth 2.0 support \(including refresh token issuing flow via UI or CLI\) -* Airbyte Java HTTP CDK -* CDK for Async HTTP endpoints \(request-poll-wait style endpoints\) -* CDK for other protocols -* Don't see a feature you need? [Create an issue and let us know how we can help!](https://github.com/airbytehq/airbyte/issues/new?assignees=&labels=type%2Fenhancement&template=feature-request.md&title=) diff --git a/airbyte-cdk/python/airbyte_cdk/connector_builder/main.py b/airbyte-cdk/python/airbyte_cdk/connector_builder/main.py index c26e5292d7f9..02ba043e937f 100644 --- a/airbyte-cdk/python/airbyte_cdk/connector_builder/main.py +++ b/airbyte-cdk/python/airbyte_cdk/connector_builder/main.py @@ -15,6 +15,8 @@ def get_config_and_catalog_from_args(args: List[str]) -> Tuple[str, Mapping[str, Any], Optional[ConfiguredAirbyteCatalog]]: + # TODO: Add functionality for the `debug` logger. + # Currently, no one `debug` level log will be displayed during `read` a stream for a connector created through `connector-builder`. parsed_args = AirbyteEntrypoint.parse_args(args) config_path, catalog_path = parsed_args.config, parsed_args.catalog if parsed_args.command != "read": @@ -42,7 +44,11 @@ def get_config_and_catalog_from_args(args: List[str]) -> Tuple[str, Mapping[str, def handle_connector_builder_request( - source: ManifestDeclarativeSource, command: str, config: Mapping[str, Any], catalog: Optional[ConfiguredAirbyteCatalog], limits: TestReadLimits + source: ManifestDeclarativeSource, + command: str, + config: Mapping[str, Any], + catalog: Optional[ConfiguredAirbyteCatalog], + limits: TestReadLimits, ) -> AirbyteMessage: if command == "resolve_manifest": return resolve_manifest(source) diff --git a/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py b/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py index b787fe5d43c9..42a0e1051b52 100644 --- a/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py +++ b/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py @@ -19,8 +19,8 @@ StreamReadSlices, ) from airbyte_cdk.entrypoint import AirbyteEntrypoint -from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource +from airbyte_cdk.sources.utils.slice_logger import SliceLogger from airbyte_cdk.sources.utils.types import JsonType from airbyte_cdk.utils import AirbyteTracedException from airbyte_cdk.utils.datetime_format_inferrer import DatetimeFormatInferrer @@ -67,10 +67,10 @@ def get_message_groups( latest_config_update: AirbyteControlMessage = None auxiliary_requests = [] for message_group in self._get_message_groups( - self._read_stream(source, config, configured_catalog), - schema_inferrer, - datetime_format_inferrer, - record_limit, + self._read_stream(source, config, configured_catalog), + schema_inferrer, + datetime_format_inferrer, + record_limit, ): if isinstance(message_group, AirbyteLogMessage): log_messages.append(LogMessage(**{"message": message_group.message, "level": message_group.level.value})) @@ -101,7 +101,11 @@ def get_message_groups( ) def _get_message_groups( - self, messages: Iterator[AirbyteMessage], schema_inferrer: SchemaInferrer, datetime_format_inferrer: DatetimeFormatInferrer, limit: int + self, + messages: Iterator[AirbyteMessage], + schema_inferrer: SchemaInferrer, + datetime_format_inferrer: DatetimeFormatInferrer, + limit: int, ) -> Iterable[Union[StreamReadPages, AirbyteControlMessage, AirbyteLogMessage, AirbyteTraceMessage, AuxiliaryRequest]]: """ Message groups are partitioned according to when request log messages are received. Subsequent response log messages @@ -136,12 +140,16 @@ def _get_message_groups( current_page_request = None current_page_response = None - if at_least_one_page_in_group and message.type == MessageType.LOG and message.log.message.startswith(AbstractSource.SLICE_LOG_PREFIX): + if ( + at_least_one_page_in_group + and message.type == MessageType.LOG + and message.log.message.startswith(SliceLogger.SLICE_LOG_PREFIX) + ): yield StreamReadSlices(pages=current_slice_pages, slice_descriptor=current_slice_descriptor) current_slice_descriptor = self._parse_slice_description(message.log.message) current_slice_pages = [] at_least_one_page_in_group = False - elif message.type == MessageType.LOG and message.log.message.startswith(AbstractSource.SLICE_LOG_PREFIX): + elif message.type == MessageType.LOG and message.log.message.startswith(SliceLogger.SLICE_LOG_PREFIX): # parsing the first slice current_slice_descriptor = self._parse_slice_description(message.log.message) elif message.type == MessageType.LOG: @@ -153,9 +161,7 @@ def _get_message_groups( stream = airbyte_cdk.get("stream", {}) if not isinstance(stream, dict): raise ValueError(f"Expected stream to be a dict, got {stream} of type {type(stream)}") - title_prefix = ( - "Parent stream: " if stream.get("is_substream", False) else "" - ) + title_prefix = "Parent stream: " if stream.get("is_substream", False) else "" http = json_message.get("http", {}) if not isinstance(http, dict): raise ValueError(f"Expected http to be a dict, got {http} of type {type(http)}") @@ -220,7 +226,12 @@ def _is_auxiliary_http_request(message: Optional[Dict[str, Any]]) -> bool: return is_http and message.get("http", {}).get("is_auxiliary", False) @staticmethod - def _close_page(current_page_request: Optional[HttpRequest], current_page_response: Optional[HttpResponse], current_slice_pages: List[StreamReadPages], current_page_records: List[Mapping[str, Any]]) -> None: + def _close_page( + current_page_request: Optional[HttpRequest], + current_page_response: Optional[HttpResponse], + current_slice_pages: List[StreamReadPages], + current_page_records: List[Mapping[str, Any]], + ) -> None: """ Close a page when parsing message groups """ @@ -229,7 +240,9 @@ def _close_page(current_page_request: Optional[HttpRequest], current_page_respon ) current_page_records.clear() - def _read_stream(self, source: DeclarativeSource, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog) -> Iterator[AirbyteMessage]: + def _read_stream( + self, source: DeclarativeSource, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog + ) -> Iterator[AirbyteMessage]: # the generator can raise an exception # iterate over the generated messages. if next raise an exception, catch it and yield it as an AirbyteLogMessage try: @@ -285,7 +298,7 @@ def _has_reached_limit(self, slices: List[StreamReadSlices]) -> bool: return False def _parse_slice_description(self, log_message: str) -> Dict[str, Any]: - return json.loads(log_message.replace(AbstractSource.SLICE_LOG_PREFIX, "", 1)) # type: ignore + return json.loads(log_message.replace(SliceLogger.SLICE_LOG_PREFIX, "", 1)) # type: ignore @staticmethod def _clean_config(config: Dict[str, Any]) -> Dict[str, Any]: diff --git a/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/README.md b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/README.md new file mode 100644 index 000000000000..b07b42e9457c --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/README.md @@ -0,0 +1,37 @@ +# Vector DB based destinations + +## Note: All helpers in this directory are experimental and subject to change + +This directory contains several helpers that can be used to create a destination that processes and chunks records, embeds their text part and loads them into a vector database. +The specific loading behavior is defined by the destination connector itself, but chunking and embedding behavior is handled by the helpers. + +To use these helpers, install the CDK with the `vector-db-based` extra: + +```bash +pip install airbyte-cdk[vector-db-based] +``` + + +The helpers can be used in the following way: +* Add the config models to the spec of the connector +* Implement the `Indexer` interface for your specific database +* In the check implementation of the destination, initialize the indexer and the embedder and call `check` on them +* In the write implementation of the destination, initialize the indexer, the embedder and pass them to a new instance of the writer. Then call the writers `write` method with the iterable for incoming messages + +If there are no connector-specific embedders, the `airbyte_cdk.destinations.vector_db_based.embedder.create_from_config` function can be used to get an embedder instance from the config. + +This is how the components interact: + +```text +┌─────────────┐ +│MyDestination│ +└┬────────────┘ +┌▽───────────────────────────────┐ +│Writer │ +└┬─────────┬──────────┬──────────┘ +┌▽───────┐┌▽────────┐┌▽────────────────┐ +│Embedder││MyIndexer││DocumentProcessor│ +└────────┘└─────────┘└─────────────────┘ +``` + +Normally, only the `MyDestination` class and the `MyIndexer` class has to be implemented specifically for the destination. The other classes are provided as is by the helpers. \ No newline at end of file diff --git a/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/__init__.py b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/__init__.py new file mode 100644 index 000000000000..86ae207f69d8 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/__init__.py @@ -0,0 +1,38 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from .config import ( + AzureOpenAIEmbeddingConfigModel, + CohereEmbeddingConfigModel, + FakeEmbeddingConfigModel, + FromFieldEmbeddingConfigModel, + OpenAICompatibleEmbeddingConfigModel, + OpenAIEmbeddingConfigModel, + ProcessingConfigModel, +) +from .document_processor import Chunk, DocumentProcessor +from .embedder import CohereEmbedder, Embedder, FakeEmbedder, OpenAIEmbedder +from .indexer import Indexer +from .writer import Writer + +__all__ = [ + "AzureOpenAIEmbedder", + "AzureOpenAIEmbeddingConfigModel", + "Chunk", + "CohereEmbedder", + "CohereEmbeddingConfigModel", + "DocumentProcessor", + "Embedder", + "FakeEmbedder", + "FakeEmbeddingConfigModel", + "FromFieldEmbedder", + "FromFieldEmbeddingConfigModel", + "Indexer", + "OpenAICompatibleEmbedder", + "OpenAICompatibleEmbeddingConfigModel", + "OpenAIEmbedder", + "OpenAIEmbeddingConfigModel", + "ProcessingConfigModel", + "Writer", +] diff --git a/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/config.py b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/config.py new file mode 100644 index 000000000000..0f42e151653a --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/config.py @@ -0,0 +1,275 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Dict, List, Literal, Optional, Union + +import dpath.util +from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig +from airbyte_cdk.utils.spec_schema_transformations import resolve_refs +from pydantic import BaseModel, Field + + +class SeparatorSplitterConfigModel(BaseModel): + mode: Literal["separator"] = Field("separator", const=True) + separators: List[str] = Field( + default=['"\\n\\n"', '"\\n"', '" "', '""'], + title="Separators", + description='List of separator strings to split text fields by. The separator itself needs to be wrapped in double quotes, e.g. to split by the dot character, use ".". To split by a newline, use "\\n".', + ) + keep_separator: bool = Field(default=False, title="Keep separator", description="Whether to keep the separator in the resulting chunks") + + class Config(OneOfOptionConfig): + title = "By Separator" + description = "Split the text by the list of separators until the chunk size is reached, using the earlier mentioned separators where possible. This is useful for splitting text fields by paragraphs, sentences, words, etc." + discriminator = "mode" + + +class MarkdownHeaderSplitterConfigModel(BaseModel): + mode: Literal["markdown"] = Field("markdown", const=True) + split_level: int = Field( + default=1, + title="Split level", + description="Level of markdown headers to split text fields by. Headings down to the specified level will be used as split points", + le=6, + ge=1, + ) + + class Config(OneOfOptionConfig): + title = "By Markdown header" + description = "Split the text by Markdown headers down to the specified header level. If the chunk size fits multiple sections, they will be combined into a single chunk." + discriminator = "mode" + + +class CodeSplitterConfigModel(BaseModel): + mode: Literal["code"] = Field("code", const=True) + language: str = Field( + title="Language", + description="Split code in suitable places based on the programming language", + enum=[ + "cpp", + "go", + "java", + "js", + "php", + "proto", + "python", + "rst", + "ruby", + "rust", + "scala", + "swift", + "markdown", + "latex", + "html", + "sol", + ], + ) + + class Config(OneOfOptionConfig): + title = "By Programming Language" + description = ( + "Split the text by suitable delimiters based on the programming language. This is useful for splitting code into chunks." + ) + discriminator = "mode" + + +TextSplitterConfigModel = Union[SeparatorSplitterConfigModel, MarkdownHeaderSplitterConfigModel, CodeSplitterConfigModel] + + +class FieldNameMappingConfigModel(BaseModel): + from_field: str = Field(title="From field name", description="The field name in the source") + to_field: str = Field(title="To field name", description="The field name to use in the destination") + + +class ProcessingConfigModel(BaseModel): + chunk_size: int = Field( + ..., + title="Chunk size", + maximum=8191, + minimum=1, + description="Size of chunks in tokens to store in vector store (make sure it is not too big for the context if your LLM)", + ) + chunk_overlap: int = Field( + title="Chunk overlap", + description="Size of overlap between chunks in tokens to store in vector store to better capture relevant context", + default=0, + ) + text_fields: Optional[List[str]] = Field( + default=[], + title="Text fields to embed", + description="List of fields in the record that should be used to calculate the embedding. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered text fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.", + always_show=True, + examples=["text", "user.name", "users.*.name"], + ) + metadata_fields: Optional[List[str]] = Field( + default=[], + title="Fields to store as metadata", + description="List of fields in the record that should be stored as metadata. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered metadata fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. When specifying nested paths, all matching values are flattened into an array set to a field named by the path.", + always_show=True, + examples=["age", "user", "user.name"], + ) + text_splitter: TextSplitterConfigModel = Field( + default=None, + title="Text splitter", + discriminator="mode", + type="object", + description="Split text fields into chunks based on the specified method.", + ) + field_name_mappings: Optional[List[FieldNameMappingConfigModel]] = Field( + default=[], + title="Field name mappings", + description="List of fields to rename. Not applicable for nested fields, but can be used to rename fields already flattened via dot notation.", + ) + + class Config: + schema_extra = {"group": "processing"} + + +class OpenAIEmbeddingConfigModel(BaseModel): + mode: Literal["openai"] = Field("openai", const=True) + openai_key: str = Field(..., title="OpenAI API key", airbyte_secret=True) + + class Config(OneOfOptionConfig): + title = "OpenAI" + description = ( + "Use the OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." + ) + discriminator = "mode" + + +class OpenAICompatibleEmbeddingConfigModel(BaseModel): + mode: Literal["openai_compatible"] = Field("openai_compatible", const=True) + api_key: str = Field(title="API key", default="", airbyte_secret=True) + base_url: str = Field( + ..., title="Base URL", description="The base URL for your OpenAI-compatible service", examples=["https://your-service-name.com"] + ) + model_name: str = Field( + title="Model name", + description="The name of the model to use for embedding", + default="text-embedding-ada-002", + examples=["text-embedding-ada-002"], + ) + dimensions: int = Field( + title="Embedding dimensions", description="The number of dimensions the embedding model is generating", examples=[1536, 384] + ) + + class Config(OneOfOptionConfig): + title = "OpenAI-compatible" + description = "Use a service that's compatible with the OpenAI API to embed text." + discriminator = "mode" + + +class AzureOpenAIEmbeddingConfigModel(BaseModel): + mode: Literal["azure_openai"] = Field("azure_openai", const=True) + openai_key: str = Field( + ..., + title="Azure OpenAI API key", + airbyte_secret=True, + description="The API key for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", + ) + api_base: str = Field( + ..., + title="Resource base URL", + description="The base URL for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", + examples=["https://your-resource-name.openai.azure.com"], + ) + deployment: str = Field( + ..., + title="Deployment", + description="The deployment for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", + examples=["your-resource-name"], + ) + + class Config(OneOfOptionConfig): + title = "Azure OpenAI" + description = "Use the Azure-hosted OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." + discriminator = "mode" + + +class FakeEmbeddingConfigModel(BaseModel): + mode: Literal["fake"] = Field("fake", const=True) + + class Config(OneOfOptionConfig): + title = "Fake" + description = "Use a fake embedding made out of random vectors with 1536 embedding dimensions. This is useful for testing the data pipeline without incurring any costs." + discriminator = "mode" + + +class FromFieldEmbeddingConfigModel(BaseModel): + mode: Literal["from_field"] = Field("from_field", const=True) + field_name: str = Field( + ..., title="Field name", description="Name of the field in the record that contains the embedding", examples=["embedding", "vector"] + ) + dimensions: int = Field( + ..., title="Embedding dimensions", description="The number of dimensions the embedding model is generating", examples=[1536, 384] + ) + + class Config(OneOfOptionConfig): + title = "From Field" + description = "Use a field in the record as the embedding. This is useful if you already have an embedding for your data and want to store it in the vector store." + discriminator = "mode" + + +class CohereEmbeddingConfigModel(BaseModel): + mode: Literal["cohere"] = Field("cohere", const=True) + cohere_key: str = Field(..., title="Cohere API key", airbyte_secret=True) + + class Config(OneOfOptionConfig): + title = "Cohere" + description = "Use the Cohere API to embed text." + discriminator = "mode" + + +class VectorDBConfigModel(BaseModel): + """ + The configuration model for the Vector DB based destinations. This model is used to generate the UI for the destination configuration, + as well as to provide type safety for the configuration passed to the destination. + + The configuration model is composed of four parts: + * Processing configuration + * Embedding configuration + * Indexing configuration + * Advanced configuration + + Processing, embedding and advanced configuration are provided by this base class, while the indexing configuration is provided by the destination connector in the sub class. + """ + + embedding: Union[ + OpenAIEmbeddingConfigModel, + CohereEmbeddingConfigModel, + FakeEmbeddingConfigModel, + AzureOpenAIEmbeddingConfigModel, + OpenAICompatibleEmbeddingConfigModel, + ] = Field(..., title="Embedding", description="Embedding configuration", discriminator="mode", group="embedding", type="object") + processing: ProcessingConfigModel + omit_raw_text: bool = Field( + default=False, + title="Do not store raw text", + group="advanced", + description="Do not store the text that gets embedded along with the vector and the metadata in the destination. If set to true, only the vector and the metadata will be stored - in this case raw text for LLM use cases needs to be retrieved from another source.", + ) + + class Config: + title = "Destination Config" + schema_extra = { + "groups": [ + {"id": "processing", "title": "Processing"}, + {"id": "embedding", "title": "Embedding"}, + {"id": "indexing", "title": "Indexing"}, + {"id": "advanced", "title": "Advanced"}, + ] + } + + @staticmethod + def remove_discriminator(schema: Dict[str, Any]) -> None: + """pydantic adds "discriminator" to the schema for oneOfs, which is not treated right by the platform as we inline all references""" + dpath.util.delete(schema, "properties/**/discriminator") + + @classmethod + def schema(cls, by_alias: bool = True, ref_template: str = "") -> Dict[str, Any]: + """we're overriding the schema classmethod to enable some post-processing""" + schema: Dict[str, Any] = super().schema() + schema = resolve_refs(schema) + cls.remove_discriminator(schema) + return schema diff --git a/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/document_processor.py b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/document_processor.py new file mode 100644 index 000000000000..06d3c892dd5d --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/document_processor.py @@ -0,0 +1,184 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import logging +from dataclasses import dataclass +from typing import Any, Dict, List, Mapping, Optional, Tuple + +import dpath.util +from airbyte_cdk.destinations.vector_db_based.config import ProcessingConfigModel, SeparatorSplitterConfigModel, TextSplitterConfigModel +from airbyte_cdk.destinations.vector_db_based.utils import create_stream_identifier +from airbyte_cdk.models import AirbyteRecordMessage, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode +from airbyte_cdk.utils.traced_exception import AirbyteTracedException, FailureType +from langchain.document_loaders.base import Document +from langchain.text_splitter import Language, RecursiveCharacterTextSplitter +from langchain.utils import stringify_dict + +METADATA_STREAM_FIELD = "_ab_stream" +METADATA_RECORD_ID_FIELD = "_ab_record_id" + +CDC_DELETED_FIELD = "_ab_cdc_deleted_at" + + +@dataclass +class Chunk: + page_content: Optional[str] + metadata: Dict[str, Any] + record: AirbyteRecordMessage + embedding: Optional[List[float]] = None + + +headers_to_split_on = ["(?:^|\n)# ", "(?:^|\n)## ", "(?:^|\n)### ", "(?:^|\n)#### ", "(?:^|\n)##### ", "(?:^|\n)###### "] + + +class DocumentProcessor: + """ + DocumentProcessor is a helper class that generates documents from Airbyte records. + + It is used to generate documents from records before writing them to the destination: + * The text fields are extracted from the record and concatenated to a single string. + * The metadata fields are extracted from the record and added to the document metadata. + * The document is split into chunks of a given size using a langchain text splitter. + + The Writer class uses the DocumentProcessor class to internally generate documents from records - in most cases you don't need to use it directly, + except if you want to implement a custom writer. + + The config parameters specified by the ProcessingConfigModel has to be made part of the connector spec to allow the user to configure the document processor. + Calling DocumentProcessor.check_config(config) will validate the config and return an error message if the config is invalid. + """ + + streams: Mapping[str, ConfiguredAirbyteStream] + + @staticmethod + def check_config(config: ProcessingConfigModel) -> Optional[str]: + if config.text_splitter is not None and config.text_splitter.mode == "separator": + for s in config.text_splitter.separators: + try: + separator = json.loads(s) + if not isinstance(separator, str): + return f"Invalid separator: {s}. Separator needs to be a valid JSON string using double quotes." + except json.decoder.JSONDecodeError: + return f"Invalid separator: {s}. Separator needs to be a valid JSON string using double quotes." + return None + + def _get_text_splitter( + self, chunk_size: int, chunk_overlap: int, splitter_config: Optional[TextSplitterConfigModel] + ) -> RecursiveCharacterTextSplitter: + if splitter_config is None: + splitter_config = SeparatorSplitterConfigModel(mode="separator") + if splitter_config.mode == "separator": + return RecursiveCharacterTextSplitter.from_tiktoken_encoder( + chunk_size=chunk_size, + chunk_overlap=chunk_overlap, + separators=[json.loads(s) for s in splitter_config.separators], + keep_separator=splitter_config.keep_separator, + disallowed_special=(), + ) + if splitter_config.mode == "markdown": + return RecursiveCharacterTextSplitter.from_tiktoken_encoder( + chunk_size=chunk_size, + chunk_overlap=chunk_overlap, + separators=headers_to_split_on[: splitter_config.split_level], + is_separator_regex=True, + keep_separator=True, + disallowed_special=(), + ) + if splitter_config.mode == "code": + return RecursiveCharacterTextSplitter.from_tiktoken_encoder( + chunk_size=chunk_size, + chunk_overlap=chunk_overlap, + separators=RecursiveCharacterTextSplitter.get_separators_for_language(Language(splitter_config.language)), + disallowed_special=(), + ) + + def __init__(self, config: ProcessingConfigModel, catalog: ConfiguredAirbyteCatalog): + self.streams = {create_stream_identifier(stream.stream): stream for stream in catalog.streams} + + self.splitter = self._get_text_splitter(config.chunk_size, config.chunk_overlap, config.text_splitter) + self.text_fields = config.text_fields + self.metadata_fields = config.metadata_fields + self.field_name_mappings = config.field_name_mappings + self.logger = logging.getLogger("airbyte.document_processor") + + def process(self, record: AirbyteRecordMessage) -> Tuple[List[Chunk], Optional[str]]: + """ + Generate documents from records. + :param records: List of AirbyteRecordMessages + :return: Tuple of (List of document chunks, record id to delete if a stream is in dedup mode to avoid stale documents in the vector store) + """ + if CDC_DELETED_FIELD in record.data and record.data[CDC_DELETED_FIELD]: + return [], self._extract_primary_key(record) + doc = self._generate_document(record) + if doc is None: + text_fields = ", ".join(self.text_fields) if self.text_fields else "all fields" + raise AirbyteTracedException( + internal_message="No text fields found in record", + message=f"Record {str(record.data)[:250]}... does not contain any of the configured text fields: {text_fields}. Please check your processing configuration, there has to be at least one text field set in each record.", + failure_type=FailureType.config_error, + ) + chunks = [ + Chunk(page_content=chunk_document.page_content, metadata=chunk_document.metadata, record=record) + for chunk_document in self._split_document(doc) + ] + id_to_delete = doc.metadata[METADATA_RECORD_ID_FIELD] if METADATA_RECORD_ID_FIELD in doc.metadata else None + return chunks, id_to_delete + + def _generate_document(self, record: AirbyteRecordMessage) -> Optional[Document]: + relevant_fields = self._extract_relevant_fields(record, self.text_fields) + if len(relevant_fields) == 0: + return None + text = stringify_dict(relevant_fields) + metadata = self._extract_metadata(record) + return Document(page_content=text, metadata=metadata) + + def _extract_relevant_fields(self, record: AirbyteRecordMessage, fields: Optional[List[str]]) -> Dict[str, Any]: + relevant_fields = {} + if fields and len(fields) > 0: + for field in fields: + values = dpath.util.values(record.data, field, separator=".") + if values and len(values) > 0: + relevant_fields[field] = values if len(values) > 1 else values[0] + else: + relevant_fields = record.data + return self._remap_field_names(relevant_fields) + + def _extract_metadata(self, record: AirbyteRecordMessage) -> Dict[str, Any]: + metadata = self._extract_relevant_fields(record, self.metadata_fields) + metadata[METADATA_STREAM_FIELD] = create_stream_identifier(record) + primary_key = self._extract_primary_key(record) + if primary_key: + metadata[METADATA_RECORD_ID_FIELD] = primary_key + return metadata + + def _extract_primary_key(self, record: AirbyteRecordMessage) -> Optional[str]: + stream_identifier = create_stream_identifier(record) + current_stream: ConfiguredAirbyteStream = self.streams[stream_identifier] + # if the sync mode is deduping, use the primary key to upsert existing records instead of appending new ones + if not current_stream.primary_key or current_stream.destination_sync_mode != DestinationSyncMode.append_dedup: + return None + + primary_key = [] + for key in current_stream.primary_key: + try: + primary_key.append(str(dpath.util.get(record.data, key))) + except KeyError: + primary_key.append("__not_found__") + stringified_primary_key = "_".join(primary_key) + return f"{stream_identifier}_{stringified_primary_key}" + + def _split_document(self, doc: Document) -> List[Document]: + chunks: List[Document] = self.splitter.split_documents([doc]) + return chunks + + def _remap_field_names(self, fields: Dict[str, Any]) -> Dict[str, Any]: + if not self.field_name_mappings: + return fields + + new_fields = fields.copy() + for mapping in self.field_name_mappings: + if mapping.from_field in new_fields: + new_fields[mapping.to_field] = new_fields.pop(mapping.from_field) + + return new_fields diff --git a/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/embedder.py b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/embedder.py new file mode 100644 index 000000000000..7fb880fadaae --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/embedder.py @@ -0,0 +1,261 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import os +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import List, Optional, Union, cast + +from airbyte_cdk.destinations.vector_db_based.config import ( + AzureOpenAIEmbeddingConfigModel, + CohereEmbeddingConfigModel, + FakeEmbeddingConfigModel, + FromFieldEmbeddingConfigModel, + OpenAICompatibleEmbeddingConfigModel, + OpenAIEmbeddingConfigModel, + ProcessingConfigModel, +) +from airbyte_cdk.destinations.vector_db_based.utils import create_chunks, format_exception +from airbyte_cdk.models import AirbyteRecordMessage +from airbyte_cdk.utils.traced_exception import AirbyteTracedException, FailureType +from langchain.embeddings.cohere import CohereEmbeddings +from langchain.embeddings.fake import FakeEmbeddings +from langchain.embeddings.localai import LocalAIEmbeddings +from langchain.embeddings.openai import OpenAIEmbeddings + + +@dataclass +class Document: + page_content: str + record: AirbyteRecordMessage + + +class Embedder(ABC): + """ + Embedder is an abstract class that defines the interface for embedding text. + + The Indexer class uses the Embedder class to internally embed text - each indexer is responsible to pass the text of all documents to the embedder and store the resulting embeddings in the destination. + The destination connector is responsible to create an embedder instance and pass it to the writer. + The CDK defines basic embedders that should be supported in each destination. It is possible to implement custom embedders for special destinations if needed. + """ + + def __init__(self) -> None: + pass + + @abstractmethod + def check(self) -> Optional[str]: + pass + + @abstractmethod + def embed_documents(self, documents: List[Document]) -> List[Optional[List[float]]]: + """ + Embed the text of each chunk and return the resulting embedding vectors. + If a chunk cannot be embedded or is configured to not be embedded, return None for that chunk. + """ + pass + + @property + @abstractmethod + def embedding_dimensions(self) -> int: + pass + + +OPEN_AI_VECTOR_SIZE = 1536 + +OPEN_AI_TOKEN_LIMIT = 150_000 # limit of tokens per minute + + +class BaseOpenAIEmbedder(Embedder): + def __init__(self, embeddings: OpenAIEmbeddings, chunk_size: int): + super().__init__() + self.embeddings = embeddings + self.chunk_size = chunk_size + + def check(self) -> Optional[str]: + try: + self.embeddings.embed_query("test") + except Exception as e: + return format_exception(e) + return None + + def embed_documents(self, documents: List[Document]) -> List[Optional[List[float]]]: + """ + Embed the text of each chunk and return the resulting embedding vectors. + + As the OpenAI API will fail if more than the per-minute limit worth of tokens is sent at once, we split the request into batches and embed each batch separately. + It's still possible to run into the rate limit between each embed call because the available token budget hasn't recovered between the calls, + but the built-in retry mechanism of the OpenAI client handles that. + """ + # Each chunk can hold at most self.chunk_size tokens, so tokens-per-minute by maximum tokens per chunk is the number of documents that can be embedded at once without exhausting the limit in a single request + embedding_batch_size = OPEN_AI_TOKEN_LIMIT // self.chunk_size + batches = create_chunks(documents, batch_size=embedding_batch_size) + embeddings: List[Optional[List[float]]] = [] + for batch in batches: + embeddings.extend(self.embeddings.embed_documents([chunk.page_content for chunk in batch])) + return embeddings + + @property + def embedding_dimensions(self) -> int: + # vector size produced by text-embedding-ada-002 model + return OPEN_AI_VECTOR_SIZE + + +class OpenAIEmbedder(BaseOpenAIEmbedder): + def __init__(self, config: OpenAIEmbeddingConfigModel, chunk_size: int): + super().__init__(OpenAIEmbeddings(openai_api_key=config.openai_key, max_retries=15, disallowed_special=()), chunk_size) # type: ignore + + +class AzureOpenAIEmbedder(BaseOpenAIEmbedder): + def __init__(self, config: AzureOpenAIEmbeddingConfigModel, chunk_size: int): + # Azure OpenAI API has — as of 20230927 — a limit of 16 documents per request + super().__init__(OpenAIEmbeddings(openai_api_key=config.openai_key, chunk_size=16, max_retries=15, openai_api_type="azure", openai_api_version="2023-05-15", openai_api_base=config.api_base, deployment=config.deployment, disallowed_special=()), chunk_size) # type: ignore + + +COHERE_VECTOR_SIZE = 1024 + + +class CohereEmbedder(Embedder): + def __init__(self, config: CohereEmbeddingConfigModel): + super().__init__() + # Client is set internally + self.embeddings = CohereEmbeddings(cohere_api_key=config.cohere_key, model="embed-english-light-v2.0") # type: ignore + + def check(self) -> Optional[str]: + try: + self.embeddings.embed_query("test") + except Exception as e: + return format_exception(e) + return None + + def embed_documents(self, documents: List[Document]) -> List[Optional[List[float]]]: + return cast(List[Optional[List[float]]], self.embeddings.embed_documents([document.page_content for document in documents])) + + @property + def embedding_dimensions(self) -> int: + # vector size produced by text-embedding-ada-002 model + return COHERE_VECTOR_SIZE + + +class FakeEmbedder(Embedder): + def __init__(self, config: FakeEmbeddingConfigModel): + super().__init__() + self.embeddings = FakeEmbeddings(size=OPEN_AI_VECTOR_SIZE) + + def check(self) -> Optional[str]: + try: + self.embeddings.embed_query("test") + except Exception as e: + return format_exception(e) + return None + + def embed_documents(self, documents: List[Document]) -> List[Optional[List[float]]]: + return cast(List[Optional[List[float]]], self.embeddings.embed_documents([document.page_content for document in documents])) + + @property + def embedding_dimensions(self) -> int: + # use same vector size as for OpenAI embeddings to keep it realistic + return OPEN_AI_VECTOR_SIZE + + +CLOUD_DEPLOYMENT_MODE = "cloud" + + +class OpenAICompatibleEmbedder(Embedder): + def __init__(self, config: OpenAICompatibleEmbeddingConfigModel): + super().__init__() + self.config = config + # Client is set internally + # Always set an API key even if there is none defined in the config because the validator will fail otherwise. Embedding APIs that don't require an API key don't fail if one is provided, so this is not breaking usage. + self.embeddings = LocalAIEmbeddings(model=config.model_name, openai_api_key=config.api_key or "dummy-api-key", openai_api_base=config.base_url, max_retries=15, disallowed_special=()) # type: ignore + + def check(self) -> Optional[str]: + deployment_mode = os.environ.get("DEPLOYMENT_MODE", "") + if deployment_mode.casefold() == CLOUD_DEPLOYMENT_MODE and not self.config.base_url.startswith("https://"): + return "Base URL must start with https://" + + try: + self.embeddings.embed_query("test") + except Exception as e: + return format_exception(e) + return None + + def embed_documents(self, documents: List[Document]) -> List[Optional[List[float]]]: + return cast(List[Optional[List[float]]], self.embeddings.embed_documents([document.page_content for document in documents])) + + @property + def embedding_dimensions(self) -> int: + # vector size produced by the model + return self.config.dimensions + + +class FromFieldEmbedder(Embedder): + def __init__(self, config: FromFieldEmbeddingConfigModel): + super().__init__() + self.config = config + + def check(self) -> Optional[str]: + return None + + def embed_documents(self, documents: List[Document]) -> List[Optional[List[float]]]: + """ + From each chunk, pull the embedding from the field specified in the config. + Check that the field exists, is a list of numbers and is the correct size. If not, raise an AirbyteTracedException explaining the problem. + """ + embeddings: List[Optional[List[float]]] = [] + for document in documents: + data = document.record.data + if self.config.field_name not in data: + raise AirbyteTracedException( + internal_message="Embedding vector field not found", + failure_type=FailureType.config_error, + message=f"Record {str(data)[:250]}... in stream {document.record.stream} does not contain embedding vector field {self.config.field_name}. Please check your embedding configuration, the embedding vector field has to be set correctly on every record.", + ) + field = data[self.config.field_name] + if not isinstance(field, list) or not all(isinstance(x, (int, float)) for x in field): + raise AirbyteTracedException( + internal_message="Embedding vector field not a list of numbers", + failure_type=FailureType.config_error, + message=f"Record {str(data)[:250]}... in stream {document.record.stream} does contain embedding vector field {self.config.field_name}, but it is not a list of numbers. Please check your embedding configuration, the embedding vector field has to be a list of numbers of length {self.config.dimensions} on every record.", + ) + if len(field) != self.config.dimensions: + raise AirbyteTracedException( + internal_message="Embedding vector field has wrong length", + failure_type=FailureType.config_error, + message=f"Record {str(data)[:250]}... in stream {document.record.stream} does contain embedding vector field {self.config.field_name}, but it has length {len(field)} instead of the configured {self.config.dimensions}. Please check your embedding configuration, the embedding vector field has to be a list of numbers of length {self.config.dimensions} on every record.", + ) + embeddings.append(field) + + return embeddings + + @property + def embedding_dimensions(self) -> int: + return self.config.dimensions + + +embedder_map = { + "openai": OpenAIEmbedder, + "cohere": CohereEmbedder, + "fake": FakeEmbedder, + "azure_openai": AzureOpenAIEmbedder, + "from_field": FromFieldEmbedder, + "openai_compatible": OpenAICompatibleEmbedder, +} + + +def create_from_config( + embedding_config: Union[ + AzureOpenAIEmbeddingConfigModel, + CohereEmbeddingConfigModel, + FakeEmbeddingConfigModel, + FromFieldEmbeddingConfigModel, + OpenAIEmbeddingConfigModel, + OpenAICompatibleEmbeddingConfigModel, + ], + processing_config: ProcessingConfigModel, +) -> Embedder: + + if embedding_config.mode == "azure_openai" or embedding_config.mode == "openai": + return cast(Embedder, embedder_map[embedding_config.mode](embedding_config, processing_config.chunk_size)) + else: + return cast(Embedder, embedder_map[embedding_config.mode](embedding_config)) diff --git a/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/indexer.py b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/indexer.py new file mode 100644 index 000000000000..c49f576a6709 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/indexer.py @@ -0,0 +1,78 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import itertools +from abc import ABC, abstractmethod +from typing import Any, Generator, Iterable, List, Optional, Tuple, TypeVar + +from airbyte_cdk.destinations.vector_db_based.document_processor import Chunk +from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog + + +class Indexer(ABC): + """ + Indexer is an abstract class that defines the interface for indexing documents. + + The Writer class uses the Indexer class to internally index documents generated by the document processor. + In a destination connector, implement a custom indexer by extending this class and implementing the abstract methods. + """ + + def __init__(self, config: Any): + self.config = config + pass + + def pre_sync(self, catalog: ConfiguredAirbyteCatalog) -> None: + """ + Run before the sync starts. This method should be used to make sure all records in the destination that belong to streams with a destination mode of overwrite are deleted. + + Each record has a metadata field with the name airbyte_cdk.destinations.vector_db_based.document_processor.METADATA_STREAM_FIELD which can be used to filter documents for deletion. + Use the airbyte_cdk.destinations.vector_db_based.utils.create_stream_identifier method to create the stream identifier based on the stream definition to use for filtering. + """ + pass + + def post_sync(self) -> List[AirbyteMessage]: + """ + Run after the sync finishes. This method should be used to perform any cleanup operations and can return a list of AirbyteMessages to be logged. + """ + return [] + + @abstractmethod + def index(self, document_chunks: List[Chunk], namespace: str, stream: str) -> None: + """ + Index a list of document chunks. + + This method should be used to index the documents in the destination. If page_content is None, the document should be indexed without the raw text. + All chunks belong to the stream and namespace specified in the parameters. + """ + pass + + @abstractmethod + def delete(self, delete_ids: List[str], namespace: str, stream: str) -> None: + """ + Delete document chunks belonging to certain record ids. + + This method should be used to delete documents from the destination. + The delete_ids parameter contains a list of record ids - all chunks with a record id in this list should be deleted from the destination. + All ids belong to the stream and namespace specified in the parameters. + """ + pass + + @abstractmethod + def check(self) -> Optional[str]: + """ + Check if the indexer is configured correctly. This method should be used to check if the indexer is configured correctly and return an error message if it is not. + """ + pass + + +T = TypeVar("T") + + +def chunks(iterable: Iterable[T], batch_size: int) -> Generator[Tuple[T, ...], None, None]: + """A helper function to break an iterable into chunks of size batch_size.""" + it = iter(iterable) + chunk = tuple(itertools.islice(it, batch_size)) + while chunk: + yield chunk + chunk = tuple(itertools.islice(it, batch_size)) diff --git a/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/test_utils.py b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/test_utils.py new file mode 100644 index 000000000000..7f8cfe5fbd8a --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/test_utils.py @@ -0,0 +1,53 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import unittest +from typing import Any, Dict + +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStateMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + SyncMode, + Type, +) + + +class BaseIntegrationTest(unittest.TestCase): + """ + BaseIntegrationTest is a base class for integration tests for vector db destinations. + + It provides helper methods to create Airbyte catalogs, records and state messages. + """ + + def _get_configured_catalog(self, destination_mode: DestinationSyncMode) -> ConfiguredAirbyteCatalog: + stream_schema = {"type": "object", "properties": {"str_col": {"type": "str"}, "int_col": {"type": "integer"}}} + + overwrite_stream = ConfiguredAirbyteStream( + stream=AirbyteStream( + name="mystream", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh] + ), + primary_key=[["int_col"]], + sync_mode=SyncMode.incremental, + destination_sync_mode=destination_mode, + ) + + return ConfiguredAirbyteCatalog(streams=[overwrite_stream]) + + def _state(self, data: Dict[str, Any]) -> AirbyteMessage: + return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data=data)) + + def _record(self, stream: str, str_value: str, int_value: int) -> AirbyteMessage: + return AirbyteMessage( + type=Type.RECORD, record=AirbyteRecordMessage(stream=stream, data={"str_col": str_value, "int_col": int_value}, emitted_at=0) + ) + + def setUp(self) -> None: + with open("secrets/config.json", "r") as f: + self.config = json.loads(f.read()) diff --git a/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/utils.py b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/utils.py new file mode 100644 index 000000000000..b0d4edebf890 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/utils.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import itertools +import traceback +from typing import Any, Iterable, Iterator, Tuple, Union + +from airbyte_cdk.models import AirbyteRecordMessage, AirbyteStream + + +def format_exception(exception: Exception) -> str: + return str(exception) + "\n" + "".join(traceback.TracebackException.from_exception(exception).format()) + + +def create_chunks(iterable: Iterable[Any], batch_size: int) -> Iterator[Tuple[Any, ...]]: + """A helper function to break an iterable into chunks of size batch_size.""" + it = iter(iterable) + chunk = tuple(itertools.islice(it, batch_size)) + while chunk: + yield chunk + chunk = tuple(itertools.islice(it, batch_size)) + + +def create_stream_identifier(stream: Union[AirbyteStream, AirbyteRecordMessage]) -> str: + if isinstance(stream, AirbyteStream): + return str(stream.name if stream.namespace is None else f"{stream.namespace}_{stream.name}") + else: + return str(stream.stream if stream.namespace is None else f"{stream.namespace}_{stream.stream}") diff --git a/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/writer.py b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/writer.py new file mode 100644 index 000000000000..0f764c366b54 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based/writer.py @@ -0,0 +1,85 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from collections import defaultdict +from typing import Dict, Iterable, List, Tuple + +from airbyte_cdk.destinations.vector_db_based.config import ProcessingConfigModel +from airbyte_cdk.destinations.vector_db_based.document_processor import Chunk, DocumentProcessor +from airbyte_cdk.destinations.vector_db_based.embedder import Document, Embedder +from airbyte_cdk.destinations.vector_db_based.indexer import Indexer +from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, Type + + +class Writer: + """ + The Writer class is orchestrating the document processor, the embedder and the indexer: + * Incoming records are passed through the document processor to generate chunks + * One the configured batch size is reached, the chunks are passed to the embedder to generate embeddings + * The embedder embeds the chunks + * The indexer deletes old chunks by the associated record id before indexing the new ones + + The destination connector is responsible to create a writer instance and pass the input messages iterable to the write method. + The batch size can be configured by the destination connector to give the freedom of either letting the user configure it or hardcoding it to a sensible value depending on the destination. + The omit_raw_text parameter can be used to omit the raw text from the chunks. This can be useful if the raw text is very large and not needed for the destination. + """ + + def __init__( + self, processing_config: ProcessingConfigModel, indexer: Indexer, embedder: Embedder, batch_size: int, omit_raw_text: bool + ) -> None: + self.processing_config = processing_config + self.indexer = indexer + self.embedder = embedder + self.batch_size = batch_size + self.omit_raw_text = omit_raw_text + self._init_batch() + + def _init_batch(self) -> None: + self.chunks: Dict[Tuple[str, str], List[Chunk]] = defaultdict(list) + self.ids_to_delete: Dict[Tuple[str, str], List[str]] = defaultdict(list) + self.number_of_chunks = 0 + + def _convert_to_document(self, chunk: Chunk) -> Document: + """ + Convert a chunk to a document for the embedder. + """ + if chunk.page_content is None: + raise ValueError("Cannot embed a chunk without page content") + return Document(page_content=chunk.page_content, record=chunk.record) + + def _process_batch(self) -> None: + for (namespace, stream), ids in self.ids_to_delete.items(): + self.indexer.delete(ids, namespace, stream) + + for (namespace, stream), chunks in self.chunks.items(): + embeddings = self.embedder.embed_documents([self._convert_to_document(chunk) for chunk in chunks]) + for i, document in enumerate(chunks): + document.embedding = embeddings[i] + if self.omit_raw_text: + document.page_content = None + self.indexer.index(chunks, namespace, stream) + + self._init_batch() + + def write(self, configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage]) -> Iterable[AirbyteMessage]: + self.processor = DocumentProcessor(self.processing_config, configured_catalog) + self.indexer.pre_sync(configured_catalog) + for message in input_messages: + if message.type == Type.STATE: + # Emitting a state message indicates that all records which came before it have been written to the destination. So we flush + # the queue to ensure writes happen, then output the state message to indicate it's safe to checkpoint state + self._process_batch() + yield message + elif message.type == Type.RECORD: + record_chunks, record_id_to_delete = self.processor.process(message.record) + self.chunks[(message.record.namespace, message.record.stream)].extend(record_chunks) + if record_id_to_delete is not None: + self.ids_to_delete[(message.record.namespace, message.record.stream)].append(record_id_to_delete) + self.number_of_chunks += len(record_chunks) + if self.number_of_chunks >= self.batch_size: + self._process_batch() + + self._process_batch() + yield from self.indexer.post_sync() diff --git a/airbyte-cdk/python/airbyte_cdk/entrypoint.py b/airbyte-cdk/python/airbyte_cdk/entrypoint.py index 3590d48bded1..37b2590b7ea6 100644 --- a/airbyte-cdk/python/airbyte_cdk/entrypoint.py +++ b/airbyte-cdk/python/airbyte_cdk/entrypoint.py @@ -14,6 +14,7 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union from urllib.parse import urlparse +import requests from airbyte_cdk.connector import TConfig from airbyte_cdk.exception_handler import init_uncaught_exception_handler from airbyte_cdk.logger import init_logger @@ -21,8 +22,11 @@ from airbyte_cdk.models.airbyte_protocol import ConnectorSpecification # type: ignore [attr-defined] from airbyte_cdk.sources import Source from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit, split_config +from airbyte_cdk.utils import is_cloud_environment from airbyte_cdk.utils.airbyte_secrets_utils import get_secrets, update_secrets +from airbyte_cdk.utils.constants import ENV_REQUEST_CACHE_PATH from airbyte_cdk.utils.traced_exception import AirbyteTracedException +from airbyte_protocol.models import FailureType from requests import PreparedRequest, Response, Session logger = init_logger("airbyte") @@ -35,10 +39,8 @@ class AirbyteEntrypoint(object): def __init__(self, source: Source): init_uncaught_exception_handler(logger) - # DEPLOYMENT_MODE is read when instantiating the entrypoint because it is the common path shared by syncs and connector - # builder test requests - deployment_mode = os.environ.get("DEPLOYMENT_MODE", "") - if deployment_mode.casefold() == CLOUD_DEPLOYMENT_MODE: + # deployment mode is read when instantiating the entrypoint because it is the common path shared by syncs and connector builder test requests + if is_cloud_environment(): _init_internal_request_filter() self.source = source @@ -86,6 +88,7 @@ def run(self, parsed_args: argparse.Namespace) -> Iterable[str]: if hasattr(parsed_args, "debug") and parsed_args.debug: self.logger.setLevel(logging.DEBUG) + logger.setLevel(logging.DEBUG) self.logger.debug("Debug logs enabled") else: self.logger.setLevel(logging.INFO) @@ -93,6 +96,7 @@ def run(self, parsed_args: argparse.Namespace) -> Iterable[str]: source_spec: ConnectorSpecification = self.source.spec(self.logger) try: with tempfile.TemporaryDirectory() as temp_dir: + os.environ[ENV_REQUEST_CACHE_PATH] = temp_dir # set this as default directory for request_cache to store *.sqlite files if cmd == "spec": message = AirbyteMessage(type=Type.SPEC, spec=source_spec) yield from [ @@ -103,6 +107,9 @@ def run(self, parsed_args: argparse.Namespace) -> Iterable[str]: raw_config = self.source.read_config(parsed_args.config) config = self.source.configure(raw_config, temp_dir) + yield from [ + self.airbyte_message_to_string(queued_message) for queued_message in self._emit_queued_messages(self.source) + ] if cmd == "check": yield from map(AirbyteEntrypoint.airbyte_message_to_string, self.check(source_spec, config)) elif cmd == "discover": @@ -174,6 +181,13 @@ def set_up_secret_filter(config: TConfig, connection_specification: Mapping[str, def airbyte_message_to_string(airbyte_message: AirbyteMessage) -> Any: return airbyte_message.json(exclude_unset=True) + @classmethod + def extract_state(cls, args: List[str]) -> Optional[Any]: + parsed_args = cls.parse_args(args) + if hasattr(parsed_args, "state"): + return parsed_args.state + return None + @classmethod def extract_catalog(cls, args: List[str]) -> Optional[Any]: parsed_args = cls.parse_args(args) @@ -212,25 +226,27 @@ def filtered_send(self: Any, request: PreparedRequest, **kwargs: Any) -> Respons parsed_url = urlparse(request.url) if parsed_url.scheme not in VALID_URL_SCHEMES: - raise ValueError( + raise requests.exceptions.InvalidSchema( "Invalid Protocol Scheme: The endpoint that data is being requested from is using an invalid or insecure " + f"protocol {parsed_url.scheme!r}. Valid protocol schemes: {','.join(VALID_URL_SCHEMES)}" ) if not parsed_url.hostname: - raise ValueError("Invalid URL specified: The endpoint that data is being requested from is not a valid URL") + raise requests.exceptions.InvalidURL("Invalid URL specified: The endpoint that data is being requested from is not a valid URL") try: is_private = _is_private_url(parsed_url.hostname, parsed_url.port) # type: ignore [arg-type] if is_private: - raise ValueError( - "Invalid URL endpoint: The endpoint that data is being requested from belongs to a private network. Source " - + "connectors only support requesting data from public API endpoints." + raise AirbyteTracedException( + internal_message=f"Invalid URL endpoint: `{parsed_url.hostname!r}` belongs to a private network", + failure_type=FailureType.config_error, + message="Invalid URL endpoint: The endpoint that data is being requested from belongs to a private network. Source connectors only support requesting data from public API endpoints.", ) - except socket.gaierror: + except socket.gaierror as exception: # This is a special case where the developer specifies an IP address string that is not formatted correctly like trailing # whitespace which will fail the socket IP lookup. This only happens when using IP addresses and not text hostnames. - raise ValueError(f"Invalid hostname or IP address '{parsed_url.hostname!r}' specified.") + # Knowing that this is a request using the requests library, we will mock the exception without calling the lib + raise requests.exceptions.InvalidURL(f"Invalid URL {parsed_url}: {exception}") return wrapped_fn(self, request, **kwargs) diff --git a/airbyte-cdk/python/airbyte_cdk/exception_handler.py b/airbyte-cdk/python/airbyte_cdk/exception_handler.py index 8ef7ed0ed55c..f8d3e2603e87 100644 --- a/airbyte-cdk/python/airbyte_cdk/exception_handler.py +++ b/airbyte-cdk/python/airbyte_cdk/exception_handler.py @@ -4,17 +4,25 @@ import logging import sys +from types import TracebackType +from typing import Any, Optional from airbyte_cdk.utils.traced_exception import AirbyteTracedException +def assemble_uncaught_exception(exception_type: type[BaseException], exception_value: BaseException) -> AirbyteTracedException: + if issubclass(exception_type, AirbyteTracedException): + return exception_value # type: ignore # validated as part of the previous line + return AirbyteTracedException.from_exception(exception_value) + + def init_uncaught_exception_handler(logger: logging.Logger) -> None: """ Handles uncaught exceptions by emitting an AirbyteTraceMessage and making sure they are not printed to the console without having secrets removed. """ - def hook_fn(exception_type, exception_value, traceback_): + def hook_fn(exception_type: type[BaseException], exception_value: BaseException, traceback_: Optional[TracebackType]) -> Any: # For developer ergonomics, we want to see the stack trace in the logs when we do a ctrl-c if issubclass(exception_type, KeyboardInterrupt): sys.__excepthook__(exception_type, exception_value, traceback_) @@ -23,11 +31,7 @@ def hook_fn(exception_type, exception_value, traceback_): logger.fatal(exception_value, exc_info=exception_value) # emit an AirbyteTraceMessage for any exception that gets to this spot - traced_exc = ( - exception_value - if issubclass(exception_type, AirbyteTracedException) - else AirbyteTracedException.from_exception(exception_value) - ) + traced_exc = assemble_uncaught_exception(exception_type, exception_value) traced_exc.emit_message() diff --git a/airbyte-cdk/python/airbyte_cdk/models/__init__.py b/airbyte-cdk/python/airbyte_cdk/models/__init__.py index 9545af7b044c..89b5cff1cb47 100644 --- a/airbyte-cdk/python/airbyte_cdk/models/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/models/__init__.py @@ -7,6 +7,7 @@ # of airbyte-cdk rather than a standalone package. from .airbyte_protocol import ( AdvancedAuth, + AirbyteAnalyticsTraceMessage, AirbyteCatalog, AirbyteConnectionStatus, AirbyteControlConnectorConfigMessage, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/__init__.py index 8516119a9179..396513539fcd 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/__init__.py @@ -2,8 +2,21 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # +import dpath.options + from .abstract_source import AbstractSource from .config import BaseConfig from .source import Source +# As part of the CDK sources, we do not control what the APIs return and it is possible that a key is empty. +# Reasons why we are doing this at the airbyte_cdk level: +# * As of today, all the use cases should allow for empty keys +# * Cases as of 2023-08-31: oauth/session token provider responses, extractor, transformation and substream) +# * The behavior is explicit at the package level and not hidden in every package that needs dpath.options.ALLOW_EMPTY_STRING_KEYS = True +# There is a downside in enforcing this option preemptively in the module __init__.py: the runtime code will import dpath even though the it +# might not need dpath leading to longer initialization time. +# There is a downside in using dpath as a library since the options are global: if we have two pieces of code that want different options, +# this will not be thread-safe. +dpath.options.ALLOW_EMPTY_STRING_KEYS = True + __all__ = ["AbstractSource", "BaseConfig", "Source"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py b/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py index 68d804666f42..0f8bf716cc10 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py @@ -2,37 +2,37 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import json import logging from abc import ABC, abstractmethod -from typing import Any, Dict, Iterator, List, Mapping, MutableMapping, Optional, Tuple, Union +from typing import Any, Dict, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Tuple, Union from airbyte_cdk.models import ( AirbyteCatalog, AirbyteConnectionStatus, - AirbyteLogMessage, AirbyteMessage, AirbyteStateMessage, AirbyteStreamStatus, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, - Level, Status, SyncMode, ) from airbyte_cdk.models import Type as MessageType from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager -from airbyte_cdk.sources.message import MessageRepository +from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository from airbyte_cdk.sources.source import Source from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.core import StreamData from airbyte_cdk.sources.streams.http.http import HttpStream from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message from airbyte_cdk.sources.utils.schema_helpers import InternalConfig, split_config +from airbyte_cdk.sources.utils.slice_logger import DebugSliceLogger, SliceLogger from airbyte_cdk.utils.event_timing import create_timer from airbyte_cdk.utils.stream_status_utils import as_airbyte_message as stream_status_as_airbyte_message from airbyte_cdk.utils.traced_exception import AirbyteTracedException +_default_message_repository = InMemoryMessageRepository() + class AbstractSource(Source, ABC): """ @@ -40,8 +40,6 @@ class AbstractSource(Source, ABC): in this class to create an Airbyte Specification compliant Source. """ - SLICE_LOG_PREFIX = "slice:" - @abstractmethod def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: """ @@ -65,6 +63,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: # Stream name to instance map for applying output object transformation _stream_to_instance_map: Dict[str, Stream] = {} + _slice_logger: SliceLogger = DebugSliceLogger() @property def name(self) -> str: @@ -92,7 +91,7 @@ def read( logger: logging.Logger, config: Mapping[str, Any], catalog: ConfiguredAirbyteCatalog, - state: Union[List[AirbyteStateMessage], MutableMapping[str, Any]] = None, + state: Optional[Union[List[AirbyteStateMessage], MutableMapping[str, Any]]] = None, ) -> Iterator[AirbyteMessage]: """Implements the Read operation from the Airbyte Specification. See https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/.""" logger.info(f"Starting syncing {self.name}") @@ -102,13 +101,18 @@ def read( stream_instances = {s.name: s for s in self.streams(config)} state_manager = ConnectorStateManager(stream_instance_map=stream_instances, state=state) self._stream_to_instance_map = stream_instances + + stream_name_to_exception: MutableMapping[str, AirbyteTracedException] = {} + with create_timer(self.name) as timer: for configured_stream in catalog.streams: stream_instance = stream_instances.get(configured_stream.stream.name) if not stream_instance: + if not self.raise_exception_on_missing_stream: + continue raise KeyError( - f"The requested stream {configured_stream.stream.name} was not found in the source." - f" Available streams: {stream_instances.keys()}" + f"The stream {configured_stream.stream.name} no longer exists in the configuration. " + f"Refresh the schema in replication settings and remove this stream from future sync attempts." ) try: @@ -118,7 +122,7 @@ def read( logger.warning(f"Skipped syncing stream '{stream_instance.name}' because it was unavailable. {reason}") continue logger.info(f"Marking stream {configured_stream.stream.name} as STARTED") - yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.STARTED) + yield stream_status_as_airbyte_message(configured_stream.stream, AirbyteStreamStatus.STARTED) yield from self._read_stream( logger=logger, stream_instance=stream_instance, @@ -127,15 +131,18 @@ def read( internal_config=internal_config, ) logger.info(f"Marking stream {configured_stream.stream.name} as STOPPED") - yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.COMPLETE) + yield stream_status_as_airbyte_message(configured_stream.stream, AirbyteStreamStatus.COMPLETE) except AirbyteTracedException as e: - yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.INCOMPLETE) - raise e + yield stream_status_as_airbyte_message(configured_stream.stream, AirbyteStreamStatus.INCOMPLETE) + if self.continue_sync_on_stream_failure: + stream_name_to_exception[stream_instance.name] = e + else: + raise e except Exception as e: yield from self._emit_queued_messages() logger.exception(f"Encountered an exception while reading stream {configured_stream.stream.name}") logger.info(f"Marking stream {configured_stream.stream.name} as STOPPED") - yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.INCOMPLETE) + yield stream_status_as_airbyte_message(configured_stream.stream, AirbyteStreamStatus.INCOMPLETE) display_message = stream_instance.get_error_display_message(e) if display_message: raise AirbyteTracedException.from_exception(e, message=display_message) from e @@ -145,8 +152,14 @@ def read( logger.info(f"Finished syncing {configured_stream.stream.name}") logger.info(timer.report()) + if self.continue_sync_on_stream_failure and len(stream_name_to_exception) > 0: + raise AirbyteTracedException(message=self._generate_failed_streams_error_message(stream_name_to_exception)) logger.info(f"Finished syncing {self.name}") + @property + def raise_exception_on_missing_stream(self) -> bool: + return True + @property def per_stream_state_enabled(self) -> bool: return True @@ -159,7 +172,6 @@ def _read_stream( state_manager: ConnectorStateManager, internal_config: InternalConfig, ) -> Iterator[AirbyteMessage]: - self._apply_log_level_to_stream_logger(logger, stream_instance) if internal_config.page_size and isinstance(stream_instance, HttpStream): logger.info(f"Setting page size for {stream_instance.name} to {internal_config.page_size}") stream_instance.page_size = internal_config.page_size @@ -171,13 +183,7 @@ def _read_stream( "cursor_field": configured_stream.cursor_field, }, ) - logger.debug( - f"Syncing stream instance: {stream_instance.name}", - extra={ - "primary_key": stream_instance.primary_key, - "cursor_field": stream_instance.cursor_field, - }, - ) + stream_instance.log_stream_sync_configuration() use_incremental = configured_stream.sync_mode == SyncMode.incremental and stream_instance.supports_incremental if use_incremental: @@ -200,25 +206,12 @@ def _read_stream( if record_counter == 1: logger.info(f"Marking stream {stream_name} as RUNNING") # If we just read the first record of the stream, emit the transition to the RUNNING state - yield stream_status_as_airbyte_message(configured_stream, AirbyteStreamStatus.RUNNING) + yield stream_status_as_airbyte_message(configured_stream.stream, AirbyteStreamStatus.RUNNING) yield from self._emit_queued_messages() yield record logger.info(f"Read {record_counter} records from {stream_name} stream") - @staticmethod - def _limit_reached(internal_config: InternalConfig, records_counter: int) -> bool: - """ - Check if record count reached limit set by internal config. - :param internal_config - internal CDK configuration separated from user defined config - :records_counter - number of records already red - :return True if limit reached, False otherwise - """ - if internal_config.limit: - if records_counter >= internal_config.limit: - return True - return False - def _read_incremental( self, logger: logging.Logger, @@ -240,67 +233,21 @@ def _read_incremental( stream_state = state_manager.get_stream_state(stream_name, stream_instance.namespace) if stream_state and "state" in dir(stream_instance): - stream_instance.state = stream_state - logger.info(f"Setting state of {stream_name} stream to {stream_state}") - - slices = stream_instance.stream_slices( - cursor_field=configured_stream.cursor_field, - sync_mode=SyncMode.incremental, - stream_state=stream_state, - ) - logger.debug(f"Processing stream slices for {stream_name} (sync_mode: incremental)", extra={"stream_slices": slices}) - - total_records_counter = 0 - has_slices = False - for _slice in slices: - has_slices = True - if self.should_log_slice_message(logger): - yield self._create_slice_log_message(_slice) - records = stream_instance.read_records( - sync_mode=SyncMode.incremental, - stream_slice=_slice, - stream_state=stream_state, - cursor_field=configured_stream.cursor_field or None, - ) - record_counter = 0 - for message_counter, record_data_or_message in enumerate(records, start=1): - message = self._get_message(record_data_or_message, stream_instance) - yield from self._emit_queued_messages() - yield message - if message.type == MessageType.RECORD: - record = message.record - stream_state = stream_instance.get_updated_state(stream_state, record.data) - checkpoint_interval = stream_instance.state_checkpoint_interval - record_counter += 1 - if checkpoint_interval and record_counter % checkpoint_interval == 0: - yield self._checkpoint_state(stream_instance, stream_state, state_manager) - - total_records_counter += 1 - # This functionality should ideally live outside of this method - # but since state is managed inside this method, we keep track - # of it here. - if self._limit_reached(internal_config, total_records_counter): - # Break from slice loop to save state and exit from _read_incremental function. - break - - yield self._checkpoint_state(stream_instance, stream_state, state_manager) - if self._limit_reached(internal_config, total_records_counter): - return - - if not has_slices: - # Safety net to ensure we always emit at least one state message even if there are no slices - checkpoint = self._checkpoint_state(stream_instance, stream_state, state_manager) - yield checkpoint - - def should_log_slice_message(self, logger: logging.Logger): - """ - - :param logger: - :return: - """ - return logger.isEnabledFor(logging.DEBUG) - - def _emit_queued_messages(self): + stream_instance.state = stream_state # type: ignore # we check that state in the dir(stream_instance) + logger.info(f"Setting state of {self.name} stream to {stream_state}") + + for record_data_or_message in stream_instance.read_incremental( + configured_stream.cursor_field, + logger, + self._slice_logger, + stream_state, + state_manager, + self.per_stream_state_enabled, + internal_config, + ): + yield self._get_message(record_data_or_message, stream_instance) + + def _emit_queued_messages(self) -> Iterable[AirbyteMessage]: if self.message_repository: yield from self.message_repository.consume_queue() return @@ -312,59 +259,16 @@ def _read_full_refresh( configured_stream: ConfiguredAirbyteStream, internal_config: InternalConfig, ) -> Iterator[AirbyteMessage]: - slices = stream_instance.stream_slices(sync_mode=SyncMode.full_refresh, cursor_field=configured_stream.cursor_field) - logger.debug( - f"Processing stream slices for {configured_stream.stream.name} (sync_mode: full_refresh)", extra={"stream_slices": slices} - ) total_records_counter = 0 - for _slice in slices: - if self.should_log_slice_message(logger): - yield self._create_slice_log_message(_slice) - record_data_or_messages = stream_instance.read_records( - stream_slice=_slice, - sync_mode=SyncMode.full_refresh, - cursor_field=configured_stream.cursor_field, - ) - for record_data_or_message in record_data_or_messages: - message = self._get_message(record_data_or_message, stream_instance) - yield message - if message.type == MessageType.RECORD: - total_records_counter += 1 - if self._limit_reached(internal_config, total_records_counter): - return - - def _create_slice_log_message(self, _slice: Optional[Mapping[str, Any]]) -> AirbyteMessage: - """ - Mapping is an interface that can be implemented in various ways. However, json.dumps will just do a `str()` if - the slice is a class implementing Mapping. Therefore, we want to cast this as a dict before passing this to json.dump - """ - printable_slice = dict(_slice) if _slice else _slice - return AirbyteMessage( - type=MessageType.LOG, - log=AirbyteLogMessage(level=Level.INFO, message=f"{self.SLICE_LOG_PREFIX}{json.dumps(printable_slice, default=str)}"), - ) - - def _checkpoint_state(self, stream: Stream, stream_state, state_manager: ConnectorStateManager): - # First attempt to retrieve the current state using the stream's state property. We receive an AttributeError if the state - # property is not implemented by the stream instance and as a fallback, use the stream_state retrieved from the stream - # instance's deprecated get_updated_state() method. - try: - state_manager.update_state_for_stream(stream.name, stream.namespace, stream.state) - - except AttributeError: - state_manager.update_state_for_stream(stream.name, stream.namespace, stream_state) - return state_manager.create_state_message(stream.name, stream.namespace, send_per_stream_state=self.per_stream_state_enabled) - - @staticmethod - def _apply_log_level_to_stream_logger(logger: logging.Logger, stream_instance: Stream): - """ - Necessary because we use different loggers at the source and stream levels. We must - apply the source's log level to each stream's logger. - """ - if hasattr(logger, "level"): - stream_instance.logger.setLevel(logger.level) - - def _get_message(self, record_data_or_message: Union[StreamData, AirbyteMessage], stream: Stream): + for record_data_or_message in stream_instance.read_full_refresh(configured_stream.cursor_field, logger, self._slice_logger): + message = self._get_message(record_data_or_message, stream_instance) + yield message + if message.type == MessageType.RECORD: + total_records_counter += 1 + if internal_config.is_limit_reached(total_records_counter): + return + + def _get_message(self, record_data_or_message: Union[StreamData, AirbyteMessage], stream: Stream) -> AirbyteMessage: """ Converts the input to an AirbyteMessage if it is a StreamData. Returns the input as is if it is already an AirbyteMessage """ @@ -375,4 +279,20 @@ def _get_message(self, record_data_or_message: Union[StreamData, AirbyteMessage] @property def message_repository(self) -> Union[None, MessageRepository]: - return None + return _default_message_repository + + @property + def continue_sync_on_stream_failure(self) -> bool: + """ + WARNING: This function is in-development which means it is subject to change. Use at your own risk. + + By default, a source should raise an exception and stop the sync when it encounters an error while syncing a stream. This + method can be overridden on a per-source basis so that a source will continue syncing streams other streams even if an + exception is raised for a stream. + """ + return False + + @staticmethod + def _generate_failed_streams_error_message(stream_failures: Mapping[str, AirbyteTracedException]) -> str: + failures = ", ".join([f"{stream}: {exception.__repr__()}" for stream, exception in stream_failures.items()]) + return f"During the sync, the following streams did not sync successfully: {failures}" diff --git a/airbyte-ci/connectors/pipelines/pipelines/commands/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/__init__.py similarity index 100% rename from airbyte-ci/connectors/pipelines/pipelines/commands/__init__.py rename to airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/__init__.py diff --git a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py new file mode 100644 index 000000000000..acfc0c039694 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py @@ -0,0 +1,189 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import logging +from typing import Dict, Iterable, List, Optional, Set + +from airbyte_cdk.models import AirbyteMessage, AirbyteStreamStatus +from airbyte_cdk.models import Type as MessageType +from airbyte_cdk.sources.concurrent_source.partition_generation_completed_sentinel import PartitionGenerationCompletedSentinel +from airbyte_cdk.sources.concurrent_source.thread_pool_manager import ThreadPoolManager +from airbyte_cdk.sources.message import MessageRepository +from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream +from airbyte_cdk.sources.streams.concurrent.partition_enqueuer import PartitionEnqueuer +from airbyte_cdk.sources.streams.concurrent.partition_reader import PartitionReader +from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition +from airbyte_cdk.sources.streams.concurrent.partitions.record import Record +from airbyte_cdk.sources.streams.concurrent.partitions.types import PartitionCompleteSentinel +from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message +from airbyte_cdk.sources.utils.slice_logger import SliceLogger +from airbyte_cdk.utils.stream_status_utils import as_airbyte_message as stream_status_as_airbyte_message + + +class ConcurrentReadProcessor: + def __init__( + self, + stream_instances_to_read_from: List[AbstractStream], + partition_enqueuer: PartitionEnqueuer, + thread_pool_manager: ThreadPoolManager, + logger: logging.Logger, + slice_logger: SliceLogger, + message_repository: MessageRepository, + partition_reader: PartitionReader, + ): + """ + This class is responsible for handling items from a concurrent stream read process. + :param stream_instances_to_read_from: List of streams to read from + :param partition_enqueuer: PartitionEnqueuer instance + :param thread_pool_manager: ThreadPoolManager instance + :param logger: Logger instance + :param slice_logger: SliceLogger instance + :param message_repository: MessageRepository instance + :param partition_reader: PartitionReader instance + """ + self._stream_name_to_instance = {s.name: s for s in stream_instances_to_read_from} + self._record_counter = {} + self._streams_to_running_partitions: Dict[str, Set[Partition]] = {} + for stream in stream_instances_to_read_from: + self._streams_to_running_partitions[stream.name] = set() + self._record_counter[stream.name] = 0 + self._thread_pool_manager = thread_pool_manager + self._partition_enqueuer = partition_enqueuer + self._stream_instances_to_start_partition_generation = stream_instances_to_read_from + self._streams_currently_generating_partitions: List[str] = [] + self._logger = logger + self._slice_logger = slice_logger + self._message_repository = message_repository + self._partition_reader = partition_reader + self._streams_done: Set[str] = set() + + def on_partition_generation_completed(self, sentinel: PartitionGenerationCompletedSentinel) -> Iterable[AirbyteMessage]: + """ + This method is called when a partition generation is completed. + 1. Remove the stream from the list of streams currently generating partitions + 2. If the stream is done, mark it as such and return a stream status message + 3. If there are more streams to read from, start the next partition generator + """ + stream_name = sentinel.stream.name + self._streams_currently_generating_partitions.remove(sentinel.stream.name) + ret = [] + # It is possible for the stream to already be done if no partitions were generated + # If the partition generation process was completed and there are no partitions left to process, the stream is done + if self._is_stream_done(stream_name) or len(self._streams_to_running_partitions[stream_name]) == 0: + ret.append(self._on_stream_is_done(stream_name)) + if self._stream_instances_to_start_partition_generation: + ret.append(self.start_next_partition_generator()) + return ret + + def on_partition(self, partition: Partition) -> None: + """ + This method is called when a partition is generated. + 1. Add the partition to the set of partitions for the stream + 2. Log the slice if necessary + 3. Submit the partition to the thread pool manager + """ + stream_name = partition.stream_name() + self._streams_to_running_partitions[stream_name].add(partition) + if self._slice_logger.should_log_slice_message(self._logger): + self._message_repository.emit_message(self._slice_logger.create_slice_log_message(partition.to_slice())) + self._thread_pool_manager.submit(self._partition_reader.process_partition, partition) + + def on_partition_complete_sentinel(self, sentinel: PartitionCompleteSentinel) -> Iterable[AirbyteMessage]: + """ + This method is called when a partition is completed. + 1. Close the partition + 2. If the stream is done, mark it as such and return a stream status message + 3. Emit messages that were added to the message repository + """ + partition = sentinel.partition + partition.close() + partitions_running = self._streams_to_running_partitions[partition.stream_name()] + if partition in partitions_running: + partitions_running.remove(partition) + # If all partitions were generated and this was the last one, the stream is done + if partition.stream_name() not in self._streams_currently_generating_partitions and len(partitions_running) == 0: + yield self._on_stream_is_done(partition.stream_name()) + yield from self._message_repository.consume_queue() + + def on_record(self, record: Record) -> Iterable[AirbyteMessage]: + """ + This method is called when a record is read from a partition. + 1. Convert the record to an AirbyteMessage + 2. If this is the first record for the stream, mark the stream as RUNNING + 3. Increment the record counter for the stream + 4. Emit the message + 5. Emit messages that were added to the message repository + """ + # Do not pass a transformer or a schema + # AbstractStreams are expected to return data as they are expected. + # Any transformation on the data should be done before reaching this point + message = stream_data_to_airbyte_message(record.stream_name, record.data) + stream = self._stream_name_to_instance[record.stream_name] + + if message.type == MessageType.RECORD: + if self._record_counter[stream.name] == 0: + self._logger.info(f"Marking stream {stream.name} as RUNNING") + yield stream_status_as_airbyte_message(stream.as_airbyte_stream(), AirbyteStreamStatus.RUNNING) + self._record_counter[stream.name] += 1 + yield message + yield from self._message_repository.consume_queue() + + def on_exception(self, exception: Exception) -> Iterable[AirbyteMessage]: + """ + This method is called when an exception is raised. + 1. Stop all running streams + 2. Raise the exception + """ + yield from self._stop_streams() + raise exception + + def start_next_partition_generator(self) -> Optional[AirbyteMessage]: + """ + Start the next partition generator. + 1. Pop the next stream to read from + 2. Submit the partition generator to the thread pool manager + 3. Add the stream to the list of streams currently generating partitions + 4. Return a stream status message + """ + if self._stream_instances_to_start_partition_generation: + stream = self._stream_instances_to_start_partition_generation.pop(0) + self._thread_pool_manager.submit(self._partition_enqueuer.generate_partitions, stream) + self._streams_currently_generating_partitions.append(stream.name) + self._logger.info(f"Marking stream {stream.name} as STARTED") + self._logger.info(f"Syncing stream: {stream.name} ") + return stream_status_as_airbyte_message( + stream.as_airbyte_stream(), + AirbyteStreamStatus.STARTED, + ) + else: + return None + + def is_done(self) -> bool: + """ + This method is called to check if the sync is done. + The sync is done when: + 1. There are no more streams generating partitions + 2. There are no more streams to read from + 3. All partitions for all streams are closed + """ + return all([self._is_stream_done(stream_name) for stream_name in self._stream_name_to_instance.keys()]) + + def _is_stream_done(self, stream_name: str) -> bool: + return stream_name in self._streams_done + + def _on_stream_is_done(self, stream_name: str) -> AirbyteMessage: + self._logger.info(f"Read {self._record_counter[stream_name]} records from {stream_name} stream") + self._logger.info(f"Marking stream {stream_name} as STOPPED") + stream = self._stream_name_to_instance[stream_name] + self._logger.info(f"Finished syncing {stream.name}") + self._streams_done.add(stream_name) + return stream_status_as_airbyte_message(stream.as_airbyte_stream(), AirbyteStreamStatus.COMPLETE) + + def _stop_streams(self) -> Iterable[AirbyteMessage]: + self._thread_pool_manager.shutdown() + for stream_name in self._streams_to_running_partitions.keys(): + stream = self._stream_name_to_instance[stream_name] + if not self._is_stream_done(stream_name): + self._logger.info(f"Marking stream {stream.name} as STOPPED") + self._logger.info(f"Finished syncing {stream.name}") + yield stream_status_as_airbyte_message(stream.as_airbyte_stream(), AirbyteStreamStatus.INCOMPLETE) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_source.py b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_source.py new file mode 100644 index 000000000000..f37b78960a81 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_source.py @@ -0,0 +1,163 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import concurrent +import logging +from queue import Queue +from typing import Iterable, Iterator, List + +from airbyte_cdk.models import AirbyteMessage +from airbyte_cdk.sources.concurrent_source.concurrent_read_processor import ConcurrentReadProcessor +from airbyte_cdk.sources.concurrent_source.partition_generation_completed_sentinel import PartitionGenerationCompletedSentinel +from airbyte_cdk.sources.concurrent_source.thread_pool_manager import ThreadPoolManager +from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository +from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream +from airbyte_cdk.sources.streams.concurrent.partition_enqueuer import PartitionEnqueuer +from airbyte_cdk.sources.streams.concurrent.partition_reader import PartitionReader +from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition +from airbyte_cdk.sources.streams.concurrent.partitions.record import Record +from airbyte_cdk.sources.streams.concurrent.partitions.throttled_queue import ThrottledQueue +from airbyte_cdk.sources.streams.concurrent.partitions.types import PartitionCompleteSentinel, QueueItem +from airbyte_cdk.sources.utils.slice_logger import DebugSliceLogger, SliceLogger + + +class ConcurrentSource: + """ + A Source that reads data from multiple AbstractStreams concurrently. + It does so by submitting partition generation, and partition read tasks to a thread pool. + The tasks asynchronously add their output to a shared queue. + The read is done when all partitions for all streams were generated and read. + """ + + DEFAULT_TIMEOUT_SECONDS = 900 + + @staticmethod + def create( + num_workers: int, + initial_number_of_partitions_to_generate: int, + logger: logging.Logger, + slice_logger: SliceLogger, + message_repository: MessageRepository, + timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS, + ) -> "ConcurrentSource": + threadpool = ThreadPoolManager( + concurrent.futures.ThreadPoolExecutor(max_workers=num_workers, thread_name_prefix="workerpool"), + logger, + ) + return ConcurrentSource( + threadpool, logger, slice_logger, message_repository, initial_number_of_partitions_to_generate, timeout_seconds + ) + + def __init__( + self, + threadpool: ThreadPoolManager, + logger: logging.Logger, + slice_logger: SliceLogger = DebugSliceLogger(), + message_repository: MessageRepository = InMemoryMessageRepository(), + initial_number_partitions_to_generate: int = 1, + timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS, + ) -> None: + """ + :param threadpool: The threadpool to submit tasks to + :param logger: The logger to log to + :param slice_logger: The slice logger used to create messages on new slices + :param message_repository: The repository to emit messages to + :param initial_number_partitions_to_generate: The initial number of concurrent partition generation tasks. Limiting this number ensures will limit the latency of the first records emitted. While the latency is not critical, emitting the records early allows the platform and the destination to process them as early as possible. + :param timeout_seconds: The maximum number of seconds to wait for a record to be read from the queue. If no record is read within this time, the source will stop reading and return. + """ + self._threadpool = threadpool + self._logger = logger + self._slice_logger = slice_logger + self._message_repository = message_repository + self._initial_number_partitions_to_generate = initial_number_partitions_to_generate + self._timeout_seconds = timeout_seconds + + def read( + self, + streams: List[AbstractStream], + ) -> Iterator[AirbyteMessage]: + self._logger.info("Starting syncing") + stream_instances_to_read_from = self._get_streams_to_read_from(streams) + + # Return early if there are no streams to read from + if not stream_instances_to_read_from: + return + + queue: ThrottledQueue = ThrottledQueue(Queue(), self._threadpool.get_throttler(), self._timeout_seconds) + concurrent_stream_processor = ConcurrentReadProcessor( + stream_instances_to_read_from, + PartitionEnqueuer(queue), + self._threadpool, + self._logger, + self._slice_logger, + self._message_repository, + PartitionReader(queue), + ) + + # Enqueue initial partition generation tasks + yield from self._submit_initial_partition_generators(concurrent_stream_processor) + + # Read from the queue until all partitions were generated and read + yield from self._consume_from_queue( + queue, + concurrent_stream_processor, + ) + self._threadpool.check_for_errors_and_shutdown() + self._logger.info("Finished syncing") + + def _submit_initial_partition_generators(self, concurrent_stream_processor: ConcurrentReadProcessor) -> Iterable[AirbyteMessage]: + for _ in range(self._initial_number_partitions_to_generate): + status_message = concurrent_stream_processor.start_next_partition_generator() + if status_message: + yield status_message + + def _consume_from_queue( + self, + queue: ThrottledQueue, + concurrent_stream_processor: ConcurrentReadProcessor, + ) -> Iterable[AirbyteMessage]: + while airbyte_message_or_record_or_exception := queue.get(): + yield from self._handle_item( + airbyte_message_or_record_or_exception, + concurrent_stream_processor, + ) + if concurrent_stream_processor.is_done() and queue.empty(): + # all partitions were generated and processed. we're done here + break + + def _handle_item( + self, + queue_item: QueueItem, + concurrent_stream_processor: ConcurrentReadProcessor, + ) -> Iterable[AirbyteMessage]: + # handle queue item and call the appropriate handler depending on the type of the queue item + if isinstance(queue_item, Exception): + yield from concurrent_stream_processor.on_exception(queue_item) + + elif isinstance(queue_item, PartitionGenerationCompletedSentinel): + yield from concurrent_stream_processor.on_partition_generation_completed(queue_item) + + elif isinstance(queue_item, Partition): + concurrent_stream_processor.on_partition(queue_item) + elif isinstance(queue_item, PartitionCompleteSentinel): + yield from concurrent_stream_processor.on_partition_complete_sentinel(queue_item) + elif isinstance(queue_item, Record): + yield from concurrent_stream_processor.on_record(queue_item) + else: + raise ValueError(f"Unknown queue item type: {type(queue_item)}") + + def _get_streams_to_read_from(self, streams: List[AbstractStream]) -> List[AbstractStream]: + """ + Iterate over the configured streams and return a list of streams to read from. + If a stream is not configured, it will be skipped. + If a stream is configured but does not exist in the source and self.raise_exception_on_missing_stream is True, an exception will be raised + If a stream is not available, it will be skipped + """ + stream_instances_to_read_from = [] + for stream in streams: + stream_availability = stream.check_availability() + if not stream_availability.is_available(): + self._logger.warning(f"Skipped syncing stream '{stream.name}' because it was unavailable. {stream_availability.message()}") + continue + stream_instances_to_read_from.append(stream) + return stream_instances_to_read_from diff --git a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py new file mode 100644 index 000000000000..8e2ea80b79ae --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_source_adapter.py @@ -0,0 +1,63 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import logging +from abc import ABC +from typing import Any, Iterator, List, Mapping, MutableMapping, Optional, Union + +from airbyte_cdk.models import AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream +from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade + + +class ConcurrentSourceAdapter(AbstractSource, ABC): + def __init__(self, concurrent_source: ConcurrentSource, **kwargs: Any) -> None: + """ + ConcurrentSourceAdapter is a Source that wraps a concurrent source and exposes it as a regular source. + + The source's streams are still defined through the streams() method. + Streams wrapped in a StreamFacade will be processed concurrently. + Other streams will be processed sequentially as a later step. + """ + self._concurrent_source = concurrent_source + super().__init__(**kwargs) + + def read( + self, + logger: logging.Logger, + config: Mapping[str, Any], + catalog: ConfiguredAirbyteCatalog, + state: Optional[Union[List[AirbyteStateMessage], MutableMapping[str, Any]]] = None, + ) -> Iterator[AirbyteMessage]: + abstract_streams = self._select_abstract_streams(config, catalog) + concurrent_stream_names = {stream.name for stream in abstract_streams} + configured_catalog_for_regular_streams = ConfiguredAirbyteCatalog( + streams=[stream for stream in catalog.streams if stream.stream.name not in concurrent_stream_names] + ) + if abstract_streams: + yield from self._concurrent_source.read(abstract_streams) + if configured_catalog_for_regular_streams.streams: + yield from super().read(logger, config, configured_catalog_for_regular_streams, state) + + def _select_abstract_streams(self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog) -> List[AbstractStream]: + """ + Selects streams that can be processed concurrently and returns their abstract representations. + """ + all_streams = self.streams(config) + stream_name_to_instance: Mapping[str, Stream] = {s.name: s for s in all_streams} + abstract_streams: List[AbstractStream] = [] + for configured_stream in configured_catalog.streams: + stream_instance = stream_name_to_instance.get(configured_stream.stream.name) + if not stream_instance: + if not self.raise_exception_on_missing_stream: + continue + raise KeyError( + f"The stream {configured_stream.stream.name} no longer exists in the configuration. " + f"Refresh the schema in replication settings and remove this stream from future sync attempts." + ) + if isinstance(stream_instance, StreamFacade): + abstract_streams.append(stream_instance._abstract_stream) + return abstract_streams diff --git a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py new file mode 100644 index 000000000000..6c351850e62d --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/partition_generation_completed_sentinel.py @@ -0,0 +1,17 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream + + +class PartitionGenerationCompletedSentinel: + """ + A sentinel object indicating all partitions for a stream were produced. + Includes a pointer to the stream that was processed. + """ + + def __init__(self, stream: AbstractStream): + """ + :param stream: The stream that was processed + """ + self.stream = stream diff --git a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/thread_pool_manager.py b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/thread_pool_manager.py new file mode 100644 index 000000000000..3c0eec206c54 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/thread_pool_manager.py @@ -0,0 +1,94 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import logging +from concurrent.futures import Future, ThreadPoolExecutor +from typing import Any, Callable, List + +from airbyte_cdk.sources.concurrent_source.throttler import Throttler + + +class ThreadPoolManager: + """ + Wrapper to abstract away the threadpool and the logic to wait for pending tasks to be completed. + """ + + DEFAULT_SLEEP_TIME = 0.1 + DEFAULT_MAX_QUEUE_SIZE = 10_000 + + def __init__( + self, + threadpool: ThreadPoolExecutor, + logger: logging.Logger, + sleep_time: float = DEFAULT_SLEEP_TIME, + max_concurrent_tasks: int = DEFAULT_MAX_QUEUE_SIZE, + ): + """ + :param threadpool: The threadpool to use + :param logger: The logger to use + :param max_concurrent_tasks: The maximum number of tasks that can be pending at the same time + :param sleep_time: How long to sleep if there are too many pending tasks + """ + self._threadpool = threadpool + self._logger = logger + self._max_concurrent_tasks = max_concurrent_tasks + self._futures: List[Future[Any]] = [] + self._throttler = Throttler(self._futures, sleep_time, max_concurrent_tasks) + + def get_throttler(self) -> Throttler: + return self._throttler + + def submit(self, function: Callable[..., Any], *args: Any) -> None: + # Submit a task to the threadpool, removing completed tasks if there are too many tasks in self._futures. + self._prune_futures(self._futures) + self._futures.append(self._threadpool.submit(function, *args)) + + def _prune_futures(self, futures: List[Future[Any]]) -> None: + """ + Take a list in input and remove the futures that are completed. If a future has an exception, it'll raise and kill the stream + operation. + + Pruning this list safely relies on the assumptions that only the main thread can modify the list of futures. + """ + if len(futures) < self._max_concurrent_tasks: + return + + for index in reversed(range(len(futures))): + future = futures[index] + + if future.done(): + # Only call future.exception() if the future is known to be done because it will block until the future is done. + # See https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.Future.exception + optional_exception = future.exception() + if optional_exception: + exception = RuntimeError(f"Failed reading with error: {optional_exception}") + self._stop_and_raise_exception(exception) + futures.pop(index) + + def shutdown(self) -> None: + self._threadpool.shutdown(wait=False, cancel_futures=True) + + def is_done(self) -> bool: + return all([f.done() for f in self._futures]) + + def check_for_errors_and_shutdown(self) -> None: + """ + Check if any of the futures have an exception, and raise it if so. If all futures are done, shutdown the threadpool. + If the futures are not done, raise an exception. + :return: + """ + exceptions_from_futures = [f for f in [future.exception() for future in self._futures] if f is not None] + if exceptions_from_futures: + exception = RuntimeError(f"Failed reading with errors: {exceptions_from_futures}") + self._stop_and_raise_exception(exception) + else: + futures_not_done = [f for f in self._futures if not f.done()] + if futures_not_done: + exception = RuntimeError(f"Failed reading with futures not done: {futures_not_done}") + self._stop_and_raise_exception(exception) + else: + self.shutdown() + + def _stop_and_raise_exception(self, exception: BaseException) -> None: + self.shutdown() + raise exception diff --git a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/throttler.py b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/throttler.py new file mode 100644 index 000000000000..5b343caef1d7 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/throttler.py @@ -0,0 +1,25 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import time +from concurrent.futures import Future +from typing import Any, List + + +class Throttler: + """ + A throttler that waits until the number of concurrent tasks is below a certain threshold. + """ + + def __init__(self, futures_list: List[Future[Any]], sleep_time: float, max_concurrent_tasks: int): + """ + :param futures_list: The list of futures to monitor + :param sleep_time: How long to sleep if there are too many pending tasks + :param max_concurrent_tasks: The maximum number of tasks that can be pending at the same time + """ + self._futures_list = futures_list + self._sleep_time = sleep_time + self._max_concurrent_tasks = max_concurrent_tasks + + def wait_and_acquire(self) -> None: + while len(self._futures_list) >= self._max_concurrent_tasks: + time.sleep(self._sleep_time) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/connector_state_manager.py b/airbyte-cdk/python/airbyte_cdk/sources/connector_state_manager.py index 8e90deb1581b..575756f5a706 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/connector_state_manager.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/connector_state_manager.py @@ -28,7 +28,9 @@ class ConnectorStateManager: interface. It also provides methods to extract and update state """ - def __init__(self, stream_instance_map: Mapping[str, Stream], state: Union[List[AirbyteStateMessage], MutableMapping[str, Any]] = None): + def __init__( + self, stream_instance_map: Mapping[str, Stream], state: Optional[Union[List[AirbyteStateMessage], MutableMapping[str, Any]]] = None + ): shared_state, per_stream_states = self._extract_from_state_message(state, stream_instance_map) # We explicitly throw an error if we receive a GLOBAL state message that contains a shared_state because API sources are @@ -43,7 +45,7 @@ def __init__(self, stream_instance_map: Mapping[str, Stream], state: Union[List[ ) self.per_stream_states = per_stream_states - def get_stream_state(self, stream_name: str, namespace: Optional[str]) -> Mapping[str, Any]: + def get_stream_state(self, stream_name: str, namespace: Optional[str]) -> MutableMapping[str, Any]: """ Retrieves the state of a given stream based on its descriptor (name + namespace). :param stream_name: Name of the stream being fetched @@ -52,10 +54,10 @@ def get_stream_state(self, stream_name: str, namespace: Optional[str]) -> Mappin """ stream_state = self.per_stream_states.get(HashableStreamDescriptor(name=stream_name, namespace=namespace)) if stream_state: - return stream_state.dict() + return stream_state.dict() # type: ignore # mypy thinks dict() returns any, but it returns a dict return {} - def update_state_for_stream(self, stream_name: str, namespace: Optional[str], value: Mapping[str, Any]): + def update_state_for_stream(self, stream_name: str, namespace: Optional[str], value: Mapping[str, Any]) -> None: """ Overwrites the state blob of a specific stream based on the provided stream name and optional namespace :param stream_name: The name of the stream whose state is being updated @@ -95,7 +97,7 @@ def create_state_message(self, stream_name: str, namespace: Optional[str], send_ @classmethod def _extract_from_state_message( - cls, state: Union[List[AirbyteStateMessage], MutableMapping[str, Any]], stream_instance_map: Mapping[str, Stream] + cls, state: Optional[Union[List[AirbyteStateMessage], MutableMapping[str, Any]]], stream_instance_map: Mapping[str, Stream] ) -> Tuple[Optional[AirbyteStateBlob], MutableMapping[HashableStreamDescriptor, Optional[AirbyteStateBlob]]]: """ Takes an incoming list of state messages or the legacy state format and extracts state attributes according to type @@ -113,17 +115,17 @@ def _extract_from_state_message( # Incoming pure legacy object format if is_legacy: - streams = cls._create_descriptor_to_stream_state_mapping(state, stream_instance_map) + streams = cls._create_descriptor_to_stream_state_mapping(state, stream_instance_map) # type: ignore # We verified state is a dict in _is_legacy_dict_state return None, streams # When processing incoming state in source.read_state(), legacy state gets deserialized into List[AirbyteStateMessage] # which can be translated into independent per-stream state values if is_migrated_legacy: - streams = cls._create_descriptor_to_stream_state_mapping(state[0].data, stream_instance_map) + streams = cls._create_descriptor_to_stream_state_mapping(state[0].data, stream_instance_map) # type: ignore # We verified that state is a list in _is_migrated_legacy_state return None, streams if is_global: - global_state = state[0].global_ + global_state = state[0].global_ # type: ignore # We verified state is a list in _is_global_state shared_state = copy.deepcopy(global_state.shared_state, {}) streams = { HashableStreamDescriptor( @@ -139,7 +141,7 @@ def _extract_from_state_message( name=per_stream_state.stream.stream_descriptor.name, namespace=per_stream_state.stream.stream_descriptor.namespace ): per_stream_state.stream.stream_state for per_stream_state in state - if per_stream_state.type == AirbyteStateType.STREAM and hasattr(per_stream_state, "stream") + if per_stream_state.type == AirbyteStateType.STREAM and hasattr(per_stream_state, "stream") # type: ignore # state is always a list of AirbyteStateMessage if is_per_stream is True } return None, streams else: @@ -170,7 +172,7 @@ def _get_legacy_state(self) -> Mapping[str, Any]: return {descriptor.name: state.dict() if state else {} for descriptor, state in self.per_stream_states.items()} @staticmethod - def _is_legacy_dict_state(state: Union[List[AirbyteStateMessage], MutableMapping[str, Any]]): + def _is_legacy_dict_state(state: Union[List[AirbyteStateMessage], MutableMapping[str, Any]]) -> bool: return isinstance(state, dict) @staticmethod diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py index a7621693f0a2..d858677b6324 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py @@ -3,7 +3,7 @@ # from dataclasses import InitVar, dataclass, field -from typing import Any, List, Mapping, Optional, Tuple, Union +from typing import Any, List, Mapping, Optional, Union import pendulum from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator @@ -32,6 +32,7 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut scopes (Optional[List[str]]): The scopes to request token_expiry_date (Optional[Union[InterpolatedString, str]]): The access token expiration date token_expiry_date_format str: format of the datetime; provide it if expires_in is returned in datetime instead of seconds + token_expiry_is_time_of_expiration bool: set True it if expires_in is returned as time of expiration instead of the number seconds until expiration refresh_request_body (Optional[Mapping[str, Any]]): The request body to send in the refresh request grant_type: The grant_type to request for access_token. If set to refresh_token, the refresh_token parameter has to be provided message_repository (MessageRepository): the message repository used to emit logs on HTTP requests @@ -45,97 +46,88 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAut refresh_token: Optional[Union[InterpolatedString, str]] = None scopes: Optional[List[str]] = None token_expiry_date: Optional[Union[InterpolatedString, str]] = None - _token_expiry_date: pendulum.DateTime = field(init=False, repr=False, default=None) - token_expiry_date_format: str = None + _token_expiry_date: Optional[pendulum.DateTime] = field(init=False, repr=False, default=None) + token_expiry_date_format: Optional[str] = None + token_expiry_is_time_of_expiration: bool = False access_token_name: Union[InterpolatedString, str] = "access_token" expires_in_name: Union[InterpolatedString, str] = "expires_in" refresh_request_body: Optional[Mapping[str, Any]] = None grant_type: Union[InterpolatedString, str] = "refresh_token" message_repository: MessageRepository = NoopMessageRepository() - def __post_init__(self, parameters: Mapping[str, Any]): - self.token_refresh_endpoint = InterpolatedString.create(self.token_refresh_endpoint, parameters=parameters) - self.client_id = InterpolatedString.create(self.client_id, parameters=parameters) - self.client_secret = InterpolatedString.create(self.client_secret, parameters=parameters) + def __post_init__(self, parameters: Mapping[str, Any]) -> None: + super().__init__() + self._token_refresh_endpoint = InterpolatedString.create(self.token_refresh_endpoint, parameters=parameters) + self._client_id = InterpolatedString.create(self.client_id, parameters=parameters) + self._client_secret = InterpolatedString.create(self.client_secret, parameters=parameters) if self.refresh_token is not None: - self.refresh_token = InterpolatedString.create(self.refresh_token, parameters=parameters) + self._refresh_token = InterpolatedString.create(self.refresh_token, parameters=parameters) + else: + self._refresh_token = None self.access_token_name = InterpolatedString.create(self.access_token_name, parameters=parameters) self.expires_in_name = InterpolatedString.create(self.expires_in_name, parameters=parameters) self.grant_type = InterpolatedString.create(self.grant_type, parameters=parameters) self._refresh_request_body = InterpolatedMapping(self.refresh_request_body or {}, parameters=parameters) - self._token_expiry_date = ( - pendulum.parse(InterpolatedString.create(self.token_expiry_date, parameters=parameters).eval(self.config)) + self._token_expiry_date: pendulum.DateTime = ( + pendulum.parse(InterpolatedString.create(self.token_expiry_date, parameters=parameters).eval(self.config)) # type: ignore # pendulum.parse returns a datetime in this context if self.token_expiry_date - else pendulum.now().subtract(days=1) + else pendulum.now().subtract(days=1) # type: ignore # substract does not have type hints ) - self._access_token = None + self._access_token: Optional[str] = None # access_token is initialized by a setter - if self.get_grant_type() == "refresh_token" and self.refresh_token is None: + if self.get_grant_type() == "refresh_token" and self._refresh_token is None: raise ValueError("OAuthAuthenticator needs a refresh_token parameter if grant_type is set to `refresh_token`") def get_token_refresh_endpoint(self) -> str: - return self.token_refresh_endpoint.eval(self.config) + refresh_token: str = self._token_refresh_endpoint.eval(self.config) + if not refresh_token: + raise ValueError("OAuthAuthenticator was unable to evaluate token_refresh_endpoint parameter") + return refresh_token def get_client_id(self) -> str: - return self.client_id.eval(self.config) + client_id: str = self._client_id.eval(self.config) + if not client_id: + raise ValueError("OAuthAuthenticator was unable to evaluate client_id parameter") + return client_id def get_client_secret(self) -> str: - return self.client_secret.eval(self.config) + client_secret: str = self._client_secret.eval(self.config) + if not client_secret: + raise ValueError("OAuthAuthenticator was unable to evaluate client_secret parameter") + return client_secret def get_refresh_token(self) -> Optional[str]: - return None if self.refresh_token is None else self.refresh_token.eval(self.config) + return None if self._refresh_token is None else self._refresh_token.eval(self.config) - def get_scopes(self) -> [str]: - return self.scopes + def get_scopes(self) -> List[str]: + return self.scopes or [] - def get_access_token_name(self) -> InterpolatedString: - return self.access_token_name.eval(self.config) + def get_access_token_name(self) -> str: + return self.access_token_name.eval(self.config) # type: ignore # eval returns a string in this context - def get_expires_in_name(self) -> InterpolatedString: - return self.expires_in_name.eval(self.config) + def get_expires_in_name(self) -> str: + return self.expires_in_name.eval(self.config) # type: ignore # eval returns a string in this context - def get_grant_type(self) -> InterpolatedString: - return self.grant_type.eval(self.config) + def get_grant_type(self) -> str: + return self.grant_type.eval(self.config) # type: ignore # eval returns a string in this context def get_refresh_request_body(self) -> Mapping[str, Any]: - return self._refresh_request_body.eval(self.config) - - def refresh_access_token(self) -> Tuple[str, Any]: - """ - This overrides the parent class method because the parent class assumes the "expires_in" field is always an int representing - seconds till token expiry. - - However, this class provides the ability to determine the expiry date of an access token either by using (pseudocode): - * expiry_datetime = datetime.now() + seconds_till_access_token_expiry # in this option we have to calculate expiry timestamp, OR - * expiry_datetime = parse(response.body["expires_at"]) # in this option the API tells us exactly when access token expires - - :return: a tuple of (access_token, either token_lifespan_in_seconds or datetime_of_token_expiry) - - # TODO this is a hack and should be better encapsulated/enabled by the AbstractOAuthAuthenticator i.e: that class should have - a method which takes the HTTP response and returns a timestamp for when the access token will expire which subclasses - such as this one can override or just configure directly. - """ - response_json = self._get_refresh_access_token_response() - return response_json[self.get_access_token_name()], response_json[self.get_expires_in_name()] + return self._refresh_request_body.eval(self.config) # type: ignore # eval should return a Mapping in this context def get_token_expiry_date(self) -> pendulum.DateTime: - return self._token_expiry_date + return self._token_expiry_date # type: ignore # _token_expiry_date is a pendulum.DateTime. It is never None despite what mypy thinks - def set_token_expiry_date(self, value: Union[str, int]): - if self.token_expiry_date_format: - self._token_expiry_date = pendulum.from_format(value, self.token_expiry_date_format) - else: - try: - self._token_expiry_date = pendulum.now().add(seconds=int(float(value))) - except ValueError: - raise ValueError(f"Invalid token expiry value {value}; a number is required.") + def set_token_expiry_date(self, value: Union[str, int]) -> None: + self._token_expiry_date = self._parse_token_expiration_date(value) @property def access_token(self) -> str: + if self._access_token is None: + raise ValueError("access_token is not set") return self._access_token @access_token.setter - def access_token(self, value: str): + def access_token(self, value: str) -> None: self._access_token = value @property @@ -152,5 +144,5 @@ class DeclarativeSingleUseRefreshTokenOauth2Authenticator(SingleUseRefreshTokenO Declarative version of SingleUseRefreshTokenOauth2Authenticator which can be used in declarative connectors. """ - def __init__(self, *args, **kwargs): + def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py new file mode 100644 index 000000000000..6a9d6128706b --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/selective_authenticator.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import dataclass +from typing import Any, List, Mapping + +import dpath +from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator + + +@dataclass +class SelectiveAuthenticator(DeclarativeAuthenticator): + """Authenticator that selects concrete implementation based on specific config value.""" + + config: Mapping[str, Any] + authenticators: Mapping[str, DeclarativeAuthenticator] + authenticator_selection_path: List[str] + + # returns "DeclarativeAuthenticator", but must return a subtype of "SelectiveAuthenticator" + def __new__( # type: ignore[misc] + cls, + config: Mapping[str, Any], + authenticators: Mapping[str, DeclarativeAuthenticator], + authenticator_selection_path: List[str], + *arg: Any, + **kwargs: Any, + ) -> DeclarativeAuthenticator: + try: + selected_key = str(dpath.util.get(config, authenticator_selection_path)) + except KeyError as err: + raise ValueError("The path from `authenticator_selection_path` is not found in the config.") from err + + try: + return authenticators[selected_key] + except KeyError as err: + raise ValueError(f"The authenticator `{selected_key}` is not found.") from err diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 87c1ef911e18..c2ceb343d6f9 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -69,6 +69,10 @@ definitions: - "{{ record['updates'] }}" - "{{ record['MetaData']['LastUpdatedTime'] }}" - "{{ stream_partition['segment_id'] }}" + value_type: + title: Value Type + description: Type of the value. If not specified, the type will be inferred from the value. + "$ref": "#/definitions/ValueType" $parameters: type: object additionalProperties: true @@ -220,6 +224,49 @@ definitions: $parameters: type: object additionalProperties: true + SelectiveAuthenticator: + title: Selective Authenticator + description: Authenticator that selects concrete authenticator based on config property. + type: object + additionalProperties: true + required: + - type + - authenticators + - authenticator_selection_path + properties: + type: + type: string + enum: [SelectiveAuthenticator] + authenticator_selection_path: + title: Authenticator Selection Path + description: Path of the field in config with selected authenticator name + type: array + items: + type: string + examples: + - ["auth"] + - ["auth", "type"] + authenticators: + title: Authenticators + description: Authenticators to select from. + type: object + additionalProperties: + anyOf: + - "$ref": "#/definitions/ApiKeyAuthenticator" + - "$ref": "#/definitions/BasicHttpAuthenticator" + - "$ref": "#/definitions/BearerAuthenticator" + - "$ref": "#/definitions/CustomAuthenticator" + - "$ref": "#/definitions/OAuthAuthenticator" + - "$ref": "#/definitions/NoAuth" + - "$ref": "#/definitions/SessionTokenAuthenticator" + - "$ref": "#/definitions/LegacySessionTokenAuthenticator" + examples: + - authenticators: + token: "#/definitions/ApiKeyAuthenticator" + oauth: "#/definitions/OAuthAuthenticator" + $parameters: + type: object + additionalProperties: true CheckStream: title: Streams to Check description: Defines the streams to try reading when running a check operation. @@ -1145,6 +1192,7 @@ definitions: - "$ref": "#/definitions/NoAuth" - "$ref": "#/definitions/SessionTokenAuthenticator" - "$ref": "#/definitions/LegacySessionTokenAuthenticator" + - "$ref": "#/definitions/SelectiveAuthenticator" error_handler: title: Error Handler description: Error handler component that defines how to handle errors. @@ -1155,12 +1203,10 @@ definitions: http_method: title: HTTP Method description: The HTTP method used to fetch data from the source (can be GET or POST). - anyOf: - - type: string - - type: string - enum: - - GET - - POST + type: string + enum: + - GET + - POST default: GET examples: - GET @@ -1240,6 +1286,11 @@ definitions: - query: 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' - searchIn: "{{ ','.join(config.get('search_in', [])) }}" - sort_by[asc]: updated_at + use_cache: + title: Use Cache + description: Enables stream requests caching. This field is automatically set by the CDK. + type: boolean + default: false $parameters: type: object additionalProperties: true @@ -1626,6 +1677,11 @@ definitions: examples: - 100 - "{{ config['page_size'] }}" + inject_on_first_request: + title: Inject Offset + description: Using the `offset` with value `0` during the first request + type: boolean + default: false $parameters: type: object additionalProperties: true @@ -1654,6 +1710,11 @@ definitions: examples: - 0 - 1 + inject_on_first_request: + title: Inject Page Number + description: Using the `page number` with value defined by `start_from_page` during the first request + type: boolean + default: false $parameters: type: object additionalProperties: true @@ -1759,9 +1820,22 @@ definitions: title: Record Filter description: Responsible for filtering records to be emitted by the Source. "$ref": "#/definitions/RecordFilter" + schema_normalization: + "$ref": "#/definitions/SchemaNormalization" + default: None $parameters: type: object additionalProperties: true + SchemaNormalization: + title: Schema Normalization + description: Responsible for normalization according to the schema. + type: string + enum: + - None + - Default + examples: + - None + - Default RemoveFields: title: Remove Fields description: A transformation which removes fields from a record. The fields removed are designated using FieldPointers. During transformation, if a field or any of its parents does not exist in the record, no error is thrown. @@ -1977,6 +2051,15 @@ definitions: $parameters: type: object additionalProperties: true + ValueType: + title: Value Type + description: A schema type. + type: string + enum: + - string + - number + - integer + - boolean WaitTimeFromHeader: title: Wait Time Extracted From Response Header description: Extract wait time from a HTTP header in the response. @@ -2190,3 +2273,26 @@ interpolation: examples: - "{{ format_datetime(config['start_time'], '%Y-%m-%d') }}" - "{{ format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%S.%fZ') }}" + filters: + - title: Hash + description: Convert the specified value to a hashed string. + arguments: + hash_type: Valid hash type for converts ('md5' as default value). + salt: An additional value to further protect sensitive data. + return_type: str + examples: + - "{{ 'Test client_secret' | hash() }} -> '3032d57a12f76b61a820e47b9a5a0cbb'" + - "{{ 'Test client_secret' | hash('md5') }} -> '3032d57a12f76b61a820e47b9a5a0cbb'" + - "{{ 'Test client_secret' | hash('md5', salt='salt') }} -> '5011a0168579c2d94cbbe1c6ad14327c'" + - title: Base64 encoder + description: Convert the specified value to a string in the base64 format. + arguments: {} + return_type: str + examples: + - "{{ 'Test client_secret' | base64encode }} -> 'VGVzdCBjbGllbnRfc2VjcmV0'" + - title: Base64 decoder + description: Decodes the specified base64 format value into a common string. + arguments: {} + return_type: str + examples: + - "{{ 'ZmFrZSByZWZyZXNoX3Rva2VuIHZhbHVl' | base64decode }} -> 'fake refresh_token value'" diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py index 56d92dfc5639..f74ed377c4ab 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py @@ -101,7 +101,7 @@ def read_records( """ :param: stream_state We knowingly avoid using stream_state as we want cursors to manage their own state. """ - yield from self.retriever.read_records(stream_slice) + yield from self.retriever.read_records(self.get_json_schema(), stream_slice) def get_json_schema(self) -> Mapping[str, Any]: # type: ignore """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/http_selector.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/http_selector.py index 1b52cb03ba4f..0da7125868f1 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/http_selector.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/http_selector.py @@ -22,6 +22,7 @@ def select_records( self, response: requests.Response, stream_state: StreamState, + records_schema: Mapping[str, Any], stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> List[Record]: @@ -29,6 +30,7 @@ def select_records( Selects records from the response :param response: The response to select the records from :param stream_state: The stream state + :param records_schema: json schema of records to return :param stream_slice: The stream slice :param next_page_token: The paginator token :return: List of Records selected from the response diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py index d08068a952e0..33ad173d5484 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_selector.py @@ -9,8 +9,15 @@ from airbyte_cdk.sources.declarative.extractors.http_selector import HttpSelector from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor from airbyte_cdk.sources.declarative.extractors.record_filter import RecordFilter +from airbyte_cdk.sources.declarative.models import SchemaNormalization from airbyte_cdk.sources.declarative.transformations import RecordTransformation from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer + +SCHEMA_TRANSFORMER_TYPE_MAPPING = { + SchemaNormalization.None_: TransformConfig.NoTransform, + SchemaNormalization.Default: TransformConfig.DefaultSchemaNormalization, +} @dataclass @@ -21,6 +28,7 @@ class RecordSelector(HttpSelector): Attributes: extractor (RecordExtractor): The record extractor responsible for extracting records from a response + schema_normalization (TypeTransformer): The record normalizer responsible for casting record values to stream schema types record_filter (RecordFilter): The record filter responsible for filtering extracted records transformations (List[RecordTransformation]): The transformations to be done on the records """ @@ -28,6 +36,7 @@ class RecordSelector(HttpSelector): extractor: RecordExtractor config: Config parameters: InitVar[Mapping[str, Any]] + schema_normalization: TypeTransformer record_filter: Optional[RecordFilter] = None transformations: List[RecordTransformation] = field(default_factory=lambda: []) @@ -38,14 +47,31 @@ def select_records( self, response: requests.Response, stream_state: StreamState, + records_schema: Mapping[str, Any], stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> List[Record]: + """ + Selects records from the response + :param response: The response to select the records from + :param stream_state: The stream state + :param records_schema: json schema of records to return + :param stream_slice: The stream slice + :param next_page_token: The paginator token + :return: List of Records selected from the response + """ all_data = self.extractor.extract_records(response) filtered_data = self._filter(all_data, stream_state, stream_slice, next_page_token) self._transform(filtered_data, stream_state, stream_slice) + self._normalize_by_schema(filtered_data, schema=records_schema) return [Record(data, stream_slice) for data in filtered_data] + def _normalize_by_schema(self, records: List[Mapping[str, Any]], schema: Optional[Mapping[str, Any]]) -> List[Mapping[str, Any]]: + if schema: + # record has type Mapping[str, Any], but dict[str, Any] expected + return [self.schema_normalization.transform(record, schema) for record in records] # type: ignore + return records + def _filter( self, records: List[Mapping[str, Any]], @@ -67,4 +93,5 @@ def _transform( ) -> None: for record in records: for transformation in self.transformations: - transformation.transform(record, config=self.config, stream_state=stream_state, stream_slice=stream_slice) + # record has type Mapping[str, Any], but Record expected + transformation.transform(record, config=self.config, stream_state=stream_state, stream_slice=stream_slice) # type: ignore diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/filters.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/filters.py index 5009426cd805..eac515b03301 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/filters.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/filters.py @@ -1,11 +1,12 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - +import base64 import hashlib +from typing import Any, Optional -def hash(value, hash_type="md5", salt=None): +def hash(value: Any, hash_type: str = "md5", salt: Optional[str] = None) -> str: """ Implementation of a custom Jinja2 hash filter Hash type defaults to 'md5' if one is not specified. @@ -44,12 +45,50 @@ def hash(value, hash_type="md5", salt=None): hash_obj.update(str(value).encode("utf-8")) if salt: hash_obj.update(str(salt).encode("utf-8")) - computed_hash = hash_obj.hexdigest() + computed_hash: str = hash_obj.hexdigest() else: raise AttributeError("No hashing function named {hname}".format(hname=hash_type)) return computed_hash -_filters_list = [hash] +def base64encode(value: str) -> str: + """ + Implementation of a custom Jinja2 base64encode filter + + For example: + + OAuthAuthenticator: + $ref: "#/definitions/OAuthAuthenticator" + $parameters: + name: "client_id" + value: "{{ config['client_id'] | base64encode }}" + + :param value: value to be encoded in base64 + :return: base64 encoded string + """ + + return base64.b64encode(value.encode("utf-8")).decode() + + +def base64decode(value: str) -> str: + """ + Implementation of a custom Jinja2 base64decode filter + + For example: + + OAuthAuthenticator: + $ref: "#/definitions/OAuthAuthenticator" + $parameters: + name: "client_id" + value: "{{ config['client_id'] | base64decode }}" + + :param value: value to be decoded from base64 + :return: base64 decoded string + """ + + return base64.b64decode(value.encode("utf-8")).decode() + + +_filters_list = [hash, base64encode, base64decode] filters = {f.__name__: f for f in _filters_list} diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py index f2fadadffa82..72043e017497 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/interpolated_mapping.py @@ -34,13 +34,14 @@ def eval(self, config: Config, **additional_parameters): :param additional_parameters: Optional parameters used for interpolation :return: The interpolated string """ - interpolated_values = { - self._interpolation.eval(name, config, parameters=self._parameters, **additional_parameters): self._eval( - value, config, **additional_parameters - ) + valid_key_types = additional_parameters.pop("valid_key_types", (str,)) + valid_value_types = additional_parameters.pop("valid_value_types", None) + return { + self._interpolation.eval( + name, config, valid_types=valid_key_types, parameters=self._parameters, **additional_parameters + ): self._eval(value, config, valid_types=valid_value_types, **additional_parameters) for name, value in self.mapping.items() } - return interpolated_values def _eval(self, value, config, **kwargs): # The values in self._mapping can be of Any type diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/jinja.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/jinja.py index 5b3a9e67eb86..91d52c7579f4 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/jinja.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/jinja.py @@ -3,7 +3,7 @@ # import ast -from typing import Optional +from typing import Any, Optional, Tuple, Type from airbyte_cdk.sources.declarative.interpolation.filters import filters from airbyte_cdk.sources.declarative.interpolation.interpolation import Interpolation @@ -58,7 +58,14 @@ def __init__(self): for builtin in self.RESTRICTED_BUILTIN_FUNCTIONS: self._environment.globals.pop(builtin, None) - def eval(self, input_str: str, config: Config, default: Optional[str] = None, **additional_parameters): + def eval( + self, + input_str: str, + config: Config, + default: Optional[str] = None, + valid_types: Optional[Tuple[Type[Any]]] = None, + **additional_parameters, + ): context = {"config": config, **additional_parameters} for alias, equivalent in self.ALIASES.items(): @@ -74,20 +81,23 @@ def eval(self, input_str: str, config: Config, default: Optional[str] = None, ** if isinstance(input_str, str): result = self._eval(input_str, context) if result: - return self._literal_eval(result) + return self._literal_eval(result, valid_types) else: # If input is not a string, return it as is raise Exception(f"Expected a string. got {input_str}") except UndefinedError: pass # If result is empty or resulted in an undefined error, evaluate and return the default string - return self._literal_eval(self._eval(default, context)) + return self._literal_eval(self._eval(default, context), valid_types) - def _literal_eval(self, result): + def _literal_eval(self, result, valid_types: Optional[Tuple[Type[Any]]]): try: - return ast.literal_eval(result) + evaluated = ast.literal_eval(result) except (ValueError, SyntaxError): return result + if not valid_types or (valid_types and isinstance(evaluated, valid_types)): + return evaluated + return result def _eval(self, s: str, context): try: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py index ee014912c68a..50b0ace58326 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/manifest_declarative_source.py @@ -6,8 +6,9 @@ import logging import pkgutil import re +from copy import deepcopy from importlib import metadata -from typing import Any, Iterator, List, Mapping, MutableMapping, Union +from typing import Any, Dict, Iterator, List, Mapping, MutableMapping, Optional, Tuple, Union import yaml from airbyte_cdk.models import ( @@ -28,6 +29,7 @@ from airbyte_cdk.sources.declarative.types import ConnectionDefinition from airbyte_cdk.sources.message import MessageRepository from airbyte_cdk.sources.streams.core import Stream +from airbyte_cdk.sources.utils.slice_logger import AlwaysLogSliceLogger, DebugSliceLogger, SliceLogger from jsonschema.exceptions import ValidationError from jsonschema.validators import validate @@ -42,7 +44,7 @@ def __init__( source_config: ConnectionDefinition, debug: bool = False, emit_connector_builder_messages: bool = False, - component_factory: ModelToComponentFactory = None, + component_factory: Optional[ModelToComponentFactory] = None, ): """ :param source_config(Mapping[str, Any]): The manifest of low-code components that describe the source connector @@ -63,6 +65,7 @@ def __init__( self._emit_connector_builder_messages = emit_connector_builder_messages self._constructor = component_factory if component_factory else ModelToComponentFactory(emit_connector_builder_messages) self._message_repository = self._constructor.get_message_repository() + self._slice_logger: SliceLogger = AlwaysLogSliceLogger() if emit_connector_builder_messages else DebugSliceLogger() self._validate_source() @@ -89,19 +92,47 @@ def connection_checker(self) -> ConnectionChecker: def streams(self, config: Mapping[str, Any]) -> List[Stream]: self._emit_manifest_debug_message(extra_args={"source_name": self.name, "parsed_config": json.dumps(self._source_config)}) + stream_configs = self._stream_configs(self._source_config) source_streams = [ self._constructor.create_component( DeclarativeStreamModel, stream_config, config, emit_connector_builder_messages=self._emit_connector_builder_messages ) - for stream_config in self._stream_configs(self._source_config) + for stream_config in self._initialize_cache_for_parent_streams(deepcopy(stream_configs)) ] - for stream in source_streams: - # make sure the log level is always applied to the stream's logger - self._apply_log_level_to_stream_logger(self.logger, stream) return source_streams + @staticmethod + def _initialize_cache_for_parent_streams(stream_configs: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + parent_streams = set() + + def update_with_cache_parent_configs(parent_configs: list[dict[str, Any]]) -> None: + for parent_config in parent_configs: + parent_streams.add(parent_config["stream"]["name"]) + parent_config["stream"]["retriever"]["requester"]["use_cache"] = True + + for stream_config in stream_configs: + if stream_config.get("incremental_sync", {}).get("parent_stream"): + parent_streams.add(stream_config["incremental_sync"]["parent_stream"]["name"]) + stream_config["incremental_sync"]["parent_stream"]["retriever"]["requester"]["use_cache"] = True + + elif stream_config.get("retriever", {}).get("partition_router", {}): + partition_router = stream_config["retriever"]["partition_router"] + + if isinstance(partition_router, dict) and partition_router.get("parent_stream_configs"): + update_with_cache_parent_configs(partition_router["parent_stream_configs"]) + elif isinstance(partition_router, list): + for router in partition_router: + if router.get("parent_stream_configs"): + update_with_cache_parent_configs(router["parent_stream_configs"]) + + for stream_config in stream_configs: + if stream_config["name"] in parent_streams: + stream_config["retriever"]["requester"]["use_cache"] = True + + return stream_configs + def spec(self, logger: logging.Logger) -> ConnectorSpecification: """ Returns the connector specification (spec) as defined in the Airbyte Protocol. The spec is an object describing the possible @@ -130,28 +161,28 @@ def read( logger: logging.Logger, config: Mapping[str, Any], catalog: ConfiguredAirbyteCatalog, - state: Union[List[AirbyteStateMessage], MutableMapping[str, Any]] = None, + state: Optional[Union[List[AirbyteStateMessage], MutableMapping[str, Any]]] = None, ) -> Iterator[AirbyteMessage]: self._configure_logger_level(logger) yield from super().read(logger, config, catalog, state) - def should_log_slice_message(self, logger: logging.Logger): - return self._emit_connector_builder_messages or super().should_log_slice_message(logger) - - def _configure_logger_level(self, logger: logging.Logger): + def _configure_logger_level(self, logger: logging.Logger) -> None: """ Set the log level to logging.DEBUG if debug mode is enabled """ if self._debug: logger.setLevel(logging.DEBUG) - def _validate_source(self): + def _validate_source(self) -> None: """ Validates the connector manifest against the declarative component schema """ try: raw_component_schema = pkgutil.get_data("airbyte_cdk", "sources/declarative/declarative_component_schema.yaml") - declarative_component_schema = yaml.load(raw_component_schema, Loader=yaml.SafeLoader) + if raw_component_schema is not None: + declarative_component_schema = yaml.load(raw_component_schema, Loader=yaml.SafeLoader) + else: + raise RuntimeError("Failed to read manifest component json schema required for validation") except FileNotFoundError as e: raise FileNotFoundError(f"Failed to read manifest component json schema required for validation: {e}") @@ -167,6 +198,10 @@ def _validate_source(self): cdk_version = metadata.version("airbyte_cdk") cdk_major, cdk_minor, cdk_patch = self._get_version_parts(cdk_version, "airbyte-cdk") manifest_version = self._source_config.get("version") + if manifest_version is None: + raise RuntimeError( + "Manifest version is not defined in the manifest. This is unexpected since it should be a required field. Please contact support." + ) manifest_major, manifest_minor, manifest_patch = self._get_version_parts(manifest_version, "manifest") if cdk_major < manifest_major or (cdk_major == manifest_major and cdk_minor < manifest_minor): @@ -182,22 +217,22 @@ def _validate_source(self): ) @staticmethod - def _get_version_parts(version: str, version_type: str) -> (int, int, int): + def _get_version_parts(version: str, version_type: str) -> Tuple[int, int, int]: """ Takes a semantic version represented as a string and splits it into a tuple of its major, minor, and patch versions. """ version_parts = re.split(r"\.", version) if len(version_parts) != 3 or not all([part.isdigit() for part in version_parts]): raise ValidationError(f"The {version_type} version {version} specified is not a valid version format (ex. 1.2.3)") - return (int(part) for part in version_parts) + return tuple(int(part) for part in version_parts) # type: ignore # We already verified there were 3 parts and they are all digits - def _stream_configs(self, manifest: Mapping[str, Any]): + def _stream_configs(self, manifest: Mapping[str, Any]) -> List[Dict[str, Any]]: # This has a warning flag for static, but after we finish part 4 we'll replace manifest with self._source_config - stream_configs = manifest.get("streams", []) + stream_configs: List[Dict[str, Any]] = manifest.get("streams", []) for s in stream_configs: if "type" not in s: s["type"] = "DeclarativeStream" return stream_configs - def _emit_manifest_debug_message(self, extra_args: dict): + def _emit_manifest_debug_message(self, extra_args: dict[str, Any]) -> None: self.logger.debug("declarative source created from manifest", extra=extra_args) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 24e3a907b115..c53385bf36af 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -10,617 +10,607 @@ from typing_extensions import Literal -class AddedFieldDefinition(BaseModel): - type: Literal["AddedFieldDefinition"] - path: List[str] = Field( - ..., - description="List of strings defining the path where to add the value on the record.", - examples=[["segment_id"], ["metadata", "segment_id"]], - title="Path", - ) - value: str = Field( - ..., - description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", - examples=[ - "{{ record['updates'] }}", - "{{ record['MetaData']['LastUpdatedTime'] }}", - "{{ stream_partition['segment_id'] }}", - ], - title="Value", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - -class AddFields(BaseModel): - type: Literal["AddFields"] - fields: List[AddedFieldDefinition] = Field( - ..., - description="List of transformations (path and corresponding value) that will be added to the record.", - title="Fields", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class AuthFlowType(Enum): - oauth2_0 = "oauth2.0" - oauth1_0 = "oauth1.0" + oauth2_0 = 'oauth2.0' + oauth1_0 = 'oauth1.0' class BasicHttpAuthenticator(BaseModel): - type: Literal["BasicHttpAuthenticator"] + type: Literal['BasicHttpAuthenticator'] username: str = Field( ..., - description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", + description='The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.', examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], - title="Username", + title='Username', ) password: Optional[str] = Field( - "", - description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", - examples=["{{ config['password'] }}", ""], - title="Password", + '', + description='The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.', + examples=["{{ config['password'] }}", ''], + title='Password', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class BearerAuthenticator(BaseModel): - type: Literal["BearerAuthenticator"] + type: Literal['BearerAuthenticator'] api_token: str = Field( ..., - description="Token to inject as request header for authenticating with the API.", + description='Token to inject as request header for authenticating with the API.', examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], - title="Bearer Token", + title='Bearer Token', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class CheckStream(BaseModel): - type: Literal["CheckStream"] + type: Literal['CheckStream'] stream_names: List[str] = Field( ..., - description="Names of the streams to try reading from when running a check operation.", - examples=[["users"], ["users", "contacts"]], - title="Stream Names", + description='Names of the streams to try reading from when running a check operation.', + examples=[['users'], ['users', 'contacts']], + title='Stream Names', ) class ConstantBackoffStrategy(BaseModel): - type: Literal["ConstantBackoffStrategy"] + type: Literal['ConstantBackoffStrategy'] backoff_time_in_seconds: Union[float, str] = Field( ..., - description="Backoff time in seconds.", + description='Backoff time in seconds.', examples=[30, 30.5, "{{ config['backoff_time'] }}"], - title="Backoff Time", + title='Backoff Time', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class CustomAuthenticator(BaseModel): class Config: extra = Extra.allow - type: Literal["CustomAuthenticator"] + type: Literal['CustomAuthenticator'] class_name: str = Field( ..., - description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_..`.", - examples=["source_railz.components.ShortLivedTokenAuthenticator"], - title="Class Name", + description='Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_..`.', + examples=['source_railz.components.ShortLivedTokenAuthenticator'], + title='Class Name', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class CustomBackoffStrategy(BaseModel): class Config: extra = Extra.allow - type: Literal["CustomBackoffStrategy"] + type: Literal['CustomBackoffStrategy'] class_name: str = Field( ..., - description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_..`.", - examples=["source_railz.components.MyCustomBackoffStrategy"], - title="Class Name", + description='Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_..`.', + examples=['source_railz.components.MyCustomBackoffStrategy'], + title='Class Name', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class CustomErrorHandler(BaseModel): class Config: extra = Extra.allow - type: Literal["CustomErrorHandler"] + type: Literal['CustomErrorHandler'] class_name: str = Field( ..., - description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_..`.", - examples=["source_railz.components.MyCustomErrorHandler"], - title="Class Name", + description='Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_..`.', + examples=['source_railz.components.MyCustomErrorHandler'], + title='Class Name', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class CustomIncrementalSync(BaseModel): class Config: extra = Extra.allow - type: Literal["CustomIncrementalSync"] + type: Literal['CustomIncrementalSync'] class_name: str = Field( ..., - description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_..`.", - examples=["source_railz.components.MyCustomIncrementalSync"], - title="Class Name", + description='Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_..`.', + examples=['source_railz.components.MyCustomIncrementalSync'], + title='Class Name', ) cursor_field: str = Field( ..., - description="The location of the value on a record that will be used as a bookmark during sync.", + description='The location of the value on a record that will be used as a bookmark during sync.', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class CustomPaginationStrategy(BaseModel): class Config: extra = Extra.allow - type: Literal["CustomPaginationStrategy"] + type: Literal['CustomPaginationStrategy'] class_name: str = Field( ..., - description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_..`.", - examples=["source_railz.components.MyCustomPaginationStrategy"], - title="Class Name", + description='Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_..`.', + examples=['source_railz.components.MyCustomPaginationStrategy'], + title='Class Name', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class CustomRecordExtractor(BaseModel): class Config: extra = Extra.allow - type: Literal["CustomRecordExtractor"] + type: Literal['CustomRecordExtractor'] class_name: str = Field( ..., - description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_..`.", - examples=["source_railz.components.MyCustomRecordExtractor"], - title="Class Name", + description='Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_..`.', + examples=['source_railz.components.MyCustomRecordExtractor'], + title='Class Name', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class CustomRequester(BaseModel): class Config: extra = Extra.allow - type: Literal["CustomRequester"] + type: Literal['CustomRequester'] class_name: str = Field( ..., - description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_..`.", - examples=["source_railz.components.MyCustomRecordExtractor"], - title="Class Name", + description='Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_..`.', + examples=['source_railz.components.MyCustomRecordExtractor'], + title='Class Name', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class CustomRetriever(BaseModel): class Config: extra = Extra.allow - type: Literal["CustomRetriever"] + type: Literal['CustomRetriever'] class_name: str = Field( ..., - description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_..`.", - examples=["source_railz.components.MyCustomRetriever"], - title="Class Name", + description='Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_..`.', + examples=['source_railz.components.MyCustomRetriever'], + title='Class Name', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class CustomPartitionRouter(BaseModel): class Config: extra = Extra.allow - type: Literal["CustomPartitionRouter"] + type: Literal['CustomPartitionRouter'] class_name: str = Field( ..., - description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_..`.", - examples=["source_railz.components.MyCustomPartitionRouter"], - title="Class Name", + description='Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_..`.', + examples=['source_railz.components.MyCustomPartitionRouter'], + title='Class Name', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class CustomTransformation(BaseModel): class Config: extra = Extra.allow - type: Literal["CustomTransformation"] + type: Literal['CustomTransformation'] class_name: str = Field( ..., - description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_..`.", - examples=["source_railz.components.MyCustomTransformation"], - title="Class Name", + description='Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_..`.', + examples=['source_railz.components.MyCustomTransformation'], + title='Class Name', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class RefreshTokenUpdater(BaseModel): refresh_token_name: Optional[str] = Field( - "refresh_token", - description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", - examples=["refresh_token"], - title="Refresh Token Property Name", + 'refresh_token', + description='The name of the property which contains the updated refresh token in the response from the token refresh endpoint.', + examples=['refresh_token'], + title='Refresh Token Property Name', ) access_token_config_path: Optional[List[str]] = Field( - ["credentials", "access_token"], - description="Config path to the access token. Make sure the field actually exists in the config.", - examples=[["credentials", "access_token"], ["access_token"]], - title="Config Path To Access Token", + ['credentials', 'access_token'], + description='Config path to the access token. Make sure the field actually exists in the config.', + examples=[['credentials', 'access_token'], ['access_token']], + title='Config Path To Access Token', ) refresh_token_config_path: Optional[List[str]] = Field( - ["credentials", "refresh_token"], - description="Config path to the access token. Make sure the field actually exists in the config.", - examples=[["credentials", "refresh_token"], ["refresh_token"]], - title="Config Path To Refresh Token", + ['credentials', 'refresh_token'], + description='Config path to the access token. Make sure the field actually exists in the config.', + examples=[['credentials', 'refresh_token'], ['refresh_token']], + title='Config Path To Refresh Token', ) token_expiry_date_config_path: Optional[List[str]] = Field( - ["credentials", "token_expiry_date"], - description="Config path to the expiry date. Make sure actually exists in the config.", - examples=[["credentials", "token_expiry_date"]], - title="Config Path To Expiry Date", + ['credentials', 'token_expiry_date'], + description='Config path to the expiry date. Make sure actually exists in the config.', + examples=[['credentials', 'token_expiry_date']], + title='Config Path To Expiry Date', ) class OAuthAuthenticator(BaseModel): - type: Literal["OAuthAuthenticator"] + type: Literal['OAuthAuthenticator'] client_id: str = Field( ..., - description="The OAuth client ID. Fill it in the user inputs.", + description='The OAuth client ID. Fill it in the user inputs.', examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"], - title="Client ID", + title='Client ID', ) client_secret: str = Field( ..., - description="The OAuth client secret. Fill it in the user inputs.", + description='The OAuth client secret. Fill it in the user inputs.', examples=[ "{{ config['client_secret }}", "{{ config['credentials']['client_secret }}", ], - title="Client Secret", + title='Client Secret', ) refresh_token: Optional[str] = Field( None, - description="Credential artifact used to get a new access token.", + description='Credential artifact used to get a new access token.', examples=[ "{{ config['refresh_token'] }}", "{{ config['credentials]['refresh_token'] }}", ], - title="Refresh Token", + title='Refresh Token', ) token_refresh_endpoint: str = Field( ..., - description="The full URL to call to obtain a new access token.", - examples=["https://connect.squareup.com/oauth2/token"], - title="Token Refresh Endpoint", + description='The full URL to call to obtain a new access token.', + examples=['https://connect.squareup.com/oauth2/token'], + title='Token Refresh Endpoint', ) access_token_name: Optional[str] = Field( - "access_token", - description="The name of the property which contains the access token in the response from the token refresh endpoint.", - examples=["access_token"], - title="Access Token Property Name", + 'access_token', + description='The name of the property which contains the access token in the response from the token refresh endpoint.', + examples=['access_token'], + title='Access Token Property Name', ) expires_in_name: Optional[str] = Field( - "expires_in", - description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", - examples=["expires_in"], - title="Token Expiry Property Name", + 'expires_in', + description='The name of the property which contains the expiry date in the response from the token refresh endpoint.', + examples=['expires_in'], + title='Token Expiry Property Name', ) grant_type: Optional[str] = Field( - "refresh_token", - description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", - examples=["refresh_token", "client_credentials"], - title="Grant Type", + 'refresh_token', + description='Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.', + examples=['refresh_token', 'client_credentials'], + title='Grant Type', ) refresh_request_body: Optional[Dict[str, Any]] = Field( None, - description="Body of the request sent to get a new access token.", + description='Body of the request sent to get a new access token.', examples=[ { - "applicationId": "{{ config['application_id'] }}", - "applicationSecret": "{{ config['application_secret'] }}", - "token": "{{ config['token'] }}", + 'applicationId': "{{ config['application_id'] }}", + 'applicationSecret': "{{ config['application_secret'] }}", + 'token': "{{ config['token'] }}", } ], - title="Refresh Request Body", + title='Refresh Request Body', ) scopes: Optional[List[str]] = Field( None, - description="List of scopes that should be granted to the access token.", - examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], - title="Scopes", + description='List of scopes that should be granted to the access token.', + examples=[ + ['crm.list.read', 'crm.objects.contacts.read', 'crm.schema.contacts.read'] + ], + title='Scopes', ) token_expiry_date: Optional[str] = Field( None, - description="The access token expiry date.", - examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], - title="Token Expiry Date", + description='The access token expiry date.', + examples=['2023-04-06T07:12:10.421833+00:00', 1680842386], + title='Token Expiry Date', ) token_expiry_date_format: Optional[str] = Field( None, - description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", - examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], - title="Token Expiry Date Format", + description='The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.', + examples=['%Y-%m-%d %H:%M:%S.%f+00:00'], + title='Token Expiry Date Format', ) refresh_token_updater: Optional[RefreshTokenUpdater] = Field( None, - description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", - title="Token Updater", + description='When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.', + title='Token Updater', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class ExponentialBackoffStrategy(BaseModel): - type: Literal["ExponentialBackoffStrategy"] + type: Literal['ExponentialBackoffStrategy'] factor: Optional[Union[float, str]] = Field( 5, - description="Multiplicative constant applied on each retry.", - examples=[5, 5.5, "10"], - title="Factor", + description='Multiplicative constant applied on each retry.', + examples=[5, 5.5, '10'], + title='Factor', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class SessionTokenRequestBearerAuthenticator(BaseModel): - type: Literal["Bearer"] + type: Literal['Bearer'] -class HttpMethodEnum(Enum): - GET = "GET" - POST = "POST" +class HttpMethod(Enum): + GET = 'GET' + POST = 'POST' class Action(Enum): - SUCCESS = "SUCCESS" - FAIL = "FAIL" - RETRY = "RETRY" - IGNORE = "IGNORE" + SUCCESS = 'SUCCESS' + FAIL = 'FAIL' + RETRY = 'RETRY' + IGNORE = 'IGNORE' class HttpResponseFilter(BaseModel): - type: Literal["HttpResponseFilter"] + type: Literal['HttpResponseFilter'] action: Action = Field( ..., - description="Action to execute if a response matches the filter.", - examples=["SUCCESS", "FAIL", "RETRY", "IGNORE"], - title="Action", + description='Action to execute if a response matches the filter.', + examples=['SUCCESS', 'FAIL', 'RETRY', 'IGNORE'], + title='Action', ) error_message: Optional[str] = Field( None, - description="Error Message to display if the response matches the filter.", - title="Error Message", + description='Error Message to display if the response matches the filter.', + title='Error Message', ) error_message_contains: Optional[str] = Field( None, - description="Match the response if its error message contains the substring.", - example=["This API operation is not enabled for this site"], - title="Error Message Substring", + description='Match the response if its error message contains the substring.', + example=['This API operation is not enabled for this site'], + title='Error Message Substring', ) http_codes: Optional[List[int]] = Field( None, - description="Match the response if its HTTP code is included in this list.", + description='Match the response if its HTTP code is included in this list.', examples=[[420, 429], [500]], - title="HTTP Codes", + title='HTTP Codes', ) predicate: Optional[str] = Field( None, - description="Match the response if the predicate evaluates to true.", + description='Match the response if the predicate evaluates to true.', examples=[ "{{ 'Too much requests' in response }}", "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", ], - title="Predicate", + title='Predicate', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class InlineSchemaLoader(BaseModel): - type: Literal["InlineSchemaLoader"] + type: Literal['InlineSchemaLoader'] schema_: Optional[Dict[str, Any]] = Field( None, - alias="schema", + alias='schema', description='Describes a streams\' schema. Refer to the Data Types documentation for more details on which types are valid.', - title="Schema", + title='Schema', ) class JsonFileSchemaLoader(BaseModel): - type: Literal["JsonFileSchemaLoader"] + type: Literal['JsonFileSchemaLoader'] file_path: Optional[str] = Field( None, description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", - example=["./schemas/users.json"], - title="File Path", + example=['./schemas/users.json'], + title='File Path', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class JsonDecoder(BaseModel): - type: Literal["JsonDecoder"] + type: Literal['JsonDecoder'] class MinMaxDatetime(BaseModel): - type: Literal["MinMaxDatetime"] + type: Literal['MinMaxDatetime'] datetime: str = Field( ..., - description="Datetime value.", - examples=["2021-01-01", "2021-01-01T00:00:00Z", "{{ config['start_time'] }}"], - title="Datetime", + description='Datetime value.', + examples=['2021-01-01', '2021-01-01T00:00:00Z', "{{ config['start_time'] }}"], + title='Datetime', ) datetime_format: Optional[str] = Field( - "", + '', description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', - examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], - title="Datetime Format", + examples=['%Y-%m-%dT%H:%M:%S.%f%z', '%Y-%m-%d', '%s'], + title='Datetime Format', ) max_datetime: Optional[str] = Field( None, - description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", - examples=["2021-01-01T00:00:00Z", "2021-01-01"], - title="Max Datetime", + description='Ceiling applied on the datetime value. Must be formatted with the datetime_format field.', + examples=['2021-01-01T00:00:00Z', '2021-01-01'], + title='Max Datetime', ) min_datetime: Optional[str] = Field( None, - description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", - examples=["2010-01-01T00:00:00Z", "2010-01-01"], - title="Min Datetime", + description='Floor applied on the datetime value. Must be formatted with the datetime_format field.', + examples=['2010-01-01T00:00:00Z', '2010-01-01'], + title='Min Datetime', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class NoAuth(BaseModel): - type: Literal["NoAuth"] - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + type: Literal['NoAuth'] + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class NoPagination(BaseModel): - type: Literal["NoPagination"] + type: Literal['NoPagination'] class OAuthConfigSpecification(BaseModel): class Config: extra = Extra.allow - oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( + oauth_user_input_from_connector_config_specification: Optional[ + Dict[str, Any] + ] = Field( None, description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", examples=[ - {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, + {'app_id': {'type': 'string', 'path_in_connector_config': ['app_id']}}, { - "app_id": { - "type": "string", - "path_in_connector_config": ["info", "app_id"], + 'app_id': { + 'type': 'string', + 'path_in_connector_config': ['info', 'app_id'], } }, ], - title="OAuth user input", + title='OAuth user input', ) complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( None, description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", examples=[ { - "refresh_token": { - "type": "string,", - "path_in_connector_config": ["credentials", "refresh_token"], + 'refresh_token': { + 'type': 'string,', + 'path_in_connector_config': ['credentials', 'refresh_token'], } } ], - title="OAuth output specification", + title='OAuth output specification', ) complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( None, - description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", - examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], - title="OAuth input specification", + description='OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }', + examples=[ + {'client_id': {'type': 'string'}, 'client_secret': {'type': 'string'}} + ], + title='OAuth input specification', ) complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( None, description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", examples=[ { - "client_id": { - "type": "string,", - "path_in_connector_config": ["credentials", "client_id"], + 'client_id': { + 'type': 'string,', + 'path_in_connector_config': ['credentials', 'client_id'], }, - "client_secret": { - "type": "string,", - "path_in_connector_config": ["credentials", "client_secret"], + 'client_secret': { + 'type': 'string,', + 'path_in_connector_config': ['credentials', 'client_secret'], }, } ], - title="OAuth server output specification", + title='OAuth server output specification', ) class OffsetIncrement(BaseModel): - type: Literal["OffsetIncrement"] + type: Literal['OffsetIncrement'] page_size: Optional[Union[int, str]] = Field( None, - description="The number of records to include in each pages.", + description='The number of records to include in each pages.', examples=[100, "{{ config['page_size'] }}"], - title="Limit", + title='Limit', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + inject_on_first_request: Optional[bool] = Field( + False, + description='Using the `offset` with value `0` during the first request', + title='Inject Offset', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class PageIncrement(BaseModel): - type: Literal["PageIncrement"] + type: Literal['PageIncrement'] page_size: Optional[int] = Field( None, - description="The number of records to include in each pages.", - examples=[100, "100"], - title="Page Size", + description='The number of records to include in each pages.', + examples=[100, '100'], + title='Page Size', ) start_from_page: Optional[int] = Field( 0, - description="Index of the first page to request.", + description='Index of the first page to request.', examples=[0, 1], - title="Start From Page", + title='Start From Page', + ) + inject_on_first_request: Optional[bool] = Field( + False, + description='Using the `page number` with value defined by `start_from_page` during the first request', + title='Inject Page Number', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class PrimaryKey(BaseModel): __root__: Union[str, List[str], List[List[str]]] = Field( ..., - description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", - examples=["id", ["code", "type"]], - title="Primary Key", + description='The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.', + examples=['id', ['code', 'type']], + title='Primary Key', ) class RecordFilter(BaseModel): - type: Literal["RecordFilter"] + type: Literal['RecordFilter'] condition: Optional[str] = Field( - "", - description="The predicate to filter a record. Records will be removed if evaluated to False.", + '', + description='The predicate to filter a record. Records will be removed if evaluated to False.', examples=[ "{{ record['created_at'] >= stream_interval['start_time'] }}", "{{ record.status in ['active', 'expired'] }}", ], ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class SchemaNormalization(Enum): + None_ = 'None' + Default = 'Default' class RemoveFields(BaseModel): - type: Literal["RemoveFields"] + type: Literal['RemoveFields'] field_pointers: List[List[str]] = Field( ..., - description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", - examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], - title="Field Paths", + description='Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.', + examples=[['tags'], [['content', 'html'], ['content', 'plain_text']]], + title='Field Paths', ) class RequestPath(BaseModel): - type: Literal["RequestPath"] + type: Literal['RequestPath'] class InjectInto(Enum): - request_parameter = "request_parameter" - header = "header" - body_data = "body_data" - body_json = "body_json" + request_parameter = 'request_parameter' + header = 'header' + body_data = 'body_data' + body_json = 'body_json' class RequestOption(BaseModel): - type: Literal["RequestOption"] + type: Literal['RequestOption'] field_name: str = Field( ..., - description="Configures which key should be used in the location that the descriptor is being injected into", - examples=["segment_id"], - title="Request Option", + description='Configures which key should be used in the location that the descriptor is being injected into', + examples=['segment_id'], + title='Request Option', ) inject_into: InjectInto = Field( ..., - description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", - examples=["request_parameter", "header", "body_data", "body_json"], - title="Inject Into", + description='Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.', + examples=['request_parameter', 'header', 'body_data', 'body_json'], + title='Inject Into', ) @@ -632,251 +622,296 @@ class Config: class LegacySessionTokenAuthenticator(BaseModel): - type: Literal["LegacySessionTokenAuthenticator"] + type: Literal['LegacySessionTokenAuthenticator'] header: str = Field( ..., - description="The name of the session token header that will be injected in the request", - examples=["X-Session"], - title="Session Request Header", + description='The name of the session token header that will be injected in the request', + examples=['X-Session'], + title='Session Request Header', ) login_url: str = Field( ..., - description="Path of the login URL (do not include the base URL)", - examples=["session"], - title="Login Path", + description='Path of the login URL (do not include the base URL)', + examples=['session'], + title='Login Path', ) session_token: Optional[str] = Field( None, - description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", + description='Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair', example=["{{ config['session_token'] }}"], - title="Session Token", + title='Session Token', ) session_token_response_key: str = Field( ..., - description="Name of the key of the session token to be extracted from the response", - examples=["id"], - title="Response Token Response Key", + description='Name of the key of the session token to be extracted from the response', + examples=['id'], + title='Response Token Response Key', ) username: Optional[str] = Field( None, - description="Username used to authenticate and obtain a session token", + description='Username used to authenticate and obtain a session token', examples=[" {{ config['username'] }}"], - title="Username", + title='Username', ) password: Optional[str] = Field( - "", - description="Password used to authenticate and obtain a session token", - examples=["{{ config['password'] }}", ""], - title="Password", + '', + description='Password used to authenticate and obtain a session token', + examples=["{{ config['password'] }}", ''], + title='Password', ) validate_session_url: str = Field( ..., - description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", - examples=["user/current"], - title="Validate Session Path", + description='Path of the URL to use to validate that the session token is valid (do not include the base URL)', + examples=['user/current'], + title='Validate Session Path', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class ValueType(Enum): + string = 'string' + number = 'number' + integer = 'integer' + boolean = 'boolean' class WaitTimeFromHeader(BaseModel): - type: Literal["WaitTimeFromHeader"] + type: Literal['WaitTimeFromHeader'] header: str = Field( ..., - description="The name of the response header defining how long to wait before retrying.", - examples=["Retry-After"], - title="Response Header Name", + description='The name of the response header defining how long to wait before retrying.', + examples=['Retry-After'], + title='Response Header Name', ) regex: Optional[str] = Field( None, - description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", - examples=["([-+]?\\d+)"], - title="Extraction Regex", + description='Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.', + examples=['([-+]?\\d+)'], + title='Extraction Regex', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class WaitUntilTimeFromHeader(BaseModel): - type: Literal["WaitUntilTimeFromHeader"] + type: Literal['WaitUntilTimeFromHeader'] header: str = Field( ..., - description="The name of the response header defining how long to wait before retrying.", - examples=["wait_time"], - title="Response Header", + description='The name of the response header defining how long to wait before retrying.', + examples=['wait_time'], + title='Response Header', ) min_wait: Optional[Union[float, str]] = Field( None, - description="Minimum time to wait before retrying.", - examples=[10, "60"], - title="Minimum Wait Time", + description='Minimum time to wait before retrying.', + examples=[10, '60'], + title='Minimum Wait Time', ) regex: Optional[str] = Field( None, - description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", - examples=["([-+]?\\d+)"], - title="Extraction Regex", + description='Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.', + examples=['([-+]?\\d+)'], + title='Extraction Regex', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class AddedFieldDefinition(BaseModel): + type: Literal['AddedFieldDefinition'] + path: List[str] = Field( + ..., + description='List of strings defining the path where to add the value on the record.', + examples=[['segment_id'], ['metadata', 'segment_id']], + title='Path', + ) + value: str = Field( + ..., + description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", + examples=[ + "{{ record['updates'] }}", + "{{ record['MetaData']['LastUpdatedTime'] }}", + "{{ stream_partition['segment_id'] }}", + ], + title='Value', + ) + value_type: Optional[ValueType] = Field( + None, + description='Type of the value. If not specified, the type will be inferred from the value.', + title='Value Type', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class AddFields(BaseModel): + type: Literal['AddFields'] + fields: List[AddedFieldDefinition] = Field( + ..., + description='List of transformations (path and corresponding value) that will be added to the record.', + title='Fields', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class ApiKeyAuthenticator(BaseModel): - type: Literal["ApiKeyAuthenticator"] + type: Literal['ApiKeyAuthenticator'] api_token: Optional[str] = Field( None, - description="The API key to inject in the request. Fill it in the user inputs.", + description='The API key to inject in the request. Fill it in the user inputs.', examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], - title="API Key", + title='API Key', ) header: Optional[str] = Field( None, - description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", - examples=["Authorization", "Api-Token", "X-Auth-Token"], - title="Header Name", + description='The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.', + examples=['Authorization', 'Api-Token', 'X-Auth-Token'], + title='Header Name', ) inject_into: Optional[RequestOption] = Field( None, - description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", + description='Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.', examples=[ - {"inject_into": "header", "field_name": "Authorization"}, - {"inject_into": "request_parameter", "field_name": "authKey"}, + {'inject_into': 'header', 'field_name': 'Authorization'}, + {'inject_into': 'request_parameter', 'field_name': 'authKey'}, ], - title="Inject API Key Into Outgoing HTTP Request", + title='Inject API Key Into Outgoing HTTP Request', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class AuthFlow(BaseModel): - auth_flow_type: Optional[AuthFlowType] = Field(None, description="The type of auth to use", title="Auth flow type") + auth_flow_type: Optional[AuthFlowType] = Field( + None, description='The type of auth to use', title='Auth flow type' + ) predicate_key: Optional[List[str]] = Field( None, - description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", - examples=[["credentials", "auth_type"]], - title="Predicate key", + description='JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.', + examples=[['credentials', 'auth_type']], + title='Predicate key', ) predicate_value: Optional[str] = Field( None, - description="Value of the predicate_key fields for the advanced auth to be applicable.", - examples=["Oauth"], - title="Predicate value", + description='Value of the predicate_key fields for the advanced auth to be applicable.', + examples=['Oauth'], + title='Predicate value', ) oauth_config_specification: Optional[OAuthConfigSpecification] = None class CursorPagination(BaseModel): - type: Literal["CursorPagination"] + type: Literal['CursorPagination'] cursor_value: str = Field( ..., - description="Value of the cursor defining the next page to fetch.", + description='Value of the cursor defining the next page to fetch.', examples=[ - "{{ headers.link.next.cursor }}", + '{{ headers.link.next.cursor }}', "{{ last_records[-1]['key'] }}", "{{ response['nextPage'] }}", ], - title="Cursor Value", + title='Cursor Value', ) page_size: Optional[int] = Field( None, - description="The number of records to include in each pages.", + description='The number of records to include in each pages.', examples=[100], - title="Page Size", + title='Page Size', ) stop_condition: Optional[str] = Field( None, - description="Template string evaluating when to stop paginating.", + description='Template string evaluating when to stop paginating.', examples=[ - "{{ response.data.has_more is false }}", + '{{ response.data.has_more is false }}', "{{ 'next' not in headers['link'] }}", ], - title="Stop Condition", + title='Stop Condition', ) decoder: Optional[JsonDecoder] = Field( None, - description="Component decoding the response so records can be extracted.", - title="Decoder", + description='Component decoding the response so records can be extracted.', + title='Decoder', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class DatetimeBasedCursor(BaseModel): - type: Literal["DatetimeBasedCursor"] + type: Literal['DatetimeBasedCursor'] cursor_field: str = Field( ..., - description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", - examples=["created_at", "{{ config['record_cursor'] }}"], - title="Cursor Field", + description='The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.', + examples=['created_at', "{{ config['record_cursor'] }}"], + title='Cursor Field', ) datetime_format: str = Field( ..., - description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", - examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms"], - title="Outgoing Datetime Format", + description='The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', + examples=['%Y-%m-%dT%H:%M:%S.%f%z', '%Y-%m-%d', '%s', '%ms'], + title='Outgoing Datetime Format', ) start_datetime: Union[str, MinMaxDatetime] = Field( ..., - description="The datetime that determines the earliest record that should be synced.", - examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], - title="Start Datetime", + description='The datetime that determines the earliest record that should be synced.', + examples=['2020-01-1T00:00:00Z', "{{ config['start_time'] }}"], + title='Start Datetime', ) cursor_datetime_formats: Optional[List[str]] = Field( None, - description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.", - title="Cursor Datetime Formats", + description='The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.', + title='Cursor Datetime Formats', ) cursor_granularity: Optional[str] = Field( None, - description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.", - examples=["PT1S"], - title="Cursor Granularity", + description='Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.', + examples=['PT1S'], + title='Cursor Granularity', ) end_datetime: Optional[Union[str, MinMaxDatetime]] = Field( None, - description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", - examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], - title="End Datetime", + description='The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.', + examples=['2021-01-1T00:00:00Z', '{{ now_utc() }}', '{{ day_delta(-1) }}'], + title='End Datetime', ) end_time_option: Optional[RequestOption] = Field( None, - description="Optionally configures how the end datetime will be sent in requests to the source API.", - title="Inject End Time Into Outgoing HTTP Request", + description='Optionally configures how the end datetime will be sent in requests to the source API.', + title='Inject End Time Into Outgoing HTTP Request', ) is_data_feed: Optional[bool] = Field( None, - description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", - title="Whether the target API is formatted as a data feed", + description='A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.', + title='Whether the target API is formatted as a data feed', ) lookback_window: Optional[str] = Field( None, - description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.", - examples=["P1D", "P{{ config['lookback_days'] }}D"], - title="Lookback Window", + description='Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.', + examples=['P1D', "P{{ config['lookback_days'] }}D"], + title='Lookback Window', ) partition_field_end: Optional[str] = Field( None, - description="Name of the partition start time field.", - examples=["ending_time"], - title="Partition Field End", + description='Name of the partition start time field.', + examples=['ending_time'], + title='Partition Field End', ) partition_field_start: Optional[str] = Field( None, - description="Name of the partition end time field.", - examples=["starting_time"], - title="Partition Field Start", + description='Name of the partition end time field.', + examples=['starting_time'], + title='Partition Field Start', ) start_time_option: Optional[RequestOption] = Field( None, - description="Optionally configures how the start datetime will be sent in requests to the source API.", - title="Inject Start Time Into Outgoing HTTP Request", + description='Optionally configures how the start datetime will be sent in requests to the source API.', + title='Inject Start Time Into Outgoing HTTP Request', ) step: Optional[str] = Field( None, - description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.", - examples=["P1W", "{{ config['step_increment'] }}"], - title="Step", + description='The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.', + examples=['P1W', "{{ config['step_increment'] }}"], + title='Step', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class DefaultErrorHandler(BaseModel): - type: Literal["DefaultErrorHandler"] + type: Literal['DefaultErrorHandler'] backoff_strategies: Optional[ List[ Union[ @@ -889,142 +924,145 @@ class DefaultErrorHandler(BaseModel): ] ] = Field( None, - description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", - title="Backoff Strategies", + description='List of backoff strategies to use to determine how long to wait before retrying a retryable request.', + title='Backoff Strategies', ) max_retries: Optional[int] = Field( 5, - description="The maximum number of time to retry a retryable request before giving up and failing.", + description='The maximum number of time to retry a retryable request before giving up and failing.', examples=[5, 0, 10], - title="Max Retry Count", + title='Max Retry Count', ) response_filters: Optional[List[HttpResponseFilter]] = Field( None, description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", - title="Response Filters", + title='Response Filters', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class DefaultPaginator(BaseModel): - type: Literal["DefaultPaginator"] - pagination_strategy: Union[CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement] = Field( + type: Literal['DefaultPaginator'] + pagination_strategy: Union[ + CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement + ] = Field( ..., - description="Strategy defining how records are paginated.", - title="Pagination Strategy", + description='Strategy defining how records are paginated.', + title='Pagination Strategy', ) decoder: Optional[JsonDecoder] = Field( None, - description="Component decoding the response so records can be extracted.", - title="Decoder", + description='Component decoding the response so records can be extracted.', + title='Decoder', ) page_size_option: Optional[RequestOption] = None page_token_option: Optional[Union[RequestOption, RequestPath]] = None - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class DpathExtractor(BaseModel): - type: Literal["DpathExtractor"] + type: Literal['DpathExtractor'] field_path: List[str] = Field( ..., description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', examples=[ - ["data"], - ["data", "records"], - ["data", "{{ parameters.name }}"], - ["data", "*", "record"], + ['data'], + ['data', 'records'], + ['data', '{{ parameters.name }}'], + ['data', '*', 'record'], ], - title="Field Path", + title='Field Path', ) decoder: Optional[JsonDecoder] = Field( None, - description="Component decoding the response so records can be extracted.", - title="Decoder", + description='Component decoding the response so records can be extracted.', + title='Decoder', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class SessionTokenRequestApiKeyAuthenticator(BaseModel): - type: Literal["ApiKey"] + type: Literal['ApiKey'] inject_into: RequestOption = Field( ..., - description="Configure how the API Key will be sent in requests to the source API.", + description='Configure how the API Key will be sent in requests to the source API.', examples=[ - {"inject_into": "header", "field_name": "Authorization"}, - {"inject_into": "request_parameter", "field_name": "authKey"}, + {'inject_into': 'header', 'field_name': 'Authorization'}, + {'inject_into': 'request_parameter', 'field_name': 'authKey'}, ], - title="Inject API Key Into Outgoing HTTP Request", + title='Inject API Key Into Outgoing HTTP Request', ) class ListPartitionRouter(BaseModel): - type: Literal["ListPartitionRouter"] + type: Literal['ListPartitionRouter'] cursor_field: str = Field( ..., description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', - examples=["section", "{{ config['section_key'] }}"], - title="Current Partition Value Identifier", + examples=['section', "{{ config['section_key'] }}"], + title='Current Partition Value Identifier', ) values: Union[str, List[str]] = Field( ..., - description="The list of attributes being iterated over and used as input for the requests made to the source API.", - examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], - title="Partition Values", + description='The list of attributes being iterated over and used as input for the requests made to the source API.', + examples=[['section_a', 'section_b', 'section_c'], "{{ config['sections'] }}"], + title='Partition Values', ) request_option: Optional[RequestOption] = Field( None, - description="A request option describing where the list value should be injected into and under what field name if applicable.", - title="Inject Partition Value Into Outgoing HTTP Request", + description='A request option describing where the list value should be injected into and under what field name if applicable.', + title='Inject Partition Value Into Outgoing HTTP Request', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class RecordSelector(BaseModel): - type: Literal["RecordSelector"] + type: Literal['RecordSelector'] extractor: Union[CustomRecordExtractor, DpathExtractor] record_filter: Optional[RecordFilter] = Field( None, - description="Responsible for filtering records to be emitted by the Source.", - title="Record Filter", + description='Responsible for filtering records to be emitted by the Source.', + title='Record Filter', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + schema_normalization: Optional[SchemaNormalization] = SchemaNormalization.None_ + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class Spec(BaseModel): - type: Literal["Spec"] + type: Literal['Spec'] connection_specification: Dict[str, Any] = Field( ..., - description="A connection specification describing how a the connector can be configured.", - title="Connection Specification", + description='A connection specification describing how a the connector can be configured.', + title='Connection Specification', ) documentation_url: Optional[str] = Field( None, description="URL of the connector's documentation page.", - examples=["https://docs.airbyte.com/integrations/sources/dremio"], - title="Documentation URL", + examples=['https://docs.airbyte.com/integrations/sources/dremio'], + title='Documentation URL', ) advanced_auth: Optional[AuthFlow] = Field( None, - description="Advanced specification for configuring the authentication flow.", - title="Advanced Auth", + description='Advanced specification for configuring the authentication flow.', + title='Advanced Auth', ) class CompositeErrorHandler(BaseModel): - type: Literal["CompositeErrorHandler"] + type: Literal['CompositeErrorHandler'] error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( ..., - description="List of error handlers to iterate on to determine how to handle a failed response.", - title="Error Handlers", + description='List of error handlers to iterate on to determine how to handle a failed response.', + title='Error Handlers', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class DeclarativeSource(BaseModel): class Config: extra = Extra.forbid - type: Literal["DeclarativeSource"] + type: Literal['DeclarativeSource'] check: CheckStream streams: List[DeclarativeStream] version: str @@ -1033,99 +1071,148 @@ class Config: spec: Optional[Spec] = None metadata: Optional[Dict[str, Any]] = Field( None, - description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", + description='For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.', ) +class SelectiveAuthenticator(BaseModel): + class Config: + extra = Extra.allow + + type: Literal['SelectiveAuthenticator'] + authenticator_selection_path: List[str] = Field( + ..., + description='Path of the field in config with selected authenticator name', + examples=[['auth'], ['auth', 'type']], + title='Authenticator Selection Path', + ) + authenticators: Dict[ + str, + Union[ + ApiKeyAuthenticator, + BasicHttpAuthenticator, + BearerAuthenticator, + CustomAuthenticator, + OAuthAuthenticator, + NoAuth, + SessionTokenAuthenticator, + LegacySessionTokenAuthenticator, + ], + ] = Field( + ..., + description='Authenticators to select from.', + examples=[ + { + 'authenticators': { + 'token': '#/definitions/ApiKeyAuthenticator', + 'oauth': '#/definitions/OAuthAuthenticator', + } + } + ], + title='Authenticators', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + class DeclarativeStream(BaseModel): class Config: extra = Extra.allow - type: Literal["DeclarativeStream"] + type: Literal['DeclarativeStream'] retriever: Union[CustomRetriever, SimpleRetriever] = Field( ..., - description="Component used to coordinate how records are extracted across stream slices and request pages.", - title="Retriever", + description='Component used to coordinate how records are extracted across stream slices and request pages.', + title='Retriever', ) - incremental_sync: Optional[Union[CustomIncrementalSync, DatetimeBasedCursor]] = Field( + incremental_sync: Optional[ + Union[CustomIncrementalSync, DatetimeBasedCursor] + ] = Field( None, - description="Component used to fetch data incrementally based on a time field in the data.", - title="Incremental Sync", + description='Component used to fetch data incrementally based on a time field in the data.', + title='Incremental Sync', + ) + name: Optional[str] = Field( + '', description='The stream name.', example=['Users'], title='Name' + ) + primary_key: Optional[PrimaryKey] = Field( + '', description='The primary key of the stream.', title='Primary Key' ) - name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") - primary_key: Optional[PrimaryKey] = Field("", description="The primary key of the stream.", title="Primary Key") schema_loader: Optional[Union[InlineSchemaLoader, JsonFileSchemaLoader]] = Field( None, - description="Component used to retrieve the schema for the current stream.", - title="Schema Loader", + description='Component used to retrieve the schema for the current stream.', + title='Schema Loader', ) - transformations: Optional[List[Union[AddFields, CustomTransformation, RemoveFields]]] = Field( + transformations: Optional[ + List[Union[AddFields, CustomTransformation, RemoveFields]] + ] = Field( None, - description="A list of transformations to be applied to each output record.", - title="Transformations", + description='A list of transformations to be applied to each output record.', + title='Transformations', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class SessionTokenAuthenticator(BaseModel): - type: Literal["SessionTokenAuthenticator"] + type: Literal['SessionTokenAuthenticator'] login_requester: HttpRequester = Field( ..., - description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", + description='Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.', examples=[ { - "type": "HttpRequester", - "url_base": "https://my_api.com", - "path": "/login", - "authenticator": { - "type": "BasicHttpAuthenticator", - "username": "{{ config.username }}", - "password": "{{ config.password }}", + 'type': 'HttpRequester', + 'url_base': 'https://my_api.com', + 'path': '/login', + 'authenticator': { + 'type': 'BasicHttpAuthenticator', + 'username': '{{ config.username }}', + 'password': '{{ config.password }}', }, } ], - title="Login Requester", + title='Login Requester', ) session_token_path: List[str] = Field( ..., - description="The path in the response body returned from the login requester to the session token.", - examples=[["access_token"], ["result", "token"]], - title="Session Token Path", + description='The path in the response body returned from the login requester to the session token.', + examples=[['access_token'], ['result', 'token']], + title='Session Token Path', ) expiration_duration: Optional[str] = Field( None, - description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.", - examples=["PT1H", "P1D"], - title="Expiration Duration", + description='The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.', + examples=['PT1H', 'P1D'], + title='Expiration Duration', ) - request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator] = Field( + request_authentication: Union[ + SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator + ] = Field( ..., - description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", - title="Data Request Authentication", + description='Authentication method to use for requests sent to the API, specifying how to inject the session token.', + title='Data Request Authentication', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class HttpRequester(BaseModel): - type: Literal["HttpRequester"] + type: Literal['HttpRequester'] url_base: str = Field( ..., - description="Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", + description='Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.', examples=[ - "https://connect.squareup.com/v2", + 'https://connect.squareup.com/v2', "{{ config['base_url'] or 'https://app.posthog.com'}}/api/", ], - title="API Base URL", + title='API Base URL', ) path: str = Field( ..., - description="Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", + description='Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.', examples=[ - "/products", + '/products', "/quotes/{{ stream_partition['id'] }}/quote_line_groups", "/trades/{{ config['symbol_id'] }}/history", ], - title="URL Path", + title='URL Path', ) authenticator: Optional[ Union[ @@ -1137,95 +1224,105 @@ class HttpRequester(BaseModel): NoAuth, SessionTokenAuthenticator, LegacySessionTokenAuthenticator, + SelectiveAuthenticator, ] ] = Field( None, - description="Authentication method to use for requests sent to the API.", - title="Authenticator", + description='Authentication method to use for requests sent to the API.', + title='Authenticator', ) - error_handler: Optional[Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler]] = Field( + error_handler: Optional[ + Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler] + ] = Field( None, - description="Error handler component that defines how to handle errors.", - title="Error Handler", + description='Error handler component that defines how to handle errors.', + title='Error Handler', ) - http_method: Optional[Union[str, HttpMethodEnum]] = Field( - "GET", - description="The HTTP method used to fetch data from the source (can be GET or POST).", - examples=["GET", "POST"], - title="HTTP Method", + http_method: Optional[HttpMethod] = Field( + HttpMethod.GET, + description='The HTTP method used to fetch data from the source (can be GET or POST).', + examples=['GET', 'POST'], + title='HTTP Method', ) request_body_data: Optional[Union[str, Dict[str, str]]] = Field( None, - description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", + description='Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.', examples=[ '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' ], - title="Request Body Payload (Non-JSON)", + title='Request Body Payload (Non-JSON)', ) request_body_json: Optional[Union[str, Dict[str, Any]]] = Field( None, - description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", + description='Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.', examples=[ - {"sort_order": "ASC", "sort_field": "CREATED_AT"}, - {"key": "{{ config['value'] }}"}, - {"sort": {"field": "updated_at", "order": "ascending"}}, + {'sort_order': 'ASC', 'sort_field': 'CREATED_AT'}, + {'key': "{{ config['value'] }}"}, + {'sort': {'field': 'updated_at', 'order': 'ascending'}}, ], - title="Request Body JSON Payload", + title='Request Body JSON Payload', ) request_headers: Optional[Union[str, Dict[str, str]]] = Field( None, - description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", - examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], - title="Request Headers", + description='Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.', + examples=[{'Output-Format': 'JSON'}, {'Version': "{{ config['version'] }}"}], + title='Request Headers', ) request_parameters: Optional[Union[str, Dict[str, str]]] = Field( None, - description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", + description='Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.', examples=[ - {"unit": "day"}, + {'unit': 'day'}, { - "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' + 'query': 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' }, - {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, - {"sort_by[asc]": "updated_at"}, + {'searchIn': "{{ ','.join(config.get('search_in', [])) }}"}, + {'sort_by[asc]': 'updated_at'}, ], - title="Query Parameters", + title='Query Parameters', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + use_cache: Optional[bool] = Field( + False, + description='Enables stream requests caching. This field is automatically set by the CDK.', + title='Use Cache', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class ParentStreamConfig(BaseModel): - type: Literal["ParentStreamConfig"] + type: Literal['ParentStreamConfig'] parent_key: str = Field( ..., - description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", - examples=["id", "{{ config['parent_record_id'] }}"], - title="Parent Key", + description='The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.', + examples=['id', "{{ config['parent_record_id'] }}"], + title='Parent Key', + ) + stream: DeclarativeStream = Field( + ..., description='Reference to the parent stream.', title='Parent Stream' ) - stream: DeclarativeStream = Field(..., description="Reference to the parent stream.", title="Parent Stream") partition_field: str = Field( ..., - description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", - examples=["parent_id", "{{ config['parent_partition_field'] }}"], - title="Current Parent Key Value Identifier", + description='While iterating over parent records during a sync, the parent_key value can be referenced by using this field.', + examples=['parent_id', "{{ config['parent_partition_field'] }}"], + title='Current Parent Key Value Identifier', ) request_option: Optional[RequestOption] = Field( None, - description="A request option describing where the parent key value should be injected into and under what field name if applicable.", - title="Request Option", + description='A request option describing where the parent key value should be injected into and under what field name if applicable.', + title='Request Option', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class SimpleRetriever(BaseModel): - type: Literal["SimpleRetriever"] + type: Literal['SimpleRetriever'] record_selector: RecordSelector = Field( ..., - description="Component that describes how to extract records from a HTTP response.", + description='Component that describes how to extract records from a HTTP response.', ) requester: Union[CustomRequester, HttpRequester] = Field( ..., - description="Requester component that describes how to prepare HTTP requests to send to the source API.", + description='Requester component that describes how to prepare HTTP requests to send to the source API.', ) paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( None, @@ -1236,28 +1333,33 @@ class SimpleRetriever(BaseModel): CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter, - List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]], + List[ + Union[ + CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter + ] + ], ] ] = Field( [], - description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", - title="Partition Router", + description='PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.', + title='Partition Router', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') class SubstreamPartitionRouter(BaseModel): - type: Literal["SubstreamPartitionRouter"] + type: Literal['SubstreamPartitionRouter'] parent_stream_configs: List[ParentStreamConfig] = Field( ..., - description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", - title="Parent Stream Configs", + description='Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.', + title='Parent Stream Configs', ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') CompositeErrorHandler.update_forward_refs() DeclarativeSource.update_forward_refs() +SelectiveAuthenticator.update_forward_refs() DeclarativeStream.update_forward_refs() SessionTokenAuthenticator.update_forward_refs() SimpleRetriever.update_forward_refs() diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py index 90f7335de004..11fc12b2c3f4 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/class_types_registry.py @@ -2,7 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import Mapping, Type +from typing import Mapping from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth from airbyte_cdk.sources.declarative.auth.oauth import DeclarativeOauth2Authenticator @@ -10,7 +10,7 @@ ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, - SessionTokenAuthenticator, + LegacySessionTokenAuthenticator, ) from airbyte_cdk.sources.declarative.checks import CheckStream from airbyte_cdk.sources.declarative.datetime.min_max_datetime import MinMaxDatetime @@ -56,7 +56,7 @@ """ CLASS_TYPES_REGISTRY contains a mapping of developer-friendly string -> class to abstract the specific class referred to """ -CLASS_TYPES_REGISTRY: Mapping[str, Type] = { +CLASS_TYPES_REGISTRY: Mapping[str, type] = { "AddedFieldDefinition": AddedFieldDefinition, "AddFields": AddFields, "ApiKeyAuthenticator": ApiKeyAuthenticator, @@ -96,7 +96,7 @@ "SimpleRetriever": SimpleRetriever, "Spec": Spec, "SubstreamPartitionRouter": SubstreamPartitionRouter, - "SessionTokenAuthenticator": SessionTokenAuthenticator, + "SessionTokenAuthenticator": LegacySessionTokenAuthenticator, "WaitUntilTimeFromHeader": WaitUntilTimeFromHeaderBackoffStrategy, "WaitTimeFromHeader": WaitTimeFromHeaderBackoffStrategy, } diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 4025b752ed88..03e43e455947 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -11,8 +11,9 @@ from airbyte_cdk.models import Level from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator -from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth +from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator, NoAuth from airbyte_cdk.sources.declarative.auth.oauth import DeclarativeSingleUseRefreshTokenOauth2Authenticator +from airbyte_cdk.sources.declarative.auth.selective_authenticator import SelectiveAuthenticator from airbyte_cdk.sources.declarative.auth.token import ( ApiKeyAuthenticator, BasicHttpAuthenticator, @@ -25,6 +26,7 @@ from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream from airbyte_cdk.sources.declarative.decoders import JsonDecoder from airbyte_cdk.sources.declarative.extractors import DpathExtractor, RecordFilter, RecordSelector +from airbyte_cdk.sources.declarative.extractors.record_selector import SCHEMA_TRANSFORMER_TYPE_MAPPING from airbyte_cdk.sources.declarative.incremental import Cursor, CursorFactory, DatetimeBasedCursor, PerPartitionCursor from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping @@ -76,10 +78,12 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import RemoveFields as RemoveFieldsModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import RequestOption as RequestOptionModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import RequestPath as RequestPathModel +from airbyte_cdk.sources.declarative.models.declarative_component_schema import SelectiveAuthenticator as SelectiveAuthenticatorModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import SessionTokenAuthenticator as SessionTokenAuthenticatorModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import SimpleRetriever as SimpleRetrieverModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import Spec as SpecModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import SubstreamPartitionRouter as SubstreamPartitionRouterModel +from airbyte_cdk.sources.declarative.models.declarative_component_schema import ValueType from airbyte_cdk.sources.declarative.models.declarative_component_schema import WaitTimeFromHeader as WaitTimeFromHeaderModel from airbyte_cdk.sources.declarative.models.declarative_component_schema import WaitUntilTimeFromHeader as WaitUntilTimeFromHeaderModel from airbyte_cdk.sources.declarative.partition_routers import ListPartitionRouter, SinglePartitionRouter, SubstreamPartitionRouter @@ -104,6 +108,7 @@ from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType from airbyte_cdk.sources.declarative.requesters.request_options import InterpolatedRequestOptionsProvider from airbyte_cdk.sources.declarative.requesters.request_path import RequestPath +from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod from airbyte_cdk.sources.declarative.retrievers import SimpleRetriever, SimpleRetrieverTestReadDecorator from airbyte_cdk.sources.declarative.schema import DefaultSchemaLoader, InlineSchemaLoader, JsonFileSchemaLoader from airbyte_cdk.sources.declarative.spec import Spec @@ -112,6 +117,7 @@ from airbyte_cdk.sources.declarative.transformations.add_fields import AddedFieldDefinition from airbyte_cdk.sources.declarative.types import Config from airbyte_cdk.sources.message import InMemoryMessageRepository, LogAppenderMessageRepositoryDecorator, MessageRepository +from airbyte_cdk.sources.utils.transform import TypeTransformer from isodate import parse_duration from pydantic import BaseModel @@ -186,6 +192,7 @@ def _init_mappings(self) -> None: RequestPathModel: self.create_request_path, RequestOptionModel: self.create_request_option, LegacySessionTokenAuthenticatorModel: self.create_legacy_session_token_authenticator, + SelectiveAuthenticatorModel: self.create_selective_authenticator, SimpleRetrieverModel: self.create_simple_retriever, SpecModel: self.create_spec, SubstreamPartitionRouterModel: self.create_substream_partition_router, @@ -232,15 +239,36 @@ def _create_component_from_model(self, model: BaseModel, config: Config, **kwarg @staticmethod def create_added_field_definition(model: AddedFieldDefinitionModel, config: Config, **kwargs: Any) -> AddedFieldDefinition: interpolated_value = InterpolatedString.create(model.value, parameters=model.parameters or {}) - return AddedFieldDefinition(path=model.path, value=interpolated_value, parameters=model.parameters or {}) + return AddedFieldDefinition( + path=model.path, + value=interpolated_value, + value_type=ModelToComponentFactory._json_schema_type_name_to_type(model.value_type), + parameters=model.parameters or {}, + ) def create_add_fields(self, model: AddFieldsModel, config: Config, **kwargs: Any) -> AddFields: added_field_definitions = [ - self._create_component_from_model(model=added_field_definition_model, config=config) + self._create_component_from_model( + model=added_field_definition_model, + value_type=ModelToComponentFactory._json_schema_type_name_to_type(added_field_definition_model.value_type), + config=config, + ) for added_field_definition_model in model.fields ] return AddFields(fields=added_field_definitions, parameters=model.parameters or {}) + @staticmethod + def _json_schema_type_name_to_type(value_type: Optional[ValueType]) -> Optional[Type[Any]]: + if not value_type: + return None + names_to_types = { + ValueType.string: str, + ValueType.number: float, + ValueType.integer: int, + ValueType.boolean: bool, + } + return names_to_types[value_type] + @staticmethod def create_api_key_authenticator( model: ApiKeyAuthenticatorModel, config: Config, token_provider: Optional[TokenProvider] = None, **kwargs: Any @@ -289,11 +317,13 @@ def create_session_token_authenticator( ) if model.request_authentication.type == "Bearer": return ModelToComponentFactory.create_bearer_authenticator( - BearerAuthenticatorModel(type="BearerAuthenticator", api_token=""), config, token_provider=token_provider + BearerAuthenticatorModel(type="BearerAuthenticator", api_token=""), + config, + token_provider=token_provider, # type: ignore # $parameters defaults to None ) else: return ModelToComponentFactory.create_api_key_authenticator( - ApiKeyAuthenticatorModel(type="ApiKeyAuthenticator", api_token="", inject_into=model.request_authentication.inject_into), + ApiKeyAuthenticatorModel(type="ApiKeyAuthenticator", api_token="", inject_into=model.request_authentication.inject_into), # type: ignore # $parameters and headers default to None config=config, token_provider=token_provider, ) @@ -683,9 +713,10 @@ def create_http_requester(self, model: HttpRequesterModel, config: Config, *, na parameters=model.parameters or {}, ) - model_http_method = ( - model.http_method if isinstance(model.http_method, str) else model.http_method.value if model.http_method is not None else "GET" - ) + assert model.use_cache is not None # for mypy + assert model.http_method is not None # for mypy + + assert model.use_cache is not None # for mypy return HttpRequester( name=name, @@ -693,12 +724,13 @@ def create_http_requester(self, model: HttpRequesterModel, config: Config, *, na path=model.path, authenticator=authenticator, error_handler=error_handler, - http_method=model_http_method, + http_method=HttpMethod[model.http_method.value], request_options_provider=request_options_provider, config=config, disable_retries=self._disable_retries, parameters=model.parameters or {}, message_repository=self._message_repository, + use_cache=model.use_cache, ) @staticmethod @@ -791,7 +823,7 @@ def create_oauth_authenticator(self, model: OAuthAuthenticatorModel, config: Con token_expiry_date_format=model.token_expiry_date_format, message_repository=self._message_repository, ) - # ignore type error beause fixing it would have a lot of dependencies, revisit later + # ignore type error because fixing it would have a lot of dependencies, revisit later return DeclarativeOauth2Authenticator( # type: ignore access_token_name=model.access_token_name or "access_token", client_id=model.client_id, @@ -803,6 +835,7 @@ def create_oauth_authenticator(self, model: OAuthAuthenticatorModel, config: Con scopes=model.scopes, token_expiry_date=model.token_expiry_date, token_expiry_date_format=model.token_expiry_date_format, # type: ignore + token_expiry_is_time_of_expiration=bool(model.token_expiry_date_format), token_refresh_endpoint=model.token_refresh_endpoint, config=config, parameters=model.parameters or {}, @@ -811,11 +844,21 @@ def create_oauth_authenticator(self, model: OAuthAuthenticatorModel, config: Con @staticmethod def create_offset_increment(model: OffsetIncrementModel, config: Config, **kwargs: Any) -> OffsetIncrement: - return OffsetIncrement(page_size=model.page_size, config=config, parameters=model.parameters or {}) + return OffsetIncrement( + page_size=model.page_size, + config=config, + inject_on_first_request=model.inject_on_first_request or False, + parameters=model.parameters or {}, + ) @staticmethod def create_page_increment(model: PageIncrementModel, config: Config, **kwargs: Any) -> PageIncrement: - return PageIncrement(page_size=model.page_size, start_from_page=model.start_from_page or 0, parameters=model.parameters or {}) + return PageIncrement( + page_size=model.page_size, + start_from_page=model.start_from_page or 0, + inject_on_first_request=model.inject_on_first_request or False, + parameters=model.parameters or {}, + ) def create_parent_stream_config(self, model: ParentStreamConfigModel, config: Config, **kwargs: Any) -> ParentStreamConfig: declarative_stream = self._create_component_from_model(model.stream, config=config) @@ -843,16 +886,24 @@ def create_request_option(model: RequestOptionModel, config: Config, **kwargs: A return RequestOption(field_name=model.field_name, inject_into=inject_into, parameters={}) def create_record_selector( - self, model: RecordSelectorModel, config: Config, *, transformations: List[RecordTransformation], **kwargs: Any + self, + model: RecordSelectorModel, + config: Config, + *, + transformations: List[RecordTransformation], + **kwargs: Any, ) -> RecordSelector: + assert model.schema_normalization is not None # for mypy extractor = self._create_component_from_model(model=model.extractor, config=config) record_filter = self._create_component_from_model(model.record_filter, config=config) if model.record_filter else None + schema_normalization = TypeTransformer(SCHEMA_TRANSFORMER_TYPE_MAPPING[model.schema_normalization]) return RecordSelector( extractor=extractor, config=config, record_filter=record_filter, transformations=transformations, + schema_normalization=schema_normalization, parameters=model.parameters or {}, ) @@ -860,6 +911,16 @@ def create_record_selector( def create_remove_fields(model: RemoveFieldsModel, config: Config, **kwargs: Any) -> RemoveFields: return RemoveFields(field_pointers=model.field_pointers, parameters={}) + def create_selective_authenticator(self, model: SelectiveAuthenticatorModel, config: Config, **kwargs: Any) -> DeclarativeAuthenticator: + authenticators = {name: self._create_component_from_model(model=auth, config=config) for name, auth in model.authenticators.items()} + # SelectiveAuthenticator will return instance of DeclarativeAuthenticator or raise ValueError error + return SelectiveAuthenticator( # type: ignore[abstract] + config=config, + authenticators=authenticators, + authenticator_selection_path=model.authenticator_selection_path, + **kwargs, + ) + @staticmethod def create_legacy_session_token_authenticator( model: LegacySessionTokenAuthenticatorModel, config: Config, *, url_base: str, **kwargs: Any diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py index a13d8cd2b838..5c6232039d83 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py @@ -40,7 +40,7 @@ class CompositeErrorHandler(ErrorHandler): error_handlers: List[ErrorHandler] parameters: InitVar[Mapping[str, Any]] - def __post_init__(self, parameters: Mapping[str, Any]): + def __post_init__(self, parameters: Mapping[str, Any]) -> None: if not self.error_handlers: raise ValueError("CompositeErrorHandler expects at least 1 underlying error handler") @@ -48,8 +48,12 @@ def __post_init__(self, parameters: Mapping[str, Any]): def max_retries(self) -> Union[int, None]: return self.error_handlers[0].max_retries + @property + def max_time(self) -> Union[int, None]: + return max([error_handler.max_time or 0 for error_handler in self.error_handlers]) + def interpret_response(self, response: requests.Response) -> ResponseStatus: - should_retry = None + should_retry = ResponseStatus(ResponseAction.FAIL) for retrier in self.error_handlers: should_retry = retrier.interpret_response(response) if should_retry.action == ResponseAction.SUCCESS: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py index 4c982f94a09d..89e13d5c4859 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/default_error_handler.py @@ -95,10 +95,12 @@ class DefaultErrorHandler(ErrorHandler): config: Config response_filters: Optional[List[HttpResponseFilter]] = None max_retries: Optional[int] = 5 + max_time: int = 60 * 10 _max_retries: int = field(init=False, repr=False, default=5) + _max_time: int = field(init=False, repr=False, default=60 * 10) backoff_strategies: Optional[List[BackoffStrategy]] = None - def __post_init__(self, parameters: Mapping[str, Any]): + def __post_init__(self, parameters: Mapping[str, Any]) -> None: self.response_filters = self.response_filters or [] if not self.response_filters: @@ -114,12 +116,12 @@ def __post_init__(self, parameters: Mapping[str, Any]): self._last_request_to_attempt_count: MutableMapping[requests.PreparedRequest, int] = {} - @property + @property # type: ignore # overwrite the property to handle the case where max_retries is not provided in the constructor def max_retries(self) -> Union[int, None]: return self._max_retries @max_retries.setter - def max_retries(self, value: Union[int, None]): + def max_retries(self, value: int) -> None: # Covers the case where max_retries is not provided in the constructor, which causes the property object # to be set which we need to avoid doing if not isinstance(value, property): @@ -132,12 +134,13 @@ def interpret_response(self, response: requests.Response) -> ResponseStatus: self._last_request_to_attempt_count = {request: 1} else: self._last_request_to_attempt_count[request] += 1 - for response_filter in self.response_filters: - matched_status = response_filter.matches( - response=response, backoff_time=self._backoff_time(response, self._last_request_to_attempt_count[request]) - ) - if matched_status is not None: - return matched_status + if self.response_filters: + for response_filter in self.response_filters: + matched_status = response_filter.matches( + response=response, backoff_time=self._backoff_time(response, self._last_request_to_attempt_count[request]) + ) + if matched_status is not None: + return matched_status if response.ok: return response_status.SUCCESS @@ -146,8 +149,9 @@ def interpret_response(self, response: requests.Response) -> ResponseStatus: def _backoff_time(self, response: requests.Response, attempt_count: int) -> Optional[float]: backoff = None - for backoff_strategies in self.backoff_strategies: - backoff = backoff_strategies.backoff(response, attempt_count) - if backoff: - return backoff + if self.backoff_strategies: + for backoff_strategies in self.backoff_strategies: + backoff = backoff_strategies.backoff(response, attempt_count) + if backoff: + return backoff return backoff diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py index b451b214a8da..0a6e8a10166b 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/error_handler.py @@ -24,6 +24,14 @@ def max_retries(self) -> Union[int, None]: """ pass + @property + @abstractmethod + def max_time(self) -> Union[int, None]: + """ + Specifies maximum total waiting time (in seconds) for backoff policy. Return None for no limit. + """ + pass + @abstractmethod def interpret_response(self, response: requests.Response) -> ResponseStatus: """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py index 3887613ee652..20c18ec9ba6c 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py @@ -7,10 +7,12 @@ import urllib from dataclasses import InitVar, dataclass from functools import lru_cache +from pathlib import Path from typing import Any, Callable, Mapping, MutableMapping, Optional, Union from urllib.parse import urljoin import requests +import requests_cache from airbyte_cdk.models import Level from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator, NoAuth from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder @@ -24,10 +26,12 @@ ) from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod, Requester from airbyte_cdk.sources.declarative.types import Config, StreamSlice, StreamState +from airbyte_cdk.sources.http_config import MAX_CONNECTION_POOL_SIZE from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, RequestBodyException, UserDefinedBackoffException from airbyte_cdk.sources.streams.http.http import BODY_REQUEST_METHODS from airbyte_cdk.sources.streams.http.rate_limiting import default_backoff_handler, user_defined_backoff_handler +from airbyte_cdk.utils.constants import ENV_REQUEST_CACHE_PATH from airbyte_cdk.utils.mapping_helpers import combine_mappings from requests.auth import AuthBase @@ -46,6 +50,7 @@ class HttpRequester(Requester): authenticator (DeclarativeAuthenticator): Authenticator defining how to authenticate to the source error_handler (Optional[ErrorHandler]): Error handler defining how to detect and handle errors config (Config): The user-provided configuration as specified by the source's spec + use_cache (bool): Indicates that data should be cached for this stream """ name: str @@ -59,9 +64,11 @@ class HttpRequester(Requester): error_handler: Optional[ErrorHandler] = None disable_retries: bool = False message_repository: MessageRepository = NoopMessageRepository() + use_cache: bool = False _DEFAULT_MAX_RETRY = 5 _DEFAULT_RETRY_FACTOR = 5 + _DEFAULT_MAX_TIME = 60 * 10 def __post_init__(self, parameters: Mapping[str, Any]) -> None: self._url_base = InterpolatedString.create(self.url_base, parameters=parameters) @@ -77,7 +84,10 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None: self.error_handler = self.error_handler self._parameters = parameters self.decoder = JsonDecoder(parameters={}) - self._session = requests.Session() + self._session = self.request_cache() + self._session.mount( + "https://", requests.adapters.HTTPAdapter(pool_connections=MAX_CONNECTION_POOL_SIZE, pool_maxsize=MAX_CONNECTION_POOL_SIZE) + ) if isinstance(self._authenticator, AuthBase): self._session.auth = self._authenticator @@ -88,6 +98,33 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None: def __hash__(self) -> int: return hash(tuple(self.__dict__)) + @property + def cache_filename(self) -> str: + """ + Note that if the environment variable REQUEST_CACHE_PATH is not set, the cache will be in-memory only. + """ + return f"{self.name}.sqlite" + + def request_cache(self) -> requests.Session: + if self.use_cache: + cache_dir = os.getenv(ENV_REQUEST_CACHE_PATH) + # Use in-memory cache if cache_dir is not set + # This is a non-obvious interface, but it ensures we don't write sql files when running unit tests + if cache_dir: + sqlite_path = str(Path(cache_dir) / self.cache_filename) + else: + sqlite_path = "file::memory:?cache=shared" + return requests_cache.CachedSession(sqlite_path, backend="sqlite") # type: ignore # there are no typeshed stubs for requests_cache + else: + return requests.Session() + + def clear_cache(self) -> None: + """ + Clear cached requests for current session, can be called any time + """ + if isinstance(self._session, requests_cache.CachedSession): + self._session.cache.clear() # type: ignore # cache.clear is not typed + def get_authenticator(self) -> DeclarativeAuthenticator: return self._authenticator @@ -104,13 +141,15 @@ def get_path( def get_method(self) -> HttpMethod: return self._http_method - # use a tiny cache to limit the memory footprint. It doesn't have to be large because we mostly - # only care about the status of the last response received - @lru_cache(maxsize=10) def interpret_response_status(self, response: requests.Response) -> ResponseStatus: - # Cache the result because the HttpStream first checks if we should retry before looking at the backoff time if self.error_handler is None: raise ValueError("Cannot interpret response status without an error handler") + + # Change CachedRequest to PreparedRequest for response + request = response.request + if isinstance(request, requests_cache.CachedRequest): + response.request = request.prepare() + return self.error_handler.interpret_response(response) def get_request_params( @@ -170,6 +209,15 @@ def max_retries(self) -> Union[int, None]: return self._DEFAULT_MAX_RETRY return self.error_handler.max_retries + @property + def max_time(self) -> Union[int, None]: + """ + Override if needed. Specifies maximum total waiting time (in seconds) for backoff policy. Return None for no limit. + """ + if self.error_handler is None: + return self._DEFAULT_MAX_TIME + return self.error_handler.max_time + @property def logger(self) -> logging.Logger: return logging.getLogger(f"airbyte.HttpRequester.{self.name}") @@ -186,7 +234,16 @@ def _should_retry(self, response: requests.Response) -> bool: """ if self.error_handler is None: return response.status_code == 429 or 500 <= response.status_code < 600 - return bool(self.interpret_response_status(response).action == ResponseAction.RETRY) + + if self.use_cache: + interpret_response_status = self.interpret_response_status + else: + # Use a tiny cache to limit the memory footprint. It doesn't have to be large because we mostly + # only care about the status of the last response received + # Cache the result because the HttpStream first checks if we should retry before looking at the backoff time + interpret_response_status = lru_cache(maxsize=10)(self.interpret_response_status) + + return bool(interpret_response_status(response).action == ResponseAction.RETRY) def _backoff_time(self, response: requests.Response) -> Optional[float]: """ @@ -277,6 +334,11 @@ def _request_params( ) if isinstance(options, str): raise ValueError("Request params cannot be a string") + + for k, v in options.items(): + if isinstance(v, (list, dict)): + raise ValueError(f"Invalid value for `{k}` parameter. The values of request params cannot be an array or object.") + return options def _request_body_data( @@ -427,11 +489,19 @@ def _send_with_retry( Add this condition to avoid an endless loop if it hasn't been set explicitly (i.e. max_retries is not None). """ + max_time = self.max_time + """ + According to backoff max_time docstring: + max_time: The maximum total amount of time to try for before + giving up. Once expired, the exception will be allowed to + escape. If a callable is passed, it will be + evaluated at runtime and its return value used. + """ if max_tries is not None: max_tries = max(0, max_tries) + 1 - user_backoff_handler = user_defined_backoff_handler(max_tries=max_tries)(self._send) # type: ignore # we don't pass in kwargs to the backoff handler - backoff_handler = default_backoff_handler(max_tries=max_tries, factor=self._DEFAULT_RETRY_FACTOR) + user_backoff_handler = user_defined_backoff_handler(max_tries=max_tries, max_time=max_time)(self._send) # type: ignore # we don't pass in kwargs to the backoff handler + backoff_handler = default_backoff_handler(max_tries=max_tries, max_time=max_time, factor=self._DEFAULT_RETRY_FACTOR) # backoff handlers wrap _send, so it will always return a response return backoff_handler(user_backoff_handler)(request, log_formatter=log_formatter) # type: ignore @@ -516,7 +586,8 @@ def _try_get_error(value: Any) -> Any: if isinstance(value, str): return value elif isinstance(value, list): - return ", ".join(_try_get_error(v) for v in value) + error_list = [_try_get_error(v) for v in value] + return ", ".join(v for v in error_list if v is not None) elif isinstance(value, dict): new_value = ( value.get("message") @@ -525,6 +596,8 @@ def _try_get_error(value: Any) -> Any: or value.get("errors") or value.get("failures") or value.get("failure") + or value.get("details") + or value.get("detail") ) return _try_get_error(new_value) return None diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py index 5f4bf69dd306..e23b948859fc 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py @@ -2,8 +2,8 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from dataclasses import InitVar, dataclass, field -from typing import Any, List, Mapping, Optional, Union +from dataclasses import InitVar, dataclass +from typing import Any, List, Mapping, MutableMapping, Optional, Union import requests from airbyte_cdk.sources.declarative.decoders.decoder import Decoder @@ -91,15 +91,15 @@ class DefaultPaginator(Paginator): url_base: Union[InterpolatedString, str] parameters: InitVar[Mapping[str, Any]] decoder: Decoder = JsonDecoder(parameters={}) - _token: Optional[Any] = field(init=False, repr=False, default=None) page_size_option: Optional[RequestOption] = None page_token_option: Optional[Union[RequestPath, RequestOption]] = None - def __post_init__(self, parameters: Mapping[str, Any]): + def __post_init__(self, parameters: Mapping[str, Any]) -> None: if self.page_size_option and not self.pagination_strategy.get_page_size(): raise ValueError("page_size_option cannot be set if the pagination strategy does not have a page_size") if isinstance(self.url_base, str): self.url_base = InterpolatedString(string=self.url_base, parameters=parameters) + self._token = self.pagination_strategy.initial_token def next_page_token(self, response: requests.Response, last_records: List[Record]) -> Optional[Mapping[str, Any]]: self._token = self.pagination_strategy.next_page_token(response, last_records) @@ -108,10 +108,10 @@ def next_page_token(self, response: requests.Response, last_records: List[Record else: return None - def path(self): + def path(self) -> Optional[str]: if self._token and self.page_token_option and isinstance(self.page_token_option, RequestPath): # Replace url base to only return the path - return str(self._token).replace(self.url_base.eval(self.config), "") + return str(self._token).replace(self.url_base.eval(self.config), "") # type: ignore # url_base is casted to a InterpolatedString in __post_init__ else: return None @@ -121,7 +121,7 @@ def get_request_params( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, - ) -> Mapping[str, Any]: + ) -> MutableMapping[str, Any]: return self._get_request_options(RequestOptionType.request_parameter) def get_request_headers( @@ -151,16 +151,16 @@ def get_request_body_json( ) -> Mapping[str, Any]: return self._get_request_options(RequestOptionType.body_json) - def reset(self): + def reset(self) -> None: self.pagination_strategy.reset() - self._token = None + self._token = self.pagination_strategy.initial_token - def _get_request_options(self, option_type: RequestOptionType) -> Mapping[str, Any]: + def _get_request_options(self, option_type: RequestOptionType) -> MutableMapping[str, Any]: options = {} if ( self.page_token_option - and self._token + and self._token is not None and isinstance(self.page_token_option, RequestOption) and self.page_token_option.inject_into == option_type ): @@ -178,7 +178,7 @@ class PaginatorTestReadDecorator(Paginator): _PAGE_COUNT_BEFORE_FIRST_NEXT_CALL = 1 - def __init__(self, decorated, maximum_number_of_pages: int = 5): + def __init__(self, decorated: Paginator, maximum_number_of_pages: int = 5) -> None: if maximum_number_of_pages and maximum_number_of_pages < 1: raise ValueError(f"The maximum number of pages on a test read needs to be strictly positive. Got {maximum_number_of_pages}") self._maximum_number_of_pages = maximum_number_of_pages @@ -192,7 +192,7 @@ def next_page_token(self, response: requests.Response, last_records: List[Record self._page_count += 1 return self._decorated.next_page_token(response, last_records) - def path(self): + def path(self) -> Optional[str]: return self._decorated.path() def get_request_params( @@ -201,7 +201,7 @@ def get_request_params( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, - ) -> Mapping[str, Any]: + ) -> MutableMapping[str, Any]: return self._decorated.get_request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) def get_request_headers( @@ -219,7 +219,7 @@ def get_request_body_data( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, - ) -> Mapping[str, Any]: + ) -> Optional[Union[Mapping[str, Any], str]]: return self._decorated.get_request_body_data(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) def get_request_body_json( @@ -228,9 +228,9 @@ def get_request_body_json( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, - ) -> Mapping[str, Any]: + ) -> Optional[Mapping[str, Any]]: return self._decorated.get_request_body_json(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - def reset(self): + def reset(self) -> None: self._decorated.reset() self._page_count = self._PAGE_COUNT_BEFORE_FIRST_NEXT_CALL diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py index 1de2899a1201..bea36fc8e323 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/cursor_pagination_strategy.py @@ -40,6 +40,10 @@ def __post_init__(self, parameters: Mapping[str, Any]): if isinstance(self.stop_condition, str): self.stop_condition = InterpolatedBoolean(condition=self.stop_condition, parameters=parameters) + @property + def initial_token(self) -> Optional[Any]: + return None + def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: decoded_response = self.decoder.decode(response) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py index 261258201242..4a3224a4b427 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/offset_increment.py @@ -40,6 +40,7 @@ class OffsetIncrement(PaginationStrategy): page_size: Optional[Union[str, int]] parameters: InitVar[Mapping[str, Any]] decoder: Decoder = JsonDecoder(parameters={}) + inject_on_first_request: bool = False def __post_init__(self, parameters: Mapping[str, Any]): self._offset = 0 @@ -49,6 +50,12 @@ def __post_init__(self, parameters: Mapping[str, Any]): else: self._page_size = None + @property + def initial_token(self) -> Optional[Any]: + if self.inject_on_first_request: + return self._offset + return None + def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: decoded_response = self.decoder.decode(response) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py index 546592a9427a..64216e016eea 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/page_increment.py @@ -22,10 +22,17 @@ class PageIncrement(PaginationStrategy): page_size: Optional[int] parameters: InitVar[Mapping[str, Any]] start_from_page: int = 0 + inject_on_first_request: bool = False def __post_init__(self, parameters: Mapping[str, Any]): self._page = self.start_from_page + @property + def initial_token(self) -> Optional[Any]: + if self.inject_on_first_request: + return self._page + return None + def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: # Stop paginating when there are fewer records than the page size or the current page has no records if (self.page_size and len(last_records) < self.page_size) or len(last_records) == 0: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py index e8b00e54b5d3..3ebe49a059b5 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/pagination_strategy.py @@ -4,9 +4,10 @@ from abc import abstractmethod from dataclasses import dataclass -from typing import Any, List, Mapping, Optional +from typing import Any, List, Optional import requests +from airbyte_cdk.sources.declarative.types import Record @dataclass @@ -15,8 +16,15 @@ class PaginationStrategy: Defines how to get the next page token """ + @property @abstractmethod - def next_page_token(self, response: requests.Response, last_records: List[Mapping[str, Any]]) -> Optional[Any]: + def initial_token(self) -> Optional[Any]: + """ + Return the initial value of the token + """ + + @abstractmethod + def next_page_token(self, response: requests.Response, last_records: List[Record]) -> Optional[Any]: """ :param response: response to process :param last_records: records extracted from the response @@ -25,7 +33,7 @@ def next_page_token(self, response: requests.Response, last_records: List[Mappin pass @abstractmethod - def reset(self): + def reset(self) -> None: """ Reset the pagination's inner state """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py index 827171bcf705..8732e39ffef5 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/paginators/strategies/stop_condition.py @@ -42,8 +42,12 @@ def next_page_token(self, response: requests.Response, last_records: List[Record return None return self._delegate.next_page_token(response, last_records) - def reset(self): + def reset(self) -> None: self._delegate.reset() def get_page_size(self) -> Optional[int]: return self._delegate.get_page_size() + + @property + def initial_token(self) -> Optional[Any]: + return self._delegate.initial_token diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py index c901c88d4590..48ef8d5edf92 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_input_provider.py @@ -3,7 +3,7 @@ # from dataclasses import InitVar, dataclass, field -from typing import Any, Mapping, Optional, Union +from typing import Any, Mapping, Optional, Tuple, Type, Union from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString @@ -31,7 +31,12 @@ def __post_init__(self, parameters: Mapping[str, Any]): self._interpolator = InterpolatedMapping(self._request_inputs, parameters=parameters) def eval_request_inputs( - self, stream_state: StreamState, stream_slice: Optional[StreamSlice] = None, next_page_token: Mapping[str, Any] = None + self, + stream_state: StreamState, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Mapping[str, Any] = None, + valid_key_types: Tuple[Type[Any]] = None, + valid_value_types: Tuple[Type[Any]] = None, ) -> Mapping[str, Any]: """ Returns the request inputs to set on an outgoing HTTP request @@ -39,10 +44,14 @@ def eval_request_inputs( :param stream_state: The stream state :param stream_slice: The stream slice :param next_page_token: The pagination token + :param valid_key_types: A tuple of types that the interpolator should allow + :param valid_value_types: A tuple of types that the interpolator should allow :return: The request inputs to set on an outgoing HTTP request """ kwargs = {"stream_state": stream_state, "stream_slice": stream_slice, "next_page_token": next_page_token} - interpolated_value = self._interpolator.eval(self.config, **kwargs) + interpolated_value = self._interpolator.eval( + self.config, valid_key_types=valid_key_types, valid_value_types=valid_value_types, **kwargs + ) if isinstance(interpolated_value, dict): non_null_tokens = {k: v for k, v in interpolated_value.items() if v is not None} diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py index 050927a80198..0f1b6e3daf3e 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py @@ -14,6 +14,7 @@ from airbyte_cdk.sources.declarative.types import Config, StreamSlice, StreamState RequestInput = Union[str, Mapping[str, str]] +ValidRequestTypes = (str, list) @dataclass @@ -69,7 +70,9 @@ def get_request_params( stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> MutableMapping[str, Any]: - interpolated_value = self._parameter_interpolator.eval_request_inputs(stream_state, stream_slice, next_page_token) + interpolated_value = self._parameter_interpolator.eval_request_inputs( + stream_state, stream_slice, next_page_token, valid_key_types=(str,), valid_value_types=ValidRequestTypes + ) if isinstance(interpolated_value, dict): return interpolated_value return {} @@ -90,7 +93,13 @@ def get_request_body_data( stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Optional[Union[Mapping, str]]: - return self._body_data_interpolator.eval_request_inputs(stream_state, stream_slice, next_page_token) + return self._body_data_interpolator.eval_request_inputs( + stream_state, + stream_slice, + next_page_token, + valid_key_types=(str,), + valid_value_types=ValidRequestTypes, + ) def get_request_body_json( self, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py index 754bb04406c7..b07ffb3f6f08 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py @@ -53,7 +53,7 @@ def get_request_body_data( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, - ) -> Optional[Union[Mapping, str]]: + ) -> Optional[Union[Mapping[str, Any], str]]: """ Specifies how to populate the body of the request with a non-JSON payload. @@ -71,7 +71,7 @@ def get_request_body_json( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, - ) -> Optional[Mapping]: + ) -> Optional[Mapping[str, Any]]: """ Specifies how to populate the body of the request with a JSON payload. diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/retriever.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/retriever.py index d46dc9463487..bf4247a4f441 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/retriever.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/retriever.py @@ -4,7 +4,7 @@ from abc import abstractmethod from dataclasses import dataclass -from typing import Iterable, Optional +from typing import Any, Iterable, Mapping, Optional from airbyte_cdk.sources.declarative.types import StreamSlice, StreamState from airbyte_cdk.sources.streams.core import StreamData @@ -19,15 +19,14 @@ class Retriever: @abstractmethod def read_records( self, + records_schema: Mapping[str, Any], stream_slice: Optional[StreamSlice] = None, ) -> Iterable[StreamData]: """ Fetch a stream's records from an HTTP API source - :param sync_mode: Unused but currently necessary for integrating with HttpStream - :param cursor_field: Unused but currently necessary for integrating with HttpStream + :param records_schema: json schema to describe record :param stream_slice: The stream slice to read data for - :param stream_state: The initial stream state :return: The records read from the API source """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index f269e35ebeab..549a694fd059 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -3,6 +3,7 @@ # from dataclasses import InitVar, dataclass, field +from functools import partial from itertools import islice from typing import Any, Callable, Iterable, List, Mapping, Optional, Set, Tuple, Union @@ -215,6 +216,7 @@ def _parse_response( self, response: Optional[requests.Response], stream_state: StreamState, + records_schema: Mapping[str, Any], stream_slice: Optional[StreamSlice] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> Iterable[Record]: @@ -225,7 +227,11 @@ def _parse_response( self._last_response = response records = self.record_selector.select_records( - response=response, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token + response=response, + stream_state=stream_state, + records_schema=records_schema, + stream_slice=stream_slice, + next_page_token=next_page_token, ) self._records_from_last_response = records return records @@ -271,16 +277,15 @@ def _fetch_next_page( # This logic is similar to _read_pages in the HttpStream class. When making changes here, consider making changes there as well. def _read_pages( self, - records_generator_fn: Callable[[Optional[requests.Response], Mapping[str, Any], Mapping[str, Any]], Iterable[StreamData]], + records_generator_fn: Callable[[Optional[requests.Response]], Iterable[StreamData]], stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any], ) -> Iterable[StreamData]: - stream_state = stream_state or {} pagination_complete = False next_page_token = None while not pagination_complete: response = self._fetch_next_page(stream_state, stream_slice, next_page_token) - yield from records_generator_fn(response, stream_state, stream_slice) + yield from records_generator_fn(response) if not response: pagination_complete = True @@ -294,14 +299,28 @@ def _read_pages( def read_records( self, + records_schema: Mapping[str, Any], stream_slice: Optional[StreamSlice] = None, ) -> Iterable[StreamData]: + """ + Fetch a stream's records from an HTTP API source + + :param records_schema: json schema to describe record + :param stream_slice: The stream slice to read data for + :return: The records read from the API source + """ stream_slice = stream_slice or {} # None-check # Fixing paginator types has a long tail of dependencies self._paginator.reset() most_recent_record_from_slice = None - for stream_data in self._read_pages(self._parse_records, self.state, stream_slice): + record_generator = partial( + self._parse_records, + stream_state=self.state or {}, + stream_slice=stream_slice, + records_schema=records_schema, + ) + for stream_data in self._read_pages(record_generator, self.state, stream_slice): most_recent_record_from_slice = self._get_most_recent_record(most_recent_record_from_slice, stream_data, stream_slice) yield stream_data @@ -361,9 +380,15 @@ def _parse_records( self, response: Optional[requests.Response], stream_state: Mapping[str, Any], + records_schema: Mapping[str, Any], stream_slice: Optional[Mapping[str, Any]], ) -> Iterable[StreamData]: - yield from self._parse_response(response, stream_slice=stream_slice, stream_state=stream_state) + yield from self._parse_response( + response, + stream_slice=stream_slice, + stream_state=stream_state, + records_schema=records_schema, + ) def must_deduplicate_query_params(self) -> bool: return True diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/add_fields.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/add_fields.py index 7802e4edbc85..109f4fb8ca70 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/add_fields.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/add_fields.py @@ -3,7 +3,7 @@ # from dataclasses import InitVar, dataclass, field -from typing import Any, List, Mapping, Optional, Union +from typing import Any, List, Mapping, Optional, Type, Union import dpath.util from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString @@ -17,6 +17,7 @@ class AddedFieldDefinition: path: FieldPointer value: Union[InterpolatedString, str] + value_type: Optional[Type[Any]] parameters: InitVar[Mapping[str, Any]] @@ -26,6 +27,7 @@ class ParsedAddFieldDefinition: path: FieldPointer value: InterpolatedString + value_type: Optional[Type[Any]] parameters: InitVar[Mapping[str, Any]] @@ -85,10 +87,10 @@ class AddFields(RecordTransformation): parameters: InitVar[Mapping[str, Any]] _parsed_fields: List[ParsedAddFieldDefinition] = field(init=False, repr=False, default_factory=list) - def __post_init__(self, parameters: Mapping[str, Any]): + def __post_init__(self, parameters: Mapping[str, Any]) -> None: for add_field in self.fields: if len(add_field.path) < 1: - raise f"Expected a non-zero-length path for the AddFields transformation {add_field}" + raise ValueError(f"Expected a non-zero-length path for the AddFields transformation {add_field}") if not isinstance(add_field.value, InterpolatedString): if not isinstance(add_field.value, str): @@ -96,11 +98,16 @@ def __post_init__(self, parameters: Mapping[str, Any]): else: self._parsed_fields.append( ParsedAddFieldDefinition( - add_field.path, InterpolatedString.create(add_field.value, parameters=parameters), parameters=parameters + add_field.path, + InterpolatedString.create(add_field.value, parameters=parameters), + value_type=add_field.value_type, + parameters=parameters, ) ) else: - self._parsed_fields.append(ParsedAddFieldDefinition(add_field.path, add_field.value, parameters={})) + self._parsed_fields.append( + ParsedAddFieldDefinition(add_field.path, add_field.value, value_type=add_field.value_type, parameters={}) + ) def transform( self, @@ -109,12 +116,15 @@ def transform( stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, ) -> Record: + if config is None: + config = {} kwargs = {"record": record, "stream_state": stream_state, "stream_slice": stream_slice} for parsed_field in self._parsed_fields: - value = parsed_field.value.eval(config, **kwargs) + valid_types = (parsed_field.value_type,) if parsed_field.value_type else None + value = parsed_field.value.eval(config, valid_types=valid_types, **kwargs) dpath.util.new(record, parsed_field.path, value) return record - def __eq__(self, other): - return self.__dict__ == other.__dict__ + def __eq__(self, other: Any) -> bool: + return bool(self.__dict__ == other.__dict__) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/transformation.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/transformation.py index 560bf39e1b08..dd91864a2537 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/transformation.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/transformations/transformation.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from typing import Any, Mapping, Optional -from airbyte_cdk.sources.declarative.types import Config, StreamSlice, StreamState +from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState @dataclass @@ -18,7 +18,7 @@ class RecordTransformation: @abstractmethod def transform( self, - record: Mapping[str, Any], + record: Record, config: Optional[Config] = None, stream_state: Optional[StreamState] = None, stream_slice: Optional[StreamSlice] = None, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/README.md b/airbyte-cdk/python/airbyte_cdk/sources/file_based/README.md index 558f189fd85c..469260b0cbd0 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/README.md +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/README.md @@ -72,6 +72,21 @@ CSV is a format loosely described by [RFC 4180](https://www.rfc-editor.org/rfc/r Parquet is a file format defined by [Apache](https://parquet.apache.org/). Configuration options are: * `decimal_as_float`: Whether to convert decimal fields to floats. There is a loss of precision when converting decimals to floats, so this is not recommended. +### Document file types (PDF, DOCX, Markdown) + +For file share source connectors, the `unstructured` parser can be used to parse document file types. The textual content of the whole file will be parsed as a single record with a `content` field containing the text encoded as markdown. + +To use the unstructured parser, the libraries `poppler` and `tesseract` need to be installed on the system running the connector. For example, on Ubuntu, you can install them with the following command: +``` +apt-get install -y tesseract-ocr poppler-utils +``` + +on Mac, you can install these via brew: +``` +brew install poppler +brew install tesseract +``` + ## Schema Having a schema allows for the file-based CDK to take action when there is a discrepancy between a record and what are the expected types of the record fields. diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py index ff7390167068..19977b58a4fa 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/availability_strategy/default_file_based_availability_strategy.py @@ -4,11 +4,11 @@ import logging import traceback -from typing import TYPE_CHECKING, List, Optional, Tuple +from typing import TYPE_CHECKING, Optional, Tuple from airbyte_cdk.sources import Source from airbyte_cdk.sources.file_based.availability_strategy import AbstractFileBasedAvailabilityStrategy -from airbyte_cdk.sources.file_based.exceptions import CheckAvailabilityError, FileBasedSourceError +from airbyte_cdk.sources.file_based.exceptions import CheckAvailabilityError, CustomFileBasedException, FileBasedSourceError from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader from airbyte_cdk.sources.file_based.remote_file import RemoteFile from airbyte_cdk.sources.file_based.schema_helpers import conforms_to_schema @@ -43,8 +43,9 @@ def check_availability_and_parsability( Returns (True, None) if successful, otherwise (False, ). For the stream: + - Verify the parser config is valid per check_config method of the parser. - Verify that we can list files from the stream using the configured globs. - - Verify that we can read one file from the stream. + - Verify that we can read one file from the stream as long as the stream parser is not setting parser_max_n_files_for_parsability to 0. This method will also check that the files and their contents are consistent with the configured options, as follows: @@ -53,27 +54,42 @@ def check_availability_and_parsability( - If the user provided a schema in the config, check that a subset of records in one file conform to the schema via a call to stream.conforms_to_schema(schema). """ + parser = stream.get_parser() + config_check_result, config_check_error_message = parser.check_config(stream.config) + if config_check_result is False: + return False, config_check_error_message try: - files = self._check_list_files(stream) - self._check_parse_record(stream, files[0], logger) + file = self._check_list_files(stream) + if not parser.parser_max_n_files_for_parsability == 0: + self._check_parse_record(stream, file, logger) + else: + # If the parser is set to not check parsability, we still want to check that we can open the file. + handle = stream.stream_reader.open_file(file, parser.file_read_mode, None, logger) + handle.close() except CheckAvailabilityError: return False, "".join(traceback.format_exc()) return True, None - def _check_list_files(self, stream: "AbstractFileBasedStream") -> List[RemoteFile]: + def _check_list_files(self, stream: "AbstractFileBasedStream") -> RemoteFile: + """ + Check that we can list files from the stream. + + Returns the first file if successful, otherwise raises a CheckAvailabilityError. + """ try: - files = stream.list_files() + file = next(iter(stream.get_files())) + except StopIteration: + raise CheckAvailabilityError(FileBasedSourceError.EMPTY_STREAM, stream=stream.name) + except CustomFileBasedException as exc: + raise CheckAvailabilityError(str(exc), stream=stream.name) from exc except Exception as exc: raise CheckAvailabilityError(FileBasedSourceError.ERROR_LISTING_FILES, stream=stream.name) from exc - if not files: - raise CheckAvailabilityError(FileBasedSourceError.EMPTY_STREAM, stream=stream.name) - - return files + return file def _check_parse_record(self, stream: "AbstractFileBasedStream", file: RemoteFile, logger: logging.Logger) -> None: - parser = stream.get_parser(stream.config.file_type) + parser = stream.get_parser() try: record = next(iter(parser.parse_records(stream.config, file, self.stream_reader, logger, discovered_schema=None))) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py index 531502e66206..b667343add80 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/abstract_file_based_spec.py @@ -6,6 +6,7 @@ from abc import abstractmethod from typing import Any, Dict, List, Optional +import dpath.util from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig from airbyte_cdk.sources.utils import schema_helpers from pydantic import AnyUrl, BaseModel, Field @@ -46,12 +47,18 @@ def schema(cls, *args: Any, **kwargs: Any) -> Dict[str, Any]: Generates the mapping comprised of the config fields """ schema = super().schema(*args, **kwargs) - transformed_schema = copy.deepcopy(schema) + transformed_schema: Dict[str, Any] = copy.deepcopy(schema) schema_helpers.expand_refs(transformed_schema) cls.replace_enum_allOf_and_anyOf(transformed_schema) + cls.remove_discriminator(transformed_schema) return transformed_schema + @staticmethod + def remove_discriminator(schema: Dict[str, Any]) -> None: + """pydantic adds "discriminator" to the schema for oneOfs, which is not treated right by the platform as we inline all references""" + dpath.util.delete(schema, "properties/**/discriminator") + @staticmethod def replace_enum_allOf_and_anyOf(schema: Dict[str, Any]) -> Dict[str, Any]: """ @@ -66,9 +73,7 @@ def replace_enum_allOf_and_anyOf(schema: Dict[str, Any]) -> Dict[str, Any]: for format in objects_to_check["oneOf"]: for key in format["properties"]: object_property = format["properties"][key] - if "allOf" in object_property and "enum" in object_property["allOf"][0]: - object_property["enum"] = object_property["allOf"][0]["enum"] - object_property.pop("allOf") + AbstractFileBasedSpec.move_enum_to_root(object_property) properties_to_change = ["validation_policy"] for property_to_change in properties_to_change: @@ -76,7 +81,24 @@ def replace_enum_allOf_and_anyOf(schema: Dict[str, Any]) -> Dict[str, Any]: if "anyOf" in property_object: schema["properties"]["streams"]["items"]["properties"][property_to_change]["type"] = "object" schema["properties"]["streams"]["items"]["properties"][property_to_change]["oneOf"] = property_object.pop("anyOf") - if "allOf" in property_object and "enum" in property_object["allOf"][0]: - property_object["enum"] = property_object["allOf"][0]["enum"] - property_object.pop("allOf") + AbstractFileBasedSpec.move_enum_to_root(property_object) + + csv_format_schemas = list( + filter( + lambda format: format["properties"]["filetype"]["default"] == "csv", + schema["properties"]["streams"]["items"]["properties"]["format"]["oneOf"], + ) + ) + if len(csv_format_schemas) != 1: + raise ValueError(f"Expecting only one CSV format but got {csv_format_schemas}") + csv_format_schemas[0]["properties"]["header_definition"]["oneOf"] = csv_format_schemas[0]["properties"]["header_definition"].pop( + "anyOf", [] + ) + csv_format_schemas[0]["properties"]["header_definition"]["type"] = "object" return schema + + @staticmethod + def move_enum_to_root(object_property: Dict[str, Any]) -> None: + if "allOf" in object_property and "enum" in object_property["allOf"][0]: + object_property["enum"] = object_property["allOf"][0]["enum"] + object_property.pop("allOf") diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/avro_format.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/avro_format.py index ee7b955a325b..a5bef76f6176 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/avro_format.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/avro_format.py @@ -2,15 +2,20 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + +from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig from pydantic import BaseModel, Field -from typing_extensions import Literal class AvroFormat(BaseModel): - class Config: + class Config(OneOfOptionConfig): title = "Avro Format" + discriminator = "filetype" - filetype: Literal["avro"] = "avro" + filetype: str = Field( + "avro", + const=True, + ) double_as_string: bool = Field( title="Convert Double Fields to Strings", diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/csv_format.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/csv_format.py index 2d8106131c70..fab52aeefd28 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/csv_format.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/csv_format.py @@ -4,10 +4,10 @@ import codecs from enum import Enum -from typing import Optional, Set +from typing import Any, Dict, List, Optional, Set, Union -from pydantic import BaseModel, Field, validator -from typing_extensions import Literal +from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig +from pydantic import BaseModel, Field, ValidationError, root_validator, validator class InferenceType(Enum): @@ -15,15 +15,77 @@ class InferenceType(Enum): PRIMITIVE_TYPES_ONLY = "Primitive Types Only" +class CsvHeaderDefinitionType(Enum): + FROM_CSV = "From CSV" + AUTOGENERATED = "Autogenerated" + USER_PROVIDED = "User Provided" + + +class CsvHeaderFromCsv(BaseModel): + class Config(OneOfOptionConfig): + title = "From CSV" + discriminator = "header_definition_type" + + header_definition_type: str = Field( + CsvHeaderDefinitionType.FROM_CSV.value, + const=True, + ) + + def has_header_row(self) -> bool: + return True + + +class CsvHeaderAutogenerated(BaseModel): + class Config(OneOfOptionConfig): + title = "Autogenerated" + discriminator = "header_definition_type" + + header_definition_type: str = Field( + CsvHeaderDefinitionType.AUTOGENERATED.value, + const=True, + ) + + def has_header_row(self) -> bool: + return False + + +class CsvHeaderUserProvided(BaseModel): + class Config(OneOfOptionConfig): + title = "User Provided" + discriminator = "header_definition_type" + + header_definition_type: str = Field( + CsvHeaderDefinitionType.USER_PROVIDED.value, + const=True, + ) + column_names: List[str] = Field( + title="Column Names", + description="The column names that will be used while emitting the CSV records", + ) + + def has_header_row(self) -> bool: + return False + + @validator("column_names") + def validate_column_names(cls, v: List[str]) -> List[str]: + if not v: + raise ValueError("At least one column name needs to be provided when using user provided headers") + return v + + DEFAULT_TRUE_VALUES = ["y", "yes", "t", "true", "on", "1"] DEFAULT_FALSE_VALUES = ["n", "no", "f", "false", "off", "0"] class CsvFormat(BaseModel): - class Config: + class Config(OneOfOptionConfig): title = "CSV Format" + discriminator = "filetype" - filetype: Literal["csv"] = "csv" + filetype: str = Field( + "csv", + const=True, + ) delimiter: str = Field( title="Delimiter", description="The character delimiting individual cells in the CSV data. This may only be a 1-character string. For tab-delimited data enter '\\t'.", @@ -64,10 +126,10 @@ class Config: skip_rows_after_header: int = Field( title="Skip Rows After Header", default=0, description="The number of rows to skip after the header row." ) - autogenerate_column_names: bool = Field( - title="Autogenerate Column Names", - default=False, - description="Whether to autogenerate column names if column_names is empty. If true, column names will be of the form “f0”, “f1”… If false, column names will be read from the first CSV row after skip_rows_before_header.", + header_definition: Union[CsvHeaderFromCsv, CsvHeaderAutogenerated, CsvHeaderUserProvided] = Field( + title="CSV Header Definition", + default=CsvHeaderFromCsv(header_definition_type=CsvHeaderDefinitionType.FROM_CSV.value), + description="How headers will be defined. `User Provided` assumes the CSV does not have a header row and uses the headers provided and `Autogenerated` assumes the CSV does not have a header row and the CDK will generate headers using for `f{i}` where `i` is the index starting from 0. Else, the default behavior is to use the header from the CSV file. If a user wants to autogenerate or provide column names for a CSV having headers, they can skip rows.", ) true_values: Set[str] = Field( title="True Values", @@ -113,3 +175,15 @@ def validate_encoding(cls, v: str) -> str: except LookupError: raise ValueError(f"invalid encoding format: {v}") return v + + @root_validator + def validate_optional_args(cls, values: Dict[str, Any]) -> Dict[str, Any]: + definition_type = values.get("header_definition_type") + column_names = values.get("user_provided_column_names") + if definition_type == CsvHeaderDefinitionType.USER_PROVIDED and not column_names: + raise ValidationError("`user_provided_column_names` should be defined if the definition 'User Provided'.", model=CsvFormat) + if definition_type != CsvHeaderDefinitionType.USER_PROVIDED and column_names: + raise ValidationError( + "`user_provided_column_names` should not be defined if the definition is not 'User Provided'.", model=CsvFormat + ) + return values diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/file_based_stream_config.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/file_based_stream_config.py index 6f38ed4abf56..0b42838ea27c 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/file_based_stream_config.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/file_based_stream_config.py @@ -3,12 +3,13 @@ # from enum import Enum -from typing import Any, List, Mapping, Optional, Type, Union +from typing import Any, List, Mapping, Optional, Union from airbyte_cdk.sources.file_based.config.avro_format import AvroFormat from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat from airbyte_cdk.sources.file_based.config.jsonl_format import JsonlFormat from airbyte_cdk.sources.file_based.config.parquet_format import ParquetFormat +from airbyte_cdk.sources.file_based.config.unstructured_format import UnstructuredFormat from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError from airbyte_cdk.sources.file_based.schema_helpers import type_mapping_to_jsonschema from pydantic import BaseModel, Field, validator @@ -16,9 +17,6 @@ PrimaryKeyType = Optional[Union[str, List[str]]] -VALID_FILE_TYPES: Mapping[str, Type[BaseModel]] = {"avro": AvroFormat, "csv": CsvFormat, "jsonl": JsonlFormat, "parquet": ParquetFormat} - - class ValidationPolicy(Enum): emit_record = "Emit Record" skip_record = "Skip Record" @@ -27,10 +25,11 @@ class ValidationPolicy(Enum): class FileBasedStreamConfig(BaseModel): name: str = Field(title="Name", description="The name of the stream.") - file_type: str = Field(title="File Type", description="The data file type that is being extracted for a stream.") globs: Optional[List[str]] = Field( + default=["**"], title="Globs", description='The pattern used to specify which files should be selected from the file system. For more information on glob pattern matching look here.', + order=1, ) legacy_prefix: Optional[str] = Field( title="Legacy Prefix", @@ -47,14 +46,16 @@ class FileBasedStreamConfig(BaseModel): description="The schema that will be used to validate records extracted from the file. This will override the stream schema that is auto-detected from incoming files.", ) primary_key: Optional[str] = Field( - title="Primary Key", description="The column or columns (for a composite key) that serves as the unique identifier of a record." + title="Primary Key", + description="The column or columns (for a composite key) that serves as the unique identifier of a record. If empty, the primary key will default to the parser's default primary key.", + airbyte_hidden=True, # Users can create/modify primary keys in the connection configuration so we shouldn't duplicate it here. ) days_to_sync_if_history_is_full: int = Field( title="Days To Sync If History Is Full", description="When the state history of the file store is full, syncs will only read files that were last modified in the provided day range.", default=3, ) - format: Optional[Union[AvroFormat, CsvFormat, JsonlFormat, ParquetFormat]] = Field( + format: Union[AvroFormat, CsvFormat, JsonlFormat, ParquetFormat, UnstructuredFormat] = Field( title="Format", description="The configuration options that are used to alter how to read incoming files that deviate from the standard formatting.", ) @@ -64,37 +65,6 @@ class FileBasedStreamConfig(BaseModel): default=False, ) - @validator("file_type", pre=True) - def validate_file_type(cls, v: str) -> str: - if v not in VALID_FILE_TYPES: - raise ValueError(f"Format filetype {v} is not a supported file type") - return v - - @classmethod - def _transform_legacy_config(cls, legacy_config: Mapping[str, Any], file_type: str) -> Mapping[str, Any]: - if file_type.casefold() not in VALID_FILE_TYPES: - raise ValueError(f"Format filetype {file_type} is not a supported file type") - if file_type.casefold() == "parquet" or file_type.casefold() == "avro": - legacy_config = cls._transform_legacy_parquet_or_avro_config(legacy_config) - return {file_type: VALID_FILE_TYPES[file_type.casefold()].parse_obj({key: val for key, val in legacy_config.items()})} - - @classmethod - def _transform_legacy_parquet_or_avro_config(cls, config: Mapping[str, Any]) -> Mapping[str, Any]: - """ - The legacy parquet parser converts decimal fields to numbers. This isn't desirable because it can lead to precision loss. - To avoid introducing a breaking change with the new default, we will set decimal_as_float to True in the legacy configs. - """ - filetype = config.get("filetype") - if filetype != "parquet" and filetype != "avro": - raise ValueError( - f"Expected {filetype} format, got {config}. This is probably due to a CDK bug. Please reach out to the Airbyte team for support." - ) - if config.get("decimal_as_float"): - raise ValueError( - f"Received legacy {filetype} file form with 'decimal_as_float' set. This is unexpected. Please reach out to the Airbyte team for support." - ) - return {**config, **{"decimal_as_float": True}} - @validator("input_schema", pre=True) def validate_input_schema(cls, v: Optional[str]) -> Optional[str]: if v: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/jsonl_format.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/jsonl_format.py index 99010d3aeca5..b7a01189d38c 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/jsonl_format.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/jsonl_format.py @@ -2,12 +2,16 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from pydantic import BaseModel -from typing_extensions import Literal +from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig +from pydantic import BaseModel, Field class JsonlFormat(BaseModel): - class Config: + class Config(OneOfOptionConfig): title = "Jsonl Format" + discriminator = "filetype" - filetype: Literal["jsonl"] = "jsonl" + filetype: str = Field( + "jsonl", + const=True, + ) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/parquet_format.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/parquet_format.py index de7f1b62969d..b462e78bba03 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/parquet_format.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/parquet_format.py @@ -2,15 +2,20 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + +from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig from pydantic import BaseModel, Field -from typing_extensions import Literal class ParquetFormat(BaseModel): - class Config: + class Config(OneOfOptionConfig): title = "Parquet Format" + discriminator = "filetype" - filetype: Literal["parquet"] = "parquet" + filetype: str = Field( + "parquet", + const=True, + ) # This option is not recommended, but necessary for backwards compatibility decimal_as_float: bool = Field( title="Convert Decimal Fields to Floats", diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/unstructured_format.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/unstructured_format.py new file mode 100644 index 000000000000..687ef8e5cf8d --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/config/unstructured_format.py @@ -0,0 +1,94 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import List, Literal, Optional, Union + +from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig +from pydantic import BaseModel, Field + + +class LocalProcessingConfigModel(BaseModel): + mode: Literal["local"] = Field("local", const=True) + + class Config(OneOfOptionConfig): + title = "Local" + description = "Process files locally, supporting `fast` and `ocr` modes. This is the default option." + discriminator = "mode" + + +class APIParameterConfigModel(BaseModel): + name: str = Field( + title="Parameter name", + description="The name of the unstructured API parameter to use", + examples=["combine_under_n_chars", "languages"], + ) + value: str = Field(title="Value", description="The value of the parameter", examples=["true", "hi_res"]) + + +class APIProcessingConfigModel(BaseModel): + mode: Literal["api"] = Field("api", const=True) + + api_key: str = Field( + default="", + always_show=True, + title="API Key", + airbyte_secret=True, + description="The API key to use matching the environment", + ) + + api_url: str = Field( + default="https://api.unstructured.io", + title="API URL", + always_show=True, + description="The URL of the unstructured API to use", + examples=["https://api.unstructured.com"], + ) + + parameters: Optional[List[APIParameterConfigModel]] = Field( + default=[], + always_show=True, + title="Additional URL Parameters", + description="List of parameters send to the API", + ) + + class Config(OneOfOptionConfig): + title = "via API" + description = "Process files via an API, using the `hi_res` mode. This option is useful for increased performance and accuracy, but requires an API key and a hosted instance of unstructured." + discriminator = "mode" + + +class UnstructuredFormat(BaseModel): + class Config(OneOfOptionConfig): + title = "Document File Type Format (Experimental)" + description = "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file." + discriminator = "filetype" + + filetype: str = Field( + "unstructured", + const=True, + ) + + skip_unprocessable_files: bool = Field( + default=True, + title="Skip Unprocessable Files", + description="If true, skip files that cannot be parsed and pass the error message along as the _ab_source_file_parse_error field. If false, fail the sync.", + always_show=True, + ) + + strategy: str = Field( + always_show=True, + order=0, + default="auto", + title="Parsing Strategy", + enum=["auto", "fast", "ocr_only", "hi_res"], + description="The strategy used to parse documents. `fast` extracts text directly from the document which doesn't work for all files. `ocr_only` is more reliable, but slower. `hi_res` is the most reliable, but requires an API key and a hosted instance of unstructured and can't be used with local mode. See the unstructured.io documentation for more details: https://unstructured-io.github.io/unstructured/core/partition.html#partition-pdf", + ) + + processing: Union[LocalProcessingConfigModel, APIProcessingConfigModel,] = Field( + default=LocalProcessingConfigModel(mode="local"), + title="Processing", + description="Processing configuration", + discriminator="mode", + type="object", + ) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py index d8cc5d2c4a74..ca2645787dc7 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/discovery_policy/abstract_discovery_policy.py @@ -4,6 +4,8 @@ from abc import ABC, abstractmethod +from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser + class AbstractDiscoveryPolicy(ABC): """ @@ -16,7 +18,6 @@ class AbstractDiscoveryPolicy(ABC): def n_concurrent_requests(self) -> int: ... - @property @abstractmethod - def max_n_files_for_schema_inference(self) -> int: + def get_max_n_files_for_schema_inference(self, parser: FileTypeParser) -> int: ... diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py index 56bd19d01f16..3ce098899fc0 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/discovery_policy/default_discovery_policy.py @@ -3,6 +3,7 @@ # from airbyte_cdk.sources.file_based.discovery_policy.abstract_discovery_policy import AbstractDiscoveryPolicy +from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser DEFAULT_N_CONCURRENT_REQUESTS = 10 DEFAULT_MAX_N_FILES_FOR_STREAM_SCHEMA_INFERENCE = 10 @@ -18,6 +19,10 @@ class DefaultDiscoveryPolicy(AbstractDiscoveryPolicy): def n_concurrent_requests(self) -> int: return DEFAULT_N_CONCURRENT_REQUESTS - @property - def max_n_files_for_schema_inference(self) -> int: - return DEFAULT_MAX_N_FILES_FOR_STREAM_SCHEMA_INFERENCE + def get_max_n_files_for_schema_inference(self, parser: FileTypeParser) -> int: + return min( + filter( + None, + (DEFAULT_MAX_N_FILES_FOR_STREAM_SCHEMA_INFERENCE, parser.parser_max_n_files_for_schema_inference), + ) + ) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/exceptions.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/exceptions.py index 3d2cf212fe58..18073be07b26 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/exceptions.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/exceptions.py @@ -3,6 +3,10 @@ # from enum import Enum +from typing import Any, List, Union + +from airbyte_cdk.models import AirbyteMessage, FailureType +from airbyte_cdk.utils import AirbyteTracedException class FileBasedSourceError(Enum): @@ -22,6 +26,8 @@ class FileBasedSourceError(Enum): ERROR_PARSING_RECORD = "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable." ERROR_PARSING_USER_PROVIDED_SCHEMA = "The provided schema could not be transformed into valid JSON Schema." ERROR_VALIDATING_RECORD = "One or more records do not pass the schema validation policy. Please modify your input schema, or select a more lenient validation policy." + ERROR_PARSING_RECORD_MISMATCHED_COLUMNS = "A header field has resolved to `None`. This indicates that the CSV has more rows than the number of header fields. If you input your schema or headers, please verify that the number of columns corresponds to the number of columns in your CSV's rows." + ERROR_PARSING_RECORD_MISMATCHED_ROWS = "A row's value has resolved to `None`. This indicates that the CSV has more columns in the header field than the number of columns in the row(s). If you input your schema or headers, please verify that the number of columns corresponds to the number of columns in your CSV's rows." STOP_SYNC_PER_SCHEMA_VALIDATION_POLICY = ( "Stopping sync in accordance with the configured validation policy. Records in file did not conform to the schema." ) @@ -35,11 +41,35 @@ class FileBasedSourceError(Enum): UNDEFINED_VALIDATION_POLICY = "The validation policy defined in the config does not exist for the source." +class FileBasedErrorsCollector: + """ + The placeholder for all errors collected. + """ + + errors: List[AirbyteMessage] = [] + + def yield_and_raise_collected(self) -> Any: + if self.errors: + # emit collected logged messages + yield from self.errors + # clean the collector + self.errors.clear() + # raising the single exception + raise AirbyteTracedException( + internal_message="Please check the logged errors for more information.", + message="Some errors occured while reading from the source.", + failure_type=FailureType.config_error, + ) + + def collect(self, logged_error: AirbyteMessage) -> None: + self.errors.append(logged_error) + + class BaseFileBasedSourceError(Exception): - def __init__(self, error: FileBasedSourceError, **kwargs): # type: ignore # noqa - super().__init__( - f"{FileBasedSourceError(error).value} Contact Support if you need assistance.\n{' '.join([f'{k}={v}' for k, v in kwargs.items()])}" - ) + def __init__(self, error: Union[FileBasedSourceError, str], **kwargs): # type: ignore # noqa + if isinstance(error, FileBasedSourceError): + error = FileBasedSourceError(error).value + super().__init__(f"{error} Contact Support if you need assistance.\n{' '.join([f'{k}={v}' for k, v in kwargs.items()])}") class ConfigValidationError(BaseFileBasedSourceError): @@ -54,6 +84,10 @@ class MissingSchemaError(BaseFileBasedSourceError): pass +class NoFilesMatchingError(BaseFileBasedSourceError): + pass + + class RecordParseError(BaseFileBasedSourceError): pass @@ -76,3 +110,13 @@ class StopSyncPerValidationPolicy(BaseFileBasedSourceError): class ErrorListingFiles(BaseFileBasedSourceError): pass + + +class CustomFileBasedException(AirbyteTracedException): + """ + A specialized exception for file-based connectors. + + This exception is designed to bypass the default error handling in the file-based CDK, allowing the use of custom error messages. + """ + + pass diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_based_source.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_based_source.py index c5b643d6fa63..9904e4a8be97 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_based_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_based_source.py @@ -5,15 +5,16 @@ import logging import traceback from abc import ABC -from typing import Any, List, Mapping, Optional, Tuple, Type +from collections import Counter +from typing import Any, Iterator, List, Mapping, MutableMapping, Optional, Tuple, Type, Union -from airbyte_cdk.models import ConnectorSpecification +from airbyte_cdk.models import AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog, ConnectorSpecification from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.file_based.availability_strategy import AbstractFileBasedAvailabilityStrategy, DefaultFileBasedAvailabilityStrategy from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig, ValidationPolicy from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy, DefaultDiscoveryPolicy -from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError +from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedErrorsCollector, FileBasedSourceError from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader from airbyte_cdk.sources.file_based.file_types import default_parsers from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser @@ -22,6 +23,7 @@ from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor from airbyte_cdk.sources.file_based.stream.cursor.default_file_based_cursor import DefaultFileBasedCursor from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.utils.analytics_message import create_analytics_message from pydantic.error_wrappers import ValidationError @@ -33,7 +35,7 @@ def __init__( catalog_path: Optional[str] = None, availability_strategy: Optional[AbstractFileBasedAvailabilityStrategy] = None, discovery_policy: AbstractDiscoveryPolicy = DefaultDiscoveryPolicy(), - parsers: Mapping[str, FileTypeParser] = default_parsers, + parsers: Mapping[Type[Any], FileTypeParser] = default_parsers, validation_policies: Mapping[ValidationPolicy, AbstractSchemaValidationPolicy] = DEFAULT_SCHEMA_VALIDATION_POLICIES, cursor_cls: Type[AbstractFileBasedCursor] = DefaultFileBasedCursor, ): @@ -47,6 +49,7 @@ def __init__( self.stream_schemas = {s.stream.name: s.stream.json_schema for s in catalog.streams} if catalog else {} self.cursor_cls = cursor_cls self.logger = logging.getLogger(f"airbyte.{self.name}") + self.errors_collector: FileBasedErrorsCollector = FileBasedErrorsCollector() def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: """ @@ -89,7 +92,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Return a list of this source's streams. """ try: - parsed_config = self.spec_class(**config) + parsed_config = self._get_parsed_config(config) self.stream_reader.config = parsed_config streams: List[Stream] = [] for stream_config in parsed_config.streams: @@ -104,6 +107,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: parsers=self.parsers, validation_policy=self._validate_and_get_validation_policy(stream_config), cursor=self.cursor_cls(stream_config), + errors_collector=self.errors_collector, ) ) return streams @@ -111,6 +115,21 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: except ValidationError as exc: raise ConfigValidationError(FileBasedSourceError.CONFIG_VALIDATION_ERROR) from exc + def read( + self, + logger: logging.Logger, + config: Mapping[str, Any], + catalog: ConfiguredAirbyteCatalog, + state: Optional[Union[List[AirbyteStateMessage], MutableMapping[str, Any]]] = None, + ) -> Iterator[AirbyteMessage]: + yield from super().read(logger, config, catalog, state) + # emit all the errors collected + yield from self.errors_collector.yield_and_raise_collected() + # count streams using a certain parser + parsed_config = self._get_parsed_config(config) + for parser, count in Counter(stream.format.filetype for stream in parsed_config.streams).items(): + yield create_analytics_message(f"file-cdk-{parser}-stream-count", count) + def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: """ Returns the specification describing what fields can be configured by a user when setting up a file-based source. @@ -121,6 +140,9 @@ def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: connectionSpecification=self.spec_class.schema(), ) + def _get_parsed_config(self, config: Mapping[str, Any]) -> AbstractFileBasedSpec: + return self.spec_class(**config) + def _validate_and_get_validation_policy(self, stream_config: FileBasedStreamConfig) -> AbstractSchemaValidationPolicy: if stream_config.validation_policy not in self.validation_policies: # This should never happen because we validate the config against the schema's validation_policy enum diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/__init__.py index 80922439a45f..77265c05473c 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/__init__.py @@ -1,16 +1,24 @@ -from typing import Mapping +from typing import Any, Mapping, Type + +from airbyte_cdk.sources.file_based.config.avro_format import AvroFormat +from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat +from airbyte_cdk.sources.file_based.config.jsonl_format import JsonlFormat +from airbyte_cdk.sources.file_based.config.parquet_format import ParquetFormat +from airbyte_cdk.sources.file_based.config.unstructured_format import UnstructuredFormat from .avro_parser import AvroParser from .csv_parser import CsvParser from .file_type_parser import FileTypeParser from .jsonl_parser import JsonlParser from .parquet_parser import ParquetParser +from .unstructured_parser import UnstructuredParser -default_parsers: Mapping[str, FileTypeParser] = { - "avro": AvroParser(), - "csv": CsvParser(), - "jsonl": JsonlParser(), - "parquet": ParquetParser(), +default_parsers: Mapping[Type[Any], FileTypeParser] = { + AvroFormat: AvroParser(), + CsvFormat: CsvParser(), + JsonlFormat: JsonlParser(), + ParquetFormat: ParquetParser(), + UnstructuredFormat: UnstructuredParser(), } -__all__ = ["AvroParser", "CsvParser", "JsonlParser", "ParquetParser", "default_parsers"] +__all__ = ["AvroParser", "CsvParser", "JsonlParser", "ParquetParser", "UnstructuredParser", "default_parsers"] diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/avro_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/avro_parser.py index d578ce957c79..25267b9a5c23 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/avro_parser.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/avro_parser.py @@ -3,12 +3,12 @@ # import logging -import uuid -from typing import Any, Dict, Iterable, Mapping, Optional +from typing import Any, Dict, Iterable, Mapping, Optional, Tuple import fastavro from airbyte_cdk.sources.file_based.config.avro_format import AvroFormat from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig +from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, RecordParseError from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser from airbyte_cdk.sources.file_based.remote_file import RemoteFile @@ -42,6 +42,12 @@ class AvroParser(FileTypeParser): ENCODING = None + def check_config(self, config: FileBasedStreamConfig) -> Tuple[bool, Optional[str]]: + """ + AvroParser does not require config checks, implicit pydantic validation is enough. + """ + return True, None + async def infer_schema( self, config: FileBasedStreamConfig, @@ -49,7 +55,7 @@ async def infer_schema( stream_reader: AbstractFileBasedStreamReader, logger: logging.Logger, ) -> SchemaType: - avro_format = config.format or AvroFormat() + avro_format = config.format if not isinstance(avro_format, AvroFormat): raise ValueError(f"Expected ParquetFormat, got {avro_format}") @@ -135,19 +141,24 @@ def parse_records( logger: logging.Logger, discovered_schema: Optional[Mapping[str, SchemaType]], ) -> Iterable[Dict[str, Any]]: - avro_format = config.format or AvroFormat() + avro_format = config.format or AvroFormat(filetype="avro") if not isinstance(avro_format, AvroFormat): raise ValueError(f"Expected ParquetFormat, got {avro_format}") - with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp: - avro_reader = fastavro.reader(fp) - schema = avro_reader.writer_schema - schema_field_name_to_type = {field["name"]: field["type"] for field in schema["fields"]} - for record in avro_reader: - yield { - record_field: self._to_output_value(avro_format, schema_field_name_to_type[record_field], record[record_field]) - for record_field, record_value in schema_field_name_to_type.items() - } + line_no = 0 + try: + with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp: + avro_reader = fastavro.reader(fp) + schema = avro_reader.writer_schema + schema_field_name_to_type = {field["name"]: field["type"] for field in schema["fields"]} + for record in avro_reader: + line_no += 1 + yield { + record_field: self._to_output_value(avro_format, schema_field_name_to_type[record_field], record[record_field]) + for record_field, record_value in schema_field_name_to_type.items() + } + except Exception as exc: + raise RecordParseError(FileBasedSourceError.ERROR_PARSING_RECORD, filename=file.uri, lineno=line_no) from exc @property def file_read_mode(self) -> FileReadMode: @@ -159,9 +170,7 @@ def _to_output_value(avro_format: AvroFormat, record_type: Mapping[str, Any], re if record_type == "double" and avro_format.double_as_string: return str(record_value) return record_value - if record_type.get("logicalType") == "uuid": - return uuid.UUID(bytes=record_value) - elif record_type.get("logicalType") == "decimal": + if record_type.get("logicalType") in ("decimal", "uuid"): return str(record_value) elif record_type.get("logicalType") == "date": return record_value.isoformat() diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/csv_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/csv_parser.py index 5468a4cc6144..b67aebcd723e 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/csv_parser.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/csv_parser.py @@ -9,15 +9,17 @@ from collections import defaultdict from functools import partial from io import IOBase -from typing import Any, Callable, Dict, Generator, Iterable, List, Mapping, Optional, Set +from typing import Any, Callable, Dict, Generator, Iterable, List, Mapping, Optional, Set, Tuple -from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat, InferenceType +from airbyte_cdk.models import FailureType +from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat, CsvHeaderAutogenerated, CsvHeaderUserProvided, InferenceType from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, RecordParseError from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser from airbyte_cdk.sources.file_based.remote_file import RemoteFile from airbyte_cdk.sources.file_based.schema_helpers import TYPE_PYTHON_MAPPING, SchemaType +from airbyte_cdk.utils.traced_exception import AirbyteTracedException DIALECT_NAME = "_config_dialect" @@ -32,6 +34,7 @@ def read_data( file_read_mode: FileReadMode, ) -> Generator[Dict[str, Any], None, None]: config_format = _extract_format(config) + lineno = 0 # Formats are configured individually per-stream so a unique dialect should be registered for each stream. # We don't unregister the dialect because we are lazily parsing each csv file to generate records @@ -48,22 +51,29 @@ def read_data( with stream_reader.open_file(file, file_read_mode, config_format.encoding, logger) as fp: headers = self._get_headers(fp, config_format, dialect_name) - # we assume that if we autogenerate columns, it is because we don't have headers - # if a user wants to autogenerate_column_names with a CSV having headers, he can skip rows rows_to_skip = ( config_format.skip_rows_before_header - + (0 if config_format.autogenerate_column_names else 1) + + (1 if config_format.header_definition.has_header_row() else 0) + config_format.skip_rows_after_header ) self._skip_rows(fp, rows_to_skip) + lineno += rows_to_skip reader = csv.DictReader(fp, dialect=dialect_name, fieldnames=headers) # type: ignore try: for row in reader: + lineno += 1 + # The row was not properly parsed if any of the values are None. This will most likely occur if there are more columns # than headers or more headers dans columns - if None in row or None in row.values(): - raise RecordParseError(FileBasedSourceError.ERROR_PARSING_RECORD) + if None in row: + raise RecordParseError( + FileBasedSourceError.ERROR_PARSING_RECORD_MISMATCHED_COLUMNS, + filename=file.uri, + lineno=lineno, + ) + if None in row.values(): + raise RecordParseError(FileBasedSourceError.ERROR_PARSING_RECORD_MISMATCHED_ROWS, filename=file.uri, lineno=lineno) yield row finally: # due to RecordParseError or GeneratorExit @@ -74,11 +84,15 @@ def _get_headers(self, fp: IOBase, config_format: CsvFormat, dialect_name: str) Assumes the fp is pointing to the beginning of the files and will reset it as such """ # Note that this method assumes the dialect has already been registered if we're parsing the headers - self._skip_rows(fp, config_format.skip_rows_before_header) - if config_format.autogenerate_column_names: + if isinstance(config_format.header_definition, CsvHeaderUserProvided): + return config_format.header_definition.column_names # type: ignore # should be CsvHeaderUserProvided given the type + + if isinstance(config_format.header_definition, CsvHeaderAutogenerated): + self._skip_rows(fp, config_format.skip_rows_before_header + config_format.skip_rows_after_header) headers = self._auto_generate_headers(fp, dialect_name) else: # Then read the header + self._skip_rows(fp, config_format.skip_rows_before_header) reader = csv.reader(fp, dialect=dialect_name) # type: ignore headers = list(next(reader)) @@ -109,6 +123,12 @@ class CsvParser(FileTypeParser): def __init__(self, csv_reader: Optional[_CsvReader] = None): self._csv_reader = csv_reader if csv_reader else _CsvReader() + def check_config(self, config: FileBasedStreamConfig) -> Tuple[bool, Optional[str]]: + """ + CsvParser does not require config checks, implicit pydantic validation is enough. + """ + return True, None + async def infer_schema( self, config: FileBasedStreamConfig, @@ -140,6 +160,12 @@ async def infer_schema( if read_bytes >= self._MAX_BYTES_PER_FILE_FOR_SCHEMA_INFERENCE: break + if not type_inferrer_by_field: + raise AirbyteTracedException( + message=f"Could not infer schema as there are no rows in {file.uri}. If having an empty CSV file is expected, ignore this. " + f"Else, please contact Airbyte.", + failure_type=FailureType.config_error, + ) schema = {header.strip(): {"type": type_inferred.infer()} for header, type_inferred in type_inferrer_by_field.items()} data_generator.close() return schema @@ -152,17 +178,25 @@ def parse_records( logger: logging.Logger, discovered_schema: Optional[Mapping[str, SchemaType]], ) -> Iterable[Dict[str, Any]]: - config_format = _extract_format(config) - if discovered_schema: - property_types = {col: prop["type"] for col, prop in discovered_schema["properties"].items()} # type: ignore # discovered_schema["properties"] is known to be a mapping - deduped_property_types = CsvParser._pre_propcess_property_types(property_types) - else: - deduped_property_types = {} - cast_fn = CsvParser._get_cast_function(deduped_property_types, config_format, logger, config.schemaless) - data_generator = self._csv_reader.read_data(config, file, stream_reader, logger, self.file_read_mode) - for row in data_generator: - yield CsvParser._to_nullable(cast_fn(row), deduped_property_types, config_format.null_values, config_format.strings_can_be_null) - data_generator.close() + line_no = 0 + try: + config_format = _extract_format(config) + if discovered_schema: + property_types = {col: prop["type"] for col, prop in discovered_schema["properties"].items()} # type: ignore # discovered_schema["properties"] is known to be a mapping + deduped_property_types = CsvParser._pre_propcess_property_types(property_types) + else: + deduped_property_types = {} + cast_fn = CsvParser._get_cast_function(deduped_property_types, config_format, logger, config.schemaless) + data_generator = self._csv_reader.read_data(config, file, stream_reader, logger, self.file_read_mode) + for row in data_generator: + line_no += 1 + yield CsvParser._to_nullable( + cast_fn(row), deduped_property_types, config_format.null_values, config_format.strings_can_be_null + ) + except RecordParseError as parse_err: + raise RecordParseError(FileBasedSourceError.ERROR_PARSING_RECORD, filename=file.uri, lineno=line_no) from parse_err + finally: + data_generator.close() @property def file_read_mode(self) -> FileReadMode: @@ -183,7 +217,7 @@ def _get_cast_function( def _to_nullable( row: Mapping[str, str], deduped_property_types: Mapping[str, str], null_values: Set[str], strings_can_be_null: bool ) -> Dict[str, Optional[str]]: - nullable = row | { + nullable = { k: None if CsvParser._value_is_none(v, deduped_property_types.get(k), null_values, strings_can_be_null) else v for k, v in row.items() } @@ -402,7 +436,7 @@ def _no_cast(row: Mapping[str, str]) -> Mapping[str, str]: def _extract_format(config: FileBasedStreamConfig) -> CsvFormat: - config_format = config.format or CsvFormat() + config_format = config.format if not isinstance(config_format, CsvFormat): raise ValueError(f"Invalid format config: {config_format}") return config_format diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/file_type_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/file_type_parser.py index 0e52fb5e04df..d334621ada66 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/file_type_parser.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/file_type_parser.py @@ -4,7 +4,7 @@ import logging from abc import ABC, abstractmethod -from typing import Any, Dict, Iterable, Mapping, Optional +from typing import Any, Dict, Iterable, Mapping, Optional, Tuple from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode @@ -20,6 +20,33 @@ class FileTypeParser(ABC): supported file type. """ + @property + def parser_max_n_files_for_schema_inference(self) -> Optional[int]: + """ + The discovery policy decides how many files are loaded for schema inference. This method can provide a parser-specific override. If it's defined, the smaller of the two values will be used. + """ + return None + + @property + def parser_max_n_files_for_parsability(self) -> Optional[int]: + """ + The availability policy decides how many files are loaded for checking whether parsing works correctly. This method can provide a parser-specific override. If it's defined, the smaller of the two values will be used. + """ + return None + + def get_parser_defined_primary_key(self, config: FileBasedStreamConfig) -> Optional[str]: + """ + The parser can define a primary key. If no user-defined primary key is provided, this will be used. + """ + return None + + @abstractmethod + def check_config(self, config: FileBasedStreamConfig) -> Tuple[bool, Optional[str]]: + """ + Check whether the config is valid for this file type. If it is, return True and None. If it's not, return False and an error message explaining why it's invalid. + """ + return True, None + @abstractmethod async def infer_schema( self, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/jsonl_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/jsonl_parser.py index 27efe8a005b4..122103c5739d 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/jsonl_parser.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/jsonl_parser.py @@ -4,7 +4,7 @@ import json import logging -from typing import Any, Dict, Iterable, Mapping, Optional, Union +from typing import Any, Dict, Iterable, Mapping, Optional, Tuple, Union from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, RecordParseError @@ -19,6 +19,12 @@ class JsonlParser(FileTypeParser): MAX_BYTES_PER_FILE_FOR_SCHEMA_INFERENCE = 1_000_000 ENCODING = "utf8" + def check_config(self, config: FileBasedStreamConfig) -> Tuple[bool, Optional[str]]: + """ + JsonlParser does not require config checks, implicit pydantic validation is enough. + """ + return True, None + async def infer_schema( self, config: FileBasedStreamConfig, @@ -113,7 +119,7 @@ def _parse_jsonl_entries( break if had_json_parsing_error and not yielded_at_least_once: - raise RecordParseError(FileBasedSourceError.ERROR_PARSING_RECORD) + raise RecordParseError(FileBasedSourceError.ERROR_PARSING_RECORD, filename=file.uri, lineno=line) @staticmethod def _instantiate_accumulator(line: Union[bytes, str]) -> Union[bytes, str]: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/parquet_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/parquet_parser.py index 851608cfffd5..00b78c489801 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/parquet_parser.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/parquet_parser.py @@ -5,13 +5,13 @@ import json import logging import os -from typing import Any, Dict, Iterable, List, Mapping, Optional +from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple from urllib.parse import unquote import pyarrow as pa import pyarrow.parquet as pq from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig, ParquetFormat -from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError +from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError, RecordParseError from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser from airbyte_cdk.sources.file_based.remote_file import RemoteFile @@ -23,6 +23,12 @@ class ParquetParser(FileTypeParser): ENCODING = None + def check_config(self, config: FileBasedStreamConfig) -> Tuple[bool, Optional[str]]: + """ + ParquetParser does not require config checks, implicit pydantic validation is enough. + """ + return True, None + async def infer_schema( self, config: FileBasedStreamConfig, @@ -30,7 +36,7 @@ async def infer_schema( stream_reader: AbstractFileBasedStreamReader, logger: logging.Logger, ) -> SchemaType: - parquet_format = config.format or ParquetFormat() + parquet_format = config.format if not isinstance(parquet_format, ParquetFormat): raise ValueError(f"Expected ParquetFormat, got {parquet_format}") @@ -54,23 +60,31 @@ def parse_records( logger: logging.Logger, discovered_schema: Optional[Mapping[str, SchemaType]], ) -> Iterable[Dict[str, Any]]: - parquet_format = config.format or ParquetFormat() + parquet_format = config.format if not isinstance(parquet_format, ParquetFormat): logger.info(f"Expected ParquetFormat, got {parquet_format}") raise ConfigValidationError(FileBasedSourceError.CONFIG_VALIDATION_ERROR) - with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp: - reader = pq.ParquetFile(fp) - partition_columns = {x.split("=")[0]: x.split("=")[1] for x in self._extract_partitions(file.uri)} - for row_group in range(reader.num_row_groups): - batch = reader.read_row_group(row_group) - for row in range(batch.num_rows): - yield { - **{ - column: ParquetParser._to_output_value(batch.column(column)[row], parquet_format) - for column in batch.column_names - }, - **partition_columns, - } + + line_no = 0 + try: + with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp: + reader = pq.ParquetFile(fp) + partition_columns = {x.split("=")[0]: x.split("=")[1] for x in self._extract_partitions(file.uri)} + for row_group in range(reader.num_row_groups): + batch = reader.read_row_group(row_group) + for row in range(batch.num_rows): + line_no += 1 + yield { + **{ + column: ParquetParser._to_output_value(batch.column(column)[row], parquet_format) + for column in batch.column_names + }, + **partition_columns, + } + except Exception as exc: + raise RecordParseError( + FileBasedSourceError.ERROR_PARSING_RECORD, filename=file.uri, lineno=f"{row_group=}, {line_no=}" + ) from exc @staticmethod def _extract_partitions(filepath: str) -> List[str]: @@ -95,7 +109,10 @@ def _to_output_value(parquet_value: Scalar, parquet_format: ParquetFormat) -> An # Decode binary strings to utf-8 if ParquetParser._is_binary(parquet_value.type): - return parquet_value.as_py().decode("utf-8") + py_value = parquet_value.as_py() + if py_value is None: + return py_value + return py_value.decode("utf-8") if pa.types.is_decimal(parquet_value.type): if parquet_format.decimal_as_float: return parquet_value.as_py() diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/unstructured_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/unstructured_parser.py new file mode 100644 index 000000000000..7c117b208672 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/unstructured_parser.py @@ -0,0 +1,357 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import logging +import traceback +from datetime import datetime +from io import BytesIO, IOBase +from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple, Union + +import backoff +import dpath.util +import requests +from airbyte_cdk.models import FailureType +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig +from airbyte_cdk.sources.file_based.config.unstructured_format import ( + APIParameterConfigModel, + APIProcessingConfigModel, + LocalProcessingConfigModel, + UnstructuredFormat, +) +from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, RecordParseError +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.schema_helpers import SchemaType +from airbyte_cdk.utils import is_cloud_environment +from airbyte_cdk.utils.traced_exception import AirbyteTracedException +from unstructured.file_utils.filetype import FILETYPE_TO_MIMETYPE, STR_TO_FILETYPE, FileType, detect_filetype + +unstructured_partition_pdf = None +unstructured_partition_docx = None +unstructured_partition_pptx = None + + +def optional_decode(contents: Union[str, bytes]) -> str: + if isinstance(contents, bytes): + return contents.decode("utf-8") + return contents + + +def _import_unstructured() -> None: + """Dynamically imported as needed, due to slow import speed.""" + global unstructured_partition_pdf + global unstructured_partition_docx + global unstructured_partition_pptx + from unstructured.partition.docx import partition_docx + from unstructured.partition.pdf import partition_pdf + from unstructured.partition.pptx import partition_pptx + + # separate global variables to properly propagate typing + unstructured_partition_pdf = partition_pdf + unstructured_partition_docx = partition_docx + unstructured_partition_pptx = partition_pptx + + +def user_error(e: Exception) -> bool: + """ + Return True if this exception is caused by user error, False otherwise. + """ + if not isinstance(e, RecordParseError): + return False + if not isinstance(e, requests.exceptions.RequestException): + return False + return bool(e.response and 400 <= e.response.status_code < 500) + + +CLOUD_DEPLOYMENT_MODE = "cloud" + + +class UnstructuredParser(FileTypeParser): + @property + def parser_max_n_files_for_schema_inference(self) -> Optional[int]: + """ + Just check one file as the schema is static + """ + return 1 + + @property + def parser_max_n_files_for_parsability(self) -> Optional[int]: + """ + Do not check any files for parsability because it might be an expensive operation and doesn't give much confidence whether the sync will succeed. + """ + return 0 + + def get_parser_defined_primary_key(self, config: FileBasedStreamConfig) -> Optional[str]: + """ + Return the document_key field as the primary key. + + his will pre-select the document key column as the primary key when setting up a connection, making it easier for the user to configure normalization in the destination. + """ + return "document_key" + + async def infer_schema( + self, + config: FileBasedStreamConfig, + file: RemoteFile, + stream_reader: AbstractFileBasedStreamReader, + logger: logging.Logger, + ) -> SchemaType: + format = _extract_format(config) + with stream_reader.open_file(file, self.file_read_mode, None, logger) as file_handle: + filetype = self._get_filetype(file_handle, file) + + if filetype not in self._supported_file_types() and not format.skip_unprocessable_files: + raise self._create_parse_error(file, self._get_file_type_error_message(filetype)) + + return { + "content": { + "type": "string", + "description": "Content of the file as markdown. Might be null if the file could not be parsed", + }, + "document_key": {"type": "string", "description": "Unique identifier of the document, e.g. the file path"}, + "_ab_source_file_parse_error": { + "type": "string", + "description": "Error message if the file could not be parsed even though the file is supported", + }, + } + + def parse_records( + self, + config: FileBasedStreamConfig, + file: RemoteFile, + stream_reader: AbstractFileBasedStreamReader, + logger: logging.Logger, + discovered_schema: Optional[Mapping[str, SchemaType]], + ) -> Iterable[Dict[str, Any]]: + format = _extract_format(config) + with stream_reader.open_file(file, self.file_read_mode, None, logger) as file_handle: + try: + markdown = self._read_file(file_handle, file, format, logger) + yield { + "content": markdown, + "document_key": file.uri, + "_ab_source_file_parse_error": None, + } + except RecordParseError as e: + # RecordParseError is raised when the file can't be parsed because of a problem with the file content (either the file is not supported or the file is corrupted) + # if the skip_unprocessable_files flag is set, we log a warning and pass the error as part of the document + # otherwise, we raise the error to fail the sync + if format.skip_unprocessable_files: + exception_str = str(e) + logger.warn(f"File {file.uri} caused an error during parsing: {exception_str}.") + yield { + "content": None, + "document_key": file.uri, + "_ab_source_file_parse_error": exception_str, + } + logger.warn(f"File {file.uri} cannot be parsed. Skipping it.") + else: + raise e + + def _read_file(self, file_handle: IOBase, remote_file: RemoteFile, format: UnstructuredFormat, logger: logging.Logger) -> str: + _import_unstructured() + if (not unstructured_partition_pdf) or (not unstructured_partition_docx) or (not unstructured_partition_pptx): + # check whether unstructured library is actually available for better error message and to ensure proper typing (can't be None after this point) + raise Exception("unstructured library is not available") + + filetype = self._get_filetype(file_handle, remote_file) + + if filetype == FileType.MD or filetype == FileType.TXT: + file_content: bytes = file_handle.read() + decoded_content: str = optional_decode(file_content) + return decoded_content + if filetype not in self._supported_file_types(): + raise self._create_parse_error(remote_file, self._get_file_type_error_message(filetype)) + if format.processing.mode == "local": + return self._read_file_locally(file_handle, filetype, format.strategy, remote_file) + elif format.processing.mode == "api": + try: + result: str = self._read_file_remotely_with_retries(file_handle, format.processing, filetype, format.strategy, remote_file) + except Exception as e: + # If a parser error happens during remotely processing the file, this means the file is corrupted. This case is handled by the parse_records method, so just rethrow. + # + # For other exceptions, re-throw as config error so the sync is stopped as problems with the external API need to be resolved by the user and are not considered part of the SLA. + # Once this parser leaves experimental stage, we should consider making this a system error instead for issues that might be transient. + if isinstance(e, RecordParseError): + raise e + raise AirbyteTracedException.from_exception(e, failure_type=FailureType.config_error) + + return result + + def _params_to_dict(self, params: Optional[List[APIParameterConfigModel]], strategy: str) -> Dict[str, Union[str, List[str]]]: + result_dict: Dict[str, Union[str, List[str]]] = {"strategy": strategy} + if params is None: + return result_dict + for item in params: + key = item.name + value = item.value + if key in result_dict: + existing_value = result_dict[key] + # If the key already exists, append the new value to its list + if isinstance(existing_value, list): + existing_value.append(value) + else: + result_dict[key] = [existing_value, value] + else: + # If the key doesn't exist, add it to the dictionary + result_dict[key] = value + + return result_dict + + def check_config(self, config: FileBasedStreamConfig) -> Tuple[bool, Optional[str]]: + """ + Perform a connection check for the parser config: + - Verify that encryption is enabled if the API is hosted on a cloud instance. + - Verify that the API can extract text from a file. + + For local processing, we don't need to perform any additional checks, implicit pydantic validation is enough. + """ + format_config = _extract_format(config) + if isinstance(format_config.processing, LocalProcessingConfigModel): + if format_config.strategy == "hi_res": + return False, "Hi-res strategy is not supported for local processing" + return True, None + + if is_cloud_environment() and not format_config.processing.api_url.startswith("https://"): + return False, "Base URL must start with https://" + + try: + self._read_file_remotely( + BytesIO(b"# Airbyte source connection test"), + format_config.processing, + FileType.MD, + "auto", + RemoteFile(uri="test", last_modified=datetime.now()), + ) + except Exception: + return False, "".join(traceback.format_exc()) + + return True, None + + @backoff.on_exception(backoff.expo, requests.exceptions.RequestException, max_tries=5, giveup=user_error) + def _read_file_remotely_with_retries( + self, file_handle: IOBase, format: APIProcessingConfigModel, filetype: FileType, strategy: str, remote_file: RemoteFile + ) -> str: + """ + Read a file remotely, retrying up to 5 times if the error is not caused by user error. This is useful for transient network errors or the API server being overloaded temporarily. + """ + return self._read_file_remotely(file_handle, format, filetype, strategy, remote_file) + + def _read_file_remotely( + self, file_handle: IOBase, format: APIProcessingConfigModel, filetype: FileType, strategy: str, remote_file: RemoteFile + ) -> str: + headers = {"accept": "application/json", "unstructured-api-key": format.api_key} + + data = self._params_to_dict(format.parameters, strategy) + + file_data = {"files": ("filename", file_handle, FILETYPE_TO_MIMETYPE[filetype])} + + response = requests.post(f"{format.api_url}/general/v0/general", headers=headers, data=data, files=file_data) + + if response.status_code == 422: + # 422 means the file couldn't be processed, but the API is working. Treat this as a parsing error (passing an error record to the destination). + raise self._create_parse_error(remote_file, response.json()) + else: + # Other error statuses are raised as requests exceptions (retry everything except user errors) + response.raise_for_status() + + json_response = response.json() + + return self._render_markdown(json_response) + + def _read_file_locally(self, file_handle: IOBase, filetype: FileType, strategy: str, remote_file: RemoteFile) -> str: + _import_unstructured() + if (not unstructured_partition_pdf) or (not unstructured_partition_docx) or (not unstructured_partition_pptx): + # check whether unstructured library is actually available for better error message and to ensure proper typing (can't be None after this point) + raise Exception("unstructured library is not available") + + file: Any = file_handle + + # before the parsing logic is entered, the file is read completely to make sure it is in local memory + file_handle.seek(0) + file_handle.read() + file_handle.seek(0) + + try: + if filetype == FileType.PDF: + # for PDF, read the file into a BytesIO object because some code paths in pdf parsing are doing an instance check on the file object and don't work with file-like objects + file_handle.seek(0) + with BytesIO(file_handle.read()) as file: + file_handle.seek(0) + elements = unstructured_partition_pdf(file=file, strategy=strategy) + elif filetype == FileType.DOCX: + elements = unstructured_partition_docx(file=file) + elif filetype == FileType.PPTX: + elements = unstructured_partition_pptx(file=file) + except Exception as e: + raise self._create_parse_error(remote_file, str(e)) + + return self._render_markdown([element.to_dict() for element in elements]) + + def _create_parse_error(self, remote_file: RemoteFile, message: str) -> RecordParseError: + return RecordParseError(FileBasedSourceError.ERROR_PARSING_RECORD, filename=remote_file.uri, message=message) + + def _get_filetype(self, file: IOBase, remote_file: RemoteFile) -> Optional[FileType]: + """ + Detect the file type based on the file name and the file content. + + There are three strategies to determine the file type: + 1. Use the mime type if available (only some sources support it) + 2. Use the file name if available + 3. Use the file content + """ + if remote_file.mime_type and remote_file.mime_type in STR_TO_FILETYPE: + return STR_TO_FILETYPE[remote_file.mime_type] + + # set name to none, otherwise unstructured will try to get the modified date from the local file system + if hasattr(file, "name"): + file.name = None + + # detect_filetype is either using the file name or file content + # if possible, try to leverage the file name to detect the file type + # if the file name is not available, use the file content + file_type = detect_filetype( + filename=remote_file.uri, + ) + if file_type is not None and not file_type == FileType.UNK: + return file_type + + type_based_on_content = detect_filetype(file=file) + + # detect_filetype is reading to read the file content + file.seek(0) + + return type_based_on_content + + def _supported_file_types(self) -> List[Any]: + return [FileType.MD, FileType.PDF, FileType.DOCX, FileType.PPTX, FileType.TXT] + + def _get_file_type_error_message(self, file_type: FileType) -> str: + supported_file_types = ", ".join([str(type) for type in self._supported_file_types()]) + return f"File type {file_type} is not supported. Supported file types are {supported_file_types}" + + def _render_markdown(self, elements: List[Any]) -> str: + return "\n\n".join((self._convert_to_markdown(el) for el in elements)) + + def _convert_to_markdown(self, el: Dict[str, Any]) -> str: + if dpath.util.get(el, "type") == "Title": + heading_str = "#" * (dpath.util.get(el, "metadata/category_depth", default=1) or 1) + return f"{heading_str} {dpath.util.get(el, 'text')}" + elif dpath.util.get(el, "type") == "ListItem": + return f"- {dpath.util.get(el, 'text')}" + elif dpath.util.get(el, "type") == "Formula": + return f"```\n{dpath.util.get(el, 'text')}\n```" + else: + return str(dpath.util.get(el, "text", default="")) + + @property + def file_read_mode(self) -> FileReadMode: + return FileReadMode.READ_BINARY + + +def _extract_format(config: FileBasedStreamConfig) -> UnstructuredFormat: + config_format = config.format + if not isinstance(config_format, UnstructuredFormat): + raise ValueError(f"Invalid format config: {config_format}") + return config_format diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/remote_file.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/remote_file.py index c78065f8d2d3..d7b739e7dc3f 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/remote_file.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/remote_file.py @@ -15,11 +15,4 @@ class RemoteFile(BaseModel): uri: str last_modified: datetime - - def extension_agrees_with_file_type(self, file_type: Optional[str]) -> bool: - extensions = self.uri.split(".")[1:] - if not extensions: - return True - if not file_type: - return True - return any(file_type.casefold() in e.casefold() for e in extensions) + mime_type: Optional[str] = None diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py index 7499316ea2d7..420cf7ef6988 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/abstract_file_based_stream.py @@ -3,14 +3,14 @@ # from abc import abstractmethod -from functools import cached_property, lru_cache -from typing import Any, Dict, Iterable, List, Mapping, Optional +from functools import cache, cached_property, lru_cache +from typing import Any, Dict, Iterable, List, Mapping, Optional, Type from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.file_based.availability_strategy import AbstractFileBasedAvailabilityStrategy from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig, PrimaryKeyType from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy -from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, RecordParseError, UndefinedParserError +from airbyte_cdk.sources.file_based.exceptions import FileBasedErrorsCollector, FileBasedSourceError, RecordParseError, UndefinedParserError from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser from airbyte_cdk.sources.file_based.remote_file import RemoteFile @@ -42,27 +42,40 @@ def __init__( stream_reader: AbstractFileBasedStreamReader, availability_strategy: AbstractFileBasedAvailabilityStrategy, discovery_policy: AbstractDiscoveryPolicy, - parsers: Dict[str, FileTypeParser], + parsers: Dict[Type[Any], FileTypeParser], validation_policy: AbstractSchemaValidationPolicy, + errors_collector: FileBasedErrorsCollector, ): super().__init__() self.config = config self.catalog_schema = catalog_schema self.validation_policy = validation_policy - self._stream_reader = stream_reader + self.stream_reader = stream_reader self._discovery_policy = discovery_policy self._availability_strategy = availability_strategy self._parsers = parsers + self.errors_collector = errors_collector @property @abstractmethod def primary_key(self) -> PrimaryKeyType: ... - @abstractmethod + @cache def list_files(self) -> List[RemoteFile]: """ List all files that belong to the stream. + + The output of this method is cached so we don't need to list the files more than once. + This means we won't pick up changes to the files during a sync. This meethod uses the + get_files method which is implemented by the concrete stream class. + """ + return list(self.get_files()) + + @abstractmethod + def get_files(self) -> Iterable[RemoteFile]: + """ + List all files that belong to the stream as defined by the stream's globs. """ ... @@ -121,11 +134,11 @@ def infer_schema(self, files: List[RemoteFile]) -> Mapping[str, Any]: """ ... - def get_parser(self, file_type: str) -> FileTypeParser: + def get_parser(self) -> FileTypeParser: try: - return self._parsers[file_type] + return self._parsers[type(self.config.format)] except KeyError: - raise UndefinedParserError(FileBasedSourceError.UNDEFINED_PARSER, stream=self.name, file_type=file_type) + raise UndefinedParserError(FileBasedSourceError.UNDEFINED_PARSER, stream=self.name, format=type(self.config.format)) def record_passes_validation_policy(self, record: Mapping[str, Any]) -> bool: if self.validation_policy: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/default_file_based_stream.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/default_file_based_stream.py index 087ea525b3af..f6e0ac8e0fe7 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/default_file_based_stream.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/stream/default_file_based_stream.py @@ -5,16 +5,18 @@ import asyncio import itertools import traceback +from copy import deepcopy from functools import cache from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Set, Union -from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, FailureType, Level from airbyte_cdk.models import Type as MessageType from airbyte_cdk.sources.file_based.config.file_based_stream_config import PrimaryKeyType from airbyte_cdk.sources.file_based.exceptions import ( FileBasedSourceError, InvalidSchemaError, MissingSchemaError, + NoFilesMatchingError, RecordParseError, SchemaInferenceError, StopSyncPerValidationPolicy, @@ -27,6 +29,7 @@ from airbyte_cdk.sources.streams import IncrementalMixin from airbyte_cdk.sources.streams.core import JsonSchema from airbyte_cdk.sources.utils.record_helper import stream_data_to_airbyte_message +from airbyte_cdk.utils.traced_exception import AirbyteTracedException class DefaultFileBasedStream(AbstractFileBasedStream, IncrementalMixin): @@ -55,7 +58,7 @@ def state(self, value: MutableMapping[str, Any]) -> None: @property def primary_key(self) -> PrimaryKeyType: - return self.config.primary_key + return self.config.primary_key or self.get_parser().get_parser_defined_primary_key(self.config) def compute_slices(self) -> Iterable[Optional[Mapping[str, Any]]]: # Sort files by last_modified, uri and return them grouped by last_modified @@ -77,14 +80,14 @@ def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[Airbyte # On read requests we should always have the catalog available raise MissingSchemaError(FileBasedSourceError.MISSING_SCHEMA, stream=self.name) # The stream only supports a single file type, so we can use the same parser for all files - parser = self.get_parser(self.config.file_type) + parser = self.get_parser() for file in stream_slice["files"]: # only serialize the datetime once file_datetime_string = file.last_modified.strftime(self.DATE_TIME_FORMAT) n_skipped = line_no = 0 try: - for record in parser.parse_records(self.config, file, self._stream_reader, self.logger, schema): + for record in parser.parse_records(self.config, file, self.stream_reader, self.logger, schema): line_no += 1 if self.config.schemaless: record = {"data": record} @@ -109,15 +112,21 @@ def read_records_from_slice(self, stream_slice: StreamSlice) -> Iterable[Airbyte except RecordParseError: # Increment line_no because the exception was raised before we could increment it line_no += 1 - yield AirbyteMessage( - type=MessageType.LOG, - log=AirbyteLogMessage( - level=Level.ERROR, - message=f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream={self.name} file={file.uri} line_no={line_no} n_skipped={n_skipped}", - stack_trace=traceback.format_exc(), + self.errors_collector.collect( + AirbyteMessage( + type=MessageType.LOG, + log=AirbyteLogMessage( + level=Level.ERROR, + message=f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream={self.name} file={file.uri} line_no={line_no} n_skipped={n_skipped}", + stack_trace=traceback.format_exc(), + ), ), ) + except AirbyteTracedException as exc: + # Re-raise the exception to stop the whole sync immediately as this is a fatal error + raise exc + except Exception: yield AirbyteMessage( type=MessageType.LOG, @@ -154,6 +163,10 @@ def get_json_schema(self) -> JsonSchema: } try: schema = self._get_raw_json_schema() + except (InvalidSchemaError, NoFilesMatchingError) as config_exception: + raise AirbyteTracedException( + message=FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value, exception=config_exception, failure_type=FailureType.config_error + ) from config_exception except Exception as exc: raise SchemaInferenceError(FileBasedSourceError.SCHEMA_INFERENCE_ERROR, stream=self.name) from exc else: @@ -169,9 +182,9 @@ def _get_raw_json_schema(self) -> JsonSchema: total_n_files = len(files) if total_n_files == 0: - raise SchemaInferenceError(FileBasedSourceError.EMPTY_STREAM, stream=self.name) + raise NoFilesMatchingError(FileBasedSourceError.EMPTY_STREAM, stream=self.name) - max_n_files_for_schema_inference = self._discovery_policy.max_n_files_for_schema_inference + max_n_files_for_schema_inference = self._discovery_policy.get_max_n_files_for_schema_inference(self.get_parser()) if total_n_files > max_n_files_for_schema_inference: # Use the most recent files for schema inference, so we pick up schema changes during discovery. files = sorted(files, key=lambda x: x.last_modified, reverse=True)[:max_n_files_for_schema_inference] @@ -184,7 +197,7 @@ def _get_raw_json_schema(self) -> JsonSchema: if not inferred_schema: raise InvalidSchemaError( FileBasedSourceError.INVALID_SCHEMA_ERROR, - details=f"Empty schema. Please check that the files are valid {self.config.file_type}", + details=f"Empty schema. Please check that the files are valid for format {self.config.format}", stream=self.name, ) @@ -192,19 +205,17 @@ def _get_raw_json_schema(self) -> JsonSchema: return schema - @cache - def list_files(self) -> List[RemoteFile]: + def get_files(self) -> Iterable[RemoteFile]: """ - List all files that belong to the stream as defined by the stream's globs. - The output of this method is cached so we don't need to list the files more than once. - This means we won't pick up changes to the files during a sync. + Return all files that belong to the stream as defined by the stream's globs. """ - return list(self._stream_reader.get_matching_files(self.config.globs or [], self.config.legacy_prefix, self.logger)) + return self.stream_reader.get_matching_files(self.config.globs or [], self.config.legacy_prefix, self.logger) def infer_schema(self, files: List[RemoteFile]) -> Mapping[str, Any]: loop = asyncio.get_event_loop() schema = loop.run_until_complete(self._infer_schema(files)) - return self._fill_nulls(schema) + # as infer schema returns a Mapping that is assumed to be immutable, we need to create a deepcopy to avoid modifying the reference + return self._fill_nulls(deepcopy(schema)) @staticmethod def _fill_nulls(schema: Mapping[str, Any]) -> Mapping[str, Any]: @@ -252,11 +263,11 @@ async def _infer_schema(self, files: List[RemoteFile]) -> Mapping[str, Any]: async def _infer_file_schema(self, file: RemoteFile) -> SchemaType: try: - return await self.get_parser(self.config.file_type).infer_schema(self.config, file, self._stream_reader, self.logger) + return await self.get_parser().infer_schema(self.config, file, self.stream_reader, self.logger) except Exception as exc: raise SchemaInferenceError( FileBasedSourceError.SCHEMA_INFERENCE_ERROR, file=file.uri, - stream_file_type=self.config.file_type, + format=str(self.config.format), stream=self.name, ) from exc diff --git a/airbyte-cdk/python/airbyte_cdk/sources/http_config.py b/airbyte-cdk/python/airbyte_cdk/sources/http_config.py new file mode 100644 index 000000000000..289ed9a923fb --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/http_config.py @@ -0,0 +1,10 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +# The goal of this variable is to make an implicit dependency explicit. As part of of the Concurrent CDK work, we are facing a situation +# where the connection pool size is too small to serve all the threads (see https://github.com/airbytehq/airbyte/issues/32072). In +# order to fix that, we will increase the requests library pool_maxsize. As there are many pieces of code that sets a requests.Session, we +# are creating this variable here so that a change in one affects the other. This can be removed once we merge how we do HTTP requests in +# one piece of code or once we make connection pool size configurable for each piece of code +MAX_CONNECTION_POOL_SIZE = 20 diff --git a/airbyte-cdk/python/airbyte_cdk/sources/message/repository.py b/airbyte-cdk/python/airbyte_cdk/sources/message/repository.py index 124d8ec416c0..bf90830900ae 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/message/repository.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/message/repository.py @@ -75,12 +75,6 @@ def __init__(self, log_level: Level = Level.INFO) -> None: self._log_level = log_level def emit_message(self, message: AirbyteMessage) -> None: - """ - :param message: As of today, only AirbyteControlMessages are supported given that supporting other types of message will need more - work and therefore this work has been postponed - """ - if message.type not in _SUPPORTED_MESSAGE_TYPES: - raise ValueError(f"As of today, only {_SUPPORTED_MESSAGE_TYPES} are supported as part of the InMemoryMessageRepository") self._message_queue.append(message) def log_message(self, level: Level, message_provider: Callable[[], LogMessage]) -> None: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/source.py b/airbyte-cdk/python/airbyte_cdk/sources/source.py index f6c374140f28..33b5b8575f31 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/source.py @@ -6,7 +6,7 @@ import logging from abc import ABC, abstractmethod from collections import defaultdict -from typing import Any, Generic, Iterable, List, Mapping, MutableMapping, TypeVar, Union +from typing import Any, Dict, Generic, Iterable, List, Mapping, MutableMapping, Optional, TypeVar, Union from airbyte_cdk.connector import BaseConnector, DefaultConnectorMixin, TConfig from airbyte_cdk.models import AirbyteCatalog, AirbyteMessage, AirbyteStateMessage, AirbyteStateType, ConfiguredAirbyteCatalog @@ -25,7 +25,7 @@ def read_catalog(self, catalog_path: str) -> TCatalog: ... @abstractmethod - def read(self, logger: logging.Logger, config: TConfig, catalog: TCatalog, state: TState = None) -> Iterable[AirbyteMessage]: + def read(self, logger: logging.Logger, config: TConfig, catalog: TCatalog, state: Optional[TState] = None) -> Iterable[AirbyteMessage]: """ Returns a generator of the AirbyteMessages generated by reading the source with the given configuration, catalog, and state. """ @@ -43,8 +43,9 @@ class Source( BaseSource[Mapping[str, Any], Union[List[AirbyteStateMessage], MutableMapping[str, Any]], ConfiguredAirbyteCatalog], ABC, ): - # can be overridden to change an input state - def read_state(self, state_path: str) -> Union[List[AirbyteStateMessage], MutableMapping[str, Any]]: + # can be overridden to change an input state. + @classmethod + def read_state(cls, state_path: str) -> Union[List[AirbyteStateMessage], MutableMapping[str, Any]]: """ Retrieves the input state of a sync by reading from the specified JSON file. Incoming state can be deserialized into either a JSON object for legacy state input or as a list of AirbyteStateMessages for the per-stream state format. Regardless of the @@ -53,30 +54,30 @@ def read_state(self, state_path: str) -> Union[List[AirbyteStateMessage], Mutabl :return: The complete stream state based on the connector's previous sync """ if state_path: - state_obj = self._read_json_file(state_path) + state_obj = BaseConnector._read_json_file(state_path) if not state_obj: - return self._emit_legacy_state_format({}) - is_per_stream_state = isinstance(state_obj, List) - if is_per_stream_state: + return cls._emit_legacy_state_format({}) + if isinstance(state_obj, List): parsed_state_messages = [] - for state in state_obj: + for state in state_obj: # type: ignore # `isinstance(state_obj, List)` ensures that this is a list parsed_message = AirbyteStateMessage.parse_obj(state) if not parsed_message.stream and not parsed_message.data and not parsed_message.global_: raise ValueError("AirbyteStateMessage should contain either a stream, global, or state field") parsed_state_messages.append(parsed_message) return parsed_state_messages else: - return self._emit_legacy_state_format(state_obj) - return self._emit_legacy_state_format({}) + return cls._emit_legacy_state_format(state_obj) # type: ignore # assuming it is a dict + return cls._emit_legacy_state_format({}) - def _emit_legacy_state_format(self, state_obj) -> Union[List[AirbyteStateMessage], MutableMapping[str, Any]]: + @classmethod + def _emit_legacy_state_format(cls, state_obj: Dict[str, Any]) -> Union[List[AirbyteStateMessage], MutableMapping[str, Any]]: """ Existing connectors that override read() might not be able to interpret the new state format. We temporarily send state in the old format for these connectors, but once all have been upgraded, this method can be removed, and we can then emit state in the list format. """ # vars(self.__class__) checks if the current class directly overrides the read() function - if "read" in vars(self.__class__): + if "read" in vars(cls): return defaultdict(dict, state_obj) else: if state_obj: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/__init__.py index 0df89f871a52..9326fd1bdca7 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # # Initialize Streams Package diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/call_rate.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/call_rate.py new file mode 100644 index 000000000000..eb33754504ee --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/call_rate.py @@ -0,0 +1,523 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import abc +import dataclasses +import datetime +import logging +import time +from datetime import timedelta +from threading import RLock +from typing import TYPE_CHECKING, Any, Mapping, Optional +from urllib import parse + +import requests +import requests_cache +from pyrate_limiter import InMemoryBucket, Limiter +from pyrate_limiter import Rate as PyRateRate +from pyrate_limiter import RateItem, TimeClock +from pyrate_limiter.exceptions import BucketFullException + +# prevents mypy from complaining about missing session attributes in LimiterMixin +if TYPE_CHECKING: + MIXIN_BASE = requests.Session +else: + MIXIN_BASE = object + +logger = logging.getLogger("airbyte") + + +@dataclasses.dataclass +class Rate: + """Call rate limit""" + + limit: int + interval: timedelta + + +class CallRateLimitHit(Exception): + def __init__(self, error: str, item: Any, weight: int, rate: str, time_to_wait: timedelta): + """Constructor + + :param error: error message + :param item: object passed into acquire_call + :param weight: how many credits were requested + :param rate: string representation of the rate violated + :param time_to_wait: how long should wait util more call will be available + """ + self.item = item + self.weight = weight + self.rate = rate + self.time_to_wait = time_to_wait + super().__init__(error) + + +class AbstractCallRatePolicy(abc.ABC): + """Call rate policy interface. + Should be configurable with different rules, like N per M for endpoint X. Endpoint X is matched with APIBudget. + """ + + @abc.abstractmethod + def matches(self, request: Any) -> bool: + """Tells if this policy matches specific request and should apply to it + + :param request: + :return: True if policy should apply to this request, False - otherwise + """ + + @abc.abstractmethod + def try_acquire(self, request: Any, weight: int) -> None: + """Try to acquire request + + :param request: a request object representing a single call to API + :param weight: number of requests to deduct from credit + :return: + """ + + @abc.abstractmethod + def update(self, available_calls: Optional[int], call_reset_ts: Optional[datetime.datetime]) -> None: + """Update call rate counting with current values + + :param available_calls: + :param call_reset_ts: + """ + + +class RequestMatcher(abc.ABC): + """Callable that help to match a request object with call rate policies.""" + + @abc.abstractmethod + def __call__(self, request: Any) -> bool: + """ + + :param request: + :return: True if matches the provided request object, False - otherwise + """ + + +class HttpRequestMatcher(RequestMatcher): + """Simple implementation of RequestMatcher for http requests case""" + + def __init__( + self, + method: Optional[str] = None, + url: Optional[str] = None, + params: Optional[Mapping[str, Any]] = None, + headers: Optional[Mapping[str, Any]] = None, + ): + """Constructor + + :param method: + :param url: + :param params: + :param headers: + """ + self._method = method + self._url = url + self._params = {str(k): str(v) for k, v in (params or {}).items()} + self._headers = {str(k): str(v) for k, v in (headers or {}).items()} + + @staticmethod + def _match_dict(obj: Mapping[str, Any], pattern: Mapping[str, Any]) -> bool: + """Check that all elements from pattern dict present and have the same values in obj dict + + :param obj: + :param pattern: + :return: + """ + return pattern.items() <= obj.items() + + def __call__(self, request: Any) -> bool: + """ + + :param request: + :return: True if matches the provided request object, False - otherwise + """ + if isinstance(request, requests.Request): + prepared_request = request.prepare() + elif isinstance(request, requests.PreparedRequest): + prepared_request = request + else: + return False + + if self._method is not None: + if prepared_request.method != self._method: + return False + if self._url is not None and prepared_request.url is not None: + url_without_params = prepared_request.url.split("?")[0] + if url_without_params != self._url: + return False + if self._params is not None: + parsed_url = parse.urlsplit(prepared_request.url) + params = dict(parse.parse_qsl(str(parsed_url.query))) + if not self._match_dict(params, self._params): + return False + if self._headers is not None: + if not self._match_dict(prepared_request.headers, self._headers): + return False + return True + + +class BaseCallRatePolicy(AbstractCallRatePolicy, abc.ABC): + def __init__(self, matchers: list[RequestMatcher]): + self._matchers = matchers + + def matches(self, request: Any) -> bool: + """Tell if this policy matches specific request and should apply to it + + :param request: + :return: True if policy should apply to this request, False - otherwise + """ + + if not self._matchers: + return True + return any(matcher(request) for matcher in self._matchers) + + +class UnlimitedCallRatePolicy(BaseCallRatePolicy): + """ + This policy is for explicit unlimited call rates. + It can be used when we want to match a specific group of requests and don't apply any limits. + + Example: + + APICallBudget( + [ + UnlimitedCallRatePolicy( + matchers=[HttpRequestMatcher(url="/some/method", headers={"sandbox": true})], + ), + FixedWindowCallRatePolicy( + matchers=[HttpRequestMatcher(url="/some/method")], + next_reset_ts=datetime.now(), + period=timedelta(hours=1) + call_limit=1000, + ), + ] + ) + + The code above will limit all calls to /some/method except calls that have header sandbox=True + """ + + def try_acquire(self, request: Any, weight: int) -> None: + """Do nothing""" + + def update(self, available_calls: Optional[int], call_reset_ts: Optional[datetime.datetime]) -> None: + """Do nothing""" + + +class FixedWindowCallRatePolicy(BaseCallRatePolicy): + def __init__(self, next_reset_ts: datetime.datetime, period: timedelta, call_limit: int, matchers: list[RequestMatcher]): + """A policy that allows {call_limit} calls within a {period} time interval + + :param next_reset_ts: next call rate reset time point + :param period: call rate reset period + :param call_limit: + :param matchers: + """ + + self._next_reset_ts = next_reset_ts + self._offset = period + self._call_limit = call_limit + self._calls_num = 0 + self._lock = RLock() + super().__init__(matchers=matchers) + + def try_acquire(self, request: Any, weight: int) -> None: + if weight > self._call_limit: + raise ValueError("Weight can not exceed the call limit") + if not self.matches(request): + raise ValueError("Request does not match the policy") + + with self._lock: + self._update_current_window() + + if self._calls_num + weight > self._call_limit: + reset_in = self._next_reset_ts - datetime.datetime.now() + error_message = ( + f"reached maximum number of allowed calls {self._call_limit} " f"per {self._offset} interval, next reset in {reset_in}." + ) + raise CallRateLimitHit( + error=error_message, + item=request, + weight=weight, + rate=f"{self._call_limit} per {self._offset}", + time_to_wait=reset_in, + ) + + self._calls_num += weight + + def update(self, available_calls: Optional[int], call_reset_ts: Optional[datetime.datetime]) -> None: + """Update call rate counters, by default, only reacts to decreasing updates of available_calls and changes to call_reset_ts. + We ignore updates with available_calls > current_available_calls to support call rate limits that are lower than API limits. + + :param available_calls: + :param call_reset_ts: + """ + with self._lock: + self._update_current_window() + current_available_calls = self._call_limit - self._calls_num + + if available_calls is not None and current_available_calls > available_calls: + logger.debug( + "got rate limit update from api, adjusting available calls from %s to %s", current_available_calls, available_calls + ) + self._calls_num = self._call_limit - available_calls + + if call_reset_ts is not None and call_reset_ts != self._next_reset_ts: + logger.debug("got rate limit update from api, adjusting reset time from %s to %s", self._next_reset_ts, call_reset_ts) + self._next_reset_ts = call_reset_ts + + def _update_current_window(self) -> None: + now = datetime.datetime.now() + if now > self._next_reset_ts: + logger.debug("started new window, %s calls available now", self._call_limit) + self._next_reset_ts = self._next_reset_ts + self._offset + self._calls_num = 0 + + +class MovingWindowCallRatePolicy(BaseCallRatePolicy): + """ + Policy to control requests rate implemented on top of PyRateLimiter lib. + The main difference between this policy and FixedWindowCallRatePolicy is that the rate-limiting window + is moving along requests that we made, and there is no moment when we reset an available number of calls. + This strategy requires saving of timestamps of all requests within a window. + """ + + def __init__(self, rates: list[Rate], matchers: list[RequestMatcher]): + """Constructor + + :param rates: list of rates, the order is important and must be ascending + :param matchers: + """ + if not rates: + raise ValueError("The list of rates can not be empty") + pyrate_rates = [PyRateRate(limit=rate.limit, interval=int(rate.interval.total_seconds() * 1000)) for rate in rates] + self._bucket = InMemoryBucket(pyrate_rates) + # Limiter will create the background task that clears old requests in the bucket + self._limiter = Limiter(self._bucket) + super().__init__(matchers=matchers) + + def try_acquire(self, request: Any, weight: int) -> None: + if not self.matches(request): + raise ValueError("Request does not match the policy") + + try: + self._limiter.try_acquire(request, weight=weight) + except BucketFullException as exc: + item = self._limiter.bucket_factory.wrap_item(request, weight) + assert isinstance(item, RateItem) + + with self._limiter.lock: + time_to_wait = self._bucket.waiting(item) + assert isinstance(time_to_wait, int) + + raise CallRateLimitHit( + error=str(exc.meta_info["error"]), + item=request, + weight=int(exc.meta_info["weight"]), + rate=str(exc.meta_info["rate"]), + time_to_wait=timedelta(milliseconds=time_to_wait), + ) + + def update(self, available_calls: Optional[int], call_reset_ts: Optional[datetime.datetime]) -> None: + """Adjust call bucket to reflect the state of the API server + + :param available_calls: + :param call_reset_ts: + :return: + """ + if available_calls is not None and call_reset_ts is None: # we do our best to sync buckets with API + if available_calls == 0: + with self._limiter.lock: + items_to_add = self._bucket.count() < self._bucket.rates[0].limit + if items_to_add > 0: + now: int = TimeClock().now() # type: ignore[no-untyped-call] + self._bucket.put(RateItem(name="dummy", timestamp=now, weight=items_to_add)) + # TODO: add support if needed, it might be that it is not possible to make a good solution for this case + # if available_calls is not None and call_reset_ts is not None: + # ts = call_reset_ts.timestamp() + + +class AbstractAPIBudget(abc.ABC): + """Interface to some API where a client allowed to have N calls per T interval. + + Important: APIBudget is not doing any API calls, the end user code is responsible to call this interface + to respect call rate limitation of the API. + + It supports multiple policies applied to different group of requests. To distinct these groups we use RequestMatchers. + Individual policy represented by MovingWindowCallRatePolicy and currently supports only moving window strategy. + """ + + @abc.abstractmethod + def acquire_call(self, request: Any, block: bool = True, timeout: Optional[float] = None) -> None: + """Try to get a call from budget, will block by default + + :param request: + :param block: when true (default) will block the current thread until call credit is available + :param timeout: if set will limit maximum time in block, otherwise will wait until credit is available + :raises: CallRateLimitHit - when no credits left and if timeout was set the waiting time exceed the timeout + """ + + @abc.abstractmethod + def get_matching_policy(self, request: Any) -> Optional[AbstractCallRatePolicy]: + """Find matching call rate policy for specific request""" + + @abc.abstractmethod + def update_from_response(self, request: Any, response: Any) -> None: + """Update budget information based on response from API + + :param request: the initial request that triggered this response + :param response: response from the API + """ + + +class APIBudget(AbstractAPIBudget): + """Default APIBudget implementation""" + + def __init__(self, policies: list[AbstractCallRatePolicy], maximum_attempts_to_acquire: int = 100000) -> None: + """Constructor + + :param policies: list of policies in this budget + :param maximum_attempts_to_acquire: number of attempts before throwing hit ratelimit exception, we put some big number here + to avoid situations when many threads compete with each other for a few lots over a significant amount of time + """ + + self._policies = policies + self._maximum_attempts_to_acquire = maximum_attempts_to_acquire + + def get_matching_policy(self, request: Any) -> Optional[AbstractCallRatePolicy]: + for policy in self._policies: + if policy.matches(request): + return policy + return None + + def acquire_call(self, request: Any, block: bool = True, timeout: Optional[float] = None) -> None: + """Try to get a call from budget, will block by default. + Matchers will be called sequentially in the same order they were added. + The first matcher that returns True will + + :param request: + :param block: when true (default) will block the current thread until call credit is available + :param timeout: if provided will limit maximum time in block, otherwise will wait until credit is available + :raises: CallRateLimitHit - when no calls left and if timeout was set the waiting time exceed the timeout + """ + + policy = self.get_matching_policy(request) + if policy: + self._do_acquire(request=request, policy=policy, block=block, timeout=timeout) + elif self._policies: + logger.info("no policies matched with requests, allow call by default") + + def update_from_response(self, request: Any, response: Any) -> None: + """Update budget information based on response from API + + :param request: the initial request that triggered this response + :param response: response from the API + """ + pass + + def _do_acquire(self, request: Any, policy: AbstractCallRatePolicy, block: bool, timeout: Optional[float]) -> None: + """Internal method to try to acquire a call credit + + :param request: + :param policy: + :param block: + :param timeout: + """ + last_exception = None + # sometimes we spend all budget before a second attempt, so we have few more here + for attempt in range(1, self._maximum_attempts_to_acquire): + try: + policy.try_acquire(request, weight=1) + return + except CallRateLimitHit as exc: + last_exception = exc + if block: + if timeout is not None: + time_to_wait = min(timedelta(seconds=timeout), exc.time_to_wait) + else: + time_to_wait = exc.time_to_wait + + time_to_wait = max(timedelta(0), time_to_wait) # sometimes we get negative duration + logger.info("reached call limit %s. going to sleep for %s", exc.rate, time_to_wait) + time.sleep(time_to_wait.total_seconds()) + else: + raise + + if last_exception: + logger.info("we used all %s attempts to acquire and failed", self._maximum_attempts_to_acquire) + raise last_exception + + +class HttpAPIBudget(APIBudget): + """Implementation of AbstractAPIBudget for HTTP""" + + def __init__( + self, + ratelimit_reset_header: str = "ratelimit-reset", + ratelimit_remaining_header: str = "ratelimit-remaining", + status_codes_for_ratelimit_hit: tuple[int] = (429,), + **kwargs: Any, + ): + """Constructor + + :param ratelimit_reset_header: name of the header that has a timestamp of the next reset of call budget + :param ratelimit_remaining_header: name of the header that has the number of calls left + :param status_codes_for_ratelimit_hit: list of HTTP status codes that signal about rate limit being hit + """ + self._ratelimit_reset_header = ratelimit_reset_header + self._ratelimit_remaining_header = ratelimit_remaining_header + self._status_codes_for_ratelimit_hit = status_codes_for_ratelimit_hit + super().__init__(**kwargs) + + def update_from_response(self, request: Any, response: Any) -> None: + policy = self.get_matching_policy(request) + if not policy: + return + + if isinstance(response, requests.Response): + available_calls = self.get_calls_left_from_response(response) + reset_ts = self.get_reset_ts_from_response(response) + policy.update(available_calls=available_calls, call_reset_ts=reset_ts) + + def get_reset_ts_from_response(self, response: requests.Response) -> Optional[datetime.datetime]: + if response.headers.get(self._ratelimit_reset_header): + return datetime.datetime.fromtimestamp(int(response.headers[self._ratelimit_reset_header])) + return None + + def get_calls_left_from_response(self, response: requests.Response) -> Optional[int]: + if response.headers.get(self._ratelimit_remaining_header): + return int(response.headers[self._ratelimit_remaining_header]) + + if response.status_code in self._status_codes_for_ratelimit_hit: + return 0 + + return None + + +class LimiterMixin(MIXIN_BASE): + """Mixin class that adds rate-limiting behavior to requests.""" + + def __init__( + self, + api_budget: AbstractAPIBudget, + **kwargs: Any, + ): + self._api_budget = api_budget + super().__init__(**kwargs) # type: ignore # Base Session doesn't take any kwargs + + def send(self, request: requests.PreparedRequest, **kwargs: Any) -> requests.Response: + """Send a request with rate-limiting.""" + self._api_budget.acquire_call(request) + response = super().send(request, **kwargs) + self._api_budget.update_from_response(request, response) + return response + + +class LimiterSession(LimiterMixin, requests.Session): + """Session that adds rate-limiting behavior to requests.""" + + +class CachedLimiterSession(requests_cache.CacheMixin, LimiterMixin, requests.Session): + """Session class with caching and rate-limiting behavior.""" diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/README.md b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/README.md new file mode 100644 index 000000000000..436230cbd614 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/README.md @@ -0,0 +1,7 @@ +## Breaking Changes & Limitations + +* [bigger scope than Concurrent CDK] checkpointing state was acting on the number of records per slice. This has been changed to consider the number of records per syncs +* `Source.read_state` and `Source._emit_legacy_state_format` are now classmethods to allow for developers to have access to the state before instantiating the source +* send_per_stream_state is always True for Concurrent CDK +* Using stream_state during read_records: The concern is that today, stream_instance.get_updated_state is called on every record and read_records on every slice. The implication is that the argument stream_state passed to read_records will have the value after the last stream_instance.get_updated_state of the previous slice. For Concurrent CDK, this is not possible as slices are processed in an unordered way. +* Cursor fields can only be data-time formatted as epoch. Eventually, we want to move to ISO 8601 as it provides more flexibility but for the first iteration on Stripe, it was easier to use the same format that was already used diff --git a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/__init__.py similarity index 100% rename from airbyte-ci/connectors/pipelines/pipelines/commands/groups/__init__.py rename to airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/__init__.py diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/abstract_stream.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/abstract_stream.py new file mode 100644 index 000000000000..d98e7a7b5498 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/abstract_stream.py @@ -0,0 +1,83 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import ABC, abstractmethod +from typing import Any, Iterable, Mapping, Optional + +from airbyte_cdk.models import AirbyteStream +from airbyte_cdk.sources.streams.concurrent.availability_strategy import StreamAvailability +from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition +from deprecated.classic import deprecated + + +@deprecated("This class is experimental. Use at your own risk.") +class AbstractStream(ABC): + """ + AbstractStream is an experimental interface for streams developed as part of the Concurrent CDK. + This interface is not yet stable and may change in the future. Use at your own risk. + + Why create a new interface instead of adding concurrency capabilities the existing Stream? + We learnt a lot since the initial design of the Stream interface, and we wanted to take the opportunity to improve. + + High level, the changes we are targeting are: + - Removing superfluous or leaky parameters from the methods' interfaces + - Using composition instead of inheritance to add new capabilities + + To allow us to iterate fast while ensuring backwards compatibility, we are creating a new interface with a facade object that will bridge the old and the new interfaces. + Source connectors that wish to leverage concurrency need to implement this new interface. An example will be available shortly + + Current restrictions on sources that implement this interface. Not all of these restrictions will be lifted in the future, but most will as we iterate on the design. + - Only full refresh is supported. This will be addressed in the future. + - The read method does not accept a cursor_field. Streams must be internally aware of the cursor field to use. User-defined cursor fields can be implemented by modifying the connector's main method to instantiate the streams with the configured cursor field. + - Streams cannot return user-friendly messages by overriding Stream.get_error_display_message. This will be addressed in the future. + - The Stream's behavior cannot depend on a namespace + - TypeTransformer is not supported. This will be addressed in the future. + - Nested cursor and primary keys are not supported + """ + + @abstractmethod + def generate_partitions(self) -> Iterable[Partition]: + """ + Generates the partitions that will be read by this stream. + :return: An iterable of partitions. + """ + + @property + @abstractmethod + def name(self) -> str: + """ + :return: The stream name + """ + + @property + @abstractmethod + def cursor_field(self) -> Optional[str]: + """ + Override to return the default cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. + :return: The name of the field used as a cursor. Nested cursor fields are not supported. + """ + + @abstractmethod + def check_availability(self) -> StreamAvailability: + """ + :return: The stream's availability + """ + + @abstractmethod + def get_json_schema(self) -> Mapping[str, Any]: + """ + :return: A dict of the JSON schema representing this stream. + """ + + @abstractmethod + def as_airbyte_stream(self) -> AirbyteStream: + """ + :return: A dict of the JSON schema representing this stream. + """ + + @abstractmethod + def log_stream_sync_configuration(self) -> None: + """ + Logs the stream's configuration for debugging purposes. + """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/adapters.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/adapters.py new file mode 100644 index 000000000000..f8a5e3ed65e3 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/adapters.py @@ -0,0 +1,437 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import copy +import json +import logging +from functools import lru_cache +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union + +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, AirbyteStream, Level, SyncMode, Type +from airbyte_cdk.sources import AbstractSource, Source +from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager +from airbyte_cdk.sources.message import MessageRepository +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy +from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream +from airbyte_cdk.sources.streams.concurrent.availability_strategy import ( + AbstractAvailabilityStrategy, + StreamAvailability, + StreamAvailable, + StreamUnavailable, +) +from airbyte_cdk.sources.streams.concurrent.cursor import Cursor, NoopCursor +from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream +from airbyte_cdk.sources.streams.concurrent.exceptions import ExceptionWithDisplayMessage +from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition +from airbyte_cdk.sources.streams.concurrent.partitions.partition_generator import PartitionGenerator +from airbyte_cdk.sources.streams.concurrent.partitions.record import Record +from airbyte_cdk.sources.streams.core import StreamData +from airbyte_cdk.sources.utils.schema_helpers import InternalConfig +from airbyte_cdk.sources.utils.slice_logger import SliceLogger +from deprecated.classic import deprecated + +""" +This module contains adapters to help enabling concurrency on Stream objects without needing to migrate to AbstractStream +""" + + +@deprecated("This class is experimental. Use at your own risk.") +class StreamFacade(Stream): + """ + The StreamFacade is a Stream that wraps an AbstractStream and exposes it as a Stream. + + All methods either delegate to the wrapped AbstractStream or provide a default implementation. + The default implementations define restrictions imposed on Streams migrated to the new interface. For instance, only source-defined cursors are supported. + """ + + @classmethod + def create_from_stream( + cls, + stream: Stream, + source: AbstractSource, + logger: logging.Logger, + state: Optional[MutableMapping[str, Any]], + cursor: Cursor, + ) -> Stream: + """ + Create a ConcurrentStream from a Stream object. + :param source: The source + :param stream: The stream + :param max_workers: The maximum number of worker thread to use + :return: + """ + pk = cls._get_primary_key_from_stream(stream.primary_key) + cursor_field = cls._get_cursor_field_from_stream(stream) + + if not source.message_repository: + raise ValueError( + "A message repository is required to emit non-record messages. Please set the message repository on the source." + ) + + message_repository = source.message_repository + return StreamFacade( + DefaultStream( + partition_generator=StreamPartitionGenerator( + stream, + message_repository, + SyncMode.full_refresh if isinstance(cursor, NoopCursor) else SyncMode.incremental, + [cursor_field] if cursor_field is not None else None, + state, + cursor, + ), + name=stream.name, + namespace=stream.namespace, + json_schema=stream.get_json_schema(), + availability_strategy=StreamAvailabilityStrategy(stream, source), + primary_key=pk, + cursor_field=cursor_field, + logger=logger, + ), + stream, + cursor, + slice_logger=source._slice_logger, + logger=logger, + ) + + @property + def state(self) -> MutableMapping[str, Any]: + raise NotImplementedError("This should not be called as part of the Concurrent CDK code. Please report the problem to Airbyte") + + @state.setter + def state(self, value: Mapping[str, Any]) -> None: + if "state" in dir(self._legacy_stream): + self._legacy_stream.state = value # type: ignore # validating `state` is attribute of stream using `if` above + + @classmethod + def _get_primary_key_from_stream(cls, stream_primary_key: Optional[Union[str, List[str], List[List[str]]]]) -> List[str]: + if stream_primary_key is None: + return [] + elif isinstance(stream_primary_key, str): + return [stream_primary_key] + elif isinstance(stream_primary_key, list): + if len(stream_primary_key) > 0 and all(isinstance(k, str) for k in stream_primary_key): + return stream_primary_key # type: ignore # We verified all items in the list are strings + else: + raise ValueError(f"Nested primary keys are not supported. Found {stream_primary_key}") + else: + raise ValueError(f"Invalid type for primary key: {stream_primary_key}") + + @classmethod + def _get_cursor_field_from_stream(cls, stream: Stream) -> Optional[str]: + if isinstance(stream.cursor_field, list): + if len(stream.cursor_field) > 1: + raise ValueError(f"Nested cursor fields are not supported. Got {stream.cursor_field} for {stream.name}") + elif len(stream.cursor_field) == 0: + return None + else: + return stream.cursor_field[0] + else: + return stream.cursor_field + + def __init__(self, stream: AbstractStream, legacy_stream: Stream, cursor: Cursor, slice_logger: SliceLogger, logger: logging.Logger): + """ + :param stream: The underlying AbstractStream + """ + self._abstract_stream = stream + self._legacy_stream = legacy_stream + self._cursor = cursor + self._slice_logger = slice_logger + self._logger = logger + + def read_full_refresh( + self, + cursor_field: Optional[List[str]], + logger: logging.Logger, + slice_logger: SliceLogger, + ) -> Iterable[StreamData]: + """ + Read full refresh. Delegate to the underlying AbstractStream, ignoring all the parameters + :param cursor_field: (ignored) + :param logger: (ignored) + :param slice_logger: (ignored) + :return: Iterable of StreamData + """ + yield from self._read_records() + + def read_incremental( + self, + cursor_field: Optional[List[str]], + logger: logging.Logger, + slice_logger: SliceLogger, + stream_state: MutableMapping[str, Any], + state_manager: ConnectorStateManager, + per_stream_state_enabled: bool, + internal_config: InternalConfig, + ) -> Iterable[StreamData]: + yield from self._read_records() + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: Optional[List[str]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + stream_state: Optional[Mapping[str, Any]] = None, + ) -> Iterable[StreamData]: + try: + yield from self._read_records() + except Exception as exc: + if hasattr(self._cursor, "state"): + state = self._cursor.state + else: + # This shouldn't happen if the ConcurrentCursor was used + state = "unknown; no state attribute was available on the cursor" + yield AirbyteMessage( + type=Type.LOG, log=AirbyteLogMessage(level=Level.ERROR, message=f"Cursor State at time of exception: {state}") + ) + raise exc + + def _read_records(self) -> Iterable[StreamData]: + for partition in self._abstract_stream.generate_partitions(): + if self._slice_logger.should_log_slice_message(self._logger): + yield self._slice_logger.create_slice_log_message(partition.to_slice()) + for record in partition.read(): + yield record.data + + @property + def name(self) -> str: + return self._abstract_stream.name + + @property + def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: + # This method is not expected to be called directly. It is only implemented for backward compatibility with the old interface + return self.as_airbyte_stream().source_defined_primary_key # type: ignore # source_defined_primary_key is known to be an Optional[List[List[str]]] + + @property + def cursor_field(self) -> Union[str, List[str]]: + if self._abstract_stream.cursor_field is None: + return [] + else: + return self._abstract_stream.cursor_field + + @property + def source_defined_cursor(self) -> bool: + # Streams must be aware of their cursor at instantiation time + return True + + @lru_cache(maxsize=None) + def get_json_schema(self) -> Mapping[str, Any]: + return self._abstract_stream.get_json_schema() + + @property + def supports_incremental(self) -> bool: + return self._legacy_stream.supports_incremental + + def check_availability(self, logger: logging.Logger, source: Optional["Source"] = None) -> Tuple[bool, Optional[str]]: + """ + Verifies the stream is available. Delegates to the underlying AbstractStream and ignores the parameters + :param logger: (ignored) + :param source: (ignored) + :return: + """ + availability = self._abstract_stream.check_availability() + return availability.is_available(), availability.message() + + def get_error_display_message(self, exception: BaseException) -> Optional[str]: + """ + Retrieves the user-friendly display message that corresponds to an exception. + This will be called when encountering an exception while reading records from the stream, and used to build the AirbyteTraceMessage. + + A display message will be returned if the exception is an instance of ExceptionWithDisplayMessage. + + :param exception: The exception that was raised + :return: A user-friendly message that indicates the cause of the error + """ + if isinstance(exception, ExceptionWithDisplayMessage): + return exception.display_message + else: + return None + + def as_airbyte_stream(self) -> AirbyteStream: + return self._abstract_stream.as_airbyte_stream() + + def log_stream_sync_configuration(self) -> None: + self._abstract_stream.log_stream_sync_configuration() + + +class StreamPartition(Partition): + """ + This class acts as an adapter between the new Partition interface and the Stream's stream_slice interface + + StreamPartitions are instantiated from a Stream and a stream_slice. + + This class can be used to help enable concurrency on existing connectors without having to rewrite everything as AbstractStream. + In the long-run, it would be preferable to update the connectors, but we don't have the tooling or need to justify the effort at this time. + """ + + def __init__( + self, + stream: Stream, + _slice: Optional[Mapping[str, Any]], + message_repository: MessageRepository, + sync_mode: SyncMode, + cursor_field: Optional[List[str]], + state: Optional[MutableMapping[str, Any]], + cursor: Cursor, + ): + """ + :param stream: The stream to delegate to + :param _slice: The partition's stream_slice + :param message_repository: The message repository to use to emit non-record messages + """ + self._stream = stream + self._slice = _slice + self._message_repository = message_repository + self._sync_mode = sync_mode + self._cursor_field = cursor_field + self._state = state + self._cursor = cursor + self._is_closed = False + + def read(self) -> Iterable[Record]: + """ + Read messages from the stream. + If the StreamData is a Mapping, it will be converted to a Record. + Otherwise, the message will be emitted on the message repository. + """ + try: + # using `stream_state=self._state` have a very different behavior than the current one as today the state is updated slice + # by slice incrementally. We don't have this guarantee with Concurrent CDK. For HttpStream, `stream_state` is passed to: + # * fetch_next_page + # * parse_response + # Both are not used for Stripe so we should be good for the first iteration of Concurrent CDK. However, Stripe still do + # `if not stream_state` to know if it calls the Event stream or not + for record_data in self._stream.read_records( + cursor_field=self._cursor_field, + sync_mode=SyncMode.full_refresh, + stream_slice=copy.deepcopy(self._slice), + stream_state=self._state, + ): + if isinstance(record_data, Mapping): + data_to_return = dict(record_data) + self._stream.transformer.transform(data_to_return, self._stream.get_json_schema()) + record = Record(data_to_return, self._stream.name) + self._cursor.observe(record) + yield Record(data_to_return, self._stream.name) + else: + self._message_repository.emit_message(record_data) + except Exception as e: + display_message = self._stream.get_error_display_message(e) + if display_message: + raise ExceptionWithDisplayMessage(display_message) from e + else: + raise e + + def to_slice(self) -> Optional[Mapping[str, Any]]: + return self._slice + + def __hash__(self) -> int: + if self._slice: + # Convert the slice to a string so that it can be hashed + s = json.dumps(self._slice, sort_keys=True) + return hash((self._stream.name, s)) + else: + return hash(self._stream.name) + + def stream_name(self) -> str: + return self._stream.name + + def close(self) -> None: + self._cursor.close_partition(self) + self._is_closed = True + + def is_closed(self) -> bool: + return self._is_closed + + def __repr__(self) -> str: + return f"StreamPartition({self._stream.name}, {self._slice})" + + +class StreamPartitionGenerator(PartitionGenerator): + """ + This class acts as an adapter between the new PartitionGenerator and Stream.stream_slices + + This class can be used to help enable concurrency on existing connectors without having to rewrite everything as AbstractStream. + In the long-run, it would be preferable to update the connectors, but we don't have the tooling or need to justify the effort at this time. + """ + + def __init__( + self, + stream: Stream, + message_repository: MessageRepository, + sync_mode: SyncMode, + cursor_field: Optional[List[str]], + state: Optional[MutableMapping[str, Any]], + cursor: Cursor, + ): + """ + :param stream: The stream to delegate to + :param message_repository: The message repository to use to emit non-record messages + """ + self.message_repository = message_repository + self._stream = stream + self._sync_mode = sync_mode + self._cursor_field = cursor_field + self._state = state + self._cursor = cursor + + def generate(self) -> Iterable[Partition]: + for s in self._stream.stream_slices(sync_mode=self._sync_mode, cursor_field=self._cursor_field, stream_state=self._state): + yield StreamPartition( + self._stream, copy.deepcopy(s), self.message_repository, self._sync_mode, self._cursor_field, self._state, self._cursor + ) + + +@deprecated("This class is experimental. Use at your own risk.") +class AvailabilityStrategyFacade(AvailabilityStrategy): + def __init__(self, abstract_availability_strategy: AbstractAvailabilityStrategy): + self._abstract_availability_strategy = abstract_availability_strategy + + def check_availability(self, stream: Stream, logger: logging.Logger, source: Optional[Source]) -> Tuple[bool, Optional[str]]: + """ + Checks stream availability. + + Important to note that the stream and source parameters are not used by the underlying AbstractAvailabilityStrategy. + + :param stream: (unused) + :param logger: logger object to use + :param source: (unused) + :return: A tuple of (boolean, str). If boolean is true, then the stream + """ + stream_availability = self._abstract_availability_strategy.check_availability(logger) + return stream_availability.is_available(), stream_availability.message() + + +class StreamAvailabilityStrategy(AbstractAvailabilityStrategy): + """ + This class acts as an adapter between the existing AvailabilityStrategy and the new AbstractAvailabilityStrategy. + StreamAvailabilityStrategy is instantiated with a Stream and a Source to allow the existing AvailabilityStrategy to be used with the new AbstractAvailabilityStrategy interface. + + A more convenient implementation would not depend on the docs URL instead of the Source itself, and would support running on an AbstractStream instead of only on a Stream. + + This class can be used to help enable concurrency on existing connectors without having to rewrite everything as AbstractStream and AbstractAvailabilityStrategy. + In the long-run, it would be preferable to update the connectors, but we don't have the tooling or need to justify the effort at this time. + """ + + def __init__(self, stream: Stream, source: Source): + """ + :param stream: The stream to delegate to + :param source: The source to delegate to + """ + self._stream = stream + self._source = source + + def check_availability(self, logger: logging.Logger) -> StreamAvailability: + try: + available, message = self._stream.check_availability(logger, self._source) + if available: + return StreamAvailable() + else: + return StreamUnavailable(str(message)) + except Exception as e: + display_message = self._stream.get_error_display_message(e) + if display_message: + raise ExceptionWithDisplayMessage(display_message) + else: + raise e diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/availability_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/availability_strategy.py new file mode 100644 index 000000000000..b65803e09df2 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/availability_strategy.py @@ -0,0 +1,66 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from abc import ABC, abstractmethod +from typing import Optional + +from deprecated.classic import deprecated + + +class StreamAvailability(ABC): + @abstractmethod + def is_available(self) -> bool: + """ + :return: True if the stream is available. False if the stream is not + """ + + @abstractmethod + def message(self) -> Optional[str]: + """ + :return: A message describing why the stream is not available. If the stream is available, this should return None. + """ + + +class StreamAvailable(StreamAvailability): + def is_available(self) -> bool: + return True + + def message(self) -> Optional[str]: + return None + + +class StreamUnavailable(StreamAvailability): + def __init__(self, message: str): + self._message = message + + def is_available(self) -> bool: + return False + + def message(self) -> Optional[str]: + return self._message + + +# Singleton instances of StreamAvailability to avoid the overhead of creating new dummy objects +STREAM_AVAILABLE = StreamAvailable() + + +@deprecated("This class is experimental. Use at your own risk.") +class AbstractAvailabilityStrategy(ABC): + """ + AbstractAvailabilityStrategy is an experimental interface developed as part of the Concurrent CDK. + This interface is not yet stable and may change in the future. Use at your own risk. + + Why create a new interface instead of using the existing AvailabilityStrategy? + The existing AvailabilityStrategy is tightly coupled with Stream and Source, which yields to circular dependencies and makes it difficult to move away from the Stream interface to AbstractStream. + """ + + @abstractmethod + def check_availability(self, logger: logging.Logger) -> StreamAvailability: + """ + Checks stream availability. + + :param logger: logger object to use + :return: A StreamAvailability object describing the stream's availability + """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/cursor.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/cursor.py new file mode 100644 index 000000000000..282498db1783 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/cursor.py @@ -0,0 +1,181 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import functools +from abc import ABC, abstractmethod +from datetime import datetime +from typing import Any, List, Mapping, MutableMapping, Optional, Protocol, Tuple + +from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager +from airbyte_cdk.sources.message import MessageRepository +from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition +from airbyte_cdk.sources.streams.concurrent.partitions.record import Record +from airbyte_cdk.sources.streams.concurrent.state_converters.abstract_stream_state_converter import AbstractStreamStateConverter + + +def _extract_value(mapping: Mapping[str, Any], path: List[str]) -> Any: + return functools.reduce(lambda a, b: a[b], path, mapping) + + +class Comparable(Protocol): + """Protocol for annotating comparable types.""" + + @abstractmethod + def __lt__(self: "Comparable", other: "Comparable") -> bool: + pass + + +class CursorField: + def __init__(self, cursor_field_key: str) -> None: + self.cursor_field_key = cursor_field_key + + def extract_value(self, record: Record) -> Comparable: + cursor_value = record.data.get(self.cursor_field_key) + if cursor_value is None: + raise ValueError(f"Could not find cursor field {self.cursor_field_key} in record") + return cursor_value # type: ignore # we assume that the value the path points at is a comparable + + +class Cursor(ABC): + @property + @abstractmethod + def state(self) -> MutableMapping[str, Any]: + ... + + @abstractmethod + def observe(self, record: Record) -> None: + """ + Indicate to the cursor that the record has been emitted + """ + raise NotImplementedError() + + @abstractmethod + def close_partition(self, partition: Partition) -> None: + """ + Indicate to the cursor that the partition has been successfully processed + """ + raise NotImplementedError() + + +class NoopCursor(Cursor): + @property + def state(self) -> MutableMapping[str, Any]: + return {} + + def observe(self, record: Record) -> None: + pass + + def close_partition(self, partition: Partition) -> None: + pass + + +class ConcurrentCursor(Cursor): + _START_BOUNDARY = 0 + _END_BOUNDARY = 1 + + def __init__( + self, + stream_name: str, + stream_namespace: Optional[str], + stream_state: Any, + message_repository: MessageRepository, + connector_state_manager: ConnectorStateManager, + connector_state_converter: AbstractStreamStateConverter, + cursor_field: CursorField, + slice_boundary_fields: Optional[Tuple[str, str]], + start: Optional[Any], + ) -> None: + self._stream_name = stream_name + self._stream_namespace = stream_namespace + self._message_repository = message_repository + self._connector_state_converter = connector_state_converter + self._connector_state_manager = connector_state_manager + self._cursor_field = cursor_field + # To see some example where the slice boundaries might not be defined, check https://github.com/airbytehq/airbyte/blob/1ce84d6396e446e1ac2377362446e3fb94509461/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py#L363-L379 + self._slice_boundary_fields = slice_boundary_fields if slice_boundary_fields else tuple() + self._start = start + self._most_recent_record: Optional[Record] = None + self._has_closed_at_least_one_slice = False + self.start, self._concurrent_state = self._get_concurrent_state(stream_state) + + @property + def state(self) -> MutableMapping[str, Any]: + return self._concurrent_state + + def _get_concurrent_state(self, state: MutableMapping[str, Any]) -> Tuple[datetime, MutableMapping[str, Any]]: + if self._connector_state_converter.is_state_message_compatible(state): + return self._start or self._connector_state_converter.zero_value, self._connector_state_converter.deserialize(state) + return self._connector_state_converter.convert_from_sequential_state(self._cursor_field, state, self._start) + + def observe(self, record: Record) -> None: + if self._slice_boundary_fields: + # Given that slicing is done using the cursor field, we don't need to observe the record as we assume slices will describe what + # has been emitted. Assuming there is a chance that records might not be yet populated for the most recent slice, use a lookback + # window + return + + if not self._most_recent_record or self._extract_cursor_value(self._most_recent_record) < self._extract_cursor_value(record): + self._most_recent_record = record + + def _extract_cursor_value(self, record: Record) -> Any: + return self._connector_state_converter.parse_value(self._cursor_field.extract_value(record)) + + def close_partition(self, partition: Partition) -> None: + slice_count_before = len(self.state.get("slices", [])) + self._add_slice_to_state(partition) + if slice_count_before < len(self.state["slices"]): # only emit if at least one slice has been processed + self._merge_partitions() + self._emit_state_message() + self._has_closed_at_least_one_slice = True + + def _add_slice_to_state(self, partition: Partition) -> None: + if self._slice_boundary_fields: + if "slices" not in self.state: + raise RuntimeError( + f"The state for stream {self._stream_name} should have at least one slice to delineate the sync start time, but no slices are present. This is unexpected. Please contact Support." + ) + self.state["slices"].append( + { + "start": self._extract_from_slice(partition, self._slice_boundary_fields[self._START_BOUNDARY]), + "end": self._extract_from_slice(partition, self._slice_boundary_fields[self._END_BOUNDARY]), + } + ) + elif self._most_recent_record: + if self._has_closed_at_least_one_slice: + raise ValueError( + "Given that slice_boundary_fields is not defined and that per-partition state is not supported, only one slice is " + "expected." + ) + + self.state["slices"].append( + { + self._connector_state_converter.START_KEY: self.start, + self._connector_state_converter.END_KEY: self._extract_cursor_value(self._most_recent_record), + } + ) + + def _emit_state_message(self) -> None: + self._connector_state_manager.update_state_for_stream( + self._stream_name, + self._stream_namespace, + self._connector_state_converter.convert_to_sequential_state(self._cursor_field, self.state), + ) + # TODO: if we migrate stored state to the concurrent state format + # (aka stop calling self._connector_state_converter.convert_to_sequential_state`), we'll need to cast datetimes to string or + # int before emitting state + state_message = self._connector_state_manager.create_state_message( + self._stream_name, self._stream_namespace, send_per_stream_state=True + ) + self._message_repository.emit_message(state_message) + + def _merge_partitions(self) -> None: + self.state["slices"] = self._connector_state_converter.merge_intervals(self.state["slices"]) + + def _extract_from_slice(self, partition: Partition, key: str) -> Comparable: + try: + _slice = partition.to_slice() + if not _slice: + raise KeyError(f"Could not find key `{key}` in empty slice") + return self._connector_state_converter.parse_value(_slice[key]) # type: ignore # we expect the devs to specify a key that would return a Comparable + except KeyError as exception: + raise KeyError(f"Partition is expected to have key `{key}` but could not be found") from exception diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/default_stream.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/default_stream.py new file mode 100644 index 000000000000..8606d273bb4f --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/default_stream.py @@ -0,0 +1,79 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from functools import lru_cache +from logging import Logger +from typing import Any, Iterable, List, Mapping, Optional + +from airbyte_cdk.models import AirbyteStream, SyncMode +from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream +from airbyte_cdk.sources.streams.concurrent.availability_strategy import AbstractAvailabilityStrategy, StreamAvailability +from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition +from airbyte_cdk.sources.streams.concurrent.partitions.partition_generator import PartitionGenerator + + +class DefaultStream(AbstractStream): + def __init__( + self, + partition_generator: PartitionGenerator, + name: str, + json_schema: Mapping[str, Any], + availability_strategy: AbstractAvailabilityStrategy, + primary_key: List[str], + cursor_field: Optional[str], + logger: Logger, + namespace: Optional[str] = None, + ) -> None: + self._stream_partition_generator = partition_generator + self._name = name + self._json_schema = json_schema + self._availability_strategy = availability_strategy + self._primary_key = primary_key + self._cursor_field = cursor_field + self._logger = logger + self._namespace = namespace + + def generate_partitions(self) -> Iterable[Partition]: + yield from self._stream_partition_generator.generate() + + @property + def name(self) -> str: + return self._name + + def check_availability(self) -> StreamAvailability: + return self._availability_strategy.check_availability(self._logger) + + @property + def cursor_field(self) -> Optional[str]: + return self._cursor_field + + @lru_cache(maxsize=None) + def get_json_schema(self) -> Mapping[str, Any]: + return self._json_schema + + def as_airbyte_stream(self) -> AirbyteStream: + stream = AirbyteStream(name=self.name, json_schema=dict(self._json_schema), supported_sync_modes=[SyncMode.full_refresh]) + + if self._namespace: + stream.namespace = self._namespace + + if self._cursor_field: + stream.source_defined_cursor = True + stream.supported_sync_modes.append(SyncMode.incremental) + stream.default_cursor_field = [self._cursor_field] + + keys = self._primary_key + if keys and len(keys) > 0: + stream.source_defined_primary_key = [keys] + + return stream + + def log_stream_sync_configuration(self) -> None: + self._logger.debug( + f"Syncing stream instance: {self.name}", + extra={ + "primary_key": self._primary_key, + "cursor_field": self.cursor_field, + }, + ) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/exceptions.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/exceptions.py new file mode 100644 index 000000000000..c67c2c58311d --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/exceptions.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any + + +class ExceptionWithDisplayMessage(Exception): + """ + Exception that can be used to display a custom message to the user. + """ + + def __init__(self, display_message: str, **kwargs: Any): + super().__init__(**kwargs) + self.display_message = display_message diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py new file mode 100644 index 000000000000..342c7ae3eec2 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partition_enqueuer.py @@ -0,0 +1,38 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.concurrent_source.partition_generation_completed_sentinel import PartitionGenerationCompletedSentinel +from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream +from airbyte_cdk.sources.streams.concurrent.partitions.throttled_queue import ThrottledQueue + + +class PartitionEnqueuer: + """ + Generates partitions from a partition generator and puts them in a queue. + """ + + def __init__(self, queue: ThrottledQueue) -> None: + """ + :param queue: The queue to put the partitions in. + :param throttler: The throttler to use to throttle the partition generation. + """ + self._queue = queue + + def generate_partitions(self, stream: AbstractStream) -> None: + """ + Generate partitions from a partition generator and put them in a queue. + When all the partitions are added to the queue, a sentinel is added to the queue to indicate that all the partitions have been generated. + + If an exception is encountered, the exception will be caught and put in the queue. + + This method is meant to be called in a separate thread. + :param partition_generator: The partition Generator + :return: + """ + try: + for partition in stream.generate_partitions(): + self._queue.put(partition) + self._queue.put(PartitionGenerationCompletedSentinel(stream)) + except Exception as e: + self._queue.put(e) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partition_reader.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partition_reader.py new file mode 100644 index 000000000000..28f920326cf4 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partition_reader.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition +from airbyte_cdk.sources.streams.concurrent.partitions.throttled_queue import ThrottledQueue +from airbyte_cdk.sources.streams.concurrent.partitions.types import PartitionCompleteSentinel + + +class PartitionReader: + """ + Generates records from a partition and puts them in a queue. + """ + + def __init__(self, queue: ThrottledQueue) -> None: + """ + :param queue: The queue to put the records in. + """ + self._queue = queue + + def process_partition(self, partition: Partition) -> None: + """ + Process a partition and put the records in the output queue. + When all the partitions are added to the queue, a sentinel is added to the queue to indicate that all the partitions have been generated. + + If an exception is encountered, the exception will be caught and put in the queue. + + This method is meant to be called from a thread. + :param partition: The partition to read data from + :return: None + """ + try: + for record in partition.read(): + self._queue.put(record) + self._queue.put(PartitionCompleteSentinel(partition)) + except Exception as e: + self._queue.put(e) diff --git a/airbyte-ci/connectors/pipelines/pipelines/pipelines/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partitions/__init__.py similarity index 100% rename from airbyte-ci/connectors/pipelines/pipelines/pipelines/__init__.py rename to airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partitions/__init__.py diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partitions/partition.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partitions/partition.py new file mode 100644 index 000000000000..09f83d8f85f2 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partitions/partition.py @@ -0,0 +1,63 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import ABC, abstractmethod +from typing import Any, Iterable, Mapping, Optional + +from airbyte_cdk.sources.streams.concurrent.partitions.record import Record + + +class Partition(ABC): + """ + A partition is responsible for reading a specific set of data from a source. + """ + + @abstractmethod + def read(self) -> Iterable[Record]: + """ + Reads the data from the partition. + :return: An iterable of records. + """ + pass + + @abstractmethod + def to_slice(self) -> Optional[Mapping[str, Any]]: + """ + Converts the partition to a slice that can be serialized and deserialized. + + Note: it would have been interesting to have a type of `Mapping[str, Comparable]` to simplify typing but some slices can have nested + values ([example](https://github.com/airbytehq/airbyte/blob/1ce84d6396e446e1ac2377362446e3fb94509461/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py#L584-L596)) + :return: A mapping representing a slice + """ + pass + + @abstractmethod + def stream_name(self) -> str: + """ + Returns the name of the stream that this partition is reading from. + :return: The name of the stream. + """ + pass + + @abstractmethod + def close(self) -> None: + """ + Closes the partition. + """ + pass + + @abstractmethod + def is_closed(self) -> bool: + """ + Returns whether the partition is closed. + :return: + """ + pass + + @abstractmethod + def __hash__(self) -> int: + """ + Returns a hash of the partition. + Partitions must be hashable so that they can be used as keys in a dictionary. + """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partitions/partition_generator.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partitions/partition_generator.py new file mode 100644 index 000000000000..eff978564772 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partitions/partition_generator.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import ABC, abstractmethod +from typing import Iterable + +from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition + + +class PartitionGenerator(ABC): + @abstractmethod + def generate(self) -> Iterable[Partition]: + """ + Generates partitions for a given sync mode. + :return: An iterable of partitions + """ + pass diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partitions/record.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partitions/record.py new file mode 100644 index 000000000000..19f3454eefab --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partitions/record.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Mapping + + +class Record: + """ + Represents a record read from a stream. + """ + + def __init__(self, data: Mapping[str, Any], stream_name: str): + self.data = data + self.stream_name = stream_name + + def __eq__(self, other: Any) -> bool: + if not isinstance(other, Record): + return False + return self.data == other.data and self.stream_name == other.stream_name + + def __repr__(self) -> str: + return f"Record(data={self.data}, stream_name={self.stream_name})" diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partitions/throttled_queue.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partitions/throttled_queue.py new file mode 100644 index 000000000000..27a1757e47f5 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partitions/throttled_queue.py @@ -0,0 +1,41 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from queue import Queue + +from airbyte_cdk.sources.concurrent_source.throttler import Throttler +from airbyte_cdk.sources.streams.concurrent.partitions.types import QueueItem + + +class ThrottledQueue: + """ + A queue that throttles the number of items that can be added to it. + + We throttle the queue using custom logic instead of relying on the queue's max size + because the main thread can continuously dequeue before submitting a future. + + Since the main thread doesn't wait, it'll be able to remove items from the queue even if the tasks should be throttled, + so the tasks won't wait. + + This class solves this issue by checking if we should throttle the queue before adding an item to it. + An example implementation of a throttler would check if the number of pending futures is greater than a certain threshold. + """ + + def __init__(self, queue: Queue[QueueItem], throttler: Throttler, timeout: float) -> None: + """ + :param queue: The queue to throttle + :param throttler: The throttler to use to throttle the queue + :param timeout: The timeout to use when getting items from the queue + """ + self._queue = queue + self._throttler = throttler + self._timeout = timeout + + def put(self, item: QueueItem) -> None: + self._throttler.wait_and_acquire() + self._queue.put(item) + + def get(self) -> QueueItem: + return self._queue.get(block=True, timeout=self._timeout) + + def empty(self) -> bool: + return self._queue.empty() diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partitions/types.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partitions/types.py new file mode 100644 index 000000000000..fe16b2b0f9ab --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/partitions/types.py @@ -0,0 +1,28 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Union + +from airbyte_cdk.sources.concurrent_source.partition_generation_completed_sentinel import PartitionGenerationCompletedSentinel +from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition +from airbyte_cdk.sources.streams.concurrent.partitions.record import Record + + +class PartitionCompleteSentinel: + """ + A sentinel object indicating all records for a partition were produced. + Includes a pointer to the partition that was processed. + """ + + def __init__(self, partition: Partition): + """ + :param partition: The partition that was processed + """ + self.partition = partition + + +""" +Typedef representing the items that can be added to the ThreadBasedConcurrentStream +""" +QueueItem = Union[Record, Partition, PartitionCompleteSentinel, PartitionGenerationCompletedSentinel, Exception] diff --git a/airbyte-integrations/connectors/source-file-secure/integration_tests/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/state_converters/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-file-secure/integration_tests/__init__.py rename to airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/state_converters/__init__.py diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py new file mode 100644 index 000000000000..843f477ddb16 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/state_converters/abstract_stream_state_converter.py @@ -0,0 +1,89 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import ABC, abstractmethod +from enum import Enum +from typing import TYPE_CHECKING, Any, List, MutableMapping, Tuple + +if TYPE_CHECKING: + from airbyte_cdk.sources.streams.concurrent.cursor import CursorField + + +class ConcurrencyCompatibleStateType(Enum): + date_range = "date-range" + + +class AbstractStreamStateConverter(ABC): + START_KEY = "start" + END_KEY = "end" + + @abstractmethod + def deserialize(self, state: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + """ + Perform any transformations needed for compatibility with the converter. + """ + ... + + @staticmethod + def is_state_message_compatible(state: MutableMapping[str, Any]) -> bool: + return bool(state) and state.get("state_type") in [t.value for t in ConcurrencyCompatibleStateType] + + @abstractmethod + def convert_from_sequential_state( + self, + cursor_field: "CursorField", + stream_state: MutableMapping[str, Any], + start: Any, + ) -> Tuple[Any, MutableMapping[str, Any]]: + """ + Convert the state message to the format required by the ConcurrentCursor. + + e.g. + { + "state_type": ConcurrencyCompatibleStateType.date_range.value, + "metadata": { … }, + "slices": [ + {starts: 0, end: 1617030403, finished_processing: true}] + } + """ + ... + + @abstractmethod + def convert_to_sequential_state(self, cursor_field: "CursorField", stream_state: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + """ + Convert the state message from the concurrency-compatible format to the stream's original format. + + e.g. + { "created": 1617030403 } + """ + ... + + @abstractmethod + def increment(self, timestamp: Any) -> Any: + """ + Increment a timestamp by a single unit. + """ + ... + + @abstractmethod + def merge_intervals(self, intervals: List[MutableMapping[str, Any]]) -> List[MutableMapping[str, Any]]: + """ + Compute and return a list of merged intervals. + + Intervals may be merged if the start time of the second interval is 1 unit or less (as defined by the + `increment` method) than the end time of the first interval. + """ + ... + + @abstractmethod + def parse_value(self, value: Any) -> Any: + """ + Parse the value of the cursor field into a comparable value. + """ + ... + + @property + @abstractmethod + def zero_value(self) -> Any: + ... diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py new file mode 100644 index 000000000000..83f8a44b23db --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/concurrent/state_converters/datetime_stream_state_converter.py @@ -0,0 +1,196 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import abstractmethod +from datetime import datetime, timedelta +from typing import Any, List, MutableMapping, Optional, Tuple + +import pendulum +from airbyte_cdk.sources.streams.concurrent.cursor import CursorField +from airbyte_cdk.sources.streams.concurrent.state_converters.abstract_stream_state_converter import ( + AbstractStreamStateConverter, + ConcurrencyCompatibleStateType, +) +from pendulum.datetime import DateTime + + +class DateTimeStreamStateConverter(AbstractStreamStateConverter): + @property + @abstractmethod + def _zero_value(self) -> Any: + ... + + @property + def zero_value(self) -> datetime: + return self.parse_timestamp(self._zero_value) + + @abstractmethod + def increment(self, timestamp: datetime) -> datetime: + ... + + @abstractmethod + def parse_timestamp(self, timestamp: Any) -> datetime: + ... + + @abstractmethod + def output_format(self, timestamp: datetime) -> Any: + ... + + def deserialize(self, state: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + for stream_slice in state.get("slices", []): + stream_slice[self.START_KEY] = self.parse_timestamp(stream_slice[self.START_KEY]) + stream_slice[self.END_KEY] = self.parse_timestamp(stream_slice[self.END_KEY]) + return state + + def parse_value(self, value: Any) -> Any: + """ + Parse the value of the cursor field into a comparable value. + """ + return self.parse_timestamp(value) + + def merge_intervals(self, intervals: List[MutableMapping[str, datetime]]) -> List[MutableMapping[str, datetime]]: + if not intervals: + return [] + + sorted_intervals = sorted(intervals, key=lambda x: (x[self.START_KEY], x[self.END_KEY])) + merged_intervals = [sorted_intervals[0]] + + for interval in sorted_intervals[1:]: + last_end_time = merged_intervals[-1][self.END_KEY] + current_start_time = interval[self.START_KEY] + if self._compare_intervals(last_end_time, current_start_time): + merged_end_time = max(last_end_time, interval[self.END_KEY]) + merged_intervals[-1][self.END_KEY] = merged_end_time + else: + merged_intervals.append(interval) + + return merged_intervals + + def _compare_intervals(self, end_time: Any, start_time: Any) -> bool: + return bool(self.increment(end_time) >= start_time) + + def convert_from_sequential_state( + self, cursor_field: CursorField, stream_state: MutableMapping[str, Any], start: datetime + ) -> Tuple[datetime, MutableMapping[str, Any]]: + """ + Convert the state message to the format required by the ConcurrentCursor. + + e.g. + { + "state_type": ConcurrencyCompatibleStateType.date_range.value, + "metadata": { … }, + "slices": [ + {"start": "2021-01-18T21:18:20.000+00:00", "end": "2021-01-18T21:18:20.000+00:00"}, + ] + } + """ + sync_start = self._get_sync_start(cursor_field, stream_state, start) + if self.is_state_message_compatible(stream_state): + return sync_start, stream_state + + # Create a slice to represent the records synced during prior syncs. + # The start and end are the same to avoid confusion as to whether the records for this slice + # were actually synced + slices = [{self.START_KEY: sync_start, self.END_KEY: sync_start}] + + return sync_start, { + "state_type": ConcurrencyCompatibleStateType.date_range.value, + "slices": slices, + "legacy": stream_state, + } + + def _get_sync_start(self, cursor_field: CursorField, stream_state: MutableMapping[str, Any], start: Optional[Any]) -> datetime: + sync_start = self.parse_timestamp(start) if start is not None else self.zero_value + prev_sync_low_water_mark = ( + self.parse_timestamp(stream_state[cursor_field.cursor_field_key]) if cursor_field.cursor_field_key in stream_state else None + ) + if prev_sync_low_water_mark and prev_sync_low_water_mark >= sync_start: + return prev_sync_low_water_mark + else: + return sync_start + + def convert_to_sequential_state(self, cursor_field: CursorField, stream_state: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + """ + Convert the state message from the concurrency-compatible format to the stream's original format. + + e.g. + { "created": "2021-01-18T21:18:20.000Z" } + """ + if self.is_state_message_compatible(stream_state): + legacy_state = stream_state.get("legacy", {}) + latest_complete_time = self._get_latest_complete_time(stream_state.get("slices", [])) + if latest_complete_time is not None: + legacy_state.update({cursor_field.cursor_field_key: self.output_format(latest_complete_time)}) + return legacy_state or {} + else: + return stream_state + + def _get_latest_complete_time(self, slices: List[MutableMapping[str, Any]]) -> Optional[datetime]: + """ + Get the latest time before which all records have been processed. + """ + if not slices: + raise RuntimeError("Expected at least one slice but there were none. This is unexpected; please contact Support.") + + merged_intervals = self.merge_intervals(slices) + first_interval = merged_intervals[0] + return first_interval[self.END_KEY] + + +class EpochValueConcurrentStreamStateConverter(DateTimeStreamStateConverter): + """ + e.g. + { "created": 1617030403 } + => + { + "state_type": "date-range", + "metadata": { … }, + "slices": [ + {starts: 0, end: 1617030403, finished_processing: true} + ] + } + """ + + _zero_value = 0 + + def increment(self, timestamp: datetime) -> datetime: + return timestamp + timedelta(seconds=1) + + def output_format(self, timestamp: datetime) -> int: + return int(timestamp.timestamp()) + + def parse_timestamp(self, timestamp: int) -> datetime: + dt_object = pendulum.from_timestamp(timestamp) + if not isinstance(dt_object, DateTime): + raise ValueError(f"DateTime object was expected but got {type(dt_object)} from pendulum.parse({timestamp})") + return dt_object # type: ignore # we are manually type checking because pendulum.parse may return different types + + +class IsoMillisConcurrentStreamStateConverter(DateTimeStreamStateConverter): + """ + e.g. + { "created": "2021-01-18T21:18:20.000Z" } + => + { + "state_type": "date-range", + "metadata": { … }, + "slices": [ + {starts: "2020-01-18T21:18:20.000Z", end: "2021-01-18T21:18:20.000Z", finished_processing: true} + ] + } + """ + + _zero_value = "0001-01-01T00:00:00.000Z" + + def increment(self, timestamp: datetime) -> datetime: + return timestamp + timedelta(milliseconds=1) + + def output_format(self, timestamp: datetime) -> Any: + return timestamp.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + def parse_timestamp(self, timestamp: str) -> datetime: + dt_object = pendulum.parse(timestamp) + if not isinstance(dt_object, DateTime): + raise ValueError(f"DateTime object was expected but got {type(dt_object)} from pendulum.parse({timestamp})") + return dt_object # type: ignore # we are manually type checking because pendulum.parse may return different types diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/core.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/core.py index 03698afa5747..8d6ba15fdcd1 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/core.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/core.py @@ -12,9 +12,11 @@ import airbyte_cdk.sources.utils.casing as casing from airbyte_cdk.models import AirbyteMessage, AirbyteStream, SyncMode +from airbyte_cdk.models import Type as MessageType # list of all possible HTTP methods which can be used for sending of request bodies -from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader +from airbyte_cdk.sources.utils.schema_helpers import InternalConfig, ResourceSchemaLoader +from airbyte_cdk.sources.utils.slice_logger import SliceLogger from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from deprecated.classic import deprecated @@ -105,6 +107,74 @@ def get_error_display_message(self, exception: BaseException) -> Optional[str]: """ return None + def read_full_refresh( + self, + cursor_field: Optional[List[str]], + logger: logging.Logger, + slice_logger: SliceLogger, + ) -> Iterable[StreamData]: + slices = self.stream_slices(sync_mode=SyncMode.full_refresh, cursor_field=cursor_field) + logger.debug(f"Processing stream slices for {self.name} (sync_mode: full_refresh)", extra={"stream_slices": slices}) + for _slice in slices: + if slice_logger.should_log_slice_message(logger): + yield slice_logger.create_slice_log_message(_slice) + yield from self.read_records( + stream_slice=_slice, + sync_mode=SyncMode.full_refresh, + cursor_field=cursor_field, + ) + + def read_incremental( # type: ignore # ignoring typing for ConnectorStateManager because of circular dependencies + self, + cursor_field: Optional[List[str]], + logger: logging.Logger, + slice_logger: SliceLogger, + stream_state: MutableMapping[str, Any], + state_manager, + per_stream_state_enabled: bool, + internal_config: InternalConfig, + ) -> Iterable[StreamData]: + slices = self.stream_slices( + cursor_field=cursor_field, + sync_mode=SyncMode.incremental, + stream_state=stream_state, + ) + logger.debug(f"Processing stream slices for {self.name} (sync_mode: incremental)", extra={"stream_slices": slices}) + + has_slices = False + record_counter = 0 + for _slice in slices: + has_slices = True + if slice_logger.should_log_slice_message(logger): + yield slice_logger.create_slice_log_message(_slice) + records = self.read_records( + sync_mode=SyncMode.incremental, + stream_slice=_slice, + stream_state=stream_state, + cursor_field=cursor_field or None, + ) + for record_data_or_message in records: + yield record_data_or_message + if isinstance(record_data_or_message, Mapping) or ( + hasattr(record_data_or_message, "type") and record_data_or_message.type == MessageType.RECORD + ): + record_data = record_data_or_message if isinstance(record_data_or_message, Mapping) else record_data_or_message.record + stream_state = self.get_updated_state(stream_state, record_data) + checkpoint_interval = self.state_checkpoint_interval + record_counter += 1 + if checkpoint_interval and record_counter % checkpoint_interval == 0: + yield self._checkpoint_state(stream_state, state_manager, per_stream_state_enabled) + + if internal_config.is_limit_reached(record_counter): + break + + yield self._checkpoint_state(stream_state, state_manager, per_stream_state_enabled) + + if not has_slices: + # Safety net to ensure we always emit at least one state message even if there are no slices + checkpoint = self._checkpoint_state(stream_state, state_manager, per_stream_state_enabled) + yield checkpoint + @abstractmethod def read_records( self, @@ -252,6 +322,18 @@ def get_updated_state( """ return {} + def log_stream_sync_configuration(self) -> None: + """ + Logs the configuration of this stream. + """ + self.logger.debug( + f"Syncing stream instance: {self.name}", + extra={ + "primary_key": self.primary_key, + "cursor_field": self.cursor_field, + }, + ) + @staticmethod def _wrapped_primary_key(keys: Optional[Union[str, List[str], List[List[str]]]]) -> Optional[List[List[str]]]: """ @@ -274,3 +356,21 @@ def _wrapped_primary_key(keys: Optional[Union[str, List[str], List[List[str]]]]) return wrapped_keys else: raise ValueError(f"Element must be either list or str. Got: {type(keys)}") + + def _checkpoint_state( # type: ignore # ignoring typing for ConnectorStateManager because of circular dependencies + self, + stream_state: Mapping[str, Any], + state_manager, + per_stream_state_enabled: bool, + ) -> AirbyteMessage: + # First attempt to retrieve the current state using the stream's state property. We receive an AttributeError if the state + # property is not implemented by the stream instance and as a fallback, use the stream_state retrieved from the stream + # instance's deprecated get_updated_state() method. + try: + state_manager.update_state_for_stream( + self.name, self.namespace, self.state # type: ignore # we know the field might not exist... + ) + + except AttributeError: + state_manager.update_state_for_stream(self.name, self.namespace, stream_state) + return state_manager.create_state_message(self.name, self.namespace, send_per_stream_state=per_stream_state_enabled) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/__init__.py index 3929f8378340..3460da053ae5 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # # Initialize Streams Package diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py index af0936e13cb8..e5784cd25c03 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py @@ -7,19 +7,22 @@ import os import urllib from abc import ABC, abstractmethod -from contextlib import suppress +from pathlib import Path from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union from urllib.parse import urljoin import requests import requests_cache from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.http_config import MAX_CONNECTION_POOL_SIZE from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy +from airbyte_cdk.sources.streams.call_rate import APIBudget, CachedLimiterSession, LimiterSession from airbyte_cdk.sources.streams.core import Stream, StreamData from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy +from airbyte_cdk.sources.utils.types import JsonType +from airbyte_cdk.utils.constants import ENV_REQUEST_CACHE_PATH from requests.auth import AuthBase -from ...utils.types import JsonType from .auth.core import HttpAuthenticator, NoAuth from .exceptions import DefaultBackoffException, RequestBodyException, UserDefinedBackoffException from .rate_limiting import default_backoff_handler, user_defined_backoff_handler @@ -37,12 +40,12 @@ class HttpStream(Stream, ABC): page_size: Optional[int] = None # Use this variable to define page size for API http requests with pagination support # TODO: remove legacy HttpAuthenticator authenticator references - def __init__(self, authenticator: Optional[Union[AuthBase, HttpAuthenticator]] = None): - if self.use_cache: - self._session = self.request_cache() - else: - self._session = requests.Session() - + def __init__(self, authenticator: Optional[Union[AuthBase, HttpAuthenticator]] = None, api_budget: Optional[APIBudget] = None): + self._api_budget: APIBudget = api_budget or APIBudget(policies=[]) + self._session = self.request_session() + self._session.mount( + "https://", requests.adapters.HTTPAdapter(pool_connections=MAX_CONNECTION_POOL_SIZE, pool_maxsize=MAX_CONNECTION_POOL_SIZE) + ) self._authenticator: HttpAuthenticator = NoAuth() if isinstance(authenticator, AuthBase): self._session.auth = authenticator @@ -53,6 +56,7 @@ def __init__(self, authenticator: Optional[Union[AuthBase, HttpAuthenticator]] = def cache_filename(self) -> str: """ Override if needed. Return the name of cache file + Note that if the environment variable REQUEST_CACHE_PATH is not set, the cache will be in-memory only. """ return f"{self.name}.sqlite" @@ -60,22 +64,33 @@ def cache_filename(self) -> str: def use_cache(self) -> bool: """ Override if needed. If True, all records will be cached. + Note that if the environment variable REQUEST_CACHE_PATH is not set, the cache will be in-memory only. """ return False - def request_cache(self) -> requests.Session: - self.clear_cache() - return requests_cache.CachedSession(self.cache_filename) + def request_session(self) -> requests.Session: + """ + Session factory based on use_cache property and call rate limits (api_budget parameter) + :return: instance of request-based session + """ + if self.use_cache: + cache_dir = os.getenv(ENV_REQUEST_CACHE_PATH) + # Use in-memory cache if cache_dir is not set + # This is a non-obvious interface, but it ensures we don't write sql files when running unit tests + if cache_dir: + sqlite_path = str(Path(cache_dir) / self.cache_filename) + else: + sqlite_path = "file::memory:?cache=shared" + return CachedLimiterSession(sqlite_path, backend="sqlite", api_budget=self._api_budget) # type: ignore # there are no typeshed stubs for requests_cache + else: + return LimiterSession(api_budget=self._api_budget) def clear_cache(self) -> None: """ - remove cache file only once + Clear cached requests for current session, can be called any time """ - STREAM_CACHE_FILES = globals().setdefault("STREAM_CACHE_FILES", set()) - if self.cache_filename not in STREAM_CACHE_FILES: - with suppress(FileNotFoundError): - os.remove(self.cache_filename) - STREAM_CACHE_FILES.add(self.cache_filename) + if isinstance(self._session, requests_cache.CachedSession): + self._session.cache.clear() # type: ignore # cache.clear is not typed @property @abstractmethod @@ -105,6 +120,13 @@ def max_retries(self) -> Union[int, None]: """ return 5 + @property + def max_time(self) -> Union[int, None]: + """ + Override if needed. Specifies maximum total waiting time (in seconds) for backoff policy. Return None for no limit. + """ + return 60 * 10 + @property def retry_factor(self) -> float: """ @@ -303,7 +325,9 @@ def _create_prepared_request( args["json"] = json elif data: args["data"] = data - return self._session.prepare_request(requests.Request(**args)) + prepared_request: requests.PreparedRequest = self._session.prepare_request(requests.Request(**args)) + + return prepared_request @classmethod def _join_url(cls, url_base: str, path: str) -> str: @@ -382,11 +406,19 @@ def _send_request(self, request: requests.PreparedRequest, request_kwargs: Mappi Add this condition to avoid an endless loop if it hasn't been set explicitly (i.e. max_retries is not None). """ + max_time = self.max_time + """ + According to backoff max_time docstring: + max_time: The maximum total amount of time to try for before + giving up. Once expired, the exception will be allowed to + escape. If a callable is passed, it will be + evaluated at runtime and its return value used. + """ if max_tries is not None: max_tries = max(0, max_tries) + 1 - user_backoff_handler = user_defined_backoff_handler(max_tries=max_tries)(self._send) - backoff_handler = default_backoff_handler(max_tries=max_tries, factor=self.retry_factor) + user_backoff_handler = user_defined_backoff_handler(max_tries=max_tries, max_time=max_time)(self._send) + backoff_handler = default_backoff_handler(max_tries=max_tries, max_time=max_time, factor=self.retry_factor) return backoff_handler(user_backoff_handler)(request, request_kwargs) @classmethod @@ -436,7 +468,7 @@ def get_error_display_message(self, exception: BaseException) -> Optional[str]: :param exception: The exception that was raised :return: A user-friendly message that indicates the cause of the error """ - if isinstance(exception, requests.HTTPError): + if isinstance(exception, requests.HTTPError) and exception.response is not None: return self.parse_response_error_message(exception.response) return None diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/rate_limiting.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/rate_limiting.py index 9bc580d500fe..84d320345294 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/rate_limiting.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/rate_limiting.py @@ -27,7 +27,7 @@ def default_backoff_handler( - max_tries: Optional[int], factor: float, **kwargs: Any + max_tries: Optional[int], factor: float, max_time: Optional[int] = None, **kwargs: Any ) -> Callable[[SendRequestCallableType], SendRequestCallableType]: def log_retry_attempt(details: Mapping[str, Any]) -> None: _, exc, _ = sys.exc_info() @@ -56,12 +56,15 @@ def should_give_up(exc: Exception) -> bool: on_backoff=log_retry_attempt, giveup=should_give_up, max_tries=max_tries, + max_time=max_time, factor=factor, **kwargs, ) -def user_defined_backoff_handler(max_tries: Optional[int], **kwargs: Any) -> Callable[[SendRequestCallableType], SendRequestCallableType]: +def user_defined_backoff_handler( + max_tries: Optional[int], max_time: Optional[int] = None, **kwargs: Any +) -> Callable[[SendRequestCallableType], SendRequestCallableType]: def sleep_on_ratelimit(details: Mapping[str, Any]) -> None: _, exc, _ = sys.exc_info() if isinstance(exc, UserDefinedBackoffException): @@ -86,5 +89,6 @@ def log_give_up(details: Mapping[str, Any]) -> None: on_giveup=log_give_up, jitter=None, max_tries=max_tries, + max_time=max_time, **kwargs, ) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py index 371f06b34d51..0dd450413dd4 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/abstract_oauth.py @@ -4,14 +4,16 @@ import logging from abc import abstractmethod +from json import JSONDecodeError from typing import Any, List, Mapping, MutableMapping, Optional, Tuple, Union import backoff import pendulum import requests -from airbyte_cdk.models import Level +from airbyte_cdk.models import FailureType, Level from airbyte_cdk.sources.http_logger import format_http_message from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository +from airbyte_cdk.utils import AirbyteTracedException from requests.auth import AuthBase from ..exceptions import DefaultBackoffException @@ -29,7 +31,21 @@ class AbstractOauth2Authenticator(AuthBase): _NO_STREAM_NAME = None - def __call__(self, request: requests.Request) -> requests.Request: + def __init__( + self, + refresh_token_error_status_codes: Tuple[int, ...] = (), + refresh_token_error_key: str = "", + refresh_token_error_values: Tuple[str, ...] = (), + ) -> None: + """ + If all of refresh_token_error_status_codes, refresh_token_error_key, and refresh_token_error_values are set, + then http errors with such params will be wrapped in AirbyteTracedException. + """ + self._refresh_token_error_status_codes = refresh_token_error_status_codes + self._refresh_token_error_key = refresh_token_error_key + self._refresh_token_error_values = refresh_token_error_values + + def __call__(self, request: requests.PreparedRequest) -> requests.PreparedRequest: """Attach the HTTP headers required to authenticate on the HTTP request""" request.headers.update(self.get_auth_header()) return request @@ -49,7 +65,7 @@ def get_access_token(self) -> str: def token_has_expired(self) -> bool: """Returns True if the token is expired""" - return pendulum.now() > self.get_token_expiry_date() + return pendulum.now() > self.get_token_expiry_date() # type: ignore # this is always a bool despite what mypy thinks def build_refresh_request_body(self) -> Mapping[str, Any]: """ @@ -64,7 +80,7 @@ def build_refresh_request_body(self) -> Mapping[str, Any]: "refresh_token": self.get_refresh_token(), } - if self.get_scopes: + if self.get_scopes(): payload["scopes"] = self.get_scopes() if self.get_refresh_request_body(): @@ -75,6 +91,19 @@ def build_refresh_request_body(self) -> Mapping[str, Any]: return payload + def _wrap_refresh_token_exception(self, exception: requests.exceptions.RequestException) -> bool: + try: + if exception.response is not None: + exception_content = exception.response.json() + else: + return False + except JSONDecodeError: + return False + return ( + exception.response.status_code in self._refresh_token_error_status_codes + and exception_content.get(self._refresh_token_error_key) in self._refresh_token_error_values + ) + @backoff.on_exception( backoff.expo, DefaultBackoffException, @@ -83,27 +112,64 @@ def build_refresh_request_body(self) -> Mapping[str, Any]: ), max_time=300, ) - def _get_refresh_access_token_response(self): + def _get_refresh_access_token_response(self) -> Any: try: response = requests.request(method="POST", url=self.get_token_refresh_endpoint(), data=self.build_refresh_request_body()) self._log_response(response) response.raise_for_status() return response.json() except requests.exceptions.RequestException as e: - if e.response.status_code == 429 or e.response.status_code >= 500: - raise DefaultBackoffException(request=e.response.request, response=e.response) + if e.response is not None: + if e.response.status_code == 429 or e.response.status_code >= 500: + raise DefaultBackoffException(request=e.response.request, response=e.response) + if self._wrap_refresh_token_exception(e): + message = "Refresh token is invalid or expired. Please re-authenticate from Sources//Settings." + raise AirbyteTracedException(internal_message=message, message=message, failure_type=FailureType.config_error) raise except Exception as e: raise Exception(f"Error while refreshing access token: {e}") from e - def refresh_access_token(self) -> Tuple[str, int]: + def refresh_access_token(self) -> Tuple[str, Union[str, int]]: """ - Returns the refresh token and its lifespan in seconds + Returns the refresh token and its expiration datetime - :return: a tuple of (access_token, token_lifespan_in_seconds) + :return: a tuple of (access_token, token_lifespan) """ response_json = self._get_refresh_access_token_response() - return response_json[self.get_access_token_name()], int(response_json[self.get_expires_in_name()]) + + return response_json[self.get_access_token_name()], response_json[self.get_expires_in_name()] + + def _parse_token_expiration_date(self, value: Union[str, int]) -> pendulum.DateTime: + """ + Return the expiration datetime of the refresh token + + :return: expiration datetime + """ + + if self.token_expiry_is_time_of_expiration: + if not self.token_expiry_date_format: + raise ValueError( + f"Invalid token expiry date format {self.token_expiry_date_format}; a string representing the format is required." + ) + return pendulum.from_format(str(value), self.token_expiry_date_format) + else: + return pendulum.now().add(seconds=int(float(value))) + + @property + def token_expiry_is_time_of_expiration(self) -> bool: + """ + Indicates that the Token Expiry returns the date until which the token will be valid, not the amount of time it will be valid. + """ + + return False + + @property + def token_expiry_date_format(self) -> Optional[str]: + """ + Format of the datetime; exists it if expires_in is returned as the expiration datetime instead of seconds until it expires + """ + + return None @abstractmethod def get_token_refresh_endpoint(self) -> str: @@ -130,7 +196,7 @@ def get_token_expiry_date(self) -> pendulum.DateTime: """Expiration date of the access token""" @abstractmethod - def set_token_expiry_date(self, value: Union[str, int]): + def set_token_expiry_date(self, value: Union[str, int]) -> None: """Setter for access token expiration date""" @abstractmethod @@ -166,14 +232,15 @@ def _message_repository(self) -> Optional[MessageRepository]: """ return _NOOP_MESSAGE_REPOSITORY - def _log_response(self, response: requests.Response): - self._message_repository.log_message( - Level.DEBUG, - lambda: format_http_message( - response, - "Refresh token", - "Obtains access token", - self._NO_STREAM_NAME, - is_auxiliary=True, - ), - ) + def _log_response(self, response: requests.Response) -> None: + if self._message_repository: + self._message_repository.log_message( + Level.DEBUG, + lambda: format_http_message( + response, + "Refresh token", + "Obtains access token", + self._NO_STREAM_NAME, + is_auxiliary=True, + ), + ) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py index d7a93157ed99..48a855fa515f 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/requests_native_auth/oauth.py @@ -31,6 +31,10 @@ def __init__( expires_in_name: str = "expires_in", refresh_request_body: Mapping[str, Any] = None, grant_type: str = "refresh_token", + token_expiry_is_time_of_expiration: bool = False, + refresh_token_error_status_codes: Tuple[int, ...] = (), + refresh_token_error_key: str = "", + refresh_token_error_values: Tuple[str, ...] = (), ): self._token_refresh_endpoint = token_refresh_endpoint self._client_secret = client_secret @@ -44,7 +48,9 @@ def __init__( self._token_expiry_date = token_expiry_date or pendulum.now().subtract(days=1) self._token_expiry_date_format = token_expiry_date_format + self._token_expiry_is_time_of_expiration = token_expiry_is_time_of_expiration self._access_token = None + super().__init__(refresh_token_error_status_codes, refresh_token_error_key, refresh_token_error_values) def get_token_refresh_endpoint(self) -> str: return self._token_refresh_endpoint @@ -77,10 +83,15 @@ def get_token_expiry_date(self) -> pendulum.DateTime: return self._token_expiry_date def set_token_expiry_date(self, value: Union[str, int]): - if self._token_expiry_date_format: - self._token_expiry_date = pendulum.from_format(value, self._token_expiry_date_format) - else: - self._token_expiry_date = pendulum.now().add(seconds=value) + self._token_expiry_date = self._parse_token_expiration_date(value) + + @property + def token_expiry_is_time_of_expiration(self) -> bool: + return self._token_expiry_is_time_of_expiration + + @property + def token_expiry_date_format(self) -> Optional[str]: + return self._token_expiry_date_format @property def access_token(self) -> str: @@ -96,8 +107,9 @@ class SingleUseRefreshTokenOauth2Authenticator(Oauth2Authenticator): Authenticator that should be used for API implementing single use refresh tokens: when refreshing access token some API returns a new refresh token that needs to used in the next refresh flow. This authenticator updates the configuration with new refresh token by emitting Airbyte control message from an observed mutation. - By default this authenticator expects a connector config with a"credentials" field with the following nested fields: client_id, client_secret, refresh_token. - This behavior can be changed by defining custom config path (using dpath paths) in client_id_config_path, client_secret_config_path, refresh_token_config_path constructor arguments. + By default, this authenticator expects a connector config with a "credentials" field with the following nested fields: client_id, + client_secret, refresh_token. This behavior can be changed by defining custom config path (using dpath paths) in client_id_config_path, + client_secret_config_path, refresh_token_config_path constructor arguments. """ def __init__( @@ -117,9 +129,12 @@ def __init__( token_expiry_date_config_path: Sequence[str] = ("credentials", "token_expiry_date"), token_expiry_date_format: Optional[str] = None, message_repository: MessageRepository = NoopMessageRepository(), + token_expiry_is_time_of_expiration: bool = False, + refresh_token_error_status_codes: Tuple[int, ...] = (), + refresh_token_error_key: str = "", + refresh_token_error_values: Tuple[str, ...] = (), ): """ - Args: connector_config (Mapping[str, Any]): The full connector configuration token_refresh_endpoint (str): Full URL to the token refresh endpoint @@ -135,6 +150,7 @@ def __init__( refresh_token_config_path (Sequence[str]): Dpath to the refresh_token field in the connector configuration. Defaults to ("credentials", "refresh_token"). token_expiry_date_config_path (Sequence[str]): Dpath to the token_expiry_date field in the connector configuration. Defaults to ("credentials", "token_expiry_date"). token_expiry_date_format (Optional[str]): Date format of the token expiry date field (set by expires_in_name). If not specified the token expiry date is interpreted as number of seconds until expiration. + token_expiry_is_time_of_expiration bool: set True it if expires_in is returned as time of expiration instead of the number seconds until expiration message_repository (MessageRepository): the message repository used to emit logs on HTTP requests and control message on config update """ self._client_id = client_id if client_id is not None else dpath.util.get(connector_config, ("credentials", "client_id")) @@ -160,6 +176,10 @@ def __init__( refresh_request_body=refresh_request_body, grant_type=grant_type, token_expiry_date_format=token_expiry_date_format, + token_expiry_is_time_of_expiration=token_expiry_is_time_of_expiration, + refresh_token_error_status_codes=refresh_token_error_status_codes, + refresh_token_error_key=refresh_token_error_key, + refresh_token_error_values=refresh_token_error_values, ) def get_refresh_token_name(self) -> str: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/utils/stream_helper.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/utils/stream_helper.py index 98edab1e9557..c5b11812b448 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/utils/stream_helper.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/utils/stream_helper.py @@ -34,5 +34,7 @@ def get_first_record_for_slice(stream: Stream, stream_slice: Optional[Mapping[st :raises StopIteration: if there is no first record to return (the read_records generator is empty) :return: StreamData containing the first record in the slice """ - records_for_slice = stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) + # We wrap the return output of read_records() because some implementations return types that are iterable, + # but not iterators such as lists or tuples + records_for_slice = iter(stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice)) return next(records_for_slice) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py b/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py index 7207b6129778..2fd26ca9cae5 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_helpers.py @@ -7,7 +7,7 @@ import json import os import pkgutil -from typing import Any, ClassVar, Dict, List, Mapping, MutableMapping, Optional, Tuple, Union +from typing import Any, ClassVar, Dict, List, Mapping, MutableMapping, Optional, Tuple import jsonref from airbyte_cdk.models import ConnectorSpecification, FailureType @@ -30,10 +30,15 @@ def __init__(self, uri_base: str, shared: str): def __call__(self, uri: str) -> Dict[str, Any]: uri = uri.replace(self.uri_base, f"{self.uri_base}/{self.shared}/") - return json.load(open(uri)) + with open(uri) as f: + data = json.load(f) + if isinstance(data, dict): + return data + else: + raise ValueError(f"Expected to read a dictionary from {uri}. Got: {data}") -def resolve_ref_links(obj: Any) -> Union[Dict[str, Any], List[Any]]: +def resolve_ref_links(obj: Any) -> Any: """ Scan resolved schema and convert jsonref.JsonRef object to JSON serializable dict. @@ -44,8 +49,11 @@ def resolve_ref_links(obj: Any) -> Union[Dict[str, Any], List[Any]]: obj = resolve_ref_links(obj.__subject__) # Omit existing definitions for external resource since # we dont need it anymore. - obj.pop("definitions", None) - return obj + if isinstance(obj, dict): + obj.pop("definitions", None) + return obj + else: + raise ValueError(f"Expected obj to be a dict. Got {obj}") elif isinstance(obj, dict): return {k: resolve_ref_links(v) for k, v in obj.items()} elif isinstance(obj, list): @@ -107,7 +115,7 @@ class ResourceSchemaLoader: def __init__(self, package_name: str): self.package_name = package_name - def get_schema(self, name: str) -> dict: + def get_schema(self, name: str) -> dict[str, Any]: """ This method retrieves a JSON schema from the schemas/ folder. @@ -131,7 +139,7 @@ def get_schema(self, name: str) -> dict: return self._resolve_schema_references(raw_schema) - def _resolve_schema_references(self, raw_schema: dict) -> dict: + def _resolve_schema_references(self, raw_schema: dict[str, Any]) -> dict[str, Any]: """ Resolve links to external references and move it to local "definitions" map. @@ -140,13 +148,19 @@ def _resolve_schema_references(self, raw_schema: dict) -> dict: """ package = importlib.import_module(self.package_name) - base = os.path.dirname(package.__file__) + "/" + if package.__file__: + base = os.path.dirname(package.__file__) + "/" + else: + raise ValueError(f"Package {package} does not have a valid __file__ field") resolved = jsonref.JsonRef.replace_refs(raw_schema, loader=JsonFileLoader(base, "schemas/shared"), base_uri=base) resolved = resolve_ref_links(resolved) - return resolved + if isinstance(resolved, dict): + return resolved + else: + raise ValueError(f"Expected resolved to be a dict. Got {resolved}") -def check_config_against_spec_or_exit(config: Mapping[str, Any], spec: ConnectorSpecification): +def check_config_against_spec_or_exit(config: Mapping[str, Any], spec: ConnectorSpecification) -> None: """ Check config object against spec. In case of spec is invalid, throws an exception with validation error description. @@ -166,17 +180,28 @@ def check_config_against_spec_or_exit(config: Mapping[str, Any], spec: Connector class InternalConfig(BaseModel): - KEYWORDS: ClassVar[set] = {"_limit", "_page_size"} + KEYWORDS: ClassVar[set[str]] = {"_limit", "_page_size"} limit: int = Field(None, alias="_limit") page_size: int = Field(None, alias="_page_size") - def dict(self, *args, **kwargs): + def dict(self, *args: Any, **kwargs: Any) -> dict[str, Any]: kwargs["by_alias"] = True kwargs["exclude_unset"] = True return super().dict(*args, **kwargs) + def is_limit_reached(self, records_counter: int) -> bool: + """ + Check if record count reached limit set by internal config. + :param records_counter - number of records already red + :return True if limit reached, False otherwise + """ + if self.limit: + if records_counter >= self.limit: + return True + return False + -def split_config(config: Mapping[str, Any]) -> Tuple[dict, InternalConfig]: +def split_config(config: Mapping[str, Any]) -> Tuple[dict[str, Any], InternalConfig]: """ Break config map object into 2 instances: first is a dict with user defined configuration and second is internal config that contains private keys for diff --git a/airbyte-cdk/python/airbyte_cdk/sources/utils/slice_logger.py b/airbyte-cdk/python/airbyte_cdk/sources/utils/slice_logger.py new file mode 100644 index 000000000000..6981cdde88fa --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/utils/slice_logger.py @@ -0,0 +1,54 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import logging +from abc import ABC, abstractmethod +from typing import Any, Mapping, Optional + +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level +from airbyte_cdk.models import Type as MessageType + + +class SliceLogger(ABC): + """ + SliceLogger is an interface that allows us to log slices of data in a uniform way. + It is responsible for determining whether or not a slice should be logged and for creating the log message. + """ + + SLICE_LOG_PREFIX = "slice:" + + def create_slice_log_message(self, _slice: Optional[Mapping[str, Any]]) -> AirbyteMessage: + """ + Mapping is an interface that can be implemented in various ways. However, json.dumps will just do a `str()` if + the slice is a class implementing Mapping. Therefore, we want to cast this as a dict before passing this to json.dump + """ + printable_slice = dict(_slice) if _slice else _slice + return AirbyteMessage( + type=MessageType.LOG, + log=AirbyteLogMessage(level=Level.INFO, message=f"{SliceLogger.SLICE_LOG_PREFIX}{json.dumps(printable_slice, default=str)}"), + ) + + @abstractmethod + def should_log_slice_message(self, logger: logging.Logger) -> bool: + """ + + :param logger: + :return: + """ + + +class DebugSliceLogger(SliceLogger): + def should_log_slice_message(self, logger: logging.Logger) -> bool: + """ + + :param logger: + :return: + """ + return logger.isEnabledFor(logging.DEBUG) + + +class AlwaysLogSliceLogger(SliceLogger): + def should_log_slice_message(self, logger: logging.Logger) -> bool: + return True diff --git a/airbyte-cdk/python/airbyte_cdk/test/__init__.py b/airbyte-cdk/python/airbyte_cdk/test/__init__.py new file mode 100644 index 000000000000..6d3fabb5a354 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/test/__init__.py @@ -0,0 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +""" +This package is provided as tooling to help sources test their implementation. It is not expected to be used as production code. +""" diff --git a/airbyte-cdk/python/airbyte_cdk/test/catalog_builder.py b/airbyte-cdk/python/airbyte_cdk/test/catalog_builder.py new file mode 100644 index 000000000000..522e3dd68ab2 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/test/catalog_builder.py @@ -0,0 +1,29 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Any, Dict, List + +from airbyte_protocol.models import ConfiguredAirbyteCatalog, SyncMode + + +class CatalogBuilder: + def __init__(self) -> None: + self._streams: List[Dict[str, Any]] = [] + + def with_stream(self, name: str, sync_mode: SyncMode) -> "CatalogBuilder": + self._streams.append( + { + "stream": { + "name": name, + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]], + }, + "primary_key": [["id"]], + "sync_mode": sync_mode.name, + "destination_sync_mode": "overwrite", + } + ) + return self + + def build(self) -> ConfiguredAirbyteCatalog: + return ConfiguredAirbyteCatalog.parse_obj({"streams": self._streams}) diff --git a/airbyte-cdk/python/airbyte_cdk/test/entrypoint_wrapper.py b/airbyte-cdk/python/airbyte_cdk/test/entrypoint_wrapper.py new file mode 100644 index 000000000000..06d5e0ebeb20 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/test/entrypoint_wrapper.py @@ -0,0 +1,166 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +""" +The AirbyteEntrypoint is important because it is a service layer that orchestrate how we execute commands from the +[common interface](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol#common-interface) through the source Python +implementation. There is some logic about which message we send to the platform and when which is relevant for integration testing. Other +than that, there are integrations point that are annoying to integrate with using Python code: +* Sources communicate with the platform using stdout. The implication is that the source could just print every message instead of + returning things to source. or to using the message repository. WARNING: As part of integration testing, we will not support + messages that are simply printed. The reason is that capturing stdout relies on overriding sys.stdout (see + https://docs.python.org/3/library/contextlib.html#contextlib.redirect_stdout) which clashes with how pytest captures logs and brings + considerations for multithreaded applications. If code you work with uses `print` statements, please migrate to + source.message_repository to emit those messages +* The entrypoint interface relies on file being written on the file system +""" + +import json +import logging +import tempfile +import traceback +from io import StringIO +from pathlib import Path +from typing import Any, List, Mapping, Optional, Union + +from airbyte_cdk.entrypoint import AirbyteEntrypoint +from airbyte_cdk.exception_handler import assemble_uncaught_exception +from airbyte_cdk.logger import AirbyteLogFormatter +from airbyte_cdk.sources import Source +from airbyte_protocol.models import AirbyteLogMessage, AirbyteMessage, AirbyteStreamStatus, ConfiguredAirbyteCatalog, Level, TraceType, Type +from pydantic.error_wrappers import ValidationError + + +class EntrypointOutput: + def __init__(self, messages: List[str], uncaught_exception: Optional[BaseException] = None): + try: + self._messages = [self._parse_message(message) for message in messages] + except ValidationError as exception: + raise ValueError("All messages are expected to be AirbyteMessage") from exception + + if uncaught_exception: + self._messages.append(assemble_uncaught_exception(type(uncaught_exception), uncaught_exception).as_airbyte_message()) + + @staticmethod + def _parse_message(message: str) -> AirbyteMessage: + try: + return AirbyteMessage.parse_obj(json.loads(message)) + except (json.JSONDecodeError, ValidationError): + # The platform assumes that logs that are not of AirbyteMessage format are log messages + return AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message=message)) + + @property + def records_and_state_messages(self) -> List[AirbyteMessage]: + return self._get_message_by_types([Type.RECORD, Type.STATE]) + + @property + def records(self) -> List[AirbyteMessage]: + return self._get_message_by_types([Type.RECORD]) + + @property + def state_messages(self) -> List[AirbyteMessage]: + return self._get_message_by_types([Type.STATE]) + + @property + def most_recent_state(self) -> Any: + state_messages = self._get_message_by_types([Type.STATE]) + if not state_messages: + raise ValueError("Can't provide most recent state as there are no state messages") + return state_messages[-1].state.data + + @property + def logs(self) -> List[AirbyteMessage]: + return self._get_message_by_types([Type.LOG]) + + @property + def trace_messages(self) -> List[AirbyteMessage]: + return self._get_message_by_types([Type.TRACE]) + + @property + def analytics_messages(self) -> List[AirbyteMessage]: + return self._get_trace_message_by_trace_type(TraceType.ANALYTICS) + + @property + def errors(self) -> List[AirbyteMessage]: + return self._get_trace_message_by_trace_type(TraceType.ERROR) + + def get_stream_statuses(self, stream_name: str) -> List[AirbyteStreamStatus]: + status_messages = map( + lambda message: message.trace.stream_status.status, + filter( + lambda message: message.trace.stream_status.stream_descriptor.name == stream_name, + self._get_trace_message_by_trace_type(TraceType.STREAM_STATUS), + ), + ) + return list(status_messages) + + def _get_message_by_types(self, message_types: List[Type]) -> List[AirbyteMessage]: + return [message for message in self._messages if message.type in message_types] + + def _get_trace_message_by_trace_type(self, trace_type: TraceType) -> List[AirbyteMessage]: + return [message for message in self._get_message_by_types([Type.TRACE]) if message.trace.type == trace_type] + + +def read( + source: Source, + config: Mapping[str, Any], + catalog: ConfiguredAirbyteCatalog, + state: Optional[Any] = None, + expecting_exception: bool = False, +) -> EntrypointOutput: + """ + config and state must be json serializable + + :param expecting_exception: By default if there is an uncaught exception, the exception will be printed out. If this is expected, please + provide expecting_exception=True so that the test output logs are cleaner + """ + log_capture_buffer = StringIO() + stream_handler = logging.StreamHandler(log_capture_buffer) + stream_handler.setLevel(logging.INFO) + stream_handler.setFormatter(AirbyteLogFormatter()) + parent_logger = logging.getLogger("") + parent_logger.addHandler(stream_handler) + + with tempfile.TemporaryDirectory() as tmp_directory: + tmp_directory_path = Path(tmp_directory) + args = [ + "read", + "--config", + make_file(tmp_directory_path / "config.json", config), + "--catalog", + make_file(tmp_directory_path / "catalog.json", catalog.json()), + ] + if state is not None: + args.extend( + [ + "--state", + make_file(tmp_directory_path / "state.json", state), + ] + ) + args.append("--debug") + source_entrypoint = AirbyteEntrypoint(source) + parsed_args = source_entrypoint.parse_args(args) + + messages = [] + uncaught_exception = None + try: + for message in source_entrypoint.run(parsed_args): + messages.append(message) + except Exception as exception: + if not expecting_exception: + print("Printing unexpected error from entrypoint_wrapper") + print("".join(traceback.format_exception(None, exception, exception.__traceback__))) + uncaught_exception = exception + + captured_logs = log_capture_buffer.getvalue().split("\n")[:-1] + + parent_logger.removeHandler(stream_handler) + + return EntrypointOutput(messages + captured_logs, uncaught_exception) + + +def make_file(path: Path, file_contents: Optional[Union[str, Mapping[str, Any], List[Mapping[str, Any]]]]) -> str: + if isinstance(file_contents, str): + path.write_text(file_contents) + else: + path.write_text(json.dumps(file_contents)) + return str(path) diff --git a/airbyte-cdk/python/airbyte_cdk/test/mock_http/__init__.py b/airbyte-cdk/python/airbyte_cdk/test/mock_http/__init__.py new file mode 100644 index 000000000000..88b28b0225e8 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/test/mock_http/__init__.py @@ -0,0 +1,6 @@ +from airbyte_cdk.test.mock_http.matcher import HttpRequestMatcher +from airbyte_cdk.test.mock_http.request import HttpRequest +from airbyte_cdk.test.mock_http.response import HttpResponse +from airbyte_cdk.test.mock_http.mocker import HttpMocker + +__all__ = ["HttpMocker", "HttpRequest", "HttpRequestMatcher", "HttpResponse"] diff --git a/airbyte-cdk/python/airbyte_cdk/test/mock_http/matcher.py b/airbyte-cdk/python/airbyte_cdk/test/mock_http/matcher.py new file mode 100644 index 000000000000..441a765b7321 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/test/mock_http/matcher.py @@ -0,0 +1,35 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_cdk.test.mock_http.request import HttpRequest + + +class HttpRequestMatcher: + def __init__(self, request: HttpRequest, minimum_number_of_expected_match: int): + self._request_to_match = request + self._minimum_number_of_expected_match = minimum_number_of_expected_match + self._actual_number_of_matches = 0 + + def matches(self, request: HttpRequest) -> bool: + hit = request.matches(self._request_to_match) + if hit: + self._actual_number_of_matches += 1 + return hit + + def has_expected_match_count(self) -> bool: + return self._actual_number_of_matches >= self._minimum_number_of_expected_match + + @property + def actual_number_of_matches(self) -> int: + return self._actual_number_of_matches + + @property + def request(self) -> HttpRequest: + return self._request_to_match + + def __str__(self) -> str: + return ( + f"HttpRequestMatcher(" + f"request_to_match={self._request_to_match}, " + f"minimum_number_of_expected_match={self._minimum_number_of_expected_match}, " + f"actual_number_of_matches={self._actual_number_of_matches})" + ) diff --git a/airbyte-cdk/python/airbyte_cdk/test/mock_http/mocker.py b/airbyte-cdk/python/airbyte_cdk/test/mock_http/mocker.py new file mode 100644 index 000000000000..c2ca734047ac --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/test/mock_http/mocker.py @@ -0,0 +1,123 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import contextlib +import functools +from enum import Enum +from types import TracebackType +from typing import Callable, List, Optional, Union + +import requests_mock +from airbyte_cdk.test.mock_http import HttpRequest, HttpRequestMatcher, HttpResponse + + +class SupportedHttpMethods(str, Enum): + GET = "get" + POST = "post" + + +class HttpMocker(contextlib.ContextDecorator): + """ + WARNING 1: This implementation only works if the lib used to perform HTTP requests is `requests`. + + WARNING 2: Given multiple requests that are not mutually exclusive, the request will match the first one. This can happen in scenarios + where the same request is added twice (in which case there will always be an exception because we will never match the second + request) or in a case like this: + ``` + http_mocker.get(HttpRequest(_A_URL, headers={"less_granular": "1", "more_granular": "2"}), <...>) + http_mocker.get(HttpRequest(_A_URL, headers={"less_granular": "1"}), <...>) + requests.get(_A_URL, headers={"less_granular": "1", "more_granular": "2"}) + ``` + In the example above, the matcher would match the second mock as requests_mock iterate over the matcher in reverse order (see + https://github.com/jamielennox/requests-mock/blob/c06f124a33f56e9f03840518e19669ba41b93202/requests_mock/adapter.py#L246) even + though the request sent is a better match for the first `http_mocker.get`. + """ + + def __init__(self) -> None: + self._mocker = requests_mock.Mocker() + self._matchers: List[HttpRequestMatcher] = [] + + def __enter__(self) -> "HttpMocker": + self._mocker.__enter__() + return self + + def __exit__(self, exc_type: Optional[BaseException], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType]) -> None: + self._mocker.__exit__(exc_type, exc_val, exc_tb) + + def _validate_all_matchers_called(self) -> None: + for matcher in self._matchers: + if not matcher.has_expected_match_count(): + raise ValueError(f"Invalid number of matches for `{matcher}`") + + def _mock_request_method( + self, method: SupportedHttpMethods, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]] + ) -> None: + if isinstance(responses, HttpResponse): + responses = [responses] + + matcher = HttpRequestMatcher(request, len(responses)) + self._matchers.append(matcher) + + getattr(self._mocker, method)( + requests_mock.ANY, + additional_matcher=self._matches_wrapper(matcher), + response_list=[{"text": response.body, "status_code": response.status_code} for response in responses], + ) + + def get(self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]) -> None: + self._mock_request_method(SupportedHttpMethods.GET, request, responses) + + def post(self, request: HttpRequest, responses: Union[HttpResponse, List[HttpResponse]]) -> None: + self._mock_request_method(SupportedHttpMethods.POST, request, responses) + + @staticmethod + def _matches_wrapper(matcher: HttpRequestMatcher) -> Callable[[requests_mock.request._RequestObjectProxy], bool]: + def matches(requests_mock_request: requests_mock.request._RequestObjectProxy) -> bool: + # query_params are provided as part of `requests_mock_request.url` + http_request = HttpRequest( + requests_mock_request.url, query_params={}, headers=requests_mock_request.headers, body=requests_mock_request.body + ) + return matcher.matches(http_request) + + return matches + + def assert_number_of_calls(self, request: HttpRequest, number_of_calls: int) -> None: + corresponding_matchers = list(filter(lambda matcher: matcher.request == request, self._matchers)) + if len(corresponding_matchers) != 1: + raise ValueError(f"Was expecting only one matcher to match the request but got `{corresponding_matchers}`") + + assert corresponding_matchers[0].actual_number_of_matches == number_of_calls + + # trying to type that using callables provides the error `incompatible with return type "_F" in supertype "ContextDecorator"` + def __call__(self, f): # type: ignore + @functools.wraps(f) + def wrapper(*args, **kwargs): # type: ignore # this is a very generic wrapper that does not need to be typed + with self: + assertion_error = None + + kwargs["http_mocker"] = self + try: + result = f(*args, **kwargs) + except requests_mock.NoMockAddress as no_mock_exception: + matchers_as_string = "\n\t".join(map(lambda matcher: str(matcher.request), self._matchers)) + raise ValueError( + f"No matcher matches {no_mock_exception.args[0]} with headers `{no_mock_exception.request.headers}` " + f"and body `{no_mock_exception.request.body}`. " + f"Matchers currently configured are:\n\t{matchers_as_string}." + ) from no_mock_exception + except AssertionError as test_assertion: + assertion_error = test_assertion + + # We validate the matchers before raising the assertion error because we want to show the tester if an HTTP request wasn't + # mocked correctly + try: + self._validate_all_matchers_called() + except ValueError as http_mocker_exception: + # This seems useless as it catches ValueError and raises ValueError but without this, the prevailing error message in + # the output is the function call that failed the assertion, whereas raising `ValueError(http_mocker_exception)` + # like we do here provides additional context for the exception. + raise ValueError(http_mocker_exception) from None + if assertion_error: + raise assertion_error + return result + + return wrapper diff --git a/airbyte-cdk/python/airbyte_cdk/test/mock_http/request.py b/airbyte-cdk/python/airbyte_cdk/test/mock_http/request.py new file mode 100644 index 000000000000..a2b6bdb9430a --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/test/mock_http/request.py @@ -0,0 +1,87 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from typing import Any, List, Mapping, Optional, Union +from urllib.parse import parse_qs, urlencode, urlparse + +ANY_QUERY_PARAMS = "any query_parameters" + + +def _is_subdict(small: Mapping[str, str], big: Mapping[str, str]) -> bool: + return dict(big, **small) == big + + +class HttpRequest: + def __init__( + self, + url: str, + query_params: Optional[Union[str, Mapping[str, Union[str, List[str]]]]] = None, + headers: Optional[Mapping[str, str]] = None, + body: Optional[Union[str, bytes, Mapping[str, Any]]] = None, + ) -> None: + self._parsed_url = urlparse(url) + self._query_params = query_params + if not self._parsed_url.query and query_params: + self._parsed_url = urlparse(f"{url}?{self._encode_qs(query_params)}") + elif self._parsed_url.query and query_params: + raise ValueError("If query params are provided as part of the url, `query_params` should be empty") + + self._headers = headers or {} + self._body = body + + @staticmethod + def _encode_qs(query_params: Union[str, Mapping[str, Union[str, List[str]]]]) -> str: + if isinstance(query_params, str): + return query_params + return urlencode(query_params, doseq=True) + + def matches(self, other: Any) -> bool: + """ + If the body of any request is a Mapping, we compare as Mappings which means that the order is not important. + If the body is a string, encoding ISO-8859-1 will be assumed + Headers only need to be a subset of `other` in order to match + """ + if isinstance(other, HttpRequest): + # if `other` is a mapping, we match as an object and formatting is not considers + if isinstance(self._body, Mapping) or isinstance(other._body, Mapping): + body_match = self._to_mapping(self._body) == self._to_mapping(other._body) + else: + body_match = self._to_bytes(self._body) == self._to_bytes(other._body) + + return ( + self._parsed_url.scheme == other._parsed_url.scheme + and self._parsed_url.hostname == other._parsed_url.hostname + and self._parsed_url.path == other._parsed_url.path + and ( + ANY_QUERY_PARAMS in (self._query_params, other._query_params) + or parse_qs(self._parsed_url.query) == parse_qs(other._parsed_url.query) + ) + and _is_subdict(other._headers, self._headers) + and body_match + ) + return False + + @staticmethod + def _to_mapping(body: Optional[Union[str, bytes, Mapping[str, Any]]]) -> Optional[Mapping[str, Any]]: + if isinstance(body, Mapping): + return body + elif isinstance(body, bytes): + return json.loads(body.decode()) # type: ignore # assumes return type of Mapping[str, Any] + elif isinstance(body, str): + return json.loads(body) # type: ignore # assumes return type of Mapping[str, Any] + return None + + @staticmethod + def _to_bytes(body: Optional[Union[str, bytes]]) -> bytes: + if isinstance(body, bytes): + return body + elif isinstance(body, str): + # `ISO-8859-1` is the default encoding used by requests + return body.encode("ISO-8859-1") + return b"" + + def __str__(self) -> str: + return f"{self._parsed_url} with headers {self._headers} and body {self._body!r})" + + def __repr__(self) -> str: + return f"HttpRequest(request={self._parsed_url}, headers={self._headers}, body={self._body!r})" diff --git a/airbyte-cdk/python/airbyte_cdk/test/mock_http/response.py b/airbyte-cdk/python/airbyte_cdk/test/mock_http/response.py new file mode 100644 index 000000000000..3aea355cd4e0 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/test/mock_http/response.py @@ -0,0 +1,15 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + + +class HttpResponse: + def __init__(self, body: str, status_code: int = 200): + self._body = body + self._status_code = status_code + + @property + def body(self) -> str: + return self._body + + @property + def status_code(self) -> int: + return self._status_code diff --git a/airbyte-cdk/python/airbyte_cdk/test/mock_http/response_builder.py b/airbyte-cdk/python/airbyte_cdk/test/mock_http/response_builder.py new file mode 100644 index 000000000000..02dd3d285107 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/test/mock_http/response_builder.py @@ -0,0 +1,213 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import functools +import json +from abc import ABC, abstractmethod +from pathlib import Path as FilePath +from typing import Any, Dict, List, Optional, Union + +from airbyte_cdk.test.mock_http import HttpResponse + + +def _extract(path: List[str], response_template: Dict[str, Any]) -> Any: + return functools.reduce(lambda a, b: a[b], path, response_template) + + +def _replace_value(dictionary: Dict[str, Any], path: List[str], value: Any) -> None: + current = dictionary + for key in path[:-1]: + current = current[key] + current[path[-1]] = value + + +def _write(dictionary: Dict[str, Any], path: List[str], value: Any) -> None: + current = dictionary + for key in path[:-1]: + current = current.setdefault(key, {}) + current[path[-1]] = value + + +class Path(ABC): + @abstractmethod + def write(self, template: Dict[str, Any], value: Any) -> None: + pass + + @abstractmethod + def update(self, template: Dict[str, Any], value: Any) -> None: + pass + + def extract(self, template: Dict[str, Any]) -> Any: + pass + + +class FieldPath(Path): + def __init__(self, field: str): + self._path = [field] + + def write(self, template: Dict[str, Any], value: Any) -> None: + _write(template, self._path, value) + + def update(self, template: Dict[str, Any], value: Any) -> None: + _replace_value(template, self._path, value) + + def extract(self, template: Dict[str, Any]) -> Any: + return _extract(self._path, template) + + def __str__(self) -> str: + return f"FieldPath(field={self._path[0]})" + + +class NestedPath(Path): + def __init__(self, path: List[str]): + self._path = path + + def write(self, template: Dict[str, Any], value: Any) -> None: + _write(template, self._path, value) + + def update(self, template: Dict[str, Any], value: Any) -> None: + _replace_value(template, self._path, value) + + def extract(self, template: Dict[str, Any]) -> Any: + return _extract(self._path, template) + + def __str__(self) -> str: + return f"NestedPath(path={self._path})" + + +class PaginationStrategy(ABC): + @abstractmethod + def update(self, response: Dict[str, Any]) -> None: + pass + + +class FieldUpdatePaginationStrategy(PaginationStrategy): + def __init__(self, path: Path, value: Any): + self._path = path + self._value = value + + def update(self, response: Dict[str, Any]) -> None: + self._path.update(response, self._value) + + +class RecordBuilder: + def __init__(self, template: Dict[str, Any], id_path: Optional[Path], cursor_path: Optional[Union[FieldPath, NestedPath]]): + self._record = template + self._id_path = id_path + self._cursor_path = cursor_path + + self._validate_template() + + def _validate_template(self) -> None: + paths_to_validate = [ + ("_id_path", self._id_path), + ("_cursor_path", self._cursor_path), + ] + for field_name, field_path in paths_to_validate: + self._validate_field(field_name, field_path) + + def _validate_field(self, field_name: str, path: Optional[Path]) -> None: + try: + if path and not path.extract(self._record): + raise ValueError(f"{field_name} `{path}` was provided but it is not part of the template `{self._record}`") + except (IndexError, KeyError) as exception: + raise ValueError(f"{field_name} `{path}` was provided but it is not part of the template `{self._record}`") from exception + + def with_id(self, identifier: Any) -> "RecordBuilder": + self._set_field("id", self._id_path, identifier) + return self + + def with_cursor(self, cursor_value: Any) -> "RecordBuilder": + self._set_field("cursor", self._cursor_path, cursor_value) + return self + + def with_field(self, path: Path, value: Any) -> "RecordBuilder": + path.write(self._record, value) + return self + + def _set_field(self, field_name: str, path: Optional[Path], value: Any) -> None: + if not path: + raise ValueError( + f"{field_name}_path was not provided and hence, the record {field_name} can't be modified. Please provide `id_field` while " + f"instantiating RecordBuilder to leverage this capability" + ) + path.update(self._record, value) + + def build(self) -> Dict[str, Any]: + return self._record + + +class HttpResponseBuilder: + def __init__( + self, + template: Dict[str, Any], + records_path: Union[FieldPath, NestedPath], + pagination_strategy: Optional[PaginationStrategy] + ): + self._response = template + self._records: List[RecordBuilder] = [] + self._records_path = records_path + self._pagination_strategy = pagination_strategy + self._status_code = 200 + + def with_record(self, record: RecordBuilder) -> "HttpResponseBuilder": + self._records.append(record) + return self + + def with_pagination(self) -> "HttpResponseBuilder": + if not self._pagination_strategy: + raise ValueError( + "`pagination_strategy` was not provided and hence, fields related to the pagination can't be modified. Please provide " + "`pagination_strategy` while instantiating ResponseBuilder to leverage this capability" + ) + self._pagination_strategy.update(self._response) + return self + + def with_status_code(self, status_code: int) -> "HttpResponseBuilder": + self._status_code = status_code + return self + + def build(self) -> HttpResponse: + self._records_path.update(self._response, [record.build() for record in self._records]) + return HttpResponse(json.dumps(self._response), self._status_code) + + +def _get_unit_test_folder(execution_folder: str) -> FilePath: + path = FilePath(execution_folder) + while path.name != "unit_tests": + if path.name == path.root or path.name == path.drive: + raise ValueError(f"Could not find `unit_tests` folder as a parent of {execution_folder}") + path = path.parent + return path + + +def find_template(resource: str, execution_folder: str) -> Dict[str, Any]: + response_template_filepath = str(_get_unit_test_folder(execution_folder) / "resource" / "http" / "response" / f"{resource}.json") + with open(response_template_filepath, "r") as template_file: + return json.load(template_file) # type: ignore # we assume the dev correctly set up the resource file + + +def create_record_builder( + response_template: Dict[str, Any], + records_path: Union[FieldPath, NestedPath], + record_id_path: Optional[Path] = None, + record_cursor_path: Optional[Union[FieldPath, NestedPath]] = None, +) -> RecordBuilder: + """ + This will use the first record define at `records_path` as a template for the records. If more records are defined, they will be ignored + """ + try: + record_template = records_path.extract(response_template)[0] + if not record_template: + raise ValueError(f"Could not extract any record from template at path `{records_path}`. " + f"Please fix the template to provide a record sample or fix `records_path`.") + return RecordBuilder(record_template, record_id_path, record_cursor_path) + except (IndexError, KeyError): + raise ValueError(f"Error while extracting records at path `{records_path}` from response template `{response_template}`") + + +def create_response_builder( + response_template: Dict[str, Any], + records_path: Union[FieldPath, NestedPath], + pagination_strategy: Optional[PaginationStrategy] = None +) -> HttpResponseBuilder: + return HttpResponseBuilder(response_template, records_path, pagination_strategy) diff --git a/airbyte-cdk/python/airbyte_cdk/test/state_builder.py b/airbyte-cdk/python/airbyte_cdk/test/state_builder.py new file mode 100644 index 000000000000..96c9a6161172 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/test/state_builder.py @@ -0,0 +1,23 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Any, Dict, List + + +class StateBuilder: + def __init__(self) -> None: + self._state: List[Dict[str, Any]] = [] + + def with_stream_state(self, stream_name: str, state: Any) -> "StateBuilder": + self._state.append({ + "type": "STREAM", + "stream": { + "stream_state": state, + "stream_descriptor": { + "name": stream_name + } + } + }) + return self + + def build(self) -> List[Dict[str, Any]]: + return self._state diff --git a/airbyte-cdk/python/airbyte_cdk/utils/__init__.py b/airbyte-cdk/python/airbyte_cdk/utils/__init__.py index 3b0185ef0173..46c30ee58e88 100644 --- a/airbyte-cdk/python/airbyte_cdk/utils/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/utils/__init__.py @@ -2,7 +2,8 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from .is_cloud_environment import is_cloud_environment from .schema_inferrer import SchemaInferrer from .traced_exception import AirbyteTracedException -__all__ = ["AirbyteTracedException", "SchemaInferrer"] +__all__ = ["AirbyteTracedException", "SchemaInferrer", "is_cloud_environment"] diff --git a/airbyte-cdk/python/airbyte_cdk/utils/analytics_message.py b/airbyte-cdk/python/airbyte_cdk/utils/analytics_message.py new file mode 100644 index 000000000000..54c3e984f93c --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/utils/analytics_message.py @@ -0,0 +1,17 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import time +from typing import Any, Optional + +from airbyte_cdk.models import AirbyteAnalyticsTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type + + +def create_analytics_message(type: str, value: Optional[Any]) -> AirbyteMessage: + return AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.ANALYTICS, + emitted_at=time.time() * 1000, + analytics=AirbyteAnalyticsTraceMessage(type=type, value=str(value) if value is not None else None), + ), + ) diff --git a/airbyte-cdk/python/airbyte_cdk/utils/constants.py b/airbyte-cdk/python/airbyte_cdk/utils/constants.py new file mode 100644 index 000000000000..1d6345cbd8f4 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/utils/constants.py @@ -0,0 +1,5 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +ENV_REQUEST_CACHE_PATH = "REQUEST_CACHE_PATH" diff --git a/airbyte-cdk/python/airbyte_cdk/utils/datetime_format_inferrer.py b/airbyte-cdk/python/airbyte_cdk/utils/datetime_format_inferrer.py index 8e29a274d25d..cd423db9c201 100644 --- a/airbyte-cdk/python/airbyte_cdk/utils/datetime_format_inferrer.py +++ b/airbyte-cdk/python/airbyte_cdk/utils/datetime_format_inferrer.py @@ -36,10 +36,14 @@ def _can_be_datetime(self, value: Any) -> bool: This is the case if the value is a string or an integer between 1_000_000_000 and 2_000_000_000 for seconds or between 1_000_000_000_000 and 2_000_000_000_000 for milliseconds. This is separate from the format check for performance reasons""" - for timestamp_range in self._timestamp_heuristic_ranges: - if isinstance(value, str) and (not value.isdecimal() or int(value) in timestamp_range): - return True - if isinstance(value, int) and value in timestamp_range: + if isinstance(value, (str, int)): + try: + value_as_int = int(value) + for timestamp_range in self._timestamp_heuristic_ranges: + if value_as_int in timestamp_range: + return True + except ValueError: + # given that it's not parsable as an int, it can represent a datetime with one of the self._formats return True return False diff --git a/airbyte-cdk/python/airbyte_cdk/utils/is_cloud_environment.py b/airbyte-cdk/python/airbyte_cdk/utils/is_cloud_environment.py new file mode 100644 index 000000000000..25b1eee87fad --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/utils/is_cloud_environment.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import os + +CLOUD_DEPLOYMENT_MODE = "cloud" + + +def is_cloud_environment() -> bool: + """ + Returns True if the connector is running in a cloud environment, False otherwise. + + The function checks the value of the DEPLOYMENT_MODE environment variable which is set by the platform. + This function can be used to determine whether stricter security measures should be applied. + """ + deployment_mode = os.environ.get("DEPLOYMENT_MODE", "") + return deployment_mode.casefold() == CLOUD_DEPLOYMENT_MODE diff --git a/airbyte-cdk/python/airbyte_cdk/utils/oneof_option_config.py b/airbyte-cdk/python/airbyte_cdk/utils/oneof_option_config.py new file mode 100644 index 000000000000..17ebf0511beb --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/utils/oneof_option_config.py @@ -0,0 +1,33 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Dict + + +class OneOfOptionConfig: + """ + Base class to configure a Pydantic model that's used as a oneOf option in a parent model in a way that's compatible with all Airbyte consumers. + + Inherit from this class in the nested Config class in a model and set title and description (these show up in the UI) and discriminator (this is making sure it's marked as required in the schema). + + Usage: + + ```python + class OptionModel(BaseModel): + mode: Literal["option_a"] = Field("option_a", const=True) + option_a_field: str = Field(...) + + class Config(OneOfOptionConfig): + title = "Option A" + description = "Option A description" + discriminator = "mode" + ``` + """ + + @staticmethod + def schema_extra(schema: Dict[str, Any], model: Any) -> None: + if hasattr(model.Config, "description"): + schema["description"] = model.Config.description + if hasattr(model.Config, "discriminator"): + schema.setdefault("required", []).append(model.Config.discriminator) diff --git a/airbyte-cdk/python/airbyte_cdk/utils/stream_status_utils.py b/airbyte-cdk/python/airbyte_cdk/utils/stream_status_utils.py index cd9e4527688d..bdd36acf9b34 100644 --- a/airbyte-cdk/python/airbyte_cdk/utils/stream_status_utils.py +++ b/airbyte-cdk/python/airbyte_cdk/utils/stream_status_utils.py @@ -7,17 +7,17 @@ from airbyte_cdk.models import ( AirbyteMessage, + AirbyteStream, AirbyteStreamStatus, AirbyteStreamStatusTraceMessage, AirbyteTraceMessage, - ConfiguredAirbyteStream, StreamDescriptor, TraceType, ) from airbyte_cdk.models import Type as MessageType -def as_airbyte_message(stream: ConfiguredAirbyteStream, current_status: AirbyteStreamStatus) -> AirbyteMessage: +def as_airbyte_message(stream: AirbyteStream, current_status: AirbyteStreamStatus) -> AirbyteMessage: """ Builds an AirbyteStreamStatusTraceMessage for the provided stream """ @@ -28,7 +28,7 @@ def as_airbyte_message(stream: ConfiguredAirbyteStream, current_status: AirbyteS type=TraceType.STREAM_STATUS, emitted_at=now_millis, stream_status=AirbyteStreamStatusTraceMessage( - stream_descriptor=StreamDescriptor(name=stream.stream.name, namespace=stream.stream.namespace), + stream_descriptor=StreamDescriptor(name=stream.name, namespace=stream.namespace), status=current_status, ), ) diff --git a/airbyte-cdk/python/airbyte_cdk/utils/traced_exception.py b/airbyte-cdk/python/airbyte_cdk/utils/traced_exception.py index ad6d7a3e02b9..dec09fcf1929 100644 --- a/airbyte-cdk/python/airbyte_cdk/utils/traced_exception.py +++ b/airbyte-cdk/python/airbyte_cdk/utils/traced_exception.py @@ -4,6 +4,7 @@ import traceback from datetime import datetime +from typing import Optional from airbyte_cdk.models import ( AirbyteConnectionStatus, @@ -25,10 +26,10 @@ class AirbyteTracedException(Exception): def __init__( self, - internal_message: str = None, - message: str = None, + internal_message: Optional[str] = None, + message: Optional[str] = None, failure_type: FailureType = FailureType.system_error, - exception: BaseException = None, + exception: Optional[BaseException] = None, ): """ :param internal_message: the internal error that caused the failure @@ -71,7 +72,7 @@ def as_connection_status_message(self) -> AirbyteMessage: ) return output_message - def emit_message(self): + def emit_message(self) -> None: """ Prints the exception as an AirbyteTraceMessage. Note that this will be called automatically on uncaught exceptions when using the airbyte_cdk entrypoint. @@ -81,9 +82,9 @@ def emit_message(self): print(filtered_message) @classmethod - def from_exception(cls, exc: Exception, *args, **kwargs) -> "AirbyteTracedException": + def from_exception(cls, exc: BaseException, *args, **kwargs) -> "AirbyteTracedException": # type: ignore # ignoring because of args and kwargs """ Helper to create an AirbyteTracedException from an existing exception :param exc: the exception that caused the error """ - return cls(internal_message=str(exc), exception=exc, *args, **kwargs) + return cls(internal_message=str(exc), exception=exc, *args, **kwargs) # type: ignore # ignoring because of args and kwargs diff --git a/airbyte-cdk/python/bin/generate-component-manifest-files.sh b/airbyte-cdk/python/bin/generate-component-manifest-files.sh index a1939345e649..d366d3ca9cde 100755 --- a/airbyte-cdk/python/bin/generate-component-manifest-files.sh +++ b/airbyte-cdk/python/bin/generate-component-manifest-files.sh @@ -19,7 +19,8 @@ function main() { --input "/airbyte/$YAML_DIR/$filename_wo_ext.yaml" \ --output "/airbyte/$OUTPUT_DIR/$filename_wo_ext.py" \ --disable-timestamp \ - --enum-field-as-literal one + --enum-field-as-literal one \ + --set-default-enum-member # There is a limitation of Pydantic where a model's private fields starting with an underscore are inaccessible. # The Pydantic model generator replaces special characters like $ with the underscore which results in all diff --git a/airbyte-cdk/python/bin/run-cats-with-local-cdk.sh b/airbyte-cdk/python/bin/run-cats-with-local-cdk.sh deleted file mode 100755 index ca29f946ba2a..000000000000 --- a/airbyte-cdk/python/bin/run-cats-with-local-cdk.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env sh - -ROOT_DIR="$(git rev-parse --show-toplevel)" - -REPO_NAME="$(basename $ROOT_DIR)" -if [ "$REPO_NAME" != "airbyte" ]; then - echo "This script must be run from the airbyte repo." 1>&2 - exit 1 -fi - -source "$ROOT_DIR/airbyte-integrations/scripts/utils.sh" - -USAGE="$(basename "$0") [-h] [-c connector1,connector2,...] -- Run connector acceptance tests (CATs) against the local CDK, if relevant.\n - -h show help text\n - -c comma-separated connector names (defaults to all connectors)" - -OUTPUT_DIR=/tmp/cat-output -SCRIPT=/tmp/run-cats-with-local-cdk.sh -# Clean up from previous test runs -rm -rf $OUTPUT_DIR && mkdir $OUTPUT_DIR - -while getopts ":hc:" opt; do - case $opt in - h ) echo $USAGE - exit 0 ;; - c ) connectors="${OPTARG}" ;; - * ) die "Unrecognized argument" ;; - esac -done - -[ -n "$connectors" ] || die "Please specify one or more connectors." - -connectors=$(echo $connectors | tr ',' ' ') - -echo "Running CATs for ${connectors}" -echo "" -echo $connectors | xargs -P 0 -n 1 -I % $ROOT_DIR/airbyte-integrations/scripts/run-acceptance-test-docker.sh % $OUTPUT_DIR - -# Print connectors with CATs that passed -for directory in $OUTPUT_DIR/*; do - SOURCE_NAME="$(basename $directory)" - CONNECTOR_OUTPUT_LOC="$OUTPUT_DIR/$SOURCE_NAME/$SOURCE_NAME" - if [ "$(cat $CONNECTOR_OUTPUT_LOC.exit-code)" = 0 ]; then - echo "$SOURCE_NAME: CATs ran successfully!" - fi -done - -echo "" - -# Print errors -for directory in $OUTPUT_DIR/*; do - SOURCE_NAME="$(basename $directory)" - CONNECTOR_OUTPUT_LOC="$OUTPUT_DIR/$SOURCE_NAME/$SOURCE_NAME" - if [ "$(cat $CONNECTOR_OUTPUT_LOC.exit-code)" != 0 ]; then - echo "$SOURCE_NAME errors:" - echo "$(cat $CONNECTOR_OUTPUT_LOC.out)" - echo "$(cat $CONNECTOR_OUTPUT_LOC.err)" - echo "" - fi -done - -echo "Done." diff --git a/airbyte-cdk/python/bin/run-mypy-on-modified-files.sh b/airbyte-cdk/python/bin/run-mypy-on-modified-files.sh index 6b45a7548f9d..0b42bc8a7a8f 100755 --- a/airbyte-cdk/python/bin/run-mypy-on-modified-files.sh +++ b/airbyte-cdk/python/bin/run-mypy-on-modified-files.sh @@ -1,3 +1,13 @@ +#!/usr/bin/env sh + set -e + +# Ensure script always runs from the project directory. +cd "$(dirname "${0}")/.." || exit 1 + # TODO change this to include unit_tests as well once it's in a good state -{ git diff --name-only --relative ':(exclude)unit_tests'; git diff --name-only --staged --relative ':(exclude)unit_tests'; git diff --name-only master... --relative ':(exclude)unit_tests'; } | grep -E '\.py$' | sort | uniq | xargs .venv/bin/python -m mypy --config-file mypy.ini --install-types --non-interactive +{ + git diff --name-only --relative ':(exclude)unit_tests' + git diff --name-only --staged --relative ':(exclude)unit_tests' + git diff --name-only master... --relative ':(exclude)unit_tests' +} | grep -E '\.py$' | sort | uniq | xargs .venv/bin/python -m mypy --config-file mypy.ini --install-types --non-interactive diff --git a/airbyte-cdk/python/build.gradle b/airbyte-cdk/python/build.gradle index 9624bcac5804..63cc9992a73b 100644 --- a/airbyte-cdk/python/build.gradle +++ b/airbyte-cdk/python/build.gradle @@ -1,35 +1,25 @@ plugins { id 'airbyte-python' - id 'airbyte-docker' + id 'airbyte-docker-legacy' } -airbytePython { - moduleDirectory 'airbyte_cdk' -} - -task generateComponentManifestClassFiles(type: Exec) { +def generateComponentManifestClassFiles = tasks.register('generateComponentManifestClassFiles', Exec) { environment 'ROOT_DIR', rootDir.absolutePath commandLine 'bin/generate-component-manifest-files.sh' - dependsOn ':tools:code-generator:airbyteDocker' +} +generateComponentManifestClassFiles.configure { + dependsOn project(':tools:code-generator').tasks.named('assemble') +} +tasks.register('generate').configure { + dependsOn generateComponentManifestClassFiles } -task validateSourceYamlManifest(type: Exec) { +tasks.register('validateSourceYamlManifest', Exec) { environment 'ROOT_DIR', rootDir.absolutePath commandLine 'bin/validate-yaml-schema.sh' } -task runLowCodeConnectorUnitTests(type: Exec) { +tasks.register('runLowCodeConnectorUnitTests', Exec) { environment 'ROOT_DIR', rootDir.absolutePath commandLine 'bin/low-code-unit-tests.sh' } - -task runMypyOnModifiedFiles(type: Exec) { - environment 'ROOT_DIR', rootDir.absolutePath - commandLine 'bin/run-mypy-on-modified-files.sh' -} - -blackFormat.dependsOn runMypyOnModifiedFiles -isortFormat.dependsOn generateComponentManifestClassFiles -flakeCheck.dependsOn generateComponentManifestClassFiles -installReqs.dependsOn generateComponentManifestClassFiles -runMypyOnModifiedFiles.dependsOn generateComponentManifestClassFiles \ No newline at end of file diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index a2860804f5e8..483305c34469 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -17,11 +17,26 @@ fastavro_dependency = "fastavro~=1.8.0" pyarrow_dependency = "pyarrow==12.0.1" +langchain_dependency = "langchain==0.0.271" +openai_dependency = "openai[embeddings]==0.27.9" +cohere_dependency = "cohere==4.21" +tiktoken_dependency = "tiktoken==0.4.0" + +unstructured_dependencies = [ + "unstructured==0.10.27", # can't be bumped higher due to transitive dependencies we can't provide + "unstructured[docx,pptx]==0.10.27", + "pdf2image==1.16.3", + "pdfminer.six==20221105", + "unstructured.pytesseract>=0.3.12", + "pytesseract==0.3.10", + "markdown", +] + setup( name="airbyte-cdk", # The version of the airbyte-cdk package is used at runtime to validate manifests. That validation must be # updated if our semver format changes such as using release candidate versions. - version="0.51.6", + version="0.59.0", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", @@ -50,15 +65,16 @@ packages=find_packages(exclude=("unit_tests",)), package_data={"airbyte_cdk": ["py.typed", "sources/declarative/declarative_component_schema.yaml"]}, install_requires=[ - "airbyte-protocol-models==0.4.0", + "airbyte-protocol-models==0.5.1", "backoff", "dpath~=2.0.1", "isodate~=0.6.1", "jsonschema~=3.2.0", "jsonref~=0.2", - "pendulum", + "pendulum<3.0.0", "genson==1.2.2", - "pydantic>=1.9.2,<2.0.0", + "pydantic>=1.10.8,<2.0.0", + "pyrate-limiter~=3.1.0", "python-dateutil", "PyYAML>=6.0.1", "requests", @@ -82,6 +98,11 @@ "pytest-httpserver", "pandas==2.0.3", pyarrow_dependency, + langchain_dependency, + openai_dependency, + cohere_dependency, + tiktoken_dependency, + *unstructured_dependencies, ], "sphinx-docs": [ "Sphinx~=4.2", @@ -91,6 +112,8 @@ avro_dependency, fastavro_dependency, pyarrow_dependency, + *unstructured_dependencies, ], + "vector-db-based": [langchain_dependency, openai_dependency, cohere_dependency, tiktoken_dependency], }, ) diff --git a/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py b/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py index f9cd37e2750a..190f8d4bcb56 100644 --- a/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py +++ b/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py @@ -59,12 +59,16 @@ "page_size": _page_size, "page_size_option": {"inject_into": "request_parameter", "field_name": "page_size"}, "page_token_option": {"inject_into": "path", "type": "RequestPath"}, - "pagination_strategy": {"type": "CursorPagination", "cursor_value": "{{ response._metadata.next }}", "page_size": _page_size}, + "pagination_strategy": { + "type": "CursorPagination", + "cursor_value": "{{ response._metadata.next }}", + "page_size": _page_size, + }, }, "partition_router": { "type": "ListPartitionRouter", "values": ["0", "1", "2", "3", "4", "5", "6", "7"], - "cursor_field": "item_id" + "cursor_field": "item_id", }, "" "requester": { @@ -89,10 +93,10 @@ "type": "object", "required": [], "properties": {}, - "additionalProperties": True + "additionalProperties": True, }, - "type": "Spec" - } + "type": "Spec", + }, } OAUTH_MANIFEST = { @@ -104,20 +108,21 @@ "page_size": _page_size, "page_size_option": {"inject_into": "request_parameter", "field_name": "page_size"}, "page_token_option": {"inject_into": "path", "type": "RequestPath"}, - "pagination_strategy": {"type": "CursorPagination", "cursor_value": "{{ response._metadata.next }}", "page_size": _page_size}, + "pagination_strategy": { + "type": "CursorPagination", + "cursor_value": "{{ response._metadata.next }}", + "page_size": _page_size, + }, }, "partition_router": { "type": "ListPartitionRouter", "values": ["0", "1", "2", "3", "4", "5", "6", "7"], - "cursor_field": "item_id" + "cursor_field": "item_id", }, "" "requester": { "path": "/v3/marketing/lists", - "authenticator": { - "type": "OAuthAuthenticator", - "api_token": "{{ config.apikey }}" - }, + "authenticator": {"type": "OAuthAuthenticator", "api_token": "{{ config.apikey }}"}, "request_parameters": {"a_param": "10"}, }, "record_selector": {"extractor": {"field_path": ["result"]}}, @@ -137,10 +142,10 @@ "type": "object", "required": [], "properties": {}, - "additionalProperties": True + "additionalProperties": True, }, - "type": "Spec" - } + "type": "Spec", + }, } RESOLVE_MANIFEST_CONFIG = { @@ -273,7 +278,11 @@ def test_resolve_manifest(valid_resolve_manifest_config_file): "page_size": _page_size, "page_size_option": {"inject_into": "request_parameter", "field_name": "page_size"}, "page_token_option": {"inject_into": "path", "type": "RequestPath"}, - "pagination_strategy": {"type": "CursorPagination", "cursor_value": "{{ response._metadata.next }}", "page_size": _page_size}, + "pagination_strategy": { + "type": "CursorPagination", + "cursor_value": "{{ response._metadata.next }}", + "page_size": _page_size, + }, }, "partition_router": { "type": "ListPartitionRouter", @@ -386,10 +395,10 @@ def test_resolve_manifest(valid_resolve_manifest_config_file): "type": "object", "required": [], "properties": {}, - "additionalProperties": True + "additionalProperties": True, }, - "type": "Spec" - } + "type": "Spec", + }, } assert resolved_manifest.record.data["manifest"] == expected_resolved_manifest assert resolved_manifest.record.stream == "resolve_manifest" @@ -424,7 +433,7 @@ def test_read(): test_read_limit_reached=False, inferred_schema=None, inferred_datetime_formats=None, - latest_config_update={} + latest_config_update={}, ) expected_airbyte_message = AirbyteMessage( @@ -440,7 +449,7 @@ def test_read(): "auxiliary_requests": [], "inferred_schema": None, "inferred_datetime_formats": None, - "latest_config_update": {} + "latest_config_update": {}, }, emitted_at=1, ), @@ -462,7 +471,7 @@ def test_config_update(): "client_id": "{{ config['credentials']['client_id'] }}", "client_secret": "{{ config['credentials']['client_secret'] }}", "refresh_token": "{{ config['credentials']['refresh_token'] }}", - "refresh_token_updater": {} + "refresh_token_updater": {}, } config = copy.deepcopy(TEST_READ_CONFIG) config["__injected_declarative_manifest"] = manifest @@ -478,7 +487,10 @@ def test_config_update(): "refresh_token": "an updated refresh token", "expires_in": 3600, } - with patch("airbyte_cdk.sources.streams.http.requests_native_auth.SingleUseRefreshTokenOauth2Authenticator._get_refresh_access_token_response", return_value=refresh_request_response): + with patch( + "airbyte_cdk.sources.streams.http.requests_native_auth.SingleUseRefreshTokenOauth2Authenticator._get_refresh_access_token_response", + return_value=refresh_request_response, + ): output = handle_connector_builder_request( source, "test_read", config, ConfiguredAirbyteCatalog.parse_obj(CONFIGURED_CATALOG), TestReadLimits() ) @@ -507,13 +519,15 @@ def check_config_against_spec(self): limits = TestReadLimits() response = read_stream(source, TEST_READ_CONFIG, ConfiguredAirbyteCatalog.parse_obj(CONFIGURED_CATALOG), limits) - expected_stream_read = StreamRead(logs=[LogMessage("error_message - a stack trace", "ERROR")], - slices=[], - test_read_limit_reached=False, - auxiliary_requests=[], - inferred_schema=None, - inferred_datetime_formats={}, - latest_config_update=None) + expected_stream_read = StreamRead( + logs=[LogMessage("error_message - a stack trace", "ERROR")], + slices=[], + test_read_limit_reached=False, + auxiliary_requests=[], + inferred_schema=None, + inferred_datetime_formats={}, + latest_config_update=None, + ) expected_message = AirbyteMessage( type=MessageType.RECORD, @@ -584,8 +598,20 @@ def create_mock_declarative_stream(http_stream): @pytest.mark.parametrize( "test_name, config, expected_max_records, expected_max_slices, expected_max_pages_per_slice", [ - ("test_no_test_read_config", {}, DEFAULT_MAXIMUM_RECORDS, DEFAULT_MAXIMUM_NUMBER_OF_SLICES, DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE), - ("test_no_values_set", {"__test_read_config": {}}, DEFAULT_MAXIMUM_RECORDS, DEFAULT_MAXIMUM_NUMBER_OF_SLICES, DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE), + ( + "test_no_test_read_config", + {}, + DEFAULT_MAXIMUM_RECORDS, + DEFAULT_MAXIMUM_NUMBER_OF_SLICES, + DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE, + ), + ( + "test_no_values_set", + {"__test_read_config": {}}, + DEFAULT_MAXIMUM_RECORDS, + DEFAULT_MAXIMUM_NUMBER_OF_SLICES, + DEFAULT_MAXIMUM_NUMBER_OF_PAGES_PER_SLICE, + ), ("test_values_are_set", {"__test_read_config": {"max_slices": 1, "max_pages_per_slice": 2, "max_records": 3}}, 3, 1, 2), ], ) @@ -622,8 +648,8 @@ def response_log_message(response: dict) -> AirbyteMessage: def _create_request(): url = "https://example.com/api" - headers = {'Content-Type': 'application/json'} - return requests.Request('POST', url, headers=headers, json={"key": "value"}).prepare() + headers = {"Content-Type": "application/json"} + return requests.Request("POST", url, headers=headers, json={"key": "value"}).prepare() def _create_response(body, request): @@ -640,7 +666,15 @@ def _create_page_response(response_body): return _create_response(response_body, request) -@patch.object(requests.Session, "send", side_effect=(_create_page_response({"result": [{"id": 0}, {"id": 1}],"_metadata": {"next": "next"}}), _create_page_response({"result": [{"id": 2}],"_metadata": {"next": "next"}})) * 10) +@patch.object( + requests.Session, + "send", + side_effect=( + _create_page_response({"result": [{"id": 0}, {"id": 1}], "_metadata": {"next": "next"}}), + _create_page_response({"result": [{"id": 2}], "_metadata": {"next": "next"}}), + ) + * 10, +) def test_read_source(mock_http_stream): """ This test sort of acts as an integration test for the connector builder. @@ -656,9 +690,15 @@ def test_read_source(mock_http_stream): max_slices = 3 limits = TestReadLimits(max_records, max_pages_per_slice, max_slices) - catalog = ConfiguredAirbyteCatalog(streams=[ - ConfiguredAirbyteStream(stream=AirbyteStream(name=_stream_name, json_schema={}, supported_sync_modes=[SyncMode.full_refresh]), sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.append) - ]) + catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream(name=_stream_name, json_schema={}, supported_sync_modes=[SyncMode.full_refresh]), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.append, + ) + ] + ) config = {"__injected_declarative_manifest": MANIFEST} @@ -681,16 +721,29 @@ def test_read_source(mock_http_stream): assert isinstance(s.retriever, SimpleRetrieverTestReadDecorator) -@patch.object(requests.Session, "send", side_effect=(_create_page_response({"result": [{"id": 0}, {"id": 1}],"_metadata": {"next": "next"}}), _create_page_response({"result": [{"id": 2}],"_metadata": {"next": "next"}}))) +@patch.object( + requests.Session, + "send", + side_effect=( + _create_page_response({"result": [{"id": 0}, {"id": 1}], "_metadata": {"next": "next"}}), + _create_page_response({"result": [{"id": 2}], "_metadata": {"next": "next"}}), + ), +) def test_read_source_single_page_single_slice(mock_http_stream): max_records = 100 max_pages_per_slice = 1 max_slices = 1 limits = TestReadLimits(max_records, max_pages_per_slice, max_slices) - catalog = ConfiguredAirbyteCatalog(streams=[ - ConfiguredAirbyteStream(stream=AirbyteStream(name=_stream_name, json_schema={}, supported_sync_modes=[SyncMode.full_refresh]), sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.append) - ]) + catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream(name=_stream_name, json_schema={}, supported_sync_modes=[SyncMode.full_refresh]), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.append, + ) + ] + ) config = {"__injected_declarative_manifest": MANIFEST} @@ -716,12 +769,13 @@ def test_read_source_single_page_single_slice(mock_http_stream): "deployment_mode, url_base, expected_error", [ pytest.param("CLOUD", "https://airbyte.com/api/v1/characters", None, id="test_cloud_read_with_public_endpoint"), - pytest.param("CLOUD", "https://10.0.27.27", "ValueError", id="test_cloud_read_with_private_endpoint"), - pytest.param("CLOUD", "https://localhost:80/api/v1/cast", "ValueError", id="test_cloud_read_with_localhost"), - pytest.param("CLOUD", "http://unsecured.protocol/api/v1", "ValueError", id="test_cloud_read_with_unsecured_endpoint"), + pytest.param("CLOUD", "https://10.0.27.27", "AirbyteTracedException", id="test_cloud_read_with_private_endpoint"), + pytest.param("CLOUD", "https://localhost:80/api/v1/cast", "AirbyteTracedException", id="test_cloud_read_with_localhost"), + pytest.param("CLOUD", "http://unsecured.protocol/api/v1", "InvalidSchema", id="test_cloud_read_with_unsecured_endpoint"), + pytest.param("CLOUD", "https://domainwithoutextension", "Invalid URL", id="test_cloud_read_with_invalid_url_endpoint"), pytest.param("OSS", "https://airbyte.com/api/v1/", None, id="test_oss_read_with_public_endpoint"), pytest.param("OSS", "https://10.0.27.27/api/v1/", None, id="test_oss_read_with_private_endpoint"), - ] + ], ) @patch.object(requests.Session, "send", _mocked_send) def test_handle_read_external_requests(deployment_mode, url_base, expected_error): @@ -734,16 +788,15 @@ def test_handle_read_external_requests(deployment_mode, url_base, expected_error limits = TestReadLimits(max_records=100, max_pages_per_slice=1, max_slices=1) - catalog = ConfiguredAirbyteCatalog(streams=[ - ConfiguredAirbyteStream( - stream=AirbyteStream( - name=_stream_name, - json_schema={}, - supported_sync_modes=[SyncMode.full_refresh]), - sync_mode=SyncMode.full_refresh, - destination_sync_mode=DestinationSyncMode.append, - ) - ]) + catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream(name=_stream_name, json_schema={}, supported_sync_modes=[SyncMode.full_refresh]), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.append, + ) + ] + ) test_manifest = MANIFEST test_manifest["streams"][0]["$parameters"]["url_base"] = url_base @@ -767,11 +820,12 @@ def test_handle_read_external_requests(deployment_mode, url_base, expected_error "deployment_mode, token_url, expected_error", [ pytest.param("CLOUD", "https://airbyte.com/tokens/bearer", None, id="test_cloud_read_with_public_endpoint"), - pytest.param("CLOUD", "https://10.0.27.27/tokens/bearer", "ValueError", id="test_cloud_read_with_private_endpoint"), - pytest.param("CLOUD", "http://unsecured.protocol/tokens/bearer", "ValueError", id="test_cloud_read_with_unsecured_endpoint"), + pytest.param("CLOUD", "https://10.0.27.27/tokens/bearer", "AirbyteTracedException", id="test_cloud_read_with_private_endpoint"), + pytest.param("CLOUD", "http://unsecured.protocol/tokens/bearer", "InvalidSchema", id="test_cloud_read_with_unsecured_endpoint"), + pytest.param("CLOUD", "https://domainwithoutextension", "Invalid URL", id="test_cloud_read_with_invalid_url_endpoint"), pytest.param("OSS", "https://airbyte.com/tokens/bearer", None, id="test_oss_read_with_public_endpoint"), pytest.param("OSS", "https://10.0.27.27/tokens/bearer", None, id="test_oss_read_with_private_endpoint"), - ] + ], ) @patch.object(requests.Session, "send", _mocked_send) def test_handle_read_external_oauth_request(deployment_mode, token_url, expected_error): @@ -784,16 +838,15 @@ def test_handle_read_external_oauth_request(deployment_mode, token_url, expected limits = TestReadLimits(max_records=100, max_pages_per_slice=1, max_slices=1) - catalog = ConfiguredAirbyteCatalog(streams=[ - ConfiguredAirbyteStream( - stream=AirbyteStream( - name=_stream_name, - json_schema={}, - supported_sync_modes=[SyncMode.full_refresh]), - sync_mode=SyncMode.full_refresh, - destination_sync_mode=DestinationSyncMode.append, - ) - ]) + catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream(name=_stream_name, json_schema={}, supported_sync_modes=[SyncMode.full_refresh]), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.append, + ) + ] + ) oauth_authenticator_config: dict[str, str] = { "type": "OAuthAuthenticator", diff --git a/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py b/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py index 67d437dfac91..ae98f6ad70ab 100644 --- a/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py +++ b/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py @@ -85,7 +85,7 @@ A_SOURCE = MagicMock() -@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') +@patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") def test_get_grouped_messages(mock_entrypoint_read: Mock) -> None: url = "https://demonslayers.com/api/v1/hashiras?era=taisho" request = { @@ -94,7 +94,11 @@ def test_get_grouped_messages(mock_entrypoint_read: Mock) -> None: "body": {"content": '{"custom": "field"}'}, } response = {"status_code": 200, "headers": {"field": "value"}, "body": {"content": '{"name": "field"}'}} - expected_schema = {"$schema": "http://json-schema.org/schema#", "properties": {"name": {"type": "string"}, "date": {"type": "string"}}, "type": "object"} + expected_schema = { + "$schema": "http://json-schema.org/schema#", + "properties": {"name": {"type": "string"}, "date": {"type": "string"}}, + "type": "object", + } expected_datetime_fields = {"date": "%Y-%m-%d"} expected_pages = [ StreamReadPages( @@ -121,15 +125,18 @@ def test_get_grouped_messages(mock_entrypoint_read: Mock) -> None: ), ] - mock_source = make_mock_source(mock_entrypoint_read, iter( - [ - request_response_log_message(request, response, url), - record_message("hashiras", {"name": "Shinobu Kocho", "date": "2023-03-03"}), - record_message("hashiras", {"name": "Muichiro Tokito", "date": "2023-03-04"}), - request_response_log_message(request, response, url), - record_message("hashiras", {"name": "Mitsuri Kanroji", "date": "2023-03-05"}), - ] - )) + mock_source = make_mock_source( + mock_entrypoint_read, + iter( + [ + request_response_log_message(request, response, url), + record_message("hashiras", {"name": "Shinobu Kocho", "date": "2023-03-03"}), + record_message("hashiras", {"name": "Muichiro Tokito", "date": "2023-03-04"}), + request_response_log_message(request, response, url), + record_message("hashiras", {"name": "Mitsuri Kanroji", "date": "2023-03-05"}), + ] + ), + ) connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) actual_response: StreamRead = connector_builder_handler.get_message_groups( @@ -144,7 +151,7 @@ def test_get_grouped_messages(mock_entrypoint_read: Mock) -> None: assert actual_page == expected_pages[i] -@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') +@patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") def test_get_grouped_messages_with_logs(mock_entrypoint_read: Mock) -> None: url = "https://demonslayers.com/api/v1/hashiras?era=taisho" request = { @@ -183,7 +190,9 @@ def test_get_grouped_messages_with_logs(mock_entrypoint_read: Mock) -> None: LogMessage(**{"message": "log message after the response", "level": "INFO"}), ] - mock_source = make_mock_source(mock_entrypoint_read, iter( + mock_source = make_mock_source( + mock_entrypoint_read, + iter( [ AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=Level.INFO, message="log message before the request")), request_response_log_message(request, response, url), @@ -192,7 +201,7 @@ def test_get_grouped_messages_with_logs(mock_entrypoint_read: Mock) -> None: record_message("hashiras", {"name": "Muichiro Tokito"}), AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=Level.INFO, message="log message after the response")), ] - ) + ), ) connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) @@ -215,7 +224,7 @@ def test_get_grouped_messages_with_logs(mock_entrypoint_read: Mock) -> None: pytest.param(3, 1, id="test_create_request_record_limit_exceeds_max"), ], ) -@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') +@patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") def test_get_grouped_messages_record_limit(mock_entrypoint_read: Mock, request_record_limit: int, max_record_limit: int) -> None: url = "https://demonslayers.com/api/v1/hashiras?era=taisho" request = { @@ -224,7 +233,9 @@ def test_get_grouped_messages_record_limit(mock_entrypoint_read: Mock, request_r "body": {"content": '{"custom": "field"}'}, } response = {"status_code": 200, "headers": {"field": "value"}, "body": {"content": '{"name": "field"}'}} - mock_source = make_mock_source(mock_entrypoint_read, iter( + mock_source = make_mock_source( + mock_entrypoint_read, + iter( [ request_response_log_message(request, response, url), record_message("hashiras", {"name": "Shinobu Kocho"}), @@ -232,7 +243,7 @@ def test_get_grouped_messages_record_limit(mock_entrypoint_read: Mock, request_r request_response_log_message(request, response, url), record_message("hashiras", {"name": "Mitsuri Kanroji"}), ] - ) + ), ) n_records = 2 record_limit = min(request_record_limit, max_record_limit) @@ -257,7 +268,7 @@ def test_get_grouped_messages_record_limit(mock_entrypoint_read: Mock, request_r pytest.param(1, id="test_create_request_no_record_limit_n_records_exceed_max"), ], ) -@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') +@patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") def test_get_grouped_messages_default_record_limit(mock_entrypoint_read: Mock, max_record_limit: int) -> None: url = "https://demonslayers.com/api/v1/hashiras?era=taisho" request = { @@ -266,7 +277,9 @@ def test_get_grouped_messages_default_record_limit(mock_entrypoint_read: Mock, m "body": {"content": '{"custom": "field"}'}, } response = {"status_code": 200, "headers": {"field": "value"}, "body": {"content": '{"name": "field"}'}} - mock_source = make_mock_source(mock_entrypoint_read, iter( + mock_source = make_mock_source( + mock_entrypoint_read, + iter( [ request_response_log_message(request, response, url), record_message("hashiras", {"name": "Shinobu Kocho"}), @@ -274,7 +287,7 @@ def test_get_grouped_messages_default_record_limit(mock_entrypoint_read: Mock, m request_response_log_message(request, response, url), record_message("hashiras", {"name": "Mitsuri Kanroji"}), ] - ) + ), ) n_records = 2 @@ -289,7 +302,7 @@ def test_get_grouped_messages_default_record_limit(mock_entrypoint_read: Mock, m assert total_records == min([max_record_limit, n_records]) -@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') +@patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") def test_get_grouped_messages_limit_0(mock_entrypoint_read: Mock) -> None: url = "https://demonslayers.com/api/v1/hashiras?era=taisho" request = { @@ -298,7 +311,9 @@ def test_get_grouped_messages_limit_0(mock_entrypoint_read: Mock) -> None: "body": {"content": '{"custom": "field"}'}, } response = {"status_code": 200, "headers": {"field": "value"}, "body": {"content": '{"name": "field"}'}} - mock_source = make_mock_source(mock_entrypoint_read, iter( + mock_source = make_mock_source( + mock_entrypoint_read, + iter( [ request_response_log_message(request, response, url), record_message("hashiras", {"name": "Shinobu Kocho"}), @@ -306,7 +321,7 @@ def test_get_grouped_messages_limit_0(mock_entrypoint_read: Mock) -> None: request_response_log_message(request, response, url), record_message("hashiras", {"name": "Mitsuri Kanroji"}), ] - ) + ), ) api = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) @@ -314,7 +329,7 @@ def test_get_grouped_messages_limit_0(mock_entrypoint_read: Mock) -> None: api.get_message_groups(source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), record_limit=0) -@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') +@patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") def test_get_grouped_messages_no_records(mock_entrypoint_read: Mock) -> None: url = "https://demonslayers.com/api/v1/hashiras?era=taisho" request = { @@ -348,12 +363,14 @@ def test_get_grouped_messages_no_records(mock_entrypoint_read: Mock) -> None: ), ] - mock_source = make_mock_source(mock_entrypoint_read, iter( + mock_source = make_mock_source( + mock_entrypoint_read, + iter( [ request_response_log_message(request, response, url), request_response_log_message(request, response, url), ] - ) + ), ) message_grouper = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) @@ -371,7 +388,15 @@ def test_get_grouped_messages_no_records(mock_entrypoint_read: Mock) -> None: "log_message, expected_response", [ pytest.param( - {"http": {"response": {"status_code": 200, "headers": {"field": "name"}, "body": {"content": '{"id": "fire", "owner": "kyojuro_rengoku"}'}}}}, + { + "http": { + "response": { + "status_code": 200, + "headers": {"field": "name"}, + "body": {"content": '{"id": "fire", "owner": "kyojuro_rengoku"}'}, + } + } + }, HttpResponse(status=200, headers={"field": "name"}, body='{"id": "fire", "owner": "kyojuro_rengoku"}'), id="test_create_response_with_all_fields", ), @@ -421,13 +446,15 @@ def test_create_response_from_log_message(log_message: str, expected_response: H assert actual_response == expected_response -@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') +@patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") def test_get_grouped_messages_with_many_slices(mock_entrypoint_read: Mock) -> None: url = "http://a-url.com" request: Mapping[str, Any] = {} response = {"status_code": 200} - mock_source = make_mock_source(mock_entrypoint_read, iter( + mock_source = make_mock_source( + mock_entrypoint_read, + iter( [ slice_message('{"descriptor": "first_slice"}'), request_response_log_message(request, response, url), @@ -440,7 +467,7 @@ def test_get_grouped_messages_with_many_slices(mock_entrypoint_read: Mock) -> No record_message("hashiras", {"name": "Obanai Iguro"}), request_response_log_message(request, response, url), ] - ) + ), ) connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) @@ -463,12 +490,14 @@ def test_get_grouped_messages_with_many_slices(mock_entrypoint_read: Mock) -> No assert len(stream_read.slices[1].pages[2].records) == 0 -@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') +@patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") def test_get_grouped_messages_given_maximum_number_of_slices_then_test_read_limit_reached(mock_entrypoint_read: Mock) -> None: maximum_number_of_slices = 5 request: Mapping[str, Any] = {} response = {"status_code": 200} - mock_source = make_mock_source(mock_entrypoint_read, iter([slice_message(), request_response_log_message(request, response, "a_url")] * maximum_number_of_slices)) + mock_source = make_mock_source( + mock_entrypoint_read, iter([slice_message(), request_response_log_message(request, response, "a_url")] * maximum_number_of_slices) + ) api = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) @@ -479,12 +508,15 @@ def test_get_grouped_messages_given_maximum_number_of_slices_then_test_read_limi assert stream_read.test_read_limit_reached -@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') +@patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") def test_get_grouped_messages_given_maximum_number_of_pages_then_test_read_limit_reached(mock_entrypoint_read: Mock) -> None: maximum_number_of_pages_per_slice = 5 request: Mapping[str, Any] = {} response = {"status_code": 200} - mock_source = make_mock_source(mock_entrypoint_read, iter([slice_message()] + [request_response_log_message(request, response, "a_url")] * maximum_number_of_pages_per_slice)) + mock_source = make_mock_source( + mock_entrypoint_read, + iter([slice_message()] + [request_response_log_message(request, response, "a_url")] * maximum_number_of_pages_per_slice), + ) api = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) @@ -502,20 +534,21 @@ def test_read_stream_returns_error_if_stream_does_not_exist() -> None: full_config: Mapping[str, Any] = {**CONFIG, **{"__injected_declarative_manifest": MANIFEST}} message_grouper = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) - actual_response = message_grouper.get_message_groups(source=mock_source, config=full_config, - configured_catalog=create_configured_catalog("not_in_manifest")) + actual_response = message_grouper.get_message_groups( + source=mock_source, config=full_config, configured_catalog=create_configured_catalog("not_in_manifest") + ) assert 1 == len(actual_response.logs) assert "Traceback" in actual_response.logs[0].message assert "ERROR" in actual_response.logs[0].level -@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') +@patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") def test_given_control_message_then_stream_read_has_config_update(mock_entrypoint_read: Mock) -> None: updated_config = {"x": 1} - mock_source = make_mock_source(mock_entrypoint_read, iter( - any_request_and_response_with_a_record() + [connector_configuration_control_message(1, updated_config)] - )) + mock_source = make_mock_source( + mock_entrypoint_read, iter(any_request_and_response_with_a_record() + [connector_configuration_control_message(1, updated_config)]) + ) connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = connector_builder_handler.get_message_groups( source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras") @@ -524,21 +557,23 @@ def test_given_control_message_then_stream_read_has_config_update(mock_entrypoin assert stream_read.latest_config_update == updated_config -@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') +@patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") def test_given_multiple_control_messages_then_stream_read_has_latest_based_on_emitted_at(mock_entrypoint_read: Mock) -> None: earliest = 0 earliest_config = {"earliest": 0} latest = 1 latest_config = {"latest": 1} - mock_source = make_mock_source(mock_entrypoint_read, iter( - any_request_and_response_with_a_record() + - [ - # here, we test that even if messages are emitted in a different order, we still rely on `emitted_at` - connector_configuration_control_message(latest, latest_config), - connector_configuration_control_message(earliest, earliest_config), - ] + mock_source = make_mock_source( + mock_entrypoint_read, + iter( + any_request_and_response_with_a_record() + + [ + # here, we test that even if messages are emitted in a different order, we still rely on `emitted_at` + connector_configuration_control_message(latest, latest_config), + connector_configuration_control_message(earliest, earliest_config), + ] + ), ) - ) connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = connector_builder_handler.get_message_groups( source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras") @@ -547,19 +582,23 @@ def test_given_multiple_control_messages_then_stream_read_has_latest_based_on_em assert stream_read.latest_config_update == latest_config -@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') -def test_given_multiple_control_messages_with_same_timestamp_then_stream_read_has_latest_based_on_message_order(mock_entrypoint_read: Mock) -> None: +@patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") +def test_given_multiple_control_messages_with_same_timestamp_then_stream_read_has_latest_based_on_message_order( + mock_entrypoint_read: Mock, +) -> None: emitted_at = 0 earliest_config = {"earliest": 0} latest_config = {"latest": 1} - mock_source = make_mock_source(mock_entrypoint_read, iter( - any_request_and_response_with_a_record() + - [ - connector_configuration_control_message(emitted_at, earliest_config), - connector_configuration_control_message(emitted_at, latest_config), - ] + mock_source = make_mock_source( + mock_entrypoint_read, + iter( + any_request_and_response_with_a_record() + + [ + connector_configuration_control_message(emitted_at, earliest_config), + connector_configuration_control_message(emitted_at, latest_config), + ] + ), ) - ) connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = connector_builder_handler.get_message_groups( source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras") @@ -568,14 +607,9 @@ def test_given_multiple_control_messages_with_same_timestamp_then_stream_read_ha assert stream_read.latest_config_update == latest_config -@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') +@patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") def test_given_auxiliary_requests_then_return_auxiliary_request(mock_entrypoint_read: Mock) -> None: - mock_source = make_mock_source(mock_entrypoint_read, iter( - any_request_and_response_with_a_record() + - [ - auxiliary_request_log_message() - ] - )) + mock_source = make_mock_source(mock_entrypoint_read, iter(any_request_and_response_with_a_record() + [auxiliary_request_log_message()])) connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = connector_builder_handler.get_message_groups( source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras") @@ -584,7 +618,7 @@ def test_given_auxiliary_requests_then_return_auxiliary_request(mock_entrypoint_ assert len(stream_read.auxiliary_requests) == 1 -@patch('airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read') +@patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") def test_given_no_slices_then_return_empty_slices(mock_entrypoint_read: Mock) -> None: mock_source = make_mock_source(mock_entrypoint_read, iter([auxiliary_request_log_message()])) connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) @@ -624,34 +658,45 @@ def connector_configuration_control_message(emitted_at: float, config: Mapping[s type=OrchestratorType.CONNECTOR_CONFIG, emitted_at=emitted_at, connectorConfig=AirbyteControlConnectorConfigMessage(config=config), - ) + ), ) def auxiliary_request_log_message() -> AirbyteMessage: - return AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=Level.INFO, message=json.dumps({ - "http": { - "is_auxiliary": True, - "title": "a title", - "description": "a description", - "request": {}, - "response": {}, - }, - "url": {"full": "https://a-url.com"} - }))) + return AirbyteMessage( + type=MessageType.LOG, + log=AirbyteLogMessage( + level=Level.INFO, + message=json.dumps( + { + "http": { + "is_auxiliary": True, + "title": "a title", + "description": "a description", + "request": {}, + "response": {}, + }, + "url": {"full": "https://a-url.com"}, + } + ), + ), + ) def request_response_log_message(request: Mapping[str, Any], response: Mapping[str, Any], url: str) -> AirbyteMessage: - return AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=Level.INFO, message=json.dumps({ - "airbyte_cdk": {"stream": {"name": "a stream name"}}, - "http": { - "title": "a title", - "description": "a description", - "request": request, - "response": response - }, - "url": {"full": url} - }))) + return AirbyteMessage( + type=MessageType.LOG, + log=AirbyteLogMessage( + level=Level.INFO, + message=json.dumps( + { + "airbyte_cdk": {"stream": {"name": "a stream name"}}, + "http": {"title": "a title", "description": "a description", "request": request, "response": response}, + "url": {"full": url}, + } + ), + ), + ) def any_request_and_response_with_a_record() -> List[AirbyteMessage]: diff --git a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/config_test.py b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/config_test.py new file mode 100644 index 000000000000..c6ccf6da1985 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/config_test.py @@ -0,0 +1,385 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Union + +import dpath.util +from airbyte_cdk.destinations.vector_db_based.config import ( + AzureOpenAIEmbeddingConfigModel, + CohereEmbeddingConfigModel, + FakeEmbeddingConfigModel, + OpenAICompatibleEmbeddingConfigModel, + OpenAIEmbeddingConfigModel, + ProcessingConfigModel, +) +from airbyte_cdk.utils.spec_schema_transformations import resolve_refs +from pydantic import BaseModel, Field + + +class IndexingModel(BaseModel): + foo: str = Field( + ..., + title="Foo", + description="Foo", + ) + + +class ConfigModel(BaseModel): + indexing: IndexingModel + + embedding: Union[ + OpenAIEmbeddingConfigModel, + CohereEmbeddingConfigModel, + FakeEmbeddingConfigModel, + AzureOpenAIEmbeddingConfigModel, + OpenAICompatibleEmbeddingConfigModel, + ] = Field( + ..., + title="Embedding", + description="Embedding configuration", + discriminator="mode", + group="embedding", + type="object", + ) + processing: ProcessingConfigModel + + class Config: + title = "My Destination Config" + schema_extra = { + "groups": [ + {"id": "processing", "title": "Processing"}, + {"id": "embedding", "title": "Embedding"}, + {"id": "indexing", "title": "Indexing"}, + ] + } + + @staticmethod + def remove_discriminator(schema: dict) -> None: + """pydantic adds "discriminator" to the schema for oneOfs, which is not treated right by the platform as we inline all references""" + dpath.util.delete(schema, "properties/**/discriminator") + + @classmethod + def schema(cls): + """we're overriding the schema classmethod to enable some post-processing""" + schema = super().schema() + schema = resolve_refs(schema) + cls.remove_discriminator(schema) + return schema + + +def test_json_schema_generation(): + # This is the expected output of the schema generation + expected = { + "title": "My Destination Config", + "type": "object", + "properties": { + "indexing": { + "title": "IndexingModel", + "type": "object", + "properties": {"foo": {"title": "Foo", "description": "Foo", "type": "string"}}, + "required": ["foo"], + }, + "embedding": { + "title": "Embedding", + "description": "Embedding configuration", + "group": "embedding", + "type": "object", + "oneOf": [ + { + "title": "OpenAI", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "openai", + "const": "openai", + "enum": ["openai"], + "type": "string", + }, + "openai_key": { + "title": "OpenAI API key", + "airbyte_secret": True, + "type": "string", + }, + }, + "required": ["openai_key", "mode"], + "description": "Use the OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions.", + }, + { + "title": "Cohere", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "cohere", + "const": "cohere", + "enum": ["cohere"], + "type": "string", + }, + "cohere_key": { + "title": "Cohere API key", + "airbyte_secret": True, + "type": "string", + }, + }, + "required": ["cohere_key", "mode"], + "description": "Use the Cohere API to embed text.", + }, + { + "title": "Fake", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "fake", + "const": "fake", + "enum": ["fake"], + "type": "string", + } + }, + "description": "Use a fake embedding made out of random vectors with 1536 embedding dimensions. This is useful for testing the data pipeline without incurring any costs.", + "required": ["mode"], + }, + { + "title": "Azure OpenAI", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "azure_openai", + "const": "azure_openai", + "enum": ["azure_openai"], + "type": "string", + }, + "openai_key": { + "title": "Azure OpenAI API key", + "description": "The API key for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", + "airbyte_secret": True, + "type": "string", + }, + "api_base": { + "title": "Resource base URL", + "description": "The base URL for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", + "examples": ["https://your-resource-name.openai.azure.com"], + "type": "string", + }, + "deployment": { + "title": "Deployment", + "description": "The deployment for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", + "examples": ["your-resource-name"], + "type": "string", + }, + }, + "required": ["openai_key", "api_base", "deployment", "mode"], + "description": "Use the Azure-hosted OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions.", + }, + { + "title": "OpenAI-compatible", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "openai_compatible", + "const": "openai_compatible", + "enum": ["openai_compatible"], + "type": "string", + }, + "api_key": { + "title": "API key", + "default": "", + "airbyte_secret": True, + "type": "string", + }, + "base_url": { + "title": "Base URL", + "description": "The base URL for your OpenAI-compatible service", + "examples": ["https://your-service-name.com"], + "type": "string", + }, + "model_name": { + "title": "Model name", + "description": "The name of the model to use for embedding", + "default": "text-embedding-ada-002", + "examples": ["text-embedding-ada-002"], + "type": "string", + }, + "dimensions": { + "title": "Embedding dimensions", + "description": "The number of dimensions the embedding model is generating", + "examples": [1536, 384], + "type": "integer", + }, + }, + "required": ["base_url", "dimensions", "mode"], + "description": "Use a service that's compatible with the OpenAI API to embed text.", + }, + ], + }, + "processing": { + "title": "ProcessingConfigModel", + "type": "object", + "properties": { + "chunk_size": { + "title": "Chunk size", + "description": "Size of chunks in tokens to store in vector store (make sure it is not too big for the context if your LLM)", + "maximum": 8191, + "minimum": 1, + "type": "integer", + }, + "chunk_overlap": { + "title": "Chunk overlap", + "description": "Size of overlap between chunks in tokens to store in vector store to better capture relevant context", + "default": 0, + "type": "integer", + }, + "text_fields": { + "title": "Text fields to embed", + "description": "List of fields in the record that should be used to calculate the embedding. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered text fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.", + "default": [], + "always_show": True, + "examples": ["text", "user.name", "users.*.name"], + "type": "array", + "items": {"type": "string"}, + }, + "metadata_fields": { + "title": "Fields to store as metadata", + "description": "List of fields in the record that should be stored as metadata. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered metadata fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. When specifying nested paths, all matching values are flattened into an array set to a field named by the path.", + "default": [], + "always_show": True, + "examples": ["age", "user", "user.name"], + "type": "array", + "items": {"type": "string"}, + }, + "text_splitter": { + "title": "Text splitter", + "description": "Split text fields into chunks based on the specified method.", + "type": "object", + "oneOf": [ + { + "title": "By Separator", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "separator", + "const": "separator", + "enum": ["separator"], + "type": "string", + }, + "separators": { + "title": "Separators", + "description": 'List of separator strings to split text fields by. The separator itself needs to be wrapped in double quotes, e.g. to split by the dot character, use ".". To split by a newline, use "\\n".', + "default": ['"\\n\\n"', '"\\n"', '" "', '""'], + "type": "array", + "items": {"type": "string"}, + }, + "keep_separator": { + "title": "Keep separator", + "description": "Whether to keep the separator in the resulting chunks", + "default": False, + "type": "boolean", + }, + }, + "description": "Split the text by the list of separators until the chunk size is reached, using the earlier mentioned separators where possible. This is useful for splitting text fields by paragraphs, sentences, words, etc.", + "required": ["mode"], + }, + { + "title": "By Markdown header", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "markdown", + "const": "markdown", + "enum": ["markdown"], + "type": "string", + }, + "split_level": { + "title": "Split level", + "description": "Level of markdown headers to split text fields by. Headings down to the specified level will be used as split points", + "default": 1, + "minimum": 1, + "maximum": 6, + "type": "integer", + }, + }, + "description": "Split the text by Markdown headers down to the specified header level. If the chunk size fits multiple sections, they will be combined into a single chunk.", + "required": ["mode"], + }, + { + "title": "By Programming Language", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "code", + "const": "code", + "enum": ["code"], + "type": "string", + }, + "language": { + "title": "Language", + "description": "Split code in suitable places based on the programming language", + "enum": [ + "cpp", + "go", + "java", + "js", + "php", + "proto", + "python", + "rst", + "ruby", + "rust", + "scala", + "swift", + "markdown", + "latex", + "html", + "sol", + ], + "type": "string", + }, + }, + "required": ["language", "mode"], + "description": "Split the text by suitable delimiters based on the programming language. This is useful for splitting code into chunks.", + }, + ], + }, + "field_name_mappings": { + "title": "Field name mappings", + "description": "List of fields to rename. Not applicable for nested fields, but can be used to rename fields already flattened via dot notation.", + "default": [], + "type": "array", + "items": { + "title": "FieldNameMappingConfigModel", + "type": "object", + "properties": { + "from_field": { + "title": "From field name", + "description": "The field name in the source", + "type": "string", + }, + "to_field": { + "title": "To field name", + "description": "The field name to use in the destination", + "type": "string", + }, + }, + "required": ["from_field", "to_field"], + }, + }, + }, + "required": ["chunk_size"], + "group": "processing", + }, + }, + "required": ["indexing", "embedding", "processing"], + "groups": [ + {"id": "processing", "title": "Processing"}, + {"id": "embedding", "title": "Embedding"}, + {"id": "indexing", "title": "Indexing"}, + ], + } + assert ConfigModel.schema() == expected diff --git a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/document_processor_test.py b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/document_processor_test.py new file mode 100644 index 000000000000..0e8760b73cc7 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/document_processor_test.py @@ -0,0 +1,658 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, List, Mapping, Optional +from unittest.mock import MagicMock + +import pytest +from airbyte_cdk.destinations.vector_db_based.config import ( + CodeSplitterConfigModel, + FieldNameMappingConfigModel, + MarkdownHeaderSplitterConfigModel, + ProcessingConfigModel, + SeparatorSplitterConfigModel, +) +from airbyte_cdk.destinations.vector_db_based.document_processor import DocumentProcessor +from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream +from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage, DestinationSyncMode, SyncMode +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + + +def initialize_processor(config=ProcessingConfigModel(chunk_size=48, chunk_overlap=0, text_fields=None, metadata_fields=None)): + catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream( + name="stream1", + json_schema={}, + namespace="namespace1", + supported_sync_modes=[SyncMode.full_refresh], + ), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + primary_key=[["id"]], + ), + ConfiguredAirbyteStream( + stream=AirbyteStream( + name="stream2", + json_schema={}, + supported_sync_modes=[SyncMode.full_refresh], + ), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ), + ] + ) + return DocumentProcessor(config=config, catalog=catalog) + + +@pytest.mark.parametrize( + "metadata_fields, expected_metadata", + [ + ( + None, + { + "_ab_stream": "namespace1_stream1", + "id": 1, + "text": "This is the text", + "complex": {"test": "abc"}, + "arr": [{"test": "abc"}, {"test": "def"}], + }, + ), + (["id"], {"_ab_stream": "namespace1_stream1", "id": 1}), + (["id", "non_existing"], {"_ab_stream": "namespace1_stream1", "id": 1}), + ( + ["id", "complex.test"], + {"_ab_stream": "namespace1_stream1", "id": 1, "complex.test": "abc"}, + ), + ( + ["id", "arr.*.test"], + {"_ab_stream": "namespace1_stream1", "id": 1, "arr.*.test": ["abc", "def"]}, + ), + ], +) +def test_process_single_chunk_with_metadata(metadata_fields, expected_metadata): + processor = initialize_processor() + processor.metadata_fields = metadata_fields + + record = AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={ + "id": 1, + "text": "This is the text", + "complex": {"test": "abc"}, + "arr": [{"test": "abc"}, {"test": "def"}], + }, + emitted_at=1234, + ) + + chunks, id_to_delete = processor.process(record) + + assert len(chunks) == 1 + # natural id is only set for dedup mode + assert "_ab_record_id" not in chunks[0].metadata + assert chunks[0].metadata == expected_metadata + assert id_to_delete is None + + +def test_process_single_chunk_limited_metadata(): + processor = initialize_processor() + + record = AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={ + "id": 1, + "text": "This is the text", + }, + emitted_at=1234, + ) + + chunks, id_to_delete = processor.process(record) + + assert len(chunks) == 1 + # natural id is only set for dedup mode + assert "_ab_record_id" not in chunks[0].metadata + assert chunks[0].metadata["_ab_stream"] == "namespace1_stream1" + assert chunks[0].metadata["id"] == 1 + assert chunks[0].metadata["text"] == "This is the text" + assert chunks[0].page_content == "id: 1\ntext: This is the text" + assert id_to_delete is None + + +def test_process_single_chunk_without_namespace(): + config = ProcessingConfigModel(chunk_size=48, chunk_overlap=0, text_fields=None, metadata_fields=None) + catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream( + name="stream1", + json_schema={}, + supported_sync_modes=[SyncMode.full_refresh], + ), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ), + ] + ) + processor = DocumentProcessor(config=config, catalog=catalog) + + record = AirbyteRecordMessage( + stream="stream1", + data={ + "id": 1, + "text": "This is the text", + }, + emitted_at=1234, + ) + + chunks, _ = processor.process(record) + assert chunks[0].metadata["_ab_stream"] == "stream1" + + +def test_complex_text_fields(): + processor = initialize_processor() + + record = AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={ + "id": 1, + "nested": { + "texts": [ + {"text": "This is the text"}, + {"text": "And another"}, + ] + }, + "non_text": "a", + "non_text_2": 1, + "text": "This is the regular text", + "other_nested": {"non_text": {"a": "xyz", "b": "abc"}}, + }, + emitted_at=1234, + ) + + processor.text_fields = [ + "nested.texts.*.text", + "text", + "other_nested.non_text", + "non.*.existing", + ] + processor.metadata_fields = ["non_text", "non_text_2", "id"] + + chunks, _ = processor.process(record) + + assert len(chunks) == 1 + assert ( + chunks[0].page_content + == """nested.texts.*.text: This is the text +And another +text: This is the regular text +other_nested.non_text: \na: xyz +b: abc""" + ) + assert chunks[0].metadata == { + "id": 1, + "non_text": "a", + "non_text_2": 1, + "_ab_stream": "namespace1_stream1", + } + + +def test_no_text_fields(): + processor = initialize_processor() + + record = AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={ + "id": 1, + "text": "This is the regular text", + }, + emitted_at=1234, + ) + + processor.text_fields = ["another_field"] + processor.logger = MagicMock() + + # assert process is throwing with no text fields found + with pytest.raises(AirbyteTracedException): + processor.process(record) + + +def test_process_multiple_chunks_with_relevant_fields(): + processor = initialize_processor() + + record = AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={ + "id": 1, + "name": "John Doe", + "text": "This is the text and it is long enough to be split into multiple chunks. This is the text and it is long enough to be split into multiple chunks. This is the text and it is long enough to be split into multiple chunks", + "age": 25, + }, + emitted_at=1234, + ) + + processor.text_fields = ["text"] + + chunks, id_to_delete = processor.process(record) + + assert len(chunks) == 2 + + for chunk in chunks: + assert chunk.metadata["age"] == 25 + assert id_to_delete is None + + +@pytest.mark.parametrize( + "label, text, chunk_size, chunk_overlap, splitter_config, expected_chunks", + [ + ( + "Default splitting", + "By default, splits are done \non multi newlines,\n\n then single newlines, then spaces", + 10, + 0, + None, + [ + "text: By default, splits are done", + "on multi newlines,", + "then single newlines, then spaces", + ], + ), + ( + "Overlap splitting", + "One two three four five six seven eight nine ten eleven twelve thirteen", + 15, + 5, + None, + [ + "text: One two three four five six", + "four five six seven eight nine ten", + "eight nine ten eleven twelve thirteen", + ], + ), + ( + "Special tokens", + "Special tokens like <|endoftext|> are treated like regular text", + 15, + 0, + None, + [ + "text: Special tokens like", + "<|endoftext|> are treated like regular", + "text", + ] + ), + ( + "Custom separator", + "Custom \nseparatorxxxDoes not split on \n\nnewlines", + 10, + 0, + SeparatorSplitterConfigModel(mode="separator", separators=['"xxx"']), + [ + "text: Custom \nseparator", + "Does not split on \n\nnewlines\n", + ], + ), + ( + "Only splits if chunks dont fit", + "Does yyynot usexxxseparators yyyif not needed", + 10, + 0, + SeparatorSplitterConfigModel(mode="separator", separators=['"xxx"', '"yyy"']), + [ + "text: Does yyynot use", + "separators yyyif not needed", + ], + ), + ( + "Use first separator first", + "Does alwaysyyy usexxxmain separators yyyfirst", + 10, + 0, + SeparatorSplitterConfigModel(mode="separator", separators=['"yyy"', '"xxx"']), + [ + "text: Does always", + "usexxxmain separators yyyfirst", + ], + ), + ( + "Basic markdown splitting", + "# Heading 1\nText 1\n\n# Heading 2\nText 2\n\n# Heading 3\nText 3", + 10, + 0, + MarkdownHeaderSplitterConfigModel(mode="markdown", split_level=1), + [ + "text: # Heading 1\nText 1\n", + "# Heading 2\nText 2", + "# Heading 3\nText 3", + ], + ), + ( + "Split multiple levels", + "# Heading 1\nText 1\n\n## Sub-Heading 1\nText 2\n\n# Heading 2\nText 3", + 10, + 0, + MarkdownHeaderSplitterConfigModel(mode="markdown", split_level=2), + [ + "text: # Heading 1\nText 1\n", + "\n## Sub-Heading 1\nText 2\n", + "# Heading 2\nText 3", + ], + ), + ( + "Do not split if split level does not allow", + "## Heading 1\nText 1\n\n## Heading 2\nText 2\n\n## Heading 3\nText 3", + 10, + 0, + MarkdownHeaderSplitterConfigModel(mode="markdown", split_level=1), + [ + "text: ## Heading 1\nText 1\n\n## Heading 2\nText 2\n\n## Heading 3\nText 3\n", + ], + ), + ( + "Do not split if everything fits", + "## Does not split if everything fits. Heading 1\nText 1\n\n## Heading 2\nText 2\n\n## Heading 3\nText 3", + 1000, + 0, + MarkdownHeaderSplitterConfigModel(mode="markdown", split_level=5), + [ + "text: ## Does not split if everything fits. Heading 1\nText 1\n\n## Heading 2\nText 2\n\n## Heading 3\nText 3", + ], + ), + ( + "Split Java code, respecting class boundaries", + "class A { /* \n\nthis is the first class */ }\nclass B {}", + 20, + 0, + CodeSplitterConfigModel(mode="code", language="java"), + [ + "text: class A { /* \n\nthis is the first class */ }", + "class B {}", + ], + ), + ( + "Split Java code as proto, not respecting class boundaries", + "class A { /* \n\nthis is the first class */ }\nclass B {}", + 20, + 0, + CodeSplitterConfigModel(mode="code", language="proto"), + [ + "text: class A { /*", + "this is the first class */ }\nclass B {}", + ], + ), + ], +) +def test_text_splitters(label, text, chunk_size, chunk_overlap, splitter_config, expected_chunks): + processor = initialize_processor( + ProcessingConfigModel( + chunk_size=chunk_size, + chunk_overlap=chunk_overlap, + text_fields=["text"], + metadata_fields=None, + text_splitter=splitter_config, + ) + ) + + record = AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={ + "id": 1, + "name": "John Doe", + "text": text, + "age": 25, + }, + emitted_at=1234, + ) + + processor.text_fields = ["text"] + + chunks, id_to_delete = processor.process(record) + + assert len(chunks) == len(expected_chunks) + + # check that the page_content in each chunk equals the expected chunk + for i, chunk in enumerate(chunks): + print(chunk.page_content) + assert chunk.page_content == expected_chunks[i] + assert id_to_delete is None + + +@pytest.mark.parametrize( + "label, split_config, has_error_message", + [ + ( + "Invalid separator", + SeparatorSplitterConfigModel(mode="separator", separators=['"xxx']), + True, + ), + ( + "Missing quotes", + SeparatorSplitterConfigModel(mode="separator", separators=["xxx"]), + True, + ), + ( + "Non-string separator", + SeparatorSplitterConfigModel(mode="separator", separators=["123"]), + True, + ), + ( + "Object separator", + SeparatorSplitterConfigModel(mode="separator", separators=["{}"]), + True, + ), + ( + "Proper separator", + SeparatorSplitterConfigModel(mode="separator", separators=['"xxx"', '"\\n\\n"']), + False, + ), + ], +) +def test_text_splitter_check(label, split_config, has_error_message): + error = DocumentProcessor.check_config( + ProcessingConfigModel( + chunk_size=48, + chunk_overlap=0, + text_fields=None, + metadata_fields=None, + text_splitter=split_config, + ) + ) + if has_error_message: + assert error is not None + else: + assert error is None + + +@pytest.mark.parametrize( + "mappings, fields, expected_chunk_metadata", + [ + (None, {"abc": "def", "xyz": 123}, {"abc": "def", "xyz": 123}), + ([], {"abc": "def", "xyz": 123}, {"abc": "def", "xyz": 123}), + ( + [FieldNameMappingConfigModel(from_field="abc", to_field="AAA")], + {"abc": "def", "xyz": 123}, + {"AAA": "def", "xyz": 123}, + ), + ( + [FieldNameMappingConfigModel(from_field="non_existing", to_field="AAA")], + {"abc": "def", "xyz": 123}, + {"abc": "def", "xyz": 123}, + ), + ], +) +def test_rename_metadata_fields( + mappings: Optional[List[FieldNameMappingConfigModel]], + fields: Mapping[str, Any], + expected_chunk_metadata: Mapping[str, Any], +): + processor = initialize_processor() + + record = AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={**fields, "text": "abc"}, + emitted_at=1234, + ) + + processor.field_name_mappings = mappings + processor.text_fields = ["text"] + + chunks, id_to_delete = processor.process(record) + + assert len(chunks) == 1 + assert chunks[0].metadata == { + **expected_chunk_metadata, + "_ab_stream": "namespace1_stream1", + "text": "abc", + } + + +@pytest.mark.parametrize( + "primary_key_value, stringified_primary_key, primary_key", + [ + ({"id": 99}, "namespace1_stream1_99", [["id"]]), + ( + {"id": 99, "name": "John Doe"}, + "namespace1_stream1_99_John Doe", + [["id"], ["name"]], + ), + ( + {"id": 99, "name": "John Doe", "age": 25}, + "namespace1_stream1_99_John Doe_25", + [["id"], ["name"], ["age"]], + ), + ( + {"nested": {"id": "abc"}, "name": "John Doe"}, + "namespace1_stream1_abc_John Doe", + [["nested", "id"], ["name"]], + ), + ( + {"nested": {"id": "abc"}}, + "namespace1_stream1_abc___not_found__", + [["nested", "id"], ["name"]], + ), + ], +) +def test_process_multiple_chunks_with_dedupe_mode( + primary_key_value: Mapping[str, Any], + stringified_primary_key: str, + primary_key: List[List[str]], +): + processor = initialize_processor() + + record = AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={ + "text": "This is the text and it is long enough to be split into multiple chunks. This is the text and it is long enough to be split into multiple chunks. This is the text and it is long enough to be split into multiple chunks", + "age": 25, + **primary_key_value, + }, + emitted_at=1234, + ) + + processor.text_fields = ["text"] + + processor.streams["namespace1_stream1"].destination_sync_mode = DestinationSyncMode.append_dedup + processor.streams["namespace1_stream1"].primary_key = primary_key + + chunks, id_to_delete = processor.process(record) + + assert len(chunks) > 1 + for chunk in chunks: + assert chunk.metadata["_ab_record_id"] == stringified_primary_key + assert id_to_delete == stringified_primary_key + + +@pytest.mark.parametrize( + "record, sync_mode, has_chunks, raises, expected_id_to_delete", + [ + pytest.param( + AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={"text": "This is the text", "id": "1"}, + emitted_at=1234, + ), + DestinationSyncMode.append_dedup, + True, + False, + "namespace1_stream1_1", + id="update", + ), + pytest.param( + AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={"text": "This is the text", "id": "1"}, + emitted_at=1234, + ), + DestinationSyncMode.append, + True, + False, + None, + id="append", + ), + pytest.param( + AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={"text": "This is the text", "id": "1", "_ab_cdc_deleted_at": 1234}, + emitted_at=1234, + ), + DestinationSyncMode.append_dedup, + False, + False, + "namespace1_stream1_1", + id="cdc_delete", + ), + pytest.param( + AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={"id": "1", "_ab_cdc_deleted_at": 1234}, + emitted_at=1234, + ), + DestinationSyncMode.append_dedup, + False, + False, + "namespace1_stream1_1", + id="cdc_delete_without_text", + ), + pytest.param( + AirbyteRecordMessage( + stream="stream1", + namespace="namespace1", + data={"id": "1"}, + emitted_at=1234, + ), + DestinationSyncMode.append_dedup, + False, + True, + "namespace1_stream1_1", + id="update_without_text", + ), + ], +) +def test_process_cdc_records(record, sync_mode, has_chunks, raises, expected_id_to_delete): + processor = initialize_processor() + + processor.text_fields = ["text"] + + processor.streams["namespace1_stream1"].destination_sync_mode = sync_mode + + if raises: + with pytest.raises(AirbyteTracedException): + processor.process(record) + else: + chunks, id_to_delete = processor.process(record) + if has_chunks: + assert len(chunks) > 0 + assert id_to_delete == expected_id_to_delete diff --git a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/embedder_test.py b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/embedder_test.py new file mode 100644 index 000000000000..3cf8e4114e5b --- /dev/null +++ b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/embedder_test.py @@ -0,0 +1,125 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock, call + +import pytest +from airbyte_cdk.destinations.vector_db_based.config import ( + AzureOpenAIEmbeddingConfigModel, + CohereEmbeddingConfigModel, + FakeEmbeddingConfigModel, + FromFieldEmbeddingConfigModel, + OpenAICompatibleEmbeddingConfigModel, + OpenAIEmbeddingConfigModel, +) +from airbyte_cdk.destinations.vector_db_based.embedder import ( + COHERE_VECTOR_SIZE, + OPEN_AI_VECTOR_SIZE, + AzureOpenAIEmbedder, + CohereEmbedder, + Document, + FakeEmbedder, + FromFieldEmbedder, + OpenAICompatibleEmbedder, + OpenAIEmbedder, +) +from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + + +@pytest.mark.parametrize( + "embedder_class, args, dimensions", + ( + (OpenAIEmbedder, [OpenAIEmbeddingConfigModel(**{"mode": "openai", "openai_key": "abc"}), 1000], OPEN_AI_VECTOR_SIZE), + (CohereEmbedder, [CohereEmbeddingConfigModel(**{"mode": "cohere", "cohere_key": "abc"})], COHERE_VECTOR_SIZE), + (FakeEmbedder, [FakeEmbeddingConfigModel(**{"mode": "fake"})], OPEN_AI_VECTOR_SIZE), + ( + AzureOpenAIEmbedder, + [ + AzureOpenAIEmbeddingConfigModel( + **{ + "mode": "azure_openai", + "openai_key": "abc", + "api_base": "https://my-resource.openai.azure.com", + "deployment": "my-deployment", + } + ), + 1000, + ], + OPEN_AI_VECTOR_SIZE, + ), + ( + OpenAICompatibleEmbedder, + [ + OpenAICompatibleEmbeddingConfigModel( + **{ + "mode": "openai_compatible", + "api_key": "abc", + "base_url": "https://my-service.com", + "model_name": "text-embedding-ada-002", + "dimensions": 50, + } + ) + ], + 50, + ), + ), +) +def test_embedder(embedder_class, args, dimensions): + embedder = embedder_class(*args) + mock_embedding_instance = MagicMock() + embedder.embeddings = mock_embedding_instance + + mock_embedding_instance.embed_query.side_effect = Exception("Some error") + assert embedder.check().startswith("Some error") + + mock_embedding_instance.embed_query.side_effect = None + assert embedder.check() is None + + assert embedder.embedding_dimensions == dimensions + + mock_embedding_instance.embed_documents.return_value = [[0] * dimensions] * 2 + + chunks = [ + Document(page_content="a", record=AirbyteRecordMessage(stream="mystream", data={}, emitted_at=0)), + Document(page_content="b", record=AirbyteRecordMessage(stream="mystream", data={}, emitted_at=0)), + ] + assert embedder.embed_documents(chunks) == mock_embedding_instance.embed_documents.return_value + mock_embedding_instance.embed_documents.assert_called_with(["a", "b"]) + + +@pytest.mark.parametrize( + "field_name, dimensions, metadata, expected_embedding, expected_error", + ( + ("a", 2, {"a": [1, 2]}, [1, 2], False), + ("a", 2, {"b": "b"}, None, True), + ("a", 2, {}, None, True), + ("a", 2, {"a": []}, None, True), + ("a", 2, {"a": [1, 2, 3]}, None, True), + ("a", 2, {"a": [1, "2", 3]}, None, True), + ), +) +def test_from_field_embedder(field_name, dimensions, metadata, expected_embedding, expected_error): + embedder = FromFieldEmbedder(FromFieldEmbeddingConfigModel(mode="from_field", dimensions=dimensions, field_name=field_name)) + chunks = [Document(page_content="a", record=AirbyteRecordMessage(stream="mystream", data=metadata, emitted_at=0))] + if expected_error: + with pytest.raises(AirbyteTracedException): + embedder.embed_documents(chunks) + else: + assert embedder.embed_documents(chunks) == [expected_embedding] + + +def test_openai_chunking(): + config = OpenAIEmbeddingConfigModel(**{"mode": "openai", "openai_key": "abc"}) + embedder = OpenAIEmbedder(config, 150) + mock_embedding_instance = MagicMock() + embedder.embeddings = mock_embedding_instance + + mock_embedding_instance.embed_documents.side_effect = lambda texts: [[0] * OPEN_AI_VECTOR_SIZE] * len(texts) + + chunks = [ + Document(page_content="a", record=AirbyteRecordMessage(stream="mystream", data={}, emitted_at=0)) for _ in range(1005) + ] + assert embedder.embed_documents(chunks) == [[0] * OPEN_AI_VECTOR_SIZE] * 1005 + mock_embedding_instance.embed_documents.assert_has_calls([call(["a"] * 1000), call(["a"] * 5)]) diff --git a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/writer_test.py b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/writer_test.py new file mode 100644 index 000000000000..c906d0f3e9b5 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/writer_test.py @@ -0,0 +1,172 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Optional +from unittest.mock import ANY, MagicMock, call + +import pytest +from airbyte_cdk.destinations.vector_db_based import ProcessingConfigModel, Writer +from airbyte_cdk.models.airbyte_protocol import ( + AirbyteLogMessage, + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStateMessage, + ConfiguredAirbyteCatalog, + Level, + Type, +) + + +def _generate_record_message(index: int, stream: str = "example_stream", namespace: Optional[str] = None): + return AirbyteMessage( + type=Type.RECORD, + record=AirbyteRecordMessage( + stream=stream, namespace=namespace, emitted_at=1234, data={"column_name": f"value {index}", "id": index} + ), + ) + + +BATCH_SIZE = 32 + + +def generate_stream(name: str = "example_stream", namespace: Optional[str] = None): + return { + "stream": { + "name": name, + "namespace": namespace, + "json_schema": {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": {}}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": False, + "default_cursor_field": ["column_name"], + }, + "primary_key": [["id"]], + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup", + } + + +def generate_mock_embedder(): + mock_embedder = MagicMock() + mock_embedder.embed_documents.return_value = [[0] * 1536] * (BATCH_SIZE + 5 + 5) + mock_embedder.embed_documents.side_effect = lambda chunks: [[0] * 1536] * len(chunks) + + return mock_embedder + + +@pytest.mark.parametrize("omit_raw_text", [True, False]) +def test_write(omit_raw_text: bool): + """ + Basic test for the write method, batcher and document processor. + """ + config_model = ProcessingConfigModel(chunk_overlap=0, chunk_size=1000, metadata_fields=None, text_fields=["column_name"]) + + configured_catalog: ConfiguredAirbyteCatalog = ConfiguredAirbyteCatalog.parse_obj({"streams": [generate_stream()]}) + # messages are flushed after 32 records or after a state message, so this will trigger two batches to be processed + input_messages = [_generate_record_message(i) for i in range(BATCH_SIZE + 5)] + state_message = AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage()) + input_messages.append(state_message) + # messages are also flushed once the input messages are exhausted, so this will trigger another batch + input_messages.extend([_generate_record_message(i) for i in range(5)]) + + mock_embedder = generate_mock_embedder() + + mock_indexer = MagicMock() + post_sync_log_message = AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message="post sync")) + mock_indexer.post_sync.return_value = [post_sync_log_message] + + # Create the DestinationLangchain instance + writer = Writer(config_model, mock_indexer, mock_embedder, BATCH_SIZE, omit_raw_text) + + output_messages = writer.write(configured_catalog, input_messages) + output_message = next(output_messages) + # assert state message is + assert output_message == state_message + + mock_indexer.pre_sync.assert_called_with(configured_catalog) + + # 1 batches due to max batch size reached and 1 batch due to state message + assert mock_indexer.index.call_count == 2 + assert mock_indexer.delete.call_count == 2 + assert mock_embedder.embed_documents.call_count == 2 + + if omit_raw_text: + for call_args in mock_indexer.index.call_args_list: + for chunk in call_args[0][0]: + if omit_raw_text: + assert chunk.page_content is None + else: + assert chunk.page_content is not None + + output_message = next(output_messages) + assert output_message == post_sync_log_message + + try: + next(output_messages) + assert False, "Expected end of message stream" + except StopIteration: + pass + + # 1 batch due to end of message stream + assert mock_indexer.index.call_count == 3 + assert mock_indexer.delete.call_count == 3 + assert mock_embedder.embed_documents.call_count == 3 + + mock_indexer.post_sync.assert_called() + + +def test_write_stream_namespace_split(): + """ + Test separate handling of streams and namespaces in the writer + + generate BATCH_SIZE - 10 records for example_stream, 5 records for example_stream with namespace abc and 10 records for example_stream2 + messages are flushed after 32 records or after a state message, so this will trigger 4 calls to the indexer: + * out of the first batch of 32, example_stream, example stream with namespace abd and the first 5 records for example_stream2 + * in the second batch, the remaining 5 records for example_stream2 + """ + config_model = ProcessingConfigModel(chunk_overlap=0, chunk_size=1000, metadata_fields=None, text_fields=["column_name"]) + + configured_catalog: ConfiguredAirbyteCatalog = ConfiguredAirbyteCatalog.parse_obj( + { + "streams": [ + generate_stream(), + generate_stream(namespace="abc"), + generate_stream("example_stream2"), + ] + } + ) + + input_messages = [_generate_record_message(i, "example_stream", None) for i in range(BATCH_SIZE - 10)] + input_messages.extend([_generate_record_message(i, "example_stream", "abc") for i in range(5)]) + input_messages.extend([_generate_record_message(i, "example_stream2", None) for i in range(10)]) + state_message = AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage()) + input_messages.append(state_message) + + mock_embedder = generate_mock_embedder() + + mock_indexer = MagicMock() + mock_indexer.post_sync.return_value = [] + + # Create the DestinationLangchain instance + writer = Writer(config_model, mock_indexer, mock_embedder, BATCH_SIZE, False) + + output_messages = writer.write(configured_catalog, input_messages) + next(output_messages) + + mock_indexer.index.assert_has_calls( + [ + call(ANY, None, "example_stream"), + call(ANY, "abc", "example_stream"), + call(ANY, None, "example_stream2"), + call(ANY, None, "example_stream2"), + ] + ) + mock_indexer.index.assert_has_calls( + [ + call(ANY, None, "example_stream"), + call(ANY, "abc", "example_stream"), + call(ANY, None, "example_stream2"), + call(ANY, None, "example_stream2"), + ] + ) + assert mock_embedder.embed_documents.call_count == 4 diff --git a/airbyte-cdk/python/unit_tests/resource/http/response/test-resource.json b/airbyte-cdk/python/unit_tests/resource/http/response/test-resource.json new file mode 100644 index 000000000000..667ec0669008 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/resource/http/response/test-resource.json @@ -0,0 +1,3 @@ +{ + "test-source template": "this is a template for test-resource" +} diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/__init__.py b/airbyte-cdk/python/unit_tests/sources/concurrent_source/__init__.py similarity index 100% rename from airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/__init__.py rename to airbyte-cdk/python/unit_tests/sources/concurrent_source/__init__.py diff --git a/airbyte-cdk/python/unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py b/airbyte-cdk/python/unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py new file mode 100644 index 000000000000..96da2b383955 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py @@ -0,0 +1,105 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import logging +from typing import Any, List, Mapping, Optional, Tuple +from unittest.mock import Mock + +import freezegun +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + SyncMode, +) +from airbyte_cdk.models import Type as MessageType +from airbyte_cdk.sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter +from airbyte_cdk.sources.message import InMemoryMessageRepository +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade +from airbyte_cdk.sources.streams.concurrent.cursor import NoopCursor + + +class _MockSource(ConcurrentSourceAdapter): + def __init__(self, concurrent_source, _streams_to_is_concurrent, logger): + super().__init__(concurrent_source) + self._streams_to_is_concurrent = _streams_to_is_concurrent + self._logger = logger + + message_repository = InMemoryMessageRepository() + + def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: + raise NotImplementedError + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + return [ + StreamFacade.create_from_stream(s, self, self._logger, None, NoopCursor()) if is_concurrent else s + for s, is_concurrent in self._streams_to_is_concurrent.items() + ] + + +@freezegun.freeze_time("2020-01-01T00:00:00") +def test_concurrent_source_adapter(): + concurrent_source = Mock() + message_from_concurrent_stream = AirbyteMessage( + type=MessageType.RECORD, + record=AirbyteRecordMessage( + stream="s2", + data={"data": 2}, + emitted_at=1577836800000, + ), + ) + concurrent_source.read.return_value = iter([message_from_concurrent_stream]) + regular_stream = _mock_stream("s1", [{"data": 1}]) + concurrent_stream = _mock_stream("s2", []) + unavailable_stream = _mock_stream("s3", [{"data": 3}], False) + concurrent_stream.name = "s2" + logger = Mock() + adapter = _MockSource(concurrent_source, {regular_stream: False, concurrent_stream: True, unavailable_stream: False}, logger) + + messages = list(adapter.read(logger, {}, _configured_catalog([regular_stream, concurrent_stream, unavailable_stream]))) + records = [m for m in messages if m.type == MessageType.RECORD] + + expected_records = [ + message_from_concurrent_stream, + AirbyteMessage( + type=MessageType.RECORD, + record=AirbyteRecordMessage( + stream="s1", + data={"data": 1}, + emitted_at=1577836800000, + ), + ), + ] + + assert records == expected_records + + +def _mock_stream(name: str, data=[], available: bool = True): + s = Mock() + s.name = name + s.as_airbyte_stream.return_value = AirbyteStream( + name=name, + json_schema={}, + supported_sync_modes=[SyncMode.full_refresh], + ) + s.check_availability.return_value = (True, None) if available else (False, "not available") + s.read_full_refresh.return_value = iter(data) + s.primary_key = None + return s + + +def _configured_catalog(streams: List[Stream]): + return ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=stream.as_airbyte_stream(), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + for stream in streams + ] + ) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_oauth.py b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_oauth.py index ff425380ed3c..bd019d374987 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_oauth.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_oauth.py @@ -1,7 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - +import base64 import logging from unittest.mock import Mock @@ -66,6 +66,46 @@ def test_refresh_request_body(self): } assert body == expected + def test_refresh_with_encode_config_params(self): + oauth = DeclarativeOauth2Authenticator( + token_refresh_endpoint="{{ config['refresh_endpoint'] }}", + client_id="{{ config['client_id'] | base64encode }}", + client_secret="{{ config['client_secret'] | base64encode }}", + config=config, + parameters={}, + grant_type="client_credentials", + ) + body = oauth.build_refresh_request_body() + expected = { + "grant_type": "client_credentials", + "client_id": base64.b64encode(config["client_id"].encode("utf-8")).decode(), + "client_secret": base64.b64encode(config["client_secret"].encode("utf-8")).decode(), + "refresh_token": None, + } + assert body == expected + + def test_refresh_with_decode_config_params(self): + updated_config_fields = { + "client_id": base64.b64encode(config["client_id"].encode("utf-8")).decode(), + "client_secret": base64.b64encode(config["client_secret"].encode("utf-8")).decode(), + } + oauth = DeclarativeOauth2Authenticator( + token_refresh_endpoint="{{ config['refresh_endpoint'] }}", + client_id="{{ config['client_id'] | base64decode }}", + client_secret="{{ config['client_secret'] | base64decode }}", + config=config | updated_config_fields, + parameters={}, + grant_type="client_credentials", + ) + body = oauth.build_refresh_request_body() + expected = { + "grant_type": "client_credentials", + "client_id": "some_client_id", + "client_secret": "some_client_secret", + "refresh_token": None, + } + assert body == expected + def test_refresh_without_refresh_token(self): """ Should work fine for grant_type client_credentials. @@ -84,7 +124,6 @@ def test_refresh_without_refresh_token(self): "client_id": "some_client_id", "client_secret": "some_client_secret", "refresh_token": None, - "scopes": None, } assert body == expected @@ -126,6 +165,35 @@ def test_refresh_access_token(self, mocker): assert ("access_token", 1000) == token + @pytest.mark.parametrize( + "timestamp, expected_date", + [ + (1640995200, "2022-01-01T00:00:00Z"), + ("1650758400", "2022-04-24T00:00:00Z"), + ], + ids=["timestamp_as_integer", "timestamp_as_integer_inside_string"], + ) + def test_initialize_declarative_oauth_with_token_expiry_date_as_timestamp(self, timestamp, expected_date): + # TODO: should be fixed inside DeclarativeOauth2Authenticator, remove next line after fixing + with pytest.raises(TypeError): + oauth = DeclarativeOauth2Authenticator( + token_refresh_endpoint="{{ config['refresh_endpoint'] }}", + client_id="{{ config['client_id'] }}", + client_secret="{{ config['client_secret'] }}", + refresh_token="{{ parameters['refresh_token'] }}", + config=config | {"token_expiry_date": timestamp}, + scopes=["scope1", "scope2"], + token_expiry_date="{{ config['token_expiry_date'] }}", + refresh_request_body={ + "custom_field": "{{ config['custom_field'] }}", + "another_field": "{{ config['another_field'] }}", + "scopes": ["no_override"], + }, + parameters={}, + ) + + assert oauth.get_token_expiry_date() == pendulum.parse(expected_date) + @pytest.mark.parametrize( "expires_in_response, token_expiry_date_format", [ @@ -149,6 +217,7 @@ def test_refresh_access_token_expire_format(self, mocker, expires_in_response, t scopes=["scope1", "scope2"], token_expiry_date="{{ config['token_expiry_date'] }}", token_expiry_date_format=token_expiry_date_format, + token_expiry_is_time_of_expiration=True, refresh_request_body={ "custom_field": "{{ config['custom_field'] }}", "another_field": "{{ config['another_field'] }}", @@ -206,6 +275,28 @@ def test_set_token_expiry_date_no_format(self, mocker, expires_in_response, next assert "access_token" == token assert oauth.get_token_expiry_date() == pendulum.parse(next_day) + def test_error_handling(self, mocker): + oauth = DeclarativeOauth2Authenticator( + token_refresh_endpoint="{{ config['refresh_endpoint'] }}", + client_id="{{ config['client_id'] }}", + client_secret="{{ config['client_secret'] }}", + refresh_token="{{ config['refresh_token'] }}", + config=config, + scopes=["scope1", "scope2"], + refresh_request_body={ + "custom_field": "{{ config['custom_field'] }}", + "another_field": "{{ config['another_field'] }}", + "scopes": ["no_override"], + }, + parameters={}, + ) + resp.status_code = 400 + mocker.patch.object(resp, "json", return_value={"access_token": "access_token", "expires_in": 123}) + mocker.patch.object(requests, "request", side_effect=mock_request, autospec=True) + with pytest.raises(requests.exceptions.HTTPError) as e: + oauth.refresh_access_token() + assert e.value.errno == 400 + def mock_request(method, url, data): if url == "refresh_end": diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_selective_authenticator.py b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_selective_authenticator.py new file mode 100644 index 000000000000..346b284c3786 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_selective_authenticator.py @@ -0,0 +1,39 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import pytest +from airbyte_cdk.sources.declarative.auth.selective_authenticator import SelectiveAuthenticator + + +def test_authenticator_selected(mocker): + authenticators = {"one": mocker.Mock(), "two": mocker.Mock()} + auth = SelectiveAuthenticator( + config={"auth": {"type": "one"}}, + authenticators=authenticators, + authenticator_selection_path=["auth", "type"], + ) + + assert auth is authenticators["one"] + + +def test_selection_path_not_found(mocker): + authenticators = {"one": mocker.Mock(), "two": mocker.Mock()} + + with pytest.raises(ValueError, match="The path from `authenticator_selection_path` is not found in the config"): + _ = SelectiveAuthenticator( + config={"auth": {"type": "one"}}, + authenticators=authenticators, + authenticator_selection_path=["auth_type"], + ) + + +def test_selected_auth_not_found(mocker): + authenticators = {"one": mocker.Mock(), "two": mocker.Mock()} + + with pytest.raises(ValueError, match="The authenticator `unknown` is not found"): + _ = SelectiveAuthenticator( + config={"auth": {"type": "unknown"}}, + authenticators=authenticators, + authenticator_selection_path=["auth", "type"], + ) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_auth.py b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_auth.py index 6217d72fe9c7..4db4a1ea0b0a 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_auth.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_auth.py @@ -83,14 +83,10 @@ def test_api_key_authenticator(test_name, header, token, expected_header, expect """ token_provider = InterpolatedStringTokenProvider(config=config, api_token=token, parameters=parameters) token_auth = ApiKeyAuthenticator( - request_option=RequestOption( - inject_into=RequestOptionType.header, - field_name=header, - parameters={} - ), + request_option=RequestOption(inject_into=RequestOptionType.header, field_name=header, parameters={}), token_provider=token_provider, config=config, - parameters=parameters + parameters=parameters, ) header1 = token_auth.get_auth_header() header2 = token_auth.get_auth_header() @@ -107,15 +103,87 @@ def test_api_key_authenticator(test_name, header, token, expected_header, expect @pytest.mark.parametrize( "test_name, field_name, token, expected_field_name, expected_field_value, inject_type, validation_fn", [ - ("test_static_token", "Authorization", "test-token", "Authorization", "test-token", RequestOptionType.request_parameter, "get_request_params"), - ("test_token_from_config", "{{ config.header }}", "{{ config.username }}", "header", "user", RequestOptionType.request_parameter, "get_request_params"), - ("test_token_from_parameters", "{{ parameters.header }}", "{{ parameters.username }}", "header", "user", RequestOptionType.request_parameter, "get_request_params"), - ("test_static_token", "Authorization", "test-token", "Authorization", "test-token", RequestOptionType.body_data, "get_request_body_data"), - ("test_token_from_config", "{{ config.header }}", "{{ config.username }}", "header", "user", RequestOptionType.body_data, "get_request_body_data"), - ("test_token_from_parameters", "{{ parameters.header }}", "{{ parameters.username }}", "header", "user", RequestOptionType.body_data, "get_request_body_data"), - ("test_static_token", "Authorization", "test-token", "Authorization", "test-token", RequestOptionType.body_json, "get_request_body_json"), - ("test_token_from_config", "{{ config.header }}", "{{ config.username }}", "header", "user", RequestOptionType.body_json, "get_request_body_json"), - ("test_token_from_parameters", "{{ parameters.header }}", "{{ parameters.username }}", "header", "user", RequestOptionType.body_json, "get_request_body_json"), + ( + "test_static_token", + "Authorization", + "test-token", + "Authorization", + "test-token", + RequestOptionType.request_parameter, + "get_request_params", + ), + ( + "test_token_from_config", + "{{ config.header }}", + "{{ config.username }}", + "header", + "user", + RequestOptionType.request_parameter, + "get_request_params", + ), + ( + "test_token_from_parameters", + "{{ parameters.header }}", + "{{ parameters.username }}", + "header", + "user", + RequestOptionType.request_parameter, + "get_request_params", + ), + ( + "test_static_token", + "Authorization", + "test-token", + "Authorization", + "test-token", + RequestOptionType.body_data, + "get_request_body_data", + ), + ( + "test_token_from_config", + "{{ config.header }}", + "{{ config.username }}", + "header", + "user", + RequestOptionType.body_data, + "get_request_body_data", + ), + ( + "test_token_from_parameters", + "{{ parameters.header }}", + "{{ parameters.username }}", + "header", + "user", + RequestOptionType.body_data, + "get_request_body_data", + ), + ( + "test_static_token", + "Authorization", + "test-token", + "Authorization", + "test-token", + RequestOptionType.body_json, + "get_request_body_json", + ), + ( + "test_token_from_config", + "{{ config.header }}", + "{{ config.username }}", + "header", + "user", + RequestOptionType.body_json, + "get_request_body_json", + ), + ( + "test_token_from_parameters", + "{{ parameters.header }}", + "{{ parameters.username }}", + "header", + "user", + RequestOptionType.body_json, + "get_request_body_json", + ), ], ) def test_api_key_authenticator_inject(test_name, field_name, token, expected_field_name, expected_field_value, inject_type, validation_fn): @@ -124,13 +192,9 @@ def test_api_key_authenticator_inject(test_name, field_name, token, expected_fie """ token_provider = InterpolatedStringTokenProvider(config=config, api_token=token, parameters=parameters) token_auth = ApiKeyAuthenticator( - request_option=RequestOption( - inject_into=inject_type, - field_name=field_name, - parameters={} - ), + request_option=RequestOption(inject_into=inject_type, field_name=field_name, parameters={}), token_provider=token_provider, config=config, - parameters=parameters + parameters=parameters, ) assert {expected_field_name: expected_field_value} == getattr(token_auth, validation_fn)() diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_provider.py b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_provider.py index 7badce21801f..e73e5eef0838 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_provider.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_token_provider.py @@ -17,11 +17,18 @@ def create_session_token_provider(): login_response.json.return_value = {"nested": {"token": "my_token"}} login_requester.send_request.return_value = login_response - return SessionTokenProvider(login_requester=login_requester, session_token_path=["nested", "token"], expiration_duration=parse_duration("PT1H"), parameters={"test": "test"}) + return SessionTokenProvider( + login_requester=login_requester, + session_token_path=["nested", "token"], + expiration_duration=parse_duration("PT1H"), + parameters={"test": "test"}, + ) def test_interpolated_string_token_provider(): - provider = InterpolatedStringTokenProvider(config={"config_key": "val"}, api_token="{{ config.config_key }}-{{ parameters.test }}", parameters={"test": "test"}) + provider = InterpolatedStringTokenProvider( + config={"config_key": "val"}, api_token="{{ config.config_key }}-{{ parameters.test }}", parameters={"test": "test"} + ) assert provider.get_token() == "val-test" diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_min_max_datetime.py b/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_min_max_datetime.py index 8aff0921bb6b..b23f6e2fffe9 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_min_max_datetime.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_min_max_datetime.py @@ -104,5 +104,9 @@ def test_min_max_datetime_lazy_eval(): } assert datetime.datetime(2022, 1, 10, 0, 0, tzinfo=datetime.timezone.utc) == MinMaxDatetime(**kwargs, parameters={}).get_datetime({}) - assert datetime.datetime(2022, 1, 20, 0, 0, tzinfo=datetime.timezone.utc) == MinMaxDatetime(**kwargs, parameters={"min_datetime": "2022-01-20T00:00:00"}).get_datetime({}) - assert datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc) == MinMaxDatetime(**kwargs, parameters={"max_datetime": "2021-01-01T00:00:00"}).get_datetime({}) + assert datetime.datetime(2022, 1, 20, 0, 0, tzinfo=datetime.timezone.utc) == MinMaxDatetime( + **kwargs, parameters={"min_datetime": "2022-01-20T00:00:00"} + ).get_datetime({}) + assert datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc) == MinMaxDatetime( + **kwargs, parameters={"max_datetime": "2021-01-01T00:00:00"} + ).get_datetime({}) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py index a03dc4becbb9..76fad421ea76 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py @@ -32,7 +32,12 @@ ), ("test_field_does_not_exist", ["record"], {"id": 1}, []), ("test_nested_list", ["list", "*", "item"], {"list": [{"item": {"id": "1"}}]}, [{"id": "1"}]), - ("test_complex_nested_list", ['data', '*', 'list', 'data2', '*'], {"data": [{"list": {"data2": [{"id": 1}, {"id": 2}]}},{"list": {"data2": [{"id": 3}, {"id": 4}]}}]}, [{"id": 1}, {"id": 2}, {"id": 3}, {"id": 4}]) + ( + "test_complex_nested_list", + ["data", "*", "list", "data2", "*"], + {"data": [{"list": {"data2": [{"id": 1}, {"id": 2}]}}, {"list": {"data2": [{"id": 3}, {"id": 4}]}}]}, + [{"id": 1}, {"id": 2}, {"id": 3}, {"id": 4}], + ), ], ) def test_dpath_extractor(test_name, field_path, body, expected_records): diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py index 619b228b009f..5fa6af43d831 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_record_selector.py @@ -13,6 +13,7 @@ from airbyte_cdk.sources.declarative.extractors.record_selector import RecordSelector from airbyte_cdk.sources.declarative.transformations import RecordTransformation from airbyte_cdk.sources.declarative.types import Record +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer @pytest.mark.parametrize( @@ -68,6 +69,7 @@ def test_record_filter(test_name, field_path, filter_template, body, expected_da stream_state = {"created_at": "06-06-21"} stream_slice = {"last_seen": "06-10-21"} next_page_token = {"last_seen_id": 14} + schema = create_schema() first_transformation = Mock(spec=RecordTransformation) second_transformation = Mock(spec=RecordTransformation) transformations = [first_transformation, second_transformation] @@ -84,13 +86,15 @@ def test_record_filter(test_name, field_path, filter_template, body, expected_da record_filter=record_filter, transformations=transformations, config=config, - parameters=parameters + parameters=parameters, + schema_normalization=TypeTransformer(TransformConfig.NoTransform), ) actual_records = record_selector.select_records( - response=response, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token + response=response, records_schema=schema, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token ) assert actual_records == [Record(data, stream_slice) for data in expected_data] + calls = [] for record in expected_data: calls.append(call(record, config=config, stream_state=stream_state, stream_slice=stream_slice)) @@ -99,7 +103,77 @@ def test_record_filter(test_name, field_path, filter_template, body, expected_da transformation.transform.assert_has_calls(calls) +@pytest.mark.parametrize( + "test_name, schema, schema_transformation, body, expected_data", + [ + ( + "test_with_empty_schema", + {}, + TransformConfig.NoTransform, + {"data": [{"id": 1, "created_at": "06-06-21", "field_int": "100", "field_float": "123.3"}]}, + [{"id": 1, "created_at": "06-06-21", "field_int": "100", "field_float": "123.3"}], + ), + ( + "test_with_schema_none_normalizer", + {}, + TransformConfig.NoTransform, + {"data": [{"id": 1, "created_at": "06-06-21", "field_int": "100", "field_float": "123.3"}]}, + [{"id": 1, "created_at": "06-06-21", "field_int": "100", "field_float": "123.3"}], + ), + ( + "test_with_schema_and_default_normalizer", + {}, + TransformConfig.DefaultSchemaNormalization, + {"data": [{"id": 1, "created_at": "06-06-21", "field_int": "100", "field_float": "123.3"}]}, + [{"id": "1", "created_at": "06-06-21", "field_int": 100, "field_float": 123.3}], + ), + ], +) +def test_schema_normalization(test_name, schema, schema_transformation, body, expected_data): + config = {"response_override": "stop_if_you_see_me"} + parameters = {"parameters_field": "data", "created_at": "06-07-21"} + stream_state = {"created_at": "06-06-21"} + stream_slice = {"last_seen": "06-10-21"} + next_page_token = {"last_seen_id": 14} + + response = create_response(body) + schema = create_schema() + decoder = JsonDecoder(parameters={}) + extractor = DpathExtractor(field_path=["data"], decoder=decoder, config=config, parameters=parameters) + record_selector = RecordSelector( + extractor=extractor, + record_filter=None, + transformations=[], + config=config, + parameters=parameters, + schema_normalization=TypeTransformer(schema_transformation), + ) + + actual_records = record_selector.select_records( + response=response, + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + records_schema=schema, + ) + + assert actual_records == [Record(data, stream_slice) for data in expected_data] + + def create_response(body): response = requests.Response() response._content = json.dumps(body).encode("utf-8") return response + + +def create_schema(): + return { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": {"type": "string"}, + "created_at": {"type": "string"}, + "field_int": {"type": "integer"}, + "field_float": {"type": "number"}, + }, + } diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py index 67617a9f0124..c128f04f391d 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py @@ -509,7 +509,9 @@ def test_request_option(test_name, inject_into, field_name, expected_req_params, ("test_parse_date_number", "20210101", "%Y%m%d", "P1D", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc)), ], ) -def test_parse_date_legacy_merge_datetime_format_in_cursor_datetime_format(test_name, input_date, date_format, date_format_granularity, expected_output_date): +def test_parse_date_legacy_merge_datetime_format_in_cursor_datetime_format( + test_name, input_date, date_format, date_format_granularity, expected_output_date +): slicer = DatetimeBasedCursor( start_datetime=MinMaxDatetime("2021-01-01T00:00:00.000000+0000", parameters={}), end_datetime=MinMaxDatetime("2021-01-10T00:00:00.000000+0000", parameters={}), diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor.py b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor.py index 2f1b8eccba18..cb7857c9352a 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor.py @@ -20,10 +20,7 @@ "partition_key int": 1, "partition_key list str": ["list item 1", "list item 2"], "partition_key list dict": [ - { - "dict within list key 1-1": "dict within list value 1-1", - "dict within list key 1-2": "dict within list value 1-2" - }, + {"dict within list key 1-1": "dict within list value 1-1", "dict within list key 1-2": "dict within list value 1-2"}, {"dict within list key 2": "dict within list value 2"}, ], "partition_key nested dict": { @@ -43,18 +40,14 @@ "partition_router_field_1": "X1", "partition_router_field_2": "Y1", }, - "cursor": { - "cursor state field": 1 - } + "cursor": {"cursor state field": 1}, }, { "partition": { "partition_router_field_1": "X2", "partition_router_field_2": "Y2", }, - "cursor": { - "cursor state field": 2 - } + "cursor": {"cursor state field": 2}, }, ] } @@ -135,8 +128,9 @@ def test_given_no_partition_when_stream_slices_then_no_slices(mocked_cursor_fact assert not next(slices, None) -def test_given_partition_router_without_state_has_one_partition_then_return_one_slice_per_cursor_slice(mocked_cursor_factory, - mocked_partition_router): +def test_given_partition_router_without_state_has_one_partition_then_return_one_slice_per_cursor_slice( + mocked_cursor_factory, mocked_partition_router +): partition = {"partition_field_1": "a value", "partition_field_2": "another value"} mocked_partition_router.stream_slices.return_value = [partition] cursor_slices = [{"start_datetime": 1}, {"start_datetime": 2}] @@ -148,20 +142,16 @@ def test_given_partition_router_without_state_has_one_partition_then_return_one_ assert list(slices) == [PerPartitionStreamSlice(partition, cursor_slice) for cursor_slice in cursor_slices] -def test_given_partition_associated_with_state_when_stream_slices_then_do_not_recreate_cursor(mocked_cursor_factory, - mocked_partition_router): +def test_given_partition_associated_with_state_when_stream_slices_then_do_not_recreate_cursor( + mocked_cursor_factory, mocked_partition_router +): partition = {"partition_field_1": "a value", "partition_field_2": "another value"} mocked_partition_router.stream_slices.return_value = [partition] cursor_slices = [{"start_datetime": 1}] mocked_cursor_factory.create.return_value = MockedCursorBuilder().with_stream_slices(cursor_slices).build() cursor = PerPartitionCursor(mocked_cursor_factory, mocked_partition_router) - cursor.set_initial_state({ - "states": [{ - "partition": partition, - "cursor": CURSOR_STATE - }] - }) + cursor.set_initial_state({"states": [{"partition": partition, "cursor": CURSOR_STATE}]}) mocked_cursor_factory.create.assert_called_once() slices = list(cursor.stream_slices()) @@ -171,33 +161,23 @@ def test_given_partition_associated_with_state_when_stream_slices_then_do_not_re def test_given_multiple_partitions_then_each_have_their_state(mocked_cursor_factory, mocked_partition_router): first_partition = {"first_partition_key": "first_partition_value"} - mocked_partition_router.stream_slices.return_value = [ - first_partition, - {"second_partition_key": "second_partition_value"} - ] + mocked_partition_router.stream_slices.return_value = [first_partition, {"second_partition_key": "second_partition_value"}] first_cursor = MockedCursorBuilder().with_stream_slices([{CURSOR_SLICE_FIELD: "first slice cursor value"}]).build() second_cursor = MockedCursorBuilder().with_stream_slices([{CURSOR_SLICE_FIELD: "second slice cursor value"}]).build() mocked_cursor_factory.create.side_effect = [first_cursor, second_cursor] cursor = PerPartitionCursor(mocked_cursor_factory, mocked_partition_router) - cursor.set_initial_state({ - "states": [{ - "partition": first_partition, - "cursor": CURSOR_STATE - }] - }) + cursor.set_initial_state({"states": [{"partition": first_partition, "cursor": CURSOR_STATE}]}) slices = list(cursor.stream_slices()) first_cursor.stream_slices.assert_called_once() second_cursor.stream_slices.assert_called_once() assert slices == [ PerPartitionStreamSlice( - partition={"first_partition_key": "first_partition_value"}, - cursor_slice={CURSOR_SLICE_FIELD: "first slice cursor value"} + partition={"first_partition_key": "first_partition_value"}, cursor_slice={CURSOR_SLICE_FIELD: "first slice cursor value"} ), PerPartitionStreamSlice( - partition={"second_partition_key": "second_partition_value"}, - cursor_slice={CURSOR_SLICE_FIELD: "second slice cursor value"} + partition={"second_partition_key": "second_partition_value"}, cursor_slice={CURSOR_SLICE_FIELD: "second slice cursor value"} ), ] @@ -205,21 +185,15 @@ def test_given_multiple_partitions_then_each_have_their_state(mocked_cursor_fact def test_given_stream_slices_when_get_stream_state_then_return_updated_state(mocked_cursor_factory, mocked_partition_router): mocked_cursor_factory.create.side_effect = [ MockedCursorBuilder().with_stream_state({CURSOR_STATE_KEY: "first slice cursor value"}).build(), - MockedCursorBuilder().with_stream_state({CURSOR_STATE_KEY: "second slice cursor value"}).build() + MockedCursorBuilder().with_stream_state({CURSOR_STATE_KEY: "second slice cursor value"}).build(), ] mocked_partition_router.stream_slices.return_value = [{"partition key": "first partition"}, {"partition key": "second partition"}] cursor = PerPartitionCursor(mocked_cursor_factory, mocked_partition_router) list(cursor.stream_slices()) assert cursor.get_stream_state() == { "states": [ - { - "partition": {"partition key": "first partition"}, - "cursor": {CURSOR_STATE_KEY: "first slice cursor value"} - }, - { - "partition": {"partition key": "second partition"}, - "cursor": {CURSOR_STATE_KEY: "second slice cursor value"} - } + {"partition": {"partition key": "first partition"}, "cursor": {CURSOR_STATE_KEY: "first slice cursor value"}}, + {"partition": {"partition key": "second partition"}, "cursor": {CURSOR_STATE_KEY: "second slice cursor value"}}, ] } @@ -231,19 +205,9 @@ def test_when_get_stream_state_then_delegate_to_underlying_cursor(mocked_cursor_ cursor = PerPartitionCursor(mocked_cursor_factory, mocked_partition_router) first_slice = list(cursor.stream_slices())[0] - cursor.should_be_synced( - Record( - {}, - first_slice - ) - ) + cursor.should_be_synced(Record({}, first_slice)) - underlying_cursor.should_be_synced.assert_called_once_with( - Record( - {}, - first_slice.cursor_slice - ) - ) + underlying_cursor.should_be_synced.assert_called_once_with(Record({}, first_slice.cursor_slice)) def test_close_slice(mocked_cursor_factory, mocked_partition_router): @@ -279,10 +243,7 @@ def test_given_unknown_partition_when_close_slice_then_raise_error(): cursor = PerPartitionCursor(any_cursor_factory, any_partition_router) stream_slice = PerPartitionStreamSlice(partition={"unknown_partition": "unknown"}, cursor_slice={}) with pytest.raises(ValueError): - cursor.close_slice( - stream_slice, - Record({}, stream_slice) - ) + cursor.close_slice(stream_slice, Record({}, stream_slice)) def test_given_unknown_partition_when_should_be_synced_then_raise_error(): @@ -290,15 +251,7 @@ def test_given_unknown_partition_when_should_be_synced_then_raise_error(): any_partition_router = Mock() cursor = PerPartitionCursor(any_cursor_factory, any_partition_router) with pytest.raises(ValueError): - cursor.should_be_synced( - Record( - {}, - PerPartitionStreamSlice( - partition={"unknown_partition": "unknown"}, - cursor_slice={} - ) - ) - ) + cursor.should_be_synced(Record({}, PerPartitionStreamSlice(partition={"unknown_partition": "unknown"}, cursor_slice={}))) def test_given_records_with_different_slice_when_is_greater_than_or_equal_then_raise_error(): @@ -308,7 +261,7 @@ def test_given_records_with_different_slice_when_is_greater_than_or_equal_then_r with pytest.raises(ValueError): cursor.is_greater_than_or_equal( Record({}, PerPartitionStreamSlice(partition={"a slice": "value"}, cursor_slice={})), - Record({}, PerPartitionStreamSlice(partition={"another slice": "value"}, cursor_slice={})) + Record({}, PerPartitionStreamSlice(partition={"another slice": "value"}, cursor_slice={})), ) @@ -319,7 +272,7 @@ def test_given_slice_is_unknown_when_is_greater_than_or_equal_then_raise_error() with pytest.raises(ValueError): cursor.is_greater_than_or_equal( Record({}, PerPartitionStreamSlice(partition={"a slice": "value"}, cursor_slice={})), - Record({}, PerPartitionStreamSlice(partition={"a slice": "value"}, cursor_slice={})) + Record({}, PerPartitionStreamSlice(partition={"a slice": "value"}, cursor_slice={})), ) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py index 0dd19c66fc3c..ef1f123fd124 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py @@ -35,7 +35,7 @@ def with_incremental_sync(self, start_datetime, end_datetime, datetime_format, c "datetime_format": datetime_format, "cursor_field": cursor_field, "step": step, - "cursor_granularity": cursor_granularity + "cursor_granularity": cursor_granularity, } return self @@ -43,12 +43,7 @@ def build(self): manifest = { "version": "0.34.2", "type": "DeclarativeSource", - "check": { - "type": "CheckStream", - "stream_names": [ - "Rates" - ] - }, + "check": {"type": "CheckStream", "stream_names": ["Rates"]}, "streams": [ { "type": "DeclarativeStream", @@ -56,11 +51,7 @@ def build(self): "primary_key": [], "schema_loader": { "type": "InlineSchemaLoader", - "schema": { - "$schema": "http://json-schema.org/schema#", - "properties": {}, - "type": "object" - } + "schema": {"$schema": "http://json-schema.org/schema#", "properties": {}, "type": "object"}, }, "retriever": { "type": "SimpleRetriever", @@ -70,14 +61,8 @@ def build(self): "path": "/exchangerates_data/latest", "http_method": "GET", }, - "record_selector": { - "type": "RecordSelector", - "extractor": { - "type": "DpathExtractor", - "field_path": [] - } - }, - } + "record_selector": {"type": "RecordSelector", "extractor": {"type": "DpathExtractor", "field_path": []}}, + }, } ], "spec": { @@ -86,11 +71,11 @@ def build(self): "type": "object", "required": [], "properties": {}, - "additionalProperties": True + "additionalProperties": True, }, "documentation_url": "https://example.org", - "type": "Spec" - } + "type": "Spec", + }, } if self._incremental_sync: manifest["streams"][0]["incremental_sync"] = self._incremental_sync @@ -101,14 +86,17 @@ def build(self): def test_given_state_for_only_some_partition_when_stream_slices_then_create_slices_using_state_or_start_from_start_datetime(): source = ManifestDeclarativeSource( - source_config=ManifestBuilder().with_list_partition_router("partition_field", ["1", "2"]).with_incremental_sync( - start_datetime="2022-01-01", - end_datetime="2022-02-28", - datetime_format="%Y-%m-%d", - cursor_field=CURSOR_FIELD, - step="P1M", - cursor_granularity="P1D", - ).build() + source_config=ManifestBuilder() + .with_list_partition_router("partition_field", ["1", "2"]) + .with_incremental_sync( + start_datetime="2022-01-01", + end_datetime="2022-02-28", + datetime_format="%Y-%m-%d", + cursor_field=CURSOR_FIELD, + step="P1M", + cursor_granularity="P1D", + ) + .build() ) stream_instance = source.streams({})[0] stream_instance.state = { @@ -134,20 +122,25 @@ def test_given_state_for_only_some_partition_when_stream_slices_then_create_slic def test_given_record_for_partition_when_read_then_update_state(): source = ManifestDeclarativeSource( - source_config=ManifestBuilder().with_list_partition_router("partition_field", ["1", "2"]).with_incremental_sync( - start_datetime="2022-01-01", - end_datetime="2022-02-28", - datetime_format="%Y-%m-%d", - cursor_field=CURSOR_FIELD, - step="P1M", - cursor_granularity="P1D", - ).build() + source_config=ManifestBuilder() + .with_list_partition_router("partition_field", ["1", "2"]) + .with_incremental_sync( + start_datetime="2022-01-01", + end_datetime="2022-02-28", + datetime_format="%Y-%m-%d", + cursor_field=CURSOR_FIELD, + step="P1M", + cursor_granularity="P1D", + ) + .build() ) stream_instance = source.streams({})[0] list(stream_instance.stream_slices(sync_mode=SYNC_MODE)) stream_slice = PerPartitionStreamSlice({"partition_field": "1"}, {"start_time": "2022-01-01", "end_time": "2022-01-31"}) - with patch.object(SimpleRetriever, "_read_pages", side_effect=[[Record({"a record key": "a record value", CURSOR_FIELD: "2022-01-15"}, stream_slice)]]): + with patch.object( + SimpleRetriever, "_read_pages", side_effect=[[Record({"a record key": "a record value", CURSOR_FIELD: "2022-01-15"}, stream_slice)]] + ): list( stream_instance.read_records( sync_mode=SYNC_MODE, @@ -157,9 +150,11 @@ def test_given_record_for_partition_when_read_then_update_state(): ) ) - assert stream_instance.state == {"states": [ - { - "partition": {"partition_field": "1"}, - "cursor": {CURSOR_FIELD: "2022-01-31"}, - } - ]} + assert stream_instance.state == { + "states": [ + { + "partition": {"partition_field": "1"}, + "cursor": {CURSOR_FIELD: "2022-01-31"}, + } + ] + } diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_filters.py b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_filters.py index 34d3f9c8cdd1..65e22a445047 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_filters.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_filters.py @@ -1,9 +1,10 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - +import base64 import hashlib +import pytest from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation interpolation = JinjaInterpolation() @@ -48,3 +49,32 @@ def test_hash_md5_with_salt(): hashlib_computed_hash = hash_obj.hexdigest() assert filter_hash == hashlib_computed_hash + + +@pytest.mark.parametrize( + "input_string", + ["test_input_client_id", "some_client_secret_1", "12345", "775.78"], +) +def test_base64encode(input_string: str): + s = "{{ '%s' | base64encode }}" % input_string + filter_base64encode = interpolation.eval(s, config={}) + + # compute expected base64encode calling base64 library directly + base64_obj = base64.b64encode(input_string.encode("utf-8")).decode() + + assert filter_base64encode == base64_obj + + +@pytest.mark.parametrize( + "input_string, expected_string", + [ + ("aW5wdXRfc3RyaW5n", "input_string"), + ("YWlyYnl0ZQ==", "airbyte"), + ("cGFzc3dvcmQ=", "password"), + ], +) +def test_base64decode(input_string: str, expected_string: str): + s = "{{ '%s' | base64decode }}" % input_string + filter_base64decode = interpolation.eval(s, config={}) + + assert filter_base64decode == expected_string diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_interpolated_nested_mapping.py b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_interpolated_nested_mapping.py index f94b07907946..cb0476c7a3ca 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_interpolated_nested_mapping.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_interpolated_nested_mapping.py @@ -22,19 +22,19 @@ ) def test(test_name, path, expected_value): d = { - "nested": { - "field": "value", - "number": 100, - "nested_array": [ - {"{{ parameters.k }}": "VALUE"}, - {"value": "{{ config['num_value'] | int + 2 }}"}, - {"value": "{{ True }}"}, - ], - "config_value": "{{ config['c'] }}", - "parameters_value": "{{ parameters['b'] }}", - "kwargs_value": "{{ kwargs['a'] }}", - } + "nested": { + "field": "value", + "number": 100, + "nested_array": [ + {"{{ parameters.k }}": "VALUE"}, + {"value": "{{ config['num_value'] | int + 2 }}"}, + {"value": "{{ True }}"}, + ], + "config_value": "{{ config['c'] }}", + "parameters_value": "{{ parameters['b'] }}", + "kwargs_value": "{{ kwargs['a'] }}", } + } config = {"c": "VALUE_FROM_CONFIG", "num_value": 3} kwargs = {"a": "VALUE_FROM_KWARGS"} diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_jinja.py b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_jinja.py index 34105f99f497..097afbb3487f 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_jinja.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_jinja.py @@ -19,6 +19,20 @@ def test_get_value_from_config(): assert val == "2022-01-01" +@pytest.mark.parametrize( + "valid_types, expected_value", + [ + pytest.param((str,), "1234J", id="test_value_is_a_string_if_valid_types_is_str"), + pytest.param(None, 1234j, id="test_value_is_interpreted_as_complex_number_by_default"), + ], +) +def test_get_value_with_complex_number(valid_types, expected_value): + s = "{{ config['value'] }}" + config = {"value": "1234J"} + val = interpolation.eval(s, config, valid_types=valid_types) + assert val == expected_value + + def test_get_value_from_stream_slice(): s = "{{ stream_slice['date'] }}" config = {"date": "2022-01-01"} @@ -193,25 +207,32 @@ def test_undeclared_variables(template_string, expected_error, expected_value): @freeze_time("2021-09-01") -@pytest.mark.parametrize("template_string, expected_value",[ - pytest.param("{{ now_utc() }}", "2021-09-01 00:00:00+00:00", id="test_now_utc"), - pytest.param("{{ now_utc().strftime('%Y-%m-%d') }}", "2021-09-01", id="test_now_utc_strftime"), - pytest.param("{{ today_utc() }}", "2021-09-01", id="test_today_utc"), - pytest.param("{{ today_utc().strftime('%Y/%m/%d') }}", "2021/09/01", id="test_todat_utc_stftime"), - pytest.param("{{ timestamp(1646006400) }}", 1646006400, id="test_timestamp_from_timestamp"), - pytest.param("{{ timestamp('2022-02-28') }}", 1646006400, id="test_timestamp_from_timestamp"), - pytest.param("{{ timestamp('2022-02-28T00:00:00Z') }}", 1646006400, id="test_timestamp_from_timestamp"), - pytest.param("{{ timestamp('2022-02-28 00:00:00Z') }}", 1646006400, id="test_timestamp_from_timestamp"), - pytest.param("{{ timestamp('2022-02-28T00:00:00-08:00') }}", 1646035200, id="test_timestamp_from_date_with_tz"), - pytest.param("{{ max(2, 3) }}", 3, id="test_max_with_arguments"), - pytest.param("{{ max([2, 3]) }}", 3, id="test_max_with_list"), - pytest.param("{{ day_delta(1) }}", "2021-09-02T00:00:00.000000+0000", id="test_day_delta"), - pytest.param("{{ day_delta(-1) }}", "2021-08-31T00:00:00.000000+0000", id="test_day_delta_negative"), - pytest.param("{{ day_delta(1, format='%Y-%m-%d') }}", "2021-09-02", id="test_day_delta_with_format"), - pytest.param("{{ duration('P1D') }}", "1 day, 0:00:00", id="test_duration_one_day"), - pytest.param("{{ duration('P6DT23H') }}", "6 days, 23:00:00", id="test_duration_six_days_and_23_hours"), - pytest.param("{{ (now_utc() - duration('P1D')).strftime('%Y-%m-%dT%H:%M:%SZ') }}", "2021-08-31T00:00:00Z", id="test_now_utc_with_duration_and_format"), -]) +@pytest.mark.parametrize( + "template_string, expected_value", + [ + pytest.param("{{ now_utc() }}", "2021-09-01 00:00:00+00:00", id="test_now_utc"), + pytest.param("{{ now_utc().strftime('%Y-%m-%d') }}", "2021-09-01", id="test_now_utc_strftime"), + pytest.param("{{ today_utc() }}", "2021-09-01", id="test_today_utc"), + pytest.param("{{ today_utc().strftime('%Y/%m/%d') }}", "2021/09/01", id="test_todat_utc_stftime"), + pytest.param("{{ timestamp(1646006400) }}", 1646006400, id="test_timestamp_from_timestamp"), + pytest.param("{{ timestamp('2022-02-28') }}", 1646006400, id="test_timestamp_from_timestamp"), + pytest.param("{{ timestamp('2022-02-28T00:00:00Z') }}", 1646006400, id="test_timestamp_from_timestamp"), + pytest.param("{{ timestamp('2022-02-28 00:00:00Z') }}", 1646006400, id="test_timestamp_from_timestamp"), + pytest.param("{{ timestamp('2022-02-28T00:00:00-08:00') }}", 1646035200, id="test_timestamp_from_date_with_tz"), + pytest.param("{{ max(2, 3) }}", 3, id="test_max_with_arguments"), + pytest.param("{{ max([2, 3]) }}", 3, id="test_max_with_list"), + pytest.param("{{ day_delta(1) }}", "2021-09-02T00:00:00.000000+0000", id="test_day_delta"), + pytest.param("{{ day_delta(-1) }}", "2021-08-31T00:00:00.000000+0000", id="test_day_delta_negative"), + pytest.param("{{ day_delta(1, format='%Y-%m-%d') }}", "2021-09-02", id="test_day_delta_with_format"), + pytest.param("{{ duration('P1D') }}", "1 day, 0:00:00", id="test_duration_one_day"), + pytest.param("{{ duration('P6DT23H') }}", "6 days, 23:00:00", id="test_duration_six_days_and_23_hours"), + pytest.param( + "{{ (now_utc() - duration('P1D')).strftime('%Y-%m-%dT%H:%M:%SZ') }}", + "2021-08-31T00:00:00Z", + id="test_now_utc_with_duration_and_format", + ), + ], +) def test_macros_examples(template_string, expected_value): # The outputs of this test are referenced in declarative_component_schema.yaml # If you change the expected output, you must also change the expected output in declarative_component_schema.yaml diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_macros.py b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_macros.py index 0a19c9ab4f86..bfd1fbc137d0 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_macros.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_macros.py @@ -27,26 +27,27 @@ def test_macros_export(test_name, fn_name, found_in_macros): assert fn_name not in macros -@pytest.mark.parametrize("test_name, input_value, format, expected_output", [ - ("test_datetime_string_to_date", "2022-01-01T01:01:01Z", "%Y-%m-%d", "2022-01-01"), - ("test_date_string_to_date", "2022-01-01", "%Y-%m-%d", "2022-01-01"), - ("test_datetime_string_to_date", "2022-01-01T00:00:00Z", "%Y-%m-%d", "2022-01-01"), - ("test_datetime_with_tz_string_to_date", "2022-01-01T00:00:00Z", "%Y-%m-%d", "2022-01-01"), - ("test_datetime_string_to_datetime", "2022-01-01T01:01:01Z", "%Y-%m-%dT%H:%M:%SZ", "2022-01-01T01:01:01Z"), - ("test_datetime_string_with_tz_to_datetime", "2022-01-01T01:01:01-0800", "%Y-%m-%dT%H:%M:%SZ", "2022-01-01T09:01:01Z"), - ("test_datetime_object_tz_to_date", datetime.datetime(2022, 1, 1, 1, 1, 1), "%Y-%m-%d", "2022-01-01"), - ("test_datetime_object_tz_to_datetime", datetime.datetime(2022, 1, 1, 1, 1, 1), "%Y-%m-%dT%H:%M:%SZ", "2022-01-01T01:01:01Z"), -]) +@pytest.mark.parametrize( + "test_name, input_value, format, expected_output", + [ + ("test_datetime_string_to_date", "2022-01-01T01:01:01Z", "%Y-%m-%d", "2022-01-01"), + ("test_date_string_to_date", "2022-01-01", "%Y-%m-%d", "2022-01-01"), + ("test_datetime_string_to_date", "2022-01-01T00:00:00Z", "%Y-%m-%d", "2022-01-01"), + ("test_datetime_with_tz_string_to_date", "2022-01-01T00:00:00Z", "%Y-%m-%d", "2022-01-01"), + ("test_datetime_string_to_datetime", "2022-01-01T01:01:01Z", "%Y-%m-%dT%H:%M:%SZ", "2022-01-01T01:01:01Z"), + ("test_datetime_string_with_tz_to_datetime", "2022-01-01T01:01:01-0800", "%Y-%m-%dT%H:%M:%SZ", "2022-01-01T09:01:01Z"), + ("test_datetime_object_tz_to_date", datetime.datetime(2022, 1, 1, 1, 1, 1), "%Y-%m-%d", "2022-01-01"), + ("test_datetime_object_tz_to_datetime", datetime.datetime(2022, 1, 1, 1, 1, 1), "%Y-%m-%dT%H:%M:%SZ", "2022-01-01T01:01:01Z"), + ], +) def test_format_datetime(test_name, input_value, format, expected_output): format_datetime = macros["format_datetime"] assert format_datetime(input_value, format) == expected_output @pytest.mark.parametrize( - "test_name, input_value, expected_output", [ - ("test_one_day", "P1D", datetime.timedelta(days=1)), - ("test_6_days_23_hours", "P6DT23H", datetime.timedelta(days=6, hours=23)) - ] + "test_name, input_value, expected_output", + [("test_one_day", "P1D", datetime.timedelta(days=1)), ("test_6_days_23_hours", "P6DT23H", datetime.timedelta(days=6, hours=23))], ) def test_duration(test_name, input_value, expected_output): duration_fn = macros["duration"] @@ -54,7 +55,8 @@ def test_duration(test_name, input_value, expected_output): @pytest.mark.parametrize( - "test_name, input_value, expected_output", [ + "test_name, input_value, expected_output", + [ ("test_int_input", 1646006400, 1646006400), ("test_float_input", 100.0, 100), ("test_float_input_is_floored", 100.9, 100), @@ -63,7 +65,7 @@ def test_duration(test_name, input_value, expected_output): ("test_string_datetime_midnight_iso8601_with_tz", "2022-02-28T00:00:00-08:00", 1646035200), ("test_string_datetime_midnight_iso8601_no_t", "2022-02-28 00:00:00Z", 1646006400), ("test_string_datetime_iso8601", "2022-02-28T10:11:12", 1646043072), - ] + ], ) def test_timestamp(test_name, input_value, expected_output): timestamp_function = macros["timestamp"] diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index 7d9fb6bbd0ac..08cea962086e 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -37,6 +37,9 @@ from airbyte_cdk.sources.declarative.models import SimpleRetriever as SimpleRetrieverModel from airbyte_cdk.sources.declarative.models import Spec as SpecModel from airbyte_cdk.sources.declarative.models import SubstreamPartitionRouter as SubstreamPartitionRouterModel +from airbyte_cdk.sources.declarative.models.declarative_component_schema import OffsetIncrement as OffsetIncrementModel +from airbyte_cdk.sources.declarative.models.declarative_component_schema import PageIncrement as PageIncrementModel +from airbyte_cdk.sources.declarative.models.declarative_component_schema import SelectiveAuthenticator from airbyte_cdk.sources.declarative.parsers.manifest_component_transformer import ManifestComponentTransformer from airbyte_cdk.sources.declarative.parsers.manifest_reference_resolver import ManifestReferenceResolver from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import ModelToComponentFactory @@ -242,7 +245,7 @@ def test_full_config_stream(): assert stream.retriever.paginator.pagination_strategy.page_size == 10 assert isinstance(stream.retriever.requester, HttpRequester) - assert stream.retriever.requester._http_method == HttpMethod.GET + assert stream.retriever.requester.http_method == HttpMethod.GET assert stream.retriever.requester.name == stream.name assert stream.retriever.requester._path.string == "{{ next_page_token['next_page_url'] }}" assert stream.retriever.requester._path.default == "{{ next_page_token['next_page_url'] }}" @@ -300,16 +303,46 @@ def test_interpolate_config(): ) assert isinstance(authenticator, DeclarativeOauth2Authenticator) - assert authenticator.client_id.eval(input_config) == "some_client_id" - assert authenticator.client_secret.string == "some_client_secret" - assert authenticator.token_refresh_endpoint.eval(input_config) == "https://api.sendgrid.com/v3/auth" - assert authenticator.refresh_token.eval(input_config) == "verysecrettoken" + assert authenticator._client_id.eval(input_config) == "some_client_id" + assert authenticator._client_secret.string == "some_client_secret" + assert authenticator._token_refresh_endpoint.eval(input_config) == "https://api.sendgrid.com/v3/auth" + assert authenticator._refresh_token.eval(input_config) == "verysecrettoken" assert authenticator._refresh_request_body.mapping == {"body_field": "yoyoyo", "interpolated_body_field": "{{ config['apikey'] }}"} assert authenticator.get_refresh_request_body() == {"body_field": "yoyoyo", "interpolated_body_field": "verysecrettoken"} +def test_interpolate_config_with_token_expiry_date_format(): + content = """ + authenticator: + type: OAuthAuthenticator + client_id: "some_client_id" + client_secret: "some_client_secret" + token_refresh_endpoint: "https://api.sendgrid.com/v3/auth" + refresh_token: "{{ config['apikey'] }}" + token_expiry_date_format: "%Y-%m-%d %H:%M:%S.%f+00:00" + """ + parsed_manifest = YamlDeclarativeSource._parse(content) + resolved_manifest = resolver.preprocess_manifest(parsed_manifest) + authenticator_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["authenticator"], {}) + + authenticator = factory.create_component( + model_type=OAuthAuthenticatorModel, component_definition=authenticator_manifest, config=input_config + ) + + assert isinstance(authenticator, DeclarativeOauth2Authenticator) + assert authenticator.token_expiry_date_format == "%Y-%m-%d %H:%M:%S.%f+00:00" + assert authenticator.token_expiry_is_time_of_expiration + assert authenticator._client_id.eval(input_config) == "some_client_id" + assert authenticator._client_secret.string == "some_client_secret" + assert authenticator._token_refresh_endpoint.eval(input_config) == "https://api.sendgrid.com/v3/auth" + + def test_single_use_oauth_branch(): - single_use_input_config = {"apikey": "verysecrettoken", "repos": ["airbyte", "airbyte-cloud"], "credentials": {"access_token": "access_token", "token_expiry_date": "1970-01-01"}} + single_use_input_config = { + "apikey": "verysecrettoken", + "repos": ["airbyte", "airbyte-cloud"], + "credentials": {"access_token": "access_token", "token_expiry_date": "1970-01-01"}, + } content = """ authenticator: @@ -684,9 +717,7 @@ def test_given_data_feed_and_incremental_then_raise_error(): with pytest.raises(ValueError): factory.create_component( - model_type=DatetimeBasedCursorModel, - component_definition=datetime_based_cursor_definition, - config=input_config + model_type=DatetimeBasedCursorModel, component_definition=datetime_based_cursor_definition, config=input_config ) @@ -713,7 +744,9 @@ def test_create_record_selector(test_name, record_selector, expected_runtime_sel resolved_manifest = resolver.preprocess_manifest(parsed_manifest) selector_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["selector"], {}) - selector = factory.create_component(model_type=RecordSelectorModel, component_definition=selector_manifest, transformations=[], config=input_config) + selector = factory.create_component( + model_type=RecordSelectorModel, component_definition=selector_manifest, transformations=[], config=input_config + ) assert isinstance(selector, RecordSelector) assert isinstance(selector.extractor, DpathExtractor) @@ -796,7 +829,7 @@ def test_create_requester(test_name, error_handler, expected_backoff_strategy_ty ) assert isinstance(selector, HttpRequester) - assert selector._http_method == HttpMethod.GET + assert selector.http_method == HttpMethod.GET assert selector.name == "name" assert selector._path.string == "/v3/marketing/lists" assert selector._url_base.string == "https://api.sendgrid.com" @@ -898,7 +931,54 @@ def test_create_request_with_session_authenticator(): assert isinstance(selector.authenticator.token_provider.login_requester, HttpRequester) assert selector.authenticator.token_provider.session_token_path == ["id"] assert selector.authenticator.token_provider.login_requester._url_base.eval(input_config) == "https://api.sendgrid.com" - assert selector.authenticator.token_provider.login_requester.get_request_body_json() == {"username": "lists", "password": "verysecrettoken"} + assert selector.authenticator.token_provider.login_requester.get_request_body_json() == { + "username": "lists", + "password": "verysecrettoken", + } + + +@pytest.mark.parametrize("input_config, expected_authenticator_class", [ + pytest.param( + {"auth": {"type": "token"}, "credentials": {"api_key": "some_key"}}, + ApiKeyAuthenticator, + id="test_create_requester_with_selective_authenticator_and_token_selected", + ), + pytest.param( + {"auth": {"type": "oauth"}, "credentials": {"client_id": "ABC"}}, + DeclarativeOauth2Authenticator, + id="test_create_requester_with_selective_authenticator_and_oauth_selected", + ), +] +) +def test_create_requester_with_selective_authenticator(input_config, expected_authenticator_class): + content = """ +authenticator: + type: SelectiveAuthenticator + authenticator_selection_path: + - auth + - type + authenticators: + token: + type: ApiKeyAuthenticator + header: "Authorization" + api_token: "api_key={{ config['credentials']['api_key'] }}" + oauth: + type: OAuthAuthenticator + token_refresh_endpoint: https://api.url.com + client_id: "{{ config['credentials']['client_id'] }}" + client_secret: some_secret + refresh_token: some_token + """ + name = "name" + parsed_manifest = YamlDeclarativeSource._parse(content) + resolved_manifest = resolver.preprocess_manifest(parsed_manifest) + authenticator_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["authenticator"], {}) + + authenticator = factory.create_component( + model_type=SelectiveAuthenticator, component_definition=authenticator_manifest, config=input_config, name=name + ) + + assert isinstance(authenticator, expected_authenticator_class) def test_create_composite_error_handler(): @@ -995,7 +1075,7 @@ def test_config_with_defaults(): assert stream.schema_loader.file_path.default == "./source_sendgrid/schemas/{{ parameters.name }}.yaml" assert isinstance(stream.retriever.requester, HttpRequester) - assert stream.retriever.requester._http_method == HttpMethod.GET + assert stream.retriever.requester.http_method == HttpMethod.GET assert isinstance(stream.retriever.requester.authenticator, BearerAuthenticator) assert stream.retriever.requester.authenticator.token_provider.get_token() == "verysecrettoken" @@ -1329,7 +1409,7 @@ def test_remove_fields(self): expected = [RemoveFields(field_pointers=[["path", "to", "field1"], ["path2"]], parameters={})] assert stream.retriever.record_selector.transformations == expected - def test_add_fields(self): + def test_add_fields_no_value_type(self): content = f""" the_stream: type: DeclarativeStream @@ -1341,26 +1421,142 @@ def test_add_fields(self): - path: ["field1"] value: "static_value" """ - parsed_manifest = YamlDeclarativeSource._parse(content) - resolved_manifest = resolver.preprocess_manifest(parsed_manifest) - resolved_manifest["type"] = "DeclarativeSource" - stream_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["the_stream"], {}) - - stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config) + expected = [ + AddFields( + fields=[ + AddedFieldDefinition( + path=["field1"], + value=InterpolatedString(string="static_value", default="static_value", parameters={}), + value_type=None, + parameters={}, + ) + ], + parameters={}, + ) + ] + self._test_add_fields(content, expected) - assert isinstance(stream, DeclarativeStream) + def test_add_fields_value_type_is_string(self): + content = f""" + the_stream: + type: DeclarativeStream + $parameters: + {self.base_parameters} + transformations: + - type: AddFields + fields: + - path: ["field1"] + value: "static_value" + value_type: string + """ expected = [ AddFields( fields=[ AddedFieldDefinition( path=["field1"], value=InterpolatedString(string="static_value", default="static_value", parameters={}), + value_type=str, + parameters={}, + ) + ], + parameters={}, + ) + ] + self._test_add_fields(content, expected) + + def test_add_fields_value_type_is_number(self): + content = f""" + the_stream: + type: DeclarativeStream + $parameters: + {self.base_parameters} + transformations: + - type: AddFields + fields: + - path: ["field1"] + value: "1" + value_type: number + """ + expected = [ + AddFields( + fields=[ + AddedFieldDefinition( + path=["field1"], + value=InterpolatedString(string="1", default="1", parameters={}), + value_type=float, + parameters={}, + ) + ], + parameters={}, + ) + ] + self._test_add_fields(content, expected) + + def test_add_fields_value_type_is_integer(self): + content = f""" + the_stream: + type: DeclarativeStream + $parameters: + {self.base_parameters} + transformations: + - type: AddFields + fields: + - path: ["field1"] + value: "1" + value_type: integer + """ + expected = [ + AddFields( + fields=[ + AddedFieldDefinition( + path=["field1"], + value=InterpolatedString(string="1", default="1", parameters={}), + value_type=int, + parameters={}, + ) + ], + parameters={}, + ) + ] + self._test_add_fields(content, expected) + + def test_add_fields_value_type_is_boolean(self): + content = f""" + the_stream: + type: DeclarativeStream + $parameters: + {self.base_parameters} + transformations: + - type: AddFields + fields: + - path: ["field1"] + value: False + value_type: boolean + """ + expected = [ + AddFields( + fields=[ + AddedFieldDefinition( + path=["field1"], + value=InterpolatedString(string="False", default="False", parameters={}), + value_type=bool, parameters={}, ) ], parameters={}, ) ] + self._test_add_fields(content, expected) + + def _test_add_fields(self, content, expected): + parsed_manifest = YamlDeclarativeSource._parse(content) + resolved_manifest = resolver.preprocess_manifest(parsed_manifest) + resolved_manifest["type"] = "DeclarativeSource" + stream_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["the_stream"], {}) + + stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config=input_config) + + assert isinstance(stream, DeclarativeStream) assert stream.retriever.record_selector.transformations == expected def test_default_schema_loader(self): @@ -1542,7 +1738,10 @@ def test_simple_retriever_emit_log_messages(): def test_ignore_retry(): requester_model = { - "type": "HttpRequester", "name": "list", "url_base": "orange.com", "path": "/v1/api", + "type": "HttpRequester", + "name": "list", + "url_base": "orange.com", + "path": "/v1/api", } connector_builder_factory = ModelToComponentFactory(disable_retries=True) @@ -1554,3 +1753,34 @@ def test_ignore_retry(): ) assert requester.max_retries == 0 + + +def test_create_page_increment(): + model = PageIncrementModel( + type="PageIncrement", + page_size=10, + start_from_page=1, + inject_on_first_request=True, + ) + expected_strategy = PageIncrement(page_size=10, start_from_page=1, inject_on_first_request=True, parameters={}) + + strategy = factory.create_page_increment(model, input_config) + + assert strategy.page_size == expected_strategy.page_size + assert strategy.start_from_page == expected_strategy.start_from_page + assert strategy.inject_on_first_request == expected_strategy.inject_on_first_request + + +def test_create_offset_increment(): + model = OffsetIncrementModel( + type="OffsetIncrement", + page_size=10, + inject_on_first_request=True, + ) + expected_strategy = OffsetIncrement(page_size=10, inject_on_first_request=True, parameters={}, config=input_config) + + strategy = factory.create_offset_increment(model, input_config) + + assert strategy.page_size == expected_strategy.page_size + assert strategy.inject_on_first_request == expected_strategy.inject_on_first_request + assert strategy.config == input_config diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_list_partition_router.py b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_list_partition_router.py index c8c8ef1dd6eb..ce1b93b9d75b 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_list_partition_router.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_list_partition_router.py @@ -78,7 +78,9 @@ def test_list_partition_router(test_name, partition_values, cursor_field, expect ], ) def test_request_option(test_name, request_option, expected_req_params, expected_headers, expected_body_json, expected_body_data): - partition_router = ListPartitionRouter(values=partition_values, cursor_field=cursor_field, config={}, request_option=request_option, parameters={}) + partition_router = ListPartitionRouter( + values=partition_values, cursor_field=cursor_field, config={}, request_option=request_option, parameters={} + ) stream_slice = {cursor_field: "customer"} assert expected_req_params == partition_router.get_request_params(stream_slice=stream_slice) @@ -89,7 +91,9 @@ def test_request_option(test_name, request_option, expected_req_params, expected def test_request_option_before_updating_cursor(): request_option = RequestOption(inject_into=RequestOptionType.request_parameter, parameters={}, field_name="owner_resource") - partition_router = ListPartitionRouter(values=partition_values, cursor_field=cursor_field, config={}, request_option=request_option, parameters={}) + partition_router = ListPartitionRouter( + values=partition_values, cursor_field=cursor_field, config={}, request_option=request_option, parameters={} + ) stream_slice = {cursor_field: "customer"} assert {} == partition_router.get_request_params(stream_slice) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py index f8c7f0cb332b..e677666f46eb 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py @@ -64,7 +64,11 @@ def read_records( "test_single_parent_slices_no_records", [ ParentStreamConfig( - stream=MockStream([{}], [], "first_stream"), parent_key="id", partition_field="first_stream_id", parameters={}, config={} + stream=MockStream([{}], [], "first_stream"), + parent_key="id", + partition_field="first_stream_id", + parameters={}, + config={}, ) ], [], @@ -136,7 +140,11 @@ def read_records( config={}, ) ], - [{"first_stream_id": 0, "parent_slice": {}}, {"first_stream_id": 1, "parent_slice": {}}, {"first_stream_id": 3, "parent_slice": {}}], + [ + {"first_stream_id": 0, "parent_slice": {}}, + {"first_stream_id": 1, "parent_slice": {}}, + {"first_stream_id": 3, "parent_slice": {}}, + ], ), ( "test_dpath_extraction", @@ -149,7 +157,11 @@ def read_records( config={}, ) ], - [{"first_stream_id": 0, "parent_slice": {}}, {"first_stream_id": 1, "parent_slice": {}}, {"first_stream_id": 3, "parent_slice": {}}], + [ + {"first_stream_id": 0, "parent_slice": {}}, + {"first_stream_id": 1, "parent_slice": {}}, + {"first_stream_id": 3, "parent_slice": {}}, + ], ), ], ) @@ -268,15 +280,23 @@ def test_given_record_is_airbyte_message_when_stream_slices_then_use_record_data partition_router = SubstreamPartitionRouter( parent_stream_configs=[ ParentStreamConfig( - stream=MockStream([parent_slice], [AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(data={"id": "record value"}, emitted_at=0, stream="stream"))], "first_stream"), + stream=MockStream( + [parent_slice], + [ + AirbyteMessage( + type=Type.RECORD, record=AirbyteRecordMessage(data={"id": "record value"}, emitted_at=0, stream="stream") + ) + ], + "first_stream", + ), parent_key="id", partition_field="partition_field", parameters={}, - config={} + config={}, ) ], parameters={}, - config={} + config={}, ) slices = list(partition_router.stream_slices()) @@ -292,11 +312,11 @@ def test_given_record_is_record_object_when_stream_slices_then_use_record_data() parent_key="id", partition_field="partition_field", parameters={}, - config={} + config={}, ) ], parameters={}, - config={} + config={}, ) slices = list(partition_router.stream_slices()) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py index d57beee7749d..c5943e94d180 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py @@ -116,39 +116,59 @@ def test_error_handler_compatibility_simple(): status_code = 403 expected_action = ResponseAction.IGNORE response_mock = create_response(status_code) - default_error_handler = DefaultErrorHandler(config={}, parameters={}, response_filters=[ - HttpResponseFilter(action=ResponseAction.IGNORE, http_codes={403}, config={}, parameters={})]) - composite_error_handler = CompositeErrorHandler(error_handlers=[ - DefaultErrorHandler( - response_filters=[HttpResponseFilter(action=ResponseAction.IGNORE, http_codes={403}, parameters={}, config={})], - parameters={}, config={}) - ], parameters={}) + default_error_handler = DefaultErrorHandler( + config={}, + parameters={}, + response_filters=[HttpResponseFilter(action=ResponseAction.IGNORE, http_codes={403}, config={}, parameters={})], + ) + composite_error_handler = CompositeErrorHandler( + error_handlers=[ + DefaultErrorHandler( + response_filters=[HttpResponseFilter(action=ResponseAction.IGNORE, http_codes={403}, parameters={}, config={})], + parameters={}, + config={}, + ) + ], + parameters={}, + ) assert default_error_handler.interpret_response(response_mock).action == expected_action assert composite_error_handler.interpret_response(response_mock).action == expected_action -@pytest.mark.parametrize("test_name, status_code, expected_action", [ - ("test_first_filter", 403, ResponseAction.IGNORE), - ("test_second_filter", 404, ResponseAction.FAIL) -]) +@pytest.mark.parametrize( + "test_name, status_code, expected_action", + [("test_first_filter", 403, ResponseAction.IGNORE), ("test_second_filter", 404, ResponseAction.FAIL)], +) def test_error_handler_compatibility_multiple_filters(test_name, status_code, expected_action): response_mock = create_response(status_code) - error_handler_with_multiple_filters = CompositeErrorHandler(error_handlers=[ - DefaultErrorHandler( - response_filters=[HttpResponseFilter(action=ResponseAction.IGNORE, http_codes={403}, parameters={}, config={}), - HttpResponseFilter(action=ResponseAction.FAIL, http_codes={404}, parameters={}, config={}) - ], parameters={}, config={}), - ], parameters={}) - composite_error_handler_with_single_filters = CompositeErrorHandler(error_handlers=[ - DefaultErrorHandler( - response_filters=[HttpResponseFilter(action=ResponseAction.IGNORE, http_codes={403}, parameters={}, config={})], - parameters={}, config={}), - DefaultErrorHandler( - response_filters=[ - HttpResponseFilter(action=ResponseAction.FAIL, http_codes={404}, parameters={}, config={}) - ], - parameters={}, config={}), - ], parameters={}) + error_handler_with_multiple_filters = CompositeErrorHandler( + error_handlers=[ + DefaultErrorHandler( + response_filters=[ + HttpResponseFilter(action=ResponseAction.IGNORE, http_codes={403}, parameters={}, config={}), + HttpResponseFilter(action=ResponseAction.FAIL, http_codes={404}, parameters={}, config={}), + ], + parameters={}, + config={}, + ), + ], + parameters={}, + ) + composite_error_handler_with_single_filters = CompositeErrorHandler( + error_handlers=[ + DefaultErrorHandler( + response_filters=[HttpResponseFilter(action=ResponseAction.IGNORE, http_codes={403}, parameters={}, config={})], + parameters={}, + config={}, + ), + DefaultErrorHandler( + response_filters=[HttpResponseFilter(action=ResponseAction.FAIL, http_codes={404}, parameters={}, config={})], + parameters={}, + config={}, + ), + ], + parameters={}, + ) actual_action_multiple_filters = error_handler_with_multiple_filters.interpret_response(response_mock).action assert actual_action_multiple_filters == expected_action @@ -166,3 +186,28 @@ def create_response(status_code: int, headers=None, json_body=None): response_mock.headers = headers or {} response_mock.json.return_value = json_body or {} return response_mock + + +@pytest.mark.parametrize( + "test_name, max_times, expected_max_time", + [ + ("test_single_handler", [10], 10), + ("test_multiple_handlers", [10, 15], 15), + ], +) +def test_max_time_is_max_of_underlying_handlers(test_name, max_times, expected_max_time): + composite_error_handler = CompositeErrorHandler( + error_handlers=[ + DefaultErrorHandler( + response_filters=[HttpResponseFilter(action=ResponseAction.IGNORE, http_codes={403}, parameters={}, config={})], + max_time=max_time, + parameters={}, + config={}, + ) + for max_time in max_times + ], + parameters={}, + ) + + max_time = composite_error_handler.max_time + assert max_time == expected_max_time diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py index 8a53bc615d74..17ef223171c3 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py @@ -179,12 +179,19 @@ def test_page_size_option_cannot_be_set_if_strategy_has_no_limit(): pass -def test_reset(): +@pytest.mark.parametrize( + "test_name, inject_on_first_request", + [ + pytest.param("test_reset_inject_on_first_request", True), + pytest.param("test_reset_no_inject_on_first_request", False), + ], +) +def test_reset(test_name, inject_on_first_request): page_size_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="limit", parameters={}) page_token_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="offset", parameters={}) url_base = "https://airbyte.io" config = {} - strategy = OffsetIncrement(config={}, page_size=2, parameters={}) + strategy = OffsetIncrement(config={}, page_size=2, inject_on_first_request=inject_on_first_request, parameters={}) paginator = DefaultPaginator( strategy, config, url_base, parameters={}, page_size_option=page_size_request_option, page_token_option=page_token_request_option ) @@ -197,6 +204,20 @@ def test_reset(): assert request_parameters_for_second_request != request_parameters_after_reset +def test_initial_token_with_offset_pagination(): + page_size_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="limit", parameters={}) + page_token_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="offset", parameters={}) + url_base = "https://airbyte.io" + config = {} + strategy = OffsetIncrement(config={}, page_size=2, parameters={}, inject_on_first_request=True) + paginator = DefaultPaginator( + strategy, config, url_base, parameters={}, page_size_option=page_size_request_option, page_token_option=page_token_request_option + ) + initial_request_parameters = paginator.get_request_params() + + assert initial_request_parameters == {"limit": 2, "offset": 0} + + def test_limit_page_fetched(): maximum_number_of_pages = 5 number_of_next_performed = maximum_number_of_pages - 1 diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py index 0fb16077d236..37f26a2af420 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_offset_increment.py @@ -3,6 +3,7 @@ # import json +from typing import Any, Optional import pytest import requests @@ -42,3 +43,16 @@ def test_offset_increment_paginator_strategy_rises(): with pytest.raises(Exception) as exc: paginator_strategy.get_page_size() assert str(exc.value) == "invalid value is of type . Expected " + + +@pytest.mark.parametrize( + "inject_on_first_request, expected_initial_token", + [ + pytest.param(True, 0, id="test_with_inject_offset"), + pytest.param(False, None, id="test_without_inject_offset"), + ], +) +def test_offset_increment_paginator_strategy_initial_token(inject_on_first_request: bool, expected_initial_token: Optional[Any]): + paginator_strategy = OffsetIncrement(page_size=20, parameters={}, config={}, inject_on_first_request=inject_on_first_request) + + assert paginator_strategy.initial_token == expected_initial_token diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py index 5603c10dacc2..52477fedc21e 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py @@ -3,6 +3,7 @@ # import json +from typing import Any, Optional import pytest import requests @@ -35,3 +36,21 @@ def test_page_increment_paginator_strategy(page_size, start_from, last_records, paginator_strategy.reset() assert start_from == paginator_strategy._page + + +@pytest.mark.parametrize( + "inject_on_first_request, start_from_page, expected_initial_token", + [ + pytest.param(True, 0, 0, id="test_with_inject_offset_page_start_from_0"), + pytest.param(True, 12, 12, id="test_with_inject_offset_page_start_from_12"), + pytest.param(False, 2, None, id="test_without_inject_offset"), + ], +) +def test_page_increment_paginator_strategy_initial_token( + inject_on_first_request: bool, start_from_page: int, expected_initial_token: Optional[Any] +): + paginator_strategy = PageIncrement( + page_size=20, parameters={}, start_from_page=start_from_page, inject_on_first_request=inject_on_first_request + ) + + assert paginator_strategy.initial_token == expected_initial_token diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py index 41d71e75623f..b2aa1117362d 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_stop_condition.py @@ -55,13 +55,14 @@ def test_given_stop_condition_is_met_when_next_page_token_then_return_none(mocke mocked_stop_condition.is_met.assert_has_calls([call(last_record), call(first_record)]) -def test_given_last_record_meets_condition_when_next_page_token_then_do_not_check_for_other_records(mocked_pagination_strategy, mocked_stop_condition): +def test_given_last_record_meets_condition_when_next_page_token_then_do_not_check_for_other_records( + mocked_pagination_strategy, mocked_stop_condition +): mocked_stop_condition.is_met.return_value = True last_record = Mock(spec=Record) StopConditionPaginationStrategyDecorator(mocked_pagination_strategy, mocked_stop_condition).next_page_token( - ANY_RESPONSE, - [Mock(spec=Record), last_record] + ANY_RESPONSE, [Mock(spec=Record), last_record] ) mocked_stop_condition.is_met.assert_called_once_with(last_record) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py index 218289ee5cad..19759832452b 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py @@ -19,18 +19,18 @@ ("test_static_param", {"a_static_request_param": "a_static_value"}, {"a_static_request_param": "a_static_value"}), ("test_value_depends_on_state", {"read_from_state": "{{ stream_state['date'] }}"}, {"read_from_state": "2021-01-01"}), ("test_value_depends_on_stream_slice", {"read_from_slice": "{{ stream_slice['start_date'] }}"}, {"read_from_slice": "2020-01-01"}), - ("test_value_depends_on_next_page_token", {"read_from_token": "{{ next_page_token['offset'] }}"}, {"read_from_token": 12345}), + ("test_value_depends_on_next_page_token", {"read_from_token": "{{ next_page_token['offset'] }}"}, {"read_from_token": "12345"}), ("test_value_depends_on_config", {"read_from_config": "{{ config['option'] }}"}, {"read_from_config": "OPTION"}), ( "test_parameter_is_interpolated", {"{{ stream_state['date'] }} - {{stream_slice['start_date']}} - {{next_page_token['offset']}} - {{config['option']}}": "ABC"}, {"2021-01-01 - 2020-01-01 - 12345 - OPTION": "ABC"}, ), - ("test_boolean_false_value", {"boolean_false": "{{ False }}"}, {"boolean_false": False}), - ("test_integer_falsy_value", {"integer_falsy": "{{ 0 }}"}, {"integer_falsy": 0}), - ("test_number_falsy_value", {"number_falsy": "{{ 0.0 }}"}, {"number_falsy": 0.0}), + ("test_boolean_false_value", {"boolean_false": "{{ False }}"}, {"boolean_false": "False"}), + ("test_integer_falsy_value", {"integer_falsy": "{{ 0 }}"}, {"integer_falsy": "0"}), + ("test_number_falsy_value", {"number_falsy": "{{ 0.0 }}"}, {"number_falsy": "0.0"}), ("test_string_falsy_value", {"string_falsy": "{{ '' }}"}, {}), - ("test_none_value", {"none_value": "{{ None }}"}, {}), + ("test_none_value", {"none_value": "{{ None }}"}, {"none_value": "None"}), ], ) def test_interpolated_request_params(test_name, input_request_params, expected_request_params): @@ -61,7 +61,11 @@ def test_interpolated_request_params(test_name, input_request_params, expected_r ("test_none_value", {"none_value": "{{ None }}"}, {}), ("test_string", """{"nested": { "key": "{{ config['option'] }}" }}""", {"nested": {"key": "OPTION"}}), ("test_nested_objects", {"nested": {"key": "{{ config['option'] }}"}}, {"nested": {"key": "OPTION"}}), - ("test_nested_objects_interpolated keys", {"nested": {"{{ stream_state['date'] }}": "{{ config['option'] }}"}}, {"nested": {"2021-01-01": "OPTION"}}), + ( + "test_nested_objects_interpolated keys", + {"nested": {"{{ stream_state['date'] }}": "{{ config['option'] }}"}}, + {"nested": {"2021-01-01": "OPTION"}}, + ), ], ) def test_interpolated_request_json(test_name, input_request_json, expected_request_json): diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py index b3a5bc772261..a861e76f2a0c 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py @@ -10,6 +10,7 @@ import pytest as pytest import requests +import requests_cache from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator, NoAuth from airbyte_cdk.sources.declarative.auth.token import BearerAuthenticator from airbyte_cdk.sources.declarative.exceptions import ReadException @@ -17,13 +18,50 @@ from airbyte_cdk.sources.declarative.requesters.error_handlers.default_error_handler import DefaultErrorHandler from airbyte_cdk.sources.declarative.requesters.error_handlers.error_handler import ErrorHandler from airbyte_cdk.sources.declarative.requesters.http_requester import HttpMethod, HttpRequester +from airbyte_cdk.sources.declarative.requesters.request_options import InterpolatedRequestOptionsProvider from airbyte_cdk.sources.declarative.types import Config +from airbyte_cdk.sources.message import MessageRepository from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, RequestBodyException, UserDefinedBackoffException from requests import PreparedRequest +from requests_cache import CachedResponse + + +@pytest.fixture +def http_requester_factory(): + def factory( + name: str = "name", + url_base: str = "https://test_base_url.com", + path: str = "/", + http_method: str = HttpMethod.GET, + request_options_provider: Optional[InterpolatedRequestOptionsProvider] = None, + authenticator: Optional[DeclarativeAuthenticator] = None, + error_handler: Optional[ErrorHandler] = None, + config: Optional[Config] = None, + parameters: Mapping[str, Any] = None, + disable_retries: bool = False, + message_repository: Optional[MessageRepository] = None, + use_cache: bool = False, + ) -> HttpRequester: + return HttpRequester( + name=name, + url_base=url_base, + path=path, + config=config or {}, + parameters=parameters or {}, + authenticator=authenticator, + http_method=http_method, + request_options_provider=request_options_provider, + error_handler=error_handler, + disable_retries=disable_retries, + message_repository=message_repository or MagicMock(), + use_cache=use_cache, + ) + + return factory def test_http_requester(): - http_method = "GET" + http_method = HttpMethod.GET request_options_provider = MagicMock() request_params = {"param": "value"} @@ -68,7 +106,7 @@ def test_http_requester(): assert requester.get_url_base() == "https://airbyte.io/" assert requester.get_path(stream_state={}, stream_slice=stream_slice, next_page_token={}) == "v1/1234" assert requester.get_authenticator() == authenticator - assert requester.get_method() == HttpMethod.GET + assert requester.get_method() == http_method assert requester.get_request_params(stream_state={}, stream_slice=None, next_page_token=None) == request_params assert requester.get_request_body_data(stream_state={}, stream_slice=None, next_page_token=None) == request_body_data assert requester.get_request_body_json(stream_state={}, stream_slice=None, next_page_token=None) == request_body_json @@ -124,7 +162,14 @@ def test_path(test_name, path, expected_path): assert requester.get_path(stream_state={}, stream_slice={}, next_page_token={}) == expected_path -def create_requester(url_base: Optional[str] = None, parameters: Optional[Mapping[str, Any]] = {}, config: Optional[Config] = None, path: Optional[str] = None, authenticator: Optional[DeclarativeAuthenticator] = None, error_handler: Optional[ErrorHandler] = None) -> HttpRequester: +def create_requester( + url_base: Optional[str] = None, + parameters: Optional[Mapping[str, Any]] = {}, + config: Optional[Config] = None, + path: Optional[str] = None, + authenticator: Optional[DeclarativeAuthenticator] = None, + error_handler: Optional[ErrorHandler] = None, +) -> HttpRequester: requester = HttpRequester( name="name", url_base=url_base or "https://example.com", @@ -169,7 +214,16 @@ def test_basic_send_request(): # merging json params from the three sources (None, {"field": "value"}, None, None, None, None, None, '{"field": "value"}'), (None, {"field": "value"}, None, {"field2": "value"}, None, None, None, '{"field": "value", "field2": "value"}'), - (None, {"field": "value"}, None, {"field2": "value"}, None, {"authfield": "val"}, None, '{"field": "value", "field2": "value", "authfield": "val"}'), + ( + None, + {"field": "value"}, + None, + {"field2": "value"}, + None, + {"authfield": "val"}, + None, + '{"field": "value", "field2": "value", "authfield": "val"}', + ), (None, {"field": "value"}, None, {"field": "value"}, None, None, ValueError, None), (None, {"field": "value"}, None, None, None, {"field": "value"}, ValueError, None), # raise on mixed data and json params @@ -178,9 +232,11 @@ def test_basic_send_request(): (None, None, {"field": "value"}, {"field": "value"}, None, None, RequestBodyException, None), (None, None, None, None, {"field": "value"}, {"field": "value"}, RequestBodyException, None), ({"field": "value"}, None, None, None, None, {"field": "value"}, RequestBodyException, None), - - ]) -def test_send_request_data_json(provider_data, provider_json, param_data, param_json, authenticator_data, authenticator_json, expected_exception, expected_body): + ], +) +def test_send_request_data_json( + provider_data, provider_json, param_data, param_json, authenticator_data, authenticator_json, expected_exception, expected_body +): options_provider = MagicMock() options_provider.get_request_body_data.return_value = provider_data options_provider.get_request_body_json.return_value = provider_json @@ -196,7 +252,7 @@ def test_send_request_data_json(provider_data, provider_json, param_data, param_ requester.send_request(request_body_data=param_data, request_body_json=param_json) sent_request: PreparedRequest = requester._session.send.call_args_list[0][0][0] if expected_body is not None: - assert sent_request.body == expected_body.decode('UTF-8') if not isinstance(expected_body, str) else expected_body + assert sent_request.body == expected_body.decode("UTF-8") if not isinstance(expected_body, str) else expected_body @pytest.mark.parametrize( @@ -215,7 +271,7 @@ def test_send_request_data_json(provider_data, provider_json, param_data, param_ ("field=value", {"abc": "def"}, None, ValueError, None), ({"abc": "def"}, "field=value", None, ValueError, None), ("field=value", None, {"abc": "def"}, ValueError, None), - ] + ], ) def test_send_request_string_data(provider_data, param_data, authenticator_data, expected_exception, expected_body): options_provider = MagicMock() @@ -240,15 +296,22 @@ def test_send_request_string_data(provider_data, param_data, authenticator_data, # merging headers from the three sources ({"header": "value"}, None, None, None, {"header": "value"}), ({"header": "value"}, {"header2": "value"}, None, None, {"header": "value", "header2": "value"}), - ({"header": "value"}, {"header2": "value"}, {"authheader": "val"}, None, {"header": "value", "header2": "value", "authheader": "val"}), + ( + {"header": "value"}, + {"header2": "value"}, + {"authheader": "val"}, + None, + {"header": "value", "header2": "value", "authheader": "val"}, + ), # raise on conflicting headers ({"header": "value"}, {"header": "value"}, None, ValueError, None), ({"header": "value"}, None, {"header": "value"}, ValueError, None), ({"header": "value"}, {"header2": "value"}, {"header": "value"}, ValueError, None), - ]) + ], +) def test_send_request_headers(provider_headers, param_headers, authenticator_headers, expected_exception, expected_headers): # headers set by the requests framework, do not validate - default_headers = {'User-Agent': mock.ANY, 'Accept-Encoding': mock.ANY, 'Accept': mock.ANY, 'Connection': mock.ANY} + default_headers = {"User-Agent": mock.ANY, "Accept-Encoding": mock.ANY, "Accept": mock.ANY, "Connection": mock.ANY} options_provider = MagicMock() options_provider.get_request_headers.return_value = provider_headers authenticator = MagicMock() @@ -275,7 +338,8 @@ def test_send_request_headers(provider_headers, param_headers, authenticator_hea ({"param": "value"}, {"param": "value"}, None, ValueError, None), ({"param": "value"}, None, {"param": "value"}, ValueError, None), ({"param": "value"}, {"param2": "value"}, {"param": "value"}, ValueError, None), - ]) + ], +) def test_send_request_params(provider_params, param_params, authenticator_params, expected_exception, expected_params): options_provider = MagicMock() options_provider.get_request_params.return_value = provider_params @@ -294,24 +358,239 @@ def test_send_request_params(provider_params, param_params, authenticator_params assert query_params == expected_params +@pytest.mark.parametrize( + "request_parameters, config, expected_query_params", + [ + pytest.param( + {"k": '{"updatedDateFrom": "2023-08-20T00:00:00Z", "updatedDateTo": "2023-08-20T23:59:59Z"}'}, + {}, + "k=%7B%22updatedDateFrom%22%3A+%222023-08-20T00%3A00%3A00Z%22%2C+%22updatedDateTo%22%3A+%222023-08-20T23%3A59%3A59Z%22%7D", + id="test-request-parameter-dictionary", + ), + pytest.param( + {"k": "1,2"}, + {}, + "k=1%2C2", # k=1,2 + id="test-request-parameter-comma-separated-numbers", + ), + pytest.param( + {"k": "a,b"}, + {}, + "k=a%2Cb", # k=a,b + id="test-request-parameter-comma-separated-strings", + ), + pytest.param( + {"k": '{{ config["k"] }}'}, + {"k": {"updatedDateFrom": "2023-08-20T00:00:00Z", "updatedDateTo": "2023-08-20T23:59:59Z"}}, + # {'updatedDateFrom': '2023-08-20T00:00:00Z', 'updatedDateTo': '2023-08-20T23:59:59Z'} + "k=%7B%27updatedDateFrom%27%3A+%272023-08-20T00%3A00%3A00Z%27%2C+%27updatedDateTo%27%3A+%272023-08-20T23%3A59%3A59Z%27%7D", + id="test-request-parameter-from-config-object", + ), + ], +) +def test_request_param_interpolation(request_parameters, config, expected_query_params): + options_provider = InterpolatedRequestOptionsProvider( + config=config, + request_parameters=request_parameters, + request_body_data={}, + request_headers={}, + parameters={}, + ) + requester = create_requester() + requester._request_options_provider = options_provider + requester.send_request() + sent_request: PreparedRequest = requester._session.send.call_args_list[0][0][0] + assert sent_request.url.split("?", 1)[-1] == expected_query_params + + +@pytest.mark.parametrize( + "request_parameters, config, invalid_value_for_key", + [ + pytest.param( + {"k": "[1,2]"}, + {}, + "k", + id="test-request-parameter-list-of-numbers", + ), + pytest.param( + {"k": {"updatedDateFrom": "2023-08-20T00:00:00Z", "updatedDateTo": "2023-08-20T23:59:59Z"}}, + {}, + "k", + id="test-request-parameter-object-of-the-updated-info", + ), + pytest.param( + {"k": '["a", "b"]'}, + {}, + "k", + id="test-request-parameter-list-of-strings", + ), + pytest.param( + {"k": '{{ config["k"] }}'}, + {"k": [1, 2]}, + "k", + id="test-request-parameter-from-config-list-of-numbers", + ), + pytest.param( + {"k": '{{ config["k"] }}'}, + {"k": ["a", "b"]}, + "k", + id="test-request-parameter-from-config-list-of-strings", + ), + pytest.param( + {"k": '{{ config["k"] }}'}, + {"k": ["a,b"]}, + "k", + id="test-request-parameter-from-config-comma-separated-strings", + ), + pytest.param( + {'["a", "b"]': '{{ config["k"] }}'}, + {"k": [1, 2]}, + '["a", "b"]', + id="test-key-with-list-is-not-interpolated", + ), + pytest.param( + {"a": '{{ config["k"] }}', "b": {"end_timestamp": 1699109113}}, + {"k": 1699108113}, + "b", + id="test-key-with-multiple-keys", + ), + ], +) +def test_request_param_interpolation_with_incorrect_values(request_parameters, config, invalid_value_for_key): + options_provider = InterpolatedRequestOptionsProvider( + config=config, + request_parameters=request_parameters, + request_body_data={}, + request_headers={}, + parameters={}, + ) + requester = create_requester() + requester._request_options_provider = options_provider + with pytest.raises(ValueError) as error: + requester.send_request() + + assert ( + error.value.args[0] + == f"Invalid value for `{invalid_value_for_key}` parameter. The values of request params cannot be an array or object." + ) + + +@pytest.mark.parametrize( + "request_body_data, config, expected_request_body_data", + [ + pytest.param( + {"k": '{"updatedDateFrom": "2023-08-20T00:00:00Z", "updatedDateTo": "2023-08-20T23:59:59Z"}'}, + {}, + # k={"updatedDateFrom": "2023-08-20T00:00:00Z", "updatedDateTo": "2023-08-20T23:59:59Z"} + "k=%7B%22updatedDateFrom%22%3A+%222023-08-20T00%3A00%3A00Z%22%2C+%22updatedDateTo%22%3A+%222023-08-20T23%3A59%3A59Z%22%7D", + id="test-request-body-dictionary", + ), + pytest.param( + {"k": "1,2"}, + {}, + "k=1%2C2", # k=1,2 + id="test-request-body-comma-separated-numbers", + ), + pytest.param( + {"k": "a,b"}, + {}, + "k=a%2Cb", # k=a,b + id="test-request-body-comma-separated-strings", + ), + pytest.param( + {"k": "[1,2]"}, + {}, + "k=1&k=2", + id="test-request-body-list-of-numbers", + ), + pytest.param( + {"k": '["a", "b"]'}, + {}, + "k=a&k=b", + id="test-request-body-list-of-strings", + ), + pytest.param( + {"k": '{{ config["k"] }}'}, + {"k": {"updatedDateFrom": "2023-08-20T00:00:00Z", "updatedDateTo": "2023-08-20T23:59:59Z"}}, + # k={'updatedDateFrom': '2023-08-20T00:00:00Z', 'updatedDateTo': '2023-08-20T23:59:59Z'} + "k=%7B%27updatedDateFrom%27%3A+%272023-08-20T00%3A00%3A00Z%27%2C+%27updatedDateTo%27%3A+%272023-08-20T23%3A59%3A59Z%27%7D", + id="test-request-body-from-config-object", + ), + pytest.param( + {"k": '{{ config["k"] }}'}, + {"k": [1, 2]}, + "k=1&k=2", + id="test-request-body-from-config-list-of-numbers", + ), + pytest.param( + {"k": '{{ config["k"] }}'}, + {"k": ["a", "b"]}, + "k=a&k=b", + id="test-request-body-from-config-list-of-strings", + ), + pytest.param( + {"k": '{{ config["k"] }}'}, + {"k": ["a,b"]}, + "k=a%2Cb", # k=a,b + id="test-request-body-from-config-comma-separated-strings", + ), + pytest.param( + {'["a", "b"]': '{{ config["k"] }}'}, + {"k": [1, 2]}, + "%5B%22a%22%2C+%22b%22%5D=1&%5B%22a%22%2C+%22b%22%5D=2", # ["a", "b"]=1&["a", "b"]=2 + id="test-key-with-list-is-not-interpolated", + ), + pytest.param( + {"k": "{'updatedDateFrom': '2023-08-20T00:00:00Z', 'updatedDateTo': '2023-08-20T23:59:59Z'}"}, + {}, + # k={'updatedDateFrom': '2023-08-20T00:00:00Z', 'updatedDateTo': '2023-08-20T23:59:59Z'} + "k=%7B%27updatedDateFrom%27%3A+%272023-08-20T00%3A00%3A00Z%27%2C+%27updatedDateTo%27%3A+%272023-08-20T23%3A59%3A59Z%27%7D", + id="test-single-quotes-are-retained", + ), + ], +) +def test_request_body_interpolation(request_body_data, config, expected_request_body_data): + options_provider = InterpolatedRequestOptionsProvider( + config=config, + request_parameters={}, + request_body_data=request_body_data, + request_headers={}, + parameters={}, + ) + requester = create_requester() + requester._request_options_provider = options_provider + requester.send_request() + sent_request: PreparedRequest = requester._session.send.call_args_list[0][0][0] + assert sent_request.body == expected_request_body_data + + @pytest.mark.parametrize( "requester_path, param_path, expected_path", [ ("deals", None, "/deals"), ("deals", "deals2", "/deals2"), ("deals", "/deals2", "/deals2"), - ("deals/{{ stream_slice.start }}/{{ next_page_token.next_page_token }}/{{ config.config_key }}/{{ parameters.param_key }}", None, "/deals/2012/pagetoken/config_value/param_value"), - ]) + ( + "deals/{{ stream_slice.start }}/{{ next_page_token.next_page_token }}/{{ config.config_key }}/{{ parameters.param_key }}", + None, + "/deals/2012/pagetoken/config_value/param_value", + ), + ], +) def test_send_request_path(requester_path, param_path, expected_path): requester = create_requester(config={"config_key": "config_value"}, path=requester_path, parameters={"param_key": "param_value"}) - requester.send_request(stream_slice={"start": "2012"}, next_page_token={"next_page_token": "pagetoken"},path=param_path) + requester.send_request(stream_slice={"start": "2012"}, next_page_token={"next_page_token": "pagetoken"}, path=param_path) sent_request: PreparedRequest = requester._session.send.call_args_list[0][0][0] parsed_url = urlparse(sent_request.url) assert parsed_url.path == expected_path def test_send_request_url_base(): - requester = create_requester(url_base="https://example.org/{{ config.config_key }}/{{ parameters.param_key }}", config={"config_key": "config_value"}, parameters={"param_key": "param_value"}) + requester = create_requester( + url_base="https://example.org/{{ config.config_key }}/{{ parameters.param_key }}", + config={"config_key": "config_value"}, + parameters={"param_key": "param_value"}, + ) requester.send_request() sent_request: PreparedRequest = requester._session.send.call_args_list[0][0][0] assert sent_request.url == "https://example.org/config_value/param_value/deals" @@ -324,10 +603,18 @@ def test_send_request_stream_slice_next_page_token(): stream_slice = {"id": "1234"} next_page_token = {"next_page_token": "next_page_token"} requester.send_request(stream_slice=stream_slice, next_page_token=next_page_token) - options_provider.get_request_params.assert_called_once_with(stream_state=None, stream_slice=stream_slice, next_page_token=next_page_token) - options_provider.get_request_body_data.assert_called_once_with(stream_state=None, stream_slice=stream_slice, next_page_token=next_page_token) - options_provider.get_request_body_json.assert_called_once_with(stream_state=None, stream_slice=stream_slice, next_page_token=next_page_token) - options_provider.get_request_headers.assert_called_once_with(stream_state=None, stream_slice=stream_slice, next_page_token=next_page_token) + options_provider.get_request_params.assert_called_once_with( + stream_state=None, stream_slice=stream_slice, next_page_token=next_page_token + ) + options_provider.get_request_body_data.assert_called_once_with( + stream_state=None, stream_slice=stream_slice, next_page_token=next_page_token + ) + options_provider.get_request_body_json.assert_called_once_with( + stream_state=None, stream_slice=stream_slice, next_page_token=next_page_token + ) + options_provider.get_request_headers.assert_called_once_with( + stream_state=None, stream_slice=stream_slice, next_page_token=next_page_token + ) def test_default_authenticator(): @@ -472,14 +759,19 @@ def test_raise_on_http_errors(mocker, error): ({"error": {"message": "something broke"}}, "something broke"), ({"error": "err-001", "message": "something broke"}, "something broke"), ({"failure": {"message": "something broke"}}, "something broke"), + ({"detail": {"message": "something broke"}}, "something broke"), ({"error": {"errors": [{"message": "one"}, {"message": "two"}, {"message": "three"}]}}, "one, two, three"), ({"errors": ["one", "two", "three"]}, "one, two, three"), + ({"errors": [None, {}, "third error", 9002.09]}, "third error"), ({"messages": ["one", "two", "three"]}, "one, two, three"), ({"errors": [{"message": "one"}, {"message": "two"}, {"message": "three"}]}, "one, two, three"), ({"error": [{"message": "one"}, {"message": "two"}, {"message": "three"}]}, "one, two, three"), ({"errors": [{"error": "one"}, {"error": "two"}, {"error": "three"}]}, "one, two, three"), ({"failures": [{"message": "one"}, {"message": "two"}, {"message": "three"}]}, "one, two, three"), + ({"details": [{"message": "one"}, {"message": "two"}, {"message": "three"}]}, "one, two, three"), + ({"details": ["one", 10087, True]}, "one"), (["one", "two", "three"], "one, two, three"), + ({"detail": False}, None), ([{"error": "one"}, {"error": "two"}, {"error": "three"}], "one, two, three"), ({"error": True}, None), ({"something_else": "hi"}, None), @@ -503,15 +795,21 @@ def test_default_parse_response_error_message_not_json(requests_mock): @pytest.mark.parametrize( - "test_name, base_url, path, expected_full_url",[ + "test_name, base_url, path, expected_full_url", + [ ("test_no_slashes", "https://airbyte.io", "my_endpoint", "https://airbyte.io/my_endpoint"), ("test_trailing_slash_on_base_url", "https://airbyte.io/", "my_endpoint", "https://airbyte.io/my_endpoint"), - ("test_trailing_slash_on_base_url_and_leading_slash_on_path", "https://airbyte.io/", "/my_endpoint", "https://airbyte.io/my_endpoint"), + ( + "test_trailing_slash_on_base_url_and_leading_slash_on_path", + "https://airbyte.io/", + "/my_endpoint", + "https://airbyte.io/my_endpoint", + ), ("test_leading_slash_on_path", "https://airbyte.io", "/my_endpoint", "https://airbyte.io/my_endpoint"), ("test_trailing_slash_on_path", "https://airbyte.io", "/my_endpoint/", "https://airbyte.io/my_endpoint/"), ("test_nested_path_no_leading_slash", "https://airbyte.io", "v1/my_endpoint", "https://airbyte.io/v1/my_endpoint"), ("test_nested_path_with_leading_slash", "https://airbyte.io", "/v1/my_endpoint", "https://airbyte.io/v1/my_endpoint"), - ] + ], ) def test_join_url(test_name, base_url, path, expected_full_url): requester = HttpRequester( @@ -533,16 +831,44 @@ def test_join_url(test_name, base_url, path, expected_full_url): @pytest.mark.parametrize( - "path, params, expected_url", [ + "path, params, expected_url", + [ pytest.param("v1/endpoint?param1=value1", {}, "https://test_base_url.com/v1/endpoint?param1=value1", id="test_params_only_in_path"), - pytest.param("v1/endpoint", {"param1": "value1"}, "https://test_base_url.com/v1/endpoint?param1=value1", id="test_params_only_in_path"), + pytest.param( + "v1/endpoint", {"param1": "value1"}, "https://test_base_url.com/v1/endpoint?param1=value1", id="test_params_only_in_path" + ), pytest.param("v1/endpoint", None, "https://test_base_url.com/v1/endpoint", id="test_params_is_none_and_no_params_in_path"), - pytest.param("v1/endpoint?param1=value1", None, "https://test_base_url.com/v1/endpoint?param1=value1", id="test_params_is_none_and_no_params_in_path"), - pytest.param("v1/endpoint?param1=value1", {"param2": "value2"}, "https://test_base_url.com/v1/endpoint?param1=value1¶m2=value2", id="test_no_duplicate_params"), - pytest.param("v1/endpoint?param1=value1", {"param1": "value1"}, "https://test_base_url.com/v1/endpoint?param1=value1", id="test_duplicate_params_same_value"), - pytest.param("v1/endpoint?param1=1", {"param1": 1}, "https://test_base_url.com/v1/endpoint?param1=1", id="test_duplicate_params_same_value_not_string"), - pytest.param("v1/endpoint?param1=value1", {"param1": "value2"}, "https://test_base_url.com/v1/endpoint?param1=value1¶m1=value2", id="test_duplicate_params_different_value"), - ] + pytest.param( + "v1/endpoint?param1=value1", + None, + "https://test_base_url.com/v1/endpoint?param1=value1", + id="test_params_is_none_and_no_params_in_path", + ), + pytest.param( + "v1/endpoint?param1=value1", + {"param2": "value2"}, + "https://test_base_url.com/v1/endpoint?param1=value1¶m2=value2", + id="test_no_duplicate_params", + ), + pytest.param( + "v1/endpoint?param1=value1", + {"param1": "value1"}, + "https://test_base_url.com/v1/endpoint?param1=value1", + id="test_duplicate_params_same_value", + ), + pytest.param( + "v1/endpoint?param1=1", + {"param1": 1}, + "https://test_base_url.com/v1/endpoint?param1=1", + id="test_duplicate_params_same_value_not_string", + ), + pytest.param( + "v1/endpoint?param1=value1", + {"param1": "value2"}, + "https://test_base_url.com/v1/endpoint?param1=value1¶m1=value2", + id="test_duplicate_params_different_value", + ), + ], ) def test_duplicate_request_params_are_deduped(path, params, expected_url): requester = HttpRequester( @@ -564,14 +890,15 @@ def test_duplicate_request_params_are_deduped(path, params, expected_url): @pytest.mark.parametrize( - "should_log, status_code, should_throw", [ + "should_log, status_code, should_throw", + [ (True, 200, False), (True, 400, False), (True, 500, True), (False, 200, False), (False, 400, False), (False, 500, True), - ] + ], ) def test_log_requests(should_log, status_code, should_throw): repository = MagicMock() @@ -584,7 +911,7 @@ def test_log_requests(should_log, status_code, should_throw): config={}, parameters={}, message_repository=repository, - disable_retries=True + disable_retries=True, ) requester._session.send = MagicMock() response = requests.Response() @@ -600,3 +927,48 @@ def test_log_requests(should_log, status_code, should_throw): if should_log: assert repository.log_message.call_args_list[0].args[1]() == "formatted_response" formatter.assert_called_once_with(response) + + +def test_connection_pool(): + requester = HttpRequester( + name="name", + url_base="https://test_base_url.com", + path="/", + http_method=HttpMethod.GET, + request_options_provider=None, + config={}, + parameters={}, + message_repository=MagicMock(), + disable_retries=True, + ) + assert requester._session.adapters["https://"]._pool_connections == 20 + + +def test_caching_filename(http_requester_factory): + http_requester = http_requester_factory() + assert http_requester.cache_filename == f"{http_requester.name}.sqlite" + + +def test_caching_session_with_enable_use_cache(http_requester_factory): + http_requester = http_requester_factory(use_cache=True) + assert isinstance(http_requester._session, requests_cache.CachedSession) + + +def test_response_caching_with_enable_use_cache(http_requester_factory, requests_mock): + http_requester = http_requester_factory(use_cache=True) + + requests_mock.register_uri("GET", http_requester.url_base, json=[{"id": 12, "title": "test_record"}]) + http_requester.clear_cache() + + response = http_requester.send_request() + + assert requests_mock.called + assert isinstance(response, requests.Response) + + requests_mock.reset_mock() + new_response = http_requester.send_request() + + assert not requests_mock.called + assert isinstance(new_response, CachedResponse) + + assert len(response.json()) == len(new_response.json()) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index ebdc7a6201fd..1747c8ec4d07 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -1,7 +1,6 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - from unittest.mock import MagicMock, Mock, patch import pytest @@ -94,7 +93,7 @@ def test_simple_retriever_full(mock_http_stream): assert retriever._last_response is None assert retriever._records_from_last_response == [] - assert retriever._parse_response(response, stream_state={}) == records + assert retriever._parse_response(response, stream_state={}, records_schema={}) == records assert retriever._last_response == response assert retriever._records_from_last_response == records @@ -170,7 +169,7 @@ def test_simple_retriever_with_request_response_log_last_records(mock_http_strea assert retriever._last_response is None assert retriever._records_from_last_response == [] - assert retriever._parse_response(response, stream_state={}) == request_response_logs + assert retriever._parse_response(response, stream_state={}, records_schema={}) == request_response_logs assert retriever._last_response == response assert retriever._records_from_last_response == request_response_logs @@ -396,8 +395,16 @@ def test_when_read_records_then_cursor_close_slice_with_greater_record(test_name ) stream_slice = {"repository": "airbyte"} - with patch.object(SimpleRetriever, "_read_pages", return_value=iter([first_record, second_record]), side_effect=lambda _, __, ___: retriever._parse_records(response=MagicMock(), stream_state=None, stream_slice=stream_slice)): - list(retriever.read_records(stream_slice=stream_slice)) + def retriever_read_pages(_, __, ___): + return retriever._parse_records(response=MagicMock(), stream_state={}, stream_slice=stream_slice, records_schema={}) + + with patch.object( + SimpleRetriever, + "_read_pages", + return_value=iter([first_record, second_record]), + side_effect=retriever_read_pages, + ): + list(retriever.read_records(stream_slice=stream_slice, records_schema={})) cursor.close_slice.assert_called_once_with(stream_slice, first_record if first_greater_than_second else second_record) @@ -420,8 +427,16 @@ def test_given_stream_data_is_not_record_when_read_records_then_update_slice_wit ) stream_slice = {"repository": "airbyte"} - with patch.object(SimpleRetriever, "_read_pages", return_value=iter(stream_data), side_effect=lambda _, __, ___: retriever._parse_records(response=MagicMock(), stream_state=None, stream_slice=stream_slice)): - list(retriever.read_records(stream_slice=stream_slice)) + def retriever_read_pages(_, __, ___): + return retriever._parse_records(response=MagicMock(), stream_state={}, stream_slice=stream_slice, records_schema={}) + + with patch.object( + SimpleRetriever, + "_read_pages", + return_value=iter(stream_data), + side_effect=retriever_read_pages, + ): + list(retriever.read_records(stream_slice=stream_slice, records_schema={})) cursor.close_slice.assert_called_once_with(stream_slice, None) @@ -430,7 +445,7 @@ def _generate_slices(number_of_slices): @patch.object(SimpleRetriever, "_read_pages", return_value=iter([])) -def test_given_state_selector_when_read_records_use_stream_state(http_stream_read_pages): +def test_given_state_selector_when_read_records_use_stream_state(http_stream_read_pages, mocker): requester = MagicMock() paginator = MagicMock() record_selector = MagicMock() @@ -449,9 +464,10 @@ def test_given_state_selector_when_read_records_use_stream_state(http_stream_rea parameters={}, config={}, ) - list(retriever.read_records(stream_slice=A_STREAM_SLICE)) - http_stream_read_pages.assert_called_once_with(retriever._parse_records, A_STREAM_STATE, A_STREAM_SLICE) + list(retriever.read_records(stream_slice=A_STREAM_SLICE, records_schema={})) + + http_stream_read_pages.assert_called_once_with(mocker.ANY, A_STREAM_STATE, A_STREAM_SLICE) def test_emit_log_request_response_messages(mocker): diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/spec/test_spec.py b/airbyte-cdk/python/unit_tests/sources/declarative/spec/test_spec.py index 27797d67be32..46b892256a25 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/spec/test_spec.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/spec/test_spec.py @@ -11,9 +11,23 @@ @pytest.mark.parametrize( "test_name, spec, expected_connection_specification", [ - ("test_only_connection_specification", Spec(connection_specification={"client_id": "my_client_id"}, parameters={}), ConnectorSpecification(connectionSpecification={"client_id": "my_client_id"})), - ("test_with_doc_url", Spec(connection_specification={"client_id": "my_client_id"}, parameters={}, documentation_url="https://airbyte.io"), ConnectorSpecification(connectionSpecification={"client_id": "my_client_id"}, documentationUrl="https://airbyte.io")), - ("test_auth_flow", Spec(connection_specification={"client_id": "my_client_id"}, parameters={}, advanced_auth=AuthFlow(auth_flow_type="oauth2.0")), ConnectorSpecification(connectionSpecification={"client_id": "my_client_id"}, advanced_auth=AdvancedAuth(auth_flow_type="oauth2.0"))), + ( + "test_only_connection_specification", + Spec(connection_specification={"client_id": "my_client_id"}, parameters={}), + ConnectorSpecification(connectionSpecification={"client_id": "my_client_id"}), + ), + ( + "test_with_doc_url", + Spec(connection_specification={"client_id": "my_client_id"}, parameters={}, documentation_url="https://airbyte.io"), + ConnectorSpecification(connectionSpecification={"client_id": "my_client_id"}, documentationUrl="https://airbyte.io"), + ), + ( + "test_auth_flow", + Spec(connection_specification={"client_id": "my_client_id"}, parameters={}, advanced_auth=AuthFlow(auth_flow_type="oauth2.0")), + ConnectorSpecification( + connectionSpecification={"client_id": "my_client_id"}, advanced_auth=AdvancedAuth(auth_flow_type="oauth2.0") + ), + ), ], ) def test_spec(test_name, spec, expected_connection_specification): diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py index 74e4d5fece7c..e91574f86d1e 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/stream_slicers/test_cartesian_product_stream_slicer.py @@ -22,9 +22,7 @@ ( "test_two_stream_slicers", [ - ListPartitionRouter( - values=["customer", "store", "subscription"], cursor_field="owner_resource", config={}, parameters={} - ), + ListPartitionRouter(values=["customer", "store", "subscription"], cursor_field="owner_resource", config={}, parameters={}), ListPartitionRouter(values=["A", "B"], cursor_field="letter", config={}, parameters={}), ], [ @@ -39,9 +37,7 @@ ( "test_list_and_datetime", [ - ListPartitionRouter( - values=["customer", "store", "subscription"], cursor_field="owner_resource", config={}, parameters={} - ), + ListPartitionRouter(values=["customer", "store", "subscription"], cursor_field="owner_resource", config={}, parameters={}), DatetimeBasedCursor( start_datetime=MinMaxDatetime(datetime="2021-01-01", datetime_format="%Y-%m-%d", parameters={}), end_datetime=MinMaxDatetime(datetime="2021-01-03", datetime_format="%Y-%m-%d", parameters={}), diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py index e139e4ac2062..8252089b5d4c 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py @@ -6,6 +6,7 @@ import logging import os import sys +from copy import deepcopy from typing import Any, List, Mapping from unittest.mock import call, patch @@ -754,8 +755,8 @@ def response_log_message(response: dict) -> AirbyteMessage: def _create_request(): url = "https://example.com/api" - headers = {'Content-Type': 'application/json'} - return requests.Request('POST', url, headers=headers, json={"key": "value"}).prepare() + headers = {"Content-Type": "application/json"} + return requests.Request("POST", url, headers=headers, json={"key": "value"}).prepare() def _create_response(body): @@ -772,475 +773,352 @@ def _create_page(response_body): return response -@pytest.mark.parametrize("test_name, manifest, pages, expected_records, expected_calls",[ - ("test_read_manifest_no_pagination_no_partitions", - { - "version": "0.34.2", - "type": "DeclarativeSource", - "check": { - "type": "CheckStream", - "stream_names": [ - "Rates" - ] - }, - "streams": [ - { - "type": "DeclarativeStream", - "name": "Rates", - "primary_key": [], - "schema_loader": { - "type": "InlineSchemaLoader", - "schema": { - "$schema": "http://json-schema.org/schema#", - "properties": { - "ABC": { - "type": "number" - }, - "AED": { - "type": "number" - }, - }, - "type": "object" - } - }, - "retriever": { - "type": "SimpleRetriever", - "requester": { - "type": "HttpRequester", - "url_base": "https://api.apilayer.com", - "path": "/exchangerates_data/latest", - "http_method": "GET", - "request_parameters": {}, - "request_headers": {}, - "request_body_json": {}, - "authenticator": { - "type": "ApiKeyAuthenticator", - "header": "apikey", - "api_token": "{{ config['api_key'] }}" - } - }, - "record_selector": { - "type": "RecordSelector", - "extractor": { - "type": "DpathExtractor", - "field_path": [ - "rates" - ] - } - }, - "paginator": { - "type": "NoPagination" - } - } - } - ], - "spec": { - "connection_specification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "required": [ - "api_key" - ], - "properties": { - "api_key": { - "type": "string", - "title": "API Key", - "airbyte_secret": True - } - }, - "additionalProperties": True - }, - "documentation_url": "https://example.org", - "type": "Spec" - } - }, - (_create_page({"rates": [{"ABC": 0}, {"AED": 1}],"_metadata": {"next": "next"}}), _create_page({"rates": [{"USD": 2}],"_metadata": {"next": "next"}})) * 10, - [{"ABC": 0}, {"AED": 1}], - [call({}, {}, None)]), - ("test_read_manifest_with_added_fields", - { - "version": "0.34.2", - "type": "DeclarativeSource", - "check": { - "type": "CheckStream", - "stream_names": [ - "Rates" - ] - }, - "streams": [ - { - "type": "DeclarativeStream", - "name": "Rates", - "primary_key": [], - "schema_loader": { - "type": "InlineSchemaLoader", - "schema": { - "$schema": "http://json-schema.org/schema#", - "properties": { - "ABC": { - "type": "number" - }, - "AED": { - "type": "number" - }, - }, - "type": "object" - } - }, - "transformations": [ - { - "type": "AddFields", - "fields": [ - { - "type": "AddedFieldDefinition", - "path": ["added_field_key"], - "value": "added_field_value" - } - ] - } - ], - "retriever": { - "type": "SimpleRetriever", - "requester": { - "type": "HttpRequester", - "url_base": "https://api.apilayer.com", - "path": "/exchangerates_data/latest", - "http_method": "GET", - "request_parameters": {}, - "request_headers": {}, - "request_body_json": {}, - "authenticator": { - "type": "ApiKeyAuthenticator", - "header": "apikey", - "api_token": "{{ config['api_key'] }}" - } - }, - "record_selector": { - "type": "RecordSelector", - "extractor": { - "type": "DpathExtractor", - "field_path": [ - "rates" - ] - } - }, - "paginator": { - "type": "NoPagination" - } - } - } - ], - "spec": { - "connection_specification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "required": [ - "api_key" - ], - "properties": { - "api_key": { - "type": "string", - "title": "API Key", - "airbyte_secret": True - } - }, - "additionalProperties": True - }, - "documentation_url": "https://example.org", - "type": "Spec" - } - }, - (_create_page({"rates": [{"ABC": 0}, {"AED": 1}],"_metadata": {"next": "next"}}), _create_page({"rates": [{"USD": 2}],"_metadata": {"next": "next"}})) * 10, - [{"ABC": 0, "added_field_key": "added_field_value"}, {"AED": 1, "added_field_key": "added_field_value"}], - [call({}, {}, None)]), - ("test_read_with_pagination_no_partitions", - { - "version": "0.34.2", - "type": "DeclarativeSource", - "check": { - "type": "CheckStream", - "stream_names": [ - "Rates" - ] - }, - "streams": [ - { - "type": "DeclarativeStream", - "name": "Rates", - "primary_key": [], - "schema_loader": { - "type": "InlineSchemaLoader", - "schema": { - "$schema": "http://json-schema.org/schema#", - "properties": { - "ABC": { - "type": "number" - }, - "AED": { - "type": "number" - }, - "USD": { - "type": "number" - }, - }, - "type": "object" - } - }, - "retriever": { - "type": "SimpleRetriever", - "requester": { - "type": "HttpRequester", - "url_base": "https://api.apilayer.com", - "path": "/exchangerates_data/latest", - "http_method": "GET", - "request_parameters": {}, - "request_headers": {}, - "request_body_json": {}, - "authenticator": { - "type": "ApiKeyAuthenticator", - "header": "apikey", - "api_token": "{{ config['api_key'] }}" - } - }, - "record_selector": { - "type": "RecordSelector", - "extractor": { - "type": "DpathExtractor", - "field_path": [ - "rates" - ] - } - }, - "paginator": { - "type": "DefaultPaginator", - "page_size": 2, - "page_size_option": {"inject_into": "request_parameter", "field_name": "page_size"}, - "page_token_option": {"inject_into": "path", "type": "RequestPath"}, - "pagination_strategy": {"type": "CursorPagination", "cursor_value": "{{ response._metadata.next }}", "page_size": 2}, - }, - } - } - ], - "spec": { - "connection_specification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "required": [ - "api_key" - ], - "properties": { - "api_key": { - "type": "string", - "title": "API Key", - "airbyte_secret": True - } - }, - "additionalProperties": True - }, - "documentation_url": "https://example.org", - "type": "Spec" - } - }, - (_create_page({"rates": [{"ABC": 0}, {"AED": 1}],"_metadata": {"next": "next"}}), _create_page({"rates": [{"USD": 2}],"_metadata": {}})) * 10, - [{"ABC": 0}, {"AED": 1}, {"USD": 2}], - [call({}, {}, None), call({}, {}, {"next_page_token": "next"})]), - ( - "test_no_pagination_with_partition_router", - { - "version": "0.34.2", - "type": "DeclarativeSource", - "check": { - "type": "CheckStream", - "stream_names": [ - "Rates" - ] - }, - "streams": [ - { - "type": "DeclarativeStream", - "name": "Rates", - "primary_key": [], - "schema_loader": { - "type": "InlineSchemaLoader", - "schema": { - "$schema": "http://json-schema.org/schema#", - "properties": { - "ABC": { - "type": "number" +@pytest.mark.parametrize( + "test_name, manifest, pages, expected_records, expected_calls", + [ + ( + "test_read_manifest_no_pagination_no_partitions", + { + "version": "0.34.2", + "type": "DeclarativeSource", + "check": {"type": "CheckStream", "stream_names": ["Rates"]}, + "streams": [ + { + "type": "DeclarativeStream", + "name": "Rates", + "primary_key": [], + "schema_loader": { + "type": "InlineSchemaLoader", + "schema": { + "$schema": "http://json-schema.org/schema#", + "properties": { + "ABC": {"type": "number"}, + "AED": {"type": "number"}, }, - "AED": { - "type": "number" + "type": "object", + }, + }, + "retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "url_base": "https://api.apilayer.com", + "path": "/exchangerates_data/latest", + "http_method": "GET", + "request_parameters": {}, + "request_headers": {}, + "request_body_json": {}, + "authenticator": { + "type": "ApiKeyAuthenticator", + "header": "apikey", + "api_token": "{{ config['api_key'] }}", }, - "partition": { - "type": "number" - } }, - "type": "object" - } + "record_selector": {"type": "RecordSelector", "extractor": {"type": "DpathExtractor", "field_path": ["rates"]}}, + "paginator": {"type": "NoPagination"}, + }, + } + ], + "spec": { + "connection_specification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["api_key"], + "properties": {"api_key": {"type": "string", "title": "API Key", "airbyte_secret": True}}, + "additionalProperties": True, }, - "retriever": { - "type": "SimpleRetriever", - "requester": { - "type": "HttpRequester", - "url_base": "https://api.apilayer.com", - "path": "/exchangerates_data/latest", - "http_method": "GET", - "request_parameters": {}, - "request_headers": {}, - "request_body_json": {}, - "authenticator": { - "type": "ApiKeyAuthenticator", - "header": "apikey", - "api_token": "{{ config['api_key'] }}" + "documentation_url": "https://example.org", + "type": "Spec", + }, + }, + ( + _create_page({"rates": [{"ABC": 0}, {"AED": 1}], "_metadata": {"next": "next"}}), + _create_page({"rates": [{"USD": 2}], "_metadata": {"next": "next"}}), + ) + * 10, + [{"ABC": 0}, {"AED": 1}], + [call({}, {}, None)], + ), + ( + "test_read_manifest_with_added_fields", + { + "version": "0.34.2", + "type": "DeclarativeSource", + "check": {"type": "CheckStream", "stream_names": ["Rates"]}, + "streams": [ + { + "type": "DeclarativeStream", + "name": "Rates", + "primary_key": [], + "schema_loader": { + "type": "InlineSchemaLoader", + "schema": { + "$schema": "http://json-schema.org/schema#", + "properties": { + "ABC": {"type": "number"}, + "AED": {"type": "number"}, + }, + "type": "object", + }, + }, + "transformations": [ + { + "type": "AddFields", + "fields": [{"type": "AddedFieldDefinition", "path": ["added_field_key"], "value": "added_field_value"}], } + ], + "retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "url_base": "https://api.apilayer.com", + "path": "/exchangerates_data/latest", + "http_method": "GET", + "request_parameters": {}, + "request_headers": {}, + "request_body_json": {}, + "authenticator": { + "type": "ApiKeyAuthenticator", + "header": "apikey", + "api_token": "{{ config['api_key'] }}", + }, + }, + "record_selector": {"type": "RecordSelector", "extractor": {"type": "DpathExtractor", "field_path": ["rates"]}}, + "paginator": {"type": "NoPagination"}, }, - "partition_router": { - "type": "ListPartitionRouter", - "values": ["0", "1"], - "cursor_field": "partition" + } + ], + "spec": { + "connection_specification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["api_key"], + "properties": {"api_key": {"type": "string", "title": "API Key", "airbyte_secret": True}}, + "additionalProperties": True, + }, + "documentation_url": "https://example.org", + "type": "Spec", + }, + }, + ( + _create_page({"rates": [{"ABC": 0}, {"AED": 1}], "_metadata": {"next": "next"}}), + _create_page({"rates": [{"USD": 2}], "_metadata": {"next": "next"}}), + ) + * 10, + [{"ABC": 0, "added_field_key": "added_field_value"}, {"AED": 1, "added_field_key": "added_field_value"}], + [call({}, {}, None)], + ), + ( + "test_read_with_pagination_no_partitions", + { + "version": "0.34.2", + "type": "DeclarativeSource", + "check": {"type": "CheckStream", "stream_names": ["Rates"]}, + "streams": [ + { + "type": "DeclarativeStream", + "name": "Rates", + "primary_key": [], + "schema_loader": { + "type": "InlineSchemaLoader", + "schema": { + "$schema": "http://json-schema.org/schema#", + "properties": { + "ABC": {"type": "number"}, + "AED": {"type": "number"}, + "USD": {"type": "number"}, + }, + "type": "object", + }, }, - "record_selector": { - "type": "RecordSelector", - "extractor": { - "type": "DpathExtractor", - "field_path": [ - "rates" - ] - } + "retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "url_base": "https://api.apilayer.com", + "path": "/exchangerates_data/latest", + "http_method": "GET", + "request_parameters": {}, + "request_headers": {}, + "request_body_json": {}, + "authenticator": { + "type": "ApiKeyAuthenticator", + "header": "apikey", + "api_token": "{{ config['api_key'] }}", + }, + }, + "record_selector": {"type": "RecordSelector", "extractor": {"type": "DpathExtractor", "field_path": ["rates"]}}, + "paginator": { + "type": "DefaultPaginator", + "page_size": 2, + "page_size_option": {"inject_into": "request_parameter", "field_name": "page_size"}, + "page_token_option": {"inject_into": "path", "type": "RequestPath"}, + "pagination_strategy": { + "type": "CursorPagination", + "cursor_value": "{{ response._metadata.next }}", + "page_size": 2, + }, + }, }, - "paginator": { - "type": "NoPagination" - } } - } - ], - "spec": { - "connection_specification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "required": [ - "api_key" - ], - "properties": { - "api_key": { - "type": "string", - "title": "API Key", - "airbyte_secret": True - } + ], + "spec": { + "connection_specification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["api_key"], + "properties": {"api_key": {"type": "string", "title": "API Key", "airbyte_secret": True}}, + "additionalProperties": True, }, - "additionalProperties": True + "documentation_url": "https://example.org", + "type": "Spec", }, - "documentation_url": "https://example.org", - "type": "Spec" - } - }, - (_create_page({"rates": [{"ABC": 0, "partition": 0}, {"AED": 1, "partition": 0}], "_metadata": {"next": "next"}}), - _create_page({"rates": [{"ABC": 2, "partition": 1}], "_metadata": {"next": "next"}})), - [{"ABC": 0, "partition": 0}, {"AED": 1, "partition": 0}, {"ABC": 2, "partition": 1}], - [call({}, {"partition": "0"}, None), call({}, {"partition": "1"}, None)] - ), - ("test_with_pagination_and_partition_router", - { - "version": "0.34.2", - "type": "DeclarativeSource", - "check": { - "type": "CheckStream", - "stream_names": [ - "Rates" - ] - }, - "streams": [ - { - "type": "DeclarativeStream", - "name": "Rates", - "primary_key": [], - "schema_loader": { - "type": "InlineSchemaLoader", - "schema": { - "$schema": "http://json-schema.org/schema#", - "properties": { - "ABC": { - "type": "number" - }, - "AED": { - "type": "number" - }, - "partition": { - "type": "number" - } - }, - "type": "object" - } - }, - "retriever": { - "type": "SimpleRetriever", - "requester": { - "type": "HttpRequester", - "url_base": "https://api.apilayer.com", - "path": "/exchangerates_data/latest", - "http_method": "GET", - "request_parameters": {}, - "request_headers": {}, - "request_body_json": {}, - "authenticator": { - "type": "ApiKeyAuthenticator", - "header": "apikey", - "api_token": "{{ config['api_key'] }}" - } - }, - "partition_router": { - "type": "ListPartitionRouter", - "values": ["0", "1"], - "cursor_field": "partition" - }, - "record_selector": { - "type": "RecordSelector", - "extractor": { - "type": "DpathExtractor", - "field_path": [ - "rates" - ] - } - }, - "paginator": { - "type": "DefaultPaginator", - "page_size": 2, - "page_size_option": {"inject_into": "request_parameter", "field_name": "page_size"}, - "page_token_option": {"inject_into": "path", "type": "RequestPath"}, - "pagination_strategy": {"type": "CursorPagination", "cursor_value": "{{ response._metadata.next }}", "page_size": 2}, - }, - } - } - ], - "spec": { - "connection_specification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "required": [ - "api_key" - ], - "properties": { - "api_key": { - "type": "string", - "title": "API Key", - "airbyte_secret": True - } - }, - "additionalProperties": True - }, - "documentation_url": "https://example.org", - "type": "Spec" - } - }, - ( - _create_page({"rates": [{"ABC": 0, "partition": 0}, {"AED": 1, "partition": 0}], "_metadata": {"next": "next"}}), - _create_page({"rates": [{"USD": 3, "partition": 0}], "_metadata": {}}), - _create_page({"rates": [{"ABC": 2, "partition": 1}], "_metadata": {}}), - ), - [{"ABC": 0, "partition": 0}, {"AED": 1, "partition": 0}, {"USD": 3, "partition": 0}, {"ABC": 2, "partition": 1}], - [call({}, {"partition": "0"}, None), call({}, {"partition": "0"},{"next_page_token": "next"}), call({}, {"partition": "1"},None),] - ) -]) + }, + ( + _create_page({"rates": [{"ABC": 0}, {"AED": 1}], "_metadata": {"next": "next"}}), + _create_page({"rates": [{"USD": 2}], "_metadata": {}}), + ) + * 10, + [{"ABC": 0}, {"AED": 1}, {"USD": 2}], + [call({}, {}, None), call({}, {}, {"next_page_token": "next"})], + ), + ( + "test_no_pagination_with_partition_router", + { + "version": "0.34.2", + "type": "DeclarativeSource", + "check": {"type": "CheckStream", "stream_names": ["Rates"]}, + "streams": [ + { + "type": "DeclarativeStream", + "name": "Rates", + "primary_key": [], + "schema_loader": { + "type": "InlineSchemaLoader", + "schema": { + "$schema": "http://json-schema.org/schema#", + "properties": {"ABC": {"type": "number"}, "AED": {"type": "number"}, "partition": {"type": "number"}}, + "type": "object", + }, + }, + "retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "url_base": "https://api.apilayer.com", + "path": "/exchangerates_data/latest", + "http_method": "GET", + "request_parameters": {}, + "request_headers": {}, + "request_body_json": {}, + "authenticator": { + "type": "ApiKeyAuthenticator", + "header": "apikey", + "api_token": "{{ config['api_key'] }}", + }, + }, + "partition_router": {"type": "ListPartitionRouter", "values": ["0", "1"], "cursor_field": "partition"}, + "record_selector": {"type": "RecordSelector", "extractor": {"type": "DpathExtractor", "field_path": ["rates"]}}, + "paginator": {"type": "NoPagination"}, + }, + } + ], + "spec": { + "connection_specification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["api_key"], + "properties": {"api_key": {"type": "string", "title": "API Key", "airbyte_secret": True}}, + "additionalProperties": True, + }, + "documentation_url": "https://example.org", + "type": "Spec", + }, + }, + ( + _create_page({"rates": [{"ABC": 0, "partition": 0}, {"AED": 1, "partition": 0}], "_metadata": {"next": "next"}}), + _create_page({"rates": [{"ABC": 2, "partition": 1}], "_metadata": {"next": "next"}}), + ), + [{"ABC": 0, "partition": 0}, {"AED": 1, "partition": 0}, {"ABC": 2, "partition": 1}], + [call({}, {"partition": "0"}, None), call({}, {"partition": "1"}, None)], + ), + ( + "test_with_pagination_and_partition_router", + { + "version": "0.34.2", + "type": "DeclarativeSource", + "check": {"type": "CheckStream", "stream_names": ["Rates"]}, + "streams": [ + { + "type": "DeclarativeStream", + "name": "Rates", + "primary_key": [], + "schema_loader": { + "type": "InlineSchemaLoader", + "schema": { + "$schema": "http://json-schema.org/schema#", + "properties": {"ABC": {"type": "number"}, "AED": {"type": "number"}, "partition": {"type": "number"}}, + "type": "object", + }, + }, + "retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "url_base": "https://api.apilayer.com", + "path": "/exchangerates_data/latest", + "http_method": "GET", + "request_parameters": {}, + "request_headers": {}, + "request_body_json": {}, + "authenticator": { + "type": "ApiKeyAuthenticator", + "header": "apikey", + "api_token": "{{ config['api_key'] }}", + }, + }, + "partition_router": {"type": "ListPartitionRouter", "values": ["0", "1"], "cursor_field": "partition"}, + "record_selector": {"type": "RecordSelector", "extractor": {"type": "DpathExtractor", "field_path": ["rates"]}}, + "paginator": { + "type": "DefaultPaginator", + "page_size": 2, + "page_size_option": {"inject_into": "request_parameter", "field_name": "page_size"}, + "page_token_option": {"inject_into": "path", "type": "RequestPath"}, + "pagination_strategy": { + "type": "CursorPagination", + "cursor_value": "{{ response._metadata.next }}", + "page_size": 2, + }, + }, + }, + } + ], + "spec": { + "connection_specification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "required": ["api_key"], + "properties": {"api_key": {"type": "string", "title": "API Key", "airbyte_secret": True}}, + "additionalProperties": True, + }, + "documentation_url": "https://example.org", + "type": "Spec", + }, + }, + ( + _create_page({"rates": [{"ABC": 0, "partition": 0}, {"AED": 1, "partition": 0}], "_metadata": {"next": "next"}}), + _create_page({"rates": [{"USD": 3, "partition": 0}], "_metadata": {}}), + _create_page({"rates": [{"ABC": 2, "partition": 1}], "_metadata": {}}), + ), + [{"ABC": 0, "partition": 0}, {"AED": 1, "partition": 0}, {"USD": 3, "partition": 0}, {"ABC": 2, "partition": 1}], + [ + call({}, {"partition": "0"}, None), + call({}, {"partition": "0"}, {"next_page_token": "next"}), + call({}, {"partition": "1"}, None), + ], + ), + ], +) def test_read_manifest_declarative_source(test_name, manifest, pages, expected_records, expected_calls): _stream_name = "Rates" with patch.object(SimpleRetriever, "_fetch_next_page", side_effect=pages) as mock_retriever: @@ -1249,9 +1127,134 @@ def test_read_manifest_declarative_source(test_name, manifest, pages, expected_r mock_retriever.assert_has_calls(expected_calls) +def test_only_parent_streams_use_cache(): + applications_stream = { + "type": "DeclarativeStream", + "$parameters": {"name": "applications", "primary_key": "id", "url_base": "https://harvest.greenhouse.io/v1/"}, + "schema_loader": { + "name": "{{ parameters.stream_name }}", + "file_path": "./source_sendgrid/schemas/{{ parameters.name }}.yaml", + }, + "retriever": { + "paginator": { + "type": "DefaultPaginator", + "page_size": 10, + "page_size_option": {"type": "RequestOption", "inject_into": "request_parameter", "field_name": "per_page"}, + "page_token_option": {"type": "RequestPath"}, + "pagination_strategy": { + "type": "CursorPagination", + "cursor_value": "{{ headers['link']['next']['url'] }}", + "stop_condition": "{{ 'next' not in headers['link'] }}", + "page_size": 100, + }, + }, + "requester": { + "path": "applications", + "authenticator": {"type": "BasicHttpAuthenticator", "username": "{{ config['api_key'] }}"}, + }, + "record_selector": {"extractor": {"type": "DpathExtractor", "field_path": []}}, + }, + } + + manifest = { + "version": "0.29.3", + "definitions": {}, + "streams": [ + deepcopy(applications_stream), + { + "type": "DeclarativeStream", + "$parameters": {"name": "applications_interviews", "primary_key": "id", "url_base": "https://harvest.greenhouse.io/v1/"}, + "schema_loader": { + "name": "{{ parameters.stream_name }}", + "file_path": "./source_sendgrid/schemas/{{ parameters.name }}.yaml", + }, + "retriever": { + "paginator": { + "type": "DefaultPaginator", + "page_size": 10, + "page_size_option": {"type": "RequestOption", "inject_into": "request_parameter", "field_name": "per_page"}, + "page_token_option": {"type": "RequestPath"}, + "pagination_strategy": { + "type": "CursorPagination", + "cursor_value": "{{ headers['link']['next']['url'] }}", + "stop_condition": "{{ 'next' not in headers['link'] }}", + "page_size": 100, + }, + }, + "requester": { + "path": "applications_interviews", + "authenticator": {"type": "BasicHttpAuthenticator", "username": "{{ config['api_key'] }}"}, + }, + "record_selector": {"extractor": {"type": "DpathExtractor", "field_path": []}}, + "partition_router": { + "parent_stream_configs": [ + {"parent_key": "id", "partition_field": "parent_id", "stream": deepcopy(applications_stream)} + ], + "type": "SubstreamPartitionRouter", + }, + }, + }, + { + "type": "DeclarativeStream", + "$parameters": {"name": "jobs", "primary_key": "id", "url_base": "https://harvest.greenhouse.io/v1/"}, + "schema_loader": { + "name": "{{ parameters.stream_name }}", + "file_path": "./source_sendgrid/schemas/{{ parameters.name }}.yaml", + }, + "retriever": { + "paginator": { + "type": "DefaultPaginator", + "page_size": 10, + "page_size_option": {"type": "RequestOption", "inject_into": "request_parameter", "field_name": "per_page"}, + "page_token_option": {"type": "RequestPath"}, + "pagination_strategy": { + "type": "CursorPagination", + "cursor_value": "{{ headers['link']['next']['url'] }}", + "stop_condition": "{{ 'next' not in headers['link'] }}", + "page_size": 100, + }, + }, + "requester": { + "path": "jobs", + "authenticator": {"type": "BasicHttpAuthenticator", "username": "{{ config['api_key'] }}"}, + }, + "record_selector": {"extractor": {"type": "DpathExtractor", "field_path": []}}, + }, + }, + ], + "check": {"type": "CheckStream", "stream_names": ["applications"]}, + } + source = ManifestDeclarativeSource(source_config=manifest) + + streams = source.streams({}) + assert len(streams) == 3 + + # Main stream with caching (parent for substream `applications_interviews`) + assert streams[0].name == "applications" + assert streams[0].retriever.requester.use_cache + + # Substream + assert streams[1].name == "applications_interviews" + assert not streams[1].retriever.requester.use_cache + + # Parent stream created for substream + assert streams[1].retriever.stream_slicer.parent_stream_configs[0].stream.name == "applications" + assert streams[1].retriever.stream_slicer.parent_stream_configs[0].stream.retriever.requester.use_cache + + # Main stream without caching + assert streams[2].name == "jobs" + assert not streams[2].retriever.requester.use_cache + + def _run_read(manifest: Mapping[str, Any], stream_name: str) -> List[AirbyteMessage]: source = ManifestDeclarativeSource(source_config=manifest) - catalog = ConfiguredAirbyteCatalog(streams=[ - ConfiguredAirbyteStream(stream=AirbyteStream(name=stream_name, json_schema={}, supported_sync_modes=[SyncMode.full_refresh]), sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.append) - ]) + catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream(name=stream_name, json_schema={}, supported_sync_modes=[SyncMode.full_refresh]), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.append, + ) + ] + ) return list(source.read(logger, {}, catalog, {})) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/transformations/test_add_fields.py b/airbyte-cdk/python/unit_tests/sources/declarative/transformations/test_add_fields.py index 1386f2847652..a10422fd71bf 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/transformations/test_add_fields.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/transformations/test_add_fields.py @@ -2,7 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import Any, List, Mapping, Tuple +from typing import Any, List, Mapping, Optional, Tuple import pytest from airbyte_cdk.sources.declarative.transformations import AddFields @@ -11,12 +11,22 @@ @pytest.mark.parametrize( - ["input_record", "field", "kwargs", "expected"], + ["input_record", "field", "field_type", "kwargs", "expected"], [ - pytest.param({"k": "v"}, [(["path"], "static_value")], {}, {"k": "v", "path": "static_value"}, id="add new static value"), + pytest.param({"k": "v"}, [(["path"], "static_value")], None, {}, {"k": "v", "path": "static_value"}, id="add new static value"), + pytest.param({"k": "v"}, [(["path"], "{{ 1 }}")], None, {}, {"k": "v", "path": 1}, id="add an expression evaluated as a number"), + pytest.param( + {"k": "v"}, + [(["path"], "{{ 1 }}")], + str, + {}, + {"k": "v", "path": "1"}, + id="add an expression evaluated as a string using the value_type field", + ), pytest.param( {"k": "v"}, [(["path"], "static_value"), (["path2"], "static_value2")], + None, {}, {"k": "v", "path": "static_value", "path2": "static_value2"}, id="add new multiple static values", @@ -24,15 +34,17 @@ pytest.param( {"k": "v"}, [(["nested", "path"], "static_value")], + None, {}, {"k": "v", "nested": {"path": "static_value"}}, id="set static value at nested path", ), - pytest.param({"k": "v"}, [(["k"], "new_value")], {}, {"k": "new_value"}, id="update value which already exists"), - pytest.param({"k": [0, 1]}, [(["k", 3], "v")], {}, {"k": [0, 1, None, "v"]}, id="Set element inside array"), + pytest.param({"k": "v"}, [(["k"], "new_value")], None, {}, {"k": "new_value"}, id="update value which already exists"), + pytest.param({"k": [0, 1]}, [(["k", 3], "v")], None, {}, {"k": [0, 1, None, "v"]}, id="Set element inside array"), pytest.param( {"k": "v"}, [(["k2"], '{{ config["shop"] }}')], + None, {"config": {"shop": "in-n-out"}}, {"k": "v", "k2": "in-n-out"}, id="set a value from the config using bracket notation", @@ -40,6 +52,7 @@ pytest.param( {"k": "v"}, [(["k2"], "{{ config.shop }}")], + None, {"config": {"shop": "in-n-out"}}, {"k": "v", "k2": "in-n-out"}, id="set a value from the config using dot notation", @@ -47,6 +60,7 @@ pytest.param( {"k": "v"}, [(["k2"], '{{ stream_state["cursor"] }}')], + None, {"stream_state": {"cursor": "t0"}}, {"k": "v", "k2": "t0"}, id="set a value from the state using bracket notation", @@ -54,6 +68,7 @@ pytest.param( {"k": "v"}, [(["k2"], "{{ stream_state.cursor }}")], + None, {"stream_state": {"cursor": "t0"}}, {"k": "v", "k2": "t0"}, id="set a value from the state using dot notation", @@ -61,6 +76,7 @@ pytest.param( {"k": "v"}, [(["k2"], '{{ stream_slice["start_date"] }}')], + None, {"stream_slice": {"start_date": "oct1"}}, {"k": "v", "k2": "oct1"}, id="set a value from the stream slice using bracket notation", @@ -68,6 +84,7 @@ pytest.param( {"k": "v"}, [(["k2"], "{{ stream_slice.start_date }}")], + None, {"stream_slice": {"start_date": "oct1"}}, {"k": "v", "k2": "oct1"}, id="set a value from the stream slice using dot notation", @@ -75,6 +92,7 @@ pytest.param( {"k": "v"}, [(["k2"], "{{ record.k }}")], + None, {}, {"k": "v", "k2": "v"}, id="set a value from a field in the record using dot notation", @@ -82,6 +100,7 @@ pytest.param( {"k": "v"}, [(["k2"], '{{ record["k"] }}')], + None, {}, {"k": "v", "k2": "v"}, id="set a value from a field in the record using bracket notation", @@ -89,6 +108,7 @@ pytest.param( {"k": {"nested": "v"}}, [(["k2"], "{{ record.k.nested }}")], + None, {}, {"k": {"nested": "v"}, "k2": "v"}, id="set a value from a nested field in the record using bracket notation", @@ -96,15 +116,20 @@ pytest.param( {"k": {"nested": "v"}}, [(["k2"], '{{ record["k"]["nested"] }}')], + None, {}, {"k": {"nested": "v"}, "k2": "v"}, id="set a value from a nested field in the record using bracket notation", ), - pytest.param({"k": "v"}, [(["k2"], "{{ 2 + 2 }}")], {}, {"k": "v", "k2": 4}, id="set a value from a jinja expression"), + pytest.param({"k": "v"}, [(["k2"], "{{ 2 + 2 }}")], None, {}, {"k": "v", "k2": 4}, id="set a value from a jinja expression"), ], ) def test_add_fields( - input_record: Mapping[str, Any], field: List[Tuple[FieldPointer, str]], kwargs: Mapping[str, Any], expected: Mapping[str, Any] + input_record: Mapping[str, Any], + field: List[Tuple[FieldPointer, str]], + field_type: Optional[str], + kwargs: Mapping[str, Any], + expected: Mapping[str, Any], ): - inputs = [AddedFieldDefinition(path=v[0], value=v[1], parameters={}) for v in field] + inputs = [AddedFieldDefinition(path=v[0], value=v[1], value_type=field_type, parameters={}) for v in field] assert AddFields(fields=inputs, parameters={"alas": "i live"}).transform(input_record, **kwargs) == expected diff --git a/airbyte-cdk/python/unit_tests/sources/embedded/test_embedded_integration.py b/airbyte-cdk/python/unit_tests/sources/embedded/test_embedded_integration.py index 28ff10bc8660..d2bad84128e2 100644 --- a/airbyte-cdk/python/unit_tests/sources/embedded/test_embedded_integration.py +++ b/airbyte-cdk/python/unit_tests/sources/embedded/test_embedded_integration.py @@ -35,13 +35,15 @@ def setUp(self): self.source_class = MagicMock() self.source = MagicMock() self.source_class.return_value = self.source - self.source.spec.return_value = ConnectorSpecification(connectionSpecification={ - "properties": { - "test": { - "type": "string", + self.source.spec.return_value = ConnectorSpecification( + connectionSpecification={ + "properties": { + "test": { + "type": "string", + } } } - }) + ) self.config = {"test": "abc"} self.integration = TestIntegration(self.source, self.config) self.stream1 = AirbyteStream( diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py b/airbyte-cdk/python/unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py index fc6656d5983e..7206d234939d 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/availability_strategy/test_default_file_based_availability_strategy.py @@ -11,6 +11,7 @@ ) from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig from airbyte_cdk.sources.file_based.config.jsonl_format import JsonlFormat +from airbyte_cdk.sources.file_based.exceptions import CustomFileBasedException from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser from airbyte_cdk.sources.file_based.remote_file import RemoteFile @@ -26,17 +27,18 @@ class DefaultFileBasedAvailabilityStrategyTest(unittest.TestCase): - def setUp(self) -> None: self._stream_reader = Mock(spec=AbstractFileBasedStreamReader) self._strategy = DefaultFileBasedAvailabilityStrategy(self._stream_reader) self._parser = Mock(spec=FileTypeParser) + self._parser.check_config.return_value = (True, None) self._stream = Mock(spec=AbstractFileBasedStream) self._stream.get_parser.return_value = self._parser self._stream.catalog_schema = _ANY_SCHEMA self._stream.config = _ANY_CONFIG self._stream.validation_policy = PropertyMock(validate_schema_before_sync=False) + self._stream.stream_reader = self._stream_reader def test_given_file_extension_does_not_match_when_check_availability_and_parsability_then_stream_is_still_available(self) -> None: """ @@ -44,9 +46,55 @@ def test_given_file_extension_does_not_match_when_check_availability_and_parsabi example we've seen was for JSONL parser but the file extension was just `.json`. Note that there we more than one record extracted from this stream so it's not just that the file is one JSON object """ - self._stream.list_files.return_value = [_FILE_WITH_UNKNOWN_EXTENSION] + self._stream.get_files.return_value = [_FILE_WITH_UNKNOWN_EXTENSION] self._parser.parse_records.return_value = [{"a record": 1}] is_available, reason = self._strategy.check_availability_and_parsability(self._stream, Mock(), Mock()) assert is_available + + def test_not_available_given_no_files(self) -> None: + """ + If no files are returned, then the stream is not available. + """ + self._stream.get_files.return_value = [] + + is_available, reason = self._strategy.check_availability_and_parsability(self._stream, Mock(), Mock()) + + assert not is_available + assert "No files were identified in the stream" in reason + + def test_parse_records_is_not_called_with_parser_max_n_files_for_parsability_set(self) -> None: + """ + If the stream parser sets parser_max_n_files_for_parsability to 0, then we should not call parse_records on it + """ + self._parser.parser_max_n_files_for_parsability = 0 + self._stream.get_files.return_value = [_FILE_WITH_UNKNOWN_EXTENSION] + + is_available, reason = self._strategy.check_availability_and_parsability(self._stream, Mock(), Mock()) + + assert is_available + assert not self._parser.parse_records.called + assert self._stream_reader.open_file.called + + def test_passing_config_check(self) -> None: + """ + Test if the DefaultFileBasedAvailabilityStrategy correctly handles the check_config method defined on the parser. + """ + self._parser.check_config.return_value = (False, "Ran into error") + is_available, error_message = self._strategy.check_availability_and_parsability(self._stream, Mock(), Mock()) + assert not is_available + assert "Ran into error" in error_message + + def test_catching_and_raising_custom_file_based_exception(self) -> None: + """ + Test if the DefaultFileBasedAvailabilityStrategy correctly handles the CustomFileBasedException + by raising a CheckAvailabilityError when the get_files method is called. + """ + # Mock the get_files method to raise CustomFileBasedException when called + self._stream.get_files.side_effect = CustomFileBasedException("Custom exception for testing.") + + # Invoke the check_availability_and_parsability method and check if it correctly handles the exception + is_available, error_message = self._strategy.check_availability_and_parsability(self._stream, Mock(), Mock()) + assert not is_available + assert "Custom exception for testing." in error_message diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/config/test_abstract_file_based_spec.py b/airbyte-cdk/python/unit_tests/sources/file_based/config/test_abstract_file_based_spec.py index 961a52d49782..dca44b86a944 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/config/test_abstract_file_based_spec.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/config/test_abstract_file_based_spec.py @@ -16,15 +16,10 @@ pytest.param(ParquetFormat, "parquet", None, id="test_parquet_format_is_a_valid_parquet_file_type"), pytest.param(AvroFormat, "avro", None, id="test_avro_format_is_a_valid_avro_file_type"), pytest.param(CsvFormat, "parquet", ValidationError, id="test_csv_format_is_not_a_valid_parquet_file_type"), - ] + ], ) def test_parquet_file_type_is_not_a_valid_csv_file_type(file_format: BaseModel, file_type: str, expected_error: Type[Exception]) -> None: - format_config = { - file_type: { - "filetype": file_type, - "decimal_as_float": True - } - } + format_config = {file_type: {"filetype": file_type, "decimal_as_float": True}} if expected_error: with pytest.raises(expected_error): diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/config/test_csv_format.py b/airbyte-cdk/python/unit_tests/sources/file_based/config/test_csv_format.py new file mode 100644 index 000000000000..4b1b2bb9fcad --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/config/test_csv_format.py @@ -0,0 +1,28 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import unittest + +import pytest +from airbyte_cdk.sources.file_based.config.csv_format import CsvHeaderAutogenerated, CsvHeaderFromCsv, CsvHeaderUserProvided +from pydantic import ValidationError + + +class CsvHeaderDefinitionTest(unittest.TestCase): + def test_given_user_provided_and_not_column_names_provided_then_raise_exception(self) -> None: + with pytest.raises(ValidationError): + CsvHeaderUserProvided(column_names=[]) + + def test_given_user_provided_and_column_names_then_config_is_valid(self) -> None: + # no error means that this test succeeds + CsvHeaderUserProvided(column_names=["1", "2", "3"]) + + def test_given_user_provided_then_csv_does_not_have_header_row(self) -> None: + assert not CsvHeaderUserProvided(column_names=["1", "2", "3"]).has_header_row() + + def test_given_autogenerated_then_csv_does_not_have_header_row(self) -> None: + assert not CsvHeaderAutogenerated().has_header_row() + + def test_given_from_csv_then_csv_has_header_row(self) -> None: + assert CsvHeaderFromCsv().has_header_row() diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/config/test_file_based_stream_config.py b/airbyte-cdk/python/unit_tests/sources/file_based/config/test_file_based_stream_config.py index ebd6a2571d0d..9158720c31e6 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/config/test_file_based_stream_config.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/config/test_file_based_stream_config.py @@ -12,23 +12,45 @@ @pytest.mark.parametrize( "file_type, input_format, expected_format, expected_error", [ - pytest.param("csv", {"filetype": "csv", "delimiter": "d", "quote_char": "q", "escape_char": "e", "encoding": "ascii", "double_quote": True}, {"filetype": "csv", "delimiter": "d", "quote_char": "q", "escape_char": "e", "encoding": "ascii", "double_quote": True}, None, id="test_valid_format"), - pytest.param("csv", {"filetype": "csv", "double_quote": False}, {"delimiter": ",", "quote_char": "\"", "encoding": "utf8", "double_quote": False}, None, id="test_default_format_values"), - pytest.param("csv", {"filetype": "csv", "delimiter": "nope", "double_quote": True}, None, ValidationError, id="test_invalid_delimiter"), - pytest.param("csv", {"filetype": "csv", "quote_char": "nope", "double_quote": True}, None, ValidationError, id="test_invalid_quote_char"), - pytest.param("csv", {"filetype": "csv", "escape_char": "nope", "double_quote": True}, None, ValidationError, id="test_invalid_escape_char"), - pytest.param("csv", {"filetype": "csv", "delimiter": ",", "quote_char": "\"", "encoding": "not_a_format", "double_quote": True}, {}, ValidationError, id="test_invalid_encoding_type"), - pytest.param("invalid", {"filetype": "invalid", "double_quote": False}, {}, ValidationError, id="test_config_format_file_type_mismatch"), - ] + pytest.param( + "csv", + {"filetype": "csv", "delimiter": "d", "quote_char": "q", "escape_char": "e", "encoding": "ascii", "double_quote": True}, + {"filetype": "csv", "delimiter": "d", "quote_char": "q", "escape_char": "e", "encoding": "ascii", "double_quote": True}, + None, + id="test_valid_format", + ), + pytest.param( + "csv", + {"filetype": "csv", "double_quote": False}, + {"delimiter": ",", "quote_char": '"', "encoding": "utf8", "double_quote": False}, + None, + id="test_default_format_values", + ), + pytest.param( + "csv", {"filetype": "csv", "delimiter": "nope", "double_quote": True}, None, ValidationError, id="test_invalid_delimiter" + ), + pytest.param( + "csv", {"filetype": "csv", "quote_char": "nope", "double_quote": True}, None, ValidationError, id="test_invalid_quote_char" + ), + pytest.param( + "csv", {"filetype": "csv", "escape_char": "nope", "double_quote": True}, None, ValidationError, id="test_invalid_escape_char" + ), + pytest.param( + "csv", + {"filetype": "csv", "delimiter": ",", "quote_char": '"', "encoding": "not_a_format", "double_quote": True}, + {}, + ValidationError, + id="test_invalid_encoding_type", + ), + pytest.param( + "invalid", {"filetype": "invalid", "double_quote": False}, {}, ValidationError, id="test_config_format_file_type_mismatch" + ), + ], ) -def test_csv_config(file_type: str, input_format: Mapping[str, Any], expected_format: Mapping[str, Any], expected_error: Type[Exception]) -> None: - stream_config = { - "name": "stream1", - "file_type": file_type, - "globs": ["*"], - "validation_policy": "Emit Record", - "format": input_format - } +def test_csv_config( + file_type: str, input_format: Mapping[str, Any], expected_format: Mapping[str, Any], expected_error: Type[Exception] +) -> None: + stream_config = {"name": "stream1", "file_type": file_type, "globs": ["*"], "validation_policy": "Emit Record", "format": input_format} if expected_error: with pytest.raises(expected_error): diff --git a/buildSrc/settings.gradle b/airbyte-cdk/python/unit_tests/sources/file_based/discovery_policy/__init__.py similarity index 100% rename from buildSrc/settings.gradle rename to airbyte-cdk/python/unit_tests/sources/file_based/discovery_policy/__init__.py diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/discovery_policy/test_default_discovery_policy.py b/airbyte-cdk/python/unit_tests/sources/file_based/discovery_policy/test_default_discovery_policy.py new file mode 100644 index 000000000000..b7ad67115cff --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/discovery_policy/test_default_discovery_policy.py @@ -0,0 +1,31 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import unittest +from unittest.mock import Mock + +from airbyte_cdk.sources.file_based.discovery_policy.default_discovery_policy import DefaultDiscoveryPolicy +from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser + + +class DefaultDiscoveryPolicyTest(unittest.TestCase): + def setUp(self) -> None: + self._policy = DefaultDiscoveryPolicy() + + self._parser = Mock(spec=FileTypeParser) + self._parser.parser_max_n_files_for_schema_inference = None + + def test_hardcoded_schema_inference_file_limit_is_returned(self) -> None: + """ + If the parser is not providing a limit, then we should use the hardcoded limit + """ + assert self._policy.get_max_n_files_for_schema_inference(self._parser) == 10 + + def test_parser_limit_is_respected(self) -> None: + """ + If the parser is providing a limit, then we should use that limit + """ + self._parser.parser_max_n_files_for_schema_inference = 1 + + assert self._policy.get_max_n_files_for_schema_inference(self._parser) == 1 diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_avro_parser.py b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_avro_parser.py index 106e9dc1e68c..c5c1242b895d 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_avro_parser.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_avro_parser.py @@ -11,6 +11,7 @@ _default_avro_format = AvroFormat() _double_as_string_avro_format = AvroFormat(double_as_string=True) +_uuid_value = uuid.uuid4() @pytest.mark.parametrize( @@ -81,8 +82,9 @@ None, id="test_record_with_nested_record", ), - pytest.param(_default_avro_format, {"type": "array", "items": "float"}, {"type": "array", "items": {"type": "number"}}, None, - id="test_array"), + pytest.param( + _default_avro_format, {"type": "array", "items": "float"}, {"type": "array", "items": {"type": "number"}}, None, id="test_array" + ), pytest.param( _default_avro_format, {"type": "array", "items": {"type": "record", "name": "SubRecord", "fields": [{"name": "precise", "type": "double"}]}}, @@ -99,8 +101,9 @@ id="test_array_of_records", ), pytest.param(_default_avro_format, {"type": "array", "not_items": "string"}, None, ValueError, id="test_array_missing_items"), - pytest.param(_default_avro_format, {"type": "array", "items": "invalid_avro_type"}, None, ValueError, - id="test_array_invalid_item_type"), + pytest.param( + _default_avro_format, {"type": "array", "items": "invalid_avro_type"}, None, ValueError, id="test_array_invalid_item_type" + ), pytest.param( _default_avro_format, {"type": "enum", "name": "IMF", "symbols": ["Ethan", "Benji", "Luther"]}, @@ -109,11 +112,15 @@ id="test_enum", ), pytest.param(_default_avro_format, {"type": "enum", "name": "IMF"}, None, ValueError, id="test_enum_missing_symbols"), - pytest.param(_default_avro_format, {"type": "enum", "symbols": ["mission", "not", "accepted"]}, None, ValueError, - id="test_enum_missing_name"), + pytest.param( + _default_avro_format, {"type": "enum", "symbols": ["mission", "not", "accepted"]}, None, ValueError, id="test_enum_missing_name" + ), pytest.param( _default_avro_format, - {"type": "map", "values": "int"}, {"type": "object", "additionalProperties": {"type": "integer"}}, None, id="test_map" + {"type": "map", "values": "int"}, + {"type": "object", "additionalProperties": {"type": "integer"}}, + None, + id="test_map", ), pytest.param( _default_avro_format, @@ -125,11 +132,15 @@ pytest.param(_default_avro_format, {"type": "map"}, None, ValueError, id="test_map_missing_values"), pytest.param( _default_avro_format, - {"type": "fixed", "name": "limit", "size": 12}, {"type": "string", "pattern": "^[0-9A-Fa-f]{24}$"}, None, id="test_fixed" + {"type": "fixed", "name": "limit", "size": 12}, + {"type": "string", "pattern": "^[0-9A-Fa-f]{24}$"}, + None, + id="test_fixed", ), pytest.param(_default_avro_format, {"type": "fixed", "name": "limit"}, None, ValueError, id="test_fixed_missing_size"), - pytest.param(_default_avro_format, {"type": "fixed", "name": "limit", "size": "50"}, None, ValueError, - id="test_fixed_size_not_integer"), + pytest.param( + _default_avro_format, {"type": "fixed", "name": "limit", "size": "50"}, None, ValueError, id="test_fixed_size_not_integer" + ), # Logical types pytest.param( _default_avro_format, @@ -138,30 +149,52 @@ None, id="test_decimal", ), - pytest.param(_default_avro_format, {"type": "bytes", "logicalType": "decimal", "scale": 4}, None, ValueError, - id="test_decimal_missing_precision"), - pytest.param(_default_avro_format, {"type": "bytes", "logicalType": "decimal", "precision": 9}, None, ValueError, - id="test_decimal_missing_scale"), - pytest.param(_default_avro_format, {"type": "bytes", "logicalType": "uuid"}, {"type": ["null", "string"]}, None, id="test_uuid"), - pytest.param(_default_avro_format, {"type": "int", "logicalType": "date"}, {"type": ["null", "string"], "format": "date"}, None, - id="test_date"), - pytest.param(_default_avro_format, {"type": "int", "logicalType": "time-millis"}, {"type": ["null", "integer"]}, None, id="test_time_millis"), - pytest.param(_default_avro_format, {"type": "long", "logicalType": "time-micros"}, {"type": ["null", "integer"]}, None, - id="test_time_micros"), pytest.param( _default_avro_format, - {"type": "long", "logicalType": "timestamp-millis"}, {"type": ["null", "string"], "format": "date-time"}, None, id="test_timestamp_millis" + {"type": "bytes", "logicalType": "decimal", "scale": 4}, + None, + ValueError, + id="test_decimal_missing_precision", ), - pytest.param(_default_avro_format, {"type": "long", "logicalType": "timestamp-micros"}, {"type": ["null", "string"]}, None, - id="test_timestamp_micros"), pytest.param( _default_avro_format, - {"type": "long", "logicalType": "local-timestamp-millis"}, {"type": "string", "format": "date-time"}, None, - id="test_local_timestamp_millis" + {"type": "bytes", "logicalType": "decimal", "precision": 9}, + None, + ValueError, + id="test_decimal_missing_scale", + ), + pytest.param(_default_avro_format, {"type": "bytes", "logicalType": "uuid"}, {"type": "string"}, None, id="test_uuid"), + pytest.param( + _default_avro_format, {"type": "int", "logicalType": "date"}, {"type": "string", "format": "date"}, None, id="test_date" + ), + pytest.param(_default_avro_format, {"type": "int", "logicalType": "time-millis"}, {"type": "integer"}, None, id="test_time_millis"), + pytest.param( + _default_avro_format, {"type": "long", "logicalType": "time-micros"}, {"type": "integer"}, None, id="test_time_micros" + ), + pytest.param( + _default_avro_format, + {"type": "long", "logicalType": "timestamp-millis"}, + {"type": "string", "format": "date-time"}, + None, + id="test_timestamp_millis", + ), + pytest.param( + _default_avro_format, {"type": "long", "logicalType": "timestamp-micros"}, {"type": "string"}, None, id="test_timestamp_micros" + ), + pytest.param( + _default_avro_format, + {"type": "long", "logicalType": "local-timestamp-millis"}, + {"type": "string", "format": "date-time"}, + None, + id="test_local_timestamp_millis", + ), + pytest.param( + _default_avro_format, + {"type": "long", "logicalType": "local-timestamp-micros"}, + {"type": "string"}, + None, + id="test_local_timestamp_micros", ), - pytest.param(_default_avro_format, {"type": "long", "logicalType": "local-timestamp-micros"}, {"type": "string"}, None, - id="test_local_timestamp_micros"), - ], ) def test_convert_primitive_avro_type_to_json(avro_format, avro_type, expected_json_type, expected_error): @@ -174,7 +207,8 @@ def test_convert_primitive_avro_type_to_json(avro_format, avro_type, expected_js @pytest.mark.parametrize( - "avro_format, record_type, record_value, expected_value", [ + "avro_format, record_type, record_value, expected_value", + [ pytest.param(_default_avro_format, "boolean", True, True, id="test_boolean"), pytest.param(_default_avro_format, "int", 123, 123, id="test_int"), pytest.param(_default_avro_format, "long", 123, 123, id="test_long"), @@ -184,25 +218,25 @@ def test_convert_primitive_avro_type_to_json(avro_format, avro_type, expected_js pytest.param(_default_avro_format, "bytes", b"hello world", b"hello world", id="test_bytes"), pytest.param(_default_avro_format, "string", "hello world", "hello world", id="test_string"), pytest.param(_default_avro_format, {"logicalType": "decimal"}, 3.1415, "3.1415", id="test_decimal"), - pytest.param(_default_avro_format, {"logicalType": "uuid"}, b"abcdefghijklmnop", uuid.UUID(bytes=b"abcdefghijklmnop"), id="test_uuid"), - pytest.param(_default_avro_format, - {"logicalType": "date"}, - datetime.date(2023, 8, 7), - "2023-08-07", - id="test_date"), + pytest.param(_default_avro_format, {"logicalType": "uuid"}, _uuid_value, str(_uuid_value), id="test_uuid"), + pytest.param(_default_avro_format, {"logicalType": "date"}, datetime.date(2023, 8, 7), "2023-08-07", id="test_date"), pytest.param(_default_avro_format, {"logicalType": "time-millis"}, 70267068, 70267068, id="test_time_millis"), pytest.param(_default_avro_format, {"logicalType": "time-micros"}, 70267068, 70267068, id="test_time_micros"), - pytest.param(_default_avro_format, - {"logicalType": "local-timestamp-millis"}, - datetime.datetime(2023, 8, 7, 19, 31, 7, 68000, tzinfo=datetime.timezone.utc), - "2023-08-07T19:31:07.068+00:00", - id="test_timestamp_millis"), - pytest.param(_default_avro_format, - {"logicalType": "local-timestamp-micros"}, - datetime.datetime(2023, 8, 7, 19, 31, 7, 68000, tzinfo=datetime.timezone.utc), - "2023-08-07T19:31:07.068000+00:00", - id="test_timestamo_micros"), - ] + pytest.param( + _default_avro_format, + {"logicalType": "local-timestamp-millis"}, + datetime.datetime(2023, 8, 7, 19, 31, 7, 68000, tzinfo=datetime.timezone.utc), + "2023-08-07T19:31:07.068+00:00", + id="test_timestamp_millis", + ), + pytest.param( + _default_avro_format, + {"logicalType": "local-timestamp-micros"}, + datetime.datetime(2023, 8, 7, 19, 31, 7, 68000, tzinfo=datetime.timezone.utc), + "2023-08-07T19:31:07.068000+00:00", + id="test_timestamo_micros", + ), + ], ) def test_to_output_value(avro_format, record_type, record_value, expected_value): parser = AvroParser() diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_csv_parser.py b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_csv_parser.py index de86ad200bc2..9596cd84c598 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_csv_parser.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_csv_parser.py @@ -13,12 +13,21 @@ from unittest.mock import Mock import pytest -from airbyte_cdk.sources.file_based.config.csv_format import DEFAULT_FALSE_VALUES, DEFAULT_TRUE_VALUES, CsvFormat, InferenceType +from airbyte_cdk.models import FailureType +from airbyte_cdk.sources.file_based.config.csv_format import ( + DEFAULT_FALSE_VALUES, + DEFAULT_TRUE_VALUES, + CsvFormat, + CsvHeaderAutogenerated, + CsvHeaderUserProvided, + InferenceType, +) from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig from airbyte_cdk.sources.file_based.exceptions import RecordParseError from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode from airbyte_cdk.sources.file_based.file_types.csv_parser import CsvParser, _CsvReader from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.utils.traced_exception import AirbyteTracedException PROPERTY_TYPES = { "col1": "null", @@ -162,7 +171,7 @@ def setUp(self) -> None: self._config.get_input_schema.return_value = None self._config.format = self._config_format - self._file = Mock(spec=RemoteFile) + self._file = RemoteFile(uri="a uri", last_modified=datetime.now()) self._stream_reader = Mock(spec=AbstractFileBasedStreamReader) self._logger = Mock(spec=logging.Logger) self._csv_reader = Mock(spec=_CsvReader) @@ -215,6 +224,12 @@ def test_given_big_file_when_infer_schema_then_stop_early(self) -> None: # since the type is number, we know the string at the end was not considered assert inferred_schema == {self._HEADER_NAME: {"type": "number"}} + def test_given_empty_csv_file_when_infer_schema_then_raise_config_error(self) -> None: + self._csv_reader.read_data.return_value = [] + with pytest.raises(AirbyteTracedException) as exception: + self._infer_schema() + assert exception.value.failure_type == FailureType.config_error + def _test_infer_schema(self, rows: List[str], expected_type: str) -> None: self._csv_reader.read_data.return_value = ({self._HEADER_NAME: row} for row in rows) inferred_schema = self._infer_schema() @@ -253,7 +268,7 @@ def setUp(self) -> None: self._config.name = self._CONFIG_NAME self._config.format = self._config_format - self._file = Mock(spec=RemoteFile) + self._file = RemoteFile(uri="a uri", last_modified=datetime.now()) self._stream_reader = Mock(spec=AbstractFileBasedStreamReader) self._logger = Mock(spec=logging.Logger) self._csv_reader = _CsvReader() @@ -278,13 +293,40 @@ def test_given_skip_rows_when_read_data_then_do_not_considered_prefixed_rows(sel assert list(data_generator) == [{"header": "a value"}, {"header": "another value"}] def test_given_autogenerated_headers_when_read_data_then_generate_headers_with_format_fX(self) -> None: - self._config_format.autogenerate_column_names = True + self._config_format.header_definition = CsvHeaderAutogenerated() self._stream_reader.open_file.return_value = CsvFileBuilder().with_data(["0,1,2,3,4,5,6"]).build() data_generator = self._read_data() assert list(data_generator) == [{"f0": "0", "f1": "1", "f2": "2", "f3": "3", "f4": "4", "f5": "5", "f6": "6"}] + def test_given_skip_row_before_and_after_and_autogenerated_headers_when_read_data_then_generate_headers_with_format_fX(self) -> None: + self._config_format.header_definition = CsvHeaderAutogenerated() + self._config_format.skip_rows_before_header = 1 + self._config_format.skip_rows_after_header = 2 + self._stream_reader.open_file.return_value = ( + CsvFileBuilder().with_data(["skip before", "skip after 1", "skip after 2", "0,1,2,3,4,5,6"]).build() + ) + + data_generator = self._read_data() + + assert list(data_generator) == [{"f0": "0", "f1": "1", "f2": "2", "f3": "3", "f4": "4", "f5": "5", "f6": "6"}] + + def test_given_user_provided_headers_when_read_data_then_use_user_provided_headers(self) -> None: + self._config_format.header_definition = CsvHeaderUserProvided(column_names=["first", "second", "third", "fourth"]) + self._stream_reader.open_file.return_value = CsvFileBuilder().with_data(["0,1,2,3"]).build() + + data_generator = self._read_data() + + assert list(data_generator) == [{"first": "0", "second": "1", "third": "2", "fourth": "3"}] + + def test_given_len_mistmatch_on_user_provided_headers_when_read_data_then_raise_error(self) -> None: + self._config_format.header_definition = CsvHeaderUserProvided(column_names=["missing", "one", "column"]) + self._stream_reader.open_file.return_value = CsvFileBuilder().with_data(["0,1,2,3"]).build() + + with pytest.raises(RecordParseError): + list(self._read_data()) + def test_given_skip_rows_after_header_when_read_data_then_do_not_parse_skipped_rows(self) -> None: self._config_format.skip_rows_after_header = 1 self._stream_reader.open_file.return_value = ( diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_parquet_parser.py b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_parquet_parser.py index 028be78b5b5e..984a782c5925 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_parquet_parser.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_parquet_parser.py @@ -42,12 +42,21 @@ pytest.param(pa.time64("us"), {"type": "string"}, _default_parquet_format, id="test_parquet_time64us"), pytest.param(pa.time64("ns"), {"type": "string"}, _default_parquet_format, id="test_parquet_time64us"), pytest.param(pa.timestamp("s"), {"type": "string", "format": "date-time"}, _default_parquet_format, id="test_parquet_timestamps_s"), - pytest.param(pa.timestamp("ms"), {"type": "string", "format": "date-time"}, _default_parquet_format, - id="test_parquet_timestamp_ms"), - pytest.param(pa.timestamp("s", "utc"), {"type": "string", "format": "date-time"}, _default_parquet_format, - id="test_parquet_timestamps_s_with_tz"), - pytest.param(pa.timestamp("ms", "est"), {"type": "string", "format": "date-time"}, _default_parquet_format, - id="test_parquet_timestamps_ms_with_tz"), + pytest.param( + pa.timestamp("ms"), {"type": "string", "format": "date-time"}, _default_parquet_format, id="test_parquet_timestamp_ms" + ), + pytest.param( + pa.timestamp("s", "utc"), + {"type": "string", "format": "date-time"}, + _default_parquet_format, + id="test_parquet_timestamps_s_with_tz", + ), + pytest.param( + pa.timestamp("ms", "est"), + {"type": "string", "format": "date-time"}, + _default_parquet_format, + id="test_parquet_timestamps_ms_with_tz", + ), pytest.param(pa.date32(), {"type": "string", "format": "date"}, _default_parquet_format, id="test_parquet_date32"), pytest.param(pa.date64(), {"type": "string", "format": "date"}, _default_parquet_format, id="test_parquet_date64"), pytest.param(pa.duration("s"), {"type": "integer"}, _default_parquet_format, id="test_duration_s"), @@ -72,7 +81,7 @@ pytest.param(pa.decimal256(2), {"type": "number"}, _decimal_as_float_parquet_format, id="test_decimal256_as_float"), pytest.param(pa.map_(pa.int32(), pa.int32()), {"type": "object"}, _default_parquet_format, id="test_map"), pytest.param(pa.null(), {"type": "null"}, _default_parquet_format, id="test_null"), - ] + ], ) def test_type_mapping(parquet_type: pa.DataType, expected_type: Mapping[str, str], parquet_format: ParquetFormat) -> None: if expected_type is None: @@ -100,23 +109,47 @@ def test_type_mapping(parquet_type: pa.DataType, expected_type: Mapping[str, str pytest.param(pa.time32("ms"), _default_parquet_format, datetime.time(3, 4, 5), "03:04:05", id="test_parquet_time32ms"), pytest.param(pa.time64("us"), _default_parquet_format, datetime.time(6, 7, 8), "06:07:08", id="test_parquet_time64us"), pytest.param(pa.time64("ns"), _default_parquet_format, datetime.time(9, 10, 11), "09:10:11", id="test_parquet_time64us"), - pytest.param(pa.timestamp("s"), _default_parquet_format, datetime.datetime(2023, 7, 7, 10, 11, 12), "2023-07-07T10:11:12", - id="test_parquet_timestamps_s"), - pytest.param(pa.timestamp("ms"), _default_parquet_format, datetime.datetime(2024, 8, 8, 11, 12, 13), "2024-08-08T11:12:13", - id="test_parquet_timestamp_ms"), - pytest.param(pa.timestamp("s", "utc"), _default_parquet_format, - datetime.datetime(2020, 1, 1, 1, 1, 1, tzinfo=datetime.timezone.utc), - "2020-01-01T01:01:01+00:00", id="test_parquet_timestamps_s_with_tz"), - pytest.param(pa.timestamp("ms", "utc"), _default_parquet_format, datetime.datetime(2021, 2, 3, 4, 5, tzinfo=datetime.timezone.utc), - "2021-02-03T04:05:00+00:00", id="test_parquet_timestamps_ms_with_tz"), + pytest.param( + pa.timestamp("s"), + _default_parquet_format, + datetime.datetime(2023, 7, 7, 10, 11, 12), + "2023-07-07T10:11:12", + id="test_parquet_timestamps_s", + ), + pytest.param( + pa.timestamp("ms"), + _default_parquet_format, + datetime.datetime(2024, 8, 8, 11, 12, 13), + "2024-08-08T11:12:13", + id="test_parquet_timestamp_ms", + ), + pytest.param( + pa.timestamp("s", "utc"), + _default_parquet_format, + datetime.datetime(2020, 1, 1, 1, 1, 1, tzinfo=datetime.timezone.utc), + "2020-01-01T01:01:01+00:00", + id="test_parquet_timestamps_s_with_tz", + ), + pytest.param( + pa.timestamp("ms", "utc"), + _default_parquet_format, + datetime.datetime(2021, 2, 3, 4, 5, tzinfo=datetime.timezone.utc), + "2021-02-03T04:05:00+00:00", + id="test_parquet_timestamps_ms_with_tz", + ), pytest.param(pa.date32(), _default_parquet_format, datetime.date(2023, 7, 7), "2023-07-07", id="test_parquet_date32"), pytest.param(pa.date64(), _default_parquet_format, datetime.date(2023, 7, 8), "2023-07-08", id="test_parquet_date64"), pytest.param(pa.duration("s"), _default_parquet_format, 12345, 12345, id="test_duration_s"), pytest.param(pa.duration("ms"), _default_parquet_format, 12345, 12345, id="test_duration_ms"), pytest.param(pa.duration("us"), _default_parquet_format, 12345, 12345, id="test_duration_us"), pytest.param(pa.duration("ns"), _default_parquet_format, 12345, 12345, id="test_duration_ns"), - pytest.param(pa.month_day_nano_interval(), _default_parquet_format, datetime.timedelta(days=3, microseconds=4), [0, 3, 4000], - id="test_parquet_month_day_nano_interval"), + pytest.param( + pa.month_day_nano_interval(), + _default_parquet_format, + datetime.timedelta(days=3, microseconds=4), + [0, 3, 4000], + id="test_parquet_month_day_nano_interval", + ), pytest.param(pa.binary(), _default_parquet_format, b"this is a binary string", "this is a binary string", id="test_binary"), pytest.param(pa.binary(2), _default_parquet_format, b"t1", "t1", id="test_fixed_size_binary"), pytest.param(pa.string(), _default_parquet_format, "this is a string", "this is a string", id="test_parquet_string"), @@ -131,13 +164,15 @@ def test_type_mapping(parquet_type: pa.DataType, expected_type: Mapping[str, str pytest.param(pa.decimal256(8, 2), _default_parquet_format, 13, "13.00", id="test_decimal256"), pytest.param(pa.decimal128(5, 3), _decimal_as_float_parquet_format, 12, 12.000, id="test_decimal128"), pytest.param(pa.decimal256(8, 2), _decimal_as_float_parquet_format, 13, 13.00, id="test_decimal256"), - pytest.param(pa.map_(pa.string(), pa.int32()), _default_parquet_format, {"hello": 1, "world": 2}, {"hello": 1, "world": 2}, - id="test_map"), + pytest.param( + pa.map_(pa.string(), pa.int32()), _default_parquet_format, {"hello": 1, "world": 2}, {"hello": 1, "world": 2}, id="test_map" + ), pytest.param(pa.null(), _default_parquet_format, None, None, id="test_null"), - ] + ], ) -def test_value_transformation(pyarrow_type: pa.DataType, parquet_format: ParquetFormat, parquet_object: Scalar, - expected_value: Any) -> None: +def test_value_transformation( + pyarrow_type: pa.DataType, parquet_format: ParquetFormat, parquet_object: Scalar, expected_value: Any +) -> None: pyarrow_value = pa.array([parquet_object], type=pyarrow_type)[0] py_value = ParquetParser._to_output_value(pyarrow_value, parquet_format) if isinstance(py_value, float): @@ -156,25 +191,39 @@ def test_value_dictionary() -> None: assert py_value == {"indices": [0, 1, 2, 0, 1], "values": ["apple", "banana", "cherry"]} +def test_value_none_binary() -> None: + none_binary_scalar = pa.scalar(None, type=pa.binary()) + try: + ParquetParser._to_output_value(none_binary_scalar, _default_parquet_format) + except AttributeError: + assert False, "`None` type binary should be handled properly" + + @pytest.mark.parametrize( - "file_format", [ - pytest.param(CsvFormat( - filetype="csv", - delimiter=",", - escape_char="\\", - quote_char='"', - ), id="test_csv_format"), + "file_format", + [ + pytest.param( + CsvFormat( + filetype="csv", + delimiter=",", + escape_char="\\", + quote_char='"', + ), + id="test_csv_format", + ), pytest.param(JsonlFormat(), id="test_jsonl_format"), - ] + ], ) def test_wrong_file_format(file_format: Union[CsvFormat, JsonlFormat]) -> None: parser = ParquetParser() - config = FileBasedStreamConfig(name="test.parquet", file_type=file_format.filetype, format={file_format.filetype: file_format}, - validation_policy=ValidationPolicy.emit_record) + config = FileBasedStreamConfig( + name="test.parquet", + file_type=file_format.filetype, + format={file_format.filetype: file_format}, + validation_policy=ValidationPolicy.emit_record, + ) file = RemoteFile(uri="s3://mybucket/test.parquet", last_modified=datetime.datetime.now()) stream_reader = Mock() logger = Mock() with pytest.raises(ValueError): - asyncio.get_event_loop().run_until_complete( - parser.infer_schema(config, file, stream_reader, logger) - ) + asyncio.get_event_loop().run_until_complete(parser.infer_schema(config, file, stream_reader, logger)) diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_unstructured_parser.py b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_unstructured_parser.py new file mode 100644 index 000000000000..311d4a0ad158 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_unstructured_parser.py @@ -0,0 +1,532 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import asyncio +from datetime import datetime +from unittest import mock +from unittest.mock import MagicMock, call, mock_open, patch + +import pytest +import requests +from airbyte_cdk.models import FailureType +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig +from airbyte_cdk.sources.file_based.config.unstructured_format import APIParameterConfigModel, APIProcessingConfigModel, UnstructuredFormat +from airbyte_cdk.sources.file_based.exceptions import RecordParseError +from airbyte_cdk.sources.file_based.file_types import UnstructuredParser +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.utils.traced_exception import AirbyteTracedException +from unstructured.documents.elements import ElementMetadata, Formula, ListItem, Text, Title +from unstructured.file_utils.filetype import FileType + +FILE_URI = "path/to/file.xyz" + + +@pytest.mark.parametrize( + "filetype, format_config, raises", + [ + pytest.param( + FileType.MD, + UnstructuredFormat(skip_unprocessable_files=False), + False, + id="markdown_file", + ), + pytest.param( + FileType.CSV, + UnstructuredFormat(skip_unprocessable_files=False), + True, + id="wrong_file_format", + ), + pytest.param( + FileType.CSV, + UnstructuredFormat(skip_unprocessable_files=True), + False, + id="wrong_file_format_skipping", + ), + pytest.param( + FileType.PDF, + UnstructuredFormat(skip_unprocessable_files=False), + False, + id="pdf_file", + ), + pytest.param( + FileType.DOCX, + UnstructuredFormat(skip_unprocessable_files=False), + False, + id="docx_file", + ), + pytest.param( + FileType.PPTX, + UnstructuredFormat(skip_unprocessable_files=False), + False, + id="pptx_file", + ), + ], +) +@patch("airbyte_cdk.sources.file_based.file_types.unstructured_parser.detect_filetype") +def test_infer_schema(mock_detect_filetype, filetype, format_config, raises): + # use a fresh event loop to avoid leaking into other tests + main_loop = asyncio.get_event_loop() + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + stream_reader = MagicMock() + mock_open(stream_reader.open_file) + fake_file = MagicMock() + fake_file.uri = FILE_URI + logger = MagicMock() + mock_detect_filetype.return_value = filetype + config = MagicMock() + config.format = format_config + if raises: + with pytest.raises(RecordParseError): + loop.run_until_complete(UnstructuredParser().infer_schema(config, fake_file, stream_reader, logger)) + else: + schema = loop.run_until_complete(UnstructuredParser().infer_schema(config, MagicMock(), MagicMock(), MagicMock())) + assert schema == { + "content": {"type": "string", "description": "Content of the file as markdown. Might be null if the file could not be parsed"}, + "document_key": {"type": "string", "description": "Unique identifier of the document, e.g. the file path"}, + "_ab_source_file_parse_error": {"type": "string", "description": "Error message if the file could not be parsed even though the file is supported"}, + } + loop.close() + asyncio.set_event_loop(main_loop) + + +@pytest.mark.parametrize( + "filetype, format_config, parse_result, raises, expected_records, parsing_error", + [ + pytest.param( + FileType.MD, + UnstructuredFormat(skip_unprocessable_files=False), + "test", + False, + [ + { + "content": "test", + "document_key": FILE_URI, + "_ab_source_file_parse_error": None, + } + ], + False, + id="markdown_file", + ), + pytest.param( + FileType.CSV, + UnstructuredFormat(skip_unprocessable_files=False), + None, + True, + None, + False, + id="wrong_file_format", + ), + pytest.param( + FileType.CSV, + UnstructuredFormat(skip_unprocessable_files=True), + None, + False, + [ + { + "content": None, + "document_key": FILE_URI, + "_ab_source_file_parse_error": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. Contact Support if you need assistance.\nfilename=path/to/file.xyz message=File type FileType.CSV is not supported. Supported file types are FileType.MD, FileType.PDF, FileType.DOCX, FileType.PPTX, FileType.TXT", + } + ], + False, + id="skip_unprocessable_files", + ), + pytest.param( + FileType.PDF, + UnstructuredFormat(skip_unprocessable_files=False), + [ + Title("heading"), + Text("This is the text"), + ListItem("This is a list item"), + Formula("This is a formula"), + ], + False, + [ + { + "content": "# heading\n\nThis is the text\n\n- This is a list item\n\n```\nThis is a formula\n```", + "document_key": FILE_URI, + "_ab_source_file_parse_error": None, + } + ], + False, + id="pdf_file", + ), + pytest.param( + FileType.PDF, + UnstructuredFormat(skip_unprocessable_files=False), + [ + Title("first level heading", metadata=ElementMetadata(category_depth=1)), + Title("second level heading", metadata=ElementMetadata(category_depth=2)), + ], + False, + [ + { + "content": "# first level heading\n\n## second level heading", + "document_key": FILE_URI, + "_ab_source_file_parse_error": None, + } + ], + False, + id="multi_level_headings", + ), + pytest.param( + FileType.DOCX, + UnstructuredFormat(skip_unprocessable_files=False), + [ + Title("heading"), + Text("This is the text"), + ListItem("This is a list item"), + Formula("This is a formula"), + ], + False, + [ + { + "content": "# heading\n\nThis is the text\n\n- This is a list item\n\n```\nThis is a formula\n```", + "document_key": FILE_URI, + "_ab_source_file_parse_error": None, + } + ], + False, + id="docx_file", + ), + pytest.param( + FileType.DOCX, + UnstructuredFormat(skip_unprocessable_files=True), + "", + False, + [ + { + "content": None, + "document_key": FILE_URI, + "_ab_source_file_parse_error": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. Contact Support if you need assistance.\nfilename=path/to/file.xyz message=weird parsing error" + } + ], + True, + id="exception_during_parsing", + ), + ], +) +@patch("unstructured.partition.pdf.partition_pdf") +@patch("unstructured.partition.pptx.partition_pptx") +@patch("unstructured.partition.docx.partition_docx") +@patch("airbyte_cdk.sources.file_based.file_types.unstructured_parser.detect_filetype") +def test_parse_records( + mock_detect_filetype, + mock_partition_docx, + mock_partition_pptx, + mock_partition_pdf, + filetype, + format_config, + parse_result, + raises, + expected_records, + parsing_error, +): + stream_reader = MagicMock() + mock_open(stream_reader.open_file, read_data=bytes(str(parse_result), "utf-8")) + fake_file = RemoteFile(uri=FILE_URI, last_modified=datetime.now()) + fake_file.uri = FILE_URI + logger = MagicMock() + config = MagicMock() + config.format = format_config + mock_detect_filetype.return_value = filetype + if parsing_error: + mock_partition_docx.side_effect = Exception("weird parsing error") + mock_partition_pptx.side_effect = Exception("weird parsing error") + mock_partition_pdf.side_effect = Exception("weird parsing error") + else: + mock_partition_docx.return_value = parse_result + mock_partition_pptx.return_value = parse_result + mock_partition_pdf.return_value = parse_result + if raises: + with pytest.raises(RecordParseError): + list(UnstructuredParser().parse_records(config, fake_file, stream_reader, logger, MagicMock())) + else: + assert list(UnstructuredParser().parse_records(config, fake_file, stream_reader, logger, MagicMock())) == expected_records + + +@pytest.mark.parametrize( + "format_config, raises_for_status, json_response, is_ok, expected_error", + [ + pytest.param( + UnstructuredFormat(skip_unprocessable_file_types=False), + False, + {"status": "ok"}, + True, + None, + id="local", + ), + pytest.param( + UnstructuredFormat(skip_unprocessable_file_types=False, strategy="fast"), + False, + {"status": "ok"}, + True, + None, + id="local_ok_strategy", + ), + pytest.param( + UnstructuredFormat(skip_unprocessable_file_types=False, strategy="hi_res"), + False, + {"status": "ok"}, + False, + "Hi-res strategy is not supported for local processing", + id="local_unsupported_strategy", + ), + pytest.param( + UnstructuredFormat(skip_unprocessable_file_types=False, processing=APIProcessingConfigModel(mode="api", api_key="test")), + False, + [{"type": "Title", "text": "Airbyte source connection test"}], + True, + None, + id="api_ok", + ), + pytest.param( + UnstructuredFormat(skip_unprocessable_file_types=False, processing=APIProcessingConfigModel(mode="api", api_key="test")), + True, + None, + False, + "API error", + id="api_error", + ), + pytest.param( + UnstructuredFormat(skip_unprocessable_file_types=False, processing=APIProcessingConfigModel(mode="api", api_key="test")), + False, + {"unexpected": "response"}, + False, + "Error", + id="unexpected_handling_error", + ), + ], +) +@patch("airbyte_cdk.sources.file_based.file_types.unstructured_parser.requests") +def test_check_config(requests_mock, format_config, raises_for_status, json_response, is_ok, expected_error): + mock_response = MagicMock() + mock_response.json.return_value = json_response + if raises_for_status: + mock_response.raise_for_status.side_effect = Exception("API error") + requests_mock.post.return_value = mock_response + result, error = UnstructuredParser().check_config(FileBasedStreamConfig(name="test", format=format_config)) + assert result == is_ok + if expected_error: + assert expected_error in error + + +@pytest.mark.parametrize( + "filetype, format_config, raises_for_status, file_content, json_response, expected_requests, raises, expected_records, http_status_code", + [ + pytest.param( + FileType.PDF, + UnstructuredFormat(skip_unprocessable_file_types=False, processing=APIProcessingConfigModel(mode="api", api_key="test")), + None, + "test", + [{"type": "Text", "text": "test"}], + [call("https://api.unstructured.io/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "auto"}, files={"files": ("filename", mock.ANY, "application/pdf")})], + False, + [ + { + "content": "test", + "document_key": FILE_URI, + "_ab_source_file_parse_error": None + } + ], + 200, + id="basic_request", + ), + pytest.param( + FileType.PDF, + UnstructuredFormat(skip_unprocessable_file_types=False, strategy="hi_res", processing=APIProcessingConfigModel(mode="api", api_key="test", api_url="http://localhost:8000", parameters=[APIParameterConfigModel(name="include_page_breaks", value="true"), APIParameterConfigModel(name="ocr_languages", value="eng"), APIParameterConfigModel(name="ocr_languages", value="kor")])), + None, + "test", + [{"type": "Text", "text": "test"}], + [call("http://localhost:8000/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "hi_res", "include_page_breaks": "true", "ocr_languages": ["eng", "kor"]}, files={"files": ("filename", mock.ANY, "application/pdf")})], + False, + [ + { + "content": "test", + "document_key": FILE_URI, + "_ab_source_file_parse_error": None + } + ], + 200, + id="request_with_params", + ), + pytest.param( + FileType.MD, + UnstructuredFormat(skip_unprocessable_file_types=False, processing=APIProcessingConfigModel(mode="api", api_key="test")), + None, + "# Mymarkdown", + None, + None, + False, + [ + { + "content": "# Mymarkdown", + "document_key": FILE_URI, + "_ab_source_file_parse_error": None + } + ], + 200, + id="handle_markdown_locally", + ), + pytest.param( + FileType.PDF, + UnstructuredFormat(skip_unprocessable_file_types=False, processing=APIProcessingConfigModel(mode="api", api_key="test")), + [ + requests.exceptions.RequestException("API error"), + requests.exceptions.RequestException("API error"), + requests.exceptions.RequestException("API error"), + requests.exceptions.RequestException("API error"), + requests.exceptions.RequestException("API error"), + ], + "test", + None, + [ + call("https://api.unstructured.io/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "auto"}, files={"files": ("filename", mock.ANY, "application/pdf")}), + call().raise_for_status(), + call("https://api.unstructured.io/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "auto"}, files={"files": ("filename", mock.ANY, "application/pdf")}), + call().raise_for_status(), + call("https://api.unstructured.io/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "auto"}, files={"files": ("filename", mock.ANY, "application/pdf")}), + call().raise_for_status(), + call("https://api.unstructured.io/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "auto"}, files={"files": ("filename", mock.ANY, "application/pdf")}), + call().raise_for_status(), + call("https://api.unstructured.io/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "auto"}, files={"files": ("filename", mock.ANY, "application/pdf")}), + call().raise_for_status(), + ], + True, + None, + 200, + id="retry_and_raise_on_api_error", + ), + pytest.param( + FileType.PDF, + UnstructuredFormat(skip_unprocessable_file_types=False, processing=APIProcessingConfigModel(mode="api", api_key="test")), + [ + requests.exceptions.RequestException("API error"), + requests.exceptions.RequestException("API error"), + None, + ], + "test", + [{"type": "Text", "text": "test"}], + [ + call("https://api.unstructured.io/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "auto"}, files={"files": ("filename", mock.ANY, "application/pdf")}), + call().raise_for_status(), + call("https://api.unstructured.io/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "auto"}, files={"files": ("filename", mock.ANY, "application/pdf")}), + call().raise_for_status(), + call("https://api.unstructured.io/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "auto"}, files={"files": ("filename", mock.ANY, "application/pdf")}), + call().raise_for_status(), + ], + False, + [ + { + "content": "test", + "document_key": FILE_URI, + "_ab_source_file_parse_error": None + } + ], + 200, + id="retry_and_recover", + ), + pytest.param( + FileType.PDF, + UnstructuredFormat(skip_unprocessable_file_types=False, processing=APIProcessingConfigModel(mode="api", api_key="test")), + [ + Exception("Unexpected error"), + ], + "test", + [{"type": "Text", "text": "test"}], + [ + call("https://api.unstructured.io/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "auto"}, files={"files": ("filename", mock.ANY, "application/pdf")}), + call().raise_for_status(), + ], + True, + None, + 200, + id="no_retry_on_unexpected_error", + ), + pytest.param( + FileType.PDF, + UnstructuredFormat(skip_unprocessable_file_types=False, processing=APIProcessingConfigModel(mode="api", api_key="test")), + [ + requests.exceptions.RequestException("API error", response=MagicMock(status_code=400)), + ], + "test", + [{"type": "Text", "text": "test"}], + [ + call("https://api.unstructured.io/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "auto"}, files={"files": ("filename", mock.ANY, "application/pdf")}), + call().raise_for_status(), + ], + True, + None, + 400, + id="no_retry_on_400_error", + ), + pytest.param( + FileType.PDF, + UnstructuredFormat(skip_unprocessable_file_types=False, processing=APIProcessingConfigModel(mode="api", api_key="test")), + None, + "test", + [{"detail": "Something went wrong"}], + [ + call("https://api.unstructured.io/general/v0/general", headers={"accept": "application/json", "unstructured-api-key": "test"}, data={"strategy": "auto"}, files={"files": ("filename", mock.ANY, "application/pdf")}), + ], + False, + [ + { + "content": None, + "document_key": FILE_URI, + "_ab_source_file_parse_error": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. Contact Support if you need assistance.\nfilename=path/to/file.xyz message=[{'detail': 'Something went wrong'}]", + } + ], + 422, + id="error_record_on_422_error", + ), + ], +) +@patch("airbyte_cdk.sources.file_based.file_types.unstructured_parser.requests") +@patch("airbyte_cdk.sources.file_based.file_types.unstructured_parser.detect_filetype") +@patch('time.sleep', side_effect=lambda _: None) +def test_parse_records_remotely( + time_mock, + mock_detect_filetype, + requests_mock, + filetype, + format_config, + raises_for_status, + file_content, + json_response, + expected_requests, + raises, + expected_records, + http_status_code +): + stream_reader = MagicMock() + mock_open(stream_reader.open_file, read_data=bytes(str(file_content), "utf-8")) + fake_file = RemoteFile(uri=FILE_URI, last_modified=datetime.now()) + fake_file.uri = FILE_URI + logger = MagicMock() + config = MagicMock() + config.format = format_config + mock_detect_filetype.return_value = filetype + mock_response = MagicMock() + mock_response.json.return_value = json_response + mock_response.status_code = http_status_code + if raises_for_status: + mock_response.raise_for_status.side_effect = raises_for_status + requests_mock.post.return_value = mock_response + requests_mock.exceptions.RequestException = requests.exceptions.RequestException + + if raises: + with pytest.raises(AirbyteTracedException) as exc: + list(UnstructuredParser().parse_records(config, fake_file, stream_reader, logger, MagicMock())) + # Failures from the API are treated as config errors + assert exc.value.failure_type == FailureType.config_error + else: + assert list(UnstructuredParser().parse_records(config, fake_file, stream_reader, logger, MagicMock())) == expected_records + + if expected_requests: + requests_mock.post.assert_has_calls(expected_requests) + else: + requests_mock.post.assert_not_called() diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/helpers.py b/airbyte-cdk/python/unit_tests/sources/file_based/helpers.py index 491b5658ad0c..d3e528e32105 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/helpers.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/helpers.py @@ -11,6 +11,7 @@ from airbyte_cdk.sources.file_based.discovery_policy import DefaultDiscoveryPolicy from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode from airbyte_cdk.sources.file_based.file_types.csv_parser import CsvParser +from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser from airbyte_cdk.sources.file_based.file_types.jsonl_parser import JsonlParser from airbyte_cdk.sources.file_based.remote_file import RemoteFile from airbyte_cdk.sources.file_based.schema_validation_policies import AbstractSchemaValidationPolicy @@ -19,13 +20,14 @@ class EmptySchemaParser(CsvParser): - async def infer_schema(self, config: FileBasedStreamConfig, file: RemoteFile, stream_reader: AbstractFileBasedStreamReader, logger: logging.Logger) -> Dict[str, Any]: + async def infer_schema( + self, config: FileBasedStreamConfig, file: RemoteFile, stream_reader: AbstractFileBasedStreamReader, logger: logging.Logger + ) -> Dict[str, Any]: return {} class LowInferenceLimitDiscoveryPolicy(DefaultDiscoveryPolicy): - @property - def max_n_files_for_schema_inference(self) -> int: + def get_max_n_files_for_schema_inference(self, parser: FileTypeParser) -> int: return 1 @@ -60,7 +62,4 @@ class LowHistoryLimitCursor(DefaultFileBasedCursor): def make_remote_files(files: List[str]) -> List[RemoteFile]: - return [ - RemoteFile(uri=f, last_modified=datetime.strptime("2023-06-05T03:54:07.000Z", "%Y-%m-%dT%H:%M:%S.%fZ")) - for f in files - ] + return [RemoteFile(uri=f, last_modified=datetime.strptime("2023-06-05T03:54:07.000Z", "%Y-%m-%dT%H:%M:%S.%fZ")) for f in files] diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/in_memory_files_source.py b/airbyte-cdk/python/unit_tests/sources/file_based/in_memory_files_source.py index ca25289bbf2c..643461471fd5 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/in_memory_files_source.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/in_memory_files_source.py @@ -88,16 +88,25 @@ def get_matching_files( prefix: Optional[str], logger: logging.Logger, ) -> Iterable[RemoteFile]: - yield from self.filter_files_by_globs_and_start_date([ - RemoteFile(uri=f, last_modified=datetime.strptime(data["last_modified"], "%Y-%m-%dT%H:%M:%S.%fZ")) - for f, data in self.files.items() - ], globs) + yield from self.filter_files_by_globs_and_start_date( + [ + RemoteFile( + uri=f, + mime_type=data.get("mime_type", None), + last_modified=datetime.strptime(data["last_modified"], "%Y-%m-%dT%H:%M:%S.%fZ"), + ) + for f, data in self.files.items() + ], + globs, + ) def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger) -> IOBase: if self.file_type == "csv": return self._make_csv_file_contents(file.uri) elif self.file_type == "jsonl": return self._make_jsonl_file_contents(file.uri) + elif self.file_type == "unstructured": + return self._make_binary_file_contents(file.uri) else: raise NotImplementedError(f"No implementation for file type: {self.file_type}") @@ -133,6 +142,13 @@ def _make_jsonl_file_contents(self, file_name: str) -> IOBase: fh.seek(0) return fh + def _make_binary_file_contents(self, file_name: str) -> IOBase: + fh = io.BytesIO() + + fh.write(self.files[file_name]["contents"]) + fh.seek(0) + return fh + class InMemorySpec(AbstractFileBasedSpec): @classmethod diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/avro_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/avro_scenarios.py index 27ef71fd9005..f1cdac5838b2 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/avro_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/avro_scenarios.py @@ -4,9 +4,9 @@ import datetime import decimal -import uuid from unit_tests.sources.file_based.in_memory_files_source import TemporaryAvroFilesStreamReader +from unit_tests.sources.file_based.scenarios.file_based_source_builder import FileBasedSourceBuilder from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder _single_avro_file = { @@ -95,7 +95,7 @@ {"name": "col_fixed", "type": {"type": "fixed", "name": "MyFixed", "size": 4}}, # Logical Types {"name": "col_decimal", "type": {"type": "bytes", "logicalType": "decimal", "precision": 10, "scale": 5}}, - {"name": "col_uuid", "type": {"type": "bytes", "logicalType": "uuid"}}, + {"name": "col_uuid", "type": {"type": "string", "logicalType": "uuid"}}, {"name": "col_date", "type": {"type": "int", "logicalType": "date"}}, {"name": "col_time_millis", "type": {"type": "int", "logicalType": "time-millis"}}, {"name": "col_time_micros", "type": {"type": "long", "logicalType": "time-micros"}}, @@ -124,7 +124,7 @@ {"lead_singer": "Matty Healy", "lead_guitar": "Adam Hann", "bass_guitar": "Ross MacDonald", "drummer": "George Daniel"}, b"\x12\x34\x56\x78", decimal.Decimal("1234.56789"), - uuid.UUID('123e4567-e89b-12d3-a456-426655440000').bytes, + "123e4567-e89b-12d3-a456-426655440000", datetime.date(2022, 5, 29), datetime.time(6, 0, 0, 456000), datetime.time(12, 0, 0, 456789), @@ -203,15 +203,18 @@ "streams": [ { "name": "stream1", - "file_type": "avro", + "format": {"filetype": "avro"}, "globs": ["*"], "validation_policy": "Emit Record", } ] } ) - .set_stream_reader(TemporaryAvroFilesStreamReader(files=_single_avro_file, file_type="avro")) - .set_file_type("avro") + .set_source_builder( + FileBasedSourceBuilder() + .set_stream_reader(TemporaryAvroFilesStreamReader(files=_single_avro_file, file_type="avro")) + .set_file_type("avro") + ) .set_expected_check_status("SUCCEEDED") .set_expected_records( [ @@ -266,15 +269,18 @@ "streams": [ { "name": "stream1", - "file_type": "avro", + "format": {"filetype": "avro"}, "globs": ["*"], "validation_policy": "Emit Record", } ] } ) - .set_stream_reader(TemporaryAvroFilesStreamReader(files=_multiple_avro_combine_schema_file, file_type="avro")) - .set_file_type("avro") + .set_source_builder( + FileBasedSourceBuilder() + .set_stream_reader(TemporaryAvroFilesStreamReader(files=_multiple_avro_combine_schema_file, file_type="avro")) + .set_file_type("avro") + ) .set_expected_records( [ { @@ -362,15 +368,18 @@ "streams": [ { "name": "stream1", - "file_type": "avro", + "format": {"filetype": "avro"}, "globs": ["*"], "validation_policy": "Emit Record", } ] } ) - .set_stream_reader(TemporaryAvroFilesStreamReader(files=_avro_all_types_file, file_type="avro")) - .set_file_type("avro") + .set_source_builder( + FileBasedSourceBuilder() + .set_stream_reader(TemporaryAvroFilesStreamReader(files=_avro_all_types_file, file_type="avro")) + .set_file_type("avro") + ) .set_expected_records( [ { @@ -431,7 +440,11 @@ "col_long": {"type": ["null", "integer"]}, "col_map": {"additionalProperties": {"type": ["null", "string"]}, "type": ["null", "object"]}, "col_record": { - "properties": {"artist": {"type": ["null", "string"]}, "song": {"type": ["null", "string"]}, "year": {"type": ["null", "integer"]}}, + "properties": { + "artist": {"type": ["null", "string"]}, + "song": {"type": ["null", "string"]}, + "year": {"type": ["null", "integer"]}, + }, "type": ["null", "object"], }, "col_string": {"type": ["null", "string"]}, @@ -463,21 +476,24 @@ "streams": [ { "name": "songs_stream", - "file_type": "avro", + "format": {"filetype": "avro"}, "globs": ["*_songs.avro"], "validation_policy": "Emit Record", }, { "name": "festivals_stream", - "file_type": "avro", + "format": {"filetype": "avro"}, "globs": ["*_festivals.avro"], "validation_policy": "Emit Record", }, ] } ) - .set_stream_reader(TemporaryAvroFilesStreamReader(files=_multiple_avro_stream_file, file_type="avro")) - .set_file_type("avro") + .set_source_builder( + FileBasedSourceBuilder() + .set_stream_reader(TemporaryAvroFilesStreamReader(files=_multiple_avro_stream_file, file_type="avro")) + .set_file_type("avro") + ) .set_expected_records( [ { @@ -586,7 +602,10 @@ "type": "object", "properties": { "col_title": {"type": ["null", "string"]}, - "col_album": {"type": ["null", "string"], "enum": ["SUMMERS_GONE", "IN_RETURN", "A_MOMENT_APART", "THE_LAST_GOODBYE"]}, + "col_album": { + "type": ["null", "string"], + "enum": ["SUMMERS_GONE", "IN_RETURN", "A_MOMENT_APART", "THE_LAST_GOODBYE"], + }, "col_year": {"type": ["null", "integer"]}, "col_vocals": {"type": ["null", "boolean"]}, "_ab_source_file_last_modified": {"type": "string"}, @@ -604,7 +623,11 @@ "properties": { "col_name": {"type": ["null", "string"]}, "col_location": { - "properties": {"country": {"type": ["null", "string"]}, "state": {"type": ["null", "string"]}, "city": {"type": ["null", "string"]}}, + "properties": { + "country": {"type": ["null", "string"]}, + "state": {"type": ["null", "string"]}, + "city": {"type": ["null", "string"]}, + }, "type": ["null", "object"], }, "col_attendance": {"type": ["null", "integer"]}, @@ -629,19 +652,18 @@ "streams": [ { "name": "stream1", - "file_type": "avro", "globs": ["*"], "validation_policy": "Emit Record", - "format": { - "filetype": "avro", - "double_as_string": False - } + "format": {"filetype": "avro", "double_as_string": False}, } ] } ) - .set_stream_reader(TemporaryAvroFilesStreamReader(files=_multiple_avro_combine_schema_file, file_type="avro")) - .set_file_type("avro") + .set_source_builder( + FileBasedSourceBuilder() + .set_stream_reader(TemporaryAvroFilesStreamReader(files=_multiple_avro_combine_schema_file, file_type="avro")) + .set_file_type("avro") + ) .set_expected_records( [ { diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/check_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/check_scenarios.py index 880046bb1ef2..26136d9cf025 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/check_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/check_scenarios.py @@ -8,6 +8,7 @@ TestErrorListMatchingFilesInMemoryFilesStreamReader, TestErrorOpenFileInMemoryFilesStreamReader, ) +from unit_tests.sources.file_based.scenarios.file_based_source_builder import FileBasedSourceBuilder from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder _base_success_scenario = ( @@ -17,34 +18,34 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "Emit Record", } ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11", "val12"), - ("val21", "val22"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_check_status("SUCCEEDED") ) -success_csv_scenario = ( - _base_success_scenario.copy() - .set_name("success_csv_scenario") -).build() +success_csv_scenario = (_base_success_scenario.copy().set_name("success_csv_scenario")).build() success_multi_stream_scenario = ( @@ -55,16 +56,16 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv", "*.gz"], "validation_policy": "Emit Record", }, { "name": "stream2", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv", "*.gz"], "validation_policy": "Emit Record", - } + }, ] } ) @@ -79,24 +80,26 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*"], "validation_policy": "Emit Record", } ] } ) - .set_files( - { - "a": { - "contents": [ - ("col1", "col2"), - ("val11", "val12"), - ("val21", "val22"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + _base_success_scenario.source_builder.copy().set_files( + { + "a": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) ) ).build() @@ -109,7 +112,7 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "Emit Record", "input_schema": '{"col1": "string", "col2": "string"}', @@ -120,16 +123,13 @@ ).build() -_base_failure_scenario = ( - _base_success_scenario.copy() - .set_expected_check_status("FAILED") -) +_base_failure_scenario = _base_success_scenario.copy().set_expected_check_status("FAILED") error_empty_stream_scenario = ( _base_failure_scenario.copy() .set_name("error_empty_stream_scenario") - .set_files({}) + .set_source_builder(_base_failure_scenario.copy().source_builder.copy().set_files({})) .set_expected_check_error(None, FileBasedSourceError.EMPTY_STREAM.value) ).build() @@ -137,7 +137,11 @@ error_listing_files_scenario = ( _base_failure_scenario.copy() .set_name("error_listing_files_scenario") - .set_stream_reader(TestErrorListMatchingFilesInMemoryFilesStreamReader(files=_base_failure_scenario._files, file_type="csv")) + .set_source_builder( + _base_failure_scenario.source_builder.copy().set_stream_reader( + TestErrorListMatchingFilesInMemoryFilesStreamReader(files=_base_failure_scenario.source_builder._files, file_type="csv") + ) + ) .set_expected_check_error(None, FileBasedSourceError.ERROR_LISTING_FILES.value) ).build() @@ -145,7 +149,11 @@ error_reading_file_scenario = ( _base_failure_scenario.copy() .set_name("error_reading_file_scenario") - .set_stream_reader(TestErrorOpenFileInMemoryFilesStreamReader(files=_base_failure_scenario._files, file_type="csv")) + .set_source_builder( + _base_failure_scenario.source_builder.copy().set_stream_reader( + TestErrorOpenFileInMemoryFilesStreamReader(files=_base_failure_scenario.source_builder._files, file_type="csv") + ) + ) .set_expected_check_error(None, FileBasedSourceError.ERROR_READING_FILE.value) ).build() @@ -158,7 +166,7 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "always_fail", "input_schema": '{"col1": "number", "col2": "string"}', @@ -166,7 +174,23 @@ ], } ) - .set_validation_policies({FailingSchemaValidationPolicy.ALWAYS_FAIL: FailingSchemaValidationPolicy()}) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } + } + ) + .set_file_type("csv") + .set_validation_policies({FailingSchemaValidationPolicy.ALWAYS_FAIL: FailingSchemaValidationPolicy()}) + ) .set_expected_check_error(None, FileBasedSourceError.ERROR_VALIDATING_RECORD.value) ).build() @@ -179,16 +203,16 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "Emit Record", }, { "name": "stream2", - "file_type": "jsonl", + "format": {"filetype": "jsonl"}, "globs": ["*.csv"], "validation_policy": "Emit Record", - } + }, ], } ) diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py index 585cb1b403d7..77164c83d8d8 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py @@ -2,19 +2,26 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError, SchemaInferenceError +from airbyte_cdk.models import AirbyteAnalyticsTraceMessage +from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat +from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.utils.traced_exception import AirbyteTracedException +from airbyte_protocol.models import SyncMode from unit_tests.sources.file_based.helpers import EmptySchemaParser, LowInferenceLimitDiscoveryPolicy -from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder +from unit_tests.sources.file_based.in_memory_files_source import InMemoryFilesSource +from unit_tests.sources.file_based.scenarios.file_based_source_builder import FileBasedSourceBuilder +from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenario, TestScenarioBuilder -single_csv_scenario = ( - TestScenarioBuilder() +single_csv_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("single_csv_scenario") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*"], "validation_policy": "Emit Record", } @@ -22,19 +29,22 @@ "start_date": "2023-06-04T03:54:07.000000Z", } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11", "val12"), - ("val21", "val22"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_spec( { "documentationUrl": "https://docs.airbyte.com/integrations/sources/in_memory_files", @@ -63,16 +73,13 @@ "type": "object", "properties": { "name": {"title": "Name", "description": "The name of the stream.", "type": "string"}, - "file_type": { - "title": "File Type", - "description": "The data file type that is being extracted for a stream.", - "type": "string", - }, "globs": { "title": "Globs", "description": 'The pattern used to specify which files should be selected from the file system. For more information on glob pattern matching look here.', "type": "array", "items": {"type": "string"}, + "order": 1, + "default": ["**"], }, "legacy_prefix": { "title": "Legacy Prefix", @@ -93,8 +100,9 @@ }, "primary_key": { "title": "Primary Key", - "description": "The column or columns (for a composite key) that serves as the unique identifier of a record.", + "description": "The column or columns (for a composite key) that serves as the unique identifier of a record. If empty, the primary key will default to the parser's default primary key.", "type": "string", + "airbyte_hidden": True, }, "days_to_sync_if_history_is_full": { "title": "Days To Sync If History Is Full", @@ -111,7 +119,7 @@ "title": "Avro Format", "type": "object", "properties": { - "filetype": {"title": "Filetype", "default": "avro", "enum": ["avro"], "type": "string"}, + "filetype": {"title": "Filetype", "default": "avro", "const": "avro", "type": "string"}, "double_as_string": { "title": "Convert Double Fields to Strings", "description": "Whether to convert double fields to strings. This is recommended if you have decimal numbers with a high degree of precision because there can be a loss precision when handling floating point numbers.", @@ -119,12 +127,13 @@ "type": "boolean", }, }, + "required": ["filetype"], }, { "title": "CSV Format", "type": "object", "properties": { - "filetype": {"title": "Filetype", "default": "csv", "enum": ["csv"], "type": "string"}, + "filetype": {"title": "Filetype", "default": "csv", "const": "csv", "type": "string"}, "delimiter": { "title": "Delimiter", "description": "The character delimiting individual cells in the CSV data. This may only be a 1-character string. For tab-delimited data enter '\\t'.", @@ -180,11 +189,58 @@ "default": 0, "type": "integer", }, - "autogenerate_column_names": { - "title": "Autogenerate Column Names", - "description": "Whether to autogenerate column names if column_names is empty. If true, column names will be of the form \u201cf0\u201d, \u201cf1\u201d\u2026 If false, column names will be read from the first CSV row after skip_rows_before_header.", - "default": False, - "type": "boolean", + "header_definition": { + "title": "CSV Header Definition", + "type": "object", + "description": "How headers will be defined. `User Provided` assumes the CSV does not have a header row and uses the headers provided and `Autogenerated` assumes the CSV does not have a header row and the CDK will generate headers using for `f{i}` where `i` is the index starting from 0. Else, the default behavior is to use the header from the CSV file. If a user wants to autogenerate or provide column names for a CSV having headers, they can skip rows.", + "default": {"header_definition_type": "From CSV"}, + "oneOf": [ + { + "title": "From CSV", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "From CSV", + "const": "From CSV", + "type": "string", + }, + }, + "required": ["header_definition_type"], + }, + { + "title": "Autogenerated", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "Autogenerated", + "const": "Autogenerated", + "type": "string", + }, + }, + "required": ["header_definition_type"], + }, + { + "title": "User Provided", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "User Provided", + "const": "User Provided", + "type": "string", + }, + "column_names": { + "title": "Column Names", + "description": "The column names that will be used while emitting the CSV records", + "type": "array", + "items": {"type": "string"}, + }, + }, + "required": ["column_names", "header_definition_type"], + }, + ], }, "true_values": { "title": "True Values", @@ -210,13 +266,15 @@ "enum": ["None", "Primitive Types Only"], }, }, + "required": ["filetype"], }, { "title": "Jsonl Format", "type": "object", "properties": { - "filetype": {"title": "Filetype", "default": "jsonl", "enum": ["jsonl"], "type": "string"} + "filetype": {"title": "Filetype", "default": "jsonl", "const": "jsonl", "type": "string"} }, + "required": ["filetype"], }, { "title": "Parquet Format", @@ -225,7 +283,7 @@ "filetype": { "title": "Filetype", "default": "parquet", - "enum": ["parquet"], + "const": "parquet", "type": "string", }, "decimal_as_float": { @@ -235,6 +293,138 @@ "type": "boolean", }, }, + "required": ["filetype"], + }, + { + "title": "Document File Type Format (Experimental)", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "unstructured", + "const": "unstructured", + "type": "string", + }, + "skip_unprocessable_files": { + "type": "boolean", + "default": True, + "title": "Skip Unprocessable Files", + "description": "If true, skip files that cannot be parsed and pass the error message along as the _ab_source_file_parse_error field. If false, fail the sync.", + "always_show": True, + }, + "strategy": { + "type": "string", + "always_show": True, + "order": 0, + "default": "auto", + "title": "Parsing Strategy", + "enum": ["auto", "fast", "ocr_only", "hi_res"], + "description": "The strategy used to parse documents. `fast` extracts text directly from the document which doesn't work for all files. `ocr_only` is more reliable, but slower. `hi_res` is the most reliable, but requires an API key and a hosted instance of unstructured and can't be used with local mode. See the unstructured.io documentation for more details: https://unstructured-io.github.io/unstructured/core/partition.html#partition-pdf", + }, + "processing": { + "title": "Processing", + "description": "Processing configuration", + "default": { + "mode": "local" + }, + "type": "object", + "oneOf": [ + { + "title": "Local", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "local", + "const": "local", + "enum": [ + "local" + ], + "type": "string" + } + }, + "description": "Process files locally, supporting `fast` and `ocr` modes. This is the default option.", + "required": [ + "mode" + ] + }, + { + "title": "via API", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "api", + "const": "api", + "enum": [ + "api" + ], + "type": "string" + }, + "api_key": { + "title": "API Key", + "description": "The API key to use matching the environment", + "default": "", + "always_show": True, + "airbyte_secret": True, + "type": "string" + }, + "api_url": { + "title": "API URL", + "description": "The URL of the unstructured API to use", + "default": "https://api.unstructured.io", + "always_show": True, + "examples": [ + "https://api.unstructured.com" + ], + "type": "string" + }, + "parameters": { + "title": "Additional URL Parameters", + "description": "List of parameters send to the API", + "default": [], + "always_show": True, + "type": "array", + "items": { + "title": "APIParameterConfigModel", + "type": "object", + "properties": { + "name": { + "title": "Parameter name", + "description": "The name of the unstructured API parameter to use", + "examples": [ + "combine_under_n_chars", + "languages" + ], + "type": "string" + }, + "value": { + "title": "Value", + "description": "The value of the parameter", + "examples": [ + "true", + "hi_res" + ], + "type": "string" + } + }, + "required": [ + "name", + "value" + ] + } + } + }, + "description": "Process files via an API, using the `hi_res` mode. This option is useful for increased performance and accuracy, but requires an API key and a hosted instance of unstructured.", + "required": [ + "mode" + ] + } + ] + }, + }, + "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file.", + "required": ["filetype"], }, ], }, @@ -245,7 +435,7 @@ "type": "boolean", }, }, - "required": ["name", "file_type"], + "required": ["name", "format"], }, }, }, @@ -298,42 +488,137 @@ ) ).build() -multi_csv_scenario = ( - TestScenarioBuilder() +multi_format_analytics_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() + .set_name("multi_format_analytics") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["file1.csv"], + "validation_policy": "Emit Record", + }, + { + "name": "stream2", + "format": {"filetype": "csv"}, + "globs": ["file2.csv"], + "validation_policy": "Emit Record", + }, + { + "name": "stream3", + "format": {"filetype": "jsonl"}, + "globs": ["file3.jsonl"], + "validation_policy": "Emit Record", + }, + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "file1.csv": { + "contents": [], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "file2.csv": { + "contents": [], + "last_modified": "2023-06-06T03:54:07.000Z", + }, + "file3.jsonl": { + "contents": [], + "last_modified": "2023-06-07T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": {}, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + }, + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": {}, + }, + "name": "stream2", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + }, + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": {}, + }, + "name": "stream3", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + }, + ] + } + ) + .set_expected_records([]) + .set_expected_analytics( + [ + AirbyteAnalyticsTraceMessage(type="file-cdk-csv-stream-count", value="2"), + AirbyteAnalyticsTraceMessage(type="file-cdk-jsonl-stream-count", value="1"), + ] + ) +).build() + +multi_csv_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("multi_csv_stream") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*"], "validation_policy": "Emit Record", } ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11a", "val12a"), - ("val21a", "val22a"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "b.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11b", "val12b", "val13b"), - ("val21b", "val22b", "val23b"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -401,41 +686,45 @@ ).build() multi_csv_stream_n_file_exceeds_limit_for_inference = ( - TestScenarioBuilder() + TestScenarioBuilder[InMemoryFilesSource]() .set_name("multi_csv_stream_n_file_exceeds_limit_for_inference") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*"], "validation_policy": "Emit Record", } ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11a", "val12a"), - ("val21a", "val22a"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "b.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11b", "val12b", "val13b"), - ("val21b", "val22b", "val23b"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") + .set_discovery_policy(LowInferenceLimitDiscoveryPolicy()) ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -497,37 +786,39 @@ }, ] ) - .set_discovery_policy(LowInferenceLimitDiscoveryPolicy()) ).build() -invalid_csv_scenario = ( - TestScenarioBuilder() - .set_name("invalid_csv_scenario") +invalid_csv_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() + .set_name("invalid_csv_scenario") # too many values for the number of headers .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*"], "validation_policy": "Emit Record", } ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1",), - ("val11", "val12"), - ("val21", "val22"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1",), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -550,7 +841,7 @@ } ) .set_expected_records([]) - .set_expected_discover_error(SchemaInferenceError, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value) + .set_expected_discover_error(AirbyteTracedException, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value) .set_expected_logs( { "read": [ @@ -561,43 +852,149 @@ ] } ) + .set_expected_read_error( + AirbyteTracedException, + "Please check the logged errors for more information.", + ) ).build() -csv_single_stream_scenario = ( - TestScenarioBuilder() +invalid_csv_multi_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() + .set_name("invalid_csv_multi_scenario") # too many values for the number of headers + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "csv"}, + "globs": ["*"], + "validation_policy": "Emit Record", + }, + { + "name": "stream2", + "format": {"filetype": "csv"}, + "globs": ["b.csv"], + "validation_policy": "Emit Record", + }, + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1",), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.csv": { + "contents": [ + ("col3",), + ("val13b", "val14b"), + ("val23b", "val24b"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": { + "type": "object", + "properties": { + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream1", + "source_defined_cursor": True, + "supported_sync_modes": ["full_refresh", "incremental"], + }, + { + "json_schema": { + "type": "object", + "properties": { + "col3": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, + }, + "name": "stream2", + "source_defined_cursor": True, + "default_cursor_field": ["_ab_source_file_last_modified"], + "supported_sync_modes": ["full_refresh", "incremental"], + }, + ] + } + ) + .set_expected_records([]) + .set_expected_discover_error(AirbyteTracedException, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value) + .set_expected_logs( + { + "read": [ + { + "level": "ERROR", + "message": f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream=stream1 file=a.csv line_no=1 n_skipped=0", + }, + { + "level": "ERROR", + "message": f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream=stream2 file=b.csv line_no=1 n_skipped=0", + }, + ] + } + ) + .set_expected_read_error(AirbyteTracedException, "Please check the logged errors for more information.") +).build() + +csv_single_stream_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("csv_single_stream_scenario") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "Emit Record", } ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11a", "val12a"), - ("val21a", "val22a"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "b.jsonl": { - "contents": [ - {"col1": "val11b", "col2": "val12b", "col3": "val13b"}, - {"col1": "val12b", "col2": "val22b", "col3": "val23b"}, - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.jsonl": { + "contents": [ + {"col1": "val11b", "col2": "val12b", "col3": "val13b"}, + {"col1": "val12b", "col2": "val22b", "col3": "val23b"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -643,48 +1040,51 @@ ) ).build() -csv_multi_stream_scenario = ( - TestScenarioBuilder() +csv_multi_stream_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("csv_multi_stream") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "Emit Record", }, { "name": "stream2", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["b.csv"], "validation_policy": "Emit Record", }, ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11a", "val12a"), - ("val21a", "val22a"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "b.csv": { - "contents": [ - ("col3",), - ("val13b",), - ("val23b",), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.csv": { + "contents": [ + ("col3",), + ("val13b",), + ("val23b",), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -761,16 +1161,14 @@ ) ).build() - -csv_custom_format_scenario = ( - TestScenarioBuilder() +csv_custom_format_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("csv_custom_format") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", "globs": ["*"], "validation_policy": "Emit Record", "format": { @@ -784,20 +1182,29 @@ ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11", "val12", "val |13|"), - ("val21", "val22", "val23"), - ("val,31", "val |,32|", "val, !!!! 33"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11", "val12", "val |13|"), + ("val21", "val22", "val23"), + ("val,31", "val |,32|", "val, !!!! 33"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("csv") + .set_file_write_options( + { + "delimiter": "#", + "quotechar": "|", + } + ) ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -860,31 +1267,22 @@ }, ] ) - .set_file_write_options( - { - "delimiter": "#", - "quotechar": "|", - } - ) ).build() - multi_stream_custom_format = ( - TestScenarioBuilder() + TestScenarioBuilder[InMemoryFilesSource]() .set_name("multi_stream_custom_format_scenario") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", "globs": ["*.csv"], "validation_policy": "Emit Record", "format": {"filetype": "csv", "delimiter": "#", "escape_char": "!", "double_quote": True, "newlines_in_values": False}, }, { "name": "stream2", - "file_type": "csv", "globs": ["b.csv"], "validation_policy": "Emit Record", "format": { @@ -898,27 +1296,35 @@ ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11a", "val !! 12a"), - ("val !! 21a", "val22a"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "b.csv": { - "contents": [ - ("col3",), - ("val @@@@ 13b",), - ("val23b",), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val !! 12a"), + ("val !! 21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.csv": { + "contents": [ + ("col3",), + ("val @@@@ 13b",), + ("val23b",), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") + .set_file_write_options( + { + "delimiter": "#", + } + ) ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -1009,42 +1415,40 @@ }, ] ) - .set_file_write_options( - { - "delimiter": "#", - } - ) ).build() - -empty_schema_inference_scenario = ( - TestScenarioBuilder() +empty_schema_inference_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("empty_schema_inference_scenario") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*"], "validation_policy": "Emit Record", } ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11", "val12"), - ("val21", "val22"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("csv") + .set_parsers({CsvFormat: EmptySchemaParser()}) ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -1066,8 +1470,7 @@ ] } ) - .set_parsers({"csv": EmptySchemaParser()}) - .set_expected_discover_error(SchemaInferenceError, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value) + .set_expected_discover_error(AirbyteTracedException, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value) .set_expected_records( [ { @@ -1092,16 +1495,15 @@ ) ).build() - -schemaless_csv_scenario = ( - TestScenarioBuilder() +schemaless_csv_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("schemaless_csv_scenario") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*"], "validation_policy": "Skip Record", "schemaless": True, @@ -1109,27 +1511,30 @@ ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11a", "val12a"), - ("val21a", "val22a"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "b.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11b", "val12b", "val13b"), - ("val21b", "val22b", "val23b"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -1188,50 +1593,52 @@ ) ).build() - -schemaless_csv_multi_stream_scenario = ( - TestScenarioBuilder() +schemaless_csv_multi_stream_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("schemaless_csv_multi_stream_scenario") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["a.csv"], "validation_policy": "Skip Record", "schemaless": True, }, { "name": "stream2", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["b.csv"], "validation_policy": "Skip Record", }, ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11a", "val12a"), - ("val21a", "val22a"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "b.csv": { - "contents": [ - ("col3",), - ("val13b",), - ("val23b",), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.csv": { + "contents": [ + ("col3",), + ("val13b",), + ("val23b",), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -1296,16 +1703,15 @@ ) ).build() - -schemaless_with_user_input_schema_fails_connection_check_scenario = ( - TestScenarioBuilder() +schemaless_with_user_input_schema_fails_connection_check_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("schemaless_with_user_input_schema_fails_connection_check_scenario") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*"], "validation_policy": "Skip Record", "input_schema": '{"col1": "string", "col2": "string", "col3": "string"}', @@ -1314,27 +1720,31 @@ ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11a", "val12a"), - ("val21a", "val22a"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "b.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11b", "val12b", "val13b"), - ("val21b", "val22b", "val23b"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") ) - .set_file_type("csv") + .set_catalog(CatalogBuilder().with_stream("stream1", SyncMode.full_refresh).build()) .set_expected_catalog( { "streams": [ @@ -1361,16 +1771,15 @@ .set_expected_read_error(ConfigValidationError, FileBasedSourceError.CONFIG_VALIDATION_ERROR.value) ).build() - -schemaless_with_user_input_schema_fails_connection_check_multi_stream_scenario = ( - TestScenarioBuilder() +schemaless_with_user_input_schema_fails_connection_check_multi_stream_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("schemaless_with_user_input_schema_fails_connection_check_multi_stream_scenario") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["a.csv"], "validation_policy": "Skip Record", "schemaless": True, @@ -1378,34 +1787,38 @@ }, { "name": "stream2", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["b.csv"], "validation_policy": "Skip Record", }, ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11a", "val12a"), - ("val21a", "val22a"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "b.csv": { - "contents": [ - ("col3",), - ("val13b",), - ("val23b",), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.csv": { + "contents": [ + ("col3",), + ("val13b",), + ("val23b",), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") ) - .set_file_type("csv") + .set_catalog(CatalogBuilder().with_stream("stream1", SyncMode.full_refresh).with_stream("stream2", SyncMode.full_refresh).build()) .set_expected_catalog( { "streams": [ @@ -1446,16 +1859,14 @@ .set_expected_read_error(ConfigValidationError, FileBasedSourceError.CONFIG_VALIDATION_ERROR.value) ).build() - -csv_string_can_be_null_with_input_schemas_scenario = ( - TestScenarioBuilder() +csv_string_can_be_null_with_input_schemas_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("csv_string_can_be_null_with_input_schema") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", "globs": ["*"], "validation_policy": "Emit Record", "input_schema": '{"col1": "string", "col2": "string"}', @@ -1468,18 +1879,21 @@ "start_date": "2023-06-04T03:54:07.000000Z", } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("2", "null"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("2", "null"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -1516,15 +1930,14 @@ ) ).build() -csv_string_are_not_null_if_strings_can_be_null_is_false_scenario = ( - TestScenarioBuilder() +csv_string_are_not_null_if_strings_can_be_null_is_false_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("csv_string_are_not_null_if_strings_can_be_null_is_false") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", "globs": ["*"], "validation_policy": "Emit Record", "input_schema": '{"col1": "string", "col2": "string"}', @@ -1538,18 +1951,21 @@ "start_date": "2023-06-04T03:54:07.000000Z", } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("2", "null"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("2", "null"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -1586,15 +2002,14 @@ ) ).build() -csv_string_not_null_if_no_null_values_scenario = ( - TestScenarioBuilder() +csv_string_not_null_if_no_null_values_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("csv_string_not_null_if_no_null_values") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", "globs": ["*"], "validation_policy": "Emit Record", "format": { @@ -1605,18 +2020,21 @@ "start_date": "2023-06-04T03:54:07.000000Z", } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("2", "null"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("2", "null"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -1653,15 +2071,14 @@ ) ).build() -csv_strings_can_be_null_not_quoted_scenario = ( - TestScenarioBuilder() +csv_strings_can_be_null_not_quoted_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("csv_strings_can_be_null_no_input_schema") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", "globs": ["*"], "validation_policy": "Emit Record", "format": {"filetype": "csv", "null_values": ["null"]}, @@ -1670,18 +2087,21 @@ "start_date": "2023-06-04T03:54:07.000000Z", } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("2", "null"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("2", "null"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -1718,37 +2138,39 @@ ) ).build() -csv_newline_in_values_quoted_value_scenario = ( - TestScenarioBuilder() +csv_newline_in_values_quoted_value_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("csv_newline_in_values_quoted_value") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", "globs": ["*"], "validation_policy": "Emit Record", "format": { "filetype": "csv", - } + }, } ], "start_date": "2023-06-04T03:54:07.000000Z", } ) - .set_files( - { - "a.csv": { - "contents": [ - '''"col1","col2"''', - '''"2","val\n2"''', - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + '''"col1","col2"''', + '''"2","val\n2"''', + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -1785,15 +2207,14 @@ ) ).build() -csv_newline_in_values_not_quoted_scenario = ( - TestScenarioBuilder() +csv_newline_in_values_not_quoted_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("csv_newline_in_values_not_quoted") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", "globs": ["*"], "validation_policy": "Emit Record", "format": { @@ -1804,18 +2225,21 @@ "start_date": "2023-06-04T03:54:07.000000Z", } ) - .set_files( - { - "a.csv": { - "contents": [ - """col1,col2""", - """2,val\n2""", - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + """col1,col2""", + """2,val\n2""", + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -1851,28 +2275,25 @@ }, ] ) - .set_expected_logs( - { - "read": [ - { - "level": "ERROR", - "message": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. stream=stream1 file=a.csv line_no=2 n_skipped=0", - } - ] - } + .set_expected_read_error( + AirbyteTracedException, + f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream=stream1 file=a.csv line_no=2 n_skipped=0", + ) + .set_expected_discover_error(AirbyteTracedException, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value) + .set_expected_read_error( + AirbyteTracedException, + "Please check the logged errors for more information.", ) - .set_expected_discover_error(SchemaInferenceError, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value) ).build() -csv_escape_char_is_set_scenario = ( - TestScenarioBuilder() +csv_escape_char_is_set_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("csv_escape_char_is_set") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", "globs": ["*"], "validation_policy": "Emit Record", "format": { @@ -1881,24 +2302,27 @@ "quote_char": '"', "delimiter": ",", "escape_char": "\\", - } + }, } ], "start_date": "2023-06-04T03:54:07.000000Z", } ) - .set_files( - { - "a.csv": { - "contents": [ - """col1,col2""", - '''val11,"val\\"2"''', - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + """col1,col2""", + '''val11,"val\\"2"''', + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -1935,8 +2359,8 @@ ) ).build() -csv_double_quote_is_set_scenario = ( - TestScenarioBuilder() +csv_double_quote_is_set_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("csv_doublequote_is_set") # This scenario tests that quotes are properly escaped when double_quotes is True .set_config( @@ -1944,7 +2368,6 @@ "streams": [ { "name": "stream1", - "file_type": "csv", "globs": ["*"], "validation_policy": "Emit Record", "format": { @@ -1952,24 +2375,27 @@ "double_quotes": True, "quote_char": '"', "delimiter": ",", - } + }, } ], "start_date": "2023-06-04T03:54:07.000000Z", } ) - .set_files( - { - "a.csv": { - "contents": [ - """col1,col2""", - '''val11,"val""2"''', - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + """col1,col2""", + '''val11,"val""2"''', + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -2006,8 +2432,8 @@ ) ).build() -csv_custom_delimiter_with_escape_char_scenario = ( - TestScenarioBuilder() +csv_custom_delimiter_with_escape_char_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("csv_custom_delimiter_with_escape_char") # This scenario tests that a value can contain the delimiter if it is wrapped in the quote_char .set_config( @@ -2015,7 +2441,6 @@ "streams": [ { "name": "stream1", - "file_type": "csv", "globs": ["*"], "validation_policy": "Emit Record", "format": {"filetype": "csv", "double_quotes": True, "quote_char": "@", "delimiter": "|", "escape_char": "+"}, @@ -2024,18 +2449,21 @@ "start_date": "2023-06-04T03:54:07.000000Z", } ) - .set_files( - { - "a.csv": { - "contents": [ - """col1|col2""", - """val"1,1|val+|2""", - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + """col1|col2""", + """val"1,1|val+|2""", + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -2072,8 +2500,8 @@ ) ).build() -csv_custom_delimiter_in_double_quotes_scenario = ( - TestScenarioBuilder() +csv_custom_delimiter_in_double_quotes_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("csv_custom_delimiter_in_double_quotes") # This scenario tests that a value can contain the delimiter if it is wrapped in the quote_char .set_config( @@ -2081,7 +2509,6 @@ "streams": [ { "name": "stream1", - "file_type": "csv", "globs": ["*"], "validation_policy": "Emit Record", "format": { @@ -2095,18 +2522,21 @@ "start_date": "2023-06-04T03:54:07.000000Z", } ) - .set_files( - { - "a.csv": { - "contents": [ - """col1|col2""", - """val"1,1|@val|2@""", - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + """col1|col2""", + """val"1,1|@val|2@""", + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -2143,16 +2573,14 @@ ) ).build() - -csv_skip_before_header_scenario = ( - TestScenarioBuilder() +csv_skip_before_header_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("csv_skip_before_header") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", "globs": ["*"], "validation_policy": "Emit Record", "format": {"filetype": "csv", "skip_rows_before_header": 2}, @@ -2161,20 +2589,23 @@ "start_date": "2023-06-04T03:54:07.000000Z", } ) - .set_files( - { - "a.csv": { - "contents": [ - ("skip_this", "skip_this"), - ("skip_this_too", "skip_this_too"), - ("col1", "col2"), - ("val11", "val12"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("skip_this", "skip_this"), + ("skip_this_too", "skip_this_too"), + ("col1", "col2"), + ("val11", "val12"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -2211,15 +2642,14 @@ ) ).build() -csv_skip_after_header_scenario = ( - TestScenarioBuilder() +csv_skip_after_header_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("csv_skip_after_header") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", "globs": ["*"], "validation_policy": "Emit Record", "format": {"filetype": "csv", "skip_rows_after_header": 2}, @@ -2228,20 +2658,23 @@ "start_date": "2023-06-04T03:54:07.000000Z", } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("skip_this", "skip_this"), - ("skip_this_too", "skip_this_too"), - ("val11", "val12"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("skip_this", "skip_this"), + ("skip_this_too", "skip_this_too"), + ("val11", "val12"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -2278,16 +2711,14 @@ ) ).build() - -csv_skip_before_and_after_header_scenario = ( - TestScenarioBuilder() +csv_skip_before_and_after_header_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("csv_skip_before_after_header") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", "globs": ["*"], "validation_policy": "Emit Record", "format": { @@ -2300,20 +2731,23 @@ "start_date": "2023-06-04T03:54:07.000000Z", } ) - .set_files( - { - "a.csv": { - "contents": [ - ("skip_this", "skip_this"), - ("col1", "col2"), - ("skip_this_too", "skip_this_too"), - ("val11", "val12"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("skip_this", "skip_this"), + ("col1", "col2"), + ("skip_this_too", "skip_this_too"), + ("val11", "val12"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -2350,37 +2784,39 @@ ) ).build() -csv_autogenerate_column_names_scenario = ( - TestScenarioBuilder() +csv_autogenerate_column_names_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("csv_autogenerate_column_names") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", "globs": ["*"], "validation_policy": "Emit Record", "format": { "filetype": "csv", - "autogenerate_column_names": True, + "header_definition": {"header_definition_type": "Autogenerated"}, }, } ], "start_date": "2023-06-04T03:54:07.000000Z", } ) - .set_files( - { - "a.csv": { - "contents": [ - ("val11", "val12"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("val11", "val12"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -2417,15 +2853,14 @@ ) ).build() -csv_custom_bool_values_scenario = ( - TestScenarioBuilder() +csv_custom_bool_values_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("csv_custom_bool_values") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", "globs": ["*"], "validation_policy": "Emit Record", "input_schema": '{"col1": "boolean", "col2": "boolean"}', @@ -2439,18 +2874,21 @@ "start_date": "2023-06-04T03:54:07.000000Z", } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("this_is_true", "this_is_false"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("this_is_true", "this_is_false"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -2487,15 +2925,14 @@ ) ).build() -csv_custom_null_values_scenario = ( - TestScenarioBuilder() +csv_custom_null_values_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("csv_custom_null_values") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", "globs": ["*"], "validation_policy": "Emit Record", "input_schema": '{"col1": "boolean", "col2": "string"}', @@ -2508,18 +2945,21 @@ "start_date": "2023-06-04T03:54:07.000000Z", } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("null", "na"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("null", "na"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -2556,16 +2996,15 @@ ) ).build() - -earlier_csv_scenario = ( - TestScenarioBuilder() +earlier_csv_scenario: TestScenario[InMemoryFilesSource] = ( + TestScenarioBuilder[InMemoryFilesSource]() .set_name("earlier_csv_stream") .set_config( { "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*"], "validation_policy": "Emit Record", } @@ -2573,19 +3012,22 @@ "start_date": "2023-06-10T03:54:07.000000Z", } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11", "val12"), - ("val21", "val22"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_check_status("FAILED") .set_expected_catalog( { @@ -2609,5 +3051,5 @@ } ) .set_expected_records([]) - .set_expected_discover_error(SchemaInferenceError, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value) + .set_expected_discover_error(AirbyteTracedException, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value) ).build() diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/file_based_source_builder.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/file_based_source_builder.py new file mode 100644 index 000000000000..90deb31fe41b --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/file_based_source_builder.py @@ -0,0 +1,86 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from copy import deepcopy +from typing import Any, Mapping, Optional, Type + +from airbyte_cdk.sources.file_based.availability_strategy.abstract_file_based_availability_strategy import ( + AbstractFileBasedAvailabilityStrategy, +) +from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy, DefaultDiscoveryPolicy +from airbyte_cdk.sources.file_based.file_based_source import default_parsers +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader +from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser +from airbyte_cdk.sources.file_based.schema_validation_policies import AbstractSchemaValidationPolicy +from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor +from unit_tests.sources.file_based.in_memory_files_source import InMemoryFilesSource +from unit_tests.sources.file_based.scenarios.scenario_builder import SourceBuilder + + +class FileBasedSourceBuilder(SourceBuilder[InMemoryFilesSource]): + def __init__(self) -> None: + self._files: Mapping[str, Any] = {} + self._file_type: Optional[str] = None + self._availability_strategy: Optional[AbstractFileBasedAvailabilityStrategy] = None + self._discovery_policy: AbstractDiscoveryPolicy = DefaultDiscoveryPolicy() + self._validation_policies: Optional[Mapping[str, AbstractSchemaValidationPolicy]] = None + self._parsers = default_parsers + self._stream_reader: Optional[AbstractFileBasedStreamReader] = None + self._file_write_options: Mapping[str, Any] = {} + self._cursor_cls: Optional[Type[AbstractFileBasedCursor]] = None + + def build(self, configured_catalog: Optional[Mapping[str, Any]]) -> InMemoryFilesSource: + if self._file_type is None: + raise ValueError("file_type is not set") + return InMemoryFilesSource( + self._files, + self._file_type, + self._availability_strategy, + self._discovery_policy, + self._validation_policies, + self._parsers, + self._stream_reader, + configured_catalog, + self._file_write_options, + self._cursor_cls, + ) + + def set_files(self, files: Mapping[str, Any]) -> "FileBasedSourceBuilder": + self._files = files + return self + + def set_file_type(self, file_type: str) -> "FileBasedSourceBuilder": + self._file_type = file_type + return self + + def set_parsers(self, parsers: Mapping[Type[Any], FileTypeParser]) -> "FileBasedSourceBuilder": + self._parsers = parsers + return self + + def set_availability_strategy(self, availability_strategy: AbstractFileBasedAvailabilityStrategy) -> "FileBasedSourceBuilder": + self._availability_strategy = availability_strategy + return self + + def set_discovery_policy(self, discovery_policy: AbstractDiscoveryPolicy) -> "FileBasedSourceBuilder": + self._discovery_policy = discovery_policy + return self + + def set_validation_policies(self, validation_policies: Mapping[str, AbstractSchemaValidationPolicy]) -> "FileBasedSourceBuilder": + self._validation_policies = validation_policies + return self + + def set_stream_reader(self, stream_reader: AbstractFileBasedStreamReader) -> "FileBasedSourceBuilder": + self._stream_reader = stream_reader + return self + + def set_cursor_cls(self, cursor_cls: AbstractFileBasedCursor) -> "FileBasedSourceBuilder": + self._cursor_cls = cursor_cls + return self + + def set_file_write_options(self, file_write_options: Mapping[str, Any]) -> "FileBasedSourceBuilder": + self._file_write_options = file_write_options + return self + + def copy(self) -> "FileBasedSourceBuilder": + return deepcopy(self) diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/incremental_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/incremental_scenarios.py index 3f4a95933b02..3c3195fbac61 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/incremental_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/incremental_scenarios.py @@ -3,6 +3,7 @@ # from unit_tests.sources.file_based.helpers import LowHistoryLimitCursor +from unit_tests.sources.file_based.scenarios.file_based_source_builder import FileBasedSourceBuilder from unit_tests.sources.file_based.scenarios.scenario_builder import IncrementalScenarioConfig, TestScenarioBuilder single_csv_input_state_is_earlier_scenario = ( @@ -13,53 +14,70 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "Emit Record", } ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11", "val12"), - ("val21", "val22"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") - .set_incremental_scenario_config(IncrementalScenarioConfig( - input_state=[{ - "type": "STREAM", - "stream": { - "stream_state": { - "history": { - "some_old_file.csv": "2023-06-01T03:54:07.000000Z" + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "history": {"some_old_file.csv": "2023-06-01T03:54:07.000000Z"}, + }, + "stream_descriptor": {"name": "stream1"}, }, - }, - "stream_descriptor": {"name": "stream1"} - } - } - ], - )) + } + ], + ) + ) .set_expected_records( [ - {"data": {"col1": "val11", "col2": "val12", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21", "col2": "val22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val11", + "col2": "val12", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21", + "col2": "val22", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, { "stream1": { - "history": { - "some_old_file.csv": "2023-06-01T03:54:07.000000Z", - "a.csv": "2023-06-05T03:54:07.000000Z" - }, + "history": {"some_old_file.csv": "2023-06-01T03:54:07.000000Z", "a.csv": "2023-06-05T03:54:07.000000Z"}, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv", } - } + }, ] ) .set_expected_catalog( @@ -74,16 +92,13 @@ "properties": { "col1": { "type": ["null", "string"], - }, "col2": { - "type": ["null", "string"], }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" + "col2": { + "type": ["null", "string"], }, - } + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream1", } @@ -100,47 +115,49 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "Emit Record", } ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11", "val12"), - ("val21", "val22"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") - .set_incremental_scenario_config(IncrementalScenarioConfig( - input_state=[{ - "type": "STREAM", - "stream": { - "stream_state": { - "history": { - "a.csv": "2023-06-05T03:54:07.000000Z" + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "history": {"a.csv": "2023-06-05T03:54:07.000000Z"}, + }, + "stream_descriptor": {"name": "stream1"}, }, - }, - "stream_descriptor": {"name": "stream1"} - } - } - ], - )) + } + ], + ) + ) .set_expected_records( [ { "stream1": { - "history": { - "a.csv": "2023-06-05T03:54:07.000000Z" - }, + "history": {"a.csv": "2023-06-05T03:54:07.000000Z"}, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv", } } @@ -158,16 +175,13 @@ "properties": { "col1": { "type": ["null", "string"], - }, "col2": { - "type": ["null", "string"], }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" + "col2": { + "type": ["null", "string"], }, - } + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream1", } @@ -184,52 +198,70 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "Emit Record", } ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11", "val12"), - ("val21", "val22"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") - .set_incremental_scenario_config(IncrementalScenarioConfig( - input_state=[{ - "type": "STREAM", - "stream": { - "stream_state": { - "history": { - "a.csv": "2023-06-01T03:54:07.000000Z" + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "history": {"a.csv": "2023-06-01T03:54:07.000000Z"}, + }, + "stream_descriptor": {"name": "stream1"}, }, - }, - "stream_descriptor": {"name": "stream1"} - } - } - ], - )) + } + ], + ) + ) .set_expected_records( [ - {"data": {"col1": "val11", "col2": "val12", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21", "col2": "val22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val11", + "col2": "val12", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21", + "col2": "val22", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, { "stream1": { - "history": { - "a.csv": "2023-06-05T03:54:07.000000Z" - }, + "history": {"a.csv": "2023-06-05T03:54:07.000000Z"}, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv", } - } + }, ] ) .set_expected_catalog( @@ -244,16 +276,13 @@ "properties": { "col1": { "type": ["null", "string"], - }, "col2": { - "type": ["null", "string"], - }, - "_ab_source_file_last_modified": { - "type": "string" }, - "_ab_source_file_url": { - "type": "string" + "col2": { + "type": ["null", "string"], }, - } + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream1", } @@ -270,26 +299,29 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "Emit Record", } ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11", "val12"), - ("val21", "val22"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -302,16 +334,13 @@ "properties": { "col1": { "type": ["null", "string"], - }, "col2": { - "type": ["null", "string"], - }, - "_ab_source_file_last_modified": { - "type": "string" }, - "_ab_source_file_url": { - "type": "string" + "col2": { + "type": ["null", "string"], }, - } + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream1", } @@ -320,21 +349,38 @@ ) .set_expected_records( [ - {"data": {"col1": "val11", "col2": "val12", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21", "col2": "val22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val11", + "col2": "val12", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21", + "col2": "val22", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, { "stream1": { - "history": { - "a.csv": "2023-06-05T03:54:07.000000Z" - }, + "history": {"a.csv": "2023-06-05T03:54:07.000000Z"}, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_a.csv", } - } + }, ] ) - .set_incremental_scenario_config(IncrementalScenarioConfig( - input_state=[], - ))).build() + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[], + ) + ) +).build() multi_csv_same_timestamp_scenario = ( TestScenarioBuilder() @@ -344,34 +390,37 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "Emit Record", } ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11a", "val12a"), - ("val21a", "val22a"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", - }, - "b.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11b", "val12b", "val13b"), - ("val21b", "val22b", "val23b"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -389,13 +438,9 @@ "col3": { "type": ["null", "string"], }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, - } + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream1", "source_defined_cursor": True, @@ -406,26 +451,58 @@ ) .set_expected_records( [ - {"data": {"col1": "val11a", "col2": "val12a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21a", "col2": "val22a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, - {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, { "stream1": { - "history": { - "a.csv": "2023-06-05T03:54:07.000000Z", - "b.csv": "2023-06-05T03:54:07.000000Z" - }, + "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"}, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv", } - } + }, ] ) - .set_incremental_scenario_config(IncrementalScenarioConfig( - input_state=[], - ))).build() + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[], + ) + ) +).build() single_csv_input_state_is_later_scenario = ( TestScenarioBuilder() @@ -435,26 +512,29 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "Emit Record", } ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11", "val12"), - ("val21", "val22"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -467,16 +547,13 @@ "properties": { "col1": { "type": ["null", "string"], - }, "col2": { - "type": ["null", "string"], }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" + "col2": { + "type": ["null", "string"], }, - } + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream1", } @@ -485,8 +562,24 @@ ) .set_expected_records( [ - {"data": {"col1": "val11", "col2": "val12", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21", "col2": "val22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val11", + "col2": "val12", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21", + "col2": "val22", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, { "stream1": { "history": { @@ -495,23 +588,23 @@ }, "_ab_source_file_last_modified": "2023-07-15T23:59:59.000000Z_recent_file.csv", } - } + }, ] ) - .set_incremental_scenario_config(IncrementalScenarioConfig( - input_state=[{ - "type": "STREAM", - "stream": { - "stream_state": { - "history": { - "recent_file.csv": "2023-07-15T23:59:59.000000Z" - } - }, - "stream_descriptor": {"name": "stream1"} - } - } - ], - ))).build() + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[ + { + "type": "STREAM", + "stream": { + "stream_state": {"history": {"recent_file.csv": "2023-07-15T23:59:59.000000Z"}}, + "stream_descriptor": {"name": "stream1"}, + }, + } + ], + ) + ) +).build() multi_csv_different_timestamps_scenario = ( TestScenarioBuilder() @@ -521,34 +614,37 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "Emit Record", } ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11a", "val12a"), - ("val21a", "val22a"), - ], - "last_modified": "2023-06-04T03:54:07.000000Z", - }, - "b.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11b", "val12b", "val13b"), - ("val21b", "val22b", "val23b"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-04T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -566,13 +662,9 @@ "col3": { "type": ["null", "string"], }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, - } + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream1", "source_defined_cursor": True, @@ -583,8 +675,24 @@ ) .set_expected_records( [ - {"data": {"col1": "val11a", "col2": "val12a", "_ab_source_file_last_modified": "2023-06-04T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21a", "col2": "val22a", "_ab_source_file_last_modified": "2023-06-04T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-04T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-04T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, { "stream1": { "history": { @@ -593,24 +701,40 @@ "_ab_source_file_last_modified": "2023-06-04T03:54:07.000000Z_a.csv", } }, - {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, { "stream1": { - "history": { - "a.csv": "2023-06-04T03:54:07.000000Z", - "b.csv": "2023-06-05T03:54:07.000000Z" - }, + "history": {"a.csv": "2023-06-04T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"}, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv", } - } + }, ] ) - .set_incremental_scenario_config(IncrementalScenarioConfig( - input_state=[], - ))).build() + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[], + ) + ) +).build() multi_csv_per_timestamp_scenario = ( TestScenarioBuilder() @@ -620,42 +744,45 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "Emit Record", } ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11a", "val12a"), - ("val21a", "val22a"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", - }, - "b.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11b", "val12b", "val13b"), - ("val21b", "val22b", "val23b"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", - }, - "c.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11c", "val12c", "val13c"), - ("val21c", "val22c", "val23c"), - ], - "last_modified": "2023-06-06T03:54:07.000000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-06T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -673,13 +800,9 @@ "col3": { "type": ["null", "string"], }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, - } + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream1", "source_defined_cursor": True, @@ -690,40 +813,88 @@ ) .set_expected_records( [ - {"data": {"col1": "val11a", "col2": "val12a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21a", "col2": "val22a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, - {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, { "stream1": { - "history": { - "a.csv": "2023-06-05T03:54:07.000000Z", - "b.csv": "2023-06-05T03:54:07.000000Z" - }, + "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"}, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv", } }, - {"data": {"col1": "val11c", "col2": "val12c", "col3": "val13c", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", - "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21c", "col2": "val22c", "col3": "val23c", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", - "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val11c", + "col2": "val12c", + "col3": "val13c", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "c.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21c", + "col2": "val22c", + "col3": "val23c", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "c.csv", + }, + "stream": "stream1", + }, { "stream1": { "history": { "a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z", - "c.csv": "2023-06-06T03:54:07.000000Z" + "c.csv": "2023-06-06T03:54:07.000000Z", }, "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv", } }, ] ) - .set_incremental_scenario_config(IncrementalScenarioConfig( - input_state=[], - ))).build() + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[], + ) + ) +).build() multi_csv_skip_file_if_already_in_history = ( TestScenarioBuilder() @@ -733,42 +904,45 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "Emit Record", } ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11a", "val12a"), - ("val21a", "val22a"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", - }, - "b.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11b", "val12b", "val13b"), - ("val21b", "val22b", "val23b"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", - }, - "c.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11c", "val12c", "val13c"), - ("val21c", "val22c", "val23c"), - ], - "last_modified": "2023-06-06T03:54:07.000000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-06T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -786,13 +960,9 @@ "col3": { "type": ["null", "string"], }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, - } + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream1", "source_defined_cursor": True, @@ -805,47 +975,78 @@ [ # {"data": {"col1": "val11a", "col2": "val12a"}, "stream": "stream1"}, # this file is skipped # {"data": {"col1": "val21a", "col2": "val22a"}, "stream": "stream1"}, # this file is skipped - {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, { "stream1": { - "history": { - "a.csv": "2023-06-05T03:54:07.000000Z", - "b.csv": "2023-06-05T03:54:07.000000Z" - }, + "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z"}, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_b.csv", } }, - {"data": {"col1": "val11c", "col2": "val12c", "col3": "val13c", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", - "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21c", "col2": "val22c", "col3": "val23c", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", - "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val11c", + "col2": "val12c", + "col3": "val13c", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "c.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21c", + "col2": "val22c", + "col3": "val23c", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "c.csv", + }, + "stream": "stream1", + }, { "stream1": { "history": { "a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z", - "c.csv": "2023-06-06T03:54:07.000000Z" + "c.csv": "2023-06-06T03:54:07.000000Z", }, "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv", } }, ] ) - .set_incremental_scenario_config(IncrementalScenarioConfig( - input_state=[{ - "type": "STREAM", - "stream": { - "stream_state": { - "history": {"a.csv": "2023-06-05T03:54:07.000000Z"} - }, - "stream_descriptor": {"name": "stream1"} - } - } - ], - ))).build() + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[ + { + "type": "STREAM", + "stream": { + "stream_state": {"history": {"a.csv": "2023-06-05T03:54:07.000000Z"}}, + "stream_descriptor": {"name": "stream1"}, + }, + } + ], + ) + ) +).build() multi_csv_include_missing_files_within_history_range = ( TestScenarioBuilder() @@ -855,42 +1056,45 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "Emit Record", } ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11a", "val12a"), - ("val21a", "val22a"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", - }, - "b.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11b", "val12b", "val13b"), - ("val21b", "val22b", "val23b"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", - }, - "c.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11c", "val12c", "val13c"), - ("val21c", "val22c", "val23c"), - ], - "last_modified": "2023-06-06T03:54:07.000000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-06T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -908,12 +1112,8 @@ "col3": { "type": ["null", "string"], }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, }, }, "name": "stream1", @@ -927,10 +1127,26 @@ [ # {"data": {"col1": "val11a", "col2": "val12a"}, "stream": "stream1"}, # this file is skipped # {"data": {"col1": "val21a", "col2": "val22a"}, "stream": "stream1"}, # this file is skipped - {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, # {"data": {"col1": "val11c", "col2": "val12c", "col3": "val13c"}, "stream": "stream1"}, # this file is skipped # {"data": {"col1": "val21c", "col2": "val22c", "col3": "val23c"}, "stream": "stream1"}, # this file is skipped { @@ -938,28 +1154,29 @@ "history": { "a.csv": "2023-06-05T03:54:07.000000Z", "b.csv": "2023-06-05T03:54:07.000000Z", - "c.csv": "2023-06-06T03:54:07.000000Z" + "c.csv": "2023-06-06T03:54:07.000000Z", }, "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_c.csv", } }, ] ) - .set_incremental_scenario_config(IncrementalScenarioConfig( - input_state=[{ - "type": "STREAM", - "stream": { - "stream_state": { - "history": { - "a.csv": "2023-06-05T03:54:07.000000Z", - "c.csv": "2023-06-06T03:54:07.000000Z" + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "history": {"a.csv": "2023-06-05T03:54:07.000000Z", "c.csv": "2023-06-06T03:54:07.000000Z"}, + }, + "stream_descriptor": {"name": "stream1"}, }, - }, - "stream_descriptor": {"name": "stream1"} - } - } - ], - ))).build() + } + ], + ) + ) +).build() multi_csv_remove_old_files_if_history_is_full_scenario = ( TestScenarioBuilder() @@ -969,43 +1186,46 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "Emit Record", } ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11a", "val12a"), - ("val21a", "val22a"), - ], - "last_modified": "2023-06-06T03:54:07.000000Z", - }, - "b.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11b", "val12b", "val13b"), - ("val21b", "val22b", "val23b"), - ], - "last_modified": "2023-06-07T03:54:07.000000Z", - }, - "c.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11c", "val12c", "val13c"), - ("val21c", "val22c", "val23c"), - ], - "last_modified": "2023-06-10T03:54:07.000000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-06T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-07T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-10T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_cursor_cls(LowHistoryLimitCursor) ) - .set_file_type("csv") - .set_cursor_cls(LowHistoryLimitCursor) .set_expected_catalog( { "streams": [ @@ -1023,13 +1243,9 @@ "col3": { "type": ["null", "string"], }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, - } + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream1", "source_defined_cursor": True, @@ -1040,8 +1256,24 @@ ) .set_expected_records( [ - {"data": {"col1": "val11a", "col2": "val12a", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21a", "col2": "val22a", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, { "stream1": { "history": { @@ -1052,10 +1284,26 @@ "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z_old_file_same_timestamp_as_a.csv", } }, - {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z", - "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z", - "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, { "stream1": { "history": { @@ -1066,38 +1314,58 @@ "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z_b.csv", } }, - {"data": {"col1": "val11c", "col2": "val12c", "col3": "val13c", "_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z", - "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21c", "col2": "val22c", "col3": "val23c", "_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z", - "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val11c", + "col2": "val12c", + "col3": "val13c", + "_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z", + "_ab_source_file_url": "c.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21c", + "col2": "val22c", + "col3": "val23c", + "_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z", + "_ab_source_file_url": "c.csv", + }, + "stream": "stream1", + }, { "stream1": { "history": { "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z", "b.csv": "2023-06-07T03:54:07.000000Z", - "c.csv": "2023-06-10T03:54:07.000000Z" + "c.csv": "2023-06-10T03:54:07.000000Z", }, "_ab_source_file_last_modified": "2023-06-10T03:54:07.000000Z_c.csv", } }, ] ) - .set_incremental_scenario_config(IncrementalScenarioConfig( - input_state=[{ - "type": "STREAM", - "stream": { - "stream_state": { - "history": { - "very_very_old_file.csv": "2023-06-01T03:54:07.000000Z", - "very_old_file.csv": "2023-06-02T03:54:07.000000Z", - "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z", + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "history": { + "very_very_old_file.csv": "2023-06-01T03:54:07.000000Z", + "very_old_file.csv": "2023-06-02T03:54:07.000000Z", + "old_file_same_timestamp_as_a.csv": "2023-06-06T03:54:07.000000Z", + }, + }, + "stream_descriptor": {"name": "stream1"}, }, - }, - "stream_descriptor": {"name": "stream1"} - } - } - ], - ))).build() + } + ], + ) + ) +).build() multi_csv_same_timestamp_more_files_than_history_size_scenario = ( TestScenarioBuilder() @@ -1107,7 +1375,7 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "Emit Record", "days_to_sync_if_history_is_full": 3, @@ -1115,44 +1383,47 @@ ] } ) - .set_files( - { - "b.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11b", "val12b", "val13b"), - ("val21b", "val22b", "val23b"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", - }, - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11a", "val12a"), - ("val21a", "val22a"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", - }, - "c.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11c", "val12c", "val13c"), - ("val21c", "val22c", "val23c"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", - }, - "d.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11d", "val12d", "val13d"), - ("val21d", "val22d", "val23d"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "d.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11d", "val12d", "val13d"), + ("val21d", "val22d", "val23d"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_cursor_cls(LowHistoryLimitCursor) ) - .set_file_type("csv") - .set_cursor_cls(LowHistoryLimitCursor) .set_expected_catalog( { "streams": [ @@ -1170,13 +1441,9 @@ "col3": { "type": ["null", "string"], }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, - } + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream1", "source_defined_cursor": True, @@ -1187,20 +1454,84 @@ ) .set_expected_records( [ - {"data": {"col1": "val11a", "col2": "val12a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21a", "col2": "val22a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, - {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, - {"data": {"col1": "val11c", "col2": "val12c", "col3": "val13c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21c", "col2": "val22c", "col3": "val23c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, - {"data": {"col1": "val11d", "col2": "val12d", "col3": "val13d", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "d.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21d", "col2": "val22d", "col3": "val23d", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "d.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val11c", + "col2": "val12c", + "col3": "val13c", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "c.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21c", + "col2": "val22c", + "col3": "val23c", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "c.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val11d", + "col2": "val12d", + "col3": "val13d", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "d.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21d", + "col2": "val22d", + "col3": "val23d", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "d.csv", + }, + "stream": "stream1", + }, { "stream1": { "history": { @@ -1210,12 +1541,15 @@ }, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv", } - } + }, ] ) - .set_incremental_scenario_config(IncrementalScenarioConfig( - input_state=[], - ))).build() + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[], + ) + ) +).build() multi_csv_sync_recent_files_if_history_is_incomplete_scenario = ( TestScenarioBuilder() @@ -1225,7 +1559,7 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "Emit Record", "days_to_sync_if_history_is_full": 3, @@ -1233,44 +1567,47 @@ ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11a", "val12a"), - ("val21a", "val22a"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", - }, - "b.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11b", "val12b", "val13b"), - ("val21b", "val22b", "val23b"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", - }, - "c.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11c", "val12c", "val13c"), - ("val21c", "val22c", "val23c"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", - }, - "d.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11d", "val12d", "val13d"), - ("val21d", "val22d", "val23d"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "d.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11d", "val12d", "val13d"), + ("val21d", "val22d", "val23d"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + } + ) + .set_cursor_cls(LowHistoryLimitCursor) + .set_file_type("csv") ) - .set_cursor_cls(LowHistoryLimitCursor) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -1288,13 +1625,9 @@ "col3": { "type": ["null", "string"], }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, - } + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream1", "source_defined_cursor": True, @@ -1317,22 +1650,26 @@ } ] ) - .set_incremental_scenario_config(IncrementalScenarioConfig( - input_state=[{ - "type": "STREAM", - "stream": { - "stream_state": { - "history": { - "b.csv": "2023-06-05T03:54:07.000000Z", - "c.csv": "2023-06-05T03:54:07.000000Z", - "d.csv": "2023-06-05T03:54:07.000000Z", + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "history": { + "b.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-05T03:54:07.000000Z", + "d.csv": "2023-06-05T03:54:07.000000Z", + }, + }, + "stream_descriptor": {"name": "stream1"}, }, - }, - "stream_descriptor": {"name": "stream1"} - } - } - ], - ))).build() + } + ], + ) + ) +).build() multi_csv_sync_files_within_time_window_if_history_is_incomplete__different_timestamps_scenario = ( TestScenarioBuilder() @@ -1342,7 +1679,7 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "Emit Record", "days_to_sync_if_history_is_full": 3, @@ -1350,44 +1687,47 @@ ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11a", "val12a"), - ("val21a", "val22a"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", - }, - "b.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11b", "val12b", "val13b"), - ("val21b", "val22b", "val23b"), - ], - "last_modified": "2023-06-06T03:54:07.000000Z", - }, - "c.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11c", "val12c", "val13c"), - ("val21c", "val22c", "val23c"), - ], - "last_modified": "2023-06-07T03:54:07.000000Z", - }, - "d.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11d", "val12d", "val13d"), - ("val21d", "val22d", "val23d"), - ], - "last_modified": "2023-06-08T03:54:07.000000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-06T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-07T03:54:07.000000Z", + }, + "d.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11d", "val12d", "val13d"), + ("val21d", "val22d", "val23d"), + ], + "last_modified": "2023-06-08T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_cursor_cls(LowHistoryLimitCursor) ) - .set_file_type("csv") - .set_cursor_cls(LowHistoryLimitCursor) .set_expected_catalog( { "streams": [ @@ -1405,13 +1745,9 @@ "col3": { "type": ["null", "string"], }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, - } + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream1", "source_defined_cursor": True, @@ -1424,10 +1760,26 @@ [ # {"data": {"col1": "val11a", "col2": "val12a"}, "stream": "stream1"}, # This file is skipped because it is older than the time_window # {"data": {"col1": "val21a", "col2": "val22a"}, "stream": "stream1"}, - {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", - "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", - "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, { "stream1": { "history": { @@ -1440,22 +1792,26 @@ }, ] ) - .set_incremental_scenario_config(IncrementalScenarioConfig( - input_state=[{ - "type": "STREAM", - "stream": { - "stream_state": { - "history": { - "c.csv": "2023-06-07T03:54:07.000000Z", - "d.csv": "2023-06-08T03:54:07.000000Z", - "e.csv": "2023-06-08T03:54:07.000000Z", + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "history": { + "c.csv": "2023-06-07T03:54:07.000000Z", + "d.csv": "2023-06-08T03:54:07.000000Z", + "e.csv": "2023-06-08T03:54:07.000000Z", + }, + }, + "stream_descriptor": {"name": "stream1"}, }, - }, - "stream_descriptor": {"name": "stream1"} - } - } - ], - ))).build() + } + ], + ) + ) +).build() multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_different_timestamps_scenario = ( TestScenarioBuilder() @@ -1465,7 +1821,7 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "Emit Record", "days_to_sync_if_history_is_full": 3, @@ -1473,44 +1829,47 @@ ] } ) - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11a", "val12a"), - ("val21a", "val22a"), - ], - "last_modified": "2023-06-05T03:54:07.000000Z", - }, - "b.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11b", "val12b", "val13b"), - ("val21b", "val22b", "val23b"), - ], - "last_modified": "2023-06-06T03:54:07.000000Z", - }, - "c.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11c", "val12c", "val13c"), - ("val21c", "val22c", "val23c"), - ], - "last_modified": "2023-06-07T03:54:07.000000Z", - }, - "d.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11d", "val12d", "val13d"), - ("val21d", "val22d", "val23d"), - ], - "last_modified": "2023-06-08T03:54:07.000000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", "val12a"), + ("val21a", "val22a"), + ], + "last_modified": "2023-06-05T03:54:07.000000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-06T03:54:07.000000Z", + }, + "c.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11c", "val12c", "val13c"), + ("val21c", "val22c", "val23c"), + ], + "last_modified": "2023-06-07T03:54:07.000000Z", + }, + "d.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11d", "val12d", "val13d"), + ("val21d", "val22d", "val23d"), + ], + "last_modified": "2023-06-08T03:54:07.000000Z", + }, + } + ) + .set_file_type("csv") + .set_cursor_cls(LowHistoryLimitCursor) ) - .set_file_type("csv") - .set_cursor_cls(LowHistoryLimitCursor) .set_expected_catalog( { "streams": [ @@ -1528,13 +1887,9 @@ "col3": { "type": ["null", "string"], }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, - } + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream1", "source_defined_cursor": True, @@ -1545,10 +1900,24 @@ ) .set_expected_records( [ - {"data": {"col1": "val11a", "col2": "val12a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21a", "col2": "val22a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, { "stream1": { "history": { @@ -1559,10 +1928,26 @@ "_ab_source_file_last_modified": "2023-06-08T03:54:07.000000Z_d.csv", } }, - {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", - "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", - "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, { "stream1": { "history": { @@ -1575,19 +1960,23 @@ }, ] ) - .set_incremental_scenario_config(IncrementalScenarioConfig( - input_state=[{ - "type": "STREAM", - "stream": { - "stream_state": { - "history": { - "old_file.csv": "2023-06-05T00:00:00.000000Z", - "c.csv": "2023-06-07T03:54:07.000000Z", - "d.csv": "2023-06-08T03:54:07.000000Z", + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "history": { + "old_file.csv": "2023-06-05T00:00:00.000000Z", + "c.csv": "2023-06-07T03:54:07.000000Z", + "d.csv": "2023-06-08T03:54:07.000000Z", + }, + }, + "stream_descriptor": {"name": "stream1"}, }, - }, - "stream_descriptor": {"name": "stream1"} - } - } - ], - ))).build() + } + ], + ) + ) +).build() diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/jsonl_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/jsonl_scenarios.py index 6e305f75ba0a..b4a447c4f0c0 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/jsonl_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/jsonl_scenarios.py @@ -2,8 +2,11 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError, SchemaInferenceError +from airbyte_cdk.sources.file_based.config.jsonl_format import JsonlFormat +from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError +from airbyte_cdk.utils.traced_exception import AirbyteTracedException from unit_tests.sources.file_based.helpers import LowInferenceBytesJsonlParser, LowInferenceLimitDiscoveryPolicy +from unit_tests.sources.file_based.scenarios.file_based_source_builder import FileBasedSourceBuilder from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder single_jsonl_scenario = ( @@ -14,25 +17,28 @@ "streams": [ { "name": "stream1", - "file_type": "jsonl", + "format": {"filetype": "jsonl"}, "globs": ["*"], "validation_policy": "Emit Record", } ] } ) - .set_files( - { - "a.jsonl": { - "contents": [ - {"col1": "val11", "col2": "val12"}, - {"col1": "val21", "col2": "val22"}, - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.jsonl": { + "contents": [ + {"col1": "val11", "col2": "val12"}, + {"col1": "val21", "col2": "val22"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("jsonl") ) - .set_file_type("jsonl") .set_expected_catalog( { "streams": [ @@ -64,10 +70,24 @@ ) .set_expected_records( [ - {"data": {"col1": "val11", "col2": "val12", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, - {"data": {"col1": "val21", "col2": "val22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, + { + "data": { + "col1": "val11", + "col2": "val12", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21", + "col2": "val22", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl", + }, + "stream": "stream1", + }, ] ) ).build() @@ -81,32 +101,35 @@ "streams": [ { "name": "stream1", - "file_type": "jsonl", + "format": {"filetype": "jsonl"}, "globs": ["*"], "validation_policy": "Emit Record", } ] } ) - .set_files( - { - "a.jsonl": { - "contents": [ - {"col1": "val11a", "col2": "val12a"}, - {"col1": "val21a", "col2": "val22a"}, - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "b.jsonl": { - "contents": [ - {"col1": "val11b", "col2": "val12b", "col3": "val13b"}, - {"col1": "val21b", "col3": "val23b"}, - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.jsonl": { + "contents": [ + {"col1": "val11a", "col2": "val12a"}, + {"col1": "val21a", "col2": "val22a"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.jsonl": { + "contents": [ + {"col1": "val11b", "col2": "val12b", "col3": "val13b"}, + {"col1": "val21b", "col3": "val23b"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("jsonl") ) - .set_file_type("jsonl") .set_expected_catalog( { "streams": [ @@ -130,7 +153,7 @@ "_ab_source_file_url": { "type": "string", }, - } + }, }, "name": "stream1", "source_defined_cursor": True, @@ -141,14 +164,43 @@ ) .set_expected_records( [ - {"data": {"col1": "val11a", "col2": "val12a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, - {"data": {"col1": "val21a", "col2": "val22a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, - {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.jsonl"}, "stream": "stream1"}, - {"data": {"col1": "val21b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.jsonl"}, "stream": "stream1"}, + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.jsonl", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.jsonl", + }, + "stream": "stream1", + }, ] ) ).build() @@ -162,32 +214,36 @@ "streams": [ { "name": "stream1", - "file_type": "jsonl", + "format": {"filetype": "jsonl"}, "globs": ["*"], "validation_policy": "Emit Record", } ] } ) - .set_files( - { - "a.jsonl": { - "contents": [ - {"col1": "val11a", "col2": "val12a"}, - {"col1": "val21a", "col2": "val22a"}, - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "b.jsonl": { - "contents": [ - {"col1": "val11b", "col2": "val12b", "col3": "val13b"}, - {"col1": "val21b", "col3": "val23b"}, - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.jsonl": { + "contents": [ + {"col1": "val11a", "col2": "val12a"}, + {"col1": "val21a", "col2": "val22a"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.jsonl": { + "contents": [ + {"col1": "val11b", "col2": "val12b", "col3": "val13b"}, + {"col1": "val21b", "col3": "val23b"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("jsonl") + .set_discovery_policy(LowInferenceLimitDiscoveryPolicy()) ) - .set_file_type("jsonl") .set_expected_catalog( { "streams": [ @@ -198,7 +254,8 @@ "properties": { "col1": { "type": ["null", "string"], - }, "col2": { + }, + "col2": { "type": ["null", "string"], }, "_ab_source_file_last_modified": { @@ -207,7 +264,7 @@ "_ab_source_file_url": { "type": "string", }, - } + }, }, "name": "stream1", "source_defined_cursor": True, @@ -218,17 +275,45 @@ ) .set_expected_records( [ - {"data": {"col1": "val11a", "col2": "val12a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, - {"data": {"col1": "val21a", "col2": "val22a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, - {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.jsonl"}, "stream": "stream1"}, - {"data": {"col1": "val21b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.jsonl"}, "stream": "stream1"}, + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.jsonl", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.jsonl", + }, + "stream": "stream1", + }, ] ) - .set_discovery_policy(LowInferenceLimitDiscoveryPolicy()) ).build() @@ -240,32 +325,36 @@ "streams": [ { "name": "stream1", - "file_type": "jsonl", + "format": {"filetype": "jsonl"}, "globs": ["*"], "validation_policy": "Emit Record", } ] } ) - .set_files( - { - "a.jsonl": { - "contents": [ - {"col1": "val11a", "col2": "val12a"}, - {"col1": "val21a", "col2": "val22a"}, - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "b.jsonl": { - "contents": [ - {"col1": "val11b", "col2": "val12b"}, - {"col1": "val21b", "col2": "val22b", "col3": "val23b"}, - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.jsonl": { + "contents": [ + {"col1": "val11a", "col2": "val12a"}, + {"col1": "val21a", "col2": "val22a"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.jsonl": { + "contents": [ + {"col1": "val11b", "col2": "val12b"}, + {"col1": "val21b", "col2": "val22b", "col3": "val23b"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("jsonl") + .set_parsers({JsonlFormat: LowInferenceBytesJsonlParser()}) ) - .set_file_type("jsonl") .set_expected_catalog( { "streams": [ @@ -276,7 +365,8 @@ "properties": { "col1": { "type": ["null", "string"], - }, "col2": { + }, + "col2": { "type": ["null", "string"], }, "_ab_source_file_last_modified": { @@ -285,7 +375,7 @@ "_ab_source_file_url": { "type": "string", }, - } + }, }, "name": "stream1", "source_defined_cursor": True, @@ -296,17 +386,45 @@ ) .set_expected_records( [ - {"data": {"col1": "val11a", "col2": "val12a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, - {"data": {"col1": "val21a", "col2": "val22a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, - {"data": {"col1": "val11b", "col2": "val12b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.jsonl"}, "stream": "stream1"}, - {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.jsonl"}, "stream": "stream1"}, + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.jsonl", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.jsonl", + }, + "stream": "stream1", + }, ] ) - .set_parsers({"jsonl": LowInferenceBytesJsonlParser()}) ).build() @@ -318,25 +436,28 @@ "streams": [ { "name": "stream1", - "file_type": "jsonl", + "format": {"filetype": "jsonl"}, "globs": ["*"], "validation_policy": "Emit Record", } ] } ) - .set_files( - { - "a.jsonl": { - "contents": [ - {"col1": "val1"}, - "invalid", - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.jsonl": { + "contents": [ + {"col1": "val1"}, + "invalid", + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("jsonl") ) - .set_file_type("jsonl") .set_expected_catalog( { "streams": [ @@ -354,7 +475,7 @@ "_ab_source_file_url": { "type": "string", }, - } + }, }, "name": "stream1", "source_defined_cursor": True, @@ -363,11 +484,15 @@ ] } ) - .set_expected_records([ - {"data": {"col1": "val1", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, - ]) - .set_expected_discover_error(SchemaInferenceError, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value) + .set_expected_records( + [ + { + "data": {"col1": "val1", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.jsonl"}, + "stream": "stream1", + }, + ] + ) + .set_expected_discover_error(AirbyteTracedException, FileBasedSourceError.SCHEMA_INFERENCE_ERROR.value) .set_expected_logs( { "read": [ @@ -389,38 +514,41 @@ "streams": [ { "name": "stream1", - "file_type": "jsonl", + "format": {"filetype": "jsonl"}, "globs": ["*.jsonl"], "validation_policy": "Emit Record", }, { "name": "stream2", - "file_type": "jsonl", + "format": {"filetype": "jsonl"}, "globs": ["b.jsonl"], "validation_policy": "Emit Record", - } + }, ] } ) - .set_files( - { - "a.jsonl": { - "contents": [ - {"col1": 1, "col2": "record1"}, - {"col1": 2, "col2": "record2"}, - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "b.jsonl": { - "contents": [ - {"col3": 1.1}, - {"col3": 2.2}, - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.jsonl": { + "contents": [ + {"col1": 1, "col2": "record1"}, + {"col1": 2, "col2": "record2"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.jsonl": { + "contents": [ + {"col3": 1.1}, + {"col3": 2.2}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("jsonl") ) - .set_file_type("jsonl") .set_expected_catalog( { "streams": [ @@ -428,15 +556,11 @@ "json_schema": { "type": "object", "properties": { - "col1": { - "type": ["null", "integer"] - }, + "col1": {"type": ["null", "integer"]}, "col2": { "type": ["null", "string"], }, - "col3": { - "type": ["null", "number"] - }, + "col3": {"type": ["null", "number"]}, "_ab_source_file_last_modified": { "type": "string", }, @@ -454,9 +578,7 @@ "json_schema": { "type": "object", "properties": { - "col3": { - "type": ["null", "number"] - }, + "col3": {"type": ["null", "number"]}, "_ab_source_file_last_modified": { "type": "string", }, @@ -475,18 +597,40 @@ ) .set_expected_records( [ - {"data": {"col1": 1, "col2": "record1", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, - {"data": {"col1": 2, "col2": "record2", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, - {"data": {"col3": 1.1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.jsonl"}, - "stream": "stream1"}, - {"data": {"col3": 2.2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.jsonl"}, - "stream": "stream1"}, - {"data": {"col3": 1.1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.jsonl"}, - "stream": "stream2"}, - {"data": {"col3": 2.2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.jsonl"}, - "stream": "stream2"}, + { + "data": { + "col1": 1, + "col2": "record1", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl", + }, + "stream": "stream1", + }, + { + "data": { + "col1": 2, + "col2": "record2", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl", + }, + "stream": "stream1", + }, + { + "data": {"col3": 1.1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.jsonl"}, + "stream": "stream1", + }, + { + "data": {"col3": 2.2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.jsonl"}, + "stream": "stream1", + }, + { + "data": {"col3": 1.1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.jsonl"}, + "stream": "stream2", + }, + { + "data": {"col3": 2.2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.jsonl"}, + "stream": "stream2", + }, ] ) ).build() @@ -500,7 +644,7 @@ "streams": [ { "name": "stream1", - "file_type": "jsonl", + "format": {"filetype": "jsonl"}, "globs": ["*"], "validation_policy": "Skip Record", "schemaless": True, @@ -508,25 +652,28 @@ ] } ) - .set_files( - { - "a.jsonl": { - "contents": [ - {"col1": 1, "col2": "record1"}, - {"col1": 2, "col2": "record2"}, - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "b.jsonl": { - "contents": [ - {"col1": 3, "col2": "record3", "col3": 1.1}, - {"col1": 4, "col2": "record4", "col3": 1.1}, - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.jsonl": { + "contents": [ + {"col1": 1, "col2": "record1"}, + {"col1": 2, "col2": "record2"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.jsonl": { + "contents": [ + {"col1": 3, "col2": "record3", "col3": 1.1}, + {"col1": 4, "col2": "record4", "col3": 1.1}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("jsonl") ) - .set_file_type("jsonl") .set_expected_catalog( { "streams": [ @@ -535,16 +682,14 @@ "json_schema": { "type": "object", "properties": { - "data": { - "type": "object" - }, + "data": {"type": "object"}, "_ab_source_file_last_modified": { "type": "string", }, "_ab_source_file_url": { "type": "string", }, - } + }, }, "name": "stream1", "source_defined_cursor": True, @@ -555,14 +700,38 @@ ) .set_expected_records( [ - {"data": {"data": {"col1": 1, "col2": "record1"}, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, - {"data": {"data": {"col1": 2, "col2": "record2"}, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, - {"data": {"data": {"col1": 3, "col2": "record3", "col3": 1.1}, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.jsonl"}, "stream": "stream1"}, - {"data": {"data": {"col1": 4, "col2": "record4", "col3": 1.1}, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.jsonl"}, "stream": "stream1"}, + { + "data": { + "data": {"col1": 1, "col2": "record1"}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl", + }, + "stream": "stream1", + }, + { + "data": { + "data": {"col1": 2, "col2": "record2"}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl", + }, + "stream": "stream1", + }, + { + "data": { + "data": {"col1": 3, "col2": "record3", "col3": 1.1}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.jsonl", + }, + "stream": "stream1", + }, + { + "data": { + "data": {"col1": 4, "col2": "record4", "col3": 1.1}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.jsonl", + }, + "stream": "stream1", + }, ] ) ).build() @@ -576,39 +745,42 @@ "streams": [ { "name": "stream1", - "file_type": "jsonl", + "format": {"filetype": "jsonl"}, "globs": ["a.jsonl"], "validation_policy": "Skip Record", "schemaless": True, }, { "name": "stream2", - "file_type": "jsonl", + "format": {"filetype": "jsonl"}, "globs": ["b.jsonl"], "validation_policy": "Skip Record", - } + }, ] } ) - .set_files( - { - "a.jsonl": { - "contents": [ - {"col1": 1, "col2": "record1"}, - {"col1": 2, "col2": "record2"}, - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "b.jsonl": { - "contents": [ - {"col3": 1.1}, - {"col3": 2.2}, - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.jsonl": { + "contents": [ + {"col1": 1, "col2": "record1"}, + {"col1": 2, "col2": "record2"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.jsonl": { + "contents": [ + {"col3": 1.1}, + {"col3": 2.2}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("jsonl") ) - .set_file_type("jsonl") .set_expected_catalog( { "streams": [ @@ -616,9 +788,7 @@ "json_schema": { "type": "object", "properties": { - "data": { - "type": "object" - }, + "data": {"type": "object"}, "_ab_source_file_last_modified": { "type": "string", }, @@ -636,9 +806,7 @@ "json_schema": { "type": "object", "properties": { - "col3": { - "type": ["null", "number"] - }, + "col3": {"type": ["null", "number"]}, "_ab_source_file_last_modified": { "type": "string", }, @@ -657,14 +825,30 @@ ) .set_expected_records( [ - {"data": {"data": {"col1": 1, "col2": "record1"}, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, - {"data": {"data": {"col1": 2, "col2": "record2"}, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, - {"data": {"col3": 1.1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.jsonl"}, - "stream": "stream2"}, - {"data": {"col3": 2.2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.jsonl"}, - "stream": "stream2"}, + { + "data": { + "data": {"col1": 1, "col2": "record1"}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl", + }, + "stream": "stream1", + }, + { + "data": { + "data": {"col1": 2, "col2": "record2"}, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl", + }, + "stream": "stream1", + }, + { + "data": {"col3": 1.1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.jsonl"}, + "stream": "stream2", + }, + { + "data": {"col3": 2.2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.jsonl"}, + "stream": "stream2", + }, ] ) ).build() @@ -677,26 +861,29 @@ "streams": [ { "name": "stream1", - "file_type": "jsonl", + "format": {"filetype": "jsonl"}, "globs": ["*"], "validation_policy": "Emit Record", - "input_schema": '{"col1": "integer", "col2": "string"}' + "input_schema": '{"col1": "integer", "col2": "string"}', } ] } ) - .set_files( - { - "a.jsonl": { - "contents": [ - {"col1": 1, "col2": "val12"}, - {"col1": 2, "col2": "val22"}, - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.jsonl": { + "contents": [ + {"col1": 1, "col2": "val12"}, + {"col1": 2, "col2": "val22"}, + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("jsonl") ) - .set_file_type("jsonl") .set_expected_catalog( { "streams": [ @@ -705,9 +892,7 @@ "json_schema": { "type": "object", "properties": { - "col1": { - "type": "integer" - }, + "col1": {"type": "integer"}, "col2": { "type": "string", }, @@ -728,10 +913,24 @@ ) .set_expected_records( [ - {"data": {"col1": 1, "col2": "val12", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, - {"data": {"col1": 2, "col2": "val22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.jsonl"}, "stream": "stream1"}, + { + "data": { + "col1": 1, + "col2": "val12", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl", + }, + "stream": "stream1", + }, + { + "data": { + "col1": 2, + "col2": "val22", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.jsonl", + }, + "stream": "stream1", + }, ] ) ).build() diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/parquet_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/parquet_scenarios.py index efba1c4bc043..0852de4a361a 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/parquet_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/parquet_scenarios.py @@ -6,8 +6,9 @@ import decimal import pyarrow as pa -from airbyte_cdk.sources.file_based.exceptions import SchemaInferenceError +from airbyte_cdk.utils.traced_exception import AirbyteTracedException from unit_tests.sources.file_based.in_memory_files_source import TemporaryParquetFilesStreamReader +from unit_tests.sources.file_based.scenarios.file_based_source_builder import FileBasedSourceBuilder from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder _single_parquet_file = { @@ -35,12 +36,14 @@ _parquet_file_with_decimal = { "a.parquet": { "contents": [ - ("col1", ), + ("col1",), (decimal.Decimal("13.00"),), ], - "schema": pa.schema([ - pa.field("col1", pa.decimal128(5, 2)), - ]), + "schema": pa.schema( + [ + pa.field("col1", pa.decimal128(5, 2)), + ] + ), "last_modified": "2023-06-05T03:54:07.000Z", } } @@ -67,98 +70,81 @@ _parquet_file_with_various_types = { "a.parquet": { "contents": [ - ("col_bool", - - "col_int8", - "col_int16", - "col_int32", - - "col_uint8", - "col_uint16", - "col_uint32", - "col_uint64", - - "col_float32", - "col_float64", - - "col_string", - - "col_date32", - "col_date64", - - "col_timestamp_without_tz", - "col_timestamp_with_tz", - - "col_time32s", - "col_time32ms", - "col_time64us", - - "col_struct", - "col_list", - "col_duration", - "col_binary", - ), - (True, - - -1, - 1, - 2, - - 2, - 3, - 4, - 5, - - 3.14, - 5.0, - - "2020-01-01", - - datetime.date(2021, 1, 1), - datetime.date(2022, 1, 1), - datetime.datetime(2023, 1, 1, 1, 2, 3), - datetime.datetime(2024, 3, 4, 5, 6, 7, tzinfo=datetime.timezone.utc), - - datetime.time(1, 2, 3), - datetime.time(2, 3, 4), - datetime.time(1, 2, 3, 4), - {"struct_key": "struct_value"}, - [1, 2, 3, 4], - 12345, - b"binary string. Hello world!", - ), + ( + "col_bool", + "col_int8", + "col_int16", + "col_int32", + "col_uint8", + "col_uint16", + "col_uint32", + "col_uint64", + "col_float32", + "col_float64", + "col_string", + "col_date32", + "col_date64", + "col_timestamp_without_tz", + "col_timestamp_with_tz", + "col_time32s", + "col_time32ms", + "col_time64us", + "col_struct", + "col_list", + "col_duration", + "col_binary", + ), + ( + True, + -1, + 1, + 2, + 2, + 3, + 4, + 5, + 3.14, + 5.0, + "2020-01-01", + datetime.date(2021, 1, 1), + datetime.date(2022, 1, 1), + datetime.datetime(2023, 1, 1, 1, 2, 3), + datetime.datetime(2024, 3, 4, 5, 6, 7, tzinfo=datetime.timezone.utc), + datetime.time(1, 2, 3), + datetime.time(2, 3, 4), + datetime.time(1, 2, 3, 4), + {"struct_key": "struct_value"}, + [1, 2, 3, 4], + 12345, + b"binary string. Hello world!", + ), ], - "schema": pa.schema([ - pa.field("col_bool", pa.bool_()), - - pa.field("col_int8", pa.int8()), - pa.field("col_int16", pa.int16()), - pa.field("col_int32", pa.int32()), - - pa.field("col_uint8", pa.uint8()), - pa.field("col_uint16", pa.uint16()), - pa.field("col_uint32", pa.uint32()), - pa.field("col_uint64", pa.uint64()), - - pa.field("col_float32", pa.float32()), - pa.field("col_float64", pa.float64()), - - pa.field("col_string", pa.string()), - - pa.field("col_date32", pa.date32()), - pa.field("col_date64", pa.date64()), - pa.field("col_timestamp_without_tz", pa.timestamp("s")), - pa.field("col_timestamp_with_tz", pa.timestamp("s", tz="UTC")), - - pa.field("col_time32s", pa.time32("s")), - pa.field("col_time32ms", pa.time32("ms")), - pa.field("col_time64us", pa.time64("us")), - - pa.field("col_struct", pa.struct([pa.field("struct_key", pa.string())])), - pa.field("col_list", pa.list_(pa.int32())), - pa.field("col_duration", pa.duration("s")), - pa.field("col_binary", pa.binary()) - ]), + "schema": pa.schema( + [ + pa.field("col_bool", pa.bool_()), + pa.field("col_int8", pa.int8()), + pa.field("col_int16", pa.int16()), + pa.field("col_int32", pa.int32()), + pa.field("col_uint8", pa.uint8()), + pa.field("col_uint16", pa.uint16()), + pa.field("col_uint32", pa.uint32()), + pa.field("col_uint64", pa.uint64()), + pa.field("col_float32", pa.float32()), + pa.field("col_float64", pa.float64()), + pa.field("col_string", pa.string()), + pa.field("col_date32", pa.date32()), + pa.field("col_date64", pa.date64()), + pa.field("col_timestamp_without_tz", pa.timestamp("s")), + pa.field("col_timestamp_with_tz", pa.timestamp("s", tz="UTC")), + pa.field("col_time32s", pa.time32("s")), + pa.field("col_time32ms", pa.time32("ms")), + pa.field("col_time64us", pa.time64("us")), + pa.field("col_struct", pa.struct([pa.field("struct_key", pa.string())])), + pa.field("col_list", pa.list_(pa.int32())), + pa.field("col_duration", pa.duration("s")), + pa.field("col_binary", pa.binary()), + ] + ), "last_modified": "2023-06-05T03:54:07.000Z", } } @@ -171,21 +157,38 @@ "streams": [ { "name": "stream1", - "file_type": "parquet", + "format": {"filetype": "parquet"}, "globs": ["*"], "validation_policy": "Emit Record", } ] } ) - .set_stream_reader(TemporaryParquetFilesStreamReader(files=_single_parquet_file, file_type="parquet")) - .set_file_type("parquet") + .set_source_builder( + FileBasedSourceBuilder() + .set_stream_reader(TemporaryParquetFilesStreamReader(files=_single_parquet_file, file_type="parquet")) + .set_file_type("parquet") + ) .set_expected_records( [ - {"data": {"col1": "val11", "col2": "val12", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.parquet"}, "stream": "stream1"}, - {"data": {"col1": "val21", "col2": "val22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.parquet"}, "stream": "stream1"}, + { + "data": { + "col1": "val11", + "col2": "val12", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.parquet", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21", + "col2": "val22", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.parquet", + }, + "stream": "stream1", + }, ] ) .set_expected_catalog( @@ -196,19 +199,11 @@ "json_schema": { "type": "object", "properties": { - "col1": { - "type": ["null", "string"] - }, - "col2": { - "type": ["null", "string"] - }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, - } + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream1", "source_defined_cursor": True, @@ -227,21 +222,42 @@ "streams": [ { "name": "stream1", - "file_type": "parquet", + "format": {"filetype": "parquet"}, "globs": ["path_prefix/**/*"], "validation_policy": "Emit Record", } ] } ) - .set_stream_reader(TemporaryParquetFilesStreamReader(files=_single_partitioned_parquet_file, file_type="parquet")) - .set_file_type("parquet") + .set_source_builder( + FileBasedSourceBuilder() + .set_stream_reader(TemporaryParquetFilesStreamReader(files=_single_partitioned_parquet_file, file_type="parquet")) + .set_file_type("parquet") + ) .set_expected_records( [ - {"data": {"col1": "val11", "col2": "val12", "partition1": "1", "partition2": "2","_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "path_prefix/partition1=1/partition2=2/a.parquet"}, "stream": "stream1"}, - {"data": {"col1": "val21", "col2": "val22", "partition1": "1", "partition2": "2", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "path_prefix/partition1=1/partition2=2/a.parquet"}, "stream": "stream1"}, + { + "data": { + "col1": "val11", + "col2": "val12", + "partition1": "1", + "partition2": "2", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "path_prefix/partition1=1/partition2=2/a.parquet", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21", + "col2": "val22", + "partition1": "1", + "partition2": "2", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "path_prefix/partition1=1/partition2=2/a.parquet", + }, + "stream": "stream1", + }, ] ) .set_expected_catalog( @@ -252,25 +268,13 @@ "json_schema": { "type": "object", "properties": { - "col1": { - "type": ["null", "string"] - }, - "col2": { - "type": ["null", "string"] - }, - "partition1": { - "type": ["null", "string"] - }, - "partition2": { - "type": ["null", "string"] - }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, - } + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "partition1": {"type": ["null", "string"]}, + "partition2": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream1", "source_defined_cursor": True, @@ -289,15 +293,18 @@ "streams": [ { "name": "stream1", - "file_type": "parquet", + "format": {"filetype": "parquet"}, "globs": ["*"], "validation_policy": "Emit Record", } ] } ) - .set_file_type("parquet") - .set_stream_reader(TemporaryParquetFilesStreamReader(files=_multiple_parquet_file, file_type="parquet")) + .set_source_builder( + FileBasedSourceBuilder() + .set_file_type("parquet") + .set_stream_reader(TemporaryParquetFilesStreamReader(files=_multiple_parquet_file, file_type="parquet")) + ) .set_expected_catalog( { "streams": [ @@ -306,22 +313,12 @@ "json_schema": { "type": "object", "properties": { - "col1": { - "type": ["null", "string"] - }, - "col2": { - "type": ["null", "string"] - }, - "col3": { - "type": ["null", "string"] - }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, - } + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "col3": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream1", "source_defined_cursor": True, @@ -332,14 +329,44 @@ ) .set_expected_records( [ - {"data": {"col1": "val11a", "col2": "val12a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.parquet"}, "stream": "stream1"}, - {"data": {"col1": "val21a", "col2": "val22a", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.parquet"}, "stream": "stream1"}, - {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.parquet"}, "stream": "stream1"}, - {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.parquet"}, "stream": "stream1"}, + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.parquet", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.parquet", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.parquet", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.parquet", + }, + "stream": "stream1", + }, ] ) ).build() @@ -352,15 +379,18 @@ "streams": [ { "name": "stream1", - "file_type": "parquet", + "format": {"filetype": "parquet"}, "globs": ["*"], "validation_policy": "Emit Record", } ] } ) - .set_stream_reader(TemporaryParquetFilesStreamReader(files=_parquet_file_with_various_types, file_type="parquet")) - .set_file_type("parquet") + .set_source_builder( + FileBasedSourceBuilder() + .set_stream_reader(TemporaryParquetFilesStreamReader(files=_parquet_file_with_various_types, file_type="parquet")) + .set_file_type("parquet") + ) .set_expected_catalog( { "streams": [ @@ -402,22 +432,10 @@ "col_string": { "type": ["null", "string"], }, - "col_date32": { - "type": ["null", "string"], - "format": "date" - }, - "col_date64": { - "type": ["null", "string"], - "format": "date" - }, - "col_timestamp_without_tz": { - "type": ["null", "string"], - "format": "date-time" - }, - "col_timestamp_with_tz": { - "type": ["null", "string"], - "format": "date-time" - }, + "col_date32": {"type": ["null", "string"], "format": "date"}, + "col_date64": {"type": ["null", "string"], "format": "date"}, + "col_timestamp_without_tz": {"type": ["null", "string"], "format": "date-time"}, + "col_timestamp_with_tz": {"type": ["null", "string"], "format": "date-time"}, "col_time32s": { "type": ["null", "string"], }, @@ -456,31 +474,35 @@ ) .set_expected_records( [ - {"data": {"col_bool": True, - "col_int8": -1, - "col_int16": 1, - "col_int32": 2, - "col_uint8": 2, - "col_uint16": 3, - "col_uint32": 4, - "col_uint64": 5, - "col_float32": 3.14, - "col_float64": 5.0, - "col_string": "2020-01-01", - "col_date32": "2021-01-01", - "col_date64": "2022-01-01", - "col_timestamp_without_tz": "2023-01-01T01:02:03", - "col_timestamp_with_tz": "2024-03-04T05:06:07+00:00", - "col_time32s": "01:02:03", - "col_time32ms": "02:03:04", - "col_time64us": "01:02:03.000004", - "col_struct": {"struct_key": "struct_value"}, - "col_list": [1, 2, 3, 4], - "col_duration": 12345, - "col_binary": "binary string. Hello world!", - "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.parquet"}, "stream": "stream1" - }, + { + "data": { + "col_bool": True, + "col_int8": -1, + "col_int16": 1, + "col_int32": 2, + "col_uint8": 2, + "col_uint16": 3, + "col_uint32": 4, + "col_uint64": 5, + "col_float32": 3.14, + "col_float64": 5.0, + "col_string": "2020-01-01", + "col_date32": "2021-01-01", + "col_date64": "2022-01-01", + "col_timestamp_without_tz": "2023-01-01T01:02:03", + "col_timestamp_with_tz": "2024-03-04T05:06:07+00:00", + "col_time32s": "01:02:03", + "col_time32ms": "02:03:04", + "col_time64us": "01:02:03.000004", + "col_struct": {"struct_key": "struct_value"}, + "col_list": [1, 2, 3, 4], + "col_duration": 12345, + "col_binary": "binary string. Hello world!", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.parquet", + }, + "stream": "stream1", + }, ] ) ).build() @@ -493,19 +515,28 @@ "streams": [ { "name": "stream1", - "file_type": "parquet", + "format": {"filetype": "parquet"}, "globs": ["*"], "validation_policy": "Emit Record", } ] } ) - .set_stream_reader(TemporaryParquetFilesStreamReader(files=_parquet_file_with_decimal, file_type="parquet")) - .set_file_type("parquet") + .set_source_builder( + FileBasedSourceBuilder() + .set_stream_reader(TemporaryParquetFilesStreamReader(files=_parquet_file_with_decimal, file_type="parquet")) + .set_file_type("parquet") + ) .set_expected_records( [ - {"data": {"col1": "13.00", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.parquet"}, "stream": "stream1"}, + { + "data": { + "col1": "13.00", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.parquet", + }, + "stream": "stream1", + }, ] ) .set_expected_catalog( @@ -516,16 +547,10 @@ "json_schema": { "type": "object", "properties": { - "col1": { - "type": ["null", "string"] - }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, - } + "col1": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream1", "source_defined_cursor": True, @@ -544,23 +569,28 @@ "streams": [ { "name": "stream1", - "file_type": "parquet", "globs": ["*"], "validation_policy": "Emit Record", - "format": { - "filetype": "parquet", - "decimal_as_float": False - } + "format": {"filetype": "parquet", "decimal_as_float": False}, } ] } ) - .set_stream_reader(TemporaryParquetFilesStreamReader(files=_parquet_file_with_decimal, file_type="parquet")) - .set_file_type("parquet") + .set_source_builder( + FileBasedSourceBuilder() + .set_stream_reader(TemporaryParquetFilesStreamReader(files=_parquet_file_with_decimal, file_type="parquet")) + .set_file_type("parquet") + ) .set_expected_records( [ - {"data": {"col1": "13.00", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.parquet"}, "stream": "stream1"}, + { + "data": { + "col1": "13.00", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.parquet", + }, + "stream": "stream1", + }, ] ) .set_expected_catalog( @@ -571,16 +601,10 @@ "json_schema": { "type": "object", "properties": { - "col1": { - "type": ["null", "string"] - }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, - } + "col1": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream1", "source_defined_cursor": True, @@ -599,23 +623,24 @@ "streams": [ { "name": "stream1", - "file_type": "parquet", "globs": ["*"], "validation_policy": "Emit Record", - "format": { - "filetype": "parquet", - "decimal_as_float": True - } + "format": {"filetype": "parquet", "decimal_as_float": True}, } ] } ) - .set_stream_reader(TemporaryParquetFilesStreamReader(files=_parquet_file_with_decimal, file_type="parquet")) - .set_file_type("parquet") + .set_source_builder( + FileBasedSourceBuilder() + .set_stream_reader(TemporaryParquetFilesStreamReader(files=_parquet_file_with_decimal, file_type="parquet")) + .set_file_type("parquet") + ) .set_expected_records( [ - {"data": {"col1": 13.00, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.parquet"}, "stream": "stream1"}, + { + "data": {"col1": 13.00, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.parquet"}, + "stream": "stream1", + }, ] ) .set_expected_catalog( @@ -626,16 +651,10 @@ "json_schema": { "type": "object", "properties": { - "col1": { - "type": ["null", "number"] - }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, - } + "col1": {"type": ["null", "number"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream1", "source_defined_cursor": True, @@ -654,7 +673,6 @@ "streams": [ { "name": "stream1", - "file_type": "parquet", "format": { "filetype": "parquet", }, @@ -664,12 +682,17 @@ ] } ) - .set_stream_reader(TemporaryParquetFilesStreamReader(files=_parquet_file_with_decimal, file_type="parquet")) - .set_file_type("parquet") + .set_source_builder( + FileBasedSourceBuilder() + .set_stream_reader(TemporaryParquetFilesStreamReader(files=_parquet_file_with_decimal, file_type="parquet")) + .set_file_type("parquet") + ) .set_expected_records( [ - {"data": {"col1": 13.00, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.parquet"}, "stream": "stream1"}, + { + "data": {"col1": 13.00, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.parquet"}, + "stream": "stream1", + }, ] ) .set_expected_catalog( @@ -680,16 +703,10 @@ "json_schema": { "type": "object", "properties": { - "col1": { - "type": ["null", "number"] - }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, - } + "col1": {"type": ["null", "number"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream1", "source_defined_cursor": True, @@ -703,34 +720,15 @@ parquet_with_invalid_config_scenario = ( TestScenarioBuilder() .set_name("parquet_with_invalid_config") - .set_config( - { - "streams": [ - { - "name": "stream1", - "file_type": "parquet", - "globs": ["*"], - "validation_policy": "Emit Record", - "format": { - "filetype": "csv" - } - } - ] - } + .set_config({"streams": [{"name": "stream1", "globs": ["*"], "validation_policy": "Emit Record", "format": {"filetype": "csv"}}]}) + .set_source_builder( + FileBasedSourceBuilder() + .set_stream_reader(TemporaryParquetFilesStreamReader(files=_single_parquet_file, file_type="parquet")) + .set_file_type("parquet") ) - .set_stream_reader(TemporaryParquetFilesStreamReader(files=_single_parquet_file, file_type="parquet")) - .set_file_type("parquet") - .set_expected_records( - [ - ] - ) - .set_expected_logs({"read": [ - { - "level": "ERROR", - "message": "Error parsing record" - } - ]}) - .set_expected_discover_error(SchemaInferenceError, "Error inferring schema from files") + .set_expected_records([]) + .set_expected_logs({"read": [{"level": "ERROR", "message": "Error parsing record"}]}) + .set_expected_discover_error(AirbyteTracedException, "Error inferring schema from files") .set_expected_catalog( { "streams": [ @@ -739,19 +737,11 @@ "json_schema": { "type": "object", "properties": { - "col1": { - "type": ["null", "string"] - }, - "col2": { - "type": ["null", "string"] - }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, - } + "col1": {"type": ["null", "string"]}, + "col2": {"type": ["null", "string"]}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream1", "source_defined_cursor": True, diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/scenario_builder.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/scenario_builder.py index b3cf8c44037e..75feaf360595 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/scenario_builder.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/scenario_builder.py @@ -1,22 +1,14 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - +from abc import ABC, abstractmethod from copy import deepcopy from dataclasses import dataclass, field -from typing import Any, List, Mapping, Optional, Tuple, Type +from typing import Any, Generic, List, Mapping, Optional, Set, Tuple, Type, TypeVar -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.file_based.availability_strategy.abstract_file_based_availability_strategy import ( - AbstractFileBasedAvailabilityStrategy, -) -from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy, DefaultDiscoveryPolicy -from airbyte_cdk.sources.file_based.file_based_source import default_parsers -from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader -from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser -from airbyte_cdk.sources.file_based.schema_validation_policies import AbstractSchemaValidationPolicy -from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor -from unit_tests.sources.file_based.in_memory_files_source import InMemoryFilesSource +from airbyte_cdk.models import AirbyteAnalyticsTraceMessage, SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_protocol.models import ConfiguredAirbyteCatalog @dataclass @@ -25,32 +17,44 @@ class IncrementalScenarioConfig: expected_output_state: Optional[Mapping[str, Any]] = None -class TestScenario: +SourceType = TypeVar("SourceType", bound=AbstractSource) + + +class SourceBuilder(ABC, Generic[SourceType]): + """ + A builder that creates a source instance of type SourceType + """ + + @abstractmethod + def build(self, configured_catalog: Optional[Mapping[str, Any]]) -> SourceType: + raise NotImplementedError() + + +class TestScenario(Generic[SourceType]): def __init__( - self, - name: str, - config: Mapping[str, Any], - files: Mapping[str, Any], - file_type: str, - expected_spec: Optional[Mapping[str, Any]], - expected_check_status: Optional[str], - expected_catalog: Optional[Mapping[str, Any]], - expected_logs: Optional[Mapping[str, List[Mapping[str, Any]]]], - expected_records: List[Mapping[str, Any]], - availability_strategy: Optional[AbstractFileBasedAvailabilityStrategy], - discovery_policy: Optional[AbstractDiscoveryPolicy], - validation_policies: Mapping[str, AbstractSchemaValidationPolicy], - parsers: Mapping[str, FileTypeParser], - stream_reader: Optional[AbstractFileBasedStreamReader], - expected_check_error: Tuple[Optional[Type[Exception]], Optional[str]], - expected_discover_error: Tuple[Optional[Type[Exception]], Optional[str]], - expected_read_error: Tuple[Optional[Type[Exception]], Optional[str]], - incremental_scenario_config: Optional[IncrementalScenarioConfig], - file_write_options: Mapping[str, Any], - cursor_cls: Optional[Type[AbstractFileBasedCursor]], + self, + name: str, + config: Mapping[str, Any], + source: SourceType, + expected_spec: Optional[Mapping[str, Any]], + expected_check_status: Optional[str], + expected_catalog: Optional[Mapping[str, Any]], + expected_logs: Optional[Mapping[str, List[Mapping[str, Any]]]], + expected_records: List[Mapping[str, Any]], + expected_check_error: Tuple[Optional[Type[Exception]], Optional[str]], + expected_discover_error: Tuple[Optional[Type[Exception]], Optional[str]], + expected_read_error: Tuple[Optional[Type[Exception]], Optional[str]], + incremental_scenario_config: Optional[IncrementalScenarioConfig], + expected_analytics: Optional[List[AirbyteAnalyticsTraceMessage]] = None, + log_levels: Optional[Set[str]] = None, + catalog: Optional[ConfiguredAirbyteCatalog] = None, ): + if log_levels is None: + log_levels = {"ERROR", "WARN", "WARNING"} self.name = name self.config = config + self.catalog = catalog + self.source = source self.expected_spec = expected_spec self.expected_check_status = expected_check_status self.expected_catalog = expected_catalog @@ -59,39 +63,33 @@ def __init__( self.expected_check_error = expected_check_error self.expected_discover_error = expected_discover_error self.expected_read_error = expected_read_error - self.source = InMemoryFilesSource( - files, - file_type, - availability_strategy, - discovery_policy, - validation_policies, - parsers, - stream_reader, - self.configured_catalog(SyncMode.incremental if incremental_scenario_config else SyncMode.full_refresh), - file_write_options, - cursor_cls, - ) self.incremental_scenario_config = incremental_scenario_config + self.expected_analytics = expected_analytics + self.log_levels = log_levels self.validate() def validate(self) -> None: assert self.name - if not self.expected_catalog: - return - streams = {s["name"] for s in self.config["streams"]} - expected_streams = {s["name"] for s in self.expected_catalog["streams"]} - assert expected_streams <= streams def configured_catalog(self, sync_mode: SyncMode) -> Optional[Mapping[str, Any]]: - if not self.expected_catalog: - return None + # The preferred way of returning the catalog for the TestScenario is by providing it at the initialization. The previous solution + # relied on `self.source.streams` which might raise an exception hence screwing the tests results as the user might expect the + # exception to be raised as part of the actual check/discover/read commands + # Note that to avoid a breaking change, we still attempt to automatically generate the catalog based on the streams + if self.catalog: + return self.catalog.dict() # type: ignore # dict() is not typed + catalog: Mapping[str, Any] = {"streams": []} - for stream in self.expected_catalog["streams"]: + for stream in self.source.streams(self.config): catalog["streams"].append( { - "stream": stream, + "stream": { + "name": stream.name, + "json_schema": {}, + "supported_sync_modes": [sync_mode.value], + }, "sync_mode": sync_mode.value, - "destination_sync_mode": "append", + "destination_sync_mode": "append" } ) @@ -104,134 +102,126 @@ def input_state(self) -> List[Mapping[str, Any]]: return [] -class TestScenarioBuilder: +class TestScenarioBuilder(Generic[SourceType]): + """ + A builder that creates a TestScenario instance for a source of type SourceType + """ + def __init__(self) -> None: self._name = "" self._config: Mapping[str, Any] = {} - self._files: Mapping[str, Any] = {} - self._file_type: Optional[str] = None + self._catalog: Optional[ConfiguredAirbyteCatalog] = None self._expected_spec: Optional[Mapping[str, Any]] = None self._expected_check_status: Optional[str] = None self._expected_catalog: Mapping[str, Any] = {} self._expected_logs: Optional[Mapping[str, Any]] = None self._expected_records: List[Mapping[str, Any]] = [] - self._availability_strategy: Optional[AbstractFileBasedAvailabilityStrategy] = None - self._discovery_policy: AbstractDiscoveryPolicy = DefaultDiscoveryPolicy() - self._validation_policies: Optional[Mapping[str, AbstractSchemaValidationPolicy]] = None - self._parsers = default_parsers - self._stream_reader: Optional[AbstractFileBasedStreamReader] = None self._expected_check_error: Tuple[Optional[Type[Exception]], Optional[str]] = None, None self._expected_discover_error: Tuple[Optional[Type[Exception]], Optional[str]] = None, None self._expected_read_error: Tuple[Optional[Type[Exception]], Optional[str]] = None, None self._incremental_scenario_config: Optional[IncrementalScenarioConfig] = None - self._file_write_options: Mapping[str, Any] = {} - self._cursor_cls: Optional[Type[AbstractFileBasedCursor]] = None + self._expected_analytics: Optional[List[AirbyteAnalyticsTraceMessage]] = None + self.source_builder: Optional[SourceBuilder[SourceType]] = None + self._log_levels = None - def set_name(self, name: str) -> "TestScenarioBuilder": + def set_name(self, name: str) -> "TestScenarioBuilder[SourceType]": self._name = name return self - def set_config(self, config: Mapping[str, Any]) -> "TestScenarioBuilder": + def set_config(self, config: Mapping[str, Any]) -> "TestScenarioBuilder[SourceType]": self._config = config return self - def set_files(self, files: Mapping[str, Any]) -> "TestScenarioBuilder": - self._files = files - return self - - def set_file_type(self, file_type: str) -> "TestScenarioBuilder": - self._file_type = file_type + def set_expected_spec(self, expected_spec: Mapping[str, Any]) -> "TestScenarioBuilder[SourceType]": + self._expected_spec = expected_spec return self - def set_expected_spec(self, expected_spec: Mapping[str, Any]) -> "TestScenarioBuilder": - self._expected_spec = expected_spec + def set_catalog(self, catalog: ConfiguredAirbyteCatalog) -> "TestScenarioBuilder[SourceType]": + self._catalog = catalog return self - def set_expected_check_status(self, expected_check_status: str) -> "TestScenarioBuilder": + def set_expected_check_status(self, expected_check_status: str) -> "TestScenarioBuilder[SourceType]": self._expected_check_status = expected_check_status return self - def set_expected_catalog(self, expected_catalog: Mapping[str, Any]) -> "TestScenarioBuilder": + def set_expected_catalog(self, expected_catalog: Mapping[str, Any]) -> "TestScenarioBuilder[SourceType]": self._expected_catalog = expected_catalog return self - def set_expected_logs(self, expected_logs: Mapping[str, List[Mapping[str, Any]]]) -> "TestScenarioBuilder": + def set_expected_logs(self, expected_logs: Mapping[str, List[Mapping[str, Any]]]) -> "TestScenarioBuilder[SourceType]": self._expected_logs = expected_logs return self - def set_expected_records(self, expected_records: List[Mapping[str, Any]]) -> "TestScenarioBuilder": + def set_expected_records(self, expected_records: List[Mapping[str, Any]]) -> "TestScenarioBuilder[SourceType]": self._expected_records = expected_records return self - def set_parsers(self, parsers: Mapping[str, FileTypeParser]) -> "TestScenarioBuilder": - self._parsers = parsers - return self - - def set_availability_strategy(self, availability_strategy: AbstractFileBasedAvailabilityStrategy) -> "TestScenarioBuilder": - self._availability_strategy = availability_strategy - return self - - def set_discovery_policy(self, discovery_policy: AbstractDiscoveryPolicy) -> "TestScenarioBuilder": - self._discovery_policy = discovery_policy - return self - - def set_validation_policies(self, validation_policies: Mapping[str, AbstractSchemaValidationPolicy]) -> "TestScenarioBuilder": - self._validation_policies = validation_policies - return self - - def set_stream_reader(self, stream_reader: AbstractFileBasedStreamReader) -> "TestScenarioBuilder": - self._stream_reader = stream_reader - return self - - def set_cursor_cls(self, cursor_cls: AbstractFileBasedCursor) -> "TestScenarioBuilder": - self._cursor_cls = cursor_cls - return self - - def set_incremental_scenario_config(self, incremental_scenario_config: IncrementalScenarioConfig) -> "TestScenarioBuilder": + def set_incremental_scenario_config(self, incremental_scenario_config: IncrementalScenarioConfig) -> "TestScenarioBuilder[SourceType]": self._incremental_scenario_config = incremental_scenario_config return self - def set_expected_check_error(self, error: Optional[Type[Exception]], message: str) -> "TestScenarioBuilder": + def set_expected_check_error(self, error: Optional[Type[Exception]], message: str) -> "TestScenarioBuilder[SourceType]": self._expected_check_error = error, message return self - def set_expected_discover_error(self, error: Type[Exception], message: str) -> "TestScenarioBuilder": + def set_expected_discover_error(self, error: Type[Exception], message: str) -> "TestScenarioBuilder[SourceType]": self._expected_discover_error = error, message return self - def set_expected_read_error(self, error: Type[Exception], message: str) -> "TestScenarioBuilder": + def set_expected_read_error(self, error: Type[Exception], message: str) -> "TestScenarioBuilder[SourceType]": self._expected_read_error = error, message return self - def set_file_write_options(self, file_write_options: Mapping[str, Any]) -> "TestScenarioBuilder": - self._file_write_options = file_write_options + def set_log_levels(self, levels: Set[str]) -> "TestScenarioBuilder": + self._log_levels = levels return self - def copy(self) -> "TestScenarioBuilder": + def set_source_builder(self, source_builder: SourceBuilder[SourceType]) -> "TestScenarioBuilder[SourceType]": + self.source_builder = source_builder + return self + + def set_expected_analytics(self, expected_analytics: Optional[List[AirbyteAnalyticsTraceMessage]]) -> "TestScenarioBuilder[SourceType]": + self._expected_analytics = expected_analytics + return self + + def copy(self) -> "TestScenarioBuilder[SourceType]": return deepcopy(self) - def build(self) -> TestScenario: - if self._file_type is None: - raise ValueError("file_type is not set") + def build(self) -> "TestScenario[SourceType]": + if self.source_builder is None: + raise ValueError("source_builder is not set") + source = self.source_builder.build( + self._configured_catalog(SyncMode.incremental if self._incremental_scenario_config else SyncMode.full_refresh) + ) return TestScenario( self._name, self._config, - self._files, - self._file_type, + source, self._expected_spec, self._expected_check_status, self._expected_catalog, self._expected_logs, self._expected_records, - self._availability_strategy, - self._discovery_policy, - self._validation_policies or {}, - self._parsers, - self._stream_reader, self._expected_check_error, self._expected_discover_error, self._expected_read_error, self._incremental_scenario_config, - self._file_write_options, - self._cursor_cls, + self._expected_analytics, + self._log_levels, + self._catalog, ) + + def _configured_catalog(self, sync_mode: SyncMode) -> Optional[Mapping[str, Any]]: + if not self._expected_catalog: + return None + catalog: Mapping[str, Any] = {"streams": []} + for stream in self._expected_catalog["streams"]: + catalog["streams"].append( + { + "stream": stream, + "sync_mode": sync_mode.value, + "destination_sync_mode": "append", + } + ) + + return catalog diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/unstructured_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/unstructured_scenarios.py new file mode 100644 index 000000000000..dc0824512a43 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/unstructured_scenarios.py @@ -0,0 +1,606 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import base64 + +import nltk +from airbyte_cdk.utils.traced_exception import AirbyteTracedException +from unit_tests.sources.file_based.scenarios.file_based_source_builder import FileBasedSourceBuilder +from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder + +# import nltk data for pdf parser +nltk.download("punkt") +nltk.download("averaged_perceptron_tagger") + +json_schema = { + "type": "object", + "properties": { + "content": {"type": ["null", "string"], "description": "Content of the file as markdown. Might be null if the file could not be parsed"}, + "document_key": {"type": ["null", "string"], "description": "Unique identifier of the document, e.g. the file path"}, + "_ab_source_file_parse_error": {"type": ["null", "string"], "description": "Error message if the file could not be parsed even though the file is supported"}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + } +} + +simple_markdown_scenario = ( + TestScenarioBuilder() + .set_name("simple_markdown_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "unstructured"}, + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.md": { + "contents": bytes( + "# Title 1\n\n## Title 2\n\n### Title 3\n\n#### Title 4\n\n##### Title 5\n\n###### Title 6\n\n", "UTF-8" + ), + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.md": { + "contents": bytes("Just some text", "UTF-8"), + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "c": { + "contents": bytes("Detected via mime type", "UTF-8"), + "last_modified": "2023-06-05T03:54:07.000Z", + "mime_type": "text/markdown", + }, + } + ) + .set_file_type("unstructured") + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": json_schema, + "name": "stream1", + "source_defined_cursor": True, + 'source_defined_primary_key': [["document_key"]], + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "document_key": "a.md", + "content": "# Title 1\n\n## Title 2\n\n### Title 3\n\n#### Title 4\n\n##### Title 5\n\n###### Title 6\n\n", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.md", + "_ab_source_file_parse_error": None, + }, + "stream": "stream1", + }, + { + "data": { + "document_key": "b.md", + "content": "Just some text", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.md", + "_ab_source_file_parse_error": None, + }, + "stream": "stream1", + }, + { + "data": { + "document_key": "c", + "content": "Detected via mime type", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "c", + "_ab_source_file_parse_error": None, + + }, + "stream": "stream1", + }, + ] + ) +).build() + +simple_txt_scenario = ( + TestScenarioBuilder() + .set_name("simple_txt_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "unstructured"}, + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.txt": { + "contents": bytes( + "Just some raw text", "UTF-8" + ), + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b": { + "contents": bytes("Detected via mime type", "UTF-8"), + "last_modified": "2023-06-05T03:54:07.000Z", + "mime_type": "text/plain", + }, + } + ) + .set_file_type("unstructured") + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": json_schema, + "name": "stream1", + "source_defined_cursor": True, + 'source_defined_primary_key': [["document_key"]], + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "document_key": "a.txt", + "content": "Just some raw text", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.txt", + "_ab_source_file_parse_error": None, + }, + "stream": "stream1", + }, + { + "data": { + "document_key": "b", + "content": "Detected via mime type", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b", + "_ab_source_file_parse_error": None, + + }, + "stream": "stream1", + }, + ] + ) +).build() + +# If skip unprocessable file types is set to false, then discover will fail if it encounters a non-matching file type +unstructured_invalid_file_type_discover_scenario_no_skip = ( + TestScenarioBuilder() + .set_name("unstructured_invalid_file_type_discover_scenario_no_skip") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "unstructured", "skip_unprocessable_files": False}, + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": bytes("Just a humble text file", "UTF-8"), + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("unstructured") + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": json_schema, + "name": "stream1", + "source_defined_cursor": True, + 'source_defined_primary_key': [["document_key"]], + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records([]) + .set_expected_discover_error(AirbyteTracedException, "Error inferring schema from files") + .set_expected_read_error( + AirbyteTracedException, + "Please check the logged errors for more information.", + ) +).build() + +# If skip unprocessable file types is set to true, then discover will succeed even if there are non-matching file types +unstructured_invalid_file_type_discover_scenario_skip = ( + TestScenarioBuilder() + .set_name("unstructured_invalid_file_type_discover_scenario_skip") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "unstructured", "skip_unprocessable_files": True}, + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": bytes("Just a humble text file", "UTF-8"), + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("unstructured") + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": json_schema, + "name": "stream1", + "source_defined_cursor": True, + 'source_defined_primary_key': [["document_key"]], + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "document_key": "a.csv", + "content": None, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + "_ab_source_file_parse_error": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. Contact Support if you need assistance.\nfilename=a.csv message=File type FileType.CSV is not supported. Supported file types are FileType.MD, FileType.PDF, FileType.DOCX, FileType.PPTX, FileType.TXT", + }, + "stream": "stream1", + } + ] + ) +).build() + +# TODO When working on https://github.com/airbytehq/airbyte/issues/31605, this test should be split into two tests: +# 1. Test that the file is skipped if skip_unprocessable_files is set to true +# 2. Test that the sync fails if skip_unprocessable_files is set to false +unstructured_invalid_file_type_read_scenario = ( + TestScenarioBuilder() + .set_name("unstructured_invalid_file_type_read_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "unstructured", "skip_unprocessable_files": False}, + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.md": { + "contents": bytes("A harmless markdown file", "UTF-8"), + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.csv": { + "contents": bytes("An evil text file", "UTF-8"), + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("unstructured") + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": json_schema, + "name": "stream1", + "source_defined_cursor": True, + 'source_defined_primary_key': [["document_key"]], + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "document_key": "a.md", + "content": "A harmless markdown file", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.md", + "_ab_source_file_parse_error": None, + }, + "stream": "stream1", + }, + ] + ) +).build() + +pdf_file = base64.b64decode( + "JVBERi0xLjEKJcKlwrHDqwoKMSAwIG9iagogIDw8IC9UeXBlIC9DYXRhbG9nCiAgICAgL1BhZ2VzIDIgMCBSCiAgPj4KZW5kb2JqCgoyIDAgb2JqCiAgPDwgL1R5cGUgL1BhZ2VzCiAgICAgL0tpZHMgWzMgMCBSXQogICAgIC9Db3VudCAxCiAgICAgL01lZGlhQm94IFswIDAgMzAwIDE0NF0KICA+PgplbmRvYmoKCjMgMCBvYmoKICA8PCAgL1R5cGUgL1BhZ2UKICAgICAgL1BhcmVudCAyIDAgUgogICAgICAvUmVzb3VyY2VzCiAgICAgICA8PCAvRm9udAogICAgICAgICAgIDw8IC9GMQogICAgICAgICAgICAgICA8PCAvVHlwZSAvRm9udAogICAgICAgICAgICAgICAgICAvU3VidHlwZSAvVHlwZTEKICAgICAgICAgICAgICAgICAgL0Jhc2VGb250IC9UaW1lcy1Sb21hbgogICAgICAgICAgICAgICA+PgogICAgICAgICAgID4+CiAgICAgICA+PgogICAgICAvQ29udGVudHMgNCAwIFIKICA+PgplbmRvYmoKCjQgMCBvYmoKICA8PCAvTGVuZ3RoIDU1ID4+CnN0cmVhbQogIEJUCiAgICAvRjEgMTggVGYKICAgIDAgMCBUZAogICAgKEhlbGxvIFdvcmxkKSBUagogIEVUCmVuZHN0cmVhbQplbmRvYmoKCnhyZWYKMCA1CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAxOCAwMDAwMCBuIAowMDAwMDAwMDc3IDAwMDAwIG4gCjAwMDAwMDAxNzggMDAwMDAgbiAKMDAwMDAwMDQ1NyAwMDAwMCBuIAp0cmFpbGVyCiAgPDwgIC9Sb290IDEgMCBSCiAgICAgIC9TaXplIDUKICA+PgpzdGFydHhyZWYKNTY1CiUlRU9GCg==" +) + +docx_file = base64.b64decode( + "UEsDBBQACAgIAEkqVFcAAAAAAAAAAAAAAAASAAAAd29yZC9udW1iZXJpbmcueG1spZNNTsMwEIVPwB0i79skFSAUNe2CCjbsgAO4jpNYtT3W2Eno7XGbv1IklIZV5Izf98bj5/X2S8mg5mgF6JTEy4gEXDPIhC5S8vnxsngigXVUZ1SC5ik5cku2m7t1k+hK7Tn6fYFHaJsolpLSOZOEoWUlV9QuwXDtizmgos4vsQgVxUNlFgyUoU7shRTuGK6i6JF0GEhJhTrpEAslGIKF3J0kCeS5YLz79Aqc4ttKdsAqxbU7O4bIpe8BtC2FsT1NzaX5YtlD6r8OUSvZ72vMFLcMaePnrGRr1ABmBoFxa/3fXVsciHE0YYAnxKCY0sJPz74TRYUeMKd0XIEG76X37oZ2Ro0HGWdh5ZRG2tKb2CPF4+8u6Ix5XuqNmJTiK4JXuQqHQM5BsJKi6wFyDkECO/DsmeqaDmHOiklxviJlghZI1RhSe9PNxtFVXN5LavhIK/5He0WozBj3+zm0ixcYP9wGWPWAcPMNUEsHCEkTQ39oAQAAPQUAAFBLAwQUAAgICABJKlRXAAAAAAAAAAAAAAAAEQAAAHdvcmQvc2V0dGluZ3MueG1spZVLbtswEIZP0DsY3Nt6xHYLIXKAJGi7aFZODzAmKYkwXyApq759qQcl2wEKxV2J/IfzzXA0Gj0+/RF8caLGMiVzlKxitKASK8JkmaPf79+X39DCOpAEuJI0R2dq0dPuy2OTWeqcP2UXniBtJnCOKud0FkUWV1SAXSlNpTcWyghwfmvKSIA51nqJldDg2IFx5s5RGsdbNGBUjmojswGxFAwbZVXhWpdMFQXDdHgEDzMnbu/yqnAtqHRdxMhQ7nNQ0lZM20AT99K8sQqQ078ucRI8nGv0nGjEQOMLLXgfqFGGaKMwtdarr71xJCbxjAK2iNFjTgrXMUMmApgcMW1z3IDG2Csfeyhah5ouMtXC8jmJ9KZf7GDAnD9mAXfU89Jfs1ldfEPwXq42Y0Peg8AVGBcA/B4CV/hIyQvIE4zNTMpZ7XxDIgxKA2JqUvupN5vEN+2yr0DTiVb+H+2HUbWe2n19D+3iC0w2nwOkAbDzI5AwqzmcnwEfS5+WJN1VF012At/NCYq6Q7SAmrt3OOyd0sH4NY17cz8Kp9W+H6sjZIP8UoLwn9fV1HxThLam2rD5N2hDRlcxudm3TvQNtO7DHsokR5yVlUtavvM74qd2tzmU6WBLO1va27oNYOyHoT89LCYtDdrFuYegPUzaOmjrSdsEbTNp26BtW606a2o4k0dfhrBs9UJxrhpKfk72D9JQj/Ar2/0FUEsHCAbSYFUWAgAADwcAAFBLAwQUAAgICABJKlRXAAAAAAAAAAAAAAAAEgAAAHdvcmQvZm9udFRhYmxlLnhtbKWUTU7DMBCFT8AdIu/bpAgQippUCAQbdsABBsdJrNoea+w09Pa4ND9QJJSGVZSM3/fG4xevNx9aRTtBTqLJ2GqZsEgYjoU0VcbeXh8XtyxyHkwBCo3I2F44tskv1m1aovEuCnLjUs0zVntv0zh2vBYa3BKtMKFYImnw4ZWqWANtG7vgqC14+S6V9Pv4MkluWIfBjDVk0g6x0JITOiz9QZJiWUouukevoCm+R8kD8kYL478cYxIq9IDG1dK6nqbn0kKx7iG7vzax06pf19opbgVBG85Cq6NRi1RYQi6cC18fjsWBuEomDPCAGBRTWvjp2XeiQZoBc0jGCWjwXgbvbmhfqHEj4yycmtLIsfQs3wlo/7sLmDHP73orJ6X4hBBUvqEhkHMQvAbyPUDNISjkW1Hcg9nBEOaimhTnE1IhoSLQY0jdWSe7Sk7i8lKDFSOt+h/tibCxY9yv5tC+/YGr6/MAlz0g7+6/qE0N6BD+O5KgWJyv4+5izD8BUEsHCK2HbQB5AQAAWgUAAFBLAwQUAAgICABJKlRXAAAAAAAAAAAAAAAADwAAAHdvcmQvc3R5bGVzLnhtbN2X7W7aMBSGr2D3gPK/TUgCQ6hp1Q91m1R11dpdwCExxMKxLduBsqufnS8gCVUakNYOfgQf+7zn+PFxbC6uXhMyWCEhMaOBNTx3rAGiIYswXQTW75f7s4k1kApoBIRRFFgbJK2ryy8X66lUG4LkQPtTOU3CwIqV4lPblmGMEpDnjCOqO+dMJKB0UyzsBMQy5WchSzgoPMMEq43tOs7YKmRYYKWCTguJswSHgkk2V8ZlyuZzHKLiUXqILnFzlzsWpgmiKotoC0R0DozKGHNZqiV91XRnXIqs3prEKiHluDXvEi0SsNaLkZA80JqJiAsWIim19S7vrBSHTgeARqLy6JLCfswykwQwrWRMadSEqtjnOnYBLZPaTmTLQpIuieRdD3gmQGyaWUAPnrv+HHeq4pqC9lKpqAqyj0QYg1ClAOmjQFi4RNEt0BVUxRwtOpVzTSnCsBCQbItUvmtlh06tXJ5j4GirtjhO7ZtgKd+Wu99HbWcHDkfvE3BLgUv9AoxYeIfmkBIlTVM8iaJZtLLHPaNKDtZTkCHGgXUtMOjw62kodxoIpLqWGHZM8TWV1XjbSMk/2rwCvVFct7TcyrqNAF2UNkSNzS6Ssesp8nor0+QQ4kyCYLOp3a9jq2j8Sok2QKpYIcsL2V0hu8ElOye0hNpw7c5BmPrisVHNun5EgfVo6jGbd5R76qMoY0whQeV0aD4oj525NuUVzAjak34xlk762cjBY4co7ZP4jsAcm03hOO8YDPMlmoFE0U9a9m4Dai/0qtrsxeIsEeKPO0MKQWN+0Aska3YOC3QjECxvkN7wVTpOUT3VSsNcIX2ODl3HzGeWDQ4s33HeXvmqyLeV6TvNysxtO1XYB6p7EKr7qaB6465QZ3XlCrLXsv1z25GQvYOQvY8NebLP2O3LOGSEiapuPfNtvHsnLe/eyQng+wfh+58JvjvpCn8P9jj7NGD7LbD9E8AeHYQ9+lSw/VPCPnirOBL2+CDs8f8JG9fC/hP4L1jpm1DjjpNZPzT18R71999BRi0oR0ehfE5nqpVm1fGhgXpuL6In/OuCayl22BBey03SO3CTLH/Jy79QSwcI2niuUysDAADPEgAAUEsDBBQACAgIAEkqVFcAAAAAAAAAAAAAAAARAAAAd29yZC9kb2N1bWVudC54bWyllV1u2zAMx0+wOwR6bx0H6VYYTfrQoMOAbQja7QCKJNtCJVGg5GTZ6Ud/t2lRuJlfZIrij3/JNHVz+8ea2V5h0OBWLL2cs5lyAqR2xYr9/nV/cc1mIXInuQGnVuyoArtdf7o5ZBJEZZWLMyK4kFmxYmWMPkuSIEplebgErxw5c0DLI5lYJJbjU+UvBFjPo95po+MxWcznn1mHgRWr0GUd4sJqgRAgj3VIBnmuheqGPgKn5G1DNp3kJmOCypAGcKHUPvQ0ey6NnGUP2b+3ib01/bqDn5JNIj/Q57CmTXQAlB5BqBBodtM6B2I6n3CANWKImCLhZc5eieXaDZi6OE5AQ+5Lyt0dWoMaNzKeRTBThLSu73qHHI+vVfAzzvN5vNeTqviEQFGxwqEgz0GIkmPsAeYcggHxpOQdd3s+FLMsJpXzCUlqXiC3Y5GGD33ZdH5SLo8l92qkFf9H+4pQ+bHcl+fQnv2B6dXHAIsesKYWuOPiqSA9Ts4OmQAD1Ivum4cljR/ksR49uanDyocVm3cP66Y2yrye3L6eetionFcmvuHZ4ovJdJl5jvybHGbTRqzfYj3gFklbMtrvCXlD8Mt0HbEZoqEle15jWJui8zRXRBY8F9QjPKqgcK/Y+g5cpPZZL4zt8lZXHRKUiG2wLx7/Ereky+nqetnIoJaVLhbtO6AmBmEBI3Id24P3xQ9eb2wHMQL9A+myXR3Bj4ZReRwt1EX5zCwVl4p2+mXRmDlA7M0uw8/K/jp6RU66H7EO7Xbda0/6AkjGy3L9D1BLBwi8KP69SwIAAHEHAABQSwMEFAAICAgASSpUVwAAAAAAAAAAAAAAABwAAAB3b3JkL19yZWxzL2RvY3VtZW50LnhtbC5yZWxzrZJNasMwEIVP0DuI2dey0x9KiZxNCGRb3AMo8viHWiMhTUp9+4qUJA4E04WX74l5882M1psfO4hvDLF3pKDIchBIxtU9tQo+q93jG4jImmo9OEIFI0bYlA/rDxw0p5rY9T6KFEJRQcfs36WMpkOrY+Y8UnppXLCakwyt9Np86RblKs9fZZhmQHmTKfa1grCvCxDV6PE/2a5peoNbZ44Wie+0kJxqMQXq0CIrOMk/s8hSGMj7DKslGSIyp+XGK8bZmUN4WhKhccSVPgyTVVysOYjnJSHoaA8Y0txXiIs1B/Gy6DF4HHB6ipM+t5c3n7z8BVBLBwiQAKvr8QAAACwDAABQSwMEFAAICAgASSpUVwAAAAAAAAAAAAAAAAsAAABfcmVscy8ucmVsc43POw7CMAwG4BNwh8g7TcuAEGrSBSF1ReUAUeKmEc1DSXj09mRgAMTAaPv3Z7ntHnYmN4zJeMegqWog6KRXxmkG5+G43gFJWTglZu+QwYIJOr5qTziLXHbSZEIiBXGJwZRz2FOa5IRWpMoHdGUy+mhFLmXUNAh5ERrppq63NL4bwD9M0isGsVcNkGEJ+I/tx9FIPHh5tejyjxNfiSKLqDEzuPuoqHq1q8IC5S39eJE/AVBLBwgtaM8isQAAACoBAABQSwMEFAAICAgASSpUVwAAAAAAAAAAAAAAABUAAAB3b3JkL3RoZW1lL3RoZW1lMS54bWztWUtv2zYcvw/YdyB0b2XZVuoEdYrYsdutTRskboceaYmW2FCiQNJJfBva44ABw7phhxXYbYdhW4EW2KX7NNk6bB3Qr7C/HpYpm86jTbcOrQ82Sf3+7wdJ+fKVw4ihfSIk5XHbci7WLERij/s0DtrW7UH/QstCUuHYx4zHpG1NiLSurH/4wWW8pkISEQT0sVzDbStUKlmzbenBMpYXeUJieDbiIsIKpiKwfYEPgG/E7HqttmJHmMYWinEEbG+NRtQjaJCytNanzHsMvmIl0wWPiV0vk6hTZFh/z0l/5ER2mUD7mLUtkOPzgwE5VBZiWCp40LZq2cey1y/bJRFTS2g1un72KegKAn+vntGJYFgSOv3m6qXNkn8957+I6/V63Z5T8ssA2PPAUmcB2+y3nM6UpwbKh4u8uzW31qziNf6NBfxqp9NxVyv4xgzfXMC3aivNjXoF35zh3UX9Oxvd7koF787wKwv4/qXVlWYVn4FCRuO9BXQazzIyJWTE2TUjvAXw1jQBZihby66cPlbLci3C97joAyALLlY0RmqSkBH2ANfFjA4FTQXgNYK1J/mSJxeWUllIeoImqm19nGCoiBnk5bMfXz57go7uPz26/8vRgwdH9382UF3DcaBTvfj+i78ffYr+evLdi4dfmfFSx//+02e//fqlGah04POvH//x9PHzbz7/84eHBviGwEMdPqARkegmOUA7PALDDALIUJyNYhBiqlNsxIHEMU5pDOieCivomxPMsAHXIVUP3hHQAkzAq+N7FYV3QzFW1AC8HkYV4BbnrMOF0abrqSzdC+M4MAsXYx23g/G+SXZ3Lr69cQK5TE0suyGpqLnNIOQ4IDFRKH3G9wgxkN2ltOLXLeoJLvlIobsUdTA1umRAh8pMdI1GEJeJSUGId8U3W3dQhzMT+02yX0VCVWBmYklYxY1X8VjhyKgxjpiOvIFVaFJydyK8isOlgkgHhHHU84mUJppbYlJR9zq0DnPYt9gkqiKFonsm5A3MuY7c5HvdEEeJUWcahzr2I7kHKYrRNldGJXi1QtI5xAHHS8N9hxJ1ttq+TYPQnCDpk7EwlQTh1XqcsBEmcdHhK706ovFxjTuCvo3Pu3FDq3z+7aP/UcveACeYama+US/DzbfnLhc+ffu78yYex9sECuJ9c37fnN/F5rysns+/Jc+6sK0ftDM20dJT94gytqsmjNyQWf+WYJ7fh8VskhGVh/wkhGEhroILBM7GSHD1CVXhbogTEONkEgJZsA4kSriEq4W1lHd2P6Vgc7bmTi+VgMZqi/v5ckO/bJZsslkgdUGNlMFphTUuvZ4wJweeUprjmqW5x0qzNW9C3SCcvkpwVuq5aEgUzIif+j1nMA3LGwyRU9NiFGKfGJY1+5zGG/GmeyYlzsfJtQUn24vVxOLqDB20rVW37lrIw0nbGsFpCYZRAvxk2mkwC+K25ancwJNrcc7iVXNWOTV3mcEVEYmQahPLMKfKHk1fpcQz/etuM/XD+RhgaCan06LRcv5DLez50JLRiHhqycpsWjzjY0XEbugfoCEbix0Mejfz7PKphE5fn04E5HazSLxq4Ra1Mf/KpqgZzJIQF9ne0mKfw7NxqUM209Szl+j+iqY0ztEU9901Jc1cOJ82/OzSBLu4wCjN0bbFhQo5dKEkpF5fwL6fyQK9EJRFqhJi6QvoVFeyP+tbOY+8yQWh2qEBEhQ6nQoFIduqsPMEZk5d3x6njIo+U6ork/x3SPYJG6TVu5Lab6Fw2k0KR2S4+aDZpuoaBv23+ODSfKWNZyaoeZbNr6k1fW0rWH09FU6zAWvi6maL6+7SnWd+q03gloHSL2jcVHhsdjwd8B2IPir3eQSJeKFVlF+5OASdW5pxKat/6xTUWhLv8zw7as5uLHH28eJe3dmuwdfu8a62F0vU1u4h2Wzhjyg+vAeyN+F6M2b5ikxglg+2RWbwkPuTYshk3hJyR0xbOot3yAhR/3Aa1jmPFv/0lJv5Ti4gtb0kbJxMWOBnm0hJXD+ZuKSY3vFK4uwWZ2LAZpJzfB7lskWWnmLx67jsFMqbXWbM3tO67BSBegWXqcPjXVZ4yjYlHjlUAnenf11B/tqzlF3/B1BLBwghWqKELAYAANsdAABQSwMEFAAICAgASSpUVwAAAAAAAAAAAAAAABMAAABbQ29udGVudF9UeXBlc10ueG1stZNNbsIwEIVP0DtE3lbE0EVVVQQW/Vm2XdADDM4ErPpPnoHC7TsJkAUCqZWajWX7zbz3eSRP5zvvii1msjFUalKOVYHBxNqGVaU+F6+jB1UQQ6jBxYCV2iOp+exmutgnpEKaA1VqzZwetSazRg9UxoRBlCZmDyzHvNIJzBesUN+Nx/faxMAYeMSth5pNn7GBjePi6XDfWlcKUnLWAAuXFjNVvOxEPGC2Z/2Lvm2oz2BGR5Ayo+tqaG0T3Z4HiEptwrtMJtsa/xQRm8YarKPZeGkpv2OuU44GiWSo3pWEzLI7pn5A5jfwYqvbSn1Sy+Mjh0HgvcNrAJ02aHwjXgtYOrxM0MuDQoSNX2KW/WWIXh4Uolc82HAZpC/5Rw6Wj3pl+J10WCenSN399tkPUEsHCDOvD7csAQAALQQAAFBLAQIUABQACAgIAEkqVFdJE0N/aAEAAD0FAAASAAAAAAAAAAAAAAAAAAAAAAB3b3JkL251bWJlcmluZy54bWxQSwECFAAUAAgICABJKlRXBtJgVRYCAAAPBwAAEQAAAAAAAAAAAAAAAACoAQAAd29yZC9zZXR0aW5ncy54bWxQSwECFAAUAAgICABJKlRXrYdtAHkBAABaBQAAEgAAAAAAAAAAAAAAAAD9AwAAd29yZC9mb250VGFibGUueG1sUEsBAhQAFAAICAgASSpUV9p4rlMrAwAAzxIAAA8AAAAAAAAAAAAAAAAAtgUAAHdvcmQvc3R5bGVzLnhtbFBLAQIUABQACAgIAEkqVFe8KP69SwIAAHEHAAARAAAAAAAAAAAAAAAAAB4JAAB3b3JkL2RvY3VtZW50LnhtbFBLAQIUABQACAgIAEkqVFeQAKvr8QAAACwDAAAcAAAAAAAAAAAAAAAAAKgLAAB3b3JkL19yZWxzL2RvY3VtZW50LnhtbC5yZWxzUEsBAhQAFAAICAgASSpUVy1ozyKxAAAAKgEAAAsAAAAAAAAAAAAAAAAA4wwAAF9yZWxzLy5yZWxzUEsBAhQAFAAICAgASSpUVyFaooQsBgAA2x0AABUAAAAAAAAAAAAAAAAAzQ0AAHdvcmQvdGhlbWUvdGhlbWUxLnhtbFBLAQIUABQACAgIAEkqVFczrw+3LAEAAC0EAAATAAAAAAAAAAAAAAAAADwUAABbQ29udGVudF9UeXBlc10ueG1sUEsFBgAAAAAJAAkAQgIAAKkVAAAAAA==" +) + +pptx_file = base64.b64decode( + "UEsDBBQAAAAIAHFwW1fGr8RntAEAALoMAAATAAAAW0NvbnRlbnRfVHlwZXNdLnhtbM2XyU7DMBCG7zxFlEsOqHHZFzXlwHJiqQQ8gEmmrcGxLc+00Ldnki6q2FKWCl8S2TPz/58nUTTpnLyUOhqDR2VNlmyl7SQCk9tCmUGW3N9dtA6TCEmaQmprIEsmgMlJd6NzN3GAERcbzOIhkTsWAvMhlBJT68BwpG99KYmXfiCczJ/kAMR2u70vcmsIDLWo0oi7nTPoy5Gm6PyFt2uQ+EGZODqd5lVWWSyd0yqXxGExNsUbk5bt91UOhc1HJZekzgPyvU4vNS8VS/lbIOKDYSw+NH10MHjjqsqKug58XONB4/dIZ61IubLOwaFyuMkJnzhUkc8NZnU3/Ai9KiDqSU/XsuQswc3oeetQcH76tUpzQ6ECKqBoOZYETwoWzF9659bD983nPaqqV3R0jkT11GvbXx/33fszE16FYF63DoiFdimVaYJBzZuXcmJHhMuLrb8mW9L+MVM7RKgQO7UdINNOgEy7ATLtBci0HyDTQYBMhwEyHf0305VEnqtwebGeb+ZUeyWmGc16OJoISD5ouKWJhj8fQpakGyl4EIfp9fdtqGWaHMcKntcyei2E5wSi/vXovgJQSwMEFAAAAAgAcXBbV/ENN+wAAQAA4QIAAAsAAABfcmVscy8ucmVsc62Sz04DIRCH7z4F2QunLttqjDFlezEmvRlTH2CE6S51gQlMTfv2ool/arZNDz3C/PjmG2C+2PlBvGPKLgYtp3UjBQYTrQudli+rx8mdFJkhWBhiQC33mOWivZo/4wBczuTeURYFErKuema6VyqbHj3kOhKGUlnH5IHLMnWKwLxBh2rWNLcq/WVU7QFTLK2u0tJOK7HaE57Djuu1M/gQzdZj4JEW/xKFDKlD1hURK0qYy+ZXui7kSo0Lzc4XOj6s8shggUFxv/WvAdzwa2OjeUqxhH5q9YawOyZ0fVkhExNOqPTHxA7ziNZn4tQN3VzyyXDHGCza00pA9G2kDn5m+wFQSwMEFAAAAAgAcXBbVwV3nA87AgAAtAwAABQAAABwcHQvcHJlc2VudGF0aW9uLnhtbO2X327aMBTG7/cUlm+4mGj+EJI0wlRaJ6RJnYQKfQDXOUBUx4lsh0GffnZwSGCa1AfIne1zvu+c/GxZzuLpVHJ0BKmKSpBJ8OBPEAhW5YXYk8nbdjVNJ0hpKnLKKwFkcgY1eVp+W9RZLUGB0FQbJTIuQmWU4IPWdeZ5ih2gpOqhqkGY2K6SJdVmKvdeLukf415yL/T92CtpIbDTy6/oq92uYPCzYk1pyl9MJPC2D3UoatW51V9xG37FbUuKHmHTvCvQq0poRXCAEW109VyVVqTWBdONGRDs46XhoXj+myoN8lf+ovTdCipygsMgSqJ0FkcpRjKzKyYSYG+58P4jvx1fTObxQJ306mHu5hOxE8GPQRT5vo8ROxMcp/O0nehzDQQrJgFEdJpZhzoTlQblZNdMK+s82qwcdrThegsnvdFnDssFtWvrtXSj17VEnJqzg0FM3zZtd8MUfuRBbXJKKl8sOET5XhDMMTI5W/q++SQ4miehrS41b1OAvogf8qPdALvNwk1N6GBKmbO0bgTTNj7oQhmnILU+HyBNicB62riqeJGvCs7biT0Z8MwlOlJTTZ8C1/JNVlu15bajzLD7Xoop1zaTZkDvAkAvAabuAkz1OF4tDu/Kw6EJezQdhJFP2POZ9Xwux3Lkc4Hi+EQ9n2CWBPEIqKPiAM0HgNIwTUdAHRUHKO4BhWEa+yOgjooDlAwAJdFsvKOvVBygtAdk6YyX9JWKA/Q4ABTPk/GSvlJpX7L/PjG923+N5V9QSwMEFAAAAAgAcXBbV1KcUMkcAQAAcQQAAB8AAABwcHQvX3JlbHMvcHJlc2VudGF0aW9uLnhtbC5yZWxzrZTBTsMwDIbvPEWUS0407YCB0NJdENIOSIiNB8hat41IkygOg709EUxbW20Vhx792/79yYqzWH63muzAo7JGsCxJGQFT2FKZWrD3zfP1AyMYpCmltgYE2wOyZX61eAMtQ+zBRjkk0cSgoE0I7pFzLBpoJSbWgYmZyvpWhhj6mjtZfMga+CxN59x3PWje8ySrUlC/KjNKNnsH//G2VaUKeLLFZwsmnBnBUasSXiQG8NFW+hqCoB2xV5El0Z/y81izKbGcVyYOXEMIce14QhskhoVZslXmEuHNtISAr966HttBGlvT7ZQQOwVfA4ijNAZxNyVEiL1wAvgN/8TR9zKflEFuNazDXkNnFR1xDOR+8nsaXNJBPW6D936K/AdQSwMEFAAAAAgAcXBbV6YtojXuBgAA0i4AACEAAABwcHQvc2xpZGVNYXN0ZXJzL3NsaWRlTWFzdGVyMS54bWztWu9u4zYS/35PIeg+5MPBK4ki9cdYp4iddW+BdBs06QPQEm3rQks6ik6TPRTYd+gb9C3a+3aPsk9yQ0q0ZMeJE6zTru8MLCxqOBrOzG9mSE727Td3C27dMlFlRT448d64JxbLkyLN8tng5MfrcS86sSpJ85TyImeDk3tWnXxz+pe3Zb/i6Xe0kkxYICKv+nRgz6Us+45TJXO2oNWbomQ5zE0LsaASXsXMSQX9CUQvuINcN3AWNMvt5nvxnO+L6TRL2HmRLBcsl7UQwTiVoH41z8rKSCufI60UrAIx+us1lU7BvuSKp+o5mdW/P7CplaV3A9tzXQ84aF9LZiMurFvKB/Zk5tnO6VunYW5G6uOqvBaMqVF++60or8pLoVf4cHspQCaItK2cLtjAVgL0RMPm1B/pgbPx+cwMaf9uKhbqCe6xQEPXtu7Vr6No7E5aSU1MWmoy/34LbzJ/t4XbMQs4nUWVVbVyD81BxpzrTHJmXXKasHnBU4gVb2Wh0b0qL4rkprLyAmxTrqhNXXHU9qtnObfkfQlipRJrG5eoSaerSLXdK5iEgLA2F4U48KN1/0QIxYHb2O152HfddetpvxSV/JYVC0sNBrZgidSBQG8vKlmzGhatUtUoJO+GRXqvOCfwBCdBwsH380J8tC3+Pq8GduxhDGtL/aI1tS3RnZmszUg+KrhGieYJyBnYiRRalxzi+2wpi2nWaFQvqaZ4Ja/kPWfa7FL9aLIAhTiFfLdZ3vvxyraqhRxxRvNVWMjTEc+SG0sWFkszaTV5r2GA6gAi1UJSL6dFsjy9pIL+sCG5cZH2jfGJYwLp8XDyV+GksOpGE9pHNCkH2U1qf0lQeRA9yHWfiCpMEIkD/+uPqhcHUqmQvuWriPnCwFLe03FVrQWWY1ZbW9J74ZJXLCny1OLslvFniEcvFH89z8TzpfsvlD4ulkLOny0ev1R8Nt0qfd8pjU1Kn1O5vkH4+0jpVIJ1HyEXKJ82qY2+JLUDn8C/jdRGnu+vUtsPiIfI15/Za/uF001mPb7lnoodymcQFVwrm7KpAl2501P+0JAUPEvHGedbjkHyrj4dySyXNSUk7Va6Yq7fWjmOWUkPG0XqcUdBHd1Tnuog+hcZjs7O3Yj03kVnQS+KMOkNz/G73miIR6Mzl8TjEf7ZNjEBkSazBRtns6Vg3y9rKJ6TFJ6DQsfz24SYqpPhvlOCmJQYF4Uqgt2kwPtIiikgrmH855IKWKFJDP/FieF7CD+dGVFM/qczwxy2vr7c2G9MBiYmr0AXZn1YLiYbkUn2EZlwlQTR24ITvzg4A0L8/++y/bWG5qpsj7zxODg/i3uuG4170RBHvRhBAR8GBE7LEQ6j4XhVtisVeTlEx3Or9edPv/3186ff91Ctne7NHcIH0G9G1lJkYMhwGAdoFA17Qw+Pe/g8Dntn44D0xsTHeDSMzkb+u59VM8HD/UQw3Wd4n5oOhYcf9CgWWSKKqpjKN0mxaJodTln8xERZZLrf4blN00RDhJAbx2FIvLjJE9DNPLW2TtvHSLj4jpbWZObBzi498O8djNIbGE1mSNGQoiFFgxFNEpZL4GgGhoIMZcXjG4pvKNhQsKEQQyGGEhgK1Jg5z/IbcIZ62Na04H+vCWZU1xioEhf0vljK92mDRIdS9x08HOLID3AMudNXFPE+9R58vcZL3A4v2sHrdXj9Hbyow4t38PodXrKDF3d4gx28pMMb7uANOrzRDt6wwxvv4I26WLg7mNeAM1vHQ+DlnS4tlR6rLsQT+7QF9emaTq4+tid6qKu6qDJ6kQ/Fje6/qR5i3rzC1BxKRJbPLpd5ItV8vbMlQ9XX06PLpCmTqxK5mp0sPxR5fTnuVGEo7yD3hon8BRXZ2ay3YKFSVBfHKWzDA/tvi3/0uGz2OLoxwWjT2Ks2JpKqkb21eq97tdT72QMXL6i4gB0Uo1gZluVQpsFVPUMwd4jX9j9IdLdhMC5gI2uNPhMZ5bUzJsvRnAorgZ+B/fnTr/YmVPUB4jWgyh+DKn8MqvxpqPQQtXCE4H3ShQNFJCSHBMcvD+BA0QHAgVo4/BYO00fu4IGi4MDTA71aJdsjHn6LB+7g0fRoDxiPLfnhHgAeuMWDtHggl4T4kPH4z78PEw7SwhF04CAeDg4Zjq3l6hDwCFo8wg4ecehFRzz+BDzCFo9o87B7xOOPxyNq8Yg7eERRcODb+YHiEZuLYudqWPYLOWdidVGELy5r1BrrHvbdWpb1W+WrINhtiR7ClWL7Dc844eif7Vcu3Ug/+ufxK5Afeq9UIg/NQdvvJF6EoujooCduCXqPPTro8WN7iP1jjX7qHA3qHov0UwfbgITHIr1+0uweLp3u34Cczn9GP/0vUEsDBBQAAAAIAHFwW1e+a0K9DQEAAMYHAAAsAAAAcHB0L3NsaWRlTWFzdGVycy9fcmVscy9zbGlkZU1hc3RlcjEueG1sLnJlbHPF1d1qwyAUB/D7PYV449VikrZpWmp6MwaFXY3uAURPPliionYsbz/ZGDSwyQYFbwQ/zv/8ODceju/TiN7AukErRoosJwiU0HJQHSMv58f7miDnuZJ81AoYmcGRY3N3eIaR+1Dj+sE4FEKUY7j33uwpdaKHibtMG1DhptV24j5sbUcNF6+8A1rmeUXtdQZuFpnoJBm2J1lgdJ4N/CVbt+0g4EGLywTK/9CCunGQ8MRnffEhltsOPMNZdn2+eFRkoQWmv8jypLQ8aks7tvjcylvafKiFherz5GuNOm7K+O+IyphslVK2isnWKWXrmGyTUraJyaqUsiom26aUbWOyOqWsjsl2KWW7bxldfL/NB1BLAwQUAAAACABxcFtXAP3sDSoEAAAFEQAAIQAAAHBwdC9zbGlkZUxheW91dHMvc2xpZGVMYXlvdXQxLnhtbM1YXY7bNhB+7ykI9cFPCvVDSbQRb2DJq6LAZncRbw7AlWhbCCWqJO3YKQLkWu1xcpJSlGR5f9o6gAP4xaKomeE3882QHL99tysZ2FIhC15NR+4bZwRolfG8qFbT0ceH1MYjIBWpcsJ4RaejPZWjd1e/vK0nkuU3ZM83CmgTlZyQqbVWqp5AKLM1LYl8w2ta6W9LLkqi9KtYwVyQz9p0yaDnOCEsSVFZnb44RZ8vl0VG5zzblLRSrRFBGVEavlwXteyt1adYqwWV2ozRfgpJ7Ws6tVShGLWAERNbPeFaV9rzbMFyUJFSTzw0EmDBipyaT7J+EJQ2o2r7m6gX9b0wGrfbewGKvLHQaVqw+9CJwVbJDOAz9VU/JJPdUpTNUwcC7KaWY4F98wubObpTIGsns2E2W9+9Iputr1+Rhv0C8GjRxqsW3Et3POtJINyDVz1eWd/w7JMEFdf+NO637h0kWp+bZ73uop4pYaxZfSSa7/B4ffl6MEIcYKf10nN9B3nB07hEUeQhp/PXRZHjtBLHXstuCbWLeb5vtB/107BCJkyqhdozal7q5sfAEDoYjOiCsWhlf1xYQJYqYZRUh2irq4QV2SegOKB5ocB7IhUVwOSXLi9tsgGhDBRjklb5PRHkwzPLLdjaIO0Rwp6ff2fJ71labB7bNb1zECU3jy1RepHdoHI6Ya4fuWHHmI9xqAvwKWOhpgsfGIsCL3Re5OlJjJnxlrlaFpRE3Ji0L6pcV78ZEraqTOZZxsDmVm92xkBOlx+6AHFd5WnBmHlpNhWaMAG2hOmNYucaRVVUqp2JAucA9SDcvg124GAfHvB1UL0BKgqiJjIXiNcb8PoD3rGL0GXi9Qe8aMB7SMPLA4wGwMERYOxhfJmAgwFwOAD2PBw6lwk4HABHR4Aj5F9ozUUDYDwAbtBeaNHhAfD4CHAYRBdadOO6Hx+dHmc47mV/+v78Ex/1J/6cKAruGcnomrNcg/DPcfLnSnv9RV+xCVv2p7/z38c//IFb1VLfrxsv/gziZDZ3cGBf41loY4wCO56jazuJUZLMnGCcJuhrf1vPtauqKGlarDaC3m2UdSpbLvQi6PoDIxrA+TkJek5Szpt0OGYFnYOVpS4cQ8sfGyL0Cj0z/3Mx+xFmzhuR8HAvbRoocLspH5/FJTjLPZXl2vSrofF+QtImbpqG89nY1ndX3T/HCNtjT6dvHAaeN8YownF6SFrZeF5pdKfm6vdvf/36/dvfZ8hVeNyu6hv3jVTdCGxEoR2J43HoJTi2YxelNpqPI3uWhoGdBj5CSYxniX/9tWl7XTTJBDVt9O9534C76EULXhaZ4JIv1ZuMl10vD2v+mYqaF6add52uATfbt++G2ImCAPsdTRpb/zRoYduMmxRh4j2p77YmSUqz4SZmqi6qVZcjgwg8+v/i6h9QSwMEFAAAAAgAcXBbV4Bl4Yi3AAAANgEAACwAAABwcHQvc2xpZGVMYXlvdXRzL19yZWxzL3NsaWRlTGF5b3V0MS54bWwucmVsc43PvQ7CIBAH8N2nICxMQutgjCntYkwcXIw+wAWuLbEFwqHRt5fRJg6O9/X755ruNU/siYlc8FrUshIMvQnW+UGL2/W43glGGbyFKXjU4o0kunbVXHCCXG5odJFYQTxpPuYc90qRGXEGkiGiL5M+pBlyKdOgIpg7DKg2VbVV6dvg7cJkJ6t5Otmas+s74j926Htn8BDMY0aff0QompzFM1DGVFhIA2bNpfzuL5ZqWSK4ahu1eLf9AFBLAwQUAAAACABxcFtXN8Y1+I0DAADNCwAAIgAAAHBwdC9zbGlkZUxheW91dHMvc2xpZGVMYXlvdXQxMC54bWy1VsGO2zYQvfcrCPXgk5aSLHtlI97AkldFgU12UTu9MxK9JkKJLEk7dooA+a32c/IlHVKS197sAnbrXkSKGr5582Yozpu324qjDVWaiXrSC6+CHqJ1IUpWP056Hxa5n/SQNqQuCRc1nfR2VPfe3vz0Ro41L+/ITqwNAohaj8nEWxkjxxjrYkUroq+EpDV8WwpVEQOv6hGXinwG6IrjKAiGuCKs9tr96pT9YrlkBZ2JYl3R2jQginJigL5eMak7NHkKmlRUA4zbfUzJ7CSdeKCLWWw95OzUBlZC7wZCL+a8RDWpYGHBDKcI9EG/gzErCEcLujXOTMuFotTO6s0vSs7lg3K7328eFGKlRWtRPNx+aM1ws8lN8LPtj92UjLdLVdkRVEHbiRd4aGef2K4BCVQ0i8XTarG6f8G2WN2+YI07B/jAqY2qIfdjOJF3JEq4j6rjq+WdKD5pVAuIx4bfhLe3aGK2o1y1KTAWyutksB/xoXPdiWW2qSh31slHGN0iGXNt5mbHqXuR9uFoKODLCRS4R2v/w9xDujIZp6TeC2JuMs6KT8gIREtm0DuiDVXIkYHjAJBWHeM0cpC0Lh+IIr89Q25UlI50xxB3Er4uZL8T8qim0AMnBV0JXgKV6BLiWqk8JBSDQ9BUuwf+t0+bz1Hc/kUAhRJL2ntFf2kF2vC90P8xH1YVlw59lA/ceTtyGZ7pck4LAeea0w3lJ8BHZ8IvVkydjt4/Ez0Xa2VWJ8PH58Kz5Yvolz4JcXcSZsTQowPQv8QBKKHg9Re4KghfdqUfXO5vs4Rrwkbx5yDNprMgGfi3yXToJ0k88NNZfOtnaZxl02AwyrP4a3frlBCqYRXN2eNa0fu1vUxOy0qIo2sc9p8yAgQun5NBl5NcCHsKD7MSXyIrS6OatPyxJgo8dJn5N3+lVzJzWUWGnSJzzkqK3q+rj890GVxCF+i4APpFaaL/oWizMM+Hs+nID4IE+sA0TvxRBOWbDgdRNEri6yTN90WrbeQ1sDu1Vr9/++vn79/+vkCt4sNOC26EO23aGVorBoGk6WgYZUnqp2Gc+/FsdO1P8+HAzwf9OM7SZJr1b7/aji2Mx4Wirh38tewayTD+oZWsWKGEFktzVYiq7UmxFJ+pkoK5tjQM2kZyQ+zVMAqDUXQ9GsZtmoBbNzq2uOkpXYlw9Y7I+40rksrdc5lbktA3tzXyZIIP+vCbfwBQSwMEFAAAAAgAcXBbV4Bl4Yi3AAAANgEAAC0AAABwcHQvc2xpZGVMYXlvdXRzL19yZWxzL3NsaWRlTGF5b3V0MTAueG1sLnJlbHONz70OwiAQB/DdpyAsTELrYIwp7WJMHFyMPsAFri2xBcKh0beX0SYOjvf1++ea7jVP7ImJXPBa1LISDL0J1vlBi9v1uN4JRhm8hSl41OKNJLp21VxwglxuaHSRWEE8aT7mHPdKkRlxBpIhoi+TPqQZcinToCKYOwyoNlW1Venb4O3CZCereTrZmrPrO+I/duh7Z/AQzGNGn39EKJqcxTNQxlRYSANmzaX87i+WalkiuGobtXi3/QBQSwMEFAAAAAgAcXBbV0uJUFfAAwAArQwAACIAAABwcHQvc2xpZGVMYXlvdXRzL3NsaWRlTGF5b3V0MTEueG1stVfRkps2FH3vV2jog59YAQaMPfFmDF46ndlkd2on7wrIayYCUUl27HQyk99qPydf0isBXtvrpPbUeTEgro7OPecKXb96vSkZWlMhC16Ne+6N00O0ynheVE/j3rt5akc9JBWpcsJ4Rce9LZW917e/vKpHkuX3ZMtXCgFEJUdkbC2VqkcYy2xJSyJveE0reLfgoiQKHsUTzgX5BNAlw57jhLgkRWW188U58/liUWR0yrNVSSvVgAjKiAL6clnUskOrz0GrBZUAY2YfUlLbmo4t0EXNC8XopMrnGwuZeLGGN651CxJkM5ajipQw8B5Ci4wwZOIRCIbmdKNMmKznglJ9V61/E/WsfhRm9tv1o0BFrtFaFAu3L9ow3EwyN/ho+lN3S0abhSj1FdRBm7HlWGirf7EeAxIoawaz59Fs+XAiNlvenYjG3QJ4b1GdVUPuZTqedVoUd5deR1zW9zz7KFHFITGtQ5PnLqJJXl/rZeuJ0lAW4qIA5xqLrE4dHYr3OcnTAoWhN/SdJnVv4If96FArzwkG5r3WIIgCN/CCYyVku4TaxDzf6tkf4AoKaEZji5L3LTMyYlLN1JZR81DrH0NKQDAjsM8sWtnvZhaSpUoYJdXOD3WbsCL7iBRHNC8UekOkogIZCWBXAqSmpAwxA0mr/JEI8scRckO9Nrw7vrhz8Ps+9l/6qBV6ZCSjS85yoOJdw1It3JGjsP7mefL5zvrBwPuBsaHjDqOfaWytlV+znYP/02jN2/gsD4zG3WoHS7oXLjmjGYfPFKNrys6A9y6Eny8LcT56/0L0lK+EWp4N718KXyxOol97i/ndFpsSRQ92Vv8aOyuHnSQ/w1FI2KLbU86PNxU+VfvfqfYFHH86i7+COJlMnSiw76JJaEeRH9jx1L+zk9hPkokTDNPE/9KdqjmkqoqSpsXTStCHlT4kz3PFxd4Au/1nR4DA9T0JOk9SzvUu3HfFv4YrCyUaW/5cEQErdM78x+fuEmeuq0jYKTJjRU7R21X54UiX4Bq6QEcJ0Cel8X5C0SZumobTydB2nAj63NiP7KEH5RuHgecNI38QxemuaKXOvAJ259bqt69///rt6z9XqFW830HCiXAvVXuHVqKAROJ4GHpJFNux66e2Px0O7EkaBnYa9H0/iaNJ0r/7ojtR1x9lgpp29/e8a5Rd/0WrXBaZ4JIv1E3Gy7bnxjX/REXNC9N2u07bKK+J/niHrud5/cGwswm4dVfDFje9sikRJt6Q+mFtiqQ051xihmr4X9DWyHMI3vufcfsvUEsDBBQAAAAIAHFwW1eAZeGItwAAADYBAAAtAAAAcHB0L3NsaWRlTGF5b3V0cy9fcmVscy9zbGlkZUxheW91dDExLnhtbC5yZWxzjc+9DsIgEAfw3acgLExC62CMKe1iTBxcjD7ABa4tsQXCodG3l9EmDo739fvnmu41T+yJiVzwWtSyEgy9Cdb5QYvb9bjeCUYZvIUpeNTijSS6dtVccIJcbmh0kVhBPGk+5hz3SpEZcQaSIaIvkz6kGXIp06AimDsMqDZVtVXp2+DtwmQnq3k62Zqz6zviP3boe2fwEMxjRp9/RCianMUzUMZUWEgDZs2l/O4vlmpZIrhqG7V4t/0AUEsDBBQAAAAIAHFwW1eTCm11IQYAAOcdAAAUAAAAcHB0L3RoZW1lL3RoZW1lMS54bWztWU1v2zYYvg/YfyB0b2XZVuoEdYrYsdutTRskboceaYmW2FCiQNJJfBva44ABw7phlwG77TBsK9ACu3S/JluHrQP6F/bqwzJl04nTZluB1gebpJ73+4OkfPXaccTQIRGS8rhtOZdrFiKxx30aB23r7qB/qWUhqXDsY8Zj0rYmRFrXNj/84CreUCGJCAL6WG7gthUqlWzYtvRgGcvLPCExPBtxEWEFUxHYvsBHwDdidr1WW7MjTGMLxTgCtndGI+oRNEhZWptT5j0GX7GS6YLHxL6XSdQpMqx/4KQ/ciK7TKBDzNoWyPH50YAcKwsxLBU8aFu17GPZm1ftkoipJbQaXT/7FHQFgX9Qz+hEMCwJnX5z/cp2yb+e81/E9Xq9bs8p+WUA7HlgqbOAbfZbTmfKUwPlw0Xe3Zpba1bxGv/GAn690+m46xV8Y4ZvLuBbtbXmVr2Cb87w7qL+na1ud62Cd2f4tQV8/8r6WrOKz0Aho/HBAjqNZxmZEjLi7IYR3gJ4a5oAM5StZVdOH6tluRbhB1z0AZAFFysaIzVJyAh7gOtiRoeCpgLwBsHak3zJkwtLqSwkPUET1bY+TjBUxAzy6vmPr54/Ra+ePzl5+Ozk4S8njx6dPPzZQHgDx4FO+PL7L/7+9lP019PvXj7+yoyXOv73nz777dcvzUClA198/eSPZ09efPP5nz88NsC3BB7q8AGNiES3yRHa4xHYZhBAhuJ8FIMQU51iKw4kjnFKY0D3VFhB355ghg24Dql68J6ALmACXh8/qCi8H4qxogbgzTCqAHc4Zx0ujDbdTGXpXhjHgVm4GOu4PYwPTbK7c/HtjRNIZ2pi2Q1JRc1dBiHHAYmJQukzfkCIgew+pRW/7lBPcMlHCt2nqIOp0SUDOlRmohs0grhMTApCvCu+2bmHOpyZ2G+TwyoSqgIzE0vCKm68jscKR0aNccR05C2sQpOS+xPhVRwuFUQ6IIyjnk+kNNHcEZOKujehe5jDvsMmURUpFD0wIW9hznXkNj/ohjhKjDrTONSxH8kDSFGMdrkyKsGrFZLOIQ44Xhrue5So89X2XRqE5gRJn4yFqSQIr9bjhI0wiYsmX2nXEY3f9+6Ve/eWoMbime/Yy3DzfbrLhU/f/ja9jcfxLoHKeN+l33fpd7FLL6vni+/Ns3Zs64fujE209AQ+ooztqwkjt2TWyCWY5/dhMZtkROWBPwlhWIir4AKBszESXH1CVbgf4gTEOJmEQBasA4kSLuGaYS3lnd1VKdicrbnTCyagsdrhfr7c0C+eJZtsFkhdUCNlsKqwxpU3E+bkwBWlOa5ZmnuqNFvzJtQNwulrBWetnouGRMGM+KnfcwbTsPyLIXJqWoxC7BPDsmaf0/hXvOmeS4mLcXJtwcn2YjWxuDpDR21r3a27FvJw0rZGcGyCYZQAP5l2GsyCuG15Kjfw7Fqcs3jdnFVOzV1mcEVEIqTaxjLMqbJH09cq8Uz/uttM/XAxBhiayWpaNFrO/6iFPR9aMhoRTy1ZmU2LZ3ysiNgP/SM0ZGOxh0HvZp5dPpXQ6evTiYDcbhaJVy3cojbmX98UNYNZEuIi21ta7HN4Ni51yGaaevYS3V/TlMYFmuK+u6akmQvn04af3Z5gFxcYpTnatrhQIYculITU6wvY9zNZoBeCskhVQix9GZ3qSg5nfSvnkTe5IFR7NECCQqdToSBkVxV2nsHMqevb45RR0WdKdWWS/w7JIWGDtHrXUvstFE67SeGIDDcfNNtUXcOg/xYfXJqvtfHMBDXPs/k1taavbQXrb6bCKhuwJq5utrjuLt155rfaBG4ZKP2Cxk2Fx2bH0wHfg+ijcp9HkIiXWkX5lYtD0LmlGZey+q9OQa0l8b7Is6Pm7MYSZ58u7vWd7Rp87Z7uanuxRG3tHpLNFv6U4sMHIHsbrjdjlq/IBGb5YFdkBg+5PymGTOYtIXfEtKWzeI+MEPWPp2Gd82jxr0+5me/lAlLbS8LG2YQFfraJlMT1s4lLiukdryTObnEmBmwmOcfnUS5bZOkpFr+Jy1ZQ3uwyY/au6rIVAvUaLlPHp7us8JRtSjxyrATuTv/Ggvy1Zym7+Q9QSwMEFAAAAAgAcXBbVwFX6IttAwAAlgsAACEAAABwcHQvc2xpZGVMYXlvdXRzL3NsaWRlTGF5b3V0Mi54bWy1VtFymzoQfb9foaEPfiICDA721OkYHO7cmbTJ1OkHKCCCWoF0Jdm12+lMf6v9nH5JJQGOnaYzzpS+ICFWZ3fPHqR9+WpbU7DBQhLWzEf+mTcCuMlZQZr7+ejdbebGIyAVagpEWYPnox2Wo1cX/7zkM0mLK7RjawU0RCNnaO5USvEZhDKvcI3kGeO40d9KJmqk9Ku4h4VAHzV0TWHgeRNYI9I43X5xyn5WliTHS5ava9yoFkRgipQOX1aEyx6Nn4LGBZYaxu4+DkntOJ477O69A6yR2OhX37nQeecrWoAG1XrhliiKgSYHpKxRGskaSH4rMDazZvOv4Ct+I+y+N5sbAUhhcLr9Duw+dGaw3WQn8NH2+36KZttS1GbUZIDt3PEcsDNPaNbwVoG8XcwfVvPq+gnbvLp8whr2DuCBU5NVG9yv6QTOER3+Pqs+XsmvWP5BgobpfEz6bXp7izZnM/KqY14ZKKenwXyEh85lT5baJqzYGSd3erSLaEalWqkdxfaFm4cNQ+h4KdK6dnDjvls5QNYqpRg1e0LURUpJ/gEoBnBBFHiNpMIC2GD0X6AhDTvKcmQhcVPcIIHePkJuWeQ26D5C2FP4eyLHPZGdmsANRTmuGC10EMGf0UqK7YPJAIxyk/KG7qn7Q4aNbC3B8ohh2Hs7cuk/0+UK50z/oxRvMD0BPngm/G1FxOno42eiZ2wtVHUyfPhceFI+iT60tsNe20uk8JGwx0OcF4XS2X3SZz6ipdOJ3RtO7aU+8k0Wn6MkXSy9OHIv48XEjeMwcpNleOmmSZimCy+aZmn4pb8+Cp2qIjXOyP1a4Ou1uR5Oq4oPg3Pojx8qogMYviZRX5OMMfMXHlYlHKIqpRJtWf5fI6E99JUZ8BwalpFJz8iKkgKDN+v67hEv0RC86NZJQz9JTfAXRJv6WTZZLqau58W6oUvC2J0GWr7JJAqCaRyex0m2F600mTc6ulO1+uPrtxc/vn4fQKvwsHfSN8KVVN0MrAXRiSTJdBKkceImfpi54XJ67i6ySeRm0TgM0yRepOPLL6YH88NZLrDt6/4r+o7QD3/pCWuSCyZZqc5yVnfNJeTsIxacEdtf+l7XEW6QuRomfjj2wyCKuzLp2PrRRgvb/tBKhIrXiF9vrEhqe8+ldonrBrjTyIMJPGioL34CUEsDBBQAAAAIAHFwW1eAZeGItwAAADYBAAAsAAAAcHB0L3NsaWRlTGF5b3V0cy9fcmVscy9zbGlkZUxheW91dDIueG1sLnJlbHONz70OwiAQB/DdpyAsTELrYIwp7WJMHFyMPsAFri2xBcKh0beX0SYOjvf1++ea7jVP7ImJXPBa1LISDL0J1vlBi9v1uN4JRhm8hSl41OKNJLp21VxwglxuaHSRWEE8aT7mHPdKkRlxBpIhoi+TPqQZcinToCKYOwyoNlW1Venb4O3CZCereTrZmrPrO+I/duh7Z/AQzGNGn39EKJqcxTNQxlRYSANmzaX87i+WalkiuGobtXi3/QBQSwMEFAAAAAgAcXBbV4tg7VpjBAAAWBEAACEAAABwcHQvc2xpZGVMYXlvdXRzL3NsaWRlTGF5b3V0My54bWzNWNtu2zYYvt9TCOqFrxRSEnUK6hSWHG0D0iSo0wdgJNoWSh1G0q69oUBfa3ucPslISrIcN2ndzgtyI1LUf/j+A/nz1+s3m5Iaa8J4UVfjkX0GRwapsjovqsV49P4utcKRwQWuckzrioxHW8JHby5+ed2cc5pf4W29EoYUUfFzPDaXQjTnAPBsSUrMz+qGVPLbvGYlFvKVLUDO8EcpuqTAgdAHJS4qs+Nnx/DX83mRkWmdrUpSiVYIIxQLCZ8vi4b30ppjpDWMcClGcz+EJLYNGZucZL8RnJuGJmRruWSbF9L2bEZzo8KlXJiRTLEbipAw/ZU3d4wQNavWv7Jm1twyzXS9vmVGkSshHbMJug8dGWiZ9AQcsC/6KT7fzFmpRukNYzM2oWls1ROoNbIRRtYuZsNqtrx5hDZbXj5CDXoFYE+psqoF97U5Tm/OXSEoMeydVT1e3lzV2QduVLW0R5nfmrejaG1WY7PsXC+UKLN3g/oI9pXzxz0ROI5ru9pEhKAfwQOnBEHgINgZa7u+AwPv0GTeqRCbuM63ivtejtJUXGXLWmapaGVSLmZiS4mer6ndKBK6qMYmNdVaTubv5BL/U2KBSue9DnyGpQcwpZ3ajrOd70ls1EObyKQQiuV2NEllvZ+ZBi9FQgmudmEUFwktsg+GqA2SF8J4i7kgzNAulJtXSlTShdahRZIqv8UMvzuQ3CJqtBd660Ef+KfD7+7Cr9x8S3FGljWVm8FwTpEJyvumVLQZyH8qIZwI+oGcfyMhPAjtMPjhhLh/OiFKzK707iqqXJ40aqoFrK7laQoO0sRRaaK9VNMiTwtK9Ys6v0hCmbHGVGbfxtY0oqhEuxJ4EPYbd0fcvg1yQK/pYdbpqTMgRV7gwCPh2uEzwnUGuO4AN7IROhqu/4xw3QEuGuDabqBRHIcXPSNeNOD19vCGThi+SLzegNcf8DpO6MMXidcf8AZ7eAPkHr/dnhNvMOANB7wK7PH77TnxhgPeaA+v7wUvc79FT9Z8hV4S7Ir7f7wDqEKnrwD8wR3gZ+o86uv8FAvyoM67p6jzuTB1HJaYzvt6D79d8MFjZflBLQY7v87ljV1Z8ZcXJ5MpDD3rMpz4Vhgiz4qn6NJKYpQkE+hFaYI+9R1ALk0VRUnSYrFi5GYlzGPDYQMnALY7eF0COP3dy+tjkta1ivd+VNApojIXrA3LHyvMpIY+Mt+5iv1IZE7rEb/3yEzuPmJcr8r7A794p/CL7H6l6Edd4/wPSZvYaepPJ5EFYSh78hiFVuTI9I19z3GiEAVhnO6SlivLK4nu2Fz98vnvV18+/3OCXAX73a88e6646GbGihXSkDiOfCcJYyu2UWqhaRRYk9T3rNRzEUricJK4l59UF22j84wR3Zr/nvdNvY2+auvLImM1r+fiLKvL7v8AaOqPhDV1oX8R2LBr6vV5HfnQR6Hb9X0aWj9qsKDt7nWGUPYWNzdrnSOlPlATvdQU1aJLkYEE7P0SufgXUEsDBBQAAAAIAHFwW1eAZeGItwAAADYBAAAsAAAAcHB0L3NsaWRlTGF5b3V0cy9fcmVscy9zbGlkZUxheW91dDMueG1sLnJlbHONz70OwiAQB/DdpyAsTELrYIwp7WJMHFyMPsAFri2xBcKh0beX0SYOjvf1++ea7jVP7ImJXPBa1LISDL0J1vlBi9v1uN4JRhm8hSl41OKNJLp21VxwglxuaHSRWEE8aT7mHPdKkRlxBpIhoi+TPqQZcinToCKYOwyoNlW1Venb4O3CZCereTrZmrPrO+I/duh7Z/AQzGNGn39EKJqcxTNQxlRYSANmzaX87i+WalkiuGobtXi3/QBQSwMEFAAAAAgAcXBbV0/KghwIBAAAaBIAACEAAABwcHQvc2xpZGVMYXlvdXRzL3NsaWRlTGF5b3V0NC54bWztWN1y2jgUvt+n0LgXXDmyjWwMU9LBJt7ZmbTJFPoAii2Ct7LllQSB7nSmr7X7OH2SlYSNIaEFtlzmBgv503f+j+3z9t2qoGBJuMhZOey4V04HkDJlWV4+DjufpokddoCQuMwwZSUZdtZEdN5d//a2Ggia3eI1W0igKEoxwENrLmU1gFCkc1JgccUqUqp7M8YLLNVf/ggzjp8UdUGh5zgBLHBeWvV5fsp5NpvlKRmzdFGQUm5IOKFYKvXFPK9Ew1adwlZxIhSNOb2vklxXZGjJJ3b38KcFDI4v1Y5rXSvT0wnNQIkLtTF9YiBmpVQ05paoppwQvSqXv/NqUt1zc+LD8p6DPNMM9UkL1jdqGNwcMgv47Phjs8SD1YwX+qo8AVZDy7HAWv9CvUdWEqSbzbTdTed3B7Dp/OYAGjYC4I5QbdVGuZfmeI0501xSAtytVY2+orpl6WcBSqbs0eZvzNsiNjbrazVv3K6prMYN+ibcFS4aZ8lVxLK1FvKgrmYTD6iQE7mmxPyp9I9Rgyt9KVZJbZHS/jSxgChkTAkutw6R1zHN089AMkCyXIL3WEjCgVFGlYCi1N6RxkeGkpTZPeb44zPmjRcro3SjIWxc+GNHdhtH1tkE7ilOyZzRTCnh/ZpbxRdVDZjOLCVp1YJ/4NsDWYb8nioOkz5u4Dh6vZdwyOmGgVMnEvI9vx90n6eTqEX8NGpmvaRurUZGZtq9Wn8vdJoM3QGopXcAi3axXovtHsA6u9hui0Uvse6eDqjF+sewfosNjmGDFts7hu212PAYNmyx/WPYDQDuB8ZUU6XTfUm3ZfOL1aUzyBSX2Ksu2EjbE+meKXJCUlZmgJIloSfQe2fST+c5P529eyZ7whZczk+mR+fS57OD7Jfua+hnfa170b7mnd/XAhS+NrbXxvba2F4b27mNzW8a2xhLstfV0CVegjNpvXhvcy73UjxTXzDair/9KB6NndC3b8JRYIch8u1ojG7sOEJxPHL8fhKjr80HUaZMlXlBkvxxwcndQn/znBYVF3o96HbbiCgFLh+ToIlJwpiuwt2o+JeIykzyTVj+WmCuJDSROfJKfU5kLuuRXuORCc0zAj4siodnfgku4RdBM0V90DVHnsr/K2ljN0mC8ahvO06Y2GGEQrvvqfSNAt/z+iHqhVGyTVqhLS+Vdqfm6vdv/7z5/u3fC+Qq3B0IqCfCrZD1Cix4rgyJon7gxWFkRy5KbDTu9+xREvh24ncRiqNwFHdvvurBgosGKSdmUvFH1sw4XPRiylHkKWeCzeRVyop6XAIr9kR4xXIzMXGdesaxxPrR0As9D6E+6tVhUro1V6Mt3Iw7TIpQ/h5Xd0uTJIV5zsVmq8rLxzpHWgjcGRFd/wdQSwMEFAAAAAgAcXBbV4Bl4Yi3AAAANgEAACwAAABwcHQvc2xpZGVMYXlvdXRzL19yZWxzL3NsaWRlTGF5b3V0NC54bWwucmVsc43PvQ7CIBAH8N2nICxMQutgjCntYkwcXIw+wAWuLbEFwqHRt5fRJg6O9/X755ruNU/siYlc8FrUshIMvQnW+UGL2/W43glGGbyFKXjU4o0kunbVXHCCXG5odJFYQTxpPuYc90qRGXEGkiGiL5M+pBlyKdOgIpg7DKg2VbVV6dvg7cJkJ6t5Otmas+s74j926Htn8BDMY0aff0QompzFM1DGVFhIA2bNpfzuL5ZqWSK4ahu1eLf9AFBLAwQUAAAACABxcFtX6aTEj+MEAAA2HAAAIQAAAHBwdC9zbGlkZUxheW91dHMvc2xpZGVMYXlvdXQ1LnhtbO1Z3ZKiOBS+36eg2AuvGAgECNbYUy3dbm1VT3fX6DxAGmLLDhA2ibbO1lTNa+0+zjzJJgiitto4erFV6w3EcPLl/H4cyfsP8yzVZoTxhOa9DnhndTSSRzRO8ude5/NoYKCOxgXOY5zSnPQ6C8I7H65+eV90eRrf4QWdCk1C5LyLe/pEiKJrmjyakAzzd7QguXw2pizDQv5kz2bM8IuEzlLTtizPzHCS69V61mY9HY+TiNzQaJqRXCxBGEmxkOrzSVLwGq1og1YwwiVMuXpTJbEoSE8XL3Q0H73Qh6c/dK0UZjM5DfQraX80TGMtx5mcCGlWYJZwmpdPeDFihKhRPvuNFcPikZUL7mePTEtiBVAt1M3qQSVmLheVA3Nr+XM9xN35mGXqLr2hzXu6pWsLdTXVHJkLLVpORs1sNHnYIRtNbndIm/UG5tqmyqqlcq/NsWtzRolIiQZWVtX68uKORl+4llNpjzJ/ad5KYmmzuheT2vUKSq/doB6a65vz2lli3qfxQm3yJO/lJO6mXAzFIiXleJaCSo2YjD8tXbs2bW6KF+pSSjNpXYplGegkNz4PdY1nIkwJzlfuE1dhmkRfNEE1EidC+4i5IEwrVZdFIxEVuij3KCFJHj9ihj9tIS81KkoTa3vM2uH73e6s3K5i/pjiiExoGksN7HNEQPlTlxvNG/E9gdiRktD1ZTWVuQZcxwXA2cxOaEELILTMOs8JfM/eTj1e7bAdYQ3n0YRKtnjS9wVbyzC7K5M6yWNZ4GpYAkzvJYmZTS5o/KtMX6g0farN3EgZObQbwNqqVqjWa1S7QXUa1ABA2BYVoNeoToMKG1Tg+MBrDeu9hoUNrLsGi2yEToF1G1ivgbVt5FmnwHoNrL8G60OndcR2wfoNLGpgFWb7kO2ARQ1ssAbruf5JIQv2MpraRAqsqOtEhlNlXBIc32C4n2ExqK9eormQVm8QmXMakSk/TXA6rmjMPoXGbOBD5LsHaMwJXCCLoy2Pvf2mathpHy/t4px9bLOLSfZxyK5c20cMB2W3qv2g7FYJH5TdqsuDslvFdlD2v1FB21uCI7cckojmsZaSGUlbwNtHwo8mCWuP7hyJPqBTJiat4eGx8Ml4J/q5uzN3b3cGz9edqQT+c4qZTKmK45zjOc6DrmW7B3s14Evmu/Rql17t0qv9n3s171Cv5p7eq21SGTyJyvb1aw2VXfq1S7926dcu/dqS2/ya226wIBvE5p2jX4uFvv13FFinft80V+4dp3FpxV9uP7y+sZBr3KJrz0AIukb/Bt4aYR+G4bXlBoMQfqu/b8fSVJFkZJA8Txl5mAq9bVSAafsmcJqISAXOHxNUx2RAqarC9aj454jKWLBdTTR444PnMZE5r0eC2iPDNImJdj/Nnrb8gs7hF57GEnqna974iPJTSRuCwcC7uQ4My0IDA/UhMgJbpm/fc207QNBH/cEqabmyPJfatc3VH9///vXH93/OkKvm+tmOfCPccVGNtClLpCH9fuDZIeobfQAHBrwJfON64LnGwHUgDPvoOnRuv6kzIgC7ESPlwdPvcX1kBeCrQ6ssiRjldCzeRTSrTr/Mgr4QVtCkPAADVnVkNcOSXYPAAi7yHa+KklStvpfKmstzqzJDUvYRFw+zMkey8jUXllNFkj9XKdKImGsHflf/AlBLAwQUAAAACABxcFtXgGXhiLcAAAA2AQAALAAAAHBwdC9zbGlkZUxheW91dHMvX3JlbHMvc2xpZGVMYXlvdXQ1LnhtbC5yZWxzjc+9DsIgEAfw3acgLExC62CMKe1iTBxcjD7ABa4tsQXCodG3l9EmDo739fvnmu41T+yJiVzwWtSyEgy9Cdb5QYvb9bjeCUYZvIUpeNTijSS6dtVccIJcbmh0kVhBPGk+5hz3SpEZcQaSIaIvkz6kGXIp06AimDsMqDZVtVXp2+DtwmQnq3k62Zqz6zviP3boe2fwEMxjRp9/RCianMUzUMZUWEgDZs2l/O4vlmpZIrhqG7V4t/0AUEsDBBQAAAAIAHFwW1cttCb1EgMAALgIAAAhAAAAcHB0L3NsaWRlTGF5b3V0cy9zbGlkZUxheW91dDYueG1stVbdbtowFL7fU1jZBVepkxAgoMFEQjNNakc12gfwEgPRHNuzDYNNlfZa2+P0SXbsEMq6TuoFu4md4/Pzne8c5+TN213N0JYqXQk+7oQXQQdRXoiy4qtx5+4295MO0obwkjDB6bizp7rzdvLqjRxpVl6RvdgYBC64HpGxtzZGjjDWxZrWRF8ISTmcLYWqiYFXtcKlIl/Bdc1wFAR9XJOKewd79RJ7sVxWBZ2JYlNTbhonijJiAL5eV1K33uRLvElFNbhx1n9CMntJx56pDKNzzvYecqpqC8LQm0D2xYKViJMaBLdWCzk1e6LlraLU7vj2nZILeaOcwYftjUJVaR0cDD18ODio4cbIbfAT81W7JaPdUtV2BS7QbuwFHtrbJ7YyujOoaITFo7RYz5/RLdaXz2jjNgA+CWqzasD9nU7k/cFDeMyqxavllSg+a8QF5GPTb9I7ajQ521WuT4n3WhrsIT4NrluyzC4V5d4G+QSrE5IR02Zh9oy6F2kfDoYCvIxAW3uU+3cLD+naZIwSfiTETDJWFZ+REYiWlUHXRBuqkAMDlwBcWnaM48i5pLy8IYp8fOK5YVE60C1C3FL4byK7LZEzYii6YaSga8FKQBCdg9PSQMrf4FoQtvQgINQ9DM7H8RLug83iey/NprMg6fmXybTvJ0nc89NZfOlnaZxl06A3zLP4vr1hJaRqqprm1Wqj6HxjvJeWKsTRAIfdx4oAgPPXJG5rkgthe+G0Kt1zVGVpVFOWLxuiIEJbmfB8lTkvI72WkQWrSoo+bOpPT3iJz8ELTBdw/Sw10X9o2izM8/5sOvSDIIGZl8aJP4ygfdN+L4qGSTxI0vzYtNpmzgHdS3v14cfP1w8/fp2hV/HpfIGP/ZU2hx3aqAoSSdNhP8qS1E/DOPfj2XDgT/N+z8973TjO0mSadS/v7ZwK41GhqBt978t2aIbxX2OzrgoltFiai0LUh/mLpfhKlRSVG8FhcBiaW8LG3iAaBNFgcGxggNauDixuZqfrEKauiZxvXY/U7mObOZGEX4RDizyq4JNfjslvUEsDBBQAAAAIAHFwW1eAZeGItwAAADYBAAAsAAAAcHB0L3NsaWRlTGF5b3V0cy9fcmVscy9zbGlkZUxheW91dDYueG1sLnJlbHONz70OwiAQB/DdpyAsTELrYIwp7WJMHFyMPsAFri2xBcKh0beX0SYOjvf1++ea7jVP7ImJXPBa1LISDL0J1vlBi9v1uN4JRhm8hSl41OKNJLp21VxwglxuaHSRWEE8aT7mHPdKkRlxBpIhoi+TPqQZcinToCKYOwyoNlW1Venb4O3CZCereTrZmrPrO+I/duh7Z/AQzGNGn39EKJqcxTNQxlRYSANmzaX87i+WalkiuGobtXi3/QBQSwMEFAAAAAgAcXBbV+sXn3fmAgAAZwcAACEAAABwcHQvc2xpZGVMYXlvdXRzL3NsaWRlTGF5b3V0Ny54bWy1VdFumzAUfd9XIPaQJ2ogJIWoSRVImSZ1bbS0H+CCSVDB9mwnSzZV6m9tn9Mv2bWBNGs7qQ/ZC7Yv917fc87V9dn5tq6sDRGyZHTc807cnkVoxvKSLse925vUCXuWVJjmuGKUjHs7Invnkw9nfCSr/BLv2FpZkILKER7bK6X4CCGZrUiN5QnjhMK/gokaKziKJcoF/g6p6wr5rjtENS6p3caL98SzoigzMmPZuiZUNUkEqbCC8uWq5LLLxt+TjQsiIY2J/rskteNkbN9VmN7blnETGzB49gSQZ4sqtyiuwRAbD22U/EYQond080nwBZ8L43u1mQurzHVsG2Oj9kfrhpogs0EvwpfdFo+2haj1ChRY27Ht2tZOf5G2ka2yssaYPVuz1fUbvtnq4g1v1F2ADi7VqJriXsPxOzgzrIg1r3BGVqzKibC8PcCudMkvWXYvLcoAmmaiQbr3aODrla9a6nNlW/IHiIirwoYLoVzPtTuGtDM6rEt2PKptzPKdvvQOVmPEo0qqhdpVxBy4/hSgoEbxcxAn05kbDpyLcDp0wjAYOPEsuHCSOEiSqTuI0iR46PohB6iqrElaLteCXK+VrXMJYATaYDm2CXVuF1B3rZKKYLqnXE085J8ir69pVoZsKMAIR/M5FvjrixSNINyA7BChTo1/a9LvNEkZU6DEoSr+MVQplGhk+bbGAm7olPGOp8xxGQk6RhZVmRPral3fveClfwxeYBZC6jep8f9D0yZemg5n08hx3RAmdByETuRD+8bDge9HYXAaxum+aaVGTqG69/bq0+Ovj0+Pv4/Qq+hwLMKMupSq3VlrUQKQOI6GfhLGTuwFqRPMolNnmg4HTjroB0ESh9Okf/Ggx6sXjDJBzKD+nHcj3gteDfm6zASTrFAnGavb1wJx9p0IzkrzYHhuO+I3uNLyeH4URaEXtjJBbd1qqkXNuDctUokvmF9vTJPAZSByYkwcXrS2R55d0MELOfkDUEsDBBQAAAAIAHFwW1eAZeGItwAAADYBAAAsAAAAcHB0L3NsaWRlTGF5b3V0cy9fcmVscy9zbGlkZUxheW91dDcueG1sLnJlbHONz70OwiAQB/DdpyAsTELrYIwp7WJMHFyMPsAFri2xBcKh0beX0SYOjvf1++ea7jVP7ImJXPBa1LISDL0J1vlBi9v1uN4JRhm8hSl41OKNJLp21VxwglxuaHSRWEE8aT7mHPdKkRlxBpIhoi+TPqQZcinToCKYOwyoNlW1Venb4O3CZCereTrZmrPrO+I/duh7Z/AQzGNGn39EKJqcxTNQxlRYSANmzaX87i+WalkiuGobtXi3/QBQSwMEFAAAAAgAcXBbV83KitWyBAAAwhIAACEAAABwcHQvc2xpZGVMYXlvdXRzL3NsaWRlTGF5b3V0OC54bWzNWN1yozYYve9TMPTCVwQE4i+zzo4hodOZbJJZZx9AAdmmC4hKstduZ2f2tdrH2SepJMB2HMfGiS96Y2T56Ejfdz4dYX34uCwLbYEpy0k1HIALa6DhKiVZXk2Hgy+PiREMNMZRlaGCVHg4WGE2+Hj1y4f6khXZLVqROdcERcUu0VCfcV5fmiZLZ7hE7ILUuBK/TQgtERdf6dTMKPomqMvCtC3LM0uUV3o7nvYZTyaTPMXXJJ2XuOINCcUF4mL5bJbXrGOr+7DVFDNBo0Y/XxJf1Xiok6c/Hpe6pmB0ITqAfiUiT8dFplWoFB0xqbhg0L7lfKbFqJZMCsPqR4qxbFWL32g9rh+oGnq3eKBankmqlkI32x9amNkMUg1zZ/i0a6LL5YSW8ikyoi2HuqVrK/lpyj685FradKab3nR2vwebzm72oM1uAnNrUhlVs7iX4dhdOI85L7AG1lF162X1LUm/Mq0iIh4ZfhPeGtHELJ/1rE0/l1R6lwb5o7k9OdufCej6QkgVou07lruTE8eyAgc4TawAeHaL2I6YtTPwZUSylRz9JJ4iUlSlMyIK9anhLBgf81WBVXtRgFpCimk11Atd9mV48ll0sb/EUiy5pqcu8DW+aW/x1PJDxUXF0AKJfajjyvgy1jVW8rjAqFprx6/iIk+/apxoOMu59gkxjqmm8iZ2rWCU7FzNoShxlT0gij7vMDcrqlXsXcxmp/brmjv6zi54KFCKZ6TIxCLs91VAni03kP7iO67vSkFfU98FAPhuW+lu4DpAlEJP9V+TfEdpR1bfjsaqab/E2sE21t5gnT1YuI11Nli4B2ttY+EG6x7DuhusdwzrbbD+May/wQbHsMEGGx7Dhq/uIbkZBWC9Wd65p2QFqS3Fnu0ps5vt2ZTgxCnHOCVVphV4gYse9PaJ9I+znPZnd05kT8icitOvLz08lT6f7GU/t5vB9Qkmpd62Mucch5n0EF0V8AwVE70xOPs9pxuAjgusQ8cb9EJgee82OK1E9Fa9H+RVJnxeNtWo+Z14JzR39ieAB/yvpeqi6MVnH/DIli8EEPbmsw74aMsHHB94fQnDA17b8QV2ELyJb8ePWz7bDjzrTXw7nt3x+dDpLUh4wNdbPknWW5DwgPd3fJ7rv02P/8f5cJoTuZ0TXSOOnzkRPIcTZfyFDwHrsBGZR+3CXOd1Iv4cySj+dqN4dG0FrnETjDwjCKBrRNfwxogjGMcjyw2TGH7v/mplIlSelzjJp3OK7+dc7ysHMG3fBM4m62IB5z8dvE6ThBCp97Yq7jlUmXDayPLnHFExQ6fMkXfgU5Q5b0b8LiPjIs+wdjcvn3by4p0jL6zIBPXe1Bw5Pd9UtDFIEu96FBriHE2MIIKBEdqifCPPte0wgH4QJeuiZTLySqyub63+/PHPrz9//HuGWjW3rxiE99wy3ra0Oc1FIFEUenYcREYEYGLA69A3RonnGonrQBhHwSh2br7LqwoAL1OK1R3I71l3ewLgi/uTMk8pYWTCL1JSthcxZk2+YVqTXN3FAKu9PVkg+Q4cQMu3PdfrvEWsrXuq1ZrNTYoqkYJ+QvX9QhVJqRw1Vl11Xk3bGtlAzK3Lp6v/AFBLAwQUAAAACABxcFtXgGXhiLcAAAA2AQAALAAAAHBwdC9zbGlkZUxheW91dHMvX3JlbHMvc2xpZGVMYXlvdXQ4LnhtbC5yZWxzjc+9DsIgEAfw3acgLExC62CMKe1iTBxcjD7ABa4tsQXCodG3l9EmDo739fvnmu41T+yJiVzwWtSyEgy9Cdb5QYvb9bjeCUYZvIUpeNTijSS6dtVccIJcbmh0kVhBPGk+5hz3SpEZcQaSIaIvkz6kGXIp06AimDsMqDZVtVXp2+DtwmQnq3k62Zqz6zviP3boe2fwEMxjRp9/RCianMUzUMZUWEgDZs2l/O4vlmpZIrhqG7V4t/0AUEsDBBQAAAAIAHFwW1da07SSeQQAADESAAAhAAAAcHB0L3NsaWRlTGF5b3V0cy9zbGlkZUxheW91dDkueG1svVjdcps4FL7fp2Doha+I+BEgMnU6Bsc7O5MmmSZ9AAVkmyl/K8mOvTud6WvtPk6fpJIAQ5ykYV1mb4wsjj6d75yjT0LvP+zyTNsSytKymE6sM3OikSIuk7RYTSef7xcGmmiM4yLBWVmQ6WRP2OTDxW/vq3OWJVd4X264JiAKdo6n+prz6hwAFq9JjtlZWZFCvFuWNMdc/KUrkFD8KKDzDNim6YEcp4XejKdDxpfLZRqTeRlvclLwGoSSDHPhPlunFWvRqiFoFSVMwKjRT13i+4pM9SqN73e6pszoVnRY+oVgHt9liVbgXHTcpjHfUKI9pnytRbiSSMqGVfeUENkqtr/T6q66pWro9faWamkioRoIHTQvGjNQD1INcDR81Tbx+W5Jc/kUEdF2U93Utb38BbKP7LgW151x1xuvb16wjdeXL1iDdgLQm1Syqp17Tsdu6dynPCOadWDV+suqqzL+wrSiFHwk/ZrewaLmLJ/Vugk/l1B6Gwb5EvQnZy9HwvID20ZIcYRIpNQ8iooLkQfNhq3reb6DjimzZgq+C8tkLwc/iKegiot4XYpKfaghM8bv+D4jqr3NrEqaZKtiqme67EvI8pPoYn+JAJlyyoeW+cG+bvdwKvmjiFExNMNiIeqkMD7f6RrLeZQRXBySxy+iLI2/aLzUSJJy7SNmnFBNBU4sW4Eo0bmaQ0GSIrnFFH86Qq49qhT3ljNo0/160h39aBncZjgm6zJLhBP2GCUgVqAuptp11qcVgmfZvu/+pA6gZcliGVoIr2Y/x/RKLaW0SIS0yKYatbkW8gmOasKxDzMeqkE17Q4Kur60GoRnoz6e3eE5HV5gQTgYD/bxnA4PdniW41veYECzDwg7QLcHiETSTgN0O0CvAxRF4JmnAXodoN8D9KEzPCdPAP0OEHWAEm14Up4Aog4w6AF6rn9iUoJXNWlc7YCHDUOux75wOGMIh1ymuqK3xtmy0RD7lzTEdcRWUe8Vr4gIMsU/+//VEAuOqyGWPa6GWObIGhKMLCHByAoSjCwgwcj6EYwsH8Ew9ZDowuBwdPnFE45cf+qAw56ccE5RIrdVojnmT48wcAwlSvgzHbLMnwsReFMuwCGuS/EtIln87YbRbG4i17hEM89ACLpGOIeXRhTCKJqZbrCI4Nf2yyYRVHmak0W6Eue2mw3Xh6bDArYPLKeLunBg/N3Ba3OyKEuZ735W3DGysuS0TsufG0zFDG1m3jhm/pfMjBsRv43IXZYmRLve5A9HcfHGiIv4qhfQL4bmjd3zpKKNrMXCm88CwzTRwkAhREZgi/INPde2AwR9FC4ORcsk80J4N7RWv3/75933b/+OUKug/0UvtOeK8aalbWgqiIRh4NkRCo3QggsDzgPfmC0811i4DoRRiGaRc/lV3gxY8DymRF05/JG0lxUWfHZdkacxLVm55GdxmTf3HqAqHwmtylRdfVhmc1mxxUJWHYQC2/ECJ2jSJHxrn8pbUF9cqBLJ6Edc3WxVkeRKUSPVVaXFqqmRzgT07noufgBQSwMEFAAAAAgAcXBbV4Bl4Yi3AAAANgEAACwAAABwcHQvc2xpZGVMYXlvdXRzL19yZWxzL3NsaWRlTGF5b3V0OS54bWwucmVsc43PvQ7CIBAH8N2nICxMQutgjCntYkwcXIw+wAWuLbEFwqHRt5fRJg6O9/X755ruNU/siYlc8FrUshIMvQnW+UGL2/W43glGGbyFKXjU4o0kunbVXHCCXG5odJFYQTxpPuYc90qRGXEGkiGiL5M+pBlyKdOgIpg7DKg2VbVV6dvg7cJkJ6t5Otmas+s74j926Htn8BDMY0aff0QompzFM1DGVFhIA2bNpfzuL5ZqWSK4ahu1eLf9AFBLAwQUAAAACABxcFtX6ORJ0TkDAACzJAAAKAAAAHBwdC9wcmludGVyU2V0dGluZ3MvcHJpbnRlclNldHRpbmdzMS5iaW7tWc9u2jAYz3orb7BbljsxUFbYlFIxKBoSbaMSKu1UuYnL3IY4cswYe6S93+5zAgETMIQd1iTqoVVw7C+/P/YX+8uJoijv+N/v94piXP6cuOoPRANMvAutqlc0FXk2cbA3vtBGVq/c1C5bJeND97ZjfTOvVN/FAVPN0ZdBv6NqZQDavu8iALpWVzUH/aGl8hgAXN1oqvadMf8zALPZTIdhL90mk7BjAExKfETZfMCDlfkA3WGOxh+ziL4Bh7c62Gat0qnxguYtHmIZzKfYY7oJx6hH6ATyy+uvhOJfxGPQvUOBAcL+fNhy+O7xDNsviOk2RZARGo85NQLGb4+F7s/kcdHXAMt7B0JihiZtSuF8HRSGP8OrNShJjMO0wpEctNtq1AwQXcijLREFDDLUc+FYjMHvozGirYoB4ssIIFjJBmLYq7bDkG8pRhww4zYWx4cdpEQFq5sKZsWKoQ1dLlNxbEgQWi2EagbXwT3PctguWD7aQSrb2SgGXLilICGWtSURTB+txXN8yN/7D9h7Ig+xZru8MK9Ns2uGfTvEQTdwgtZSrfQ5xrW0th3pm2ic6NxBFgKiAWIM0Q0Qx3slNUtwS7BL9HCF1KLQC9zo9TaMsETQcy1+CkoCvNFQzYYZFiZjmHP1JRwEPB4suxmQe2+CbTvP04AhJ2y8QzbLoxf/RjARdY/K+28tdgVndfFNFDd/bJxvNAsmZXYe8Ald8ImQZLg9EyLLytXmLk8lzY3G7hnwqZ7lGcCl6PO9Cpcn19n4OGJ5SNFr/CMPFjJHp2L4lqQlOhUqS6ej+Jamdd93ipuqZeQEoFk4vUie1Db794uy/lYlpVLRa5W0tRM299FWBKloVtKs/XUKKVYZ1LRI5UDjHVgSaQzUANE3kVbpRFGUP6UCfLHpEns6Qd6ScVjP9QlxFyrkujKXhpiwWMOh2I5qE8B3njZX7SsWTsP/Q55IOJaAk+gQH+e9eL2Xkqhehj7hbGOed4jr8mcWzYskr3Aoo1MEsuZBD9OAhSm7UA5sscrHghjAAnqRJCUqWKvWG/Xm2Xm9kVlPovMp9Apmyhar5ElLulrSmCeepF7Pyf+/8xVFPrj5/QtQSwMEFAAAAAgAcXBbV1ycRxREAQAAiQIAABEAAABwcHQvcHJlc1Byb3BzLnhtbLWSy07DMBBF90j8Q+S9aztJ81KTKmmChMSCBXyAlTitpfgh230gxL8TQgoUNt2wm9Ho3jl3NKv1SQzegRnLlcwBWWDgMdmqjsttDp6f7mACPOuo7OigJMvBC7NgXdzerHSmDbNMOupG6aPxRiNpM5qDnXM6Q8i2OyaoXSjN5DjrlRHUja3Zos7Q47hADMjHOEKCcglmvblGr/qet6xW7V6MAJ8mhg0Tid1xbc9u+hq3nzkukIoxJDu5B+vmytsbnoPXJo42TRqWMMLBBoYk9GGVNhWMahLEGBNc+vHbh5qEWcdtS013L+iWNR13NXX0DEfCP3iCt0ZZ1btFq8ScE2l1ZEYrPkUleL7XgQ45wAAVKzTBXTLWASlx5JcwTpMShoGfwrKqa1hVZbKMIh8vCf5iZD3dD25irDX/Lzz0fU30+3uKd1BLAwQUAAAACABxcFtXZzMmjZsBAACCAwAAEQAAAHBwdC92aWV3UHJvcHMueG1sjVPBTuMwEL2vxD9YvoOTCEKJmnJBcEFapIa9G2eaGjm25XFLy9fvJG5pCz1wmzfjeX5vxp7eb3rD1hBQO1vz/CrjDKxyrbZdzV+bx8sJZxilbaVxFmq+BeT3s4s/U1+tNXy8BEYEFitZ82WMvhIC1RJ6iVfOg6XawoVeRoKhE22QH0TcG1FkWSl6qS3f9Yff9LvFQit4cGrVg42JJICRkcTjUnvcs/nfsPkASDRj96kkIzH+I3c1R9M2y1X/ZqU2Q4bPyLgdSEb4EgZMPNEFaJ9hERl+0hhvyiLj4rjWOD+W7q7LciyJnzxodAsHqOamTYihlb5xT0G3NacNJfj37R1URLpuVKV2Z9cyzJU0sM/jAGZTWeGGDSsurjkjmjwbZVB6eyYtvvp85YLutGWbml/mN3nB2XaIKEjn1EFxtyIDzxi/Yka9NGLahgufnHlHaou83M0mHUnJyWR/74FEHM8gaTqdkHURsIFNPBra0Ti/GSdn54yfps8bz0bT2XfH4qyEjtY091LRS2eKmm/pMRCB2u7DxJK+z+w/UEsDBBQAAAAIAHFwW1fY/Y2PpQAAALYAAAATAAAAcHB0L3RhYmxlU3R5bGVzLnhtbA3MSQ6CMBhA4b2Jd2j+fS1DUSQUwiArd+oBKpQh6UBooxLj3WX58pIvzT9KopdY7GQ0A//gARK6Nd2kBwaPe4NjQNZx3XFptGCwCgt5tt+lPHFPeXOrFFfr0KZom3AGo3NzQohtR6G4PZhZ6O31ZlHcbbkMpFv4e9OVJIHnHYnikwbUiZ7BN6qCIKK0wKfL5YhpSANcejTGcVTW1bmp/SosfkCyP1BLAwQUAAAACABxcFtXN2scvHQBAACZAwAAFQAAAHBwdC9zbGlkZXMvc2xpZGUxLnhtbK2T30rDMBTG732KkJteuWwTRMragYre+GfQ+QBZe7YW0yTkZHV9e5O0tUMnDPQmJ8k53++cD5LF8lAL0oDBSskkmk2mEQGZq6KSuyR6Wz9c3kQELZcFF0pCErWA0TK9WOgYRUGcWGLME1paq2PGMC+h5jhRGqTLbZWpuXVHs2OF4R8OWgs2n06vWc0rSXu9PkevDSBIy60b9BTEnANR222Vw73K97VjdRADIkCxrDTS1DnLM1H4iHptAPxONo9GZ3plQvqlWRlSFQmdUSJ5DQmlrE/0ZawThQ37Jt8dlaDuCn+i5wN6XVkBZPbVoSvlTvqk8nckUjm2H6Vr9VXR9fdRl8S22qFyawKNDlP5PDvuj8Ng9nCritb32bgYLnks0Ga2FRAO2i9hEpsG6oL5rV9NWHVgDyA2mP3d8tVgOdtvbHA9/w/XuN90rl2Twyj5o3t2yh0b3wwbn1EuzDPXr00w4B6mBXMXrrT7D/38YwkLPyv9BFBLAwQUAAAACABxcFtXNuhQzbcAAAA2AQAAIAAAAHBwdC9zbGlkZXMvX3JlbHMvc2xpZGUxLnhtbC5yZWxzjc+9CsIwEAfw3acIWTKZtA4i0tRFBMFJ9AGO5NoG2yTkoti3N6MFB8f7+v255vCeRvbCRC54LWpZCYbeBOt8r8X9dlrvBKMM3sIYPGoxI4lDu2quOEIuNzS4SKwgnjQfco57pcgMOAHJENGXSRfSBLmUqVcRzAN6VJuq2qr0bfB2YbKz1Tydbc3ZbY74jx26zhk8BvOc0OcfEYpGZ/ECc3jmwkLqMWsu5Xd/sVTLEsFV26jFu+0HUEsDBBQAAAAIAHFwW1daoA6towUAAOMPAAAXAAAAZG9jUHJvcHMvdGh1bWJuYWlsLmpwZWftVmtwE1UUPrt7NyltzRAoLRQHwrsywKQtQisCJmnappQ2pC2vcYZJk00TmiZhd9OWTp2R+kD9Iw/ffywFFR1nHFS0oI6tIqCjA4gFCgxjEbX4Gh6Kr4F47m5eQBCUv707e++Xc7577vnOvXM3kWORr2F4RamtFBiGgXJ8IHJa222zWFbZHdWltkorOgC0252hkJ81ADQFZNFRZjYsX7HSoO0HFsZABuRChtMlhUx2eyVgo1y4rl06AgwdD89M7f/XluEWJBcAk4Y46JZcTYhbAXi/KyTKAJozaC9qkUOItXcizhIxQcRGihtUXEJxvYqXK5xahwUxzUXn8jrdiNsRz6hPsjckYTUHpWWVCQFB9LkMtBZ2Mejx+YWkdG/ivsXW5A/H1huHb6bUWLMIxzyq3SuWO6K40+W01iCejHh/SDZT+1TEP4Ub60yIpwOwIzxiaZ3KZ+9t89YuQ5yN2O2TbbVRe1ugvqpanct2NQYXOaKc/S7JgjWDiYhPeQVbpZoPB26hxErrhXicN1wejc9VSM011licNq+lSo3DiaudFXbEuYgfE4OOajVnrkvwlznU+NzekGyP5sANBvxVlWpMohMkRaNil7215epcMkfGTVTnkpUeX6ktym8P+ZWziLmRbWLYURflHHSK1jI1DrkgBOqiMfnRbmcJre0sxAtgKeMEAYJQj70LAnAZDOCAMjDjGAIRPR7wgR8tAnoFtPiYO6ARbal5doWj4gSjQZk9SGfjKqk56gpno5wgySFGUojvPFJJ5pMiUgwGspDcRxaQErQWk3nxufak9elaZ+Nx1kAYo1LeUjBvyA3nJdbrEFf5XAeePHfV7OB1OQuxfJIrABJWIMacmax/X/v7oxMx+kj3/Ycz97VD9c3qy5/hB/k+7Pv5kwkGf4I/iU8/mDA3v5JRE74+JQ8pKYNkDb34yuDEfgB5wSTeVSt6AhtyEx5aCWF91aUq6JiRsBqPGn829hm3GLcZf7ymyimrxG3mdnIfcLu43dznYOB6uF7uQ24v9wb3XtJe3fh8xPde0RtTSz2pai2AX2fWjdVN0pXoxuum6CoT8XQ5unxduW4aesbG9y15vWQtPliBfayqqddSeXXo9UGLokBSKhyAtdec/+hsMo7kE9s1p7aInuUYQ2PVlGhMYNBM1xRr8jUVFMfy00xDXzH21qtOnesGCoQkVrLOmcqpo2eVzm5WfBIIstAq04vWEgytFX0NXtlQYDTONZjwUyUYbAHXrBkGp99vUFySQRQkQWwW3LOAfgfVK/qiQ/m+MdkHEjZ5McD8X/DOOpiwrQwDvC4B5MxO2PLwThz1IkD3HFdYbI7e+QzzBYDkKSxQf2Wa8W46FYlcxPtKuwng8sZI5O+uSOTyVox/EqDHHxkA2drq8wAsXkxvfUgDwuQCT2fju4AZG8elTB5e4BSzAOt9QKL2quja5dHf6sh2sjEGA51cnN1DqZETYKH/Hm6r0SC3G4OJ9IA+DXoY4Bg9sHqG0zORPTAec+VVQuzDyrAc4TXatGHpGUjYORxYhuNYwvE8QWnMA+gHoudHTMg3aUYucWonrskqWLdxS9ok847eUY5D5yYX1osdw9Kzc0aPyZ0ydVreXdNn3z1nblHxPZYSa2lZua2iprZu6TLcXpdb8DR4faslOdzc0rq27aGHH3l0/WOPP7Fp81NPP/Psc8+/0LV120svv7L91dfefOvtne+8271r90cf7/lk7779n3725eGv+o4cPdZ/fOD0N2e+/e77wbM/nL9w8dffLv3+x59/UV1UZ6yl1IVFYFhCOKKluhi2hRL0hJ+QrxlhWqJ1rhk5sWBdWpZ545YdvcMmFTrOjaoXD6VnT549MOU8laYouzVhHf9LWVxYQtdxyOTwwOk5PSyEK1fyoJN9MB2GhqFhaBgahob/OET6/wFQSwMEFAAAAAgAcXBbV4sU/ON5AQAA2wIAABEAAABkb2NQcm9wcy9jb3JlLnhtbI2SzU7DMBCE7zxF1EtOqeMWSomSIAHiBBJSi0DcjL1NDYlt2dumeXucpE356YFbVjP7aTyb9HpXlcEWrJNaZSEdx2EAimshVZGFz8v7aB4GDpkSrNQKsrABF17nZyk3CdcWnqw2YFGCCzxIuYSbbLRGNAkhjq+hYm7sHcqLK20rhn60BTGMf7ICyCSOZ6QCZIIhIy0wMgNxtEcKPiDNxpYdQHACJVSg0BE6puToRbCVO7nQKd+clcTGwEnrQRzcOycHY13X43raWX1+Sl4fHxbdUyOp2qo4jPJU8AQllkC6T7d5/wCO/cAtMNTWD77ET2hqbYXrJQGOW2nQHyMvQIFlCCLYOH+NwDS41ioyBncp+eVtSSVz+OgPt5Igbpp8gbCF4JYp1aTkr9xuWNjK9u457RzDmO5b7JP6AP71Sd/VQXmZ3t4t70f5JKbTKKbR5HIZXyX0PKGztzbdj/0jsNoH+D/xIrmYfyMeAF1+7uGFto3vjvz5H/MvUEsDBBQAAAAIAHFwW1ee0I557wEAAG0EAAAQAAAAZG9jUHJvcHMvYXBwLnhtbJ1UwY7TMBC9I/EPlk9waJNChVDlZgVdrXqgNFKzy3mwJ42FY0e26W75eiYJyaZQIUFO7808vRnP2BE3T7VhJ/RBO7vmi3nKGVrplLbHNb8v7mbvOQsRrALjLK75GQO/yV6+ELl3DfqoMTCysGHNqxibVZIEWWENYU5pS5nS+RoiUX9MXFlqibdOfq/RxuRNmr5L8CmiVahmzWjIe8fVKf6vqXKy7S88FOeG/DJRuAim0DVmC5E8E/HFeRWyVCQ9EB+axmgJkaaR7bT0Lrgysh1IbaMLFcvdI/rcERPJVEvjwEDlO3bXdZft7SxIj2jZoXKP7NVy9fa1SK4IRQ4ejh6aqmtlwsTBaIVd9BcSn13sAz0QW60U2mfdBRe73cbopksMUBwkGNzQeLISTECyHgNii9CuPgftSXmKqxPK6DwL+gctf8nZVwjYDnXNT+A12Mh7WU86bJoQfVbQwsh75B2cyqZYL9u99OCvwt6rOx0rdDQY/qFEer1EMh6T8OUA+hL7klYSr8xjMZ1H1wOfdLnvLia7Poih3m8VdmDhiG1iRBtXN2DPFBrRJ22/hfumcLcQcdjiZVAcKvCo6FmMWx4DYksNe0P6j9R9e+hLPtKwqcAeUQ0WfybaB/PQ/z2yxXKe0tc9jCHW3vfhWWc/AVBLAQIUAxQAAAAIAHFwW1fGr8RntAEAALoMAAATAAAAAAAAAAAAAACAAQAAAABbQ29udGVudF9UeXBlc10ueG1sUEsBAhQDFAAAAAgAcXBbV/ENN+wAAQAA4QIAAAsAAAAAAAAAAAAAAIAB5QEAAF9yZWxzLy5yZWxzUEsBAhQDFAAAAAgAcXBbVwV3nA87AgAAtAwAABQAAAAAAAAAAAAAAIABDgMAAHBwdC9wcmVzZW50YXRpb24ueG1sUEsBAhQDFAAAAAgAcXBbV1KcUMkcAQAAcQQAAB8AAAAAAAAAAAAAAIABewUAAHBwdC9fcmVscy9wcmVzZW50YXRpb24ueG1sLnJlbHNQSwECFAMUAAAACABxcFtXpi2iNe4GAADSLgAAIQAAAAAAAAAAAAAAgAHUBgAAcHB0L3NsaWRlTWFzdGVycy9zbGlkZU1hc3RlcjEueG1sUEsBAhQDFAAAAAgAcXBbV75rQr0NAQAAxgcAACwAAAAAAAAAAAAAAIABAQ4AAHBwdC9zbGlkZU1hc3RlcnMvX3JlbHMvc2xpZGVNYXN0ZXIxLnhtbC5yZWxzUEsBAhQDFAAAAAgAcXBbVwD97A0qBAAABREAACEAAAAAAAAAAAAAAIABWA8AAHBwdC9zbGlkZUxheW91dHMvc2xpZGVMYXlvdXQxLnhtbFBLAQIUAxQAAAAIAHFwW1eAZeGItwAAADYBAAAsAAAAAAAAAAAAAACAAcETAABwcHQvc2xpZGVMYXlvdXRzL19yZWxzL3NsaWRlTGF5b3V0MS54bWwucmVsc1BLAQIUAxQAAAAIAHFwW1c3xjX4jQMAAM0LAAAiAAAAAAAAAAAAAACAAcIUAABwcHQvc2xpZGVMYXlvdXRzL3NsaWRlTGF5b3V0MTAueG1sUEsBAhQDFAAAAAgAcXBbV4Bl4Yi3AAAANgEAAC0AAAAAAAAAAAAAAIABjxgAAHBwdC9zbGlkZUxheW91dHMvX3JlbHMvc2xpZGVMYXlvdXQxMC54bWwucmVsc1BLAQIUAxQAAAAIAHFwW1dLiVBXwAMAAK0MAAAiAAAAAAAAAAAAAACAAZEZAABwcHQvc2xpZGVMYXlvdXRzL3NsaWRlTGF5b3V0MTEueG1sUEsBAhQDFAAAAAgAcXBbV4Bl4Yi3AAAANgEAAC0AAAAAAAAAAAAAAIABkR0AAHBwdC9zbGlkZUxheW91dHMvX3JlbHMvc2xpZGVMYXlvdXQxMS54bWwucmVsc1BLAQIUAxQAAAAIAHFwW1eTCm11IQYAAOcdAAAUAAAAAAAAAAAAAACAAZMeAABwcHQvdGhlbWUvdGhlbWUxLnhtbFBLAQIUAxQAAAAIAHFwW1cBV+iLbQMAAJYLAAAhAAAAAAAAAAAAAACAAeYkAABwcHQvc2xpZGVMYXlvdXRzL3NsaWRlTGF5b3V0Mi54bWxQSwECFAMUAAAACABxcFtXgGXhiLcAAAA2AQAALAAAAAAAAAAAAAAAgAGSKAAAcHB0L3NsaWRlTGF5b3V0cy9fcmVscy9zbGlkZUxheW91dDIueG1sLnJlbHNQSwECFAMUAAAACABxcFtXi2DtWmMEAABYEQAAIQAAAAAAAAAAAAAAgAGTKQAAcHB0L3NsaWRlTGF5b3V0cy9zbGlkZUxheW91dDMueG1sUEsBAhQDFAAAAAgAcXBbV4Bl4Yi3AAAANgEAACwAAAAAAAAAAAAAAIABNS4AAHBwdC9zbGlkZUxheW91dHMvX3JlbHMvc2xpZGVMYXlvdXQzLnhtbC5yZWxzUEsBAhQDFAAAAAgAcXBbV0/KghwIBAAAaBIAACEAAAAAAAAAAAAAAIABNi8AAHBwdC9zbGlkZUxheW91dHMvc2xpZGVMYXlvdXQ0LnhtbFBLAQIUAxQAAAAIAHFwW1eAZeGItwAAADYBAAAsAAAAAAAAAAAAAACAAX0zAABwcHQvc2xpZGVMYXlvdXRzL19yZWxzL3NsaWRlTGF5b3V0NC54bWwucmVsc1BLAQIUAxQAAAAIAHFwW1fppMSP4wQAADYcAAAhAAAAAAAAAAAAAACAAX40AABwcHQvc2xpZGVMYXlvdXRzL3NsaWRlTGF5b3V0NS54bWxQSwECFAMUAAAACABxcFtXgGXhiLcAAAA2AQAALAAAAAAAAAAAAAAAgAGgOQAAcHB0L3NsaWRlTGF5b3V0cy9fcmVscy9zbGlkZUxheW91dDUueG1sLnJlbHNQSwECFAMUAAAACABxcFtXLbQm9RIDAAC4CAAAIQAAAAAAAAAAAAAAgAGhOgAAcHB0L3NsaWRlTGF5b3V0cy9zbGlkZUxheW91dDYueG1sUEsBAhQDFAAAAAgAcXBbV4Bl4Yi3AAAANgEAACwAAAAAAAAAAAAAAIAB8j0AAHBwdC9zbGlkZUxheW91dHMvX3JlbHMvc2xpZGVMYXlvdXQ2LnhtbC5yZWxzUEsBAhQDFAAAAAgAcXBbV+sXn3fmAgAAZwcAACEAAAAAAAAAAAAAAIAB8z4AAHBwdC9zbGlkZUxheW91dHMvc2xpZGVMYXlvdXQ3LnhtbFBLAQIUAxQAAAAIAHFwW1eAZeGItwAAADYBAAAsAAAAAAAAAAAAAACAARhCAABwcHQvc2xpZGVMYXlvdXRzL19yZWxzL3NsaWRlTGF5b3V0Ny54bWwucmVsc1BLAQIUAxQAAAAIAHFwW1fNyorVsgQAAMISAAAhAAAAAAAAAAAAAACAARlDAABwcHQvc2xpZGVMYXlvdXRzL3NsaWRlTGF5b3V0OC54bWxQSwECFAMUAAAACABxcFtXgGXhiLcAAAA2AQAALAAAAAAAAAAAAAAAgAEKSAAAcHB0L3NsaWRlTGF5b3V0cy9fcmVscy9zbGlkZUxheW91dDgueG1sLnJlbHNQSwECFAMUAAAACABxcFtXWtO0knkEAAAxEgAAIQAAAAAAAAAAAAAAgAELSQAAcHB0L3NsaWRlTGF5b3V0cy9zbGlkZUxheW91dDkueG1sUEsBAhQDFAAAAAgAcXBbV4Bl4Yi3AAAANgEAACwAAAAAAAAAAAAAAIABw00AAHBwdC9zbGlkZUxheW91dHMvX3JlbHMvc2xpZGVMYXlvdXQ5LnhtbC5yZWxzUEsBAhQDFAAAAAgAcXBbV+jkSdE5AwAAsyQAACgAAAAAAAAAAAAAAIABxE4AAHBwdC9wcmludGVyU2V0dGluZ3MvcHJpbnRlclNldHRpbmdzMS5iaW5QSwECFAMUAAAACABxcFtXXJxHFEQBAACJAgAAEQAAAAAAAAAAAAAAgAFDUgAAcHB0L3ByZXNQcm9wcy54bWxQSwECFAMUAAAACABxcFtXZzMmjZsBAACCAwAAEQAAAAAAAAAAAAAAgAG2UwAAcHB0L3ZpZXdQcm9wcy54bWxQSwECFAMUAAAACABxcFtX2P2Nj6UAAAC2AAAAEwAAAAAAAAAAAAAAgAGAVQAAcHB0L3RhYmxlU3R5bGVzLnhtbFBLAQIUAxQAAAAIAHFwW1c3axy8dAEAAJkDAAAVAAAAAAAAAAAAAACAAVZWAABwcHQvc2xpZGVzL3NsaWRlMS54bWxQSwECFAMUAAAACABxcFtXNuhQzbcAAAA2AQAAIAAAAAAAAAAAAAAAgAH9VwAAcHB0L3NsaWRlcy9fcmVscy9zbGlkZTEueG1sLnJlbHNQSwECFAMUAAAACABxcFtXWqAOraMFAADjDwAAFwAAAAAAAAAAAAAAgAHyWAAAZG9jUHJvcHMvdGh1bWJuYWlsLmpwZWdQSwECFAMUAAAACABxcFtXixT843kBAADbAgAAEQAAAAAAAAAAAAAAgAHKXgAAZG9jUHJvcHMvY29yZS54bWxQSwECFAMUAAAACABxcFtXntCOee8BAABtBAAAEAAAAAAAAAAAAAAAgAFyYAAAZG9jUHJvcHMvYXBwLnhtbFBLBQYAAAAAJgAmAKMLAACPYgAAAAA=" +) + +simple_unstructured_scenario = ( + TestScenarioBuilder() + .set_name("simple_unstructured_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "unstructured"}, + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "sample.pdf": { + # minimal pdf file inlined as base 64 + "contents": pdf_file, + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "sample.docx": { + # minimal docx file inlined as base 64 + "contents": docx_file, + "last_modified": "2023-06-06T03:54:07.000Z", + }, + "sample.pptx": { + # minimal pptx file inlined as base 64 + "contents": pptx_file, + "last_modified": "2023-06-07T03:54:07.000Z", + }, + } + ) + .set_file_type("unstructured") + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": json_schema, + "name": "stream1", + "source_defined_cursor": True, + 'source_defined_primary_key': [["document_key"]], + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "document_key": "sample.pdf", + "content": "# Hello World", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "sample.pdf", + "_ab_source_file_parse_error": None, + }, + "stream": "stream1", + }, + { + "data": { + "document_key": "sample.docx", + "content": "# Content", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "sample.docx", + "_ab_source_file_parse_error": None, + }, + "stream": "stream1", + }, + { + "data": { + "document_key": "sample.pptx", + "content": "# Title", + "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z", + "_ab_source_file_url": "sample.pptx", + "_ab_source_file_parse_error": None, + }, + "stream": "stream1", + }, + ] + ) +).build() + +corrupted_file_scenario = ( + TestScenarioBuilder() + .set_name("corrupted_file_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "unstructured"}, + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "sample.pdf": { + # bytes that can't be parsed as pdf + "contents": bytes("___ corrupted file ___", "utf-8"), + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("unstructured") + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": json_schema, + "name": "stream1", + "source_defined_cursor": True, + 'source_defined_primary_key': [["document_key"]], + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "document_key": "sample.pdf", + "content": None, + "_ab_source_file_parse_error": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. Contact Support if you need assistance.\nfilename=sample.pdf message=No /Root object! - Is this really a PDF?", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "sample.pdf", + }, + "stream": "stream1", + }, + ] + ) +).build() + +no_file_extension_unstructured_scenario = ( + TestScenarioBuilder() + .set_name("no_file_extension_unstructured_scenario") + .set_config( + { + "streams": [ + { + "name": "stream1", + "format": {"filetype": "unstructured"}, + "globs": ["*"], + "validation_policy": "Emit Record", + } + ] + } + ) + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "pdf_without_extension": { + # same file, but can't be detected via file extension + "contents": pdf_file, + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "docx_without_extension": { + # same file, but can't be detected via file extesion + "contents": docx_file, + "last_modified": "2023-06-06T03:54:07.000Z", + }, + "pptx_without_extension": { + # minimal pptx file inlined as base 64 + "contents": pptx_file, + "last_modified": "2023-06-07T03:54:07.000Z", + }, + } + ) + .set_file_type("unstructured") + ) + .set_expected_catalog( + { + "streams": [ + { + "default_cursor_field": ["_ab_source_file_last_modified"], + "json_schema": json_schema, + "name": "stream1", + "source_defined_cursor": True, + 'source_defined_primary_key': [["document_key"]], + "supported_sync_modes": ["full_refresh", "incremental"], + } + ] + } + ) + .set_expected_records( + [ + { + "data": { + "document_key": "pdf_without_extension", + "content": "# Hello World", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "pdf_without_extension", + "_ab_source_file_parse_error": None, + }, + "stream": "stream1", + }, + { + "data": { + "document_key": "docx_without_extension", + "content": "# Content", + "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", + "_ab_source_file_url": "docx_without_extension", + "_ab_source_file_parse_error": None, + }, + "stream": "stream1", + }, + { + "data": { + "document_key": "pptx_without_extension", + "content": "# Title", + "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z", + "_ab_source_file_url": "pptx_without_extension", + "_ab_source_file_parse_error": None, + }, + "stream": "stream1", + }, + ] + ) +).build() diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py index 33c8587a04d2..58d528cb7caf 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py @@ -4,6 +4,9 @@ from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_protocol.models import SyncMode +from unit_tests.sources.file_based.scenarios.file_based_source_builder import FileBasedSourceBuilder from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder """ @@ -17,19 +20,22 @@ _base_user_input_schema_scenario = ( TestScenarioBuilder() - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11", "val12"), - ("val21", "val22"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11", "val12"), + ("val21", "val22"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + } } - } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -38,18 +44,10 @@ "json_schema": { "type": "object", "properties": { - "col1": { - "type": "string" - }, - "col2": { - "type": "string" - }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, + "col1": {"type": "string"}, + "col2": {"type": "string"}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, }, }, "name": "stream1", @@ -61,10 +59,24 @@ ) .set_expected_records( [ - {"data": {"col1": "val11", "col2": "val12", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, - {"data": {"col1": "val21", "col2": "val22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val11", + "col2": "val12", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val21", + "col2": "val22", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, ] ) ) @@ -78,7 +90,7 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*"], "validation_policy": "Emit Record", "input_schema": '{"col1": "string", "col2": "string"}', @@ -98,7 +110,7 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*"], "validation_policy": "Emit Record", "input_schema": '{"col1": "x", "col2": "string"}', @@ -106,6 +118,7 @@ ] } ) + .set_catalog(CatalogBuilder().with_stream("stream1", SyncMode.full_refresh).build()) .set_expected_check_status("FAILED") .set_expected_check_error(None, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value) .set_expected_discover_error(ConfigValidationError, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value) @@ -121,7 +134,7 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*"], "validation_policy": "Emit Record", "input_schema": '{"col1": "integer", "col2": "string"}', @@ -139,18 +152,10 @@ "json_schema": { "type": "object", "properties": { - "col1": { - "type": "integer" - }, - "col2": { - "type": "string" - }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, + "col1": {"type": "integer"}, + "col2": {"type": "string"}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, }, }, "name": "stream1", @@ -171,7 +176,7 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*"], "validation_policy": "Skip Record", "input_schema": '{"col1": "integer", "col2": "string"}', @@ -189,18 +194,10 @@ "json_schema": { "type": "object", "properties": { - "col1": { - "type": "integer" - }, - "col2": { - "type": "string" - }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, + "col1": {"type": "integer"}, + "col2": {"type": "string"}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, }, }, "name": "stream1", @@ -211,56 +208,61 @@ } ) .set_expected_records([]) - .set_expected_logs({ - "read": [ - { - 'level': 'WARN', - 'message': 'Records in file did not pass validation policy. stream=stream1 file=a.csv n_skipped=2 validation_policy=skip_record', - }, - { - 'level': "WARN", - 'message': 'Could not cast the value to the expected type.: col1: value=val11,expected_type=integer', - }, - { - 'level': "WARN", - 'message': 'Could not cast the value to the expected type.: col1: value=val21,expected_type=integer', - }, - ] - }) + .set_expected_logs( + { + "read": [ + { + "level": "WARN", + "message": "Records in file did not pass validation policy. stream=stream1 file=a.csv n_skipped=2 validation_policy=skip_record", + }, + { + "level": "WARN", + "message": "Could not cast the value to the expected type.: col1: value=val11,expected_type=integer", + }, + { + "level": "WARN", + "message": "Could not cast the value to the expected type.: col1: value=val21,expected_type=integer", + }, + ] + } + ) ).build() _base_multi_stream_user_input_schema_scenario = ( TestScenarioBuilder() - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val11a", 21), - ("val12a", 22), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "b.csv": { - "contents": [ - ("col1", "col2", "col3"), - ("val11b", "val12b", "val13b"), - ("val21b", "val22b", "val23b"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "c.csv": { - "contents": [ - ("col1",), - ("val11c",), - ("val21c",), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val11a", 21), + ("val12a", 22), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.csv": { + "contents": [ + ("col1", "col2", "col3"), + ("val11b", "val12b", "val13b"), + ("val21b", "val22b", "val23b"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "c.csv": { + "contents": [ + ("col1",), + ("val11c",), + ("val21c",), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ @@ -275,13 +277,9 @@ "col2": { "type": "integer", }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, - } + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream1", "source_defined_cursor": True, @@ -301,13 +299,9 @@ "col3": { "type": "string", }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, - } + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream2", "source_defined_cursor": True, @@ -321,13 +315,9 @@ "col1": { "type": ["null", "string"], }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, - } + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + }, }, "name": "stream3", "source_defined_cursor": True, @@ -338,19 +328,53 @@ ) .set_expected_records( [ - {"data": {"col1": "val11a", "col2": 21, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, - {"data": {"col1": "val12a", "col2": 22, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val11a", + "col2": 21, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val12a", + "col2": 22, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, # The files in b.csv are emitted despite having an invalid schema - {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.csv"}, "stream": "stream2"}, - {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.csv"}, "stream": "stream2"}, - {"data": {"col1": "val11c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "c.csv"}, "stream": "stream3"}, - {"data": {"col1": "val21c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "c.csv"}, "stream": "stream3"}, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream2", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream2", + }, + { + "data": {"col1": "val11c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"}, + "stream": "stream3", + }, + { + "data": {"col1": "val21c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"}, + "stream": "stream3", + }, ] ) ) @@ -364,25 +388,24 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["a.csv"], "validation_policy": "Emit Record", "input_schema": '{"col1": "string", "col2": "integer"}', }, { "name": "stream2", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["b.csv"], "validation_policy": "Emit Record", "input_schema": '{"col1": "string", "col2": "string", "col3": "string"}', }, { "name": "stream3", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["c.csv"], "validation_policy": "Emit Record", }, - ] } ) @@ -398,28 +421,28 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["a.csv"], "validation_policy": "Emit Record", "input_schema": '{"col1": "string", "col2": "integer"}', }, { "name": "stream2", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["b.csv"], "validation_policy": "Emit Record", "input_schema": '{"col1": "x", "col2": "string", "col3": "string"}', # this stream's schema is invalid }, { "name": "stream3", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["c.csv"], "validation_policy": "Emit Record", }, - ] } ) + .set_catalog(CatalogBuilder().with_stream("stream1", SyncMode.full_refresh).with_stream("stream2", SyncMode.full_refresh).with_stream("stream3", SyncMode.full_refresh).build()) .set_expected_check_status("FAILED") .set_expected_check_error(None, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value) .set_expected_discover_error(ConfigValidationError, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value) @@ -435,25 +458,24 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["a.csv"], "validation_policy": "Emit Record", "input_schema": '{"col1": "string", "col2": "integer"}', }, { "name": "stream2", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["b.csv"], "validation_policy": "Emit Record", "input_schema": '{"col1": "string", "col2": "integer", "col3": "string"}', # this stream's records do not conform to the schema }, { "name": "stream3", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["c.csv"], "validation_policy": "Emit Record", }, - ] } ) @@ -465,18 +487,10 @@ "json_schema": { "type": "object", "properties": { - "col1": { - "type": "string" - }, - "col2": { - "type": "integer" - }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, + "col1": {"type": "string"}, + "col2": {"type": "integer"}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, }, }, "name": "stream1", @@ -488,21 +502,11 @@ "json_schema": { "type": "object", "properties": { - "col1": { - "type": "string" - }, - "col2": { - "type": "integer" - }, - "col3": { - "type": "string" - }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, + "col1": {"type": "string"}, + "col2": {"type": "integer"}, + "col3": {"type": "string"}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, }, }, "name": "stream2", @@ -517,12 +521,8 @@ "col1": { "type": ["null", "string"], }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, }, }, "name": "stream3", @@ -536,33 +536,68 @@ .set_expected_check_error(None, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value) .set_expected_records( [ - {"data": {"col1": "val11a", "col2": 21, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, - {"data": {"col1": "val12a", "col2": 22, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, - {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.csv"}, "stream": "stream2"}, - {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.csv"}, "stream": "stream2"}, - {"data": {"col1": "val11c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "c.csv"}, "stream": "stream3"}, - {"data": {"col1": "val21c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "c.csv"}, "stream": "stream3"}, - ] - ) - .set_expected_logs({ - "read": [ { - 'level': "WARN", - 'message': 'Could not cast the value to the expected type.: col2: value=val12b,expected_type=integer', + "data": { + "col1": "val11a", + "col2": 21, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val12a", + "col2": 22, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream2", + }, + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream2", }, { - 'level': "WARN", - 'message': 'Could not cast the value to the expected type.: col2: value=val22b,expected_type=integer', + "data": {"col1": "val11c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"}, + "stream": "stream3", + }, + { + "data": {"col1": "val21c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"}, + "stream": "stream3", }, ] - - }) + ) + .set_expected_logs( + { + "read": [ + { + "level": "WARN", + "message": "Could not cast the value to the expected type.: col2: value=val12b,expected_type=integer", + }, + { + "level": "WARN", + "message": "Could not cast the value to the expected type.: col2: value=val22b,expected_type=integer", + }, + ] + } + ) ).build() @@ -574,25 +609,24 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["a.csv"], "validation_policy": "Emit Record", "input_schema": '{"col1": "string", "col2": "integer"}', }, { "name": "stream2", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["b.csv"], "validation_policy": "Skip Record", "input_schema": '{"col1": "string", "col2": "integer", "col3": "string"}', # this stream's records do not conform to the schema }, { "name": "stream3", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["c.csv"], "validation_policy": "Emit Record", }, - ] } ) @@ -604,18 +638,10 @@ "json_schema": { "type": "object", "properties": { - "col1": { - "type": "string" - }, - "col2": { - "type": "integer" - }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, + "col1": {"type": "string"}, + "col2": {"type": "integer"}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, }, }, "name": "stream1", @@ -627,21 +653,11 @@ "json_schema": { "type": "object", "properties": { - "col1": { - "type": "string" - }, - "col2": { - "type": "integer" - }, - "col3": { - "type": "string" - }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, + "col1": {"type": "string"}, + "col2": {"type": "integer"}, + "col3": {"type": "string"}, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, }, }, "name": "stream2", @@ -656,12 +672,8 @@ "col1": { "type": ["null", "string"], }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, }, }, "name": "stream3", @@ -675,34 +687,54 @@ .set_expected_check_error(None, FileBasedSourceError.ERROR_PARSING_USER_PROVIDED_SCHEMA.value) .set_expected_records( [ - {"data": {"col1": "val11a", "col2": 21, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, - {"data": {"col1": "val12a", "col2": 22, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val11a", + "col2": 21, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val12a", + "col2": 22, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, # {"data": {"col1": "val11b", "col2": "val12b", "col3": "val13b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", # "_ab_source_file_url": "b.csv"}, "stream": "stream2"}, # {"data": {"col1": "val21b", "col2": "val22b", "col3": "val23b", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", # "_ab_source_file_url": "b.csv"}, "stream": "stream2"}, - {"data": {"col1": "val11c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "c.csv"}, "stream": "stream3"}, - {"data": {"col1": "val21c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "c.csv"}, "stream": "stream3"}, - ] - ) - .set_expected_logs({ - "read": [ { - 'level': 'WARN', - 'message': 'Records in file did not pass validation policy. stream=stream2 file=b.csv n_skipped=2 validation_policy=skip_record', + "data": {"col1": "val11c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"}, + "stream": "stream3", }, { - 'level': "WARN", - 'message': 'Could not cast the value to the expected type.: col2: value=val12b,expected_type=integer', - }, - { - 'level': "WARN", - 'message': 'Could not cast the value to the expected type.: col2: value=val22b,expected_type=integer', + "data": {"col1": "val21c", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"}, + "stream": "stream3", }, ] - }) + ) + .set_expected_logs( + { + "read": [ + { + "level": "WARN", + "message": "Records in file did not pass validation policy. stream=stream2 file=b.csv n_skipped=2 validation_policy=skip_record", + }, + { + "level": "WARN", + "message": "Could not cast the value to the expected type.: col2: value=val12b,expected_type=integer", + }, + { + "level": "WARN", + "message": "Could not cast the value to the expected type.: col2: value=val22b,expected_type=integer", + }, + ] + } + ) ).build() diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py index fb2abf648e1e..9ac880b11fe5 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/validation_policy_scenarios.py @@ -2,53 +2,58 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_cdk.sources.file_based.exceptions import FileBasedSourceError + +from airbyte_cdk.utils.traced_exception import AirbyteTracedException +from unit_tests.sources.file_based.scenarios.file_based_source_builder import FileBasedSourceBuilder from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder _base_single_stream_scenario = ( TestScenarioBuilder() - .set_files( - { - "a.csv": { - "contents": [ - ("col1", "col2"), - ("val_a_11", "1"), - ("val_a_12", "2"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "b.csv": { # The records in this file do not conform to the schema - "contents": [ - ("col1", "col2"), - ("val_b_11", "this is text that will trigger validation policy"), - ("val_b_12", "2"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "c.csv": { - "contents": [ - ("col1",), - ("val_c_11",), - ("val_c_12", "val_c_22"), # This record is not parsable - ("val_c_13",), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "d.csv": { - "contents": [ - ("col1",), - ("val_d_11",), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a.csv": { + "contents": [ + ("col1", "col2"), + ("val_a_11", "1"), + ("val_a_12", "2"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b.csv": { # The records in this file do not conform to the schema + "contents": [ + ("col1", "col2"), + ("val_b_11", "this is text that will trigger validation policy"), + ("val_b_12", "2"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "c.csv": { + "contents": [ + ("col1",), + ("val_c_11",), + ("val_c_12", "val_c_22"), # This record is not parsable + ("val_c_13",), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "d.csv": { + "contents": [ + ("col1",), + ("val_d_11",), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ { - 'default_cursor_field': ['_ab_source_file_last_modified'], + "default_cursor_field": ["_ab_source_file_last_modified"], "json_schema": { "type": "object", "properties": { @@ -58,12 +63,8 @@ "col2": { "type": "integer", }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, }, }, "name": "stream1", @@ -78,74 +79,75 @@ _base_multi_stream_scenario = ( TestScenarioBuilder() - .set_files( - { - "a/a1.csv": { - "contents": [ - ("col1", "col2"), - ("val_aa1_11", "1"), - ("val_aa1_12", "2"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "a/a2.csv": { - "contents": [ - ("col1", "col2"), - ("val_aa2_11", "this is text that will trigger validation policy"), - ("val_aa2_12", "2"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "a/a3.csv": { - "contents": [ - ("col1",), - ("val_aa3_11",), - ("val_aa3_12", "val_aa3_22"), # This record is not parsable - ("val_aa3_13",), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "a/a4.csv": { - "contents": [ - ("col1",), - ("val_aa4_11",), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - - "b/b1.csv": { # The records in this file do not conform to the schema - "contents": [ - ("col1", "col2"), - ("val_bb1_11", "1"), - ("val_bb1_12", "2"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "b/b2.csv": { - "contents": [ - ("col1", "col2"), - ("val_bb2_11", "this is text that will trigger validation policy"), - ("val_bb2_12", "2"), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - "b/b3.csv": { - "contents": [ - ("col1",), - ("val_bb3_11",), - ("val_bb3_12",), - ], - "last_modified": "2023-06-05T03:54:07.000Z", - }, - - } + .set_source_builder( + FileBasedSourceBuilder() + .set_files( + { + "a/a1.csv": { + "contents": [ + ("col1", "col2"), + ("val_aa1_11", "1"), + ("val_aa1_12", "2"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "a/a2.csv": { + "contents": [ + ("col1", "col2"), + ("val_aa2_11", "this is text that will trigger validation policy"), + ("val_aa2_12", "2"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "a/a3.csv": { + "contents": [ + ("col1",), + ("val_aa3_11",), + ("val_aa3_12", "val_aa3_22"), # This record is not parsable + ("val_aa3_13",), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "a/a4.csv": { + "contents": [ + ("col1",), + ("val_aa4_11",), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b/b1.csv": { # The records in this file do not conform to the schema + "contents": [ + ("col1", "col2"), + ("val_bb1_11", "1"), + ("val_bb1_12", "2"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b/b2.csv": { + "contents": [ + ("col1", "col2"), + ("val_bb2_11", "this is text that will trigger validation policy"), + ("val_bb2_12", "2"), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + "b/b3.csv": { + "contents": [ + ("col1",), + ("val_bb3_11",), + ("val_bb3_12",), + ], + "last_modified": "2023-06-05T03:54:07.000Z", + }, + } + ) + .set_file_type("csv") ) - .set_file_type("csv") .set_expected_catalog( { "streams": [ { - 'default_cursor_field': ['_ab_source_file_last_modified'], + "default_cursor_field": ["_ab_source_file_last_modified"], "json_schema": { "type": "object", "properties": { @@ -155,12 +157,8 @@ "col2": { "type": "integer", }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, }, }, "name": "stream1", @@ -169,7 +167,7 @@ }, { "json_schema": { - 'default_cursor_field': ['_ab_source_file_last_modified'], + "default_cursor_field": ["_ab_source_file_last_modified"], "type": "object", "properties": { "col1": { @@ -178,12 +176,8 @@ "col2": { "type": "integer", }, - "_ab_source_file_last_modified": { - "type": "string" - }, - "_ab_source_file_url": { - "type": "string" - }, + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, }, }, "name": "stream2", @@ -204,7 +198,7 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "Skip Record", } @@ -213,32 +207,76 @@ ) .set_expected_records( [ - {"data": {"col1": "val_a_11", "col2": 1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, - {"data": {"col1": "val_a_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val_a_11", + "col2": 1, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val_a_12", + "col2": 2, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, # {"data": {"col1": "val_b_11", "col2": "this is text that will trigger validation policy", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, # This record is skipped because it does not conform - {"data": {"col1": "val_b_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, - {"data": {"col1": "val_c_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, - # {"data": {"col1": "val_c_12", None: "val_c_22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, # This record is malformed so should not be emitted - # {"data": {"col1": "val_c_13", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, # Skipped since previous record is malformed - {"data": {"col1": "val_d_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "d.csv"}, "stream": "stream1"}, - ] - ) - .set_expected_logs({ - "read": [ { - "level": "WARN", - "message": "Records in file did not pass validation policy. stream=stream1 file=b.csv n_skipped=1 validation_policy=skip_record", + "data": { + "col1": "val_b_12", + "col2": 2, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", }, { - "level": "ERROR", - "message": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. stream=stream1 file=c.csv line_no=2 n_skipped=0", + "data": { + "col1": "val_c_11", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "c.csv", + }, + "stream": "stream1", }, + # {"data": {"col1": "val_c_12", None: "val_c_22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, # This record is malformed so should not be emitted + # {"data": {"col1": "val_c_13", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, # Skipped since previous record is malformed { - "level": "WARN", - "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", + "data": { + "col1": "val_d_11", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "d.csv", + }, + "stream": "stream1", }, ] - }) + ) + .set_expected_logs( + { + "read": [ + { + "level": "WARN", + "message": "Records in file did not pass validation policy. stream=stream1 file=b.csv n_skipped=1 validation_policy=skip_record", + }, + { + "level": "ERROR", + "message": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. stream=stream1 file=c.csv line_no=2 n_skipped=0", + }, + { + "level": "WARN", + "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", + }, + ] + } + ) + .set_expected_read_error( + AirbyteTracedException, + "Please check the logged errors for more information.", + ) ).build() @@ -250,62 +288,143 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["a/*.csv"], "validation_policy": "Skip Record", }, { "name": "stream2", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["b/*.csv"], "validation_policy": "Skip Record", - } - + }, ] } ) .set_expected_records( [ - {"data": {"col1": "val_aa1_11", "col2": 1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a1.csv"}, "stream": "stream1"}, - {"data": {"col1": "val_aa1_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a1.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val_aa1_11", + "col2": 1, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a/a1.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val_aa1_12", + "col2": 2, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a/a1.csv", + }, + "stream": "stream1", + }, # {"data": {"col1": "val_aa2_11", "col2": "this is text that will trigger validation policy", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a2.csv"}, "stream": "stream1"}, # This record is skipped because it does not conform - {"data": {"col1": "val_aa2_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a2.csv"}, "stream": "stream1"}, - {"data": {"col1": "val_aa3_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a3.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val_aa2_12", + "col2": 2, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a/a2.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val_aa3_11", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a/a3.csv", + }, + "stream": "stream1", + }, # {"data": {"col1": "val_aa3_12", None: "val_aa3_22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a3.csv"}, "stream": "stream1"}, # This record is malformed so should not be emitted # {"data": {"col1": "val_aa3_13", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a3.csv"}, "stream": "stream1"}, # Skipped since previous record is malformed - {"data": {"col1": "val_aa4_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a4.csv"}, "stream": "stream1"}, - {"data": {"col1": "val_bb1_11", "col2": 1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b1.csv"}, "stream": "stream2"}, - {"data": {"col1": "val_bb1_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b1.csv"}, "stream": "stream2"}, - # {"data": {"col1": "val_bb2_11", "col2": "val_bb2_21", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b2.csv"}, "stream": "stream2"}, # This record is skipped because it does not conform - {"data": {"col1": "val_bb2_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b2.csv"}, "stream": "stream2"}, - {"data": {"col1": "val_bb3_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b3.csv"}, "stream": "stream2"}, - {"data": {"col1": "val_bb3_12", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b3.csv"}, "stream": "stream2"}, - ] - ) - .set_expected_logs({ - "read": [ { - "level": "WARN", - "message": "Records in file did not pass validation policy. stream=stream1 file=a/a2.csv n_skipped=1 validation_policy=skip_record", + "data": { + "col1": "val_aa4_11", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a/a4.csv", + }, + "stream": "stream1", }, { - "level": "ERROR", - "message": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. stream=stream1 file=a/a3.csv line_no=2 n_skipped=0", + "data": { + "col1": "val_bb1_11", + "col2": 1, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b/b1.csv", + }, + "stream": "stream2", }, { - "level": "WARN", - "message": "Records in file did not pass validation policy. stream=stream2 file=b/b2.csv n_skipped=1 validation_policy=skip_record", + "data": { + "col1": "val_bb1_12", + "col2": 2, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b/b1.csv", + }, + "stream": "stream2", }, + # {"data": {"col1": "val_bb2_11", "col2": "val_bb2_21", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b2.csv"}, "stream": "stream2"}, # This record is skipped because it does not conform { - "level": "WARN", - "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", + "data": { + "col1": "val_bb2_12", + "col2": 2, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b/b2.csv", + }, + "stream": "stream2", }, { - "level": "WARN", - "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", + "data": { + "col1": "val_bb3_11", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b/b3.csv", + }, + "stream": "stream2", + }, + { + "data": { + "col1": "val_bb3_12", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b/b3.csv", + }, + "stream": "stream2", }, ] - }) + ) + .set_expected_logs( + { + "read": [ + { + "level": "WARN", + "message": "Records in file did not pass validation policy. stream=stream1 file=a/a2.csv n_skipped=1 validation_policy=skip_record", + }, + { + "level": "ERROR", + "message": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. stream=stream1 file=a/a3.csv line_no=2 n_skipped=0", + }, + { + "level": "WARN", + "message": "Records in file did not pass validation policy. stream=stream2 file=b/b2.csv n_skipped=1 validation_policy=skip_record", + }, + { + "level": "WARN", + "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", + }, + { + "level": "WARN", + "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", + }, + ] + } + ) + .set_expected_read_error( + AirbyteTracedException, + "Please check the logged errors for more information.", + ) ).build() @@ -317,7 +436,7 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "Emit Record", } @@ -326,28 +445,66 @@ ) .set_expected_records( [ - {"data": {"col1": "val_a_11", "col2": 1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, - {"data": {"col1": "val_a_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, - {"data": {"col1": "val_b_11", "col2": "this is text that will trigger validation policy", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, # This record is skipped because it does not conform - {"data": {"col1": "val_b_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.csv"}, "stream": "stream1"}, - {"data": {"col1": "val_c_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, - # {"data": {"col1": "val_c_12", None: "val_c_22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, # This record is malformed so should not be emitted - # {"data": {"col1": "val_c_13", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, # No more records from this stream are emitted after we hit a parse error - {"data": {"col1": "val_d_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "d.csv"}, "stream": "stream1"}, - ] - ) - .set_expected_logs({ - "read": [ { - "level": "ERROR", - "message": f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream=stream1 file=c.csv line_no=2 n_skipped=0", + "data": { + "col1": "val_a_11", + "col2": 1, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val_a_12", + "col2": 2, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", }, { - "level": "WARN", - "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", + "data": { + "col1": "val_b_11", + "col2": "this is text that will trigger validation policy", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, # This record is skipped because it does not conform + { + "data": { + "col1": "val_b_12", + "col2": 2, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val_c_11", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "c.csv", + }, + "stream": "stream1", + }, + # {"data": {"col1": "val_c_12", None: "val_c_22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, # This record is malformed so should not be emitted + # {"data": {"col1": "val_c_13", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c.csv"}, "stream": "stream1"}, # No more records from this stream are emitted after we hit a parse error + { + "data": { + "col1": "val_d_11", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "d.csv", + }, + "stream": "stream1", }, ] - }) + ) + .set_expected_read_error( + AirbyteTracedException, + "Please check the logged errors for more information.", + ) ).build() @@ -359,54 +516,133 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["a/*.csv"], "validation_policy": "Emit Record", }, { "name": "stream2", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["b/*.csv"], "validation_policy": "Emit Record", - } - + }, ] } ) .set_expected_records( [ - {"data": {"col1": "val_aa1_11", "col2": 1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a1.csv"}, "stream": "stream1"}, - {"data": {"col1": "val_aa1_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a1.csv"}, "stream": "stream1"}, - {"data": {"col1": "val_aa2_11", "col2": "this is text that will trigger validation policy", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a2.csv"}, "stream": "stream1"}, - {"data": {"col1": "val_aa2_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a2.csv"}, "stream": "stream1"}, - {"data": {"col1": "val_aa3_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a3.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val_aa1_11", + "col2": 1, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a/a1.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val_aa1_12", + "col2": 2, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a/a1.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val_aa2_11", + "col2": "this is text that will trigger validation policy", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a/a2.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val_aa2_12", + "col2": 2, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a/a2.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val_aa3_11", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a/a3.csv", + }, + "stream": "stream1", + }, # {"data": {"col1": "val_aa3_12", None: "val_aa3_22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a3.csv"}, "stream": "stream1"}, # This record is malformed so should not be emitted # {"data": {"col1": "val_aa3_13", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a3.csv"}, "stream": "stream1"}, # Skipped since previous record is malformed - {"data": {"col1": "val_aa4_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a4.csv"}, "stream": "stream1"}, - {"data": {"col1": "val_bb1_11", "col2": 1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b1.csv"}, "stream": "stream2"}, - {"data": {"col1": "val_bb1_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b1.csv"}, "stream": "stream2"}, - {"data": {"col1": "val_bb2_11", "col2": "this is text that will trigger validation policy", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b2.csv"}, "stream": "stream2"}, - {"data": {"col1": "val_bb2_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b2.csv"}, "stream": "stream2"}, - {"data": {"col1": "val_bb3_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b3.csv"}, "stream": "stream2"}, - {"data": {"col1": "val_bb3_12", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b3.csv"}, "stream": "stream2"}, - ] - ) - .set_expected_logs({ - "read": [ { - "level": "ERROR", - "message": f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream=stream1 file=a/a3.csv line_no=2 n_skipped=0", + "data": { + "col1": "val_aa4_11", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a/a4.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val_bb1_11", + "col2": 1, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b/b1.csv", + }, + "stream": "stream2", }, { - "level": "WARN", - "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", + "data": { + "col1": "val_bb1_12", + "col2": 2, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b/b1.csv", + }, + "stream": "stream2", }, { - "level": "WARN", - "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", + "data": { + "col1": "val_bb2_11", + "col2": "this is text that will trigger validation policy", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b/b2.csv", + }, + "stream": "stream2", + }, + { + "data": { + "col1": "val_bb2_12", + "col2": 2, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b/b2.csv", + }, + "stream": "stream2", + }, + { + "data": { + "col1": "val_bb3_11", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b/b3.csv", + }, + "stream": "stream2", + }, + { + "data": { + "col1": "val_bb3_12", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b/b3.csv", + }, + "stream": "stream2", }, ] - }) + ) + .set_expected_read_error( + AirbyteTracedException, + "Please check the logged errors for more information.", + ) ).build() @@ -418,30 +654,50 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["*.csv"], "validation_policy": "Wait for Discover", } ] } ) - .set_expected_records([ - {"data": {"col1": "val_a_11", "col2": 1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, - {"data": {"col1": "val_a_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv"}, "stream": "stream1"}, - # No records past that because the first record for the second file did not conform to the schema - ]) - .set_expected_logs({ - "read": [ + .set_expected_records( + [ { - "level": "WARN", - "message": "Stopping sync in accordance with the configured validation policy. Records in file did not conform to the schema. stream=stream1 file=b.csv validation_policy=Wait for Discover n_skipped=0", + "data": { + "col1": "val_a_11", + "col2": 1, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", }, { - "level": "WARN", - "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", + "data": { + "col1": "val_a_12", + "col2": 2, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", }, + # No records past that because the first record for the second file did not conform to the schema ] - }) + ) + .set_expected_logs( + { + "read": [ + { + "level": "WARN", + "message": "Stopping sync in accordance with the configured validation policy. Records in file did not conform to the schema. stream=stream1 file=b.csv validation_policy=Wait for Discover n_skipped=0", + }, + { + "level": "WARN", + "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", + }, + ] + } + ) ).build() @@ -453,56 +709,89 @@ "streams": [ { "name": "stream1", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["a/*.csv"], "validation_policy": "Wait for Discover", }, { "name": "stream2", - "file_type": "csv", + "format": {"filetype": "csv"}, "globs": ["b/*.csv"], "validation_policy": "Wait for Discover", - } - + }, ] } ) .set_expected_records( [ - {"data": {"col1": "val_aa1_11", "col2": 1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a1.csv"}, "stream": "stream1"}, - {"data": {"col1": "val_aa1_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a1.csv"}, "stream": "stream1"}, + { + "data": { + "col1": "val_aa1_11", + "col2": 1, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a/a1.csv", + }, + "stream": "stream1", + }, + { + "data": { + "col1": "val_aa1_12", + "col2": 2, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a/a1.csv", + }, + "stream": "stream1", + }, # {"data": {"col1": "val_aa2_11", "col2": "this is text that will trigger validation policy", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a2.csv"}, "stream": "stream1"}, # {"data": {"col1": "val_aa2_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a2.csv"}, "stream": "stream1"}, # {"data": {"col1": "val_aa3_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a3.csv"}, "stream": "stream1"}, # {"data": {"col1": "val_aa3_12", None: "val_aa3_22", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a3.csv"}, "stream": "stream1"}, # {"data": {"col1": "val_aa3_13", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a3.csv"}, "stream": "stream1"}, # {"data": {"col1": "val_aa4_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a/a4.csv"}, "stream": "stream1"}, - {"data": {"col1": "val_bb1_11", "col2": 1, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b1.csv"}, "stream": "stream2"}, - {"data": {"col1": "val_bb1_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b1.csv"}, "stream": "stream2"}, + { + "data": { + "col1": "val_bb1_11", + "col2": 1, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b/b1.csv", + }, + "stream": "stream2", + }, + { + "data": { + "col1": "val_bb1_12", + "col2": 2, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b/b1.csv", + }, + "stream": "stream2", + }, # {"data": {"col1": "val_bb2_11", "col2": "this is text that will trigger validation policy", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b2.csv"}, "stream": "stream2"}, # {"data": {"col1": "val_bb2_12", "col2": 2, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b2.csv"}, "stream": "stream2"}, # {"data": {"col1": "val_bb3_11", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b3.csv"}, "stream": "stream2"}, # {"data": {"col1": "val_bb3_12", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b/b3.csv"}, "stream": "stream2"}, ] ) - .set_expected_logs({ - "read": [ - { - "level": "WARN", - "message": "Stopping sync in accordance with the configured validation policy. Records in file did not conform to the schema. stream=stream1 file=a/a2.csv validation_policy=Wait for Discover n_skipped=0", - }, - { - "level": "WARN", - "message": "Stopping sync in accordance with the configured validation policy. Records in file did not conform to the schema. stream=stream2 file=b/b2.csv validation_policy=Wait for Discover n_skipped=0", - }, - { - "level": "WARN", - "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", - }, - { - "level": "WARN", - "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", - }, - ] - }) + .set_expected_logs( + { + "read": [ + { + "level": "WARN", + "message": "Stopping sync in accordance with the configured validation policy. Records in file did not conform to the schema. stream=stream1 file=a/a2.csv validation_policy=Wait for Discover n_skipped=0", + }, + { + "level": "WARN", + "message": "Stopping sync in accordance with the configured validation policy. Records in file did not conform to the schema. stream=stream2 file=b/b2.csv validation_policy=Wait for Discover n_skipped=0", + }, + { + "level": "WARN", + "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", + }, + { + "level": "WARN", + "message": "Could not cast the value to the expected type.: col2: value=this is text that will trigger validation policy,expected_type=integer", + }, + ] + } + ) ).build() diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/schema_validation_policies/test_default_schema_validation_policy.py b/airbyte-cdk/python/unit_tests/sources/file_based/schema_validation_policies/test_default_schema_validation_policy.py index 957dcd356bad..a7d2bfb7c019 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/schema_validation_policies/test_default_schema_validation_policy.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/schema_validation_policies/test_default_schema_validation_policy.py @@ -23,13 +23,9 @@ SCHEMA = { "type": "object", "properties": { - "col1": { - "type": "string" - }, - "col2": { - "type": "integer" - }, - } + "col1": {"type": "string"}, + "col2": {"type": "integer"}, + }, } @@ -42,13 +38,10 @@ pytest.param(NONCONFORMING_RECORD, SCHEMA, ValidationPolicy.skip_record, False, id="nonconforming_skip_record"), pytest.param(CONFORMING_RECORD, SCHEMA, ValidationPolicy.wait_for_discover, True, id="record-conforms_wait_for_discover"), pytest.param(NONCONFORMING_RECORD, SCHEMA, ValidationPolicy.wait_for_discover, False, id="nonconforming_wait_for_discover"), - ] + ], ) def test_record_passes_validation_policy( - record: Mapping[str, Any], - schema: Mapping[str, Any], - validation_policy: ValidationPolicy, - expected_result: bool + record: Mapping[str, Any], schema: Mapping[str, Any], validation_policy: ValidationPolicy, expected_result: bool ) -> None: if validation_policy == ValidationPolicy.wait_for_discover and expected_result is False: with pytest.raises(StopSyncPerValidationPolicy): diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/stream/test_default_file_based_cursor.py b/airbyte-cdk/python/unit_tests/sources/file_based/stream/test_default_file_based_cursor.py index fdf755b42cdd..2088097f7ef4 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/stream/test_default_file_based_cursor.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/stream/test_default_file_based_cursor.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock import pytest +from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig, ValidationPolicy from airbyte_cdk.sources.file_based.remote_file import RemoteFile from airbyte_cdk.sources.file_based.stream.cursor.default_file_based_cursor import DefaultFileBasedCursor @@ -16,21 +17,19 @@ @pytest.mark.parametrize( "files_to_add, expected_start_time, expected_state_dict", [ - pytest.param([ - RemoteFile(uri="a.csv", - last_modified=datetime.strptime("2021-01-01T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv"), - RemoteFile(uri="b.csv", - last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv"), - RemoteFile(uri="c.csv", - last_modified=datetime.strptime("2020-12-31T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv") - - ], - [datetime(2021, 1, 1), - datetime(2021, 1, 1), - datetime(2020, 12, 31)], + pytest.param( + [ + RemoteFile( + uri="a.csv", last_modified=datetime.strptime("2021-01-01T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), file_type="csv" + ), + RemoteFile( + uri="b.csv", last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), file_type="csv" + ), + RemoteFile( + uri="c.csv", last_modified=datetime.strptime("2020-12-31T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), file_type="csv" + ), + ], + [datetime(2021, 1, 1), datetime(2021, 1, 1), datetime(2020, 12, 31)], { "history": { "a.csv": "2021-01-01T00:00:00.000000Z", @@ -39,26 +38,24 @@ }, "_ab_source_file_last_modified": "2021-01-02T00:00:00.000000Z_b.csv", }, - id="test_file_start_time_is_earliest_time_in_history"), - pytest.param([ - RemoteFile(uri="a.csv", - last_modified=datetime.strptime("2021-01-01T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv"), - RemoteFile(uri="b.csv", - last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv"), - RemoteFile(uri="c.csv", - last_modified=datetime.strptime("2021-01-03T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv"), - RemoteFile(uri="d.csv", - last_modified=datetime.strptime("2021-01-04T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv"), - - ], - [datetime(2021, 1, 1), - datetime(2021, 1, 1), - datetime(2021, 1, 1), - datetime(2021, 1, 2)], + id="test_file_start_time_is_earliest_time_in_history", + ), + pytest.param( + [ + RemoteFile( + uri="a.csv", last_modified=datetime.strptime("2021-01-01T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), file_type="csv" + ), + RemoteFile( + uri="b.csv", last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), file_type="csv" + ), + RemoteFile( + uri="c.csv", last_modified=datetime.strptime("2021-01-03T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), file_type="csv" + ), + RemoteFile( + uri="d.csv", last_modified=datetime.strptime("2021-01-04T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), file_type="csv" + ), + ], + [datetime(2021, 1, 1), datetime(2021, 1, 1), datetime(2021, 1, 1), datetime(2021, 1, 2)], { "history": { "b.csv": "2021-01-02T00:00:00.000000Z", @@ -67,31 +64,35 @@ }, "_ab_source_file_last_modified": "2021-01-04T00:00:00.000000Z_d.csv", }, - id="test_earliest_file_is_removed_from_history_if_history_is_full"), - pytest.param([ - RemoteFile(uri="a.csv", - last_modified=datetime.strptime("2021-01-01T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv"), - RemoteFile(uri="file_with_same_timestamp_as_b.csv", - last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv"), - RemoteFile(uri="b.csv", - last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv"), - RemoteFile(uri="c.csv", - last_modified=datetime.strptime("2021-01-03T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv"), - RemoteFile(uri="d.csv", - last_modified=datetime.strptime("2021-01-04T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv"), - - ], - [datetime(2021, 1, 1), - datetime(2021, 1, 1), - datetime(2021, 1, 1), - datetime(2021, 1, 2), - datetime(2021, 1, 2), - ], + id="test_earliest_file_is_removed_from_history_if_history_is_full", + ), + pytest.param( + [ + RemoteFile( + uri="a.csv", last_modified=datetime.strptime("2021-01-01T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), file_type="csv" + ), + RemoteFile( + uri="file_with_same_timestamp_as_b.csv", + last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), + file_type="csv", + ), + RemoteFile( + uri="b.csv", last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), file_type="csv" + ), + RemoteFile( + uri="c.csv", last_modified=datetime.strptime("2021-01-03T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), file_type="csv" + ), + RemoteFile( + uri="d.csv", last_modified=datetime.strptime("2021-01-04T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), file_type="csv" + ), + ], + [ + datetime(2021, 1, 1), + datetime(2021, 1, 1), + datetime(2021, 1, 1), + datetime(2021, 1, 2), + datetime(2021, 1, 2), + ], { "history": { "file_with_same_timestamp_as_b.csv": "2021-01-02T00:00:00.000000Z", @@ -100,7 +101,8 @@ }, "_ab_source_file_last_modified": "2021-01-04T00:00:00.000000Z_d.csv", }, - id="test_files_are_sorted_by_timestamp_and_by_name"), + id="test_files_are_sorted_by_timestamp_and_by_name", + ), ], ) def test_add_file(files_to_add: List[RemoteFile], expected_start_time: List[datetime], expected_state_dict: Mapping[str, Any]) -> None: @@ -113,53 +115,68 @@ def test_add_file(files_to_add: List[RemoteFile], expected_start_time: List[date assert expected_state_dict == cursor.get_state() -@pytest.mark.parametrize("files, expected_files_to_sync, max_history_size, history_is_partial", [ - pytest.param([ - RemoteFile(uri="a.csv", - last_modified=datetime.strptime("2021-01-01T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv"), - RemoteFile(uri="b.csv", - last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv"), - RemoteFile(uri="c.csv", - last_modified=datetime.strptime("2020-12-31T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv") +@pytest.mark.parametrize( + "files, expected_files_to_sync, max_history_size, history_is_partial", + [ + pytest.param( + [ + RemoteFile( + uri="a.csv", last_modified=datetime.strptime("2021-01-01T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), file_type="csv" + ), + RemoteFile( + uri="b.csv", last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), file_type="csv" + ), + RemoteFile( + uri="c.csv", last_modified=datetime.strptime("2020-12-31T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), file_type="csv" + ), + ], + [ + RemoteFile( + uri="a.csv", last_modified=datetime.strptime("2021-01-01T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), file_type="csv" + ), + RemoteFile( + uri="b.csv", last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), file_type="csv" + ), + RemoteFile( + uri="c.csv", last_modified=datetime.strptime("2020-12-31T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), file_type="csv" + ), + ], + 3, + True, + id="test_all_files_should_be_synced", + ), + pytest.param( + [ + RemoteFile( + uri="a.csv", last_modified=datetime.strptime("2021-01-01T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), file_type="csv" + ), + RemoteFile( + uri="b.csv", last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), file_type="csv" + ), + RemoteFile( + uri="c.csv", last_modified=datetime.strptime("2020-12-31T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), file_type="csv" + ), + ], + [ + RemoteFile( + uri="a.csv", last_modified=datetime.strptime("2021-01-01T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), file_type="csv" + ), + RemoteFile( + uri="b.csv", last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), file_type="csv" + ), + RemoteFile( + uri="c.csv", last_modified=datetime.strptime("2020-12-31T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), file_type="csv" + ), + ], + 2, + True, + id="test_sync_more_files_than_history_size", + ), ], - [ - RemoteFile(uri="a.csv", - last_modified=datetime.strptime("2021-01-01T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv"), - RemoteFile(uri="b.csv", - last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv"), - RemoteFile(uri="c.csv", - last_modified=datetime.strptime("2020-12-31T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv") - ], 3, True, id="test_all_files_should_be_synced"), - pytest.param([ - RemoteFile(uri="a.csv", - last_modified=datetime.strptime("2021-01-01T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv"), - RemoteFile(uri="b.csv", - last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv"), - RemoteFile(uri="c.csv", - last_modified=datetime.strptime("2020-12-31T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv") - ], [ - RemoteFile(uri="a.csv", - last_modified=datetime.strptime("2021-01-01T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv"), - RemoteFile(uri="b.csv", - last_modified=datetime.strptime("2021-01-02T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv"), - RemoteFile(uri="c.csv", - last_modified=datetime.strptime("2020-12-31T00:00:00.000Z", "%Y-%m-%dT%H:%M:%S.%fZ"), - file_type="csv") - - ], 2, True, id="test_sync_more_files_than_history_size"), -]) -def test_get_files_to_sync(files: List[RemoteFile], expected_files_to_sync: List[RemoteFile], max_history_size: int, history_is_partial: bool) -> None: +) +def test_get_files_to_sync( + files: List[RemoteFile], expected_files_to_sync: List[RemoteFile], max_history_size: int, history_is_partial: bool +) -> None: logger = MagicMock() cursor = get_cursor(max_history_size, 3) @@ -182,9 +199,7 @@ def test_only_recent_files_are_synced_if_history_is_full() -> None: ] state = { - "history": { - f.uri: f.last_modified.strftime(DefaultFileBasedCursor.DATE_TIME_FORMAT) for f in files_in_history - }, + "history": {f.uri: f.last_modified.strftime(DefaultFileBasedCursor.DATE_TIME_FORMAT) for f in files_in_history}, } cursor.set_initial_state(state) @@ -204,11 +219,14 @@ def test_only_recent_files_are_synced_if_history_is_full() -> None: logger.warning.assert_called_once() -@pytest.mark.parametrize("modified_at_delta, should_sync_file", [ - pytest.param(timedelta(days=-1), False, id="test_modified_at_is_earlier"), - pytest.param(timedelta(days=0), False, id="test_modified_at_is_equal"), - pytest.param(timedelta(days=1), True, id="test_modified_at_is_more_recent"), -]) +@pytest.mark.parametrize( + "modified_at_delta, should_sync_file", + [ + pytest.param(timedelta(days=-1), False, id="test_modified_at_is_earlier"), + pytest.param(timedelta(days=0), False, id="test_modified_at_is_equal"), + pytest.param(timedelta(days=1), True, id="test_modified_at_is_more_recent"), + ], +) def test_sync_file_already_present_in_history(modified_at_delta: timedelta, should_sync_file: bool) -> None: logger = MagicMock() cursor = get_cursor(2, 3) @@ -219,9 +237,7 @@ def test_sync_file_already_present_in_history(modified_at_delta: timedelta, shou ] state = { - "history": { - f.uri: f.last_modified.strftime(DefaultFileBasedCursor.DATE_TIME_FORMAT) for f in files_in_history - }, + "history": {f.uri: f.last_modified.strftime(DefaultFileBasedCursor.DATE_TIME_FORMAT) for f in files_in_history}, } cursor.set_initial_state(state) @@ -235,14 +251,33 @@ def test_sync_file_already_present_in_history(modified_at_delta: timedelta, shou @freeze_time("2023-06-06T00:00:00Z") @pytest.mark.parametrize( - "file_name, last_modified, earliest_dt_in_history, should_sync_file", [ + "file_name, last_modified, earliest_dt_in_history, should_sync_file", + [ pytest.param("a.csv", datetime(2023, 6, 3), datetime(2023, 6, 6), True, id="test_last_modified_is_equal_to_time_buffer"), pytest.param("b.csv", datetime(2023, 6, 6), datetime(2023, 6, 6), False, id="test_file_was_already_synced"), pytest.param("b.csv", datetime(2023, 6, 7), datetime(2023, 6, 6), True, id="test_file_was_synced_in_the_past"), - pytest.param("b.csv", datetime(2023, 6, 3), datetime(2023, 6, 6), False, id="test_file_was_synced_in_the_past_but_last_modified_is_earlier_in_history"), - pytest.param("a.csv", datetime(2023, 6, 3), datetime(2023, 6, 3), False, id="test_last_modified_is_equal_to_earliest_dt_in_history_and_lexicographically_smaller"), - pytest.param("c.csv", datetime(2023, 6, 3), datetime(2023, 6, 3), True, id="test_last_modified_is_equal_to_earliest_dt_in_history_and_lexicographically_greater"), - ] + pytest.param( + "b.csv", + datetime(2023, 6, 3), + datetime(2023, 6, 6), + False, + id="test_file_was_synced_in_the_past_but_last_modified_is_earlier_in_history", + ), + pytest.param( + "a.csv", + datetime(2023, 6, 3), + datetime(2023, 6, 3), + False, + id="test_last_modified_is_equal_to_earliest_dt_in_history_and_lexicographically_smaller", + ), + pytest.param( + "c.csv", + datetime(2023, 6, 3), + datetime(2023, 6, 3), + True, + id="test_last_modified_is_equal_to_earliest_dt_in_history_and_lexicographically_greater", + ), + ], ) def test_should_sync_file(file_name: str, last_modified: datetime, earliest_dt_in_history: datetime, should_sync_file: bool) -> None: logger = MagicMock() @@ -252,7 +287,10 @@ def test_should_sync_file(file_name: str, last_modified: datetime, earliest_dt_i cursor._start_time = cursor._compute_start_time() cursor._initial_earliest_file_in_history = cursor._compute_earliest_file_in_history() - assert bool(list(cursor.get_files_to_sync([RemoteFile(uri=file_name, last_modified=last_modified, file_type="csv")], logger))) == should_sync_file + assert ( + bool(list(cursor.get_files_to_sync([RemoteFile(uri=file_name, last_modified=last_modified, file_type="csv")], logger))) + == should_sync_file + ) def test_set_initial_state_no_history() -> None: @@ -264,5 +302,9 @@ def get_cursor(max_history_size: int, days_to_sync_if_history_is_full: int) -> D cursor_cls = DefaultFileBasedCursor cursor_cls.DEFAULT_MAX_HISTORY_SIZE = max_history_size config = FileBasedStreamConfig( - file_type="csv", name="test", validation_policy=ValidationPolicy.emit_record, days_to_sync_if_history_is_full=days_to_sync_if_history_is_full) + format=CsvFormat(), + name="test", + validation_policy=ValidationPolicy.emit_record, + days_to_sync_if_history_is_full=days_to_sync_if_history_is_full, + ) return cursor_cls(config) diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/stream/test_default_file_based_stream.py b/airbyte-cdk/python/unit_tests/sources/file_based/stream/test_default_file_based_stream.py index 99b2ae789a4e..be36413f271b 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/stream/test_default_file_based_stream.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/stream/test_default_file_based_stream.py @@ -2,21 +2,29 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import traceback import unittest from datetime import datetime, timezone from typing import Any, Iterable, Iterator, Mapping from unittest.mock import Mock import pytest -from airbyte_cdk.models import Level +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level +from airbyte_cdk.models import Type as MessageType from airbyte_cdk.sources.file_based.availability_strategy import AbstractFileBasedAvailabilityStrategy from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy +from airbyte_cdk.sources.file_based.exceptions import FileBasedErrorsCollector, FileBasedSourceError from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader from airbyte_cdk.sources.file_based.file_types.file_type_parser import FileTypeParser from airbyte_cdk.sources.file_based.remote_file import RemoteFile from airbyte_cdk.sources.file_based.schema_validation_policies import AbstractSchemaValidationPolicy from airbyte_cdk.sources.file_based.stream.cursor import AbstractFileBasedCursor from airbyte_cdk.sources.file_based.stream.default_file_based_stream import DefaultFileBasedStream +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + + +class MockFormat: + pass @pytest.mark.parametrize( @@ -50,23 +58,27 @@ ), pytest.param( {"type": "object", "properties": {"prop": {"type": "string"}}}, - {"type": ["null", "object"], "properties": {"prop": {"type": ["null", "string"]}}}, + { + "type": ["null", "object"], + "properties": {"prop": {"type": ["null", "string"]}}, + }, id="deeply-nested-schema", ), ], ) -def test_fill_nulls(input_schema: Mapping[str, Any], expected_output: Mapping[str, Any]) -> None: +def test_fill_nulls( + input_schema: Mapping[str, Any], expected_output: Mapping[str, Any] +) -> None: assert DefaultFileBasedStream._fill_nulls(input_schema) == expected_output class DefaultFileBasedStreamTest(unittest.TestCase): - _FILE_TYPE = "file_type" _NOW = datetime(2022, 10, 22, tzinfo=timezone.utc) _A_RECORD = {"a_record": 1} def setUp(self) -> None: self._stream_config = Mock() - self._stream_config.file_type = self._FILE_TYPE + self._stream_config.format = MockFormat() self._stream_config.name = "a stream name" self._catalog_schema = Mock() self._stream_reader = Mock(spec=AbstractFileBasedStreamReader) @@ -83,48 +95,168 @@ def setUp(self) -> None: stream_reader=self._stream_reader, availability_strategy=self._availability_strategy, discovery_policy=self._discovery_policy, - parsers={self._FILE_TYPE: self._parser}, + parsers={MockFormat: self._parser}, validation_policy=self._validation_policy, cursor=self._cursor, + errors_collector=FileBasedErrorsCollector(), ) def test_when_read_records_from_slice_then_return_records(self) -> None: self._parser.parse_records.return_value = [self._A_RECORD] - messages = list(self._stream.read_records_from_slice({"files": [RemoteFile(uri="uri", last_modified=self._NOW)]})) - assert list(map(lambda message: message.record.data["data"], messages)) == [self._A_RECORD] + messages = list( + self._stream.read_records_from_slice( + {"files": [RemoteFile(uri="uri", last_modified=self._NOW)]} + ) + ) + assert list(map(lambda message: message.record.data["data"], messages)) == [ + self._A_RECORD + ] - def test_given_exception_when_read_records_from_slice_then_do_process_other_files(self) -> None: + def test_given_exception_when_read_records_from_slice_then_do_process_other_files( + self, + ) -> None: """ The current behavior for source-s3 v3 does not fail sync on some errors and hence, we will keep this behaviour for now. One example we can easily reproduce this is by having a file with gzip extension that is not actually a gzip file. The reader will fail to open the file but the sync won't fail. Ticket: https://github.com/airbytehq/airbyte/issues/29680 """ - self._parser.parse_records.side_effect = [ValueError("An error"), [self._A_RECORD]] + self._parser.parse_records.side_effect = [ + ValueError("An error"), + [self._A_RECORD], + ] - messages = list(self._stream.read_records_from_slice({"files": [ - RemoteFile(uri="invalid_file", last_modified=self._NOW), - RemoteFile(uri="valid_file", last_modified=self._NOW), - ]})) + messages = list( + self._stream.read_records_from_slice( + { + "files": [ + RemoteFile(uri="invalid_file", last_modified=self._NOW), + RemoteFile(uri="valid_file", last_modified=self._NOW), + ] + } + ) + ) assert messages[0].log.level == Level.ERROR assert messages[1].record.data["data"] == self._A_RECORD - def test_given_exception_after_skipping_records_when_read_records_from_slice_then_send_warning(self) -> None: + def test_given_traced_exception_when_read_records_from_slice_then_fail( + self, + ) -> None: + """ + When a traced exception is raised, the stream shouldn't try to handle but pass it on to the caller. + """ + self._parser.parse_records.side_effect = [AirbyteTracedException("An error")] + + with pytest.raises(AirbyteTracedException): + list( + self._stream.read_records_from_slice( + { + "files": [ + RemoteFile(uri="invalid_file", last_modified=self._NOW), + RemoteFile(uri="valid_file", last_modified=self._NOW), + ] + } + ) + ) + + def test_given_exception_after_skipping_records_when_read_records_from_slice_then_send_warning( + self, + ) -> None: self._stream_config.schemaless = False self._validation_policy.record_passes_validation_policy.return_value = False - self._parser.parse_records.side_effect = [self._iter([self._A_RECORD, ValueError("An error")])] + self._parser.parse_records.side_effect = [ + self._iter([self._A_RECORD, ValueError("An error")]) + ] - messages = list(self._stream.read_records_from_slice({"files": [ - RemoteFile(uri="invalid_file", last_modified=self._NOW), - RemoteFile(uri="valid_file", last_modified=self._NOW), - ]})) + messages = list( + self._stream.read_records_from_slice( + { + "files": [ + RemoteFile(uri="invalid_file", last_modified=self._NOW), + RemoteFile(uri="valid_file", last_modified=self._NOW), + ] + } + ) + ) assert messages[0].log.level == Level.ERROR assert messages[1].log.level == Level.WARN + def test_override_max_n_files_for_schema_inference_is_respected(self) -> None: + self._discovery_policy.n_concurrent_requests = 1 + self._discovery_policy.get_max_n_files_for_schema_inference.return_value = 3 + self._stream.config.input_schema = None + self._stream.config.schemaless = None + self._parser.infer_schema.return_value = {"data": {"type": "string"}} + files = [RemoteFile(uri=f"file{i}", last_modified=self._NOW) for i in range(10)] + self._stream_reader.get_matching_files.return_value = files + + schema = self._stream.get_json_schema() + + assert schema == { + "type": "object", + "properties": { + "_ab_source_file_last_modified": {"type": "string"}, + "_ab_source_file_url": {"type": "string"}, + "data": {"type": ["null", "string"]}, + }, + } + assert self._parser.infer_schema.call_count == 3 + def _iter(self, x: Iterable[Any]) -> Iterator[Any]: for item in x: if isinstance(item, Exception): raise item yield item + + +class TestFileBasedErrorCollector: + test_error_collector: FileBasedErrorsCollector = FileBasedErrorsCollector() + + @pytest.mark.parametrize( + "stream, file, line_no, n_skipped, collector_expected_len", + ( + ("stream_1", "test.csv", 1, 1, 1), + ("stream_2", "test2.csv", 2, 2, 2), + ), + ids=[ + "Single error", + "Multiple errors", + ], + ) + def test_collect_parsing_error( + self, stream, file, line_no, n_skipped, collector_expected_len + ) -> None: + test_error_pattern = "Error parsing record." + # format the error body + test_error = ( + AirbyteMessage( + type=MessageType.LOG, + log=AirbyteLogMessage( + level=Level.ERROR, + message=f"{FileBasedSourceError.ERROR_PARSING_RECORD.value} stream={stream} file={file} line_no={line_no} n_skipped={n_skipped}", + stack_trace=traceback.format_exc(), + ), + ), + ) + # collecting the error + self.test_error_collector.collect(test_error) + # check the error has been collected + assert len(self.test_error_collector.errors) == collector_expected_len + # check for the patern presence for the collected errors + for error in self.test_error_collector.errors: + assert test_error_pattern in error[0].log.message + + def test_yield_and_raise_collected(self) -> None: + # we expect the following method will raise the AirbyteTracedException + with pytest.raises(AirbyteTracedException) as parse_error: + list(self.test_error_collector.yield_and_raise_collected()) + assert ( + parse_error.value.message + == "Some errors occured while reading from the source." + ) + assert ( + parse_error.value.internal_message + == "Please check the logged errors for more information." + ) diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/test_file_based_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/test_file_based_scenarios.py new file mode 100644 index 000000000000..e39c4b02c5a8 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/file_based/test_file_based_scenarios.py @@ -0,0 +1,269 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from pathlib import PosixPath + +import pytest +from _pytest.capture import CaptureFixture +from airbyte_cdk.sources.abstract_source import AbstractSource +from freezegun import freeze_time +from unit_tests.sources.file_based.scenarios.avro_scenarios import ( + avro_all_types_scenario, + avro_file_with_double_as_number_scenario, + multiple_avro_combine_schema_scenario, + multiple_streams_avro_scenario, + single_avro_scenario, +) +from unit_tests.sources.file_based.scenarios.check_scenarios import ( + error_empty_stream_scenario, + error_listing_files_scenario, + error_multi_stream_scenario, + error_reading_file_scenario, + error_record_validation_user_provided_schema_scenario, + success_csv_scenario, + success_extensionless_scenario, + success_multi_stream_scenario, + success_user_provided_schema_scenario, +) +from unit_tests.sources.file_based.scenarios.csv_scenarios import ( + csv_autogenerate_column_names_scenario, + csv_custom_bool_values_scenario, + csv_custom_delimiter_in_double_quotes_scenario, + csv_custom_delimiter_with_escape_char_scenario, + csv_custom_format_scenario, + csv_custom_null_values_scenario, + csv_double_quote_is_set_scenario, + csv_escape_char_is_set_scenario, + csv_multi_stream_scenario, + csv_newline_in_values_not_quoted_scenario, + csv_newline_in_values_quoted_value_scenario, + csv_single_stream_scenario, + csv_skip_after_header_scenario, + csv_skip_before_and_after_header_scenario, + csv_skip_before_header_scenario, + csv_string_are_not_null_if_strings_can_be_null_is_false_scenario, + csv_string_can_be_null_with_input_schemas_scenario, + csv_string_not_null_if_no_null_values_scenario, + csv_strings_can_be_null_not_quoted_scenario, + earlier_csv_scenario, + empty_schema_inference_scenario, + invalid_csv_multi_scenario, + invalid_csv_scenario, + multi_csv_scenario, + multi_csv_stream_n_file_exceeds_limit_for_inference, + multi_format_analytics_scenario, + multi_stream_custom_format, + schemaless_csv_multi_stream_scenario, + schemaless_csv_scenario, + schemaless_with_user_input_schema_fails_connection_check_multi_stream_scenario, + schemaless_with_user_input_schema_fails_connection_check_scenario, + single_csv_scenario, +) +from unit_tests.sources.file_based.scenarios.incremental_scenarios import ( + multi_csv_different_timestamps_scenario, + multi_csv_include_missing_files_within_history_range, + multi_csv_per_timestamp_scenario, + multi_csv_remove_old_files_if_history_is_full_scenario, + multi_csv_same_timestamp_more_files_than_history_size_scenario, + multi_csv_same_timestamp_scenario, + multi_csv_skip_file_if_already_in_history, + multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_different_timestamps_scenario, + multi_csv_sync_files_within_time_window_if_history_is_incomplete__different_timestamps_scenario, + multi_csv_sync_recent_files_if_history_is_incomplete_scenario, + single_csv_file_is_skipped_if_same_modified_at_as_in_history, + single_csv_file_is_synced_if_modified_at_is_more_recent_than_in_history, + single_csv_input_state_is_earlier_scenario, + single_csv_input_state_is_later_scenario, + single_csv_no_input_state_scenario, +) +from unit_tests.sources.file_based.scenarios.jsonl_scenarios import ( + invalid_jsonl_scenario, + jsonl_multi_stream_scenario, + jsonl_user_input_schema_scenario, + multi_jsonl_stream_n_bytes_exceeds_limit_for_inference, + multi_jsonl_stream_n_file_exceeds_limit_for_inference, + multi_jsonl_with_different_keys_scenario, + schemaless_jsonl_multi_stream_scenario, + schemaless_jsonl_scenario, + single_jsonl_scenario, +) +from unit_tests.sources.file_based.scenarios.parquet_scenarios import ( + multi_parquet_scenario, + parquet_file_with_decimal_as_float_scenario, + parquet_file_with_decimal_as_string_scenario, + parquet_file_with_decimal_no_config_scenario, + parquet_various_types_scenario, + parquet_with_invalid_config_scenario, + single_parquet_scenario, + single_partitioned_parquet_scenario, +) +from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenario +from unit_tests.sources.file_based.scenarios.unstructured_scenarios import ( + corrupted_file_scenario, + no_file_extension_unstructured_scenario, + simple_markdown_scenario, + simple_txt_scenario, + simple_unstructured_scenario, + unstructured_invalid_file_type_discover_scenario_no_skip, + unstructured_invalid_file_type_discover_scenario_skip, + unstructured_invalid_file_type_read_scenario, +) +from unit_tests.sources.file_based.scenarios.user_input_schema_scenarios import ( + multi_stream_user_input_schema_scenario_emit_nonconforming_records, + multi_stream_user_input_schema_scenario_schema_is_invalid, + multi_stream_user_input_schema_scenario_skip_nonconforming_records, + single_stream_user_input_schema_scenario_emit_nonconforming_records, + single_stream_user_input_schema_scenario_schema_is_invalid, + single_stream_user_input_schema_scenario_skip_nonconforming_records, + valid_multi_stream_user_input_schema_scenario, + valid_single_stream_user_input_schema_scenario, +) +from unit_tests.sources.file_based.scenarios.validation_policy_scenarios import ( + emit_record_scenario_multi_stream, + emit_record_scenario_single_stream, + skip_record_scenario_multi_stream, + skip_record_scenario_single_stream, + wait_for_rediscovery_scenario_multi_stream, + wait_for_rediscovery_scenario_single_stream, +) +from unit_tests.sources.file_based.test_scenarios import verify_check, verify_discover, verify_read, verify_spec + +discover_scenarios = [ + csv_multi_stream_scenario, + csv_single_stream_scenario, + invalid_csv_scenario, + invalid_csv_multi_scenario, + single_csv_scenario, + multi_csv_scenario, + multi_csv_stream_n_file_exceeds_limit_for_inference, + single_csv_input_state_is_earlier_scenario, + single_csv_no_input_state_scenario, + single_csv_input_state_is_later_scenario, + multi_csv_same_timestamp_scenario, + multi_csv_different_timestamps_scenario, + multi_csv_per_timestamp_scenario, + multi_csv_skip_file_if_already_in_history, + multi_csv_include_missing_files_within_history_range, + multi_csv_remove_old_files_if_history_is_full_scenario, + multi_csv_same_timestamp_more_files_than_history_size_scenario, + multi_csv_sync_recent_files_if_history_is_incomplete_scenario, + multi_csv_sync_files_within_time_window_if_history_is_incomplete__different_timestamps_scenario, + multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_different_timestamps_scenario, + single_csv_file_is_skipped_if_same_modified_at_as_in_history, + single_csv_file_is_synced_if_modified_at_is_more_recent_than_in_history, + csv_custom_format_scenario, + earlier_csv_scenario, + multi_stream_custom_format, + empty_schema_inference_scenario, + single_parquet_scenario, + multi_parquet_scenario, + parquet_various_types_scenario, + parquet_file_with_decimal_no_config_scenario, + parquet_file_with_decimal_as_string_scenario, + parquet_file_with_decimal_as_float_scenario, + schemaless_csv_scenario, + schemaless_csv_multi_stream_scenario, + schemaless_with_user_input_schema_fails_connection_check_multi_stream_scenario, + schemaless_with_user_input_schema_fails_connection_check_scenario, + single_stream_user_input_schema_scenario_schema_is_invalid, + single_stream_user_input_schema_scenario_emit_nonconforming_records, + single_stream_user_input_schema_scenario_skip_nonconforming_records, + multi_stream_user_input_schema_scenario_emit_nonconforming_records, + multi_stream_user_input_schema_scenario_skip_nonconforming_records, + multi_stream_user_input_schema_scenario_schema_is_invalid, + valid_multi_stream_user_input_schema_scenario, + valid_single_stream_user_input_schema_scenario, + single_jsonl_scenario, + multi_jsonl_with_different_keys_scenario, + multi_jsonl_stream_n_file_exceeds_limit_for_inference, + multi_jsonl_stream_n_bytes_exceeds_limit_for_inference, + invalid_jsonl_scenario, + jsonl_multi_stream_scenario, + jsonl_user_input_schema_scenario, + schemaless_jsonl_scenario, + schemaless_jsonl_multi_stream_scenario, + csv_string_can_be_null_with_input_schemas_scenario, + csv_string_are_not_null_if_strings_can_be_null_is_false_scenario, + csv_string_not_null_if_no_null_values_scenario, + csv_strings_can_be_null_not_quoted_scenario, + csv_newline_in_values_quoted_value_scenario, + csv_escape_char_is_set_scenario, + csv_double_quote_is_set_scenario, + csv_custom_delimiter_with_escape_char_scenario, + csv_custom_delimiter_in_double_quotes_scenario, + csv_skip_before_header_scenario, + csv_skip_after_header_scenario, + csv_skip_before_and_after_header_scenario, + csv_custom_bool_values_scenario, + csv_custom_null_values_scenario, + single_avro_scenario, + avro_all_types_scenario, + multiple_avro_combine_schema_scenario, + multiple_streams_avro_scenario, + avro_file_with_double_as_number_scenario, + csv_newline_in_values_not_quoted_scenario, + csv_autogenerate_column_names_scenario, + parquet_with_invalid_config_scenario, + single_partitioned_parquet_scenario, + simple_markdown_scenario, + simple_txt_scenario, + simple_unstructured_scenario, + corrupted_file_scenario, + no_file_extension_unstructured_scenario, + unstructured_invalid_file_type_discover_scenario_no_skip, + unstructured_invalid_file_type_discover_scenario_skip, + unstructured_invalid_file_type_read_scenario, +] + +read_scenarios = discover_scenarios + [ + emit_record_scenario_multi_stream, + emit_record_scenario_single_stream, + skip_record_scenario_multi_stream, + skip_record_scenario_single_stream, + multi_format_analytics_scenario, + wait_for_rediscovery_scenario_multi_stream, + wait_for_rediscovery_scenario_single_stream, +] + +spec_scenarios = [ + single_csv_scenario, +] + +check_scenarios = [ + error_empty_stream_scenario, + error_listing_files_scenario, + error_reading_file_scenario, + error_record_validation_user_provided_schema_scenario, + error_multi_stream_scenario, + success_csv_scenario, + success_extensionless_scenario, + success_multi_stream_scenario, + success_user_provided_schema_scenario, + schemaless_with_user_input_schema_fails_connection_check_multi_stream_scenario, + schemaless_with_user_input_schema_fails_connection_check_scenario, + valid_single_stream_user_input_schema_scenario, + single_avro_scenario, + earlier_csv_scenario, +] + + +@pytest.mark.parametrize("scenario", discover_scenarios, ids=[s.name for s in discover_scenarios]) +def test_file_based_discover(capsys: CaptureFixture[str], tmp_path: PosixPath, scenario: TestScenario[AbstractSource]) -> None: + verify_discover(capsys, tmp_path, scenario) + + +@pytest.mark.parametrize("scenario", read_scenarios, ids=[s.name for s in read_scenarios]) +@freeze_time("2023-06-09T00:00:00Z") +def test_file_based_read(scenario: TestScenario[AbstractSource]) -> None: + verify_read(scenario) + + +@pytest.mark.parametrize("scenario", spec_scenarios, ids=[c.name for c in spec_scenarios]) +def test_file_based_spec(capsys: CaptureFixture[str], scenario: TestScenario[AbstractSource]) -> None: + verify_spec(capsys, scenario) + + +@pytest.mark.parametrize("scenario", check_scenarios, ids=[c.name for c in check_scenarios]) +def test_file_based_check(capsys: CaptureFixture[str], tmp_path: PosixPath, scenario: TestScenario[AbstractSource]) -> None: + verify_check(capsys, tmp_path, scenario) diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/test_file_based_stream_reader.py b/airbyte-cdk/python/unit_tests/sources/file_based/test_file_based_stream_reader.py index 0f32399fd6df..677f7f419ea6 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/test_file_based_stream_reader.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/test_file_based_stream_reader.py @@ -26,12 +26,30 @@ """ FILEPATHS = [ - "a", "a.csv", "a.csv.gz", "a.jsonl", - "a/b", "a/b.csv", "a/b.csv.gz", "a/b.jsonl", - "a/c", "a/c.csv", "a/c.csv.gz", "a/c.jsonl", - "a/b/c", "a/b/c.csv", "a/b/c.csv.gz", "a/b/c.jsonl", - "a/c/c", "a/c/c.csv", "a/c/c.csv.gz", "a/c/c.jsonl", - "a/b/c/d", "a/b/c/d.csv", "a/b/c/d.csv.gz", "a/b/c/d.jsonl" + "a", + "a.csv", + "a.csv.gz", + "a.jsonl", + "a/b", + "a/b.csv", + "a/b.csv.gz", + "a/b.jsonl", + "a/c", + "a/c.csv", + "a/c.csv.gz", + "a/c.jsonl", + "a/b/c", + "a/b/c.csv", + "a/b/c.csv.gz", + "a/b/c.jsonl", + "a/c/c", + "a/c/c.csv", + "a/c/c.csv.gz", + "a/c/c.jsonl", + "a/b/c/d", + "a/b/c/d.csv", + "a/b/c/d.csv.gz", + "a/b/c/d.jsonl", ] FILES = make_remote_files(FILEPATHS) @@ -68,59 +86,186 @@ def documentation_url(cls) -> AnyUrl: pytest.param([], DEFAULT_CONFIG, set(), set(), id="no-globs"), pytest.param([""], DEFAULT_CONFIG, set(), set(), id="empty-string"), pytest.param(["**"], DEFAULT_CONFIG, set(FILEPATHS), set(), id="**"), - pytest.param(["**/*.csv"], DEFAULT_CONFIG, {"a.csv", "a/b.csv", "a/c.csv", "a/b/c.csv", "a/c/c.csv", "a/b/c/d.csv"}, set(), id="**/*.csv"), - pytest.param(["**/*.csv*"], DEFAULT_CONFIG, - {"a.csv", "a.csv.gz", "a/b.csv", "a/b.csv.gz", "a/c.csv", "a/c.csv.gz", "a/b/c.csv", "a/b/c.csv.gz", "a/c/c.csv", - "a/c/c.csv.gz", "a/b/c/d.csv", "a/b/c/d.csv.gz"}, set(), id="**/*.csv*"), + pytest.param( + ["**/*.csv"], DEFAULT_CONFIG, {"a.csv", "a/b.csv", "a/c.csv", "a/b/c.csv", "a/c/c.csv", "a/b/c/d.csv"}, set(), id="**/*.csv" + ), + pytest.param( + ["**/*.csv*"], + DEFAULT_CONFIG, + { + "a.csv", + "a.csv.gz", + "a/b.csv", + "a/b.csv.gz", + "a/c.csv", + "a/c.csv.gz", + "a/b/c.csv", + "a/b/c.csv.gz", + "a/c/c.csv", + "a/c/c.csv.gz", + "a/b/c/d.csv", + "a/b/c/d.csv.gz", + }, + set(), + id="**/*.csv*", + ), pytest.param(["*"], DEFAULT_CONFIG, {"a", "a.csv", "a.csv.gz", "a.jsonl"}, set(), id="*"), pytest.param(["*.csv"], DEFAULT_CONFIG, {"a.csv"}, set(), id="*.csv"), pytest.param(["*.csv*"], DEFAULT_CONFIG, {"a.csv", "a.csv.gz"}, set(), id="*.csv*"), - pytest.param(["*/*"], DEFAULT_CONFIG, {"a/b", "a/b.csv", "a/b.csv.gz", "a/b.jsonl", "a/c", "a/c.csv", "a/c.csv.gz", "a/c.jsonl"}, set(), id="*/*"), + pytest.param( + ["*/*"], + DEFAULT_CONFIG, + {"a/b", "a/b.csv", "a/b.csv.gz", "a/b.jsonl", "a/c", "a/c.csv", "a/c.csv.gz", "a/c.jsonl"}, + set(), + id="*/*", + ), pytest.param(["*/*.csv"], DEFAULT_CONFIG, {"a/b.csv", "a/c.csv"}, set(), id="*/*.csv"), pytest.param(["*/*.csv*"], DEFAULT_CONFIG, {"a/b.csv", "a/b.csv.gz", "a/c.csv", "a/c.csv.gz"}, set(), id="*/*.csv*"), - pytest.param(["*/**"], DEFAULT_CONFIG, - {"a/b", "a/b.csv", "a/b.csv.gz", "a/b.jsonl", "a/c", "a/c.csv", "a/c.csv.gz", "a/c.jsonl", "a/b/c", "a/b/c.csv", - "a/b/c.csv.gz", "a/b/c.jsonl", "a/c/c", "a/c/c.csv", "a/c/c.csv.gz", "a/c/c.jsonl", "a/b/c/d", "a/b/c/d.csv", - "a/b/c/d.csv.gz", "a/b/c/d.jsonl"}, set(), id="*/**"), - pytest.param(["a/*"], DEFAULT_CONFIG, {"a/b", "a/b.csv", "a/b.csv.gz", "a/b.jsonl", "a/c", "a/c.csv", "a/c.csv.gz", "a/c.jsonl"}, - {"a/"}, id="a/*"), + pytest.param( + ["*/**"], + DEFAULT_CONFIG, + { + "a/b", + "a/b.csv", + "a/b.csv.gz", + "a/b.jsonl", + "a/c", + "a/c.csv", + "a/c.csv.gz", + "a/c.jsonl", + "a/b/c", + "a/b/c.csv", + "a/b/c.csv.gz", + "a/b/c.jsonl", + "a/c/c", + "a/c/c.csv", + "a/c/c.csv.gz", + "a/c/c.jsonl", + "a/b/c/d", + "a/b/c/d.csv", + "a/b/c/d.csv.gz", + "a/b/c/d.jsonl", + }, + set(), + id="*/**", + ), + pytest.param( + ["a/*"], + DEFAULT_CONFIG, + {"a/b", "a/b.csv", "a/b.csv.gz", "a/b.jsonl", "a/c", "a/c.csv", "a/c.csv.gz", "a/c.jsonl"}, + {"a/"}, + id="a/*", + ), pytest.param(["a/*.csv"], DEFAULT_CONFIG, {"a/b.csv", "a/c.csv"}, {"a/"}, id="a/*.csv"), pytest.param(["a/*.csv*"], DEFAULT_CONFIG, {"a/b.csv", "a/b.csv.gz", "a/c.csv", "a/c.csv.gz"}, {"a/"}, id="a/*.csv*"), pytest.param(["a/b/*"], DEFAULT_CONFIG, {"a/b/c", "a/b/c.csv", "a/b/c.csv.gz", "a/b/c.jsonl"}, {"a/b/"}, id="a/b/*"), pytest.param(["a/b/*.csv"], DEFAULT_CONFIG, {"a/b/c.csv"}, {"a/b/"}, id="a/b/*.csv"), pytest.param(["a/b/*.csv*"], DEFAULT_CONFIG, {"a/b/c.csv", "a/b/c.csv.gz"}, {"a/b/"}, id="a/b/*.csv*"), - pytest.param(["a/*/*"], DEFAULT_CONFIG, - {"a/b/c", "a/b/c.csv", "a/b/c.csv.gz", "a/b/c.jsonl", "a/c/c", "a/c/c.csv", "a/c/c.csv.gz", "a/c/c.jsonl"}, - {"a/"}, id="a/*/*"), + pytest.param( + ["a/*/*"], + DEFAULT_CONFIG, + {"a/b/c", "a/b/c.csv", "a/b/c.csv.gz", "a/b/c.jsonl", "a/c/c", "a/c/c.csv", "a/c/c.csv.gz", "a/c/c.jsonl"}, + {"a/"}, + id="a/*/*", + ), pytest.param(["a/*/*.csv"], DEFAULT_CONFIG, {"a/b/c.csv", "a/c/c.csv"}, {"a/"}, id="a/*/*.csv"), pytest.param(["a/*/*.csv*"], DEFAULT_CONFIG, {"a/b/c.csv", "a/b/c.csv.gz", "a/c/c.csv", "a/c/c.csv.gz"}, {"a/"}, id="a/*/*.csv*"), - pytest.param(["a/**/*"], DEFAULT_CONFIG, - {"a/b", "a/b.csv", "a/b.csv.gz", "a/b.jsonl", "a/c", "a/c.csv", "a/c.csv.gz", "a/c.jsonl", "a/b/c", "a/b/c.csv", - "a/b/c.csv.gz", "a/b/c.jsonl", "a/c/c", "a/c/c.csv", "a/c/c.csv.gz", "a/c/c.jsonl", "a/b/c/d", "a/b/c/d.csv", - "a/b/c/d.csv.gz", "a/b/c/d.jsonl"}, {"a/"}, id="a/**/*"), - pytest.param(["a/**/*.csv"], DEFAULT_CONFIG, {"a/b.csv", "a/c.csv", "a/b/c.csv", "a/c/c.csv", "a/b/c/d.csv"}, {"a/"}, - id="a/**/*.csv"), - pytest.param(["a/**/*.csv*"], DEFAULT_CONFIG, - {"a/b.csv", "a/b.csv.gz", "a/c.csv", "a/c.csv.gz", "a/b/c.csv", "a/b/c.csv.gz", "a/c/c.csv", "a/c/c.csv.gz", - "a/b/c/d.csv", "a/b/c/d.csv.gz"}, {"a/"}, id="a/**/*.csv*"), - pytest.param(["**/*.csv", "**/*.gz"], DEFAULT_CONFIG, - {"a.csv", "a.csv.gz", "a/b.csv", "a/b.csv.gz", "a/c.csv", "a/c.csv.gz", "a/b/c.csv", "a/b/c.csv.gz", "a/c/c.csv", - "a/c/c.csv.gz", "a/b/c/d.csv", "a/b/c/d.csv.gz"}, set(), id="**/*.csv,**/*.gz"), + pytest.param( + ["a/**/*"], + DEFAULT_CONFIG, + { + "a/b", + "a/b.csv", + "a/b.csv.gz", + "a/b.jsonl", + "a/c", + "a/c.csv", + "a/c.csv.gz", + "a/c.jsonl", + "a/b/c", + "a/b/c.csv", + "a/b/c.csv.gz", + "a/b/c.jsonl", + "a/c/c", + "a/c/c.csv", + "a/c/c.csv.gz", + "a/c/c.jsonl", + "a/b/c/d", + "a/b/c/d.csv", + "a/b/c/d.csv.gz", + "a/b/c/d.jsonl", + }, + {"a/"}, + id="a/**/*", + ), + pytest.param( + ["a/**/*.csv"], DEFAULT_CONFIG, {"a/b.csv", "a/c.csv", "a/b/c.csv", "a/c/c.csv", "a/b/c/d.csv"}, {"a/"}, id="a/**/*.csv" + ), + pytest.param( + ["a/**/*.csv*"], + DEFAULT_CONFIG, + { + "a/b.csv", + "a/b.csv.gz", + "a/c.csv", + "a/c.csv.gz", + "a/b/c.csv", + "a/b/c.csv.gz", + "a/c/c.csv", + "a/c/c.csv.gz", + "a/b/c/d.csv", + "a/b/c/d.csv.gz", + }, + {"a/"}, + id="a/**/*.csv*", + ), + pytest.param( + ["**/*.csv", "**/*.gz"], + DEFAULT_CONFIG, + { + "a.csv", + "a.csv.gz", + "a/b.csv", + "a/b.csv.gz", + "a/c.csv", + "a/c.csv.gz", + "a/b/c.csv", + "a/b/c.csv.gz", + "a/c/c.csv", + "a/c/c.csv.gz", + "a/b/c/d.csv", + "a/b/c/d.csv.gz", + }, + set(), + id="**/*.csv,**/*.gz", + ), pytest.param(["*.csv", "*.gz"], DEFAULT_CONFIG, {"a.csv", "a.csv.gz"}, set(), id="*.csv,*.gz"), - pytest.param(["a/*.csv", "a/*/*.csv"], DEFAULT_CONFIG, {"a/b.csv", "a/c.csv", "a/b/c.csv", "a/c/c.csv"}, {"a/"}, - id="a/*.csv,a/*/*.csv"), + pytest.param( + ["a/*.csv", "a/*/*.csv"], DEFAULT_CONFIG, {"a/b.csv", "a/c.csv", "a/b/c.csv", "a/c/c.csv"}, {"a/"}, id="a/*.csv,a/*/*.csv" + ), pytest.param(["a/*.csv", "a/b/*.csv"], DEFAULT_CONFIG, {"a/b.csv", "a/c.csv", "a/b/c.csv"}, {"a/", "a/b/"}, id="a/*.csv,a/b/*.csv"), - pytest.param(["**/*.csv"], {"start_date": "2023-06-01T03:54:07.000Z", "streams": []}, - {"a.csv", "a/b.csv", "a/c.csv", "a/b/c.csv", "a/c/c.csv", "a/b/c/d.csv"}, set(), - id="all_csvs_modified_after_start_date"), - pytest.param(["**/*.csv"], {"start_date": "2023-06-10T03:54:07.000Z", "streams": []}, set(), set(), - id="all_csvs_modified_before_start_date"), - pytest.param(["**/*.csv"], {"start_date": "2023-06-05T03:54:07.000Z", "streams": []}, - {"a.csv", "a/b.csv", "a/c.csv", "a/b/c.csv", "a/c/c.csv", "a/b/c/d.csv"}, set(), - id="all_csvs_modified_exactly_on_start_date"), + pytest.param( + ["**/*.csv"], + {"start_date": "2023-06-01T03:54:07.000Z", "streams": []}, + {"a.csv", "a/b.csv", "a/c.csv", "a/b/c.csv", "a/c/c.csv", "a/b/c/d.csv"}, + set(), + id="all_csvs_modified_after_start_date", + ), + pytest.param( + ["**/*.csv"], {"start_date": "2023-06-10T03:54:07.000Z", "streams": []}, set(), set(), id="all_csvs_modified_before_start_date" + ), + pytest.param( + ["**/*.csv"], + {"start_date": "2023-06-05T03:54:07.000Z", "streams": []}, + {"a.csv", "a/b.csv", "a/c.csv", "a/b/c.csv", "a/c/c.csv", "a/b/c/d.csv"}, + set(), + id="all_csvs_modified_exactly_on_start_date", + ), ], ) -def test_globs_and_prefixes_from_globs(globs: List[str], config: Mapping[str, Any], expected_matches: Set[str], expected_path_prefixes: Set[str]) -> None: +def test_globs_and_prefixes_from_globs( + globs: List[str], config: Mapping[str, Any], expected_matches: Set[str], expected_path_prefixes: Set[str] +) -> None: reader = TestStreamReader() reader.config = TestSpec(**config) assert set([f.uri for f in reader.filter_files_by_globs_and_start_date(FILES, globs)]) == expected_matches diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/test_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/test_scenarios.py index e89e320430a6..747d22a31a1f 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/test_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/test_scenarios.py @@ -11,199 +11,16 @@ from _pytest.capture import CaptureFixture from _pytest.reports import ExceptionInfo from airbyte_cdk.entrypoint import launch -from airbyte_cdk.logger import AirbyteLogFormatter -from airbyte_cdk.models import SyncMode -from freezegun import freeze_time -from pytest import LogCaptureFixture -from unit_tests.sources.file_based.scenarios.avro_scenarios import ( - avro_all_types_scenario, - avro_file_with_double_as_number_scenario, - multiple_avro_combine_schema_scenario, - multiple_streams_avro_scenario, - single_avro_scenario, -) -from unit_tests.sources.file_based.scenarios.check_scenarios import ( - error_empty_stream_scenario, - error_listing_files_scenario, - error_multi_stream_scenario, - error_reading_file_scenario, - error_record_validation_user_provided_schema_scenario, - success_csv_scenario, - success_extensionless_scenario, - success_multi_stream_scenario, - success_user_provided_schema_scenario, -) -from unit_tests.sources.file_based.scenarios.csv_scenarios import ( - csv_autogenerate_column_names_scenario, - csv_custom_bool_values_scenario, - csv_custom_delimiter_in_double_quotes_scenario, - csv_custom_delimiter_with_escape_char_scenario, - csv_custom_format_scenario, - csv_custom_null_values_scenario, - csv_double_quote_is_set_scenario, - csv_escape_char_is_set_scenario, - csv_multi_stream_scenario, - csv_newline_in_values_not_quoted_scenario, - csv_newline_in_values_quoted_value_scenario, - csv_single_stream_scenario, - csv_skip_after_header_scenario, - csv_skip_before_and_after_header_scenario, - csv_skip_before_header_scenario, - csv_string_are_not_null_if_strings_can_be_null_is_false_scenario, - csv_string_can_be_null_with_input_schemas_scenario, - csv_string_not_null_if_no_null_values_scenario, - csv_strings_can_be_null_not_quoted_scenario, - earlier_csv_scenario, - empty_schema_inference_scenario, - invalid_csv_scenario, - multi_csv_scenario, - multi_csv_stream_n_file_exceeds_limit_for_inference, - multi_stream_custom_format, - schemaless_csv_multi_stream_scenario, - schemaless_csv_scenario, - schemaless_with_user_input_schema_fails_connection_check_multi_stream_scenario, - schemaless_with_user_input_schema_fails_connection_check_scenario, - single_csv_scenario, -) -from unit_tests.sources.file_based.scenarios.incremental_scenarios import ( - multi_csv_different_timestamps_scenario, - multi_csv_include_missing_files_within_history_range, - multi_csv_per_timestamp_scenario, - multi_csv_remove_old_files_if_history_is_full_scenario, - multi_csv_same_timestamp_more_files_than_history_size_scenario, - multi_csv_same_timestamp_scenario, - multi_csv_skip_file_if_already_in_history, - multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_different_timestamps_scenario, - multi_csv_sync_files_within_time_window_if_history_is_incomplete__different_timestamps_scenario, - multi_csv_sync_recent_files_if_history_is_incomplete_scenario, - single_csv_file_is_skipped_if_same_modified_at_as_in_history, - single_csv_file_is_synced_if_modified_at_is_more_recent_than_in_history, - single_csv_input_state_is_earlier_scenario, - single_csv_input_state_is_later_scenario, - single_csv_no_input_state_scenario, -) -from unit_tests.sources.file_based.scenarios.jsonl_scenarios import ( - invalid_jsonl_scenario, - jsonl_multi_stream_scenario, - jsonl_user_input_schema_scenario, - multi_jsonl_stream_n_bytes_exceeds_limit_for_inference, - multi_jsonl_stream_n_file_exceeds_limit_for_inference, - multi_jsonl_with_different_keys_scenario, - schemaless_jsonl_multi_stream_scenario, - schemaless_jsonl_scenario, - single_jsonl_scenario, -) -from unit_tests.sources.file_based.scenarios.parquet_scenarios import ( - multi_parquet_scenario, - parquet_file_with_decimal_as_float_scenario, - parquet_file_with_decimal_as_string_scenario, - parquet_file_with_decimal_no_config_scenario, - parquet_various_types_scenario, - parquet_with_invalid_config_scenario, - single_parquet_scenario, - single_partitioned_parquet_scenario, -) +from airbyte_cdk.models import AirbyteAnalyticsTraceMessage, SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput +from airbyte_cdk.test.entrypoint_wrapper import read as entrypoint_read +from airbyte_cdk.utils.traced_exception import AirbyteTracedException +from airbyte_protocol.models import AirbyteLogMessage, AirbyteMessage, ConfiguredAirbyteCatalog from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenario -from unit_tests.sources.file_based.scenarios.user_input_schema_scenarios import ( - multi_stream_user_input_schema_scenario_emit_nonconforming_records, - multi_stream_user_input_schema_scenario_schema_is_invalid, - multi_stream_user_input_schema_scenario_skip_nonconforming_records, - single_stream_user_input_schema_scenario_emit_nonconforming_records, - single_stream_user_input_schema_scenario_schema_is_invalid, - single_stream_user_input_schema_scenario_skip_nonconforming_records, - valid_multi_stream_user_input_schema_scenario, - valid_single_stream_user_input_schema_scenario, -) -from unit_tests.sources.file_based.scenarios.validation_policy_scenarios import ( - emit_record_scenario_multi_stream, - emit_record_scenario_single_stream, - skip_record_scenario_multi_stream, - skip_record_scenario_single_stream, - wait_for_rediscovery_scenario_multi_stream, - wait_for_rediscovery_scenario_single_stream, -) - -discover_scenarios = [ - csv_multi_stream_scenario, - csv_single_stream_scenario, - invalid_csv_scenario, - single_csv_scenario, - multi_csv_scenario, - multi_csv_stream_n_file_exceeds_limit_for_inference, - single_csv_input_state_is_earlier_scenario, - single_csv_no_input_state_scenario, - single_csv_input_state_is_later_scenario, - multi_csv_same_timestamp_scenario, - multi_csv_different_timestamps_scenario, - multi_csv_per_timestamp_scenario, - multi_csv_skip_file_if_already_in_history, - multi_csv_include_missing_files_within_history_range, - multi_csv_remove_old_files_if_history_is_full_scenario, - multi_csv_same_timestamp_more_files_than_history_size_scenario, - multi_csv_sync_recent_files_if_history_is_incomplete_scenario, - multi_csv_sync_files_within_time_window_if_history_is_incomplete__different_timestamps_scenario, - multi_csv_sync_files_within_history_time_window_if_history_is_incomplete_different_timestamps_scenario, - single_csv_file_is_skipped_if_same_modified_at_as_in_history, - single_csv_file_is_synced_if_modified_at_is_more_recent_than_in_history, - csv_custom_format_scenario, - earlier_csv_scenario, - multi_stream_custom_format, - empty_schema_inference_scenario, - single_parquet_scenario, - multi_parquet_scenario, - parquet_various_types_scenario, - parquet_file_with_decimal_no_config_scenario, - parquet_file_with_decimal_as_string_scenario, - parquet_file_with_decimal_as_float_scenario, - schemaless_csv_scenario, - schemaless_csv_multi_stream_scenario, - schemaless_with_user_input_schema_fails_connection_check_multi_stream_scenario, - schemaless_with_user_input_schema_fails_connection_check_scenario, - single_stream_user_input_schema_scenario_schema_is_invalid, - single_stream_user_input_schema_scenario_emit_nonconforming_records, - single_stream_user_input_schema_scenario_skip_nonconforming_records, - multi_stream_user_input_schema_scenario_emit_nonconforming_records, - multi_stream_user_input_schema_scenario_skip_nonconforming_records, - multi_stream_user_input_schema_scenario_schema_is_invalid, - valid_multi_stream_user_input_schema_scenario, - valid_single_stream_user_input_schema_scenario, - single_jsonl_scenario, - multi_jsonl_with_different_keys_scenario, - multi_jsonl_stream_n_file_exceeds_limit_for_inference, - multi_jsonl_stream_n_bytes_exceeds_limit_for_inference, - invalid_jsonl_scenario, - jsonl_multi_stream_scenario, - jsonl_user_input_schema_scenario, - schemaless_jsonl_scenario, - schemaless_jsonl_multi_stream_scenario, - csv_string_can_be_null_with_input_schemas_scenario, - csv_string_are_not_null_if_strings_can_be_null_is_false_scenario, - csv_string_not_null_if_no_null_values_scenario, - csv_strings_can_be_null_not_quoted_scenario, - csv_newline_in_values_quoted_value_scenario, - csv_escape_char_is_set_scenario, - csv_double_quote_is_set_scenario, - csv_custom_delimiter_with_escape_char_scenario, - csv_custom_delimiter_in_double_quotes_scenario, - csv_skip_before_header_scenario, - csv_skip_after_header_scenario, - csv_skip_before_and_after_header_scenario, - csv_custom_bool_values_scenario, - csv_custom_null_values_scenario, - single_avro_scenario, - avro_all_types_scenario, - multiple_avro_combine_schema_scenario, - multiple_streams_avro_scenario, - avro_file_with_double_as_number_scenario, - csv_newline_in_values_not_quoted_scenario, - csv_autogenerate_column_names_scenario, - parquet_with_invalid_config_scenario, - single_partitioned_parquet_scenario, -] - - -@pytest.mark.parametrize("scenario", discover_scenarios, ids=[s.name for s in discover_scenarios]) -def test_discover(capsys: CaptureFixture[str], tmp_path: PosixPath, scenario: TestScenario) -> None: + + +def verify_discover(capsys: CaptureFixture[str], tmp_path: PosixPath, scenario: TestScenario[AbstractSource]) -> None: expected_exc, expected_msg = scenario.expected_discover_error expected_logs = scenario.expected_logs if expected_exc: @@ -211,7 +28,7 @@ def test_discover(capsys: CaptureFixture[str], tmp_path: PosixPath, scenario: Te discover(capsys, tmp_path, scenario) if expected_msg: assert expected_msg in get_error_message_from_exc(exc) - else: + elif scenario.expected_catalog: output = discover(capsys, tmp_path, scenario) catalog, logs = output["catalog"], output["logs"] assert catalog == scenario.expected_catalog @@ -221,111 +38,90 @@ def test_discover(capsys: CaptureFixture[str], tmp_path: PosixPath, scenario: Te _verify_expected_logs(logs, discover_logs) -read_scenarios = discover_scenarios + [ - emit_record_scenario_multi_stream, - emit_record_scenario_single_stream, - skip_record_scenario_multi_stream, - skip_record_scenario_single_stream, - wait_for_rediscovery_scenario_multi_stream, - wait_for_rediscovery_scenario_single_stream, -] - - -@pytest.mark.parametrize("scenario", read_scenarios, ids=[s.name for s in read_scenarios]) -@freeze_time("2023-06-09T00:00:00Z") -def test_read(capsys: CaptureFixture[str], caplog: LogCaptureFixture, tmp_path: PosixPath, scenario: TestScenario) -> None: - caplog.handler.setFormatter(AirbyteLogFormatter()) +def verify_read(scenario: TestScenario[AbstractSource]) -> None: if scenario.incremental_scenario_config: - run_test_read_incremental(capsys, caplog, tmp_path, scenario) + run_test_read_incremental(scenario) else: - run_test_read_full_refresh(capsys, caplog, tmp_path, scenario) + run_test_read_full_refresh(scenario) -def run_test_read_full_refresh(capsys: CaptureFixture[str], caplog: LogCaptureFixture, tmp_path: PosixPath, scenario: TestScenario) -> None: +def run_test_read_full_refresh(scenario: TestScenario[AbstractSource]) -> None: expected_exc, expected_msg = scenario.expected_read_error + output = read(scenario) if expected_exc: - with pytest.raises(expected_exc) as exc: # noqa - read(capsys, caplog, tmp_path, scenario) + assert_exception(expected_exc, output) if expected_msg: - assert expected_msg in get_error_message_from_exc(exc) + assert expected_msg in output.errors[-1].trace.error.internal_message else: - output = read(capsys, caplog, tmp_path, scenario) _verify_read_output(output, scenario) -def run_test_read_incremental(capsys: CaptureFixture[str], caplog: LogCaptureFixture, tmp_path: PosixPath, scenario: TestScenario) -> None: +def run_test_read_incremental(scenario: TestScenario[AbstractSource]) -> None: expected_exc, expected_msg = scenario.expected_read_error + output = read_with_state(scenario) if expected_exc: - with pytest.raises(expected_exc): - read_with_state(capsys, caplog, tmp_path, scenario) + assert_exception(expected_exc, output) else: - output = read_with_state(capsys, caplog, tmp_path, scenario) _verify_read_output(output, scenario) -def _verify_read_output(output: Dict[str, Any], scenario: TestScenario) -> None: - records, logs = output["records"], output["logs"] - logs = [log for log in logs if log.get("level") in ("ERROR", "WARN", "WARNING")] +def assert_exception(expected_exception: type[BaseException], output: EntrypointOutput) -> None: + assert expected_exception.__name__ in output.errors[-1].trace.error.stack_trace + + +def _verify_read_output(output: EntrypointOutput, scenario: TestScenario[AbstractSource]) -> None: + records, log_messages = output.records_and_state_messages, output.logs + logs = [message.log for message in log_messages if message.log.level.value in scenario.log_levels] expected_records = scenario.expected_records assert len(records) == len(expected_records) for actual, expected in zip(records, expected_records): - if "record" in actual: - assert len(actual["record"]["data"]) == len(expected["data"]) - for key, value in actual["record"]["data"].items(): + if actual.record: + assert len(actual.record.data) == len(expected["data"]) + for key, value in actual.record.data.items(): if isinstance(value, float): assert math.isclose(value, expected["data"][key], abs_tol=1e-04) else: assert value == expected["data"][key] - assert actual["record"]["stream"] == expected["stream"] - elif "state" in actual: - assert actual["state"]["data"] == expected + assert actual.record.stream == expected["stream"] + elif actual.state: + assert actual.state.data == expected if scenario.expected_logs: read_logs = scenario.expected_logs.get("read") assert len(logs) == (len(read_logs) if read_logs else 0) _verify_expected_logs(logs, read_logs) + if scenario.expected_analytics: + analytics = output.analytics_messages + + _verify_analytics(analytics, scenario.expected_analytics) + + +def _verify_analytics(analytics: List[AirbyteMessage], expected_analytics: Optional[List[AirbyteAnalyticsTraceMessage]]) -> None: + if expected_analytics: + for actual, expected in zip(analytics, expected_analytics): + actual_type, actual_value = actual.trace.analytics.type, actual.trace.analytics.value + expected_type = expected.type + expected_value = expected.value + assert actual_type == expected_type + assert actual_value == expected_value -def _verify_expected_logs(logs: List[Dict[str, Any]], expected_logs: Optional[List[Mapping[str, Any]]]) -> None: + +def _verify_expected_logs(logs: List[AirbyteLogMessage], expected_logs: Optional[List[Mapping[str, Any]]]) -> None: if expected_logs: for actual, expected in zip(logs, expected_logs): - actual_level, actual_message = actual["level"], actual["message"] + actual_level, actual_message = actual.level.value, actual.message expected_level = expected["level"] expected_message = expected["message"] assert actual_level == expected_level assert expected_message in actual_message -spec_scenarios = [ - single_csv_scenario, -] - - -@pytest.mark.parametrize("scenario", spec_scenarios, ids=[c.name for c in spec_scenarios]) -def test_spec(capsys: CaptureFixture[str], scenario: TestScenario) -> None: +def verify_spec(capsys: CaptureFixture[str], scenario: TestScenario[AbstractSource]) -> None: assert spec(capsys, scenario) == scenario.expected_spec -check_scenarios = [ - error_empty_stream_scenario, - error_listing_files_scenario, - error_reading_file_scenario, - error_record_validation_user_provided_schema_scenario, - error_multi_stream_scenario, - success_csv_scenario, - success_extensionless_scenario, - success_multi_stream_scenario, - success_user_provided_schema_scenario, - schemaless_with_user_input_schema_fails_connection_check_multi_stream_scenario, - schemaless_with_user_input_schema_fails_connection_check_scenario, - valid_single_stream_user_input_schema_scenario, - single_avro_scenario, - earlier_csv_scenario, -] - - -@pytest.mark.parametrize("scenario", check_scenarios, ids=[c.name for c in check_scenarios]) -def test_check(capsys: CaptureFixture[str], tmp_path: PosixPath, scenario: TestScenario) -> None: +def verify_check(capsys: CaptureFixture[str], tmp_path: PosixPath, scenario: TestScenario[AbstractSource]) -> None: expected_exc, expected_msg = scenario.expected_check_error if expected_exc: @@ -341,7 +137,7 @@ def test_check(capsys: CaptureFixture[str], tmp_path: PosixPath, scenario: TestS assert output["status"] == scenario.expected_check_status -def spec(capsys: CaptureFixture[str], scenario: TestScenario) -> Mapping[str, Any]: +def spec(capsys: CaptureFixture[str], scenario: TestScenario[AbstractSource]) -> Mapping[str, Any]: launch( scenario.source, ["spec"], @@ -350,7 +146,7 @@ def spec(capsys: CaptureFixture[str], scenario: TestScenario) -> Mapping[str, An return json.loads(captured.out.splitlines()[0])["spec"] # type: ignore -def check(capsys: CaptureFixture[str], tmp_path: PosixPath, scenario: TestScenario) -> Dict[str, Any]: +def check(capsys: CaptureFixture[str], tmp_path: PosixPath, scenario: TestScenario[AbstractSource]) -> Dict[str, Any]: launch( scenario.source, ["check", "--config", make_file(tmp_path / "config.json", scenario.config)], @@ -359,7 +155,7 @@ def check(capsys: CaptureFixture[str], tmp_path: PosixPath, scenario: TestScenar return json.loads(captured.out.splitlines()[0])["connectionStatus"] # type: ignore -def discover(capsys: CaptureFixture[str], tmp_path: PosixPath, scenario: TestScenario) -> Dict[str, Any]: +def discover(capsys: CaptureFixture[str], tmp_path: PosixPath, scenario: TestScenario[AbstractSource]) -> Dict[str, Any]: launch( scenario.source, ["discover", "--config", make_file(tmp_path / "config.json", scenario.config)], @@ -372,48 +168,21 @@ def discover(capsys: CaptureFixture[str], tmp_path: PosixPath, scenario: TestSce } -def read(capsys: CaptureFixture[str], caplog: LogCaptureFixture, tmp_path: PosixPath, scenario: TestScenario) -> Dict[str, Any]: - with caplog.handler.stream as logger_stream: - launch( - scenario.source, - [ - "read", - "--config", - make_file(tmp_path / "config.json", scenario.config), - "--catalog", - make_file(tmp_path / "catalog.json", scenario.configured_catalog(SyncMode.full_refresh)), - ], - ) - captured = capsys.readouterr().out.splitlines() + logger_stream.getvalue().split("\n")[:-1] - - return { - "records": [msg for msg in (json.loads(line) for line in captured) if msg["type"] == "RECORD"], - "logs": [msg["log"] for msg in (json.loads(line) for line in captured) if msg["type"] == "LOG"], - } - - -def read_with_state( - capsys: CaptureFixture[str], caplog: LogCaptureFixture, tmp_path: PosixPath, scenario: TestScenario -) -> Dict[str, List[Any]]: - launch( +def read(scenario: TestScenario[AbstractSource]) -> EntrypointOutput: + return entrypoint_read( scenario.source, - [ - "read", - "--config", - make_file(tmp_path / "config.json", scenario.config), - "--catalog", - make_file(tmp_path / "catalog.json", scenario.configured_catalog(SyncMode.incremental)), - "--state", - make_file(tmp_path / "state.json", scenario.input_state()), - ], + scenario.config, + ConfiguredAirbyteCatalog.parse_obj(scenario.configured_catalog(SyncMode.full_refresh)), + ) + + +def read_with_state(scenario: TestScenario[AbstractSource]) -> EntrypointOutput: + return entrypoint_read( + scenario.source, + scenario.config, + ConfiguredAirbyteCatalog.parse_obj(scenario.configured_catalog(SyncMode.incremental)), + scenario.input_state(), ) - captured = capsys.readouterr() - logs = caplog.records - return { - "records": [msg for msg in (json.loads(line) for line in captured.out.splitlines()) if msg["type"] in ("RECORD", "STATE")], - "logs": [msg["log"] for msg in (json.loads(line) for line in captured.out.splitlines()) if msg["type"] == "LOG"] - + [{"level": log.levelname, "message": log.message} for log in logs], - } def make_file(path: Path, file_contents: Optional[Union[Mapping[str, Any], List[Mapping[str, Any]]]]) -> str: @@ -422,4 +191,6 @@ def make_file(path: Path, file_contents: Optional[Union[Mapping[str, Any], List[ def get_error_message_from_exc(exc: ExceptionInfo[Any]) -> str: + if isinstance(exc.value, AirbyteTracedException): + return exc.value.message return str(exc.value.args[0]) diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/test_schema_helpers.py b/airbyte-cdk/python/unit_tests/sources/file_based/test_schema_helpers.py index 625fdb23cf87..3292c9e41826 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/test_schema_helpers.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/test_schema_helpers.py @@ -33,7 +33,7 @@ "string_field": "val1", "array_field": [1.1, 2.2], "object_field": {"col": "val"}, - "column_x": "extra" + "column_x": "extra", } CONFORMING_WITH_MISSING_COLUMN_RECORD = { @@ -161,31 +161,19 @@ SCHEMA = { "type": "object", "properties": { - "null_field": { - "type": "null" - }, - "boolean_field": { - "type": "boolean" - }, - "integer_field": { - "type": "integer" - }, - "number_field": { - "type": "number" - }, - "string_field": { - "type": "string" - }, + "null_field": {"type": "null"}, + "boolean_field": {"type": "boolean"}, + "integer_field": {"type": "integer"}, + "number_field": {"type": "number"}, + "string_field": {"type": "string"}, "array_field": { "type": "array", "items": { "type": "number", }, }, - "object_field": { - "type": "object" - }, - } + "object_field": {"type": "object"}, + }, } @@ -203,13 +191,9 @@ pytest.param(CONFORMING_NARROWER_ARRAY_RECORD, SCHEMA, True, id="conforming-array-values-narrower-than-schema"), pytest.param(NONCONFORMING_INVALID_ARRAY_RECORD, SCHEMA, False, id="nonconforming-array-is-not-a-string"), pytest.param(NONCONFORMING_INVALID_OBJECT_RECORD, SCHEMA, False, id="nonconforming-object-is-not-a-string"), - ] + ], ) -def test_conforms_to_schema( - record: Mapping[str, Any], - schema: Mapping[str, Any], - expected_result: bool -) -> None: +def test_conforms_to_schema(record: Mapping[str, Any], schema: Mapping[str, Any], expected_result: bool) -> None: assert conforms_to_schema(record, schema) == expected_result @@ -233,15 +217,50 @@ def test_comparable_types() -> None: pytest.param({"a": {"type": "number"}}, {"a": {"type": "integer"}}, {"a": {"type": "number"}}, id="single-key-schema1-is-wider"), pytest.param({"a": {"type": "array"}}, {"a": {"type": "integer"}}, None, id="single-key-with-array-schema1"), pytest.param({"a": {"type": "integer"}}, {"a": {"type": "array"}}, None, id="single-key-with-array-schema2"), - pytest.param({"a": {"type": "object", "properties": {"b": {"type": "integer"}}}}, {"a": {"type": "object", "properties": {"b": {"type": "integer"}}}}, {"a": {"type": "object", "properties": {"b": {"type": "integer"}}}}, id="single-key-same-object"), - pytest.param({"a": {"type": "object", "properties": {"b": {"type": "integer"}}}}, {"a": {"type": "object", "properties": {"b": {"type": "string"}}}}, None, id="single-key-different-objects"), - pytest.param({"a": {"type": "object", "properties": {"b": {"type": "integer"}}}}, {"a": {"type": "number"}}, None, id="single-key-with-object-schema1"), - pytest.param({"a": {"type": "number"}}, {"a": {"type": "object", "properties": {"b": {"type": "integer"}}}}, None, id="single-key-with-object-schema2"), - pytest.param({"a": {"type": "array", "items": {"type": "number"}}}, {"a": {"type": "array", "items": {"type": "number"}}}, {"a": {"type": "array", "items": {"type": "number"}}}, id="equal-arrays-in-both-schemas"), - pytest.param({"a": {"type": "array", "items": {"type": "integer"}}}, {"a": {"type": "array", "items": {"type": "number"}}}, None, id="different-arrays-in-both-schemas"), - pytest.param({"a": {"type": "integer"}, "b": {"type": "string"}}, {"c": {"type": "number"}}, {"a": {"type": "integer"}, "b": {"type": "string"}, "c": {"type": "number"}}, id=""), + pytest.param( + {"a": {"type": "object", "properties": {"b": {"type": "integer"}}}}, + {"a": {"type": "object", "properties": {"b": {"type": "integer"}}}}, + {"a": {"type": "object", "properties": {"b": {"type": "integer"}}}}, + id="single-key-same-object", + ), + pytest.param( + {"a": {"type": "object", "properties": {"b": {"type": "integer"}}}}, + {"a": {"type": "object", "properties": {"b": {"type": "string"}}}}, + None, + id="single-key-different-objects", + ), + pytest.param( + {"a": {"type": "object", "properties": {"b": {"type": "integer"}}}}, + {"a": {"type": "number"}}, + None, + id="single-key-with-object-schema1", + ), + pytest.param( + {"a": {"type": "number"}}, + {"a": {"type": "object", "properties": {"b": {"type": "integer"}}}}, + None, + id="single-key-with-object-schema2", + ), + pytest.param( + {"a": {"type": "array", "items": {"type": "number"}}}, + {"a": {"type": "array", "items": {"type": "number"}}}, + {"a": {"type": "array", "items": {"type": "number"}}}, + id="equal-arrays-in-both-schemas", + ), + pytest.param( + {"a": {"type": "array", "items": {"type": "integer"}}}, + {"a": {"type": "array", "items": {"type": "number"}}}, + None, + id="different-arrays-in-both-schemas", + ), + pytest.param( + {"a": {"type": "integer"}, "b": {"type": "string"}}, + {"c": {"type": "number"}}, + {"a": {"type": "integer"}, "b": {"type": "string"}, "c": {"type": "number"}}, + id="", + ), pytest.param({"a": {"type": "invalid_type"}}, {"b": {"type": "integer"}}, None, id="invalid-type"), - ] + ], ) def test_merge_schemas(schema1: SchemaType, schema2: SchemaType, expected_result: Optional[SchemaType]) -> None: if expected_result is not None: @@ -259,48 +278,22 @@ def test_merge_schemas(schema1: SchemaType, schema2: SchemaType, expected_result { "type": "object", "properties": { - "col1": { - "type": "null" - }, - "col2": { - "type": "array" - }, - "col3": { - "type": "boolean" - }, - "col4": { - "type": "number" - }, - "col5": { - "type": "integer" - }, - "col6": { - "type": "number" - }, - "col7": { - "type": "object" - }, - "col8": { - "type": "string" - } - } + "col1": {"type": "null"}, + "col2": {"type": "array"}, + "col3": {"type": "boolean"}, + "col4": {"type": "number"}, + "col5": {"type": "integer"}, + "col6": {"type": "number"}, + "col7": {"type": "object"}, + "col8": {"type": "string"}, + }, }, None, - id="valid_all_types" + id="valid_all_types", ), pytest.param( '{"col1 ": " string", "col2": " integer"}', - { - "type": "object", - "properties": { - "col1": { - "type": "string" - }, - "col2": { - "type": "integer" - } - } - }, + {"type": "object", "properties": {"col1": {"type": "string"}, "col2": {"type": "integer"}}}, None, id="valid_extra_spaces", ), @@ -342,7 +335,9 @@ def test_merge_schemas(schema1: SchemaType, schema2: SchemaType, expected_result ), ], ) -def test_type_mapping_to_jsonschema(type_mapping: Mapping[str, Any], expected_schema: Optional[Mapping[str, Any]], expected_exc_msg: Optional[str]) -> None: +def test_type_mapping_to_jsonschema( + type_mapping: Mapping[str, Any], expected_schema: Optional[Mapping[str, Any]], expected_exc_msg: Optional[str] +) -> None: if expected_exc_msg: with pytest.raises(ConfigValidationError) as exc: type_mapping_to_jsonschema(type_mapping) diff --git a/airbyte-cdk/python/unit_tests/sources/fixtures/source_test_fixture.py b/airbyte-cdk/python/unit_tests/sources/fixtures/source_test_fixture.py index 101b5dced98c..a62f0e164bef 100644 --- a/airbyte-cdk/python/unit_tests/sources/fixtures/source_test_fixture.py +++ b/airbyte-cdk/python/unit_tests/sources/fixtures/source_test_fixture.py @@ -29,47 +29,50 @@ class SourceTestFixture(AbstractSource): operations. For simplicity, it also overrides functions that read from files in favor of returning the data directly avoiding the need to load static files (ex. spec.yaml, config.json, configured_catalog.json) into the unit-test package. """ + def __init__(self, streams: Optional[List[Stream]] = None, authenticator: Optional[AuthBase] = None): self._streams = streams self._authenticator = authenticator def spec(self, logger: logging.Logger) -> ConnectorSpecification: - return ConnectorSpecification(connectionSpecification={ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Test Fixture Spec", - "type": "object", - "required": ["api_token"], - "properties": { - "api_token": { - "type": "string", - "title": "API token", - "description": "The token used to authenticate requests to the API.", - "airbyte_secret": True - } + return ConnectorSpecification( + connectionSpecification={ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Test Fixture Spec", + "type": "object", + "required": ["api_token"], + "properties": { + "api_token": { + "type": "string", + "title": "API token", + "description": "The token used to authenticate requests to the API.", + "airbyte_secret": True, + } + }, } - }) + ) def read_config(self, config_path: str) -> Mapping[str, Any]: - return { - "api_token": "just_some_token" - } + return {"api_token": "just_some_token"} @classmethod def read_catalog(cls, catalog_path: str) -> ConfiguredAirbyteCatalog: - return ConfiguredAirbyteCatalog(streams=[ - ConfiguredAirbyteStream( - stream=AirbyteStream( - name="http_test_stream", - json_schema={}, - supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental], - default_cursor_field=["updated_at"], - source_defined_cursor=True, - source_defined_primary_key=[["id"]] - ), - sync_mode=SyncMode.full_refresh, - destination_sync_mode=DestinationSyncMode.overwrite, - ) - ]) + return ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream( + name="http_test_stream", + json_schema={}, + supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental], + default_cursor_field=["updated_at"], + source_defined_cursor=True, + source_defined_primary_key=[["id"]], + ), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + ] + ) def check_connection(self, *args, **kwargs) -> Tuple[bool, Optional[Any]]: return True, "" @@ -92,21 +95,21 @@ def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: return "id" def path( - self, - *, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, + self, + *, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, ) -> str: return "cast" def parse_response( - self, - response: requests.Response, - *, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, + self, + response: requests.Response, + *, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, ) -> Iterable[Mapping]: body = response.json() or {} return body["records"] @@ -127,12 +130,14 @@ def fixture_mock_send(self, request, **kwargs) -> requests.Response: response.request = request response.status_code = 200 response.headers = {"header": "value"} - response_body = {"records": [ - {"id": 1, "name": "Celine Song", "position": "director"}, - {"id": 2, "name": "Shabier Kirchner", "position": "cinematographer"}, - {"id": 3, "name": "Christopher Bear", "position": "composer"}, - {"id": 4, "name": "Daniel Rossen", "position": "composer"} - ]} + response_body = { + "records": [ + {"id": 1, "name": "Celine Song", "position": "director"}, + {"id": 2, "name": "Shabier Kirchner", "position": "cinematographer"}, + {"id": 3, "name": "Christopher Bear", "position": "composer"}, + {"id": 4, "name": "Daniel Rossen", "position": "composer"}, + ] + } response._content = json.dumps(response_body).encode("utf-8") return response @@ -141,6 +146,7 @@ class SourceFixtureOauthAuthenticator(Oauth2Authenticator): """ Test OAuth authenticator that only overrides the request and response aspect of the authenticator flow """ + def refresh_access_token(self) -> Tuple[str, int]: response = requests.request(method="POST", url=self.get_token_refresh_endpoint(), params={}) response.raise_for_status() diff --git a/airbyte-cdk/python/unit_tests/sources/message/test_repository.py b/airbyte-cdk/python/unit_tests/sources/message/test_repository.py index b8db5e08e53f..95c8f96a154d 100644 --- a/airbyte-cdk/python/unit_tests/sources/message/test_repository.py +++ b/airbyte-cdk/python/unit_tests/sources/message/test_repository.py @@ -5,15 +5,7 @@ from unittest.mock import Mock import pytest -from airbyte_cdk.models import ( - AirbyteControlConnectorConfigMessage, - AirbyteControlMessage, - AirbyteMessage, - AirbyteStateMessage, - Level, - OrchestratorType, - Type, -) +from airbyte_cdk.models import AirbyteControlConnectorConfigMessage, AirbyteControlMessage, AirbyteMessage, Level, OrchestratorType, Type from airbyte_cdk.sources.message import ( InMemoryMessageRepository, LogAppenderMessageRepositoryDecorator, @@ -27,11 +19,14 @@ emitted_at=0, connectorConfig=AirbyteControlConnectorConfigMessage(config={"a config": "value"}), ) -ANY_MESSAGE = AirbyteMessage(type=Type.CONTROL, control=AirbyteControlMessage( - type=OrchestratorType.CONNECTOR_CONFIG, - emitted_at=0, - connectorConfig=AirbyteControlConnectorConfigMessage(config={"any message": "value"}), -)) +ANY_MESSAGE = AirbyteMessage( + type=Type.CONTROL, + control=AirbyteControlMessage( + type=OrchestratorType.CONNECTOR_CONFIG, + emitted_at=0, + connectorConfig=AirbyteControlConnectorConfigMessage(config={"any message": "value"}), + ), +) ANOTHER_CONTROL = AirbyteControlMessage( type=OrchestratorType.CONNECTOR_CONFIG, emitted_at=0, @@ -71,14 +66,9 @@ def test_given_message_is_consumed_when_consume_queue_then_remove_message_from_q second_message_generator = repo.consume_queue() assert list(second_message_generator) == [second_message] - def test_given_message_is_not_control_nor_log_message_when_emit_message_then_raise_error(self): - repo = InMemoryMessageRepository() - with pytest.raises(ValueError): - repo.emit_message(AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data={"state": "state value"}))) - def test_given_log_level_is_severe_enough_when_log_message_then_allow_message_to_be_consumed(self): repo = InMemoryMessageRepository(Level.DEBUG) - repo.log_message(Level.INFO, lambda: "this is a log message") + repo.log_message(Level.INFO, lambda: {"message": "this is a log message"}) assert list(repo.consume_queue()) def test_given_log_level_is_severe_enough_when_log_message_then_filter_secrets(self, mocker): @@ -86,18 +76,18 @@ def test_given_log_level_is_severe_enough_when_log_message_then_filter_secrets(s mocker.patch("airbyte_cdk.sources.message.repository.filter_secrets", return_value=filtered_message) repo = InMemoryMessageRepository(Level.DEBUG) - repo.log_message(Level.INFO, lambda: "this is a log message") + repo.log_message(Level.INFO, lambda: {"message": "this is a log message"}) assert list(repo.consume_queue())[0].log.message == filtered_message def test_given_log_level_not_severe_enough_when_log_message_then_do_not_allow_message_to_be_consumed(self): repo = InMemoryMessageRepository(Level.ERROR) - repo.log_message(Level.INFO, lambda: "this is a log message") + repo.log_message(Level.INFO, lambda: {"message": "this is a log message"}) assert not list(repo.consume_queue()) def test_given_unknown_log_level_as_threshold_when_log_message_then_allow_message_to_be_consumed(self): repo = InMemoryMessageRepository(UNKNOWN_LEVEL) - repo.log_message(Level.DEBUG, lambda: "this is a log message") + repo.log_message(Level.DEBUG, lambda: {"message": "this is a log message"}) assert list(repo.consume_queue()) def test_given_unknown_log_level_for_log_when_log_message_then_raise_error(self): @@ -106,14 +96,14 @@ def test_given_unknown_log_level_for_log_when_log_message_then_raise_error(self) """ repo = InMemoryMessageRepository(Level.ERROR) with pytest.raises(ValidationError): - repo.log_message(UNKNOWN_LEVEL, lambda: "this is a log message") + repo.log_message(UNKNOWN_LEVEL, lambda: {"message": "this is a log message"}) class TestNoopMessageRepository: def test_given_message_emitted_when_consume_queue_then_return_empty(self): repo = NoopMessageRepository() repo.emit_message(AirbyteMessage(type=Type.CONTROL, control=A_CONTROL)) - repo.log_message(Level.INFO, lambda: "this is a log message") + repo.log_message(Level.INFO, lambda: {"message": "this is a log message"}) assert not list(repo.consume_queue()) @@ -135,10 +125,7 @@ def test_when_log_message_then_append(self, decorated): repo = LogAppenderMessageRepositoryDecorator({"a": {"dict_to_append": "appended value"}}, decorated, Level.DEBUG) repo.log_message(Level.INFO, lambda: {"a": {"original": "original value"}}) assert decorated.log_message.call_args_list[0].args[1]() == { - "a": { - "dict_to_append": "appended value", - "original": "original value" - } + "a": {"dict_to_append": "appended value", "original": "original value"} } def test_given_value_clash_when_log_message_then_overwrite_value(self, decorated): diff --git a/airbyte-integrations/connectors/source-clockify/unit_tests/__init__.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-clockify/unit_tests/__init__.py rename to airbyte-cdk/python/unit_tests/sources/streams/concurrent/__init__.py diff --git a/airbyte-integrations/connectors/source-copper/unit_tests/__init__.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-copper/unit_tests/__init__.py rename to airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/__init__.py diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py new file mode 100644 index 000000000000..72a0425bc098 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/incremental_scenarios.py @@ -0,0 +1,247 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from airbyte_cdk.sources.streams.concurrent.cursor import CursorField +from airbyte_cdk.sources.streams.concurrent.state_converters.abstract_stream_state_converter import ConcurrencyCompatibleStateType +from unit_tests.sources.file_based.scenarios.scenario_builder import IncrementalScenarioConfig, TestScenarioBuilder +from unit_tests.sources.streams.concurrent.scenarios.stream_facade_builder import StreamFacadeSourceBuilder +from unit_tests.sources.streams.concurrent.scenarios.utils import MockStream + +_NO_SLICE_BOUNDARIES = None +_NO_INPUT_STATE = [] +test_incremental_stream_without_slice_boundaries_no_input_state = ( + TestScenarioBuilder() + .set_name("test_incremental_stream_without_slice_boundaries_no_input_state") + .set_config({}) + .set_source_builder( + StreamFacadeSourceBuilder() + .set_streams( + [ + MockStream( + [ + ({"from": 0, "to": 1}, [{"id": "1", "cursor_field": 0}, {"id": "2", "cursor_field": 1}]), + ({"from": 1, "to": 2}, [{"id": "3", "cursor_field": 2}, {"id": "4", "cursor_field": 3}]), + ], + "stream1", + json_schema={ + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + ) + ] + ) + .set_incremental(CursorField("cursor_field"), _NO_SLICE_BOUNDARIES) + .set_input_state(_NO_INPUT_STATE) + ) + .set_expected_read_error(ValueError, "test exception") + .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"}) + .set_incremental_scenario_config(IncrementalScenarioConfig(input_state=_NO_INPUT_STATE)) + .build() +) + + +test_incremental_stream_with_slice_boundaries_no_input_state = ( + TestScenarioBuilder() + .set_name("test_incremental_stream_with_slice_boundaries_no_input_state") + .set_config({}) + .set_source_builder( + StreamFacadeSourceBuilder() + .set_streams( + [ + MockStream( + [ + ({"from": 0, "to": 1}, [{"id": "1", "cursor_field": 0}, {"id": "2", "cursor_field": 1}]), + ({"from": 1, "to": 2}, [{"id": "3", "cursor_field": 2}, {"id": "4", "cursor_field": 3}]), + ], + "stream1", + json_schema={ + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + ) + ] + ) + .set_incremental(CursorField("cursor_field"), ("from", "to")) + .set_input_state(_NO_INPUT_STATE) + ) + .set_expected_records( + [ + {"data": {"id": "1", "cursor_field": 0}, "stream": "stream1"}, + {"data": {"id": "2", "cursor_field": 1}, "stream": "stream1"}, + {"stream1": {"cursor_field": 1}}, + {"data": {"id": "3", "cursor_field": 2}, "stream": "stream1"}, + {"data": {"id": "4", "cursor_field": 3}, "stream": "stream1"}, + {"stream1": {"cursor_field": 2}}, + ] + ) + .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"}) + .set_incremental_scenario_config(IncrementalScenarioConfig(input_state=_NO_INPUT_STATE)) + .build() +) + + +LEGACY_STATE = [{"type": "STREAM", "stream": {"stream_state": {"cursor_field": 0}, "stream_descriptor": {"name": "stream1"}}}] +test_incremental_stream_without_slice_boundaries_with_legacy_state = ( + TestScenarioBuilder() + .set_name("test_incremental_stream_without_slice_boundaries_with_legacy_state") + .set_config({}) + .set_source_builder( + StreamFacadeSourceBuilder() + .set_streams( + [ + MockStream( + [ + ({"from": 0, "to": 1}, [{"id": "1", "cursor_field": 0}, {"id": "2", "cursor_field": 1}]), + ({"from": 1, "to": 2}, [{"id": "3", "cursor_field": 2}, {"id": "4", "cursor_field": 3}]), + ], + "stream1", + json_schema={ + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + ) + ] + ) + .set_incremental(CursorField("cursor_field"), _NO_SLICE_BOUNDARIES) + .set_input_state(LEGACY_STATE) + ) + .set_expected_read_error(ValueError, "test exception") + .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"}) + .set_incremental_scenario_config(IncrementalScenarioConfig(input_state=LEGACY_STATE)) + .build() +) + + +test_incremental_stream_with_slice_boundaries_with_legacy_state = ( + TestScenarioBuilder() + .set_name("test_incremental_stream_with_slice_boundaries_with_legacy_state") + .set_config({}) + .set_source_builder( + StreamFacadeSourceBuilder() + .set_streams( + [ + MockStream( + [ + ({"from": 0, "to": 1}, [{"id": "1", "cursor_field": 0}, {"id": "2", "cursor_field": 1}]), + ({"from": 1, "to": 2}, [{"id": "3", "cursor_field": 2}, {"id": "4", "cursor_field": 3}]), + ], + "stream1", + json_schema={ + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + ) + ] + ) + .set_incremental(CursorField("cursor_field"), ("from", "to")) + .set_input_state(LEGACY_STATE) + ) + .set_expected_records( + [ + {"data": {"id": "1", "cursor_field": 0}, "stream": "stream1"}, + {"data": {"id": "2", "cursor_field": 1}, "stream": "stream1"}, + {"stream1": {"cursor_field": 1}}, + {"data": {"id": "3", "cursor_field": 2}, "stream": "stream1"}, + {"data": {"id": "4", "cursor_field": 3}, "stream": "stream1"}, + {"stream1": {"cursor_field": 2}}, + ] + ) + .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"}) + .set_incremental_scenario_config(IncrementalScenarioConfig(input_state=LEGACY_STATE)) + .build() +) + + +CONCURRENT_STATE = [ + { + "type": "STREAM", + "stream": { + "stream_state": { + "slices": [{"start": 0, "end": 0}], + "state_type": ConcurrencyCompatibleStateType.date_range.value, + }, + "stream_descriptor": {"name": "stream1"}, + }, + }, +] +test_incremental_stream_without_slice_boundaries_with_concurrent_state = ( + TestScenarioBuilder() + .set_name("test_incremental_stream_without_slice_boundaries_with_concurrent_state") + .set_config({}) + .set_source_builder( + StreamFacadeSourceBuilder() + .set_streams( + [ + MockStream( + [ + ({"from": 0, "to": 1}, [{"id": "1", "cursor_field": 0}, {"id": "2", "cursor_field": 1}]), + ({"from": 1, "to": 2}, [{"id": "3", "cursor_field": 2}, {"id": "4", "cursor_field": 3}]), + ], + "stream1", + json_schema={ + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + ) + ] + ) + .set_incremental(CursorField("cursor_field"), _NO_SLICE_BOUNDARIES) + .set_input_state(CONCURRENT_STATE) + ) + .set_expected_read_error(ValueError, "test exception") + .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"}) + .set_incremental_scenario_config(IncrementalScenarioConfig(input_state=CONCURRENT_STATE)) + .build() +) + + +test_incremental_stream_with_slice_boundaries_with_concurrent_state = ( + TestScenarioBuilder() + .set_name("test_incremental_stream_with_slice_boundaries_with_concurrent_state") + .set_config({}) + .set_source_builder( + StreamFacadeSourceBuilder() + .set_streams( + [ + MockStream( + [ + ({"from": 0, "to": 1}, [{"id": "1", "cursor_field": 0}, {"id": "2", "cursor_field": 1}]), + ({"from": 1, "to": 2}, [{"id": "3", "cursor_field": 2}, {"id": "4", "cursor_field": 3}]), + ], + "stream1", + json_schema={ + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + ) + ] + ) + .set_incremental(CursorField("cursor_field"), ("from", "to")) + .set_input_state(CONCURRENT_STATE) + ) + .set_expected_records( + [ + {"data": {"id": "1", "cursor_field": 0}, "stream": "stream1"}, + {"data": {"id": "2", "cursor_field": 1}, "stream": "stream1"}, + {"stream1": {"cursor_field": 1}}, + {"data": {"id": "3", "cursor_field": 2}, "stream": "stream1"}, + {"data": {"id": "4", "cursor_field": 3}, "stream": "stream1"}, + {"stream1": {"cursor_field": 2}}, + ] + ) + .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"}) + .set_incremental_scenario_config(IncrementalScenarioConfig(input_state=CONCURRENT_STATE)) + .build() +) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py new file mode 100644 index 000000000000..30ec297b0b4f --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import concurrent +import logging +from typing import Any, List, Mapping, Optional, Tuple, Union + +from airbyte_cdk.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, ConnectorSpecification, DestinationSyncMode, SyncMode +from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource +from airbyte_cdk.sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter +from airbyte_cdk.sources.concurrent_source.thread_pool_manager import ThreadPoolManager +from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager +from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade +from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor, CursorField, NoopCursor +from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_state_converter import EpochValueConcurrentStreamStateConverter +from airbyte_protocol.models import ConfiguredAirbyteStream +from unit_tests.sources.file_based.scenarios.scenario_builder import SourceBuilder +from unit_tests.sources.streams.concurrent.scenarios.thread_based_concurrent_stream_source_builder import NeverLogSliceLogger + +_CURSOR_FIELD = "cursor_field" +_NO_STATE = None + + +class StreamFacadeConcurrentConnectorStateConverter(EpochValueConcurrentStreamStateConverter): + pass + + +class StreamFacadeSource(ConcurrentSourceAdapter): + def __init__( + self, + streams: List[Stream], + threadpool: concurrent.futures.ThreadPoolExecutor, + cursor_field: Optional[CursorField] = None, + cursor_boundaries: Optional[Tuple[str, str]] = None, + input_state: Optional[List[Mapping[str, Any]]] = _NO_STATE, + ): + self._message_repository = InMemoryMessageRepository() + threadpool_manager = ThreadPoolManager(threadpool, streams[0].logger) + concurrent_source = ConcurrentSource(threadpool_manager, streams[0].logger, NeverLogSliceLogger(), self._message_repository) + super().__init__(concurrent_source) + self._streams = streams + self._threadpool = threadpool_manager + self._cursor_field = cursor_field + self._cursor_boundaries = cursor_boundaries + self._state = [AirbyteStateMessage.parse_obj(s) for s in input_state] if input_state else None + + def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: + return True, None + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + state_manager = ConnectorStateManager(stream_instance_map={s.name: s for s in self._streams}, state=self._state) + state_converter = StreamFacadeConcurrentConnectorStateConverter() + stream_states = [state_manager.get_stream_state(stream.name, stream.namespace) for stream in self._streams] + return [ + StreamFacade.create_from_stream( + stream, + self, + stream.logger, + state, + ConcurrentCursor( + stream.name, + stream.namespace, + state, + self.message_repository, # type: ignore # for this source specifically, we always return `InMemoryMessageRepository` + state_manager, + state_converter, + self._cursor_field, + self._cursor_boundaries, + None, + ) + if self._cursor_field + else NoopCursor(), + ) + for stream, state in zip(self._streams, stream_states) + ] + + @property + def message_repository(self) -> Union[None, MessageRepository]: + return self._message_repository + + def spec(self, logger: logging.Logger) -> ConnectorSpecification: + return ConnectorSpecification(connectionSpecification={}) + + def read_catalog(self, catalog_path: str) -> ConfiguredAirbyteCatalog: + return ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=s.as_airbyte_stream(), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + for s in self._streams + ] + ) + + +class StreamFacadeSourceBuilder(SourceBuilder[StreamFacadeSource]): + def __init__(self): + self._source = None + self._streams = [] + self._max_workers = 1 + self._cursor_field = None + self._cursor_boundaries = None + self._input_state = None + self._raw_input_state = None + + def set_streams(self, streams: List[Stream]) -> "StreamFacadeSourceBuilder": + self._streams = streams + return self + + def set_max_workers(self, max_workers: int) -> "StreamFacadeSourceBuilder": + self._max_workers = max_workers + return self + + def set_incremental(self, cursor_field: CursorField, cursor_boundaries: Optional[Tuple[str, str]]) -> "StreamFacadeSourceBuilder": + self._cursor_field = cursor_field + self._cursor_boundaries = cursor_boundaries + return self + + def set_input_state(self, state: List[Mapping[str, Any]]) -> "StreamFacadeSourceBuilder": + self._input_state = state + return self + + def build(self, configured_catalog: Optional[Mapping[str, Any]]) -> StreamFacadeSource: + threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=self._max_workers, thread_name_prefix="workerpool") + return StreamFacadeSource(self._streams, threadpool, self._cursor_field, self._cursor_boundaries, self._input_state) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py new file mode 100644 index 000000000000..ae66d3a44374 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_scenarios.py @@ -0,0 +1,450 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from airbyte_cdk.sources.streams.concurrent.cursor import CursorField +from unit_tests.sources.file_based.scenarios.scenario_builder import IncrementalScenarioConfig, TestScenarioBuilder +from unit_tests.sources.streams.concurrent.scenarios.stream_facade_builder import StreamFacadeSourceBuilder +from unit_tests.sources.streams.concurrent.scenarios.utils import MockStream + +_stream1 = MockStream( + [ + (None, [{"id": "1"}, {"id": "2"}]), + ], + "stream1", + json_schema={ + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, +) + +_stream_raising_exception = MockStream( + [ + (None, [{"id": "1"}, ValueError("test exception")]), + ], + "stream1", + json_schema={ + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, +) + +_stream_with_primary_key = MockStream( + [ + (None, [{"id": "1"}, {"id": "2"}]), + ], + "stream1", + json_schema={ + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + primary_key="id", +) + +_stream2 = MockStream( + [ + (None, [{"id": "A"}, {"id": "B"}]), + ], + "stream2", + json_schema={ + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, +) + +_stream_with_single_slice = MockStream( + [ + ({"slice_key": "s1"}, [{"id": "1"}, {"id": "2"}]), + ], + "stream1", + json_schema={ + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, +) + +_stream_with_multiple_slices = MockStream( + [ + ({"slice_key": "s1"}, [{"id": "1"}, {"id": "2"}]), + ({"slice_key": "s2"}, [{"id": "3"}, {"id": "4"}]), + ], + "stream1", + json_schema={ + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, +) + +test_stream_facade_single_stream = ( + TestScenarioBuilder() + .set_name("test_stream_facade_single_stream") + .set_config({}) + .set_source_builder(StreamFacadeSourceBuilder().set_streams([_stream1])) + .set_expected_records( + [ + {"data": {"id": "1"}, "stream": "stream1"}, + {"data": {"id": "2"}, "stream": "stream1"}, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "json_schema": { + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + "name": "stream1", + "supported_sync_modes": ["full_refresh"], + } + ] + } + ) + .set_expected_logs( + { + "read": [ + {"level": "INFO", "message": "Starting syncing"}, + {"level": "INFO", "message": "Marking stream stream1 as STARTED"}, + {"level": "INFO", "message": "Syncing stream: stream1"}, + {"level": "INFO", "message": "Marking stream stream1 as RUNNING"}, + {"level": "INFO", "message": "Read 2 records from stream1 stream"}, + {"level": "INFO", "message": "Marking stream stream1 as STOPPED"}, + {"level": "INFO", "message": "Finished syncing stream1"}, + {"level": "INFO", "message": "Finished syncing"}, + ] + } + ) + .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"}) + .build() +) + +test_stream_facade_raises_exception = ( + TestScenarioBuilder() + .set_name("test_stream_facade_raises_exception") + .set_config({}) + .set_source_builder(StreamFacadeSourceBuilder().set_streams([_stream_raising_exception])) + .set_expected_records( + [ + {"data": {"id": "1"}, "stream": "stream1"}, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "json_schema": { + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + "name": "stream1", + "supported_sync_modes": ["full_refresh"], + } + ] + } + ) + .set_expected_read_error(ValueError, "test exception") + .build() +) + +test_stream_facade_single_stream_with_primary_key = ( + TestScenarioBuilder() + .set_name("test_stream_facade_stream_with_primary_key") + .set_config({}) + .set_source_builder(StreamFacadeSourceBuilder().set_streams([_stream1])) + .set_expected_records( + [ + {"data": {"id": "1"}, "stream": "stream1"}, + {"data": {"id": "2"}, "stream": "stream1"}, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "json_schema": { + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + "name": "stream1", + "supported_sync_modes": ["full_refresh"], + } + ] + } + ) + .build() +) + +test_stream_facade_multiple_streams = ( + TestScenarioBuilder() + .set_name("test_stream_facade_multiple_streams") + .set_config({}) + .set_source_builder(StreamFacadeSourceBuilder().set_streams([_stream1, _stream2])) + .set_expected_records( + [ + {"data": {"id": "1"}, "stream": "stream1"}, + {"data": {"id": "2"}, "stream": "stream1"}, + {"data": {"id": "A"}, "stream": "stream2"}, + {"data": {"id": "B"}, "stream": "stream2"}, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "json_schema": { + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + "name": "stream1", + "supported_sync_modes": ["full_refresh"], + }, + { + "json_schema": { + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + "name": "stream2", + "supported_sync_modes": ["full_refresh"], + }, + ] + } + ) + .build() +) + +test_stream_facade_single_stream_with_single_slice = ( + TestScenarioBuilder() + .set_name("test_stream_facade_single_stream_with_single_slice") + .set_config({}) + .set_source_builder(StreamFacadeSourceBuilder().set_streams([_stream1])) + .set_expected_records( + [ + {"data": {"id": "1"}, "stream": "stream1"}, + {"data": {"id": "2"}, "stream": "stream1"}, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "json_schema": { + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + "name": "stream1", + "supported_sync_modes": ["full_refresh"], + } + ] + } + ) + .build() +) + +test_stream_facade_single_stream_with_multiple_slices = ( + TestScenarioBuilder() + .set_name("test_stream_facade_single_stream_with_multiple_slice") + .set_config({}) + .set_source_builder(StreamFacadeSourceBuilder().set_streams([_stream_with_multiple_slices])) + .set_expected_records( + [ + {"data": {"id": "1"}, "stream": "stream1"}, + {"data": {"id": "2"}, "stream": "stream1"}, + {"data": {"id": "3"}, "stream": "stream1"}, + {"data": {"id": "4"}, "stream": "stream1"}, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "json_schema": { + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + "name": "stream1", + "supported_sync_modes": ["full_refresh"], + } + ] + } + ) + .build() +) + +test_stream_facade_single_stream_with_multiple_slices_with_concurrency_level_two = ( + TestScenarioBuilder() + .set_name("test_stream_facade_single_stream_with_multiple_slice_with_concurrency_level_two") + .set_config({}) + .set_source_builder(StreamFacadeSourceBuilder().set_streams([_stream_with_multiple_slices])) + .set_expected_records( + [ + {"data": {"id": "1"}, "stream": "stream1"}, + {"data": {"id": "2"}, "stream": "stream1"}, + {"data": {"id": "3"}, "stream": "stream1"}, + {"data": {"id": "4"}, "stream": "stream1"}, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "json_schema": { + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + "name": "stream1", + "supported_sync_modes": ["full_refresh"], + } + ] + } + ) + .build() +) + + +test_incremental_stream_with_slice_boundaries = ( + TestScenarioBuilder() + .set_name("test_incremental_stream_with_slice_boundaries") + .set_config({}) + .set_source_builder( + StreamFacadeSourceBuilder() + .set_streams( + [ + MockStream( + [ + ({"from": 0, "to": 1}, [{"id": "1", "cursor_field": 0}, {"id": "2", "cursor_field": 1}]), + ({"from": 1, "to": 2}, [{"id": "3", "cursor_field": 2}, {"id": "4", "cursor_field": 3}]), + ], + "stream1", + json_schema={ + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + ) + ] + ) + .set_incremental(CursorField("cursor_field"), ("from", "to")) + ) + .set_expected_records( + [ + {"data": {"id": "1", "cursor_field": 0}, "stream": "stream1"}, + {"data": {"id": "2", "cursor_field": 1}, "stream": "stream1"}, + {"stream1": {'cursor_field': 1}}, + {"data": {"id": "3", "cursor_field": 2}, "stream": "stream1"}, + {"data": {"id": "4", "cursor_field": 3}, "stream": "stream1"}, + {"stream1": {"cursor_field": 2}}, + ] + ) + .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"}) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[], + ) + ) + .build() +) + + +_NO_SLICE_BOUNDARIES = None +test_incremental_stream_without_slice_boundaries = ( + TestScenarioBuilder() + .set_name("test_incremental_stream_without_slice_boundaries") + .set_config({}) + .set_source_builder( + StreamFacadeSourceBuilder() + .set_streams( + [ + MockStream( + [ + (None, [{"id": "1", "cursor_field": 0}, {"id": "2", "cursor_field": 3}]), + ], + "stream1", + json_schema={ + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + ) + ] + ) + .set_incremental(CursorField("cursor_field"), _NO_SLICE_BOUNDARIES) + ) + .set_expected_records( + [ + {"data": {"id": "1", "cursor_field": 0}, "stream": "stream1"}, + {"data": {"id": "2", "cursor_field": 3}, "stream": "stream1"}, + {"stream1": {"cursor_field": 3}}, + ] + ) + .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"}) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[], + ) + ) + .build() +) + +test_incremental_stream_with_many_slices_but_without_slice_boundaries = ( + TestScenarioBuilder() + .set_name("test_incremental_stream_with_many_slices_byt_without_slice_boundaries") + .set_config({}) + .set_source_builder( + StreamFacadeSourceBuilder() + .set_streams( + [ + MockStream( + [ + ({"parent_id": 1}, [{"id": "1", "cursor_field": 0}]), + ({"parent_id": 309}, [{"id": "3", "cursor_field": 0}]), + ], + "stream1", + json_schema={ + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + ) + ] + ) + .set_incremental(CursorField("cursor_field"), _NO_SLICE_BOUNDARIES) + ) + .set_expected_read_error(ValueError, "test exception") + .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"}) + .set_incremental_scenario_config( + IncrementalScenarioConfig( + input_state=[], + ) + ) + .build() +) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py new file mode 100644 index 000000000000..af2249873035 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/test_concurrent_scenarios.py @@ -0,0 +1,76 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from pathlib import PosixPath + +import pytest +from _pytest.capture import CaptureFixture +from freezegun import freeze_time +from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenario +from unit_tests.sources.file_based.test_scenarios import verify_discover, verify_read +from unit_tests.sources.streams.concurrent.scenarios.incremental_scenarios import ( + test_incremental_stream_with_slice_boundaries_no_input_state, + test_incremental_stream_with_slice_boundaries_with_concurrent_state, + test_incremental_stream_with_slice_boundaries_with_legacy_state, + test_incremental_stream_without_slice_boundaries_no_input_state, + test_incremental_stream_without_slice_boundaries_with_concurrent_state, + test_incremental_stream_without_slice_boundaries_with_legacy_state, +) +from unit_tests.sources.streams.concurrent.scenarios.stream_facade_scenarios import ( + test_incremental_stream_with_many_slices_but_without_slice_boundaries, + test_incremental_stream_with_slice_boundaries, + test_incremental_stream_without_slice_boundaries, + test_stream_facade_multiple_streams, + test_stream_facade_raises_exception, + test_stream_facade_single_stream, + test_stream_facade_single_stream_with_multiple_slices, + test_stream_facade_single_stream_with_multiple_slices_with_concurrency_level_two, + test_stream_facade_single_stream_with_primary_key, + test_stream_facade_single_stream_with_single_slice, +) +from unit_tests.sources.streams.concurrent.scenarios.thread_based_concurrent_stream_scenarios import ( + test_concurrent_cdk_multiple_streams, + test_concurrent_cdk_partition_raises_exception, + test_concurrent_cdk_single_stream, + test_concurrent_cdk_single_stream_multiple_partitions, + test_concurrent_cdk_single_stream_multiple_partitions_concurrency_level_two, + test_concurrent_cdk_single_stream_with_primary_key, +) + +scenarios = [ + test_concurrent_cdk_single_stream, + test_concurrent_cdk_multiple_streams, + test_concurrent_cdk_single_stream_multiple_partitions, + test_concurrent_cdk_single_stream_multiple_partitions_concurrency_level_two, + test_concurrent_cdk_single_stream_with_primary_key, + test_concurrent_cdk_partition_raises_exception, + # test streams built using the facade + test_stream_facade_single_stream, + test_stream_facade_multiple_streams, + test_stream_facade_single_stream_with_primary_key, + test_stream_facade_single_stream_with_single_slice, + test_stream_facade_single_stream_with_multiple_slices, + test_stream_facade_single_stream_with_multiple_slices_with_concurrency_level_two, + test_stream_facade_raises_exception, + test_incremental_stream_with_slice_boundaries, + test_incremental_stream_without_slice_boundaries, + test_incremental_stream_with_many_slices_but_without_slice_boundaries, + test_incremental_stream_with_slice_boundaries_no_input_state, + test_incremental_stream_with_slice_boundaries_with_concurrent_state, + test_incremental_stream_with_slice_boundaries_with_legacy_state, + test_incremental_stream_without_slice_boundaries_no_input_state, + test_incremental_stream_without_slice_boundaries_with_concurrent_state, + test_incremental_stream_without_slice_boundaries_with_legacy_state, +] + + +@pytest.mark.parametrize("scenario", scenarios, ids=[s.name for s in scenarios]) +@freeze_time("2023-06-09T00:00:00Z") +def test_concurrent_read(scenario: TestScenario) -> None: + verify_read(scenario) + + +@pytest.mark.parametrize("scenario", scenarios, ids=[s.name for s in scenarios]) +def test_concurrent_discover(capsys: CaptureFixture[str], tmp_path: PosixPath, scenario: TestScenario) -> None: + verify_discover(capsys, tmp_path, scenario) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py new file mode 100644 index 000000000000..2f4ab9b9fccb --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_scenarios.py @@ -0,0 +1,408 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import logging + +from airbyte_cdk.sources.message import InMemoryMessageRepository +from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream +from airbyte_cdk.sources.streams.concurrent.partitions.record import Record +from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder +from unit_tests.sources.streams.concurrent.scenarios.thread_based_concurrent_stream_source_builder import ( + AlwaysAvailableAvailabilityStrategy, + ConcurrentSourceBuilder, + InMemoryPartition, + InMemoryPartitionGenerator, +) + +_id_only_stream = DefaultStream( + partition_generator=InMemoryPartitionGenerator( + [InMemoryPartition("partition1", "stream1", None, [Record({"id": "1"}, "stream1"), Record({"id": "2"}, "stream1")])] + ), + name="stream1", + json_schema={ + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + availability_strategy=AlwaysAvailableAvailabilityStrategy(), + primary_key=[], + cursor_field=None, + logger=logging.getLogger("test_logger"), +) + +_id_only_stream_with_slice_logger = DefaultStream( + partition_generator=InMemoryPartitionGenerator( + [InMemoryPartition("partition1", "stream1", None, [Record({"id": "1"}, "stream1"), Record({"id": "2"}, "stream1")])] + ), + name="stream1", + json_schema={ + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + availability_strategy=AlwaysAvailableAvailabilityStrategy(), + primary_key=[], + cursor_field=None, + logger=logging.getLogger("test_logger"), +) + +_id_only_stream_with_primary_key = DefaultStream( + partition_generator=InMemoryPartitionGenerator( + [InMemoryPartition("partition1", "stream1", None, [Record({"id": "1"}, "stream1"), Record({"id": "2"}, "stream1")])] + ), + name="stream1", + json_schema={ + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + availability_strategy=AlwaysAvailableAvailabilityStrategy(), + primary_key=["id"], + cursor_field=None, + logger=logging.getLogger("test_logger"), +) + +_id_only_stream_multiple_partitions = DefaultStream( + partition_generator=InMemoryPartitionGenerator( + [ + InMemoryPartition("partition1", "stream1", {"p": "1"}, [Record({"id": "1"}, "stream1"), Record({"id": "2"}, "stream1")]), + InMemoryPartition("partition2", "stream1", {"p": "2"}, [Record({"id": "3"}, "stream1"), Record({"id": "4"}, "stream1")]), + ] + ), + name="stream1", + json_schema={ + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + availability_strategy=AlwaysAvailableAvailabilityStrategy(), + primary_key=[], + cursor_field=None, + logger=logging.getLogger("test_logger"), +) + +_id_only_stream_multiple_partitions_concurrency_level_two = DefaultStream( + partition_generator=InMemoryPartitionGenerator( + [ + InMemoryPartition("partition1", "stream1", {"p": "1"}, [Record({"id": "1"}, "stream1"), Record({"id": "2"}, "stream1")]), + InMemoryPartition("partition2", "stream1", {"p": "2"}, [Record({"id": "3"}, "stream1"), Record({"id": "4"}, "stream1")]), + ] + ), + name="stream1", + json_schema={ + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + availability_strategy=AlwaysAvailableAvailabilityStrategy(), + primary_key=[], + cursor_field=None, + logger=logging.getLogger("test_logger"), +) + +_stream_raising_exception = DefaultStream( + partition_generator=InMemoryPartitionGenerator( + [InMemoryPartition("partition1", "stream1", None, [Record({"id": "1"}, "stream1"), ValueError("test exception")])] + ), + name="stream1", + json_schema={ + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + availability_strategy=AlwaysAvailableAvailabilityStrategy(), + primary_key=[], + cursor_field=None, + logger=logging.getLogger("test_logger"), +) + +test_concurrent_cdk_single_stream = ( + TestScenarioBuilder() + .set_name("test_concurrent_cdk_single_stream") + .set_config({}) + .set_source_builder( + ConcurrentSourceBuilder() + .set_streams( + [ + _id_only_stream, + ] + ) + .set_message_repository(InMemoryMessageRepository()) + ) + .set_expected_records( + [ + {"data": {"id": "1"}, "stream": "stream1"}, + {"data": {"id": "2"}, "stream": "stream1"}, + ] + ) + .set_expected_logs( + { + "read": [ + {"level": "INFO", "message": "Starting syncing"}, + {"level": "INFO", "message": "Marking stream stream1 as STARTED"}, + {"level": "INFO", "message": "Syncing stream: stream1"}, + {"level": "INFO", "message": "Marking stream stream1 as RUNNING"}, + {"level": "INFO", "message": "Read 2 records from stream1 stream"}, + {"level": "INFO", "message": "Marking stream stream1 as STOPPED"}, + {"level": "INFO", "message": "Finished syncing stream1"}, + {"level": "INFO", "message": "Finished syncing"}, + ] + } + ) + .set_log_levels({"ERROR", "WARN", "WARNING", "INFO", "DEBUG"}) + .set_expected_catalog( + { + "streams": [ + { + "json_schema": { + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + "name": "stream1", + "supported_sync_modes": ["full_refresh"], + } + ] + } + ) + .build() +) + +test_concurrent_cdk_single_stream_with_primary_key = ( + TestScenarioBuilder() + .set_name("test_concurrent_cdk_single_stream_with_primary_key") + .set_config({}) + .set_source_builder( + ConcurrentSourceBuilder() + .set_streams( + [ + _id_only_stream_with_primary_key, + ] + ) + .set_message_repository(InMemoryMessageRepository()) + ) + .set_expected_records( + [ + {"data": {"id": "1"}, "stream": "stream1"}, + {"data": {"id": "2"}, "stream": "stream1"}, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "json_schema": { + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + "name": "stream1", + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]], + } + ] + } + ) + .build() +) + +test_concurrent_cdk_multiple_streams = ( + TestScenarioBuilder() + .set_name("test_concurrent_cdk_multiple_streams") + .set_config({}) + .set_source_builder( + ConcurrentSourceBuilder() + .set_streams( + [ + _id_only_stream, + DefaultStream( + partition_generator=InMemoryPartitionGenerator( + [ + InMemoryPartition( + "partition1", + "stream2", + None, + [Record({"id": "10", "key": "v1"}, "stream2"), Record({"id": "20", "key": "v2"}, "stream2")], + ) + ] + ), + name="stream2", + json_schema={ + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + "key": {"type": ["null", "string"]}, + }, + }, + availability_strategy=AlwaysAvailableAvailabilityStrategy(), + primary_key=[], + cursor_field=None, + logger=logging.getLogger("test_logger"), + ), + ] + ) + .set_message_repository(InMemoryMessageRepository()) + ) + .set_expected_records( + [ + {"data": {"id": "1"}, "stream": "stream1"}, + {"data": {"id": "2"}, "stream": "stream1"}, + {"data": {"id": "10", "key": "v1"}, "stream": "stream2"}, + {"data": {"id": "20", "key": "v2"}, "stream": "stream2"}, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "json_schema": { + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + "name": "stream1", + "supported_sync_modes": ["full_refresh"], + }, + { + "json_schema": { + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + "key": {"type": ["null", "string"]}, + }, + }, + "name": "stream2", + "supported_sync_modes": ["full_refresh"], + }, + ] + } + ) + .build() +) + +test_concurrent_cdk_partition_raises_exception = ( + TestScenarioBuilder() + .set_name("test_concurrent_partition_raises_exception") + .set_config({}) + .set_source_builder( + ConcurrentSourceBuilder() + .set_streams( + [ + _stream_raising_exception, + ] + ) + .set_message_repository(InMemoryMessageRepository()) + ) + .set_expected_records( + [ + {"data": {"id": "1"}, "stream": "stream1"}, + ] + ) + .set_expected_read_error(ValueError, "test exception") + .set_expected_catalog( + { + "streams": [ + { + "json_schema": { + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + "name": "stream1", + "supported_sync_modes": ["full_refresh"], + } + ] + } + ) + .build() +) + +test_concurrent_cdk_single_stream_multiple_partitions = ( + TestScenarioBuilder() + .set_name("test_concurrent_cdk_single_stream_multiple_partitions") + .set_config({}) + .set_source_builder( + ConcurrentSourceBuilder() + .set_streams( + [ + _id_only_stream_multiple_partitions, + ] + ) + .set_message_repository(InMemoryMessageRepository()) + ) + .set_expected_records( + [ + {"data": {"id": "1"}, "stream": "stream1"}, + {"data": {"id": "2"}, "stream": "stream1"}, + {"data": {"id": "3"}, "stream": "stream1"}, + {"data": {"id": "4"}, "stream": "stream1"}, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "json_schema": { + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + "name": "stream1", + "supported_sync_modes": ["full_refresh"], + } + ] + } + ) + .build() +) + +test_concurrent_cdk_single_stream_multiple_partitions_concurrency_level_two = ( + TestScenarioBuilder() + .set_name("test_concurrent_cdk_single_stream_multiple_partitions_concurrency_level_2") + .set_config({}) + .set_source_builder( + ConcurrentSourceBuilder() + .set_streams( + [ + _id_only_stream_multiple_partitions_concurrency_level_two, + ] + ) + .set_message_repository(InMemoryMessageRepository()) + ) + .set_expected_records( + [ + {"data": {"id": "1"}, "stream": "stream1"}, + {"data": {"id": "2"}, "stream": "stream1"}, + {"data": {"id": "3"}, "stream": "stream1"}, + {"data": {"id": "4"}, "stream": "stream1"}, + ] + ) + .set_expected_catalog( + { + "streams": [ + { + "json_schema": { + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + }, + }, + "name": "stream1", + "supported_sync_modes": ["full_refresh"], + } + ] + } + ) + .build() +) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py new file mode 100644 index 000000000000..943aea30dbba --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py @@ -0,0 +1,141 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import json +import logging +from typing import Any, Iterable, List, Mapping, Optional, Tuple, Union + +from airbyte_cdk.models import ConfiguredAirbyteCatalog, ConnectorSpecification, DestinationSyncMode, SyncMode +from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource +from airbyte_cdk.sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter +from airbyte_cdk.sources.message import MessageRepository +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade +from airbyte_cdk.sources.streams.concurrent.availability_strategy import AbstractAvailabilityStrategy, StreamAvailability, StreamAvailable +from airbyte_cdk.sources.streams.concurrent.cursor import NoopCursor +from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream +from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition +from airbyte_cdk.sources.streams.concurrent.partitions.partition_generator import PartitionGenerator +from airbyte_cdk.sources.streams.concurrent.partitions.record import Record +from airbyte_cdk.sources.streams.core import StreamData +from airbyte_cdk.sources.utils.slice_logger import SliceLogger +from airbyte_protocol.models import ConfiguredAirbyteStream +from unit_tests.sources.file_based.scenarios.scenario_builder import SourceBuilder + + +class LegacyStream(Stream): + def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: + return None + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: Optional[List[str]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + stream_state: Optional[Mapping[str, Any]] = None, + ) -> Iterable[StreamData]: + yield from [] + + +class ConcurrentCdkSource(ConcurrentSourceAdapter): + def __init__(self, streams: List[DefaultStream], message_repository: Optional[MessageRepository], max_workers, timeout_in_seconds): + concurrent_source = ConcurrentSource.create(1, 1, streams[0]._logger, NeverLogSliceLogger(), message_repository) + super().__init__(concurrent_source) + self._streams = streams + + def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: + # Check is not verified because it is up to the source to implement this method + return True, None + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + return [StreamFacade(s, LegacyStream(), NoopCursor(), NeverLogSliceLogger(), s._logger) for s in self._streams] + + def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: + return ConnectorSpecification(connectionSpecification={}) + + def read_catalog(self, catalog_path: str) -> ConfiguredAirbyteCatalog: + return ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=StreamFacade(s, LegacyStream(), NoopCursor(), NeverLogSliceLogger(), s._logger).as_airbyte_stream(), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + for s in self._streams + ] + ) + + @property + def message_repository(self) -> Union[None, MessageRepository]: + return self._message_repository + + +class InMemoryPartitionGenerator(PartitionGenerator): + def __init__(self, partitions: List[Partition]): + self._partitions = partitions + + def generate(self) -> Iterable[Partition]: + yield from self._partitions + + +class InMemoryPartition(Partition): + def stream_name(self) -> str: + return self._stream_name + + def __init__(self, name, stream_name, _slice, records): + self._name = name + self._stream_name = stream_name + self._slice = _slice + self._records = records + self._is_closed = False + + def read(self) -> Iterable[Record]: + for record_or_exception in self._records: + if isinstance(record_or_exception, Exception): + raise record_or_exception + else: + yield record_or_exception + + def to_slice(self) -> Optional[Mapping[str, Any]]: + return self._slice + + def __hash__(self) -> int: + if self._slice: + # Convert the slice to a string so that it can be hashed + s = json.dumps(self._slice, sort_keys=True) + return hash((self._name, s)) + else: + return hash(self._name) + + def close(self) -> None: + self._is_closed = True + + def is_closed(self) -> bool: + return self._is_closed + + +class ConcurrentSourceBuilder(SourceBuilder[ConcurrentCdkSource]): + def __init__(self): + self._streams: List[DefaultStream] = [] + self._message_repository = None + + def build(self, configured_catalog: Optional[Mapping[str, Any]]) -> ConcurrentCdkSource: + return ConcurrentCdkSource(self._streams, self._message_repository, 1, 1) + + def set_streams(self, streams: List[DefaultStream]) -> "ConcurrentSourceBuilder": + self._streams = streams + return self + + def set_message_repository(self, message_repository: MessageRepository) -> "ConcurrentSourceBuilder": + self._message_repository = message_repository + return self + + +class AlwaysAvailableAvailabilityStrategy(AbstractAvailabilityStrategy): + def check_availability(self, logger: logging.Logger) -> StreamAvailability: + return StreamAvailable() + + +class NeverLogSliceLogger(SliceLogger): + def should_log_slice_message(self, logger: logging.Logger) -> bool: + return False diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/utils.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/utils.py new file mode 100644 index 000000000000..87c5234b5596 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/utils.py @@ -0,0 +1,55 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from typing import Any, Iterable, List, Mapping, Optional, Tuple, Union + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.core import StreamData + + +class MockStream(Stream): + def __init__( + self, + slices_and_records_or_exception: Iterable[Tuple[Optional[Mapping[str, Any]], Iterable[Union[Exception, Mapping[str, Any]]]]], + name, + json_schema, + primary_key=None, + ): + self._slices_and_records_or_exception = slices_and_records_or_exception + self._name = name + self._json_schema = json_schema + self._primary_key = primary_key + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: Optional[List[str]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + stream_state: Optional[Mapping[str, Any]] = None, + ) -> Iterable[StreamData]: + for _slice, records_or_exception in self._slices_and_records_or_exception: + if stream_slice == _slice: + for item in records_or_exception: + if isinstance(item, Exception): + raise item + yield item + + @property + def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: + return self._primary_key + + @property + def name(self) -> str: + return self._name + + def get_json_schema(self) -> Mapping[str, Any]: + return self._json_schema + + def stream_slices( + self, *, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + if self._slices_and_records_or_exception: + yield from [_slice for _slice, records_or_exception in self._slices_and_records_or_exception] + else: + yield None diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_adapters.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_adapters.py new file mode 100644 index 000000000000..345d3c4b09cd --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_adapters.py @@ -0,0 +1,391 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import logging +import unittest +from unittest.mock import Mock + +import pytest +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, AirbyteStream, Level, SyncMode +from airbyte_cdk.models import Type as MessageType +from airbyte_cdk.sources.message import InMemoryMessageRepository +from airbyte_cdk.sources.streams.concurrent.adapters import ( + AvailabilityStrategyFacade, + StreamAvailabilityStrategy, + StreamFacade, + StreamPartition, + StreamPartitionGenerator, +) +from airbyte_cdk.sources.streams.concurrent.availability_strategy import STREAM_AVAILABLE, StreamAvailable, StreamUnavailable +from airbyte_cdk.sources.streams.concurrent.cursor import Cursor +from airbyte_cdk.sources.streams.concurrent.exceptions import ExceptionWithDisplayMessage +from airbyte_cdk.sources.streams.concurrent.partitions.record import Record +from airbyte_cdk.sources.streams.core import Stream +from airbyte_cdk.sources.utils.slice_logger import SliceLogger +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer + +_ANY_SYNC_MODE = SyncMode.full_refresh +_ANY_STATE = {"state_key": "state_value"} +_ANY_CURSOR_FIELD = ["a", "cursor", "key"] +_STREAM_NAME = "stream" +_ANY_CURSOR = Mock(spec=Cursor) + + +@pytest.mark.parametrize( + "stream_availability, expected_available, expected_message", + [ + pytest.param(StreamAvailable(), True, None, id="test_stream_is_available"), + pytest.param(STREAM_AVAILABLE, True, None, id="test_stream_is_available_using_singleton"), + pytest.param(StreamUnavailable("message"), False, "message", id="test_stream_is_available"), + ], +) +def test_availability_strategy_facade(stream_availability, expected_available, expected_message): + strategy = Mock() + strategy.check_availability.return_value = stream_availability + facade = AvailabilityStrategyFacade(strategy) + + logger = Mock() + available, message = facade.check_availability(Mock(), logger, Mock()) + + assert available == expected_available + assert message == expected_message + + strategy.check_availability.assert_called_once_with(logger) + + +def test_stream_availability_strategy(): + stream = Mock() + source = Mock() + stream.check_availability.return_value = True, None + logger = Mock() + availability_strategy = StreamAvailabilityStrategy(stream, source) + + stream_availability = availability_strategy.check_availability(logger) + assert stream_availability.is_available() + assert stream_availability.message() is None + + stream.check_availability.assert_called_once_with(logger, source) + + +@pytest.mark.parametrize( + "sync_mode", + [ + pytest.param(SyncMode.full_refresh, id="test_full_refresh"), + pytest.param(SyncMode.incremental, id="test_incremental"), + ], +) +def test_stream_partition_generator(sync_mode): + stream = Mock() + message_repository = Mock() + stream_slices = [{"slice": 1}, {"slice": 2}] + stream.stream_slices.return_value = stream_slices + + partition_generator = StreamPartitionGenerator(stream, message_repository, _ANY_SYNC_MODE, _ANY_CURSOR_FIELD, _ANY_STATE, _ANY_CURSOR) + + partitions = list(partition_generator.generate()) + slices = [partition.to_slice() for partition in partitions] + assert slices == stream_slices + stream.stream_slices.assert_called_once_with(sync_mode=_ANY_SYNC_MODE, cursor_field=_ANY_CURSOR_FIELD, stream_state=_ANY_STATE) + + +@pytest.mark.parametrize( + "transformer, expected_records", + [ + pytest.param( + TypeTransformer(TransformConfig.NoTransform), + [Record({"data": "1"}, _STREAM_NAME), Record({"data": "2"}, _STREAM_NAME)], + id="test_no_transform", + ), + pytest.param( + TypeTransformer(TransformConfig.DefaultSchemaNormalization), + [Record({"data": 1}, _STREAM_NAME), Record({"data": 2}, _STREAM_NAME)], + id="test_default_transform", + ), + ], +) +def test_stream_partition(transformer, expected_records): + stream = Mock() + stream.name = _STREAM_NAME + stream.get_json_schema.return_value = {"type": "object", "properties": {"data": {"type": ["integer"]}}} + stream.transformer = transformer + message_repository = InMemoryMessageRepository() + _slice = None + sync_mode = SyncMode.full_refresh + cursor_field = None + state = None + partition = StreamPartition(stream, _slice, message_repository, sync_mode, cursor_field, state, _ANY_CURSOR) + + a_log_message = AirbyteMessage( + type=MessageType.LOG, + log=AirbyteLogMessage( + level=Level.INFO, + message='slice:{"partition": 1}', + ), + ) + + stream_data = [a_log_message, {"data": "1"}, {"data": "2"}] + stream.read_records.return_value = stream_data + + records = list(partition.read()) + messages = list(message_repository.consume_queue()) + + assert records == expected_records + assert messages == [a_log_message] + + +@pytest.mark.parametrize( + "exception_type, expected_display_message", + [ + pytest.param(Exception, None, id="test_exception_no_display_message"), + pytest.param(ExceptionWithDisplayMessage, "display_message", id="test_exception_no_display_message"), + ], +) +def test_stream_partition_raising_exception(exception_type, expected_display_message): + stream = Mock() + stream.get_error_display_message.return_value = expected_display_message + + message_repository = InMemoryMessageRepository() + _slice = None + + partition = StreamPartition(stream, _slice, message_repository, _ANY_SYNC_MODE, _ANY_CURSOR_FIELD, _ANY_STATE, _ANY_CURSOR) + + stream.read_records.side_effect = Exception() + + with pytest.raises(exception_type) as e: + list(partition.read()) + if isinstance(e, ExceptionWithDisplayMessage): + assert e.display_message == "display message" + + +@pytest.mark.parametrize( + "_slice, expected_hash", + [ + pytest.param({"partition": 1, "k": "v"}, hash(("stream", '{"k": "v", "partition": 1}')), id="test_hash_with_slice"), + pytest.param(None, hash("stream"), id="test_hash_no_slice"), + ], +) +def test_stream_partition_hash(_slice, expected_hash): + stream = Mock() + stream.name = "stream" + partition = StreamPartition(stream, _slice, Mock(), _ANY_SYNC_MODE, _ANY_CURSOR_FIELD, _ANY_STATE, _ANY_CURSOR) + + _hash = partition.__hash__() + assert _hash == expected_hash + + +class StreamFacadeTest(unittest.TestCase): + def setUp(self): + self._abstract_stream = Mock() + self._abstract_stream.name = "stream" + self._abstract_stream.as_airbyte_stream.return_value = AirbyteStream( + name="stream", + json_schema={"type": "object"}, + supported_sync_modes=[SyncMode.full_refresh], + ) + self._legacy_stream = Mock(spec=Stream) + self._cursor = Mock(spec=Cursor) + self._logger = Mock() + self._slice_logger = Mock() + self._slice_logger.should_log_slice_message.return_value = False + self._facade = StreamFacade(self._abstract_stream, self._legacy_stream, self._cursor, self._slice_logger, self._logger) + self._source = Mock() + + self._stream = Mock() + self._stream.primary_key = "id" + + def test_name_is_delegated_to_wrapped_stream(self): + assert self._facade.name == self._abstract_stream.name + + def test_cursor_field_is_a_string(self): + self._abstract_stream.cursor_field = "cursor_field" + assert self._facade.cursor_field == "cursor_field" + + def test_none_cursor_field_is_converted_to_an_empty_list(self): + self._abstract_stream.cursor_field = None + assert self._facade.cursor_field == [] + + def test_source_defined_cursor_is_true(self): + assert self._facade.source_defined_cursor + + def test_json_schema_is_delegated_to_wrapped_stream(self): + json_schema = {"type": "object"} + self._abstract_stream.get_json_schema.return_value = json_schema + assert self._facade.get_json_schema() == json_schema + self._abstract_stream.get_json_schema.assert_called_once_with() + + def test_given_cursor_is_noop_when_supports_incremental_then_return_legacy_stream_response(self): + assert ( + StreamFacade( + self._abstract_stream, self._legacy_stream, _ANY_CURSOR, Mock(spec=SliceLogger), Mock(spec=logging.Logger) + ).supports_incremental + == self._legacy_stream.supports_incremental + ) + + def test_given_cursor_is_not_noop_when_supports_incremental_then_return_true(self): + assert StreamFacade( + self._abstract_stream, self._legacy_stream, Mock(spec=Cursor), Mock(spec=SliceLogger), Mock(spec=logging.Logger) + ).supports_incremental + + def test_check_availability_is_delegated_to_wrapped_stream(self): + availability = StreamAvailable() + self._abstract_stream.check_availability.return_value = availability + assert self._facade.check_availability(Mock(), Mock()) == (availability.is_available(), availability.message()) + self._abstract_stream.check_availability.assert_called_once_with() + + def test_full_refresh(self): + expected_stream_data = [{"data": 1}, {"data": 2}] + records = [Record(data, "stream") for data in expected_stream_data] + + partition = Mock() + partition.read.return_value = records + self._abstract_stream.generate_partitions.return_value = [partition] + + actual_stream_data = list(self._facade.read_records(SyncMode.full_refresh, None, None, None)) + + assert actual_stream_data == expected_stream_data + + def test_read_records_full_refresh(self): + expected_stream_data = [{"data": 1}, {"data": 2}] + records = [Record(data, "stream") for data in expected_stream_data] + partition = Mock() + partition.read.return_value = records + self._abstract_stream.generate_partitions.return_value = [partition] + + actual_stream_data = list(self._facade.read_full_refresh(None, None, None)) + + assert actual_stream_data == expected_stream_data + + def test_read_records_incremental(self): + expected_stream_data = [{"data": 1}, {"data": 2}] + records = [Record(data, "stream") for data in expected_stream_data] + partition = Mock() + partition.read.return_value = records + self._abstract_stream.generate_partitions.return_value = [partition] + + actual_stream_data = list(self._facade.read_incremental(None, None, None, None, None, None, None)) + + assert actual_stream_data == expected_stream_data + + def test_create_from_stream_stream(self): + stream = Mock() + stream.name = "stream" + stream.primary_key = "id" + stream.cursor_field = "cursor" + + facade = StreamFacade.create_from_stream(stream, self._source, self._logger, _ANY_STATE, self._cursor) + + assert facade.name == "stream" + assert facade.cursor_field == "cursor" + assert facade._abstract_stream._primary_key == ["id"] + + def test_create_from_stream_stream_with_none_primary_key(self): + stream = Mock() + stream.name = "stream" + stream.primary_key = None + stream.cursor_field = [] + + facade = StreamFacade.create_from_stream(stream, self._source, self._logger, _ANY_STATE, self._cursor) + assert facade._abstract_stream._primary_key == [] + + def test_create_from_stream_with_composite_primary_key(self): + stream = Mock() + stream.name = "stream" + stream.primary_key = ["id", "name"] + stream.cursor_field = [] + + facade = StreamFacade.create_from_stream(stream, self._source, self._logger, _ANY_STATE, self._cursor) + assert facade._abstract_stream._primary_key == ["id", "name"] + + def test_create_from_stream_with_empty_list_cursor(self): + stream = Mock() + stream.primary_key = "id" + stream.cursor_field = [] + + facade = StreamFacade.create_from_stream(stream, self._source, self._logger, _ANY_STATE, self._cursor) + + assert facade.cursor_field == [] + + def test_create_from_stream_raises_exception_if_primary_key_is_nested(self): + stream = Mock() + stream.name = "stream" + stream.primary_key = [["field", "id"]] + + with self.assertRaises(ValueError): + StreamFacade.create_from_stream(stream, self._source, self._logger, _ANY_STATE, self._cursor) + + def test_create_from_stream_raises_exception_if_primary_key_has_invalid_type(self): + stream = Mock() + stream.name = "stream" + stream.primary_key = 123 + + with self.assertRaises(ValueError): + StreamFacade.create_from_stream(stream, self._source, self._logger, _ANY_STATE, self._cursor) + + def test_create_from_stream_raises_exception_if_cursor_field_is_nested(self): + stream = Mock() + stream.name = "stream" + stream.primary_key = "id" + stream.cursor_field = ["field", "cursor"] + + with self.assertRaises(ValueError): + StreamFacade.create_from_stream(stream, self._source, self._logger, _ANY_STATE, self._cursor) + + def test_create_from_stream_with_cursor_field_as_list(self): + stream = Mock() + stream.name = "stream" + stream.primary_key = "id" + stream.cursor_field = ["cursor"] + + facade = StreamFacade.create_from_stream(stream, self._source, self._logger, _ANY_STATE, self._cursor) + assert facade.cursor_field == "cursor" + + def test_create_from_stream_none_message_repository(self): + self._stream.name = "stream" + self._stream.primary_key = "id" + self._stream.cursor_field = "cursor" + self._source.message_repository = None + + with self.assertRaises(ValueError): + StreamFacade.create_from_stream(self._stream, self._source, self._logger, {}, self._cursor) + + def test_get_error_display_message_no_display_message(self): + self._stream.get_error_display_message.return_value = "display_message" + + facade = StreamFacade.create_from_stream(self._stream, self._source, self._logger, _ANY_STATE, self._cursor) + + expected_display_message = None + e = Exception() + + display_message = facade.get_error_display_message(e) + + assert expected_display_message == display_message + + def test_get_error_display_message_with_display_message(self): + self._stream.get_error_display_message.return_value = "display_message" + + facade = StreamFacade.create_from_stream(self._stream, self._source, self._logger, _ANY_STATE, self._cursor) + + expected_display_message = "display_message" + e = ExceptionWithDisplayMessage("display_message") + + display_message = facade.get_error_display_message(e) + + assert expected_display_message == display_message + + +@pytest.mark.parametrize( + "exception, expected_display_message", + [ + pytest.param(Exception("message"), None, id="test_no_display_message"), + pytest.param(ExceptionWithDisplayMessage("message"), "message", id="test_no_display_message"), + ], +) +def test_get_error_display_message(exception, expected_display_message): + stream = Mock() + legacy_stream = Mock() + cursor = Mock(spec=Cursor) + facade = StreamFacade(stream, legacy_stream, cursor, Mock().Mock(), Mock()) + + display_message = facade.get_error_display_message(exception) + + assert display_message == expected_display_message diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_partition_generator.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_partition_generator.py new file mode 100644 index 000000000000..d41b92fb8afa --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_partition_generator.py @@ -0,0 +1,33 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import Mock, call + +import pytest +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.concurrent_source.partition_generation_completed_sentinel import PartitionGenerationCompletedSentinel +from airbyte_cdk.sources.streams.concurrent.adapters import StreamPartition +from airbyte_cdk.sources.streams.concurrent.partition_enqueuer import PartitionEnqueuer +from airbyte_cdk.sources.streams.concurrent.partitions.throttled_queue import ThrottledQueue + + +@pytest.mark.parametrize( + "slices", [pytest.param([], id="test_no_partitions"), pytest.param([{"partition": 1}, {"partition": 2}], id="test_two_partitions")] +) +def test_partition_generator(slices): + queue = Mock(spec=ThrottledQueue) + partition_generator = PartitionEnqueuer(queue) + + stream = Mock() + message_repository = Mock() + sync_mode = SyncMode.full_refresh + cursor_field = None + state = None + cursor = Mock() + partitions = [StreamPartition(stream, s, message_repository, sync_mode, cursor_field, state, cursor) for s in slices] + stream.generate_partitions.return_value = iter(partitions) + + partition_generator.generate_partitions(stream) + + assert queue.put.has_calls([call(p) for p in partitions] + [call(PartitionGenerationCompletedSentinel(stream))]) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py new file mode 100644 index 000000000000..e33ce5b4df72 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py @@ -0,0 +1,674 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import logging +import unittest +from unittest.mock import Mock + +import freezegun +from airbyte_cdk.models import ( + AirbyteLogMessage, + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStream, + AirbyteStreamStatus, + AirbyteStreamStatusTraceMessage, + AirbyteTraceMessage, +) +from airbyte_cdk.models import Level as LogLevel +from airbyte_cdk.models import StreamDescriptor, SyncMode, TraceType +from airbyte_cdk.models import Type as MessageType +from airbyte_cdk.sources.concurrent_source.concurrent_read_processor import ConcurrentReadProcessor +from airbyte_cdk.sources.concurrent_source.partition_generation_completed_sentinel import PartitionGenerationCompletedSentinel +from airbyte_cdk.sources.concurrent_source.thread_pool_manager import ThreadPoolManager +from airbyte_cdk.sources.message import LogMessage, MessageRepository +from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream +from airbyte_cdk.sources.streams.concurrent.partition_enqueuer import PartitionEnqueuer +from airbyte_cdk.sources.streams.concurrent.partition_reader import PartitionReader +from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition +from airbyte_cdk.sources.streams.concurrent.partitions.record import Record +from airbyte_cdk.sources.streams.concurrent.partitions.types import PartitionCompleteSentinel +from airbyte_cdk.sources.utils.slice_logger import SliceLogger + +_STREAM_NAME = "stream" +_ANOTHER_STREAM_NAME = "stream2" + + +class TestConcurrentReadProcessor(unittest.TestCase): + def setUp(self): + self._partition_enqueuer = Mock(spec=PartitionEnqueuer) + self._thread_pool_manager = Mock(spec=ThreadPoolManager) + + self._an_open_partition = Mock(spec=Partition) + self._log_message = Mock(spec=LogMessage) + self._an_open_partition.to_slice.return_value = self._log_message + self._an_open_partition.stream_name.return_value = _STREAM_NAME + + self._a_closed_partition = Mock(spec=Partition) + self._a_closed_partition.stream_name.return_value = _ANOTHER_STREAM_NAME + + self._logger = Mock(spec=logging.Logger) + self._slice_logger = Mock(spec=SliceLogger) + self._slice_logger.create_slice_log_message.return_value = self._log_message + self._message_repository = Mock(spec=MessageRepository) + self._message_repository.consume_queue.return_value = [] + self._partition_reader = Mock(spec=PartitionReader) + + self._stream = Mock(spec=AbstractStream) + self._stream.name = _STREAM_NAME + self._stream.as_airbyte_stream.return_value = AirbyteStream( + name=_STREAM_NAME, + json_schema={}, + supported_sync_modes=[SyncMode.full_refresh], + ) + self._another_stream = Mock(spec=AbstractStream) + self._another_stream.name = _ANOTHER_STREAM_NAME + self._another_stream.as_airbyte_stream.return_value = AirbyteStream( + name=_ANOTHER_STREAM_NAME, + json_schema={}, + supported_sync_modes=[SyncMode.full_refresh], + ) + + self._record_data = {"id": 1, "value": "A"} + self._record = Mock(spec=Record) + self._record.stream_name = _STREAM_NAME + self._record.data = self._record_data + + def test_stream_is_not_done_initially(self): + stream_instances_to_read_from = [self._stream] + handler = ConcurrentReadProcessor( + stream_instances_to_read_from, + self._partition_enqueuer, + self._thread_pool_manager, + self._logger, + self._slice_logger, + self._message_repository, + self._partition_reader, + ) + assert not handler._is_stream_done(self._stream.name) + + def test_handle_partition_done_no_other_streams_to_generate_partitions_for(self): + stream_instances_to_read_from = [self._stream] + + handler = ConcurrentReadProcessor( + stream_instances_to_read_from, + self._partition_enqueuer, + self._thread_pool_manager, + self._logger, + self._slice_logger, + self._message_repository, + self._partition_reader, + ) + handler.start_next_partition_generator() + handler.on_partition(self._an_open_partition) + + sentinel = PartitionGenerationCompletedSentinel(self._stream) + messages = list(handler.on_partition_generation_completed(sentinel)) + + expected_messages = [] + assert expected_messages == messages + + @freezegun.freeze_time("2020-01-01T00:00:00") + def test_handle_last_stream_partition_done(self): + stream_instances_to_read_from = [self._another_stream] + + handler = ConcurrentReadProcessor( + stream_instances_to_read_from, + self._partition_enqueuer, + self._thread_pool_manager, + self._logger, + self._slice_logger, + self._message_repository, + self._partition_reader, + ) + handler.start_next_partition_generator() + + sentinel = PartitionGenerationCompletedSentinel(self._another_stream) + messages = handler.on_partition_generation_completed(sentinel) + + expected_messages = [ + AirbyteMessage( + type=MessageType.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1577836800000.0, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name=_ANOTHER_STREAM_NAME), + status=AirbyteStreamStatus(AirbyteStreamStatus.COMPLETE), + ), + ), + ) + ] + assert expected_messages == messages + + def test_handle_partition(self): + stream_instances_to_read_from = [self._another_stream] + + handler = ConcurrentReadProcessor( + stream_instances_to_read_from, + self._partition_enqueuer, + self._thread_pool_manager, + self._logger, + self._slice_logger, + self._message_repository, + self._partition_reader, + ) + + handler.on_partition(self._a_closed_partition) + + self._thread_pool_manager.submit.assert_called_with(self._partition_reader.process_partition, self._a_closed_partition) + assert self._a_closed_partition in handler._streams_to_running_partitions[_ANOTHER_STREAM_NAME] + + def test_handle_partition_emits_log_message_if_it_should_be_logged(self): + stream_instances_to_read_from = [self._stream] + self._slice_logger = Mock(spec=SliceLogger) + self._slice_logger.should_log_slice_message.return_value = True + self._slice_logger.create_slice_log_message.return_value = self._log_message + + handler = ConcurrentReadProcessor( + stream_instances_to_read_from, + self._partition_enqueuer, + self._thread_pool_manager, + self._logger, + self._slice_logger, + self._message_repository, + self._partition_reader, + ) + + handler.on_partition(self._an_open_partition) + + self._thread_pool_manager.submit.assert_called_with(self._partition_reader.process_partition, self._an_open_partition) + self._message_repository.emit_message.assert_called_with(self._log_message) + + assert self._an_open_partition in handler._streams_to_running_partitions[_STREAM_NAME] + + @freezegun.freeze_time("2020-01-01T00:00:00") + def test_handle_on_partition_complete_sentinel_with_messages_from_repository(self): + stream_instances_to_read_from = [self._stream] + partition = Mock(spec=Partition) + log_message = Mock(spec=LogMessage) + partition.to_slice.return_value = log_message + partition.stream_name.return_value = _STREAM_NAME + + handler = ConcurrentReadProcessor( + stream_instances_to_read_from, + self._partition_enqueuer, + self._thread_pool_manager, + self._logger, + self._slice_logger, + self._message_repository, + self._partition_reader, + ) + handler.start_next_partition_generator() + handler.on_partition(partition) + + sentinel = PartitionCompleteSentinel(partition) + + self._message_repository.consume_queue.return_value = [ + AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=LogLevel.INFO, message="message emitted from the repository")) + ] + + messages = list(handler.on_partition_complete_sentinel(sentinel)) + + expected_messages = [ + AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=LogLevel.INFO, message="message emitted from the repository")) + ] + assert expected_messages == messages + + partition.close.assert_called_once() + + @freezegun.freeze_time("2020-01-01T00:00:00") + def test_handle_on_partition_complete_sentinel_yields_status_message_if_the_stream_is_done(self): + self._streams_currently_generating_partitions = [self._another_stream] + stream_instances_to_read_from = [self._another_stream] + log_message = Mock(spec=LogMessage) + self._a_closed_partition.to_slice.return_value = log_message + self._message_repository.consume_queue.return_value = [] + + handler = ConcurrentReadProcessor( + stream_instances_to_read_from, + self._partition_enqueuer, + self._thread_pool_manager, + self._logger, + self._slice_logger, + self._message_repository, + self._partition_reader, + ) + handler.start_next_partition_generator() + handler.on_partition(self._a_closed_partition) + handler.on_partition_generation_completed(PartitionGenerationCompletedSentinel(self._another_stream)) + + sentinel = PartitionCompleteSentinel(self._a_closed_partition) + + messages = list(handler.on_partition_complete_sentinel(sentinel)) + + expected_messages = [ + AirbyteMessage( + type=MessageType.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor( + name=_ANOTHER_STREAM_NAME, + ), + status=AirbyteStreamStatus.COMPLETE, + ), + emitted_at=1577836800000.0, + ), + ) + ] + assert expected_messages == messages + self._a_closed_partition.close.assert_called_once() + + @freezegun.freeze_time("2020-01-01T00:00:00") + def test_handle_on_partition_complete_sentinel_yields_no_status_message_if_the_stream_is_not_done(self): + stream_instances_to_read_from = [self._stream] + partition = Mock(spec=Partition) + log_message = Mock(spec=LogMessage) + partition.to_slice.return_value = log_message + partition.stream_name.return_value = _STREAM_NAME + + handler = ConcurrentReadProcessor( + stream_instances_to_read_from, + self._partition_enqueuer, + self._thread_pool_manager, + self._logger, + self._slice_logger, + self._message_repository, + self._partition_reader, + ) + handler.start_next_partition_generator() + + sentinel = PartitionCompleteSentinel(partition) + + messages = list(handler.on_partition_complete_sentinel(sentinel)) + + expected_messages = [] + assert expected_messages == messages + partition.close.assert_called_once() + + @freezegun.freeze_time("2020-01-01T00:00:00") + def test_on_record_no_status_message_no_repository_messge(self): + stream_instances_to_read_from = [self._stream] + partition = Mock(spec=Partition) + log_message = Mock(spec=LogMessage) + partition.to_slice.return_value = log_message + partition.stream_name.return_value = _STREAM_NAME + self._message_repository.consume_queue.return_value = [] + + handler = ConcurrentReadProcessor( + stream_instances_to_read_from, + self._partition_enqueuer, + self._thread_pool_manager, + self._logger, + self._slice_logger, + self._message_repository, + self._partition_reader, + ) + + # Simulate a first record + list(handler.on_record(self._record)) + + messages = list(handler.on_record(self._record)) + + expected_messages = [ + AirbyteMessage( + type=MessageType.RECORD, + record=AirbyteRecordMessage( + stream=_STREAM_NAME, + data=self._record_data, + emitted_at=1577836800000, + ), + ) + ] + assert expected_messages == messages + + @freezegun.freeze_time("2020-01-01T00:00:00") + def test_on_record_with_repository_messge(self): + stream_instances_to_read_from = [self._stream] + partition = Mock(spec=Partition) + log_message = Mock(spec=LogMessage) + partition.to_slice.return_value = log_message + partition.stream_name.return_value = _STREAM_NAME + slice_logger = Mock(spec=SliceLogger) + slice_logger.should_log_slice_message.return_value = True + slice_logger.create_slice_log_message.return_value = log_message + self._message_repository.consume_queue.return_value = [ + AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=LogLevel.INFO, message="message emitted from the repository")) + ] + + handler = ConcurrentReadProcessor( + stream_instances_to_read_from, + self._partition_enqueuer, + self._thread_pool_manager, + self._logger, + self._slice_logger, + self._message_repository, + self._partition_reader, + ) + + stream = Mock(spec=AbstractStream) + stream.name = _STREAM_NAME + stream.as_airbyte_stream.return_value = AirbyteStream( + name=_STREAM_NAME, + json_schema={}, + supported_sync_modes=[SyncMode.full_refresh], + ) + + # Simulate a first record + list(handler.on_record(self._record)) + + messages = list(handler.on_record(self._record)) + + expected_messages = [ + AirbyteMessage( + type=MessageType.RECORD, + record=AirbyteRecordMessage( + stream=_STREAM_NAME, + data=self._record_data, + emitted_at=1577836800000, + ), + ), + AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=LogLevel.INFO, message="message emitted from the repository")), + ] + assert expected_messages == messages + assert handler._record_counter[_STREAM_NAME] == 2 + + @freezegun.freeze_time("2020-01-01T00:00:00") + def test_on_record_emits_status_message_on_first_record_no_repository_message(self): + self._streams_currently_generating_partitions = [_STREAM_NAME] + stream_instances_to_read_from = [self._stream] + partition = Mock(spec=Partition) + partition.stream_name.return_value = _STREAM_NAME + + handler = ConcurrentReadProcessor( + stream_instances_to_read_from, + self._partition_enqueuer, + self._thread_pool_manager, + self._logger, + self._slice_logger, + self._message_repository, + self._partition_reader, + ) + + messages = list(handler.on_record(self._record)) + + expected_messages = [ + AirbyteMessage( + type=MessageType.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1577836800000.0, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name=_STREAM_NAME), status=AirbyteStreamStatus(AirbyteStreamStatus.RUNNING) + ), + ), + ), + AirbyteMessage( + type=MessageType.RECORD, + record=AirbyteRecordMessage( + stream=_STREAM_NAME, + data=self._record_data, + emitted_at=1577836800000, + ), + ), + ] + assert expected_messages == messages + + @freezegun.freeze_time("2020-01-01T00:00:00") + def test_on_record_emits_status_message_on_first_record_with_repository_message(self): + stream_instances_to_read_from = [self._stream] + partition = Mock(spec=Partition) + log_message = Mock(spec=LogMessage) + partition.to_slice.return_value = log_message + partition.stream_name.return_value = _STREAM_NAME + self._message_repository.consume_queue.return_value = [ + AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=LogLevel.INFO, message="message emitted from the repository")) + ] + + handler = ConcurrentReadProcessor( + stream_instances_to_read_from, + self._partition_enqueuer, + self._thread_pool_manager, + self._logger, + self._slice_logger, + self._message_repository, + self._partition_reader, + ) + + stream = Mock(spec=AbstractStream) + stream.name = _STREAM_NAME + stream.as_airbyte_stream.return_value = AirbyteStream( + name=_STREAM_NAME, + json_schema={}, + supported_sync_modes=[SyncMode.full_refresh], + ) + + messages = list(handler.on_record(self._record)) + + expected_messages = [ + AirbyteMessage( + type=MessageType.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1577836800000.0, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name=_STREAM_NAME), status=AirbyteStreamStatus(AirbyteStreamStatus.RUNNING) + ), + ), + ), + AirbyteMessage( + type=MessageType.RECORD, + record=AirbyteRecordMessage( + stream=_STREAM_NAME, + data=self._record_data, + emitted_at=1577836800000, + ), + ), + AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=LogLevel.INFO, message="message emitted from the repository")), + ] + assert expected_messages == messages + + @freezegun.freeze_time("2020-01-01T00:00:00") + def test_on_exception_stops_streams_and_raises_an_exception(self): + stream_instances_to_read_from = [self._stream, self._another_stream] + + handler = ConcurrentReadProcessor( + stream_instances_to_read_from, + self._partition_enqueuer, + self._thread_pool_manager, + self._logger, + self._slice_logger, + self._message_repository, + self._partition_reader, + ) + + handler.start_next_partition_generator() + + another_stream = Mock(spec=AbstractStream) + another_stream.name = _STREAM_NAME + another_stream.as_airbyte_stream.return_value = AirbyteStream( + name=_ANOTHER_STREAM_NAME, + json_schema={}, + supported_sync_modes=[SyncMode.full_refresh], + ) + + exception = RuntimeError("Something went wrong") + + messages = [] + + with self.assertRaises(RuntimeError): + for m in handler.on_exception(exception): + messages.append(m) + + expected_message = [ + AirbyteMessage( + type=MessageType.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1577836800000.0, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name=_STREAM_NAME), status=AirbyteStreamStatus(AirbyteStreamStatus.INCOMPLETE) + ), + ), + ), + AirbyteMessage( + type=MessageType.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1577836800000.0, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name=_ANOTHER_STREAM_NAME), status=AirbyteStreamStatus(AirbyteStreamStatus.INCOMPLETE) + ), + ), + ) + ] + + assert messages == expected_message + self._thread_pool_manager.shutdown.assert_called_once() + + @freezegun.freeze_time("2020-01-01T00:00:00") + def test_on_exception_does_not_stop_streams_that_are_already_done(self): + stream_instances_to_read_from = [self._stream, self._another_stream] + + handler = ConcurrentReadProcessor( + stream_instances_to_read_from, + self._partition_enqueuer, + self._thread_pool_manager, + self._logger, + self._slice_logger, + self._message_repository, + self._partition_reader, + ) + + handler.start_next_partition_generator() + handler.on_partition(self._an_open_partition) + handler.on_partition_generation_completed(PartitionGenerationCompletedSentinel(self._stream)) + handler.on_partition_generation_completed(PartitionGenerationCompletedSentinel(self._another_stream)) + + another_stream = Mock(spec=AbstractStream) + another_stream.name = _STREAM_NAME + another_stream.as_airbyte_stream.return_value = AirbyteStream( + name=_ANOTHER_STREAM_NAME, + json_schema={}, + supported_sync_modes=[SyncMode.full_refresh], + ) + + exception = RuntimeError("Something went wrong") + + messages = [] + + with self.assertRaises(RuntimeError): + for m in handler.on_exception(exception): + messages.append(m) + + expected_message = [ + AirbyteMessage( + type=MessageType.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1577836800000.0, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name=_STREAM_NAME), status=AirbyteStreamStatus(AirbyteStreamStatus.INCOMPLETE) + ), + ), + ) + ] + + assert messages == expected_message + self._thread_pool_manager.shutdown.assert_called_once() + + def test_is_done_is_false_if_there_are_any_instances_to_read_from(self): + stream_instances_to_read_from = [self._stream] + + handler = ConcurrentReadProcessor( + stream_instances_to_read_from, + self._partition_enqueuer, + self._thread_pool_manager, + self._logger, + self._slice_logger, + self._message_repository, + self._partition_reader, + ) + + assert not handler.is_done() + + def test_is_done_is_false_if_there_are_streams_still_generating_partitions(self): + stream_instances_to_read_from = [self._stream] + + handler = ConcurrentReadProcessor( + stream_instances_to_read_from, + self._partition_enqueuer, + self._thread_pool_manager, + self._logger, + self._slice_logger, + self._message_repository, + self._partition_reader, + ) + + handler.start_next_partition_generator() + + assert not handler.is_done() + + def test_is_done_is_false_if_all_partitions_are_not_closed(self): + stream_instances_to_read_from = [self._stream] + + handler = ConcurrentReadProcessor( + stream_instances_to_read_from, + self._partition_enqueuer, + self._thread_pool_manager, + self._logger, + self._slice_logger, + self._message_repository, + self._partition_reader, + ) + + handler.start_next_partition_generator() + handler.on_partition(self._an_open_partition) + handler.on_partition_generation_completed(PartitionGenerationCompletedSentinel(self._stream)) + + assert not handler.is_done() + + def test_is_done_is_true_if_all_partitions_are_closed_and_no_streams_are_generating_partitions_and_none_are_still_to_run(self): + stream_instances_to_read_from = [] + + handler = ConcurrentReadProcessor( + stream_instances_to_read_from, + self._partition_enqueuer, + self._thread_pool_manager, + self._logger, + self._slice_logger, + self._message_repository, + self._partition_reader, + ) + + assert handler.is_done() + + @freezegun.freeze_time("2020-01-01T00:00:00") + def test_start_next_partition_generator(self): + stream_instances_to_read_from = [self._stream] + handler = ConcurrentReadProcessor( + stream_instances_to_read_from, + self._partition_enqueuer, + self._thread_pool_manager, + self._logger, + self._slice_logger, + self._message_repository, + self._partition_reader, + ) + + status_message = handler.start_next_partition_generator() + + assert status_message == AirbyteMessage( + type=MessageType.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1577836800000.0, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name=_STREAM_NAME), status=AirbyteStreamStatus(AirbyteStreamStatus.STARTED) + ), + ), + ) + + assert _STREAM_NAME in handler._streams_currently_generating_partitions + self._thread_pool_manager.submit.assert_called_with(self._partition_enqueuer.generate_partitions, self._stream) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_cursor.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_cursor.py new file mode 100644 index 000000000000..dd1246fbc2a8 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_cursor.py @@ -0,0 +1,139 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from typing import Any, Mapping, Optional +from unittest import TestCase +from unittest.mock import Mock + +import pytest +from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager +from airbyte_cdk.sources.message import MessageRepository +from airbyte_cdk.sources.streams.concurrent.cursor import Comparable, ConcurrentCursor, CursorField +from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition +from airbyte_cdk.sources.streams.concurrent.partitions.record import Record +from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_state_converter import EpochValueConcurrentStreamStateConverter + +_A_STREAM_NAME = "a stream name" +_A_STREAM_NAMESPACE = "a stream namespace" +_A_CURSOR_FIELD_KEY = "a_cursor_field_key" +_NO_PARTITION_IDENTIFIER = None +_NO_SLICE = None +_NO_SLICE_BOUNDARIES = None +_LOWER_SLICE_BOUNDARY_FIELD = "lower_boundary" +_UPPER_SLICE_BOUNDARY_FIELD = "upper_boundary" +_SLICE_BOUNDARY_FIELDS = (_LOWER_SLICE_BOUNDARY_FIELD, _UPPER_SLICE_BOUNDARY_FIELD) +_A_VERY_HIGH_CURSOR_VALUE = 1000000000 + + +def _partition(_slice: Optional[Mapping[str, Any]]) -> Partition: + partition = Mock(spec=Partition) + partition.to_slice.return_value = _slice + return partition + + +def _record(cursor_value: Comparable) -> Record: + return Record(data={_A_CURSOR_FIELD_KEY: cursor_value}, stream_name=_A_STREAM_NAME) + + +class ConcurrentCursorTest(TestCase): + def setUp(self) -> None: + self._message_repository = Mock(spec=MessageRepository) + self._state_manager = Mock(spec=ConnectorStateManager) + self._state_converter = EpochValueConcurrentStreamStateConverter() + + def _cursor_with_slice_boundary_fields(self) -> ConcurrentCursor: + return ConcurrentCursor( + _A_STREAM_NAME, + _A_STREAM_NAMESPACE, + {}, + self._message_repository, + self._state_manager, + self._state_converter, + CursorField(_A_CURSOR_FIELD_KEY), + _SLICE_BOUNDARY_FIELDS, + None, + ) + + def _cursor_without_slice_boundary_fields(self) -> ConcurrentCursor: + return ConcurrentCursor( + _A_STREAM_NAME, + _A_STREAM_NAMESPACE, + {}, + self._message_repository, + self._state_manager, + self._state_converter, + CursorField(_A_CURSOR_FIELD_KEY), + None, + None, + ) + + def test_given_boundary_fields_when_close_partition_then_emit_state(self) -> None: + cursor = self._cursor_with_slice_boundary_fields() + cursor.close_partition( + _partition( + {_LOWER_SLICE_BOUNDARY_FIELD: 12, _UPPER_SLICE_BOUNDARY_FIELD: 30}, + ) + ) + + self._message_repository.emit_message.assert_called_once_with(self._state_manager.create_state_message.return_value) + self._state_manager.update_state_for_stream.assert_called_once_with( + _A_STREAM_NAME, + _A_STREAM_NAMESPACE, + {_A_CURSOR_FIELD_KEY: 0}, # State message is updated to the legacy format before being emitted + ) + + def test_given_boundary_fields_when_close_partition_then_emit_updated_state(self) -> None: + self._cursor_with_slice_boundary_fields().close_partition( + _partition( + {_LOWER_SLICE_BOUNDARY_FIELD: 0, _UPPER_SLICE_BOUNDARY_FIELD: 30}, + ) + ) + + self._message_repository.emit_message.assert_called_once_with(self._state_manager.create_state_message.return_value) + self._state_manager.update_state_for_stream.assert_called_once_with( + _A_STREAM_NAME, + _A_STREAM_NAMESPACE, + {_A_CURSOR_FIELD_KEY: 30}, # State message is updated to the legacy format before being emitted + ) + + def test_given_boundary_fields_and_record_observed_when_close_partition_then_ignore_records(self) -> None: + cursor = self._cursor_with_slice_boundary_fields() + cursor.observe(_record(_A_VERY_HIGH_CURSOR_VALUE)) + + cursor.close_partition(_partition({_LOWER_SLICE_BOUNDARY_FIELD: 12, _UPPER_SLICE_BOUNDARY_FIELD: 30})) + + assert self._state_manager.update_state_for_stream.call_args_list[0].args[2][_A_CURSOR_FIELD_KEY] != _A_VERY_HIGH_CURSOR_VALUE + + def test_given_no_boundary_fields_when_close_partition_then_emit_state(self) -> None: + cursor = self._cursor_without_slice_boundary_fields() + cursor.observe(_record(10)) + cursor.close_partition(_partition(_NO_SLICE)) + + self._state_manager.update_state_for_stream.assert_called_once_with( + _A_STREAM_NAME, + _A_STREAM_NAMESPACE, + {'a_cursor_field_key': 10}, + ) + + def test_given_no_boundary_fields_when_close_multiple_partitions_then_raise_exception(self) -> None: + cursor = self._cursor_without_slice_boundary_fields() + cursor.observe(_record(10)) + cursor.close_partition(_partition(_NO_SLICE)) + + with pytest.raises(ValueError): + cursor.close_partition(_partition(_NO_SLICE)) + + def test_given_no_records_observed_when_close_partition_then_do_not_emit_state(self) -> None: + cursor = self._cursor_without_slice_boundary_fields() + cursor.close_partition(_partition(_NO_SLICE)) + assert self._message_repository.emit_message.call_count == 0 + + def test_given_slice_boundaries_and_no_slice_when_close_partition_then_raise_error(self) -> None: + cursor = self._cursor_with_slice_boundary_fields() + with pytest.raises(KeyError): + cursor.close_partition(_partition(_NO_SLICE)) + + def test_given_slice_boundaries_not_matching_slice_when_close_partition_then_raise_error(self) -> None: + cursor = self._cursor_with_slice_boundary_fields() + with pytest.raises(KeyError): + cursor.close_partition(_partition({"not_matching_key": "value"})) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_datetime_state_converter.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_datetime_state_converter.py new file mode 100644 index 000000000000..b516afaeef62 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_datetime_state_converter.py @@ -0,0 +1,325 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from datetime import datetime, timezone + +import pytest +from airbyte_cdk.sources.streams.concurrent.cursor import CursorField +from airbyte_cdk.sources.streams.concurrent.state_converters.abstract_stream_state_converter import ConcurrencyCompatibleStateType +from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_state_converter import ( + EpochValueConcurrentStreamStateConverter, + IsoMillisConcurrentStreamStateConverter, +) + + +@pytest.mark.parametrize( + "converter, input_state, is_compatible", + [ + pytest.param( + EpochValueConcurrentStreamStateConverter(), + {"state_type": "date-range"}, + True, + id="no-input-state-is-compatible-epoch", + ), + pytest.param( + EpochValueConcurrentStreamStateConverter(), + { + "created_at": "2022_05_22", + "state_type": ConcurrencyCompatibleStateType.date_range.value, + }, + True, + id="input-state-with-date_range-is-compatible-epoch", + ), + pytest.param( + EpochValueConcurrentStreamStateConverter(), + { + "created_at": "2022_05_22", + "state_type": "fake", + }, + False, + id="input-state-with-fake-state-type-is-not-compatible-epoch", + ), + pytest.param( + EpochValueConcurrentStreamStateConverter(), + { + "created_at": "2022_05_22", + }, + False, + id="input-state-without-state_type-is-not-compatible-epoch", + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + {"state_type": "date-range"}, + True, + id="no-input-state-is-compatible-isomillis", + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + { + "created_at": "2022_05_22", + "state_type": ConcurrencyCompatibleStateType.date_range.value, + }, + True, + id="input-state-with-date_range-is-compatible-isomillis", + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + { + "created_at": "2022_05_22", + "state_type": "fake", + }, + False, + id="input-state-with-fake-state-type-is-not-compatible-isomillis", + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + { + "created_at": "2022_05_22", + }, + False, + id="input-state-without-state_type-is-not-compatible-isomillis", + ), + ], +) +def test_concurrent_stream_state_converter_is_state_message_compatible(converter, input_state, is_compatible): + assert converter.is_state_message_compatible(input_state) == is_compatible + + +@pytest.mark.parametrize( + "converter,start,state,expected_start", + [ + pytest.param( + EpochValueConcurrentStreamStateConverter(), + None, + {}, + EpochValueConcurrentStreamStateConverter().zero_value, + id="epoch-converter-no-state-no-start-start-is-zero-value" + ), + pytest.param( + EpochValueConcurrentStreamStateConverter(), + 1617030403, + {}, + datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc), + id="epoch-converter-no-state-with-start-start-is-start" + ), + pytest.param( + EpochValueConcurrentStreamStateConverter(), + None, + {"created_at": 1617030404}, + datetime(2021, 3, 29, 15, 6, 44, tzinfo=timezone.utc), + id="epoch-converter-state-without-start-start-is-from-state" + ), + pytest.param( + EpochValueConcurrentStreamStateConverter(), + 1617030404, + {"created_at": 1617030403}, + datetime(2021, 3, 29, 15, 6, 44, tzinfo=timezone.utc), + id="epoch-converter-state-before-start-start-is-start" + ), + pytest.param( + EpochValueConcurrentStreamStateConverter(), + 1617030403, + {"created_at": 1617030404}, + datetime(2021, 3, 29, 15, 6, 44, tzinfo=timezone.utc), + id="epoch-converter-state-after-start-start-is-from-state" + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + None, + {}, + IsoMillisConcurrentStreamStateConverter().zero_value, + id="isomillis-converter-no-state-no-start-start-is-zero-value" + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + "2021-08-22T05:03:27.000Z", + {}, + datetime(2021, 8, 22, 5, 3, 27, tzinfo=timezone.utc), + id="isomillis-converter-no-state-with-start-start-is-start" + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + None, + {"created_at": "2021-08-22T05:03:27.000Z"}, + datetime(2021, 8, 22, 5, 3, 27, tzinfo=timezone.utc), + id="isomillis-converter-state-without-start-start-is-from-state" + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + "2022-08-22T05:03:27.000Z", + {"created_at": "2021-08-22T05:03:27.000Z"}, + datetime(2022, 8, 22, 5, 3, 27, tzinfo=timezone.utc), + id="isomillis-converter-state-before-start-start-is-start" + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + "2022-08-22T05:03:27.000Z", + {"created_at": "2023-08-22T05:03:27.000Z"}, + datetime(2023, 8, 22, 5, 3, 27, tzinfo=timezone.utc), + id="isomillis-converter-state-after-start-start-is-from-state" + ), + ] +) +def test_get_sync_start(converter, start, state, expected_start): + assert converter._get_sync_start(CursorField("created_at"), state, start) == expected_start + + +@pytest.mark.parametrize( + "converter, start, sequential_state, expected_output_state", + [ + pytest.param( + EpochValueConcurrentStreamStateConverter(), + 0, + {}, + { + "legacy": {}, + "slices": [{"start": EpochValueConcurrentStreamStateConverter().zero_value, + "end": EpochValueConcurrentStreamStateConverter().zero_value}], + "state_type": "date-range", + }, + id="empty-input-state-epoch", + ), + pytest.param( + EpochValueConcurrentStreamStateConverter(), + 1617030403, + {"created": 1617030403}, + { + "state_type": "date-range", + "slices": [{"start": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc), + "end": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc)}], + "legacy": {"created": 1617030403}, + }, + id="with-input-state-epoch", + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + "2020-01-01T00:00:00.000Z", + {"created": "2021-08-22T05:03:27.000Z"}, + { + "state_type": "date-range", + "slices": [{"start": datetime(2021, 8, 22, 5, 3, 27, tzinfo=timezone.utc), + "end": datetime(2021, 8, 22, 5, 3, 27, tzinfo=timezone.utc)}], + "legacy": {"created": "2021-08-22T05:03:27.000Z"}, + }, + id="with-input-state-isomillis", + ), + ], +) +def test_convert_from_sequential_state(converter, start, sequential_state, expected_output_state): + comparison_format = "%Y-%m-%dT%H:%M:%S.%f" + if expected_output_state["slices"]: + _, conversion = converter.convert_from_sequential_state(CursorField("created"), sequential_state, start) + assert conversion["state_type"] == expected_output_state["state_type"] + assert conversion["legacy"] == expected_output_state["legacy"] + for actual, expected in zip(conversion["slices"], expected_output_state["slices"]): + assert actual["start"].strftime(comparison_format) == expected["start"].strftime(comparison_format) + assert actual["end"].strftime(comparison_format) == expected["end"].strftime(comparison_format) + else: + _, conversion = converter.convert_from_sequential_state(CursorField("created"), sequential_state, start) + assert conversion == expected_output_state + + +@pytest.mark.parametrize( + "converter, concurrent_state, expected_output_state", + [ + pytest.param( + EpochValueConcurrentStreamStateConverter(), + { + "state_type": "date-range", + "slices": [{"start": datetime(1970, 1, 3, 0, 0, 0, tzinfo=timezone.utc), + "end": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc)}], + }, + {"created": 1617030403}, + id="epoch-single-slice", + ), + pytest.param( + EpochValueConcurrentStreamStateConverter(), + { + "state_type": "date-range", + "slices": [{"start": datetime(1970, 1, 1, 0, 0, 0, tzinfo=timezone.utc), + "end": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc)}, + {"start": datetime(2020, 1, 1, 0, 0, 0, tzinfo=timezone.utc), + "end": datetime(2022, 3, 29, 15, 6, 43, tzinfo=timezone.utc)}], + }, + {"created": 1648566403}, + id="epoch-overlapping-slices", + ), + pytest.param( + EpochValueConcurrentStreamStateConverter(), + { + "state_type": "date-range", + "slices": [{"start": datetime(1970, 1, 1, 0, 0, 0, tzinfo=timezone.utc), + "end": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc)}, + {"start": datetime(2022, 1, 1, 0, 0, 0, tzinfo=timezone.utc), + "end": datetime(2023, 3, 29, 15, 6, 43, tzinfo=timezone.utc)}], + }, + {"created": 1617030403}, + id="epoch-multiple-slices", + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + { + "state_type": "date-range", + "slices": [{"start": datetime(1970, 1, 3, 0, 0, 0, tzinfo=timezone.utc), + "end": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc)}], + }, + {"created": "2021-03-29T15:06:43.000Z"}, + id="isomillis-single-slice", + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + { + "state_type": "date-range", + "slices": [{"start": datetime(1970, 1, 1, 0, 0, 0, tzinfo=timezone.utc), + "end": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc)}, + {"start": datetime(2020, 1, 1, 0, 0, 0, tzinfo=timezone.utc), + "end": datetime(2022, 3, 29, 15, 6, 43, tzinfo=timezone.utc)}], + }, + {"created": "2022-03-29T15:06:43.000Z"}, + id="isomillis-overlapping-slices", + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + { + "state_type": "date-range", + "slices": [{"start": datetime(1970, 1, 1, 0, 0, 0, tzinfo=timezone.utc), + "end": datetime(2021, 3, 29, 15, 6, 43, tzinfo=timezone.utc)}, + {"start": datetime(2022, 1, 1, 0, 0, 0, tzinfo=timezone.utc), + "end": datetime(2023, 3, 29, 15, 6, 43, tzinfo=timezone.utc)}], + }, + {"created": "2021-03-29T15:06:43.000Z"}, + id="isomillis-multiple-slices", + ), + ], +) +def test_convert_to_sequential_state(converter, concurrent_state, expected_output_state): + assert converter.convert_to_sequential_state(CursorField("created"), concurrent_state) == expected_output_state + + +@pytest.mark.parametrize( + "converter, concurrent_state, expected_output_state", + [ + pytest.param( + EpochValueConcurrentStreamStateConverter(), + { + "state_type": ConcurrencyCompatibleStateType.date_range.value, + "start": EpochValueConcurrentStreamStateConverter().zero_value, + }, + {"created": 0}, + id="empty-slices-epoch", + ), + pytest.param( + IsoMillisConcurrentStreamStateConverter(), + { + "state_type": ConcurrencyCompatibleStateType.date_range.value, + "start": datetime(2021, 8, 22, 5, 3, 27, tzinfo=timezone.utc), + }, + {"created": "2021-08-22T05:03:27.000Z"}, + id="empty-slices-isomillis", + ), + ], +) +def test_convert_to_sequential_state_no_slices_returns_legacy_state(converter, concurrent_state, expected_output_state): + with pytest.raises(RuntimeError): + converter.convert_to_sequential_state(CursorField("created"), concurrent_state) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_default_stream.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_default_stream.py new file mode 100644 index 000000000000..818c2862bb8b --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_default_stream.py @@ -0,0 +1,190 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import unittest +from unittest.mock import Mock + +from airbyte_cdk.models import AirbyteStream, SyncMode +from airbyte_cdk.sources.streams.concurrent.availability_strategy import STREAM_AVAILABLE +from airbyte_cdk.sources.streams.concurrent.cursor import Cursor +from airbyte_cdk.sources.streams.concurrent.default_stream import DefaultStream + + +class ThreadBasedConcurrentStreamTest(unittest.TestCase): + def setUp(self): + self._partition_generator = Mock() + self._name = "name" + self._json_schema = {} + self._availability_strategy = Mock() + self._primary_key = [] + self._cursor_field = None + self._logger = Mock() + self._cursor = Mock(spec=Cursor) + self._stream = DefaultStream( + self._partition_generator, + self._name, + self._json_schema, + self._availability_strategy, + self._primary_key, + self._cursor_field, + self._logger, + ) + + def test_get_json_schema(self): + json_schema = self._stream.get_json_schema() + assert json_schema == self._json_schema + + def test_check_availability(self): + self._availability_strategy.check_availability.return_value = STREAM_AVAILABLE + availability = self._stream.check_availability() + assert availability == STREAM_AVAILABLE + self._availability_strategy.check_availability.assert_called_once_with(self._logger) + + def test_check_for_error_raises_an_exception_if_any_of_the_futures_are_not_done(self): + futures = [Mock() for _ in range(3)] + for f in futures: + f.exception.return_value = None + futures[0].done.return_value = False + + with self.assertRaises(Exception): + self._stream._check_for_errors(futures) + + def test_check_for_error_raises_an_exception_if_any_of_the_futures_raised_an_exception(self): + futures = [Mock() for _ in range(3)] + for f in futures: + f.exception.return_value = None + futures[0].exception.return_value = Exception("error") + + with self.assertRaises(Exception): + self._stream._check_for_errors(futures) + + def test_as_airbyte_stream(self): + expected_airbyte_stream = AirbyteStream( + name=self._name, + json_schema=self._json_schema, + supported_sync_modes=[SyncMode.full_refresh], + source_defined_cursor=None, + default_cursor_field=None, + source_defined_primary_key=None, + namespace=None, + ) + actual_airbyte_stream = self._stream.as_airbyte_stream() + + assert expected_airbyte_stream == actual_airbyte_stream + + def test_as_airbyte_stream_with_primary_key(self): + json_schema = { + "type": "object", + "properties": { + "id_a": {"type": ["null", "string"]}, + "id_b": {"type": ["null", "string"]}, + }, + } + stream = DefaultStream( + self._partition_generator, + self._name, + json_schema, + self._availability_strategy, + ["id"], + self._cursor_field, + self._logger, + ) + + expected_airbyte_stream = AirbyteStream( + name=self._name, + json_schema=json_schema, + supported_sync_modes=[SyncMode.full_refresh], + source_defined_cursor=None, + default_cursor_field=None, + source_defined_primary_key=[["id"]], + namespace=None, + ) + + airbyte_stream = stream.as_airbyte_stream() + assert expected_airbyte_stream == airbyte_stream + + def test_as_airbyte_stream_with_composite_primary_key(self): + json_schema = { + "type": "object", + "properties": { + "id_a": {"type": ["null", "string"]}, + "id_b": {"type": ["null", "string"]}, + }, + } + stream = DefaultStream( + self._partition_generator, + self._name, + json_schema, + self._availability_strategy, + ["id_a", "id_b"], + self._cursor_field, + self._logger, + ) + + expected_airbyte_stream = AirbyteStream( + name=self._name, + json_schema=json_schema, + supported_sync_modes=[SyncMode.full_refresh], + source_defined_cursor=None, + default_cursor_field=None, + source_defined_primary_key=[["id_a", "id_b"]], + namespace=None, + ) + + airbyte_stream = stream.as_airbyte_stream() + assert expected_airbyte_stream == airbyte_stream + + def test_as_airbyte_stream_with_a_cursor(self): + json_schema = { + "type": "object", + "properties": { + "id": {"type": ["null", "string"]}, + "date": {"type": ["null", "string"]}, + }, + } + stream = DefaultStream( + self._partition_generator, + self._name, + json_schema, + self._availability_strategy, + self._primary_key, + "date", + self._logger, + ) + + expected_airbyte_stream = AirbyteStream( + name=self._name, + json_schema=json_schema, + supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental], + source_defined_cursor=True, + default_cursor_field=["date"], + source_defined_primary_key=None, + namespace=None, + ) + + airbyte_stream = stream.as_airbyte_stream() + assert expected_airbyte_stream == airbyte_stream + + def test_as_airbyte_stream_with_namespace(self): + stream = DefaultStream( + self._partition_generator, + self._name, + self._json_schema, + self._availability_strategy, + self._primary_key, + self._cursor_field, + self._logger, + namespace="test", + ) + expected_airbyte_stream = AirbyteStream( + name=self._name, + json_schema=self._json_schema, + supported_sync_modes=[SyncMode.full_refresh], + source_defined_cursor=None, + default_cursor_field=None, + source_defined_primary_key=None, + namespace="test", + ) + actual_airbyte_stream = stream.as_airbyte_stream() + + assert expected_airbyte_stream == actual_airbyte_stream diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_reader.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_reader.py new file mode 100644 index 000000000000..df82432c415f --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_reader.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from queue import Queue +from unittest.mock import Mock + +from airbyte_cdk.sources.streams.concurrent.partition_reader import PartitionReader +from airbyte_cdk.sources.streams.concurrent.partitions.record import Record +from airbyte_cdk.sources.streams.concurrent.partitions.types import PartitionCompleteSentinel + + +def test_partition_reader(): + queue = Queue() + partition_reader = PartitionReader(queue) + + stream_partition = Mock() + records = [ + Record({"id": 1, "name": "Jack"}, "stream"), + Record({"id": 2, "name": "John"}, "stream"), + ] + stream_partition.read.return_value = iter(records) + + partition_reader.process_partition(stream_partition) + + actual_records = [] + while record := queue.get(): + if isinstance(record, PartitionCompleteSentinel): + break + actual_records.append(record) + + assert records == actual_records diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_thread_pool_manager.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_thread_pool_manager.py new file mode 100644 index 000000000000..db950f33f79c --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_thread_pool_manager.py @@ -0,0 +1,85 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from concurrent.futures import Future, ThreadPoolExecutor +from unittest import TestCase +from unittest.mock import Mock + +from airbyte_cdk.sources.concurrent_source.thread_pool_manager import ThreadPoolManager + +_SLEEP_TIME = 2 + + +class ThreadPoolManagerTest(TestCase): + def setUp(self): + self._threadpool = Mock(spec=ThreadPoolExecutor) + self._thread_pool_manager = ThreadPoolManager(self._threadpool, Mock(), max_concurrent_tasks=1, sleep_time=_SLEEP_TIME) + self._fn = lambda x: x + self._arg = "arg" + + def test_submit_calls_underlying_thread_pool(self): + self._thread_pool_manager.submit(self._fn, self._arg) + self._threadpool.submit.assert_called_with(self._fn, self._arg) + + assert len(self._thread_pool_manager._futures) == 1 + + def test_submit_task_previous_task_failed(self): + future = Mock(spec=Future) + future.exception.return_value = RuntimeError + future.done.side_effect = [True, True] + + self._thread_pool_manager._futures = [future] + + with self.assertRaises(RuntimeError): + self._thread_pool_manager.submit(self._fn, self._arg) + + def test_shutdown(self): + self._thread_pool_manager.shutdown() + self._threadpool.shutdown.assert_called_with(wait=False, cancel_futures=True) + + def test_is_done_is_false_if_not_all_futures_are_done(self): + future = Mock(spec=Future) + future.done.return_value = False + + self._thread_pool_manager._futures = [future] + + assert not self._thread_pool_manager.is_done() + + def test_is_done_is_true_if_all_futures_are_done(self): + future = Mock(spec=Future) + future.done.return_value = True + + self._thread_pool_manager._futures = [future] + + assert self._thread_pool_manager.is_done() + + def test_threadpool_shutdown_if_errors(self): + future = Mock(spec=Future) + future.exception.return_value = RuntimeError + + self._thread_pool_manager._futures = [future] + + with self.assertRaises(RuntimeError): + self._thread_pool_manager.check_for_errors_and_shutdown() + self._threadpool.shutdown.assert_called_with(wait=False, cancel_futures=True) + + def test_check_for_errors_and_shutdown_raises_error_if_futures_are_not_done(self): + future = Mock(spec=Future) + future.exception.return_value = None + future.done.return_value = False + + self._thread_pool_manager._futures = [future] + + with self.assertRaises(RuntimeError): + self._thread_pool_manager.check_for_errors_and_shutdown() + self._threadpool.shutdown.assert_called_with(wait=False, cancel_futures=True) + + def test_check_for_errors_and_shutdown_does_not_raise_error_if_futures_are_done(self): + future = Mock(spec=Future) + future.exception.return_value = None + future.done.return_value = True + + self._thread_pool_manager._futures = [future] + + self._thread_pool_manager.check_for_errors_and_shutdown() + self._threadpool.shutdown.assert_called_with(wait=False, cancel_futures=True) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_throttled_queue.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_throttled_queue.py new file mode 100644 index 000000000000..33e4c9f0a058 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_throttled_queue.py @@ -0,0 +1,65 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from queue import Queue +from unittest.mock import Mock + +import pytest +from _queue import Empty +from airbyte_cdk.sources.concurrent_source.throttler import Throttler +from airbyte_cdk.sources.streams.concurrent.partitions.throttled_queue import ThrottledQueue + +_AN_ITEM = Mock() + + +def test_new_throttled_queue_is_empty(): + queue = Queue() + throttler = Mock(spec=Throttler) + timeout = 100 + throttled_queue = ThrottledQueue(queue, throttler, timeout) + + assert throttled_queue.empty() + + +def test_throttled_queue_is_not_empty_after_putting_an_item(): + queue = Queue() + throttler = Mock(spec=Throttler) + timeout = 100 + throttled_queue = ThrottledQueue(queue, throttler, timeout) + + throttled_queue.put(_AN_ITEM) + + assert not throttled_queue.empty() + + +def test_throttled_queue_get_returns_item_if_any(): + queue = Queue() + throttler = Mock(spec=Throttler) + timeout = 100 + throttled_queue = ThrottledQueue(queue, throttler, timeout) + + throttled_queue.put(_AN_ITEM) + item = throttled_queue.get() + + assert item == _AN_ITEM + assert throttled_queue.empty() + + +def test_throttled_queue_blocks_for_timeout_seconds_if_no_items(): + queue = Mock(spec=Queue) + throttler = Mock(spec=Throttler) + timeout = 100 + throttled_queue = ThrottledQueue(queue, throttler, timeout) + + throttled_queue.get() + + assert queue.get.is_called_once_with(block=True, timeout=timeout) + + +def test_throttled_queue_raises_an_error_if_no_items_after_timeout(): + queue = Queue() + throttler = Mock(spec=Throttler) + timeout = 0.001 + throttled_queue = ThrottledQueue(queue, throttler, timeout) + + with pytest.raises(Empty): + throttled_queue.get() diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_throttler.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_throttler.py new file mode 100644 index 000000000000..fbe006771244 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_throttler.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from unittest.mock import patch + +from airbyte_cdk.sources.concurrent_source.throttler import Throttler + + +@patch('time.sleep', side_effect=lambda _: None) +@patch('airbyte_cdk.sources.concurrent_source.throttler.len', side_effect=[1, 1, 0]) +def test_throttler(sleep_mock, len_mock): + throttler = Throttler([], 0.1, 1) + throttler.wait_and_acquire() + assert sleep_mock.call_count == 3 diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/auth/test_auth.py b/airbyte-cdk/python/unit_tests/sources/streams/http/auth/test_auth.py index 7a664ac841a2..444031526a90 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/auth/test_auth.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/auth/test_auth.py @@ -151,13 +151,11 @@ def test_refresh_access_token_retry(self, error_code, requests_mock): TestOauth2Authenticator.refresh_endpoint, TestOauth2Authenticator.client_id, TestOauth2Authenticator.client_secret, - TestOauth2Authenticator.refresh_token + TestOauth2Authenticator.refresh_token, ) requests_mock.post( TestOauth2Authenticator.refresh_endpoint, - [ - {"status_code": error_code}, {"status_code": error_code}, {"json": {"access_token": "token", "expires_in": 10}} - ] + [{"status_code": error_code}, {"status_code": error_code}, {"json": {"access_token": "token", "expires_in": 10}}], ) token, expires_in = oauth.refresh_access_token() assert (token, expires_in) == ("token", 10) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py index df2eb08e8464..8af1199dea7e 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/requests_native_auth/test_requests_native_auth.py @@ -4,13 +4,14 @@ import json import logging +from typing import Optional, Union from unittest.mock import Mock import freezegun import pendulum import pytest import requests -from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.models import FailureType, OrchestratorType, Type from airbyte_cdk.sources.streams.http.requests_native_auth import ( BasicHttpAuthenticator, MultipleTokenAuthenticator, @@ -18,7 +19,9 @@ SingleUseRefreshTokenOauth2Authenticator, TokenAuthenticator, ) +from airbyte_cdk.utils import AirbyteTracedException from requests import Response +from requests.exceptions import RequestException LOGGER = logging.getLogger(__name__) @@ -170,8 +173,54 @@ def test_refresh_access_token(self, mocker): mocker.patch.object(resp, "json", return_value={"access_token": "access_token", "expires_in": "2000"}) token, expires_in = oauth.refresh_access_token() - assert isinstance(expires_in, int) - assert ("access_token", 2000) == (token, expires_in) + assert isinstance(expires_in, str) + assert ("access_token", "2000") == (token, expires_in) + + # Test with expires_in as str + mocker.patch.object(resp, "json", return_value={"access_token": "access_token", "expires_in": "2022-04-24T00:00:00Z"}) + token, expires_in = oauth.refresh_access_token() + + assert isinstance(expires_in, str) + assert ("access_token", "2022-04-24T00:00:00Z") == (token, expires_in) + + @pytest.mark.parametrize( + "expires_in_response, token_expiry_date_format, expected_token_expiry_date", + [ + (3600, None, pendulum.datetime(year=2022, month=1, day=1, hour=1)), + ("90012", None, pendulum.datetime(year=2022, month=1, day=2, hour=1, second=12)), + ("2024-02-28", "YYYY-MM-DD", pendulum.datetime(year=2024, month=2, day=28)), + ("2022-02-12T00:00:00.000000+00:00", "YYYY-MM-DDTHH:mm:ss.SSSSSSZ", pendulum.datetime(year=2022, month=2, day=12)), + ], + ids=["seconds", "string_of_seconds", "simple_date", "simple_datetime"], + ) + @freezegun.freeze_time("2022-01-01") + def test_parse_refresh_token_lifespan( + self, + mocker, + expires_in_response: Union[str, int], + token_expiry_date_format: Optional[str], + expected_token_expiry_date: pendulum.DateTime, + ): + oauth = Oauth2Authenticator( + token_refresh_endpoint="refresh_end", + client_id="some_client_id", + client_secret="some_client_secret", + refresh_token="some_refresh_token", + scopes=["scope1", "scope2"], + token_expiry_date=pendulum.now().subtract(days=3), + token_expiry_date_format=token_expiry_date_format, + token_expiry_is_time_of_expiration=bool(token_expiry_date_format), + refresh_request_body={"custom_field": "in_outbound_request", "another_field": "exists_in_body", "scopes": ["no_override"]}, + ) + + resp.status_code = 200 + mocker.patch.object(resp, "json", return_value={"access_token": "access_token", "expires_in": expires_in_response}) + mocker.patch.object(requests, "request", side_effect=mock_request, autospec=True) + token, expire_in = oauth.refresh_access_token() + expires_datetime = oauth._parse_token_expiration_date(expire_in) + + assert isinstance(expires_datetime, pendulum.DateTime) + assert ("access_token", expected_token_expiry_date) == (token, expires_datetime) @pytest.mark.parametrize("error_code", (429, 500, 502, 504)) def test_refresh_access_token_retry(self, error_code, requests_mock): @@ -179,13 +228,11 @@ def test_refresh_access_token_retry(self, error_code, requests_mock): f"https://{TestOauth2Authenticator.refresh_endpoint}", TestOauth2Authenticator.client_id, TestOauth2Authenticator.client_secret, - TestOauth2Authenticator.refresh_token + TestOauth2Authenticator.refresh_token, ) requests_mock.post( f"https://{TestOauth2Authenticator.refresh_endpoint}", - [ - {"status_code": error_code}, {"status_code": error_code}, {"json": {"access_token": "token", "expires_in": 10}} - ] + [{"status_code": error_code}, {"status_code": error_code}, {"json": {"access_token": "token", "expires_in": 10}}], ) token, expires_in = oauth.refresh_access_token() assert isinstance(expires_in, int) @@ -207,6 +254,41 @@ def test_auth_call_method(self, mocker): assert {"Authorization": "Bearer access_token"} == prepared_request.headers + @pytest.mark.parametrize( + ("config_codes", "response_code", "config_key", "response_key", "config_values", "response_value", "wrapped"), + ( + ((400,), 400, "error", "error", ("invalid_grant",), "invalid_grant", True), + ((401,), 400, "error", "error", ("invalid_grant",), "invalid_grant", False), + ((400,), 400, "error_key", "error", ("invalid_grant",), "invalid_grant", False), + ((400,), 400, "error", "error", ("invalid_grant",), "valid_grant", False), + ((), 400, "", "error", (), "valid_grant", False), + ), + ) + def test_refresh_access_token_wrapped( + self, requests_mock, config_codes, response_code, config_key, response_key, config_values, response_value, wrapped + ): + oauth = Oauth2Authenticator( + f"https://{TestOauth2Authenticator.refresh_endpoint}", + TestOauth2Authenticator.client_id, + TestOauth2Authenticator.client_secret, + TestOauth2Authenticator.refresh_token, + refresh_token_error_status_codes=config_codes, + refresh_token_error_key=config_key, + refresh_token_error_values=config_values, + ) + error_content = {response_key: response_value} + requests_mock.post(f"https://{TestOauth2Authenticator.refresh_endpoint}", status_code=response_code, json=error_content) + + exception_to_raise = AirbyteTracedException if wrapped else RequestException + with pytest.raises(exception_to_raise) as exc_info: + oauth.refresh_access_token() + + if wrapped: + error_message = "Refresh token is invalid or expired. Please re-authenticate from Sources//Settings." + assert exc_info.value.internal_message == error_message + assert exc_info.value.message == error_message + assert exc_info.value.failure_type == FailureType.config_error + class TestSingleUseRefreshTokenOauth2Authenticator: @pytest.fixture @@ -217,7 +299,7 @@ def connector_config(self): "refresh_token": "my_refresh_token", "client_id": "my_client_id", "client_secret": "my_client_secret", - "token_expiry_date": "2022-12-31T00:00:00+00:00" + "token_expiry_date": "2022-12-31T00:00:00+00:00", } } @@ -243,9 +325,11 @@ def test_init(self, connector_config): ("number_of_seconds", 42, None, "2022-12-31T00:00:42+00:00"), ("string_of_seconds", "42", None, "2022-12-31T00:00:42+00:00"), ("date_format", "2023-04-04", "YYYY-MM-DD", "2023-04-04T00:00:00+00:00"), - ] + ], ) - def test_given_no_message_repository_get_access_token(self, test_name, expires_in_value, expiry_date_format, expected_expiry_date, capsys, mocker, connector_config): + def test_given_no_message_repository_get_access_token( + self, test_name, expires_in_value, expiry_date_format, expected_expiry_date, capsys, mocker, connector_config + ): authenticator = SingleUseRefreshTokenOauth2Authenticator( connector_config, token_refresh_endpoint="foobar", @@ -306,7 +390,9 @@ def test_given_message_repository_when_get_access_token_then_log_request(self, m message_repository=message_repository, ) mocker.patch("airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth.requests.request") - mocker.patch("airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth.format_http_message", return_value="formatted json") + mocker.patch( + "airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth.format_http_message", return_value="formatted json" + ) authenticator.token_has_expired = mocker.Mock(return_value=True) authenticator.get_access_token() diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py b/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py index 392ef132612c..b63af7973854 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py @@ -34,6 +34,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp stub_resp = {"data": self.resp_counter} self.resp_counter += 1 yield stub_resp + pass def retry_factor(self) -> float: @@ -43,21 +44,44 @@ def retry_factor(self) -> float: @pytest.mark.parametrize( ("status_code", "json_contents", "expected_is_available", "expected_messages"), [ - (403, {"error": "Something went wrong"}, False, [ - "This is most likely due to insufficient permissions on the credentials in use.", - "Something went wrong", - ]), - (200, {}, True, []) - ] + ( + 403, + {"error": "Something went wrong"}, + False, + [ + "This is most likely due to insufficient permissions on the credentials in use.", + "Something went wrong", + ], + ), + (200, {}, True, []), + ], ) @pytest.mark.parametrize( - ("include_source", "expected_docs_url_messages"), [ + ("include_source", "expected_docs_url_messages"), + [ (True, ["Please visit https://docs.airbyte.com/integrations/sources/MockSource to learn more."]), (False, ["Please visit the connector's documentation to learn more."]), - ] + ], ) -def test_default_http_availability_strategy(mocker, status_code, json_contents, expected_is_available, expected_messages, include_source, expected_docs_url_messages): - http_stream = MockHttpStream() +@pytest.mark.parametrize("records_as_list", [True, False]) +def test_default_http_availability_strategy( + mocker, + status_code, + json_contents, + expected_is_available, + expected_messages, + include_source, + expected_docs_url_messages, + records_as_list, +): + class MockListHttpStream(MockHttpStream): + def read_records(self, *args, **kvargs): + if records_as_list: + return list(super().read_records(*args, **kvargs)) + else: + return super().read_records(*args, **kvargs) + + http_stream = MockListHttpStream() assert isinstance(http_stream.availability_strategy, HttpAvailabilityStrategy) class MockResponseWithJsonContents(requests.Response, mocker.MagicMock): @@ -131,8 +155,8 @@ def test_send_handles_retries_when_checking_availability(mocker, caplog): assert message in caplog.text -def test_http_availability_strategy_on_empty_stream(mocker): - +@pytest.mark.parametrize("records_as_list", [True, False]) +def test_http_availability_strategy_on_empty_stream(mocker, records_as_list): class MockEmptyHttpStream(mocker.MagicMock, MockHttpStream): def __init__(self, *args, **kvargs): mocker.MagicMock.__init__(self) @@ -144,7 +168,10 @@ def __init__(self, *args, **kvargs): assert isinstance(empty_stream.availability_strategy, HttpAvailabilityStrategy) # Generator should have no values to generate - empty_stream.read_records.return_value = iter([]) + if records_as_list: + empty_stream.read_records.return_value = [] + else: + empty_stream.read_records.return_value = iter([]) logger = logging.getLogger("airbyte.test-source") stream_is_available, _ = empty_stream.check_availability(logger) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py b/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py index 9cd89fb59967..e826e74a47ee 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py @@ -388,6 +388,17 @@ def test_caching_sessions_are_different(): assert stream_1.cache_filename == stream_2.cache_filename +# def test_cached_streams_wortk_when_request_path_is_not_set(mocker, requests_mock): +# This test verifies that HttpStreams with a cached session work even if the path is not set +# For instance, when running in a unit test +# stream = CacheHttpStream() +# with mocker.patch.object(stream._session, "send", wraps=stream._session.send): +# requests_mock.register_uri("GET", stream.url_base) +# records = list(stream.read_records(sync_mode=SyncMode.full_refresh)) +# assert records == [{"data": 1}] +# "" + + def test_parent_attribute_exist(): parent_stream = CacheHttpStream() child_stream = CacheHttpSubStream(parent=parent_stream) @@ -395,13 +406,20 @@ def test_parent_attribute_exist(): assert child_stream.parent == parent_stream -def test_cache_response(mocker): +def test_that_response_was_cached(mocker, requests_mock): + requests_mock.register_uri("GET", "https://google.com/", text="text") stream = CacheHttpStream() + stream.clear_cache() mocker.patch.object(stream, "url_base", "https://google.com/") - list(stream.read_records(sync_mode=SyncMode.full_refresh)) + records = list(stream.read_records(sync_mode=SyncMode.full_refresh)) - with open(stream.cache_filename, "rb") as f: - assert f.read() + assert requests_mock.called + + requests_mock.reset_mock() + new_records = list(stream.read_records(sync_mode=SyncMode.full_refresh)) + + assert len(records) == len(new_records) + assert not requests_mock.called class CacheHttpStreamWithSlices(CacheHttpStream): @@ -425,15 +443,16 @@ def test_using_cache(mocker, requests_mock): parent_stream = CacheHttpStreamWithSlices() mocker.patch.object(parent_stream, "url_base", "https://google.com/") + parent_stream.clear_cache() assert requests_mock.call_count == 0 - assert parent_stream._session.cache.response_count() == 0 + assert len(parent_stream._session.cache.responses) == 0 for _slice in parent_stream.stream_slices(): list(parent_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=_slice)) assert requests_mock.call_count == 2 - assert parent_stream._session.cache.response_count() == 2 + assert len(parent_stream._session.cache.responses) == 2 child_stream = CacheHttpSubStream(parent=parent_stream) @@ -441,9 +460,9 @@ def test_using_cache(mocker, requests_mock): pass assert requests_mock.call_count == 2 - assert parent_stream._session.cache.response_count() == 2 - assert parent_stream._session.cache.has_url("https://google.com/") - assert parent_stream._session.cache.has_url("https://google.com/search") + assert len(parent_stream._session.cache.responses) == 2 + assert parent_stream._session.cache.contains(url="https://google.com/") + assert parent_stream._session.cache.contains(url="https://google.com/search") class AutoFailTrueHttpStream(StubBasicReadHttpStream): @@ -511,20 +530,28 @@ def test_default_get_error_display_message_handles_http_error(mocker): non_http_err_msg = stream.get_error_display_message(RuntimeError("not me")) assert non_http_err_msg is None - http_err_msg = stream.get_error_display_message(requests.HTTPError()) + response = requests.Response() + http_exception = requests.HTTPError(response=response) + http_err_msg = stream.get_error_display_message(http_exception) assert http_err_msg == "my custom message" @pytest.mark.parametrize( - "test_name, base_url, path, expected_full_url",[ + "test_name, base_url, path, expected_full_url", + [ ("test_no_slashes", "https://airbyte.io", "my_endpoint", "https://airbyte.io/my_endpoint"), ("test_trailing_slash_on_base_url", "https://airbyte.io/", "my_endpoint", "https://airbyte.io/my_endpoint"), - ("test_trailing_slash_on_base_url_and_leading_slash_on_path", "https://airbyte.io/", "/my_endpoint", "https://airbyte.io/my_endpoint"), + ( + "test_trailing_slash_on_base_url_and_leading_slash_on_path", + "https://airbyte.io/", + "/my_endpoint", + "https://airbyte.io/my_endpoint", + ), ("test_leading_slash_on_path", "https://airbyte.io", "/my_endpoint", "https://airbyte.io/my_endpoint"), ("test_trailing_slash_on_path", "https://airbyte.io", "/my_endpoint/", "https://airbyte.io/my_endpoint/"), ("test_nested_path_no_leading_slash", "https://airbyte.io", "v1/my_endpoint", "https://airbyte.io/v1/my_endpoint"), ("test_nested_path_with_leading_slash", "https://airbyte.io", "/v1/my_endpoint", "https://airbyte.io/v1/my_endpoint"), - ] + ], ) def test_join_url(test_name, base_url, path, expected_full_url): actual_url = HttpStream._join_url(base_url, path) @@ -532,18 +559,65 @@ def test_join_url(test_name, base_url, path, expected_full_url): @pytest.mark.parametrize( - "deduplicate_query_params, path, params, expected_url", [ - pytest.param(True, "v1/endpoint?param1=value1", {}, "https://test_base_url.com/v1/endpoint?param1=value1", id="test_params_only_in_path"), - pytest.param(True, "v1/endpoint", {"param1": "value1"}, "https://test_base_url.com/v1/endpoint?param1=value1", id="test_params_only_in_path"), + "deduplicate_query_params, path, params, expected_url", + [ + pytest.param( + True, "v1/endpoint?param1=value1", {}, "https://test_base_url.com/v1/endpoint?param1=value1", id="test_params_only_in_path" + ), + pytest.param( + True, "v1/endpoint", {"param1": "value1"}, "https://test_base_url.com/v1/endpoint?param1=value1", id="test_params_only_in_path" + ), pytest.param(True, "v1/endpoint", None, "https://test_base_url.com/v1/endpoint", id="test_params_is_none_and_no_params_in_path"), - pytest.param(True, "v1/endpoint?param1=value1", None, "https://test_base_url.com/v1/endpoint?param1=value1", id="test_params_is_none_and_no_params_in_path"), - pytest.param(True, "v1/endpoint?param1=value1", {"param2": "value2"}, "https://test_base_url.com/v1/endpoint?param1=value1¶m2=value2", id="test_no_duplicate_params"), - pytest.param(True, "v1/endpoint?param1=value1", {"param1": "value1"}, "https://test_base_url.com/v1/endpoint?param1=value1", id="test_duplicate_params_same_value"), - pytest.param(True, "v1/endpoint?param1=1", {"param1": 1}, "https://test_base_url.com/v1/endpoint?param1=1", id="test_duplicate_params_same_value_not_string"), - pytest.param(True, "v1/endpoint?param1=value1", {"param1": "value2"}, "https://test_base_url.com/v1/endpoint?param1=value1¶m1=value2", id="test_duplicate_params_different_value"), - pytest.param(False, "v1/endpoint?param1=value1", {"param1": "value2"}, "https://test_base_url.com/v1/endpoint?param1=value1¶m1=value2", id="test_same_params_different_value_no_deduplication"), - pytest.param(False, "v1/endpoint?param1=value1", {"param1": "value1"}, "https://test_base_url.com/v1/endpoint?param1=value1¶m1=value1", id="test_same_params_same_value_no_deduplication"), - ] + pytest.param( + True, + "v1/endpoint?param1=value1", + None, + "https://test_base_url.com/v1/endpoint?param1=value1", + id="test_params_is_none_and_no_params_in_path", + ), + pytest.param( + True, + "v1/endpoint?param1=value1", + {"param2": "value2"}, + "https://test_base_url.com/v1/endpoint?param1=value1¶m2=value2", + id="test_no_duplicate_params", + ), + pytest.param( + True, + "v1/endpoint?param1=value1", + {"param1": "value1"}, + "https://test_base_url.com/v1/endpoint?param1=value1", + id="test_duplicate_params_same_value", + ), + pytest.param( + True, + "v1/endpoint?param1=1", + {"param1": 1}, + "https://test_base_url.com/v1/endpoint?param1=1", + id="test_duplicate_params_same_value_not_string", + ), + pytest.param( + True, + "v1/endpoint?param1=value1", + {"param1": "value2"}, + "https://test_base_url.com/v1/endpoint?param1=value1¶m1=value2", + id="test_duplicate_params_different_value", + ), + pytest.param( + False, + "v1/endpoint?param1=value1", + {"param1": "value2"}, + "https://test_base_url.com/v1/endpoint?param1=value1¶m1=value2", + id="test_same_params_different_value_no_deduplication", + ), + pytest.param( + False, + "v1/endpoint?param1=value1", + {"param1": "value1"}, + "https://test_base_url.com/v1/endpoint?param1=value1¶m1=value1", + id="test_same_params_same_value_no_deduplication", + ), + ], ) def test_duplicate_request_params_are_deduped(deduplicate_query_params, path, params, expected_url): stream = StubBasicReadHttpStream(deduplicate_query_params) @@ -554,3 +628,8 @@ def test_duplicate_request_params_are_deduped(deduplicate_query_params, path, pa else: prepared_request = stream._create_prepared_request(path=path, params=params) assert prepared_request.url == expected_url + + +def test_connection_pool(): + stream = StubBasicReadHttpStream(authenticator=HttpTokenAuthenticator("test-token")) + assert stream._session.adapters["https://"]._pool_connections == 20 diff --git a/airbyte-cdk/python/unit_tests/sources/streams/test_call_rate.py b/airbyte-cdk/python/unit_tests/sources/streams/test_call_rate.py new file mode 100644 index 000000000000..3072f884542d --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/test_call_rate.py @@ -0,0 +1,300 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import os +import tempfile +import time +from datetime import datetime, timedelta +from typing import Iterable, Mapping + +import pytest +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.call_rate import ( + APIBudget, + CallRateLimitHit, + FixedWindowCallRatePolicy, + HttpRequestMatcher, + MovingWindowCallRatePolicy, + Rate, + UnlimitedCallRatePolicy, +) +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +from airbyte_cdk.utils.constants import ENV_REQUEST_CACHE_PATH +from requests import Request + + +class StubDummyHttpStream(HttpStream): + url_base = "https://test_base_url.com" + primary_key = "some_key" + + def next_page_token(self, *args, **kwargs): + return True # endless pages + + def path(self, **kwargs) -> str: + return "" + + def parse_response(self, *args, **kwargs) -> Iterable[Mapping]: + yield {"data": "some_data"} + + +class StubDummyCacheHttpStream(StubDummyHttpStream): + use_cache = True + + +@pytest.fixture(name="enable_cache") +def enable_cache_fixture(): + prev_cache_path = os.environ.get(ENV_REQUEST_CACHE_PATH) + with tempfile.TemporaryDirectory() as temp_dir: + os.environ[ENV_REQUEST_CACHE_PATH] = temp_dir + yield + + if prev_cache_path is not None: + os.environ[ENV_REQUEST_CACHE_PATH] = prev_cache_path + + +class TestHttpRequestMatcher: + try_all_types_of_requests = pytest.mark.parametrize( + "request_factory", + [Request, lambda *args, **kwargs: Request(*args, **kwargs).prepare()], + ) + + @try_all_types_of_requests + def test_url(self, request_factory): + matcher = HttpRequestMatcher(url="http://some_url/") + assert not matcher(request_factory(url="http://some_wrong_url")) + assert matcher(request_factory(url="http://some_url")) + + @try_all_types_of_requests + def test_method(self, request_factory): + matcher = HttpRequestMatcher(method="GET") + assert not matcher(request_factory(url="http://some_url")) + assert not matcher(request_factory(url="http://some_url", method="POST")) + assert matcher(request_factory(url="http://some_url", method="GET")) + + @try_all_types_of_requests + def test_params(self, request_factory): + matcher = HttpRequestMatcher(params={"param1": 10, "param2": 15}) + assert not matcher(request_factory(url="http://some_url/")) + assert not matcher(request_factory(url="http://some_url/", params={"param1": 10, "param3": 100})) + assert not matcher(request_factory(url="http://some_url/", params={"param1": 10, "param2": 10})) + assert matcher(request_factory(url="http://some_url/", params={"param1": 10, "param2": 15, "param3": 100})) + + @try_all_types_of_requests + def test_header(self, request_factory): + matcher = HttpRequestMatcher(headers={"header1": 10, "header2": 15}) + assert not matcher(request_factory(url="http://some_url")) + assert not matcher(request_factory(url="http://some_url", headers={"header1": "10", "header3": "100"})) + assert not matcher(request_factory(url="http://some_url", headers={"header1": "10", "header2": "10"})) + assert matcher(request_factory(url="http://some_url", headers={"header1": "10", "header2": "15", "header3": "100"})) + + @try_all_types_of_requests + def test_combination(self, request_factory): + matcher = HttpRequestMatcher(method="GET", url="http://some_url/", headers={"header1": 10}, params={"param2": "test"}) + assert matcher(request_factory(method="GET", url="http://some_url", headers={"header1": "10"}, params={"param2": "test"})) + assert not matcher(request_factory(method="GET", url="http://some_url", headers={"header1": "10"})) + assert not matcher(request_factory(method="GET", url="http://some_url")) + assert not matcher(request_factory(url="http://some_url")) + + +def test_http_request_matching(mocker): + """Test policy lookup based on matchers.""" + users_policy = mocker.Mock(spec=MovingWindowCallRatePolicy) + groups_policy = mocker.Mock(spec=MovingWindowCallRatePolicy) + root_policy = mocker.Mock(spec=MovingWindowCallRatePolicy) + + users_policy.matches.side_effect = HttpRequestMatcher(url="http://domain/api/users", method="GET") + groups_policy.matches.side_effect = HttpRequestMatcher(url="http://domain/api/groups", method="POST") + root_policy.matches.side_effect = HttpRequestMatcher(method="GET") + api_budget = APIBudget( + policies=[ + users_policy, + groups_policy, + root_policy, + ] + ) + + api_budget.acquire_call(Request("POST", url="http://domain/unmatched_endpoint"), block=False), "unrestricted call" + users_policy.try_acquire.assert_not_called() + groups_policy.try_acquire.assert_not_called() + root_policy.try_acquire.assert_not_called() + + users_request = Request("GET", url="http://domain/api/users") + api_budget.acquire_call(users_request, block=False), "first call, first matcher" + users_policy.try_acquire.assert_called_once_with(users_request, weight=1) + groups_policy.try_acquire.assert_not_called() + root_policy.try_acquire.assert_not_called() + + api_budget.acquire_call(Request("GET", url="http://domain/api/users"), block=False), "second call, first matcher" + assert users_policy.try_acquire.call_count == 2 + groups_policy.try_acquire.assert_not_called() + root_policy.try_acquire.assert_not_called() + + group_request = Request("POST", url="http://domain/api/groups") + api_budget.acquire_call(group_request, block=False), "first call, second matcher" + assert users_policy.try_acquire.call_count == 2 + groups_policy.try_acquire.assert_called_once_with(group_request, weight=1) + root_policy.try_acquire.assert_not_called() + + api_budget.acquire_call(Request("POST", url="http://domain/api/groups"), block=False), "second call, second matcher" + assert users_policy.try_acquire.call_count == 2 + assert groups_policy.try_acquire.call_count == 2 + root_policy.try_acquire.assert_not_called() + + any_get_request = Request("GET", url="http://domain/api/") + api_budget.acquire_call(any_get_request, block=False), "first call, third matcher" + assert users_policy.try_acquire.call_count == 2 + assert groups_policy.try_acquire.call_count == 2 + root_policy.try_acquire.assert_called_once_with(any_get_request, weight=1) + + +class TestUnlimitedCallRatePolicy: + def test_try_acquire(self, mocker): + policy = UnlimitedCallRatePolicy(matchers=[]) + assert policy.matches(mocker.Mock()), "should match anything" + policy.try_acquire(mocker.Mock(), weight=1) + policy.try_acquire(mocker.Mock(), weight=10) + + def test_update(self): + policy = UnlimitedCallRatePolicy(matchers=[]) + policy.update(available_calls=10, call_reset_ts=datetime.now()) + policy.update(available_calls=None, call_reset_ts=datetime.now()) + policy.update(available_calls=10, call_reset_ts=None) + + +class TestFixedWindowCallRatePolicy: + def test_limit_rate(self, mocker): + policy = FixedWindowCallRatePolicy(matchers=[], next_reset_ts=datetime.now(), period=timedelta(hours=1), call_limit=100) + policy.try_acquire(mocker.Mock(), weight=1) + policy.try_acquire(mocker.Mock(), weight=20) + with pytest.raises(ValueError, match="Weight can not exceed the call limit"): + policy.try_acquire(mocker.Mock(), weight=101) + + with pytest.raises(CallRateLimitHit) as exc: + policy.try_acquire(mocker.Mock(), weight=100 - 20 - 1 + 1) + + assert exc.value.time_to_wait + assert exc.value.weight == 100 - 20 - 1 + 1 + assert exc.value.item + + def test_update_available_calls(self, mocker): + policy = FixedWindowCallRatePolicy(matchers=[], next_reset_ts=datetime.now(), period=timedelta(hours=1), call_limit=100) + # update to decrease number of calls available + policy.update(available_calls=2, call_reset_ts=None) + # hit the limit with weight=3 + with pytest.raises(CallRateLimitHit): + policy.try_acquire(mocker.Mock(), weight=3) + # ok with less weight=1 + policy.try_acquire(mocker.Mock(), weight=1) + + # update to increase number of calls available, ignored + policy.update(available_calls=20, call_reset_ts=None) + # so we still hit the limit with weight=3 + with pytest.raises(CallRateLimitHit): + policy.try_acquire(mocker.Mock(), weight=3) + + +class TestMovingWindowCallRatePolicy: + def test_no_rates(self): + """should raise a ValueError when no rates provided""" + with pytest.raises(ValueError, match="The list of rates can not be empty"): + MovingWindowCallRatePolicy(rates=[], matchers=[]) + + def test_limit_rate(self): + """try_acquire must respect configured call rate and throw CallRateLimitHit when hit the limit.""" + policy = MovingWindowCallRatePolicy(rates=[Rate(10, timedelta(minutes=1))], matchers=[]) + + for i in range(10): + policy.try_acquire("call", weight=1), f"{i + 1} call" + + with pytest.raises(CallRateLimitHit) as excinfo1: + policy.try_acquire("call", weight=1), "call over limit" + assert excinfo1.value.time_to_wait.total_seconds() == pytest.approx(60, 0.1) + + time.sleep(0.5) + + with pytest.raises(CallRateLimitHit) as excinfo2: + policy.try_acquire("call", weight=1), "call over limit" + assert excinfo2.value.time_to_wait < excinfo1.value.time_to_wait, "time to wait must decrease over time" + + def test_limit_rate_support_custom_weight(self): + """try_acquire must take into account provided weight and throw CallRateLimitHit when hit the limit.""" + policy = MovingWindowCallRatePolicy(rates=[Rate(10, timedelta(minutes=1))], matchers=[]) + + policy.try_acquire("call", weight=2), "1st call with weight of 2" + with pytest.raises(CallRateLimitHit) as excinfo: + policy.try_acquire("call", weight=9), "2nd call, over limit since 2 + 9 = 11 > 10" + assert excinfo.value.time_to_wait.total_seconds() == pytest.approx(60, 0.1), "should wait 1 minute before next call" + + def test_multiple_limit_rates(self): + """try_acquire must take into all call rates and apply stricter.""" + policy = MovingWindowCallRatePolicy( + matchers=[], + rates=[ + Rate(10, timedelta(minutes=10)), + Rate(3, timedelta(seconds=10)), + Rate(2, timedelta(hours=1)), + ], + ) + + policy.try_acquire("call", weight=2), "1 call" + + with pytest.raises(CallRateLimitHit) as excinfo: + policy.try_acquire("call", weight=1), "1 call" + + assert excinfo.value.time_to_wait.total_seconds() == pytest.approx(3600, 0.1) + assert str(excinfo.value) == "Bucket for item=call with Rate limit=2/1.0h is already full" + + +class TestHttpStreamIntegration: + def test_without_cache(self, mocker, requests_mock): + """Test that HttpStream will use call budget when provided""" + requests_mock.get(f"{StubDummyHttpStream.url_base}/", json={"data": "test"}) + + mocker.patch.object(MovingWindowCallRatePolicy, "try_acquire") + + api_budget = APIBudget( + policies=[ + MovingWindowCallRatePolicy( + matchers=[HttpRequestMatcher(url=f"{StubDummyHttpStream.url_base}/", method="GET")], + rates=[ + Rate(2, timedelta(minutes=1)), + ], + ), + ] + ) + + stream = StubDummyHttpStream(api_budget=api_budget, authenticator=TokenAuthenticator(token="ABCD")) + records = stream.read_records(SyncMode.full_refresh) + for i in range(10): + assert next(records) == {"data": "some_data"} + + assert MovingWindowCallRatePolicy.try_acquire.call_count == 10 + + @pytest.mark.usefixtures("enable_cache") + def test_with_cache(self, mocker, requests_mock): + """Test that HttpStream will use call budget when provided and not cached""" + requests_mock.get(f"{StubDummyHttpStream.url_base}/", json={"data": "test"}) + + mocker.patch.object(MovingWindowCallRatePolicy, "try_acquire") + + api_budget = APIBudget( + policies=[ + MovingWindowCallRatePolicy( + matchers=[ + HttpRequestMatcher(url=f"{StubDummyHttpStream.url_base}/", method="GET"), + ], + rates=[ + Rate(2, timedelta(minutes=1)), + ], + ) + ] + ) + + stream = StubDummyCacheHttpStream(api_budget=api_budget) + records = stream.read_records(SyncMode.full_refresh) + + for i in range(10): + assert next(records) == {"data": "some_data"} + + assert MovingWindowCallRatePolicy.try_acquire.call_count == 1 diff --git a/airbyte-cdk/python/unit_tests/sources/streams/test_stream_read.py b/airbyte-cdk/python/unit_tests/sources/streams/test_stream_read.py new file mode 100644 index 000000000000..02b9cec19bab --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/streams/test_stream_read.py @@ -0,0 +1,194 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import logging +from typing import Any, Iterable, List, Mapping, Optional, Union +from unittest.mock import Mock + +import pytest +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, SyncMode +from airbyte_cdk.models import Type as MessageType +from airbyte_cdk.sources.message import InMemoryMessageRepository +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade +from airbyte_cdk.sources.streams.concurrent.cursor import NoopCursor +from airbyte_cdk.sources.streams.core import StreamData +from airbyte_cdk.sources.utils.schema_helpers import InternalConfig +from airbyte_cdk.sources.utils.slice_logger import DebugSliceLogger + +_A_CURSOR_FIELD = ["NESTED", "CURSOR"] +_DEFAULT_INTERNAL_CONFIG = InternalConfig() +_STREAM_NAME = "STREAM" +_NO_STATE = None + + +class _MockStream(Stream): + def __init__(self, slice_to_records: Mapping[str, List[Mapping[str, Any]]]): + self._slice_to_records = slice_to_records + + @property + def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: + return None + + def stream_slices( + self, *, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + for partition in self._slice_to_records.keys(): + yield {"partition": partition} + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: Optional[List[str]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + stream_state: Optional[Mapping[str, Any]] = None, + ) -> Iterable[StreamData]: + yield from self._slice_to_records[stream_slice["partition"]] + + def get_json_schema(self) -> Mapping[str, Any]: + return {} + + +def _stream(slice_to_partition_mapping, slice_logger, logger, message_repository): + return _MockStream(slice_to_partition_mapping) + + +def _concurrent_stream(slice_to_partition_mapping, slice_logger, logger, message_repository): + stream = _stream(slice_to_partition_mapping, slice_logger, logger, message_repository) + source = Mock() + source._slice_logger = slice_logger + source.message_repository = message_repository + stream = StreamFacade.create_from_stream(stream, source, logger, _NO_STATE, NoopCursor()) + stream.logger.setLevel(logger.level) + return stream + + +@pytest.mark.parametrize( + "constructor", + [ + pytest.param(_stream, id="synchronous_reader"), + pytest.param(_concurrent_stream, id="concurrent_reader"), + ], +) +def test_full_refresh_read_a_single_slice_with_debug(constructor): + # This test verifies that a concurrent stream adapted from a Stream behaves the same as the Stream object. + # It is done by running the same test cases on both streams + records = [ + {"id": 1, "partition": 1}, + {"id": 2, "partition": 1}, + ] + slice_to_partition = {1: records} + slice_logger = DebugSliceLogger() + logger = _mock_logger(True) + message_repository = InMemoryMessageRepository(Level.DEBUG) + stream = constructor(slice_to_partition, slice_logger, logger, message_repository) + + expected_records = [ + AirbyteMessage( + type=MessageType.LOG, + log=AirbyteLogMessage( + level=Level.INFO, + message='slice:{"partition": 1}', + ), + ), + *records, + ] + + actual_records = _read(stream, logger, slice_logger, message_repository) + + assert expected_records == actual_records + + +@pytest.mark.parametrize( + "constructor", + [ + pytest.param(_stream, id="synchronous_reader"), + pytest.param(_concurrent_stream, id="concurrent_reader"), + ], +) +def test_full_refresh_read_a_single_slice(constructor): + # This test verifies that a concurrent stream adapted from a Stream behaves the same as the Stream object. + # It is done by running the same test cases on both streams + logger = _mock_logger() + slice_logger = DebugSliceLogger() + message_repository = InMemoryMessageRepository(Level.INFO) + + records = [ + {"id": 1, "partition": 1}, + {"id": 2, "partition": 1}, + ] + slice_to_partition = {1: records} + stream = constructor(slice_to_partition, slice_logger, logger, message_repository) + + expected_records = [*records] + + actual_records = _read(stream, logger, slice_logger, message_repository) + + assert expected_records == actual_records + + +@pytest.mark.parametrize( + "constructor", + [ + pytest.param(_stream, id="synchronous_reader"), + pytest.param(_concurrent_stream, id="concurrent_reader"), + ], +) +def test_full_refresh_read_a_two_slices(constructor): + # This test verifies that a concurrent stream adapted from a Stream behaves the same as the Stream object + # It is done by running the same test cases on both streams + logger = _mock_logger() + slice_logger = DebugSliceLogger() + message_repository = InMemoryMessageRepository(Level.INFO) + + records_partition_1 = [ + {"id": 1, "partition": 1}, + {"id": 2, "partition": 1}, + ] + records_partition_2 = [ + {"id": 3, "partition": 2}, + {"id": 4, "partition": 2}, + ] + slice_to_partition = {1: records_partition_1, 2: records_partition_2} + stream = constructor(slice_to_partition, slice_logger, logger, message_repository) + + expected_records = [ + *records_partition_1, + *records_partition_2, + ] + + actual_records = _read(stream, logger, slice_logger, message_repository) + + for record in expected_records: + assert record in actual_records + assert len(expected_records) == len(actual_records) + + +def _read(stream, logger, slice_logger, message_repository): + records = [] + for record in stream.read_full_refresh(_A_CURSOR_FIELD, logger, slice_logger): + for message in message_repository.consume_queue(): + records.append(message) + records.append(record) + return records + + +def _mock_partition_generator(name: str, slices, records_per_partition, *, available=True, debug_log=False): + stream = Mock() + stream.name = name + stream.get_json_schema.return_value = {} + stream.generate_partitions.return_value = iter(slices) + stream.read_records.side_effect = [iter(records) for records in records_per_partition] + stream.logger.isEnabledFor.return_value = debug_log + if available: + stream.check_availability.return_value = True, None + else: + stream.check_availability.return_value = False, "A reason why the stream is unavailable" + return stream + + +def _mock_logger(enabled_for_debug=False): + logger = Mock() + logger.isEnabledFor.return_value = enabled_for_debug + logger.level = logging.DEBUG if enabled_for_debug else logging.INFO + return logger diff --git a/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py b/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py index 7bbcc0e1813e..4315f488112d 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py +++ b/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py @@ -52,11 +52,13 @@ def __init__( check_lambda: Callable[[], Tuple[bool, Optional[Any]]] = None, streams: List[Stream] = None, per_stream: bool = True, - message_repository: MessageRepository = None + message_repository: MessageRepository = None, + exception_on_missing_stream: bool = True, ): self._streams = streams self.check_lambda = check_lambda self.per_stream = per_stream + self.exception_on_missing_stream = exception_on_missing_stream self._message_repository = message_repository def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: @@ -69,6 +71,10 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: raise Exception("Stream is not set") return self._streams + @property + def raise_exception_on_missing_stream(self) -> bool: + return self.exception_on_missing_stream + @property def per_stream_state_enabled(self) -> bool: return self.per_stream @@ -105,6 +111,14 @@ def state(self, value: MutableMapping[str, Any]): self._cursor_value = value.get(self.cursor_field, self.start_date) +class StreamRaisesException(Stream): + name = "lamentations" + primary_key = None + + def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]: + raise AirbyteTracedException(message="I was born only to crash like Icarus") + + MESSAGE_FROM_REPOSITORY = Mock() @@ -239,6 +253,21 @@ def test_read_nonexistent_stream_raises_exception(mocker): list(src.read(logger, {}, catalog)) +def test_read_nonexistent_stream_without_raises_exception(mocker): + """Tests that attempting to sync a stream which the source does not return from the `streams` method raises an exception""" + s1 = MockStream(name="s1") + s2 = MockStream(name="this_stream_doesnt_exist_in_the_source") + + mocker.patch.object(MockStream, "get_json_schema", return_value={}) + + src = MockSource(streams=[s1], exception_on_missing_stream=False) + + catalog = ConfiguredAirbyteCatalog(streams=[_configured_stream(s2, SyncMode.full_refresh)]) + messages = list(src.read(logger, {}, catalog)) + + assert messages == [] + + def test_read_stream_emits_repository_message_before_record(mocker, message_repository): stream = MockStream(name="my_stream") mocker.patch.object(MockStream, "get_json_schema", return_value={}) @@ -371,8 +400,9 @@ def test_valid_full_refresh_read_no_slices(mocker): _as_stream_status("s2", AirbyteStreamStatus.STARTED), _as_stream_status("s2", AirbyteStreamStatus.RUNNING), *_as_records("s2", stream_output), - _as_stream_status("s2", AirbyteStreamStatus.COMPLETE) - ]) + _as_stream_status("s2", AirbyteStreamStatus.COMPLETE), + ] + ) messages = _fix_emitted_at(list(src.read(logger, {}, catalog))) assert expected == messages @@ -411,8 +441,9 @@ def test_valid_full_refresh_read_with_slices(mocker): _as_stream_status("s2", AirbyteStreamStatus.STARTED), _as_stream_status("s2", AirbyteStreamStatus.RUNNING), *_as_records("s2", slices), - _as_stream_status("s2", AirbyteStreamStatus.COMPLETE) - ]) + _as_stream_status("s2", AirbyteStreamStatus.COMPLETE), + ] + ) messages = _fix_emitted_at(list(src.read(logger, {}, catalog))) @@ -534,24 +565,26 @@ def test_with_state_attribute(self, mocker, use_legacy, per_stream_enabled): ] ) - expected = _fix_emitted_at([ - _as_stream_status("s1", AirbyteStreamStatus.STARTED), - _as_stream_status("s1", AirbyteStreamStatus.RUNNING), - _as_record("s1", stream_output[0]), - _as_record("s1", stream_output[1]), - _as_state({"s1": new_state_from_connector}, "s1", new_state_from_connector) - if per_stream_enabled - else _as_state({"s1": new_state_from_connector}), - _as_stream_status("s1", AirbyteStreamStatus.COMPLETE), - _as_stream_status("s2", AirbyteStreamStatus.STARTED), - _as_stream_status("s2", AirbyteStreamStatus.RUNNING), - _as_record("s2", stream_output[0]), - _as_record("s2", stream_output[1]), - _as_state({"s1": new_state_from_connector, "s2": new_state_from_connector}, "s2", new_state_from_connector) - if per_stream_enabled - else _as_state({"s1": new_state_from_connector, "s2": new_state_from_connector}), - _as_stream_status("s2", AirbyteStreamStatus.COMPLETE), - ]) + expected = _fix_emitted_at( + [ + _as_stream_status("s1", AirbyteStreamStatus.STARTED), + _as_stream_status("s1", AirbyteStreamStatus.RUNNING), + _as_record("s1", stream_output[0]), + _as_record("s1", stream_output[1]), + _as_state({"s1": new_state_from_connector}, "s1", new_state_from_connector) + if per_stream_enabled + else _as_state({"s1": new_state_from_connector}), + _as_stream_status("s1", AirbyteStreamStatus.COMPLETE), + _as_stream_status("s2", AirbyteStreamStatus.STARTED), + _as_stream_status("s2", AirbyteStreamStatus.RUNNING), + _as_record("s2", stream_output[0]), + _as_record("s2", stream_output[1]), + _as_state({"s1": new_state_from_connector, "s2": new_state_from_connector}, "s2", new_state_from_connector) + if per_stream_enabled + else _as_state({"s1": new_state_from_connector, "s2": new_state_from_connector}), + _as_stream_status("s2", AirbyteStreamStatus.COMPLETE), + ] + ) messages = _fix_emitted_at(list(src.read(logger, {}, catalog, state=input_state))) assert messages == expected @@ -613,24 +646,26 @@ def test_with_checkpoint_interval(self, mocker, use_legacy, per_stream_enabled): ] ) - expected = _fix_emitted_at([ - _as_stream_status("s1", AirbyteStreamStatus.STARTED), - _as_stream_status("s1", AirbyteStreamStatus.RUNNING), - _as_record("s1", stream_output[0]), - _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), - _as_record("s1", stream_output[1]), - _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), - _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), - _as_stream_status("s1", AirbyteStreamStatus.COMPLETE), - _as_stream_status("s2", AirbyteStreamStatus.STARTED), - _as_stream_status("s2", AirbyteStreamStatus.RUNNING), - _as_record("s2", stream_output[0]), - _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), - _as_record("s2", stream_output[1]), - _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), - _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), - _as_stream_status("s2", AirbyteStreamStatus.COMPLETE), - ]) + expected = _fix_emitted_at( + [ + _as_stream_status("s1", AirbyteStreamStatus.STARTED), + _as_stream_status("s1", AirbyteStreamStatus.RUNNING), + _as_record("s1", stream_output[0]), + _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), + _as_record("s1", stream_output[1]), + _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), + _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), + _as_stream_status("s1", AirbyteStreamStatus.COMPLETE), + _as_stream_status("s2", AirbyteStreamStatus.STARTED), + _as_stream_status("s2", AirbyteStreamStatus.RUNNING), + _as_record("s2", stream_output[0]), + _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), + _as_record("s2", stream_output[1]), + _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), + _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), + _as_stream_status("s2", AirbyteStreamStatus.COMPLETE), + ] + ) messages = _fix_emitted_at(list(src.read(logger, {}, catalog, state=input_state))) assert expected == messages @@ -680,18 +715,20 @@ def test_with_no_interval(self, mocker, use_legacy, per_stream_enabled): ] ) - expected = _fix_emitted_at([ - _as_stream_status("s1", AirbyteStreamStatus.STARTED), - _as_stream_status("s1", AirbyteStreamStatus.RUNNING), - *_as_records("s1", stream_output), - _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), - _as_stream_status("s1", AirbyteStreamStatus.COMPLETE), - _as_stream_status("s2", AirbyteStreamStatus.STARTED), - _as_stream_status("s2", AirbyteStreamStatus.RUNNING), - *_as_records("s2", stream_output), - _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), - _as_stream_status("s2", AirbyteStreamStatus.COMPLETE), - ]) + expected = _fix_emitted_at( + [ + _as_stream_status("s1", AirbyteStreamStatus.STARTED), + _as_stream_status("s1", AirbyteStreamStatus.RUNNING), + *_as_records("s1", stream_output), + _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), + _as_stream_status("s1", AirbyteStreamStatus.COMPLETE), + _as_stream_status("s2", AirbyteStreamStatus.STARTED), + _as_stream_status("s2", AirbyteStreamStatus.RUNNING), + *_as_records("s2", stream_output), + _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), + _as_stream_status("s2", AirbyteStreamStatus.COMPLETE), + ] + ) messages = _fix_emitted_at(list(src.read(logger, {}, catalog, state=input_state))) @@ -762,26 +799,28 @@ def test_with_slices(self, mocker, use_legacy, per_stream_enabled): ] ) - expected = _fix_emitted_at([ - _as_stream_status("s1", AirbyteStreamStatus.STARTED), - _as_stream_status("s1", AirbyteStreamStatus.RUNNING), - # stream 1 slice 1 - *_as_records("s1", stream_output), - _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), - # stream 1 slice 2 - *_as_records("s1", stream_output), - _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), - _as_stream_status("s1", AirbyteStreamStatus.COMPLETE), - _as_stream_status("s2", AirbyteStreamStatus.STARTED), - _as_stream_status("s2", AirbyteStreamStatus.RUNNING), - # stream 2 slice 1 - *_as_records("s2", stream_output), - _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), - # stream 2 slice 2 - *_as_records("s2", stream_output), - _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), - _as_stream_status("s2", AirbyteStreamStatus.COMPLETE), - ]) + expected = _fix_emitted_at( + [ + _as_stream_status("s1", AirbyteStreamStatus.STARTED), + _as_stream_status("s1", AirbyteStreamStatus.RUNNING), + # stream 1 slice 1 + *_as_records("s1", stream_output), + _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), + # stream 1 slice 2 + *_as_records("s1", stream_output), + _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), + _as_stream_status("s1", AirbyteStreamStatus.COMPLETE), + _as_stream_status("s2", AirbyteStreamStatus.STARTED), + _as_stream_status("s2", AirbyteStreamStatus.RUNNING), + # stream 2 slice 1 + *_as_records("s2", stream_output), + _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), + # stream 2 slice 2 + *_as_records("s2", stream_output), + _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), + _as_stream_status("s2", AirbyteStreamStatus.COMPLETE), + ] + ) messages = _fix_emitted_at(list(src.read(logger, {}, catalog, state=input_state))) @@ -863,14 +902,16 @@ def test_no_slices(self, mocker, use_legacy, per_stream_enabled, slices): ] ) - expected = _fix_emitted_at([ - _as_stream_status("s1", AirbyteStreamStatus.STARTED), - _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), - _as_stream_status("s1", AirbyteStreamStatus.COMPLETE), - _as_stream_status("s2", AirbyteStreamStatus.STARTED), - _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), - _as_stream_status("s2", AirbyteStreamStatus.COMPLETE), - ]) + expected = _fix_emitted_at( + [ + _as_stream_status("s1", AirbyteStreamStatus.STARTED), + _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), + _as_stream_status("s1", AirbyteStreamStatus.COMPLETE), + _as_stream_status("s2", AirbyteStreamStatus.STARTED), + _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), + _as_stream_status("s2", AirbyteStreamStatus.COMPLETE), + ] + ) messages = _fix_emitted_at(list(src.read(logger, {}, catalog, state=input_state))) @@ -951,42 +992,46 @@ def test_with_slices_and_interval(self, mocker, use_legacy, per_stream_enabled): ] ) - expected = _fix_emitted_at([ - # stream 1 slice 1 - _as_stream_status("s1", AirbyteStreamStatus.STARTED), - _as_stream_status("s1", AirbyteStreamStatus.RUNNING), - _as_record("s1", stream_output[0]), - _as_record("s1", stream_output[1]), - _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), - _as_record("s1", stream_output[2]), - _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), - # stream 1 slice 2 - _as_record("s1", stream_output[0]), - _as_record("s1", stream_output[1]), - _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), - _as_record("s1", stream_output[2]), - _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), - _as_stream_status("s1", AirbyteStreamStatus.COMPLETE), - # stream 2 slice 1 - _as_stream_status("s2", AirbyteStreamStatus.STARTED), - _as_stream_status("s2", AirbyteStreamStatus.RUNNING), - _as_record("s2", stream_output[0]), - _as_record("s2", stream_output[1]), - _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), - _as_record("s2", stream_output[2]), - _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), - # stream 2 slice 2 - _as_record("s2", stream_output[0]), - _as_record("s2", stream_output[1]), - _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), - _as_record("s2", stream_output[2]), - _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), - _as_stream_status("s2", AirbyteStreamStatus.COMPLETE), - ]) + expected = _fix_emitted_at( + [ + # stream 1 slice 1 + _as_stream_status("s1", AirbyteStreamStatus.STARTED), + _as_stream_status("s1", AirbyteStreamStatus.RUNNING), + _as_record("s1", stream_output[0]), + _as_record("s1", stream_output[1]), + _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), + _as_record("s1", stream_output[2]), + _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), + # stream 1 slice 2 + _as_record("s1", stream_output[0]), + _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), + _as_record("s1", stream_output[1]), + _as_record("s1", stream_output[2]), + _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), + _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), + _as_stream_status("s1", AirbyteStreamStatus.COMPLETE), + # stream 2 slice 1 + _as_stream_status("s2", AirbyteStreamStatus.STARTED), + _as_stream_status("s2", AirbyteStreamStatus.RUNNING), + _as_record("s2", stream_output[0]), + _as_record("s2", stream_output[1]), + _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), + _as_record("s2", stream_output[2]), + _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), + # stream 2 slice 2 + _as_record("s2", stream_output[0]), + _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), + _as_record("s2", stream_output[1]), + _as_record("s2", stream_output[2]), + _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), + _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), + _as_stream_status("s2", AirbyteStreamStatus.COMPLETE), + ] + ) messages = _fix_emitted_at(list(src.read(logger, {}, catalog, state=input_state))) - assert expected == messages + assert messages == expected @pytest.mark.parametrize( "per_stream_enabled", @@ -1073,11 +1118,12 @@ def test_emit_non_records(self, mocker, per_stream_enabled): _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), # stream 1 slice 2 stream_data_to_airbyte_message("s1", stream_output[0]), + _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), stream_data_to_airbyte_message("s1", stream_output[1]), stream_data_to_airbyte_message("s1", stream_output[2]), - _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), stream_data_to_airbyte_message("s1", stream_output[3]), _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), + _as_state({"s1": state}, "s1", state) if per_stream_enabled else _as_state({"s1": state}), _as_stream_status("s1", AirbyteStreamStatus.COMPLETE), # stream 2 slice 1 _as_stream_status("s2", AirbyteStreamStatus.STARTED), @@ -1090,33 +1136,127 @@ def test_emit_non_records(self, mocker, per_stream_enabled): _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), # stream 2 slice 2 stream_data_to_airbyte_message("s2", stream_output[0]), + _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), stream_data_to_airbyte_message("s2", stream_output[1]), stream_data_to_airbyte_message("s2", stream_output[2]), - _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), stream_data_to_airbyte_message("s2", stream_output[3]), _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), + _as_state({"s1": state, "s2": state}, "s2", state) if per_stream_enabled else _as_state({"s1": state, "s2": state}), _as_stream_status("s2", AirbyteStreamStatus.COMPLETE), ] ) messages = _fix_emitted_at(list(src.read(logger, {}, catalog, state=input_state))) - assert expected == messages + assert messages == expected def test_checkpoint_state_from_stream_instance(): teams_stream = MockStreamOverridesStateMethod() managers_stream = StreamNoStateMethod() - src = MockSource(streams=[teams_stream, managers_stream]) state_manager = ConnectorStateManager({"teams": teams_stream, "managers": managers_stream}, []) # The stream_state passed to checkpoint_state() should be ignored since stream implements state function teams_stream.state = {"updated_at": "2022-09-11"} - actual_message = src._checkpoint_state(teams_stream, {"ignored": "state"}, state_manager) + actual_message = teams_stream._checkpoint_state({"ignored": "state"}, state_manager, True) assert actual_message == _as_state({"teams": {"updated_at": "2022-09-11"}}, "teams", {"updated_at": "2022-09-11"}) # The stream_state passed to checkpoint_state() should be used since the stream does not implement state function - actual_message = src._checkpoint_state(managers_stream, {"updated": "expected_here"}, state_manager) + actual_message = managers_stream._checkpoint_state({"updated": "expected_here"}, state_manager, True) assert actual_message == _as_state( {"teams": {"updated_at": "2022-09-11"}, "managers": {"updated": "expected_here"}}, "managers", {"updated": "expected_here"} ) + + +def test_continue_sync_with_failed_streams(mocker): + """ + Tests that running a sync for a connector with multiple streams and continue_sync_on_stream_failure enabled continues + syncing even when one stream fails with an error. + """ + stream_output = [{"k1": "v1"}, {"k2": "v2"}] + s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1") + s2 = StreamRaisesException() + s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3") + + mocker.patch.object(MockStream, "get_json_schema", return_value={}) + mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={}) + + src = MockSource(streams=[s1, s2, s3]) + mocker.patch.object(MockSource, "continue_sync_on_stream_failure", return_value=True) + catalog = ConfiguredAirbyteCatalog( + streams=[ + _configured_stream(s1, SyncMode.full_refresh), + _configured_stream(s2, SyncMode.full_refresh), + _configured_stream(s3, SyncMode.full_refresh), + ] + ) + + expected = _fix_emitted_at( + [ + _as_stream_status("s1", AirbyteStreamStatus.STARTED), + _as_stream_status("s1", AirbyteStreamStatus.RUNNING), + *_as_records("s1", stream_output), + _as_stream_status("s1", AirbyteStreamStatus.COMPLETE), + _as_stream_status("lamentations", AirbyteStreamStatus.STARTED), + _as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE), + _as_stream_status("s3", AirbyteStreamStatus.STARTED), + _as_stream_status("s3", AirbyteStreamStatus.RUNNING), + *_as_records("s3", stream_output), + _as_stream_status("s3", AirbyteStreamStatus.COMPLETE), + ] + ) + + messages = [] + with pytest.raises(AirbyteTracedException) as exc: + # We can't use list comprehension or list() here because we are still raising a final exception for the + # failed streams and that disrupts parsing the generator into the messages emitted before + for message in src.read(logger, {}, catalog): + messages.append(message) + + messages = _fix_emitted_at(messages) + assert expected == messages + assert "lamentations" in exc.value.message + + +def test_stop_sync_with_failed_streams(mocker): + """ + Tests that running a sync for a connector with multiple streams and continue_sync_on_stream_failure disabled stops + syncing once a stream fails with an error. + """ + stream_output = [{"k1": "v1"}, {"k2": "v2"}] + s1 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s1") + s2 = StreamRaisesException() + s3 = MockStream([({"sync_mode": SyncMode.full_refresh}, stream_output)], name="s3") + + mocker.patch.object(MockStream, "get_json_schema", return_value={}) + mocker.patch.object(StreamRaisesException, "get_json_schema", return_value={}) + + src = MockSource(streams=[s1, s2, s3]) + catalog = ConfiguredAirbyteCatalog( + streams=[ + _configured_stream(s1, SyncMode.full_refresh), + _configured_stream(s2, SyncMode.full_refresh), + _configured_stream(s3, SyncMode.full_refresh), + ] + ) + + expected = _fix_emitted_at( + [ + _as_stream_status("s1", AirbyteStreamStatus.STARTED), + _as_stream_status("s1", AirbyteStreamStatus.RUNNING), + *_as_records("s1", stream_output), + _as_stream_status("s1", AirbyteStreamStatus.COMPLETE), + _as_stream_status("lamentations", AirbyteStreamStatus.STARTED), + _as_stream_status("lamentations", AirbyteStreamStatus.INCOMPLETE), + ] + ) + + messages = [] + with pytest.raises(AirbyteTracedException): + # We can't use list comprehension or list() here because we are still raising a final exception for the + # failed streams and that disrupts parsing the generator into the messages emitted before + for message in src.read(logger, {}, catalog): + messages.append(message) + + messages = _fix_emitted_at(messages) + assert expected == messages diff --git a/airbyte-cdk/python/unit_tests/sources/test_concurrent_source.py b/airbyte-cdk/python/unit_tests/sources/test_concurrent_source.py new file mode 100644 index 000000000000..ca5c669a27c6 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/test_concurrent_source.py @@ -0,0 +1,105 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import concurrent +import logging +from typing import Any, Callable, Dict, Iterable, Mapping, Optional, Tuple +from unittest.mock import Mock + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource +from airbyte_cdk.sources.concurrent_source.thread_pool_manager import ThreadPoolManager +from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository +from airbyte_cdk.sources.streams.concurrent.abstract_stream import AbstractStream +from airbyte_cdk.sources.streams.concurrent.availability_strategy import StreamAvailability, StreamAvailable, StreamUnavailable +from airbyte_cdk.sources.streams.concurrent.partitions.partition import Partition +from airbyte_cdk.sources.streams.concurrent.partitions.record import Record +from airbyte_protocol.models import AirbyteStream + +logger = logging.getLogger("airbyte") + + +class _MockSource(ConcurrentSource): + def __init__( + self, + check_lambda: Callable[[], Tuple[bool, Optional[Any]]] = None, + per_stream: bool = True, + message_repository: MessageRepository = InMemoryMessageRepository(), + threadpool: ThreadPoolManager = ThreadPoolManager( + concurrent.futures.ThreadPoolExecutor(max_workers=1, thread_name_prefix="workerpool"), logger + ), + exception_on_missing_stream: bool = True, + ): + super().__init__(threadpool, Mock(), Mock(), message_repository) + self.check_lambda = check_lambda + self.per_stream = per_stream + self.exception_on_missing_stream = exception_on_missing_stream + self._message_repository = message_repository + + +MESSAGE_FROM_REPOSITORY = Mock() + + +class _MockStream(AbstractStream): + def __init__(self, name: str, available: bool = True, json_schema: Dict[str, Any] = {}): + self._name = name + self._available = available + self._json_schema = json_schema + + def generate_partitions(self) -> Iterable[Partition]: + yield _MockPartition(self._name) + + @property + def name(self) -> str: + return self._name + + @property + def cursor_field(self) -> Optional[str]: + raise NotImplementedError + + def check_availability(self) -> StreamAvailability: + if self._available: + return StreamAvailable() + else: + return StreamUnavailable("stream is unavailable") + + def get_json_schema(self) -> Mapping[str, Any]: + return self._json_schema + + def as_airbyte_stream(self) -> AirbyteStream: + return AirbyteStream(name=self.name, json_schema=self.get_json_schema(), supported_sync_modes=[SyncMode.full_refresh]) + + def log_stream_sync_configuration(self) -> None: + raise NotImplementedError + + +class _MockPartition(Partition): + def __init__(self, name: str): + self._name = name + self._closed = False + + def read(self) -> Iterable[Record]: + yield from [Record({"key": "value"}, self._name)] + + def to_slice(self) -> Optional[Mapping[str, Any]]: + return {} + + def stream_name(self) -> str: + return self._name + + def close(self) -> None: + self._closed = True + + def is_closed(self) -> bool: + return self._closed + + def __hash__(self) -> int: + return hash(self._name) + + +def test_concurrent_source_reading_from_no_streams(): + stream = _MockStream("my_stream", False, {}) + source = _MockSource() + messages = [] + for m in source.read([stream]): + messages.append(m) diff --git a/airbyte-cdk/python/unit_tests/sources/test_config.py b/airbyte-cdk/python/unit_tests/sources/test_config.py index e9617da3684a..ac8b6aac4874 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_config.py +++ b/airbyte-cdk/python/unit_tests/sources/test_config.py @@ -43,7 +43,12 @@ class TestBaseConfig: "properties": { "count": {"title": "Count", "type": "integer"}, "name": {"title": "Name", "type": "string"}, - "selected_strategy": {"const": "option1", "title": "Selected " "Strategy", "type": "string", "default": "option1"}, + "selected_strategy": { + "const": "option1", + "title": "Selected " "Strategy", + "type": "string", + "default": "option1", + }, }, "required": ["name", "count"], "title": "Choice1", @@ -51,7 +56,12 @@ class TestBaseConfig: }, { "properties": { - "selected_strategy": {"const": "option2", "title": "Selected " "Strategy", "type": "string", "default": "option2"}, + "selected_strategy": { + "const": "option2", + "title": "Selected " "Strategy", + "type": "string", + "default": "option2", + }, "sequence": {"items": {"type": "string"}, "title": "Sequence", "type": "array"}, }, "required": ["sequence"], diff --git a/airbyte-cdk/python/unit_tests/sources/test_http_logger.py b/airbyte-cdk/python/unit_tests/sources/test_http_logger.py index a79a5216e2eb..5711d3529211 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_http_logger.py +++ b/airbyte-cdk/python/unit_tests/sources/test_http_logger.py @@ -58,7 +58,17 @@ def build(self): {}, {}, {}, - {"airbyte_cdk": {"stream": {"name": A_STREAM_NAME}}, "http": {"title": A_TITLE, "description": A_DESCRIPTION, "request": {"method": "GET", "body": {"content": None}, "headers": {}}, "response": EMPTY_RESPONSE}, "log": {"level": "debug"}, "url": {"full": "https://airbyte.io/"}}, + { + "airbyte_cdk": {"stream": {"name": A_STREAM_NAME}}, + "http": { + "title": A_TITLE, + "description": A_DESCRIPTION, + "request": {"method": "GET", "body": {"content": None}, "headers": {}}, + "response": EMPTY_RESPONSE, + }, + "log": {"level": "debug"}, + "url": {"full": "https://airbyte.io/"}, + }, ), ( "test_get_request_with_headers", @@ -68,7 +78,17 @@ def build(self): {}, {}, {}, - {"airbyte_cdk": {"stream": {"name": A_STREAM_NAME}}, "http": {"title": A_TITLE, "description": A_DESCRIPTION, "request": {"method": "GET", "body": {"content": None}, "headers": {"h1": "v1", "h2": "v2"}}, "response": EMPTY_RESPONSE}, "log": {"level": "debug"}, "url": {"full": "https://airbyte.io/"}}, + { + "airbyte_cdk": {"stream": {"name": A_STREAM_NAME}}, + "http": { + "title": A_TITLE, + "description": A_DESCRIPTION, + "request": {"method": "GET", "body": {"content": None}, "headers": {"h1": "v1", "h2": "v2"}}, + "response": EMPTY_RESPONSE, + }, + "log": {"level": "debug"}, + "url": {"full": "https://airbyte.io/"}, + }, ), ( "test_get_request_with_request_params", @@ -78,7 +98,17 @@ def build(self): {"p1": "v1", "p2": "v2"}, {}, {}, - {"airbyte_cdk": {"stream": {"name": A_STREAM_NAME}}, "http": {"title": A_TITLE, "description": A_DESCRIPTION, "request": {"method": "GET", "body": {"content": None}, "headers": {}}, "response": EMPTY_RESPONSE}, "log": {"level": "debug"}, "url": {"full": "https://airbyte.io/?p1=v1&p2=v2"}}, + { + "airbyte_cdk": {"stream": {"name": A_STREAM_NAME}}, + "http": { + "title": A_TITLE, + "description": A_DESCRIPTION, + "request": {"method": "GET", "body": {"content": None}, "headers": {}}, + "response": EMPTY_RESPONSE, + }, + "log": {"level": "debug"}, + "url": {"full": "https://airbyte.io/?p1=v1&p2=v2"}, + }, ), ( "test_get_request_with_request_body_json", @@ -88,7 +118,21 @@ def build(self): {}, {"b1": "v1", "b2": "v2"}, {}, - {"airbyte_cdk": {"stream": {"name": A_STREAM_NAME}}, "http": {"title": A_TITLE, "description": A_DESCRIPTION, "request": {"method": "GET", "body": {"content": '{"b1": "v1", "b2": "v2"}'}, "headers": {"Content-Type": "application/json", "Content-Length": "24"}}, "response": EMPTY_RESPONSE}, "log": {"level": "debug"}, "url": {"full": "https://airbyte.io/"}} + { + "airbyte_cdk": {"stream": {"name": A_STREAM_NAME}}, + "http": { + "title": A_TITLE, + "description": A_DESCRIPTION, + "request": { + "method": "GET", + "body": {"content": '{"b1": "v1", "b2": "v2"}'}, + "headers": {"Content-Type": "application/json", "Content-Length": "24"}, + }, + "response": EMPTY_RESPONSE, + }, + "log": {"level": "debug"}, + "url": {"full": "https://airbyte.io/"}, + }, ), ( "test_get_request_with_headers_params_and_body", @@ -98,7 +142,21 @@ def build(self): {"p1": "v1", "p2": "v2"}, {"b1": "v1", "b2": "v2"}, {}, - {"airbyte_cdk": {"stream": {"name": A_STREAM_NAME}}, "http": {"title": A_TITLE, "description": A_DESCRIPTION, "request": {"method": "GET", "body": {"content": '{"b1": "v1", "b2": "v2"}'}, "headers": {"Content-Type": "application/json", "Content-Length": "24", "h1": "v1"}}, "response": EMPTY_RESPONSE}, "log": {"level": "debug"}, "url": {"full": "https://airbyte.io/?p1=v1&p2=v2"}}, + { + "airbyte_cdk": {"stream": {"name": A_STREAM_NAME}}, + "http": { + "title": A_TITLE, + "description": A_DESCRIPTION, + "request": { + "method": "GET", + "body": {"content": '{"b1": "v1", "b2": "v2"}'}, + "headers": {"Content-Type": "application/json", "Content-Length": "24", "h1": "v1"}, + }, + "response": EMPTY_RESPONSE, + }, + "log": {"level": "debug"}, + "url": {"full": "https://airbyte.io/?p1=v1&p2=v2"}, + }, ), ( "test_get_request_with_request_body_data", @@ -108,7 +166,21 @@ def build(self): {}, {}, {"b1": "v1", "b2": "v2"}, - {"airbyte_cdk": {"stream": {"name": A_STREAM_NAME}}, "http": {"title": A_TITLE, "description": A_DESCRIPTION, "request": {"method": "GET", "body": {"content": "b1=v1&b2=v2"}, "headers": {"Content-Type": "application/x-www-form-urlencoded", "Content-Length": "11"}}, "response": EMPTY_RESPONSE}, "log": {"level": "debug"}, "url": {"full": "https://airbyte.io/"}}, + { + "airbyte_cdk": {"stream": {"name": A_STREAM_NAME}}, + "http": { + "title": A_TITLE, + "description": A_DESCRIPTION, + "request": { + "method": "GET", + "body": {"content": "b1=v1&b2=v2"}, + "headers": {"Content-Type": "application/x-www-form-urlencoded", "Content-Length": "11"}, + }, + "response": EMPTY_RESPONSE, + }, + "log": {"level": "debug"}, + "url": {"full": "https://airbyte.io/"}, + }, ), ( "test_basic_post_request", @@ -118,7 +190,17 @@ def build(self): {}, {}, {}, - {"airbyte_cdk": {"stream": {"name": A_STREAM_NAME}}, "http": {"title": A_TITLE, "description": A_DESCRIPTION, "request": {"method": "POST", "body": {"content": None}, "headers": {"Content-Length": "0"}}, "response": EMPTY_RESPONSE}, "log": {"level": "debug"}, "url": {"full": "https://airbyte.io/"}} + { + "airbyte_cdk": {"stream": {"name": A_STREAM_NAME}}, + "http": { + "title": A_TITLE, + "description": A_DESCRIPTION, + "request": {"method": "POST", "body": {"content": None}, "headers": {"Content-Length": "0"}}, + "response": EMPTY_RESPONSE, + }, + "log": {"level": "debug"}, + "url": {"full": "https://airbyte.io/"}, + }, ), ], ) @@ -138,33 +220,27 @@ def test_prepared_request_to_airbyte_message(test_name, http_method, url, header @pytest.mark.parametrize( "test_name, response_body, response_headers, status_code, expected_airbyte_message", [ - ( - "test_response_no_body_no_headers", - b"", - {}, - 200, - {"body": {"content": ""}, "headers": {}, "status_code": 200} - ), + ("test_response_no_body_no_headers", b"", {}, 200, {"body": {"content": ""}, "headers": {}, "status_code": 200}), ( "test_response_no_body_with_headers", b"", {"h1": "v1", "h2": "v2"}, 200, - {"body": {"content": ""}, "headers": {"h1": "v1", "h2": "v2"}, "status_code": 200} + {"body": {"content": ""}, "headers": {"h1": "v1", "h2": "v2"}, "status_code": 200}, ), ( "test_response_with_body_no_headers", b'{"b1": "v1", "b2": "v2"}', {}, 200, - {"body": {"content": '{"b1": "v1", "b2": "v2"}'}, "headers": {}, "status_code": 200} + {"body": {"content": '{"b1": "v1", "b2": "v2"}'}, "headers": {}, "status_code": 200}, ), ( "test_response_with_body_and_headers", b'{"b1": "v1", "b2": "v2"}', {"h1": "v1", "h2": "v2"}, 200, - {"body": {"content": '{"b1": "v1", "b2": "v2"}'}, "headers": {"h1": "v1", "h2": "v2"}, "status_code": 200} + {"body": {"content": '{"b1": "v1", "b2": "v2"}'}, "headers": {"h1": "v1", "h2": "v2"}, "status_code": 200}, ), ], ) diff --git a/airbyte-cdk/python/unit_tests/sources/test_integration_source.py b/airbyte-cdk/python/unit_tests/sources/test_integration_source.py index 01f58b5adec7..048864f12c90 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_integration_source.py +++ b/airbyte-cdk/python/unit_tests/sources/test_integration_source.py @@ -9,6 +9,7 @@ import pytest import requests from airbyte_cdk.entrypoint import launch +from airbyte_cdk.utils import AirbyteTracedException from unit_tests.sources.fixtures.source_test_fixture import ( HttpTestStream, SourceFixtureOauthAuthenticator, @@ -22,19 +23,19 @@ [ pytest.param("CLOUD", "https://airbyte.com/api/v1/", [], None, id="test_cloud_read_with_public_endpoint"), pytest.param("CLOUD", "http://unsecured.com/api/v1/", [], ValueError, id="test_cloud_read_with_unsecured_url"), - pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], ValueError, id="test_cloud_read_with_private_endpoint"), - pytest.param("CLOUD", "https://localhost:80/api/v1/", [], ValueError, id="test_cloud_read_with_localhost"), + pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], AirbyteTracedException, id="test_cloud_read_with_private_endpoint"), + pytest.param("CLOUD", "https://localhost:80/api/v1/", [], AirbyteTracedException, id="test_cloud_read_with_localhost"), pytest.param("OSS", "https://airbyte.com/api/v1/", [], None, id="test_oss_read_with_public_endpoint"), pytest.param("OSS", "https://172.20.105.99/api/v1/", [], None, id="test_oss_read_with_private_endpoint"), - ] + ], ) @patch.object(requests.Session, "send", fixture_mock_send) def test_external_request_source(capsys, deployment_mode, url_base, expected_records, expected_error): source = SourceTestFixture() with mock.patch.dict(os.environ, {"DEPLOYMENT_MODE": deployment_mode}, clear=False): # clear=True clears the existing os.environ dict - with mock.patch.object(HttpTestStream, 'url_base', url_base): - args = ['read', '--config', 'config.json', '--catalog', 'configured_catalog.json'] + with mock.patch.object(HttpTestStream, "url_base", url_base): + args = ["read", "--config", "config.json", "--catalog", "configured_catalog.json"] if expected_error: with pytest.raises(expected_error): launch(source, args) @@ -47,23 +48,20 @@ def test_external_request_source(capsys, deployment_mode, url_base, expected_rec [ pytest.param("CLOUD", "https://airbyte.com/api/v1/", [], None, id="test_cloud_read_with_public_endpoint"), pytest.param("CLOUD", "http://unsecured.com/api/v1/", [], ValueError, id="test_cloud_read_with_unsecured_url"), - pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], ValueError, id="test_cloud_read_with_private_endpoint"), + pytest.param("CLOUD", "https://172.20.105.99/api/v1/", [], AirbyteTracedException, id="test_cloud_read_with_private_endpoint"), pytest.param("OSS", "https://airbyte.com/api/v1/", [], None, id="test_oss_read_with_public_endpoint"), pytest.param("OSS", "https://172.20.105.99/api/v1/", [], None, id="test_oss_read_with_private_endpoint"), - ] + ], ) @patch.object(requests.Session, "send", fixture_mock_send) def test_external_oauth_request_source(deployment_mode, token_refresh_url, expected_records, expected_error): oauth_authenticator = SourceFixtureOauthAuthenticator( - client_id="nora", - client_secret="hae_sung", - refresh_token="arthur", - token_refresh_endpoint=token_refresh_url + client_id="nora", client_secret="hae_sung", refresh_token="arthur", token_refresh_endpoint=token_refresh_url ) source = SourceTestFixture(authenticator=oauth_authenticator) with mock.patch.dict(os.environ, {"DEPLOYMENT_MODE": deployment_mode}, clear=False): # clear=True clears the existing os.environ dict - args = ['read', '--config', 'config.json', '--catalog', 'configured_catalog.json'] + args = ["read", "--config", "config.json", "--catalog", "configured_catalog.json"] if expected_error: with pytest.raises(expected_error): launch(source, args) diff --git a/airbyte-cdk/python/unit_tests/sources/test_source.py b/airbyte-cdk/python/unit_tests/sources/test_source.py index 1996c56914c7..3657a1c03c14 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_source.py +++ b/airbyte-cdk/python/unit_tests/sources/test_source.py @@ -591,7 +591,7 @@ def __init__(self, *args, **kvargs): f"Skipped syncing stream '{http_stream.name}' because it was unavailable.", f"Unable to read {http_stream.name} stream.", "This is most likely due to insufficient permissions on the credentials in use.", - f"Please visit https://docs.airbyte.com/integrations/sources/{source.name} to learn more." + f"Please visit https://docs.airbyte.com/integrations/sources/{source.name} to learn more.", ] for message in expected_logs: assert message in caplog.text @@ -674,7 +674,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp f"Skipped syncing stream '{http_stream.name}' because it was unavailable.", f"Unable to get slices for {http_stream.name} stream, because of error in parent stream", "This is most likely due to insufficient permissions on the credentials in use.", - f"Please visit https://docs.airbyte.com/integrations/sources/{source.name} to learn more." + f"Please visit https://docs.airbyte.com/integrations/sources/{source.name} to learn more.", ] for message in expected_logs: assert message in caplog.text diff --git a/airbyte-cdk/python/unit_tests/sources/test_source_read.py b/airbyte-cdk/python/unit_tests/sources/test_source_read.py new file mode 100644 index 000000000000..752c4640d3ed --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/test_source_read.py @@ -0,0 +1,461 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import logging +from typing import Any, Iterable, List, Mapping, Optional, Tuple, Union +from unittest.mock import Mock + +import freezegun +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStream, + AirbyteStreamStatus, + AirbyteStreamStatusTraceMessage, + AirbyteTraceMessage, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + StreamDescriptor, + SyncMode, + TraceType, +) +from airbyte_cdk.models import Type as MessageType +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource +from airbyte_cdk.sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter +from airbyte_cdk.sources.message import InMemoryMessageRepository +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade +from airbyte_cdk.sources.streams.concurrent.cursor import NoopCursor +from airbyte_cdk.sources.streams.core import StreamData +from airbyte_cdk.utils import AirbyteTracedException +from unit_tests.sources.streams.concurrent.scenarios.thread_based_concurrent_stream_source_builder import NeverLogSliceLogger + + +class _MockStream(Stream): + def __init__(self, slice_to_records: Mapping[str, List[Mapping[str, Any]]], name: str): + self._slice_to_records = slice_to_records + self._name = name + + @property + def name(self) -> str: + return self._name + + @property + def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: + return None + + def stream_slices( + self, *, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + for partition in self._slice_to_records.keys(): + yield {"partition": partition} + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: Optional[List[str]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + stream_state: Optional[Mapping[str, Any]] = None, + ) -> Iterable[StreamData]: + for record_or_exception in self._slice_to_records[stream_slice["partition"]]: + if isinstance(record_or_exception, Exception): + raise record_or_exception + else: + yield record_or_exception + + def get_json_schema(self) -> Mapping[str, Any]: + return {} + + +class _MockSource(AbstractSource): + message_repository = InMemoryMessageRepository() + + def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: + pass + + def set_streams(self, streams): + self._streams = streams + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + return self._streams + + +class _MockConcurrentSource(ConcurrentSourceAdapter): + message_repository = InMemoryMessageRepository() + + def __init__(self, logger): + concurrent_source = ConcurrentSource.create(1, 1, logger, NeverLogSliceLogger(), self.message_repository) + super().__init__(concurrent_source) + + def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: + pass + + def set_streams(self, streams): + self._streams = streams + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + return self._streams + + +@freezegun.freeze_time("2020-01-01T00:00:00") +def test_concurrent_source_yields_the_same_messages_as_abstract_source_when_no_exceptions_are_raised(): + records_stream_1_partition_1 = [ + {"id": 1, "partition": "1"}, + {"id": 2, "partition": "1"}, + ] + records_stream_1_partition_2 = [ + {"id": 3, "partition": "2"}, + {"id": 4, "partition": "2"}, + ] + records_stream_2_partition_1 = [ + {"id": 100, "partition": "A"}, + {"id": 200, "partition": "A"}, + ] + records_stream_2_partition_2 = [ + {"id": 300, "partition": "B"}, + {"id": 400, "partition": "B"}, + ] + stream_1_slice_to_partition = {"1": records_stream_1_partition_1, "2": records_stream_1_partition_2} + stream_2_slice_to_partition = {"A": records_stream_2_partition_1, "B": records_stream_2_partition_2} + state = None + logger = _init_logger() + + source, concurrent_source = _init_sources([stream_1_slice_to_partition, stream_2_slice_to_partition], state, logger) + + config = {} + catalog = _create_configured_catalog(source._streams) + messages_from_abstract_source = _read_from_source(source, logger, config, catalog, state, None) + messages_from_concurrent_source = _read_from_source(concurrent_source, logger, config, catalog, state, None) + + expected_messages = [ + AirbyteMessage( + type=MessageType.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1577836800000.0, + error=None, + estimate=None, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="stream0"), status=AirbyteStreamStatus(AirbyteStreamStatus.STARTED) + ), + ), + ), + AirbyteMessage( + type=MessageType.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1577836800000.0, + error=None, + estimate=None, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="stream0"), status=AirbyteStreamStatus(AirbyteStreamStatus.RUNNING) + ), + ), + ), + AirbyteMessage( + type=MessageType.RECORD, + record=AirbyteRecordMessage( + stream="stream0", + data=records_stream_1_partition_1[0], + emitted_at=1577836800000, + ), + ), + AirbyteMessage( + type=MessageType.RECORD, + record=AirbyteRecordMessage( + stream="stream0", + data=records_stream_1_partition_1[1], + emitted_at=1577836800000, + ), + ), + AirbyteMessage( + type=MessageType.RECORD, + record=AirbyteRecordMessage( + stream="stream0", + data=records_stream_1_partition_2[0], + emitted_at=1577836800000, + ), + ), + AirbyteMessage( + type=MessageType.RECORD, + record=AirbyteRecordMessage( + stream="stream0", + data=records_stream_1_partition_2[1], + emitted_at=1577836800000, + ), + ), + AirbyteMessage( + type=MessageType.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1577836800000.0, + error=None, + estimate=None, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="stream0"), status=AirbyteStreamStatus(AirbyteStreamStatus.COMPLETE) + ), + ), + ), + AirbyteMessage( + type=MessageType.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1577836800000.0, + error=None, + estimate=None, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="stream1"), status=AirbyteStreamStatus(AirbyteStreamStatus.STARTED) + ), + ), + ), + AirbyteMessage( + type=MessageType.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1577836800000.0, + error=None, + estimate=None, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="stream1"), status=AirbyteStreamStatus(AirbyteStreamStatus.RUNNING) + ), + ), + ), + AirbyteMessage( + type=MessageType.RECORD, + record=AirbyteRecordMessage( + stream="stream1", + data=records_stream_2_partition_1[0], + emitted_at=1577836800000, + ), + ), + AirbyteMessage( + type=MessageType.RECORD, + record=AirbyteRecordMessage( + stream="stream1", + data=records_stream_2_partition_1[1], + emitted_at=1577836800000, + ), + ), + AirbyteMessage( + type=MessageType.RECORD, + record=AirbyteRecordMessage( + stream="stream1", + data=records_stream_2_partition_2[0], + emitted_at=1577836800000, + ), + ), + AirbyteMessage( + type=MessageType.RECORD, + record=AirbyteRecordMessage( + stream="stream1", + data=records_stream_2_partition_2[1], + emitted_at=1577836800000, + ), + ), + AirbyteMessage( + type=MessageType.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1577836800000.0, + error=None, + estimate=None, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="stream1"), status=AirbyteStreamStatus(AirbyteStreamStatus.COMPLETE) + ), + ), + ), + ] + _verify_messages(expected_messages, messages_from_abstract_source, messages_from_concurrent_source) + + +@freezegun.freeze_time("2020-01-01T00:00:00") +def test_concurrent_source_yields_the_same_messages_as_abstract_source_when_a_traced_exception_is_raised(): + records = [{"id": 1, "partition": "1"}, AirbyteTracedException()] + stream_slice_to_partition = {"1": records} + + logger = _init_logger() + state = None + source, concurrent_source = _init_sources([stream_slice_to_partition], state, logger) + config = {} + catalog = _create_configured_catalog(source._streams) + messages_from_abstract_source = _read_from_source(source, logger, config, catalog, state, AirbyteTracedException) + messages_from_concurrent_source = _read_from_source(concurrent_source, logger, config, catalog, state, AirbyteTracedException) + + expected_messages = [ + AirbyteMessage( + type=MessageType.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1577836800000.0, + error=None, + estimate=None, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="stream0"), status=AirbyteStreamStatus(AirbyteStreamStatus.STARTED) + ), + ), + ), + AirbyteMessage( + type=MessageType.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1577836800000.0, + error=None, + estimate=None, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="stream0"), status=AirbyteStreamStatus(AirbyteStreamStatus.RUNNING) + ), + ), + ), + AirbyteMessage( + type=MessageType.RECORD, + record=AirbyteRecordMessage( + stream="stream0", + data=records[0], + emitted_at=1577836800000, + ), + ), + AirbyteMessage( + type=MessageType.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1577836800000.0, + error=None, + estimate=None, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="stream0"), status=AirbyteStreamStatus(AirbyteStreamStatus.INCOMPLETE) + ), + ), + ), + ] + _verify_messages(expected_messages, messages_from_abstract_source, messages_from_concurrent_source) + + +@freezegun.freeze_time("2020-01-01T00:00:00") +def test_concurrent_source_yields_the_same_messages_as_abstract_source_when_an_exception_is_raised(): + records = [{"id": 1, "partition": "1"}, RuntimeError()] + stream_slice_to_partition = {"1": records} + logger = _init_logger() + + state = None + + source, concurrent_source = _init_sources([stream_slice_to_partition], state, logger) + config = {} + catalog = _create_configured_catalog(source._streams) + messages_from_abstract_source = _read_from_source(source, logger, config, catalog, state, RuntimeError) + messages_from_concurrent_source = _read_from_source(concurrent_source, logger, config, catalog, state, RuntimeError) + + expected_messages = [ + AirbyteMessage( + type=MessageType.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1577836800000.0, + error=None, + estimate=None, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="stream0"), status=AirbyteStreamStatus(AirbyteStreamStatus.STARTED) + ), + ), + ), + AirbyteMessage( + type=MessageType.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1577836800000.0, + error=None, + estimate=None, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="stream0"), status=AirbyteStreamStatus(AirbyteStreamStatus.RUNNING) + ), + ), + ), + AirbyteMessage( + type=MessageType.RECORD, + record=AirbyteRecordMessage( + stream="stream0", + data=records[0], + emitted_at=1577836800000, + ), + ), + AirbyteMessage( + type=MessageType.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=1577836800000.0, + error=None, + estimate=None, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="stream0"), status=AirbyteStreamStatus(AirbyteStreamStatus.INCOMPLETE) + ), + ), + ), + ] + _verify_messages(expected_messages, messages_from_abstract_source, messages_from_concurrent_source) + + +def _init_logger(): + logger = Mock() + logger.level = logging.INFO + logger.isEnabledFor.return_value = False + return logger + + +def _init_sources(stream_slice_to_partitions, state, logger): + source = _init_source(stream_slice_to_partitions, state, logger, _MockSource()) + concurrent_source = _init_source(stream_slice_to_partitions, state, logger, _MockConcurrentSource(logger)) + return source, concurrent_source + + +def _init_source(stream_slice_to_partitions, state, logger, source): + cursor = NoopCursor() + streams = [ + StreamFacade.create_from_stream(_MockStream(stream_slices, f"stream{i}"), source, logger, state, cursor) + for i, stream_slices in enumerate(stream_slice_to_partitions) + ] + source.set_streams(streams) + return source + + +def _create_configured_catalog(streams): + return ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream(name=s.name, json_schema={}, supported_sync_modes=[SyncMode.full_refresh]), + sync_mode=SyncMode.full_refresh, + cursor_field=None, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + for s in streams + ] + ) + + +def _read_from_source(source, logger, config, catalog, state, expected_exception): + messages = [] + try: + for m in source.read(logger, config, catalog, state): + messages.append(m) + except Exception as e: + if expected_exception: + assert isinstance(e, expected_exception) + return messages + + +def _verify_messages(expected_messages, messages_from_abstract_source, messages_from_concurrent_source): + assert _compare(expected_messages, messages_from_concurrent_source) + + +def _compare(s, t): + # Use a compare method that does not require ordering or hashing the elements + # We can't rely on the ordering because of the multithreading + # AirbyteMessage does not implement __eq__ and __hash__ + t = list(t) + try: + for elem in s: + t.remove(elem) + except ValueError: + print(f"ValueError: {elem}") + return False + return not t diff --git a/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py b/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py index eb3b87c7c7cf..0b76f5eef5c2 100644 --- a/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py +++ b/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py @@ -13,8 +13,9 @@ from pathlib import Path import jsonref +import pytest from airbyte_cdk.models.airbyte_protocol import ConnectorSpecification, FailureType -from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader, check_config_against_spec_or_exit +from airbyte_cdk.sources.utils.schema_helpers import InternalConfig, ResourceSchemaLoader, check_config_against_spec_or_exit from airbyte_cdk.utils.traced_exception import AirbyteTracedException from pytest import fixture from pytest import raises as pytest_raises @@ -189,3 +190,17 @@ def test_shared_schemas_resolves_nested(): # Make sure generated schema is JSON serializable assert json.dumps(actual_schema) assert jsonref.JsonRef.replace_refs(actual_schema) + + +@pytest.mark.parametrize( + "limit, record_count, expected", + [ + pytest.param(None, sys.maxsize, False, id="test_no_limit"), + pytest.param(1, 1, True, id="test_record_count_is_exactly_the_limit"), + pytest.param(1, 2, True, id="test_record_count_is_more_than_the_limit"), + pytest.param(1, 0, False, id="test_record_count_is_less_than_the_limit"), + ], +) +def test_internal_config(limit, record_count, expected): + config = InternalConfig(_limit=limit) + assert config.is_limit_reached(record_count) == expected diff --git a/airbyte-cdk/python/unit_tests/sources/utils/test_slice_logger.py b/airbyte-cdk/python/unit_tests/sources/utils/test_slice_logger.py new file mode 100644 index 000000000000..0796e8769179 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/utils/test_slice_logger.py @@ -0,0 +1,46 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging + +import pytest +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level +from airbyte_cdk.models import Type as MessageType +from airbyte_cdk.sources.utils.slice_logger import AlwaysLogSliceLogger, DebugSliceLogger + + +@pytest.mark.parametrize( + "slice_logger, level, should_log", + [ + pytest.param(DebugSliceLogger(), logging.DEBUG, True, id="debug_logger_should_log_if_level_is_debug"), + pytest.param(DebugSliceLogger(), logging.INFO, False, id="debug_logger_should_not_log_if_level_is_info"), + pytest.param(DebugSliceLogger(), logging.WARN, False, id="debug_logger_should_not_log_if_level_is_warn"), + pytest.param(DebugSliceLogger(), logging.WARNING, False, id="debug_logger_should_not_log_if_level_is_warning"), + pytest.param(DebugSliceLogger(), logging.ERROR, False, id="debug_logger_should_not_log_if_level_is_error"), + pytest.param(DebugSliceLogger(), logging.CRITICAL, False, id="always_log_logger_should_not_log_if_level_is_critical"), + pytest.param(AlwaysLogSliceLogger(), logging.DEBUG, True, id="always_log_logger_should_log_if_level_is_debug"), + pytest.param(AlwaysLogSliceLogger(), logging.INFO, True, id="always_log_logger_should_log_if_level_is_info"), + pytest.param(AlwaysLogSliceLogger(), logging.WARN, True, id="always_log_logger_should_log_if_level_is_warn"), + pytest.param(AlwaysLogSliceLogger(), logging.WARNING, True, id="always_log_logger_should_log_if_level_is_warning"), + pytest.param(AlwaysLogSliceLogger(), logging.ERROR, True, id="always_log_logger_should_log_if_level_is_error"), + pytest.param(AlwaysLogSliceLogger(), logging.CRITICAL, True, id="always_log_logger_should_log_if_level_is_critical"), + ], +) +def test_should_log_slice_message(slice_logger, level, should_log): + logger = logging.Logger(name="name", level=level) + assert slice_logger.should_log_slice_message(logger) == should_log + + +@pytest.mark.parametrize( + "_slice, expected_message", + [ + pytest.param(None, "slice:null", id="test_none_slice"), + pytest.param({}, "slice:{}", id="test_empty_slice"), + pytest.param({"key": "value"}, 'slice:{"key": "value"}', id="test_dict"), + ], +) +def test_create_slice_log_message(_slice, expected_message): + expected_log_message = AirbyteMessage(type=MessageType.LOG, log=AirbyteLogMessage(level=Level.INFO, message=expected_message)) + log_message = DebugSliceLogger().create_slice_log_message(_slice) + assert log_message == expected_log_message diff --git a/airbyte-cdk/python/unit_tests/test/__init__.py b/airbyte-cdk/python/unit_tests/test/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-cdk/python/unit_tests/test/mock_http/__init__.py b/airbyte-cdk/python/unit_tests/test/mock_http/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-cdk/python/unit_tests/test/mock_http/test_matcher.py b/airbyte-cdk/python/unit_tests/test/mock_http/test_matcher.py new file mode 100644 index 000000000000..61a9ecfec2f9 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/test/mock_http/test_matcher.py @@ -0,0 +1,53 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from unittest import TestCase +from unittest.mock import Mock + +from airbyte_cdk.test.mock_http.matcher import HttpRequestMatcher +from airbyte_cdk.test.mock_http.request import HttpRequest + + +class HttpRequestMatcherTest(TestCase): + def setUp(self) -> None: + self._a_request = Mock(spec=HttpRequest) + self._another_request = Mock(spec=HttpRequest) + self._request_to_match = Mock(spec=HttpRequest) + self._matcher = HttpRequestMatcher(self._request_to_match, 1) + + def test_given_request_matches_when_matches_then_has_expected_match_count(self): + self._a_request.matches.return_value = True + self._matcher.matches(self._a_request) + assert self._matcher.has_expected_match_count() + + def test_given_request_does_not_match_when_matches_then_does_not_have_expected_match_count(self): + self._a_request.matches.return_value = False + self._matcher.matches(self._a_request) + + assert not self._matcher.has_expected_match_count() + assert self._matcher.actual_number_of_matches == 0 + + def test_given_many_requests_with_some_match_when_matches_then_has_expected_match_count(self): + self._a_request.matches.return_value = True + self._another_request.matches.return_value = False + self._matcher.matches(self._a_request) + self._matcher.matches(self._another_request) + + assert self._matcher.has_expected_match_count() + assert self._matcher.actual_number_of_matches == 1 + + def test_given_expected_number_of_requests_met_when_matches_then_has_expected_match_count(self): + _matcher = HttpRequestMatcher(self._request_to_match, 2) + self._a_request.matches.return_value = True + _matcher.matches(self._a_request) + _matcher.matches(self._a_request) + + assert _matcher.has_expected_match_count() + assert _matcher.actual_number_of_matches == 2 + + def test_given_expected_number_of_requests_not_met_when_matches_then_does_not_have_expected_match_count(self): + _matcher = HttpRequestMatcher(self._request_to_match, 2) + self._a_request.matches.side_effect = [True, False] + _matcher.matches(self._a_request) + _matcher.matches(self._a_request) + + assert not _matcher.has_expected_match_count() diff --git a/airbyte-cdk/python/unit_tests/test/mock_http/test_mocker.py b/airbyte-cdk/python/unit_tests/test/mock_http/test_mocker.py new file mode 100644 index 000000000000..cec689566e90 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/test/mock_http/test_mocker.py @@ -0,0 +1,212 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from unittest import TestCase + +import pytest +import requests +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse + +# Ensure that the scheme is HTTP as requests only partially supports other schemes +# see https://github.com/psf/requests/blob/0b4d494192de489701d3a2e32acef8fb5d3f042e/src/requests/models.py#L424-L429 +_A_URL = "http://test.com/" +_ANOTHER_URL = "http://another-test.com/" +_A_RESPONSE_BODY = "a body" +_ANOTHER_RESPONSE_BODY = "another body" +_A_RESPONSE = HttpResponse("any response") +_SOME_QUERY_PARAMS = {"q1": "query value"} +_SOME_HEADERS = {"h1": "header value"} +_SOME_REQUEST_BODY_MAPPING = {"first_field": "first_value", "second_field": 2} +_SOME_REQUEST_BODY_STR = "some_request_body" + + +class HttpMockerTest(TestCase): + @HttpMocker() + def test_given_get_request_match_when_decorate_then_return_response(self, http_mocker): + http_mocker.get( + HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS), + HttpResponse(_A_RESPONSE_BODY, 474), + ) + + response = requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS) + + assert response.text == _A_RESPONSE_BODY + assert response.status_code == 474 + + @HttpMocker() + def test_given_loose_headers_matching_when_decorate_then_match(self, http_mocker): + http_mocker.get( + HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS), + HttpResponse(_A_RESPONSE_BODY, 474), + ) + + requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS | {"more strict query param key": "any value"}) + + @HttpMocker() + def test_given_post_request_match_when_decorate_then_return_response(self, http_mocker): + http_mocker.post( + HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS, _SOME_REQUEST_BODY_STR), + HttpResponse(_A_RESPONSE_BODY, 474), + ) + + response = requests.post(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS, data=_SOME_REQUEST_BODY_STR) + + assert response.text == _A_RESPONSE_BODY + assert response.status_code == 474 + + @HttpMocker() + def test_given_multiple_responses_when_decorate_get_request_then_return_response(self, http_mocker): + http_mocker.get( + HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS), + [HttpResponse(_A_RESPONSE_BODY, 1), HttpResponse(_ANOTHER_RESPONSE_BODY, 2)], + ) + + first_response = requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS) + second_response = requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS) + + assert first_response.text == _A_RESPONSE_BODY + assert first_response.status_code == 1 + assert second_response.text == _ANOTHER_RESPONSE_BODY + assert second_response.status_code == 2 + + @HttpMocker() + def test_given_multiple_responses_when_decorate_post_request_then_return_response(self, http_mocker): + http_mocker.post( + HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS, _SOME_REQUEST_BODY_STR), + [HttpResponse(_A_RESPONSE_BODY, 1), HttpResponse(_ANOTHER_RESPONSE_BODY, 2)], + ) + + first_response = requests.post(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS, data=_SOME_REQUEST_BODY_STR) + second_response = requests.post(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS, data=_SOME_REQUEST_BODY_STR) + + assert first_response.text == _A_RESPONSE_BODY + assert first_response.status_code == 1 + assert second_response.text == _ANOTHER_RESPONSE_BODY + assert second_response.status_code == 2 + + @HttpMocker() + def test_given_more_requests_than_responses_when_decorate_then_raise_error(self, http_mocker): + http_mocker.get( + HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS), + [HttpResponse(_A_RESPONSE_BODY, 1), HttpResponse(_ANOTHER_RESPONSE_BODY, 2)], + ) + + last_response = [requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS) for _ in range(10)][-1] + + assert last_response.text == _ANOTHER_RESPONSE_BODY + assert last_response.status_code == 2 + + @HttpMocker() + def test_given_all_requests_match_when_decorate_then_do_not_raise(self, http_mocker): + http_mocker.get( + HttpRequest(_A_URL, _SOME_QUERY_PARAMS, _SOME_HEADERS), + _A_RESPONSE, + ) + requests.get(_A_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS) + + def test_given_missing_requests_when_decorate_then_raise(self): + @HttpMocker() + def decorated_function(http_mocker): + http_mocker.get( + HttpRequest(_A_URL), + _A_RESPONSE, + ) + + with pytest.raises(ValueError) as exc_info: + decorated_function() + assert "Invalid number of matches" in str(exc_info.value) + + def test_given_assertion_error_when_decorate_then_raise_assertion_error(self): + @HttpMocker() + def decorated_function(http_mocker): + http_mocker.get( + HttpRequest(_A_URL), + _A_RESPONSE, + ) + requests.get(_A_URL) + assert False + + with pytest.raises(AssertionError): + decorated_function() + + def test_given_assertion_error_but_missing_request_when_decorate_then_raise_missing_http_request(self): + @HttpMocker() + def decorated_function(http_mocker): + http_mocker.get( + HttpRequest(_A_URL), + _A_RESPONSE, + ) + assert False + + with pytest.raises(ValueError) as exc_info: + decorated_function() + assert "Invalid number of matches" in str(exc_info.value) + + def test_given_request_does_not_match_when_decorate_then_raise(self): + @HttpMocker() + def decorated_function(http_mocker): + http_mocker.get( + HttpRequest(_A_URL), + _A_RESPONSE, + ) + requests.get(_ANOTHER_URL, params=_SOME_QUERY_PARAMS, headers=_SOME_HEADERS) + + with pytest.raises(ValueError) as exc_info: + decorated_function() + assert "No matcher matches" in str(exc_info.value) + + def test_given_request_matches_multiple_matchers_when_decorate_then_match_first_one(self): + less_granular_headers = {"less_granular": "1"} + more_granular_headers = {"more_granular": "2"} | less_granular_headers + + @HttpMocker() + def decorated_function(http_mocker): + http_mocker.get( + HttpRequest(_A_URL, headers=more_granular_headers), + _A_RESPONSE, + ) + http_mocker.get( + HttpRequest(_A_URL, headers=less_granular_headers), + _A_RESPONSE, + ) + requests.get(_A_URL, headers=more_granular_headers) + + with pytest.raises(ValueError) as exc_info: + decorated_function() + assert "more_granular" in str(exc_info.value) # the matcher corresponding to the first `http_mocker.get` is not matched + + def test_given_exact_number_of_call_provided_when_assert_number_of_calls_then_do_not_raise(self): + @HttpMocker() + def decorated_function(http_mocker): + request = HttpRequest(_A_URL) + http_mocker.get(request, _A_RESPONSE) + + requests.get(_A_URL) + requests.get(_A_URL) + + http_mocker.assert_number_of_calls(request, 2) + + decorated_function() + # then do not raise + + def test_given_invalid_number_of_call_provided_when_assert_number_of_calls_then_raise(self): + @HttpMocker() + def decorated_function(http_mocker): + request = HttpRequest(_A_URL) + http_mocker.get(request, _A_RESPONSE) + + requests.get(_A_URL) + requests.get(_A_URL) + + http_mocker.assert_number_of_calls(request, 1) + + with pytest.raises(AssertionError): + decorated_function() + + def test_given_unknown_request_when_assert_number_of_calls_then_raise(self): + @HttpMocker() + def decorated_function(http_mocker): + http_mocker.get(HttpRequest(_A_URL), _A_RESPONSE) + http_mocker.assert_number_of_calls(HttpRequest(_ANOTHER_URL), 1) + + with pytest.raises(ValueError): + decorated_function() diff --git a/airbyte-cdk/python/unit_tests/test/mock_http/test_request.py b/airbyte-cdk/python/unit_tests/test/mock_http/test_request.py new file mode 100644 index 000000000000..a5a94ea05580 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/test/mock_http/test_request.py @@ -0,0 +1,117 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from unittest import TestCase + +import pytest +from airbyte_cdk.test.mock_http.request import ANY_QUERY_PARAMS, HttpRequest + + +class HttpRequestMatcherTest(TestCase): + def test_given_query_params_as_dict_and_string_then_query_params_are_properly_considered(self): + with_string = HttpRequest("mock://test.com/path", query_params="a_query_param=q1&a_list_param=first&a_list_param=second") + with_dict = HttpRequest("mock://test.com/path", query_params={"a_query_param": "q1", "a_list_param": ["first", "second"]}) + assert with_string.matches(with_dict) and with_dict.matches(with_string) + + def test_given_query_params_in_url_and_also_provided_then_raise_error(self): + with pytest.raises(ValueError): + HttpRequest("mock://test.com/path?a_query_param=1", query_params={"another_query_param": "2"}) + + def test_given_same_url_query_params_and_subset_headers_when_matches_then_return_true(self): + request_to_match = HttpRequest("mock://test.com/path", {"a_query_param": "q1"}, {"first_header": "h1"}) + actual_request = HttpRequest("mock://test.com/path", {"a_query_param": "q1"}, {"first_header": "h1", "second_header": "h2"}) + assert actual_request.matches(request_to_match) + + def test_given_url_differs_when_matches_then_return_false(self): + assert not HttpRequest("mock://test.com/another_path").matches(HttpRequest("mock://test.com/path")) + + def test_given_query_params_differs_when_matches_then_return_false(self): + request_to_match = HttpRequest("mock://test.com/path", {"a_query_param": "q1"}) + actual_request = HttpRequest("mock://test.com/path", {"another_query_param": "q2"}) + assert not actual_request.matches(request_to_match) + + def test_given_query_params_is_subset_differs_when_matches_then_return_false(self): + request_to_match = HttpRequest("mock://test.com/path", {"a_query_param": "q1"}) + actual_request = HttpRequest("mock://test.com/path", {"a_query_param": "q1", "another_query_param": "q2"}) + assert not actual_request.matches(request_to_match) + + def test_given_headers_is_subset_differs_when_matches_then_return_true(self): + request_to_match = HttpRequest("mock://test.com/path", headers={"first_header": "h1"}) + actual_request = HttpRequest("mock://test.com/path", headers={"first_header": "h1", "second_header": "h2"}) + assert actual_request.matches(request_to_match) + + def test_given_headers_value_does_not_match_differs_when_matches_then_return_false(self): + request_to_match = HttpRequest("mock://test.com/path", headers={"first_header": "h1"}) + actual_request = HttpRequest("mock://test.com/path", headers={"first_header": "value does not match"}) + assert not actual_request.matches(request_to_match) + + def test_given_same_body_mappings_value_when_matches_then_return_true(self): + request_to_match = HttpRequest("mock://test.com/path", body={"first_field": "first_value", "second_field": 2}) + actual_request = HttpRequest("mock://test.com/path", body={"first_field": "first_value", "second_field": 2}) + assert actual_request.matches(request_to_match) + + def test_given_bodies_are_mapping_and_differs_when_matches_then_return_false(self): + request_to_match = HttpRequest("mock://test.com/path", body={"first_field": "first_value"}) + actual_request = HttpRequest("mock://test.com/path", body={"first_field": "value does not match"}) + assert not actual_request.matches(request_to_match) + + def test_given_same_mapping_and_bytes_when_matches_then_return_true(self): + request_to_match = HttpRequest("mock://test.com/path", body={"first_field": "first_value"}) + actual_request = HttpRequest("mock://test.com/path", body=b'{"first_field": "first_value"}') + assert actual_request.matches(request_to_match) + + def test_given_different_mapping_and_bytes_when_matches_then_return_false(self): + request_to_match = HttpRequest("mock://test.com/path", body={"first_field": "first_value"}) + actual_request = HttpRequest("mock://test.com/path", body=b'{"first_field": "another value"}') + assert not actual_request.matches(request_to_match) + + def test_given_same_mapping_and_str_when_matches_then_return_true(self): + request_to_match = HttpRequest("mock://test.com/path", body={"first_field": "first_value"}) + actual_request = HttpRequest("mock://test.com/path", body='{"first_field": "first_value"}') + assert actual_request.matches(request_to_match) + + def test_given_different_mapping_and_str_when_matches_then_return_false(self): + request_to_match = HttpRequest("mock://test.com/path", body={"first_field": "first_value"}) + actual_request = HttpRequest("mock://test.com/path", body='{"first_field": "another value"}') + assert not actual_request.matches(request_to_match) + + def test_given_same_bytes_and_mapping_when_matches_then_return_true(self): + request_to_match = HttpRequest("mock://test.com/path", body=b'{"first_field": "first_value"}') + actual_request = HttpRequest("mock://test.com/path", body={"first_field": "first_value"}) + assert actual_request.matches(request_to_match) + + def test_given_different_bytes_and_mapping_when_matches_then_return_false(self): + request_to_match = HttpRequest("mock://test.com/path", body=b'{"first_field": "first_value"}') + actual_request = HttpRequest("mock://test.com/path", body={"first_field": "another value"}) + assert not actual_request.matches(request_to_match) + + def test_given_same_str_and_mapping_when_matches_then_return_true(self): + request_to_match = HttpRequest("mock://test.com/path", body='{"first_field": "first_value"}') + actual_request = HttpRequest("mock://test.com/path", body={"first_field": "first_value"}) + assert actual_request.matches(request_to_match) + + def test_given_different_str_and_mapping_when_matches_then_return_false(self): + request_to_match = HttpRequest("mock://test.com/path", body='{"first_field": "first_value"}') + actual_request = HttpRequest("mock://test.com/path", body={"first_field": "another value"}) + assert not actual_request.matches(request_to_match) + + def test_given_same_body_str_value_when_matches_then_return_true(self): + request_to_match = HttpRequest("mock://test.com/path", body="some_request_body") + actual_request = HttpRequest("mock://test.com/path", body="some_request_body") + assert actual_request.matches(request_to_match) + + def test_given_body_str_value_differs_when_matches_then_return_false(self): + request_to_match = HttpRequest("mock://test.com/path", body="some_request_body") + actual_request = HttpRequest("mock://test.com/path", body="another_request_body") + assert not actual_request.matches(request_to_match) + + def test_given_any_matcher_for_query_param_when_matches_then_return_true(self): + request_to_match = HttpRequest("mock://test.com/path", {"a_query_param": "q1"}) + actual_request = HttpRequest("mock://test.com/path", ANY_QUERY_PARAMS) + + assert actual_request.matches(request_to_match) + assert request_to_match.matches(actual_request) + + def test_given_any_matcher_for_both_when_matches_then_return_true(self): + request_to_match = HttpRequest("mock://test.com/path", ANY_QUERY_PARAMS) + actual_request = HttpRequest("mock://test.com/path", ANY_QUERY_PARAMS) + assert actual_request.matches(request_to_match) diff --git a/airbyte-cdk/python/unit_tests/test/mock_http/test_response_builder.py b/airbyte-cdk/python/unit_tests/test/mock_http/test_response_builder.py new file mode 100644 index 000000000000..328db535ca36 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/test/mock_http/test_response_builder.py @@ -0,0 +1,177 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +import json +from copy import deepcopy +from pathlib import Path as FilePath +from typing import Any, Dict, Optional, Union +from unittest import TestCase +from unittest.mock import Mock + +import pytest +from airbyte_cdk.test.mock_http.response import HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + FieldUpdatePaginationStrategy, + HttpResponseBuilder, + NestedPath, + PaginationStrategy, + Path, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) + +_RECORDS_FIELD = "records_field" +_ID_FIELD = "record_id" +_CURSOR_FIELD = "record_cursor" +_ANY_RECORD = {"a_record_field": "a record value"} +_SOME_RECORDS = {_RECORDS_FIELD: [_ANY_RECORD]} +_A_RESPONSE_TEMPLATE = _SOME_RECORDS + +_RECORD_BUILDER = 0 +_RESPONSE_BUILDER = 1 + + +def _record_builder( + response_template: Dict[str, Any], + records_path: Union[FieldPath, NestedPath], + record_id_path: Optional[Path] = None, + record_cursor_path: Optional[Union[FieldPath, NestedPath]] = None, +) -> RecordBuilder: + return create_record_builder(deepcopy(response_template), records_path, record_id_path, record_cursor_path) + + +def _any_record_builder() -> RecordBuilder: + return create_record_builder({"record_path": [{"a_record": "record value"}]}, FieldPath("record_path")) + + +def _response_builder( + response_template: Dict[str, Any], + records_path: Union[FieldPath, NestedPath], + pagination_strategy: Optional[PaginationStrategy] = None +) -> HttpResponseBuilder: + return create_response_builder(deepcopy(response_template), records_path, pagination_strategy=pagination_strategy) + + +def _body(response: HttpResponse) -> Dict[str, Any]: + return json.loads(response.body) + + +class RecordBuilderTest(TestCase): + def test_given_with_id_when_build_then_set_id(self) -> None: + builder = _record_builder({_RECORDS_FIELD: [{_ID_FIELD: "an id"}]}, FieldPath(_RECORDS_FIELD), FieldPath(_ID_FIELD)) + record = builder.with_id("another id").build() + assert record[_ID_FIELD] == "another id" + + def test_given_nested_id_when_build_then_set_id(self) -> None: + builder = _record_builder({_RECORDS_FIELD: [{"nested": {_ID_FIELD: "id"}}]}, FieldPath(_RECORDS_FIELD), NestedPath(["nested", _ID_FIELD])) + record = builder.with_id("another id").build() + assert record["nested"][_ID_FIELD] == "another id" + + def test_given_id_path_not_provided_but_with_id_when_build_then_raise_error(self) -> None: + builder = _record_builder(_A_RESPONSE_TEMPLATE, FieldPath(_RECORDS_FIELD), None) + with pytest.raises(ValueError): + builder.with_id("any_id").build() + + def test_given_no_id_in_template_for_path_when_build_then_raise_error(self) -> None: + with pytest.raises(ValueError): + _record_builder({_RECORDS_FIELD: [{"record without id": "should fail"}]}, FieldPath(_RECORDS_FIELD), FieldPath(_ID_FIELD)) + + def test_given_with_cursor_when_build_then_set_id(self) -> None: + builder = _record_builder( + {_RECORDS_FIELD: [{_CURSOR_FIELD: "a cursor"}]}, + FieldPath(_RECORDS_FIELD), + record_cursor_path=FieldPath(_CURSOR_FIELD) + ) + record = builder.with_cursor("another cursor").build() + assert record[_CURSOR_FIELD] == "another cursor" + + def test_given_nested_cursor_when_build_then_set_cursor(self) -> None: + builder = _record_builder( + {_RECORDS_FIELD: [{"nested": {_CURSOR_FIELD: "a cursor"}}]}, + FieldPath(_RECORDS_FIELD), + record_cursor_path=NestedPath(["nested", _CURSOR_FIELD]) + ) + record = builder.with_cursor("another cursor").build() + assert record["nested"][_CURSOR_FIELD] == "another cursor" + + def test_given_with_field_when_build_then_write_field(self) -> None: + builder = _any_record_builder() + record = builder.with_field(FieldPath("to_write_field"), "a field value").build() + assert record["to_write_field"] == "a field value" + + def test_given_nested_cursor_when_build_then_write_field(self) -> None: + builder = _any_record_builder() + record = builder.with_field(NestedPath(["path", "to_write_field"]), "a field value").build() + assert record["path"]["to_write_field"] == "a field value" + + def test_given_cursor_path_not_provided_but_with_id_when_build_then_raise_error(self) -> None: + builder = _record_builder(_A_RESPONSE_TEMPLATE, FieldPath(_RECORDS_FIELD)) + with pytest.raises(ValueError): + builder.with_cursor("any cursor").build() + + def test_given_no_cursor_in_template_for_path_when_build_then_raise_error(self) -> None: + with pytest.raises(ValueError): + _record_builder( + {_RECORDS_FIELD: [{"record without cursor": "should fail"}]}, + FieldPath(_RECORDS_FIELD), + record_cursor_path=FieldPath(_ID_FIELD) + ) + + +class HttpResponseBuilderTest(TestCase): + def test_given_records_in_template_but_no_with_records_when_build_then_no_records(self) -> None: + builder = _response_builder({_RECORDS_FIELD: [{"a_record_field": "a record value"}]}, FieldPath(_RECORDS_FIELD)) + response = builder.build() + assert len(_body(response)[_RECORDS_FIELD]) == 0 + + def test_given_many_records_when_build_then_response_has_records(self) -> None: + builder = _response_builder(_A_RESPONSE_TEMPLATE, FieldPath(_RECORDS_FIELD)) + a_record_builder = Mock(spec=RecordBuilder) + a_record_builder.build.return_value = {"a record": 1} + another_record_builder = Mock(spec=RecordBuilder) + another_record_builder.build.return_value = {"another record": 2} + + response = builder.with_record(a_record_builder).with_record(another_record_builder).build() + + assert len(_body(response)[_RECORDS_FIELD]) == 2 + + def test_when_build_then_default_status_code_is_200(self) -> None: + builder = _response_builder(_A_RESPONSE_TEMPLATE, FieldPath(_RECORDS_FIELD)) + response = builder.build() + assert response.status_code == 200 + + def test_given_status_code_when_build_then_status_code_is_set(self) -> None: + builder = _response_builder(_A_RESPONSE_TEMPLATE, FieldPath(_RECORDS_FIELD)) + response = builder.with_status_code(239).build() + assert response.status_code == 239 + + def test_given_pagination_with_strategy_when_build_then_apply_strategy(self) -> None: + builder = _response_builder( + {"has_more_pages": False} | _SOME_RECORDS, + FieldPath(_RECORDS_FIELD), + pagination_strategy=FieldUpdatePaginationStrategy(FieldPath("has_more_pages"), "yes more page") + ) + + response = builder.with_pagination().build() + + assert _body(response)["has_more_pages"] == "yes more page" + + def test_given_no_pagination_strategy_but_pagination_when_build_then_raise_error(self) -> None: + builder = _response_builder(_A_RESPONSE_TEMPLATE, FieldPath(_RECORDS_FIELD)) + with pytest.raises(ValueError): + builder.with_pagination() + + +class UtilMethodsTest(TestCase): + def test_from_resource_file(self) -> None: + template = find_template("test-resource", __file__) + assert template == {"test-source template": "this is a template for test-resource"} + + def test_given_cwd_doesnt_have_unit_tests_as_parent_when_from_resource_file__then_raise_error(self) -> None: + with pytest.raises(ValueError): + find_template("test-resource", str(FilePath(__file__).parent.parent.parent.parent)) + + def test_given_records_path_invalid_when_create_builders_from_resource_then_raise_exception(self) -> None: + with pytest.raises(ValueError): + create_record_builder(_A_RESPONSE_TEMPLATE, NestedPath(["invalid", "record", "path"])) diff --git a/airbyte-cdk/python/unit_tests/test/test_entrypoint_wrapper.py b/airbyte-cdk/python/unit_tests/test/test_entrypoint_wrapper.py new file mode 100644 index 000000000000..9f3256600b50 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/test/test_entrypoint_wrapper.py @@ -0,0 +1,241 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +import logging +import os +from typing import Any, Iterator, List +from unittest import TestCase +from unittest.mock import Mock, patch + +from airbyte_cdk.sources.abstract_source import AbstractSource +from airbyte_cdk.test.entrypoint_wrapper import read +from airbyte_protocol.models import ( + AirbyteAnalyticsTraceMessage, + AirbyteErrorTraceMessage, + AirbyteLogMessage, + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStateMessage, + AirbyteStreamStatus, + AirbyteStreamStatusTraceMessage, + AirbyteTraceMessage, + ConfiguredAirbyteCatalog, + Level, + StreamDescriptor, + TraceType, + Type, +) + + +def _a_state_message(state: Any) -> AirbyteMessage: + return AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage(data=state) + ) + + +def _a_status_message(stream_name: str, status: AirbyteStreamStatus) -> AirbyteMessage: + return AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + emitted_at=0, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name=stream_name), + status=status, + ), + ), + ) + + +_A_RECORD = AirbyteMessage( + type=Type.RECORD, + record=AirbyteRecordMessage(stream="stream", data={"record key": "record value"}, emitted_at=0) +) +_A_STATE_MESSAGE = _a_state_message({"state key": "state value for _A_STATE_MESSAGE"}) +_A_LOG = AirbyteMessage( + type=Type.LOG, + log=AirbyteLogMessage(level=Level.INFO, message="This is an Airbyte log message") +) +_AN_ERROR_MESSAGE = AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.ERROR, + emitted_at=0, + error=AirbyteErrorTraceMessage(message="AirbyteErrorTraceMessage message"), + ), +) +_AN_ANALYTIC_MESSAGE = AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.ANALYTICS, + emitted_at=0, + analytics=AirbyteAnalyticsTraceMessage(type="an analytic type", value="an analytic value"), + ), +) + +_A_STREAM_NAME = "a stream name" +_A_CONFIG = {"config_key": "config_value"} +_A_CATALOG = ConfiguredAirbyteCatalog.parse_obj( + { + "streams": [ + { + "stream": { + "name": "a_stream_name", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append", + } + ] + } +) +_A_STATE = {"state_key": "state_value"} +_A_LOG_MESSAGE = "a log message" + + +def _to_entrypoint_output(messages: List[AirbyteMessage]) -> Iterator[str]: + return (message.json(exclude_unset=True) for message in messages) + + +def _a_mocked_source() -> AbstractSource: + source = Mock(spec=AbstractSource) + source.message_repository = None + return source + + +def _validate_tmp_json_file(expected, file_path) -> None: + with open(file_path) as file: + assert json.load(file) == expected + + +def _validate_tmp_catalog(expected, file_path) -> None: + assert ConfiguredAirbyteCatalog.parse_file(file_path) == expected + + +def _create_tmp_file_validation(entrypoint, expected_config, expected_catalog, expected_state): + def _validate_tmp_files(self): + _validate_tmp_json_file(expected_config, entrypoint.return_value.parse_args.call_args.args[0][2]) + _validate_tmp_catalog(expected_catalog, entrypoint.return_value.parse_args.call_args.args[0][4]) + _validate_tmp_json_file(expected_state, entrypoint.return_value.parse_args.call_args.args[0][6]) + return entrypoint.return_value.run.return_value + return _validate_tmp_files + + +class EntrypointWrapperTest(TestCase): + def setUp(self) -> None: + self._a_source = _a_mocked_source() + + @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") + def test_when_read_then_ensure_parameters(self, entrypoint): + entrypoint.return_value.run.side_effect = _create_tmp_file_validation(entrypoint, _A_CONFIG, _A_CATALOG, _A_STATE) + + read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) + + entrypoint.assert_called_once_with(self._a_source) + entrypoint.return_value.run.assert_called_once_with(entrypoint.return_value.parse_args.return_value) + assert entrypoint.return_value.parse_args.call_count == 1 + assert entrypoint.return_value.parse_args.call_args.args[0][0] == "read" + assert entrypoint.return_value.parse_args.call_args.args[0][1] == "--config" + assert entrypoint.return_value.parse_args.call_args.args[0][3] == "--catalog" + + @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") + def test_when_read_then_ensure_files_are_temporary(self, entrypoint): + read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) + + assert not os.path.exists(entrypoint.return_value.parse_args.call_args.args[0][2]) + assert not os.path.exists(entrypoint.return_value.parse_args.call_args.args[0][4]) + assert not os.path.exists(entrypoint.return_value.parse_args.call_args.args[0][6]) + + @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") + def test_given_logging_during_run_when_read_then_output_has_logs(self, entrypoint): + def _do_some_logging(self): + logging.getLogger("any logger").info(_A_LOG_MESSAGE) + return entrypoint.return_value.run.return_value + entrypoint.return_value.run.side_effect = _do_some_logging + + output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) + + assert len(output.logs) == 1 + assert output.logs[0].log.message == _A_LOG_MESSAGE + + @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") + def test_given_record_when_read_then_output_has_record(self, entrypoint): + entrypoint.return_value.run.return_value = _to_entrypoint_output([_A_RECORD]) + output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) + assert output.records == [_A_RECORD] + + @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") + def test_given_state_message_when_read_then_output_has_state_message(self, entrypoint): + entrypoint.return_value.run.return_value = _to_entrypoint_output([_A_STATE_MESSAGE]) + output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) + assert output.state_messages == [_A_STATE_MESSAGE] + + @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") + def test_given_state_message_and_records_when_read_then_output_has_records_and_state_message(self, entrypoint): + entrypoint.return_value.run.return_value = _to_entrypoint_output([_A_RECORD, _A_STATE_MESSAGE]) + output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) + assert output.records_and_state_messages == [_A_RECORD, _A_STATE_MESSAGE] + + @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") + def test_given_many_state_messages_and_records_when_read_then_output_has_records_and_state_message(self, entrypoint): + last_emitted_state = {"last state key": "last state value"} + entrypoint.return_value.run.return_value = _to_entrypoint_output([_A_STATE_MESSAGE, _a_state_message(last_emitted_state)]) + + output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) + + assert output.most_recent_state == last_emitted_state + + @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") + def test_given_log_when_read_then_output_has_log(self, entrypoint): + entrypoint.return_value.run.return_value = _to_entrypoint_output([_A_LOG]) + output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) + assert output.logs == [_A_LOG] + + @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") + def test_given_trace_message_when_read_then_output_has_trace_messages(self, entrypoint): + entrypoint.return_value.run.return_value = _to_entrypoint_output([_AN_ANALYTIC_MESSAGE]) + output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) + assert output.analytics_messages == [_AN_ANALYTIC_MESSAGE] + + @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") + def test_given_stream_statuses_when_read_then_return_statuses(self, entrypoint): + status_messages = [ + _a_status_message(_A_STREAM_NAME, AirbyteStreamStatus.STARTED), + _a_status_message(_A_STREAM_NAME, AirbyteStreamStatus.COMPLETE) + ] + entrypoint.return_value.run.return_value = _to_entrypoint_output(status_messages) + output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) + assert output.get_stream_statuses(_A_STREAM_NAME) == [AirbyteStreamStatus.STARTED, AirbyteStreamStatus.COMPLETE] + + @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") + def test_given_stream_statuses_for_many_streams_when_read_then_filter_other_streams(self, entrypoint): + status_messages = [ + _a_status_message(_A_STREAM_NAME, AirbyteStreamStatus.STARTED), + _a_status_message("another stream name", AirbyteStreamStatus.INCOMPLETE), + _a_status_message(_A_STREAM_NAME, AirbyteStreamStatus.COMPLETE) + ] + entrypoint.return_value.run.return_value = _to_entrypoint_output(status_messages) + output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) + assert len(output.get_stream_statuses(_A_STREAM_NAME)) == 2 + + @patch('airbyte_cdk.test.entrypoint_wrapper.print', create=True) + @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") + def test_given_unexpected_exception_when_read_then_print(self, entrypoint, print_mock): + entrypoint.return_value.run.side_effect = ValueError("This error should be printed") + read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) + assert print_mock.call_count > 0 + + @patch('airbyte_cdk.test.entrypoint_wrapper.print', create=True) + @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") + def test_given_expected_exception_when_read_then_do_not_print(self, entrypoint, print_mock): + entrypoint.return_value.run.side_effect = ValueError("This error should be printed") + read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE, expecting_exception=True) + assert print_mock.call_count == 0 + + @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") + def test_given_uncaught_exception_when_read_then_output_has_error(self, entrypoint): + entrypoint.return_value.run.side_effect = ValueError("This error should be printed") + output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) + assert output.errors diff --git a/airbyte-cdk/python/unit_tests/test_entrypoint.py b/airbyte-cdk/python/unit_tests/test_entrypoint.py index 5cd37e5d503b..61e7e7ec142a 100644 --- a/airbyte-cdk/python/unit_tests/test_entrypoint.py +++ b/airbyte-cdk/python/unit_tests/test_entrypoint.py @@ -28,6 +28,7 @@ Type, ) from airbyte_cdk.sources import Source +from airbyte_cdk.utils import AirbyteTracedException class MockSource(Source): @@ -68,14 +69,14 @@ def spec_mock(mocker): type=OrchestratorType.CONNECTOR_CONFIG, emitted_at=10, connectorConfig=AirbyteControlConnectorConfigMessage(config={"any config": "a config value"}), - ) + ), ) @pytest.fixture def entrypoint(mocker) -> AirbyteEntrypoint: message_repository = MagicMock() - message_repository.consume_queue.side_effect = [[message for message in [MESSAGE_FROM_REPOSITORY]], []] + message_repository.consume_queue.side_effect = [[message for message in [MESSAGE_FROM_REPOSITORY]], [], []] mocker.patch.object(MockSource, "message_repository", new_callable=mocker.PropertyMock, return_value=message_repository) return AirbyteEntrypoint(MockSource()) @@ -242,10 +243,20 @@ def test_run_read(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock) messages = list(entrypoint.run(parsed_args)) - assert [_wrap_message(expected), MESSAGE_FROM_REPOSITORY.json(exclude_unset=True)] == messages + assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True), _wrap_message(expected)] == messages assert spec_mock.called +def test_given_message_emitted_during_config_when_read_then_emit_message_before_next_steps(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock): + parsed_args = Namespace(command="read", config="config_path", state="statepath", catalog="catalogpath") + mocker.patch.object(MockSource, "read_catalog", side_effect=ValueError) + + messages = entrypoint.run(parsed_args) + assert next(messages) == MESSAGE_FROM_REPOSITORY.json(exclude_unset=True) + with pytest.raises(ValueError): + next(messages) + + def test_run_read_with_exception(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock): parsed_args = Namespace(command="read", config="config_path", state="statepath", catalog="catalogpath") mocker.patch.object(MockSource, "read_state", return_value={}) @@ -266,17 +277,17 @@ def test_invalid_command(entrypoint: AirbyteEntrypoint, config_mock): "deployment_mode, url, expected_error", [ pytest.param("CLOUD", "https://airbyte.com", None, id="test_cloud_public_endpoint_is_successful"), - pytest.param("CLOUD", "https://192.168.27.30", ValueError, id="test_cloud_private_ip_address_is_rejected"), - pytest.param("CLOUD", "https://localhost:8080/api/v1/cast", ValueError, id="test_cloud_private_endpoint_is_rejected"), + pytest.param("CLOUD", "https://192.168.27.30", AirbyteTracedException, id="test_cloud_private_ip_address_is_rejected"), + pytest.param("CLOUD", "https://localhost:8080/api/v1/cast", AirbyteTracedException, id="test_cloud_private_endpoint_is_rejected"), pytest.param("CLOUD", "http://past.lives.net/api/v1/inyun", ValueError, id="test_cloud_unsecured_endpoint_is_rejected"), pytest.param("CLOUD", "https://not:very/cash:443.money", ValueError, id="test_cloud_invalid_url_format"), pytest.param("CLOUD", "https://192.168.27.30 ", ValueError, id="test_cloud_incorrect_ip_format_is_rejected"), - pytest.param("cloud", "https://192.168.27.30", ValueError, id="test_case_insensitive_cloud_environment_variable"), + pytest.param("cloud", "https://192.168.27.30", AirbyteTracedException, id="test_case_insensitive_cloud_environment_variable"), pytest.param("OSS", "https://airbyte.com", None, id="test_oss_public_endpoint_is_successful"), pytest.param("OSS", "https://192.168.27.30", None, id="test_oss_private_endpoint_is_successful"), pytest.param("OSS", "https://localhost:8080/api/v1/cast", None, id="test_oss_private_endpoint_is_successful"), pytest.param("OSS", "http://past.lives.net/api/v1/inyun", None, id="test_oss_unsecured_endpoint_is_successful"), - ] + ], ) @patch.object(requests.Session, "send", lambda self, request, **kwargs: requests.Response()) def test_filter_internal_requests(deployment_mode, url, expected_error): diff --git a/airbyte-cdk/python/unit_tests/test_exception_handler.py b/airbyte-cdk/python/unit_tests/test_exception_handler.py index 73981848e973..3c6466dd46c7 100644 --- a/airbyte-cdk/python/unit_tests/test_exception_handler.py +++ b/airbyte-cdk/python/unit_tests/test_exception_handler.py @@ -8,7 +8,21 @@ import sys import pytest +from airbyte_cdk.exception_handler import assemble_uncaught_exception from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteLogMessage, AirbyteMessage, AirbyteTraceMessage +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + + +def test_given_exception_is_traced_exception_when_assemble_uncaught_exception_then_return_same_exception(): + exception = AirbyteTracedException() + assembled_exception = assemble_uncaught_exception(type(exception), exception) + assert exception == assembled_exception + + +def test_given_exception_not_traced_exception_when_assemble_uncaught_exception_then_return_traced_exception(): + exception = ValueError("any error") + assembled_exception = assemble_uncaught_exception(type(exception), exception) + assert isinstance(assembled_exception, AirbyteTracedException) def test_uncaught_exception_handler(): diff --git a/airbyte-cdk/python/unit_tests/utils/test_datetime_format_inferrer.py b/airbyte-cdk/python/unit_tests/utils/test_datetime_format_inferrer.py index 68152184b66f..766007467184 100644 --- a/airbyte-cdk/python/unit_tests/utils/test_datetime_format_inferrer.py +++ b/airbyte-cdk/python/unit_tests/utils/test_datetime_format_inferrer.py @@ -22,6 +22,7 @@ ("timestamp_ms_match_string", [{"d": "1686058051000"}], {"d": "%ms"}), ("timestamp_no_match_integer", [{"d": 99}], {}), ("timestamp_no_match_string", [{"d": "99999999999999999999"}], {}), + ("timestamp_overflow", [{"d": f"{10**100}_100"}], {}), # this case was previously causing OverflowError hence this test ("simple_no_match", [{"d": "20220203"}], {}), ("multiple_match", [{"d": "2022-02-03", "e": "2022-02-03"}], {"d": "%Y-%m-%d", "e": "%Y-%m-%d"}), ( diff --git a/airbyte-cdk/python/unit_tests/utils/test_rate_limiting.py b/airbyte-cdk/python/unit_tests/utils/test_rate_limiting.py new file mode 100644 index 000000000000..d1ed294b930d --- /dev/null +++ b/airbyte-cdk/python/unit_tests/utils/test_rate_limiting.py @@ -0,0 +1,26 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import pytest +from airbyte_cdk.sources.streams.http.rate_limiting import default_backoff_handler +from requests import exceptions + + +def helper_with_exceptions(exception_type): + raise exception_type + + +@pytest.mark.parametrize( + "max_tries, max_time, factor, exception_to_raise", + [ + (1, None, 1, exceptions.ConnectTimeout), + (1, 1, 0, exceptions.ReadTimeout), + (2, 2, 1, exceptions.ConnectionError), + (3, 3, 1, exceptions.ChunkedEncodingError), + ], +) +def test_default_backoff_handler(max_tries: int, max_time: int, factor: int, exception_to_raise: Exception): + backoff_handler = default_backoff_handler(max_tries=max_tries, max_time=max_time, factor=factor)(helper_with_exceptions) + with pytest.raises(exception_to_raise): + backoff_handler(exception_to_raise) diff --git a/airbyte-cdk/python/unit_tests/utils/test_stream_status_utils.py b/airbyte-cdk/python/unit_tests/utils/test_stream_status_utils.py index d89a400d8f44..5d41ab7a5700 100644 --- a/airbyte-cdk/python/unit_tests/utils/test_stream_status_utils.py +++ b/airbyte-cdk/python/unit_tests/utils/test_stream_status_utils.py @@ -2,69 +2,60 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_cdk.models import ( - AirbyteMessage, - AirbyteStream, - AirbyteStreamStatus, - ConfiguredAirbyteStream, - DestinationSyncMode, - SyncMode, - TraceType, -) +from airbyte_cdk.models import AirbyteMessage, AirbyteStream, AirbyteStreamStatus, SyncMode, TraceType from airbyte_cdk.models import Type as MessageType from airbyte_cdk.utils.stream_status_utils import as_airbyte_message as stream_status_as_airbyte_message stream = AirbyteStream(name="name", namespace="namespace", json_schema={}, supported_sync_modes=[SyncMode.full_refresh]) -configured_stream = ConfiguredAirbyteStream(stream=stream, sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.overwrite) def test_started_as_message(): stream_status = AirbyteStreamStatus.STARTED - airbyte_message = stream_status_as_airbyte_message(configured_stream, stream_status) + airbyte_message = stream_status_as_airbyte_message(stream, stream_status) assert type(airbyte_message) == AirbyteMessage assert airbyte_message.type == MessageType.TRACE assert airbyte_message.trace.type == TraceType.STREAM_STATUS assert airbyte_message.trace.emitted_at > 0 - assert airbyte_message.trace.stream_status.stream_descriptor.name == configured_stream.stream.name - assert airbyte_message.trace.stream_status.stream_descriptor.namespace == configured_stream.stream.namespace + assert airbyte_message.trace.stream_status.stream_descriptor.name == stream.name + assert airbyte_message.trace.stream_status.stream_descriptor.namespace == stream.namespace assert airbyte_message.trace.stream_status.status == stream_status def test_running_as_message(): stream_status = AirbyteStreamStatus.RUNNING - airbyte_message = stream_status_as_airbyte_message(configured_stream, stream_status) + airbyte_message = stream_status_as_airbyte_message(stream, stream_status) assert type(airbyte_message) == AirbyteMessage assert airbyte_message.type == MessageType.TRACE assert airbyte_message.trace.type == TraceType.STREAM_STATUS assert airbyte_message.trace.emitted_at > 0 - assert airbyte_message.trace.stream_status.stream_descriptor.name == configured_stream.stream.name - assert airbyte_message.trace.stream_status.stream_descriptor.namespace == configured_stream.stream.namespace + assert airbyte_message.trace.stream_status.stream_descriptor.name == stream.name + assert airbyte_message.trace.stream_status.stream_descriptor.namespace == stream.namespace assert airbyte_message.trace.stream_status.status == stream_status def test_complete_as_message(): stream_status = AirbyteStreamStatus.COMPLETE - airbyte_message = stream_status_as_airbyte_message(configured_stream, stream_status) + airbyte_message = stream_status_as_airbyte_message(stream, stream_status) assert type(airbyte_message) == AirbyteMessage assert airbyte_message.type == MessageType.TRACE assert airbyte_message.trace.type == TraceType.STREAM_STATUS assert airbyte_message.trace.emitted_at > 0 - assert airbyte_message.trace.stream_status.stream_descriptor.name == configured_stream.stream.name - assert airbyte_message.trace.stream_status.stream_descriptor.namespace == configured_stream.stream.namespace + assert airbyte_message.trace.stream_status.stream_descriptor.name == stream.name + assert airbyte_message.trace.stream_status.stream_descriptor.namespace == stream.namespace assert airbyte_message.trace.stream_status.status == stream_status def test_incomplete_failed_as_message(): stream_status = AirbyteStreamStatus.INCOMPLETE - airbyte_message = stream_status_as_airbyte_message(configured_stream, stream_status) + airbyte_message = stream_status_as_airbyte_message(stream, stream_status) assert type(airbyte_message) == AirbyteMessage assert airbyte_message.type == MessageType.TRACE assert airbyte_message.trace.type == TraceType.STREAM_STATUS assert airbyte_message.trace.emitted_at > 0 - assert airbyte_message.trace.stream_status.stream_descriptor.name == configured_stream.stream.name - assert airbyte_message.trace.stream_status.stream_descriptor.namespace == configured_stream.stream.namespace + assert airbyte_message.trace.stream_status.stream_descriptor.name == stream.name + assert airbyte_message.trace.stream_status.stream_descriptor.namespace == stream.namespace assert airbyte_message.trace.stream_status.status == stream_status diff --git a/airbyte-ci/.python-version b/airbyte-ci/.python-version new file mode 100644 index 000000000000..7c7a975f4c47 --- /dev/null +++ b/airbyte-ci/.python-version @@ -0,0 +1 @@ +3.10 \ No newline at end of file diff --git a/airbyte-ci/connectors/base_images/README.md b/airbyte-ci/connectors/base_images/README.md new file mode 100644 index 000000000000..9aea896e936f --- /dev/null +++ b/airbyte-ci/connectors/base_images/README.md @@ -0,0 +1,87 @@ +# airbyte-connectors-base-images + +This python package contains the base images used by Airbyte connectors. +It is intended to be used as a python library. +Our connector build pipeline ([`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1)) uses this library to build the connector images. +Our base images are declared in code, using the [Dagger Python SDK](https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/). + +- [Python base image code declaration](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/base_images/base_images/python/bases.py) +- ~Java base image code declaration~ *TODO* + + +## Where are the Dockerfiles? +Our base images are not declared using Dockerfiles. +They are declared in code using the [Dagger Python SDK](https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/). +We prefer this approach because it allows us to interact with base images container as code: we can use python to declare the base images and use the full power of the language to build and test them. +However, we do artificially generate Dockerfiles for debugging and documentation purposes. + + + +### Example for `airbyte/python-connector-base`: +```dockerfile +FROM docker.io/python:3.9.18-slim-bookworm@sha256:44b7f161ed03f85e96d423b9916cdc8cb0509fb970fd643bdbc9896d49e1cad0 +RUN ln -snf /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN pip install --upgrade pip==23.2.1 +ENV POETRY_VIRTUALENVS_CREATE=false +ENV POETRY_VIRTUALENVS_IN_PROJECT=false +ENV POETRY_NO_INTERACTION=1 +RUN pip install poetry==1.6.1 +RUN sh -c apt update && apt-get install -y socat=1.7.4.4-2 +RUN sh -c apt-get update && apt-get install -y tesseract-ocr=5.3.0-2 poppler-utils=22.12.0-2+b1 +RUN mkdir /usr/share/nltk_data +``` + + + +## Base images + + +### `airbyte/python-connector-base` + +| Version | Published | Docker Image Address | Changelog | +|---------|-----------|--------------|-----------| +| 1.2.0 | ✅| docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 | Add CDK system dependencies: nltk data, tesseract, poppler. | +| 1.1.0 | ✅| docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c | Install socat | +| 1.0.0 | ✅| docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27 | Initial release: based on Python 3.9.18, on slim-bookworm system, with pip==23.2.1 and poetry==1.6.1 | + + +## How to release a new base image version (example for Python) + +### Requirements +* [Docker](https://docs.docker.com/get-docker/) +* [Poetry](https://python-poetry.org/docs/#installation) +* Dockerhub logins + +### Steps +1. `poetry install` +2. Open `base_images/python/bases.py`. +3. Make changes to the `AirbytePythonConnectorBaseImage`, you're likely going to change the `get_container` method to change the base image. +4. Implement the `container` property which must return a `dagger.Container` object. +5. **Recommended**: Add new sanity checks to `run_sanity_check` to confirm that the new version is working as expected. +6. Cut a new base image version by running `poetry run generate-release`. You'll need your DockerHub credentials. + +It will: + - Prompt you to pick which base image you'd like to publish. + - Prompt you for a major/minor/patch/pre-release version bump. + - Prompt you for a changelog message. + - Run the sanity checks on the new version. + - Optional: Publish the new version to DockerHub. + - Regenerate the docs and the registry json file. +7. Commit and push your changes. +8. Create a PR and ask for a review from the Connector Operations team. + +**Please note that if you don't publish your image while cutting the new version you can publish it later with `poetry run publish `.** +No connector will use the new base image version until its metadata is updated to use it. +If you're not fully confident with the new base image version please: + - please publish it as a pre-release version + - try out the new version on a couple of connectors + - cut a new version with a major/minor/patch bump and publish it + - This steps can happen in different PRs. + + +## Running tests locally +```bash +poetry run pytest +# Static typing checks +poetry run mypy base_images --check-untyped-defs +``` diff --git a/airbyte-ci/connectors/base_images/base_images/__init__.py b/airbyte-ci/connectors/base_images/base_images/__init__.py new file mode 100644 index 000000000000..1168a935e475 --- /dev/null +++ b/airbyte-ci/connectors/base_images/base_images/__init__.py @@ -0,0 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from rich.console import Console + +console = Console() diff --git a/airbyte-ci/connectors/base_images/base_images/bases.py b/airbyte-ci/connectors/base_images/base_images/bases.py new file mode 100644 index 000000000000..7a8ffcf4e17e --- /dev/null +++ b/airbyte-ci/connectors/base_images/base_images/bases.py @@ -0,0 +1,101 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This module declares common (abstract) classes and methods used by all base images.""" +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import final + +import dagger +import semver + +from .published_image import PublishedImage + + +class AirbyteConnectorBaseImage(ABC): + """An abstract class that represents an Airbyte base image. + Please do not declare any Dagger with_exec instruction in this class as in the abstract class context we have no guarantee about the underlying system used in the base image. + """ + + @final + def __init__(self, dagger_client: dagger.Client, version: semver.VersionInfo): + """Initializes the Airbyte base image. + + Args: + dagger_client (dagger.Client): The dagger client used to build the base image. + version (semver.VersionInfo): The version of the base image. + """ + self.dagger_client = dagger_client + self.version = version + + # INSTANCE PROPERTIES: + + @property + def name_with_tag(self) -> str: + """Returns the full name of the Airbyte base image, with its tag. + + Returns: + str: The full name of the Airbyte base image, with its tag. + """ + return f"{self.repository}:{self.version}" + + # MANDATORY SUBCLASSES ATTRIBUTES / PROPERTIES: + + @property + @abstractmethod + def root_image(self) -> PublishedImage: + """Returns the base image used to build the Airbyte base image. + + Raises: + NotImplementedError: Raised if a subclass does not define a 'root_image' attribute. + + Returns: + PublishedImage: The base image used to build the Airbyte base image. + """ + raise NotImplementedError("Subclasses must define a 'root_image' attribute.") + + @property + @abstractmethod + def repository(self) -> str: + """This is the name of the repository where the image will be hosted. + e.g: airbyte/python-connector-base + + Raises: + NotImplementedError: Raised if a subclass does not define an 'repository' attribute. + + Returns: + str: The repository name where the image will be hosted. + """ + raise NotImplementedError("Subclasses must define an 'repository' attribute.") + + # MANDATORY SUBCLASSES METHODS: + + @abstractmethod + def get_container(self, platform: dagger.Platform) -> dagger.Container: + """Returns the container of the Airbyte connector base image.""" + raise NotImplementedError("Subclasses must define a 'get_container' method.") + + @abstractmethod + async def run_sanity_checks(self, platform: dagger.Platform): + """Runs sanity checks on the base image container. + This method is called before image publication. + + Args: + base_image_version (AirbyteConnectorBaseImage): The base image version on which the sanity checks should run. + + Raises: + SanityCheckError: Raised if a sanity check fails. + """ + raise NotImplementedError("Subclasses must define a 'run_sanity_checks' method.") + + # INSTANCE METHODS: + @final + def get_base_container(self, platform: dagger.Platform) -> dagger.Container: + """Returns a container using the base image. This container is used to build the Airbyte base image. + + Returns: + dagger.Container: The container using the base python image. + """ + return self.dagger_client.pipeline(self.name_with_tag).container(platform=platform).from_(self.root_image.address) diff --git a/airbyte-ci/connectors/base_images/base_images/commands.py b/airbyte-ci/connectors/base_images/base_images/commands.py new file mode 100644 index 000000000000..90d86ce383bd --- /dev/null +++ b/airbyte-ci/connectors/base_images/base_images/commands.py @@ -0,0 +1,190 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import argparse +import sys +from typing import Callable, Type + +import anyio +import dagger +import inquirer # type: ignore +import semver +from base_images import bases, console, consts, errors, hacks, publish, utils, version_registry +from jinja2 import Environment, FileSystemLoader + + +async def _generate_docs(dagger_client: dagger.Client): + """This function will generate the README.md file from the templates/README.md.j2 template. + It will first load all the registries to render the template with up to date information. + """ + docker_credentials = utils.docker.get_credentials() + env = Environment(loader=FileSystemLoader("base_images/templates")) + template = env.get_template("README.md.j2") + rendered_template = template.render({"registries": await version_registry.get_all_registries(dagger_client, docker_credentials)}) + with open("README.md", "w") as readme: + readme.write(rendered_template) + console.log("README.md generated successfully.") + + +async def _generate_release(dagger_client: dagger.Client): + """This function will cut a new version on top of the previous one. It will prompt the user for release details: version bump, changelog entry. + The user can optionally publish the new version to our remote registry. + If the version is not published its changelog entry is still persisted. + It can later be published by running the publish command. + In the future we might only allow publishing new pre-release versions from this flow. + """ + docker_credentials = utils.docker.get_credentials() + select_base_image_class_answers = inquirer.prompt( + [ + inquirer.List( + "BaseImageClass", + message="Which base image would you like to release a new version for?", + choices=[(BaseImageClass.repository, BaseImageClass) for BaseImageClass in version_registry.MANAGED_BASE_IMAGES], + ) + ] + ) + BaseImageClass = select_base_image_class_answers["BaseImageClass"] + registry = await version_registry.VersionRegistry.load(BaseImageClass, dagger_client, docker_credentials) + latest_entry = registry.latest_entry + + # If theres in no latest entry, it means we have no version yet: the registry is empty + # New version will be cut on top of 0.0.0 so this one will actually never be published + seed_version = semver.VersionInfo.parse("0.0.0") + if latest_entry is None: + latest_version = seed_version + else: + latest_version = latest_entry.version + + if latest_version != seed_version and not latest_entry.published: # type: ignore + console.log( + f"The latest version of {BaseImageClass.repository} ({latest_version}) has not been published yet. Please publish it first before cutting a new version." + ) + sys.exit(1) + + new_version_answers = inquirer.prompt( + [ + inquirer.List( + "new_version", + message=f"Which kind of new version would you like to cut? (latest version is {latest_version}))", + choices=[ + ("prerelease", latest_version.bump_prerelease()), + ("patch", latest_version.bump_patch()), + ("minor", latest_version.bump_minor()), + ("major", latest_version.bump_major()), + ], + ), + inquirer.Text("changelog_entry", message="What should the changelog entry be?", validate=lambda _, entry: len(entry) > 0), + inquirer.Confirm("publish_now", message="Would you like to publish it to our remote registry now?"), + ] + ) + new_version, changelog_entry, publish_now = ( + new_version_answers["new_version"], + new_version_answers["changelog_entry"], + new_version_answers["publish_now"], + ) + + base_image_version = BaseImageClass(dagger_client, new_version) + + try: + await publish.run_sanity_checks(base_image_version) + console.log("Sanity checks passed.") + except errors.SanityCheckError as e: + console.log(f"Sanity checks failed: {e}") + console.log("Aborting.") + sys.exit(1) + dockerfile_example = hacks.get_container_dockerfile(base_image_version.get_container(consts.PLATFORMS_WE_PUBLISH_FOR[0])) + + # Add this step we can create a changelog entry: sanity checks passed, image built successfully and sanity checks passed. + changelog_entry = version_registry.ChangelogEntry(new_version, changelog_entry, dockerfile_example) + if publish_now: + published_docker_image = await publish.publish_to_remote_registry(base_image_version) + console.log(f"Published {published_docker_image.address} successfully.") + else: + published_docker_image = None + console.log( + f"Skipping publication. You can publish it later by running `poetry run publish {base_image_version.repository} {new_version}`." + ) + + new_registry_entry = version_registry.VersionRegistryEntry(published_docker_image, changelog_entry, new_version) + registry.add_entry(new_registry_entry) + console.log(f"Added {new_version} to the registry.") + await _generate_docs(dagger_client) + console.log("Generated docs successfully.") + + +async def _publish( + dagger_client: dagger.Client, BaseImageClassToPublish: Type[bases.AirbyteConnectorBaseImage], version: semver.VersionInfo +): + """This function will publish a specific version of a base image to our remote registry. + Users are prompted for confirmation before overwriting an existing version. + If the version does not exist in the registry, the flow is aborted and user is suggested to cut a new version first. + """ + docker_credentials = utils.docker.get_credentials() + registry = await version_registry.VersionRegistry.load(BaseImageClassToPublish, dagger_client, docker_credentials) + registry_entry = registry.get_entry_for_version(version) + if not registry_entry: + console.log(f"No entry found for version {version} in the registry. Please cut a new version first: `poetry run generate-release`") + sys.exit(1) + if registry_entry.published: + force_answers = inquirer.prompt( + [ + inquirer.Confirm( + "force", message="This version has already been published to our remote registry. Would you like to overwrite it?" + ), + ] + ) + if not force_answers["force"]: + console.log("Not overwriting the already exiting image.") + sys.exit(0) + + base_image_version = BaseImageClassToPublish(dagger_client, version) + published_docker_image = await publish.publish_to_remote_registry(base_image_version) + console.log(f"Published {published_docker_image.address} successfully.") + await _generate_docs(dagger_client) + console.log("Generated docs successfully.") + + +async def execute_async_command(command_fn: Callable, *args, **kwargs): + """This is a helper function that will execute a command function in an async context, required by the use of Dagger.""" + async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: + await command_fn(dagger_client, *args, **kwargs) + + +def generate_docs(): + """This command will generate the README.md file from the templates/README.md.j2 template. + It will first load all the registries to render the template with up to date information. + """ + anyio.run(execute_async_command, _generate_docs) + + +def generate_release(): + """This command will cut a new version on top of the previous one. It will prompt the user for release details: version bump, changelog entry. + The user can optionally publish the new version to our remote registry. + If the version is not published its changelog entry is still persisted. + It can later be published by running the publish command. + In the future we might only allow publishing new pre-release versions from this flow. + """ + anyio.run(execute_async_command, _generate_release) + + +def publish_existing_version(): + """This command is intended to be used when: + - We have a changelog entry for a new version but it's not published yet (for future publish on merge flows). + - We have a good reason to overwrite an existing version in the remote registry. + """ + parser = argparse.ArgumentParser(description="Publish a specific version of a base image to our remote registry.") + parser.add_argument("repository", help="The base image repository name") + parser.add_argument("version", help="The version to publish") + args = parser.parse_args() + + version = semver.VersionInfo.parse(args.version) + BaseImageClassToPublish = None + for BaseImageClass in version_registry.MANAGED_BASE_IMAGES: + if BaseImageClass.repository == args.repository: + BaseImageClassToPublish = BaseImageClass + if BaseImageClassToPublish is None: + console.log(f"Unknown base image name: {args.repository}") + sys.exit(1) + + anyio.run(execute_async_command, _publish, BaseImageClassToPublish, version) diff --git a/airbyte-ci/connectors/base_images/base_images/consts.py b/airbyte-ci/connectors/base_images/base_images/consts.py new file mode 100644 index 000000000000..ce3701c300f1 --- /dev/null +++ b/airbyte-ci/connectors/base_images/base_images/consts.py @@ -0,0 +1,11 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This module declares constants used by the base_images module. +""" + +import dagger + +REMOTE_REGISTRY = "docker.io" +PLATFORMS_WE_PUBLISH_FOR = (dagger.Platform("linux/amd64"), dagger.Platform("linux/arm64")) diff --git a/airbyte-ci/connectors/base_images/base_images/errors.py b/airbyte-ci/connectors/base_images/base_images/errors.py new file mode 100644 index 000000000000..3e009806c80b --- /dev/null +++ b/airbyte-ci/connectors/base_images/base_images/errors.py @@ -0,0 +1,17 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This module contains the exceptions used by the base_images module. +""" + +from typing import Union + +import dagger + + +class SanityCheckError(Exception): + """Raised when a sanity check fails.""" + + def __init__(self, error: Union[str, dagger.ExecError], *args: object) -> None: + super().__init__(error, *args) diff --git a/airbyte-ci/connectors/base_images/base_images/hacks.py b/airbyte-ci/connectors/base_images/base_images/hacks.py new file mode 100644 index 000000000000..cbdd964ad941 --- /dev/null +++ b/airbyte-ci/connectors/base_images/base_images/hacks.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import dagger + +# If we perform addition dagger operations on the container, we need to make sure that a mapping exists for the new field name. +DAGGER_FIELD_NAME_TO_DOCKERFILE_INSTRUCTION = { + "from": lambda field: f'FROM {field.args.get("address")}', + "withExec": lambda field: f'RUN {" ".join(field.args.get("args"))}', + "withEnvVariable": lambda field: f'ENV {field.args.get("name")}={field.args.get("value")}', + "withLabel": lambda field: f'LABEL {field.args.get("name")}={field.args.get("value")}', +} + + +def get_container_dockerfile(container) -> str: + """Returns the Dockerfile of the base image container. + Disclaimer: THIS IS HIGHLY EXPERIMENTAL, HACKY AND BRITTLE. + TODO: CONFIRM WITH THE DAGGER TEAM WHAT CAN GO WRONG HERE. + Returns: + str: The Dockerfile of the base image container. + """ + + lineage = [ + field for field in list(container._ctx.selections) if isinstance(field, dagger.api.base.Field) and field.type_name == "Container" + ] + dockerfile = [] + + for field in lineage: + if field.name in DAGGER_FIELD_NAME_TO_DOCKERFILE_INSTRUCTION: + try: + dockerfile.append(DAGGER_FIELD_NAME_TO_DOCKERFILE_INSTRUCTION[field.name](field)) + except KeyError: + raise KeyError( + f"Unknown field name: {field.name}, please add it to the DAGGER_FIELD_NAME_TO_DOCKERFILE_INSTRUCTION mapping." + ) + return "\n".join(dockerfile) diff --git a/airbyte-ci/connectors/base_images/base_images/publish.py b/airbyte-ci/connectors/base_images/base_images/publish.py new file mode 100644 index 000000000000..4f4ebcfaf3e9 --- /dev/null +++ b/airbyte-ci/connectors/base_images/base_images/publish.py @@ -0,0 +1,34 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import dagger +from base_images import bases, consts, published_image + + +async def run_sanity_checks(base_image_version: bases.AirbyteConnectorBaseImage): + for platform in consts.PLATFORMS_WE_PUBLISH_FOR: + await base_image_version.run_sanity_checks(platform) + + +async def publish_to_remote_registry(base_image_version: bases.AirbyteConnectorBaseImage) -> published_image.PublishedImage: + """Publishes a base image to the remote registry. + + Args: + base_image_version (common.AirbyteConnectorBaseImage): The base image to publish. + + Returns: + models.PublishedImage: The published image as a PublishedImage instance. + """ + + address = f"{consts.REMOTE_REGISTRY}/{base_image_version.repository}:{base_image_version.version}" + variants_to_publish = [] + for platform in consts.PLATFORMS_WE_PUBLISH_FOR: + await base_image_version.run_sanity_checks(platform) + variants_to_publish.append(base_image_version.get_container(platform)) + # Publish with forced compression to ensure backward compatibility with older versions of docker + published_address = await variants_to_publish[0].publish( + address, platform_variants=variants_to_publish[1:], forced_compression=dagger.ImageLayerCompression.Gzip + ) + return published_image.PublishedImage.from_address(published_address) diff --git a/airbyte-ci/connectors/base_images/base_images/published_image.py b/airbyte-ci/connectors/base_images/base_images/published_image.py new file mode 100644 index 000000000000..7e8f5f0af5eb --- /dev/null +++ b/airbyte-ci/connectors/base_images/base_images/published_image.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from __future__ import annotations + +from dataclasses import dataclass + +import semver + + +@dataclass +class PublishedImage: + registry: str + repository: str + tag: str + sha: str + + @property + def address(self) -> str: + return f"{self.registry}/{self.repository}:{self.tag}@sha256:{self.sha}" + + @classmethod + def from_address(cls, address: str) -> PublishedImage: + """Creates a PublishedImage instance from a docker image address. + A docker image address is a string of the form: + registry/repository:tag@sha256:sha + + Args: + address (str): _description_ + + Returns: + PublishedImage: _description_ + """ + parts = address.split("/") + registry = parts.pop(0) + without_registry = "/".join(parts) + repository, tag, sha = without_registry.replace("@sha256", "").split(":") + return cls(registry, repository, tag, sha) + + @property + def name_with_tag(self) -> str: + return f"{self.repository}:{self.tag}" + + @property + def version(self) -> semver.VersionInfo: + return semver.VersionInfo.parse(self.tag) diff --git a/airbyte-integrations/connectors/source-datadog/unit_tests/__init__.py b/airbyte-ci/connectors/base_images/base_images/python/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-datadog/unit_tests/__init__.py rename to airbyte-ci/connectors/base_images/base_images/python/__init__.py diff --git a/airbyte-ci/connectors/base_images/base_images/python/bases.py b/airbyte-ci/connectors/base_images/base_images/python/bases.py new file mode 100644 index 000000000000..59bff804a381 --- /dev/null +++ b/airbyte-ci/connectors/base_images/base_images/python/bases.py @@ -0,0 +1,126 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from __future__ import annotations + +from typing import Callable, Final + +import dagger +from base_images import bases, published_image +from base_images import sanity_checks as base_sanity_checks +from base_images.python import sanity_checks as python_sanity_checks +from base_images.root_images import PYTHON_3_9_18 + + +class AirbytePythonConnectorBaseImage(bases.AirbyteConnectorBaseImage): + + root_image: Final[published_image.PublishedImage] = PYTHON_3_9_18 + repository: Final[str] = "airbyte/python-connector-base" + pip_cache_name: Final[str] = "pip_cache" + nltk_data_path: Final[str] = "/usr/share/nltk_data" + ntlk_data = { + "tokenizers": {"https://github.com/nltk/nltk_data/raw/5db857e6f7df11eabb5e5665836db9ec8df07e28/packages/tokenizers/punkt.zip"}, + "taggers": { + "https://github.com/nltk/nltk_data/raw/5db857e6f7df11eabb5e5665836db9ec8df07e28/packages/taggers/averaged_perceptron_tagger.zip" + }, + } + + def install_cdk_system_dependencies(self) -> Callable: + def get_nltk_data_dir() -> dagger.Directory: + """Returns a dagger directory containing the nltk data. + + Returns: + dagger.Directory: A dagger directory containing the nltk data. + """ + data_container = self.dagger_client.container().from_("bash:latest") + + for nltk_data_subfolder, nltk_data_urls in self.ntlk_data.items(): + full_nltk_data_path = f"{self.nltk_data_path}/{nltk_data_subfolder}" + for nltk_data_url in nltk_data_urls: + zip_file = self.dagger_client.http(nltk_data_url) + data_container = ( + data_container.with_file("/tmp/data.zip", zip_file) + .with_exec(["mkdir", "-p", full_nltk_data_path], skip_entrypoint=True) + .with_exec(["unzip", "-o", "/tmp/data.zip", "-d", full_nltk_data_path], skip_entrypoint=True) + .with_exec(["rm", "/tmp/data.zip"], skip_entrypoint=True) + ) + return data_container.directory(self.nltk_data_path) + + def with_tesseract_and_poppler(container: dagger.Container) -> dagger.Container: + """ + Installs Tesseract-OCR and Poppler-utils in the base image. + These tools are necessary for OCR (Optical Character Recognition) processes and working with PDFs, respectively. + """ + + container = container.with_exec( + ["sh", "-c", "apt-get update && apt-get install -y tesseract-ocr=5.3.0-2 poppler-utils=22.12.0-2+b1"], skip_entrypoint=True + ) + + return container + + def with_file_based_connector_dependencies(container: dagger.Container) -> dagger.Container: + """ + Installs the dependencies for file-based connectors. This includes: + - tesseract-ocr + - poppler-utils + - nltk data + """ + container = with_tesseract_and_poppler(container) + container = container.with_exec(["mkdir", self.nltk_data_path], skip_entrypoint=True).with_directory( + self.nltk_data_path, get_nltk_data_dir() + ) + return container + + return with_file_based_connector_dependencies + + def get_container(self, platform: dagger.Platform) -> dagger.Container: + """Returns the container used to build the base image. + We currently use the python:3.9.18-slim-bookworm image as a base. + We set the container system timezone to UTC. + We then upgrade pip and install poetry. + + Args: + platform (dagger.Platform): The platform this container should be built for. + + Returns: + dagger.Container: The container used to build the base image. + """ + pip_cache_volume: dagger.CacheVolume = self.dagger_client.cache_volume(AirbytePythonConnectorBaseImage.pip_cache_name) + + return ( + self.get_base_container(platform) + .with_mounted_cache("/root/.cache/pip", pip_cache_volume) + # Set the timezone to UTC + .with_exec(["ln", "-snf", "/usr/share/zoneinfo/Etc/UTC", "/etc/localtime"]) + # Upgrade pip to the expected version + .with_exec(["pip", "install", "--upgrade", "pip==23.2.1"]) + # Declare poetry specific environment variables + .with_env_variable("POETRY_VIRTUALENVS_CREATE", "false") + .with_env_variable("POETRY_VIRTUALENVS_IN_PROJECT", "false") + .with_env_variable("POETRY_NO_INTERACTION", "1") + .with_exec(["pip", "install", "poetry==1.6.1"], skip_entrypoint=True) + # Install socat 1.7.4.4 + .with_exec(["sh", "-c", "apt update && apt-get install -y socat=1.7.4.4-2"]) + # Install CDK system dependencies + .with_(self.install_cdk_system_dependencies()) + ) + + async def run_sanity_checks(self, platform: dagger.Platform): + """Runs sanity checks on the base image container. + This method is called before image publication. + Consider it like a pre-flight check before take-off to the remote registry. + + Args: + platform (dagger.Platform): The platform on which the sanity checks should run. + """ + container = self.get_container(platform) + await base_sanity_checks.check_timezone_is_utc(container) + await base_sanity_checks.check_a_command_is_available_using_version_option(container, "bash") + await python_sanity_checks.check_python_version(container, "3.9.18") + await python_sanity_checks.check_pip_version(container, "23.2.1") + await python_sanity_checks.check_poetry_version(container, "1.6.1") + await python_sanity_checks.check_python_image_has_expected_env_vars(container) + await base_sanity_checks.check_a_command_is_available_using_version_option(container, "socat", "-V") + await base_sanity_checks.check_socat_version(container, "1.7.4.4") + await python_sanity_checks.check_cdk_system_dependencies(container) diff --git a/airbyte-ci/connectors/base_images/base_images/python/sanity_checks.py b/airbyte-ci/connectors/base_images/base_images/python/sanity_checks.py new file mode 100644 index 000000000000..5411c8c269dc --- /dev/null +++ b/airbyte-ci/connectors/base_images/base_images/python/sanity_checks.py @@ -0,0 +1,149 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import dagger +from base_images import errors +from base_images import sanity_checks as base_sanity_checks + + +async def check_python_version(container: dagger.Container, expected_python_version: str): + """Checks that the python version is the expected one. + + Args: + container (dagger.Container): The container on which the sanity checks should run. + expected_python_version (str): The expected python version. + + Raises: + errors.SanityCheckError: Raised if the python --version command could not be executed or if the outputted version is not the expected one. + """ + try: + python_version_output: str = await container.with_exec(["python", "--version"], skip_entrypoint=True).stdout() + except dagger.ExecError as e: + raise errors.SanityCheckError(e) + if python_version_output != f"Python {expected_python_version}\n": + raise errors.SanityCheckError(f"unexpected python version: {python_version_output}") + + +async def check_pip_version(container: dagger.Container, expected_pip_version: str): + """Checks that the pip version is the expected one. + + Args: + container (dagger.Container): The container on which the sanity checks should run. + expected_pip_version (str): The expected pip version. + + Raises: + errors.SanityCheckError: Raised if the pip --version command could not be executed or if the outputted version is not the expected one. + """ + try: + pip_version_output: str = await container.with_exec(["pip", "--version"], skip_entrypoint=True).stdout() + except dagger.ExecError as e: + raise errors.SanityCheckError(e) + if not pip_version_output.startswith(f"pip {expected_pip_version}"): + raise errors.SanityCheckError(f"unexpected pip version: {pip_version_output}") + + +async def check_poetry_version(container: dagger.Container, expected_poetry_version: str): + """Checks that the poetry version is the expected one. + + Args: + container (dagger.Container): The container on which the sanity checks should run. + expected_poetry_version (str): The expected poetry version. + + Raises: + errors.SanityCheckError: Raised if the poetry --version command could not be executed or if the outputted version is not the expected one. + """ + try: + poetry_version_output: str = await container.with_exec(["poetry", "--version"], skip_entrypoint=True).stdout() + except dagger.ExecError as e: + raise errors.SanityCheckError(e) + if not poetry_version_output.startswith(f"Poetry (version {expected_poetry_version}"): + raise errors.SanityCheckError(f"unexpected poetry version: {poetry_version_output}") + + +async def check_python_image_has_expected_env_vars(python_image_container: dagger.Container): + """Check a python container has the set of env var we always expect on python images. + + Args: + python_image_container (dagger.Container): The container on which the sanity checks should run. + """ + expected_env_vars = { + "PYTHON_VERSION", + "PYTHON_PIP_VERSION", + "PYTHON_GET_PIP_SHA256", + "PYTHON_GET_PIP_URL", + "HOME", + "PATH", + "LANG", + "GPG_KEY", + "PYTHON_SETUPTOOLS_VERSION", + } + # It's not suboptimal to call printenv multiple times because the printenv output is cached. + for expected_env_var in expected_env_vars: + await base_sanity_checks.check_env_var_with_printenv(python_image_container, expected_env_var) + + +async def check_nltk_data(python_image_container: dagger.Container): + """Install nltk and check that the required data is available. + As of today the required data is: + - taggers/averaged_perceptron_tagger + - tokenizers/punkt + + Args: + python_image_container (dagger.Container): The container on which the sanity checks should run. + + Raises: + errors.SanityCheckError: Raised if the nltk data is not available. + """ + with_nltk = await python_image_container.with_exec(["pip", "install", "nltk==3.8.1"], skip_entrypoint=True) + try: + await with_nltk.with_exec( + ["python", "-c", 'import nltk;nltk.data.find("taggers/averaged_perceptron_tagger");nltk.data.find("tokenizers/punkt")'], + skip_entrypoint=True, + ) + except dagger.ExecError as e: + raise errors.SanityCheckError(e) + + +async def check_tesseract_version(python_image_container: dagger.Container, tesseract_version: str): + """Check that the tesseract version is the expected one. + + Args: + python_image_container (dagger.Container): The container on which the sanity checks should run. + tesseract_version (str): The expected tesseract version. + + Raises: + errors.SanityCheckError: Raised if the tesseract --version command could not be executed or if the outputted version is not the expected one. + """ + try: + tesseract_version_output = await python_image_container.with_exec(["tesseract", "--version"], skip_entrypoint=True).stdout() + except dagger.ExecError as e: + raise errors.SanityCheckError(e) + if not tesseract_version_output.startswith(f"tesseract {tesseract_version}"): + raise errors.SanityCheckError(f"unexpected tesseract version: {tesseract_version_output}") + + +async def check_poppler_utils_version(python_image_container: dagger.Container, poppler_version: str): + """Check that the poppler version is the expected one. + The poppler version can be checked by running a pdftotext -v command. + + Args: + python_image_container (dagger.Container): The container on which the sanity checks should run. + poppler_version (str): The expected poppler version. + + Raises: + errors.SanityCheckError: Raised if the pdftotext -v command could not be executed or if the outputted version is not the expected one. + """ + try: + pdf_to_text_version_output = await python_image_container.with_exec(["pdftotext", "-v"], skip_entrypoint=True).stderr() + except dagger.ExecError as e: + raise errors.SanityCheckError(e) + + if f"pdftotext version {poppler_version}" not in pdf_to_text_version_output: + raise errors.SanityCheckError(f"unexpected poppler version: {pdf_to_text_version_output}") + + +async def check_cdk_system_dependencies(python_image_container: dagger.Container): + await check_nltk_data(python_image_container) + await check_tesseract_version(python_image_container, "5.3.0") + await check_poppler_utils_version(python_image_container, "22.12.0") diff --git a/airbyte-ci/connectors/base_images/base_images/root_images.py b/airbyte-ci/connectors/base_images/base_images/root_images.py new file mode 100644 index 000000000000..2dc3fc28ab58 --- /dev/null +++ b/airbyte-ci/connectors/base_images/base_images/root_images.py @@ -0,0 +1,12 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from .published_image import PublishedImage + +PYTHON_3_9_18 = PublishedImage( + registry="docker.io", + repository="python", + tag="3.9.18-slim-bookworm", + sha="44b7f161ed03f85e96d423b9916cdc8cb0509fb970fd643bdbc9896d49e1cad0", +) diff --git a/airbyte-ci/connectors/base_images/base_images/sanity_checks.py b/airbyte-ci/connectors/base_images/base_images/sanity_checks.py new file mode 100644 index 000000000000..0b141bdd1abc --- /dev/null +++ b/airbyte-ci/connectors/base_images/base_images/sanity_checks.py @@ -0,0 +1,101 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import re +from typing import Optional + +import dagger +from base_images import errors + + +async def check_env_var_with_printenv( + container: dagger.Container, expected_env_var_name: str, expected_env_var_value: Optional[str] = None +): + """This checks if an environment variable is correctly defined by calling the printenv command in a container. + + Args: + container (dagger.Container): The container on which the sanity checks should run. + expected_env_var_name (str): The name of the environment variable to check. + expected_env_var_value (Optional[str], optional): The expected value of the environment variable. Defaults to None. + + Raises: + errors.SanityCheckError: Raised if the environment variable is not defined or if it has an unexpected value. + """ + try: + printenv_output = await container.with_exec(["printenv"], skip_entrypoint=True).stdout() + except dagger.ExecError as e: + raise errors.SanityCheckError(e) + env_vars = {line.split("=")[0]: line.split("=")[1] for line in printenv_output.splitlines()} + if expected_env_var_name not in env_vars: + raise errors.SanityCheckError(f"the {expected_env_var_name} environment variable is not defined.") + if expected_env_var_value is not None and env_vars[expected_env_var_name] != expected_env_var_value: + raise errors.SanityCheckError( + f"the {expected_env_var_name} environment variable is defined but has an unexpected value: {env_vars[expected_env_var_name]}." + ) + + +async def check_timezone_is_utc(container: dagger.Container): + """Check that the system timezone is UTC. + + Args: + container (dagger.Container): The container on which the sanity checks should run. + + Raises: + errors.SanityCheckError: Raised if the date command could not be executed or if the outputted timezone is not UTC. + """ + try: + tz_output: str = await container.with_exec(["date"], skip_entrypoint=True).stdout() + except dagger.ExecError as e: + raise errors.SanityCheckError(e) + if "UTC" not in tz_output: + raise errors.SanityCheckError(f"unexpected timezone: {tz_output}") + + +async def check_a_command_is_available_using_version_option(container: dagger.Container, command: str, version_option: str = "--version"): + """Checks that a command is available in the container by calling it with the --version option. + + Args: + container (dagger.Container): The container on which the sanity checks should run. + command (str): The command to check. + + Raises: + errors.SanityCheckError: Raised if the command could not be executed or if the outputted version is not the expected one. + """ + try: + command_version_output: str = await container.with_exec([command, version_option], skip_entrypoint=True).stdout() + except dagger.ExecError as e: + raise errors.SanityCheckError(e) + if command_version_output == "": + raise errors.SanityCheckError(f"unexpected {command} version: {command_version_output}") + + +async def check_socat_version(container: dagger.Container, expected_socat_version: str): + """Checks that the socat version is the expected one. + + Args: + container (dagger.Container): The container on which the sanity checks should run. + expected_socat_version (str): The expected socat version. + + Raises: + errors.SanityCheckError: Raised if the socat --version command could not be executed or if the outputted version is not the expected one. + """ + try: + socat_version_output: str = await container.with_exec(["socat", "-V"], skip_entrypoint=True).stdout() + except dagger.ExecError as e: + raise errors.SanityCheckError(e) + socat_version_line = None + for line in socat_version_output.splitlines(): + if line.startswith("socat version"): + socat_version_line = line + break + if socat_version_line is None: + raise errors.SanityCheckError(f"Could not parse the socat version from the output: {socat_version_output}") + version_pattern = r"version (\d+\.\d+\.\d+\.\d+)" + match = re.search(version_pattern, socat_version_line) + if match: + version_number = match.group(1) + if version_number != expected_socat_version: + raise errors.SanityCheckError(f"unexpected socat version: {version_number}") + else: + raise errors.SanityCheckError(f"Could not find the socat version in the version output: {socat_version_line}") diff --git a/airbyte-ci/connectors/base_images/base_images/templates/README.md.j2 b/airbyte-ci/connectors/base_images/base_images/templates/README.md.j2 new file mode 100644 index 000000000000..ad8dc7387047 --- /dev/null +++ b/airbyte-ci/connectors/base_images/base_images/templates/README.md.j2 @@ -0,0 +1,79 @@ +# airbyte-connectors-base-images + +This python package contains the base images used by Airbyte connectors. +It is intended to be used as a python library. +Our connector build pipeline ([`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1)) uses this library to build the connector images. +Our base images are declared in code, using the [Dagger Python SDK](https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/). + +- [Python base image code declaration](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/base_images/base_images/python/bases.py) +- ~Java base image code declaration~ *TODO* + + +## Where are the Dockerfiles? +Our base images are not declared using Dockerfiles. +They are declared in code using the [Dagger Python SDK](https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/). +We prefer this approach because it allows us to interact with base images container as code: we can use python to declare the base images and use the full power of the language to build and test them. +However, we do artificially generate Dockerfiles for debugging and documentation purposes. + +{% for registry in registries %} +{% if registry.entries %} +### Example for `{{ registry.ConnectorBaseImageClass.repository }}`: +```dockerfile +{{ registry.entries[0].changelog_entry.dockerfile_example }} +``` +{% endif %} +{% endfor %} + +## Base images +{% for registry in registries %} + +### `{{ registry.ConnectorBaseImageClass.repository }}` + +| Version | Published | Docker Image Address | Changelog | +|---------|-----------|--------------|-----------| +{%- for entry in registry.entries %} +| {{ entry.version }} | {{ "✅" if entry.published else "❌" }}| {{ entry.published_docker_image.address }} | {{ entry.changelog_entry.changelog_entry }} | +{%- endfor %} +{% endfor %} + +## How to release a new base image version (example for Python) + +### Requirements +* [Docker](https://docs.docker.com/get-docker/) +* [Poetry](https://python-poetry.org/docs/#installation) +* Dockerhub logins + +### Steps +1. `poetry install` +2. Open `base_images/python/bases.py`. +3. Make changes to the `AirbytePythonConnectorBaseImage`, you're likely going to change the `get_container` method to change the base image. +4. Implement the `container` property which must return a `dagger.Container` object. +5. **Recommended**: Add new sanity checks to `run_sanity_check` to confirm that the new version is working as expected. +6. Cut a new base image version by running `poetry run generate-release`. You'll need your DockerHub credentials. + +It will: + - Prompt you to pick which base image you'd like to publish. + - Prompt you for a major/minor/patch/pre-release version bump. + - Prompt you for a changelog message. + - Run the sanity checks on the new version. + - Optional: Publish the new version to DockerHub. + - Regenerate the docs and the registry json file. +7. Commit and push your changes. +8. Create a PR and ask for a review from the Connector Operations team. + +**Please note that if you don't publish your image while cutting the new version you can publish it later with `poetry run publish `.** +No connector will use the new base image version until its metadata is updated to use it. +If you're not fully confident with the new base image version please: + - please publish it as a pre-release version + - try out the new version on a couple of connectors + - cut a new version with a major/minor/patch bump and publish it + - This steps can happen in different PRs. + + +## Running tests locally +```bash +poetry run pytest +# Static typing checks +poetry run mypy base_images --check-untyped-defs +``` + diff --git a/airbyte-integrations/connectors/source-glassfrog/unit_tests/__init__.py b/airbyte-ci/connectors/base_images/base_images/utils/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-glassfrog/unit_tests/__init__.py rename to airbyte-ci/connectors/base_images/base_images/utils/__init__.py diff --git a/airbyte-ci/connectors/base_images/base_images/utils/docker.py b/airbyte-ci/connectors/base_images/base_images/utils/docker.py new file mode 100644 index 000000000000..31f03fea4c03 --- /dev/null +++ b/airbyte-ci/connectors/base_images/base_images/utils/docker.py @@ -0,0 +1,95 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import getpass +import os +import uuid +from typing import List, Tuple + +import dagger +from base_images import console, published_image + + +def get_credentials() -> Tuple[str, str]: + """This function will prompt the user for docker credentials. + If the user has set the DOCKER_HUB_USERNAME and DOCKER_HUB_PASSWORD environment variables, it will use those instead. + Returns: + Tuple[str, str]: (username, password) + """ + if os.environ.get("DOCKER_HUB_USERNAME") and os.environ.get("DOCKER_HUB_PASSWORD"): + console.log("Using docker credentials from environment variables.") + return os.environ["DOCKER_HUB_USERNAME"], os.environ["DOCKER_HUB_PASSWORD"] + else: + console.log("Please enter your docker credentials.") + console.log("You can set them as environment variables to avoid being prompted again: DOCKER_HUB_USERNAME, DOCKER_HUB_PASSWORD") + # Not using inquirer here because of the sensitive nature of the information + docker_username = input("Dockerhub username: ") + docker_password = getpass.getpass("Dockerhub Password: ") + return docker_username, docker_password + + +class CraneClient: + + CRANE_IMAGE_ADDRESS = ( + "gcr.io/go-containerregistry/crane/debug:v0.15.1@sha256:f6ddf8e2c47df889e06e33c3e83b84251ac19c8728a670ff39f2ca9e90c4f905" + ) + + def __init__(self, dagger_client: dagger.Client, docker_credentials: Tuple[str, str]): + self.docker_hub_username_secret = dagger_client.set_secret("DOCKER_HUB_USERNAME", docker_credentials[0]) + self.docker_hub_username_password = dagger_client.set_secret("DOCKER_HUB_PASSWORD", docker_credentials[1]) + + self.bare_container = ( + dagger_client.container().from_(self.CRANE_IMAGE_ADDRESS) + # We don't want to cache any subsequent commands that might run in this container + # because we want to have fresh output data every time we run this command. + .with_env_variable("CACHE_BUSTER", str(uuid.uuid4())) + ) + + self.authenticated_container = self.login() + + def login(self) -> dagger.Container: + return ( + self.bare_container.with_secret_variable("DOCKER_HUB_USERNAME", self.docker_hub_username_secret) + .with_secret_variable("DOCKER_HUB_PASSWORD", self.docker_hub_username_password) + .with_exec( + ["sh", "-c", "crane auth login index.docker.io -u $DOCKER_HUB_USERNAME -p $DOCKER_HUB_PASSWORD"], skip_entrypoint=True + ) + ) + + async def digest(self, repository_and_tag: str) -> str: + console.log(f"Fetching digest for {repository_and_tag}...") + return (await self.authenticated_container.with_exec(["digest", repository_and_tag]).stdout()).strip() + + async def ls(self, registry_name: str, repository_name: str) -> List[str]: + repository_address = f"{registry_name}/{repository_name}" + console.log(f"Fetching published images in {repository_address}...") + try: + crane_ls_output = await self.authenticated_container.with_exec(["ls", repository_address]).stdout() + return crane_ls_output.splitlines() + except dagger.ExecError as exec_error: + # When the repository does not exist, crane ls returns an error with NAME_UNKNOWN in the stderr. + if "NAME_UNKNOWN" in exec_error.stderr: + console.log(f"Repository {repository_address} does not exist. Returning an empty list.") + return [] + else: + raise exec_error + + +class RemoteRepository: + def __init__(self, crane_client: CraneClient, registry_name: str, repository_name: str): + self.crane_client = crane_client + self.registry_name = registry_name + self.repository_name = repository_name + + async def get_all_images(self) -> List[published_image.PublishedImage]: + repository_address = f"{self.registry_name}/{self.repository_name}" + all_tags = await self.crane_client.ls(self.registry_name, self.repository_name) + # CraneClient ls lists the tags available for a repository, but not the digests. + # We want the digest to uniquely identify the image, so we need to fetch it separately with `crane digest` + available_addresses_without_digest = [f"{repository_address}:{tag}" for tag in all_tags] + available_addresses_with_digest = [] + for address in available_addresses_without_digest: + digest = await self.crane_client.digest(address) + available_addresses_with_digest.append(f"{address}@{digest}") + return [published_image.PublishedImage.from_address(address) for address in available_addresses_with_digest] diff --git a/airbyte-ci/connectors/base_images/base_images/version_registry.py b/airbyte-ci/connectors/base_images/base_images/version_registry.py new file mode 100644 index 000000000000..bd7fd27260c0 --- /dev/null +++ b/airbyte-ci/connectors/base_images/base_images/version_registry.py @@ -0,0 +1,274 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Optional, Tuple, Type + +import dagger +import semver +from base_images import consts, published_image +from base_images.bases import AirbyteConnectorBaseImage +from base_images.python.bases import AirbytePythonConnectorBaseImage +from base_images.utils import docker +from connector_ops.utils import ConnectorLanguage # type: ignore + +MANAGED_BASE_IMAGES = [AirbytePythonConnectorBaseImage] + + +@dataclass +class ChangelogEntry: + version: semver.VersionInfo + changelog_entry: str + dockerfile_example: str + + def to_serializable_dict(self): + return { + "version": str(self.version), + "changelog_entry": self.changelog_entry, + "dockerfile_example": self.dockerfile_example, + } + + @staticmethod + def from_dict(entry_dict: Dict): + return ChangelogEntry( + version=semver.VersionInfo.parse(entry_dict["version"]), + changelog_entry=entry_dict["changelog_entry"], + dockerfile_example=entry_dict["dockerfile_example"], + ) + + +@dataclass +class VersionRegistryEntry: + published_docker_image: Optional[published_image.PublishedImage] + changelog_entry: Optional[ChangelogEntry] + version: semver.VersionInfo + + @property + def published(self) -> bool: + return self.published_docker_image is not None + + +class VersionRegistry: + def __init__( + self, + ConnectorBaseImageClass: Type[AirbyteConnectorBaseImage], + entries: List[VersionRegistryEntry], + ) -> None: + self.ConnectorBaseImageClass: Type[AirbyteConnectorBaseImage] = ConnectorBaseImageClass + self._entries: List[VersionRegistryEntry] = entries + + @staticmethod + def get_changelog_dump_path(ConnectorBaseImageClass: Type[AirbyteConnectorBaseImage]) -> Path: + """Returns the path where the changelog is dumped to disk. + + Args: + ConnectorBaseImageClass (Type[AirbyteConnectorBaseImage]): The base image version class bound to the registry. + + Returns: + Path: The path where the changelog JSON is dumped to disk. + """ + registries_dir = Path("generated/changelogs") + registries_dir.mkdir(exist_ok=True, parents=True) + return registries_dir / f'{ConnectorBaseImageClass.repository.replace("-", "_").replace("/", "_")}.json' # type: ignore + + @property + def changelog_dump_path(self) -> Path: + """Returns the path where the changelog JSON is dumped to disk. + + Returns: + Path: The path where the changelog JSON is dumped to disk. + """ + return self.get_changelog_dump_path(self.ConnectorBaseImageClass) + + @staticmethod + def get_changelog_entries(ConnectorBaseImageClass: Type[AirbyteConnectorBaseImage]) -> List[ChangelogEntry]: + """Returns the changelog entries for a given base image version class. + The changelog entries are loaded from the checked in changelog dump JSON file. + + Args: + ConnectorBaseImageClass (Type[AirbyteConnectorBaseImage]): The base image version class bound to the registry. + + Returns: + List[ChangelogEntry]: The changelog entries for a given base image version class. + """ + change_log_dump_path = VersionRegistry.get_changelog_dump_path(ConnectorBaseImageClass) + if not change_log_dump_path.exists(): + changelog_entries = [] + else: + changelog_entries = [ChangelogEntry.from_dict(raw_entry) for raw_entry in json.loads(change_log_dump_path.read_text())] + return changelog_entries + + @staticmethod + async def get_all_published_base_images( + dagger_client: dagger.Client, docker_credentials: Tuple[str, str], ConnectorBaseImageClass: Type[AirbyteConnectorBaseImage] + ) -> List[published_image.PublishedImage]: + """Returns all the published base images for a given base image version class. + + Args: + dagger_client (dagger.Client): The dagger client used to build the registry. + docker_credentials (Tuple[str, str]): The docker credentials used to fetch published images from DockerHub. + ConnectorBaseImageClass (Type[AirbyteConnectorBaseImage]): The base image version class bound to the registry. + + Returns: + List[published_image.PublishedImage]: The published base images for a given base image version class. + """ + crane_client = docker.CraneClient(dagger_client, docker_credentials) + remote_registry = docker.RemoteRepository(crane_client, consts.REMOTE_REGISTRY, ConnectorBaseImageClass.repository) # type: ignore + return await remote_registry.get_all_images() + + @staticmethod + async def load( + ConnectorBaseImageClass: Type[AirbyteConnectorBaseImage], dagger_client: dagger.Client, docker_credentials: Tuple[str, str] + ) -> VersionRegistry: + """Instantiates a registry by fetching available versions from the remote registry and loading the changelog from disk. + + Args: + ConnectorBaseImageClass (Type[AirbyteConnectorBaseImage]): The base image version class bound to the registry. + + Returns: + VersionRegistry: The registry. + """ + # Loading the local structured changelog file which is stored as a json file. + changelog_entries = VersionRegistry.get_changelog_entries(ConnectorBaseImageClass) + + # Build a dict of changelog entries by version number for easier lookup + changelog_entries_by_version = {entry.version: entry for entry in changelog_entries} + + # Instantiate a crane client and a remote registry to fetch published images from DockerHub + published_docker_images = await VersionRegistry.get_all_published_base_images( + dagger_client, docker_credentials, ConnectorBaseImageClass + ) + + # Build a dict of published images by version number for easier lookup + published_docker_images_by_version = {image.version: image for image in published_docker_images} + + # We union the set of versions from the changelog and the published images to get all the versions we have to consider + all_versions = set(changelog_entries_by_version.keys()) | set(published_docker_images_by_version.keys()) + + registry_entries = [] + # Iterate over all the versions we have to consider and build a registry entry for each of them + # The registry entry will contain the published image if available, and the changelog entry if available + # If the version is not published, the published image will be None + # If the version is not in the changelog, the changelog entry will be None + for version in all_versions: + published_docker_image = published_docker_images_by_version.get(version) + changelog_entry = changelog_entries_by_version.get(version) + registry_entries.append(VersionRegistryEntry(published_docker_image, changelog_entry, version)) + return VersionRegistry(ConnectorBaseImageClass, registry_entries) + + def save_changelog(self): + """Writes the changelog to disk. The changelog is dumped as a json file with a list of ChangelogEntry objects.""" + as_json = json.dumps([entry.changelog_entry.to_serializable_dict() for entry in self.entries if entry.changelog_entry]) + self.changelog_dump_path.write_text(as_json) + + def add_entry(self, new_entry: VersionRegistryEntry) -> List[VersionRegistryEntry]: + """Registers a new entry in the registry and saves the changelog locally. + + Args: + new_entry (VersionRegistryEntry): The new entry to register. + + Returns: + List[VersionRegistryEntry]: All the entries sorted by version number in descending order. + """ + self._entries.append(new_entry) + self.save_changelog() + return self.entries + + @property + def entries(self) -> List[VersionRegistryEntry]: + """Returns all the base image versions sorted by version number in descending order. + + Returns: + List[Type[VersionRegistryEntry]]: All the published versions sorted by version number in descending order. + """ + return sorted(self._entries, key=lambda entry: entry.version, reverse=True) + + @property + def latest_entry(self) -> Optional[VersionRegistryEntry]: + """Returns the latest entry this registry. + The latest entry is the one with the highest version number. + If no entry is available, returns None. + Returns: + Optional[VersionRegistryEntry]: The latest registry entry, or None if no entry is available. + """ + try: + return self.entries[0] + except IndexError: + return None + + @property + def latest_published_entry(self) -> Optional[VersionRegistryEntry]: + """Returns the latest published entry this registry. + The latest published entry is the one with the highest version number among the published entries. + If no entry is available, returns None. + Returns: + Optional[VersionRegistryEntry]: The latest published registry entry, or None if no entry is available. + """ + try: + return [entry for entry in self.entries if entry.published][0] + except IndexError: + return None + + def get_entry_for_version(self, version: semver.VersionInfo) -> Optional[VersionRegistryEntry]: + """Returns the entry for a given version. + If no entry is available, returns None. + Returns: + Optional[VersionRegistryEntry]: The registry entry for the given version, or None if no entry is available. + """ + for entry in self.entries: + if entry.version == version: + return entry + return None + + @property + def latest_not_pre_released_published_entry(self) -> Optional[VersionRegistryEntry]: + """Returns the latest entry with a not pre-released version in this registry which is published. + If no entry is available, returns None. + It is meant to be used externally to get the latest published version. + Returns: + Optional[VersionRegistryEntry]: The latest registry entry with a not pre-released version, or None if no entry is available. + """ + try: + not_pre_release_published_entries = [entry for entry in self.entries if not entry.version.prerelease and entry.published] + return not_pre_release_published_entries[0] + except IndexError: + return None + + +async def get_python_registry(dagger_client: dagger.Client, docker_credentials: Tuple[str, str]) -> VersionRegistry: + return await VersionRegistry.load(AirbytePythonConnectorBaseImage, dagger_client, docker_credentials) + + +async def get_registry_for_language( + dagger_client: dagger.Client, language: ConnectorLanguage, docker_credentials: Tuple[str, str] +) -> VersionRegistry: + """Returns the registry for a given language. + It is meant to be used externally to get the registry for a given connector language. + + Args: + dagger_client (dagger.Client): The dagger client used to build the registry. + language (ConnectorLanguage): The connector language. + docker_credentials (Tuple[str, str]): The docker credentials used to fetch published images from DockerHub. + + Raises: + NotImplementedError: Raised if the registry for the given language is not implemented yet. + + Returns: + VersionRegistry: The registry for the given language. + """ + if language in [ConnectorLanguage.PYTHON, ConnectorLanguage.LOW_CODE]: + return await get_python_registry(dagger_client, docker_credentials) + else: + raise NotImplementedError(f"Registry for language {language} is not implemented yet.") + + +async def get_all_registries(dagger_client: dagger.Client, docker_credentials: Tuple[str, str]) -> List[VersionRegistry]: + return [ + await get_python_registry(dagger_client, docker_credentials), + # await get_java_registry(dagger_client), + ] diff --git a/airbyte-ci/connectors/base_images/generated/changelogs/airbyte_python_connector_base.json b/airbyte-ci/connectors/base_images/generated/changelogs/airbyte_python_connector_base.json new file mode 100644 index 000000000000..d1b57e403a9f --- /dev/null +++ b/airbyte-ci/connectors/base_images/generated/changelogs/airbyte_python_connector_base.json @@ -0,0 +1,17 @@ +[ + { + "version": "1.2.0", + "changelog_entry": "Add CDK system dependencies: nltk data, tesseract, poppler.", + "dockerfile_example": "FROM docker.io/python:3.9.18-slim-bookworm@sha256:44b7f161ed03f85e96d423b9916cdc8cb0509fb970fd643bdbc9896d49e1cad0\nRUN ln -snf /usr/share/zoneinfo/Etc/UTC /etc/localtime\nRUN pip install --upgrade pip==23.2.1\nENV POETRY_VIRTUALENVS_CREATE=false\nENV POETRY_VIRTUALENVS_IN_PROJECT=false\nENV POETRY_NO_INTERACTION=1\nRUN pip install poetry==1.6.1\nRUN sh -c apt update && apt-get install -y socat=1.7.4.4-2\nRUN sh -c apt-get update && apt-get install -y tesseract-ocr=5.3.0-2 poppler-utils=22.12.0-2+b1\nRUN mkdir /usr/share/nltk_data" + }, + { + "version": "1.1.0", + "changelog_entry": "Install socat", + "dockerfile_example": "FROM docker.io/python:3.9.18-slim-bookworm@sha256:44b7f161ed03f85e96d423b9916cdc8cb0509fb970fd643bdbc9896d49e1cad0\nRUN ln -snf /usr/share/zoneinfo/Etc/UTC /etc/localtime\nRUN pip install --upgrade pip==23.2.1\nENV POETRY_VIRTUALENVS_CREATE=false\nENV POETRY_VIRTUALENVS_IN_PROJECT=false\nENV POETRY_NO_INTERACTION=1\nRUN pip install poetry==1.6.1\nRUN sh -c apt update && apt-get install -y socat=1.7.4.4-2" + }, + { + "version": "1.0.0", + "changelog_entry": "Initial release: based on Python 3.9.18, on slim-bookworm system, with pip==23.2.1 and poetry==1.6.1", + "dockerfile_example": "FROM docker.io/python:3.9.18-slim-bookworm@sha256:44b7f161ed03f85e96d423b9916cdc8cb0509fb970fd643bdbc9896d49e1cad0\nRUN ln -snf /usr/share/zoneinfo/Etc/UTC /etc/localtime\nRUN pip install --upgrade pip==23.2.1\nENV POETRY_VIRTUALENVS_CREATE=false\nENV POETRY_VIRTUALENVS_IN_PROJECT=false\nENV POETRY_NO_INTERACTION=1\nRUN pip install poetry==1.6.1" + } +] diff --git a/airbyte-ci/connectors/base_images/poetry.lock b/airbyte-ci/connectors/base_images/poetry.lock new file mode 100644 index 000000000000..44a8b475dca2 --- /dev/null +++ b/airbyte-ci/connectors/base_images/poetry.lock @@ -0,0 +1,2167 @@ +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. + +[[package]] +name = "ansicon" +version = "1.89.0" +description = "Python wrapper for loading Jason Hood's ANSICON" +optional = false +python-versions = "*" +files = [ + {file = "ansicon-1.89.0-py2.py3-none-any.whl", hash = "sha256:f1def52d17f65c2c9682cf8370c03f541f410c1752d6a14029f97318e4b9dfec"}, + {file = "ansicon-1.89.0.tar.gz", hash = "sha256:e4d039def5768a47e4afec8e89e83ec3ae5a26bf00ad851f914d1240b444d2b1"}, +] + +[[package]] +name = "anyio" +version = "4.2.0" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +optional = false +python-versions = ">=3.8" +files = [ + {file = "anyio-4.2.0-py3-none-any.whl", hash = "sha256:745843b39e829e108e518c489b31dc757de7d2131d53fac32bd8df268227bfee"}, + {file = "anyio-4.2.0.tar.gz", hash = "sha256:e1875bb4b4e2de1669f4bc7869b6d3f54231cdced71605e6e64c9be77e3be50f"}, +] + +[package.dependencies] +exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} +idna = ">=2.8" +sniffio = ">=1.1" +typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""} + +[package.extras] +doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] +trio = ["trio (>=0.23)"] + +[[package]] +name = "atomicwrites" +version = "1.4.1" +description = "Atomic file writes." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "beartype" +version = "0.16.4" +description = "Unbearably fast runtime type checking in pure Python." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "beartype-0.16.4-py3-none-any.whl", hash = "sha256:64865952f9dff1e17f22684b3c7286fc79754553b47eaefeb1286224ae8c1bd9"}, + {file = "beartype-0.16.4.tar.gz", hash = "sha256:1ada89cf2d6eb30eb6e156eed2eb5493357782937910d74380918e53c2eae0bf"}, +] + +[package.extras] +all = ["typing-extensions (>=3.10.0.0)"] +dev = ["autoapi (>=0.9.0)", "coverage (>=5.5)", "mypy (>=0.800)", "numpy", "pandera", "pydata-sphinx-theme (<=0.7.2)", "pytest (>=4.0.0)", "sphinx", "sphinx (>=4.2.0,<6.0.0)", "sphinxext-opengraph (>=0.7.5)", "tox (>=3.20.1)", "typing-extensions (>=3.10.0.0)"] +doc-rtd = ["autoapi (>=0.9.0)", "pydata-sphinx-theme (<=0.7.2)", "sphinx (>=4.2.0,<6.0.0)", "sphinxext-opengraph (>=0.7.5)"] +test-tox = ["mypy (>=0.800)", "numpy", "pandera", "pytest (>=4.0.0)", "sphinx", "typing-extensions (>=3.10.0.0)"] +test-tox-coverage = ["coverage (>=5.5)"] + +[[package]] +name = "blessed" +version = "1.20.0" +description = "Easy, practical library for making terminal apps, by providing an elegant, well-documented interface to Colors, Keyboard input, and screen Positioning capabilities." +optional = false +python-versions = ">=2.7" +files = [ + {file = "blessed-1.20.0-py2.py3-none-any.whl", hash = "sha256:0c542922586a265e699188e52d5f5ac5ec0dd517e5a1041d90d2bbf23f906058"}, + {file = "blessed-1.20.0.tar.gz", hash = "sha256:2cdd67f8746e048f00df47a2880f4d6acbcdb399031b604e34ba8f71d5787680"}, +] + +[package.dependencies] +jinxed = {version = ">=1.1.0", markers = "platform_system == \"Windows\""} +six = ">=1.9.0" +wcwidth = ">=0.1.4" + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2023.11.17" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2023.11.17-py3-none-any.whl", hash = "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"}, + {file = "certifi-2023.11.17.tar.gz", hash = "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1"}, +] + +[[package]] +name = "cffi" +version = "1.16.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "ci-credentials" +version = "1.1.0" +description = "CLI tooling to read and manage GSM secrets" +optional = false +python-versions = "^3.10" +files = [] +develop = false + +[package.dependencies] +click = "^8.1.3" +common_utils = {path = "../common_utils", develop = true} +pyyaml = "^6.0" +requests = "^2.28.2" + +[package.source] +type = "directory" +url = "../ci_credentials" + +[[package]] +name = "click" +version = "8.1.7" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "common-utils" +version = "0.0.0" +description = "Suite of all often used classes and common functions" +optional = false +python-versions = "^3.10" +files = [] +develop = true + +[package.dependencies] +cryptography = "^3.4.7" +pyjwt = "^2.1.0" +requests = "^2.28.2" + +[package.source] +type = "directory" +url = "../common_utils" + +[[package]] +name = "connector-ops" +version = "0.3.3" +description = "Packaged maintained by the connector operations team to perform CI for connectors" +optional = false +python-versions = "^3.10" +files = [] +develop = true + +[package.dependencies] +ci-credentials = {path = "../ci_credentials"} +click = "^8.1.3" +GitPython = "^3.1.29" +google-cloud-storage = "^2.8.0" +pandas = "^2.0.3" +pydantic = "^1.9" +pydash = "^7.0.4" +PyGithub = "^1.58.0" +PyYAML = "^6.0" +requests = "^2.28.2" +rich = "^13.0.0" +simpleeval = "^0.9.13" + +[package.source] +type = "directory" +url = "../connector_ops" + +[[package]] +name = "coverage" +version = "7.4.0" +description = "Code coverage measurement for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "coverage-7.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:36b0ea8ab20d6a7564e89cb6135920bc9188fb5f1f7152e94e8300b7b189441a"}, + {file = "coverage-7.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0676cd0ba581e514b7f726495ea75aba3eb20899d824636c6f59b0ed2f88c471"}, + {file = "coverage-7.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0ca5c71a5a1765a0f8f88022c52b6b8be740e512980362f7fdbb03725a0d6b9"}, + {file = "coverage-7.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7c97726520f784239f6c62506bc70e48d01ae71e9da128259d61ca5e9788516"}, + {file = "coverage-7.4.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:815ac2d0f3398a14286dc2cea223a6f338109f9ecf39a71160cd1628786bc6f5"}, + {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:80b5ee39b7f0131ebec7968baa9b2309eddb35b8403d1869e08f024efd883566"}, + {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5b2ccb7548a0b65974860a78c9ffe1173cfb5877460e5a229238d985565574ae"}, + {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:995ea5c48c4ebfd898eacb098164b3cc826ba273b3049e4a889658548e321b43"}, + {file = "coverage-7.4.0-cp310-cp310-win32.whl", hash = "sha256:79287fd95585ed36e83182794a57a46aeae0b64ca53929d1176db56aacc83451"}, + {file = "coverage-7.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:5b14b4f8760006bfdb6e08667af7bc2d8d9bfdb648351915315ea17645347137"}, + {file = "coverage-7.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:04387a4a6ecb330c1878907ce0dc04078ea72a869263e53c72a1ba5bbdf380ca"}, + {file = "coverage-7.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ea81d8f9691bb53f4fb4db603203029643caffc82bf998ab5b59ca05560f4c06"}, + {file = "coverage-7.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74775198b702868ec2d058cb92720a3c5a9177296f75bd97317c787daf711505"}, + {file = "coverage-7.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76f03940f9973bfaee8cfba70ac991825611b9aac047e5c80d499a44079ec0bc"}, + {file = "coverage-7.4.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:485e9f897cf4856a65a57c7f6ea3dc0d4e6c076c87311d4bc003f82cfe199d25"}, + {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6ae8c9d301207e6856865867d762a4b6fd379c714fcc0607a84b92ee63feff70"}, + {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bf477c355274a72435ceb140dc42de0dc1e1e0bf6e97195be30487d8eaaf1a09"}, + {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:83c2dda2666fe32332f8e87481eed056c8b4d163fe18ecc690b02802d36a4d26"}, + {file = "coverage-7.4.0-cp311-cp311-win32.whl", hash = "sha256:697d1317e5290a313ef0d369650cfee1a114abb6021fa239ca12b4849ebbd614"}, + {file = "coverage-7.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:26776ff6c711d9d835557ee453082025d871e30b3fd6c27fcef14733f67f0590"}, + {file = "coverage-7.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:13eaf476ec3e883fe3e5fe3707caeb88268a06284484a3daf8250259ef1ba143"}, + {file = "coverage-7.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846f52f46e212affb5bcf131c952fb4075b55aae6b61adc9856222df89cbe3e2"}, + {file = "coverage-7.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26f66da8695719ccf90e794ed567a1549bb2644a706b41e9f6eae6816b398c4a"}, + {file = "coverage-7.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:164fdcc3246c69a6526a59b744b62e303039a81e42cfbbdc171c91a8cc2f9446"}, + {file = "coverage-7.4.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:316543f71025a6565677d84bc4df2114e9b6a615aa39fb165d697dba06a54af9"}, + {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bb1de682da0b824411e00a0d4da5a784ec6496b6850fdf8c865c1d68c0e318dd"}, + {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:0e8d06778e8fbffccfe96331a3946237f87b1e1d359d7fbe8b06b96c95a5407a"}, + {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a56de34db7b7ff77056a37aedded01b2b98b508227d2d0979d373a9b5d353daa"}, + {file = "coverage-7.4.0-cp312-cp312-win32.whl", hash = "sha256:51456e6fa099a8d9d91497202d9563a320513fcf59f33991b0661a4a6f2ad450"}, + {file = "coverage-7.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:cd3c1e4cb2ff0083758f09be0f77402e1bdf704adb7f89108007300a6da587d0"}, + {file = "coverage-7.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e9d1bf53c4c8de58d22e0e956a79a5b37f754ed1ffdbf1a260d9dcfa2d8a325e"}, + {file = "coverage-7.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:109f5985182b6b81fe33323ab4707011875198c41964f014579cf82cebf2bb85"}, + {file = "coverage-7.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cc9d4bc55de8003663ec94c2f215d12d42ceea128da8f0f4036235a119c88ac"}, + {file = "coverage-7.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc6d65b21c219ec2072c1293c505cf36e4e913a3f936d80028993dd73c7906b1"}, + {file = "coverage-7.4.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a10a4920def78bbfff4eff8a05c51be03e42f1c3735be42d851f199144897ba"}, + {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b8e99f06160602bc64da35158bb76c73522a4010f0649be44a4e167ff8555952"}, + {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7d360587e64d006402b7116623cebf9d48893329ef035278969fa3bbf75b697e"}, + {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:29f3abe810930311c0b5d1a7140f6395369c3db1be68345638c33eec07535105"}, + {file = "coverage-7.4.0-cp38-cp38-win32.whl", hash = "sha256:5040148f4ec43644702e7b16ca864c5314ccb8ee0751ef617d49aa0e2d6bf4f2"}, + {file = "coverage-7.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:9864463c1c2f9cb3b5db2cf1ff475eed2f0b4285c2aaf4d357b69959941aa555"}, + {file = "coverage-7.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:936d38794044b26c99d3dd004d8af0035ac535b92090f7f2bb5aa9c8e2f5cd42"}, + {file = "coverage-7.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:799c8f873794a08cdf216aa5d0531c6a3747793b70c53f70e98259720a6fe2d7"}, + {file = "coverage-7.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7defbb9737274023e2d7af02cac77043c86ce88a907c58f42b580a97d5bcca9"}, + {file = "coverage-7.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a1526d265743fb49363974b7aa8d5899ff64ee07df47dd8d3e37dcc0818f09ed"}, + {file = "coverage-7.4.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf635a52fc1ea401baf88843ae8708591aa4adff875e5c23220de43b1ccf575c"}, + {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:756ded44f47f330666843b5781be126ab57bb57c22adbb07d83f6b519783b870"}, + {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:0eb3c2f32dabe3a4aaf6441dde94f35687224dfd7eb2a7f47f3fd9428e421058"}, + {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bfd5db349d15c08311702611f3dccbef4b4e2ec148fcc636cf8739519b4a5c0f"}, + {file = "coverage-7.4.0-cp39-cp39-win32.whl", hash = "sha256:53d7d9158ee03956e0eadac38dfa1ec8068431ef8058fe6447043db1fb40d932"}, + {file = "coverage-7.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:cfd2a8b6b0d8e66e944d47cdec2f47c48fef2ba2f2dff5a9a75757f64172857e"}, + {file = "coverage-7.4.0-pp38.pp39.pp310-none-any.whl", hash = "sha256:c530833afc4707fe48524a44844493f36d8727f04dcce91fb978c414a8556cc6"}, + {file = "coverage-7.4.0.tar.gz", hash = "sha256:707c0f58cb1712b8809ece32b68996ee1e609f71bd14615bd8f87a1293cb610e"}, +] + +[package.dependencies] +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} + +[package.extras] +toml = ["tomli"] + +[[package]] +name = "cryptography" +version = "3.4.8" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.6" +files = [ + {file = "cryptography-3.4.8-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:a00cf305f07b26c351d8d4e1af84ad7501eca8a342dedf24a7acb0e7b7406e14"}, + {file = "cryptography-3.4.8-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:f44d141b8c4ea5eb4dbc9b3ad992d45580c1d22bf5e24363f2fbf50c2d7ae8a7"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0a7dcbcd3f1913f664aca35d47c1331fce738d44ec34b7be8b9d332151b0b01e"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34dae04a0dce5730d8eb7894eab617d8a70d0c97da76b905de9efb7128ad7085"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1eb7bb0df6f6f583dd8e054689def236255161ebbcf62b226454ab9ec663746b"}, + {file = "cryptography-3.4.8-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:9965c46c674ba8cc572bc09a03f4c649292ee73e1b683adb1ce81e82e9a6a0fb"}, + {file = "cryptography-3.4.8-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:3c4129fc3fdc0fa8e40861b5ac0c673315b3c902bbdc05fc176764815b43dd1d"}, + {file = "cryptography-3.4.8-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:695104a9223a7239d155d7627ad912953b540929ef97ae0c34c7b8bf30857e89"}, + {file = "cryptography-3.4.8-cp36-abi3-win32.whl", hash = "sha256:21ca464b3a4b8d8e86ba0ee5045e103a1fcfac3b39319727bc0fc58c09c6aff7"}, + {file = "cryptography-3.4.8-cp36-abi3-win_amd64.whl", hash = "sha256:3520667fda779eb788ea00080124875be18f2d8f0848ec00733c0ec3bb8219fc"}, + {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d2a6e5ef66503da51d2110edf6c403dc6b494cc0082f85db12f54e9c5d4c3ec5"}, + {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a305600e7a6b7b855cd798e00278161b681ad6e9b7eca94c721d5f588ab212af"}, + {file = "cryptography-3.4.8-pp36-pypy36_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:3fa3a7ccf96e826affdf1a0a9432be74dc73423125c8f96a909e3835a5ef194a"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:d9ec0e67a14f9d1d48dd87a2531009a9b251c02ea42851c060b25c782516ff06"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5b0fbfae7ff7febdb74b574055c7466da334a5371f253732d7e2e7525d570498"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94fff993ee9bc1b2440d3b7243d488c6a3d9724cc2b09cdb297f6a886d040ef7"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:8695456444f277af73a4877db9fc979849cd3ee74c198d04fc0776ebc3db52b9"}, + {file = "cryptography-3.4.8-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:cd65b60cfe004790c795cc35f272e41a3df4631e2fb6b35aa7ac6ef2859d554e"}, + {file = "cryptography-3.4.8.tar.gz", hash = "sha256:94cc5ed4ceaefcbe5bf38c8fba6a21fc1d365bb8fb826ea1688e3370b2e24a1c"}, +] + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] +docstest = ["doc8", "pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +sdist = ["setuptools-rust (>=0.11.4)"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] + +[[package]] +name = "dagger-io" +version = "0.9.6" +description = "A client package for running Dagger pipelines in Python." +optional = false +python-versions = ">=3.10" +files = [ + {file = "dagger_io-0.9.6-py3-none-any.whl", hash = "sha256:e2f1e4bbc252071a314fa5b0bad11a910433a9ee043972b716f6fcc5f9fc8236"}, + {file = "dagger_io-0.9.6.tar.gz", hash = "sha256:147b5a33c44d17f602a4121679893655e91308beb8c46a466afed39cf40f789b"}, +] + +[package.dependencies] +anyio = ">=3.6.2" +beartype = ">=0.11.0" +cattrs = ">=22.2.0" +gql = ">=3.4.0" +graphql-core = ">=3.2.3" +httpx = ">=0.23.1" +platformdirs = ">=2.6.2" +rich = ">=10.11.0" +typing-extensions = ">=4.8.0" + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "editor" +version = "1.6.5" +description = "🖋 Open the default text editor 🖋" +optional = false +python-versions = ">=3.8" +files = [ + {file = "editor-1.6.5-py3-none-any.whl", hash = "sha256:53c26dd78333b50b8cdcf67748956afa75fabcb5bb25e96a00515504f58e49a8"}, + {file = "editor-1.6.5.tar.gz", hash = "sha256:5a8ad611d2a05de34994df3781605e26e63492f82f04c2e93abdd330eed6fa8d"}, +] + +[package.dependencies] +runs = "*" +xmod = "*" + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "gitdb" +version = "4.0.11" +description = "Git Object Database" +optional = false +python-versions = ">=3.7" +files = [ + {file = "gitdb-4.0.11-py3-none-any.whl", hash = "sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4"}, + {file = "gitdb-4.0.11.tar.gz", hash = "sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b"}, +] + +[package.dependencies] +smmap = ">=3.0.1,<6" + +[[package]] +name = "gitpython" +version = "3.1.41" +description = "GitPython is a Python library used to interact with Git repositories" +optional = false +python-versions = ">=3.7" +files = [ + {file = "GitPython-3.1.41-py3-none-any.whl", hash = "sha256:c36b6634d069b3f719610175020a9aed919421c87552185b085e04fbbdb10b7c"}, + {file = "GitPython-3.1.41.tar.gz", hash = "sha256:ed66e624884f76df22c8e16066d567aaa5a37d5b5fa19db2c6df6f7156db9048"}, +] + +[package.dependencies] +gitdb = ">=4.0.1,<5" + +[package.extras] +test = ["black", "coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "sumtypes"] + +[[package]] +name = "google-api-core" +version = "2.15.0" +description = "Google API client core library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-api-core-2.15.0.tar.gz", hash = "sha256:abc978a72658f14a2df1e5e12532effe40f94f868f6e23d95133bd6abcca35ca"}, + {file = "google_api_core-2.15.0-py3-none-any.whl", hash = "sha256:2aa56d2be495551e66bbff7f729b790546f87d5c90e74781aa77233bcb395a8a"}, +] + +[package.dependencies] +google-auth = ">=2.14.1,<3.0.dev0" +googleapis-common-protos = ">=1.56.2,<2.0.dev0" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +requests = ">=2.18.0,<3.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0)"] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] + +[[package]] +name = "google-auth" +version = "2.26.2" +description = "Google Authentication Library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-auth-2.26.2.tar.gz", hash = "sha256:97327dbbf58cccb58fc5a1712bba403ae76668e64814eb30f7316f7e27126b81"}, + {file = "google_auth-2.26.2-py2.py3-none-any.whl", hash = "sha256:3f445c8ce9b61ed6459aad86d8ccdba4a9afed841b2d1451a11ef4db08957424"}, +] + +[package.dependencies] +cachetools = ">=2.0.0,<6.0" +pyasn1-modules = ">=0.2.1" +rsa = ">=3.1.4,<5" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] +enterprise-cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"] +pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] +reauth = ["pyu2f (>=0.1.5)"] +requests = ["requests (>=2.20.0,<3.0.0.dev0)"] + +[[package]] +name = "google-cloud-core" +version = "2.4.1" +description = "Google Cloud API client core library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-cloud-core-2.4.1.tar.gz", hash = "sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073"}, + {file = "google_cloud_core-2.4.1-py2.py3-none-any.whl", hash = "sha256:a9e6a4422b9ac5c29f79a0ede9485473338e2ce78d91f2370c01e730eab22e61"}, +] + +[package.dependencies] +google-api-core = ">=1.31.6,<2.0.dev0 || >2.3.0,<3.0.0dev" +google-auth = ">=1.25.0,<3.0dev" + +[package.extras] +grpc = ["grpcio (>=1.38.0,<2.0dev)", "grpcio-status (>=1.38.0,<2.0.dev0)"] + +[[package]] +name = "google-cloud-storage" +version = "2.14.0" +description = "Google Cloud Storage API client library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-cloud-storage-2.14.0.tar.gz", hash = "sha256:2d23fcf59b55e7b45336729c148bb1c464468c69d5efbaee30f7201dd90eb97e"}, + {file = "google_cloud_storage-2.14.0-py2.py3-none-any.whl", hash = "sha256:8641243bbf2a2042c16a6399551fbb13f062cbc9a2de38d6c0bb5426962e9dbd"}, +] + +[package.dependencies] +google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" +google-auth = ">=2.23.3,<3.0dev" +google-cloud-core = ">=2.3.0,<3.0dev" +google-crc32c = ">=1.0,<2.0dev" +google-resumable-media = ">=2.6.0" +requests = ">=2.18.0,<3.0.0dev" + +[package.extras] +protobuf = ["protobuf (<5.0.0dev)"] + +[[package]] +name = "google-crc32c" +version = "1.5.0" +description = "A python wrapper of the C library 'Google CRC32C'" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-crc32c-1.5.0.tar.gz", hash = "sha256:89284716bc6a5a415d4eaa11b1726d2d60a0cd12aadf5439828353662ede9dd7"}, + {file = "google_crc32c-1.5.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:596d1f98fc70232fcb6590c439f43b350cb762fb5d61ce7b0e9db4539654cc13"}, + {file = "google_crc32c-1.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:be82c3c8cfb15b30f36768797a640e800513793d6ae1724aaaafe5bf86f8f346"}, + {file = "google_crc32c-1.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:461665ff58895f508e2866824a47bdee72497b091c730071f2b7575d5762ab65"}, + {file = "google_crc32c-1.5.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2096eddb4e7c7bdae4bd69ad364e55e07b8316653234a56552d9c988bd2d61b"}, + {file = "google_crc32c-1.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:116a7c3c616dd14a3de8c64a965828b197e5f2d121fedd2f8c5585c547e87b02"}, + {file = "google_crc32c-1.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:5829b792bf5822fd0a6f6eb34c5f81dd074f01d570ed7f36aa101d6fc7a0a6e4"}, + {file = "google_crc32c-1.5.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:64e52e2b3970bd891309c113b54cf0e4384762c934d5ae56e283f9a0afcd953e"}, + {file = "google_crc32c-1.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:02ebb8bf46c13e36998aeaad1de9b48f4caf545e91d14041270d9dca767b780c"}, + {file = "google_crc32c-1.5.0-cp310-cp310-win32.whl", hash = "sha256:2e920d506ec85eb4ba50cd4228c2bec05642894d4c73c59b3a2fe20346bd00ee"}, + {file = "google_crc32c-1.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:07eb3c611ce363c51a933bf6bd7f8e3878a51d124acfc89452a75120bc436289"}, + {file = "google_crc32c-1.5.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:cae0274952c079886567f3f4f685bcaf5708f0a23a5f5216fdab71f81a6c0273"}, + {file = "google_crc32c-1.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1034d91442ead5a95b5aaef90dbfaca8633b0247d1e41621d1e9f9db88c36298"}, + {file = "google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c42c70cd1d362284289c6273adda4c6af8039a8ae12dc451dcd61cdabb8ab57"}, + {file = "google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8485b340a6a9e76c62a7dce3c98e5f102c9219f4cfbf896a00cf48caf078d438"}, + {file = "google_crc32c-1.5.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77e2fd3057c9d78e225fa0a2160f96b64a824de17840351b26825b0848022906"}, + {file = "google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f583edb943cf2e09c60441b910d6a20b4d9d626c75a36c8fcac01a6c96c01183"}, + {file = "google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:a1fd716e7a01f8e717490fbe2e431d2905ab8aa598b9b12f8d10abebb36b04dd"}, + {file = "google_crc32c-1.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:72218785ce41b9cfd2fc1d6a017dc1ff7acfc4c17d01053265c41a2c0cc39b8c"}, + {file = "google_crc32c-1.5.0-cp311-cp311-win32.whl", hash = "sha256:66741ef4ee08ea0b2cc3c86916ab66b6aef03768525627fd6a1b34968b4e3709"}, + {file = "google_crc32c-1.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:ba1eb1843304b1e5537e1fca632fa894d6f6deca8d6389636ee5b4797affb968"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:98cb4d057f285bd80d8778ebc4fde6b4d509ac3f331758fb1528b733215443ae"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd8536e902db7e365f49e7d9029283403974ccf29b13fc7028b97e2295b33556"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:19e0a019d2c4dcc5e598cd4a4bc7b008546b0358bd322537c74ad47a5386884f"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02c65b9817512edc6a4ae7c7e987fea799d2e0ee40c53ec573a692bee24de876"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6ac08d24c1f16bd2bf5eca8eaf8304812f44af5cfe5062006ec676e7e1d50afc"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3359fc442a743e870f4588fcf5dcbc1bf929df1fad8fb9905cd94e5edb02e84c"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e986b206dae4476f41bcec1faa057851f3889503a70e1bdb2378d406223994a"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:de06adc872bcd8c2a4e0dc51250e9e65ef2ca91be023b9d13ebd67c2ba552e1e"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-win32.whl", hash = "sha256:d3515f198eaa2f0ed49f8819d5732d70698c3fa37384146079b3799b97667a94"}, + {file = "google_crc32c-1.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:67b741654b851abafb7bc625b6d1cdd520a379074e64b6a128e3b688c3c04740"}, + {file = "google_crc32c-1.5.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:c02ec1c5856179f171e032a31d6f8bf84e5a75c45c33b2e20a3de353b266ebd8"}, + {file = "google_crc32c-1.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:edfedb64740750e1a3b16152620220f51d58ff1b4abceb339ca92e934775c27a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84e6e8cd997930fc66d5bb4fde61e2b62ba19d62b7abd7a69920406f9ecca946"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:024894d9d3cfbc5943f8f230e23950cd4906b2fe004c72e29b209420a1e6b05a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:998679bf62b7fb599d2878aa3ed06b9ce688b8974893e7223c60db155f26bd8d"}, + {file = "google_crc32c-1.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:83c681c526a3439b5cf94f7420471705bbf96262f49a6fe546a6db5f687a3d4a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4c6fdd4fccbec90cc8a01fc00773fcd5fa28db683c116ee3cb35cd5da9ef6c37"}, + {file = "google_crc32c-1.5.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5ae44e10a8e3407dbe138984f21e536583f2bba1be9491239f942c2464ac0894"}, + {file = "google_crc32c-1.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:37933ec6e693e51a5b07505bd05de57eee12f3e8c32b07da7e73669398e6630a"}, + {file = "google_crc32c-1.5.0-cp38-cp38-win32.whl", hash = "sha256:fe70e325aa68fa4b5edf7d1a4b6f691eb04bbccac0ace68e34820d283b5f80d4"}, + {file = "google_crc32c-1.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:74dea7751d98034887dbd821b7aae3e1d36eda111d6ca36c206c44478035709c"}, + {file = "google_crc32c-1.5.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c6c777a480337ac14f38564ac88ae82d4cd238bf293f0a22295b66eb89ffced7"}, + {file = "google_crc32c-1.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:759ce4851a4bb15ecabae28f4d2e18983c244eddd767f560165563bf9aefbc8d"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f13cae8cc389a440def0c8c52057f37359014ccbc9dc1f0827936bcd367c6100"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e560628513ed34759456a416bf86b54b2476c59144a9138165c9a1575801d0d9"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1674e4307fa3024fc897ca774e9c7562c957af85df55efe2988ed9056dc4e57"}, + {file = "google_crc32c-1.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:278d2ed7c16cfc075c91378c4f47924c0625f5fc84b2d50d921b18b7975bd210"}, + {file = "google_crc32c-1.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d5280312b9af0976231f9e317c20e4a61cd2f9629b7bfea6a693d1878a264ebd"}, + {file = "google_crc32c-1.5.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:8b87e1a59c38f275c0e3676fc2ab6d59eccecfd460be267ac360cc31f7bcde96"}, + {file = "google_crc32c-1.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7c074fece789b5034b9b1404a1f8208fc2d4c6ce9decdd16e8220c5a793e6f61"}, + {file = "google_crc32c-1.5.0-cp39-cp39-win32.whl", hash = "sha256:7f57f14606cd1dd0f0de396e1e53824c371e9544a822648cd76c034d209b559c"}, + {file = "google_crc32c-1.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:a2355cba1f4ad8b6988a4ca3feed5bff33f6af2d7f134852cf279c2aebfde541"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:f314013e7dcd5cf45ab1945d92e713eec788166262ae8deb2cfacd53def27325"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b747a674c20a67343cb61d43fdd9207ce5da6a99f629c6e2541aa0e89215bcd"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8f24ed114432de109aa9fd317278518a5af2d31ac2ea6b952b2f7782b43da091"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8667b48e7a7ef66afba2c81e1094ef526388d35b873966d8a9a447974ed9178"}, + {file = "google_crc32c-1.5.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:1c7abdac90433b09bad6c43a43af253e688c9cfc1c86d332aed13f9a7c7f65e2"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:6f998db4e71b645350b9ac28a2167e6632c239963ca9da411523bb439c5c514d"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c99616c853bb585301df6de07ca2cadad344fd1ada6d62bb30aec05219c45d2"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2ad40e31093a4af319dadf503b2467ccdc8f67c72e4bcba97f8c10cb078207b5"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd67cf24a553339d5062eff51013780a00d6f97a39ca062781d06b3a73b15462"}, + {file = "google_crc32c-1.5.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:398af5e3ba9cf768787eef45c803ff9614cc3e22a5b2f7d7ae116df8b11e3314"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:b1f8133c9a275df5613a451e73f36c2aea4fe13c5c8997e22cf355ebd7bd0728"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ba053c5f50430a3fcfd36f75aff9caeba0440b2d076afdb79a318d6ca245f88"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:272d3892a1e1a2dbc39cc5cde96834c236d5327e2122d3aaa19f6614531bb6eb"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:635f5d4dd18758a1fbd1049a8e8d2fee4ffed124462d837d1a02a0e009c3ab31"}, + {file = "google_crc32c-1.5.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:c672d99a345849301784604bfeaeba4db0c7aae50b95be04dd651fd2a7310b93"}, +] + +[package.extras] +testing = ["pytest"] + +[[package]] +name = "google-resumable-media" +version = "2.7.0" +description = "Utilities for Google Media Downloads and Resumable Uploads" +optional = false +python-versions = ">= 3.7" +files = [ + {file = "google-resumable-media-2.7.0.tar.gz", hash = "sha256:5f18f5fa9836f4b083162064a1c2c98c17239bfda9ca50ad970ccf905f3e625b"}, + {file = "google_resumable_media-2.7.0-py2.py3-none-any.whl", hash = "sha256:79543cfe433b63fd81c0844b7803aba1bb8950b47bedf7d980c38fa123937e08"}, +] + +[package.dependencies] +google-crc32c = ">=1.0,<2.0dev" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)", "google-auth (>=1.22.0,<2.0dev)"] +requests = ["requests (>=2.18.0,<3.0.0dev)"] + +[[package]] +name = "googleapis-common-protos" +version = "1.62.0" +description = "Common protobufs used in Google APIs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "googleapis-common-protos-1.62.0.tar.gz", hash = "sha256:83f0ece9f94e5672cced82f592d2a5edf527a96ed1794f0bab36d5735c996277"}, + {file = "googleapis_common_protos-1.62.0-py2.py3-none-any.whl", hash = "sha256:4750113612205514f9f6aa4cb00d523a94f3e8c06c5ad2fee466387dc4875f07"}, +] + +[package.dependencies] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] + +[[package]] +name = "gql" +version = "3.5.0" +description = "GraphQL client for Python" +optional = false +python-versions = "*" +files = [ + {file = "gql-3.5.0-py2.py3-none-any.whl", hash = "sha256:70dda5694a5b194a8441f077aa5fb70cc94e4ec08016117523f013680901ecb7"}, + {file = "gql-3.5.0.tar.gz", hash = "sha256:ccb9c5db543682b28f577069950488218ed65d4ac70bb03b6929aaadaf636de9"}, +] + +[package.dependencies] +anyio = ">=3.0,<5" +backoff = ">=1.11.1,<3.0" +graphql-core = ">=3.2,<3.3" +yarl = ">=1.6,<2.0" + +[package.extras] +aiohttp = ["aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)"] +all = ["aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "botocore (>=1.21,<2)", "httpx (>=0.23.1,<1)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "websockets (>=10,<12)"] +botocore = ["botocore (>=1.21,<2)"] +dev = ["aiofiles", "aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "black (==22.3.0)", "botocore (>=1.21,<2)", "check-manifest (>=0.42,<1)", "flake8 (==3.8.1)", "httpx (>=0.23.1,<1)", "isort (==4.3.21)", "mock (==4.0.2)", "mypy (==0.910)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "sphinx (>=5.3.0,<6)", "sphinx-argparse (==0.2.5)", "sphinx-rtd-theme (>=0.4,<1)", "types-aiofiles", "types-mock", "types-requests", "vcrpy (==4.4.0)", "websockets (>=10,<12)"] +httpx = ["httpx (>=0.23.1,<1)"] +requests = ["requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)"] +test = ["aiofiles", "aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "botocore (>=1.21,<2)", "httpx (>=0.23.1,<1)", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "vcrpy (==4.4.0)", "websockets (>=10,<12)"] +test-no-transport = ["aiofiles", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "vcrpy (==4.4.0)"] +websockets = ["websockets (>=10,<12)"] + +[[package]] +name = "graphql-core" +version = "3.2.3" +description = "GraphQL implementation for Python, a port of GraphQL.js, the JavaScript reference implementation for GraphQL." +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "graphql-core-3.2.3.tar.gz", hash = "sha256:06d2aad0ac723e35b1cb47885d3e5c45e956a53bc1b209a9fc5369007fe46676"}, + {file = "graphql_core-3.2.3-py3-none-any.whl", hash = "sha256:5766780452bd5ec8ba133f8bf287dc92713e3868ddd83aee4faab9fc3e303dc3"}, +] + +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] + +[[package]] +name = "httpcore" +version = "1.0.2" +description = "A minimal low-level HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpcore-1.0.2-py3-none-any.whl", hash = "sha256:096cc05bca73b8e459a1fc3dcf585148f63e534eae4339559c9b8a8d6399acc7"}, + {file = "httpcore-1.0.2.tar.gz", hash = "sha256:9fc092e4799b26174648e54b74ed5f683132a464e95643b226e00c2ed2fa6535"}, +] + +[package.dependencies] +certifi = "*" +h11 = ">=0.13,<0.15" + +[package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<0.23.0)"] + +[[package]] +name = "httpx" +version = "0.26.0" +description = "The next generation HTTP client." +optional = false +python-versions = ">=3.8" +files = [ + {file = "httpx-0.26.0-py3-none-any.whl", hash = "sha256:8915f5a3627c4d47b73e8202457cb28f1266982d1159bd5779d86a80c0eab1cd"}, + {file = "httpx-0.26.0.tar.gz", hash = "sha256:451b55c30d5185ea6b23c2c793abf9bb237d2a7dfb901ced6ff69ad37ec1dfaf"}, +] + +[package.dependencies] +anyio = "*" +certifi = "*" +httpcore = "==1.*" +idna = "*" +sniffio = "*" + +[package.extras] +brotli = ["brotli", "brotlicffi"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "inquirer" +version = "3.2.1" +description = "Collection of common interactive command line user interfaces, based on Inquirer.js" +optional = false +python-versions = ">=3.8.1" +files = [ + {file = "inquirer-3.2.1-py3-none-any.whl", hash = "sha256:e1a0a001b499633ca69d2ea64da712b449939e8fad8fa47caebc92b0ee212df4"}, + {file = "inquirer-3.2.1.tar.gz", hash = "sha256:d5ff9bb8cd07bd3f076eabad8ae338280886e93998ff10461975b768e3854fbc"}, +] + +[package.dependencies] +blessed = ">=1.19.0" +editor = ">=1.6.0" +readchar = ">=3.0.6" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jinxed" +version = "1.2.1" +description = "Jinxed Terminal Library" +optional = false +python-versions = "*" +files = [ + {file = "jinxed-1.2.1-py2.py3-none-any.whl", hash = "sha256:37422659c4925969c66148c5e64979f553386a4226b9484d910d3094ced37d30"}, + {file = "jinxed-1.2.1.tar.gz", hash = "sha256:30c3f861b73279fea1ed928cfd4dfb1f273e16cd62c8a32acfac362da0f78f3f"}, +] + +[package.dependencies] +ansicon = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "markdown-it-py" +version = "3.0.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +optional = false +python-versions = ">=3.8" +files = [ + {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, + {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, +] + +[package.dependencies] +mdurl = ">=0.1,<1.0" + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +code-style = ["pre-commit (>=3.0,<4.0)"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + +[[package]] +name = "markupsafe" +version = "2.1.3" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win32.whl", hash = "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win32.whl", hash = "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win_amd64.whl", hash = "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win32.whl", hash = "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win_amd64.whl", hash = "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win32.whl", hash = "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba"}, + {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] + +[[package]] +name = "multidict" +version = "6.0.4" +description = "multidict implementation" +optional = false +python-versions = ">=3.7" +files = [ + {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8"}, + {file = "multidict-6.0.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171"}, + {file = "multidict-6.0.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93"}, + {file = "multidict-6.0.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0"}, + {file = "multidict-6.0.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5"}, + {file = "multidict-6.0.4-cp310-cp310-win32.whl", hash = "sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8"}, + {file = "multidict-6.0.4-cp310-cp310-win_amd64.whl", hash = "sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3"}, + {file = "multidict-6.0.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710"}, + {file = "multidict-6.0.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed"}, + {file = "multidict-6.0.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461"}, + {file = "multidict-6.0.4-cp311-cp311-win32.whl", hash = "sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636"}, + {file = "multidict-6.0.4-cp311-cp311-win_amd64.whl", hash = "sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0"}, + {file = "multidict-6.0.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9"}, + {file = "multidict-6.0.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87"}, + {file = "multidict-6.0.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d"}, + {file = "multidict-6.0.4-cp37-cp37m-win32.whl", hash = "sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775"}, + {file = "multidict-6.0.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161"}, + {file = "multidict-6.0.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258"}, + {file = "multidict-6.0.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d"}, + {file = "multidict-6.0.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1"}, + {file = "multidict-6.0.4-cp38-cp38-win32.whl", hash = "sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779"}, + {file = "multidict-6.0.4-cp38-cp38-win_amd64.whl", hash = "sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35"}, + {file = "multidict-6.0.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1"}, + {file = "multidict-6.0.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176"}, + {file = "multidict-6.0.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95"}, + {file = "multidict-6.0.4-cp39-cp39-win32.whl", hash = "sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313"}, + {file = "multidict-6.0.4-cp39-cp39-win_amd64.whl", hash = "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2"}, + {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, +] + +[[package]] +name = "mypy" +version = "1.8.0" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mypy-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:485a8942f671120f76afffff70f259e1cd0f0cfe08f81c05d8816d958d4577d3"}, + {file = "mypy-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:df9824ac11deaf007443e7ed2a4a26bebff98d2bc43c6da21b2b64185da011c4"}, + {file = "mypy-1.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2afecd6354bbfb6e0160f4e4ad9ba6e4e003b767dd80d85516e71f2e955ab50d"}, + {file = "mypy-1.8.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8963b83d53ee733a6e4196954502b33567ad07dfd74851f32be18eb932fb1cb9"}, + {file = "mypy-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:e46f44b54ebddbeedbd3d5b289a893219065ef805d95094d16a0af6630f5d410"}, + {file = "mypy-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:855fe27b80375e5c5878492f0729540db47b186509c98dae341254c8f45f42ae"}, + {file = "mypy-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4c886c6cce2d070bd7df4ec4a05a13ee20c0aa60cb587e8d1265b6c03cf91da3"}, + {file = "mypy-1.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d19c413b3c07cbecf1f991e2221746b0d2a9410b59cb3f4fb9557f0365a1a817"}, + {file = "mypy-1.8.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9261ed810972061388918c83c3f5cd46079d875026ba97380f3e3978a72f503d"}, + {file = "mypy-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:51720c776d148bad2372ca21ca29256ed483aa9a4cdefefcef49006dff2a6835"}, + {file = "mypy-1.8.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:52825b01f5c4c1c4eb0db253ec09c7aa17e1a7304d247c48b6f3599ef40db8bd"}, + {file = "mypy-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f5ac9a4eeb1ec0f1ccdc6f326bcdb464de5f80eb07fb38b5ddd7b0de6bc61e55"}, + {file = "mypy-1.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afe3fe972c645b4632c563d3f3eff1cdca2fa058f730df2b93a35e3b0c538218"}, + {file = "mypy-1.8.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:42c6680d256ab35637ef88891c6bd02514ccb7e1122133ac96055ff458f93fc3"}, + {file = "mypy-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:720a5ca70e136b675af3af63db533c1c8c9181314d207568bbe79051f122669e"}, + {file = "mypy-1.8.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:028cf9f2cae89e202d7b6593cd98db6759379f17a319b5faf4f9978d7084cdc6"}, + {file = "mypy-1.8.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4e6d97288757e1ddba10dd9549ac27982e3e74a49d8d0179fc14d4365c7add66"}, + {file = "mypy-1.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f1478736fcebb90f97e40aff11a5f253af890c845ee0c850fe80aa060a267c6"}, + {file = "mypy-1.8.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42419861b43e6962a649068a61f4a4839205a3ef525b858377a960b9e2de6e0d"}, + {file = "mypy-1.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:2b5b6c721bd4aabaadead3a5e6fa85c11c6c795e0c81a7215776ef8afc66de02"}, + {file = "mypy-1.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5c1538c38584029352878a0466f03a8ee7547d7bd9f641f57a0f3017a7c905b8"}, + {file = "mypy-1.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ef4be7baf08a203170f29e89d79064463b7fc7a0908b9d0d5114e8009c3a259"}, + {file = "mypy-1.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7178def594014aa6c35a8ff411cf37d682f428b3b5617ca79029d8ae72f5402b"}, + {file = "mypy-1.8.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ab3c84fa13c04aeeeabb2a7f67a25ef5d77ac9d6486ff33ded762ef353aa5592"}, + {file = "mypy-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:99b00bc72855812a60d253420d8a2eae839b0afa4938f09f4d2aa9bb4654263a"}, + {file = "mypy-1.8.0-py3-none-any.whl", hash = "sha256:538fd81bb5e430cc1381a443971c0475582ff9f434c16cd46d2c66763ce85d9d"}, + {file = "mypy-1.8.0.tar.gz", hash = "sha256:6ff8b244d7085a0b425b56d327b480c3b29cafbd2eff27316a004f9a7391ae07"}, +] + +[package.dependencies] +mypy-extensions = ">=1.0.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=4.1.0" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "numpy" +version = "1.26.3" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "numpy-1.26.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:806dd64230dbbfaca8a27faa64e2f414bf1c6622ab78cc4264f7f5f028fee3bf"}, + {file = "numpy-1.26.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02f98011ba4ab17f46f80f7f8f1c291ee7d855fcef0a5a98db80767a468c85cd"}, + {file = "numpy-1.26.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d45b3ec2faed4baca41c76617fcdcfa4f684ff7a151ce6fc78ad3b6e85af0a6"}, + {file = "numpy-1.26.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdd2b45bf079d9ad90377048e2747a0c82351989a2165821f0c96831b4a2a54b"}, + {file = "numpy-1.26.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:211ddd1e94817ed2d175b60b6374120244a4dd2287f4ece45d49228b4d529178"}, + {file = "numpy-1.26.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b1240f767f69d7c4c8a29adde2310b871153df9b26b5cb2b54a561ac85146485"}, + {file = "numpy-1.26.3-cp310-cp310-win32.whl", hash = "sha256:21a9484e75ad018974a2fdaa216524d64ed4212e418e0a551a2d83403b0531d3"}, + {file = "numpy-1.26.3-cp310-cp310-win_amd64.whl", hash = "sha256:9e1591f6ae98bcfac2a4bbf9221c0b92ab49762228f38287f6eeb5f3f55905ce"}, + {file = "numpy-1.26.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b831295e5472954104ecb46cd98c08b98b49c69fdb7040483aff799a755a7374"}, + {file = "numpy-1.26.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9e87562b91f68dd8b1c39149d0323b42e0082db7ddb8e934ab4c292094d575d6"}, + {file = "numpy-1.26.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c66d6fec467e8c0f975818c1796d25c53521124b7cfb760114be0abad53a0a2"}, + {file = "numpy-1.26.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f25e2811a9c932e43943a2615e65fc487a0b6b49218899e62e426e7f0a57eeda"}, + {file = "numpy-1.26.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:af36e0aa45e25c9f57bf684b1175e59ea05d9a7d3e8e87b7ae1a1da246f2767e"}, + {file = "numpy-1.26.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:51c7f1b344f302067b02e0f5b5d2daa9ed4a721cf49f070280ac202738ea7f00"}, + {file = "numpy-1.26.3-cp311-cp311-win32.whl", hash = "sha256:7ca4f24341df071877849eb2034948459ce3a07915c2734f1abb4018d9c49d7b"}, + {file = "numpy-1.26.3-cp311-cp311-win_amd64.whl", hash = "sha256:39763aee6dfdd4878032361b30b2b12593fb445ddb66bbac802e2113eb8a6ac4"}, + {file = "numpy-1.26.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a7081fd19a6d573e1a05e600c82a1c421011db7935ed0d5c483e9dd96b99cf13"}, + {file = "numpy-1.26.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12c70ac274b32bc00c7f61b515126c9205323703abb99cd41836e8125ea0043e"}, + {file = "numpy-1.26.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f784e13e598e9594750b2ef6729bcd5a47f6cfe4a12cca13def35e06d8163e3"}, + {file = "numpy-1.26.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f24750ef94d56ce6e33e4019a8a4d68cfdb1ef661a52cdaee628a56d2437419"}, + {file = "numpy-1.26.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:77810ef29e0fb1d289d225cabb9ee6cf4d11978a00bb99f7f8ec2132a84e0166"}, + {file = "numpy-1.26.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8ed07a90f5450d99dad60d3799f9c03c6566709bd53b497eb9ccad9a55867f36"}, + {file = "numpy-1.26.3-cp312-cp312-win32.whl", hash = "sha256:f73497e8c38295aaa4741bdfa4fda1a5aedda5473074369eca10626835445511"}, + {file = "numpy-1.26.3-cp312-cp312-win_amd64.whl", hash = "sha256:da4b0c6c699a0ad73c810736303f7fbae483bcb012e38d7eb06a5e3b432c981b"}, + {file = "numpy-1.26.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1666f634cb3c80ccbd77ec97bc17337718f56d6658acf5d3b906ca03e90ce87f"}, + {file = "numpy-1.26.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:18c3319a7d39b2c6a9e3bb75aab2304ab79a811ac0168a671a62e6346c29b03f"}, + {file = "numpy-1.26.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b7e807d6888da0db6e7e75838444d62495e2b588b99e90dd80c3459594e857b"}, + {file = "numpy-1.26.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4d362e17bcb0011738c2d83e0a65ea8ce627057b2fdda37678f4374a382a137"}, + {file = "numpy-1.26.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b8c275f0ae90069496068c714387b4a0eba5d531aace269559ff2b43655edd58"}, + {file = "numpy-1.26.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:cc0743f0302b94f397a4a65a660d4cd24267439eb16493fb3caad2e4389bccbb"}, + {file = "numpy-1.26.3-cp39-cp39-win32.whl", hash = "sha256:9bc6d1a7f8cedd519c4b7b1156d98e051b726bf160715b769106661d567b3f03"}, + {file = "numpy-1.26.3-cp39-cp39-win_amd64.whl", hash = "sha256:867e3644e208c8922a3be26fc6bbf112a035f50f0a86497f98f228c50c607bb2"}, + {file = "numpy-1.26.3-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3c67423b3703f8fbd90f5adaa37f85b5794d3366948efe9a5190a5f3a83fc34e"}, + {file = "numpy-1.26.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46f47ee566d98849323f01b349d58f2557f02167ee301e5e28809a8c0e27a2d0"}, + {file = "numpy-1.26.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a8474703bffc65ca15853d5fd4d06b18138ae90c17c8d12169968e998e448bb5"}, + {file = "numpy-1.26.3.tar.gz", hash = "sha256:697df43e2b6310ecc9d95f05d5ef20eacc09c7c4ecc9da3f235d39e71b7da1e4"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pandas" +version = "2.1.4" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pandas-2.1.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bdec823dc6ec53f7a6339a0e34c68b144a7a1fd28d80c260534c39c62c5bf8c9"}, + {file = "pandas-2.1.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:294d96cfaf28d688f30c918a765ea2ae2e0e71d3536754f4b6de0ea4a496d034"}, + {file = "pandas-2.1.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b728fb8deba8905b319f96447a27033969f3ea1fea09d07d296c9030ab2ed1d"}, + {file = "pandas-2.1.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00028e6737c594feac3c2df15636d73ace46b8314d236100b57ed7e4b9ebe8d9"}, + {file = "pandas-2.1.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:426dc0f1b187523c4db06f96fb5c8d1a845e259c99bda74f7de97bd8a3bb3139"}, + {file = "pandas-2.1.4-cp310-cp310-win_amd64.whl", hash = "sha256:f237e6ca6421265643608813ce9793610ad09b40154a3344a088159590469e46"}, + {file = "pandas-2.1.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b7d852d16c270e4331f6f59b3e9aa23f935f5c4b0ed2d0bc77637a8890a5d092"}, + {file = "pandas-2.1.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bd7d5f2f54f78164b3d7a40f33bf79a74cdee72c31affec86bfcabe7e0789821"}, + {file = "pandas-2.1.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0aa6e92e639da0d6e2017d9ccff563222f4eb31e4b2c3cf32a2a392fc3103c0d"}, + {file = "pandas-2.1.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d797591b6846b9db79e65dc2d0d48e61f7db8d10b2a9480b4e3faaddc421a171"}, + {file = "pandas-2.1.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d2d3e7b00f703aea3945995ee63375c61b2e6aa5aa7871c5d622870e5e137623"}, + {file = "pandas-2.1.4-cp311-cp311-win_amd64.whl", hash = "sha256:dc9bf7ade01143cddc0074aa6995edd05323974e6e40d9dbde081021ded8510e"}, + {file = "pandas-2.1.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:482d5076e1791777e1571f2e2d789e940dedd927325cc3cb6d0800c6304082f6"}, + {file = "pandas-2.1.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8a706cfe7955c4ca59af8c7a0517370eafbd98593155b48f10f9811da440248b"}, + {file = "pandas-2.1.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0513a132a15977b4a5b89aabd304647919bc2169eac4c8536afb29c07c23540"}, + {file = "pandas-2.1.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9f17f2b6fc076b2a0078862547595d66244db0f41bf79fc5f64a5c4d635bead"}, + {file = "pandas-2.1.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:45d63d2a9b1b37fa6c84a68ba2422dc9ed018bdaa668c7f47566a01188ceeec1"}, + {file = "pandas-2.1.4-cp312-cp312-win_amd64.whl", hash = "sha256:f69b0c9bb174a2342818d3e2778584e18c740d56857fc5cdb944ec8bbe4082cf"}, + {file = "pandas-2.1.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3f06bda01a143020bad20f7a85dd5f4a1600112145f126bc9e3e42077c24ef34"}, + {file = "pandas-2.1.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ab5796839eb1fd62a39eec2916d3e979ec3130509930fea17fe6f81e18108f6a"}, + {file = "pandas-2.1.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edbaf9e8d3a63a9276d707b4d25930a262341bca9874fcb22eff5e3da5394732"}, + {file = "pandas-2.1.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ebfd771110b50055712b3b711b51bee5d50135429364d0498e1213a7adc2be8"}, + {file = "pandas-2.1.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8ea107e0be2aba1da619cc6ba3f999b2bfc9669a83554b1904ce3dd9507f0860"}, + {file = "pandas-2.1.4-cp39-cp39-win_amd64.whl", hash = "sha256:d65148b14788b3758daf57bf42725caa536575da2b64df9964c563b015230984"}, + {file = "pandas-2.1.4.tar.gz", hash = "sha256:fcb68203c833cc735321512e13861358079a96c174a61f5116a1de89c58c0ef7"}, +] + +[package.dependencies] +numpy = [ + {version = ">=1.22.4,<2", markers = "python_version < \"3.11\""}, + {version = ">=1.23.2,<2", markers = "python_version == \"3.11\""}, + {version = ">=1.26.0,<2", markers = "python_version >= \"3.12\""}, +] +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.1" + +[package.extras] +all = ["PyQt5 (>=5.15.6)", "SQLAlchemy (>=1.4.36)", "beautifulsoup4 (>=4.11.1)", "bottleneck (>=1.3.4)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=0.8.1)", "fsspec (>=2022.05.0)", "gcsfs (>=2022.05.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.8.0)", "matplotlib (>=3.6.1)", "numba (>=0.55.2)", "numexpr (>=2.8.0)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pandas-gbq (>=0.17.5)", "psycopg2 (>=2.9.3)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.5)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "pyxlsb (>=1.0.9)", "qtpy (>=2.2.0)", "s3fs (>=2022.05.0)", "scipy (>=1.8.1)", "tables (>=3.7.0)", "tabulate (>=0.8.10)", "xarray (>=2022.03.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)", "zstandard (>=0.17.0)"] +aws = ["s3fs (>=2022.05.0)"] +clipboard = ["PyQt5 (>=5.15.6)", "qtpy (>=2.2.0)"] +compression = ["zstandard (>=0.17.0)"] +computation = ["scipy (>=1.8.1)", "xarray (>=2022.03.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pyxlsb (>=1.0.9)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)"] +feather = ["pyarrow (>=7.0.0)"] +fss = ["fsspec (>=2022.05.0)"] +gcp = ["gcsfs (>=2022.05.0)", "pandas-gbq (>=0.17.5)"] +hdf5 = ["tables (>=3.7.0)"] +html = ["beautifulsoup4 (>=4.11.1)", "html5lib (>=1.1)", "lxml (>=4.8.0)"] +mysql = ["SQLAlchemy (>=1.4.36)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.8.10)"] +parquet = ["pyarrow (>=7.0.0)"] +performance = ["bottleneck (>=1.3.4)", "numba (>=0.55.2)", "numexpr (>=2.8.0)"] +plot = ["matplotlib (>=3.6.1)"] +postgresql = ["SQLAlchemy (>=1.4.36)", "psycopg2 (>=2.9.3)"] +spss = ["pyreadstat (>=1.1.5)"] +sql-other = ["SQLAlchemy (>=1.4.36)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.8.0)"] + +[[package]] +name = "platformdirs" +version = "4.1.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.1.0-py3-none-any.whl", hash = "sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380"}, + {file = "platformdirs-4.1.0.tar.gz", hash = "sha256:906d548203468492d432bcb294d4bc2fff751bf84971fbb2c10918cc206ee420"}, +] + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] + +[[package]] +name = "pluggy" +version = "1.3.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, + {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "protobuf" +version = "4.25.2" +description = "" +optional = false +python-versions = ">=3.8" +files = [ + {file = "protobuf-4.25.2-cp310-abi3-win32.whl", hash = "sha256:b50c949608682b12efb0b2717f53256f03636af5f60ac0c1d900df6213910fd6"}, + {file = "protobuf-4.25.2-cp310-abi3-win_amd64.whl", hash = "sha256:8f62574857ee1de9f770baf04dde4165e30b15ad97ba03ceac65f760ff018ac9"}, + {file = "protobuf-4.25.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:2db9f8fa64fbdcdc93767d3cf81e0f2aef176284071507e3ede160811502fd3d"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:10894a2885b7175d3984f2be8d9850712c57d5e7587a2410720af8be56cdaf62"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fc381d1dd0516343f1440019cedf08a7405f791cd49eef4ae1ea06520bc1c020"}, + {file = "protobuf-4.25.2-cp38-cp38-win32.whl", hash = "sha256:33a1aeef4b1927431d1be780e87b641e322b88d654203a9e9d93f218ee359e61"}, + {file = "protobuf-4.25.2-cp38-cp38-win_amd64.whl", hash = "sha256:47f3de503fe7c1245f6f03bea7e8d3ec11c6c4a2ea9ef910e3221c8a15516d62"}, + {file = "protobuf-4.25.2-cp39-cp39-win32.whl", hash = "sha256:5e5c933b4c30a988b52e0b7c02641760a5ba046edc5e43d3b94a74c9fc57c1b3"}, + {file = "protobuf-4.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:d66a769b8d687df9024f2985d5137a337f957a0916cf5464d1513eee96a63ff0"}, + {file = "protobuf-4.25.2-py3-none-any.whl", hash = "sha256:a8b7a98d4ce823303145bf3c1a8bdb0f2f4642a414b196f04ad9853ed0c8f830"}, + {file = "protobuf-4.25.2.tar.gz", hash = "sha256:fe599e175cb347efc8ee524bcd4b902d11f7262c0e569ececcb89995c15f0a5e"}, +] + +[[package]] +name = "py" +version = "1.11.0" +description = "library with cross-python path, ini-parsing, io, code, log facilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"}, + {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, +] + +[[package]] +name = "pyasn1" +version = "0.5.1" +description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1-0.5.1-py2.py3-none-any.whl", hash = "sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58"}, + {file = "pyasn1-0.5.1.tar.gz", hash = "sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c"}, +] + +[[package]] +name = "pyasn1-modules" +version = "0.3.0" +description = "A collection of ASN.1-based protocols modules" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1_modules-0.3.0-py2.py3-none-any.whl", hash = "sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d"}, + {file = "pyasn1_modules-0.3.0.tar.gz", hash = "sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c"}, +] + +[package.dependencies] +pyasn1 = ">=0.4.6,<0.6.0" + +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] + +[[package]] +name = "pydantic" +version = "1.10.13" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.13-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:efff03cc7a4f29d9009d1c96ceb1e7a70a65cfe86e89d34e4a5f2ab1e5693737"}, + {file = "pydantic-1.10.13-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3ecea2b9d80e5333303eeb77e180b90e95eea8f765d08c3d278cd56b00345d01"}, + {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1740068fd8e2ef6eb27a20e5651df000978edce6da6803c2bef0bc74540f9548"}, + {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84bafe2e60b5e78bc64a2941b4c071a4b7404c5c907f5f5a99b0139781e69ed8"}, + {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bc0898c12f8e9c97f6cd44c0ed70d55749eaf783716896960b4ecce2edfd2d69"}, + {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:654db58ae399fe6434e55325a2c3e959836bd17a6f6a0b6ca8107ea0571d2e17"}, + {file = "pydantic-1.10.13-cp310-cp310-win_amd64.whl", hash = "sha256:75ac15385a3534d887a99c713aa3da88a30fbd6204a5cd0dc4dab3d770b9bd2f"}, + {file = "pydantic-1.10.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c553f6a156deb868ba38a23cf0df886c63492e9257f60a79c0fd8e7173537653"}, + {file = "pydantic-1.10.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5e08865bc6464df8c7d61439ef4439829e3ab62ab1669cddea8dd00cd74b9ffe"}, + {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e31647d85a2013d926ce60b84f9dd5300d44535a9941fe825dc349ae1f760df9"}, + {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:210ce042e8f6f7c01168b2d84d4c9eb2b009fe7bf572c2266e235edf14bacd80"}, + {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8ae5dd6b721459bfa30805f4c25880e0dd78fc5b5879f9f7a692196ddcb5a580"}, + {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f8e81fc5fb17dae698f52bdd1c4f18b6ca674d7068242b2aff075f588301bbb0"}, + {file = "pydantic-1.10.13-cp311-cp311-win_amd64.whl", hash = "sha256:61d9dce220447fb74f45e73d7ff3b530e25db30192ad8d425166d43c5deb6df0"}, + {file = "pydantic-1.10.13-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4b03e42ec20286f052490423682016fd80fda830d8e4119f8ab13ec7464c0132"}, + {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59ef915cac80275245824e9d771ee939133be38215555e9dc90c6cb148aaeb5"}, + {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a1f9f747851338933942db7af7b6ee8268568ef2ed86c4185c6ef4402e80ba8"}, + {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:97cce3ae7341f7620a0ba5ef6cf043975cd9d2b81f3aa5f4ea37928269bc1b87"}, + {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854223752ba81e3abf663d685f105c64150873cc6f5d0c01d3e3220bcff7d36f"}, + {file = "pydantic-1.10.13-cp37-cp37m-win_amd64.whl", hash = "sha256:b97c1fac8c49be29486df85968682b0afa77e1b809aff74b83081cc115e52f33"}, + {file = "pydantic-1.10.13-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c958d053453a1c4b1c2062b05cd42d9d5c8eb67537b8d5a7e3c3032943ecd261"}, + {file = "pydantic-1.10.13-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c5370a7edaac06daee3af1c8b1192e305bc102abcbf2a92374b5bc793818599"}, + {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d6f6e7305244bddb4414ba7094ce910560c907bdfa3501e9db1a7fd7eaea127"}, + {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3a3c792a58e1622667a2837512099eac62490cdfd63bd407993aaf200a4cf1f"}, + {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c636925f38b8db208e09d344c7aa4f29a86bb9947495dd6b6d376ad10334fb78"}, + {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:678bcf5591b63cc917100dc50ab6caebe597ac67e8c9ccb75e698f66038ea953"}, + {file = "pydantic-1.10.13-cp38-cp38-win_amd64.whl", hash = "sha256:6cf25c1a65c27923a17b3da28a0bdb99f62ee04230c931d83e888012851f4e7f"}, + {file = "pydantic-1.10.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8ef467901d7a41fa0ca6db9ae3ec0021e3f657ce2c208e98cd511f3161c762c6"}, + {file = "pydantic-1.10.13-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:968ac42970f57b8344ee08837b62f6ee6f53c33f603547a55571c954a4225691"}, + {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9849f031cf8a2f0a928fe885e5a04b08006d6d41876b8bbd2fc68a18f9f2e3fd"}, + {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:56e3ff861c3b9c6857579de282ce8baabf443f42ffba355bf070770ed63e11e1"}, + {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f00790179497767aae6bcdc36355792c79e7bbb20b145ff449700eb076c5f96"}, + {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:75b297827b59bc229cac1a23a2f7a4ac0031068e5be0ce385be1462e7e17a35d"}, + {file = "pydantic-1.10.13-cp39-cp39-win_amd64.whl", hash = "sha256:e70ca129d2053fb8b728ee7d1af8e553a928d7e301a311094b8a0501adc8763d"}, + {file = "pydantic-1.10.13-py3-none-any.whl", hash = "sha256:b87326822e71bd5f313e7d3bfdc77ac3247035ac10b0c0618bd99dcf95b1e687"}, + {file = "pydantic-1.10.13.tar.gz", hash = "sha256:32c8b48dcd3b2ac4e78b0ba4af3a2c2eb6048cb75202f0ea7b34feb740efc340"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pydash" +version = "7.0.6" +description = "The kitchen sink of Python utility libraries for doing \"stuff\" in a functional way. Based on the Lo-Dash Javascript library." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydash-7.0.6-py3-none-any.whl", hash = "sha256:10e506935953fde4b0d6fe21a88e17783cd1479256ae96f285b5f89063b4efd6"}, + {file = "pydash-7.0.6.tar.gz", hash = "sha256:7d9df7e9f36f2bbb08316b609480e7c6468185473a21bdd8e65dda7915565a26"}, +] + +[package.dependencies] +typing-extensions = ">=3.10,<4.6.0 || >4.6.0" + +[package.extras] +dev = ["Sphinx", "black", "build", "coverage", "docformatter", "flake8", "flake8-black", "flake8-bugbear", "flake8-isort", "furo", "importlib-metadata (<5)", "invoke", "isort", "mypy", "pylint", "pytest", "pytest-cov", "pytest-mypy-testing", "sphinx-autodoc-typehints", "tox", "twine", "wheel"] + +[[package]] +name = "pygithub" +version = "1.59.1" +description = "Use the full Github API v3" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyGithub-1.59.1-py3-none-any.whl", hash = "sha256:3d87a822e6c868142f0c2c4bf16cce4696b5a7a4d142a7bd160e1bdf75bc54a9"}, + {file = "PyGithub-1.59.1.tar.gz", hash = "sha256:c44e3a121c15bf9d3a5cc98d94c9a047a5132a9b01d22264627f58ade9ddc217"}, +] + +[package.dependencies] +deprecated = "*" +pyjwt = {version = ">=2.4.0", extras = ["crypto"]} +pynacl = ">=1.4.0" +requests = ">=2.14.0" + +[[package]] +name = "pygments" +version = "2.17.2" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pygments-2.17.2-py3-none-any.whl", hash = "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c"}, + {file = "pygments-2.17.2.tar.gz", hash = "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"}, +] + +[package.extras] +plugins = ["importlib-metadata"] +windows-terminal = ["colorama (>=0.4.6)"] + +[[package]] +name = "pyjwt" +version = "2.8.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, + {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, +] + +[package.dependencies] +cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + +[[package]] +name = "pynacl" +version = "1.5.0" +description = "Python binding to the Networking and Cryptography (NaCl) library" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyNaCl-1.5.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d"}, + {file = "PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858"}, + {file = "PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b"}, + {file = "PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff"}, + {file = "PyNaCl-1.5.0-cp36-abi3-win32.whl", hash = "sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543"}, + {file = "PyNaCl-1.5.0-cp36-abi3-win_amd64.whl", hash = "sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93"}, + {file = "PyNaCl-1.5.0.tar.gz", hash = "sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba"}, +] + +[package.dependencies] +cffi = ">=1.4.1" + +[package.extras] +docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] +tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] + +[[package]] +name = "pytest" +version = "6.2.5" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"}, + {file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"}, +] + +[package.dependencies] +atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""} +attrs = ">=19.2.0" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +py = ">=1.8.2" +toml = "*" + +[package.extras] +testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"] + +[[package]] +name = "pytest-cov" +version = "4.1.0" +description = "Pytest plugin for measuring coverage." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-cov-4.1.0.tar.gz", hash = "sha256:3904b13dfbfec47f003b8e77fd5b589cd11904a21ddf1ab38a64f204d6a10ef6"}, + {file = "pytest_cov-4.1.0-py3-none-any.whl", hash = "sha256:6ba70b9e97e69fcc3fb45bfeab2d0a138fb65c4d0d6a41ef33983ad114be8c3a"}, +] + +[package.dependencies] +coverage = {version = ">=5.2.1", extras = ["toml"]} +pytest = ">=4.6" + +[package.extras] +testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] + +[[package]] +name = "pytest-mock" +version = "3.12.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, +] + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2023.3.post1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2023.3.post1-py2.py3-none-any.whl", hash = "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7"}, + {file = "pytz-2023.3.post1.tar.gz", hash = "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "readchar" +version = "4.0.5" +description = "Library to easily read single chars and key strokes" +optional = false +python-versions = ">=3.7" +files = [ + {file = "readchar-4.0.5-py3-none-any.whl", hash = "sha256:76ec784a5dd2afac3b7da8003329834cdd9824294c260027f8c8d2e4d0a78f43"}, + {file = "readchar-4.0.5.tar.gz", hash = "sha256:08a456c2d7c1888cde3f4688b542621b676eb38cd6cfed7eb6cb2e2905ddc826"}, +] + +[package.dependencies] +setuptools = ">=41.0" + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "rich" +version = "13.7.0" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "rich-13.7.0-py3-none-any.whl", hash = "sha256:6da14c108c4866ee9520bbffa71f6fe3962e193b7da68720583850cd4548e235"}, + {file = "rich-13.7.0.tar.gz", hash = "sha256:5cb5123b5cf9ee70584244246816e9114227e0b98ad9176eede6ad54bf5403fa"}, +] + +[package.dependencies] +markdown-it-py = ">=2.2.0" +pygments = ">=2.13.0,<3.0.0" + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<9)"] + +[[package]] +name = "rsa" +version = "4.9" +description = "Pure-Python RSA implementation" +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"}, + {file = "rsa-4.9.tar.gz", hash = "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"}, +] + +[package.dependencies] +pyasn1 = ">=0.1.3" + +[[package]] +name = "runs" +version = "1.2.0" +description = "🏃 Run a block of text as a subprocess 🏃" +optional = false +python-versions = ">=3.8" +files = [ + {file = "runs-1.2.0-py3-none-any.whl", hash = "sha256:ec6fe3b24dfa20c5c4e5c4806d3b35bb880aad0e787a8610913c665c5a7cc07c"}, + {file = "runs-1.2.0.tar.gz", hash = "sha256:8804271011b7a2eeb0d77c3e3f556e5ce5f602fa0dd2a31ed0c1222893be69b7"}, +] + +[package.dependencies] +xmod = "*" + +[[package]] +name = "semver" +version = "3.0.2" +description = "Python helper for Semantic Versioning (https://semver.org)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "semver-3.0.2-py3-none-any.whl", hash = "sha256:b1ea4686fe70b981f85359eda33199d60c53964284e0cfb4977d243e37cf4bf4"}, + {file = "semver-3.0.2.tar.gz", hash = "sha256:6253adb39c70f6e51afed2fa7152bcd414c411286088fb4b9effb133885ab4cc"}, +] + +[[package]] +name = "setuptools" +version = "69.0.3" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.0.3-py3-none-any.whl", hash = "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05"}, + {file = "setuptools-69.0.3.tar.gz", hash = "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "simpleeval" +version = "0.9.13" +description = "A simple, safe single expression evaluator library." +optional = false +python-versions = "*" +files = [ + {file = "simpleeval-0.9.13-py2.py3-none-any.whl", hash = "sha256:22a2701a5006e4188d125d34accf2405c2c37c93f6b346f2484b6422415ae54a"}, + {file = "simpleeval-0.9.13.tar.gz", hash = "sha256:4a30f9cc01825fe4c719c785e3762623e350c4840d5e6855c2a8496baaa65fac"}, +] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "smmap" +version = "5.0.1" +description = "A pure Python implementation of a sliding window memory map manager" +optional = false +python-versions = ">=3.7" +files = [ + {file = "smmap-5.0.1-py3-none-any.whl", hash = "sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da"}, + {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, +] + +[[package]] +name = "sniffio" +version = "1.3.0" +description = "Sniff out which async library your code is running under" +optional = false +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, + {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, +] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "tzdata" +version = "2023.4" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2023.4-py2.py3-none-any.whl", hash = "sha256:aa3ace4329eeacda5b7beb7ea08ece826c28d761cda36e747cfbf97996d39bf3"}, + {file = "tzdata-2023.4.tar.gz", hash = "sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9"}, +] + +[[package]] +name = "urllib3" +version = "2.1.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.1.0-py3-none-any.whl", hash = "sha256:55901e917a5896a349ff771be919f8bd99aff50b79fe58fec595eb37bbc56bb3"}, + {file = "urllib3-2.1.0.tar.gz", hash = "sha256:df7aa8afb0148fa78488e7899b2c59b5f4ffcfa82e6c54ccb9dd37c1d7b52d54"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "vulture" +version = "2.10" +description = "Find dead code" +optional = false +python-versions = ">=3.8" +files = [ + {file = "vulture-2.10-py2.py3-none-any.whl", hash = "sha256:568a4176db7468d0157817ae3bb1847a19f1ddc629849af487f9d3b279bff77d"}, + {file = "vulture-2.10.tar.gz", hash = "sha256:2a5c3160bffba77595b6e6dfcc412016bd2a09cd4b66cdf7fbba913684899f6f"}, +] + +[package.dependencies] +toml = "*" + +[[package]] +name = "wcwidth" +version = "0.2.13" +description = "Measures the displayed width of unicode strings in a terminal" +optional = false +python-versions = "*" +files = [ + {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, + {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, +] + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[[package]] +name = "xmod" +version = "1.8.1" +description = "🌱 Turn any object into a module 🌱" +optional = false +python-versions = ">=3.8" +files = [ + {file = "xmod-1.8.1-py3-none-any.whl", hash = "sha256:a24e9458a4853489042522bdca9e50ee2eac5ab75c809a91150a8a7f40670d48"}, + {file = "xmod-1.8.1.tar.gz", hash = "sha256:38c76486b9d672c546d57d8035df0beb7f4a9b088bc3fb2de5431ae821444377"}, +] + +[[package]] +name = "yarl" +version = "1.9.4" +description = "Yet another URL library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541"}, + {file = "yarl-1.9.4-cp310-cp310-win32.whl", hash = "sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d"}, + {file = "yarl-1.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98"}, + {file = "yarl-1.9.4-cp311-cp311-win32.whl", hash = "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31"}, + {file = "yarl-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10"}, + {file = "yarl-1.9.4-cp312-cp312-win32.whl", hash = "sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7"}, + {file = "yarl-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984"}, + {file = "yarl-1.9.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434"}, + {file = "yarl-1.9.4-cp37-cp37m-win32.whl", hash = "sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749"}, + {file = "yarl-1.9.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3"}, + {file = "yarl-1.9.4-cp38-cp38-win32.whl", hash = "sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece"}, + {file = "yarl-1.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0"}, + {file = "yarl-1.9.4-cp39-cp39-win32.whl", hash = "sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575"}, + {file = "yarl-1.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15"}, + {file = "yarl-1.9.4-py3-none-any.whl", hash = "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad"}, + {file = "yarl-1.9.4.tar.gz", hash = "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf"}, +] + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + +[metadata] +lock-version = "2.0" +python-versions = "^3.10" +content-hash = "e6f67b753371bdbe515e2326b68d32e46a492722b13a8b32a2636fe1e0c39028" diff --git a/airbyte-ci/connectors/base_images/pyproject.toml b/airbyte-ci/connectors/base_images/pyproject.toml new file mode 100644 index 000000000000..206408e58c2d --- /dev/null +++ b/airbyte-ci/connectors/base_images/pyproject.toml @@ -0,0 +1,34 @@ +[tool.poetry] +name = "airbyte-connectors-base-images" +version = "1.0.1" +description = "This package is used to generate and publish the base images for Airbyte Connectors." +authors = ["Augustin Lafanechere "] +readme = "README.md" +packages = [{include = "base_images"}] +include = ["generated"] +[tool.poetry.dependencies] +python = "^3.10" +dagger-io = "==0.9.6" +gitpython = "^3.1.35" +rich = "^13.5.2" +semver = "^3.0.1" +connector-ops = {path = "../connector_ops", develop = true} +inquirer = "^3.1.3" +jinja2 = "^3.1.2" + +[tool.poetry.group.dev.dependencies] +pytest = "^6.2.5" +pytest-mock = "^3.10.0" +pytest-cov = "^4.1.0" +mypy = "^1.5.1" +vulture = "^2.9.1" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry.scripts] +generate-docs = "base_images.commands:generate_docs" +generate-release = "base_images.commands:generate_release" +publish = "base_images.commands:publish_existing_version" + diff --git a/airbyte-ci/connectors/base_images/pytest.ini b/airbyte-ci/connectors/base_images/pytest.ini new file mode 100644 index 000000000000..f14609688a09 --- /dev/null +++ b/airbyte-ci/connectors/base_images/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --cov=base_images --cov-report=term-missing diff --git a/airbyte-integrations/connectors/source-insightly/unit_tests/__init__.py b/airbyte-ci/connectors/base_images/tests/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-insightly/unit_tests/__init__.py rename to airbyte-ci/connectors/base_images/tests/__init__.py diff --git a/airbyte-ci/connectors/base_images/tests/conftest.py b/airbyte-ci/connectors/base_images/tests/conftest.py new file mode 100644 index 000000000000..3f08d090096e --- /dev/null +++ b/airbyte-ci/connectors/base_images/tests/conftest.py @@ -0,0 +1,25 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import platform +import sys + +import dagger +import pytest + + +@pytest.fixture(scope="module") +def anyio_backend(): + return "asyncio" + + +@pytest.fixture(scope="module") +async def dagger_client(): + async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client: + yield client + + +@pytest.fixture(scope="session") +def current_platform(): + return dagger.Platform(f"linux/{platform.machine()}") diff --git a/airbyte-integrations/connectors/source-klarna/unit_tests/__init__.py b/airbyte-ci/connectors/base_images/tests/test_python/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-klarna/unit_tests/__init__.py rename to airbyte-ci/connectors/base_images/tests/test_python/__init__.py diff --git a/airbyte-ci/connectors/base_images/tests/test_python/test_bases.py b/airbyte-ci/connectors/base_images/tests/test_python/test_bases.py new file mode 100644 index 000000000000..d535991bc86b --- /dev/null +++ b/airbyte-ci/connectors/base_images/tests/test_python/test_bases.py @@ -0,0 +1,39 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import pytest +import semver +from base_images import root_images +from base_images.python import bases + +pytestmark = [ + pytest.mark.anyio, +] + + +class TestAirbytePythonConnectorBaseImage: + @pytest.fixture + def dummy_version(self): + return semver.VersionInfo.parse("0.0.0-rc.1") + + def test_class_attributes(self): + """Spot any regression in the class attributes.""" + assert bases.AirbytePythonConnectorBaseImage.root_image == root_images.PYTHON_3_9_18 + assert bases.AirbytePythonConnectorBaseImage.repository == "airbyte/python-connector-base" + assert bases.AirbytePythonConnectorBaseImage.pip_cache_name == "pip_cache" + + async def test_run_sanity_checks(self, dagger_client, current_platform, dummy_version): + base_image_version = bases.AirbytePythonConnectorBaseImage(dagger_client, dummy_version) + await base_image_version.run_sanity_checks(current_platform) + + async def test_pip_cache_volume(self, dagger_client, current_platform, dummy_version): + base_image_version = bases.AirbytePythonConnectorBaseImage(dagger_client, dummy_version) + container = base_image_version.get_container(current_platform) + assert "/root/.cache/pip" in await container.mounts() + + async def test_is_using_bookworm(self, dagger_client, current_platform, dummy_version): + base_image_version = bases.AirbytePythonConnectorBaseImage(dagger_client, dummy_version) + container = base_image_version.get_container(current_platform) + cat_output = await container.with_exec(["cat", "/etc/os-release"], skip_entrypoint=True).stdout() + assert "Debian GNU/Linux 12 (bookworm)" in [kv.split("=")[1].replace('"', "") for kv in cat_output.splitlines()] diff --git a/airbyte-ci/connectors/base_images/tests/test_python/test_sanity_checks.py b/airbyte-ci/connectors/base_images/tests/test_python/test_sanity_checks.py new file mode 100644 index 000000000000..b438d9953939 --- /dev/null +++ b/airbyte-ci/connectors/base_images/tests/test_python/test_sanity_checks.py @@ -0,0 +1,56 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from contextlib import nullcontext as does_not_raise + +import pytest +from base_images import root_images +from base_images.errors import SanityCheckError +from base_images.python import sanity_checks + +pytestmark = [ + pytest.mark.anyio, +] + + +@pytest.mark.parametrize( + "docker_image, python_version, expected_error", + [ + (root_images.PYTHON_3_9_18.address, "3.9.18", does_not_raise()), + (root_images.PYTHON_3_9_18.address, "3.9.19", pytest.raises(SanityCheckError)), + ("hello-world:latest", "3.9.19", pytest.raises(SanityCheckError)), + ], +) +async def test_check_python_version(dagger_client, docker_image, python_version, expected_error): + container_with_python = dagger_client.container().from_(docker_image) + with expected_error: + await sanity_checks.check_python_version(container_with_python, python_version) + + +@pytest.mark.parametrize( + "docker_image, pip_version, expected_error", + [ + (root_images.PYTHON_3_9_18.address, "23.0.1", does_not_raise()), + (root_images.PYTHON_3_9_18.address, "23.0.2", pytest.raises(SanityCheckError)), + ("hello-world:latest", "23.0.1", pytest.raises(SanityCheckError)), + ], +) +async def test_check_pip_version(dagger_client, docker_image, pip_version, expected_error): + container_with_python = dagger_client.container().from_(docker_image) + with expected_error: + await sanity_checks.check_pip_version(container_with_python, pip_version) + + +@pytest.mark.parametrize( + "docker_image, poetry_version, expected_error", + [ + ("pfeiffermax/python-poetry:1.6.0-poetry1.6.1-python3.9.18-slim-bookworm", "1.6.1", does_not_raise()), + ("pfeiffermax/python-poetry:1.6.0-poetry1.4.2-python3.9.18-bookworm", "1.6.1", pytest.raises(SanityCheckError)), + (root_images.PYTHON_3_9_18.address, "23.0.2", pytest.raises(SanityCheckError)), + ], +) +async def test_check_poetry_version(dagger_client, docker_image, poetry_version, expected_error): + container_with_python = dagger_client.container().from_(docker_image) + with expected_error: + await sanity_checks.check_poetry_version(container_with_python, poetry_version) diff --git a/airbyte-ci/connectors/base_images/tests/test_sanity_checks.py b/airbyte-ci/connectors/base_images/tests/test_sanity_checks.py new file mode 100644 index 000000000000..8fd4703b146c --- /dev/null +++ b/airbyte-ci/connectors/base_images/tests/test_sanity_checks.py @@ -0,0 +1,112 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from contextlib import nullcontext as does_not_raise + +import pytest +from base_images import root_images, sanity_checks +from base_images.errors import SanityCheckError + +pytestmark = [ + pytest.mark.anyio, +] + + +@pytest.mark.parametrize( + "docker_image, expected_env_var_name, expected_env_var_value, expected_error", + [ + (root_images.PYTHON_3_9_18.address, "PYTHON_VERSION", "3.9.18", does_not_raise()), + (root_images.PYTHON_3_9_18.address, "PYTHON_VERSION", "3.9.19", pytest.raises(SanityCheckError)), + (root_images.PYTHON_3_9_18.address, "NOT_EXISTING_ENV_VAR", "3.9.19", pytest.raises(SanityCheckError)), + ], +) +async def test_check_env_var_with_printenv(dagger_client, docker_image, expected_env_var_name, expected_env_var_value, expected_error): + container = dagger_client.container().from_(docker_image) + with expected_error: + await sanity_checks.check_env_var_with_printenv(container, expected_env_var_name, expected_env_var_value) + container_without_printenv = container.with_exec(["rm", "/usr/bin/printenv"], skip_entrypoint=True) + with pytest.raises(SanityCheckError): + await sanity_checks.check_env_var_with_printenv(container_without_printenv, expected_env_var_name, expected_env_var_value) + + +async def test_check_timezone_is_utc(dagger_client): + container = dagger_client.container().from_(root_images.PYTHON_3_9_18.address) + # This containers has UTC as timezone by default + await sanity_checks.check_timezone_is_utc(container) + container_not_on_utc = container.with_exec(["ln", "-sf", "/usr/share/zoneinfo/Europe/Paris", "/etc/localtime"], skip_entrypoint=True) + with pytest.raises(SanityCheckError): + await sanity_checks.check_timezone_is_utc(container_not_on_utc) + container_without_date = container.with_exec(["rm", "/usr/bin/date"], skip_entrypoint=True) + with pytest.raises(SanityCheckError): + await sanity_checks.check_timezone_is_utc(container_without_date) + + +async def test_check_a_command_is_available_using_version_option(dagger_client): + container = dagger_client.container().from_(root_images.PYTHON_3_9_18.address) + await sanity_checks.check_a_command_is_available_using_version_option(container, "bash") + container_without_bash = container.with_exec(["rm", "/usr/bin/bash"], skip_entrypoint=True) + with pytest.raises(SanityCheckError): + await sanity_checks.check_a_command_is_available_using_version_option(container_without_bash, "bash") + container_without_ls = container.with_exec(["rm", "/usr/bin/ls"], skip_entrypoint=True) + with pytest.raises(SanityCheckError): + await sanity_checks.check_a_command_is_available_using_version_option(container_without_ls, "ls") + container_without_date = container.with_exec(["rm", "/usr/bin/date"], skip_entrypoint=True) + with pytest.raises(SanityCheckError): + await sanity_checks.check_a_command_is_available_using_version_option(container_without_date, "date") + container_without_printenv = container.with_exec(["rm", "/usr/bin/printenv"], skip_entrypoint=True) + with pytest.raises(SanityCheckError): + await sanity_checks.check_a_command_is_available_using_version_option(container_without_printenv, "printenv") + + +async def test_check_socat_version(mocker): + # Mocking is used in this test because it's hard to install a different socat version in the PYTHON_3_9_18 container + + # Mock the container and its 'with_exec' method + mock_container = mocker.Mock() + # Set the expected version + expected_version = "1.2.3.4" + + # Mock the 'stdout' method and return an output different from the socat -V command + mock_stdout = mocker.AsyncMock(return_value="foobar") + mock_container.with_exec.return_value.stdout = mock_stdout + + # Run the function + with pytest.raises(SanityCheckError) as exc_info: + await sanity_checks.check_socat_version(mock_container, expected_version) + + # Check the error message + assert str(exc_info.value) == "Could not parse the socat version from the output: foobar" + + # Mock the 'stdout' method and return a "socat version" line but with a version structure not matching the pattern from the socat -V command + mock_stdout = mocker.AsyncMock( + return_value="socat by Gerhard Rieger and contributors - see www.dest-unreach.org\nsocat version 1.1 on 06 Nov 2022 08:15:51" + ) + mock_container.with_exec.return_value.stdout = mock_stdout + + # Run the function + with pytest.raises(SanityCheckError) as exc_info: + await sanity_checks.check_socat_version(mock_container, expected_version) + # Check the error message + assert str(exc_info.value) == "Could not find the socat version in the version output: socat version 1.1 on 06 Nov 2022 08:15:51" + + # Mock the 'stdout' method and return a correct "socat version" line but with a version different from the expected one + mock_stdout = mocker.AsyncMock( + return_value="socat by Gerhard Rieger and contributors - see www.dest-unreach.org\nsocat version 1.7.4.4 on 06 Nov 2022 08:15:51" + ) + mock_container.with_exec.return_value.stdout = mock_stdout + + # Run the function + with pytest.raises(SanityCheckError) as exc_info: + await sanity_checks.check_socat_version(mock_container, expected_version) + # Check the error message + assert str(exc_info.value) == "unexpected socat version: 1.7.4.4" + + # Mock the 'stdout' method and return a correct "socat version" matching the expected one + mock_stdout = mocker.AsyncMock( + return_value=f"socat by Gerhard Rieger and contributors - see www.dest-unreach.org\nsocat version {expected_version} on 06 Nov 2022 08:15:51" + ) + mock_container.with_exec.return_value.stdout = mock_stdout + + # No exception should be raised by this function call + await sanity_checks.check_socat_version(mock_container, expected_version) diff --git a/airbyte-ci/connectors/base_images/tests/test_version_registry.py b/airbyte-ci/connectors/base_images/tests/test_version_registry.py new file mode 100644 index 000000000000..a7ce3114ac4c --- /dev/null +++ b/airbyte-ci/connectors/base_images/tests/test_version_registry.py @@ -0,0 +1,152 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +from pathlib import Path + +import pytest +import semver +from base_images import version_registry +from base_images.python.bases import AirbytePythonConnectorBaseImage + + +class TestChangelogEntry: + def test_to_serializable_dict(self): + changelog_entry = version_registry.ChangelogEntry(semver.VersionInfo.parse("1.0.0"), "first version", "Dockerfile example") + assert ( + json.dumps(changelog_entry.to_serializable_dict()) + == '{"version": "1.0.0", "changelog_entry": "first version", "dockerfile_example": "Dockerfile example"}' + ), "The changelog entry should be serializable to JSON" + + +class TestVersionRegistry: + @pytest.fixture + def fake_entries(self, mocker): + # Please keep this list ordered by version + return [ + version_registry.VersionRegistryEntry( + published_docker_image=mocker.Mock(), changelog_entry="first version", version=semver.VersionInfo.parse("1.0.0") + ), + version_registry.VersionRegistryEntry( + published_docker_image=mocker.Mock(), changelog_entry="second version", version=semver.VersionInfo.parse("2.0.0") + ), + version_registry.VersionRegistryEntry( + published_docker_image=mocker.Mock(), changelog_entry="pre-release", version=semver.VersionInfo.parse("3.0.0-rc.1") + ), + version_registry.VersionRegistryEntry( + published_docker_image=None, changelog_entry="third version", version=semver.VersionInfo.parse("3.0.0") + ), + ] + + def test_entries(self, fake_entries): + entries = version_registry.VersionRegistry(AirbytePythonConnectorBaseImage, fake_entries).entries + versions = [entry.version for entry in entries] + assert set(versions) == set( + [entry.version for entry in fake_entries] + ), "The entries should be unique by version and contain all the entries passed as argument" + assert versions == sorted(versions, reverse=True), "The entries should be sorted by version in descending order" + + def test_latest_entry(self, fake_entries): + vr = version_registry.VersionRegistry(AirbytePythonConnectorBaseImage, fake_entries) + assert vr.latest_entry == fake_entries[-1] + + def test_get_entry_for_version(self, fake_entries): + vr = version_registry.VersionRegistry(AirbytePythonConnectorBaseImage, fake_entries) + entry = vr.get_entry_for_version(semver.VersionInfo.parse("1.0.0")) + assert entry.version == semver.VersionInfo.parse("1.0.0") + + def test_latest_published_entry(self, fake_entries): + vr = version_registry.VersionRegistry(AirbytePythonConnectorBaseImage, fake_entries) + assert vr.latest_published_entry == fake_entries[-2] + + def latest_not_pre_released_published_entry(self, fake_entries): + vr = version_registry.VersionRegistry(AirbytePythonConnectorBaseImage, fake_entries) + assert vr.latest_not_pre_released_published_entry == fake_entries[1] + + def test_get_changelog_dump_path(self, mocker): + mock_connector_class = mocker.Mock() + mock_connector_class.repository = "example-repo" + + path = version_registry.VersionRegistry.get_changelog_dump_path(mock_connector_class) + expected_changelog_dump_path = Path("generated/changelogs/example_repo.json") + assert path == expected_changelog_dump_path + assert version_registry.VersionRegistry(mock_connector_class, []).changelog_dump_path == expected_changelog_dump_path + + def test_get_changelog_entries_with_existing_json(self, mocker, tmp_path): + dummy_change_log_path = tmp_path / "changelog.json" + dummy_changelog_entry = version_registry.ChangelogEntry(semver.VersionInfo.parse("1.0.0"), "Initial release", "") + dummy_change_log_path.write_text(json.dumps([dummy_changelog_entry.to_serializable_dict()])) + + mock_connector_class = mocker.Mock() + mocker.patch.object(version_registry.VersionRegistry, "get_changelog_dump_path", return_value=dummy_change_log_path) + + changelog_entries = version_registry.VersionRegistry.get_changelog_entries(mock_connector_class) + + assert len(changelog_entries) == 1 + assert isinstance(changelog_entries[0], version_registry.ChangelogEntry) + assert changelog_entries[0].version == semver.VersionInfo.parse("1.0.0") + assert changelog_entries[0].changelog_entry == "Initial release" + + def test_get_changelog_entries_without_json(self, mocker, tmp_path): + dummy_change_log_path = tmp_path / "changelog.json" + + mock_connector_class = mocker.Mock() + mocker.patch.object(version_registry.VersionRegistry, "get_changelog_dump_path", return_value=dummy_change_log_path) + + changelog_entries = version_registry.VersionRegistry.get_changelog_entries(mock_connector_class) + assert len(changelog_entries) == 0 + + @pytest.fixture + def mock_dagger_client(self, mocker): + return mocker.Mock() + + @pytest.fixture + def fake_docker_credentials(self): + return ("username", "password") + + @pytest.fixture + def fake_changelog_entries(self): + return [ + version_registry.ChangelogEntry(semver.VersionInfo.parse("1.0.0"), "first version", ""), + version_registry.ChangelogEntry(semver.VersionInfo.parse("2.0.0"), "second unpublished version", ""), + ] + + @pytest.fixture + def fake_published_images(self, mocker): + # Mock the published images to include only one version (2.0.0) + return [mocker.Mock(version=semver.VersionInfo.parse("1.0.0"))] + + @pytest.mark.anyio + async def test_get_all_published_base_images(self, mocker, mock_dagger_client, fake_docker_credentials): + mock_crane_client = mocker.Mock() + mocker.patch.object(version_registry.docker, "CraneClient", return_value=mock_crane_client) + + mock_remote_registry = mocker.AsyncMock() + mocker.patch.object(version_registry.docker, "RemoteRepository", return_value=mock_remote_registry) + + sample_published_images = [mocker.Mock(), mocker.Mock()] + mock_remote_registry.get_all_images.return_value = sample_published_images + + published_images = await version_registry.VersionRegistry.get_all_published_base_images( + mock_dagger_client, fake_docker_credentials, mocker.Mock() + ) + + assert published_images == sample_published_images + + @pytest.mark.anyio + async def test_load_with_mocks( + self, mocker, mock_dagger_client, fake_docker_credentials, fake_changelog_entries, fake_published_images + ): + mocker.patch.object(version_registry.VersionRegistry, "get_changelog_entries", return_value=fake_changelog_entries) + mocker.patch.object(version_registry.VersionRegistry, "get_all_published_base_images", return_value=fake_published_images) + + registry = await version_registry.VersionRegistry.load(mocker.Mock(), mock_dagger_client, fake_docker_credentials) + + assert len(registry.entries) == 2, "Two entries should be in the registry" + version_1_entry = registry.get_entry_for_version(semver.VersionInfo.parse("1.0.0")) + assert version_1_entry is not None, "The version 1.0.0 should be in the registry even if not published" + assert version_1_entry.published, "The version 1.0.0 should be published" + version_2_entry = registry.get_entry_for_version(semver.VersionInfo.parse("2.0.0")) + assert version_2_entry is not None, "The version 2.0.0 should be in the registry even if not published" + assert version_2_entry.published is False, "The version 2.0.0 should not be published" diff --git a/airbyte-ci/connectors/ci_credentials/README.md b/airbyte-ci/connectors/ci_credentials/README.md index 238ae35351f6..511d8cb005a6 100644 --- a/airbyte-ci/connectors/ci_credentials/README.md +++ b/airbyte-ci/connectors/ci_credentials/README.md @@ -11,20 +11,36 @@ This project requires Python 3.10 and pipx. The recommended way to install `ci_credentials` is using pipx. This ensures the tool and its dependencies are isolated from your other Python projects. +If you havent installed pyenv, you can do it with brew: + +```bash +brew update +brew install pyenv +``` + If you haven't installed pipx, you can do it with pip: ```bash +cd airbyte-ci/connectors/ci_credentials/ +pyenv install # ensure you have the correct python version python -m pip install --user pipx python -m pipx ensurepath ``` -Once pipx is installed, navigate to the root directory of the project, then run: +Once pyenv and pipx is installed then run the following: ```bash -pipx install airbyte-ci/connectors/ci_credentials/ +pipx install --editable --force --python=python3.10 airbyte-ci/connectors/ci_credentials/ ``` -This command installs ci_credentials and makes it globally available in your terminal. +This command installs `ci_credentials` and makes it globally available in your terminal. + +_Note: `--force` is required to ensure updates are applied on subsequent installs._ +_Note: `--python=python3.10` is required to ensure the correct python version is used._ +_Note: `--editable` is required to ensure the correct python version is used._ + +If you face any installation problem feel free to reach out the Airbyte Connectors Operations team. + ## Get GSM access Download a Service account json key that has access to Google Secrets Manager. @@ -52,6 +68,11 @@ pipx install --editable airbyte-ci/connectors/ci_credentials/ This is useful when you are making changes to the package and want to test them in real-time. +Note: + +- The package name is `pipelines`, not `airbyte-ci`. You will need this when uninstalling or reinstalling. +- Even with the above `--editable` method, live changes to the code in the sibling project `/airbyte-ci/connectors/connector_ops/` are not automatically captured. To ensure you are using the latest code, use the command `pipx reinstall pipelines`. + ## Usage After installation, you can use the ci_credentials command in your terminal. diff --git a/airbyte-ci/connectors/ci_credentials/ci_credentials/secrets_manager.py b/airbyte-ci/connectors/ci_credentials/ci_credentials/secrets_manager.py index 2c02785957b4..c024caf01587 100644 --- a/airbyte-ci/connectors/ci_credentials/ci_credentials/secrets_manager.py +++ b/airbyte-ci/connectors/ci_credentials/ci_credentials/secrets_manager.py @@ -117,6 +117,9 @@ def __load_gsm_secrets(self) -> List[RemoteSecret]: enabled_versions = [version["name"] for version in versions_data["versions"] if version["state"] == "ENABLED"] if len(enabled_versions) > 1: self.logger.critical(f"{log_name} should have one enabled version at the same time!!!") + if not enabled_versions: + self.logger.warning(f"{log_name} doesn't have enabled versions for {secret_name}") + continue enabled_version = enabled_versions[0] secret_url = f"https://secretmanager.googleapis.com/v1/{enabled_version}:access" secret_data = self.api.get(secret_url) diff --git a/airbyte-ci/connectors/ci_credentials/tests/__init__.py b/airbyte-ci/connectors/ci_credentials/tests/__init__.py index e69de29bb2d1..f70ecfc3a89e 100644 --- a/airbyte-ci/connectors/ci_credentials/tests/__init__.py +++ b/airbyte-ci/connectors/ci_credentials/tests/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. diff --git a/airbyte-ci/connectors/common_utils/common_utils/__init__.py b/airbyte-ci/connectors/common_utils/common_utils/__init__.py index 0078efdb784a..dc90ae912c83 100644 --- a/airbyte-ci/connectors/common_utils/common_utils/__init__.py +++ b/airbyte-ci/connectors/common_utils/common_utils/__init__.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + from .google_api import GoogleApi from .logger import Logger diff --git a/airbyte-ci/connectors/common_utils/tests/__init__.py b/airbyte-ci/connectors/common_utils/tests/__init__.py index e69de29bb2d1..f70ecfc3a89e 100644 --- a/airbyte-ci/connectors/common_utils/tests/__init__.py +++ b/airbyte-ci/connectors/common_utils/tests/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. diff --git a/airbyte-ci/connectors/connector_ops/.python-version b/airbyte-ci/connectors/connector_ops/.python-version deleted file mode 100644 index 36435ac696df..000000000000 --- a/airbyte-ci/connectors/connector_ops/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.10.8 diff --git a/airbyte-ci/connectors/connector_ops/README.md b/airbyte-ci/connectors/connector_ops/README.md index ef306478b7cb..344d985bc717 100644 --- a/airbyte-ci/connectors/connector_ops/README.md +++ b/airbyte-ci/connectors/connector_ops/README.md @@ -1,35 +1,46 @@ # connector_ops -A collection of tools and checks run by Github Actions +A collection of utilities for working with Airbyte connectors. -## Running Locally +# Setup -From this directory, create a virtual environment: +## Prerequisites -``` -python3 -m venv .venv +#### Poetry + +Before you can start working on this project, you will need to have Poetry installed on your system. Please follow the instructions below to install Poetry: + +1. Open your terminal or command prompt. +2. Install Poetry using the recommended installation method: + +```bash +curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.5.1 python3 - ``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: +Alternatively, you can use `pip` to install Poetry: ```bash -source .venv/bin/activate -pip install -e . # assuming you are in the ./airbyte-ci/connectors/connector_ops directory +pip install --user poetry ``` -pip will make binaries for all the commands in setup.py, so you can run `allowed-hosts-checks` directly from the virtual-env +3. After the installation is complete, close and reopen your terminal to ensure the newly installed `poetry` command is available in your system's PATH. -## Testing Locally +For more detailed instructions and alternative installation methods, please refer to the official Poetry documentation: https://python-poetry.org/docs/#installation -To install requirements to run unit tests, use: +### Using Poetry in the Project -``` -pip install -e ".[tests]" -``` +Once Poetry is installed, you can use it to manage the project's dependencies and virtual environment. To get started, navigate to the project's root directory in your terminal and follow these steps: -Unit tests are currently configured to be run from the base `airbyte` directory. You can run the tests from that directory with the following command: +## Installation +```bash +poetry install ``` -pytest -s airbyte-ci/connector_ops/connectors/tests + + +## Testing Locally + +Simply run +```bash +poetry run pytest ``` \ No newline at end of file diff --git a/airbyte-ci/connectors/connector_ops/connector_ops/acceptance_test_config_checks.py b/airbyte-ci/connectors/connector_ops/connector_ops/acceptance_test_config_checks.py index 7d2b064468df..9bc70e3b17a6 100644 --- a/airbyte-ci/connectors/connector_ops/connector_ops/acceptance_test_config_checks.py +++ b/airbyte-ci/connectors/connector_ops/connector_ops/acceptance_test_config_checks.py @@ -4,17 +4,10 @@ import logging import sys -from typing import Dict, List, Set, Union +from typing import List -import yaml from connector_ops import utils -BACKWARD_COMPATIBILITY_REVIEWERS = {"connector-operations", "connector-extensibility"} -TEST_STRICTNESS_LEVEL_REVIEWERS = {"connector-operations"} -GA_BYPASS_REASON_REVIEWERS = {"connector-operations"} -GA_CONNECTOR_REVIEWERS = {"gl-python"} -REVIEW_REQUIREMENTS_FILE_PATH = ".github/connector_org_review_requirements.yaml" - def find_connectors_with_bad_strictness_level() -> List[utils.Connector]: """Check if changed connectors have the expected connector acceptance test strictness level according to their release stage. @@ -38,43 +31,6 @@ def find_connectors_with_bad_strictness_level() -> List[utils.Connector]: return connectors_with_bad_strictness_level -def find_changed_important_connectors() -> Set[utils.Connector]: - """Find important connectors modified on the current branch. - - Returns: - Set[utils.Connector]: The set of GA connectors that were modified on the current branch. - """ - changed_connectors = utils.get_changed_connectors(destination=False, third_party=False) - return {connector for connector in changed_connectors if connector.is_important_connector} - - -def get_bypass_reason_changes() -> Set[utils.Connector]: - """Find connectors that have modified bypass_reasons. - - Returns: - Set[str]: Set of connector names e.g {"source-github"}: The set of GA connectors that have changed bypass_reasons. - """ - bypass_reason_changes = utils.get_changed_acceptance_test_config(diff_regex="bypass_reason") - return bypass_reason_changes.intersection(find_changed_important_connectors()) - - -def find_mandatory_reviewers() -> List[Union[str, Dict[str, List]]]: - important_connector_changes = find_changed_important_connectors() - backward_compatibility_changes = utils.get_changed_acceptance_test_config(diff_regex="disable_for_version") - test_strictness_level_changes = utils.get_changed_acceptance_test_config(diff_regex="test_strictness_level") - ga_bypass_reason_changes = get_bypass_reason_changes() - - if backward_compatibility_changes: - return [{"any-of": list(BACKWARD_COMPATIBILITY_REVIEWERS)}] - if test_strictness_level_changes: - return [{"any-of": list(TEST_STRICTNESS_LEVEL_REVIEWERS)}] - if ga_bypass_reason_changes: - return [{"any-of": list(GA_BYPASS_REASON_REVIEWERS)}] - if important_connector_changes: - return list(GA_CONNECTOR_REVIEWERS) - return [] - - def check_test_strictness_level(): connectors_with_bad_strictness_level = find_connectors_with_bad_strictness_level() if connectors_with_bad_strictness_level: @@ -84,28 +40,3 @@ def check_test_strictness_level(): sys.exit(1) else: sys.exit(0) - - -def write_review_requirements_file(): - mandatory_reviewers = find_mandatory_reviewers() - - if mandatory_reviewers: - requirements_file_content = [ - {"name": "Required reviewers from the connector org teams", "paths": "unmatched", "teams": mandatory_reviewers} - ] - with open(REVIEW_REQUIREMENTS_FILE_PATH, "w") as requirements_file: - yaml.safe_dump(requirements_file_content, requirements_file) - print("CREATED_REQUIREMENTS_FILE=true") - else: - print("CREATED_REQUIREMENTS_FILE=false") - - -def print_mandatory_reviewers(): - teams = [] - mandatory_reviewers = find_mandatory_reviewers() - for mandatory_reviewer in mandatory_reviewers: - if isinstance(mandatory_reviewer, dict): - teams += mandatory_reviewer["any-of"] - else: - teams.append(mandatory_reviewer) - print(f"MANDATORY_REVIEWERS=A review is required from these teams: {','.join(teams)}") diff --git a/airbyte-ci/connectors/connector_ops/connector_ops/qa_checks.py b/airbyte-ci/connectors/connector_ops/connector_ops/qa_checks.py index 462a05c1c4d0..99bd8e1786b0 100644 --- a/airbyte-ci/connectors/connector_ops/connector_ops/qa_checks.py +++ b/airbyte-ci/connectors/connector_ops/connector_ops/qa_checks.py @@ -5,29 +5,30 @@ import sys from pathlib import Path -from typing import Iterable, Optional, Set, Tuple +from typing import Callable, Iterable, Optional, Set, Tuple -from connector_ops.utils import Connector +from connector_ops.utils import Connector, ConnectorLanguage from pydash.objects import get def check_migration_guide(connector: Connector) -> bool: """Check if a migration guide is available for the connector if a breaking change was introduced.""" - breaking_changes = get(connector.metadata, f"releases.breakingChanges") + breaking_changes = get(connector.metadata, "releases.breakingChanges") if not breaking_changes: return True migration_guide_file_path = connector.migration_guide_file_path - if not migration_guide_file_path.exists(): + migration_guide_exists = migration_guide_file_path is not None and migration_guide_file_path.exists() + if not migration_guide_exists: print( - f"Migration guide file is missing for {connector.name}. Please create a {connector.migration_guide_file_name} file in the docs folder." + f"Migration guide file is missing for {connector.name}. Please create a migration guide at {connector.migration_guide_file_path}" ) return False # Check that the migration guide begins with # {connector name} Migration Guide expected_title = f"# {connector.name_from_metadata} Migration Guide" - expected_version_header_start = f"## Upgrading to " + expected_version_header_start = "## Upgrading to " with open(migration_guide_file_path) as f: first_line = f.readline().strip() if not first_line == expected_title: @@ -55,7 +56,7 @@ def check_migration_guide(connector: Connector) -> bool: if ordered_breaking_changes != ordered_heading_versions: print(f"Migration guide file for {connector.name} has incorrect version headings.") - print(f"Check for missing, extra, or misordered headings, or headers with typos.") + print("Check for missing, extra, or misordered headings, or headers with typos.") print(f"Expected headings: {ordered_expected_headings}") return False @@ -72,8 +73,9 @@ def check_documentation_file_exists(connector: Connector) -> bool: Returns: bool: Wether a documentation file was found. """ + file_path = connector.documentation_file_path - return connector.documentation_file_path.exists() + return file_path is not None and file_path.exists() def check_documentation_follows_guidelines(connector: Connector) -> bool: @@ -168,7 +170,15 @@ def read_all_files_in_directory( ".hypothesis", } -IGNORED_FILENAME_PATTERN_FOR_HTTPS_CHECKS = {"*Test.java", "*.jar", "*.pyc", "*.gz", "*.svg"} +IGNORED_FILENAME_PATTERN_FOR_HTTPS_CHECKS = { + "*Test.java", + "*.jar", + "*.pyc", + "*.gz", + "*.svg", + "expected_records.jsonl", + "expected_records.json", +} IGNORED_URLS_PREFIX = { "http://json-schema.org", "http://localhost", @@ -202,12 +212,16 @@ def check_connector_https_url_only(connector: Connector) -> bool: bool: Wether the connector code contains only https url. """ files_with_http_url = set() + ignore_comment = "# ignore-https-check" # Define the ignore comment pattern + for filename, line in read_all_files_in_directory( connector.code_directory, IGNORED_DIRECTORIES_FOR_HTTPS_CHECKS, IGNORED_FILENAME_PATTERN_FOR_HTTPS_CHECKS ): line = line.lower() if is_comment(line, filename): continue + if ignore_comment in line: + continue for prefix in IGNORED_URLS_PREFIX: line = line.replace(prefix, "") if "http://" in line: @@ -235,10 +249,14 @@ def check_connector_has_no_critical_vulnerabilities(connector: Connector) -> boo def check_metadata_version_matches_dockerfile_label(connector: Connector) -> bool: - return connector.version_in_dockerfile_label == connector.version + version_in_dockerfile = connector.version_in_dockerfile_label + if version_in_dockerfile is None: + # Java connectors don't have Dockerfiles. + return connector.language == ConnectorLanguage.JAVA + return version_in_dockerfile == connector.version -QA_CHECKS = [ +DEFAULT_QA_CHECKS = ( check_documentation_file_exists, check_migration_guide, # Disabling the following check because it's likely to not pass on a lot of connectors. @@ -250,8 +268,13 @@ def check_metadata_version_matches_dockerfile_label(connector: Connector) -> boo # https://github.com/airbytehq/airbyte/issues/21606 check_connector_https_url_only, check_connector_has_no_critical_vulnerabilities, - check_metadata_version_matches_dockerfile_label, -] +) + + +def get_qa_checks_to_run(connector: Connector) -> Tuple[Callable]: + if connector.has_dockerfile: + return DEFAULT_QA_CHECKS + (check_metadata_version_matches_dockerfile_label,) + return DEFAULT_QA_CHECKS def remove_strict_encrypt_suffix(connector_technical_name: str) -> str: @@ -285,7 +308,7 @@ def run_qa_checks(): connector_technical_name = remove_strict_encrypt_suffix(connector_technical_name) connector = Connector(connector_technical_name) print(f"Running QA checks for {connector_technical_name}:{connector.version}") - qa_check_results = {qa_check.__name__: qa_check(connector) for qa_check in QA_CHECKS} + qa_check_results = {qa_check.__name__: qa_check(connector) for qa_check in get_qa_checks_to_run(connector)} if not all(qa_check_results.values()): print(f"QA checks failed for {connector_technical_name}:{connector.version}:") for check_name, check_result in qa_check_results.items(): diff --git a/airbyte-ci/connectors/connector_ops/connector_ops/required_reviewer_checks.py b/airbyte-ci/connectors/connector_ops/connector_ops/required_reviewer_checks.py new file mode 100644 index 000000000000..3d96341910fd --- /dev/null +++ b/airbyte-ci/connectors/connector_ops/connector_ops/required_reviewer_checks.py @@ -0,0 +1,89 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Dict, List, Set, Tuple, Union + +import yaml +from connector_ops import utils + +BACKWARD_COMPATIBILITY_REVIEWERS = {"connector-operations", "connector-extensibility"} +TEST_STRICTNESS_LEVEL_REVIEWERS = {"connector-operations"} +BYPASS_REASON_REVIEWERS = {"connector-operations"} +STRATEGIC_PYTHON_CONNECTOR_REVIEWERS = {"gl-python"} +BREAKING_CHANGE_REVIEWERS = {"breaking-change-reviewers"} +REVIEW_REQUIREMENTS_FILE_PATH = ".github/connector_org_review_requirements.yaml" + + +def find_changed_strategic_connectors( + languages: Tuple[utils.ConnectorLanguage] = ( + utils.ConnectorLanguage.JAVA, + utils.ConnectorLanguage.LOW_CODE, + utils.ConnectorLanguage.PYTHON, + ) +) -> Set[utils.Connector]: + """Find important connectors modified on the current branch. + + Returns: + Set[utils.Connector]: The set of important connectors that were modified on the current branch. + """ + changed_connectors = utils.get_changed_connectors(destination=False, third_party=False) + return {connector for connector in changed_connectors if connector.is_strategic_connector and connector.language in languages} + + +def get_bypass_reason_changes() -> Set[utils.Connector]: + """Find connectors that have modified bypass_reasons. + + Returns: + Set[str]: Set of connector names e.g {"source-github"}: The set of important connectors that have changed bypass_reasons. + """ + bypass_reason_changes = utils.get_changed_acceptance_test_config(diff_regex="bypass_reason") + return bypass_reason_changes.intersection(find_changed_strategic_connectors()) + + +def find_mandatory_reviewers() -> List[Dict[str, Union[str, Dict[str, List]]]]: + requirements = [ + { + "name": "Backwards compatibility test skip", + "teams": list(BACKWARD_COMPATIBILITY_REVIEWERS), + "is_required": utils.get_changed_acceptance_test_config(diff_regex="disable_for_version"), + }, + { + "name": "Acceptance test strictness level", + "teams": list(TEST_STRICTNESS_LEVEL_REVIEWERS), + "is_required": utils.get_changed_acceptance_test_config(diff_regex="test_strictness_level"), + }, + {"name": "Strategic connector bypass reasons", "teams": list(BYPASS_REASON_REVIEWERS), "is_required": get_bypass_reason_changes()}, + { + "name": "Strategic python connectors", + "teams": list(STRATEGIC_PYTHON_CONNECTOR_REVIEWERS), + "is_required": find_changed_strategic_connectors((utils.ConnectorLanguage.PYTHON, utils.ConnectorLanguage.LOW_CODE)), + }, + { + "name": "Breaking changes", + "teams": list(BREAKING_CHANGE_REVIEWERS), + "is_required": utils.get_changed_metadata(diff_regex="upgradeDeadline"), + }, + ] + + return [{"name": r["name"], "teams": r["teams"]} for r in requirements if r["is_required"]] + + +def write_review_requirements_file(): + mandatory_reviewers = find_mandatory_reviewers() + + if mandatory_reviewers: + requirements_file_content = [dict(r, paths=["**"]) for r in mandatory_reviewers] + with open(REVIEW_REQUIREMENTS_FILE_PATH, "w") as requirements_file: + yaml.safe_dump(requirements_file_content, requirements_file) + print("CREATED_REQUIREMENTS_FILE=true") + else: + print("CREATED_REQUIREMENTS_FILE=false") + + +def print_mandatory_reviewers(): + teams = set() + mandatory_reviewers = find_mandatory_reviewers() + for reviewers in mandatory_reviewers: + teams.update(reviewers["teams"]) + print(f"MANDATORY_REVIEWERS=A review is required from these teams: {', '.join(teams)}") diff --git a/airbyte-ci/connectors/connector_ops/connector_ops/utils.py b/airbyte-ci/connectors/connector_ops/connector_ops/utils.py index ef3b83e9dd80..2993288283ad 100644 --- a/airbyte-ci/connectors/connector_ops/connector_ops/utils.py +++ b/airbyte-ci/connectors/connector_ops/connector_ops/utils.py @@ -18,23 +18,35 @@ from ci_credentials import SecretsManager from pydash.objects import get from rich.console import Console +from simpleeval import simple_eval console = Console() DIFFED_BRANCH = os.environ.get("DIFFED_BRANCH", "origin/master") OSS_CATALOG_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json" +BASE_AIRBYTE_DOCS_URL = "https://docs.airbyte.com" CONNECTOR_PATH_PREFIX = "airbyte-integrations/connectors" SOURCE_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/source-" DESTINATION_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/destination-" -THIRD_PARTY_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + "/third_party/" + +THIRD_PARTY_GLOB = "third-party" +THIRD_PARTY_CONNECTOR_PATH_PREFIX = CONNECTOR_PATH_PREFIX + f"/{THIRD_PARTY_GLOB}/" SCAFFOLD_CONNECTOR_GLOB = "-scaffold-" ACCEPTANCE_TEST_CONFIG_FILE_NAME = "acceptance-test-config.yml" +METADATA_FILE_NAME = "metadata.yaml" AIRBYTE_DOCKER_REPO = "airbyte" AIRBYTE_REPO_DIRECTORY_NAME = "airbyte" GRADLE_PROJECT_RE_PATTERN = r"project\((['\"])(.+?)\1\)" -TEST_GRADLE_DEPENDENCIES = ["testImplementation", "integrationTestJavaImplementation", "performanceTestJavaImplementation"] +TEST_GRADLE_DEPENDENCIES = [ + "testImplementation", + "testCompileOnly", + "integrationTestJavaImplementation", + "performanceTestJavaImplementation", + "testFixturesCompileOnly", + "testFixturesImplementation", +] def download_catalog(catalog_url): @@ -46,8 +58,8 @@ def download_catalog(catalog_url): METADATA_FILE_NAME = "metadata.yaml" ICON_FILE_NAME = "icon.svg" -IMPORTANT_CONNECTOR_THRESHOLDS = { - "sl": 300, +STRATEGIC_CONNECTOR_THRESHOLDS = { + "sl": 200, "ql": 400, } @@ -71,6 +83,30 @@ def get_connector_name_from_path(path): def get_changed_acceptance_test_config(diff_regex: Optional[str] = None) -> Set[str]: """Retrieve the set of connectors for which the acceptance_test_config file was changed in the current branch (compared to master). + Args: + diff_regex (str): Find the edited files that contain the following regex in their change. + + Returns: + Set[Connector]: Set of connectors that were changed + """ + return get_changed_file(ACCEPTANCE_TEST_CONFIG_FILE_NAME, diff_regex) + + +def get_changed_metadata(diff_regex: Optional[str] = None) -> Set[str]: + """Retrieve the set of connectors for which the metadata file was changed in the current branch (compared to master). + + Args: + diff_regex (str): Find the edited files that contain the following regex in their change. + + Returns: + Set[Connector]: Set of connectors that were changed + """ + return get_changed_file(METADATA_FILE_NAME, diff_regex) + + +def get_changed_file(file_name: str, diff_regex: Optional[str] = None) -> Set[str]: + """Retrieve the set of connectors for which the given file was changed in the current branch (compared to master). + Args: diff_regex (str): Find the edited files that contain the following regex in their change. @@ -87,11 +123,31 @@ def get_changed_acceptance_test_config(diff_regex: Optional[str] = None) -> Set[ changed_acceptance_test_config_paths = { file_path for file_path in airbyte_repo.git.diff(*diff_command_args).split("\n") - if file_path.startswith(SOURCE_CONNECTOR_PATH_PREFIX) and file_path.endswith(ACCEPTANCE_TEST_CONFIG_FILE_NAME) + if file_path.startswith(SOURCE_CONNECTOR_PATH_PREFIX) and file_path.endswith(file_name) } return {Connector(get_connector_name_from_path(changed_file)) for changed_file in changed_acceptance_test_config_paths} +def has_local_cdk_ref(build_file: Path) -> bool: + """Return true if the build file uses the local CDK. + + Args: + build_file (Path): Path to the build.gradle file of the project. + + Returns: + bool: True if using local CDK. + """ + contents = "\n".join( + [ + # Return contents without inline code comments + line.split("//")[0] + for line in build_file.read_text().split("\n") + ] + ) + contents = contents.replace(" ", "") + return "useLocalCdk=true" in contents + + def get_gradle_dependencies_block(build_file: Path) -> str: """Get the dependencies block of a Gradle file. @@ -135,7 +191,7 @@ def parse_gradle_dependencies(build_file: Path) -> Tuple[List[Path], List[Path]] # Find all matches for test dependencies and regular dependencies matches = re.findall( - r"(testImplementation|integrationTestJavaImplementation|performanceTestJavaImplementation|implementation|api).*?project\(['\"](.*?)['\"]\)", + r"(compileOnly|testCompileOnly|testFixturesCompileOnly|testFixturesImplementation|testImplementation|integrationTestJavaImplementation|performanceTestJavaImplementation|implementation|api).*?project\(['\"](.*?)['\"]\)", dependencies_block, ) if matches: @@ -150,10 +206,33 @@ def parse_gradle_dependencies(build_file: Path) -> Tuple[List[Path], List[Path]] else: project_dependencies.append(path) - project_dependencies.append(Path("airbyte-cdk", "java", "airbyte-cdk")) + # Dedupe dependencies: + project_dependencies = list(set(project_dependencies)) + test_dependencies = list(set(test_dependencies)) + return project_dependencies, test_dependencies +def get_local_cdk_gradle_dependencies(with_test_dependencies: bool) -> List[Path]: + """Recursively retrieve all transitive dependencies of a Gradle project. + + Args: + with_test_dependencies: True to include test dependencies. + + Returns: + List[Path]: All dependencies of the project. + """ + base_path = Path("airbyte-cdk/java/airbyte-cdk") + found: List[Path] = [base_path] + for submodule in ["core", "db-sources", "db-destinations"]: + found.append(base_path / submodule) + project_dependencies, test_dependencies = parse_gradle_dependencies(base_path / Path(submodule) / Path("build.gradle")) + found += project_dependencies + if with_test_dependencies: + found += test_dependencies + return list(set(found)) + + def get_all_gradle_dependencies( build_file: Path, with_test_dependencies: bool = True, found_dependencies: Optional[List[Path]] = None ) -> List[Path]: @@ -169,6 +248,12 @@ def get_all_gradle_dependencies( if found_dependencies is None: found_dependencies = [] project_dependencies, test_dependencies = parse_gradle_dependencies(build_file) + + # Since first party project folders are transitive (compileOnly) in the + # CDK, we always need to add them as the project dependencies. + project_dependencies += get_local_cdk_gradle_dependencies(False) + test_dependencies += get_local_cdk_gradle_dependencies(with_test_dependencies=True) + all_dependencies = project_dependencies + test_dependencies if with_test_dependencies else project_dependencies for dependency_path in all_dependencies: if dependency_path not in found_dependencies and Path(dependency_path / "build.gradle").exists(): @@ -192,7 +277,9 @@ class ConnectorLanguageError(Exception): class Connector: """Utility class to gather metadata about a connector.""" - technical_name: str + # Path to the connector directory relative to the CONNECTOR_PATH_PREFIX + # e.g source-google-sheets or third-party/farosai/airbyte-pagerduty-source + relative_connector_path: str def _get_type_and_name_from_technical_name(self) -> Tuple[str, str]: if "-" not in self.technical_name: @@ -201,22 +288,58 @@ def _get_type_and_name_from_technical_name(self) -> Tuple[str, str]: name = self.technical_name[len(_type) + 1 :] return _type, name + @property + def technical_name(self) -> str: + """ + Return the technical name of the connector from the given relative_connector_path + e.g. source-google-sheets -> source-google-sheets or third-party/farosai/airbyte-pagerduty-source -> airbyte-pagerduty-source + """ + return self.relative_connector_path.split("/")[-1] + @property def name(self): return self._get_type_and_name_from_technical_name()[1] @property def connector_type(self) -> str: - return self._get_type_and_name_from_technical_name()[0] + return self.metadata["connectorType"] if self.metadata else None @property - def documentation_directory(self) -> Path: + def is_third_party(self) -> bool: + return THIRD_PARTY_GLOB in self.relative_connector_path + + @property + def has_airbyte_docs(self) -> bool: + return ( + self.metadata + and self.metadata.get("documentationUrl") is not None + and BASE_AIRBYTE_DOCS_URL in str(self.metadata.get("documentationUrl")) + ) + + @property + def local_connector_documentation_directory(self) -> Path: return Path(f"./docs/integrations/{self.connector_type}s") @property - def documentation_file_path(self) -> Path: - readme_file_name = f"{self.name}.md" - return self.documentation_directory / readme_file_name + def relative_documentation_path_str(self) -> str: + documentation_url = self.metadata["documentationUrl"] + relative_documentation_path = documentation_url.replace(BASE_AIRBYTE_DOCS_URL, "") + + # strip leading and trailing slashes + relative_documentation_path = relative_documentation_path.strip("/") + + return f"./docs/{relative_documentation_path}" + + @property + def documentation_file_path(self) -> Optional[Path]: + return Path(f"{self.relative_documentation_path_str}.md") if self.has_airbyte_docs else None + + @property + def inapp_documentation_file_path(self) -> Path: + if not self.has_airbyte_docs: + return None + + return Path(f"{self.relative_documentation_path_str}.inapp.md") @property def migration_guide_file_name(self) -> str: @@ -224,7 +347,7 @@ def migration_guide_file_name(self) -> str: @property def migration_guide_file_path(self) -> Path: - return self.documentation_directory / self.migration_guide_file_name + return self.local_connector_documentation_directory / self.migration_guide_file_name @property def icon_path(self) -> Path: @@ -233,7 +356,11 @@ def icon_path(self) -> Path: @property def code_directory(self) -> Path: - return Path(f"./airbyte-integrations/connectors/{self.technical_name}") + return Path(f"./{CONNECTOR_PATH_PREFIX}/{self.relative_connector_path}") + + @property + def has_dockerfile(self) -> bool: + return (self.code_directory / "Dockerfile").is_file() @property def metadata_file_path(self) -> Path: @@ -250,25 +377,22 @@ def metadata(self) -> Optional[dict]: def language(self) -> ConnectorLanguage: if Path(self.code_directory / self.technical_name.replace("-", "_") / "manifest.yaml").is_file(): return ConnectorLanguage.LOW_CODE - if Path(self.code_directory / "setup.py").is_file(): + if Path(self.code_directory / "setup.py").is_file() or Path(self.code_directory / "pyproject.toml").is_file(): return ConnectorLanguage.PYTHON - try: - with open(self.code_directory / "Dockerfile") as dockerfile: - if "FROM airbyte/integration-base-java" in dockerfile.read(): - return ConnectorLanguage.JAVA - except FileNotFoundError: - pass + if Path(self.code_directory / "src" / "main" / "java").exists(): + return ConnectorLanguage.JAVA return None - # raise ConnectorLanguageError(f"We could not infer {self.technical_name} connector language") @property - def version(self) -> str: + def version(self) -> Optional[str]: if self.metadata is None: return self.version_in_dockerfile_label return self.metadata["dockerImageTag"] @property - def version_in_dockerfile_label(self) -> str: + def version_in_dockerfile_label(self) -> Optional[str]: + if not self.has_dockerfile: + return None with open(self.code_directory / "Dockerfile") as f: for line in f: if "io.airbyte.version" in line: @@ -288,6 +412,37 @@ def name_from_metadata(self) -> Optional[str]: def support_level(self) -> Optional[str]: return self.metadata.get("supportLevel") if self.metadata else None + def metadata_query_match(self, query_string: str) -> bool: + """Evaluate a query string against the connector metadata. + + Based on the simpleeval library: + https://github.com/danthedeckie/simpleeval + + Examples + -------- + >>> connector.metadata_query_match("'s3' in data.name") + True + + >>> connector.metadata_query_match("data.supportLevel == 'certified'") + False + + >>> connector.metadata_query_match("data.ab_internal.ql >= 100") + True + + Args: + query_string (str): The query string to evaluate. + + Returns: + bool: True if the query string matches the connector metadata, False otherwise. + """ + try: + matches = simple_eval(query_string, names={"data": self.metadata}) + return bool(matches) + except Exception as e: + # Skip on error as we not all fields are present in all connectors. + logging.debug(f"Failed to evaluate query string {query_string} for connector {self.technical_name}, error: {e}") + return False + @property def ab_internal_sl(self) -> int: """Airbyte Internal Field. @@ -329,16 +484,16 @@ def ab_internal_ql(self) -> int: return ql_value @property - def is_important_connector(self) -> bool: - """Check if a connector qualifies as an important connector. + def is_strategic_connector(self) -> bool: + """Check if a connector qualifies as a strategic connector. Returns: bool: True if the connector is a high value connector, False otherwise. """ - if self.ab_internal_sl >= IMPORTANT_CONNECTOR_THRESHOLDS["sl"]: + if self.ab_internal_sl >= STRATEGIC_CONNECTOR_THRESHOLDS["sl"]: return True - if self.ab_internal_ql >= IMPORTANT_CONNECTOR_THRESHOLDS["ql"]: + if self.ab_internal_ql >= STRATEGIC_CONNECTOR_THRESHOLDS["ql"]: return True return False @@ -350,7 +505,7 @@ def requires_high_test_strictness_level(self) -> bool: Returns: bool: True if the connector requires high test strictness level, False otherwise. """ - return self.ab_internal_ql >= IMPORTANT_CONNECTOR_THRESHOLDS["ql"] + return self.ab_internal_ql >= STRATEGIC_CONNECTOR_THRESHOLDS["ql"] @property def requires_allowed_hosts_check(self) -> bool: @@ -396,6 +551,10 @@ def normalization_tag(self) -> Optional[str]: if self.supports_normalization: return f"{self.metadata['normalizationConfig']['normalizationTag']}" + @property + def is_using_poetry(self) -> bool: + return Path(self.code_directory / "pyproject.toml").exists() + def get_secret_manager(self, gsm_credentials: str): return SecretsManager(connector_name=self.technical_name, gsm_credentials=gsm_credentials) @@ -436,6 +595,26 @@ def get_changed_connectors( return {Connector(get_connector_name_from_path(changed_file)) for changed_file in changed_source_connector_files} +def _get_relative_connector_folder_name_from_metadata_path(metadata_file_path: str) -> str: + """Get the relative connector folder name from the metadata file path. + + Args: + metadata_file_path (Path): Path to the metadata file. + + Returns: + str: The relative connector folder name. + """ + # remove CONNECTOR_PATH_PREFIX and anything before + metadata_file_path = metadata_file_path.split(CONNECTOR_PATH_PREFIX)[-1] + + # remove metadata.yaml + metadata_file_path = metadata_file_path.replace(METADATA_FILE_NAME, "") + + # remove leading and trailing slashes + metadata_file_path = metadata_file_path.strip("/") + return metadata_file_path + + def get_all_connectors_in_repo() -> Set[Connector]: """Retrieve a set of all Connectors in the repo. We globe the connectors folder for metadata.yaml files and construct Connectors from the directory name. @@ -447,8 +626,8 @@ def get_all_connectors_in_repo() -> Set[Connector]: repo_path = repo.working_tree_dir return { - Connector(Path(metadata_file).parent.name) - for metadata_file in glob(f"{repo_path}/airbyte-integrations/connectors/**/metadata.yaml", recursive=True) + Connector(_get_relative_connector_folder_name_from_metadata_path(metadata_file)) + for metadata_file in glob(f"{repo_path}/{CONNECTOR_PATH_PREFIX}/**/metadata.yaml", recursive=True) if SCAFFOLD_CONNECTOR_GLOB not in metadata_file } diff --git a/airbyte-ci/connectors/connector_ops/poetry.lock b/airbyte-ci/connectors/connector_ops/poetry.lock index 1a1e547ab392..f852e65d56c7 100644 --- a/airbyte-ci/connectors/connector_ops/poetry.lock +++ b/airbyte-ci/connectors/connector_ops/poetry.lock @@ -24,75 +24,63 @@ files = [ [[package]] name = "cffi" -version = "1.15.1" +version = "1.16.0" description = "Foreign Function Interface for Python calling C code." optional = false -python-versions = "*" +python-versions = ">=3.8" files = [ - {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, - {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, - {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, - {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, - {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, - {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, - {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, - {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, - {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, - {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, - {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, - {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, - {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, - {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, - {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, - {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, - {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, - {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, - {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, ] [package.dependencies] @@ -100,86 +88,101 @@ pycparser = "*" [[package]] name = "charset-normalizer" -version = "3.2.0" +version = "3.3.0" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7.0" files = [ - {file = "charset-normalizer-3.2.0.tar.gz", hash = "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-win32.whl", hash = "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-win32.whl", hash = "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-win32.whl", hash = "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-win32.whl", hash = "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-win32.whl", hash = "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80"}, - {file = "charset_normalizer-3.2.0-py3-none-any.whl", hash = "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6"}, + {file = "charset-normalizer-3.3.0.tar.gz", hash = "sha256:63563193aec44bce707e0c5ca64ff69fa72ed7cf34ce6e11d5127555756fd2f6"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:effe5406c9bd748a871dbcaf3ac69167c38d72db8c9baf3ff954c344f31c4cbe"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4162918ef3098851fcd8a628bf9b6a98d10c380725df9e04caf5ca6dd48c847a"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0570d21da019941634a531444364f2482e8db0b3425fcd5ac0c36565a64142c8"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5707a746c6083a3a74b46b3a631d78d129edab06195a92a8ece755aac25a3f3d"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:278c296c6f96fa686d74eb449ea1697f3c03dc28b75f873b65b5201806346a69"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a4b71f4d1765639372a3b32d2638197f5cd5221b19531f9245fcc9ee62d38f56"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5969baeaea61c97efa706b9b107dcba02784b1601c74ac84f2a532ea079403e"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a3f93dab657839dfa61025056606600a11d0b696d79386f974e459a3fbc568ec"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:db756e48f9c5c607b5e33dd36b1d5872d0422e960145b08ab0ec7fd420e9d649"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:232ac332403e37e4a03d209a3f92ed9071f7d3dbda70e2a5e9cff1c4ba9f0678"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e5c1502d4ace69a179305abb3f0bb6141cbe4714bc9b31d427329a95acfc8bdd"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:2502dd2a736c879c0f0d3e2161e74d9907231e25d35794584b1ca5284e43f596"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23e8565ab7ff33218530bc817922fae827420f143479b753104ab801145b1d5b"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-win32.whl", hash = "sha256:1872d01ac8c618a8da634e232f24793883d6e456a66593135aeafe3784b0848d"}, + {file = "charset_normalizer-3.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:557b21a44ceac6c6b9773bc65aa1b4cc3e248a5ad2f5b914b91579a32e22204d"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d7eff0f27edc5afa9e405f7165f85a6d782d308f3b6b9d96016c010597958e63"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6a685067d05e46641d5d1623d7c7fdf15a357546cbb2f71b0ebde91b175ffc3e"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0d3d5b7db9ed8a2b11a774db2bbea7ba1884430a205dbd54a32d61d7c2a190fa"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2935ffc78db9645cb2086c2f8f4cfd23d9b73cc0dc80334bc30aac6f03f68f8c"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fe359b2e3a7729010060fbca442ca225280c16e923b37db0e955ac2a2b72a05"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:380c4bde80bce25c6e4f77b19386f5ec9db230df9f2f2ac1e5ad7af2caa70459"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f0d1e3732768fecb052d90d62b220af62ead5748ac51ef61e7b32c266cac9293"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1b2919306936ac6efb3aed1fbf81039f7087ddadb3160882a57ee2ff74fd2382"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f8888e31e3a85943743f8fc15e71536bda1c81d5aa36d014a3c0c44481d7db6e"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:82eb849f085624f6a607538ee7b83a6d8126df6d2f7d3b319cb837b289123078"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7b8b8bf1189b3ba9b8de5c8db4d541b406611a71a955bbbd7385bbc45fcb786c"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:5adf257bd58c1b8632046bbe43ee38c04e1038e9d37de9c57a94d6bd6ce5da34"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c350354efb159b8767a6244c166f66e67506e06c8924ed74669b2c70bc8735b1"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-win32.whl", hash = "sha256:02af06682e3590ab952599fbadac535ede5d60d78848e555aa58d0c0abbde786"}, + {file = "charset_normalizer-3.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:86d1f65ac145e2c9ed71d8ffb1905e9bba3a91ae29ba55b4c46ae6fc31d7c0d4"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:3b447982ad46348c02cb90d230b75ac34e9886273df3a93eec0539308a6296d7"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:abf0d9f45ea5fb95051c8bfe43cb40cda383772f7e5023a83cc481ca2604d74e"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b09719a17a2301178fac4470d54b1680b18a5048b481cb8890e1ef820cb80455"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3d9b48ee6e3967b7901c052b670c7dda6deb812c309439adaffdec55c6d7b78"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:edfe077ab09442d4ef3c52cb1f9dab89bff02f4524afc0acf2d46be17dc479f5"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3debd1150027933210c2fc321527c2299118aa929c2f5a0a80ab6953e3bd1908"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86f63face3a527284f7bb8a9d4f78988e3c06823f7bea2bd6f0e0e9298ca0403"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:24817cb02cbef7cd499f7c9a2735286b4782bd47a5b3516a0e84c50eab44b98e"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c71f16da1ed8949774ef79f4a0260d28b83b3a50c6576f8f4f0288d109777989"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:9cf3126b85822c4e53aa28c7ec9869b924d6fcfb76e77a45c44b83d91afd74f9"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:b3b2316b25644b23b54a6f6401074cebcecd1244c0b8e80111c9a3f1c8e83d65"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:03680bb39035fbcffe828eae9c3f8afc0428c91d38e7d61aa992ef7a59fb120e"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4cc152c5dd831641e995764f9f0b6589519f6f5123258ccaca8c6d34572fefa8"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-win32.whl", hash = "sha256:b8f3307af845803fb0b060ab76cf6dd3a13adc15b6b451f54281d25911eb92df"}, + {file = "charset_normalizer-3.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:8eaf82f0eccd1505cf39a45a6bd0a8cf1c70dcfc30dba338207a969d91b965c0"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:dc45229747b67ffc441b3de2f3ae5e62877a282ea828a5bdb67883c4ee4a8810"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f4a0033ce9a76e391542c182f0d48d084855b5fcba5010f707c8e8c34663d77"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ada214c6fa40f8d800e575de6b91a40d0548139e5dc457d2ebb61470abf50186"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b1121de0e9d6e6ca08289583d7491e7fcb18a439305b34a30b20d8215922d43c"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1063da2c85b95f2d1a430f1c33b55c9c17ffaf5e612e10aeaad641c55a9e2b9d"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:70f1d09c0d7748b73290b29219e854b3207aea922f839437870d8cc2168e31cc"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:250c9eb0f4600361dd80d46112213dff2286231d92d3e52af1e5a6083d10cad9"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:750b446b2ffce1739e8578576092179160f6d26bd5e23eb1789c4d64d5af7dc7"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:fc52b79d83a3fe3a360902d3f5d79073a993597d48114c29485e9431092905d8"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:588245972aca710b5b68802c8cad9edaa98589b1b42ad2b53accd6910dad3545"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e39c7eb31e3f5b1f88caff88bcff1b7f8334975b46f6ac6e9fc725d829bc35d4"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-win32.whl", hash = "sha256:abecce40dfebbfa6abf8e324e1860092eeca6f7375c8c4e655a8afb61af58f2c"}, + {file = "charset_normalizer-3.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:24a91a981f185721542a0b7c92e9054b7ab4fea0508a795846bc5b0abf8118d4"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:67b8cc9574bb518ec76dc8e705d4c39ae78bb96237cb533edac149352c1f39fe"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ac71b2977fb90c35d41c9453116e283fac47bb9096ad917b8819ca8b943abecd"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3ae38d325b512f63f8da31f826e6cb6c367336f95e418137286ba362925c877e"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:542da1178c1c6af8873e143910e2269add130a299c9106eef2594e15dae5e482"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:30a85aed0b864ac88309b7d94be09f6046c834ef60762a8833b660139cfbad13"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aae32c93e0f64469f74ccc730a7cb21c7610af3a775157e50bbd38f816536b38"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15b26ddf78d57f1d143bdf32e820fd8935d36abe8a25eb9ec0b5a71c82eb3895"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f5d10bae5d78e4551b7be7a9b29643a95aded9d0f602aa2ba584f0388e7a557"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:249c6470a2b60935bafd1d1d13cd613f8cd8388d53461c67397ee6a0f5dce741"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c5a74c359b2d47d26cdbbc7845e9662d6b08a1e915eb015d044729e92e7050b7"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:b5bcf60a228acae568e9911f410f9d9e0d43197d030ae5799e20dca8df588287"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:187d18082694a29005ba2944c882344b6748d5be69e3a89bf3cc9d878e548d5a"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:81bf654678e575403736b85ba3a7867e31c2c30a69bc57fe88e3ace52fb17b89"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-win32.whl", hash = "sha256:85a32721ddde63c9df9ebb0d2045b9691d9750cb139c161c80e500d210f5e26e"}, + {file = "charset_normalizer-3.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:468d2a840567b13a590e67dd276c570f8de00ed767ecc611994c301d0f8c014f"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e0fc42822278451bc13a2e8626cf2218ba570f27856b536e00cfa53099724828"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:09c77f964f351a7369cc343911e0df63e762e42bac24cd7d18525961c81754f4"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:12ebea541c44fdc88ccb794a13fe861cc5e35d64ed689513a5c03d05b53b7c82"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:805dfea4ca10411a5296bcc75638017215a93ffb584c9e344731eef0dcfb026a"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:96c2b49eb6a72c0e4991d62406e365d87067ca14c1a729a870d22354e6f68115"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aaf7b34c5bc56b38c931a54f7952f1ff0ae77a2e82496583b247f7c969eb1479"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:619d1c96099be5823db34fe89e2582b336b5b074a7f47f819d6b3a57ff7bdb86"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a0ac5e7015a5920cfce654c06618ec40c33e12801711da6b4258af59a8eff00a"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:93aa7eef6ee71c629b51ef873991d6911b906d7312c6e8e99790c0f33c576f89"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7966951325782121e67c81299a031f4c115615e68046f79b85856b86ebffc4cd"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:02673e456dc5ab13659f85196c534dc596d4ef260e4d86e856c3b2773ce09843"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:c2af80fb58f0f24b3f3adcb9148e6203fa67dd3f61c4af146ecad033024dde43"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:153e7b6e724761741e0974fc4dcd406d35ba70b92bfe3fedcb497226c93b9da7"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-win32.whl", hash = "sha256:d47ecf253780c90ee181d4d871cd655a789da937454045b17b5798da9393901a"}, + {file = "charset_normalizer-3.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:d97d85fa63f315a8bdaba2af9a6a686e0eceab77b3089af45133252618e70884"}, + {file = "charset_normalizer-3.3.0-py3-none-any.whl", hash = "sha256:e46cd37076971c1040fc8c41273a8b3e2c624ce4f2be3f5dfcb7a430c1d3acc2"}, ] [[package]] @@ -203,13 +206,13 @@ url = "../ci_credentials" [[package]] name = "click" -version = "8.1.6" +version = "8.1.7" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" files = [ - {file = "click-8.1.6-py3-none-any.whl", hash = "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"}, - {file = "click-8.1.6.tar.gz", hash = "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd"}, + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, ] [package.dependencies] @@ -244,20 +247,6 @@ requests = "^2.28.2" type = "directory" url = "../common_utils" -[[package]] -name = "commonmark" -version = "0.9.1" -description = "Python parser for the CommonMark Markdown spec" -optional = false -python-versions = "*" -files = [ - {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, - {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, -] - -[package.extras] -test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] - [[package]] name = "cryptography" version = "3.4.8" @@ -358,27 +347,30 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.32" +version = "3.1.37" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" files = [ - {file = "GitPython-3.1.32-py3-none-any.whl", hash = "sha256:e3d59b1c2c6ebb9dfa7a184daf3b6dd4914237e7488a1730a6d8f6f5d0b4187f"}, - {file = "GitPython-3.1.32.tar.gz", hash = "sha256:8d9b8cb1e80b9735e8717c9362079d3ce4c6e5ddeebedd0361b228c3a67a62f6"}, + {file = "GitPython-3.1.37-py3-none-any.whl", hash = "sha256:5f4c4187de49616d710a77e98ddf17b4782060a1788df441846bddefbb89ab33"}, + {file = "GitPython-3.1.37.tar.gz", hash = "sha256:f9b9ddc0761c125d5780eab2d64be4873fc6817c2899cbcb34b02344bdc7bc54"}, ] [package.dependencies] gitdb = ">=4.0.1,<5" +[package.extras] +test = ["black", "coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mypy", "pre-commit", "pytest", "pytest-cov", "pytest-sugar"] + [[package]] name = "google-api-core" -version = "2.11.1" +version = "2.12.0" description = "Google API client core library" optional = false python-versions = ">=3.7" files = [ - {file = "google-api-core-2.11.1.tar.gz", hash = "sha256:25d29e05a0058ed5f19c61c0a78b1b53adea4d9364b464d014fbda941f6d1c9a"}, - {file = "google_api_core-2.11.1-py3-none-any.whl", hash = "sha256:d92a5a92dc36dd4f4b9ee4e55528a90e432b059f93aee6ad857f9de8cc7ae94a"}, + {file = "google-api-core-2.12.0.tar.gz", hash = "sha256:c22e01b1e3c4dcd90998494879612c38d0a3411d1f7b679eb89e2abe3ce1f553"}, + {file = "google_api_core-2.12.0-py3-none-any.whl", hash = "sha256:ec6054f7d64ad13b41e43d96f735acbd763b0f3b695dabaa2d579673f6a6e160"}, ] [package.dependencies] @@ -394,21 +386,19 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] [[package]] name = "google-auth" -version = "2.22.0" +version = "2.23.3" description = "Google Authentication Library" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "google-auth-2.22.0.tar.gz", hash = "sha256:164cba9af4e6e4e40c3a4f90a1a6c12ee56f14c0b4868d1ca91b32826ab334ce"}, - {file = "google_auth-2.22.0-py2.py3-none-any.whl", hash = "sha256:d61d1b40897407b574da67da1a833bdc10d5a11642566e506565d1b1a46ba873"}, + {file = "google-auth-2.23.3.tar.gz", hash = "sha256:6864247895eea5d13b9c57c9e03abb49cb94ce2dc7c58e91cba3248c7477c9e3"}, + {file = "google_auth-2.23.3-py2.py3-none-any.whl", hash = "sha256:a8f4608e65c244ead9e0538f181a96c6e11199ec114d41f1d7b1bffa96937bda"}, ] [package.dependencies] cachetools = ">=2.0.0,<6.0" pyasn1-modules = ">=0.2.1" rsa = ">=3.1.4,<5" -six = ">=1.9.0" -urllib3 = "<2.0" [package.extras] aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] @@ -437,20 +427,20 @@ grpc = ["grpcio (>=1.38.0,<2.0dev)"] [[package]] name = "google-cloud-storage" -version = "2.10.0" +version = "2.11.0" description = "Google Cloud Storage API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-storage-2.10.0.tar.gz", hash = "sha256:934b31ead5f3994e5360f9ff5750982c5b6b11604dc072bc452c25965e076dc7"}, - {file = "google_cloud_storage-2.10.0-py2.py3-none-any.whl", hash = "sha256:9433cf28801671de1c80434238fb1e7e4a1ba3087470e90f70c928ea77c2b9d7"}, + {file = "google-cloud-storage-2.11.0.tar.gz", hash = "sha256:6fbf62659b83c8f3a0a743af0d661d2046c97c3a5bfb587c4662c4bc68de3e31"}, + {file = "google_cloud_storage-2.11.0-py2.py3-none-any.whl", hash = "sha256:88cbd7fb3d701c780c4272bc26952db99f25eb283fb4c2208423249f00b5fe53"}, ] [package.dependencies] google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" google-auth = ">=1.25.0,<3.0dev" google-cloud-core = ">=2.3.0,<3.0dev" -google-resumable-media = ">=2.3.2" +google-resumable-media = ">=2.6.0" requests = ">=2.18.0,<3.0.0dev" [package.extras] @@ -538,20 +528,20 @@ testing = ["pytest"] [[package]] name = "google-resumable-media" -version = "2.5.0" +version = "2.6.0" description = "Utilities for Google Media Downloads and Resumable Uploads" optional = false python-versions = ">= 3.7" files = [ - {file = "google-resumable-media-2.5.0.tar.gz", hash = "sha256:218931e8e2b2a73a58eb354a288e03a0fd5fb1c4583261ac6e4c078666468c93"}, - {file = "google_resumable_media-2.5.0-py2.py3-none-any.whl", hash = "sha256:da1bd943e2e114a56d85d6848497ebf9be6a14d3db23e9fc57581e7c3e8170ec"}, + {file = "google-resumable-media-2.6.0.tar.gz", hash = "sha256:972852f6c65f933e15a4a210c2b96930763b47197cdf4aa5f5bea435efb626e7"}, + {file = "google_resumable_media-2.6.0-py2.py3-none-any.whl", hash = "sha256:fc03d344381970f79eebb632a3c18bb1828593a2dc5572b5f90115ef7d11e81b"}, ] [package.dependencies] google-crc32c = ">=1.0,<2.0dev" [package.extras] -aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)"] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)", "google-auth (>=1.22.0,<2.0dev)"] requests = ["requests (>=2.18.0,<3.0.0dev)"] [[package]] @@ -593,6 +583,41 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] +[[package]] +name = "markdown-it-py" +version = "3.0.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +optional = false +python-versions = ">=3.8" +files = [ + {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, + {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, +] + +[package.dependencies] +mdurl = ">=0.1,<1.0" + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +code-style = ["pre-commit (>=3.0,<4.0)"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] + [[package]] name = "numpy" version = "1.25.2" @@ -627,92 +652,192 @@ files = [ {file = "numpy-1.25.2.tar.gz", hash = "sha256:fd608e19c8d7c55021dffd43bfe5492fab8cc105cc8986f813f8c3c048b38760"}, ] +[[package]] +name = "numpy" +version = "1.26.0" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = "<3.13,>=3.9" +files = [ + {file = "numpy-1.26.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f8db2f125746e44dce707dd44d4f4efeea8d7e2b43aace3f8d1f235cfa2733dd"}, + {file = "numpy-1.26.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0621f7daf973d34d18b4e4bafb210bbaf1ef5e0100b5fa750bd9cde84c7ac292"}, + {file = "numpy-1.26.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51be5f8c349fdd1a5568e72713a21f518e7d6707bcf8503b528b88d33b57dc68"}, + {file = "numpy-1.26.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:767254ad364991ccfc4d81b8152912e53e103ec192d1bb4ea6b1f5a7117040be"}, + {file = "numpy-1.26.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:436c8e9a4bdeeee84e3e59614d38c3dbd3235838a877af8c211cfcac8a80b8d3"}, + {file = "numpy-1.26.0-cp310-cp310-win32.whl", hash = "sha256:c2e698cb0c6dda9372ea98a0344245ee65bdc1c9dd939cceed6bb91256837896"}, + {file = "numpy-1.26.0-cp310-cp310-win_amd64.whl", hash = "sha256:09aaee96c2cbdea95de76ecb8a586cb687d281c881f5f17bfc0fb7f5890f6b91"}, + {file = "numpy-1.26.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:637c58b468a69869258b8ae26f4a4c6ff8abffd4a8334c830ffb63e0feefe99a"}, + {file = "numpy-1.26.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:306545e234503a24fe9ae95ebf84d25cba1fdc27db971aa2d9f1ab6bba19a9dd"}, + {file = "numpy-1.26.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c6adc33561bd1d46f81131d5352348350fc23df4d742bb246cdfca606ea1208"}, + {file = "numpy-1.26.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e062aa24638bb5018b7841977c360d2f5917268d125c833a686b7cbabbec496c"}, + {file = "numpy-1.26.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:546b7dd7e22f3c6861463bebb000646fa730e55df5ee4a0224408b5694cc6148"}, + {file = "numpy-1.26.0-cp311-cp311-win32.whl", hash = "sha256:c0b45c8b65b79337dee5134d038346d30e109e9e2e9d43464a2970e5c0e93229"}, + {file = "numpy-1.26.0-cp311-cp311-win_amd64.whl", hash = "sha256:eae430ecf5794cb7ae7fa3808740b015aa80747e5266153128ef055975a72b99"}, + {file = "numpy-1.26.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:166b36197e9debc4e384e9c652ba60c0bacc216d0fc89e78f973a9760b503388"}, + {file = "numpy-1.26.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f042f66d0b4ae6d48e70e28d487376204d3cbf43b84c03bac57e28dac6151581"}, + {file = "numpy-1.26.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5e18e5b14a7560d8acf1c596688f4dfd19b4f2945b245a71e5af4ddb7422feb"}, + {file = "numpy-1.26.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f6bad22a791226d0a5c7c27a80a20e11cfe09ad5ef9084d4d3fc4a299cca505"}, + {file = "numpy-1.26.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4acc65dd65da28060e206c8f27a573455ed724e6179941edb19f97e58161bb69"}, + {file = "numpy-1.26.0-cp312-cp312-win32.whl", hash = "sha256:bb0d9a1aaf5f1cb7967320e80690a1d7ff69f1d47ebc5a9bea013e3a21faec95"}, + {file = "numpy-1.26.0-cp312-cp312-win_amd64.whl", hash = "sha256:ee84ca3c58fe48b8ddafdeb1db87388dce2c3c3f701bf447b05e4cfcc3679112"}, + {file = "numpy-1.26.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4a873a8180479bc829313e8d9798d5234dfacfc2e8a7ac188418189bb8eafbd2"}, + {file = "numpy-1.26.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:914b28d3215e0c721dc75db3ad6d62f51f630cb0c277e6b3bcb39519bed10bd8"}, + {file = "numpy-1.26.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c78a22e95182fb2e7874712433eaa610478a3caf86f28c621708d35fa4fd6e7f"}, + {file = "numpy-1.26.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86f737708b366c36b76e953c46ba5827d8c27b7a8c9d0f471810728e5a2fe57c"}, + {file = "numpy-1.26.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b44e6a09afc12952a7d2a58ca0a2429ee0d49a4f89d83a0a11052da696440e49"}, + {file = "numpy-1.26.0-cp39-cp39-win32.whl", hash = "sha256:5671338034b820c8d58c81ad1dafc0ed5a00771a82fccc71d6438df00302094b"}, + {file = "numpy-1.26.0-cp39-cp39-win_amd64.whl", hash = "sha256:020cdbee66ed46b671429c7265cf00d8ac91c046901c55684954c3958525dab2"}, + {file = "numpy-1.26.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0792824ce2f7ea0c82ed2e4fecc29bb86bee0567a080dacaf2e0a01fe7654369"}, + {file = "numpy-1.26.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d484292eaeb3e84a51432a94f53578689ffdea3f90e10c8b203a99be5af57d8"}, + {file = "numpy-1.26.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:186ba67fad3c60dbe8a3abff3b67a91351100f2661c8e2a80364ae6279720299"}, + {file = "numpy-1.26.0.tar.gz", hash = "sha256:f93fc78fe8bf15afe2b8d6b6499f1c73953169fad1e9a8dd086cdff3190e7fdf"}, +] + [[package]] name = "packaging" -version = "23.1" +version = "23.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, - {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, ] [[package]] name = "pandas" -version = "2.0.3" +version = "2.1.0" description = "Powerful data structures for data analysis, time series, and statistics" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +files = [ + {file = "pandas-2.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:40dd20439ff94f1b2ed55b393ecee9cb6f3b08104c2c40b0cb7186a2f0046242"}, + {file = "pandas-2.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d4f38e4fedeba580285eaac7ede4f686c6701a9e618d8a857b138a126d067f2f"}, + {file = "pandas-2.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6e6a0fe052cf27ceb29be9429428b4918f3740e37ff185658f40d8702f0b3e09"}, + {file = "pandas-2.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d81e1813191070440d4c7a413cb673052b3b4a984ffd86b8dd468c45742d3cc"}, + {file = "pandas-2.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:eb20252720b1cc1b7d0b2879ffc7e0542dd568f24d7c4b2347cb035206936421"}, + {file = "pandas-2.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:38f74ef7ebc0ffb43b3d633e23d74882bce7e27bfa09607f3c5d3e03ffd9a4a5"}, + {file = "pandas-2.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cda72cc8c4761c8f1d97b169661f23a86b16fdb240bdc341173aee17e4d6cedd"}, + {file = "pandas-2.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d97daeac0db8c993420b10da4f5f5b39b01fc9ca689a17844e07c0a35ac96b4b"}, + {file = "pandas-2.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8c58b1113892e0c8078f006a167cc210a92bdae23322bb4614f2f0b7a4b510f"}, + {file = "pandas-2.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:629124923bcf798965b054a540f9ccdfd60f71361255c81fa1ecd94a904b9dd3"}, + {file = "pandas-2.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:70cf866af3ab346a10debba8ea78077cf3a8cd14bd5e4bed3d41555a3280041c"}, + {file = "pandas-2.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:d53c8c1001f6a192ff1de1efe03b31a423d0eee2e9e855e69d004308e046e694"}, + {file = "pandas-2.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:86f100b3876b8c6d1a2c66207288ead435dc71041ee4aea789e55ef0e06408cb"}, + {file = "pandas-2.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:28f330845ad21c11db51e02d8d69acc9035edfd1116926ff7245c7215db57957"}, + {file = "pandas-2.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9a6ccf0963db88f9b12df6720e55f337447aea217f426a22d71f4213a3099a6"}, + {file = "pandas-2.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d99e678180bc59b0c9443314297bddce4ad35727a1a2656dbe585fd78710b3b9"}, + {file = "pandas-2.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b31da36d376d50a1a492efb18097b9101bdbd8b3fbb3f49006e02d4495d4c644"}, + {file = "pandas-2.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:0164b85937707ec7f70b34a6c3a578dbf0f50787f910f21ca3b26a7fd3363437"}, + {file = "pandas-2.1.0.tar.gz", hash = "sha256:62c24c7fc59e42b775ce0679cfa7b14a5f9bfb7643cfbe708c960699e05fb918"}, +] + +[package.dependencies] +numpy = {version = ">=1.23.2", markers = "python_version >= \"3.11\""} +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.1" + +[package.extras] +all = ["PyQt5 (>=5.15.6)", "SQLAlchemy (>=1.4.36)", "beautifulsoup4 (>=4.11.1)", "bottleneck (>=1.3.4)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=0.8.1)", "fsspec (>=2022.05.0)", "gcsfs (>=2022.05.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.8.0)", "matplotlib (>=3.6.1)", "numba (>=0.55.2)", "numexpr (>=2.8.0)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pandas-gbq (>=0.17.5)", "psycopg2 (>=2.9.3)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.5)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "pyxlsb (>=1.0.9)", "qtpy (>=2.2.0)", "s3fs (>=2022.05.0)", "scipy (>=1.8.1)", "tables (>=3.7.0)", "tabulate (>=0.8.10)", "xarray (>=2022.03.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)", "zstandard (>=0.17.0)"] +aws = ["s3fs (>=2022.05.0)"] +clipboard = ["PyQt5 (>=5.15.6)", "qtpy (>=2.2.0)"] +compression = ["zstandard (>=0.17.0)"] +computation = ["scipy (>=1.8.1)", "xarray (>=2022.03.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pyxlsb (>=1.0.9)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)"] +feather = ["pyarrow (>=7.0.0)"] +fss = ["fsspec (>=2022.05.0)"] +gcp = ["gcsfs (>=2022.05.0)", "pandas-gbq (>=0.17.5)"] +hdf5 = ["tables (>=3.7.0)"] +html = ["beautifulsoup4 (>=4.11.1)", "html5lib (>=1.1)", "lxml (>=4.8.0)"] +mysql = ["SQLAlchemy (>=1.4.36)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.8.10)"] +parquet = ["pyarrow (>=7.0.0)"] +performance = ["bottleneck (>=1.3.4)", "numba (>=0.55.2)", "numexpr (>=2.8.0)"] +plot = ["matplotlib (>=3.6.1)"] +postgresql = ["SQLAlchemy (>=1.4.36)", "psycopg2 (>=2.9.3)"] +spss = ["pyreadstat (>=1.1.5)"] +sql-other = ["SQLAlchemy (>=1.4.36)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.8.0)"] + +[[package]] +name = "pandas" +version = "2.1.1" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.9" files = [ - {file = "pandas-2.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e4c7c9f27a4185304c7caf96dc7d91bc60bc162221152de697c98eb0b2648dd8"}, - {file = "pandas-2.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f167beed68918d62bffb6ec64f2e1d8a7d297a038f86d4aed056b9493fca407f"}, - {file = "pandas-2.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce0c6f76a0f1ba361551f3e6dceaff06bde7514a374aa43e33b588ec10420183"}, - {file = "pandas-2.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba619e410a21d8c387a1ea6e8a0e49bb42216474436245718d7f2e88a2f8d7c0"}, - {file = "pandas-2.0.3-cp310-cp310-win32.whl", hash = "sha256:3ef285093b4fe5058eefd756100a367f27029913760773c8bf1d2d8bebe5d210"}, - {file = "pandas-2.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:9ee1a69328d5c36c98d8e74db06f4ad518a1840e8ccb94a4ba86920986bb617e"}, - {file = "pandas-2.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b084b91d8d66ab19f5bb3256cbd5ea661848338301940e17f4492b2ce0801fe8"}, - {file = "pandas-2.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37673e3bdf1551b95bf5d4ce372b37770f9529743d2498032439371fc7b7eb26"}, - {file = "pandas-2.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9cb1e14fdb546396b7e1b923ffaeeac24e4cedd14266c3497216dd4448e4f2d"}, - {file = "pandas-2.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9cd88488cceb7635aebb84809d087468eb33551097d600c6dad13602029c2df"}, - {file = "pandas-2.0.3-cp311-cp311-win32.whl", hash = "sha256:694888a81198786f0e164ee3a581df7d505024fbb1f15202fc7db88a71d84ebd"}, - {file = "pandas-2.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6a21ab5c89dcbd57f78d0ae16630b090eec626360085a4148693def5452d8a6b"}, - {file = "pandas-2.0.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9e4da0d45e7f34c069fe4d522359df7d23badf83abc1d1cef398895822d11061"}, - {file = "pandas-2.0.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:32fca2ee1b0d93dd71d979726b12b61faa06aeb93cf77468776287f41ff8fdc5"}, - {file = "pandas-2.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:258d3624b3ae734490e4d63c430256e716f488c4fcb7c8e9bde2d3aa46c29089"}, - {file = "pandas-2.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eae3dc34fa1aa7772dd3fc60270d13ced7346fcbcfee017d3132ec625e23bb0"}, - {file = "pandas-2.0.3-cp38-cp38-win32.whl", hash = "sha256:f3421a7afb1a43f7e38e82e844e2bca9a6d793d66c1a7f9f0ff39a795bbc5e02"}, - {file = "pandas-2.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:69d7f3884c95da3a31ef82b7618af5710dba95bb885ffab339aad925c3e8ce78"}, - {file = "pandas-2.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5247fb1ba347c1261cbbf0fcfba4a3121fbb4029d95d9ef4dc45406620b25c8b"}, - {file = "pandas-2.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:81af086f4543c9d8bb128328b5d32e9986e0c84d3ee673a2ac6fb57fd14f755e"}, - {file = "pandas-2.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1994c789bf12a7c5098277fb43836ce090f1073858c10f9220998ac74f37c69b"}, - {file = "pandas-2.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ec591c48e29226bcbb316e0c1e9423622bc7a4eaf1ef7c3c9fa1a3981f89641"}, - {file = "pandas-2.0.3-cp39-cp39-win32.whl", hash = "sha256:04dbdbaf2e4d46ca8da896e1805bc04eb85caa9a82e259e8eed00254d5e0c682"}, - {file = "pandas-2.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:1168574b036cd8b93abc746171c9b4f1b83467438a5e45909fed645cf8692dbc"}, - {file = "pandas-2.0.3.tar.gz", hash = "sha256:c02f372a88e0d17f36d3093a644c73cfc1788e876a7c4bcb4020a77512e2043c"}, + {file = "pandas-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:58d997dbee0d4b64f3cb881a24f918b5f25dd64ddf31f467bb9b67ae4c63a1e4"}, + {file = "pandas-2.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02304e11582c5d090e5a52aec726f31fe3f42895d6bfc1f28738f9b64b6f0614"}, + {file = "pandas-2.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffa8f0966de2c22de408d0e322db2faed6f6e74265aa0856f3824813cf124363"}, + {file = "pandas-2.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1f84c144dee086fe4f04a472b5cd51e680f061adf75c1ae4fc3a9275560f8f4"}, + {file = "pandas-2.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:75ce97667d06d69396d72be074f0556698c7f662029322027c226fd7a26965cb"}, + {file = "pandas-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:4c3f32fd7c4dccd035f71734df39231ac1a6ff95e8bdab8d891167197b7018d2"}, + {file = "pandas-2.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9e2959720b70e106bb1d8b6eadd8ecd7c8e99ccdbe03ee03260877184bb2877d"}, + {file = "pandas-2.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:25e8474a8eb258e391e30c288eecec565bfed3e026f312b0cbd709a63906b6f8"}, + {file = "pandas-2.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b8bd1685556f3374520466998929bade3076aeae77c3e67ada5ed2b90b4de7f0"}, + {file = "pandas-2.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc3657869c7902810f32bd072f0740487f9e030c1a3ab03e0af093db35a9d14e"}, + {file = "pandas-2.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:05674536bd477af36aa2effd4ec8f71b92234ce0cc174de34fd21e2ee99adbc2"}, + {file = "pandas-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:b407381258a667df49d58a1b637be33e514b07f9285feb27769cedb3ab3d0b3a"}, + {file = "pandas-2.1.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c747793c4e9dcece7bb20156179529898abf505fe32cb40c4052107a3c620b49"}, + {file = "pandas-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3bcad1e6fb34b727b016775bea407311f7721db87e5b409e6542f4546a4951ea"}, + {file = "pandas-2.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5ec7740f9ccb90aec64edd71434711f58ee0ea7f5ed4ac48be11cfa9abf7317"}, + {file = "pandas-2.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29deb61de5a8a93bdd033df328441a79fcf8dd3c12d5ed0b41a395eef9cd76f0"}, + {file = "pandas-2.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4f99bebf19b7e03cf80a4e770a3e65eee9dd4e2679039f542d7c1ace7b7b1daa"}, + {file = "pandas-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:84e7e910096416adec68075dc87b986ff202920fb8704e6d9c8c9897fe7332d6"}, + {file = "pandas-2.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:366da7b0e540d1b908886d4feb3d951f2f1e572e655c1160f5fde28ad4abb750"}, + {file = "pandas-2.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9e50e72b667415a816ac27dfcfe686dc5a0b02202e06196b943d54c4f9c7693e"}, + {file = "pandas-2.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc1ab6a25da197f03ebe6d8fa17273126120874386b4ac11c1d687df288542dd"}, + {file = "pandas-2.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0dbfea0dd3901ad4ce2306575c54348d98499c95be01b8d885a2737fe4d7a98"}, + {file = "pandas-2.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0489b0e6aa3d907e909aef92975edae89b1ee1654db5eafb9be633b0124abe97"}, + {file = "pandas-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:4cdb0fab0400c2cb46dafcf1a0fe084c8bb2480a1fa8d81e19d15e12e6d4ded2"}, + {file = "pandas-2.1.1.tar.gz", hash = "sha256:fecb198dc389429be557cde50a2d46da8434a17fe37d7d41ff102e3987fd947b"}, ] [package.dependencies] numpy = [ - {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, - {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, + {version = ">=1.22.4", markers = "python_version < \"3.11\""}, + {version = ">=1.23.2", markers = "python_version == \"3.11\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" tzdata = ">=2022.1" [package.extras] -all = ["PyQt5 (>=5.15.1)", "SQLAlchemy (>=1.4.16)", "beautifulsoup4 (>=4.9.3)", "bottleneck (>=1.3.2)", "brotlipy (>=0.7.0)", "fastparquet (>=0.6.3)", "fsspec (>=2021.07.0)", "gcsfs (>=2021.07.0)", "html5lib (>=1.1)", "hypothesis (>=6.34.2)", "jinja2 (>=3.0.0)", "lxml (>=4.6.3)", "matplotlib (>=3.6.1)", "numba (>=0.53.1)", "numexpr (>=2.7.3)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pandas-gbq (>=0.15.0)", "psycopg2 (>=2.8.6)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "python-snappy (>=0.6.0)", "pyxlsb (>=1.0.8)", "qtpy (>=2.2.0)", "s3fs (>=2021.08.0)", "scipy (>=1.7.1)", "tables (>=3.6.1)", "tabulate (>=0.8.9)", "xarray (>=0.21.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)", "zstandard (>=0.15.2)"] -aws = ["s3fs (>=2021.08.0)"] -clipboard = ["PyQt5 (>=5.15.1)", "qtpy (>=2.2.0)"] -compression = ["brotlipy (>=0.7.0)", "python-snappy (>=0.6.0)", "zstandard (>=0.15.2)"] -computation = ["scipy (>=1.7.1)", "xarray (>=0.21.0)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pyxlsb (>=1.0.8)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)"] +all = ["PyQt5 (>=5.15.6)", "SQLAlchemy (>=1.4.36)", "beautifulsoup4 (>=4.11.1)", "bottleneck (>=1.3.4)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=0.8.1)", "fsspec (>=2022.05.0)", "gcsfs (>=2022.05.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.8.0)", "matplotlib (>=3.6.1)", "numba (>=0.55.2)", "numexpr (>=2.8.0)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pandas-gbq (>=0.17.5)", "psycopg2 (>=2.9.3)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.5)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "pyxlsb (>=1.0.9)", "qtpy (>=2.2.0)", "s3fs (>=2022.05.0)", "scipy (>=1.8.1)", "tables (>=3.7.0)", "tabulate (>=0.8.10)", "xarray (>=2022.03.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)", "zstandard (>=0.17.0)"] +aws = ["s3fs (>=2022.05.0)"] +clipboard = ["PyQt5 (>=5.15.6)", "qtpy (>=2.2.0)"] +compression = ["zstandard (>=0.17.0)"] +computation = ["scipy (>=1.8.1)", "xarray (>=2022.03.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pyxlsb (>=1.0.9)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)"] feather = ["pyarrow (>=7.0.0)"] -fss = ["fsspec (>=2021.07.0)"] -gcp = ["gcsfs (>=2021.07.0)", "pandas-gbq (>=0.15.0)"] -hdf5 = ["tables (>=3.6.1)"] -html = ["beautifulsoup4 (>=4.9.3)", "html5lib (>=1.1)", "lxml (>=4.6.3)"] -mysql = ["SQLAlchemy (>=1.4.16)", "pymysql (>=1.0.2)"] -output-formatting = ["jinja2 (>=3.0.0)", "tabulate (>=0.8.9)"] +fss = ["fsspec (>=2022.05.0)"] +gcp = ["gcsfs (>=2022.05.0)", "pandas-gbq (>=0.17.5)"] +hdf5 = ["tables (>=3.7.0)"] +html = ["beautifulsoup4 (>=4.11.1)", "html5lib (>=1.1)", "lxml (>=4.8.0)"] +mysql = ["SQLAlchemy (>=1.4.36)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.8.10)"] parquet = ["pyarrow (>=7.0.0)"] -performance = ["bottleneck (>=1.3.2)", "numba (>=0.53.1)", "numexpr (>=2.7.1)"] +performance = ["bottleneck (>=1.3.4)", "numba (>=0.55.2)", "numexpr (>=2.8.0)"] plot = ["matplotlib (>=3.6.1)"] -postgresql = ["SQLAlchemy (>=1.4.16)", "psycopg2 (>=2.8.6)"] -spss = ["pyreadstat (>=1.1.2)"] -sql-other = ["SQLAlchemy (>=1.4.16)"] -test = ["hypothesis (>=6.34.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] -xml = ["lxml (>=4.6.3)"] +postgresql = ["SQLAlchemy (>=1.4.36)", "psycopg2 (>=2.9.3)"] +spss = ["pyreadstat (>=1.1.5)"] +sql-other = ["SQLAlchemy (>=1.4.36)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.8.0)"] [[package]] name = "pluggy" -version = "1.2.0" +version = "1.3.0" description = "plugin and hook calling mechanisms for python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, - {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, + {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, + {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, ] [package.extras] @@ -721,24 +846,24 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "protobuf" -version = "4.24.0" +version = "4.24.4" description = "" optional = false python-versions = ">=3.7" files = [ - {file = "protobuf-4.24.0-cp310-abi3-win32.whl", hash = "sha256:81cb9c4621d2abfe181154354f63af1c41b00a4882fb230b4425cbaed65e8f52"}, - {file = "protobuf-4.24.0-cp310-abi3-win_amd64.whl", hash = "sha256:6c817cf4a26334625a1904b38523d1b343ff8b637d75d2c8790189a4064e51c3"}, - {file = "protobuf-4.24.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:ae97b5de10f25b7a443b40427033e545a32b0e9dda17bcd8330d70033379b3e5"}, - {file = "protobuf-4.24.0-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:567fe6b0647494845d0849e3d5b260bfdd75692bf452cdc9cb660d12457c055d"}, - {file = "protobuf-4.24.0-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:a6b1ca92ccabfd9903c0c7dde8876221dc7d8d87ad5c42e095cc11b15d3569c7"}, - {file = "protobuf-4.24.0-cp37-cp37m-win32.whl", hash = "sha256:a38400a692fd0c6944c3c58837d112f135eb1ed6cdad5ca6c5763336e74f1a04"}, - {file = "protobuf-4.24.0-cp37-cp37m-win_amd64.whl", hash = "sha256:5ab19ee50037d4b663c02218a811a5e1e7bb30940c79aac385b96e7a4f9daa61"}, - {file = "protobuf-4.24.0-cp38-cp38-win32.whl", hash = "sha256:e8834ef0b4c88666ebb7c7ec18045aa0f4325481d724daa624a4cf9f28134653"}, - {file = "protobuf-4.24.0-cp38-cp38-win_amd64.whl", hash = "sha256:8bb52a2be32db82ddc623aefcedfe1e0eb51da60e18fcc908fb8885c81d72109"}, - {file = "protobuf-4.24.0-cp39-cp39-win32.whl", hash = "sha256:ae7a1835721086013de193311df858bc12cd247abe4ef9710b715d930b95b33e"}, - {file = "protobuf-4.24.0-cp39-cp39-win_amd64.whl", hash = "sha256:44825e963008f8ea0d26c51911c30d3e82e122997c3c4568fd0385dd7bacaedf"}, - {file = "protobuf-4.24.0-py3-none-any.whl", hash = "sha256:82e6e9ebdd15b8200e8423676eab38b774624d6a1ad696a60d86a2ac93f18201"}, - {file = "protobuf-4.24.0.tar.gz", hash = "sha256:5d0ceb9de6e08311832169e601d1fc71bd8e8c779f3ee38a97a78554945ecb85"}, + {file = "protobuf-4.24.4-cp310-abi3-win32.whl", hash = "sha256:ec9912d5cb6714a5710e28e592ee1093d68c5ebfeda61983b3f40331da0b1ebb"}, + {file = "protobuf-4.24.4-cp310-abi3-win_amd64.whl", hash = "sha256:1badab72aa8a3a2b812eacfede5020472e16c6b2212d737cefd685884c191085"}, + {file = "protobuf-4.24.4-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:8e61a27f362369c2f33248a0ff6896c20dcd47b5d48239cb9720134bef6082e4"}, + {file = "protobuf-4.24.4-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:bffa46ad9612e6779d0e51ae586fde768339b791a50610d85eb162daeb23661e"}, + {file = "protobuf-4.24.4-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:b493cb590960ff863743b9ff1452c413c2ee12b782f48beca77c8da3e2ffe9d9"}, + {file = "protobuf-4.24.4-cp37-cp37m-win32.whl", hash = "sha256:dbbed8a56e56cee8d9d522ce844a1379a72a70f453bde6243e3c86c30c2a3d46"}, + {file = "protobuf-4.24.4-cp37-cp37m-win_amd64.whl", hash = "sha256:6b7d2e1c753715dcfe9d284a25a52d67818dd43c4932574307daf836f0071e37"}, + {file = "protobuf-4.24.4-cp38-cp38-win32.whl", hash = "sha256:02212557a76cd99574775a81fefeba8738d0f668d6abd0c6b1d3adcc75503dbe"}, + {file = "protobuf-4.24.4-cp38-cp38-win_amd64.whl", hash = "sha256:2fa3886dfaae6b4c5ed2730d3bf47c7a38a72b3a1f0acb4d4caf68e6874b947b"}, + {file = "protobuf-4.24.4-cp39-cp39-win32.whl", hash = "sha256:b77272f3e28bb416e2071186cb39efd4abbf696d682cbb5dc731308ad37fa6dd"}, + {file = "protobuf-4.24.4-cp39-cp39-win_amd64.whl", hash = "sha256:9fee5e8aa20ef1b84123bb9232b3f4a5114d9897ed89b4b8142d81924e05d79b"}, + {file = "protobuf-4.24.4-py3-none-any.whl", hash = "sha256:80797ce7424f8c8d2f2547e2d42bfbb6c08230ce5832d6c099a37335c9c90a92"}, + {file = "protobuf-4.24.4.tar.gz", hash = "sha256:5a70731910cd9104762161719c3d883c960151eea077134458503723b60e3667"}, ] [[package]] @@ -779,47 +904,47 @@ files = [ [[package]] name = "pydantic" -version = "1.10.12" +version = "1.10.13" description = "Data validation and settings management using python type hints" optional = false python-versions = ">=3.7" files = [ - {file = "pydantic-1.10.12-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a1fcb59f2f355ec350073af41d927bf83a63b50e640f4dbaa01053a28b7a7718"}, - {file = "pydantic-1.10.12-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b7ccf02d7eb340b216ec33e53a3a629856afe1c6e0ef91d84a4e6f2fb2ca70fe"}, - {file = "pydantic-1.10.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8fb2aa3ab3728d950bcc885a2e9eff6c8fc40bc0b7bb434e555c215491bcf48b"}, - {file = "pydantic-1.10.12-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:771735dc43cf8383959dc9b90aa281f0b6092321ca98677c5fb6125a6f56d58d"}, - {file = "pydantic-1.10.12-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ca48477862372ac3770969b9d75f1bf66131d386dba79506c46d75e6b48c1e09"}, - {file = "pydantic-1.10.12-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a5e7add47a5b5a40c49b3036d464e3c7802f8ae0d1e66035ea16aa5b7a3923ed"}, - {file = "pydantic-1.10.12-cp310-cp310-win_amd64.whl", hash = "sha256:e4129b528c6baa99a429f97ce733fff478ec955513630e61b49804b6cf9b224a"}, - {file = "pydantic-1.10.12-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b0d191db0f92dfcb1dec210ca244fdae5cbe918c6050b342d619c09d31eea0cc"}, - {file = "pydantic-1.10.12-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:795e34e6cc065f8f498c89b894a3c6da294a936ee71e644e4bd44de048af1405"}, - {file = "pydantic-1.10.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:69328e15cfda2c392da4e713443c7dbffa1505bc9d566e71e55abe14c97ddc62"}, - {file = "pydantic-1.10.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2031de0967c279df0d8a1c72b4ffc411ecd06bac607a212892757db7462fc494"}, - {file = "pydantic-1.10.12-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ba5b2e6fe6ca2b7e013398bc7d7b170e21cce322d266ffcd57cca313e54fb246"}, - {file = "pydantic-1.10.12-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2a7bac939fa326db1ab741c9d7f44c565a1d1e80908b3797f7f81a4f86bc8d33"}, - {file = "pydantic-1.10.12-cp311-cp311-win_amd64.whl", hash = "sha256:87afda5539d5140cb8ba9e8b8c8865cb5b1463924d38490d73d3ccfd80896b3f"}, - {file = "pydantic-1.10.12-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:549a8e3d81df0a85226963611950b12d2d334f214436a19537b2efed61b7639a"}, - {file = "pydantic-1.10.12-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598da88dfa127b666852bef6d0d796573a8cf5009ffd62104094a4fe39599565"}, - {file = "pydantic-1.10.12-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba5c4a8552bff16c61882db58544116d021d0b31ee7c66958d14cf386a5b5350"}, - {file = "pydantic-1.10.12-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c79e6a11a07da7374f46970410b41d5e266f7f38f6a17a9c4823db80dadf4303"}, - {file = "pydantic-1.10.12-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab26038b8375581dc832a63c948f261ae0aa21f1d34c1293469f135fa92972a5"}, - {file = "pydantic-1.10.12-cp37-cp37m-win_amd64.whl", hash = "sha256:e0a16d274b588767602b7646fa05af2782576a6cf1022f4ba74cbb4db66f6ca8"}, - {file = "pydantic-1.10.12-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6a9dfa722316f4acf4460afdf5d41d5246a80e249c7ff475c43a3a1e9d75cf62"}, - {file = "pydantic-1.10.12-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a73f489aebd0c2121ed974054cb2759af8a9f747de120acd2c3394cf84176ccb"}, - {file = "pydantic-1.10.12-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b30bcb8cbfccfcf02acb8f1a261143fab622831d9c0989707e0e659f77a18e0"}, - {file = "pydantic-1.10.12-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2fcfb5296d7877af406ba1547dfde9943b1256d8928732267e2653c26938cd9c"}, - {file = "pydantic-1.10.12-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:2f9a6fab5f82ada41d56b0602606a5506aab165ca54e52bc4545028382ef1c5d"}, - {file = "pydantic-1.10.12-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:dea7adcc33d5d105896401a1f37d56b47d443a2b2605ff8a969a0ed5543f7e33"}, - {file = "pydantic-1.10.12-cp38-cp38-win_amd64.whl", hash = "sha256:1eb2085c13bce1612da8537b2d90f549c8cbb05c67e8f22854e201bde5d98a47"}, - {file = "pydantic-1.10.12-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ef6c96b2baa2100ec91a4b428f80d8f28a3c9e53568219b6c298c1125572ebc6"}, - {file = "pydantic-1.10.12-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6c076be61cd0177a8433c0adcb03475baf4ee91edf5a4e550161ad57fc90f523"}, - {file = "pydantic-1.10.12-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d5a58feb9a39f481eda4d5ca220aa8b9d4f21a41274760b9bc66bfd72595b86"}, - {file = "pydantic-1.10.12-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5f805d2d5d0a41633651a73fa4ecdd0b3d7a49de4ec3fadf062fe16501ddbf1"}, - {file = "pydantic-1.10.12-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:1289c180abd4bd4555bb927c42ee42abc3aee02b0fb2d1223fb7c6e5bef87dbe"}, - {file = "pydantic-1.10.12-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5d1197e462e0364906cbc19681605cb7c036f2475c899b6f296104ad42b9f5fb"}, - {file = "pydantic-1.10.12-cp39-cp39-win_amd64.whl", hash = "sha256:fdbdd1d630195689f325c9ef1a12900524dceb503b00a987663ff4f58669b93d"}, - {file = "pydantic-1.10.12-py3-none-any.whl", hash = "sha256:b749a43aa51e32839c9d71dc67eb1e4221bb04af1033a32e3923d46f9effa942"}, - {file = "pydantic-1.10.12.tar.gz", hash = "sha256:0fe8a415cea8f340e7a9af9c54fc71a649b43e8ca3cc732986116b3cb135d303"}, + {file = "pydantic-1.10.13-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:efff03cc7a4f29d9009d1c96ceb1e7a70a65cfe86e89d34e4a5f2ab1e5693737"}, + {file = "pydantic-1.10.13-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3ecea2b9d80e5333303eeb77e180b90e95eea8f765d08c3d278cd56b00345d01"}, + {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1740068fd8e2ef6eb27a20e5651df000978edce6da6803c2bef0bc74540f9548"}, + {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84bafe2e60b5e78bc64a2941b4c071a4b7404c5c907f5f5a99b0139781e69ed8"}, + {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bc0898c12f8e9c97f6cd44c0ed70d55749eaf783716896960b4ecce2edfd2d69"}, + {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:654db58ae399fe6434e55325a2c3e959836bd17a6f6a0b6ca8107ea0571d2e17"}, + {file = "pydantic-1.10.13-cp310-cp310-win_amd64.whl", hash = "sha256:75ac15385a3534d887a99c713aa3da88a30fbd6204a5cd0dc4dab3d770b9bd2f"}, + {file = "pydantic-1.10.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c553f6a156deb868ba38a23cf0df886c63492e9257f60a79c0fd8e7173537653"}, + {file = "pydantic-1.10.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5e08865bc6464df8c7d61439ef4439829e3ab62ab1669cddea8dd00cd74b9ffe"}, + {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e31647d85a2013d926ce60b84f9dd5300d44535a9941fe825dc349ae1f760df9"}, + {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:210ce042e8f6f7c01168b2d84d4c9eb2b009fe7bf572c2266e235edf14bacd80"}, + {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8ae5dd6b721459bfa30805f4c25880e0dd78fc5b5879f9f7a692196ddcb5a580"}, + {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f8e81fc5fb17dae698f52bdd1c4f18b6ca674d7068242b2aff075f588301bbb0"}, + {file = "pydantic-1.10.13-cp311-cp311-win_amd64.whl", hash = "sha256:61d9dce220447fb74f45e73d7ff3b530e25db30192ad8d425166d43c5deb6df0"}, + {file = "pydantic-1.10.13-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4b03e42ec20286f052490423682016fd80fda830d8e4119f8ab13ec7464c0132"}, + {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59ef915cac80275245824e9d771ee939133be38215555e9dc90c6cb148aaeb5"}, + {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a1f9f747851338933942db7af7b6ee8268568ef2ed86c4185c6ef4402e80ba8"}, + {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:97cce3ae7341f7620a0ba5ef6cf043975cd9d2b81f3aa5f4ea37928269bc1b87"}, + {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854223752ba81e3abf663d685f105c64150873cc6f5d0c01d3e3220bcff7d36f"}, + {file = "pydantic-1.10.13-cp37-cp37m-win_amd64.whl", hash = "sha256:b97c1fac8c49be29486df85968682b0afa77e1b809aff74b83081cc115e52f33"}, + {file = "pydantic-1.10.13-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c958d053453a1c4b1c2062b05cd42d9d5c8eb67537b8d5a7e3c3032943ecd261"}, + {file = "pydantic-1.10.13-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c5370a7edaac06daee3af1c8b1192e305bc102abcbf2a92374b5bc793818599"}, + {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d6f6e7305244bddb4414ba7094ce910560c907bdfa3501e9db1a7fd7eaea127"}, + {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3a3c792a58e1622667a2837512099eac62490cdfd63bd407993aaf200a4cf1f"}, + {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c636925f38b8db208e09d344c7aa4f29a86bb9947495dd6b6d376ad10334fb78"}, + {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:678bcf5591b63cc917100dc50ab6caebe597ac67e8c9ccb75e698f66038ea953"}, + {file = "pydantic-1.10.13-cp38-cp38-win_amd64.whl", hash = "sha256:6cf25c1a65c27923a17b3da28a0bdb99f62ee04230c931d83e888012851f4e7f"}, + {file = "pydantic-1.10.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8ef467901d7a41fa0ca6db9ae3ec0021e3f657ce2c208e98cd511f3161c762c6"}, + {file = "pydantic-1.10.13-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:968ac42970f57b8344ee08837b62f6ee6f53c33f603547a55571c954a4225691"}, + {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9849f031cf8a2f0a928fe885e5a04b08006d6d41876b8bbd2fc68a18f9f2e3fd"}, + {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:56e3ff861c3b9c6857579de282ce8baabf443f42ffba355bf070770ed63e11e1"}, + {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f00790179497767aae6bcdc36355792c79e7bbb20b145ff449700eb076c5f96"}, + {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:75b297827b59bc229cac1a23a2f7a4ac0031068e5be0ce385be1462e7e17a35d"}, + {file = "pydantic-1.10.13-cp39-cp39-win_amd64.whl", hash = "sha256:e70ca129d2053fb8b728ee7d1af8e553a928d7e301a311094b8a0501adc8763d"}, + {file = "pydantic-1.10.13-py3-none-any.whl", hash = "sha256:b87326822e71bd5f313e7d3bfdc77ac3247035ac10b0c0618bd99dcf95b1e687"}, + {file = "pydantic-1.10.13.tar.gz", hash = "sha256:32c8b48dcd3b2ac4e78b0ba4af3a2c2eb6048cb75202f0ea7b34feb740efc340"}, ] [package.dependencies] @@ -925,13 +1050,13 @@ tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] [[package]] name = "pytest" -version = "7.4.0" +version = "7.4.2" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32"}, - {file = "pytest-7.4.0.tar.gz", hash = "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"}, + {file = "pytest-7.4.2-py3-none-any.whl", hash = "sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002"}, + {file = "pytest-7.4.2.tar.gz", hash = "sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069"}, ] [package.dependencies] @@ -978,13 +1103,13 @@ six = ">=1.5" [[package]] name = "pytz" -version = "2023.3" +version = "2023.3.post1" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" files = [ - {file = "pytz-2023.3-py2.py3-none-any.whl", hash = "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb"}, - {file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"}, + {file = "pytz-2023.3.post1-py2.py3-none-any.whl", hash = "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7"}, + {file = "pytz-2023.3.post1.tar.gz", hash = "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b"}, ] [[package]] @@ -1059,22 +1184,21 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "rich" -version = "11.2.0" +version = "13.6.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false -python-versions = ">=3.6.2,<4.0.0" +python-versions = ">=3.7.0" files = [ - {file = "rich-11.2.0-py3-none-any.whl", hash = "sha256:d5f49ad91fb343efcae45a2b2df04a9755e863e50413623ab8c9e74f05aee52b"}, - {file = "rich-11.2.0.tar.gz", hash = "sha256:1a6266a5738115017bb64a66c59c717e7aa047b3ae49a011ede4abdeffc6536e"}, + {file = "rich-13.6.0-py3-none-any.whl", hash = "sha256:2b38e2fe9ca72c9a00170a1a2d20c63c790d0e10ef1fe35eba76e1e7b1d7d245"}, + {file = "rich-13.6.0.tar.gz", hash = "sha256:5c14d22737e6d5084ef4771b62d5d4363165b403455a30a1c8ca39dc7b644bef"}, ] [package.dependencies] -colorama = ">=0.4.0,<0.5.0" -commonmark = ">=0.9.0,<0.10.0" -pygments = ">=2.6.0,<3.0.0" +markdown-it-py = ">=2.2.0" +pygments = ">=2.13.0,<3.0.0" [package.extras] -jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] +jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "rsa" @@ -1090,6 +1214,17 @@ files = [ [package.dependencies] pyasn1 = ">=0.1.3" +[[package]] +name = "simpleeval" +version = "0.9.13" +description = "A simple, safe single expression evaluator library." +optional = false +python-versions = "*" +files = [ + {file = "simpleeval-0.9.13-py2.py3-none-any.whl", hash = "sha256:22a2701a5006e4188d125d34accf2405c2c37c93f6b346f2484b6422415ae54a"}, + {file = "simpleeval-0.9.13.tar.gz", hash = "sha256:4a30f9cc01825fe4c719c785e3762623e350c4840d5e6855c2a8496baaa65fac"}, +] + [[package]] name = "six" version = "1.16.0" @@ -1103,13 +1238,13 @@ files = [ [[package]] name = "smmap" -version = "5.0.0" +version = "5.0.1" description = "A pure Python implementation of a sliding window memory map manager" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "smmap-5.0.0-py3-none-any.whl", hash = "sha256:2aba19d6a040e78d8b09de5c57e96207b09ed71d8e55ce0959eeee6c8e190d94"}, - {file = "smmap-5.0.0.tar.gz", hash = "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936"}, + {file = "smmap-5.0.1-py3-none-any.whl", hash = "sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da"}, + {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, ] [[package]] @@ -1125,13 +1260,13 @@ files = [ [[package]] name = "typing-extensions" -version = "4.7.1" -description = "Backported and Experimental Type Hints for Python 3.7+" +version = "4.8.0" +description = "Backported and Experimental Type Hints for Python 3.8+" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, - {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, + {file = "typing_extensions-4.8.0-py3-none-any.whl", hash = "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0"}, + {file = "typing_extensions-4.8.0.tar.gz", hash = "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"}, ] [[package]] @@ -1147,19 +1282,20 @@ files = [ [[package]] name = "urllib3" -version = "1.26.16" +version = "2.0.6" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +python-versions = ">=3.7" files = [ - {file = "urllib3-1.26.16-py2.py3-none-any.whl", hash = "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f"}, - {file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"}, + {file = "urllib3-2.0.6-py3-none-any.whl", hash = "sha256:7a7c7003b000adf9e7ca2a377c9688bbc54ed41b985789ed576570342a375cd2"}, + {file = "urllib3-2.0.6.tar.gz", hash = "sha256:b19e1a85d206b56d7df1d5e683df4a7725252a964e3993648dd0fb5a1c157564"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] [[package]] name = "wrapt" @@ -1248,4 +1384,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "e601115553c94d23c6d25303190d48216b7a6ccc9d93fd3dec2369ffc5094a5b" +content-hash = "3d508b33cc7418c89765c36e163354db9f9f47c08c002a977725b27cc76e5c62" diff --git a/airbyte-ci/connectors/connector_ops/pyproject.toml b/airbyte-ci/connectors/connector_ops/pyproject.toml index 3faca6122351..f4e5460b2493 100644 --- a/airbyte-ci/connectors/connector_ops/pyproject.toml +++ b/airbyte-ci/connectors/connector_ops/pyproject.toml @@ -1,10 +1,10 @@ [build-system] -requires = ["poetry-core>=1.0.0"] +requires = ["poetry-core>=1.1.0"] build-backend = "poetry.core.masonry.api" [tool.poetry] name = "connector_ops" -version = "0.2.2" +version = "0.3.3" description = "Packaged maintained by the connector operations team to perform CI for connectors" authors = ["Airbyte "] @@ -16,11 +16,12 @@ PyYAML = "^6.0" GitPython = "^3.1.29" pydantic = "^1.9" PyGithub = "^1.58.0" -rich = "^11.0.1" +rich = "^13.0.0" pydash = "^7.0.4" google-cloud-storage = "^2.8.0" ci-credentials = {path = "../ci_credentials"} pandas = "^2.0.3" +simpleeval = "^0.9.13" [tool.poetry.group.test.dependencies] pytest = "^7.4.0" @@ -29,7 +30,7 @@ freezegun = "^1.1.0" [tool.poetry.scripts] check-test-strictness-level = "connector_ops.acceptance_test_config_checks:check_test_strictness_level" -write-review-requirements-file = "connector_ops.acceptance_test_config_checks:write_review_requirements_file" -print-mandatory-reviewers = "connector_ops.acceptance_test_config_checks:print_mandatory_reviewers" +write-review-requirements-file = "connector_ops.required_reviewer_checks:write_review_requirements_file" +print-mandatory-reviewers = "connector_ops.required_reviewer_checks:print_mandatory_reviewers" allowed-hosts-checks = "connector_ops.allowed_hosts_checks:check_allowed_hosts" run-qa-checks = "connector_ops.qa_checks:run_qa_checks" diff --git a/airbyte-ci/connectors/connector_ops/tests/conftest.py b/airbyte-ci/connectors/connector_ops/tests/conftest.py index fbc90e8a1f59..78aad3d1c104 100644 --- a/airbyte-ci/connectors/connector_ops/tests/conftest.py +++ b/airbyte-ci/connectors/connector_ops/tests/conftest.py @@ -3,6 +3,7 @@ # +import os from datetime import datetime import pandas as pd @@ -53,3 +54,12 @@ def dummy_qa_report() -> pd.DataFrame: } ] ) + + +@pytest.fixture(autouse=True) +def set_working_dir_to_repo_root(monkeypatch): + """Set working directory to the root of the repository. + + HACK: This is a workaround for the fact that these tests are not run from the root of the repository. + """ + monkeypatch.chdir(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..")) diff --git a/airbyte-ci/connectors/connector_ops/tests/test_acceptance_test_config_checks.py b/airbyte-ci/connectors/connector_ops/tests/test_acceptance_test_config_checks.py deleted file mode 100644 index c100cd3fb483..000000000000 --- a/airbyte-ci/connectors/connector_ops/tests/test_acceptance_test_config_checks.py +++ /dev/null @@ -1,177 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import shutil -from typing import List - -import git -import pytest -import yaml -from connector_ops import acceptance_test_config_checks - - -@pytest.fixture -def mock_diffed_branched(mocker): - airbyte_repo = git.Repo(search_parent_directories=True) - mocker.patch.object(acceptance_test_config_checks.utils, "DIFFED_BRANCH", airbyte_repo.active_branch) - return airbyte_repo.active_branch - - -@pytest.fixture -def pokeapi_acceptance_test_config_path(): - return "airbyte-integrations/connectors/source-pokeapi/acceptance-test-config.yml" - - -@pytest.fixture -def ga_connector_file(): - return "airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml" - - -@pytest.fixture -def not_ga_backward_compatibility_change_expected_team(tmp_path, pokeapi_acceptance_test_config_path) -> List: - expected_teams = [{"any-of": list(acceptance_test_config_checks.BACKWARD_COMPATIBILITY_REVIEWERS)}] - backup_path = tmp_path / "backup_poke_acceptance" - shutil.copyfile(pokeapi_acceptance_test_config_path, backup_path) - with open(pokeapi_acceptance_test_config_path, "a") as acceptance_test_config_file: - acceptance_test_config_file.write("disable_for_version: 0.0.0") - yield expected_teams - shutil.copyfile(backup_path, pokeapi_acceptance_test_config_path) - - -@pytest.fixture -def not_ga_test_strictness_level_change_expected_team(tmp_path, pokeapi_acceptance_test_config_path) -> List: - expected_teams = [{"any-of": list(acceptance_test_config_checks.TEST_STRICTNESS_LEVEL_REVIEWERS)}] - backup_path = tmp_path / "non_ga_acceptance_test_config.backup" - shutil.copyfile(pokeapi_acceptance_test_config_path, backup_path) - with open(pokeapi_acceptance_test_config_path, "a") as acceptance_test_config_file: - acceptance_test_config_file.write("test_strictness_level: foo") - yield expected_teams - shutil.copyfile(backup_path, pokeapi_acceptance_test_config_path) - - -@pytest.fixture -def not_ga_bypass_reason_file_change_expected_team(tmp_path, pokeapi_acceptance_test_config_path): - expected_teams = [] - backup_path = tmp_path / "non_ga_acceptance_test_config.backup" - shutil.copyfile(pokeapi_acceptance_test_config_path, backup_path) - with open(pokeapi_acceptance_test_config_path, "a") as acceptance_test_config_file: - acceptance_test_config_file.write("bypass_reason:") - yield expected_teams - shutil.copyfile(backup_path, pokeapi_acceptance_test_config_path) - - -@pytest.fixture -def not_ga_not_tracked_change_expected_team(tmp_path, pokeapi_acceptance_test_config_path): - expected_teams = [] - backup_path = tmp_path / "non_ga_acceptance_test_config.backup" - shutil.copyfile(pokeapi_acceptance_test_config_path, backup_path) - with open(pokeapi_acceptance_test_config_path, "a") as acceptance_test_config_file: - acceptance_test_config_file.write("not_tracked") - yield expected_teams - shutil.copyfile(backup_path, pokeapi_acceptance_test_config_path) - - -@pytest.fixture -def ga_connector_file_change_expected_team(tmp_path, ga_connector_file): - expected_teams = list(acceptance_test_config_checks.GA_CONNECTOR_REVIEWERS) - backup_path = tmp_path / "ga_acceptance_test_config.backup" - shutil.copyfile(ga_connector_file, backup_path) - with open(ga_connector_file, "a") as ga_acceptance_test_config_file: - ga_acceptance_test_config_file.write("foobar") - yield expected_teams - shutil.copyfile(backup_path, ga_connector_file) - - -@pytest.fixture -def ga_connector_backward_compatibility_file_change_expected_team(tmp_path, ga_connector_file): - expected_teams = [{"any-of": list(acceptance_test_config_checks.BACKWARD_COMPATIBILITY_REVIEWERS)}] - backup_path = tmp_path / "ga_acceptance_test_config.backup" - shutil.copyfile(ga_connector_file, backup_path) - with open(ga_connector_file, "a") as ga_acceptance_test_config_file: - ga_acceptance_test_config_file.write("disable_for_version: 0.0.0") - yield expected_teams - shutil.copyfile(backup_path, ga_connector_file) - - -@pytest.fixture -def ga_connector_bypass_reason_file_change_expected_team(tmp_path, ga_connector_file): - expected_teams = [{"any-of": list(acceptance_test_config_checks.GA_BYPASS_REASON_REVIEWERS)}] - backup_path = tmp_path / "ga_acceptance_test_config.backup" - shutil.copyfile(ga_connector_file, backup_path) - with open(ga_connector_file, "a") as ga_acceptance_test_config_file: - ga_acceptance_test_config_file.write("bypass_reason:") - yield expected_teams - shutil.copyfile(backup_path, ga_connector_file) - - -@pytest.fixture -def ga_connector_test_strictness_level_file_change_expected_team(tmp_path, ga_connector_file): - expected_teams = [{"any-of": list(acceptance_test_config_checks.TEST_STRICTNESS_LEVEL_REVIEWERS)}] - backup_path = tmp_path / "ga_acceptance_test_config.backup" - shutil.copyfile(ga_connector_file, backup_path) - with open(ga_connector_file, "a") as ga_acceptance_test_config_file: - ga_acceptance_test_config_file.write("test_strictness_level: 0.0.0") - yield expected_teams - shutil.copyfile(backup_path, ga_connector_file) - - -def verify_no_requirements_file_was_generated(captured: str): - assert captured.out.split("\n")[0].split("=")[-1] == "false" - - -def verify_requirements_file_was_generated(captured: str): - assert captured.out.split("\n")[0].split("=")[-1] == "true" - - -def verify_review_requirements_file_contains_expected_teams(requirements_file_path: str, expected_teams: List): - with open(requirements_file_path, "r") as requirements_file: - requirements = yaml.safe_load(requirements_file) - assert requirements[0]["teams"] == expected_teams - - -def check_review_requirements_file(capsys, expected_teams: List): - acceptance_test_config_checks.write_review_requirements_file() - captured = capsys.readouterr() - if not expected_teams: - verify_no_requirements_file_was_generated(captured) - else: - verify_requirements_file_was_generated(captured) - requirements_file_path = acceptance_test_config_checks.REVIEW_REQUIREMENTS_FILE_PATH - verify_review_requirements_file_contains_expected_teams(requirements_file_path, expected_teams) - - -def test_find_mandatory_reviewers_backward_compatibility(mock_diffed_branched, capsys, not_ga_backward_compatibility_change_expected_team): - check_review_requirements_file(capsys, not_ga_backward_compatibility_change_expected_team) - - -def test_find_mandatory_reviewers_test_strictness_level(mock_diffed_branched, capsys, not_ga_test_strictness_level_change_expected_team): - check_review_requirements_file(capsys, not_ga_test_strictness_level_change_expected_team) - - -def test_find_mandatory_reviewers_not_ga_bypass_reason(mock_diffed_branched, capsys, not_ga_bypass_reason_file_change_expected_team): - check_review_requirements_file(capsys, not_ga_bypass_reason_file_change_expected_team) - - -def test_find_mandatory_reviewers_ga(mock_diffed_branched, capsys, ga_connector_file_change_expected_team): - check_review_requirements_file(capsys, ga_connector_file_change_expected_team) - - -def test_find_mandatory_reviewers_ga_backward_compatibility( - mock_diffed_branched, capsys, ga_connector_backward_compatibility_file_change_expected_team -): - check_review_requirements_file(capsys, ga_connector_backward_compatibility_file_change_expected_team) - - -def test_find_mandatory_reviewers_ga_bypass_reason(mock_diffed_branched, capsys, ga_connector_bypass_reason_file_change_expected_team): - check_review_requirements_file(capsys, ga_connector_bypass_reason_file_change_expected_team) - - -def test_find_mandatory_reviewers_ga_test_strictness_level( - mock_diffed_branched, capsys, ga_connector_test_strictness_level_file_change_expected_team -): - check_review_requirements_file(capsys, ga_connector_test_strictness_level_file_change_expected_team) - - -def test_find_mandatory_reviewers_no_tracked_changed(mock_diffed_branched, capsys, not_ga_not_tracked_change_expected_team): - check_review_requirements_file(capsys, not_ga_not_tracked_change_expected_team) diff --git a/airbyte-ci/connectors/connector_ops/tests/test_qa_checks.py b/airbyte-ci/connectors/connector_ops/tests/test_qa_checks.py index a7860b65505a..f87bdbba384f 100644 --- a/airbyte-ci/connectors/connector_ops/tests/test_qa_checks.py +++ b/airbyte-ci/connectors/connector_ops/tests/test_qa_checks.py @@ -80,7 +80,7 @@ def test_run_qa_checks_success(capsys, mocker, user_input, expect_qa_checks_to_r mocker.patch.object(qa_checks, "Connector") mock_qa_check = mocker.Mock(return_value=True, __name__="mock_qa_check") if expect_qa_checks_to_run: - mocker.patch.object(qa_checks, "QA_CHECKS", [mock_qa_check]) + mocker.patch.object(qa_checks, "get_qa_checks_to_run", return_value=[mock_qa_check]) with pytest.raises(SystemExit) as wrapped_error: qa_checks.run_qa_checks() assert wrapped_error.value.code == 0 @@ -101,7 +101,7 @@ def test_run_qa_checks_error(capsys, mocker): mocker.patch.object(qa_checks.sys, "argv", ["", "source-faker"]) mocker.patch.object(qa_checks, "Connector") mock_qa_check = mocker.Mock(return_value=False, __name__="mock_qa_check") - mocker.patch.object(qa_checks, "QA_CHECKS", [mock_qa_check]) + mocker.patch.object(qa_checks, "DEFAULT_QA_CHECKS", (mock_qa_check,)) with pytest.raises(SystemExit) as wrapped_error: qa_checks.run_qa_checks() assert wrapped_error.value.code == 1 @@ -114,6 +114,7 @@ def test_run_qa_checks_error(capsys, mocker): "file_name, file_line, expected_in_stdout", [ ("file_with_http_url.foo", "http://foo.bar", True), + ("file_with_http_url_and_ignore_comment.foo", "http://foo.bar # ignore-https-check", False), ("file_without_https_url.foo", "", False), ("file_with_https_url.foo", "https://airbyte.com", False), ("file_with_http_url_and_ignored.foo", "http://localhost http://airbyte.com", True), @@ -185,8 +186,8 @@ def test_is_comment(tmp_path, file_name, line, expect_is_comment): def test_check_missing_migration_guide(mocker, tmp_path, capsys): connector = qa_checks.Connector("source-foobar") - mock_documentation_directory_path = Path(tmp_path) - mocker.patch.object(qa_checks.Connector, "documentation_directory", mock_documentation_directory_path) + local_connector_documentation_directory = Path(tmp_path) + mocker.patch.object(qa_checks.Connector, "local_connector_documentation_directory", local_connector_documentation_directory) mock_metadata_dict = { "documentationUrl": tmp_path, @@ -201,9 +202,14 @@ def test_check_missing_migration_guide(mocker, tmp_path, capsys): } mocker.patch.object(qa_checks.Connector, "metadata", mock_metadata_dict) - assert qa_checks.check_migration_guide(connector) == False + assert qa_checks.check_migration_guide(connector) is False stdout, _ = capsys.readouterr() - assert "Migration guide file is missing for foobar. Please create a foobar-migrations.md file in the docs folder" in stdout + # f"Migration guide file is missing for {connector.name}. Please create a migration guide at {connector.migration_guide_file_path}" + + assert ( + f"Migration guide file is missing for foobar. Please create a migration guide at {local_connector_documentation_directory}/foobar-migrations.md" + in stdout + ) @pytest.mark.parametrize( @@ -218,9 +224,9 @@ def test_check_missing_migration_guide(mocker, tmp_path, capsys): ) def test_check_invalid_migration_guides(mocker, tmp_path, capsys, test_file, expected_stdout): connector = qa_checks.Connector("source-foobar") - mock_documentation_directory_path = Path(tmp_path) - mocker.patch.object(qa_checks.Connector, "documentation_directory", mock_documentation_directory_path) - mock_migration_file = mock_documentation_directory_path / f"{connector.name}-migrations.md" + local_connector_documentation_directory = Path(tmp_path) + mocker.patch.object(qa_checks.Connector, "local_connector_documentation_directory", local_connector_documentation_directory) + mock_migration_file = local_connector_documentation_directory / f"{connector.name}-migrations.md" mock_breaking_change_value = { "upgradeDeadline": "2021-01-01", @@ -241,6 +247,28 @@ def test_check_invalid_migration_guides(mocker, tmp_path, capsys, test_file, exp mocker.patch.object(qa_checks.Connector, "metadata", mock_metadata_dict) - assert qa_checks.check_migration_guide(connector) == False + assert qa_checks.check_migration_guide(connector) is False stdout, _ = capsys.readouterr() assert expected_stdout in stdout + + +def test_get_qa_checks_to_run(mocker): + mocker.patch.object(utils.Connector, "has_dockerfile", False) + connector = utils.Connector("source-faker") + + assert ( + qa_checks.get_qa_checks_to_run(connector) == qa_checks.DEFAULT_QA_CHECKS + ), "A connector without a Dockerfile should run the default set of QA checks" + mocker.patch.object(utils.Connector, "has_dockerfile", True) + connector = utils.Connector("source-faker") + assert qa_checks.get_qa_checks_to_run(connector) == qa_checks.DEFAULT_QA_CHECKS + ( + qa_checks.check_metadata_version_matches_dockerfile_label, + ), "A connector with a Dockerfile should run the default set of QA checks plus check_metadata_version_matches_dockerfile_label" + + +def test_check_metadata_version_matches_dockerfile_label_without_dockerfile(mocker): + mocker.patch.object(utils.Connector, "has_dockerfile", False) + connector_without_dockerfile = utils.Connector("source-faker") + assert ( + qa_checks.check_metadata_version_matches_dockerfile_label(connector_without_dockerfile) is False + ), "A connector without a Dockerfile should fail check_metadata_version_matches_dockerfile_label" diff --git a/airbyte-ci/connectors/connector_ops/tests/test_required_reviewer_checks.py b/airbyte-ci/connectors/connector_ops/tests/test_required_reviewer_checks.py new file mode 100644 index 000000000000..42ac60aedf5b --- /dev/null +++ b/airbyte-ci/connectors/connector_ops/tests/test_required_reviewer_checks.py @@ -0,0 +1,207 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import shutil +from typing import List + +import git +import pytest +import yaml +from connector_ops import required_reviewer_checks + + +@pytest.fixture +def mock_diffed_branched(mocker): + airbyte_repo = git.Repo(search_parent_directories=True) + mocker.patch.object(required_reviewer_checks.utils, "DIFFED_BRANCH", airbyte_repo.active_branch) + return airbyte_repo.active_branch + + +@pytest.fixture +def pokeapi_acceptance_test_config_path(): + return "airbyte-integrations/connectors/source-pokeapi/acceptance-test-config.yml" + + +@pytest.fixture +def pokeapi_metadata_path(): + return "airbyte-integrations/connectors/source-pokeapi/metadata.yaml" + + +@pytest.fixture +def strategic_connector_file(): + return "airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml" + + +@pytest.fixture +def not_strategic_backward_compatibility_change_expected_team(tmp_path, pokeapi_acceptance_test_config_path) -> List: + expected_teams = list(required_reviewer_checks.BACKWARD_COMPATIBILITY_REVIEWERS) + backup_path = tmp_path / "backup_poke_acceptance" + shutil.copyfile(pokeapi_acceptance_test_config_path, backup_path) + with open(pokeapi_acceptance_test_config_path, "a") as acceptance_test_config_file: + acceptance_test_config_file.write("disable_for_version: 0.0.0") + yield expected_teams + shutil.copyfile(backup_path, pokeapi_acceptance_test_config_path) + + +@pytest.fixture +def not_strategic_test_strictness_level_change_expected_team(tmp_path, pokeapi_acceptance_test_config_path) -> List: + expected_teams = list(required_reviewer_checks.TEST_STRICTNESS_LEVEL_REVIEWERS) + backup_path = tmp_path / "non_strategic_acceptance_test_config.backup" + shutil.copyfile(pokeapi_acceptance_test_config_path, backup_path) + with open(pokeapi_acceptance_test_config_path, "a") as acceptance_test_config_file: + acceptance_test_config_file.write("test_strictness_level: foo") + yield expected_teams + shutil.copyfile(backup_path, pokeapi_acceptance_test_config_path) + + +@pytest.fixture +def not_strategic_bypass_reason_file_change_expected_team(tmp_path, pokeapi_acceptance_test_config_path): + expected_teams = [] + backup_path = tmp_path / "non_strategic_acceptance_test_config.backup" + shutil.copyfile(pokeapi_acceptance_test_config_path, backup_path) + with open(pokeapi_acceptance_test_config_path, "a") as acceptance_test_config_file: + acceptance_test_config_file.write("bypass_reason:") + yield expected_teams + shutil.copyfile(backup_path, pokeapi_acceptance_test_config_path) + + +@pytest.fixture +def not_strategic_not_tracked_change_expected_team(tmp_path, pokeapi_acceptance_test_config_path): + expected_teams = [] + backup_path = tmp_path / "non_strategic_acceptance_test_config.backup" + shutil.copyfile(pokeapi_acceptance_test_config_path, backup_path) + with open(pokeapi_acceptance_test_config_path, "a") as acceptance_test_config_file: + acceptance_test_config_file.write("not_tracked") + yield expected_teams + shutil.copyfile(backup_path, pokeapi_acceptance_test_config_path) + + +@pytest.fixture +def strategic_connector_file_change_expected_team(tmp_path, strategic_connector_file): + expected_teams = list(required_reviewer_checks.STRATEGIC_PYTHON_CONNECTOR_REVIEWERS) + backup_path = tmp_path / "strategic_acceptance_test_config.backup" + shutil.copyfile(strategic_connector_file, backup_path) + with open(strategic_connector_file, "a") as strategic_acceptance_test_config_file: + strategic_acceptance_test_config_file.write("foobar") + yield expected_teams + shutil.copyfile(backup_path, strategic_connector_file) + + +@pytest.fixture +def strategic_connector_backward_compatibility_file_change_expected_team(tmp_path, strategic_connector_file): + expected_teams = list(required_reviewer_checks.BACKWARD_COMPATIBILITY_REVIEWERS) + backup_path = tmp_path / "strategic_acceptance_test_config.backup" + shutil.copyfile(strategic_connector_file, backup_path) + with open(strategic_connector_file, "a") as strategic_acceptance_test_config_file: + strategic_acceptance_test_config_file.write("disable_for_version: 0.0.0") + yield expected_teams + shutil.copyfile(backup_path, strategic_connector_file) + + +@pytest.fixture +def strategic_connector_bypass_reason_file_change_expected_team(tmp_path, strategic_connector_file): + expected_teams = list(required_reviewer_checks.BYPASS_REASON_REVIEWERS) + backup_path = tmp_path / "strategic_acceptance_test_config.backup" + shutil.copyfile(strategic_connector_file, backup_path) + with open(strategic_connector_file, "a") as strategic_acceptance_test_config_file: + strategic_acceptance_test_config_file.write("bypass_reason:") + yield expected_teams + shutil.copyfile(backup_path, strategic_connector_file) + + +@pytest.fixture +def strategic_connector_test_strictness_level_file_change_expected_team(tmp_path, strategic_connector_file): + expected_teams = list(required_reviewer_checks.TEST_STRICTNESS_LEVEL_REVIEWERS) + backup_path = tmp_path / "strategic_acceptance_test_config.backup" + shutil.copyfile(strategic_connector_file, backup_path) + with open(strategic_connector_file, "a") as strategic_acceptance_test_config_file: + strategic_acceptance_test_config_file.write("test_strictness_level: 0.0.0") + yield expected_teams + shutil.copyfile(backup_path, strategic_connector_file) + + +@pytest.fixture +def test_breaking_change_release_expected_team(tmp_path, pokeapi_metadata_path) -> List: + expected_teams = list(required_reviewer_checks.BREAKING_CHANGE_REVIEWERS) + backup_path = tmp_path / "backup_poke_metadata" + shutil.copyfile(pokeapi_metadata_path, backup_path) + with open(pokeapi_metadata_path, "a") as acceptance_test_config_file: + acceptance_test_config_file.write( + "releases:\n breakingChanges:\n 23.0.0:\n message: hi\n upgradeDeadline: 2025-01-01" + ) + yield expected_teams + shutil.copyfile(backup_path, pokeapi_metadata_path) + + +def verify_no_requirements_file_was_generated(captured: str): + assert captured.out.split("\n")[0].split("=")[-1] == "false" + + +def verify_requirements_file_was_generated(captured: str): + assert captured.out.split("\n")[0].split("=")[-1] == "true" + + +def verify_review_requirements_file_contains_expected_teams(requirements_file_path: str, expected_teams: List): + with open(requirements_file_path, "r") as requirements_file: + requirements = yaml.safe_load(requirements_file) + assert any([r["teams"] == expected_teams for r in requirements]) + + +def check_review_requirements_file(capsys, expected_teams: List): + required_reviewer_checks.write_review_requirements_file() + captured = capsys.readouterr() + if not expected_teams: + verify_no_requirements_file_was_generated(captured) + else: + verify_requirements_file_was_generated(captured) + requirements_file_path = required_reviewer_checks.REVIEW_REQUIREMENTS_FILE_PATH + verify_review_requirements_file_contains_expected_teams(requirements_file_path, expected_teams) + + +def test_find_mandatory_reviewers_backward_compatibility( + mock_diffed_branched, capsys, not_strategic_backward_compatibility_change_expected_team +): + check_review_requirements_file(capsys, not_strategic_backward_compatibility_change_expected_team) + + +def test_find_mandatory_reviewers_test_strictness_level( + mock_diffed_branched, capsys, not_strategic_test_strictness_level_change_expected_team +): + check_review_requirements_file(capsys, not_strategic_test_strictness_level_change_expected_team) + + +def test_find_mandatory_reviewers_not_strategic_bypass_reason( + mock_diffed_branched, capsys, not_strategic_bypass_reason_file_change_expected_team +): + check_review_requirements_file(capsys, not_strategic_bypass_reason_file_change_expected_team) + + +def test_find_mandatory_reviewers_ga(mock_diffed_branched, capsys, strategic_connector_file_change_expected_team): + check_review_requirements_file(capsys, strategic_connector_file_change_expected_team) + + +def test_find_mandatory_reviewers_strategic_backward_compatibility( + mock_diffed_branched, capsys, strategic_connector_backward_compatibility_file_change_expected_team +): + check_review_requirements_file(capsys, strategic_connector_backward_compatibility_file_change_expected_team) + + +def test_find_mandatory_reviewers_strategic_bypass_reason( + mock_diffed_branched, capsys, strategic_connector_bypass_reason_file_change_expected_team +): + check_review_requirements_file(capsys, strategic_connector_bypass_reason_file_change_expected_team) + + +def test_find_mandatory_reviewers_strategic_test_strictness_level( + mock_diffed_branched, capsys, strategic_connector_test_strictness_level_file_change_expected_team +): + check_review_requirements_file(capsys, strategic_connector_test_strictness_level_file_change_expected_team) + + +def test_find_mandatory_reviewers_breaking_change_release(mock_diffed_branched, capsys, test_breaking_change_release_expected_team): + check_review_requirements_file(capsys, test_breaking_change_release_expected_team) + + +def test_find_mandatory_reviewers_no_tracked_changed(mock_diffed_branched, capsys, not_strategic_not_tracked_change_expected_team): + check_review_requirements_file(capsys, not_strategic_not_tracked_change_expected_team) diff --git a/airbyte-ci/connectors/connector_ops/tests/test_utils.py b/airbyte-ci/connectors/connector_ops/tests/test_utils.py index c2cc09db0e19..8350a580f0ea 100644 --- a/airbyte-ci/connectors/connector_ops/tests/test_utils.py +++ b/airbyte-ci/connectors/connector_ops/tests/test_utils.py @@ -30,42 +30,75 @@ def test__get_type_and_name_from_technical_name(self, technical_name, expected_t "connector, exists", [ (utils.Connector("source-faker"), True), - (utils.Connector("source-notpublished"), False), + (utils.Connector("source-doesnotexist"), False), ], ) def test_init(self, connector, exists, mocker, tmp_path): assert str(connector) == connector.technical_name - assert connector.connector_type, connector.name == connector._get_type_and_name_from_technical_name() assert connector.code_directory == Path(f"./airbyte-integrations/connectors/{connector.technical_name}") assert connector.acceptance_test_config_path == connector.code_directory / utils.ACCEPTANCE_TEST_CONFIG_FILE_NAME - assert connector.documentation_file_path == Path(f"./docs/integrations/{connector.connector_type}s/{connector.name}.md") if exists: + assert connector.connector_type, connector.name == connector._get_type_and_name_from_technical_name() + assert connector.documentation_file_path == Path(f"./docs/integrations/{connector.connector_type}s/{connector.name}.md") assert isinstance(connector.metadata, dict) assert isinstance(connector.support_level, str) assert isinstance(connector.acceptance_test_config, dict) - assert connector.icon_path == Path(f"./airbyte-config-oss/init-oss/src/main/resources/icons/{connector.metadata['icon']}") + assert connector.icon_path == Path(f"./airbyte-integrations/connectors/{connector.technical_name}/icon.svg") assert len(connector.version.split(".")) == 3 else: assert connector.metadata is None assert connector.support_level is None assert connector.acceptance_test_config is None - assert connector.icon_path == Path(f"./airbyte-config-oss/init-oss/src/main/resources/icons/{connector.name}.svg") - with pytest.raises(FileNotFoundError): - connector.version + assert connector.icon_path == Path(f"./airbyte-integrations/connectors/{connector.technical_name}/icon.svg") + assert connector.version is None with pytest.raises(utils.ConnectorVersionNotFound): Path(tmp_path / "Dockerfile").touch() mocker.patch.object(utils.Connector, "code_directory", tmp_path) utils.Connector(connector.technical_name).version + def test_metadata_query_match(self, mocker): + connector = utils.Connector("source-faker") + mocker.patch.object(utils.Connector, "metadata", {"dockerRepository": "airbyte/source-faker", "ab_internal": {"ql": 100}}) + assert connector.metadata_query_match("data.dockerRepository == 'airbyte/source-faker'") + assert connector.metadata_query_match("'source' in data.dockerRepository") + assert not connector.metadata_query_match("data.dockerRepository == 'airbyte/source-faker2'") + assert not connector.metadata_query_match("'destination' in data.dockerRepository") + assert connector.metadata_query_match("data.ab_internal.ql == 100") + assert connector.metadata_query_match("data.ab_internal.ql >= 100") + assert connector.metadata_query_match("data.ab_internal.ql > 1") + assert not connector.metadata_query_match("data.ab_internal.ql == 101") + assert not connector.metadata_query_match("data.ab_internal.ql >= 101") + assert not connector.metadata_query_match("data.ab_internal.ql > 101") + assert not connector.metadata_query_match("data.ab_internal == whatever") + + @pytest.fixture + def connector_without_dockerfile(self, mocker, tmp_path): + mocker.patch.object(utils.Connector, "code_directory", tmp_path) + connector = utils.Connector("source-faker") + return connector + + def test_has_dockerfile_without_dockerfile(self, connector_without_dockerfile): + assert not connector_without_dockerfile.has_dockerfile + + @pytest.fixture + def connector_with_dockerfile(self, mocker, tmp_path): + mocker.patch.object(utils.Connector, "code_directory", tmp_path) + connector = utils.Connector("source-faker") + tmp_path.joinpath("Dockerfile").touch() + return connector + + def test_has_dockerfile_with_dockerfile(self, connector_with_dockerfile): + assert connector_with_dockerfile.has_dockerfile + @pytest.fixture() -def gradle_file_with_dependencies(tmpdir) -> Path: +def gradle_file_with_dependencies(tmpdir) -> tuple[Path, list[Path], list[Path]]: test_gradle_file = Path(tmpdir) / "build.gradle" test_gradle_file.write_text( """ plugins { - id 'java' + id 'java' } dependencies { @@ -83,56 +116,68 @@ def gradle_file_with_dependencies(tmpdir) -> Path: return test_gradle_file, expected_dependencies, expected_test_dependencies +@pytest.fixture() +def gradle_file_with_local_cdk_dependencies(tmpdir) -> tuple[Path, list[Path], list[Path]]: + test_gradle_file = Path(tmpdir) / "build.gradle" + test_gradle_file.write_text( + """ + plugins { + id 'java' + id 'airbyte-java-connector' + } + + airbyteJavaConnector { + cdkVersionRequired = '0.1.0' + features = ['db-destinations'] + useLocalCdk = true + } + + airbyteJavaConnector.addCdkDependencies() + + dependencies { + implementation project(':path:to:dependency1') + implementation project(':path:to:dependency2') + testImplementation project(':path:to:test:dependency') + integrationTestJavaImplementation project(':path:to:test:dependency1') + performanceTestJavaImplementation project(':path:to:test:dependency2') + } + """ + ) + expected_dependencies = [ + Path("path/to/dependency1"), + Path("path/to/dependency2"), + ] + expected_test_dependencies = [ + Path("path/to/test/dependency"), + Path("path/to/test/dependency1"), + Path("path/to/test/dependency2"), + ] + return test_gradle_file, expected_dependencies, expected_test_dependencies + + def test_parse_dependencies(gradle_file_with_dependencies): gradle_file, expected_regular_dependencies, expected_test_dependencies = gradle_file_with_dependencies - regular_dependencies, test_dependencies = utils.parse_dependencies(gradle_file) + regular_dependencies, test_dependencies = utils.parse_gradle_dependencies(gradle_file) + assert len(regular_dependencies) == len(expected_regular_dependencies) + assert all([regular_dependency in expected_regular_dependencies for regular_dependency in regular_dependencies]) + assert len(test_dependencies) == len(expected_test_dependencies) + assert all([test_dependency in expected_test_dependencies for test_dependency in test_dependencies]) + + +def test_parse_dependencies_with_cdk(gradle_file_with_local_cdk_dependencies): + gradle_file, expected_regular_dependencies, expected_test_dependencies = gradle_file_with_local_cdk_dependencies + regular_dependencies, test_dependencies = utils.parse_gradle_dependencies(gradle_file) assert len(regular_dependencies) == len(expected_regular_dependencies) assert all([regular_dependency in expected_regular_dependencies for regular_dependency in regular_dependencies]) assert len(test_dependencies) == len(expected_test_dependencies) assert all([test_dependency in expected_test_dependencies for test_dependency in test_dependencies]) -@pytest.mark.parametrize("with_test_dependencies", [True, False]) -def test_get_all_gradle_dependencies(with_test_dependencies): - build_file = Path("airbyte-integrations/connectors/source-postgres-strict-encrypt/build.gradle") - if with_test_dependencies: - all_dependencies = utils.get_all_gradle_dependencies(build_file) - expected_dependencies = [ - Path("airbyte-cdk/java/airbyte-cdk"), - Path("airbyte-db/db-lib"), - Path("airbyte-json-validation"), - Path("airbyte-config-oss/config-models-oss"), - Path("airbyte-commons"), - Path("airbyte-test-utils"), - Path("airbyte-api"), - Path("airbyte-connector-test-harnesses/acceptance-test-harness"), - Path("airbyte-commons-protocol"), - Path("airbyte-integrations/bases/base-java"), - Path("airbyte-commons-cli"), - Path("airbyte-integrations/bases/base"), - Path("airbyte-integrations/connectors/source-postgres"), - Path("airbyte-integrations/bases/debezium"), - Path("airbyte-integrations/connectors/source-jdbc"), - Path("airbyte-integrations/connectors/source-relational-db"), - Path("airbyte-integrations/bases/standard-source-test"), - ] - assert len(all_dependencies) == len(expected_dependencies) - assert all([dependency in expected_dependencies for dependency in all_dependencies]) - else: - all_dependencies = utils.get_all_gradle_dependencies(build_file, with_test_dependencies=False) - expected_dependencies = [ - Path("airbyte-cdk/java/airbyte-cdk"), - Path("airbyte-db/db-lib"), - Path("airbyte-json-validation"), - Path("airbyte-config-oss/config-models-oss"), - Path("airbyte-commons"), - Path("airbyte-integrations/bases/base-java"), - Path("airbyte-commons-cli"), - Path("airbyte-integrations/bases/base"), - Path("airbyte-integrations/connectors/source-postgres"), - Path("airbyte-integrations/bases/debezium"), - Path("airbyte-integrations/connectors/source-jdbc"), - Path("airbyte-integrations/connectors/source-relational-db"), - ] - assert len(all_dependencies) == len(expected_dependencies) - assert all([dependency in expected_dependencies for dependency in all_dependencies]) +def test_get_all_connectors_in_repo(): + all_connectors = utils.get_all_connectors_in_repo() + assert len(all_connectors) > 0 + for connector in all_connectors: + assert isinstance(connector, utils.Connector) + assert connector.metadata is not None + if connector.has_airbyte_docs: + assert connector.documentation_file_path.exists() diff --git a/airbyte-ci/connectors/metadata_service/lib/README.md b/airbyte-ci/connectors/metadata_service/lib/README.md index d573fb6bf581..5b2016c3f921 100644 --- a/airbyte-ci/connectors/metadata_service/lib/README.md +++ b/airbyte-ci/connectors/metadata_service/lib/README.md @@ -31,8 +31,12 @@ poetry run pytest ``` ## Validating Metadata Files +To be considered valid, a connector must have a metadata.yaml file which must conform to the [ConnectorMetadataDefinitionV0](./metadata_service/models/src/ConnectorMetadataDefinitionV0.yaml) schema, and a documentation file. + +The paths to both files must be passed to the validate command. + ```bash -poetry run metadata_service validate tests/fixtures/valid/metadata_registry_override.yaml +poetry run metadata_service validate tests/fixtures/metadata_validate/valid/metadata_simple.yaml tests/fixtures/doc.md ``` ## Useful Commands diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/commands.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/commands.py index de86081a18cb..12604b08baa3 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/commands.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/commands.py @@ -12,20 +12,11 @@ def log_metadata_upload_info(metadata_upload_info: MetadataUploadInfo): - if metadata_upload_info.version_uploaded: - click.secho( - f"The metadata file {metadata_upload_info.metadata_file_path} was uploaded to {metadata_upload_info.version_blob_id}.", - color="green", - ) - if metadata_upload_info.latest_uploaded: - click.secho( - f"The metadata file {metadata_upload_info.metadata_file_path} was uploaded to {metadata_upload_info.latest_blob_id}.", - color="green", - ) - if metadata_upload_info.icon_uploaded: - click.secho( - f"The icon file {metadata_upload_info.metadata_file_path} was uploaded to {metadata_upload_info.icon_blob_id}.", color="green" - ) + for file in metadata_upload_info.uploaded_files: + if file.uploaded: + click.secho( + f"The {file.description} file for {metadata_upload_info.metadata_file_path} was uploaded to {file.blob_id}.", color="green" + ) @click.group(help="Airbyte Metadata Service top-level command group.") @@ -34,35 +25,37 @@ def metadata_service(): @metadata_service.command(help="Validate a given metadata YAML file.") -@click.argument("file_path", type=click.Path(exists=True, path_type=pathlib.Path)) -def validate(file_path: pathlib.Path): - file_path = file_path if not file_path.is_dir() else file_path / METADATA_FILE_NAME +@click.argument("metadata_file_path", type=click.Path(exists=True, path_type=pathlib.Path), required=True) +@click.argument("docs_path", type=click.Path(exists=True, path_type=pathlib.Path), required=True) +def validate(metadata_file_path: pathlib.Path, docs_path: pathlib.Path): + metadata_file_path = metadata_file_path if not metadata_file_path.is_dir() else metadata_file_path / METADATA_FILE_NAME - click.echo(f"Validating {file_path}...") + click.echo(f"Validating {metadata_file_path}...") - metadata, error = validate_and_load(file_path, PRE_UPLOAD_VALIDATORS) + metadata, error = validate_and_load(metadata_file_path, PRE_UPLOAD_VALIDATORS, ValidatorOptions(docs_path=str(docs_path))) if metadata: - click.echo(f"{file_path} is a valid ConnectorMetadataDefinitionV0 YAML file.") + click.echo(f"{metadata_file_path} is a valid ConnectorMetadataDefinitionV0 YAML file.") else: - click.echo(f"{file_path} is not a valid ConnectorMetadataDefinitionV0 YAML file.") + click.echo(f"{metadata_file_path} is not a valid ConnectorMetadataDefinitionV0 YAML file.") click.echo(str(error)) exit(1) @metadata_service.command(help="Upload a metadata YAML file to a GCS bucket.") -@click.argument("metadata-file-path", type=click.Path(exists=True, path_type=pathlib.Path)) -@click.argument("bucket-name", type=click.STRING) +@click.argument("metadata-file-path", type=click.Path(exists=True, path_type=pathlib.Path), required=True) +@click.argument("docs-path", type=click.Path(exists=True, path_type=pathlib.Path), required=True) +@click.argument("bucket-name", type=click.STRING, required=True) @click.option("--prerelease", type=click.STRING, required=False, default=None, help="The prerelease tag of the connector.") -def upload(metadata_file_path: pathlib.Path, bucket_name: str, prerelease: str): +def upload(metadata_file_path: pathlib.Path, docs_path: pathlib.Path, bucket_name: str, prerelease: str): metadata_file_path = metadata_file_path if not metadata_file_path.is_dir() else metadata_file_path / METADATA_FILE_NAME - validator_opts = ValidatorOptions(prerelease_tag=prerelease) + validator_opts = ValidatorOptions(docs_path=str(docs_path), prerelease_tag=prerelease) try: upload_info = upload_metadata_to_gcs(bucket_name, metadata_file_path, validator_opts) log_metadata_upload_info(upload_info) except (ValidationError, FileNotFoundError) as e: click.secho(f"The metadata file could not be uploaded: {str(e)}", color="red") exit(1) - if upload_info.uploaded: + if upload_info.metadata_uploaded: exit(0) else: click.secho(f"The metadata file {metadata_file_path} was not uploaded.", color="yellow") diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/constants.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/constants.py index b8ce22e7406c..ef563008a9ed 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/constants.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/constants.py @@ -5,3 +5,6 @@ METADATA_FILE_NAME = "metadata.yaml" ICON_FILE_NAME = "icon.svg" METADATA_FOLDER = "metadata" +DOCS_FOLDER_PATH = "docs/integrations" +DOC_FILE_NAME = "doc.md" +DOC_INAPP_FILE_NAME = "doc.inapp.md" diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/docker_hub.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/docker_hub.py index ff0bdc82e94f..1b01d20328d3 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/docker_hub.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/docker_hub.py @@ -3,7 +3,8 @@ # import os -from typing import List +import time +from typing import Optional import requests @@ -26,13 +27,15 @@ def get_docker_hub_auth_token() -> str: return token -def is_image_on_docker_hub(image_name: str, version: str) -> bool: +def is_image_on_docker_hub(image_name: str, version: str, digest: Optional[str] = None, retries: int = 0, wait_sec: int = 30) -> bool: """Check if a given image and version exists on Docker Hub. Args: image_name (str): The name of the image to check. version (str): The version of the image to check. - + digest (str, optional): The digest of the image to check. Defaults to None. + retries (int, optional): The number of times to retry the request. Defaults to 0. + wait_sec (int, optional): The number of seconds to wait between retries. Defaults to 30. Returns: bool: True if the image and version exists on Docker Hub, False otherwise. """ @@ -40,6 +43,19 @@ def is_image_on_docker_hub(image_name: str, version: str) -> bool: token = get_docker_hub_auth_token() headers = {"Authorization": f"JWT {token}"} tag_url = f"https://registry.hub.docker.com/v2/repositories/{image_name}/tags/{version}" - response = requests.get(tag_url, headers=headers) - return response.ok + # Allow for retries as the DockerHub API is not always reliable with returning the latest publish. + for _ in range(retries + 1): + response = requests.get(tag_url, headers=headers) + if response.ok: + break + time.sleep(wait_sec) + + if not response.ok: + response.raise_for_status() + return False + + # If a digest is provided, check that it matches the digest of the image on Docker Hub. + if digest is not None: + return f"sha256:{digest}" == response.json()["digest"] + return True diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/gcs_upload.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/gcs_upload.py index 74de80534cec..695e90a46d81 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/gcs_upload.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/gcs_upload.py @@ -6,14 +6,22 @@ import hashlib import json import os +import re from dataclasses import dataclass from pathlib import Path -from typing import Optional, Tuple +from typing import List, Optional, Tuple import yaml from google.cloud import storage from google.oauth2 import service_account -from metadata_service.constants import ICON_FILE_NAME, METADATA_FILE_NAME, METADATA_FOLDER +from metadata_service.constants import ( + DOC_FILE_NAME, + DOC_INAPP_FILE_NAME, + DOCS_FOLDER_PATH, + ICON_FILE_NAME, + METADATA_FILE_NAME, + METADATA_FOLDER, +) from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 from metadata_service.models.transform import to_json_sanitized_dict from metadata_service.validators.metadata_validator import POST_UPLOAD_VALIDATORS, ValidatorOptions, validate_and_load @@ -21,15 +29,18 @@ @dataclass(frozen=True) -class MetadataUploadInfo: +class UploadedFile: + id: str uploaded: bool - latest_uploaded: bool - latest_blob_id: Optional[str] - version_uploaded: bool - version_blob_id: Optional[str] - icon_uploaded: bool - icon_blob_id: Optional[str] + description: str + blob_id: Optional[str] + + +@dataclass(frozen=True) +class MetadataUploadInfo: + metadata_uploaded: bool metadata_file_path: str + uploaded_files: List[UploadedFile] def get_metadata_remote_file_path(dockerRepository: str, version: str) -> str: @@ -56,6 +67,27 @@ def get_icon_remote_file_path(dockerRepository: str, version: str) -> str: return f"{METADATA_FOLDER}/{dockerRepository}/{version}/{ICON_FILE_NAME}" +def get_doc_remote_file_path(dockerRepository: str, version: str, inapp: bool) -> str: + """Get the path to the icon file for a specific version of a connector. + + Args: + dockerRepository (str): Name of the connector docker image. + version (str): Version of the connector. + Returns: + str: Path to the icon file. + """ + return f"{METADATA_FOLDER}/{dockerRepository}/{version}/{DOC_INAPP_FILE_NAME if inapp else DOC_FILE_NAME}" + + +def get_doc_local_file_path(metadata: ConnectorMetadataDefinitionV0, docs_path: Path, inapp: bool) -> Path: + pattern = re.compile(r"^https://docs\.airbyte\.com/(.+)$") + match = pattern.search(metadata.data.documentationUrl) + if match: + extension = ".inapp.md" if inapp else ".md" + return (docs_path / match.group(1)).with_suffix(extension) + return None + + def compute_gcs_md5(file_name: str) -> str: hash_md5 = hashlib.md5() with open(file_name, "rb") as f: @@ -120,6 +152,26 @@ def _icon_upload(metadata: ConnectorMetadataDefinitionV0, bucket: storage.bucket return upload_file_if_changed(local_icon_path, bucket, latest_icon_path) +def _doc_upload( + metadata: ConnectorMetadataDefinitionV0, bucket: storage.bucket.Bucket, docs_path: Path, latest: bool, inapp: bool +) -> Tuple[bool, str]: + local_doc_path = get_doc_local_file_path(metadata, docs_path, inapp) + if not local_doc_path: + return False, f"Metadata does not contain a valid Airbyte documentation url, skipping doc upload." + + remote_doc_path = get_doc_remote_file_path(metadata.data.dockerRepository, "latest" if latest else metadata.data.dockerImageTag, inapp) + + if local_doc_path.exists(): + doc_uploaded, doc_blob_id = upload_file_if_changed(local_doc_path, bucket, remote_doc_path) + else: + if inapp: + doc_uploaded, doc_blob_id = False, f"No inapp doc found at {local_doc_path}, skipping inapp doc upload." + else: + raise FileNotFoundError(f"Expected to find connector doc file at {local_doc_path}, but none was found.") + + return doc_uploaded, doc_blob_id + + def create_prerelease_metadata_file(metadata_file_path: Path, validator_opts: ValidatorOptions) -> Path: metadata, error = validate_and_load(metadata_file_path, [], validator_opts) if metadata is None: @@ -143,9 +195,7 @@ def create_prerelease_metadata_file(metadata_file_path: Path, validator_opts: Va return tmp_metadata_file_path -def upload_metadata_to_gcs( - bucket_name: str, metadata_file_path: Path, validator_opts: ValidatorOptions = ValidatorOptions() -) -> MetadataUploadInfo: +def upload_metadata_to_gcs(bucket_name: str, metadata_file_path: Path, validator_opts: ValidatorOptions) -> MetadataUploadInfo: """Upload a metadata file to a GCS bucket. If the per 'version' key already exists it won't be overwritten. @@ -167,26 +217,76 @@ def upload_metadata_to_gcs( if metadata is None: raise ValueError(f"Metadata file {metadata_file_path} is invalid for uploading: {error}") - service_account_info = json.loads(os.environ.get("GCS_CREDENTIALS")) + gcs_creds = os.environ.get("GCS_CREDENTIALS") + if not gcs_creds: + raise ValueError("Please set the GCS_CREDENTIALS env var.") + + service_account_info = json.loads(gcs_creds) credentials = service_account.Credentials.from_service_account_info(service_account_info) storage_client = storage.Client(credentials=credentials) bucket = storage_client.bucket(bucket_name) + docs_path = Path(validator_opts.docs_path) icon_uploaded, icon_blob_id = _icon_upload(metadata, bucket, metadata_file_path) version_uploaded, version_blob_id = _version_upload(metadata, bucket, metadata_file_path) + + doc_version_uploaded, doc_version_blob_id = _doc_upload(metadata, bucket, docs_path, False, False) + doc_inapp_version_uploaded, doc_inapp_version_blob_id = _doc_upload(metadata, bucket, docs_path, False, True) + if not validator_opts.prerelease_tag: latest_uploaded, latest_blob_id = _latest_upload(metadata, bucket, metadata_file_path) + doc_latest_uploaded, doc_latest_blob_id = _doc_upload(metadata, bucket, docs_path, True, False) + doc_inapp_latest_uploaded, doc_inapp_latest_blob_id = _doc_upload(metadata, bucket, docs_path, True, True) else: latest_uploaded, latest_blob_id = False, None + doc_latest_uploaded, doc_latest_blob_id = doc_inapp_latest_uploaded, doc_inapp_latest_blob_id = False, None return MetadataUploadInfo( - uploaded=version_uploaded or latest_uploaded, - latest_uploaded=latest_uploaded, - version_uploaded=version_uploaded, - version_blob_id=version_blob_id, - latest_blob_id=latest_blob_id, - icon_blob_id=icon_blob_id, - icon_uploaded=icon_uploaded, + metadata_uploaded=version_uploaded or latest_uploaded, metadata_file_path=str(metadata_file_path), + uploaded_files=[ + UploadedFile( + id="version_metadata", + uploaded=version_uploaded, + description="versioned metadata", + blob_id=version_blob_id, + ), + UploadedFile( + id="latest_metadata", + uploaded=latest_uploaded, + description="latest metadata", + blob_id=latest_blob_id, + ), + UploadedFile( + id="icon", + uploaded=icon_uploaded, + description="icon", + blob_id=icon_blob_id, + ), + UploadedFile( + id="doc_version", + uploaded=doc_version_uploaded, + description="versioned doc", + blob_id=doc_version_blob_id, + ), + UploadedFile( + id="doc_latest", + uploaded=doc_latest_uploaded, + description="latest doc", + blob_id=doc_latest_blob_id, + ), + UploadedFile( + id="doc_inapp_version", + uploaded=doc_inapp_version_uploaded, + description="versioned inapp doc", + blob_id=doc_inapp_version_blob_id, + ), + UploadedFile( + id="doc_inapp_latest", + uploaded=doc_inapp_latest_uploaded, + description="latest inapp doc", + blob_id=doc_inapp_latest_blob_id, + ), + ], ) diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ActorDefinitionResourceRequirements.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ActorDefinitionResourceRequirements.py index e3ec41f010b2..1f6e484eef73 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ActorDefinitionResourceRequirements.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ActorDefinitionResourceRequirements.py @@ -1,7 +1,3 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - # generated by datamodel-codegen: # filename: ActorDefinitionResourceRequirements.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/AirbyteInternal.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/AirbyteInternal.py index 9714260f2670..c70266521316 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/AirbyteInternal.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/AirbyteInternal.py @@ -1,7 +1,3 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - # generated by datamodel-codegen: # filename: AirbyteInternal.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/AllowedHosts.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/AllowedHosts.py index c7401961b290..ce4534f3adca 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/AllowedHosts.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/AllowedHosts.py @@ -1,7 +1,3 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - # generated by datamodel-codegen: # filename: AllowedHosts.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorBuildOptions.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorBuildOptions.py new file mode 100644 index 000000000000..ad7a10d5ab56 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorBuildOptions.py @@ -0,0 +1,15 @@ +# generated by datamodel-codegen: +# filename: ConnectorBuildOptions.yaml + +from __future__ import annotations + +from typing import Optional + +from pydantic import BaseModel, Extra + + +class ConnectorBuildOptions(BaseModel): + class Config: + extra = Extra.forbid + + baseImage: Optional[str] = None diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py index 4721e9f15f21..40719df845f3 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorMetadataDefinitionV0.py @@ -1,20 +1,23 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - # generated by datamodel-codegen: # filename: ConnectorMetadataDefinitionV0.yaml from __future__ import annotations from datetime import date -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional from uuid import UUID from pydantic import AnyUrl, BaseModel, Extra, Field, constr from typing_extensions import Literal +class ConnectorBuildOptions(BaseModel): + class Config: + extra = Extra.forbid + + baseImage: Optional[str] = None + + class ReleaseStage(BaseModel): __root__: Literal["alpha", "beta", "generally_available", "custom"] = Field( ..., @@ -95,20 +98,15 @@ class JobType(BaseModel): ) -class VersionBreakingChange(BaseModel): +class StreamBreakingChangeScope(BaseModel): class Config: extra = Extra.forbid - upgradeDeadline: date = Field( + scopeType: Any = Field("stream", const=True) + impactedScopes: List[str] = Field( ..., - description="The deadline by which to upgrade before the breaking change takes effect.", - ) - message: str = Field( - ..., description="Descriptive message detailing the breaking change." - ) - migrationDocumentationUrl: Optional[AnyUrl] = Field( - None, - description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", + description="List of streams that are impacted by the breaking change.", + min_items=1, ) @@ -120,6 +118,14 @@ class Config: ql: Optional[Literal[100, 200, 300, 400, 500, 600]] = None +class PyPi(BaseModel): + class Config: + extra = Extra.forbid + + enabled: bool + packageName: str = Field(..., description="The name of the package on PyPi.") + + class JobTypeResourceLimit(BaseModel): class Config: extra = Extra.forbid @@ -128,14 +134,18 @@ class Config: resourceRequirements: ResourceRequirements -class ConnectorBreakingChanges(BaseModel): +class BreakingChangeScope(BaseModel): + __root__: StreamBreakingChangeScope = Field( + ..., + description="A scope that can be used to limit the impact of a breaking change.", + ) + + +class RemoteRegistries(BaseModel): class Config: extra = Extra.forbid - __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( - ..., - description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", - ) + pypi: Optional[PyPi] = None class ActorDefinitionResourceRequirements(BaseModel): @@ -149,14 +159,25 @@ class Config: jobSpecific: Optional[List[JobTypeResourceLimit]] = None -class ConnectorReleases(BaseModel): +class VersionBreakingChange(BaseModel): class Config: extra = Extra.forbid - breakingChanges: ConnectorBreakingChanges + upgradeDeadline: date = Field( + ..., + description="The deadline by which to upgrade before the breaking change takes effect.", + ) + message: str = Field( + ..., description="Descriptive message detailing the breaking change." + ) migrationDocumentationUrl: Optional[AnyUrl] = Field( None, - description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", + description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", + ) + scopedImpact: Optional[List[BreakingChangeScope]] = Field( + None, + description="List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", + min_items=1, ) @@ -179,6 +200,16 @@ class Config: resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None +class ConnectorBreakingChanges(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( + ..., + description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", + ) + + class Registry(BaseModel): class Config: extra = Extra.forbid @@ -187,13 +218,25 @@ class Config: cloud: Optional[RegistryOverrides] = None +class ConnectorReleases(BaseModel): + class Config: + extra = Extra.forbid + + breakingChanges: ConnectorBreakingChanges + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate from the previous version to the current version. Defaults to ${documentationUrl}-migrations", + ) + + class Data(BaseModel): class Config: - extra = Extra.allow + extra = Extra.forbid name: str icon: Optional[str] = None definitionId: UUID + connectorBuildOptions: Optional[ConnectorBuildOptions] = None connectorType: Literal["destination", "source"] dockerRepository: str dockerImageTag: str @@ -214,7 +257,14 @@ class Config: None, description="the Airbyte Protocol version supported by the connector" ) connectorSubtype: Literal[ - "api", "database", "file", "custom", "message_queue", "unknown" + "api", + "database", + "datalake", + "file", + "custom", + "message_queue", + "unknown", + "vectorstore", ] releaseStage: ReleaseStage supportLevel: Optional[SupportLevel] = None @@ -229,6 +279,7 @@ class Config: suggestedStreams: Optional[SuggestedStreams] = None resourceRequirements: Optional[ActorDefinitionResourceRequirements] = None ab_internal: Optional[AirbyteInternal] = None + remoteRegistries: Optional[RemoteRegistries] = None class ConnectorMetadataDefinitionV0(BaseModel): diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryDestinationDefinition.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryDestinationDefinition.py index 5b7e71acb9f1..e1954ad41369 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryDestinationDefinition.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryDestinationDefinition.py @@ -1,7 +1,3 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - # generated by datamodel-codegen: # filename: ConnectorRegistryDestinationDefinition.yaml @@ -85,20 +81,15 @@ class Config: ) -class VersionBreakingChange(BaseModel): +class StreamBreakingChangeScope(BaseModel): class Config: extra = Extra.forbid - upgradeDeadline: date = Field( + scopeType: Any = Field("stream", const=True) + impactedScopes: List[str] = Field( ..., - description="The deadline by which to upgrade before the breaking change takes effect.", - ) - message: str = Field( - ..., description="Descriptive message detailing the breaking change." - ) - migrationDocumentationUrl: Optional[AnyUrl] = Field( - None, - description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", + description="List of streams that are impacted by the breaking change.", + min_items=1, ) @@ -118,13 +109,10 @@ class Config: resourceRequirements: ResourceRequirements -class ConnectorBreakingChanges(BaseModel): - class Config: - extra = Extra.forbid - - __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( +class BreakingChangeScope(BaseModel): + __root__: StreamBreakingChangeScope = Field( ..., - description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", + description="A scope that can be used to limit the impact of a breaking change.", ) @@ -139,6 +127,38 @@ class Config: jobSpecific: Optional[List[JobTypeResourceLimit]] = None +class VersionBreakingChange(BaseModel): + class Config: + extra = Extra.forbid + + upgradeDeadline: date = Field( + ..., + description="The deadline by which to upgrade before the breaking change takes effect.", + ) + message: str = Field( + ..., description="Descriptive message detailing the breaking change." + ) + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", + ) + scopedImpact: Optional[List[BreakingChangeScope]] = Field( + None, + description="List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", + min_items=1, + ) + + +class ConnectorBreakingChanges(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( + ..., + description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", + ) + + class ConnectorReleases(BaseModel): class Config: extra = Extra.forbid diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistrySourceDefinition.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistrySourceDefinition.py index 7836ffe5ed44..60ea7a4a970c 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistrySourceDefinition.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistrySourceDefinition.py @@ -1,7 +1,3 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - # generated by datamodel-codegen: # filename: ConnectorRegistrySourceDefinition.yaml @@ -77,20 +73,15 @@ class Config: ) -class VersionBreakingChange(BaseModel): +class StreamBreakingChangeScope(BaseModel): class Config: extra = Extra.forbid - upgradeDeadline: date = Field( + scopeType: Any = Field("stream", const=True) + impactedScopes: List[str] = Field( ..., - description="The deadline by which to upgrade before the breaking change takes effect.", - ) - message: str = Field( - ..., description="Descriptive message detailing the breaking change." - ) - migrationDocumentationUrl: Optional[AnyUrl] = Field( - None, - description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", + description="List of streams that are impacted by the breaking change.", + min_items=1, ) @@ -110,13 +101,10 @@ class Config: resourceRequirements: ResourceRequirements -class ConnectorBreakingChanges(BaseModel): - class Config: - extra = Extra.forbid - - __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( +class BreakingChangeScope(BaseModel): + __root__: StreamBreakingChangeScope = Field( ..., - description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", + description="A scope that can be used to limit the impact of a breaking change.", ) @@ -131,6 +119,38 @@ class Config: jobSpecific: Optional[List[JobTypeResourceLimit]] = None +class VersionBreakingChange(BaseModel): + class Config: + extra = Extra.forbid + + upgradeDeadline: date = Field( + ..., + description="The deadline by which to upgrade before the breaking change takes effect.", + ) + message: str = Field( + ..., description="Descriptive message detailing the breaking change." + ) + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", + ) + scopedImpact: Optional[List[BreakingChangeScope]] = Field( + None, + description="List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", + min_items=1, + ) + + +class ConnectorBreakingChanges(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( + ..., + description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", + ) + + class ConnectorReleases(BaseModel): class Config: extra = Extra.forbid diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryV0.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryV0.py index 290b76e527ea..b1bbd935d6e6 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryV0.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryV0.py @@ -1,7 +1,3 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - # generated by datamodel-codegen: # filename: ConnectorRegistryV0.yaml @@ -85,20 +81,15 @@ class Config: ) -class VersionBreakingChange(BaseModel): +class StreamBreakingChangeScope(BaseModel): class Config: extra = Extra.forbid - upgradeDeadline: date = Field( + scopeType: Any = Field("stream", const=True) + impactedScopes: List[str] = Field( ..., - description="The deadline by which to upgrade before the breaking change takes effect.", - ) - message: str = Field( - ..., description="Descriptive message detailing the breaking change." - ) - migrationDocumentationUrl: Optional[AnyUrl] = Field( - None, - description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", + description="List of streams that are impacted by the breaking change.", + min_items=1, ) @@ -128,13 +119,10 @@ class Config: resourceRequirements: ResourceRequirements -class ConnectorBreakingChanges(BaseModel): - class Config: - extra = Extra.forbid - - __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( +class BreakingChangeScope(BaseModel): + __root__: StreamBreakingChangeScope = Field( ..., - description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", + description="A scope that can be used to limit the impact of a breaking change.", ) @@ -149,6 +137,38 @@ class Config: jobSpecific: Optional[List[JobTypeResourceLimit]] = None +class VersionBreakingChange(BaseModel): + class Config: + extra = Extra.forbid + + upgradeDeadline: date = Field( + ..., + description="The deadline by which to upgrade before the breaking change takes effect.", + ) + message: str = Field( + ..., description="Descriptive message detailing the breaking change." + ) + migrationDocumentationUrl: Optional[AnyUrl] = Field( + None, + description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", + ) + scopedImpact: Optional[List[BreakingChangeScope]] = Field( + None, + description="List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", + min_items=1, + ) + + +class ConnectorBreakingChanges(BaseModel): + class Config: + extra = Extra.forbid + + __root__: Dict[constr(regex=r"^\d+\.\d+\.\d+$"), VersionBreakingChange] = Field( + ..., + description="Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade.", + ) + + class ConnectorReleases(BaseModel): class Config: extra = Extra.forbid diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorReleases.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorReleases.py index 493f56fcb0c1..8db22c0f403d 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorReleases.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorReleases.py @@ -1,18 +1,33 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - # generated by datamodel-codegen: # filename: ConnectorReleases.yaml from __future__ import annotations from datetime import date -from typing import Dict, Optional +from typing import Any, Dict, List, Optional from pydantic import AnyUrl, BaseModel, Extra, Field, constr +class StreamBreakingChangeScope(BaseModel): + class Config: + extra = Extra.forbid + + scopeType: Any = Field("stream", const=True) + impactedScopes: List[str] = Field( + ..., + description="List of streams that are impacted by the breaking change.", + min_items=1, + ) + + +class BreakingChangeScope(BaseModel): + __root__: StreamBreakingChangeScope = Field( + ..., + description="A scope that can be used to limit the impact of a breaking change.", + ) + + class VersionBreakingChange(BaseModel): class Config: extra = Extra.forbid @@ -28,6 +43,11 @@ class Config: None, description="URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version}", ) + scopedImpact: Optional[List[BreakingChangeScope]] = Field( + None, + description="List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types.", + min_items=1, + ) class ConnectorBreakingChanges(BaseModel): diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/JobType.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/JobType.py index 860a77ac7c27..aef4f7ad5f99 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/JobType.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/JobType.py @@ -1,7 +1,3 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - # generated by datamodel-codegen: # filename: JobType.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/NormalizationDestinationDefinitionConfig.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/NormalizationDestinationDefinitionConfig.py index 0e719498056b..00a642bfaeb1 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/NormalizationDestinationDefinitionConfig.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/NormalizationDestinationDefinitionConfig.py @@ -1,7 +1,3 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - # generated by datamodel-codegen: # filename: NormalizationDestinationDefinitionConfig.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/RegistryOverrides.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/RegistryOverrides.py index 1af53d9865bc..eb6908bc65b2 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/RegistryOverrides.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/RegistryOverrides.py @@ -1,7 +1,3 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - # generated by datamodel-codegen: # filename: RegistryOverrides.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ReleaseStage.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ReleaseStage.py index 59587282973d..cb7c9b909b0b 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ReleaseStage.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ReleaseStage.py @@ -1,7 +1,3 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - # generated by datamodel-codegen: # filename: ReleaseStage.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/RemoteRegistries.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/RemoteRegistries.py new file mode 100644 index 000000000000..b44447eb9c76 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/RemoteRegistries.py @@ -0,0 +1,23 @@ +# generated by datamodel-codegen: +# filename: RemoteRegistries.yaml + +from __future__ import annotations + +from typing import Optional + +from pydantic import BaseModel, Extra, Field + + +class PyPi(BaseModel): + class Config: + extra = Extra.forbid + + enabled: bool + packageName: str = Field(..., description="The name of the package on PyPi.") + + +class RemoteRegistries(BaseModel): + class Config: + extra = Extra.forbid + + pypi: Optional[PyPi] = None diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ResourceRequirements.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ResourceRequirements.py index ad2ebf0da9a2..abc7e6173d05 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ResourceRequirements.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ResourceRequirements.py @@ -1,7 +1,3 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - # generated by datamodel-codegen: # filename: ResourceRequirements.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/SuggestedStreams.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/SuggestedStreams.py index 589441346978..9a3d7cdf4012 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/SuggestedStreams.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/SuggestedStreams.py @@ -1,7 +1,3 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - # generated by datamodel-codegen: # filename: SuggestedStreams.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/SupportLevel.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/SupportLevel.py index 9367e03ddbd6..4a0f7d77c87e 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/SupportLevel.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/SupportLevel.py @@ -1,7 +1,3 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - # generated by datamodel-codegen: # filename: SupportLevel.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/__init__.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/__init__.py index ec5d6b7b85cf..8947dcdaac58 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/__init__.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/__init__.py @@ -2,6 +2,7 @@ from .ActorDefinitionResourceRequirements import * from .AirbyteInternal import * from .AllowedHosts import * +from .ConnectorBuildOptions import * from .ConnectorMetadataDefinitionV0 import * from .ConnectorRegistryDestinationDefinition import * from .ConnectorRegistrySourceDefinition import * @@ -11,6 +12,7 @@ from .NormalizationDestinationDefinitionConfig import * from .RegistryOverrides import * from .ReleaseStage import * +from .RemoteRegistries import * from .ResourceRequirements import * from .SuggestedStreams import * from .SupportLevel import * diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorBuildOptions.yaml b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorBuildOptions.yaml new file mode 100644 index 000000000000..c040dd5404be --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorBuildOptions.yaml @@ -0,0 +1,10 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorBuildOptions.yaml +title: ConnectorBuildOptions +description: metadata specific to the build process. +type: object +additionalProperties: false +properties: + baseImage: + type: string diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorMetadataDefinitionV0.yaml b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorMetadataDefinitionV0.yaml index d35b63633b69..17411405fb6d 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorMetadataDefinitionV0.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorMetadataDefinitionV0.yaml @@ -25,7 +25,7 @@ properties: - githubIssueLabel - connectorSubtype - releaseStage - additionalProperties: true + additionalProperties: false properties: name: type: string @@ -34,6 +34,8 @@ properties: definitionId: type: string format: uuid + connectorBuildOptions: + "$ref": ConnectorBuildOptions.yaml connectorType: type: string enum: @@ -69,10 +71,12 @@ properties: enum: - api - database + - datalake - file - custom - message_queue - unknown + - vectorstore releaseStage: "$ref": ReleaseStage.yaml supportLevel: @@ -107,3 +111,5 @@ properties: "$ref": ActorDefinitionResourceRequirements.yaml ab_internal: "$ref": AirbyteInternal.yaml + remoteRegistries: + "$ref": RemoteRegistries.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorReleases.yaml b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorReleases.yaml index ec1f48611ec1..ff29e780783e 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorReleases.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorReleases.yaml @@ -2,7 +2,7 @@ "$schema": http://json-schema.org/draft-07/schema# "$id": https://github.com/airbytehq/airbyte/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorReleases.yaml title: ConnectorReleases -description: Each entry denotes a breaking change in a specific version of a connector that requires user action to upgrade. +description: Contains information about different types of releases for a connector. type: object additionalProperties: false required: @@ -42,3 +42,31 @@ definitions: description: URL to documentation on how to migrate to the current version. Defaults to ${documentationUrl}-migrations#${version} type: string format: uri + scopedImpact: + description: List of scopes that are impacted by the breaking change. If not specified, the breaking change cannot be scoped to reduce impact via the supported scope types. + type: array + minItems: 1 + items: + $ref: "#/definitions/BreakingChangeScope" + BreakingChangeScope: + description: A scope that can be used to limit the impact of a breaking change. + type: object + oneOf: + - $ref: "#/definitions/StreamBreakingChangeScope" + StreamBreakingChangeScope: + description: A scope that can be used to limit the impact of a breaking change to specific streams. + type: object + additionalProperties: false + required: + - scopeType + - impactedScopes + properties: + scopeType: + type: const + const: stream + impactedScopes: + description: List of streams that are impacted by the breaking change. + type: array + minItems: 1 + items: + type: string diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/RemoteRegistries.yaml b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/RemoteRegistries.yaml new file mode 100644 index 000000000000..474dc3d0a312 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/RemoteRegistries.yaml @@ -0,0 +1,25 @@ +--- +"$schema": http://json-schema.org/draft-07/schema# +"$id": https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/RemoteRegistries.yml +title: RemoteRegistries +description: describes how the connector is published to remote registries +type: object +additionalProperties: false +properties: + pypi: + $ref: "#/definitions/PyPi" +definitions: + PyPi: + title: PyPi + description: describes the PyPi publishing options + type: object + additionalProperties: false + required: + - enabled + - packageName + properties: + enabled: + type: boolean + packageName: + type: string + description: The name of the package on PyPi. diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/spec_cache.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/spec_cache.py index 51df94c9fa7a..6495b34f9026 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/spec_cache.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/spec_cache.py @@ -4,13 +4,30 @@ import json from dataclasses import dataclass +from enum import Enum from typing import List from google.cloud import storage -SPEC_CACHE_BUCKET_NAME = "io-airbyte-cloud-spec-cache" +PROD_SPEC_CACHE_BUCKET_NAME = "io-airbyte-cloud-spec-cache" CACHE_FOLDER = "specs" -SPEC_FILE_NAME = "spec.json" + + +class Registries(str, Enum): + OSS = "oss" + CLOUD = "cloud" + + @classmethod + def _missing_(cls, value): + """Returns the registry from the string value. (case insensitive)""" + value = value.lower() + for member in cls: + if member.lower() == value: + return member + return None + + +SPEC_FILE_NAMES = {Registries.OSS: "spec.json", Registries.CLOUD: "spec.cloud.json"} @dataclass @@ -18,19 +35,35 @@ class CachedSpec: docker_repository: str docker_image_tag: str spec_cache_path: str + registry: Registries + + def __str__(self) -> str: + return self.spec_cache_path + +def get_spec_file_name(registry: Registries) -> str: + return SPEC_FILE_NAMES[registry] -def get_spec_cache_path(docker_repository: str, docker_image_tag: str) -> str: - """Returns the path to the spec.json file in the spec cache bucket.""" - return f"{CACHE_FOLDER}/{docker_repository}/{docker_image_tag}/{SPEC_FILE_NAME}" + +def get_registry_from_spec_cache_path(spec_cache_path: str) -> Registries: + """Returns the registry from the spec cache path.""" + for registry in Registries: + file_name = get_spec_file_name(registry) + if file_name in spec_cache_path: + return registry + + raise Exception(f"Could not find any registry file name in spec cache path: {spec_cache_path}") def get_docker_info_from_spec_cache_path(spec_cache_path: str) -> CachedSpec: """Returns the docker repository and tag from the spec cache path.""" + registry = get_registry_from_spec_cache_path(spec_cache_path) + registry_file_name = get_spec_file_name(registry) + # remove the leading "specs/" from the path using CACHE_FOLDER without_folder = spec_cache_path.replace(f"{CACHE_FOLDER}/", "") - without_file = without_folder.replace(f"/{SPEC_FILE_NAME}", "") + without_file = without_folder.replace(f"/{registry_file_name}", "") # split on only the last "/" to get the docker repository and tag # this is because the docker repository can have "/" in it @@ -38,33 +71,50 @@ def get_docker_info_from_spec_cache_path(spec_cache_path: str) -> CachedSpec: docker_repository = without_file.replace(f"/{docker_image_tag}", "") return CachedSpec( - docker_repository=docker_repository, - docker_image_tag=docker_image_tag, - spec_cache_path=spec_cache_path, + docker_repository=docker_repository, docker_image_tag=docker_image_tag, spec_cache_path=spec_cache_path, registry=registry ) -def is_spec_cached(docker_repository: str, docker_image_tag: str) -> bool: - """Returns True if the spec.json file exists in the spec cache bucket.""" - spec_path = get_spec_cache_path(docker_repository, docker_image_tag) +class SpecCache: + def __init__(self, bucket_name: str = PROD_SPEC_CACHE_BUCKET_NAME): + self.client = storage.Client.create_anonymous_client() + self.bucket = self.client.bucket(bucket_name) + self.cached_specs = self.get_all_cached_specs() + + def get_all_cached_specs(self) -> List[CachedSpec]: + """Returns a list of all the specs in the spec cache bucket.""" + + blobs = self.bucket.list_blobs(prefix=CACHE_FOLDER) + + return [get_docker_info_from_spec_cache_path(blob.name) for blob in blobs if blob.name.endswith(".json")] - client = storage.Client.create_anonymous_client() - bucket = client.bucket(SPEC_CACHE_BUCKET_NAME) - blob = bucket.blob(spec_path) + def _find_spec_cache(self, docker_repository: str, docker_image_tag: str, registry: Registries) -> CachedSpec: + """Returns the spec cache path for a given docker repository and tag.""" - return blob.exists() + # find the spec cache path for the given docker repository and tag + for cached_spec in self.cached_specs: + if ( + cached_spec.docker_repository == docker_repository + and cached_spec.registry == registry + and cached_spec.docker_image_tag == docker_image_tag + ): + return cached_spec + return None -def list_cached_specs() -> List[CachedSpec]: - """Returns a list of all the specs in the spec cache bucket.""" - client = storage.Client.create_anonymous_client() - bucket = client.bucket(SPEC_CACHE_BUCKET_NAME) - blobs = bucket.list_blobs(prefix=CACHE_FOLDER) + def find_spec_cache_with_fallback(self, docker_repository: str, docker_image_tag: str, registry_str: str) -> CachedSpec: + """Returns the spec cache path for a given docker repository and tag and fallback to OSS if none found""" + registry = Registries(registry_str) - return [get_docker_info_from_spec_cache_path(blob.name) for blob in blobs] + # if the registry is cloud try to return the cloud spec first + if registry == Registries.CLOUD: + spec_cache = self._find_spec_cache(docker_repository, docker_image_tag, registry) + if spec_cache: + return spec_cache + # fallback to OSS + return self._find_spec_cache(docker_repository, docker_image_tag, Registries.OSS) -def get_cached_spec(spec_cache_path: str) -> dict: - client = storage.Client.create_anonymous_client() - bucket = client.bucket(SPEC_CACHE_BUCKET_NAME) - return json.loads(bucket.blob(spec_cache_path).download_as_string()) + def download_spec(self, spec: CachedSpec) -> dict: + """Downloads the spec from the spec cache bucket.""" + return json.loads(self.bucket.blob(spec.spec_cache_path).download_as_string()) diff --git a/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py b/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py index bc8840eb019c..2e5e38f50e9c 100644 --- a/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py +++ b/airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py @@ -16,6 +16,7 @@ @dataclass(frozen=True) class ValidatorOptions: + docs_path: str prerelease_tag: Optional[str] = None @@ -25,15 +26,8 @@ class ValidatorOptions: # TODO: Remove these when each of these connectors ship any new version ALREADY_ON_MAJOR_VERSION_EXCEPTIONS = [ ("airbyte/source-prestashop", "1.0.0"), - ("airbyte/source-onesignal", "1.0.0"), ("airbyte/source-yandex-metrica", "1.0.0"), - ("airbyte/destination-meilisearch", "1.0.0"), ("airbyte/destination-csv", "1.0.0"), - ("airbyte/source-metabase", "1.0.0"), - ("airbyte/source-typeform", "1.0.0"), - ("airbyte/source-recharge", "1.0.0"), - ("airbyte/source-pipedrive", "1.0.0"), - ("airbyte/source-paypal-transaction", "2.0.0"), ] @@ -70,7 +64,7 @@ def validate_metadata_images_in_dockerhub( print(f"Checking that the following images are on dockerhub: {images_to_check}") for image, version in images_to_check: - if not is_image_on_docker_hub(image, version): + if not is_image_on_docker_hub(image, version, retries=3): return False, f"Image {image}:{version} does not exist in DockerHub" return True, None @@ -137,10 +131,62 @@ def validate_major_version_bump_has_breaking_change_entry( return True, None +def validate_docs_path_exists(metadata_definition: ConnectorMetadataDefinitionV0, validator_opts: ValidatorOptions) -> ValidationResult: + """Ensure that the doc_path exists.""" + if not pathlib.Path(validator_opts.docs_path).exists(): + return False, f"Could not find {validator_opts.docs_path}." + + return True, None + + +def validate_metadata_base_images_in_dockerhub( + metadata_definition: ConnectorMetadataDefinitionV0, validator_opts: ValidatorOptions +) -> ValidationResult: + metadata_definition_dict = metadata_definition.dict() + + image_address = get(metadata_definition_dict, "data.connectorBuildOptions.baseImage") + if image_address is None: + return True, None + + try: + image_name, tag_with_sha_prefix, digest = image_address.split(":") + # As we query the DockerHub API we need to remove the docker.io prefix + image_name = image_name.replace("docker.io/", "") + except ValueError: + return False, f"Image {image_address} is not in the format :@" + tag = tag_with_sha_prefix.split("@")[0] + + print(f"Checking that the base images is on dockerhub: {image_address}") + + if not is_image_on_docker_hub(image_name, tag, digest, retries=3): + return False, f"Image {image_address} does not exist in DockerHub" + + return True, None + + +def validate_pypi_only_for_python( + metadata_definition: ConnectorMetadataDefinitionV0, _validator_opts: ValidatorOptions +) -> ValidationResult: + """Ensure that if pypi publishing is enabled for a connector, it has a python language tag.""" + + pypi_enabled = get(metadata_definition, "data.remoteRegistries.pypi.enabled", False) + if not pypi_enabled: + return True, None + + tags = get(metadata_definition, "data.tags", []) + if "language:python" not in tags and "language:low-code" not in tags: + return False, "If pypi publishing is enabled, the connector must have a python language tag." + + return True, None + + PRE_UPLOAD_VALIDATORS = [ validate_all_tags_are_keyvalue_pairs, validate_at_least_one_language_tag, validate_major_version_bump_has_breaking_change_entry, + validate_docs_path_exists, + validate_metadata_base_images_in_dockerhub, + validate_pypi_only_for_python, ] POST_UPLOAD_VALIDATORS = PRE_UPLOAD_VALIDATORS + [ @@ -151,7 +197,7 @@ def validate_major_version_bump_has_breaking_change_entry( def validate_and_load( file_path: pathlib.Path, validators_to_run: List[Validator], - validator_opts: ValidatorOptions = ValidatorOptions(), + validator_opts: ValidatorOptions, ) -> Tuple[Optional[ConnectorMetadataDefinitionV0], Optional[ValidationError]]: """Load a metadata file from a path (runs jsonschema validation) and run optional extra validators. @@ -167,6 +213,7 @@ def validate_and_load( return None, f"Validation error: {e}" for validator in validators_to_run: + print(f"Running validator: {validator.__name__}") is_valid, error = validator(metadata_model, validator_opts) if not is_valid: return None, f"Validation error: {error}" diff --git a/airbyte-ci/connectors/metadata_service/lib/pyproject.toml b/airbyte-ci/connectors/metadata_service/lib/pyproject.toml index 860a58c88bdb..e787e3469ac2 100644 --- a/airbyte-ci/connectors/metadata_service/lib/pyproject.toml +++ b/airbyte-ci/connectors/metadata_service/lib/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "metadata-service" -version = "0.1.4" +version = "0.3.3" description = "" authors = ["Ben Church "] readme = "README.md" diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/__init__.py b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/__init__.py index b6f56dc87332..71d7203499ae 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/__init__.py +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/__init__.py @@ -1,33 +1,53 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import os from typing import Callable, List import pytest +from metadata_service.constants import DOC_FILE_NAME def list_all_paths_in_fixture_directory(folder_name: str) -> List[str]: file_path = os.path.join(os.path.dirname(__file__), folder_name) + + # If folder_name has subdirectories, os.walk will return a list of tuples, + # one for folder_name and one for each of its subdirectories. + fixture_files = [] for root, dirs, files in os.walk(file_path): - return [os.path.join(root, file_name) for file_name in files] + fixture_files.extend(os.path.join(root, file_name) for file_name in files) + return fixture_files @pytest.fixture(scope="session") def valid_metadata_yaml_files() -> List[str]: - return list_all_paths_in_fixture_directory("metadata_validate/valid") + files = list_all_paths_in_fixture_directory("metadata_validate/valid") + if not files: + pytest.fail("No files found in metadata_validate/valid") + return files @pytest.fixture(scope="session") def invalid_metadata_yaml_files() -> List[str]: - return list_all_paths_in_fixture_directory("metadata_validate/invalid") + files = list_all_paths_in_fixture_directory("metadata_validate/invalid") + if not files: + pytest.fail("No files found in metadata_validate/invalid") + return files @pytest.fixture(scope="session") def valid_metadata_upload_files() -> List[str]: - return list_all_paths_in_fixture_directory("metadata_upload/valid") + files = list_all_paths_in_fixture_directory("metadata_upload/valid") + if not files: + pytest.fail("No files found in metadata_upload/valid") + return files @pytest.fixture(scope="session") def invalid_metadata_upload_files() -> List[str]: - return list_all_paths_in_fixture_directory("metadata_upload/invalid") + files = list_all_paths_in_fixture_directory("metadata_upload/invalid") + if not files: + pytest.fail("No files found in metadata_upload/invalid") + return files @pytest.fixture(scope="session") diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/doc.md b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/doc.md new file mode 100644 index 000000000000..8c70aca89908 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/doc.md @@ -0,0 +1 @@ +# The test doc for metadata_validate \ No newline at end of file diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/repo_nonexistent/metadata_cloud_repo_does_not_exist.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/repo_nonexistent/metadata_cloud_repo_does_not_exist.yaml similarity index 97% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/repo_nonexistent/metadata_cloud_repo_does_not_exist.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/repo_nonexistent/metadata_cloud_repo_does_not_exist.yaml index 3e7a69aca256..7c64d2c6da62 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/repo_nonexistent/metadata_cloud_repo_does_not_exist.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/repo_nonexistent/metadata_cloud_repo_does_not_exist.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/repo_nonexistent/metadata_main_repo_does_not_exist.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/repo_nonexistent/metadata_main_repo_does_not_exist.yaml similarity index 97% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/repo_nonexistent/metadata_main_repo_does_not_exist.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/repo_nonexistent/metadata_main_repo_does_not_exist.yaml index c54651bab24d..d8ae617736c3 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/repo_nonexistent/metadata_main_repo_does_not_exist.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/repo_nonexistent/metadata_main_repo_does_not_exist.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/does-not-exist-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/repo_nonexistent/metadata_normalization_repo_does_not_exist.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/repo_nonexistent/metadata_normalization_repo_does_not_exist.yaml similarity index 97% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/repo_nonexistent/metadata_normalization_repo_does_not_exist.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/repo_nonexistent/metadata_normalization_repo_does_not_exist.yaml index 4c882b0b9110..df801eebb8ec 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/repo_nonexistent/metadata_normalization_repo_does_not_exist.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/repo_nonexistent/metadata_normalization_repo_does_not_exist.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/repo_nonexistent/metadata_oss_repo_does_not_exist.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/repo_nonexistent/metadata_oss_repo_does_not_exist.yaml new file mode 100644 index 000000000000..022f4ac965b3 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/repo_nonexistent/metadata_oss_repo_does_not_exist.yaml @@ -0,0 +1,27 @@ +metadataSpecVersion: 1.0 +data: + name: AlloyDB for PostgreSQL + definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 + connectorType: source + dockerRepository: airbyte/exists-1 + githubIssueLabel: source-alloydb-strict-encrypt + dockerImageTag: 0.0.1 + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource + connectorSubtype: database + releaseStage: generally_available + license: MIT + normalizationConfig: + normalizationIntegrationType: postgres + normalizationRepository: airbyte/exists-2 + normalizationTag: 0.0.1 + registries: + cloud: + enabled: true + dockerRepository: airbyte/exists-3 + dockerImageTag: 0.0.1 + oss: + enabled: true + dockerRepository: airbyte/does-not-exist-4 + dockerImageTag: 0.0.1 + tags: + - language:java diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/tag_nonexistent/metadata_breaking_change_image_tag_does_not_exist.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/tag_nonexistent/metadata_breaking_change_image_tag_does_not_exist.yaml similarity index 94% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/tag_nonexistent/metadata_breaking_change_image_tag_does_not_exist.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/tag_nonexistent/metadata_breaking_change_image_tag_does_not_exist.yaml index 0b4174048ec0..685f75ed08e1 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/tag_nonexistent/metadata_breaking_change_image_tag_does_not_exist.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/tag_nonexistent/metadata_breaking_change_image_tag_does_not_exist.yaml @@ -16,7 +16,7 @@ data: releaseStage: alpha releases: breakingChanges: - 6.0.0: # tag does not exist + 0.0.0: # tag does not exist upgradeDeadline: 2023-08-22 message: "This version made a change." documentationUrl: https://docs.airbyte.com/integrations/sources/onesignal diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/tag_nonexistent/metadata_cloud_image_tag_does_not_exist.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/tag_nonexistent/metadata_cloud_image_tag_does_not_exist.yaml similarity index 91% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/tag_nonexistent/metadata_cloud_image_tag_does_not_exist.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/tag_nonexistent/metadata_cloud_image_tag_does_not_exist.yaml index 1b74d41fc198..799a7d4f3424 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/tag_nonexistent/metadata_cloud_image_tag_does_not_exist.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/tag_nonexistent/metadata_cloud_image_tag_does_not_exist.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT @@ -18,7 +18,7 @@ data: cloud: enabled: true dockerRepository: airbyte/exists-3 - dockerImageTag: 6.6.6 # tag does not exist + dockerImageTag: 99.99.99 # tag does not exist oss: enabled: true dockerRepository: airbyte/exists-4 diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/tag_nonexistent/metadata_main_image_tag_does_not_exist.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/tag_nonexistent/metadata_main_image_tag_does_not_exist.yaml similarity index 90% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/tag_nonexistent/metadata_main_image_tag_does_not_exist.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/tag_nonexistent/metadata_main_image_tag_does_not_exist.yaml index dfada6f04316..5e5732263165 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/tag_nonexistent/metadata_main_image_tag_does_not_exist.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/tag_nonexistent/metadata_main_image_tag_does_not_exist.yaml @@ -5,8 +5,8 @@ data: connectorType: source dockerRepository: airbyte/exists-1 githubIssueLabel: source-alloydb-strict-encrypt - dockerImageTag: 6.6.6 # tag does not exist - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + dockerImageTag: 99.99.99 # tag does not exist + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/tag_nonexistent/metadata_normalization_image_tag_does_not_exist.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/tag_nonexistent/metadata_normalization_image_tag_does_not_exist.yaml similarity index 91% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/tag_nonexistent/metadata_normalization_image_tag_does_not_exist.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/tag_nonexistent/metadata_normalization_image_tag_does_not_exist.yaml index 5e9bec13dbaa..7f4375c486f2 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/tag_nonexistent/metadata_normalization_image_tag_does_not_exist.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/tag_nonexistent/metadata_normalization_image_tag_does_not_exist.yaml @@ -6,14 +6,14 @@ data: dockerRepository: airbyte/exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT normalizationConfig: normalizationIntegrationType: postgres normalizationRepository: airbyte/exists-2 - dockerImageTag: 6.6.6 # tag does not exist + normalizationTag: 99.99.99 # tag does not exist registries: cloud: enabled: true diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/tag_nonexistent/metadata_oss_repo_does_not_exist.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/tag_nonexistent/metadata_oss_repo_does_not_exist.yaml new file mode 100644 index 000000000000..ffb3735e5655 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/tag_nonexistent/metadata_oss_repo_does_not_exist.yaml @@ -0,0 +1,27 @@ +metadataSpecVersion: 1.0 +data: + name: AlloyDB for PostgreSQL + definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 + connectorType: source + dockerRepository: airbyte/exists-1 + githubIssueLabel: source-alloydb-strict-encrypt + dockerImageTag: 0.0.1 + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource + connectorSubtype: database + releaseStage: generally_available + license: MIT + normalizationConfig: + normalizationIntegrationType: postgres + normalizationRepository: airbyte/exists-2 + normalizationTag: 0.0.1 + registries: + cloud: + enabled: true + dockerRepository: airbyte/exists-3 + dockerImageTag: 0.0.1 + oss: + enabled: true + dockerRepository: airbyte/exists-4 + dockerImageTag: 99.99.99 # tag does not exist + tags: + - language:java diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/valid_overrides_but_base_image_nonexistent/metadata_main_image_tag_does_not_exist_but_is_overrode.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/valid_overrides_but_image_nonexistent/metadata_main_image_tag_does_not_exist_but_is_overrode.yaml similarity index 92% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/valid_overrides_but_base_image_nonexistent/metadata_main_image_tag_does_not_exist_but_is_overrode.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/valid_overrides_but_image_nonexistent/metadata_main_image_tag_does_not_exist_but_is_overrode.yaml index 3ef1246722b1..81ee107bd267 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/valid_overrides_but_base_image_nonexistent/metadata_main_image_tag_does_not_exist_but_is_overrode.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/valid_overrides_but_image_nonexistent/metadata_main_image_tag_does_not_exist_but_is_overrode.yaml @@ -5,8 +5,8 @@ data: connectorType: source dockerRepository: airbyte/exists-1 githubIssueLabel: source-alloydb-strict-encrypt - dockerImageTag: 6.6.6 # tag does not exist - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + dockerImageTag: 99.99.99 # tag does not exist + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/valid_overrides_but_base_image_nonexistent/metadata_main_repo_does_not_exist_but_is_overrode.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/valid_overrides_but_image_nonexistent/metadata_main_repo_does_not_exist_but_is_overrode.yaml similarity index 97% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/valid_overrides_but_base_image_nonexistent/metadata_main_repo_does_not_exist_but_is_overrode.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/valid_overrides_but_image_nonexistent/metadata_main_repo_does_not_exist_but_is_overrode.yaml index d1522b9e8fe1..f06ea13cecd1 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/valid_overrides_but_base_image_nonexistent/metadata_main_repo_does_not_exist_but_is_overrode.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/referenced_image_not_in_dockerhub/valid_overrides_but_image_nonexistent/metadata_main_repo_does_not_exist_but_is_overrode.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/does-not-exist-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/repo_nonexistent/metadata_oss_repo_does_not_exist.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/repo_nonexistent/metadata_oss_repo_does_not_exist.yaml deleted file mode 100644 index 2e233dcd11d4..000000000000 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/repo_nonexistent/metadata_oss_repo_does_not_exist.yaml +++ /dev/null @@ -1,27 +0,0 @@ -metadataSpecVersion: 1.0 -data: - name: AlloyDB for PostgreSQL - definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 - connectorType: source - dockerRepository: airbyte/exists-1 - githubIssueLabel: source-alloydb-strict-encrypt - dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb - connectorSubtype: database - releaseStage: generally_available - license: MIT - normalizationConfig: - normalizationIntegrationType: postgres - normalizationRepository: airbyte/exists-2 - normalizationTag: 0.0.1 - registries: - cloud: - enabled: true - dockerRepository: airbyte/exists-3 - dockerImageTag: 0.0.1 - oss: - enabled: true - dockerRepository: airbyte/does-not-exist-4 - dockerImageTag: 0.0.1 - tags: - - language:java diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/tag_nonexistent/metadata_oss_repo_does_not_exist.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/tag_nonexistent/metadata_oss_repo_does_not_exist.yaml deleted file mode 100644 index 0e9f4a20d1b8..000000000000 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/invalid/tag_nonexistent/metadata_oss_repo_does_not_exist.yaml +++ /dev/null @@ -1,27 +0,0 @@ -metadataSpecVersion: 1.0 -data: - name: AlloyDB for PostgreSQL - definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 - connectorType: source - dockerRepository: airbyte/exists-1 - githubIssueLabel: source-alloydb-strict-encrypt - dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb - connectorSubtype: database - releaseStage: generally_available - license: MIT - normalizationConfig: - normalizationIntegrationType: postgres - normalizationRepository: airbyte/exists-2 - normalizationTag: 0.0.1 - registries: - cloud: - enabled: true - dockerRepository: airbyte/exists-3 - dockerImageTag: 0.0.1 - oss: - enabled: true - dockerRepository: airbyte/exists-4 - dockerImageTag: 6.6.6 # tag does not exist - tags: - - language:java diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/valid/metadata_all_images_exist.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/valid/referenced_image_in_dockerhub/metadata_all_images_exist.yaml similarity index 98% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/valid/metadata_all_images_exist.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/valid/referenced_image_in_dockerhub/metadata_all_images_exist.yaml index 725b3288fe18..1f5cda30a331 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/valid/metadata_all_images_exist.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/valid/referenced_image_in_dockerhub/metadata_all_images_exist.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/valid/metadata_all_images_exist_no_overrides.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/valid/referenced_image_in_dockerhub/metadata_all_images_exist_no_overrides.yaml similarity index 96% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/valid/metadata_all_images_exist_no_overrides.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/valid/referenced_image_in_dockerhub/metadata_all_images_exist_no_overrides.yaml index e2be9955653d..d2067be338f0 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/valid/metadata_all_images_exist_no_overrides.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/valid/referenced_image_in_dockerhub/metadata_all_images_exist_no_overrides.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/valid/metadata_all_images_exist_no_overrides_with_normalization.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/valid/referenced_image_in_dockerhub/metadata_all_images_exist_no_overrides_with_normalization.yaml similarity index 97% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/valid/metadata_all_images_exist_no_overrides_with_normalization.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/valid/referenced_image_in_dockerhub/metadata_all_images_exist_no_overrides_with_normalization.yaml index 238b2f572614..74be91d596ef 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/valid/metadata_all_images_exist_no_overrides_with_normalization.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/valid/referenced_image_in_dockerhub/metadata_all_images_exist_no_overrides_with_normalization.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/valid/referenced_image_in_dockerhub/metadata_base_image_exists.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/valid/referenced_image_in_dockerhub/metadata_base_image_exists.yaml new file mode 100644 index 000000000000..9fae02a082aa --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_upload/valid/referenced_image_in_dockerhub/metadata_base_image_exists.yaml @@ -0,0 +1,29 @@ +data: + ab_internal: + ql: 400 + sl: 200 + allowedHosts: + hosts: + - zopim.com + connectorBuildOptions: + baseImage: docker.io/airbyte/base-repo-exists:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + connectorSubtype: api + connectorType: source + definitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4 + dockerImageTag: 0.2.1 + dockerRepository: airbyte/source-exists-1 + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource + githubIssueLabel: source-alloy-db + icon: alloy-db.svg + license: MIT + name: AlloyDB for PostgreSQL + registries: + cloud: + enabled: true + oss: + enabled: true + releaseStage: generally_available + supportLevel: certified + tags: + - language:python +metadataSpecVersion: "1.0" diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_extra_data.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_extra_data.yaml similarity index 96% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_extra_data.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_extra_data.yaml index 62217ec5b6b5..d412853bb354 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_extra_data.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_extra_data.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_no_tags.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_no_tags.yaml deleted file mode 100644 index 5bc45e353d4a..000000000000 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_no_tags.yaml +++ /dev/null @@ -1,12 +0,0 @@ -metadataSpecVersion: 1.0 -data: - name: AlloyDB for PostgreSQL - definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 - connectorType: source - dockerRepository: airbyte/source-alloydb-strict-encrypt - githubIssueLabel: source-alloydb-strict-encrypt - dockerImageTag: 2.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb - connectorSubtype: database - releaseStage: generally_available - license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/connector_build_options_invalid/metadata_base_image_digest_does_not_exists.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/connector_build_options_invalid/metadata_base_image_digest_does_not_exists.yaml new file mode 100644 index 000000000000..0f20fe4d28c8 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/connector_build_options_invalid/metadata_base_image_digest_does_not_exists.yaml @@ -0,0 +1,29 @@ +data: + ab_internal: + ql: 400 + sl: 200 + allowedHosts: + hosts: + - zopim.com + connectorBuildOptions: + baseImage: docker.io/airbyte/base-repo-exists:1.1.0@sha256:MISSINGSHA + connectorSubtype: api + connectorType: source + definitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4 + dockerImageTag: 0.2.1 + dockerRepository: airbyte/source-exists-1 + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource + githubIssueLabel: source-alloy-db + icon: alloy-db.svg + license: MIT + name: AlloyDB for PostgreSQL + registries: + cloud: + enabled: true + oss: + enabled: true + releaseStage: generally_available + supportLevel: certified + tags: + - language:python +metadataSpecVersion: "1.0" diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/connector_build_options_invalid/metadata_base_image_name_does_not_exists.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/connector_build_options_invalid/metadata_base_image_name_does_not_exists.yaml new file mode 100644 index 000000000000..9937ef11cda9 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/connector_build_options_invalid/metadata_base_image_name_does_not_exists.yaml @@ -0,0 +1,29 @@ +data: + ab_internal: + ql: 400 + sl: 200 + allowedHosts: + hosts: + - zopim.com + connectorBuildOptions: + baseImage: docker.io/airbyte/foobar-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + connectorSubtype: api + connectorType: source + definitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4 + dockerImageTag: 0.2.1 + dockerRepository: airbyte/source-exists-1 + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource + githubIssueLabel: source-alloy-db + icon: alloy-db.svg + license: MIT + name: AlloyDB for PostgreSQL + registries: + cloud: + enabled: true + oss: + enabled: true + releaseStage: generally_available + supportLevel: certified + tags: + - language:python +metadataSpecVersion: "1.0" diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/connector_build_options_invalid/metadata_base_image_tag_does_not_exists.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/connector_build_options_invalid/metadata_base_image_tag_does_not_exists.yaml new file mode 100644 index 000000000000..fed77cbd1399 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/connector_build_options_invalid/metadata_base_image_tag_does_not_exists.yaml @@ -0,0 +1,29 @@ +data: + ab_internal: + ql: 400 + sl: 200 + allowedHosts: + hosts: + - zopim.com + connectorBuildOptions: + baseImage: docker.io/airbyte/base-repo-exists:99.99.99@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + connectorSubtype: api + connectorType: source + definitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4 + dockerImageTag: 0.2.1 + dockerRepository: airbyte/source-exists-1 + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource + githubIssueLabel: source-alloy-db + icon: alloy-db.svg + license: MIT + name: AlloyDB for PostgreSQL + registries: + cloud: + enabled: true + oss: + enabled: true + releaseStage: generally_available + supportLevel: certified + tags: + - language:python +metadataSpecVersion: "1.0" diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/connector_build_options_invalid/metadata_build_base_image_wrong_type.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/connector_build_options_invalid/metadata_build_base_image_wrong_type.yaml new file mode 100644 index 000000000000..b3ecd5aee1f7 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/connector_build_options_invalid/metadata_build_base_image_wrong_type.yaml @@ -0,0 +1,29 @@ +data: + allowedHosts: + hosts: + - "*.googleapis.com" + connectorBuildOptions: + unexpectedField: additionalProperties are not allowed ('unexpectedField' was unexpected) + connectorSubtype: file + connectorType: source + definitionId: 71607ba1-c0ac-4799-8049-7f4b90dd50f7 + dockerImageTag: 0.3.7 + dockerRepository: airbyte/source-google-sheets + githubIssueLabel: source-google-sheets + icon: google-sheets.svg + license: Elv2 + name: Google Sheets + registries: + cloud: + enabled: true + oss: + enabled: true + releaseStage: generally_available + documentationUrl: https://docs.airbyte.com/integrations/sources/google-sheets + tags: + - language:python + ab_internal: + sl: 300 + ql: 400 + supportLevel: certified +metadataSpecVersion: "1.0" diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/connector_build_options_invalid/metadata_invalid_base_image_no_sha.yml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/connector_build_options_invalid/metadata_invalid_base_image_no_sha.yml new file mode 100644 index 000000000000..91de8734a4bf --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/connector_build_options_invalid/metadata_invalid_base_image_no_sha.yml @@ -0,0 +1,29 @@ +data: + allowedHosts: + hosts: + - "*.googleapis.com" + connectorBuildOptions: + baseImage: docker.io/airbyte/base-repo-exists:1.1.0 + connectorSubtype: file + connectorType: source + definitionId: 71607ba1-c0ac-4799-8049-7f4b90dd50f7 + dockerImageTag: 0.3.7 + dockerRepository: airbyte/source-google-sheets + githubIssueLabel: source-google-sheets + icon: google-sheets.svg + license: Elv2 + name: Google Sheets + registries: + cloud: + enabled: true + oss: + enabled: true + releaseStage: generally_available + documentationUrl: https://docs.airbyte.com/integrations/sources/google-sheets + tags: + - language:python + ab_internal: + sl: 300 + ql: 400 + supportLevel: certified +metadataSpecVersion: "1.0" diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_internal_fields.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/metadata_invalid_internal_fields.yaml similarity index 96% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_internal_fields.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/metadata_invalid_internal_fields.yaml index a7d1e8a1ee8a..db4b261bd977 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_internal_fields.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/metadata_invalid_internal_fields.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/metadata_invalid_remote_registries.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/metadata_invalid_remote_registries.yaml new file mode 100644 index 000000000000..1e1d095e6333 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/metadata_invalid_remote_registries.yaml @@ -0,0 +1,16 @@ +metadataSpecVersion: 1.0 +data: + name: AlloyDB for PostgreSQL + definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 + connectorType: source + dockerRepository: airbyte/image-exists-1 + githubIssueLabel: source-alloydb-strict-encrypt + dockerImageTag: 0.0.1 + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource + connectorSubtype: database + releaseStage: generally_available + remoteRegistries: + maven: enabled + license: MIT + tags: + - language:java diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_unknown_support_level.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/metadata_unknown_support_level.yaml similarity index 96% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_unknown_support_level.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/metadata_unknown_support_level.yaml index 0e4d785ac6e7..9fe7b5ededaf 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_unknown_support_level.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/metadata_unknown_support_level.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available supportLevel: dne diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/metadata_wrong_language_remote_registries.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/metadata_wrong_language_remote_registries.yaml new file mode 100644 index 000000000000..c77bfde70bd4 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/metadata_wrong_language_remote_registries.yaml @@ -0,0 +1,18 @@ +metadataSpecVersion: 1.0 +data: + name: AlloyDB for PostgreSQL + definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 + connectorType: source + dockerRepository: airbyte/image-exists-1 + githubIssueLabel: source-alloydb-strict-encrypt + dockerImageTag: 0.0.1 + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource + connectorSubtype: database + releaseStage: generally_available + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-alloydb-strict-encrypt + license: MIT + tags: + - language:java diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_registry_no_id_override.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/overrides_invalid/metadata_registry_no_id_override.yaml similarity index 97% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_registry_no_id_override.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/overrides_invalid/metadata_registry_no_id_override.yaml index 50a5b70a2e5d..a33e5913e889 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_registry_no_id_override.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/overrides_invalid/metadata_registry_no_id_override.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_registry_no_type_override.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/overrides_invalid/metadata_registry_no_type_override.yaml similarity index 97% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_registry_no_type_override.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/overrides_invalid/metadata_registry_no_type_override.yaml index d40962371471..812ac0ef70c7 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_registry_no_type_override.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/overrides_invalid/metadata_registry_no_type_override.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_registry_unknown_override.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/overrides_invalid/metadata_registry_unknown_override.yaml similarity index 97% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_registry_unknown_override.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/overrides_invalid/metadata_registry_unknown_override.yaml index e8184ce98828..06e5710cf868 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_registry_unknown_override.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/overrides_invalid/metadata_registry_unknown_override.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/impact_scopes_invalid/metadata_breaking_changes_empty_impacted_stream.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/impact_scopes_invalid/metadata_breaking_changes_empty_impacted_stream.yaml new file mode 100644 index 000000000000..3351eb5ddee5 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/impact_scopes_invalid/metadata_breaking_changes_empty_impacted_stream.yaml @@ -0,0 +1,23 @@ +metadataSpecVersion: 1.0 +data: + name: AlloyDB for PostgreSQL + definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 + connectorType: source + dockerRepository: airbyte/image-exists-1 + githubIssueLabel: source-alloydb-strict-encrypt + dockerImageTag: 0.0.1 + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource + connectorSubtype: database + releaseStage: generally_available + license: MIT + releases: + breakingChanges: + 2.0.0: + upgradeDeadline: 2023-08-22 + message: "This version changes the connector’s authentication method from `ApiKey` to `oAuth`, per the [API guide](https://amazon-sqs.com/api/someguide)." + scopedImpact: + - scopeType: stream + impactedScopes: [] + + tags: + - language:java diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/impact_scopes_invalid/metadata_breaking_changes_impact_scopes_unknown_scope_type.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/impact_scopes_invalid/metadata_breaking_changes_impact_scopes_unknown_scope_type.yaml new file mode 100644 index 000000000000..9b4176acfd5c --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/impact_scopes_invalid/metadata_breaking_changes_impact_scopes_unknown_scope_type.yaml @@ -0,0 +1,23 @@ +metadataSpecVersion: 1.0 +data: + name: AlloyDB for PostgreSQL + definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 + connectorType: source + dockerRepository: airbyte/image-exists-1 + githubIssueLabel: source-alloydb-strict-encrypt + dockerImageTag: 0.0.1 + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource + connectorSubtype: database + releaseStage: generally_available + license: MIT + releases: + breakingChanges: + 2.0.0: + upgradeDeadline: 2023-08-22 + message: "This version changes the connector’s authentication method from `ApiKey` to `oAuth`, per the [API guide](https://amazon-sqs.com/api/someguide)." + scopedImpact: + - type: foo + impactedScopes: ["bar"] + + tags: + - language:java diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_breaking_change_versions_under_releases.yml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_breaking_change_versions_under_releases.yml new file mode 100644 index 000000000000..27e2c17d1abf --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_breaking_change_versions_under_releases.yml @@ -0,0 +1,18 @@ +metadataSpecVersion: 1.0 +data: + name: AlloyDB for PostgreSQL + definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 + connectorType: source + dockerRepository: airbyte/image-exists-1 + githubIssueLabel: source-alloydb-strict-encrypt + dockerImageTag: 0.0.1 + documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + connectorSubtype: database + releaseStage: generally_available + license: MIT + releasestests/fixtures/metadata_validate/invalid/metadata_breaking_change_versions_under_releases.yml: + 2.1.3: + message: "This version changes the connector’s authentication method from `ApiKey` to `oAuth`, per the [API guide](https://amazon-sqs.com/api/someguide)." + upgradeDeadline: 2023-08-22 + tags: + - language:java diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_breaking_changes_not_under_releases.yml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_breaking_changes_not_under_releases.yml new file mode 100644 index 000000000000..e2b9918cccf0 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_breaking_changes_not_under_releases.yml @@ -0,0 +1,18 @@ +metadataSpecVersion: 1.0 +data: + name: AlloyDB for PostgreSQL + definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 + connectorType: source + dockerRepository: airbyte/image-exists-1 + githubIssueLabel: source-alloydb-strict-encrypt + dockerImageTag: 0.0.1 + documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + connectorSubtype: database + releaseStage: generally_available + license: MIT + breakingChanges: + 2.1.3: + message: "This version changes the connector’s authentication method from `ApiKey` to `oAuth`, per the [API guide](https://amazon-sqs.com/api/someguide)." + upgradeDeadline: 2023-08-22 + tags: + - language:java diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_additional_property.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_invalid_breaking_change_additional_property.yaml similarity index 97% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_additional_property.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_invalid_breaking_change_additional_property.yaml index aa60bb77ce29..03f3579470fa 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_additional_property.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_invalid_breaking_change_additional_property.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_invalid_deadline.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_invalid_breaking_change_invalid_deadline.yaml similarity index 97% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_invalid_deadline.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_invalid_breaking_change_invalid_deadline.yaml index cfa128302081..a4b3891435b1 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_invalid_deadline.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_invalid_breaking_change_invalid_deadline.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_no_deadline.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_invalid_breaking_change_no_deadline.yaml similarity index 97% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_no_deadline.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_invalid_breaking_change_no_deadline.yaml index 237a3295bc44..458fdccce291 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_no_deadline.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_invalid_breaking_change_no_deadline.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_no_message.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_invalid_breaking_change_no_message.yaml similarity index 96% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_no_message.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_invalid_breaking_change_no_message.yaml index 250c5ff6fe78..73572c41a5d6 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_no_message.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_invalid_breaking_change_no_message.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_version.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_invalid_breaking_change_version.yaml similarity index 97% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_version.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_invalid_breaking_change_version.yaml index c623e4898e3e..76c15e44d7c2 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_breaking_change_version.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_invalid_breaking_change_version.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_major_version_no_breaking_change_entry_for_version.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_major_version_no_breaking_change_entry_for_version.yaml similarity index 81% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_major_version_no_breaking_change_entry_for_version.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_major_version_no_breaking_change_entry_for_version.yaml index 46f419ed2838..86bdde803417 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_major_version_no_breaking_change_entry_for_version.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_major_version_no_breaking_change_entry_for_version.yaml @@ -16,9 +16,10 @@ data: enabled: true releaseStage: alpha releases: - 1.0.0: - upgradeDeadline: 2023-08-22 - message: "This version made a change." + breakingChanges: + 1.0.0: + upgradeDeadline: 2023-08-22 + message: "This version made a change." documentationUrl: https://docs.airbyte.com/integrations/sources/onesignal tags: - language:python diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_major_version_no_releases.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_major_version_no_releases.yaml similarity index 100% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_major_version_no_releases.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/optional_top_level_property_invalid/releases_invalid/metadata_major_version_no_releases.yaml diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_empty_tags.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_invalid/tags_invalid/metadata_empty_tags.yaml similarity index 96% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_empty_tags.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_invalid/tags_invalid/metadata_empty_tags.yaml index c5d336b6ac5e..90f558ad64e3 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_empty_tags.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_invalid/tags_invalid/metadata_empty_tags.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/source-alloydb-strict-encrypt githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 2.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_tag_format.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_invalid/tags_invalid/metadata_invalid_tag_format.yaml similarity index 96% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_tag_format.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_invalid/tags_invalid/metadata_invalid_tag_format.yaml index 72642a22f465..f5416208fa3e 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_invalid_tag_format.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_invalid/tags_invalid/metadata_invalid_tag_format.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/source-alloydb-strict-encrypt githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 2.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_no_lang_tag.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_invalid/tags_invalid/metadata_no_lang_tag.yaml similarity index 96% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_no_lang_tag.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_invalid/tags_invalid/metadata_no_lang_tag.yaml index 6abe37436019..a065f3e742c7 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_no_lang_tag.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_invalid/tags_invalid/metadata_no_lang_tag.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/source-alloydb-strict-encrypt githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 2.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_connector_type.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_missing/metadata_missing_connector_type.yaml similarity index 95% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_connector_type.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_missing/metadata_missing_connector_type.yaml index 609d95d29518..3222f6bdd974 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_connector_type.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_missing/metadata_missing_connector_type.yaml @@ -5,7 +5,7 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_definition_id.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_missing/metadata_missing_definition_id.yaml similarity index 96% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_definition_id.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_missing/metadata_missing_definition_id.yaml index e8a8e40c780b..b1eb9f113e32 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_definition_id.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_missing/metadata_missing_definition_id.yaml @@ -5,7 +5,7 @@ data: githubIssueLabel: source-alloydb-strict-encrypt dockerRepository: airbyte/image-exists-1 dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_docker_image_tag.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_missing/metadata_missing_docker_image_tag.yaml similarity index 96% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_docker_image_tag.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_missing/metadata_missing_docker_image_tag.yaml index 296dd68ec4ac..28b6807ee54a 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_docker_image_tag.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_missing/metadata_missing_docker_image_tag.yaml @@ -5,7 +5,7 @@ data: connectorType: source githubIssueLabel: source-alloydb-strict-encrypt dockerRepository: airbyte/image-exists-1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_docker_repo.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_missing/metadata_missing_docker_repo.yaml similarity index 96% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_docker_repo.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_missing/metadata_missing_docker_repo.yaml index 70d59d6c87c4..5bc5ca48cdd8 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_docker_repo.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_missing/metadata_missing_docker_repo.yaml @@ -5,7 +5,7 @@ data: connectorType: source githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_missing/metadata_missing_tags.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_missing/metadata_missing_tags.yaml new file mode 100644 index 000000000000..4b704e001f8c --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_missing/metadata_missing_tags.yaml @@ -0,0 +1,12 @@ +metadataSpecVersion: 1.0 +data: + name: AlloyDB for PostgreSQL + definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 + connectorType: source + dockerRepository: airbyte/source-alloydb-strict-encrypt + githubIssueLabel: source-alloydb-strict-encrypt + dockerImageTag: 2.0.1 + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource + connectorSubtype: database + releaseStage: generally_available + license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_version.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_missing/metadata_missing_version.yaml similarity index 96% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_version.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_missing/metadata_missing_version.yaml index 504dfd2ccd3f..8142a7133c5c 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/metadata_missing_version.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/invalid/required_top_level_property_missing/metadata_missing_version.yaml @@ -5,7 +5,7 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_simple.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_simple.yaml index 0b778c8dd456..c8bd4e079df0 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_simple.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_simple.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/metadata_build_base_image.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/metadata_build_base_image.yaml new file mode 100644 index 000000000000..4b2c65d6ebea --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/metadata_build_base_image.yaml @@ -0,0 +1,29 @@ +data: + allowedHosts: + hosts: + - "*.googleapis.com" + connectorBuildOptions: + baseImage: docker.io/airbyte/base-repo-exists:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + connectorSubtype: file + connectorType: source + definitionId: 71607ba1-c0ac-4799-8049-7f4b90dd50f7 + dockerImageTag: 0.3.7 + dockerRepository: airbyte/source-google-sheets + githubIssueLabel: source-google-sheets + icon: google-sheets.svg + license: Elv2 + name: Google Sheets + registries: + cloud: + enabled: true + oss: + enabled: true + releaseStage: generally_available + documentationUrl: https://docs.airbyte.com/integrations/sources/google-sheets + tags: + - language:python + ab_internal: + sl: 300 + ql: 400 + supportLevel: certified +metadataSpecVersion: "1.0" diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_internal_fields.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/metadata_internal_fields.yaml similarity index 96% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_internal_fields.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/metadata_internal_fields.yaml index 44e7a81350cd..2d788f819f7b 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_internal_fields.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/metadata_internal_fields.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_registry_allowed_hosts.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/metadata_registry_allowed_hosts.yaml similarity index 96% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_registry_allowed_hosts.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/metadata_registry_allowed_hosts.yaml index 8ee5e0a0292e..fe9279d3c8de 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_registry_allowed_hosts.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/metadata_registry_allowed_hosts.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_registry_required_resources.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/metadata_registry_required_resources.yaml similarity index 97% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_registry_required_resources.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/metadata_registry_required_resources.yaml index cd4e4f055b80..a49ca2841bed 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_registry_required_resources.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/metadata_registry_required_resources.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/metadata_remote_registries.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/metadata_remote_registries.yaml new file mode 100644 index 000000000000..b942106789e2 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/metadata_remote_registries.yaml @@ -0,0 +1,18 @@ +metadataSpecVersion: 1.0 +data: + name: AlloyDB for PostgreSQL + definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 + connectorType: source + dockerRepository: airbyte/image-exists-1 + githubIssueLabel: source-alloydb-strict-encrypt + dockerImageTag: 0.0.1 + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource + connectorSubtype: database + releaseStage: generally_available + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-alloydb-strict-encrypt + license: MIT + tags: + - language:python diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_support_level.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/metadata_support_level.yaml similarity index 96% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_support_level.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/metadata_support_level.yaml index 6a26f44bf42f..0fe30754f681 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_support_level.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/metadata_support_level.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available supportLevel: community diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_registry_complex_override.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_overrides/metadata_registry_complex_override.yaml similarity index 97% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_registry_complex_override.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_overrides/metadata_registry_complex_override.yaml index 2b421617ba84..b0b617446eb7 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_registry_complex_override.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_overrides/metadata_registry_complex_override.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_registry_enabled.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_overrides/metadata_registry_enabled.yaml similarity index 97% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_registry_enabled.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_overrides/metadata_registry_enabled.yaml index d773c1ad08d2..713515a0a358 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_registry_enabled.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_overrides/metadata_registry_enabled.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_breaking_change_prerelease.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_releases/metadata_breaking_change_prerelease.yaml similarity index 97% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_breaking_change_prerelease.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_releases/metadata_breaking_change_prerelease.yaml index 9dadab96b112..47cc776c6769 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_breaking_change_prerelease.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_releases/metadata_breaking_change_prerelease.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 2.0.0-dev.cf3628ccf3 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_breaking_changes.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_releases/metadata_breaking_changes.yaml similarity index 97% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_breaking_changes.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_releases/metadata_breaking_changes.yaml index 17a588d4a91f..230bbf43d09c 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_breaking_changes.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_releases/metadata_breaking_changes.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_releases/metadata_breaking_changes_with_impact_scopes.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_releases/metadata_breaking_changes_with_impact_scopes.yaml new file mode 100644 index 000000000000..215d8f18c578 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_releases/metadata_breaking_changes_with_impact_scopes.yaml @@ -0,0 +1,22 @@ +metadataSpecVersion: 1.0 +data: + name: AlloyDB for PostgreSQL + definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 + connectorType: source + dockerRepository: airbyte/image-exists-1 + githubIssueLabel: source-alloydb-strict-encrypt + dockerImageTag: 0.0.1 + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource + connectorSubtype: database + releaseStage: generally_available + license: MIT + releases: + breakingChanges: + 2.0.0: + upgradeDeadline: 2023-08-22 + message: "This version changes the connector’s authentication method from `ApiKey` to `oAuth`, per the [API guide](https://amazon-sqs.com/api/someguide)." + scopedImpact: + - scopeType: stream + impactedScopes: ["affected_stream"] + tags: + - language:java diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_breaking_changes_with_migration_doc_url.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_releases/metadata_breaking_changes_with_migration_doc_url.yaml similarity index 88% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_breaking_changes_with_migration_doc_url.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_releases/metadata_breaking_changes_with_migration_doc_url.yaml index bfcbf95644e9..4745efd02a9f 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_breaking_changes_with_migration_doc_url.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_releases/metadata_breaking_changes_with_migration_doc_url.yaml @@ -6,15 +6,15 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 0.0.1 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT releases: - migrationDocumentationUrl: https://docs.airbyte.com/integrations/sources/alloydb-migrations + migrationDocumentationUrl: https://docs.airbyte.com/integrations/sources/existingsource-migrations breakingChanges: 2.0.0: - migrationDocumentationUrl: https://docs.airbyte.com/integrations/sources/alloydb-migrations#2.0.0 + migrationDocumentationUrl: https://docs.airbyte.com/integrations/sources/existingsource-migrations#2.0.0 upgradeDeadline: 2023-08-22 message: "This version changes the connector’s authentication method from `ApiKey` to `oAuth`, per the [API guide](https://amazon-sqs.com/api/someguide)." tags: diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_releases/metadata_breaking_changes_with_multipel_impact_scopes.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_releases/metadata_breaking_changes_with_multipel_impact_scopes.yaml new file mode 100644 index 000000000000..0c19362d12e1 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_releases/metadata_breaking_changes_with_multipel_impact_scopes.yaml @@ -0,0 +1,24 @@ +metadataSpecVersion: 1.0 +data: + name: AlloyDB for PostgreSQL + definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 + connectorType: source + dockerRepository: airbyte/image-exists-1 + githubIssueLabel: source-alloydb-strict-encrypt + dockerImageTag: 0.0.1 + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource + connectorSubtype: database + releaseStage: generally_available + license: MIT + releases: + breakingChanges: + 2.0.0: + upgradeDeadline: 2023-08-22 + message: "This version changes the connector’s authentication method from `ApiKey` to `oAuth`, per the [API guide](https://amazon-sqs.com/api/someguide)." + scopedImpact: + - scopeType: stream + impactedScopes: ["affected_stream", "one_more"] + - scopeType: stream + impactedScopes: ["another_affected_stream"] + tags: + - language:java diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_major_version_with_breaking_change_for_version.yaml b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_releases/metadata_major_version_with_breaking_change_for_version.yaml similarity index 97% rename from airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_major_version_with_breaking_change_for_version.yaml rename to airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_releases/metadata_major_version_with_breaking_change_for_version.yaml index d0fd1bc65772..515df162bebb 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/metadata_major_version_with_breaking_change_for_version.yaml +++ b/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/metadata_validate/valid/with_optional_field/with_releases/metadata_major_version_with_breaking_change_for_version.yaml @@ -6,7 +6,7 @@ data: dockerRepository: airbyte/image-exists-1 githubIssueLabel: source-alloydb-strict-encrypt dockerImageTag: 2.0.0 - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb + documentationUrl: https://docs.airbyte.com/integrations/sources/existingsource connectorSubtype: database releaseStage: generally_available license: MIT diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/test_commands.py b/airbyte-ci/connectors/metadata_service/lib/tests/test_commands.py index ddd44b4ab6fb..2ef3ec4188fa 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/test_commands.py +++ b/airbyte-ci/connectors/metadata_service/lib/tests/test_commands.py @@ -7,93 +7,196 @@ import pytest from click.testing import CliRunner from metadata_service import commands -from metadata_service.gcs_upload import MetadataUploadInfo +from metadata_service.gcs_upload import MetadataUploadInfo, UploadedFile from metadata_service.validators.metadata_validator import ValidatorOptions from pydantic import BaseModel, ValidationError, error_wrappers +from test_gcs_upload import stub_is_image_on_docker_hub # TEST VALIDATE COMMAND -def test_valid_metadata_yaml_files(valid_metadata_yaml_files): +def test_valid_metadata_yaml_files(mocker, valid_metadata_yaml_files, tmp_path): runner = CliRunner() + # Mock dockerhub for base image checks + mocker.patch("metadata_service.validators.metadata_validator.is_image_on_docker_hub", side_effect=stub_is_image_on_docker_hub) + assert len(valid_metadata_yaml_files) > 0, "No files found" for file_path in valid_metadata_yaml_files: - result = runner.invoke(commands.validate, [file_path]) + result = runner.invoke(commands.validate, [file_path, str(tmp_path)]) assert result.exit_code == 0, f"Validation failed for {file_path} with error: {result.output}" -def test_invalid_metadata_yaml_files(invalid_metadata_yaml_files): +def test_invalid_metadata_yaml_files(invalid_metadata_yaml_files, tmp_path): runner = CliRunner() assert len(invalid_metadata_yaml_files) > 0, "No files found" for file_path in invalid_metadata_yaml_files: - result = runner.invoke(commands.validate, [file_path]) - assert result.exit_code != 0, f"Validation succeeded (when it shouldve failed) for {file_path}" + result = runner.invoke(commands.validate, [file_path, str(tmp_path)]) + assert result.exit_code != 0, f"Validation succeeded (when it should have failed) for {file_path}" -def test_file_not_found_fails(): +def test_metadata_file_not_found_fails(tmp_path): runner = CliRunner() - result = runner.invoke(commands.validate, ["non_existent_file.yaml"]) - assert result.exit_code != 0, "Validation succeeded (when it shouldve failed) for non_existent_file.yaml" + result = runner.invoke(commands.validate, ["non_existent_file.yaml", str(tmp_path)]) + assert result.exit_code != 0, "Validation succeeded (when it should have failed) for non_existent_file.yaml" + + +def test_docs_path_not_found_fails(valid_metadata_yaml_files): + runner = CliRunner() + + assert len(valid_metadata_yaml_files) > 0, "No files found" + + result = runner.invoke(commands.validate, [valid_metadata_yaml_files[0], "non_existent_docs_path"]) + assert result.exit_code != 0, "Validation succeeded (when it should have failed) for non_existent_docs_path" def mock_metadata_upload_info( - latest_uploaded: bool, version_uploaded: bool, icon_uploaded: bool, metadata_file_path: str + latest_uploaded: bool, + version_uploaded: bool, + icon_uploaded: bool, + doc_version_uploaded: bool, + doc_inapp_version_uploaded: bool, + doc_latest_uploaded: bool, + doc_inapp_latest_uploaded: bool, + metadata_file_path: str, ) -> MetadataUploadInfo: return MetadataUploadInfo( - uploaded=(latest_uploaded or version_uploaded), - latest_uploaded=latest_uploaded, - latest_blob_id="latest_blob_id" if latest_uploaded else None, - version_uploaded=version_uploaded, - version_blob_id="version_blob_id" if version_uploaded else None, - icon_uploaded=icon_uploaded, - icon_blob_id="icon_blob_id" if icon_uploaded else None, + metadata_uploaded=(latest_uploaded or version_uploaded), metadata_file_path=metadata_file_path, + uploaded_files=[ + UploadedFile( + id="version_metadata", + uploaded=version_uploaded, + description="versioned metadata", + blob_id="version_blob_id" if version_uploaded else None, + ), + UploadedFile( + id="latest_metadata", + uploaded=latest_uploaded, + description="latest metadata", + blob_id="latest_blob_id" if latest_uploaded else None, + ), + UploadedFile( + id="icon", + uploaded=icon_uploaded, + description="icon", + blob_id="icon_blob_id" if icon_uploaded else None, + ), + UploadedFile( + id="doc_version", + uploaded=doc_version_uploaded, + description="versioned doc", + blob_id="doc_version_blob_id" if doc_version_uploaded else None, + ), + UploadedFile( + id="doc_latest", + uploaded=doc_latest_uploaded, + description="latest doc", + blob_id="doc_latest_blob_id" if doc_latest_uploaded else None, + ), + UploadedFile( + id="doc_inapp_version", + uploaded=doc_inapp_version_uploaded, + description="versioned inapp doc", + blob_id="doc_inapp_version_blob_id" if doc_inapp_version_uploaded else None, + ), + UploadedFile( + id="doc_inapp_latest", + uploaded=doc_inapp_latest_uploaded, + description="latest inapp doc", + blob_id="doc_inapp_latest_blob_id" if doc_inapp_latest_uploaded else None, + ), + ], ) # TEST UPLOAD COMMAND @pytest.mark.parametrize( - "latest_uploaded, version_uploaded, icon_uploaded", + "latest_uploaded, version_uploaded, icon_uploaded, doc_version_uploaded, doc_inapp_version_uploaded, doc_latest_uploaded, doc_inapp_latest_uploaded", [ - (False, False, False), - (True, False, False), - (False, True, False), - (False, False, True), - (True, True, False), - (True, False, True), - (False, True, True), - (True, True, True), + (False, False, False, False, False, False, False), + (True, False, False, False, False, False, False), + (False, True, False, False, False, False, False), + (False, False, True, False, False, False, False), + (True, True, False, False, False, False, False), + (True, False, True, False, False, False, False), + (False, True, True, False, False, False, False), + (True, True, True, False, False, False, False), + (True, True, True, True, True, True, True), ], ) -def test_upload(mocker, valid_metadata_yaml_files, latest_uploaded, version_uploaded, icon_uploaded): +def test_upload( + mocker, + tmp_path, + valid_metadata_yaml_files, + latest_uploaded, + version_uploaded, + icon_uploaded, + doc_version_uploaded, + doc_inapp_version_uploaded, + doc_latest_uploaded, + doc_inapp_latest_uploaded, +): runner = CliRunner() mocker.patch.object(commands.click, "secho") mocker.patch.object(commands, "upload_metadata_to_gcs") metadata_file_path = valid_metadata_yaml_files[0] - upload_info = mock_metadata_upload_info(latest_uploaded, version_uploaded, icon_uploaded, metadata_file_path) + upload_info = mock_metadata_upload_info( + latest_uploaded, + version_uploaded, + icon_uploaded, + doc_version_uploaded, + doc_inapp_version_uploaded, + doc_latest_uploaded, + doc_inapp_latest_uploaded, + metadata_file_path, + ) commands.upload_metadata_to_gcs.return_value = upload_info result = runner.invoke( - commands.upload, [metadata_file_path, "my-bucket"] + commands.upload, [metadata_file_path, str(tmp_path), "my-bucket"] ) # Using valid_metadata_yaml_files[0] as SA because it exists... if latest_uploaded: commands.click.secho.assert_has_calls( - [mocker.call(f"The metadata file {metadata_file_path} was uploaded to latest_blob_id.", color="green")] + [mocker.call(f"The latest metadata file for {metadata_file_path} was uploaded to latest_blob_id.", color="green")] ) assert result.exit_code == 0 if version_uploaded: commands.click.secho.assert_has_calls( - [mocker.call(f"The metadata file {metadata_file_path} was uploaded to version_blob_id.", color="green")] + [mocker.call(f"The versioned metadata file for {metadata_file_path} was uploaded to version_blob_id.", color="green")] ) assert result.exit_code == 0 if icon_uploaded: commands.click.secho.assert_has_calls( - [mocker.call(f"The icon file {metadata_file_path} was uploaded to icon_blob_id.", color="green")] + [mocker.call(f"The icon file for {metadata_file_path} was uploaded to icon_blob_id.", color="green")] + ) + + if doc_version_uploaded: + commands.click.secho.assert_has_calls( + [mocker.call(f"The versioned doc file for {metadata_file_path} was uploaded to doc_version_blob_id.", color="green")] + ) + + if doc_inapp_version_uploaded: + commands.click.secho.assert_has_calls( + [ + mocker.call( + f"The versioned inapp doc file for {metadata_file_path} was uploaded to doc_inapp_version_blob_id.", color="green" + ) + ] + ) + + if doc_latest_uploaded: + commands.click.secho.assert_has_calls( + [mocker.call(f"The latest doc file for {metadata_file_path} was uploaded to doc_latest_blob_id.", color="green")] + ) + + if doc_inapp_latest_uploaded: + commands.click.secho.assert_has_calls( + [mocker.call(f"The latest inapp doc file for {metadata_file_path} was uploaded to doc_inapp_latest_blob_id.", color="green")] ) if not (latest_uploaded or version_uploaded): @@ -102,7 +205,7 @@ def test_upload(mocker, valid_metadata_yaml_files, latest_uploaded, version_uplo assert result.exit_code == 5 -def test_upload_prerelease(mocker, valid_metadata_yaml_files): +def test_upload_prerelease(mocker, valid_metadata_yaml_files, tmp_path): runner = CliRunner() mocker.patch.object(commands.click, "secho") mocker.patch.object(commands, "upload_metadata_to_gcs") @@ -110,12 +213,12 @@ def test_upload_prerelease(mocker, valid_metadata_yaml_files): prerelease_tag = "0.3.0-dev.6d33165120" bucket = "my-bucket" metadata_file_path = valid_metadata_yaml_files[0] - validator_opts = ValidatorOptions(prerelease_tag=prerelease_tag) + validator_opts = ValidatorOptions(docs_path=str(tmp_path), prerelease_tag=prerelease_tag) - upload_info = mock_metadata_upload_info(False, True, False, metadata_file_path) + upload_info = mock_metadata_upload_info(False, True, False, True, False, False, False, metadata_file_path) commands.upload_metadata_to_gcs.return_value = upload_info result = runner.invoke( - commands.upload, [metadata_file_path, bucket, "--prerelease", prerelease_tag] + commands.upload, [metadata_file_path, str(tmp_path), bucket, "--prerelease", prerelease_tag] ) # Using valid_metadata_yaml_files[0] as SA because it exists... commands.upload_metadata_to_gcs.assert_has_calls([mocker.call(bucket, pathlib.Path(metadata_file_path), validator_opts)]) @@ -130,13 +233,13 @@ def test_upload_prerelease(mocker, valid_metadata_yaml_files): (ValueError("Boom!"), False), ], ) -def test_upload_with_errors(mocker, valid_metadata_yaml_files, error, handled): +def test_upload_with_errors(mocker, valid_metadata_yaml_files, tmp_path, error, handled): runner = CliRunner() mocker.patch.object(commands.click, "secho") mocker.patch.object(commands, "upload_metadata_to_gcs") commands.upload_metadata_to_gcs.side_effect = error result = runner.invoke( - commands.upload, [valid_metadata_yaml_files[0], "my-bucket"] + commands.upload, [valid_metadata_yaml_files[0], str(tmp_path), "my-bucket"] ) # Using valid_metadata_yaml_files[0] as SA because it exists... assert result.exit_code == 1 if handled: diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/test_gcs_upload.py b/airbyte-ci/connectors/metadata_service/lib/tests/test_gcs_upload.py index a4213787176b..26c6260adf8e 100644 --- a/airbyte-ci/connectors/metadata_service/lib/tests/test_gcs_upload.py +++ b/airbyte-ci/connectors/metadata_service/lib/tests/test_gcs_upload.py @@ -3,26 +3,81 @@ # import json -import pathlib +from pathlib import Path +from typing import Optional import pytest import yaml from metadata_service import gcs_upload -from metadata_service.constants import METADATA_FILE_NAME +from metadata_service.constants import DOC_FILE_NAME, METADATA_FILE_NAME from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 from metadata_service.models.transform import to_json_sanitized_dict from metadata_service.validators.metadata_validator import ValidatorOptions from pydash.objects import get -# Version exists by default, but "666" is bad! (6.0.0 too since breaking changes regex tho) -MOCK_VERSIONS_THAT_DO_NOT_EXIST = ["6.6.6", "6.0.0"] +MOCK_VERSIONS_THAT_DO_NOT_EXIST = ["99.99.99", "0.0.0"] +MISSING_SHA = "MISSINGSHA" +DOCS_PATH = "/docs" +MOCK_DOC_URL_PATH = "integrations/sources/existingsource.md" +VALID_DOC_FILE_PATH = Path(DOCS_PATH) / MOCK_DOC_URL_PATH -def stub_is_image_on_docker_hub(image_name: str, version: str) -> bool: - return "exists" in image_name and version not in MOCK_VERSIONS_THAT_DO_NOT_EXIST +def stub_is_image_on_docker_hub(image_name: str, version: str, digest: Optional[str] = None, retries: int = 0, wait_sec: int = 30) -> bool: + image_repo_exists = "exists" in image_name + version_exists = version not in MOCK_VERSIONS_THAT_DO_NOT_EXIST + sha_is_valid = (digest != MISSING_SHA) if digest is not None else True + image_exists = all([image_repo_exists, version_exists, sha_is_valid]) + return image_exists -def setup_upload_mocks(mocker, version_blob_md5_hash, latest_blob_md5_hash, local_file_md5_hash): +@pytest.fixture(autouse=True) +def mock_local_doc_path_exists(monkeypatch): + original_exists = Path.exists + mocked_doc_path = Path(DOCS_PATH) / MOCK_DOC_URL_PATH + + def fake_exists(self): + if self == Path(DOCS_PATH) or self == mocked_doc_path: + return True + return original_exists(self) + + monkeypatch.setattr(Path, "exists", fake_exists) + + +def assert_upload_invalid_metadata_fails_correctly(metadata_file_path: Path, expected_error_match: str, validate_success_error_match: str): + """ + When attempting to upload invalid metadata, we expect it to fail in a predictable way, depending on what is exactly invalid + about the file. This helper aims to make it easier for a developer who is adding new test cases to figure out that their test + is failing because the test data that should be invalid is passing all of the validation steps. + + Because we don't exit the uploading process if validation fails, this case often looks like a weird error message that is hard to + grok. + """ + try: + with pytest.raises(ValueError, match=expected_error_match) as exc_info: + gcs_upload.upload_metadata_to_gcs( + "my_bucket", + metadata_file_path, + validator_opts=ValidatorOptions(docs_path=DOCS_PATH), + ) + print(f"Upload raised {exc_info.value}") + except AssertionError as e: + if validate_success_error_match in str(e): + raise AssertionError(f"Validation succeeded (when it should have failed) for {metadata_file_path}") from e + else: + raise e + + +def setup_upload_mocks( + mocker, + version_blob_md5_hash, + latest_blob_md5_hash, + local_file_md5_hash, + doc_local_file_md5_hash, + doc_version_blob_md5_hash, + doc_latest_blob_md5_hash, + metadata_file_path, + doc_file_path, +): # Mock dockerhub mocker.patch("metadata_service.validators.metadata_validator.is_image_on_docker_hub", side_effect=stub_is_image_on_docker_hub) @@ -34,17 +89,29 @@ def setup_upload_mocks(mocker, version_blob_md5_hash, latest_blob_md5_hash, loca latest_blob_exists = latest_blob_md5_hash is not None version_blob_exists = version_blob_md5_hash is not None + doc_version_blob_exists = doc_version_blob_md5_hash is not None + doc_latest_blob_exists = doc_latest_blob_md5_hash is not None mock_version_blob = mocker.Mock(exists=mocker.Mock(return_value=version_blob_exists), md5_hash=version_blob_md5_hash) mock_latest_blob = mocker.Mock(exists=mocker.Mock(return_value=latest_blob_exists), md5_hash=latest_blob_md5_hash) + mock_doc_version_blob = mocker.Mock(exists=mocker.Mock(return_value=doc_version_blob_exists), md5_hash=doc_version_blob_md5_hash) + mock_doc_latest_blob = mocker.Mock(exists=mocker.Mock(return_value=doc_latest_blob_exists), md5_hash=doc_latest_blob_md5_hash) mock_bucket = mock_storage_client.bucket.return_value - mock_bucket.blob.side_effect = [mock_version_blob, mock_latest_blob] + mock_bucket.blob.side_effect = [mock_version_blob, mock_doc_version_blob, mock_latest_blob, mock_doc_latest_blob] mocker.patch.object(gcs_upload.service_account.Credentials, "from_service_account_info", mocker.Mock(return_value=mock_credentials)) mocker.patch.object(gcs_upload.storage, "Client", mocker.Mock(return_value=mock_storage_client)) # Mock md5 hash - mocker.patch.object(gcs_upload, "compute_gcs_md5", mocker.Mock(return_value=local_file_md5_hash)) + def side_effect_compute_gcs_md5(file_path): + if str(file_path) == str(metadata_file_path): + return local_file_md5_hash + elif str(file_path) == str(doc_file_path): + return doc_local_file_md5_hash + else: + raise ValueError(f"Unexpected path: {file_path}") + + mocker.patch.object(gcs_upload, "compute_gcs_md5", side_effect=side_effect_compute_gcs_md5) return { "mock_credentials": mock_credentials, @@ -52,141 +119,298 @@ def setup_upload_mocks(mocker, version_blob_md5_hash, latest_blob_md5_hash, loca "mock_bucket": mock_bucket, "mock_version_blob": mock_version_blob, "mock_latest_blob": mock_latest_blob, + "mock_doc_version_blob": mock_doc_version_blob, + "mock_doc_latest_blob": mock_doc_latest_blob, "service_account_json": service_account_json, } @pytest.mark.parametrize( - "version_blob_md5_hash, latest_blob_md5_hash, local_file_md5_hash", + "version_blob_md5_hash, latest_blob_md5_hash, local_file_md5_hash, local_doc_file_md5_hash, doc_version_blob_md5_hash, doc_latest_blob_md5_hash", [ - pytest.param(None, "same_md5_hash", "same_md5_hash", id="Version blob does not exist: Version blob should be uploaded."), - pytest.param("same_md5_hash", None, "same_md5_hash", id="Latest blob does not exist: Latest blob should be uploaded."), - pytest.param(None, None, "same_md5_hash", id="Latest blob and Version blob does not exist: both should be uploaded."), pytest.param( - "different_md5_hash", "same_md5_hash", "same_md5_hash", id="Version blob does not match: Version blob should be uploaded." + None, + "same_md5_hash", + "same_md5_hash", + "same_doc_md5_hash", + "same_doc_md5_hash", + "same_doc_md5_hash", + id="Version blob does not exist: Version blob should be uploaded.", ), pytest.param( "same_md5_hash", + None, "same_md5_hash", + "same_doc_md5_hash", + "same_doc_md5_hash", + "same_doc_md5_hash", + id="Latest blob does not exist: Latest blob should be uploaded.", + ), + pytest.param( + None, + None, "same_md5_hash", - id="Version blob and Latest blob match: no upload should happen.", + "same_doc_md5_hash", + "same_doc_md5_hash", + "same_doc_md5_hash", + id="Latest blob and Version blob does not exist: both should be uploaded.", ), pytest.param( - "same_md5_hash", "different_md5_hash", "same_md5_hash", id="Latest blob does not match: Latest blob should be uploaded." + "different_md5_hash", + "same_md5_hash", + "same_md5_hash", + "same_doc_md5_hash", + "same_doc_md5_hash", + "same_doc_md5_hash", + id="Version blob does not match: Version blob should be uploaded.", ), pytest.param( "same_md5_hash", + "same_md5_hash", + "same_md5_hash", + "same_doc_md5_hash", + "same_doc_md5_hash", + "same_doc_md5_hash", + id="Version blob and Latest blob match, and version and latest doc blobs match: no upload should happen.", + ), + pytest.param( "same_md5_hash", "different_md5_hash", + "same_md5_hash", + "same_doc_md5_hash", + "same_doc_md5_hash", + "same_doc_md5_hash", + id="Latest blob does not match: Latest blob should be uploaded.", + ), + pytest.param( + "same_md5_hash", + "same_md5_hash", + "different_md5_hash", + "same_doc_md5_hash", + "same_doc_md5_hash", + "same_doc_md5_hash", id="Latest blob and Version blob does not match: both should be uploaded.", ), + pytest.param( + "same_md5_hash", + "same_md5_hash", + "same_md5_hash", + "same_doc_md5_hash", + None, + "same_doc_md5_hash", + id="Version doc blob does not exist: Doc version blob should be uploaded.", + ), + pytest.param( + "same_md5_hash", + "same_md5_hash", + "same_md5_hash", + "same_doc_md5_hash", + "same_doc_md5_hash", + None, + id="Latest doc blob does not exist: Doc latest blob should be uploaded.", + ), + pytest.param( + "same_md5_hash", + "same_md5_hash", + "same_md5_hash", + "same_doc_md5_hash", + "different_doc_md5_hash", + "same_doc_md5_hash", + id="Version doc blob does not match: Doc version blob should be uploaded.", + ), + pytest.param( + "same_md5_hash", + "same_md5_hash", + "same_md5_hash", + "same_doc_md5_hash", + "same_doc_md5_hash", + "different_doc_md5_hash", + id="Latest doc blob does not match: Doc version blob should be uploaded.", + ), ], ) def test_upload_metadata_to_gcs_valid_metadata( - mocker, valid_metadata_upload_files, version_blob_md5_hash, latest_blob_md5_hash, local_file_md5_hash + mocker, + valid_metadata_upload_files, + version_blob_md5_hash, + latest_blob_md5_hash, + local_file_md5_hash, + local_doc_file_md5_hash, + doc_version_blob_md5_hash, + doc_latest_blob_md5_hash, ): mocker.spy(gcs_upload, "_version_upload") mocker.spy(gcs_upload, "_latest_upload") + mocker.spy(gcs_upload, "_doc_upload") for valid_metadata_upload_file in valid_metadata_upload_files: - mocks = setup_upload_mocks(mocker, version_blob_md5_hash, latest_blob_md5_hash, local_file_md5_hash) - - metadata_file_path = pathlib.Path(valid_metadata_upload_file) + print(f"\nTesting upload of valid metadata file: " + valid_metadata_upload_file) + metadata_file_path = Path(valid_metadata_upload_file) metadata = ConnectorMetadataDefinitionV0.parse_obj(yaml.safe_load(metadata_file_path.read_text())) + + mocks = setup_upload_mocks( + mocker, + version_blob_md5_hash, + latest_blob_md5_hash, + local_file_md5_hash, + local_doc_file_md5_hash, + doc_version_blob_md5_hash, + doc_latest_blob_md5_hash, + metadata_file_path, + VALID_DOC_FILE_PATH, + ) + expected_version_key = f"metadata/{metadata.data.dockerRepository}/{metadata.data.dockerImageTag}/{METADATA_FILE_NAME}" expected_latest_key = f"metadata/{metadata.data.dockerRepository}/latest/{METADATA_FILE_NAME}" + expected_version_doc_key = f"metadata/{metadata.data.dockerRepository}/{metadata.data.dockerImageTag}/{DOC_FILE_NAME}" + expected_latest_doc_key = f"metadata/{metadata.data.dockerRepository}/latest/{DOC_FILE_NAME}" latest_blob_exists = latest_blob_md5_hash is not None version_blob_exists = version_blob_md5_hash is not None + doc_version_blob_exists = doc_version_blob_md5_hash is not None + doc_latest_blob_exists = doc_latest_blob_md5_hash is not None # Call function under tests upload_info = gcs_upload.upload_metadata_to_gcs( - "my_bucket", - metadata_file_path, + "my_bucket", metadata_file_path, validator_opts=ValidatorOptions(docs_path=DOCS_PATH) ) # Assertions gcs_upload._version_upload.assert_called() gcs_upload._latest_upload.assert_called() + gcs_upload._doc_upload.assert_called() gcs_upload.service_account.Credentials.from_service_account_info.assert_called_with(json.loads(mocks["service_account_json"])) mocks["mock_storage_client"].bucket.assert_called_with("my_bucket") - mocks["mock_bucket"].blob.assert_has_calls([mocker.call(expected_version_key), mocker.call(expected_latest_key)]) - assert upload_info.version_blob_id == mocks["mock_version_blob"].id + mocks["mock_bucket"].blob.assert_has_calls( + [ + mocker.call(expected_version_key), + mocker.call(expected_version_doc_key), + mocker.call(expected_latest_key), + mocker.call(expected_latest_doc_key), + ] + ) + + version_metadata_uploaded_file = next((file for file in upload_info.uploaded_files if file.id == "version_metadata"), None) + assert version_metadata_uploaded_file, "version_metadata not found in uploaded files." + assert version_metadata_uploaded_file.blob_id == mocks["mock_version_blob"].id + + doc_version_uploaded_file = next((file for file in upload_info.uploaded_files if file.id == "doc_version"), None) + assert doc_version_uploaded_file, "doc_version not found in uploaded files." + assert doc_version_uploaded_file.blob_id == mocks["mock_doc_version_blob"].id + + doc_latest_uploaded_file = next((file for file in upload_info.uploaded_files if file.id == "doc_latest"), None) + assert doc_latest_uploaded_file, "doc_latest not found in uploaded files." + assert doc_latest_uploaded_file.blob_id == mocks["mock_doc_latest_blob"].id if not version_blob_exists: mocks["mock_version_blob"].upload_from_filename.assert_called_with(metadata_file_path) - assert upload_info.uploaded + assert upload_info.metadata_uploaded if not latest_blob_exists: mocks["mock_latest_blob"].upload_from_filename.assert_called_with(metadata_file_path) - assert upload_info.uploaded + assert upload_info.metadata_uploaded + + if not doc_version_blob_exists: + mocks["mock_doc_version_blob"].upload_from_filename.assert_called_with(VALID_DOC_FILE_PATH) + assert doc_version_uploaded_file.uploaded + + if not doc_latest_blob_exists: + mocks["mock_doc_latest_blob"].upload_from_filename.assert_called_with(VALID_DOC_FILE_PATH) + assert doc_latest_uploaded_file.uploaded if version_blob_md5_hash != local_file_md5_hash: mocks["mock_version_blob"].upload_from_filename.assert_called_with(metadata_file_path) - assert upload_info.uploaded + assert upload_info.metadata_uploaded if latest_blob_md5_hash != local_file_md5_hash: mocks["mock_latest_blob"].upload_from_filename.assert_called_with(metadata_file_path) - assert upload_info.uploaded + assert upload_info.metadata_uploaded + + if doc_version_blob_md5_hash != local_doc_file_md5_hash: + mocks["mock_doc_version_blob"].upload_from_filename.assert_called_with(VALID_DOC_FILE_PATH) + assert doc_version_uploaded_file.uploaded + + if doc_latest_blob_md5_hash != local_doc_file_md5_hash: + mocks["mock_doc_latest_blob"].upload_from_filename.assert_called_with(VALID_DOC_FILE_PATH) + assert doc_latest_uploaded_file.uploaded # clear the call count gcs_upload._latest_upload.reset_mock() gcs_upload._version_upload.reset_mock() + gcs_upload._doc_upload.reset_mock() def test_upload_metadata_to_gcs_non_existent_metadata_file(): - metadata_file_path = pathlib.Path("./i_dont_exist.yaml") + metadata_file_path = Path("./i_dont_exist.yaml") with pytest.raises(FileNotFoundError): gcs_upload.upload_metadata_to_gcs( "my_bucket", metadata_file_path, + validator_opts=ValidatorOptions(docs_path=DOCS_PATH), ) -def test_upload_invalid_metadata_to_gcs(invalid_metadata_yaml_files): +def test_upload_invalid_metadata_to_gcs(mocker, invalid_metadata_yaml_files): + # Mock dockerhub + mocker.patch("metadata_service.validators.metadata_validator.is_image_on_docker_hub", side_effect=stub_is_image_on_docker_hub) + + # Test that all invalid metadata files throw a ValueError for invalid_metadata_file in invalid_metadata_yaml_files: - metadata_file_path = pathlib.Path(invalid_metadata_file) - # If your test fails with 'Please set the DOCKER_HUB_USERNAME and DOCKER_HUB_PASSWORD environment variables.' - # then your test data passed validation when it shouldn't have! - with pytest.raises(ValueError, match="Validation error"): - gcs_upload.upload_metadata_to_gcs( - "my_bucket", - metadata_file_path, - ) + print(f"\nTesting upload of invalid metadata file: " + invalid_metadata_file) + metadata_file_path = Path(invalid_metadata_file) + + error_match_if_validation_fails_as_expected = "Validation error" + + # If validation succeeds, it goes on to upload any new/changed files. + # We don't mock the gcs stuff in this test, so it fails trying to + # mock compute the md5 hash. + error_match_if_validation_succeeds = "Please set the GCS_CREDENTIALS env var." + + assert_upload_invalid_metadata_fails_correctly( + metadata_file_path, error_match_if_validation_fails_as_expected, error_match_if_validation_succeeds + ) def test_upload_metadata_to_gcs_invalid_docker_images(mocker, invalid_metadata_upload_files): - setup_upload_mocks(mocker, None, None, "new_md5_hash") + setup_upload_mocks(mocker, None, None, "new_md5_hash", None, None, None, None, None) - # Test that all invalid metadata files throw a ValueError + # Test that valid metadata files that reference invalid docker images throw a ValueError for invalid_metadata_file in invalid_metadata_upload_files: - metadata_file_path = pathlib.Path(invalid_metadata_file) - with pytest.raises(ValueError, match="does not exist in DockerHub"): - gcs_upload.upload_metadata_to_gcs( - "my_bucket", - metadata_file_path, - ) + print(f"\nTesting upload of valid metadata file with invalid docker image: " + invalid_metadata_file) + metadata_file_path = Path(invalid_metadata_file) + + error_match_if_validation_fails_as_expected = "does not exist in DockerHub" + + # If validation succeeds, it goes on to upload any new/changed files. + # We mock gcs stuff in this test, so it fails trying to compare the md5 hashes. + error_match_if_validation_succeeds = "Unexpected path" + + assert_upload_invalid_metadata_fails_correctly( + metadata_file_path, error_match_if_validation_fails_as_expected, error_match_if_validation_succeeds + ) def test_upload_metadata_to_gcs_with_prerelease(mocker, valid_metadata_upload_files): # Arrange - setup_upload_mocks(mocker, "new_md5_hash1", "new_md5_hash2", "new_md5_hash3") + setup_upload_mocks(mocker, "new_md5_hash1", "new_md5_hash2", "new_md5_hash3", None, None, None, None, None) mocker.patch("metadata_service.gcs_upload._latest_upload", return_value=(True, "someid")) mocker.patch("metadata_service.gcs_upload.upload_file_if_changed", return_value=(True, "someid")) mocker.spy(gcs_upload, "_version_upload") + doc_upload_spy = mocker.spy(gcs_upload, "_doc_upload") for valid_metadata_upload_file in valid_metadata_upload_files: + print(f"\nTesting prerelease upload of valid metadata file: " + valid_metadata_upload_file) # Assuming there is a valid metadata file in the list, if not, you might need to create one - metadata_file_path = pathlib.Path(valid_metadata_upload_file) + metadata_file_path = Path(valid_metadata_upload_file) prerelease_image_tag = "1.5.6-dev.f80318f754" gcs_upload.upload_metadata_to_gcs( "my_bucket", metadata_file_path, - ValidatorOptions(prerelease_tag=prerelease_image_tag), + ValidatorOptions(docs_path=DOCS_PATH, prerelease_tag=prerelease_image_tag), ) gcs_upload._latest_upload.assert_not_called() @@ -196,11 +420,17 @@ def test_upload_metadata_to_gcs_with_prerelease(mocker, valid_metadata_upload_fi _, __, tmp_metadata_file_path = gcs_upload._version_upload.call_args[0] assert prerelease_image_tag in str(tmp_metadata_file_path) + # Assert that _doc_upload is only called twice, both with latest set to False + assert doc_upload_spy.call_count == 2 + assert doc_upload_spy.call_args_list[0].args[-2] == False + assert doc_upload_spy.call_args_list[1].args[-2] == False + doc_upload_spy.reset_mock() + # Verify the tmp metadata is overridden assert tmp_metadata_file_path.exists(), f"{tmp_metadata_file_path} does not exist" # verify that the metadata is overrode - tmp_metadata, error = gcs_upload.validate_and_load(tmp_metadata_file_path, []) + tmp_metadata, error = gcs_upload.validate_and_load(tmp_metadata_file_path, [], validator_opts=ValidatorOptions(docs_path=DOCS_PATH)) tmp_metadata_dict = to_json_sanitized_dict(tmp_metadata, exclude_none=True) assert tmp_metadata_dict["data"]["dockerImageTag"] == prerelease_image_tag for registry in get(tmp_metadata_dict, "data.registries", {}).values(): diff --git a/airbyte-ci/connectors/metadata_service/lib/tests/test_spec_cache.py b/airbyte-ci/connectors/metadata_service/lib/tests/test_spec_cache.py new file mode 100644 index 000000000000..9ce15092fcb7 --- /dev/null +++ b/airbyte-ci/connectors/metadata_service/lib/tests/test_spec_cache.py @@ -0,0 +1,101 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import patch + +import pytest +from metadata_service.spec_cache import CachedSpec, Registries, SpecCache, get_docker_info_from_spec_cache_path + + +@pytest.fixture +def mock_spec_cache(): + with patch("google.cloud.storage.Client.create_anonymous_client") as MockClient, patch( + "google.cloud.storage.Client.bucket" + ) as MockBucket: + + # Create stub mock client and bucket + MockClient.return_value + MockBucket.return_value + + # Create a list of 4 test CachedSpecs + test_specs = [ + CachedSpec("image1", "tag-has-override", "path1", Registries.OSS), + CachedSpec("image1", "tag-has-override", "path2", Registries.CLOUD), + CachedSpec("image2", "tag-no-override", "path3", Registries.OSS), + CachedSpec("image3", "tag-no-override", "path4", Registries.CLOUD), + ] + + # Mock get_all_cached_specs to return test_specs + with patch.object(SpecCache, "get_all_cached_specs", return_value=test_specs): + yield SpecCache() + + +@pytest.mark.parametrize( + "image,tag,given_registry,expected_registry", + [ + ("image1", "tag-has-override", "OSS", Registries.OSS), + ("image1", "tag-has-override", "CLOUD", Registries.CLOUD), + ("image2", "tag-no-override", "OSS", Registries.OSS), + ("image2", "tag-no-override", "CLOUD", Registries.OSS), + ("image3", "tag-no-override", "OSS", None), + ("image3", "tag-no-override", "CLOUD", Registries.CLOUD), + ("nonexistent", "tag", "OSS", None), + ("nonexistent", "tag", "CLOUD", None), + ], +) +def test_find_spec_cache_with_fallback(mock_spec_cache, image, tag, given_registry, expected_registry): + spec = mock_spec_cache.find_spec_cache_with_fallback(image, tag, given_registry) + if expected_registry == None: + assert spec == None + else: + assert spec.docker_repository == image + assert spec.docker_image_tag == tag + assert spec.registry == expected_registry + + +@pytest.mark.parametrize( + "spec_cache_path,expected_spec", + [ + ( + "specs/airbyte/destination-azure-blob-storage/0.1.1/spec.json", + CachedSpec( + "airbyte/destination-azure-blob-storage", + "0.1.1", + "specs/airbyte/destination-azure-blob-storage/0.1.1/spec.json", + Registries.OSS, + ), + ), + ( + "specs/airbyte/destination-azure-blob-storage/0.1.1/spec.cloud.json", + CachedSpec( + "airbyte/destination-azure-blob-storage", + "0.1.1", + "specs/airbyte/destination-azure-blob-storage/0.1.1/spec.cloud.json", + Registries.CLOUD, + ), + ), + ( + "specs/airbyte/source-azure-blob-storage/1.1.1/spec.json", + CachedSpec( + "airbyte/source-azure-blob-storage", "1.1.1", "specs/airbyte/source-azure-blob-storage/1.1.1/spec.json", Registries.OSS + ), + ), + ( + "specs/faros/some-name/1.1.1/spec.json", + CachedSpec("faros/some-name", "1.1.1", "specs/faros/some-name/1.1.1/spec.json", Registries.OSS), + ), + ], +) +def test_get_docker_info_from_spec_cache_path(spec_cache_path, expected_spec): + actual_spec = get_docker_info_from_spec_cache_path(spec_cache_path) + + assert actual_spec.docker_repository == expected_spec.docker_repository + assert actual_spec.docker_image_tag == expected_spec.docker_image_tag + assert actual_spec.spec_cache_path == expected_spec.spec_cache_path + assert actual_spec.registry == expected_spec.registry + + +def test_get_docker_info_from_spec_cache_path_invalid(): + with pytest.raises(Exception): + get_docker_info_from_spec_cache_path("specs/airbyte/destination-azure-blob-storage/0.1.1/spec") diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/__init__.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/__init__.py index 81805b44a77d..336652744150 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/__init__.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/__init__.py @@ -113,13 +113,18 @@ ), } -CONNECTOR_TEST_REPORT_RESOURCE_TREE = { - **SLACK_RESOURCE_TREE, - **GITHUB_RESOURCE_TREE, +CONNECTOR_TEST_REPORT_SENSOR_RESOURCE_TREE = { **GCS_RESOURCE_TREE, "latest_nightly_complete_file_blobs": gcs_directory_blobs.configured( {"gcs_bucket": {"env": "CI_REPORT_BUCKET"}, "prefix": NIGHTLY_FOLDER, "match_regex": f".*{NIGHTLY_COMPLETE_REPORT_FILE_NAME}$"} ), +} + +CONNECTOR_TEST_REPORT_RESOURCE_TREE = { + **SLACK_RESOURCE_TREE, + **GITHUB_RESOURCE_TREE, + **GCS_RESOURCE_TREE, + **CONNECTOR_TEST_REPORT_SENSOR_RESOURCE_TREE, "latest_nightly_test_output_file_blobs": gcs_directory_blobs.configured( { "gcs_bucket": {"env": "CI_REPORT_BUCKET"}, @@ -140,7 +145,7 @@ } SENSORS = [ - registry_updated_sensor(job=generate_registry_reports, resources_def=RESOURCES), + registry_updated_sensor(job=generate_registry_reports, resources_def=REGISTRY_RESOURCE_TREE), new_gcs_blobs_sensor( job=generate_oss_registry, resources_def=REGISTRY_ENTRY_RESOURCE_TREE, @@ -155,7 +160,7 @@ ), new_gcs_blobs_sensor( job=generate_nightly_reports, - resources_def=CONNECTOR_TEST_REPORT_RESOURCE_TREE, + resources_def=CONNECTOR_TEST_REPORT_SENSOR_RESOURCE_TREE, gcs_blobs_resource_key="latest_nightly_complete_file_blobs", interval=(1 * 60 * 60), ), diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_entry.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_entry.py index c00aa7e8e655..80f01d15956f 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_entry.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_entry.py @@ -17,7 +17,7 @@ from metadata_service.constants import ICON_FILE_NAME, METADATA_FILE_NAME from metadata_service.models.generated.ConnectorRegistryDestinationDefinition import ConnectorRegistryDestinationDefinition from metadata_service.models.generated.ConnectorRegistrySourceDefinition import ConnectorRegistrySourceDefinition -from metadata_service.spec_cache import get_cached_spec, list_cached_specs +from metadata_service.spec_cache import SpecCache from orchestrator.config import MAX_METADATA_PARTITION_RUN_REQUEST, VALID_REGISTRIES, get_public_url_for_gcs_file from orchestrator.logging import sentry from orchestrator.logging.publish_connector_lifecycle import PublishConnectorLifecycle, PublishConnectorLifecycleStage, StageStatus @@ -45,20 +45,17 @@ class MissingCachedSpecError(Exception): @sentry_sdk.trace -def apply_spec_to_registry_entry(registry_entry: dict, cached_specs: OutputDataFrame) -> dict: - cached_connector_version = { - (cached_spec["docker_repository"], cached_spec["docker_image_tag"]): cached_spec["spec_cache_path"] - for cached_spec in cached_specs.to_dict(orient="records") - } - - try: - spec_path = cached_connector_version[(registry_entry["dockerRepository"], registry_entry["dockerImageTag"])] - entry_with_spec = copy.deepcopy(registry_entry) - entry_with_spec["spec"] = get_cached_spec(spec_path) - return entry_with_spec - except KeyError: +def apply_spec_to_registry_entry(registry_entry: dict, spec_cache: SpecCache, registry_name: str) -> dict: + cached_spec = spec_cache.find_spec_cache_with_fallback( + registry_entry["dockerRepository"], registry_entry["dockerImageTag"], registry_name + ) + if cached_spec is None: raise MissingCachedSpecError(f"No cached spec found for {registry_entry['dockerRepository']}:{registry_entry['dockerImageTag']}") + entry_with_spec = copy.deepcopy(registry_entry) + entry_with_spec["spec"] = spec_cache.download_spec(cached_spec) + return entry_with_spec + def calculate_migration_documentation_url(releases_or_breaking_change: dict, documentation_url: str, version: Optional[str] = None) -> str: """Calculate the migration documentation url for the connector releases. @@ -217,8 +214,8 @@ def get_connector_type_from_registry_entry(registry_entry: dict) -> TaggedRegist raise Exception("Could not determine connector type from registry entry") -def get_registry_entry_write_path(metadata_entry: LatestMetadataEntry, registry_name: str): - metadata_path = metadata_entry.file_path +def _get_latest_entry_write_path(metadata_path: Optional[str], registry_name: str) -> str: + """Get the write path for the registry entry, assuming the metadata entry is the latest version.""" if metadata_path is None: raise Exception(f"Metadata entry {metadata_entry} does not have a file path") @@ -226,6 +223,23 @@ def get_registry_entry_write_path(metadata_entry: LatestMetadataEntry, registry_ return os.path.join(metadata_folder, registry_name) +def get_registry_entry_write_path( + registry_entry: Optional[PolymorphicRegistryEntry], metadata_entry: LatestMetadataEntry, registry_name: str +) -> str: + """Get the write path for the registry entry.""" + if metadata_entry.is_latest_version_path: + # if the metadata entry is the latest version, write the registry entry to the same path as the metadata entry + return _get_latest_entry_write_path(metadata_entry.file_path, registry_name) + else: + if registry_entry is None: + raise Exception(f"Could not determine write path for registry entry {registry_entry} because it is None") + + # if the metadata entry is not the latest version, write the registry entry to its own version specific path + # this is handle the case when a dockerImageTag is overridden + + return HACKS.construct_registry_entry_write_path(registry_entry, registry_name) + + @sentry_sdk.trace def persist_registry_entry_to_json( registry_entry: PolymorphicRegistryEntry, @@ -244,17 +258,16 @@ def persist_registry_entry_to_json( Returns: GCSFileHandle: The registry_entry directory manager. """ - registry_entry_write_path = get_registry_entry_write_path(metadata_entry, registry_name) + registry_entry_write_path = get_registry_entry_write_path(registry_entry, metadata_entry, registry_name) registry_entry_json = registry_entry.json(exclude_none=True) file_handle = registry_directory_manager.write_data(registry_entry_json.encode("utf-8"), ext="json", key=registry_entry_write_path) - HACKS.write_registry_to_overrode_file_paths(registry_entry, registry_name, metadata_entry, registry_directory_manager) return file_handle @sentry_sdk.trace def generate_and_persist_registry_entry( metadata_entry: LatestMetadataEntry, - cached_specs: OutputDataFrame, + spec_cache: SpecCache, metadata_directory_manager: GCSFileManager, registry_name: str, ) -> str: @@ -269,7 +282,7 @@ def generate_and_persist_registry_entry( Output[ConnectorRegistryV0]: The registry. """ raw_entry_dict = metadata_to_registry_entry(metadata_entry, registry_name) - registry_entry_with_spec = apply_spec_to_registry_entry(raw_entry_dict, cached_specs) + registry_entry_with_spec = apply_spec_to_registry_entry(raw_entry_dict, spec_cache, registry_name) _, ConnectorModel = get_connector_type_from_registry_entry(registry_entry_with_spec) @@ -306,14 +319,14 @@ def get_registry_status_lists(registry_entry: LatestMetadataEntry) -> Tuple[List return valid_enabled_registries, valid_disabled_registries -def delete_registry_entry(registry_name, registry_entry: LatestMetadataEntry, metadata_directory_manager: GCSFileManager) -> str: +def delete_registry_entry(registry_name, metadata_entry: LatestMetadataEntry, metadata_directory_manager: GCSFileManager) -> str: """Delete the given registry entry from GCS. Args: - registry_entry (LatestMetadataEntry): The registry entry. + metadata_entry (LatestMetadataEntry): The registry entry. metadata_directory_manager (GCSFileManager): The metadata directory manager. """ - registry_entry_write_path = get_registry_entry_write_path(registry_entry, registry_name) + registry_entry_write_path = get_registry_entry_write_path(None, metadata_entry, registry_name) file_handle = metadata_directory_manager.delete_by_key(key=registry_entry_write_path, ext="json") return file_handle.public_url if file_handle else None @@ -436,20 +449,25 @@ def registry_entry(context: OpExecutionContext, metadata_entry: Optional[LatestM f"Generating registry entry for {metadata_entry.file_path}", ) - cached_specs = pd.DataFrame(list_cached_specs()) + spec_cache = SpecCache() root_metadata_directory_manager = context.resources.root_metadata_directory_manager enabled_registries, disabled_registries = get_registry_status_lists(metadata_entry) persisted_registry_entries = { - registry_name: generate_and_persist_registry_entry(metadata_entry, cached_specs, root_metadata_directory_manager, registry_name) + registry_name: generate_and_persist_registry_entry(metadata_entry, spec_cache, root_metadata_directory_manager, registry_name) for registry_name in enabled_registries } - deleted_registry_entries = { - registry_name: delete_registry_entry(registry_name, metadata_entry, root_metadata_directory_manager) - for registry_name in disabled_registries - } + # Only delete the registry entry if it is the latest version + # This is to preserve any registry specific overrides even if they were removed + deleted_registry_entries = {} + if metadata_entry.is_latest_version_path: + context.log.debug(f"Deleting previous registry entries enabled {metadata_entry.file_path}") + deleted_registry_entries = { + registry_name: delete_registry_entry(registry_name, metadata_entry, root_metadata_directory_manager) + for registry_name in disabled_registries + } dagster_metadata_persist = { f"create_{registry_name}": MetadataValue.url(registry_url) for registry_name, registry_url in persisted_registry_entries.items() diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_report.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_report.py index 8895b963eff1..a1e26af19329 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_report.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_report.py @@ -324,7 +324,6 @@ def connector_registry_report(context, all_destinations_dataframe, all_sources_d metadata = { "first_10_preview": MetadataValue.md(all_connectors_dataframe.head(10).to_markdown()), - "json": MetadataValue.json(json_string), "json_gcs_url": MetadataValue.url(json_file_handle.public_url), "html_gcs_url": MetadataValue.url(html_file_handle.public_url), } diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/hacks.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/hacks.py index 3cce26dd2a1f..cd8d30950126 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/hacks.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/hacks.py @@ -4,27 +4,14 @@ from typing import Union -from dagster import get_dagster_logger -from dagster_gcp.gcs.file_manager import GCSFileHandle, GCSFileManager from metadata_service.constants import METADATA_FILE_NAME from metadata_service.gcs_upload import get_metadata_remote_file_path from metadata_service.models.generated.ConnectorRegistryDestinationDefinition import ConnectorRegistryDestinationDefinition from metadata_service.models.generated.ConnectorRegistrySourceDefinition import ConnectorRegistrySourceDefinition -from orchestrator.models.metadata import LatestMetadataEntry PolymorphicRegistryEntry = Union[ConnectorRegistrySourceDefinition, ConnectorRegistryDestinationDefinition] -def _is_docker_repository_overridden( - metadata_entry: LatestMetadataEntry, - registry_entry: PolymorphicRegistryEntry, -) -> bool: - """Check if the docker repository is overridden in the registry entry.""" - registry_entry_docker_repository = registry_entry.dockerRepository - metadata_docker_repository = metadata_entry.metadata_definition.data.dockerRepository - return registry_entry_docker_repository != metadata_docker_repository - - def _get_version_specific_registry_entry_file_path(registry_entry, registry_name): """Get the file path for the version specific registry entry file.""" docker_reposiory = registry_entry.dockerRepository @@ -44,48 +31,56 @@ def _check_for_invalid_write_path(write_path: str): ) -def write_registry_to_overrode_file_paths( +def construct_registry_entry_write_path( registry_entry: PolymorphicRegistryEntry, registry_name: str, - metadata_entry: LatestMetadataEntry, - registry_directory_manager: GCSFileManager, -) -> GCSFileHandle: +) -> str: """ - Write the registry entry to the docker repository and version specific file paths - in the event that the docker repository is overridden. + Construct a registry entry write path from its parts. Underlying issue: - The registry entry files (oss.json and cloud.json) are traditionally written to - the same path as the metadata.yaml file that created them. This is fine for the - most cases, but when the docker repository is overridden, the registry entry - files need to be written to a different path. + This is barely a hack. + + But it is related to a few imperfect design decisions that we have to work around. + 1. Metadata files and the registry entries are saved to the same top level folder. + 2. That save path is determined by the docker repository and version of the image + 3. A metadata file can include overrides for the docker repository and version of the image depending on the registry + 4. The platform looks up registry entries by docker repository and version of the image. + 5. The registry generation depends on what ever registry entry is written to a path ending in latest/{registry_name}.json + + This means that when a metadata file overrides the docker repository and version of the image, + the registry entry needs to be written to a different path than the metadata file. + + *But only in the case that its a versioned path and NOT a latest path.* + + Example: + If metadata file for source-posgres is at version 2.0.0 but there is a override for the cloud registry + that changes the docker repository to source-postgres-strict-encrypt and the version to 1.0.0 + + Then we will have a metadata file written to: + gs://my-bucket/metadata/source-postgres/2.0.0/metadata.yaml + + and registry entries written to: + gs://my-bucket/metadata/source-postgres/2.0.0/oss.json + gs://my-bucket/metadata/source-postgres-strict-encrypt/1.0.0/cloud.json - For example if source-postgres:dev.123 is overridden to source-postgres-strict-encrypt:dev.123 - then the oss.json file needs to be written to the path that would be assumed - by the platform when looking for a specific registry entry. In this case, for cloud, it would be - gs://my-bucket/metadata/source-postgres-strict-encrypt/dev.123/cloud.json + But if the metadata file is written to a latest path, then the registry entry will be written to the same path: + gs://my-bucket/metadata/source-postgres/latest/oss.json + gs://my-bucket/metadata/source-postgres/latest/cloud.json - Ideally we would not have to do this, but the combination of prereleases and common overrides - make this nessesary. + Future Solution: + To resolve this properly we need to + 1. Separate the save paths for metadata files and registry entries + 2. Have the paths determined by definitionId and a metadata version + 3. Allow for references to other metadata files in the metadata file instead of overrides Args: registry_entry (PolymorphicRegistryEntry): The registry entry to write registry_name (str): The name of the registry entry (oss or cloud) - metadata_entry (LatestMetadataEntry): The metadata entry that created the registry entry - registry_directory_manager (GCSFileManager): The file manager to use to write the registry entry Returns: - GCSFileHandle: The file handle of the written registry entry + str: The registry entry write path corresponding to the registry entry """ - if not _is_docker_repository_overridden(metadata_entry, registry_entry): - return None - logger = get_dagster_logger() - registry_entry_json = registry_entry.json(exclude_none=True) overrode_registry_entry_version_write_path = _get_version_specific_registry_entry_file_path(registry_entry, registry_name) _check_for_invalid_write_path(overrode_registry_entry_version_write_path) - logger.info(f"Writing registry entry to {overrode_registry_entry_version_write_path}") - file_handle = registry_directory_manager.write_data( - registry_entry_json.encode("utf-8"), ext="json", key=overrode_registry_entry_version_write_path - ) - logger.info(f"Successfully wrote registry entry to {file_handle.public_url}") - return file_handle + return overrode_registry_entry_version_write_path diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/models/metadata.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/models/metadata.py index 4e5ac4b38299..f73d3aa96346 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/models/metadata.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/models/metadata.py @@ -2,9 +2,9 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from dataclasses import dataclass from typing import Any, Optional, Tuple +from metadata_service.constants import METADATA_FILE_NAME from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0 from pydantic import BaseModel, ValidationError @@ -51,3 +51,11 @@ class LatestMetadataEntry(BaseModel): icon_url: Optional[str] = None bucket_name: Optional[str] = None file_path: Optional[str] = None + + @property + def is_latest_version_path(self) -> bool: + """ + Path is considered a latest version path if the subfolder containing METADATA_FILE_NAME is "latest" + """ + ending_path = f"latest/{METADATA_FILE_NAME}" + return self.file_path.endswith(ending_path) diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/templates/render.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/templates/render.py index 3153a86f3959..eaa76f7b30c7 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/templates/render.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/templates/render.py @@ -43,10 +43,8 @@ def test_badge_html(test_summary_url: str) -> str: def internal_level_html(level_value: float) -> str: - level = level_value / 100 - - # remove trailing zeros - level = f"{level:.2f}".rstrip("0").rstrip(".") + # cast to int to remove decimal places + level = int(level_value) return f"Level {level}" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/dagster_helpers.py b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/dagster_helpers.py index 3ccc4fa0841b..dc59a1d13c1a 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/dagster_helpers.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/dagster_helpers.py @@ -16,7 +16,16 @@ def output_dataframe(result_df: pd.DataFrame) -> Output[pd.DataFrame]: """ Returns a Dagster Output object with a dataframe as the result and a markdown preview. """ - return Output(result_df, metadata={"count": len(result_df), "preview": MetadataValue.md(result_df.to_markdown())}) + + # Truncate to 100 rows to avoid dagster throwing a "too large" error + MAX_PREVIEW_ROWS = 100 + is_truncated = len(result_df) > MAX_PREVIEW_ROWS + preview_result_df = result_df.head(MAX_PREVIEW_ROWS) + + return Output( + result_df, + metadata={"count": len(result_df), "preview": MetadataValue.md(preview_result_df.to_markdown()), "is_truncated": is_truncated}, + ) def string_array_to_hash(strings: List[str]) -> str: diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/poetry.lock b/airbyte-ci/connectors/metadata_service/orchestrator/poetry.lock index fafe163819a5..d350d91b4064 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/poetry.lock +++ b/airbyte-ci/connectors/metadata_service/orchestrator/poetry.lock @@ -2,13 +2,13 @@ [[package]] name = "alembic" -version = "1.11.2" +version = "1.13.1" description = "A database migration tool for SQLAlchemy." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "alembic-1.11.2-py3-none-any.whl", hash = "sha256:7981ab0c4fad4fe1be0cf183aae17689fe394ff874fd2464adb774396faf0796"}, - {file = "alembic-1.11.2.tar.gz", hash = "sha256:678f662130dc540dac12de0ea73de9f89caea9dbea138f60ef6263149bf84657"}, + {file = "alembic-1.13.1-py3-none-any.whl", hash = "sha256:2edcc97bed0bd3272611ce3a98d98279e9c209e7186e43e75bbb1b2bdfdbcc43"}, + {file = "alembic-1.13.1.tar.gz", hash = "sha256:4932c8558bf68f2ee92b9bbcb8218671c627064d5b08939437af6d77dc05e595"}, ] [package.dependencies] @@ -17,7 +17,7 @@ SQLAlchemy = ">=1.3.0" typing-extensions = ">=4" [package.extras] -tz = ["python-dateutil"] +tz = ["backports.zoneinfo"] [[package]] name = "aniso8601" @@ -35,24 +35,25 @@ dev = ["black", "coverage", "isort", "pre-commit", "pyenchant", "pylint"] [[package]] name = "anyio" -version = "3.7.1" +version = "4.2.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "anyio-3.7.1-py3-none-any.whl", hash = "sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5"}, - {file = "anyio-3.7.1.tar.gz", hash = "sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780"}, + {file = "anyio-4.2.0-py3-none-any.whl", hash = "sha256:745843b39e829e108e518c489b31dc757de7d2131d53fac32bd8df268227bfee"}, + {file = "anyio-4.2.0.tar.gz", hash = "sha256:e1875bb4b4e2de1669f4bc7869b6d3f54231cdced71605e6e64c9be77e3be50f"}, ] [package.dependencies] -exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} +exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} idna = ">=2.8" sniffio = ">=1.1" +typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""} [package.extras] -doc = ["Sphinx", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme (>=1.2.2)", "sphinxcontrib-jquery"] -test = ["anyio[trio]", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] -trio = ["trio (<0.22)"] +doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] +trio = ["trio (>=0.23)"] [[package]] name = "appdirs" @@ -67,21 +68,22 @@ files = [ [[package]] name = "attrs" -version = "23.1.0" +version = "23.2.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.7" files = [ - {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"}, - {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"}, + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, ] [package.extras] cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] -dev = ["attrs[docs,tests]", "pre-commit"] +dev = ["attrs[tests]", "pre-commit"] docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] tests = ["attrs[tests-no-zope]", "zope-interface"] -tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] [[package]] name = "backoff" @@ -114,140 +116,130 @@ lxml = ["lxml"] [[package]] name = "build" -version = "0.10.0" +version = "1.0.3" description = "A simple, correct Python build frontend" optional = false python-versions = ">= 3.7" files = [ - {file = "build-0.10.0-py3-none-any.whl", hash = "sha256:af266720050a66c893a6096a2f410989eeac74ff9a68ba194b3f6473e8e26171"}, - {file = "build-0.10.0.tar.gz", hash = "sha256:d5b71264afdb5951d6704482aac78de887c80691c52b88a9ad195983ca2c9269"}, + {file = "build-1.0.3-py3-none-any.whl", hash = "sha256:589bf99a67df7c9cf07ec0ac0e5e2ea5d4b37ac63301c4986d1acb126aa83f8f"}, + {file = "build-1.0.3.tar.gz", hash = "sha256:538aab1b64f9828977f84bc63ae570b060a8ed1be419e7870b8b4fc5e6ea553b"}, ] [package.dependencies] colorama = {version = "*", markers = "os_name == \"nt\""} +importlib-metadata = {version = ">=4.6", markers = "python_version < \"3.10\""} packaging = ">=19.0" pyproject_hooks = "*" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} [package.extras] -docs = ["furo (>=2021.08.31)", "sphinx (>=4.0,<5.0)", "sphinx-argparse-cli (>=1.5)", "sphinx-autodoc-typehints (>=1.10)"] -test = ["filelock (>=3)", "pytest (>=6.2.4)", "pytest-cov (>=2.12)", "pytest-mock (>=2)", "pytest-rerunfailures (>=9.1)", "pytest-xdist (>=1.34)", "setuptools (>=42.0.0)", "setuptools (>=56.0.0)", "toml (>=0.10.0)", "wheel (>=0.36.0)"] -typing = ["importlib-metadata (>=5.1)", "mypy (==0.991)", "tomli", "typing-extensions (>=3.7.4.3)"] +docs = ["furo (>=2023.08.17)", "sphinx (>=7.0,<8.0)", "sphinx-argparse-cli (>=1.5)", "sphinx-autodoc-typehints (>=1.10)", "sphinx-issues (>=3.0.0)"] +test = ["filelock (>=3)", "pytest (>=6.2.4)", "pytest-cov (>=2.12)", "pytest-mock (>=2)", "pytest-rerunfailures (>=9.1)", "pytest-xdist (>=1.34)", "setuptools (>=42.0.0)", "setuptools (>=56.0.0)", "setuptools (>=56.0.0)", "setuptools (>=67.8.0)", "wheel (>=0.36.0)"] +typing = ["importlib-metadata (>=5.1)", "mypy (>=1.5.0,<1.6.0)", "tomli", "typing-extensions (>=3.7.4.3)"] virtualenv = ["virtualenv (>=20.0.35)"] [[package]] name = "cachecontrol" -version = "0.12.14" +version = "0.13.1" description = "httplib2 caching for requests" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "CacheControl-0.12.14-py2.py3-none-any.whl", hash = "sha256:1c2939be362a70c4e5f02c6249462b3b7a24441e4f1ced5e9ef028172edf356a"}, - {file = "CacheControl-0.12.14.tar.gz", hash = "sha256:d1087f45781c0e00616479bfd282c78504371ca71da017b49df9f5365a95feba"}, + {file = "cachecontrol-0.13.1-py3-none-any.whl", hash = "sha256:95dedbec849f46dda3137866dc28b9d133fc9af55f5b805ab1291833e4457aa4"}, + {file = "cachecontrol-0.13.1.tar.gz", hash = "sha256:f012366b79d2243a6118309ce73151bf52a38d4a5dac8ea57f09bd29087e506b"}, ] [package.dependencies] -lockfile = {version = ">=0.9", optional = true, markers = "extra == \"filecache\""} +filelock = {version = ">=3.8.0", optional = true, markers = "extra == \"filecache\""} msgpack = ">=0.5.2" -requests = "*" +requests = ">=2.16.0" [package.extras] -filecache = ["lockfile (>=0.9)"] +dev = ["CacheControl[filecache,redis]", "black", "build", "cherrypy", "mypy", "pytest", "pytest-cov", "sphinx", "tox", "types-redis", "types-requests"] +filecache = ["filelock (>=3.8.0)"] redis = ["redis (>=2.10.5)"] [[package]] name = "cachetools" -version = "5.3.1" +version = "5.3.2" description = "Extensible memoizing collections and decorators" optional = false python-versions = ">=3.7" files = [ - {file = "cachetools-5.3.1-py3-none-any.whl", hash = "sha256:95ef631eeaea14ba2e36f06437f36463aac3a096799e876ee55e5cdccb102590"}, - {file = "cachetools-5.3.1.tar.gz", hash = "sha256:dce83f2d9b4e1f732a8cd44af8e8fab2dbe46201467fc98b3ef8f269092bf62b"}, + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, ] [[package]] name = "certifi" -version = "2023.7.22" +version = "2023.11.17" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"}, - {file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"}, + {file = "certifi-2023.11.17-py3-none-any.whl", hash = "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"}, + {file = "certifi-2023.11.17.tar.gz", hash = "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1"}, ] [[package]] name = "cffi" -version = "1.15.1" +version = "1.16.0" description = "Foreign Function Interface for Python calling C code." optional = false -python-versions = "*" +python-versions = ">=3.8" files = [ - {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, - {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, - {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, - {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, - {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, - {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, - {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, - {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, - {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, - {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, - {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, - {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, - {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, - {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, - {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, - {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, - {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, - {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, - {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, ] [package.dependencies] @@ -255,112 +247,127 @@ pycparser = "*" [[package]] name = "charset-normalizer" -version = "3.2.0" +version = "3.3.2" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7.0" files = [ - {file = "charset-normalizer-3.2.0.tar.gz", hash = "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-win32.whl", hash = "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-win32.whl", hash = "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-win32.whl", hash = "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-win32.whl", hash = "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-win32.whl", hash = "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80"}, - {file = "charset_normalizer-3.2.0-py3-none-any.whl", hash = "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6"}, + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, ] [[package]] name = "cleo" -version = "2.0.1" +version = "2.1.0" description = "Cleo allows you to create beautiful and testable command-line interfaces." optional = false python-versions = ">=3.7,<4.0" files = [ - {file = "cleo-2.0.1-py3-none-any.whl", hash = "sha256:6eb133670a3ed1f3b052d53789017b6e50fca66d1287e6e6696285f4cb8ea448"}, - {file = "cleo-2.0.1.tar.gz", hash = "sha256:eb4b2e1f3063c11085cebe489a6e9124163c226575a3c3be69b2e51af4a15ec5"}, + {file = "cleo-2.1.0-py3-none-any.whl", hash = "sha256:4a31bd4dd45695a64ee3c4758f583f134267c2bc518d8ae9a29cf237d009b07e"}, + {file = "cleo-2.1.0.tar.gz", hash = "sha256:0b2c880b5d13660a7ea651001fb4acb527696c01f15c9ee650f377aa543fd523"}, ] [package.dependencies] crashtest = ">=0.4.1,<0.5.0" -rapidfuzz = ">=2.2.0,<3.0.0" +rapidfuzz = ">=3.0.0,<4.0.0" [[package]] name = "click" -version = "8.1.6" +version = "8.1.7" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" files = [ - {file = "click-8.1.6-py3-none-any.whl", hash = "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"}, - {file = "click-8.1.6.tar.gz", hash = "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd"}, + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, ] [package.dependencies] @@ -407,48 +414,49 @@ files = [ [[package]] name = "croniter" -version = "1.4.1" +version = "2.0.1" description = "croniter provides iteration for datetime object with cron like format" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ - {file = "croniter-1.4.1-py2.py3-none-any.whl", hash = "sha256:9595da48af37ea06ec3a9f899738f1b2c1c13da3c38cea606ef7cd03ea421128"}, - {file = "croniter-1.4.1.tar.gz", hash = "sha256:1a6df60eacec3b7a0aa52a8f2ef251ae3dd2a7c7c8b9874e73e791636d55a361"}, + {file = "croniter-2.0.1-py2.py3-none-any.whl", hash = "sha256:4cb064ce2d8f695b3b078be36ff50115cf8ac306c10a7e8653ee2a5b534673d7"}, + {file = "croniter-2.0.1.tar.gz", hash = "sha256:d199b2ec3ea5e82988d1f72022433c5f9302b3b3ea9e6bfd6a1518f6ea5e700a"}, ] [package.dependencies] python-dateutil = "*" +pytz = ">2021.1" [[package]] name = "cryptography" -version = "41.0.3" +version = "41.0.7" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-41.0.3-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:652627a055cb52a84f8c448185922241dd5217443ca194d5739b44612c5e6507"}, - {file = "cryptography-41.0.3-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:8f09daa483aedea50d249ef98ed500569841d6498aa9c9f4b0531b9964658922"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fd871184321100fb400d759ad0cddddf284c4b696568204d281c902fc7b0d81"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84537453d57f55a50a5b6835622ee405816999a7113267739a1b4581f83535bd"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3fb248989b6363906827284cd20cca63bb1a757e0a2864d4c1682a985e3dca47"}, - {file = "cryptography-41.0.3-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:42cb413e01a5d36da9929baa9d70ca90d90b969269e5a12d39c1e0d475010116"}, - {file = "cryptography-41.0.3-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:aeb57c421b34af8f9fe830e1955bf493a86a7996cc1338fe41b30047d16e962c"}, - {file = "cryptography-41.0.3-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:6af1c6387c531cd364b72c28daa29232162010d952ceb7e5ca8e2827526aceae"}, - {file = "cryptography-41.0.3-cp37-abi3-win32.whl", hash = "sha256:0d09fb5356f975974dbcb595ad2d178305e5050656affb7890a1583f5e02a306"}, - {file = "cryptography-41.0.3-cp37-abi3-win_amd64.whl", hash = "sha256:a983e441a00a9d57a4d7c91b3116a37ae602907a7618b882c8013b5762e80574"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5259cb659aa43005eb55a0e4ff2c825ca111a0da1814202c64d28a985d33b087"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:67e120e9a577c64fe1f611e53b30b3e69744e5910ff3b6e97e935aeb96005858"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:7efe8041897fe7a50863e51b77789b657a133c75c3b094e51b5e4b5cec7bf906"}, - {file = "cryptography-41.0.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ce785cf81a7bdade534297ef9e490ddff800d956625020ab2ec2780a556c313e"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:57a51b89f954f216a81c9d057bf1a24e2f36e764a1ca9a501a6964eb4a6800dd"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:4c2f0d35703d61002a2bbdcf15548ebb701cfdd83cdc12471d2bae80878a4207"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:23c2d778cf829f7d0ae180600b17e9fceea3c2ef8b31a99e3c694cbbf3a24b84"}, - {file = "cryptography-41.0.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:95dd7f261bb76948b52a5330ba5202b91a26fbac13ad0e9fc8a3ac04752058c7"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:41d7aa7cdfded09b3d73a47f429c298e80796c8e825ddfadc84c8a7f12df212d"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d0d651aa754ef58d75cec6edfbd21259d93810b73f6ec246436a21b7841908de"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:ab8de0d091acbf778f74286f4989cf3d1528336af1b59f3e5d2ebca8b5fe49e1"}, - {file = "cryptography-41.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a74fbcdb2a0d46fe00504f571a2a540532f4c188e6ccf26f1f178480117b33c4"}, - {file = "cryptography-41.0.3.tar.gz", hash = "sha256:6d192741113ef5e30d89dcb5b956ef4e1578f304708701b8b73d38e3e1461f34"}, + {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf"}, + {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1"}, + {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157"}, + {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406"}, + {file = "cryptography-41.0.7-cp37-abi3-win32.whl", hash = "sha256:f983596065a18a2183e7f79ab3fd4c475205b839e02cbc0efbbf9666c4b3083d"}, + {file = "cryptography-41.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:90452ba79b8788fa380dfb587cca692976ef4e757b194b093d845e8d99f612f2"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:079b85658ea2f59c4f43b70f8119a52414cdb7be34da5d019a77bf96d473b960"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d6c391c021ab1f7a82da5d8d0b3cee2f4b2c455ec86c8aebbc84837a631ff309"}, + {file = "cryptography-41.0.7.tar.gz", hash = "sha256:13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc"}, ] [package.dependencies] @@ -466,31 +474,31 @@ test-randomorder = ["pytest-randomly"] [[package]] name = "dagit" -version = "1.4.4" +version = "1.5.14" description = "Web UI for dagster." optional = false python-versions = "*" files = [ - {file = "dagit-1.4.4-py3-none-any.whl", hash = "sha256:cf10a16546c6e81618af9cc6cbe8a1914c8e60df191c1fdd38c3ce8e874f64a5"}, - {file = "dagit-1.4.4.tar.gz", hash = "sha256:83778973f07b97ae415ecc67c86ee502395e7d882d474827a4e914766122dbf3"}, + {file = "dagit-1.5.14-py3-none-any.whl", hash = "sha256:2353eb039c99409adc2935593cbdf23cc10d0436ac68e54c548e7203b9412528"}, + {file = "dagit-1.5.14.tar.gz", hash = "sha256:7ce254bdee417e8e63730258f4de2551894dba3fa86b58a4236a26561d92d560"}, ] [package.dependencies] -dagster-webserver = "1.4.4" +dagster-webserver = "1.5.14" [package.extras] -notebook = ["dagster-webserver[notebook] (==1.4.4)"] -test = ["dagster-webserver[test] (==1.4.4)"] +notebook = ["dagster-webserver[notebook] (==1.5.14)"] +test = ["dagster-webserver[test] (==1.5.14)"] [[package]] name = "dagster" -version = "1.4.4" -description = "The data orchestration platform built for productivity." +version = "1.5.14" +description = "Dagster is an orchestration platform for the development, production, and observation of data assets." optional = false python-versions = "*" files = [ - {file = "dagster-1.4.4-py3-none-any.whl", hash = "sha256:8790005fef7d21e65bdf206908706b486181365b908242edf6d0d06a97901a75"}, - {file = "dagster-1.4.4.tar.gz", hash = "sha256:4e4d07609489b3499ab4d3f0b24796f860c57f35d5234d73bc6869f1dda39d47"}, + {file = "dagster-1.5.14-py3-none-any.whl", hash = "sha256:951df17927a4dd5d594ccd349c9f3b82f1c671b28ac47b4ec8bf5e3ee57ad254"}, + {file = "dagster-1.5.14.tar.gz", hash = "sha256:511ecbdbbab794853791badec77580162022cc2362a2e2b5f70661ec1ac1ce79"}, ] [package.dependencies] @@ -498,15 +506,16 @@ alembic = ">=1.2.1,<1.6.3 || >1.6.3,<1.7.0 || >1.7.0,<1.11.0 || >1.11.0" click = ">=5.0" coloredlogs = ">=6.1,<=14.0" croniter = ">=0.3.34" +dagster-pipes = "1.5.14" docstring-parser = "*" grpcio = ">=1.44.0" grpcio-health-checking = ">=1.44.0" Jinja2 = "*" packaging = ">=20.9" -pendulum = "*" -protobuf = ">=3.20.0" +pendulum = ">=0.7.0,<3" +protobuf = ">=3.20.0,<5" psutil = {version = ">=1.0", markers = "platform_system == \"Windows\""} -pydantic = "<1.10.7 || >1.10.7,<2.0.0" +pydantic = ">1.10.0,<1.10.7 || >1.10.7,<3" python-dateutil = "*" python-dotenv = "*" pytz = "*" @@ -514,60 +523,59 @@ pywin32 = {version = "!=226", markers = "platform_system == \"Windows\""} PyYAML = ">=5.1" requests = "*" setuptools = "*" -sqlalchemy = ">=1.0" +sqlalchemy = ">=1.0,<3" tabulate = "*" -tomli = "*" +tomli = "<3" toposort = ">=1.0" -tqdm = "*" -typing-extensions = ">=4.4.0" -universal-pathlib = "<0.1.0" +tqdm = "<5" +typing-extensions = ">=4.4.0,<5" +universal-pathlib = "*" watchdog = ">=0.8.3" [package.extras] -black = ["black[jupyter] (==22.12.0)"] docker = ["docker"] mypy = ["mypy (==0.991)"] -pyright = ["pandas-stubs", "pyright (==1.1.316)", "types-PyYAML", "types-backports", "types-certifi", "types-chardet", "types-croniter", "types-cryptography", "types-mock", "types-paramiko", "types-pkg-resources", "types-pyOpenSSL", "types-python-dateutil", "types-pytz", "types-requests", "types-simplejson", "types-six", "types-sqlalchemy (==1.4.53.34)", "types-tabulate", "types-toml", "types-tzlocal"] -ruff = ["ruff (==0.0.277)"] -test = ["buildkite-test-collector", "docker", "grpcio-tools (>=1.44.0)", "mock (==3.0.5)", "morefs[asynclocal]", "objgraph", "pytest (==7.0.1)", "pytest-cov (==2.10.1)", "pytest-dependency (==0.5.1)", "pytest-mock (==3.3.1)", "pytest-rerunfailures (==10.0)", "pytest-runner (==5.2)", "pytest-xdist (==2.1.0)", "responses (<=0.23.1)", "syrupy (<4)", "tox (==3.25.0)", "yamllint"] +pyright = ["pandas-stubs", "pyright (==1.1.339)", "types-PyYAML", "types-backports", "types-certifi", "types-chardet", "types-croniter", "types-cryptography", "types-mock", "types-paramiko", "types-pkg-resources", "types-pyOpenSSL", "types-python-dateutil", "types-pytz", "types-requests", "types-simplejson", "types-six", "types-sqlalchemy (==1.4.53.34)", "types-tabulate", "types-toml", "types-tzlocal"] +ruff = ["ruff (==0.1.7)"] +test = ["buildkite-test-collector", "docker", "grpcio-tools (>=1.44.0)", "mock (==3.0.5)", "morefs[asynclocal]", "mypy-protobuf", "objgraph", "pytest (>=7.0.1)", "pytest-cov (==2.10.1)", "pytest-dependency (==0.5.1)", "pytest-mock (==3.3.1)", "pytest-rerunfailures (==10.0)", "pytest-runner (==5.2)", "pytest-xdist (==3.3.1)", "responses (<=0.23.1)", "syrupy (<4)", "tox (==3.25.0)"] [[package]] name = "dagster-cloud" -version = "1.4.4" +version = "1.5.14" description = "" optional = false python-versions = "*" files = [ - {file = "dagster_cloud-1.4.4-py3-none-any.whl", hash = "sha256:fe0c1a098530d33cdb440dc29d6ae55fdcc02eb1e7ce3a6ea4582342881a6842"}, - {file = "dagster_cloud-1.4.4.tar.gz", hash = "sha256:047cf1dacac012311252cfb505f1229e912e3e175a9cbe0549ae6b3facfd5417"}, + {file = "dagster-cloud-1.5.14.tar.gz", hash = "sha256:d21c3f445d775feb1ba28049c75b2098e11bab27dc0f23633595875367b96d8b"}, + {file = "dagster_cloud-1.5.14-py3-none-any.whl", hash = "sha256:928c66649efd86acd959482741eb4b7eb92522f68c4cc49620f2d7a259d70aff"}, ] [package.dependencies] -dagster = "1.4.4" -dagster-cloud-cli = "1.4.4" -pex = "*" +dagster = "1.5.14" +dagster-cloud-cli = "1.5.14" +pex = ">=2.1.132" questionary = "*" requests = "*" typer = {version = "*", extras = ["all"]} [package.extras] -docker = ["dagster-docker (==0.20.4)", "docker"] -ecs = ["boto3", "dagster-aws (==0.20.4)"] -kubernetes = ["dagster-k8s (==0.20.4)", "kubernetes"] +docker = ["dagster-docker (==0.21.14)", "docker"] +ecs = ["boto3", "dagster-aws (==0.21.14)"] +kubernetes = ["dagster-k8s (==0.21.14)", "kubernetes"] pex = ["boto3"] sandbox = ["supervisor"] serverless = ["boto3"] -tests = ["black", "dagster-cloud-test-infra", "dagster-k8s (==0.20.4)", "docker", "httpretty", "isort", "kubernetes", "moto[all]", "mypy", "paramiko", "pylint", "pytest", "types-PyYAML", "types-requests"] +tests = ["dagster-cloud-test-infra", "dagster-dbt (==0.21.14)", "dagster-k8s (==0.21.14)", "dbt-core", "dbt-duckdb", "dbt-postgres", "dbt-snowflake", "docker", "httpretty", "isort", "kubernetes", "moto[all]", "mypy", "paramiko", "psutil", "pylint", "pytest", "types-PyYAML", "types-requests"] [[package]] name = "dagster-cloud-cli" -version = "1.4.4" +version = "1.5.14" description = "" optional = false python-versions = "*" files = [ - {file = "dagster_cloud_cli-1.4.4-py3-none-any.whl", hash = "sha256:f38f230bb21a4535765762f92b5d06438a507da7bab57fe7db91c27cc70fe60f"}, - {file = "dagster_cloud_cli-1.4.4.tar.gz", hash = "sha256:6ae9f5bd1b9235108c6131551752953a88613e71c20d9b4086597c8a9966f2a4"}, + {file = "dagster-cloud-cli-1.5.14.tar.gz", hash = "sha256:75045561a77e97c6b223b71d0fa7c2fe371c72e9818696849ec5b15df9a3e60c"}, + {file = "dagster_cloud_cli-1.5.14-py3-none-any.whl", hash = "sha256:6f9150ff2a8e0c2ddde1f831f0cee17932c0cc0e2e1b27cd701f61b2d7f42749"}, ] [package.dependencies] @@ -583,18 +591,18 @@ tests = ["freezegun"] [[package]] name = "dagster-gcp" -version = "0.20.4" +version = "0.21.14" description = "Package for GCP-specific Dagster framework op and resource components." optional = false python-versions = "*" files = [ - {file = "dagster-gcp-0.20.4.tar.gz", hash = "sha256:b3c76ea8398a41016e58374cd9699514ae1903e503b426347dea17adca0ea758"}, - {file = "dagster_gcp-0.20.4-py3-none-any.whl", hash = "sha256:2cb241f47e98cfbc3f3c2af64e7260923c6ba717929f672f4a039ec988b0de61"}, + {file = "dagster-gcp-0.21.14.tar.gz", hash = "sha256:caa8196d51f56cba658179a28ad6efb844493f705d4318d05b5edee82c1966c1"}, + {file = "dagster_gcp-0.21.14-py3-none-any.whl", hash = "sha256:28a7cebcbedc7b54b67f04281424a36cd89f722bdf5411173187855e2938cb69"}, ] [package.dependencies] -dagster = "1.4.4" -dagster-pandas = "0.20.4" +dagster = "1.5.14" +dagster-pandas = "0.21.14" db-dtypes = "*" google-api-python-client = "*" google-cloud-bigquery = "*" @@ -606,68 +614,78 @@ pyarrow = ["pyarrow"] [[package]] name = "dagster-graphql" -version = "1.4.4" +version = "1.5.14" description = "The GraphQL frontend to python dagster." optional = false python-versions = "*" files = [ - {file = "dagster-graphql-1.4.4.tar.gz", hash = "sha256:7ca85756393aa6a4d0c2a43044e3a0d3e3a61bffb527fa82c936126296bfb5c6"}, - {file = "dagster_graphql-1.4.4-py3-none-any.whl", hash = "sha256:f919459f1edb8be2e1d02a28fa3600869a27be5d52d66eb253902e155d1a5a04"}, + {file = "dagster-graphql-1.5.14.tar.gz", hash = "sha256:208c66bfd68021c1606b97f830e87d8f1c16051cbaaa9a1570682344a2c60999"}, + {file = "dagster_graphql-1.5.14-py3-none-any.whl", hash = "sha256:991b9f3342a6dc139b84e9f836acec4f47c925993b2208685c6518e4620e7015"}, ] [package.dependencies] -dagster = "1.4.4" -gql = {version = ">=3.0.0", extras = ["requests"]} -graphene = ">=3" +dagster = "1.5.14" +gql = {version = ">=3,<4", extras = ["requests"]} +graphene = ">=3,<4" requests = "*" starlette = "*" -urllib3 = "<2.0.0" [[package]] name = "dagster-pandas" -version = "0.20.4" +version = "0.21.14" description = "Utilities and examples for working with pandas and dagster, an opinionated framework for expressing data pipelines" optional = false python-versions = "*" files = [ - {file = "dagster-pandas-0.20.4.tar.gz", hash = "sha256:954055ce711017e151f3a3f0466d99d55ffc16bf4554e357777d7a02e3413993"}, - {file = "dagster_pandas-0.20.4-py3-none-any.whl", hash = "sha256:f5e37ad885cd44e79f06eae412792b6284f9a0568f4ba606f895fe467cccaf74"}, + {file = "dagster-pandas-0.21.14.tar.gz", hash = "sha256:f3a961086f8386939248a97ed3991996307c9fbe8cbcb2bad2af372635d33347"}, + {file = "dagster_pandas-0.21.14-py3-none-any.whl", hash = "sha256:eadd0f380f3331f7e6d135990c3f154e1f9b2b147d098c2dd019308335ac0ab9"}, ] [package.dependencies] -dagster = "1.4.4" +dagster = "1.5.14" pandas = "*" +[[package]] +name = "dagster-pipes" +version = "1.5.14" +description = "Toolkit for Dagster integrations with transform logic outside of Dagster" +optional = false +python-versions = "*" +files = [ + {file = "dagster-pipes-1.5.14.tar.gz", hash = "sha256:a3302565cfcdb7d7e697f813a6950f96259c4190cb19b76f256932b937d3a80d"}, + {file = "dagster_pipes-1.5.14-py3-none-any.whl", hash = "sha256:f65883a618595f49c84172b4e93d4fcba9e29e5d102f8dc064af5c56f51ab1a6"}, +] + [[package]] name = "dagster-slack" -version = "0.20.4" +version = "0.21.14" description = "A Slack client resource for posting to Slack" optional = false python-versions = "*" files = [ - {file = "dagster-slack-0.20.4.tar.gz", hash = "sha256:c0a8dcedd722f4d0f15eb4322d6a0160f0360e24e1bfffc612624f967b99e3d2"}, - {file = "dagster_slack-0.20.4-py3-none-any.whl", hash = "sha256:4e418012bd94fda8303044282aedaec1d11ce697f7495161f23b745885223914"}, + {file = "dagster-slack-0.21.14.tar.gz", hash = "sha256:7a77776f1eb088fe66a061c92f61aab294d42d0e867c97fb0a82132b1bbf8f52"}, + {file = "dagster_slack-0.21.14-py3-none-any.whl", hash = "sha256:5c6e2344c4fc4036184881d0a1809b446e24e88753cbf3333bc93787962832bc"}, ] [package.dependencies] -dagster = "1.4.4" +dagster = "1.5.14" slack-sdk = "*" [[package]] name = "dagster-webserver" -version = "1.4.4" +version = "1.5.14" description = "Web UI for dagster." optional = false python-versions = "*" files = [ - {file = "dagster_webserver-1.4.4-py3-none-any.whl", hash = "sha256:80ebb430617a1949c7d3019fd2cc29178467d1d6b8136bd09b64fb13ba09103a"}, - {file = "dagster_webserver-1.4.4.tar.gz", hash = "sha256:3b1b0316d5937478f8ff734c2de10e2f5ae3da500fbdea47948947496fc60646"}, + {file = "dagster-webserver-1.5.14.tar.gz", hash = "sha256:33f692f382bced3e05d8a78bc15fc086aa15de542c455efa2675fd9ca1f3e7b1"}, + {file = "dagster_webserver-1.5.14-py3-none-any.whl", hash = "sha256:d75a3e64ea6b5ea77e738905bf662e9fb1302789661421826c15cd4fe00685bd"}, ] [package.dependencies] click = ">=7.0,<9.0" -dagster = "1.4.4" -dagster-graphql = "1.4.4" +dagster = "1.5.14" +dagster-graphql = "1.5.14" starlette = "*" uvicorn = {version = "*", extras = ["standard"]} @@ -677,13 +695,13 @@ test = ["starlette[full]"] [[package]] name = "db-dtypes" -version = "1.1.1" +version = "1.2.0" description = "Pandas Data Types for SQL systems (BigQuery, Spanner)" optional = false python-versions = ">=3.7" files = [ - {file = "db-dtypes-1.1.1.tar.gz", hash = "sha256:ab485c85fef2454f3182427def0b0a3ab179b2871542787d33ba519d62078883"}, - {file = "db_dtypes-1.1.1-py2.py3-none-any.whl", hash = "sha256:23be34ea2bc91065447ecea4d5f107e46d1de223d152e69fa73673a62d5bd27d"}, + {file = "db-dtypes-1.2.0.tar.gz", hash = "sha256:3531bb1fb8b5fbab33121fe243ccc2ade16ab2524f4c113b05cc702a1908e6ea"}, + {file = "db_dtypes-1.2.0-py2.py3-none-any.whl", hash = "sha256:6320bddd31d096447ef749224d64aab00972ed20e4392d86f7d8b81ad79f7ff0"}, ] [package.dependencies] @@ -694,20 +712,20 @@ pyarrow = ">=3.0.0" [[package]] name = "deepdiff" -version = "6.3.1" +version = "6.7.1" description = "Deep Difference and Search of any Python object/data. Recreate objects by adding adding deltas to each other." optional = false python-versions = ">=3.7" files = [ - {file = "deepdiff-6.3.1-py3-none-any.whl", hash = "sha256:eae2825b2e1ea83df5fc32683d9aec5a56e38b756eb2b280e00863ce4def9d33"}, - {file = "deepdiff-6.3.1.tar.gz", hash = "sha256:e8c1bb409a2caf1d757799add53b3a490f707dd792ada0eca7cac1328055097a"}, + {file = "deepdiff-6.7.1-py3-none-any.whl", hash = "sha256:58396bb7a863cbb4ed5193f548c56f18218060362311aa1dc36397b2f25108bd"}, + {file = "deepdiff-6.7.1.tar.gz", hash = "sha256:b367e6fa6caac1c9f500adc79ada1b5b1242c50d5f716a1a4362030197847d30"}, ] [package.dependencies] ordered-set = ">=4.0.2,<4.2.0" [package.extras] -cli = ["click (==8.1.3)", "pyyaml (==6.0)"] +cli = ["click (==8.1.3)", "pyyaml (==6.0.1)"] optimize = ["orjson"] [[package]] @@ -729,13 +747,13 @@ dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] [[package]] name = "distlib" -version = "0.3.7" +version = "0.3.8" description = "Distribution utilities" optional = false python-versions = "*" files = [ - {file = "distlib-0.3.7-py2.py3-none-any.whl", hash = "sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057"}, - {file = "distlib-0.3.7.tar.gz", hash = "sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8"}, + {file = "distlib-0.3.8-py2.py3-none-any.whl", hash = "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784"}, + {file = "distlib-0.3.8.tar.gz", hash = "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64"}, ] [[package]] @@ -762,67 +780,80 @@ files = [ [[package]] name = "dulwich" -version = "0.21.5" +version = "0.21.7" description = "Python Git Library" optional = false python-versions = ">=3.7" files = [ - {file = "dulwich-0.21.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8864719bc176cdd27847332a2059127e2f7bab7db2ff99a999873cb7fff54116"}, - {file = "dulwich-0.21.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3800cdc17d144c1f7e114972293bd6c46688f5bcc2c9228ed0537ded72394082"}, - {file = "dulwich-0.21.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e2f676bfed8146966fe934ee734969d7d81548fbd250a8308582973670a9dab1"}, - {file = "dulwich-0.21.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4db330fb59fe3b9d253bdf0e49a521739db83689520c4921ab1c5242aaf77b82"}, - {file = "dulwich-0.21.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3e8f6d4f4f4d01dd1d3c968e486d4cd77f96f772da7265941bc506de0944ddb9"}, - {file = "dulwich-0.21.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:1cc0c9ba19ac1b2372598802bc9201a9c45e5d6f1f7a80ec40deeb10acc4e9ae"}, - {file = "dulwich-0.21.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:61e10242b5a7a82faa8996b2c76239cfb633620b02cdd2946e8af6e7eb31d651"}, - {file = "dulwich-0.21.5-cp310-cp310-win32.whl", hash = "sha256:7f357639b56146a396f48e5e0bc9bbaca3d6d51c8340bd825299272b588fff5f"}, - {file = "dulwich-0.21.5-cp310-cp310-win_amd64.whl", hash = "sha256:891d5c73e2b66d05dbb502e44f027dc0dbbd8f6198bc90dae348152e69d0befc"}, - {file = "dulwich-0.21.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:45d6198e804b539708b73a003419e48fb42ff2c3c6dd93f63f3b134dff6dd259"}, - {file = "dulwich-0.21.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c2a565d4e704d7f784cdf9637097141f6d47129c8fffc2fac699d57cb075a169"}, - {file = "dulwich-0.21.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:823091d6b6a1ea07dc4839c9752198fb39193213d103ac189c7669736be2eaff"}, - {file = "dulwich-0.21.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b2c9931b657f2206abec0964ec2355ee2c1e04d05f8864e823ffa23c548c4548"}, - {file = "dulwich-0.21.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7dc358c2ee727322a09b7c6da43d47a1026049dbd3ad8d612eddca1f9074b298"}, - {file = "dulwich-0.21.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6155ab7388ee01c670f7c5d8003d4e133eebebc7085a856c007989f0ba921b36"}, - {file = "dulwich-0.21.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a605e10d72f90a39ea2e634fbfd80f866fc4df29a02ea6db52ae92e5fd4a2003"}, - {file = "dulwich-0.21.5-cp311-cp311-win32.whl", hash = "sha256:daa607370722c3dce99a0022397c141caefb5ed32032a4f72506f4817ea6405b"}, - {file = "dulwich-0.21.5-cp311-cp311-win_amd64.whl", hash = "sha256:5e56b2c1911c344527edb2bf1a4356e2fb7e086b1ba309666e1e5c2224cdca8a"}, - {file = "dulwich-0.21.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:85d3401d08b1ec78c7d58ae987c4bb7b768a438f3daa74aeb8372bebc7fb16fa"}, - {file = "dulwich-0.21.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:90479608e49db93d8c9e4323bc0ec5496678b535446e29d8fd67dc5bbb5d51bf"}, - {file = "dulwich-0.21.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9a6bf99f57bcac4c77fc60a58f1b322c91cc4d8c65dc341f76bf402622f89cb"}, - {file = "dulwich-0.21.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3e68b162af2aae995355e7920f89d50d72b53d56021e5ac0a546d493b17cbf7e"}, - {file = "dulwich-0.21.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0ab86d6d42e385bf3438e70f3c9b16de68018bd88929379e3484c0ef7990bd3c"}, - {file = "dulwich-0.21.5-cp37-cp37m-win32.whl", hash = "sha256:f2eeca6d61366cf5ee8aef45bed4245a67d4c0f0d731dc2383eabb80fa695683"}, - {file = "dulwich-0.21.5-cp37-cp37m-win_amd64.whl", hash = "sha256:1b20a3656b48c941d49c536824e1e5278a695560e8de1a83b53a630143c4552e"}, - {file = "dulwich-0.21.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:3932b5e17503b265a85f1eda77ede647681c3bab53bc9572955b6b282abd26ea"}, - {file = "dulwich-0.21.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6616132d219234580de88ceb85dd51480dc43b1bdc05887214b8dd9cfd4a9d40"}, - {file = "dulwich-0.21.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:eaf6c7fb6b13495c19c9aace88821c2ade3c8c55b4e216cd7cc55d3e3807d7fa"}, - {file = "dulwich-0.21.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be12a46f73023970125808a4a78f610c055373096c1ecea3280edee41613eba8"}, - {file = "dulwich-0.21.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baecef0d8b9199822c7912876a03a1af17833f6c0d461efb62decebd45897e49"}, - {file = "dulwich-0.21.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:82f632afb9c7c341a875d46aaa3e6c5e586c7a64ce36c9544fa400f7e4f29754"}, - {file = "dulwich-0.21.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82cdf482f8f51fcc965ffad66180b54a9abaea9b1e985a32e1acbfedf6e0e363"}, - {file = "dulwich-0.21.5-cp38-cp38-win32.whl", hash = "sha256:c8ded43dc0bd2e65420eb01e778034be5ca7f72e397a839167eda7dcb87c4248"}, - {file = "dulwich-0.21.5-cp38-cp38-win_amd64.whl", hash = "sha256:2aba0fdad2a19bd5bb3aad6882580cb33359c67b48412ccd4cfccd932012b35e"}, - {file = "dulwich-0.21.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:fd4ad079758514375f11469e081723ba8831ce4eaa1a64b41f06a3a866d5ac34"}, - {file = "dulwich-0.21.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7fe62685bf356bfb4d0738f84a3fcf0d1fc9e11fee152e488a20b8c66a52429e"}, - {file = "dulwich-0.21.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:aae448da7d80306dda4fc46292fed7efaa466294571ab3448be16714305076f1"}, - {file = "dulwich-0.21.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b24cb1fad0525dba4872e9381bc576ea2a6dcdf06b0ed98f8e953e3b1d719b89"}, - {file = "dulwich-0.21.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e39b7c2c9bda6acae83b25054650a8bb7e373e886e2334721d384e1479bf04b"}, - {file = "dulwich-0.21.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:26456dba39d1209fca17187db06967130e27eeecad2b3c2bbbe63467b0bf09d6"}, - {file = "dulwich-0.21.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:281310644e02e3aa6d76bcaffe2063b9031213c4916b5f1a6e68c25bdecfaba4"}, - {file = "dulwich-0.21.5-cp39-cp39-win32.whl", hash = "sha256:4814ca3209dabe0fe7719e9545fbdad7f8bb250c5a225964fe2a31069940c4cf"}, - {file = "dulwich-0.21.5-cp39-cp39-win_amd64.whl", hash = "sha256:c922a4573267486be0ef85216f2da103fb38075b8465dc0e90457843884e4860"}, - {file = "dulwich-0.21.5-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:e52b20c4368171b7d32bd3ab0f1d2402e76ad4f2ea915ff9aa73bc9fa2b54d6d"}, - {file = "dulwich-0.21.5-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aeb736d777ee21f2117a90fc453ee181aa7eedb9e255b5ef07c51733f3fe5cb6"}, - {file = "dulwich-0.21.5-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5e8a79c1ed7166f32ad21974fa98d11bf6fd74e94a47e754c777c320e01257c6"}, - {file = "dulwich-0.21.5-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:b943517e30bd651fbc275a892bb96774f3893d95fe5a4dedd84496a98eaaa8ab"}, - {file = "dulwich-0.21.5-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:32493a456358a3a6c15bbda07106fc3d4cc50834ee18bc7717968d18be59b223"}, - {file = "dulwich-0.21.5-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0aa44b812d978fc22a04531f5090c3c369d5facd03fa6e0501d460a661800c7f"}, - {file = "dulwich-0.21.5-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f46bcb6777e5f9f4af24a2bd029e88b77316269d24ce66be590e546a0d8f7b7"}, - {file = "dulwich-0.21.5-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:a917fd3b4493db3716da2260f16f6b18f68d46fbe491d851d154fc0c2d984ae4"}, - {file = "dulwich-0.21.5-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:684c52cff867d10c75a7238151ca307582b3d251bbcd6db9e9cffbc998ef804e"}, - {file = "dulwich-0.21.5-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9019189d7a8f7394df6a22cd5b484238c5776e42282ad5d6d6c626b4c5f43597"}, - {file = "dulwich-0.21.5-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:494024f74c2eef9988adb4352b3651ac1b6c0466176ec62b69d3d3672167ba68"}, - {file = "dulwich-0.21.5-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:f9b6ac1b1c67fc6083c42b7b6cd3b211292c8a6517216c733caf23e8b103ab6d"}, - {file = "dulwich-0.21.5.tar.gz", hash = "sha256:70955e4e249ddda6e34a4636b90f74e931e558f993b17c52570fa6144b993103"}, + {file = "dulwich-0.21.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d4c0110798099bb7d36a110090f2688050703065448895c4f53ade808d889dd3"}, + {file = "dulwich-0.21.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2bc12697f0918bee324c18836053644035362bb3983dc1b210318f2fed1d7132"}, + {file = "dulwich-0.21.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:471305af74790827fcbafe330fc2e8bdcee4fb56ca1177c8c481b1c8f806c4a4"}, + {file = "dulwich-0.21.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d54c9d0e845be26f65f954dff13a1cd3f2b9739820c19064257b8fd7435ab263"}, + {file = "dulwich-0.21.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:12d61334a575474e707614f2e93d6ed4cdae9eb47214f9277076d9e5615171d3"}, + {file = "dulwich-0.21.7-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e274cebaf345f0b1e3b70197f2651de92b652386b68020cfd3bf61bc30f6eaaa"}, + {file = "dulwich-0.21.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:817822f970e196e757ae01281ecbf21369383285b9f4a83496312204cf889b8c"}, + {file = "dulwich-0.21.7-cp310-cp310-win32.whl", hash = "sha256:7836da3f4110ce684dcd53489015fb7fa94ed33c5276e3318b8b1cbcb5b71e08"}, + {file = "dulwich-0.21.7-cp310-cp310-win_amd64.whl", hash = "sha256:4a043b90958cec866b4edc6aef5fe3c2c96a664d0b357e1682a46f6c477273c4"}, + {file = "dulwich-0.21.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ce8db196e79c1f381469410d26fb1d8b89c6b87a4e7f00ff418c22a35121405c"}, + {file = "dulwich-0.21.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:62bfb26bdce869cd40be443dfd93143caea7089b165d2dcc33de40f6ac9d812a"}, + {file = "dulwich-0.21.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c01a735b9a171dcb634a97a3cec1b174cfbfa8e840156870384b633da0460f18"}, + {file = "dulwich-0.21.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fa4d14767cf7a49c9231c2e52cb2a3e90d0c83f843eb6a2ca2b5d81d254cf6b9"}, + {file = "dulwich-0.21.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7bca4b86e96d6ef18c5bc39828ea349efb5be2f9b1f6ac9863f90589bac1084d"}, + {file = "dulwich-0.21.7-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a7b5624b02ef808cdc62dabd47eb10cd4ac15e8ac6df9e2e88b6ac6b40133673"}, + {file = "dulwich-0.21.7-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c3a539b4696a42fbdb7412cb7b66a4d4d332761299d3613d90a642923c7560e1"}, + {file = "dulwich-0.21.7-cp311-cp311-win32.whl", hash = "sha256:675a612ce913081beb0f37b286891e795d905691dfccfb9bf73721dca6757cde"}, + {file = "dulwich-0.21.7-cp311-cp311-win_amd64.whl", hash = "sha256:460ba74bdb19f8d498786ae7776745875059b1178066208c0fd509792d7f7bfc"}, + {file = "dulwich-0.21.7-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:4c51058ec4c0b45dc5189225b9e0c671b96ca9713c1daf71d622c13b0ab07681"}, + {file = "dulwich-0.21.7-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:4bc4c5366eaf26dda3fdffe160a3b515666ed27c2419f1d483da285ac1411de0"}, + {file = "dulwich-0.21.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a0650ec77d89cb947e3e4bbd4841c96f74e52b4650830112c3057a8ca891dc2f"}, + {file = "dulwich-0.21.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f18f0a311fb7734b033a3101292b932158cade54b74d1c44db519e42825e5a2"}, + {file = "dulwich-0.21.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c589468e5c0cd84e97eb7ec209ab005a2cb69399e8c5861c3edfe38989ac3a8"}, + {file = "dulwich-0.21.7-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d62446797163317a397a10080c6397ffaaca51a7804c0120b334f8165736c56a"}, + {file = "dulwich-0.21.7-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e84cc606b1f581733df4350ca4070e6a8b30be3662bbb81a590b177d0c996c91"}, + {file = "dulwich-0.21.7-cp312-cp312-win32.whl", hash = "sha256:c3d1685f320907a52c40fd5890627945c51f3a5fa4bcfe10edb24fec79caadec"}, + {file = "dulwich-0.21.7-cp312-cp312-win_amd64.whl", hash = "sha256:6bd69921fdd813b7469a3c77bc75c1783cc1d8d72ab15a406598e5a3ba1a1503"}, + {file = "dulwich-0.21.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7d8ab29c660125db52106775caa1f8f7f77a69ed1fe8bc4b42bdf115731a25bf"}, + {file = "dulwich-0.21.7-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0d2e4485b98695bf95350ce9d38b1bb0aaac2c34ad00a0df789aa33c934469b"}, + {file = "dulwich-0.21.7-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e138d516baa6b5bafbe8f030eccc544d0d486d6819b82387fc0e285e62ef5261"}, + {file = "dulwich-0.21.7-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f34bf9b9fa9308376263fd9ac43143c7c09da9bc75037bb75c6c2423a151b92c"}, + {file = "dulwich-0.21.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2e2c66888207b71cd1daa2acb06d3984a6bc13787b837397a64117aa9fc5936a"}, + {file = "dulwich-0.21.7-cp37-cp37m-win32.whl", hash = "sha256:10893105c6566fc95bc2a67b61df7cc1e8f9126d02a1df6a8b2b82eb59db8ab9"}, + {file = "dulwich-0.21.7-cp37-cp37m-win_amd64.whl", hash = "sha256:460b3849d5c3d3818a80743b4f7a0094c893c559f678e56a02fff570b49a644a"}, + {file = "dulwich-0.21.7-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:74700e4c7d532877355743336c36f51b414d01e92ba7d304c4f8d9a5946dbc81"}, + {file = "dulwich-0.21.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c92e72c43c9e9e936b01a57167e0ea77d3fd2d82416edf9489faa87278a1cdf7"}, + {file = "dulwich-0.21.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d097e963eb6b9fa53266146471531ad9c6765bf390849230311514546ed64db2"}, + {file = "dulwich-0.21.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:808e8b9cc0aa9ac74870b49db4f9f39a52fb61694573f84b9c0613c928d4caf8"}, + {file = "dulwich-0.21.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1957b65f96e36c301e419d7adaadcff47647c30eb072468901bb683b1000bc5"}, + {file = "dulwich-0.21.7-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:4b09bc3a64fb70132ec14326ecbe6e0555381108caff3496898962c4136a48c6"}, + {file = "dulwich-0.21.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5882e70b74ac3c736a42d3fdd4f5f2e6570637f59ad5d3e684760290b58f041"}, + {file = "dulwich-0.21.7-cp38-cp38-win32.whl", hash = "sha256:29bb5c1d70eba155ded41ed8a62be2f72edbb3c77b08f65b89c03976292f6d1b"}, + {file = "dulwich-0.21.7-cp38-cp38-win_amd64.whl", hash = "sha256:25c3ab8fb2e201ad2031ddd32e4c68b7c03cb34b24a5ff477b7a7dcef86372f5"}, + {file = "dulwich-0.21.7-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8929c37986c83deb4eb500c766ee28b6670285b512402647ee02a857320e377c"}, + {file = "dulwich-0.21.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cc1e11be527ac06316539b57a7688bcb1b6a3e53933bc2f844397bc50734e9ae"}, + {file = "dulwich-0.21.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0fc3078a1ba04c588fabb0969d3530efd5cd1ce2cf248eefb6baf7cbc15fc285"}, + {file = "dulwich-0.21.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40dcbd29ba30ba2c5bfbab07a61a5f20095541d5ac66d813056c122244df4ac0"}, + {file = "dulwich-0.21.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8869fc8ec3dda743e03d06d698ad489b3705775fe62825e00fa95aa158097fc0"}, + {file = "dulwich-0.21.7-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d96ca5e0dde49376fbcb44f10eddb6c30284a87bd03bb577c59bb0a1f63903fa"}, + {file = "dulwich-0.21.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e0064363bd5e814359657ae32517fa8001e8573d9d040bd997908d488ab886ed"}, + {file = "dulwich-0.21.7-cp39-cp39-win32.whl", hash = "sha256:869eb7be48243e695673b07905d18b73d1054a85e1f6e298fe63ba2843bb2ca1"}, + {file = "dulwich-0.21.7-cp39-cp39-win_amd64.whl", hash = "sha256:404b8edeb3c3a86c47c0a498699fc064c93fa1f8bab2ffe919e8ab03eafaaad3"}, + {file = "dulwich-0.21.7-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:e598d743c6c0548ebcd2baf94aa9c8bfacb787ea671eeeb5828cfbd7d56b552f"}, + {file = "dulwich-0.21.7-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4a2d76c96426e791556836ef43542b639def81be4f1d6d4322cd886c115eae1"}, + {file = "dulwich-0.21.7-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6c88acb60a1f4d31bd6d13bfba465853b3df940ee4a0f2a3d6c7a0778c705b7"}, + {file = "dulwich-0.21.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ecd315847dea406a4decfa39d388a2521e4e31acde3bd9c2609c989e817c6d62"}, + {file = "dulwich-0.21.7-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:d05d3c781bc74e2c2a2a8f4e4e2ed693540fbe88e6ac36df81deac574a6dad99"}, + {file = "dulwich-0.21.7-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6de6f8de4a453fdbae8062a6faa652255d22a3d8bce0cd6d2d6701305c75f2b3"}, + {file = "dulwich-0.21.7-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e25953c7acbbe4e19650d0225af1c0c0e6882f8bddd2056f75c1cc2b109b88ad"}, + {file = "dulwich-0.21.7-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:4637cbd8ed1012f67e1068aaed19fcc8b649bcf3e9e26649826a303298c89b9d"}, + {file = "dulwich-0.21.7-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:858842b30ad6486aacaa607d60bab9c9a29e7c59dc2d9cb77ae5a94053878c08"}, + {file = "dulwich-0.21.7-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:739b191f61e1c4ce18ac7d520e7a7cbda00e182c3489552408237200ce8411ad"}, + {file = "dulwich-0.21.7-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:274c18ec3599a92a9b67abaf110e4f181a4f779ee1aaab9e23a72e89d71b2bd9"}, + {file = "dulwich-0.21.7-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:2590e9b431efa94fc356ae33b38f5e64f1834ec3a94a6ac3a64283b206d07aa3"}, + {file = "dulwich-0.21.7-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ed60d1f610ef6437586f7768254c2a93820ccbd4cfdac7d182cf2d6e615969bb"}, + {file = "dulwich-0.21.7-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8278835e168dd097089f9e53088c7a69c6ca0841aef580d9603eafe9aea8c358"}, + {file = "dulwich-0.21.7-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffc27fb063f740712e02b4d2f826aee8bbed737ed799962fef625e2ce56e2d29"}, + {file = "dulwich-0.21.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:61e3451bd3d3844f2dca53f131982553be4d1b1e1ebd9db701843dd76c4dba31"}, + {file = "dulwich-0.21.7.tar.gz", hash = "sha256:a9e9c66833cea580c3ac12927e4b9711985d76afca98da971405d414de60e968"}, ] [package.dependencies] @@ -836,13 +867,13 @@ pgp = ["gpg"] [[package]] name = "exceptiongroup" -version = "1.1.2" +version = "1.2.0" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"}, - {file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"}, + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, ] [package.extras] @@ -863,30 +894,45 @@ files = [ pyreadline = {version = "*", markers = "platform_system == \"Windows\""} pyrepl = ">=0.8.2" +[[package]] +name = "fastjsonschema" +version = "2.19.1" +description = "Fastest Python implementation of JSON schema" +optional = false +python-versions = "*" +files = [ + {file = "fastjsonschema-2.19.1-py3-none-any.whl", hash = "sha256:3672b47bc94178c9f23dbb654bf47440155d4db9df5f7bc47643315f9c405cd0"}, + {file = "fastjsonschema-2.19.1.tar.gz", hash = "sha256:e3126a94bdc4623d3de4485f8d468a12f02a67921315ddc87836d6e456dc789d"}, +] + +[package.extras] +devel = ["colorama", "json-spec", "jsonschema", "pylint", "pytest", "pytest-benchmark", "pytest-cache", "validictory"] + [[package]] name = "filelock" -version = "3.12.2" +version = "3.13.1" description = "A platform independent file lock." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "filelock-3.12.2-py3-none-any.whl", hash = "sha256:cbb791cdea2a72f23da6ac5b5269ab0a0d161e9ef0100e653b69049a7706d1ec"}, - {file = "filelock-3.12.2.tar.gz", hash = "sha256:002740518d8aa59a26b0c76e10fb8c6e15eae825d34b6fdf670333fd7b938d81"}, + {file = "filelock-3.13.1-py3-none-any.whl", hash = "sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c"}, + {file = "filelock-3.13.1.tar.gz", hash = "sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e"}, ] [package.extras] -docs = ["furo (>=2023.5.20)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] -testing = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "diff-cover (>=7.5)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)", "pytest-timeout (>=2.1)"] +docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.24)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"] +typing = ["typing-extensions (>=4.8)"] [[package]] name = "fsspec" -version = "2023.6.0" +version = "2023.12.2" description = "File-system specification" optional = false python-versions = ">=3.8" files = [ - {file = "fsspec-2023.6.0-py3-none-any.whl", hash = "sha256:1cbad1faef3e391fba6dc005ae9b5bdcbf43005c9167ce78c915549c352c869a"}, - {file = "fsspec-2023.6.0.tar.gz", hash = "sha256:d0b2f935446169753e7a5c5c55681c54ea91996cc67be93c39a154fb3a2742af"}, + {file = "fsspec-2023.12.2-py3-none-any.whl", hash = "sha256:d800d87f72189a745fa3d6b033b9dc4a34ad069f60ca60b943a63599f5501960"}, + {file = "fsspec-2023.12.2.tar.gz", hash = "sha256:8548d39e8810b59c38014934f6b31e57f40c1b20f911f4cc2b85389c7e9bf0cb"}, ] [package.extras] @@ -1018,26 +1064,18 @@ beautifulsoup4 = "*" [[package]] name = "google-api-core" -version = "2.11.1" +version = "2.15.0" description = "Google API client core library" optional = false python-versions = ">=3.7" files = [ - {file = "google-api-core-2.11.1.tar.gz", hash = "sha256:25d29e05a0058ed5f19c61c0a78b1b53adea4d9364b464d014fbda941f6d1c9a"}, - {file = "google_api_core-2.11.1-py3-none-any.whl", hash = "sha256:d92a5a92dc36dd4f4b9ee4e55528a90e432b059f93aee6ad857f9de8cc7ae94a"}, + {file = "google-api-core-2.15.0.tar.gz", hash = "sha256:abc978a72658f14a2df1e5e12532effe40f94f868f6e23d95133bd6abcca35ca"}, + {file = "google_api_core-2.15.0-py3-none-any.whl", hash = "sha256:2aa56d2be495551e66bbff7f729b790546f87d5c90e74781aa77233bcb395a8a"}, ] [package.dependencies] google-auth = ">=2.14.1,<3.0.dev0" googleapis-common-protos = ">=1.56.2,<2.0.dev0" -grpcio = [ - {version = ">=1.33.2,<2.0dev", optional = true, markers = "extra == \"grpc\""}, - {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, -] -grpcio-status = [ - {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "extra == \"grpc\""}, - {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, -] protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" requests = ">=2.18.0,<3.0.0.dev0" @@ -1048,13 +1086,13 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] [[package]] name = "google-api-python-client" -version = "2.95.0" +version = "2.113.0" description = "Google API Client Library for Python" optional = false python-versions = ">=3.7" files = [ - {file = "google-api-python-client-2.95.0.tar.gz", hash = "sha256:d2731ede12f79e53fbe11fdb913dfe986440b44c0a28431c78a8ec275f4c1541"}, - {file = "google_api_python_client-2.95.0-py2.py3-none-any.whl", hash = "sha256:a8aab2da678f42a01f2f52108f787fef4310f23f9dd917c4e64664c3f0c885ba"}, + {file = "google-api-python-client-2.113.0.tar.gz", hash = "sha256:bcffbc8ffbad631f699cf85aa91993f3dc03060b234ca9e6e2f9135028bd9b52"}, + {file = "google_api_python_client-2.113.0-py2.py3-none-any.whl", hash = "sha256:25659d488df6c8a69615b2a510af0e63b4c47ab2cb87d71c1e13b28715906e27"}, ] [package.dependencies] @@ -1066,21 +1104,19 @@ uritemplate = ">=3.0.1,<5" [[package]] name = "google-auth" -version = "2.22.0" +version = "2.26.1" description = "Google Authentication Library" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "google-auth-2.22.0.tar.gz", hash = "sha256:164cba9af4e6e4e40c3a4f90a1a6c12ee56f14c0b4868d1ca91b32826ab334ce"}, - {file = "google_auth-2.22.0-py2.py3-none-any.whl", hash = "sha256:d61d1b40897407b574da67da1a833bdc10d5a11642566e506565d1b1a46ba873"}, + {file = "google-auth-2.26.1.tar.gz", hash = "sha256:54385acca5c0fbdda510cd8585ba6f3fcb06eeecf8a6ecca39d3ee148b092590"}, + {file = "google_auth-2.26.1-py2.py3-none-any.whl", hash = "sha256:2c8b55e3e564f298122a02ab7b97458ccfcc5617840beb5d0ac757ada92c9780"}, ] [package.dependencies] cachetools = ">=2.0.0,<6.0" pyasn1-modules = ">=0.2.1" rsa = ">=3.1.4,<5" -six = ">=1.9.0" -urllib3 = "<2.0" [package.extras] aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] @@ -1091,64 +1127,58 @@ requests = ["requests (>=2.20.0,<3.0.0.dev0)"] [[package]] name = "google-auth-httplib2" -version = "0.1.0" +version = "0.2.0" description = "Google Authentication Library: httplib2 transport" optional = false python-versions = "*" files = [ - {file = "google-auth-httplib2-0.1.0.tar.gz", hash = "sha256:a07c39fd632becacd3f07718dfd6021bf396978f03ad3ce4321d060015cc30ac"}, - {file = "google_auth_httplib2-0.1.0-py2.py3-none-any.whl", hash = "sha256:31e49c36c6b5643b57e82617cb3e021e3e1d2df9da63af67252c02fa9c1f4a10"}, + {file = "google-auth-httplib2-0.2.0.tar.gz", hash = "sha256:38aa7badf48f974f1eb9861794e9c0cb2a0511a4ec0679b1f886d108f5640e05"}, + {file = "google_auth_httplib2-0.2.0-py2.py3-none-any.whl", hash = "sha256:b65a0a2123300dd71281a7bf6e64d65a0759287df52729bdd1ae2e47dc311a3d"}, ] [package.dependencies] google-auth = "*" -httplib2 = ">=0.15.0" -six = "*" +httplib2 = ">=0.19.0" [[package]] name = "google-cloud-bigquery" -version = "3.11.4" +version = "3.14.1" description = "Google BigQuery API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-bigquery-3.11.4.tar.gz", hash = "sha256:697df117241a2283bcbb93b21e10badc14e51c9a90800d2a7e1a3e1c7d842974"}, - {file = "google_cloud_bigquery-3.11.4-py2.py3-none-any.whl", hash = "sha256:5fa7897743a0ed949ade25a0942fc9e7557d8fce307c6f8a76d1b604cf27f1b1"}, + {file = "google-cloud-bigquery-3.14.1.tar.gz", hash = "sha256:aa15bd86f79ea76824c7d710f5ae532323c4b3ba01ef4abff42d4ee7a2e9b142"}, + {file = "google_cloud_bigquery-3.14.1-py2.py3-none-any.whl", hash = "sha256:a8ded18455da71508db222b7c06197bc12b6dbc6ed5b0b64e7007b76d7016957"}, ] [package.dependencies] -google-api-core = {version = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev", extras = ["grpc"]} +google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" google-cloud-core = ">=1.6.0,<3.0.0dev" google-resumable-media = ">=0.6.0,<3.0dev" -grpcio = [ - {version = ">=1.47.0,<2.0dev", markers = "python_version < \"3.11\""}, - {version = ">=1.49.1,<2.0dev", markers = "python_version >= \"3.11\""}, -] packaging = ">=20.0.0" -proto-plus = ">=1.15.0,<2.0.0dev" -protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" python-dateutil = ">=2.7.2,<3.0dev" requests = ">=2.21.0,<3.0.0dev" [package.extras] -all = ["Shapely (>=1.8.4,<2.0dev)", "db-dtypes (>=0.3.0,<2.0.0dev)", "geopandas (>=0.9.0,<1.0dev)", "google-cloud-bigquery-storage (>=2.6.0,<3.0.0dev)", "grpcio (>=1.47.0,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "ipykernel (>=6.0.0)", "ipython (>=7.23.1,!=8.1.0)", "ipywidgets (>=7.7.0)", "opentelemetry-api (>=1.1.0)", "opentelemetry-instrumentation (>=0.20b0)", "opentelemetry-sdk (>=1.1.0)", "pandas (>=1.1.0)", "pyarrow (>=3.0.0)", "tqdm (>=4.7.4,<5.0.0dev)"] +all = ["Shapely (>=1.8.4,<3.0.0dev)", "db-dtypes (>=0.3.0,<2.0.0dev)", "geopandas (>=0.9.0,<1.0dev)", "google-cloud-bigquery-storage (>=2.6.0,<3.0.0dev)", "grpcio (>=1.47.0,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "importlib-metadata (>=1.0.0)", "ipykernel (>=6.0.0)", "ipython (>=7.23.1,!=8.1.0)", "ipywidgets (>=7.7.0)", "opentelemetry-api (>=1.1.0)", "opentelemetry-instrumentation (>=0.20b0)", "opentelemetry-sdk (>=1.1.0)", "pandas (>=1.1.0)", "proto-plus (>=1.15.0,<2.0.0dev)", "protobuf (>=3.19.5,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev)", "pyarrow (>=3.0.0)", "tqdm (>=4.7.4,<5.0.0dev)"] +bigquery-v2 = ["proto-plus (>=1.15.0,<2.0.0dev)", "protobuf (>=3.19.5,!=3.20.0,!=3.20.1,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5,<5.0.0dev)"] bqstorage = ["google-cloud-bigquery-storage (>=2.6.0,<3.0.0dev)", "grpcio (>=1.47.0,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "pyarrow (>=3.0.0)"] -geopandas = ["Shapely (>=1.8.4,<2.0dev)", "geopandas (>=0.9.0,<1.0dev)"] +geopandas = ["Shapely (>=1.8.4,<3.0.0dev)", "geopandas (>=0.9.0,<1.0dev)"] ipython = ["ipykernel (>=6.0.0)", "ipython (>=7.23.1,!=8.1.0)"] ipywidgets = ["ipykernel (>=6.0.0)", "ipywidgets (>=7.7.0)"] opentelemetry = ["opentelemetry-api (>=1.1.0)", "opentelemetry-instrumentation (>=0.20b0)", "opentelemetry-sdk (>=1.1.0)"] -pandas = ["db-dtypes (>=0.3.0,<2.0.0dev)", "pandas (>=1.1.0)", "pyarrow (>=3.0.0)"] +pandas = ["db-dtypes (>=0.3.0,<2.0.0dev)", "importlib-metadata (>=1.0.0)", "pandas (>=1.1.0)", "pyarrow (>=3.0.0)"] tqdm = ["tqdm (>=4.7.4,<5.0.0dev)"] [[package]] name = "google-cloud-core" -version = "2.3.3" +version = "2.4.1" description = "Google Cloud API client core library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-core-2.3.3.tar.gz", hash = "sha256:37b80273c8d7eee1ae816b3a20ae43585ea50506cb0e60f3cf5be5f87f1373cb"}, - {file = "google_cloud_core-2.3.3-py2.py3-none-any.whl", hash = "sha256:fbd11cad3e98a7e5b0343dc07cb1039a5ffd7a5bb96e1f1e27cee4bda4a90863"}, + {file = "google-cloud-core-2.4.1.tar.gz", hash = "sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073"}, + {file = "google_cloud_core-2.4.1-py2.py3-none-any.whl", hash = "sha256:a9e6a4422b9ac5c29f79a0ede9485473338e2ce78d91f2370c01e730eab22e61"}, ] [package.dependencies] @@ -1156,24 +1186,25 @@ google-api-core = ">=1.31.6,<2.0.dev0 || >2.3.0,<3.0.0dev" google-auth = ">=1.25.0,<3.0dev" [package.extras] -grpc = ["grpcio (>=1.38.0,<2.0dev)"] +grpc = ["grpcio (>=1.38.0,<2.0dev)", "grpcio-status (>=1.38.0,<2.0.dev0)"] [[package]] name = "google-cloud-storage" -version = "2.10.0" +version = "2.14.0" description = "Google Cloud Storage API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-storage-2.10.0.tar.gz", hash = "sha256:934b31ead5f3994e5360f9ff5750982c5b6b11604dc072bc452c25965e076dc7"}, - {file = "google_cloud_storage-2.10.0-py2.py3-none-any.whl", hash = "sha256:9433cf28801671de1c80434238fb1e7e4a1ba3087470e90f70c928ea77c2b9d7"}, + {file = "google-cloud-storage-2.14.0.tar.gz", hash = "sha256:2d23fcf59b55e7b45336729c148bb1c464468c69d5efbaee30f7201dd90eb97e"}, + {file = "google_cloud_storage-2.14.0-py2.py3-none-any.whl", hash = "sha256:8641243bbf2a2042c16a6399551fbb13f062cbc9a2de38d6c0bb5426962e9dbd"}, ] [package.dependencies] google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" -google-auth = ">=1.25.0,<3.0dev" +google-auth = ">=2.23.3,<3.0dev" google-cloud-core = ">=2.3.0,<3.0dev" -google-resumable-media = ">=2.3.2" +google-crc32c = ">=1.0,<2.0dev" +google-resumable-media = ">=2.6.0" requests = ">=2.18.0,<3.0.0dev" [package.extras] @@ -1278,31 +1309,31 @@ protobuf = ">=3.0.0b3" [[package]] name = "google-resumable-media" -version = "2.5.0" +version = "2.7.0" description = "Utilities for Google Media Downloads and Resumable Uploads" optional = false python-versions = ">= 3.7" files = [ - {file = "google-resumable-media-2.5.0.tar.gz", hash = "sha256:218931e8e2b2a73a58eb354a288e03a0fd5fb1c4583261ac6e4c078666468c93"}, - {file = "google_resumable_media-2.5.0-py2.py3-none-any.whl", hash = "sha256:da1bd943e2e114a56d85d6848497ebf9be6a14d3db23e9fc57581e7c3e8170ec"}, + {file = "google-resumable-media-2.7.0.tar.gz", hash = "sha256:5f18f5fa9836f4b083162064a1c2c98c17239bfda9ca50ad970ccf905f3e625b"}, + {file = "google_resumable_media-2.7.0-py2.py3-none-any.whl", hash = "sha256:79543cfe433b63fd81c0844b7803aba1bb8950b47bedf7d980c38fa123937e08"}, ] [package.dependencies] google-crc32c = ">=1.0,<2.0dev" [package.extras] -aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)"] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)", "google-auth (>=1.22.0,<2.0dev)"] requests = ["requests (>=2.18.0,<3.0.0dev)"] [[package]] name = "googleapis-common-protos" -version = "1.60.0" +version = "1.62.0" description = "Common protobufs used in Google APIs" optional = false python-versions = ">=3.7" files = [ - {file = "googleapis-common-protos-1.60.0.tar.gz", hash = "sha256:e73ebb404098db405ba95d1e1ae0aa91c3e15a71da031a2eeb6b2e23e7bc3708"}, - {file = "googleapis_common_protos-1.60.0-py2.py3-none-any.whl", hash = "sha256:69f9bbcc6acde92cab2db95ce30a70bd2b81d20b12eff3f1aabaffcbe8a93918"}, + {file = "googleapis-common-protos-1.62.0.tar.gz", hash = "sha256:83f0ece9f94e5672cced82f592d2a5edf527a96ed1794f0bab36d5735c996277"}, + {file = "googleapis_common_protos-1.62.0-py2.py3-none-any.whl", hash = "sha256:4750113612205514f9f6aa4cb00d523a94f3e8c06c5ad2fee466387dc4875f07"}, ] [package.dependencies] @@ -1313,32 +1344,33 @@ grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] [[package]] name = "gql" -version = "3.4.1" +version = "3.5.0" description = "GraphQL client for Python" optional = false python-versions = "*" files = [ - {file = "gql-3.4.1-py2.py3-none-any.whl", hash = "sha256:315624ca0f4d571ef149d455033ebd35e45c1a13f18a059596aeddcea99135cf"}, - {file = "gql-3.4.1.tar.gz", hash = "sha256:11dc5d8715a827f2c2899593439a4f36449db4f0eafa5b1ea63948f8a2f8c545"}, + {file = "gql-3.5.0-py2.py3-none-any.whl", hash = "sha256:70dda5694a5b194a8441f077aa5fb70cc94e4ec08016117523f013680901ecb7"}, + {file = "gql-3.5.0.tar.gz", hash = "sha256:ccb9c5db543682b28f577069950488218ed65d4ac70bb03b6929aaadaf636de9"}, ] [package.dependencies] +anyio = ">=3.0,<5" backoff = ">=1.11.1,<3.0" graphql-core = ">=3.2,<3.3" requests = {version = ">=2.26,<3", optional = true, markers = "extra == \"requests\""} -requests-toolbelt = {version = ">=0.9.1,<1", optional = true, markers = "extra == \"requests\""} -urllib3 = {version = ">=1.26,<2", optional = true, markers = "extra == \"requests\""} +requests-toolbelt = {version = ">=1.0.0,<2", optional = true, markers = "extra == \"requests\""} yarl = ">=1.6,<2.0" [package.extras] -aiohttp = ["aiohttp (>=3.7.1,<3.9.0)"] -all = ["aiohttp (>=3.7.1,<3.9.0)", "botocore (>=1.21,<2)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] +aiohttp = ["aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)"] +all = ["aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "botocore (>=1.21,<2)", "httpx (>=0.23.1,<1)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "websockets (>=10,<12)"] botocore = ["botocore (>=1.21,<2)"] -dev = ["aiofiles", "aiohttp (>=3.7.1,<3.9.0)", "black (==22.3.0)", "botocore (>=1.21,<2)", "check-manifest (>=0.42,<1)", "flake8 (==3.8.1)", "isort (==4.3.21)", "mock (==4.0.2)", "mypy (==0.910)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "sphinx (>=3.0.0,<4)", "sphinx-argparse (==0.2.5)", "sphinx-rtd-theme (>=0.4,<1)", "types-aiofiles", "types-mock", "types-requests", "urllib3 (>=1.26,<2)", "vcrpy (==4.0.2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] -requests = ["requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)"] -test = ["aiofiles", "aiohttp (>=3.7.1,<3.9.0)", "botocore (>=1.21,<2)", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)", "vcrpy (==4.0.2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] -test-no-transport = ["aiofiles", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "vcrpy (==4.0.2)"] -websockets = ["websockets (>=10,<11)", "websockets (>=9,<10)"] +dev = ["aiofiles", "aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "black (==22.3.0)", "botocore (>=1.21,<2)", "check-manifest (>=0.42,<1)", "flake8 (==3.8.1)", "httpx (>=0.23.1,<1)", "isort (==4.3.21)", "mock (==4.0.2)", "mypy (==0.910)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "sphinx (>=5.3.0,<6)", "sphinx-argparse (==0.2.5)", "sphinx-rtd-theme (>=0.4,<1)", "types-aiofiles", "types-mock", "types-requests", "vcrpy (==4.4.0)", "websockets (>=10,<12)"] +httpx = ["httpx (>=0.23.1,<1)"] +requests = ["requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)"] +test = ["aiofiles", "aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "botocore (>=1.21,<2)", "httpx (>=0.23.1,<1)", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "vcrpy (==4.4.0)", "websockets (>=10,<12)"] +test-no-transport = ["aiofiles", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "vcrpy (==4.4.0)"] +websockets = ["websockets (>=10,<12)"] [[package]] name = "graphene" @@ -1387,75 +1419,73 @@ graphql-core = ">=3.2,<3.3" [[package]] name = "greenlet" -version = "2.0.2" +version = "3.0.3" description = "Lightweight in-process concurrent programming" optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" -files = [ - {file = "greenlet-2.0.2-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:bdfea8c661e80d3c1c99ad7c3ff74e6e87184895bbaca6ee8cc61209f8b9b85d"}, - {file = "greenlet-2.0.2-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:9d14b83fab60d5e8abe587d51c75b252bcc21683f24699ada8fb275d7712f5a9"}, - {file = "greenlet-2.0.2-cp27-cp27m-win32.whl", hash = "sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74"}, - {file = "greenlet-2.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343"}, - {file = "greenlet-2.0.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae"}, - {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df"}, - {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088"}, - {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb"}, - {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d75209eed723105f9596807495d58d10b3470fa6732dd6756595e89925ce2470"}, - {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3a51c9751078733d88e013587b108f1b7a1fb106d402fb390740f002b6f6551a"}, - {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91"}, - {file = "greenlet-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645"}, - {file = "greenlet-2.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c"}, - {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca"}, - {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0"}, - {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2"}, - {file = "greenlet-2.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:eff4eb9b7eb3e4d0cae3d28c283dc16d9bed6b193c2e1ace3ed86ce48ea8df19"}, - {file = "greenlet-2.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5454276c07d27a740c5892f4907c86327b632127dd9abec42ee62e12427ff7e3"}, - {file = "greenlet-2.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:7cafd1208fdbe93b67c7086876f061f660cfddc44f404279c1585bbf3cdc64c5"}, - {file = "greenlet-2.0.2-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:910841381caba4f744a44bf81bfd573c94e10b3045ee00de0cbf436fe50673a6"}, - {file = "greenlet-2.0.2-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:18a7f18b82b52ee85322d7a7874e676f34ab319b9f8cce5de06067384aa8ff43"}, - {file = "greenlet-2.0.2-cp35-cp35m-win32.whl", hash = "sha256:03a8f4f3430c3b3ff8d10a2a86028c660355ab637cee9333d63d66b56f09d52a"}, - {file = "greenlet-2.0.2-cp35-cp35m-win_amd64.whl", hash = "sha256:4b58adb399c4d61d912c4c331984d60eb66565175cdf4a34792cd9600f21b394"}, - {file = "greenlet-2.0.2-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:703f18f3fda276b9a916f0934d2fb6d989bf0b4fb5a64825260eb9bfd52d78f0"}, - {file = "greenlet-2.0.2-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:32e5b64b148966d9cccc2c8d35a671409e45f195864560829f395a54226408d3"}, - {file = "greenlet-2.0.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dd11f291565a81d71dab10b7033395b7a3a5456e637cf997a6f33ebdf06f8db"}, - {file = "greenlet-2.0.2-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e0f72c9ddb8cd28532185f54cc1453f2c16fb417a08b53a855c4e6a418edd099"}, - {file = "greenlet-2.0.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd021c754b162c0fb55ad5d6b9d960db667faad0fa2ff25bb6e1301b0b6e6a75"}, - {file = "greenlet-2.0.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:3c9b12575734155d0c09d6c3e10dbd81665d5c18e1a7c6597df72fd05990c8cf"}, - {file = "greenlet-2.0.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b9ec052b06a0524f0e35bd8790686a1da006bd911dd1ef7d50b77bfbad74e292"}, - {file = "greenlet-2.0.2-cp36-cp36m-win32.whl", hash = "sha256:dbfcfc0218093a19c252ca8eb9aee3d29cfdcb586df21049b9d777fd32c14fd9"}, - {file = "greenlet-2.0.2-cp36-cp36m-win_amd64.whl", hash = "sha256:9f35ec95538f50292f6d8f2c9c9f8a3c6540bbfec21c9e5b4b751e0a7c20864f"}, - {file = "greenlet-2.0.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:d5508f0b173e6aa47273bdc0a0b5ba055b59662ba7c7ee5119528f466585526b"}, - {file = "greenlet-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:f82d4d717d8ef19188687aa32b8363e96062911e63ba22a0cff7802a8e58e5f1"}, - {file = "greenlet-2.0.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9c59a2120b55788e800d82dfa99b9e156ff8f2227f07c5e3012a45a399620b7"}, - {file = "greenlet-2.0.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2780572ec463d44c1d3ae850239508dbeb9fed38e294c68d19a24d925d9223ca"}, - {file = "greenlet-2.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:937e9020b514ceedb9c830c55d5c9872abc90f4b5862f89c0887033ae33c6f73"}, - {file = "greenlet-2.0.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:36abbf031e1c0f79dd5d596bfaf8e921c41df2bdf54ee1eed921ce1f52999a86"}, - {file = "greenlet-2.0.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:18e98fb3de7dba1c0a852731c3070cf022d14f0d68b4c87a19cc1016f3bb8b33"}, - {file = "greenlet-2.0.2-cp37-cp37m-win32.whl", hash = "sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7"}, - {file = "greenlet-2.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3"}, - {file = "greenlet-2.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30"}, - {file = "greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b"}, - {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"}, - {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b"}, - {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acd2162a36d3de67ee896c43effcd5ee3de247eb00354db411feb025aa319857"}, - {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:0bf60faf0bc2468089bdc5edd10555bab6e85152191df713e2ab1fcc86382b5a"}, - {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a"}, - {file = "greenlet-2.0.2-cp38-cp38-win32.whl", hash = "sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249"}, - {file = "greenlet-2.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40"}, - {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8"}, - {file = "greenlet-2.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6"}, - {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df"}, - {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be4ed120b52ae4d974aa40215fcdfde9194d63541c7ded40ee12eb4dda57b76b"}, - {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94c817e84245513926588caf1152e3b559ff794d505555211ca041f032abbb6b"}, - {file = "greenlet-2.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1a819eef4b0e0b96bb0d98d797bef17dc1b4a10e8d7446be32d1da33e095dbb8"}, - {file = "greenlet-2.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7efde645ca1cc441d6dc4b48c0f7101e8d86b54c8530141b09fd31cef5149ec9"}, - {file = "greenlet-2.0.2-cp39-cp39-win32.whl", hash = "sha256:ea9872c80c132f4663822dd2a08d404073a5a9b5ba6155bea72fb2a79d1093b5"}, - {file = "greenlet-2.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:db1a39669102a1d8d12b57de2bb7e2ec9066a6f2b3da35ae511ff93b01b5d564"}, - {file = "greenlet-2.0.2.tar.gz", hash = "sha256:e7c8dc13af7db097bed64a051d2dd49e9f0af495c26995c00a9ee842690d34c0"}, +python-versions = ">=3.7" +files = [ + {file = "greenlet-3.0.3-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:9da2bd29ed9e4f15955dd1595ad7bc9320308a3b766ef7f837e23ad4b4aac31a"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d353cadd6083fdb056bb46ed07e4340b0869c305c8ca54ef9da3421acbdf6881"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dca1e2f3ca00b84a396bc1bce13dd21f680f035314d2379c4160c98153b2059b"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3ed7fb269f15dc662787f4119ec300ad0702fa1b19d2135a37c2c4de6fadfd4a"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd4f49ae60e10adbc94b45c0b5e6a179acc1736cf7a90160b404076ee283cf83"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73a411ef564e0e097dbe7e866bb2dda0f027e072b04da387282b02c308807405"}, + {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7f362975f2d179f9e26928c5b517524e89dd48530a0202570d55ad6ca5d8a56f"}, + {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:649dde7de1a5eceb258f9cb00bdf50e978c9db1b996964cd80703614c86495eb"}, + {file = "greenlet-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:68834da854554926fbedd38c76e60c4a2e3198c6fbed520b106a8986445caaf9"}, + {file = "greenlet-3.0.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:b1b5667cced97081bf57b8fa1d6bfca67814b0afd38208d52538316e9422fc61"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52f59dd9c96ad2fc0d5724107444f76eb20aaccb675bf825df6435acb7703559"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:afaff6cf5200befd5cec055b07d1c0a5a06c040fe5ad148abcd11ba6ab9b114e"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe754d231288e1e64323cfad462fcee8f0288654c10bdf4f603a39ed923bef33"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2797aa5aedac23af156bbb5a6aa2cd3427ada2972c828244eb7d1b9255846379"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7f009caad047246ed379e1c4dbcb8b020f0a390667ea74d2387be2998f58a22"}, + {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c5e1536de2aad7bf62e27baf79225d0d64360d4168cf2e6becb91baf1ed074f3"}, + {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:894393ce10ceac937e56ec00bb71c4c2f8209ad516e96033e4b3b1de270e200d"}, + {file = "greenlet-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:1ea188d4f49089fc6fb283845ab18a2518d279c7cd9da1065d7a84e991748728"}, + {file = "greenlet-3.0.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:70fb482fdf2c707765ab5f0b6655e9cfcf3780d8d87355a063547b41177599be"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4d1ac74f5c0c0524e4a24335350edad7e5f03b9532da7ea4d3c54d527784f2e"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:149e94a2dd82d19838fe4b2259f1b6b9957d5ba1b25640d2380bea9c5df37676"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:15d79dd26056573940fcb8c7413d84118086f2ec1a8acdfa854631084393efcc"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881b7db1ebff4ba09aaaeae6aa491daeb226c8150fc20e836ad00041bcb11230"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fcd2469d6a2cf298f198f0487e0a5b1a47a42ca0fa4dfd1b6862c999f018ebbf"}, + {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1f672519db1796ca0d8753f9e78ec02355e862d0998193038c7073045899f305"}, + {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2516a9957eed41dd8f1ec0c604f1cdc86758b587d964668b5b196a9db5bfcde6"}, + {file = "greenlet-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:bba5387a6975598857d86de9eac14210a49d554a77eb8261cc68b7d082f78ce2"}, + {file = "greenlet-3.0.3-cp37-cp37m-macosx_11_0_universal2.whl", hash = "sha256:5b51e85cb5ceda94e79d019ed36b35386e8c37d22f07d6a751cb659b180d5274"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:daf3cb43b7cf2ba96d614252ce1684c1bccee6b2183a01328c98d36fcd7d5cb0"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99bf650dc5d69546e076f413a87481ee1d2d09aaaaaca058c9251b6d8c14783f"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dd6e660effd852586b6a8478a1d244b8dc90ab5b1321751d2ea15deb49ed414"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3391d1e16e2a5a1507d83e4a8b100f4ee626e8eca43cf2cadb543de69827c4c"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1f145462f1fa6e4a4ae3c0f782e580ce44d57c8f2c7aae1b6fa88c0b2efdb41"}, + {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1a7191e42732df52cb5f39d3527217e7ab73cae2cb3694d241e18f53d84ea9a7"}, + {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0448abc479fab28b00cb472d278828b3ccca164531daab4e970a0458786055d6"}, + {file = "greenlet-3.0.3-cp37-cp37m-win32.whl", hash = "sha256:b542be2440edc2d48547b5923c408cbe0fc94afb9f18741faa6ae970dbcb9b6d"}, + {file = "greenlet-3.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:01bc7ea167cf943b4c802068e178bbf70ae2e8c080467070d01bfa02f337ee67"}, + {file = "greenlet-3.0.3-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:1996cb9306c8595335bb157d133daf5cf9f693ef413e7673cb07e3e5871379ca"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ddc0f794e6ad661e321caa8d2f0a55ce01213c74722587256fb6566049a8b04"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9db1c18f0eaad2f804728c67d6c610778456e3e1cc4ab4bbd5eeb8e6053c6fc"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7170375bcc99f1a2fbd9c306f5be8764eaf3ac6b5cb968862cad4c7057756506"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b66c9c1e7ccabad3a7d037b2bcb740122a7b17a53734b7d72a344ce39882a1b"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:098d86f528c855ead3479afe84b49242e174ed262456c342d70fc7f972bc13c4"}, + {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:81bb9c6d52e8321f09c3d165b2a78c680506d9af285bfccbad9fb7ad5a5da3e5"}, + {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fd096eb7ffef17c456cfa587523c5f92321ae02427ff955bebe9e3c63bc9f0da"}, + {file = "greenlet-3.0.3-cp38-cp38-win32.whl", hash = "sha256:d46677c85c5ba00a9cb6f7a00b2bfa6f812192d2c9f7d9c4f6a55b60216712f3"}, + {file = "greenlet-3.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:419b386f84949bf0e7c73e6032e3457b82a787c1ab4a0e43732898a761cc9dbf"}, + {file = "greenlet-3.0.3-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:da70d4d51c8b306bb7a031d5cff6cc25ad253affe89b70352af5f1cb68e74b53"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:086152f8fbc5955df88382e8a75984e2bb1c892ad2e3c80a2508954e52295257"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d73a9fe764d77f87f8ec26a0c85144d6a951a6c438dfe50487df5595c6373eac"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7dcbe92cc99f08c8dd11f930de4d99ef756c3591a5377d1d9cd7dd5e896da71"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1551a8195c0d4a68fac7a4325efac0d541b48def35feb49d803674ac32582f61"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64d7675ad83578e3fc149b617a444fab8efdafc9385471f868eb5ff83e446b8b"}, + {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b37eef18ea55f2ffd8f00ff8fe7c8d3818abd3e25fb73fae2ca3b672e333a7a6"}, + {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:77457465d89b8263bca14759d7c1684df840b6811b2499838cc5b040a8b5b113"}, + {file = "greenlet-3.0.3-cp39-cp39-win32.whl", hash = "sha256:57e8974f23e47dac22b83436bdcf23080ade568ce77df33159e019d161ce1d1e"}, + {file = "greenlet-3.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:c5ee858cfe08f34712f548c3c363e807e7186f03ad7a5039ebadb29e8c6be067"}, + {file = "greenlet-3.0.3.tar.gz", hash = "sha256:43374442353259554ce33599da8b692d5aa96f8976d567d4badf263371fbe491"}, ] [package.extras] -docs = ["Sphinx", "docutils (<0.18)"] +docs = ["Sphinx", "furo"] test = ["objgraph", "psutil"] [[package]] @@ -1490,90 +1520,83 @@ oauth2client = ">=1.4.11" [[package]] name = "grpcio" -version = "1.56.2" +version = "1.60.0" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.7" files = [ - {file = "grpcio-1.56.2-cp310-cp310-linux_armv7l.whl", hash = "sha256:bf0b9959e673505ee5869950642428046edb91f99942607c2ecf635f8a4b31c9"}, - {file = "grpcio-1.56.2-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:5144feb20fe76e73e60c7d73ec3bf54f320247d1ebe737d10672480371878b48"}, - {file = "grpcio-1.56.2-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:a72797549935c9e0b9bc1def1768c8b5a709538fa6ab0678e671aec47ebfd55e"}, - {file = "grpcio-1.56.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c3f3237a57e42f79f1e560726576aedb3a7ef931f4e3accb84ebf6acc485d316"}, - {file = "grpcio-1.56.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:900bc0096c2ca2d53f2e5cebf98293a7c32f532c4aeb926345e9747452233950"}, - {file = "grpcio-1.56.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:97e0efaebbfd222bcaac2f1735c010c1d3b167112d9d237daebbeedaaccf3d1d"}, - {file = "grpcio-1.56.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c0c85c5cbe8b30a32fa6d802588d55ffabf720e985abe9590c7c886919d875d4"}, - {file = "grpcio-1.56.2-cp310-cp310-win32.whl", hash = "sha256:06e84ad9ae7668a109e970c7411e7992751a116494cba7c4fb877656527f9a57"}, - {file = "grpcio-1.56.2-cp310-cp310-win_amd64.whl", hash = "sha256:10954662f77dc36c9a1fb5cc4a537f746580d6b5734803be1e587252682cda8d"}, - {file = "grpcio-1.56.2-cp311-cp311-linux_armv7l.whl", hash = "sha256:c435f5ce1705de48e08fcbcfaf8aee660d199c90536e3e06f2016af7d6a938dd"}, - {file = "grpcio-1.56.2-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:6108e5933eb8c22cd3646e72d5b54772c29f57482fd4c41a0640aab99eb5071d"}, - {file = "grpcio-1.56.2-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:8391cea5ce72f4a12368afd17799474015d5d3dc00c936a907eb7c7eaaea98a5"}, - {file = "grpcio-1.56.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:750de923b456ca8c0f1354d6befca45d1f3b3a789e76efc16741bd4132752d95"}, - {file = "grpcio-1.56.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fda2783c12f553cdca11c08e5af6eecbd717280dc8fbe28a110897af1c15a88c"}, - {file = "grpcio-1.56.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9e04d4e4cfafa7c5264e535b5d28e786f0571bea609c3f0aaab13e891e933e9c"}, - {file = "grpcio-1.56.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:89a49cc5ad08a38b6141af17e00d1dd482dc927c7605bc77af457b5a0fca807c"}, - {file = "grpcio-1.56.2-cp311-cp311-win32.whl", hash = "sha256:6a007a541dff984264981fbafeb052bfe361db63578948d857907df9488d8774"}, - {file = "grpcio-1.56.2-cp311-cp311-win_amd64.whl", hash = "sha256:af4063ef2b11b96d949dccbc5a987272f38d55c23c4c01841ea65a517906397f"}, - {file = "grpcio-1.56.2-cp37-cp37m-linux_armv7l.whl", hash = "sha256:a6ff459dac39541e6a2763a4439c4ca6bc9ecb4acc05a99b79246751f9894756"}, - {file = "grpcio-1.56.2-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:f20fd21f7538f8107451156dd1fe203300b79a9ddceba1ee0ac8132521a008ed"}, - {file = "grpcio-1.56.2-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:d1fbad1f9077372b6587ec589c1fc120b417b6c8ad72d3e3cc86bbbd0a3cee93"}, - {file = "grpcio-1.56.2-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ee26e9dfb3996aff7c870f09dc7ad44a5f6732b8bdb5a5f9905737ac6fd4ef1"}, - {file = "grpcio-1.56.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a4c60abd950d6de3e4f1ddbc318075654d275c29c846ab6a043d6ed2c52e4c8c"}, - {file = "grpcio-1.56.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1c31e52a04e62c8577a7bf772b3e7bed4df9c9e0dd90f92b6ffa07c16cab63c9"}, - {file = "grpcio-1.56.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:345356b307cce5d14355e8e055b4ca5f99bc857c33a3dc1ddbc544fca9cd0475"}, - {file = "grpcio-1.56.2-cp37-cp37m-win_amd64.whl", hash = "sha256:42e63904ee37ae46aa23de50dac8b145b3596f43598fa33fe1098ab2cbda6ff5"}, - {file = "grpcio-1.56.2-cp38-cp38-linux_armv7l.whl", hash = "sha256:7c5ede2e2558f088c49a1ddda19080e4c23fb5d171de80a726b61b567e3766ed"}, - {file = "grpcio-1.56.2-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:33971197c47965cc1d97d78d842163c283e998223b151bab0499b951fd2c0b12"}, - {file = "grpcio-1.56.2-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:d39f5d4af48c138cb146763eda14eb7d8b3ccbbec9fe86fb724cd16e0e914c64"}, - {file = "grpcio-1.56.2-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ded637176addc1d3eef35331c39acc598bac550d213f0a1bedabfceaa2244c87"}, - {file = "grpcio-1.56.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c90da4b124647547a68cf2f197174ada30c7bb9523cb976665dfd26a9963d328"}, - {file = "grpcio-1.56.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3ccb621749a81dc7755243665a70ce45536ec413ef5818e013fe8dfbf5aa497b"}, - {file = "grpcio-1.56.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4eb37dd8dd1aa40d601212afa27ca5be255ba792e2e0b24d67b8af5e012cdb7d"}, - {file = "grpcio-1.56.2-cp38-cp38-win32.whl", hash = "sha256:ddb4a6061933bd9332b74eac0da25f17f32afa7145a33a0f9711ad74f924b1b8"}, - {file = "grpcio-1.56.2-cp38-cp38-win_amd64.whl", hash = "sha256:8940d6de7068af018dfa9a959a3510e9b7b543f4c405e88463a1cbaa3b2b379a"}, - {file = "grpcio-1.56.2-cp39-cp39-linux_armv7l.whl", hash = "sha256:51173e8fa6d9a2d85c14426bdee5f5c4a0654fd5fddcc21fe9d09ab0f6eb8b35"}, - {file = "grpcio-1.56.2-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:373b48f210f43327a41e397391715cd11cfce9ded2fe76a5068f9bacf91cc226"}, - {file = "grpcio-1.56.2-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:42a3bbb2bc07aef72a7d97e71aabecaf3e4eb616d39e5211e2cfe3689de860ca"}, - {file = "grpcio-1.56.2-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5344be476ac37eb9c9ad09c22f4ea193c1316bf074f1daf85bddb1b31fda5116"}, - {file = "grpcio-1.56.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3fa3ab0fb200a2c66493828ed06ccd1a94b12eddbfb985e7fd3e5723ff156c6"}, - {file = "grpcio-1.56.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b975b85d1d5efc36cf8b237c5f3849b64d1ba33d6282f5e991f28751317504a1"}, - {file = "grpcio-1.56.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:cbdf2c498e077282cd427cfd88bdce4668019791deef0be8155385ab2ba7837f"}, - {file = "grpcio-1.56.2-cp39-cp39-win32.whl", hash = "sha256:139f66656a762572ae718fa0d1f2dce47c05e9fbf7a16acd704c354405b97df9"}, - {file = "grpcio-1.56.2-cp39-cp39-win_amd64.whl", hash = "sha256:830215173ad45d670140ff99aac3b461f9be9a6b11bee1a17265aaaa746a641a"}, - {file = "grpcio-1.56.2.tar.gz", hash = "sha256:0ff789ae7d8ddd76d2ac02e7d13bfef6fc4928ac01e1dcaa182be51b6bcc0aaa"}, + {file = "grpcio-1.60.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:d020cfa595d1f8f5c6b343530cd3ca16ae5aefdd1e832b777f9f0eb105f5b139"}, + {file = "grpcio-1.60.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:b98f43fcdb16172dec5f4b49f2fece4b16a99fd284d81c6bbac1b3b69fcbe0ff"}, + {file = "grpcio-1.60.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:20e7a4f7ded59097c84059d28230907cd97130fa74f4a8bfd1d8e5ba18c81491"}, + {file = "grpcio-1.60.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:452ca5b4afed30e7274445dd9b441a35ece656ec1600b77fff8c216fdf07df43"}, + {file = "grpcio-1.60.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43e636dc2ce9ece583b3e2ca41df5c983f4302eabc6d5f9cd04f0562ee8ec1ae"}, + {file = "grpcio-1.60.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6e306b97966369b889985a562ede9d99180def39ad42c8014628dd3cc343f508"}, + {file = "grpcio-1.60.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f897c3b127532e6befdcf961c415c97f320d45614daf84deba0a54e64ea2457b"}, + {file = "grpcio-1.60.0-cp310-cp310-win32.whl", hash = "sha256:b87efe4a380887425bb15f220079aa8336276398dc33fce38c64d278164f963d"}, + {file = "grpcio-1.60.0-cp310-cp310-win_amd64.whl", hash = "sha256:a9c7b71211f066908e518a2ef7a5e211670761651039f0d6a80d8d40054047df"}, + {file = "grpcio-1.60.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:fb464479934778d7cc5baf463d959d361954d6533ad34c3a4f1d267e86ee25fd"}, + {file = "grpcio-1.60.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:4b44d7e39964e808b071714666a812049765b26b3ea48c4434a3b317bac82f14"}, + {file = "grpcio-1.60.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:90bdd76b3f04bdb21de5398b8a7c629676c81dfac290f5f19883857e9371d28c"}, + {file = "grpcio-1.60.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:91229d7203f1ef0ab420c9b53fe2ca5c1fbeb34f69b3bc1b5089466237a4a134"}, + {file = "grpcio-1.60.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b36a2c6d4920ba88fa98075fdd58ff94ebeb8acc1215ae07d01a418af4c0253"}, + {file = "grpcio-1.60.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:297eef542156d6b15174a1231c2493ea9ea54af8d016b8ca7d5d9cc65cfcc444"}, + {file = "grpcio-1.60.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:87c9224acba0ad8bacddf427a1c2772e17ce50b3042a789547af27099c5f751d"}, + {file = "grpcio-1.60.0-cp311-cp311-win32.whl", hash = "sha256:95ae3e8e2c1b9bf671817f86f155c5da7d49a2289c5cf27a319458c3e025c320"}, + {file = "grpcio-1.60.0-cp311-cp311-win_amd64.whl", hash = "sha256:467a7d31554892eed2aa6c2d47ded1079fc40ea0b9601d9f79204afa8902274b"}, + {file = "grpcio-1.60.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:a7152fa6e597c20cb97923407cf0934e14224af42c2b8d915f48bc3ad2d9ac18"}, + {file = "grpcio-1.60.0-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:7db16dd4ea1b05ada504f08d0dca1cd9b926bed3770f50e715d087c6f00ad748"}, + {file = "grpcio-1.60.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:b0571a5aef36ba9177e262dc88a9240c866d903a62799e44fd4aae3f9a2ec17e"}, + {file = "grpcio-1.60.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6fd9584bf1bccdfff1512719316efa77be235469e1e3295dce64538c4773840b"}, + {file = "grpcio-1.60.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d6a478581b1a1a8fdf3318ecb5f4d0cda41cacdffe2b527c23707c9c1b8fdb55"}, + {file = "grpcio-1.60.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:77c8a317f0fd5a0a2be8ed5cbe5341537d5c00bb79b3bb27ba7c5378ba77dbca"}, + {file = "grpcio-1.60.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1c30bb23a41df95109db130a6cc1b974844300ae2e5d68dd4947aacba5985aa5"}, + {file = "grpcio-1.60.0-cp312-cp312-win32.whl", hash = "sha256:2aef56e85901c2397bd557c5ba514f84de1f0ae5dd132f5d5fed042858115951"}, + {file = "grpcio-1.60.0-cp312-cp312-win_amd64.whl", hash = "sha256:e381fe0c2aa6c03b056ad8f52f8efca7be29fb4d9ae2f8873520843b6039612a"}, + {file = "grpcio-1.60.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:92f88ca1b956eb8427a11bb8b4a0c0b2b03377235fc5102cb05e533b8693a415"}, + {file = "grpcio-1.60.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:e278eafb406f7e1b1b637c2cf51d3ad45883bb5bd1ca56bc05e4fc135dfdaa65"}, + {file = "grpcio-1.60.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:a48edde788b99214613e440fce495bbe2b1e142a7f214cce9e0832146c41e324"}, + {file = "grpcio-1.60.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:de2ad69c9a094bf37c1102b5744c9aec6cf74d2b635558b779085d0263166454"}, + {file = "grpcio-1.60.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:073f959c6f570797272f4ee9464a9997eaf1e98c27cb680225b82b53390d61e6"}, + {file = "grpcio-1.60.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c826f93050c73e7769806f92e601e0efdb83ec8d7c76ddf45d514fee54e8e619"}, + {file = "grpcio-1.60.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:9e30be89a75ee66aec7f9e60086fadb37ff8c0ba49a022887c28c134341f7179"}, + {file = "grpcio-1.60.0-cp37-cp37m-win_amd64.whl", hash = "sha256:b0fb2d4801546598ac5cd18e3ec79c1a9af8b8f2a86283c55a5337c5aeca4b1b"}, + {file = "grpcio-1.60.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:9073513ec380434eb8d21970e1ab3161041de121f4018bbed3146839451a6d8e"}, + {file = "grpcio-1.60.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:74d7d9fa97809c5b892449b28a65ec2bfa458a4735ddad46074f9f7d9550ad13"}, + {file = "grpcio-1.60.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:1434ca77d6fed4ea312901122dc8da6c4389738bf5788f43efb19a838ac03ead"}, + {file = "grpcio-1.60.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e61e76020e0c332a98290323ecfec721c9544f5b739fab925b6e8cbe1944cf19"}, + {file = "grpcio-1.60.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675997222f2e2f22928fbba640824aebd43791116034f62006e19730715166c0"}, + {file = "grpcio-1.60.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5208a57eae445ae84a219dfd8b56e04313445d146873117b5fa75f3245bc1390"}, + {file = "grpcio-1.60.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:428d699c8553c27e98f4d29fdc0f0edc50e9a8a7590bfd294d2edb0da7be3629"}, + {file = "grpcio-1.60.0-cp38-cp38-win32.whl", hash = "sha256:83f2292ae292ed5a47cdcb9821039ca8e88902923198f2193f13959360c01860"}, + {file = "grpcio-1.60.0-cp38-cp38-win_amd64.whl", hash = "sha256:705a68a973c4c76db5d369ed573fec3367d7d196673fa86614b33d8c8e9ebb08"}, + {file = "grpcio-1.60.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:c193109ca4070cdcaa6eff00fdb5a56233dc7610216d58fb81638f89f02e4968"}, + {file = "grpcio-1.60.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:676e4a44e740deaba0f4d95ba1d8c5c89a2fcc43d02c39f69450b1fa19d39590"}, + {file = "grpcio-1.60.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:5ff21e000ff2f658430bde5288cb1ac440ff15c0d7d18b5fb222f941b46cb0d2"}, + {file = "grpcio-1.60.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c86343cf9ff7b2514dd229bdd88ebba760bd8973dac192ae687ff75e39ebfab"}, + {file = "grpcio-1.60.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fd3b3968ffe7643144580f260f04d39d869fcc2cddb745deef078b09fd2b328"}, + {file = "grpcio-1.60.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:30943b9530fe3620e3b195c03130396cd0ee3a0d10a66c1bee715d1819001eaf"}, + {file = "grpcio-1.60.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b10241250cb77657ab315270b064a6c7f1add58af94befa20687e7c8d8603ae6"}, + {file = "grpcio-1.60.0-cp39-cp39-win32.whl", hash = "sha256:79a050889eb8d57a93ed21d9585bb63fca881666fc709f5d9f7f9372f5e7fd03"}, + {file = "grpcio-1.60.0-cp39-cp39-win_amd64.whl", hash = "sha256:8a97a681e82bc11a42d4372fe57898d270a2707f36c45c6676e49ce0d5c41353"}, + {file = "grpcio-1.60.0.tar.gz", hash = "sha256:2199165a1affb666aa24adf0c97436686d0a61bc5fc113c037701fb7c7fceb96"}, ] [package.extras] -protobuf = ["grpcio-tools (>=1.56.2)"] +protobuf = ["grpcio-tools (>=1.60.0)"] [[package]] name = "grpcio-health-checking" -version = "1.56.2" +version = "1.60.0" description = "Standard Health Checking Service for gRPC" optional = false python-versions = ">=3.6" files = [ - {file = "grpcio-health-checking-1.56.2.tar.gz", hash = "sha256:5cda1d8a1368be2cda04f9284a8b73cee09ff3e277eec8ddd9abcf2fef76b372"}, - {file = "grpcio_health_checking-1.56.2-py3-none-any.whl", hash = "sha256:d0aedbcdbb365c08a5bd860384098502e35045e31fdd9d80e440bb58487e83d7"}, -] - -[package.dependencies] -grpcio = ">=1.56.2" -protobuf = ">=4.21.6" - -[[package]] -name = "grpcio-status" -version = "1.56.2" -description = "Status proto mapping for gRPC" -optional = false -python-versions = ">=3.6" -files = [ - {file = "grpcio-status-1.56.2.tar.gz", hash = "sha256:a046b2c0118df4a5687f4585cca9d3c3bae5c498c4dff055dcb43fb06a1180c8"}, - {file = "grpcio_status-1.56.2-py3-none-any.whl", hash = "sha256:63f3842867735f59f5d70e723abffd2e8501a6bcd915612a1119e52f10614782"}, + {file = "grpcio-health-checking-1.60.0.tar.gz", hash = "sha256:478b5300778120fed9f6d134d72b157a59f9c06689789218cbff47fafca2f119"}, + {file = "grpcio_health_checking-1.60.0-py3-none-any.whl", hash = "sha256:13caf28bc93795bd6bdb580b21832ebdd1aa3f5b648ea47ed17362d85bed96d3"}, ] [package.dependencies] -googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.56.2" +grpcio = ">=1.60.0" protobuf = ">=4.21.6" [[package]] @@ -1587,27 +1610,6 @@ files = [ {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, ] -[[package]] -name = "html5lib" -version = "1.1" -description = "HTML parser based on the WHATWG HTML specification" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -files = [ - {file = "html5lib-1.1-py2.py3-none-any.whl", hash = "sha256:0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d"}, - {file = "html5lib-1.1.tar.gz", hash = "sha256:b2e5b40261e20f354d198eae92afc10d750afb487ed5e50f9c4eaf07c184146f"}, -] - -[package.dependencies] -six = ">=1.9" -webencodings = "*" - -[package.extras] -all = ["chardet (>=2.2)", "genshi", "lxml"] -chardet = ["chardet (>=2.2)"] -genshi = ["genshi"] -lxml = ["lxml"] - [[package]] name = "httplib2" version = "0.22.0" @@ -1624,46 +1626,47 @@ pyparsing = {version = ">=2.4.2,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.0.2 || >3.0 [[package]] name = "httptools" -version = "0.6.0" +version = "0.6.1" description = "A collection of framework independent HTTP protocol utils." optional = false -python-versions = ">=3.5.0" -files = [ - {file = "httptools-0.6.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:818325afee467d483bfab1647a72054246d29f9053fd17cc4b86cda09cc60339"}, - {file = "httptools-0.6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72205730bf1be875003692ca54a4a7c35fac77b4746008966061d9d41a61b0f5"}, - {file = "httptools-0.6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33eb1d4e609c835966e969a31b1dedf5ba16b38cab356c2ce4f3e33ffa94cad3"}, - {file = "httptools-0.6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bdc6675ec6cb79d27e0575750ac6e2b47032742e24eed011b8db73f2da9ed40"}, - {file = "httptools-0.6.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:463c3bc5ef64b9cf091be9ac0e0556199503f6e80456b790a917774a616aff6e"}, - {file = "httptools-0.6.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:82f228b88b0e8c6099a9c4757ce9fdbb8b45548074f8d0b1f0fc071e35655d1c"}, - {file = "httptools-0.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:0781fedc610293a2716bc7fa142d4c85e6776bc59d617a807ff91246a95dea35"}, - {file = "httptools-0.6.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:721e503245d591527cddd0f6fd771d156c509e831caa7a57929b55ac91ee2b51"}, - {file = "httptools-0.6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:274bf20eeb41b0956e34f6a81f84d26ed57c84dd9253f13dcb7174b27ccd8aaf"}, - {file = "httptools-0.6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:259920bbae18740a40236807915def554132ad70af5067e562f4660b62c59b90"}, - {file = "httptools-0.6.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:03bfd2ae8a2d532952ac54445a2fb2504c804135ed28b53fefaf03d3a93eb1fd"}, - {file = "httptools-0.6.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f959e4770b3fc8ee4dbc3578fd910fab9003e093f20ac8c621452c4d62e517cb"}, - {file = "httptools-0.6.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6e22896b42b95b3237eccc42278cd72c0df6f23247d886b7ded3163452481e38"}, - {file = "httptools-0.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:38f3cafedd6aa20ae05f81f2e616ea6f92116c8a0f8dcb79dc798df3356836e2"}, - {file = "httptools-0.6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:47043a6e0ea753f006a9d0dd076a8f8c99bc0ecae86a0888448eb3076c43d717"}, - {file = "httptools-0.6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35a541579bed0270d1ac10245a3e71e5beeb1903b5fbbc8d8b4d4e728d48ff1d"}, - {file = "httptools-0.6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65d802e7b2538a9756df5acc062300c160907b02e15ed15ba035b02bce43e89c"}, - {file = "httptools-0.6.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:26326e0a8fe56829f3af483200d914a7cd16d8d398d14e36888b56de30bec81a"}, - {file = "httptools-0.6.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e41ccac9e77cd045f3e4ee0fc62cbf3d54d7d4b375431eb855561f26ee7a9ec4"}, - {file = "httptools-0.6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:4e748fc0d5c4a629988ef50ac1aef99dfb5e8996583a73a717fc2cac4ab89932"}, - {file = "httptools-0.6.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:cf8169e839a0d740f3d3c9c4fa630ac1a5aaf81641a34575ca6773ed7ce041a1"}, - {file = "httptools-0.6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5dcc14c090ab57b35908d4a4585ec5c0715439df07be2913405991dbb37e049d"}, - {file = "httptools-0.6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d0b0571806a5168013b8c3d180d9f9d6997365a4212cb18ea20df18b938aa0b"}, - {file = "httptools-0.6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fb4a608c631f7dcbdf986f40af7a030521a10ba6bc3d36b28c1dc9e9035a3c0"}, - {file = "httptools-0.6.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:93f89975465133619aea8b1952bc6fa0e6bad22a447c6d982fc338fbb4c89649"}, - {file = "httptools-0.6.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:73e9d66a5a28b2d5d9fbd9e197a31edd02be310186db423b28e6052472dc8201"}, - {file = "httptools-0.6.0-cp38-cp38-win_amd64.whl", hash = "sha256:22c01fcd53648162730a71c42842f73b50f989daae36534c818b3f5050b54589"}, - {file = "httptools-0.6.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:3f96d2a351b5625a9fd9133c95744e8ca06f7a4f8f0b8231e4bbaae2c485046a"}, - {file = "httptools-0.6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:72ec7c70bd9f95ef1083d14a755f321d181f046ca685b6358676737a5fecd26a"}, - {file = "httptools-0.6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b703d15dbe082cc23266bf5d9448e764c7cb3fcfe7cb358d79d3fd8248673ef9"}, - {file = "httptools-0.6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82c723ed5982f8ead00f8e7605c53e55ffe47c47465d878305ebe0082b6a1755"}, - {file = "httptools-0.6.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b0a816bb425c116a160fbc6f34cece097fd22ece15059d68932af686520966bd"}, - {file = "httptools-0.6.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:dea66d94e5a3f68c5e9d86e0894653b87d952e624845e0b0e3ad1c733c6cc75d"}, - {file = "httptools-0.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:23b09537086a5a611fad5696fc8963d67c7e7f98cb329d38ee114d588b0b74cd"}, - {file = "httptools-0.6.0.tar.gz", hash = "sha256:9fc6e409ad38cbd68b177cd5158fc4042c796b82ca88d99ec78f07bed6c6b796"}, +python-versions = ">=3.8.0" +files = [ + {file = "httptools-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d2f6c3c4cb1948d912538217838f6e9960bc4a521d7f9b323b3da579cd14532f"}, + {file = "httptools-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:00d5d4b68a717765b1fabfd9ca755bd12bf44105eeb806c03d1962acd9b8e563"}, + {file = "httptools-0.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:639dc4f381a870c9ec860ce5c45921db50205a37cc3334e756269736ff0aac58"}, + {file = "httptools-0.6.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e57997ac7fb7ee43140cc03664de5f268813a481dff6245e0075925adc6aa185"}, + {file = "httptools-0.6.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:0ac5a0ae3d9f4fe004318d64b8a854edd85ab76cffbf7ef5e32920faef62f142"}, + {file = "httptools-0.6.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:3f30d3ce413088a98b9db71c60a6ada2001a08945cb42dd65a9a9fe228627658"}, + {file = "httptools-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:1ed99a373e327f0107cb513b61820102ee4f3675656a37a50083eda05dc9541b"}, + {file = "httptools-0.6.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7a7ea483c1a4485c71cb5f38be9db078f8b0e8b4c4dc0210f531cdd2ddac1ef1"}, + {file = "httptools-0.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:85ed077c995e942b6f1b07583e4eb0a8d324d418954fc6af913d36db7c05a5a0"}, + {file = "httptools-0.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b0bb634338334385351a1600a73e558ce619af390c2b38386206ac6a27fecfc"}, + {file = "httptools-0.6.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d9ceb2c957320def533671fc9c715a80c47025139c8d1f3797477decbc6edd2"}, + {file = "httptools-0.6.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:4f0f8271c0a4db459f9dc807acd0eadd4839934a4b9b892f6f160e94da309837"}, + {file = "httptools-0.6.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6a4f5ccead6d18ec072ac0b84420e95d27c1cdf5c9f1bc8fbd8daf86bd94f43d"}, + {file = "httptools-0.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:5cceac09f164bcba55c0500a18fe3c47df29b62353198e4f37bbcc5d591172c3"}, + {file = "httptools-0.6.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:75c8022dca7935cba14741a42744eee13ba05db00b27a4b940f0d646bd4d56d0"}, + {file = "httptools-0.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:48ed8129cd9a0d62cf4d1575fcf90fb37e3ff7d5654d3a5814eb3d55f36478c2"}, + {file = "httptools-0.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f58e335a1402fb5a650e271e8c2d03cfa7cea46ae124649346d17bd30d59c90"}, + {file = "httptools-0.6.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93ad80d7176aa5788902f207a4e79885f0576134695dfb0fefc15b7a4648d503"}, + {file = "httptools-0.6.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9bb68d3a085c2174c2477eb3ffe84ae9fb4fde8792edb7bcd09a1d8467e30a84"}, + {file = "httptools-0.6.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b512aa728bc02354e5ac086ce76c3ce635b62f5fbc32ab7082b5e582d27867bb"}, + {file = "httptools-0.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:97662ce7fb196c785344d00d638fc9ad69e18ee4bfb4000b35a52efe5adcc949"}, + {file = "httptools-0.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:8e216a038d2d52ea13fdd9b9c9c7459fb80d78302b257828285eca1c773b99b3"}, + {file = "httptools-0.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3e802e0b2378ade99cd666b5bffb8b2a7cc8f3d28988685dc300469ea8dd86cb"}, + {file = "httptools-0.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4bd3e488b447046e386a30f07af05f9b38d3d368d1f7b4d8f7e10af85393db97"}, + {file = "httptools-0.6.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe467eb086d80217b7584e61313ebadc8d187a4d95bb62031b7bab4b205c3ba3"}, + {file = "httptools-0.6.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3c3b214ce057c54675b00108ac42bacf2ab8f85c58e3f324a4e963bbc46424f4"}, + {file = "httptools-0.6.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8ae5b97f690badd2ca27cbf668494ee1b6d34cf1c464271ef7bfa9ca6b83ffaf"}, + {file = "httptools-0.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:405784577ba6540fa7d6ff49e37daf104e04f4b4ff2d1ac0469eaa6a20fde084"}, + {file = "httptools-0.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:95fb92dd3649f9cb139e9c56604cc2d7c7bf0fc2e7c8d7fbd58f96e35eddd2a3"}, + {file = "httptools-0.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dcbab042cc3ef272adc11220517278519adf8f53fd3056d0e68f0a6f891ba94e"}, + {file = "httptools-0.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cf2372e98406efb42e93bfe10f2948e467edfd792b015f1b4ecd897903d3e8d"}, + {file = "httptools-0.6.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:678fcbae74477a17d103b7cae78b74800d795d702083867ce160fc202104d0da"}, + {file = "httptools-0.6.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e0b281cf5a125c35f7f6722b65d8542d2e57331be573e9e88bc8b0115c4a7a81"}, + {file = "httptools-0.6.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:95658c342529bba4e1d3d2b1a874db16c7cca435e8827422154c9da76ac4e13a"}, + {file = "httptools-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:7ebaec1bf683e4bf5e9fbb49b8cc36da482033596a415b3e4ebab5a4c0d7ec5e"}, + {file = "httptools-0.6.1.tar.gz", hash = "sha256:c6e26c30455600b95d94b1b836085138e82f177351454ee841c148f93a9bad5a"}, ] [package.extras] @@ -1685,13 +1688,13 @@ pyreadline3 = {version = "*", markers = "sys_platform == \"win32\" and python_ve [[package]] name = "humanize" -version = "4.7.0" +version = "4.9.0" description = "Python humanize utilities" optional = false python-versions = ">=3.8" files = [ - {file = "humanize-4.7.0-py3-none-any.whl", hash = "sha256:df7c429c2d27372b249d3f26eb53b07b166b661326e0325793e0a988082e3889"}, - {file = "humanize-4.7.0.tar.gz", hash = "sha256:7ca0e43e870981fa684acb5b062deb307218193bca1a01f2b2676479df849b3a"}, + {file = "humanize-4.9.0-py3-none-any.whl", hash = "sha256:ce284a76d5b1377fd8836733b983bfb0b76f1aa1c090de2566fcf008d7f6ab16"}, + {file = "humanize-4.9.0.tar.gz", hash = "sha256:582a265c931c683a7e9b8ed9559089dea7edcf6cc95be39a3cbc2c5d5ac2bcfa"}, ] [package.extras] @@ -1699,31 +1702,31 @@ tests = ["freezegun", "pytest", "pytest-cov"] [[package]] name = "idna" -version = "3.4" +version = "3.6" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.5" files = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, ] [[package]] name = "importlib-metadata" -version = "6.8.0" +version = "7.0.1" description = "Read metadata from Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "importlib_metadata-6.8.0-py3-none-any.whl", hash = "sha256:3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb"}, - {file = "importlib_metadata-6.8.0.tar.gz", hash = "sha256:dbace7892d8c0c4ac1ad096662232f831d4e64f4c4545bd53016a3e9d4654743"}, + {file = "importlib_metadata-7.0.1-py3-none-any.whl", hash = "sha256:4805911c3a4ec7c3966410053e9ec6a1fecd629117df5adee56dfc9432a1081e"}, + {file = "importlib_metadata-7.0.1.tar.gz", hash = "sha256:f238736bb06590ae52ac1fab06a3a9ef1d8dce2b7a35b5ab329371d6c8f5d2cc"}, ] [package.dependencies] zipp = ">=0.5" [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] perf = ["ipython"] testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)", "pytest-ruff"] @@ -1769,13 +1772,13 @@ testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", [[package]] name = "jedi" -version = "0.19.0" +version = "0.19.1" description = "An autocompletion tool for Python that can be used for text editors." optional = false python-versions = ">=3.6" files = [ - {file = "jedi-0.19.0-py2.py3-none-any.whl", hash = "sha256:cb8ce23fbccff0025e9386b5cf85e892f94c9b822378f8da49970471335ac64e"}, - {file = "jedi-0.19.0.tar.gz", hash = "sha256:bcf9894f1753969cbac8022a8c2eaee06bfa3724e4192470aaffe7eb6272b0c4"}, + {file = "jedi-0.19.1-py2.py3-none-any.whl", hash = "sha256:e983c654fe5c02867aef4cdfce5a2fbb4a50adc0af145f70504238f18ef5e7e0"}, + {file = "jedi-0.19.1.tar.gz", hash = "sha256:cf0496f3651bc65d7174ac1b7d043eff454892c708a87d1b683e57b569927ffd"}, ] [package.dependencies] @@ -1784,7 +1787,7 @@ parso = ">=0.8.3,<0.9.0" [package.extras] docs = ["Jinja2 (==2.11.3)", "MarkupSafe (==1.1.1)", "Pygments (==2.8.1)", "alabaster (==0.7.12)", "babel (==2.9.1)", "chardet (==4.0.0)", "commonmark (==0.8.1)", "docutils (==0.17.1)", "future (==0.18.2)", "idna (==2.10)", "imagesize (==1.2.0)", "mock (==1.0.1)", "packaging (==20.9)", "pyparsing (==2.4.7)", "pytz (==2021.1)", "readthedocs-sphinx-ext (==2.1.4)", "recommonmark (==0.5.0)", "requests (==2.25.1)", "six (==1.15.0)", "snowballstemmer (==2.1.0)", "sphinx (==1.8.5)", "sphinx-rtd-theme (==0.4.3)", "sphinxcontrib-serializinghtml (==1.1.4)", "sphinxcontrib-websupport (==1.2.4)", "urllib3 (==1.26.4)"] qa = ["flake8 (==5.0.4)", "mypy (==0.971)", "types-setuptools (==67.2.0.1)"] -testing = ["Django (<3.1)", "attrs", "colorama", "docopt", "pytest (<7.0.0)"] +testing = ["Django", "attrs", "colorama", "docopt", "pytest (<7.0.0)"] [[package]] name = "jeepney" @@ -1818,50 +1821,15 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] -[[package]] -name = "jsonschema" -version = "4.18.6" -description = "An implementation of JSON Schema validation for Python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "jsonschema-4.18.6-py3-none-any.whl", hash = "sha256:dc274409c36175aad949c68e5ead0853aaffbe8e88c830ae66bb3c7a1728ad2d"}, - {file = "jsonschema-4.18.6.tar.gz", hash = "sha256:ce71d2f8c7983ef75a756e568317bf54bc531dc3ad7e66a128eae0d51623d8a3"}, -] - -[package.dependencies] -attrs = ">=22.2.0" -jsonschema-specifications = ">=2023.03.6" -referencing = ">=0.28.4" -rpds-py = ">=0.7.1" - -[package.extras] -format = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3987", "uri-template", "webcolors (>=1.11)"] -format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "uri-template", "webcolors (>=1.11)"] - -[[package]] -name = "jsonschema-specifications" -version = "2023.7.1" -description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" -optional = false -python-versions = ">=3.8" -files = [ - {file = "jsonschema_specifications-2023.7.1-py3-none-any.whl", hash = "sha256:05adf340b659828a004220a9613be00fa3f223f2b82002e273dee62fd50524b1"}, - {file = "jsonschema_specifications-2023.7.1.tar.gz", hash = "sha256:c91a50404e88a1f6ba40636778e2ee08f6e24c5613fe4c53ac24578a5a7f72bb"}, -] - -[package.dependencies] -referencing = ">=0.28.0" - [[package]] name = "keyring" -version = "23.13.1" +version = "24.3.0" description = "Store and access your passwords safely." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "keyring-23.13.1-py3-none-any.whl", hash = "sha256:771ed2a91909389ed6148631de678f82ddc73737d85a927f382a8a1b157898cd"}, - {file = "keyring-23.13.1.tar.gz", hash = "sha256:ba2e15a9b35e21908d0aaf4e0a47acc52d6ae33444df0da2b49d41a46ef6d678"}, + {file = "keyring-24.3.0-py3-none-any.whl", hash = "sha256:4446d35d636e6a10b8bce7caa66913dd9eca5fd222ca03a3d42c38608ac30836"}, + {file = "keyring-24.3.0.tar.gz", hash = "sha256:e730ecffd309658a08ee82535a3b5ec4b4c8669a9be11efb66249d8e0aeb9a25"}, ] [package.dependencies] @@ -1872,30 +1840,19 @@ pywin32-ctypes = {version = ">=0.2.0", markers = "sys_platform == \"win32\""} SecretStorage = {version = ">=3.2", markers = "sys_platform == \"linux\""} [package.extras] -completion = ["shtab"] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] -testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] - -[[package]] -name = "lockfile" -version = "0.12.2" -description = "Platform-independent file locking module" -optional = false -python-versions = "*" -files = [ - {file = "lockfile-0.12.2-py2.py3-none-any.whl", hash = "sha256:6c3cb24f344923d30b2785d5ad75182c8ea7ac1b6171b08657258ec7429d50fa"}, - {file = "lockfile-0.12.2.tar.gz", hash = "sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799"}, -] +completion = ["shtab (>=1.1.0)"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-ruff"] [[package]] name = "mako" -version = "1.2.4" +version = "1.3.0" description = "A super-fast templating language that borrows the best ideas from the existing templating languages." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "Mako-1.2.4-py3-none-any.whl", hash = "sha256:c97c79c018b9165ac9922ae4f32da095ffd3c4e6872b45eded42926deea46818"}, - {file = "Mako-1.2.4.tar.gz", hash = "sha256:d60a3903dc3bb01a18ad6a89cdbe2e4eadc69c0bc8ef1e3773ba53d44c3f7a34"}, + {file = "Mako-1.3.0-py3-none-any.whl", hash = "sha256:57d4e997349f1a92035aa25c17ace371a4213f2ca42f99bee9a602500cfd54d9"}, + {file = "Mako-1.3.0.tar.gz", hash = "sha256:e3a9d388fd00e87043edbe8792f45880ac0114e9c4adc69f6e9bfb2c55e3b11b"}, ] [package.dependencies] @@ -2013,7 +1970,7 @@ files = [ [[package]] name = "metadata-service" -version = "0.1.4" +version = "0.3.3" description = "" optional = false python-versions = "^3.9" @@ -2036,85 +1993,78 @@ url = "../lib" [[package]] name = "more-itertools" -version = "10.1.0" +version = "10.2.0" description = "More routines for operating on iterables, beyond itertools" optional = false python-versions = ">=3.8" files = [ - {file = "more-itertools-10.1.0.tar.gz", hash = "sha256:626c369fa0eb37bac0291bce8259b332fd59ac792fa5497b59837309cd5b114a"}, - {file = "more_itertools-10.1.0-py3-none-any.whl", hash = "sha256:64e0735fcfdc6f3464ea133afe8ea4483b1c5fe3a3d69852e6503b43a0b222e6"}, + {file = "more-itertools-10.2.0.tar.gz", hash = "sha256:8fccb480c43d3e99a00087634c06dd02b0d50fbf088b380de5a41a015ec239e1"}, + {file = "more_itertools-10.2.0-py3-none-any.whl", hash = "sha256:686b06abe565edfab151cb8fd385a05651e1fdf8f0a14191e4439283421f8684"}, ] [[package]] name = "msgpack" -version = "1.0.5" +version = "1.0.7" description = "MessagePack serializer" optional = false -python-versions = "*" +python-versions = ">=3.8" files = [ - {file = "msgpack-1.0.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:525228efd79bb831cf6830a732e2e80bc1b05436b086d4264814b4b2955b2fa9"}, - {file = "msgpack-1.0.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4f8d8b3bf1ff2672567d6b5c725a1b347fe838b912772aa8ae2bf70338d5a198"}, - {file = "msgpack-1.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cdc793c50be3f01106245a61b739328f7dccc2c648b501e237f0699fe1395b81"}, - {file = "msgpack-1.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cb47c21a8a65b165ce29f2bec852790cbc04936f502966768e4aae9fa763cb7"}, - {file = "msgpack-1.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e42b9594cc3bf4d838d67d6ed62b9e59e201862a25e9a157019e171fbe672dd3"}, - {file = "msgpack-1.0.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:55b56a24893105dc52c1253649b60f475f36b3aa0fc66115bffafb624d7cb30b"}, - {file = "msgpack-1.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:1967f6129fc50a43bfe0951c35acbb729be89a55d849fab7686004da85103f1c"}, - {file = "msgpack-1.0.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:20a97bf595a232c3ee6d57ddaadd5453d174a52594bf9c21d10407e2a2d9b3bd"}, - {file = "msgpack-1.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d25dd59bbbbb996eacf7be6b4ad082ed7eacc4e8f3d2df1ba43822da9bfa122a"}, - {file = "msgpack-1.0.5-cp310-cp310-win32.whl", hash = "sha256:382b2c77589331f2cb80b67cc058c00f225e19827dbc818d700f61513ab47bea"}, - {file = "msgpack-1.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:4867aa2df9e2a5fa5f76d7d5565d25ec76e84c106b55509e78c1ede0f152659a"}, - {file = "msgpack-1.0.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9f5ae84c5c8a857ec44dc180a8b0cc08238e021f57abdf51a8182e915e6299f0"}, - {file = "msgpack-1.0.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9e6ca5d5699bcd89ae605c150aee83b5321f2115695e741b99618f4856c50898"}, - {file = "msgpack-1.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5494ea30d517a3576749cad32fa27f7585c65f5f38309c88c6d137877fa28a5a"}, - {file = "msgpack-1.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ab2f3331cb1b54165976a9d976cb251a83183631c88076613c6c780f0d6e45a"}, - {file = "msgpack-1.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28592e20bbb1620848256ebc105fc420436af59515793ed27d5c77a217477705"}, - {file = "msgpack-1.0.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fe5c63197c55bce6385d9aee16c4d0641684628f63ace85f73571e65ad1c1e8d"}, - {file = "msgpack-1.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ed40e926fa2f297e8a653c954b732f125ef97bdd4c889f243182299de27e2aa9"}, - {file = "msgpack-1.0.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:b2de4c1c0538dcb7010902a2b97f4e00fc4ddf2c8cda9749af0e594d3b7fa3d7"}, - {file = "msgpack-1.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:bf22a83f973b50f9d38e55c6aade04c41ddda19b00c4ebc558930d78eecc64ed"}, - {file = "msgpack-1.0.5-cp311-cp311-win32.whl", hash = "sha256:c396e2cc213d12ce017b686e0f53497f94f8ba2b24799c25d913d46c08ec422c"}, - {file = "msgpack-1.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:6c4c68d87497f66f96d50142a2b73b97972130d93677ce930718f68828b382e2"}, - {file = "msgpack-1.0.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a2b031c2e9b9af485d5e3c4520f4220d74f4d222a5b8dc8c1a3ab9448ca79c57"}, - {file = "msgpack-1.0.5-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f837b93669ce4336e24d08286c38761132bc7ab29782727f8557e1eb21b2080"}, - {file = "msgpack-1.0.5-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1d46dfe3832660f53b13b925d4e0fa1432b00f5f7210eb3ad3bb9a13c6204a6"}, - {file = "msgpack-1.0.5-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:366c9a7b9057e1547f4ad51d8facad8b406bab69c7d72c0eb6f529cf76d4b85f"}, - {file = "msgpack-1.0.5-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:4c075728a1095efd0634a7dccb06204919a2f67d1893b6aa8e00497258bf926c"}, - {file = "msgpack-1.0.5-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:f933bbda5a3ee63b8834179096923b094b76f0c7a73c1cfe8f07ad608c58844b"}, - {file = "msgpack-1.0.5-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:36961b0568c36027c76e2ae3ca1132e35123dcec0706c4b7992683cc26c1320c"}, - {file = "msgpack-1.0.5-cp36-cp36m-win32.whl", hash = "sha256:b5ef2f015b95f912c2fcab19c36814963b5463f1fb9049846994b007962743e9"}, - {file = "msgpack-1.0.5-cp36-cp36m-win_amd64.whl", hash = "sha256:288e32b47e67f7b171f86b030e527e302c91bd3f40fd9033483f2cacc37f327a"}, - {file = "msgpack-1.0.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:137850656634abddfb88236008339fdaba3178f4751b28f270d2ebe77a563b6c"}, - {file = "msgpack-1.0.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c05a4a96585525916b109bb85f8cb6511db1c6f5b9d9cbcbc940dc6b4be944b"}, - {file = "msgpack-1.0.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56a62ec00b636583e5cb6ad313bbed36bb7ead5fa3a3e38938503142c72cba4f"}, - {file = "msgpack-1.0.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef8108f8dedf204bb7b42994abf93882da1159728a2d4c5e82012edd92c9da9f"}, - {file = "msgpack-1.0.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1835c84d65f46900920b3708f5ba829fb19b1096c1800ad60bae8418652a951d"}, - {file = "msgpack-1.0.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:e57916ef1bd0fee4f21c4600e9d1da352d8816b52a599c46460e93a6e9f17086"}, - {file = "msgpack-1.0.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:17358523b85973e5f242ad74aa4712b7ee560715562554aa2134d96e7aa4cbbf"}, - {file = "msgpack-1.0.5-cp37-cp37m-win32.whl", hash = "sha256:cb5aaa8c17760909ec6cb15e744c3ebc2ca8918e727216e79607b7bbce9c8f77"}, - {file = "msgpack-1.0.5-cp37-cp37m-win_amd64.whl", hash = "sha256:ab31e908d8424d55601ad7075e471b7d0140d4d3dd3272daf39c5c19d936bd82"}, - {file = "msgpack-1.0.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:b72d0698f86e8d9ddf9442bdedec15b71df3598199ba33322d9711a19f08145c"}, - {file = "msgpack-1.0.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:379026812e49258016dd84ad79ac8446922234d498058ae1d415f04b522d5b2d"}, - {file = "msgpack-1.0.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:332360ff25469c346a1c5e47cbe2a725517919892eda5cfaffe6046656f0b7bb"}, - {file = "msgpack-1.0.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:476a8fe8fae289fdf273d6d2a6cb6e35b5a58541693e8f9f019bfe990a51e4ba"}, - {file = "msgpack-1.0.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9985b214f33311df47e274eb788a5893a761d025e2b92c723ba4c63936b69b1"}, - {file = "msgpack-1.0.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:48296af57cdb1d885843afd73c4656be5c76c0c6328db3440c9601a98f303d87"}, - {file = "msgpack-1.0.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:addab7e2e1fcc04bd08e4eb631c2a90960c340e40dfc4a5e24d2ff0d5a3b3edb"}, - {file = "msgpack-1.0.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:916723458c25dfb77ff07f4c66aed34e47503b2eb3188b3adbec8d8aa6e00f48"}, - {file = "msgpack-1.0.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:821c7e677cc6acf0fd3f7ac664c98803827ae6de594a9f99563e48c5a2f27eb0"}, - {file = "msgpack-1.0.5-cp38-cp38-win32.whl", hash = "sha256:1c0f7c47f0087ffda62961d425e4407961a7ffd2aa004c81b9c07d9269512f6e"}, - {file = "msgpack-1.0.5-cp38-cp38-win_amd64.whl", hash = "sha256:bae7de2026cbfe3782c8b78b0db9cbfc5455e079f1937cb0ab8d133496ac55e1"}, - {file = "msgpack-1.0.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:20c784e66b613c7f16f632e7b5e8a1651aa5702463d61394671ba07b2fc9e025"}, - {file = "msgpack-1.0.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:266fa4202c0eb94d26822d9bfd7af25d1e2c088927fe8de9033d929dd5ba24c5"}, - {file = "msgpack-1.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:18334484eafc2b1aa47a6d42427da7fa8f2ab3d60b674120bce7a895a0a85bdd"}, - {file = "msgpack-1.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57e1f3528bd95cc44684beda696f74d3aaa8a5e58c816214b9046512240ef437"}, - {file = "msgpack-1.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:586d0d636f9a628ddc6a17bfd45aa5b5efaf1606d2b60fa5d87b8986326e933f"}, - {file = "msgpack-1.0.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a740fa0e4087a734455f0fc3abf5e746004c9da72fbd541e9b113013c8dc3282"}, - {file = "msgpack-1.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3055b0455e45810820db1f29d900bf39466df96ddca11dfa6d074fa47054376d"}, - {file = "msgpack-1.0.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a61215eac016f391129a013c9e46f3ab308db5f5ec9f25811e811f96962599a8"}, - {file = "msgpack-1.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:362d9655cd369b08fda06b6657a303eb7172d5279997abe094512e919cf74b11"}, - {file = "msgpack-1.0.5-cp39-cp39-win32.whl", hash = "sha256:ac9dd47af78cae935901a9a500104e2dea2e253207c924cc95de149606dc43cc"}, - {file = "msgpack-1.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:06f5174b5f8ed0ed919da0e62cbd4ffde676a374aba4020034da05fab67b9164"}, - {file = "msgpack-1.0.5.tar.gz", hash = "sha256:c075544284eadc5cddc70f4757331d99dcbc16b2bbd4849d15f8aae4cf36d31c"}, + {file = "msgpack-1.0.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:04ad6069c86e531682f9e1e71b71c1c3937d6014a7c3e9edd2aa81ad58842862"}, + {file = "msgpack-1.0.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cca1b62fe70d761a282496b96a5e51c44c213e410a964bdffe0928e611368329"}, + {file = "msgpack-1.0.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e50ebce52f41370707f1e21a59514e3375e3edd6e1832f5e5235237db933c98b"}, + {file = "msgpack-1.0.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a7b4f35de6a304b5533c238bee86b670b75b03d31b7797929caa7a624b5dda6"}, + {file = "msgpack-1.0.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28efb066cde83c479dfe5a48141a53bc7e5f13f785b92ddde336c716663039ee"}, + {file = "msgpack-1.0.7-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4cb14ce54d9b857be9591ac364cb08dc2d6a5c4318c1182cb1d02274029d590d"}, + {file = "msgpack-1.0.7-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b573a43ef7c368ba4ea06050a957c2a7550f729c31f11dd616d2ac4aba99888d"}, + {file = "msgpack-1.0.7-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ccf9a39706b604d884d2cb1e27fe973bc55f2890c52f38df742bc1d79ab9f5e1"}, + {file = "msgpack-1.0.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cb70766519500281815dfd7a87d3a178acf7ce95390544b8c90587d76b227681"}, + {file = "msgpack-1.0.7-cp310-cp310-win32.whl", hash = "sha256:b610ff0f24e9f11c9ae653c67ff8cc03c075131401b3e5ef4b82570d1728f8a9"}, + {file = "msgpack-1.0.7-cp310-cp310-win_amd64.whl", hash = "sha256:a40821a89dc373d6427e2b44b572efc36a2778d3f543299e2f24eb1a5de65415"}, + {file = "msgpack-1.0.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:576eb384292b139821c41995523654ad82d1916da6a60cff129c715a6223ea84"}, + {file = "msgpack-1.0.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:730076207cb816138cf1af7f7237b208340a2c5e749707457d70705715c93b93"}, + {file = "msgpack-1.0.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:85765fdf4b27eb5086f05ac0491090fc76f4f2b28e09d9350c31aac25a5aaff8"}, + {file = "msgpack-1.0.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3476fae43db72bd11f29a5147ae2f3cb22e2f1a91d575ef130d2bf49afd21c46"}, + {file = "msgpack-1.0.7-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d4c80667de2e36970ebf74f42d1088cc9ee7ef5f4e8c35eee1b40eafd33ca5b"}, + {file = "msgpack-1.0.7-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5b0bf0effb196ed76b7ad883848143427a73c355ae8e569fa538365064188b8e"}, + {file = "msgpack-1.0.7-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:f9a7c509542db4eceed3dcf21ee5267ab565a83555c9b88a8109dcecc4709002"}, + {file = "msgpack-1.0.7-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:84b0daf226913133f899ea9b30618722d45feffa67e4fe867b0b5ae83a34060c"}, + {file = "msgpack-1.0.7-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ec79ff6159dffcc30853b2ad612ed572af86c92b5168aa3fc01a67b0fa40665e"}, + {file = "msgpack-1.0.7-cp311-cp311-win32.whl", hash = "sha256:3e7bf4442b310ff154b7bb9d81eb2c016b7d597e364f97d72b1acc3817a0fdc1"}, + {file = "msgpack-1.0.7-cp311-cp311-win_amd64.whl", hash = "sha256:3f0c8c6dfa6605ab8ff0611995ee30d4f9fcff89966cf562733b4008a3d60d82"}, + {file = "msgpack-1.0.7-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f0936e08e0003f66bfd97e74ee530427707297b0d0361247e9b4f59ab78ddc8b"}, + {file = "msgpack-1.0.7-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:98bbd754a422a0b123c66a4c341de0474cad4a5c10c164ceed6ea090f3563db4"}, + {file = "msgpack-1.0.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b291f0ee7961a597cbbcc77709374087fa2a9afe7bdb6a40dbbd9b127e79afee"}, + {file = "msgpack-1.0.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebbbba226f0a108a7366bf4b59bf0f30a12fd5e75100c630267d94d7f0ad20e5"}, + {file = "msgpack-1.0.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e2d69948e4132813b8d1131f29f9101bc2c915f26089a6d632001a5c1349672"}, + {file = "msgpack-1.0.7-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bdf38ba2d393c7911ae989c3bbba510ebbcdf4ecbdbfec36272abe350c454075"}, + {file = "msgpack-1.0.7-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:993584fc821c58d5993521bfdcd31a4adf025c7d745bbd4d12ccfecf695af5ba"}, + {file = "msgpack-1.0.7-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:52700dc63a4676669b341ba33520f4d6e43d3ca58d422e22ba66d1736b0a6e4c"}, + {file = "msgpack-1.0.7-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e45ae4927759289c30ccba8d9fdce62bb414977ba158286b5ddaf8df2cddb5c5"}, + {file = "msgpack-1.0.7-cp312-cp312-win32.whl", hash = "sha256:27dcd6f46a21c18fa5e5deed92a43d4554e3df8d8ca5a47bf0615d6a5f39dbc9"}, + {file = "msgpack-1.0.7-cp312-cp312-win_amd64.whl", hash = "sha256:7687e22a31e976a0e7fc99c2f4d11ca45eff652a81eb8c8085e9609298916dcf"}, + {file = "msgpack-1.0.7-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5b6ccc0c85916998d788b295765ea0e9cb9aac7e4a8ed71d12e7d8ac31c23c95"}, + {file = "msgpack-1.0.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:235a31ec7db685f5c82233bddf9858748b89b8119bf4538d514536c485c15fe0"}, + {file = "msgpack-1.0.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cab3db8bab4b7e635c1c97270d7a4b2a90c070b33cbc00c99ef3f9be03d3e1f7"}, + {file = "msgpack-1.0.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0bfdd914e55e0d2c9e1526de210f6fe8ffe9705f2b1dfcc4aecc92a4cb4b533d"}, + {file = "msgpack-1.0.7-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36e17c4592231a7dbd2ed09027823ab295d2791b3b1efb2aee874b10548b7524"}, + {file = "msgpack-1.0.7-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38949d30b11ae5f95c3c91917ee7a6b239f5ec276f271f28638dec9156f82cfc"}, + {file = "msgpack-1.0.7-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ff1d0899f104f3921d94579a5638847f783c9b04f2d5f229392ca77fba5b82fc"}, + {file = "msgpack-1.0.7-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:dc43f1ec66eb8440567186ae2f8c447d91e0372d793dfe8c222aec857b81a8cf"}, + {file = "msgpack-1.0.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:dd632777ff3beaaf629f1ab4396caf7ba0bdd075d948a69460d13d44357aca4c"}, + {file = "msgpack-1.0.7-cp38-cp38-win32.whl", hash = "sha256:4e71bc4416de195d6e9b4ee93ad3f2f6b2ce11d042b4d7a7ee00bbe0358bd0c2"}, + {file = "msgpack-1.0.7-cp38-cp38-win_amd64.whl", hash = "sha256:8f5b234f567cf76ee489502ceb7165c2a5cecec081db2b37e35332b537f8157c"}, + {file = "msgpack-1.0.7-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:bfef2bb6ef068827bbd021017a107194956918ab43ce4d6dc945ffa13efbc25f"}, + {file = "msgpack-1.0.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:484ae3240666ad34cfa31eea7b8c6cd2f1fdaae21d73ce2974211df099a95d81"}, + {file = "msgpack-1.0.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3967e4ad1aa9da62fd53e346ed17d7b2e922cba5ab93bdd46febcac39be636fc"}, + {file = "msgpack-1.0.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8dd178c4c80706546702c59529ffc005681bd6dc2ea234c450661b205445a34d"}, + {file = "msgpack-1.0.7-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6ffbc252eb0d229aeb2f9ad051200668fc3a9aaa8994e49f0cb2ffe2b7867e7"}, + {file = "msgpack-1.0.7-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:822ea70dc4018c7e6223f13affd1c5c30c0f5c12ac1f96cd8e9949acddb48a61"}, + {file = "msgpack-1.0.7-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:384d779f0d6f1b110eae74cb0659d9aa6ff35aaf547b3955abf2ab4c901c4819"}, + {file = "msgpack-1.0.7-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f64e376cd20d3f030190e8c32e1c64582eba56ac6dc7d5b0b49a9d44021b52fd"}, + {file = "msgpack-1.0.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5ed82f5a7af3697b1c4786053736f24a0efd0a1b8a130d4c7bfee4b9ded0f08f"}, + {file = "msgpack-1.0.7-cp39-cp39-win32.whl", hash = "sha256:f26a07a6e877c76a88e3cecac8531908d980d3d5067ff69213653649ec0f60ad"}, + {file = "msgpack-1.0.7-cp39-cp39-win_amd64.whl", hash = "sha256:1dc93e8e4653bdb5910aed79f11e165c85732067614f180f70534f056da97db3"}, + {file = "msgpack-1.0.7.tar.gz", hash = "sha256:572efc93db7a4d27e404501975ca6d2d9775705c2d922390d878fcf768d92c87"}, ] [[package]] @@ -2202,36 +2152,47 @@ files = [ [[package]] name = "numpy" -version = "1.25.2" +version = "1.26.3" description = "Fundamental package for array computing in Python" optional = false python-versions = ">=3.9" files = [ - {file = "numpy-1.25.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:db3ccc4e37a6873045580d413fe79b68e47a681af8db2e046f1dacfa11f86eb3"}, - {file = "numpy-1.25.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:90319e4f002795ccfc9050110bbbaa16c944b1c37c0baeea43c5fb881693ae1f"}, - {file = "numpy-1.25.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfe4a913e29b418d096e696ddd422d8a5d13ffba4ea91f9f60440a3b759b0187"}, - {file = "numpy-1.25.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f08f2e037bba04e707eebf4bc934f1972a315c883a9e0ebfa8a7756eabf9e357"}, - {file = "numpy-1.25.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bec1e7213c7cb00d67093247f8c4db156fd03075f49876957dca4711306d39c9"}, - {file = "numpy-1.25.2-cp310-cp310-win32.whl", hash = "sha256:7dc869c0c75988e1c693d0e2d5b26034644399dd929bc049db55395b1379e044"}, - {file = "numpy-1.25.2-cp310-cp310-win_amd64.whl", hash = "sha256:834b386f2b8210dca38c71a6e0f4fd6922f7d3fcff935dbe3a570945acb1b545"}, - {file = "numpy-1.25.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c5462d19336db4560041517dbb7759c21d181a67cb01b36ca109b2ae37d32418"}, - {file = "numpy-1.25.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c5652ea24d33585ea39eb6a6a15dac87a1206a692719ff45d53c5282e66d4a8f"}, - {file = "numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2"}, - {file = "numpy-1.25.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60e7f0f7f6d0eee8364b9a6304c2845b9c491ac706048c7e8cf47b83123b8dbf"}, - {file = "numpy-1.25.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:bb33d5a1cf360304754913a350edda36d5b8c5331a8237268c48f91253c3a364"}, - {file = "numpy-1.25.2-cp311-cp311-win32.whl", hash = "sha256:5883c06bb92f2e6c8181df7b39971a5fb436288db58b5a1c3967702d4278691d"}, - {file = "numpy-1.25.2-cp311-cp311-win_amd64.whl", hash = "sha256:5c97325a0ba6f9d041feb9390924614b60b99209a71a69c876f71052521d42a4"}, - {file = "numpy-1.25.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b79e513d7aac42ae918db3ad1341a015488530d0bb2a6abcbdd10a3a829ccfd3"}, - {file = "numpy-1.25.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eb942bfb6f84df5ce05dbf4b46673ffed0d3da59f13635ea9b926af3deb76926"}, - {file = "numpy-1.25.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e0746410e73384e70d286f93abf2520035250aad8c5714240b0492a7302fdca"}, - {file = "numpy-1.25.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7806500e4f5bdd04095e849265e55de20d8cc4b661b038957354327f6d9b295"}, - {file = "numpy-1.25.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8b77775f4b7df768967a7c8b3567e309f617dd5e99aeb886fa14dc1a0791141f"}, - {file = "numpy-1.25.2-cp39-cp39-win32.whl", hash = "sha256:2792d23d62ec51e50ce4d4b7d73de8f67a2fd3ea710dcbc8563a51a03fb07b01"}, - {file = "numpy-1.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:76b4115d42a7dfc5d485d358728cdd8719be33cc5ec6ec08632a5d6fca2ed380"}, - {file = "numpy-1.25.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1a1329e26f46230bf77b02cc19e900db9b52f398d6722ca853349a782d4cff55"}, - {file = "numpy-1.25.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c3abc71e8b6edba80a01a52e66d83c5d14433cbcd26a40c329ec7ed09f37901"}, - {file = "numpy-1.25.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:1b9735c27cea5d995496f46a8b1cd7b408b3f34b6d50459d9ac8fe3a20cc17bf"}, - {file = "numpy-1.25.2.tar.gz", hash = "sha256:fd608e19c8d7c55021dffd43bfe5492fab8cc105cc8986f813f8c3c048b38760"}, + {file = "numpy-1.26.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:806dd64230dbbfaca8a27faa64e2f414bf1c6622ab78cc4264f7f5f028fee3bf"}, + {file = "numpy-1.26.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02f98011ba4ab17f46f80f7f8f1c291ee7d855fcef0a5a98db80767a468c85cd"}, + {file = "numpy-1.26.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d45b3ec2faed4baca41c76617fcdcfa4f684ff7a151ce6fc78ad3b6e85af0a6"}, + {file = "numpy-1.26.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdd2b45bf079d9ad90377048e2747a0c82351989a2165821f0c96831b4a2a54b"}, + {file = "numpy-1.26.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:211ddd1e94817ed2d175b60b6374120244a4dd2287f4ece45d49228b4d529178"}, + {file = "numpy-1.26.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b1240f767f69d7c4c8a29adde2310b871153df9b26b5cb2b54a561ac85146485"}, + {file = "numpy-1.26.3-cp310-cp310-win32.whl", hash = "sha256:21a9484e75ad018974a2fdaa216524d64ed4212e418e0a551a2d83403b0531d3"}, + {file = "numpy-1.26.3-cp310-cp310-win_amd64.whl", hash = "sha256:9e1591f6ae98bcfac2a4bbf9221c0b92ab49762228f38287f6eeb5f3f55905ce"}, + {file = "numpy-1.26.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b831295e5472954104ecb46cd98c08b98b49c69fdb7040483aff799a755a7374"}, + {file = "numpy-1.26.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9e87562b91f68dd8b1c39149d0323b42e0082db7ddb8e934ab4c292094d575d6"}, + {file = "numpy-1.26.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c66d6fec467e8c0f975818c1796d25c53521124b7cfb760114be0abad53a0a2"}, + {file = "numpy-1.26.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f25e2811a9c932e43943a2615e65fc487a0b6b49218899e62e426e7f0a57eeda"}, + {file = "numpy-1.26.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:af36e0aa45e25c9f57bf684b1175e59ea05d9a7d3e8e87b7ae1a1da246f2767e"}, + {file = "numpy-1.26.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:51c7f1b344f302067b02e0f5b5d2daa9ed4a721cf49f070280ac202738ea7f00"}, + {file = "numpy-1.26.3-cp311-cp311-win32.whl", hash = "sha256:7ca4f24341df071877849eb2034948459ce3a07915c2734f1abb4018d9c49d7b"}, + {file = "numpy-1.26.3-cp311-cp311-win_amd64.whl", hash = "sha256:39763aee6dfdd4878032361b30b2b12593fb445ddb66bbac802e2113eb8a6ac4"}, + {file = "numpy-1.26.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a7081fd19a6d573e1a05e600c82a1c421011db7935ed0d5c483e9dd96b99cf13"}, + {file = "numpy-1.26.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12c70ac274b32bc00c7f61b515126c9205323703abb99cd41836e8125ea0043e"}, + {file = "numpy-1.26.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f784e13e598e9594750b2ef6729bcd5a47f6cfe4a12cca13def35e06d8163e3"}, + {file = "numpy-1.26.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f24750ef94d56ce6e33e4019a8a4d68cfdb1ef661a52cdaee628a56d2437419"}, + {file = "numpy-1.26.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:77810ef29e0fb1d289d225cabb9ee6cf4d11978a00bb99f7f8ec2132a84e0166"}, + {file = "numpy-1.26.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8ed07a90f5450d99dad60d3799f9c03c6566709bd53b497eb9ccad9a55867f36"}, + {file = "numpy-1.26.3-cp312-cp312-win32.whl", hash = "sha256:f73497e8c38295aaa4741bdfa4fda1a5aedda5473074369eca10626835445511"}, + {file = "numpy-1.26.3-cp312-cp312-win_amd64.whl", hash = "sha256:da4b0c6c699a0ad73c810736303f7fbae483bcb012e38d7eb06a5e3b432c981b"}, + {file = "numpy-1.26.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1666f634cb3c80ccbd77ec97bc17337718f56d6658acf5d3b906ca03e90ce87f"}, + {file = "numpy-1.26.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:18c3319a7d39b2c6a9e3bb75aab2304ab79a811ac0168a671a62e6346c29b03f"}, + {file = "numpy-1.26.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b7e807d6888da0db6e7e75838444d62495e2b588b99e90dd80c3459594e857b"}, + {file = "numpy-1.26.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4d362e17bcb0011738c2d83e0a65ea8ce627057b2fdda37678f4374a382a137"}, + {file = "numpy-1.26.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b8c275f0ae90069496068c714387b4a0eba5d531aace269559ff2b43655edd58"}, + {file = "numpy-1.26.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:cc0743f0302b94f397a4a65a660d4cd24267439eb16493fb3caad2e4389bccbb"}, + {file = "numpy-1.26.3-cp39-cp39-win32.whl", hash = "sha256:9bc6d1a7f8cedd519c4b7b1156d98e051b726bf160715b769106661d567b3f03"}, + {file = "numpy-1.26.3-cp39-cp39-win_amd64.whl", hash = "sha256:867e3644e208c8922a3be26fc6bbf112a035f50f0a86497f98f228c50c607bb2"}, + {file = "numpy-1.26.3-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3c67423b3703f8fbd90f5adaa37f85b5794d3366948efe9a5190a5f3a83fc34e"}, + {file = "numpy-1.26.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46f47ee566d98849323f01b349d58f2557f02167ee301e5e28809a8c0e27a2d0"}, + {file = "numpy-1.26.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a8474703bffc65ca15853d5fd4d06b18138ae90c17c8d12169968e998e448bb5"}, + {file = "numpy-1.26.3.tar.gz", hash = "sha256:697df43e2b6310ecc9d95f05d5ef20eacc09c7c4ecc9da3f235d39e71b7da1e4"}, ] [[package]] @@ -2268,13 +2229,13 @@ dev = ["black", "mypy", "pytest"] [[package]] name = "packaging" -version = "23.1" +version = "23.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, - {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, ] [[package]] @@ -2396,29 +2357,27 @@ pytzdata = ">=2020.1" [[package]] name = "pex" -version = "2.0.3" +version = "2.1.156" description = "The PEX packaging toolchain." optional = false -python-versions = "*" +python-versions = ">=2.7,<3.13,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" files = [ - {file = "pex-2.0.3-py2.py3-none-any.whl", hash = "sha256:4ca62e27fd30cd1d4acbcdadfc52739c6e73c9292613d98cb35d88065174fa63"}, - {file = "pex-2.0.3.tar.gz", hash = "sha256:a8a35e7eb212616b2964d70d8a134d41d16649c943ab206b90c749c005e60999"}, + {file = "pex-2.1.156-py2.py3-none-any.whl", hash = "sha256:e7c00fe6f12f6b2ed57ab8e55c4d422647b30e25a4a275cfbc3d3b0bc26e774a"}, + {file = "pex-2.1.156.tar.gz", hash = "sha256:542ecb457c21f5ae8fa749894098e1c54e8639628efee70ece7f89da602aa4c2"}, ] [package.extras] -cachecontrol = ["CacheControl (>=0.12.3)"] -requests = ["requests (>=2.8.14)"] subprocess = ["subprocess32 (>=3.2.7)"] [[package]] name = "pexpect" -version = "4.8.0" +version = "4.9.0" description = "Pexpect allows easy control of interactive console applications." optional = false python-versions = "*" files = [ - {file = "pexpect-4.8.0-py2.py3-none-any.whl", hash = "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937"}, - {file = "pexpect-4.8.0.tar.gz", hash = "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c"}, + {file = "pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523"}, + {file = "pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f"}, ] [package.dependencies] @@ -2440,13 +2399,13 @@ testing = ["pytest", "pytest-cov"] [[package]] name = "platformdirs" -version = "3.10.0" +version = "3.11.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." optional = false python-versions = ">=3.7" files = [ - {file = "platformdirs-3.10.0-py3-none-any.whl", hash = "sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d"}, - {file = "platformdirs-3.10.0.tar.gz", hash = "sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d"}, + {file = "platformdirs-3.11.0-py3-none-any.whl", hash = "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e"}, + {file = "platformdirs-3.11.0.tar.gz", hash = "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3"}, ] [package.extras] @@ -2455,13 +2414,13 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-co [[package]] name = "pluggy" -version = "1.2.0" +version = "1.3.0" description = "plugin and hook calling mechanisms for python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, - {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, + {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, + {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, ] [package.extras] @@ -2480,70 +2439,66 @@ files = [ [[package]] name = "poetry" -version = "1.5.1" +version = "1.7.1" description = "Python dependency management and packaging made easy." optional = false -python-versions = ">=3.7,<4.0" +python-versions = ">=3.8,<4.0" files = [ - {file = "poetry-1.5.1-py3-none-any.whl", hash = "sha256:dfc7ce3a38ae216c0465694e2e674bef6eb1a2ba81aa47a26f9dc03362fe2f5f"}, - {file = "poetry-1.5.1.tar.gz", hash = "sha256:cc7ea4524d1a11558006224bfe8ba8ed071417d4eb5ef6c89decc6a37d437eeb"}, + {file = "poetry-1.7.1-py3-none-any.whl", hash = "sha256:03d3807a0fb3bc1028cc3707dfd646aae629d58e476f7e7f062437680741c561"}, + {file = "poetry-1.7.1.tar.gz", hash = "sha256:b348a70e7d67ad9c0bd3d0ea255bc6df84c24cf4b16f8d104adb30b425d6ff32"}, ] [package.dependencies] -build = ">=0.10.0,<0.11.0" -cachecontrol = {version = ">=0.12.9,<0.13.0", extras = ["filecache"]} -cleo = ">=2.0.0,<3.0.0" +build = ">=1.0.3,<2.0.0" +cachecontrol = {version = ">=0.13.0,<0.14.0", extras = ["filecache"]} +cleo = ">=2.1.0,<3.0.0" crashtest = ">=0.4.1,<0.5.0" dulwich = ">=0.21.2,<0.22.0" -filelock = ">=3.8.0,<4.0.0" -html5lib = ">=1.0,<2.0" +fastjsonschema = ">=2.18.0,<3.0.0" importlib-metadata = {version = ">=4.4", markers = "python_version < \"3.10\""} installer = ">=0.7.0,<0.8.0" -jsonschema = ">=4.10.0,<5.0.0" -keyring = ">=23.9.0,<24.0.0" -lockfile = ">=0.12.2,<0.13.0" -packaging = ">=20.4" +keyring = ">=24.0.0,<25.0.0" +packaging = ">=20.5" pexpect = ">=4.7.0,<5.0.0" pkginfo = ">=1.9.4,<2.0.0" platformdirs = ">=3.0.0,<4.0.0" -poetry-core = "1.6.1" -poetry-plugin-export = ">=1.4.0,<2.0.0" +poetry-core = "1.8.1" +poetry-plugin-export = ">=1.6.0,<2.0.0" pyproject-hooks = ">=1.0.0,<2.0.0" -requests = ">=2.18,<3.0" +requests = ">=2.26,<3.0" requests-toolbelt = ">=0.9.1,<2" shellingham = ">=1.5,<2.0" tomli = {version = ">=2.0.1,<3.0.0", markers = "python_version < \"3.11\""} tomlkit = ">=0.11.4,<1.0.0" trove-classifiers = ">=2022.5.19" -urllib3 = ">=1.26.0,<2.0.0" -virtualenv = ">=20.22.0,<21.0.0" +virtualenv = ">=20.23.0,<21.0.0" xattr = {version = ">=0.10.0,<0.11.0", markers = "sys_platform == \"darwin\""} [[package]] name = "poetry-core" -version = "1.6.1" +version = "1.8.1" description = "Poetry PEP 517 Build Backend" optional = false -python-versions = ">=3.7,<4.0" +python-versions = ">=3.8,<4.0" files = [ - {file = "poetry_core-1.6.1-py3-none-any.whl", hash = "sha256:70707340447dee0e7f334f9495ae652481c67b32d8d218f296a376ac2ed73573"}, - {file = "poetry_core-1.6.1.tar.gz", hash = "sha256:0f9b0de39665f36d6594657e7d57b6f463cc10f30c28e6d1c3b9ff54c26c9ac3"}, + {file = "poetry_core-1.8.1-py3-none-any.whl", hash = "sha256:194832b24f3283e01c5402eae71a6aae850ecdfe53f50a979c76bf7aa5010ffa"}, + {file = "poetry_core-1.8.1.tar.gz", hash = "sha256:67a76c671da2a70e55047cddda83566035b701f7e463b32a2abfeac6e2a16376"}, ] [[package]] name = "poetry-plugin-export" -version = "1.4.0" +version = "1.6.0" description = "Poetry plugin to export the dependencies to various formats" optional = false -python-versions = ">=3.7,<4.0" +python-versions = ">=3.8,<4.0" files = [ - {file = "poetry_plugin_export-1.4.0-py3-none-any.whl", hash = "sha256:5d9186d6f77cf2bf35fc96bd11fe650cc7656e515b17d99cb65018d50ba22589"}, - {file = "poetry_plugin_export-1.4.0.tar.gz", hash = "sha256:f16974cd9f222d4ef640fa97a8d661b04d4fb339e51da93973f1bc9d578e183f"}, + {file = "poetry_plugin_export-1.6.0-py3-none-any.whl", hash = "sha256:2dce6204c9318f1f6509a11a03921fb3f461b201840b59f1c237b6ab454dabcf"}, + {file = "poetry_plugin_export-1.6.0.tar.gz", hash = "sha256:091939434984267a91abf2f916a26b00cff4eee8da63ec2a24ba4b17cf969a59"}, ] [package.dependencies] -poetry = ">=1.5.0,<2.0.0" -poetry-core = ">=1.6.0,<2.0.0" +poetry = ">=1.6.0,<2.0.0" +poetry-core = ">=1.7.0,<2.0.0" [[package]] name = "poetry2setup" @@ -2561,78 +2516,61 @@ poetry-core = ">=1.0.0,<2.0.0" [[package]] name = "prompt-toolkit" -version = "3.0.39" +version = "3.0.43" description = "Library for building powerful interactive command lines in Python" optional = false python-versions = ">=3.7.0" files = [ - {file = "prompt_toolkit-3.0.39-py3-none-any.whl", hash = "sha256:9dffbe1d8acf91e3de75f3b544e4842382fc06c6babe903ac9acb74dc6e08d88"}, - {file = "prompt_toolkit-3.0.39.tar.gz", hash = "sha256:04505ade687dc26dc4284b1ad19a83be2f2afe83e7a828ace0c72f3a1df72aac"}, + {file = "prompt_toolkit-3.0.43-py3-none-any.whl", hash = "sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6"}, + {file = "prompt_toolkit-3.0.43.tar.gz", hash = "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d"}, ] [package.dependencies] wcwidth = "*" -[[package]] -name = "proto-plus" -version = "1.22.3" -description = "Beautiful, Pythonic protocol buffers." -optional = false -python-versions = ">=3.6" -files = [ - {file = "proto-plus-1.22.3.tar.gz", hash = "sha256:fdcd09713cbd42480740d2fe29c990f7fbd885a67efc328aa8be6ee3e9f76a6b"}, - {file = "proto_plus-1.22.3-py3-none-any.whl", hash = "sha256:a49cd903bc0b6ab41f76bf65510439d56ca76f868adf0274e738bfdd096894df"}, -] - -[package.dependencies] -protobuf = ">=3.19.0,<5.0.0dev" - -[package.extras] -testing = ["google-api-core[grpc] (>=1.31.5)"] - [[package]] name = "protobuf" -version = "4.23.4" +version = "4.25.1" description = "" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "protobuf-4.23.4-cp310-abi3-win32.whl", hash = "sha256:5fea3c64d41ea5ecf5697b83e41d09b9589e6f20b677ab3c48e5f242d9b7897b"}, - {file = "protobuf-4.23.4-cp310-abi3-win_amd64.whl", hash = "sha256:7b19b6266d92ca6a2a87effa88ecc4af73ebc5cfde194dc737cf8ef23a9a3b12"}, - {file = "protobuf-4.23.4-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:8547bf44fe8cec3c69e3042f5c4fb3e36eb2a7a013bb0a44c018fc1e427aafbd"}, - {file = "protobuf-4.23.4-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:fee88269a090ada09ca63551bf2f573eb2424035bcf2cb1b121895b01a46594a"}, - {file = "protobuf-4.23.4-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:effeac51ab79332d44fba74660d40ae79985901ac21bca408f8dc335a81aa597"}, - {file = "protobuf-4.23.4-cp37-cp37m-win32.whl", hash = "sha256:c3e0939433c40796ca4cfc0fac08af50b00eb66a40bbbc5dee711998fb0bbc1e"}, - {file = "protobuf-4.23.4-cp37-cp37m-win_amd64.whl", hash = "sha256:9053df6df8e5a76c84339ee4a9f5a2661ceee4a0dab019e8663c50ba324208b0"}, - {file = "protobuf-4.23.4-cp38-cp38-win32.whl", hash = "sha256:e1c915778d8ced71e26fcf43c0866d7499891bca14c4368448a82edc61fdbc70"}, - {file = "protobuf-4.23.4-cp38-cp38-win_amd64.whl", hash = "sha256:351cc90f7d10839c480aeb9b870a211e322bf05f6ab3f55fcb2f51331f80a7d2"}, - {file = "protobuf-4.23.4-cp39-cp39-win32.whl", hash = "sha256:6dd9b9940e3f17077e820b75851126615ee38643c2c5332aa7a359988820c720"}, - {file = "protobuf-4.23.4-cp39-cp39-win_amd64.whl", hash = "sha256:0a5759f5696895de8cc913f084e27fd4125e8fb0914bb729a17816a33819f474"}, - {file = "protobuf-4.23.4-py3-none-any.whl", hash = "sha256:e9d0be5bf34b275b9f87ba7407796556abeeba635455d036c7351f7c183ef8ff"}, - {file = "protobuf-4.23.4.tar.gz", hash = "sha256:ccd9430c0719dce806b93f89c91de7977304729e55377f872a92465d548329a9"}, + {file = "protobuf-4.25.1-cp310-abi3-win32.whl", hash = "sha256:193f50a6ab78a970c9b4f148e7c750cfde64f59815e86f686c22e26b4fe01ce7"}, + {file = "protobuf-4.25.1-cp310-abi3-win_amd64.whl", hash = "sha256:3497c1af9f2526962f09329fd61a36566305e6c72da2590ae0d7d1322818843b"}, + {file = "protobuf-4.25.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:0bf384e75b92c42830c0a679b0cd4d6e2b36ae0cf3dbb1e1dfdda48a244f4bcd"}, + {file = "protobuf-4.25.1-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:0f881b589ff449bf0b931a711926e9ddaad3b35089cc039ce1af50b21a4ae8cb"}, + {file = "protobuf-4.25.1-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:ca37bf6a6d0046272c152eea90d2e4ef34593aaa32e8873fc14c16440f22d4b7"}, + {file = "protobuf-4.25.1-cp38-cp38-win32.whl", hash = "sha256:abc0525ae2689a8000837729eef7883b9391cd6aa7950249dcf5a4ede230d5dd"}, + {file = "protobuf-4.25.1-cp38-cp38-win_amd64.whl", hash = "sha256:1484f9e692091450e7edf418c939e15bfc8fc68856e36ce399aed6889dae8bb0"}, + {file = "protobuf-4.25.1-cp39-cp39-win32.whl", hash = "sha256:8bdbeaddaac52d15c6dce38c71b03038ef7772b977847eb6d374fc86636fa510"}, + {file = "protobuf-4.25.1-cp39-cp39-win_amd64.whl", hash = "sha256:becc576b7e6b553d22cbdf418686ee4daa443d7217999125c045ad56322dda10"}, + {file = "protobuf-4.25.1-py3-none-any.whl", hash = "sha256:a19731d5e83ae4737bb2a089605e636077ac001d18781b3cf489b9546c7c80d6"}, + {file = "protobuf-4.25.1.tar.gz", hash = "sha256:57d65074b4f5baa4ab5da1605c02be90ac20c8b40fb137d6a8df9f416b0d0ce2"}, ] [[package]] name = "psutil" -version = "5.9.5" +version = "5.9.7" description = "Cross-platform lib for process and system monitoring in Python." optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ - {file = "psutil-5.9.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:be8929ce4313f9f8146caad4272f6abb8bf99fc6cf59344a3167ecd74f4f203f"}, - {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ab8ed1a1d77c95453db1ae00a3f9c50227ebd955437bcf2a574ba8adbf6a74d5"}, - {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:4aef137f3345082a3d3232187aeb4ac4ef959ba3d7c10c33dd73763fbc063da4"}, - {file = "psutil-5.9.5-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48"}, - {file = "psutil-5.9.5-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:acf2aef9391710afded549ff602b5887d7a2349831ae4c26be7c807c0a39fac4"}, - {file = "psutil-5.9.5-cp27-none-win32.whl", hash = "sha256:5b9b8cb93f507e8dbaf22af6a2fd0ccbe8244bf30b1baad6b3954e935157ae3f"}, - {file = "psutil-5.9.5-cp27-none-win_amd64.whl", hash = "sha256:8c5f7c5a052d1d567db4ddd231a9d27a74e8e4a9c3f44b1032762bd7b9fdcd42"}, - {file = "psutil-5.9.5-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:3c6f686f4225553615612f6d9bc21f1c0e305f75d7d8454f9b46e901778e7217"}, - {file = "psutil-5.9.5-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a7dd9997128a0d928ed4fb2c2d57e5102bb6089027939f3b722f3a210f9a8da"}, - {file = "psutil-5.9.5-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89518112647f1276b03ca97b65cc7f64ca587b1eb0278383017c2a0dcc26cbe4"}, - {file = "psutil-5.9.5-cp36-abi3-win32.whl", hash = "sha256:104a5cc0e31baa2bcf67900be36acde157756b9c44017b86b2c049f11957887d"}, - {file = "psutil-5.9.5-cp36-abi3-win_amd64.whl", hash = "sha256:b258c0c1c9d145a1d5ceffab1134441c4c5113b2417fafff7315a917a026c3c9"}, - {file = "psutil-5.9.5-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30"}, - {file = "psutil-5.9.5.tar.gz", hash = "sha256:5410638e4df39c54d957fc51ce03048acd8e6d60abc0f5107af51e5fb566eb3c"}, + {file = "psutil-5.9.7-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:0bd41bf2d1463dfa535942b2a8f0e958acf6607ac0be52265ab31f7923bcd5e6"}, + {file = "psutil-5.9.7-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:5794944462509e49d4d458f4dbfb92c47539e7d8d15c796f141f474010084056"}, + {file = "psutil-5.9.7-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:fe361f743cb3389b8efda21980d93eb55c1f1e3898269bc9a2a1d0bb7b1f6508"}, + {file = "psutil-5.9.7-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:e469990e28f1ad738f65a42dcfc17adaed9d0f325d55047593cb9033a0ab63df"}, + {file = "psutil-5.9.7-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:3c4747a3e2ead1589e647e64aad601981f01b68f9398ddf94d01e3dc0d1e57c7"}, + {file = "psutil-5.9.7-cp27-none-win32.whl", hash = "sha256:1d4bc4a0148fdd7fd8f38e0498639ae128e64538faa507df25a20f8f7fb2341c"}, + {file = "psutil-5.9.7-cp27-none-win_amd64.whl", hash = "sha256:4c03362e280d06bbbfcd52f29acd79c733e0af33d707c54255d21029b8b32ba6"}, + {file = "psutil-5.9.7-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ea36cc62e69a13ec52b2f625c27527f6e4479bca2b340b7a452af55b34fcbe2e"}, + {file = "psutil-5.9.7-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1132704b876e58d277168cd729d64750633d5ff0183acf5b3c986b8466cd0284"}, + {file = "psutil-5.9.7-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe8b7f07948f1304497ce4f4684881250cd859b16d06a1dc4d7941eeb6233bfe"}, + {file = "psutil-5.9.7-cp36-cp36m-win32.whl", hash = "sha256:b27f8fdb190c8c03914f908a4555159327d7481dac2f01008d483137ef3311a9"}, + {file = "psutil-5.9.7-cp36-cp36m-win_amd64.whl", hash = "sha256:44969859757f4d8f2a9bd5b76eba8c3099a2c8cf3992ff62144061e39ba8568e"}, + {file = "psutil-5.9.7-cp37-abi3-win32.whl", hash = "sha256:c727ca5a9b2dd5193b8644b9f0c883d54f1248310023b5ad3e92036c5e2ada68"}, + {file = "psutil-5.9.7-cp37-abi3-win_amd64.whl", hash = "sha256:f37f87e4d73b79e6c5e749440c3113b81d1ee7d26f21c19c47371ddea834f414"}, + {file = "psutil-5.9.7-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:032f4f2c909818c86cea4fe2cc407f1c0f0cde8e6c6d702b28b8ce0c0d143340"}, + {file = "psutil-5.9.7.tar.gz", hash = "sha256:3f02134e82cfb5d089fddf20bb2e03fd5cd52395321d1c8458a9e58500ff417c"}, ] [package.extras] @@ -2640,19 +2578,19 @@ test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] [[package]] name = "ptpython" -version = "3.0.23" +version = "3.0.25" description = "Python REPL build on top of prompt_toolkit" optional = false python-versions = ">=3.7" files = [ - {file = "ptpython-3.0.23-py2.py3-none-any.whl", hash = "sha256:51069503684169b21e1980734a9ba2e104643b7e6a50d3ca0e5669ea70d9e21c"}, - {file = "ptpython-3.0.23.tar.gz", hash = "sha256:9fc9bec2cc51bc4000c1224d8c56241ce8a406b3d49ec8dc266f78cd3cd04ba4"}, + {file = "ptpython-3.0.25-py2.py3-none-any.whl", hash = "sha256:16654143dea960dcefb9d6e69af5f92f01c7a783dd28ff99e78bc7449fba805c"}, + {file = "ptpython-3.0.25.tar.gz", hash = "sha256:887f0a91a576bc26585a0dcec41cd03f004ac7c46a2c88576c87fc51d6c06cd7"}, ] [package.dependencies] appdirs = "*" jedi = ">=0.16.0" -prompt-toolkit = ">=3.0.28,<3.1.0" +prompt-toolkit = ">=3.0.34,<3.1.0" pygments = "*" [package.extras] @@ -2672,36 +2610,47 @@ files = [ [[package]] name = "pyarrow" -version = "12.0.1" +version = "14.0.2" description = "Python library for Apache Arrow" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pyarrow-12.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:6d288029a94a9bb5407ceebdd7110ba398a00412c5b0155ee9813a40d246c5df"}, - {file = "pyarrow-12.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:345e1828efdbd9aa4d4de7d5676778aba384a2c3add896d995b23d368e60e5af"}, - {file = "pyarrow-12.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d6009fdf8986332b2169314da482baed47ac053311c8934ac6651e614deacd6"}, - {file = "pyarrow-12.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d3c4cbbf81e6dd23fe921bc91dc4619ea3b79bc58ef10bce0f49bdafb103daf"}, - {file = "pyarrow-12.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:cdacf515ec276709ac8042c7d9bd5be83b4f5f39c6c037a17a60d7ebfd92c890"}, - {file = "pyarrow-12.0.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:749be7fd2ff260683f9cc739cb862fb11be376de965a2a8ccbf2693b098db6c7"}, - {file = "pyarrow-12.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6895b5fb74289d055c43db3af0de6e16b07586c45763cb5e558d38b86a91e3a7"}, - {file = "pyarrow-12.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1887bdae17ec3b4c046fcf19951e71b6a619f39fa674f9881216173566c8f718"}, - {file = "pyarrow-12.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2c9cb8eeabbadf5fcfc3d1ddea616c7ce893db2ce4dcef0ac13b099ad7ca082"}, - {file = "pyarrow-12.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:ce4aebdf412bd0eeb800d8e47db854f9f9f7e2f5a0220440acf219ddfddd4f63"}, - {file = "pyarrow-12.0.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:e0d8730c7f6e893f6db5d5b86eda42c0a130842d101992b581e2138e4d5663d3"}, - {file = "pyarrow-12.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43364daec02f69fec89d2315f7fbfbeec956e0d991cbbef471681bd77875c40f"}, - {file = "pyarrow-12.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:051f9f5ccf585f12d7de836e50965b3c235542cc896959320d9776ab93f3b33d"}, - {file = "pyarrow-12.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:be2757e9275875d2a9c6e6052ac7957fbbfc7bc7370e4a036a9b893e96fedaba"}, - {file = "pyarrow-12.0.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:cf812306d66f40f69e684300f7af5111c11f6e0d89d6b733e05a3de44961529d"}, - {file = "pyarrow-12.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:459a1c0ed2d68671188b2118c63bac91eaef6fc150c77ddd8a583e3c795737bf"}, - {file = "pyarrow-12.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:85e705e33eaf666bbe508a16fd5ba27ca061e177916b7a317ba5a51bee43384c"}, - {file = "pyarrow-12.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9120c3eb2b1f6f516a3b7a9714ed860882d9ef98c4b17edcdc91d95b7528db60"}, - {file = "pyarrow-12.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:c780f4dc40460015d80fcd6a6140de80b615349ed68ef9adb653fe351778c9b3"}, - {file = "pyarrow-12.0.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:a3c63124fc26bf5f95f508f5d04e1ece8cc23a8b0af2a1e6ab2b1ec3fdc91b24"}, - {file = "pyarrow-12.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b13329f79fa4472324f8d32dc1b1216616d09bd1e77cfb13104dec5463632c36"}, - {file = "pyarrow-12.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb656150d3d12ec1396f6dde542db1675a95c0cc8366d507347b0beed96e87ca"}, - {file = "pyarrow-12.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6251e38470da97a5b2e00de5c6a049149f7b2bd62f12fa5dbb9ac674119ba71a"}, - {file = "pyarrow-12.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:3de26da901216149ce086920547dfff5cd22818c9eab67ebc41e863a5883bac7"}, - {file = "pyarrow-12.0.1.tar.gz", hash = "sha256:cce317fc96e5b71107bf1f9f184d5e54e2bd14bbf3f9a3d62819961f0af86fec"}, + {file = "pyarrow-14.0.2-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:ba9fe808596c5dbd08b3aeffe901e5f81095baaa28e7d5118e01354c64f22807"}, + {file = "pyarrow-14.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:22a768987a16bb46220cef490c56c671993fbee8fd0475febac0b3e16b00a10e"}, + {file = "pyarrow-14.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dbba05e98f247f17e64303eb876f4a80fcd32f73c7e9ad975a83834d81f3fda"}, + {file = "pyarrow-14.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a898d134d00b1eca04998e9d286e19653f9d0fcb99587310cd10270907452a6b"}, + {file = "pyarrow-14.0.2-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:87e879323f256cb04267bb365add7208f302df942eb943c93a9dfeb8f44840b1"}, + {file = "pyarrow-14.0.2-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:76fc257559404ea5f1306ea9a3ff0541bf996ff3f7b9209fc517b5e83811fa8e"}, + {file = "pyarrow-14.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:b0c4a18e00f3a32398a7f31da47fefcd7a927545b396e1f15d0c85c2f2c778cd"}, + {file = "pyarrow-14.0.2-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:87482af32e5a0c0cce2d12eb3c039dd1d853bd905b04f3f953f147c7a196915b"}, + {file = "pyarrow-14.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:059bd8f12a70519e46cd64e1ba40e97eae55e0cbe1695edd95384653d7626b23"}, + {file = "pyarrow-14.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f16111f9ab27e60b391c5f6d197510e3ad6654e73857b4e394861fc79c37200"}, + {file = "pyarrow-14.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06ff1264fe4448e8d02073f5ce45a9f934c0f3db0a04460d0b01ff28befc3696"}, + {file = "pyarrow-14.0.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:6dd4f4b472ccf4042f1eab77e6c8bce574543f54d2135c7e396f413046397d5a"}, + {file = "pyarrow-14.0.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:32356bfb58b36059773f49e4e214996888eeea3a08893e7dbde44753799b2a02"}, + {file = "pyarrow-14.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:52809ee69d4dbf2241c0e4366d949ba035cbcf48409bf404f071f624ed313a2b"}, + {file = "pyarrow-14.0.2-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:c87824a5ac52be210d32906c715f4ed7053d0180c1060ae3ff9b7e560f53f944"}, + {file = "pyarrow-14.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a25eb2421a58e861f6ca91f43339d215476f4fe159eca603c55950c14f378cc5"}, + {file = "pyarrow-14.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c1da70d668af5620b8ba0a23f229030a4cd6c5f24a616a146f30d2386fec422"}, + {file = "pyarrow-14.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2cc61593c8e66194c7cdfae594503e91b926a228fba40b5cf25cc593563bcd07"}, + {file = "pyarrow-14.0.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:78ea56f62fb7c0ae8ecb9afdd7893e3a7dbeb0b04106f5c08dbb23f9c0157591"}, + {file = "pyarrow-14.0.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:37c233ddbce0c67a76c0985612fef27c0c92aef9413cf5aa56952f359fcb7379"}, + {file = "pyarrow-14.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:e4b123ad0f6add92de898214d404e488167b87b5dd86e9a434126bc2b7a5578d"}, + {file = "pyarrow-14.0.2-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:e354fba8490de258be7687f341bc04aba181fc8aa1f71e4584f9890d9cb2dec2"}, + {file = "pyarrow-14.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:20e003a23a13da963f43e2b432483fdd8c38dc8882cd145f09f21792e1cf22a1"}, + {file = "pyarrow-14.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc0de7575e841f1595ac07e5bc631084fd06ca8b03c0f2ecece733d23cd5102a"}, + {file = "pyarrow-14.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66e986dc859712acb0bd45601229021f3ffcdfc49044b64c6d071aaf4fa49e98"}, + {file = "pyarrow-14.0.2-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:f7d029f20ef56673a9730766023459ece397a05001f4e4d13805111d7c2108c0"}, + {file = "pyarrow-14.0.2-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:209bac546942b0d8edc8debda248364f7f668e4aad4741bae58e67d40e5fcf75"}, + {file = "pyarrow-14.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:1e6987c5274fb87d66bb36816afb6f65707546b3c45c44c28e3c4133c010a881"}, + {file = "pyarrow-14.0.2-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:a01d0052d2a294a5f56cc1862933014e696aa08cc7b620e8c0cce5a5d362e976"}, + {file = "pyarrow-14.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a51fee3a7db4d37f8cda3ea96f32530620d43b0489d169b285d774da48ca9785"}, + {file = "pyarrow-14.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64df2bf1ef2ef14cee531e2dfe03dd924017650ffaa6f9513d7a1bb291e59c15"}, + {file = "pyarrow-14.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c0fa3bfdb0305ffe09810f9d3e2e50a2787e3a07063001dcd7adae0cee3601a"}, + {file = "pyarrow-14.0.2-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:c65bf4fd06584f058420238bc47a316e80dda01ec0dfb3044594128a6c2db794"}, + {file = "pyarrow-14.0.2-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:63ac901baec9369d6aae1cbe6cca11178fb018a8d45068aaf5bb54f94804a866"}, + {file = "pyarrow-14.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:75ee0efe7a87a687ae303d63037d08a48ef9ea0127064df18267252cfe2e9541"}, + {file = "pyarrow-14.0.2.tar.gz", hash = "sha256:36cef6ba12b499d864d1def3e990f97949e0b79400d08b7cf74504ffbd3eb025"}, ] [package.dependencies] @@ -2709,13 +2658,13 @@ numpy = ">=1.16.6" [[package]] name = "pyasn1" -version = "0.5.0" +version = "0.5.1" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ - {file = "pyasn1-0.5.0-py2.py3-none-any.whl", hash = "sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57"}, - {file = "pyasn1-0.5.0.tar.gz", hash = "sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde"}, + {file = "pyasn1-0.5.1-py2.py3-none-any.whl", hash = "sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58"}, + {file = "pyasn1-0.5.1.tar.gz", hash = "sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c"}, ] [[package]] @@ -2745,47 +2694,47 @@ files = [ [[package]] name = "pydantic" -version = "1.10.12" +version = "1.10.13" description = "Data validation and settings management using python type hints" optional = false python-versions = ">=3.7" files = [ - {file = "pydantic-1.10.12-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a1fcb59f2f355ec350073af41d927bf83a63b50e640f4dbaa01053a28b7a7718"}, - {file = "pydantic-1.10.12-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b7ccf02d7eb340b216ec33e53a3a629856afe1c6e0ef91d84a4e6f2fb2ca70fe"}, - {file = "pydantic-1.10.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8fb2aa3ab3728d950bcc885a2e9eff6c8fc40bc0b7bb434e555c215491bcf48b"}, - {file = "pydantic-1.10.12-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:771735dc43cf8383959dc9b90aa281f0b6092321ca98677c5fb6125a6f56d58d"}, - {file = "pydantic-1.10.12-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ca48477862372ac3770969b9d75f1bf66131d386dba79506c46d75e6b48c1e09"}, - {file = "pydantic-1.10.12-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a5e7add47a5b5a40c49b3036d464e3c7802f8ae0d1e66035ea16aa5b7a3923ed"}, - {file = "pydantic-1.10.12-cp310-cp310-win_amd64.whl", hash = "sha256:e4129b528c6baa99a429f97ce733fff478ec955513630e61b49804b6cf9b224a"}, - {file = "pydantic-1.10.12-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b0d191db0f92dfcb1dec210ca244fdae5cbe918c6050b342d619c09d31eea0cc"}, - {file = "pydantic-1.10.12-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:795e34e6cc065f8f498c89b894a3c6da294a936ee71e644e4bd44de048af1405"}, - {file = "pydantic-1.10.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:69328e15cfda2c392da4e713443c7dbffa1505bc9d566e71e55abe14c97ddc62"}, - {file = "pydantic-1.10.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2031de0967c279df0d8a1c72b4ffc411ecd06bac607a212892757db7462fc494"}, - {file = "pydantic-1.10.12-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ba5b2e6fe6ca2b7e013398bc7d7b170e21cce322d266ffcd57cca313e54fb246"}, - {file = "pydantic-1.10.12-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:2a7bac939fa326db1ab741c9d7f44c565a1d1e80908b3797f7f81a4f86bc8d33"}, - {file = "pydantic-1.10.12-cp311-cp311-win_amd64.whl", hash = "sha256:87afda5539d5140cb8ba9e8b8c8865cb5b1463924d38490d73d3ccfd80896b3f"}, - {file = "pydantic-1.10.12-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:549a8e3d81df0a85226963611950b12d2d334f214436a19537b2efed61b7639a"}, - {file = "pydantic-1.10.12-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:598da88dfa127b666852bef6d0d796573a8cf5009ffd62104094a4fe39599565"}, - {file = "pydantic-1.10.12-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ba5c4a8552bff16c61882db58544116d021d0b31ee7c66958d14cf386a5b5350"}, - {file = "pydantic-1.10.12-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c79e6a11a07da7374f46970410b41d5e266f7f38f6a17a9c4823db80dadf4303"}, - {file = "pydantic-1.10.12-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ab26038b8375581dc832a63c948f261ae0aa21f1d34c1293469f135fa92972a5"}, - {file = "pydantic-1.10.12-cp37-cp37m-win_amd64.whl", hash = "sha256:e0a16d274b588767602b7646fa05af2782576a6cf1022f4ba74cbb4db66f6ca8"}, - {file = "pydantic-1.10.12-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6a9dfa722316f4acf4460afdf5d41d5246a80e249c7ff475c43a3a1e9d75cf62"}, - {file = "pydantic-1.10.12-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:a73f489aebd0c2121ed974054cb2759af8a9f747de120acd2c3394cf84176ccb"}, - {file = "pydantic-1.10.12-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b30bcb8cbfccfcf02acb8f1a261143fab622831d9c0989707e0e659f77a18e0"}, - {file = "pydantic-1.10.12-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2fcfb5296d7877af406ba1547dfde9943b1256d8928732267e2653c26938cd9c"}, - {file = "pydantic-1.10.12-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:2f9a6fab5f82ada41d56b0602606a5506aab165ca54e52bc4545028382ef1c5d"}, - {file = "pydantic-1.10.12-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:dea7adcc33d5d105896401a1f37d56b47d443a2b2605ff8a969a0ed5543f7e33"}, - {file = "pydantic-1.10.12-cp38-cp38-win_amd64.whl", hash = "sha256:1eb2085c13bce1612da8537b2d90f549c8cbb05c67e8f22854e201bde5d98a47"}, - {file = "pydantic-1.10.12-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ef6c96b2baa2100ec91a4b428f80d8f28a3c9e53568219b6c298c1125572ebc6"}, - {file = "pydantic-1.10.12-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6c076be61cd0177a8433c0adcb03475baf4ee91edf5a4e550161ad57fc90f523"}, - {file = "pydantic-1.10.12-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d5a58feb9a39f481eda4d5ca220aa8b9d4f21a41274760b9bc66bfd72595b86"}, - {file = "pydantic-1.10.12-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e5f805d2d5d0a41633651a73fa4ecdd0b3d7a49de4ec3fadf062fe16501ddbf1"}, - {file = "pydantic-1.10.12-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:1289c180abd4bd4555bb927c42ee42abc3aee02b0fb2d1223fb7c6e5bef87dbe"}, - {file = "pydantic-1.10.12-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5d1197e462e0364906cbc19681605cb7c036f2475c899b6f296104ad42b9f5fb"}, - {file = "pydantic-1.10.12-cp39-cp39-win_amd64.whl", hash = "sha256:fdbdd1d630195689f325c9ef1a12900524dceb503b00a987663ff4f58669b93d"}, - {file = "pydantic-1.10.12-py3-none-any.whl", hash = "sha256:b749a43aa51e32839c9d71dc67eb1e4221bb04af1033a32e3923d46f9effa942"}, - {file = "pydantic-1.10.12.tar.gz", hash = "sha256:0fe8a415cea8f340e7a9af9c54fc71a649b43e8ca3cc732986116b3cb135d303"}, + {file = "pydantic-1.10.13-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:efff03cc7a4f29d9009d1c96ceb1e7a70a65cfe86e89d34e4a5f2ab1e5693737"}, + {file = "pydantic-1.10.13-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3ecea2b9d80e5333303eeb77e180b90e95eea8f765d08c3d278cd56b00345d01"}, + {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1740068fd8e2ef6eb27a20e5651df000978edce6da6803c2bef0bc74540f9548"}, + {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84bafe2e60b5e78bc64a2941b4c071a4b7404c5c907f5f5a99b0139781e69ed8"}, + {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bc0898c12f8e9c97f6cd44c0ed70d55749eaf783716896960b4ecce2edfd2d69"}, + {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:654db58ae399fe6434e55325a2c3e959836bd17a6f6a0b6ca8107ea0571d2e17"}, + {file = "pydantic-1.10.13-cp310-cp310-win_amd64.whl", hash = "sha256:75ac15385a3534d887a99c713aa3da88a30fbd6204a5cd0dc4dab3d770b9bd2f"}, + {file = "pydantic-1.10.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c553f6a156deb868ba38a23cf0df886c63492e9257f60a79c0fd8e7173537653"}, + {file = "pydantic-1.10.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5e08865bc6464df8c7d61439ef4439829e3ab62ab1669cddea8dd00cd74b9ffe"}, + {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e31647d85a2013d926ce60b84f9dd5300d44535a9941fe825dc349ae1f760df9"}, + {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:210ce042e8f6f7c01168b2d84d4c9eb2b009fe7bf572c2266e235edf14bacd80"}, + {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8ae5dd6b721459bfa30805f4c25880e0dd78fc5b5879f9f7a692196ddcb5a580"}, + {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f8e81fc5fb17dae698f52bdd1c4f18b6ca674d7068242b2aff075f588301bbb0"}, + {file = "pydantic-1.10.13-cp311-cp311-win_amd64.whl", hash = "sha256:61d9dce220447fb74f45e73d7ff3b530e25db30192ad8d425166d43c5deb6df0"}, + {file = "pydantic-1.10.13-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4b03e42ec20286f052490423682016fd80fda830d8e4119f8ab13ec7464c0132"}, + {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59ef915cac80275245824e9d771ee939133be38215555e9dc90c6cb148aaeb5"}, + {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a1f9f747851338933942db7af7b6ee8268568ef2ed86c4185c6ef4402e80ba8"}, + {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:97cce3ae7341f7620a0ba5ef6cf043975cd9d2b81f3aa5f4ea37928269bc1b87"}, + {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854223752ba81e3abf663d685f105c64150873cc6f5d0c01d3e3220bcff7d36f"}, + {file = "pydantic-1.10.13-cp37-cp37m-win_amd64.whl", hash = "sha256:b97c1fac8c49be29486df85968682b0afa77e1b809aff74b83081cc115e52f33"}, + {file = "pydantic-1.10.13-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c958d053453a1c4b1c2062b05cd42d9d5c8eb67537b8d5a7e3c3032943ecd261"}, + {file = "pydantic-1.10.13-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c5370a7edaac06daee3af1c8b1192e305bc102abcbf2a92374b5bc793818599"}, + {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d6f6e7305244bddb4414ba7094ce910560c907bdfa3501e9db1a7fd7eaea127"}, + {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3a3c792a58e1622667a2837512099eac62490cdfd63bd407993aaf200a4cf1f"}, + {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c636925f38b8db208e09d344c7aa4f29a86bb9947495dd6b6d376ad10334fb78"}, + {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:678bcf5591b63cc917100dc50ab6caebe597ac67e8c9ccb75e698f66038ea953"}, + {file = "pydantic-1.10.13-cp38-cp38-win_amd64.whl", hash = "sha256:6cf25c1a65c27923a17b3da28a0bdb99f62ee04230c931d83e888012851f4e7f"}, + {file = "pydantic-1.10.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8ef467901d7a41fa0ca6db9ae3ec0021e3f657ce2c208e98cd511f3161c762c6"}, + {file = "pydantic-1.10.13-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:968ac42970f57b8344ee08837b62f6ee6f53c33f603547a55571c954a4225691"}, + {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9849f031cf8a2f0a928fe885e5a04b08006d6d41876b8bbd2fc68a18f9f2e3fd"}, + {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:56e3ff861c3b9c6857579de282ce8baabf443f42ffba355bf070770ed63e11e1"}, + {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f00790179497767aae6bcdc36355792c79e7bbb20b145ff449700eb076c5f96"}, + {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:75b297827b59bc229cac1a23a2f7a4ac0031068e5be0ce385be1462e7e17a35d"}, + {file = "pydantic-1.10.13-cp39-cp39-win_amd64.whl", hash = "sha256:e70ca129d2053fb8b728ee7d1af8e553a928d7e301a311094b8a0501adc8763d"}, + {file = "pydantic-1.10.13-py3-none-any.whl", hash = "sha256:b87326822e71bd5f313e7d3bfdc77ac3247035ac10b0c0618bd99dcf95b1e687"}, + {file = "pydantic-1.10.13.tar.gz", hash = "sha256:32c8b48dcd3b2ac4e78b0ba4af3a2c2eb6048cb75202f0ea7b34feb740efc340"}, ] [package.dependencies] @@ -2828,17 +2777,18 @@ requests = ">=2.14.0" [[package]] name = "pygments" -version = "2.15.1" +version = "2.17.2" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.7" files = [ - {file = "Pygments-2.15.1-py3-none-any.whl", hash = "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"}, - {file = "Pygments-2.15.1.tar.gz", hash = "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c"}, + {file = "pygments-2.17.2-py3-none-any.whl", hash = "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c"}, + {file = "pygments-2.17.2.tar.gz", hash = "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"}, ] [package.extras] plugins = ["importlib-metadata"] +windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pyjwt" @@ -2947,13 +2897,13 @@ files = [ [[package]] name = "pytest" -version = "7.4.0" +version = "7.4.4" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32"}, - {file = "pytest-7.4.0.tar.gz", hash = "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"}, + {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, + {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, ] [package.dependencies] @@ -2997,13 +2947,13 @@ cli = ["click (>=5.0)"] [[package]] name = "pytz" -version = "2023.3" +version = "2023.3.post1" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" files = [ - {file = "pytz-2023.3-py2.py3-none-any.whl", hash = "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb"}, - {file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"}, + {file = "pytz-2023.3.post1-py2.py3-none-any.whl", hash = "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7"}, + {file = "pytz-2023.3.post1.tar.gz", hash = "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b"}, ] [[package]] @@ -3119,123 +3069,106 @@ docs = ["Sphinx (>=3.3,<4.0)", "sphinx-autobuild (>=2020.9.1,<2021.0.0)", "sphin [[package]] name = "rapidfuzz" -version = "2.15.1" +version = "3.6.1" description = "rapid fuzzy string matching" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "rapidfuzz-2.15.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fc0bc259ebe3b93e7ce9df50b3d00e7345335d35acbd735163b7c4b1957074d3"}, - {file = "rapidfuzz-2.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d59fb3a410d253f50099d7063855c2b95df1ef20ad93ea3a6b84115590899f25"}, - {file = "rapidfuzz-2.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c525a3da17b6d79d61613096c8683da86e3573e807dfaecf422eea09e82b5ba6"}, - {file = "rapidfuzz-2.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4deae6a918ecc260d0c4612257be8ba321d8e913ccb43155403842758c46fbe"}, - {file = "rapidfuzz-2.15.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2577463d10811386e704a3ab58b903eb4e2a31b24dfd9886d789b0084d614b01"}, - {file = "rapidfuzz-2.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f67d5f56aa48c0da9de4ab81bffb310683cf7815f05ea38e5aa64f3ba4368339"}, - {file = "rapidfuzz-2.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d7927722ff43690e52b3145b5bd3089151d841d350c6f8378c3cfac91f67573a"}, - {file = "rapidfuzz-2.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6534afc787e32c4104f65cdeb55f6abe4d803a2d0553221d00ef9ce12788dcde"}, - {file = "rapidfuzz-2.15.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d0ae6ec79a1931929bb9dd57bc173eb5ba4c7197461bf69e3a34b6dd314feed2"}, - {file = "rapidfuzz-2.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:be7ccc45c4d1a7dfb595f260e8022a90c6cb380c2a346ee5aae93f85c96d362b"}, - {file = "rapidfuzz-2.15.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:8ba013500a2b68c64b2aecc5fb56a2dad6c2872cf545a0308fd044827b6e5f6a"}, - {file = "rapidfuzz-2.15.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:4d9f7d10065f657f960b48699e7dddfce14ab91af4bab37a215f0722daf0d716"}, - {file = "rapidfuzz-2.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7e24a1b802cea04160b3fccd75d2d0905065783ebc9de157d83c14fb9e1c6ce2"}, - {file = "rapidfuzz-2.15.1-cp310-cp310-win32.whl", hash = "sha256:dffdf03499e0a5b3442951bb82b556333b069e0661e80568752786c79c5b32de"}, - {file = "rapidfuzz-2.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:7d150d90a7c6caae7962f29f857a4e61d42038cfd82c9df38508daf30c648ae7"}, - {file = "rapidfuzz-2.15.1-cp310-cp310-win_arm64.whl", hash = "sha256:87c30e9184998ff6eb0fa9221f94282ce7c908fd0da96a1ef66ecadfaaa4cdb7"}, - {file = "rapidfuzz-2.15.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6986413cb37035eb796e32f049cbc8c13d8630a4ac1e0484e3e268bb3662bd1b"}, - {file = "rapidfuzz-2.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a72f26e010d4774b676f36e43c0fc8a2c26659efef4b3be3fd7714d3491e9957"}, - {file = "rapidfuzz-2.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b5cd54c98a387cca111b3b784fc97a4f141244bbc28a92d4bde53f164464112e"}, - {file = "rapidfuzz-2.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da7fac7c3da39f93e6b2ebe386ed0ffe1cefec91509b91857f6e1204509e931f"}, - {file = "rapidfuzz-2.15.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f976e76ac72f650790b3a5402431612175b2ac0363179446285cb3c901136ca9"}, - {file = "rapidfuzz-2.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:abde47e1595902a490ed14d4338d21c3509156abb2042a99e6da51f928e0c117"}, - {file = "rapidfuzz-2.15.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca8f1747007a3ce919739a60fa95c5325f7667cccf6f1c1ef18ae799af119f5e"}, - {file = "rapidfuzz-2.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c35da09ab9797b020d0d4f07a66871dfc70ea6566363811090353ea971748b5a"}, - {file = "rapidfuzz-2.15.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a3a769ca7580686a66046b77df33851b3c2d796dc1eb60c269b68f690f3e1b65"}, - {file = "rapidfuzz-2.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d50622efefdb03a640a51a6123748cd151d305c1f0431af762e833d6ffef71f0"}, - {file = "rapidfuzz-2.15.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:b7461b0a7651d68bc23f0896bffceea40f62887e5ab8397bf7caa883592ef5cb"}, - {file = "rapidfuzz-2.15.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:074ee9e17912e025c72a5780ee4c7c413ea35cd26449719cc399b852d4e42533"}, - {file = "rapidfuzz-2.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7025fb105a11f503943f17718cdb8241ea3bb4d812c710c609e69bead40e2ff0"}, - {file = "rapidfuzz-2.15.1-cp311-cp311-win32.whl", hash = "sha256:2084d36b95139413cef25e9487257a1cc892b93bd1481acd2a9656f7a1d9930c"}, - {file = "rapidfuzz-2.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:5a738fcd24e34bce4b19126b92fdae15482d6d3a90bd687fd3d24ce9d28ce82d"}, - {file = "rapidfuzz-2.15.1-cp311-cp311-win_arm64.whl", hash = "sha256:dc3cafa68cfa54638632bdcadf9aab89a3d182b4a3f04d2cad7585ed58ea8731"}, - {file = "rapidfuzz-2.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3c53d57ba7a88f7bf304d4ea5a14a0ca112db0e0178fff745d9005acf2879f7d"}, - {file = "rapidfuzz-2.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6ee758eec4cf2215dc8d8eafafcea0d1f48ad4b0135767db1b0f7c5c40a17dd"}, - {file = "rapidfuzz-2.15.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2d93ba3ae59275e7a3a116dac4ffdb05e9598bf3ee0861fecc5b60fb042d539e"}, - {file = "rapidfuzz-2.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7c3ff75e647908ddbe9aa917fbe39a112d5631171f3fcea5809e2363e525a59d"}, - {file = "rapidfuzz-2.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6d89c421702474c6361245b6b199e6e9783febacdbfb6b002669e6cb3ef17a09"}, - {file = "rapidfuzz-2.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f69e6199fec0f58f9a89afbbaea78d637c7ce77f656a03a1d6ea6abdc1d44f8"}, - {file = "rapidfuzz-2.15.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:41dfea282844d0628279b4db2929da0dacb8ac317ddc5dcccc30093cf16357c1"}, - {file = "rapidfuzz-2.15.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2dd03477feefeccda07b7659dd614f6738cfc4f9b6779dd61b262a73b0a9a178"}, - {file = "rapidfuzz-2.15.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:5efe035aa76ff37d1b5fa661de3c4b4944de9ff227a6c0b2e390a95c101814c0"}, - {file = "rapidfuzz-2.15.1-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:ed2cf7c69102c7a0a06926d747ed855bc836f52e8d59a5d1e3adfd980d1bd165"}, - {file = "rapidfuzz-2.15.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a0e441d4c2025110ec3eba5d54f11f78183269a10152b3a757a739ffd1bb12bf"}, - {file = "rapidfuzz-2.15.1-cp37-cp37m-win32.whl", hash = "sha256:a4a54efe17cc9f53589c748b53f28776dfdfb9bc83619685740cb7c37985ac2f"}, - {file = "rapidfuzz-2.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:bb8318116ecac4dfb84841d8b9b461f9bb0c3be5b616418387d104f72d2a16d1"}, - {file = "rapidfuzz-2.15.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e9296c530e544f68858c3416ad1d982a1854f71e9d2d3dcedb5b216e6d54f067"}, - {file = "rapidfuzz-2.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:49c4bcdb9238f11f8c4eba1b898937f09b92280d6f900023a8216008f299b41a"}, - {file = "rapidfuzz-2.15.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ebb40a279e134bb3fef099a8b58ed5beefb201033d29bdac005bddcdb004ef71"}, - {file = "rapidfuzz-2.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a7381c11cb590bbd4e6f2d8779a0b34fdd2234dfa13d0211f6aee8ca166d9d05"}, - {file = "rapidfuzz-2.15.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cfdcdedfd12a0077193f2cf3626ff6722c5a184adf0d2d51f1ec984bf21c23c3"}, - {file = "rapidfuzz-2.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f85bece1ec59bda8b982bd719507d468d4df746dfb1988df11d916b5e9fe19e8"}, - {file = "rapidfuzz-2.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b1b393f4a1eaa6867ffac6aef58cfb04bab2b3d7d8e40b9fe2cf40dd1d384601"}, - {file = "rapidfuzz-2.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53de456ef020a77bf9d7c6c54860a48e2e902584d55d3001766140ac45c54bc7"}, - {file = "rapidfuzz-2.15.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2492330bc38b76ed967eab7bdaea63a89b6ceb254489e2c65c3824efcbf72993"}, - {file = "rapidfuzz-2.15.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:099e4c6befaa8957a816bdb67ce664871f10aaec9bebf2f61368cf7e0869a7a1"}, - {file = "rapidfuzz-2.15.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:46599b2ad4045dd3f794a24a6db1e753d23304699d4984462cf1ead02a51ddf3"}, - {file = "rapidfuzz-2.15.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:591f19d16758a3c55c9d7a0b786b40d95599a5b244d6eaef79c7a74fcf5104d8"}, - {file = "rapidfuzz-2.15.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ed17359061840eb249f8d833cb213942e8299ffc4f67251a6ed61833a9f2ea20"}, - {file = "rapidfuzz-2.15.1-cp38-cp38-win32.whl", hash = "sha256:aa1e5aad325168e29bf8e17006479b97024aa9d2fdbe12062bd2f8f09080acf8"}, - {file = "rapidfuzz-2.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:c2bb68832b140c551dbed691290bef4ee6719d4e8ce1b7226a3736f61a9d1a83"}, - {file = "rapidfuzz-2.15.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:3fac40972cf7b6c14dded88ae2331eb50dfbc278aa9195473ef6fc6bfe49f686"}, - {file = "rapidfuzz-2.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0e456cbdc0abf39352800309dab82fd3251179fa0ff6573fa117f51f4e84be8"}, - {file = "rapidfuzz-2.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:22b9d22022b9d09fd4ece15102270ab9b6a5cfea8b6f6d1965c1df7e3783f5ff"}, - {file = "rapidfuzz-2.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:46754fe404a9a6f5cbf7abe02d74af390038d94c9b8c923b3f362467606bfa28"}, - {file = "rapidfuzz-2.15.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:91abb8bf7610efe326394adc1d45e1baca8f360e74187f3fa0ef3df80cdd3ba6"}, - {file = "rapidfuzz-2.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e40a2f60024f9d3c15401e668f732800114a023f3f8d8c40f1521a62081ff054"}, - {file = "rapidfuzz-2.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a48ee83916401ac73938526d7bd804e01d2a8fe61809df7f1577b0b3b31049a3"}, - {file = "rapidfuzz-2.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c71580052f9dbac443c02f60484e5a2e5f72ad4351b84b2009fbe345b1f38422"}, - {file = "rapidfuzz-2.15.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:82b86d5b8c1b9bcbc65236d75f81023c78d06a721c3e0229889ff4ed5c858169"}, - {file = "rapidfuzz-2.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:fc4528b7736e5c30bc954022c2cf410889abc19504a023abadbc59cdf9f37cae"}, - {file = "rapidfuzz-2.15.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:e1e0e569108a5760d8f01d0f2148dd08cc9a39ead79fbefefca9e7c7723c7e88"}, - {file = "rapidfuzz-2.15.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:94e1c97f0ad45b05003806f8a13efc1fc78983e52fa2ddb00629003acf4676ef"}, - {file = "rapidfuzz-2.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47e81767a962e41477a85ad7ac937e34d19a7d2a80be65614f008a5ead671c56"}, - {file = "rapidfuzz-2.15.1-cp39-cp39-win32.whl", hash = "sha256:79fc574aaf2d7c27ec1022e29c9c18f83cdaf790c71c05779528901e0caad89b"}, - {file = "rapidfuzz-2.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:f3dd4bcef2d600e0aa121e19e6e62f6f06f22a89f82ef62755e205ce14727874"}, - {file = "rapidfuzz-2.15.1-cp39-cp39-win_arm64.whl", hash = "sha256:cac095cbdf44bc286339a77214bbca6d4d228c9ebae3da5ff6a80aaeb7c35634"}, - {file = "rapidfuzz-2.15.1-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:b89d1126be65c85763d56e3b47d75f1a9b7c5529857b4d572079b9a636eaa8a7"}, - {file = "rapidfuzz-2.15.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19b7460e91168229768be882ea365ba0ac7da43e57f9416e2cfadc396a7df3c2"}, - {file = "rapidfuzz-2.15.1-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:93c33c03e7092642c38f8a15ca2d8fc38da366f2526ec3b46adf19d5c7aa48ba"}, - {file = "rapidfuzz-2.15.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:040faca2e26d9dab5541b45ce72b3f6c0e36786234703fc2ac8c6f53bb576743"}, - {file = "rapidfuzz-2.15.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:6e2a3b23e1e9aa13474b3c710bba770d0dcc34d517d3dd6f97435a32873e3f28"}, - {file = "rapidfuzz-2.15.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:2e597b9dfd6dd180982684840975c458c50d447e46928efe3e0120e4ec6f6686"}, - {file = "rapidfuzz-2.15.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d14752c9dd2036c5f36ebe8db5f027275fa7d6b3ec6484158f83efb674bab84e"}, - {file = "rapidfuzz-2.15.1-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:558224b6fc6124d13fa32d57876f626a7d6188ba2a97cbaea33a6ee38a867e31"}, - {file = "rapidfuzz-2.15.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c89cfa88dc16fd8c9bcc0c7f0b0073f7ef1e27cceb246c9f5a3f7004fa97c4d"}, - {file = "rapidfuzz-2.15.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:509c5b631cd64df69f0f011893983eb15b8be087a55bad72f3d616b6ae6a0f96"}, - {file = "rapidfuzz-2.15.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0f73a04135a03a6e40393ecd5d46a7a1049d353fc5c24b82849830d09817991f"}, - {file = "rapidfuzz-2.15.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c99d53138a2dfe8ada67cb2855719f934af2733d726fbf73247844ce4dd6dd5"}, - {file = "rapidfuzz-2.15.1-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f01fa757f0fb332a1f045168d29b0d005de6c39ee5ce5d6c51f2563bb53c601b"}, - {file = "rapidfuzz-2.15.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60368e1add6e550faae65614844c43f8a96e37bf99404643b648bf2dba92c0fb"}, - {file = "rapidfuzz-2.15.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:785744f1270828cc632c5a3660409dee9bcaac6931a081bae57542c93e4d46c4"}, - {file = "rapidfuzz-2.15.1.tar.gz", hash = "sha256:d62137c2ca37aea90a11003ad7dc109c8f1739bfbe5a9a217f3cdb07d7ac00f6"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ac434fc71edda30d45db4a92ba5e7a42c7405e1a54cb4ec01d03cc668c6dcd40"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2a791168e119cfddf4b5a40470620c872812042f0621e6a293983a2d52372db0"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5a2f3e9df346145c2be94e4d9eeffb82fab0cbfee85bd4a06810e834fe7c03fa"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23de71e7f05518b0bbeef55d67b5dbce3bcd3e2c81e7e533051a2e9401354eb0"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d056e342989248d2bdd67f1955bb7c3b0ecfa239d8f67a8dfe6477b30872c607"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:01835d02acd5d95c1071e1da1bb27fe213c84a013b899aba96380ca9962364bc"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ed0f712e0bb5fea327e92aec8a937afd07ba8de4c529735d82e4c4124c10d5a0"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96cd19934f76a1264e8ecfed9d9f5291fde04ecb667faef5f33bdbfd95fe2d1f"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e06c4242a1354cf9d48ee01f6f4e6e19c511d50bb1e8d7d20bcadbb83a2aea90"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:d73dcfe789d37c6c8b108bf1e203e027714a239e50ad55572ced3c004424ed3b"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:06e98ff000e2619e7cfe552d086815671ed09b6899408c2c1b5103658261f6f3"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:08b6fb47dd889c69fbc0b915d782aaed43e025df6979b6b7f92084ba55edd526"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a1788ebb5f5b655a15777e654ea433d198f593230277e74d51a2a1e29a986283"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-win32.whl", hash = "sha256:c65f92881753aa1098c77818e2b04a95048f30edbe9c3094dc3707d67df4598b"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:4243a9c35667a349788461aae6471efde8d8800175b7db5148a6ab929628047f"}, + {file = "rapidfuzz-3.6.1-cp310-cp310-win_arm64.whl", hash = "sha256:f59d19078cc332dbdf3b7b210852ba1f5db8c0a2cd8cc4c0ed84cc00c76e6802"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:fbc07e2e4ac696497c5f66ec35c21ddab3fc7a406640bffed64c26ab2f7ce6d6"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:40cced1a8852652813f30fb5d4b8f9b237112a0bbaeebb0f4cc3611502556764"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:82300e5f8945d601c2daaaac139d5524d7c1fdf719aa799a9439927739917460"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edf97c321fd641fea2793abce0e48fa4f91f3c202092672f8b5b4e781960b891"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7420e801b00dee4a344ae2ee10e837d603461eb180e41d063699fb7efe08faf0"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:060bd7277dc794279fa95522af355034a29c90b42adcb7aa1da358fc839cdb11"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7e3375e4f2bfec77f907680328e4cd16cc64e137c84b1886d547ab340ba6928"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a490cd645ef9d8524090551016f05f052e416c8adb2d8b85d35c9baa9d0428ab"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2e03038bfa66d2d7cffa05d81c2f18fd6acbb25e7e3c068d52bb7469e07ff382"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:2b19795b26b979c845dba407fe79d66975d520947b74a8ab6cee1d22686f7967"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:064c1d66c40b3a0f488db1f319a6e75616b2e5fe5430a59f93a9a5e40a656d15"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:3c772d04fb0ebeece3109d91f6122b1503023086a9591a0b63d6ee7326bd73d9"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:841eafba6913c4dfd53045835545ba01a41e9644e60920c65b89c8f7e60c00a9"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-win32.whl", hash = "sha256:266dd630f12696ea7119f31d8b8e4959ef45ee2cbedae54417d71ae6f47b9848"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:d79aec8aeee02ab55d0ddb33cea3ecd7b69813a48e423c966a26d7aab025cdfe"}, + {file = "rapidfuzz-3.6.1-cp311-cp311-win_arm64.whl", hash = "sha256:484759b5dbc5559e76fefaa9170147d1254468f555fd9649aea3bad46162a88b"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b2ef4c0fd3256e357b70591ffb9e8ed1d439fb1f481ba03016e751a55261d7c1"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:588c4b20fa2fae79d60a4e438cf7133d6773915df3cc0a7f1351da19eb90f720"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7142ee354e9c06e29a2636b9bbcb592bb00600a88f02aa5e70e4f230347b373e"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1dfc557c0454ad22382373ec1b7df530b4bbd974335efe97a04caec936f2956a"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:03f73b381bdeccb331a12c3c60f1e41943931461cdb52987f2ecf46bfc22f50d"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6b0ccc2ec1781c7e5370d96aef0573dd1f97335343e4982bdb3a44c133e27786"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da3e8c9f7e64bb17faefda085ff6862ecb3ad8b79b0f618a6cf4452028aa2222"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fde9b14302a31af7bdafbf5cfbb100201ba21519be2b9dedcf4f1048e4fbe65d"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c1a23eee225dfb21c07f25c9fcf23eb055d0056b48e740fe241cbb4b22284379"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e49b9575d16c56c696bc7b06a06bf0c3d4ef01e89137b3ddd4e2ce709af9fe06"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:0a9fc714b8c290261669f22808913aad49553b686115ad0ee999d1cb3df0cd66"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:a3ee4f8f076aa92184e80308fc1a079ac356b99c39408fa422bbd00145be9854"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:f056ba42fd2f32e06b2c2ba2443594873cfccc0c90c8b6327904fc2ddf6d5799"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-win32.whl", hash = "sha256:5d82b9651e3d34b23e4e8e201ecd3477c2baa17b638979deeabbb585bcb8ba74"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-win_amd64.whl", hash = "sha256:dad55a514868dae4543ca48c4e1fc0fac704ead038dafedf8f1fc0cc263746c1"}, + {file = "rapidfuzz-3.6.1-cp312-cp312-win_arm64.whl", hash = "sha256:3c84294f4470fcabd7830795d754d808133329e0a81d62fcc2e65886164be83b"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e19d519386e9db4a5335a4b29f25b8183a1c3f78cecb4c9c3112e7f86470e37f"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:01eb03cd880a294d1bf1a583fdd00b87169b9cc9c9f52587411506658c864d73"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:be368573255f8fbb0125a78330a1a40c65e9ba3c5ad129a426ff4289099bfb41"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3e5af946f419c30f5cb98b69d40997fe8580efe78fc83c2f0f25b60d0e56efb"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f382f7ffe384ce34345e1c0b2065451267d3453cadde78946fbd99a59f0cc23c"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be156f51f3a4f369e758505ed4ae64ea88900dcb2f89d5aabb5752676d3f3d7e"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1936d134b6c513fbe934aeb668b0fee1ffd4729a3c9d8d373f3e404fbb0ce8a0"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:12ff8eaf4a9399eb2bebd838f16e2d1ded0955230283b07376d68947bbc2d33d"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:ae598a172e3a95df3383634589660d6b170cc1336fe7578115c584a99e0ba64d"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:cd4ba4c18b149da11e7f1b3584813159f189dc20833709de5f3df8b1342a9759"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:0402f1629e91a4b2e4aee68043a30191e5e1b7cd2aa8dacf50b1a1bcf6b7d3ab"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:1e12319c6b304cd4c32d5db00b7a1e36bdc66179c44c5707f6faa5a889a317c0"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:0bbfae35ce4de4c574b386c43c78a0be176eeddfdae148cb2136f4605bebab89"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-win32.whl", hash = "sha256:7fec74c234d3097612ea80f2a80c60720eec34947066d33d34dc07a3092e8105"}, + {file = "rapidfuzz-3.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:a553cc1a80d97459d587529cc43a4c7c5ecf835f572b671107692fe9eddf3e24"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:757dfd7392ec6346bd004f8826afb3bf01d18a723c97cbe9958c733ab1a51791"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2963f4a3f763870a16ee076796be31a4a0958fbae133dbc43fc55c3968564cf5"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d2f0274595cc5b2b929c80d4e71b35041104b577e118cf789b3fe0a77b37a4c5"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f211e366e026de110a4246801d43a907cd1a10948082f47e8a4e6da76fef52"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a59472b43879012b90989603aa5a6937a869a72723b1bf2ff1a0d1edee2cc8e6"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a03863714fa6936f90caa7b4b50ea59ea32bb498cc91f74dc25485b3f8fccfe9"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5dd95b6b7bfb1584f806db89e1e0c8dbb9d25a30a4683880c195cc7f197eaf0c"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7183157edf0c982c0b8592686535c8b3e107f13904b36d85219c77be5cefd0d8"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ad9d74ef7c619b5b0577e909582a1928d93e07d271af18ba43e428dc3512c2a1"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b53137d81e770c82189e07a8f32722d9e4260f13a0aec9914029206ead38cac3"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:49b9ed2472394d306d5dc967a7de48b0aab599016aa4477127b20c2ed982dbf9"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:dec307b57ec2d5054d77d03ee4f654afcd2c18aee00c48014cb70bfed79597d6"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4381023fa1ff32fd5076f5d8321249a9aa62128eb3f21d7ee6a55373e672b261"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-win32.whl", hash = "sha256:8d7a072f10ee57c8413c8ab9593086d42aaff6ee65df4aa6663eecdb7c398dca"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:ebcfb5bfd0a733514352cfc94224faad8791e576a80ffe2fd40b2177bf0e7198"}, + {file = "rapidfuzz-3.6.1-cp39-cp39-win_arm64.whl", hash = "sha256:1c47d592e447738744905c18dda47ed155620204714e6df20eb1941bb1ba315e"}, + {file = "rapidfuzz-3.6.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:eef8b346ab331bec12bbc83ac75641249e6167fab3d84d8f5ca37fd8e6c7a08c"}, + {file = "rapidfuzz-3.6.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:53251e256017e2b87f7000aee0353ba42392c442ae0bafd0f6b948593d3f68c6"}, + {file = "rapidfuzz-3.6.1-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6dede83a6b903e3ebcd7e8137e7ff46907ce9316e9d7e7f917d7e7cdc570ee05"}, + {file = "rapidfuzz-3.6.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4da90e4c2b444d0a171d7444ea10152e07e95972bb40b834a13bdd6de1110c"}, + {file = "rapidfuzz-3.6.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:ca3dfcf74f2b6962f411c33dd95b0adf3901266e770da6281bc96bb5a8b20de9"}, + {file = "rapidfuzz-3.6.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:bcc957c0a8bde8007f1a8a413a632a1a409890f31f73fe764ef4eac55f59ca87"}, + {file = "rapidfuzz-3.6.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:692c9a50bea7a8537442834f9bc6b7d29d8729a5b6379df17c31b6ab4df948c2"}, + {file = "rapidfuzz-3.6.1-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76c23ceaea27e790ddd35ef88b84cf9d721806ca366199a76fd47cfc0457a81b"}, + {file = "rapidfuzz-3.6.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2b155e67fff215c09f130555002e42f7517d0ea72cbd58050abb83cb7c880cec"}, + {file = "rapidfuzz-3.6.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:3028ee8ecc48250607fa8a0adce37b56275ec3b1acaccd84aee1f68487c8557b"}, + {file = "rapidfuzz-3.6.1.tar.gz", hash = "sha256:35660bee3ce1204872574fa041c7ad7ec5175b3053a4cb6e181463fc07013de7"}, ] [package.extras] full = ["numpy"] -[[package]] -name = "referencing" -version = "0.30.1" -description = "JSON Referencing + Python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "referencing-0.30.1-py3-none-any.whl", hash = "sha256:185d4a29f001c6e8ae4dad3861e61282a81cb01b9f0ef70a15450c45c6513a0d"}, - {file = "referencing-0.30.1.tar.gz", hash = "sha256:9370c77ceefd39510d70948bbe7375ce2d0125b9c11fd380671d4de959a8e3ce"}, -] - -[package.dependencies] -attrs = ">=22.2.0" -rpds-py = ">=0.7.0" - [[package]] name = "requests" version = "2.31.0" @@ -3259,13 +3192,13 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "requests-toolbelt" -version = "0.10.1" +version = "1.0.0" description = "A utility belt for advanced users of python-requests" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ - {file = "requests-toolbelt-0.10.1.tar.gz", hash = "sha256:62e09f7ff5ccbda92772a29f394a49c3ad6cb181d568b1337626b2abb628a63d"}, - {file = "requests_toolbelt-0.10.1-py2.py3-none-any.whl", hash = "sha256:18565aa58116d9951ac39baa288d3adb5b3ff975c4f25eee78555d89e8f247f7"}, + {file = "requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6"}, + {file = "requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06"}, ] [package.dependencies] @@ -3273,13 +3206,13 @@ requests = ">=2.0.1,<3.0.0" [[package]] name = "rich" -version = "13.5.2" +version = "13.7.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.7.0" files = [ - {file = "rich-13.5.2-py3-none-any.whl", hash = "sha256:146a90b3b6b47cac4a73c12866a499e9817426423f57c5a66949c086191a8808"}, - {file = "rich-13.5.2.tar.gz", hash = "sha256:fb9d6c0a0f643c99eed3875b5377a184132ba9be4d61516a55273d3554d75a39"}, + {file = "rich-13.7.0-py3-none-any.whl", hash = "sha256:6da14c108c4866ee9520bbffa71f6fe3962e193b7da68720583850cd4548e235"}, + {file = "rich-13.7.0.tar.gz", hash = "sha256:5cb5123b5cf9ee70584244246816e9114227e0b98ad9176eede6ad54bf5403fa"}, ] [package.dependencies] @@ -3289,112 +3222,6 @@ pygments = ">=2.13.0,<3.0.0" [package.extras] jupyter = ["ipywidgets (>=7.5.1,<9)"] -[[package]] -name = "rpds-py" -version = "0.9.2" -description = "Python bindings to Rust's persistent data structures (rpds)" -optional = false -python-versions = ">=3.8" -files = [ - {file = "rpds_py-0.9.2-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:ab6919a09c055c9b092798ce18c6c4adf49d24d4d9e43a92b257e3f2548231e7"}, - {file = "rpds_py-0.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d55777a80f78dd09410bd84ff8c95ee05519f41113b2df90a69622f5540c4f8b"}, - {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a216b26e5af0a8e265d4efd65d3bcec5fba6b26909014effe20cd302fd1138fa"}, - {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29cd8bfb2d716366a035913ced99188a79b623a3512292963d84d3e06e63b496"}, - {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:44659b1f326214950a8204a248ca6199535e73a694be8d3e0e869f820767f12f"}, - {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:745f5a43fdd7d6d25a53ab1a99979e7f8ea419dfefebcab0a5a1e9095490ee5e"}, - {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a987578ac5214f18b99d1f2a3851cba5b09f4a689818a106c23dbad0dfeb760f"}, - {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bf4151acb541b6e895354f6ff9ac06995ad9e4175cbc6d30aaed08856558201f"}, - {file = "rpds_py-0.9.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:03421628f0dc10a4119d714a17f646e2837126a25ac7a256bdf7c3943400f67f"}, - {file = "rpds_py-0.9.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:13b602dc3e8dff3063734f02dcf05111e887f301fdda74151a93dbbc249930fe"}, - {file = "rpds_py-0.9.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:fae5cb554b604b3f9e2c608241b5d8d303e410d7dfb6d397c335f983495ce7f6"}, - {file = "rpds_py-0.9.2-cp310-none-win32.whl", hash = "sha256:47c5f58a8e0c2c920cc7783113df2fc4ff12bf3a411d985012f145e9242a2764"}, - {file = "rpds_py-0.9.2-cp310-none-win_amd64.whl", hash = "sha256:4ea6b73c22d8182dff91155af018b11aac9ff7eca085750455c5990cb1cfae6e"}, - {file = "rpds_py-0.9.2-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:e564d2238512c5ef5e9d79338ab77f1cbbda6c2d541ad41b2af445fb200385e3"}, - {file = "rpds_py-0.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f411330a6376fb50e5b7a3e66894e4a39e60ca2e17dce258d53768fea06a37bd"}, - {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e7521f5af0233e89939ad626b15278c71b69dc1dfccaa7b97bd4cdf96536bb7"}, - {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8d3335c03100a073883857e91db9f2e0ef8a1cf42dc0369cbb9151c149dbbc1b"}, - {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d25b1c1096ef0447355f7293fbe9ad740f7c47ae032c2884113f8e87660d8f6e"}, - {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6a5d3fbd02efd9cf6a8ffc2f17b53a33542f6b154e88dd7b42ef4a4c0700fdad"}, - {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5934e2833afeaf36bd1eadb57256239785f5af0220ed8d21c2896ec4d3a765f"}, - {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:095b460e117685867d45548fbd8598a8d9999227e9061ee7f012d9d264e6048d"}, - {file = "rpds_py-0.9.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:91378d9f4151adc223d584489591dbb79f78814c0734a7c3bfa9c9e09978121c"}, - {file = "rpds_py-0.9.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:24a81c177379300220e907e9b864107614b144f6c2a15ed5c3450e19cf536fae"}, - {file = "rpds_py-0.9.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:de0b6eceb46141984671802d412568d22c6bacc9b230174f9e55fc72ef4f57de"}, - {file = "rpds_py-0.9.2-cp311-none-win32.whl", hash = "sha256:700375326ed641f3d9d32060a91513ad668bcb7e2cffb18415c399acb25de2ab"}, - {file = "rpds_py-0.9.2-cp311-none-win_amd64.whl", hash = "sha256:0766babfcf941db8607bdaf82569ec38107dbb03c7f0b72604a0b346b6eb3298"}, - {file = "rpds_py-0.9.2-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:b1440c291db3f98a914e1afd9d6541e8fc60b4c3aab1a9008d03da4651e67386"}, - {file = "rpds_py-0.9.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0f2996fbac8e0b77fd67102becb9229986396e051f33dbceada3debaacc7033f"}, - {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9f30d205755566a25f2ae0382944fcae2f350500ae4df4e795efa9e850821d82"}, - {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:159fba751a1e6b1c69244e23ba6c28f879a8758a3e992ed056d86d74a194a0f3"}, - {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1f044792e1adcea82468a72310c66a7f08728d72a244730d14880cd1dabe36b"}, - {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9251eb8aa82e6cf88510530b29eef4fac825a2b709baf5b94a6094894f252387"}, - {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01899794b654e616c8625b194ddd1e5b51ef5b60ed61baa7a2d9c2ad7b2a4238"}, - {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b0c43f8ae8f6be1d605b0465671124aa8d6a0e40f1fb81dcea28b7e3d87ca1e1"}, - {file = "rpds_py-0.9.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:207f57c402d1f8712618f737356e4b6f35253b6d20a324d9a47cb9f38ee43a6b"}, - {file = "rpds_py-0.9.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b52e7c5ae35b00566d244ffefba0f46bb6bec749a50412acf42b1c3f402e2c90"}, - {file = "rpds_py-0.9.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:978fa96dbb005d599ec4fd9ed301b1cc45f1a8f7982d4793faf20b404b56677d"}, - {file = "rpds_py-0.9.2-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:6aa8326a4a608e1c28da191edd7c924dff445251b94653988efb059b16577a4d"}, - {file = "rpds_py-0.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:aad51239bee6bff6823bbbdc8ad85136c6125542bbc609e035ab98ca1e32a192"}, - {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4bd4dc3602370679c2dfb818d9c97b1137d4dd412230cfecd3c66a1bf388a196"}, - {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dd9da77c6ec1f258387957b754f0df60766ac23ed698b61941ba9acccd3284d1"}, - {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:190ca6f55042ea4649ed19c9093a9be9d63cd8a97880106747d7147f88a49d18"}, - {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:876bf9ed62323bc7dcfc261dbc5572c996ef26fe6406b0ff985cbcf460fc8a4c"}, - {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa2818759aba55df50592ecbc95ebcdc99917fa7b55cc6796235b04193eb3c55"}, - {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9ea4d00850ef1e917815e59b078ecb338f6a8efda23369677c54a5825dbebb55"}, - {file = "rpds_py-0.9.2-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:5855c85eb8b8a968a74dc7fb014c9166a05e7e7a8377fb91d78512900aadd13d"}, - {file = "rpds_py-0.9.2-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:14c408e9d1a80dcb45c05a5149e5961aadb912fff42ca1dd9b68c0044904eb32"}, - {file = "rpds_py-0.9.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:65a0583c43d9f22cb2130c7b110e695fff834fd5e832a776a107197e59a1898e"}, - {file = "rpds_py-0.9.2-cp38-none-win32.whl", hash = "sha256:71f2f7715935a61fa3e4ae91d91b67e571aeb5cb5d10331ab681256bda2ad920"}, - {file = "rpds_py-0.9.2-cp38-none-win_amd64.whl", hash = "sha256:674c704605092e3ebbbd13687b09c9f78c362a4bc710343efe37a91457123044"}, - {file = "rpds_py-0.9.2-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:07e2c54bef6838fa44c48dfbc8234e8e2466d851124b551fc4e07a1cfeb37260"}, - {file = "rpds_py-0.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f7fdf55283ad38c33e35e2855565361f4bf0abd02470b8ab28d499c663bc5d7c"}, - {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:890ba852c16ace6ed9f90e8670f2c1c178d96510a21b06d2fa12d8783a905193"}, - {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:50025635ba8b629a86d9d5474e650da304cb46bbb4d18690532dd79341467846"}, - {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:517cbf6e67ae3623c5127206489d69eb2bdb27239a3c3cc559350ef52a3bbf0b"}, - {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0836d71ca19071090d524739420a61580f3f894618d10b666cf3d9a1688355b1"}, - {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c439fd54b2b9053717cca3de9583be6584b384d88d045f97d409f0ca867d80f"}, - {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f68996a3b3dc9335037f82754f9cdbe3a95db42bde571d8c3be26cc6245f2324"}, - {file = "rpds_py-0.9.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7d68dc8acded354c972116f59b5eb2e5864432948e098c19fe6994926d8e15c3"}, - {file = "rpds_py-0.9.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:f963c6b1218b96db85fc37a9f0851eaf8b9040aa46dec112611697a7023da535"}, - {file = "rpds_py-0.9.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5a46859d7f947061b4010e554ccd1791467d1b1759f2dc2ec9055fa239f1bc26"}, - {file = "rpds_py-0.9.2-cp39-none-win32.whl", hash = "sha256:e07e5dbf8a83c66783a9fe2d4566968ea8c161199680e8ad38d53e075df5f0d0"}, - {file = "rpds_py-0.9.2-cp39-none-win_amd64.whl", hash = "sha256:682726178138ea45a0766907957b60f3a1bf3acdf212436be9733f28b6c5af3c"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:196cb208825a8b9c8fc360dc0f87993b8b260038615230242bf18ec84447c08d"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:c7671d45530fcb6d5e22fd40c97e1e1e01965fc298cbda523bb640f3d923b387"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83b32f0940adec65099f3b1c215ef7f1d025d13ff947975a055989cb7fd019a4"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f67da97f5b9eac838b6980fc6da268622e91f8960e083a34533ca710bec8611"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:03975db5f103997904c37e804e5f340c8fdabbb5883f26ee50a255d664eed58c"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:987b06d1cdb28f88a42e4fb8a87f094e43f3c435ed8e486533aea0bf2e53d931"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c861a7e4aef15ff91233751619ce3a3d2b9e5877e0fcd76f9ea4f6847183aa16"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:02938432352359805b6da099c9c95c8a0547fe4b274ce8f1a91677401bb9a45f"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:ef1f08f2a924837e112cba2953e15aacfccbbfcd773b4b9b4723f8f2ddded08e"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:35da5cc5cb37c04c4ee03128ad59b8c3941a1e5cd398d78c37f716f32a9b7f67"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:141acb9d4ccc04e704e5992d35472f78c35af047fa0cfae2923835d153f091be"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:79f594919d2c1a0cc17d1988a6adaf9a2f000d2e1048f71f298b056b1018e872"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:a06418fe1155e72e16dddc68bb3780ae44cebb2912fbd8bb6ff9161de56e1798"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b2eb034c94b0b96d5eddb290b7b5198460e2d5d0c421751713953a9c4e47d10"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8b08605d248b974eb02f40bdcd1a35d3924c83a2a5e8f5d0fa5af852c4d960af"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a0805911caedfe2736935250be5008b261f10a729a303f676d3d5fea6900c96a"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ab2299e3f92aa5417d5e16bb45bb4586171c1327568f638e8453c9f8d9e0f020"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c8d7594e38cf98d8a7df25b440f684b510cf4627fe038c297a87496d10a174f"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8b9ec12ad5f0a4625db34db7e0005be2632c1013b253a4a60e8302ad4d462afd"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:1fcdee18fea97238ed17ab6478c66b2095e4ae7177e35fb71fbe561a27adf620"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-musllinux_1_2_i686.whl", hash = "sha256:933a7d5cd4b84f959aedeb84f2030f0a01d63ae6cf256629af3081cf3e3426e8"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:686ba516e02db6d6f8c279d1641f7067ebb5dc58b1d0536c4aaebb7bf01cdc5d"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:0173c0444bec0a3d7d848eaeca2d8bd32a1b43f3d3fde6617aac3731fa4be05f"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:d576c3ef8c7b2d560e301eb33891d1944d965a4d7a2eacb6332eee8a71827db6"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed89861ee8c8c47d6beb742a602f912b1bb64f598b1e2f3d758948721d44d468"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1054a08e818f8e18910f1bee731583fe8f899b0a0a5044c6e680ceea34f93876"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99e7c4bb27ff1aab90dcc3e9d37ee5af0231ed98d99cb6f5250de28889a3d502"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c545d9d14d47be716495076b659db179206e3fd997769bc01e2d550eeb685596"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9039a11bca3c41be5a58282ed81ae422fa680409022b996032a43badef2a3752"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fb39aca7a64ad0c9490adfa719dbeeb87d13be137ca189d2564e596f8ba32c07"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:2d8b3b3a2ce0eaa00c5bbbb60b6713e94e7e0becab7b3db6c5c77f979e8ed1f1"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:99b1c16f732b3a9971406fbfe18468592c5a3529585a45a35adbc1389a529a03"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:c27ee01a6c3223025f4badd533bea5e87c988cb0ba2811b690395dfe16088cfe"}, - {file = "rpds_py-0.9.2.tar.gz", hash = "sha256:8d70e8f14900f2657c249ea4def963bed86a29b81f81f5b76b5a9215680de945"}, -] - [[package]] name = "rsa" version = "4.9" @@ -3426,24 +3253,24 @@ jeepney = ">=0.6" [[package]] name = "semver" -version = "3.0.1" +version = "3.0.2" description = "Python helper for Semantic Versioning (https://semver.org)" optional = false python-versions = ">=3.7" files = [ - {file = "semver-3.0.1-py3-none-any.whl", hash = "sha256:2a23844ba1647362c7490fe3995a86e097bb590d16f0f32dfc383008f19e4cdf"}, - {file = "semver-3.0.1.tar.gz", hash = "sha256:9ec78c5447883c67b97f98c3b6212796708191d22e4ad30f4570f840171cbce1"}, + {file = "semver-3.0.2-py3-none-any.whl", hash = "sha256:b1ea4686fe70b981f85359eda33199d60c53964284e0cfb4977d243e37cf4bf4"}, + {file = "semver-3.0.2.tar.gz", hash = "sha256:6253adb39c70f6e51afed2fa7152bcd414c411286088fb4b9effb133885ab4cc"}, ] [[package]] name = "sentry-sdk" -version = "1.29.2" +version = "1.39.1" description = "Python client for Sentry (https://sentry.io)" optional = false python-versions = "*" files = [ - {file = "sentry-sdk-1.29.2.tar.gz", hash = "sha256:a99ee105384788c3f228726a88baf515fe7b5f1d2d0f215a03d194369f158df7"}, - {file = "sentry_sdk-1.29.2-py2.py3-none-any.whl", hash = "sha256:3e17215d8006612e2df02b0e73115eb8376c37e3f586d8436fa41644e605074d"}, + {file = "sentry-sdk-1.39.1.tar.gz", hash = "sha256:320a55cdf9da9097a0bead239c35b7e61f53660ef9878861824fd6d9b2eaf3b5"}, + {file = "sentry_sdk-1.39.1-py2.py3-none-any.whl", hash = "sha256:81b5b9ffdd1a374e9eb0c053b5d2012155db9cbe76393a8585677b753bd5fdc1"}, ] [package.dependencies] @@ -3453,10 +3280,12 @@ urllib3 = {version = ">=1.26.11", markers = "python_version >= \"3.6\""} [package.extras] aiohttp = ["aiohttp (>=3.5)"] arq = ["arq (>=0.23)"] +asyncpg = ["asyncpg (>=0.23)"] beam = ["apache-beam (>=2.12)"] bottle = ["bottle (>=0.12.13)"] celery = ["celery (>=3)"] chalice = ["chalice (>=1.16.0)"] +clickhouse-driver = ["clickhouse-driver (>=0.2.0)"] django = ["django (>=1.8)"] falcon = ["falcon (>=1.4)"] fastapi = ["fastapi (>=0.79.0)"] @@ -3466,6 +3295,7 @@ httpx = ["httpx (>=0.16.0)"] huey = ["huey (>=2)"] loguru = ["loguru (>=0.5)"] opentelemetry = ["opentelemetry-distro (>=0.35b0)"] +opentelemetry-experimental = ["opentelemetry-distro (>=0.40b0,<1.0)", "opentelemetry-instrumentation-aiohttp-client (>=0.40b0,<1.0)", "opentelemetry-instrumentation-django (>=0.40b0,<1.0)", "opentelemetry-instrumentation-fastapi (>=0.40b0,<1.0)", "opentelemetry-instrumentation-flask (>=0.40b0,<1.0)", "opentelemetry-instrumentation-requests (>=0.40b0,<1.0)", "opentelemetry-instrumentation-sqlite3 (>=0.40b0,<1.0)", "opentelemetry-instrumentation-urllib (>=0.40b0,<1.0)"] pure-eval = ["asttokens", "executing", "pure-eval"] pymongo = ["pymongo (>=3.1)"] pyspark = ["pyspark (>=2.4.4)"] @@ -3479,29 +3309,29 @@ tornado = ["tornado (>=5)"] [[package]] name = "setuptools" -version = "68.0.0" +version = "69.0.3" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "setuptools-68.0.0-py3-none-any.whl", hash = "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f"}, - {file = "setuptools-68.0.0.tar.gz", hash = "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235"}, + {file = "setuptools-69.0.3-py3-none-any.whl", hash = "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05"}, + {file = "setuptools-69.0.3.tar.gz", hash = "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] name = "shellingham" -version = "1.5.0.post1" +version = "1.5.4" description = "Tool to Detect Surrounding Shell" optional = false python-versions = ">=3.7" files = [ - {file = "shellingham-1.5.0.post1-py2.py3-none-any.whl", hash = "sha256:368bf8c00754fd4f55afb7bbb86e272df77e4dc76ac29dbcbb81a59e9fc15744"}, - {file = "shellingham-1.5.0.post1.tar.gz", hash = "sha256:823bc5fb5c34d60f285b624e7264f4dda254bc803a3774a147bf99c0e3004a28"}, + {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, + {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, ] [[package]] @@ -3517,19 +3347,15 @@ files = [ [[package]] name = "slack-sdk" -version = "3.21.3" +version = "3.26.2" description = "The Slack API Platform SDK for Python" optional = false -python-versions = ">=3.6.0" +python-versions = ">=3.6" files = [ - {file = "slack_sdk-3.21.3-py2.py3-none-any.whl", hash = "sha256:de3c07b92479940b61cd68c566f49fbc9974c8f38f661d26244078f3903bb9cc"}, - {file = "slack_sdk-3.21.3.tar.gz", hash = "sha256:20829bdc1a423ec93dac903470975ebf3bc76fd3fd91a4dadc0eeffc940ecb0c"}, + {file = "slack_sdk-3.26.2-py2.py3-none-any.whl", hash = "sha256:a10e8ee69ca17d274989d0c2bbecb875f19898da3052d8d57de0898a00b1ab52"}, + {file = "slack_sdk-3.26.2.tar.gz", hash = "sha256:bcdac5e688fa50e9357ecd00b803b6a8bad766aa614d35d8dc0636f40adc48bf"}, ] -[package.extras] -optional = ["SQLAlchemy (>=1.4,<3)", "aiodns (>1.0)", "aiohttp (>=3.7.3,<4)", "boto3 (<=2)", "websocket-client (>=1,<2)", "websockets (>=10,<11)"] -testing = ["Flask (>=1,<2)", "Flask-Sockets (>=0.2,<1)", "Jinja2 (==3.0.3)", "Werkzeug (<2)", "black (==22.8.0)", "boto3 (<=2)", "click (==8.0.4)", "databases (>=0.5)", "flake8 (>=5,<6)", "itsdangerous (==1.1.0)", "moto (>=3,<4)", "psutil (>=5,<6)", "pytest (>=6.2.5,<7)", "pytest-asyncio (<1)", "pytest-cov (>=2,<3)"] - [[package]] name = "sniffio" version = "1.3.0" @@ -3543,72 +3369,81 @@ files = [ [[package]] name = "soupsieve" -version = "2.4.1" +version = "2.5" description = "A modern CSS selector implementation for Beautiful Soup." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "soupsieve-2.4.1-py3-none-any.whl", hash = "sha256:1c1bfee6819544a3447586c889157365a27e10d88cde3ad3da0cf0ddf646feb8"}, - {file = "soupsieve-2.4.1.tar.gz", hash = "sha256:89d12b2d5dfcd2c9e8c22326da9d9aa9cb3dfab0a83a024f05704076ee8d35ea"}, + {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, + {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, ] [[package]] name = "sqlalchemy" -version = "2.0.19" +version = "2.0.25" description = "Database Abstraction Library" optional = false python-versions = ">=3.7" files = [ - {file = "SQLAlchemy-2.0.19-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9deaae357edc2091a9ed5d25e9ee8bba98bcfae454b3911adeaf159c2e9ca9e3"}, - {file = "SQLAlchemy-2.0.19-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0bf0fd65b50a330261ec7fe3d091dfc1c577483c96a9fa1e4323e932961aa1b5"}, - {file = "SQLAlchemy-2.0.19-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d90ccc15ba1baa345796a8fb1965223ca7ded2d235ccbef80a47b85cea2d71a"}, - {file = "SQLAlchemy-2.0.19-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb4e688f6784427e5f9479d1a13617f573de8f7d4aa713ba82813bcd16e259d1"}, - {file = "SQLAlchemy-2.0.19-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:584f66e5e1979a7a00f4935015840be627e31ca29ad13f49a6e51e97a3fb8cae"}, - {file = "SQLAlchemy-2.0.19-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2c69ce70047b801d2aba3e5ff3cba32014558966109fecab0c39d16c18510f15"}, - {file = "SQLAlchemy-2.0.19-cp310-cp310-win32.whl", hash = "sha256:96f0463573469579d32ad0c91929548d78314ef95c210a8115346271beeeaaa2"}, - {file = "SQLAlchemy-2.0.19-cp310-cp310-win_amd64.whl", hash = "sha256:22bafb1da60c24514c141a7ff852b52f9f573fb933b1e6b5263f0daa28ce6db9"}, - {file = "SQLAlchemy-2.0.19-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d6894708eeb81f6d8193e996257223b6bb4041cb05a17cd5cf373ed836ef87a2"}, - {file = "SQLAlchemy-2.0.19-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8f2afd1aafded7362b397581772c670f20ea84d0a780b93a1a1529da7c3d369"}, - {file = "SQLAlchemy-2.0.19-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b15afbf5aa76f2241184c1d3b61af1a72ba31ce4161013d7cb5c4c2fca04fd6e"}, - {file = "SQLAlchemy-2.0.19-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8fc05b59142445a4efb9c1fd75c334b431d35c304b0e33f4fa0ff1ea4890f92e"}, - {file = "SQLAlchemy-2.0.19-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5831138f0cc06b43edf5f99541c64adf0ab0d41f9a4471fd63b54ae18399e4de"}, - {file = "SQLAlchemy-2.0.19-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3afa8a21a9046917b3a12ffe016ba7ebe7a55a6fc0c7d950beb303c735c3c3ad"}, - {file = "SQLAlchemy-2.0.19-cp311-cp311-win32.whl", hash = "sha256:c896d4e6ab2eba2afa1d56be3d0b936c56d4666e789bfc59d6ae76e9fcf46145"}, - {file = "SQLAlchemy-2.0.19-cp311-cp311-win_amd64.whl", hash = "sha256:024d2f67fb3ec697555e48caeb7147cfe2c08065a4f1a52d93c3d44fc8e6ad1c"}, - {file = "SQLAlchemy-2.0.19-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:89bc2b374ebee1a02fd2eae6fd0570b5ad897ee514e0f84c5c137c942772aa0c"}, - {file = "SQLAlchemy-2.0.19-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dd4d410a76c3762511ae075d50f379ae09551d92525aa5bb307f8343bf7c2c12"}, - {file = "SQLAlchemy-2.0.19-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f469f15068cd8351826df4080ffe4cc6377c5bf7d29b5a07b0e717dddb4c7ea2"}, - {file = "SQLAlchemy-2.0.19-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:cda283700c984e699e8ef0fcc5c61f00c9d14b6f65a4f2767c97242513fcdd84"}, - {file = "SQLAlchemy-2.0.19-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:43699eb3f80920cc39a380c159ae21c8a8924fe071bccb68fc509e099420b148"}, - {file = "SQLAlchemy-2.0.19-cp37-cp37m-win32.whl", hash = "sha256:61ada5831db36d897e28eb95f0f81814525e0d7927fb51145526c4e63174920b"}, - {file = "SQLAlchemy-2.0.19-cp37-cp37m-win_amd64.whl", hash = "sha256:57d100a421d9ab4874f51285c059003292433c648df6abe6c9c904e5bd5b0828"}, - {file = "SQLAlchemy-2.0.19-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:16a310f5bc75a5b2ce7cb656d0e76eb13440b8354f927ff15cbaddd2523ee2d1"}, - {file = "SQLAlchemy-2.0.19-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:cf7b5e3856cbf1876da4e9d9715546fa26b6e0ba1a682d5ed2fc3ca4c7c3ec5b"}, - {file = "SQLAlchemy-2.0.19-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e7b69d9ced4b53310a87117824b23c509c6fc1f692aa7272d47561347e133b6"}, - {file = "SQLAlchemy-2.0.19-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f9eb4575bfa5afc4b066528302bf12083da3175f71b64a43a7c0badda2be365"}, - {file = "SQLAlchemy-2.0.19-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6b54d1ad7a162857bb7c8ef689049c7cd9eae2f38864fc096d62ae10bc100c7d"}, - {file = "SQLAlchemy-2.0.19-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5d6afc41ca0ecf373366fd8e10aee2797128d3ae45eb8467b19da4899bcd1ee0"}, - {file = "SQLAlchemy-2.0.19-cp38-cp38-win32.whl", hash = "sha256:430614f18443b58ceb9dedec323ecddc0abb2b34e79d03503b5a7579cd73a531"}, - {file = "SQLAlchemy-2.0.19-cp38-cp38-win_amd64.whl", hash = "sha256:eb60699de43ba1a1f77363f563bb2c652f7748127ba3a774f7cf2c7804aa0d3d"}, - {file = "SQLAlchemy-2.0.19-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a752b7a9aceb0ba173955d4f780c64ee15a1a991f1c52d307d6215c6c73b3a4c"}, - {file = "SQLAlchemy-2.0.19-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7351c05db355da112e056a7b731253cbeffab9dfdb3be1e895368513c7d70106"}, - {file = "SQLAlchemy-2.0.19-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fa51ce4aea583b0c6b426f4b0563d3535c1c75986c4373a0987d84d22376585b"}, - {file = "SQLAlchemy-2.0.19-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ae7473a67cd82a41decfea58c0eac581209a0aa30f8bc9190926fbf628bb17f7"}, - {file = "SQLAlchemy-2.0.19-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:851a37898a8a39783aab603c7348eb5b20d83c76a14766a43f56e6ad422d1ec8"}, - {file = "SQLAlchemy-2.0.19-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:539010665c90e60c4a1650afe4ab49ca100c74e6aef882466f1de6471d414be7"}, - {file = "SQLAlchemy-2.0.19-cp39-cp39-win32.whl", hash = "sha256:f82c310ddf97b04e1392c33cf9a70909e0ae10a7e2ddc1d64495e3abdc5d19fb"}, - {file = "SQLAlchemy-2.0.19-cp39-cp39-win_amd64.whl", hash = "sha256:8e712cfd2e07b801bc6b60fdf64853bc2bd0af33ca8fa46166a23fe11ce0dbb0"}, - {file = "SQLAlchemy-2.0.19-py3-none-any.whl", hash = "sha256:314145c1389b021a9ad5aa3a18bac6f5d939f9087d7fc5443be28cba19d2c972"}, - {file = "SQLAlchemy-2.0.19.tar.gz", hash = "sha256:77a14fa20264af73ddcdb1e2b9c5a829b8cc6b8304d0f093271980e36c200a3f"}, + {file = "SQLAlchemy-2.0.25-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4344d059265cc8b1b1be351bfb88749294b87a8b2bbe21dfbe066c4199541ebd"}, + {file = "SQLAlchemy-2.0.25-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6f9e2e59cbcc6ba1488404aad43de005d05ca56e069477b33ff74e91b6319735"}, + {file = "SQLAlchemy-2.0.25-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:84daa0a2055df9ca0f148a64fdde12ac635e30edbca80e87df9b3aaf419e144a"}, + {file = "SQLAlchemy-2.0.25-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc8b7dabe8e67c4832891a5d322cec6d44ef02f432b4588390017f5cec186a84"}, + {file = "SQLAlchemy-2.0.25-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:f5693145220517b5f42393e07a6898acdfe820e136c98663b971906120549da5"}, + {file = "SQLAlchemy-2.0.25-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:db854730a25db7c956423bb9fb4bdd1216c839a689bf9cc15fada0a7fb2f4570"}, + {file = "SQLAlchemy-2.0.25-cp310-cp310-win32.whl", hash = "sha256:14a6f68e8fc96e5e8f5647ef6cda6250c780612a573d99e4d881581432ef1669"}, + {file = "SQLAlchemy-2.0.25-cp310-cp310-win_amd64.whl", hash = "sha256:87f6e732bccd7dcf1741c00f1ecf33797383128bd1c90144ac8adc02cbb98643"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:342d365988ba88ada8af320d43df4e0b13a694dbd75951f537b2d5e4cb5cd002"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f37c0caf14b9e9b9e8f6dbc81bc56db06acb4363eba5a633167781a48ef036ed"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa9373708763ef46782d10e950b49d0235bfe58facebd76917d3f5cbf5971aed"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d24f571990c05f6b36a396218f251f3e0dda916e0c687ef6fdca5072743208f5"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:75432b5b14dc2fff43c50435e248b45c7cdadef73388e5610852b95280ffd0e9"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:884272dcd3ad97f47702965a0e902b540541890f468d24bd1d98bcfe41c3f018"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-win32.whl", hash = "sha256:e607cdd99cbf9bb80391f54446b86e16eea6ad309361942bf88318bcd452363c"}, + {file = "SQLAlchemy-2.0.25-cp311-cp311-win_amd64.whl", hash = "sha256:7d505815ac340568fd03f719446a589162d55c52f08abd77ba8964fbb7eb5b5f"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:0dacf67aee53b16f365c589ce72e766efaabd2b145f9de7c917777b575e3659d"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b801154027107461ee992ff4b5c09aa7cc6ec91ddfe50d02bca344918c3265c6"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59a21853f5daeb50412d459cfb13cb82c089ad4c04ec208cd14dddd99fc23b39"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29049e2c299b5ace92cbed0c1610a7a236f3baf4c6b66eb9547c01179f638ec5"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b64b183d610b424a160b0d4d880995e935208fc043d0302dd29fee32d1ee3f95"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4f7a7d7fcc675d3d85fbf3b3828ecd5990b8d61bd6de3f1b260080b3beccf215"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-win32.whl", hash = "sha256:cf18ff7fc9941b8fc23437cc3e68ed4ebeff3599eec6ef5eebf305f3d2e9a7c2"}, + {file = "SQLAlchemy-2.0.25-cp312-cp312-win_amd64.whl", hash = "sha256:91f7d9d1c4dd1f4f6e092874c128c11165eafcf7c963128f79e28f8445de82d5"}, + {file = "SQLAlchemy-2.0.25-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:bb209a73b8307f8fe4fe46f6ad5979649be01607f11af1eb94aa9e8a3aaf77f0"}, + {file = "SQLAlchemy-2.0.25-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:798f717ae7c806d67145f6ae94dc7c342d3222d3b9a311a784f371a4333212c7"}, + {file = "SQLAlchemy-2.0.25-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5fdd402169aa00df3142149940b3bf9ce7dde075928c1886d9a1df63d4b8de62"}, + {file = "SQLAlchemy-2.0.25-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0d3cab3076af2e4aa5693f89622bef7fa770c6fec967143e4da7508b3dceb9b9"}, + {file = "SQLAlchemy-2.0.25-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:74b080c897563f81062b74e44f5a72fa44c2b373741a9ade701d5f789a10ba23"}, + {file = "SQLAlchemy-2.0.25-cp37-cp37m-win32.whl", hash = "sha256:87d91043ea0dc65ee583026cb18e1b458d8ec5fc0a93637126b5fc0bc3ea68c4"}, + {file = "SQLAlchemy-2.0.25-cp37-cp37m-win_amd64.whl", hash = "sha256:75f99202324383d613ddd1f7455ac908dca9c2dd729ec8584c9541dd41822a2c"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:420362338681eec03f53467804541a854617faed7272fe71a1bfdb07336a381e"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7c88f0c7dcc5f99bdb34b4fd9b69b93c89f893f454f40219fe923a3a2fd11625"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3be4987e3ee9d9a380b66393b77a4cd6d742480c951a1c56a23c335caca4ce3"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2a159111a0f58fb034c93eeba211b4141137ec4b0a6e75789ab7a3ef3c7e7e3"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8b8cb63d3ea63b29074dcd29da4dc6a97ad1349151f2d2949495418fd6e48db9"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:736ea78cd06de6c21ecba7416499e7236a22374561493b456a1f7ffbe3f6cdb4"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-win32.whl", hash = "sha256:10331f129982a19df4284ceac6fe87353ca3ca6b4ca77ff7d697209ae0a5915e"}, + {file = "SQLAlchemy-2.0.25-cp38-cp38-win_amd64.whl", hash = "sha256:c55731c116806836a5d678a70c84cb13f2cedba920212ba7dcad53260997666d"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:605b6b059f4b57b277f75ace81cc5bc6335efcbcc4ccb9066695e515dbdb3900"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:665f0a3954635b5b777a55111ababf44b4fc12b1f3ba0a435b602b6387ffd7cf"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecf6d4cda1f9f6cb0b45803a01ea7f034e2f1aed9475e883410812d9f9e3cfcf"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c51db269513917394faec5e5c00d6f83829742ba62e2ac4fa5c98d58be91662f"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:790f533fa5c8901a62b6fef5811d48980adeb2f51f1290ade8b5e7ba990ba3de"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1b1180cda6df7af84fe72e4530f192231b1f29a7496951db4ff38dac1687202d"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-win32.whl", hash = "sha256:555651adbb503ac7f4cb35834c5e4ae0819aab2cd24857a123370764dc7d7e24"}, + {file = "SQLAlchemy-2.0.25-cp39-cp39-win_amd64.whl", hash = "sha256:dc55990143cbd853a5d038c05e79284baedf3e299661389654551bd02a6a68d7"}, + {file = "SQLAlchemy-2.0.25-py3-none-any.whl", hash = "sha256:a86b4240e67d4753dc3092d9511886795b3c2852abe599cffe108952f7af7ac3"}, + {file = "SQLAlchemy-2.0.25.tar.gz", hash = "sha256:a2c69a7664fb2d54b8682dd774c3b54f67f84fa123cf84dda2a5f40dcaa04e08"}, ] [package.dependencies] greenlet = {version = "!=0.4.17", markers = "platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\""} -typing-extensions = ">=4.2.0" +typing-extensions = ">=4.6.0" [package.extras] -aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] -aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing-extensions (!=3.10.0.1)"] +aiomysql = ["aiomysql (>=0.2.0)", "greenlet (!=0.4.17)"] +aioodbc = ["aioodbc", "greenlet (!=0.4.17)"] +aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] asyncio = ["greenlet (!=0.4.17)"] asyncmy = ["asyncmy (>=0.2.3,!=0.2.4,!=0.2.6)", "greenlet (!=0.4.17)"] mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5)"] @@ -3618,7 +3453,7 @@ mssql-pyodbc = ["pyodbc"] mypy = ["mypy (>=0.910)"] mysql = ["mysqlclient (>=1.4.0)"] mysql-connector = ["mysql-connector-python"] -oracle = ["cx-oracle (>=7)"] +oracle = ["cx_oracle (>=8)"] oracle-oracledb = ["oracledb (>=1.0.1)"] postgresql = ["psycopg2 (>=2.7)"] postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] @@ -3628,17 +3463,17 @@ postgresql-psycopg2binary = ["psycopg2-binary"] postgresql-psycopg2cffi = ["psycopg2cffi"] postgresql-psycopgbinary = ["psycopg[binary] (>=3.0.7)"] pymysql = ["pymysql"] -sqlcipher = ["sqlcipher3-binary"] +sqlcipher = ["sqlcipher3_binary"] [[package]] name = "starlette" -version = "0.31.0" +version = "0.34.0" description = "The little ASGI library that shines." optional = false python-versions = ">=3.8" files = [ - {file = "starlette-0.31.0-py3-none-any.whl", hash = "sha256:1aab7e04bcbafbb1867c1ce62f6b21c60a6e3cecb5a08dcee8abac7457fbcfbf"}, - {file = "starlette-0.31.0.tar.gz", hash = "sha256:7df0a3d8fa2c027d641506204ef69239d19bf9406ad2e77b319926e476ac3042"}, + {file = "starlette-0.34.0-py3-none-any.whl", hash = "sha256:2e14ee943f2df59eb8c141326240ce601643f1a97b577db44634f6d05d368c37"}, + {file = "starlette-0.34.0.tar.gz", hash = "sha256:ed050aaf3896945bfaae93bdf337e53ef3f29115a9d9c153e402985115cd9c8e"}, ] [package.dependencies] @@ -3675,13 +3510,13 @@ files = [ [[package]] name = "tomlkit" -version = "0.12.1" +version = "0.12.3" description = "Style preserving TOML library" optional = false python-versions = ">=3.7" files = [ - {file = "tomlkit-0.12.1-py3-none-any.whl", hash = "sha256:712cbd236609acc6a3e2e97253dfc52d4c2082982a88f61b640ecf0817eab899"}, - {file = "tomlkit-0.12.1.tar.gz", hash = "sha256:38e1ff8edb991273ec9f6181244a6a391ac30e9f5098e7535640ea6be97a7c86"}, + {file = "tomlkit-0.12.3-py3-none-any.whl", hash = "sha256:b0a645a9156dc7cb5d3a1f0d4bab66db287fcb8e0430bdd4664a095ea16414ba"}, + {file = "tomlkit-0.12.3.tar.gz", hash = "sha256:75baf5012d06501f07bee5bf8e801b9f343e7aac5a92581f20f80ce632e6b5a4"}, ] [[package]] @@ -3697,33 +3532,33 @@ files = [ [[package]] name = "tqdm" -version = "4.65.0" +version = "4.66.1" description = "Fast, Extensible Progress Meter" optional = false python-versions = ">=3.7" files = [ - {file = "tqdm-4.65.0-py3-none-any.whl", hash = "sha256:c4f53a17fe37e132815abceec022631be8ffe1b9381c2e6e30aa70edc99e9671"}, - {file = "tqdm-4.65.0.tar.gz", hash = "sha256:1871fb68a86b8fb3b59ca4cdd3dcccbc7e6d613eeed31f4c332531977b89beb5"}, + {file = "tqdm-4.66.1-py3-none-any.whl", hash = "sha256:d302b3c5b53d47bce91fea46679d9c3c6508cf6332229aa1e7d8653723793386"}, + {file = "tqdm-4.66.1.tar.gz", hash = "sha256:d88e651f9db8d8551a62556d3cff9e3034274ca5d66e93197cf2490e2dcb69c7"}, ] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} [package.extras] -dev = ["py-make (>=0.1.0)", "twine", "wheel"] +dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] notebook = ["ipywidgets (>=6)"] slack = ["slack-sdk"] telegram = ["requests"] [[package]] name = "trove-classifiers" -version = "2023.7.6" +version = "2024.1.8" description = "Canonical source for classifiers on PyPI (pypi.org)." optional = false python-versions = "*" files = [ - {file = "trove-classifiers-2023.7.6.tar.gz", hash = "sha256:8a8e168b51d20fed607043831d37632bb50919d1c80a64e0f1393744691a8b22"}, - {file = "trove_classifiers-2023.7.6-py3-none-any.whl", hash = "sha256:b420d5aa048ee7c456233a49203f7d58d1736af4a6cde637657d78c13ab7969b"}, + {file = "trove-classifiers-2024.1.8.tar.gz", hash = "sha256:6e36caf430ff6485c4b57a4c6b364a13f6a898d16b9417c6c37467e59c14b05a"}, + {file = "trove_classifiers-2024.1.8-py3-none-any.whl", hash = "sha256:3c1ff4deb10149c7e39ede6e5bbc107def64362ef1ee7590ec98d71fb92f1b6a"}, ] [[package]] @@ -3752,32 +3587,32 @@ test = ["black (>=22.3.0,<23.0.0)", "coverage (>=6.2,<7.0)", "isort (>=5.0.6,<6. [[package]] name = "typing-extensions" -version = "4.7.1" -description = "Backported and Experimental Type Hints for Python 3.7+" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, - {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, ] [[package]] name = "universal-pathlib" -version = "0.0.24" +version = "0.1.4" description = "pathlib api extended to use fsspec backends" optional = false python-versions = ">=3.8" files = [ - {file = "universal_pathlib-0.0.24-py3-none-any.whl", hash = "sha256:a2e907b11b1b3f6e982275e5ac0c58a4d34dba2b9e703ecbe2040afa572c741b"}, - {file = "universal_pathlib-0.0.24.tar.gz", hash = "sha256:fcbffb95e4bc69f704af5dde4f9a624b2269f251a38c81ab8bec19dfeaad830f"}, + {file = "universal_pathlib-0.1.4-py3-none-any.whl", hash = "sha256:f99186cf950bde1262de9a590bb019613ef84f9fabd9f276e8b019722201943a"}, + {file = "universal_pathlib-0.1.4.tar.gz", hash = "sha256:82e5d86d16a27e0ea1adc7d88acbcba9d02d5a45488163174f96d9ac289db2e4"}, ] [package.dependencies] -fsspec = "*" +fsspec = ">=2022.1.0" [package.extras] -dev = ["adlfs", "aiohttp", "cheroot", "gcsfs", "hadoop-test-cluster", "moto[s3,server]", "mypy (==1.3.0)", "pyarrow", "pylint (==2.17.4)", "pytest (==7.3.2)", "pytest-cov (==4.1.0)", "pytest-mock (==3.11.1)", "pytest-sugar (==0.9.6)", "requests", "s3fs", "webdav4[fsspec]", "wsgidav"] -tests = ["mypy (==1.3.0)", "pylint (==2.17.4)", "pytest (==7.3.2)", "pytest-cov (==4.1.0)", "pytest-mock (==3.11.1)", "pytest-sugar (==0.9.6)"] +dev = ["adlfs", "aiohttp", "cheroot", "gcsfs", "hadoop-test-cluster", "moto[s3,server]", "mypy (==1.3.0)", "packaging", "pyarrow", "pydantic", "pydantic-settings", "pylint (==2.17.4)", "pytest (==7.3.2)", "pytest-cov (==4.1.0)", "pytest-mock (==3.11.1)", "pytest-sugar (==0.9.6)", "requests", "s3fs", "webdav4[fsspec]", "wsgidav"] +tests = ["mypy (==1.3.0)", "packaging", "pylint (==2.17.4)", "pytest (==7.3.2)", "pytest-cov (==4.1.0)", "pytest-mock (==3.11.1)", "pytest-sugar (==0.9.6)"] [[package]] name = "uritemplate" @@ -3792,29 +3627,29 @@ files = [ [[package]] name = "urllib3" -version = "1.26.16" +version = "2.1.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +python-versions = ">=3.8" files = [ - {file = "urllib3-1.26.16-py2.py3-none-any.whl", hash = "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f"}, - {file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"}, + {file = "urllib3-2.1.0-py3-none-any.whl", hash = "sha256:55901e917a5896a349ff771be919f8bd99aff50b79fe58fec595eb37bbc56bb3"}, + {file = "urllib3-2.1.0.tar.gz", hash = "sha256:df7aa8afb0148fa78488e7899b2c59b5f4ffcfa82e6c54ccb9dd37c1d7b52d54"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] [[package]] name = "uvicorn" -version = "0.23.2" +version = "0.25.0" description = "The lightning-fast ASGI server." optional = false python-versions = ">=3.8" files = [ - {file = "uvicorn-0.23.2-py3-none-any.whl", hash = "sha256:1f9be6558f01239d4fdf22ef8126c39cb1ad0addf76c40e760549d2c2f43ab53"}, - {file = "uvicorn-0.23.2.tar.gz", hash = "sha256:4d3cc12d7727ba72b64d12d3cc7743124074c0a69f7b201512fc50c3e3f1569a"}, + {file = "uvicorn-0.25.0-py3-none-any.whl", hash = "sha256:ce107f5d9bd02b4636001a77a4e74aab5e1e2b146868ebbad565237145af444c"}, + {file = "uvicorn-0.25.0.tar.gz", hash = "sha256:6dddbad1d7ee0f5140aba5ec138ddc9612c5109399903828b4874c9937f009c2"}, ] [package.dependencies] @@ -3834,66 +3669,66 @@ standard = ["colorama (>=0.4)", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", [[package]] name = "uvloop" -version = "0.17.0" +version = "0.19.0" description = "Fast implementation of asyncio event loop on top of libuv" optional = false -python-versions = ">=3.7" -files = [ - {file = "uvloop-0.17.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ce9f61938d7155f79d3cb2ffa663147d4a76d16e08f65e2c66b77bd41b356718"}, - {file = "uvloop-0.17.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:68532f4349fd3900b839f588972b3392ee56042e440dd5873dfbbcd2cc67617c"}, - {file = "uvloop-0.17.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0949caf774b9fcefc7c5756bacbbbd3fc4c05a6b7eebc7c7ad6f825b23998d6d"}, - {file = "uvloop-0.17.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ff3d00b70ce95adce264462c930fbaecb29718ba6563db354608f37e49e09024"}, - {file = "uvloop-0.17.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:a5abddb3558d3f0a78949c750644a67be31e47936042d4f6c888dd6f3c95f4aa"}, - {file = "uvloop-0.17.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8efcadc5a0003d3a6e887ccc1fb44dec25594f117a94e3127954c05cf144d811"}, - {file = "uvloop-0.17.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3378eb62c63bf336ae2070599e49089005771cc651c8769aaad72d1bd9385a7c"}, - {file = "uvloop-0.17.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6aafa5a78b9e62493539456f8b646f85abc7093dd997f4976bb105537cf2635e"}, - {file = "uvloop-0.17.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c686a47d57ca910a2572fddfe9912819880b8765e2f01dc0dd12a9bf8573e539"}, - {file = "uvloop-0.17.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:864e1197139d651a76c81757db5eb199db8866e13acb0dfe96e6fc5d1cf45fc4"}, - {file = "uvloop-0.17.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2a6149e1defac0faf505406259561bc14b034cdf1d4711a3ddcdfbaa8d825a05"}, - {file = "uvloop-0.17.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6708f30db9117f115eadc4f125c2a10c1a50d711461699a0cbfaa45b9a78e376"}, - {file = "uvloop-0.17.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:23609ca361a7fc587031429fa25ad2ed7242941adec948f9d10c045bfecab06b"}, - {file = "uvloop-0.17.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2deae0b0fb00a6af41fe60a675cec079615b01d68beb4cc7b722424406b126a8"}, - {file = "uvloop-0.17.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45cea33b208971e87a31c17622e4b440cac231766ec11e5d22c76fab3bf9df62"}, - {file = "uvloop-0.17.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:9b09e0f0ac29eee0451d71798878eae5a4e6a91aa275e114037b27f7db72702d"}, - {file = "uvloop-0.17.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:dbbaf9da2ee98ee2531e0c780455f2841e4675ff580ecf93fe5c48fe733b5667"}, - {file = "uvloop-0.17.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a4aee22ece20958888eedbad20e4dbb03c37533e010fb824161b4f05e641f738"}, - {file = "uvloop-0.17.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:307958f9fc5c8bb01fad752d1345168c0abc5d62c1b72a4a8c6c06f042b45b20"}, - {file = "uvloop-0.17.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ebeeec6a6641d0adb2ea71dcfb76017602ee2bfd8213e3fcc18d8f699c5104f"}, - {file = "uvloop-0.17.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1436c8673c1563422213ac6907789ecb2b070f5939b9cbff9ef7113f2b531595"}, - {file = "uvloop-0.17.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8887d675a64cfc59f4ecd34382e5b4f0ef4ae1da37ed665adba0c2badf0d6578"}, - {file = "uvloop-0.17.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:3db8de10ed684995a7f34a001f15b374c230f7655ae840964d51496e2f8a8474"}, - {file = "uvloop-0.17.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7d37dccc7ae63e61f7b96ee2e19c40f153ba6ce730d8ba4d3b4e9738c1dccc1b"}, - {file = "uvloop-0.17.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:cbbe908fda687e39afd6ea2a2f14c2c3e43f2ca88e3a11964b297822358d0e6c"}, - {file = "uvloop-0.17.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3d97672dc709fa4447ab83276f344a165075fd9f366a97b712bdd3fee05efae8"}, - {file = "uvloop-0.17.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1e507c9ee39c61bfddd79714e4f85900656db1aec4d40c6de55648e85c2799c"}, - {file = "uvloop-0.17.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c092a2c1e736086d59ac8e41f9c98f26bbf9b9222a76f21af9dfe949b99b2eb9"}, - {file = "uvloop-0.17.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:30babd84706115626ea78ea5dbc7dd8d0d01a2e9f9b306d24ca4ed5796c66ded"}, - {file = "uvloop-0.17.0.tar.gz", hash = "sha256:0ddf6baf9cf11a1a22c71487f39f15b2cf78eb5bde7e5b45fbb99e8a9d91b9e1"}, +python-versions = ">=3.8.0" +files = [ + {file = "uvloop-0.19.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:de4313d7f575474c8f5a12e163f6d89c0a878bc49219641d49e6f1444369a90e"}, + {file = "uvloop-0.19.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5588bd21cf1fcf06bded085f37e43ce0e00424197e7c10e77afd4bbefffef428"}, + {file = "uvloop-0.19.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b1fd71c3843327f3bbc3237bedcdb6504fd50368ab3e04d0410e52ec293f5b8"}, + {file = "uvloop-0.19.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a05128d315e2912791de6088c34136bfcdd0c7cbc1cf85fd6fd1bb321b7c849"}, + {file = "uvloop-0.19.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:cd81bdc2b8219cb4b2556eea39d2e36bfa375a2dd021404f90a62e44efaaf957"}, + {file = "uvloop-0.19.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5f17766fb6da94135526273080f3455a112f82570b2ee5daa64d682387fe0dcd"}, + {file = "uvloop-0.19.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4ce6b0af8f2729a02a5d1575feacb2a94fc7b2e983868b009d51c9a9d2149bef"}, + {file = "uvloop-0.19.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:31e672bb38b45abc4f26e273be83b72a0d28d074d5b370fc4dcf4c4eb15417d2"}, + {file = "uvloop-0.19.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:570fc0ed613883d8d30ee40397b79207eedd2624891692471808a95069a007c1"}, + {file = "uvloop-0.19.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5138821e40b0c3e6c9478643b4660bd44372ae1e16a322b8fc07478f92684e24"}, + {file = "uvloop-0.19.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:91ab01c6cd00e39cde50173ba4ec68a1e578fee9279ba64f5221810a9e786533"}, + {file = "uvloop-0.19.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:47bf3e9312f63684efe283f7342afb414eea4d3011542155c7e625cd799c3b12"}, + {file = "uvloop-0.19.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:da8435a3bd498419ee8c13c34b89b5005130a476bda1d6ca8cfdde3de35cd650"}, + {file = "uvloop-0.19.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:02506dc23a5d90e04d4f65c7791e65cf44bd91b37f24cfc3ef6cf2aff05dc7ec"}, + {file = "uvloop-0.19.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2693049be9d36fef81741fddb3f441673ba12a34a704e7b4361efb75cf30befc"}, + {file = "uvloop-0.19.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7010271303961c6f0fe37731004335401eb9075a12680738731e9c92ddd96ad6"}, + {file = "uvloop-0.19.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:5daa304d2161d2918fa9a17d5635099a2f78ae5b5960e742b2fcfbb7aefaa593"}, + {file = "uvloop-0.19.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:7207272c9520203fea9b93843bb775d03e1cf88a80a936ce760f60bb5add92f3"}, + {file = "uvloop-0.19.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:78ab247f0b5671cc887c31d33f9b3abfb88d2614b84e4303f1a63b46c046c8bd"}, + {file = "uvloop-0.19.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:472d61143059c84947aa8bb74eabbace30d577a03a1805b77933d6bd13ddebbd"}, + {file = "uvloop-0.19.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45bf4c24c19fb8a50902ae37c5de50da81de4922af65baf760f7c0c42e1088be"}, + {file = "uvloop-0.19.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:271718e26b3e17906b28b67314c45d19106112067205119dddbd834c2b7ce797"}, + {file = "uvloop-0.19.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:34175c9fd2a4bc3adc1380e1261f60306344e3407c20a4d684fd5f3be010fa3d"}, + {file = "uvloop-0.19.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e27f100e1ff17f6feeb1f33968bc185bf8ce41ca557deee9d9bbbffeb72030b7"}, + {file = "uvloop-0.19.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:13dfdf492af0aa0a0edf66807d2b465607d11c4fa48f4a1fd41cbea5b18e8e8b"}, + {file = "uvloop-0.19.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6e3d4e85ac060e2342ff85e90d0c04157acb210b9ce508e784a944f852a40e67"}, + {file = "uvloop-0.19.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ca4956c9ab567d87d59d49fa3704cf29e37109ad348f2d5223c9bf761a332e7"}, + {file = "uvloop-0.19.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f467a5fd23b4fc43ed86342641f3936a68ded707f4627622fa3f82a120e18256"}, + {file = "uvloop-0.19.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:492e2c32c2af3f971473bc22f086513cedfc66a130756145a931a90c3958cb17"}, + {file = "uvloop-0.19.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2df95fca285a9f5bfe730e51945ffe2fa71ccbfdde3b0da5772b4ee4f2e770d5"}, + {file = "uvloop-0.19.0.tar.gz", hash = "sha256:0246f4fd1bf2bf702e06b0d45ee91677ee5c31242f39aab4ea6fe0c51aedd0fd"}, ] [package.extras] -dev = ["Cython (>=0.29.32,<0.30.0)", "Sphinx (>=4.1.2,<4.2.0)", "aiohttp", "flake8 (>=3.9.2,<3.10.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=22.0.0,<22.1.0)", "pycodestyle (>=2.7.0,<2.8.0)", "pytest (>=3.6.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"] docs = ["Sphinx (>=4.1.2,<4.2.0)", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-asyncio (>=0.3.0,<0.4.0)"] -test = ["Cython (>=0.29.32,<0.30.0)", "aiohttp", "flake8 (>=3.9.2,<3.10.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=22.0.0,<22.1.0)", "pycodestyle (>=2.7.0,<2.8.0)"] +test = ["Cython (>=0.29.36,<0.30.0)", "aiohttp (==3.9.0b0)", "aiohttp (>=3.8.1)", "flake8 (>=5.0,<6.0)", "mypy (>=0.800)", "psutil", "pyOpenSSL (>=23.0.0,<23.1.0)", "pycodestyle (>=2.9.0,<2.10.0)"] [[package]] name = "virtualenv" -version = "20.24.2" +version = "20.25.0" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.7" files = [ - {file = "virtualenv-20.24.2-py3-none-any.whl", hash = "sha256:43a3052be36080548bdee0b42919c88072037d50d56c28bd3f853cbe92b953ff"}, - {file = "virtualenv-20.24.2.tar.gz", hash = "sha256:fd8a78f46f6b99a67b7ec5cf73f92357891a7b3a40fd97637c27f854aae3b9e0"}, + {file = "virtualenv-20.25.0-py3-none-any.whl", hash = "sha256:4238949c5ffe6876362d9c0180fc6c3a824a7b12b80604eeb8085f2ed7460de3"}, + {file = "virtualenv-20.25.0.tar.gz", hash = "sha256:bf51c0d9c7dd63ea8e44086fa1e4fb1093a31e963b86959257378aef020e1f1b"}, ] [package.dependencies] distlib = ">=0.3.7,<1" filelock = ">=3.12.2,<4" -platformdirs = ">=3.9.1,<4" +platformdirs = ">=3.9.1,<5" [package.extras] -docs = ["furo (>=2023.5.20)", "proselint (>=0.13)", "sphinx (>=7.0.1)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"] [[package]] @@ -3937,33 +3772,86 @@ watchmedo = ["PyYAML (>=3.10)"] [[package]] name = "watchfiles" -version = "0.19.0" +version = "0.21.0" description = "Simple, modern and high performance file watching and code reload in python." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "watchfiles-0.19.0-cp37-abi3-macosx_10_7_x86_64.whl", hash = "sha256:91633e64712df3051ca454ca7d1b976baf842d7a3640b87622b323c55f3345e7"}, - {file = "watchfiles-0.19.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:b6577b8c6c8701ba8642ea9335a129836347894b666dd1ec2226830e263909d3"}, - {file = "watchfiles-0.19.0-cp37-abi3-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:18b28f6ad871b82df9542ff958d0c86bb0d8310bb09eb8e87d97318a3b5273af"}, - {file = "watchfiles-0.19.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fac19dc9cbc34052394dbe81e149411a62e71999c0a19e1e09ce537867f95ae0"}, - {file = "watchfiles-0.19.0-cp37-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:09ea3397aecbc81c19ed7f025e051a7387feefdb789cf768ff994c1228182fda"}, - {file = "watchfiles-0.19.0-cp37-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c0376deac92377817e4fb8f347bf559b7d44ff556d9bc6f6208dd3f79f104aaf"}, - {file = "watchfiles-0.19.0-cp37-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c75eff897786ee262c9f17a48886f4e98e6cfd335e011c591c305e5d083c056"}, - {file = "watchfiles-0.19.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cb5d45c4143c1dd60f98a16187fd123eda7248f84ef22244818c18d531a249d1"}, - {file = "watchfiles-0.19.0-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:79c533ff593db861ae23436541f481ec896ee3da4e5db8962429b441bbaae16e"}, - {file = "watchfiles-0.19.0-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:3d7d267d27aceeeaa3de0dd161a0d64f0a282264d592e335fff7958cc0cbae7c"}, - {file = "watchfiles-0.19.0-cp37-abi3-win32.whl", hash = "sha256:176a9a7641ec2c97b24455135d58012a5be5c6217fc4d5fef0b2b9f75dbf5154"}, - {file = "watchfiles-0.19.0-cp37-abi3-win_amd64.whl", hash = "sha256:945be0baa3e2440151eb3718fd8846751e8b51d8de7b884c90b17d271d34cae8"}, - {file = "watchfiles-0.19.0-cp37-abi3-win_arm64.whl", hash = "sha256:0089c6dc24d436b373c3c57657bf4f9a453b13767150d17284fc6162b2791911"}, - {file = "watchfiles-0.19.0-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:cae3dde0b4b2078f31527acff6f486e23abed307ba4d3932466ba7cdd5ecec79"}, - {file = "watchfiles-0.19.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:7f3920b1285a7d3ce898e303d84791b7bf40d57b7695ad549dc04e6a44c9f120"}, - {file = "watchfiles-0.19.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9afd0d69429172c796164fd7fe8e821ade9be983f51c659a38da3faaaaac44dc"}, - {file = "watchfiles-0.19.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68dce92b29575dda0f8d30c11742a8e2b9b8ec768ae414b54f7453f27bdf9545"}, - {file = "watchfiles-0.19.0-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:5569fc7f967429d4bc87e355cdfdcee6aabe4b620801e2cf5805ea245c06097c"}, - {file = "watchfiles-0.19.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:5471582658ea56fca122c0f0d0116a36807c63fefd6fdc92c71ca9a4491b6b48"}, - {file = "watchfiles-0.19.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b538014a87f94d92f98f34d3e6d2635478e6be6423a9ea53e4dd96210065e193"}, - {file = "watchfiles-0.19.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20b44221764955b1e703f012c74015306fb7e79a00c15370785f309b1ed9aa8d"}, - {file = "watchfiles-0.19.0.tar.gz", hash = "sha256:d9b073073e048081e502b6c6b0b88714c026a1a4c890569238d04aca5f9ca74b"}, + {file = "watchfiles-0.21.0-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:27b4035013f1ea49c6c0b42d983133b136637a527e48c132d368eb19bf1ac6aa"}, + {file = "watchfiles-0.21.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c81818595eff6e92535ff32825f31c116f867f64ff8cdf6562cd1d6b2e1e8f3e"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6c107ea3cf2bd07199d66f156e3ea756d1b84dfd43b542b2d870b77868c98c03"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d9ac347653ebd95839a7c607608703b20bc07e577e870d824fa4801bc1cb124"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5eb86c6acb498208e7663ca22dbe68ca2cf42ab5bf1c776670a50919a56e64ab"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f564bf68404144ea6b87a78a3f910cc8de216c6b12a4cf0b27718bf4ec38d303"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3d0f32ebfaa9c6011f8454994f86108c2eb9c79b8b7de00b36d558cadcedaa3d"}, + {file = "watchfiles-0.21.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6d45d9b699ecbac6c7bd8e0a2609767491540403610962968d258fd6405c17c"}, + {file = "watchfiles-0.21.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:aff06b2cac3ef4616e26ba17a9c250c1fe9dd8a5d907d0193f84c499b1b6e6a9"}, + {file = "watchfiles-0.21.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d9792dff410f266051025ecfaa927078b94cc7478954b06796a9756ccc7e14a9"}, + {file = "watchfiles-0.21.0-cp310-none-win32.whl", hash = "sha256:214cee7f9e09150d4fb42e24919a1e74d8c9b8a9306ed1474ecaddcd5479c293"}, + {file = "watchfiles-0.21.0-cp310-none-win_amd64.whl", hash = "sha256:1ad7247d79f9f55bb25ab1778fd47f32d70cf36053941f07de0b7c4e96b5d235"}, + {file = "watchfiles-0.21.0-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:668c265d90de8ae914f860d3eeb164534ba2e836811f91fecc7050416ee70aa7"}, + {file = "watchfiles-0.21.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3a23092a992e61c3a6a70f350a56db7197242f3490da9c87b500f389b2d01eef"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e7941bbcfdded9c26b0bf720cb7e6fd803d95a55d2c14b4bd1f6a2772230c586"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:11cd0c3100e2233e9c53106265da31d574355c288e15259c0d40a4405cbae317"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d78f30cbe8b2ce770160d3c08cff01b2ae9306fe66ce899b73f0409dc1846c1b"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6674b00b9756b0af620aa2a3346b01f8e2a3dc729d25617e1b89cf6af4a54eb1"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fd7ac678b92b29ba630d8c842d8ad6c555abda1b9ef044d6cc092dacbfc9719d"}, + {file = "watchfiles-0.21.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c873345680c1b87f1e09e0eaf8cf6c891b9851d8b4d3645e7efe2ec20a20cc7"}, + {file = "watchfiles-0.21.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:49f56e6ecc2503e7dbe233fa328b2be1a7797d31548e7a193237dcdf1ad0eee0"}, + {file = "watchfiles-0.21.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:02d91cbac553a3ad141db016e3350b03184deaafeba09b9d6439826ee594b365"}, + {file = "watchfiles-0.21.0-cp311-none-win32.whl", hash = "sha256:ebe684d7d26239e23d102a2bad2a358dedf18e462e8808778703427d1f584400"}, + {file = "watchfiles-0.21.0-cp311-none-win_amd64.whl", hash = "sha256:4566006aa44cb0d21b8ab53baf4b9c667a0ed23efe4aaad8c227bfba0bf15cbe"}, + {file = "watchfiles-0.21.0-cp311-none-win_arm64.whl", hash = "sha256:c550a56bf209a3d987d5a975cdf2063b3389a5d16caf29db4bdddeae49f22078"}, + {file = "watchfiles-0.21.0-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:51ddac60b96a42c15d24fbdc7a4bfcd02b5a29c047b7f8bf63d3f6f5a860949a"}, + {file = "watchfiles-0.21.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:511f0b034120cd1989932bf1e9081aa9fb00f1f949fbd2d9cab6264916ae89b1"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:cfb92d49dbb95ec7a07511bc9efb0faff8fe24ef3805662b8d6808ba8409a71a"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f92944efc564867bbf841c823c8b71bb0be75e06b8ce45c084b46411475a915"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:642d66b75eda909fd1112d35c53816d59789a4b38c141a96d62f50a3ef9b3360"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d23bcd6c8eaa6324fe109d8cac01b41fe9a54b8c498af9ce464c1aeeb99903d6"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18d5b4da8cf3e41895b34e8c37d13c9ed294954907929aacd95153508d5d89d7"}, + {file = "watchfiles-0.21.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b8d1eae0f65441963d805f766c7e9cd092f91e0c600c820c764a4ff71a0764c"}, + {file = "watchfiles-0.21.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1fd9a5205139f3c6bb60d11f6072e0552f0a20b712c85f43d42342d162be1235"}, + {file = "watchfiles-0.21.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a1e3014a625bcf107fbf38eece0e47fa0190e52e45dc6eee5a8265ddc6dc5ea7"}, + {file = "watchfiles-0.21.0-cp312-none-win32.whl", hash = "sha256:9d09869f2c5a6f2d9df50ce3064b3391d3ecb6dced708ad64467b9e4f2c9bef3"}, + {file = "watchfiles-0.21.0-cp312-none-win_amd64.whl", hash = "sha256:18722b50783b5e30a18a8a5db3006bab146d2b705c92eb9a94f78c72beb94094"}, + {file = "watchfiles-0.21.0-cp312-none-win_arm64.whl", hash = "sha256:a3b9bec9579a15fb3ca2d9878deae789df72f2b0fdaf90ad49ee389cad5edab6"}, + {file = "watchfiles-0.21.0-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:4ea10a29aa5de67de02256a28d1bf53d21322295cb00bd2d57fcd19b850ebd99"}, + {file = "watchfiles-0.21.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:40bca549fdc929b470dd1dbfcb47b3295cb46a6d2c90e50588b0a1b3bd98f429"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9b37a7ba223b2f26122c148bb8d09a9ff312afca998c48c725ff5a0a632145f7"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec8c8900dc5c83650a63dd48c4d1d245343f904c4b64b48798c67a3767d7e165"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8ad3fe0a3567c2f0f629d800409cd528cb6251da12e81a1f765e5c5345fd0137"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d353c4cfda586db2a176ce42c88f2fc31ec25e50212650c89fdd0f560ee507b"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:83a696da8922314ff2aec02987eefb03784f473281d740bf9170181829133765"}, + {file = "watchfiles-0.21.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a03651352fc20975ee2a707cd2d74a386cd303cc688f407296064ad1e6d1562"}, + {file = "watchfiles-0.21.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3ad692bc7792be8c32918c699638b660c0de078a6cbe464c46e1340dadb94c19"}, + {file = "watchfiles-0.21.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06247538e8253975bdb328e7683f8515ff5ff041f43be6c40bff62d989b7d0b0"}, + {file = "watchfiles-0.21.0-cp38-none-win32.whl", hash = "sha256:9a0aa47f94ea9a0b39dd30850b0adf2e1cd32a8b4f9c7aa443d852aacf9ca214"}, + {file = "watchfiles-0.21.0-cp38-none-win_amd64.whl", hash = "sha256:8d5f400326840934e3507701f9f7269247f7c026d1b6cfd49477d2be0933cfca"}, + {file = "watchfiles-0.21.0-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:7f762a1a85a12cc3484f77eee7be87b10f8c50b0b787bb02f4e357403cad0c0e"}, + {file = "watchfiles-0.21.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:6e9be3ef84e2bb9710f3f777accce25556f4a71e15d2b73223788d528fcc2052"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:4c48a10d17571d1275701e14a601e36959ffada3add8cdbc9e5061a6e3579a5d"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c889025f59884423428c261f212e04d438de865beda0b1e1babab85ef4c0f01"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:66fac0c238ab9a2e72d026b5fb91cb902c146202bbd29a9a1a44e8db7b710b6f"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b4a21f71885aa2744719459951819e7bf5a906a6448a6b2bbce8e9cc9f2c8128"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1c9198c989f47898b2c22201756f73249de3748e0fc9de44adaf54a8b259cc0c"}, + {file = "watchfiles-0.21.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d8f57c4461cd24fda22493109c45b3980863c58a25b8bec885ca8bea6b8d4b28"}, + {file = "watchfiles-0.21.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:853853cbf7bf9408b404754b92512ebe3e3a83587503d766d23e6bf83d092ee6"}, + {file = "watchfiles-0.21.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d5b1dc0e708fad9f92c296ab2f948af403bf201db8fb2eb4c8179db143732e49"}, + {file = "watchfiles-0.21.0-cp39-none-win32.whl", hash = "sha256:59137c0c6826bd56c710d1d2bda81553b5e6b7c84d5a676747d80caf0409ad94"}, + {file = "watchfiles-0.21.0-cp39-none-win_amd64.whl", hash = "sha256:6cb8fdc044909e2078c248986f2fc76f911f72b51ea4a4fbbf472e01d14faa58"}, + {file = "watchfiles-0.21.0-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:ab03a90b305d2588e8352168e8c5a1520b721d2d367f31e9332c4235b30b8994"}, + {file = "watchfiles-0.21.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:927c589500f9f41e370b0125c12ac9e7d3a2fd166b89e9ee2828b3dda20bfe6f"}, + {file = "watchfiles-0.21.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1bd467213195e76f838caf2c28cd65e58302d0254e636e7c0fca81efa4a2e62c"}, + {file = "watchfiles-0.21.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:02b73130687bc3f6bb79d8a170959042eb56eb3a42df3671c79b428cd73f17cc"}, + {file = "watchfiles-0.21.0-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:08dca260e85ffae975448e344834d765983237ad6dc308231aa16e7933db763e"}, + {file = "watchfiles-0.21.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:3ccceb50c611c433145502735e0370877cced72a6c70fd2410238bcbc7fe51d8"}, + {file = "watchfiles-0.21.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:57d430f5fb63fea141ab71ca9c064e80de3a20b427ca2febcbfcef70ff0ce895"}, + {file = "watchfiles-0.21.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0dd5fad9b9c0dd89904bbdea978ce89a2b692a7ee8a0ce19b940e538c88a809c"}, + {file = "watchfiles-0.21.0-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:be6dd5d52b73018b21adc1c5d28ac0c68184a64769052dfeb0c5d9998e7f56a2"}, + {file = "watchfiles-0.21.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:b3cab0e06143768499384a8a5efb9c4dc53e19382952859e4802f294214f36ec"}, + {file = "watchfiles-0.21.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c6ed10c2497e5fedadf61e465b3ca12a19f96004c15dcffe4bd442ebadc2d85"}, + {file = "watchfiles-0.21.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43babacef21c519bc6631c5fce2a61eccdfc011b4bcb9047255e9620732c8097"}, + {file = "watchfiles-0.21.0.tar.gz", hash = "sha256:c76c635fabf542bb78524905718c39f736a98e5ab25b23ec6d4abede1a85a6a3"}, ] [package.dependencies] @@ -3971,197 +3859,190 @@ anyio = ">=3.0.0" [[package]] name = "wcwidth" -version = "0.2.6" +version = "0.2.13" description = "Measures the displayed width of unicode strings in a terminal" optional = false python-versions = "*" files = [ - {file = "wcwidth-0.2.6-py2.py3-none-any.whl", hash = "sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e"}, - {file = "wcwidth-0.2.6.tar.gz", hash = "sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0"}, -] - -[[package]] -name = "webencodings" -version = "0.5.1" -description = "Character encoding aliases for legacy web content" -optional = false -python-versions = "*" -files = [ - {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"}, - {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"}, + {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, + {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, ] [[package]] name = "websockets" -version = "11.0.3" +version = "12.0" description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "websockets-11.0.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3ccc8a0c387629aec40f2fc9fdcb4b9d5431954f934da3eaf16cdc94f67dbfac"}, - {file = "websockets-11.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d67ac60a307f760c6e65dad586f556dde58e683fab03323221a4e530ead6f74d"}, - {file = "websockets-11.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:84d27a4832cc1a0ee07cdcf2b0629a8a72db73f4cf6de6f0904f6661227f256f"}, - {file = "websockets-11.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffd7dcaf744f25f82190856bc26ed81721508fc5cbf2a330751e135ff1283564"}, - {file = "websockets-11.0.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7622a89d696fc87af8e8d280d9b421db5133ef5b29d3f7a1ce9f1a7bf7fcfa11"}, - {file = "websockets-11.0.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bceab846bac555aff6427d060f2fcfff71042dba6f5fca7dc4f75cac815e57ca"}, - {file = "websockets-11.0.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:54c6e5b3d3a8936a4ab6870d46bdd6ec500ad62bde9e44462c32d18f1e9a8e54"}, - {file = "websockets-11.0.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:41f696ba95cd92dc047e46b41b26dd24518384749ed0d99bea0a941ca87404c4"}, - {file = "websockets-11.0.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:86d2a77fd490ae3ff6fae1c6ceaecad063d3cc2320b44377efdde79880e11526"}, - {file = "websockets-11.0.3-cp310-cp310-win32.whl", hash = "sha256:2d903ad4419f5b472de90cd2d40384573b25da71e33519a67797de17ef849b69"}, - {file = "websockets-11.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:1d2256283fa4b7f4c7d7d3e84dc2ece74d341bce57d5b9bf385df109c2a1a82f"}, - {file = "websockets-11.0.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e848f46a58b9fcf3d06061d17be388caf70ea5b8cc3466251963c8345e13f7eb"}, - {file = "websockets-11.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:aa5003845cdd21ac0dc6c9bf661c5beddd01116f6eb9eb3c8e272353d45b3288"}, - {file = "websockets-11.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b58cbf0697721120866820b89f93659abc31c1e876bf20d0b3d03cef14faf84d"}, - {file = "websockets-11.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:660e2d9068d2bedc0912af508f30bbeb505bbbf9774d98def45f68278cea20d3"}, - {file = "websockets-11.0.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c1f0524f203e3bd35149f12157438f406eff2e4fb30f71221c8a5eceb3617b6b"}, - {file = "websockets-11.0.3-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:def07915168ac8f7853812cc593c71185a16216e9e4fa886358a17ed0fd9fcf6"}, - {file = "websockets-11.0.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b30c6590146e53149f04e85a6e4fcae068df4289e31e4aee1fdf56a0dead8f97"}, - {file = "websockets-11.0.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:619d9f06372b3a42bc29d0cd0354c9bb9fb39c2cbc1a9c5025b4538738dbffaf"}, - {file = "websockets-11.0.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:01f5567d9cf6f502d655151645d4e8b72b453413d3819d2b6f1185abc23e82dd"}, - {file = "websockets-11.0.3-cp311-cp311-win32.whl", hash = "sha256:e1459677e5d12be8bbc7584c35b992eea142911a6236a3278b9b5ce3326f282c"}, - {file = "websockets-11.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:e7837cb169eca3b3ae94cc5787c4fed99eef74c0ab9506756eea335e0d6f3ed8"}, - {file = "websockets-11.0.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:9f59a3c656fef341a99e3d63189852be7084c0e54b75734cde571182c087b152"}, - {file = "websockets-11.0.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2529338a6ff0eb0b50c7be33dc3d0e456381157a31eefc561771ee431134a97f"}, - {file = "websockets-11.0.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:34fd59a4ac42dff6d4681d8843217137f6bc85ed29722f2f7222bd619d15e95b"}, - {file = "websockets-11.0.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:332d126167ddddec94597c2365537baf9ff62dfcc9db4266f263d455f2f031cb"}, - {file = "websockets-11.0.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6505c1b31274723ccaf5f515c1824a4ad2f0d191cec942666b3d0f3aa4cb4007"}, - {file = "websockets-11.0.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f467ba0050b7de85016b43f5a22b46383ef004c4f672148a8abf32bc999a87f0"}, - {file = "websockets-11.0.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:9d9acd80072abcc98bd2c86c3c9cd4ac2347b5a5a0cae7ed5c0ee5675f86d9af"}, - {file = "websockets-11.0.3-cp37-cp37m-win32.whl", hash = "sha256:e590228200fcfc7e9109509e4d9125eace2042fd52b595dd22bbc34bb282307f"}, - {file = "websockets-11.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:b16fff62b45eccb9c7abb18e60e7e446998093cdcb50fed33134b9b6878836de"}, - {file = "websockets-11.0.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:fb06eea71a00a7af0ae6aefbb932fb8a7df3cb390cc217d51a9ad7343de1b8d0"}, - {file = "websockets-11.0.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8a34e13a62a59c871064dfd8ffb150867e54291e46d4a7cf11d02c94a5275bae"}, - {file = "websockets-11.0.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4841ed00f1026dfbced6fca7d963c4e7043aa832648671b5138008dc5a8f6d99"}, - {file = "websockets-11.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a073fc9ab1c8aff37c99f11f1641e16da517770e31a37265d2755282a5d28aa"}, - {file = "websockets-11.0.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:68b977f21ce443d6d378dbd5ca38621755f2063d6fdb3335bda981d552cfff86"}, - {file = "websockets-11.0.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1a99a7a71631f0efe727c10edfba09ea6bee4166a6f9c19aafb6c0b5917d09c"}, - {file = "websockets-11.0.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:bee9fcb41db2a23bed96c6b6ead6489702c12334ea20a297aa095ce6d31370d0"}, - {file = "websockets-11.0.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:4b253869ea05a5a073ebfdcb5cb3b0266a57c3764cf6fe114e4cd90f4bfa5f5e"}, - {file = "websockets-11.0.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:1553cb82942b2a74dd9b15a018dce645d4e68674de2ca31ff13ebc2d9f283788"}, - {file = "websockets-11.0.3-cp38-cp38-win32.whl", hash = "sha256:f61bdb1df43dc9c131791fbc2355535f9024b9a04398d3bd0684fc16ab07df74"}, - {file = "websockets-11.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:03aae4edc0b1c68498f41a6772d80ac7c1e33c06c6ffa2ac1c27a07653e79d6f"}, - {file = "websockets-11.0.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:777354ee16f02f643a4c7f2b3eff8027a33c9861edc691a2003531f5da4f6bc8"}, - {file = "websockets-11.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8c82f11964f010053e13daafdc7154ce7385ecc538989a354ccc7067fd7028fd"}, - {file = "websockets-11.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3580dd9c1ad0701169e4d6fc41e878ffe05e6bdcaf3c412f9d559389d0c9e016"}, - {file = "websockets-11.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f1a3f10f836fab6ca6efa97bb952300b20ae56b409414ca85bff2ad241d2a61"}, - {file = "websockets-11.0.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:df41b9bc27c2c25b486bae7cf42fccdc52ff181c8c387bfd026624a491c2671b"}, - {file = "websockets-11.0.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:279e5de4671e79a9ac877427f4ac4ce93751b8823f276b681d04b2156713b9dd"}, - {file = "websockets-11.0.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1fdf26fa8a6a592f8f9235285b8affa72748dc12e964a5518c6c5e8f916716f7"}, - {file = "websockets-11.0.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:69269f3a0b472e91125b503d3c0b3566bda26da0a3261c49f0027eb6075086d1"}, - {file = "websockets-11.0.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:97b52894d948d2f6ea480171a27122d77af14ced35f62e5c892ca2fae9344311"}, - {file = "websockets-11.0.3-cp39-cp39-win32.whl", hash = "sha256:c7f3cb904cce8e1be667c7e6fef4516b98d1a6a0635a58a57528d577ac18a128"}, - {file = "websockets-11.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:c792ea4eabc0159535608fc5658a74d1a81020eb35195dd63214dcf07556f67e"}, - {file = "websockets-11.0.3-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:f2e58f2c36cc52d41f2659e4c0cbf7353e28c8c9e63e30d8c6d3494dc9fdedcf"}, - {file = "websockets-11.0.3-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de36fe9c02995c7e6ae6efe2e205816f5f00c22fd1fbf343d4d18c3d5ceac2f5"}, - {file = "websockets-11.0.3-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0ac56b661e60edd453585f4bd68eb6a29ae25b5184fd5ba51e97652580458998"}, - {file = "websockets-11.0.3-pp37-pypy37_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e052b8467dd07d4943936009f46ae5ce7b908ddcac3fda581656b1b19c083d9b"}, - {file = "websockets-11.0.3-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:42cc5452a54a8e46a032521d7365da775823e21bfba2895fb7b77633cce031bb"}, - {file = "websockets-11.0.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:e6316827e3e79b7b8e7d8e3b08f4e331af91a48e794d5d8b099928b6f0b85f20"}, - {file = "websockets-11.0.3-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8531fdcad636d82c517b26a448dcfe62f720e1922b33c81ce695d0edb91eb931"}, - {file = "websockets-11.0.3-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c114e8da9b475739dde229fd3bc6b05a6537a88a578358bc8eb29b4030fac9c9"}, - {file = "websockets-11.0.3-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e063b1865974611313a3849d43f2c3f5368093691349cf3c7c8f8f75ad7cb280"}, - {file = "websockets-11.0.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:92b2065d642bf8c0a82d59e59053dd2fdde64d4ed44efe4870fa816c1232647b"}, - {file = "websockets-11.0.3-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0ee68fe502f9031f19d495dae2c268830df2760c0524cbac5d759921ba8c8e82"}, - {file = "websockets-11.0.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dcacf2c7a6c3a84e720d1bb2b543c675bf6c40e460300b628bab1b1efc7c034c"}, - {file = "websockets-11.0.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b67c6f5e5a401fc56394f191f00f9b3811fe843ee93f4a70df3c389d1adf857d"}, - {file = "websockets-11.0.3-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d5023a4b6a5b183dc838808087033ec5df77580485fc533e7dab2567851b0a4"}, - {file = "websockets-11.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:ed058398f55163a79bb9f06a90ef9ccc063b204bb346c4de78efc5d15abfe602"}, - {file = "websockets-11.0.3-py3-none-any.whl", hash = "sha256:6681ba9e7f8f3b19440921e99efbb40fc89f26cd71bf539e45d8c8a25c976dc6"}, - {file = "websockets-11.0.3.tar.gz", hash = "sha256:88fc51d9a26b10fc331be344f1781224a375b78488fc343620184e95a4b27016"}, + {file = "websockets-12.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d554236b2a2006e0ce16315c16eaa0d628dab009c33b63ea03f41c6107958374"}, + {file = "websockets-12.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2d225bb6886591b1746b17c0573e29804619c8f755b5598d875bb4235ea639be"}, + {file = "websockets-12.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:eb809e816916a3b210bed3c82fb88eaf16e8afcf9c115ebb2bacede1797d2547"}, + {file = "websockets-12.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c588f6abc13f78a67044c6b1273a99e1cf31038ad51815b3b016ce699f0d75c2"}, + {file = "websockets-12.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5aa9348186d79a5f232115ed3fa9020eab66d6c3437d72f9d2c8ac0c6858c558"}, + {file = "websockets-12.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6350b14a40c95ddd53e775dbdbbbc59b124a5c8ecd6fbb09c2e52029f7a9f480"}, + {file = "websockets-12.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:70ec754cc2a769bcd218ed8d7209055667b30860ffecb8633a834dde27d6307c"}, + {file = "websockets-12.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6e96f5ed1b83a8ddb07909b45bd94833b0710f738115751cdaa9da1fb0cb66e8"}, + {file = "websockets-12.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4d87be612cbef86f994178d5186add3d94e9f31cc3cb499a0482b866ec477603"}, + {file = "websockets-12.0-cp310-cp310-win32.whl", hash = "sha256:befe90632d66caaf72e8b2ed4d7f02b348913813c8b0a32fae1cc5fe3730902f"}, + {file = "websockets-12.0-cp310-cp310-win_amd64.whl", hash = "sha256:363f57ca8bc8576195d0540c648aa58ac18cf85b76ad5202b9f976918f4219cf"}, + {file = "websockets-12.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5d873c7de42dea355d73f170be0f23788cf3fa9f7bed718fd2830eefedce01b4"}, + {file = "websockets-12.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3f61726cae9f65b872502ff3c1496abc93ffbe31b278455c418492016e2afc8f"}, + {file = "websockets-12.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ed2fcf7a07334c77fc8a230755c2209223a7cc44fc27597729b8ef5425aa61a3"}, + {file = "websockets-12.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e332c210b14b57904869ca9f9bf4ca32f5427a03eeb625da9b616c85a3a506c"}, + {file = "websockets-12.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5693ef74233122f8ebab026817b1b37fe25c411ecfca084b29bc7d6efc548f45"}, + {file = "websockets-12.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e9e7db18b4539a29cc5ad8c8b252738a30e2b13f033c2d6e9d0549b45841c04"}, + {file = "websockets-12.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6e2df67b8014767d0f785baa98393725739287684b9f8d8a1001eb2839031447"}, + {file = "websockets-12.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bea88d71630c5900690fcb03161ab18f8f244805c59e2e0dc4ffadae0a7ee0ca"}, + {file = "websockets-12.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dff6cdf35e31d1315790149fee351f9e52978130cef6c87c4b6c9b3baf78bc53"}, + {file = "websockets-12.0-cp311-cp311-win32.whl", hash = "sha256:3e3aa8c468af01d70332a382350ee95f6986db479ce7af14d5e81ec52aa2b402"}, + {file = "websockets-12.0-cp311-cp311-win_amd64.whl", hash = "sha256:25eb766c8ad27da0f79420b2af4b85d29914ba0edf69f547cc4f06ca6f1d403b"}, + {file = "websockets-12.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0e6e2711d5a8e6e482cacb927a49a3d432345dfe7dea8ace7b5790df5932e4df"}, + {file = "websockets-12.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:dbcf72a37f0b3316e993e13ecf32f10c0e1259c28ffd0a85cee26e8549595fbc"}, + {file = "websockets-12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12743ab88ab2af1d17dd4acb4645677cb7063ef4db93abffbf164218a5d54c6b"}, + {file = "websockets-12.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b645f491f3c48d3f8a00d1fce07445fab7347fec54a3e65f0725d730d5b99cb"}, + {file = "websockets-12.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9893d1aa45a7f8b3bc4510f6ccf8db8c3b62120917af15e3de247f0780294b92"}, + {file = "websockets-12.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f38a7b376117ef7aff996e737583172bdf535932c9ca021746573bce40165ed"}, + {file = "websockets-12.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:f764ba54e33daf20e167915edc443b6f88956f37fb606449b4a5b10ba42235a5"}, + {file = "websockets-12.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:1e4b3f8ea6a9cfa8be8484c9221ec0257508e3a1ec43c36acdefb2a9c3b00aa2"}, + {file = "websockets-12.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9fdf06fd06c32205a07e47328ab49c40fc1407cdec801d698a7c41167ea45113"}, + {file = "websockets-12.0-cp312-cp312-win32.whl", hash = "sha256:baa386875b70cbd81798fa9f71be689c1bf484f65fd6fb08d051a0ee4e79924d"}, + {file = "websockets-12.0-cp312-cp312-win_amd64.whl", hash = "sha256:ae0a5da8f35a5be197f328d4727dbcfafa53d1824fac3d96cdd3a642fe09394f"}, + {file = "websockets-12.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5f6ffe2c6598f7f7207eef9a1228b6f5c818f9f4d53ee920aacd35cec8110438"}, + {file = "websockets-12.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9edf3fc590cc2ec20dc9d7a45108b5bbaf21c0d89f9fd3fd1685e223771dc0b2"}, + {file = "websockets-12.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8572132c7be52632201a35f5e08348137f658e5ffd21f51f94572ca6c05ea81d"}, + {file = "websockets-12.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:604428d1b87edbf02b233e2c207d7d528460fa978f9e391bd8aaf9c8311de137"}, + {file = "websockets-12.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1a9d160fd080c6285e202327aba140fc9a0d910b09e423afff4ae5cbbf1c7205"}, + {file = "websockets-12.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87b4aafed34653e465eb77b7c93ef058516cb5acf3eb21e42f33928616172def"}, + {file = "websockets-12.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b2ee7288b85959797970114deae81ab41b731f19ebcd3bd499ae9ca0e3f1d2c8"}, + {file = "websockets-12.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7fa3d25e81bfe6a89718e9791128398a50dec6d57faf23770787ff441d851967"}, + {file = "websockets-12.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a571f035a47212288e3b3519944f6bf4ac7bc7553243e41eac50dd48552b6df7"}, + {file = "websockets-12.0-cp38-cp38-win32.whl", hash = "sha256:3c6cc1360c10c17463aadd29dd3af332d4a1adaa8796f6b0e9f9df1fdb0bad62"}, + {file = "websockets-12.0-cp38-cp38-win_amd64.whl", hash = "sha256:1bf386089178ea69d720f8db6199a0504a406209a0fc23e603b27b300fdd6892"}, + {file = "websockets-12.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ab3d732ad50a4fbd04a4490ef08acd0517b6ae6b77eb967251f4c263011a990d"}, + {file = "websockets-12.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a1d9697f3337a89691e3bd8dc56dea45a6f6d975f92e7d5f773bc715c15dde28"}, + {file = "websockets-12.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1df2fbd2c8a98d38a66f5238484405b8d1d16f929bb7a33ed73e4801222a6f53"}, + {file = "websockets-12.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23509452b3bc38e3a057382c2e941d5ac2e01e251acce7adc74011d7d8de434c"}, + {file = "websockets-12.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e5fc14ec6ea568200ea4ef46545073da81900a2b67b3e666f04adf53ad452ec"}, + {file = "websockets-12.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46e71dbbd12850224243f5d2aeec90f0aaa0f2dde5aeeb8fc8df21e04d99eff9"}, + {file = "websockets-12.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b81f90dcc6c85a9b7f29873beb56c94c85d6f0dac2ea8b60d995bd18bf3e2aae"}, + {file = "websockets-12.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a02413bc474feda2849c59ed2dfb2cddb4cd3d2f03a2fedec51d6e959d9b608b"}, + {file = "websockets-12.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bbe6013f9f791944ed31ca08b077e26249309639313fff132bfbf3ba105673b9"}, + {file = "websockets-12.0-cp39-cp39-win32.whl", hash = "sha256:cbe83a6bbdf207ff0541de01e11904827540aa069293696dd528a6640bd6a5f6"}, + {file = "websockets-12.0-cp39-cp39-win_amd64.whl", hash = "sha256:fc4e7fa5414512b481a2483775a8e8be7803a35b30ca805afa4998a84f9fd9e8"}, + {file = "websockets-12.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:248d8e2446e13c1d4326e0a6a4e9629cb13a11195051a73acf414812700badbd"}, + {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f44069528d45a933997a6fef143030d8ca8042f0dfaad753e2906398290e2870"}, + {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c4e37d36f0d19f0a4413d3e18c0d03d0c268ada2061868c1e6f5ab1a6d575077"}, + {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d829f975fc2e527a3ef2f9c8f25e553eb7bc779c6665e8e1d52aa22800bb38b"}, + {file = "websockets-12.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:2c71bd45a777433dd9113847af751aae36e448bc6b8c361a566cb043eda6ec30"}, + {file = "websockets-12.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0bee75f400895aef54157b36ed6d3b308fcab62e5260703add87f44cee9c82a6"}, + {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:423fc1ed29f7512fceb727e2d2aecb952c46aa34895e9ed96071821309951123"}, + {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27a5e9964ef509016759f2ef3f2c1e13f403725a5e6a1775555994966a66e931"}, + {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3181df4583c4d3994d31fb235dc681d2aaad744fbdbf94c4802485ececdecf2"}, + {file = "websockets-12.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:b067cb952ce8bf40115f6c19f478dc71c5e719b7fbaa511359795dfd9d1a6468"}, + {file = "websockets-12.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:00700340c6c7ab788f176d118775202aadea7602c5cc6be6ae127761c16d6b0b"}, + {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e469d01137942849cff40517c97a30a93ae79917752b34029f0ec72df6b46399"}, + {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffefa1374cd508d633646d51a8e9277763a9b78ae71324183693959cf94635a7"}, + {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba0cab91b3956dfa9f512147860783a1829a8d905ee218a9837c18f683239611"}, + {file = "websockets-12.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2cb388a5bfb56df4d9a406783b7f9dbefb888c09b71629351cc6b036e9259370"}, + {file = "websockets-12.0-py3-none-any.whl", hash = "sha256:dc284bbc8d7c78a6c69e0c7325ab46ee5e40bb4d50e494d8131a07ef47500e9e"}, + {file = "websockets-12.0.tar.gz", hash = "sha256:81df9cbcbb6c260de1e007e58c011bfebe2dafc8435107b0537f393dd38c8b1b"}, ] [[package]] name = "wmctrl" -version = "0.4" +version = "0.5" description = "A tool to programmatically control windows inside X" optional = false -python-versions = "*" +python-versions = ">=2.7" files = [ - {file = "wmctrl-0.4.tar.gz", hash = "sha256:66cbff72b0ca06a22ec3883ac3a4d7c41078bdae4fb7310f52951769b10e14e0"}, + {file = "wmctrl-0.5-py2.py3-none-any.whl", hash = "sha256:ae695c1863a314c899e7cf113f07c0da02a394b968c4772e1936219d9234ddd7"}, + {file = "wmctrl-0.5.tar.gz", hash = "sha256:7839a36b6fe9e2d6fd22304e5dc372dbced2116ba41283ea938b2da57f53e962"}, ] +[package.dependencies] +attrs = "*" + +[package.extras] +test = ["pytest"] + [[package]] name = "wrapt" -version = "1.15.0" +version = "1.16.0" description = "Module for decorators, wrappers and monkey patching." optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -files = [ - {file = "wrapt-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a"}, - {file = "wrapt-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923"}, - {file = "wrapt-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975"}, - {file = "wrapt-1.15.0-cp310-cp310-win32.whl", hash = "sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1"}, - {file = "wrapt-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e"}, - {file = "wrapt-1.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7"}, - {file = "wrapt-1.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98"}, - {file = "wrapt-1.15.0-cp311-cp311-win32.whl", hash = "sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416"}, - {file = "wrapt-1.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248"}, - {file = "wrapt-1.15.0-cp35-cp35m-win32.whl", hash = "sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559"}, - {file = "wrapt-1.15.0-cp35-cp35m-win_amd64.whl", hash = "sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639"}, - {file = "wrapt-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2"}, - {file = "wrapt-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1"}, - {file = "wrapt-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420"}, - {file = "wrapt-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653"}, - {file = "wrapt-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0"}, - {file = "wrapt-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e"}, - {file = "wrapt-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145"}, - {file = "wrapt-1.15.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7"}, - {file = "wrapt-1.15.0-cp38-cp38-win32.whl", hash = "sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b"}, - {file = "wrapt-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1"}, - {file = "wrapt-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86"}, - {file = "wrapt-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9"}, - {file = "wrapt-1.15.0-cp39-cp39-win32.whl", hash = "sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff"}, - {file = "wrapt-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6"}, - {file = "wrapt-1.15.0-py3-none-any.whl", hash = "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640"}, - {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, ] [[package]] @@ -4250,85 +4131,101 @@ cffi = ">=1.0" [[package]] name = "yarl" -version = "1.9.2" +version = "1.9.4" description = "Yet another URL library" optional = false python-versions = ">=3.7" files = [ - {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c2ad583743d16ddbdf6bb14b5cd76bf43b0d0006e918809d5d4ddf7bde8dd82"}, - {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:82aa6264b36c50acfb2424ad5ca537a2060ab6de158a5bd2a72a032cc75b9eb8"}, - {file = "yarl-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c0c77533b5ed4bcc38e943178ccae29b9bcf48ffd1063f5821192f23a1bd27b9"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee4afac41415d52d53a9833ebae7e32b344be72835bbb589018c9e938045a560"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9bf345c3a4f5ba7f766430f97f9cc1320786f19584acc7086491f45524a551ac"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2a96c19c52ff442a808c105901d0bdfd2e28575b3d5f82e2f5fd67e20dc5f4ea"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:891c0e3ec5ec881541f6c5113d8df0315ce5440e244a716b95f2525b7b9f3608"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c3a53ba34a636a256d767c086ceb111358876e1fb6b50dfc4d3f4951d40133d5"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:566185e8ebc0898b11f8026447eacd02e46226716229cea8db37496c8cdd26e0"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:2b0738fb871812722a0ac2154be1f049c6223b9f6f22eec352996b69775b36d4"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:32f1d071b3f362c80f1a7d322bfd7b2d11e33d2adf395cc1dd4df36c9c243095"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:e9fdc7ac0d42bc3ea78818557fab03af6181e076a2944f43c38684b4b6bed8e3"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:56ff08ab5df8429901ebdc5d15941b59f6253393cb5da07b4170beefcf1b2528"}, - {file = "yarl-1.9.2-cp310-cp310-win32.whl", hash = "sha256:8ea48e0a2f931064469bdabca50c2f578b565fc446f302a79ba6cc0ee7f384d3"}, - {file = "yarl-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:50f33040f3836e912ed16d212f6cc1efb3231a8a60526a407aeb66c1c1956dde"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:646d663eb2232d7909e6601f1a9107e66f9791f290a1b3dc7057818fe44fc2b6"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:aff634b15beff8902d1f918012fc2a42e0dbae6f469fce134c8a0dc51ca423bb"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a83503934c6273806aed765035716216cc9ab4e0364f7f066227e1aaea90b8d0"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b25322201585c69abc7b0e89e72790469f7dad90d26754717f3310bfe30331c2"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:22a94666751778629f1ec4280b08eb11815783c63f52092a5953faf73be24191"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ec53a0ea2a80c5cd1ab397925f94bff59222aa3cf9c6da938ce05c9ec20428d"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:159d81f22d7a43e6eabc36d7194cb53f2f15f498dbbfa8edc8a3239350f59fe7"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:832b7e711027c114d79dffb92576acd1bd2decc467dec60e1cac96912602d0e6"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:95d2ecefbcf4e744ea952d073c6922e72ee650ffc79028eb1e320e732898d7e8"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d4e2c6d555e77b37288eaf45b8f60f0737c9efa3452c6c44626a5455aeb250b9"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:783185c75c12a017cc345015ea359cc801c3b29a2966c2655cd12b233bf5a2be"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:b8cc1863402472f16c600e3e93d542b7e7542a540f95c30afd472e8e549fc3f7"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:822b30a0f22e588b32d3120f6d41e4ed021806418b4c9f0bc3048b8c8cb3f92a"}, - {file = "yarl-1.9.2-cp311-cp311-win32.whl", hash = "sha256:a60347f234c2212a9f0361955007fcf4033a75bf600a33c88a0a8e91af77c0e8"}, - {file = "yarl-1.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:be6b3fdec5c62f2a67cb3f8c6dbf56bbf3f61c0f046f84645cd1ca73532ea051"}, - {file = "yarl-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38a3928ae37558bc1b559f67410df446d1fbfa87318b124bf5032c31e3447b74"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac9bb4c5ce3975aeac288cfcb5061ce60e0d14d92209e780c93954076c7c4367"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3da8a678ca8b96c8606bbb8bfacd99a12ad5dd288bc6f7979baddd62f71c63ef"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13414591ff516e04fcdee8dc051c13fd3db13b673c7a4cb1350e6b2ad9639ad3"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf74d08542c3a9ea97bb8f343d4fcbd4d8f91bba5ec9d5d7f792dbe727f88938"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e7221580dc1db478464cfeef9b03b95c5852cc22894e418562997df0d074ccc"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:494053246b119b041960ddcd20fd76224149cfea8ed8777b687358727911dd33"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:52a25809fcbecfc63ac9ba0c0fb586f90837f5425edfd1ec9f3372b119585e45"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:e65610c5792870d45d7b68c677681376fcf9cc1c289f23e8e8b39c1485384185"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:1b1bba902cba32cdec51fca038fd53f8beee88b77efc373968d1ed021024cc04"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:662e6016409828ee910f5d9602a2729a8a57d74b163c89a837de3fea050c7582"}, - {file = "yarl-1.9.2-cp37-cp37m-win32.whl", hash = "sha256:f364d3480bffd3aa566e886587eaca7c8c04d74f6e8933f3f2c996b7f09bee1b"}, - {file = "yarl-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6a5883464143ab3ae9ba68daae8e7c5c95b969462bbe42e2464d60e7e2698368"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5610f80cf43b6202e2c33ba3ec2ee0a2884f8f423c8f4f62906731d876ef4fac"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b9a4e67ad7b646cd6f0938c7ebfd60e481b7410f574c560e455e938d2da8e0f4"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:83fcc480d7549ccebe9415d96d9263e2d4226798c37ebd18c930fce43dfb9574"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fcd436ea16fee7d4207c045b1e340020e58a2597301cfbcfdbe5abd2356c2fb"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84e0b1599334b1e1478db01b756e55937d4614f8654311eb26012091be109d59"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3458a24e4ea3fd8930e934c129b676c27452e4ebda80fbe47b56d8c6c7a63a9e"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:838162460b3a08987546e881a2bfa573960bb559dfa739e7800ceeec92e64417"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f4e2d08f07a3d7d3e12549052eb5ad3eab1c349c53ac51c209a0e5991bbada78"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:de119f56f3c5f0e2fb4dee508531a32b069a5f2c6e827b272d1e0ff5ac040333"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:149ddea5abf329752ea5051b61bd6c1d979e13fbf122d3a1f9f0c8be6cb6f63c"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:674ca19cbee4a82c9f54e0d1eee28116e63bc6fd1e96c43031d11cbab8b2afd5"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:9b3152f2f5677b997ae6c804b73da05a39daa6a9e85a512e0e6823d81cdad7cc"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5415d5a4b080dc9612b1b63cba008db84e908b95848369aa1da3686ae27b6d2b"}, - {file = "yarl-1.9.2-cp38-cp38-win32.whl", hash = "sha256:f7a3d8146575e08c29ed1cd287068e6d02f1c7bdff8970db96683b9591b86ee7"}, - {file = "yarl-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:63c48f6cef34e6319a74c727376e95626f84ea091f92c0250a98e53e62c77c72"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:75df5ef94c3fdc393c6b19d80e6ef1ecc9ae2f4263c09cacb178d871c02a5ba9"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c027a6e96ef77d401d8d5a5c8d6bc478e8042f1e448272e8d9752cb0aff8b5c8"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3b078dbe227f79be488ffcfc7a9edb3409d018e0952cf13f15fd6512847f3f7"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59723a029760079b7d991a401386390c4be5bfec1e7dd83e25a6a0881859e716"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b03917871bf859a81ccb180c9a2e6c1e04d2f6a51d953e6a5cdd70c93d4e5a2a"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c1012fa63eb6c032f3ce5d2171c267992ae0c00b9e164efe4d73db818465fac3"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a74dcbfe780e62f4b5a062714576f16c2f3493a0394e555ab141bf0d746bb955"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c56986609b057b4839968ba901944af91b8e92f1725d1a2d77cbac6972b9ed1"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2c315df3293cd521033533d242d15eab26583360b58f7ee5d9565f15fee1bef4"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b7232f8dfbd225d57340e441d8caf8652a6acd06b389ea2d3222b8bc89cbfca6"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:53338749febd28935d55b41bf0bcc79d634881195a39f6b2f767870b72514caf"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:066c163aec9d3d073dc9ffe5dd3ad05069bcb03fcaab8d221290ba99f9f69ee3"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8288d7cd28f8119b07dd49b7230d6b4562f9b61ee9a4ab02221060d21136be80"}, - {file = "yarl-1.9.2-cp39-cp39-win32.whl", hash = "sha256:b124e2a6d223b65ba8768d5706d103280914d61f5cae3afbc50fc3dfcc016623"}, - {file = "yarl-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:61016e7d582bc46a5378ffdd02cd0314fb8ba52f40f9cf4d9a5e7dbef88dee18"}, - {file = "yarl-1.9.2.tar.gz", hash = "sha256:04ab9d4b9f587c06d801c2abfe9317b77cdf996c65a90d5e84ecc45010823571"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541"}, + {file = "yarl-1.9.4-cp310-cp310-win32.whl", hash = "sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d"}, + {file = "yarl-1.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98"}, + {file = "yarl-1.9.4-cp311-cp311-win32.whl", hash = "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31"}, + {file = "yarl-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10"}, + {file = "yarl-1.9.4-cp312-cp312-win32.whl", hash = "sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7"}, + {file = "yarl-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984"}, + {file = "yarl-1.9.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434"}, + {file = "yarl-1.9.4-cp37-cp37m-win32.whl", hash = "sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749"}, + {file = "yarl-1.9.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3"}, + {file = "yarl-1.9.4-cp38-cp38-win32.whl", hash = "sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece"}, + {file = "yarl-1.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0"}, + {file = "yarl-1.9.4-cp39-cp39-win32.whl", hash = "sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575"}, + {file = "yarl-1.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15"}, + {file = "yarl-1.9.4-py3-none-any.whl", hash = "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad"}, + {file = "yarl-1.9.4.tar.gz", hash = "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf"}, ] [package.dependencies] @@ -4337,20 +4234,20 @@ multidict = ">=4.0" [[package]] name = "zipp" -version = "3.16.2" +version = "3.17.0" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" files = [ - {file = "zipp-3.16.2-py3-none-any.whl", hash = "sha256:679e51dd4403591b2d6838a48de3d283f3d188412a9782faadf845f298736ba0"}, - {file = "zipp-3.16.2.tar.gz", hash = "sha256:ebc15946aa78bd63458992fc81ec3b6f7b1e92d51c35e6de1c3804e73b799147"}, + {file = "zipp-3.17.0-py3-none-any.whl", hash = "sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31"}, + {file = "zipp-3.17.0.tar.gz", hash = "sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy (>=0.9.1)", "pytest-ruff"] [metadata] lock-version = "2.0" -python-versions = "^3.9" -content-hash = "8c6fa8dc9750af9e32ac39bfb45a960721098d735bd81f5baf8134921127f16d" +python-versions = "^3.9, <3.13" +content-hash = "bb5bbfdca5cf2dd2c8040275e5ae8ff9ec78719f2aad3bdddb0f652b9f2bd893" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/pyproject.toml b/airbyte-ci/connectors/metadata_service/orchestrator/pyproject.toml index 36eb30b42eff..755dffa05469 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/pyproject.toml +++ b/airbyte-ci/connectors/metadata_service/orchestrator/pyproject.toml @@ -7,11 +7,11 @@ readme = "README.md" packages = [{include = "orchestrator"}] [tool.poetry.dependencies] -python = "^3.9" # This is set to 3.9 as currently there is an issue when deploying via dagster-cloud where a dependency does not have a prebuild wheel file for 3.10 -dagit = "^1.4.1" -dagster = "^1.4.1" +python = "^3.9, <3.13" # This is set to 3.9 as currently there is an issue when deploying via dagster-cloud where a dependency does not have a prebuild wheel file for 3.10 +dagit = "^1.5.14" +dagster = "^1.5.14" pandas = "^1.5.3" -dagster-gcp = "^0.20.2" +dagster-gcp = "^0.21.14" google = "^3.0.0" jinja2 = "^3.1.2" pygithub = "^1.58.0" @@ -20,16 +20,17 @@ deepdiff = "^6.3.0" mergedeep = "^1.3.4" pydash = "^6.0.2" dpath = "^2.1.5" -dagster-cloud = "^1.2.6" +dagster-cloud = "^1.5.14" grpcio = "^1.47.0" poetry2setup = "^1.1.0" poetry = "^1.5.1" -pydantic = "^1.10.6" -dagster-slack = "^0.20.2" +pydantic = "^1.10.8" +dagster-slack = "^0.21.14" sentry-sdk = "^1.28.1" semver = "^3.0.1" python-dateutil = "^2.8.2" humanize = "^4.7.0" +pendulum = "<3.0.0" [tool.poetry.group.dev.dependencies] diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/__init__.py b/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/__init__.py index 57104ac0b2e6..9ca33c3955c2 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/__init__.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/__init__.py @@ -1,3 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + import json import os diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/cloud_registry.json b/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/cloud_registry.json index bf8ccafeeeb5..dc8795894588 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/cloud_registry.json +++ b/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/cloud_registry.json @@ -6493,7 +6493,7 @@ "lsn_commit_behaviour": { "type": "string", "title": "LSN commit behaviour", - "description": "Determines when Airbtye should flush the LSN of processed WAL logs in the source database. `After loading Data in the destination` is default. If `While reading Data` is selected, in case of a downstream failure (while loading data into the destination), next sync would result in a full sync.", + "description": "Determines when Airbyte should flush the LSN of processed WAL logs in the source database. `After loading Data in the destination` is default. If `While reading Data` is selected, in case of a downstream failure (while loading data into the destination), next sync would result in a full sync.", "enum": [ "While reading Data", "After loading Data in the destination" @@ -15842,7 +15842,7 @@ "lsn_commit_behaviour": { "type": "string", "title": "LSN commit behaviour", - "description": "Determines when Airbtye should flush the LSN of processed WAL logs in the source database. `After loading Data in the destination` is default. If `While reading Data` is selected, in case of a downstream failure (while loading data into the destination), next sync would result in a full sync.", + "description": "Determines when Airbyte should flush the LSN of processed WAL logs in the source database. `After loading Data in the destination` is default. If `While reading Data` is selected, in case of a downstream failure (while loading data into the destination), next sync would result in a full sync.", "enum": [ "While reading Data", "After loading Data in the destination" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/oss_registry.json b/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/oss_registry.json index 13cb1ec84560..a1271847cddb 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/oss_registry.json +++ b/airbyte-ci/connectors/metadata_service/orchestrator/tests/fixtures/oss_registry.json @@ -9252,7 +9252,7 @@ "lsn_commit_behaviour": { "type": "string", "title": "LSN commit behaviour", - "description": "Determines when Airbtye should flush the LSN of processed WAL logs in the source database. `After loading Data in the destination` is default. If `While reading Data` is selected, in case of a downstream failure (while loading data into the destination), next sync would result in a full sync.", + "description": "Determines when Airbyte should flush the LSN of processed WAL logs in the source database. `After loading Data in the destination` is default. If `While reading Data` is selected, in case of a downstream failure (while loading data into the destination), next sync would result in a full sync.", "enum": [ "While reading Data", "After loading Data in the destination" @@ -22995,7 +22995,7 @@ "lsn_commit_behaviour": { "type": "string", "title": "LSN commit behaviour", - "description": "Determines when Airbtye should flush the LSN of processed WAL logs in the source database. `After loading Data in the destination` is default. If `While reading Data` is selected, in case of a downstream failure (while loading data into the destination), next sync would result in a full sync.", + "description": "Determines when Airbyte should flush the LSN of processed WAL logs in the source database. `After loading Data in the destination` is default. If `While reading Data` is selected, in case of a downstream failure (while loading data into the destination), next sync would result in a full sync.", "enum": [ "While reading Data", "After loading Data in the destination" diff --git a/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_registry.py b/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_registry.py index b5ed2d349d86..151e49b4ab8a 100644 --- a/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_registry.py +++ b/airbyte-ci/connectors/metadata_service/orchestrator/tests/test_registry.py @@ -13,6 +13,7 @@ from metadata_service.models.generated.ConnectorRegistryV0 import ConnectorRegistryV0 from orchestrator.assets.registry_entry import ( get_connector_type_from_registry_entry, + get_registry_entry_write_path, get_registry_status_lists, metadata_to_registry_entry, safe_parse_metadata_definition, @@ -277,11 +278,15 @@ def test_overrides_application(registry_type, expected_docker_image_tag, expecte mock_metadata_entry = mock.Mock() mock_metadata_entry.metadata_definition.dict.return_value = metadata + mock_metadata_entry.file_path = f"metadata/{expected_docker_image_tag}/metadata.yaml" mock_metadata_entry.icon_url = "test-icon-url" - result = metadata_to_registry_entry(mock_metadata_entry, registry_type) - assert result["dockerImageTag"] == expected_docker_image_tag - assert result["additionalField"] == expected_additional_field + registry_entry = metadata_to_registry_entry(mock_metadata_entry, registry_type) + assert registry_entry["dockerImageTag"] == expected_docker_image_tag + assert registry_entry["additionalField"] == expected_additional_field + + expected_write_path = f"metadata/{expected_docker_image_tag}/{registry_type}" + assert get_registry_entry_write_path(registry_entry, mock_metadata_entry, registry_type) == expected_write_path def test_source_type_extraction(): diff --git a/airbyte-ci/connectors/pipelines/.gitignore b/airbyte-ci/connectors/pipelines/.gitignore index a93f5bbc51ac..d17bbbefa193 100644 --- a/airbyte-ci/connectors/pipelines/.gitignore +++ b/airbyte-ci/connectors/pipelines/.gitignore @@ -1 +1,2 @@ -pipeline_reports \ No newline at end of file +pipeline_reports +.venv diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index fc8ffec73cef..c979b0e8c0f9 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -1,6 +1,7 @@ # Airbyte CI CLI ## What is it? + `airbyte-ci` is a command line interface to run CI/CD pipelines. The goal of this CLI is to offer developers a tool to run these pipelines locally and in a CI context with the same guarantee. It can prevent unnecessary commit -> push cycles developers typically go through when they when to test their changes against a remote CI. @@ -9,45 +10,69 @@ Our pipeline are declared with Python code, the main entrypoint is [here](https: This documentation should be helpful for both local and CI use of the CLI. We indeed [power connector testing in the CI with this CLI](https://github.com/airbytehq/airbyte/blob/master/.github/workflows/connector_integration_test_single_dagger.yml#L78). ## How to install + ### Requirements -* A running Docker engine -* Python >= 3.10 -* [pipx](https://pypa.github.io/pipx/installation/) -## Requirements +- A running Docker engine with version >= 20.10.23 + +## Install or Update -This project requires Python 3.10 and pipx. +The recommended way to install `airbyte-ci` is using the [Makefile](../../../Makefile). -## General Installation +```sh +# from the root of the airbyte repository +make tools.airbyte-ci.install +``` -The recommended way to install `airbyte-ci` is using pipx. This ensures the tool and its dependencies are isolated from your other Python projects. +### Setting up connector secrets access -If you haven't installed pipx, you can do it with pip: +If you plan to use Airbyte CI to run CAT (Connector Acceptance Tests), we recommend setting up GSM +access so that Airbyte CI can pull remote secrets from GSM. For setup instructions, see the +CI Credentials package (which Airbyte CI uses under the hood) README's +[Get GSM Access](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/ci_credentials/README.md#get-gsm-access) +instructions. -```bash -python -m pip install --user pipx -python -m pipx ensurepath +### Updating the airbyte-ci tool + +To reinstall airbyte-ci, run the following command: + +```sh +airbyte-ci update ``` -Once pipx is installed, navigate to the root directory of the project, then run: +or if that fails, you can reinstall it with the following command: -```bash -pipx install airbyte-ci/connectors/pipelines/ +```sh +# from the root of the airbyte repository +make tools.airbyte-ci.install +``` + +## Checking the airbyte-ci install + +To check that airbyte-ci is installed correctly, run the following command: + +```sh +make tools.airbyte-ci.check ``` -This command installs `airbyte-ci` and makes it globally available in your terminal. +## Cleaning the airbyte-ci install -If you face any installation problem feel free to reach out the Airbyte Connectors Operations team. +To clean the airbyte-ci install, run the following command: + +```sh +make tools.airbyte-ci.clean +``` ## Installation for development #### Pre-requisites -* Poetry >= 1.1.8 -* Python >= 3.10 + +- Poetry >= 1.1.8 +- Python >= 3.10 #### Installation -If you are developing on pipelines, we recommend installing airbyte-ci in editable mode: +If you are developing on pipelines, we recommend installing airbyte-ci with poetry: ```bash cd airbyte-ci/connectors/pipelines/ @@ -56,80 +81,110 @@ poetry shell cd ../../ ``` -At this point you can run `airbyte-ci` commands from the root of the repository. +**Alternatively**, you can install airbyte-ci with pipx so that the entrypoint is available in your PATH: + +```bash +make tools.airbyte-ci.install +``` + +However, this will not automatically install the dependencies for the local dependencies of airbyte-ci, or respect the lockfile. + +Its often best to use the `poetry` steps instead. ## Commands reference + +At this point you can run `airbyte-ci` commands. + - [`airbyte-ci` command group](#airbyte-ci) - * [Options](#options) + - [Options](#options) - [`connectors` command subgroup](#connectors-command-subgroup) - * [Options](#options-1) + - [Options](#options-1) - [`connectors list` command](#connectors-list-command) -- [`connectors format` command](#connectors-format-command) - [`connectors test` command](#connectors-test-command) - * [Examples](#examples-) - * [What it runs](#what-it-runs-) + - [Examples](#examples-) + - [What it runs](#what-it-runs-) - [`connectors build` command](#connectors-build-command) - * [What it runs](#what-it-runs) + - [What it runs](#what-it-runs) - [`connectors publish` command](#connectors-publish-command) - [Examples](#examples) - [Options](#options-2) - * [What it runs](#what-it-runs-1) +- [`connectors bump_version` command](#connectors-bump_version) +- [`connectors upgrade_cdk` command](#connectors-upgrade_cdk) +- [`connectors upgrade_base_image` command](#connectors-upgrade_base_image) +- [`connectors migrate_to_base_image` command](#connectors-migrate_to_base_image) +- [`format` command subgroup](#format-subgroup) + - [`format check` command](#format-check-command) + - [`format fix` command](#format-fix-command) - [`metadata` command subgroup](#metadata-command-subgroup) - [`metadata validate` command](#metadata-validate-command) - * [Example](#example) - * [Options](#options-3) + - [Example](#example) + - [Options](#options-3) - [`metadata upload` command](#metadata-upload-command) - * [Example](#example-1) - * [Options](#options-4) + - [Example](#example-1) + - [Options](#options-4) - [`metadata deploy orchestrator` command](#metadata-deploy-orchestrator-command) - * [Example](#example-2) - * [What it runs](#what-it-runs--1) + - [Example](#example-2) + - [What it runs](#what-it-runs--1) - [`metadata test lib` command](#metadata-test-lib-command) - * [Example](#example-3) + - [Example](#example-3) - [`metadata test orchestrator` command](#metadata-test-orchestrator-command) - * [Example](#example-4) + - [Example](#example-4) - [`tests` command](#test-command) - * [Example](#example-5) + - [Example](#example-5) + ### `airbyte-ci` command group + **The main command group option has sensible defaults. In local use cases you're not likely to pass options to the `airbyte-ci` command group.** #### Options -| Option | Default value | Mapped environment variable | Description | -| --------------------------------------- | ------------------------------- | ----------------------------- | ------------------------------------------------------------------------------------------- | -| `--no-tui` | | | Disables the Dagger terminal UI. | -| `--is-local/--is-ci` | `--is-local` | | Determines the environment in which the CLI runs: local environment or CI environment. | -| `--git-branch` | The checked out git branch name | `CI_GIT_BRANCH` | The git branch on which the pipelines will run. | -| `--git-revision` | The current branch head | `CI_GIT_REVISION` | The commit hash on which the pipelines will run. | -| `--diffed-branch` | `origin/master` | | Branch to which the git diff will happen to detect new or modified files. | -| `--gha-workflow-run-id` | | | GHA CI only - The run id of the GitHub action workflow | -| `--ci-context` | `manual` | | The current CI context: `manual` for manual run, `pull_request`, `nightly_builds`, `master` | -| `--pipeline-start-timestamp` | Current epoch time | `CI_PIPELINE_START_TIMESTAMP` | Start time of the pipeline as epoch time. Used for pipeline run duration computation. | -| `--show-dagger-logs/--hide-dagger-logs` | `--hide-dagger-logs` | | Flag to show or hide the dagger logs. | +| Option | Default value | Mapped environment variable | Description | +| ---------------------------------------------- | ------------------------------- | ----------------------------- | ------------------------------------------------------------------------------------------- | +| `--yes/--y` | False | | Agrees to all prompts. | +| `--yes-auto-update` | False | | Agrees to the auto update prompts. | +| `--enable-update-check/--disable-update-check` | True | | Turns on the update check feature | +| `--enable-dagger-run/--disable-dagger-run` | `--enable-dagger-run` | | Disables the Dagger terminal UI. | +| `--is-local/--is-ci` | `--is-local` | | Determines the environment in which the CLI runs: local environment or CI environment. | +| `--git-branch` | The checked out git branch name | `CI_GIT_BRANCH` | The git branch on which the pipelines will run. | +| `--git-revision` | The current branch head | `CI_GIT_REVISION` | The commit hash on which the pipelines will run. | +| `--diffed-branch` | `origin/master` | | Branch to which the git diff will happen to detect new or modified files. | +| `--gha-workflow-run-id` | | | GHA CI only - The run id of the GitHub action workflow | +| `--ci-context` | `manual` | | The current CI context: `manual` for manual run, `pull_request`, `nightly_builds`, `master` | +| `--pipeline-start-timestamp` | Current epoch time | `CI_PIPELINE_START_TIMESTAMP` | Start time of the pipeline as epoch time. Used for pipeline run duration computation. | +| `--show-dagger-logs/--hide-dagger-logs` | `--hide-dagger-logs` | | Flag to show or hide the dagger logs. | + ### `connectors` command subgroup Available commands: -* `airbyte-ci connectors test`: Run tests for one or multiple connectors. -* `airbyte-ci connectors build`: Build docker images for one or multiple connectors. -* `airbyte-ci connectors publish`: Publish a connector to Airbyte's DockerHub. + +- `airbyte-ci connectors test`: Run tests for one or multiple connectors. +- `airbyte-ci connectors build`: Build docker images for one or multiple connectors. +- `airbyte-ci connectors publish`: Publish a connector to Airbyte's DockerHub. #### Options -| Option | Multiple | Default value | Description | -| -------------------------------------------------------------- | -------- | -------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--use-remote-secrets` | False | True | If True, connectors configuration will be pulled from Google Secret Manager. Requires the GCP_GSM_CREDENTIALS environment variable to be set with a service account with permission to read GSM secrets. If False the connector configuration will be read from the local connector `secrets` folder. | -| `--name` | True | | Select a specific connector for which the pipeline will run. Can be used multiple time to select multiple connectors. The expected name is the connector technical name. e.g. `source-pokeapi` | -| `--support-level` | True | | Select connectors with a specific support level: `community`, `certified`. Can be used multiple times to select multiple support levels. | -| `--language` | True | | Select connectors with a specific language: `python`, `low-code`, `java`. Can be used multiple times to select multiple languages. | -| `--modified` | False | False | Run the pipeline on only the modified connectors on the branch or previous commit (depends on the pipeline implementation). | -| `--concurrency` | False | 5 | Control the number of connector pipelines that can run in parallel. Useful to speed up pipelines or control their resource usage. | -| `--metadata-change-only/--not-metadata-change-only` | False | `--not-metadata-change-only` | Only run the pipeline on connectors with changes on their metadata.yaml file. | -| `--enable-dependency-scanning / --disable-dependency-scanning` | False | ` --disable-dependency-scanning` | When enabled the dependency scanning will be performed to detect the connectors to select according to a dependency change. | + +| Option | Multiple | Default value | Mapped Environment Variable | Description | +| -------------------------------------------------------------- | -------- | -------------------------------- | --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--use-remote-secrets/--use-local-secrets` | False | | | If --use-remote-secrets, connectors configuration will be pulled from Google Secret Manager. Requires the `GCP_GSM_CREDENTIALS` environment variable to be set with a service account with permission to read GSM secrets. If --use-local-secrets the connector configuration will be read from the local connector `secrets` folder. If this flag is not used and a `GCP_GSM_CREDENTIALS` environment variable is set remote secrets will be used, local secrets will be used otherwise. | +| `--name` | True | | | Select a specific connector for which the pipeline will run. Can be used multiple times to select multiple connectors. The expected name is the connector technical name. e.g. `source-pokeapi` | +| `--support-level` | True | | | Select connectors with a specific support level: `community`, `certified`. Can be used multiple times to select multiple support levels. | +| `--metadata-query` | False | | | Filter connectors by the `data` field in the metadata file using a [simpleeval](https://github.com/danthedeckie/simpleeval) query. e.g. 'data.ab_internal.ql == 200' | +| `--use-local-cdk` | False | False | | Build with the airbyte-cdk from the local repository. " "This is useful for testing changes to the CDK. | +| `--language` | True | | | Select connectors with a specific language: `python`, `low-code`, `java`. Can be used multiple times to select multiple languages. | +| `--modified` | False | False | | Run the pipeline on only the modified connectors on the branch or previous commit (depends on the pipeline implementation). | +| `--concurrency` | False | 5 | | Control the number of connector pipelines that can run in parallel. Useful to speed up pipelines or control their resource usage. | +| `--metadata-change-only/--not-metadata-change-only` | False | `--not-metadata-change-only` | | Only run the pipeline on connectors with changes on their metadata.yaml file. | +| `--enable-dependency-scanning / --disable-dependency-scanning` | False | ` --disable-dependency-scanning` | | When enabled the dependency scanning will be performed to detect the connectors to select according to a dependency change. | +| `--docker-hub-username` | | | DOCKER_HUB_USERNAME | Your username to connect to DockerHub. Required for the publish subcommand. | +| `--docker-hub-password` | | | DOCKER_HUB_PASSWORD | Your password to connect to DockerHub. Required for the publish subcommand. | ### `connectors list` command + Retrieve the list of connectors satisfying the provided filters. #### Examples + List all connectors: `airbyte-ci connectors list` @@ -150,34 +205,8 @@ List connectors with multiple filters: `airbyte-ci connectors --language=low-code --support-level=certified list` -### `connectors format` command -Run a code formatter on one or multiple connectors. - -For Python connectors we run the following tools, using the configuration defined in `pyproject.toml`: -* `black` for code formatting -* `isort` for import sorting -* `licenseheaders` for adding license headers - -For Java connectors we run `./gradlew format`. - -In local CLI execution the formatted code is exported back the local repository. No commit or push is performed. -In CI execution the formatted code is pushed to the remote branch. One format commit per connector. - -#### Examples -Format a specific connector: - -`airbyte-ci connectors --name=source-pokeapi format` - -Format all Python connectors: - -`airbyte-ci connectors --language=python format` - -Format all connectors modified on the current branch: - -`airbyte-ci connectors --modified format` - - ### `connectors test` command + Run a test pipeline for one or multiple connectors. #### Examples @@ -194,13 +223,16 @@ Test certified connectors: Test connectors changed on the current branch: `airbyte-ci connectors --modified test` +Run acceptance test only on the modified connectors, just run its full refresh tests: +`airbyte-ci connectors --modified test --only-step="acceptance" --acceptance.-k=test_full_refresh` + #### What it runs + ```mermaid flowchart TD entrypoint[[For each selected connector]] subgraph static ["Static code analysis"] qa[Run QA checks] - fmt[Run code format checks] sem["Check version follows semantic versionning"] incr["Check version is incremented"] metadata_validation["Run metadata validation on metadata.yaml"] @@ -229,23 +261,51 @@ flowchart TD #### Options -| Option | Multiple | Default value | Description | -| ------------------- | -------- | ------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--fail-fast` | False | False | Abort after any tests fail, rather than continuing to run additional tests. Use this setting to confirm a known bug is fixed (or not), or when you only require a pass/fail result. | -| `--fast-tests-only` | True | False | Run unit tests only, skipping integration tests or any tests explicitly tagged as slow. Use this for more frequent checks, when it is not feasible to run the entire test suite. | -| `--code-tests-only` | True | False | Skip any tests not directly related to code updates. For instance, metadata checks, version bump checks, changelog verification, etc. Use this setting to help focus on code quality during development.| +| Option | Multiple | Default value | Description | +| ------------------------------------------------------- | -------- | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--skip-step/-x` | True | | Skip steps by id e.g. `-x unit -x acceptance` | +| `--only-step/-k` | True | | Only run specific steps by id e.g. `-k unit -k acceptance` | +| `--fail-fast` | False | False | Abort after any tests fail, rather than continuing to run additional tests. Use this setting to confirm a known bug is fixed (or not), or when you only require a pass/fail result. | +| `--code-tests-only` | True | False | Skip any tests not directly related to code updates. For instance, metadata checks, version bump checks, changelog verification, etc. Use this setting to help focus on code quality during development. | +| `--concurrent-cat` | False | False | Make CAT tests run concurrently using pytest-xdist. Be careful about source or destination API rate limits. | +| `--.=` | True | | You can pass extra parameters for specific test steps. More details in the extra parameters section below | +| `--ci-requirements` | False | | | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. Note: -* The above options are implemented for Java connectors but may not be available for Python connectors. If an option is not supported, the pipeline will not fail but instead the 'default' behavior will be executed. +- The above options are implemented for Java connectors but may not be available for Python connectors. If an option is not supported, the pipeline will not fail but instead the 'default' behavior will be executed. + +#### Extra parameters +You can pass extra parameters to the following steps: +* `unit` +* `integration` +* `acceptance` + +This allows you to override the default parameters of these steps. +For example, you can only run the `test_read` test of the acceptance test suite with: +`airbyte-ci connectors --name=source-pokeapi test --acceptance.-k=test_read` +Here the `-k` parameter is passed to the pytest command running acceptance tests. +Please keep in mind that the extra parameters are not validated by the CLI: if you pass an invalid parameter, you'll face a late failure during the pipeline execution. ### `connectors build` command + Run a build pipeline for one or multiple connectors and export the built docker image to the local docker host. It's mainly purposed for local use. Build a single connector: `airbyte-ci connectors --name=source-pokeapi build` +Build a single connector with a custom image tag: +`airbyte-ci connectors --name=source-pokeapi build --tag=my-custom-tag` + +Build a single connector for multiple architectures: +`airbyte-ci connectors --name=source-pokeapi build --architecture=linux/amd64 --architecture=linux/arm64` + +You will get: + +- `airbyte/source-pokeapi:dev-linux-amd64` +- `airbyte/source-pokeapi:dev-linux-arm64` + Build multiple connectors: `airbyte-ci connectors --name=source-pokeapi --name=source-bigquery build` @@ -269,6 +329,7 @@ flowchart TD ``` For Java connectors: + ```mermaid flowchart TD arch(For each platform amd64/arm64) @@ -282,16 +343,25 @@ flowchart TD distTar-->connector normalization--"if supports normalization"-->connector - load[Load to docker host with :dev tag, current platform] + load[Load to docker host with :dev tag] spec[Get spec] connector-->spec--"if success"-->load ``` +### Options + +| Option | Multiple | Default value | Description | +| --------------------- | -------- | -------------- | -------------------------------------------------------------------- | +| `--architecture`/`-a` | True | Local platform | Defines for which architecture(s) the connector image will be built. | +| `--tag` | False | `dev` | Image tag for the built image. | + ### `connectors publish` command + Run a publish pipeline for one or multiple connectors. It's mainly purposed for CI use to release a connector update. ### Examples + Publish all connectors modified in the head commit: `airbyte-ci connectors --modified publish` ### Options @@ -299,17 +369,19 @@ Publish all connectors modified in the head commit: `airbyte-ci connectors --mod | Option | Required | Default | Mapped environment variable | Description | | ------------------------------------ | -------- | --------------- | ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `--pre-release/--main-release` | False | `--pre-release` | | Whether to publish the pre-release or the main release version of a connector. Defaults to pre-release. For main release you have to set the credentials to interact with the GCS bucket. | -| `--docker-hub-username` | True | | `DOCKER_HUB_USERNAME` | Your username to connect to DockerHub. | -| `--docker-hub-password` | True | | `DOCKER_HUB_PASSWORD` | Your password to connect to DockerHub. | | `--spec-cache-gcs-credentials` | False | | `SPEC_CACHE_GCS_CREDENTIALS` | The service account key to upload files to the GCS bucket hosting spec cache. | | `--spec-cache-bucket-name` | False | | `SPEC_CACHE_BUCKET_NAME` | The name of the GCS bucket where specs will be cached. | | `--metadata-service-gcs-credentials` | False | | `METADATA_SERVICE_GCS_CREDENTIALS` | The service account key to upload files to the GCS bucket hosting the metadata files. | | `--metadata-service-bucket-name` | False | | `METADATA_SERVICE_BUCKET_NAME` | The name of the GCS bucket where metadata files will be uploaded. | | `--slack-webhook` | False | | `SLACK_WEBHOOK` | The Slack webhook URL to send notifications to. | | `--slack-channel` | False | | `SLACK_CHANNEL` | The Slack channel name to send notifications to. | +| `--ci-requirements` | False | | | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. | + I've added an empty "Default" column, and you can fill in the default values as needed. + #### What it runs + ```mermaid flowchart TD validate[Validate the metadata file] @@ -323,107 +395,377 @@ flowchart TD validate-->check-->build-->upload_spec-->push-->pull-->upload_metadata ``` -### `metadata` command subgroup +### `connectors bump_version` command + +Bump the version of the selected connectors. + +### Examples + +Bump source-openweather: `airbyte-ci connectors --name=source-openweather bump_version patch ""` + +#### Arguments + +| Argument | Description | +| --------------------- | ---------------------------------------------------------------------- | +| `BUMP_TYPE` | major, minor or patch | +| `PULL_REQUEST_NUMBER` | The GitHub pull request number, used in the changelog entry | +| `CHANGELOG_ENTRY` | The changelog entry that will get added to the connector documentation | + +### `connectors upgrade_cdk` command + +Upgrade the CDK version of the selected connectors by updating the dependency in the setup.py file. + +### Examples + +Upgrade for source-openweather: `airbyte-ci connectors --name=source-openweather upgrade_cdk ` + +#### Arguments + +| Argument | Description | +| ------------- | ------------------------------------------------------- | +| `CDK_VERSION` | CDK version to set (default to the most recent version) | + +### `connectors upgrade_base_image` command + +Modify the selected connector metadata to use the latest base image version. + +### Examples + +Upgrade the base image for source-openweather: `airbyte-ci connectors --name=source-openweather upgrade_base_image` + +### Options + +| Option | Required | Default | Mapped environment variable | Description | +| ----------------------- | -------- | ------- | --------------------------- | --------------------------------------------------------------------------------------------------------------- | +| `--docker-hub-username` | True | | `DOCKER_HUB_USERNAME` | Your username to connect to DockerHub. It's used to read the base image registry. | +| `--docker-hub-password` | True | | `DOCKER_HUB_PASSWORD` | Your password to connect to DockerHub. It's used to read the base image registry. | +| `--set-if-not-exists` | False | True | | Whether to set or not the baseImage metadata if no connectorBuildOptions is declared in the connector metadata. | + +### `connectors migrate_to_base_image` command + +Make a connector using a Dockerfile migrate to the base image by: + +- Removing its Dockerfile +- Updating its metadata to use the latest base image version +- Updating its documentation to explain the build process +- Bumping by a patch version + +### Examples + +Migrate source-openweather to use the base image: `airbyte-ci connectors --name=source-openweather migrate_to_base_image` + +### Arguments + +| Argument | Description | +| --------------------- | ----------------------------------------------------------- | +| `PULL_REQUEST_NUMBER` | The GitHub pull request number, used in the changelog entry | + +### `format` command subgroup Available commands: -* `airbyte-ci metadata validate` -* `airbyte-ci metadata upload` -* `airbyte-ci metadata test lib` -* `airbyte-ci metadata test orchestrator` -* `airbyte-ci metadata deploy orchestrator` -### `metadata validate` command -This commands validates the modified `metadata.yaml` files in the head commit, or all the `metadata.yaml` files. +- `airbyte-ci format check all` +- `airbyte-ci format fix all` -#### Example -Validate all `metadata.yaml` files in the repo: -`airbyte-ci metadata validate --all` +### Options -#### Options -| Option | Default | Description | -| ------------------ | ------------ | -------------------------------------------------------------------------------------------------------------------------- | -| `--modified/--all` | `--modified` | Flag to run validation of `metadata.yaml` files on the modified files in the head commit or all the `metadata.yaml` files. | +| Option | Required | Default | Mapped environment variable | Description | +| ------------------- | -------- | ------- | --------------------------- | ------------------------------------------------------------------------------------------- | +| `--quiet/-q` | False | False | | Hide formatter execution details in reporting. | +| `--ci-requirements` | False | | | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. | + +### Examples -### `metadata upload` command -This command upload the modified `metadata.yaml` files in the head commit, or all the `metadata.yaml` files, to a GCS bucket. +- Check for formatting errors in the repository: `airbyte-ci format check all` +- Fix formatting for only python files: `airbyte-ci format fix python` -#### Example -Upload all the `metadata.yaml` files to a GCS bucket: -`airbyte-ci metadata upload --all ` +### `format check all` command -#### Options -| Option | Required | Default | Mapped environment variable | Description | -| ------------------- | -------- | ------------ | --------------------------- | ------------------------------------------------------------------------------------------------------------------------ | -| `--gcs-credentials` | True | | `GCS_CREDENTIALS` | Service account credentials in JSON format with permission to get and upload on the GCS bucket | -| `--modified/--all` | True | `--modified` | | Flag to upload the modified `metadata.yaml` files in the head commit or all the `metadata.yaml` files to a GCS bucket. | +This command runs formatting checks, but does not format the code in place. It will exit 1 as soon as a failure is encountered. To fix errors, use `airbyte-ci format fix all`. + +Running `airbyte-ci format check` will run checks on all different types of code. Run `airbyte-ci format check --help` for subcommands to check formatting for only certain types of files. + +### `format fix all` command + +This command runs formatting checks and reformats any code that would be reformatted, so it's recommended to stage changes you might have before running this command. + +Running `airbyte-ci format fix all` will format all of the different types of code. Run `airbyte-ci format fix --help` for subcommands to format only certain types of files. + +### `metadata` command subgroup + +Available commands: + +- `airbyte-ci metadata deploy orchestrator` ### `metadata deploy orchestrator` command + This command deploys the metadata service orchestrator to production. The `DAGSTER_CLOUD_METADATA_API_TOKEN` environment variable must be set. #### Example + `airbyte-ci metadata deploy orchestrator` #### What it runs + ```mermaid flowchart TD test[Run orchestrator tests] --> deploy[Deploy orchestrator to Dagster Cloud] ``` -### `metadata test lib` command -This command runs tests for the metadata service library. - -#### Example -`airbyte-ci metadata test lib` - -### `metadata test orchestrator` command -This command runs tests for the metadata service orchestrator. - -#### Example -`airbyte-ci metadata test orchestrator` - ### `tests` command + This command runs the Python tests for a airbyte-ci poetry package. #### Arguments -| Option | Required | Default | Mapped environment variable | Description | -| ------------------ | -------- | ------- | --------------------------- | ---------------------------------------------------------------- | -| `poetry_package_path` | True | | | The path to poetry package to test. | + +| Option | Required | Default | Mapped environment variable | Description | +| --------------------- | -------- | ------- | --------------------------- | ----------------------------------- | +| `poetry_package_path` | True | | | The path to poetry package to test. | #### Options -| Option | Required | Default | Mapped environment variable | Description | -| ------------------ | -------- | ------- | --------------------------- | ---------------------------------------------------------------- | -| `--test-directory` | False | tests | | The path to the directory on which pytest should discover tests, relative to the poetry package. | +| Option | Required | Default | Mapped environment variable | Description | +| ------------------------- | -------- | ------- | --------------------------- | ------------------------------------------------------------------------------------------- | +| `-c/--poetry-run-command` | True | None | | The command to run with `poetry run` | +| `-e/--pass-env-var` | False | None | | Host environment variable that is passed to the container running the poetry command | +| `--ci-requirements` | False | | | Output the CI requirements as a JSON payload. It is used to determine the CI runner to use. | -#### Example -`airbyte-ci test airbyte-ci/connectors/pipelines --test-directory=tests` -`airbyte-ci tests airbyte-integrations/bases/connector-acceptance-test --test-directory=unit_tests` +#### Examples +You can pass multiple `-c/--poetry-run-command` options to run multiple commands. + +E.G.: running `pytest` and `mypy`: +`airbyte-ci test airbyte-ci/connectors/pipelines --poetry-run-command='pytest tests' --poetry-run-command='mypy pipelines'` + +E.G.: passing the environment variable `GCP_GSM_CREDENTIALS` environment variable to the container running the poetry command: +`airbyte-ci test airbyte-lib --pass-env-var='GCP_GSM_CREDENTIALS'` + +E.G.: running `pytest` on a specific test folder: +`airbyte-ci tests airbyte-integrations/bases/connector-acceptance-test --poetry-run-command='pytest tests/unit_tests'` ## Changelog -| Version | PR | Description | -| ------- | --------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | -| 1.1.0 | [#29509](https://github.com/airbytehq/airbyte/pull/29509) | Refactor the airbyte-ci test command to run tests on any poetry package. | -| 1.0.0 | [#28000](https://github.com/airbytehq/airbyte/pull/29232) | Remove release stages in favor of support level from airbyte-ci. | -| 0.5.0 | [#28000](https://github.com/airbytehq/airbyte/pull/28000) | Run connector acceptance tests with dagger-in-dagger. | -| 0.4.7 | [#29156](https://github.com/airbytehq/airbyte/pull/29156) | Improve how we check existence of requirement.txt or setup.py file to not raise early pip install errors. | -| 0.4.6 | [#28729](https://github.com/airbytehq/airbyte/pull/28729) | Use keyword args instead of positional argument for optional paramater in Dagger's API | -| 0.4.5 | [#29034](https://github.com/airbytehq/airbyte/pull/29034) | Disable Dagger terminal UI when running publish. | -| 0.4.4 | [#29064](https://github.com/airbytehq/airbyte/pull/29064) | Make connector modified files a frozen set. | -| 0.4.3 | [#29033](https://github.com/airbytehq/airbyte/pull/29033) | Disable dependency scanning for Java connectors. | -| 0.4.2 | [#29030](https://github.com/airbytehq/airbyte/pull/29030) | Make report path always have the same prefix: `airbyte-ci/`. | -| 0.4.1 | [#28855](https://github.com/airbytehq/airbyte/pull/28855) | Improve the selected connectors detection for connectors commands. | -| 0.4.0 | [#28947](https://github.com/airbytehq/airbyte/pull/28947) | Show Dagger Cloud run URLs in CI | -| 0.3.2 | [#28789](https://github.com/airbytehq/airbyte/pull/28789) | Do not consider empty reports as successfull. | -| 0.3.1 | [#28938](https://github.com/airbytehq/airbyte/pull/28938) | Handle 5 status code on MetadataUpload as skipped | -| 0.3.0 | [#28869](https://github.com/airbytehq/airbyte/pull/28869) | Enable the Dagger terminal UI on local `airbyte-ci` execution | -| 0.2.3 | [#28907](https://github.com/airbytehq/airbyte/pull/28907) | Make dagger-in-dagger work for `airbyte-ci tests` command | -| 0.2.2 | [#28897](https://github.com/airbytehq/airbyte/pull/28897) | Sentry: Ignore error logs without exceptions from reporting | -| 0.2.1 | [#28767](https://github.com/airbytehq/airbyte/pull/28767) | Improve pytest step result evaluation to prevent false negative/positive. | -| 0.2.0 | [#28857](https://github.com/airbytehq/airbyte/pull/28857) | Add the `airbyte-ci tests` command to run the test suite on any `airbyte-ci` poetry package. | -| 0.1.1 | [#28858](https://github.com/airbytehq/airbyte/pull/28858) | Increase the max duration of Connector Package install to 20mn. | -| 0.1.0 | | Alpha version not in production yet. All the commands described in this doc are available. | + +| Version | PR | Description | +| ------- | ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- | +| 3.5.1 | [#34321](https://github.com/airbytehq/airbyte/pull/34321) | Upgrade to Dagger 0.9.6 . | +| 3.5.0 | [#33313](https://github.com/airbytehq/airbyte/pull/33313) | Pass extra params after Gradle tasks. | +| 3.4.2 | [#34301](https://github.com/airbytehq/airbyte/pull/34301) | Pass extra params after Gradle tasks. | +| 3.4.1 | [#34067](https://github.com/airbytehq/airbyte/pull/34067) | Use dagster-cloud 1.5.7 for deploy | +| 3.4.0 | [#34276](https://github.com/airbytehq/airbyte/pull/34276) | Introduce `--only-step` option for connector tests. | +| 3.3.0 | [#34218](https://github.com/airbytehq/airbyte/pull/34218) | Introduce `--ci-requirements` option for client defined CI runners. | +| 3.2.0 | [#34050](https://github.com/airbytehq/airbyte/pull/34050) | Connector test steps can take extra parameters | +| 3.1.3 | [#34136](https://github.com/airbytehq/airbyte/pull/34136) | Fix issue where dagger excludes were not being properly applied | +| 3.1.2 | [#33972](https://github.com/airbytehq/airbyte/pull/33972) | Remove secrets scrubbing hack for --is-local and other small tweaks. | +| 3.1.1 | [#33979](https://github.com/airbytehq/airbyte/pull/33979) | Fix AssertionError on report existence again | +| 3.1.0 | [#33994](https://github.com/airbytehq/airbyte/pull/33994) | Log more context information in CI. | +| 3.0.2 | [#33987](https://github.com/airbytehq/airbyte/pull/33987) | Fix type checking issue when running --help | +| 3.0.1 | [#33981](https://github.com/airbytehq/airbyte/pull/33981) | Fix issues with deploying dagster, pin pendulum version in dagster-cli install | +| 3.0.0 | [#33582](https://github.com/airbytehq/airbyte/pull/33582) | Upgrade to Dagger 0.9.5 | +| 2.14.3 | [#33964](https://github.com/airbytehq/airbyte/pull/33964) | Reintroduce mypy with fixes for AssertionError on publish and missing report URL on connector test commit status. | +| 2.14.2 | [#33954](https://github.com/airbytehq/airbyte/pull/33954) | Revert mypy changes | +| 2.14.1 | [#33956](https://github.com/airbytehq/airbyte/pull/33956) | Exclude pnpm lock files from auto-formatting | +| 2.14.0 | [#33941](https://github.com/airbytehq/airbyte/pull/33941) | Enable in-connector normalization in destination-postgres | +| 2.13.1 | [#33920](https://github.com/airbytehq/airbyte/pull/33920) | Report different sentry environments | +| 2.13.0 | [#33784](https://github.com/airbytehq/airbyte/pull/33784) | Make `airbyte-ci test` able to run any poetry command | +| 2.12.0 | [#33313](https://github.com/airbytehq/airbyte/pull/33313) | Add upgrade CDK command | +| 2.11.0 | [#32188](https://github.com/airbytehq/airbyte/pull/32188) | Add -x option to connector test to allow for skipping steps | +| 2.10.12 | [#33419](https://github.com/airbytehq/airbyte/pull/33419) | Make ClickPipelineContext handle dagger logging. | +| 2.10.11 | [#33497](https://github.com/airbytehq/airbyte/pull/33497) | Consider nested .gitignore rules in format. | +| 2.10.10 | [#33449](https://github.com/airbytehq/airbyte/pull/33449) | Add generated metadata models to the default format ignore list. | +| 2.10.9 | [#33370](https://github.com/airbytehq/airbyte/pull/33370) | Fix bug that broke airbyte-ci test | +| 2.10.8 | [#33249](https://github.com/airbytehq/airbyte/pull/33249) | Exclude git ignored files from formatting. | +| 2.10.7 | [#33248](https://github.com/airbytehq/airbyte/pull/33248) | Fix bug which broke airbyte-ci connectors tests when optional DockerHub credentials env vars are not set. | +| 2.10.6 | [#33170](https://github.com/airbytehq/airbyte/pull/33170) | Remove Dagger logs from console output of `format`. | +| 2.10.5 | [#33097](https://github.com/airbytehq/airbyte/pull/33097) | Improve `format` performances, exit with 1 status code when `fix` changes files. | +| 2.10.4 | [#33206](https://github.com/airbytehq/airbyte/pull/33206) | Add "-y/--yes" Flag to allow preconfirmation of prompts | +| 2.10.3 | [#33080](https://github.com/airbytehq/airbyte/pull/33080) | Fix update failing due to SSL error on install. | +| 2.10.2 | [#33008](https://github.com/airbytehq/airbyte/pull/33008) | Fix local `connector build`. | +| 2.10.1 | [#32928](https://github.com/airbytehq/airbyte/pull/32928) | Fix BuildConnectorImages constructor. | +| 2.10.0 | [#32819](https://github.com/airbytehq/airbyte/pull/32819) | Add `--tag` option to connector build. | +| 2.9.0 | [#32816](https://github.com/airbytehq/airbyte/pull/32816) | Add `--architecture` option to connector build. | +| 2.8.1 | [#32999](https://github.com/airbytehq/airbyte/pull/32999) | Improve Java code formatting speed | +| 2.8.0 | [#31930](https://github.com/airbytehq/airbyte/pull/31930) | Move pipx install to `airbyte-ci-dev`, and add auto-update feature targeting binary | +| 2.7.3 | [#32847](https://github.com/airbytehq/airbyte/pull/32847) | Improve --modified behaviour for pull requests. | +| 2.7.2 | [#32839](https://github.com/airbytehq/airbyte/pull/32839) | Revert changes in v2.7.1. | +| 2.7.1 | [#32806](https://github.com/airbytehq/airbyte/pull/32806) | Improve --modified behaviour for pull requests. | +| 2.7.0 | [#31930](https://github.com/airbytehq/airbyte/pull/31930) | Merge airbyte-ci-internal into airbyte-ci | +| 2.6.0 | [#31831](https://github.com/airbytehq/airbyte/pull/31831) | Add `airbyte-ci format` commands, remove connector-specific formatting check | +| 2.5.9 | [#32427](https://github.com/airbytehq/airbyte/pull/32427) | Re-enable caching for source-postgres | +| 2.5.8 | [#32402](https://github.com/airbytehq/airbyte/pull/32402) | Set Dagger Cloud token for airbyters only | +| 2.5.7 | [#31628](https://github.com/airbytehq/airbyte/pull/31628) | Add ClickPipelineContext class | +| 2.5.6 | [#32139](https://github.com/airbytehq/airbyte/pull/32139) | Test coverage report on Python connector UnitTest. | +| 2.5.5 | [#32114](https://github.com/airbytehq/airbyte/pull/32114) | Create cache mount for `/var/lib/docker` to store images in `dind` context. | +| 2.5.4 | [#32090](https://github.com/airbytehq/airbyte/pull/32090) | Do not cache `docker login`. | +| 2.5.3 | [#31974](https://github.com/airbytehq/airbyte/pull/31974) | Fix latest CDK install and pip cache mount on connector install. | +| 2.5.2 | [#31871](https://github.com/airbytehq/airbyte/pull/31871) | Deactivate PR comments, add HTML report links to the PR status when its ready. | +| 2.5.1 | [#31774](https://github.com/airbytehq/airbyte/pull/31774) | Add a docker configuration check on `airbyte-ci` startup. | +| 2.5.0 | [#31766](https://github.com/airbytehq/airbyte/pull/31766) | Support local connectors secrets. | +| 2.4.0 | [#31716](https://github.com/airbytehq/airbyte/pull/31716) | Enable pre-release publish with local CDK. | +| 2.3.1 | [#31748](https://github.com/airbytehq/airbyte/pull/31748) | Use AsyncClick library instead of base Click. | +| 2.3.0 | [#31699](https://github.com/airbytehq/airbyte/pull/31699) | Support optional concurrent CAT execution. | +| 2.2.6 | [#31752](https://github.com/airbytehq/airbyte/pull/31752) | Only authenticate when secrets are available. | +| 2.2.5 | [#31718](https://github.com/airbytehq/airbyte/pull/31718) | Authenticate the sidecar docker daemon to DockerHub. | +| 2.2.4 | [#31535](https://github.com/airbytehq/airbyte/pull/31535) | Improve gradle caching when building java connectors. | +| 2.2.3 | [#31688](https://github.com/airbytehq/airbyte/pull/31688) | Fix failing `CheckBaseImageUse` step when not running on PR. | +| 2.2.2 | [#31659](https://github.com/airbytehq/airbyte/pull/31659) | Support builds on x86_64 platform | +| 2.2.1 | [#31653](https://github.com/airbytehq/airbyte/pull/31653) | Fix CheckBaseImageIsUsed failing on non certified connectors. | +| 2.2.0 | [#30527](https://github.com/airbytehq/airbyte/pull/30527) | Add a new check for python connectors to make sure certified connectors use our base image. | +| 2.1.1 | [#31488](https://github.com/airbytehq/airbyte/pull/31488) | Improve `airbyte-ci` start time with Click Lazy load | +| 2.1.0 | [#31412](https://github.com/airbytehq/airbyte/pull/31412) | Run airbyte-ci from any where in airbyte project | +| 2.0.4 | [#31487](https://github.com/airbytehq/airbyte/pull/31487) | Allow for third party connector selections | +| 2.0.3 | [#31525](https://github.com/airbytehq/airbyte/pull/31525) | Refactor folder structure | +| 2.0.2 | [#31533](https://github.com/airbytehq/airbyte/pull/31533) | Pip cache volume by python version. | +| 2.0.1 | [#31545](https://github.com/airbytehq/airbyte/pull/31545) | Reword the changelog entry when using `migrate_to_base_image`. | +| 2.0.0 | [#31424](https://github.com/airbytehq/airbyte/pull/31424) | Remove `airbyte-ci connectors format` command. | +| 1.9.4 | [#31478](https://github.com/airbytehq/airbyte/pull/31478) | Fix running tests for connector-ops package. | +| 1.9.3 | [#31457](https://github.com/airbytehq/airbyte/pull/31457) | Improve the connector documentation for connectors migrated to our base image. | +| 1.9.2 | [#31426](https://github.com/airbytehq/airbyte/pull/31426) | Concurrent execution of java connectors tests. | +| 1.9.1 | [#31455](https://github.com/airbytehq/airbyte/pull/31455) | Fix `None` docker credentials on publish. | +| 1.9.0 | [#30520](https://github.com/airbytehq/airbyte/pull/30520) | New commands: `bump_version`, `upgrade_base_image`, `migrate_to_base_image`. | +| 1.8.0 | [#30520](https://github.com/airbytehq/airbyte/pull/30520) | New commands: `bump_version`, `upgrade_base_image`, `migrate_to_base_image`. | +| 1.7.2 | [#31343](https://github.com/airbytehq/airbyte/pull/31343) | Bind Pytest integration tests to a dockerhost. | +| 1.7.1 | [#31332](https://github.com/airbytehq/airbyte/pull/31332) | Disable Gradle step caching on source-postgres. | +| 1.7.0 | [#30526](https://github.com/airbytehq/airbyte/pull/30526) | Implement pre/post install hooks support. | +| 1.6.0 | [#30474](https://github.com/airbytehq/airbyte/pull/30474) | Test connector inside their containers. | +| 1.5.1 | [#31227](https://github.com/airbytehq/airbyte/pull/31227) | Use python 3.11 in amazoncorretto-bazed gradle containers, run 'test' gradle task instead of 'check'. | +| 1.5.0 | [#30456](https://github.com/airbytehq/airbyte/pull/30456) | Start building Python connectors using our base images. | +| 1.4.6 | [ #31087](https://github.com/airbytehq/airbyte/pull/31087) | Throw error if airbyte-ci tools is out of date | +| 1.4.5 | [#31133](https://github.com/airbytehq/airbyte/pull/31133) | Fix bug when building containers using `with_integration_base_java_and_normalization`. | +| 1.4.4 | [#30743](https://github.com/airbytehq/airbyte/pull/30743) | Add `--disable-report-auto-open` and `--use-host-gradle-dist-tar` to allow gradle integration. | +| 1.4.3 | [#30595](https://github.com/airbytehq/airbyte/pull/30595) | Add --version and version check | +| 1.4.2 | [#30595](https://github.com/airbytehq/airbyte/pull/30595) | Remove directory name requirement | +| 1.4.1 | [#30595](https://github.com/airbytehq/airbyte/pull/30595) | Load base migration guide into QA Test container for strict encrypt variants | +| 1.4.0 | [#30330](https://github.com/airbytehq/airbyte/pull/30330) | Add support for pyproject.toml as the prefered entry point for a connector package | +| 1.3.0 | [#30461](https://github.com/airbytehq/airbyte/pull/30461) | Add `--use-local-cdk` flag to all connectors commands | +| 1.2.3 | [#30477](https://github.com/airbytehq/airbyte/pull/30477) | Fix a test regression introduced the previous version. | +| 1.2.2 | [#30438](https://github.com/airbytehq/airbyte/pull/30438) | Add workaround to always stream logs properly with --is-local. | +| 1.2.1 | [#30384](https://github.com/airbytehq/airbyte/pull/30384) | Java connector test performance fixes. | +| 1.2.0 | [#30330](https://github.com/airbytehq/airbyte/pull/30330) | Add `--metadata-query` option to connectors command | +| 1.1.3 | [#30314](https://github.com/airbytehq/airbyte/pull/30314) | Stop patching gradle files to make them work with airbyte-ci. | +| 1.1.2 | [#30279](https://github.com/airbytehq/airbyte/pull/30279) | Fix correctness issues in layer caching by making atomic execution groupings | +| 1.1.1 | [#30252](https://github.com/airbytehq/airbyte/pull/30252) | Fix redundancies and broken logic in GradleTask, to speed up the CI runs. | +| 1.1.0 | [#29509](https://github.com/airbytehq/airbyte/pull/29509) | Refactor the airbyte-ci test command to run tests on any poetry package. | +| 1.0.0 | [#28000](https://github.com/airbytehq/airbyte/pull/29232) | Remove release stages in favor of support level from airbyte-ci. | +| 0.5.0 | [#28000](https://github.com/airbytehq/airbyte/pull/28000) | Run connector acceptance tests with dagger-in-dagger. | +| 0.4.7 | [#29156](https://github.com/airbytehq/airbyte/pull/29156) | Improve how we check existence of requirement.txt or setup.py file to not raise early pip install errors. | +| 0.4.6 | [#28729](https://github.com/airbytehq/airbyte/pull/28729) | Use keyword args instead of positional argument for optional paramater in Dagger's API | +| 0.4.5 | [#29034](https://github.com/airbytehq/airbyte/pull/29034) | Disable Dagger terminal UI when running publish. | +| 0.4.4 | [#29064](https://github.com/airbytehq/airbyte/pull/29064) | Make connector modified files a frozen set. | +| 0.4.3 | [#29033](https://github.com/airbytehq/airbyte/pull/29033) | Disable dependency scanning for Java connectors. | +| 0.4.2 | [#29030](https://github.com/airbytehq/airbyte/pull/29030) | Make report path always have the same prefix: `airbyte-ci/`. | +| 0.4.1 | [#28855](https://github.com/airbytehq/airbyte/pull/28855) | Improve the selected connectors detection for connectors commands. | +| 0.4.0 | [#28947](https://github.com/airbytehq/airbyte/pull/28947) | Show Dagger Cloud run URLs in CI | +| 0.3.2 | [#28789](https://github.com/airbytehq/airbyte/pull/28789) | Do not consider empty reports as successfull. | +| 0.3.1 | [#28938](https://github.com/airbytehq/airbyte/pull/28938) | Handle 5 status code on MetadataUpload as skipped | +| 0.3.0 | [#28869](https://github.com/airbytehq/airbyte/pull/28869) | Enable the Dagger terminal UI on local `airbyte-ci` execution | +| 0.2.3 | [#28907](https://github.com/airbytehq/airbyte/pull/28907) | Make dagger-in-dagger work for `airbyte-ci tests` command | +| 0.2.2 | [#28897](https://github.com/airbytehq/airbyte/pull/28897) | Sentry: Ignore error logs without exceptions from reporting | +| 0.2.1 | [#28767](https://github.com/airbytehq/airbyte/pull/28767) | Improve pytest step result evaluation to prevent false negative/positive. | +| 0.2.0 | [#28857](https://github.com/airbytehq/airbyte/pull/28857) | Add the `airbyte-ci tests` command to run the test suite on any `airbyte-ci` poetry package. | +| 0.1.1 | [#28858](https://github.com/airbytehq/airbyte/pull/28858) | Increase the max duration of Connector Package install to 20mn. | +| 0.1.0 | | Alpha version not in production yet. All the commands described in this doc are available. | ## More info + This project is owned by the Connectors Operations team. -We share project updates and remaining stories before its release to production in this [EPIC](https://github.com/airbytehq/airbyte/issues/24403). \ No newline at end of file +We share project updates and remaining stories before its release to production in this [EPIC](https://github.com/airbytehq/airbyte/issues/24403). + +# Troubleshooting + +## Commands + +### `make tools.airbyte-ci.check` + +This command checks if the `airbyte-ci` command is appropriately installed. + +### `make tools.airbyte-ci.clean` + +This command removes the `airbyte-ci` command from your system. + +## Common issues + +### `airbyte-ci` is not found + +If you get the following error when running `airbyte-ci`: + +```bash +$ airbyte-ci +zsh: command not found: airbyte-ci +``` + +It means that the `airbyte-ci` command is not in your PATH. + +Try running + +```bash +make make tools.airbyte-ci.check +``` + +For some hints on how to fix this. + +But when in doubt it can be best to run + +```bash +make tools.airbyte-ci.clean +``` + +Then reinstall the CLI with + +```bash +make tools.airbyte-ci.install +``` + +## Development + +### `airbyte-ci` is not found + +To fix this, you can either: + +- Ensure that airbyte-ci is installed with pipx. Run `pipx list` to check if airbyte-ci is installed. +- Run `pipx ensurepath` to add the pipx binary directory to your PATH. +- Add the pipx binary directory to your PATH manually. The pipx binary directory is usually `~/.local/bin`. + +### python3.10 not found + +If you get the following error when running `pipx install --editable --force --python=python3.10 airbyte-ci/connectors/pipelines/`: + +```bash +$ pipx install --editable --force --python=python3.10 airbyte-ci/connectors/pipelines/ +Error: Python 3.10 not found on your system. +``` + +It means that you don't have Python 3.10 installed on your system. + +To fix this, you can either: + +- Install Python 3.10 with pyenv. Run `pyenv install 3.10` to install the latest Python version. +- Install Python 3.10 with your system package manager. For instance, on Ubuntu you can run `sudo apt install python3.10`. +- Ensure that Python 3.10 is in your PATH. Run `which python3.10` to check if Python 3.10 is installed and in your PATH. + +### Any type of pipeline failure + +First you should check that the version of the CLI you are using is the latest one. +You can check the version of the CLI with the `--version` option: + +```bash +$ airbyte-ci --version +airbyte-ci, version 0.1.0 +``` + +and compare it with the version in the pyproject.toml file: + +```bash +$ cat airbyte-ci/connectors/pipelines/pyproject.toml | grep version +``` + +If you get any type of pipeline failure, you can run the pipeline with the `--show-dagger-logs` option to get more information about the failure. + +```bash +$ airbyte-ci --show-dagger-logs connectors --name=source-pokeapi test +``` + +and when in doubt, you can reinstall the CLI with the `--force` option: + +```bash +$ pipx reinstall pipelines --force +``` diff --git a/airbyte-ci/connectors/pipelines/pipelines/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/__init__.py index 371bafaa1370..4b1a6ecc74dd 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/__init__.py +++ b/airbyte-ci/connectors/pipelines/pipelines/__init__.py @@ -5,26 +5,26 @@ """The pipelines package.""" import logging import os - +from typing import Union from rich.logging import RichHandler -from . import sentry_utils +from .helpers import sentry_utils sentry_utils.initialize() logging.getLogger("requests").setLevel(logging.WARNING) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("httpx").setLevel(logging.WARNING) -logging_handlers = [RichHandler(rich_tracebacks=True)] -if "CI" in os.environ: - # RichHandler does not work great in the CI - logging_handlers = [logging.StreamHandler()] + +# RichHandler does not work great in the CI environment, so we use a StreamHandler instead +logging_handler: Union[RichHandler, logging.StreamHandler] = RichHandler(rich_tracebacks=True) if "CI" not in os.environ else logging.StreamHandler() + logging.basicConfig( level=logging.INFO, format="%(name)s: %(message)s", datefmt="[%X]", - handlers=logging_handlers, + handlers=[logging_handler], ) main_logger = logging.getLogger(__name__) diff --git a/airbyte-ci/connectors/pipelines/pipelines/actions/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/actions/__init__.py deleted file mode 100644 index 09bf0600a802..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/actions/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# -"""The actions package is made to declare reusable pipeline components.""" diff --git a/airbyte-ci/connectors/pipelines/pipelines/actions/environments.py b/airbyte-ci/connectors/pipelines/pipelines/actions/environments.py deleted file mode 100644 index 3f75ffbdc3d6..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/actions/environments.py +++ /dev/null @@ -1,1009 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -"""This modules groups functions made to create reusable environments packaged in dagger containers.""" - -from __future__ import annotations - -import importlib.util -import json -import re -import uuid -from pathlib import Path -from typing import TYPE_CHECKING, Callable, List, Optional - -import toml -from dagger import CacheVolume, Client, Container, DaggerError, Directory, File, Platform, Secret -from dagger.engine._version import CLI_VERSION as dagger_engine_version -from pipelines import consts -from pipelines.consts import ( - CI_CREDENTIALS_SOURCE_PATH, - CONNECTOR_OPS_SOURCE_PATHSOURCE_PATH, - CONNECTOR_TESTING_REQUIREMENTS, - LICENSE_SHORT_FILE_PATH, - PYPROJECT_TOML_FILE_PATH, -) -from pipelines.utils import check_path_in_workdir, get_file_contents - -if TYPE_CHECKING: - from pipelines.contexts import ConnectorContext, PipelineContext - - -def with_python_base(context: PipelineContext, python_version: str = "3.10") -> Container: - """Build a Python container with a cache volume for pip cache. - - Args: - context (PipelineContext): The current test context, providing a dagger client and a repository directory. - python_image_name (str, optional): The python image to use to build the python base environment. Defaults to "python:3.9-slim". - - Raises: - ValueError: Raised if the python_image_name is not a python image. - - Returns: - Container: The python base environment container. - """ - - pip_cache: CacheVolume = context.dagger_client.cache_volume("pip_cache") - - base_container = ( - context.dagger_client.container() - .from_(f"python:{python_version}-slim") - .with_exec(["apt-get", "update"]) - .with_exec(["apt-get", "install", "-y", "build-essential", "cmake", "g++", "libffi-dev", "libstdc++6", "git"]) - .with_mounted_cache("/root/.cache/pip", pip_cache) - .with_exec(["pip", "install", "pip==23.1.2"]) - ) - - return base_container - - -def with_testing_dependencies(context: PipelineContext) -> Container: - """Build a testing environment by installing testing dependencies on top of a python base environment. - - Args: - context (PipelineContext): The current test context, providing a dagger client and a repository directory. - - Returns: - Container: The testing environment container. - """ - python_environment: Container = with_python_base(context) - pyproject_toml_file = context.get_repo_dir(".", include=[PYPROJECT_TOML_FILE_PATH]).file(PYPROJECT_TOML_FILE_PATH) - license_short_file = context.get_repo_dir(".", include=[LICENSE_SHORT_FILE_PATH]).file(LICENSE_SHORT_FILE_PATH) - - return ( - python_environment.with_exec(["pip", "install"] + CONNECTOR_TESTING_REQUIREMENTS) - .with_file(f"/{PYPROJECT_TOML_FILE_PATH}", pyproject_toml_file) - .with_file(f"/{LICENSE_SHORT_FILE_PATH}", license_short_file) - ) - - -def with_git(dagger_client, ci_github_access_token_secret, ci_git_user) -> Container: - return ( - dagger_client.container() - .from_("alpine:latest") - .with_secret_variable("GITHUB_TOKEN", ci_github_access_token_secret) - .with_exec(["apk", "update"]) - .with_exec(["apk", "add", "git", "tar", "wget"]) - .with_workdir("/ghcli") - .with_exec(["wget", "https://github.com/cli/cli/releases/download/v2.30.0/gh_2.30.0_linux_amd64.tar.gz", "-O", "ghcli.tar.gz"]) - .with_exec(["tar", "--strip-components=1", "-xf", "ghcli.tar.gz"]) - .with_exec(["rm", "ghcli.tar.gz"]) - .with_exec(["cp", "bin/gh", "/usr/local/bin/gh"]) - .with_exec(["git", "config", "--global", "user.email", f"{ci_git_user}@users.noreply.github.com"]) - .with_exec(["git", "config", "--global", "user.name", ci_git_user]) - .with_exec(["git", "config", "--global", "--add", "--bool", "push.autoSetupRemote", "true"]) - ) - - -async def with_installed_pipx_package( - context: PipelineContext, - python_environment: Container, - package_source_code_path: str, - exclude: Optional[List] = None, -) -> Container: - """Install a python package in a python environment container using pipx. - - Args: - context (PipelineContext): The current test context, providing the repository directory from which the python sources will be pulled. - python_environment (Container): An existing python environment in which the package will be installed. - package_source_code_path (str): The local path to the package source code. - exclude (Optional[List]): A list of file or directory to exclude from the python package source code. - - Returns: - Container: A python environment container with the python package installed. - """ - pipx_python_environment = with_pipx(python_environment) - container = with_python_package(context, pipx_python_environment, package_source_code_path, exclude=exclude) - - local_dependencies = await find_local_dependencies_in_pyproject_toml(context, container, package_source_code_path, exclude=exclude) - for dependency_directory in local_dependencies: - container = container.with_mounted_directory("/" + dependency_directory, context.get_repo_dir(dependency_directory)) - - container = container.with_exec(["pipx", "install", f"/{package_source_code_path}"]) - - return container - - -def with_python_package( - context: PipelineContext, - python_environment: Container, - package_source_code_path: str, - exclude: Optional[List] = None, -) -> Container: - """Load a python package source code to a python environment container. - - Args: - context (PipelineContext): The current test context, providing the repository directory from which the python sources will be pulled. - python_environment (Container): An existing python environment in which the package will be installed. - package_source_code_path (str): The local path to the package source code. - additional_dependency_groups (Optional[List]): extra_requires dependency of setup.py to install. Defaults to None. - exclude (Optional[List]): A list of file or directory to exclude from the python package source code. - - Returns: - Container: A python environment container with the python package source code. - """ - package_source_code_directory: Directory = context.get_repo_dir(package_source_code_path, exclude=exclude) - work_dir_path = f"/{package_source_code_path}" - container = python_environment.with_mounted_directory(work_dir_path, package_source_code_directory).with_workdir(work_dir_path) - return container - - -async def find_local_python_dependencies( - context: PipelineContext, - package_source_code_path: str, - search_dependencies_in_setup_py: bool = True, - search_dependencies_in_requirements_txt: bool = True, -) -> List[str]: - """Find local python dependencies of a python package. The dependencies are found in the setup.py and requirements.txt files. - - Args: - context (PipelineContext): The current pipeline context, providing a dagger client and a repository directory. - package_source_code_path (str): The local path to the python package source code. - search_dependencies_in_setup_py (bool, optional): Whether to search for local dependencies in the setup.py file. Defaults to True. - search_dependencies_in_requirements_txt (bool, optional): Whether to search for local dependencies in the requirements.txt file. Defaults to True. - - Returns: - List[str]: Paths to the local dependencies relative to the airbyte repo. - """ - python_environment = with_python_base(context) - container = with_python_package(context, python_environment, package_source_code_path) - - local_dependency_paths = [] - if search_dependencies_in_setup_py: - local_dependency_paths += await find_local_dependencies_in_setup_py(container) - if search_dependencies_in_requirements_txt: - local_dependency_paths += await find_local_dependencies_in_requirements_txt(container, package_source_code_path) - - transitive_dependency_paths = [] - for local_dependency_path in local_dependency_paths: - # Transitive local dependencies installation is achieved by calling their setup.py file, not their requirements.txt file. - transitive_dependency_paths += await find_local_python_dependencies(context, local_dependency_path, True, False) - - all_dependency_paths = local_dependency_paths + transitive_dependency_paths - if all_dependency_paths: - context.logger.debug(f"Found local dependencies for {package_source_code_path}: {all_dependency_paths}") - return all_dependency_paths - - -async def find_local_dependencies_in_setup_py(python_package: Container) -> List[str]: - """Find local dependencies of a python package in its setup.py file. - - Args: - python_package (Container): A python package container. - - Returns: - List[str]: Paths to the local dependencies relative to the airbyte repo. - """ - setup_file_content = await get_file_contents(python_package, "setup.py") - if not setup_file_content: - return [] - - local_setup_dependency_paths = [] - with_egg_info = python_package.with_exec(["python", "setup.py", "egg_info"]) - egg_info_output = await with_egg_info.stdout() - dependency_in_requires_txt = [] - for line in egg_info_output.split("\n"): - if line.startswith("writing requirements to"): - # Find the path to the requirements.txt file that was generated by calling egg_info - requires_txt_path = line.replace("writing requirements to", "").strip() - requirements_txt_content = await with_egg_info.file(requires_txt_path).contents() - dependency_in_requires_txt = requirements_txt_content.split("\n") - - for dependency_line in dependency_in_requires_txt: - if "file://" in dependency_line: - match = re.search(r"file:///(.+)", dependency_line) - if match: - local_setup_dependency_paths.append([match.group(1)][0]) - return local_setup_dependency_paths - - -async def find_local_dependencies_in_requirements_txt(python_package: Container, package_source_code_path: str) -> List[str]: - """Find local dependencies of a python package in a requirements.txt file. - - Args: - python_package (Container): A python environment container with the python package source code. - package_source_code_path (str): The local path to the python package source code. - - Returns: - List[str]: Paths to the local dependencies relative to the airbyte repo. - """ - requirements_txt_content = await get_file_contents(python_package, "requirements.txt") - if not requirements_txt_content: - return [] - - local_requirements_dependency_paths = [] - for line in requirements_txt_content.split("\n"): - # Some package declare themselves as a requirement in requirements.txt, - # #Without line != "-e ." the package will be considered a dependency of itself which can cause an infinite loop - if line.startswith("-e .") and line != "-e .": - local_dependency_path = Path(line[3:]) - package_source_code_path = Path(package_source_code_path) - local_dependency_path = str((package_source_code_path / local_dependency_path).resolve().relative_to(Path.cwd())) - local_requirements_dependency_paths.append(local_dependency_path) - return local_requirements_dependency_paths - - -async def find_local_dependencies_in_pyproject_toml( - context: PipelineContext, - base_container: Container, - pyproject_file_path: str, - exclude: Optional[List] = None, -) -> list: - """Find local dependencies of a python package in a pyproject.toml file. - - Args: - python_package (Container): A python environment container with the python package source code. - pyproject_file_path (str): The path to the pyproject.toml file. - - Returns: - list: Paths to the local dependencies relative to the current directory. - """ - python_package = with_python_package(context, base_container, pyproject_file_path) - pyproject_content_raw = await get_file_contents(python_package, "pyproject.toml") - if not pyproject_content_raw: - return [] - - pyproject_content = toml.loads(pyproject_content_raw) - local_dependency_paths = [] - for dep, value in pyproject_content["tool"]["poetry"]["dependencies"].items(): - if isinstance(value, dict) and "path" in value: - local_dependency_path = Path(value["path"]) - pyproject_file_path = Path(pyproject_file_path) - local_dependency_path = str((pyproject_file_path / local_dependency_path).resolve().relative_to(Path.cwd())) - local_dependency_paths.append(local_dependency_path) - - # Ensure we parse the child dependencies - # TODO handle more than pyproject.toml - child_local_dependencies = await find_local_dependencies_in_pyproject_toml( - context, base_container, local_dependency_path, exclude=exclude - ) - local_dependency_paths += child_local_dependencies - - return local_dependency_paths - - -async def with_installed_python_package( - context: PipelineContext, - python_environment: Container, - package_source_code_path: str, - additional_dependency_groups: Optional[List] = None, - exclude: Optional[List] = None, -) -> Container: - """Install a python package in a python environment container. - - Args: - context (PipelineContext): The current test context, providing the repository directory from which the python sources will be pulled. - python_environment (Container): An existing python environment in which the package will be installed. - package_source_code_path (str): The local path to the package source code. - additional_dependency_groups (Optional[List]): extra_requires dependency of setup.py to install. Defaults to None. - exclude (Optional[List]): A list of file or directory to exclude from the python package source code. - - Returns: - Container: A python environment container with the python package installed. - """ - install_requirements_cmd = ["python", "-m", "pip", "install", "-r", "requirements.txt"] - install_connector_package_cmd = ["python", "-m", "pip", "install", "."] - - container = with_python_package(context, python_environment, package_source_code_path, exclude=exclude) - - local_dependencies = await find_local_python_dependencies(context, package_source_code_path) - - for dependency_directory in local_dependencies: - container = container.with_mounted_directory("/" + dependency_directory, context.get_repo_dir(dependency_directory)) - - has_setup_py, has_requirements_txt = await check_path_in_workdir(container, "setup.py"), await check_path_in_workdir( - container, "requirements.txt" - ) - - if has_setup_py: - container = container.with_exec(install_connector_package_cmd) - if has_requirements_txt: - container = container.with_exec(install_requirements_cmd) - - if additional_dependency_groups: - container = container.with_exec( - install_connector_package_cmd[:-1] + [install_connector_package_cmd[-1] + f"[{','.join(additional_dependency_groups)}]"] - ) - - return container - - -def with_python_connector_source(context: ConnectorContext) -> Container: - """Load an airbyte connector source code in a testing environment. - - Args: - context (ConnectorContext): The current test context, providing the repository directory from which the connector sources will be pulled. - Returns: - Container: A python environment container (with the connector source code). - """ - connector_source_path = str(context.connector.code_directory) - testing_environment: Container = with_testing_dependencies(context) - - return with_python_package(context, testing_environment, connector_source_path) - - -async def with_python_connector_installed(context: ConnectorContext) -> Container: - """Install an airbyte connector python package in a testing environment. - - Args: - context (ConnectorContext): The current test context, providing the repository directory from which the connector sources will be pulled. - Returns: - Container: A python environment container (with the connector installed). - """ - connector_source_path = str(context.connector.code_directory) - testing_environment: Container = with_testing_dependencies(context) - exclude = [ - f"{context.connector.code_directory}/{item}" - for item in [ - "secrets", - "metadata.yaml", - "bootstrap.md", - "icon.svg", - "README.md", - "Dockerfile", - "acceptance-test-docker.sh", - "build.gradle", - ".hypothesis", - ".dockerignore", - ] - ] - return await with_installed_python_package( - context, testing_environment, connector_source_path, additional_dependency_groups=["dev", "tests", "main"], exclude=exclude - ) - - -async def with_ci_credentials(context: PipelineContext, gsm_secret: Secret) -> Container: - """Install the ci_credentials package in a python environment. - - Args: - context (PipelineContext): The current test context, providing the repository directory from which the ci_credentials sources will be pulled. - gsm_secret (Secret): The secret holding GCP_GSM_CREDENTIALS env variable value. - - Returns: - Container: A python environment with the ci_credentials package installed. - """ - python_base_environment: Container = with_python_base(context) - ci_credentials = await with_installed_pipx_package(context, python_base_environment, CI_CREDENTIALS_SOURCE_PATH) - ci_credentials = ci_credentials.with_env_variable("VERSION", "dagger_ci") - return ci_credentials.with_secret_variable("GCP_GSM_CREDENTIALS", gsm_secret).with_workdir("/") - - -def with_alpine_packages(base_container: Container, packages_to_install: List[str]) -> Container: - """Installs packages using apk-get. - Args: - context (Container): A alpine based container. - - Returns: - Container: A container with the packages installed. - - """ - package_install_command = ["apk", "add"] - return base_container.with_exec(package_install_command + packages_to_install) - - -def with_debian_packages(base_container: Container, packages_to_install: List[str]) -> Container: - """Installs packages using apt-get. - Args: - context (Container): A alpine based container. - - Returns: - Container: A container with the packages installed. - - """ - update_packages_command = ["apt-get", "update"] - package_install_command = ["apt-get", "install", "-y"] - return base_container.with_exec(update_packages_command).with_exec(package_install_command + packages_to_install) - - -def with_pip_packages(base_container: Container, packages_to_install: List[str]) -> Container: - """Installs packages using pip - Args: - context (Container): A container with python installed - - Returns: - Container: A container with the pip packages installed. - - """ - package_install_command = ["pip", "install"] - return base_container.with_exec(package_install_command + packages_to_install) - - -async def with_connector_ops(context: PipelineContext) -> Container: - """Installs the connector_ops package in a Container running Python > 3.10 with git.. - - Args: - context (PipelineContext): The current test context, providing the repository directory from which the ci_connector_sources sources will be pulled. - - Returns: - Container: A python environment container with connector_ops installed. - """ - python_base_environment: Container = with_python_base(context) - - return await with_installed_pipx_package(context, python_base_environment, CONNECTOR_OPS_SOURCE_PATHSOURCE_PATH) - - -def with_global_dockerd_service(dagger_client: Client) -> Container: - """Create a container with a docker daemon running. - We expose its 2375 port to use it as a docker host for docker-in-docker use cases. - Args: - dagger_client (Client): The dagger client used to create the container. - Returns: - Container: The container running dockerd as a service - """ - return ( - dagger_client.container() - .from_(consts.DOCKER_DIND_IMAGE) - .with_mounted_cache( - "/tmp", - dagger_client.cache_volume("shared-tmp"), - ) - .with_exposed_port(2375) - .with_exec(["dockerd", "--log-level=error", "--host=tcp://0.0.0.0:2375", "--tls=false"], insecure_root_capabilities=True) - ) - - -def with_bound_docker_host( - context: ConnectorContext, - container: Container, -) -> Container: - """Bind a container to a docker host. It will use the dockerd service as a docker host. - - Args: - context (ConnectorContext): The current connector context. - container (Container): The container to bind to the docker host. - Returns: - Container: The container bound to the docker host. - """ - dockerd = context.dockerd_service - docker_hostname = "global-docker-host" - return ( - container.with_env_variable("DOCKER_HOST", f"tcp://{docker_hostname}:2375") - .with_service_binding(docker_hostname, dockerd) - .with_mounted_cache("/tmp", context.dagger_client.cache_volume("shared-tmp")) - ) - - -def bound_docker_host(context: ConnectorContext) -> Container: - def bound_docker_host_inner(container: Container) -> Container: - return with_bound_docker_host(context, container) - - return bound_docker_host_inner - - -def with_docker_cli(context: ConnectorContext) -> Container: - """Create a container with the docker CLI installed and bound to a persistent docker host. - - Args: - context (ConnectorContext): The current connector context. - - Returns: - Container: A docker cli container bound to a docker host. - """ - docker_cli = context.dagger_client.container().from_(consts.DOCKER_CLI_IMAGE) - return with_bound_docker_host(context, docker_cli) - - -def with_gradle( - context: ConnectorContext, - sources_to_include: List[str] = None, - bind_to_docker_host: bool = True, -) -> Container: - """Create a container with Gradle installed and bound to a persistent docker host. - - Args: - context (ConnectorContext): The current connector context. - sources_to_include (List[str], optional): List of additional source path to mount to the container. Defaults to None. - bind_to_docker_host (bool): Whether to bind the gradle container to a docker host. - - Returns: - Container: A container with Gradle installed and Java sources from the repository. - """ - - include = [ - ".root", - ".env", - "build.gradle", - "deps.toml", - "gradle.properties", - "gradle", - "gradlew", - "LICENSE_SHORT", - "publish-repositories.gradle", - "settings.gradle", - "build.gradle", - "tools/gradle", - "spotbugs-exclude-filter-file.xml", - "buildSrc", - "tools/bin/build_image.sh", - "tools/lib/lib.sh", - "tools/gradle/codestyle", - "pyproject.toml", - ] - - if sources_to_include: - include += sources_to_include - # TODO re-enable once we have fixed the over caching issue - # gradle_dependency_cache: CacheVolume = context.dagger_client.cache_volume("gradle-dependencies-caching") - # gradle_build_cache: CacheVolume = context.dagger_client.cache_volume(f"{context.connector.technical_name}-gradle-build-cache") - - openjdk_with_docker = ( - context.dagger_client.container() - .from_("openjdk:17.0.1-jdk-slim") - .with_exec(["apt-get", "update"]) - .with_exec(["apt-get", "install", "-y", "curl", "jq", "rsync", "npm", "pip"]) - .with_env_variable("VERSION", consts.DOCKER_VERSION) - .with_exec(["sh", "-c", "curl -fsSL https://get.docker.com | sh"]) - .with_env_variable("GRADLE_HOME", "/root/.gradle") - .with_exec(["mkdir", "/airbyte"]) - .with_workdir("/airbyte") - .with_mounted_directory("/airbyte", context.get_repo_dir(".", include=include)) - .with_exec(["mkdir", "-p", consts.GRADLE_READ_ONLY_DEPENDENCY_CACHE_PATH]) - # TODO (ben) reenable once we have fixed the over caching issue - # .with_mounted_cache(consts.GRADLE_BUILD_CACHE_PATH, gradle_build_cache, sharing=CacheSharingMode.LOCKED) - # .with_mounted_cache(consts.GRADLE_READ_ONLY_DEPENDENCY_CACHE_PATH, gradle_dependency_cache) - .with_env_variable("GRADLE_RO_DEP_CACHE", consts.GRADLE_READ_ONLY_DEPENDENCY_CACHE_PATH) - ) - - if bind_to_docker_host: - return with_bound_docker_host(context, openjdk_with_docker) - else: - return openjdk_with_docker - - -async def load_image_to_docker_host(context: ConnectorContext, tar_file: File, image_tag: str): - """Load a docker image tar archive to the docker host. - - Args: - context (ConnectorContext): The current connector context. - tar_file (File): The file object holding the docker image tar archive. - image_tag (str): The tag to create on the image if it has no tag. - """ - # Hacky way to make sure the image is always loaded - tar_name = f"{str(uuid.uuid4())}.tar" - docker_cli = with_docker_cli(context).with_mounted_file(tar_name, tar_file) - - image_load_output = await docker_cli.with_exec(["docker", "load", "--input", tar_name]).stdout() - # Not tagged images only have a sha256 id the load output shares. - if "sha256:" in image_load_output: - image_id = image_load_output.replace("\n", "").replace("Loaded image ID: sha256:", "") - await docker_cli.with_exec(["docker", "tag", image_id, image_tag]) - image_sha = json.loads(await docker_cli.with_exec(["docker", "inspect", image_tag]).stdout())[0].get("Id") - return image_sha - - -def with_pipx(base_python_container: Container) -> Container: - """Installs pipx in a python container. - - Args: - base_python_container (Container): The container to install pipx on. - - Returns: - Container: A python environment with pipx installed. - """ - python_with_pipx = with_pip_packages(base_python_container, ["pipx"]).with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") - - return python_with_pipx - - -def with_poetry(context: PipelineContext) -> Container: - """Install poetry in a python environment. - - Args: - context (PipelineContext): The current test context, providing the repository directory from which the ci_credentials sources will be pulled. - Returns: - Container: A python environment with poetry installed. - """ - python_base_environment: Container = with_python_base(context) - python_with_git = with_debian_packages(python_base_environment, ["git"]) - python_with_poetry = with_pip_packages(python_with_git, ["poetry"]) - - # poetry_cache: CacheVolume = context.dagger_client.cache_volume("poetry_cache") - # poetry_with_cache = python_with_poetry.with_mounted_cache("/root/.cache/pypoetry", poetry_cache, sharing=CacheSharingMode.SHARED) - - return python_with_poetry - - -def with_poetry_module(context: PipelineContext, parent_dir: Directory, module_path: str) -> Container: - """Sets up a Poetry module. - - Args: - context (PipelineContext): The current test context, providing the repository directory from which the ci_credentials sources will be pulled. - Returns: - Container: A python environment with dependencies installed using poetry. - """ - poetry_install_dependencies_cmd = ["poetry", "install"] - - python_with_poetry = with_poetry(context) - return ( - python_with_poetry.with_mounted_directory("/src", parent_dir) - .with_workdir(f"/src/{module_path}") - .with_exec(poetry_install_dependencies_cmd) - .with_env_variable("CACHEBUSTER", str(uuid.uuid4())) - ) - - -def with_integration_base(context: PipelineContext, build_platform: Platform) -> Container: - return ( - context.dagger_client.container(platform=build_platform) - .from_("amazonlinux:2022.0.20220831.1") - .with_workdir("/airbyte") - .with_file("base.sh", context.get_repo_dir("airbyte-integrations/bases/base", include=["base.sh"]).file("base.sh")) - .with_env_variable("AIRBYTE_ENTRYPOINT", "/airbyte/base.sh") - .with_label("io.airbyte.version", "0.1.0") - .with_label("io.airbyte.name", "airbyte/integration-base") - ) - - -def with_integration_base_java(context: PipelineContext, build_platform: Platform, jdk_version: str = "17.0.4") -> Container: - integration_base = with_integration_base(context, build_platform) - return ( - context.dagger_client.container(platform=build_platform) - .from_(f"amazoncorretto:{jdk_version}") - .with_directory("/airbyte", integration_base.directory("/airbyte")) - .with_exec(["yum", "install", "-y", "tar", "openssl"]) - .with_exec(["yum", "clean", "all"]) - .with_workdir("/airbyte") - .with_file("dd-java-agent.jar", context.dagger_client.http("https://dtdg.co/latest-java-tracer")) - .with_file("javabase.sh", context.get_repo_dir("airbyte-integrations/bases/base-java", include=["javabase.sh"]).file("javabase.sh")) - .with_env_variable("AIRBYTE_SPEC_CMD", "/airbyte/javabase.sh --spec") - .with_env_variable("AIRBYTE_CHECK_CMD", "/airbyte/javabase.sh --check") - .with_env_variable("AIRBYTE_DISCOVER_CMD", "/airbyte/javabase.sh --discover") - .with_env_variable("AIRBYTE_READ_CMD", "/airbyte/javabase.sh --read") - .with_env_variable("AIRBYTE_WRITE_CMD", "/airbyte/javabase.sh --write") - .with_env_variable("AIRBYTE_ENTRYPOINT", "/airbyte/base.sh") - .with_label("io.airbyte.version", "0.1.2") - .with_label("io.airbyte.name", "airbyte/integration-base-java") - ) - - -BASE_DESTINATION_NORMALIZATION_BUILD_CONFIGURATION = { - "destination-bigquery": { - "dockerfile": "Dockerfile", - "dbt_adapter": "dbt-bigquery==1.0.0", - "integration_name": "bigquery", - "normalization_image": "airbyte/normalization:0.4.3", - "supports_in_connector_normalization": True, - "yum_packages": [], - }, - "destination-clickhouse": { - "dockerfile": "clickhouse.Dockerfile", - "dbt_adapter": "dbt-clickhouse>=1.4.0", - "integration_name": "clickhouse", - "normalization_image": "airbyte/normalization-clickhouse:0.4.3", - "supports_in_connector_normalization": False, - "yum_packages": [], - }, - "destination-duckdb": { - "dockerfile": "duckdb.Dockerfile", - "dbt_adapter": "dbt-duckdb==1.0.1", - "integration_name": "duckdb", - "normalization_image": "airbyte/normalization-duckdb:0.4.3", - "supports_in_connector_normalization": False, - "yum_packages": [], - }, - "destination-mssql": { - "dockerfile": "mssql.Dockerfile", - "dbt_adapter": "dbt-sqlserver==1.0.0", - "integration_name": "mssql", - "normalization_image": "airbyte/normalization-mssql:0.4.3", - "supports_in_connector_normalization": True, - "yum_packages": [], - }, - "destination-mysql": { - "dockerfile": "mysql.Dockerfile", - "dbt_adapter": "dbt-mysql==1.0.0", - "integration_name": "mysql", - "normalization_image": "airbyte/normalization-mysql:0.4.3", - "supports_in_connector_normalization": False, - "yum_packages": [], - }, - "destination-oracle": { - "dockerfile": "oracle.Dockerfile", - "dbt_adapter": "dbt-oracle==0.4.3", - "integration_name": "oracle", - "normalization_image": "airbyte/normalization-oracle:0.4.3", - "supports_in_connector_normalization": False, - "yum_packages": [], - }, - "destination-postgres": { - "dockerfile": "Dockerfile", - "dbt_adapter": "dbt-postgres==1.0.0", - "integration_name": "postgres", - "normalization_image": "airbyte/normalization:0.4.3", - "supports_in_connector_normalization": False, - "yum_packages": [], - }, - "destination-redshift": { - "dockerfile": "redshift.Dockerfile", - "dbt_adapter": "dbt-redshift==1.0.0", - "integration_name": "redshift", - "normalization_image": "airbyte/normalization-redshift:0.4.3", - "supports_in_connector_normalization": True, - "yum_packages": [], - }, - "destination-snowflake": { - "dockerfile": "snowflake.Dockerfile", - "dbt_adapter": "dbt-snowflake==1.0.0", - "integration_name": "snowflake", - "normalization_image": "airbyte/normalization-snowflake:0.4.3", - "supports_in_connector_normalization": True, - "yum_packages": ["gcc-c++"], - }, - "destination-tidb": { - "dockerfile": "tidb.Dockerfile", - "dbt_adapter": "dbt-tidb==1.0.1", - "integration_name": "tidb", - "normalization_image": "airbyte/normalization-tidb:0.4.3", - "supports_in_connector_normalization": True, - "yum_packages": [], - }, -} - -DESTINATION_NORMALIZATION_BUILD_CONFIGURATION = { - **BASE_DESTINATION_NORMALIZATION_BUILD_CONFIGURATION, - **{f"{k}-strict-encrypt": v for k, v in BASE_DESTINATION_NORMALIZATION_BUILD_CONFIGURATION.items()}, -} - - -def with_normalization(context: ConnectorContext, build_platform: Platform) -> Container: - return context.dagger_client.container(platform=build_platform).from_( - DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["normalization_image"] - ) - - -def with_integration_base_java_and_normalization(context: PipelineContext, build_platform: Platform) -> Container: - yum_packages_to_install = [ - "python3", - "python3-devel", - "jq", - "sshpass", - "git", - ] - - additional_yum_packages = DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["yum_packages"] - yum_packages_to_install += additional_yum_packages - - dbt_adapter_package = DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["dbt_adapter"] - normalization_integration_name = DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["integration_name"] - - pip_cache: CacheVolume = context.dagger_client.cache_volume("pip_cache") - - return ( - with_integration_base_java(context, build_platform) - .with_exec(["yum", "install", "-y"] + yum_packages_to_install) - .with_exec(["yum", "clean", "all"]) - .with_exec(["alternatives", "--install", "/usr/bin/python", "python", "/usr/bin/python3", "60"]) - .with_mounted_cache("/root/.cache/pip", pip_cache) - .with_exec(["python", "-m", "ensurepip", "--upgrade"]) - # Workaround for https://github.com/yaml/pyyaml/issues/601 - .with_exec(["pip3", "install", "Cython<3.0", "pyyaml~=5.4", "--no-build-isolation"]) - .with_exec(["pip3", "install", dbt_adapter_package]) - .with_directory("airbyte_normalization", with_normalization(context, build_platform).directory("/airbyte")) - .with_workdir("airbyte_normalization") - .with_exec(["sh", "-c", "mv * .."]) - .with_workdir("/airbyte") - .with_exec(["rm", "-rf", "airbyte_normalization"]) - # We don't install the airbyte-protocol legacy package as its not used anymore and not compatible with Cython 3.x - # .with_workdir("/airbyte/base_python_structs") - # .with_exec(["pip3", "install", "--force-reinstall", "Cython<3.0", ".",]) - .with_workdir("/airbyte/normalization_code") - .with_exec(["pip3", "install", "."]) - .with_workdir("/airbyte/normalization_code/dbt-template/") - # amazon linux 2 isn't compatible with urllib3 2.x, so force 1.x - .with_exec(["pip3", "install", "urllib3<2"]) - .with_exec(["dbt", "deps"]) - .with_workdir("/airbyte") - .with_file( - "run_with_normalization.sh", - context.get_repo_dir("airbyte-integrations/bases/base-java", include=["run_with_normalization.sh"]).file( - "run_with_normalization.sh" - ), - ) - .with_env_variable("AIRBYTE_NORMALIZATION_INTEGRATION", normalization_integration_name) - .with_env_variable("AIRBYTE_ENTRYPOINT", "/airbyte/run_with_normalization.sh") - ) - - -async def with_airbyte_java_connector(context: ConnectorContext, connector_java_tar_file: File, build_platform: Platform) -> Container: - application = context.connector.technical_name - - build_stage = ( - with_integration_base_java(context, build_platform) - .with_workdir("/airbyte") - .with_env_variable("APPLICATION", context.connector.technical_name) - .with_file(f"{application}.tar", connector_java_tar_file) - .with_exec(["tar", "xf", f"{application}.tar", "--strip-components=1"]) - .with_exec(["rm", "-rf", f"{application}.tar"]) - ) - - if ( - context.connector.supports_normalization - and DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["supports_in_connector_normalization"] - ): - base = with_integration_base_java_and_normalization(context, build_platform) - entrypoint = ["/airbyte/run_with_normalization.sh"] - else: - base = with_integration_base_java(context, build_platform) - entrypoint = ["/airbyte/base.sh"] - - connector_container = ( - base.with_workdir("/airbyte") - .with_env_variable("APPLICATION", application) - .with_mounted_directory("builts_artifacts", build_stage.directory("/airbyte")) - .with_exec(["sh", "-c", "mv builts_artifacts/* ."]) - .with_label("io.airbyte.version", context.metadata["dockerImageTag"]) - .with_label("io.airbyte.name", context.metadata["dockerRepository"]) - .with_entrypoint(entrypoint) - ) - return await finalize_build(context, connector_container) - - -async def get_cdk_version_from_python_connector(python_connector: Container) -> Optional[str]: - pip_freeze_stdout = await python_connector.with_entrypoint("pip").with_exec(["freeze"]).stdout() - pip_dependencies = [dep.split("==") for dep in pip_freeze_stdout.split("\n")] - for package_name, package_version in pip_dependencies: - if package_name == "airbyte-cdk": - return package_version - return None - - -async def with_airbyte_python_connector(context: ConnectorContext, build_platform: Platform) -> Container: - if context.connector.technical_name == "source-file-secure": - return await with_airbyte_python_connector_full_dagger(context, build_platform) - - pip_cache: CacheVolume = context.dagger_client.cache_volume("pip_cache") - connector_container = ( - context.dagger_client.container(platform=build_platform) - .with_mounted_cache("/root/.cache/pip", pip_cache) - .build(await context.get_connector_dir()) - .with_label("io.airbyte.name", context.metadata["dockerRepository"]) - ) - cdk_version = await get_cdk_version_from_python_connector(connector_container) - if cdk_version: - connector_container = connector_container.with_label("io.airbyte.cdk_version", cdk_version) - context.cdk_version = cdk_version - if not await connector_container.label("io.airbyte.version") == context.metadata["dockerImageTag"]: - raise DaggerError( - "Abusive caching might be happening. The connector container should have been built with the correct version as defined in metadata.yaml" - ) - return await finalize_build(context, connector_container) - - -async def finalize_build(context: ConnectorContext, connector_container: Container) -> Container: - """Finalize build by adding dagger engine version label and running finalize_build.sh or finalize_build.py if present in the connector directory.""" - connector_container = connector_container.with_label("io.dagger.engine_version", dagger_engine_version) - connector_dir_with_finalize_script = await context.get_connector_dir(include=["finalize_build.sh", "finalize_build.py"]) - finalize_scripts = await connector_dir_with_finalize_script.entries() - if not finalize_scripts: - return connector_container - - # We don't want finalize scripts to override the entrypoint so we keep it in memory to reset it after finalization - original_entrypoint = await connector_container.entrypoint() - - has_finalize_bash_script = "finalize_build.sh" in finalize_scripts - has_finalize_python_script = "finalize_build.py" in finalize_scripts - if has_finalize_python_script and has_finalize_bash_script: - raise Exception("Connector has both finalize_build.sh and finalize_build.py, please remove one of them") - - if has_finalize_python_script: - context.logger.info(f"{context.connector.technical_name} has a finalize_build.py script, running it to finalize build...") - module_path = context.connector.code_directory / "finalize_build.py" - connector_finalize_module_spec = importlib.util.spec_from_file_location( - f"{context.connector.code_directory.name}_finalize", module_path - ) - connector_finalize_module = importlib.util.module_from_spec(connector_finalize_module_spec) - connector_finalize_module_spec.loader.exec_module(connector_finalize_module) - try: - connector_container = await connector_finalize_module.finalize_build(context, connector_container) - except AttributeError: - raise Exception("Connector has a finalize_build.py script but it doesn't have a finalize_build function.") - - if has_finalize_bash_script: - context.logger.info(f"{context.connector.technical_name} has finalize_build.sh script, running it to finalize build...") - connector_container = ( - connector_container.with_file("/tmp/finalize_build.sh", connector_dir_with_finalize_script.file("finalize_build.sh")) - .with_entrypoint("sh") - .with_exec(["/tmp/finalize_build.sh"]) - ) - - return connector_container.with_entrypoint(original_entrypoint) - - -async def with_airbyte_python_connector_full_dagger(context: ConnectorContext, build_platform: Platform) -> Container: - setup_dependencies_to_mount = await find_local_python_dependencies( - context, str(context.connector.code_directory), search_dependencies_in_setup_py=True, search_dependencies_in_requirements_txt=False - ) - - pip_cache: CacheVolume = context.dagger_client.cache_volume("pip_cache") - base = context.dagger_client.container(platform=build_platform).from_("python:3.9-slim") - snake_case_name = context.connector.technical_name.replace("-", "_") - entrypoint = ["python", "/airbyte/integration_code/main.py"] - builder = ( - base.with_workdir("/airbyte/integration_code") - .with_env_variable("DAGGER_BUILD", "True") - .with_exec(["apt-get", "update"]) - .with_mounted_cache("/root/.cache/pip", pip_cache) - .with_exec(["pip", "install", "--upgrade", "pip"]) - .with_exec(["apt-get", "install", "-y", "tzdata"]) - .with_file("setup.py", (await context.get_connector_dir(include="setup.py")).file("setup.py")) - ) - - for dependency_path in setup_dependencies_to_mount: - in_container_dependency_path = f"/local_dependencies/{Path(dependency_path).name}" - builder = builder.with_mounted_directory(in_container_dependency_path, context.get_repo_dir(dependency_path)) - - builder = builder.with_exec(["pip", "install", "--prefix=/install", "."]) - - connector_container = ( - base.with_workdir("/airbyte/integration_code") - .with_directory("/usr/local", builder.directory("/install")) - .with_file("/usr/localtime", builder.file("/usr/share/zoneinfo/Etc/UTC")) - .with_new_file("/etc/timezone", contents="Etc/UTC") - .with_exec(["apt-get", "install", "-y", "bash"]) - .with_file("main.py", (await context.get_connector_dir(include="main.py")).file("main.py")) - .with_directory(snake_case_name, (await context.get_connector_dir(include=snake_case_name)).directory(snake_case_name)) - .with_env_variable("AIRBYTE_ENTRYPOINT", " ".join(entrypoint)) - .with_entrypoint(entrypoint) - .with_label("io.airbyte.version", context.metadata["dockerImageTag"]) - .with_label("io.airbyte.name", context.metadata["dockerRepository"]) - ) - return await finalize_build(context, connector_container) - - -def with_crane( - context: PipelineContext, -) -> Container: - """Crane is a tool to analyze and manipulate container images. - We can use it to extract the image manifest and the list of layers or list the existing tags on an image repository. - https://github.com/google/go-containerregistry/tree/main/cmd/crane - """ - - # We use the debug image as it contains a shell which we need to properly use environment variables - # https://github.com/google/go-containerregistry/tree/main/cmd/crane#images - base_container = context.dagger_client.container().from_("gcr.io/go-containerregistry/crane/debug:v0.15.1") - - if context.docker_hub_username_secret and context.docker_hub_password_secret: - base_container = ( - base_container.with_secret_variable("DOCKER_HUB_USERNAME", context.docker_hub_username_secret).with_secret_variable( - "DOCKER_HUB_PASSWORD", context.docker_hub_password_secret - ) - # We need to use skip_entrypoint=True to avoid the entrypoint to be overridden by the crane command - # We use sh -c to be able to use environment variables in the command - # This is a workaround as the default crane entrypoint doesn't support environment variables - .with_exec( - ["sh", "-c", "crane auth login index.docker.io -u $DOCKER_HUB_USERNAME -p $DOCKER_HUB_PASSWORD"], skip_entrypoint=True - ) - ) - - return base_container - - -def mounted_connector_secrets(context: PipelineContext, secret_directory_path="secrets") -> Callable: - def mounted_connector_secrets_inner(container: Container): - container = container.with_exec(["mkdir", secret_directory_path], skip_entrypoint=True) - for secret_file_name, secret in context.connector_secrets.items(): - container = container.with_mounted_secret(f"{secret_directory_path}/{secret_file_name}", secret) - return container - - return mounted_connector_secrets_inner diff --git a/airbyte-ci/connectors/pipelines/pipelines/actions/secrets.py b/airbyte-ci/connectors/pipelines/pipelines/actions/secrets.py deleted file mode 100644 index 985ca064b5b9..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/actions/secrets.py +++ /dev/null @@ -1,104 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -"""This modules groups functions made to download/upload secrets from/to a remote secret service and provide these secret in a dagger Directory.""" -from __future__ import annotations - -import datetime -from typing import TYPE_CHECKING - -from dagger import Secret -from pipelines.actions import environments -from pipelines.utils import get_file_contents, get_secret_host_variable - -if TYPE_CHECKING: - from dagger import Container - from pipelines.contexts import ConnectorContext - - -async def get_secrets_to_mask(ci_credentials_with_downloaded_secrets: Container) -> list[str]: - """This function will print the secrets to mask in the GitHub actions logs with the ::add-mask:: prefix. - We're not doing it directly from the ci_credentials tool because its stdout is wrapped around the dagger logger, - And GHA will only interpret lines starting with ::add-mask:: as secrets to mask. - """ - secrets_to_mask = [] - if secrets_to_mask_file := await get_file_contents(ci_credentials_with_downloaded_secrets, "/tmp/secrets_to_mask.txt"): - for secret_to_mask in secrets_to_mask_file.splitlines(): - # We print directly to stdout because the GHA runner will mask only if the log line starts with "::add-mask::" - # If we use the dagger logger, or context logger, the log line will start with other stuff and will not be masked - print(f"::add-mask::{secret_to_mask}") - secrets_to_mask.append(secret_to_mask) - return secrets_to_mask - - -async def download(context: ConnectorContext, gcp_gsm_env_variable_name: str = "GCP_GSM_CREDENTIALS") -> dict[str, Secret]: - """Use the ci-credentials tool to download the secrets stored for a specific connector to a Directory. - - Args: - context (ConnectorContext): The context providing a connector object. - gcp_gsm_env_variable_name (str, optional): The name of the environment variable holding credentials to connect to Google Secret Manager. Defaults to "GCP_GSM_CREDENTIALS". - - Returns: - Directory: A directory with the downloaded secrets. - """ - gsm_secret = get_secret_host_variable(context.dagger_client, gcp_gsm_env_variable_name) - secrets_path = f"/{context.connector.code_directory}/secrets" - ci_credentials = await environments.with_ci_credentials(context, gsm_secret) - with_downloaded_secrets = ( - ci_credentials.with_exec(["mkdir", "-p", secrets_path]) - .with_env_variable( - "CACHEBUSTER", datetime.datetime.now().isoformat() - ) # Secrets can be updated on GSM anytime, we can't cache this step... - .with_exec(["ci_credentials", context.connector.technical_name, "write-to-storage"]) - ) - # We don't want to print secrets in the logs when running locally. - if context.is_ci: - context.secrets_to_mask = await get_secrets_to_mask(with_downloaded_secrets) - connector_secrets = {} - for secret_file in await with_downloaded_secrets.directory(secrets_path).entries(): - secret_plaintext = await with_downloaded_secrets.directory(secrets_path).file(secret_file).contents() - # We have to namespace secrets as Dagger derives session wide secret ID from their name - unique_secret_name = f"{context.connector.technical_name}_{secret_file}" - connector_secrets[secret_file] = context.dagger_client.set_secret(unique_secret_name, secret_plaintext) - - return connector_secrets - - -async def upload(context: ConnectorContext, gcp_gsm_env_variable_name: str = "GCP_GSM_CREDENTIALS"): - """Use the ci-credentials tool to upload the secrets stored in the context's updated_secrets-dir. - - Args: - context (ConnectorContext): The context providing a connector object and the update secrets dir. - gcp_gsm_env_variable_name (str, optional): The name of the environment variable holding credentials to connect to Google Secret Manager. Defaults to "GCP_GSM_CREDENTIALS". - - Returns: - container (Container): The executed ci-credentials update-secrets command. - - Raises: - ExecError: If the command returns a non-zero exit code. - """ - gsm_secret = get_secret_host_variable(context.dagger_client, gcp_gsm_env_variable_name) - secrets_path = f"/{context.connector.code_directory}/secrets" - - ci_credentials = await environments.with_ci_credentials(context, gsm_secret) - - return await ci_credentials.with_directory(secrets_path, context.updated_secrets_dir).with_exec( - ["ci_credentials", context.connector.technical_name, "update-secrets"] - ) - - -async def get_connector_secrets(context: ConnectorContext) -> dict[str, Secret]: - """Download the secrets from GSM or use the local secrets directory for a connector. - - Args: - context (ConnectorContext): The context providing the connector directory and the use_remote_secrets flag. - - Returns: - Directory: A directory with the downloaded connector secrets. - """ - if context.use_remote_secrets: - connector_secrets = await download(context) - else: - raise NotImplementedError("Local secrets are not implemented yet. See https://github.com/airbytehq/airbyte/issues/25621") - return connector_secrets diff --git a/airbyte-integrations/connectors/source-nasa/unit_tests/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-nasa/unit_tests/__init__.py rename to airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/__init__.py diff --git a/airbyte-integrations/connectors/source-orbit/unit_tests/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-orbit/unit_tests/__init__.py rename to airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/__init__.py diff --git a/airbyte-integrations/connectors/source-public-apis/unit_tests/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-public-apis/unit_tests/__init__.py rename to airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/__init__.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py new file mode 100644 index 000000000000..6b6624391f47 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py @@ -0,0 +1,82 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import List + +import asyncclick as click +import dagger +from pipelines import main_logger +from pipelines.airbyte_ci.connectors.build_image.steps import run_connector_build_pipeline +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines +from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand +from pipelines.consts import BUILD_PLATFORMS, LOCAL_BUILD_PLATFORM + + +@click.command(cls=DaggerPipelineCommand, help="Build all images for the selected connectors.") +@click.option( + "--use-host-gradle-dist-tar", + is_flag=True, + help="Use gradle distTar output from host for java connectors.", + default=False, + type=bool, +) +@click.option( + "-a", + "--architecture", + "build_architectures", + help="Architecture for which to build the connector image. If not specified, the image will be built for the local architecture.", + multiple=True, + default=[LOCAL_BUILD_PLATFORM], + type=click.Choice(BUILD_PLATFORMS, case_sensitive=True), +) +@click.option( + "-t", + "--tag", + help="The tag to use for the built image.", + default="dev", + type=str, +) +@click.pass_context +async def build(ctx: click.Context, use_host_gradle_dist_tar: bool, build_architectures: List[str], tag: str) -> bool: + """Runs a build pipeline for the selected connectors.""" + build_platforms = [dagger.Platform(architecture) for architecture in build_architectures] + main_logger.info(f"Building connectors for {build_platforms}, use --architecture to change this.") + connectors_contexts = [ + ConnectorContext( + pipeline_name=f"Build connector {connector.technical_name}", + connector=connector, + is_local=ctx.obj["is_local"], + git_branch=ctx.obj["git_branch"], + git_revision=ctx.obj["git_revision"], + ci_report_bucket=ctx.obj["ci_report_bucket_name"], + report_output_prefix=ctx.obj["report_output_prefix"], + use_remote_secrets=ctx.obj["use_remote_secrets"], + gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), + dagger_logs_url=ctx.obj.get("dagger_logs_url"), + pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), + ci_context=ctx.obj.get("ci_context"), + ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], + use_local_cdk=ctx.obj.get("use_local_cdk"), + enable_report_auto_open=ctx.obj.get("enable_report_auto_open"), + use_host_gradle_dist_tar=use_host_gradle_dist_tar, + s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"), + s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"), + targeted_platforms=build_platforms, + ) + for connector in ctx.obj["selected_connectors_with_modified_files"] + ] + if use_host_gradle_dist_tar and not ctx.obj["is_local"]: + raise Exception("flag --use-host-gradle-dist-tar requires --is-local") + await run_connectors_pipelines( + connectors_contexts, + run_connector_build_pipeline, + "Build Pipeline", + ctx.obj["concurrency"], + ctx.obj["dagger_logs_path"], + ctx.obj["execute_timeout"], + tag, + ) + + return True diff --git a/airbyte-integrations/connectors/source-qonto/unit_tests/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/pipeline.py similarity index 100% rename from airbyte-integrations/connectors/source-qonto/unit_tests/__init__.py rename to airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/pipeline.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/__init__.py new file mode 100644 index 000000000000..7a47568639a6 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/__init__.py @@ -0,0 +1,57 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +"""This module groups factory like functions to dispatch builds steps according to the connector language.""" + +from __future__ import annotations + +import anyio +from connector_ops.utils import ConnectorLanguage # type: ignore +from pipelines.airbyte_ci.connectors.build_image.steps import java_connectors, python_connectors +from pipelines.airbyte_ci.connectors.build_image.steps.common import LoadContainerToLocalDockerHost, StepStatus +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.connectors.reports import ConnectorReport, Report +from pipelines.models.steps import StepResult + + + +class NoBuildStepForLanguageError(Exception): + pass + + +LANGUAGE_BUILD_CONNECTOR_MAPPING = { + ConnectorLanguage.PYTHON: python_connectors.run_connector_build, + ConnectorLanguage.LOW_CODE: python_connectors.run_connector_build, + ConnectorLanguage.JAVA: java_connectors.run_connector_build, +} + + +async def run_connector_build(context: ConnectorContext) -> StepResult: + """Run a build pipeline for a single connector.""" + if context.connector.language not in LANGUAGE_BUILD_CONNECTOR_MAPPING: + raise NoBuildStepForLanguageError(f"No build step for connector language {context.connector.language}.") + return await LANGUAGE_BUILD_CONNECTOR_MAPPING[context.connector.language](context) + + +async def run_connector_build_pipeline(context: ConnectorContext, semaphore: anyio.Semaphore, image_tag: str) -> Report: + """Run a build pipeline for a single connector. + + Args: + context (ConnectorContext): The initialized connector context. + semaphore (anyio.Semaphore): The semaphore to use to limit the number of concurrent builds. + image_tag (str): The tag to use for the built image. + Returns: + ConnectorReport: The reports holding builds results. + """ + step_results = [] + async with semaphore: + async with context: + build_result = await run_connector_build(context) + per_platform_built_containers = build_result.output_artifact + step_results.append(build_result) + if context.is_local and build_result.status is StepStatus.SUCCESS: + load_image_result = await LoadContainerToLocalDockerHost(context, per_platform_built_containers, image_tag).run() + step_results.append(load_image_result) + report = ConnectorReport(context, step_results, name="BUILD RESULTS") + context.report = report + return report diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/build_customization.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/build_customization.py new file mode 100644 index 000000000000..818aa3163843 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/build_customization.py @@ -0,0 +1,96 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import importlib +from logging import Logger +from types import ModuleType +from typing import List, Optional + +from connector_ops.utils import Connector # type: ignore +from dagger import Container + +BUILD_CUSTOMIZATION_MODULE_NAME = "build_customization" +BUILD_CUSTOMIZATION_SPEC_NAME = f"{BUILD_CUSTOMIZATION_MODULE_NAME}.py" +DEFAULT_MAIN_FILE_NAME = "main.py" + + +def get_build_customization_module(connector: Connector) -> Optional[ModuleType]: + """Import the build_customization.py file from the connector directory if it exists. + Returns: + Optional[ModuleType]: The build_customization.py module if it exists, None otherwise. + """ + build_customization_spec_path = connector.code_directory / BUILD_CUSTOMIZATION_SPEC_NAME + + if not build_customization_spec_path.exists() or not (build_customization_spec := importlib.util.spec_from_file_location( + f"{connector.code_directory.name}_{BUILD_CUSTOMIZATION_MODULE_NAME}", build_customization_spec_path + )): + return None + + if build_customization_spec.loader is None: + return None + + build_customization_module = importlib.util.module_from_spec(build_customization_spec) + build_customization_spec.loader.exec_module(build_customization_module) + return build_customization_module + + +def get_main_file_name(connector: Connector) -> str: + """Get the main file name from the build_customization.py module if it exists, DEFAULT_MAIN_FILE_NAME otherwise. + + Args: + connector (Connector): The connector to build. + + Returns: + str: The main file name. + """ + build_customization_module = get_build_customization_module(connector) + + return ( + build_customization_module.MAIN_FILE_NAME + if build_customization_module and hasattr(build_customization_module, "MAIN_FILE_NAME") + else DEFAULT_MAIN_FILE_NAME + ) + + +def get_entrypoint(connector: Connector) -> List[str]: + main_file_name = get_main_file_name(connector) + return ["python", f"/airbyte/integration_code/{main_file_name}"] + + +async def pre_install_hooks(connector: Connector, base_container: Container, logger: Logger) -> Container: + """Run the pre_connector_install hook if it exists in the build_customization.py module. + It will mutate the base_container and return it. + + Args: + connector (Connector): The connector to build. + base_container (Container): The base container to mutate. + logger (Logger): The logger to use. + + Returns: + Container: The mutated base_container. + """ + build_customization_module = get_build_customization_module(connector) + if build_customization_module and hasattr(build_customization_module, "pre_connector_install"): + base_container = await build_customization_module.pre_connector_install(base_container) + logger.info(f"Connector {connector.technical_name} pre install hook executed.") + return base_container + + +async def post_install_hooks(connector: Connector, connector_container: Container, logger: Logger) -> Container: + """Run the post_connector_install hook if it exists in the build_customization.py module. + It will mutate the connector_container and return it. + + Args: + connector (Connector): The connector to build. + connector_container (Container): The connector container to mutate. + logger (Logger): The logger to use. + + Returns: + Container: The mutated connector_container. + """ + build_customization_module = get_build_customization_module(connector) + if build_customization_module and hasattr(build_customization_module, "post_connector_install"): + connector_container = await build_customization_module.post_connector_install(connector_container) + logger.info(f"Connector {connector.technical_name} post install hook executed.") + return connector_container diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py new file mode 100644 index 000000000000..2b03a45b3f75 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py @@ -0,0 +1,110 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from __future__ import annotations + +from abc import ABC +from typing import TYPE_CHECKING + +import docker # type: ignore +from dagger import Container, ExecError, Platform, QueryError +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.helpers.utils import export_container_to_tarball +from pipelines.models.steps import Step, StepResult, StepStatus + +if TYPE_CHECKING: + from typing import Any + +class BuildConnectorImagesBase(Step, ABC): + """ + A step to build connector images for a set of platforms. + """ + + context: ConnectorContext + + @property + def title(self) -> str: + return f"Build {self.context.connector.technical_name} docker image for platform(s) {', '.join(self.build_platforms)}" + + def __init__(self, context: ConnectorContext) -> None: + self.build_platforms = context.targeted_platforms + super().__init__(context) + + async def _run(self, *args: Any) -> StepResult: + build_results_per_platform = {} + for platform in self.build_platforms: + try: + connector = await self._build_connector(platform, *args) + try: + await connector.with_exec(["spec"]) + except ExecError: + return StepResult( + self, StepStatus.FAILURE, stderr=f"Failed to run spec on the connector built for platform {platform}." + ) + build_results_per_platform[platform] = connector + except QueryError as e: + return StepResult(self, StepStatus.FAILURE, stderr=f"Failed to build connector image for platform {platform}: {e}") + success_message = ( + f"The {self.context.connector.technical_name} docker image " + f"was successfully built for platform(s) {', '.join(self.build_platforms)}" + ) + return StepResult(self, StepStatus.SUCCESS, stdout=success_message, output_artifact=build_results_per_platform) + + async def _build_connector(self, platform: Platform, *args: Any, **kwargs: Any) -> Container: + """Implement the generation of the image for the platform and return the corresponding container. + + Returns: + Container: The container to package as a docker image for this platform. + """ + raise NotImplementedError("`BuildConnectorImagesBase`s must define a '_build_connector' attribute.") + + +class LoadContainerToLocalDockerHost(Step): + context: ConnectorContext + + def __init__(self, context: ConnectorContext, containers: dict[Platform, Container], image_tag: str = "dev") -> None: + super().__init__(context) + self.image_tag = image_tag + self.containers = containers + + def _generate_dev_tag(self, platform: Platform, multi_platforms: bool) -> str: + """ + When building for multiple platforms, we need to tag the image with the platform name. + There's no way to locally build a multi-arch image, so we need to tag the image with the platform name when the user passed multiple architecture options. + """ + return f"{self.image_tag}-{platform.replace('/', '-')}" if multi_platforms else self.image_tag + + @property + def title(self) -> str: + return f"Load {self.image_name}:{self.image_tag} to the local docker host." + + @property + def image_name(self) -> str: + return f"airbyte/{self.context.connector.technical_name}" + + async def _run(self) -> StepResult: + loaded_images = [] + multi_platforms = len(self.containers) > 1 + for platform, container in self.containers.items(): + _, exported_tar_path = await export_container_to_tarball(self.context, container, platform) + if not exported_tar_path: + return StepResult( + self, + StepStatus.FAILURE, + stderr=f"Failed to export the connector image {self.image_name}:{self.image_tag} to a tarball.", + ) + try: + client = docker.from_env() + image_tag = self._generate_dev_tag(platform, multi_platforms) + full_image_name = f"{self.image_name}:{image_tag}" + with open(exported_tar_path, "rb") as tarball_content: + new_image = client.images.load(tarball_content.read())[0] + new_image.tag(self.image_name, tag=image_tag) + image_sha = new_image.id + loaded_images.append(full_image_name) + except docker.errors.DockerException as e: + return StepResult( + self, StepStatus.FAILURE, stderr=f"Something went wrong while interacting with the local docker client: {e}" + ) + + return StepResult(self, StepStatus.SUCCESS, stdout=f"Loaded image {','.join(loaded_images)} to your Docker host ({image_sha}).") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/java_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/java_connectors.py new file mode 100644 index 000000000000..f8a4c7ed0d61 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/java_connectors.py @@ -0,0 +1,67 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dagger import Container, Directory, File, Platform, QueryError +from pipelines.airbyte_ci.connectors.build_image.steps.common import BuildConnectorImagesBase +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.steps.gradle import GradleTask +from pipelines.dagger.containers import java +from pipelines.models.steps import StepResult, StepStatus + + +class BuildConnectorDistributionTar(GradleTask): + """ + A step to build a Java connector image using the distTar Gradle task. + """ + + title = "Build connector tar" + gradle_task_name = "distTar" + + +class BuildConnectorImages(BuildConnectorImagesBase): + """ + A step to build Java connector images using the distTar Gradle task. + """ + + async def _run(self, dist_dir: Directory) -> StepResult: + dist_tar: File + try: + dir_files = await dist_dir.entries() + tar_files = [f for f in dir_files if f.endswith(".tar")] + num_files = len(tar_files) + if num_files != 1: + error_message = ( + "The distribution tar file for the current java connector was not built." + if num_files == 0 + else "More than one distribution tar file was built for the current java connector." + ) + return StepResult(self, StepStatus.FAILURE, stderr=error_message) + dist_tar = dist_dir.file(tar_files[0]) + except QueryError as e: + return StepResult(self, StepStatus.FAILURE, stderr=str(e)) + return await super()._run(dist_tar) + + async def _build_connector(self, platform: Platform, dist_tar: File) -> Container: + return await java.with_airbyte_java_connector(self.context, dist_tar, platform) + + +async def run_connector_build(context: ConnectorContext) -> StepResult: + """Create the java connector distribution tar file and build the connector image.""" + + if context.use_host_gradle_dist_tar and context.is_local: + # Special case: use a local dist tar to speed up local development. + dist_dir = await context.dagger_client.host().directory(dist_tar_directory_path(context), include=["*.tar"]) + # Speed things up by only building for the local platform. + return await BuildConnectorImages(context).run(dist_dir) + + # Default case: distribution tar is built by the dagger pipeline. + build_connector_tar_result = await BuildConnectorDistributionTar(context).run() + if build_connector_tar_result.status is not StepStatus.SUCCESS: + return build_connector_tar_result + dist_dir = await build_connector_tar_result.output_artifact.directory(dist_tar_directory_path(context)) + return await BuildConnectorImages(context).run(dist_dir) + + +def dist_tar_directory_path(context: ConnectorContext) -> str: + return f"{context.connector.code_directory}/build/distributions" diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/normalization.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/normalization.py new file mode 100644 index 000000000000..4774c4fe78f3 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/normalization.py @@ -0,0 +1,39 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dagger import Platform +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.dagger.actions.connector import normalization +from pipelines.models.steps import Step, StepResult, StepStatus + + +# TODO this class could be deleted +# if java connectors tests are not relying on an existing local normalization image to run +class BuildOrPullNormalization(Step): + """A step to build or pull the normalization image for a connector according to the image name.""" + + context: ConnectorContext + + def __init__(self, context: ConnectorContext, normalization_image: str, build_platform: Platform) -> None: + """Initialize the step to build or pull the normalization image. + + Args: + context (ConnectorContext): The current connector context. + normalization_image (str): The normalization image to build (if :dev) or pull. + """ + super().__init__(context) + self.build_platform = build_platform + self.use_dev_normalization = normalization_image.endswith(":dev") + self.normalization_image = normalization_image + + @property + def title(self) -> str: + return f"Build {self.normalization_image}" if self.use_dev_normalization else f"Pull {self.normalization_image}" + + async def _run(self) -> StepResult: + if self.use_dev_normalization: + build_normalization_container = normalization.with_normalization(self.context, self.build_platform) + else: + build_normalization_container = self.context.dagger_client.container().from_(self.normalization_image) + return StepResult(self, StepStatus.SUCCESS, output_artifact=build_normalization_container) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py new file mode 100644 index 000000000000..f8eac5b8db07 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py @@ -0,0 +1,108 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from typing import Any + +from dagger import Container, Platform +from pipelines.airbyte_ci.connectors.build_image.steps import build_customization +from pipelines.airbyte_ci.connectors.build_image.steps.common import BuildConnectorImagesBase +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.dagger.actions.python.common import apply_python_development_overrides, with_python_connector_installed +from pipelines.models.steps import StepResult + + +class BuildConnectorImages(BuildConnectorImagesBase): + """ + A step to build a Python connector image. + A spec command is run on the container to validate it was built successfully. + """ + + context: ConnectorContext + PATH_TO_INTEGRATION_CODE = "/airbyte/integration_code" + + async def _build_connector(self, platform: Platform, *args: Any) -> Container: + if ( + "connectorBuildOptions" in self.context.connector.metadata + and "baseImage" in self.context.connector.metadata["connectorBuildOptions"] + ): + return await self._build_from_base_image(platform) + else: + return await self._build_from_dockerfile(platform) + + def _get_base_container(self, platform: Platform) -> Container: + base_image_name = self.context.connector.metadata["connectorBuildOptions"]["baseImage"] + self.logger.info(f"Building connector from base image {base_image_name}") + return self.dagger_client.container(platform=platform).from_(base_image_name) + + async def _create_builder_container(self, base_container: Container) -> Container: + """Pre install the connector dependencies in a builder container. + + Args: + base_container (Container): The base container to use to build the connector. + + Returns: + Container: The builder container, with installed dependencies. + """ + ONLY_PYTHON_BUILD_FILES = ["setup.py", "requirements.txt", "pyproject.toml", "poetry.lock"] + builder = await with_python_connector_installed( + self.context, + base_container, + str(self.context.connector.code_directory), + include=ONLY_PYTHON_BUILD_FILES, + ) + + return builder + + async def _build_from_base_image(self, platform: Platform) -> Container: + """Build the connector container using the base image defined in the metadata, in the connectorBuildOptions.baseImage field. + + Returns: + Container: The connector container built from the base image. + """ + self.logger.info(f"Building connector from base image in metadata for {platform}") + base = self._get_base_container(platform) + customized_base = await build_customization.pre_install_hooks(self.context.connector, base, self.logger) + entrypoint = build_customization.get_entrypoint(self.context.connector) + main_file_name = build_customization.get_main_file_name(self.context.connector) + + builder = await self._create_builder_container(customized_base) + + # The snake case name of the connector corresponds to the python package name of the connector + # We want to mount it to the container under PATH_TO_INTEGRATION_CODE/connector_snake_case_name + connector_snake_case_name = self.context.connector.technical_name.replace("-", "_") + + connector_container = ( + # copy python dependencies from builder to connector container + customized_base.with_directory("/usr/local", builder.directory("/usr/local")) + .with_workdir(self.PATH_TO_INTEGRATION_CODE) + .with_file(main_file_name, (await self.context.get_connector_dir(include=[main_file_name])).file(main_file_name)) + .with_directory( + connector_snake_case_name, + (await self.context.get_connector_dir(include=[connector_snake_case_name])).directory(connector_snake_case_name), + ) + .with_env_variable("AIRBYTE_ENTRYPOINT", " ".join(entrypoint)) + .with_entrypoint(entrypoint) + .with_label("io.airbyte.version", self.context.connector.metadata["dockerImageTag"]) + .with_label("io.airbyte.name", self.context.connector.metadata["dockerRepository"]) + ) + customized_connector = await build_customization.post_install_hooks(self.context.connector, connector_container, self.logger) + return customized_connector + + async def _build_from_dockerfile(self, platform: Platform) -> Container: + """Build the connector container using its Dockerfile. + + Returns: + Container: The connector container built from its Dockerfile. + """ + self.logger.warn( + "This connector is built from its Dockerfile. This is now deprecated. Please set connectorBuildOptions.baseImage metadata field to use our new build process." + ) + container = self.dagger_client.container(platform=platform).build(await self.context.get_connector_dir()) + container = await apply_python_development_overrides(self.context, container) + return container + + +async def run_connector_build(context: ConnectorContext) -> StepResult: + return await BuildConnectorImages(context).run() diff --git a/airbyte-integrations/connectors/source-wrike/unit_tests/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-wrike/unit_tests/__init__.py rename to airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/__init__.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py new file mode 100644 index 000000000000..d6ddbe5360bb --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py @@ -0,0 +1,61 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import asyncclick as click +from pipelines.airbyte_ci.connectors.bump_version.pipeline import run_connector_version_bump_pipeline +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines +from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand + + +@click.command(cls=DaggerPipelineCommand, short_help="Bump a connector version: update metadata.yaml and changelog.") +@click.argument("bump-type", type=click.Choice(["patch", "minor", "major"])) +@click.argument("pull-request-number", type=str) +@click.argument("changelog-entry", type=str) +@click.pass_context +async def bump_version( + ctx: click.Context, + bump_type: str, + pull_request_number: str, + changelog_entry: str, +) -> bool: + """Bump a connector version: update metadata.yaml and changelog.""" + + connectors_contexts = [ + ConnectorContext( + pipeline_name=f"Upgrade base image versions of connector {connector.technical_name}", + connector=connector, + is_local=ctx.obj["is_local"], + git_branch=ctx.obj["git_branch"], + git_revision=ctx.obj["git_revision"], + ci_report_bucket=ctx.obj["ci_report_bucket_name"], + report_output_prefix=ctx.obj["report_output_prefix"], + use_remote_secrets=ctx.obj["use_remote_secrets"], + gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), + dagger_logs_url=ctx.obj.get("dagger_logs_url"), + pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), + ci_context=ctx.obj.get("ci_context"), + ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], + ci_git_user=ctx.obj["ci_git_user"], + ci_github_access_token=ctx.obj["ci_github_access_token"], + enable_report_auto_open=False, + s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"), + s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"), + ) + for connector in ctx.obj["selected_connectors_with_modified_files"] + ] + + await run_connectors_pipelines( + connectors_contexts, + run_connector_version_bump_pipeline, + "Version bump pipeline pipeline", + ctx.obj["concurrency"], + ctx.obj["dagger_logs_path"], + ctx.obj["execute_timeout"], + bump_type, + changelog_entry, + pull_request_number, + ) + + return True diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/pipeline.py new file mode 100644 index 000000000000..cf330b5c5a9e --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/pipeline.py @@ -0,0 +1,182 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import datetime +from copy import deepcopy +from typing import TYPE_CHECKING + +import semver +from dagger import Container, Directory +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.connectors.reports import ConnectorReport, Report +from pipelines.helpers import git +from pipelines.helpers.connectors import metadata_change_helpers +from pipelines.models.steps import Step, StepResult, StepStatus + +if TYPE_CHECKING: + from anyio import Semaphore + + +def get_bumped_version(version: str, bump_type: str) -> str: + current_version = semver.VersionInfo.parse(version) + if bump_type == "patch": + new_version = current_version.bump_patch() + elif bump_type == "minor": + new_version = current_version.bump_minor() + elif bump_type == "major": + new_version = current_version.bump_major() + else: + raise ValueError(f"Unknown bump type: {bump_type}") + return str(new_version) + + +class AddChangelogEntry(Step): + context: ConnectorContext + title = "Add changelog entry" + + def __init__( + self, + context: ConnectorContext, + repo_dir: Container, + new_version: str, + changelog_entry: str, + pull_request_number: str, + ) -> None: + super().__init__(context) + self.repo_dir = repo_dir + self.new_version = new_version + self.changelog_entry = changelog_entry + self.pull_request_number = pull_request_number + + async def _run(self) -> StepResult: + doc_path = self.context.connector.documentation_file_path + if not doc_path.exists(): + return StepResult( + self, + StepStatus.SKIPPED, + stdout="Connector does not have a documentation file.", + output_artifact=self.repo_dir, + ) + try: + updated_doc = self.add_changelog_entry(doc_path.read_text()) + except Exception as e: + return StepResult( + self, + StepStatus.FAILURE, + stdout=f"Could not add changelog entry: {e}", + output_artifact=self.repo_dir, + ) + updated_repo_dir = self.repo_dir.with_new_file(str(doc_path), contents=updated_doc) + return StepResult( + self, + StepStatus.SUCCESS, + stdout=f"Added changelog entry to {doc_path}", + output_artifact=updated_repo_dir, + ) + + def find_line_index_for_new_entry(self, markdown_text: str) -> int: + lines = markdown_text.splitlines() + for line_index, line in enumerate(lines): + if "version" in line.lower() and "date" in line.lower() and "pull request" in line.lower() and "subject" in line.lower(): + return line_index + 2 + raise Exception("Could not find the changelog section table in the documentation file.") + + def add_changelog_entry(self, og_doc_content: str) -> str: + today = datetime.date.today().strftime("%Y-%m-%d") + lines = og_doc_content.splitlines() + line_index_for_new_entry = self.find_line_index_for_new_entry(og_doc_content) + new_entry = f"| {self.new_version} | {today} | [{self.pull_request_number}](https://github.com/airbytehq/airbyte/pull/{self.pull_request_number}) | {self.changelog_entry} |" + lines.insert(line_index_for_new_entry, new_entry) + return "\n".join(lines) + + +class BumpDockerImageTagInMetadata(Step): + context: ConnectorContext + title = "Upgrade the dockerImageTag to the latest version in metadata.yaml" + + def __init__( + self, + context: ConnectorContext, + repo_dir: Directory, + new_version: str, + ) -> None: + super().__init__(context) + self.repo_dir = repo_dir + self.new_version = new_version + + @staticmethod + def get_metadata_with_bumped_version(previous_version: str, new_version: str, current_metadata: dict) -> dict: + updated_metadata = deepcopy(current_metadata) + updated_metadata["data"]["dockerImageTag"] = new_version + # Bump strict versions + if current_metadata["data"].get("registries", {}).get("cloud", {}).get("dockerImageTag") == previous_version: + updated_metadata["data"]["registries"]["cloud"]["dockerImageTag"] = new_version + return updated_metadata + + async def _run(self) -> StepResult: + metadata_path = self.context.connector.metadata_file_path + current_metadata = await metadata_change_helpers.get_current_metadata(self.repo_dir, metadata_path) + current_version = metadata_change_helpers.get_current_version(current_metadata) + if current_version is None: + return StepResult( + self, + StepStatus.SKIPPED, + stdout="Can't retrieve the connector current version.", + output_artifact=self.repo_dir, + ) + updated_metadata = self.get_metadata_with_bumped_version(current_version, self.new_version, current_metadata) + repo_dir_with_updated_metadata = metadata_change_helpers.get_repo_dir_with_updated_metadata( + self.repo_dir, metadata_path, updated_metadata + ) + + return StepResult( + self, + StepStatus.SUCCESS, + stdout=f"Updated dockerImageTag from {current_version} to {self.new_version} in {metadata_path}", + output_artifact=repo_dir_with_updated_metadata, + ) + + +async def run_connector_version_bump_pipeline( + context: ConnectorContext, + semaphore: "Semaphore", + bump_type: str, + changelog_entry: str, + pull_request_number: str, +) -> Report: + """Run a pipeline to upgrade for a single connector. + + Args: + context (ConnectorContext): The initialized connector context. + + Returns: + Report: The reports holding the base image version upgrade results. + """ + async with semaphore: + steps_results = [] + async with context: + og_repo_dir = await context.get_repo_dir() + new_version = get_bumped_version(context.connector.version, bump_type) + update_docker_image_tag_in_metadata = BumpDockerImageTagInMetadata( + context, + og_repo_dir, + new_version, + ) + update_docker_image_tag_in_metadata_result = await update_docker_image_tag_in_metadata.run() + repo_dir_with_updated_metadata = update_docker_image_tag_in_metadata_result.output_artifact + steps_results.append(update_docker_image_tag_in_metadata_result) + + add_changelog_entry = AddChangelogEntry( + context, + repo_dir_with_updated_metadata, + new_version, + changelog_entry, + pull_request_number, + ) + add_changelog_entry_result = await add_changelog_entry.run() + steps_results.append(add_changelog_entry_result) + final_repo_dir = add_changelog_entry_result.output_artifact + await og_repo_dir.diff(final_repo_dir).export(str(git.get_git_repo_path())) + report = ConnectorReport(context, steps_results, name="CONNECTOR VERSION BUMP RESULTS") + context.report = report + return report diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py new file mode 100644 index 000000000000..e3f6a5e0922c --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py @@ -0,0 +1,280 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import os +from pathlib import Path +from typing import List, Optional, Set, Tuple + +import asyncclick as click +from connector_ops.utils import ConnectorLanguage, SupportLevelEnum, get_all_connectors_in_repo # type: ignore +from pipelines import main_logger +from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.lazy_group import LazyGroup +from pipelines.consts import CIContext +from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles, get_connector_modified_files, get_modified_connectors +from pipelines.helpers.git import get_modified_files_in_branch, get_modified_files_in_commit +from pipelines.helpers.utils import transform_strs_to_paths + +ALL_CONNECTORS = get_all_connectors_in_repo() + + +def log_selected_connectors(selected_connectors_with_modified_files: List[ConnectorWithModifiedFiles]) -> None: + if selected_connectors_with_modified_files: + selected_connectors_names = [c.technical_name for c in selected_connectors_with_modified_files] + main_logger.info(f"Will run on the following {len(selected_connectors_names)} connectors: {', '.join(selected_connectors_names)}.") + else: + main_logger.info("No connectors to run.") + + +def get_selected_connectors_with_modified_files( + selected_names: Tuple[str], + selected_support_levels: Tuple[str], + selected_languages: Tuple[str], + modified: bool, + metadata_changes_only: bool, + metadata_query: str, + modified_files: Set[Path], + enable_dependency_scanning: bool = False, +) -> List[ConnectorWithModifiedFiles]: + """Get the connectors that match the selected criteria. + + Args: + selected_names (Tuple[str]): Selected connector names. + selected_support_levels (Tuple[str]): Selected connector support levels. + selected_languages (Tuple[str]): Selected connector languages. + modified (bool): Whether to select the modified connectors. + metadata_changes_only (bool): Whether to select only the connectors with metadata changes. + modified_files (Set[Path]): The modified files. + enable_dependency_scanning (bool): Whether to enable the dependency scanning. + Returns: + List[ConnectorWithModifiedFiles]: The connectors that match the selected criteria. + """ + + if metadata_changes_only and not modified: + main_logger.info("--metadata-changes-only overrides --modified") + modified = True + + selected_modified_connectors = ( + get_modified_connectors(modified_files, ALL_CONNECTORS, enable_dependency_scanning) if modified else set() + ) + selected_connectors_by_name = {c for c in ALL_CONNECTORS if c.technical_name in selected_names} + selected_connectors_by_support_level = {connector for connector in ALL_CONNECTORS if connector.support_level in selected_support_levels} + selected_connectors_by_language = {connector for connector in ALL_CONNECTORS if connector.language in selected_languages} + selected_connectors_by_query = ( + {connector for connector in ALL_CONNECTORS if connector.metadata_query_match(metadata_query)} if metadata_query else set() + ) + + non_empty_connector_sets = [ + connector_set + for connector_set in [ + selected_connectors_by_name, + selected_connectors_by_support_level, + selected_connectors_by_language, + selected_connectors_by_query, + selected_modified_connectors, + ] + if connector_set + ] + # The selected connectors are the intersection of the selected connectors by name, support_level, language, simpleeval query and modified. + selected_connectors = set.intersection(*non_empty_connector_sets) if non_empty_connector_sets else set() + + selected_connectors_with_modified_files = [] + for connector in selected_connectors: + connector_with_modified_files = ConnectorWithModifiedFiles( + relative_connector_path=connector.relative_connector_path, + modified_files=get_connector_modified_files(connector, modified_files), + ) + if not metadata_changes_only: + selected_connectors_with_modified_files.append(connector_with_modified_files) + else: + if connector_with_modified_files.has_metadata_change: + selected_connectors_with_modified_files.append(connector_with_modified_files) + return selected_connectors_with_modified_files + + +def validate_environment(is_local: bool) -> None: + """Check if the required environment variables exist.""" + if is_local: + if not Path(".git").is_dir(): + raise click.UsageError("You need to run this command from the repository root.") + else: + required_env_vars_for_ci = [ + "GCP_GSM_CREDENTIALS", + "CI_REPORT_BUCKET_NAME", + "CI_GITHUB_ACCESS_TOKEN", + "DOCKER_HUB_USERNAME", + "DOCKER_HUB_PASSWORD", + ] + for required_env_var in required_env_vars_for_ci: + if os.getenv(required_env_var) is None: + raise click.UsageError(f"When running in a CI context a {required_env_var} environment variable must be set.") + + +def should_use_remote_secrets(use_remote_secrets: Optional[bool]) -> bool: + """Check if the connector secrets should be loaded from Airbyte GSM or from the local secrets directory. + + Args: + use_remote_secrets (Optional[bool]): Whether to use remote connector secrets or local connector secrets according to user inputs. + + Raises: + click.UsageError: If the --use-remote-secrets flag was provided but no GCP_GSM_CREDENTIALS environment variable was found. + + Returns: + bool: Whether to use remote connector secrets (True) or local connector secrets (False). + """ + gcp_gsm_credentials_is_set = bool(os.getenv("GCP_GSM_CREDENTIALS")) + if use_remote_secrets is None: + if gcp_gsm_credentials_is_set: + main_logger.info("GCP_GSM_CREDENTIALS environment variable found, using remote connector secrets.") + return True + else: + main_logger.info("No GCP_GSM_CREDENTIALS environment variable found, using local connector secrets.") + return False + if use_remote_secrets: + if gcp_gsm_credentials_is_set: + main_logger.info("GCP_GSM_CREDENTIALS environment variable found, using remote connector secrets.") + return True + else: + raise click.UsageError("The --use-remote-secrets flag was provided but no GCP_GSM_CREDENTIALS environment variable was found.") + else: + main_logger.info("Using local connector secrets as the --use-local-secrets flag was provided") + return False + + +@click.group( + cls=LazyGroup, + help="Commands related to connectors and connector acceptance tests.", + lazy_subcommands={ + "build": "pipelines.airbyte_ci.connectors.build_image.commands.build", + "test": "pipelines.airbyte_ci.connectors.test.commands.test", + "list": "pipelines.airbyte_ci.connectors.list.commands.list_connectors", + "publish": "pipelines.airbyte_ci.connectors.publish.commands.publish", + "bump_version": "pipelines.airbyte_ci.connectors.bump_version.commands.bump_version", + "migrate_to_base_image": "pipelines.airbyte_ci.connectors.migrate_to_base_image.commands.migrate_to_base_image", + "upgrade_base_image": "pipelines.airbyte_ci.connectors.upgrade_base_image.commands.upgrade_base_image", + "upgrade_cdk": "pipelines.airbyte_ci.connectors.upgrade_cdk.commands.bump_version", + }, +) +@click.option( + "--use-remote-secrets/--use-local-secrets", + help="Use Airbyte GSM connector secrets or local connector secrets.", + type=bool, + default=None, +) +@click.option( + "--name", + "names", + multiple=True, + help="Only test a specific connector. Use its technical name. e.g source-pokeapi.", + type=click.Choice([c.technical_name for c in ALL_CONNECTORS]), +) +@click.option("--language", "languages", multiple=True, help="Filter connectors to test by language.", type=click.Choice(ConnectorLanguage)) +@click.option( + "--support-level", + "support_levels", + multiple=True, + help="Filter connectors to test by support_level.", + type=click.Choice(SupportLevelEnum), +) +@click.option("--modified/--not-modified", help="Only test modified connectors in the current branch.", default=False, type=bool) +@click.option( + "--metadata-changes-only/--not-metadata-changes-only", + help="Only test connectors with modified metadata files in the current branch.", + default=False, + type=bool, +) +@click.option( + "--metadata-query", + help="Filter connectors by metadata query using `simpleeval`. e.g. 'data.ab_internal.ql == 200'", + type=str, +) +@click.option("--concurrency", help="Number of connector tests pipeline to run in parallel.", default=5, type=int) +@click.option( + "--execute-timeout", + help="The maximum time in seconds for the execution of a Dagger request before an ExecuteTimeoutError is raised. Passing None results in waiting forever.", + default=None, + type=int, +) +@click.option( + "--enable-dependency-scanning/--disable-dependency-scanning", + help="When enabled, the dependency scanning will be performed to detect the connectors to test according to a dependency change.", + default=False, + type=bool, +) +@click.option( + "--use-local-cdk", + is_flag=True, + help=("Build with the airbyte-cdk from the local repository. " "This is useful for testing changes to the CDK."), + default=False, + type=bool, +) +@click.option( + "--enable-report-auto-open/--disable-report-auto-open", + is_flag=True, + help=("When enabled, finishes by opening a browser window to display an HTML report."), + default=True, + type=bool, +) +@click.option( + "--docker-hub-username", + help="Your username to connect to DockerHub.", + type=click.STRING, + required=False, + envvar="DOCKER_HUB_USERNAME", +) +@click.option( + "--docker-hub-password", + help="Your password to connect to DockerHub.", + type=click.STRING, + required=False, + envvar="DOCKER_HUB_PASSWORD", +) +@click_merge_args_into_context_obj +@click_append_to_context_object("use_remote_secrets", lambda ctx: should_use_remote_secrets(ctx.obj["use_remote_secrets"])) +@click.pass_context +@click_ignore_unused_kwargs +async def connectors( + ctx: click.Context, +) -> None: + """Group all the connectors-ci command.""" + validate_environment(ctx.obj["is_local"]) + + modified_files = [] + if ctx.obj["modified"] or ctx.obj["metadata_changes_only"]: + modified_files = transform_strs_to_paths( + await get_modified_files( + ctx.obj["git_branch"], + ctx.obj["git_revision"], + ctx.obj["diffed_branch"], + ctx.obj["is_local"], + ctx.obj["ci_context"], + ) + ) + + ctx.obj["selected_connectors_with_modified_files"] = get_selected_connectors_with_modified_files( + ctx.obj["names"], + ctx.obj["support_levels"], + ctx.obj["languages"], + ctx.obj["modified"], + ctx.obj["metadata_changes_only"], + ctx.obj["metadata_query"], + set(modified_files), + ctx.obj["enable_dependency_scanning"], + ) + log_selected_connectors(ctx.obj["selected_connectors_with_modified_files"]) + + +async def get_modified_files(git_branch: str, git_revision: str, diffed_branch: str, is_local: bool, ci_context: CIContext) -> Set[str]: + """Get the list of modified files in the current git branch. + If the current branch is master, it will return the list of modified files in the head commit. + The head commit on master should be the merge commit of the latest merged pull request as we squash commits on merge. + Pipelines like "publish on merge" are triggered on each new commit on master. + + If the CI context is a pull request, it will return the list of modified files in the pull request, without using git diff. + If the current branch is not master, it will return the list of modified files in the current branch. + This latest case is the one we encounter when running the pipeline locally, on a local branch, or manually on GHA with a workflow dispatch event. + """ + if ci_context is CIContext.MASTER or (ci_context is CIContext.MANUAL and git_branch == "master"): + return await get_modified_files_in_commit(git_branch, git_revision, is_local) + return await get_modified_files_in_branch(git_branch, git_revision, diffed_branch, is_local) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/consts.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/consts.py new file mode 100644 index 000000000000..10e00bc67dad --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/consts.py @@ -0,0 +1,26 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from enum import Enum + + +class CONNECTOR_TEST_STEP_ID(str, Enum): + """ + An enum for the different step ids of the connector test pipeline. + """ + + ACCEPTANCE = "acceptance" + BUILD_NORMALIZATION = "build_normalization" + BUILD_TAR = "build_tar" + BUILD = "build" + CHECK_BASE_IMAGE = "check_base_image" + INTEGRATION = "integration" + METADATA_VALIDATION = "metadata_validation" + QA_CHECKS = "qa_checks" + UNIT = "unit" + VERSION_FOLLOW_CHECK = "version_follow_check" + VERSION_INC_CHECK = "version_inc_check" + TEST_ORCHESTRATOR = "test_orchestrator" + DEPLOY_ORCHESTRATOR = "deploy_orchestrator" + + def __str__(self) -> str: + return self.value diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py new file mode 100644 index 000000000000..dff4f9b2a736 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/context.py @@ -0,0 +1,278 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""Module declaring context related classes.""" + +from __future__ import annotations + +from datetime import datetime +from types import TracebackType +from typing import TYPE_CHECKING + +import yaml # type: ignore +from anyio import Path +from asyncer import asyncify +from dagger import Directory, Platform, Secret +from github import PullRequest +from pipelines.airbyte_ci.connectors.reports import ConnectorReport +from pipelines.consts import BUILD_PLATFORMS +from pipelines.dagger.actions import secrets +from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles +from pipelines.helpers.execution.run_steps import RunStepOptions +from pipelines.helpers.github import update_commit_status_check +from pipelines.helpers.slack import send_message_to_webhook +from pipelines.helpers.utils import METADATA_FILE_NAME +from pipelines.models.contexts.pipeline_context import PipelineContext + +if TYPE_CHECKING: + from pathlib import Path as NativePath + from typing import Dict, FrozenSet, List, Optional, Sequence + + +class ConnectorContext(PipelineContext): + """The connector context is used to store configuration for a specific connector pipeline run.""" + + DEFAULT_CONNECTOR_ACCEPTANCE_TEST_IMAGE = "airbyte/connector-acceptance-test:dev" + + def __init__( + self, + pipeline_name: str, + connector: ConnectorWithModifiedFiles, + is_local: bool, + git_branch: str, + git_revision: str, + report_output_prefix: str, + use_remote_secrets: bool = True, + ci_report_bucket: Optional[str] = None, + ci_gcs_credentials: Optional[str] = None, + ci_git_user: Optional[str] = None, + ci_github_access_token: Optional[str] = None, + connector_acceptance_test_image: str = DEFAULT_CONNECTOR_ACCEPTANCE_TEST_IMAGE, + gha_workflow_run_url: Optional[str] = None, + dagger_logs_url: Optional[str] = None, + pipeline_start_timestamp: Optional[int] = None, + ci_context: Optional[str] = None, + slack_webhook: Optional[str] = None, + reporting_slack_channel: Optional[str] = None, + pull_request: Optional[PullRequest.PullRequest] = None, + should_save_report: bool = True, + code_tests_only: bool = False, + use_local_cdk: bool = False, + use_host_gradle_dist_tar: bool = False, + enable_report_auto_open: bool = True, + docker_hub_username: Optional[str] = None, + docker_hub_password: Optional[str] = None, + s3_build_cache_access_key_id: Optional[str] = None, + s3_build_cache_secret_key: Optional[str] = None, + concurrent_cat: Optional[bool] = False, + run_step_options: RunStepOptions = RunStepOptions(), + targeted_platforms: Sequence[Platform] = BUILD_PLATFORMS, + ) -> None: + """Initialize a connector context. + + Args: + connector (Connector): The connector under test. + is_local (bool): Whether the context is for a local run or a CI run. + git_branch (str): The current git branch name. + git_revision (str): The current git revision, commit hash. + report_output_prefix (str): The S3 key to upload the test report to. + use_remote_secrets (bool, optional): Whether to download secrets for GSM or use the local secrets. Defaults to True. + connector_acceptance_test_image (Optional[str], optional): The image to use to run connector acceptance tests. Defaults to DEFAULT_CONNECTOR_ACCEPTANCE_TEST_IMAGE. + gha_workflow_run_url (Optional[str], optional): URL to the github action workflow run. Only valid for CI run. Defaults to None. + dagger_logs_url (Optional[str], optional): URL to the dagger logs. Only valid for CI run. Defaults to None. + pipeline_start_timestamp (Optional[int], optional): Timestamp at which the pipeline started. Defaults to None. + ci_context (Optional[str], optional): Pull requests, workflow dispatch or nightly build. Defaults to None. + slack_webhook (Optional[str], optional): The slack webhook to send messages to. Defaults to None. + reporting_slack_channel (Optional[str], optional): The slack channel to send messages to. Defaults to None. + pull_request (PullRequest, optional): The pull request object if the pipeline was triggered by a pull request. Defaults to None. + code_tests_only (bool, optional): Whether to ignore non-code tests like QA and metadata checks. Defaults to False. + use_host_gradle_dist_tar (bool, optional): Used when developing java connectors with gradle. Defaults to False. + enable_report_auto_open (bool, optional): Open HTML report in browser window. Defaults to True. + docker_hub_username (Optional[str], optional): Docker Hub username to use to read registries. Defaults to None. + docker_hub_password (Optional[str], optional): Docker Hub password to use to read registries. Defaults to None. + s3_build_cache_access_key_id (Optional[str], optional): Gradle S3 Build Cache credentials. Defaults to None. + s3_build_cache_secret_key (Optional[str], optional): Gradle S3 Build Cache credentials. Defaults to None. + concurrent_cat (bool, optional): Whether to run the CAT tests in parallel. Defaults to False. + targeted_platforms (Optional[Iterable[Platform]], optional): The platforms to build the connector image for. Defaults to BUILD_PLATFORMS. + """ + + self.pipeline_name = pipeline_name + self.connector = connector + self.use_remote_secrets = use_remote_secrets + self.connector_acceptance_test_image = connector_acceptance_test_image + self._secrets_dir: Optional[Directory] = None + self._updated_secrets_dir: Optional[Directory] = None + self.cdk_version: Optional[str] = None + self.should_save_report = should_save_report + self.code_tests_only = code_tests_only + self.use_local_cdk = use_local_cdk + self.use_host_gradle_dist_tar = use_host_gradle_dist_tar + self.enable_report_auto_open = enable_report_auto_open + self.docker_hub_username = docker_hub_username + self.docker_hub_password = docker_hub_password + self.s3_build_cache_access_key_id = s3_build_cache_access_key_id + self.s3_build_cache_secret_key = s3_build_cache_secret_key + self.concurrent_cat = concurrent_cat + self._connector_secrets: Optional[Dict[str, Secret]] = None + self.targeted_platforms = targeted_platforms + + super().__init__( + pipeline_name=pipeline_name, + is_local=is_local, + git_branch=git_branch, + git_revision=git_revision, + report_output_prefix=report_output_prefix, + gha_workflow_run_url=gha_workflow_run_url, + dagger_logs_url=dagger_logs_url, + pipeline_start_timestamp=pipeline_start_timestamp, + ci_context=ci_context, + slack_webhook=slack_webhook, + reporting_slack_channel=reporting_slack_channel, + pull_request=pull_request, + ci_report_bucket=ci_report_bucket, + ci_gcs_credentials=ci_gcs_credentials, + ci_git_user=ci_git_user, + ci_github_access_token=ci_github_access_token, + run_step_options=run_step_options, + enable_report_auto_open=enable_report_auto_open, + ) + + @property + def s3_build_cache_access_key_id_secret(self) -> Optional[Secret]: + if self.s3_build_cache_access_key_id: + return self.dagger_client.set_secret("s3_build_cache_access_key_id", self.s3_build_cache_access_key_id) + return None + + @property + def s3_build_cache_secret_key_secret(self) -> Optional[Secret]: + if self.s3_build_cache_access_key_id and self.s3_build_cache_secret_key: + return self.dagger_client.set_secret("s3_build_cache_secret_key", self.s3_build_cache_secret_key) + return None + + @property + def modified_files(self) -> FrozenSet[NativePath]: + return self.connector.modified_files + + @property + def secrets_dir(self) -> Optional[Directory]: + return self._secrets_dir + + @secrets_dir.setter + def secrets_dir(self, secrets_dir: Directory) -> None: + self._secrets_dir = secrets_dir + + @property + def updated_secrets_dir(self) -> Optional[Directory]: + return self._updated_secrets_dir + + @updated_secrets_dir.setter + def updated_secrets_dir(self, updated_secrets_dir: Directory) -> None: + self._updated_secrets_dir = updated_secrets_dir + + @property + def connector_acceptance_test_source_dir(self) -> Directory: + return self.get_repo_dir("airbyte-integrations/bases/connector-acceptance-test") + + @property + def should_save_updated_secrets(self) -> bool: + return self.use_remote_secrets and self.updated_secrets_dir is not None + + @property + def host_image_export_dir_path(self) -> str: + return "." if self.is_ci else "/tmp" + + @property + def metadata_path(self) -> Path: + return self.connector.code_directory / METADATA_FILE_NAME + + @property + def metadata(self) -> dict: + return yaml.safe_load(self.metadata_path.read_text())["data"] + + @property + def docker_repository(self) -> str: + return self.metadata["dockerRepository"] + + @property + def docker_image_tag(self) -> str: + return self.metadata["dockerImageTag"] + + @property + def docker_image(self) -> str: + return f"{self.docker_repository}:{self.docker_image_tag}" + + @property + def docker_hub_username_secret(self) -> Optional[Secret]: + if self.docker_hub_username is None: + return None + return self.dagger_client.set_secret("docker_hub_username", self.docker_hub_username) + + @property + def docker_hub_password_secret(self) -> Optional[Secret]: + if self.docker_hub_password is None: + return None + return self.dagger_client.set_secret("docker_hub_password", self.docker_hub_password) + + async def get_connector_secrets(self) -> Dict[str, Secret]: + if self._connector_secrets is None: + self._connector_secrets = await secrets.get_connector_secrets(self) + return self._connector_secrets + + async def get_connector_dir(self, exclude: Optional[List[str]] = None, include: Optional[List[str]] = None) -> Directory: + """Get the connector under test source code directory. + + Args: + exclude ([List[str], optional): List of files or directories to exclude from the directory. Defaults to None. + include ([List[str], optional): List of files or directories to include in the directory. Defaults to None. + + Returns: + Directory: The connector under test source code directory. + """ + vanilla_connector_dir = self.get_repo_dir(str(self.connector.code_directory), exclude=exclude, include=include) + return await vanilla_connector_dir.with_timestamps(1) + + async def __aexit__( + self, exception_type: Optional[type[BaseException]], exception_value: Optional[BaseException], traceback: Optional[TracebackType] + ) -> bool: + """Perform teardown operation for the ConnectorContext. + + On the context exit the following operations will happen: + - Upload updated connector secrets back to Google Secret Manager + - Write a test report in JSON format locally and to S3 if running in a CI environment + - Update the commit status check on GitHub if running in a CI environment. + It should gracefully handle the execution error that happens and always upload a test report and update commit status check. + Args: + exception_type (Optional[type[BaseException]]): The exception type if an exception was raised in the context execution, None otherwise. + exception_value (Optional[BaseException]): The exception value if an exception was raised in the context execution, None otherwise. + traceback (Optional[TracebackType]): The traceback if an exception was raised in the context execution, None otherwise. + Returns: + bool: Whether the teardown operation ran successfully. + """ + self.stopped_at = datetime.utcnow() + self.state = self.determine_final_state(self.report, exception_value) + if exception_value: + self.logger.error("An error got handled by the ConnectorContext", exc_info=True) + if self.report is None: + self.logger.error("No test report was provided. This is probably due to an upstream error") + self.report = ConnectorReport(self, []) + + if self.should_save_updated_secrets: + await secrets.upload(self) + + self.report.print() + + if self.should_save_report: + await self.report.save() + + await asyncify(update_commit_status_check)(**self.github_commit_status) + + if self.should_send_slack_message: + # Using a type ignore here because the should_send_slack_message property is checking for non nullity of the slack_webhook and reporting_slack_channel + await asyncify(send_message_to_webhook)(self.create_slack_message(), self.reporting_slack_channel, self.slack_webhook) # type: ignore + + # Supress the exception if any + return True + + def create_slack_message(self) -> str: + raise NotImplementedError diff --git a/airbyte-integrations/connectors/source-younium/unit_tests/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-younium/unit_tests/__init__.py rename to airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/__init__.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py new file mode 100644 index 000000000000..5c5b97ca5621 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/commands.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import asyncclick as click +from connector_ops.utils import console # type: ignore +from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand +from rich.table import Table +from rich.text import Text + + +@click.command(cls=DaggerPipelineCommand, help="List all selected connectors.", name="list") +@click.pass_context +async def list_connectors( + ctx: click.Context, +) -> bool: + selected_connectors = sorted(ctx.obj["selected_connectors_with_modified_files"], key=lambda x: x.technical_name) + table = Table(title=f"{len(selected_connectors)} selected connectors") + table.add_column("Modified") + table.add_column("Connector") + table.add_column("Language") + table.add_column("Release stage") + table.add_column("Version") + table.add_column("Folder") + + for connector in selected_connectors: + modified = "X" if connector.modified_files else "" + connector_name = Text(connector.technical_name) + language: Text = Text(connector.language.value) if connector.language else Text("N/A") + try: + support_level: Text = Text(connector.support_level) + except Exception: + support_level = Text("N/A") + try: + version: Text = Text(connector.version) + except Exception: + version = Text("N/A") + folder = Text(str(connector.code_directory)) + table.add_row(modified, connector_name, language, support_level, version, folder) + + console.print(table) + return True diff --git a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/pipeline.py similarity index 100% rename from airbyte-integrations/connectors/source-zendesk-sell/unit_tests/__init__.py rename to airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/list/pipeline.py diff --git a/airbyte-integrations/connectors/source-zenefits/unit_tests/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/__init__.py similarity index 100% rename from airbyte-integrations/connectors/source-zenefits/unit_tests/__init__.py rename to airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/__init__.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/commands.py new file mode 100644 index 000000000000..7b6196e6eee5 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/commands.py @@ -0,0 +1,63 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import asyncclick as click +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.connectors.migrate_to_base_image.pipeline import run_connector_migration_to_base_image_pipeline +from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines +from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand +from pipelines.helpers.utils import fail_if_missing_docker_hub_creds + + +@click.command( + cls=DaggerPipelineCommand, + short_help="Make the selected connectors use our base image: remove dockerfile, update metadata.yaml and update documentation.", +) +@click.argument("pull-request-number", type=str) +@click.pass_context +async def migrate_to_base_image( + ctx: click.Context, + pull_request_number: str, +) -> bool: + """Bump a connector version: update metadata.yaml, changelog and delete legacy files.""" + + fail_if_missing_docker_hub_creds(ctx) + + connectors_contexts = [ + ConnectorContext( + pipeline_name=f"Upgrade base image versions of connector {connector.technical_name}", + connector=connector, + is_local=ctx.obj["is_local"], + git_branch=ctx.obj["git_branch"], + git_revision=ctx.obj["git_revision"], + ci_report_bucket=ctx.obj["ci_report_bucket_name"], + report_output_prefix=ctx.obj["report_output_prefix"], + use_remote_secrets=ctx.obj["use_remote_secrets"], + gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), + dagger_logs_url=ctx.obj.get("dagger_logs_url"), + pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), + ci_context=ctx.obj.get("ci_context"), + ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], + ci_git_user=ctx.obj["ci_git_user"], + ci_github_access_token=ctx.obj["ci_github_access_token"], + enable_report_auto_open=False, + docker_hub_username=ctx.obj.get("docker_hub_username"), + docker_hub_password=ctx.obj.get("docker_hub_password"), + s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"), + s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"), + ) + for connector in ctx.obj["selected_connectors_with_modified_files"] + ] + + await run_connectors_pipelines( + connectors_contexts, + run_connector_migration_to_base_image_pipeline, + "Migration to base image pipeline", + ctx.obj["concurrency"], + ctx.obj["dagger_logs_path"], + ctx.obj["execute_timeout"], + pull_request_number, + ) + + return True diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/pipeline.py new file mode 100644 index 000000000000..38f0bf477713 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_base_image/pipeline.py @@ -0,0 +1,357 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import textwrap +from copy import deepcopy +from typing import TYPE_CHECKING + +from base_images import version_registry # type: ignore +from connector_ops.utils import ConnectorLanguage # type: ignore +from dagger import Directory +from jinja2 import Template +from pipelines.airbyte_ci.connectors.bump_version.pipeline import AddChangelogEntry, BumpDockerImageTagInMetadata, get_bumped_version +from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext +from pipelines.airbyte_ci.connectors.reports import ConnectorReport, Report +from pipelines.helpers import git +from pipelines.helpers.connectors import metadata_change_helpers +from pipelines.models.steps import Step, StepResult, StepStatus + +if TYPE_CHECKING: + from typing import Optional + + from anyio import Semaphore + + +class UpgradeBaseImageMetadata(Step): + context: ConnectorContext + + title = "Upgrade the base image to the latest version in metadata.yaml" + + def __init__( + self, + context: ConnectorContext, + repo_dir: Directory, + set_if_not_exists: bool = True, + ) -> None: + super().__init__(context) + self.repo_dir = repo_dir + self.set_if_not_exists = set_if_not_exists + + async def get_latest_base_image_address(self) -> "Optional[str]": + try: + version_registry_for_language = await version_registry.get_registry_for_language( + self.dagger_client, self.context.connector.language, (self.context.docker_hub_username, self.context.docker_hub_password) + ) + return version_registry_for_language.latest_not_pre_released_published_entry.published_docker_image.address + except NotImplementedError: + return None + + @staticmethod + def update_base_image_in_metadata(current_metadata: dict, latest_base_image_version_address: str) -> dict: + current_connector_build_options = current_metadata["data"].get("connectorBuildOptions", {}) + updated_metadata = deepcopy(current_metadata) + updated_metadata["data"]["connectorBuildOptions"] = { + **current_connector_build_options, + **{"baseImage": latest_base_image_version_address}, + } + return updated_metadata + + async def _run(self) -> StepResult: + latest_base_image_address = await self.get_latest_base_image_address() + if latest_base_image_address is None: + return StepResult( + self, + StepStatus.SKIPPED, + stdout="Could not find a base image for this connector language.", + output_artifact=self.repo_dir, + ) + + metadata_path = self.context.connector.metadata_file_path + current_metadata = await metadata_change_helpers.get_current_metadata(self.repo_dir, metadata_path) + current_base_image_address = current_metadata.get("data", {}).get("connectorBuildOptions", {}).get("baseImage") + + if current_base_image_address is None and not self.set_if_not_exists: + return StepResult( + self, + StepStatus.SKIPPED, + stdout="Connector does not have a base image metadata field.", + output_artifact=self.repo_dir, + ) + + if current_base_image_address == latest_base_image_address: + return StepResult( + self, + StepStatus.SKIPPED, + stdout="Connector already uses latest base image", + output_artifact=self.repo_dir, + ) + updated_metadata = self.update_base_image_in_metadata(current_metadata, latest_base_image_address) + updated_repo_dir = metadata_change_helpers.get_repo_dir_with_updated_metadata(self.repo_dir, metadata_path, updated_metadata) + + return StepResult( + self, + StepStatus.SUCCESS, + stdout=f"Updated base image to {latest_base_image_address} in {metadata_path}", + output_artifact=updated_repo_dir, + ) + + +class DeleteConnectorFile(Step): + context: ConnectorContext + + def __init__( + self, + context: ConnectorContext, + file_to_delete: str, + ) -> None: + super().__init__(context) + self.file_to_delete = file_to_delete + + @property + def title(self) -> str: + return f"Delete {self.file_to_delete}" + + async def _run(self) -> StepResult: + file_to_delete_path = self.context.connector.code_directory / self.file_to_delete + if not file_to_delete_path.exists(): + return StepResult( + self, + StepStatus.SKIPPED, + stdout=f"Connector does not have a {self.file_to_delete}", + ) + # As this is a deletion of a file, this has to happen on the host fs + # Deleting the file in a Directory container would not work because the directory.export method would not export the deleted file from the Directory back to host. + file_to_delete_path.unlink() + return StepResult( + self, + StepStatus.SUCCESS, + stdout=f"Deleted {file_to_delete_path}", + ) + + +class AddBuildInstructionsToReadme(Step): + context: ConnectorContext + + title = "Add build instructions to README.md" + + def __init__(self, context: PipelineContext, repo_dir: Directory) -> None: + super().__init__(context) + self.repo_dir = repo_dir + + async def _run(self) -> StepResult: + readme_path = self.context.connector.code_directory / "README.md" + if not readme_path.exists(): + return StepResult( + self, + StepStatus.SKIPPED, + stdout="Connector does not have a documentation file.", + output_artifact=self.repo_dir, + ) + current_readme = await (await self.context.get_connector_dir(include=["README.md"])).file("README.md").contents() + try: + updated_readme = self.add_build_instructions(current_readme) + except Exception as e: + return StepResult( + self, + StepStatus.FAILURE, + stdout=str(e), + output_artifact=self.repo_dir, + ) + updated_repo_dir = await self.repo_dir.with_new_file(str(readme_path), contents=updated_readme) + return StepResult( + self, + StepStatus.SUCCESS, + stdout=f"Added build instructions to {readme_path}", + output_artifact=updated_repo_dir, + ) + + def add_build_instructions(self, og_doc_content: str) -> str: + + build_instructions_template = Template( + textwrap.dedent( + """ + + #### Use `airbyte-ci` to build your connector + The Airbyte way of building this connector is to use our `airbyte-ci` tool. + You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). + Then running the following command will build your connector: + + ```bash + airbyte-ci connectors --name {{ connector_technical_name }} build + ``` + Once the command is done, you will find your connector image in your local docker registry: `{{ connector_image }}:dev`. + + ##### Customizing our build process + When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. + You can customize our build process by adding a `build_customization.py` module to your connector. + This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. + It will be imported at runtime by our build process and the functions will be called if they exist. + + Here is an example of a `build_customization.py` module: + ```python + from __future__ import annotations + + from typing import TYPE_CHECKING + + if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + + async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + + async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") + ``` + + #### Build your own connector image + This connector is built using our dynamic built process in `airbyte-ci`. + The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. + The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). + It does not rely on a Dockerfile. + + If you would like to patch our connector and build your own a simple approach would be to: + + 1. Create your own Dockerfile based on the latest version of the connector image. + ```Dockerfile + FROM {{ connector_image }}:latest + + COPY . ./airbyte/integration_code + RUN pip install ./airbyte/integration_code + + # The entrypoint and default env vars are already set in the base image + # ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" + # ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + ``` + Please use this as an example. This is not optimized. + + 2. Build your image: + ```bash + docker build -t {{ connector_image }}:dev . + # Running the spec command against your patched connector + docker run {{ connector_image }}:dev spec + ``` + """ + ) + ) + + build_instructions = build_instructions_template.render( + { + "connector_image": self.context.connector.metadata["dockerRepository"], + "connector_technical_name": self.context.connector.technical_name, + } + ) + + og_lines = og_doc_content.splitlines() + build_instructions_index = None + run_instructions_index = None + + for line_no, line in enumerate(og_lines): + if "#### Build" in line: + build_instructions_index = line_no + if "#### Run" in line: + run_instructions_index = line_no + break + + if build_instructions_index is None or run_instructions_index is None: + raise Exception("Could not find build or run instructions in README.md") + + new_doc = "\n".join(og_lines[:build_instructions_index] + build_instructions.splitlines() + og_lines[run_instructions_index:]) + return new_doc + + +async def run_connector_base_image_upgrade_pipeline(context: ConnectorContext, semaphore: "Semaphore", set_if_not_exists: bool) -> Report: + """Run a pipeline to upgrade for a single connector to use our base image.""" + async with semaphore: + steps_results = [] + async with context: + og_repo_dir = await context.get_repo_dir() + update_base_image_in_metadata = UpgradeBaseImageMetadata( + context, + og_repo_dir, + set_if_not_exists=set_if_not_exists, + ) + update_base_image_in_metadata_result = await update_base_image_in_metadata.run() + steps_results.append(update_base_image_in_metadata_result) + final_repo_dir = update_base_image_in_metadata_result.output_artifact + await og_repo_dir.diff(final_repo_dir).export(str(git.get_git_repo_path())) + report = ConnectorReport(context, steps_results, name="BASE IMAGE UPGRADE RESULTS") + context.report = report + return report + + +async def run_connector_migration_to_base_image_pipeline( + context: ConnectorContext, semaphore: "Semaphore", pull_request_number: str +) -> Report: + async with semaphore: + steps_results = [] + async with context: + # DELETE DOCKERFILE + delete_docker_file = DeleteConnectorFile( + context, + "Dockerfile", + ) + delete_docker_file_result = await delete_docker_file.run() + steps_results.append(delete_docker_file_result) + + # DELETE BUILD.GRADLE IF NOT JAVA + if context.connector.language is not ConnectorLanguage.JAVA: + delete_gradle_file = DeleteConnectorFile( + context, + "build.gradle", + ) + delete_gradle_file_result = await delete_gradle_file.run() + steps_results.append(delete_gradle_file_result) + + og_repo_dir = await context.get_repo_dir() + + # UPDATE BASE IMAGE IN METADATA + update_base_image_in_metadata = UpgradeBaseImageMetadata( + context, + og_repo_dir, + set_if_not_exists=True, + ) + update_base_image_in_metadata_result = await update_base_image_in_metadata.run() + steps_results.append(update_base_image_in_metadata_result) + if update_base_image_in_metadata_result.status is not StepStatus.SUCCESS: + context.report = ConnectorReport(context, steps_results, name="BASE IMAGE UPGRADE RESULTS") + return context.report + + # BUMP CONNECTOR VERSION IN METADATA + new_version = get_bumped_version(context.connector.version, "patch") + bump_version_in_metadata = BumpDockerImageTagInMetadata( + context, + update_base_image_in_metadata_result.output_artifact, + new_version, + ) + bump_version_in_metadata_result = await bump_version_in_metadata.run() + steps_results.append(bump_version_in_metadata_result) + + # ADD CHANGELOG ENTRY + add_changelog_entry = AddChangelogEntry( + context, + bump_version_in_metadata_result.output_artifact, + new_version, + "Base image migration: remove Dockerfile and use the python-connector-base image", + pull_request_number, + ) + add_changelog_entry_result = await add_changelog_entry.run() + steps_results.append(add_changelog_entry_result) + + # UPDATE DOC + add_build_instructions_to_doc = AddBuildInstructionsToReadme( + context, + add_changelog_entry_result.output_artifact, + ) + add_build_instructions_to_doc_results = await add_build_instructions_to_doc.run() + steps_results.append(add_build_instructions_to_doc_results) + + # EXPORT MODIFIED FILES BACK TO HOST + final_repo_dir = add_build_instructions_to_doc_results.output_artifact + await og_repo_dir.diff(final_repo_dir).export(str(git.get_git_repo_path())) + report = ConnectorReport(context, steps_results, name="MIGRATE TO BASE IMAGE RESULTS") + context.report = report + return report diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/pipeline.py new file mode 100644 index 000000000000..b4055029a74f --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/pipeline.py @@ -0,0 +1,124 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This module groups the functions to run full pipelines for connector testing.""" +from __future__ import annotations + +import sys +from pathlib import Path +from typing import TYPE_CHECKING, Any, Callable, List, Optional, Union + +import anyio +import dagger +from connector_ops.utils import ConnectorLanguage # type: ignore +from dagger import Config +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext +from pipelines.airbyte_ci.steps.no_op import NoOpStep +from pipelines.consts import ContextState +from pipelines.dagger.actions.system import docker +from pipelines.helpers.utils import create_and_open_file +from pipelines.models.reports import Report +from pipelines.models.steps import StepResult, StepStatus + +if TYPE_CHECKING: + from pipelines.models.contexts.pipeline_context import PipelineContext + +GITHUB_GLOBAL_CONTEXT = "[POC please ignore] Connectors CI" +GITHUB_GLOBAL_DESCRIPTION = "Running connectors tests" + +CONNECTOR_LANGUAGE_TO_FORCED_CONCURRENCY_MAPPING = { + # We run the Java connectors tests sequentially because we currently have memory issues when Java integration tests are run in parallel. + # See https://github.com/airbytehq/airbyte/issues/27168 + ConnectorLanguage.JAVA: anyio.Semaphore(1), +} + + +async def context_to_step_result(context: PipelineContext) -> StepResult: + if context.state == ContextState.SUCCESSFUL: + return await NoOpStep(context, StepStatus.SUCCESS).run() + + if context.state == ContextState.FAILURE: + return await NoOpStep(context, StepStatus.FAILURE).run() + + if context.state == ContextState.ERROR: + return await NoOpStep(context, StepStatus.FAILURE).run() + + raise ValueError(f"Could not convert context state: {context.state} to step status") + + +# HACK: This is to avoid wrapping the whole pipeline in a dagger pipeline to avoid instability just prior to launch +# TODO (ben): Refactor run_connectors_pipelines to wrap the whole pipeline in a dagger pipeline once Steps are refactored +async def run_report_complete_pipeline( + dagger_client: dagger.Client, contexts: List[ConnectorContext] | List[PublishConnectorContext] | List[PipelineContext] +) -> None: + """Create and Save a report representing the run of the encompassing pipeline. + + This is to denote when the pipeline is complete, useful for long running pipelines like nightlies. + """ + + if not contexts: + return + + # Repurpose the first context to be the pipeline upload context to preserve timestamps + first_connector_context = contexts[0] + + pipeline_name = f"Report upload {first_connector_context.report_output_prefix}" + first_connector_context.pipeline_name = pipeline_name + + # Transform contexts into a list of steps + steps_results = [await context_to_step_result(context) for context in contexts] + + report = Report( + name=pipeline_name, + pipeline_context=first_connector_context, + steps_results=steps_results, + filename="complete", + ) + + await report.save() + + +async def run_connectors_pipelines( + contexts: Union[List[ConnectorContext], List[PublishConnectorContext]], + connector_pipeline: Callable, + pipeline_name: str, + concurrency: int, + dagger_logs_path: Optional[Path], + execute_timeout: Optional[int], + *args: Any, +) -> List[ConnectorContext] | List[PublishConnectorContext]: + """Run a connector pipeline for all the connector contexts.""" + + default_connectors_semaphore = anyio.Semaphore(concurrency) + dagger_logs_output = sys.stderr if not dagger_logs_path else create_and_open_file(dagger_logs_path) + async with dagger.Connection(Config(log_output=dagger_logs_output, execute_timeout=execute_timeout)) as dagger_client: + docker_hub_username = contexts[0].docker_hub_username + docker_hub_password = contexts[0].docker_hub_password + + if docker_hub_username and docker_hub_password: + docker_hub_username_secret = dagger_client.set_secret("DOCKER_HUB_USERNAME", docker_hub_username) + docker_hub_password_secret = dagger_client.set_secret("DOCKER_HUB_PASSWORD", docker_hub_password) + dockerd_service = docker.with_global_dockerd_service(dagger_client, docker_hub_username_secret, docker_hub_password_secret) + else: + dockerd_service = docker.with_global_dockerd_service(dagger_client) + + await dockerd_service.start() + + async with anyio.create_task_group() as tg_connectors: + for context in contexts: + context.dagger_client = dagger_client.pipeline(f"{pipeline_name} - {context.connector.technical_name}") + context.dockerd_service = dockerd_service + tg_connectors.start_soon( + connector_pipeline, + context, + CONNECTOR_LANGUAGE_TO_FORCED_CONCURRENCY_MAPPING.get(context.connector.language, default_connectors_semaphore), + *args, + ) + + # When the connectors pipelines are done, we can stop the dockerd service + await dockerd_service.stop() + await run_report_complete_pipeline(dagger_client, contexts) + + return contexts diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py new file mode 100644 index 000000000000..0de1d7a2032f --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/commands.py @@ -0,0 +1,127 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import asyncclick as click +from pipelines import main_logger +from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines +from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext +from pipelines.airbyte_ci.connectors.publish.pipeline import reorder_contexts, run_connector_publish_pipeline +from pipelines.cli.click_decorators import click_ci_requirements_option +from pipelines.cli.confirm_prompt import confirm +from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand +from pipelines.consts import ContextState +from pipelines.helpers.utils import fail_if_missing_docker_hub_creds + + +@click.command(cls=DaggerPipelineCommand, help="Publish all images for the selected connectors.") +@click_ci_requirements_option() +@click.option("--pre-release/--main-release", help="Use this flag if you want to publish pre-release images.", default=True, type=bool) +@click.option( + "--spec-cache-gcs-credentials", + help="The service account key to upload files to the GCS bucket hosting spec cache.", + type=click.STRING, + required=True, + envvar="SPEC_CACHE_GCS_CREDENTIALS", +) +@click.option( + "--spec-cache-bucket-name", + help="The name of the GCS bucket where specs will be cached.", + type=click.STRING, + required=True, + envvar="SPEC_CACHE_BUCKET_NAME", +) +@click.option( + "--metadata-service-gcs-credentials", + help="The service account key to upload files to the GCS bucket hosting the metadata files.", + type=click.STRING, + required=True, + envvar="METADATA_SERVICE_GCS_CREDENTIALS", +) +@click.option( + "--metadata-service-bucket-name", + help="The name of the GCS bucket where metadata files will be uploaded.", + type=click.STRING, + required=True, + envvar="METADATA_SERVICE_BUCKET_NAME", +) +@click.option( + "--slack-webhook", + help="The Slack webhook URL to send notifications to.", + type=click.STRING, + envvar="SLACK_WEBHOOK", +) +@click.option( + "--slack-channel", + help="The Slack webhook URL to send notifications to.", + type=click.STRING, + envvar="SLACK_CHANNEL", + default="#connector-publish-updates", +) +@click.pass_context +async def publish( + ctx: click.Context, + pre_release: bool, + spec_cache_gcs_credentials: str, + spec_cache_bucket_name: str, + metadata_service_bucket_name: str, + metadata_service_gcs_credentials: str, + slack_webhook: str, + slack_channel: str, +) -> bool: + ctx.obj["spec_cache_gcs_credentials"] = spec_cache_gcs_credentials + ctx.obj["spec_cache_bucket_name"] = spec_cache_bucket_name + ctx.obj["metadata_service_bucket_name"] = metadata_service_bucket_name + ctx.obj["metadata_service_gcs_credentials"] = metadata_service_gcs_credentials + if ctx.obj["is_local"]: + confirm( + "Publishing from a local environment is not recommended and requires to be logged in Airbyte's DockerHub registry, do you want to continue?", + abort=True, + ) + + fail_if_missing_docker_hub_creds(ctx) + + publish_connector_contexts = reorder_contexts( + [ + PublishConnectorContext( + connector=connector, + pre_release=pre_release, + spec_cache_gcs_credentials=spec_cache_gcs_credentials, + spec_cache_bucket_name=spec_cache_bucket_name, + metadata_service_gcs_credentials=metadata_service_gcs_credentials, + metadata_bucket_name=metadata_service_bucket_name, + docker_hub_username=ctx.obj["docker_hub_username"], + docker_hub_password=ctx.obj["docker_hub_password"], + slack_webhook=slack_webhook, + reporting_slack_channel=slack_channel, + ci_report_bucket=ctx.obj["ci_report_bucket_name"], + report_output_prefix=ctx.obj["report_output_prefix"], + is_local=ctx.obj["is_local"], + git_branch=ctx.obj["git_branch"], + git_revision=ctx.obj["git_revision"], + gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), + dagger_logs_url=ctx.obj.get("dagger_logs_url"), + pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), + ci_context=ctx.obj.get("ci_context"), + ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], + pull_request=ctx.obj.get("pull_request"), + s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"), + s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"), + use_local_cdk=ctx.obj.get("use_local_cdk"), + ) + for connector in ctx.obj["selected_connectors_with_modified_files"] + ] + ) + main_logger.warn("Concurrency is forced to 1. For stability reasons we disable parallel publish pipelines.") + ctx.obj["concurrency"] = 1 + + ran_publish_connector_contexts = await run_connectors_pipelines( + publish_connector_contexts, + run_connector_publish_pipeline, + "Publishing connectors", + ctx.obj["concurrency"], + ctx.obj["dagger_logs_path"], + ctx.obj["execute_timeout"], + ) + return all(context.state is ContextState.SUCCESSFUL for context in ran_publish_connector_contexts) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py new file mode 100644 index 000000000000..a4471bac7eca --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py @@ -0,0 +1,127 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""Module declaring context related classes.""" + +from typing import Optional + +import asyncclick as click +from dagger import Secret +from github import PullRequest +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.consts import ContextState +from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles +from pipelines.helpers.gcs import sanitize_gcs_credentials +from pipelines.helpers.utils import format_duration + + +class PublishConnectorContext(ConnectorContext): + docker_hub_username_secret: Secret + docker_hub_password_secret: Secret + + def __init__( + self, + connector: ConnectorWithModifiedFiles, + pre_release: bool, + spec_cache_gcs_credentials: str, + spec_cache_bucket_name: str, + metadata_service_gcs_credentials: str, + metadata_bucket_name: str, + docker_hub_username: str, + docker_hub_password: str, + slack_webhook: str, + reporting_slack_channel: str, + ci_report_bucket: str, + report_output_prefix: str, + is_local: bool, + git_branch: str, + git_revision: str, + gha_workflow_run_url: Optional[str] = None, + dagger_logs_url: Optional[str] = None, + pipeline_start_timestamp: Optional[int] = None, + ci_context: Optional[str] = None, + ci_gcs_credentials: Optional[str] = None, + pull_request: Optional[PullRequest.PullRequest] = None, + s3_build_cache_access_key_id: Optional[str] = None, + s3_build_cache_secret_key: Optional[str] = None, + use_local_cdk: bool = False, + ) -> None: + self.pre_release = pre_release + self.spec_cache_bucket_name = spec_cache_bucket_name + self.metadata_bucket_name = metadata_bucket_name + self.spec_cache_gcs_credentials = sanitize_gcs_credentials(spec_cache_gcs_credentials) + self.metadata_service_gcs_credentials = sanitize_gcs_credentials(metadata_service_gcs_credentials) + pipeline_name = f"Publish {connector.technical_name}" + pipeline_name = pipeline_name + " (pre-release)" if pre_release else pipeline_name + + if use_local_cdk and not self.pre_release: + raise click.UsageError("Publishing with the local CDK is only supported for pre-release publishing.") + + super().__init__( + pipeline_name=pipeline_name, + connector=connector, + report_output_prefix=report_output_prefix, + ci_report_bucket=ci_report_bucket, + is_local=is_local, + git_branch=git_branch, + git_revision=git_revision, + gha_workflow_run_url=gha_workflow_run_url, + dagger_logs_url=dagger_logs_url, + pipeline_start_timestamp=pipeline_start_timestamp, + ci_context=ci_context, + slack_webhook=slack_webhook, + reporting_slack_channel=reporting_slack_channel, + ci_gcs_credentials=ci_gcs_credentials, + should_save_report=True, + use_local_cdk=use_local_cdk, + docker_hub_username=docker_hub_username, + docker_hub_password=docker_hub_password, + s3_build_cache_access_key_id=s3_build_cache_access_key_id, + s3_build_cache_secret_key=s3_build_cache_secret_key, + ) + + @property + def metadata_service_gcs_credentials_secret(self) -> Secret: + return self.dagger_client.set_secret("metadata_service_gcs_credentials", self.metadata_service_gcs_credentials) + + @property + def spec_cache_gcs_credentials_secret(self) -> Secret: + return self.dagger_client.set_secret("spec_cache_gcs_credentials", self.spec_cache_gcs_credentials) + + @property + def docker_image_tag(self) -> str: + # get the docker image tag from the parent class + metadata_tag = super().docker_image_tag + if self.pre_release: + return f"{metadata_tag}-dev.{self.git_revision[:10]}" + else: + return metadata_tag + + def create_slack_message(self) -> str: + + docker_hub_url = f"https://hub.docker.com/r/{self.connector.metadata['dockerRepository']}/tags" + message = f"*Publish <{docker_hub_url}|{self.docker_image}>*\n" + if self.is_ci: + message += f"🤖 <{self.gha_workflow_run_url}|GitHub Action workflow>\n" + else: + message += "🧑‍💻 Local run\n" + message += f"*Connector:* {self.connector.technical_name}\n" + message += f"*Version:* {self.connector.version}\n" + branch_url = f"https://github.com/airbytehq/airbyte/tree/{self.git_branch}" + message += f"*Branch:* <{branch_url}|{self.git_branch}>\n" + commit_url = f"https://github.com/airbytehq/airbyte/commit/{self.git_revision}" + message += f"*Commit:* <{commit_url}|{self.git_revision[:10]}>\n" + if self.state in [ContextState.INITIALIZED, ContextState.RUNNING]: + message += "🟠" + if self.state is ContextState.SUCCESSFUL: + message += "🟢" + if self.state in [ContextState.FAILURE, ContextState.ERROR]: + message += "🔴" + message += f" {self.state.value['description']}\n" + if self.state is ContextState.SUCCESSFUL: + assert self.report is not None, "Report should be set when state is successful" + message += f"⏲️ Run duration: {format_duration(self.report.run_duration)}\n" + if self.state is ContextState.FAILURE: + message += "\ncc. " # @dev-connector-ops + return message diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py new file mode 100644 index 000000000000..ba61a521ec66 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -0,0 +1,327 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import uuid +from typing import List, Tuple + +import anyio +from airbyte_protocol.models.airbyte_protocol import ConnectorSpecification # type: ignore +from dagger import Container, ExecError, File, ImageLayerCompression, QueryError +from pipelines import consts +from pipelines.airbyte_ci.connectors.build_image import steps +from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext +from pipelines.airbyte_ci.connectors.reports import ConnectorReport +from pipelines.airbyte_ci.metadata.pipeline import MetadataUpload, MetadataValidation +from pipelines.dagger.actions.remote_storage import upload_to_gcs +from pipelines.dagger.actions.system import docker +from pipelines.models.steps import Step, StepResult, StepStatus +from pydantic import ValidationError + + +class InvalidSpecOutputError(Exception): + pass + + +class CheckConnectorImageDoesNotExist(Step): + context: PublishConnectorContext + title = "Check if the connector docker image does not exist on the registry." + + async def _run(self) -> StepResult: + docker_repository, docker_tag = self.context.docker_image.split(":") + crane_ls = ( + docker.with_crane( + self.context, + ) + .with_env_variable("CACHEBUSTER", str(uuid.uuid4())) + .with_exec(["ls", docker_repository]) + ) + try: + crane_ls_stdout = await crane_ls.stdout() + except ExecError as e: + if "NAME_UNKNOWN" in e.stderr: + return StepResult(self, status=StepStatus.SUCCESS, stdout=f"The docker repository {docker_repository} does not exist.") + else: + return StepResult(self, status=StepStatus.FAILURE, stderr=e.stderr, stdout=e.stdout) + else: # The docker repo exists and ls was successful + existing_tags = crane_ls_stdout.split("\n") + docker_tag_already_exists = docker_tag in existing_tags + if docker_tag_already_exists: + return StepResult(self, status=StepStatus.SKIPPED, stderr=f"{self.context.docker_image} already exists.") + return StepResult(self, status=StepStatus.SUCCESS, stdout=f"No manifest found for {self.context.docker_image}.") + + +class PushConnectorImageToRegistry(Step): + context: PublishConnectorContext + title = "Push connector image to registry" + + @property + def latest_docker_image_name(self) -> str: + return f"{self.context.docker_repository}:latest" + + async def _run(self, built_containers_per_platform: List[Container], attempts: int = 3) -> StepResult: + try: + image_ref = await built_containers_per_platform[0].publish( + f"docker.io/{self.context.docker_image}", + platform_variants=built_containers_per_platform[1:], + forced_compression=ImageLayerCompression.Gzip, + ) + if not self.context.pre_release: + image_ref = await built_containers_per_platform[0].publish( + f"docker.io/{self.latest_docker_image_name}", + platform_variants=built_containers_per_platform[1:], + forced_compression=ImageLayerCompression.Gzip, + ) + return StepResult(self, status=StepStatus.SUCCESS, stdout=f"Published {image_ref}") + except QueryError as e: + if attempts > 0: + self.context.logger.error(str(e)) + self.context.logger.warn(f"Failed to publish {self.context.docker_image}. Retrying. {attempts} attempts left.") + await anyio.sleep(5) + return await self._run(built_containers_per_platform, attempts - 1) + return StepResult(self, status=StepStatus.FAILURE, stderr=str(e)) + + +class PullConnectorImageFromRegistry(Step): + context: PublishConnectorContext + title = "Pull connector image from registry" + + async def check_if_image_only_has_gzip_layers(self) -> bool: + """Check if the image only has gzip layers. + Docker version > 21 can create images that has some layers compressed with zstd. + These layers are not supported by previous docker versions. + We want to make sure that the image we are about to release is compatible with all docker versions. + We use crane to inspect the manifest of the image and check if it only has gzip layers. + """ + has_only_gzip_layers = True + for platform in consts.BUILD_PLATFORMS: + inspect = docker.with_crane(self.context).with_exec( + ["manifest", "--platform", f"{str(platform)}", f"docker.io/{self.context.docker_image}"] + ) + try: + inspect_stdout = await inspect.stdout() + except ExecError as e: + raise Exception(f"Failed to inspect {self.context.docker_image}: {e.stderr}") from e + try: + for layer in json.loads(inspect_stdout)["layers"]: + if not layer["mediaType"].endswith("gzip"): + has_only_gzip_layers = False + break + except (KeyError, json.JSONDecodeError) as e: + raise Exception(f"Failed to parse manifest for {self.context.docker_image}: {inspect_stdout}") from e + return has_only_gzip_layers + + async def _run(self, attempt: int = 3) -> StepResult: + try: + try: + await self.context.dagger_client.container().from_(f"docker.io/{self.context.docker_image}").with_exec(["spec"]) + except ExecError: + if attempt > 0: + await anyio.sleep(10) + return await self._run(attempt - 1) + else: + return StepResult(self, status=StepStatus.FAILURE, stderr=f"Failed to pull {self.context.docker_image}") + if not await self.check_if_image_only_has_gzip_layers(): + return StepResult( + self, + status=StepStatus.FAILURE, + stderr=f"Image {self.context.docker_image} does not only have gzip compressed layers. Please rebuild the connector with Docker < 21.", + ) + else: + return StepResult( + self, + status=StepStatus.SUCCESS, + stdout=f"Pulled {self.context.docker_image} and validated it has gzip only compressed layers and we can run spec on it.", + ) + except QueryError as e: + if attempt > 0: + await anyio.sleep(10) + return await self._run(attempt - 1) + return StepResult(self, status=StepStatus.FAILURE, stderr=str(e)) + + +class UploadSpecToCache(Step): + context: PublishConnectorContext + title = "Upload connector spec to spec cache bucket" + default_spec_file_name = "spec.json" + cloud_spec_file_name = "spec.cloud.json" + + @property + def spec_key_prefix(self) -> str: + return "specs/" + self.context.docker_image.replace(":", "/") + + @property + def cloud_spec_key(self) -> str: + return f"{self.spec_key_prefix}/{self.cloud_spec_file_name}" + + @property + def oss_spec_key(self) -> str: + return f"{self.spec_key_prefix}/{self.default_spec_file_name}" + + def _parse_spec_output(self, spec_output: str) -> str: + parsed_spec_message = None + for line in spec_output.split("\n"): + try: + parsed_json = json.loads(line) + if parsed_json["type"] == "SPEC": + parsed_spec_message = parsed_json + break + except (json.JSONDecodeError, KeyError): + continue + if parsed_spec_message: + parsed_spec = parsed_spec_message["spec"] + try: + ConnectorSpecification.parse_obj(parsed_spec) + return json.dumps(parsed_spec) + except (ValidationError, ValueError) as e: + raise InvalidSpecOutputError(f"The SPEC message did not pass schema validation: {str(e)}.") + raise InvalidSpecOutputError("No spec found in the output of the SPEC command.") + + async def _get_connector_spec(self, connector: Container, deployment_mode: str) -> str: + spec_output = await connector.with_env_variable("DEPLOYMENT_MODE", deployment_mode).with_exec(["spec"]).stdout() + return self._parse_spec_output(spec_output) + + async def _get_spec_as_file(self, spec: str, name: str = "spec_to_cache.json") -> File: + return (await self.context.get_connector_dir()).with_new_file(name, contents=spec).file(name) + + async def _run(self, built_connector: Container) -> StepResult: + try: + oss_spec: str = await self._get_connector_spec(built_connector, "OSS") + cloud_spec: str = await self._get_connector_spec(built_connector, "CLOUD") + except InvalidSpecOutputError as e: + return StepResult(self, status=StepStatus.FAILURE, stderr=str(e)) + + specs_to_uploads: List[Tuple[str, File]] = [(self.oss_spec_key, await self._get_spec_as_file(oss_spec))] + + if oss_spec != cloud_spec: + specs_to_uploads.append((self.cloud_spec_key, await self._get_spec_as_file(cloud_spec, "cloud_spec_to_cache.json"))) + + for key, file in specs_to_uploads: + exit_code, stdout, stderr = await upload_to_gcs( + self.context.dagger_client, + file, + key, + self.context.spec_cache_bucket_name, + self.context.spec_cache_gcs_credentials_secret, + flags=['--cache-control="no-cache"'], + ) + if exit_code != 0: + return StepResult(self, status=StepStatus.FAILURE, stdout=stdout, stderr=stderr) + return StepResult(self, status=StepStatus.SUCCESS, stdout="Uploaded connector spec to spec cache bucket.") + + +# Pipeline + + +async def run_connector_publish_pipeline(context: PublishConnectorContext, semaphore: anyio.Semaphore) -> ConnectorReport: + """Run a publish pipeline for a single connector. + + 1. Validate the metadata file. + 2. Check if the connector image already exists. + 3. Build the connector, with platform variants. + 4. Push the connector to DockerHub, with platform variants. + 5. Upload its spec to the spec cache bucket. + 6. Upload its metadata file to the metadata service bucket. + + Returns: + ConnectorReport: The reports holding publish results. + """ + + metadata_upload_step = MetadataUpload( + context=context, + metadata_service_gcs_credentials_secret=context.metadata_service_gcs_credentials_secret, + docker_hub_username_secret=context.docker_hub_username_secret, + docker_hub_password_secret=context.docker_hub_password_secret, + metadata_bucket_name=context.metadata_bucket_name, + pre_release=context.pre_release, + pre_release_tag=context.docker_image_tag, + ) + + def create_connector_report(results: List[StepResult]) -> ConnectorReport: + report = ConnectorReport(context, results, name="PUBLISH RESULTS") + context.report = report + return report + + async with semaphore: + async with context: + # TODO add a strucutre to hold the results of each step. and perform skips and failures + + results = [] + + metadata_validation_results = await MetadataValidation(context).run() + results.append(metadata_validation_results) + + # Exit early if the metadata file is invalid. + if metadata_validation_results.status is not StepStatus.SUCCESS: + return create_connector_report(results) + + check_connector_image_results = await CheckConnectorImageDoesNotExist(context).run() + results.append(check_connector_image_results) + + # If the connector image already exists, we don't need to build it, but we still need to upload the metadata file. + # We also need to upload the spec to the spec cache bucket. + if check_connector_image_results.status is StepStatus.SKIPPED: + context.logger.info( + "The connector version is already published. Let's upload metadata.yaml and spec to GCS even if no version bump happened." + ) + already_published_connector = context.dagger_client.container().from_(context.docker_image) + upload_to_spec_cache_results = await UploadSpecToCache(context).run(already_published_connector) + results.append(upload_to_spec_cache_results) + if upload_to_spec_cache_results.status is not StepStatus.SUCCESS: + return create_connector_report(results) + metadata_upload_results = await metadata_upload_step.run() + results.append(metadata_upload_results) + + # Exit early if the connector image already exists or has failed to build + if check_connector_image_results.status is not StepStatus.SUCCESS: + return create_connector_report(results) + + build_connector_results = await steps.run_connector_build(context) + results.append(build_connector_results) + + # Exit early if the connector image failed to build + if build_connector_results.status is not StepStatus.SUCCESS: + return create_connector_report(results) + + built_connector_platform_variants = list(build_connector_results.output_artifact.values()) + push_connector_image_results = await PushConnectorImageToRegistry(context).run(built_connector_platform_variants) + results.append(push_connector_image_results) + + # Exit early if the connector image failed to push + if push_connector_image_results.status is not StepStatus.SUCCESS: + return create_connector_report(results) + + # Make sure the image published is healthy by pulling it and running SPEC on it. + # See https://github.com/airbytehq/airbyte/issues/26085 + pull_connector_image_results = await PullConnectorImageFromRegistry(context).run() + results.append(pull_connector_image_results) + + # Exit early if the connector image failed to pull + if pull_connector_image_results.status is not StepStatus.SUCCESS: + return create_connector_report(results) + + upload_to_spec_cache_results = await UploadSpecToCache(context).run(built_connector_platform_variants[0]) + results.append(upload_to_spec_cache_results) + if upload_to_spec_cache_results.status is not StepStatus.SUCCESS: + return create_connector_report(results) + + metadata_upload_results = await metadata_upload_step.run() + results.append(metadata_upload_results) + connector_report = create_connector_report(results) + return connector_report + + +def reorder_contexts(contexts: List[PublishConnectorContext]) -> List[PublishConnectorContext]: + """Reorder contexts so that the ones that are for strict-encrypt/secure connectors come first. + The metadata upload on publish checks if the the connectors referenced in the metadata file are already published to DockerHub. + Non strict-encrypt variant reference the strict-encrypt variant in their metadata file for cloud. + So if we publish the non strict-encrypt variant first, the metadata upload will fail if the strict-encrypt variant is not published yet. + As strict-encrypt variant are often modified in the same PR as the non strict-encrypt variant, we want to publish them first. + """ + + def is_secure_variant(context: PublishConnectorContext) -> bool: + SECURE_VARIANT_KEYS = ["secure", "strict-encrypt"] + return any(key in context.connector.technical_name for key in SECURE_VARIANT_KEYS) + + return sorted(contexts, key=lambda context: (is_secure_variant(context), context.connector.technical_name), reverse=True) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py new file mode 100644 index 000000000000..b8265c4385a1 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/reports.py @@ -0,0 +1,157 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from __future__ import annotations + +import json +import webbrowser +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from anyio import Path +from connector_ops.utils import console # type: ignore +from jinja2 import Environment, PackageLoader, select_autoescape +from pipelines.consts import GCS_PUBLIC_DOMAIN +from pipelines.helpers.utils import format_duration +from pipelines.models.reports import Report +from pipelines.models.steps import StepStatus +from rich.console import Group +from rich.panel import Panel +from rich.style import Style +from rich.table import Table +from rich.text import Text + +if TYPE_CHECKING: + from typing import List + + from pipelines.airbyte_ci.connectors.context import ConnectorContext + from rich.tree import RenderableType + + +@dataclass(frozen=True) +class ConnectorReport(Report): + """A dataclass to build connector test reports to share pipelines executions results with the user.""" + + pipeline_context: ConnectorContext + + @property + def report_output_prefix(self) -> str: + return f"{self.pipeline_context.report_output_prefix}/{self.pipeline_context.connector.technical_name}/{self.pipeline_context.connector.version}" + + @property + def html_report_file_name(self) -> str: + return self.filename + ".html" + + @property + def html_report_remote_storage_key(self) -> str: + return f"{self.report_output_prefix}/{self.html_report_file_name}" + + @property + def html_report_url(self) -> str: + return f"{GCS_PUBLIC_DOMAIN}/{self.pipeline_context.ci_report_bucket}/{self.html_report_remote_storage_key}" + + def to_json(self) -> str: + """Create a JSON representation of the connector test report. + + Returns: + str: The JSON representation of the report. + """ + assert self.pipeline_context.pipeline_start_timestamp is not None, "The pipeline start timestamp must be set to save reports." + + return json.dumps( + { + "connector_technical_name": self.pipeline_context.connector.technical_name, + "connector_version": self.pipeline_context.connector.version, + "run_timestamp": self.created_at.isoformat(), + "run_duration": self.run_duration.total_seconds(), + "success": self.success, + "failed_steps": [s.step.__class__.__name__ for s in self.failed_steps], # type: ignore + "successful_steps": [s.step.__class__.__name__ for s in self.successful_steps], # type: ignore + "skipped_steps": [s.step.__class__.__name__ for s in self.skipped_steps], # type: ignore + "gha_workflow_run_url": self.pipeline_context.gha_workflow_run_url, + "pipeline_start_timestamp": self.pipeline_context.pipeline_start_timestamp, + "pipeline_end_timestamp": round(self.created_at.timestamp()), + "pipeline_duration": round(self.created_at.timestamp()) - self.pipeline_context.pipeline_start_timestamp, + "git_branch": self.pipeline_context.git_branch, + "git_revision": self.pipeline_context.git_revision, + "ci_context": self.pipeline_context.ci_context, + "cdk_version": self.pipeline_context.cdk_version, + "html_report_url": self.html_report_url, + "dagger_cloud_url": self.pipeline_context.dagger_cloud_url, + } + ) + + async def to_html(self) -> str: + env = Environment( + loader=PackageLoader("pipelines.airbyte_ci.connectors.test.steps"), + autoescape=select_autoescape(), + trim_blocks=False, + lstrip_blocks=True, + ) + template = env.get_template("test_report.html.j2") + template.globals["StepStatus"] = StepStatus + template.globals["format_duration"] = format_duration + local_icon_path = await Path(f"{self.pipeline_context.connector.code_directory}/icon.svg").resolve() + template_context = { + "connector_name": self.pipeline_context.connector.technical_name, + "step_results": self.steps_results, + "run_duration": self.run_duration, + "created_at": self.created_at.isoformat(), + "connector_version": self.pipeline_context.connector.version, + "gha_workflow_run_url": None, + "dagger_logs_url": None, + "git_branch": self.pipeline_context.git_branch, + "git_revision": self.pipeline_context.git_revision, + "commit_url": None, + "icon_url": local_icon_path.as_uri(), + } + + if self.pipeline_context.is_ci: + template_context["commit_url"] = f"https://github.com/airbytehq/airbyte/commit/{self.pipeline_context.git_revision}" + template_context["gha_workflow_run_url"] = self.pipeline_context.gha_workflow_run_url + template_context["dagger_logs_url"] = self.pipeline_context.dagger_logs_url + template_context["dagger_cloud_url"] = self.pipeline_context.dagger_cloud_url + template_context[ + "icon_url" + ] = f"https://raw.githubusercontent.com/airbytehq/airbyte/{self.pipeline_context.git_revision}/{self.pipeline_context.connector.code_directory}/icon.svg" + return template.render(template_context) + + async def save(self) -> None: + local_html_path = await self.save_local(self.html_report_file_name, await self.to_html()) + absolute_path = await local_html_path.resolve() + if self.pipeline_context.enable_report_auto_open: + self.pipeline_context.logger.info(f"HTML report saved locally: {absolute_path}") + if self.pipeline_context.enable_report_auto_open: + self.pipeline_context.logger.info("Opening HTML report in browser.") + webbrowser.open(absolute_path.as_uri()) + if self.remote_storage_enabled: + await self.save_remote(local_html_path, self.html_report_remote_storage_key, "text/html") + self.pipeline_context.logger.info(f"HTML report uploaded to {self.html_report_url}") + await super().save() + + def print(self) -> None: + """Print the test report to the console in a nice way.""" + connector_name = self.pipeline_context.connector.technical_name + main_panel_title = Text(f"{connector_name.upper()} - {self.name}") + main_panel_title.stylize(Style(color="blue", bold=True)) + duration_subtitle = Text(f"⏲️ Total pipeline duration for {connector_name}: {format_duration(self.run_duration)}") + step_results_table = Table(title="Steps results") + step_results_table.add_column("Step") + step_results_table.add_column("Result") + step_results_table.add_column("Duration") + + for step_result in self.steps_results: + step = Text(step_result.step.title) + step.stylize(step_result.status.get_rich_style()) + result = Text(step_result.status.value) + result.stylize(step_result.status.get_rich_style()) + step_results_table.add_row(step, result, format_duration(step_result.step.run_duration)) + + details_instructions = Text("ℹ️ You can find more details with step executions logs in the saved HTML report.") + to_render: List[RenderableType] = [step_results_table, details_instructions] + + if self.pipeline_context.dagger_cloud_url: + self.pipeline_context.logger.info(f"🔗 View runs for commit in Dagger Cloud: {self.pipeline_context.dagger_cloud_url}") + + main_panel = Panel(Group(*to_render), title=main_panel_title, subtitle=duration_subtitle) + console.print(main_panel) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py new file mode 100644 index 000000000000..245f6156228b --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/commands.py @@ -0,0 +1,158 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import sys +from typing import Dict, List + +import asyncclick as click +from pipelines import main_logger +from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines +from pipelines.airbyte_ci.connectors.test.pipeline import run_connector_test_pipeline +from pipelines.cli.click_decorators import click_ci_requirements_option +from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand +from pipelines.consts import LOCAL_BUILD_PLATFORM, ContextState +from pipelines.helpers.execution import argument_parsing +from pipelines.helpers.execution.run_steps import RunStepOptions +from pipelines.helpers.github import update_global_commit_status_check_for_tests +from pipelines.helpers.utils import fail_if_missing_docker_hub_creds +from pipelines.models.steps import STEP_PARAMS + + +@click.command( + cls=DaggerPipelineCommand, + help="Test all the selected connectors.", + context_settings=dict( + ignore_unknown_options=True, + ), +) +@click_ci_requirements_option() +@click.option( + "--code-tests-only", + is_flag=True, + help=("Only execute code tests. " "Metadata checks, QA, and acceptance tests will be skipped."), + default=False, + type=bool, +) +@click.option( + "--fail-fast", + help="When enabled, tests will fail fast.", + default=False, + type=bool, + is_flag=True, +) +@click.option( + "--concurrent-cat", + help="When enabled, the CAT tests will run concurrently. Be careful about rate limits", + default=False, + type=bool, + is_flag=True, +) +@click.option( + "--skip-step", + "-x", + "skip_steps", + multiple=True, + type=click.Choice([step_id.value for step_id in CONNECTOR_TEST_STEP_ID]), + help="Skip a step by name. Can be used multiple times to skip multiple steps.", +) +@click.option( + "--only-step", + "-k", + "only_steps", + multiple=True, + type=click.Choice([step_id.value for step_id in CONNECTOR_TEST_STEP_ID]), + help="Only run specific step by name. Can be used multiple times to keep multiple steps.", +) +@click.argument( + "extra_params", nargs=-1, type=click.UNPROCESSED, callback=argument_parsing.build_extra_params_mapping(CONNECTOR_TEST_STEP_ID) +) +@click.pass_context +async def test( + ctx: click.Context, + code_tests_only: bool, + fail_fast: bool, + concurrent_cat: bool, + skip_steps: List[str], + only_steps: List[str], + extra_params: Dict[CONNECTOR_TEST_STEP_ID, STEP_PARAMS], +) -> bool: + """Runs a test pipeline for the selected connectors. + + Args: + ctx (click.Context): The click context. + """ + if only_steps and skip_steps: + raise click.UsageError("Cannot use both --only-step and --skip-step at the same time.") + if ctx.obj["is_ci"]: + fail_if_missing_docker_hub_creds(ctx) + if ctx.obj["is_ci"] and ctx.obj["pull_request"] and ctx.obj["pull_request"].draft: + main_logger.info("Skipping connectors tests for draft pull request.") + sys.exit(0) + + if ctx.obj["selected_connectors_with_modified_files"]: + update_global_commit_status_check_for_tests(ctx.obj, "pending") + else: + main_logger.warn("No connector were selected for testing.") + update_global_commit_status_check_for_tests(ctx.obj, "success") + return True + + run_step_options = RunStepOptions( + fail_fast=fail_fast, + skip_steps=[CONNECTOR_TEST_STEP_ID(step_id) for step_id in skip_steps], + keep_steps=[CONNECTOR_TEST_STEP_ID(step_id) for step_id in only_steps], + step_params=extra_params, + ) + connectors_tests_contexts = [ + ConnectorContext( + pipeline_name=f"Testing connector {connector.technical_name}", + connector=connector, + is_local=ctx.obj["is_local"], + git_branch=ctx.obj["git_branch"], + git_revision=ctx.obj["git_revision"], + ci_report_bucket=ctx.obj["ci_report_bucket_name"], + report_output_prefix=ctx.obj["report_output_prefix"], + use_remote_secrets=ctx.obj["use_remote_secrets"], + gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), + dagger_logs_url=ctx.obj.get("dagger_logs_url"), + pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), + ci_context=ctx.obj.get("ci_context"), + pull_request=ctx.obj.get("pull_request"), + ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], + code_tests_only=code_tests_only, + use_local_cdk=ctx.obj.get("use_local_cdk"), + s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"), + s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"), + docker_hub_username=ctx.obj.get("docker_hub_username"), + docker_hub_password=ctx.obj.get("docker_hub_password"), + concurrent_cat=concurrent_cat, + run_step_options=run_step_options, + targeted_platforms=[LOCAL_BUILD_PLATFORM], + ) + for connector in ctx.obj["selected_connectors_with_modified_files"] + ] + + try: + await run_connectors_pipelines( + [connector_context for connector_context in connectors_tests_contexts], + run_connector_test_pipeline, + "Test Pipeline", + ctx.obj["concurrency"], + ctx.obj["dagger_logs_path"], + ctx.obj["execute_timeout"], + ) + except Exception as e: + main_logger.error("An error occurred while running the test pipeline", exc_info=e) + update_global_commit_status_check_for_tests(ctx.obj, "failure") + return False + + @ctx.call_on_close + def send_commit_status_check() -> None: + if ctx.obj["is_ci"]: + global_success = all(connector_context.state is ContextState.SUCCESSFUL for connector_context in connectors_tests_contexts) + update_global_commit_status_check_for_tests(ctx.obj, "success" if global_success else "failure") + + # If we reach this point, it means that all the connectors have been tested so the pipeline did its job and can exit with success. + return True diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/pipeline.py new file mode 100644 index 000000000000..d1b875a1c180 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/pipeline.py @@ -0,0 +1,79 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +"""This module groups factory like functions to dispatch tests steps according to the connector under test language.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import anyio +from connector_ops.utils import ConnectorLanguage # type: ignore +from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.connectors.reports import ConnectorReport +from pipelines.airbyte_ci.connectors.test.steps import java_connectors, python_connectors +from pipelines.airbyte_ci.connectors.test.steps.common import QaChecks, VersionFollowsSemverCheck, VersionIncrementCheck +from pipelines.airbyte_ci.metadata.pipeline import MetadataValidation +from pipelines.helpers.execution.run_steps import StepToRun, run_steps + +if TYPE_CHECKING: + + from pipelines.helpers.execution.run_steps import STEP_TREE + +LANGUAGE_MAPPING = { + "get_test_steps": { + ConnectorLanguage.PYTHON: python_connectors.get_test_steps, + ConnectorLanguage.LOW_CODE: python_connectors.get_test_steps, + ConnectorLanguage.JAVA: java_connectors.get_test_steps, + }, +} + + +def get_test_steps(context: ConnectorContext) -> STEP_TREE: + """Get all the tests steps according to the connector language. + + Args: + context (ConnectorContext): The current connector context. + + Returns: + STEP_TREE: The list of tests steps. + """ + if _get_test_steps := LANGUAGE_MAPPING["get_test_steps"].get(context.connector.language): + return _get_test_steps(context) + else: + context.logger.warning(f"No tests defined for connector language {context.connector.language}!") + return [] + + +async def run_connector_test_pipeline(context: ConnectorContext, semaphore: anyio.Semaphore) -> ConnectorReport: + """ + Compute the steps to run for a connector test pipeline. + """ + all_steps_to_run: STEP_TREE = [] + + all_steps_to_run += get_test_steps(context) + + if not context.code_tests_only: + static_analysis_steps_to_run = [ + [ + StepToRun(id=CONNECTOR_TEST_STEP_ID.METADATA_VALIDATION, step=MetadataValidation(context)), + StepToRun(id=CONNECTOR_TEST_STEP_ID.VERSION_FOLLOW_CHECK, step=VersionFollowsSemverCheck(context)), + StepToRun(id=CONNECTOR_TEST_STEP_ID.VERSION_INC_CHECK, step=VersionIncrementCheck(context)), + StepToRun(id=CONNECTOR_TEST_STEP_ID.QA_CHECKS, step=QaChecks(context)), + ] + ] + all_steps_to_run += static_analysis_steps_to_run + + async with semaphore: + async with context: + result_dict = await run_steps( + runnables=all_steps_to_run, + options=context.run_step_options, + ) + + results = list(result_dict.values()) + report = ConnectorReport(context, steps_results=results, name="TEST RESULTS") + context.report = report + + return report diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py new file mode 100644 index 000000000000..dc780ac50f1a --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py @@ -0,0 +1,347 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This module groups steps made to run tests agnostic to a connector language.""" + +import datetime +import os +from abc import ABC, abstractmethod +from functools import cached_property +from typing import Any, ClassVar, List, Optional + +import requests # type: ignore +import semver +import yaml # type: ignore +from connector_ops.utils import Connector # type: ignore +from dagger import Container, Directory +from pipelines import hacks +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.consts import CIContext +from pipelines.dagger.actions import secrets +from pipelines.dagger.containers import internal_tools +from pipelines.helpers.utils import METADATA_FILE_NAME +from pipelines.models.steps import STEP_PARAMS, Step, StepResult, StepStatus + + +class VersionCheck(Step, ABC): + """A step to validate the connector version was bumped if files were modified""" + + context: ConnectorContext + GITHUB_URL_PREFIX_FOR_CONNECTORS = "https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-integrations/connectors" + failure_message: ClassVar + + @property + def should_run(self) -> bool: + return True + + @property + def github_master_metadata_url(self) -> str: + return f"{self.GITHUB_URL_PREFIX_FOR_CONNECTORS}/{self.context.connector.technical_name}/{METADATA_FILE_NAME}" + + @cached_property + def master_metadata(self) -> Optional[dict]: + response = requests.get(self.github_master_metadata_url) + + # New connectors will not have a metadata file in master + if not response.ok: + return None + return yaml.safe_load(response.text) + + @property + def master_connector_version(self) -> semver.Version: + metadata = self.master_metadata + if not metadata: + return semver.Version.parse("0.0.0") + + return semver.Version.parse(str(metadata["data"]["dockerImageTag"])) + + @property + def current_connector_version(self) -> semver.Version: + return semver.Version.parse(str(self.context.metadata["dockerImageTag"])) + + @property + def success_result(self) -> StepResult: + return StepResult(self, status=StepStatus.SUCCESS) + + @property + def failure_result(self) -> StepResult: + return StepResult(self, status=StepStatus.FAILURE, stderr=self.failure_message) + + @abstractmethod + def validate(self) -> StepResult: + raise NotImplementedError() + + async def _run(self) -> StepResult: + if not self.should_run: + return StepResult(self, status=StepStatus.SKIPPED, stdout="No modified files required a version bump.") + if self.context.ci_context == CIContext.MASTER: + return StepResult(self, status=StepStatus.SKIPPED, stdout="Version check are not running in master context.") + try: + return self.validate() + except (requests.HTTPError, ValueError, TypeError) as e: + return StepResult(self, status=StepStatus.FAILURE, stderr=str(e)) + + +class VersionIncrementCheck(VersionCheck): + context: ConnectorContext + title = "Connector version increment check" + + BYPASS_CHECK_FOR = [ + METADATA_FILE_NAME, + "acceptance-test-config.yml", + "README.md", + "bootstrap.md", + ".dockerignore", + "unit_tests", + "integration_tests", + "src/test", + "src/test-integration", + "src/test-performance", + "build.gradle", + ] + + @property + def failure_message(self) -> str: + return f"The dockerImageTag in {METADATA_FILE_NAME} was not incremented. The files you modified should lead to a version bump. Master version is {self.master_connector_version}, current version is {self.current_connector_version}" + + @property + def should_run(self) -> bool: + for filename in self.context.modified_files: + relative_path = str(filename).replace(str(self.context.connector.code_directory) + "/", "") + if not any([relative_path.startswith(to_bypass) for to_bypass in self.BYPASS_CHECK_FOR]): + return True + return False + + def validate(self) -> StepResult: + if not self.current_connector_version > self.master_connector_version: + return self.failure_result + return self.success_result + + +class VersionFollowsSemverCheck(VersionCheck): + context: ConnectorContext + title = "Connector version semver check" + + @property + def failure_message(self) -> str: + return f"The dockerImageTag in {METADATA_FILE_NAME} is not following semantic versioning or was decremented. Master version is {self.master_connector_version}, current version is {self.current_connector_version}" + + def validate(self) -> StepResult: + try: + if not self.current_connector_version >= self.master_connector_version: + return self.failure_result + except ValueError: + return self.failure_result + return self.success_result + + +class QaChecks(Step): + """A step to run QA checks for a connector.""" + + context: ConnectorContext + title = "QA checks" + + async def _run(self) -> StepResult: + """Run QA checks on a connector. + + The QA checks are defined in this module: + https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connector_ops/connector_ops/qa_checks.py + + Args: + context (ConnectorContext): The current test context, providing a connector object, a dagger client and a repository directory. + Returns: + StepResult: Failure or success of the QA checks with stdout and stderr. + """ + connector_ops = await internal_tools.with_connector_ops(self.context) + include = [ + str(self.context.connector.code_directory), + str(self.context.connector.documentation_file_path), + str(self.context.connector.migration_guide_file_path), + str(self.context.connector.icon_path), + ] + if ( + self.context.connector.technical_name.endswith("strict-encrypt") + or self.context.connector.technical_name == "source-file-secure" + ): + original_connector = Connector(self.context.connector.technical_name.replace("-strict-encrypt", "").replace("-secure", "")) + include += [ + str(original_connector.code_directory), + str(original_connector.documentation_file_path), + str(original_connector.icon_path), + str(original_connector.migration_guide_file_path), + ] + + filtered_repo = self.context.get_repo_dir( + include=include, + ) + + qa_checks = ( + connector_ops.with_mounted_directory("/airbyte", filtered_repo) + .with_workdir("/airbyte") + .with_exec(["run-qa-checks", f"connectors/{self.context.connector.technical_name}"]) + ) + + return await self.get_step_result(qa_checks) + + +class AcceptanceTests(Step): + """A step to run acceptance tests for a connector if it has an acceptance test config file.""" + + context: ConnectorContext + title = "Acceptance tests" + CONTAINER_TEST_INPUT_DIRECTORY = "/test_input" + CONTAINER_SECRETS_DIRECTORY = "/test_input/secrets" + skipped_exit_code = 5 + accept_extra_params = True + + @property + def default_params(self) -> STEP_PARAMS: + """Default pytest options. + + Returns: + dict: The default pytest options. + """ + return super().default_params | { + "-ra": [], # Show extra test summary info in the report for all but the passed tests + "--disable-warnings": [], # Disable warnings in the pytest report + "--durations": ["3"], # Show the 3 slowest tests in the report + } + + @property + def base_cat_command(self) -> List[str]: + command = [ + "python", + "-m", + "pytest", + "-p", # Load the connector_acceptance_test plugin + "connector_acceptance_test.plugin", + "--acceptance-test-config", + self.CONTAINER_TEST_INPUT_DIRECTORY, + ] + + if self.concurrent_test_run: + command += ["--numprocesses=auto"] # Using pytest-xdist to run tests in parallel, auto means using all available cores + return command + + def __init__(self, context: ConnectorContext, concurrent_test_run: Optional[bool] = False) -> None: + """Create a step to run acceptance tests for a connector if it has an acceptance test config file. + + Args: + context (ConnectorContext): The current test context, providing a connector object, a dagger client and a repository directory. + concurrent_test_run (Optional[bool], optional): Whether to run acceptance tests in parallel. Defaults to False. + """ + super().__init__(context) + self.concurrent_test_run = concurrent_test_run + + async def get_cat_command(self, connector_dir: Directory) -> List[str]: + """ + Connectors can optionally setup or teardown resources before and after the acceptance tests are run. + This is done via the acceptance.py file in their integration_tests directory. + We append this module as a plugin the acceptance will use. + """ + cat_command = self.base_cat_command + if "integration_tests" in await connector_dir.entries(): + if "acceptance.py" in await connector_dir.directory("integration_tests").entries(): + cat_command += ["-p", "integration_tests.acceptance"] + return cat_command + self.params_as_cli_options + + async def _run(self, connector_under_test_container: Container) -> StepResult: + """Run the acceptance test suite on a connector dev image. Build the connector acceptance test image if the tag is :dev. + + Args: + connector_under_test_container (Container): The container holding the connector under test image. + + Returns: + StepResult: Failure or success of the acceptances tests with stdout and stderr. + """ + + if not self.context.connector.acceptance_test_config: + return StepResult(self, StepStatus.SKIPPED) + connector_dir = await self.context.get_connector_dir() + cat_container = await self._build_connector_acceptance_test(connector_under_test_container, connector_dir) + cat_command = await self.get_cat_command(connector_dir) + cat_container = cat_container.with_(hacks.never_fail_exec(cat_command)) + step_result = await self.get_step_result(cat_container) + secret_dir = cat_container.directory(self.CONTAINER_SECRETS_DIRECTORY) + + if secret_files := await secret_dir.entries(): + for file_path in secret_files: + if file_path.startswith("updated_configurations"): + self.context.updated_secrets_dir = secret_dir + break + return step_result + + async def get_cache_buster(self) -> str: + """ + This bursts the CAT cached results everyday and on new version or image size change. + It's cool because in case of a partially failing nightly build the connectors that already ran CAT won't re-run CAT. + We keep the guarantee that a CAT runs everyday. + + Returns: + str: A string representing the cachebuster value. + """ + return datetime.datetime.utcnow().strftime("%Y%m%d") + self.context.connector.version + + async def _build_connector_acceptance_test(self, connector_under_test_container: Container, test_input: Directory) -> Container: + """Create a container to run connector acceptance tests. + + Args: + connector_under_test_container (Container): The container holding the connector under test image. + test_input (Directory): The connector under test directory. + Returns: + Container: A container with connector acceptance tests installed. + """ + + if self.context.connector_acceptance_test_image.endswith(":dev"): + cat_container = self.context.connector_acceptance_test_source_dir.docker_build() + else: + cat_container = self.dagger_client.container().from_(self.context.connector_acceptance_test_image) + + connector_container_id = await connector_under_test_container.id() + + cat_container = ( + cat_container.with_env_variable("RUN_IN_AIRBYTE_CI", "1") + .with_exec(["mkdir", "/dagger_share"], skip_entrypoint=True) + .with_env_variable("CACHEBUSTER", await self.get_cache_buster()) + .with_new_file("/tmp/container_id.txt", contents=str(connector_container_id)) + .with_workdir("/test_input") + .with_mounted_directory("/test_input", test_input) + .with_(await secrets.mounted_connector_secrets(self.context, self.CONTAINER_SECRETS_DIRECTORY)) + ) + if "_EXPERIMENTAL_DAGGER_RUNNER_HOST" in os.environ: + self.context.logger.info("Using experimental dagger runner host to run CAT with dagger-in-dagger") + cat_container = cat_container.with_env_variable( + "_EXPERIMENTAL_DAGGER_RUNNER_HOST", "unix:///var/run/buildkit/buildkitd.sock" + ).with_unix_socket( + "/var/run/buildkit/buildkitd.sock", self.context.dagger_client.host().unix_socket("/var/run/buildkit/buildkitd.sock") + ) + + return cat_container.with_unix_socket("/var/run/docker.sock", self.context.dagger_client.host().unix_socket("/var/run/docker.sock")) + + +class CheckBaseImageIsUsed(Step): + context: ConnectorContext + title = "Check our base image is used" + + async def _run(self, *args: Any, **kwargs: Any) -> StepResult: + is_certified = self.context.connector.metadata.get("supportLevel") == "certified" + if not is_certified: + return self.skip("Connector is not certified, it does not require the use of our base image.") + + is_using_base_image = self.context.connector.metadata.get("connectorBuildOptions", {}).get("baseImage") is not None + migration_hint = f"Please run 'airbyte-ci connectors --name={self.context.connector.technical_name} migrate_to_base_image ' and commit the changes." + if not is_using_base_image: + return StepResult( + self, + StepStatus.FAILURE, + stdout=f"Connector is certified but does not use our base image. {migration_hint}", + ) + has_dockerfile = "Dockerfile" in await (await self.context.get_connector_dir(include=["Dockerfile"])).entries() + if has_dockerfile: + return StepResult( + self, + StepStatus.FAILURE, + stdout=f"Connector is certified but is still using a Dockerfile. {migration_hint}", + ) + return StepResult(self, StepStatus.SUCCESS, stdout="Connector is certified and uses our base image.") diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/java_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/java_connectors.py new file mode 100644 index 000000000000..06b0aaea4314 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/java_connectors.py @@ -0,0 +1,170 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This module groups steps made to run tests for a specific Java connector given a test context.""" +from __future__ import annotations + +from typing import TYPE_CHECKING + +import anyio +from dagger import File, QueryError +from pipelines.airbyte_ci.connectors.build_image.steps.java_connectors import ( + BuildConnectorDistributionTar, + BuildConnectorImages, + dist_tar_directory_path, +) +from pipelines.airbyte_ci.connectors.build_image.steps.normalization import BuildOrPullNormalization +from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.connectors.test.steps.common import AcceptanceTests +from pipelines.airbyte_ci.steps.gradle import GradleTask +from pipelines.consts import LOCAL_BUILD_PLATFORM +from pipelines.dagger.actions.system import docker +from pipelines.helpers.execution.run_steps import StepToRun +from pipelines.helpers.utils import export_container_to_tarball +from pipelines.models.steps import STEP_PARAMS, StepResult, StepStatus + +if TYPE_CHECKING: + from typing import Callable, Dict, List, Optional + + from pipelines.helpers.execution.run_steps import RESULTS_DICT, STEP_TREE + + +class IntegrationTests(GradleTask): + """A step to run integrations tests for Java connectors using the integrationTestJava Gradle task.""" + + title = "Java Connector Integration Tests" + gradle_task_name = "integrationTestJava" + mount_connector_secrets = True + bind_to_docker_host = True + + @property + def default_params(self) -> STEP_PARAMS: + return super().default_params | { + "-x": ["buildConnectorImage", "assemble"], # Exclude the buildConnectorImage and assemble tasks + } + + async def _load_normalization_image(self, normalization_tar_file: File) -> None: + normalization_image_tag = f"{self.context.connector.normalization_repository}:dev" + self.context.logger.info("Load the normalization image to the docker host.") + await docker.load_image_to_docker_host(self.context, normalization_tar_file, normalization_image_tag) + self.context.logger.info("Successfully loaded the normalization image to the docker host.") + + async def _load_connector_image(self, connector_tar_file: File) -> None: + connector_image_tag = f"airbyte/{self.context.connector.technical_name}:dev" + self.context.logger.info("Load the connector image to the docker host") + await docker.load_image_to_docker_host(self.context, connector_tar_file, connector_image_tag) + self.context.logger.info("Successfully loaded the connector image to the docker host.") + + async def _run(self, connector_tar_file: File, normalization_tar_file: Optional[File]) -> StepResult: + try: + async with anyio.create_task_group() as tg: + if normalization_tar_file: + tg.start_soon(self._load_normalization_image, normalization_tar_file) + tg.start_soon(self._load_connector_image, connector_tar_file) + except QueryError as e: + return StepResult(self, StepStatus.FAILURE, stderr=str(e)) + # Run the gradle integration test task now that the required docker images have been loaded. + return await super()._run() + + +class UnitTests(GradleTask): + """A step to run unit tests for Java connectors.""" + + title = "Java Connector Unit Tests" + gradle_task_name = "test" + bind_to_docker_host = True + + +def _create_integration_step_args_factory(context: ConnectorContext) -> Callable: + """ + Create a function that can process the args for the integration step. + """ + + async def _create_integration_step_args(results: RESULTS_DICT) -> Dict[str, Optional[File]]: + + connector_container = results["build"].output_artifact[LOCAL_BUILD_PLATFORM] + connector_image_tar_file, _ = await export_container_to_tarball(context, connector_container, LOCAL_BUILD_PLATFORM) + + if context.connector.supports_normalization: + tar_file_name = f"{context.connector.normalization_repository}_{context.git_revision}.tar" + build_normalization_results = results["build_normalization"] + + normalization_container = build_normalization_results.output_artifact + normalization_tar_file, _ = await export_container_to_tarball( + context, normalization_container, LOCAL_BUILD_PLATFORM, tar_file_name=tar_file_name + ) + else: + normalization_tar_file = None + + return {"connector_tar_file": connector_image_tar_file, "normalization_tar_file": normalization_tar_file} + + return _create_integration_step_args + + +def _get_normalization_steps(context: ConnectorContext) -> List[StepToRun]: + normalization_image = f"{context.connector.normalization_repository}:dev" + context.logger.info(f"This connector supports normalization: will build {normalization_image}.") + normalization_steps = [ + StepToRun( + id=CONNECTOR_TEST_STEP_ID.BUILD_NORMALIZATION, + step=BuildOrPullNormalization(context, normalization_image, LOCAL_BUILD_PLATFORM), + depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + ) + ] + + return normalization_steps + + +def _get_acceptance_test_steps(context: ConnectorContext) -> List[StepToRun]: + """ + Generate the steps to run the acceptance tests for a Java connector. + """ + # Run tests in parallel + return [ + StepToRun( + id=CONNECTOR_TEST_STEP_ID.INTEGRATION, + step=IntegrationTests(context), + args=_create_integration_step_args_factory(context), + depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + ), + StepToRun( + id=CONNECTOR_TEST_STEP_ID.ACCEPTANCE, + step=AcceptanceTests(context, True), + args=lambda results: { + "connector_under_test_container": results[CONNECTOR_TEST_STEP_ID.BUILD].output_artifact[LOCAL_BUILD_PLATFORM] + }, + depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + ), + ] + + +def get_test_steps(context: ConnectorContext) -> STEP_TREE: + """ + Get all the tests steps for a Java connector. + """ + + steps: STEP_TREE = [ + [StepToRun(id=CONNECTOR_TEST_STEP_ID.BUILD_TAR, step=BuildConnectorDistributionTar(context))], + [StepToRun(id=CONNECTOR_TEST_STEP_ID.UNIT, step=UnitTests(context), depends_on=[CONNECTOR_TEST_STEP_ID.BUILD_TAR])], + [ + StepToRun( + id=CONNECTOR_TEST_STEP_ID.BUILD, + step=BuildConnectorImages(context), + args=lambda results: { + "dist_dir": results[CONNECTOR_TEST_STEP_ID.BUILD_TAR].output_artifact.directory(dist_tar_directory_path(context)) + }, + depends_on=[CONNECTOR_TEST_STEP_ID.BUILD_TAR], + ), + ], + ] + + if context.connector.supports_normalization: + normalization_steps = _get_normalization_steps(context) + steps.append(normalization_steps) + + acceptance_test_steps = _get_acceptance_test_steps(context) + steps.append(acceptance_test_steps) + + return steps diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py new file mode 100644 index 000000000000..5b2d71c465c6 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/python_connectors.py @@ -0,0 +1,233 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This module groups steps made to run tests for a specific Python connector given a test context.""" + +from abc import ABC, abstractmethod +from typing import List, Sequence, Tuple + +import pipelines.dagger.actions.python.common +import pipelines.dagger.actions.system.docker +from dagger import Container, File +from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages +from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.connectors.test.steps.common import AcceptanceTests, CheckBaseImageIsUsed +from pipelines.consts import LOCAL_BUILD_PLATFORM +from pipelines.dagger.actions import secrets +from pipelines.helpers.execution.run_steps import STEP_TREE, StepToRun +from pipelines.models.steps import STEP_PARAMS, Step, StepResult + + +class PytestStep(Step, ABC): + """An abstract class to run pytest tests and evaluate success or failure according to pytest logs.""" + + context: ConnectorContext + + PYTEST_INI_FILE_NAME = "pytest.ini" + PYPROJECT_FILE_NAME = "pyproject.toml" + common_test_dependencies: List[str] = [] + + skipped_exit_code = 5 + bind_to_docker_host = False + accept_extra_params = True + + @property + def default_params(self) -> STEP_PARAMS: + """Default pytest options. + + Returns: + dict: The default pytest options. + """ + return super().default_params | { + "-s": [], # Disable capturing stdout/stderr in pytest + } + + @property + @abstractmethod + def test_directory_name(self) -> str: + raise NotImplementedError("test_directory_name must be implemented in the child class.") + + @property + def extra_dependencies_names(self) -> Sequence[str]: + if self.context.connector.is_using_poetry: + return ("dev",) + return ("dev", "tests") + + async def _run(self, connector_under_test: Container) -> StepResult: + """Run all pytest tests declared in the test directory of the connector code. + + Args: + connector_under_test (Container): The connector under test container. + + Returns: + StepResult: Failure or success of the unit tests with stdout and stdout. + """ + if not await self.check_if_tests_are_available(self.test_directory_name): + return self.skip(f"No {self.test_directory_name} directory found in the connector.") + + test_config_file_name, test_config_file = await self.get_config_file_name_and_file() + test_environment = await self.install_testing_environment( + connector_under_test, test_config_file_name, test_config_file, self.extra_dependencies_names + ) + pytest_command = self.get_pytest_command(test_config_file_name) + + if self.bind_to_docker_host: + test_environment = pipelines.dagger.actions.system.docker.with_bound_docker_host(self.context, test_environment) + + test_execution = test_environment.with_exec(pytest_command) + + return await self.get_step_result(test_execution) + + def get_pytest_command(self, test_config_file_name: str) -> List[str]: + """Get the pytest command to run. + + Returns: + List[str]: The pytest command to run. + """ + cmd = ["pytest", self.test_directory_name, "-c", test_config_file_name] + self.params_as_cli_options + if self.context.connector.is_using_poetry: + return ["poetry", "run"] + cmd + return cmd + + async def check_if_tests_are_available(self, test_directory_name: str) -> bool: + """Check if the tests are available in the connector directory. + + Returns: + bool: True if the tests are available. + """ + connector_dir = await self.context.get_connector_dir() + connector_dir_entries = await connector_dir.entries() + return test_directory_name in connector_dir_entries + + async def get_config_file_name_and_file(self) -> Tuple[str, File]: + """Get the config file name and file to use for pytest. + + The order of priority is: + - pytest.ini file in the connector directory + - pyproject.toml file in the connector directory + - pyproject.toml file in the repository directory + + Returns: + Tuple[str, File]: The config file name and file to use for pytest. + """ + connector_dir = await self.context.get_connector_dir() + connector_dir_entries = await connector_dir.entries() + if self.PYTEST_INI_FILE_NAME in connector_dir_entries: + config_file_name = self.PYTEST_INI_FILE_NAME + test_config = (await self.context.get_connector_dir(include=[self.PYTEST_INI_FILE_NAME])).file(self.PYTEST_INI_FILE_NAME) + self.logger.info(f"Found {self.PYTEST_INI_FILE_NAME}, using it for testing.") + elif self.PYPROJECT_FILE_NAME in connector_dir_entries: + config_file_name = self.PYPROJECT_FILE_NAME + test_config = (await self.context.get_connector_dir(include=[self.PYPROJECT_FILE_NAME])).file(self.PYPROJECT_FILE_NAME) + self.logger.info(f"Found {self.PYPROJECT_FILE_NAME} at connector level, using it for testing.") + else: + config_file_name = f"global_{self.PYPROJECT_FILE_NAME}" + test_config = (await self.context.get_repo_dir(include=[self.PYPROJECT_FILE_NAME])).file(self.PYPROJECT_FILE_NAME) + self.logger.info(f"Found {self.PYPROJECT_FILE_NAME} at repo level, using it for testing.") + return config_file_name, test_config + + async def install_testing_environment( + self, + built_connector_container: Container, + test_config_file_name: str, + test_config_file: File, + extra_dependencies_names: Sequence[str], + ) -> Container: + """Install the connector with the extra dependencies in /test_environment. + + Args: + extra_dependencies_names (List[str]): Extra dependencies to install. + + Returns: + Container: The container with the test environment installed. + """ + secret_mounting_function = await secrets.mounted_connector_secrets(self.context, "secrets") + + container_with_test_deps = ( + # Install the connector python package in /test_environment with the extra dependencies + await pipelines.dagger.actions.python.common.with_python_connector_installed( + self.context, + # Reset the entrypoint to run non airbyte commands + built_connector_container.with_entrypoint([]), + str(self.context.connector.code_directory), + additional_dependency_groups=extra_dependencies_names, + ) + ) + if self.common_test_dependencies: + container_with_test_deps = container_with_test_deps.with_exec( + ["pip", "install", f'{" ".join(self.common_test_dependencies)}'], skip_entrypoint=True + ) + return ( + container_with_test_deps + # Mount the test config file + .with_mounted_file(test_config_file_name, test_config_file) + # Mount the secrets + .with_(secret_mounting_function).with_env_variable("PYTHONPATH", ".") + ) + + +class UnitTests(PytestStep): + """A step to run the connector unit tests with Pytest.""" + + title = "Unit tests" + test_directory_name = "unit_tests" + common_test_dependencies = ["pytest-cov==4.1.0"] + MINIMUM_COVERAGE_FOR_CERTIFIED_CONNECTORS = 90 + + @property + def default_params(self) -> STEP_PARAMS: + """Make sure the coverage computation is run for the unit tests. + + Returns: + dict: The default pytest options. + """ + coverage_options = {"--cov": [self.context.connector.technical_name.replace("-", "_")]} + if self.context.connector.support_level == "certified": + coverage_options["--cov-fail-under"] = [str(self.MINIMUM_COVERAGE_FOR_CERTIFIED_CONNECTORS)] + return super().default_params | coverage_options + + +class IntegrationTests(PytestStep): + """A step to run the connector integration tests with Pytest.""" + + title = "Integration tests" + test_directory_name = "integration_tests" + bind_to_docker_host = True + + +def get_test_steps(context: ConnectorContext) -> STEP_TREE: + """ + Get all the tests steps for a Python connector. + """ + return [ + [StepToRun(id=CONNECTOR_TEST_STEP_ID.BUILD, step=BuildConnectorImages(context))], + [ + StepToRun( + id=CONNECTOR_TEST_STEP_ID.UNIT, + step=UnitTests(context), + args=lambda results: {"connector_under_test": results[CONNECTOR_TEST_STEP_ID.BUILD].output_artifact[LOCAL_BUILD_PLATFORM]}, + depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + ) + ], + [ + StepToRun( + id=CONNECTOR_TEST_STEP_ID.INTEGRATION, + step=IntegrationTests(context), + args=lambda results: {"connector_under_test": results[CONNECTOR_TEST_STEP_ID.BUILD].output_artifact[LOCAL_BUILD_PLATFORM]}, + depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + ), + StepToRun( + id=CONNECTOR_TEST_STEP_ID.ACCEPTANCE, + step=AcceptanceTests(context, context.concurrent_cat), + args=lambda results: { + "connector_under_test_container": results[CONNECTOR_TEST_STEP_ID.BUILD].output_artifact[LOCAL_BUILD_PLATFORM] + }, + depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + ), + StepToRun( + id=CONNECTOR_TEST_STEP_ID.CHECK_BASE_IMAGE, step=CheckBaseImageIsUsed(context), depends_on=[CONNECTOR_TEST_STEP_ID.BUILD] + ), + ], + ] diff --git a/airbyte-ci/connectors/pipelines/pipelines/tests/templates/test_report.html.j2 b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/templates/test_report.html.j2 similarity index 100% rename from airbyte-ci/connectors/pipelines/pipelines/tests/templates/test_report.html.j2 rename to airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/templates/test_report.html.j2 diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_base_image/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_base_image/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_base_image/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_base_image/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_base_image/commands.py new file mode 100644 index 000000000000..e43d084ac074 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_base_image/commands.py @@ -0,0 +1,57 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import asyncclick as click +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.connectors.migrate_to_base_image.pipeline import run_connector_base_image_upgrade_pipeline +from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines +from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand +from pipelines.helpers.utils import fail_if_missing_docker_hub_creds + + +@click.command(cls=DaggerPipelineCommand, short_help="Upgrades the base image version used by the selected connectors.") +@click.option("--set-if-not-exists", default=True) +@click.pass_context +async def upgrade_base_image(ctx: click.Context, set_if_not_exists: bool, docker_hub_username: str, docker_hub_password: str) -> bool: + """Upgrades the base image version used by the selected connectors.""" + + fail_if_missing_docker_hub_creds(ctx) + + connectors_contexts = [ + ConnectorContext( + pipeline_name=f"Upgrade base image versions of connector {connector.technical_name}", + connector=connector, + is_local=ctx.obj["is_local"], + git_branch=ctx.obj["git_branch"], + git_revision=ctx.obj["git_revision"], + ci_report_bucket=ctx.obj["ci_report_bucket_name"], + report_output_prefix=ctx.obj["report_output_prefix"], + use_remote_secrets=ctx.obj["use_remote_secrets"], + gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), + dagger_logs_url=ctx.obj.get("dagger_logs_url"), + pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), + ci_context=ctx.obj.get("ci_context"), + ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], + ci_git_user=ctx.obj["ci_git_user"], + ci_github_access_token=ctx.obj["ci_github_access_token"], + enable_report_auto_open=False, + docker_hub_username=ctx.obj.get("docker_hub_username"), + docker_hub_password=ctx.obj.get("docker_hub_password"), + s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"), + s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"), + ) + for connector in ctx.obj["selected_connectors_with_modified_files"] + ] + + await run_connectors_pipelines( + connectors_contexts, + run_connector_base_image_upgrade_pipeline, + "Upgrade base image pipeline", + ctx.obj["concurrency"], + ctx.obj["dagger_logs_path"], + ctx.obj["execute_timeout"], + set_if_not_exists, + ) + + return True diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_base_image/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_base_image/pipeline.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_base_image/pipeline.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/commands.py new file mode 100644 index 000000000000..ad1fd2e0e80a --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/commands.py @@ -0,0 +1,67 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import asyncclick as click +import requests # type: ignore +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines +from pipelines.airbyte_ci.connectors.upgrade_cdk.pipeline import run_connector_cdk_upgrade_pipeline +from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand + + +def latest_cdk_version() -> str: + """ + Get the latest version of airbyte-cdk from pypi + """ + cdk_pypi_url = "https://pypi.org/pypi/airbyte-cdk/json" + response = requests.get(cdk_pypi_url) + response.raise_for_status() + package_info = response.json() + return package_info["info"]["version"] + + +@click.command(cls=DaggerPipelineCommand, short_help="Upgrade CDK version") +@click.argument("target-cdk-version", type=str, default=latest_cdk_version) +@click.pass_context +async def bump_version( + ctx: click.Context, + target_cdk_version: str, +) -> bool: + """Upgrade CDK version""" + + connectors_contexts = [ + ConnectorContext( + pipeline_name=f"Upgrade CDK version of connector {connector.technical_name}", + connector=connector, + is_local=ctx.obj["is_local"], + git_branch=ctx.obj["git_branch"], + git_revision=ctx.obj["git_revision"], + ci_report_bucket=ctx.obj["ci_report_bucket_name"], + report_output_prefix=ctx.obj["report_output_prefix"], + use_remote_secrets=ctx.obj["use_remote_secrets"], + gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), + dagger_logs_url=ctx.obj.get("dagger_logs_url"), + pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), + ci_context=ctx.obj.get("ci_context"), + ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], + ci_git_user=ctx.obj["ci_git_user"], + ci_github_access_token=ctx.obj["ci_github_access_token"], + enable_report_auto_open=False, + s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"), + s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"), + ) + for connector in ctx.obj["selected_connectors_with_modified_files"] + ] + + await run_connectors_pipelines( + connectors_contexts, + run_connector_cdk_upgrade_pipeline, + "Upgrade CDK version pipeline", + ctx.obj["concurrency"], + ctx.obj["dagger_logs_path"], + ctx.obj["execute_timeout"], + target_cdk_version, + ) + + return True diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/pipeline.py new file mode 100644 index 000000000000..50390ca281ed --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/pipeline.py @@ -0,0 +1,94 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from __future__ import annotations + +import os +import re +from typing import TYPE_CHECKING + +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.connectors.reports import ConnectorReport, Report +from pipelines.helpers import git +from pipelines.models.steps import Step, StepResult, StepStatus + +if TYPE_CHECKING: + from anyio import Semaphore + + +class SetCDKVersion(Step): + context: ConnectorContext + title = "Set CDK Version" + + def __init__( + self, + context: ConnectorContext, + new_version: str, + ) -> None: + super().__init__(context) + self.new_version = new_version + + async def _run(self) -> StepResult: + context = self.context + og_connector_dir = await context.get_connector_dir() + if "setup.py" not in await og_connector_dir.entries(): + return self.skip("Connector does not have a setup.py file.") + setup_py = og_connector_dir.file("setup.py") + setup_py_content = await setup_py.contents() + try: + updated_setup_py = self.update_cdk_version(setup_py_content) + updated_connector_dir = og_connector_dir.with_new_file("setup.py", updated_setup_py) + diff = og_connector_dir.diff(updated_connector_dir) + exported_successfully = await diff.export(os.path.join(git.get_git_repo_path(), context.connector.code_directory)) + if not exported_successfully: + return StepResult( + self, + StepStatus.FAILURE, + stdout="Could not export diff to local git repo.", + ) + return StepResult(self, StepStatus.SUCCESS, stdout=f"Updated CDK version to {self.new_version}", output_artifact=diff) + except ValueError as e: + return StepResult( + self, + StepStatus.FAILURE, + stderr=f"Could not set CDK version: {e}", + exc_info=e, + ) + + def update_cdk_version(self, og_setup_py_content: str) -> str: + airbyte_cdk_dependency = re.search( + r"airbyte-cdk(?P\[[a-zA-Z0-9-]*\])?(?P[<>=!~]+[0-9]*\.[0-9]*\.[0-9]*)?", og_setup_py_content + ) + # If there is no airbyte-cdk dependency, add the version + if airbyte_cdk_dependency is not None: + new_version = f"airbyte-cdk{airbyte_cdk_dependency.group('extra') or ''}>={self.new_version}" + return og_setup_py_content.replace(airbyte_cdk_dependency.group(), new_version) + else: + raise ValueError("Could not find airbyte-cdk dependency in setup.py") + + +async def run_connector_cdk_upgrade_pipeline( + context: ConnectorContext, + semaphore: Semaphore, + target_version: str, +) -> Report: + """Run a pipeline to upgrade the CDK version for a single connector. + + Args: + context (ConnectorContext): The initialized connector context. + + Returns: + Report: The reports holding the CDK version set results. + """ + async with semaphore: + steps_results = [] + async with context: + set_cdk_version = SetCDKVersion( + context, + target_version, + ) + set_cdk_version_result = await set_cdk_version.run() + steps_results.append(set_cdk_version_result) + report = ConnectorReport(context, steps_results, name="CONNECTOR VERSION CDK UPGRADE RESULTS") + context.report = report + return report diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py new file mode 100644 index 000000000000..df81810b4e27 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/actions.py @@ -0,0 +1,20 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from typing import List + +import dagger + + +async def list_files_in_directory(dagger_client: dagger.Client, directory: dagger.Directory) -> List[str]: + """ + List all files in a directory. + """ + return ( + await dagger_client.container() + .from_("bash:latest") + .with_mounted_directory("/to_list", directory) + .with_workdir("/to_list") + .with_exec(["find", ".", "-type", "f"]) + .stdout() + ).splitlines() diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py new file mode 100644 index 000000000000..a1f2d3cc613f --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/commands.py @@ -0,0 +1,133 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +""" +Module exposing the format commands. +""" +from __future__ import annotations + +import logging +import sys +from typing import Dict, List + +import asyncclick as click +from pipelines.airbyte_ci.format.configuration import FORMATTERS_CONFIGURATIONS, Formatter +from pipelines.airbyte_ci.format.format_command import FormatCommand +from pipelines.cli.click_decorators import click_ci_requirements_option, click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.helpers.cli import LogOptions, invoke_commands_concurrently, invoke_commands_sequentially, log_command_results +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context +from pipelines.models.steps import StepStatus + + +@click.group( + name="format", + help="Commands related to formatting.", +) +@click.option("--quiet", "-q", help="Hide details of the formatter execution.", default=False, is_flag=True) +@click_ci_requirements_option() +@click_merge_args_into_context_obj +@pass_pipeline_context +@click_ignore_unused_kwargs +async def format_code(pipeline_context: ClickPipelineContext) -> None: + pass + + +@format_code.group( + help="Run code format checks and fail if any checks fail.", + chain=True, +) +async def check() -> None: + pass + + +@format_code.group( + help="Run code format checks and fix any failures.", + chain=True, +) +async def fix() -> None: + pass + + +# Check and fix commands only differ in the export_formatted_code parameter value: check does not export, fix does. +FORMATTERS_CHECK_COMMANDS: Dict[Formatter, FormatCommand] = { + config.formatter: FormatCommand( + config.formatter, config.file_filter, config.get_format_container_fn, config.format_commands, export_formatted_code=False + ) + for config in FORMATTERS_CONFIGURATIONS +} + +FORMATTERS_FIX_COMMANDS: Dict[Formatter, FormatCommand] = { + config.formatter: FormatCommand( + config.formatter, config.file_filter, config.get_format_container_fn, config.format_commands, export_formatted_code=True + ) + for config in FORMATTERS_CONFIGURATIONS +} + +# Register language specific check commands +for formatter, check_command in FORMATTERS_CHECK_COMMANDS.items(): + check.add_command(check_command, name=formatter.value) + +# Register language specific fix commands +for formatter, fix_command in FORMATTERS_FIX_COMMANDS.items(): + fix.add_command(fix_command, name=formatter.value) + + +@check.command(name="all", help="Run all format checks and fail if any checks fail.") +@click.pass_context +async def all_checks(ctx: click.Context) -> None: + """ + Run all format checks and fail if any checks fail. + """ + + # We disable logging and exit on failure because its this the current command that takes care of reporting. + all_commands: List[click.Command] = [ + command.set_enable_logging(False).set_exit_on_failure(False) for command in FORMATTERS_CHECK_COMMANDS.values() + ] + command_results = await invoke_commands_concurrently(ctx, all_commands) + failure = any([r.status is StepStatus.FAILURE for r in command_results]) + logger = logging.getLogger(check.commands["all"].name) + log_options = LogOptions( + quiet=ctx.obj["quiet"], + help_message="Run `airbyte-ci format fix all` to fix the code format.", + ) + log_command_results(ctx, command_results, logger, log_options) + if failure: + sys.exit(1) + + +@fix.command(name="all", help="Fix all format failures. Exits with status 1 if any file was modified.") +@click.pass_context +async def all_fix(ctx: click.Context) -> None: + """Run code format checks and fix any failures.""" + logger = logging.getLogger(fix.commands["all"].name) + + # We have to run license command sequentially because it modifies the same set of files as other commands. + # If we ran it concurrently with language commands, we face race condition issues. + # We also want to run it before language specific formatter as they might reformat the license header. + sequential_commands: List[click.Command] = [ + FORMATTERS_FIX_COMMANDS[Formatter.LICENSE].set_enable_logging(False).set_exit_on_failure(False), + ] + command_results = await invoke_commands_sequentially(ctx, sequential_commands) + + # We can run language commands concurrently because they modify different set of files. + # We disable logging and exit on failure because its this the current command that takes care of reporting. + concurrent_commands: List[click.Command] = [ + FORMATTERS_FIX_COMMANDS[Formatter.JAVA].set_enable_logging(False).set_exit_on_failure(False), + FORMATTERS_FIX_COMMANDS[Formatter.PYTHON].set_enable_logging(False).set_exit_on_failure(False), + FORMATTERS_FIX_COMMANDS[Formatter.JS].set_enable_logging(False).set_exit_on_failure(False), + ] + + command_results += await invoke_commands_concurrently(ctx, concurrent_commands) + failure = any([r.status is StepStatus.FAILURE for r in command_results]) + + log_options = LogOptions( + quiet=ctx.obj["quiet"], + help_message="You can stage the formatted files `git add .` and commit them with `git commit -m 'chore: format code'`.", + ) + + log_command_results(ctx, command_results, logger, log_options) + if failure: + # We exit this command with status 1 because we want to make it fail when fix is modifying files. + # It allows us to run it in a Git hook and fail the commit/push if the code is not formatted. + sys.exit(1) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/configuration.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/configuration.py new file mode 100644 index 000000000000..0ee6e5441b0b --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/configuration.py @@ -0,0 +1,53 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import dataclass +from typing import Callable, List + +from pipelines.airbyte_ci.format.consts import CACHE_MOUNT_PATH, LICENSE_FILE_NAME, Formatter +from pipelines.airbyte_ci.format.containers import ( + format_java_container, + format_js_container, + format_license_container, + format_python_container, +) + + +@dataclass +class FormatConfiguration: + """A class to store the configuration of a formatter.""" + + formatter: Formatter + file_filter: List[str] + get_format_container_fn: Callable + format_commands: List[str] + + +FORMATTERS_CONFIGURATIONS: List[FormatConfiguration] = [ + # Run spotless on all java and gradle files. + FormatConfiguration( + Formatter.JAVA, ["**/*.java", "**/*.gradle"], format_java_container, ["mvn -f spotless-maven-pom.xml spotless:apply clean"] + ), + # Run prettier on all json and yaml files. + FormatConfiguration( + Formatter.JS, + ["**/*.json", "**/*.yaml", "**/*.yml"], + format_js_container, + [f"prettier --write . --list-different --cache --cache-location={CACHE_MOUNT_PATH}/.prettier_cache"], + ), + # Add license header to java and python files. The license header is stored in LICENSE_SHORT file. + FormatConfiguration( + Formatter.LICENSE, + ["**/*.java", "**/*.py"], + format_license_container, + [f"addlicense -c 'Airbyte, Inc.' -l apache -v -f {LICENSE_FILE_NAME} ."], + ), + # Run isort and black on all python files. + FormatConfiguration( + Formatter.PYTHON, + ["**/*.py"], + format_python_container, + ["poetry run isort --settings-file pyproject.toml .", "poetry run black --config pyproject.toml ."], + ), +] diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py new file mode 100644 index 000000000000..37d321f0f398 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/consts.py @@ -0,0 +1,67 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from enum import Enum + +REPO_MOUNT_PATH = "/src" +CACHE_MOUNT_PATH = "/cache" + +LICENSE_FILE_NAME = "LICENSE_SHORT" + +# TODO create .airbyte_ci_ignore files? +DEFAULT_FORMAT_IGNORE_LIST = [ + "**/__init__.py", # These files has never been formatted and we don't want to start now (for now) see https://github.com/airbytehq/airbyte/issues/33296 + "**/__pycache__", + "**/.eggs", + "**/.git", + "**/.gradle", + "**/.mypy_cache", + "**/.pytest_cache", + "**/.tox", + "**/.venv", + "**/*.egg-info", + "**/build", + "**/charts", # Helm charts often have injected template strings that will fail general linting. Helm linting is done separately. + "**/dbt_test_config", + "**/dbt-project-template-clickhouse", + "**/dbt-project-template-duckdb", + "**/dbt-project-template-mssql", + "**/dbt-project-template-mysql", + "**/dbt-project-template-oracle", + "**/dbt-project-template-snowflake", + "**/dbt-project-template-tidb", + "**/dbt-project-template", + "**/node_modules", + "**/pnpm-lock.yaml", # This file is generated and should not be formatted + "**/normalization_test_output", + "**/source-amplitude/unit_tests/api_data/zipped.json", # Zipped file presents as non-UTF-8 making spotless sad + "**/tools/git_hooks/tests/test_spec_linter.py", + "airbyte-cdk/python/airbyte_cdk/sources/declarative/models/**", # These files are generated and should not be formatted + "airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/**", # These files are generated and should not be formatted + "**/airbyte-ci/connectors/metadata_service/lib/tests/fixtures/**/invalid", # This is a test directory with invalid and sometimes unformatted code + "airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code", # This is a test directory with badly formatted code +] + + +class Formatter(Enum): + """An enum for the formatter values which can be ["java", "js", "python", "license"].""" + + JAVA = "java" + JS = "js" + PYTHON = "python" + LICENSE = "license" + + +# This files are dependencies to be mounted in formatter containers. +# They are used as configuration files for the formatter. +# We mount them to formatter containers because they can be required to install dependencies. +# We use them to "warmup" containers because they are not likely to change often. +# The mount will be cached and we won't re-install dependencies unless these files change. +WARM_UP_INCLUSIONS = { + Formatter.JAVA: [ + "spotless-maven-pom.xml", + "tools/gradle/codestyle/java-google-style.xml", + ], + Formatter.PYTHON: ["pyproject.toml", "poetry.lock"], + Formatter.LICENSE: [LICENSE_FILE_NAME], +} diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py new file mode 100644 index 000000000000..9528bc0f6eb9 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/containers.py @@ -0,0 +1,148 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Dict, List, Optional + +import dagger +from pipelines.airbyte_ci.format.consts import CACHE_MOUNT_PATH, DEFAULT_FORMAT_IGNORE_LIST, REPO_MOUNT_PATH, WARM_UP_INCLUSIONS, Formatter +from pipelines.consts import GO_IMAGE, MAVEN_IMAGE, NODE_IMAGE, PYTHON_3_10_IMAGE +from pipelines.helpers import cache_keys +from pipelines.helpers.utils import sh_dash_c + + +def build_container( + dagger_client: dagger.Client, + base_image: str, + dir_to_format: dagger.Directory, + warmup_dir: Optional[dagger.Directory] = None, + install_commands: Optional[List[str]] = None, + env_vars: Dict[str, Any] = {}, + cache_volume: Optional[dagger.CacheVolume] = None, +) -> dagger.Container: + """Build a container for formatting code. + Args: + ctx (ClickPipelineContext): The context of the pipeline + base_image (str): The base image to use for the container + dir_to_format (Directory): A directory with the source code to format + warmup_dir (Optional[Directory], optional): A directory with the source code to warm up the container cache. Defaults to None. + install_commands (Optional[List[str]], optional): A list of commands to run to install dependencies. Defaults to None. + env_vars (Optional[Dict[str, Any]], optional): A dictionary of environment variables to set in the container. Defaults to {}. + cache_volume (Optional[CacheVolume], optional): A cache volume to mount in the container. Defaults to None. + + Returns: + dagger.Container: The container to use for formatting + """ + # Create container from base image + container = dagger_client.container().from_(base_image) + + # Add any environment variables + for key, value in env_vars.items(): + container = container.with_env_variable(key, value) + + # Set the working directory to the code to format + container = container.with_workdir(REPO_MOUNT_PATH) + + # Mount files to be referenced by the install_commands, if requested. + # These should only be files which do not change very often, to avoid invalidating the layer cache. + if warmup_dir: + container = container.with_mounted_directory( + REPO_MOUNT_PATH, + warmup_dir, + ) + + # Install any dependencies of the formatter + if install_commands: + container = container.with_exec(sh_dash_c(install_commands), skip_entrypoint=True) + + # Mount the relevant parts of the repository: the code to format and the formatting config + # Exclude the default ignore list to keep things as small as possible. + # The mount path is the same as for the earlier volume mount, this will cause those directory + # contents to be overwritten. This is intentional and not a concern as the current file set is + # a superset of the earlier one. + if warmup_dir: + container = container.with_mounted_directory(REPO_MOUNT_PATH, dir_to_format.with_directory(".", warmup_dir)) + else: + container = container.with_mounted_directory(REPO_MOUNT_PATH, dir_to_format) + if cache_volume: + container = container.with_mounted_cache(CACHE_MOUNT_PATH, cache_volume) + return container + + +def format_java_container(dagger_client: dagger.Client, java_code: dagger.Directory) -> dagger.Container: + """ + Create a Maven container with spotless installed with mounted code to format and a cache volume. + We warm up the container cache with the spotless configuration and dependencies. + """ + warmup_dir = dagger_client.host().directory( + ".", + include=WARM_UP_INCLUSIONS[Formatter.JAVA], + exclude=DEFAULT_FORMAT_IGNORE_LIST, + ) + return build_container( + dagger_client, + base_image=MAVEN_IMAGE, + warmup_dir=warmup_dir, + install_commands=[ + # Run maven before mounting the sources to download all its dependencies. + # Dagger will cache the resulting layer to minimize the downloads. + # The go-offline goal purportedly downloads all dependencies. + # This isn't quite the case, we still need to add spotless goals. + "mvn -f spotless-maven-pom.xml" + " org.apache.maven.plugins:maven-dependency-plugin:3.6.1:go-offline" + " spotless:apply" + " spotless:check" + " clean" + ], + dir_to_format=java_code, + ) + + +def format_js_container(dagger_client: dagger.Client, js_code: dagger.Directory, prettier_version: str = "3.0.3") -> dagger.Container: + """Create a Node container with prettier installed with mounted code to format and a cache volume.""" + return build_container( + dagger_client, + base_image=NODE_IMAGE, + dir_to_format=js_code, + install_commands=[f"npm install -g npm@10.1.0 prettier@{prettier_version}"], + cache_volume=dagger_client.cache_volume(cache_keys.get_prettier_cache_key(prettier_version)), + ) + + +def format_license_container(dagger_client: dagger.Client, license_code: dagger.Directory) -> dagger.Container: + """Create a Go container with addlicense installed with mounted code to format.""" + warmup_dir = dagger_client.host().directory(".", include=WARM_UP_INCLUSIONS[Formatter.LICENSE], exclude=DEFAULT_FORMAT_IGNORE_LIST) + return build_container( + dagger_client, + base_image=GO_IMAGE, + dir_to_format=license_code, + install_commands=["go get -u github.com/google/addlicense"], + warmup_dir=warmup_dir, + ) + + +def format_python_container( + dagger_client: dagger.Client, python_code: dagger.Directory, black_version: str = "~22.3.0" +) -> dagger.Container: + """Create a Python container with pipx and the global pyproject.toml installed with mounted code to format and a cache volume. + We warm up the container with the pyproject.toml and poetry.lock files to not repeat the pyproject.toml installation. + """ + + warmup_dir = dagger_client.host().directory(".", include=WARM_UP_INCLUSIONS[Formatter.PYTHON], exclude=DEFAULT_FORMAT_IGNORE_LIST) + return build_container( + dagger_client, + base_image=PYTHON_3_10_IMAGE, + env_vars={"PIPX_BIN_DIR": "/usr/local/bin", "BLACK_CACHE_DIR": f"{CACHE_MOUNT_PATH}/black"}, + install_commands=[ + "pip install pipx", + "pipx ensurepath", + "pipx install poetry", + "poetry install --no-root", + ], + dir_to_format=python_code, + warmup_dir=warmup_dir, + # Namespacing the cache volume by black version is likely overkill: + # Black already manages cache directories by version internally. + # https://github.com/psf/black/blob/e4ae213f06050e7f76ebcf01578c002e412dafdc/src/black/cache.py#L42 + cache_volume=dagger_client.cache_volume(cache_keys.get_black_cache_key(black_version)), + ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/format_command.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/format_command.py new file mode 100644 index 000000000000..07b4c72381a9 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/format/format_command.py @@ -0,0 +1,209 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from __future__ import annotations + +import logging +import sys +from typing import Callable, List, Tuple + +import asyncclick as click +import dagger +from pipelines import main_logger +from pipelines.airbyte_ci.format.actions import list_files_in_directory +from pipelines.airbyte_ci.format.configuration import Formatter +from pipelines.airbyte_ci.format.consts import DEFAULT_FORMAT_IGNORE_LIST, REPO_MOUNT_PATH, WARM_UP_INCLUSIONS +from pipelines.consts import GIT_IMAGE +from pipelines.helpers import sentry_utils +from pipelines.helpers.cli import LogOptions, log_command_results +from pipelines.helpers.utils import sh_dash_c +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context +from pipelines.models.steps import CommandResult, StepStatus + + +class FormatCommand(click.Command): + """Generic command to run a formatter.""" + + # This constant is useful for mocking in tests + LOCAL_REPO_PATH = "." + + def __init__( + self, + formatter: Formatter, + file_filter: List[str], + get_format_container_fn: Callable, + format_commands: List[str], + export_formatted_code: bool, + enable_logging: bool = True, + exit_on_failure: bool = True, + ) -> None: + """Initialize a FormatCommand. + + Args: + formatter (Formatter): The formatter to run + file_filter (List[str]): The list of files to include in the formatter + get_format_container_fn (Callable): A function to get the container to run the formatter in + format_commands (List[str]): The list of commands to run in the container to format the repository + export_formatted_code (bool): Whether to export the formatted code back to the host + enable_logging (bool, optional): Make the command log its output. Defaults to True. + exit_on_failure (bool, optional): Exit the process with status code 1 if the command fails. Defaults to True. + """ + super().__init__(formatter.value) + self.formatter = formatter + self.file_filter = file_filter + self.get_format_container_fn = get_format_container_fn + self.format_commands = format_commands + self.export_formatted_code = export_formatted_code + self.help = self.get_help_message() + self._enable_logging = enable_logging + self._exit_on_failure = exit_on_failure + self.logger = logging.getLogger(self.name) + + def get_help_message(self) -> str: + """Get the help message for the command. + + Returns: + str: The help message + """ + message = f"Run {self.formatter.value} formatter" + if self.export_formatted_code: + message = f"{message}, will fix any failures." + else: + message = f"{message}." + return message + + def get_dir_to_format(self, dagger_client: dagger.Client) -> dagger.Directory: + """Get a directory with all the source code to format according to the file_filter. + We mount the files to format in a git container and remove all gitignored files. + It ensures we're not formatting files that are gitignored. + + Args: + dagger_client (dagger.Client): The dagger client to use to get the directory. + + Returns: + Directory: The directory with the files to format that are not gitignored. + """ + # Load a directory from the host with all the files to format according to the file_filter and the .gitignore files + dir_to_format = dagger_client.host().directory( + self.LOCAL_REPO_PATH, include=self.file_filter + ["**/.gitignore"], exclude=DEFAULT_FORMAT_IGNORE_LIST + ) + + return ( + dagger_client.container() + .from_(GIT_IMAGE) + .with_workdir(REPO_MOUNT_PATH) + .with_mounted_directory(REPO_MOUNT_PATH, dir_to_format) + # All with_exec commands below will re-run if the to_format directory changes + .with_exec(["init"]) + # Remove all gitignored files + .with_exec(["clean", "-dfqX"]) + # Delete all .gitignore files + .with_exec(sh_dash_c(['find . -type f -name ".gitignore" -exec rm {} \;']), skip_entrypoint=True) + # Delete .git + .with_exec(["rm", "-rf", ".git"], skip_entrypoint=True) + .directory(REPO_MOUNT_PATH) + .with_timestamps(0) + ) + + @pass_pipeline_context + @sentry_utils.with_command_context + async def invoke(self, ctx: click.Context, click_pipeline_context: ClickPipelineContext) -> CommandResult: + """Run the command. If _exit_on_failure is True, exit the process with status code 1 if the command fails. + + Args: + ctx (click.Context): The click context + click_pipeline_context (ClickPipelineContext): The pipeline context + + Returns: + Any: The result of running the command + """ + + dagger_client = await click_pipeline_context.get_dagger_client(pipeline_name=f"Format {self.formatter.value}") + dir_to_format = self.get_dir_to_format(dagger_client) + + container = self.get_format_container_fn(dagger_client, dir_to_format) + command_result = await self.get_format_command_result(dagger_client, container, dir_to_format) + + if (formatted_code_dir := command_result.output_artifact) and self.export_formatted_code: + await formatted_code_dir.export(self.LOCAL_REPO_PATH) + + if self._enable_logging: + log_command_results(ctx, [command_result], main_logger, LogOptions(quiet=ctx.obj["quiet"])) + + if command_result.status is StepStatus.FAILURE and self._exit_on_failure: + sys.exit(1) + + self.logger.info(f"Finished running formatter - {command_result.status}") + return command_result + + def set_enable_logging(self, value: bool) -> FormatCommand: + """Set _enable_logging to the given value. + Args: + value (bool): The value to set + Returns: + FormatCommand: The command with logging disabled + """ + self._enable_logging = False + return self + + def set_exit_on_failure(self, value: bool) -> FormatCommand: + """Set _exit_on_failure to the given value. + + Args: + value (bool): The value to set + Returns: + FormatCommand: The command with _exit_on_failure disabled + """ + self._exit_on_failure = value + return self + + async def run_format( + self, dagger_client: dagger.Client, container: dagger.Container, format_commands: List[str], not_formatted_code: dagger.Directory + ) -> Tuple[dagger.Directory, str, str]: + """Run the format commands in the container. Return the directory with the modified files, stdout and stderr. + + Args: + dagger_client (dagger.Client): The dagger client to use + container (dagger.Container): The container to run the format_commands in + format_commands (List[str]): The list of commands to run to format the repository + not_formatted_code (dagger.Directory): The directory with the code to format + """ + format_container = container.with_exec(sh_dash_c(format_commands), skip_entrypoint=True) + formatted_directory = format_container.directory(REPO_MOUNT_PATH) + if warmup_inclusion := WARM_UP_INCLUSIONS.get(self.formatter): + warmup_dir = dagger_client.host().directory(".", include=warmup_inclusion, exclude=DEFAULT_FORMAT_IGNORE_LIST) + not_formatted_code = not_formatted_code.with_directory(".", warmup_dir) + formatted_directory = formatted_directory.with_directory(".", warmup_dir) + return ( + await not_formatted_code.with_timestamps(0).diff(formatted_directory.with_timestamps(0)), + await format_container.stdout(), + await format_container.stderr(), + ) + + async def get_format_command_result( + self, + dagger_client: dagger.Client, + container: dagger.Container, + not_formatted_code: dagger.Directory, + ) -> CommandResult: + """Run a format command and return the CommandResult. + A command is considered successful if the export operation of run_format is successful and no files were modified. + + Args: + click_command (click.Command): The click command to run + container (dagger.Container): The container to run the format_commands in + not_formatted_code (dagger.Directory): The directory with the code to format + Returns: + CommandResult: The result of running the command + """ + try: + dir_with_modified_files, stdout, stderr = await self.run_format( + dagger_client, container, self.format_commands, not_formatted_code + ) + if await dir_with_modified_files.entries(): + modified_files = await list_files_in_directory(dagger_client, dir_with_modified_files) + self.logger.debug(f"Modified files: {modified_files}") + return CommandResult(self, status=StepStatus.FAILURE, stdout=stdout, stderr=stderr, output_artifact=dir_with_modified_files) + return CommandResult(self, status=StepStatus.SUCCESS, stdout=stdout, stderr=stderr) + except dagger.ExecError as e: + return CommandResult(self, status=StepStatus.FAILURE, stderr=e.stderr, stdout=e.stdout, exc_info=e) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/commands.py new file mode 100644 index 000000000000..ca856d9bbb67 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/commands.py @@ -0,0 +1,40 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import asyncclick as click +from pipelines.cli.click_decorators import click_ci_requirements_option +from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand + +# MAIN GROUP + + +@click.group(help="Commands related to the metadata service.") +@click_ci_requirements_option() +@click.pass_context +def metadata(ctx: click.Context) -> None: + pass + + +@metadata.group(help="Commands related to deploying components of the metadata service.") +@click.pass_context +def deploy(ctx: click.Context) -> None: + pass + + +@deploy.command(cls=DaggerPipelineCommand, name="orchestrator", help="Deploy the metadata service orchestrator to production") +@click.pass_context +async def deploy_orchestrator(ctx: click.Context) -> None: + # Import locally to speed up CLI. + from pipelines.airbyte_ci.metadata.pipeline import run_metadata_orchestrator_deploy_pipeline + + await run_metadata_orchestrator_deploy_pipeline( + ctx.obj["is_local"], + ctx.obj["git_branch"], + ctx.obj["git_revision"], + ctx.obj["report_output_prefix"], + ctx.obj.get("gha_workflow_run_url"), + ctx.obj.get("dagger_logs_url"), + ctx.obj.get("pipeline_start_timestamp"), + ctx.obj.get("ci_context"), + ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/pipeline.py new file mode 100644 index 000000000000..2bd32b1fcaae --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/metadata/pipeline.py @@ -0,0 +1,205 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import uuid +from typing import Optional + +import dagger +from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID +from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext +from pipelines.airbyte_ci.steps.docker import SimpleDockerStep +from pipelines.airbyte_ci.steps.poetry import PoetryRunStep +from pipelines.consts import DOCS_DIRECTORY_ROOT_PATH, INTERNAL_TOOL_PATHS +from pipelines.dagger.actions.python.common import with_pip_packages +from pipelines.dagger.containers.python import with_python_base +from pipelines.helpers.execution.run_steps import STEP_TREE, StepToRun, run_steps +from pipelines.helpers.utils import DAGGER_CONFIG, get_secret_host_variable +from pipelines.models.reports import Report +from pipelines.models.steps import MountPath, Step, StepResult + +# STEPS + + +class MetadataValidation(SimpleDockerStep): + def __init__(self, context: ConnectorContext) -> None: + super().__init__( + title=f"Validate metadata for {context.connector.technical_name}", + context=context, + paths_to_mount=[ + MountPath(context.connector.metadata_file_path), + MountPath(DOCS_DIRECTORY_ROOT_PATH), + MountPath(context.connector.icon_path, optional=True), + ], + internal_tools=[ + MountPath(INTERNAL_TOOL_PATHS.METADATA_SERVICE.value), + ], + secrets={ + k: v + for k, v in { + "DOCKER_HUB_USERNAME": context.docker_hub_username_secret, + "DOCKER_HUB_PASSWORD": context.docker_hub_password_secret, + }.items() + if v + }, + command=[ + "metadata_service", + "validate", + str(context.connector.metadata_file_path), + DOCS_DIRECTORY_ROOT_PATH, + ], + ) + + +class MetadataUpload(SimpleDockerStep): + # When the metadata service exits with this code, it means the metadata is valid but the upload was skipped because the metadata is already uploaded + skipped_exit_code = 5 + + def __init__( + self, + context: ConnectorContext, + metadata_bucket_name: str, + metadata_service_gcs_credentials_secret: dagger.Secret, + docker_hub_username_secret: dagger.Secret, + docker_hub_password_secret: dagger.Secret, + pre_release: bool = False, + pre_release_tag: Optional[str] = None, + ) -> None: + title = f"Upload metadata for {context.connector.technical_name} v{context.connector.version}" + command_to_run = [ + "metadata_service", + "upload", + str(context.connector.metadata_file_path), + DOCS_DIRECTORY_ROOT_PATH, + metadata_bucket_name, + ] + + if pre_release and pre_release_tag: + command_to_run += ["--prerelease", pre_release_tag] + + super().__init__( + title=title, + context=context, + paths_to_mount=[ + MountPath(context.connector.metadata_file_path), + MountPath(DOCS_DIRECTORY_ROOT_PATH), + MountPath(context.connector.icon_path, optional=True), + ], + internal_tools=[ + MountPath(INTERNAL_TOOL_PATHS.METADATA_SERVICE.value), + ], + secrets={ + "DOCKER_HUB_USERNAME": docker_hub_username_secret, + "DOCKER_HUB_PASSWORD": docker_hub_password_secret, + "GCS_CREDENTIALS": metadata_service_gcs_credentials_secret, + }, + env_variables={ + # The cache buster ensures we always run the upload command (in case of remote bucket change) + "CACHEBUSTER": str(uuid.uuid4()), + }, + command=command_to_run, + ) + + +class DeployOrchestrator(Step): + title = "Deploy Metadata Orchestrator to Dagster Cloud" + deploy_dagster_command = [ + "dagster-cloud", + "serverless", + "deploy-python-executable", + "--location-name", + "metadata_service_orchestrator", + "--location-file", + "dagster_cloud.yaml", + "--organization", + "airbyte-connectors", + "--deployment", + "prod", + "--python-version", + "3.9", + ] + + async def _run(self) -> StepResult: + # mount metadata_service/lib and metadata_service/orchestrator + parent_dir = self.context.get_repo_dir("airbyte-ci/connectors/metadata_service") + python_base = with_python_base(self.context, "3.9") + python_with_dependencies = with_pip_packages(python_base, ["dagster-cloud==1.5.14", "poetry2setup==1.1.0"]) + dagster_cloud_api_token_secret: dagger.Secret = get_secret_host_variable( + self.context.dagger_client, "DAGSTER_CLOUD_METADATA_API_TOKEN" + ) + + container_to_run = ( + python_with_dependencies.with_mounted_directory("/src", parent_dir) + .with_secret_variable("DAGSTER_CLOUD_API_TOKEN", dagster_cloud_api_token_secret) + .with_workdir("/src/orchestrator") + .with_exec(["/bin/sh", "-c", "poetry2setup >> setup.py"]) + .with_exec(self.deploy_dagster_command) + ) + return await self.get_step_result(container_to_run) + + +class TestOrchestrator(PoetryRunStep): + def __init__(self, context: PipelineContext) -> None: + super().__init__( + context=context, + title="Test Metadata Orchestrator", + parent_dir_path="airbyte-ci/connectors/metadata_service", + module_path="orchestrator", + poetry_run_args=["pytest"], + ) + + +# PIPELINES + + +async def run_metadata_orchestrator_deploy_pipeline( + is_local: bool, + git_branch: str, + git_revision: str, + report_output_prefix: str, + gha_workflow_run_url: Optional[str], + dagger_logs_url: Optional[str], + pipeline_start_timestamp: Optional[int], + ci_context: Optional[str], +) -> bool: + success: bool = False + + metadata_pipeline_context = PipelineContext( + pipeline_name="Metadata Service Orchestrator Unit Test Pipeline", + is_local=is_local, + git_branch=git_branch, + git_revision=git_revision, + report_output_prefix=report_output_prefix, + gha_workflow_run_url=gha_workflow_run_url, + dagger_logs_url=dagger_logs_url, + pipeline_start_timestamp=pipeline_start_timestamp, + ci_context=ci_context, + ) + async with dagger.Connection(DAGGER_CONFIG) as dagger_client: + metadata_pipeline_context.dagger_client = dagger_client.pipeline(metadata_pipeline_context.pipeline_name) + + async with metadata_pipeline_context: + steps: STEP_TREE = [ + [ + StepToRun( + id=CONNECTOR_TEST_STEP_ID.TEST_ORCHESTRATOR, + step=TestOrchestrator(context=metadata_pipeline_context), + ) + ], + [ + StepToRun( + id=CONNECTOR_TEST_STEP_ID.DEPLOY_ORCHESTRATOR, + step=DeployOrchestrator(context=metadata_pipeline_context), + depends_on=[CONNECTOR_TEST_STEP_ID.TEST_ORCHESTRATOR], + ) + ], + ] + steps_results = await run_steps(steps) + report = Report( + pipeline_context=metadata_pipeline_context, + steps_results=list(steps_results.values()), + name="METADATA ORCHESTRATOR DEPLOY RESULTS", + ) + metadata_pipeline_context.report = report + success = report.success + return success diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py new file mode 100644 index 000000000000..71c692c37fae --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py @@ -0,0 +1,102 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import List, Optional + +import dagger +from pipelines.dagger.actions.python.pipx import with_installed_pipx_package +from pipelines.dagger.containers.python import with_python_base +from pipelines.models.contexts.pipeline_context import PipelineContext +from pipelines.models.steps import MountPath, Step, StepResult + + +class SimpleDockerStep(Step): + def __init__( + self, + title: str, + context: PipelineContext, + paths_to_mount: List[MountPath] = [], + internal_tools: List[MountPath] = [], + secrets: dict[str, dagger.Secret] = {}, + env_variables: dict[str, str] = {}, + working_directory: str = "/", + command: Optional[List[str]] = None, + ) -> None: + """A simple step that runs a given command in a container. + + Args: + title (str): name of the step + context (PipelineContext): context of the step + paths_to_mount (List[MountPath], optional): directory paths to mount. Defaults to []. + internal_tools (List[MountPath], optional): internal tools to install. Defaults to []. + secrets (dict[str, dagger.Secret], optional): secrets to add to container. Defaults to {}. + env_variables (dict[str, str], optional): env variables to set in container. Defaults to {}. + working_directory (str, optional): working directory to run the command in. Defaults to "/". + command (Optional[List[str]], optional): The default command to run. Defaults to None. + """ + self._title = title + super().__init__(context) + + self.paths_to_mount = paths_to_mount + self.working_directory = working_directory + self.internal_tools = internal_tools + self.secrets = secrets + self.env_variables = env_variables + self.command = command + + @property + def title(self) -> str: + return self._title + + def _mount_paths(self, container: dagger.Container) -> dagger.Container: + for path_to_mount in self.paths_to_mount: + if path_to_mount.optional and not path_to_mount.get_path().exists(): + continue + + path_string = str(path_to_mount) + destination_path = f"/{path_string}" + if path_to_mount.is_file: + file_to_load = self.context.get_repo_file(path_string) + container = container.with_mounted_file(destination_path, file_to_load) + else: + container = container.with_mounted_directory(destination_path, self.context.get_repo_dir(path_string)) + return container + + async def _install_internal_tools(self, container: dagger.Container) -> dagger.Container: + for internal_tool in self.internal_tools: + container = await with_installed_pipx_package(self.context, container, str(internal_tool)) + return container + + def _set_workdir(self, container: dagger.Container) -> dagger.Container: + return container.with_workdir(self.working_directory) + + def _set_env_variables(self, container: dagger.Container) -> dagger.Container: + for key, value in self.env_variables.items(): + container = container.with_env_variable(key, value) + return container + + def _set_secrets(self, container: dagger.Container) -> dagger.Container: + for key, value in self.secrets.items(): + container = container.with_secret_variable(key, value) + return container + + async def init_container(self) -> dagger.Container: + # TODO (ben): Replace with python base container when available + container = with_python_base(self.context) + + container = self._mount_paths(container) + container = self._set_env_variables(container) + container = self._set_secrets(container) + container = await self._install_internal_tools(container) + container = self._set_workdir(container) + + return container + + async def _run(self, command: Optional[List[str]] = None) -> StepResult: + command_to_run = command or self.command + if not command_to_run: + raise ValueError(f"No command given to the {self.title} step") + + container_to_run = await self.init_container() + return await self.get_step_result(container_to_run.with_exec(command_to_run)) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py new file mode 100644 index 000000000000..08b110dabc0a --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/gradle.py @@ -0,0 +1,203 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import ABC +from typing import Any, ClassVar, List, Optional, Tuple + +import pipelines.dagger.actions.system.docker +from dagger import CacheSharingMode, CacheVolume +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.consts import AMAZONCORRETTO_IMAGE +from pipelines.dagger.actions import secrets +from pipelines.helpers.utils import sh_dash_c +from pipelines.models.steps import Step, StepResult + + +class GradleTask(Step, ABC): + """ + A step to run a Gradle task. + + Attributes: + title (str): The step title. + gradle_task_name (str): The Gradle task name to run. + bind_to_docker_host (bool): Whether to install the docker client and bind it to the host. + mount_connector_secrets (bool): Whether to mount connector secrets. + """ + + context: ConnectorContext + + LOCAL_MAVEN_REPOSITORY_PATH = "/root/.m2" + GRADLE_DEP_CACHE_PATH = "/root/gradle-cache" + GRADLE_HOME_PATH = "/root/.gradle" + STATIC_GRADLE_OPTIONS = ("--no-daemon", "--no-watch-fs", "--build-cache", "--scan", "--console=plain") + gradle_task_name: ClassVar[str] + bind_to_docker_host: ClassVar[bool] = False + mount_connector_secrets: ClassVar[bool] = False + accept_extra_params = True + + @property + def gradle_task_options(self) -> Tuple[str, ...]: + return self.STATIC_GRADLE_OPTIONS + (f"-Ds3BuildCachePrefix={self.context.connector.technical_name}",) + + @property + def dependency_cache_volume(self) -> CacheVolume: + """This cache volume is for sharing gradle dependencies (jars and poms) across all pipeline runs.""" + return self.context.dagger_client.cache_volume("gradle-dependency-cache") + + @property + def build_include(self) -> List[str]: + """Retrieve the list of source code directory required to run a Java connector Gradle task. + + The list is different according to the connector type. + + Returns: + List[str]: List of directories or files to be mounted to the container to run a Java connector Gradle task. + """ + return [ + str(dependency_directory) + for dependency_directory in self.context.connector.get_local_dependency_paths(with_test_dependencies=True) + ] + + def _get_gradle_command(self, task: str, *args: Any, task_options: Optional[List[str]] = None) -> str: + task_options = task_options or [] + return f"./gradlew {' '.join(self.gradle_task_options + args)} {task} {' '.join(task_options)}" + + async def _run(self, *args: Any, **kwargs: Any) -> StepResult: + include = [ + ".root", + ".env", + "build.gradle", + "deps.toml", + "gradle.properties", + "gradle", + "gradlew", + "settings.gradle", + "build.gradle", + "tools/gradle", + "spotbugs-exclude-filter-file.xml", + "buildSrc", + "tools/bin/build_image.sh", + "tools/lib/lib.sh", + "pyproject.toml", + ] + self.build_include + + yum_packages_to_install = [ + "docker", # required by :integrationTestJava. + "findutils", # gradle requires xargs, which is shipped in findutils. + "jq", # required by :acceptance-test-harness to inspect docker images. + "rsync", # required for gradle cache synchronization. + ] + + # Common base container. + gradle_container_base = ( + self.dagger_client.container() + # Use a linux+jdk base image with long-term support, such as amazoncorretto. + .from_(AMAZONCORRETTO_IMAGE) + # Mount the dependency cache volume, but not to $GRADLE_HOME, because gradle doesn't expect concurrent modifications. + .with_mounted_cache(self.GRADLE_DEP_CACHE_PATH, self.dependency_cache_volume, sharing=CacheSharingMode.LOCKED) + # Set GRADLE_HOME to the directory which will be rsync-ed with the gradle cache volume. + .with_env_variable("GRADLE_HOME", self.GRADLE_HOME_PATH) + # Same for GRADLE_USER_HOME. + .with_env_variable("GRADLE_USER_HOME", self.GRADLE_HOME_PATH) + # Install a bunch of packages as early as possible. + .with_exec( + sh_dash_c( + [ + # Update first, but in the same .with_exec step as the package installation. + # Otherwise, we risk caching stale package URLs. + "yum update -y", + f"yum install -y {' '.join(yum_packages_to_install)}", + # Remove any dangly bits. + "yum clean all", + # Deliberately soft-remove docker, so that the `docker` CLI is unavailable by default. + # This is a defensive choice to enforce the expectation that, as a general rule, gradle tasks do not rely on docker. + "yum remove -y --noautoremove docker", # remove docker package but not its dependencies + "yum install -y --downloadonly docker", # have docker package in place for quick install + ] + ) + ) + # Set RUN_IN_AIRBYTE_CI to tell gradle how to configure its build cache. + # This is consumed by settings.gradle in the repo root. + .with_env_variable("RUN_IN_AIRBYTE_CI", "1") + # Disable the Ryuk container because it needs privileged docker access which it can't have. + .with_env_variable("TESTCONTAINERS_RYUK_DISABLED", "true") + # Set the current working directory. + .with_workdir("/airbyte") + ) + + # Augment the base container with S3 build cache secrets when available. + if self.context.s3_build_cache_access_key_id_secret: + gradle_container_base = gradle_container_base.with_secret_variable( + "S3_BUILD_CACHE_ACCESS_KEY_ID", self.context.s3_build_cache_access_key_id_secret + ) + if self.context.s3_build_cache_secret_key_secret: + gradle_container_base = gradle_container_base.with_secret_variable( + "S3_BUILD_CACHE_SECRET_KEY", self.context.s3_build_cache_secret_key_secret + ) + + # Running a gradle task like "help" with these arguments will trigger updating all dependencies. + # When the cache is cold, this downloads many gigabytes of jars and poms from all over the internet. + warm_dependency_cache_args = ["--write-verification-metadata", "sha256", "--dry-run"] + if self.context.is_local: + # When running locally, this dependency update is slower and less useful than within a CI runner. Skip it. + warm_dependency_cache_args = ["--dry-run"] + + # Mount the whole git repo to update the cache volume contents and build the CDK. + with_whole_git_repo = ( + gradle_container_base + # Mount the whole repo. + .with_directory("/airbyte", self.context.get_repo_dir(".")) + # Update the cache in place by executing a gradle task which will update all dependencies and build the CDK. + .with_exec( + sh_dash_c( + [ + # Ensure that the .m2 directory exists. + f"mkdir -p {self.LOCAL_MAVEN_REPOSITORY_PATH}", + # Load from the cache volume. + f"(rsync -a --stats --mkpath {self.GRADLE_DEP_CACHE_PATH}/ {self.GRADLE_HOME_PATH} || true)", + # Resolve all dependencies and write their checksums to './gradle/verification-metadata.dryrun.xml'. + self._get_gradle_command("help", *warm_dependency_cache_args), + # Build the CDK and publish it to the local maven repository. + self._get_gradle_command(":airbyte-cdk:java:airbyte-cdk:publishSnapshotIfNeeded"), + # Store to the cache volume. + f"(rsync -a --stats {self.GRADLE_HOME_PATH}/ {self.GRADLE_DEP_CACHE_PATH} || true)", + ] + ) + ) + ) + + # Mount only the code needed to build the connector. + gradle_container = ( + gradle_container_base + # Copy the local maven repository and force evaluation of `with_whole_git_repo` container. + .with_directory(self.LOCAL_MAVEN_REPOSITORY_PATH, await with_whole_git_repo.directory(self.LOCAL_MAVEN_REPOSITORY_PATH)) + # Mount the connector-agnostic whitelisted files in the git repo. + .with_mounted_directory("/airbyte", self.context.get_repo_dir(".", include=include)) + # Mount the sources for the connector and its dependencies in the git repo. + .with_mounted_directory(str(self.context.connector.code_directory), await self.context.get_connector_dir()) + ) + + # From this point on, we add layers which are task-dependent. + if self.mount_connector_secrets: + secrets_dir = f"{self.context.connector.code_directory}/secrets" + gradle_container = gradle_container.with_(await secrets.mounted_connector_secrets(self.context, secrets_dir)) + if self.bind_to_docker_host: + # If this GradleTask subclass needs docker, then install it and bind it to the existing global docker host container. + gradle_container = pipelines.dagger.actions.system.docker.with_bound_docker_host(self.context, gradle_container) + # This installation should be cheap, as the package has already been downloaded, and its dependencies are already installed. + gradle_container = gradle_container.with_exec(["yum", "install", "-y", "docker"]) + + # Run the gradle task that we actually care about. + connector_task = f":airbyte-integrations:connectors:{self.context.connector.technical_name}:{self.gradle_task_name}" + gradle_container = gradle_container.with_exec( + sh_dash_c( + [ + # Warm the gradle cache. + f"(rsync -a --stats --mkpath {self.GRADLE_DEP_CACHE_PATH}/ {self.GRADLE_HOME_PATH} || true)", + # Run the gradle task. + self._get_gradle_command(connector_task, task_options=self.params_as_cli_options), + ] + ) + ) + return await self.get_step_result(gradle_container) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/no_op.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/no_op.py new file mode 100644 index 000000000000..2d1629e05672 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/no_op.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any + +from pipelines.models.contexts.pipeline_context import PipelineContext +from pipelines.models.steps import Step, StepResult, StepStatus + + +class NoOpStep(Step): + """A step that does nothing.""" + + title = "No Op" + should_log = False + + def __init__(self, context: PipelineContext, step_status: StepStatus) -> None: + super().__init__(context) + self.step_status = step_status + + async def _run(self, *args: Any, **kwargs: Any) -> StepResult: + return StepResult(self, self.step_status) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/poetry.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/poetry.py new file mode 100644 index 000000000000..43cb05ead074 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/poetry.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import List + +from pipelines.dagger.actions.python.poetry import with_poetry_module +from pipelines.models.contexts.pipeline_context import PipelineContext +from pipelines.models.steps import Step, StepResult + + +class PoetryRunStep(Step): + def __init__(self, context: PipelineContext, title: str, parent_dir_path: str, module_path: str, poetry_run_args: List[str]) -> None: + """A simple step that runs a given command inside a poetry project. + + Args: + context (PipelineContext): context of the step + title (str): name of the step + parent_dir_path (str): The path to the parent directory of the poetry project + module_path (str): The path to the poetry project + poetry_run_args (List[str]): The arguments to pass to the poetry run command + """ + self._title = title + super().__init__(context) + + parent_dir = self.context.get_repo_dir(parent_dir_path) + module_path = module_path + self.poetry_run_args = poetry_run_args + self.poetry_run_container = with_poetry_module(self.context, parent_dir, module_path).with_entrypoint(["poetry", "run"]) + + @property + def title(self) -> str: + return self._title + + async def _run(self) -> StepResult: + poetry_run_exec = self.poetry_run_container.with_exec(self.poetry_run_args) + return await self.get_step_result(poetry_run_exec) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py new file mode 100644 index 000000000000..4dc24ae0a197 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/commands.py @@ -0,0 +1,147 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from __future__ import annotations + +import logging +import os +from pathlib import Path +from typing import TYPE_CHECKING + +import asyncclick as click +import asyncer +from pipelines.cli.click_decorators import click_ci_requirements_option, click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.consts import DOCKER_VERSION +from pipelines.helpers.utils import sh_dash_c +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context + +if TYPE_CHECKING: + from typing import List, Tuple + + import dagger + +## HELPERS +async def run_poetry_command(container: dagger.Container, command: str) -> Tuple[str, str]: + """Run a poetry command in a container and return the stdout and stderr. + + Args: + container (dagger.Container): The container to run the command in. + command (str): The command to run. + + Returns: + Tuple[str, str]: The stdout and stderr of the command. + """ + container = container.with_exec(["poetry", "run", *command.split(" ")]) + return await container.stdout(), await container.stderr() + + +def validate_env_vars_exist(_ctx: dict, _param: dict, value: List[str]) -> List[str]: + for var in value: + if var not in os.environ: + raise click.BadParameter(f"Environment variable {var} does not exist.") + return value + + +@click.command() +@click.argument("poetry_package_path") +@click_ci_requirements_option() +@click.option( + "-c", + "--poetry-run-command", + multiple=True, + help="The poetry run command to run.", + required=True, +) +@click.option( + "--pass-env-var", + "-e", + "passed_env_vars", + multiple=True, + help="The environment variables to pass to the container.", + required=False, + callback=validate_env_vars_exist, +) +@click_merge_args_into_context_obj +@pass_pipeline_context +@click_ignore_unused_kwargs +async def test(pipeline_context: ClickPipelineContext) -> None: + """Runs the tests for the given airbyte-ci package + + Args: + pipeline_context (ClickPipelineContext): The context object. + """ + poetry_package_path = pipeline_context.params["poetry_package_path"] + if not Path(f"{poetry_package_path}/pyproject.toml").exists(): + raise click.UsageError(f"Could not find pyproject.toml in {poetry_package_path}") + + commands_to_run: List[str] = pipeline_context.params["poetry_run_command"] + + logger = logging.getLogger(f"{poetry_package_path}.tests") + logger.info(f"Running tests for {poetry_package_path}") + + # The following directories are always mounted because a lot of tests rely on them + directories_to_always_mount = [ + ".git", # This is needed as some package tests rely on being in a git repo + ".github", + "docs", + "airbyte-integrations", + "airbyte-ci", + "airbyte-cdk", + "pyproject.toml", + "LICENSE_SHORT", + "poetry.lock", + "spotless-maven-pom.xml", + "tools/gradle/codestyle/java-google-style.xml", + ] + directories_to_mount = list(set([poetry_package_path, *directories_to_always_mount])) + + pipeline_name = f"Unit tests for {poetry_package_path}" + dagger_client = await pipeline_context.get_dagger_client(pipeline_name=pipeline_name) + test_container = await ( + dagger_client.container() + .from_("python:3.10.12") + .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y bash git curl", + "pip install pipx", + "pipx ensurepath", + "pipx install poetry", + ] + ) + ) + .with_env_variable("VERSION", DOCKER_VERSION) + .with_exec(sh_dash_c(["curl -fsSL https://get.docker.com | sh"])) + .with_mounted_directory( + "/airbyte", + dagger_client.host().directory( + ".", + exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**.log", "**/.gradle"], + include=directories_to_mount, + ), + ) + .with_workdir(f"/airbyte/{poetry_package_path}") + .with_exec(["poetry", "install", "--with=dev"]) + .with_unix_socket("/var/run/docker.sock", dagger_client.host().unix_socket("/var/run/docker.sock")) + .with_env_variable("CI", str(pipeline_context.params["is_ci"])) + .with_workdir(f"/airbyte/{poetry_package_path}") + ) + + # register passed env vars as secrets and add them to the container + for var in pipeline_context.params["passed_env_vars"]: + secret = dagger_client.set_secret(var, os.environ[var]) + test_container = test_container.with_secret_variable(var, secret) + + soon_command_executions_results = [] + async with asyncer.create_task_group() as poetry_commands_task_group: + for command in commands_to_run: + logger.info(f"Running command: {command}") + soon_command_execution_result = poetry_commands_task_group.soonify(run_poetry_command)(test_container, command) + soon_command_executions_results.append(soon_command_execution_result) + + for result in soon_command_executions_results: + stdout, stderr = result.value + logger.info(stdout) + logger.error(stderr) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/update/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/update/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/update/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/update/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/update/commands.py new file mode 100644 index 000000000000..c633f59db1d2 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/update/commands.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging + +import asyncclick as click +from pipelines.cli.auto_update import is_dev_command +from pipelines.external_scripts.airbyte_ci_dev_install import main as install_airbyte_ci_dev_pipx +from pipelines.external_scripts.airbyte_ci_install import main as install_airbyte_ci_binary + + +@click.command() +@click.option("--version", default="latest", type=str, help="The version to update to.") +async def update(version: str) -> None: + """Updates airbyte-ci to the latest version.""" + is_dev = is_dev_command() + if is_dev: + logging.info("Updating to the latest development version of airbyte-ci...") + install_airbyte_ci_dev_pipx() + else: + logging.info("Updating to the latest version of airbyte-ci...") + install_airbyte_ci_binary(version) diff --git a/airbyte-ci/connectors/pipelines/pipelines/bases.py b/airbyte-ci/connectors/pipelines/pipelines/bases.py deleted file mode 100644 index b5f397d69ff2..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/bases.py +++ /dev/null @@ -1,659 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -"""This module declare base / abstract models to be reused in a pipeline lifecycle.""" - -from __future__ import annotations - -import json -import logging -import webbrowser -from abc import ABC, abstractmethod -from dataclasses import dataclass, field -from datetime import datetime, timedelta -from enum import Enum -from typing import TYPE_CHECKING, Any, ClassVar, List, Optional, Set - -import anyio -import asyncer -from anyio import Path -from connector_ops.utils import Connector, console -from dagger import Container, DaggerError -from jinja2 import Environment, PackageLoader, select_autoescape -from pipelines import sentry_utils -from pipelines.actions import remote_storage -from pipelines.consts import GCS_PUBLIC_DOMAIN, LOCAL_REPORTS_PATH_ROOT, PYPROJECT_TOML_FILE_PATH -from pipelines.utils import METADATA_FILE_NAME, check_path_in_workdir, format_duration, get_exec_result -from rich.console import Group -from rich.panel import Panel -from rich.style import Style -from rich.table import Table -from rich.text import Text -from tabulate import tabulate - -if TYPE_CHECKING: - from pipelines.contexts import PipelineContext - - -@dataclass(frozen=True) -class ConnectorWithModifiedFiles(Connector): - modified_files: Set[Path] = field(default_factory=frozenset) - - @property - def has_metadata_change(self) -> bool: - return any(path.name == METADATA_FILE_NAME for path in self.modified_files) - - -class CIContext(str, Enum): - """An enum for Ci context values which can be ["manual", "pull_request", "nightly_builds"].""" - - MANUAL = "manual" - PULL_REQUEST = "pull_request" - NIGHTLY_BUILDS = "nightly_builds" - MASTER = "master" - - def __str__(self) -> str: - return self.value - - -class StepStatus(Enum): - """An Enum to characterize the success, failure or skipping of a Step.""" - - SUCCESS = "Successful" - FAILURE = "Failed" - SKIPPED = "Skipped" - - def get_rich_style(self) -> Style: - """Match color used in the console output to the step status.""" - if self is StepStatus.SUCCESS: - return Style(color="green") - if self is StepStatus.FAILURE: - return Style(color="red", bold=True) - if self is StepStatus.SKIPPED: - return Style(color="yellow") - - def get_emoji(self) -> str: - """Match emoji used in the console output to the step status.""" - if self is StepStatus.SUCCESS: - return "✅" - if self is StepStatus.FAILURE: - return "❌" - if self is StepStatus.SKIPPED: - return "🟡" - - def __str__(self) -> str: # noqa D105 - return self.value - - -class Step(ABC): - """An abstract class to declare and run pipeline step.""" - - title: ClassVar[str] - max_retries: ClassVar[int] = 0 - max_dagger_error_retries: ClassVar[int] = 3 - should_log: ClassVar[bool] = True - success_exit_code: ClassVar[int] = 0 - skipped_exit_code: ClassVar[int] = None - # The max duration of a step run. If the step run for more than this duration it will be considered as timed out. - # The default of 5 hours is arbitrary and can be changed if needed. - max_duration: ClassVar[timedelta] = timedelta(hours=5) - - retry_delay = timedelta(seconds=10) - - def __init__(self, context: PipelineContext) -> None: # noqa D107 - self.context = context - self.retry_count = 0 - self.started_at = None - self.stopped_at = None - - @property - def run_duration(self) -> timedelta: - if self.started_at and self.stopped_at: - return self.stopped_at - self.started_at - else: - return timedelta(seconds=0) - - @property - def logger(self) -> logging.Logger: - if self.should_log: - return logging.getLogger(f"{self.context.pipeline_name} - {self.title}") - else: - disabled_logger = logging.getLogger() - disabled_logger.disabled = True - return disabled_logger - - @property - def dagger_client(self) -> Container: - return self.context.dagger_client.pipeline(self.title) - - async def log_progress(self, completion_event: anyio.Event) -> None: - """Log the step progress every 30 seconds until the step is done.""" - while not completion_event.is_set(): - duration = datetime.utcnow() - self.started_at - elapsed_seconds = duration.total_seconds() - if elapsed_seconds > 30 and round(elapsed_seconds) % 30 == 0: - self.logger.info(f"⏳ Still running... (duration: {format_duration(duration)})") - await anyio.sleep(1) - - async def run_with_completion(self, completion_event: anyio.Event, *args, **kwargs) -> StepResult: - """Run the step with a timeout and set the completion event when the step is done.""" - try: - with anyio.fail_after(self.max_duration.total_seconds()): - result = await self._run(*args, **kwargs) - completion_event.set() - return result - except TimeoutError: - self.retry_count = self.max_retries + 1 - self.logger.error(f"🚨 {self.title} timed out after {self.max_duration}. No additional retry will happen.") - completion_event.set() - return self._get_timed_out_step_result() - - @sentry_utils.with_step_context - async def run(self, *args, **kwargs) -> StepResult: - """Public method to run the step. It output a step result. - - If an unexpected dagger error happens it outputs a failed step result with the exception payload. - - Returns: - StepResult: The step result following the step run. - """ - self.logger.info(f"🚀 Start {self.title}") - self.started_at = datetime.utcnow() - completion_event = anyio.Event() - try: - async with asyncer.create_task_group() as task_group: - soon_result = task_group.soonify(self.run_with_completion)(completion_event, *args, **kwargs) - task_group.soonify(self.log_progress)(completion_event) - step_result = soon_result.value - except DaggerError as e: - self.logger.error("Step failed with an unexpected dagger error", exc_info=e) - step_result = StepResult(self, StepStatus.FAILURE, stderr=str(e), exc_info=e) - - self.stopped_at = datetime.utcnow() - self.log_step_result(step_result) - - lets_retry = self.should_retry(step_result) - step_result = await self.retry(step_result, *args, **kwargs) if lets_retry else step_result - return step_result - - def should_retry(self, step_result: StepResult) -> bool: - """Return True if the step should be retried.""" - if step_result.status is not StepStatus.FAILURE: - return False - max_retries = self.max_dagger_error_retries if step_result.exc_info else self.max_retries - return self.retry_count < max_retries and max_retries > 0 - - async def retry(self, step_result, *args, **kwargs) -> StepResult: - self.retry_count += 1 - self.logger.warn( - f"Failed with error: {step_result.stderr}.\nRetry #{self.retry_count} in {self.retry_delay.total_seconds()} seconds..." - ) - await anyio.sleep(self.retry_delay.total_seconds()) - return await self.run(*args, **kwargs) - - def log_step_result(self, result: StepResult) -> None: - """Log the step result. - - Args: - result (StepResult): The step result to log. - """ - duration = format_duration(self.run_duration) - if result.status is StepStatus.FAILURE: - self.logger.info(f"{result.status.get_emoji()} failed (duration: {duration})") - if result.status is StepStatus.SKIPPED: - self.logger.info(f"{result.status.get_emoji()} was skipped (duration: {duration})") - if result.status is StepStatus.SUCCESS: - self.logger.info(f"{result.status.get_emoji()} was successful (duration: {duration})") - - @abstractmethod - async def _run(self, *args, **kwargs) -> StepResult: - """Implement the execution of the step and return a step result. - - Returns: - StepResult: The result of the step run. - """ - ... - - def skip(self, reason: str = None) -> StepResult: - """Declare a step as skipped. - - Args: - reason (str, optional): Reason why the step was skipped. - - Returns: - StepResult: A skipped step result. - """ - return StepResult(self, StepStatus.SKIPPED, stdout=reason) - - def get_step_status_from_exit_code( - self, - exit_code: int, - ) -> StepStatus: - """Map an exit code to a step status. - - Args: - exit_code (int): A process exit code. - - Raises: - ValueError: Raised if the exit code is not mapped to a step status. - - Returns: - StepStatus: The step status inferred from the exit code. - """ - if exit_code == self.success_exit_code: - return StepStatus.SUCCESS - elif self.skipped_exit_code is not None and exit_code == self.skipped_exit_code: - return StepStatus.SKIPPED - else: - return StepStatus.FAILURE - - async def get_step_result(self, container: Container) -> StepResult: - """Concurrent retrieval of exit code, stdout and stdout of a container. - - Create a StepResult object from these objects. - - Args: - container (Container): The container from which we want to infer a step result/ - - Returns: - StepResult: Failure or success with stdout and stderr. - """ - exit_code, stdout, stderr = await get_exec_result(container) - return StepResult( - self, - self.get_step_status_from_exit_code(exit_code), - stderr=stderr, - stdout=stdout, - output_artifact=container, - ) - - def _get_timed_out_step_result(self) -> StepResult: - return StepResult( - self, - StepStatus.FAILURE, - stdout=f"Timed out after the max duration of {format_duration(self.max_duration)}. Please checkout the Dagger logs to see what happened.", - ) - - -class PytestStep(Step, ABC): - """An abstract class to run pytest tests and evaluate success or failure according to pytest logs.""" - - skipped_exit_code = 5 - - async def _run_tests_in_directory(self, connector_under_test: Container, test_directory: str) -> StepResult: - """Run the pytest tests in the test_directory that was passed. - - A StepStatus.SKIPPED is returned if no tests were discovered. - - Args: - connector_under_test (Container): The connector under test container. - test_directory (str): The directory in which the python test modules are declared - - Returns: - Tuple[StepStatus, Optional[str], Optional[str]]: Tuple of StepStatus, stderr and stdout. - """ - test_config = "pytest.ini" if await check_path_in_workdir(connector_under_test, "pytest.ini") else "/" + PYPROJECT_TOML_FILE_PATH - if await check_path_in_workdir(connector_under_test, test_directory): - tester = connector_under_test.with_exec( - [ - "python", - "-m", - "pytest", - "-s", - test_directory, - "-c", - test_config, - ] - ) - return await self.get_step_result(tester) - - else: - return StepResult(self, StepStatus.SKIPPED) - - -class NoOpStep(Step): - """A step that does nothing.""" - - title = "No Op" - should_log = False - - def __init__(self, context: PipelineContext, step_status: StepStatus) -> None: - super().__init__(context) - self.step_status = step_status - - async def _run(self, *args, **kwargs) -> StepResult: - return StepResult(self, self.step_status) - - -@dataclass(frozen=True) -class StepResult: - """A dataclass to capture the result of a step.""" - - step: Step - status: StepStatus - created_at: datetime = field(default_factory=datetime.utcnow) - stderr: Optional[str] = None - stdout: Optional[str] = None - output_artifact: Any = None - exc_info: Optional[Exception] = None - - def __repr__(self) -> str: # noqa D105 - return f"{self.step.title}: {self.status.value}" - - def __str__(self) -> str: # noqa D105 - return f"{self.step.title}: {self.status.value}\n\nSTDOUT:\n{self.stdout}\n\nSTDERR:\n{self.stderr}" - - def __post_init__(self): - if self.stderr: - super().__setattr__("stderr", self.redact_secrets_from_string(self.stderr)) - if self.stdout: - super().__setattr__("stdout", self.redact_secrets_from_string(self.stdout)) - - def redact_secrets_from_string(self, value: str) -> str: - for secret in self.step.context.secrets_to_mask: - value = value.replace(secret, "********") - return value - - -@dataclass(frozen=True) -class Report: - """A dataclass to build reports to share pipelines executions results with the user.""" - - pipeline_context: PipelineContext - steps_results: List[StepResult] - created_at: datetime = field(default_factory=datetime.utcnow) - name: str = "REPORT" - filename: str = "output" - - @property - def report_output_prefix(self) -> str: # noqa D102 - return self.pipeline_context.report_output_prefix - - @property - def json_report_file_name(self) -> str: # noqa D102 - return self.filename + ".json" - - @property - def json_report_remote_storage_key(self) -> str: # noqa D102 - return f"{self.report_output_prefix}/{self.json_report_file_name}" - - @property - def failed_steps(self) -> List[StepResult]: # noqa D102 - return [step_result for step_result in self.steps_results if step_result.status is StepStatus.FAILURE] - - @property - def successful_steps(self) -> List[StepResult]: # noqa D102 - return [step_result for step_result in self.steps_results if step_result.status is StepStatus.SUCCESS] - - @property - def skipped_steps(self) -> List[StepResult]: # noqa D102 - return [step_result for step_result in self.steps_results if step_result.status is StepStatus.SKIPPED] - - @property - def success(self) -> bool: # noqa D102 - return len(self.failed_steps) == 0 and (len(self.skipped_steps) > 0 or len(self.successful_steps) > 0) - - @property - def run_duration(self) -> timedelta: # noqa D102 - return self.pipeline_context.stopped_at - self.pipeline_context.started_at - - @property - def lead_duration(self) -> timedelta: # noqa D102 - return self.pipeline_context.stopped_at - self.pipeline_context.created_at - - @property - def remote_storage_enabled(self) -> bool: # noqa D102 - return self.pipeline_context.is_ci - - async def save_local(self, filename: str, content: str) -> Path: - """Save the report files locally.""" - local_path = anyio.Path(f"{LOCAL_REPORTS_PATH_ROOT}/{self.report_output_prefix}/{filename}") - await local_path.parents[0].mkdir(parents=True, exist_ok=True) - await local_path.write_text(content) - return local_path - - async def save_remote(self, local_path: Path, remote_key: str, content_type: str = None) -> int: - gcs_cp_flags = None if content_type is None else [f"--content-type={content_type}"] - local_file = self.pipeline_context.dagger_client.host().directory(".", include=[str(local_path)]).file(str(local_path)) - report_upload_exit_code, _, _ = await remote_storage.upload_to_gcs( - dagger_client=self.pipeline_context.dagger_client, - file_to_upload=local_file, - key=remote_key, - bucket=self.pipeline_context.ci_report_bucket, - gcs_credentials=self.pipeline_context.ci_gcs_credentials_secret, - flags=gcs_cp_flags, - ) - gcs_uri = "gs://" + self.pipeline_context.ci_report_bucket + "/" + remote_key - public_url = f"{GCS_PUBLIC_DOMAIN}/{self.pipeline_context.ci_report_bucket}/{remote_key}" - if report_upload_exit_code != 0: - self.pipeline_context.logger.error(f"Uploading {local_path} to {gcs_uri} failed.") - else: - self.pipeline_context.logger.info(f"Uploading {local_path} to {gcs_uri} succeeded. Public URL: {public_url}") - return report_upload_exit_code - - async def save(self) -> None: - """Save the report files.""" - local_json_path = await self.save_local(self.json_report_file_name, self.to_json()) - absolute_path = await local_json_path.absolute() - self.pipeline_context.logger.info(f"Report saved locally at {absolute_path}") - if self.remote_storage_enabled: - await self.save_remote(local_json_path, self.json_report_remote_storage_key, "application/json") - - def to_json(self) -> str: - """Create a JSON representation of the report. - - Returns: - str: The JSON representation of the report. - """ - return json.dumps( - { - "pipeline_name": self.pipeline_context.pipeline_name, - "run_timestamp": self.pipeline_context.started_at.isoformat(), - "run_duration": self.run_duration.total_seconds(), - "success": self.success, - "failed_steps": [s.step.__class__.__name__ for s in self.failed_steps], - "successful_steps": [s.step.__class__.__name__ for s in self.successful_steps], - "skipped_steps": [s.step.__class__.__name__ for s in self.skipped_steps], - "gha_workflow_run_url": self.pipeline_context.gha_workflow_run_url, - "pipeline_start_timestamp": self.pipeline_context.pipeline_start_timestamp, - "pipeline_end_timestamp": round(self.pipeline_context.stopped_at.timestamp()), - "pipeline_duration": round(self.pipeline_context.stopped_at.timestamp()) - self.pipeline_context.pipeline_start_timestamp, - "git_branch": self.pipeline_context.git_branch, - "git_revision": self.pipeline_context.git_revision, - "ci_context": self.pipeline_context.ci_context, - "pull_request_url": self.pipeline_context.pull_request.html_url if self.pipeline_context.pull_request else None, - "dagger_cloud_url": self.pipeline_context.dagger_cloud_url, - } - ) - - def print(self): - """Print the test report to the console in a nice way.""" - pipeline_name = self.pipeline_context.pipeline_name - main_panel_title = Text(f"{pipeline_name.upper()} - {self.name}") - main_panel_title.stylize(Style(color="blue", bold=True)) - duration_subtitle = Text(f"⏲️ Total pipeline duration for {pipeline_name}: {format_duration(self.run_duration)}") - step_results_table = Table(title="Steps results") - step_results_table.add_column("Step") - step_results_table.add_column("Result") - step_results_table.add_column("Finished after") - - for step_result in self.steps_results: - step = Text(step_result.step.title) - step.stylize(step_result.status.get_rich_style()) - result = Text(step_result.status.value) - result.stylize(step_result.status.get_rich_style()) - - if step_result.status is StepStatus.SKIPPED: - step_results_table.add_row(step, result, "N/A") - else: - run_time = format_duration((step_result.created_at - step_result.step.started_at)) - step_results_table.add_row(step, result, run_time) - - to_render = [step_results_table] - if self.failed_steps: - sub_panels = [] - for failed_step in self.failed_steps: - errors = Text(failed_step.stderr) - panel_title = Text(f"{pipeline_name} {failed_step.step.title.lower()} failures") - panel_title.stylize(Style(color="red", bold=True)) - sub_panel = Panel(errors, title=panel_title) - sub_panels.append(sub_panel) - failures_group = Group(*sub_panels) - to_render.append(failures_group) - - if self.pipeline_context.dagger_cloud_url: - self.pipeline_context.logger.info(f"🔗 View runs for commit in Dagger Cloud: {self.pipeline_context.dagger_cloud_url}") - - main_panel = Panel(Group(*to_render), title=main_panel_title, subtitle=duration_subtitle) - console.print(main_panel) - - -@dataclass(frozen=True) -class ConnectorReport(Report): - """A dataclass to build connector test reports to share pipelines executions results with the user.""" - - @property - def report_output_prefix(self) -> str: # noqa D102 - return f"{self.pipeline_context.report_output_prefix}/{self.pipeline_context.connector.technical_name}/{self.pipeline_context.connector.version}" - - @property - def html_report_file_name(self) -> str: # noqa D102 - return self.filename + ".html" - - @property - def html_report_remote_storage_key(self) -> str: # noqa D102 - return f"{self.report_output_prefix}/{self.html_report_file_name}" - - @property - def html_report_url(self) -> str: # noqa D102 - return f"{GCS_PUBLIC_DOMAIN}/{self.pipeline_context.ci_report_bucket}/{self.html_report_remote_storage_key}" - - @property - def should_be_commented_on_pr(self) -> bool: # noqa D102 - return ( - self.pipeline_context.should_save_report - and self.pipeline_context.is_ci - and self.pipeline_context.pull_request - and self.pipeline_context.PRODUCTION - ) - - def to_json(self) -> str: - """Create a JSON representation of the connector test report. - - Returns: - str: The JSON representation of the report. - """ - return json.dumps( - { - "connector_technical_name": self.pipeline_context.connector.technical_name, - "connector_version": self.pipeline_context.connector.version, - "run_timestamp": self.created_at.isoformat(), - "run_duration": self.run_duration.total_seconds(), - "success": self.success, - "failed_steps": [s.step.__class__.__name__ for s in self.failed_steps], - "successful_steps": [s.step.__class__.__name__ for s in self.successful_steps], - "skipped_steps": [s.step.__class__.__name__ for s in self.skipped_steps], - "gha_workflow_run_url": self.pipeline_context.gha_workflow_run_url, - "pipeline_start_timestamp": self.pipeline_context.pipeline_start_timestamp, - "pipeline_end_timestamp": round(self.created_at.timestamp()), - "pipeline_duration": round(self.created_at.timestamp()) - self.pipeline_context.pipeline_start_timestamp, - "git_branch": self.pipeline_context.git_branch, - "git_revision": self.pipeline_context.git_revision, - "ci_context": self.pipeline_context.ci_context, - "cdk_version": self.pipeline_context.cdk_version, - "html_report_url": self.html_report_url, - "dagger_cloud_url": self.pipeline_context.dagger_cloud_url, - } - ) - - def post_comment_on_pr(self) -> None: - icon_url = f"https://raw.githubusercontent.com/airbytehq/airbyte/{self.pipeline_context.git_revision}/{self.pipeline_context.connector.code_directory}/icon.svg" - global_status_emoji = "✅" if self.success else "❌" - commit_url = f"{self.pipeline_context.pull_request.html_url}/commits/{self.pipeline_context.git_revision}" - markdown_comment = f'## {self.pipeline_context.connector.technical_name} test report (commit [`{self.pipeline_context.git_revision[:10]}`]({commit_url})) - {global_status_emoji}\n\n' - markdown_comment += f"⏲️ Total pipeline duration: {format_duration(self.run_duration)} \n\n" - report_data = [ - [step_result.step.title, step_result.status.get_emoji()] - for step_result in self.steps_results - if step_result.status is not StepStatus.SKIPPED - ] - markdown_comment += tabulate(report_data, headers=["Step", "Result"], tablefmt="pipe") + "\n\n" - markdown_comment += f"🔗 [View the logs here]({self.html_report_url})\n\n" - - if self.pipeline_context.dagger_cloud_url: - markdown_comment += f"☁️ [View runs for commit in Dagger Cloud]({self.pipeline_context.dagger_cloud_url})\n\n" - - markdown_comment += "*Please note that tests are only run on PR ready for review. Please set your PR to draft mode to not flood the CI engine and upstream service on following commits.*\n" - markdown_comment += "**You can run the same pipeline locally on this branch with the [airbyte-ci](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) tool with the following command**\n" - markdown_comment += f"```bash\nairbyte-ci connectors --name={self.pipeline_context.connector.technical_name} test\n```\n\n" - self.pipeline_context.pull_request.create_issue_comment(markdown_comment) - - async def to_html(self) -> str: - env = Environment(loader=PackageLoader("pipelines.tests"), autoescape=select_autoescape(), trim_blocks=False, lstrip_blocks=True) - template = env.get_template("test_report.html.j2") - template.globals["StepStatus"] = StepStatus - template.globals["format_duration"] = format_duration - local_icon_path = await Path(f"{self.pipeline_context.connector.code_directory}/icon.svg").resolve() - template_context = { - "connector_name": self.pipeline_context.connector.technical_name, - "step_results": self.steps_results, - "run_duration": self.run_duration, - "created_at": self.created_at.isoformat(), - "connector_version": self.pipeline_context.connector.version, - "gha_workflow_run_url": None, - "dagger_logs_url": None, - "git_branch": self.pipeline_context.git_branch, - "git_revision": self.pipeline_context.git_revision, - "commit_url": None, - "icon_url": local_icon_path.as_uri(), - } - - if self.pipeline_context.is_ci: - template_context["commit_url"] = f"https://github.com/airbytehq/airbyte/commit/{self.pipeline_context.git_revision}" - template_context["gha_workflow_run_url"] = self.pipeline_context.gha_workflow_run_url - template_context["dagger_logs_url"] = self.pipeline_context.dagger_logs_url - template_context["dagger_cloud_url"] = self.pipeline_context.dagger_cloud_url - template_context[ - "icon_url" - ] = f"https://raw.githubusercontent.com/airbytehq/airbyte/{self.pipeline_context.git_revision}/{self.pipeline_context.connector.code_directory}/icon.svg" - return template.render(template_context) - - async def save(self) -> None: - local_html_path = await self.save_local(self.html_report_file_name, await self.to_html()) - absolute_path = await local_html_path.resolve() - if self.pipeline_context.is_local: - self.pipeline_context.logger.info(f"HTML report saved locally: {absolute_path}") - self.pipeline_context.logger.info("Opening HTML report in browser.") - webbrowser.open(absolute_path.as_uri()) - if self.remote_storage_enabled: - await self.save_remote(local_html_path, self.html_report_remote_storage_key, "text/html") - self.pipeline_context.logger.info(f"HTML report uploaded to {self.html_report_url}") - await super().save() - - def print(self): - """Print the test report to the console in a nice way.""" - connector_name = self.pipeline_context.connector.technical_name - main_panel_title = Text(f"{connector_name.upper()} - {self.name}") - main_panel_title.stylize(Style(color="blue", bold=True)) - duration_subtitle = Text(f"⏲️ Total pipeline duration for {connector_name}: {format_duration(self.run_duration)}") - step_results_table = Table(title="Steps results") - step_results_table.add_column("Step") - step_results_table.add_column("Result") - step_results_table.add_column("Duration") - - for step_result in self.steps_results: - step = Text(step_result.step.title) - step.stylize(step_result.status.get_rich_style()) - result = Text(step_result.status.value) - result.stylize(step_result.status.get_rich_style()) - step_results_table.add_row(step, result, format_duration(step_result.step.run_duration)) - - details_instructions = Text("ℹ️ You can find more details with step executions logs in the saved HTML report.") - to_render = [step_results_table, details_instructions] - - if self.pipeline_context.dagger_cloud_url: - self.pipeline_context.logger.info(f"🔗 View runs for commit in Dagger Cloud: {self.pipeline_context.dagger_cloud_url}") - - main_panel = Panel(Group(*to_render), title=main_panel_title, subtitle=duration_subtitle) - console.print(main_panel) diff --git a/airbyte-ci/connectors/pipelines/pipelines/builds/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/builds/__init__.py deleted file mode 100644 index 3d92eabf6234..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/builds/__init__.py +++ /dev/null @@ -1,58 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# -"""This module groups factory like functions to dispatch builds steps according to the connector language.""" - -from __future__ import annotations - -import platform - -import anyio -from connector_ops.utils import ConnectorLanguage -from dagger import Platform -from pipelines.bases import ConnectorReport, StepResult -from pipelines.builds import common, java_connectors, python_connectors -from pipelines.contexts import ConnectorContext - - -class NoBuildStepForLanguageError(Exception): - pass - - -LANGUAGE_BUILD_CONNECTOR_MAPPING = { - ConnectorLanguage.PYTHON: python_connectors.run_connector_build, - ConnectorLanguage.LOW_CODE: python_connectors.run_connector_build, - ConnectorLanguage.JAVA: java_connectors.run_connector_build, -} - -BUILD_PLATFORMS = [Platform("linux/amd64"), Platform("linux/arm64")] -LOCAL_BUILD_PLATFORM = Platform(f"linux/{platform.machine()}") - - -async def run_connector_build(context: ConnectorContext) -> StepResult: - """Run a build pipeline for a single connector.""" - if context.connector.language not in LANGUAGE_BUILD_CONNECTOR_MAPPING: - raise NoBuildStepForLanguageError(f"No build step for connector language {context.connector.language}.") - return await LANGUAGE_BUILD_CONNECTOR_MAPPING[context.connector.language](context) - - -async def run_connector_build_pipeline(context: ConnectorContext, semaphore: anyio.Semaphore) -> ConnectorReport: - """Run a build pipeline for a single connector. - - Args: - context (ConnectorContext): The initialized connector context. - - Returns: - ConnectorReport: The reports holding builds results. - """ - step_results = [] - async with semaphore: - async with context: - build_result = await run_connector_build(context) - step_results.append(build_result) - if context.is_local and build_result.status is common.StepStatus.SUCCESS: - connector_to_load_to_local_docker_host = build_result.output_artifact[LOCAL_BUILD_PLATFORM] - load_image_result = await common.LoadContainerToLocalDockerHost(context, connector_to_load_to_local_docker_host).run() - step_results.append(load_image_result) - context.report = ConnectorReport(context, step_results, name="BUILD RESULTS") - return context.report diff --git a/airbyte-ci/connectors/pipelines/pipelines/builds/common.py b/airbyte-ci/connectors/pipelines/pipelines/builds/common.py deleted file mode 100644 index d8de2bffc4a1..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/builds/common.py +++ /dev/null @@ -1,65 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from abc import ABC -from typing import Tuple - -import docker -from dagger import Container, Platform -from pipelines.bases import Step, StepResult, StepStatus -from pipelines.consts import BUILD_PLATFORMS -from pipelines.contexts import ConnectorContext -from pipelines.utils import export_container_to_tarball - - -class BuildConnectorImageBase(Step, ABC): - @property - def title(self): - return f"Build {self.context.connector.technical_name} docker image for platform {self.build_platform}" - - def __init__(self, context: ConnectorContext, build_platform: Platform) -> None: - self.build_platform = build_platform - super().__init__(context) - - -class BuildConnectorImageForAllPlatformsBase(Step, ABC): - - ALL_PLATFORMS = BUILD_PLATFORMS - - title = f"Build connector image for {BUILD_PLATFORMS}" - - def get_success_result(self, build_results_per_platform: dict[Platform, Container]) -> StepResult: - return StepResult( - self, - StepStatus.SUCCESS, - stdout="The connector image was successfully built for all platforms.", - output_artifact=build_results_per_platform, - ) - - -class LoadContainerToLocalDockerHost(Step): - IMAGE_TAG = "dev" - - def __init__(self, context: ConnectorContext, container: Container) -> None: - super().__init__(context) - self.container = container - - @property - def title(self): - return f"Load {self.image_name}:{self.IMAGE_TAG} to the local docker host." - - @property - def image_name(self) -> Tuple: - return f"airbyte/{self.context.connector.technical_name}" - - async def _run(self) -> StepResult: - _, exported_tarball_path = await export_container_to_tarball(self.context, self.container) - client = docker.from_env() - try: - with open(exported_tarball_path, "rb") as tarball_content: - new_image = client.images.load(tarball_content.read())[0] - new_image.tag(self.image_name, tag=self.IMAGE_TAG) - return StepResult(self, StepStatus.SUCCESS) - except ConnectionError: - return StepResult(self, StepStatus.FAILURE, stderr="The connection to the local docker host failed.") diff --git a/airbyte-ci/connectors/pipelines/pipelines/builds/java_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/builds/java_connectors.py deleted file mode 100644 index ffd6ff1e8bd4..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/builds/java_connectors.py +++ /dev/null @@ -1,88 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from dagger import ExecError, File, QueryError -from pipelines.actions import environments -from pipelines.bases import StepResult, StepStatus -from pipelines.builds.common import BuildConnectorImageBase, BuildConnectorImageForAllPlatformsBase -from pipelines.contexts import ConnectorContext -from pipelines.gradle import GradleTask - - -class BuildConnectorDistributionTar(GradleTask): - title = "Build connector tar" - gradle_task_name = "distTar" - - async def _run(self) -> StepResult: - cdk_includes = ["./airbyte-cdk/java/airbyte-cdk/**"] - with_built_tar = ( - environments.with_gradle( - self.context, - self.build_include + cdk_includes, - ) - .with_exec(["./gradlew", ":airbyte-cdk:java:airbyte-cdk:publishSnapshotIfNeeded"]) - .with_mounted_directory(str(self.context.connector.code_directory), await self.context.get_connector_dir()) - .with_exec(self._get_gradle_command()) - .with_workdir(f"{self.context.connector.code_directory}/build/distributions") - ) - distributions = await with_built_tar.directory(".").entries() - tar_files = [f for f in distributions if f.endswith(".tar")] - await self._export_gradle_dependency_cache(with_built_tar) - if len(tar_files) == 1: - return StepResult( - self, - StepStatus.SUCCESS, - stdout="The tar file for the current connector was successfully built.", - output_artifact=with_built_tar.file(tar_files[0]), - ) - else: - return StepResult( - self, - StepStatus.FAILURE, - stderr="The distributions directory contains multiple connector tar files. We can't infer which one should be used. Please review and delete any unnecessary tar files.", - ) - - -class BuildConnectorImage(BuildConnectorImageBase): - """ - A step to build a Java connector image using the distTar Gradle task. - """ - - async def _run(self, distribution_tar: File) -> StepResult: - try: - java_connector = await environments.with_airbyte_java_connector(self.context, distribution_tar, self.build_platform) - try: - await java_connector.with_exec(["spec"]) - except ExecError: - return StepResult( - self, StepStatus.FAILURE, stderr=f"Failed to run spec on the connector built for platform {self.build_platform}." - ) - return StepResult( - self, StepStatus.SUCCESS, stdout="The connector image was successfully built.", output_artifact=java_connector - ) - except QueryError as e: - return StepResult(self, StepStatus.FAILURE, stderr=str(e)) - - -class BuildConnectorImageForAllPlatforms(BuildConnectorImageForAllPlatformsBase): - """Build a Java connector image for all platforms.""" - - async def _run(self, distribution_tar: File) -> StepResult: - build_results_per_platform = {} - for platform in self.ALL_PLATFORMS: - build_connector_step_result = await BuildConnectorImage(self.context, platform).run(distribution_tar) - if build_connector_step_result.status is not StepStatus.SUCCESS: - return build_connector_step_result - build_results_per_platform[platform] = build_connector_step_result.output_artifact - return self.get_success_result(build_results_per_platform) - - -async def run_connector_build(context: ConnectorContext) -> StepResult: - """Create the java connector distribution tar file and build the connector image.""" - - build_connector_tar_result = await BuildConnectorDistributionTar(context).run() - if build_connector_tar_result.status is not StepStatus.SUCCESS: - return build_connector_tar_result - - return await BuildConnectorImageForAllPlatforms(context).run(build_connector_tar_result.output_artifact) diff --git a/airbyte-ci/connectors/pipelines/pipelines/builds/normalization.py b/airbyte-ci/connectors/pipelines/pipelines/builds/normalization.py deleted file mode 100644 index 3494086eee8c..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/builds/normalization.py +++ /dev/null @@ -1,34 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from dagger import Platform -from pipelines.actions import environments -from pipelines.bases import Step, StepResult, StepStatus -from pipelines.contexts import ConnectorContext - - -# TODO this class could be deleted -# if java connectors tests are not relying on an existing local normalization image to run -class BuildOrPullNormalization(Step): - """A step to build or pull the normalization image for a connector according to the image name.""" - - def __init__(self, context: ConnectorContext, normalization_image: str, build_platform: Platform) -> None: - """Initialize the step to build or pull the normalization image. - - Args: - context (ConnectorContext): The current connector context. - normalization_image (str): The normalization image to build (if :dev) or pull. - """ - super().__init__(context) - self.build_platform = build_platform - self.use_dev_normalization = normalization_image.endswith(":dev") - self.normalization_image = normalization_image - self.title = f"Build {self.normalization_image}" if self.use_dev_normalization else f"Pull {self.normalization_image}" - - async def _run(self) -> StepResult: - if self.use_dev_normalization: - build_normalization_container = environments.with_normalization(self.context, self.build_platform) - else: - build_normalization_container = self.context.dagger_client.container().from_(self.normalization_image) - return StepResult(self, StepStatus.SUCCESS, output_artifact=build_normalization_container) diff --git a/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py deleted file mode 100644 index d18dc9537d8d..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from dagger import QueryError -from pipelines.actions.environments import with_airbyte_python_connector -from pipelines.bases import StepResult, StepStatus -from pipelines.builds.common import BuildConnectorImageBase, BuildConnectorImageForAllPlatformsBase -from pipelines.contexts import ConnectorContext - - -class BuildConnectorImage(BuildConnectorImageBase): - """ - A step to build a Python connector image. - A spec command is run on the container to validate it was built successfully. - """ - - async def _run(self) -> StepResult: - connector = await with_airbyte_python_connector(self.context, self.build_platform) - try: - return await self.get_step_result(connector.with_exec(["spec"])) - except QueryError as e: - return StepResult(self, StepStatus.FAILURE, stderr=str(e)) - - -class BuildConnectorImageForAllPlatforms(BuildConnectorImageForAllPlatformsBase): - """Build a Python connector image for all platforms.""" - - async def _run(self) -> StepResult: - build_results_per_platform = {} - for platform in self.ALL_PLATFORMS: - build_connector_step_result = await BuildConnectorImage(self.context, platform).run() - if build_connector_step_result.status is not StepStatus.SUCCESS: - return build_connector_step_result - build_results_per_platform[platform] = build_connector_step_result.output_artifact - return self.get_success_result(build_results_per_platform) - - -async def run_connector_build(context: ConnectorContext) -> StepResult: - return await BuildConnectorImageForAllPlatforms(context).run() diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py new file mode 100644 index 000000000000..8779fee5eab1 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -0,0 +1,206 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This module is the CLI entrypoint to the airbyte-ci commands.""" + +from __future__ import annotations + +# HACK! IMPORTANT! This import and function call must be the first import in this file +# This is needed to ensure that the working directory is the root of the airbyte repo +# ruff: noqa: E402 +from pipelines.cli.ensure_repo_root import set_working_directory_to_root + +set_working_directory_to_root() + +import logging +import multiprocessing +import os +import sys +from typing import Optional + +import asyncclick as click +import docker # type: ignore +from github import PullRequest +from pipelines import main_logger +from pipelines.cli.auto_update import __installed_version__, check_for_upgrade, pre_confirm_auto_update_flag +from pipelines.cli.click_decorators import ( + CI_REQUIREMENTS_OPTION_NAME, + click_append_to_context_object, + click_ignore_unused_kwargs, + click_merge_args_into_context_obj, +) +from pipelines.cli.confirm_prompt import pre_confirm_all_flag +from pipelines.cli.lazy_group import LazyGroup +from pipelines.cli.telemetry import click_track_command +from pipelines.consts import DAGGER_WRAP_ENV_VAR_NAME, CIContext +from pipelines.dagger.actions.connector.hooks import get_dagger_sdk_version +from pipelines.helpers import github +from pipelines.helpers.git import get_current_git_branch, get_current_git_revision +from pipelines.helpers.utils import get_current_epoch_time + + +def log_context_info(ctx: click.Context) -> None: + main_logger.info(f"Running airbyte-ci version {__installed_version__}") + main_logger.info(f"Running dagger version {get_dagger_sdk_version()}") + main_logger.info("Running airbyte-ci in CI mode.") + main_logger.info(f"CI Context: {ctx.obj['ci_context']}") + main_logger.info(f"CI Report Bucket Name: {ctx.obj['ci_report_bucket_name']}") + main_logger.info(f"Git Branch: {ctx.obj['git_branch']}") + main_logger.info(f"Git Revision: {ctx.obj['git_revision']}") + main_logger.info(f"GitHub Workflow Run ID: {ctx.obj['gha_workflow_run_id']}") + main_logger.info(f"GitHub Workflow Run URL: {ctx.obj['gha_workflow_run_url']}") + main_logger.info(f"Pull Request Number: {ctx.obj['pull_request_number']}") + main_logger.info(f"Pipeline Start Timestamp: {ctx.obj['pipeline_start_timestamp']}") + + +def _get_gha_workflow_run_url(ctx: click.Context) -> Optional[str]: + gha_workflow_run_id = ctx.obj["gha_workflow_run_id"] + if not gha_workflow_run_id: + return None + + return f"https://github.com/airbytehq/airbyte/actions/runs/{gha_workflow_run_id}" + + +def _get_pull_request(ctx: click.Context) -> Optional[PullRequest.PullRequest]: + pull_request_number = ctx.obj["pull_request_number"] + ci_github_access_token = ctx.obj["ci_github_access_token"] + + can_get_pull_request = pull_request_number and ci_github_access_token + if not can_get_pull_request: + return None + + return github.get_pull_request(pull_request_number, ci_github_access_token) + + +def check_local_docker_configuration() -> None: + try: + docker_client = docker.from_env() + except Exception as e: + raise click.UsageError(f"Could not connect to docker daemon: {e}") + daemon_info = docker_client.info() + docker_cpus_count = daemon_info["NCPU"] + local_cpus_count = multiprocessing.cpu_count() + if docker_cpus_count < local_cpus_count: + logging.warning( + f"Your docker daemon is configured with less CPUs than your local machine ({docker_cpus_count} vs. {local_cpus_count}). This may slow down the airbyte-ci execution. Please consider increasing the number of CPUs allocated to your docker daemon in the Resource Allocation settings of Docker." + ) + + +def is_dagger_run_enabled_by_default() -> bool: + if CI_REQUIREMENTS_OPTION_NAME in sys.argv: + return False + + dagger_run_by_default = [ + ["connectors", "test"], + ["connectors", "build"], + ["test"], + ["metadata_service"], + ] + + for command_tokens in dagger_run_by_default: + if all(token in sys.argv for token in command_tokens): + return True + + return False + + +def check_dagger_wrap() -> bool: + """ + Check if the command is already wrapped by dagger run. + This is useful to avoid infinite recursion when calling dagger run from dagger run. + """ + return os.getenv(DAGGER_WRAP_ENV_VAR_NAME) == "true" + + +def is_current_process_wrapped_by_dagger_run() -> bool: + """ + Check if the current process is wrapped by dagger run. + """ + called_with_dagger_run = check_dagger_wrap() + main_logger.info(f"Called with dagger run: {called_with_dagger_run}") + return called_with_dagger_run + + +# COMMANDS + + +@click.group( + cls=LazyGroup, + help="Airbyte CI top-level command group.", + lazy_subcommands={ + "connectors": "pipelines.airbyte_ci.connectors.commands.connectors", + "format": "pipelines.airbyte_ci.format.commands.format_code", + "metadata": "pipelines.airbyte_ci.metadata.commands.metadata", + "test": "pipelines.airbyte_ci.test.commands.test", + "update": "pipelines.airbyte_ci.update.commands.update", + }, +) +@click.version_option(__installed_version__) +@pre_confirm_all_flag +@pre_confirm_auto_update_flag +@click.option("--enable-dagger-run/--disable-dagger-run", default=is_dagger_run_enabled_by_default) +@click.option("--enable-update-check/--disable-update-check", default=True) +@click.option("--enable-auto-update/--disable-auto-update", default=True) +@click.option("--is-local/--is-ci", default=True) +@click.option("--git-branch", default=get_current_git_branch, envvar="CI_GIT_BRANCH") +@click.option("--git-revision", default=get_current_git_revision, envvar="CI_GIT_REVISION") +@click.option( + "--diffed-branch", + help="Branch to which the git diff will happen to detect new or modified connectors", + default="origin/master", + type=str, +) +@click.option("--gha-workflow-run-id", help="[CI Only] The run id of the GitHub action workflow", default=None, type=str) +@click.option("--ci-context", default=CIContext.MANUAL, envvar="CI_CONTEXT", type=click.Choice([c for c in CIContext])) +@click.option("--pipeline-start-timestamp", default=get_current_epoch_time, envvar="CI_PIPELINE_START_TIMESTAMP", type=int) +@click.option("--pull-request-number", envvar="PULL_REQUEST_NUMBER", type=int) +@click.option("--ci-git-user", default="octavia-squidington-iii", envvar="CI_GIT_USER", type=str) +@click.option("--ci-github-access-token", envvar="CI_GITHUB_ACCESS_TOKEN", type=str) +@click.option("--ci-report-bucket-name", envvar="CI_REPORT_BUCKET_NAME", type=str) +@click.option("--ci-artifact-bucket-name", envvar="CI_ARTIFACT_BUCKET_NAME", type=str) +@click.option( + "--ci-gcs-credentials", + help="The service account to use during CI.", + type=click.STRING, + required=False, # Not required for pre-release or local pipelines + envvar="GCP_GSM_CREDENTIALS", +) +@click.option("--ci-job-key", envvar="CI_JOB_KEY", type=str) +@click.option("--s3-build-cache-access-key-id", envvar="S3_BUILD_CACHE_ACCESS_KEY_ID", type=str) +@click.option("--s3-build-cache-secret-key", envvar="S3_BUILD_CACHE_SECRET_KEY", type=str) +@click.option("--show-dagger-logs/--hide-dagger-logs", default=False, type=bool) +@click_track_command +@click_merge_args_into_context_obj +@click_append_to_context_object("is_ci", lambda ctx: not ctx.obj["is_local"]) +@click_append_to_context_object("gha_workflow_run_url", _get_gha_workflow_run_url) +@click_append_to_context_object("pull_request", _get_pull_request) +@click.pass_context +@click_ignore_unused_kwargs +async def airbyte_ci(ctx: click.Context) -> None: # noqa D103 + # Check that the command being run is not upgrade + is_update_command = ctx.invoked_subcommand == "update" + if ctx.obj["enable_update_check"] and ctx.obj["is_local"] and not is_update_command: + check_for_upgrade( + require_update=ctx.obj["is_local"], + enable_auto_update=ctx.obj["is_local"] and ctx.obj["enable_auto_update"], + ) + + if ctx.obj["enable_dagger_run"] and not is_current_process_wrapped_by_dagger_run(): + main_logger.debug("Re-Running airbyte-ci with dagger run.") + from pipelines.cli.dagger_run import call_current_command_with_dagger_run + + call_current_command_with_dagger_run() + return + + if ctx.obj["is_local"]: + # This check is meaningful only when running locally + # In our CI the docker host used by the Dagger Engine is different from the one used by the runner. + check_local_docker_configuration() + + if not ctx.obj["is_local"]: + log_context_info(ctx) + + +if __name__ == "__main__": + airbyte_ci() diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/auto_update.py b/airbyte-ci/connectors/pipelines/pipelines/cli/auto_update.py new file mode 100644 index 000000000000..e1ac37ee68d9 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/auto_update.py @@ -0,0 +1,140 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +# HELPERS +from __future__ import annotations + +import importlib +import logging +import os +import sys +from typing import TYPE_CHECKING + +import asyncclick as click +import requests # type: ignore +from pipelines import main_logger +from pipelines.cli.confirm_prompt import confirm +from pipelines.consts import LOCAL_PIPELINE_PACKAGE_PATH +from pipelines.external_scripts.airbyte_ci_install import RELEASE_URL, get_airbyte_os_name + +if TYPE_CHECKING: + from typing import Callable + +__installed_version__ = importlib.metadata.version("pipelines") + +PROD_COMMAND = "airbyte-ci" +DEV_COMMAND = "airbyte-ci-dev" +AUTO_UPDATE_AGREE_KEY = "yes_auto_update" + + +def pre_confirm_auto_update_flag(f: Callable) -> Callable: + """Decorator to add a --yes-auto-update flag to a command.""" + return click.option( + "--yes-auto-update", AUTO_UPDATE_AGREE_KEY, is_flag=True, default=False, help="Skip prompts and automatically upgrade pipelines" + )(f) + + +def _is_version_available(version: str, is_dev: bool) -> bool: + """ + Check if an given version is available. + """ + + # Given that they can install from source, we don't need to check for upgrades + if is_dev: + return True + + os_name = get_airbyte_os_name() + url = f"{RELEASE_URL}/{os_name}/{version}/airbyte-ci" + + # Just check if the URL exists, but dont download it + return requests.head(url).ok + + +def _get_latest_version() -> str: + """ + Get the version of the latest release, which is just in the pyproject.toml file of the pipelines package + as this is an internal tool, we don't need to check for the latest version on PyPI + """ + path_to_pyproject_toml = LOCAL_PIPELINE_PACKAGE_PATH + "pyproject.toml" + with open(path_to_pyproject_toml, "r") as f: + for line in f.readlines(): + if "version" in line: + return line.split("=")[1].strip().replace('"', "") + raise Exception("Could not find version in pyproject.toml. Please ensure you are running from the root of the airbyte repo.") + + +def is_dev_command() -> bool: + """ + Check if the current command is the dev version of the command + """ + current_command = " ".join(sys.argv) + return DEV_COMMAND in current_command + + +def check_for_upgrade( + require_update: bool = True, + enable_auto_update: bool = True, +) -> None: + """Check if the installed version of pipelines is up to date.""" + current_command = " ".join(sys.argv) + latest_version = _get_latest_version() + is_out_of_date = latest_version != __installed_version__ + if not is_out_of_date: + main_logger.info(f"airbyte-ci is up to date. Installed version: {__installed_version__}. Latest version: {latest_version}") + return + + is_dev_version = is_dev_command() + upgrade_available = _is_version_available(latest_version, is_dev_version) + if not upgrade_available: + main_logger.warning( + f"airbyte-ci is out of date, but no upgrade is available yet. This likely means that a release is still being built. Installed version: {__installed_version__}. Latest version: {latest_version}" + ) + return + + parent_command = DEV_COMMAND if is_dev_version else PROD_COMMAND + upgrade_command = f"{parent_command} update" + + # Tack on the specific version if it is not the latest version and it is not the dev version + # This is because the dev version always corresponds to the version in the local repository + if not is_dev_version: + upgrade_command = f"{upgrade_command} --version {latest_version}" + + upgrade_error_message = f""" + 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 + + This version of `airbyte-ci` does not match that of your local airbyte repository. + + Installed Version: {__installed_version__}. + Local Repository Version: {latest_version} + + Please upgrade your local airbyte repository to the latest version using the following command: + $ {upgrade_command} + + Alternatively you can skip this with the `--disable-update-check` flag. + + 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨 + """ + logging.warning(upgrade_error_message) + + # Ask the user if they want to upgrade + if enable_auto_update and confirm( + "Do you want to automatically upgrade?", default=True, additional_pre_confirm_key=AUTO_UPDATE_AGREE_KEY + ): + # if the current command contains `airbyte-ci-dev` is the dev version of the command + logging.info(f"[{'DEV' if is_dev_version else 'BINARY'}] Upgrading pipelines...") + + upgrade_exit_code = os.system(upgrade_command) + if upgrade_exit_code != 0: + raise Exception(f"Failed to upgrade pipelines. Exit code: {upgrade_exit_code}") + + logging.info(f"Re-running command: {current_command}") + + # Re-run the command + command_exit_code = os.system(current_command) + sys.exit(command_exit_code) + + if require_update: + raise Exception(upgrade_error_message) + + return diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py new file mode 100644 index 000000000000..b88f582c6e37 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/click_decorators.py @@ -0,0 +1,152 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import functools +import inspect +from functools import wraps +from typing import Any, Callable, Type, TypeVar + +import asyncclick as click +from pipelines.models.ci_requirements import CIRequirements + +_AnyCallable = Callable[..., Any] +FC = TypeVar("FC", bound="_AnyCallable | click.core.Command") +CI_REQUIREMENTS_OPTION_NAME = "--ci-requirements" + + +def _contains_var_kwarg(f: Callable) -> bool: + return any(param.kind is inspect.Parameter.VAR_KEYWORD for param in inspect.signature(f).parameters.values()) + + +def _is_kwarg_of(key: str, f: Callable) -> bool: + param = inspect.signature(f).parameters.get(key) + if not param: + return False + + return bool(param) and (param.kind is inspect.Parameter.KEYWORD_ONLY or param.kind is inspect.Parameter.POSITIONAL_OR_KEYWORD) + + +def click_ignore_unused_kwargs(f: Callable) -> Callable: + """Make function ignore unmatched kwargs. + + If the function already has the catch all **kwargs, do nothing. + + Useful in the case that the argument is meant to be passed to a child command + and is not used by the parent command + """ + if _contains_var_kwarg(f): + return f + + @functools.wraps(f) + def inner(*args: Any, **kwargs: Any) -> Callable: + filtered_kwargs = {key: value for key, value in kwargs.items() if _is_kwarg_of(key, f)} + return f(*args, **filtered_kwargs) + + return inner + + +def click_merge_args_into_context_obj(f: Callable) -> Callable: + """ + Decorator to pass click context and args to children commands. + """ + + def wrapper(*args: Any, **kwargs: Any) -> Callable: + ctx = click.get_current_context() + ctx.ensure_object(dict) + click_obj = ctx.obj + click_params = ctx.params + command_name = ctx.command.name + + # Error if click_obj and click_params have the same key + intersection = set(click_obj.keys()) & set(click_params.keys()) + if intersection: + raise ValueError(f"Your command '{command_name}' has defined options/arguments with the same key as its parent: {intersection}") + + ctx.obj = {**click_obj, **click_params} + return f(*args, **kwargs) + + return wrapper + + +def click_append_to_context_object(key: str, value: Callable) -> Callable: + """ + Decorator to append a value to the click context object. + """ + + def decorator(f: Callable) -> Callable: + async def wrapper(*args: Any, **kwargs: Any) -> Any: # noqa: ANN401 + ctx = click.get_current_context() + ctx.ensure_object(dict) + + # if async, get the value, cannot use await + if inspect.iscoroutinefunction(value): + ctx.obj[key] = await value(ctx) + elif callable(value): + ctx.obj[key] = value(ctx) + else: + ctx.obj[key] = value + return await f(*args, **kwargs) + + return wrapper + + return decorator + + +class LazyPassDecorator: + """ + Used to create a decorator that will pass an instance of the given class to the decorated function. + """ + + def __init__(self, cls: Type[Any], *args: Any, **kwargs: Any) -> None: + """ + Initialize the decorator with the given source class + """ + self.cls = cls + self.args = args + self.kwargs = kwargs + + def __call__(self, f: Callable[..., Any]) -> Callable[..., Any]: + """ + Create a decorator that will pass an instance of the given class to the decorated function. + """ + + @wraps(f) + def decorated_function(*args: Any, **kwargs: Any) -> Any: # noqa: ANN401 + # Check if the kwargs already contain the arguments being passed by the decorator + decorator_kwargs = {k: v for k, v in self.kwargs.items() if k not in kwargs} + # Create an instance of the class + instance = self.cls(*self.args, **decorator_kwargs) + # If function has **kwargs, we can put the instance there + if "kwargs" in kwargs: + kwargs["kwargs"] = instance + # Otherwise, add it to positional arguments + else: + args = (*args, instance) + return f(*args, **kwargs) + + return decorated_function + + +def click_ci_requirements_option() -> Callable[[FC], FC]: + """Add a --ci-requirements option to the command. + + Returns: + Callable[[FC], FC]: The decorated command. + """ + + def callback(ctx: click.Context, param: click.Parameter, value: bool) -> None: + if value: + ci_requirements = CIRequirements() + click.echo(ci_requirements.to_json()) + ctx.exit() + + return click.decorators.option( + CI_REQUIREMENTS_OPTION_NAME, + is_flag=True, + expose_value=False, + is_eager=True, + flag_value=True, + help="Show the CI requirements and exit. It used to make airbyte-ci client define the CI runners it will run on.", + callback=callback, + ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/confirm_prompt.py b/airbyte-ci/connectors/pipelines/pipelines/cli/confirm_prompt.py new file mode 100644 index 000000000000..b7504cb452f1 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/confirm_prompt.py @@ -0,0 +1,33 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import asyncclick as click + +if TYPE_CHECKING: + from typing import Any, Callable + +PRE_CONFIRM_ALL_KEY = "yes" + + +def pre_confirm_all_flag(f: Callable) -> Callable: + """Decorator to add a --yes flag to a command.""" + return click.option("-y", "--yes", PRE_CONFIRM_ALL_KEY, is_flag=True, default=False, help="Skip prompts and use default values")(f) + + +def confirm(*args: Any, **kwargs: Any) -> bool: + """Confirm a prompt with the user, with support for a --yes flag.""" + additional_pre_confirm_key = kwargs.pop("additional_pre_confirm_key", None) + ctx = click.get_current_context() + if ctx.obj.get(PRE_CONFIRM_ALL_KEY, False): + return True + + if additional_pre_confirm_key: + if ctx.obj.get(additional_pre_confirm_key, False): + return True + + return click.confirm(*args, **kwargs) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_pipeline_command.py b/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_pipeline_command.py new file mode 100644 index 000000000000..96c8dbb89419 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_pipeline_command.py @@ -0,0 +1,104 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This module groups util function used in pipelines.""" +from __future__ import annotations + +import sys +from pathlib import Path + +import asyncclick as click +from dagger import DaggerError +from pipelines import consts, main_logger +from pipelines.consts import GCS_PUBLIC_DOMAIN, STATIC_REPORT_PREFIX +from pipelines.helpers import sentry_utils +from pipelines.helpers.gcs import upload_to_gcs +from pipelines.helpers.utils import slugify + + +class DaggerPipelineCommand(click.Command): + @sentry_utils.with_command_context + async def invoke(self, ctx: click.Context) -> None: + """Wrap parent invoke in a try catch suited to handle pipeline failures. + Args: + ctx (click.Context): The invocation context. + Raises: + e: Raise whatever exception that was caught. + """ + command_name = self.name + main_logger.info(f"Running Dagger Command {command_name}...") + main_logger.info( + "If you're running this command for the first time the Dagger engine image will be pulled, it can take a short minute..." + ) + ctx.obj["report_output_prefix"] = self.render_report_output_prefix(ctx) + dagger_logs_gcs_key = f"{ctx.obj['report_output_prefix']}/dagger-logs.txt" + try: + if not ctx.obj["show_dagger_logs"]: + dagger_log_dir = Path(f"{consts.LOCAL_REPORTS_PATH_ROOT}/{ctx.obj['report_output_prefix']}") + dagger_log_path = Path(f"{dagger_log_dir}/dagger.log").resolve() + ctx.obj["dagger_logs_path"] = dagger_log_path + main_logger.info(f"Saving dagger logs to: {dagger_log_path}") + if ctx.obj["is_ci"]: + ctx.obj["dagger_logs_url"] = f"{GCS_PUBLIC_DOMAIN}/{ctx.obj['ci_report_bucket_name']}/{dagger_logs_gcs_key}" + else: + ctx.obj["dagger_logs_url"] = None + else: + ctx.obj["dagger_logs_path"] = None + pipeline_success = await super().invoke(ctx) + if not pipeline_success: + raise DaggerError(f"Dagger Command {command_name} failed.") + except DaggerError as e: + main_logger.error(f"Dagger Command {command_name} failed", exc_info=e) + sys.exit(1) + finally: + if ctx.obj.get("dagger_logs_path"): + if ctx.obj["is_local"]: + main_logger.info(f"Dagger logs saved to {ctx.obj['dagger_logs_path']}") + if ctx.obj["is_ci"]: + gcs_uri, public_url = upload_to_gcs( + ctx.obj["dagger_logs_path"], ctx.obj["ci_report_bucket_name"], dagger_logs_gcs_key, ctx.obj["ci_gcs_credentials"] + ) + main_logger.info(f"Dagger logs saved to {gcs_uri}. Public URL: {public_url}") + + @staticmethod + def render_report_output_prefix(ctx: click.Context) -> str: + """Render the report output prefix for any command in the Connector CLI. + + The goal is to standardize the output of all logs and reports generated by the CLI + related to a specific command, and to a specific CI context. + + Note: We cannot hoist this higher in the command hierarchy because only one level of + subcommands are available at the time the context is created. + """ + + git_branch = ctx.obj["git_branch"] + git_revision = ctx.obj["git_revision"] + pipeline_start_timestamp = ctx.obj["pipeline_start_timestamp"] + ci_context = ctx.obj["ci_context"] + ci_job_key = ctx.obj["ci_job_key"] if ctx.obj.get("ci_job_key") else ci_context + + sanitized_branch = slugify(git_branch.replace("/", "_")) + + # get the command name for the current context, if a group then prepend the parent command name + if ctx.command_path: + cmd_components = ctx.command_path.split(" ") + cmd_components[0] = STATIC_REPORT_PREFIX + cmd = "/".join(cmd_components) + else: + cmd = None + + path_values = [ + cmd, + ci_job_key, + sanitized_branch, + pipeline_start_timestamp, + git_revision, + ] + + # check all values are defined + if None in path_values: + raise ValueError(f"Missing value required to render the report output prefix: {path_values}") + + # join all values with a slash, and convert all values to string + return "/".join(map(str, path_values)) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py b/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py new file mode 100644 index 000000000000..e290d41c4766 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/dagger_run.py @@ -0,0 +1,126 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This module execute the airbyte-ci-internal CLI wrapped in a dagger run command to use the Dagger Terminal UI.""" + +import logging +import os +import re +import subprocess +import sys +from pathlib import Path +from typing import Optional + +import pkg_resources # type: ignore +import requests # type: ignore +from pipelines.consts import DAGGER_WRAP_ENV_VAR_NAME + +LOGGER = logging.getLogger(__name__) +BIN_DIR = Path.home() / "bin" +BIN_DIR.mkdir(exist_ok=True) +DAGGER_TELEMETRY_TOKEN_ENV_VAR_NAME_VALUE = ( + # The _EXPERIMENTAL_DAGGER_CLOUD_TOKEN is used for telemetry only at the moment. + # It will eventually be renamed to a more specific name in future Dagger versions. + "_EXPERIMENTAL_DAGGER_CLOUD_TOKEN", + "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k", +) + +ARGS_DISABLING_TUI = ["--no-tui", "--version", "publish", "upgrade-base-image", "--help", "format", "bump-version", "migrate-to-base-image"] + + +def get_dagger_path() -> Optional[str]: + try: + return ( + subprocess.run(["which", "dagger"], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.decode("utf-8").strip() + ) + except subprocess.CalledProcessError: + if Path(BIN_DIR / "dagger").exists(): + return str(Path(BIN_DIR / "dagger")) + return None + + +def get_current_dagger_sdk_version() -> str: + version = pkg_resources.get_distribution("dagger-io").version + return version + + +def install_dagger_cli(dagger_version: str) -> None: + install_script_path = "/tmp/install_dagger.sh" + with open(install_script_path, "w") as f: + response = requests.get("https://dl.dagger.io/dagger/install.sh") + response.raise_for_status() + f.write(response.text) + subprocess.run(["chmod", "+x", install_script_path], check=True) + os.environ["BIN_DIR"] = str(BIN_DIR) + os.environ["DAGGER_VERSION"] = dagger_version + subprocess.run([install_script_path], check=True) + + +def get_dagger_cli_version(dagger_path: Optional[str]) -> Optional[str]: + if not dagger_path: + return None + version_output = ( + subprocess.run([dagger_path, "version"], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.decode("utf-8").strip() + ) + version_pattern = r"v(\d+\.\d+\.\d+)" + + match = re.search(version_pattern, version_output) + + if match: + version = match.group(1) + return version + else: + raise Exception("Could not find dagger version in output: " + version_output) + + +def check_dagger_cli_install() -> str: + """ + If the dagger CLI is not installed, install it. + """ + + expected_dagger_cli_version = get_current_dagger_sdk_version() + dagger_path = get_dagger_path() + if dagger_path is None: + LOGGER.info(f"The Dagger CLI is not installed. Installing {expected_dagger_cli_version}...") + install_dagger_cli(expected_dagger_cli_version) + dagger_path = get_dagger_path() + assert dagger_path is not None, "Dagger CLI installation failed, dagger not found in path" + + cli_version = get_dagger_cli_version(dagger_path) + if cli_version != expected_dagger_cli_version: + LOGGER.warning( + f"The Dagger CLI version '{cli_version}' does not match the expected version '{expected_dagger_cli_version}'. Installing Dagger CLI '{expected_dagger_cli_version}'..." + ) + install_dagger_cli(expected_dagger_cli_version) + return check_dagger_cli_install() + return dagger_path + + +def mark_dagger_wrap() -> None: + """ + Mark that the dagger wrap has been applied. + """ + os.environ[DAGGER_WRAP_ENV_VAR_NAME] = "true" + + +def call_current_command_with_dagger_run() -> None: + mark_dagger_wrap() + # We're enabling telemetry only for local runs. + # CI runs already have telemetry as DAGGER_CLOUD_TOKEN env var is set on the CI. + if (os.environ.get("AIRBYTE_ROLE") == "airbyter") and not os.environ.get("CI"): + os.environ[DAGGER_TELEMETRY_TOKEN_ENV_VAR_NAME_VALUE[0]] = DAGGER_TELEMETRY_TOKEN_ENV_VAR_NAME_VALUE[1] + + exit_code = 0 + dagger_path = check_dagger_cli_install() + command = [dagger_path, "run"] + sys.argv + try: + try: + LOGGER.info(f"Running command: {command}") + subprocess.run(command, check=True) + except KeyboardInterrupt: + LOGGER.info("Keyboard interrupt detected. Exiting...") + exit_code = 1 + except subprocess.CalledProcessError as e: + exit_code = e.returncode + sys.exit(exit_code) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/ensure_repo_root.py b/airbyte-ci/connectors/pipelines/pipelines/cli/ensure_repo_root.py new file mode 100644 index 000000000000..5970979d9d71 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/ensure_repo_root.py @@ -0,0 +1,59 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import logging +import os +from pathlib import Path + +import git + + +def _validate_airbyte_repo(repo: git.Repo) -> bool: + """Check if any of the remotes are the airbyte repo.""" + expected_repo_name = "airbytehq/airbyte" + for remote in repo.remotes: + if expected_repo_name in remote.url: + return True + + warning_message = f""" + ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️ + + It looks like you are not running this command from the airbyte repo ({expected_repo_name}). + + If this command is run from outside the airbyte repo, it will not work properly. + + Please run this command your local airbyte project. + + ⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️ + """ + + logging.warning(warning_message) + + return False + + +def get_airbyte_repo() -> git.Repo: + """Get the airbyte repo.""" + repo = git.Repo(search_parent_directories=True) + _validate_airbyte_repo(repo) + return repo + + +def get_airbyte_repo_path_with_fallback() -> Path: + """Get the path to the airbyte repo.""" + try: + repo_path = get_airbyte_repo().working_tree_dir + if repo_path is not None: + return Path(str(get_airbyte_repo().working_tree_dir)) + except git.exc.InvalidGitRepositoryError: + pass + logging.warning("Could not find the airbyte repo, falling back to the current working directory.") + path = Path.cwd() + logging.warning(f"Using {path} as the airbyte repo path.") + return path + + +def set_working_directory_to_root() -> None: + """Set the working directory to the root of the airbyte repo.""" + working_dir = get_airbyte_repo_path_with_fallback() + logging.info(f"Setting working directory to {working_dir}") + os.chdir(working_dir) diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/lazy_group.py b/airbyte-ci/connectors/pipelines/pipelines/cli/lazy_group.py new file mode 100644 index 000000000000..def24edb1b6d --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/lazy_group.py @@ -0,0 +1,46 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +# Source: https://click.palletsprojects.com/en/8.1.x/complex/ + +import importlib +from typing import Any, Dict, List, Optional + +import asyncclick as click + + +class LazyGroup(click.Group): + """ + A click Group that can lazily load subcommands. + """ + + def __init__(self, *args: Any, lazy_subcommands: Optional[Dict[str, str]] = None, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + # lazy_subcommands is a map of the form: + # + # {command-name} -> {module-name}.{command-object-name} + # + self.lazy_subcommands = lazy_subcommands or {} + + def list_commands(self, ctx: click.Context) -> List[str]: + base = super().list_commands(ctx) + lazy = sorted(self.lazy_subcommands.keys()) + return base + lazy + + def get_command(self, ctx: click.Context, cmd_name: str) -> Optional[click.Command]: + if cmd_name in self.lazy_subcommands: + return self._lazy_load(cmd_name) + return super().get_command(ctx, cmd_name) + + def _lazy_load(self, cmd_name: str) -> click.Command: + # lazily loading a command, first get the module name and attribute name + import_path = self.lazy_subcommands[cmd_name] + modname, cmd_object_name = import_path.rsplit(".", 1) + # do the import + mod = importlib.import_module(modname) + # get the Command object from that module + cmd_object = getattr(mod, cmd_object_name) + # check the result to make debugging easier + if not isinstance(cmd_object, click.Command): + print(f"{cmd_object} is of instance {type(cmd_object)}") + raise ValueError(f"Lazy loading of {import_path} failed by returning " "a non-command object") + return cmd_object diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py b/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py new file mode 100644 index 000000000000..a6998724897a --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/telemetry.py @@ -0,0 +1,72 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from __future__ import annotations + +import getpass +import hashlib +import os +import platform +import sys +from typing import TYPE_CHECKING + +import segment.analytics as analytics # type: ignore +from asyncclick import get_current_context + +if TYPE_CHECKING: + from typing import Any, Callable, Dict, Tuple + + from asyncclick import Command + +analytics.write_key = "G6G7whgro81g9xM00kN2buclGKvcOjFd" +analytics.send = True +analytics.debug = False + + +def _is_airbyte_user() -> bool: + """Returns True if the user is airbyter, False otherwise.""" + return os.getenv("AIRBYTE_ROLE") == "airbyter" + + +def _get_anonymous_system_id() -> str: + """Returns a unique anonymous hashid of the current system info.""" + # Collect machine-specific information + machine_info = platform.node() + username = getpass.getuser() + + unique_system_info = f"{machine_info}-{username}" + + # Generate a unique hash + unique_id = hashlib.sha256(unique_system_info.encode()).hexdigest() + + return unique_id + + +def click_track_command(f: Callable) -> Callable: + """ + Decorator to track CLI commands with segment.io + """ + + def wrapper(*args: Tuple, **kwargs: Dict[str, Any]) -> Command: + ctx = get_current_context() + top_level_command = ctx.command_path + full_cmd = " ".join(sys.argv) + + # remove anything prior to the command name f.__name__ + # to avoid logging inline secrets + santized_cmd = full_cmd[full_cmd.find(top_level_command) :] + + sys_id = _get_anonymous_system_id() + sys_user_name = f"anonymous:{sys_id}" + airbyter = _is_airbyte_user() + + is_local = kwargs.get("is_local", False) + user_id = "local-user" if is_local else "ci-user" + event = f"airbyte-ci:{f.__name__}" + + # IMPORTANT! do not log kwargs as they may contain secrets + analytics.track(user_id, event, {"username": sys_user_name, "command": santized_cmd, "airbyter": airbyter}) + + return f(*args, **kwargs) + + return wrapper diff --git a/airbyte-ci/connectors/pipelines/pipelines/commands/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/commands/airbyte_ci.py deleted file mode 100644 index 3ff8a0531c98..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/commands/airbyte_ci.py +++ /dev/null @@ -1,146 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -"""This module is the CLI entrypoint to the airbyte-ci commands.""" - -from typing import List - -import click -from github import PullRequest -from pipelines import github, main_logger -from pipelines.bases import CIContext -from pipelines.utils import ( - get_current_epoch_time, - get_current_git_branch, - get_current_git_revision, - get_modified_files_in_branch, - get_modified_files_in_commit, - get_modified_files_in_pull_request, - transform_strs_to_paths, -) - -from .groups.connectors import connectors -from .groups.metadata import metadata -from .groups.tests import test - -# HELPERS - - -def get_modified_files( - git_branch: str, git_revision: str, diffed_branch: str, is_local: bool, ci_context: CIContext, pull_request: PullRequest -) -> List[str]: - """Get the list of modified files in the current git branch. - If the current branch is master, it will return the list of modified files in the head commit. - The head commit on master should be the merge commit of the latest merged pull request as we squash commits on merge. - Pipelines like "publish on merge" are triggered on each new commit on master. - - If the CI context is a pull request, it will return the list of modified files in the pull request, without using git diff. - If the current branch is not master, it will return the list of modified files in the current branch. - This latest case is the one we encounter when running the pipeline locally, on a local branch, or manually on GHA with a workflow dispatch event. - """ - if ci_context is CIContext.MASTER or ci_context is CIContext.NIGHTLY_BUILDS: - return get_modified_files_in_commit(git_branch, git_revision, is_local) - if ci_context is CIContext.PULL_REQUEST and pull_request is not None: - return get_modified_files_in_pull_request(pull_request) - if ci_context is CIContext.MANUAL: - if git_branch == "master": - return get_modified_files_in_commit(git_branch, git_revision, is_local) - else: - return get_modified_files_in_branch(git_branch, git_revision, diffed_branch, is_local) - return get_modified_files_in_branch(git_branch, git_revision, diffed_branch, is_local) - - -# COMMANDS - - -@click.group(help="Airbyte CI top-level command group.") -@click.option("--is-local/--is-ci", default=True) -@click.option("--git-branch", default=get_current_git_branch, envvar="CI_GIT_BRANCH") -@click.option("--git-revision", default=get_current_git_revision, envvar="CI_GIT_REVISION") -@click.option( - "--diffed-branch", - help="Branch to which the git diff will happen to detect new or modified connectors", - default="origin/master", - type=str, -) -@click.option("--gha-workflow-run-id", help="[CI Only] The run id of the GitHub action workflow", default=None, type=str) -@click.option("--ci-context", default=CIContext.MANUAL, envvar="CI_CONTEXT", type=click.Choice(CIContext)) -@click.option("--pipeline-start-timestamp", default=get_current_epoch_time, envvar="CI_PIPELINE_START_TIMESTAMP", type=int) -@click.option("--pull-request-number", envvar="PULL_REQUEST_NUMBER", type=int) -@click.option("--ci-git-user", default="octavia-squidington-iii", envvar="CI_GIT_USER", type=str) -@click.option("--ci-github-access-token", envvar="CI_GITHUB_ACCESS_TOKEN", type=str) -@click.option("--ci-report-bucket-name", envvar="CI_REPORT_BUCKET_NAME", type=str) -@click.option( - "--ci-gcs-credentials", - help="The service account to use during CI.", - type=click.STRING, - required=False, # Not required for pre-release or local pipelines - envvar="GCP_GSM_CREDENTIALS", -) -@click.option("--ci-job-key", envvar="CI_JOB_KEY", type=str) -@click.option("--show-dagger-logs/--hide-dagger-logs", default=False, type=bool) -@click.pass_context -def airbyte_ci( - ctx: click.Context, - is_local: bool, - git_branch: str, - git_revision: str, - diffed_branch: str, - gha_workflow_run_id: str, - ci_context: str, - pipeline_start_timestamp: int, - pull_request_number: int, - ci_git_user: str, - ci_github_access_token: str, - ci_report_bucket_name: str, - ci_gcs_credentials: str, - ci_job_key: str, - show_dagger_logs: bool, -): # noqa D103 - ctx.ensure_object(dict) - ctx.obj["is_local"] = is_local - ctx.obj["is_ci"] = not is_local - ctx.obj["git_branch"] = git_branch - ctx.obj["git_revision"] = git_revision - ctx.obj["gha_workflow_run_id"] = gha_workflow_run_id - ctx.obj["gha_workflow_run_url"] = ( - f"https://github.com/airbytehq/airbyte/actions/runs/{gha_workflow_run_id}" if gha_workflow_run_id else None - ) - ctx.obj["ci_context"] = ci_context - ctx.obj["ci_report_bucket_name"] = ci_report_bucket_name - ctx.obj["ci_gcs_credentials"] = ci_gcs_credentials - ctx.obj["ci_git_user"] = ci_git_user - ctx.obj["ci_github_access_token"] = ci_github_access_token - ctx.obj["ci_job_key"] = ci_job_key - ctx.obj["pipeline_start_timestamp"] = pipeline_start_timestamp - ctx.obj["show_dagger_logs"] = show_dagger_logs - - if pull_request_number and ci_github_access_token: - ctx.obj["pull_request"] = github.get_pull_request(pull_request_number, ci_github_access_token) - else: - ctx.obj["pull_request"] = None - - ctx.obj["modified_files"] = transform_strs_to_paths( - get_modified_files(git_branch, git_revision, diffed_branch, is_local, ci_context, ctx.obj["pull_request"]) - ) - - if not is_local: - main_logger.info("Running airbyte-ci in CI mode.") - main_logger.info(f"CI Context: {ci_context}") - main_logger.info(f"CI Report Bucket Name: {ci_report_bucket_name}") - main_logger.info(f"Git Branch: {git_branch}") - main_logger.info(f"Git Revision: {git_revision}") - main_logger.info(f"GitHub Workflow Run ID: {gha_workflow_run_id}") - main_logger.info(f"GitHub Workflow Run URL: {ctx.obj['gha_workflow_run_url']}") - main_logger.info(f"Pull Request Number: {pull_request_number}") - main_logger.info(f"Pipeline Start Timestamp: {pipeline_start_timestamp}") - main_logger.info(f"Modified Files: {ctx.obj['modified_files']}") - - -airbyte_ci.add_command(connectors) -airbyte_ci.add_command(metadata) -airbyte_ci.add_command(test) - -if __name__ == "__main__": - airbyte_ci() diff --git a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py deleted file mode 100644 index 1fe82c244cff..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/connectors.py +++ /dev/null @@ -1,505 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -"""This module declares the CLI commands to run the connectors CI pipelines.""" - -import os -import sys -from pathlib import Path -from typing import List, Set, Tuple - -import anyio -import click -from connector_ops.utils import ConnectorLanguage, SupportLevelEnum, console, get_all_connectors_in_repo -from pipelines import main_logger -from pipelines.bases import ConnectorWithModifiedFiles -from pipelines.builds import run_connector_build_pipeline -from pipelines.contexts import ConnectorContext, ContextState, PublishConnectorContext -from pipelines.format import run_connectors_format_pipelines -from pipelines.github import update_global_commit_status_check_for_tests -from pipelines.pipelines.connectors import run_connectors_pipelines -from pipelines.publish import reorder_contexts, run_connector_publish_pipeline -from pipelines.tests import run_connector_test_pipeline -from pipelines.utils import DaggerPipelineCommand, get_connector_modified_files, get_modified_connectors - -# HELPERS - -ALL_CONNECTORS = get_all_connectors_in_repo() - - -def validate_environment(is_local: bool, use_remote_secrets: bool): - """Check if the required environment variables exist.""" - if is_local: - if not (os.getcwd().endswith("/airbyte") and Path(".git").is_dir()): - raise click.UsageError("You need to run this command from the airbyte repository root.") - else: - required_env_vars_for_ci = [ - "GCP_GSM_CREDENTIALS", - "CI_REPORT_BUCKET_NAME", - "CI_GITHUB_ACCESS_TOKEN", - ] - for required_env_var in required_env_vars_for_ci: - if os.getenv(required_env_var) is None: - raise click.UsageError(f"When running in a CI context a {required_env_var} environment variable must be set.") - if use_remote_secrets and os.getenv("GCP_GSM_CREDENTIALS") is None: - raise click.UsageError( - "You have to set the GCP_GSM_CREDENTIALS if you want to download secrets from GSM. Set the --use-remote-secrets option to false otherwise." - ) - - -def get_selected_connectors_with_modified_files( - selected_names: Tuple[str], - selected_support_levels: Tuple[str], - selected_languages: Tuple[str], - modified: bool, - metadata_changes_only: bool, - modified_files: Set[Path], - enable_dependency_scanning: bool = False, -) -> List[ConnectorWithModifiedFiles]: - """Get the connectors that match the selected criteria. - - Args: - selected_names (Tuple[str]): Selected connector names. - selected_support_levels (Tuple[str]): Selected connector support levels. - selected_languages (Tuple[str]): Selected connector languages. - modified (bool): Whether to select the modified connectors. - metadata_changes_only (bool): Whether to select only the connectors with metadata changes. - modified_files (Set[Path]): The modified files. - enable_dependency_scanning (bool): Whether to enable the dependency scanning. - Returns: - List[ConnectorWithModifiedFiles]: The connectors that match the selected criteria. - """ - - if metadata_changes_only and not modified: - main_logger.info("--metadata-changes-only overrides --modified") - modified = True - - selected_modified_connectors = ( - get_modified_connectors(modified_files, ALL_CONNECTORS, enable_dependency_scanning) if modified else set() - ) - selected_connectors_by_name = {c for c in ALL_CONNECTORS if c.technical_name in selected_names} - selected_connectors_by_support_level = {connector for connector in ALL_CONNECTORS if connector.support_level in selected_support_levels} - selected_connectors_by_language = {connector for connector in ALL_CONNECTORS if connector.language in selected_languages} - non_empty_connector_sets = [ - connector_set - for connector_set in [ - selected_connectors_by_name, - selected_connectors_by_support_level, - selected_connectors_by_language, - selected_modified_connectors, - ] - if connector_set - ] - # The selected connectors are the intersection of the selected connectors by name, support_level, language and modified. - selected_connectors = set.intersection(*non_empty_connector_sets) if non_empty_connector_sets else set() - - selected_connectors_with_modified_files = [] - for connector in selected_connectors: - connector_with_modified_files = ConnectorWithModifiedFiles( - technical_name=connector.technical_name, modified_files=get_connector_modified_files(connector, modified_files) - ) - if not metadata_changes_only: - selected_connectors_with_modified_files.append(connector_with_modified_files) - else: - if connector_with_modified_files.has_metadata_change: - selected_connectors_with_modified_files.append(connector_with_modified_files) - return selected_connectors_with_modified_files - - -# COMMANDS - - -@click.group(help="Commands related to connectors and connector acceptance tests.") -@click.option("--use-remote-secrets", default=True) # specific to connectors -@click.option( - "--name", - "names", - multiple=True, - help="Only test a specific connector. Use its technical name. e.g source-pokeapi.", - type=click.Choice([c.technical_name for c in ALL_CONNECTORS]), -) -@click.option("--language", "languages", multiple=True, help="Filter connectors to test by language.", type=click.Choice(ConnectorLanguage)) -@click.option( - "--support-level", - "support_levels", - multiple=True, - help="Filter connectors to test by support_level.", - type=click.Choice(SupportLevelEnum), -) -@click.option("--modified/--not-modified", help="Only test modified connectors in the current branch.", default=False, type=bool) -@click.option( - "--metadata-changes-only/--not-metadata-changes-only", - help="Only test connectors with modified metadata files in the current branch.", - default=False, - type=bool, -) -@click.option("--concurrency", help="Number of connector tests pipeline to run in parallel.", default=5, type=int) -@click.option( - "--execute-timeout", - help="The maximum time in seconds for the execution of a Dagger request before an ExecuteTimeoutError is raised. Passing None results in waiting forever.", - default=None, - type=int, -) -@click.option( - "--enable-dependency-scanning/--disable-dependency-scanning", - help="When enabled, the dependency scanning will be performed to detect the connectors to test according to a dependency change.", - default=False, - type=bool, -) -@click.pass_context -def connectors( - ctx: click.Context, - use_remote_secrets: bool, - names: Tuple[str], - languages: Tuple[ConnectorLanguage], - support_levels: Tuple[str], - modified: bool, - metadata_changes_only: bool, - concurrency: int, - execute_timeout: int, - enable_dependency_scanning: bool, -): - """Group all the connectors-ci command.""" - validate_environment(ctx.obj["is_local"], use_remote_secrets) - - ctx.ensure_object(dict) - ctx.obj["use_remote_secrets"] = use_remote_secrets - ctx.obj["concurrency"] = concurrency - ctx.obj["execute_timeout"] = execute_timeout - ctx.obj["selected_connectors_with_modified_files"] = get_selected_connectors_with_modified_files( - names, support_levels, languages, modified, metadata_changes_only, ctx.obj["modified_files"], enable_dependency_scanning - ) - log_selected_connectors(ctx.obj["selected_connectors_with_modified_files"]) - - -@connectors.command(cls=DaggerPipelineCommand, help="Test all the selected connectors.") -@click.option( - "--code-tests-only", - is_flag=True, - help=("Only execute code tests. " "Metadata checks, QA, and acceptance tests will be skipped."), - default=False, - type=bool, -) -@click.option( - "--fail-fast", - help="When enabled, tests will fail fast.", - default=False, - type=bool, - is_flag=True, -) -@click.option( - "--fast-tests-only", - help="When enabled, slow tests are skipped.", - default=False, - type=bool, - is_flag=True, -) -@click.pass_context -def test( - ctx: click.Context, - code_tests_only: bool, - fail_fast: bool, - fast_tests_only: bool, -) -> bool: - """Runs a test pipeline for the selected connectors. - - Args: - ctx (click.Context): The click context. - """ - if ctx.obj["is_ci"] and ctx.obj["pull_request"] and ctx.obj["pull_request"].draft: - main_logger.info("Skipping connectors tests for draft pull request.") - sys.exit(0) - - if ctx.obj["selected_connectors_with_modified_files"]: - update_global_commit_status_check_for_tests(ctx.obj, "pending") - else: - main_logger.warn("No connector were selected for testing.") - update_global_commit_status_check_for_tests(ctx.obj, "success") - return True - - connectors_tests_contexts = [ - ConnectorContext( - pipeline_name=f"Testing connector {connector.technical_name}", - connector=connector, - is_local=ctx.obj["is_local"], - git_branch=ctx.obj["git_branch"], - git_revision=ctx.obj["git_revision"], - ci_report_bucket=ctx.obj["ci_report_bucket_name"], - report_output_prefix=ctx.obj["report_output_prefix"], - use_remote_secrets=ctx.obj["use_remote_secrets"], - gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), - dagger_logs_url=ctx.obj.get("dagger_logs_url"), - pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), - ci_context=ctx.obj.get("ci_context"), - pull_request=ctx.obj.get("pull_request"), - ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], - fail_fast=fail_fast, - fast_tests_only=fast_tests_only, - code_tests_only=code_tests_only, - ) - for connector in ctx.obj["selected_connectors_with_modified_files"] - ] - try: - anyio.run( - run_connectors_pipelines, - [connector_context for connector_context in connectors_tests_contexts], - run_connector_test_pipeline, - "Test Pipeline", - ctx.obj["concurrency"], - ctx.obj["dagger_logs_path"], - ctx.obj["execute_timeout"], - ) - except Exception as e: - main_logger.error("An error occurred while running the test pipeline", exc_info=e) - update_global_commit_status_check_for_tests(ctx.obj, "failure") - return False - - @ctx.call_on_close - def send_commit_status_check() -> None: - if ctx.obj["is_ci"]: - global_success = all(connector_context.state is ContextState.SUCCESSFUL for connector_context in connectors_tests_contexts) - update_global_commit_status_check_for_tests(ctx.obj, "success" if global_success else "failure") - - # If we reach this point, it means that all the connectors have been tested so the pipeline did its job and can exit with success. - return True - - -@connectors.command(cls=DaggerPipelineCommand, help="Build all images for the selected connectors.") -@click.pass_context -def build(ctx: click.Context) -> bool: - """Runs a build pipeline for the selected connectors.""" - - connectors_contexts = [ - ConnectorContext( - pipeline_name=f"Build connector {connector.technical_name}", - connector=connector, - is_local=ctx.obj["is_local"], - git_branch=ctx.obj["git_branch"], - git_revision=ctx.obj["git_revision"], - ci_report_bucket=ctx.obj["ci_report_bucket_name"], - report_output_prefix=ctx.obj["report_output_prefix"], - use_remote_secrets=ctx.obj["use_remote_secrets"], - gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), - dagger_logs_url=ctx.obj.get("dagger_logs_url"), - pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), - ci_context=ctx.obj.get("ci_context"), - ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], - ) - for connector in ctx.obj["selected_connectors_with_modified_files"] - ] - anyio.run( - run_connectors_pipelines, - connectors_contexts, - run_connector_build_pipeline, - "Build Pipeline", - ctx.obj["concurrency"], - ctx.obj["dagger_logs_path"], - ctx.obj["execute_timeout"], - ) - - return True - - -@connectors.command(cls=DaggerPipelineCommand, help="Publish all images for the selected connectors.") -@click.option("--pre-release/--main-release", help="Use this flag if you want to publish pre-release images.", default=True, type=bool) -@click.option( - "--spec-cache-gcs-credentials", - help="The service account key to upload files to the GCS bucket hosting spec cache.", - type=click.STRING, - required=True, - envvar="SPEC_CACHE_GCS_CREDENTIALS", -) -@click.option( - "--spec-cache-bucket-name", - help="The name of the GCS bucket where specs will be cached.", - type=click.STRING, - required=True, - envvar="SPEC_CACHE_BUCKET_NAME", -) -@click.option( - "--metadata-service-gcs-credentials", - help="The service account key to upload files to the GCS bucket hosting the metadata files.", - type=click.STRING, - required=True, - envvar="METADATA_SERVICE_GCS_CREDENTIALS", -) -@click.option( - "--metadata-service-bucket-name", - help="The name of the GCS bucket where metadata files will be uploaded.", - type=click.STRING, - required=True, - envvar="METADATA_SERVICE_BUCKET_NAME", -) -@click.option( - "--docker-hub-username", - help="Your username to connect to DockerHub.", - type=click.STRING, - required=True, - envvar="DOCKER_HUB_USERNAME", -) -@click.option( - "--docker-hub-password", - help="Your password to connect to DockerHub.", - type=click.STRING, - required=True, - envvar="DOCKER_HUB_PASSWORD", -) -@click.option( - "--slack-webhook", - help="The Slack webhook URL to send notifications to.", - type=click.STRING, - envvar="SLACK_WEBHOOK", -) -@click.option( - "--slack-channel", - help="The Slack webhook URL to send notifications to.", - type=click.STRING, - envvar="SLACK_CHANNEL", - default="#publish-on-merge-updates", -) -@click.pass_context -def publish( - ctx: click.Context, - pre_release: bool, - spec_cache_gcs_credentials: str, - spec_cache_bucket_name: str, - metadata_service_bucket_name: str, - metadata_service_gcs_credentials: str, - docker_hub_username: str, - docker_hub_password: str, - slack_webhook: str, - slack_channel: str, -): - ctx.obj["spec_cache_gcs_credentials"] = spec_cache_gcs_credentials - ctx.obj["spec_cache_bucket_name"] = spec_cache_bucket_name - ctx.obj["metadata_service_bucket_name"] = metadata_service_bucket_name - ctx.obj["metadata_service_gcs_credentials"] = metadata_service_gcs_credentials - if ctx.obj["is_local"]: - click.confirm( - "Publishing from a local environment is not recommended and requires to be logged in Airbyte's DockerHub registry, do you want to continue?", - abort=True, - ) - - publish_connector_contexts = reorder_contexts( - [ - PublishConnectorContext( - connector=connector, - pre_release=pre_release, - spec_cache_gcs_credentials=spec_cache_gcs_credentials, - spec_cache_bucket_name=spec_cache_bucket_name, - metadata_service_gcs_credentials=metadata_service_gcs_credentials, - metadata_bucket_name=metadata_service_bucket_name, - docker_hub_username=docker_hub_username, - docker_hub_password=docker_hub_password, - slack_webhook=slack_webhook, - reporting_slack_channel=slack_channel, - ci_report_bucket=ctx.obj["ci_report_bucket_name"], - report_output_prefix=ctx.obj["report_output_prefix"], - is_local=ctx.obj["is_local"], - git_branch=ctx.obj["git_branch"], - git_revision=ctx.obj["git_revision"], - gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), - dagger_logs_url=ctx.obj.get("dagger_logs_url"), - pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), - ci_context=ctx.obj.get("ci_context"), - ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], - pull_request=ctx.obj.get("pull_request"), - ) - for connector in ctx.obj["selected_connectors_with_modified_files"] - ] - ) - - main_logger.warn("Concurrency is forced to 1. For stability reasons we disable parallel publish pipelines.") - ctx.obj["concurrency"] = 1 - - publish_connector_contexts = anyio.run( - run_connectors_pipelines, - publish_connector_contexts, - run_connector_publish_pipeline, - "Publishing connectors", - ctx.obj["concurrency"], - ctx.obj["dagger_logs_path"], - ctx.obj["execute_timeout"], - ) - return all(context.state is ContextState.SUCCESSFUL for context in publish_connector_contexts) - - -@connectors.command(cls=DaggerPipelineCommand, help="List all selected connectors.") -@click.pass_context -def list( - ctx: click.Context, -): - selected_connectors = sorted(ctx.obj["selected_connectors_with_modified_files"], key=lambda x: x.technical_name) - table = Table(title=f"{len(selected_connectors)} selected connectors") - table.add_column("Modified") - table.add_column("Connector") - table.add_column("Language") - table.add_column("Release stage") - table.add_column("Version") - table.add_column("Folder") - - for connector in selected_connectors: - modified = "X" if connector.modified_files else "" - connector_name = Text(connector.technical_name) - language = Text(connector.language.value) if connector.language else "N/A" - try: - support_level = Text(connector.support_level) - except Exception: - support_level = "N/A" - try: - version = Text(connector.version) - except Exception: - version = "N/A" - folder = Text(str(connector.code_directory)) - table.add_row(modified, connector_name, language, support_level, version, folder) - - console.print(table) - return True - - -@connectors.command(name="format", cls=DaggerPipelineCommand, help="Autoformat connector code.") -@click.pass_context -def format_code(ctx: click.Context) -> bool: - connectors_contexts = [ - ConnectorContext( - pipeline_name=f"Format connector {connector.technical_name}", - connector=connector, - is_local=ctx.obj["is_local"], - git_branch=ctx.obj["git_branch"], - git_revision=ctx.obj["git_revision"], - ci_report_bucket=ctx.obj["ci_report_bucket_name"], - report_output_prefix=ctx.obj["report_output_prefix"], - use_remote_secrets=ctx.obj["use_remote_secrets"], - gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), - dagger_logs_url=ctx.obj.get("dagger_logs_url"), - pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), - ci_context=ctx.obj.get("ci_context"), - ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], - ci_git_user=ctx.obj["ci_git_user"], - ci_github_access_token=ctx.obj["ci_github_access_token"], - pull_request=ctx.obj.get("pull_request"), - should_save_report=False, - ) - for connector in ctx.obj["selected_connectors_with_modified_files"] - ] - - anyio.run( - run_connectors_format_pipelines, - connectors_contexts, - ctx.obj["ci_git_user"], - ctx.obj["ci_github_access_token"], - ctx.obj["git_branch"], - ctx.obj["is_local"], - ctx.obj["execute_timeout"], - ) - - return True - - -def log_selected_connectors(selected_connectors_with_modified_files: List[ConnectorWithModifiedFiles]) -> None: - if selected_connectors_with_modified_files: - selected_connectors_names = [c.technical_name for c in selected_connectors_with_modified_files] - main_logger.info(f"Will run on the following connectors: {', '.join(selected_connectors_names)}.") - else: - main_logger.info("No connectors to run.") diff --git a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/metadata.py b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/metadata.py deleted file mode 100644 index a9988db89fe2..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/metadata.py +++ /dev/null @@ -1,148 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import anyio -import click -from pipelines.bases import CIContext -from pipelines.pipelines.metadata import ( - run_metadata_lib_test_pipeline, - run_metadata_orchestrator_deploy_pipeline, - run_metadata_orchestrator_test_pipeline, - run_metadata_upload_pipeline, - run_metadata_validation_pipeline, -) -from pipelines.utils import DaggerPipelineCommand, get_all_metadata_files, get_expected_metadata_files, get_modified_metadata_files - -# MAIN GROUP - - -@click.group(help="Commands related to the metadata service.") -@click.pass_context -def metadata(ctx: click.Context): - pass - - -# VALIDATE COMMAND - - -@metadata.command(cls=DaggerPipelineCommand, help="Commands related to validating the metadata files.") -@click.option("--modified-only/--all", default=True) -@click.pass_context -def validate(ctx: click.Context, modified_only: bool) -> bool: - if modified_only: - metadata_to_validate = get_expected_metadata_files(ctx.obj["modified_files"]) - else: - click.secho("Will run metadata validation on all the metadata files found in the repo.") - metadata_to_validate = get_all_metadata_files() - - click.secho(f"Will validate {len(metadata_to_validate)} metadata files.") - - return anyio.run( - run_metadata_validation_pipeline, - ctx.obj["is_local"], - ctx.obj["git_branch"], - ctx.obj["git_revision"], - ctx.obj.get("gha_workflow_run_url"), - ctx.obj.get("pipeline_start_timestamp"), - ctx.obj.get("ci_context"), - metadata_to_validate, - ) - - -# UPLOAD COMMAND - - -@metadata.command(cls=DaggerPipelineCommand, help="Commands related to uploading the metadata files to remote storage.") -@click.argument("gcs-bucket-name", type=click.STRING) -@click.option("--modified-only/--all", default=True) -@click.pass_context -def upload(ctx: click.Context, gcs_bucket_name: str, modified_only: bool) -> bool: - if modified_only: - if ctx.obj["ci_context"] is not CIContext.MASTER and ctx.obj["git_branch"] != "master": - click.secho("Not on the master branch. Skipping metadata upload.") - return True - metadata_to_upload = get_modified_metadata_files(ctx.obj["modified_files"]) - if not metadata_to_upload: - click.secho("No modified metadata found. Skipping metadata upload.") - return True - else: - metadata_to_upload = get_all_metadata_files() - - click.secho(f"Will upload {len(metadata_to_upload)} metadata files.") - - return anyio.run( - run_metadata_upload_pipeline, - ctx.obj["is_local"], - ctx.obj["git_branch"], - ctx.obj["git_revision"], - ctx.obj.get("gha_workflow_run_url"), - ctx.obj.get("dagger_logs_url"), - ctx.obj.get("pipeline_start_timestamp"), - ctx.obj.get("ci_context"), - metadata_to_upload, - gcs_bucket_name, - ) - - -# DEPLOY GROUP - - -@metadata.group(help="Commands related to deploying components of the metadata service.") -@click.pass_context -def deploy(ctx: click.Context): - pass - - -@deploy.command(cls=DaggerPipelineCommand, name="orchestrator", help="Deploy the metadata service orchestrator to production") -@click.pass_context -def deploy_orchestrator(ctx: click.Context) -> bool: - return anyio.run( - run_metadata_orchestrator_deploy_pipeline, - ctx.obj["is_local"], - ctx.obj["git_branch"], - ctx.obj["git_revision"], - ctx.obj.get("gha_workflow_run_url"), - ctx.obj.get("dagger_logs_url"), - ctx.obj.get("pipeline_start_timestamp"), - ctx.obj.get("ci_context"), - ) - - -# TEST GROUP - - -@metadata.group(help="Commands related to testing the metadata service.") -@click.pass_context -def test(ctx: click.Context): - pass - - -@test.command(cls=DaggerPipelineCommand, name="lib", help="Run tests for the metadata service library.") -@click.pass_context -def test_lib(ctx: click.Context) -> bool: - return anyio.run( - run_metadata_lib_test_pipeline, - ctx.obj["is_local"], - ctx.obj["git_branch"], - ctx.obj["git_revision"], - ctx.obj.get("gha_workflow_run_url"), - ctx.obj.get("dagger_logs_url"), - ctx.obj.get("pipeline_start_timestamp"), - ctx.obj.get("ci_context"), - ) - - -@test.command(cls=DaggerPipelineCommand, name="orchestrator", help="Run tests for the metadata service orchestrator.") -@click.pass_context -def test_orchestrator(ctx: click.Context) -> bool: - return anyio.run( - run_metadata_orchestrator_test_pipeline, - ctx.obj["is_local"], - ctx.obj["git_branch"], - ctx.obj["git_revision"], - ctx.obj.get("gha_workflow_run_url"), - ctx.obj.get("dagger_logs_url"), - ctx.obj.get("pipeline_start_timestamp"), - ctx.obj.get("ci_context"), - ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/tests.py b/airbyte-ci/connectors/pipelines/pipelines/commands/groups/tests.py deleted file mode 100644 index c9cd4248a170..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/commands/groups/tests.py +++ /dev/null @@ -1,87 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -""" -Module exposing the tests command to test airbyte-ci projects. -""" - -import logging -import os -import sys - -import anyio -import click -import dagger - - -@click.command() -@click.argument("poetry_package_path") -@click.option("--test-directory", default="tests", help="The directory containing the tests to run.") -def test( - poetry_package_path: str, - test_directory: str, -): - """Runs the tests for the given airbyte-ci package. - - Args: - poetry_package_path (str): Path to the poetry package to test, relative to airbyte-ci directory. - test_directory (str): The directory containing the tests to run. - """ - success = anyio.run(run_test, poetry_package_path, test_directory) - if not success: - click.Abort() - - -async def run_test(poetry_package_path: str, test_directory: str) -> bool: - """Runs the tests for the given airbyte-ci package in a Dagger container. - - Args: - airbyte_ci_package_path (str): Path to the airbyte-ci package to test, relative to airbyte-ci directory. - Returns: - bool: True if the tests passed, False otherwise. - """ - logger = logging.getLogger(f"{poetry_package_path}.tests") - logger.info(f"Running tests for {poetry_package_path}") - # The following directories are always mounted because a lot of tests rely on them - directories_to_always_mount = [".git", "airbyte-integrations", "airbyte-ci"] - directories_to_mount = list(set([poetry_package_path, *directories_to_always_mount])) - async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as dagger_client: - try: - docker_host_socket = dagger_client.host().unix_socket("/var/run/buildkit/buildkitd.sock") - pytest_container = await ( - dagger_client.container() - .from_("python:3.10.12") - .with_exec(["apt-get", "update"]) - .with_exec(["apt-get", "install", "-y", "bash", "git", "curl"]) - .with_env_variable("VERSION", "24.0.2") - .with_exec(["sh", "-c", "curl -fsSL https://get.docker.com | sh"]) - .with_exec(["pip", "install", "pipx"]) - .with_exec(["pipx", "ensurepath"]) - .with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") - .with_exec(["pipx", "install", "poetry"]) - .with_mounted_directory( - "/airbyte", - dagger_client.host().directory( - ".", - exclude=["**/__pycache__", "**/.pytest_cache", "**/.venv", "**.log", "**/build", "**/.gradle"], - include=directories_to_mount, - ), - ) - .with_workdir(f"/airbyte/{poetry_package_path}") - .with_exec(["poetry", "install"]) - .with_unix_socket("/var/run/docker.sock", dagger_client.host().unix_socket("/var/run/docker.sock")) - .with_exec(["poetry", "run", "pytest", test_directory]) - ) - if "_EXPERIMENTAL_DAGGER_RUNNER_HOST" in os.environ: - logger.info("Using experimental dagger runner host to run CAT with dagger-in-dagger") - pytest_container = pytest_container.with_env_variable( - "_EXPERIMENTAL_DAGGER_RUNNER_HOST", "unix:///var/run/buildkit/buildkitd.sock" - ).with_unix_socket("/var/run/buildkit/buildkitd.sock", docker_host_socket) - - await pytest_container - return True - except dagger.ExecError as e: - logger.error("Tests failed") - logger.error(e.stderr) - sys.exit(1) diff --git a/airbyte-ci/connectors/pipelines/pipelines/consts.py b/airbyte-ci/connectors/pipelines/pipelines/consts.py index 445479449ddc..851578cc7a0c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/consts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/consts.py @@ -2,8 +2,9 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import os import platform -from pathlib import Path +from enum import Enum from dagger import Platform @@ -12,25 +13,80 @@ CONNECTOR_TESTING_REQUIREMENTS = [ "pip==21.3.1", "mccabe==0.6.1", - "flake8==4.0.1", - "pyproject-flake8==0.0.1a2", - "black==22.3.0", - "isort==5.6.4", + # "flake8==4.0.1", + # "pyproject-flake8==0.0.1a2", "pytest==6.2.5", "coverage[toml]==6.3.1", "pytest-custom_exit_code", - "licenseheaders==0.8.8", ] -CI_CREDENTIALS_SOURCE_PATH = "airbyte-ci/connectors/ci_credentials" -CONNECTOR_OPS_SOURCE_PATHSOURCE_PATH = "airbyte-ci/connectors/connector_ops" -BUILD_PLATFORMS = [Platform("linux/amd64"), Platform("linux/arm64")] -LOCAL_BUILD_PLATFORM = Platform(f"linux/{platform.machine()}") +BUILD_PLATFORMS = (Platform("linux/amd64"), Platform("linux/arm64")) + +PLATFORM_MACHINE_TO_DAGGER_PLATFORM = { + "x86_64": Platform("linux/amd64"), + "arm64": Platform("linux/arm64"), + "aarch64": Platform("linux/amd64"), + "amd64": Platform("linux/amd64"), +} +LOCAL_MACHINE_TYPE = platform.machine() +LOCAL_BUILD_PLATFORM = PLATFORM_MACHINE_TO_DAGGER_PLATFORM[LOCAL_MACHINE_TYPE] +AMAZONCORRETTO_IMAGE = "amazoncorretto:17.0.8-al2023" +NODE_IMAGE = "node:18.18.0-slim" +GO_IMAGE = "golang:1.17" +PYTHON_3_10_IMAGE = "python:3.10.13-slim" +MAVEN_IMAGE = "maven:3.9.5-amazoncorretto-17-al2023" DOCKER_VERSION = "24.0.2" -DOCKER_DIND_IMAGE = "docker:24-dind" -DOCKER_CLI_IMAGE = "docker:24-cli" +DOCKER_DIND_IMAGE = f"docker:{DOCKER_VERSION}-dind" +DOCKER_CLI_IMAGE = f"docker:{DOCKER_VERSION}-cli" +DOCKER_REGISTRY_MIRROR_URL = os.getenv("DOCKER_REGISTRY_MIRROR_URL") +DOCKER_REGISTRY_ADDRESS = "docker.io" +DOCKER_VAR_LIB_VOLUME_NAME = "docker-cache" +GIT_IMAGE = "alpine/git:latest" GRADLE_CACHE_PATH = "/root/.gradle/caches" GRADLE_BUILD_CACHE_PATH = f"{GRADLE_CACHE_PATH}/build-cache-1" GRADLE_READ_ONLY_DEPENDENCY_CACHE_PATH = "/root/gradle_dependency_cache" LOCAL_REPORTS_PATH_ROOT = "airbyte-ci/connectors/pipelines/pipeline_reports/" +LOCAL_PIPELINE_PACKAGE_PATH = "airbyte-ci/connectors/pipelines/" +DOCS_DIRECTORY_ROOT_PATH = "docs/" GCS_PUBLIC_DOMAIN = "https://storage.cloud.google.com" +DOCKER_HOST_NAME = "global-docker-host" +DOCKER_HOST_PORT = 2375 +DOCKER_TMP_VOLUME_NAME = "shared-tmp" +DOCKER_VAR_LIB_VOLUME_NAME = "docker-cache" +STATIC_REPORT_PREFIX = "airbyte-ci" +PIP_CACHE_VOLUME_NAME = "pip_cache" +PIP_CACHE_PATH = "/root/.cache/pip" +POETRY_CACHE_VOLUME_NAME = "poetry_cache" +POETRY_CACHE_PATH = "/root/.cache/pypoetry" +STORAGE_DRIVER = "fuse-overlayfs" +TAILSCALE_AUTH_KEY = os.getenv("TAILSCALE_AUTH_KEY") + + +class CIContext(str, Enum): + """An enum for Ci context values which can be ["manual", "pull_request", "nightly_builds"].""" + + MANUAL = "manual" + PULL_REQUEST = "pull_request" + MASTER = "master" + + def __str__(self) -> str: + return self.value + + +class ContextState(Enum): + """Enum to characterize the current context state, values are used for external representation on GitHub commit checks.""" + + INITIALIZED = {"github_state": "pending", "description": "Pipelines are being initialized..."} + RUNNING = {"github_state": "pending", "description": "Pipelines are running..."} + ERROR = {"github_state": "error", "description": "Something went wrong while running the Pipelines."} + SUCCESSFUL = {"github_state": "success", "description": "All Pipelines ran successfully."} + FAILURE = {"github_state": "failure", "description": "Pipeline failed."} + + +class INTERNAL_TOOL_PATHS(str, Enum): + CI_CREDENTIALS = "airbyte-ci/connectors/ci_credentials" + CONNECTOR_OPS = "airbyte-ci/connectors/connector_ops" + METADATA_SERVICE = "airbyte-ci/connectors/metadata_service/lib" + + +DAGGER_WRAP_ENV_VAR_NAME = "_DAGGER_WRAP_APPLIED" diff --git a/airbyte-ci/connectors/pipelines/pipelines/contexts.py b/airbyte-ci/connectors/pipelines/pipelines/contexts.py deleted file mode 100644 index 897006696f90..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/contexts.py +++ /dev/null @@ -1,592 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -"""Module declaring context related classes.""" - -import logging -import os -from datetime import datetime -from enum import Enum -from glob import glob -from types import TracebackType -from typing import List, Optional - -import yaml -from anyio import Path -from asyncer import asyncify -from dagger import Client, Directory, Secret -from github import PullRequest -from pipelines import hacks -from pipelines.actions import secrets -from pipelines.bases import CIContext, ConnectorReport, ConnectorWithModifiedFiles, Report -from pipelines.github import update_commit_status_check -from pipelines.slack import send_message_to_webhook -from pipelines.utils import AIRBYTE_REPO_URL, METADATA_FILE_NAME, format_duration, sanitize_gcs_credentials - - -class ContextState(Enum): - """Enum to characterize the current context state, values are used for external representation on GitHub commit checks.""" - - INITIALIZED = {"github_state": "pending", "description": "Pipelines are being initialized..."} - RUNNING = {"github_state": "pending", "description": "Pipelines are running..."} - ERROR = {"github_state": "error", "description": "Something went wrong while running the Pipelines."} - SUCCESSFUL = {"github_state": "success", "description": "All Pipelines ran successfully."} - FAILURE = {"github_state": "failure", "description": "Pipeline failed."} - - -class PipelineContext: - """The pipeline context is used to store configuration for a specific pipeline run.""" - - PRODUCTION = bool(os.environ.get("PRODUCTION", False)) # Set this to True to enable production mode (e.g. to send PR comments) - - DEFAULT_EXCLUDED_FILES = ( - [".git", "airbyte-ci/connectors/pipelines/*"] - + glob("**/build", recursive=True) - + glob("**/.venv", recursive=True) - + glob("**/secrets", recursive=True) - + glob("**/__pycache__", recursive=True) - + glob("**/*.egg-info", recursive=True) - + glob("**/.vscode", recursive=True) - + glob("**/.pytest_cache", recursive=True) - + glob("**/.eggs", recursive=True) - + glob("**/.mypy_cache", recursive=True) - + glob("**/.DS_Store", recursive=True) - + glob("**/airbyte_ci_logs", recursive=True) - ) - - def __init__( - self, - pipeline_name: str, - is_local: bool, - git_branch: str, - git_revision: str, - gha_workflow_run_url: Optional[str] = None, - dagger_logs_url: Optional[str] = None, - pipeline_start_timestamp: Optional[int] = None, - ci_context: Optional[str] = None, - is_ci_optional: bool = False, - slack_webhook: Optional[str] = None, - reporting_slack_channel: Optional[str] = None, - pull_request: PullRequest = None, - ci_report_bucket: Optional[str] = None, - ci_gcs_credentials: Optional[str] = None, - ci_git_user: Optional[str] = None, - ci_github_access_token: Optional[str] = None, - ): - """Initialize a pipeline context. - - Args: - pipeline_name (str): The pipeline name. - is_local (bool): Whether the context is for a local run or a CI run. - git_branch (str): The current git branch name. - git_revision (str): The current git revision, commit hash. - gha_workflow_run_url (Optional[str], optional): URL to the github action workflow run. Only valid for CI run. Defaults to None. - dagger_logs_url (Optional[str], optional): URL to the dagger logs. Only valid for CI run. Defaults to None. - pipeline_start_timestamp (Optional[int], optional): Timestamp at which the pipeline started. Defaults to None. - ci_context (Optional[str], optional): Pull requests, workflow dispatch or nightly build. Defaults to None. - is_ci_optional (bool, optional): Whether the CI is optional. Defaults to False. - slack_webhook (Optional[str], optional): Slack webhook to send messages to. Defaults to None. - reporting_slack_channel (Optional[str], optional): Slack channel to send messages to. Defaults to None. - pull_request (PullRequest, optional): The pull request object if the pipeline was triggered by a pull request. Defaults to None. - """ - self.pipeline_name = pipeline_name - self.is_local = is_local - self.git_branch = git_branch - self.git_revision = git_revision - self.gha_workflow_run_url = gha_workflow_run_url - self.dagger_logs_url = dagger_logs_url - self.pipeline_start_timestamp = pipeline_start_timestamp - self.created_at = datetime.utcnow() - self.ci_context = ci_context - self.state = ContextState.INITIALIZED - self.is_ci_optional = is_ci_optional - self.slack_webhook = slack_webhook - self.reporting_slack_channel = reporting_slack_channel - self.pull_request = pull_request - self.logger = logging.getLogger(self.pipeline_name) - self.dagger_client = None - self._report = None - self.dockerd_service = None - self.ci_gcs_credentials = sanitize_gcs_credentials(ci_gcs_credentials) if ci_gcs_credentials else None - self.ci_report_bucket = ci_report_bucket - self.ci_git_user = ci_git_user - self.ci_github_access_token = ci_github_access_token - self.started_at = None - self.stopped_at = None - self.secrets_to_mask = [] - update_commit_status_check(**self.github_commit_status) - - @property - def dagger_client(self) -> Client: # noqa D102 - return self._dagger_client - - @dagger_client.setter - def dagger_client(self, dagger_client: Client): # noqa D102 - self._dagger_client = dagger_client - - @property - def is_ci(self): # noqa D102 - return self.is_local is False - - @property - def is_pr(self): # noqa D102 - return self.ci_context == CIContext.PULL_REQUEST - - @property - def repo(self): # noqa D102 - return self.dagger_client.git(AIRBYTE_REPO_URL, keep_git_dir=True) - - @property - def report(self) -> Report: # noqa D102 - return self._report - - @report.setter - def report(self, report: Report): # noqa D102 - self._report = report - - @property - def ci_gcs_credentials_secret(self) -> Secret: - return self.dagger_client.set_secret("ci_gcs_credentials", self.ci_gcs_credentials) - - @property - def ci_github_access_token_secret(self) -> Secret: - return self.dagger_client.set_secret("ci_github_access_token", self.ci_github_access_token) - - @property - def github_commit_status(self) -> dict: - """Build a dictionary used as kwargs to the update_commit_status_check function.""" - return { - "sha": self.git_revision, - "state": self.state.value["github_state"], - "target_url": self.gha_workflow_run_url, - "description": self.state.value["description"], - "context": self.pipeline_name, - "should_send": self.is_pr, - "logger": self.logger, - "is_optional": self.is_ci_optional, - } - - @property - def should_send_slack_message(self) -> bool: - return self.slack_webhook is not None and self.reporting_slack_channel is not None - - @property - def has_dagger_cloud_token(self) -> bool: - return "_EXPERIMENTAL_DAGGER_CLOUD_TOKEN" in os.environ - - @property - def dagger_cloud_url(self) -> str: - """Gets the link to the Dagger Cloud runs page for the current commit.""" - if self.is_local or not self.has_dagger_cloud_token: - return None - - return f"https://alpha.dagger.cloud/changeByPipelines?filter=dagger.io/git.ref:{self.git_revision}" - - def get_repo_dir(self, subdir: str = ".", exclude: Optional[List[str]] = None, include: Optional[List[str]] = None) -> Directory: - """Get a directory from the current repository. - - The directory is extracted from the host file system. - A couple of files or directories that could corrupt builds are exclude by default (check DEFAULT_EXCLUDED_FILES). - - Args: - subdir (str, optional): Path to the subdirectory to get. Defaults to "." to get the full repository. - exclude ([List[str], optional): List of files or directories to exclude from the directory. Defaults to None. - include ([List[str], optional): List of files or directories to include in the directory. Defaults to None. - - Returns: - Directory: The selected repo directory. - """ - if exclude is None: - exclude = self.DEFAULT_EXCLUDED_FILES - else: - exclude += self.DEFAULT_EXCLUDED_FILES - exclude = list(set(exclude)) - exclude.sort() # sort to make sure the order is always the same to not burst the cache. Casting exclude to set can change the order - if subdir != ".": - subdir = f"{subdir}/" if not subdir.endswith("/") else subdir - exclude = [f.replace(subdir, "") for f in exclude if subdir in f] - return self.dagger_client.host().directory(subdir, exclude=exclude, include=include) - - def create_slack_message(self) -> str: - raise NotImplementedError() - - async def __aenter__(self): - """Perform setup operation for the PipelineContext. - - Updates the current commit status on Github. - - Raises: - Exception: An error is raised when the context was not initialized with a Dagger client - Returns: - PipelineContext: A running instance of the PipelineContext. - """ - if self.dagger_client is None: - raise Exception("A Pipeline can't be entered with an undefined dagger_client") - self.state = ContextState.RUNNING - self.started_at = datetime.utcnow() - self.logger.info("Caching the latest CDK version...") - await hacks.cache_latest_cdk(self.dagger_client) - await asyncify(update_commit_status_check)(**self.github_commit_status) - if self.should_send_slack_message: - await asyncify(send_message_to_webhook)(self.create_slack_message(), self.reporting_slack_channel, self.slack_webhook) - return self - - @staticmethod - def determine_final_state(report: Optional[Report], exception_value: Optional[BaseException]) -> ContextState: - """Determine the final state of the context from the report or the exception value. - - Args: - report (Optional[Report]): The pipeline report if any. - exception_value (Optional[BaseException]): The exception value if an exception was raised in the context execution, None otherwise. - Returns: - ContextState: The final state of the context. - """ - if exception_value is not None or report is None: - return ContextState.ERROR - if report is not None and report.failed_steps: - return ContextState.FAILURE - if report is not None and report.success: - return ContextState.SUCCESSFUL - raise Exception( - f"The final state of the context could not be determined for the report and exception value provided. Report: {report}, Exception: {exception_value}" - ) - - async def __aexit__( - self, exception_type: Optional[type[BaseException]], exception_value: Optional[BaseException], traceback: Optional[TracebackType] - ) -> bool: - """Perform teardown operation for the PipelineContext. - - On the context exit the following operations will happen: - - Log the error value if an error was handled. - - Log the test report. - - Update the commit status check on GitHub if running in a CI environment. - - It should gracefully handle all the execution errors that happened and always upload a test report and update commit status check. - - Args: - exception_type (Optional[type[BaseException]]): The exception type if an exception was raised in the context execution, None otherwise. - exception_value (Optional[BaseException]): The exception value if an exception was raised in the context execution, None otherwise. - traceback (Optional[TracebackType]): The traceback if an exception was raised in the context execution, None otherwise. - Returns: - bool: Whether the teardown operation ran successfully. - """ - self.state = self.determine_final_state(self.report, exception_value) - self.stopped_at = datetime.utcnow() - - if exception_value: - self.logger.error("An error was handled by the Pipeline", exc_info=True) - if self.report is None: - self.logger.error("No test report was provided. This is probably due to an upstream error") - self.report = Report(self, steps_results=[]) - - self.report.print() - - await asyncify(update_commit_status_check)(**self.github_commit_status) - if self.should_send_slack_message: - await asyncify(send_message_to_webhook)(self.create_slack_message(), self.reporting_slack_channel, self.slack_webhook) - # supress the exception if it was handled - return True - - -class ConnectorContext(PipelineContext): - """The connector context is used to store configuration for a specific connector pipeline run.""" - - DEFAULT_CONNECTOR_ACCEPTANCE_TEST_IMAGE = "airbyte/connector-acceptance-test:dev" - - def __init__( - self, - pipeline_name: str, - connector: ConnectorWithModifiedFiles, - is_local: bool, - git_branch: bool, - git_revision: bool, - report_output_prefix: str, - use_remote_secrets: bool = True, - ci_report_bucket: Optional[str] = None, - ci_gcs_credentials: Optional[str] = None, - ci_git_user: Optional[str] = None, - ci_github_access_token: Optional[str] = None, - connector_acceptance_test_image: Optional[str] = DEFAULT_CONNECTOR_ACCEPTANCE_TEST_IMAGE, - gha_workflow_run_url: Optional[str] = None, - dagger_logs_url: Optional[str] = None, - pipeline_start_timestamp: Optional[int] = None, - ci_context: Optional[str] = None, - slack_webhook: Optional[str] = None, - reporting_slack_channel: Optional[str] = None, - pull_request: PullRequest = None, - should_save_report: bool = True, - fail_fast: bool = False, - fast_tests_only: bool = False, - code_tests_only: bool = False, - ): - """Initialize a connector context. - - Args: - connector (Connector): The connector under test. - is_local (bool): Whether the context is for a local run or a CI run. - git_branch (str): The current git branch name. - git_revision (str): The current git revision, commit hash. - report_output_prefix (str): The S3 key to upload the test report to. - use_remote_secrets (bool, optional): Whether to download secrets for GSM or use the local secrets. Defaults to True. - connector_acceptance_test_image (Optional[str], optional): The image to use to run connector acceptance tests. Defaults to DEFAULT_CONNECTOR_ACCEPTANCE_TEST_IMAGE. - gha_workflow_run_url (Optional[str], optional): URL to the github action workflow run. Only valid for CI run. Defaults to None. - dagger_logs_url (Optional[str], optional): URL to the dagger logs. Only valid for CI run. Defaults to None. - pipeline_start_timestamp (Optional[int], optional): Timestamp at which the pipeline started. Defaults to None. - ci_context (Optional[str], optional): Pull requests, workflow dispatch or nightly build. Defaults to None. - slack_webhook (Optional[str], optional): The slack webhook to send messages to. Defaults to None. - reporting_slack_channel (Optional[str], optional): The slack channel to send messages to. Defaults to None. - pull_request (PullRequest, optional): The pull request object if the pipeline was triggered by a pull request. Defaults to None. - fail_fast (bool, optional): Whether to fail fast. Defaults to False. - fast_tests_only (bool, optional): Whether to run only fast tests. Defaults to False. - code_tests_only (bool, optional): Whether to ignore non-code tests like QA and metadata checks. Defaults to False. - """ - - self.pipeline_name = pipeline_name - self.connector = connector - self.use_remote_secrets = use_remote_secrets - self.connector_acceptance_test_image = connector_acceptance_test_image - self.report_output_prefix = report_output_prefix - self._secrets_dir = None - self._updated_secrets_dir = None - self.cdk_version = None - self.should_save_report = should_save_report - self.fail_fast = fail_fast - self.fast_tests_only = fast_tests_only - self.code_tests_only = code_tests_only - - super().__init__( - pipeline_name=pipeline_name, - is_local=is_local, - git_branch=git_branch, - git_revision=git_revision, - gha_workflow_run_url=gha_workflow_run_url, - dagger_logs_url=dagger_logs_url, - pipeline_start_timestamp=pipeline_start_timestamp, - ci_context=ci_context, - slack_webhook=slack_webhook, - reporting_slack_channel=reporting_slack_channel, - pull_request=pull_request, - ci_report_bucket=ci_report_bucket, - ci_gcs_credentials=ci_gcs_credentials, - ci_git_user=ci_git_user, - ci_github_access_token=ci_github_access_token, - ) - - @property - def modified_files(self): - return self.connector.modified_files - - @property - def secrets_dir(self) -> Directory: # noqa D102 - return self._secrets_dir - - @secrets_dir.setter - def secrets_dir(self, secrets_dir: Directory): # noqa D102 - self._secrets_dir = secrets_dir - - @property - def updated_secrets_dir(self) -> Directory: # noqa D102 - return self._updated_secrets_dir - - @updated_secrets_dir.setter - def updated_secrets_dir(self, updated_secrets_dir: Directory): # noqa D102 - self._updated_secrets_dir = updated_secrets_dir - - @property - def connector_acceptance_test_source_dir(self) -> Directory: # noqa D102 - return self.get_repo_dir("airbyte-integrations/bases/connector-acceptance-test") - - @property - def should_save_updated_secrets(self) -> bool: # noqa D102 - return self.use_remote_secrets and self.updated_secrets_dir is not None - - @property - def host_image_export_dir_path(self) -> str: - return "." if self.is_ci else "/tmp" - - @property - def metadata_path(self) -> Path: - return self.connector.code_directory / METADATA_FILE_NAME - - @property - def metadata(self) -> dict: - return yaml.safe_load(self.metadata_path.read_text())["data"] - - @property - def docker_repository(self) -> str: - return self.metadata["dockerRepository"] - - @property - def docker_image_tag(self) -> str: - return self.metadata["dockerImageTag"] - - @property - def docker_image(self) -> str: - return f"{self.docker_repository}:{self.docker_image_tag}" - - async def get_connector_dir(self, exclude=None, include=None) -> Directory: - """Get the connector under test source code directory. - - Args: - exclude ([List[str], optional): List of files or directories to exclude from the directory. Defaults to None. - include ([List[str], optional): List of files or directories to include in the directory. Defaults to None. - - Returns: - Directory: The connector under test source code directory. - """ - vanilla_connector_dir = self.get_repo_dir(str(self.connector.code_directory), exclude=exclude, include=include) - return await hacks.patch_connector_dir(self, vanilla_connector_dir) - - async def __aexit__( - self, exception_type: Optional[type[BaseException]], exception_value: Optional[BaseException], traceback: Optional[TracebackType] - ) -> bool: - """Perform teardown operation for the ConnectorContext. - - On the context exit the following operations will happen: - - Upload updated connector secrets back to Google Secret Manager - - Write a test report in JSON format locally and to S3 if running in a CI environment - - Update the commit status check on GitHub if running in a CI environment. - It should gracefully handle the execution error that happens and always upload a test report and update commit status check. - Args: - exception_type (Optional[type[BaseException]]): The exception type if an exception was raised in the context execution, None otherwise. - exception_value (Optional[BaseException]): The exception value if an exception was raised in the context execution, None otherwise. - traceback (Optional[TracebackType]): The traceback if an exception was raised in the context execution, None otherwise. - Returns: - bool: Whether the teardown operation ran successfully. - """ - self.stopped_at = datetime.utcnow() - self.state = self.determine_final_state(self.report, exception_value) - if exception_value: - self.logger.error("An error got handled by the ConnectorContext", exc_info=True) - if self.report is None: - self.logger.error("No test report was provided. This is probably due to an upstream error") - self.report = ConnectorReport(self, []) - - if self.should_save_updated_secrets: - await secrets.upload(self) - - self.report.print() - - if self.should_save_report: - await self.report.save() - - if self.report.should_be_commented_on_pr: - self.report.post_comment_on_pr() - - await asyncify(update_commit_status_check)(**self.github_commit_status) - - if self.should_send_slack_message: - await asyncify(send_message_to_webhook)(self.create_slack_message(), self.reporting_slack_channel, self.slack_webhook) - - # Supress the exception if any - return True - - def create_slack_message(self) -> str: - raise NotImplementedError - - -class PublishConnectorContext(ConnectorContext): - def __init__( - self, - connector: ConnectorWithModifiedFiles, - pre_release: bool, - spec_cache_gcs_credentials: str, - spec_cache_bucket_name: str, - metadata_service_gcs_credentials: str, - metadata_bucket_name: str, - docker_hub_username: str, - docker_hub_password: str, - slack_webhook: str, - reporting_slack_channel: str, - ci_report_bucket: str, - report_output_prefix: str, - is_local: bool, - git_branch: bool, - git_revision: bool, - gha_workflow_run_url: Optional[str] = None, - dagger_logs_url: Optional[str] = None, - pipeline_start_timestamp: Optional[int] = None, - ci_context: Optional[str] = None, - ci_gcs_credentials: str = None, - pull_request: PullRequest = None, - ): - self.pre_release = pre_release - self.spec_cache_bucket_name = spec_cache_bucket_name - self.metadata_bucket_name = metadata_bucket_name - self.spec_cache_gcs_credentials = sanitize_gcs_credentials(spec_cache_gcs_credentials) - self.metadata_service_gcs_credentials = sanitize_gcs_credentials(metadata_service_gcs_credentials) - self.docker_hub_username = docker_hub_username - self.docker_hub_password = docker_hub_password - - pipeline_name = f"Publish {connector.technical_name}" - pipeline_name = pipeline_name + " (pre-release)" if pre_release else pipeline_name - - super().__init__( - pipeline_name=pipeline_name, - connector=connector, - report_output_prefix=report_output_prefix, - ci_report_bucket=ci_report_bucket, - is_local=is_local, - git_branch=git_branch, - git_revision=git_revision, - gha_workflow_run_url=gha_workflow_run_url, - dagger_logs_url=dagger_logs_url, - pipeline_start_timestamp=pipeline_start_timestamp, - ci_context=ci_context, - slack_webhook=slack_webhook, - reporting_slack_channel=reporting_slack_channel, - ci_gcs_credentials=ci_gcs_credentials, - should_save_report=True, - ) - - @property - def docker_hub_username_secret(self) -> Secret: - return self.dagger_client.set_secret("docker_hub_username", self.docker_hub_username) - - @property - def docker_hub_password_secret(self) -> Secret: - return self.dagger_client.set_secret("docker_hub_password", self.docker_hub_password) - - @property - def metadata_service_gcs_credentials_secret(self) -> Secret: - return self.dagger_client.set_secret("metadata_service_gcs_credentials", self.metadata_service_gcs_credentials) - - @property - def spec_cache_gcs_credentials_secret(self) -> Secret: - return self.dagger_client.set_secret("spec_cache_gcs_credentials", self.spec_cache_gcs_credentials) - - @property - def docker_image_tag(self): - # get the docker image tag from the parent class - metadata_tag = super().docker_image_tag - if self.pre_release: - return f"{metadata_tag}-dev.{self.git_revision[:10]}" - else: - return metadata_tag - - def create_slack_message(self) -> str: - docker_hub_url = f"https://hub.docker.com/r/{self.connector.metadata['dockerRepository']}/tags" - message = f"*Publish <{docker_hub_url}|{self.docker_image}>*\n" - if self.is_ci: - message += f"🤖 <{self.gha_workflow_run_url}|GitHub Action workflow>\n" - else: - message += "🧑‍💻 Local run\n" - message += f"*Connector:* {self.connector.technical_name}\n" - message += f"*Version:* {self.connector.version}\n" - branch_url = f"https://github.com/airbytehq/airbyte/tree/{self.git_branch}" - message += f"*Branch:* <{branch_url}|{self.git_branch}>\n" - commit_url = f"https://github.com/airbytehq/airbyte/commit/{self.git_revision}" - message += f"*Commit:* <{commit_url}|{self.git_revision[:10]}>\n" - if self.state in [ContextState.INITIALIZED, ContextState.RUNNING]: - message += "🟠" - if self.state is ContextState.SUCCESSFUL: - message += "🟢" - if self.state in [ContextState.FAILURE, ContextState.ERROR]: - message += "🔴" - message += f" {self.state.value['description']}\n" - if self.state is ContextState.SUCCESSFUL: - message += f"⏲️ Run duration: {format_duration(self.report.run_duration)}\n" - if self.state is ContextState.FAILURE: - message += "\ncc. " # @dev-connector-ops - return message diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/hooks.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/hooks.py new file mode 100644 index 000000000000..cf4abfaec5e7 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/hooks.py @@ -0,0 +1,63 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import importlib.util +from importlib import metadata +from importlib.abc import Loader + +from dagger import Container +from pipelines.airbyte_ci.connectors.context import ConnectorContext + + +def get_dagger_sdk_version() -> str: + try: + return metadata.version("dagger-io") + except metadata.PackageNotFoundError: + return "n/a" + + +async def finalize_build(context: ConnectorContext, connector_container: Container) -> Container: + """Finalize build by adding dagger engine version label and running finalize_build.sh or finalize_build.py if present in the connector directory.""" + connector_container = connector_container.with_label("io.dagger.engine_version", get_dagger_sdk_version()) + connector_dir_with_finalize_script = await context.get_connector_dir(include=["finalize_build.sh", "finalize_build.py"]) + finalize_scripts = await connector_dir_with_finalize_script.entries() + if not finalize_scripts: + return connector_container + + # We don't want finalize scripts to override the entrypoint so we keep it in memory to reset it after finalization + original_entrypoint = await connector_container.entrypoint() + if not original_entrypoint: + original_entrypoint = [] + + has_finalize_bash_script = "finalize_build.sh" in finalize_scripts + has_finalize_python_script = "finalize_build.py" in finalize_scripts + if has_finalize_python_script and has_finalize_bash_script: + raise Exception("Connector has both finalize_build.sh and finalize_build.py, please remove one of them") + + if has_finalize_python_script: + context.logger.info(f"{context.connector.technical_name} has a finalize_build.py script, running it to finalize build...") + module_path = context.connector.code_directory / "finalize_build.py" + connector_finalize_module_spec = importlib.util.spec_from_file_location( + f"{context.connector.code_directory.name}_finalize", module_path + ) + if connector_finalize_module_spec is None: + raise Exception("Connector has a finalize_build.py script but it can't be loaded.") + connector_finalize_module = importlib.util.module_from_spec(connector_finalize_module_spec) + if not isinstance(connector_finalize_module_spec.loader, Loader): + raise Exception("Connector has a finalize_build.py script but it can't be loaded.") + connector_finalize_module_spec.loader.exec_module(connector_finalize_module) + try: + connector_container = await connector_finalize_module.finalize_build(context, connector_container) + except AttributeError: + raise Exception("Connector has a finalize_build.py script but it doesn't have a finalize_build function.") + + if has_finalize_bash_script: + context.logger.info(f"{context.connector.technical_name} has finalize_build.sh script, running it to finalize build...") + connector_container = ( + connector_container.with_file("/tmp/finalize_build.sh", connector_dir_with_finalize_script.file("finalize_build.sh")) + .with_entrypoint("sh") + .with_exec(["/tmp/finalize_build.sh"]) + ) + + return connector_container.with_entrypoint(original_entrypoint) diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/normalization.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/normalization.py new file mode 100644 index 000000000000..9fe2806b7e27 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/connector/normalization.py @@ -0,0 +1,77 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Dict + +from dagger import Container, Platform +from pipelines.airbyte_ci.connectors.context import ConnectorContext + +BASE_DESTINATION_NORMALIZATION_BUILD_CONFIGURATION = { + "destination-clickhouse": { + "dockerfile": "clickhouse.Dockerfile", + "dbt_adapter": "dbt-clickhouse>=1.4.0", + "integration_name": "clickhouse", + "normalization_image": "airbyte/normalization-clickhouse:0.4.3", + "supports_in_connector_normalization": False, + "yum_packages": [], + }, + "destination-mssql": { + "dockerfile": "mssql.Dockerfile", + "dbt_adapter": "dbt-sqlserver==1.0.0", + "integration_name": "mssql", + "normalization_image": "airbyte/normalization-mssql:0.4.3", + "supports_in_connector_normalization": True, + "yum_packages": [], + }, + "destination-mysql": { + "dockerfile": "mysql.Dockerfile", + "dbt_adapter": "dbt-mysql==1.0.0", + "integration_name": "mysql", + "normalization_image": "airbyte/normalization-mysql:0.4.3", + "supports_in_connector_normalization": False, + "yum_packages": [], + }, + "destination-oracle": { + "dockerfile": "oracle.Dockerfile", + "dbt_adapter": "dbt-oracle==0.4.3", + "integration_name": "oracle", + "normalization_image": "airbyte/normalization-oracle:0.4.3", + "supports_in_connector_normalization": False, + "yum_packages": [], + }, + "destination-postgres": { + "dockerfile": "Dockerfile", + "dbt_adapter": "dbt-postgres==1.0.0", + "integration_name": "postgres", + "normalization_image": "airbyte/normalization:0.4.3", + "supports_in_connector_normalization": True, + "yum_packages": [], + }, + "destination-redshift": { + "dockerfile": "redshift.Dockerfile", + "dbt_adapter": "dbt-redshift==1.0.0", + "integration_name": "redshift", + "normalization_image": "airbyte/normalization-redshift:0.4.3", + "supports_in_connector_normalization": True, + "yum_packages": [], + }, + "destination-tidb": { + "dockerfile": "tidb.Dockerfile", + "dbt_adapter": "dbt-tidb==1.0.1", + "integration_name": "tidb", + "normalization_image": "airbyte/normalization-tidb:0.4.3", + "supports_in_connector_normalization": True, + "yum_packages": [], + }, +} +DESTINATION_NORMALIZATION_BUILD_CONFIGURATION: Dict[str, Dict[str, Any]] = { + **BASE_DESTINATION_NORMALIZATION_BUILD_CONFIGURATION, + **{f"{k}-strict-encrypt": v for k, v in BASE_DESTINATION_NORMALIZATION_BUILD_CONFIGURATION.items()}, +} + + +def with_normalization(context: ConnectorContext, build_platform: Platform) -> Container: + normalization_image_name = DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["normalization_image"] + assert isinstance(normalization_image_name, str) + return context.dagger_client.container(platform=build_platform).from_(normalization_image_name) diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py new file mode 100644 index 000000000000..741de252ae67 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/common.py @@ -0,0 +1,283 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import re +from pathlib import Path +from typing import List, Optional, Sequence + +from dagger import Container, Directory +from pipelines import hacks +from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext +from pipelines.dagger.containers.python import with_pip_cache, with_poetry_cache, with_python_base, with_testing_dependencies +from pipelines.helpers.utils import check_path_in_workdir, get_file_contents + + +def with_python_package( + context: PipelineContext, + python_environment: Container, + package_source_code_path: str, + exclude: Optional[List] = None, + include: Optional[List] = None, +) -> Container: + """Load a python package source code to a python environment container. + + Args: + context (PipelineContext): The current test context, providing the repository directory from which the python sources will be pulled. + python_environment (Container): An existing python environment in which the package will be installed. + package_source_code_path (str): The local path to the package source code. + additional_dependency_groups (Optional[List]): extra_requires dependency of setup.py to install. Defaults to None. + exclude (Optional[List]): A list of file or directory to exclude from the python package source code. + + Returns: + Container: A python environment container with the python package source code. + """ + package_source_code_directory: Directory = context.get_repo_dir(package_source_code_path, exclude=exclude, include=include) + work_dir_path = f"/{package_source_code_path}" + container = python_environment.with_mounted_directory(work_dir_path, package_source_code_directory).with_workdir(work_dir_path) + return container + + +async def find_local_dependencies_in_setup_py(python_package: Container) -> List[str]: + """Find local dependencies of a python package in its setup.py file. + + Args: + python_package (Container): A python package container. + + Returns: + List[str]: Paths to the local dependencies relative to the airbyte repo. + """ + setup_file_content = await get_file_contents(python_package, "setup.py") + if not setup_file_content: + return [] + + local_setup_dependency_paths = [] + with_egg_info = python_package.with_exec(["python", "setup.py", "egg_info"]) + egg_info_output = await with_egg_info.stdout() + dependency_in_requires_txt = [] + for line in egg_info_output.split("\n"): + if line.startswith("writing requirements to"): + # Find the path to the requirements.txt file that was generated by calling egg_info + requires_txt_path = line.replace("writing requirements to", "").strip() + requirements_txt_content = await with_egg_info.file(requires_txt_path).contents() + dependency_in_requires_txt = requirements_txt_content.split("\n") + + for dependency_line in dependency_in_requires_txt: + if "file://" in dependency_line: + match = re.search(r"file:///(.+)", dependency_line) + if match: + local_setup_dependency_paths.append([match.group(1)][0]) + return local_setup_dependency_paths + + +async def find_local_dependencies_in_requirements_txt(python_package: Container, package_source_code_path: str) -> List[str]: + """Find local dependencies of a python package in a requirements.txt file. + + Args: + python_package (Container): A python environment container with the python package source code. + package_source_code_path (str): The local path to the python package source code. + + Returns: + List[str]: Paths to the local dependencies relative to the airbyte repo. + """ + requirements_txt_content = await get_file_contents(python_package, "requirements.txt") + if not requirements_txt_content: + return [] + + local_requirements_dependency_paths = [] + for line in requirements_txt_content.split("\n"): + # Some package declare themselves as a requirement in requirements.txt, + # #Without line != "-e ." the package will be considered a dependency of itself which can cause an infinite loop + if line.startswith("-e .") and line != "-e .": + local_dependency_path = str((Path(package_source_code_path) / Path(line[3:])).resolve().relative_to(Path.cwd())) + local_requirements_dependency_paths.append(local_dependency_path) + return local_requirements_dependency_paths + + +async def find_local_python_dependencies( + context: PipelineContext, + package_source_code_path: str, + search_dependencies_in_setup_py: bool = True, + search_dependencies_in_requirements_txt: bool = True, +) -> List[str]: + """Find local python dependencies of a python package. The dependencies are found in the setup.py and requirements.txt files. + + Args: + context (PipelineContext): The current pipeline context, providing a dagger client and a repository directory. + package_source_code_path (str): The local path to the python package source code. + search_dependencies_in_setup_py (bool, optional): Whether to search for local dependencies in the setup.py file. Defaults to True. + search_dependencies_in_requirements_txt (bool, optional): Whether to search for local dependencies in the requirements.txt file. Defaults to True. + + Returns: + List[str]: Paths to the local dependencies relative to the airbyte repo. + """ + python_environment = with_python_base(context) + container = with_python_package(context, python_environment, package_source_code_path) + + local_dependency_paths = [] + if search_dependencies_in_setup_py: + local_dependency_paths += await find_local_dependencies_in_setup_py(container) + if search_dependencies_in_requirements_txt: + local_dependency_paths += await find_local_dependencies_in_requirements_txt(container, package_source_code_path) + + transitive_dependency_paths = [] + for local_dependency_path in local_dependency_paths: + # Transitive local dependencies installation is achieved by calling their setup.py file, not their requirements.txt file. + transitive_dependency_paths += await find_local_python_dependencies(context, local_dependency_path, True, False) + + all_dependency_paths = local_dependency_paths + transitive_dependency_paths + if all_dependency_paths: + context.logger.debug(f"Found local dependencies for {package_source_code_path}: {all_dependency_paths}") + return all_dependency_paths + + +def _install_python_dependencies_from_setup_py( + container: Container, + additional_dependency_groups: Optional[Sequence[str]] = None, +) -> Container: + install_connector_package_cmd = ["pip", "install", "."] + container = container.with_exec(install_connector_package_cmd) + + if additional_dependency_groups: + # e.g. .[dev,tests] + group_string = f".[{','.join(additional_dependency_groups)}]" + group_install_cmd = ["pip", "install", group_string] + + container = container.with_exec(group_install_cmd) + + return container + + +def _install_python_dependencies_from_requirements_txt(container: Container) -> Container: + install_requirements_cmd = ["pip", "install", "-r", "requirements.txt"] + return container.with_exec(install_requirements_cmd) + + +def _install_python_dependencies_from_poetry( + container: Container, + additional_dependency_groups: Optional[Sequence[str]] = None, +) -> Container: + pip_install_poetry_cmd = ["pip", "install", "poetry"] + poetry_disable_virtual_env_cmd = ["poetry", "config", "virtualenvs.create", "false"] + poetry_install_no_venv_cmd = ["poetry", "install"] + if additional_dependency_groups: + for group in additional_dependency_groups: + poetry_install_no_venv_cmd += ["--with", group] + + return container.with_exec(pip_install_poetry_cmd).with_exec(poetry_disable_virtual_env_cmd).with_exec(poetry_install_no_venv_cmd) + + +async def with_installed_python_package( + context: PipelineContext, + python_environment: Container, + package_source_code_path: str, + additional_dependency_groups: Optional[Sequence[str]] = None, + exclude: Optional[List] = None, + include: Optional[List] = None, +) -> Container: + """Install a python package in a python environment container. + + Args: + context (PipelineContext): The current test context, providing the repository directory from which the python sources will be pulled. + python_environment (Container): An existing python environment in which the package will be installed. + package_source_code_path (str): The local path to the package source code. + additional_dependency_groups (Optional[Sequence[str]]): extra_requires dependency of setup.py to install. Defaults to None. + exclude (Optional[List]): A list of file or directory to exclude from the python package source code. + + Returns: + Container: A python environment container with the python package installed. + """ + container = with_python_package(context, python_environment, package_source_code_path, exclude=exclude, include=include) + local_dependencies = await find_local_python_dependencies(context, package_source_code_path) + + for dependency_directory in local_dependencies: + container = container.with_mounted_directory("/" + dependency_directory, context.get_repo_dir(dependency_directory)) + + has_setup_py = await check_path_in_workdir(container, "setup.py") + has_requirements_txt = await check_path_in_workdir(container, "requirements.txt") + has_pyproject_toml = await check_path_in_workdir(container, "pyproject.toml") + + if has_pyproject_toml: + container = with_poetry_cache(container, context.dagger_client) + container = _install_python_dependencies_from_poetry(container, additional_dependency_groups) + elif has_setup_py: + container = with_pip_cache(container, context.dagger_client) + container = _install_python_dependencies_from_setup_py(container, additional_dependency_groups) + elif has_requirements_txt: + container = with_pip_cache(container, context.dagger_client) + container = _install_python_dependencies_from_requirements_txt(container) + + return container + + +def with_python_connector_source(context: ConnectorContext) -> Container: + """Load an airbyte connector source code in a testing environment. + + Args: + context (ConnectorContext): The current test context, providing the repository directory from which the connector sources will be pulled. + Returns: + Container: A python environment container (with the connector source code). + """ + connector_source_path = str(context.connector.code_directory) + testing_environment: Container = with_testing_dependencies(context) + + return with_python_package(context, testing_environment, connector_source_path) + + +async def apply_python_development_overrides(context: ConnectorContext, connector_container: Container) -> Container: + # Run the connector using the local cdk if flag is set + if context.use_local_cdk: + context.logger.info("Using local CDK") + # mount the local cdk + path_to_cdk = "airbyte-cdk/python/" + directory_to_mount = context.get_repo_dir(path_to_cdk) + + context.logger.info(f"Mounting CDK from {directory_to_mount}") + + # Install the airbyte-cdk package from the local directory + # We use --no-deps to avoid conflicts with the airbyte-cdk version required by the connector + connector_container = connector_container.with_mounted_directory(f"/{path_to_cdk}", directory_to_mount).with_exec( + ["pip", "install", "--no-deps", f"/{path_to_cdk}"], skip_entrypoint=True + ) + + return connector_container + + +async def with_python_connector_installed( + context: ConnectorContext, + python_container: Container, + connector_source_path: str, + additional_dependency_groups: Optional[Sequence[str]] = None, + exclude: Optional[List[str]] = None, + include: Optional[List[str]] = None, +) -> Container: + """Install an airbyte python connectors dependencies.""" + + # Download the latest CDK version to update the pip cache. + # This is a hack to ensure we always get the latest CDK version installed in connectors not pinning the CDK version. + await hacks.cache_latest_cdk(context) + container = await with_installed_python_package( + context, + python_container, + connector_source_path, + additional_dependency_groups=additional_dependency_groups, + exclude=exclude, + include=include, + ) + + container = await apply_python_development_overrides(context, container) + + return container + + +def with_pip_packages(base_container: Container, packages_to_install: List[str]) -> Container: + """Installs packages using pip + Args: + context (Container): A container with python installed + + Returns: + Container: A container with the pip packages installed. + + """ + package_install_command = ["pip", "install"] + return base_container.with_exec(package_install_command + packages_to_install) diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/pipx.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/pipx.py new file mode 100644 index 000000000000..856ce2d566cc --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/pipx.py @@ -0,0 +1,53 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import List, Optional + +from dagger import Container +from pipelines.airbyte_ci.connectors.context import PipelineContext +from pipelines.dagger.actions.python.common import with_pip_packages, with_python_package +from pipelines.dagger.actions.python.poetry import find_local_dependencies_in_pyproject_toml + + +def with_pipx(base_python_container: Container) -> Container: + """Installs pipx in a python container. + + Args: + base_python_container (Container): The container to install pipx on. + + Returns: + Container: A python environment with pipx installed. + """ + python_with_pipx = with_pip_packages(base_python_container, ["pipx"]).with_env_variable("PIPX_BIN_DIR", "/usr/local/bin") + + return python_with_pipx + + +async def with_installed_pipx_package( + context: PipelineContext, + python_environment: Container, + package_source_code_path: str, + exclude: Optional[List] = None, +) -> Container: + """Install a python package in a python environment container using pipx. + + Args: + context (PipelineContext): The current test context, providing the repository directory from which the python sources will be pulled. + python_environment (Container): An existing python environment in which the package will be installed. + package_source_code_path (str): The local path to the package source code. + exclude (Optional[List]): A list of file or directory to exclude from the python package source code. + + Returns: + Container: A python environment container with the python package installed. + """ + pipx_python_environment = with_pipx(python_environment) + container = with_python_package(context, pipx_python_environment, package_source_code_path, exclude=exclude) + + local_dependencies = await find_local_dependencies_in_pyproject_toml(context, container, package_source_code_path, exclude=exclude) + for dependency_directory in local_dependencies: + container = container.with_mounted_directory("/" + dependency_directory, context.get_repo_dir(dependency_directory)) + + container = container.with_exec(["pipx", "install", f"/{package_source_code_path}"]) + + return container diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/poetry.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/poetry.py new file mode 100644 index 000000000000..c9399a11c699 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/python/poetry.py @@ -0,0 +1,89 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import uuid +from pathlib import Path +from typing import List, Optional + +import toml # type: ignore +from dagger import Container, Directory +from pipelines.airbyte_ci.connectors.context import PipelineContext +from pipelines.dagger.actions.python.common import with_pip_packages, with_python_package +from pipelines.dagger.actions.system.common import with_debian_packages +from pipelines.dagger.containers.python import with_python_base +from pipelines.helpers.utils import get_file_contents + + +async def find_local_dependencies_in_pyproject_toml( + context: PipelineContext, + base_container: Container, + pyproject_file_path: str, + exclude: Optional[List] = None, +) -> list: + """Find local dependencies of a python package in a pyproject.toml file. + + Args: + python_package (Container): A python environment container with the python package source code. + pyproject_file_path (str): The path to the pyproject.toml file. + + Returns: + list: Paths to the local dependencies relative to the current directory. + """ + python_package = with_python_package(context, base_container, pyproject_file_path) + pyproject_content_raw = await get_file_contents(python_package, "pyproject.toml") + if not pyproject_content_raw: + return [] + + pyproject_content = toml.loads(pyproject_content_raw) + local_dependency_paths = [] + for value in pyproject_content["tool"]["poetry"]["dependencies"].values(): + if isinstance(value, dict) and "path" in value: + local_dependency_path = str((Path(pyproject_file_path) / Path(value["path"])).resolve().relative_to(Path.cwd())) + local_dependency_paths.append(local_dependency_path) + + # Ensure we parse the child dependencies + # TODO handle more than pyproject.toml + child_local_dependencies = await find_local_dependencies_in_pyproject_toml( + context, base_container, local_dependency_path, exclude=exclude + ) + local_dependency_paths += child_local_dependencies + + return local_dependency_paths + + +def with_poetry(context: PipelineContext) -> Container: + """Install poetry in a python environment. + + Args: + context (PipelineContext): The current test context, providing the repository directory from which the ci_credentials sources will be pulled. + Returns: + Container: A python environment with poetry installed. + """ + python_base_environment: Container = with_python_base(context) + python_with_git = with_debian_packages(python_base_environment, ["git"]) + python_with_poetry = with_pip_packages(python_with_git, ["poetry"]) + + # poetry_cache: CacheVolume = context.dagger_client.cache_volume("poetry_cache") + # poetry_with_cache = python_with_poetry.with_mounted_cache("/root/.cache/pypoetry", poetry_cache, sharing=CacheSharingMode.SHARED) + + return python_with_poetry + + +def with_poetry_module(context: PipelineContext, parent_dir: Directory, module_path: str) -> Container: + """Sets up a Poetry module. + + Args: + context (PipelineContext): The current test context, providing the repository directory from which the ci_credentials sources will be pulled. + Returns: + Container: A python environment with dependencies installed using poetry. + """ + poetry_install_dependencies_cmd = ["poetry", "install"] + + python_with_poetry = with_poetry(context) + return ( + python_with_poetry.with_mounted_directory("/src", parent_dir) + .with_workdir(f"/src/{module_path}") + .with_exec(poetry_install_dependencies_cmd) + .with_env_variable("CACHEBUSTER", str(uuid.uuid4())) + ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/actions/remote_storage.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/remote_storage.py similarity index 97% rename from airbyte-ci/connectors/pipelines/pipelines/actions/remote_storage.py rename to airbyte-ci/connectors/pipelines/pipelines/dagger/actions/remote_storage.py index 3024cf8378c0..7995ccb2f1b1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/actions/remote_storage.py +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/remote_storage.py @@ -9,7 +9,7 @@ from typing import List, Optional, Tuple from dagger import Client, File, Secret -from pipelines.utils import get_exec_result, secret_host_variable, with_exit_code +from pipelines.helpers.utils import get_exec_result, secret_host_variable, with_exit_code GOOGLE_CLOUD_SDK_TAG = "425.0.0-slim" diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py new file mode 100644 index 000000000000..e13ff7abd8a6 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/secrets.py @@ -0,0 +1,157 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This modules groups functions made to download/upload secrets from/to a remote secret service and provide these secret in a dagger Directory.""" +from __future__ import annotations + +import datetime +from typing import TYPE_CHECKING + +from anyio import Path +from dagger import Secret +from pipelines.helpers.utils import get_file_contents, get_secret_host_variable + +if TYPE_CHECKING: + from typing import Callable, Dict + + from dagger import Container + from pipelines.airbyte_ci.connectors.context import ConnectorContext + + +async def get_secrets_to_mask(ci_credentials_with_downloaded_secrets: Container) -> list[str]: + """This function will print the secrets to mask in the GitHub actions logs with the ::add-mask:: prefix. + We're not doing it directly from the ci_credentials tool because its stdout is wrapped around the dagger logger, + And GHA will only interpret lines starting with ::add-mask:: as secrets to mask. + """ + secrets_to_mask = [] + if secrets_to_mask_file := await get_file_contents(ci_credentials_with_downloaded_secrets, "/tmp/secrets_to_mask.txt"): + for secret_to_mask in secrets_to_mask_file.splitlines(): + # We print directly to stdout because the GHA runner will mask only if the log line starts with "::add-mask::" + # If we use the dagger logger, or context logger, the log line will start with other stuff and will not be masked + print(f"::add-mask::{secret_to_mask}") + secrets_to_mask.append(secret_to_mask) + return secrets_to_mask + + +async def download(context: ConnectorContext, gcp_gsm_env_variable_name: str = "GCP_GSM_CREDENTIALS") -> Dict[str, Secret]: + """Use the ci-credentials tool to download the secrets stored for a specific connector to a Directory. + + Args: + context (ConnectorContext): The context providing a connector object. + gcp_gsm_env_variable_name (str, optional): The name of the environment variable holding credentials to connect to Google Secret Manager. Defaults to "GCP_GSM_CREDENTIALS". + + Returns: + dict[str, Secret]: A dict mapping the secret file name to the dagger Secret object. + """ + # temp - fix circular import + from pipelines.dagger.containers.internal_tools import with_ci_credentials + + gsm_secret = get_secret_host_variable(context.dagger_client, gcp_gsm_env_variable_name) + secrets_path = f"/{context.connector.code_directory}/secrets" + ci_credentials = await with_ci_credentials(context, gsm_secret) + with_downloaded_secrets = ( + ci_credentials.with_exec(["mkdir", "-p", secrets_path]) + .with_env_variable( + "CACHEBUSTER", datetime.datetime.now().isoformat() + ) # Secrets can be updated on GSM anytime, we can't cache this step... + .with_exec(["ci_credentials", context.connector.technical_name, "write-to-storage"]) + ) + # We don't want to print secrets in the logs when running locally. + if context.is_ci: + context.secrets_to_mask = await get_secrets_to_mask(with_downloaded_secrets) + connector_secrets = {} + for secret_file in await with_downloaded_secrets.directory(secrets_path).entries(): + secret_plaintext = await with_downloaded_secrets.directory(secrets_path).file(secret_file).contents() + # We have to namespace secrets as Dagger derives session wide secret ID from their name + unique_secret_name = f"{context.connector.technical_name}_{secret_file}" + connector_secrets[secret_file] = context.dagger_client.set_secret(unique_secret_name, secret_plaintext) + + return connector_secrets + + +async def upload(context: ConnectorContext, gcp_gsm_env_variable_name: str = "GCP_GSM_CREDENTIALS") -> Container: + """Use the ci-credentials tool to upload the secrets stored in the context's updated_secrets-dir. + + Args: + context (ConnectorContext): The context providing a connector object and the update secrets dir. + gcp_gsm_env_variable_name (str, optional): The name of the environment variable holding credentials to connect to Google Secret Manager. Defaults to "GCP_GSM_CREDENTIALS". + + Returns: + container (Container): The executed ci-credentials update-secrets command. + + Raises: + ExecError: If the command returns a non-zero exit code. + """ + assert context.updated_secrets_dir is not None, "The context's updated_secrets_dir must be set to upload secrets." + # temp - fix circular import + from pipelines.dagger.containers.internal_tools import with_ci_credentials + + gsm_secret = get_secret_host_variable(context.dagger_client, gcp_gsm_env_variable_name) + secrets_path = f"/{context.connector.code_directory}/secrets" + + ci_credentials = await with_ci_credentials(context, gsm_secret) + + return await ci_credentials.with_directory(secrets_path, context.updated_secrets_dir).with_exec( + ["ci_credentials", context.connector.technical_name, "update-secrets"] + ) + + +async def load_from_local(context: ConnectorContext) -> Dict[str, Secret]: + """Load the secrets from the local secrets directory for a connector. + + Args: + context (ConnectorContext): The context providing the connector directory. + + Returns: + dict[str, Secret]: A dict mapping the secret file name to the dagger Secret object. + """ + connector_secrets: Dict[str, Secret] = {} + local_secrets_path = Path(context.connector.code_directory / "secrets") + if not await local_secrets_path.is_dir(): + context.logger.warning(f"Local secrets directory {local_secrets_path} does not exist, no secrets will be loaded.") + return connector_secrets + async for secret_file in local_secrets_path.iterdir(): + secret_plaintext = await secret_file.read_text() + unique_secret_name = f"{context.connector.technical_name}_{secret_file.name}" + connector_secrets[secret_file.name] = context.dagger_client.set_secret(unique_secret_name, secret_plaintext) + if not connector_secrets: + context.logger.warning(f"Local secrets directory {local_secrets_path} is empty, no secrets will be loaded.") + return connector_secrets + + +async def get_connector_secrets(context: ConnectorContext) -> dict[str, Secret]: + """Download the secrets from GSM or use the local secrets directory for a connector. + + Args: + context (ConnectorContext): The context providing the connector directory and the use_remote_secrets flag. + + Returns: + dict[str, Secret]: A dict mapping the secret file name to the dagger Secret object. + """ + if context.use_remote_secrets: + connector_secrets = await download(context) + else: + connector_secrets = await load_from_local(context) + return connector_secrets + + +async def mounted_connector_secrets(context: ConnectorContext, secret_directory_path: str) -> Callable[[Container], Container]: + """Returns an argument for a dagger container's with_ method which mounts all connector secrets in it. + + Args: + context (ConnectorContext): The context providing a connector object and its secrets. + secret_directory_path (str): Container directory where the secrets will be mounted, as files. + + Returns: + fn (Callable[[Container], Container]): A function to pass as argument to the connector container's with_ method. + """ + connector_secrets = await context.get_connector_secrets() + + def with_secrets_mounted_as_dagger_secrets(container: Container) -> Container: + container = container.with_exec(["mkdir", "-p", secret_directory_path], skip_entrypoint=True) + for secret_file_name, secret in connector_secrets.items(): + container = container.with_mounted_secret(f"{secret_directory_path}/{secret_file_name}", secret) + return container + + return with_secrets_mounted_as_dagger_secrets diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/system/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/system/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/system/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/system/common.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/system/common.py new file mode 100644 index 000000000000..940d56a74260 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/system/common.py @@ -0,0 +1,21 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import List + +from dagger import Container + + +def with_debian_packages(base_container: Container, packages_to_install: List[str]) -> Container: + """Installs packages using apt-get. + Args: + context (Container): A alpine based container. + + Returns: + Container: A container with the packages installed. + + """ + update_packages_command = ["apt-get", "update"] + package_install_command = ["apt-get", "install", "-y"] + return base_container.with_exec(update_packages_command).with_exec(package_install_command + packages_to_install) diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/system/docker.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/system/docker.py new file mode 100644 index 000000000000..d0f2fcbd0240 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/actions/system/docker.py @@ -0,0 +1,238 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import uuid +from typing import Callable, Dict, List, Optional, Union + +from dagger import Client, Container, File, Secret, Service +from pipelines import consts +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.consts import ( + DOCKER_HOST_NAME, + DOCKER_HOST_PORT, + DOCKER_REGISTRY_ADDRESS, + DOCKER_REGISTRY_MIRROR_URL, + DOCKER_TMP_VOLUME_NAME, + DOCKER_VAR_LIB_VOLUME_NAME, + STORAGE_DRIVER, + TAILSCALE_AUTH_KEY, +) +from pipelines.helpers.utils import sh_dash_c + + +def get_base_dockerd_container(dagger_client: Client) -> Container: + """Provision a container to run a docker daemon. + It will be used as a docker host for docker-in-docker use cases. + + Args: + dagger_client (Client): The dagger client used to create the container. + Returns: + Container: The container to run dockerd as a service + """ + apk_packages_to_install = [ + STORAGE_DRIVER, + # Curl is only used for debugging purposes. + "curl", + ] + base_container = ( + dagger_client.container() + .from_(consts.DOCKER_DIND_IMAGE) + # We set this env var because we need to use a non-default zombie reaper setting. + # The reason for this is that by default it will want to set its parent process ID to 1 when reaping. + # This won't be possible because of container-ception: dind is running inside the dagger engine. + # See https://github.com/krallin/tini#subreaping for details. + .with_env_variable("TINI_SUBREAPER", "") + .with_exec( + sh_dash_c( + [ + "apk update", + f"apk add {' '.join(apk_packages_to_install)}", + "mkdir /etc/docker", + ] + ) + ) + # Expose the docker host port. + .with_exposed_port(DOCKER_HOST_PORT) + # We cache /tmp for file sharing between client and daemon. + .with_mounted_cache("/tmp", dagger_client.cache_volume(DOCKER_TMP_VOLUME_NAME)) + ) + + # We cache /var/lib/docker to avoid downloading images and layers multiple times. + base_container = base_container.with_mounted_cache("/var/lib/docker", dagger_client.cache_volume(DOCKER_VAR_LIB_VOLUME_NAME)) + return base_container + + +def get_daemon_config_json(registry_mirror_url: Optional[str] = None) -> str: + """Get the json representation of the docker daemon config. + + Args: + registry_mirror_url (Optional[str]): The registry mirror url to use. + + Returns: + str: The json representation of the docker daemon config. + """ + daemon_config: Dict[str, Union[List[str], str]] = { + "storage-driver": STORAGE_DRIVER, + } + if registry_mirror_url: + daemon_config["registry-mirrors"] = ["http://" + registry_mirror_url] + daemon_config["insecure-registries"] = [registry_mirror_url] + return json.dumps(daemon_config) + + +def docker_login( + dockerd_container: Container, + docker_registry_username_secret: Secret, + docker_registry_password_secret: Secret, +) -> Container: + """Login to a docker registry if the username and password secrets are provided. + + Args: + dockerd_container (Container): The dockerd_container container to login to the registry. + docker_registry_username_secret (Secret): The docker registry username secret. + docker_registry_password_secret (Secret): The docker registry password secret. + docker_registry_address (Optional[str]): The docker registry address to login to. Defaults to "docker.io" (DockerHub). + Returns: + Container: The container with the docker login command executed if the username and password secrets are provided. Noop otherwise. + """ + if docker_registry_username_secret and docker_registry_username_secret: + return ( + dockerd_container + # We use a cache buster here to guarantee the docker login is always executed. + .with_env_variable("CACHEBUSTER", str(uuid.uuid4())) + .with_secret_variable("DOCKER_REGISTRY_USERNAME", docker_registry_username_secret) + .with_secret_variable("DOCKER_REGISTRY_PASSWORD", docker_registry_password_secret) + .with_exec( + sh_dash_c([f"docker login -u $DOCKER_REGISTRY_USERNAME -p $DOCKER_REGISTRY_PASSWORD {DOCKER_REGISTRY_ADDRESS}"]), + skip_entrypoint=True, + ) + ) + else: + return dockerd_container + + +def with_global_dockerd_service( + dagger_client: Client, + docker_hub_username_secret: Optional[Secret] = None, + docker_hub_password_secret: Optional[Secret] = None, +) -> Service: + """Create a container with a docker daemon running. + We expose its 2375 port to use it as a docker host for docker-in-docker use cases. + It is optionally bound to a tailscale VPN if the TAILSCALE_AUTH_KEY env var is set. + Args: + dagger_client (Client): The dagger client used to create the container. + docker_hub_username_secret (Optional[Secret]): The DockerHub username secret. + docker_hub_password_secret (Optional[Secret]): The DockerHub password secret. + Returns: + Container: The container running dockerd as a service + """ + + dockerd_container = get_base_dockerd_container(dagger_client) + if TAILSCALE_AUTH_KEY is not None: + # Ping the registry mirror host to make sure it's reachable through VPN + # We set a cache buster here to guarantee the curl command is always executed. + dockerd_container = dockerd_container.with_env_variable("CACHEBUSTER", str(uuid.uuid4())).with_exec( + ["curl", "-vvv", f"http://{DOCKER_REGISTRY_MIRROR_URL}/v2/"], skip_entrypoint=True + ) + daemon_config_json = get_daemon_config_json(DOCKER_REGISTRY_MIRROR_URL) + else: + daemon_config_json = get_daemon_config_json() + + dockerd_container = dockerd_container.with_new_file("/etc/docker/daemon.json", contents=daemon_config_json) + if docker_hub_username_secret and docker_hub_password_secret: + # Docker login happens late because there's a cache buster in the docker login command. + dockerd_container = docker_login(dockerd_container, docker_hub_username_secret, docker_hub_password_secret) + return dockerd_container.with_exec( + ["dockerd", "--log-level=error", f"--host=tcp://0.0.0.0:{DOCKER_HOST_PORT}", "--tls=false"], insecure_root_capabilities=True + ).as_service() + + +def with_bound_docker_host( + context: ConnectorContext, + container: Container, +) -> Container: + """Bind a container to a docker host. It will use the dockerd service as a docker host. + + Args: + context (ConnectorContext): The current connector context. + container (Container): The container to bind to the docker host. + Returns: + Container: The container bound to the docker host. + """ + assert context.dockerd_service is not None + return ( + container.with_env_variable("DOCKER_HOST", f"tcp://{DOCKER_HOST_NAME}:{DOCKER_HOST_PORT}") + .with_service_binding(DOCKER_HOST_NAME, context.dockerd_service) + .with_mounted_cache("/tmp", context.dagger_client.cache_volume(DOCKER_TMP_VOLUME_NAME)) + ) + + +def bound_docker_host(context: ConnectorContext) -> Callable[[Container], Container]: + def bound_docker_host_inner(container: Container) -> Container: + return with_bound_docker_host(context, container) + + return bound_docker_host_inner + + +def with_docker_cli(context: ConnectorContext) -> Container: + """Create a container with the docker CLI installed and bound to a persistent docker host. + + Args: + context (ConnectorContext): The current connector context. + + Returns: + Container: A docker cli container bound to a docker host. + """ + docker_cli = context.dagger_client.container().from_(consts.DOCKER_CLI_IMAGE) + return with_bound_docker_host(context, docker_cli) + + +async def load_image_to_docker_host(context: ConnectorContext, tar_file: File, image_tag: str) -> str: + """Load a docker image tar archive to the docker host. + + Args: + context (ConnectorContext): The current connector context. + tar_file (File): The file object holding the docker image tar archive. + image_tag (str): The tag to create on the image if it has no tag. + """ + # Hacky way to make sure the image is always loaded + tar_name = f"{str(uuid.uuid4())}.tar" + docker_cli = with_docker_cli(context).with_mounted_file(tar_name, tar_file) + + image_load_output = await docker_cli.with_exec(["docker", "load", "--input", tar_name]).stdout() + # Not tagged images only have a sha256 id the load output shares. + if "sha256:" in image_load_output: + image_id = image_load_output.replace("\n", "").replace("Loaded image ID: sha256:", "") + await docker_cli.with_exec(["docker", "tag", image_id, image_tag]) + image_sha = json.loads(await docker_cli.with_exec(["docker", "inspect", image_tag]).stdout())[0].get("Id") + return image_sha + + +def with_crane( + context: ConnectorContext, +) -> Container: + """Crane is a tool to analyze and manipulate container images. + We can use it to extract the image manifest and the list of layers or list the existing tags on an image repository. + https://github.com/google/go-containerregistry/tree/main/cmd/crane + """ + + # We use the debug image as it contains a shell which we need to properly use environment variables + # https://github.com/google/go-containerregistry/tree/main/cmd/crane#images + base_container = context.dagger_client.container().from_("gcr.io/go-containerregistry/crane/debug:v0.15.1") + + if context.docker_hub_username_secret and context.docker_hub_password_secret: + base_container = ( + base_container.with_secret_variable("DOCKER_HUB_USERNAME", context.docker_hub_username_secret).with_secret_variable( + "DOCKER_HUB_PASSWORD", context.docker_hub_password_secret + ) + # We need to use skip_entrypoint=True to avoid the entrypoint to be overridden by the crane command + # We use sh -c to be able to use environment variables in the command + # This is a workaround as the default crane entrypoint doesn't support environment variables + .with_exec( + sh_dash_c(["crane auth login index.docker.io -u $DOCKER_HUB_USERNAME -p $DOCKER_HUB_PASSWORD"]), skip_entrypoint=True + ) + ) + + return base_container diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/git.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/git.py new file mode 100644 index 000000000000..bd9a8a5b5b8d --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/git.py @@ -0,0 +1,38 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Optional + +from dagger import Client, Container +from pipelines.helpers.utils import AIRBYTE_REPO_URL + + +async def checked_out_git_container( + dagger_client: Client, + current_git_branch: str, + current_git_revision: str, + diffed_branch: Optional[str] = None, +) -> Container: + """Builds git-based container with the current branch checked out.""" + current_git_branch = current_git_branch.removeprefix("origin/") + diffed_branch = current_git_branch if diffed_branch is None else diffed_branch.removeprefix("origin/") + return await ( + dagger_client.container() + .from_("alpine/git:latest") + .with_workdir("/repo") + .with_exec(["init"]) + .with_env_variable("CACHEBUSTER", current_git_revision) + .with_exec( + [ + "remote", + "add", + "--fetch", + "--track", + current_git_branch, + "--track", + diffed_branch if diffed_branch is not None else current_git_branch, + "origin", + AIRBYTE_REPO_URL, + ] + ) + .with_exec(["checkout", "-t", f"origin/{current_git_branch}"]) + ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/internal_tools.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/internal_tools.py new file mode 100644 index 000000000000..2eb424189439 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/internal_tools.py @@ -0,0 +1,39 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dagger import Container, Secret +from pipelines.airbyte_ci.connectors.context import PipelineContext +from pipelines.consts import INTERNAL_TOOL_PATHS +from pipelines.dagger.actions.python.pipx import with_installed_pipx_package +from pipelines.dagger.containers.python import with_python_base + + +async def with_ci_credentials(context: PipelineContext, gsm_secret: Secret) -> Container: + """Install the ci_credentials package in a python environment. + + Args: + context (PipelineContext): The current test context, providing the repository directory from which the ci_credentials sources will be pulled. + gsm_secret (Secret): The secret holding GCP_GSM_CREDENTIALS env variable value. + + Returns: + Container: A python environment with the ci_credentials package installed. + """ + python_base_environment: Container = with_python_base(context) + ci_credentials = await with_installed_pipx_package(context, python_base_environment, INTERNAL_TOOL_PATHS.CI_CREDENTIALS.value) + ci_credentials = ci_credentials.with_env_variable("VERSION", "dagger_ci") + return ci_credentials.with_secret_variable("GCP_GSM_CREDENTIALS", gsm_secret).with_workdir("/") + + +async def with_connector_ops(context: PipelineContext) -> Container: + """Installs the connector_ops package in a Container running Python > 3.10 with git.. + + Args: + context (PipelineContext): The current test context, providing the repository directory from which the ci_connector_sources sources will be pulled. + + Returns: + Container: A python environment container with connector_ops installed. + """ + python_base_environment: Container = with_python_base(context) + + return await with_installed_pipx_package(context, python_base_environment, INTERNAL_TOOL_PATHS.CONNECTOR_OPS.value) diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py new file mode 100644 index 000000000000..6061f321969e --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/java.py @@ -0,0 +1,175 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dagger import CacheVolume, Container, File, Platform +from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext +from pipelines.consts import AMAZONCORRETTO_IMAGE +from pipelines.dagger.actions.connector.hooks import finalize_build +from pipelines.dagger.actions.connector.normalization import DESTINATION_NORMALIZATION_BUILD_CONFIGURATION, with_normalization +from pipelines.helpers.utils import sh_dash_c + + +def with_integration_base(context: PipelineContext, build_platform: Platform) -> Container: + return ( + context.dagger_client.container(platform=build_platform) + .from_("amazonlinux:2022.0.20220831.1") + .with_workdir("/airbyte") + .with_file("base.sh", context.get_repo_dir("airbyte-integrations/bases/base", include=["base.sh"]).file("base.sh")) + .with_env_variable("AIRBYTE_ENTRYPOINT", "/airbyte/base.sh") + .with_label("io.airbyte.version", "0.1.0") + .with_label("io.airbyte.name", "airbyte/integration-base") + ) + + +def with_integration_base_java(context: PipelineContext, build_platform: Platform) -> Container: + integration_base = with_integration_base(context, build_platform) + yum_packages_to_install = [ + "tar", # required to untar java connector binary distributions. + "openssl", # required because we need to ssh and scp sometimes. + "findutils", # required for xargs, which is shipped as part of findutils. + ] + return ( + context.dagger_client.container(platform=build_platform) + # Use a linux+jdk base image with long-term support, such as amazoncorretto. + .from_(AMAZONCORRETTO_IMAGE) + # Install a bunch of packages as early as possible. + .with_exec( + sh_dash_c( + [ + # Update first, but in the same .with_exec step as the package installation. + # Otherwise, we risk caching stale package URLs. + "yum update -y", + # + f"yum install -y {' '.join(yum_packages_to_install)}", + # Remove any dangly bits. + "yum clean all", + ] + ) + ) + # Add what files we need to the /airbyte directory. + # Copy base.sh from the airbyte/integration-base image. + .with_directory("/airbyte", integration_base.directory("/airbyte")) + .with_workdir("/airbyte") + # Download a utility jar from the internet. + .with_file("dd-java-agent.jar", context.dagger_client.http("https://dtdg.co/latest-java-tracer")) + # Copy javabase.sh from the git repo. + .with_file("javabase.sh", context.get_repo_dir("airbyte-integrations/bases/base-java", include=["javabase.sh"]).file("javabase.sh")) + # Set a bunch of env variables used by base.sh. + .with_env_variable("AIRBYTE_SPEC_CMD", "/airbyte/javabase.sh --spec") + .with_env_variable("AIRBYTE_CHECK_CMD", "/airbyte/javabase.sh --check") + .with_env_variable("AIRBYTE_DISCOVER_CMD", "/airbyte/javabase.sh --discover") + .with_env_variable("AIRBYTE_READ_CMD", "/airbyte/javabase.sh --read") + .with_env_variable("AIRBYTE_WRITE_CMD", "/airbyte/javabase.sh --write") + .with_env_variable("AIRBYTE_ENTRYPOINT", "/airbyte/base.sh") + # Set image labels. + .with_label("io.airbyte.version", "0.1.2") + .with_label("io.airbyte.name", "airbyte/integration-base-java") + ) + + +def with_integration_base_java_and_normalization(context: ConnectorContext, build_platform: Platform) -> Container: + yum_packages_to_install = [ + "python3", + "python3-devel", + "jq", + "sshpass", + "git", + ] + + additional_yum_packages = DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["yum_packages"] + yum_packages_to_install += additional_yum_packages + + dbt_adapter_package = DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["dbt_adapter"] + assert isinstance(dbt_adapter_package, str) + normalization_integration_name = DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["integration_name"] + assert isinstance(normalization_integration_name, str) + + pip_cache: CacheVolume = context.dagger_client.cache_volume("pip_cache") + + return ( + with_integration_base_java(context, build_platform) + .with_exec( + sh_dash_c( + [ + "yum update -y", + f"yum install -y {' '.join(yum_packages_to_install)}", + "yum clean all", + "alternatives --install /usr/bin/python python /usr/bin/python3 60", + ] + ) + ) + .with_mounted_cache("/root/.cache/pip", pip_cache) + .with_exec( + sh_dash_c( + [ + "python -m ensurepip --upgrade", + # Workaround for https://github.com/yaml/pyyaml/issues/601 + "pip3 install 'Cython<3.0' 'pyyaml~=5.4' --no-build-isolation", + # Required for dbt https://github.com/dbt-labs/dbt-core/issues/7075 + "pip3 install 'pytz~=2023.3'", + f"pip3 install {dbt_adapter_package}", + # amazon linux 2 isn't compatible with urllib3 2.x, so force 1.x + "pip3 install 'urllib3<2'", + ] + ) + ) + .with_directory("airbyte_normalization", with_normalization(context, build_platform).directory("/airbyte")) + .with_workdir("airbyte_normalization") + .with_exec(sh_dash_c(["mv * .."])) + .with_workdir("/airbyte") + .with_exec(["rm", "-rf", "airbyte_normalization"]) + .with_workdir("/airbyte/normalization_code") + .with_exec(["pip3", "install", "."]) + .with_workdir("/airbyte/normalization_code/dbt-template/") + .with_exec(["dbt", "deps"]) + .with_workdir("/airbyte") + .with_file( + "run_with_normalization.sh", + context.get_repo_dir("airbyte-integrations/bases/base-java", include=["run_with_normalization.sh"]).file( + "run_with_normalization.sh" + ), + ) + .with_env_variable("AIRBYTE_NORMALIZATION_INTEGRATION", normalization_integration_name) + .with_env_variable("AIRBYTE_ENTRYPOINT", "/airbyte/run_with_normalization.sh") + ) + + +async def with_airbyte_java_connector(context: ConnectorContext, connector_java_tar_file: File, build_platform: Platform) -> Container: + application = context.connector.technical_name + + build_stage = ( + with_integration_base_java(context, build_platform) + .with_workdir("/airbyte") + .with_env_variable("APPLICATION", context.connector.technical_name) + .with_file(f"{application}.tar", connector_java_tar_file) + .with_exec( + sh_dash_c( + [ + f"tar xf {application}.tar --strip-components=1", + f"rm -rf {application}.tar", + ] + ) + ) + ) + + if ( + context.connector.supports_normalization + and DESTINATION_NORMALIZATION_BUILD_CONFIGURATION[context.connector.technical_name]["supports_in_connector_normalization"] + ): + base = with_integration_base_java_and_normalization(context, build_platform) + entrypoint = ["/airbyte/run_with_normalization.sh"] + else: + base = with_integration_base_java(context, build_platform) + entrypoint = ["/airbyte/base.sh"] + + connector_container = ( + base.with_workdir("/airbyte") + .with_env_variable("APPLICATION", application) + .with_mounted_directory("built_artifacts", build_stage.directory("/airbyte")) + .with_exec(sh_dash_c(["mv built_artifacts/* ."])) + .with_label("io.airbyte.version", context.metadata["dockerImageTag"]) + .with_label("io.airbyte.name", context.metadata["dockerRepository"]) + .with_entrypoint(entrypoint) + ) + return await finalize_build(context, connector_container) diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py new file mode 100644 index 000000000000..98227fe9c81d --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/dagger/containers/python.py @@ -0,0 +1,90 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dagger import CacheSharingMode, CacheVolume, Client, Container +from pipelines.airbyte_ci.connectors.context import PipelineContext +from pipelines.consts import ( + CONNECTOR_TESTING_REQUIREMENTS, + PIP_CACHE_PATH, + PIP_CACHE_VOLUME_NAME, + POETRY_CACHE_PATH, + POETRY_CACHE_VOLUME_NAME, + PYPROJECT_TOML_FILE_PATH, +) +from pipelines.helpers.utils import sh_dash_c + + +def with_python_base(context: PipelineContext, python_version: str = "3.10") -> Container: + """Build a Python container with a cache volume for pip cache. + + Args: + context (PipelineContext): The current test context, providing a dagger client and a repository directory. + python_image_name (str, optional): The python image to use to build the python base environment. Defaults to "python:3.9-slim". + + Raises: + ValueError: Raised if the python_image_name is not a python image. + + Returns: + Container: The python base environment container. + """ + + pip_cache: CacheVolume = context.dagger_client.cache_volume("pip_cache") + + base_container = ( + context.dagger_client.container() + .from_(f"python:{python_version}-slim") + .with_mounted_cache("/root/.cache/pip", pip_cache) + .with_exec( + sh_dash_c( + [ + "apt-get update", + "apt-get install -y build-essential cmake g++ libffi-dev libstdc++6 git", + "pip install pip==23.1.2", + ] + ) + ) + ) + + return base_container + + +def with_testing_dependencies(context: PipelineContext) -> Container: + """Build a testing environment by installing testing dependencies on top of a python base environment. + + Args: + context (PipelineContext): The current test context, providing a dagger client and a repository directory. + + Returns: + Container: The testing environment container. + """ + python_environment: Container = with_python_base(context) + pyproject_toml_file = context.get_repo_dir(".", include=[PYPROJECT_TOML_FILE_PATH]).file(PYPROJECT_TOML_FILE_PATH) + + return python_environment.with_exec(["pip", "install"] + CONNECTOR_TESTING_REQUIREMENTS).with_file( + f"/{PYPROJECT_TOML_FILE_PATH}", pyproject_toml_file + ) + + +def with_pip_cache(container: Container, dagger_client: Client) -> Container: + """Mounts the pip cache in the container. + Args: + container (Container): A container with python installed + + Returns: + Container: A container with the pip cache mounted. + """ + pip_cache_volume = dagger_client.cache_volume(PIP_CACHE_VOLUME_NAME) + return container.with_mounted_cache(PIP_CACHE_PATH, pip_cache_volume, sharing=CacheSharingMode.SHARED) + + +def with_poetry_cache(container: Container, dagger_client: Client) -> Container: + """Mounts the poetry cache in the container. + Args: + container (Container): A container with python installed + + Returns: + Container: A container with the poetry cache mounted. + """ + poetry_cache_volume = dagger_client.cache_volume(POETRY_CACHE_VOLUME_NAME) + return container.with_mounted_cache(POETRY_CACHE_PATH, poetry_cache_volume, sharing=CacheSharingMode.SHARED) diff --git a/airbyte-ci/connectors/pipelines/pipelines/dagger_run.py b/airbyte-ci/connectors/pipelines/pipelines/dagger_run.py deleted file mode 100644 index d9ef70879617..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/dagger_run.py +++ /dev/null @@ -1,110 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -"""This module execute the airbyte-ci-internal CLI wrapped in a dagger run command to use the Dagger Terminal UI.""" - -import logging -import os -import re -import subprocess -import sys -from pathlib import Path -from typing import Optional - -import pkg_resources -import requests - -LOGGER = logging.getLogger(__name__) -BIN_DIR = Path.home() / "bin" -BIN_DIR.mkdir(exist_ok=True) -DAGGER_CLOUD_TOKEN_ENV_VAR_NAME_VALUE = ( - "_EXPERIMENTAL_DAGGER_CLOUD_TOKEN", - "p.eyJ1IjogIjFiZjEwMmRjLWYyZmQtNDVhNi1iNzM1LTgxNzI1NGFkZDU2ZiIsICJpZCI6ICJlNjk3YzZiYy0yMDhiLTRlMTktODBjZC0yNjIyNGI3ZDBjMDEifQ.hT6eMOYt3KZgNoVGNYI3_v4CC-s19z8uQsBkGrBhU3k", -) -ARGS_DISABLING_TUI = ["--no-tui", "publish"] - - -def get_dagger_path() -> Optional[str]: - try: - return ( - subprocess.run(["which", "dagger"], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.decode("utf-8").strip() - ) - except subprocess.CalledProcessError: - if Path(BIN_DIR / "dagger").exists(): - return str(Path(BIN_DIR / "dagger")) - - -def get_current_dagger_sdk_version() -> str: - version = pkg_resources.get_distribution("dagger-io").version - return version - - -def install_dagger_cli(dagger_version: str) -> None: - install_script_path = "/tmp/install_dagger.sh" - with open(install_script_path, "w") as f: - response = requests.get("https://dl.dagger.io/dagger/install.sh") - response.raise_for_status() - f.write(response.text) - subprocess.run(["chmod", "+x", install_script_path], check=True) - os.environ["BIN_DIR"] = str(BIN_DIR) - os.environ["DAGGER_VERSION"] = dagger_version - subprocess.run([install_script_path], check=True) - - -def get_dagger_cli_version(dagger_path: Optional[str]) -> Optional[str]: - if not dagger_path: - return None - version_output = ( - subprocess.run([dagger_path, "version"], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.decode("utf-8").strip() - ) - version_pattern = r"v(\d+\.\d+\.\d+)" - - match = re.search(version_pattern, version_output) - - if match: - version = match.group(1) - return version - else: - raise Exception("Could not find dagger version in output: " + version_output) - - -def check_dagger_cli_install() -> str: - expected_dagger_cli_version = get_current_dagger_sdk_version() - dagger_path = get_dagger_path() - if dagger_path is None: - LOGGER.info(f"The Dagger CLI is not installed. Installing {expected_dagger_cli_version}...") - install_dagger_cli(expected_dagger_cli_version) - dagger_path = get_dagger_path() - - cli_version = get_dagger_cli_version(dagger_path) - if cli_version != expected_dagger_cli_version: - LOGGER.warning( - f"The Dagger CLI version '{cli_version}' does not match the expected version '{expected_dagger_cli_version}'. Installing Dagger CLI '{expected_dagger_cli_version}'..." - ) - install_dagger_cli(expected_dagger_cli_version) - return check_dagger_cli_install() - return dagger_path - - -def main(): - os.environ[DAGGER_CLOUD_TOKEN_ENV_VAR_NAME_VALUE[0]] = DAGGER_CLOUD_TOKEN_ENV_VAR_NAME_VALUE[1] - exit_code = 0 - if len(sys.argv) > 1 and any([arg in ARGS_DISABLING_TUI for arg in sys.argv]): - command = ["airbyte-ci-internal"] + [arg for arg in sys.argv[1:] if arg != "--no-tui"] - else: - dagger_path = check_dagger_cli_install() - command = [dagger_path, "run", "airbyte-ci-internal"] + sys.argv[1:] - try: - try: - subprocess.run(command, check=True) - except KeyboardInterrupt: - LOGGER.info("Keyboard interrupt detected. Exiting...") - exit_code = 1 - except subprocess.CalledProcessError as e: - exit_code = e.returncode - sys.exit(exit_code) - - -if __name__ == "__main__": - main() diff --git a/airbyte-ci/connectors/pipelines/pipelines/external_scripts/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/external_scripts/__init__.py new file mode 100644 index 000000000000..f70ecfc3a89e --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/external_scripts/__init__.py @@ -0,0 +1 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. diff --git a/airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_check.sh b/airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_check.sh new file mode 100755 index 000000000000..b46d5a6c3313 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_check.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -o errexit -o nounset -o pipefail + +echo "Checking if airbyte-ci is correctly installed..." + +INSTALL_DIR="$HOME/.local/bin" +HUMAN_READABLE_INSTALL_DIR="\$HOME/.local/bin" + +# Check that the target directory is on the PATH +# If not print an error message and exit +if ! echo "$PATH" | grep -q "$INSTALL_DIR"; then + echo "The target directory $INSTALL_DIR is not on the PATH" + echo "Check that $HUMAN_READABLE_INSTALL_DIR is part of the PATH" + echo "" + echo "If not, please add 'export PATH=\"$HUMAN_READABLE_INSTALL_DIR:\$PATH\"' to your shell profile" + exit 1 +fi + +# Check that airbyte-ci is on the PATH +# If not print an error message and exit +if ! which airbyte-ci >/dev/null 2>&1; then + echo "airbyte-ci is not installed" + echo "" + echo "Please run 'make tools.airbyte-ci.install' to install airbyte-ci" + exit 1 +fi + +EXPECTED_PATH="$INSTALL_DIR/airbyte-ci" +AIRBYTE_CI_PATH=$(which airbyte-ci 2>/dev/null) +if [ "$AIRBYTE_CI_PATH" != "$EXPECTED_PATH" ]; then + echo "airbyte-ci is not from the expected install location: $EXPECTED_PATH" + echo "airbyte-ci is installed at: $AIRBYTE_CI_PATH" + echo "Check that airbyte-ci exists at $HUMAN_READABLE_INSTALL_DIR and $HUMAN_READABLE_INSTALL_DIR is part of the PATH" + echo "" + echo "If it is, try running 'make tools.airbyte-ci.clean', then run 'make tools.airbyte-ci.install' again" + exit 1 +fi + +# Check if the AIRBYTE_CI_PATH is a symlink +if [ -L "$AIRBYTE_CI_PATH" ]; then + echo "" + echo "#########################################################################" + echo "# #" + echo "# Warning: airbyte-ci at $AIRBYTE_CI_PATH is a symlink. #" + echo "# You are possibly using a development version of airbyte-ci. #" + echo "# To update to a release version, run 'make tools.airbyte-ci.install' #" + echo "# #" + echo "# If this warning persists, try running 'make tools.airbyte-ci.clean' #" + echo "# Then run 'make tools.airbyte-ci.install' again. #" + echo "# #" + echo "#########################################################################" + echo "" +fi + +echo "airbyte-ci is correctly installed at $EXPECTED_PATH" diff --git a/airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_clean.sh b/airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_clean.sh new file mode 100755 index 000000000000..56d2c1fbf371 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_clean.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -o errexit -o nounset -o pipefail + +# Check if pipx is on the path and if so, uninstall pipelines +if which pipx >/dev/null 2>&1; then + # ignore errors if pipelines is not installed + pipx uninstall pipelines || true + echo "Uninstalled pipelines via pipx" +else + echo "pipx not found, skipping uninstall of pipelines" +fi + +# Remove airbyte-ci if it's on the path +while which airbyte-ci >/dev/null 2>&1; do + echo "Removing $(which airbyte-ci)" + rm "$(which airbyte-ci)" +done +echo "Removed airbyte-ci" + +# Remove airbyte-ci-internal if it's on the path +while which airbyte-ci-internal >/dev/null 2>&1; do + echo "Removing $(which airbyte-ci)" + rm "$(which airbyte-ci-internal)" +done +echo "Removed airbyte-ci-internal" + +# Remove airbyte-ci-dev if it's on the path +while which airbyte-ci-dev >/dev/null 2>&1; do + echo "Removing $(which airbyte-ci)" + rm "$(which airbyte-ci-dev)" +done + echo "Removed airbyte-ci-dev" + +# Check if airbyte-ci is stashed away in pyenv +# If so, remove it +# This prevents `pyenv init -` from adding it back to the path +while pyenv whence --path airbyte-ci >/dev/null 2>&1; do + rm "$(pyenv whence --path airbyte-ci)" + echo "Uninstalled pipelines via pyenv" +done + echo "All airbyte-ci references removed from pyenv versions." + +echo "Cleanup of airbyte-ci install completed." diff --git a/airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_dev_install.py b/airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_dev_install.py new file mode 100755 index 000000000000..f938896d95a4 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_dev_install.py @@ -0,0 +1,55 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +# !IMPORTANT! This script is used to install the airbyte-ci tool on a Linux or macOS system. +# Meaning, no external dependencies are allowed as we don't want users to have to run anything +# other than this script to install the tool. + +import subprocess +import sys + + +def check_command_exists(command: str, not_found_message: str) -> None: + """ + Check if a command exists in the system path. + """ + try: + subprocess.check_call(["which", command], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + except subprocess.CalledProcessError: + print(not_found_message) + sys.exit(1) + + +def main() -> None: + # Check if Python 3.10 is on the path + check_command_exists( + "python3.10", + """python3.10 not found on the path. +Please install Python 3.10 using pyenv: +1. Install pyenv if not already installed: + brew install pyenv +2. Install Python 3.10 using pyenv: + pyenv install 3.10.12""", + ) + print("Python 3.10 is already installed.") + + # Check if pipx is installed + check_command_exists( + "pipx", + """pipx not found. Please install pipx: +1. Ensure Python 3.6 or later is installed. +2. Install pipx using Python: + python3 -m pip install --user pipx +3. Add pipx to your PATH: + python3 -m pipx ensurepath +After installation, restart your terminal or source your shell +configuration file to ensure the pipx command is available.""", + ) + print("pipx is already installed.") + + # Install airbyte-ci development version + subprocess.run(["pipx", "install", "--editable", "--force", "--python=python3.10", "airbyte-ci/connectors/pipelines/"]) + print("Development version of airbyte-ci installed.....") + + +if __name__ == "__main__": + main() diff --git a/airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_install.py b/airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_install.py new file mode 100755 index 000000000000..d1267c9840e9 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/external_scripts/airbyte_ci_install.py @@ -0,0 +1,134 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +# !IMPORTANT! This script is used to install the airbyte-ci tool on a Linux or macOS system. +# Meaning, no external dependencies are allowed as we don't want users to have to run anything +# other than this script to install the tool. + +from __future__ import annotations + +import os +import shutil +import ssl +import sys +import tempfile +import urllib.request +from typing import TYPE_CHECKING + +# !IMPORTANT! This constant is inline here instead of being imported from pipelines/consts.py +# because we don't want to introduce any dependencies on other files in the repository. +RELEASE_URL = os.getenv("RELEASE_URL", "https://connectors.airbyte.com/files/airbyte-ci/releases") + +if TYPE_CHECKING: + from typing import Optional + + +def _get_custom_certificate_path() -> Optional[str]: + """ + Returns the path to the custom certificate file if certifi is installed, otherwise None. + + HACK: This is a workaround for the fact that the pyinstaller binary does not know how or where to + find the ssl certificates file. This happens because the binary is built on a different system + than the one it is being run on. This function will return the path to the certifi certificate file + if it is installed, otherwise it will return None. This function is used in get_ssl_context() below. + + WHY: this works when certifi is not found: + If you run this file directly, it will use the system python interpreter and will be able to find + the ssl certificates file. e.g. when running in dev mode or via the makefile. + + WHY: this works when certifi is found: + When this file is run by the pyinstaller binary, it is through the pipelines project, which has + certifi installed. This means that when this file is run by the pyinstaller binary, it will be able + to find the ssl certificates file in the certifi package. + + """ + # if certifi is not installed, do nothing + try: + import certifi + + return certifi.where() + except ImportError: + return None + + +def get_ssl_context() -> ssl.SSLContext: + """ + Returns an ssl.SSLContext object with the custom certificate file if certifi is installed, otherwise + returns the default ssl.SSLContext object. + """ + certifi_path = _get_custom_certificate_path() + if certifi_path is None: + return ssl.create_default_context() + + return ssl.create_default_context(cafile=certifi_path) + + +def get_airbyte_os_name() -> Optional[str]: + """ + Returns 'ubuntu' if the system is Linux or 'macos' if the system is macOS. + """ + OS = os.uname().sysname + if OS == "Linux": + print("Linux based system detected.") + return "ubuntu" + elif OS == "Darwin": + print("macOS based system detected.") + return "macos" + else: + return None + + +def main(version: str = "latest") -> None: + # Determine the operating system + os_name = get_airbyte_os_name() + if os_name is None: + print("Unsupported operating system") + return + + url = f"{RELEASE_URL}/{os_name}/{version}/airbyte-ci" + + # Create the directory if it does not exist + destination_dir = os.path.expanduser("~/.local/bin") + os.makedirs(destination_dir, exist_ok=True) + + # Download the binary to a temporary folder + with tempfile.TemporaryDirectory() as tmp_dir: + tmp_file = os.path.join(tmp_dir, "airbyte-ci") + + # Download the file using urllib.request + print(f"Downloading from {url}") + ssl_context = get_ssl_context() + with urllib.request.urlopen(url, context=ssl_context) as response, open(tmp_file, "wb") as out_file: + shutil.copyfileobj(response, out_file) + + # Check if the destination path is a symlink and delete it if it is + destination_path = os.path.join(destination_dir, "airbyte-ci") + if os.path.islink(destination_path): + os.remove(destination_path) + + # Copy the file from the temporary folder to the destination + shutil.copy(tmp_file, destination_path) + + # Make the binary executable + os.chmod(destination_path, 0o755) + + # ASCII Art and Completion Message + install_complete_message = f""" + ╔───────────────────────────────────────────────────────────────────────────────╗ + │ │ + │ AAA IIIII RRRRRR BBBBB YY YY TTTTTTT EEEEEEE CCCCC IIIII │ + │ AAAAA III RR RR BB B YY YY TTT EE CC III │ + │ AA AA III RRRRRR BBBBBB YYYYY TTT EEEEE _____ CC III │ + │ AAAAAAA III RR RR BB BB YYY TTT EE CC III │ + │ AA AA IIIII RR RR BBBBBB YYY TTT EEEEEEE CCCCC IIIII │ + │ │ + │ === Installation complete. v({version})=== │ + │ {destination_path} │ + ╚───────────────────────────────────────────────────────────────────────────────╝ + """ + + print(install_complete_message) + + +if __name__ == "__main__": + version_arg = sys.argv[1] if len(sys.argv) > 1 else "latest" + main(version_arg) diff --git a/airbyte-ci/connectors/pipelines/pipelines/format/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/format/__init__.py deleted file mode 100644 index 730874bd6b9b..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/format/__init__.py +++ /dev/null @@ -1,101 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# -"""This module groups factory like functions to dispatch formatting steps according to the connector language.""" - -from __future__ import annotations - -import sys -from typing import List, Optional - -import anyio -import dagger -from connector_ops.utils import ConnectorLanguage -from pipelines.actions import environments -from pipelines.bases import ConnectorReport, Step, StepResult, StepStatus -from pipelines.contexts import ConnectorContext -from pipelines.format import java_connectors, python_connectors -from pipelines.git import GitPushChanges -from pipelines.pipelines.connectors import run_report_complete_pipeline - - -class NoFormatStepForLanguageError(Exception): - pass - - -FORMATTING_STEP_TO_CONNECTOR_LANGUAGE_MAPPING = { - ConnectorLanguage.PYTHON: python_connectors.FormatConnectorCode, - ConnectorLanguage.LOW_CODE: python_connectors.FormatConnectorCode, - ConnectorLanguage.JAVA: java_connectors.FormatConnectorCode, -} - - -class ExportChanges(Step): - title = "Export changes to local repository" - - async def _run(self, changed_directory: dagger.Directory, changed_directory_path_in_repo: str) -> StepResult: - await changed_directory.export(changed_directory_path_in_repo) - return StepResult(self, StepStatus.SUCCESS, stdout=f"Changes exported to {changed_directory_path_in_repo}") - - -async def run_connector_format_pipeline(context: ConnectorContext) -> ConnectorReport: - """Run a format pipeline for a single connector. - - Args: - context (ConnectorContext): The initialized connector context. - - Returns: - ConnectorReport: The reports holding formats results. - """ - steps_results = [] - async with context: - FormatConnectorCode = FORMATTING_STEP_TO_CONNECTOR_LANGUAGE_MAPPING.get(context.connector.language) - if not FormatConnectorCode: - raise NoFormatStepForLanguageError( - f"No formatting step found for connector {context.connector.technical_name} with language {context.connector.language}" - ) - format_connector_code_result = await FormatConnectorCode(context).run() - steps_results.append(format_connector_code_result) - - if context.is_local: - export_changes_results = await ExportChanges(context).run( - format_connector_code_result.output_artifact, str(context.connector.code_directory) - ) - steps_results.append(export_changes_results) - else: - git_push_changes_results = await GitPushChanges(context).run( - format_connector_code_result.output_artifact, - str(context.connector.code_directory), - f"Auto format {context.connector.technical_name} code", - skip_ci=True, - ) - steps_results.append(git_push_changes_results) - context.report = ConnectorReport(context, steps_results, name="FORMAT RESULTS") - return context.report - - -async def run_connectors_format_pipelines( - contexts: List[ConnectorContext], - ci_git_user: str, - ci_github_access_token: str, - git_branch: str, - is_local: bool, - execute_timeout: Optional[int], -) -> List[ConnectorContext]: - async with dagger.Connection(dagger.Config(log_output=sys.stderr, execute_timeout=execute_timeout)) as dagger_client: - requires_dind = any(context.connector.language == ConnectorLanguage.JAVA for context in contexts) - dockerd_service = environments.with_global_dockerd_service(dagger_client) - async with anyio.create_task_group() as tg_main: - if requires_dind: - tg_main.start_soon(dockerd_service.sync) - await anyio.sleep(10) # Wait for the docker service to be ready - for context in contexts: - context.dagger_client = dagger_client.pipeline(f"Format - {context.connector.technical_name}") - context.dockerd_service = dockerd_service - await run_connector_format_pipeline(context) - # When the connectors pipelines are done, we can stop the dockerd service - tg_main.cancel_scope.cancel() - - await run_report_complete_pipeline(dagger_client, contexts) - - return contexts diff --git a/airbyte-ci/connectors/pipelines/pipelines/format/java_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/format/java_connectors.py deleted file mode 100644 index 7d73f3ab40fb..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/format/java_connectors.py +++ /dev/null @@ -1,32 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from pipelines.actions import environments -from pipelines.bases import StepResult -from pipelines.gradle import GradleTask -from pipelines.utils import get_exec_result - - -class FormatConnectorCode(GradleTask): - """ - A step to format a Java connector code. - """ - - title = "Format connector code" - - async def _run(self) -> StepResult: - formatted = ( - environments.with_gradle(self.context, self.build_include, bind_to_docker_host=self.BIND_TO_DOCKER_HOST) - .with_mounted_directory(str(self.context.connector.code_directory), await self.context.get_connector_dir()) - .with_exec(["./gradlew", "format"]) - ) - exit_code, stdout, stderr = await get_exec_result(formatted) - return StepResult( - self, - self.get_step_status_from_exit_code(exit_code), - stderr=stderr, - stdout=stdout, - output_artifact=formatted.directory(str(self.context.connector.code_directory)), - ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/format/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/format/python_connectors.py deleted file mode 100644 index e2ebbcf68d8a..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/format/python_connectors.py +++ /dev/null @@ -1,57 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import asyncer -from pipelines.actions import environments -from pipelines.bases import Step, StepResult -from pipelines.utils import with_exit_code, with_stderr, with_stdout - - -class FormatConnectorCode(Step): - """ - A step to format a Python connector code. - """ - - title = "Format connector code" - - @property - def black_cmd(self): - return ["python", "-m", "black", f"--config=/{environments.PYPROJECT_TOML_FILE_PATH}", "."] - - @property - def isort_cmd(self): - return ["python", "-m", "isort", f"--settings-file=/{environments.PYPROJECT_TOML_FILE_PATH}", "."] - - @property - def licenseheaders_cmd(self): - return [ - "python", - "-m", - "licenseheaders", - f"--tmpl=/{environments.LICENSE_SHORT_FILE_PATH}", - "--ext=py", - "--exclude=**/models/__init__.py", - ] - - async def _run(self) -> StepResult: - formatted = ( - environments.with_testing_dependencies(self.context) - .with_mounted_directory("/connector_code", await self.context.get_connector_dir()) - .with_workdir("/connector_code") - .with_exec(self.licenseheaders_cmd) - .with_exec(self.isort_cmd) - .with_exec(self.black_cmd) - ) - async with asyncer.create_task_group() as task_group: - soon_exit_code = task_group.soonify(with_exit_code)(formatted) - soon_stderr = task_group.soonify(with_stderr)(formatted) - soon_stdout = task_group.soonify(with_stdout)(formatted) - - return StepResult( - self, - self.get_step_status_from_exit_code(await soon_exit_code), - stderr=soon_stderr.value, - stdout=soon_stdout.value, - output_artifact=formatted.directory("/connector_code"), - ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/git.py b/airbyte-ci/connectors/pipelines/pipelines/git.py deleted file mode 100644 index acf23c2e8eef..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/git.py +++ /dev/null @@ -1,117 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from dagger import Client, Directory, Secret -from pipelines.actions import environments -from pipelines.bases import Step, StepResult -from pipelines.github import AIRBYTE_GITHUB_REPO - - -class GitPushChanges(Step): - """ - A step to push changes to the remote repository. - """ - - title = "Push changes to the remote repository" - - GITHUB_REPO_URL = f"https://github.com/{AIRBYTE_GITHUB_REPO}.git" - - @property - def ci_git_user(self) -> str: - return self.context.ci_git_user - - @property - def ci_github_access_token(self) -> str: - return self.context.ci_github_access_token - - @property - def dagger_client(self) -> Client: - return self.context.dagger_client - - @property - def git_branch(self) -> str: - return self.context.git_branch - - @property - def authenticated_repo_url(self) -> Secret: - url = self.GITHUB_REPO_URL.replace("https://", f"https://{self.ci_git_user}:{self.ci_github_access_token}@") - return self.dagger_client.set_secret("authenticated_repo_url", url) - - @property - def airbyte_repo(self) -> Directory: - return self.dagger_client.git(self.GITHUB_REPO_URL, keep_git_dir=True).branch(self.git_branch).tree() - - def get_commit_message(self, commit_message: str, skip_ci: bool) -> str: - commit_message = f"🤖 {commit_message}" - return f"{commit_message} [skip ci]" if skip_ci else commit_message - - async def _run( - self, changed_directory: Directory, changed_directory_path: str, commit_message: str, skip_ci: bool = True - ) -> StepResult: - diff = ( - environments.with_git(self.dagger_client, self.context.ci_github_access_token_secret, self.ci_git_user) - .with_secret_variable("AUTHENTICATED_REPO_URL", self.authenticated_repo_url) - .with_mounted_directory("/airbyte", self.airbyte_repo) - .with_workdir("/airbyte") - .with_exec(["git", "checkout", self.git_branch]) - .with_mounted_directory(f"/airbyte/{changed_directory_path}", changed_directory) - .with_exec(["git", "diff", "--name-only"]) - ) - - if not await diff.stdout(): - return self.skip("No changes to push") - - commit_and_push = ( - diff.with_exec(["sh", "-c", "git remote set-url origin $AUTHENTICATED_REPO_URL"]) - .with_exec(["git", "add", "."]) - .with_exec(["git", "commit", "-m", self.get_commit_message(commit_message, skip_ci)]) - .with_exec(["git", "pull", "--rebase", "origin", self.git_branch]) - .with_exec(["git", "push"]) - ) - return await self.get_step_result(commit_and_push) - - -class GitPushEmptyCommit(GitPushChanges): - """ - A step to push an empty commit to the remote repository. - """ - - title = "Push empty commit to the remote repository" - - def __init__(self, dagger_client, ci_git_user, ci_github_access_token, git_branch): - self._dagger_client = dagger_client - self._ci_github_access_token = ci_github_access_token - self._ci_git_user = ci_git_user - self._git_branch = git_branch - self.ci_github_access_token_secret = dagger_client.set_secret("ci_github_access_token", ci_github_access_token) - - @property - def dagger_client(self) -> Client: - return self._dagger_client - - @property - def ci_git_user(self) -> str: - return self._ci_git_user - - @property - def ci_github_access_token(self) -> Secret: - return self._ci_github_access_token - - @property - def git_branch(self) -> str: - return self._git_branch - - async def _run(self, commit_message: str, skip_ci: bool = True) -> StepResult: - push_empty_commit = ( - environments.with_git(self.dagger_client, self.ci_github_access_token_secret, self.ci_git_user) - .with_secret_variable("AUTHENTICATED_REPO_URL", self.authenticated_repo_url) - .with_mounted_directory("/airbyte", self.airbyte_repo) - .with_workdir("/airbyte") - .with_exec(["git", "checkout", self.git_branch]) - .with_exec(["sh", "-c", "git remote set-url origin $AUTHENTICATED_REPO_URL"]) - .with_exec(["git", "commit", "--allow-empty", "-m", self.get_commit_message(commit_message, skip_ci)]) - .with_exec(["git", "pull", "--rebase", "origin", self.git_branch]) - .with_exec(["git", "push"]) - ) - return await self.get_step_result(push_empty_commit) diff --git a/airbyte-ci/connectors/pipelines/pipelines/gradle.py b/airbyte-ci/connectors/pipelines/pipelines/gradle.py deleted file mode 100644 index c1dbf283d7a5..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/gradle.py +++ /dev/null @@ -1,132 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from __future__ import annotations - -from abc import ABC -from typing import ClassVar, Tuple - -from dagger import CacheVolume, Container, Directory, QueryError -from pipelines import consts -from pipelines.actions import environments -from pipelines.bases import Step, StepResult -from pipelines.contexts import PipelineContext - - -class GradleTask(Step, ABC): - """ - A step to run a Gradle task. - - Attributes: - task_name (str): The Gradle task name to run. - title (str): The step title. - """ - - DEFAULT_TASKS_TO_EXCLUDE = ["airbyteDocker"] - BIND_TO_DOCKER_HOST = True - gradle_task_name: ClassVar - gradle_task_options: Tuple[str, ...] = () - - def __init__(self, context: PipelineContext, with_java_cdk_snapshot: bool = True) -> None: - super().__init__(context) - self.with_java_cdk_snapshot = with_java_cdk_snapshot - - @property - def connector_java_build_cache(self) -> CacheVolume: - return self.context.dagger_client.cache_volume("connector_java_build_cache") - - @property - def build_include(self) -> List[str]: - """Retrieve the list of source code directory required to run a Java connector Gradle task. - - The list is different according to the connector type. - - Returns: - List[str]: List of directories or files to be mounted to the container to run a Java connector Gradle task. - """ - return [ - str(dependency_directory) - for dependency_directory in self.context.connector.get_local_dependency_paths(with_test_dependencies=True) - ] - - async def _get_patched_build_src_dir(self) -> Directory: - """Patch some gradle plugins. - - Returns: - Directory: The patched buildSrc directory - """ - - build_src_dir = self.context.get_repo_dir("buildSrc") - cat_gradle_plugin_content = await build_src_dir.file("src/main/groovy/airbyte-connector-acceptance-test.gradle").contents() - # When running integrationTest in Dagger we don't want to run connectorAcceptanceTest - # connectorAcceptanceTest is run in the AcceptanceTest step - cat_gradle_plugin_content = cat_gradle_plugin_content.replace( - "project.integrationTest.dependsOn(project.connectorAcceptanceTest)", "" - ) - return build_src_dir.with_new_file("src/main/groovy/airbyte-connector-acceptance-test.gradle", contents=cat_gradle_plugin_content) - - def _get_gradle_command(self, extra_options: Tuple[str, ...] = ("--no-daemon", "--scan", "--build-cache")) -> List: - command = ( - ["./gradlew"] - + list(extra_options) - + [f":airbyte-integrations:connectors:{self.context.connector.technical_name}:{self.gradle_task_name}"] - + list(self.gradle_task_options) - ) - for task in self.DEFAULT_TASKS_TO_EXCLUDE: - command += ["-x", task] - return command - - async def _run(self) -> StepResult: - includes = self.build_include - if self.with_java_cdk_snapshot: - includes + ["./airbyte-cdk/java/airbyte-cdk/**"] - - connector_under_test = ( - environments.with_gradle(self.context, includes, bind_to_docker_host=self.BIND_TO_DOCKER_HOST) - .with_mounted_directory(str(self.context.connector.code_directory), await self.context.get_connector_dir()) - .with_mounted_directory("buildSrc", await self._get_patched_build_src_dir()) - # Disable the Ryuk container because it needs privileged docker access that does not work: - .with_env_variable("TESTCONTAINERS_RYUK_DISABLED", "true") - .with_(environments.mounted_connector_secrets(self.context, f"{self.context.connector.code_directory}/secrets")) - ) - if self.with_java_cdk_snapshot: - connector_under_test = connector_under_test.with_exec(["./gradlew", ":airbyte-cdk:java:airbyte-cdk:publishSnapshotIfNeeded"]) - connector_under_test = connector_under_test.with_exec(self._get_gradle_command()) - - results = await self.get_step_result(connector_under_test) - - await self._export_gradle_dependency_cache(connector_under_test) - return results - - async def _export_gradle_dependency_cache(self, gradle_container: Container) -> Container: - """Export the Gradle writable dependency cache to the read-only dependency cache path. - The read-only dependency cache is persisted thanks to mounted cache volumes in environments.with_gradle(). - You can read more about Shared readonly cache here: https://docs.gradle.org/current/userguide/dependency_resolution.html#sub:shared-readonly-cache - Args: - gradle_container (Container): The Gradle container. - - Returns: - Container: The Gradle container, with the updated cache. - """ - try: - cache_dirs = await gradle_container.directory(consts.GRADLE_CACHE_PATH).entries() - except QueryError: - cache_dirs = [] - if "modules-2" in cache_dirs: - with_cache = gradle_container.with_exec( - [ - "rsync", - "--archive", - "--quiet", - "--times", - "--exclude", - "*.lock", - "--exclude", - "gc.properties", - f"{consts.GRADLE_CACHE_PATH}/modules-2/", - f"{consts.GRADLE_READ_ONLY_DEPENDENCY_CACHE_PATH}/modules-2/", - ] - ) - return await with_cache - return gradle_container diff --git a/airbyte-ci/connectors/pipelines/pipelines/hacks.py b/airbyte-ci/connectors/pipelines/pipelines/hacks.py index b6a4d79cce9d..e11cc5664b23 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/hacks.py +++ b/airbyte-ci/connectors/pipelines/pipelines/hacks.py @@ -8,70 +8,14 @@ from typing import TYPE_CHECKING, Callable, List -import requests -from connector_ops.utils import ConnectorLanguage -from dagger import DaggerError +from pipelines import consts if TYPE_CHECKING: - from dagger import Client, Container, Directory - from pipelines.contexts import ConnectorContext + from dagger import Container + from pipelines.airbyte_ci.connectors.context import ConnectorContext -LINES_TO_REMOVE_FROM_GRADLE_FILE = [ - # Do not build normalization with Gradle - we build normalization with Dagger in the BuildOrPullNormalization step. - "project(':airbyte-integrations:bases:base-normalization').airbyteDocker.output", -] - - -async def _patch_gradle_file(context: ConnectorContext, connector_dir: Directory) -> Directory: - """Patch the build.gradle file of the connector under test by removing the lines declared in LINES_TO_REMOVE_FROM_GRADLE_FILE. - - Underlying issue: - Java connectors build.gradle declare a dependency to the normalization module. - It means every time we test a java connector the normalization is built. - This is time consuming and not required as normalization is now baked in containers. - Normalization is going away soon so hopefully this hack will be removed soon. - - Args: - context (ConnectorContext): The initialized connector context. - connector_dir (Directory): The directory containing the build.gradle file to patch. - Returns: - Directory: The directory containing the patched gradle file. - """ - if context.connector.language is not ConnectorLanguage.JAVA: - context.logger.info(f"Connector language {context.connector.language} does not require a patched build.gradle file.") - return connector_dir - - try: - gradle_file_content = await connector_dir.file("build.gradle").contents() - except DaggerError: - context.logger.info("Could not find build.gradle file in the connector directory. Skipping patching.") - return connector_dir - - context.logger.warn("Patching build.gradle file to remove normalization build.") - - patched_gradle_file = [] - - for line in gradle_file_content.splitlines(): - if not any(line_to_remove in line for line_to_remove in LINES_TO_REMOVE_FROM_GRADLE_FILE): - patched_gradle_file.append(line) - return connector_dir.with_new_file("build.gradle", contents="\n".join(patched_gradle_file)) - - -async def patch_connector_dir(context: ConnectorContext, connector_dir: Directory) -> Directory: - """Patch a connector directory: patch cat config, gradle file and dockerfile. - - Args: - context (ConnectorContext): The initialized connector context. - connector_dir (Directory): The directory containing the connector to patch. - Returns: - Directory: The directory containing the patched connector. - """ - patched_connector_dir = await _patch_gradle_file(context, connector_dir) - return patched_connector_dir.with_timestamps(1) - - -async def cache_latest_cdk(dagger_client: Client, pip_cache_volume_name: str = "pip_cache") -> None: +async def cache_latest_cdk(context: ConnectorContext) -> None: """ Download the latest CDK version to update the pip cache. @@ -89,28 +33,21 @@ async def cache_latest_cdk(dagger_client: Client, pip_cache_volume_name: str = " Args: dagger_client (Client): Dagger client. """ - - # We get the latest version of the CDK from PyPI using their API. - # It allows us to explicitly install the latest version of the CDK in the container - # while keeping buildkit layer caching when the version value does not change. - # In other words: we only update the pip cache when the latest CDK version changes. - # When the CDK version does not change, the pip cache is not updated as the with_exec command remains the same. - cdk_pypi_url = "https://pypi.org/pypi/airbyte-cdk/json" - response = requests.get(cdk_pypi_url) - response.raise_for_status() - package_info = response.json() - cdk_latest_version = package_info["info"]["version"] + # We want the CDK to be re-downloaded on every run per connector to ensure we always get the latest version. + # But we don't want to invalidate the pip cache on every run because it could lead to a different CDK version installed on different architecture build. + cachebuster_value = f"{context.connector.technical_name}_{context.pipeline_start_timestamp}" await ( - dagger_client.container() + context.dagger_client.container() .from_("python:3.9-slim") - .with_mounted_cache("/root/.cache/pip", dagger_client.cache_volume(pip_cache_volume_name)) - .with_exec(["pip", "install", "--force-reinstall", f"airbyte-cdk=={cdk_latest_version}"]) + .with_mounted_cache(consts.PIP_CACHE_PATH, context.dagger_client.cache_volume(consts.PIP_CACHE_VOLUME_NAME)) + .with_env_variable("CACHEBUSTER", cachebuster_value) + .with_exec(["pip", "install", "--force-reinstall", "airbyte-cdk", "-vvv"]) .sync() ) -def never_fail_exec(command: List[str]) -> Callable: +def never_fail_exec(command: List[str]) -> Callable[[Container], Container]: """ Wrap a command execution with some bash sugar to always exit with a 0 exit code but write the actual exit code to a file. @@ -129,7 +66,7 @@ def never_fail_exec(command: List[str]) -> Callable: Callable: _description_ """ - def never_fail_exec_inner(container: Container): + def never_fail_exec_inner(container: Container) -> Container: return container.with_exec(["sh", "-c", f"{' '.join(command)}; echo $? > /exit_code"], skip_entrypoint=True) return never_fail_exec_inner diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/cache_keys.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/cache_keys.py new file mode 100644 index 000000000000..c5fd9b75130a --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/cache_keys.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from pipelines.helpers.utils import slugify + + +def get_black_cache_key(black_version: str) -> str: + return slugify(f"black-{black_version}") + + +def get_prettier_cache_key(prettier_version: str) -> str: + return slugify(f"prettier-{prettier_version}") diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py new file mode 100644 index 000000000000..4f601b7e83dc --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/cli.py @@ -0,0 +1,100 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from dataclasses import dataclass +from logging import Logger +from typing import Any, List, Optional + +import asyncclick as click +import asyncer +from jinja2 import Template +from pipelines.models.steps import CommandResult + +ALL_RESULTS_KEY = "_run_all_results" + +SUMMARY_TEMPLATE_STR = """ +{% if command_results %} +Summary of commands results +======================== +{% for command_result in command_results %} +{{ '✅' if command_result.success else '❌' }} {{ command_result.command.name }} +{% endfor %} +{% endif %} +""" + +DETAILS_TEMPLATE_STR = """ +{% for command_result in command_results %} +{% if command_result.stdout or command_result.stderr %} +================================= + +Details for {{ command_result.command.name }} command +{% if command_result.stdout %} +STDOUT: +{{ command_result.stdout }} +{% endif %} +{% if command_result.stderr %} +STDERR: +{{ command_result.stderr }} +{% endif %} +{% endif %} +{% endfor %} + +""" + + +@dataclass +class LogOptions: + quiet: bool = True + help_message: Optional[str] = None + + +def log_command_results( + ctx: click.Context, command_results: List[CommandResult], logger: Logger, options: LogOptions = LogOptions() +) -> None: + """ + Log the output of the subcommands run by `run_all_subcommands`. + """ + + if not options.quiet: + details_template = Template(DETAILS_TEMPLATE_STR) + details_message = details_template.render(command_results=command_results) + logger.info(details_message) + + summary_template = Template(SUMMARY_TEMPLATE_STR) + summary_message = summary_template.render(command_results=command_results) + logger.info(summary_message) + + if options.help_message: + logger.info(options.help_message) + + +async def invoke_commands_concurrently(ctx: click.Context, commands: List[click.Command]) -> List[Any]: + """ + Run click commands concurrently and return a list of their return values. + """ + + soon_command_executions_results = [] + async with asyncer.create_task_group() as command_task_group: + for command in commands: + soon_command_execution_result = command_task_group.soonify(command.invoke)(ctx) + soon_command_executions_results.append(soon_command_execution_result) + return [r.value for r in soon_command_executions_results] + + +async def invoke_commands_sequentially(ctx: click.Context, commands: List[click.Command]) -> List[Any]: + """ + Run click commands sequentially and return a list of their return values. + """ + command_executions_results = [] + for command in commands: + command_executions_results.append(await command.invoke(ctx)) + return command_executions_results + + +def get_all_sibling_commands(ctx: click.Context) -> List[click.Command]: + """ + Get all sibling commands of the current command. + """ + return [c for c in ctx.parent.command.commands.values() if c.name != ctx.command.name] # type: ignore diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/metadata_change_helpers.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/metadata_change_helpers.py new file mode 100644 index 000000000000..6dd55b93bffb --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/metadata_change_helpers.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from pathlib import Path + +import yaml # type: ignore +from dagger import Directory + +# Helpers + + +async def get_current_metadata(repo_dir: Directory, metadata_path: Path) -> dict: + return yaml.safe_load(await repo_dir.file(str(metadata_path)).contents()) + + +def get_repo_dir_with_updated_metadata(repo_dir: Directory, metadata_path: Path, updated_metadata: dict) -> Directory: + return repo_dir.with_new_file(str(metadata_path), contents=yaml.safe_dump(updated_metadata)) + + +def get_current_version(current_metadata: dict) -> str: + return current_metadata.get("data", {}).get("dockerImageTag") diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/modifed.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/modifed.py new file mode 100644 index 000000000000..b58258ef3465 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/connectors/modifed.py @@ -0,0 +1,70 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import dataclass, field +from pathlib import Path +from typing import FrozenSet, Set, Union + +from connector_ops.utils import Connector # type: ignore +from pipelines import main_logger +from pipelines.helpers.utils import IGNORED_FILE_EXTENSIONS, METADATA_FILE_NAME + + +def get_connector_modified_files(connector: Connector, all_modified_files: Set[Path]) -> FrozenSet[Path]: + connector_modified_files = set() + for modified_file in all_modified_files: + modified_file_path = Path(modified_file) + if modified_file_path.is_relative_to(connector.code_directory): + connector_modified_files.add(modified_file) + return frozenset(connector_modified_files) + + +def _find_modified_connectors( + file_path: Union[str, Path], all_connectors: Set[Connector], dependency_scanning: bool = True +) -> Set[Connector]: + """Find all connectors impacted by the file change.""" + modified_connectors = set() + + for connector in all_connectors: + if Path(file_path).is_relative_to(Path(connector.code_directory)): + main_logger.info(f"Adding connector '{connector}' due to connector file modification: {file_path}.") + modified_connectors.add(connector) + + if dependency_scanning: + for connector_dependency in connector.get_local_dependency_paths(): + if Path(file_path).is_relative_to(Path(connector_dependency)): + # Add the connector to the modified connectors + modified_connectors.add(connector) + main_logger.info(f"Adding connector '{connector}' due to dependency modification: '{file_path}'.") + return modified_connectors + + +def _is_ignored_file(file_path: Union[str, Path]) -> bool: + """Check if the provided file has an ignored extension.""" + return Path(file_path).suffix in IGNORED_FILE_EXTENSIONS + + +def get_modified_connectors(modified_files: Set[Path], all_connectors: Set[Connector], dependency_scanning: bool) -> Set[Connector]: + """Create a mapping of modified connectors (key) and modified files (value). + If dependency scanning is enabled any modification to a dependency will trigger connector pipeline for all connectors that depend on it. + It currently works only for Java connectors . + It's especially useful to trigger tests of strict-encrypt variant when a change is made to the base connector. + Or to tests all jdbc connectors when a change is made to source-jdbc or base-java. + We'll consider extending the dependency resolution to Python connectors once we confirm that it's needed and feasible in term of scale. + """ + # Ignore files with certain extensions + modified_connectors = set() + for modified_file in modified_files: + if not _is_ignored_file(modified_file): + modified_connectors.update(_find_modified_connectors(modified_file, all_connectors, dependency_scanning)) + return modified_connectors + + +@dataclass(frozen=True) +class ConnectorWithModifiedFiles(Connector): + modified_files: FrozenSet[Path] = field(default_factory=frozenset) + + @property + def has_metadata_change(self) -> bool: + return any(path.name == METADATA_FILE_NAME for path in self.modified_files) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/argument_parsing.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/argument_parsing.py new file mode 100644 index 000000000000..af32aa52b213 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/argument_parsing.py @@ -0,0 +1,66 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING + +import asyncclick as click + +if TYPE_CHECKING: + from enum import Enum + from typing import Callable, Dict, Tuple, Type + + from pipelines.models.steps import STEP_PARAMS + +# Pattern for extra param options: --.= +EXTRA_PARAM_PATTERN_FOR_OPTION = re.compile(r"^--([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_-][a-zA-Z0-9_-]*)=([^=]+)$") +# Pattern for extra param flag: --. +EXTRA_PARAM_PATTERN_FOR_FLAG = re.compile(r"^--([a-zA-Z_][a-zA-Z0-9_]*)\.([a-zA-Z_-][a-zA-Z0-9_-]*)$") +EXTRA_PARAM_PATTERN_ERROR_MESSAGE = "The extra flags must be structured as --. for flags or --.= for options. You can use - or -- for option/flag names." + + +def build_extra_params_mapping(SupportedStepIds: Type[Enum]) -> Callable: + def callback(ctx: click.Context, argument: click.core.Argument, raw_extra_params: Tuple[str]) -> Dict[str, STEP_PARAMS]: + """Build a mapping of step id to extra params. + Validate the extra params and raise a ValueError if they are invalid. + Validation rules: + - The extra params must be structured as --.= for options or --. for flags. + - The step id must be one of the existing step ids. + + + Args: + ctx (click.Context): The click context. + argument (click.core.Argument): The click argument. + raw_extra_params (Tuple[str]): The extra params provided by the user. + Raises: + ValueError: Raised if the extra params format is invalid. + ValueError: Raised if the step id in the extra params is not one of the unique steps to run. + + Returns: + Dict[Literal, STEP_PARAMS]: The mapping of step id to extra params. + """ + extra_params_mapping: Dict[str, STEP_PARAMS] = {} + for param in raw_extra_params: + is_flag = "=" not in param + pattern = EXTRA_PARAM_PATTERN_FOR_FLAG if is_flag else EXTRA_PARAM_PATTERN_FOR_OPTION + matches = pattern.match(param) + if not matches: + raise ValueError(f"Invalid parameter {param}. {EXTRA_PARAM_PATTERN_ERROR_MESSAGE}") + if is_flag: + step_name, param_name = matches.groups() + param_value = None + else: + step_name, param_name, param_value = matches.groups() + try: + step_id = SupportedStepIds(step_name).value + except ValueError: + raise ValueError(f"Invalid step name {step_name}, it must be one of {[step_id.value for step_id in SupportedStepIds]}") + + extra_params_mapping.setdefault(step_id, {}).setdefault(param_name, []) + # param_value is None if the param is a flag + if param_value is not None: + extra_params_mapping[step_id][param_name].append(param_value) + return extra_params_mapping + + return callback diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/run_steps.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/run_steps.py new file mode 100644 index 000000000000..5db187af307c --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/execution/run_steps.py @@ -0,0 +1,328 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""The actions package is made to declare reusable pipeline components.""" + +from __future__ import annotations + +import inspect +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Optional, Set, Tuple, Union + +import anyio +import asyncer +from pipelines import main_logger +from pipelines.models.steps import StepStatus + +if TYPE_CHECKING: + from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID + from pipelines.models.steps import STEP_PARAMS, Step, StepResult + + RESULTS_DICT = Dict[str, StepResult] + ARGS_TYPE = Union[Dict, Callable[[RESULTS_DICT], Dict], Awaitable[Dict]] + + +class InvalidStepConfiguration(Exception): + pass + + +def _get_dependency_graph(steps: STEP_TREE) -> Dict[str, List[str]]: + """ + Get the dependency graph of a step tree. + """ + dependency_graph: Dict[str, List[str]] = {} + for step in steps: + if isinstance(step, StepToRun): + dependency_graph[step.id] = step.depends_on + elif isinstance(step, list): + nested_dependency_graph = _get_dependency_graph(list(step)) + dependency_graph = {**dependency_graph, **nested_dependency_graph} + else: + raise Exception(f"Unexpected step type: {type(step)}") + + return dependency_graph + + +def _get_transitive_dependencies_for_step_id( + dependency_graph: Dict[str, List[str]], step_id: str, visited: Optional[Set[str]] = None +) -> List[str]: + """Get the transitive dependencies for a step id. + + Args: + dependency_graph (Dict[str, str]): The dependency graph to use. + step_id (str): The step id to get the transitive dependencies for. + visited (Optional[Set[str]], optional): The set of visited step ids. Defaults to None. + + Returns: + List[str]: List of transitive dependencies as step ids. + """ + if visited is None: + visited = set() + + if step_id not in visited: + visited.add(step_id) + + dependencies: List[str] = dependency_graph.get(step_id, []) + for dependency in dependencies: + dependencies.extend(_get_transitive_dependencies_for_step_id(dependency_graph, dependency, visited)) + + return dependencies + else: + return [] + + +@dataclass +class RunStepOptions: + """Options for the run_step function.""" + + fail_fast: bool = True + skip_steps: List[str] = field(default_factory=list) + keep_steps: List[str] = field(default_factory=list) + log_step_tree: bool = True + concurrency: int = 10 + step_params: Dict[CONNECTOR_TEST_STEP_ID, STEP_PARAMS] = field(default_factory=dict) + + def __post_init__(self) -> None: + if self.skip_steps and self.keep_steps: + raise ValueError("Cannot use both skip_steps and keep_steps at the same time") + + def get_step_ids_to_skip(self, runnables: STEP_TREE) -> List[str]: + if self.skip_steps: + return self.skip_steps + if self.keep_steps: + step_ids_to_keep = set(self.keep_steps) + dependency_graph = _get_dependency_graph(runnables) + all_step_ids = set(dependency_graph.keys()) + for step_id in self.keep_steps: + step_ids_to_keep.update(_get_transitive_dependencies_for_step_id(dependency_graph, step_id)) + return list(all_step_ids - step_ids_to_keep) + return [] + + +@dataclass(frozen=True) +class StepToRun: + """ + A class to wrap a Step with its id and args. + + Used to coordinate the execution of multiple steps inside a pipeline. + """ + + id: CONNECTOR_TEST_STEP_ID + step: Step + args: ARGS_TYPE = field(default_factory=dict) + depends_on: List[str] = field(default_factory=list) + + +STEP_TREE = List[StepToRun | List[StepToRun]] + + +async def evaluate_run_args(args: ARGS_TYPE, results: RESULTS_DICT) -> Dict: + """ + Evaluate the args of a StepToRun using the results of previous steps. + """ + if inspect.iscoroutinefunction(args): + return await args(results) + elif callable(args): + return args(results) + elif isinstance(args, dict): + return args + + raise TypeError(f"Unexpected args type: {type(args)}") + + +def _skip_remaining_steps(remaining_steps: STEP_TREE) -> RESULTS_DICT: + """ + Skip all remaining steps. + """ + skipped_results: Dict[str, StepResult] = {} + for runnable_step in remaining_steps: + if isinstance(runnable_step, StepToRun): + skipped_results[runnable_step.id] = runnable_step.step.skip() + elif isinstance(runnable_step, list): + nested_skipped_results = _skip_remaining_steps(list(runnable_step)) + skipped_results = {**skipped_results, **nested_skipped_results} + else: + raise Exception(f"Unexpected step type: {type(runnable_step)}") + + return skipped_results + + +def _step_dependencies_succeeded(step_to_eval: StepToRun, results: RESULTS_DICT) -> bool: + """ + Check if all dependencies of a step have succeeded. + """ + main_logger.info(f"Checking if dependencies {step_to_eval.depends_on} have succeeded") + + # Check if all depends_on keys are in the results dict + # If not, that means a step has not been run yet + # Implying that the order of the steps are not correct + for step_id in step_to_eval.depends_on: + if step_id not in results: + raise InvalidStepConfiguration( + f"Step {step_to_eval.id} depends on {step_id} which has not been run yet. This implies that the order of the steps is not correct. Please check that the steps are in the correct order." + ) + + return all(results[step_id] and results[step_id].status is StepStatus.SUCCESS for step_id in step_to_eval.depends_on) + + +def _filter_skipped_steps(steps_to_evaluate: STEP_TREE, skip_steps: List[str], results: RESULTS_DICT) -> Tuple[STEP_TREE, RESULTS_DICT]: + """ + Filter out steps that should be skipped. + + Either because they are in the skip list or because one of their dependencies failed. + """ + steps_to_run: STEP_TREE = [] + for step_to_eval in steps_to_evaluate: + + # ignore nested steps + if isinstance(step_to_eval, list): + steps_to_run.append(step_to_eval) + continue + + # skip step if its id is in the skip list + if step_to_eval.id in skip_steps: + main_logger.info(f"Skipping step {step_to_eval.id}") + results[step_to_eval.id] = step_to_eval.step.skip("Skipped by user") + + # skip step if a dependency failed + elif not _step_dependencies_succeeded(step_to_eval, results): + main_logger.info( + f"Skipping step {step_to_eval.id} because one of the dependencies have not been met: {step_to_eval.depends_on}" + ) + results[step_to_eval.id] = step_to_eval.step.skip("Skipped because a dependency was not met") + + else: + steps_to_run.append(step_to_eval) + + return steps_to_run, results + + +def _get_next_step_group(steps: STEP_TREE) -> Tuple[STEP_TREE, STEP_TREE]: + """ + Get the next group of steps to run concurrently. + """ + if not steps: + return [], [] + + if isinstance(steps[0], list): + return list(steps[0]), list(steps[1:]) + else: + # Termination case: if the next step is not a list that means we have reached the max depth + return steps, [] + + +def _log_step_tree(step_tree: STEP_TREE, options: RunStepOptions, depth: int = 0) -> None: + """ + Log the step tree to the console. + + e.g. + Step tree + - step1 + - step2 + - step3 + - step4 (skip) + - step5 + - step6 + """ + indent = " " + for steps in step_tree: + if isinstance(steps, list): + _log_step_tree(list(steps), options, depth + 1) + else: + if steps.id in options.skip_steps: + main_logger.info(f"{indent * depth}- {steps.id} (skip)") + else: + main_logger.info(f"{indent * depth}- {steps.id}") + + +async def run_steps( + runnables: STEP_TREE, + results: RESULTS_DICT = {}, + options: RunStepOptions = RunStepOptions(), +) -> RESULTS_DICT: + """Run multiple steps sequentially, or in parallel if steps are wrapped into a sublist. + + Examples + -------- + >>> from pipelines.models.steps import Step, StepResult, StepStatus + >>> class TestStep(Step): + ... async def _run(self) -> StepResult: + ... return StepResult(self, StepStatus.SUCCESS) + >>> steps = [ + ... StepToRun(id="step1", step=TestStep()), + ... [ + ... StepToRun(id="step2", step=TestStep()), + ... StepToRun(id="step3", step=TestStep()), + ... ], + ... StepToRun(id="step4", step=TestStep()), + ... ] + >>> results = await run_steps(steps) + >>> results["step1"].status + + >>> results["step2"].status + + >>> results["step3"].status + + >>> results["step4"].status + + + + Args: + runnables (List[StepToRun]): List of steps to run. + results (RESULTS_DICT, optional): Dictionary of step results, used for recursion. + + Returns: + RESULTS_DICT: Dictionary of step results. + """ + # If there are no steps to run, return the results + if not runnables: + return results + + step_ids_to_skip = options.get_step_ids_to_skip(runnables) + # Log the step tree + if options.log_step_tree: + main_logger.info(f"STEP TREE: {runnables}") + _log_step_tree(runnables, options) + options.log_step_tree = False + + # If any of the previous steps failed, skip the remaining steps + if options.fail_fast and any(result.status is StepStatus.FAILURE for result in results.values()): + skipped_results = _skip_remaining_steps(runnables) + return {**results, **skipped_results} + + # Pop the next step to run + steps_to_evaluate, remaining_steps = _get_next_step_group(runnables) + + # Remove any skipped steps + steps_to_run, results = _filter_skipped_steps(steps_to_evaluate, step_ids_to_skip, results) + + # Run all steps in list concurrently + semaphore = anyio.Semaphore(options.concurrency) + async with semaphore: + async with asyncer.create_task_group() as task_group: + tasks = [] + for step_to_run in steps_to_run: + # if the step to run is a list, run it in parallel + if isinstance(step_to_run, list): + tasks.append(task_group.soonify(run_steps)(list(step_to_run), results, options)) + else: + step_args = await evaluate_run_args(step_to_run.args, results) + step_to_run.step.extra_params = options.step_params.get(step_to_run.id, {}) + main_logger.info(f"QUEUING STEP {step_to_run.id}") + tasks.append(task_group.soonify(step_to_run.step.run)(**step_args)) + + # Apply new results + new_results: Dict[str, Any] = {} + for i, task in enumerate(tasks): + step_to_run = steps_to_run[i] + if isinstance(step_to_run, list): + new_results = {**new_results, **task.value} + else: + new_results[step_to_run.id] = task.value + + return await run_steps( + runnables=remaining_steps, + results={**results, **new_results}, + options=options, + ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/gcs.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/gcs.py new file mode 100644 index 000000000000..71fbce43b44b --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/gcs.py @@ -0,0 +1,49 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +from pathlib import Path +from typing import Tuple + +from google.cloud import storage # type: ignore +from google.oauth2 import service_account # type: ignore +from pipelines import main_logger +from pipelines.consts import GCS_PUBLIC_DOMAIN + + +def upload_to_gcs(file_path: Path, bucket_name: str, object_name: str, credentials: str) -> Tuple[str, str]: + """Upload a file to a GCS bucket. + + Args: + file_path (Path): The path to the file to upload. + bucket_name (str): The name of the GCS bucket. + object_name (str): The name of the object in the GCS bucket. + credentials (str): The GCS credentials as a JSON string. + """ + # Exit early if file does not exist + if not file_path.exists(): + main_logger.warning(f"File {file_path} does not exist. Skipping upload to GCS.") + return "", "" + + credentials = service_account.Credentials.from_service_account_info(json.loads(credentials)) + client = storage.Client(credentials=credentials) + bucket = client.get_bucket(bucket_name) + blob = bucket.blob(object_name) + blob.upload_from_filename(str(file_path)) + gcs_uri = f"gs://{bucket_name}/{object_name}" + public_url = f"{GCS_PUBLIC_DOMAIN}/{bucket_name}/{object_name}" + return gcs_uri, public_url + + +def sanitize_gcs_credentials(raw_value: str) -> str: + """Try to parse the raw string input that should contain a json object with the GCS credentials. + It will raise an exception if the parsing fails and help us to fail fast on invalid credentials input. + + Args: + raw_value (str): A string representing a json object with the GCS credentials. + + Returns: + str: The raw value string if it was successfully parsed. + """ + return json.dumps(json.loads(raw_value)) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/git.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/git.py new file mode 100644 index 000000000000..fee2f7a0708f --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/git.py @@ -0,0 +1,85 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import functools +from typing import Set + +import git +from dagger import Connection +from pipelines.dagger.containers.git import checked_out_git_container +from pipelines.helpers.utils import DAGGER_CONFIG, DIFF_FILTER + + +def get_current_git_revision() -> str: # noqa D103 + return git.Repo(search_parent_directories=True).head.object.hexsha + + +def get_current_git_branch() -> str: # noqa D103 + return git.Repo(search_parent_directories=True).active_branch.name + + +async def get_modified_files_in_branch_remote( + current_git_branch: str, current_git_revision: str, diffed_branch: str = "origin/master" +) -> Set[str]: + """Use git diff to spot the modified files on the remote branch.""" + async with Connection(DAGGER_CONFIG) as dagger_client: + container = await checked_out_git_container(dagger_client, current_git_branch, current_git_revision, diffed_branch) + modified_files = await container.with_exec( + ["diff", f"--diff-filter={DIFF_FILTER}", "--name-only", f"{diffed_branch}...{current_git_branch}"] + ).stdout() + return set(modified_files.split("\n")) + + +def get_modified_files_local(current_git_revision: str, diffed: str = "master") -> Set[str]: + """Use git diff and git status to spot the modified files in the local repo.""" + airbyte_repo = git.Repo() + modified_files = airbyte_repo.git.diff(f"--diff-filter={DIFF_FILTER}", "--name-only", f"{diffed}...{current_git_revision}").split("\n") + status_output = airbyte_repo.git.status("--porcelain") + for not_committed_change in status_output.split("\n"): + file_path = not_committed_change.strip().split(" ")[-1] + if file_path: + modified_files.append(file_path) + return set(modified_files) + + +async def get_modified_files_in_branch( + current_git_branch: str, current_git_revision: str, diffed_branch: str, is_local: bool = True +) -> Set[str]: + """Retrieve the list of modified files on the branch.""" + if is_local: + return get_modified_files_local(current_git_revision, diffed_branch) + else: + return await get_modified_files_in_branch_remote(current_git_branch, current_git_revision, diffed_branch) + + +async def get_modified_files_in_commit_remote(current_git_branch: str, current_git_revision: str) -> Set[str]: + async with Connection(DAGGER_CONFIG) as dagger_client: + container = await checked_out_git_container(dagger_client, current_git_branch, current_git_revision) + modified_files = await container.with_exec(["diff-tree", "--no-commit-id", "--name-only", current_git_revision, "-r"]).stdout() + return set(modified_files.split("\n")) + + +def get_modified_files_in_commit_local(current_git_revision: str) -> Set[str]: + airbyte_repo = git.Repo() + modified_files = airbyte_repo.git.diff_tree("--no-commit-id", "--name-only", current_git_revision, "-r").split("\n") + return set(modified_files) + + +async def get_modified_files_in_commit(current_git_branch: str, current_git_revision: str, is_local: bool = True) -> Set[str]: + if is_local: + return get_modified_files_in_commit_local(current_git_revision) + else: + return await get_modified_files_in_commit_remote(current_git_branch, current_git_revision) + + +@functools.cache +def get_git_repo() -> git.Repo: + """Retrieve the git repo.""" + return git.Repo(search_parent_directories=True) + + +@functools.cache +def get_git_repo_path() -> str: + """Retrieve the git repo path.""" + return str(get_git_repo().working_tree_dir) diff --git a/airbyte-ci/connectors/pipelines/pipelines/github.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/github.py similarity index 90% rename from airbyte-ci/connectors/pipelines/pipelines/github.py rename to airbyte-ci/connectors/pipelines/pipelines/helpers/github.py index fd6bb7e47530..867c0fa896b7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/github.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/github.py @@ -9,9 +9,9 @@ import os from typing import TYPE_CHECKING, Optional -from connector_ops.utils import console +from connector_ops.utils import console # type: ignore from pipelines import main_logger -from pipelines.bases import CIContext +from pipelines.consts import CIContext if TYPE_CHECKING: from logging import Logger @@ -33,8 +33,15 @@ def safe_log(logger: Optional[Logger], message: str, level: str = "info") -> Non def update_commit_status_check( - sha: str, state: str, target_url: str, description: str, context: str, is_optional=False, should_send=True, logger: Logger = None -): + sha: str, + state: str, + target_url: str, + description: str, + context: str, + is_optional: bool = False, + should_send: bool = True, + logger: Optional[Logger] = None, +) -> None: """Call the GitHub API to create commit status check. Args: @@ -77,7 +84,7 @@ def update_commit_status_check( safe_log(logger, f"Created {state} status for commit {sha} on Github in {context} context with desc: {description}.") -def get_pull_request(pull_request_number: int, github_access_token: str) -> PullRequest: +def get_pull_request(pull_request_number: int, github_access_token: str) -> PullRequest.PullRequest: """Get a pull request object from its number. Args: @@ -91,7 +98,8 @@ def get_pull_request(pull_request_number: int, github_access_token: str) -> Pull return airbyte_repo.get_pull(pull_request_number) -def update_global_commit_status_check_for_tests(click_context: dict, github_state: str, logger: Logger = None): +def update_global_commit_status_check_for_tests(click_context: dict, github_state: str, logger: Optional[Logger] = None) -> None: + update_commit_status_check( click_context["git_revision"], github_state, diff --git a/airbyte-ci/connectors/pipelines/pipelines/sentry_utils.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/sentry_utils.py similarity index 75% rename from airbyte-ci/connectors/pipelines/pipelines/sentry_utils.py rename to airbyte-ci/connectors/pipelines/pipelines/helpers/sentry_utils.py index da36bb015ebd..fb1a44138a60 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/sentry_utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/sentry_utils.py @@ -1,24 +1,33 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from __future__ import annotations import importlib.metadata import os +from typing import TYPE_CHECKING import sentry_sdk -from connector_ops.utils import Connector +from connector_ops.utils import Connector # type: ignore +if TYPE_CHECKING: + from typing import Any, Callable, Dict, Optional -def initialize(): + from asyncclick import Command, Context + from pipelines.models.steps import Step + + +def initialize() -> None: if "SENTRY_DSN" in os.environ: sentry_sdk.init( dsn=os.environ.get("SENTRY_DSN"), + environment=os.environ.get("SENTRY_ENVIRONMENT") or "production", before_send=before_send, release=f"pipelines@{importlib.metadata.version('pipelines')}", ) -def before_send(event, hint): +def before_send(event: Dict[str, Any], hint: Dict[str, Any]) -> Optional[Dict[str, Any]]: # Ignore logged errors that do not contain an exception if "log_record" in hint and "exc_info" not in hint: return None @@ -26,8 +35,8 @@ def before_send(event, hint): return event -def with_step_context(func): - def wrapper(self, *args, **kwargs): +def with_step_context(func: Callable) -> Callable: + def wrapper(self: Step, *args: Any, **kwargs: Any) -> Step: with sentry_sdk.configure_scope() as scope: step_name = self.__class__.__name__ scope.set_tag("pipeline_step", step_name) @@ -61,8 +70,8 @@ def wrapper(self, *args, **kwargs): return wrapper -def with_command_context(func): - def wrapper(self, ctx, *args, **kwargs): +def with_command_context(func: Callable) -> Callable: + def wrapper(self: Command, ctx: Context, *args: Any, **kwargs: Any) -> Command: with sentry_sdk.configure_scope() as scope: scope.set_tag("pipeline_command", self.name) scope.set_context( diff --git a/airbyte-ci/connectors/pipelines/pipelines/slack.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py similarity index 94% rename from airbyte-ci/connectors/pipelines/pipelines/slack.py rename to airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py index ecc4e23e0996..affc981a60de 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/slack.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py @@ -4,7 +4,7 @@ import json -import requests +import requests # type: ignore from pipelines import main_logger diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/steps.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/steps.py deleted file mode 100644 index c2456122778b..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/steps.py +++ /dev/null @@ -1,62 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -"""The actions package is made to declare reusable pipeline components.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, List, Tuple, Union - -import asyncer -from pipelines.bases import Step, StepStatus - -if TYPE_CHECKING: - from pipelines.bases import StepResult - - -async def run_steps( - steps_and_run_args: List[Union[Step, Tuple[Step, Tuple]] | List[Union[Step, Tuple[Step, Tuple]]]], results: List[StepResult] = [] -) -> List[StepResult]: - """Run multiple steps sequentially, or in parallel if steps are wrapped into a sublist. - - Args: - steps_and_run_args (List[Union[Step, Tuple[Step, Tuple]] | List[Union[Step, Tuple[Step, Tuple]]]]): List of steps to run, if steps are wrapped in a sublist they will be executed in parallel. run function arguments can be passed as a tuple along the Step instance. - results (List[StepResult], optional): List of step results, used for recursion. - - Returns: - List[StepResult]: List of step results. - """ - # If there are no steps to run, return the results - if not steps_and_run_args: - return results - - # If any of the previous steps failed, skip the remaining steps - if any(result.status is StepStatus.FAILURE for result in results): - skipped_results = [] - for step_and_run_args in steps_and_run_args: - if isinstance(step_and_run_args, Tuple): - skipped_results.append(step_and_run_args[0].skip()) - else: - skipped_results.append(step_and_run_args.skip()) - return results + skipped_results - - # Pop the next step to run - steps_to_run, remaining_steps = steps_and_run_args[0], steps_and_run_args[1:] - - # wrap the step in a list if it is not already (allows for parallel steps) - if not isinstance(steps_to_run, list): - steps_to_run = [steps_to_run] - - async with asyncer.create_task_group() as task_group: - tasks = [] - for step in steps_to_run: - if isinstance(step, Step): - tasks.append(task_group.soonify(step.run)()) - elif isinstance(step, Tuple) and isinstance(step[0], Step) and isinstance(step[1], Tuple): - step, run_args = step - tasks.append(task_group.soonify(step.run)(*run_args)) - - new_results = [task.value for task in tasks] - - return await run_steps(remaining_steps, results + new_results) diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py new file mode 100644 index 000000000000..c796e9f6e43b --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/utils.py @@ -0,0 +1,328 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This module groups util function used in pipelines.""" +from __future__ import annotations + +import contextlib +import datetime +import os +import re +import sys +import unicodedata +from io import TextIOWrapper +from pathlib import Path +from typing import TYPE_CHECKING + +import anyio +import asyncclick as click +import asyncer +from dagger import Client, Config, Container, ExecError, File, ImageLayerCompression, Platform, Secret +from more_itertools import chunked + +if TYPE_CHECKING: + from typing import Any, Callable, Generator, List, Optional, Set, Tuple + + from pipelines.airbyte_ci.connectors.context import ConnectorContext + +DAGGER_CONFIG = Config(log_output=sys.stderr) +AIRBYTE_REPO_URL = "https://github.com/airbytehq/airbyte.git" +METADATA_FILE_NAME = "metadata.yaml" +METADATA_ICON_FILE_NAME = "icon.svg" +DIFF_FILTER = "MADRT" # Modified, Added, Deleted, Renamed, Type changed +IGNORED_FILE_EXTENSIONS = [".md"] + + +# This utils will probably be redundant once https://github.com/dagger/dagger/issues/3764 is implemented +async def check_path_in_workdir(container: Container, path: str) -> bool: + """Check if a local path is mounted to the working directory of a container. + + Args: + container (Container): The container on which we want the check the path existence. + path (str): Directory or file path we want to check the existence in the container working directory. + + Returns: + bool: Whether the path exists in the container working directory. + """ + workdir = (await container.with_exec(["pwd"], skip_entrypoint=True).stdout()).strip() + mounts = await container.mounts() + if workdir in mounts: + expected_file_path = Path(workdir[1:]) / path + return expected_file_path.is_file() or expected_file_path.is_dir() + else: + return False + + +def secret_host_variable(client: Client, name: str, default: str = "") -> Callable[[Container], Container]: + """Add a host environment variable as a secret in a container. + + Example: + container.with_(secret_host_variable(client, "MY_SECRET")) + + Args: + client (Client): The dagger client. + name (str): The name of the environment variable. The same name will be + used in the container, for the secret name and for the host variable. + default (str): The default value to use if the host variable is not set. Defaults to "". + + Returns: + Callable[[Container], Container]: A function that can be used in a `Container.with_()` method. + """ + + def _secret_host_variable(container: Container) -> Container: + return container.with_secret_variable(name, get_secret_host_variable(client, name, default)) + + return _secret_host_variable + + +def get_secret_host_variable(client: Client, name: str, default: str = "") -> Secret: + """Creates a dagger.Secret from a host environment variable. + + Args: + client (Client): The dagger client. + name (str): The name of the environment variable. The same name will be used for the secret. + default (str): The default value to use if the host variable is not set. Defaults to "". + + Returns: + Secret: A dagger secret. + """ + return client.set_secret(name, os.environ.get(name, default)) + + +# This utils will probably be redundant once https://github.com/dagger/dagger/issues/3764 is implemented +async def get_file_contents(container: Container, path: str) -> Optional[str]: + """Retrieve a container file contents. + + Args: + container (Container): The container hosting the file you want to read. + path (str): Path, in the container, to the file you want to read. + + Returns: + Optional[str]: The file content if the file exists in the container, None otherwise. + """ + dir_name, file_name = os.path.split(path) + if file_name not in set(await container.directory(dir_name).entries()): + return None + return await container.file(path).contents() + + +@contextlib.contextmanager +def catch_exec_error_group() -> Generator: + try: + yield + except anyio.ExceptionGroup as eg: + for e in eg.exceptions: + if isinstance(e, ExecError): + raise e + raise + + +async def get_container_output(container: Container) -> Tuple[str, str]: + """Retrieve both stdout and stderr of a container, concurrently. + + Args: + container (Container): The container to execute. + + Returns: + Tuple[str, str]: The stdout and stderr of the container, respectively. + """ + with catch_exec_error_group(): + async with asyncer.create_task_group() as task_group: + soon_stdout = task_group.soonify(container.stdout)() + soon_stderr = task_group.soonify(container.stderr)() + return soon_stdout.value, soon_stderr.value + + +async def get_exec_result(container: Container) -> Tuple[int, str, str]: + """Retrieve the exit_code along with stdout and stderr of a container by handling the ExecError. + + Note: It is preferrable to not worry about the exit code value and just capture + ExecError to handle errors. This is offered as a convenience when the exit code + value is actually needed. + + If the container has a file at /exit_code, the exit code will be read from it. + See hacks.never_fail_exec for more details. + + Args: + container (Container): The container to execute. + + Returns: + Tuple[int, str, str]: The exit_code, stdout and stderr of the container, respectively. + """ + try: + exit_code = 0 + in_file_exit_code = await get_file_contents(container, "/exit_code") + if in_file_exit_code: + exit_code = int(in_file_exit_code) + return exit_code, *(await get_container_output(container)) + except ExecError as e: + return e.exit_code, e.stdout, e.stderr + + +async def with_exit_code(container: Container) -> int: + """Read the container exit code. + + Args: + container (Container): The container from which you want to read the exit code. + + Returns: + int: The exit code. + """ + try: + await container + except ExecError as e: + return e.exit_code + return 0 + + +async def with_stderr(container: Container) -> str: + """Retrieve the stderr of a container even on execution error.""" + try: + return await container.stderr() + except ExecError as e: + return e.stderr + + +async def with_stdout(container: Container) -> str: + """Retrieve the stdout of a container even on execution error.""" + try: + return await container.stdout() + except ExecError as e: + return e.stdout + + +def get_current_epoch_time() -> int: # noqa D103 + return round(datetime.datetime.utcnow().timestamp()) + + +def slugify(value: object, allow_unicode: bool = False) -> str: + """ + Taken from https://github.com/django/django/blob/master/django/utils/text.py. + + Convert to ASCII if 'allow_unicode' is False. Convert spaces or repeated + dashes to single dashes. Remove characters that aren't alphanumerics, + underscores, or hyphens. Convert to lowercase. Also strip leading and + trailing whitespace, dashes, and underscores. + """ + value = str(value) + if allow_unicode: + value = unicodedata.normalize("NFKC", value) + else: + value = unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode("ascii") + value = re.sub(r"[^\w\s-]", "", value.lower()) + return re.sub(r"[-\s]+", "-", value).strip("-_") + + +def key_value_text_to_dict(text: str) -> dict: + kv = {} + for line in text.split("\n"): + if "=" in line: + try: + k, v = line.split("=") + except ValueError: + continue + kv[k] = v + return kv + + +async def key_value_file_to_dict(file: File) -> dict: + return key_value_text_to_dict(await file.contents()) + + +async def get_dockerfile_labels(dockerfile: File) -> dict: + return {k.replace("LABEL ", ""): v for k, v in (await key_value_file_to_dict(dockerfile)).items() if k.startswith("LABEL")} + + +async def get_version_from_dockerfile(dockerfile: File) -> str: + dockerfile_labels = await get_dockerfile_labels(dockerfile) + try: + return dockerfile_labels["io.airbyte.version"] + except KeyError: + raise Exception("Could not get the version from the Dockerfile labels.") + + +def create_and_open_file(file_path: Path) -> TextIOWrapper: + """Create a file and open it for writing. + + Args: + file_path (Path): The path to the file to create. + + Returns: + File: The file object. + """ + file_path.parent.mkdir(parents=True, exist_ok=True) + file_path.touch() + return file_path.open("w") + + +async def execute_concurrently(steps: List[Callable], concurrency: int = 5) -> List[Any]: + tasks = [] + # Asyncer does not have builtin semaphore, so control concurrency via chunks of steps + # Anyio has semaphores but does not have the soonify method which allow access to results via the value task attribute. + for chunk in chunked(steps, concurrency): + async with asyncer.create_task_group() as task_group: + tasks += [task_group.soonify(step)() for step in chunk] + return [task.value for task in tasks] + + +async def export_container_to_tarball( + context: ConnectorContext, container: Container, platform: Platform, tar_file_name: Optional[str] = None +) -> Tuple[Optional[File], Optional[Path]]: + """Save the container image to the host filesystem as a tar archive. + + Exports a container to a tarball file. + The tarball file is saved to the host filesystem in the directory specified by the host_image_export_dir_path attribute of the context. + + Args: + context (ConnectorContext): The current connector context. + container (Container) : The list of container variants to export. + platform (Platform): The platform of the container to export. + tar_file_name (Optional[str], optional): The name of the tar archive file. Defaults to None. + + Returns: + Tuple[Optional[File], Optional[Path]]: A tuple with the file object holding the tar archive on the host and its path. + """ + tar_file_name = ( + f"{slugify(context.connector.technical_name)}_{context.git_revision}_{platform.replace('/', '_')}.tar" + if tar_file_name is None + else tar_file_name + ) + local_path = Path(f"{context.host_image_export_dir_path}/{tar_file_name}") + export_success = await container.export(str(local_path), forced_compression=ImageLayerCompression.Gzip) + if export_success: + return context.dagger_client.host().file(str(local_path)), local_path + return None, None + + +def format_duration(time_delta: datetime.timedelta) -> str: + total_seconds = time_delta.total_seconds() + if total_seconds < 60: + return "{:.2f}s".format(total_seconds) + minutes = int(total_seconds // 60) + seconds = int(total_seconds % 60) + return "{:02d}mn{:02d}s".format(minutes, seconds) + + +def sh_dash_c(lines: List[str]) -> List[str]: + """Wrap sequence of commands in shell for safe usage of dagger Container's with_exec method.""" + return ["sh", "-c", " && ".join(["set -o xtrace"] + lines)] + + +def transform_strs_to_paths(str_paths: Set[str]) -> List[Path]: + """Transform a list of string paths to an ordered list of Path objects. + + Args: + str_paths (Set[str]): A set of string paths. + + Returns: + List[Path]: A list of Path objects. + """ + return sorted([Path(str_path) for str_path in str_paths]) + + +def fail_if_missing_docker_hub_creds(ctx: click.Context) -> None: + if ctx.obj["docker_hub_username"] is None or ctx.obj["docker_hub_password"] is None: + raise click.UsageError( + "You need to be logged to DockerHub registry to run this command. Please set DOCKER_HUB_USERNAME and DOCKER_HUB_PASSWORD environment variables." + ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/models/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/models/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/ci_requirements.py b/airbyte-ci/connectors/pipelines/pipelines/models/ci_requirements.py new file mode 100644 index 000000000000..fce2dce67124 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/models/ci_requirements.py @@ -0,0 +1,34 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from dataclasses import dataclass +from importlib import metadata + +INFRA_SUPPORTED_DAGGER_VERSIONS = { + "0.6.4", + "0.9.5", + "0.9.6", +} + + +@dataclass +class CIRequirements: + """ + A dataclass to store the CI requirements. + It used to make airbyte-ci client define the CI runners it will run on. + """ + + dagger_version = metadata.version("dagger-io") + + def __post_init__(self) -> None: + if self.dagger_version not in INFRA_SUPPORTED_DAGGER_VERSIONS: + raise ValueError( + f"Unsupported dagger version: {self.dagger_version}. " f"Supported versions are: {INFRA_SUPPORTED_DAGGER_VERSIONS}." + ) + + def to_json(self) -> str: + return json.dumps( + { + "dagger_version": self.dagger_version, + } + ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py new file mode 100644 index 000000000000..fd565105c70c --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/click_pipeline_context.py @@ -0,0 +1,121 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import io +import sys +import tempfile +from typing import Any, Callable, Dict, Optional, TextIO, Tuple + +import anyio +import dagger +from asyncclick import Context, get_current_context +from pipelines import main_logger +from pipelines.cli.click_decorators import LazyPassDecorator +from pydantic import BaseModel, Field, PrivateAttr + +from ..singleton import Singleton + + +class ClickPipelineContext(BaseModel, Singleton): + """ + A replacement class for the Click context object passed to click functions. + + This class is meant to serve as a singleton object that initializes and holds onto a single instance of the + Dagger client, which is used to create containers for running pipelines. + """ + + dockerd_service: Optional[dagger.Container] = Field(default=None) + _dagger_client: Optional[dagger.Client] = PrivateAttr(default=None) + _click_context: Callable[[], Context] = PrivateAttr(default_factory=lambda: get_current_context) + _og_click_context: Context = PrivateAttr(default=None) + + @property + def params(self) -> Dict[str, Any]: + """ + Returns a combination of the click context object and the click context params. + + This means that any arguments or options defined in the parent command will be available to the child command. + """ + ctx = self._click_context() + click_obj = ctx.obj + click_params = ctx.params + command_name = ctx.command.name + + # Error if click_obj and click_params have the same key, and not the same value + intersection = set(click_obj.keys()) & set(click_params.keys()) + if intersection: + for key in intersection: + if click_obj[key] != click_params[key]: + raise ValueError( + f"Your command '{command_name}' has defined options/arguments with the same key as its parent, but with different values: {intersection}" + ) + + return {**click_obj, **click_params} + + class Config: + arbitrary_types_allowed = True + + def __init__(self, **data: dict[str, Any]) -> None: + """ + Initialize the ClickPipelineContext instance. + + This method checks the _initialized flag for the ClickPipelineContext class in the Singleton base class. + If the flag is False, the initialization logic is executed and the flag is set to True. + If the flag is True, the initialization logic is skipped. + + This ensures that the initialization logic is only executed once, even if the ClickPipelineContext instance is retrieved multiple times. + This can be useful if the initialization logic is expensive (e.g., it involves network requests or database queries). + """ + if not Singleton._initialized[ClickPipelineContext]: + super().__init__(**data) + Singleton._initialized[ClickPipelineContext] = True + + """ + Note: Its important to hold onto the original click context object, as it is used to hold onto the Dagger client. + """ + self._og_click_context = self._click_context() + + _dagger_client_lock: anyio.Lock = PrivateAttr(default_factory=anyio.Lock) + + async def get_dagger_client(self, pipeline_name: Optional[str] = None) -> dagger.Client: + """ + Get (or initialize) the Dagger Client instance. + """ + if not self._dagger_client: + async with self._dagger_client_lock: + if not self._dagger_client: + + connection = dagger.Connection(dagger.Config(log_output=self.get_log_output())) + """ + Sets up the '_dagger_client' attribute, intended for single-threaded use within connectors. + + Caution: + Avoid using this client across multiple thread pools, as it can lead to errors. + Cross-thread pool calls are generally considered an anti-pattern. + """ + self._dagger_client = await self._og_click_context.with_async_resource(connection) # type: ignore + + assert self._dagger_client, "Error initializing Dagger client" + return self._dagger_client.pipeline(pipeline_name) if pipeline_name else self._dagger_client + + def get_log_output(self) -> TextIO: + # This `show_dagger_logs` flag is likely going to be removed in the future. + # See https://github.com/airbytehq/airbyte/issues/33487 + if self.params.get("show_dagger_logs", False): + return sys.stdout + else: + log_output, self._click_context().obj["dagger_logs_path"] = self._create_dagger_client_log_file() + return log_output + + def _create_dagger_client_log_file(self) -> Tuple[TextIO, str]: + """ + Create the dagger client log file. + """ + dagger_logs_file_descriptor, dagger_logs_temp_file_path = tempfile.mkstemp(dir="/tmp", prefix="dagger_client_", suffix=".log") + main_logger.info(f"Dagger client logs stored in {dagger_logs_temp_file_path}") + return io.TextIOWrapper(io.FileIO(dagger_logs_file_descriptor, "w+")), dagger_logs_temp_file_path + + +# Create @pass_pipeline_context decorator for use in click commands +pass_pipeline_context: LazyPassDecorator = LazyPassDecorator(ClickPipelineContext) diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py new file mode 100644 index 000000000000..0d2431560145 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/pipeline_context.py @@ -0,0 +1,322 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""Module declaring context related classes.""" + +from __future__ import annotations + +import logging +import os +from datetime import datetime +from glob import glob +from types import TracebackType +from typing import TYPE_CHECKING + +from asyncer import asyncify +from dagger import Client, Directory, File, GitRepository, Secret, Service +from github import PullRequest +from pipelines.airbyte_ci.connectors.reports import ConnectorReport +from pipelines.consts import CIContext, ContextState +from pipelines.helpers.execution.run_steps import RunStepOptions +from pipelines.helpers.gcs import sanitize_gcs_credentials +from pipelines.helpers.github import update_commit_status_check +from pipelines.helpers.slack import send_message_to_webhook +from pipelines.helpers.utils import AIRBYTE_REPO_URL +from pipelines.models.reports import Report + +if TYPE_CHECKING: + from typing import List, Optional + + +class PipelineContext: + """The pipeline context is used to store configuration for a specific pipeline run.""" + + _dagger_client: Optional[Client] + _report: Optional[Report | ConnectorReport] + dockerd_service: Optional[Service] + started_at: Optional[datetime] + stopped_at: Optional[datetime] + + secrets_to_mask: List[str] + + PRODUCTION = bool(os.environ.get("PRODUCTION", False)) # Set this to True to enable production mode (e.g. to send PR comments) + + DEFAULT_EXCLUDED_FILES = ( + [".git", "airbyte-ci/connectors/pipelines/*"] + + glob("**/build", recursive=True) + + glob("**/.venv", recursive=True) + + glob("**/secrets", recursive=True) + + glob("**/__pycache__", recursive=True) + + glob("**/*.egg-info", recursive=True) + + glob("**/.vscode", recursive=True) + + glob("**/.pytest_cache", recursive=True) + + glob("**/.eggs", recursive=True) + + glob("**/.mypy_cache", recursive=True) + + glob("**/.DS_Store", recursive=True) + + glob("**/airbyte_ci_logs", recursive=True) + + glob("**/.gradle", recursive=True) + ) + + def __init__( + self, + pipeline_name: str, + is_local: bool, + git_branch: str, + git_revision: str, + report_output_prefix: str, + gha_workflow_run_url: Optional[str] = None, + dagger_logs_url: Optional[str] = None, + pipeline_start_timestamp: Optional[int] = None, + ci_context: Optional[str] = None, + is_ci_optional: bool = False, + slack_webhook: Optional[str] = None, + reporting_slack_channel: Optional[str] = None, + pull_request: Optional[PullRequest.PullRequest] = None, + ci_report_bucket: Optional[str] = None, + ci_gcs_credentials: Optional[str] = None, + ci_git_user: Optional[str] = None, + ci_github_access_token: Optional[str] = None, + run_step_options: RunStepOptions = RunStepOptions(), + enable_report_auto_open: bool = True, + ) -> None: + """Initialize a pipeline context. + + Args: + pipeline_name (str): The pipeline name. + is_local (bool): Whether the context is for a local run or a CI run. + git_branch (str): The current git branch name. + git_revision (str): The current git revision, commit hash. + report_output_prefix (str): The prefix to use for the report output. + gha_workflow_run_url (Optional[str], optional): URL to the github action workflow run. Only valid for CI run. Defaults to None. + dagger_logs_url (Optional[str], optional): URL to the dagger logs. Only valid for CI run. Defaults to None. + pipeline_start_timestamp (Optional[int], optional): Timestamp at which the pipeline started. Defaults to None. + ci_context (Optional[str], optional): Pull requests, workflow dispatch or nightly build. Defaults to None. + is_ci_optional (bool, optional): Whether the CI is optional. Defaults to False. + slack_webhook (Optional[str], optional): Slack webhook to send messages to. Defaults to None. + reporting_slack_channel (Optional[str], optional): Slack channel to send messages to. Defaults to None. + pull_request (PullRequest, optional): The pull request object if the pipeline was triggered by a pull request. Defaults to None. + """ + self.pipeline_name = pipeline_name + self.is_local = is_local + self.git_branch = git_branch + self.git_revision = git_revision + self.report_output_prefix = report_output_prefix + self.gha_workflow_run_url = gha_workflow_run_url + self.dagger_logs_url = dagger_logs_url + self.pipeline_start_timestamp = pipeline_start_timestamp + self.created_at = datetime.utcnow() + self.ci_context = ci_context + self.state = ContextState.INITIALIZED + self.is_ci_optional = is_ci_optional + self.slack_webhook = slack_webhook + self.reporting_slack_channel = reporting_slack_channel + self.pull_request = pull_request + self.logger = logging.getLogger(self.pipeline_name) + self._dagger_client = None + self._report = None + self.dockerd_service = None + self.ci_gcs_credentials = sanitize_gcs_credentials(ci_gcs_credentials) if ci_gcs_credentials else None + self.ci_report_bucket = ci_report_bucket + self.ci_git_user = ci_git_user + self.ci_github_access_token = ci_github_access_token + self.started_at = None + self.stopped_at = None + self.secrets_to_mask = [] + self.run_step_options = run_step_options + self.enable_report_auto_open = enable_report_auto_open + update_commit_status_check(**self.github_commit_status) + + @property + def dagger_client(self) -> Client: + assert self._dagger_client is not None, "The dagger client was not set on this PipelineContext" + return self._dagger_client + + @dagger_client.setter + def dagger_client(self, dagger_client: Client) -> None: + self._dagger_client = dagger_client + + @property + def is_ci(self) -> bool: + return self.is_local is False + + @property + def is_pr(self) -> bool: + return self.ci_context == CIContext.PULL_REQUEST + + @property + def repo(self) -> GitRepository: + return self.dagger_client.git(AIRBYTE_REPO_URL, keep_git_dir=True) + + @property + def report(self) -> Report | ConnectorReport | None: + return self._report + + @report.setter + def report(self, report: Report | ConnectorReport) -> None: + self._report = report + + @property + def ci_gcs_credentials_secret(self) -> Secret: + assert self.ci_gcs_credentials is not None, "The ci_gcs_credentials was not set on this PipelineContext." + return self.dagger_client.set_secret("ci_gcs_credentials", self.ci_gcs_credentials) + + @property + def ci_github_access_token_secret(self) -> Secret: + assert self.ci_github_access_token is not None, "The ci_github_access_token was not set on this PipelineContext." + return self.dagger_client.set_secret("ci_github_access_token", self.ci_github_access_token) + + @property + def github_commit_status(self) -> dict: + """Build a dictionary used as kwargs to the update_commit_status_check function.""" + target_url: Optional[str] = self.gha_workflow_run_url + + if self.state not in [ContextState.RUNNING, ContextState.INITIALIZED] and isinstance(self.report, ConnectorReport): + target_url = self.report.html_report_url + + return { + "sha": self.git_revision, + "state": self.state.value["github_state"], + "target_url": target_url, + "description": self.state.value["description"], + "context": self.pipeline_name, + "should_send": self.is_pr, + "logger": self.logger, + "is_optional": self.is_ci_optional, + } + + @property + def should_send_slack_message(self) -> bool: + return self.slack_webhook is not None and self.reporting_slack_channel is not None + + @property + def has_dagger_cloud_token(self) -> bool: + return "_EXPERIMENTAL_DAGGER_CLOUD_TOKEN" in os.environ + + @property + def dagger_cloud_url(self) -> Optional[str]: + """Gets the link to the Dagger Cloud runs page for the current commit.""" + if self.is_local or not self.has_dagger_cloud_token: + return None + + return f"https://alpha.dagger.cloud/changeByPipelines?filter=dagger.io/git.ref:{self.git_revision}" + + def get_repo_file(self, file_path: str) -> File: + """Get a file from the current repository. + + The file is extracted from the host file system. + + Args: + file_path (str): Path to the file to get. + + Returns: + Path: The selected repo file. + """ + return self.dagger_client.host().file(file_path) + + def get_repo_dir(self, subdir: str = ".", exclude: Optional[List[str]] = None, include: Optional[List[str]] = None) -> Directory: + """Get a directory from the current repository. + + The directory is extracted from the host file system. + A couple of files or directories that could corrupt builds are exclude by default (check DEFAULT_EXCLUDED_FILES). + + Args: + subdir (str, optional): Path to the subdirectory to get. Defaults to "." to get the full repository. + exclude ([List[str], optional): List of files or directories to exclude from the directory. Defaults to None. + include ([List[str], optional): List of files or directories to include in the directory. Defaults to None. + + Returns: + Directory: The selected repo directory. + """ + if exclude is None: + exclude = self.DEFAULT_EXCLUDED_FILES + else: + exclude += self.DEFAULT_EXCLUDED_FILES + exclude = list(set(exclude)) + exclude.sort() # sort to make sure the order is always the same to not burst the cache. Casting exclude to set can change the order + if subdir != ".": + subdir = f"{subdir}/" if not subdir.endswith("/") else subdir + exclude = [f.replace(subdir, "") for f in exclude if subdir in f] + return self.dagger_client.host().directory(subdir, exclude=exclude, include=include) + + def create_slack_message(self) -> str: + raise NotImplementedError() + + async def __aenter__(self) -> PipelineContext: + """Perform setup operation for the PipelineContext. + + Updates the current commit status on Github. + + Raises: + Exception: An error is raised when the context was not initialized with a Dagger client + Returns: + PipelineContext: A running instance of the PipelineContext. + """ + if self.dagger_client is None: + raise Exception("A Pipeline can't be entered with an undefined dagger_client") + self.state = ContextState.RUNNING + self.started_at = datetime.utcnow() + self.logger.info("Caching the latest CDK version...") + await asyncify(update_commit_status_check)(**self.github_commit_status) + if self.should_send_slack_message: + # Using a type ignore here because the should_send_slack_message property is checking for non nullity of the slack_webhook and reporting_slack_channel + await asyncify(send_message_to_webhook)(self.create_slack_message(), self.reporting_slack_channel, self.slack_webhook) # type: ignore + return self + + @staticmethod + def determine_final_state(report: Optional[Report], exception_value: Optional[BaseException]) -> ContextState: + """Determine the final state of the context from the report or the exception value. + + Args: + report (Optional[Report]): The pipeline report if any. + exception_value (Optional[BaseException]): The exception value if an exception was raised in the context execution, None otherwise. + Returns: + ContextState: The final state of the context. + """ + if exception_value is not None or report is None: + return ContextState.ERROR + if report is not None and report.failed_steps: + return ContextState.FAILURE + if report is not None and report.success: + return ContextState.SUCCESSFUL + raise Exception( + f"The final state of the context could not be determined for the report and exception value provided. Report: {report}, Exception: {exception_value}" + ) + + async def __aexit__( + self, exception_type: Optional[type[BaseException]], exception_value: Optional[BaseException], traceback: Optional[TracebackType] + ) -> bool: + """Perform teardown operation for the PipelineContext. + + On the context exit the following operations will happen: + - Log the error value if an error was handled. + - Log the test report. + - Update the commit status check on GitHub if running in a CI environment. + + It should gracefully handle all the execution errors that happened and always upload a test report and update commit status check. + + Args: + exception_type (Optional[type[BaseException]]): The exception type if an exception was raised in the context execution, None otherwise. + exception_value (Optional[BaseException]): The exception value if an exception was raised in the context execution, None otherwise. + traceback (Optional[TracebackType]): The traceback if an exception was raised in the context execution, None otherwise. + Returns: + bool: Whether the teardown operation ran successfully. + """ + if exception_value: + self.logger.error("An error was handled by the Pipeline", exc_info=True) + + if self.report is None: + self.logger.error("No test report was provided. This is probably due to an upstream error") + self.report = Report(self, steps_results=[]) + + self.state = self.determine_final_state(self.report, exception_value) + self.stopped_at = datetime.utcnow() + + self.report.print() + + await asyncify(update_commit_status_check)(**self.github_commit_status) + if self.should_send_slack_message: + # Using a type ignore here because the should_send_slack_message property is checking for non nullity of the slack_webhook and reporting_slack_channel + await asyncify(send_message_to_webhook)(self.create_slack_message(), self.reporting_slack_channel, self.slack_webhook) # type: ignore + # supress the exception if it was handled + return True diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/reports.py b/airbyte-ci/connectors/pipelines/pipelines/models/reports.py new file mode 100644 index 000000000000..206e1c44e9e1 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/models/reports.py @@ -0,0 +1,194 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +"""This module declare base / abstract models to be reused in a pipeline lifecycle.""" + +from __future__ import annotations + +import json +import typing +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from typing import List + +import anyio +from anyio import Path +from connector_ops.utils import console # type: ignore +from pipelines.consts import GCS_PUBLIC_DOMAIN, LOCAL_REPORTS_PATH_ROOT +from pipelines.dagger.actions import remote_storage +from pipelines.helpers.utils import format_duration +from pipelines.models.steps import StepResult, StepStatus +from rich.console import Group +from rich.panel import Panel +from rich.style import Style +from rich.table import Table +from rich.text import Text + +if typing.TYPE_CHECKING: + from pipelines.models.contexts.pipeline_context import PipelineContext + from rich.tree import RenderableType + + +@dataclass(frozen=True) +class Report: + """A dataclass to build reports to share pipelines executions results with the user.""" + + pipeline_context: PipelineContext + steps_results: List[StepResult] + created_at: datetime = field(default_factory=datetime.utcnow) + name: str = "REPORT" + filename: str = "output" + + @property + def report_output_prefix(self) -> str: + return self.pipeline_context.report_output_prefix + + @property + def json_report_file_name(self) -> str: + return self.filename + ".json" + + @property + def json_report_remote_storage_key(self) -> str: + return f"{self.report_output_prefix}/{self.json_report_file_name}" + + @property + def failed_steps(self) -> List[StepResult]: + return [step_result for step_result in self.steps_results if step_result.status is StepStatus.FAILURE] + + @property + def successful_steps(self) -> List[StepResult]: + return [step_result for step_result in self.steps_results if step_result.status is StepStatus.SUCCESS] + + @property + def skipped_steps(self) -> List[StepResult]: + return [step_result for step_result in self.steps_results if step_result.status is StepStatus.SKIPPED] + + @property + def success(self) -> bool: + return len(self.failed_steps) == 0 and (len(self.skipped_steps) > 0 or len(self.successful_steps) > 0) + + @property + def run_duration(self) -> timedelta: + assert self.pipeline_context.started_at is not None, "The pipeline started_at timestamp must be set to save reports." + assert self.pipeline_context.stopped_at is not None, "The pipeline stopped_at timestamp must be set to save reports." + return self.pipeline_context.stopped_at - self.pipeline_context.started_at + + @property + def lead_duration(self) -> timedelta: + assert self.pipeline_context.started_at is not None, "The pipeline started_at timestamp must be set to save reports." + assert self.pipeline_context.stopped_at is not None, "The pipeline stopped_at timestamp must be set to save reports." + return self.pipeline_context.stopped_at - self.pipeline_context.created_at + + @property + def remote_storage_enabled(self) -> bool: + return self.pipeline_context.is_ci + + async def save_local(self, filename: str, content: str) -> Path: + """Save the report files locally.""" + local_path = anyio.Path(f"{LOCAL_REPORTS_PATH_ROOT}/{self.report_output_prefix}/{filename}") + await local_path.parents[0].mkdir(parents=True, exist_ok=True) + await local_path.write_text(content) + return local_path + + async def save_remote(self, local_path: Path, remote_key: str, content_type: str) -> int: + assert self.pipeline_context.ci_report_bucket is not None, "The ci_report_bucket must be set to save reports." + + gcs_cp_flags = None if content_type is None else [f"--content-type={content_type}"] + local_file = self.pipeline_context.dagger_client.host().directory(".", include=[str(local_path)]).file(str(local_path)) + report_upload_exit_code, _, _ = await remote_storage.upload_to_gcs( + dagger_client=self.pipeline_context.dagger_client, + file_to_upload=local_file, + key=remote_key, + bucket=self.pipeline_context.ci_report_bucket, + gcs_credentials=self.pipeline_context.ci_gcs_credentials_secret, + flags=gcs_cp_flags, + ) + gcs_uri = "gs://" + self.pipeline_context.ci_report_bucket + "/" + remote_key + public_url = f"{GCS_PUBLIC_DOMAIN}/{self.pipeline_context.ci_report_bucket}/{remote_key}" + if report_upload_exit_code != 0: + self.pipeline_context.logger.error(f"Uploading {local_path} to {gcs_uri} failed.") + else: + self.pipeline_context.logger.info(f"Uploading {local_path} to {gcs_uri} succeeded. Public URL: {public_url}") + return report_upload_exit_code + + async def save(self) -> None: + """Save the report files.""" + + local_json_path = await self.save_local(self.json_report_file_name, self.to_json()) + absolute_path = await local_json_path.absolute() + self.pipeline_context.logger.info(f"Report saved locally at {absolute_path}") + if self.remote_storage_enabled: + await self.save_remote(local_json_path, self.json_report_remote_storage_key, "application/json") + + def to_json(self) -> str: + """Create a JSON representation of the report. + + Returns: + str: The JSON representation of the report. + """ + assert self.pipeline_context.pipeline_start_timestamp is not None, "The pipeline start timestamp must be set to save reports." + assert self.pipeline_context.started_at is not None, "The pipeline started_at timestamp must be set to save reports." + assert self.pipeline_context.stopped_at is not None, "The pipeline stopped_at timestamp must be set to save reports." + return json.dumps( + { + "pipeline_name": self.pipeline_context.pipeline_name, + "run_timestamp": self.pipeline_context.started_at.isoformat(), + "run_duration": self.run_duration.total_seconds(), + "success": self.success, + "failed_steps": [s.step.__class__.__name__ for s in self.failed_steps], # type: ignore + "successful_steps": [s.step.__class__.__name__ for s in self.successful_steps], # type: ignore + "skipped_steps": [s.step.__class__.__name__ for s in self.skipped_steps], # type: ignore + "gha_workflow_run_url": self.pipeline_context.gha_workflow_run_url, + "pipeline_start_timestamp": self.pipeline_context.pipeline_start_timestamp, + "pipeline_end_timestamp": round(self.pipeline_context.stopped_at.timestamp()), + "pipeline_duration": round(self.pipeline_context.stopped_at.timestamp()) - self.pipeline_context.pipeline_start_timestamp, + "git_branch": self.pipeline_context.git_branch, + "git_revision": self.pipeline_context.git_revision, + "ci_context": self.pipeline_context.ci_context, + "pull_request_url": self.pipeline_context.pull_request.html_url if self.pipeline_context.pull_request else None, + "dagger_cloud_url": self.pipeline_context.dagger_cloud_url, + } + ) + + def print(self) -> None: + """Print the test report to the console in a nice way.""" + pipeline_name = self.pipeline_context.pipeline_name + main_panel_title = Text(f"{pipeline_name.upper()} - {self.name}") + main_panel_title.stylize(Style(color="blue", bold=True)) + duration_subtitle = Text(f"⏲️ Total pipeline duration for {pipeline_name}: {format_duration(self.run_duration)}") + step_results_table = Table(title="Steps results") + step_results_table.add_column("Step") + step_results_table.add_column("Result") + step_results_table.add_column("Finished after") + + for step_result in self.steps_results: + step = Text(step_result.step.title) + step.stylize(step_result.status.get_rich_style()) + result = Text(step_result.status.value) + result.stylize(step_result.status.get_rich_style()) + + if step_result.status is StepStatus.SKIPPED: + step_results_table.add_row(step, result, "N/A") + else: + assert step_result.step.started_at is not None, "The step started_at timestamp must be set to print reports." + run_time = format_duration((step_result.created_at - step_result.step.started_at)) + step_results_table.add_row(step, result, run_time) + + to_render: List[RenderableType] = [step_results_table] + if self.failed_steps: + sub_panels = [] + for failed_step in self.failed_steps: + errors = Text(failed_step.stderr) if failed_step.stderr else Text("") + panel_title = Text(f"{pipeline_name} {failed_step.step.title.lower()} failures") + panel_title.stylize(Style(color="red", bold=True)) + sub_panel = Panel(errors, title=panel_title) + sub_panels.append(sub_panel) + failures_group = Group(*sub_panels) + to_render.append(failures_group) + + if self.pipeline_context.dagger_cloud_url: + self.pipeline_context.logger.info(f"🔗 View runs for commit in Dagger Cloud: {self.pipeline_context.dagger_cloud_url}") + + main_panel = Panel(Group(*to_render), title=main_panel_title, subtitle=duration_subtitle) + console.print(main_panel) diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py new file mode 100644 index 000000000000..7fdb4b6f5dda --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/models/singleton.py @@ -0,0 +1,26 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Type + + +class Singleton: + """ + A base class for implementing the Singleton pattern. + + This class stores instances and initialization flags for each subclass in dictionaries. + This allows each subclass to have its own unique instance and control over its initialization process. + + The __new__ method ensures that only one instance of each subclass is created. + The _initialized dictionary is used to control when the initialization logic of each subclass is executed. + """ + + _instances: dict[Type["Singleton"], Any] = {} + _initialized: dict[Type["Singleton"], bool] = {} + + def __new__(cls: Type["Singleton"], *args: Any, **kwargs: Any) -> Any: # noqa: ANN401 + if cls not in cls._instances: + cls._instances[cls] = super().__new__(cls) + cls._initialized[cls] = False + return cls._instances[cls] diff --git a/airbyte-ci/connectors/pipelines/pipelines/models/steps.py b/airbyte-ci/connectors/pipelines/pipelines/models/steps.py new file mode 100644 index 000000000000..bc3acafebc06 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/models/steps.py @@ -0,0 +1,378 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from __future__ import annotations + +import logging +from abc import abstractmethod +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from enum import Enum +from pathlib import Path +from typing import TYPE_CHECKING, Dict, List + +import anyio +import asyncer +import click +from dagger import Client, Container, DaggerError +from pipelines import main_logger +from pipelines.helpers import sentry_utils +from pipelines.helpers.utils import format_duration, get_exec_result + +if TYPE_CHECKING: + from typing import Any, ClassVar, Optional, Union + from pipelines.airbyte_ci.format.format_command import FormatCommand + from pipelines.models.contexts.pipeline_context import PipelineContext + + +from abc import ABC + +from rich.style import Style + +STEP_PARAMS = Dict[str, List[str]] + + +@dataclass +class MountPath: + path: Union[Path, str] + optional: bool = False + + def _cast_fields(self) -> None: + self.path = Path(self.path) + self.optional = bool(self.optional) + + def _check_exists(self) -> None: + if not self.get_path().exists(): + message = f"{self.path} does not exist." + if self.optional: + main_logger.warning(message) + else: + raise FileNotFoundError(message) + + def get_path(self) -> Path: + return Path(self.path) + + def __post_init__(self) -> None: + self._cast_fields() + self._check_exists() + + def __str__(self) -> str: + return str(self.path) + + @property + def is_file(self) -> bool: + return self.get_path().is_file() + + +@dataclass(frozen=True) +class StepResult: + """A dataclass to capture the result of a step.""" + + step: Step + status: StepStatus + created_at: datetime = field(default_factory=datetime.utcnow) + stderr: Optional[str] = None + stdout: Optional[str] = None + output_artifact: Any = None + exc_info: Optional[Exception] = None + + def __repr__(self) -> str: # noqa D105 + return f"{self.step.title}: {self.status.value}" + + def __str__(self) -> str: # noqa D105 + return f"{self.step.title}: {self.status.value}\n\nSTDOUT:\n{self.stdout}\n\nSTDERR:\n{self.stderr}" + + def __post_init__(self) -> None: + if self.stderr: + super().__setattr__("stderr", self.redact_secrets_from_string(self.stderr)) + if self.stdout: + super().__setattr__("stdout", self.redact_secrets_from_string(self.stdout)) + + def redact_secrets_from_string(self, value: str) -> str: + for secret in self.step.context.secrets_to_mask: + value = value.replace(secret, "********") + return value + + +@dataclass(frozen=True) +class CommandResult: + """A dataclass to capture the result of a command.""" + + command: click.Command | FormatCommand + status: StepStatus + created_at: datetime = field(default_factory=datetime.utcnow) + stderr: Optional[str] = None + stdout: Optional[str] = None + exc_info: Optional[Exception] = None + output_artifact: Any = None + + def __repr__(self) -> str: # noqa D105 + return f"{self.command.name}: {self.status.value}" + + def __str__(self) -> str: # noqa D105 + return f"{self.command.name}: {self.status.value}\n\nSTDOUT:\n{self.stdout}\n\nSTDERR:\n{self.stderr}" + + @property + def success(self) -> bool: + return self.status is StepStatus.SUCCESS + + +class StepStatus(Enum): + """An Enum to characterize the success, failure or skipping of a Step.""" + + SUCCESS = "Successful" + FAILURE = "Failed" + SKIPPED = "Skipped" + + def get_rich_style(self) -> Style: + """Match color used in the console output to the step status.""" + if self is StepStatus.SUCCESS: + return Style(color="green") + if self is StepStatus.FAILURE: + return Style(color="red", bold=True) + if self is StepStatus.SKIPPED: + return Style(color="yellow") + + def get_emoji(self) -> str: + """Match emoji used in the console output to the step status.""" + if self is StepStatus.SUCCESS: + return "✅" + if self is StepStatus.FAILURE: + return "❌" + if self is StepStatus.SKIPPED: + return "🟡" + + def __str__(self) -> str: # noqa D105 + return self.value + + +class Step(ABC): + """An abstract class to declare and run pipeline step.""" + + max_retries: ClassVar[int] = 0 + max_dagger_error_retries: ClassVar[int] = 3 + should_log: ClassVar[bool] = True + success_exit_code: ClassVar[int] = 0 + skipped_exit_code: ClassVar[Optional[int]] = None + # The max duration of a step run. If the step run for more than this duration it will be considered as timed out. + # The default of 5 hours is arbitrary and can be changed if needed. + max_duration: ClassVar[timedelta] = timedelta(hours=5) + retry_delay = timedelta(seconds=10) + accept_extra_params: bool = False + + def __init__(self, context: PipelineContext) -> None: # noqa D107 + self.context = context + self.retry_count = 0 + self.started_at: Optional[datetime] = None + self.stopped_at: Optional[datetime] = None + self._extra_params: STEP_PARAMS = {} + + @property + def extra_params(self) -> STEP_PARAMS: + return self._extra_params + + @extra_params.setter + def extra_params(self, value: STEP_PARAMS) -> None: + if value and not self.accept_extra_params: + raise ValueError(f"{self.__class__.__name__} does not accept extra params.") + self._extra_params = value + self.logger.info(f"Will run with the following parameters: {self.params}") + + @property + def default_params(self) -> STEP_PARAMS: + return {} + + @property + def params(self) -> STEP_PARAMS: + return self.default_params | self.extra_params + + @property + def params_as_cli_options(self) -> List[str]: + """Return the step params as a list of CLI options. + + Returns: + List[str]: The step params as a list of CLI options. + """ + cli_options: List[str] = [] + for name, values in self.params.items(): + if not values: + # If no values are available, we assume it is a flag + cli_options.append(name) + else: + cli_options.extend(f"{name}={value}" for value in values) + return cli_options + + @property + def title(self) -> str: + """The title of the step.""" + raise NotImplementedError("Steps must define a 'title' attribute.") + + @property + def run_duration(self) -> timedelta: + if self.started_at and self.stopped_at: + return self.stopped_at - self.started_at + else: + return timedelta(seconds=0) + + @property + def logger(self) -> logging.Logger: + if self.should_log: + return logging.getLogger(f"{self.context.pipeline_name} - {self.title}") + else: + disabled_logger = logging.getLogger() + disabled_logger.disabled = True + return disabled_logger + + @property + def dagger_client(self) -> Client: + return self.context.dagger_client.pipeline(self.title) + + async def log_progress(self, completion_event: anyio.Event) -> None: + """Log the step progress every 30 seconds until the step is done.""" + while not completion_event.is_set(): + assert self.started_at is not None, "The step must be started before logging its progress." + duration = datetime.utcnow() - self.started_at + elapsed_seconds = duration.total_seconds() + if elapsed_seconds > 30 and round(elapsed_seconds) % 30 == 0: + self.logger.info(f"⏳ Still running... (duration: {format_duration(duration)})") + await anyio.sleep(1) + + async def run_with_completion(self, completion_event: anyio.Event, *args: Any, **kwargs: Any) -> StepResult: + """Run the step with a timeout and set the completion event when the step is done.""" + try: + with anyio.fail_after(self.max_duration.total_seconds()): + result = await self._run(*args, **kwargs) + completion_event.set() + return result + except TimeoutError: + self.retry_count = self.max_retries + 1 + self.logger.error(f"🚨 {self.title} timed out after {self.max_duration}. No additional retry will happen.") + completion_event.set() + return self._get_timed_out_step_result() + + @sentry_utils.with_step_context + async def run(self, *args: Any, **kwargs: Any) -> StepResult: + """Public method to run the step. It output a step result. + + If an unexpected dagger error happens it outputs a failed step result with the exception payload. + + Returns: + StepResult: The step result following the step run. + """ + self.logger.info(f"🚀 Start {self.title}") + self.started_at = datetime.utcnow() + completion_event = anyio.Event() + try: + async with asyncer.create_task_group() as task_group: + soon_result = task_group.soonify(self.run_with_completion)(completion_event, *args, **kwargs) + task_group.soonify(self.log_progress)(completion_event) + step_result = soon_result.value + except DaggerError as e: + self.logger.error("Step failed with an unexpected dagger error", exc_info=e) + step_result = StepResult(self, StepStatus.FAILURE, stderr=str(e), exc_info=e) + + self.stopped_at = datetime.utcnow() + self.log_step_result(step_result) + + lets_retry = self.should_retry(step_result) + step_result = await self.retry(step_result, *args, **kwargs) if lets_retry else step_result + return step_result + + def should_retry(self, step_result: StepResult) -> bool: + """Return True if the step should be retried.""" + if step_result.status is not StepStatus.FAILURE: + return False + max_retries = self.max_dagger_error_retries if step_result.exc_info else self.max_retries + return self.retry_count < max_retries and max_retries > 0 + + async def retry(self, step_result: StepResult, *args: Any, **kwargs: Any) -> StepResult: + self.retry_count += 1 + self.logger.warn( + f"Failed with error: {step_result.stderr}.\nRetry #{self.retry_count} in {self.retry_delay.total_seconds()} seconds..." + ) + await anyio.sleep(self.retry_delay.total_seconds()) + return await self.run(*args, **kwargs) + + def log_step_result(self, result: StepResult) -> None: + """Log the step result. + + Args: + result (StepResult): The step result to log. + """ + duration = format_duration(self.run_duration) + if result.status is StepStatus.FAILURE: + self.logger.info(f"{result.status.get_emoji()} failed (duration: {duration})") + if result.status is StepStatus.SKIPPED: + self.logger.info(f"{result.status.get_emoji()} was skipped (duration: {duration})") + if result.status is StepStatus.SUCCESS: + self.logger.info(f"{result.status.get_emoji()} was successful (duration: {duration})") + + @abstractmethod + async def _run(self, *args: Any, **kwargs: Any) -> StepResult: + """Implement the execution of the step and return a step result. + + Returns: + StepResult: The result of the step run. + """ + raise NotImplementedError("Steps must define a '_run' attribute.") + + def skip(self, reason: Optional[str] = None) -> StepResult: + """Declare a step as skipped. + + Args: + reason (str, optional): Reason why the step was skipped. + + Returns: + StepResult: A skipped step result. + """ + return StepResult(self, StepStatus.SKIPPED, stdout=reason) + + def get_step_status_from_exit_code( + self, + exit_code: int, + ) -> StepStatus: + """Map an exit code to a step status. + + Args: + exit_code (int): A process exit code. + + Raises: + ValueError: Raised if the exit code is not mapped to a step status. + + Returns: + StepStatus: The step status inferred from the exit code. + """ + if exit_code == self.success_exit_code: + return StepStatus.SUCCESS + elif self.skipped_exit_code is not None and exit_code == self.skipped_exit_code: + return StepStatus.SKIPPED + else: + return StepStatus.FAILURE + + async def get_step_result(self, container: Container) -> StepResult: + """Concurrent retrieval of exit code, stdout and stdout of a container. + + Create a StepResult object from these objects. + + Args: + container (Container): The container from which we want to infer a step result/ + + Returns: + StepResult: Failure or success with stdout and stderr. + """ + exit_code, stdout, stderr = await get_exec_result(container) + return StepResult( + self, + self.get_step_status_from_exit_code(exit_code), + stderr=stderr, + stdout=stdout, + output_artifact=container, + ) + + def _get_timed_out_step_result(self) -> StepResult: + return StepResult( + self, + StepStatus.FAILURE, + stdout=f"Timed out after the max duration of {format_duration(self.max_duration)}. Please checkout the Dagger logs to see what happened.", + ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/pipelines/connectors.py b/airbyte-ci/connectors/pipelines/pipelines/pipelines/connectors.py deleted file mode 100644 index 5acb57b9558f..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/pipelines/connectors.py +++ /dev/null @@ -1,108 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -"""This module groups the functions to run full pipelines for connector testing.""" - -import sys -from pathlib import Path -from typing import Callable, List, Optional - -import anyio -import dagger -from connector_ops.utils import ConnectorLanguage -from dagger import Config -from pipelines.actions import environments -from pipelines.bases import NoOpStep, Report, StepResult, StepStatus -from pipelines.contexts import ConnectorContext, ContextState -from pipelines.utils import create_and_open_file - -GITHUB_GLOBAL_CONTEXT = "[POC please ignore] Connectors CI" -GITHUB_GLOBAL_DESCRIPTION = "Running connectors tests" - -CONNECTOR_LANGUAGE_TO_FORCED_CONCURRENCY_MAPPING = { - # We run the Java connectors tests sequentially because we currently have memory issues when Java integration tests are run in parallel. - # See https://github.com/airbytehq/airbyte/issues/27168 - ConnectorLanguage.JAVA: anyio.Semaphore(1), -} - - -async def context_to_step_result(context: ConnectorContext) -> StepResult: - if context.state == ContextState.SUCCESSFUL: - return await NoOpStep(context, StepStatus.SUCCESS).run() - - if context.state == ContextState.FAILURE: - return await NoOpStep(context, StepStatus.FAILURE).run() - - if context.state == ContextState.ERROR: - return await NoOpStep(context, StepStatus.FAILURE).run() - - raise ValueError(f"Could not convert context state: {context.state} to step status") - - -# HACK: This is to avoid wrapping the whole pipeline in a dagger pipeline to avoid instability just prior to launch -# TODO (ben): Refactor run_connectors_pipelines to wrap the whole pipeline in a dagger pipeline once Steps are refactored -async def run_report_complete_pipeline(dagger_client: dagger.Client, contexts: List[ConnectorContext]) -> List[ConnectorContext]: - """Create and Save a report representing the run of the encompassing pipeline. - - This is to denote when the pipeline is complete, useful for long running pipelines like nightlies. - """ - - if not contexts: - return [] - - # Repurpose the first context to be the pipeline upload context to preserve timestamps - first_connector_context = contexts[0] - - pipeline_name = f"Report upload {first_connector_context.report_output_prefix}" - first_connector_context.pipeline_name = pipeline_name - - # Transform contexts into a list of steps - steps_results = [await context_to_step_result(context) for context in contexts] - - report = Report( - name=pipeline_name, - pipeline_context=first_connector_context, - steps_results=steps_results, - filename="complete", - ) - - return await report.save() - - -async def run_connectors_pipelines( - contexts: List[ConnectorContext], - connector_pipeline: Callable, - pipeline_name: str, - concurrency: int, - dagger_logs_path: Optional[Path], - execute_timeout: Optional[int], - *args, -) -> List[ConnectorContext]: - """Run a connector pipeline for all the connector contexts.""" - - default_connectors_semaphore = anyio.Semaphore(concurrency) - dagger_logs_output = sys.stderr if not dagger_logs_path else create_and_open_file(dagger_logs_path) - async with dagger.Connection(Config(log_output=dagger_logs_output, execute_timeout=execute_timeout)) as dagger_client: - # HACK: This is to get a long running dockerd service to be shared across all the connectors pipelines - # Using the "normal" service binding leads to restart of dockerd during pipeline run that can cause corrupted docker state - # See https://github.com/airbytehq/airbyte/issues/27233 - dockerd_service = environments.with_global_dockerd_service(dagger_client) - async with anyio.create_task_group() as tg_main: - tg_main.start_soon(dockerd_service.sync) - await anyio.sleep(10) # Wait for the docker service to be ready - async with anyio.create_task_group() as tg_connectors: - for context in contexts: - context.dagger_client = dagger_client.pipeline(f"{pipeline_name} - {context.connector.technical_name}") - context.dockerd_service = dockerd_service - tg_connectors.start_soon( - connector_pipeline, - context, - CONNECTOR_LANGUAGE_TO_FORCED_CONCURRENCY_MAPPING.get(context.connector.language, default_connectors_semaphore), - *args, - ) - # When the connectors pipelines are done, we can stop the dockerd service - tg_main.cancel_scope.cancel() - await run_report_complete_pipeline(dagger_client, contexts) - - return contexts diff --git a/airbyte-ci/connectors/pipelines/pipelines/pipelines/metadata.py b/airbyte-ci/connectors/pipelines/pipelines/pipelines/metadata.py deleted file mode 100644 index d5288ef05ba9..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/pipelines/metadata.py +++ /dev/null @@ -1,340 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import functools -import uuid -from pathlib import Path -from typing import Optional, Set - -import dagger -from pipelines.actions.environments import with_pip_packages, with_poetry_module, with_python_base -from pipelines.bases import Report, Step, StepResult -from pipelines.contexts import PipelineContext -from pipelines.helpers.steps import run_steps -from pipelines.utils import DAGGER_CONFIG, METADATA_FILE_NAME, METADATA_ICON_FILE_NAME, execute_concurrently, get_secret_host_variable - -METADATA_DIR = "airbyte-ci/connectors/metadata_service" -METADATA_LIB_MODULE_PATH = "lib" -METADATA_ORCHESTRATOR_MODULE_PATH = "orchestrator" - -# HELPERS - - -def get_metadata_file_from_path(context: PipelineContext, metadata_path: Path) -> dagger.File: - if metadata_path.is_file() and metadata_path.name != METADATA_FILE_NAME: - raise ValueError(f"The metadata file name is not {METADATA_FILE_NAME}, it is {metadata_path.name} .") - if metadata_path.is_dir(): - metadata_path = metadata_path / METADATA_FILE_NAME - if not metadata_path.exists(): - raise FileNotFoundError(f"{str(metadata_path)} does not exist.") - return context.get_repo_dir(str(metadata_path.parent), include=[METADATA_FILE_NAME]).file(METADATA_FILE_NAME) - - -def get_metadata_icon_file_from_path(context: PipelineContext, metadata_icon_path: Path) -> dagger.File: - return context.get_repo_dir(str(metadata_icon_path.parent), include=[METADATA_ICON_FILE_NAME]).file(METADATA_ICON_FILE_NAME) - - -# STEPS - - -class PoetryRun(Step): - def __init__(self, context: PipelineContext, title: str, parent_dir_path: str, module_path: str): - self.title = title - super().__init__(context) - self.parent_dir = self.context.get_repo_dir(parent_dir_path) - self.module_path = module_path - self.poetry_run_container = with_poetry_module(self.context, self.parent_dir, self.module_path).with_entrypoint(["poetry", "run"]) - - async def _run(self, poetry_run_args: list) -> StepResult: - poetry_run_exec = self.poetry_run_container.with_exec(poetry_run_args) - return await self.get_step_result(poetry_run_exec) - - -class MetadataValidation(PoetryRun): - def __init__(self, context: PipelineContext, metadata_path: Path): - title = f"Validate {metadata_path}" - super().__init__(context, title, METADATA_DIR, METADATA_LIB_MODULE_PATH) - self.poetry_run_container = self.poetry_run_container.with_mounted_file( - METADATA_FILE_NAME, get_metadata_file_from_path(context, metadata_path) - ) - - async def _run(self) -> StepResult: - return await super()._run(["metadata_service", "validate", METADATA_FILE_NAME]) - - -class MetadataUpload(PoetryRun): - # When the metadata service exits with this code, it means the metadata is valid but the upload was skipped because the metadata is already uploaded - skipped_exit_code = 5 - - def __init__( - self, - context: PipelineContext, - metadata_path: Path, - metadata_bucket_name: str, - metadata_service_gcs_credentials_secret: dagger.Secret, - docker_hub_username_secret: dagger.Secret, - docker_hub_password_secret: dagger.Secret, - pre_release: bool = False, - pre_release_tag: Optional[str] = None, - ): - title = f"Upload {metadata_path}" - self.gcs_bucket_name = metadata_bucket_name - self.pre_release = pre_release - self.pre_release_tag = pre_release_tag - super().__init__(context, title, METADATA_DIR, METADATA_LIB_MODULE_PATH) - - # Ensure the icon file is included in the upload - base_container = self.poetry_run_container.with_file(METADATA_FILE_NAME, get_metadata_file_from_path(context, metadata_path)) - metadata_icon_path = metadata_path.parent / METADATA_ICON_FILE_NAME - if metadata_icon_path.exists(): - base_container = base_container.with_file( - METADATA_ICON_FILE_NAME, get_metadata_icon_file_from_path(context, metadata_icon_path) - ) - - self.poetry_run_container = ( - base_container.with_secret_variable("DOCKER_HUB_USERNAME", docker_hub_username_secret) - .with_secret_variable("DOCKER_HUB_PASSWORD", docker_hub_password_secret) - .with_secret_variable("GCS_CREDENTIALS", metadata_service_gcs_credentials_secret) - # The cache buster ensures we always run the upload command (in case of remote bucket change) - .with_env_variable("CACHEBUSTER", str(uuid.uuid4())) - ) - - async def _run(self) -> StepResult: - upload_command = ["metadata_service", "upload", METADATA_FILE_NAME, self.gcs_bucket_name] - - if self.pre_release: - upload_command += ["--prerelease", self.pre_release_tag] - - return await super()._run(upload_command) - - -class DeployOrchestrator(Step): - title = "Deploy Metadata Orchestrator to Dagster Cloud" - deploy_dagster_command = [ - "dagster-cloud", - "serverless", - "deploy-python-executable", - "--location-name", - "metadata_service_orchestrator", - "--location-file", - "dagster_cloud.yaml", - "--organization", - "airbyte-connectors", - "--deployment", - "prod", - "--python-version", - "3.9", - ] - - async def _run(self) -> StepResult: - parent_dir = self.context.get_repo_dir(METADATA_DIR) - python_base = with_python_base(self.context, "3.9") - python_with_dependencies = with_pip_packages(python_base, ["dagster-cloud==1.2.6", "pydantic==1.10.6", "poetry2setup==1.1.0"]) - dagster_cloud_api_token_secret: dagger.Secret = get_secret_host_variable( - self.context.dagger_client, "DAGSTER_CLOUD_METADATA_API_TOKEN" - ) - - container_to_run = ( - python_with_dependencies.with_mounted_directory("/src", parent_dir) - .with_secret_variable("DAGSTER_CLOUD_API_TOKEN", dagster_cloud_api_token_secret) - .with_workdir(f"/src/{METADATA_ORCHESTRATOR_MODULE_PATH}") - .with_exec(["/bin/sh", "-c", "poetry2setup >> setup.py"]) - .with_exec(self.deploy_dagster_command) - ) - return await self.get_step_result(container_to_run) - - -class TestOrchestrator(PoetryRun): - def __init__(self, context: PipelineContext): - super().__init__( - context=context, - title="Test Metadata Orchestrator", - parent_dir_path=METADATA_DIR, - module_path=METADATA_ORCHESTRATOR_MODULE_PATH, - ) - - async def _run(self) -> StepResult: - return await super()._run(["pytest"]) - - -# PIPELINES - - -async def run_metadata_validation_pipeline( - is_local: bool, - git_branch: str, - git_revision: str, - gha_workflow_run_url: Optional[str], - dagger_logs_url: Optional[str], - pipeline_start_timestamp: Optional[int], - ci_context: Optional[str], - metadata_to_validate: Set[Path], -) -> bool: - metadata_pipeline_context = PipelineContext( - pipeline_name="Validate metadata.yaml files", - is_local=is_local, - git_branch=git_branch, - git_revision=git_revision, - gha_workflow_run_url=gha_workflow_run_url, - dagger_logs_url=dagger_logs_url, - pipeline_start_timestamp=pipeline_start_timestamp, - ci_context=ci_context, - ) - - async with dagger.Connection(DAGGER_CONFIG) as dagger_client: - metadata_pipeline_context.dagger_client = dagger_client.pipeline(metadata_pipeline_context.pipeline_name) - async with metadata_pipeline_context: - validation_steps = [MetadataValidation(metadata_pipeline_context, metadata_path).run for metadata_path in metadata_to_validate] - - results = await execute_concurrently(validation_steps, concurrency=10) - metadata_pipeline_context.report = Report( - pipeline_context=metadata_pipeline_context, steps_results=results, name="METADATA VALIDATION RESULTS" - ) - - return metadata_pipeline_context.report.success - - -async def run_metadata_lib_test_pipeline( - is_local: bool, - git_branch: str, - git_revision: str, - gha_workflow_run_url: Optional[str], - dagger_logs_url: Optional[str], - pipeline_start_timestamp: Optional[int], - ci_context: Optional[str], -) -> bool: - metadata_pipeline_context = PipelineContext( - pipeline_name="Metadata Service Lib Unit Test Pipeline", - is_local=is_local, - git_branch=git_branch, - git_revision=git_revision, - gha_workflow_run_url=gha_workflow_run_url, - dagger_logs_url=dagger_logs_url, - pipeline_start_timestamp=pipeline_start_timestamp, - ci_context=ci_context, - ) - - async with dagger.Connection(DAGGER_CONFIG) as dagger_client: - metadata_pipeline_context.dagger_client = dagger_client.pipeline(metadata_pipeline_context.pipeline_name) - async with metadata_pipeline_context: - test_lib_step = PoetryRun( - context=metadata_pipeline_context, - title="Test Metadata Service Lib", - parent_dir_path=METADATA_DIR, - module_path=METADATA_LIB_MODULE_PATH, - ) - result = await test_lib_step.run(["pytest"]) - metadata_pipeline_context.report = Report( - pipeline_context=metadata_pipeline_context, steps_results=[result], name="METADATA LIB TEST RESULTS" - ) - - return metadata_pipeline_context.report.success - - -async def run_metadata_orchestrator_test_pipeline( - is_local: bool, - git_branch: str, - git_revision: str, - gha_workflow_run_url: Optional[str], - dagger_logs_url: Optional[str], - pipeline_start_timestamp: Optional[int], - ci_context: Optional[str], -) -> bool: - metadata_pipeline_context = PipelineContext( - pipeline_name="Metadata Service Orchestrator Unit Test Pipeline", - is_local=is_local, - git_branch=git_branch, - git_revision=git_revision, - gha_workflow_run_url=gha_workflow_run_url, - dagger_logs_url=dagger_logs_url, - pipeline_start_timestamp=pipeline_start_timestamp, - ci_context=ci_context, - ) - - async with dagger.Connection(DAGGER_CONFIG) as dagger_client: - metadata_pipeline_context.dagger_client = dagger_client.pipeline(metadata_pipeline_context.pipeline_name) - async with metadata_pipeline_context: - test_orch_step = TestOrchestrator(context=metadata_pipeline_context) - result = await test_orch_step.run() - metadata_pipeline_context.report = Report( - pipeline_context=metadata_pipeline_context, steps_results=[result], name="METADATA ORCHESTRATOR TEST RESULTS" - ) - - return metadata_pipeline_context.report.success - - -async def run_metadata_upload_pipeline( - is_local: bool, - git_branch: str, - git_revision: str, - gha_workflow_run_url: Optional[str], - dagger_logs_url: Optional[str], - pipeline_start_timestamp: Optional[int], - ci_context: Optional[str], - metadata_to_upload: Set[Path], - gcs_bucket_name: str, -) -> bool: - pipeline_context = PipelineContext( - pipeline_name="Metadata Upload Pipeline", - is_local=is_local, - git_branch=git_branch, - git_revision=git_revision, - gha_workflow_run_url=gha_workflow_run_url, - dagger_logs_url=dagger_logs_url, - pipeline_start_timestamp=pipeline_start_timestamp, - ci_context=ci_context, - ) - - async with dagger.Connection(DAGGER_CONFIG) as dagger_client: - pipeline_context.dagger_client = dagger_client.pipeline(pipeline_context.pipeline_name) - async with pipeline_context: - get_secret = functools.partial(get_secret_host_variable, pipeline_context.dagger_client) - results = await execute_concurrently( - [ - MetadataUpload( - context=pipeline_context, - metadata_service_gcs_credentials_secret=get_secret("GCS_CREDENTIALS"), - docker_hub_username_secret=get_secret("DOCKER_HUB_USERNAME"), - docker_hub_password_secret=get_secret("DOCKER_HUB_PASSWORD"), - metadata_bucket_name=gcs_bucket_name, - metadata_path=metadata_path, - ).run - for metadata_path in metadata_to_upload - ] - ) - pipeline_context.report = Report(pipeline_context, results, name="METADATA UPLOAD RESULTS") - - return pipeline_context.report.success - - -async def run_metadata_orchestrator_deploy_pipeline( - is_local: bool, - git_branch: str, - git_revision: str, - gha_workflow_run_url: Optional[str], - dagger_logs_url: Optional[str], - pipeline_start_timestamp: Optional[int], - ci_context: Optional[str], -) -> bool: - metadata_pipeline_context = PipelineContext( - pipeline_name="Metadata Service Orchestrator Unit Test Pipeline", - is_local=is_local, - git_branch=git_branch, - git_revision=git_revision, - gha_workflow_run_url=gha_workflow_run_url, - dagger_logs_url=dagger_logs_url, - pipeline_start_timestamp=pipeline_start_timestamp, - ci_context=ci_context, - ) - - async with dagger.Connection(DAGGER_CONFIG) as dagger_client: - metadata_pipeline_context.dagger_client = dagger_client.pipeline(metadata_pipeline_context.pipeline_name) - - async with metadata_pipeline_context: - steps = [TestOrchestrator(context=metadata_pipeline_context), DeployOrchestrator(context=metadata_pipeline_context)] - steps_results = await run_steps(steps) - metadata_pipeline_context.report = Report( - pipeline_context=metadata_pipeline_context, steps_results=steps_results, name="METADATA ORCHESTRATOR DEPLOY RESULTS" - ) - return metadata_pipeline_context.report.success diff --git a/airbyte-ci/connectors/pipelines/pipelines/publish.py b/airbyte-ci/connectors/pipelines/pipelines/publish.py deleted file mode 100644 index 2948cc9ef86a..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/publish.py +++ /dev/null @@ -1,317 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import json -import uuid -from typing import List, Tuple - -import anyio -from airbyte_protocol.models.airbyte_protocol import ConnectorSpecification -from dagger import Container, ExecError, File, ImageLayerCompression, QueryError -from pipelines import builds, consts -from pipelines.actions import environments -from pipelines.actions.remote_storage import upload_to_gcs -from pipelines.bases import ConnectorReport, Step, StepResult, StepStatus -from pipelines.contexts import PublishConnectorContext -from pipelines.pipelines import metadata -from pydantic import ValidationError - - -class CheckConnectorImageDoesNotExist(Step): - title = "Check if the connector docker image does not exist on the registry." - - async def _run(self) -> StepResult: - docker_repository, docker_tag = self.context.docker_image.split(":") - crane_ls = ( - environments.with_crane( - self.context, - ) - .with_env_variable("CACHEBUSTER", str(uuid.uuid4())) - .with_exec(["ls", docker_repository]) - ) - try: - crane_ls_stdout = await crane_ls.stdout() - except ExecError as e: - if "NAME_UNKNOWN" in e.stderr: - return StepResult(self, status=StepStatus.SUCCESS, stdout=f"The docker repository {docker_repository} does not exist.") - else: - return StepResult(self, status=StepStatus.FAILURE, stderr=e.stderr, stdout=e.stdout) - else: # The docker repo exists and ls was successful - existing_tags = crane_ls_stdout.split("\n") - docker_tag_already_exists = docker_tag in existing_tags - if docker_tag_already_exists: - return StepResult(self, status=StepStatus.SKIPPED, stderr=f"{self.context.docker_image} already exists.") - return StepResult(self, status=StepStatus.SUCCESS, stdout=f"No manifest found for {self.context.docker_image}.") - - -class PushConnectorImageToRegistry(Step): - title = "Push connector image to registry" - - @property - def latest_docker_image_name(self): - return f"{self.context.docker_repository}:latest" - - async def _run(self, built_containers_per_platform: List[Container], attempts: int = 3) -> StepResult: - try: - image_ref = await built_containers_per_platform[0].publish( - f"docker.io/{self.context.docker_image}", - platform_variants=built_containers_per_platform[1:], - forced_compression=ImageLayerCompression.Gzip, - ) - if not self.context.pre_release: - image_ref = await built_containers_per_platform[0].publish( - f"docker.io/{self.latest_docker_image_name}", - platform_variants=built_containers_per_platform[1:], - forced_compression=ImageLayerCompression.Gzip, - ) - return StepResult(self, status=StepStatus.SUCCESS, stdout=f"Published {image_ref}") - except QueryError as e: - if attempts > 0: - self.context.logger.error(str(e)) - self.context.logger.warn(f"Failed to publish {self.context.docker_image}. Retrying. {attempts} attempts left.") - await anyio.sleep(5) - return await self._run(built_containers_per_platform, attempts - 1) - return StepResult(self, status=StepStatus.FAILURE, stderr=str(e)) - - -class PullConnectorImageFromRegistry(Step): - title = "Pull connector image from registry" - - async def check_if_image_only_has_gzip_layers(self) -> bool: - """Check if the image only has gzip layers. - Docker version > 21 can create images that has some layers compressed with zstd. - These layers are not supported by previous docker versions. - We want to make sure that the image we are about to release is compatible with all docker versions. - We use crane to inspect the manifest of the image and check if it only has gzip layers. - """ - for platform in consts.BUILD_PLATFORMS: - inspect = environments.with_crane(self.context).with_exec( - ["manifest", "--platform", f"{str(platform)}", f"docker.io/{self.context.docker_image}"] - ) - try: - inspect_stdout = await inspect.stdout() - except ExecError as e: - raise Exception(f"Failed to inspect {self.context.docker_image}: {e.stderr}") from e - try: - for layer in json.loads(inspect_stdout)["layers"]: - if not layer["mediaType"].endswith("gzip"): - return False - return True - except (KeyError, json.JSONDecodeError) as e: - raise Exception(f"Failed to parse manifest for {self.context.docker_image}: {inspect_stdout}") from e - - async def _run(self, attempt: int = 3) -> StepResult: - try: - try: - await self.context.dagger_client.container().from_(f"docker.io/{self.context.docker_image}").with_exec(["spec"]) - except ExecError: - if attempt > 0: - await anyio.sleep(10) - return await self._run(attempt - 1) - else: - return StepResult(self, status=StepStatus.FAILURE, stderr=f"Failed to pull {self.context.docker_image}") - if not await self.check_if_image_only_has_gzip_layers(): - return StepResult( - self, - status=StepStatus.FAILURE, - stderr=f"Image {self.context.docker_image} does not only have gzip compressed layers. Please rebuild the connector with Docker < 21.", - ) - else: - return StepResult( - self, - status=StepStatus.SUCCESS, - stdout=f"Pulled {self.context.docker_image} and validated it has gzip only compressed layers and we can run spec on it.", - ) - except QueryError as e: - if attempt > 0: - await anyio.sleep(10) - return await self._run(attempt - 1) - return StepResult(self, status=StepStatus.FAILURE, stderr=str(e)) - - -class InvalidSpecOutputError(Exception): - pass - - -class UploadSpecToCache(Step): - title = "Upload connector spec to spec cache bucket" - default_spec_file_name = "spec.json" - cloud_spec_file_name = "spec.cloud.json" - - @property - def spec_key_prefix(self): - return "specs/" + self.context.docker_image.replace(":", "/") - - @property - def cloud_spec_key(self): - return f"{self.spec_key_prefix}/{self.cloud_spec_file_name}" - - @property - def oss_spec_key(self): - return f"{self.spec_key_prefix}/{self.default_spec_file_name}" - - def _parse_spec_output(self, spec_output: str) -> str: - parsed_spec_message = None - for line in spec_output.split("\n"): - try: - parsed_json = json.loads(line) - if parsed_json["type"] == "SPEC": - parsed_spec_message = parsed_json - break - except (json.JSONDecodeError, KeyError): - continue - if parsed_spec_message: - parsed_spec = parsed_spec_message["spec"] - try: - ConnectorSpecification.parse_obj(parsed_spec) - return json.dumps(parsed_spec) - except (ValidationError, ValueError) as e: - raise InvalidSpecOutputError(f"The SPEC message did not pass schema validation: {str(e)}.") - raise InvalidSpecOutputError("No spec found in the output of the SPEC command.") - - async def _get_connector_spec(self, connector: Container, deployment_mode: str) -> str: - spec_output = await connector.with_env_variable("DEPLOYMENT_MODE", deployment_mode).with_exec(["spec"]).stdout() - return self._parse_spec_output(spec_output) - - async def _get_spec_as_file(self, spec: str, name="spec_to_cache.json") -> File: - return (await self.context.get_connector_dir()).with_new_file(name, contents=spec).file(name) - - async def _run(self, built_connector: Container) -> StepResult: - try: - oss_spec: str = await self._get_connector_spec(built_connector, "OSS") - cloud_spec: str = await self._get_connector_spec(built_connector, "CLOUD") - except InvalidSpecOutputError as e: - return StepResult(self, status=StepStatus.FAILURE, stderr=str(e)) - - specs_to_uploads: List[Tuple[str, File]] = [(self.oss_spec_key, await self._get_spec_as_file(oss_spec))] - - if oss_spec != cloud_spec: - specs_to_uploads.append((self.cloud_spec_key, await self._get_spec_as_file(cloud_spec, "cloud_spec_to_cache.json"))) - - for key, file in specs_to_uploads: - exit_code, stdout, stderr = await upload_to_gcs( - self.context.dagger_client, - file, - key, - self.context.spec_cache_bucket_name, - self.context.spec_cache_gcs_credentials_secret, - flags=['--cache-control="no-cache"'], - ) - if exit_code != 0: - return StepResult(self, status=StepStatus.FAILURE, stdout=stdout, stderr=stderr) - return StepResult(self, status=StepStatus.SUCCESS, stdout="Uploaded connector spec to spec cache bucket.") - - -async def run_connector_publish_pipeline(context: PublishConnectorContext, semaphore: anyio.Semaphore) -> ConnectorReport: - """Run a publish pipeline for a single connector. - - 1. Validate the metadata file. - 2. Check if the connector image already exists. - 3. Build the connector, with platform variants. - 4. Push the connector to DockerHub, with platform variants. - 5. Upload its spec to the spec cache bucket. - 6. Upload its metadata file to the metadata service bucket. - - Returns: - ConnectorReport: The reports holding publish results. - """ - - metadata_upload_step = metadata.MetadataUpload( - context=context, - metadata_service_gcs_credentials_secret=context.metadata_service_gcs_credentials_secret, - docker_hub_username_secret=context.docker_hub_username_secret, - docker_hub_password_secret=context.docker_hub_password_secret, - metadata_bucket_name=context.metadata_bucket_name, - metadata_path=context.metadata_path, - pre_release=context.pre_release, - pre_release_tag=context.docker_image_tag, - ) - - def create_connector_report(results: List[StepResult]) -> ConnectorReport: - report = ConnectorReport(context, results, name="PUBLISH RESULTS") - context.report = report - return report - - async with semaphore: - async with context: - # TODO add a strucutre to hold the results of each step. and perform skips and failures - - results = [] - - metadata_validation_results = await metadata.MetadataValidation(context, context.metadata_path).run() - results.append(metadata_validation_results) - - # Exit early if the metadata file is invalid. - if metadata_validation_results.status is not StepStatus.SUCCESS: - return create_connector_report(results) - - check_connector_image_results = await CheckConnectorImageDoesNotExist(context).run() - results.append(check_connector_image_results) - - # If the connector image already exists, we don't need to build it, but we still need to upload the metadata file. - # We also need to upload the spec to the spec cache bucket. - if check_connector_image_results.status is StepStatus.SKIPPED: - context.logger.info( - "The connector version is already published. Let's upload metadata.yaml and spec to GCS even if no version bump happened." - ) - already_published_connector = context.dagger_client.container().from_(context.docker_image) - upload_to_spec_cache_results = await UploadSpecToCache(context).run(already_published_connector) - results.append(upload_to_spec_cache_results) - if upload_to_spec_cache_results.status is not StepStatus.SUCCESS: - return create_connector_report(results) - metadata_upload_results = await metadata_upload_step.run() - results.append(metadata_upload_results) - - # Exit early if the connector image already exists or has failed to build - if check_connector_image_results.status is not StepStatus.SUCCESS: - return create_connector_report(results) - - build_connector_results = await builds.run_connector_build(context) - results.append(build_connector_results) - - # Exit early if the connector image failed to build - if build_connector_results.status is not StepStatus.SUCCESS: - return create_connector_report(results) - - built_connector_platform_variants = list(build_connector_results.output_artifact.values()) - push_connector_image_results = await PushConnectorImageToRegistry(context).run(built_connector_platform_variants) - results.append(push_connector_image_results) - - # Exit early if the connector image failed to push - if push_connector_image_results.status is not StepStatus.SUCCESS: - return create_connector_report(results) - - # Make sure the image published is healthy by pulling it and running SPEC on it. - # See https://github.com/airbytehq/airbyte/issues/26085 - pull_connector_image_results = await PullConnectorImageFromRegistry(context).run() - results.append(pull_connector_image_results) - - # Exit early if the connector image failed to pull - if pull_connector_image_results.status is not StepStatus.SUCCESS: - return create_connector_report(results) - - upload_to_spec_cache_results = await UploadSpecToCache(context).run(built_connector_platform_variants[0]) - results.append(upload_to_spec_cache_results) - if upload_to_spec_cache_results.status is not StepStatus.SUCCESS: - return create_connector_report(results) - - metadata_upload_results = await metadata_upload_step.run() - results.append(metadata_upload_results) - - return create_connector_report(results) - - -def reorder_contexts(contexts: List[PublishConnectorContext]) -> List[PublishConnectorContext]: - """Reorder contexts so that the ones that are for strict-encrypt/secure connectors come first. - The metadata upload on publish checks if the the connectors referenced in the metadata file are already published to DockerHub. - Non strict-encrypt variant reference the strict-encrypt variant in their metadata file for cloud. - So if we publish the non strict-encrypt variant first, the metadata upload will fail if the strict-encrypt variant is not published yet. - As strict-encrypt variant are often modified in the same PR as the non strict-encrypt variant, we want to publish them first. - """ - - def is_secure_variant(context: PublishConnectorContext) -> bool: - SECURE_VARIANT_KEYS = ["secure", "strict-encrypt"] - return any(key in context.connector.technical_name for key in SECURE_VARIANT_KEYS) - - return sorted(contexts, key=lambda context: (is_secure_variant(context), context.connector.technical_name), reverse=True) diff --git a/airbyte-ci/connectors/pipelines/pipelines/tests/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/tests/__init__.py deleted file mode 100644 index 0d3d26c27e1a..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/tests/__init__.py +++ /dev/null @@ -1,126 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# -"""This module groups factory like functions to dispatch tests steps according to the connector under test language.""" - -import itertools -from typing import List - -import anyio -import asyncer -from connector_ops.utils import METADATA_FILE_NAME, ConnectorLanguage -from pipelines.bases import ConnectorReport, StepResult -from pipelines.contexts import ConnectorContext -from pipelines.pipelines.metadata import MetadataValidation -from pipelines.tests import java_connectors, python_connectors -from pipelines.tests.common import QaChecks, VersionFollowsSemverCheck, VersionIncrementCheck - -LANGUAGE_MAPPING = { - "run_all_tests": { - ConnectorLanguage.PYTHON: python_connectors.run_all_tests, - ConnectorLanguage.LOW_CODE: python_connectors.run_all_tests, - ConnectorLanguage.JAVA: java_connectors.run_all_tests, - }, - "run_code_format_checks": { - ConnectorLanguage.PYTHON: python_connectors.run_code_format_checks, - ConnectorLanguage.LOW_CODE: python_connectors.run_code_format_checks, - # ConnectorLanguage.JAVA: java_connectors.run_code_format_checks - }, -} - - -async def run_metadata_validation(context: ConnectorContext) -> List[StepResult]: - """Run the metadata validation on a connector. - Args: - context (ConnectorContext): The current connector context. - - Returns: - List[StepResult]: The results of the metadata validation steps. - """ - return [await MetadataValidation(context, context.connector.code_directory / METADATA_FILE_NAME).run()] - - -async def run_version_checks(context: ConnectorContext) -> List[StepResult]: - """Run the version checks on a connector. - - Args: - context (ConnectorContext): The current connector context. - - Returns: - List[StepResult]: The results of the version checks steps. - """ - return [await VersionFollowsSemverCheck(context).run(), await VersionIncrementCheck(context).run()] - - -async def run_qa_checks(context: ConnectorContext) -> List[StepResult]: - """Run the QA checks on a connector. - - Args: - context (ConnectorContext): The current connector context. - - Returns: - List[StepResult]: The results of the QA checks steps. - """ - return [await QaChecks(context).run()] - - -async def run_code_format_checks(context: ConnectorContext) -> List[StepResult]: - """Run the code format checks according to the connector language. - - Args: - context (ConnectorContext): The current connector context. - - Returns: - List[StepResult]: The results of the code format checks steps. - """ - if _run_code_format_checks := LANGUAGE_MAPPING["run_code_format_checks"].get(context.connector.language): - return await _run_code_format_checks(context) - else: - context.logger.warning(f"No code format checks defined for connector language {context.connector.language}!") - return [] - - -async def run_all_tests(context: ConnectorContext) -> List[StepResult]: - """Run all the tests steps according to the connector language. - - Args: - context (ConnectorContext): The current connector context. - - Returns: - List[StepResult]: The results of the tests steps. - """ - if _run_all_tests := LANGUAGE_MAPPING["run_all_tests"].get(context.connector.language): - return await _run_all_tests(context) - else: - context.logger.warning(f"No tests defined for connector language {context.connector.language}!") - return [] - - -async def run_connector_test_pipeline(context: ConnectorContext, semaphore: anyio.Semaphore) -> ConnectorReport: - """Run a test pipeline for a single connector. - - A visual DAG can be found on the README.md file of the pipelines modules. - - Args: - context (ConnectorContext): The initialized connector context. - - Returns: - ConnectorReport: The test reports holding tests results. - """ - async with semaphore: - async with context: - async with asyncer.create_task_group() as task_group: - tasks = [ - task_group.soonify(run_all_tests)(context), - task_group.soonify(run_code_format_checks)(context), - ] - if not context.code_tests_only: - tasks += [ - task_group.soonify(run_metadata_validation)(context), - task_group.soonify(run_version_checks)(context), - task_group.soonify(run_qa_checks)(context), - ] - results = list(itertools.chain(*(task.value for task in tasks))) - context.report = ConnectorReport(context, steps_results=results, name="TEST RESULTS") - - return context.report diff --git a/airbyte-ci/connectors/pipelines/pipelines/tests/common.py b/airbyte-ci/connectors/pipelines/pipelines/tests/common.py deleted file mode 100644 index 374152d87e8a..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/tests/common.py +++ /dev/null @@ -1,283 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -"""This module groups steps made to run tests agnostic to a connector language.""" - -import datetime -import os -from abc import ABC, abstractmethod -from functools import cached_property -from typing import ClassVar, List, Optional - -import requests -import semver -import yaml -from connector_ops.utils import Connector -from dagger import Container, Directory, File -from pipelines import hacks -from pipelines.actions import environments -from pipelines.bases import CIContext, PytestStep, Step, StepResult, StepStatus -from pipelines.utils import METADATA_FILE_NAME - - -class VersionCheck(Step, ABC): - """A step to validate the connector version was bumped if files were modified""" - - GITHUB_URL_PREFIX_FOR_CONNECTORS = "https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-integrations/connectors" - failure_message: ClassVar - should_run = True - - @property - def github_master_metadata_url(self): - return f"{self.GITHUB_URL_PREFIX_FOR_CONNECTORS}/{self.context.connector.technical_name}/{METADATA_FILE_NAME}" - - @cached_property - def master_metadata(self) -> Optional[dict]: - response = requests.get(self.github_master_metadata_url) - - # New connectors will not have a metadata file in master - if not response.ok: - return None - return yaml.safe_load(response.text) - - @property - def master_connector_version(self) -> semver.Version: - metadata = self.master_metadata - if not metadata: - return semver.Version.parse("0.0.0") - - return semver.Version.parse(str(metadata["data"]["dockerImageTag"])) - - @property - def current_connector_version(self) -> semver.Version: - return semver.Version.parse(str(self.context.metadata["dockerImageTag"])) - - @property - def success_result(self) -> StepResult: - return StepResult(self, status=StepStatus.SUCCESS) - - @property - def failure_result(self) -> StepResult: - return StepResult(self, status=StepStatus.FAILURE, stderr=self.failure_message) - - @abstractmethod - def validate(self) -> StepResult: - raise NotImplementedError() - - async def _run(self) -> StepResult: - if not self.should_run: - return StepResult(self, status=StepStatus.SKIPPED, stdout="No modified files required a version bump.") - if self.context.ci_context in [CIContext.MASTER, CIContext.NIGHTLY_BUILDS]: - return StepResult(self, status=StepStatus.SKIPPED, stdout="Version check are not running in master context.") - try: - return self.validate() - except (requests.HTTPError, ValueError, TypeError) as e: - return StepResult(self, status=StepStatus.FAILURE, stderr=str(e)) - - -class VersionIncrementCheck(VersionCheck): - title = "Connector version increment check" - - BYPASS_CHECK_FOR = [ - METADATA_FILE_NAME, - "acceptance-test-config.yml", - "README.md", - "bootstrap.md", - ".dockerignore", - "unit_tests", - "integration_tests", - "src/test", - "src/test-integration", - "src/test-performance", - "build.gradle", - ] - - @property - def failure_message(self) -> str: - return f"The dockerImageTag in {METADATA_FILE_NAME} was not incremented. The files you modified should lead to a version bump. Master version is {self.master_connector_version}, current version is {self.current_connector_version}" - - @property - def should_run(self) -> bool: - for filename in self.context.modified_files: - relative_path = str(filename).replace(str(self.context.connector.code_directory) + "/", "") - if not any([relative_path.startswith(to_bypass) for to_bypass in self.BYPASS_CHECK_FOR]): - return True - return False - - def validate(self) -> StepResult: - if not self.current_connector_version > self.master_connector_version: - return self.failure_result - return self.success_result - - -class VersionFollowsSemverCheck(VersionCheck): - title = "Connector version semver check" - - @property - def failure_message(self) -> str: - return f"The dockerImageTag in {METADATA_FILE_NAME} is not following semantic versioning or was decremented. Master version is {self.master_connector_version}, current version is {self.current_connector_version}" - - def validate(self) -> StepResult: - try: - if not self.current_connector_version >= self.master_connector_version: - return self.failure_result - except ValueError: - return self.failure_result - return self.success_result - - -class QaChecks(Step): - """A step to run QA checks for a connector.""" - - title = "QA checks" - - async def _run(self) -> StepResult: - """Run QA checks on a connector. - - The QA checks are defined in this module: - https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connector_ops/connector_ops/qa_checks.py - - Args: - context (ConnectorContext): The current test context, providing a connector object, a dagger client and a repository directory. - Returns: - StepResult: Failure or success of the QA checks with stdout and stderr. - """ - connector_ops = await environments.with_connector_ops(self.context) - include = [ - str(self.context.connector.code_directory), - str(self.context.connector.documentation_file_path), - str(self.context.connector.migration_guide_file_path), - str(self.context.connector.icon_path), - ] - if ( - self.context.connector.technical_name.endswith("strict-encrypt") - or self.context.connector.technical_name == "source-file-secure" - ): - original_connector = Connector(self.context.connector.technical_name.replace("-strict-encrypt", "").replace("-secure", "")) - include += [ - str(original_connector.code_directory), - str(original_connector.documentation_file_path), - str(original_connector.icon_path), - ] - - filtered_repo = self.context.get_repo_dir( - include=include, - ) - - qa_checks = ( - connector_ops.with_mounted_directory("/airbyte", filtered_repo) - .with_workdir("/airbyte") - .with_exec(["run-qa-checks", f"connectors/{self.context.connector.technical_name}"]) - ) - - return await self.get_step_result(qa_checks) - - -class AcceptanceTests(PytestStep): - """A step to run acceptance tests for a connector if it has an acceptance test config file.""" - - title = "Acceptance tests" - CONTAINER_TEST_INPUT_DIRECTORY = "/test_input" - CONTAINER_SECRETS_DIRECTORY = "/test_input/secrets" - - @property - def base_cat_command(self) -> List[str]: - return [ - "python", - "-m", - "pytest", - "-p", - "connector_acceptance_test.plugin", - "--acceptance-test-config", - self.CONTAINER_TEST_INPUT_DIRECTORY, - ] - - async def get_cat_command(self, connector_dir: Directory) -> List[str]: - """ - Connectors can optionally setup or teardown resources before and after the acceptance tests are run. - This is done via the acceptance.py file in their integration_tests directory. - We append this module as a plugin the acceptance will use. - """ - cat_command = self.base_cat_command - if "integration_tests" in await connector_dir.entries(): - if "acceptance.py" in await connector_dir.directory("integration_tests").entries(): - cat_command += ["-p", "integration_tests.acceptance"] - return cat_command - - async def _run(self, connector_under_test_image_tar: File) -> StepResult: - """Run the acceptance test suite on a connector dev image. Build the connector acceptance test image if the tag is :dev. - - Args: - connector_under_test_image_tar (File): The file holding the tar archive of the connector image. - - Returns: - StepResult: Failure or success of the acceptances tests with stdout and stderr. - """ - if not self.context.connector.acceptance_test_config: - return StepResult(self, StepStatus.SKIPPED) - connector_dir = await self.context.get_connector_dir() - cat_container = await self._build_connector_acceptance_test(connector_under_test_image_tar, connector_dir) - cat_command = await self.get_cat_command(connector_dir) - cat_container = cat_container.with_(hacks.never_fail_exec(cat_command)) - step_result = await self.get_step_result(cat_container) - secret_dir = cat_container.directory(self.CONTAINER_SECRETS_DIRECTORY) - - if secret_files := await secret_dir.entries(): - for file_path in secret_files: - if file_path.startswith("updated_configurations"): - self.context.updated_secrets_dir = secret_dir - break - return step_result - - async def get_cache_buster(self, connector_under_test_image_tar: File) -> str: - """ - This bursts the CAT cached results everyday and on new version or image size change. - It's cool because in case of a partially failing nightly build the connectors that already ran CAT won't re-run CAT. - We keep the guarantee that a CAT runs everyday. - - Args: - connector_under_test_image_tar (File): The file holding the tar archive of the connector image. - Returns: - str: A string representing the cachebuster value. - """ - return ( - datetime.datetime.utcnow().strftime("%Y%m%d") - + self.context.connector.version - + str(await connector_under_test_image_tar.size()) - ) - - async def _build_connector_acceptance_test(self, connector_under_test_image_tar: File, test_input: Directory) -> Container: - """Create a container to run connector acceptance tests. - - Args: - connector_under_test_image_tar (File): The file containing the tar archive of the image of the connector under test. - test_input (Directory): The connector under test directory. - Returns: - Container: A container with connector acceptance tests installed. - """ - - if self.context.connector_acceptance_test_image.endswith(":dev"): - cat_container = self.context.connector_acceptance_test_source_dir.docker_build() - else: - cat_container = self.dagger_client.container().from_(self.context.connector_acceptance_test_image) - - cat_container = ( - cat_container.with_env_variable("RUN_IN_AIRBYTE_CI", "1") - .with_exec(["mkdir", "/dagger_share"], skip_entrypoint=True) - .with_env_variable("CACHEBUSTER", await self.get_cache_buster(connector_under_test_image_tar)) - .with_mounted_file("/dagger_share/connector_under_test_image.tar", connector_under_test_image_tar) - .with_env_variable("CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH", "/dagger_share/connector_under_test_image.tar") - .with_workdir("/test_input") - .with_mounted_directory("/test_input", test_input) - .with_(environments.mounted_connector_secrets(self.context, secret_directory_path="/test_input/secrets")) - ) - if "_EXPERIMENTAL_DAGGER_RUNNER_HOST" in os.environ: - self.context.logger.info("Using experimental dagger runner host to run CAT with dagger-in-dagger") - cat_container = cat_container.with_env_variable( - "_EXPERIMENTAL_DAGGER_RUNNER_HOST", "unix:///var/run/buildkit/buildkitd.sock" - ).with_unix_socket( - "/var/run/buildkit/buildkitd.sock", self.context.dagger_client.host().unix_socket("/var/run/buildkit/buildkitd.sock") - ) - - return cat_container.with_unix_socket("/var/run/docker.sock", self.context.dagger_client.host().unix_socket("/var/run/docker.sock")) diff --git a/airbyte-ci/connectors/pipelines/pipelines/tests/java_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/tests/java_connectors.py deleted file mode 100644 index 052547ac46e4..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/tests/java_connectors.py +++ /dev/null @@ -1,122 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -"""This module groups steps made to run tests for a specific Java connector given a test context.""" - -from typing import List, Optional - -import anyio -from dagger import File, QueryError -from pipelines.actions import environments, secrets -from pipelines.bases import StepResult, StepStatus -from pipelines.builds import LOCAL_BUILD_PLATFORM -from pipelines.builds.java_connectors import BuildConnectorDistributionTar, BuildConnectorImage -from pipelines.builds.normalization import BuildOrPullNormalization -from pipelines.contexts import ConnectorContext -from pipelines.gradle import GradleTask -from pipelines.tests.common import AcceptanceTests -from pipelines.utils import export_container_to_tarball - - -class IntegrationTests(GradleTask): - """A step to run integrations tests for Java connectors using the integrationTestJava Gradle task.""" - - gradle_task_name = "integrationTest" - DEFAULT_TASKS_TO_EXCLUDE = ["airbyteDocker"] - title = "Java Connector Integration Tests" - - async def _load_normalization_image(self, normalization_tar_file: File): - normalization_image_tag = f"{self.context.connector.normalization_repository}:dev" - self.context.logger.info("Load the normalization image to the docker host.") - await environments.load_image_to_docker_host(self.context, normalization_tar_file, normalization_image_tag) - self.context.logger.info("Successfully loaded the normalization image to the docker host.") - - async def _load_connector_image(self, connector_tar_file: File): - connector_image_tag = f"airbyte/{self.context.connector.technical_name}:dev" - self.context.logger.info("Load the connector image to the docker host") - await environments.load_image_to_docker_host(self.context, connector_tar_file, connector_image_tag) - self.context.logger.info("Successfully loaded the connector image to the docker host.") - - async def _run(self, connector_tar_file: File, normalization_tar_file: Optional[File]) -> StepResult: - try: - async with anyio.create_task_group() as tg: - if normalization_tar_file: - tg.start_soon(self._load_normalization_image, normalization_tar_file) - tg.start_soon(self._load_connector_image, connector_tar_file) - return await super()._run() - except QueryError as e: - return StepResult(self, StepStatus.FAILURE, stderr=str(e)) - - -class UnitTests(GradleTask): - """A step to run unit tests for Java connectors.""" - - title = "Java Connector Unit Tests" - gradle_task_name = "test" - context: ConnectorContext - - @property - def gradle_task_options(self) -> tuple[str, ...]: - """Return the Gradle task options to use when running unit tests.""" - if self.context.fail_fast: - return ("--fail-fast",) - - return () - - -async def run_all_tests(context: ConnectorContext) -> List[StepResult]: - """Run all tests for a Java connectors. - - - Build the normalization image if the connector supports it. - - Run unit tests with Gradle. - - Build connector image with Gradle. - - Run integration and acceptance test in parallel using the built connector and normalization images. - - Args: - context (ConnectorContext): The current connector context. - - Returns: - List[StepResult]: The results of all the tests steps. - """ - context.connector_secrets = await secrets.get_connector_secrets(context) - step_results = [] - - unit_tests_results = await UnitTests(context).run() - step_results.append(unit_tests_results) - - if context.fail_fast and unit_tests_results.status is StepStatus.FAILURE: - return step_results - - build_distribution_tar_results = await BuildConnectorDistributionTar(context).run() - step_results.append(build_distribution_tar_results) - if build_distribution_tar_results.status is StepStatus.FAILURE: - return step_results - - build_connector_image_results = await BuildConnectorImage(context, LOCAL_BUILD_PLATFORM).run( - build_distribution_tar_results.output_artifact - ) - step_results.append(build_connector_image_results) - if build_connector_image_results.status is StepStatus.FAILURE: - return step_results - - if context.connector.supports_normalization: - normalization_image = f"{context.connector.normalization_repository}:dev" - context.logger.info(f"This connector supports normalization: will build {normalization_image}.") - build_normalization_results = await BuildOrPullNormalization(context, normalization_image, LOCAL_BUILD_PLATFORM).run() - normalization_container = build_normalization_results.output_artifact - normalization_tar_file, _ = await export_container_to_tarball( - context, normalization_container, tar_file_name=f"{context.connector.normalization_repository}_{context.git_revision}.tar" - ) - step_results.append(build_normalization_results) - else: - normalization_tar_file = None - - connector_image_tar_file, _ = await export_container_to_tarball(context, build_connector_image_results.output_artifact) - - integration_tests_results = await IntegrationTests(context).run(connector_image_tar_file, normalization_tar_file) - step_results.append(integration_tests_results) - - acceptance_tests_results = await AcceptanceTests(context).run(connector_image_tar_file) - step_results.append(acceptance_tests_results) - return step_results diff --git a/airbyte-ci/connectors/pipelines/pipelines/tests/python_connectors.py b/airbyte-ci/connectors/pipelines/pipelines/tests/python_connectors.py deleted file mode 100644 index 8470fc6ff84b..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/tests/python_connectors.py +++ /dev/null @@ -1,161 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -"""This module groups steps made to run tests for a specific Python connector given a test context.""" - -from datetime import timedelta -from typing import List - -import asyncer -from dagger import Container -from pipelines.actions import environments, secrets -from pipelines.bases import Step, StepResult, StepStatus -from pipelines.builds import LOCAL_BUILD_PLATFORM -from pipelines.builds.python_connectors import BuildConnectorImage -from pipelines.contexts import ConnectorContext -from pipelines.helpers.steps import run_steps -from pipelines.tests.common import AcceptanceTests, PytestStep -from pipelines.utils import export_container_to_tarball - - -class CodeFormatChecks(Step): - """A step to run the code format checks on a Python connector using Black, Isort and Flake.""" - - title = "Code format checks" - - RUN_BLACK_CMD = ["python", "-m", "black", f"--config=/{environments.PYPROJECT_TOML_FILE_PATH}", "--check", "."] - RUN_ISORT_CMD = ["python", "-m", "isort", f"--settings-file=/{environments.PYPROJECT_TOML_FILE_PATH}", "--check-only", "--diff", "."] - RUN_FLAKE_CMD = ["python", "-m", "pflake8", f"--config=/{environments.PYPROJECT_TOML_FILE_PATH}", "."] - - async def _run(self) -> StepResult: - """Run a code format check on the container source code. - - We call black, isort and flake commands: - - Black formats the code: fails if the code is not formatted. - - Isort checks the import orders: fails if the import are not properly ordered. - - Flake enforces style-guides: fails if the style-guide is not followed. - - Args: - context (ConnectorContext): The current test context, providing a connector object, a dagger client and a repository directory. - step (Step): The step in which the code format checks are run. Defaults to Step.CODE_FORMAT_CHECKS - Returns: - StepResult: Failure or success of the code format checks with stdout and stderr. - """ - connector_under_test = environments.with_python_connector_source(self.context) - - formatter = ( - connector_under_test.with_exec(["echo", "Running black"]) - .with_exec(self.RUN_BLACK_CMD) - .with_exec(["echo", "Running Isort"]) - .with_exec(self.RUN_ISORT_CMD) - .with_exec(["echo", "Running Flake"]) - .with_exec(self.RUN_FLAKE_CMD) - ) - return await self.get_step_result(formatter) - - -class ConnectorPackageInstall(Step): - """A step to install the Python connector package in a container.""" - - title = "Connector package install" - max_duration = timedelta(minutes=20) - max_retries = 3 - - async def _run(self) -> StepResult: - """Install the connector under test package in a Python container. - - Returns: - StepResult: Failure or success of the package installation and the connector under test container (with the connector package installed). - """ - connector_under_test = await environments.with_python_connector_installed(self.context) - return await self.get_step_result(connector_under_test) - - -class UnitTests(PytestStep): - """A step to run the connector unit tests with Pytest.""" - - title = "Unit tests" - - async def _run(self, connector_under_test: Container) -> StepResult: - """Run all pytest tests declared in the unit_tests directory of the connector code. - - Args: - connector_under_test (Container): The connector under test container. - - Returns: - StepResult: Failure or success of the unit tests with stdout and stdout. - """ - connector_under_test_with_secrets = connector_under_test.with_(environments.mounted_connector_secrets(self.context)) - return await self._run_tests_in_directory(connector_under_test_with_secrets, "unit_tests") - - -class IntegrationTests(PytestStep): - """A step to run the connector integration tests with Pytest.""" - - title = "Integration tests" - - async def _run(self, connector_under_test: Container) -> StepResult: - """Run all pytest tests declared in the integration_tests directory of the connector code. - - Args: - connector_under_test (Container): The connector under test container. - - Returns: - StepResult: Failure or success of the integration tests with stdout and stdout. - """ - - connector_under_test = connector_under_test.with_(environments.bound_docker_host(self.context)).with_( - environments.mounted_connector_secrets(self.context) - ) - return await self._run_tests_in_directory(connector_under_test, "integration_tests") - - -async def run_all_tests(context: ConnectorContext) -> List[StepResult]: - """Run all tests for a Python connector. - - Args: - context (ConnectorContext): The current connector context. - - Returns: - List[StepResult]: The results of all the steps that ran or were skipped. - """ - - step_results = await run_steps( - [ - ConnectorPackageInstall(context), - BuildConnectorImage(context, LOCAL_BUILD_PLATFORM), - ] - ) - if any([step_result.status is StepStatus.FAILURE for step_result in step_results]): - return step_results - connector_package_install_results, build_connector_image_results = step_results[0], step_results[1] - connector_image_tar_file, _ = await export_container_to_tarball(context, build_connector_image_results.output_artifact) - connector_container = connector_package_install_results.output_artifact - - context.connector_secrets = await secrets.get_connector_secrets(context) - - unit_test_results = await UnitTests(context).run(connector_container) - - if unit_test_results.status is StepStatus.FAILURE: - return step_results + [unit_test_results] - step_results.append(unit_test_results) - async with asyncer.create_task_group() as task_group: - tasks = [ - task_group.soonify(IntegrationTests(context).run)(connector_container), - task_group.soonify(AcceptanceTests(context).run)(connector_image_tar_file), - ] - - return step_results + [task.value for task in tasks] - - -async def run_code_format_checks(context: ConnectorContext) -> List[StepResult]: - """Run the code format check steps for Python connectors. - - Args: - context (ConnectorContext): The current connector context. - - Returns: - List[StepResult]: Results of the code format checks. - """ - return [await CodeFormatChecks(context).run()] diff --git a/airbyte-ci/connectors/pipelines/pipelines/utils.py b/airbyte-ci/connectors/pipelines/pipelines/utils.py deleted file mode 100644 index d80fe8d744ed..000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/utils.py +++ /dev/null @@ -1,635 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -"""This module groups util function used in pipelines.""" -from __future__ import annotations - -import contextlib -import datetime -import json -import os -import re -import sys -import unicodedata -from glob import glob -from io import TextIOWrapper -from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, FrozenSet, List, Optional, Set, Tuple, Union - -import anyio -import asyncer -import click -import git -from connector_ops.utils import get_changed_connectors -from dagger import Client, Config, Connection, Container, DaggerError, ExecError, File, ImageLayerCompression, QueryError, Secret -from google.cloud import storage -from google.oauth2 import service_account -from more_itertools import chunked -from pipelines import consts, main_logger, sentry_utils -from pipelines.consts import GCS_PUBLIC_DOMAIN - -if TYPE_CHECKING: - from connector_ops.utils import Connector - from github import PullRequest - from pipelines.contexts import ConnectorContext - -DAGGER_CONFIG = Config(log_output=sys.stderr) -AIRBYTE_REPO_URL = "https://github.com/airbytehq/airbyte.git" -METADATA_FILE_NAME = "metadata.yaml" -METADATA_ICON_FILE_NAME = "icon.svg" -DIFF_FILTER = "MADRT" # Modified, Added, Deleted, Renamed, Type changed -IGNORED_FILE_EXTENSIONS = [".md"] -STATIC_REPORT_PREFIX = "airbyte-ci" - - -# This utils will probably be redundant once https://github.com/dagger/dagger/issues/3764 is implemented -async def check_path_in_workdir(container: Container, path: str) -> bool: - """Check if a local path is mounted to the working directory of a container. - - Args: - container (Container): The container on which we want the check the path existence. - path (str): Directory or file path we want to check the existence in the container working directory. - - Returns: - bool: Whether the path exists in the container working directory. - """ - workdir = (await container.with_exec(["pwd"]).stdout()).strip() - mounts = await container.mounts() - if workdir in mounts: - expected_file_path = Path(workdir[1:]) / path - return expected_file_path.is_file() or expected_file_path.is_dir() - else: - return False - - -def secret_host_variable(client: Client, name: str, default: str = ""): - """Add a host environment variable as a secret in a container. - - Example: - container.with_(secret_host_variable(client, "MY_SECRET")) - - Args: - client (Client): The dagger client. - name (str): The name of the environment variable. The same name will be - used in the container, for the secret name and for the host variable. - default (str): The default value to use if the host variable is not set. Defaults to "". - - Returns: - Callable[[Container], Container]: A function that can be used in a `Container.with_()` method. - """ - - def _secret_host_variable(container: Container): - return container.with_secret_variable(name, get_secret_host_variable(client, name, default)) - - return _secret_host_variable - - -def get_secret_host_variable(client: Client, name: str, default: str = "") -> Secret: - """Creates a dagger.Secret from a host environment variable. - - Args: - client (Client): The dagger client. - name (str): The name of the environment variable. The same name will be used for the secret. - default (str): The default value to use if the host variable is not set. Defaults to "". - - Returns: - Secret: A dagger secret. - """ - return client.set_secret(name, os.environ.get(name, default)) - - -# This utils will probably be redundant once https://github.com/dagger/dagger/issues/3764 is implemented -async def get_file_contents(container: Container, path: str) -> Optional[str]: - """Retrieve a container file contents. - - Args: - container (Container): The container hosting the file you want to read. - path (str): Path, in the container, to the file you want to read. - - Returns: - Optional[str]: The file content if the file exists in the container, None otherwise. - """ - try: - return await container.file(path).contents() - except QueryError as e: - if "no such file or directory" not in str(e): - # this error could come from a network issue - raise - return None - - -@contextlib.contextmanager -def catch_exec_error_group(): - try: - yield - except anyio.ExceptionGroup as eg: - for e in eg.exceptions: - if isinstance(e, ExecError): - raise e - raise - - -async def get_container_output(container: Container) -> Tuple[str, str]: - """Retrieve both stdout and stderr of a container, concurrently. - - Args: - container (Container): The container to execute. - - Returns: - Tuple[str, str]: The stdout and stderr of the container, respectively. - """ - with catch_exec_error_group(): - async with asyncer.create_task_group() as task_group: - soon_stdout = task_group.soonify(container.stdout)() - soon_stderr = task_group.soonify(container.stderr)() - return soon_stdout.value, soon_stderr.value - - -async def get_exec_result(container: Container) -> Tuple[int, str, str]: - """Retrieve the exit_code along with stdout and stderr of a container by handling the ExecError. - - Note: It is preferrable to not worry about the exit code value and just capture - ExecError to handle errors. This is offered as a convenience when the exit code - value is actually needed. - - If the container has a file at /exit_code, the exit code will be read from it. - See hacks.never_fail_exec for more details. - - Args: - container (Container): The container to execute. - - Returns: - Tuple[int, str, str]: The exit_code, stdout and stderr of the container, respectively. - """ - try: - exit_code = 0 - in_file_exit_code = await get_file_contents(container, "/exit_code") - if in_file_exit_code: - exit_code = int(in_file_exit_code) - return exit_code, *(await get_container_output(container)) - except ExecError as e: - return e.exit_code, e.stdout, e.stderr - - -async def with_exit_code(container: Container) -> int: - """Read the container exit code. - - Args: - container (Container): The container from which you want to read the exit code. - - Returns: - int: The exit code. - """ - try: - await container - except ExecError as e: - return e.exit_code - return 0 - - -async def with_stderr(container: Container) -> str: - """Retrieve the stderr of a container even on execution error.""" - try: - return await container.stderr() - except ExecError as e: - return e.stderr - - -async def with_stdout(container: Container) -> str: - """Retrieve the stdout of a container even on execution error.""" - try: - return await container.stdout() - except ExecError as e: - return e.stdout - - -def get_current_git_branch() -> str: # noqa D103 - return git.Repo().active_branch.name - - -def get_current_git_revision() -> str: # noqa D103 - return git.Repo().head.object.hexsha - - -def get_current_epoch_time() -> int: # noqa D103 - return round(datetime.datetime.utcnow().timestamp()) - - -async def get_modified_files_in_branch_remote( - current_git_branch: str, current_git_revision: str, diffed_branch: str = "origin/master" -) -> Set[str]: - """Use git diff to spot the modified files on the remote branch.""" - async with Connection(DAGGER_CONFIG) as dagger_client: - modified_files = await ( - dagger_client.container() - .from_("alpine/git:latest") - .with_workdir("/repo") - .with_exec(["init"]) - .with_env_variable("CACHEBUSTER", current_git_revision) - .with_exec( - [ - "remote", - "add", - "--fetch", - "--track", - diffed_branch.split("/")[-1], - "--track", - current_git_branch, - "origin", - AIRBYTE_REPO_URL, - ] - ) - .with_exec(["checkout", "-t", f"origin/{current_git_branch}"]) - .with_exec(["diff", f"--diff-filter={DIFF_FILTER}", "--name-only", f"{diffed_branch}...{current_git_revision}"]) - .stdout() - ) - return set(modified_files.split("\n")) - - -def get_modified_files_in_branch_local(current_git_revision: str, diffed_branch: str = "master") -> Set[str]: - """Use git diff and git status to spot the modified files on the local branch.""" - airbyte_repo = git.Repo() - modified_files = airbyte_repo.git.diff( - f"--diff-filter={DIFF_FILTER}", "--name-only", f"{diffed_branch}...{current_git_revision}" - ).split("\n") - status_output = airbyte_repo.git.status("--porcelain") - for not_committed_change in status_output.split("\n"): - file_path = not_committed_change.strip().split(" ")[-1] - if file_path: - modified_files.append(file_path) - return set(modified_files) - - -def get_modified_files_in_branch(current_git_branch: str, current_git_revision: str, diffed_branch: str, is_local: bool = True) -> Set[str]: - """Retrieve the list of modified files on the branch.""" - if is_local: - return get_modified_files_in_branch_local(current_git_revision, diffed_branch) - else: - return anyio.run(get_modified_files_in_branch_remote, current_git_branch, current_git_revision, diffed_branch) - - -async def get_modified_files_in_commit_remote(current_git_branch: str, current_git_revision: str) -> Set[str]: - async with Connection(DAGGER_CONFIG) as dagger_client: - modified_files = await ( - dagger_client.container() - .from_("alpine/git:latest") - .with_workdir("/repo") - .with_exec(["init"]) - .with_env_variable("CACHEBUSTER", current_git_revision) - .with_exec( - [ - "remote", - "add", - "--fetch", - "--track", - current_git_branch, - "origin", - AIRBYTE_REPO_URL, - ] - ) - .with_exec(["checkout", "-t", f"origin/{current_git_branch}"]) - .with_exec(["diff-tree", "--no-commit-id", "--name-only", current_git_revision, "-r"]) - .stdout() - ) - return set(modified_files.split("\n")) - - -def get_modified_files_in_commit_local(current_git_revision: str) -> Set[str]: - airbyte_repo = git.Repo() - modified_files = airbyte_repo.git.diff_tree("--no-commit-id", "--name-only", current_git_revision, "-r").split("\n") - return set(modified_files) - - -def get_modified_files_in_commit(current_git_branch: str, current_git_revision: str, is_local: bool = True) -> Set[str]: - if is_local: - return get_modified_files_in_commit_local(current_git_revision) - else: - return anyio.run(get_modified_files_in_commit_remote, current_git_branch, current_git_revision) - - -def get_modified_files_in_pull_request(pull_request: PullRequest) -> List[str]: - """Retrieve the list of modified files in a pull request.""" - return [f.filename for f in pull_request.get_files()] - - -def get_last_commit_message() -> str: - """Retrieve the last commit message.""" - return git.Repo().head.commit.message - - -def _is_ignored_file(file_path: Union[str, Path]) -> bool: - """Check if the provided file has an ignored extension.""" - return Path(file_path).suffix in IGNORED_FILE_EXTENSIONS - - -def _find_modified_connectors( - file_path: Union[str, Path], all_connectors: Set[Connector], dependency_scanning: bool = True -) -> Set[Connector]: - """Find all connectors impacted by the file change.""" - modified_connectors = set() - - for connector in all_connectors: - if Path(file_path).is_relative_to(Path(connector.code_directory)): - main_logger.info(f"Adding connector '{connector}' due to connector file modification: {file_path}.") - modified_connectors.add(connector) - - if dependency_scanning: - for connector_dependency in connector.get_local_dependency_paths(): - if Path(file_path).is_relative_to(Path(connector_dependency)): - # Add the connector to the modified connectors - modified_connectors.add(connector) - main_logger.info(f"Adding connector '{connector}' due to dependency modification: '{file_path}'.") - return modified_connectors - - -def get_modified_connectors(modified_files: Set[Path], all_connectors: Set[Connector], dependency_scanning: bool) -> Set[Connector]: - """Create a mapping of modified connectors (key) and modified files (value). - If dependency scanning is enabled any modification to a dependency will trigger connector pipeline for all connectors that depend on it. - It currently works only for Java connectors . - It's especially useful to trigger tests of strict-encrypt variant when a change is made to the base connector. - Or to tests all jdbc connectors when a change is made to source-jdbc or base-java. - We'll consider extending the dependency resolution to Python connectors once we confirm that it's needed and feasible in term of scale. - """ - # Ignore files with certain extensions - modified_connectors = set() - for modified_file in modified_files: - if not _is_ignored_file(modified_file): - modified_connectors.update(_find_modified_connectors(modified_file, all_connectors, dependency_scanning)) - return modified_connectors - - -def get_connector_modified_files(connector: Connector, all_modified_files: Set[Path]) -> FrozenSet[Path]: - connector_modified_files = set() - for modified_file in all_modified_files: - modified_file_path = Path(modified_file) - if modified_file_path.is_relative_to(connector.code_directory): - connector_modified_files.add(modified_file) - return frozenset(connector_modified_files) - - -def get_modified_metadata_files(modified_files: Set[Union[str, Path]]) -> Set[Path]: - return { - Path(str(f)) - for f in modified_files - if str(f).endswith(METADATA_FILE_NAME) and str(f).startswith("airbyte-integrations/connectors") and "-scaffold-" not in str(f) - } - - -def get_expected_metadata_files(modified_files: Set[Union[str, Path]]) -> Set[Path]: - changed_connectors = get_changed_connectors(modified_files=modified_files) - return {changed_connector.metadata_file_path for changed_connector in changed_connectors} - - -def get_all_metadata_files() -> Set[Path]: - return { - Path(metadata_file) - for metadata_file in glob("airbyte-integrations/connectors/**/metadata.yaml", recursive=True) - if "-scaffold-" not in metadata_file - } - - -def slugify(value: Any, allow_unicode: bool = False): - """ - Taken from https://github.com/django/django/blob/master/django/utils/text.py. - - Convert to ASCII if 'allow_unicode' is False. Convert spaces or repeated - dashes to single dashes. Remove characters that aren't alphanumerics, - underscores, or hyphens. Convert to lowercase. Also strip leading and - trailing whitespace, dashes, and underscores. - """ - value = str(value) - if allow_unicode: - value = unicodedata.normalize("NFKC", value) - else: - value = unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode("ascii") - value = re.sub(r"[^\w\s-]", "", value.lower()) - return re.sub(r"[-\s]+", "-", value).strip("-_") - - -def key_value_text_to_dict(text: str) -> dict: - kv = {} - for line in text.split("\n"): - if "=" in line: - try: - k, v = line.split("=") - except ValueError: - continue - kv[k] = v - return kv - - -async def key_value_file_to_dict(file: File) -> dict: - return key_value_text_to_dict(await file.contents()) - - -async def get_dockerfile_labels(dockerfile: File) -> dict: - return {k.replace("LABEL ", ""): v for k, v in (await key_value_file_to_dict(dockerfile)).items() if k.startswith("LABEL")} - - -async def get_version_from_dockerfile(dockerfile: File) -> str: - dockerfile_labels = await get_dockerfile_labels(dockerfile) - try: - return dockerfile_labels["io.airbyte.version"] - except KeyError: - raise Exception("Could not get the version from the Dockerfile labels.") - - -def create_and_open_file(file_path: Path) -> TextIOWrapper: - """Create a file and open it for writing. - - Args: - file_path (Path): The path to the file to create. - - Returns: - File: The file object. - """ - file_path.parent.mkdir(parents=True, exist_ok=True) - file_path.touch() - return file_path.open("w") - - -class DaggerPipelineCommand(click.Command): - @sentry_utils.with_command_context - def invoke(self, ctx: click.Context) -> Any: - """Wrap parent invoke in a try catch suited to handle pipeline failures. - Args: - ctx (click.Context): The invocation context. - Raises: - e: Raise whatever exception that was caught. - Returns: - Any: The invocation return value. - """ - command_name = self.name - main_logger.info(f"Running Dagger Command {command_name}...") - main_logger.info( - "If you're running this command for the first time the Dagger engine image will be pulled, it can take a short minute..." - ) - ctx.obj["report_output_prefix"] = self.render_report_output_prefix(ctx) - dagger_logs_gcs_key = f"{ctx.obj['report_output_prefix']}/dagger-logs.txt" - try: - if not ctx.obj["show_dagger_logs"]: - dagger_log_dir = Path(f"{consts.LOCAL_REPORTS_PATH_ROOT}/{ctx.obj['report_output_prefix']}") - dagger_log_path = Path(f"{dagger_log_dir}/dagger.log").resolve() - ctx.obj["dagger_logs_path"] = dagger_log_path - main_logger.info(f"Saving dagger logs to: {dagger_log_path}") - if ctx.obj["is_ci"]: - ctx.obj["dagger_logs_url"] = f"{GCS_PUBLIC_DOMAIN}/{ctx.obj['ci_report_bucket_name']}/{dagger_logs_gcs_key}" - else: - ctx.obj["dagger_logs_url"] = None - else: - ctx.obj["dagger_logs_path"] = None - pipeline_success = super().invoke(ctx) - if not pipeline_success: - raise DaggerError(f"Dagger Command {command_name} failed.") - except DaggerError as e: - main_logger.error(f"Dagger Command {command_name} failed", exc_info=e) - sys.exit(1) - finally: - if ctx.obj.get("dagger_logs_path"): - if ctx.obj["is_local"]: - main_logger.info(f"Dagger logs saved to {ctx.obj['dagger_logs_path']}") - if ctx.obj["is_ci"]: - gcs_uri, public_url = upload_to_gcs( - ctx.obj["dagger_logs_path"], ctx.obj["ci_report_bucket_name"], dagger_logs_gcs_key, ctx.obj["ci_gcs_credentials"] - ) - main_logger.info(f"Dagger logs saved to {gcs_uri}. Public URL: {public_url}") - - @staticmethod - def render_report_output_prefix(ctx: click.Context) -> str: - """Render the report output prefix for any command in the Connector CLI. - - The goal is to standardize the output of all logs and reports generated by the CLI - related to a specific command, and to a specific CI context. - - Note: We cannot hoist this higher in the command hierarchy because only one level of - subcommands are available at the time the context is created. - """ - - git_branch = ctx.obj["git_branch"] - git_revision = ctx.obj["git_revision"] - pipeline_start_timestamp = ctx.obj["pipeline_start_timestamp"] - ci_context = ctx.obj["ci_context"] - ci_job_key = ctx.obj["ci_job_key"] if ctx.obj.get("ci_job_key") else ci_context - - sanitized_branch = slugify(git_branch.replace("/", "_")) - - # get the command name for the current context, if a group then prepend the parent command name - if ctx.command_path: - cmd_components = ctx.command_path.split(" ") - cmd_components[0] = STATIC_REPORT_PREFIX - cmd = "/".join(cmd_components) - else: - cmd = None - - path_values = [ - cmd, - ci_job_key, - sanitized_branch, - pipeline_start_timestamp, - git_revision, - ] - - # check all values are defined - if None in path_values: - raise ValueError(f"Missing value required to render the report output prefix: {path_values}") - - # join all values with a slash, and convert all values to string - return "/".join(map(str, path_values)) - - -async def execute_concurrently(steps: List[Callable], concurrency=5): - tasks = [] - # Asyncer does not have builtin semaphore, so control concurrency via chunks of steps - # Anyio has semaphores but does not have the soonify method which allow access to results via the value task attribute. - for chunk in chunked(steps, concurrency): - async with asyncer.create_task_group() as task_group: - tasks += [task_group.soonify(step)() for step in chunk] - return [task.value for task in tasks] - - -async def export_container_to_tarball( - context: ConnectorContext, container: Container, tar_file_name: Optional[str] = None -) -> Tuple[Optional[File], Optional[Path]]: - """Save the container image to the host filesystem as a tar archive. - - Exporting a container image as a tar archive allows user to have a dagger built container image available on their host filesystem. - They can load this tar file to their main docker host with 'docker load'. - This mechanism is also used to share dagger built containers with other steps like AcceptanceTest that have their own dockerd service. - We 'docker load' this tar file to AcceptanceTest's docker host to make sure the container under test image is available for testing. - - Returns: - Tuple[Optional[File], Optional[Path]]: A tuple with the file object holding the tar archive on the host and its path. - """ - if tar_file_name is None: - tar_file_name = f"{context.connector.technical_name}_{context.git_revision}.tar" - tar_file_name = slugify(tar_file_name) - local_path = Path(f"{context.host_image_export_dir_path}/{tar_file_name}") - export_success = await container.export(str(local_path), forced_compression=ImageLayerCompression.Gzip) - if export_success: - exported_file = ( - context.dagger_client.host().directory(context.host_image_export_dir_path, include=[tar_file_name]).file(tar_file_name) - ) - return exported_file, local_path - else: - return None, None - - -def sanitize_gcs_credentials(raw_value: Optional[str]) -> Optional[str]: - """Try to parse the raw string input that should contain a json object with the GCS credentials. - It will raise an exception if the parsing fails and help us to fail fast on invalid credentials input. - - Args: - raw_value (str): A string representing a json object with the GCS credentials. - - Returns: - str: The raw value string if it was successfully parsed. - """ - if raw_value is None: - return None - return json.dumps(json.loads(raw_value)) - - -def format_duration(time_delta: datetime.timedelta) -> str: - total_seconds = time_delta.total_seconds() - if total_seconds < 60: - return "{:.2f}s".format(total_seconds) - minutes = int(total_seconds // 60) - seconds = int(total_seconds % 60) - return "{:02d}mn{:02d}s".format(minutes, seconds) - - -def upload_to_gcs(file_path: Path, bucket_name: str, object_name: str, credentials: str) -> Tuple[str, str]: - """Upload a file to a GCS bucket. - - Args: - file_path (Path): The path to the file to upload. - bucket_name (str): The name of the GCS bucket. - object_name (str): The name of the object in the GCS bucket. - credentials (str): The GCS credentials as a JSON string. - """ - # Exit early if file does not exist - if not file_path.exists(): - main_logger.warning(f"File {file_path} does not exist. Skipping upload to GCS.") - return "", "" - - credentials = service_account.Credentials.from_service_account_info(json.loads(credentials)) - client = storage.Client(credentials=credentials) - bucket = client.get_bucket(bucket_name) - blob = bucket.blob(object_name) - blob.upload_from_filename(str(file_path)) - gcs_uri = f"gs://{bucket_name}/{object_name}" - public_url = f"{GCS_PUBLIC_DOMAIN}/{bucket_name}/{object_name}" - return gcs_uri, public_url - - -def transform_strs_to_paths(str_paths: List[str]) -> List[Path]: - """Transform a list of string paths to a list of Path objects. - - Args: - str_paths (List[str]): A list of string paths. - - Returns: - List[Path]: A list of Path objects. - """ - return [Path(str_path) for str_path in str_paths] diff --git a/airbyte-ci/connectors/pipelines/poetry.lock b/airbyte-ci/connectors/pipelines/poetry.lock index f48430f798f6..e97b0e920b72 100644 --- a/airbyte-ci/connectors/pipelines/poetry.lock +++ b/airbyte-ci/connectors/pipelines/poetry.lock @@ -1,18 +1,62 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] -name = "airbyte-protocol-models" +name = "airbyte-connectors-base-images" version = "1.0.1" +description = "This package is used to generate and publish the base images for Airbyte Connectors." +optional = false +python-versions = "^3.10" +files = [] +develop = true + +[package.dependencies] +connector-ops = {path = "../connector_ops", develop = true} +dagger-io = "==0.9.6" +gitpython = "^3.1.35" +inquirer = "^3.1.3" +jinja2 = "^3.1.2" +rich = "^13.5.2" +semver = "^3.0.1" + +[package.source] +type = "directory" +url = "../base_images" + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.3" description = "Declares the Airbyte Protocol." optional = false -python-versions = ">=3.9" +python-versions = ">=3.8" files = [ - {file = "airbyte_protocol_models-1.0.1-py3-none-any.whl", hash = "sha256:2c214fb8cb42b74aa6408beeea2cd52f094bc8a3ba0e78af20bb358e5404f4a8"}, - {file = "airbyte_protocol_models-1.0.1.tar.gz", hash = "sha256:caa860d15c9c9073df4b221f58280b9855d36de07519e010d1e610546458d0a7"}, + {file = "airbyte_protocol_models-0.5.3-py3-none-any.whl", hash = "sha256:a913f1e86d5b2ae17d19e0135339e55fc25bb93bfc3f7ab38592677f29b56c57"}, + {file = "airbyte_protocol_models-0.5.3.tar.gz", hash = "sha256:a71bc0e98e0722d5cbd3122c40a59a7f9cbc91b6c934db7e768a57c40546f54b"}, ] [package.dependencies] -pydantic = ">=1.9.2,<1.10.0" +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "altgraph" +version = "0.17.4" +description = "Python graph (network) package" +optional = false +python-versions = "*" +files = [ + {file = "altgraph-0.17.4-py2.py3-none-any.whl", hash = "sha256:642743b4750de17e655e6711601b077bc6598dbfa3ba5fa2b2a35ce12b508dff"}, + {file = "altgraph-0.17.4.tar.gz", hash = "sha256:1b5afbb98f6c4dcadb2e2ae6ab9fa994bbb8c1d75f4fa96d340f9437ae454406"}, +] + +[[package]] +name = "ansicon" +version = "1.89.0" +description = "Python wrapper for loading Jason Hood's ANSICON" +optional = false +python-versions = "*" +files = [ + {file = "ansicon-1.89.0-py2.py3-none-any.whl", hash = "sha256:f1def52d17f65c2c9682cf8370c03f541f410c1752d6a14029f97318e4b9dfec"}, + {file = "ansicon-1.89.0.tar.gz", hash = "sha256:e4d039def5768a47e4afec8e89e83ec3ae5a26bf00ad851f914d1240b444d2b1"}, +] [[package]] name = "anyio" @@ -35,6 +79,21 @@ doc = ["Sphinx", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd- test = ["anyio[trio]", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] trio = ["trio (<0.22)"] +[[package]] +name = "asyncclick" +version = "8.1.7.1" +description = "Composable command line interface toolkit, async version" +optional = false +python-versions = ">=3.7" +files = [ + {file = "asyncclick-8.1.7.1-py3-none-any.whl", hash = "sha256:e0fea5f0223ac45cfc26153cc80a58cc65fc077ac8de79be49248c918e8c3422"}, + {file = "asyncclick-8.1.7.1.tar.gz", hash = "sha256:a47b61258a689212cf9463fbf3b4cc52d05bfd03185f6ead2315fc03fd17ef75"}, +] + +[package.dependencies] +anyio = "*" +colorama = {version = "*", markers = "platform_system == \"Windows\""} + [[package]] name = "asyncer" version = "0.0.2" @@ -61,21 +120,22 @@ files = [ [[package]] name = "attrs" -version = "23.1.0" +version = "23.2.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.7" files = [ - {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"}, - {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"}, + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, ] [package.extras] cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] -dev = ["attrs[docs,tests]", "pre-commit"] +dev = ["attrs[tests]", "pre-commit"] docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] tests = ["attrs[tests-no-zope]", "zope-interface"] -tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] [[package]] name = "backoff" @@ -90,13 +150,13 @@ files = [ [[package]] name = "beartype" -version = "0.15.0" +version = "0.16.4" description = "Unbearably fast runtime type checking in pure Python." optional = false python-versions = ">=3.8.0" files = [ - {file = "beartype-0.15.0-py3-none-any.whl", hash = "sha256:52cd2edea72fdd84e4e7f8011a9e3007bf0125c3d6d7219e937b9d8868169177"}, - {file = "beartype-0.15.0.tar.gz", hash = "sha256:2af6a8d8a7267ccf7d271e1a3bd908afbc025d2a09aa51123567d7d7b37438df"}, + {file = "beartype-0.16.4-py3-none-any.whl", hash = "sha256:64865952f9dff1e17f22684b3c7286fc79754553b47eaefeb1286224ae8c1bd9"}, + {file = "beartype-0.16.4.tar.gz", hash = "sha256:1ada89cf2d6eb30eb6e156eed2eb5493357782937910d74380918e53c2eae0bf"}, ] [package.extras] @@ -106,124 +166,128 @@ doc-rtd = ["autoapi (>=0.9.0)", "pydata-sphinx-theme (<=0.7.2)", "sphinx (>=4.2. test-tox = ["mypy (>=0.800)", "numpy", "pandera", "pytest (>=4.0.0)", "sphinx", "typing-extensions (>=3.10.0.0)"] test-tox-coverage = ["coverage (>=5.5)"] +[[package]] +name = "blessed" +version = "1.20.0" +description = "Easy, practical library for making terminal apps, by providing an elegant, well-documented interface to Colors, Keyboard input, and screen Positioning capabilities." +optional = false +python-versions = ">=2.7" +files = [ + {file = "blessed-1.20.0-py2.py3-none-any.whl", hash = "sha256:0c542922586a265e699188e52d5f5ac5ec0dd517e5a1041d90d2bbf23f906058"}, + {file = "blessed-1.20.0.tar.gz", hash = "sha256:2cdd67f8746e048f00df47a2880f4d6acbcdb399031b604e34ba8f71d5787680"}, +] + +[package.dependencies] +jinxed = {version = ">=1.1.0", markers = "platform_system == \"Windows\""} +six = ">=1.9.0" +wcwidth = ">=0.1.4" + [[package]] name = "cachetools" -version = "5.3.1" +version = "5.3.2" description = "Extensible memoizing collections and decorators" optional = false python-versions = ">=3.7" files = [ - {file = "cachetools-5.3.1-py3-none-any.whl", hash = "sha256:95ef631eeaea14ba2e36f06437f36463aac3a096799e876ee55e5cdccb102590"}, - {file = "cachetools-5.3.1.tar.gz", hash = "sha256:dce83f2d9b4e1f732a8cd44af8e8fab2dbe46201467fc98b3ef8f269092bf62b"}, + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, ] [[package]] name = "cattrs" -version = "23.1.2" +version = "23.2.3" description = "Composable complex class support for attrs and dataclasses." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "cattrs-23.1.2-py3-none-any.whl", hash = "sha256:b2bb14311ac17bed0d58785e5a60f022e5431aca3932e3fc5cc8ed8639de50a4"}, - {file = "cattrs-23.1.2.tar.gz", hash = "sha256:db1c821b8c537382b2c7c66678c3790091ca0275ac486c76f3c8f3920e83c657"}, + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, ] [package.dependencies] -attrs = ">=20" -exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} -typing_extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} [package.extras] -bson = ["pymongo (>=4.2.0,<5.0.0)"] -cbor2 = ["cbor2 (>=5.4.6,<6.0.0)"] -msgpack = ["msgpack (>=1.0.2,<2.0.0)"] -orjson = ["orjson (>=3.5.2,<4.0.0)"] -pyyaml = ["PyYAML (>=6.0,<7.0)"] -tomlkit = ["tomlkit (>=0.11.4,<0.12.0)"] -ujson = ["ujson (>=5.4.0,<6.0.0)"] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] [[package]] name = "certifi" -version = "2023.7.22" +version = "2023.11.17" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"}, - {file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"}, + {file = "certifi-2023.11.17-py3-none-any.whl", hash = "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"}, + {file = "certifi-2023.11.17.tar.gz", hash = "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1"}, ] [[package]] name = "cffi" -version = "1.15.1" +version = "1.16.0" description = "Foreign Function Interface for Python calling C code." optional = false -python-versions = "*" -files = [ - {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, - {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, - {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, - {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, - {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, - {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, - {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, - {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, - {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, - {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, - {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, - {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, - {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, - {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, - {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, - {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, - {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, - {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, - {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, - {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, - {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, - {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, - {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, - {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, - {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, - {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, - {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, - {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, - {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, - {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, - {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, - {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, - {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, ] [package.dependencies] @@ -231,86 +295,101 @@ pycparser = "*" [[package]] name = "charset-normalizer" -version = "3.2.0" +version = "3.3.2" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7.0" files = [ - {file = "charset-normalizer-3.2.0.tar.gz", hash = "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-win32.whl", hash = "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-win32.whl", hash = "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-win32.whl", hash = "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-win32.whl", hash = "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-win32.whl", hash = "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80"}, - {file = "charset_normalizer-3.2.0-py3-none-any.whl", hash = "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6"}, + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, ] [[package]] @@ -334,13 +413,13 @@ url = "../ci_credentials" [[package]] name = "click" -version = "8.1.6" +version = "8.1.7" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" files = [ - {file = "click-8.1.6-py3-none-any.whl", hash = "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"}, - {file = "click-8.1.6.tar.gz", hash = "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd"}, + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, ] [package.dependencies] @@ -375,40 +454,28 @@ requests = "^2.28.2" type = "directory" url = "../common_utils" -[[package]] -name = "commonmark" -version = "0.9.1" -description = "Python parser for the CommonMark Markdown spec" -optional = false -python-versions = "*" -files = [ - {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, - {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, -] - -[package.extras] -test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] - [[package]] name = "connector-ops" -version = "0.2.2" +version = "0.3.3" description = "Packaged maintained by the connector operations team to perform CI for connectors" optional = false python-versions = "^3.10" files = [] -develop = false +develop = true [package.dependencies] ci-credentials = {path = "../ci_credentials"} click = "^8.1.3" GitPython = "^3.1.29" google-cloud-storage = "^2.8.0" +pandas = "^2.0.3" pydantic = "^1.9" pydash = "^7.0.4" PyGithub = "^1.58.0" PyYAML = "^6.0" requests = "^2.28.2" -rich = "^11.0.1" +rich = "^13.0.0" +simpleeval = "^0.9.13" [package.source] type = "directory" @@ -416,71 +483,63 @@ url = "../connector_ops" [[package]] name = "coverage" -version = "7.2.7" +version = "7.4.0" description = "Code coverage measurement for Python" optional = false -python-versions = ">=3.7" -files = [ - {file = "coverage-7.2.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d39b5b4f2a66ccae8b7263ac3c8170994b65266797fb96cbbfd3fb5b23921db8"}, - {file = "coverage-7.2.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6d040ef7c9859bb11dfeb056ff5b3872436e3b5e401817d87a31e1750b9ae2fb"}, - {file = "coverage-7.2.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba90a9563ba44a72fda2e85302c3abc71c5589cea608ca16c22b9804262aaeb6"}, - {file = "coverage-7.2.7-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7d9405291c6928619403db1d10bd07888888ec1abcbd9748fdaa971d7d661b2"}, - {file = "coverage-7.2.7-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31563e97dae5598556600466ad9beea39fb04e0229e61c12eaa206e0aa202063"}, - {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ebba1cd308ef115925421d3e6a586e655ca5a77b5bf41e02eb0e4562a111f2d1"}, - {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cb017fd1b2603ef59e374ba2063f593abe0fc45f2ad9abdde5b4d83bd922a353"}, - {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62a5c7dad11015c66fbb9d881bc4caa5b12f16292f857842d9d1871595f4495"}, - {file = "coverage-7.2.7-cp310-cp310-win32.whl", hash = "sha256:ee57190f24fba796e36bb6d3aa8a8783c643d8fa9760c89f7a98ab5455fbf818"}, - {file = "coverage-7.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:f75f7168ab25dd93110c8a8117a22450c19976afbc44234cbf71481094c1b850"}, - {file = "coverage-7.2.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:06a9a2be0b5b576c3f18f1a241f0473575c4a26021b52b2a85263a00f034d51f"}, - {file = "coverage-7.2.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5baa06420f837184130752b7c5ea0808762083bf3487b5038d68b012e5937dbe"}, - {file = "coverage-7.2.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdec9e8cbf13a5bf63290fc6013d216a4c7232efb51548594ca3631a7f13c3a3"}, - {file = "coverage-7.2.7-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:52edc1a60c0d34afa421c9c37078817b2e67a392cab17d97283b64c5833f427f"}, - {file = "coverage-7.2.7-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63426706118b7f5cf6bb6c895dc215d8a418d5952544042c8a2d9fe87fcf09cb"}, - {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:afb17f84d56068a7c29f5fa37bfd38d5aba69e3304af08ee94da8ed5b0865833"}, - {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:48c19d2159d433ccc99e729ceae7d5293fbffa0bdb94952d3579983d1c8c9d97"}, - {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0e1f928eaf5469c11e886fe0885ad2bf1ec606434e79842a879277895a50942a"}, - {file = "coverage-7.2.7-cp311-cp311-win32.whl", hash = "sha256:33d6d3ea29d5b3a1a632b3c4e4f4ecae24ef170b0b9ee493883f2df10039959a"}, - {file = "coverage-7.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:5b7540161790b2f28143191f5f8ec02fb132660ff175b7747b95dcb77ac26562"}, - {file = "coverage-7.2.7-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f2f67fe12b22cd130d34d0ef79206061bfb5eda52feb6ce0dba0644e20a03cf4"}, - {file = "coverage-7.2.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a342242fe22407f3c17f4b499276a02b01e80f861f1682ad1d95b04018e0c0d4"}, - {file = "coverage-7.2.7-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:171717c7cb6b453aebac9a2ef603699da237f341b38eebfee9be75d27dc38e01"}, - {file = "coverage-7.2.7-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49969a9f7ffa086d973d91cec8d2e31080436ef0fb4a359cae927e742abfaaa6"}, - {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b46517c02ccd08092f4fa99f24c3b83d8f92f739b4657b0f146246a0ca6a831d"}, - {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:a3d33a6b3eae87ceaefa91ffdc130b5e8536182cd6dfdbfc1aa56b46ff8c86de"}, - {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:976b9c42fb2a43ebf304fa7d4a310e5f16cc99992f33eced91ef6f908bd8f33d"}, - {file = "coverage-7.2.7-cp312-cp312-win32.whl", hash = "sha256:8de8bb0e5ad103888d65abef8bca41ab93721647590a3f740100cd65c3b00511"}, - {file = "coverage-7.2.7-cp312-cp312-win_amd64.whl", hash = "sha256:9e31cb64d7de6b6f09702bb27c02d1904b3aebfca610c12772452c4e6c21a0d3"}, - {file = "coverage-7.2.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:58c2ccc2f00ecb51253cbe5d8d7122a34590fac9646a960d1430d5b15321d95f"}, - {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d22656368f0e6189e24722214ed8d66b8022db19d182927b9a248a2a8a2f67eb"}, - {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a895fcc7b15c3fc72beb43cdcbdf0ddb7d2ebc959edac9cef390b0d14f39f8a9"}, - {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e84606b74eb7de6ff581a7915e2dab7a28a0517fbe1c9239eb227e1354064dcd"}, - {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0a5f9e1dbd7fbe30196578ca36f3fba75376fb99888c395c5880b355e2875f8a"}, - {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:419bfd2caae268623dd469eff96d510a920c90928b60f2073d79f8fe2bbc5959"}, - {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2aee274c46590717f38ae5e4650988d1af340fe06167546cc32fe2f58ed05b02"}, - {file = "coverage-7.2.7-cp37-cp37m-win32.whl", hash = "sha256:61b9a528fb348373c433e8966535074b802c7a5d7f23c4f421e6c6e2f1697a6f"}, - {file = "coverage-7.2.7-cp37-cp37m-win_amd64.whl", hash = "sha256:b1c546aca0ca4d028901d825015dc8e4d56aac4b541877690eb76490f1dc8ed0"}, - {file = "coverage-7.2.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:54b896376ab563bd38453cecb813c295cf347cf5906e8b41d340b0321a5433e5"}, - {file = "coverage-7.2.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3d376df58cc111dc8e21e3b6e24606b5bb5dee6024f46a5abca99124b2229ef5"}, - {file = "coverage-7.2.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e330fc79bd7207e46c7d7fd2bb4af2963f5f635703925543a70b99574b0fea9"}, - {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e9d683426464e4a252bf70c3498756055016f99ddaec3774bf368e76bbe02b6"}, - {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d13c64ee2d33eccf7437961b6ea7ad8673e2be040b4f7fd4fd4d4d28d9ccb1e"}, - {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b7aa5f8a41217360e600da646004f878250a0d6738bcdc11a0a39928d7dc2050"}, - {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8fa03bce9bfbeeef9f3b160a8bed39a221d82308b4152b27d82d8daa7041fee5"}, - {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:245167dd26180ab4c91d5e1496a30be4cd721a5cf2abf52974f965f10f11419f"}, - {file = "coverage-7.2.7-cp38-cp38-win32.whl", hash = "sha256:d2c2db7fd82e9b72937969bceac4d6ca89660db0a0967614ce2481e81a0b771e"}, - {file = "coverage-7.2.7-cp38-cp38-win_amd64.whl", hash = "sha256:2e07b54284e381531c87f785f613b833569c14ecacdcb85d56b25c4622c16c3c"}, - {file = "coverage-7.2.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:537891ae8ce59ef63d0123f7ac9e2ae0fc8b72c7ccbe5296fec45fd68967b6c9"}, - {file = "coverage-7.2.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06fb182e69f33f6cd1d39a6c597294cff3143554b64b9825d1dc69d18cc2fff2"}, - {file = "coverage-7.2.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:201e7389591af40950a6480bd9edfa8ed04346ff80002cec1a66cac4549c1ad7"}, - {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f6951407391b639504e3b3be51b7ba5f3528adbf1a8ac3302b687ecababf929e"}, - {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f48351d66575f535669306aa7d6d6f71bc43372473b54a832222803eb956fd1"}, - {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b29019c76039dc3c0fd815c41392a044ce555d9bcdd38b0fb60fb4cd8e475ba9"}, - {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:81c13a1fc7468c40f13420732805a4c38a105d89848b7c10af65a90beff25250"}, - {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:975d70ab7e3c80a3fe86001d8751f6778905ec723f5b110aed1e450da9d4b7f2"}, - {file = "coverage-7.2.7-cp39-cp39-win32.whl", hash = "sha256:7ee7d9d4822c8acc74a5e26c50604dff824710bc8de424904c0982e25c39c6cb"}, - {file = "coverage-7.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:eb393e5ebc85245347950143969b241d08b52b88a3dc39479822e073a1a8eb27"}, - {file = "coverage-7.2.7-pp37.pp38.pp39-none-any.whl", hash = "sha256:b7b4c971f05e6ae490fef852c218b0e79d4e52f79ef0c8475566584a8fb3e01d"}, - {file = "coverage-7.2.7.tar.gz", hash = "sha256:924d94291ca674905fe9481f12294eb11f2d3d3fd1adb20314ba89e94f44ed59"}, +python-versions = ">=3.8" +files = [ + {file = "coverage-7.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:36b0ea8ab20d6a7564e89cb6135920bc9188fb5f1f7152e94e8300b7b189441a"}, + {file = "coverage-7.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0676cd0ba581e514b7f726495ea75aba3eb20899d824636c6f59b0ed2f88c471"}, + {file = "coverage-7.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0ca5c71a5a1765a0f8f88022c52b6b8be740e512980362f7fdbb03725a0d6b9"}, + {file = "coverage-7.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7c97726520f784239f6c62506bc70e48d01ae71e9da128259d61ca5e9788516"}, + {file = "coverage-7.4.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:815ac2d0f3398a14286dc2cea223a6f338109f9ecf39a71160cd1628786bc6f5"}, + {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:80b5ee39b7f0131ebec7968baa9b2309eddb35b8403d1869e08f024efd883566"}, + {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5b2ccb7548a0b65974860a78c9ffe1173cfb5877460e5a229238d985565574ae"}, + {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:995ea5c48c4ebfd898eacb098164b3cc826ba273b3049e4a889658548e321b43"}, + {file = "coverage-7.4.0-cp310-cp310-win32.whl", hash = "sha256:79287fd95585ed36e83182794a57a46aeae0b64ca53929d1176db56aacc83451"}, + {file = "coverage-7.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:5b14b4f8760006bfdb6e08667af7bc2d8d9bfdb648351915315ea17645347137"}, + {file = "coverage-7.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:04387a4a6ecb330c1878907ce0dc04078ea72a869263e53c72a1ba5bbdf380ca"}, + {file = "coverage-7.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ea81d8f9691bb53f4fb4db603203029643caffc82bf998ab5b59ca05560f4c06"}, + {file = "coverage-7.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74775198b702868ec2d058cb92720a3c5a9177296f75bd97317c787daf711505"}, + {file = "coverage-7.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76f03940f9973bfaee8cfba70ac991825611b9aac047e5c80d499a44079ec0bc"}, + {file = "coverage-7.4.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:485e9f897cf4856a65a57c7f6ea3dc0d4e6c076c87311d4bc003f82cfe199d25"}, + {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6ae8c9d301207e6856865867d762a4b6fd379c714fcc0607a84b92ee63feff70"}, + {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bf477c355274a72435ceb140dc42de0dc1e1e0bf6e97195be30487d8eaaf1a09"}, + {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:83c2dda2666fe32332f8e87481eed056c8b4d163fe18ecc690b02802d36a4d26"}, + {file = "coverage-7.4.0-cp311-cp311-win32.whl", hash = "sha256:697d1317e5290a313ef0d369650cfee1a114abb6021fa239ca12b4849ebbd614"}, + {file = "coverage-7.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:26776ff6c711d9d835557ee453082025d871e30b3fd6c27fcef14733f67f0590"}, + {file = "coverage-7.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:13eaf476ec3e883fe3e5fe3707caeb88268a06284484a3daf8250259ef1ba143"}, + {file = "coverage-7.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846f52f46e212affb5bcf131c952fb4075b55aae6b61adc9856222df89cbe3e2"}, + {file = "coverage-7.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26f66da8695719ccf90e794ed567a1549bb2644a706b41e9f6eae6816b398c4a"}, + {file = "coverage-7.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:164fdcc3246c69a6526a59b744b62e303039a81e42cfbbdc171c91a8cc2f9446"}, + {file = "coverage-7.4.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:316543f71025a6565677d84bc4df2114e9b6a615aa39fb165d697dba06a54af9"}, + {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bb1de682da0b824411e00a0d4da5a784ec6496b6850fdf8c865c1d68c0e318dd"}, + {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:0e8d06778e8fbffccfe96331a3946237f87b1e1d359d7fbe8b06b96c95a5407a"}, + {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a56de34db7b7ff77056a37aedded01b2b98b508227d2d0979d373a9b5d353daa"}, + {file = "coverage-7.4.0-cp312-cp312-win32.whl", hash = "sha256:51456e6fa099a8d9d91497202d9563a320513fcf59f33991b0661a4a6f2ad450"}, + {file = "coverage-7.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:cd3c1e4cb2ff0083758f09be0f77402e1bdf704adb7f89108007300a6da587d0"}, + {file = "coverage-7.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e9d1bf53c4c8de58d22e0e956a79a5b37f754ed1ffdbf1a260d9dcfa2d8a325e"}, + {file = "coverage-7.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:109f5985182b6b81fe33323ab4707011875198c41964f014579cf82cebf2bb85"}, + {file = "coverage-7.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cc9d4bc55de8003663ec94c2f215d12d42ceea128da8f0f4036235a119c88ac"}, + {file = "coverage-7.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc6d65b21c219ec2072c1293c505cf36e4e913a3f936d80028993dd73c7906b1"}, + {file = "coverage-7.4.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a10a4920def78bbfff4eff8a05c51be03e42f1c3735be42d851f199144897ba"}, + {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b8e99f06160602bc64da35158bb76c73522a4010f0649be44a4e167ff8555952"}, + {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7d360587e64d006402b7116623cebf9d48893329ef035278969fa3bbf75b697e"}, + {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:29f3abe810930311c0b5d1a7140f6395369c3db1be68345638c33eec07535105"}, + {file = "coverage-7.4.0-cp38-cp38-win32.whl", hash = "sha256:5040148f4ec43644702e7b16ca864c5314ccb8ee0751ef617d49aa0e2d6bf4f2"}, + {file = "coverage-7.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:9864463c1c2f9cb3b5db2cf1ff475eed2f0b4285c2aaf4d357b69959941aa555"}, + {file = "coverage-7.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:936d38794044b26c99d3dd004d8af0035ac535b92090f7f2bb5aa9c8e2f5cd42"}, + {file = "coverage-7.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:799c8f873794a08cdf216aa5d0531c6a3747793b70c53f70e98259720a6fe2d7"}, + {file = "coverage-7.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7defbb9737274023e2d7af02cac77043c86ce88a907c58f42b580a97d5bcca9"}, + {file = "coverage-7.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a1526d265743fb49363974b7aa8d5899ff64ee07df47dd8d3e37dcc0818f09ed"}, + {file = "coverage-7.4.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf635a52fc1ea401baf88843ae8708591aa4adff875e5c23220de43b1ccf575c"}, + {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:756ded44f47f330666843b5781be126ab57bb57c22adbb07d83f6b519783b870"}, + {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:0eb3c2f32dabe3a4aaf6441dde94f35687224dfd7eb2a7f47f3fd9428e421058"}, + {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bfd5db349d15c08311702611f3dccbef4b4e2ec148fcc636cf8739519b4a5c0f"}, + {file = "coverage-7.4.0-cp39-cp39-win32.whl", hash = "sha256:53d7d9158ee03956e0eadac38dfa1ec8068431ef8058fe6447043db1fb40d932"}, + {file = "coverage-7.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:cfd2a8b6b0d8e66e944d47cdec2f47c48fef2ba2f2dff5a9a75757f64172857e"}, + {file = "coverage-7.4.0-pp38.pp39.pp310-none-any.whl", hash = "sha256:c530833afc4707fe48524a44844493f36d8727f04dcce91fb978c414a8556cc6"}, + {file = "coverage-7.4.0.tar.gz", hash = "sha256:707c0f58cb1712b8809ece32b68996ee1e609f71bd14615bd8f87a1293cb610e"}, ] [package.dependencies] @@ -530,13 +589,13 @@ test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.0)" [[package]] name = "dagger-io" -version = "0.6.4" +version = "0.9.6" description = "A client package for running Dagger pipelines in Python." optional = false python-versions = ">=3.10" files = [ - {file = "dagger_io-0.6.4-py3-none-any.whl", hash = "sha256:b1bea624d1428a40228fffaa96407292cc3d18a7eca5bc036e6ceb9abd903d9a"}, - {file = "dagger_io-0.6.4.tar.gz", hash = "sha256:b754fd9820c41904e344377330ccca88f0a3409023eea8f0557db739b871e552"}, + {file = "dagger_io-0.9.6-py3-none-any.whl", hash = "sha256:e2f1e4bbc252071a314fa5b0bad11a910433a9ee043972b716f6fcc5f9fc8236"}, + {file = "dagger_io-0.9.6.tar.gz", hash = "sha256:147b5a33c44d17f602a4121679893655e91308beb8c46a466afed39cf40f789b"}, ] [package.dependencies] @@ -547,11 +606,8 @@ gql = ">=3.4.0" graphql-core = ">=3.2.3" httpx = ">=0.23.1" platformdirs = ">=2.6.2" -typing-extensions = ">=4.4.0" - -[package.extras] -cli = ["typer[all] (>=0.6.1)"] -server = ["strawberry-graphql (>=0.187.0)", "typer[all] (>=0.6.1)"] +rich = ">=10.11.0" +typing-extensions = ">=4.8.0" [[package]] name = "deprecated" @@ -572,33 +628,49 @@ dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] [[package]] name = "docker" -version = "5.0.3" +version = "6.1.3" description = "A Python library for the Docker Engine API." optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "docker-5.0.3-py2.py3-none-any.whl", hash = "sha256:7a79bb439e3df59d0a72621775d600bc8bc8b422d285824cb37103eab91d1ce0"}, - {file = "docker-5.0.3.tar.gz", hash = "sha256:d916a26b62970e7c2f554110ed6af04c7ccff8e9f81ad17d0d40c75637e227fb"}, + {file = "docker-6.1.3-py3-none-any.whl", hash = "sha256:aecd2277b8bf8e506e484f6ab7aec39abe0038e29fa4a6d3ba86c3fe01844ed9"}, + {file = "docker-6.1.3.tar.gz", hash = "sha256:aa6d17830045ba5ef0168d5eaa34d37beeb113948c413affe1d5991fc11f9a20"}, ] [package.dependencies] -pywin32 = {version = "227", markers = "sys_platform == \"win32\""} -requests = ">=2.14.2,<2.18.0 || >2.18.0" +packaging = ">=14.0" +pywin32 = {version = ">=304", markers = "sys_platform == \"win32\""} +requests = ">=2.26.0" +urllib3 = ">=1.26.0" websocket-client = ">=0.32.0" [package.extras] -ssh = ["paramiko (>=2.4.2)"] -tls = ["cryptography (>=3.4.7)", "idna (>=2.0.0)", "pyOpenSSL (>=17.5.0)"] +ssh = ["paramiko (>=2.4.3)"] + +[[package]] +name = "editor" +version = "1.6.5" +description = "🖋 Open the default text editor 🖋" +optional = false +python-versions = ">=3.8" +files = [ + {file = "editor-1.6.5-py3-none-any.whl", hash = "sha256:53c26dd78333b50b8cdcf67748956afa75fabcb5bb25e96a00515504f58e49a8"}, + {file = "editor-1.6.5.tar.gz", hash = "sha256:5a8ad611d2a05de34994df3781605e26e63492f82f04c2e93abdd330eed6fa8d"}, +] + +[package.dependencies] +runs = "*" +xmod = "*" [[package]] name = "exceptiongroup" -version = "1.1.2" +version = "1.2.0" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"}, - {file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"}, + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, ] [package.extras] @@ -606,13 +678,13 @@ test = ["pytest (>=6)"] [[package]] name = "freezegun" -version = "1.2.2" +version = "1.4.0" description = "Let your Python tests travel through time" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "freezegun-1.2.2-py3-none-any.whl", hash = "sha256:ea1b963b993cb9ea195adbd893a48d573fda951b0da64f60883d7e988b606c9f"}, - {file = "freezegun-1.2.2.tar.gz", hash = "sha256:cd22d1ba06941384410cd967d8a99d5ae2442f57dfafeff2fda5de8dc5c05446"}, + {file = "freezegun-1.4.0-py3-none-any.whl", hash = "sha256:55e0fc3c84ebf0a96a5aa23ff8b53d70246479e9a68863f1fcac5a3e52f19dd6"}, + {file = "freezegun-1.4.0.tar.gz", hash = "sha256:10939b0ba0ff5adaecf3b06a5c2f73071d9678e507c5eaedb23c761d56ac774b"}, ] [package.dependencies] @@ -620,13 +692,13 @@ python-dateutil = ">=2.7" [[package]] name = "gitdb" -version = "4.0.10" +version = "4.0.11" description = "Git Object Database" optional = false python-versions = ">=3.7" files = [ - {file = "gitdb-4.0.10-py3-none-any.whl", hash = "sha256:c286cf298426064079ed96a9e4a9d39e7f3e9bf15ba60701e95f5492f28415c7"}, - {file = "gitdb-4.0.10.tar.gz", hash = "sha256:6eb990b69df4e15bad899ea868dc46572c3f75339735663b81de79b06f17eb9a"}, + {file = "gitdb-4.0.11-py3-none-any.whl", hash = "sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4"}, + {file = "gitdb-4.0.11.tar.gz", hash = "sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b"}, ] [package.dependencies] @@ -634,27 +706,30 @@ smmap = ">=3.0.1,<6" [[package]] name = "gitpython" -version = "3.1.32" +version = "3.1.41" description = "GitPython is a Python library used to interact with Git repositories" optional = false python-versions = ">=3.7" files = [ - {file = "GitPython-3.1.32-py3-none-any.whl", hash = "sha256:e3d59b1c2c6ebb9dfa7a184daf3b6dd4914237e7488a1730a6d8f6f5d0b4187f"}, - {file = "GitPython-3.1.32.tar.gz", hash = "sha256:8d9b8cb1e80b9735e8717c9362079d3ce4c6e5ddeebedd0361b228c3a67a62f6"}, + {file = "GitPython-3.1.41-py3-none-any.whl", hash = "sha256:c36b6634d069b3f719610175020a9aed919421c87552185b085e04fbbdb10b7c"}, + {file = "GitPython-3.1.41.tar.gz", hash = "sha256:ed66e624884f76df22c8e16066d567aaa5a37d5b5fa19db2c6df6f7156db9048"}, ] [package.dependencies] gitdb = ">=4.0.1,<5" +[package.extras] +test = ["black", "coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre-commit", "pytest (>=7.3.1)", "pytest-cov", "pytest-instafail", "pytest-mock", "pytest-sugar", "sumtypes"] + [[package]] name = "google-api-core" -version = "2.11.1" +version = "2.15.0" description = "Google API client core library" optional = false python-versions = ">=3.7" files = [ - {file = "google-api-core-2.11.1.tar.gz", hash = "sha256:25d29e05a0058ed5f19c61c0a78b1b53adea4d9364b464d014fbda941f6d1c9a"}, - {file = "google_api_core-2.11.1-py3-none-any.whl", hash = "sha256:d92a5a92dc36dd4f4b9ee4e55528a90e432b059f93aee6ad857f9de8cc7ae94a"}, + {file = "google-api-core-2.15.0.tar.gz", hash = "sha256:abc978a72658f14a2df1e5e12532effe40f94f868f6e23d95133bd6abcca35ca"}, + {file = "google_api_core-2.15.0-py3-none-any.whl", hash = "sha256:2aa56d2be495551e66bbff7f729b790546f87d5c90e74781aa77233bcb395a8a"}, ] [package.dependencies] @@ -670,21 +745,19 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] [[package]] name = "google-auth" -version = "2.22.0" +version = "2.26.2" description = "Google Authentication Library" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "google-auth-2.22.0.tar.gz", hash = "sha256:164cba9af4e6e4e40c3a4f90a1a6c12ee56f14c0b4868d1ca91b32826ab334ce"}, - {file = "google_auth-2.22.0-py2.py3-none-any.whl", hash = "sha256:d61d1b40897407b574da67da1a833bdc10d5a11642566e506565d1b1a46ba873"}, + {file = "google-auth-2.26.2.tar.gz", hash = "sha256:97327dbbf58cccb58fc5a1712bba403ae76668e64814eb30f7316f7e27126b81"}, + {file = "google_auth-2.26.2-py2.py3-none-any.whl", hash = "sha256:3f445c8ce9b61ed6459aad86d8ccdba4a9afed841b2d1451a11ef4db08957424"}, ] [package.dependencies] cachetools = ">=2.0.0,<6.0" pyasn1-modules = ">=0.2.1" rsa = ">=3.1.4,<5" -six = ">=1.9.0" -urllib3 = "<2.0" [package.extras] aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] @@ -695,13 +768,13 @@ requests = ["requests (>=2.20.0,<3.0.0.dev0)"] [[package]] name = "google-cloud-core" -version = "2.3.3" +version = "2.4.1" description = "Google Cloud API client core library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-core-2.3.3.tar.gz", hash = "sha256:37b80273c8d7eee1ae816b3a20ae43585ea50506cb0e60f3cf5be5f87f1373cb"}, - {file = "google_cloud_core-2.3.3-py2.py3-none-any.whl", hash = "sha256:fbd11cad3e98a7e5b0343dc07cb1039a5ffd7a5bb96e1f1e27cee4bda4a90863"}, + {file = "google-cloud-core-2.4.1.tar.gz", hash = "sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073"}, + {file = "google_cloud_core-2.4.1-py2.py3-none-any.whl", hash = "sha256:a9e6a4422b9ac5c29f79a0ede9485473338e2ce78d91f2370c01e730eab22e61"}, ] [package.dependencies] @@ -709,24 +782,25 @@ google-api-core = ">=1.31.6,<2.0.dev0 || >2.3.0,<3.0.0dev" google-auth = ">=1.25.0,<3.0dev" [package.extras] -grpc = ["grpcio (>=1.38.0,<2.0dev)"] +grpc = ["grpcio (>=1.38.0,<2.0dev)", "grpcio-status (>=1.38.0,<2.0.dev0)"] [[package]] name = "google-cloud-storage" -version = "2.10.0" +version = "2.14.0" description = "Google Cloud Storage API client library" optional = false python-versions = ">=3.7" files = [ - {file = "google-cloud-storage-2.10.0.tar.gz", hash = "sha256:934b31ead5f3994e5360f9ff5750982c5b6b11604dc072bc452c25965e076dc7"}, - {file = "google_cloud_storage-2.10.0-py2.py3-none-any.whl", hash = "sha256:9433cf28801671de1c80434238fb1e7e4a1ba3087470e90f70c928ea77c2b9d7"}, + {file = "google-cloud-storage-2.14.0.tar.gz", hash = "sha256:2d23fcf59b55e7b45336729c148bb1c464468c69d5efbaee30f7201dd90eb97e"}, + {file = "google_cloud_storage-2.14.0-py2.py3-none-any.whl", hash = "sha256:8641243bbf2a2042c16a6399551fbb13f062cbc9a2de38d6c0bb5426962e9dbd"}, ] [package.dependencies] google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev" -google-auth = ">=1.25.0,<3.0dev" +google-auth = ">=2.23.3,<3.0dev" google-cloud-core = ">=2.3.0,<3.0dev" -google-resumable-media = ">=2.3.2" +google-crc32c = ">=1.0,<2.0dev" +google-resumable-media = ">=2.6.0" requests = ">=2.18.0,<3.0.0dev" [package.extras] @@ -814,31 +888,31 @@ testing = ["pytest"] [[package]] name = "google-resumable-media" -version = "2.5.0" +version = "2.7.0" description = "Utilities for Google Media Downloads and Resumable Uploads" optional = false python-versions = ">= 3.7" files = [ - {file = "google-resumable-media-2.5.0.tar.gz", hash = "sha256:218931e8e2b2a73a58eb354a288e03a0fd5fb1c4583261ac6e4c078666468c93"}, - {file = "google_resumable_media-2.5.0-py2.py3-none-any.whl", hash = "sha256:da1bd943e2e114a56d85d6848497ebf9be6a14d3db23e9fc57581e7c3e8170ec"}, + {file = "google-resumable-media-2.7.0.tar.gz", hash = "sha256:5f18f5fa9836f4b083162064a1c2c98c17239bfda9ca50ad970ccf905f3e625b"}, + {file = "google_resumable_media-2.7.0-py2.py3-none-any.whl", hash = "sha256:79543cfe433b63fd81c0844b7803aba1bb8950b47bedf7d980c38fa123937e08"}, ] [package.dependencies] google-crc32c = ">=1.0,<2.0dev" [package.extras] -aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)"] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)", "google-auth (>=1.22.0,<2.0dev)"] requests = ["requests (>=2.18.0,<3.0.0dev)"] [[package]] name = "googleapis-common-protos" -version = "1.60.0" +version = "1.62.0" description = "Common protobufs used in Google APIs" optional = false python-versions = ">=3.7" files = [ - {file = "googleapis-common-protos-1.60.0.tar.gz", hash = "sha256:e73ebb404098db405ba95d1e1ae0aa91c3e15a71da031a2eeb6b2e23e7bc3708"}, - {file = "googleapis_common_protos-1.60.0-py2.py3-none-any.whl", hash = "sha256:69f9bbcc6acde92cab2db95ce30a70bd2b81d20b12eff3f1aabaffcbe8a93918"}, + {file = "googleapis-common-protos-1.62.0.tar.gz", hash = "sha256:83f0ece9f94e5672cced82f592d2a5edf527a96ed1794f0bab36d5735c996277"}, + {file = "googleapis_common_protos-1.62.0-py2.py3-none-any.whl", hash = "sha256:4750113612205514f9f6aa4cb00d523a94f3e8c06c5ad2fee466387dc4875f07"}, ] [package.dependencies] @@ -849,29 +923,31 @@ grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] [[package]] name = "gql" -version = "3.4.1" +version = "3.5.0" description = "GraphQL client for Python" optional = false python-versions = "*" files = [ - {file = "gql-3.4.1-py2.py3-none-any.whl", hash = "sha256:315624ca0f4d571ef149d455033ebd35e45c1a13f18a059596aeddcea99135cf"}, - {file = "gql-3.4.1.tar.gz", hash = "sha256:11dc5d8715a827f2c2899593439a4f36449db4f0eafa5b1ea63948f8a2f8c545"}, + {file = "gql-3.5.0-py2.py3-none-any.whl", hash = "sha256:70dda5694a5b194a8441f077aa5fb70cc94e4ec08016117523f013680901ecb7"}, + {file = "gql-3.5.0.tar.gz", hash = "sha256:ccb9c5db543682b28f577069950488218ed65d4ac70bb03b6929aaadaf636de9"}, ] [package.dependencies] +anyio = ">=3.0,<5" backoff = ">=1.11.1,<3.0" graphql-core = ">=3.2,<3.3" yarl = ">=1.6,<2.0" [package.extras] -aiohttp = ["aiohttp (>=3.7.1,<3.9.0)"] -all = ["aiohttp (>=3.7.1,<3.9.0)", "botocore (>=1.21,<2)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] +aiohttp = ["aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)"] +all = ["aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "botocore (>=1.21,<2)", "httpx (>=0.23.1,<1)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "websockets (>=10,<12)"] botocore = ["botocore (>=1.21,<2)"] -dev = ["aiofiles", "aiohttp (>=3.7.1,<3.9.0)", "black (==22.3.0)", "botocore (>=1.21,<2)", "check-manifest (>=0.42,<1)", "flake8 (==3.8.1)", "isort (==4.3.21)", "mock (==4.0.2)", "mypy (==0.910)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "sphinx (>=3.0.0,<4)", "sphinx-argparse (==0.2.5)", "sphinx-rtd-theme (>=0.4,<1)", "types-aiofiles", "types-mock", "types-requests", "urllib3 (>=1.26,<2)", "vcrpy (==4.0.2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] -requests = ["requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)"] -test = ["aiofiles", "aiohttp (>=3.7.1,<3.9.0)", "botocore (>=1.21,<2)", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)", "vcrpy (==4.0.2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] -test-no-transport = ["aiofiles", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "vcrpy (==4.0.2)"] -websockets = ["websockets (>=10,<11)", "websockets (>=9,<10)"] +dev = ["aiofiles", "aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "black (==22.3.0)", "botocore (>=1.21,<2)", "check-manifest (>=0.42,<1)", "flake8 (==3.8.1)", "httpx (>=0.23.1,<1)", "isort (==4.3.21)", "mock (==4.0.2)", "mypy (==0.910)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "sphinx (>=5.3.0,<6)", "sphinx-argparse (==0.2.5)", "sphinx-rtd-theme (>=0.4,<1)", "types-aiofiles", "types-mock", "types-requests", "vcrpy (==4.4.0)", "websockets (>=10,<12)"] +httpx = ["httpx (>=0.23.1,<1)"] +requests = ["requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)"] +test = ["aiofiles", "aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "botocore (>=1.21,<2)", "httpx (>=0.23.1,<1)", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "vcrpy (==4.4.0)", "websockets (>=10,<12)"] +test-no-transport = ["aiofiles", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "vcrpy (==4.4.0)"] +websockets = ["websockets (>=10,<12)"] [[package]] name = "graphql-core" @@ -897,39 +973,40 @@ files = [ [[package]] name = "httpcore" -version = "0.17.3" +version = "1.0.2" description = "A minimal low-level HTTP client." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "httpcore-0.17.3-py3-none-any.whl", hash = "sha256:c2789b767ddddfa2a5782e3199b2b7f6894540b17b16ec26b2c4d8e103510b87"}, - {file = "httpcore-0.17.3.tar.gz", hash = "sha256:a6f30213335e34c1ade7be6ec7c47f19f50c56db36abef1a9dfa3815b1cb3888"}, + {file = "httpcore-1.0.2-py3-none-any.whl", hash = "sha256:096cc05bca73b8e459a1fc3dcf585148f63e534eae4339559c9b8a8d6399acc7"}, + {file = "httpcore-1.0.2.tar.gz", hash = "sha256:9fc092e4799b26174648e54b74ed5f683132a464e95643b226e00c2ed2fa6535"}, ] [package.dependencies] -anyio = ">=3.0,<5.0" certifi = "*" h11 = ">=0.13,<0.15" -sniffio = "==1.*" [package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<0.23.0)"] [[package]] name = "httpx" -version = "0.24.1" +version = "0.26.0" description = "The next generation HTTP client." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "httpx-0.24.1-py3-none-any.whl", hash = "sha256:06781eb9ac53cde990577af654bd990a4949de37a28bdb4a230d434f3a30b9bd"}, - {file = "httpx-0.24.1.tar.gz", hash = "sha256:5853a43053df830c20f8110c5e69fe44d035d850b2dfe795e196f00fdb774bdd"}, + {file = "httpx-0.26.0-py3-none-any.whl", hash = "sha256:8915f5a3627c4d47b73e8202457cb28f1266982d1159bd5779d86a80c0eab1cd"}, + {file = "httpx-0.26.0.tar.gz", hash = "sha256:451b55c30d5185ea6b23c2c793abf9bb237d2a7dfb901ced6ff69ad37ec1dfaf"}, ] [package.dependencies] +anyio = "*" certifi = "*" -httpcore = ">=0.15.0,<0.18.0" +httpcore = "==1.*" idna = "*" sniffio = "*" @@ -941,13 +1018,13 @@ socks = ["socksio (==1.*)"] [[package]] name = "idna" -version = "3.4" +version = "3.6" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.5" files = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, ] [[package]] @@ -961,15 +1038,31 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] +[[package]] +name = "inquirer" +version = "3.2.1" +description = "Collection of common interactive command line user interfaces, based on Inquirer.js" +optional = false +python-versions = ">=3.8.1" +files = [ + {file = "inquirer-3.2.1-py3-none-any.whl", hash = "sha256:e1a0a001b499633ca69d2ea64da712b449939e8fad8fa47caebc92b0ee212df4"}, + {file = "inquirer-3.2.1.tar.gz", hash = "sha256:d5ff9bb8cd07bd3f076eabad8ae338280886e93998ff10461975b768e3854fbc"}, +] + +[package.dependencies] +blessed = ">=1.19.0" +editor = ">=1.6.0" +readchar = ">=3.0.6" + [[package]] name = "jinja2" -version = "3.1.2" +version = "3.1.3" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" files = [ - {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, - {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, ] [package.dependencies] @@ -978,6 +1071,58 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "jinxed" +version = "1.2.1" +description = "Jinxed Terminal Library" +optional = false +python-versions = "*" +files = [ + {file = "jinxed-1.2.1-py2.py3-none-any.whl", hash = "sha256:37422659c4925969c66148c5e64979f553386a4226b9484d910d3094ced37d30"}, + {file = "jinxed-1.2.1.tar.gz", hash = "sha256:30c3f861b73279fea1ed928cfd4dfb1f273e16cd62c8a32acfac362da0f78f3f"}, +] + +[package.dependencies] +ansicon = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "macholib" +version = "1.16.3" +description = "Mach-O header analysis and editing" +optional = false +python-versions = "*" +files = [ + {file = "macholib-1.16.3-py2.py3-none-any.whl", hash = "sha256:0e315d7583d38b8c77e815b1ecbdbf504a8258d8b3e17b61165c6feb60d18f2c"}, + {file = "macholib-1.16.3.tar.gz", hash = "sha256:07ae9e15e8e4cd9a788013d81f5908b3609aa76f9b1421bae9c4d7606ec86a30"}, +] + +[package.dependencies] +altgraph = ">=0.17" + +[[package]] +name = "markdown-it-py" +version = "3.0.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +optional = false +python-versions = ">=3.8" +files = [ + {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, + {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, +] + +[package.dependencies] +mdurl = ">=0.1,<1.0" + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +code-style = ["pre-commit (>=3.0,<4.0)"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + [[package]] name = "markupsafe" version = "2.1.3" @@ -1005,6 +1150,16 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1037,6 +1192,28 @@ files = [ {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, ] +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] + +[[package]] +name = "monotonic" +version = "1.6" +description = "An implementation of time.monotonic() for Python 2 & < 3.3" +optional = false +python-versions = "*" +files = [ + {file = "monotonic-1.6-py2.py3-none-any.whl", hash = "sha256:68687e19a14f11f26d140dd5c86f3dba4bf5df58003000ed467e0e2a69bca96c"}, + {file = "monotonic-1.6.tar.gz", hash = "sha256:3a55207bcfed53ddd5c5bae174524062935efed17792e9de2ad0205ce9ad63f7"}, +] + [[package]] name = "more-itertools" version = "8.14.0" @@ -1131,26 +1308,215 @@ files = [ {file = "multidict-6.0.4.tar.gz", hash = "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49"}, ] +[[package]] +name = "mypy" +version = "1.8.0" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mypy-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:485a8942f671120f76afffff70f259e1cd0f0cfe08f81c05d8816d958d4577d3"}, + {file = "mypy-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:df9824ac11deaf007443e7ed2a4a26bebff98d2bc43c6da21b2b64185da011c4"}, + {file = "mypy-1.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2afecd6354bbfb6e0160f4e4ad9ba6e4e003b767dd80d85516e71f2e955ab50d"}, + {file = "mypy-1.8.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8963b83d53ee733a6e4196954502b33567ad07dfd74851f32be18eb932fb1cb9"}, + {file = "mypy-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:e46f44b54ebddbeedbd3d5b289a893219065ef805d95094d16a0af6630f5d410"}, + {file = "mypy-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:855fe27b80375e5c5878492f0729540db47b186509c98dae341254c8f45f42ae"}, + {file = "mypy-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4c886c6cce2d070bd7df4ec4a05a13ee20c0aa60cb587e8d1265b6c03cf91da3"}, + {file = "mypy-1.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d19c413b3c07cbecf1f991e2221746b0d2a9410b59cb3f4fb9557f0365a1a817"}, + {file = "mypy-1.8.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9261ed810972061388918c83c3f5cd46079d875026ba97380f3e3978a72f503d"}, + {file = "mypy-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:51720c776d148bad2372ca21ca29256ed483aa9a4cdefefcef49006dff2a6835"}, + {file = "mypy-1.8.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:52825b01f5c4c1c4eb0db253ec09c7aa17e1a7304d247c48b6f3599ef40db8bd"}, + {file = "mypy-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f5ac9a4eeb1ec0f1ccdc6f326bcdb464de5f80eb07fb38b5ddd7b0de6bc61e55"}, + {file = "mypy-1.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afe3fe972c645b4632c563d3f3eff1cdca2fa058f730df2b93a35e3b0c538218"}, + {file = "mypy-1.8.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:42c6680d256ab35637ef88891c6bd02514ccb7e1122133ac96055ff458f93fc3"}, + {file = "mypy-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:720a5ca70e136b675af3af63db533c1c8c9181314d207568bbe79051f122669e"}, + {file = "mypy-1.8.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:028cf9f2cae89e202d7b6593cd98db6759379f17a319b5faf4f9978d7084cdc6"}, + {file = "mypy-1.8.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4e6d97288757e1ddba10dd9549ac27982e3e74a49d8d0179fc14d4365c7add66"}, + {file = "mypy-1.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f1478736fcebb90f97e40aff11a5f253af890c845ee0c850fe80aa060a267c6"}, + {file = "mypy-1.8.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42419861b43e6962a649068a61f4a4839205a3ef525b858377a960b9e2de6e0d"}, + {file = "mypy-1.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:2b5b6c721bd4aabaadead3a5e6fa85c11c6c795e0c81a7215776ef8afc66de02"}, + {file = "mypy-1.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5c1538c38584029352878a0466f03a8ee7547d7bd9f641f57a0f3017a7c905b8"}, + {file = "mypy-1.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ef4be7baf08a203170f29e89d79064463b7fc7a0908b9d0d5114e8009c3a259"}, + {file = "mypy-1.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7178def594014aa6c35a8ff411cf37d682f428b3b5617ca79029d8ae72f5402b"}, + {file = "mypy-1.8.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ab3c84fa13c04aeeeabb2a7f67a25ef5d77ac9d6486ff33ded762ef353aa5592"}, + {file = "mypy-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:99b00bc72855812a60d253420d8a2eae839b0afa4938f09f4d2aa9bb4654263a"}, + {file = "mypy-1.8.0-py3-none-any.whl", hash = "sha256:538fd81bb5e430cc1381a443971c0475582ff9f434c16cd46d2c66763ce85d9d"}, + {file = "mypy-1.8.0.tar.gz", hash = "sha256:6ff8b244d7085a0b425b56d327b480c3b29cafbd2eff27316a004f9a7391ae07"}, +] + +[package.dependencies] +mypy-extensions = ">=1.0.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=4.1.0" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "numpy" +version = "1.26.3" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "numpy-1.26.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:806dd64230dbbfaca8a27faa64e2f414bf1c6622ab78cc4264f7f5f028fee3bf"}, + {file = "numpy-1.26.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02f98011ba4ab17f46f80f7f8f1c291ee7d855fcef0a5a98db80767a468c85cd"}, + {file = "numpy-1.26.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d45b3ec2faed4baca41c76617fcdcfa4f684ff7a151ce6fc78ad3b6e85af0a6"}, + {file = "numpy-1.26.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdd2b45bf079d9ad90377048e2747a0c82351989a2165821f0c96831b4a2a54b"}, + {file = "numpy-1.26.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:211ddd1e94817ed2d175b60b6374120244a4dd2287f4ece45d49228b4d529178"}, + {file = "numpy-1.26.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b1240f767f69d7c4c8a29adde2310b871153df9b26b5cb2b54a561ac85146485"}, + {file = "numpy-1.26.3-cp310-cp310-win32.whl", hash = "sha256:21a9484e75ad018974a2fdaa216524d64ed4212e418e0a551a2d83403b0531d3"}, + {file = "numpy-1.26.3-cp310-cp310-win_amd64.whl", hash = "sha256:9e1591f6ae98bcfac2a4bbf9221c0b92ab49762228f38287f6eeb5f3f55905ce"}, + {file = "numpy-1.26.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b831295e5472954104ecb46cd98c08b98b49c69fdb7040483aff799a755a7374"}, + {file = "numpy-1.26.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9e87562b91f68dd8b1c39149d0323b42e0082db7ddb8e934ab4c292094d575d6"}, + {file = "numpy-1.26.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c66d6fec467e8c0f975818c1796d25c53521124b7cfb760114be0abad53a0a2"}, + {file = "numpy-1.26.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f25e2811a9c932e43943a2615e65fc487a0b6b49218899e62e426e7f0a57eeda"}, + {file = "numpy-1.26.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:af36e0aa45e25c9f57bf684b1175e59ea05d9a7d3e8e87b7ae1a1da246f2767e"}, + {file = "numpy-1.26.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:51c7f1b344f302067b02e0f5b5d2daa9ed4a721cf49f070280ac202738ea7f00"}, + {file = "numpy-1.26.3-cp311-cp311-win32.whl", hash = "sha256:7ca4f24341df071877849eb2034948459ce3a07915c2734f1abb4018d9c49d7b"}, + {file = "numpy-1.26.3-cp311-cp311-win_amd64.whl", hash = "sha256:39763aee6dfdd4878032361b30b2b12593fb445ddb66bbac802e2113eb8a6ac4"}, + {file = "numpy-1.26.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a7081fd19a6d573e1a05e600c82a1c421011db7935ed0d5c483e9dd96b99cf13"}, + {file = "numpy-1.26.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12c70ac274b32bc00c7f61b515126c9205323703abb99cd41836e8125ea0043e"}, + {file = "numpy-1.26.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f784e13e598e9594750b2ef6729bcd5a47f6cfe4a12cca13def35e06d8163e3"}, + {file = "numpy-1.26.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f24750ef94d56ce6e33e4019a8a4d68cfdb1ef661a52cdaee628a56d2437419"}, + {file = "numpy-1.26.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:77810ef29e0fb1d289d225cabb9ee6cf4d11978a00bb99f7f8ec2132a84e0166"}, + {file = "numpy-1.26.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8ed07a90f5450d99dad60d3799f9c03c6566709bd53b497eb9ccad9a55867f36"}, + {file = "numpy-1.26.3-cp312-cp312-win32.whl", hash = "sha256:f73497e8c38295aaa4741bdfa4fda1a5aedda5473074369eca10626835445511"}, + {file = "numpy-1.26.3-cp312-cp312-win_amd64.whl", hash = "sha256:da4b0c6c699a0ad73c810736303f7fbae483bcb012e38d7eb06a5e3b432c981b"}, + {file = "numpy-1.26.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1666f634cb3c80ccbd77ec97bc17337718f56d6658acf5d3b906ca03e90ce87f"}, + {file = "numpy-1.26.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:18c3319a7d39b2c6a9e3bb75aab2304ab79a811ac0168a671a62e6346c29b03f"}, + {file = "numpy-1.26.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b7e807d6888da0db6e7e75838444d62495e2b588b99e90dd80c3459594e857b"}, + {file = "numpy-1.26.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4d362e17bcb0011738c2d83e0a65ea8ce627057b2fdda37678f4374a382a137"}, + {file = "numpy-1.26.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b8c275f0ae90069496068c714387b4a0eba5d531aace269559ff2b43655edd58"}, + {file = "numpy-1.26.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:cc0743f0302b94f397a4a65a660d4cd24267439eb16493fb3caad2e4389bccbb"}, + {file = "numpy-1.26.3-cp39-cp39-win32.whl", hash = "sha256:9bc6d1a7f8cedd519c4b7b1156d98e051b726bf160715b769106661d567b3f03"}, + {file = "numpy-1.26.3-cp39-cp39-win_amd64.whl", hash = "sha256:867e3644e208c8922a3be26fc6bbf112a035f50f0a86497f98f228c50c607bb2"}, + {file = "numpy-1.26.3-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3c67423b3703f8fbd90f5adaa37f85b5794d3366948efe9a5190a5f3a83fc34e"}, + {file = "numpy-1.26.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46f47ee566d98849323f01b349d58f2557f02167ee301e5e28809a8c0e27a2d0"}, + {file = "numpy-1.26.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a8474703bffc65ca15853d5fd4d06b18138ae90c17c8d12169968e998e448bb5"}, + {file = "numpy-1.26.3.tar.gz", hash = "sha256:697df43e2b6310ecc9d95f05d5ef20eacc09c7c4ecc9da3f235d39e71b7da1e4"}, +] + [[package]] name = "packaging" -version = "23.1" +version = "23.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, - {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pandas" +version = "2.1.4" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pandas-2.1.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bdec823dc6ec53f7a6339a0e34c68b144a7a1fd28d80c260534c39c62c5bf8c9"}, + {file = "pandas-2.1.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:294d96cfaf28d688f30c918a765ea2ae2e0e71d3536754f4b6de0ea4a496d034"}, + {file = "pandas-2.1.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b728fb8deba8905b319f96447a27033969f3ea1fea09d07d296c9030ab2ed1d"}, + {file = "pandas-2.1.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00028e6737c594feac3c2df15636d73ace46b8314d236100b57ed7e4b9ebe8d9"}, + {file = "pandas-2.1.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:426dc0f1b187523c4db06f96fb5c8d1a845e259c99bda74f7de97bd8a3bb3139"}, + {file = "pandas-2.1.4-cp310-cp310-win_amd64.whl", hash = "sha256:f237e6ca6421265643608813ce9793610ad09b40154a3344a088159590469e46"}, + {file = "pandas-2.1.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b7d852d16c270e4331f6f59b3e9aa23f935f5c4b0ed2d0bc77637a8890a5d092"}, + {file = "pandas-2.1.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bd7d5f2f54f78164b3d7a40f33bf79a74cdee72c31affec86bfcabe7e0789821"}, + {file = "pandas-2.1.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0aa6e92e639da0d6e2017d9ccff563222f4eb31e4b2c3cf32a2a392fc3103c0d"}, + {file = "pandas-2.1.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d797591b6846b9db79e65dc2d0d48e61f7db8d10b2a9480b4e3faaddc421a171"}, + {file = "pandas-2.1.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d2d3e7b00f703aea3945995ee63375c61b2e6aa5aa7871c5d622870e5e137623"}, + {file = "pandas-2.1.4-cp311-cp311-win_amd64.whl", hash = "sha256:dc9bf7ade01143cddc0074aa6995edd05323974e6e40d9dbde081021ded8510e"}, + {file = "pandas-2.1.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:482d5076e1791777e1571f2e2d789e940dedd927325cc3cb6d0800c6304082f6"}, + {file = "pandas-2.1.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8a706cfe7955c4ca59af8c7a0517370eafbd98593155b48f10f9811da440248b"}, + {file = "pandas-2.1.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0513a132a15977b4a5b89aabd304647919bc2169eac4c8536afb29c07c23540"}, + {file = "pandas-2.1.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9f17f2b6fc076b2a0078862547595d66244db0f41bf79fc5f64a5c4d635bead"}, + {file = "pandas-2.1.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:45d63d2a9b1b37fa6c84a68ba2422dc9ed018bdaa668c7f47566a01188ceeec1"}, + {file = "pandas-2.1.4-cp312-cp312-win_amd64.whl", hash = "sha256:f69b0c9bb174a2342818d3e2778584e18c740d56857fc5cdb944ec8bbe4082cf"}, + {file = "pandas-2.1.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3f06bda01a143020bad20f7a85dd5f4a1600112145f126bc9e3e42077c24ef34"}, + {file = "pandas-2.1.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ab5796839eb1fd62a39eec2916d3e979ec3130509930fea17fe6f81e18108f6a"}, + {file = "pandas-2.1.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edbaf9e8d3a63a9276d707b4d25930a262341bca9874fcb22eff5e3da5394732"}, + {file = "pandas-2.1.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ebfd771110b50055712b3b711b51bee5d50135429364d0498e1213a7adc2be8"}, + {file = "pandas-2.1.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8ea107e0be2aba1da619cc6ba3f999b2bfc9669a83554b1904ce3dd9507f0860"}, + {file = "pandas-2.1.4-cp39-cp39-win_amd64.whl", hash = "sha256:d65148b14788b3758daf57bf42725caa536575da2b64df9964c563b015230984"}, + {file = "pandas-2.1.4.tar.gz", hash = "sha256:fcb68203c833cc735321512e13861358079a96c174a61f5116a1de89c58c0ef7"}, +] + +[package.dependencies] +numpy = {version = ">=1.22.4,<2", markers = "python_version < \"3.11\""} +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.1" + +[package.extras] +all = ["PyQt5 (>=5.15.6)", "SQLAlchemy (>=1.4.36)", "beautifulsoup4 (>=4.11.1)", "bottleneck (>=1.3.4)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=0.8.1)", "fsspec (>=2022.05.0)", "gcsfs (>=2022.05.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.8.0)", "matplotlib (>=3.6.1)", "numba (>=0.55.2)", "numexpr (>=2.8.0)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pandas-gbq (>=0.17.5)", "psycopg2 (>=2.9.3)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.5)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "pyxlsb (>=1.0.9)", "qtpy (>=2.2.0)", "s3fs (>=2022.05.0)", "scipy (>=1.8.1)", "tables (>=3.7.0)", "tabulate (>=0.8.10)", "xarray (>=2022.03.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)", "zstandard (>=0.17.0)"] +aws = ["s3fs (>=2022.05.0)"] +clipboard = ["PyQt5 (>=5.15.6)", "qtpy (>=2.2.0)"] +compression = ["zstandard (>=0.17.0)"] +computation = ["scipy (>=1.8.1)", "xarray (>=2022.03.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pyxlsb (>=1.0.9)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)"] +feather = ["pyarrow (>=7.0.0)"] +fss = ["fsspec (>=2022.05.0)"] +gcp = ["gcsfs (>=2022.05.0)", "pandas-gbq (>=0.17.5)"] +hdf5 = ["tables (>=3.7.0)"] +html = ["beautifulsoup4 (>=4.11.1)", "html5lib (>=1.1)", "lxml (>=4.8.0)"] +mysql = ["SQLAlchemy (>=1.4.36)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.8.10)"] +parquet = ["pyarrow (>=7.0.0)"] +performance = ["bottleneck (>=1.3.4)", "numba (>=0.55.2)", "numexpr (>=2.8.0)"] +plot = ["matplotlib (>=3.6.1)"] +postgresql = ["SQLAlchemy (>=1.4.36)", "psycopg2 (>=2.9.3)"] +spss = ["pyreadstat (>=1.1.5)"] +sql-other = ["SQLAlchemy (>=1.4.36)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.8.0)"] + +[[package]] +name = "pastel" +version = "0.2.1" +description = "Bring colors to your terminal." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pastel-0.2.1-py2.py3-none-any.whl", hash = "sha256:4349225fcdf6c2bb34d483e523475de5bb04a5c10ef711263452cb37d7dd4364"}, + {file = "pastel-0.2.1.tar.gz", hash = "sha256:e6581ac04e973cac858828c6202c1e1e81fee1dc7de7683f3e1ffe0bfd8a573d"}, +] + +[[package]] +name = "pefile" +version = "2023.2.7" +description = "Python PE parsing module" +optional = false +python-versions = ">=3.6.0" +files = [ + {file = "pefile-2023.2.7-py3-none-any.whl", hash = "sha256:da185cd2af68c08a6cd4481f7325ed600a88f6a813bad9dea07ab3ef73d8d8d6"}, + {file = "pefile-2023.2.7.tar.gz", hash = "sha256:82e6114004b3d6911c77c3953e3838654b04511b8b66e8583db70c65998017dc"}, ] [[package]] name = "platformdirs" -version = "3.10.0" +version = "4.1.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "platformdirs-3.10.0-py3-none-any.whl", hash = "sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d"}, - {file = "platformdirs-3.10.0.tar.gz", hash = "sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d"}, + {file = "platformdirs-4.1.0-py3-none-any.whl", hash = "sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380"}, + {file = "platformdirs-4.1.0.tar.gz", hash = "sha256:906d548203468492d432bcb294d4bc2fff751bf84971fbb2c10918cc206ee420"}, ] [package.extras] @@ -1159,39 +1525,55 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-co [[package]] name = "pluggy" -version = "1.2.0" +version = "1.3.0" description = "plugin and hook calling mechanisms for python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, - {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, + {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, + {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, ] [package.extras] dev = ["pre-commit", "tox"] testing = ["pytest", "pytest-benchmark"] +[[package]] +name = "poethepoet" +version = "0.24.4" +description = "A task runner that works well with poetry." +optional = false +python-versions = ">=3.8" +files = [ + {file = "poethepoet-0.24.4-py3-none-any.whl", hash = "sha256:fb4ea35d7f40fe2081ea917d2e4102e2310fda2cde78974050ca83896e229075"}, + {file = "poethepoet-0.24.4.tar.gz", hash = "sha256:ff4220843a87c888cbcb5312c8905214701d0af60ac7271795baa8369b428fef"}, +] + +[package.dependencies] +pastel = ">=0.2.1,<0.3.0" +tomli = ">=1.2.2" + +[package.extras] +poetry-plugin = ["poetry (>=1.0,<2.0)"] + [[package]] name = "protobuf" -version = "4.24.0" +version = "4.25.2" description = "" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "protobuf-4.24.0-cp310-abi3-win32.whl", hash = "sha256:81cb9c4621d2abfe181154354f63af1c41b00a4882fb230b4425cbaed65e8f52"}, - {file = "protobuf-4.24.0-cp310-abi3-win_amd64.whl", hash = "sha256:6c817cf4a26334625a1904b38523d1b343ff8b637d75d2c8790189a4064e51c3"}, - {file = "protobuf-4.24.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:ae97b5de10f25b7a443b40427033e545a32b0e9dda17bcd8330d70033379b3e5"}, - {file = "protobuf-4.24.0-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:567fe6b0647494845d0849e3d5b260bfdd75692bf452cdc9cb660d12457c055d"}, - {file = "protobuf-4.24.0-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:a6b1ca92ccabfd9903c0c7dde8876221dc7d8d87ad5c42e095cc11b15d3569c7"}, - {file = "protobuf-4.24.0-cp37-cp37m-win32.whl", hash = "sha256:a38400a692fd0c6944c3c58837d112f135eb1ed6cdad5ca6c5763336e74f1a04"}, - {file = "protobuf-4.24.0-cp37-cp37m-win_amd64.whl", hash = "sha256:5ab19ee50037d4b663c02218a811a5e1e7bb30940c79aac385b96e7a4f9daa61"}, - {file = "protobuf-4.24.0-cp38-cp38-win32.whl", hash = "sha256:e8834ef0b4c88666ebb7c7ec18045aa0f4325481d724daa624a4cf9f28134653"}, - {file = "protobuf-4.24.0-cp38-cp38-win_amd64.whl", hash = "sha256:8bb52a2be32db82ddc623aefcedfe1e0eb51da60e18fcc908fb8885c81d72109"}, - {file = "protobuf-4.24.0-cp39-cp39-win32.whl", hash = "sha256:ae7a1835721086013de193311df858bc12cd247abe4ef9710b715d930b95b33e"}, - {file = "protobuf-4.24.0-cp39-cp39-win_amd64.whl", hash = "sha256:44825e963008f8ea0d26c51911c30d3e82e122997c3c4568fd0385dd7bacaedf"}, - {file = "protobuf-4.24.0-py3-none-any.whl", hash = "sha256:82e6e9ebdd15b8200e8423676eab38b774624d6a1ad696a60d86a2ac93f18201"}, - {file = "protobuf-4.24.0.tar.gz", hash = "sha256:5d0ceb9de6e08311832169e601d1fc71bd8e8c779f3ee38a97a78554945ecb85"}, + {file = "protobuf-4.25.2-cp310-abi3-win32.whl", hash = "sha256:b50c949608682b12efb0b2717f53256f03636af5f60ac0c1d900df6213910fd6"}, + {file = "protobuf-4.25.2-cp310-abi3-win_amd64.whl", hash = "sha256:8f62574857ee1de9f770baf04dde4165e30b15ad97ba03ceac65f760ff018ac9"}, + {file = "protobuf-4.25.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:2db9f8fa64fbdcdc93767d3cf81e0f2aef176284071507e3ede160811502fd3d"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:10894a2885b7175d3984f2be8d9850712c57d5e7587a2410720af8be56cdaf62"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fc381d1dd0516343f1440019cedf08a7405f791cd49eef4ae1ea06520bc1c020"}, + {file = "protobuf-4.25.2-cp38-cp38-win32.whl", hash = "sha256:33a1aeef4b1927431d1be780e87b641e322b88d654203a9e9d93f218ee359e61"}, + {file = "protobuf-4.25.2-cp38-cp38-win_amd64.whl", hash = "sha256:47f3de503fe7c1245f6f03bea7e8d3ec11c6c4a2ea9ef910e3221c8a15516d62"}, + {file = "protobuf-4.25.2-cp39-cp39-win32.whl", hash = "sha256:5e5c933b4c30a988b52e0b7c02641760a5ba046edc5e43d3b94a74c9fc57c1b3"}, + {file = "protobuf-4.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:d66a769b8d687df9024f2985d5137a337f957a0916cf5464d1513eee96a63ff0"}, + {file = "protobuf-4.25.2-py3-none-any.whl", hash = "sha256:a8b7a98d4ce823303145bf3c1a8bdb0f2f4642a414b196f04ad9853ed0c8f830"}, + {file = "protobuf-4.25.2.tar.gz", hash = "sha256:fe599e175cb347efc8ee524bcd4b902d11f7262c0e569ececcb89995c15f0a5e"}, ] [[package]] @@ -1207,13 +1589,13 @@ files = [ [[package]] name = "pyasn1" -version = "0.5.0" +version = "0.5.1" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ - {file = "pyasn1-0.5.0-py2.py3-none-any.whl", hash = "sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57"}, - {file = "pyasn1-0.5.0.tar.gz", hash = "sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde"}, + {file = "pyasn1-0.5.1-py2.py3-none-any.whl", hash = "sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58"}, + {file = "pyasn1-0.5.1.tar.gz", hash = "sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c"}, ] [[package]] @@ -1243,50 +1625,51 @@ files = [ [[package]] name = "pydantic" -version = "1.9.2" +version = "1.10.13" description = "Data validation and settings management using python type hints" optional = false -python-versions = ">=3.6.1" -files = [ - {file = "pydantic-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c9e04a6cdb7a363d7cb3ccf0efea51e0abb48e180c0d31dca8d247967d85c6e"}, - {file = "pydantic-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fafe841be1103f340a24977f61dee76172e4ae5f647ab9e7fd1e1fca51524f08"}, - {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afacf6d2a41ed91fc631bade88b1d319c51ab5418870802cedb590b709c5ae3c"}, - {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ee0d69b2a5b341fc7927e92cae7ddcfd95e624dfc4870b32a85568bd65e6131"}, - {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ff68fc85355532ea77559ede81f35fff79a6a5543477e168ab3a381887caea76"}, - {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c0f5e142ef8217019e3eef6ae1b6b55f09a7a15972958d44fbd228214cede567"}, - {file = "pydantic-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:615661bfc37e82ac677543704437ff737418e4ea04bef9cf11c6d27346606044"}, - {file = "pydantic-1.9.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:328558c9f2eed77bd8fffad3cef39dbbe3edc7044517f4625a769d45d4cf7555"}, - {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bd446bdb7755c3a94e56d7bdfd3ee92396070efa8ef3a34fab9579fe6aa1d84"}, - {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0b214e57623a535936005797567231a12d0da0c29711eb3514bc2b3cd008d0f"}, - {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:d8ce3fb0841763a89322ea0432f1f59a2d3feae07a63ea2c958b2315e1ae8adb"}, - {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b34ba24f3e2d0b39b43f0ca62008f7ba962cff51efa56e64ee25c4af6eed987b"}, - {file = "pydantic-1.9.2-cp36-cp36m-win_amd64.whl", hash = "sha256:84d76ecc908d917f4684b354a39fd885d69dd0491be175f3465fe4b59811c001"}, - {file = "pydantic-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4de71c718c9756d679420c69f216776c2e977459f77e8f679a4a961dc7304a56"}, - {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5803ad846cdd1ed0d97eb00292b870c29c1f03732a010e66908ff48a762f20e4"}, - {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a8c5360a0297a713b4123608a7909e6869e1b56d0e96eb0d792c27585d40757f"}, - {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:cdb4272678db803ddf94caa4f94f8672e9a46bae4a44f167095e4d06fec12979"}, - {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:19b5686387ea0d1ea52ecc4cffb71abb21702c5e5b2ac626fd4dbaa0834aa49d"}, - {file = "pydantic-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:32e0b4fb13ad4db4058a7c3c80e2569adbd810c25e6ca3bbd8b2a9cc2cc871d7"}, - {file = "pydantic-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91089b2e281713f3893cd01d8e576771cd5bfdfbff5d0ed95969f47ef6d676c3"}, - {file = "pydantic-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e631c70c9280e3129f071635b81207cad85e6c08e253539467e4ead0e5b219aa"}, - {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b3946f87e5cef3ba2e7bd3a4eb5a20385fe36521d6cc1ebf3c08a6697c6cfb3"}, - {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5565a49effe38d51882cb7bac18bda013cdb34d80ac336428e8908f0b72499b0"}, - {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:bd67cb2c2d9602ad159389c29e4ca964b86fa2f35c2faef54c3eb28b4efd36c8"}, - {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4aafd4e55e8ad5bd1b19572ea2df546ccace7945853832bb99422a79c70ce9b8"}, - {file = "pydantic-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:d70916235d478404a3fa8c997b003b5f33aeac4686ac1baa767234a0f8ac2326"}, - {file = "pydantic-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0ca86b525264daa5f6b192f216a0d1e860b7383e3da1c65a1908f9c02f42801"}, - {file = "pydantic-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1061c6ee6204f4f5a27133126854948e3b3d51fcc16ead2e5d04378c199b2f44"}, - {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e78578f0c7481c850d1c969aca9a65405887003484d24f6110458fb02cca7747"}, - {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5da164119602212a3fe7e3bc08911a89db4710ae51444b4224c2382fd09ad453"}, - {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ead3cd020d526f75b4188e0a8d71c0dbbe1b4b6b5dc0ea775a93aca16256aeb"}, - {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7d0f183b305629765910eaad707800d2f47c6ac5bcfb8c6397abdc30b69eeb15"}, - {file = "pydantic-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:f1a68f4f65a9ee64b6ccccb5bf7e17db07caebd2730109cb8a95863cfa9c4e55"}, - {file = "pydantic-1.9.2-py3-none-any.whl", hash = "sha256:78a4d6bdfd116a559aeec9a4cfe77dda62acc6233f8b56a716edad2651023e5e"}, - {file = "pydantic-1.9.2.tar.gz", hash = "sha256:8cb0bc509bfb71305d7a59d00163d5f9fc4530f0881ea32c74ff4f74c85f3d3d"}, +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.13-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:efff03cc7a4f29d9009d1c96ceb1e7a70a65cfe86e89d34e4a5f2ab1e5693737"}, + {file = "pydantic-1.10.13-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3ecea2b9d80e5333303eeb77e180b90e95eea8f765d08c3d278cd56b00345d01"}, + {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1740068fd8e2ef6eb27a20e5651df000978edce6da6803c2bef0bc74540f9548"}, + {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84bafe2e60b5e78bc64a2941b4c071a4b7404c5c907f5f5a99b0139781e69ed8"}, + {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bc0898c12f8e9c97f6cd44c0ed70d55749eaf783716896960b4ecce2edfd2d69"}, + {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:654db58ae399fe6434e55325a2c3e959836bd17a6f6a0b6ca8107ea0571d2e17"}, + {file = "pydantic-1.10.13-cp310-cp310-win_amd64.whl", hash = "sha256:75ac15385a3534d887a99c713aa3da88a30fbd6204a5cd0dc4dab3d770b9bd2f"}, + {file = "pydantic-1.10.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c553f6a156deb868ba38a23cf0df886c63492e9257f60a79c0fd8e7173537653"}, + {file = "pydantic-1.10.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5e08865bc6464df8c7d61439ef4439829e3ab62ab1669cddea8dd00cd74b9ffe"}, + {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e31647d85a2013d926ce60b84f9dd5300d44535a9941fe825dc349ae1f760df9"}, + {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:210ce042e8f6f7c01168b2d84d4c9eb2b009fe7bf572c2266e235edf14bacd80"}, + {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8ae5dd6b721459bfa30805f4c25880e0dd78fc5b5879f9f7a692196ddcb5a580"}, + {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f8e81fc5fb17dae698f52bdd1c4f18b6ca674d7068242b2aff075f588301bbb0"}, + {file = "pydantic-1.10.13-cp311-cp311-win_amd64.whl", hash = "sha256:61d9dce220447fb74f45e73d7ff3b530e25db30192ad8d425166d43c5deb6df0"}, + {file = "pydantic-1.10.13-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4b03e42ec20286f052490423682016fd80fda830d8e4119f8ab13ec7464c0132"}, + {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59ef915cac80275245824e9d771ee939133be38215555e9dc90c6cb148aaeb5"}, + {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a1f9f747851338933942db7af7b6ee8268568ef2ed86c4185c6ef4402e80ba8"}, + {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:97cce3ae7341f7620a0ba5ef6cf043975cd9d2b81f3aa5f4ea37928269bc1b87"}, + {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854223752ba81e3abf663d685f105c64150873cc6f5d0c01d3e3220bcff7d36f"}, + {file = "pydantic-1.10.13-cp37-cp37m-win_amd64.whl", hash = "sha256:b97c1fac8c49be29486df85968682b0afa77e1b809aff74b83081cc115e52f33"}, + {file = "pydantic-1.10.13-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c958d053453a1c4b1c2062b05cd42d9d5c8eb67537b8d5a7e3c3032943ecd261"}, + {file = "pydantic-1.10.13-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c5370a7edaac06daee3af1c8b1192e305bc102abcbf2a92374b5bc793818599"}, + {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d6f6e7305244bddb4414ba7094ce910560c907bdfa3501e9db1a7fd7eaea127"}, + {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3a3c792a58e1622667a2837512099eac62490cdfd63bd407993aaf200a4cf1f"}, + {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c636925f38b8db208e09d344c7aa4f29a86bb9947495dd6b6d376ad10334fb78"}, + {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:678bcf5591b63cc917100dc50ab6caebe597ac67e8c9ccb75e698f66038ea953"}, + {file = "pydantic-1.10.13-cp38-cp38-win_amd64.whl", hash = "sha256:6cf25c1a65c27923a17b3da28a0bdb99f62ee04230c931d83e888012851f4e7f"}, + {file = "pydantic-1.10.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8ef467901d7a41fa0ca6db9ae3ec0021e3f657ce2c208e98cd511f3161c762c6"}, + {file = "pydantic-1.10.13-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:968ac42970f57b8344ee08837b62f6ee6f53c33f603547a55571c954a4225691"}, + {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9849f031cf8a2f0a928fe885e5a04b08006d6d41876b8bbd2fc68a18f9f2e3fd"}, + {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:56e3ff861c3b9c6857579de282ce8baabf443f42ffba355bf070770ed63e11e1"}, + {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f00790179497767aae6bcdc36355792c79e7bbb20b145ff449700eb076c5f96"}, + {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:75b297827b59bc229cac1a23a2f7a4ac0031068e5be0ce385be1462e7e17a35d"}, + {file = "pydantic-1.10.13-cp39-cp39-win_amd64.whl", hash = "sha256:e70ca129d2053fb8b728ee7d1af8e553a928d7e301a311094b8a0501adc8763d"}, + {file = "pydantic-1.10.13-py3-none-any.whl", hash = "sha256:b87326822e71bd5f313e7d3bfdc77ac3247035ac10b0c0618bd99dcf95b1e687"}, + {file = "pydantic-1.10.13.tar.gz", hash = "sha256:32c8b48dcd3b2ac4e78b0ba4af3a2c2eb6048cb75202f0ea7b34feb740efc340"}, ] [package.dependencies] -typing-extensions = ">=3.7.4.3" +typing-extensions = ">=4.2.0" [package.extras] dotenv = ["python-dotenv (>=0.10.4)"] @@ -1309,6 +1692,44 @@ typing-extensions = ">=3.10,<4.6.0 || >4.6.0" [package.extras] dev = ["Sphinx", "black", "build", "coverage", "docformatter", "flake8", "flake8-black", "flake8-bugbear", "flake8-isort", "furo", "importlib-metadata (<5)", "invoke", "isort", "mypy", "pylint", "pytest", "pytest-cov", "pytest-mypy-testing", "sphinx-autodoc-typehints", "tox", "twine", "wheel"] +[[package]] +name = "pygit2" +version = "1.13.3" +description = "Python bindings for libgit2." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pygit2-1.13.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a3053850e29c1e102b1ab759d90b0dcc6402d7a434cbe810cfd2792294cf0ba6"}, + {file = "pygit2-1.13.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d2461db082c27231e2565e24e7ec3d6a60c7ceea5cda7364cb6eb81a6aedebd"}, + {file = "pygit2-1.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2216dc34edbe44e37c5cabc5f1530266445b87c66038fc8b3e0e7be64b3d4edb"}, + {file = "pygit2-1.13.3-cp310-cp310-win32.whl", hash = "sha256:5bc8c173ead087a4200e8763fad92105b4c9d40d03e007b9d9bbe47793716d31"}, + {file = "pygit2-1.13.3-cp310-cp310-win_amd64.whl", hash = "sha256:c305adf3a86e02db8bcd89bb92e33e896a2ff36f58a5ad7ff15675491ab6a751"}, + {file = "pygit2-1.13.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8de720ca137624d8f98c8b8d57cdb1e461034adf3e158762ee5c3085620c8075"}, + {file = "pygit2-1.13.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a85521b8e218afd4111d5bd4e106d77ffaac7ecd9a1ed8c1eea9f8d9568d287"}, + {file = "pygit2-1.13.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7c5cb78cc0a88f5cb2910e85996f33fcac99994251117f00f2e344f84d2616a"}, + {file = "pygit2-1.13.3-cp311-cp311-win32.whl", hash = "sha256:81686e3e06132f23eab32c6718c21930e8adda795c2ca76617f7562bff7b6b66"}, + {file = "pygit2-1.13.3-cp311-cp311-win_amd64.whl", hash = "sha256:93cc7ffb403688c2ec3e169096f34b2b99bc709adc54487e9d11165cfd070948"}, + {file = "pygit2-1.13.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:07b17f766c88ce1d05d264b5819e75ad261f3b60e33e4105a71f02467d0f6d39"}, + {file = "pygit2-1.13.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81bfe9ca394cdc896b632f18cd5f9c656a5f6c03c61deb1570b9081f2406776b"}, + {file = "pygit2-1.13.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c83e6e5ac357a9e87698c1eb25f430846f208bce12362d2209e7c9ac214e00c"}, + {file = "pygit2-1.13.3-cp312-cp312-win32.whl", hash = "sha256:de481a2cee7ef98143109bd9d2b30690022aeb8ba849feeba082a3b48a53c214"}, + {file = "pygit2-1.13.3-cp312-cp312-win_amd64.whl", hash = "sha256:0353b55f8bed1742dab15083ee9ee508ed91feb5c2563e2a612af277317030c6"}, + {file = "pygit2-1.13.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:60152bb30bc2ab880d3c82f113be33aac7ced571d1148c51720ccefff9dfc9ce"}, + {file = "pygit2-1.13.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcf827ebe392e2181a50ebaf724947e30a1da076a74d8a6f9cec784158faced1"}, + {file = "pygit2-1.13.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73cb821acc5cc8ad62a96154d030ff47127073b56f71157e7c65b2e7ebb4d52f"}, + {file = "pygit2-1.13.3-cp38-cp38-win32.whl", hash = "sha256:112c4efd421c3c8b4bb4406d3cd4a3a6a18a925c1f8b08d8a6dd0b591c6c6049"}, + {file = "pygit2-1.13.3-cp38-cp38-win_amd64.whl", hash = "sha256:21f73fd2863b6b21b4fbfed11a42af0ac1036472bb3716944b42a9719beaf07e"}, + {file = "pygit2-1.13.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e94f2598bdf68340609bb21fd4d21213812913b40b73e5fcba67f4fb01f4fba4"}, + {file = "pygit2-1.13.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5846aadb7b72802b3e4cb981f956965e92bc1692e7514ff4491bd7e24b20b358"}, + {file = "pygit2-1.13.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2823d097b103740d52e600ef2079e23966583edbde08ac122279f1ab3b2c3979"}, + {file = "pygit2-1.13.3-cp39-cp39-win32.whl", hash = "sha256:72fda35f88a3f5549eb9683c47ac73a6b674df943fc2490d93d539b46f518cbd"}, + {file = "pygit2-1.13.3-cp39-cp39-win_amd64.whl", hash = "sha256:2087fd130181e4ba6b8599fcd406920781555a52a3f142bd1dec4de21b9c5792"}, + {file = "pygit2-1.13.3.tar.gz", hash = "sha256:0257c626011e4afb99bdb20875443f706f84201d4c92637f02215b98eac13ded"}, +] + +[package.dependencies] +cffi = ">=1.16.0" + [[package]] name = "pygithub" version = "1.59.1" @@ -1328,17 +1749,67 @@ requests = ">=2.14.0" [[package]] name = "pygments" -version = "2.16.1" +version = "2.17.2" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.7" files = [ - {file = "Pygments-2.16.1-py3-none-any.whl", hash = "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692"}, - {file = "Pygments-2.16.1.tar.gz", hash = "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"}, + {file = "pygments-2.17.2-py3-none-any.whl", hash = "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c"}, + {file = "pygments-2.17.2.tar.gz", hash = "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"}, ] [package.extras] plugins = ["importlib-metadata"] +windows-terminal = ["colorama (>=0.4.6)"] + +[[package]] +name = "pyinstaller" +version = "6.3.0" +description = "PyInstaller bundles a Python application and all its dependencies into a single package." +optional = false +python-versions = "<3.13,>=3.8" +files = [ + {file = "pyinstaller-6.3.0-py3-none-macosx_10_13_universal2.whl", hash = "sha256:75a6f2a6f835a2e6e0899d10e60c10caf5defd25aced38b1dd48fbbabc89de07"}, + {file = "pyinstaller-6.3.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:de25beb176f73a944758553caacec46cc665bf3910ad8a174706d79cf6e95340"}, + {file = "pyinstaller-6.3.0-py3-none-manylinux2014_i686.whl", hash = "sha256:e436fcc0ea87c3f132baac916d508c24c84a8f6d8a06c3154fbc753f169b76c7"}, + {file = "pyinstaller-6.3.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:b721d793a33b6d9946c7dd95d3ea7589c0424b51cf1b9fe580f03c544f1336b2"}, + {file = "pyinstaller-6.3.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:96c37a1ee5b2fd5bb25c098ef510661d6d17b6515d0b86d8fc93727dd2475ba3"}, + {file = "pyinstaller-6.3.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:abe91106a3bbccc3f3a27af4325676ecdb6f46cb842ac663625002a870fc503b"}, + {file = "pyinstaller-6.3.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:41c937fe8f07ae02009b3b5a96ac3eb0800a4f8a97af142d4100060fe2135bb9"}, + {file = "pyinstaller-6.3.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:886b3b995b674905a20ad5b720b47cc395897d7b391117831027a4c8c5d67a58"}, + {file = "pyinstaller-6.3.0-py3-none-win32.whl", hash = "sha256:0597fb04337695e5cc5250253e0655530bf14f264b7a5b7d219cc65f6889c4bd"}, + {file = "pyinstaller-6.3.0-py3-none-win_amd64.whl", hash = "sha256:156b32ba943e0090bcc68e40ae1cb68fd92b7f1ab6fe0bdf8faf3d3cfc4e12dd"}, + {file = "pyinstaller-6.3.0-py3-none-win_arm64.whl", hash = "sha256:1eadbd1fae84e2e6c678d8b4ed6a232ec5c8fe3a839aea5a3071c4c0282f98cc"}, + {file = "pyinstaller-6.3.0.tar.gz", hash = "sha256:914d4c96cc99472e37ac552fdd82fbbe09e67bb592d0717fcffaa99ea74273df"}, +] + +[package.dependencies] +altgraph = "*" +macholib = {version = ">=1.8", markers = "sys_platform == \"darwin\""} +packaging = ">=22.0" +pefile = {version = ">=2022.5.30", markers = "sys_platform == \"win32\""} +pyinstaller-hooks-contrib = ">=2021.4" +pywin32-ctypes = {version = ">=0.2.1", markers = "sys_platform == \"win32\""} +setuptools = ">=42.0.0" + +[package.extras] +completion = ["argcomplete"] +hook-testing = ["execnet (>=1.5.0)", "psutil", "pytest (>=2.7.3)"] + +[[package]] +name = "pyinstaller-hooks-contrib" +version = "2023.12" +description = "Community maintained hooks for PyInstaller" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyinstaller-hooks-contrib-2023.12.tar.gz", hash = "sha256:11a9d59d903723dd693e8c10b054f3ea1ecad390623c9fa527c731d715fc5b3f"}, + {file = "pyinstaller_hooks_contrib-2023.12-py2.py3-none-any.whl", hash = "sha256:6a601a0d783fa725327fc6ac712779475dc8979f639419c7fcd460dd8d0a6d2a"}, +] + +[package.dependencies] +packaging = ">=22.0" +setuptools = ">=42.0.0" [[package]] name = "pyjwt" @@ -1430,13 +1901,13 @@ testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtuale [[package]] name = "pytest-mock" -version = "3.11.1" +version = "3.12.0" description = "Thin-wrapper around the mock package for easier use with pytest" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pytest-mock-3.11.1.tar.gz", hash = "sha256:7f6b125602ac6d743e523ae0bfa71e1a697a2f5534064528c6ff84c2f7c2fc7f"}, - {file = "pytest_mock-3.11.1-py3-none-any.whl", hash = "sha256:21c279fff83d70763b05f8874cc9cfb3fcacd6d354247a976f9529d19f9acf39"}, + {file = "pytest-mock-3.12.0.tar.gz", hash = "sha256:31a40f038c22cad32287bb43932054451ff5583ff094bca6f675df2f8bc1a6e9"}, + {file = "pytest_mock-3.12.0-py3-none-any.whl", hash = "sha256:0972719a7263072da3a21c7f4773069bcc7486027d7e8e1f81d98a47e701bc4f"}, ] [package.dependencies] @@ -1459,25 +1930,49 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "pytz" +version = "2023.3.post1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2023.3.post1-py2.py3-none-any.whl", hash = "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7"}, + {file = "pytz-2023.3.post1.tar.gz", hash = "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b"}, +] + [[package]] name = "pywin32" -version = "227" +version = "306" description = "Python for Window Extensions" optional = false python-versions = "*" files = [ - {file = "pywin32-227-cp27-cp27m-win32.whl", hash = "sha256:371fcc39416d736401f0274dd64c2302728c9e034808e37381b5e1b22be4a6b0"}, - {file = "pywin32-227-cp27-cp27m-win_amd64.whl", hash = "sha256:4cdad3e84191194ea6d0dd1b1b9bdda574ff563177d2adf2b4efec2a244fa116"}, - {file = "pywin32-227-cp35-cp35m-win32.whl", hash = "sha256:f4c5be1a293bae0076d93c88f37ee8da68136744588bc5e2be2f299a34ceb7aa"}, - {file = "pywin32-227-cp35-cp35m-win_amd64.whl", hash = "sha256:a929a4af626e530383a579431b70e512e736e9588106715215bf685a3ea508d4"}, - {file = "pywin32-227-cp36-cp36m-win32.whl", hash = "sha256:300a2db938e98c3e7e2093e4491439e62287d0d493fe07cce110db070b54c0be"}, - {file = "pywin32-227-cp36-cp36m-win_amd64.whl", hash = "sha256:9b31e009564fb95db160f154e2aa195ed66bcc4c058ed72850d047141b36f3a2"}, - {file = "pywin32-227-cp37-cp37m-win32.whl", hash = "sha256:47a3c7551376a865dd8d095a98deba954a98f326c6fe3c72d8726ca6e6b15507"}, - {file = "pywin32-227-cp37-cp37m-win_amd64.whl", hash = "sha256:31f88a89139cb2adc40f8f0e65ee56a8c585f629974f9e07622ba80199057511"}, - {file = "pywin32-227-cp38-cp38-win32.whl", hash = "sha256:7f18199fbf29ca99dff10e1f09451582ae9e372a892ff03a28528a24d55875bc"}, - {file = "pywin32-227-cp38-cp38-win_amd64.whl", hash = "sha256:7c1ae32c489dc012930787f06244426f8356e129184a02c25aef163917ce158e"}, - {file = "pywin32-227-cp39-cp39-win32.whl", hash = "sha256:c054c52ba46e7eb6b7d7dfae4dbd987a1bb48ee86debe3f245a2884ece46e295"}, - {file = "pywin32-227-cp39-cp39-win_amd64.whl", hash = "sha256:f27cec5e7f588c3d1051651830ecc00294f90728d19c3bf6916e6dba93ea357c"}, + {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"}, + {file = "pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8"}, + {file = "pywin32-306-cp311-cp311-win32.whl", hash = "sha256:e65028133d15b64d2ed8f06dd9fbc268352478d4f9289e69c190ecd6818b6407"}, + {file = "pywin32-306-cp311-cp311-win_amd64.whl", hash = "sha256:a7639f51c184c0272e93f244eb24dafca9b1855707d94c192d4a0b4c01e1100e"}, + {file = "pywin32-306-cp311-cp311-win_arm64.whl", hash = "sha256:70dba0c913d19f942a2db25217d9a1b726c278f483a919f1abfed79c9cf64d3a"}, + {file = "pywin32-306-cp312-cp312-win32.whl", hash = "sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b"}, + {file = "pywin32-306-cp312-cp312-win_amd64.whl", hash = "sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e"}, + {file = "pywin32-306-cp312-cp312-win_arm64.whl", hash = "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040"}, + {file = "pywin32-306-cp37-cp37m-win32.whl", hash = "sha256:1c73ea9a0d2283d889001998059f5eaaba3b6238f767c9cf2833b13e6a685f65"}, + {file = "pywin32-306-cp37-cp37m-win_amd64.whl", hash = "sha256:72c5f621542d7bdd4fdb716227be0dd3f8565c11b280be6315b06ace35487d36"}, + {file = "pywin32-306-cp38-cp38-win32.whl", hash = "sha256:e4c092e2589b5cf0d365849e73e02c391c1349958c5ac3e9d5ccb9a28e017b3a"}, + {file = "pywin32-306-cp38-cp38-win_amd64.whl", hash = "sha256:e8ac1ae3601bee6ca9f7cb4b5363bf1c0badb935ef243c4733ff9a393b1690c0"}, + {file = "pywin32-306-cp39-cp39-win32.whl", hash = "sha256:e25fd5b485b55ac9c057f67d94bc203f3f6595078d1fb3b458c9c28b7153a802"}, + {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, +] + +[[package]] +name = "pywin32-ctypes" +version = "0.2.2" +description = "A (partial) reimplementation of pywin32 using ctypes/cffi" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pywin32-ctypes-0.2.2.tar.gz", hash = "sha256:3426e063bdd5fd4df74a14fa3cf80a0b42845a87e1d1e81f6549f9daec593a60"}, + {file = "pywin32_ctypes-0.2.2-py3-none-any.whl", hash = "sha256:bf490a1a709baf35d688fe0ecf980ed4de11d2b3e37b51e5442587a75d9957e7"}, ] [[package]] @@ -1492,6 +1987,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1499,8 +1995,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1517,6 +2020,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1524,27 +2028,42 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, ] +[[package]] +name = "readchar" +version = "4.0.5" +description = "Library to easily read single chars and key strokes" +optional = false +python-versions = ">=3.7" +files = [ + {file = "readchar-4.0.5-py3-none-any.whl", hash = "sha256:76ec784a5dd2afac3b7da8003329834cdd9824294c260027f8c8d2e4d0a78f43"}, + {file = "readchar-4.0.5.tar.gz", hash = "sha256:08a456c2d7c1888cde3f4688b542621b676eb38cd6cfed7eb6cb2e2905ddc826"}, +] + +[package.dependencies] +setuptools = ">=41.0" + [[package]] name = "requests" -version = "2.31.0" +version = "2.28.2" description = "Python HTTP for Humans." optional = false -python-versions = ">=3.7" +python-versions = ">=3.7, <4" files = [ - {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, - {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, + {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, + {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, ] [package.dependencies] certifi = ">=2017.4.17" charset-normalizer = ">=2,<4" idna = ">=2.5,<4" -urllib3 = ">=1.21.1,<3" +urllib3 = ">=1.21.1,<1.27" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] @@ -1552,22 +2071,21 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "rich" -version = "11.2.0" +version = "13.7.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false -python-versions = ">=3.6.2,<4.0.0" +python-versions = ">=3.7.0" files = [ - {file = "rich-11.2.0-py3-none-any.whl", hash = "sha256:d5f49ad91fb343efcae45a2b2df04a9755e863e50413623ab8c9e74f05aee52b"}, - {file = "rich-11.2.0.tar.gz", hash = "sha256:1a6266a5738115017bb64a66c59c717e7aa047b3ae49a011ede4abdeffc6536e"}, + {file = "rich-13.7.0-py3-none-any.whl", hash = "sha256:6da14c108c4866ee9520bbffa71f6fe3962e193b7da68720583850cd4548e235"}, + {file = "rich-13.7.0.tar.gz", hash = "sha256:5cb5123b5cf9ee70584244246816e9114227e0b98ad9176eede6ad54bf5403fa"}, ] [package.dependencies] -colorama = ">=0.4.0,<0.5.0" -commonmark = ">=0.9.0,<0.10.0" -pygments = ">=2.6.0,<3.0.0" +markdown-it-py = ">=2.2.0" +pygments = ">=2.13.0,<3.0.0" [package.extras] -jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] +jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "rsa" @@ -1583,26 +2101,86 @@ files = [ [package.dependencies] pyasn1 = ">=0.1.3" +[[package]] +name = "ruff" +version = "0.1.13" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.1.13-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:e3fd36e0d48aeac672aa850045e784673449ce619afc12823ea7868fcc41d8ba"}, + {file = "ruff-0.1.13-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9fb6b3b86450d4ec6a6732f9f60c4406061b6851c4b29f944f8c9d91c3611c7a"}, + {file = "ruff-0.1.13-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b13ba5d7156daaf3fd08b6b993360a96060500aca7e307d95ecbc5bb47a69296"}, + {file = "ruff-0.1.13-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9ebb40442f7b531e136d334ef0851412410061e65d61ca8ce90d894a094feb22"}, + {file = "ruff-0.1.13-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:226b517f42d59a543d6383cfe03cccf0091e3e0ed1b856c6824be03d2a75d3b6"}, + {file = "ruff-0.1.13-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:5f0312ba1061e9b8c724e9a702d3c8621e3c6e6c2c9bd862550ab2951ac75c16"}, + {file = "ruff-0.1.13-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2f59bcf5217c661254bd6bc42d65a6fd1a8b80c48763cb5c2293295babd945dd"}, + {file = "ruff-0.1.13-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e6894b00495e00c27b6ba61af1fc666f17de6140345e5ef27dd6e08fb987259d"}, + {file = "ruff-0.1.13-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a1600942485c6e66119da294c6294856b5c86fd6df591ce293e4a4cc8e72989"}, + {file = "ruff-0.1.13-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ee3febce7863e231a467f90e681d3d89210b900d49ce88723ce052c8761be8c7"}, + {file = "ruff-0.1.13-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:dcaab50e278ff497ee4d1fe69b29ca0a9a47cd954bb17963628fa417933c6eb1"}, + {file = "ruff-0.1.13-py3-none-musllinux_1_2_i686.whl", hash = "sha256:f57de973de4edef3ad3044d6a50c02ad9fc2dff0d88587f25f1a48e3f72edf5e"}, + {file = "ruff-0.1.13-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:7a36fa90eb12208272a858475ec43ac811ac37e91ef868759770b71bdabe27b6"}, + {file = "ruff-0.1.13-py3-none-win32.whl", hash = "sha256:a623349a505ff768dad6bd57087e2461be8db58305ebd5577bd0e98631f9ae69"}, + {file = "ruff-0.1.13-py3-none-win_amd64.whl", hash = "sha256:f988746e3c3982bea7f824c8fa318ce7f538c4dfefec99cd09c8770bd33e6539"}, + {file = "ruff-0.1.13-py3-none-win_arm64.whl", hash = "sha256:6bbbc3042075871ec17f28864808540a26f0f79a4478c357d3e3d2284e832998"}, + {file = "ruff-0.1.13.tar.gz", hash = "sha256:e261f1baed6291f434ffb1d5c6bd8051d1c2a26958072d38dfbec39b3dda7352"}, +] + +[[package]] +name = "runs" +version = "1.2.0" +description = "🏃 Run a block of text as a subprocess 🏃" +optional = false +python-versions = ">=3.8" +files = [ + {file = "runs-1.2.0-py3-none-any.whl", hash = "sha256:ec6fe3b24dfa20c5c4e5c4806d3b35bb880aad0e787a8610913c665c5a7cc07c"}, + {file = "runs-1.2.0.tar.gz", hash = "sha256:8804271011b7a2eeb0d77c3e3f556e5ce5f602fa0dd2a31ed0c1222893be69b7"}, +] + +[package.dependencies] +xmod = "*" + +[[package]] +name = "segment-analytics-python" +version = "2.2.3" +description = "The hassle-free way to integrate analytics into any python application." +optional = false +python-versions = ">=3.6.0" +files = [ + {file = "segment-analytics-python-2.2.3.tar.gz", hash = "sha256:0df5908e3df74b4482f33392fdd450df4c8351bf54974376fbe6bf33b0700865"}, + {file = "segment_analytics_python-2.2.3-py2.py3-none-any.whl", hash = "sha256:06cc3d8e79103f02c3878ec66cb66152415473d0d2a142b98a0ee18da972e109"}, +] + +[package.dependencies] +backoff = ">=2.1,<3.0" +monotonic = ">=1.5,<2.0" +python-dateutil = ">=2.2,<3.0" +requests = ">=2.7,<3.0" + +[package.extras] +test = ["flake8 (==3.7.9)", "mock (==2.0.0)", "pylint (==2.8.0)"] + [[package]] name = "semver" -version = "3.0.1" +version = "3.0.2" description = "Python helper for Semantic Versioning (https://semver.org)" optional = false python-versions = ">=3.7" files = [ - {file = "semver-3.0.1-py3-none-any.whl", hash = "sha256:2a23844ba1647362c7490fe3995a86e097bb590d16f0f32dfc383008f19e4cdf"}, - {file = "semver-3.0.1.tar.gz", hash = "sha256:9ec78c5447883c67b97f98c3b6212796708191d22e4ad30f4570f840171cbce1"}, + {file = "semver-3.0.2-py3-none-any.whl", hash = "sha256:b1ea4686fe70b981f85359eda33199d60c53964284e0cfb4977d243e37cf4bf4"}, + {file = "semver-3.0.2.tar.gz", hash = "sha256:6253adb39c70f6e51afed2fa7152bcd414c411286088fb4b9effb133885ab4cc"}, ] [[package]] name = "sentry-sdk" -version = "1.29.2" +version = "1.39.2" description = "Python client for Sentry (https://sentry.io)" optional = false python-versions = "*" files = [ - {file = "sentry-sdk-1.29.2.tar.gz", hash = "sha256:a99ee105384788c3f228726a88baf515fe7b5f1d2d0f215a03d194369f158df7"}, - {file = "sentry_sdk-1.29.2-py2.py3-none-any.whl", hash = "sha256:3e17215d8006612e2df02b0e73115eb8376c37e3f586d8436fa41644e605074d"}, + {file = "sentry-sdk-1.39.2.tar.gz", hash = "sha256:24c83b0b41c887d33328a9166f5950dc37ad58f01c9f2fbff6b87a6f1094170c"}, + {file = "sentry_sdk-1.39.2-py2.py3-none-any.whl", hash = "sha256:acaf597b30258fc7663063b291aa99e58f3096e91fe1e6634f4b79f9c1943e8e"}, ] [package.dependencies] @@ -1612,10 +2190,12 @@ urllib3 = {version = ">=1.26.11", markers = "python_version >= \"3.6\""} [package.extras] aiohttp = ["aiohttp (>=3.5)"] arq = ["arq (>=0.23)"] +asyncpg = ["asyncpg (>=0.23)"] beam = ["apache-beam (>=2.12)"] bottle = ["bottle (>=0.12.13)"] celery = ["celery (>=3)"] chalice = ["chalice (>=1.16.0)"] +clickhouse-driver = ["clickhouse-driver (>=0.2.0)"] django = ["django (>=1.8)"] falcon = ["falcon (>=1.4)"] fastapi = ["fastapi (>=0.79.0)"] @@ -1625,7 +2205,8 @@ httpx = ["httpx (>=0.16.0)"] huey = ["huey (>=2)"] loguru = ["loguru (>=0.5)"] opentelemetry = ["opentelemetry-distro (>=0.35b0)"] -pure-eval = ["asttokens", "executing", "pure-eval"] +opentelemetry-experimental = ["opentelemetry-distro (>=0.40b0,<1.0)", "opentelemetry-instrumentation-aiohttp-client (>=0.40b0,<1.0)", "opentelemetry-instrumentation-django (>=0.40b0,<1.0)", "opentelemetry-instrumentation-fastapi (>=0.40b0,<1.0)", "opentelemetry-instrumentation-flask (>=0.40b0,<1.0)", "opentelemetry-instrumentation-requests (>=0.40b0,<1.0)", "opentelemetry-instrumentation-sqlite3 (>=0.40b0,<1.0)", "opentelemetry-instrumentation-urllib (>=0.40b0,<1.0)"] +pure-eval = ["asttokens", "executing", "pure_eval"] pymongo = ["pymongo (>=3.1)"] pyspark = ["pyspark (>=2.4.4)"] quart = ["blinker (>=1.1)", "quart (>=0.16.1)"] @@ -1636,6 +2217,33 @@ starlette = ["starlette (>=0.19.1)"] starlite = ["starlite (>=1.48)"] tornado = ["tornado (>=5)"] +[[package]] +name = "setuptools" +version = "69.0.3" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.0.3-py3-none-any.whl", hash = "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05"}, + {file = "setuptools-69.0.3.tar.gz", hash = "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "simpleeval" +version = "0.9.13" +description = "A simple, safe single expression evaluator library." +optional = false +python-versions = "*" +files = [ + {file = "simpleeval-0.9.13-py2.py3-none-any.whl", hash = "sha256:22a2701a5006e4188d125d34accf2405c2c37c93f6b346f2484b6422415ae54a"}, + {file = "simpleeval-0.9.13.tar.gz", hash = "sha256:4a30f9cc01825fe4c719c785e3762623e350c4840d5e6855c2a8496baaa65fac"}, +] + [[package]] name = "six" version = "1.16.0" @@ -1649,13 +2257,13 @@ files = [ [[package]] name = "smmap" -version = "5.0.0" +version = "5.0.1" description = "A pure Python implementation of a sliding window memory map manager" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "smmap-5.0.0-py3-none-any.whl", hash = "sha256:2aba19d6a040e78d8b09de5c57e96207b09ed71d8e55ce0959eeee6c8e190d94"}, - {file = "smmap-5.0.0.tar.gz", hash = "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936"}, + {file = "smmap-5.0.1-py3-none-any.whl", hash = "sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da"}, + {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, ] [[package]] @@ -1669,20 +2277,6 @@ files = [ {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, ] -[[package]] -name = "tabulate" -version = "0.8.10" -description = "Pretty-print tabular data" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -files = [ - {file = "tabulate-0.8.10-py3-none-any.whl", hash = "sha256:0ba055423dbaa164b9e456abe7920c5e8ed33fcc16f6d1b2f2d152c8e1e8b4fc"}, - {file = "tabulate-0.8.10.tar.gz", hash = "sha256:6c57f3f3dd7ac2782770155f3adb2db0b1a269637e42f27599925e64b114f519"}, -] - -[package.extras] -widechars = ["wcwidth"] - [[package]] name = "toml" version = "0.10.2" @@ -1707,212 +2301,256 @@ files = [ [[package]] name = "typing-extensions" -version = "4.7.1" -description = "Backported and Experimental Type Hints for Python 3.7+" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, - {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "tzdata" +version = "2023.4" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2023.4-py2.py3-none-any.whl", hash = "sha256:aa3ace4329eeacda5b7beb7ea08ece826c28d761cda36e747cfbf97996d39bf3"}, + {file = "tzdata-2023.4.tar.gz", hash = "sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9"}, ] [[package]] name = "urllib3" -version = "1.26.16" +version = "1.26.18" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ - {file = "urllib3-1.26.16-py2.py3-none-any.whl", hash = "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f"}, - {file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"}, + {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, + {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +[[package]] +name = "wcwidth" +version = "0.2.13" +description = "Measures the displayed width of unicode strings in a terminal" +optional = false +python-versions = "*" +files = [ + {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, + {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, +] + [[package]] name = "websocket-client" -version = "1.6.1" +version = "1.7.0" description = "WebSocket client for Python with low level API options" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "websocket-client-1.6.1.tar.gz", hash = "sha256:c951af98631d24f8df89ab1019fc365f2227c0892f12fd150e935607c79dd0dd"}, - {file = "websocket_client-1.6.1-py3-none-any.whl", hash = "sha256:f1f9f2ad5291f0225a49efad77abf9e700b6fef553900623060dad6e26503b9d"}, + {file = "websocket-client-1.7.0.tar.gz", hash = "sha256:10e511ea3a8c744631d3bd77e61eb17ed09304c413ad42cf6ddfa4c7787e8fe6"}, + {file = "websocket_client-1.7.0-py3-none-any.whl", hash = "sha256:f4c3d22fec12a2461427a29957ff07d35098ee2d976d3ba244e688b8b4057588"}, ] [package.extras] -docs = ["Sphinx (>=3.4)", "sphinx-rtd-theme (>=0.5)"] +docs = ["Sphinx (>=6.0)", "sphinx-rtd-theme (>=1.1.0)"] optional = ["python-socks", "wsaccel"] test = ["websockets"] [[package]] name = "wrapt" -version = "1.15.0" +version = "1.16.0" description = "Module for decorators, wrappers and monkey patching." optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -files = [ - {file = "wrapt-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a"}, - {file = "wrapt-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923"}, - {file = "wrapt-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975"}, - {file = "wrapt-1.15.0-cp310-cp310-win32.whl", hash = "sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1"}, - {file = "wrapt-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e"}, - {file = "wrapt-1.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7"}, - {file = "wrapt-1.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98"}, - {file = "wrapt-1.15.0-cp311-cp311-win32.whl", hash = "sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416"}, - {file = "wrapt-1.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248"}, - {file = "wrapt-1.15.0-cp35-cp35m-win32.whl", hash = "sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559"}, - {file = "wrapt-1.15.0-cp35-cp35m-win_amd64.whl", hash = "sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639"}, - {file = "wrapt-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2"}, - {file = "wrapt-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1"}, - {file = "wrapt-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420"}, - {file = "wrapt-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653"}, - {file = "wrapt-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0"}, - {file = "wrapt-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e"}, - {file = "wrapt-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145"}, - {file = "wrapt-1.15.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7"}, - {file = "wrapt-1.15.0-cp38-cp38-win32.whl", hash = "sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b"}, - {file = "wrapt-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1"}, - {file = "wrapt-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86"}, - {file = "wrapt-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9"}, - {file = "wrapt-1.15.0-cp39-cp39-win32.whl", hash = "sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff"}, - {file = "wrapt-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6"}, - {file = "wrapt-1.15.0-py3-none-any.whl", hash = "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640"}, - {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[[package]] +name = "xmod" +version = "1.8.1" +description = "🌱 Turn any object into a module 🌱" +optional = false +python-versions = ">=3.8" +files = [ + {file = "xmod-1.8.1-py3-none-any.whl", hash = "sha256:a24e9458a4853489042522bdca9e50ee2eac5ab75c809a91150a8a7f40670d48"}, + {file = "xmod-1.8.1.tar.gz", hash = "sha256:38c76486b9d672c546d57d8035df0beb7f4a9b088bc3fb2de5431ae821444377"}, ] [[package]] name = "yarl" -version = "1.9.2" +version = "1.9.4" description = "Yet another URL library" optional = false python-versions = ">=3.7" files = [ - {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c2ad583743d16ddbdf6bb14b5cd76bf43b0d0006e918809d5d4ddf7bde8dd82"}, - {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:82aa6264b36c50acfb2424ad5ca537a2060ab6de158a5bd2a72a032cc75b9eb8"}, - {file = "yarl-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c0c77533b5ed4bcc38e943178ccae29b9bcf48ffd1063f5821192f23a1bd27b9"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee4afac41415d52d53a9833ebae7e32b344be72835bbb589018c9e938045a560"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9bf345c3a4f5ba7f766430f97f9cc1320786f19584acc7086491f45524a551ac"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2a96c19c52ff442a808c105901d0bdfd2e28575b3d5f82e2f5fd67e20dc5f4ea"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:891c0e3ec5ec881541f6c5113d8df0315ce5440e244a716b95f2525b7b9f3608"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c3a53ba34a636a256d767c086ceb111358876e1fb6b50dfc4d3f4951d40133d5"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:566185e8ebc0898b11f8026447eacd02e46226716229cea8db37496c8cdd26e0"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:2b0738fb871812722a0ac2154be1f049c6223b9f6f22eec352996b69775b36d4"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:32f1d071b3f362c80f1a7d322bfd7b2d11e33d2adf395cc1dd4df36c9c243095"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:e9fdc7ac0d42bc3ea78818557fab03af6181e076a2944f43c38684b4b6bed8e3"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:56ff08ab5df8429901ebdc5d15941b59f6253393cb5da07b4170beefcf1b2528"}, - {file = "yarl-1.9.2-cp310-cp310-win32.whl", hash = "sha256:8ea48e0a2f931064469bdabca50c2f578b565fc446f302a79ba6cc0ee7f384d3"}, - {file = "yarl-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:50f33040f3836e912ed16d212f6cc1efb3231a8a60526a407aeb66c1c1956dde"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:646d663eb2232d7909e6601f1a9107e66f9791f290a1b3dc7057818fe44fc2b6"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:aff634b15beff8902d1f918012fc2a42e0dbae6f469fce134c8a0dc51ca423bb"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a83503934c6273806aed765035716216cc9ab4e0364f7f066227e1aaea90b8d0"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b25322201585c69abc7b0e89e72790469f7dad90d26754717f3310bfe30331c2"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:22a94666751778629f1ec4280b08eb11815783c63f52092a5953faf73be24191"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ec53a0ea2a80c5cd1ab397925f94bff59222aa3cf9c6da938ce05c9ec20428d"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:159d81f22d7a43e6eabc36d7194cb53f2f15f498dbbfa8edc8a3239350f59fe7"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:832b7e711027c114d79dffb92576acd1bd2decc467dec60e1cac96912602d0e6"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:95d2ecefbcf4e744ea952d073c6922e72ee650ffc79028eb1e320e732898d7e8"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d4e2c6d555e77b37288eaf45b8f60f0737c9efa3452c6c44626a5455aeb250b9"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:783185c75c12a017cc345015ea359cc801c3b29a2966c2655cd12b233bf5a2be"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:b8cc1863402472f16c600e3e93d542b7e7542a540f95c30afd472e8e549fc3f7"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:822b30a0f22e588b32d3120f6d41e4ed021806418b4c9f0bc3048b8c8cb3f92a"}, - {file = "yarl-1.9.2-cp311-cp311-win32.whl", hash = "sha256:a60347f234c2212a9f0361955007fcf4033a75bf600a33c88a0a8e91af77c0e8"}, - {file = "yarl-1.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:be6b3fdec5c62f2a67cb3f8c6dbf56bbf3f61c0f046f84645cd1ca73532ea051"}, - {file = "yarl-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38a3928ae37558bc1b559f67410df446d1fbfa87318b124bf5032c31e3447b74"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac9bb4c5ce3975aeac288cfcb5061ce60e0d14d92209e780c93954076c7c4367"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3da8a678ca8b96c8606bbb8bfacd99a12ad5dd288bc6f7979baddd62f71c63ef"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13414591ff516e04fcdee8dc051c13fd3db13b673c7a4cb1350e6b2ad9639ad3"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf74d08542c3a9ea97bb8f343d4fcbd4d8f91bba5ec9d5d7f792dbe727f88938"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e7221580dc1db478464cfeef9b03b95c5852cc22894e418562997df0d074ccc"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:494053246b119b041960ddcd20fd76224149cfea8ed8777b687358727911dd33"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:52a25809fcbecfc63ac9ba0c0fb586f90837f5425edfd1ec9f3372b119585e45"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:e65610c5792870d45d7b68c677681376fcf9cc1c289f23e8e8b39c1485384185"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:1b1bba902cba32cdec51fca038fd53f8beee88b77efc373968d1ed021024cc04"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:662e6016409828ee910f5d9602a2729a8a57d74b163c89a837de3fea050c7582"}, - {file = "yarl-1.9.2-cp37-cp37m-win32.whl", hash = "sha256:f364d3480bffd3aa566e886587eaca7c8c04d74f6e8933f3f2c996b7f09bee1b"}, - {file = "yarl-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6a5883464143ab3ae9ba68daae8e7c5c95b969462bbe42e2464d60e7e2698368"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5610f80cf43b6202e2c33ba3ec2ee0a2884f8f423c8f4f62906731d876ef4fac"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b9a4e67ad7b646cd6f0938c7ebfd60e481b7410f574c560e455e938d2da8e0f4"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:83fcc480d7549ccebe9415d96d9263e2d4226798c37ebd18c930fce43dfb9574"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fcd436ea16fee7d4207c045b1e340020e58a2597301cfbcfdbe5abd2356c2fb"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84e0b1599334b1e1478db01b756e55937d4614f8654311eb26012091be109d59"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3458a24e4ea3fd8930e934c129b676c27452e4ebda80fbe47b56d8c6c7a63a9e"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:838162460b3a08987546e881a2bfa573960bb559dfa739e7800ceeec92e64417"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f4e2d08f07a3d7d3e12549052eb5ad3eab1c349c53ac51c209a0e5991bbada78"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:de119f56f3c5f0e2fb4dee508531a32b069a5f2c6e827b272d1e0ff5ac040333"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:149ddea5abf329752ea5051b61bd6c1d979e13fbf122d3a1f9f0c8be6cb6f63c"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:674ca19cbee4a82c9f54e0d1eee28116e63bc6fd1e96c43031d11cbab8b2afd5"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:9b3152f2f5677b997ae6c804b73da05a39daa6a9e85a512e0e6823d81cdad7cc"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5415d5a4b080dc9612b1b63cba008db84e908b95848369aa1da3686ae27b6d2b"}, - {file = "yarl-1.9.2-cp38-cp38-win32.whl", hash = "sha256:f7a3d8146575e08c29ed1cd287068e6d02f1c7bdff8970db96683b9591b86ee7"}, - {file = "yarl-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:63c48f6cef34e6319a74c727376e95626f84ea091f92c0250a98e53e62c77c72"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:75df5ef94c3fdc393c6b19d80e6ef1ecc9ae2f4263c09cacb178d871c02a5ba9"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c027a6e96ef77d401d8d5a5c8d6bc478e8042f1e448272e8d9752cb0aff8b5c8"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3b078dbe227f79be488ffcfc7a9edb3409d018e0952cf13f15fd6512847f3f7"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59723a029760079b7d991a401386390c4be5bfec1e7dd83e25a6a0881859e716"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b03917871bf859a81ccb180c9a2e6c1e04d2f6a51d953e6a5cdd70c93d4e5a2a"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c1012fa63eb6c032f3ce5d2171c267992ae0c00b9e164efe4d73db818465fac3"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a74dcbfe780e62f4b5a062714576f16c2f3493a0394e555ab141bf0d746bb955"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c56986609b057b4839968ba901944af91b8e92f1725d1a2d77cbac6972b9ed1"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2c315df3293cd521033533d242d15eab26583360b58f7ee5d9565f15fee1bef4"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b7232f8dfbd225d57340e441d8caf8652a6acd06b389ea2d3222b8bc89cbfca6"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:53338749febd28935d55b41bf0bcc79d634881195a39f6b2f767870b72514caf"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:066c163aec9d3d073dc9ffe5dd3ad05069bcb03fcaab8d221290ba99f9f69ee3"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8288d7cd28f8119b07dd49b7230d6b4562f9b61ee9a4ab02221060d21136be80"}, - {file = "yarl-1.9.2-cp39-cp39-win32.whl", hash = "sha256:b124e2a6d223b65ba8768d5706d103280914d61f5cae3afbc50fc3dfcc016623"}, - {file = "yarl-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:61016e7d582bc46a5378ffdd02cd0314fb8ba52f40f9cf4d9a5e7dbef88dee18"}, - {file = "yarl-1.9.2.tar.gz", hash = "sha256:04ab9d4b9f587c06d801c2abfe9317b77cdf996c65a90d5e84ecc45010823571"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541"}, + {file = "yarl-1.9.4-cp310-cp310-win32.whl", hash = "sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d"}, + {file = "yarl-1.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98"}, + {file = "yarl-1.9.4-cp311-cp311-win32.whl", hash = "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31"}, + {file = "yarl-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10"}, + {file = "yarl-1.9.4-cp312-cp312-win32.whl", hash = "sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7"}, + {file = "yarl-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984"}, + {file = "yarl-1.9.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434"}, + {file = "yarl-1.9.4-cp37-cp37m-win32.whl", hash = "sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749"}, + {file = "yarl-1.9.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3"}, + {file = "yarl-1.9.4-cp38-cp38-win32.whl", hash = "sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece"}, + {file = "yarl-1.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0"}, + {file = "yarl-1.9.4-cp39-cp39-win32.whl", hash = "sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575"}, + {file = "yarl-1.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15"}, + {file = "yarl-1.9.4-py3-none-any.whl", hash = "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad"}, + {file = "yarl-1.9.4.tar.gz", hash = "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf"}, ] [package.dependencies] @@ -1921,5 +2559,5 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" -python-versions = "^3.10" -content-hash = "2903a2c7f1a7ebc91ada22a4e6b73aa96e7195e0d3da72310254862e49c6f22b" +python-versions = "~3.10" +content-hash = "0c7f7c9e18637d2cf9402f22c71502916cd4a1938111dd78eb7874f2c061c1fe" diff --git a/airbyte-ci/connectors/pipelines/pyinstaller_hooks/hook-certifi.py b/airbyte-ci/connectors/pipelines/pyinstaller_hooks/hook-certifi.py new file mode 100644 index 000000000000..4ebdd023f9b9 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pyinstaller_hooks/hook-certifi.py @@ -0,0 +1,6 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from PyInstaller.utils.hooks import collect_data_files + +# Get the cacert.pem +datas = collect_data_files("certifi") diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 20001f87d260..ce6db7b3942e 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,25 +4,29 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "1.1.0" +version = "3.5.1" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] [tool.poetry.dependencies] -python = "^3.10" -dagger-io = "^0.6.4" +python = "~3.10" +dagger-io = "==0.9.6" asyncer = "^0.0.2" anyio = "^3.4.1" more-itertools = "^8.11.0" -docker = "^5.0.3" +docker = "^6.0.0" semver = "^3.0.1" airbyte-protocol-models = "*" -tabulate = "^0.8.9" jinja2 = "^3.0.2" -requests = "^2.28.2" -connector-ops = {path = "../connector_ops"} +requests = "2.28.2" # Pinned as the requests 2.29.0 version is not compatible with the docker package +airbyte-connectors-base-images = {path = "../base_images", develop = true} +connector-ops = {path = "../connector_ops", develop = true} toml = "^0.10.2" sentry-sdk = "^1.28.1" +segment-analytics-python = "^2.2.3" +pygit2 = "^1.13.1" +asyncclick = "^8.1.3.4" +certifi = "^2023.11.17" [tool.poetry.group.test.dependencies] pytest = "^6.2.5" @@ -32,7 +36,17 @@ pytest-mock = "^3.10.0" [tool.poetry.group.dev.dependencies] freezegun = "^1.2.2" pytest-cov = "^4.1.0" +pyinstaller = "^6.1.0" +poethepoet = "^0.24.2" +pytest = "^6.2.5" +pytest-mock = "^3.10.0" +mypy = "^1.7.1" +ruff = "^0.1.9" [tool.poetry.scripts] -airbyte-ci-internal = "pipelines.commands.airbyte_ci:airbyte_ci" -airbyte-ci = "pipelines.dagger_run:main" +airbyte-ci = "pipelines.cli.airbyte_ci:airbyte_ci" +airbyte-ci-dev = "pipelines.cli.airbyte_ci:airbyte_ci" + +[tool.poe.tasks.build-release-binary] +shell = "pyinstaller --additional-hooks-dir=pyinstaller_hooks --collect-all pipelines --collect-all beartype --collect-all dagger --hidden-import strawberry --name $ARTIFACT_NAME --onefile pipelines/cli/airbyte_ci.py" +args = [{name = "ARTIFACT_NAME", default="airbyte-ci", positional = true}] diff --git a/airbyte-ci/connectors/pipelines/pytest.ini b/airbyte-ci/connectors/pipelines/pytest.ini index 0bd08b038c23..b228671b5fa2 100644 --- a/airbyte-ci/connectors/pipelines/pytest.ini +++ b/airbyte-ci/connectors/pipelines/pytest.ini @@ -1,2 +1,4 @@ [pytest] addopts = --cov=pipelines +markers = + slow: marks tests as slow (deselect with '-m "not slow"') diff --git a/airbyte-ci/connectors/pipelines/ruff.toml b/airbyte-ci/connectors/pipelines/ruff.toml new file mode 100644 index 000000000000..7530d724d090 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/ruff.toml @@ -0,0 +1,22 @@ +target-version = "py310" + +ignore = ["ANN101", "ANN002", "ANN003"] + + + +[lint] + +extend-select = [ + +"ANN", # flake8-annotations + +] + + + +[lint.pydocstyle] + +convention = "google" + +[lint.flake8-annotations] +allow-star-arg-any = true diff --git a/airbyte-ci/connectors/pipelines/tests/conftest.py b/airbyte-ci/connectors/pipelines/tests/conftest.py index 47cfb0fc195f..fe7314bbebe7 100644 --- a/airbyte-ci/connectors/pipelines/tests/conftest.py +++ b/airbyte-ci/connectors/pipelines/tests/conftest.py @@ -3,16 +3,17 @@ # import os +import platform import sys from pathlib import Path -from typing import Set +from typing import List import dagger import git import pytest import requests from connector_ops.utils import Connector -from pipelines import utils +from pipelines.helpers import utils from tests.utils import ALL_CONNECTORS @@ -22,8 +23,13 @@ def anyio_backend(): @pytest.fixture(scope="module") -async def dagger_client(): - async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client: +def dagger_connection(): + return dagger.Connection(dagger.Config(log_output=sys.stderr)) + + +@pytest.fixture(scope="module") +async def dagger_client(dagger_connection): + async with dagger_connection as client: yield client @@ -68,5 +74,10 @@ def from_airbyte_root(airbyte_repo_path): @pytest.fixture(scope="session") -def all_connectors() -> Set[Connector]: - return ALL_CONNECTORS +def all_connectors() -> List[Connector]: + return sorted(ALL_CONNECTORS, key=lambda connector: connector.technical_name) + + +@pytest.fixture(scope="session") +def current_platform(): + return dagger.Platform(f"linux/{platform.machine()}") diff --git a/airbyte-ci/connectors/pipelines/tests/test_actions/test_environments.py b/airbyte-ci/connectors/pipelines/tests/test_actions/test_environments.py index f48c061dbec2..fbfcb4391f37 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_actions/test_environments.py +++ b/airbyte-ci/connectors/pipelines/tests/test_actions/test_environments.py @@ -3,9 +3,8 @@ # import pytest -from connector_ops.utils import Connector -from pipelines.actions import environments -from pipelines.contexts import PipelineContext +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.dagger.actions.python import common pytestmark = [ pytest.mark.anyio, @@ -13,27 +12,31 @@ @pytest.fixture -def python_connector() -> Connector: - return Connector("source-openweather") - - -@pytest.fixture -def context(dagger_client): - context = PipelineContext( +def connector_context(dagger_client): + context = ConnectorContext( pipeline_name="test", - is_local=True, + connector="source-faker", git_branch="test", git_revision="test", + report_output_prefix="test", + is_local=True, + use_remote_secrets=True, ) context.dagger_client = dagger_client return context -async def test_with_installed_python_package(context, python_connector): - python_environment = context.dagger_client.container().from_("python:3.9") - installed_connector_package = await environments.with_installed_python_package( - context, - python_environment, - str(python_connector.code_directory), - ) - await installed_connector_package.with_exec(["python", "main.py", "spec"]) +@pytest.mark.parametrize("use_local_cdk", [True, False]) +async def test_apply_python_development_overrides(connector_context, use_local_cdk): + connector_context.use_local_cdk = use_local_cdk + fake_connector_container = connector_context.dagger_client.container().from_("airbyte/python-connector-base:1.0.0") + before_override_pip_freeze = await fake_connector_container.with_exec(["pip", "freeze"]).stdout() + + assert "airbyte-cdk" not in before_override_pip_freeze.splitlines(), "The base image should not have the airbyte-cdk installed." + connector_with_overrides = await common.apply_python_development_overrides(connector_context, fake_connector_container) + + after_override_pip_freeze = await connector_with_overrides.with_exec(["pip", "freeze"]).stdout() + if use_local_cdk: + assert "airbyte-cdk" not in after_override_pip_freeze.splitlines(), "The override should not install the airbyte-cdk package." + else: + assert "airbyte-cdk" not in after_override_pip_freeze.splitlines(), "The override should install the airbyte-cdk package." diff --git a/airbyte-ci/connectors/pipelines/tests/test_bases.py b/airbyte-ci/connectors/pipelines/tests/test_bases.py index e1f1ebee2f73..5b4547df1e45 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_bases.py +++ b/airbyte-ci/connectors/pipelines/tests/test_bases.py @@ -7,7 +7,7 @@ import anyio import pytest from dagger import DaggerError -from pipelines import bases +from pipelines.models import reports, steps pytestmark = [ pytest.mark.anyio, @@ -15,14 +15,14 @@ class TestStep: - class DummyStep(bases.Step): + class DummyStep(steps.Step): title = "Dummy step" max_retries = 3 max_duration = timedelta(seconds=2) - async def _run(self, run_duration: timedelta) -> bases.StepResult: + async def _run(self, run_duration: timedelta) -> steps.StepResult: await anyio.sleep(run_duration.total_seconds()) - return bases.StepResult(self, bases.StepStatus.SUCCESS) + return steps.StepResult(self, steps.StepStatus.SUCCESS) @pytest.fixture def test_context(self, mocker): @@ -31,7 +31,7 @@ def test_context(self, mocker): async def test_run_with_timeout(self, test_context): step = self.DummyStep(test_context) step_result = await step.run(run_duration=step.max_duration - timedelta(seconds=1)) - assert step_result.status == bases.StepStatus.SUCCESS + assert step_result.status == steps.StepStatus.SUCCESS assert step.retry_count == 0 step_result = await step.run(run_duration=step.max_duration + timedelta(seconds=1)) @@ -45,19 +45,19 @@ async def test_run_with_timeout(self, test_context): @pytest.mark.parametrize( "step_status, exc_info, max_retries, max_dagger_error_retries, expect_retry", [ - (bases.StepStatus.SUCCESS, None, 0, 0, False), - (bases.StepStatus.SUCCESS, None, 3, 0, False), - (bases.StepStatus.SUCCESS, None, 0, 3, False), - (bases.StepStatus.SUCCESS, None, 3, 3, False), - (bases.StepStatus.SKIPPED, None, 0, 0, False), - (bases.StepStatus.SKIPPED, None, 3, 0, False), - (bases.StepStatus.SKIPPED, None, 0, 3, False), - (bases.StepStatus.SKIPPED, None, 3, 3, False), - (bases.StepStatus.FAILURE, DaggerError(), 0, 0, False), - (bases.StepStatus.FAILURE, DaggerError(), 0, 3, True), - (bases.StepStatus.FAILURE, None, 0, 0, False), - (bases.StepStatus.FAILURE, None, 0, 3, False), - (bases.StepStatus.FAILURE, None, 3, 0, True), + (steps.StepStatus.SUCCESS, None, 0, 0, False), + (steps.StepStatus.SUCCESS, None, 3, 0, False), + (steps.StepStatus.SUCCESS, None, 0, 3, False), + (steps.StepStatus.SUCCESS, None, 3, 3, False), + (steps.StepStatus.SKIPPED, None, 0, 0, False), + (steps.StepStatus.SKIPPED, None, 3, 0, False), + (steps.StepStatus.SKIPPED, None, 0, 3, False), + (steps.StepStatus.SKIPPED, None, 3, 3, False), + (steps.StepStatus.FAILURE, DaggerError(), 0, 0, False), + (steps.StepStatus.FAILURE, DaggerError(), 0, 3, True), + (steps.StepStatus.FAILURE, None, 0, 0, False), + (steps.StepStatus.FAILURE, None, 0, 3, False), + (steps.StepStatus.FAILURE, None, 3, 0, True), ], ) async def test_run_with_retries(self, mocker, test_context, step_status, exc_info, max_retries, max_dagger_error_retries, expect_retry): @@ -67,7 +67,7 @@ async def test_run_with_retries(self, mocker, test_context, step_status, exc_inf step.max_duration = timedelta(seconds=60) step.retry_delay = timedelta(seconds=0) step._run = mocker.AsyncMock( - side_effect=[bases.StepResult(step, step_status, exc_info=exc_info)] * (max(max_dagger_error_retries, max_retries) + 1) + side_effect=[steps.StepResult(step, step_status, exc_info=exc_info)] * (max(max_dagger_error_retries, max_retries) + 1) ) step_result = await step.run() @@ -85,23 +85,23 @@ def test_context(self, mocker): return mocker.Mock() def test_report_failed_if_it_has_no_step_result(self, test_context): - report = bases.Report(test_context, []) + report = reports.Report(test_context, []) assert not report.success - report = bases.Report(test_context, [bases.StepResult(None, bases.StepStatus.FAILURE)]) + report = reports.Report(test_context, [steps.StepResult(None, steps.StepStatus.FAILURE)]) assert not report.success - report = bases.Report( - test_context, [bases.StepResult(None, bases.StepStatus.FAILURE), bases.StepResult(None, bases.StepStatus.SUCCESS)] + report = reports.Report( + test_context, [steps.StepResult(None, steps.StepStatus.FAILURE), steps.StepResult(None, steps.StepStatus.SUCCESS)] ) assert not report.success - report = bases.Report(test_context, [bases.StepResult(None, bases.StepStatus.SUCCESS)]) + report = reports.Report(test_context, [steps.StepResult(None, steps.StepStatus.SUCCESS)]) assert report.success - report = bases.Report( - test_context, [bases.StepResult(None, bases.StepStatus.SUCCESS), bases.StepResult(None, bases.StepStatus.SKIPPED)] + report = reports.Report( + test_context, [steps.StepResult(None, steps.StepStatus.SUCCESS), steps.StepResult(None, steps.StepStatus.SKIPPED)] ) assert report.success - report = bases.Report(test_context, [bases.StepResult(None, bases.StepStatus.SKIPPED)]) + report = reports.Report(test_context, [steps.StepResult(None, steps.StepStatus.SKIPPED)]) assert report.success diff --git a/airbyte-ci/connectors/pipelines/tests/test_build_image/__init__.py b/airbyte-ci/connectors/pipelines/tests/test_build_image/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_build_image/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/tests/test_build_image/dummy_build_customization.py b/airbyte-ci/connectors/pipelines/tests/test_build_image/dummy_build_customization.py new file mode 100644 index 000000000000..6bc6ce686adc --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_build_image/dummy_build_customization.py @@ -0,0 +1,36 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + """This function will run before the connector installation. + It can mutate the base image container. + + Args: + base_image_container (Container): The base image container to mutate. + + Returns: + Container: The mutated base image container. + """ + return base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + + +async def post_connector_install(connector_container: Container) -> Container: + """This function will run after the connector installation during the build process. + It can mutate the connector container. + + Args: + connector_container (Container): The connector container to mutate. + + Returns: + Container: The mutated connector container. + """ + return connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") diff --git a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_python_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_build_image/test_python_connectors.py new file mode 100644 index 000000000000..bb8ac23a10ea --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_build_image/test_python_connectors.py @@ -0,0 +1,169 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from pathlib import Path + +import pytest +from pipelines.airbyte_ci.connectors.build_image.steps import build_customization, python_connectors +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.consts import BUILD_PLATFORMS +from pipelines.models.steps import StepStatus + +pytestmark = [ + pytest.mark.anyio, +] + + +class TestBuildConnectorImage: + @pytest.fixture + def all_platforms(self): + return BUILD_PLATFORMS + + @pytest.fixture + def test_context(self, mocker): + return mocker.Mock(secrets_to_mask=[], targeted_platforms=BUILD_PLATFORMS) + + @pytest.fixture + def test_context_with_connector_with_base_image(self, test_context): + test_context.connector.metadata = {"connectorBuildOptions": {"baseImage": "xyz"}} + return test_context + + @pytest.fixture + def test_context_with_connector_without_base_image(self, test_context): + test_context.connector.metadata = {} + return test_context + + @pytest.fixture + def connector_with_base_image_no_build_customization(self, all_connectors): + for connector in all_connectors: + if connector.metadata and connector.metadata.get("connectorBuildOptions", {}).get("baseImage"): + if not (connector.code_directory / "build_customization.py").exists(): + return connector + pytest.skip("No connector with a connectorBuildOptions.baseImage metadata found") + + @pytest.fixture + def connector_with_base_image_with_build_customization(self, connector_with_base_image_no_build_customization): + dummy_build_customization = (Path(__file__).parent / "dummy_build_customization.py").read_text() + (connector_with_base_image_no_build_customization.code_directory / "build_customization.py").write_text(dummy_build_customization) + yield connector_with_base_image_no_build_customization + (connector_with_base_image_no_build_customization.code_directory / "build_customization.py").unlink() + + @pytest.fixture + def test_context_with_real_connector_using_base_image( + self, connector_with_base_image_no_build_customization, dagger_client, current_platform + ): + context = ConnectorContext( + pipeline_name="test build", + connector=connector_with_base_image_no_build_customization, + git_branch="test", + git_revision="test", + report_output_prefix="test", + is_local=True, + use_remote_secrets=True, + targeted_platforms=[current_platform], + ) + context.dagger_client = dagger_client + return context + + @pytest.fixture + def test_context_with_real_connector_using_base_image_with_build_customization( + self, connector_with_base_image_with_build_customization, dagger_client, current_platform + ): + context = ConnectorContext( + pipeline_name="test build", + connector=connector_with_base_image_with_build_customization, + git_branch="test", + git_revision="test", + report_output_prefix="test", + is_local=True, + use_remote_secrets=True, + targeted_platforms=[current_platform], + ) + context.dagger_client = dagger_client + return context + + @pytest.fixture + def connector_without_base_image(self, all_connectors): + for connector in all_connectors: + if connector.metadata and not connector.metadata.get("connectorBuildOptions", {}).get("baseImage"): + return connector + pytest.skip("No connector without a connectorBuildOptions.baseImage metadata found") + + @pytest.fixture + def test_context_with_real_connector_without_base_image(self, connector_without_base_image, dagger_client, current_platform): + context = ConnectorContext( + pipeline_name="test build", + connector=connector_without_base_image, + git_branch="test", + git_revision="test", + report_output_prefix="test", + is_local=True, + use_remote_secrets=True, + targeted_platforms=[current_platform], + ) + context.dagger_client = dagger_client + return context + + async def test__run_using_base_image_with_mocks(self, mocker, test_context_with_connector_with_base_image, all_platforms): + container_built_from_base = mocker.AsyncMock() + mocker.patch.object( + python_connectors.BuildConnectorImages, "_build_from_base_image", mocker.AsyncMock(return_value=container_built_from_base) + ) + mocker.patch.object(python_connectors.BuildConnectorImages, "get_step_result", mocker.AsyncMock()) + step = python_connectors.BuildConnectorImages(test_context_with_connector_with_base_image) + step_result = await step._run() + assert step._build_from_base_image.call_count == len(all_platforms) + container_built_from_base.with_exec.assert_called_with(["spec"]) + assert step_result.status is StepStatus.SUCCESS + for platform in all_platforms: + assert step_result.output_artifact[platform] == container_built_from_base + + @pytest.mark.slow + async def test_building_from_base_image_for_real(self, test_context_with_real_connector_using_base_image, current_platform): + step = python_connectors.BuildConnectorImages(test_context_with_real_connector_using_base_image) + step_result = await step._run() + step_result.status is StepStatus.SUCCESS + built_container = step_result.output_artifact[current_platform] + assert await built_container.env_variable("AIRBYTE_ENTRYPOINT") == " ".join( + build_customization.get_entrypoint(step.context.connector) + ) + assert await built_container.workdir() == step.PATH_TO_INTEGRATION_CODE + assert await built_container.entrypoint() == build_customization.get_entrypoint(step.context.connector) + assert ( + await built_container.label("io.airbyte.version") + == test_context_with_real_connector_using_base_image.connector.metadata["dockerImageTag"] + ) + assert ( + await built_container.label("io.airbyte.name") + == test_context_with_real_connector_using_base_image.connector.metadata["dockerRepository"] + ) + + @pytest.mark.slow + async def test_building_from_base_image_with_customization_for_real( + self, test_context_with_real_connector_using_base_image_with_build_customization, current_platform + ): + step = python_connectors.BuildConnectorImages(test_context_with_real_connector_using_base_image_with_build_customization) + step_result = await step._run() + step_result.status is StepStatus.SUCCESS + built_container = step_result.output_artifact[current_platform] + assert await built_container.env_variable("MY_PRE_BUILD_ENV_VAR") == "my_pre_build_env_var_value" + assert await built_container.env_variable("MY_POST_BUILD_ENV_VAR") == "my_post_build_env_var_value" + + async def test__run_using_base_dockerfile_with_mocks(self, mocker, test_context_with_connector_without_base_image, all_platforms): + container_built_from_dockerfile = mocker.AsyncMock() + mocker.patch.object( + python_connectors.BuildConnectorImages, "_build_from_dockerfile", mocker.AsyncMock(return_value=container_built_from_dockerfile) + ) + step = python_connectors.BuildConnectorImages(test_context_with_connector_without_base_image) + step_result = await step._run() + assert step._build_from_dockerfile.call_count == len(all_platforms) + container_built_from_dockerfile.with_exec.assert_called_with(["spec"]) + assert step_result.status is StepStatus.SUCCESS + for platform in all_platforms: + assert step_result.output_artifact[platform] == container_built_from_dockerfile + + async def test_building_from_dockerfile_for_real(self, test_context_with_real_connector_without_base_image): + step = python_connectors.BuildConnectorImages(test_context_with_real_connector_without_base_image) + step_result = await step._run() + step_result.status is StepStatus.SUCCESS diff --git a/airbyte-ci/connectors/pipelines/tests/test_build_image/test_steps/test_common.py b/airbyte-ci/connectors/pipelines/tests/test_build_image/test_steps/test_common.py new file mode 100644 index 000000000000..8b2fbbcf1e3d --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_build_image/test_steps/test_common.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import os + +import dagger +import docker +import pytest +from pipelines.airbyte_ci.connectors.build_image.steps import common +from pipelines.consts import LOCAL_BUILD_PLATFORM +from pipelines.models.steps import StepStatus + +pytestmark = [ + pytest.mark.anyio, +] + + +@pytest.mark.slow +class TestLoadContainerToLocalDockerHost: + @pytest.fixture(scope="class") + def faker_connector(self, all_connectors): + for connector in all_connectors: + if connector.technical_name == "source-faker": + return connector + pytest.fail("Could not find the source-faker connector.") + + @pytest.fixture + def test_context(self, mocker, dagger_client, faker_connector, tmp_path): + return mocker.Mock( + secrets_to_mask=[], + dagger_client=dagger_client, + connector=faker_connector, + host_image_export_dir_path=tmp_path, + git_revision="test-revision", + ) + + @pytest.fixture + def bad_docker_host(self): + original_docker_host = os.environ.get("DOCKER_HOST") + yield "tcp://localhost:9999" + if original_docker_host: + os.environ["DOCKER_HOST"] = original_docker_host + else: + del os.environ["DOCKER_HOST"] + + @pytest.mark.parametrize( + "platforms", + [ + [dagger.Platform("linux/arm64")], + [dagger.Platform("linux/amd64")], + [dagger.Platform("linux/amd64"), dagger.Platform("linux/arm64")], + ], + ) + async def test_run(self, dagger_client, test_context, platforms): + """Test that the step runs successfully and that the image is loaded in the local docker host.""" + built_containers = { + platform: dagger_client.container(platform=platform).from_(f'{test_context.connector.metadata["dockerRepository"]}:latest') + for platform in platforms + } + step = common.LoadContainerToLocalDockerHost(test_context, built_containers) + + assert step.image_tag == "dev" + docker_client = docker.from_env() + step.image_tag = "test-load-container" + for platform in platforms: + full_image_name = f"{test_context.connector.metadata['dockerRepository']}:{step.image_tag}-{platform.replace('/', '-')}" + try: + docker_client.images.remove(full_image_name, force=True) + except docker.errors.ImageNotFound: + pass + result = await step.run() + assert result.status is StepStatus.SUCCESS + multi_platforms = len(platforms) > 1 + for platform in platforms: + if multi_platforms: + full_image_name = f"{test_context.connector.metadata['dockerRepository']}:{step.image_tag}-{platform.replace('/', '-')}" + else: + full_image_name = f"{test_context.connector.metadata['dockerRepository']}:{step.image_tag}" + docker_client.images.get(full_image_name) + + # CI can't run docker arm64 containers + if platform is LOCAL_BUILD_PLATFORM or (os.getenv("CI") != "True"): + docker_client.containers.run(full_image_name, "spec") + docker_client.images.remove(full_image_name, force=True) + + async def test_run_export_failure(self, dagger_client, test_context, mocker): + """Test that the step fails if the export of the container fails.""" + built_containers = { + LOCAL_BUILD_PLATFORM: dagger_client.container(platform=LOCAL_BUILD_PLATFORM).from_( + f'{test_context.connector.metadata["dockerRepository"]}:latest' + ) + } + step = common.LoadContainerToLocalDockerHost(test_context, built_containers) + + mocker.patch.object(common, "export_container_to_tarball", return_value=(None, None)) + result = await step.run() + assert result.status is StepStatus.FAILURE + assert "Failed to export the connector image" in result.stderr + + async def test_run_connection_error(self, dagger_client, test_context, bad_docker_host): + """Test that the step fails if the connection to the docker host fails.""" + built_containers = { + LOCAL_BUILD_PLATFORM: dagger_client.container(platform=LOCAL_BUILD_PLATFORM).from_( + f'{test_context.connector.metadata["dockerRepository"]}:latest' + ) + } + step = common.LoadContainerToLocalDockerHost(test_context, built_containers) + os.environ["DOCKER_HOST"] = bad_docker_host + result = await step.run() + assert result.status is StepStatus.FAILURE + assert "Something went wrong while interacting with the local docker client" in result.stderr + + async def test_run_import_failure(self, dagger_client, test_context, mocker): + """Test that the step fails if the docker import of the tar fails.""" + built_containers = { + LOCAL_BUILD_PLATFORM: dagger_client.container(platform=LOCAL_BUILD_PLATFORM).from_( + f'{test_context.connector.metadata["dockerRepository"]}:latest' + ) + } + step = common.LoadContainerToLocalDockerHost(test_context, built_containers) + mock_docker_client = mocker.MagicMock() + mock_docker_client.api.import_image_from_file.return_value = "bad response" + mock_docker_client.images.load.side_effect = docker.errors.DockerException("test error") + mocker.patch.object(common.docker, "from_env", return_value=mock_docker_client) + result = await step.run() + assert result.status is StepStatus.FAILURE + assert "Something went wrong while interacting with the local docker client: test error" in result.stderr diff --git a/airbyte-ci/connectors/pipelines/tests/test_cli/__init__.py b/airbyte-ci/connectors/pipelines/tests/test_cli/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_cli/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py new file mode 100644 index 000000000000..cf503c9d6588 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_cli/test_click_decorators.py @@ -0,0 +1,84 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import asyncclick as click +import pytest +from asyncclick.testing import CliRunner +from pipelines.cli.click_decorators import click_append_to_context_object, click_ignore_unused_kwargs, click_merge_args_into_context_obj + + +@pytest.mark.anyio +async def test_click_append_to_context_object(): + runner = CliRunner() + + def get_value(ctx): + return "got" + + async def get_async_value(ctx): + return "async_got" + + @click.command(name="test-command") + @click_append_to_context_object("get", get_value) + @click_append_to_context_object("async_get", get_async_value) + @click_append_to_context_object("foo", "bar") + @click_append_to_context_object("baz", lambda _ctx: "qux") + @click_append_to_context_object("foo2", lambda ctx: ctx.obj.get("foo") + "2") + async def test_command(): + ctx = click.get_current_context() + assert ctx.obj["foo"] == "bar" + assert ctx.obj["baz"] == "qux" + assert ctx.obj["foo2"] == "bar2" + assert ctx.obj["get"] == "got" + assert ctx.obj["async_get"] == "async_got" + + @click.command(name="test-command") + @click_append_to_context_object("get", get_value) + @click_append_to_context_object("async_get", get_async_value) + @click_append_to_context_object("foo", "bar") + @click_append_to_context_object("baz", lambda _ctx: "qux") + @click_append_to_context_object("foo2", lambda ctx: ctx.obj.get("foo") + "2") + async def test_command_async(): + ctx = click.get_current_context() + assert ctx.obj["foo"] == "bar" + assert ctx.obj["baz"] == "qux" + assert ctx.obj["foo2"] == "bar2" + assert ctx.obj["get"] == "got" + assert ctx.obj["async_get"] == "async_got" + + result = await runner.invoke(test_command) + assert result.exit_code == 0 + + result_async = await runner.invoke(test_command_async) + assert result_async.exit_code == 0 + + +@pytest.mark.anyio +async def test_click_ignore_unused_kwargs(): + @click_ignore_unused_kwargs + def decorated_function(a, b): + return a + b + + # Test that the decorated function works as expected with matching kwargs + assert decorated_function(a=1, b=2) == 3 + + # Test that the decorated function ignores unmatched kwargs + assert decorated_function(a=1, b=2, c=3) == 3 + + +@pytest.mark.anyio +async def test_click_merge_args_into_context_obj(): + runner = CliRunner() + + @click.command(name="test-command") + @click.option("--foo", help="foo option") + @click.option("--bar", help="bar option") + @click_merge_args_into_context_obj + @click_ignore_unused_kwargs + async def test_command(foo, bar): + ctx = click.get_current_context() + assert ctx.obj["foo"] == foo + assert ctx.obj["bar"] == bar + + result = await runner.invoke(test_command, ["--foo", "hello", "--bar", "world"]) + assert result.exit_code == 0 diff --git a/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py index 0f9cda290e27..cf9b3a52da87 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py +++ b/airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py @@ -4,11 +4,15 @@ from typing import Callable +import asyncclick as click import pytest -from click.testing import CliRunner +from asyncclick.testing import CliRunner from connector_ops.utils import METADATA_FILE_NAME, ConnectorLanguage -from pipelines.bases import ConnectorWithModifiedFiles -from pipelines.commands.groups import connectors +from pipelines.airbyte_ci.connectors import commands as connectors_commands +from pipelines.airbyte_ci.connectors.build_image import commands as connectors_build_command +from pipelines.airbyte_ci.connectors.publish import commands as connectors_publish_command +from pipelines.airbyte_ci.connectors.test import commands as connectors_test_command +from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles from tests.utils import pick_a_random_connector @@ -19,12 +23,13 @@ def runner(): def test_get_selected_connectors_by_name_no_file_modification(): connector = pick_a_random_connector() - selected_connectors = connectors.get_selected_connectors_with_modified_files( + selected_connectors = connectors_commands.get_selected_connectors_with_modified_files( selected_names=(connector.technical_name,), selected_support_levels=(), selected_languages=(), modified=False, metadata_changes_only=False, + metadata_query=None, modified_files=set(), ) @@ -35,12 +40,13 @@ def test_get_selected_connectors_by_name_no_file_modification(): def test_get_selected_connectors_by_support_level_no_file_modification(): - selected_connectors = connectors.get_selected_connectors_with_modified_files( + selected_connectors = connectors_commands.get_selected_connectors_with_modified_files( selected_names=(), selected_support_levels=["certified"], selected_languages=(), modified=False, metadata_changes_only=False, + metadata_query=None, modified_files=set(), ) @@ -48,12 +54,13 @@ def test_get_selected_connectors_by_support_level_no_file_modification(): def test_get_selected_connectors_by_language_no_file_modification(): - selected_connectors = connectors.get_selected_connectors_with_modified_files( + selected_connectors = connectors_commands.get_selected_connectors_with_modified_files( selected_names=(), selected_support_levels=(), selected_languages=(ConnectorLanguage.LOW_CODE,), modified=False, metadata_changes_only=False, + metadata_query=None, modified_files=set(), ) @@ -63,12 +70,13 @@ def test_get_selected_connectors_by_language_no_file_modification(): def test_get_selected_connectors_by_name_with_file_modification(): connector = pick_a_random_connector() modified_files = {connector.code_directory / "setup.py"} - selected_connectors = connectors.get_selected_connectors_with_modified_files( + selected_connectors = connectors_commands.get_selected_connectors_with_modified_files( selected_names=(connector.technical_name,), selected_support_levels=(), selected_languages=(), modified=False, metadata_changes_only=False, + metadata_query=None, modified_files=modified_files, ) @@ -81,12 +89,13 @@ def test_get_selected_connectors_by_name_with_file_modification(): def test_get_selected_connectors_by_name_and_support_level_or_languages_leads_to_intersection(): connector = pick_a_random_connector() modified_files = {connector.code_directory / "setup.py"} - selected_connectors = connectors.get_selected_connectors_with_modified_files( + selected_connectors = connectors_commands.get_selected_connectors_with_modified_files( selected_names=(connector.technical_name,), selected_support_levels=(connector.support_level,), selected_languages=(connector.language,), modified=False, metadata_changes_only=False, + metadata_query=None, modified_files=modified_files, ) @@ -97,12 +106,13 @@ def test_get_selected_connectors_with_modified(): first_modified_connector = pick_a_random_connector() second_modified_connector = pick_a_random_connector(other_picked_connectors=[first_modified_connector]) modified_files = {first_modified_connector.code_directory / "setup.py", second_modified_connector.code_directory / "setup.py"} - selected_connectors = connectors.get_selected_connectors_with_modified_files( + selected_connectors = connectors_commands.get_selected_connectors_with_modified_files( selected_names=(), selected_support_levels=(), selected_languages=(), modified=True, metadata_changes_only=False, + metadata_query=None, modified_files=modified_files, ) @@ -113,12 +123,13 @@ def test_get_selected_connectors_with_modified_and_language(): first_modified_connector = pick_a_random_connector(language=ConnectorLanguage.PYTHON) second_modified_connector = pick_a_random_connector(language=ConnectorLanguage.JAVA, other_picked_connectors=[first_modified_connector]) modified_files = {first_modified_connector.code_directory / "setup.py", second_modified_connector.code_directory / "setup.py"} - selected_connectors = connectors.get_selected_connectors_with_modified_files( + selected_connectors = connectors_commands.get_selected_connectors_with_modified_files( selected_names=(), selected_support_levels=(), selected_languages=(ConnectorLanguage.JAVA,), modified=True, metadata_changes_only=False, + metadata_query=None, modified_files=modified_files, ) @@ -130,12 +141,13 @@ def test_get_selected_connectors_with_modified_and_support_level(): first_modified_connector = pick_a_random_connector(support_level="community") second_modified_connector = pick_a_random_connector(support_level="certified", other_picked_connectors=[first_modified_connector]) modified_files = {first_modified_connector.code_directory / "setup.py", second_modified_connector.code_directory / "setup.py"} - selected_connectors = connectors.get_selected_connectors_with_modified_files( + selected_connectors = connectors_commands.get_selected_connectors_with_modified_files( selected_names=(), selected_support_levels=["certified"], selected_languages=(), modified=True, metadata_changes_only=False, + metadata_query=None, modified_files=modified_files, ) @@ -151,12 +163,13 @@ def test_get_selected_connectors_with_modified_and_metadata_only(): second_modified_connector.code_directory / METADATA_FILE_NAME, second_modified_connector.code_directory / "setup.py", } - selected_connectors = connectors.get_selected_connectors_with_modified_files( + selected_connectors = connectors_commands.get_selected_connectors_with_modified_files( selected_names=(), selected_support_levels=(), selected_languages=(), modified=True, metadata_changes_only=True, + metadata_query=None, modified_files=modified_files, ) @@ -176,12 +189,13 @@ def test_get_selected_connectors_with_metadata_only(): second_modified_connector.code_directory / METADATA_FILE_NAME, second_modified_connector.code_directory / "setup.py", } - selected_connectors = connectors.get_selected_connectors_with_modified_files( + selected_connectors = connectors_commands.get_selected_connectors_with_modified_files( selected_names=(), selected_support_levels=(), selected_languages=(), modified=False, metadata_changes_only=True, + metadata_query=None, modified_files=modified_files, ) @@ -193,6 +207,25 @@ def test_get_selected_connectors_with_metadata_only(): } +def test_get_selected_connectors_with_metadata_query(): + connector = pick_a_random_connector() + metadata_query = f"data.dockerRepository == '{connector.metadata['dockerRepository']}'" + selected_connectors = connectors_commands.get_selected_connectors_with_modified_files( + selected_names=(), + selected_support_levels=(), + selected_languages=(), + modified=False, + metadata_changes_only=False, + metadata_query=metadata_query, + modified_files=set(), + ) + + assert len(selected_connectors) == 1 + assert isinstance(selected_connectors[0], ConnectorWithModifiedFiles) + assert selected_connectors[0].technical_name == connector.technical_name + assert not selected_connectors[0].modified_files + + @pytest.fixture() def click_context_obj(): return { @@ -213,15 +246,17 @@ def click_context_obj(): "concurrency": 1, "ci_git_user": None, "ci_github_access_token": None, + "docker_hub_username": "foo", + "docker_hub_password": "bar", } @pytest.mark.parametrize( "command, command_args", [ - (connectors.test, []), + (connectors_test_command.test, []), ( - connectors.publish, + connectors_publish_command.publish, [ "--spec-cache-gcs-credentials", "test", @@ -231,17 +266,13 @@ def click_context_obj(): "test", "--metadata-service-bucket-name", "test", - "--docker-hub-username", - "test", - "--docker-hub-password", - "test", ], ), - (connectors.format_code, []), - (connectors.build, []), + (connectors_build_command.build, []), ], ) -def test_commands_do_not_override_connector_selection( +@pytest.mark.anyio +async def test_commands_do_not_override_connector_selection( mocker, runner: CliRunner, click_context_obj: dict, command: Callable, command_args: list ): """ @@ -252,11 +283,35 @@ def test_commands_do_not_override_connector_selection( selected_connector = mocker.MagicMock() click_context_obj["selected_connectors_with_modified_files"] = [selected_connector] - mocker.patch.object(connectors.click, "confirm") + mocker.patch.object(click, "confirm") mock_connector_context = mocker.MagicMock() - mocker.patch.object(connectors, "ConnectorContext", mock_connector_context) - mocker.patch.object(connectors, "PublishConnectorContext", mock_connector_context) - runner.invoke(command, command_args, catch_exceptions=False, obj=click_context_obj) + mocker.patch.object(connectors_test_command, "ConnectorContext", mock_connector_context) + mocker.patch.object(connectors_build_command, "ConnectorContext", mock_connector_context) + mocker.patch.object(connectors_publish_command, "PublishConnectorContext", mock_connector_context) + await runner.invoke(command, command_args, catch_exceptions=True, obj=click_context_obj) assert mock_connector_context.call_count == 1 # If the connector selection is overriden the context won't be instantiated with the selected connector mock instance assert mock_connector_context.call_args_list[0].kwargs["connector"] == selected_connector + + +@pytest.mark.parametrize( + "use_remote_secrets_user_input, gsm_env_var_set, expected_use_remote_secrets, expect_click_usage_error", + [ + (None, True, True, False), + (None, False, False, False), + (True, False, None, True), + (True, True, True, False), + (False, True, False, False), + (False, False, False, False), + ], +) +def test_should_use_remote_secrets( + mocker, use_remote_secrets_user_input, gsm_env_var_set, expected_use_remote_secrets, expect_click_usage_error +): + mocker.patch.object(connectors_commands.os, "getenv", return_value="test" if gsm_env_var_set else None) + if expect_click_usage_error: + with pytest.raises(click.UsageError): + connectors_commands.should_use_remote_secrets(use_remote_secrets_user_input) + else: + final_use_remote_secrets = connectors_commands.should_use_remote_secrets(use_remote_secrets_user_input) + assert final_use_remote_secrets == expected_use_remote_secrets diff --git a/airbyte-ci/connectors/pipelines/tests/test_dagger/__init__.py b/airbyte-ci/connectors/pipelines/tests/test_dagger/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_dagger/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/__init__.py b/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/__init__.py b/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py b/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py new file mode 100644 index 000000000000..2bb2c3f7fa9e --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_python/test_common.py @@ -0,0 +1,72 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import pytest +import requests +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.dagger.actions.python import common + +pytestmark = [ + pytest.mark.anyio, +] + + +@pytest.fixture(scope="module") +def latest_cdk_version(): + cdk_pypi_url = "https://pypi.org/pypi/airbyte-cdk/json" + response = requests.get(cdk_pypi_url) + response.raise_for_status() + package_info = response.json() + return package_info["info"]["version"] + + +@pytest.fixture(scope="module") +def python_connector_with_setup_not_latest_cdk(all_connectors): + for connector in all_connectors: + if ( + connector.metadata.get("connectorBuildOptions", {}).get("baseImage", False) + and connector.language == "python" + and connector.code_directory.joinpath("setup.py").exists() + ): + return connector + pytest.skip("No python connector with setup.py and not latest cdk version found") + + +@pytest.fixture(scope="module") +def context_with_setup(dagger_client, python_connector_with_setup_not_latest_cdk): + context = ConnectorContext( + pipeline_name="test python common", + connector=python_connector_with_setup_not_latest_cdk, + git_branch="test", + git_revision="test", + report_output_prefix="test", + is_local=True, + use_remote_secrets=False, + ) + context.dagger_client = dagger_client + return context + + +@pytest.fixture(scope="module") +def python_connector_base_image_address(python_connector_with_setup_not_latest_cdk): + return python_connector_with_setup_not_latest_cdk.metadata["connectorBuildOptions"]["baseImage"] + + +async def test_with_python_connector_installed_from_setup(context_with_setup, python_connector_base_image_address, latest_cdk_version): + + python_container = context_with_setup.dagger_client.container().from_(python_connector_base_image_address) + + container = await common.with_python_connector_installed( + context_with_setup, python_container, str(context_with_setup.connector.code_directory) + ) + # Uninstall and reinstall the latest cdk version + cdk_install_latest_output = ( + await container.with_exec(["pip", "uninstall", "-y", f"airbyte-cdk=={latest_cdk_version}"], skip_entrypoint=True) + .with_exec(["pip", "install", f"airbyte-cdk=={latest_cdk_version}"], skip_entrypoint=True) + .stdout() + ) + # Assert that the latest cdk version is installed from cache + assert f"Using cached airbyte_cdk-{latest_cdk_version}-py3-none-any.whl" in cdk_install_latest_output + # Assert that the connector is installed + pip_freeze_output = await container.with_exec(["pip", "freeze"], skip_entrypoint=True).stdout() + assert context_with_setup.connector.technical_name in pip_freeze_output diff --git a/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_secrets.py b/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_secrets.py new file mode 100644 index 000000000000..b7af1eebc230 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_dagger/test_actions/test_secrets.py @@ -0,0 +1,54 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import json + +import pytest +from pipelines.dagger.actions import secrets + +pytestmark = [ + pytest.mark.anyio, +] + + +async def test_load_from_local_directory_not_exists(mocker, tmp_path): + context = mocker.MagicMock() + context.connector.code_directory = tmp_path + result = await secrets.load_from_local(context) + assert result == {} + context.logger.warning.assert_called_with(f"Local secrets directory {tmp_path / 'secrets'} does not exist, no secrets will be loaded.") + + +async def test_load_from_local_empty_directory(mocker, tmp_path): + secrets_dir = tmp_path / "secrets" + secrets_dir.mkdir() + context = mocker.MagicMock() + context.connector.code_directory = tmp_path + + result = await secrets.load_from_local(context) + + assert result == {} + context.logger.warning.assert_called_with(f"Local secrets directory {secrets_dir} is empty, no secrets will be loaded.") + + +async def test_load_from_local_with_secrets(mocker, tmp_path, dagger_client): + secrets_dir = tmp_path / "secrets" + secrets_dir.mkdir() + + first_dummy_config = json.dumps({"dummy": "config_a"}) + second_dummy_config = json.dumps({"dummy": "config_b"}) + (secrets_dir / "dummy_config_a.json").write_text(first_dummy_config) + (secrets_dir / "dummy_config_b.json").write_text(second_dummy_config) + + context = mocker.MagicMock() + context.dagger_client = dagger_client + context.connector.code_directory = tmp_path + + result = await secrets.load_from_local(context) + assert len(result) == 2, "All secrets should be loaded from the local secrets directory" + assert ( + await result["dummy_config_a.json"].plaintext() == first_dummy_config + ), "The content of dummy_config_a.json should be loaded as a secret" + assert ( + await result["dummy_config_b.json"].plaintext() == second_dummy_config + ), "The content of dummy_config_b.json should be loaded as a secret" diff --git a/airbyte-ci/connectors/pipelines/tests/test_environments.py b/airbyte-ci/connectors/pipelines/tests/test_environments.py deleted file mode 100644 index db48914fdeb7..000000000000 --- a/airbyte-ci/connectors/pipelines/tests/test_environments.py +++ /dev/null @@ -1,5 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -# diff --git a/airbyte-ci/connectors/pipelines/tests/test_format/__init__.py b/airbyte-ci/connectors/pipelines/tests/test_format/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_format/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/tests/test_format/formatted_code/java.java b/airbyte-ci/connectors/pipelines/tests/test_format/formatted_code/java.java new file mode 100644 index 000000000000..725920dd60c5 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_format/formatted_code/java.java @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +public class BadlyFormatted { + + public static void main(String[] args) { + System.out.println("Hello, World!"); + for (int i = 0; i < 5; i++) { + System.out.println(i); + } + } + +} diff --git a/airbyte-ci/connectors/pipelines/tests/test_format/formatted_code/json.json b/airbyte-ci/connectors/pipelines/tests/test_format/formatted_code/json.json new file mode 100644 index 000000000000..2c0b44605f43 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_format/formatted_code/json.json @@ -0,0 +1,8 @@ +{ + "name": "John Doe", + "age": 25, + "address": { + "city": "XYZ", + "street": "123 Main St" + } +} diff --git a/airbyte-ci/connectors/pipelines/tests/test_format/formatted_code/python.py b/airbyte-ci/connectors/pipelines/tests/test_format/formatted_code/python.py new file mode 100644 index 000000000000..e9c43d7a4066 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_format/formatted_code/python.py @@ -0,0 +1,9 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + + +def my_function(): + print("Using single quotes, no newlines, and no spaces between the function name.") + + +def my_other_function(): + print("Using single quotes, no newlines, and no spaces between the function name.") diff --git a/airbyte-ci/connectors/pipelines/tests/test_format/formatted_code/yaml.yaml b/airbyte-ci/connectors/pipelines/tests/test_format/formatted_code/yaml.yaml new file mode 100644 index 000000000000..44707db1e502 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_format/formatted_code/yaml.yaml @@ -0,0 +1,4 @@ +name: John Doe +age: 25 +city: XYZ +street: 123 Main St diff --git a/airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/java.java b/airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/java.java new file mode 100644 index 000000000000..674e8da69e28 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/java.java @@ -0,0 +1,8 @@ +public class BadlyFormatted { +public static void main(String[] args) { +System.out.println("Hello, World!"); +for (int i=0; i<5; i++){ +System.out.println(i); +} +} +} \ No newline at end of file diff --git a/airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/json.json b/airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/json.json new file mode 100644 index 000000000000..105254fe423d --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/json.json @@ -0,0 +1,8 @@ +{ + "name": "John Doe", + "age": 25, +"address": { + "city": "XYZ", + "street": "123 Main St" +} +} \ No newline at end of file diff --git a/airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/python.py b/airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/python.py new file mode 100644 index 000000000000..a7b87abe0702 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/python.py @@ -0,0 +1,4 @@ +def my_function(): + print("Using single quotes, no newlines, and no spaces between the function name.") +def my_other_function(): + print("Using single quotes, no newlines, and no spaces between the function name.") diff --git a/airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/yaml.yaml b/airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/yaml.yaml new file mode 100644 index 000000000000..ad5bfe703d4b --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code/yaml.yaml @@ -0,0 +1,4 @@ +name: John Doe +age: 25 +city: XYZ +street: 123 Main St \ No newline at end of file diff --git a/airbyte-ci/connectors/pipelines/tests/test_format/test_commands.py b/airbyte-ci/connectors/pipelines/tests/test_format/test_commands.py new file mode 100644 index 000000000000..61ddb9466156 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_format/test_commands.py @@ -0,0 +1,91 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import shutil + +import pytest +from asyncclick.testing import CliRunner +from pipelines.airbyte_ci.format import commands +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext + +pytestmark = [ + pytest.mark.anyio, +] + +PATH_TO_NON_FORMATTED_CODE = "airbyte-ci/connectors/pipelines/tests/test_format/non_formatted_code" +PATH_TO_FORMATTED_CODE = "airbyte-ci/connectors/pipelines/tests/test_format/formatted_code" + + +@pytest.fixture +def tmp_dir_with_non_formatted_code(tmp_path): + """ + This fixture creates a directory with non formatted code and a license file in the tmp_path. + It copies the content of non_formatted_code into the tmp_path. + The non_formatted_code directory has non formatted java, python, json, and yaml files missing license headers. + """ + shutil.copytree(PATH_TO_NON_FORMATTED_CODE, tmp_path / "non_formatted_code") + return str(tmp_path / "non_formatted_code") + + +@pytest.fixture +def tmp_dir_with_formatted_code(tmp_path): + """ + This fixture creates a directory with correctly formatted code and a license file in the tmp_path. + It copies the content of formatted_code into the tmp_path. + The formatted_code has correctly formatted java, python, json, and yaml files with license headers. + """ + shutil.copytree(PATH_TO_FORMATTED_CODE, tmp_path / "formatted_code") + return str(tmp_path / "formatted_code") + + +@pytest.fixture +def now_formatted_directory(dagger_client, tmp_dir_with_non_formatted_code): + return dagger_client.host().directory(tmp_dir_with_non_formatted_code).with_timestamps(0) + + +@pytest.fixture +def already_formatted_directory(dagger_client, tmp_dir_with_formatted_code): + return dagger_client.host().directory(tmp_dir_with_formatted_code).with_timestamps(0) + + +@pytest.fixture +def directory_with_expected_formatted_code(dagger_client): + expected_formatted_code_path = PATH_TO_FORMATTED_CODE + return dagger_client.host().directory(expected_formatted_code_path).with_timestamps(0) + + +@pytest.mark.slow +@pytest.mark.parametrize("subcommand", ["check", "fix"]) +async def test_check_and_fix_all_on_non_formatted_code( + mocker, subcommand, dagger_client, tmp_dir_with_non_formatted_code, now_formatted_directory, directory_with_expected_formatted_code +): + """ + Test that when given non formatted files the 'check' and 'fix' all command exit with status 1. + We also check that 'fix' correctly exports back the formatted code and that it matches what we expect. + """ + mocker.patch.object(ClickPipelineContext, "get_dagger_client", mocker.AsyncMock(return_value=dagger_client)) + mocker.patch.object(commands.FormatCommand, "LOCAL_REPO_PATH", tmp_dir_with_non_formatted_code) + runner = CliRunner() + result = await runner.invoke(commands.format_code, [subcommand, "all"], catch_exceptions=False) + if subcommand == "fix": + assert await now_formatted_directory.diff(directory_with_expected_formatted_code).entries() == [] + assert result.exit_code == 1 + + +@pytest.mark.slow +@pytest.mark.parametrize("subcommand", ["check", "fix"]) +async def test_check_and_fix_all_on_formatted_code( + mocker, subcommand, dagger_client, tmp_dir_with_formatted_code, already_formatted_directory, directory_with_expected_formatted_code +): + """ + Test that when given formatted files the 'check' and 'fix' all command exit with status 0. + We also check that 'fix' does not exports back any file change to the host. + """ + mocker.patch.object(ClickPipelineContext, "get_dagger_client", mocker.AsyncMock(return_value=dagger_client)) + mocker.patch.object(commands.FormatCommand, "LOCAL_REPO_PATH", tmp_dir_with_formatted_code) + runner = CliRunner() + result = await runner.invoke(commands.format_code, [subcommand, "all"], catch_exceptions=False) + if subcommand == "fix": + assert await already_formatted_directory.diff(directory_with_expected_formatted_code).entries() == [] + assert result.exit_code == 0 diff --git a/airbyte-ci/connectors/pipelines/tests/test_gradle.py b/airbyte-ci/connectors/pipelines/tests/test_gradle.py index e45027c860dc..34312ec1d0d3 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_gradle.py +++ b/airbyte-ci/connectors/pipelines/tests/test_gradle.py @@ -1,11 +1,14 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from __future__ import annotations from pathlib import Path +import pipelines.helpers.connectors.modifed import pytest -from pipelines import bases, gradle +from pipelines.airbyte_ci.steps import gradle +from pipelines.models import steps pytestmark = [ pytest.mark.anyio, @@ -15,16 +18,17 @@ class TestGradleTask: class DummyStep(gradle.GradleTask): gradle_task_name = "dummyTask" + title = "Dummy Step" - async def _run(self) -> bases.StepResult: - return bases.StepResult(self, bases.StepStatus.SUCCESS) + async def _run(self) -> steps.StepResult: + return steps.StepResult(self, steps.StepStatus.SUCCESS) @pytest.fixture def test_context(self, mocker, dagger_client): return mocker.Mock( secrets_to_mask=[], dagger_client=dagger_client, - connector=bases.ConnectorWithModifiedFiles( + connector=pipelines.helpers.connectors.modifed.ConnectorWithModifiedFiles( "source-postgres", frozenset({Path("airbyte-integrations/connectors/source-postgres/metadata.yaml")}) ), ) @@ -32,3 +36,11 @@ def test_context(self, mocker, dagger_client): async def test_build_include(self, test_context): step = self.DummyStep(test_context) assert step.build_include + + def test_params(self, test_context): + step = self.DummyStep(test_context) + step.extra_params = {"-x": ["dummyTask", "dummyTask2"]} + assert set(step.params_as_cli_options) == { + "-x=dummyTask", + "-x=dummyTask2", + } diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/__init__.py b/airbyte-ci/connectors/pipelines/tests/test_helpers/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_helpers/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/__init__.py b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_argument_parsing.py b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_argument_parsing.py new file mode 100644 index 000000000000..7201a2b83059 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_argument_parsing.py @@ -0,0 +1,36 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import enum +import time + +import anyio +import pytest +from pipelines.helpers.execution import argument_parsing + + +class SupportedStepIds(enum.Enum): + STEP1 = "step1" + STEP2 = "step2" + STEP3 = "step3" + + +def test_build_extra_params_mapping(mocker): + ctx = mocker.Mock() + argument = mocker.Mock() + + raw_extra_params = ( + "--step1.param1=value1", + "--step2.param2=value2", + "--step3.param3=value3", + "--step1.param4", + ) + + result = argument_parsing.build_extra_params_mapping(SupportedStepIds)(ctx, argument, raw_extra_params) + + expected_result = { + SupportedStepIds.STEP1.value: {"param1": ["value1"], "param4": []}, + SupportedStepIds.STEP2.value: {"param2": ["value2"]}, + SupportedStepIds.STEP3.value: {"param3": ["value3"]}, + } + + assert result == expected_result diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_run_steps.py b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_run_steps.py new file mode 100644 index 000000000000..b1975799ae35 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_execution/test_run_steps.py @@ -0,0 +1,424 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import time + +import anyio +import pytest +from pipelines.helpers.execution.run_steps import InvalidStepConfiguration, RunStepOptions, StepToRun, run_steps +from pipelines.models.contexts.pipeline_context import PipelineContext +from pipelines.models.steps import Step, StepResult, StepStatus + +test_context = PipelineContext(pipeline_name="test", is_local=True, git_branch="test", git_revision="test", report_output_prefix="test") + + +class TestStep(Step): + title = "Test Step" + + async def _run(self, result_status=StepStatus.SUCCESS) -> StepResult: + return StepResult(self, result_status) + + +@pytest.mark.anyio +@pytest.mark.parametrize( + "desc, steps, expected_results, options", + [ + ( + "All consecutive steps succeed", + [ + [StepToRun(id="step1", step=TestStep(test_context))], + [StepToRun(id="step2", step=TestStep(test_context))], + [StepToRun(id="step3", step=TestStep(test_context))], + [StepToRun(id="step4", step=TestStep(test_context))], + ], + {"step1": StepStatus.SUCCESS, "step2": StepStatus.SUCCESS, "step3": StepStatus.SUCCESS, "step4": StepStatus.SUCCESS}, + RunStepOptions(fail_fast=True), + ), + ( + "Steps all succeed with parallel steps", + [ + [StepToRun(id="step1", step=TestStep(test_context))], + [ + StepToRun(id="step2", step=TestStep(test_context)), + StepToRun(id="step3", step=TestStep(test_context)), + ], + [StepToRun(id="step4", step=TestStep(test_context))], + ], + {"step1": StepStatus.SUCCESS, "step2": StepStatus.SUCCESS, "step3": StepStatus.SUCCESS, "step4": StepStatus.SUCCESS}, + RunStepOptions(fail_fast=True), + ), + ( + "Steps after a failed step are skipped, when fail_fast is True", + [ + [StepToRun(id="step1", step=TestStep(test_context))], + [StepToRun(id="step2", step=TestStep(test_context), args={"result_status": StepStatus.FAILURE})], + [StepToRun(id="step3", step=TestStep(test_context))], + [StepToRun(id="step4", step=TestStep(test_context))], + ], + {"step1": StepStatus.SUCCESS, "step2": StepStatus.FAILURE, "step3": StepStatus.SKIPPED, "step4": StepStatus.SKIPPED}, + RunStepOptions(fail_fast=True), + ), + ( + "Steps after a failed step are not skipped, when fail_fast is False", + [ + [StepToRun(id="step1", step=TestStep(test_context))], + [StepToRun(id="step2", step=TestStep(test_context), args={"result_status": StepStatus.FAILURE})], + [StepToRun(id="step3", step=TestStep(test_context))], + [StepToRun(id="step4", step=TestStep(test_context))], + ], + {"step1": StepStatus.SUCCESS, "step2": StepStatus.FAILURE, "step3": StepStatus.SUCCESS, "step4": StepStatus.SUCCESS}, + RunStepOptions(fail_fast=False), + ), + ( + "fail fast has no effect on parallel steps", + [ + [StepToRun(id="step1", step=TestStep(test_context))], + [ + StepToRun(id="step2", step=TestStep(test_context)), + StepToRun(id="step3", step=TestStep(test_context)), + ], + [StepToRun(id="step4", step=TestStep(test_context))], + ], + {"step1": StepStatus.SUCCESS, "step2": StepStatus.SUCCESS, "step3": StepStatus.SUCCESS, "step4": StepStatus.SUCCESS}, + RunStepOptions(fail_fast=False), + ), + ( + "Nested parallel steps execute properly", + [ + [StepToRun(id="step1", step=TestStep(test_context))], + [ + [StepToRun(id="step2", step=TestStep(test_context))], + [StepToRun(id="step3", step=TestStep(test_context))], + [ + StepToRun(id="step4", step=TestStep(test_context)), + StepToRun(id="step5", step=TestStep(test_context)), + ], + ], + [StepToRun(id="step6", step=TestStep(test_context))], + ], + { + "step1": StepStatus.SUCCESS, + "step2": StepStatus.SUCCESS, + "step3": StepStatus.SUCCESS, + "step4": StepStatus.SUCCESS, + "step5": StepStatus.SUCCESS, + "step6": StepStatus.SUCCESS, + }, + RunStepOptions(fail_fast=True), + ), + ( + "When fail_fast is True, nested parallel steps skip at the first failure", + [ + [StepToRun(id="step1", step=TestStep(test_context))], + [ + [StepToRun(id="step2", step=TestStep(test_context))], + [StepToRun(id="step3", step=TestStep(test_context))], + [ + StepToRun(id="step4", step=TestStep(test_context)), + StepToRun(id="step5", step=TestStep(test_context), args={"result_status": StepStatus.FAILURE}), + ], + ], + [StepToRun(id="step6", step=TestStep(test_context))], + ], + { + "step1": StepStatus.SUCCESS, + "step2": StepStatus.SUCCESS, + "step3": StepStatus.SUCCESS, + "step4": StepStatus.SUCCESS, + "step5": StepStatus.FAILURE, + "step6": StepStatus.SKIPPED, + }, + RunStepOptions(fail_fast=True), + ), + ( + "When fail_fast is False, nested parallel steps do not skip at the first failure", + [ + [StepToRun(id="step1", step=TestStep(test_context))], + [ + [StepToRun(id="step2", step=TestStep(test_context))], + [StepToRun(id="step3", step=TestStep(test_context))], + [ + StepToRun(id="step4", step=TestStep(test_context)), + StepToRun(id="step5", step=TestStep(test_context), args={"result_status": StepStatus.FAILURE}), + ], + ], + [StepToRun(id="step6", step=TestStep(test_context))], + ], + { + "step1": StepStatus.SUCCESS, + "step2": StepStatus.SUCCESS, + "step3": StepStatus.SUCCESS, + "step4": StepStatus.SUCCESS, + "step5": StepStatus.FAILURE, + "step6": StepStatus.SUCCESS, + }, + RunStepOptions(fail_fast=False), + ), + ( + "When fail_fast is False, consecutive steps still operate as expected", + [ + StepToRun(id="step1", step=TestStep(test_context)), + StepToRun(id="step2", step=TestStep(test_context)), + StepToRun(id="step3", step=TestStep(test_context)), + StepToRun(id="step4", step=TestStep(test_context)), + ], + {"step1": StepStatus.SUCCESS, "step2": StepStatus.SUCCESS, "step3": StepStatus.SUCCESS, "step4": StepStatus.SUCCESS}, + RunStepOptions(fail_fast=False), + ), + ( + "skip_steps skips the specified steps", + [ + StepToRun(id="step1", step=TestStep(test_context)), + StepToRun(id="step2", step=TestStep(test_context)), + StepToRun(id="step3", step=TestStep(test_context)), + StepToRun(id="step4", step=TestStep(test_context)), + ], + {"step1": StepStatus.SUCCESS, "step2": StepStatus.SKIPPED, "step3": StepStatus.SUCCESS, "step4": StepStatus.SUCCESS}, + RunStepOptions(fail_fast=False, skip_steps=["step2"]), + ), + ( + "step is skipped if the dependency fails", + [ + [StepToRun(id="step1", step=TestStep(test_context))], + [StepToRun(id="step2", step=TestStep(test_context), args={"result_status": StepStatus.FAILURE})], + [StepToRun(id="step3", step=TestStep(test_context), depends_on=["step2"])], + ], + {"step1": StepStatus.SUCCESS, "step2": StepStatus.FAILURE, "step3": StepStatus.SKIPPED}, + RunStepOptions(fail_fast=False), + ), + ], +) +async def test_run_steps_output(desc, steps, expected_results, options): + results = await run_steps(steps, options=options) + + for step_id, expected_status in expected_results.items(): + assert results[step_id].status == expected_status, desc + + +@pytest.mark.anyio +async def test_run_steps_throws_on_invalid_order(): + concurrent_steps = [ + StepToRun(id="step1", step=TestStep(test_context)), + StepToRun(id="step2", step=TestStep(test_context), depends_on=["step1"]), + ] + + with pytest.raises(InvalidStepConfiguration): + await run_steps(concurrent_steps) + + +@pytest.mark.anyio +async def test_run_steps_concurrent(): + ran_at = {} + + class SleepStep(Step): + title = "Sleep Step" + + async def _run(self, name, sleep) -> StepResult: + await anyio.sleep(sleep) + ran_at[name] = time.time() + return StepResult(self, StepStatus.SUCCESS) + + steps = [ + StepToRun(id="step1", step=SleepStep(test_context), args={"name": "step1", "sleep": 2}), + StepToRun(id="step2", step=SleepStep(test_context), args={"name": "step2", "sleep": 2}), + StepToRun(id="step3", step=SleepStep(test_context), args={"name": "step3", "sleep": 2}), + StepToRun(id="step4", step=SleepStep(test_context), args={"name": "step4", "sleep": 0}), + ] + + await run_steps(steps) + + # assert that step4 is the first step to finish + assert ran_at["step4"] < ran_at["step1"] + assert ran_at["step4"] < ran_at["step2"] + assert ran_at["step4"] < ran_at["step3"] + + +@pytest.mark.anyio +async def test_run_steps_concurrency_of_1(): + ran_at = {} + + class SleepStep(Step): + title = "Sleep Step" + + async def _run(self, name, sleep) -> StepResult: + ran_at[name] = time.time() + await anyio.sleep(sleep) + return StepResult(self, StepStatus.SUCCESS) + + steps = [ + StepToRun(id="step1", step=SleepStep(test_context), args={"name": "step1", "sleep": 1}), + StepToRun(id="step2", step=SleepStep(test_context), args={"name": "step2", "sleep": 1}), + StepToRun(id="step3", step=SleepStep(test_context), args={"name": "step3", "sleep": 1}), + StepToRun(id="step4", step=SleepStep(test_context), args={"name": "step4", "sleep": 1}), + ] + + await run_steps(steps, options=RunStepOptions(concurrency=1)) + + # Assert that they run sequentially + assert ran_at["step1"] < ran_at["step2"] + assert ran_at["step2"] < ran_at["step3"] + assert ran_at["step3"] < ran_at["step4"] + + +@pytest.mark.anyio +async def test_run_steps_sequential(): + ran_at = {} + + class SleepStep(Step): + title = "Sleep Step" + + async def _run(self, name, sleep) -> StepResult: + await anyio.sleep(sleep) + ran_at[name] = time.time() + return StepResult(self, StepStatus.SUCCESS) + + steps = [ + [StepToRun(id="step1", step=SleepStep(test_context), args={"name": "step1", "sleep": 1})], + [StepToRun(id="step1", step=SleepStep(test_context), args={"name": "step2", "sleep": 1})], + [StepToRun(id="step3", step=SleepStep(test_context), args={"name": "step3", "sleep": 1})], + [StepToRun(id="step4", step=SleepStep(test_context), args={"name": "step4", "sleep": 0})], + ] + + await run_steps(steps) + + # assert that steps are run in order + assert ran_at["step1"] < ran_at["step2"] + assert ran_at["step2"] < ran_at["step3"] + assert ran_at["step3"] < ran_at["step4"] + + +@pytest.mark.anyio +async def test_run_steps_passes_results(): + """ + Example pattern + StepToRun( + id=CONNECTOR_TEST_STEP_ID.INTEGRATION, + step=IntegrationTests(context), + args=_create_integration_step_args_factory(context), + depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + ), + StepToRun( + id=CONNECTOR_TEST_STEP_ID.ACCEPTANCE, + step=AcceptanceTests(context, True), + args=lambda results: {"connector_under_test_container": results[CONNECTOR_TEST_STEP_ID.BUILD].output_artifact[LOCAL_BUILD_PLATFORM]}, + depends_on=[CONNECTOR_TEST_STEP_ID.BUILD], + ), + + """ + + class Simple(Step): + title = "Test Step" + + async def _run(self, arg1, arg2) -> StepResult: + output_artifact = f"{arg1}:{arg2}" + return StepResult(self, StepStatus.SUCCESS, output_artifact=output_artifact) + + async def async_args(results): + return {"arg1": results["step2"].output_artifact, "arg2": "4"} + + steps = [ + [StepToRun(id="step1", step=Simple(test_context), args={"arg1": "1", "arg2": "2"})], + [StepToRun(id="step2", step=Simple(test_context), args=lambda results: {"arg1": results["step1"].output_artifact, "arg2": "3"})], + [StepToRun(id="step3", step=Simple(test_context), args=async_args)], + ] + + results = await run_steps(steps) + + assert results["step1"].output_artifact == "1:2" + assert results["step2"].output_artifact == "1:2:3" + assert results["step3"].output_artifact == "1:2:3:4" + + +@pytest.mark.anyio +@pytest.mark.parametrize( + "invalid_args", + [ + 1, + True, + "string", + [1, 2], + None, + ], +) +async def test_run_steps_throws_on_invalid_args(invalid_args): + steps = [ + [StepToRun(id="step1", step=TestStep(test_context), args=invalid_args)], + ] + + with pytest.raises(TypeError): + await run_steps(steps) + + +@pytest.mark.anyio +async def test_run_steps_with_params(): + steps = [StepToRun(id="step1", step=TestStep(test_context))] + options = RunStepOptions(fail_fast=True, step_params={"step1": {"--param1": ["value1"]}}) + TestStep.accept_extra_params = False + with pytest.raises(ValueError): + await run_steps(steps, options=options) + assert steps[0].step.params_as_cli_options == [] + TestStep.accept_extra_params = True + await run_steps(steps, options=options) + assert steps[0].step.params_as_cli_options == ["--param1=value1"] + + +class TestRunStepOptions: + def test_init(self): + options = RunStepOptions() + assert options.fail_fast is True + assert options.concurrency == 10 + assert options.skip_steps == [] + assert options.step_params == {} + + options = RunStepOptions(fail_fast=False, concurrency=1, skip_steps=["step1"], step_params={"step1": {"--param1": ["value1"]}}) + assert options.fail_fast is False + assert options.concurrency == 1 + assert options.skip_steps == ["step1"] + assert options.step_params == {"step1": {"--param1": ["value1"]}} + + with pytest.raises(ValueError): + RunStepOptions(skip_steps=["step1"], keep_steps=["step2"]) + + @pytest.mark.parametrize( + "step_tree, options, expected_skipped_ids", + [ + ( + [ + [StepToRun(id="step1", step=TestStep(test_context)), StepToRun(id="step2", step=TestStep(test_context))], + StepToRun(id="step3", step=TestStep(test_context)), + StepToRun(id="step4", step=TestStep(test_context), depends_on=["step3", "step1"]), + StepToRun(id="step5", step=TestStep(test_context)), + ], + RunStepOptions(keep_steps=["step4"]), + {"step2", "step5"}, + ), + ( + [ + [StepToRun(id="step1", step=TestStep(test_context)), StepToRun(id="step2", step=TestStep(test_context))], + StepToRun(id="step3", step=TestStep(test_context)), + [ + StepToRun(id="step4", step=TestStep(test_context), depends_on=["step1"]), + StepToRun(id="step6", step=TestStep(test_context), depends_on=["step4", "step5"]), + ], + StepToRun(id="step5", step=TestStep(test_context), depends_on=["step3"]), + ], + RunStepOptions(keep_steps=["step6"]), + {"step2"}, + ), + ( + [ + [StepToRun(id="step1", step=TestStep(test_context)), StepToRun(id="step2", step=TestStep(test_context))], + StepToRun(id="step3", step=TestStep(test_context)), + [ + StepToRun(id="step4", step=TestStep(test_context), depends_on=["step1"]), + StepToRun(id="step6", step=TestStep(test_context), depends_on=["step4", "step5"]), + ], + StepToRun(id="step5", step=TestStep(test_context), depends_on=["step3"]), + ], + RunStepOptions(skip_steps=["step1"]), + {"step1"}, + ), + ], + ) + def test_get_step_ids_to_skip(self, step_tree, options, expected_skipped_ids): + skipped_ids = options.get_step_ids_to_skip(step_tree) + assert set(skipped_ids) == expected_skipped_ids diff --git a/airbyte-ci/connectors/pipelines/tests/test_helpers/test_utils.py b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_utils.py new file mode 100644 index 000000000000..f63bd000bbc7 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_helpers/test_utils.py @@ -0,0 +1,237 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from pathlib import Path +from unittest import mock + +import dagger +import pytest +from connector_ops.utils import Connector, ConnectorLanguage +from pipelines import consts +from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand +from pipelines.helpers import utils +from pipelines.helpers.connectors.modifed import get_connector_modified_files, get_modified_connectors +from tests.utils import pick_a_random_connector + + +@pytest.mark.parametrize( + "ctx, expected", + [ + ( + mock.MagicMock( + command_path="my command path", + obj={ + "git_branch": "my_branch", + "git_revision": "my_git_revision", + "pipeline_start_timestamp": "my_pipeline_start_timestamp", + "ci_context": "my_ci_context", + "ci_job_key": None, + }, + ), + f"{consts.STATIC_REPORT_PREFIX}/command/path/my_ci_context/my_branch/my_pipeline_start_timestamp/my_git_revision", + ), + ( + mock.MagicMock( + command_path="my command path", + obj={ + "git_branch": "my_branch", + "git_revision": "my_git_revision", + "pipeline_start_timestamp": "my_pipeline_start_timestamp", + "ci_context": "my_ci_context", + "ci_job_key": "my_ci_job_key", + }, + ), + f"{consts.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch/my_pipeline_start_timestamp/my_git_revision", + ), + ( + mock.MagicMock( + command_path="my command path", + obj={ + "git_branch": "my_branch", + "git_revision": "my_git_revision", + "pipeline_start_timestamp": "my_pipeline_start_timestamp", + "ci_context": "my_ci_context", + "ci_job_key": "my_ci_job_key", + }, + ), + f"{consts.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch/my_pipeline_start_timestamp/my_git_revision", + ), + ( + mock.MagicMock( + command_path="my command path", + obj={ + "git_branch": "my_branch", + "git_revision": "my_git_revision", + "pipeline_start_timestamp": "my_pipeline_start_timestamp", + "ci_context": "my_ci_context", + "ci_job_key": "my_ci_job_key", + }, + ), + f"{consts.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch/my_pipeline_start_timestamp/my_git_revision", + ), + ( + mock.MagicMock( + command_path="my command path", + obj={ + "git_branch": "my_branch/with/slashes", + "git_revision": "my_git_revision", + "pipeline_start_timestamp": "my_pipeline_start_timestamp", + "ci_context": "my_ci_context", + "ci_job_key": "my_ci_job_key", + }, + ), + f"{consts.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch_with_slashes/my_pipeline_start_timestamp/my_git_revision", + ), + ( + mock.MagicMock( + command_path="my command path", + obj={ + "git_branch": "my_branch/with/slashes#and!special@characters", + "git_revision": "my_git_revision", + "pipeline_start_timestamp": "my_pipeline_start_timestamp", + "ci_context": "my_ci_context", + "ci_job_key": "my_ci_job_key", + }, + ), + f"{consts.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch_with_slashesandspecialcharacters/my_pipeline_start_timestamp/my_git_revision", + ), + ( + mock.MagicMock( + command_path="airbyte-ci command path", + obj={ + "git_branch": "my_branch/with/slashes#and!special@characters", + "git_revision": "my_git_revision", + "pipeline_start_timestamp": "my_pipeline_start_timestamp", + "ci_context": "my_ci_context", + "ci_job_key": "my_ci_job_key", + }, + ), + f"{consts.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch_with_slashesandspecialcharacters/my_pipeline_start_timestamp/my_git_revision", + ), + ( + mock.MagicMock( + command_path="airbyte-ci-internal command path", + obj={ + "git_branch": "my_branch/with/slashes#and!special@characters", + "git_revision": "my_git_revision", + "pipeline_start_timestamp": "my_pipeline_start_timestamp", + "ci_context": "my_ci_context", + "ci_job_key": "my_ci_job_key", + }, + ), + f"{consts.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch_with_slashesandspecialcharacters/my_pipeline_start_timestamp/my_git_revision", + ), + ], +) +def test_render_report_output_prefix(ctx, expected): + assert DaggerPipelineCommand.render_report_output_prefix(ctx) == expected + + +@pytest.mark.parametrize("enable_dependency_scanning", [True, False]) +def test_get_modified_connectors_with_dependency_scanning(all_connectors, enable_dependency_scanning): + base_java_changed_file = Path("airbyte-cdk/java/airbyte-cdk/core/src/main/java/io/airbyte/cdk/integrations/BaseConnector.java") + modified_files = [base_java_changed_file] + + not_modified_java_connector = pick_a_random_connector(language=ConnectorLanguage.JAVA) + modified_java_connector = pick_a_random_connector( + language=ConnectorLanguage.JAVA, other_picked_connectors=[not_modified_java_connector] + ) + modified_files.append(modified_java_connector.code_directory / "foo.bar") + + modified_connectors = get_modified_connectors(modified_files, all_connectors, enable_dependency_scanning) + if enable_dependency_scanning: + assert not_modified_java_connector in modified_connectors + else: + assert not_modified_java_connector not in modified_connectors + assert modified_java_connector in modified_connectors + + +def test_get_connector_modified_files(): + connector = pick_a_random_connector() + other_connector = pick_a_random_connector(other_picked_connectors=[connector]) + + all_modified_files = { + connector.code_directory / "setup.py", + other_connector.code_directory / "README.md", + } + + result = get_connector_modified_files(connector, all_modified_files) + assert result == frozenset({connector.code_directory / "setup.py"}) + + +def test_no_modified_files_in_connector_directory(): + connector = pick_a_random_connector() + other_connector = pick_a_random_connector(other_picked_connectors=[connector]) + + all_modified_files = { + other_connector.code_directory / "README.md", + } + + result = get_connector_modified_files(connector, all_modified_files) + assert result == frozenset() + + +@pytest.mark.anyio +async def test_check_path_in_workdir(dagger_client): + connector = Connector("source-openweather") + container = ( + dagger_client.container() + .from_("bash") + .with_mounted_directory(str(connector.code_directory), dagger_client.host().directory(str(connector.code_directory))) + .with_workdir(str(connector.code_directory)) + ) + assert await utils.check_path_in_workdir(container, "metadata.yaml") + assert await utils.check_path_in_workdir(container, "setup.py") + assert await utils.check_path_in_workdir(container, "requirements.txt") + assert await utils.check_path_in_workdir(container, "not_existing_file") is False + + +def test_sh_dash_c(): + assert utils.sh_dash_c(["foo", "bar"]) == ["sh", "-c", "set -o xtrace && foo && bar"] + assert utils.sh_dash_c(["foo"]) == ["sh", "-c", "set -o xtrace && foo"] + assert utils.sh_dash_c([]) == ["sh", "-c", "set -o xtrace"] + + +@pytest.mark.anyio +@pytest.mark.parametrize("tar_file_name", [None, "custom_tar_name.tar"]) +async def test_export_container_to_tarball(mocker, dagger_client, tmp_path, tar_file_name): + context = mocker.Mock( + dagger_client=dagger_client, + connector=mocker.Mock(technical_name="my_connector"), + host_image_export_dir_path=tmp_path, + git_revision="my_git_revision", + ) + container = dagger_client.container().from_("bash:latest") + platform = consts.LOCAL_BUILD_PLATFORM + + expected_tar_file_path = ( + tmp_path / f"my_connector_my_git_revision_{platform.replace('/', '_')}.tar" if tar_file_name is None else tmp_path / tar_file_name + ) + exported_tar_file, exported_tar_file_path = await utils.export_container_to_tarball( + context, container, platform, tar_file_name=tar_file_name + ) + assert exported_tar_file_path == expected_tar_file_path + assert await exported_tar_file.size() == expected_tar_file_path.stat().st_size + + +@pytest.mark.anyio +async def test_export_container_to_tarball_failure(mocker, tmp_path): + + context = mocker.Mock( + connector=mocker.Mock(technical_name="my_connector"), + host_image_export_dir_path=tmp_path, + git_revision="my_git_revision", + ) + + mock_export = mocker.AsyncMock(return_value=False) + container = mocker.AsyncMock(export=mock_export) + platform = consts.LOCAL_BUILD_PLATFORM + exported_tar_file, exported_tar_file_path = await utils.export_container_to_tarball(context, container, platform) + assert exported_tar_file is None + assert exported_tar_file_path is None + + mock_export.assert_called_once_with( + str(tmp_path / f"my_connector_my_git_revision_{platform.replace('/', '_')}.tar"), + forced_compression=dagger.ImageLayerCompression.Gzip, + ) diff --git a/airbyte-ci/connectors/pipelines/tests/test_models/__init__.py b/airbyte-ci/connectors/pipelines/tests/test_models/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_models/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py b/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py new file mode 100644 index 000000000000..4efb8b9e7b0a --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_models/test_click_pipeline_context.py @@ -0,0 +1,71 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from unittest.mock import patch + +import asyncclick as click +import dagger +import pytest +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext + + +@pytest.mark.anyio +async def test_get_dagger_client_singleton(dagger_connection): + @click.command() + def cli(): + pass + + ctx = click.Context(cli) + ctx.obj = {"foo": "bar"} + ctx.params = {"baz": "qux"} + + async with ctx.scope(): + click_pipeline_context = ClickPipelineContext() + with patch("pipelines.models.contexts.click_pipeline_context.dagger.Connection", lambda _x: dagger_connection): + client1 = await click_pipeline_context.get_dagger_client() + client2 = await click_pipeline_context.get_dagger_client() + client3 = await click_pipeline_context.get_dagger_client(pipeline_name="pipeline_name") + assert isinstance(client1, dagger.Client) + assert isinstance(client2, dagger.Client) + assert isinstance(client3, dagger.Client) + + assert client1 == client2 + assert client1 != client3 + + +@pytest.mark.anyio +async def test_get_dagger_client_click_params(dagger_connection): + @click.command() + def cli(): + pass + + given_click_obj = {"foo": "bar"} + given_click_params = {"baz": "qux"} + + ctx = click.Context(cli, obj=given_click_obj) + ctx.params = given_click_params + + async with ctx.scope(): + click_pipeline_context = ClickPipelineContext() + with patch("pipelines.models.contexts.click_pipeline_context.dagger.Connection", lambda _x: dagger_connection): + pipeline_context_params = click_pipeline_context.params + assert pipeline_context_params == {**given_click_obj, **given_click_params} + + +@pytest.mark.anyio +async def test_get_dagger_client_click_params_duplicate(dagger_connection): + @click.command() + def cli(): + pass + + given_click_obj = {"foo": "bar"} + given_click_params = {"foo": "qux"} + + ctx = click.Context(cli, obj=given_click_obj) + ctx.params = given_click_params + ctx.command.params = [click.Option(["--foo"])] + + async with ctx.scope(): + click_pipeline_context = ClickPipelineContext() + with patch("pipelines.models.contexts.click_pipeline_context.dagger.Connection", lambda _x: dagger_connection): + with pytest.raises(ValueError): + click_pipeline_context.params diff --git a/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py b/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py new file mode 100644 index 000000000000..87648f432a12 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_models/test_singleton.py @@ -0,0 +1,31 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from pipelines.models.singleton import Singleton + + +class SingletonChild(Singleton): + def __init__(self): + if not self._initialized[self.__class__]: + self.value = "initialized" + self._initialized[self.__class__] = True + + +def test_singleton_instance(): + instance1 = SingletonChild() + instance2 = SingletonChild() + assert instance1 is instance2 + + +def test_singleton_unique_per_subclass(): + class AnotherSingletonChild(Singleton): + pass + + instance1 = SingletonChild() + instance2 = AnotherSingletonChild() + assert instance1 is not instance2 + + +def test_singleton_initialized(): + instance = SingletonChild() + instance.value # This should initialize the instance + assert instance._initialized[SingletonChild] diff --git a/airbyte-ci/connectors/pipelines/tests/test_publish.py b/airbyte-ci/connectors/pipelines/tests/test_publish.py index 9bcf38a9bfcd..b7fe7d764d5f 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_publish.py +++ b/airbyte-ci/connectors/pipelines/tests/test_publish.py @@ -1,15 +1,14 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - import json import random from typing import List import anyio import pytest -from pipelines import publish -from pipelines.bases import StepStatus +from pipelines.airbyte_ci.connectors.publish import pipeline as publish_pipeline +from pipelines.models.steps import StepStatus pytestmark = [ pytest.mark.anyio, @@ -38,13 +37,13 @@ async def test_run_skipped_when_already_published(self, three_random_connectors_ """We pick three random connectors from the OSS registry. They should be published. We check that the step is skipped.""" for image_name in three_random_connectors_image_names: publish_context.docker_image = image_name - step = publish.CheckConnectorImageDoesNotExist(publish_context) + step = publish_pipeline.CheckConnectorImageDoesNotExist(publish_context) step_result = await step.run() assert step_result.status == StepStatus.SKIPPED async def test_run_success_when_already_published(self, publish_context): publish_context.docker_image = "airbyte/source-pokeapi:0.0.0" - step = publish.CheckConnectorImageDoesNotExist(publish_context) + step = publish_pipeline.CheckConnectorImageDoesNotExist(publish_context) step_result = await step.run() assert step_result.status == StepStatus.SUCCESS @@ -79,17 +78,21 @@ async def test_run(self, mocker, dagger_client, valid_spec, successful_upload, r upload_exit_code = 0 if successful_upload else 1 mocker.patch.object( - publish, "upload_to_gcs", mocker.AsyncMock(return_value=(upload_exit_code, "upload_to_gcs_stdout", "upload_to_gcs_stderr")) + publish_pipeline, + "upload_to_gcs", + mocker.AsyncMock(return_value=(upload_exit_code, "upload_to_gcs_stdout", "upload_to_gcs_stderr")), ) if not valid_spec: mocker.patch.object( - publish.UploadSpecToCache, "_get_connector_spec", mocker.Mock(side_effect=publish.InvalidSpecOutputError("Invalid spec.")) + publish_pipeline.UploadSpecToCache, + "_get_connector_spec", + mocker.Mock(side_effect=publish_pipeline.InvalidSpecOutputError("Invalid spec.")), ) - step = publish.UploadSpecToCache(publish_context) + step = publish_pipeline.UploadSpecToCache(publish_context) step_result = await step.run(connector_container) if valid_spec: - publish.upload_to_gcs.assert_called_once_with( + publish_pipeline.upload_to_gcs.assert_called_once_with( publish_context.dagger_client, mocker.ANY, f"specs/{image_name.replace(':', '/')}/spec.json", @@ -98,7 +101,7 @@ async def test_run(self, mocker, dagger_client, valid_spec, successful_upload, r flags=['--cache-control="no-cache"'], ) - spec_file = publish.upload_to_gcs.call_args.args[1] + spec_file = publish_pipeline.upload_to_gcs.call_args.args[1] uploaded_content = await spec_file.contents() assert json.loads(uploaded_content) == expected_spec @@ -114,42 +117,42 @@ async def test_run(self, mocker, dagger_client, valid_spec, successful_upload, r assert step_result.status == StepStatus.FAILURE assert step_result.stderr == "Invalid spec." assert step_result.stdout is None - publish.upload_to_gcs.assert_not_called() + publish_pipeline.upload_to_gcs.assert_not_called() def test_parse_spec_output_valid(self, publish_context, random_connector): - step = publish.UploadSpecToCache(publish_context) + step = publish_pipeline.UploadSpecToCache(publish_context) correct_spec_message = json.dumps({"type": "SPEC", "spec": random_connector["spec"]}) spec_output = f'random_stuff\n{{"type": "RANDOM_MESSAGE"}}\n{correct_spec_message}' result = step._parse_spec_output(spec_output) assert json.loads(result) == random_connector["spec"] def test_parse_spec_output_invalid_json(self, publish_context): - step = publish.UploadSpecToCache(publish_context) + step = publish_pipeline.UploadSpecToCache(publish_context) spec_output = "Invalid JSON" - with pytest.raises(publish.InvalidSpecOutputError): + with pytest.raises(publish_pipeline.InvalidSpecOutputError): step._parse_spec_output(spec_output) def test_parse_spec_output_invalid_key(self, publish_context): - step = publish.UploadSpecToCache(publish_context) + step = publish_pipeline.UploadSpecToCache(publish_context) spec_output = '{"type": "SPEC", "spec": {"invalid_key": "value"}}' - with pytest.raises(publish.InvalidSpecOutputError): + with pytest.raises(publish_pipeline.InvalidSpecOutputError): step._parse_spec_output(spec_output) def test_parse_spec_output_no_spec(self, publish_context): - step = publish.UploadSpecToCache(publish_context) + step = publish_pipeline.UploadSpecToCache(publish_context) spec_output = '{"type": "OTHER"}' - with pytest.raises(publish.InvalidSpecOutputError): + with pytest.raises(publish_pipeline.InvalidSpecOutputError): step._parse_spec_output(spec_output) STEPS_TO_PATCH = [ - (publish.metadata, "MetadataValidation"), - (publish.metadata, "MetadataUpload"), - (publish, "CheckConnectorImageDoesNotExist"), - (publish, "UploadSpecToCache"), - (publish, "PushConnectorImageToRegistry"), - (publish, "PullConnectorImageFromRegistry"), - (publish.builds, "run_connector_build"), + (publish_pipeline, "MetadataValidation"), + (publish_pipeline, "MetadataUpload"), + (publish_pipeline, "CheckConnectorImageDoesNotExist"), + (publish_pipeline, "UploadSpecToCache"), + (publish_pipeline, "PushConnectorImageToRegistry"), + (publish_pipeline, "PullConnectorImageFromRegistry"), + (publish_pipeline.steps, "run_connector_build"), ] @@ -159,12 +162,12 @@ async def test_run_connector_publish_pipeline_when_failed_validation(mocker, pre for module, to_mock in STEPS_TO_PATCH: mocker.patch.object(module, to_mock, return_value=mocker.AsyncMock()) - run_metadata_validation = publish.metadata.MetadataValidation.return_value.run + run_metadata_validation = publish_pipeline.MetadataValidation.return_value.run run_metadata_validation.return_value = mocker.Mock(status=StepStatus.FAILURE) context = mocker.MagicMock(pre_release=pre_release) semaphore = anyio.Semaphore(1) - report = await publish.run_connector_publish_pipeline(context, semaphore) + report = await publish_pipeline.run_connector_publish_pipeline(context, semaphore) run_metadata_validation.assert_called_once() # Check that nothing else is called @@ -195,20 +198,20 @@ async def test_run_connector_publish_pipeline_when_image_exists_or_failed(mocker for module, to_mock in STEPS_TO_PATCH: mocker.patch.object(module, to_mock, return_value=mocker.AsyncMock()) - run_metadata_validation = publish.metadata.MetadataValidation.return_value.run + run_metadata_validation = publish_pipeline.MetadataValidation.return_value.run run_metadata_validation.return_value = mocker.Mock(status=StepStatus.SUCCESS) # ensure spec always succeeds - run_upload_spec_to_cache = publish.UploadSpecToCache.return_value.run + run_upload_spec_to_cache = publish_pipeline.UploadSpecToCache.return_value.run run_upload_spec_to_cache.return_value = mocker.Mock(status=StepStatus.SUCCESS) - run_check_connector_image_does_not_exist = publish.CheckConnectorImageDoesNotExist.return_value.run + run_check_connector_image_does_not_exist = publish_pipeline.CheckConnectorImageDoesNotExist.return_value.run run_check_connector_image_does_not_exist.return_value = mocker.Mock(status=check_image_exists_status) - run_metadata_upload = publish.metadata.MetadataUpload.return_value.run + run_metadata_upload = publish_pipeline.MetadataUpload.return_value.run semaphore = anyio.Semaphore(1) - report = await publish.run_connector_publish_pipeline(publish_context, semaphore) + report = await publish_pipeline.run_connector_publish_pipeline(publish_context, semaphore) run_metadata_validation.assert_called_once() run_check_connector_image_does_not_exist.assert_called_once() @@ -266,10 +269,10 @@ async def test_run_connector_publish_pipeline_when_image_does_not_exist( """We check that the full pipeline is executed as expected when the connector image does not exist and the metadata validation passed.""" for module, to_mock in STEPS_TO_PATCH: mocker.patch.object(module, to_mock, return_value=mocker.AsyncMock()) - publish.metadata.MetadataValidation.return_value.run.return_value = mocker.Mock( + publish_pipeline.MetadataValidation.return_value.run.return_value = mocker.Mock( name="metadata_validation_result", status=StepStatus.SUCCESS ) - publish.CheckConnectorImageDoesNotExist.return_value.run.return_value = mocker.Mock( + publish_pipeline.CheckConnectorImageDoesNotExist.return_value.run.return_value = mocker.Mock( name="check_connector_image_does_not_exist_result", status=StepStatus.SUCCESS ) @@ -277,22 +280,22 @@ async def test_run_connector_publish_pipeline_when_image_does_not_exist( built_connector_platform = mocker.Mock() built_connector_platform.values.return_value = ["linux/amd64"] - publish.builds.run_connector_build.return_value = mocker.Mock( + publish_pipeline.steps.run_connector_build.return_value = mocker.Mock( name="build_connector_for_publish_result", status=build_step_status, output_artifact=built_connector_platform ) - publish.PushConnectorImageToRegistry.return_value.run.return_value = mocker.Mock( + publish_pipeline.PushConnectorImageToRegistry.return_value.run.return_value = mocker.Mock( name="push_connector_image_to_registry_result", status=push_step_status ) - publish.PullConnectorImageFromRegistry.return_value.run.return_value = mocker.Mock( + publish_pipeline.PullConnectorImageFromRegistry.return_value.run.return_value = mocker.Mock( name="pull_connector_image_from_registry_result", status=pull_step_status ) - publish.UploadSpecToCache.return_value.run.return_value = mocker.Mock( + publish_pipeline.UploadSpecToCache.return_value.run.return_value = mocker.Mock( name="upload_spec_to_cache_result", status=upload_to_spec_cache_step_status ) - publish.metadata.MetadataUpload.return_value.run.return_value = mocker.Mock( + publish_pipeline.MetadataUpload.return_value.run.return_value = mocker.Mock( name="metadata_upload_result", status=metadata_upload_step_status ) @@ -300,14 +303,14 @@ async def test_run_connector_publish_pipeline_when_image_does_not_exist( pre_release=pre_release, ) semaphore = anyio.Semaphore(1) - report = await publish.run_connector_publish_pipeline(context, semaphore) + report = await publish_pipeline.run_connector_publish_pipeline(context, semaphore) steps_to_run = [ - publish.metadata.MetadataValidation.return_value.run, - publish.CheckConnectorImageDoesNotExist.return_value.run, - publish.builds.run_connector_build, - publish.PushConnectorImageToRegistry.return_value.run, - publish.PullConnectorImageFromRegistry.return_value.run, + publish_pipeline.MetadataValidation.return_value.run, + publish_pipeline.CheckConnectorImageDoesNotExist.return_value.run, + publish_pipeline.steps.run_connector_build, + publish_pipeline.PushConnectorImageToRegistry.return_value.run, + publish_pipeline.PullConnectorImageFromRegistry.return_value.run, ] for i, step_to_run in enumerate(steps_to_run): @@ -324,9 +327,9 @@ async def test_run_connector_publish_pipeline_when_image_does_not_exist( step_to_run.assert_not_called() break if build_step_status is StepStatus.SUCCESS: - publish.PushConnectorImageToRegistry.return_value.run.assert_called_once_with(["linux/amd64"]) + publish_pipeline.PushConnectorImageToRegistry.return_value.run.assert_called_once_with(["linux/amd64"]) else: - publish.PushConnectorImageToRegistry.return_value.run.assert_not_called() - publish.PullConnectorImageFromRegistry.return_value.run.assert_not_called() - publish.UploadSpecToCache.return_value.run.assert_not_called() - publish.metadata.MetadataUpload.return_value.run.assert_not_called() + publish_pipeline.PushConnectorImageToRegistry.return_value.run.assert_not_called() + publish_pipeline.PullConnectorImageFromRegistry.return_value.run.assert_not_called() + publish_pipeline.UploadSpecToCache.return_value.run.assert_not_called() + publish_pipeline.MetadataUpload.return_value.run.assert_not_called() diff --git a/airbyte-ci/connectors/pipelines/tests/test_steps/__init__.py b/airbyte-ci/connectors/pipelines/tests/test_steps/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_steps/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/tests/test_steps/test_simple_docker_step.py b/airbyte-ci/connectors/pipelines/tests/test_steps/test_simple_docker_step.py new file mode 100644 index 000000000000..01b83f561e1f --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_steps/test_simple_docker_step.py @@ -0,0 +1,98 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from pathlib import Path + +import pytest +from pipelines.airbyte_ci.steps.docker import SimpleDockerStep +from pipelines.helpers.utils import get_exec_result +from pipelines.models.contexts.pipeline_context import PipelineContext +from pipelines.models.steps import MountPath + +pytestmark = [ + pytest.mark.anyio, +] + + +@pytest.fixture +def context(dagger_client): + context = PipelineContext( + pipeline_name="test", + is_local=True, + git_branch="test", + git_revision="test", + report_output_prefix="test", + ) + context.dagger_client = dagger_client + return context + + +class TestSimpleDockerStep: + async def test_env_variables_set(self, context): + # Define test inputs + title = "test_env_variables_set" + env_variables = {"VAR1": "value1", "VAR2": "value2"} + + # Create SimpleDockerStep instance + step = SimpleDockerStep(title=title, context=context, env_variables=env_variables) + + # Initialize container + container = await step.init_container() + + # Check if environment variables are set + for key, expected_value in env_variables.items(): + stdout_value = await container.with_exec(["printenv", key]).stdout() + actual_value = stdout_value.strip() + assert actual_value == expected_value + + async def test_mount_paths(self, context): + # Define test inputs + title = "test_mount_paths" + + path_to_current_file = Path(__file__).relative_to(Path.cwd()) + invalid_path = Path("invalid_path") + paths_to_mount = [ + MountPath(path=path_to_current_file, optional=False), + MountPath(path=invalid_path, optional=True), + ] + + # Create SimpleDockerStep instance + step = SimpleDockerStep(title=title, context=context, paths_to_mount=paths_to_mount) + + # Initialize container + container = await step.init_container() + + for path_to_mount in paths_to_mount: + exit_code, _stdout, _stderr = await get_exec_result(container.with_exec(["test", "-f", f"{str(path_to_mount)}"])) + + expected_exit_code = 1 if path_to_mount.optional else 0 + assert exit_code == expected_exit_code + + async def test_invalid_mount_paths(self): + path_to_current_file = Path(__file__).relative_to(Path.cwd()) + invalid_path = Path("invalid_path") + + # No errors expected + MountPath(path=path_to_current_file, optional=False) + MountPath(path=invalid_path, optional=True) + + # File not found error expected + with pytest.raises(FileNotFoundError): + MountPath(path=invalid_path, optional=False) + + async def test_work_dir(self, context): + # Define test inputs + title = "test_work_dir" + working_directory = "/test" + + # Create SimpleDockerStep instance + step = SimpleDockerStep(title=title, context=context, working_directory=working_directory) + + # Initialize container + container = await step.init_container() + + # Check if working directory is set + stdout_value = await container.with_exec(["pwd"]).stdout() + actual_value = stdout_value.strip() + assert actual_value == working_directory diff --git a/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py b/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py index 0bdcd5158c93..a9c1470b6dd7 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py +++ b/airbyte-ci/connectors/pipelines/tests/test_tests/test_common.py @@ -11,8 +11,11 @@ import pytest import yaml from freezegun import freeze_time -from pipelines.bases import ConnectorWithModifiedFiles, StepStatus -from pipelines.tests import common +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.connectors.test.steps import common +from pipelines.dagger.actions.system import docker +from pipelines.helpers.connectors.modifed import ConnectorWithModifiedFiles +from pipelines.models.steps import StepStatus pytestmark = [ pytest.mark.anyio, @@ -37,24 +40,31 @@ def get_dummy_cat_container(dagger_client: dagger.Client, exit_code: int, secret return container.with_new_file("/stupid_bash_script.sh", contents=f"echo {stdout}; echo {stderr} >&2; exit {exit_code}") @pytest.fixture - def test_context(self, mocker, dagger_client): - return mocker.MagicMock(connector=ConnectorWithModifiedFiles("source-faker", frozenset()), dagger_client=dagger_client) + def test_context_ci(self, current_platform, dagger_client): + context = ConnectorContext( + pipeline_name="test", + connector=ConnectorWithModifiedFiles("source-faker", frozenset()), + git_branch="test", + git_revision="test", + report_output_prefix="test", + is_local=False, + use_remote_secrets=True, + targeted_platforms=[current_platform], + ) + context.dagger_client = dagger_client + return context @pytest.fixture - def dummy_connector_under_test_image_tar(self, dagger_client, tmpdir) -> dagger.File: - dummy_tar_file = tmpdir / "dummy.tar" - dummy_tar_file.write_text("dummy", encoding="utf8") - return dagger_client.host().directory(str(tmpdir), include=["dummy.tar"]).file("dummy.tar") + def dummy_connector_under_test_container(self, dagger_client) -> dagger.Container: + return dagger_client.container().from_("airbyte/source-faker:latest") @pytest.fixture - def another_dummy_connector_under_test_image_tar(self, dagger_client, tmpdir) -> dagger.File: - dummy_tar_file = tmpdir / "another_dummy.tar" - dummy_tar_file.write_text("another_dummy", encoding="utf8") - return dagger_client.host().directory(str(tmpdir), include=["another_dummy.tar"]).file("another_dummy.tar") - - async def test_skipped_when_no_acceptance_test_config(self, mocker, test_context): - test_context.connector = mocker.MagicMock(acceptance_test_config=None) - acceptance_test_step = common.AcceptanceTests(test_context) + def another_dummy_connector_under_test_container(self, dagger_client) -> dagger.File: + return dagger_client.container().from_("airbyte/source-pokeapi:latest") + + async def test_skipped_when_no_acceptance_test_config(self, mocker, test_context_ci): + test_context_ci.connector = mocker.MagicMock(acceptance_test_config=None) + acceptance_test_step = common.AcceptanceTests(test_context_ci) step_result = await acceptance_test_step._run(None) assert step_result.status == StepStatus.SKIPPED @@ -114,7 +124,7 @@ async def test_skipped_when_no_acceptance_test_config(self, mocker, test_context ) async def test__run( self, - test_context, + test_context_ci, mocker, exit_code: int, expected_status: StepStatus, @@ -124,23 +134,23 @@ async def test__run( ): """Test the behavior of the run function using a dummy container.""" cat_container = self.get_dummy_cat_container( - test_context.dagger_client, exit_code, secrets_file_names, stdout="hello", stderr="world" + test_context_ci.dagger_client, exit_code, secrets_file_names, stdout="hello", stderr="world" ) async_mock = mocker.AsyncMock(return_value=cat_container) mocker.patch.object(common.AcceptanceTests, "_build_connector_acceptance_test", side_effect=async_mock) mocker.patch.object(common.AcceptanceTests, "get_cat_command", return_value=["bash", "/stupid_bash_script.sh"]) - test_context.get_connector_dir = mocker.AsyncMock(return_value=test_input_dir) - acceptance_test_step = common.AcceptanceTests(test_context) + test_context_ci.get_connector_dir = mocker.AsyncMock(return_value=test_input_dir) + acceptance_test_step = common.AcceptanceTests(test_context_ci) step_result = await acceptance_test_step._run(None) assert step_result.status == expected_status assert step_result.stdout.strip() == "hello" assert step_result.stderr.strip() == "world" if expect_updated_secrets: assert ( - await test_context.updated_secrets_dir.entries() + await test_context_ci.updated_secrets_dir.entries() == await cat_container.directory(f"{common.AcceptanceTests.CONTAINER_SECRETS_DIRECTORY}").entries() ) - assert any("updated_configurations" in str(file_name) for file_name in await test_context.updated_secrets_dir.entries()) + assert any("updated_configurations" in str(file_name) for file_name in await test_context_ci.updated_secrets_dir.entries()) @pytest.fixture def test_input_dir(self, dagger_client, tmpdir): @@ -148,17 +158,18 @@ def test_input_dir(self, dagger_client, tmpdir): yaml.safe_dump({"connector_image": "airbyte/connector_under_test_image:dev"}, f) return dagger_client.host().directory(str(tmpdir)) - def get_patched_acceptance_test_step(self, dagger_client, mocker, test_context, test_input_dir): - test_context.get_connector_dir = mocker.AsyncMock(return_value=test_input_dir) - test_context.connector_acceptance_test_image = "bash:latest" - test_context.connector_secrets = {"config.json": dagger_client.set_secret("config.json", "connector_secret")} + def get_patched_acceptance_test_step(self, dagger_client, mocker, test_context_ci, test_input_dir): + test_secrets = {"config.json": dagger_client.set_secret("config.json", "connector_secret")} + test_context_ci.get_connector_dir = mocker.AsyncMock(return_value=test_input_dir) + test_context_ci.connector_acceptance_test_image = "bash:latest" + test_context_ci.get_connector_secrets = mocker.AsyncMock(return_value=test_secrets) - mocker.patch.object(common.environments, "load_image_to_docker_host", return_value="image_sha") - mocker.patch.object(common.environments, "with_bound_docker_host", lambda _, cat_container: cat_container) - return common.AcceptanceTests(test_context) + mocker.patch.object(docker, "load_image_to_docker_host", return_value="image_sha") + mocker.patch.object(docker, "with_bound_docker_host", lambda _, cat_container: cat_container) + return common.AcceptanceTests(test_context_ci) async def test_cat_container_provisioning( - self, dagger_client, mocker, test_context, test_input_dir, dummy_connector_under_test_image_tar + self, dagger_client, mocker, test_context_ci, test_input_dir, dummy_connector_under_test_container ): """Check that the acceptance test container is correctly provisioned. We check that: @@ -167,8 +178,12 @@ async def test_cat_container_provisioning( - that the entrypoint is correctly set. - the current working directory is correctly set. """ - acceptance_test_step = self.get_patched_acceptance_test_step(dagger_client, mocker, test_context, test_input_dir) - cat_container = await acceptance_test_step._build_connector_acceptance_test(dummy_connector_under_test_image_tar, test_input_dir) + # The mounted_connector_secrets behaves differently when the test is run locally or in CI. + # It is not masking the secrets when run locally. + # We want to confirm that the secrets are correctly masked when run in CI. + + acceptance_test_step = self.get_patched_acceptance_test_step(dagger_client, mocker, test_context_ci, test_input_dir) + cat_container = await acceptance_test_step._build_connector_acceptance_test(dummy_connector_under_test_container, test_input_dir) assert (await cat_container.with_exec(["pwd"]).stdout()).strip() == acceptance_test_step.CONTAINER_TEST_INPUT_DIRECTORY test_input_ls_result = await cat_container.with_exec(["ls"]).stdout() assert all( @@ -182,19 +197,19 @@ async def test_cat_container_caching( self, dagger_client, mocker, - test_context, + test_context_ci, test_input_dir, - dummy_connector_under_test_image_tar, - another_dummy_connector_under_test_image_tar, + dummy_connector_under_test_container, + another_dummy_connector_under_test_container, ): """Check that the acceptance test container caching behavior is correct.""" initial_datetime = datetime.datetime(year=1992, month=6, day=19, hour=13, minute=1, second=0) with freeze_time(initial_datetime) as frozen_datetime: - acceptance_test_step = self.get_patched_acceptance_test_step(dagger_client, mocker, test_context, test_input_dir) + acceptance_test_step = self.get_patched_acceptance_test_step(dagger_client, mocker, test_context_ci, test_input_dir) cat_container = await acceptance_test_step._build_connector_acceptance_test( - dummy_connector_under_test_image_tar, test_input_dir + dummy_connector_under_test_container, test_input_dir ) cat_container = cat_container.with_exec(["date"]) fist_date_result = await cat_container.stdout() @@ -202,7 +217,7 @@ async def test_cat_container_caching( frozen_datetime.tick(delta=datetime.timedelta(hours=5)) # Check that cache is used in the same day cat_container = await acceptance_test_step._build_connector_acceptance_test( - dummy_connector_under_test_image_tar, test_input_dir + dummy_connector_under_test_container, test_input_dir ) cat_container = cat_container.with_exec(["date"]) second_date_result = await cat_container.stdout() @@ -211,17 +226,76 @@ async def test_cat_container_caching( # Check that cache bursted after a day frozen_datetime.tick(delta=datetime.timedelta(days=1, seconds=1)) cat_container = await acceptance_test_step._build_connector_acceptance_test( - dummy_connector_under_test_image_tar, test_input_dir + dummy_connector_under_test_container, test_input_dir ) cat_container = cat_container.with_exec(["date"]) third_date_result = await cat_container.stdout() assert third_date_result != second_date_result time.sleep(1) - # Check that changing the tarball invalidates the cache + # Check that changing the container invalidates the cache cat_container = await acceptance_test_step._build_connector_acceptance_test( - another_dummy_connector_under_test_image_tar, test_input_dir + another_dummy_connector_under_test_container, test_input_dir ) cat_container = cat_container.with_exec(["date"]) fourth_date_result = await cat_container.stdout() assert fourth_date_result != third_date_result + + async def test_params(self, dagger_client, mocker, test_context_ci, test_input_dir): + acceptance_test_step = self.get_patched_acceptance_test_step(dagger_client, mocker, test_context_ci, test_input_dir) + assert set(acceptance_test_step.params_as_cli_options) == {"-ra", "--disable-warnings", "--durations=3"} + acceptance_test_step.extra_params = {"--durations": ["5"], "--collect-only": []} + assert set(acceptance_test_step.params_as_cli_options) == {"-ra", "--disable-warnings", "--durations=5", "--collect-only"} + + +class TestCheckBaseImageIsUsed: + @pytest.fixture + def certified_connector_no_base_image(self, all_connectors): + for connector in all_connectors: + if connector.metadata.get("supportLevel") == "certified": + if connector.metadata.get("connectorBuildOptions", {}).get("baseImage") is None: + return connector + pytest.skip("No certified connector without base image found") + + @pytest.fixture + def certified_connector_with_base_image(self, all_connectors): + for connector in all_connectors: + if connector.metadata.get("supportLevel") == "certified": + if connector.metadata.get("connectorBuildOptions", {}).get("baseImage") is not None: + return connector + pytest.skip("No certified connector with base image found") + + @pytest.fixture + def community_connector_no_base_image(self, all_connectors): + for connector in all_connectors: + if connector.metadata.get("supportLevel") == "community": + if connector.metadata.get("connectorBuildOptions", {}).get("baseImage") is None: + return connector + pytest.skip("No certified connector without base image found") + + @pytest.fixture + def test_context(self, mocker, dagger_client): + return mocker.MagicMock(dagger_client=dagger_client) + + async def test_pass_on_community_connector_no_base_image(self, mocker, dagger_client, community_connector_no_base_image): + test_context = mocker.MagicMock(dagger_client=dagger_client, connector=community_connector_no_base_image) + check_base_image_is_used_step = common.CheckBaseImageIsUsed(test_context) + step_result = await check_base_image_is_used_step.run() + assert step_result.status == StepStatus.SKIPPED + + async def test_pass_on_certified_connector_with_base_image(self, mocker, dagger_client, certified_connector_with_base_image): + dagger_connector_dir = dagger_client.host().directory(str(certified_connector_with_base_image.code_directory)) + test_context = mocker.MagicMock( + dagger_client=dagger_client, + connector=certified_connector_with_base_image, + get_connector_dir=mocker.AsyncMock(return_value=dagger_connector_dir), + ) + check_base_image_is_used_step = common.CheckBaseImageIsUsed(test_context) + step_result = await check_base_image_is_used_step.run() + assert step_result.status == StepStatus.SUCCESS + + async def test_fail_on_certified_connector_no_base_image(self, mocker, dagger_client, certified_connector_no_base_image): + test_context = mocker.MagicMock(dagger_client=dagger_client, connector=certified_connector_no_base_image) + check_base_image_is_used_step = common.CheckBaseImageIsUsed(test_context) + step_result = await check_base_image_is_used_step.run() + assert step_result.status == StepStatus.FAILURE diff --git a/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py b/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py new file mode 100644 index 000000000000..2d89af9ec94d --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_tests/test_python_connectors.py @@ -0,0 +1,107 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import pytest +from connector_ops.utils import Connector, ConnectorLanguage +from pipelines.airbyte_ci.connectors.build_image.steps.python_connectors import BuildConnectorImages +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.connectors.test.steps.python_connectors import UnitTests +from pipelines.models.steps import StepResult + +pytestmark = [ + pytest.mark.anyio, +] + + +class TestUnitTests: + @pytest.fixture + def connector_with_poetry(self): + return Connector("destination-duckdb") + + @pytest.fixture + def certified_connector_with_setup(self, all_connectors): + for connector in all_connectors: + if connector.support_level == "certified" and connector.language in [ConnectorLanguage.LOW_CODE, ConnectorLanguage.PYTHON]: + if connector.code_directory.joinpath("setup.py").exists(): + return connector + pytest.skip("No certified connector with setup.py found.") + + @pytest.fixture + def context_for_certified_connector_with_setup(self, mocker, certified_connector_with_setup, dagger_client, current_platform): + context = ConnectorContext( + pipeline_name="test unit tests", + connector=certified_connector_with_setup, + git_branch="test", + git_revision="test", + report_output_prefix="test", + is_local=True, + use_remote_secrets=True, + targeted_platforms=[current_platform], + ) + context.dagger_client = dagger_client + context.get_connector_secrets = mocker.AsyncMock(return_value={}) + return context + + @pytest.fixture + async def certified_container_with_setup(self, context_for_certified_connector_with_setup, current_platform): + result = await BuildConnectorImages(context_for_certified_connector_with_setup).run() + return result.output_artifact[current_platform] + + @pytest.fixture + def context_for_connector_with_poetry(self, mocker, connector_with_poetry, dagger_client, current_platform): + context = ConnectorContext( + pipeline_name="test unit tests", + connector=connector_with_poetry, + git_branch="test", + git_revision="test", + report_output_prefix="test", + is_local=True, + use_remote_secrets=True, + targeted_platforms=[current_platform], + ) + context.dagger_client = dagger_client + context.get_connector_secrets = mocker.AsyncMock(return_value={}) + return context + + @pytest.fixture + async def container_with_poetry(self, context_for_connector_with_poetry, current_platform): + result = await BuildConnectorImages(context_for_connector_with_poetry).run() + return result.output_artifact[current_platform] + + async def test__run_for_setup_py(self, context_for_certified_connector_with_setup, certified_container_with_setup): + # Assume that the tests directory is available + result = await UnitTests(context_for_certified_connector_with_setup)._run(certified_container_with_setup) + assert isinstance(result, StepResult) + assert "test session starts" in result.stdout or "test session starts" in result.stderr + assert ( + "Total coverage:" in result.stdout + ), "The pytest-cov package should be installed in the test environment and test coverage report should be displayed." + assert "Required test coverage of" in result.stdout, "A test coverage threshold should be defined for certified connectors." + pip_freeze_output = await result.output_artifact.with_exec(["pip", "freeze"], skip_entrypoint=True).stdout() + assert ( + context_for_certified_connector_with_setup.connector.technical_name in pip_freeze_output + ), "The connector should be installed in the test environment." + assert "pytest" in pip_freeze_output, "The pytest package should be installed in the test environment." + assert "pytest-cov" in pip_freeze_output, "The pytest-cov package should be installed in the test environment." + + async def test__run_for_poetry(self, context_for_connector_with_poetry, container_with_poetry): + # Assume that the tests directory is available + result = await UnitTests(context_for_connector_with_poetry).run(container_with_poetry) + assert isinstance(result, StepResult) + # We only check for the presence of "test session starts" because we have no guarantee that the tests will pass + assert "test session starts" in result.stdout or "test session starts" in result.stderr, "The pytest tests should have started." + pip_freeze_output = await result.output_artifact.with_exec(["poetry", "run", "pip", "freeze"], skip_entrypoint=True).stdout() + + assert ( + context_for_connector_with_poetry.connector.technical_name in pip_freeze_output + ), "The connector should be installed in the test environment." + assert "pytest" in pip_freeze_output, "The pytest package should be installed in the test environment." + + def test_params(self, context_for_certified_connector_with_setup): + step = UnitTests(context_for_certified_connector_with_setup) + assert step.params_as_cli_options == [ + "-s", + f"--cov={context_for_certified_connector_with_setup.connector.technical_name.replace('-', '_')}", + f"--cov-fail-under={step.MINIMUM_COVERAGE_FOR_CERTIFIED_CONNECTORS}", + ] diff --git a/airbyte-ci/connectors/pipelines/tests/test_upgrade_cdk.py b/airbyte-ci/connectors/pipelines/tests/test_upgrade_cdk.py new file mode 100644 index 000000000000..9cc3d26f9f27 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_upgrade_cdk.py @@ -0,0 +1,121 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import random +from pathlib import Path +from typing import List +from unittest.mock import AsyncMock, MagicMock + +import anyio +import pytest +from connector_ops.utils import Connector, ConnectorLanguage +from dagger import Directory +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.connectors.publish import pipeline as publish_pipeline +from pipelines.airbyte_ci.connectors.upgrade_cdk import pipeline as upgrade_cdk_pipeline +from pipelines.models.steps import StepStatus + +pytestmark = [ + pytest.mark.anyio, +] + + +@pytest.fixture +def sample_connector(): + return Connector("source-pokeapi") + + +def get_sample_setup_py(airbyte_cdk_dependency: str): + return f"""from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "{airbyte_cdk_dependency}", +] + +setup( + name="source_pokeapi", + description="Source implementation for Pokeapi.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, +) +""" + + +@pytest.fixture +def connector_context(sample_connector, dagger_client, current_platform): + context = ConnectorContext( + pipeline_name="test", + connector=sample_connector, + git_branch="test", + git_revision="test", + report_output_prefix="test", + is_local=True, + use_remote_secrets=True, + targeted_platforms=[current_platform], + ) + context.dagger_client = dagger_client + return context + + +@pytest.mark.parametrize( + "setup_py_content, expected_setup_py_content", + [ + (get_sample_setup_py("airbyte-cdk"), get_sample_setup_py("airbyte-cdk>=6.6.6")), + (get_sample_setup_py("airbyte-cdk[file-based]"), get_sample_setup_py("airbyte-cdk[file-based]>=6.6.6")), + (get_sample_setup_py("airbyte-cdk==1.2.3"), get_sample_setup_py("airbyte-cdk>=6.6.6")), + (get_sample_setup_py("airbyte-cdk>=1.2.3"), get_sample_setup_py("airbyte-cdk>=6.6.6")), + (get_sample_setup_py("airbyte-cdk[file-based]>=1.2.3"), get_sample_setup_py("airbyte-cdk[file-based]>=6.6.6")), + ], +) +async def test_run_connector_cdk_upgrade_pipeline( + connector_context: ConnectorContext, setup_py_content: str, expected_setup_py_content: str +): + full_og_connector_dir = await connector_context.get_connector_dir() + updated_connector_dir = full_og_connector_dir.with_new_file("setup.py", setup_py_content) + + # For this test, replace the actual connector dir with an updated version that sets the setup.py contents + connector_context.get_connector_dir = AsyncMock(return_value=updated_connector_dir) + + # Mock the diff method to record the resulting directory and return a mock to not actually export the diff to the repo + updated_connector_dir.diff = MagicMock(return_value=AsyncMock()) + step = upgrade_cdk_pipeline.SetCDKVersion(connector_context, "6.6.6") + step_result = await step.run() + assert step_result.status == StepStatus.SUCCESS + + # Check that the resulting directory that got passed to the mocked diff method looks as expected + resulting_directory: Directory = await full_og_connector_dir.diff(updated_connector_dir.diff.call_args[0][0]) + files = await resulting_directory.entries() + # validate only setup.py is changed + assert files == ["setup.py"] + setup_py = resulting_directory.file("setup.py") + actual_setup_py_content = await setup_py.contents() + assert expected_setup_py_content == actual_setup_py_content + + # Assert that the diff was exported to the repo + assert updated_connector_dir.diff.return_value.export.call_count == 1 + + +async def test_skip_connector_cdk_upgrade_pipeline_on_missing_setup_py(connector_context: ConnectorContext): + full_og_connector_dir = await connector_context.get_connector_dir() + updated_connector_dir = full_og_connector_dir.without_file("setup.py") + + connector_context.get_connector_dir = AsyncMock(return_value=updated_connector_dir) + + step = upgrade_cdk_pipeline.SetCDKVersion(connector_context, "6.6.6") + step_result = await step.run() + assert step_result.status == StepStatus.SKIPPED + + +async def test_fail_connector_cdk_upgrade_pipeline_on_missing_airbyte_cdk(connector_context: ConnectorContext): + full_og_connector_dir = await connector_context.get_connector_dir() + updated_connector_dir = full_og_connector_dir.with_new_file("setup.py", get_sample_setup_py("another-lib==1.2.3")) + + connector_context.get_connector_dir = AsyncMock(return_value=updated_connector_dir) + + step = upgrade_cdk_pipeline.SetCDKVersion(connector_context, "6.6.6") + step_result = await step.run() + assert step_result.status == StepStatus.FAILURE diff --git a/airbyte-ci/connectors/pipelines/tests/test_utils.py b/airbyte-ci/connectors/pipelines/tests/test_utils.py deleted file mode 100644 index 58b9c9fd78cd..000000000000 --- a/airbyte-ci/connectors/pipelines/tests/test_utils.py +++ /dev/null @@ -1,183 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from pathlib import Path -from unittest import mock - -import pytest -from connector_ops.utils import Connector, ConnectorLanguage -from pipelines import utils -from tests.utils import pick_a_random_connector - - -@pytest.mark.parametrize( - "ctx, expected", - [ - ( - mock.MagicMock( - command_path="my command path", - obj={ - "git_branch": "my_branch", - "git_revision": "my_git_revision", - "pipeline_start_timestamp": "my_pipeline_start_timestamp", - "ci_context": "my_ci_context", - "ci_job_key": None, - }, - ), - f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_context/my_branch/my_pipeline_start_timestamp/my_git_revision", - ), - ( - mock.MagicMock( - command_path="my command path", - obj={ - "git_branch": "my_branch", - "git_revision": "my_git_revision", - "pipeline_start_timestamp": "my_pipeline_start_timestamp", - "ci_context": "my_ci_context", - "ci_job_key": "my_ci_job_key", - }, - ), - f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch/my_pipeline_start_timestamp/my_git_revision", - ), - ( - mock.MagicMock( - command_path="my command path", - obj={ - "git_branch": "my_branch", - "git_revision": "my_git_revision", - "pipeline_start_timestamp": "my_pipeline_start_timestamp", - "ci_context": "my_ci_context", - "ci_job_key": "my_ci_job_key", - }, - ), - f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch/my_pipeline_start_timestamp/my_git_revision", - ), - ( - mock.MagicMock( - command_path="my command path", - obj={ - "git_branch": "my_branch", - "git_revision": "my_git_revision", - "pipeline_start_timestamp": "my_pipeline_start_timestamp", - "ci_context": "my_ci_context", - "ci_job_key": "my_ci_job_key", - }, - ), - f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch/my_pipeline_start_timestamp/my_git_revision", - ), - ( - mock.MagicMock( - command_path="my command path", - obj={ - "git_branch": "my_branch/with/slashes", - "git_revision": "my_git_revision", - "pipeline_start_timestamp": "my_pipeline_start_timestamp", - "ci_context": "my_ci_context", - "ci_job_key": "my_ci_job_key", - }, - ), - f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch_with_slashes/my_pipeline_start_timestamp/my_git_revision", - ), - ( - mock.MagicMock( - command_path="my command path", - obj={ - "git_branch": "my_branch/with/slashes#and!special@characters", - "git_revision": "my_git_revision", - "pipeline_start_timestamp": "my_pipeline_start_timestamp", - "ci_context": "my_ci_context", - "ci_job_key": "my_ci_job_key", - }, - ), - f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch_with_slashesandspecialcharacters/my_pipeline_start_timestamp/my_git_revision", - ), - ( - mock.MagicMock( - command_path="airbyte-ci command path", - obj={ - "git_branch": "my_branch/with/slashes#and!special@characters", - "git_revision": "my_git_revision", - "pipeline_start_timestamp": "my_pipeline_start_timestamp", - "ci_context": "my_ci_context", - "ci_job_key": "my_ci_job_key", - }, - ), - f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch_with_slashesandspecialcharacters/my_pipeline_start_timestamp/my_git_revision", - ), - ( - mock.MagicMock( - command_path="airbyte-ci-internal command path", - obj={ - "git_branch": "my_branch/with/slashes#and!special@characters", - "git_revision": "my_git_revision", - "pipeline_start_timestamp": "my_pipeline_start_timestamp", - "ci_context": "my_ci_context", - "ci_job_key": "my_ci_job_key", - }, - ), - f"{utils.STATIC_REPORT_PREFIX}/command/path/my_ci_job_key/my_branch_with_slashesandspecialcharacters/my_pipeline_start_timestamp/my_git_revision", - ), - ], -) -def test_render_report_output_prefix(ctx, expected): - assert utils.DaggerPipelineCommand.render_report_output_prefix(ctx) == expected - - -@pytest.mark.parametrize("enable_dependency_scanning", [True, False]) -def test_get_modified_connectors_with_dependency_scanning(all_connectors, enable_dependency_scanning): - base_java_changed_file = Path("airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/BaseConnector.java") - modified_files = [base_java_changed_file] - - not_modified_java_connector = pick_a_random_connector(language=ConnectorLanguage.JAVA) - modified_java_connector = pick_a_random_connector( - language=ConnectorLanguage.JAVA, other_picked_connectors=[not_modified_java_connector] - ) - modified_files.append(modified_java_connector.code_directory / "foo.bar") - - modified_connectors = utils.get_modified_connectors(modified_files, all_connectors, enable_dependency_scanning) - if enable_dependency_scanning: - assert not_modified_java_connector in modified_connectors - else: - assert not_modified_java_connector not in modified_connectors - assert modified_java_connector in modified_connectors - - -def test_get_connector_modified_files(): - connector = pick_a_random_connector() - other_connector = pick_a_random_connector(other_picked_connectors=[connector]) - - all_modified_files = { - connector.code_directory / "setup.py", - other_connector.code_directory / "README.md", - } - - result = utils.get_connector_modified_files(connector, all_modified_files) - assert result == frozenset({connector.code_directory / "setup.py"}) - - -def test_no_modified_files_in_connector_directory(): - connector = pick_a_random_connector() - other_connector = pick_a_random_connector(other_picked_connectors=[connector]) - - all_modified_files = { - other_connector.code_directory / "README.md", - } - - result = utils.get_connector_modified_files(connector, all_modified_files) - assert result == frozenset() - - -@pytest.mark.anyio -async def test_check_path_in_workdir(dagger_client): - connector = Connector("source-openweather") - container = ( - dagger_client.container() - .from_("bash") - .with_mounted_directory(str(connector.code_directory), dagger_client.host().directory(str(connector.code_directory))) - .with_workdir(str(connector.code_directory)) - ) - assert await utils.check_path_in_workdir(container, "metadata.yaml") - assert await utils.check_path_in_workdir(container, "setup.py") - assert await utils.check_path_in_workdir(container, "requirements.txt") - assert await utils.check_path_in_workdir(container, "not_existing_file") is False diff --git a/airbyte-ci/connectors/pipelines/tests/utils.py b/airbyte-ci/connectors/pipelines/tests/utils.py index fc1c4ae5d6e6..4038b9f7d319 100644 --- a/airbyte-ci/connectors/pipelines/tests/utils.py +++ b/airbyte-ci/connectors/pipelines/tests/utils.py @@ -13,7 +13,7 @@ def pick_a_random_connector( language: ConnectorLanguage = None, support_level: str = None, other_picked_connectors: list = None ) -> Connector: """Pick a random connector from the list of all connectors.""" - all_connectors = list(ALL_CONNECTORS) + all_connectors = [c for c in list(ALL_CONNECTORS)] if language: all_connectors = [c for c in all_connectors if c.language is language] if support_level: diff --git a/airbyte-commons-cli/build.gradle b/airbyte-commons-cli/build.gradle deleted file mode 100644 index 2b9e141d8164..000000000000 --- a/airbyte-commons-cli/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id "java-library" -} - -dependencies { - implementation 'commons-cli:commons-cli:1.4' -} - -Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project) diff --git a/airbyte-commons-protocol/build.gradle b/airbyte-commons-protocol/build.gradle deleted file mode 100644 index 5787c7af290d..000000000000 --- a/airbyte-commons-protocol/build.gradle +++ /dev/null @@ -1,12 +0,0 @@ -dependencies { - annotationProcessor libs.bundles.micronaut.annotation.processor - testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor - - implementation libs.bundles.micronaut.annotation - testImplementation libs.bundles.micronaut.test - - implementation libs.airbyte.protocol - implementation project(':airbyte-json-validation') -} - -Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project) diff --git a/airbyte-commons-protocol/src/test/resources/WellKnownTypes.json b/airbyte-commons-protocol/src/test/resources/WellKnownTypes.json deleted file mode 100644 index 95d2ff9e26fa..000000000000 --- a/airbyte-commons-protocol/src/test/resources/WellKnownTypes.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "definitions": { - "String": { - "type": "string", - "description": "Arbitrary text" - }, - "BinaryData": { - "type": "string", - "description": "Arbitrary binary data. Represented as base64-encoded strings in the JSON transport. In the future, if we support other transports, may be encoded differently.\n", - "pattern": "^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$" - }, - "Date": { - "type": "string", - "pattern": "^\\d{4}-\\d{2}-\\d{2}( BC)?$", - "description": "RFC 3339\u00a75.6's full-date format, extended with BC era support" - }, - "TimestampWithTimezone": { - "type": "string", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?(Z|[+\\-]\\d{1,2}:\\d{2})( BC)?$", - "description": "An instant in time. Frequently simply referred to as just a timestamp, or timestamptz. Uses RFC 3339\u00a75.6's date-time format, requiring a \"T\" separator, and extended with BC era support. Note that we do _not_ accept Unix epochs here.\n" - }, - "TimestampWithoutTimezone": { - "type": "string", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?( BC)?$", - "description": "Also known as a localdatetime, or just datetime. Under RFC 3339\u00a75.6, this would be represented as `full-date \"T\" partial-time`, extended with BC era support.\n" - }, - "TimeWithTimezone": { - "type": "string", - "pattern": "^\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?(Z|[+\\-]\\d{1,2}:\\d{2})$", - "description": "An RFC 3339\u00a75.6 full-time" - }, - "TimeWithoutTimezone": { - "type": "string", - "pattern": "^\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?$", - "description": "An RFC 3339\u00a75.6 partial-time" - }, - "Number": { - "type": "string", - "oneOf": [ - { - "pattern": "-?(0|[0-9]\\d*)(\\.\\d+)?" - }, - { - "enum": ["Infinity", "-Infinity", "NaN"] - } - ], - "description": "Note the mix of regex validation for normal numbers, and enum validation for special values." - }, - "Integer": { - "type": "string", - "oneOf": [ - { - "pattern": "-?(0|[0-9]\\d*)" - }, - { - "enum": ["Infinity", "-Infinity", "NaN"] - } - ] - }, - "Boolean": { - "type": "boolean", - "description": "Note the direct usage of a primitive boolean rather than string. Unlike Numbers and Integers, we don't expect unusual values here." - } - } -} diff --git a/airbyte-commons/build.gradle b/airbyte-commons/build.gradle deleted file mode 100644 index 4a2dd07625ac..000000000000 --- a/airbyte-commons/build.gradle +++ /dev/null @@ -1,13 +0,0 @@ -plugins { - id "java-library" -} - -dependencies { - // Dependencies for this module should be specified in the top-level build.gradle. See readme for more explanation. - implementation libs.airbyte.protocol - - // this dependency is an exception to the above rule because it is only used INTERNALLY to the commons library. - implementation 'com.jayway.jsonpath:json-path:2.7.0' -} - -Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project) diff --git a/airbyte-config-oss/config-models-oss/README.md b/airbyte-config-oss/config-models-oss/README.md deleted file mode 100644 index 7ad4cbdf95bf..000000000000 --- a/airbyte-config-oss/config-models-oss/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# Config Models - -This module uses `jsonschema2pojo` to generate Java config objects from [json schema](https://json-schema.org/) definitions. See [build.gradle](./build.gradle) for details. - -## How to use -- Update json schema under: - ``` - src/main/resources/types/ - ``` -- Run the following command under the project root: - ```sh - SUB_BUILD=PLATFORM ./gradlew airbyte-config-oss:config-models-oss:generateJsonSchema2Pojo - ``` - The generated file is under: - ``` - build/generated/src/gen/java/io/airbyte/config/ - ``` - -## Reference -- [`jsonschema2pojo` plugin](https://github.com/joelittlejohn/jsonschema2pojo/tree/master/jsonschema2pojo-gradle-plugin). diff --git a/airbyte-config-oss/config-models-oss/build.gradle b/airbyte-config-oss/config-models-oss/build.gradle deleted file mode 100644 index c0e92fb85ce0..000000000000 --- a/airbyte-config-oss/config-models-oss/build.gradle +++ /dev/null @@ -1,60 +0,0 @@ -import org.jsonschema2pojo.SourceType - -plugins { - id "java-library" - id "com.github.eirnym.js2p" version "1.0" -} - -dependencies { - annotationProcessor libs.bundles.micronaut.annotation.processor - api libs.bundles.micronaut.annotation - - implementation project(':airbyte-json-validation') - implementation libs.airbyte.protocol - implementation project(':airbyte-commons') -} - -jsonSchema2Pojo { - sourceType = SourceType.YAMLSCHEMA - source = files("${sourceSets.main.output.resourcesDir}/types") - targetDirectory = new File(project.buildDir, 'generated/src/gen/java/') - - targetPackage = 'io.airbyte.configoss' - useLongIntegers = true - - removeOldOutput = true - - generateBuilders = true - includeConstructors = false - includeSetters = true - serializable = true -} - -test { - useJUnitPlatform { - excludeTags 'log4j2-config', 'logger-client' - } - testLogging { - events "passed", "skipped", "failed" - } -} - -task log4j2IntegrationTest(type: Test) { - useJUnitPlatform { - includeTags 'log4j2-config' - } - testLogging { - events "passed", "skipped", "failed" - } -} - -task logClientsIntegrationTest(type: Test) { - useJUnitPlatform { - includeTags 'logger-client' - } - testLogging { - events "passed", "skipped", "failed" - } -} - -Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project) diff --git a/airbyte-config-oss/init-oss/build.gradle b/airbyte-config-oss/init-oss/build.gradle deleted file mode 100644 index a810c1ec6bb3..000000000000 --- a/airbyte-config-oss/init-oss/build.gradle +++ /dev/null @@ -1,38 +0,0 @@ -plugins { - id 'java-library' - id "de.undercouch.download" version "5.4.0" - -} - -dependencies { - annotationProcessor libs.bundles.micronaut.annotation.processor - api libs.bundles.micronaut.annotation - - implementation 'commons-cli:commons-cli:1.4' - implementation project(':airbyte-commons-cli') - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-json-validation') - implementation libs.lombok - implementation libs.micronaut.cache.caffeine - - testImplementation project(':airbyte-test-utils') - testImplementation 'com.squareup.okhttp3:mockwebserver:4.9.1' -} - -task downloadConnectorRegistry(type: Download) { - src 'https://connectors.airbyte.com/files/registries/v0/oss_registry.json' - dest new File(projectDir, 'src/main/resources/seed/oss_registry.json') - overwrite true -} - -task downloadSpecSecretMask(type: Download) { - src 'https://connectors.airbyte.com/files/registries/v0/specs_secrets_mask.yaml' - dest new File(project(":airbyte-commons").projectDir, 'src/main/resources/seed/specs_secrets_mask.yaml') - overwrite true -} - -processResources.dependsOn(downloadConnectorRegistry) -processResources.dependsOn(downloadSpecSecretMask) - -Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project) diff --git a/airbyte-connector-test-harnesses/acceptance-test-harness/build.gradle b/airbyte-connector-test-harnesses/acceptance-test-harness/build.gradle deleted file mode 100644 index 9aac14c201f2..000000000000 --- a/airbyte-connector-test-harnesses/acceptance-test-harness/build.gradle +++ /dev/null @@ -1,44 +0,0 @@ -plugins { - id "java-library" -} - -dependencies { - annotationProcessor platform(libs.micronaut.bom) - annotationProcessor libs.bundles.micronaut.annotation.processor - - implementation platform(libs.micronaut.bom) - implementation libs.bundles.micronaut - - implementation 'io.fabric8:kubernetes-client:5.12.2' - implementation 'com.auth0:java-jwt:3.19.2' - implementation libs.guava - implementation(libs.temporal.sdk) { - exclude module: 'guava' - } - implementation 'org.apache.ant:ant:1.10.10' - implementation 'org.apache.commons:commons-text:1.10.0' - implementation libs.bundles.datadog - - implementation project(':airbyte-api') - implementation project(':airbyte-commons-protocol') - implementation project(':airbyte-config-oss:config-models-oss') - implementation project(':airbyte-json-validation') - implementation libs.airbyte.protocol - - testAnnotationProcessor platform(libs.micronaut.bom) - testAnnotationProcessor libs.bundles.micronaut.test.annotation.processor - testAnnotationProcessor libs.jmh.annotations - - testImplementation libs.bundles.micronaut.test - testImplementation 'com.jayway.jsonpath:json-path:2.7.0' - testImplementation 'org.mockito:mockito-inline:4.7.0' - testImplementation libs.postgresql - testImplementation libs.platform.testcontainers - testImplementation libs.platform.testcontainers.postgresql - testImplementation libs.jmh.core - testImplementation libs.jmh.annotations - testImplementation 'com.github.docker-java:docker-java:3.2.8' - testImplementation 'com.github.docker-java:docker-java-transport-httpclient5:3.2.8' -} - -Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project) diff --git a/airbyte-db/db-lib/README.md b/airbyte-db/db-lib/README.md deleted file mode 100644 index 22d3dca69642..000000000000 --- a/airbyte-db/db-lib/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# How to Create a New Database - -Check `io.airbyte.db.instance.configs` for example. - -## Database Instance -- Create a new package under `io.airbyte.db.instance` with the name of the database. -- Create the database schema enum that defines all tables in the database. -- Write a SQL script that initializes the database. - - The default path for this file is `resource/_database/schema.sql`. -- Implement the `DatabaseInstance` interface that extends from `BaseDatabaseInstance`. This class initializes the database by executing the initialization script. - -## Database Migration -- Implement the `DatabaseMigrator` interface that extends from `BaseDatabaseMigrator`. This class will handle the database migration. -- Create a new package `migrations` under the database package. Put all migrations files there. -- Add the migration commands in `build.gradle` for the new database. - - The three commands are `newMigration`, `runMigration`, and `dumpSchema`. - -## jOOQ Code Generation -- To setup jOOQ code generation for the new database, refer to [`airbyte-db/jooq`](../jooq/README.md) for details. -- Please do not use any jOOQ generated code in this `lib` module. This is because the `jooq` module that generates the code depends on this one. - -# How to Write a Migration -- Run the `newMigration` command to create a new migration file in `io.airbyte.db.instance..migrations`. - - Configs database: `./gradlew :airbyte-db:db-lib:newConfigsMigration`. - - Jobs database: `./gradlew :airbyte-db:db-lib:newJobsMigration`. -- Write the migration using [`jOOQ`](https://www.jooq.org/). -- Use the `runMigration` command to apply your newly written migration if you want to test it. - - Configs database: `./gradlew :airbyte-db:db-lib:runConfigsMigration`. - - Jobs database: `./gradlew :airbyte-db:db-lib:runJobsMigration`. -- Run the `dumpSchema` command to update the database schema. - - Configs database: `./gradlew :airbyte-db:db-lib:dumpConfigsSchema` - - Jobs database: `./gradlew :airbyte-db:db-lib:dumpJobsSchema` - -## Migration Filename -- The name of the file should follow this pattern: `V(version)__(migration_description_in_snake_case).java`. -- This pattern is mandatory for Flyway to correctly locate and sort the migrations. -- The first part is `V`, which denotes for *versioned* migration. -- The second part is a version string with this pattern: `___`. - - The `major`, `minor`, and `patch` should match that of the Airbyte version. - - The `id` should start from `001` for each `__` combination. - - Example version: `0_29_9_001` -- The third part is a double underscore separator `__`. -- The fourth part is a brief description in snake case. Only the first letter should be capitalized for consistency. -- See original Flyway [documentation](https://flywaydb.org/documentation/concepts/migrations#naming-1) for more details. - -## Sample Migration File - -```java -/** - * This migration add an "active" column to the "airbyte_configs" table. - * This column is nullable, and default to {@code true}. - */ -public class V0_29_9_001__Add_active_column extends BaseJavaMigration { - - @Override - public void migrate(Context context) throws Exception { - DSL.using(context.getConnection()).alterTable("airbyte_configs") - .addColumn(field("active", SQLDataType.BOOLEAN.defaultValue(true).nullable(true))) - .execute(); - } - -} -``` - -# How to Run a Migration -- Automatic. Migrations will be run automatically in the server. If you prefer to manually run the migration, change `RUN_DATABASE_MIGRATION_ON_STARTUP` to `false` in `.env`. -- API. Call `api/v1/db_migrations/list` to retrieve the current migration status, and call `api/v1/db_migrations/migrate` to run the migrations. Check the API [documentation](https://airbyte-public-api-docs.s3.us-east-2.amazonaws.com/rapidoc-api-docs.html#tag--db_migration) for more details. - -# Schema Dump -- The database schema is checked in to the codebase to ensure that we don't accidentally make any schema change. -- The schema dump can be done manually and automatically. -- To dump the schema manually, run the `dumpSchema` command, as mentioned above. -- The `DatabaseMigratorTest` dumps the schema automatically for each database. Please remember to check in any change in the schema dump. diff --git a/airbyte-db/db-lib/build.gradle b/airbyte-db/db-lib/build.gradle deleted file mode 100644 index a7002bd82234..000000000000 --- a/airbyte-db/db-lib/build.gradle +++ /dev/null @@ -1,61 +0,0 @@ -plugins { - id 'java-library' -} - -// Add a configuration for our migrations tasks defined below to encapsulate their dependencies -configurations { - migrations.extendsFrom implementation -} - -configurations.all { - exclude group: 'io.micronaut.flyway' -} - -dependencies { - api libs.hikaricp - api libs.jooq.meta - api libs.jooq - api libs.postgresql - - implementation libs.airbyte.protocol - implementation project(':airbyte-json-validation') - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.flyway.core - - migrations libs.platform.testcontainers.postgresql - migrations sourceSets.main.output - - // Mark as compile only to avoid leaking transitively to connectors - compileOnly libs.platform.testcontainers.postgresql - compileOnly libs.connectors.testcontainers.mysql - - // These are required because gradle might be using lower version of Jna from other - // library transitive dependency. Can be removed if we can figure out which library is the cause. - // Refer: https://github.com/testcontainers/testcontainers-java/issues/3834#issuecomment-825409079 - implementation 'net.java.dev.jna:jna:5.8.0' - implementation 'net.java.dev.jna:jna-platform:5.8.0' - - testImplementation project(':airbyte-test-utils') - testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation libs.platform.testcontainers.postgresql - testImplementation libs.connectors.testcontainers.mysql - - // Big Query - implementation('com.google.cloud:google-cloud-bigquery:1.133.1') - - // Lombok - implementation 'org.projectlombok:lombok:1.18.20' - annotationProcessor('org.projectlombok:lombok:1.18.20') - - // MongoDB - implementation 'org.mongodb:mongodb-driver-sync:4.3.0' - - // Teradata - implementation 'com.teradata.jdbc:terajdbc4:17.20.00.12' - - // MySQL - implementation 'mysql:mysql-connector-java:8.0.30' - -} - -Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project) diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java deleted file mode 100644 index 2607e208d7bc..000000000000 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/MySqlUtils.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.db; - -import com.google.common.annotations.VisibleForTesting; -import java.io.IOException; -import org.testcontainers.containers.MySQLContainer; - -public class MySqlUtils { - - @VisibleForTesting - public static Certificate getCertificate(final MySQLContainer container, - final boolean useAllCertificates) - throws IOException, InterruptedException { - // add root and server certificates to config file - container.execInContainer("sh", "-c", "sed -i '31 a ssl' /etc/my.cnf"); - container.execInContainer("sh", "-c", "sed -i '32 a ssl-ca=/var/lib/mysql/ca.pem' /etc/my.cnf"); - container.execInContainer("sh", "-c", "sed -i '33 a ssl-cert=/var/lib/mysql/server-cert.pem' /etc/my.cnf"); - container.execInContainer("sh", "-c", "sed -i '34 a ssl-key=/var/lib/mysql/server-key.pem' /etc/my.cnf"); - container.execInContainer("sh", "-c", "sed -i '35 a require_secure_transport=ON' /etc/my.cnf"); - // add client certificates to config file - if (useAllCertificates) { - container.execInContainer("sh", "-c", "sed -i '39 a [client]' /etc/mysql/my.cnf"); - container.execInContainer("sh", "-c", "sed -i '40 a ssl-ca=/var/lib/mysql/ca.pem' /etc/my.cnf"); - container.execInContainer("sh", "-c", "sed -i '41 a ssl-cert=/var/lib/mysql/client-cert.pem' /etc/my.cnf"); - container.execInContainer("sh", "-c", "sed -i '42 a ssl-key=/var/lib/mysql/client-key.pem' /etc/my.cnf"); - } - // copy root certificate and client certificates - var caCert = container.execInContainer("sh", "-c", "cat /var/lib/mysql/ca.pem").getStdout().trim(); - - if (useAllCertificates) { - var clientKey = container.execInContainer("sh", "-c", "cat /var/lib/mysql/client-key.pem").getStdout().trim(); - var clientCert = container.execInContainer("sh", "-c", "cat /var/lib/mysql/client-cert.pem").getStdout().trim(); - return new Certificate(caCert, clientCert, clientKey); - } else { - return new Certificate(caCert); - } - } - - public static class Certificate { - - private final String caCertificate; - private final String clientCertificate; - private final String clientKey; - - public Certificate(final String caCertificate) { - this.caCertificate = caCertificate; - this.clientCertificate = null; - this.clientKey = null; - } - - public Certificate(final String caCertificate, final String clientCertificate, final String clientKey) { - this.caCertificate = caCertificate; - this.clientCertificate = clientCertificate; - this.clientKey = clientKey; - } - - public String getCaCertificate() { - return caCertificate; - } - - public String getClientCertificate() { - return clientCertificate; - } - - public String getClientKey() { - return clientKey; - } - - } - -} diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/PostgresUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/PostgresUtils.java deleted file mode 100644 index 084dfdee7940..000000000000 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/PostgresUtils.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.db; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import java.io.IOException; -import java.sql.SQLException; -import java.util.List; -import org.testcontainers.containers.PostgreSQLContainer; - -public class PostgresUtils { - - public static PgLsn getLsn(final JdbcDatabase database) throws SQLException { - // pg version >= 10. For versions < 10 use query select * from pg_current_xlog_location() - final List jsonNodes = database - .bufferedResultSetQuery(conn -> conn.createStatement().executeQuery("select * from pg_current_wal_lsn()"), - resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet)); - - Preconditions.checkState(jsonNodes.size() == 1); - return PgLsn.fromPgString(jsonNodes.get(0).get("pg_current_wal_lsn").asText()); - } - - @VisibleForTesting - public static Certificate getCertificate(final PostgreSQLContainer container) throws IOException, InterruptedException { - container.execInContainer("su", "-c", "psql -U test -c \"CREATE USER postgres WITH PASSWORD 'postgres';\""); - container.execInContainer("su", "-c", "psql -U test -c \"GRANT CONNECT ON DATABASE \"test\" TO postgres;\""); - container.execInContainer("su", "-c", "psql -U test -c \"ALTER USER postgres WITH SUPERUSER;\""); - - container.execInContainer("su", "-c", "openssl ecparam -name prime256v1 -genkey -noout -out ca.key"); - container.execInContainer("su", "-c", "openssl req -new -x509 -sha256 -key ca.key -out ca.crt -subj \"/CN=127.0.0.1\""); - container.execInContainer("su", "-c", "openssl ecparam -name prime256v1 -genkey -noout -out server.key"); - container.execInContainer("su", "-c", "openssl req -new -sha256 -key server.key -out server.csr -subj \"/CN=localhost\""); - container.execInContainer("su", "-c", - "openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 365 -sha256"); - container.execInContainer("su", "-c", "cp server.key /etc/ssl/private/"); - container.execInContainer("su", "-c", "cp server.crt /etc/ssl/private/"); - container.execInContainer("su", "-c", "cp ca.crt /etc/ssl/private/"); - container.execInContainer("su", "-c", "chmod og-rwx /etc/ssl/private/server.* /etc/ssl/private/ca.*"); - container.execInContainer("su", "-c", "chown postgres:postgres /etc/ssl/private/server.crt /etc/ssl/private/server.key /etc/ssl/private/ca.crt"); - container.execInContainer("su", "-c", "echo \"ssl = on\" >> /var/lib/postgresql/data/postgresql.conf"); - container.execInContainer("su", "-c", "echo \"ssl_cert_file = '/etc/ssl/private/server.crt'\" >> /var/lib/postgresql/data/postgresql.conf"); - container.execInContainer("su", "-c", "echo \"ssl_key_file = '/etc/ssl/private/server.key'\" >> /var/lib/postgresql/data/postgresql.conf"); - container.execInContainer("su", "-c", "echo \"ssl_ca_file = '/etc/ssl/private/ca.crt'\" >> /var/lib/postgresql/data/postgresql.conf"); - container.execInContainer("su", "-c", "mkdir root/.postgresql"); - container.execInContainer("su", "-c", - "echo \"hostssl all all 127.0.0.1/32 cert clientcert=verify-full\" >> /var/lib/postgresql/data/pg_hba.conf"); - - final var caCert = container.execInContainer("su", "-c", "cat ca.crt").getStdout().trim(); - - container.execInContainer("su", "-c", "openssl ecparam -name prime256v1 -genkey -noout -out client.key"); - container.execInContainer("su", "-c", "openssl req -new -sha256 -key client.key -out client.csr -subj \"/CN=postgres\""); - container.execInContainer("su", "-c", - "openssl x509 -req -in client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out client.crt -days 365 -sha256"); - container.execInContainer("su", "-c", "cp client.crt ~/.postgresql/postgresql.crt"); - container.execInContainer("su", "-c", "cp client.key ~/.postgresql/postgresql.key"); - container.execInContainer("su", "-c", "chmod 0600 ~/.postgresql/postgresql.crt ~/.postgresql/postgresql.key"); - container.execInContainer("su", "-c", "cp ca.crt root/.postgresql/ca.crt"); - container.execInContainer("su", "-c", "chown postgres:postgres ~/.postgresql/ca.crt"); - - container.execInContainer("su", "-c", "psql -U test -c \"SELECT pg_reload_conf();\""); - - final var clientKey = container.execInContainer("su", "-c", "cat client.key").getStdout().trim(); - final var clientCert = container.execInContainer("su", "-c", "cat client.crt").getStdout().trim(); - return new Certificate(caCert, clientCert, clientKey); - } - - public static class Certificate { - - private final String caCertificate; - private final String clientCertificate; - private final String clientKey; - - public Certificate(final String caCertificate, final String clientCertificate, final String clientKey) { - this.caCertificate = caCertificate; - this.clientCertificate = clientCertificate; - this.clientKey = clientKey; - } - - public String getCaCertificate() { - return caCertificate; - } - - public String getClientCertificate() { - return clientCertificate; - } - - public String getClientKey() { - return clientKey; - } - - } - -} diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/DataSourceFactory.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/DataSourceFactory.java deleted file mode 100644 index d970b09fbba9..000000000000 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/factory/DataSourceFactory.java +++ /dev/null @@ -1,323 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.db.factory; - -import static org.postgresql.PGProperty.CONNECT_TIMEOUT; - -import com.google.common.base.Preconditions; -import com.zaxxer.hikari.HikariConfig; -import com.zaxxer.hikari.HikariDataSource; -import java.io.Closeable; -import java.time.Duration; -import java.util.Map; -import java.util.Objects; -import javax.sql.DataSource; - -/** - * Temporary factory class that provides convenience methods for creating a {@link DataSource} - * instance. This class will be removed once the project has been converted to leverage an - * application framework to manage the creation and injection of {@link DataSource} objects. - */ -public class DataSourceFactory { - - /** - * Constructs a new {@link DataSource} using the provided configuration. - * - * @param username The username of the database user. - * @param password The password of the database user. - * @param driverClassName The fully qualified name of the JDBC driver class. - * @param jdbcConnectionString The JDBC connection string. - * @return The configured {@link DataSource}. - */ - public static DataSource create(final String username, - final String password, - final String driverClassName, - final String jdbcConnectionString) { - return new DataSourceBuilder() - .withDriverClassName(driverClassName) - .withJdbcUrl(jdbcConnectionString) - .withPassword(password) - .withUsername(username) - .build(); - } - - /** - * Constructs a new {@link DataSource} using the provided configuration. - * - * @param username The username of the database user. - * @param password The password of the database user. - * @param driverClassName The fully qualified name of the JDBC driver class. - * @param jdbcConnectionString The JDBC connection string. - * @param connectionProperties Additional configuration properties for the underlying driver. - * @return The configured {@link DataSource}. - */ - public static DataSource create(final String username, - final String password, - final String driverClassName, - final String jdbcConnectionString, - final Map connectionProperties) { - return new DataSourceBuilder() - .withConnectionProperties(connectionProperties) - .withDriverClassName(driverClassName) - .withJdbcUrl(jdbcConnectionString) - .withPassword(password) - .withUsername(username) - .withConnectionTimeoutMs(DataSourceBuilder.getConnectionTimeoutMs(connectionProperties, driverClassName)) - .build(); - } - - /** - * Constructs a new {@link DataSource} using the provided configuration. - * - * @param username The username of the database user. - * @param password The password of the database user. - * @param host The host address of the database. - * @param port The port of the database. - * @param database The name of the database. - * @param driverClassName The fully qualified name of the JDBC driver class. - * @return The configured {@link DataSource}. - */ - public static DataSource create(final String username, - final String password, - final String host, - final int port, - final String database, - final String driverClassName) { - return new DataSourceBuilder() - .withDatabase(database) - .withDriverClassName(driverClassName) - .withHost(host) - .withPort(port) - .withPassword(password) - .withUsername(username) - .build(); - } - - /** - * Constructs a new {@link DataSource} using the provided configuration. - * - * @param username The username of the database user. - * @param password The password of the database user. - * @param host The host address of the database. - * @param port The port of the database. - * @param database The name of the database. - * @param driverClassName The fully qualified name of the JDBC driver class. - * @param connectionProperties Additional configuration properties for the underlying driver. - * @return The configured {@link DataSource}. - */ - public static DataSource create(final String username, - final String password, - final String host, - final int port, - final String database, - final String driverClassName, - final Map connectionProperties) { - return new DataSourceBuilder() - .withConnectionProperties(connectionProperties) - .withDatabase(database) - .withDriverClassName(driverClassName) - .withHost(host) - .withPort(port) - .withPassword(password) - .withUsername(username) - .build(); - } - - /** - * Convenience method that constructs a new {@link DataSource} for a PostgreSQL database using the - * provided configuration. - * - * @param username The username of the database user. - * @param password The password of the database user. - * @param host The host address of the database. - * @param port The port of the database. - * @param database The name of the database. - * @return The configured {@link DataSource}. - */ - public static DataSource createPostgres(final String username, - final String password, - final String host, - final int port, - final String database) { - return new DataSourceBuilder() - .withDatabase(database) - .withDriverClassName("org.postgresql.Driver") - .withHost(host) - .withPort(port) - .withPassword(password) - .withUsername(username) - .build(); - } - - /** - * Utility method that attempts to close the provided {@link DataSource} if it implements - * {@link Closeable}. - * - * @param dataSource The {@link DataSource} to close. - * @throws Exception if unable to close the data source. - */ - public static void close(final DataSource dataSource) throws Exception { - if (dataSource != null) { - if (dataSource instanceof AutoCloseable closeable) { - closeable.close(); - } - } - } - - /** - * Builder class used to configure and construct {@link DataSource} instances. - */ - private static class DataSourceBuilder { - - private Map connectionProperties = Map.of(); - private String database; - private String driverClassName; - private String host; - private String jdbcUrl; - private int maximumPoolSize = 10; - private int minimumPoolSize = 0; - private long connectionTimeoutMs; - private String password; - private int port = 5432; - private String username; - private static final String CONNECT_TIMEOUT_KEY = "connectTimeout"; - private static final Duration CONNECT_TIMEOUT_DEFAULT = Duration.ofSeconds(60); - - private DataSourceBuilder() {} - - /** - * Retrieves connectionTimeout value from connection properties in seconds, default minimum timeout - * is 60 seconds since Hikari default of 30 seconds is not enough for acceptance tests. In the case - * the value is 0, pass the value along as Hikari and Postgres use default max value for 0 timeout - * value - * - * NOTE: HikariCP uses milliseconds for all time values: - * https://github.com/brettwooldridge/HikariCP#gear-configuration-knobs-baby whereas Postgres is - * measured in seconds: https://jdbc.postgresql.org/documentation/head/connect.html - * - * @param connectionProperties custom jdbc_url_parameters containing information on connection - * properties - * @param driverClassName name of the JDBC driver - * @return DataSourceBuilder class used to create dynamic fields for DataSource - */ - private static long getConnectionTimeoutMs(final Map connectionProperties, String driverClassName) { - // TODO: the usage of CONNECT_TIMEOUT is Postgres specific, may need to extend for other databases - if (driverClassName.equals(DatabaseDriver.POSTGRESQL.getDriverClassName())) { - final String pgPropertyConnectTimeout = CONNECT_TIMEOUT.getName(); - // If the PGProperty.CONNECT_TIMEOUT was set by the user, then take its value, if not take the - // default - if (connectionProperties.containsKey(pgPropertyConnectTimeout) - && (Long.parseLong(connectionProperties.get(pgPropertyConnectTimeout)) >= 0)) { - return Duration.ofSeconds(Long.parseLong(connectionProperties.get(pgPropertyConnectTimeout))).toMillis(); - } else { - return Duration.ofSeconds(Long.parseLong(Objects.requireNonNull(CONNECT_TIMEOUT.getDefaultValue()))).toMillis(); - } - } - final Duration connectionTimeout; - connectionTimeout = - connectionProperties.containsKey(CONNECT_TIMEOUT_KEY) ? Duration.ofSeconds(Long.parseLong(connectionProperties.get(CONNECT_TIMEOUT_KEY))) - : CONNECT_TIMEOUT_DEFAULT; - if (connectionTimeout.getSeconds() == 0) { - return connectionTimeout.toMillis(); - } else { - return (connectionTimeout.compareTo(CONNECT_TIMEOUT_DEFAULT) > 0 ? connectionTimeout : CONNECT_TIMEOUT_DEFAULT).toMillis(); - } - } - - public DataSourceBuilder withConnectionProperties(final Map connectionProperties) { - if (connectionProperties != null) { - this.connectionProperties = connectionProperties; - } - return this; - } - - public DataSourceBuilder withDatabase(final String database) { - this.database = database; - return this; - } - - public DataSourceBuilder withDriverClassName(final String driverClassName) { - this.driverClassName = driverClassName; - return this; - } - - public DataSourceBuilder withHost(final String host) { - this.host = host; - return this; - } - - public DataSourceBuilder withJdbcUrl(final String jdbcUrl) { - this.jdbcUrl = jdbcUrl; - return this; - } - - public DataSourceBuilder withMaximumPoolSize(final Integer maximumPoolSize) { - if (maximumPoolSize != null) { - this.maximumPoolSize = maximumPoolSize; - } - return this; - } - - public DataSourceBuilder withMinimumPoolSize(final Integer minimumPoolSize) { - if (minimumPoolSize != null) { - this.minimumPoolSize = minimumPoolSize; - } - return this; - } - - public DataSourceBuilder withConnectionTimeoutMs(final Long connectionTimeoutMs) { - if (connectionTimeoutMs != null) { - this.connectionTimeoutMs = connectionTimeoutMs; - } - return this; - } - - public DataSourceBuilder withPassword(final String password) { - this.password = password; - return this; - } - - public DataSourceBuilder withPort(final Integer port) { - if (port != null) { - this.port = port; - } - return this; - } - - public DataSourceBuilder withUsername(final String username) { - this.username = username; - return this; - } - - public DataSource build() { - final DatabaseDriver databaseDriver = DatabaseDriver.findByDriverClassName(driverClassName); - - Preconditions.checkNotNull(databaseDriver, "Unknown or blank driver class name: '" + driverClassName + "'."); - - final HikariConfig config = new HikariConfig(); - - config.setDriverClassName(databaseDriver.getDriverClassName()); - config.setJdbcUrl(jdbcUrl != null ? jdbcUrl : String.format(databaseDriver.getUrlFormatString(), host, port, database)); - config.setMaximumPoolSize(maximumPoolSize); - config.setMinimumIdle(minimumPoolSize); - config.setConnectionTimeout(connectionTimeoutMs); - config.setPassword(password); - config.setUsername(username); - - /* - * Disable to prevent failing on startup. Applications may start prior to the database container - * being available. To avoid failing to create the connection pool, disable the fail check. This - * will preserve existing behavior that tests for the connection on first use, not on creation. - */ - config.setInitializationFailTimeout(Integer.MIN_VALUE); - - connectionProperties.forEach(config::addDataSourceProperty); - - return new HikariDataSource(config); - } - - } - -} diff --git a/airbyte-integrations/bases/base-java-s3/.dockerignore b/airbyte-integrations/bases/base-java-s3/.dockerignore deleted file mode 100644 index efa02761302a..000000000000 --- a/airbyte-integrations/bases/base-java-s3/.dockerignore +++ /dev/null @@ -1,4 +0,0 @@ -* -!Dockerfile -!build -!javabase.sh diff --git a/airbyte-integrations/bases/base-java-s3/build.gradle b/airbyte-integrations/bases/base-java-s3/build.gradle deleted file mode 100644 index 55da35f87c44..000000000000 --- a/airbyte-integrations/bases/base-java-s3/build.gradle +++ /dev/null @@ -1,39 +0,0 @@ -plugins { - id 'java-library' -} - -dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - - implementation 'org.apache.commons:commons-csv:1.4' - implementation 'com.github.alexmojaki:s3-stream-upload:2.2.2' - - implementation ('org.apache.parquet:parquet-avro:1.12.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} - implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} - implementation group: 'com.hadoop.gplcompression', name: 'hadoop-lzo', version: '0.4.20' - - // parquet - implementation ('org.apache.hadoop:hadoop-common:3.3.3') { - exclude group: 'org.slf4j', module: 'slf4j-log4j12' - exclude group: 'org.slf4j', module: 'slf4j-reload4j' - } - implementation ('org.apache.hadoop:hadoop-aws:3.3.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} - - implementation ('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.3') { - exclude group: 'org.slf4j', module: 'slf4j-log4j12' - exclude group: 'org.slf4j', module: 'slf4j-reload4j' - } - - implementation ('org.apache.parquet:parquet-avro:1.12.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} - implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} - - testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation 'org.xerial.snappy:snappy-java:1.1.8.4' - testImplementation "org.mockito:mockito-inline:4.1.0" - - testImplementation 'org.junit.jupiter:junit-jupiter-api:5.8.1' - testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.8.1' -} diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/NoEncryption.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/NoEncryption.java deleted file mode 100644 index 4129f753f7fe..000000000000 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/NoEncryption.java +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.s3; - -public final class NoEncryption implements EncryptionConfig { - -} diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/SerializedBufferFactory.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/SerializedBufferFactory.java deleted file mode 100644 index d1eb12f9be27..000000000000 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/SerializedBufferFactory.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.s3; - -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.record_buffer.BufferCreateFunction; -import io.airbyte.integrations.destination.record_buffer.BufferStorage; -import io.airbyte.integrations.destination.record_buffer.SerializableBuffer; -import io.airbyte.integrations.destination.s3.avro.AvroSerializedBuffer; -import io.airbyte.integrations.destination.s3.avro.S3AvroFormatConfig; -import io.airbyte.integrations.destination.s3.csv.CsvSerializedBuffer; -import io.airbyte.integrations.destination.s3.csv.S3CsvFormatConfig; -import io.airbyte.integrations.destination.s3.jsonl.JsonLSerializedBuffer; -import io.airbyte.integrations.destination.s3.jsonl.S3JsonlFormatConfig; -import io.airbyte.integrations.destination.s3.parquet.ParquetSerializedBuffer; -import java.util.concurrent.Callable; -import java.util.function.Function; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class SerializedBufferFactory { - - protected static final Logger LOGGER = LoggerFactory.getLogger(SerializedBufferFactory.class); - - /** - * When running a - * {@link io.airbyte.integrations.destination.record_buffer.SerializedBufferingStrategy}, it would - * usually need to instantiate new buffers when flushing data or when it receives data for a - * brand-new stream. This factory fills this need and @return the function to be called on such - * events. - *

      - * The factory is responsible for choosing the correct constructor function for a new - * {@link SerializableBuffer} that handles the correct serialized format of the data. It is - * configured by composition with another function to create a new {@link BufferStorage} where to - * store it. - *

      - * This factory determines which {@link S3FormatConfig} to use depending on the user provided @param - * config, The @param createStorageFunctionWithoutExtension is the constructor function to call when - * creating a new buffer where to store data. Note that we typically associate which format is being - * stored in the storage object thanks to its file extension. - */ - public static BufferCreateFunction getCreateFunction(final S3DestinationConfig config, - final Function createStorageFunctionWithoutExtension) { - final S3FormatConfig formatConfig = config.getFormatConfig(); - LOGGER.info("S3 format config: {}", formatConfig.toString()); - switch (formatConfig.getFormat()) { - case AVRO -> { - final Callable createStorageFunctionWithExtension = - () -> createStorageFunctionWithoutExtension.apply(formatConfig.getFileExtension()); - return AvroSerializedBuffer.createFunction((S3AvroFormatConfig) formatConfig, createStorageFunctionWithExtension); - } - case CSV -> { - final Callable createStorageFunctionWithExtension = - () -> createStorageFunctionWithoutExtension.apply(formatConfig.getFileExtension()); - return CsvSerializedBuffer.createFunction((S3CsvFormatConfig) formatConfig, createStorageFunctionWithExtension); - } - case JSONL -> { - final Callable createStorageFunctionWithExtension = - () -> createStorageFunctionWithoutExtension.apply(formatConfig.getFileExtension()); - return JsonLSerializedBuffer.createBufferFunction((S3JsonlFormatConfig) formatConfig, createStorageFunctionWithExtension); - } - case PARQUET -> { - // we can't choose the type of buffer storage with parquet because of how the underlying hadoop - // library is imposing file usage. - return ParquetSerializedBuffer.createFunction(config); - } - default -> { - throw new RuntimeException("Unexpected output format: " + Jsons.serialize(config)); - } - } - } - -} diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/WriteConfig.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/WriteConfig.java deleted file mode 100644 index 4b24580fac36..000000000000 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/WriteConfig.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.s3; - -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import java.util.ArrayList; -import java.util.List; - -/** - * Write configuration POJO for blob storage destinations - */ -public class WriteConfig { - - private final String namespace; - private final String streamName; - private final String outputBucketPath; - private final String pathFormat; - private final String fullOutputPath; - private final DestinationSyncMode syncMode; - private final List storedFiles; - - public WriteConfig(final String namespace, - final String streamName, - final String outputBucketPath, - final String pathFormat, - final String fullOutputPath, - final DestinationSyncMode syncMode) { - this.namespace = namespace; - this.streamName = streamName; - this.outputBucketPath = outputBucketPath; - this.pathFormat = pathFormat; - this.fullOutputPath = fullOutputPath; - this.syncMode = syncMode; - this.storedFiles = new ArrayList<>(); - } - - public String getNamespace() { - return namespace; - } - - public String getStreamName() { - return streamName; - } - - public String getOutputBucketPath() { - return outputBucketPath; - } - - public String getPathFormat() { - return pathFormat; - } - - public String getFullOutputPath() { - return fullOutputPath; - } - - public DestinationSyncMode getSyncMode() { - return syncMode; - } - - public List getStoredFiles() { - return storedFiles; - } - - public void addStoredFile(final String file) { - storedFiles.add(file); - } - - public void clearStoredFiles() { - storedFiles.clear(); - } - - @Override - public String toString() { - return "WriteConfig{" + - "streamName=" + streamName + - ", namespace=" + namespace + - ", outputBucketPath=" + outputBucketPath + - ", pathFormat=" + pathFormat + - ", fullOutputPath=" + fullOutputPath + - ", syncMode=" + syncMode + - '}'; - } - -} diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/AvroRecordFactory.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/AvroRecordFactory.java deleted file mode 100644 index e5ad1755fa57..000000000000 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/AvroRecordFactory.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.s3.avro; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.commons.jackson.MoreMappers; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.base.TypingAndDedupingFlag; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import java.util.UUID; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericData; -import tech.allegro.schema.json2avro.converter.JsonAvroConverter; - -public class AvroRecordFactory { - - private static final ObjectMapper MAPPER = MoreMappers.initMapper(); - private static final ObjectWriter WRITER = MAPPER.writer(); - - private final Schema schema; - private final JsonAvroConverter converter; - - public AvroRecordFactory(final Schema schema, final JsonAvroConverter converter) { - this.schema = schema; - this.converter = converter; - } - - public GenericData.Record getAvroRecord(final UUID id, final AirbyteRecordMessage recordMessage) throws JsonProcessingException { - final ObjectNode jsonRecord = MAPPER.createObjectNode(); - if (TypingAndDedupingFlag.isDestinationV2()) { - jsonRecord.put(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, id.toString()); - jsonRecord.put(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT, recordMessage.getEmittedAt()); - jsonRecord.put(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT, (Long) null); - } else { - jsonRecord.put(JavaBaseConstants.COLUMN_NAME_AB_ID, id.toString()); - jsonRecord.put(JavaBaseConstants.COLUMN_NAME_EMITTED_AT, recordMessage.getEmittedAt()); - } - jsonRecord.setAll((ObjectNode) recordMessage.getData()); - - return converter.convertToGenericDataRecord(WRITER.writeValueAsBytes(jsonRecord), schema); - } - - public GenericData.Record getAvroRecord(JsonNode formattedData) throws JsonProcessingException { - var bytes = WRITER.writeValueAsBytes(formattedData); - return converter.convertToGenericDataRecord(bytes, schema); - } - -} diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/JsonSchemaType.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/JsonSchemaType.java deleted file mode 100644 index 881aaed9f06d..000000000000 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/JsonSchemaType.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.s3.avro; - -import java.util.Arrays; -import java.util.List; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import org.apache.avro.Schema; - -/** - * Mapping of JsonSchema types to Avro types. - */ -public enum JsonSchemaType { - - STRING_V1("WellKnownTypes.json#/definitions/String", Schema.Type.STRING), - INTEGER_V1("WellKnownTypes.json#/definitions/Integer", Schema.Type.LONG), - NUMBER_V1("WellKnownTypes.json#/definitions/Number", Schema.Type.DOUBLE), - BOOLEAN_V1("WellKnownTypes.json#/definitions/Boolean", Schema.Type.BOOLEAN), - BINARY_DATA_V1("WellKnownTypes.json#/definitions/BinaryData", Schema.Type.BYTES), - DATE_V1("WellKnownTypes.json#/definitions/Date", Schema.Type.INT), - TIMESTAMP_WITH_TIMEZONE_V1("WellKnownTypes.json#/definitions/TimestampWithTimezone", Schema.Type.LONG), - TIMESTAMP_WITHOUT_TIMEZONE_V1("WellKnownTypes.json#/definitions/TimestampWithoutTimezone", Schema.Type.LONG), - TIME_WITH_TIMEZONE_V1("WellKnownTypes.json#/definitions/TimeWithTimezone", Schema.Type.STRING), - TIME_WITHOUT_TIMEZONE_V1("WellKnownTypes.json#/definitions/TimeWithoutTimezone", Schema.Type.LONG), - OBJECT("object", Schema.Type.RECORD), - ARRAY("array", Schema.Type.ARRAY), - COMBINED("combined", Schema.Type.UNION), - @Deprecated - STRING_V0("string", null, Schema.Type.STRING), - @Deprecated - NUMBER_INT_V0("number", "integer", Schema.Type.LONG), - @Deprecated - NUMBER_BIGINT_V0("string", "big_integer", Schema.Type.STRING), - @Deprecated - NUMBER_FLOAT_V0("number", "float", Schema.Type.FLOAT), - @Deprecated - NUMBER_V0("number", null, Schema.Type.DOUBLE), - @Deprecated - INTEGER_V0("integer", null, Schema.Type.LONG), - @Deprecated - BOOLEAN_V0("boolean", null, Schema.Type.BOOLEAN), - @Deprecated - NULL("null", null, Schema.Type.NULL); - - private final String jsonSchemaType; - private final Schema.Type avroType; - private String jsonSchemaAirbyteType; - - JsonSchemaType(final String jsonSchemaType, final String jsonSchemaAirbyteType, final Schema.Type avroType) { - this.jsonSchemaType = jsonSchemaType; - this.jsonSchemaAirbyteType = jsonSchemaAirbyteType; - this.avroType = avroType; - } - - JsonSchemaType(final String jsonSchemaType, final Schema.Type avroType) { - this.jsonSchemaType = jsonSchemaType; - this.avroType = avroType; - } - - public static JsonSchemaType fromJsonSchemaType(final String jsonSchemaType) { - return fromJsonSchemaType(jsonSchemaType, null); - } - - public static JsonSchemaType fromJsonSchemaType(final @Nonnull String jsonSchemaType, final @Nullable String jsonSchemaAirbyteType) { - List matchSchemaType = null; - // Match by Type + airbyteType - if (jsonSchemaAirbyteType != null) { - matchSchemaType = Arrays.stream(values()) - .filter(type -> jsonSchemaType.equals(type.jsonSchemaType)) - .filter(type -> jsonSchemaAirbyteType.equals(type.jsonSchemaAirbyteType)) - .toList(); - } - - // Match by Type are no results already - if (matchSchemaType == null || matchSchemaType.isEmpty()) { - matchSchemaType = - Arrays.stream(values()).filter(format -> jsonSchemaType.equals(format.jsonSchemaType) && format.jsonSchemaAirbyteType == null).toList(); - } - - if (matchSchemaType.isEmpty()) { - throw new IllegalArgumentException( - String.format("Unexpected jsonSchemaType - %s and jsonSchemaAirbyteType - %s", jsonSchemaType, jsonSchemaAirbyteType)); - } else if (matchSchemaType.size() > 1) { - throw new RuntimeException( - String.format("Match with more than one json type! Matched types : %s, Inputs jsonSchemaType : %s, jsonSchemaAirbyteType : %s", - matchSchemaType, jsonSchemaType, jsonSchemaAirbyteType)); - } else { - return matchSchemaType.get(0); - } - } - - public String getJsonSchemaType() { - return jsonSchemaType; - } - - public Schema.Type getAvroType() { - return avroType; - } - - @Override - public String toString() { - return jsonSchemaType; - } - - public String getJsonSchemaAirbyteType() { - return jsonSchemaAirbyteType; - } - -} diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/credential/BlobStorageCredentialConfig.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/credential/BlobStorageCredentialConfig.java deleted file mode 100644 index 1a49ae23e2b6..000000000000 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/credential/BlobStorageCredentialConfig.java +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.s3.credential; - -public interface BlobStorageCredentialConfig { - - CredentialType getCredentialType(); - -} diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/credential/S3CredentialType.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/credential/S3CredentialType.java deleted file mode 100644 index cc5ecd18389f..000000000000 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/credential/S3CredentialType.java +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.s3.credential; - -public enum S3CredentialType { - - ACCESS_KEY, - DEFAULT_PROFILE - -} diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSheetGenerators.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSheetGenerators.java deleted file mode 100644 index bd189669e866..000000000000 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSheetGenerators.java +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.s3.csv; - -public class CsvSheetGenerators { - -} diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/S3JsonlFormatConfig.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/S3JsonlFormatConfig.java deleted file mode 100644 index 7b317e5d5e2e..000000000000 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/jsonl/S3JsonlFormatConfig.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.s3.jsonl; - -import static io.airbyte.integrations.destination.s3.S3DestinationConstants.COMPRESSION_ARG_NAME; -import static io.airbyte.integrations.destination.s3.S3DestinationConstants.DEFAULT_COMPRESSION_TYPE; -import static io.airbyte.integrations.destination.s3.S3DestinationConstants.FLATTENING_ARG_NAME; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.destination.s3.S3Format; -import io.airbyte.integrations.destination.s3.S3FormatConfig; -import io.airbyte.integrations.destination.s3.util.CompressionType; -import io.airbyte.integrations.destination.s3.util.CompressionTypeHelper; -import io.airbyte.integrations.destination.s3.util.Flattening; -import java.util.Objects; -import lombok.ToString; - -@ToString -public class S3JsonlFormatConfig implements S3FormatConfig { - - public static final String JSONL_SUFFIX = ".jsonl"; - - private final Flattening flattening; - - private final CompressionType compressionType; - - public S3JsonlFormatConfig(final JsonNode formatConfig) { - this( - formatConfig.has(FLATTENING_ARG_NAME) - ? Flattening.fromValue(formatConfig.get(FLATTENING_ARG_NAME).asText()) - : Flattening.NO, - formatConfig.has(COMPRESSION_ARG_NAME) - ? CompressionTypeHelper.parseCompressionType(formatConfig.get(COMPRESSION_ARG_NAME)) - : DEFAULT_COMPRESSION_TYPE); - } - - public S3JsonlFormatConfig(final Flattening flattening, final CompressionType compressionType) { - this.flattening = flattening; - this.compressionType = compressionType; - } - - @Override - public S3Format getFormat() { - return S3Format.JSONL; - } - - @Override - public String getFileExtension() { - return JSONL_SUFFIX + compressionType.getFileExtension(); - } - - public CompressionType getCompressionType() { - return compressionType; - } - - public Flattening getFlatteningType() { - return flattening; - } - - @Override - public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - final S3JsonlFormatConfig that = (S3JsonlFormatConfig) o; - return flattening == that.flattening - && Objects.equals(compressionType, that.compressionType); - } - - @Override - public int hashCode() { - return Objects.hash(flattening, compressionType); - } - -} diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/CompressionTypeHelper.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/CompressionTypeHelper.java deleted file mode 100644 index 619fb1b2a85d..000000000000 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/util/CompressionTypeHelper.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.s3.util; - -import static io.airbyte.integrations.destination.s3.S3DestinationConstants.COMPRESSION_TYPE_ARG_NAME; -import static io.airbyte.integrations.destination.s3.S3DestinationConstants.DEFAULT_COMPRESSION_TYPE; - -import com.fasterxml.jackson.databind.JsonNode; - -public class CompressionTypeHelper { - - private CompressionTypeHelper() {} - - /** - * Sample expected input: { "compression_type": "No Compression" } - */ - public static CompressionType parseCompressionType(final JsonNode compressionConfig) { - if (compressionConfig == null || compressionConfig.isNull()) { - return DEFAULT_COMPRESSION_TYPE; - } - final String compressionType = compressionConfig.get(COMPRESSION_TYPE_ARG_NAME).asText(); - if (compressionType.toUpperCase().equals(CompressionType.GZIP.name())) { - return CompressionType.GZIP; - } else { - return CompressionType.NO_COMPRESSION; - } - } - -} diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/writer/DestinationFileWriter.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/writer/DestinationFileWriter.java deleted file mode 100644 index d5bd233f3ed5..000000000000 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/writer/DestinationFileWriter.java +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.s3.writer; - -import io.airbyte.integrations.destination.s3.S3Format; - -public interface DestinationFileWriter extends DestinationWriter { - - String getFileLocation(); - - S3Format getFileFormat(); - - String getOutputPath(); - -} diff --git a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/writer/DestinationWriter.java b/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/writer/DestinationWriter.java deleted file mode 100644 index 47cf61a93790..000000000000 --- a/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/writer/DestinationWriter.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.s3.writer; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import java.io.IOException; -import java.util.UUID; - -/** - * {@link DestinationWriter} is responsible for writing Airbyte stream data to an S3 location in a - * specific format. - */ -public interface DestinationWriter { - - /** - * Prepare an S3 writer for the stream. - */ - void initialize() throws IOException; - - /** - * Write an Airbyte record message to an S3 object. - */ - void write(UUID id, AirbyteRecordMessage recordMessage) throws IOException; - - void write(JsonNode formattedData) throws IOException; - - /** - * Close the S3 writer for the stream. - */ - void close(boolean hasFailed) throws IOException; - -} diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/S3StorageOperationsTest.java b/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/S3StorageOperationsTest.java deleted file mode 100644 index ca1122ea02f8..000000000000 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/S3StorageOperationsTest.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.s3; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.ListObjectsRequest; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.s3.util.S3NameTransformer; -import java.util.List; -import java.util.regex.Pattern; -import org.joda.time.DateTime; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; - -public class S3StorageOperationsTest { - - private static final String BUCKET_NAME = "fake-bucket"; - private static final String FAKE_BUCKET_PATH = "fake-bucketPath"; - private static final String NAMESPACE = "namespace"; - private static final String STREAM_NAME = "stream_name1"; - private static final String OBJECT_TO_DELETE = NAMESPACE + "/" + STREAM_NAME + "/2022_04_04_123456789_0.csv.gz"; - private AmazonS3 s3Client; - private S3StorageOperations s3StorageOperations; - - @BeforeEach - public void setup() { - final NamingConventionTransformer nameTransformer = new S3NameTransformer(); - s3Client = mock(AmazonS3.class); - - final S3ObjectSummary objectSummary1 = mock(S3ObjectSummary.class); - final S3ObjectSummary objectSummary2 = mock(S3ObjectSummary.class); - final S3ObjectSummary objectSummary3 = mock(S3ObjectSummary.class); - when(objectSummary1.getKey()).thenReturn(OBJECT_TO_DELETE); - when(objectSummary2.getKey()).thenReturn(NAMESPACE + "/stream_name2/2022_04_04_123456789_0.csv.gz"); - when(objectSummary3.getKey()).thenReturn("other_files.txt"); - - final ObjectListing results = mock(ObjectListing.class); - when(results.isTruncated()).thenReturn(false); - when(results.getObjectSummaries()).thenReturn(List.of(objectSummary1, objectSummary2, objectSummary3)); - when(s3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(results); - - final S3DestinationConfig s3Config = S3DestinationConfig.create(BUCKET_NAME, FAKE_BUCKET_PATH, "fake-region") - .withEndpoint("fake-endpoint") - .withAccessKeyCredential("fake-accessKeyId", "fake-secretAccessKey") - .withS3Client(s3Client) - .get(); - s3StorageOperations = new S3StorageOperations(nameTransformer, s3Client, s3Config); - } - - @Test - void testRegexMatch() { - final Pattern regexFormat = - Pattern.compile(s3StorageOperations.getRegexFormat(NAMESPACE, STREAM_NAME, S3DestinationConstants.DEFAULT_PATH_FORMAT)); - assertTrue(regexFormat.matcher(OBJECT_TO_DELETE).matches()); - assertTrue(regexFormat - .matcher(s3StorageOperations.getBucketObjectPath(NAMESPACE, STREAM_NAME, DateTime.now(), S3DestinationConstants.DEFAULT_PATH_FORMAT)) - .matches()); - assertFalse(regexFormat.matcher(NAMESPACE + "/" + STREAM_NAME + "/some_random_file_0.doc").matches()); - assertFalse(regexFormat.matcher(NAMESPACE + "/stream_name2/2022_04_04_123456789_0.csv.gz").matches()); - } - - @Test - void testCustomRegexMatch() { - final String customFormat = "${NAMESPACE}_${STREAM_NAME}_${YEAR}-${MONTH}-${DAY}-${HOUR}-${MINUTE}-${SECOND}-${MILLISECOND}-${EPOCH}-${UUID}"; - assertTrue(Pattern - .compile(s3StorageOperations.getRegexFormat(NAMESPACE, STREAM_NAME, customFormat)) - .matcher(s3StorageOperations.getBucketObjectPath(NAMESPACE, STREAM_NAME, DateTime.now(), customFormat)).matches()); - } - - @Test - void testGetExtension() { - assertEquals(".csv.gz", S3StorageOperations.getExtension("test.csv.gz")); - assertEquals(".gz", S3StorageOperations.getExtension("test.gz")); - assertEquals(".avro", S3StorageOperations.getExtension("test.avro")); - assertEquals("", S3StorageOperations.getExtension("test-file")); - } - - @Test - void testCleanUpBucketObject() { - final String pathFormat = S3DestinationConstants.DEFAULT_PATH_FORMAT; - s3StorageOperations.cleanUpBucketObject(NAMESPACE, STREAM_NAME, FAKE_BUCKET_PATH, pathFormat); - final ArgumentCaptor deleteRequest = ArgumentCaptor.forClass(DeleteObjectsRequest.class); - verify(s3Client).deleteObjects(deleteRequest.capture()); - assertEquals(1, deleteRequest.getValue().getKeys().size()); - assertEquals(OBJECT_TO_DELETE, deleteRequest.getValue().getKeys().get(0).getKey()); - } - - @Test - void testGetFilename() { - assertEquals("filename", S3StorageOperations.getFilename("filename")); - assertEquals("filename", S3StorageOperations.getFilename("/filename")); - assertEquals("filename", S3StorageOperations.getFilename("/p1/p2/filename")); - assertEquals("filename.csv", S3StorageOperations.getFilename("/p1/p2/filename.csv")); - } - -} diff --git a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/JsonSchemaTypeTest.java b/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/JsonSchemaTypeTest.java deleted file mode 100644 index 29b9f00390ba..000000000000 --- a/airbyte-integrations/bases/base-java-s3/src/test/java/io/airbyte/integrations/destination/s3/avro/JsonSchemaTypeTest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.s3.avro; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import java.util.stream.Stream; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.ArgumentsProvider; -import org.junit.jupiter.params.provider.ArgumentsSource; - -public class JsonSchemaTypeTest { - - @ParameterizedTest - @ArgumentsSource(JsonSchemaTypeProvider.class) - public void testFromJsonSchemaType(String type, String airbyteType, JsonSchemaType expectedJsonSchemaType) { - assertEquals( - expectedJsonSchemaType, - JsonSchemaType.fromJsonSchemaType(type, airbyteType)); - } - - public static class JsonSchemaTypeProvider implements ArgumentsProvider { - - @Override - public Stream provideArguments(ExtensionContext context) { - return Stream.of( - Arguments.of("WellKnownTypes.json#/definitions/Number", null, JsonSchemaType.NUMBER_V1), - Arguments.of("WellKnownTypes.json#/definitions/String", null, JsonSchemaType.STRING_V1), - Arguments.of("WellKnownTypes.json#/definitions/Integer", null, JsonSchemaType.INTEGER_V1), - Arguments.of("WellKnownTypes.json#/definitions/Boolean", null, JsonSchemaType.BOOLEAN_V1), - Arguments.of("WellKnownTypes.json#/definitions/BinaryData", null, JsonSchemaType.BINARY_DATA_V1), - Arguments.of("WellKnownTypes.json#/definitions/Date", null, JsonSchemaType.DATE_V1), - Arguments.of("WellKnownTypes.json#/definitions/TimestampWithTimezone", null, JsonSchemaType.TIMESTAMP_WITH_TIMEZONE_V1), - Arguments.of("WellKnownTypes.json#/definitions/TimestampWithoutTimezone", null, JsonSchemaType.TIMESTAMP_WITHOUT_TIMEZONE_V1), - Arguments.of("WellKnownTypes.json#/definitions/TimeWithTimezone", null, JsonSchemaType.TIME_WITH_TIMEZONE_V1), - Arguments.of("WellKnownTypes.json#/definitions/TimeWithoutTimezone", null, JsonSchemaType.TIME_WITHOUT_TIMEZONE_V1), - Arguments.of("number", "integer", JsonSchemaType.NUMBER_INT_V0), - Arguments.of("string", "big_integer", JsonSchemaType.NUMBER_BIGINT_V0), - Arguments.of("number", "float", JsonSchemaType.NUMBER_FLOAT_V0), - Arguments.of("number", null, JsonSchemaType.NUMBER_V0), - Arguments.of("string", null, JsonSchemaType.STRING_V0), - Arguments.of("integer", null, JsonSchemaType.INTEGER_V0), - Arguments.of("boolean", null, JsonSchemaType.BOOLEAN_V0), - Arguments.of("null", null, JsonSchemaType.NULL), - Arguments.of("object", null, JsonSchemaType.OBJECT), - Arguments.of("array", null, JsonSchemaType.ARRAY), - Arguments.of("combined", null, JsonSchemaType.COMBINED)); - } - - } - -} diff --git a/airbyte-integrations/bases/base-java/Dockerfile b/airbyte-integrations/bases/base-java/Dockerfile index 416545c4bf92..d19438eab3f0 100644 --- a/airbyte-integrations/bases/base-java/Dockerfile +++ b/airbyte-integrations/bases/base-java/Dockerfile @@ -6,11 +6,11 @@ # If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. # Please reach out to the Connectors Operations team if you have any question. -ARG JDK_VERSION=17.0.4 +ARG JDK_VERSION=17.0.8 FROM amazoncorretto:${JDK_VERSION} COPY --from=airbyte/integration-base:dev /airbyte /airbyte -RUN yum install -y tar openssl && yum clean all +RUN yum update -y && yum install -y tar openssl && yum clean all WORKDIR /airbyte diff --git a/airbyte-integrations/bases/base-java/build.gradle b/airbyte-integrations/bases/base-java/build.gradle index 6bbbf4e847ff..a80a2274f156 100644 --- a/airbyte-integrations/bases/base-java/build.gradle +++ b/airbyte-integrations/bases/base-java/build.gradle @@ -1,13 +1,12 @@ plugins { id 'java-library' - id 'airbyte-docker' + id 'airbyte-docker-legacy' } dependencies { - implementation libs.airbyte.protocol - implementation project(':airbyte-config-oss:config-models-oss') - implementation project(':airbyte-commons-cli') - implementation project(':airbyte-json-validation') + implementation project(':airbyte-cdk:java:airbyte-cdk:config-models-oss') + implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons-cli') + implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') implementation 'commons-cli:commons-cli:1.4' implementation 'net.i2p.crypto:eddsa:0.3.0' @@ -19,12 +18,10 @@ dependencies { implementation 'org.bouncycastle:bctls-jdk15on:1.66' implementation libs.jackson.annotations - implementation libs.connectors.testcontainers - implementation libs.connectors.testcontainers.jdbc + implementation libs.testcontainers + implementation libs.testcontainers.jdbc implementation libs.bundles.datadog - implementation files(project(':airbyte-integrations:bases:base').airbyteDocker.outputs) - testImplementation 'commons-lang:commons-lang:2.6' implementation group: 'org.apache.logging.log4j', name: 'log4j-layout-template-json', version: '2.17.2' } diff --git a/airbyte-integrations/bases/base-java/javabase.sh b/airbyte-integrations/bases/base-java/javabase.sh index 34302052d40b..d835d4e9e0eb 100755 --- a/airbyte-integrations/bases/base-java/javabase.sh +++ b/airbyte-integrations/bases/base-java/javabase.sh @@ -15,6 +15,9 @@ if [[ $IS_CAPTURE_HEAP_DUMP_ON_ERROR = true ]]; then echo "APPLICATION=$APPLICATION" fi fi +#30781 - Allocate 32KB for log4j appender buffer to ensure that each line is logged in a single println +JAVA_OPTS=$JAVA_OPTS" -Dlog4j.encoder.byteBufferSize=32768" +export JAVA_OPTS # Wrap run script in a script so that we can lazy evaluate the value of APPLICATION. APPLICATION is # set by the dockerfile that inherits base-java, so it cannot be evaluated when base-java is built. diff --git a/airbyte-integrations/bases/base-java/run_with_normalization.sh b/airbyte-integrations/bases/base-java/run_with_normalization.sh index f61cfea63b9a..669763021803 100755 --- a/airbyte-integrations/bases/base-java/run_with_normalization.sh +++ b/airbyte-integrations/bases/base-java/run_with_normalization.sh @@ -36,9 +36,13 @@ then elif test "$NORMALIZATION_TECHNIQUE" = 'LEGACY' && test "$USE_1S1T_FORMAT" != "true" then echo '{"type": "LOG","log":{"level":"INFO","message":"Starting in-connector normalization"}}' + # Normalization tries to create this file from the connector config and crashes if it already exists + # so just nuke it and let normalization recreate it. + # Use -f to avoid error if it doesn't exist, since it's only created for certain SSL modes. + rm -f ca.crt # the args in a write command are `write --catalog foo.json --config bar.json` # so if we remove the `write`, we can just pass the rest directly into normalization - /airbyte/entrypoint.sh run ${@:2} --integration-type $AIRBYTE_NORMALIZATION_INTEGRATION | java -cp "/airbyte/lib/*" io.airbyte.integrations.destination.normalization.NormalizationLogParser + /airbyte/entrypoint.sh run ${@:2} --integration-type $AIRBYTE_NORMALIZATION_INTEGRATION | java -cp "/airbyte/lib/*" io.airbyte.cdk.integrations.destination.normalization.NormalizationLogParser normalization_exit_code=$? echo '{"type": "LOG","log":{"level":"INFO","message":"In-connector normalization done (exit code '"$normalization_exit_code"')"}}' else diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/AirbyteExceptionHandler.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/AirbyteExceptionHandler.java deleted file mode 100644 index be4e1a4d914c..000000000000 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/AirbyteExceptionHandler.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.base; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class AirbyteExceptionHandler implements Thread.UncaughtExceptionHandler { - - private static final Logger LOGGER = LoggerFactory.getLogger(AirbyteExceptionHandler.class); - public static final String logMessage = "Something went wrong in the connector. See the logs for more details."; - - @Override - public void uncaughtException(Thread t, Throwable e) { - // This is a naive AirbyteTraceMessage emission in order to emit one when any error occurs in a - // connector. - // If a connector implements AirbyteTraceMessage emission itself, this code will result in an - // additional one being emitted. - // this is fine tho because: - // "The earliest AirbyteTraceMessage where type=error will be used to populate the FailureReason for - // the sync." - // from the spec: - // https://docs.google.com/document/d/1ctrj3Yh_GjtQ93aND-WH3ocqGxsmxyC3jfiarrF6NY0/edit# - LOGGER.error(logMessage, e); - AirbyteTraceMessageUtility.emitSystemErrorTrace(e, logMessage); - terminate(); - } - - // by doing this in a separate method we can mock it to avoid closing the jvm and therefore test - // properly - protected void terminate() { - System.exit(1); - } - -} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/spec_modification/SpecModifyingDestination.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/spec_modification/SpecModifyingDestination.java deleted file mode 100644 index b06f5dab188d..000000000000 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/spec_modification/SpecModifyingDestination.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.base.spec_modification; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import java.util.function.Consumer; - -public abstract class SpecModifyingDestination implements Destination { - - private final Destination destination; - - public SpecModifyingDestination(final Destination destination) { - this.destination = destination; - } - - public abstract ConnectorSpecification modifySpec(ConnectorSpecification originalSpec) throws Exception; - - @Override - public ConnectorSpecification spec() throws Exception { - return modifySpec(destination.spec()); - } - - @Override - public AirbyteConnectionStatus check(final JsonNode config) throws Exception { - return destination.check(config); - } - - @Override - public AirbyteMessageConsumer getConsumer(final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final Consumer outputRecordCollector) - throws Exception { - return destination.getConsumer(config, catalog, outputRecordCollector); - } - -} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshBastionContainer.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshBastionContainer.java deleted file mode 100644 index 4a32e6e43eff..000000000000 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/base/ssh/SshBastionContainer.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.base.ssh; - -import static io.airbyte.integrations.base.ssh.SshHelpers.getInnerContainerAddress; -import static io.airbyte.integrations.base.ssh.SshHelpers.getOuterContainerAddress; -import static io.airbyte.integrations.base.ssh.SshTunnel.TunnelMethod.SSH_KEY_AUTH; -import static io.airbyte.integrations.base.ssh.SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import java.io.IOException; -import java.util.List; -import java.util.Objects; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.JdbcDatabaseContainer; -import org.testcontainers.containers.Network; -import org.testcontainers.images.builder.ImageFromDockerfile; - -public class SshBastionContainer { - - private static final String SSH_USER = "sshuser"; - private static final String SSH_PASSWORD = "secret"; - private GenericContainer bastion; - - public void initAndStartBastion(final Network network) { - bastion = new GenericContainer( - new ImageFromDockerfile("bastion-test") - .withFileFromClasspath("Dockerfile", "bastion/Dockerfile")) - .withNetwork(network) - .withExposedPorts(22); - bastion.start(); - } - - public JsonNode getTunnelConfig(final SshTunnel.TunnelMethod tunnelMethod, - final ImmutableMap.Builder builderWithSchema, - final boolean innerAddress) - throws IOException, InterruptedException { - final var containerAddress = innerAddress ? getInnerContainerAddress(bastion) : getOuterContainerAddress(bastion); - return Jsons.jsonNode(builderWithSchema - .put("tunnel_method", Jsons.jsonNode(ImmutableMap.builder() - .put("tunnel_host", - Objects.requireNonNull(containerAddress.left)) - .put("tunnel_method", tunnelMethod) - .put("tunnel_port", containerAddress.right) - .put("tunnel_user", SSH_USER) - .put("tunnel_user_password", tunnelMethod.equals(SSH_PASSWORD_AUTH) ? SSH_PASSWORD : "") - .put("ssh_key", tunnelMethod.equals(SSH_KEY_AUTH) ? bastion.execInContainer("cat", "var/bastion/id_rsa").getStdout() : "") - .build())) - .build()); - } - - public ImmutableMap.Builder getBasicDbConfigBuider(final JdbcDatabaseContainer db) { - return getBasicDbConfigBuider(db, db.getDatabaseName()); - } - - public ImmutableMap.Builder getBasicDbConfigBuider(final JdbcDatabaseContainer db, final List schemas) { - return getBasicDbConfigBuider(db, db.getDatabaseName()).put("schemas", schemas); - } - - public ImmutableMap.Builder getBasicDbConfigBuider(final JdbcDatabaseContainer db, final String schemaName) { - return ImmutableMap.builder() - .put("host", Objects.requireNonNull(db.getContainerInfo().getNetworkSettings() - .getNetworks() - .entrySet().stream().findFirst().get().getValue().getIpAddress())) - .put("username", db.getUsername()) - .put("password", db.getPassword()) - .put("port", db.getExposedPorts().get(0)) - .put("database", schemaName) - .put("ssl", false); - } - - public void stopAndCloseContainers(final JdbcDatabaseContainer db) { - bastion.stop(); - bastion.close(); - db.stop(); - db.close(); - } - - public void stopAndClose() { - bastion.close(); - } - - public GenericContainer getContainer() { - return bastion; - } - -} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/OnCloseFunction.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/OnCloseFunction.java deleted file mode 100644 index 662125938210..000000000000 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/OnCloseFunction.java +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.buffered_stream_consumer; - -import io.airbyte.commons.functional.CheckedConsumer; - -/** - * Interface allowing destination to specify clean up logic that must be executed after all - * record-related logic has finished. - */ -public interface OnCloseFunction extends CheckedConsumer { - - @Override - void accept(Boolean hasFailed) throws Exception; - -} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/OnStartFunction.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/OnStartFunction.java deleted file mode 100644 index ebef22ef9280..000000000000 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/buffered_stream_consumer/OnStartFunction.java +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.buffered_stream_consumer; - -import io.airbyte.commons.concurrency.VoidCallable; - -public interface OnStartFunction extends VoidCallable { - -} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/BufferFlushType.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/BufferFlushType.java deleted file mode 100644 index 3d2a85b77f96..000000000000 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination/record_buffer/BufferFlushType.java +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.record_buffer; - -public enum BufferFlushType { - FLUSH_ALL, - FLUSH_SINGLE_STREAM -} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/AsyncStreamConsumer.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/AsyncStreamConsumer.java deleted file mode 100644 index 435a0b11a94c..000000000000 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/AsyncStreamConsumer.java +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination_async; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.base.Strings; -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.SerializedAirbyteMessageConsumer; -import io.airbyte.integrations.destination.buffered_stream_consumer.OnStartFunction; -import io.airbyte.integrations.destination_async.buffers.BufferEnqueue; -import io.airbyte.integrations.destination_async.buffers.BufferManager; -import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteMessage; -import io.airbyte.integrations.destination_async.state.FlushFailure; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteMessage.Type; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.StreamDescriptor; -import java.util.Optional; -import java.util.Set; -import java.util.function.Consumer; -import lombok.extern.slf4j.Slf4j; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Async version of the - * {@link io.airbyte.integrations.destination.buffered_stream_consumer.BufferedStreamConsumer}. - *

      - * With this consumer, a destination is able to continue reading records until hitting the maximum - * memory limit governed by {@link GlobalMemoryManager}. Record writing is decoupled via - * {@link FlushWorkers}. See the other linked class for more detail. - */ -@Slf4j -public class AsyncStreamConsumer implements SerializedAirbyteMessageConsumer { - - private static final Logger LOGGER = LoggerFactory.getLogger(AsyncStreamConsumer.class); - - private final OnStartFunction onStart; - private final OnCloseFunction onClose; - private final ConfiguredAirbyteCatalog catalog; - private final BufferManager bufferManager; - private final BufferEnqueue bufferEnqueue; - private final FlushWorkers flushWorkers; - private final Set streamNames; - private final FlushFailure flushFailure; - private final String defaultNamespace; - - private boolean hasStarted; - private boolean hasClosed; - // This is to account for the references when deserialization to a PartialAirbyteMessage. The - // calculation is as follows: - // PartialAirbyteMessage (4) + Max( PartialRecordMessage(4), PartialStateMessage(6)) with - // PartialStateMessage being larger with more nested objects within it. Using 8 bytes as we assumed - // a 64 bit JVM. - final int PARTIAL_DESERIALIZE_REF_BYTES = 10 * 8; - - public AsyncStreamConsumer(final Consumer outputRecordCollector, - final OnStartFunction onStart, - final OnCloseFunction onClose, - final DestinationFlushFunction flusher, - final ConfiguredAirbyteCatalog catalog, - final BufferManager bufferManager, - final String defaultNamespace) { - this(outputRecordCollector, onStart, onClose, flusher, catalog, bufferManager, new FlushFailure(), defaultNamespace); - } - - @VisibleForTesting - public AsyncStreamConsumer(final Consumer outputRecordCollector, - final OnStartFunction onStart, - final OnCloseFunction onClose, - final DestinationFlushFunction flusher, - final ConfiguredAirbyteCatalog catalog, - final BufferManager bufferManager, - final FlushFailure flushFailure, - final String defaultNamespace) { - this.defaultNamespace = defaultNamespace; - hasStarted = false; - hasClosed = false; - - this.onStart = onStart; - this.onClose = onClose; - this.catalog = catalog; - this.bufferManager = bufferManager; - bufferEnqueue = bufferManager.getBufferEnqueue(); - this.flushFailure = flushFailure; - flushWorkers = new FlushWorkers(bufferManager.getBufferDequeue(), flusher, outputRecordCollector, flushFailure, bufferManager.getStateManager()); - streamNames = StreamDescriptorUtils.fromConfiguredCatalog(catalog); - } - - @Override - public void start() throws Exception { - Preconditions.checkState(!hasStarted, "Consumer has already been started."); - hasStarted = true; - - flushWorkers.start(); - - LOGGER.info("{} started.", AsyncStreamConsumer.class); - onStart.call(); - } - - @Override - public void accept(final String messageString, final Integer sizeInBytes) throws Exception { - Preconditions.checkState(hasStarted, "Cannot accept records until consumer has started"); - propagateFlushWorkerExceptionIfPresent(); - /* - * intentionally putting extractStream outside the buffer manager so that if in the future we want - * to try to use a thread pool to partially deserialize to get record type and stream name, we can - * do it without touching buffer manager. - */ - deserializeAirbyteMessage(messageString) - .ifPresent(message -> { - if (Type.RECORD.equals(message.getType())) { - if (Strings.isNullOrEmpty(message.getRecord().getNamespace())) { - message.getRecord().setNamespace(defaultNamespace); - } - validateRecord(message); - } - bufferEnqueue.addRecord(message, sizeInBytes + PARTIAL_DESERIALIZE_REF_BYTES); - }); - } - - /** - * Deserializes to a {@link PartialAirbyteMessage} which can represent both a Record or a State - * Message - * - * PartialAirbyteMessage holds either: - *

    • entire serialized message string when message is a valid State Message - *
    • serialized AirbyteRecordMessage when message is a valid Record Message
    • - * - * @param messageString the string to deserialize - * @return PartialAirbyteMessage if the message is valid, empty otherwise - */ - @VisibleForTesting - public static Optional deserializeAirbyteMessage(final String messageString) { - // TODO: (ryankfu) plumb in the serialized AirbyteStateMessage to match AirbyteRecordMessage code - // parity. https://github.com/airbytehq/airbyte/issues/27530 for additional context - final Optional messageOptional = Jsons.tryDeserialize(messageString, PartialAirbyteMessage.class) - .map(partial -> { - if (Type.RECORD.equals(partial.getType()) && partial.getRecord().getData() != null) { - return partial.withSerialized(partial.getRecord().getData().toString()); - } else if (Type.STATE.equals(partial.getType())) { - return partial.withSerialized(messageString); - } else { - return null; - } - }); - - if (messageOptional.isPresent()) { - return messageOptional; - } - throw new RuntimeException("Invalid serialized message"); - } - - @Override - public void close() throws Exception { - Preconditions.checkState(hasStarted, "Cannot close; has not started."); - Preconditions.checkState(!hasClosed, "Has already closed."); - hasClosed = true; - - // assume closing upload workers will flush all accepted records. - // we need to close the workers before closing the bufferManagers (and underlying buffers) - // or we risk in-memory data. - flushWorkers.close(); - - bufferManager.close(); - onClose.call(); - - // as this throws an exception, we need to be after all other close functions. - propagateFlushWorkerExceptionIfPresent(); - LOGGER.info("{} closed", AsyncStreamConsumer.class); - } - - private void propagateFlushWorkerExceptionIfPresent() throws Exception { - if (flushFailure.isFailed()) { - throw flushFailure.getException(); - } - } - - private void validateRecord(final PartialAirbyteMessage message) { - final StreamDescriptor streamDescriptor = new StreamDescriptor() - .withNamespace(message.getRecord().getNamespace()) - .withName(message.getRecord().getStream()); - // if stream is not part of list of streams to sync to then throw invalid stream exception - if (!streamNames.contains(streamDescriptor)) { - throwUnrecognizedStream(catalog, message); - } - } - - private static void throwUnrecognizedStream(final ConfiguredAirbyteCatalog catalog, final PartialAirbyteMessage message) { - throw new IllegalArgumentException( - String.format("Message contained record from a stream that was not in the catalog. \ncatalog: %s , \nmessage: %s", - Jsons.serialize(catalog), Jsons.serialize(message))); - } - -} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/DestinationFlushFunction.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/DestinationFlushFunction.java deleted file mode 100644 index 559fc8eaac2d..000000000000 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/DestinationFlushFunction.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination_async; - -import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.StreamDescriptor; -import java.util.stream.Stream; - -/** - * An interface meant to be used with {@link FlushWorkers}. - *

      - * A destination instructs workers how to write data by specifying - * {@link #flush(StreamDescriptor, Stream)}. This keeps the worker abstraction generic and reusable. - *

      - * e.g. A database destination's flush function likely involves parsing the stream into SQL - * statements. - *

      - * There are 2 different destination types as of this writing: - *

        - *
      • 1. Destinations that upload files. This includes warehouses and databases.
      • - *
      • 2. Destinations that upload data streams. This mostly includes various Cloud storages. This - * will include reverse-ETL in the future
      • - *
      - * In both cases, the simplest way to model the incoming data is as a stream. - */ -public interface DestinationFlushFunction { - - /** - * Flush a batch of data to the destination. - * - * @param decs the Airbyte stream the data stream belongs to - * @param stream a bounded {@link AirbyteMessage} stream ideally of - * {@link #getOptimalBatchSizeBytes()} size - * @throws Exception - */ - void flush(StreamDescriptor decs, Stream stream) throws Exception; - - /** - * When invoking {@link #flush(StreamDescriptor, Stream)}, best effort attempt to invoke flush with - * a batch of this size. Useful for Destinations that have optimal flush batch sizes. - * - * @return the optimal batch size in bytes - */ - long getOptimalBatchSizeBytes(); - -} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/GlobalMemoryManager.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/GlobalMemoryManager.java deleted file mode 100644 index 726e1c7a4cc1..000000000000 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/GlobalMemoryManager.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination_async; - -import java.util.concurrent.atomic.AtomicLong; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; - -/** - * Responsible for managing global memory across multiple queues in a thread-safe way. - *

      - * This means memory allocation and de-allocation for each queue can be dynamically adjusted - * according to the overall available memory. Memory blocks are managed in chunks of - * {@link #BLOCK_SIZE_BYTES}, and the total amount of memory managed is configured at creation time. - *

      - * As a destination has no information about incoming per-stream records, having static non-global - * queue sizes can cause unnecessary backpressure on a per-stream basis. By providing a dynamic, - * global view of memory management, this class allows each queue to free and consume memory - * dynamically, enabling effective sharing of global memory resources across all the queues. and - * avoiding accidental stream backpressure. - *

      - * This becomes particularly useful in the following scenarios: - *

        - *
      • 1. When the incoming records belong to a single stream. Dynamic allocation ensures this one - * stream can utilise all memory.
      • - *
      • 2. When the incoming records are from multiple streams, such as with Change Data Capture - * (CDC). Here, dynamic allocation let us create as many queues as possible, allowing all streams to - * be processed in parallel without accidental backpressure from unnecessary eager flushing.
      • - *
      - */ -@Slf4j -public class GlobalMemoryManager { - - // In cases where a queue is rapidly expanding, a larger block size allows less allocation calls. On - // the flip size, a smaller block size allows more granular memory management. Since this overhead - // is minimal for now, err on a smaller block sizes. - public static final long BLOCK_SIZE_BYTES = 10 * 1024 * 1024; // 10MB - private final long maxMemoryBytes; - - private final AtomicLong currentMemoryBytes = new AtomicLong(0); - - public GlobalMemoryManager(final long maxMemoryBytes) { - this.maxMemoryBytes = maxMemoryBytes; - } - - public long getMaxMemoryBytes() { - return maxMemoryBytes; - } - - public long getCurrentMemoryBytes() { - return currentMemoryBytes.get(); - } - - /** - * Requests a block of memory of {@link #BLOCK_SIZE_BYTES}. Return 0 if memory cannot be freed. - * - * @return the size of the allocated block, in bytes - */ - public synchronized long requestMemory() { - // todo(davin): what happens if the incoming record is larger than 10MB? - if (currentMemoryBytes.get() >= maxMemoryBytes) { - return 0L; - } - - final var freeMem = maxMemoryBytes - currentMemoryBytes.get(); - // Never allocate more than free memory size. - final var toAllocateBytes = Math.min(freeMem, BLOCK_SIZE_BYTES); - currentMemoryBytes.addAndGet(toAllocateBytes); - - log.debug("Memory Requested: max: {}, allocated: {}, allocated in this request: {}", - FileUtils.byteCountToDisplaySize(maxMemoryBytes), - FileUtils.byteCountToDisplaySize(currentMemoryBytes.get()), - FileUtils.byteCountToDisplaySize(toAllocateBytes)); - return toAllocateBytes; - } - - /** - * Frees a block of memory of the given size. If the amount of memory freed exceeds the current - * memory allocation, a warning will be logged. - * - * @param bytes the size of the block to free, in bytes - */ - public void free(final long bytes) { - log.info("Freeing {} bytes..", bytes); - currentMemoryBytes.addAndGet(-bytes); - - if (currentMemoryBytes.get() < 0) { - log.warn("Freed more memory than allocated. This should never happen. Please report this bug."); - } - } - -} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/OnCloseFunction.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/OnCloseFunction.java deleted file mode 100644 index f3d43e594432..000000000000 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/OnCloseFunction.java +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination_async; - -import io.airbyte.commons.concurrency.VoidCallable; - -/** - * Async version of - * {@link io.airbyte.integrations.destination.buffered_stream_consumer.OnCloseFunction}. Separately - * out for easier versioning. - */ -public interface OnCloseFunction extends VoidCallable {} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/buffers/BufferManager.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/buffers/BufferManager.java deleted file mode 100644 index 3216135fdf73..000000000000 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/buffers/BufferManager.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination_async.buffers; - -import com.google.common.annotations.VisibleForTesting; -import io.airbyte.integrations.destination_async.AirbyteFileUtils; -import io.airbyte.integrations.destination_async.FlushWorkers; -import io.airbyte.integrations.destination_async.GlobalMemoryManager; -import io.airbyte.integrations.destination_async.state.GlobalAsyncStateManager; -import io.airbyte.protocol.models.v0.StreamDescriptor; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@Slf4j -public class BufferManager { - - private static final Logger LOGGER = LoggerFactory.getLogger(BufferManager.class); - - public final long maxMemory; - private final ConcurrentMap buffers; - private final BufferEnqueue bufferEnqueue; - private final BufferDequeue bufferDequeue; - private final GlobalMemoryManager memoryManager; - - private final GlobalAsyncStateManager stateManager; - private final ScheduledExecutorService debugLoop; - - public BufferManager() { - this((long) (Runtime.getRuntime().maxMemory() * 0.8)); - } - - @VisibleForTesting - public BufferManager(final long memoryLimit) { - maxMemory = memoryLimit; - LOGGER.info("Memory available to the JVM {}", FileUtils.byteCountToDisplaySize(maxMemory)); - memoryManager = new GlobalMemoryManager(maxMemory); - this.stateManager = new GlobalAsyncStateManager(memoryManager); - buffers = new ConcurrentHashMap<>(); - bufferEnqueue = new BufferEnqueue(memoryManager, buffers, stateManager); - bufferDequeue = new BufferDequeue(memoryManager, buffers, stateManager); - debugLoop = Executors.newSingleThreadScheduledExecutor(); - debugLoop.scheduleAtFixedRate(this::printQueueInfo, 0, 10, TimeUnit.SECONDS); - } - - public GlobalAsyncStateManager getStateManager() { - return stateManager; - } - - public BufferEnqueue getBufferEnqueue() { - return bufferEnqueue; - } - - public BufferDequeue getBufferDequeue() { - return bufferDequeue; - } - - /** - * Closing a queue will flush all items from it. For this reason, this method needs to be called - * after {@link FlushWorkers#close()}. This allows the upload workers to make sure all items in the - * queue has been flushed. - */ - public void close() throws Exception { - debugLoop.shutdownNow(); - log.info("Buffers cleared.."); - } - - private void printQueueInfo() { - final var queueInfo = new StringBuilder().append("QUEUE INFO").append(System.lineSeparator()); - - queueInfo - .append(String.format(" Global Mem Manager -- max: %s, allocated: %s (%s MB), %% used: %s", - AirbyteFileUtils.byteCountToDisplaySize(memoryManager.getMaxMemoryBytes()), - AirbyteFileUtils.byteCountToDisplaySize(memoryManager.getCurrentMemoryBytes()), - (double) memoryManager.getCurrentMemoryBytes() / 1024 / 1024, - (double) memoryManager.getCurrentMemoryBytes() / memoryManager.getMaxMemoryBytes())) - .append(System.lineSeparator()); - - for (final var entry : buffers.entrySet()) { - final var queue = entry.getValue(); - queueInfo.append( - String.format(" Queue name: %s, num records: %d, num bytes: %s", - entry.getKey().getName(), queue.size(), AirbyteFileUtils.byteCountToDisplaySize(queue.getCurrentMemoryUsage()))) - .append(System.lineSeparator()); - } - log.info(queueInfo.toString()); - } - -} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/state/GlobalAsyncStateManager.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/state/GlobalAsyncStateManager.java deleted file mode 100644 index 420f892a66ef..000000000000 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/destination_async/state/GlobalAsyncStateManager.java +++ /dev/null @@ -1,338 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination_async.state; - -import static java.lang.Thread.sleep; - -import com.google.common.base.Preconditions; -import io.airbyte.integrations.destination_async.GlobalMemoryManager; -import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteMessage; -import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteStreamState; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteStateMessage; -import io.airbyte.protocol.models.v0.StreamDescriptor; -import java.util.ArrayList; -import java.util.Collection; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicLong; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.apache.mina.util.ConcurrentHashSet; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Responsible for managing state within the Destination. The general approach is a ref counter - * approach - each state message is associated with a record count. This count represents the number - * of preceding records. For a state to be emitted, all preceding records have to be written to the - * destination i.e. the counter is 0. - *

      - * A per-stream state queue is maintained internally, with each state within the queue having a - * counter. This means we *ALLOW* records succeeding an unemitted state to be written. This - * decouples record writing from state management at the cost of potentially repeating work if an - * upstream state is never written. - *

      - * One important detail here is the difference between how PER-STREAM & NON-PER-STREAM is handled. - * The PER-STREAM case is simple, and is as described above. The NON-PER-STREAM case is slightly - * tricky. Because we don't know the stream type to begin with, we always assume PER_STREAM until - * the first state message arrives. If this state message is a GLOBAL state, we alias all existing - * state ids to a single global state id via a set of alias ids. From then onwards, we use one id - - * {@link #SENTINEL_GLOBAL_DESC} regardless of stream. Read - * {@link #convertToGlobalIfNeeded(AirbyteMessage)} for more detail. - */ -@Slf4j -public class GlobalAsyncStateManager { - - private static final Logger LOGGER = LoggerFactory.getLogger(GlobalAsyncStateManager.class); - - private static final StreamDescriptor SENTINEL_GLOBAL_DESC = new StreamDescriptor().withName(UUID.randomUUID().toString()); - private final GlobalMemoryManager memoryManager; - - /** - * Memory that the manager has allocated to it to use. It can ask for more memory as needed. - */ - private final AtomicLong memoryAllocated; - /** - * Memory that the manager is currently using. - */ - private final AtomicLong memoryUsed; - - boolean preState = true; - private final ConcurrentMap stateIdToCounter = new ConcurrentHashMap<>(); - private final ConcurrentMap> streamToStateIdQ = new ConcurrentHashMap<>(); - - private final ConcurrentMap> stateIdToState = new ConcurrentHashMap<>(); - // empty in the STREAM case. - - // Alias-ing only exists in the non-STREAM case where we have to convert existing state ids to one - // single global id. - // This only happens once. - private final Set aliasIds = new ConcurrentHashSet<>(); - private long retroactiveGlobalStateId = 0; - - public GlobalAsyncStateManager(final GlobalMemoryManager memoryManager) { - this.memoryManager = memoryManager; - memoryAllocated = new AtomicLong(memoryManager.requestMemory()); - memoryUsed = new AtomicLong(); - } - - // Always assume STREAM to begin, and convert only if needed. Most state is per stream anyway. - private AirbyteStateMessage.AirbyteStateType stateType = AirbyteStateMessage.AirbyteStateType.STREAM; - - /** - * Main method to process state messages. - *

      - * The first incoming state message tells us the type of state we are dealing with. We then convert - * internal data structures if needed. - *

      - * Because state messages are a watermark, all preceding records need to be flushed before the state - * message can be processed. - */ - public void trackState(final PartialAirbyteMessage message, final long sizeInBytes) { - if (preState) { - convertToGlobalIfNeeded(message); - preState = false; - } - // stateType should not change after a conversion. - Preconditions.checkArgument(stateType == extractStateType(message)); - - closeState(message, sizeInBytes); - } - - /** - * Identical to {@link #getStateId(StreamDescriptor)} except this increments the associated counter - * by 1. Intended to be called whenever a record is ingested. - * - * @param streamDescriptor - stream to get stateId for. - * @return state id - */ - public long getStateIdAndIncrementCounter(final StreamDescriptor streamDescriptor) { - return getStateIdAndIncrement(streamDescriptor, 1); - } - - /** - * Each decrement represent one written record for a state. A zero counter means there are no more - * inflight records associated with a state and the state can be flushed. - * - * @param stateId reference to a state. - * @param count to decrement. - */ - public void decrement(final long stateId, final long count) { - log.trace("decrementing state id: {}, count: {}", stateId, count); - stateIdToCounter.get(getStateAfterAlias(stateId)).addAndGet(-count); - } - - /** - * Returns state messages with no more inflight records i.e. counter = 0 across all streams. - * Intended to be called by {@link io.airbyte.integrations.destination_async.FlushWorkers} after a - * worker has finished flushing its record batch. - *

      - * The return list of states should be emitted back to the platform. - * - * @return list of state messages with no more inflight records. - */ - public List flushStates() { - final List output = new ArrayList<>(); - Long bytesFlushed = 0L; - for (final Map.Entry> entry : streamToStateIdQ.entrySet()) { - // Remove all states with 0 counters. - // Per-stream synchronized is required to make sure the state (at the head of the queue) - // logic is applied to is the state actually removed. - synchronized (this) { - final LinkedList stateIdQueue = entry.getValue(); - while (true) { - final Long oldestState = stateIdQueue.peek(); - if (oldestState == null) { - break; - } - - // technically possible this map hasn't been updated yet. - final boolean noCorrespondingStateMsg = stateIdToState.get(oldestState) == null; - if (noCorrespondingStateMsg) { - break; - } - - final boolean noPrevRecs = !stateIdToCounter.containsKey(oldestState); - final boolean allRecsEmitted = stateIdToCounter.get(oldestState).get() == 0; - if (noPrevRecs || allRecsEmitted) { - var polled = entry.getValue().poll(); // poll to remove. no need to read as the earlier peek is still valid. - output.add(stateIdToState.get(oldestState).getLeft()); - bytesFlushed += stateIdToState.get(oldestState).getRight(); - } else { - break; - } - } - } - } - - freeBytes(bytesFlushed); - return output; - } - - private Long getStateIdAndIncrement(final StreamDescriptor streamDescriptor, final long increment) { - final StreamDescriptor resolvedDescriptor = stateType == AirbyteStateMessage.AirbyteStateType.STREAM ? streamDescriptor : SENTINEL_GLOBAL_DESC; - // As concurrent collections do not guarantee data consistency when iterating, use `get` instead of - // `containsKey`. - if (streamToStateIdQ.get(resolvedDescriptor) == null) { - registerNewStreamDescriptor(resolvedDescriptor); - } - final Long stateId = streamToStateIdQ.get(resolvedDescriptor).peekLast(); - final var update = stateIdToCounter.get(stateId).addAndGet(increment); - log.trace("State id: {}, count: {}", stateId, update); - return stateId; - } - - /** - * Return the internal id of a state message. This is the id that should be used to reference a - * state when interacting with all methods in this class. - * - * @param streamDescriptor - stream to get stateId for. - * @return state id - */ - private long getStateId(final StreamDescriptor streamDescriptor) { - return getStateIdAndIncrement(streamDescriptor, 0); - } - - /** - * Pass this the number of bytes that were flushed. It will track those internally and if the - * memoryUsed gets signficantly lower than what is allocated, then it will return it to the memory - * manager. We don't always return to the memory manager to avoid needlessly allocating / - * de-allocating memory rapidly over a few bytes. - * - * @param bytesFlushed bytes that were flushed (and should be removed from memory used). - */ - private void freeBytes(final long bytesFlushed) { - LOGGER.debug("Bytes flushed memory to store state message. Allocated: {}, Used: {}, Flushed: {}, % Used: {}", - FileUtils.byteCountToDisplaySize(memoryAllocated.get()), - FileUtils.byteCountToDisplaySize(memoryUsed.get()), - FileUtils.byteCountToDisplaySize(bytesFlushed), - (double) memoryUsed.get() / memoryAllocated.get()); - - memoryManager.free(bytesFlushed); - memoryAllocated.addAndGet(-bytesFlushed); - memoryUsed.addAndGet(-bytesFlushed); - LOGGER.debug("Returned {} of memory back to the memory manager.", FileUtils.byteCountToDisplaySize(bytesFlushed)); - } - - private void convertToGlobalIfNeeded(final PartialAirbyteMessage message) { - // instead of checking for global or legacy, check for the inverse of stream. - stateType = extractStateType(message); - if (stateType != AirbyteStateMessage.AirbyteStateType.STREAM) {// alias old stream-level state ids to single global state id - // upon conversion, all previous tracking data structures need to be cleared as we move - // into the non-STREAM world for correctness. - - aliasIds.addAll(streamToStateIdQ.values().stream().flatMap(Collection::stream).toList()); - streamToStateIdQ.clear(); - retroactiveGlobalStateId = StateIdProvider.getNextId(); - - streamToStateIdQ.put(SENTINEL_GLOBAL_DESC, new LinkedList<>()); - streamToStateIdQ.get(SENTINEL_GLOBAL_DESC).add(retroactiveGlobalStateId); - - final long combinedCounter = stateIdToCounter.values() - .stream() - .mapToLong(AtomicLong::get) - .sum(); - stateIdToCounter.clear(); - stateIdToCounter.put(retroactiveGlobalStateId, new AtomicLong(combinedCounter)); - } - } - - private AirbyteStateMessage.AirbyteStateType extractStateType(final PartialAirbyteMessage message) { - if (message.getState().getType() == null) { - // Treated the same as GLOBAL. - return AirbyteStateMessage.AirbyteStateType.LEGACY; - } else { - return message.getState().getType(); - } - } - - /** - * When a state message is received, 'close' the previous state to associate the existing state id - * to the newly arrived state message. We also increment the state id in preparation for the next - * state message. - */ - private void closeState(final PartialAirbyteMessage message, final long sizeInBytes) { - final StreamDescriptor resolvedDescriptor = extractStream(message).orElse(SENTINEL_GLOBAL_DESC); - stateIdToState.put(getStateId(resolvedDescriptor), ImmutablePair.of(message, sizeInBytes)); - registerNewStateId(resolvedDescriptor); - - allocateMemoryToState(sizeInBytes); - } - - /** - * Given the size of a state message, tracks how much memory the manager is using and requests - * additional memory from the memory manager if needed. - * - * @param sizeInBytes size of the state message - */ - @SuppressWarnings("BusyWait") - private void allocateMemoryToState(final long sizeInBytes) { - if (memoryAllocated.get() < memoryUsed.get() + sizeInBytes) { - while (memoryAllocated.get() < memoryUsed.get() + sizeInBytes) { - memoryAllocated.addAndGet(memoryManager.requestMemory()); - try { - LOGGER.debug("Insufficient memory to store state message. Allocated: {}, Used: {}, Size of State Msg: {}, Needed: {}", - FileUtils.byteCountToDisplaySize(memoryAllocated.get()), - FileUtils.byteCountToDisplaySize(memoryUsed.get()), - FileUtils.byteCountToDisplaySize(sizeInBytes), - FileUtils.byteCountToDisplaySize(sizeInBytes - (memoryAllocated.get() - memoryUsed.get()))); - sleep(1000); - } catch (final InterruptedException e) { - throw new RuntimeException(e); - } - } - } - memoryUsed.addAndGet(sizeInBytes); - LOGGER.debug("State Manager memory usage: Allocated: {}, Used: {}, % Used {}", - FileUtils.byteCountToDisplaySize(memoryAllocated.get()), - FileUtils.byteCountToDisplaySize(memoryUsed.get()), - (double) memoryUsed.get() / memoryAllocated.get()); - } - - private static Optional extractStream(final PartialAirbyteMessage message) { - return Optional.ofNullable(message.getState().getStream()).map(PartialAirbyteStreamState::getStreamDescriptor); - } - - private long getStateAfterAlias(final long stateId) { - if (aliasIds.contains(stateId)) { - return retroactiveGlobalStateId; - } else { - return stateId; - } - } - - private void registerNewStreamDescriptor(final StreamDescriptor resolvedDescriptor) { - streamToStateIdQ.put(resolvedDescriptor, new LinkedList<>()); - registerNewStateId(resolvedDescriptor); - } - - private void registerNewStateId(final StreamDescriptor resolvedDescriptor) { - final long stateId = StateIdProvider.getNextId(); - streamToStateIdQ.get(resolvedDescriptor).add(stateId); - stateIdToCounter.put(stateId, new AtomicLong(0)); - } - - /** - * Simplify internal tracking by providing a global always increasing counter for state ids. - */ - private static class StateIdProvider { - - private static long pk = 0; - - public static long getNextId() { - return pk++; - } - - } - -} diff --git a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/ConnectorExceptionUtil.java b/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/ConnectorExceptionUtil.java deleted file mode 100644 index 65d6428fcdc3..000000000000 --- a/airbyte-integrations/bases/base-java/src/main/java/io/airbyte/integrations/util/ConnectorExceptionUtil.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.util; - -import com.google.common.collect.ImmutableList; -import io.airbyte.commons.exceptions.ConfigErrorException; -import io.airbyte.commons.exceptions.ConnectionErrorException; -import io.airbyte.integrations.base.errors.messages.ErrorMessage; -import java.sql.SQLException; -import java.sql.SQLSyntaxErrorException; -import java.util.List; -import java.util.Locale; -import java.util.function.Predicate; - -/** - * Utility class defining methods for handling configuration exceptions in connectors. - */ -public class ConnectorExceptionUtil { - - public static final String COMMON_EXCEPTION_MESSAGE_TEMPLATE = "Could not connect with provided configuration. Error: %s"; - static final String RECOVERY_CONNECTION_ERROR_MESSAGE = - "We're having issues syncing from a Postgres replica that is configured as a hot standby server. " + - "Please see https://docs.airbyte.com/integrations/sources/postgres/#sync-data-from-postgres-hot-standby-server for options and workarounds"; - - public static final List HTTP_AUTHENTICATION_ERROR_CODES = ImmutableList.of(401, 403); - private static final List> configErrorPredicates = - List.of(getConfigErrorPredicate(), getConnectionErrorPredicate(), - isRecoveryConnectionExceptionPredicate(), isUnknownColumnInFieldListException()); - - public static boolean isConfigError(final Throwable e) { - return configErrorPredicates.stream().anyMatch(predicate -> predicate.test(e)); - } - - public static String getDisplayMessage(final Throwable e) { - if (e instanceof ConfigErrorException) { - return ((ConfigErrorException) e).getDisplayMessage(); - } else if (e instanceof ConnectionErrorException) { - final ConnectionErrorException connEx = (ConnectionErrorException) e; - return ErrorMessage.getErrorMessage(connEx.getStateCode(), connEx.getErrorCode(), connEx.getExceptionMessage(), connEx); - } else if (isRecoveryConnectionExceptionPredicate().test(e)) { - return RECOVERY_CONNECTION_ERROR_MESSAGE; - } else if (isUnknownColumnInFieldListException().test(e)) { - return e.getMessage(); - } else { - return String.format(COMMON_EXCEPTION_MESSAGE_TEMPLATE, e.getMessage() != null ? e.getMessage() : ""); - } - } - - /** - * Returns the first instance of an exception associated with a configuration error (if it exists). - * Otherwise, the original exception is returned. - */ - public static Throwable getRootConfigError(final Exception e) { - Throwable current = e; - while (current != null) { - if (ConnectorExceptionUtil.isConfigError(current)) { - return current; - } else { - current = current.getCause(); - } - } - return e; - } - - private static Predicate getConfigErrorPredicate() { - return e -> e instanceof ConfigErrorException; - } - - private static Predicate getConnectionErrorPredicate() { - return e -> e instanceof ConnectionErrorException; - } - - private static Predicate isRecoveryConnectionExceptionPredicate() { - return e -> e instanceof SQLException && e.getMessage() - .toLowerCase(Locale.ROOT) - .contains("due to conflict with recovery"); - } - - private static Predicate isUnknownColumnInFieldListException() { - return e -> e instanceof SQLSyntaxErrorException - && e.getMessage() - .toLowerCase(Locale.ROOT) - .contains("unknown column") - && e.getMessage() - .toLowerCase(Locale.ROOT) - .contains("in 'field list'"); - } - -} diff --git a/airbyte-integrations/bases/base-java/src/main/resources/log4j2.xml b/airbyte-integrations/bases/base-java/src/main/resources/log4j2.xml deleted file mode 100644 index 81e76194de83..000000000000 --- a/airbyte-integrations/bases/base-java/src/main/resources/log4j2.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - - diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/AirbyteExceptionHandlerTest.java b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/AirbyteExceptionHandlerTest.java deleted file mode 100644 index 8729bca1f8d9..000000000000 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/base/AirbyteExceptionHandlerTest.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.base; - -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.spy; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.Jsons; -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; -import java.nio.charset.StandardCharsets; -import lombok.SneakyThrows; -import org.junit.After; -import org.junit.Before; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; -import org.slf4j.LoggerFactory; - -public class AirbyteExceptionHandlerTest { - - PrintStream originalOut = System.out; - private volatile ByteArrayOutputStream outContent = new ByteArrayOutputStream(); - - @Before - public void setUpOut() { - System.setOut(new PrintStream(outContent, true, StandardCharsets.UTF_8)); - } - - @Test - void testTraceMessageEmission() throws Exception { - // mocking terminate() method in AirbyteExceptionHandler, so we don't kill the JVM - AirbyteExceptionHandler airbyteExceptionHandler = spy(new AirbyteExceptionHandler()); - doNothing().when(airbyteExceptionHandler).terminate(); - - // have to spawn a new thread to test the uncaught exception handling, - // because junit catches any exceptions in main thread, i.e. they're not 'uncaught' - Thread thread = new Thread() { - - @SneakyThrows - public void run() { - setUpOut(); - final IntegrationRunner runner = Mockito.mock(IntegrationRunner.class); - doThrow(new RuntimeException("error")).when(runner).run(new String[] {"write"}); - runner.run(new String[] {"write"}); - } - - }; - thread.setUncaughtExceptionHandler(airbyteExceptionHandler); - thread.start(); - thread.join(); - System.out.flush(); - revertOut(); - - // now we turn the std out from the thread into json and check it's the expected TRACE message - JsonNode traceMsgJson = Jsons.deserialize(outContent.toString(StandardCharsets.UTF_8)); - LoggerFactory.getLogger(AirbyteExceptionHandlerTest.class).debug(traceMsgJson.toString()); - Assertions.assertEquals("TRACE", traceMsgJson.get("type").asText()); - Assertions.assertEquals("ERROR", traceMsgJson.get("trace").get("type").asText()); - Assertions.assertEquals(AirbyteExceptionHandler.logMessage, traceMsgJson.get("trace").get("error").get("message").asText()); - Assertions.assertEquals("system_error", traceMsgJson.get("trace").get("error").get("failure_type").asText()); - } - - @After - public void revertOut() { - System.setOut(originalOut); - } - -} diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/DetectStreamToFlushTest.java b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/DetectStreamToFlushTest.java deleted file mode 100644 index 723aec616e8c..000000000000 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/DetectStreamToFlushTest.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination_async; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import io.airbyte.integrations.destination_async.buffers.BufferDequeue; -import io.airbyte.protocol.models.v0.StreamDescriptor; -import java.time.Instant; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.atomic.AtomicBoolean; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class DetectStreamToFlushTest { - - public static final Instant NOW = Instant.now(); - public static final Instant FIVE_MIN_AGO = NOW.minusSeconds(60 * 5); - private static final long SIZE_10MB = 10 * 1024 * 1024; - private static final long SIZE_200MB = 200 * 1024 * 1024; - - private static final StreamDescriptor DESC1 = new StreamDescriptor().withName("test1"); - - private static DestinationFlushFunction flusher; - - @BeforeEach - void setup() { - flusher = mock(DestinationFlushFunction.class); - when(flusher.getOptimalBatchSizeBytes()).thenReturn(SIZE_200MB); - } - - @Test - void testGetNextSkipsEmptyStreams() { - final BufferDequeue bufferDequeue = mock(BufferDequeue.class); - when(bufferDequeue.getBufferedStreams()).thenReturn(Set.of(DESC1)); - when(bufferDequeue.getQueueSizeBytes(DESC1)).thenReturn(Optional.of(0L)); - final RunningFlushWorkers runningFlushWorkers = mock(RunningFlushWorkers.class); - final DetectStreamToFlush detect = new DetectStreamToFlush(bufferDequeue, runningFlushWorkers, new AtomicBoolean(false), flusher); - assertEquals(Optional.empty(), detect.getNextStreamToFlush(0)); - } - - @Test - void testGetNextPicksUpOnSizeTrigger() { - final BufferDequeue bufferDequeue = mock(BufferDequeue.class); - when(bufferDequeue.getBufferedStreams()).thenReturn(Set.of(DESC1)); - when(bufferDequeue.getQueueSizeBytes(DESC1)).thenReturn(Optional.of(1L)); - final RunningFlushWorkers runningFlushWorkers = mock(RunningFlushWorkers.class); - final DetectStreamToFlush detect = new DetectStreamToFlush(bufferDequeue, runningFlushWorkers, new AtomicBoolean(false), flusher); - // if above threshold, triggers - assertEquals(Optional.of(DESC1), detect.getNextStreamToFlush(0)); - // if below threshold, no trigger - assertEquals(Optional.empty(), detect.getNextStreamToFlush(1)); - } - - @Test - void testGetNextAccountsForAlreadyRunningWorkers() { - final BufferDequeue bufferDequeue = mock(BufferDequeue.class); - when(bufferDequeue.getBufferedStreams()).thenReturn(Set.of(DESC1)); - when(bufferDequeue.getQueueSizeBytes(DESC1)).thenReturn(Optional.of(1L)); - final RunningFlushWorkers runningFlushWorkers = mock(RunningFlushWorkers.class); - when(runningFlushWorkers.getSizesOfRunningWorkerBatches(any())).thenReturn(List.of(Optional.of(SIZE_10MB))); - final DetectStreamToFlush detect = new DetectStreamToFlush(bufferDequeue, runningFlushWorkers, new AtomicBoolean(false), flusher); - assertEquals(Optional.empty(), detect.getNextStreamToFlush(0)); - } - - @Test - void testGetNextPicksUpOnTimeTrigger() { - final BufferDequeue bufferDequeue = mock(BufferDequeue.class); - when(bufferDequeue.getBufferedStreams()).thenReturn(Set.of(DESC1)); - when(bufferDequeue.getQueueSizeBytes(DESC1)).thenReturn(Optional.of(1L)); - when(bufferDequeue.getTimeOfLastRecord(DESC1)) - // because we eagerly load values and later access them again - // double the mocks for correctness; two calls here equals one test case. - .thenReturn(Optional.empty()) - .thenReturn(Optional.empty()) - .thenReturn(Optional.of(NOW)) - .thenReturn(Optional.of(NOW)) - .thenReturn(Optional.of(FIVE_MIN_AGO)) - .thenReturn(Optional.of(FIVE_MIN_AGO)); - - final RunningFlushWorkers runningFlushWorkers = mock(RunningFlushWorkers.class); - when(runningFlushWorkers.getSizesOfRunningWorkerBatches(any())).thenReturn(List.of(Optional.of(SIZE_10MB))); - final DetectStreamToFlush detect = new DetectStreamToFlush(bufferDequeue, runningFlushWorkers, new AtomicBoolean(false), flusher); - assertEquals(Optional.empty(), detect.getNextStreamToFlush(0)); - assertEquals(Optional.empty(), detect.getNextStreamToFlush(0)); - assertEquals(Optional.of(DESC1), detect.getNextStreamToFlush(0)); - } - -} diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/GlobalMemoryManagerTest.java b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/GlobalMemoryManagerTest.java deleted file mode 100644 index 025dfae25eee..000000000000 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/GlobalMemoryManagerTest.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination_async; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.junit.jupiter.api.Test; - -public class GlobalMemoryManagerTest { - - private static final long BYTES_10_MB = 10 * 1024 * 1024; - private static final long BYTES_35_MB = 35 * 1024 * 1024; - private static final long BYTES_5_MB = 5 * 1024 * 1024; - - @Test - void test() { - final GlobalMemoryManager mgr = new GlobalMemoryManager(BYTES_35_MB); - - assertEquals(BYTES_10_MB, mgr.requestMemory()); - assertEquals(BYTES_10_MB, mgr.requestMemory()); - assertEquals(BYTES_10_MB, mgr.requestMemory()); - assertEquals(BYTES_5_MB, mgr.requestMemory()); - assertEquals(0, mgr.requestMemory()); - - mgr.free(BYTES_10_MB); - - assertEquals(BYTES_10_MB, mgr.requestMemory()); - } - -} diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/TimeTriggerTest.java b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/TimeTriggerTest.java deleted file mode 100644 index 3e0632d372e1..000000000000 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/TimeTriggerTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination_async; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import io.airbyte.integrations.destination_async.buffers.BufferDequeue; -import io.airbyte.protocol.models.v0.StreamDescriptor; -import java.time.Instant; -import java.util.Optional; -import java.util.Set; -import org.junit.jupiter.api.Test; - -public class TimeTriggerTest { - - public static final Instant NOW = Instant.now(); - public static final Instant FIVE_MIN_AGO = NOW.minusSeconds(60 * 5); - - private static final StreamDescriptor DESC1 = new StreamDescriptor().withName("test1"); - - @Test - void testTimeTrigger() { - final BufferDequeue bufferDequeue = mock(BufferDequeue.class); - when(bufferDequeue.getBufferedStreams()).thenReturn(Set.of(DESC1)); - when(bufferDequeue.getTimeOfLastRecord(DESC1)) - .thenReturn(Optional.empty()) - .thenReturn(Optional.of(NOW)) - .thenReturn(Optional.of(FIVE_MIN_AGO)); - - final DetectStreamToFlush detect = new DetectStreamToFlush(bufferDequeue, null, null, null); - assertEquals(false, detect.isTimeTriggered(DESC1).getLeft()); - assertEquals(false, detect.isTimeTriggered(DESC1).getLeft()); - assertEquals(true, detect.isTimeTriggered(DESC1).getLeft()); - } - -} diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/buffers/BufferDequeueTest.java b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/buffers/BufferDequeueTest.java deleted file mode 100644 index eb76f0d8f063..000000000000 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/buffers/BufferDequeueTest.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination_async.buffers; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteMessage; -import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteRecordMessage; -import io.airbyte.protocol.models.v0.AirbyteMessage.Type; -import io.airbyte.protocol.models.v0.StreamDescriptor; -import java.time.Instant; -import java.time.temporal.ChronoUnit; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; - -public class BufferDequeueTest { - - private static final int RECORD_SIZE_20_BYTES = 20; - public static final String RECORD_20_BYTES = "abc"; - private static final String STREAM_NAME = "stream1"; - private static final StreamDescriptor STREAM_DESC = new StreamDescriptor().withName(STREAM_NAME); - private static final PartialAirbyteMessage RECORD_MSG_20_BYTES = new PartialAirbyteMessage() - .withType(Type.RECORD) - .withRecord(new PartialAirbyteRecordMessage() - .withStream(STREAM_NAME)); - - @Nested - class Take { - - @Test - void testTakeShouldBestEffortRead() { - final BufferManager bufferManager = new BufferManager(); - final BufferEnqueue enqueue = bufferManager.getBufferEnqueue(); - final BufferDequeue dequeue = bufferManager.getBufferDequeue(); - - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); - - // total size of records is 80, so we expect 50 to get us 2 records (prefer to under-pull records - // than over-pull). - try (final MemoryAwareMessageBatch take = dequeue.take(STREAM_DESC, 50)) { - assertEquals(2, take.getData().size()); - // verify it only took the records from the queue that it actually returned. - assertEquals(2, dequeue.getQueueSizeInRecords(STREAM_DESC).orElseThrow()); - } catch (final Exception e) { - throw new RuntimeException(e); - } - } - - @Test - void testTakeShouldReturnAllIfPossible() { - final BufferManager bufferManager = new BufferManager(); - final BufferEnqueue enqueue = bufferManager.getBufferEnqueue(); - final BufferDequeue dequeue = bufferManager.getBufferDequeue(); - - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); - - try (final MemoryAwareMessageBatch take = dequeue.take(STREAM_DESC, 60)) { - assertEquals(3, take.getData().size()); - } catch (final Exception e) { - throw new RuntimeException(e); - } - } - - @Test - void testTakeFewerRecordsThanSizeLimitShouldNotError() { - final BufferManager bufferManager = new BufferManager(); - final BufferEnqueue enqueue = bufferManager.getBufferEnqueue(); - final BufferDequeue dequeue = bufferManager.getBufferDequeue(); - - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); - - try (final MemoryAwareMessageBatch take = dequeue.take(STREAM_DESC, Long.MAX_VALUE)) { - assertEquals(2, take.getData().size()); - } catch (final Exception e) { - throw new RuntimeException(e); - } - } - - } - - @Test - void testMetadataOperationsCorrect() { - final BufferManager bufferManager = new BufferManager(); - final BufferEnqueue enqueue = bufferManager.getBufferEnqueue(); - final BufferDequeue dequeue = bufferManager.getBufferDequeue(); - - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); - enqueue.addRecord(RECORD_MSG_20_BYTES, RECORD_SIZE_20_BYTES); - - final var secondStream = new StreamDescriptor().withName("stream_2"); - final PartialAirbyteMessage recordFromSecondStream = Jsons.clone(RECORD_MSG_20_BYTES); - recordFromSecondStream.getRecord().withStream(secondStream.getName()); - enqueue.addRecord(recordFromSecondStream, RECORD_SIZE_20_BYTES); - - assertEquals(60, dequeue.getTotalGlobalQueueSizeBytes()); - - assertEquals(2, dequeue.getQueueSizeInRecords(STREAM_DESC).get()); - assertEquals(1, dequeue.getQueueSizeInRecords(secondStream).get()); - - assertEquals(40, dequeue.getQueueSizeBytes(STREAM_DESC).get()); - assertEquals(20, dequeue.getQueueSizeBytes(secondStream).get()); - - // Buffer of 3 sec to deal with test execution variance. - final var lastThreeSec = Instant.now().minus(3, ChronoUnit.SECONDS); - assertTrue(lastThreeSec.isBefore(dequeue.getTimeOfLastRecord(STREAM_DESC).get())); - assertTrue(lastThreeSec.isBefore(dequeue.getTimeOfLastRecord(secondStream).get())); - } - - @Test - void testMetadataOperationsError() { - final BufferManager bufferManager = new BufferManager(); - final BufferDequeue dequeue = bufferManager.getBufferDequeue(); - - final var ghostStream = new StreamDescriptor().withName("ghost stream"); - - assertEquals(0, dequeue.getTotalGlobalQueueSizeBytes()); - - assertTrue(dequeue.getQueueSizeInRecords(ghostStream).isEmpty()); - - assertTrue(dequeue.getQueueSizeBytes(ghostStream).isEmpty()); - - assertTrue(dequeue.getTimeOfLastRecord(ghostStream).isEmpty()); - } - -} diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/buffers/BufferEnqueueTest.java b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/buffers/BufferEnqueueTest.java deleted file mode 100644 index bee7dc36f75d..000000000000 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/buffers/BufferEnqueueTest.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination_async.buffers; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; - -import io.airbyte.integrations.destination_async.GlobalMemoryManager; -import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteMessage; -import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteRecordMessage; -import io.airbyte.integrations.destination_async.state.GlobalAsyncStateManager; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.StreamDescriptor; -import java.util.concurrent.ConcurrentHashMap; -import org.junit.jupiter.api.Test; - -public class BufferEnqueueTest { - - private static final int RECORD_SIZE_20_BYTES = 20; - - @Test - void testAddRecordShouldAdd() { - final var twoMB = 2 * 1024 * 1024; - final var streamToBuffer = new ConcurrentHashMap(); - final var enqueue = new BufferEnqueue(new GlobalMemoryManager(twoMB), streamToBuffer, mock(GlobalAsyncStateManager.class)); - - final var streamName = "stream"; - final var stream = new StreamDescriptor().withName(streamName); - final var record = new PartialAirbyteMessage() - .withType(AirbyteMessage.Type.RECORD) - .withRecord(new PartialAirbyteRecordMessage() - .withStream(streamName)); - - enqueue.addRecord(record, RECORD_SIZE_20_BYTES); - assertEquals(1, streamToBuffer.get(stream).size()); - assertEquals(20L, streamToBuffer.get(stream).getCurrentMemoryUsage()); - - } - - @Test - public void testAddRecordShouldExpand() { - final var oneKb = 1024; - final var streamToBuffer = new ConcurrentHashMap(); - final var enqueue = - new BufferEnqueue(new GlobalMemoryManager(oneKb), streamToBuffer, mock(GlobalAsyncStateManager.class)); - - final var streamName = "stream"; - final var stream = new StreamDescriptor().withName(streamName); - final var record = new PartialAirbyteMessage() - .withType(AirbyteMessage.Type.RECORD) - .withRecord(new PartialAirbyteRecordMessage() - .withStream(streamName)); - - enqueue.addRecord(record, RECORD_SIZE_20_BYTES); - enqueue.addRecord(record, RECORD_SIZE_20_BYTES); - assertEquals(2, streamToBuffer.get(stream).size()); - assertEquals(40, streamToBuffer.get(stream).getCurrentMemoryUsage()); - - } - -} diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/buffers/MemoryBoundedLinkedBlockingQueueTest.java b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/buffers/MemoryBoundedLinkedBlockingQueueTest.java deleted file mode 100644 index 7805d49d1716..000000000000 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/buffers/MemoryBoundedLinkedBlockingQueueTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination_async.buffers; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.concurrent.TimeUnit; -import org.junit.jupiter.api.Test; - -public class MemoryBoundedLinkedBlockingQueueTest { - - @Test - void offerAndTakeShouldReturn() throws InterruptedException { - final MemoryBoundedLinkedBlockingQueue queue = new MemoryBoundedLinkedBlockingQueue<>(1024); - - queue.offer("abc", 6); - - final var item = queue.take(); - - assertEquals("abc", item.item()); - } - - @Test - void testBlocksOnFullMemory() throws InterruptedException { - final MemoryBoundedLinkedBlockingQueue queue = new MemoryBoundedLinkedBlockingQueue<>(10); - assertTrue(queue.offer("abc", 6)); - assertFalse(queue.offer("abc", 6)); - - assertNotNull(queue.poll(1, TimeUnit.NANOSECONDS)); - assertNull(queue.poll(1, TimeUnit.NANOSECONDS)); - - } - -} diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/buffers/StreamAwareQueueTest.java b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/buffers/StreamAwareQueueTest.java deleted file mode 100644 index 446b988d30f2..000000000000 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/buffers/StreamAwareQueueTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination_async.buffers; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteMessage; -import org.junit.jupiter.api.Test; - -public class StreamAwareQueueTest { - - @Test - void test() throws InterruptedException { - final StreamAwareQueue queue = new StreamAwareQueue(1024); - - assertEquals(0, queue.getCurrentMemoryUsage()); - assertNull(queue.getTimeOfLastMessage().orElse(null)); - - queue.offer(new PartialAirbyteMessage(), 6, 1); - queue.offer(new PartialAirbyteMessage(), 6, 2); - queue.offer(new PartialAirbyteMessage(), 6, 3); - - assertEquals(18, queue.getCurrentMemoryUsage()); - assertNotNull(queue.getTimeOfLastMessage().orElse(null)); - - queue.take(); - queue.take(); - queue.take(); - - assertEquals(0, queue.getCurrentMemoryUsage()); - // This should be null because the queue is empty - assertTrue(queue.getTimeOfLastMessage().isEmpty(), "Expected empty optional; got " + queue.getTimeOfLastMessage()); - } - -} diff --git a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/state/GlobalAsyncStateManagerTest.java b/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/state/GlobalAsyncStateManagerTest.java deleted file mode 100644 index 27cd6ca7aa02..000000000000 --- a/airbyte-integrations/bases/base-java/src/test/java/io/airbyte/integrations/destination_async/state/GlobalAsyncStateManagerTest.java +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination_async.state; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import io.airbyte.integrations.destination_async.GlobalMemoryManager; -import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteMessage; -import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteStateMessage; -import io.airbyte.integrations.destination_async.partial_messages.PartialAirbyteStreamState; -import io.airbyte.protocol.models.v0.AirbyteMessage.Type; -import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; -import io.airbyte.protocol.models.v0.StreamDescriptor; -import java.util.List; -import java.util.Set; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; - -class GlobalAsyncStateManagerTest { - - private static final long TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES = 100 * 1024 * 1024; // 10MB - - private static final long STATE_MSG_SIZE = 1000; - - private static final String STREAM_NAME = "id_and_name"; - private static final String STREAM_NAME2 = STREAM_NAME + 2; - private static final String STREAM_NAME3 = STREAM_NAME + 3; - private static final StreamDescriptor STREAM1_DESC = new StreamDescriptor() - .withName(STREAM_NAME); - private static final StreamDescriptor STREAM2_DESC = new StreamDescriptor() - .withName(STREAM_NAME2); - private static final StreamDescriptor STREAM3_DESC = new StreamDescriptor() - .withName(STREAM_NAME3); - - private static final PartialAirbyteMessage GLOBAL_STATE_MESSAGE1 = new PartialAirbyteMessage() - .withType(Type.STATE) - .withState(new PartialAirbyteStateMessage() - .withType(AirbyteStateType.GLOBAL)); - private static final PartialAirbyteMessage GLOBAL_STATE_MESSAGE2 = new PartialAirbyteMessage() - .withType(Type.STATE) - .withState(new PartialAirbyteStateMessage() - .withType(AirbyteStateType.GLOBAL)); - private static final PartialAirbyteMessage STREAM1_STATE_MESSAGE1 = new PartialAirbyteMessage() - .withType(Type.STATE) - .withState(new PartialAirbyteStateMessage() - .withType(AirbyteStateType.STREAM) - .withStream(new PartialAirbyteStreamState().withStreamDescriptor(STREAM1_DESC))); - private static final PartialAirbyteMessage STREAM1_STATE_MESSAGE2 = new PartialAirbyteMessage() - .withType(Type.STATE) - .withState(new PartialAirbyteStateMessage() - .withType(AirbyteStateType.STREAM) - .withStream(new PartialAirbyteStreamState().withStreamDescriptor(STREAM1_DESC))); - - @Test - void testBasic() { - final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); - - final var firstStateId = stateManager.getStateIdAndIncrementCounter(STREAM1_DESC); - final var secondStateId = stateManager.getStateIdAndIncrementCounter(STREAM1_DESC); - assertEquals(firstStateId, secondStateId); - - stateManager.decrement(firstStateId, 2); - // because no state message has been tracked, there is nothing to flush yet. - var flushed = stateManager.flushStates(); - assertEquals(0, flushed.size()); - - stateManager.trackState(STREAM1_STATE_MESSAGE1, STATE_MSG_SIZE); - flushed = stateManager.flushStates(); - assertEquals(List.of(STREAM1_STATE_MESSAGE1), flushed); - } - - @Nested - class GlobalState { - - @Test - void testEmptyQueuesGlobalState() { - final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); - - // GLOBAL - stateManager.trackState(GLOBAL_STATE_MESSAGE1, STATE_MSG_SIZE); - assertEquals(List.of(GLOBAL_STATE_MESSAGE1), stateManager.flushStates()); - - assertThrows(IllegalArgumentException.class, () -> stateManager.trackState(STREAM1_STATE_MESSAGE1, STATE_MSG_SIZE)); - } - - @Test - void testConversion() { - final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); - - final var preConvertId0 = simulateIncomingRecords(STREAM1_DESC, 10, stateManager); - final var preConvertId1 = simulateIncomingRecords(STREAM2_DESC, 10, stateManager); - final var preConvertId2 = simulateIncomingRecords(STREAM3_DESC, 10, stateManager); - assertEquals(3, Set.of(preConvertId0, preConvertId1, preConvertId2).size()); - - stateManager.trackState(GLOBAL_STATE_MESSAGE1, STATE_MSG_SIZE); - - // Since this is actually a global state, we can only flush after all streams are done. - stateManager.decrement(preConvertId0, 10); - assertEquals(List.of(), stateManager.flushStates()); - stateManager.decrement(preConvertId1, 10); - assertEquals(List.of(), stateManager.flushStates()); - stateManager.decrement(preConvertId2, 10); - assertEquals(List.of(GLOBAL_STATE_MESSAGE1), stateManager.flushStates()); - } - - @Test - void testCorrectFlushingOneStream() { - final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); - - final var preConvertId0 = simulateIncomingRecords(STREAM1_DESC, 10, stateManager); - stateManager.trackState(GLOBAL_STATE_MESSAGE1, STATE_MSG_SIZE); - stateManager.decrement(preConvertId0, 10); - assertEquals(List.of(GLOBAL_STATE_MESSAGE1), stateManager.flushStates()); - - final var afterConvertId1 = simulateIncomingRecords(STREAM1_DESC, 10, stateManager); - stateManager.trackState(GLOBAL_STATE_MESSAGE2, STATE_MSG_SIZE); - stateManager.decrement(afterConvertId1, 10); - assertEquals(List.of(GLOBAL_STATE_MESSAGE2), stateManager.flushStates()); - } - - @Test - void testCorrectFlushingManyStreams() { - final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); - - final var preConvertId0 = simulateIncomingRecords(STREAM1_DESC, 10, stateManager); - final var preConvertId1 = simulateIncomingRecords(STREAM2_DESC, 10, stateManager); - assertNotEquals(preConvertId0, preConvertId1); - stateManager.trackState(GLOBAL_STATE_MESSAGE1, STATE_MSG_SIZE); - stateManager.decrement(preConvertId0, 10); - stateManager.decrement(preConvertId1, 10); - assertEquals(List.of(GLOBAL_STATE_MESSAGE1), stateManager.flushStates()); - - final var afterConvertId0 = simulateIncomingRecords(STREAM1_DESC, 10, stateManager); - final var afterConvertId1 = simulateIncomingRecords(STREAM2_DESC, 10, stateManager); - assertEquals(afterConvertId0, afterConvertId1); - stateManager.trackState(GLOBAL_STATE_MESSAGE2, STATE_MSG_SIZE); - stateManager.decrement(afterConvertId0, 20); - assertEquals(List.of(GLOBAL_STATE_MESSAGE2), stateManager.flushStates()); - } - - } - - @Nested - class PerStreamState { - - @Test - void testEmptyQueues() { - final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); - - // GLOBAL - stateManager.trackState(STREAM1_STATE_MESSAGE1, STATE_MSG_SIZE); - assertEquals(List.of(STREAM1_STATE_MESSAGE1), stateManager.flushStates()); - - assertThrows(IllegalArgumentException.class, () -> stateManager.trackState(GLOBAL_STATE_MESSAGE1, STATE_MSG_SIZE)); - } - - @Test - void testCorrectFlushingOneStream() { - final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); - - var stateId = simulateIncomingRecords(STREAM1_DESC, 3, stateManager); - stateManager.trackState(STREAM1_STATE_MESSAGE1, STATE_MSG_SIZE); - stateManager.decrement(stateId, 3); - assertEquals(List.of(STREAM1_STATE_MESSAGE1), stateManager.flushStates()); - - stateId = simulateIncomingRecords(STREAM1_DESC, 10, stateManager); - stateManager.trackState(STREAM1_STATE_MESSAGE2, STATE_MSG_SIZE); - stateManager.decrement(stateId, 10); - assertEquals(List.of(STREAM1_STATE_MESSAGE2), stateManager.flushStates()); - - } - - @Test - void testCorrectFlushingManyStream() { - final GlobalAsyncStateManager stateManager = new GlobalAsyncStateManager(new GlobalMemoryManager(TOTAL_QUEUES_MAX_SIZE_LIMIT_BYTES)); - - final var stream1StateId = simulateIncomingRecords(STREAM1_DESC, 3, stateManager); - final var stream2StateId = simulateIncomingRecords(STREAM2_DESC, 7, stateManager); - - stateManager.trackState(STREAM1_STATE_MESSAGE1, STATE_MSG_SIZE); - stateManager.decrement(stream1StateId, 3); - assertEquals(List.of(STREAM1_STATE_MESSAGE1), stateManager.flushStates()); - - stateManager.decrement(stream2StateId, 4); - assertEquals(List.of(), stateManager.flushStates()); - stateManager.trackState(STREAM1_STATE_MESSAGE2, STATE_MSG_SIZE); - stateManager.decrement(stream2StateId, 3); - // only flush state if counter is 0. - assertEquals(List.of(STREAM1_STATE_MESSAGE2), stateManager.flushStates()); - } - - } - - private static long simulateIncomingRecords(final StreamDescriptor desc, final long count, final GlobalAsyncStateManager manager) { - var stateId = 0L; - for (int i = 0; i < count; i++) { - stateId = manager.getStateIdAndIncrementCounter(desc); - } - return stateId; - } - -} diff --git a/airbyte-integrations/bases/base-normalization/build.gradle b/airbyte-integrations/bases/base-normalization/build.gradle index 03740e0f9019..961ff66d0097 100644 --- a/airbyte-integrations/bases/base-normalization/build.gradle +++ b/airbyte-integrations/bases/base-normalization/build.gradle @@ -1,161 +1,57 @@ import org.apache.tools.ant.taskdefs.condition.Os plugins { - id 'airbyte-docker' + id 'airbyte-docker-legacy' id 'airbyte-python' } -airbytePython { - moduleDirectory 'normalization' -} - dependencies { - implementation project(':airbyte-connector-test-harnesses:acceptance-test-harness') + project(':airbyte-cdk:java:airbyte-cdk:acceptance-test-harness') } // we need to access the sshtunneling script from airbyte-workers for ssh support -task copySshScript(type: Copy, dependsOn: [project(':airbyte-connector-test-harnesses:acceptance-test-harness').processResources]) { - from "${project(':airbyte-connector-test-harnesses:acceptance-test-harness').buildDir}/resources/main" +def copySshScript = tasks.register('copySshScript', Copy) { + from "${project(':airbyte-cdk:java:airbyte-cdk:acceptance-test-harness').buildDir}/resources/main" into "${buildDir}" include "sshtunneling.sh" } +copySshScript.configure { + dependsOn project(':airbyte-cdk:java:airbyte-cdk:acceptance-test-harness').tasks.named('processResources') +} // make sure the copy task above worked (if it fails, it fails silently annoyingly) -task checkSshScriptCopy(type: Task, dependsOn: copySshScript) { +def checkSshScriptCopy = tasks.register('checkSshScriptCopy') { doFirst { assert file("${buildDir}/sshtunneling.sh").exists() : "Copy of sshtunneling.sh failed, check that it is present in airbyte-workers." } } - -airbyteDocker.dependsOn(checkSshScriptCopy) -assemble.dependsOn(checkSshScriptCopy) -test.dependsOn(checkSshScriptCopy) - -integrationTest.dependsOn(build) - - -static def getDockerfile(String customConnector) { - return "${customConnector}.Dockerfile" +checkSshScriptCopy.configure { + dependsOn copySshScript } -static def getDockerImageName(String customConnector) { - return "airbyte/normalization-${customConnector}" +def generate = tasks.register('generate') +generate.configure { + dependsOn checkSshScriptCopy } -static def getImageNameWithTag(String customConnector) { - return "${getDockerImageName(customConnector)}:dev" +tasks.named('check').configure { + dependsOn generate } - -def buildAirbyteDocker(String customConnector) { - // def baseCommand = ['docker', 'build', '.', '-f', getDockerfile(customConnector), '-t', getImageNameWithTag(customConnector)] - // As the base dbt image (https://hub.docker.com/r/fishtownanalytics/dbt/tags) we are using is only build for amd64, we need to use buildkit to force builds for your local environment - // We are lucky that all the python code dbt uses is mutli-arch compatible - - def arch = 'linux/amd64' - if (Os.isArch("aarch_64") || Os.isArch("aarch64")) { - arch = 'linux/arm64' - } - - def cmdArray = ['docker', 'buildx', 'build', '--load', '--platform', arch, '-f', getDockerfile(customConnector), '-t', getImageNameWithTag(customConnector), '.'] - // println("Building normalization container: " + cmdArray.join(" ")) - - return { - commandLine cmdArray +[ + 'bigquery', + 'mysql', + 'postgres', + 'redshift', + 'snowflake', + 'oracle', + 'mssql', + 'clickhouse', + 'tidb', + 'duckdb', +].each {destinationName -> + tasks.matching { it.name == 'integrationTestPython' }.configureEach { + dependsOn project(":airbyte-integrations:connectors:destination-$destinationName").tasks.named('assemble') } } - -task airbyteDockerMSSql(type: Exec, dependsOn: checkSshScriptCopy) { - configure buildAirbyteDocker('mssql') - dependsOn assemble -} -task airbyteDockerMySql(type: Exec, dependsOn: checkSshScriptCopy) { - configure buildAirbyteDocker('mysql') - dependsOn assemble -} -task airbyteDockerOracle(type: Exec, dependsOn: checkSshScriptCopy) { - configure buildAirbyteDocker('oracle') - dependsOn assemble -} -task airbyteDockerClickhouse(type: Exec, dependsOn: checkSshScriptCopy) { - configure buildAirbyteDocker('clickhouse') - dependsOn assemble -} -task airbyteDockerSnowflake(type: Exec, dependsOn: checkSshScriptCopy) { - configure buildAirbyteDocker('snowflake') - dependsOn assemble -} -task airbyteDockerRedshift(type: Exec, dependsOn: checkSshScriptCopy) { - configure buildAirbyteDocker('redshift') - dependsOn assemble -} -task airbyteDockerTiDB(type: Exec, dependsOn: checkSshScriptCopy) { - configure buildAirbyteDocker('tidb') - dependsOn assemble -} -task airbyteDockerDuckDB(type: Exec, dependsOn: checkSshScriptCopy) { - configure buildAirbyteDocker('duckdb') - dependsOn assemble -} - -airbyteDocker.dependsOn(airbyteDockerMSSql) -airbyteDocker.dependsOn(airbyteDockerMySql) -airbyteDocker.dependsOn(airbyteDockerOracle) -airbyteDocker.dependsOn(airbyteDockerClickhouse) -airbyteDocker.dependsOn(airbyteDockerSnowflake) -airbyteDocker.dependsOn(airbyteDockerRedshift) -airbyteDocker.dependsOn(airbyteDockerTiDB) -airbyteDocker.dependsOn(airbyteDockerDuckDB) - -task("customIntegrationTestPython", type: PythonTask, dependsOn: installTestReqs) { - module = "pytest" - command = "-s integration_tests" - - dependsOn ':airbyte-integrations:bases:base-normalization:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-bigquery:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-mysql:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-postgres:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-redshift:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-snowflake:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-oracle:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-mssql:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-clickhouse:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-tidb:airbyteDocker' - dependsOn ':airbyte-integrations:connectors:destination-duckdb:airbyteDocker' -} - -// not really sure what this task does differently from customIntegrationTestPython, but it seems to also run integration tests -// and as such it depends on the docker images. -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:bases:base-normalization:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-bigquery:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-mysql:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-postgres:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-redshift:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-snowflake:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-oracle:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-mssql:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-clickhouse:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-tidb:airbyteDocker' -project.tasks.findByName('_customIntegrationTestsCoverage').dependsOn ':airbyte-integrations:connectors:destination-duckdb:airbyteDocker' - -// DATs have some additional tests that exercise normalization code paths, -// so we want to run these in addition to the base-normalization integration tests. -// If you add more items here, make sure to also to have CI fetch their credentials. -// See git history for an example. -// TODO reenable these - they're causing flakiness in our test results, need to figure that out -// integrationTest.dependsOn(":airbyte-integrations:connectors:destination-bigquery:integrationTest") -// integrationTest.dependsOn(":airbyte-integrations:connectors:destination-postgres:integrationTest") -// integrationTest.dependsOn(":airbyte-integrations:connectors:destination-snowflake:integrationTest") - -integrationTest.dependsOn("customIntegrationTestPython") -customIntegrationTests.dependsOn("customIntegrationTestPython") - -// TODO fix and use https://github.com/airbytehq/airbyte/issues/3192 instead -task('mypyCheck', type: PythonTask) { - module = "mypy" - command = "normalization --config-file ${project.rootProject.file('pyproject.toml').absolutePath}" - - dependsOn 'blackFormat' -} -check.dependsOn mypyCheck diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/test_sparse_nested_fields.py b/airbyte-integrations/bases/base-normalization/integration_tests/test_sparse_nested_fields.py index 9d4af44d6e12..d67547c1a352 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/test_sparse_nested_fields.py +++ b/airbyte-integrations/bases/base-normalization/integration_tests/test_sparse_nested_fields.py @@ -68,7 +68,7 @@ def test_sparse_nested_fields(destination_type: DestinationType): # TODO extract these conditions? if destination_type.value not in dbt_test_utils.get_test_targets(): pytest.skip(f"Destinations {destination_type} is not in NORMALIZATION_TEST_TARGET env variable") - if (destination_type.value in (DestinationType.ORACLE.value, DestinationType.CLICKHOUSE.value)): + if destination_type.value in (DestinationType.ORACLE.value, DestinationType.CLICKHOUSE.value): pytest.skip(f"Destinations {destination_type} does not support nested streams") if destination_type.value in [DestinationType.MYSQL.value, DestinationType.ORACLE.value]: pytest.skip(f"{destination_type} does not support incremental yet") diff --git a/airbyte-integrations/bases/base-normalization/mssql.Dockerfile b/airbyte-integrations/bases/base-normalization/mssql.Dockerfile index 449c0c60982b..1ec099724203 100644 --- a/airbyte-integrations/bases/base-normalization/mssql.Dockerfile +++ b/airbyte-integrations/bases/base-normalization/mssql.Dockerfile @@ -4,7 +4,7 @@ COPY --from=airbyte/base-airbyte-protocol-python:0.1.1 /airbyte /airbyte # Install curl & gnupg dependencies USER root WORKDIR /tmp -RUN apt-get update && apt-get install -y \ +RUN apt-get update --allow-insecure-repositories && apt-get install -y \ wget \ curl \ unzip \ diff --git a/airbyte-integrations/bases/base-normalization/setup.py b/airbyte-integrations/bases/base-normalization/setup.py index e2cd6dd874bf..cf58f57434d8 100644 --- a/airbyte-integrations/bases/base-normalization/setup.py +++ b/airbyte-integrations/bases/base-normalization/setup.py @@ -23,6 +23,6 @@ ], }, extras_require={ - "tests": ["airbyte-cdk", "pytest", "mypy", "types-PyYAML"], + "tests": ["airbyte-cdk", "pyyaml", "pytest", "mypy", "types-PyYAML"], }, ) diff --git a/airbyte-integrations/bases/base-standard-source-test-file/.dockerignore b/airbyte-integrations/bases/base-standard-source-test-file/.dockerignore deleted file mode 100644 index 6145f27e93a0..000000000000 --- a/airbyte-integrations/bases/base-standard-source-test-file/.dockerignore +++ /dev/null @@ -1,4 +0,0 @@ -* -!Dockerfile -!build -!entrypoint.sh diff --git a/airbyte-integrations/bases/base-standard-source-test-file/Dockerfile b/airbyte-integrations/bases/base-standard-source-test-file/Dockerfile deleted file mode 100644 index 82faf3f5efad..000000000000 --- a/airbyte-integrations/bases/base-standard-source-test-file/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -ARG JDK_VERSION=17.0.4 -FROM amazoncorretto:${JDK_VERSION} - -ARG DOCKER_BUILD_ARCH=amd64 - -# Install Docker to launch worker images. Eventually should be replaced with Docker-java. -# See https://gitter.im/docker-java/docker-java?at=5f3eb87ba8c1780176603f4e for more information on why we are not currently using Docker-java -RUN amazon-linux-extras install -y docker -RUN yum install -y openssl jq tar && yum clean all - -ENV APPLICATION base-standard-source-test-file - -WORKDIR /app - -COPY entrypoint.sh . -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 - -ENTRYPOINT ["/app/entrypoint.sh"] - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/base-standard-source-test-file diff --git a/airbyte-integrations/bases/base-standard-source-test-file/build.gradle b/airbyte-integrations/bases/base-standard-source-test-file/build.gradle deleted file mode 100644 index f58675f3de08..000000000000 --- a/airbyte-integrations/bases/base-standard-source-test-file/build.gradle +++ /dev/null @@ -1,15 +0,0 @@ -plugins { - id 'application' - id 'airbyte-docker' -} - -dependencies { - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:standard-source-test') - - implementation 'net.sourceforge.argparse4j:argparse4j:0.8.1' -} - -application { - mainClass = 'io.airbyte.integrations.standardtest.source.fs.TestSourceMain' -} diff --git a/airbyte-integrations/bases/base-standard-source-test-file/entrypoint.sh b/airbyte-integrations/bases/base-standard-source-test-file/entrypoint.sh deleted file mode 100755 index bec87b79c0da..000000000000 --- a/airbyte-integrations/bases/base-standard-source-test-file/entrypoint.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -set -e - -/app/bin/"${APPLICATION}" "$@" diff --git a/airbyte-integrations/bases/base-typing-deduping-test/build.gradle b/airbyte-integrations/bases/base-typing-deduping-test/build.gradle deleted file mode 100644 index 5c786c2f79c0..000000000000 --- a/airbyte-integrations/bases/base-typing-deduping-test/build.gradle +++ /dev/null @@ -1,15 +0,0 @@ -plugins { - id 'java-library' -} - -dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation project(':airbyte-connector-test-harnesses:acceptance-test-harness') - implementation project(':airbyte-integrations:bases:base-typing-deduping') - implementation libs.airbyte.protocol - - implementation(enforcedPlatform('org.junit:junit-bom:5.8.2')) - implementation 'org.junit.jupiter:junit-jupiter-api' - implementation 'org.junit.jupiter:junit-jupiter-params' - implementation 'org.mockito:mockito-core:4.6.1' -} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseSqlGeneratorIntegrationTest.java b/airbyte-integrations/bases/base-typing-deduping-test/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseSqlGeneratorIntegrationTest.java deleted file mode 100644 index 8f26b1699dc0..000000000000 --- a/airbyte-integrations/bases/base-typing-deduping-test/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseSqlGeneratorIntegrationTest.java +++ /dev/null @@ -1,975 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.base.destination.typing_deduping; - -import static java.util.Collections.emptyList; -import static java.util.Collections.singletonList; -import static org.junit.jupiter.api.Assertions.assertAll; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.Streams; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.string.Strings; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.SyncMode; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.commons.lang3.tuple.Pair; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.parallel.Execution; -import org.junit.jupiter.api.parallel.ExecutionMode; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class exercises {@link SqlGenerator} implementations. All destinations should extend this - * class for their respective implementation. Subclasses are encouraged to add additional tests with - * destination-specific behavior (for example, verifying that datasets are created in the correct - * BigQuery region). - *

      - * Subclasses should implement a {@link org.junit.jupiter.api.BeforeAll} method to load any secrets - * and connect to the destination. This test expects to be able to run - * {@link #getDestinationHandler()} in a {@link org.junit.jupiter.api.BeforeEach} method. - */ -@Execution(ExecutionMode.CONCURRENT) -public abstract class BaseSqlGeneratorIntegrationTest { - - private static final Logger LOGGER = LoggerFactory.getLogger(BaseSqlGeneratorIntegrationTest.class); - /** - * This, along with {@link #FINAL_TABLE_COLUMN_NAMES_CDC}, is the list of columns that should be in - * the final table. They're useful for generating SQL queries to insert records into the final - * table. - */ - protected static final List FINAL_TABLE_COLUMN_NAMES = List.of( - "_airbyte_raw_id", - "_airbyte_extracted_at", - "_airbyte_meta", - "id1", - "id2", - "updated_at", - "struct", - "array", - "string", - "number", - "integer", - "boolean", - "timestamp_with_timezone", - "timestamp_without_timezone", - "time_with_timezone", - "time_without_timezone", - "date", - "unknown"); - protected static final List FINAL_TABLE_COLUMN_NAMES_CDC; - - static { - FINAL_TABLE_COLUMN_NAMES_CDC = Streams.concat( - FINAL_TABLE_COLUMN_NAMES.stream(), - Stream.of("_ab_cdc_deleted_at")).toList(); - } - - protected static final RecordDiffer DIFFER = new RecordDiffer( - Pair.of("id1", AirbyteProtocolType.INTEGER), - Pair.of("id2", AirbyteProtocolType.INTEGER), - Pair.of("updated_at", AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE)); - - /** - * Subclasses may use these four StreamConfigs in their tests. - */ - protected StreamConfig incrementalDedupStream; - /** - * We intentionally don't have full refresh overwrite/append streams. Those actually behave - * identically in the sqlgenerator. Overwrite mode is actually handled in - * {@link DefaultTyperDeduper}. - */ - protected StreamConfig incrementalAppendStream; - protected StreamConfig cdcIncrementalDedupStream; - /** - * This isn't particularly realistic, but it's technically possible. - */ - protected StreamConfig cdcIncrementalAppendStream; - - protected SqlGenerator generator; - protected DestinationHandler destinationHandler; - protected String namespace; - - protected StreamId streamId; - private List primaryKey; - private ColumnId cursor; - private LinkedHashMap COLUMNS; - - protected abstract SqlGenerator getSqlGenerator(); - - protected abstract DestinationHandler getDestinationHandler(); - - /** - * Do any setup work to create a namespace for this test run. For example, this might create a - * BigQuery dataset, or a Snowflake schema. - */ - protected abstract void createNamespace(String namespace) throws Exception; - - /** - * Create a raw table using the StreamId's rawTableId. - */ - protected abstract void createRawTable(StreamId streamId) throws Exception; - - /** - * Creates a raw table in the v1 format - */ - protected abstract void createV1RawTable(StreamId v1RawTable) throws Exception; - - /** - * Create a final table usingi the StreamId's finalTableId. Subclasses are recommended to hardcode - * the columns from {@link #FINAL_TABLE_COLUMN_NAMES} or {@link #FINAL_TABLE_COLUMN_NAMES_CDC}. The - * only difference between those two column lists is the inclusion of the _ab_cdc_deleted_at column, - * which is controlled by the includeCdcDeletedAt parameter. - */ - protected abstract void createFinalTable(boolean includeCdcDeletedAt, StreamId streamId, String suffix) throws Exception; - - protected abstract void insertRawTableRecords(StreamId streamId, List records) throws Exception; - - protected abstract void insertV1RawTableRecords(StreamId streamId, List records) throws Exception; - - protected abstract void insertFinalTableRecords(boolean includeCdcDeletedAt, StreamId streamId, String suffix, List records) - throws Exception; - - /** - * The two dump methods are defined identically as in {@link BaseTypingDedupingTest}, but with - * slightly different method signature. This test expects subclasses to respect the raw/finalTableId - * on the StreamId object, rather than hardcoding e.g. the airbyte_internal dataset. - *

      - * The {@code _airbyte_data} field must be deserialized into an ObjectNode, even if it's stored in - * the destination as a string. - */ - protected abstract List dumpRawTableRecords(StreamId streamId) throws Exception; - - protected abstract List dumpFinalTableRecords(StreamId streamId, String suffix) throws Exception; - - /** - * Clean up all resources in the namespace. For example, this might delete the BigQuery dataset - * created in {@link #createNamespace(String)}. - */ - protected abstract void teardownNamespace(String namespace) throws Exception; - - /** - * This test implementation is extremely destination-specific, but all destinations must implement - * it. This test should verify that creating a table using {@link #incrementalDedupStream} works as - * expected, including column types, indexing, partitioning, etc. - *

      - * Note that subclasses must also annotate their implementation with @Test. - */ - @Test - public abstract void testCreateTableIncremental() throws Exception; - - @BeforeEach - public void setup() throws Exception { - generator = getSqlGenerator(); - destinationHandler = getDestinationHandler(); - final ColumnId id1 = generator.buildColumnId("id1"); - final ColumnId id2 = generator.buildColumnId("id2"); - primaryKey = List.of(id1, id2); - cursor = generator.buildColumnId("updated_at"); - - COLUMNS = new LinkedHashMap<>(); - COLUMNS.put(id1, AirbyteProtocolType.INTEGER); - COLUMNS.put(id2, AirbyteProtocolType.INTEGER); - COLUMNS.put(cursor, AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); - COLUMNS.put(generator.buildColumnId("struct"), new Struct(new LinkedHashMap<>())); - COLUMNS.put(generator.buildColumnId("array"), new Array(AirbyteProtocolType.UNKNOWN)); - COLUMNS.put(generator.buildColumnId("string"), AirbyteProtocolType.STRING); - COLUMNS.put(generator.buildColumnId("number"), AirbyteProtocolType.NUMBER); - COLUMNS.put(generator.buildColumnId("integer"), AirbyteProtocolType.INTEGER); - COLUMNS.put(generator.buildColumnId("boolean"), AirbyteProtocolType.BOOLEAN); - COLUMNS.put(generator.buildColumnId("timestamp_with_timezone"), AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); - COLUMNS.put(generator.buildColumnId("timestamp_without_timezone"), AirbyteProtocolType.TIMESTAMP_WITHOUT_TIMEZONE); - COLUMNS.put(generator.buildColumnId("time_with_timezone"), AirbyteProtocolType.TIME_WITH_TIMEZONE); - COLUMNS.put(generator.buildColumnId("time_without_timezone"), AirbyteProtocolType.TIME_WITHOUT_TIMEZONE); - COLUMNS.put(generator.buildColumnId("date"), AirbyteProtocolType.DATE); - COLUMNS.put(generator.buildColumnId("unknown"), AirbyteProtocolType.UNKNOWN); - - final LinkedHashMap cdcColumns = new LinkedHashMap<>(COLUMNS); - cdcColumns.put(generator.buildColumnId("_ab_cdc_deleted_at"), AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); - - namespace = Strings.addRandomSuffix("sql_generator_test", "_", 5); - // This is not a typical stream ID would look like, but SqlGenerator isn't allowed to make any - // assumptions about StreamId structure. - // In practice, the final table would be testDataset.users, and the raw table would be - // airbyte_internal.testDataset_raw__stream_users. - streamId = new StreamId(namespace, "users_final", namespace, "users_raw", namespace, "users_final"); - - incrementalDedupStream = new StreamConfig( - streamId, - SyncMode.INCREMENTAL, - DestinationSyncMode.APPEND_DEDUP, - primaryKey, - Optional.of(cursor), - COLUMNS); - incrementalAppendStream = new StreamConfig( - streamId, - SyncMode.INCREMENTAL, - DestinationSyncMode.APPEND, - primaryKey, - Optional.of(cursor), - COLUMNS); - - cdcIncrementalDedupStream = new StreamConfig( - streamId, - SyncMode.INCREMENTAL, - DestinationSyncMode.APPEND_DEDUP, - primaryKey, - Optional.of(cursor), - cdcColumns); - cdcIncrementalAppendStream = new StreamConfig( - streamId, - SyncMode.INCREMENTAL, - DestinationSyncMode.APPEND, - primaryKey, - Optional.of(cursor), - cdcColumns); - - LOGGER.info("Running with namespace {}", namespace); - createNamespace(namespace); - } - - @AfterEach - public void teardown() throws Exception { - teardownNamespace(namespace); - } - - /** - * Create a table and verify that we correctly recognize it as identical to itself. - */ - @Test - public void detectNoSchemaChange() throws Exception { - final String createTable = generator.createTable(incrementalDedupStream, "", false); - destinationHandler.execute(createTable); - - final Optional existingTable = destinationHandler.findExistingTable(streamId); - if (!existingTable.isPresent()) { - fail("Destination handler could not find existing table"); - } - - assertTrue( - generator.existingSchemaMatchesStreamConfig(incrementalDedupStream, existingTable.get()), - "Unchanged schema was incorrectly detected as a schema change."); - } - - /** - * Verify that adding a new column is detected as a schema change. - */ - @Test - public void detectColumnAdded() throws Exception { - final String createTable = generator.createTable(incrementalDedupStream, "", false); - destinationHandler.execute(createTable); - - final Optional existingTable = destinationHandler.findExistingTable(streamId); - if (!existingTable.isPresent()) { - fail("Destination handler could not find existing table"); - } - - incrementalDedupStream.columns().put( - generator.buildColumnId("new_column"), - AirbyteProtocolType.STRING); - - assertFalse( - generator.existingSchemaMatchesStreamConfig(incrementalDedupStream, existingTable.get()), - "Adding a new column was not detected as a schema change."); - } - - /** - * Verify that removing a column is detected as a schema change. - */ - @Test - public void detectColumnRemoved() throws Exception { - final String createTable = generator.createTable(incrementalDedupStream, "", false); - destinationHandler.execute(createTable); - - final Optional existingTable = destinationHandler.findExistingTable(streamId); - if (!existingTable.isPresent()) { - fail("Destination handler could not find existing table"); - } - - incrementalDedupStream.columns().remove(generator.buildColumnId("string")); - - assertFalse( - generator.existingSchemaMatchesStreamConfig(incrementalDedupStream, existingTable.get()), - "Removing a column was not detected as a schema change."); - } - - /** - * Verify that changing a column's type is detected as a schema change. - */ - @Test - public void detectColumnChanged() throws Exception { - final String createTable = generator.createTable(incrementalDedupStream, "", false); - destinationHandler.execute(createTable); - - final Optional existingTable = destinationHandler.findExistingTable(streamId); - if (!existingTable.isPresent()) { - fail("Destination handler could not find existing table"); - } - - incrementalDedupStream.columns().put( - generator.buildColumnId("string"), - AirbyteProtocolType.INTEGER); - - assertFalse( - generator.existingSchemaMatchesStreamConfig(incrementalDedupStream, existingTable.get()), - "Altering a column was not detected as a schema change."); - } - - /** - * Test that T+D throws an error for an incremental-dedup sync where at least one record has a null - * primary key, and that we don't write any final records. - */ - @Test - public void incrementalDedupInvalidPrimaryKey() throws Exception { - createRawTable(streamId); - createFinalTable(false, streamId, ""); - insertRawTableRecords( - streamId, - List.of( - Jsons.deserialize( - """ - { - "_airbyte_raw_id": "10d6e27d-ae7a-41b5-baf8-c4c277ef9c11", - "_airbyte_extracted_at": "2023-01-01T00:00:00Z", - "_airbyte_data": {} - } - """), - Jsons.deserialize( - """ - { - "_airbyte_raw_id": "5ce60e70-98aa-4fe3-8159-67207352c4f0", - "_airbyte_extracted_at": "2023-01-01T00:00:00Z", - "_airbyte_data": {"id1": 1, "id2": 100} - } - """))); - - final String sql = generator.updateTable(incrementalDedupStream, ""); - assertThrows( - Exception.class, - () -> destinationHandler.execute(sql)); - DIFFER.diffFinalTableRecords( - emptyList(), - dumpFinalTableRecords(streamId, "")); - } - - /** - * Run a full T+D update for an incremental-dedup stream, writing to a final table with "_foo" - * suffix, with values for all data types. Verifies all behaviors for all types: - *

        - *
      • A valid, nonnull value
      • - *
      • No value (i.e. the column is missing from the record)
      • - *
      • A JSON null value
      • - *
      • An invalid value
      • - *
      - *

      - * In practice, incremental streams never write to a suffixed table, but SqlGenerator isn't allowed - * to make that assumption (and we might as well exercise that code path). - */ - @Test - public void allTypes() throws Exception { - createRawTable(streamId); - createFinalTable(false, streamId, "_foo"); - insertRawTableRecords( - streamId, - BaseTypingDedupingTest.readRecords("sqlgenerator/alltypes_inputrecords.jsonl")); - - final String sql = generator.updateTable(incrementalDedupStream, "_foo"); - destinationHandler.execute(sql); - - verifyRecords( - "sqlgenerator/alltypes_expectedrecords_raw.jsonl", - dumpRawTableRecords(streamId), - "sqlgenerator/alltypes_expectedrecords_final.jsonl", - dumpFinalTableRecords(streamId, "_foo")); - } - - @Test - public void timestampFormats() throws Exception { - createRawTable(streamId); - createFinalTable(false, streamId, ""); - insertRawTableRecords( - streamId, - BaseTypingDedupingTest.readRecords("sqlgenerator/timestampformats_inputrecords.jsonl")); - - final String sql = generator.updateTable(incrementalAppendStream, ""); - destinationHandler.execute(sql); - - DIFFER.diffFinalTableRecords( - BaseTypingDedupingTest.readRecords("sqlgenerator/timestampformats_expectedrecords_final.jsonl"), - dumpFinalTableRecords(streamId, "")); - } - - @Test - public void incrementalDedup() throws Exception { - createRawTable(streamId); - createFinalTable(false, streamId, ""); - insertRawTableRecords( - streamId, - BaseTypingDedupingTest.readRecords("sqlgenerator/incrementaldedup_inputrecords.jsonl")); - - final String sql = generator.updateTable(incrementalDedupStream, ""); - destinationHandler.execute(sql); - - verifyRecords( - "sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl", - dumpRawTableRecords(streamId), - "sqlgenerator/incrementaldedup_expectedrecords_final.jsonl", - dumpFinalTableRecords(streamId, "")); - } - - /** - * We shouldn't crash on a sync with null cursor. Insert two records and verify that we keep the - * record with higher extracted_at. - */ - @Test - public void incrementalDedupNoCursor() throws Exception { - createRawTable(streamId); - createFinalTable(false, streamId, ""); - insertRawTableRecords( - streamId, - List.of( - Jsons.deserialize( - """ - { - "_airbyte_raw_id": "c5bcae50-962e-4b92-b2eb-1659eae31693", - "_airbyte_extracted_at": "2023-01-01T00:00:00Z", - "_airbyte_data": { - "id1": 1, - "id2": 100, - "string": "foo" - } - } - """), - Jsons.deserialize( - """ - { - "_airbyte_raw_id": "93f1bdd8-1916-4e6c-94dc-29a5d9701179", - "_airbyte_extracted_at": "2023-01-01T01:00:00Z", - "_airbyte_data": { - "id1": 1, - "id2": 100, - "string": "bar" - } - } - """))); - final StreamConfig streamConfig = new StreamConfig( - streamId, - SyncMode.INCREMENTAL, - DestinationSyncMode.APPEND_DEDUP, - primaryKey, - Optional.empty(), - COLUMNS); - - final String sql = generator.updateTable(streamConfig, ""); - destinationHandler.execute(sql); - - final List actualRawRecords = dumpRawTableRecords(streamId); - final List actualFinalRecords = dumpFinalTableRecords(streamId, ""); - verifyRecordCounts( - 1, - actualRawRecords, - 1, - actualFinalRecords); - assertAll( - () -> assertEquals("bar", actualRawRecords.get(0).get("_airbyte_data").get("string").asText()), - () -> assertEquals("bar", actualFinalRecords.get(0).get("string").asText())); - } - - @Test - public void incrementalAppend() throws Exception { - createRawTable(streamId); - createFinalTable(false, streamId, ""); - insertRawTableRecords( - streamId, - BaseTypingDedupingTest.readRecords("sqlgenerator/incrementaldedup_inputrecords.jsonl")); - - final String sql = generator.updateTable(incrementalAppendStream, ""); - destinationHandler.execute(sql); - - verifyRecordCounts( - 3, - dumpRawTableRecords(streamId), - 3, - dumpFinalTableRecords(streamId, "")); - } - - /** - * Create a nonempty users_final_tmp table. Overwrite users_final from users_final_tmp. Verify that - * users_final now exists and contains nonzero records. - */ - @Test - public void overwriteFinalTable() throws Exception { - createFinalTable(false, streamId, "_tmp"); - final List records = singletonList(Jsons.deserialize( - """ - { - "_airbyte_raw_id": "4fa4efe2-3097-4464-bd22-11211cc3e15b", - "_airbyte_extracted_at": "2023-01-01T00:00:00Z", - "_airbyte_meta": {} - } - """)); - insertFinalTableRecords( - false, - streamId, - "_tmp", - records); - - final String sql = generator.overwriteFinalTable(streamId, "_tmp"); - destinationHandler.execute(sql); - - assertEquals(1, dumpFinalTableRecords(streamId, "").size()); - } - - @Test - public void cdcImmediateDeletion() throws Exception { - createRawTable(streamId); - createFinalTable(true, streamId, ""); - insertRawTableRecords( - streamId, - singletonList(Jsons.deserialize( - """ - { - "_airbyte_raw_id": "4fa4efe2-3097-4464-bd22-11211cc3e15b", - "_airbyte_extracted_at": "2023-01-01T00:00:00Z", - "_airbyte_data": { - "id1": 1, - "id2": 100, - "updated_at": "2023-01-01T00:00:00Z", - "_ab_cdc_deleted_at": "2023-01-01T00:01:00Z" - } - } - """))); - - final String sql = generator.updateTable(cdcIncrementalDedupStream, ""); - destinationHandler.execute(sql); - - verifyRecordCounts( - 1, - dumpRawTableRecords(streamId), - 0, - dumpFinalTableRecords(streamId, "")); - } - - /** - * Verify that running T+D twice is idempotent. Previously there was a bug where non-dedup syncs - * with an _ab_cdc_deleted_at column would duplicate "deleted" records on each run. - */ - @Test - public void cdcIdempotent() throws Exception { - createRawTable(streamId); - createFinalTable(true, streamId, ""); - insertRawTableRecords( - streamId, - singletonList(Jsons.deserialize( - """ - { - "_airbyte_raw_id": "4fa4efe2-3097-4464-bd22-11211cc3e15b", - "_airbyte_extracted_at": "2023-01-01T00:00:00Z", - "_airbyte_data": { - "id1": 1, - "id2": 100, - "updated_at": "2023-01-01T00:00:00Z", - "_ab_cdc_deleted_at": "2023-01-01T00:01:00Z" - } - } - """))); - - final String sql = generator.updateTable(cdcIncrementalAppendStream, ""); - // Execute T+D twice - destinationHandler.execute(sql); - destinationHandler.execute(sql); - - verifyRecordCounts( - 1, - dumpRawTableRecords(streamId), - 1, - dumpFinalTableRecords(streamId, "")); - } - - @Test - public void cdcComplexUpdate() throws Exception { - createRawTable(streamId); - createFinalTable(true, streamId, ""); - insertRawTableRecords( - streamId, - BaseTypingDedupingTest.readRecords("sqlgenerator/cdcupdate_inputrecords_raw.jsonl")); - insertFinalTableRecords( - true, - streamId, - "", - BaseTypingDedupingTest.readRecords("sqlgenerator/cdcupdate_inputrecords_final.jsonl")); - - final String sql = generator.updateTable(cdcIncrementalDedupStream, ""); - destinationHandler.execute(sql); - - verifyRecordCounts( - // We keep the newest raw record per PK - 7, - dumpRawTableRecords(streamId), - 5, - dumpFinalTableRecords(streamId, "")); - } - - /** - * source operations: - *

        - *
      1. insert id=1 (lsn 10000)
      2. - *
      3. delete id=1 (lsn 10001)
      4. - *
      - *

      - * But the destination writes lsn 10001 before 10000. We should still end up with no records in the - * final table. - *

      - * All records have the same emitted_at timestamp. This means that we live or die purely based on - * our ability to use _ab_cdc_lsn. - */ - @Test - public void testCdcOrdering_updateAfterDelete() throws Exception { - createRawTable(streamId); - createFinalTable(true, streamId, ""); - insertRawTableRecords( - streamId, - BaseTypingDedupingTest.readRecords("sqlgenerator/cdcordering_updateafterdelete_inputrecords.jsonl")); - - final String sql = generator.updateTable(cdcIncrementalDedupStream, ""); - destinationHandler.execute(sql); - - verifyRecordCounts( - 1, - dumpRawTableRecords(streamId), - 0, - dumpFinalTableRecords(streamId, "")); - } - - /** - * source operations: - *

        - *
      1. arbitrary history...
      2. - *
      3. delete id=1 (lsn 10001)
      4. - *
      5. reinsert id=1 (lsn 10002)
      6. - *
      - *

      - * But the destination receives LSNs 10002 before 10001. In this case, we should keep the reinserted - * record in the final table. - *

      - * All records have the same emitted_at timestamp. This means that we live or die purely based on - * our ability to use _ab_cdc_lsn. - */ - @Test - public void testCdcOrdering_insertAfterDelete() throws Exception { - createRawTable(streamId); - createFinalTable(true, streamId, ""); - insertRawTableRecords( - streamId, - BaseTypingDedupingTest.readRecords("sqlgenerator/cdcordering_insertafterdelete_inputrecords_raw.jsonl")); - insertFinalTableRecords( - true, - streamId, - "", - BaseTypingDedupingTest.readRecords("sqlgenerator/cdcordering_insertafterdelete_inputrecords_final.jsonl")); - - final String sql = generator.updateTable(cdcIncrementalDedupStream, ""); - destinationHandler.execute(sql); - - verifyRecordCounts( - 1, - dumpRawTableRecords(streamId), - 1, - dumpFinalTableRecords(streamId, "")); - } - - /** - * Create a table which includes the _ab_cdc_deleted_at column, then soft reset it using the non-cdc - * stream config. Verify that the deleted_at column gets dropped. - */ - @Test - public void softReset() throws Exception { - createRawTable(streamId); - createFinalTable(true, streamId, ""); - insertRawTableRecords( - streamId, - singletonList(Jsons.deserialize( - """ - { - "_airbyte_raw_id": "arst", - "_airbyte_extracted_at": "2023-01-01T00:00:00Z", - "_airbyte_loaded_at": "2023-01-01T00:00:00Z", - "_airbyte_data": { - "id1": 1, - "id2": 100, - "_ab_cdc_deleted_at": "2023-01-01T00:01:00Z" - } - } - """))); - insertFinalTableRecords( - true, - streamId, - "", - singletonList(Jsons.deserialize( - """ - { - "_airbyte_raw_id": "arst", - "_airbyte_extracted_at": "2023-01-01T00:00:00Z", - "_airbyte_meta": {}, - "id1": 1, - "id2": 100, - "_ab_cdc_deleted_at": "2023-01-01T00:01:00Z" - } - """))); - - final String sql = generator.softReset(incrementalAppendStream); - destinationHandler.execute(sql); - - final List actualRawRecords = dumpRawTableRecords(streamId); - final List actualFinalRecords = dumpFinalTableRecords(streamId, ""); - assertAll( - () -> assertEquals(1, actualRawRecords.size()), - () -> assertEquals(1, actualFinalRecords.size()), - () -> assertTrue( - actualFinalRecords.stream().noneMatch(record -> record.has("_ab_cdc_deleted_at")), - "_ab_cdc_deleted_at column was expected to be dropped. Actual final table had: " + actualFinalRecords)); - } - - @Test - public void weirdColumnNames() throws Exception { - createRawTable(streamId); - insertRawTableRecords( - streamId, - BaseTypingDedupingTest.readRecords("sqlgenerator/weirdcolumnnames_inputrecords_raw.jsonl")); - final StreamConfig stream = new StreamConfig( - streamId, - SyncMode.INCREMENTAL, - DestinationSyncMode.APPEND_DEDUP, - primaryKey, - Optional.of(cursor), - new LinkedHashMap<>() { - - { - put(generator.buildColumnId("id1"), AirbyteProtocolType.INTEGER); - put(generator.buildColumnId("id2"), AirbyteProtocolType.INTEGER); - put(generator.buildColumnId("updated_at"), AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); - put(generator.buildColumnId("$starts_with_dollar_sign"), AirbyteProtocolType.STRING); - put(generator.buildColumnId("includes\"doublequote"), AirbyteProtocolType.STRING); - put(generator.buildColumnId("includes'singlequote"), AirbyteProtocolType.STRING); - put(generator.buildColumnId("includes`backtick"), AirbyteProtocolType.STRING); - put(generator.buildColumnId("includes.period"), AirbyteProtocolType.STRING); - put(generator.buildColumnId("includes$$doubledollar"), AirbyteProtocolType.STRING); - put(generator.buildColumnId("endswithbackslash\\"), AirbyteProtocolType.STRING); - } - - }); - - final String createTable = generator.createTable(stream, "", false); - destinationHandler.execute(createTable); - final String updateTable = generator.updateTable(stream, ""); - destinationHandler.execute(updateTable); - - verifyRecords( - "sqlgenerator/weirdcolumnnames_expectedrecords_raw.jsonl", - dumpRawTableRecords(streamId), - "sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl", - dumpFinalTableRecords(streamId, "")); - } - - /** - * Verify that we don't crash when there are special characters in the stream namespace, name, - * primary key, or cursor. - */ - @ParameterizedTest - @ValueSource(strings = {"$", "\"", "'", "`", ".", "$$", "\\"}) - public void noCrashOnSpecialCharacters(final String specialChars) throws Exception { - final String str = namespace + "_" + specialChars; - final StreamId originalStreamId = generator.buildStreamId(str, str, "unused"); - final StreamId modifiedStreamId = new StreamId( - originalStreamId.finalNamespace(), - originalStreamId.finalName(), - // hack for testing simplicity: put the raw tables in the final namespace. This makes cleanup - // easier. - originalStreamId.finalNamespace(), - "raw_table", - null, - null); - final ColumnId columnId = generator.buildColumnId(str); - try { - createNamespace(modifiedStreamId.finalNamespace()); - createRawTable(modifiedStreamId); - insertRawTableRecords( - modifiedStreamId, - List.of(Jsons.jsonNode(Map.of( - "_airbyte_raw_id", "758989f2-b148-4dd3-8754-30d9c17d05fb", - "_airbyte_extracted_at", "2023-01-01T00:00:00Z", - "_airbyte_data", Map.of(str, "bar"))))); - final StreamConfig stream = new StreamConfig( - modifiedStreamId, - SyncMode.INCREMENTAL, - DestinationSyncMode.APPEND_DEDUP, - List.of(columnId), - Optional.of(columnId), - new LinkedHashMap<>() { - - { - put(columnId, AirbyteProtocolType.STRING); - } - - }); - - final String createTable = generator.createTable(stream, "", false); - destinationHandler.execute(createTable); - final String updateTable = generator.updateTable(stream, ""); - // Not verifying anything about the data; let's just make sure we don't crash. - destinationHandler.execute(updateTable); - } finally { - teardownNamespace(modifiedStreamId.finalNamespace()); - } - } - - /** - * A stream with no columns is weird, but we shouldn't treat it specially in any way. It should - * create a final table as usual, and populate it with the relevant metadata columns. - */ - @Test - public void noColumns() throws Exception { - createRawTable(streamId); - insertRawTableRecords( - streamId, - List.of(Jsons.deserialize( - """ - { - "_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", - "_airbyte_extracted_at": "2023-01-01T00:00:00Z", - "_airbyte_data": {} - } - """))); - final StreamConfig stream = new StreamConfig( - streamId, - SyncMode.INCREMENTAL, - DestinationSyncMode.APPEND, - emptyList(), - Optional.empty(), - new LinkedHashMap<>()); - - final String createTable = generator.createTable(stream, "", false); - destinationHandler.execute(createTable); - final String updateTable = generator.updateTable(stream, ""); - destinationHandler.execute(updateTable); - - verifyRecords( - "sqlgenerator/nocolumns_expectedrecords_raw.jsonl", - dumpRawTableRecords(streamId), - "sqlgenerator/nocolumns_expectedrecords_final.jsonl", - dumpFinalTableRecords(streamId, "")); - } - - @Test - public void testV1V2migration() throws Exception { - // This is maybe a little hacky, but it avoids having to refactor this entire class and subclasses - // for something that is going away - final StreamId v1RawTableStreamId = new StreamId(null, null, streamId.finalNamespace(), "v1_" + streamId.rawName(), null, null); - createV1RawTable(v1RawTableStreamId); - insertV1RawTableRecords(v1RawTableStreamId, BaseTypingDedupingTest.readRecords("sqlgenerator/all_types_v1_inputrecords.jsonl")); - final String migration = generator.migrateFromV1toV2(streamId, v1RawTableStreamId.rawNamespace(), v1RawTableStreamId.rawName()); - destinationHandler.execute(migration); - final List v1RawRecords = dumpV1RawTableRecords(v1RawTableStreamId); - final List v2RawRecords = dumpRawTableRecords(streamId); - migrationAssertions(v1RawRecords, v2RawRecords); - } - - protected void migrationAssertions(final List v1RawRecords, final List v2RawRecords) { - final var v2RecordMap = v2RawRecords.stream().collect(Collectors.toMap( - record -> record.get("_airbyte_raw_id").asText(), - Function.identity())); - assertAll( - () -> assertEquals(5, v1RawRecords.size()), - () -> assertEquals(5, v2RawRecords.size())); - v1RawRecords.forEach(v1Record -> { - final var v1id = v1Record.get("_airbyte_ab_id").asText(); - assertAll( - () -> assertEquals(v1id, v2RecordMap.get(v1id).get("_airbyte_raw_id").asText()), - () -> assertEquals(v1Record.get("_airbyte_emitted_at").asText(), v2RecordMap.get(v1id).get("_airbyte_extracted_at").asText()), - () -> assertNull(v2RecordMap.get(v1id).get("_airbyte_loaded_at"))); - final JsonNode originalData = v1Record.get("_airbyte_data"); - JsonNode migratedData = v2RecordMap.get(v1id).get("_airbyte_data"); - if (migratedData.isTextual()) { - migratedData = Jsons.deserializeExact(migratedData.asText()); - } - // hacky thing because we only care about the data contents. - // diffRawTableRecords makes some assumptions about the structure of the blob. - DIFFER.diffFinalTableRecords(List.of(originalData), List.of(migratedData)); - }); - } - - protected List dumpV1RawTableRecords(final StreamId streamId) throws Exception { - return dumpRawTableRecords(streamId); - } - - @Test - public void testCreateTableForce() throws Exception { - final String createTableNoForce = generator.createTable(incrementalDedupStream, "", false); - final String createTableForce = generator.createTable(incrementalDedupStream, "", true); - - destinationHandler.execute(createTableNoForce); - assertThrows(Exception.class, () -> destinationHandler.execute(createTableNoForce)); - // This should not throw an exception - destinationHandler.execute(createTableForce); - - assertTrue(destinationHandler.findExistingTable(streamId).isPresent()); - } - - private void verifyRecords(final String expectedRawRecordsFile, - final List actualRawRecords, - final String expectedFinalRecordsFile, - final List actualFinalRecords) { - assertAll( - () -> DIFFER.diffRawTableRecords( - BaseTypingDedupingTest.readRecords(expectedRawRecordsFile), - actualRawRecords), - () -> assertEquals( - 0, - actualRawRecords.stream() - .filter(record -> !record.hasNonNull("_airbyte_loaded_at")) - .count()), - () -> DIFFER.diffFinalTableRecords( - BaseTypingDedupingTest.readRecords(expectedFinalRecordsFile), - actualFinalRecords)); - } - - private void verifyRecordCounts(final int expectedRawRecords, - final List actualRawRecords, - final int expectedFinalRecords, - final List actualFinalRecords) { - assertAll( - () -> assertEquals( - expectedRawRecords, - actualRawRecords.size(), - "Raw record count was incorrect"), - () -> assertEquals( - 0, - actualRawRecords.stream() - .filter(record -> !record.hasNonNull("_airbyte_loaded_at")) - .count()), - () -> assertEquals( - expectedFinalRecords, - actualFinalRecords.size(), - "Final record count was incorrect")); - } - -} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseTypingDedupingTest.java b/airbyte-integrations/bases/base-typing-deduping-test/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseTypingDedupingTest.java deleted file mode 100644 index 48ca40f32f88..000000000000 --- a/airbyte-integrations/bases/base-typing-deduping-test/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/BaseTypingDedupingTest.java +++ /dev/null @@ -1,670 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.base.destination.typing_deduping; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.lang.Exceptions; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.configoss.WorkerDestinationConfig; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteStream; -import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.SyncMode; -import io.airbyte.workers.internal.AirbyteDestination; -import io.airbyte.workers.internal.DefaultAirbyteDestination; -import io.airbyte.workers.process.AirbyteIntegrationLauncher; -import io.airbyte.workers.process.DockerProcessFactory; -import io.airbyte.workers.process.ProcessFactory; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.UUID; -import java.util.stream.Stream; -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.parallel.Execution; -import org.junit.jupiter.api.parallel.ExecutionMode; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This is loosely based on standard-destination-tests's DestinationAcceptanceTest class. The - * sync-running code is copy-pasted from there. - *

      - * All tests use a single stream, whose schema is defined in {@code resources/schema.json}. Each - * test case constructs a ConfiguredAirbyteCatalog dynamically. - *

      - * For sync modes which use a primary key, the stream provides a composite key of (id1, id2). For - * sync modes which use a cursor, the stream provides an updated_at field. The stream also has an - * _ab_cdc_deleted_at field. - */ -// If you're running from inside intellij, you must run your specific subclass to get concurrent -// execution. -@Execution(ExecutionMode.CONCURRENT) -public abstract class BaseTypingDedupingTest { - - private static final Logger LOGGER = LoggerFactory.getLogger(BaseTypingDedupingTest.class); - protected static final JsonNode SCHEMA; - static { - try { - SCHEMA = Jsons.deserialize(MoreResources.readResource("dat/schema.json")); - } catch (final IOException e) { - throw new RuntimeException(e); - } - } - private static final RecordDiffer DIFFER = new RecordDiffer( - Pair.of("id1", AirbyteProtocolType.INTEGER), - Pair.of("id2", AirbyteProtocolType.INTEGER), - Pair.of("updated_at", AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE), - Pair.of("old_cursor", AirbyteProtocolType.INTEGER)); - - private String randomSuffix; - private JsonNode config; - protected String streamNamespace; - protected String streamName; - private List streamsToTearDown; - - /** - * @return the docker image to run, e.g. {@code "airbyte/destination-bigquery:dev"}. - */ - protected abstract String getImageName(); - - /** - * Get the destination connector config. Subclasses may use this method for other setup work, e.g. - * opening a connection to the destination. - *

      - * Subclasses should _not_ start testcontainers in this method; that belongs in a BeforeAll method. - * The tests in this class are intended to be run concurrently on a shared database and will not - * interfere with each other. - *

      - * Sublcasses which need access to the config may use {@link #getConfig()}. - */ - protected abstract JsonNode generateConfig() throws Exception; - - /** - * For a given stream, return the records that exist in the destination's raw table. Each record - * must be in the format {"_airbyte_raw_id": "...", "_airbyte_extracted_at": "...", - * "_airbyte_loaded_at": "...", "_airbyte_data": {fields...}}. - *

      - * The {@code _airbyte_data} column must be an - * {@link com.fasterxml.jackson.databind.node.ObjectNode} (i.e. it cannot be a string value). - *

      - * streamNamespace may be null, in which case you should query from the default namespace. - */ - protected abstract List dumpRawTableRecords(String streamNamespace, String streamName) throws Exception; - - /** - * For a given stream, return the records that exist in the destination's final table. Each record - * must be in the format {"_airbyte_raw_id": "...", "_airbyte_extracted_at": "...", "_airbyte_meta": - * {...}, "field1": ..., "field2": ..., ...}. - *

      - * For JSON-valued columns, there is some nuance: a SQL null should be represented as a missing - * entry, whereas a JSON null should be represented as a - * {@link com.fasterxml.jackson.databind.node.NullNode}. For example, in the JSON blob {"name": - * null}, the `name` field is a JSON null, and the `address` field is a SQL null. - *

      - * The corresponding SQL looks like - * {@code INSERT INTO ... (name, address) VALUES ('null' :: jsonb, NULL)}. - *

      - * streamNamespace may be null, in which case you should query from the default namespace. - */ - protected abstract List dumpFinalTableRecords(String streamNamespace, String streamName) throws Exception; - - /** - * Delete any resources in the destination associated with this stream AND its namespace. We need - * this because we write raw tables to a shared {@code airbyte} namespace, which we can't drop - * wholesale. Must handle the case where the table/namespace doesn't exist (e.g. if the connector - * crashed without writing any data). - *

      - * In general, this should resemble - * {@code DROP TABLE IF EXISTS airbyte._; DROP SCHEMA IF EXISTS }. - */ - protected abstract void teardownStreamAndNamespace(String streamNamespace, String streamName) throws Exception; - - /** - * Destinations which need to clean up resources after an entire test finishes should override this - * method. For example, if you want to gracefully close a database connection, you should do that - * here. - */ - protected void globalTeardown() throws Exception {} - - /** - * @return A suffix which is different for each concurrent test, but stable within a single test. - */ - protected synchronized String getUniqueSuffix() { - if (randomSuffix == null) { - randomSuffix = "_" + RandomStringUtils.randomAlphabetic(5).toLowerCase(); - } - return randomSuffix; - } - - protected JsonNode getConfig() { - return config; - } - - @BeforeEach - public void setup() throws Exception { - config = generateConfig(); - streamNamespace = "typing_deduping_test" + getUniqueSuffix(); - streamName = "test_stream" + getUniqueSuffix(); - streamsToTearDown = new ArrayList<>(); - LOGGER.info("Using stream namespace {} and name {}", streamNamespace, streamName); - } - - @AfterEach - public void teardown() throws Exception { - for (final AirbyteStreamNameNamespacePair streamId : streamsToTearDown) { - teardownStreamAndNamespace(streamId.getNamespace(), streamId.getName()); - } - globalTeardown(); - } - - /** - * Starting with an empty destination, execute a full refresh overwrite sync. Verify that the - * records are written to the destination table. Then run a second sync, and verify that the records - * are overwritten. - */ - @Test - public void fullRefreshOverwrite() throws Exception { - final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.FULL_REFRESH) - .withDestinationSyncMode(DestinationSyncMode.OVERWRITE) - .withStream(new AirbyteStream() - .withNamespace(streamNamespace) - .withName(streamName) - .withJsonSchema(SCHEMA)))); - - // First sync - final List messages1 = readMessages("dat/sync1_messages.jsonl"); - - runSync(catalog, messages1); - - final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_raw.jsonl"); - final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_final.jsonl"); - verifySyncResult(expectedRawRecords1, expectedFinalRecords1); - - // Second sync - final List messages2 = readMessages("dat/sync2_messages.jsonl"); - - runSync(catalog, messages2); - - final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl"); - final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl"); - verifySyncResult(expectedRawRecords2, expectedFinalRecords2); - } - - /** - * Starting with an empty destination, execute a full refresh append sync. Verify that the records - * are written to the destination table. Then run a second sync, and verify that the old and new - * records are all present. - */ - @Test - public void fullRefreshAppend() throws Exception { - final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.FULL_REFRESH) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withStream(new AirbyteStream() - .withNamespace(streamNamespace) - .withName(streamName) - .withJsonSchema(SCHEMA)))); - - // First sync - final List messages1 = readMessages("dat/sync1_messages.jsonl"); - - runSync(catalog, messages1); - - final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_raw.jsonl"); - final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_final.jsonl"); - verifySyncResult(expectedRawRecords1, expectedFinalRecords1); - - // Second sync - final List messages2 = readMessages("dat/sync2_messages.jsonl"); - - runSync(catalog, messages2); - - final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_append_raw.jsonl"); - final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_append_final.jsonl"); - verifySyncResult(expectedRawRecords2, expectedFinalRecords2); - } - - /** - * Starting with an empty destination, execute an incremental append sync. - *

      - * This is (not so secretly) identical to {@link #fullRefreshAppend()}, and uses the same set of - * expected records. Incremental as a concept only exists in the source. From the destination's - * perspective, we only care about the destination sync mode. - */ - @Test - public void incrementalAppend() throws Exception { - final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( - new ConfiguredAirbyteStream() - // These two lines are literally the only difference between this test and fullRefreshAppend - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(List.of("updated_at")) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withStream(new AirbyteStream() - .withNamespace(streamNamespace) - .withName(streamName) - .withJsonSchema(SCHEMA)))); - - // First sync - final List messages1 = readMessages("dat/sync1_messages.jsonl"); - - runSync(catalog, messages1); - - final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_raw.jsonl"); - final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_final.jsonl"); - verifySyncResult(expectedRawRecords1, expectedFinalRecords1); - - // Second sync - final List messages2 = readMessages("dat/sync2_messages.jsonl"); - - runSync(catalog, messages2); - - final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_append_raw.jsonl"); - final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_append_final.jsonl"); - verifySyncResult(expectedRawRecords2, expectedFinalRecords2); - } - - /** - * Starting with an empty destination, execute an incremental dedup sync. Verify that the records - * are written to the destination table. Then run a second sync, and verify that the raw/final - * tables contain the correct records. - */ - @Test - public void incrementalDedup() throws Exception { - final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(List.of("updated_at")) - .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP) - .withPrimaryKey(List.of(List.of("id1"), List.of("id2"))) - .withStream(new AirbyteStream() - .withNamespace(streamNamespace) - .withName(streamName) - .withJsonSchema(SCHEMA)))); - - // First sync - final List messages1 = readMessages("dat/sync1_messages.jsonl"); - - runSync(catalog, messages1); - - final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_dedup_raw.jsonl"); - final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_dedup_final.jsonl"); - verifySyncResult(expectedRawRecords1, expectedFinalRecords1); - - // Second sync - final List messages2 = readMessages("dat/sync2_messages.jsonl"); - - runSync(catalog, messages2); - - final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_incremental_dedup_raw.jsonl"); - final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_incremental_dedup_final.jsonl"); - verifySyncResult(expectedRawRecords2, expectedFinalRecords2); - } - - /** - * Identical to {@link #incrementalDedup()}, except that the stream has no namespace. - */ - @Test - public void incrementalDedupDefaultNamespace() throws Exception { - final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(List.of("updated_at")) - .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP) - .withPrimaryKey(List.of(List.of("id1"), List.of("id2"))) - .withStream(new AirbyteStream() - // NB: we don't call `withNamespace` here - .withName(streamName) - .withJsonSchema(SCHEMA)))); - - // First sync - final List messages1 = readMessages("dat/sync1_messages.jsonl", null, streamName); - - runSync(catalog, messages1); - - final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_dedup_raw.jsonl"); - final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_dedup_final.jsonl"); - verifySyncResult(expectedRawRecords1, expectedFinalRecords1, null, streamName); - - // Second sync - final List messages2 = readMessages("dat/sync2_messages.jsonl", null, streamName); - - runSync(catalog, messages2); - - final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_incremental_dedup_raw.jsonl"); - final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_incremental_dedup_final.jsonl"); - verifySyncResult(expectedRawRecords2, expectedFinalRecords2, null, streamName); - } - - @Test - @Disabled("Not yet implemented") - public void testLineBreakCharacters() throws Exception { - // TODO verify that we can handle strings with interesting characters - // build an airbyterecordmessage using something like this, and add it to the input messages: - Jsons.jsonNode(ImmutableMap.builder() - .put("id", 1) - .put("currency", "USD\u2028") - .put("date", "2020-03-\n31T00:00:00Z\r") - // TODO(sherifnada) hack: write decimals with sigfigs because Snowflake stores 10.1 as "10" which - // fails destination tests - .put("HKD", 10.1) - .put("NZD", 700.1) - .build()); - } - - /** - * Run a sync, then remove the {@code name} column from the schema and run a second sync. Verify - * that the final table doesn't contain the `name` column after the second sync. - */ - @Test - public void testIncrementalSyncDropOneColumn() throws Exception { - final AirbyteStream stream = new AirbyteStream() - .withNamespace(streamNamespace) - .withName(streamName) - .withJsonSchema(SCHEMA); - final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(List.of("updated_at")) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withStream(stream))); - - // First sync - final List messages1 = readMessages("dat/sync1_messages.jsonl"); - - runSync(catalog, messages1); - - final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_raw.jsonl"); - final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_final.jsonl"); - verifySyncResult(expectedRawRecords1, expectedFinalRecords1); - - // Second sync - final List messages2 = readMessages("dat/sync2_messages.jsonl"); - final JsonNode trimmedSchema = SCHEMA.deepCopy(); - ((ObjectNode) trimmedSchema.get("properties")).remove("name"); - stream.setJsonSchema(trimmedSchema); - - runSync(catalog, messages2); - - // The raw data is unaffected by the schema, but the final table should not have a `name` column. - final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_append_raw.jsonl"); - final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_append_final.jsonl").stream() - .peek(record -> ((ObjectNode) record).remove("name")) - .toList(); - verifySyncResult(expectedRawRecords2, expectedFinalRecords2); - } - - @Test - @Disabled("Not yet implemented") - public void testSyncUsesAirbyteStreamNamespaceIfNotNull() throws Exception { - // TODO duplicate this test for each sync mode. Run 1st+2nd syncs using a stream with null - // namespace: - final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.FULL_REFRESH) - .withCursorField(List.of("updated_at")) - .withDestinationSyncMode(DestinationSyncMode.OVERWRITE) - .withPrimaryKey(List.of(List.of("id1"), List.of("id2"))) - .withStream(new AirbyteStream() - .withNamespace(null) - .withName(streamName) - .withJsonSchema(SCHEMA)))); - } - - // TODO duplicate this test for each sync mode. Run 1st+2nd syncs using two streams with the same - // name but different namespace - // TODO maybe we don't even need the single-stream versions... - /** - * Identical to {@link #incrementalDedup()}, except there are two streams with the same name and - * different namespace. - */ - @Test - public void incrementalDedupIdenticalName() throws Exception { - final String namespace1 = streamNamespace + "_1"; - final String namespace2 = streamNamespace + "_2"; - final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(List.of("updated_at")) - .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP) - .withPrimaryKey(List.of(List.of("id1"), List.of("id2"))) - .withStream(new AirbyteStream() - .withNamespace(namespace1) - .withName(streamName) - .withJsonSchema(SCHEMA)), - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(List.of("updated_at")) - .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP) - .withPrimaryKey(List.of(List.of("id1"), List.of("id2"))) - .withStream(new AirbyteStream() - .withNamespace(namespace2) - .withName(streamName) - .withJsonSchema(SCHEMA)))); - - // First sync - // Read the same set of messages for both streams - final List messages1 = Stream.concat( - readMessages("dat/sync1_messages.jsonl", namespace1, streamName).stream(), - readMessages("dat/sync1_messages.jsonl", namespace2, streamName).stream()).toList(); - - runSync(catalog, messages1); - - final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_dedup_raw.jsonl"); - final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_dedup_final.jsonl"); - verifySyncResult(expectedRawRecords1, expectedFinalRecords1, namespace1, streamName); - verifySyncResult(expectedRawRecords1, expectedFinalRecords1, namespace2, streamName); - - // Second sync - final List messages2 = Stream.concat( - readMessages("dat/sync2_messages.jsonl", namespace1, streamName).stream(), - readMessages("dat/sync2_messages.jsonl", namespace2, streamName).stream()).toList(); - - runSync(catalog, messages2); - - final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_incremental_dedup_raw.jsonl"); - final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_incremental_dedup_final.jsonl"); - verifySyncResult(expectedRawRecords2, expectedFinalRecords2, namespace1, streamName); - verifySyncResult(expectedRawRecords2, expectedFinalRecords2, namespace2, streamName); - } - - @Test - @Disabled("Not yet implemented") - public void testSyncNotFailsWithNewFields() throws Exception { - // TODO duplicate this test for each sync mode. Run a sync, then add a new field to the schema, then - // run another sync - // We might want to write a test that verifies more general schema evolution (e.g. all valid - // evolutions) - } - - /** - * Change the cursor column in the second sync to a column that doesn't exist in the first sync. - * Verify that we overwrite everything correctly. - *

      - * This essentially verifies that the destination connector correctly recognizes NULL cursors as - * older than non-NULL cursors. - */ - @Test - public void incrementalDedupChangeCursor() throws Exception { - final JsonNode mangledSchema = SCHEMA.deepCopy(); - ((ObjectNode) mangledSchema.get("properties")).remove("updated_at"); - ((ObjectNode) mangledSchema.get("properties")).set( - "old_cursor", - Jsons.deserialize( - """ - {"type": "integer"} - """)); - final ConfiguredAirbyteStream configuredStream = new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(List.of("old_cursor")) - .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP) - .withPrimaryKey(List.of(List.of("id1"), List.of("id2"))) - .withStream(new AirbyteStream() - .withNamespace(streamNamespace) - .withName(streamName) - .withJsonSchema(mangledSchema)); - final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of(configuredStream)); - - // First sync - final List messages1 = readMessages("dat/sync1_cursorchange_messages.jsonl"); - - runSync(catalog, messages1); - - final List expectedRawRecords1 = readRecords("dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl"); - final List expectedFinalRecords1 = readRecords("dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl"); - verifySyncResult(expectedRawRecords1, expectedFinalRecords1); - - // Second sync - final List messages2 = readMessages("dat/sync2_messages.jsonl"); - configuredStream.getStream().setJsonSchema(SCHEMA); - configuredStream.setCursorField(List.of("updated_at")); - - runSync(catalog, messages2); - - final List expectedRawRecords2 = readRecords("dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl"); - final List expectedFinalRecords2 = readRecords("dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl"); - verifySyncResult(expectedRawRecords2, expectedFinalRecords2); - } - - @Test - @Disabled("Not yet implemented") - public void testSyncWithLargeRecordBatch() throws Exception { - // TODO duplicate this test for each sync mode. Run a single sync with many records - /* - * copied from DATs: This serves to test MSSQL 2100 limit parameters in a single query. this means - * that for Airbyte insert data need to limit to ~ 700 records (3 columns for the raw tables) = 2100 - * params - * - * this maybe needs configuration per destination to specify that limit? - */ - } - - @Test - @Disabled("Not yet implemented") - public void testDataTypes() throws Exception { - // TODO duplicate this test for each sync mode. See DataTypeTestArgumentProvider for what this test - // does in DAT-land - // we probably don't want to do the exact same thing, but the general spirit of testing a wide range - // of values for every data type is approximately correct - // this test probably needs some configuration per destination to specify what values are supported? - } - - protected void verifySyncResult(final List expectedRawRecords, final List expectedFinalRecords) throws Exception { - verifySyncResult(expectedRawRecords, expectedFinalRecords, streamNamespace, streamName); - } - - private void verifySyncResult(final List expectedRawRecords, - final List expectedFinalRecords, - final String streamNamespace, - final String streamName) - throws Exception { - final List actualRawRecords = dumpRawTableRecords(streamNamespace, streamName); - final List actualFinalRecords = dumpFinalTableRecords(streamNamespace, streamName); - DIFFER.verifySyncResult(expectedRawRecords, actualRawRecords, expectedFinalRecords, actualFinalRecords); - } - - public static List readRecords(final String filename) throws IOException { - return MoreResources.readResource(filename).lines() - .map(String::trim) - .filter(line -> !line.isEmpty()) - .filter(line -> !line.startsWith("//")) - .map(Jsons::deserializeExact) - .toList(); - } - - protected List readMessages(final String filename) throws IOException { - return readMessages(filename, streamNamespace, streamName); - } - - private static List readMessages(final String filename, final String streamNamespace, final String streamName) throws IOException { - return readRecords(filename).stream() - .map(record -> Jsons.convertValue(record, AirbyteMessage.class)) - .peek(message -> { - message.getRecord().setNamespace(streamNamespace); - message.getRecord().setStream(streamName); - }).toList(); - } - - /* - * !!!!!! WARNING !!!!!! The code below was mostly copypasted from DestinationAcceptanceTest. If you - * make edits here, you probably want to also edit there. - */ - - // These contain some state, so they are instanced per test (i.e. cannot be static) - private Path jobRoot; - private ProcessFactory processFactory; - - @BeforeEach - public void setupProcessFactory() throws IOException { - final Path testDir = Path.of("/tmp/airbyte_tests/"); - Files.createDirectories(testDir); - final Path workspaceRoot = Files.createTempDirectory(testDir, "test"); - jobRoot = Files.createDirectories(Path.of(workspaceRoot.toString(), "job")); - final Path localRoot = Files.createTempDirectory(testDir, "output"); - processFactory = new DockerProcessFactory( - workspaceRoot, - workspaceRoot.toString(), - localRoot.toString(), - "host", - Collections.emptyMap()); - } - - protected void runSync(final ConfiguredAirbyteCatalog catalog, final List messages) throws Exception { - runSync(catalog, messages, getImageName()); - } - - protected void runSync(final ConfiguredAirbyteCatalog catalog, final List messages, final String imageName) throws Exception { - catalog.getStreams().forEach(s -> streamsToTearDown.add(AirbyteStreamNameNamespacePair.fromAirbyteStream(s.getStream()))); - - final WorkerDestinationConfig destinationConfig = new WorkerDestinationConfig() - .withConnectionId(UUID.randomUUID()) - .withCatalog(convertProtocolObject(catalog, io.airbyte.protocol.models.ConfiguredAirbyteCatalog.class)) - .withDestinationConnectionConfiguration(config); - - final AirbyteDestination destination = new DefaultAirbyteDestination(new AirbyteIntegrationLauncher( - "0", - 0, - imageName, - processFactory, - null, - null, - false, - new EnvVariableFeatureFlags())); - - destination.start(destinationConfig, jobRoot, Collections.emptyMap()); - messages.forEach( - message -> Exceptions.toRuntime(() -> destination.accept(convertProtocolObject(message, io.airbyte.protocol.models.AirbyteMessage.class)))); - destination.notifyEndOfInput(); - - while (!destination.isFinished()) { - destination.attemptRead(); - } - - destination.close(); - } - - private static V0 convertProtocolObject(final V1 v1, final Class klass) { - return Jsons.object(Jsons.jsonNode(v1), klass); - } - -} diff --git a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/alltypes_inputrecords.jsonl b/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/alltypes_inputrecords.jsonl deleted file mode 100644 index ca82be9ffdc4..000000000000 --- a/airbyte-integrations/bases/base-typing-deduping-test/src/main/resources/sqlgenerator/alltypes_inputrecords.jsonl +++ /dev/null @@ -1,6 +0,0 @@ -{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}}' -{"_airbyte_raw_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": null, "struct": null, "string": null, "number": null, "integer": null, "boolean": null, "timestamp_with_timezone": null, "timestamp_without_timezone": null, "time_with_timezone": null, "time_without_timezone": null, "date": null, "unknown": null}} -{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z"}} -// Note that array and struct have invalid values ({} and [] respectively). -{"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": {}, "number": {}, "integer": {}, "boolean": {}, "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": {}, "unknown": null}} -{"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118}} diff --git a/airbyte-integrations/bases/base-typing-deduping/build.gradle b/airbyte-integrations/bases/base-typing-deduping/build.gradle deleted file mode 100644 index 296403745343..000000000000 --- a/airbyte-integrations/bases/base-typing-deduping/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'java-library' -} - -dependencies { - implementation libs.airbyte.protocol - implementation project(path: ':airbyte-integrations:bases:base-java') -} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Array.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Array.java deleted file mode 100644 index dccd687e033e..000000000000 --- a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Array.java +++ /dev/null @@ -1,9 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.base.destination.typing_deduping; - -public record Array(AirbyteType items) implements AirbyteType { - -} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParser.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParser.java deleted file mode 100644 index 03ec4e06a8f0..000000000000 --- a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParser.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.base.destination.typing_deduping; - -import static io.airbyte.integrations.base.JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE; - -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map.Entry; -import java.util.Optional; -import org.apache.commons.codec.digest.DigestUtils; - -public class CatalogParser { - - private final SqlGenerator sqlGenerator; - private final String rawNamespace; - - public CatalogParser(final SqlGenerator sqlGenerator) { - this(sqlGenerator, DEFAULT_AIRBYTE_INTERNAL_NAMESPACE); - } - - public CatalogParser(final SqlGenerator sqlGenerator, final String rawNamespace) { - this.sqlGenerator = sqlGenerator; - this.rawNamespace = rawNamespace; - } - - public ParsedCatalog parseCatalog(final ConfiguredAirbyteCatalog catalog) { - // this code is bad and I feel bad - // it's mostly a port of the old normalization logic to prevent tablename collisions. - // tbh I have no idea if it works correctly. - final List streamConfigs = new ArrayList<>(); - for (final ConfiguredAirbyteStream stream : catalog.getStreams()) { - final StreamConfig originalStreamConfig = toStreamConfig(stream); - // Use empty string quote because we don't really care - if (streamConfigs.stream().anyMatch(s -> s.id().finalTableId("").equals(originalStreamConfig.id().finalTableId(""))) - || streamConfigs.stream().anyMatch(s -> s.id().rawTableId("").equals(originalStreamConfig.id().rawTableId("")))) { - final String originalNamespace = stream.getStream().getNamespace(); - final String originalName = stream.getStream().getName(); - // ... this logic is ported from legacy normalization, and maybe should change? - // We're taking a hash of the quoted namespace and the unquoted stream name - final String hash = DigestUtils.sha1Hex(originalStreamConfig.id().finalNamespace() + "&airbyte&" + originalName).substring(0, 3); - final String newName = originalName + "_" + hash; - streamConfigs.add(new StreamConfig( - sqlGenerator.buildStreamId(originalNamespace, newName, rawNamespace), - originalStreamConfig.syncMode(), - originalStreamConfig.destinationSyncMode(), - originalStreamConfig.primaryKey(), - originalStreamConfig.cursor(), - originalStreamConfig.columns())); - } else { - streamConfigs.add(originalStreamConfig); - } - } - return new ParsedCatalog(streamConfigs); - } - - private StreamConfig toStreamConfig(final ConfiguredAirbyteStream stream) { - final AirbyteType schema = AirbyteType.fromJsonSchema(stream.getStream().getJsonSchema()); - final LinkedHashMap airbyteColumns; - if (schema instanceof final Struct o) { - airbyteColumns = o.properties(); - } else if (schema instanceof final Union u) { - airbyteColumns = u.asColumns(); - } else { - throw new IllegalArgumentException("Top-level schema must be an object"); - } - - if (stream.getPrimaryKey().stream().anyMatch(key -> key.size() > 1)) { - throw new IllegalArgumentException("Only top-level primary keys are supported"); - } - final List primaryKey = stream.getPrimaryKey().stream().map(key -> sqlGenerator.buildColumnId(key.get(0))).toList(); - - if (stream.getCursorField().size() > 1) { - throw new IllegalArgumentException("Only top-level cursors are supported"); - } - final Optional cursor; - if (stream.getCursorField().size() > 0) { - cursor = Optional.of(sqlGenerator.buildColumnId(stream.getCursorField().get(0))); - } else { - cursor = Optional.empty(); - } - - // this code is really bad and I'm not convinced we need to preserve this behavior. - // as with the tablename collisions thing above - we're trying to preserve legacy normalization's - // naming conventions here. - final LinkedHashMap columns = new LinkedHashMap<>(); - for (final Entry entry : airbyteColumns.entrySet()) { - final ColumnId originalColumnId = sqlGenerator.buildColumnId(entry.getKey()); - ColumnId columnId; - if (columns.keySet().stream().noneMatch(c -> c.canonicalName().equals(originalColumnId.canonicalName()))) { - // None of the existing columns have the same name. We can add this new column as-is. - columnId = originalColumnId; - } else { - // One of the existing columns has the same name. We need to handle this collision. - // Append _1, _2, _3, ... to the column name until we find one that doesn't collide. - int i = 1; - while (true) { - columnId = sqlGenerator.buildColumnId(entry.getKey() + "_" + i); - final String canonicalName = columnId.canonicalName(); - if (columns.keySet().stream().noneMatch(c -> c.canonicalName().equals(canonicalName))) { - break; - } else { - i++; - } - } - // But we need to keep the original name so that we can still fetch it out of the JSON records. - columnId = new ColumnId( - columnId.name(), - originalColumnId.originalName(), - columnId.canonicalName()); - } - - columns.put(columnId, entry.getValue()); - } - - return new StreamConfig( - sqlGenerator.buildStreamId(stream.getStream().getNamespace(), stream.getStream().getName(), rawNamespace), - stream.getSyncMode(), - stream.getDestinationSyncMode(), - primaryKey, - cursor, - columns); - } - -} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java deleted file mode 100644 index 5c078e83c3c5..000000000000 --- a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduper.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.base.destination.typing_deduping; - -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import java.util.HashSet; -import java.util.Optional; -import java.util.Set; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * An abstraction over SqlGenerator and DestinationHandler. Destinations will still need to call - * {@code new CatalogParser(new FooSqlGenerator()).parseCatalog()}, but should otherwise avoid - * interacting directly with these classes. - *

      - * In a typical sync, destinations should call the methods: - *

        - *
      1. {@link #prepareTables()} once at the start of the sync
      2. - *
      3. {@link #typeAndDedupe(String, String)} as needed throughout the sync
      4. - *
      5. {@link #commitFinalTables()} once at the end of the sync
      6. - *
      - * Note that createFinalTables initializes some internal state. The other methods will throw an - * exception if that method was not called. - */ -public class DefaultTyperDeduper implements TyperDeduper { - - private static final Logger LOGGER = LoggerFactory.getLogger(TyperDeduper.class); - - private static final String NO_SUFFIX = ""; - private static final String TMP_OVERWRITE_TABLE_SUFFIX = "_airbyte_tmp"; - - private final SqlGenerator sqlGenerator; - private final DestinationHandler destinationHandler; - - private final DestinationV1V2Migrator v1V2Migrator; - private final V2RawTableMigrator v2RawTableMigrator; - private final ParsedCatalog parsedCatalog; - private Set overwriteStreamsWithTmpTable; - private final Set streamsWithSuccesfulSetup; - - public DefaultTyperDeduper(final SqlGenerator sqlGenerator, - final DestinationHandler destinationHandler, - final ParsedCatalog parsedCatalog, - final DestinationV1V2Migrator v1V2Migrator, - final V2RawTableMigrator v2RawTableMigrator) { - this.sqlGenerator = sqlGenerator; - this.destinationHandler = destinationHandler; - this.parsedCatalog = parsedCatalog; - this.v1V2Migrator = v1V2Migrator; - this.v2RawTableMigrator = v2RawTableMigrator; - this.streamsWithSuccesfulSetup = new HashSet<>(); - } - - public DefaultTyperDeduper( - final SqlGenerator sqlGenerator, - final DestinationHandler destinationHandler, - final ParsedCatalog parsedCatalog, - final DestinationV1V2Migrator v1V2Migrator) { - this(sqlGenerator, destinationHandler, parsedCatalog, v1V2Migrator, new NoopV2RawTableMigrator<>()); - } - - /** - * Create the tables that T+D will write to during the sync. In OVERWRITE mode, these might not be - * the true final tables. Specifically, other than an initial sync (i.e. table does not exist, or is - * empty) we write to a temporary final table, and swap it into the true final table at the end of - * the sync. This is to prevent user downtime during a sync. - */ - public void prepareTables() throws Exception { - if (overwriteStreamsWithTmpTable != null) { - throw new IllegalStateException("Tables were already prepared."); - } - overwriteStreamsWithTmpTable = new HashSet<>(); - LOGGER.info("Preparing final tables"); - - // For each stream, make sure that its corresponding final table exists. - // Also, for OVERWRITE streams, decide if we're writing directly to the final table, or into an - // _airbyte_tmp table. - for (final StreamConfig stream : parsedCatalog.streams()) { - // Migrate the Raw Tables if this is the first v2 sync after a v1 sync - v1V2Migrator.migrateIfNecessary(sqlGenerator, destinationHandler, stream); - v2RawTableMigrator.migrateIfNecessary(stream); - - final Optional existingTable = destinationHandler.findExistingTable(stream.id()); - if (existingTable.isPresent()) { - LOGGER.info("Final Table exists for stream {}", stream.id().finalName()); - // The table already exists. Decide whether we're writing to it directly, or using a tmp table. - if (stream.destinationSyncMode() == DestinationSyncMode.OVERWRITE) { - if (!destinationHandler.isFinalTableEmpty(stream.id()) || !sqlGenerator.existingSchemaMatchesStreamConfig(stream, existingTable.get())) { - // We want to overwrite an existing table. Write into a tmp table. We'll overwrite the table at the - // end of the sync. - overwriteStreamsWithTmpTable.add(stream.id()); - // overwrite an existing tmp table if needed. - destinationHandler.execute(sqlGenerator.createTable(stream, TMP_OVERWRITE_TABLE_SUFFIX, true)); - LOGGER.info("Using temp final table for stream {}, will overwrite existing table at end of sync", stream.id().finalName()); - } else { - LOGGER.info("Final Table for stream {} is empty and matches the expected v2 format, writing to table directly", stream.id().finalName()); - } - - } else if (!sqlGenerator.existingSchemaMatchesStreamConfig(stream, existingTable.get())) { - // We're loading data directly into the existing table. Make sure it has the right schema. - LOGGER.info("Existing schema for stream {} is different from expected schema. Executing soft reset.", stream.id().finalTableId("")); - destinationHandler.execute(sqlGenerator.softReset(stream)); - } - } else { - LOGGER.info("Final Table does not exist for stream {}, creating.", stream.id().finalName()); - // The table doesn't exist. Create it. Don't force. - destinationHandler.execute(sqlGenerator.createTable(stream, NO_SUFFIX, false)); - } - - streamsWithSuccesfulSetup.add(stream.id()); - } - } - - /** - * Execute typing and deduping for a single stream (i.e. fetch new raw records into the final table, - * etc.). - *

      - * This method is thread-safe; multiple threads can call it concurrently. - * - * @param originalNamespace The stream's namespace, as declared in the configured catalog - * @param originalName The stream's name, as declared in the configured catalog - */ - public void typeAndDedupe(final String originalNamespace, final String originalName) throws Exception { - LOGGER.info("Attempting typing and deduping for {}.{}", originalNamespace, originalName); - final var streamConfig = parsedCatalog.getStream(originalNamespace, originalName); - if (streamsWithSuccesfulSetup.stream() - .noneMatch(streamId -> streamId.originalNamespace().equals(originalNamespace) && streamId.originalName().equals(originalName))) { - // For example, if T+D setup fails, but the consumer tries to run T+D on all streams during close, - // we should skip it. - LOGGER.warn("Skipping typing and deduping for {}.{} because we could not set up the tables for this stream.", originalNamespace, originalName); - return; - } - final String suffix = getFinalTableSuffix(streamConfig.id()); - final String sql = sqlGenerator.updateTable(streamConfig, suffix); - destinationHandler.execute(sql); - } - - /** - * Does any "end of sync" work. For most streams, this is a noop. - *

      - * For OVERWRITE streams where we're writing to a temp table, this is where we swap the temp table - * into the final table. - */ - public void commitFinalTables() throws Exception { - LOGGER.info("Committing final tables"); - for (final StreamConfig streamConfig : parsedCatalog.streams()) { - if (!streamsWithSuccesfulSetup.contains(streamConfig.id())) { - LOGGER.warn("Skipping committing final table for for {}.{} because we could not set up the tables for this stream.", - streamConfig.id().originalNamespace(), streamConfig.id().originalName()); - continue; - } - if (DestinationSyncMode.OVERWRITE.equals(streamConfig.destinationSyncMode())) { - final StreamId streamId = streamConfig.id(); - final String finalSuffix = getFinalTableSuffix(streamId); - if (!StringUtils.isEmpty(finalSuffix)) { - final String overwriteFinalTable = sqlGenerator.overwriteFinalTable(streamId, finalSuffix); - LOGGER.info("Overwriting final table with tmp table for stream {}.{}", streamId.originalNamespace(), streamId.originalName()); - destinationHandler.execute(overwriteFinalTable); - } - } - } - } - - private String getFinalTableSuffix(final StreamId streamId) { - return overwriteStreamsWithTmpTable.contains(streamId) ? TMP_OVERWRITE_TABLE_SUFFIX : NO_SUFFIX; - } - -} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationHandler.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationHandler.java deleted file mode 100644 index 9ace9bd64c65..000000000000 --- a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/DestinationHandler.java +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.base.destination.typing_deduping; - -import java.util.Optional; - -public interface DestinationHandler { - - Optional findExistingTable(StreamId id) throws Exception; - - boolean isFinalTableEmpty(StreamId id) throws Exception; - - void execute(final String sql) throws Exception; - -} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoopTyperDeduper.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoopTyperDeduper.java deleted file mode 100644 index a503914efa6a..000000000000 --- a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoopTyperDeduper.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.base.destination.typing_deduping; - -public class NoopTyperDeduper implements TyperDeduper { - - @Override - public void prepareTables() throws Exception { - - } - - @Override - public void typeAndDedupe(String originalNamespace, String originalName) throws Exception { - - } - - @Override - public void commitFinalTables() throws Exception { - - } - -} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoopV2RawTableMigrator.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoopV2RawTableMigrator.java deleted file mode 100644 index 8535481d7847..000000000000 --- a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/NoopV2RawTableMigrator.java +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.base.destination.typing_deduping; - -public class NoopV2RawTableMigrator implements V2RawTableMigrator { - - @Override - public void migrateIfNecessary(final StreamConfig streamConfig) { - // do nothing - } - -} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/SqlGenerator.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/SqlGenerator.java deleted file mode 100644 index 4fe85355a931..000000000000 --- a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/SqlGenerator.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.base.destination.typing_deduping; - -public interface SqlGenerator { - - String SOFT_RESET_SUFFIX = "_ab_soft_reset"; - - StreamId buildStreamId(String namespace, String name, String rawNamespaceOverride); - - ColumnId buildColumnId(String name); - - /** - * Generate a SQL statement to create a fresh table to match the given stream. - *

      - * The generated SQL should throw an exception if the table already exists and {@code force} is - * false. Callers should use - * {@link #existingSchemaMatchesStreamConfig(StreamConfig, java.lang.Object)} if the table is known - * to exist, and potentially {@link #softReset(StreamConfig)}. - * - * @param suffix A suffix to add to the stream name. Useful for full refresh overwrite syncs, where - * we write the entire sync to a temp table. - * @param force If true, will overwrite an existing table. If false, will throw an exception if the - * table already exists. If you're passing a non-empty prefix, you likely want to set this to - * true. - */ - String createTable(final StreamConfig stream, final String suffix, boolean force); - - /** - * Check the final table's schema and compare it to what the stream config would generate. - * - * @param stream the stream/stable in question - * @param existingTable the existing table mapped to the stream - * @return whether the existing table matches the expected schema - */ - boolean existingSchemaMatchesStreamConfig(final StreamConfig stream, final DialectTableDefinition existingTable); - - /** - * SQL Statement which will rebuild the final table using the raw table data. Should not cause data - * downtime. Typically this will resemble "create tmp_table; update raw_table set loaded_at=null; - * (t+d into tmp table); (overwrite final table from tmp table);" - * - * @param stream the stream to rebuild - */ - String softReset(final StreamConfig stream); - - /** - * Generate a SQL statement to copy new data from the raw table into the final table. - *

      - * Responsible for: - *

        - *
      • Pulling new raw records from a table (i.e. records with null _airbyte_loaded_at)
      • - *
      • Extracting the JSON fields and casting to the appropriate types
      • - *
      • Handling errors in those casts
      • - *
      • Merging those typed records into an existing table
      • - *
      • Updating the raw records with SET _airbyte_loaded_at = now()
      • - *
      - *

      - * Implementing classes are recommended to break this into smaller methods, which can be tested in - * isolation. However, this interface only requires a single mega-method. - * - * @param finalSuffix the suffix of the final table to write to. If empty string, writes to the - * final table directly. Useful for full refresh overwrite syncs, where we write the entire - * sync to a temp table and then swap it into the final table at the end. - */ - String updateTable(final StreamConfig stream, String finalSuffix); - - /** - * Drop the previous final table, and rename the new final table to match the old final table. - *

      - * This method may assume that the stream is an OVERWRITE stream, and that the final suffix is - * non-empty. Callers are responsible for verifying those are true. - */ - String overwriteFinalTable(StreamId stream, String finalSuffix); - - /** - * Creates a sql query which will create a v2 raw table from the v1 raw table, then performs a soft - * reset. - * - * @param streamId the stream to migrate - * @param namespace - * @param tableName - * @return a string containing the necessary sql to migrate - */ - String migrateFromV1toV2(StreamId streamId, String namespace, String tableName); - -} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Struct.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Struct.java deleted file mode 100644 index 80eb61be79c5..000000000000 --- a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/Struct.java +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.base.destination.typing_deduping; - -import java.util.LinkedHashMap; - -/** - * @param properties Use LinkedHashMap to preserve insertion order. - */ -public record Struct(LinkedHashMap properties) implements AirbyteType { - -} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduper.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduper.java deleted file mode 100644 index 8a90791359f8..000000000000 --- a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/TyperDeduper.java +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.base.destination.typing_deduping; - -public interface TyperDeduper { - - void prepareTables() throws Exception; - - void typeAndDedupe(String originalNamespace, String originalName) throws Exception; - - void commitFinalTables() throws Exception; - -} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/V2RawTableMigrator.java b/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/V2RawTableMigrator.java deleted file mode 100644 index d17722ac1279..000000000000 --- a/airbyte-integrations/bases/base-typing-deduping/src/main/java/io/airbyte/integrations/base/destination/typing_deduping/V2RawTableMigrator.java +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.base.destination.typing_deduping; - -public interface V2RawTableMigrator { - - void migrateIfNecessary(final StreamConfig streamConfig) throws InterruptedException; - -} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParserTest.java b/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParserTest.java deleted file mode 100644 index f79da6a374f6..000000000000 --- a/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/CatalogParserTest.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.base.destination.typing_deduping; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.mockito.Mockito.*; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.protocol.models.v0.AirbyteStream; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import java.util.List; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class CatalogParserTest { - - private SqlGenerator sqlGenerator; - private CatalogParser parser; - - @BeforeEach - public void setup() { - sqlGenerator = mock(SqlGenerator.class); - // noop quoting logic - when(sqlGenerator.buildColumnId(any())).thenAnswer(invocation -> { - String fieldName = invocation.getArgument(0); - return new ColumnId(fieldName, fieldName, fieldName); - }); - when(sqlGenerator.buildStreamId(any(), any(), any())).thenAnswer(invocation -> { - String namespace = invocation.getArgument(0); - String name = invocation.getArgument(1); - String rawNamespace = invocation.getArgument(1); - return new StreamId(namespace, name, rawNamespace, namespace + "_abab_" + name, namespace, name); - }); - - parser = new CatalogParser(sqlGenerator); - } - - /** - * Both these streams will write to the same final table name ("foofoo"). Verify that they don't - * actually use the same tablename. - */ - @Test - public void finalNameCollision() { - when(sqlGenerator.buildStreamId(any(), any(), any())).thenAnswer(invocation -> { - String originalNamespace = invocation.getArgument(0); - String originalName = (invocation.getArgument(1)); - String originalRawNamespace = (invocation.getArgument(1)); - - // emulate quoting logic that causes a name collision - String quotedName = originalName.replaceAll("bar", ""); - return new StreamId(originalNamespace, quotedName, originalRawNamespace, originalNamespace + "_abab_" + quotedName, originalNamespace, - originalName); - }); - final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( - stream("a", "foobarfoo"), - stream("a", "foofoo"))); - - final ParsedCatalog parsedCatalog = parser.parseCatalog(catalog); - - assertNotEquals( - parsedCatalog.streams().get(0).id().finalName(), - parsedCatalog.streams().get(1).id().finalName()); - } - - /** - * The schema contains two fields, which will both end up named "foofoo" after quoting. Verify that - * they don't actually use the same column name. - */ - @Test - public void columnNameCollision() { - when(sqlGenerator.buildColumnId(any())).thenAnswer(invocation -> { - String originalName = invocation.getArgument(0); - - // emulate quoting logic that causes a name collision - String quotedName = originalName.replaceAll("bar", ""); - return new ColumnId(quotedName, originalName, quotedName); - }); - JsonNode schema = Jsons.deserialize(""" - { - "type": "object", - "properties": { - "foobarfoo": {"type": "string"}, - "foofoo": {"type": "string"} - } - } - """); - final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of(stream("a", "a", schema))); - - final ParsedCatalog parsedCatalog = parser.parseCatalog(catalog); - - assertEquals(2, parsedCatalog.streams().get(0).columns().size()); - } - - private static ConfiguredAirbyteStream stream(String namespace, String name) { - return stream( - namespace, - name, - Jsons.deserialize(""" - { - "type": "object", - "properties": { - "name": {"type": "string"} - } - } - """)); - } - - private static ConfiguredAirbyteStream stream(String namespace, String name, JsonNode schema) { - return new ConfiguredAirbyteStream().withStream( - new AirbyteStream() - .withNamespace(namespace) - .withName(name) - .withJsonSchema(schema)); - } - -} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduperTest.java b/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduperTest.java deleted file mode 100644 index 62fb1374eed9..000000000000 --- a/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/DefaultTyperDeduperTest.java +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.base.destination.typing_deduping; - -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.clearInvocations; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.ignoreStubs; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; -import static org.mockito.Mockito.verifyNoMoreInteractions; -import static org.mockito.Mockito.when; - -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import java.util.List; -import java.util.Optional; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class DefaultTyperDeduperTest { - - private MockSqlGenerator sqlGenerator; - private DestinationHandler destinationHandler; - - private DestinationV1V2Migrator migrator; - private TyperDeduper typerDeduper; - - @BeforeEach - void setup() { - sqlGenerator = spy(new MockSqlGenerator()); - destinationHandler = mock(DestinationHandler.class); - migrator = new NoOpDestinationV1V2Migrator<>(); - - final ParsedCatalog parsedCatalog = new ParsedCatalog(List.of( - new StreamConfig( - new StreamId("overwrite_ns", "overwrite_stream", null, null, "overwrite_ns", "overwrite_stream"), - null, - DestinationSyncMode.OVERWRITE, - null, - null, - null), - new StreamConfig( - new StreamId("append_ns", "append_stream", null, null, "append_ns", "append_stream"), - null, - DestinationSyncMode.APPEND, - null, - null, - null), - new StreamConfig( - new StreamId("dedup_ns", "dedup_stream", null, null, "dedup_ns", "dedup_stream"), - null, - DestinationSyncMode.APPEND_DEDUP, - null, - null, - null))); - - typerDeduper = new DefaultTyperDeduper<>(sqlGenerator, destinationHandler, parsedCatalog, migrator); - } - - /** - * When there are no existing tables, we should create them and write to them directly. - */ - @Test - void emptyDestination() throws Exception { - when(destinationHandler.findExistingTable(any())).thenReturn(Optional.empty()); - - typerDeduper.prepareTables(); - verify(destinationHandler).execute("CREATE TABLE overwrite_ns.overwrite_stream"); - verify(destinationHandler).execute("CREATE TABLE append_ns.append_stream"); - verify(destinationHandler).execute("CREATE TABLE dedup_ns.dedup_stream"); - verifyNoMoreInteractions(ignoreStubs(destinationHandler)); - clearInvocations(destinationHandler); - - typerDeduper.typeAndDedupe("overwrite_ns", "overwrite_stream"); - verify(destinationHandler).execute("UPDATE TABLE overwrite_ns.overwrite_stream"); - typerDeduper.typeAndDedupe("append_ns", "append_stream"); - verify(destinationHandler).execute("UPDATE TABLE append_ns.append_stream"); - typerDeduper.typeAndDedupe("dedup_ns", "dedup_stream"); - verify(destinationHandler).execute("UPDATE TABLE dedup_ns.dedup_stream"); - verifyNoMoreInteractions(ignoreStubs(destinationHandler)); - clearInvocations(destinationHandler); - - typerDeduper.commitFinalTables(); - verify(destinationHandler, never()).execute(any()); - } - - /** - * When there's an existing table but it's empty, we should ensure it has the right schema and write - * to it directly. - */ - @Test - void existingEmptyTable() throws Exception { - when(destinationHandler.findExistingTable(any())).thenReturn(Optional.of("foo")); - when(destinationHandler.isFinalTableEmpty(any())).thenReturn(true); - when(sqlGenerator.existingSchemaMatchesStreamConfig(any(), any())).thenReturn(false); - - typerDeduper.prepareTables(); - verify(destinationHandler).execute("CREATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp"); - verify(destinationHandler).execute("SOFT RESET append_ns.append_stream"); - verify(destinationHandler).execute("SOFT RESET dedup_ns.dedup_stream"); - verifyNoMoreInteractions(ignoreStubs(destinationHandler)); - clearInvocations(destinationHandler); - - typerDeduper.typeAndDedupe("overwrite_ns", "overwrite_stream"); - verify(destinationHandler).execute("UPDATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp"); - typerDeduper.typeAndDedupe("append_ns", "append_stream"); - verify(destinationHandler).execute("UPDATE TABLE append_ns.append_stream"); - typerDeduper.typeAndDedupe("dedup_ns", "dedup_stream"); - verify(destinationHandler).execute("UPDATE TABLE dedup_ns.dedup_stream"); - verifyNoMoreInteractions(ignoreStubs(destinationHandler)); - clearInvocations(destinationHandler); - - typerDeduper.commitFinalTables(); - verify(destinationHandler).execute("OVERWRITE TABLE overwrite_ns.overwrite_stream FROM overwrite_ns.overwrite_stream_airbyte_tmp"); - verifyNoMoreInteractions(ignoreStubs(destinationHandler)); - } - - /** - * When there's an existing empty table with the right schema, we don't need to do anything during - * setup. - */ - @Test - void existingEmptyTableMatchingSchema() throws Exception { - when(destinationHandler.findExistingTable(any())).thenReturn(Optional.of("foo")); - when(destinationHandler.isFinalTableEmpty(any())).thenReturn(true); - when(sqlGenerator.existingSchemaMatchesStreamConfig(any(), any())).thenReturn(true); - - typerDeduper.prepareTables(); - verify(destinationHandler, never()).execute(any()); - } - - /** - * When there's an existing nonempty table, we should alter it. For the OVERWRITE stream, we also - * need to write to a tmp table, and overwrite the real table at the end of the sync. - */ - @Test - void existingNonemptyTable() throws Exception { - when(destinationHandler.findExistingTable(any())).thenReturn(Optional.of("foo")); - when(destinationHandler.isFinalTableEmpty(any())).thenReturn(false); - - typerDeduper.prepareTables(); - // NB: We only create a tmp table for the overwrite stream, and do _not_ soft reset the existing - // overwrite stream's table. - verify(destinationHandler).execute("CREATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp"); - verify(destinationHandler).execute("SOFT RESET append_ns.append_stream"); - verify(destinationHandler).execute("SOFT RESET dedup_ns.dedup_stream"); - verifyNoMoreInteractions(ignoreStubs(destinationHandler)); - clearInvocations(destinationHandler); - - typerDeduper.typeAndDedupe("overwrite_ns", "overwrite_stream"); - // NB: no airbyte_tmp suffix on the non-overwrite streams - verify(destinationHandler).execute("UPDATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp"); - typerDeduper.typeAndDedupe("append_ns", "append_stream"); - verify(destinationHandler).execute("UPDATE TABLE append_ns.append_stream"); - typerDeduper.typeAndDedupe("dedup_ns", "dedup_stream"); - verify(destinationHandler).execute("UPDATE TABLE dedup_ns.dedup_stream"); - verifyNoMoreInteractions(ignoreStubs(destinationHandler)); - clearInvocations(destinationHandler); - - typerDeduper.commitFinalTables(); - verify(destinationHandler).execute("OVERWRITE TABLE overwrite_ns.overwrite_stream FROM overwrite_ns.overwrite_stream_airbyte_tmp"); - verifyNoMoreInteractions(ignoreStubs(destinationHandler)); - } - - /** - * When there's an existing nonempty table with the right schema, we don't need to modify it, but - * OVERWRITE streams still need to create a tmp table. - */ - @Test - void existingNonemptyTableMatchingSchema() throws Exception { - when(destinationHandler.findExistingTable(any())).thenReturn(Optional.of("foo")); - when(destinationHandler.isFinalTableEmpty(any())).thenReturn(false); - when(sqlGenerator.existingSchemaMatchesStreamConfig(any(), any())).thenReturn(true); - - typerDeduper.prepareTables(); - // NB: We only create one tmp table here. - // Also, we need to alter the existing _real_ table, not the tmp table! - verify(destinationHandler).execute("CREATE TABLE overwrite_ns.overwrite_stream_airbyte_tmp"); - verifyNoMoreInteractions(ignoreStubs(destinationHandler)); - } - - @Test - void nonexistentStream() { - assertThrows(IllegalArgumentException.class, - () -> typerDeduper.typeAndDedupe("nonexistent_ns", "nonexistent_stream")); - verifyNoInteractions(ignoreStubs(destinationHandler)); - } - - @Test - void failedSetup() throws Exception { - doThrow(new RuntimeException("foo")).when(destinationHandler).execute(any()); - - assertThrows(RuntimeException.class, () -> typerDeduper.prepareTables()); - clearInvocations(destinationHandler); - - typerDeduper.typeAndDedupe("dedup_ns", "dedup_stream"); - typerDeduper.commitFinalTables(); - - verifyNoInteractions(ignoreStubs(destinationHandler)); - } - -} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/MockSqlGenerator.java b/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/MockSqlGenerator.java deleted file mode 100644 index 3f56b61114e2..000000000000 --- a/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/MockSqlGenerator.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.base.destination.typing_deduping; - -/** - * Basic SqlGenerator mock. See {@link DefaultTyperDeduperTest} for example usage. - */ -class MockSqlGenerator implements SqlGenerator { - - @Override - public StreamId buildStreamId(final String namespace, final String name, final String rawNamespaceOverride) { - return null; - } - - @Override - public ColumnId buildColumnId(final String name) { - return null; - } - - @Override - public String createTable(final StreamConfig stream, final String suffix, final boolean force) { - return "CREATE TABLE " + stream.id().finalTableId("", suffix); - } - - @Override - public boolean existingSchemaMatchesStreamConfig(final StreamConfig stream, final String existingTable) throws TableNotMigratedException { - return false; - } - - @Override - public String softReset(final StreamConfig stream) { - return "SOFT RESET " + stream.id().finalTableId(""); - } - - @Override - public String updateTable(final StreamConfig stream, final String finalSuffix) { - return "UPDATE TABLE " + stream.id().finalTableId("", finalSuffix); - } - - @Override - public String overwriteFinalTable(final StreamId stream, final String finalSuffix) { - return "OVERWRITE TABLE " + stream.finalTableId("") + " FROM " + stream.finalTableId("", finalSuffix); - } - - @Override - public String migrateFromV1toV2(final StreamId streamId, final String namespace, final String tableName) { - return "MIGRATE TABLE " + String.join(".", namespace, tableName) + " TO " + streamId.rawTableId(""); - } - -} diff --git a/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValveTest.java b/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValveTest.java deleted file mode 100644 index 3f6b35e6eaa3..000000000000 --- a/airbyte-integrations/bases/base-typing-deduping/src/test/java/io/airbyte/integrations/base/destination/typing_deduping/TypeAndDedupeOperationValveTest.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.base.destination.typing_deduping; - -import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; -import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Supplier; -import java.util.stream.IntStream; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -public class TypeAndDedupeOperationValveTest { - - private static final AirbyteStreamNameNamespacePair STREAM_A = new AirbyteStreamNameNamespacePair("a", "a"); - private static final AirbyteStreamNameNamespacePair STREAM_B = new AirbyteStreamNameNamespacePair("b", "b"); - private static final Supplier ALWAYS_ZERO = () -> 0l; - - private Supplier minuteUpdates; - - @BeforeEach - public void setup() { - AtomicLong start = new AtomicLong(0); - minuteUpdates = () -> start.getAndUpdate(l -> l + (60 * 1000)); - } - - private void elapseTime(Supplier timing, int iterations) { - IntStream.range(0, iterations).forEach(__ -> { - timing.get(); - }); - } - - @Test - public void testAddStream() { - final var valve = new TypeAndDedupeOperationValve(ALWAYS_ZERO); - valve.addStream(STREAM_A); - Assertions.assertEquals(-1, valve.getIncrementInterval(STREAM_A)); - Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A)); - Assertions.assertEquals(valve.get(STREAM_A), 0l); - } - - @Test - public void testReadyToTypeAndDedupe() { - final var valve = new TypeAndDedupeOperationValve(minuteUpdates); - // method call increments time - valve.addStream(STREAM_A); - elapseTime(minuteUpdates, 1); - // method call increments time - valve.addStream(STREAM_B); - // method call increments time - Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A)); - elapseTime(minuteUpdates, 1); - Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_B)); - valve.updateTimeAndIncreaseInterval(STREAM_A); - Assertions.assertEquals(1000 * 60 * 15, - valve.getIncrementInterval(STREAM_A)); - // method call increments time - Assertions.assertFalse(valve.readyToTypeAndDedupe(STREAM_A)); - // More than enough time has passed now - elapseTime(minuteUpdates, 15); - Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A)); - } - - @Test - public void testIncrementInterval() { - final var valve = new TypeAndDedupeOperationValve(ALWAYS_ZERO); - valve.addStream(STREAM_A); - IntStream.rangeClosed(1, 4).forEach(i -> { - final var index = valve.incrementInterval(STREAM_A); - Assertions.assertEquals(i, index); - }); - Assertions.assertEquals(4, valve.incrementInterval(STREAM_A)); - // Twice to be sure - Assertions.assertEquals(4, valve.incrementInterval(STREAM_A)); - } - - @Test - public void testUpdateTimeAndIncreaseInterval() { - final var valve = new TypeAndDedupeOperationValve(minuteUpdates); - valve.addStream(STREAM_A); - IntStream.range(0, 1).forEach(__ -> Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A))); // start ready to T&D - Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A)); - valve.updateTimeAndIncreaseInterval(STREAM_A); - IntStream.range(0, 15).forEach(__ -> Assertions.assertFalse(valve.readyToTypeAndDedupe(STREAM_A))); - Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A)); - valve.updateTimeAndIncreaseInterval(STREAM_A); - IntStream.range(0, 60).forEach(__ -> Assertions.assertFalse(valve.readyToTypeAndDedupe(STREAM_A))); - Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A)); - valve.updateTimeAndIncreaseInterval(STREAM_A); - IntStream.range(0, 120).forEach(__ -> Assertions.assertFalse(valve.readyToTypeAndDedupe(STREAM_A))); - Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A)); - valve.updateTimeAndIncreaseInterval(STREAM_A); - IntStream.range(0, 240).forEach(__ -> Assertions.assertFalse(valve.readyToTypeAndDedupe(STREAM_A))); - Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A)); - valve.updateTimeAndIncreaseInterval(STREAM_A); - IntStream.range(0, 240).forEach(__ -> Assertions.assertFalse(valve.readyToTypeAndDedupe(STREAM_A))); - Assertions.assertTrue(valve.readyToTypeAndDedupe(STREAM_A)); - } - -} diff --git a/airbyte-integrations/bases/base/build.gradle b/airbyte-integrations/bases/base/build.gradle index c5810d7529af..0c2de175e2cc 100644 --- a/airbyte-integrations/bases/base/build.gradle +++ b/airbyte-integrations/bases/base/build.gradle @@ -1,3 +1,3 @@ plugins { - id 'airbyte-docker' + id 'airbyte-docker-legacy' } diff --git a/airbyte-integrations/bases/bases-destination-jdbc/build.gradle b/airbyte-integrations/bases/bases-destination-jdbc/build.gradle deleted file mode 100644 index 4b731d310b42..000000000000 --- a/airbyte-integrations/bases/bases-destination-jdbc/build.gradle +++ /dev/null @@ -1,35 +0,0 @@ -plugins { - id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' -} - -dependencies { - implementation 'com.google.cloud:google-cloud-storage:1.113.16' - implementation 'com.google.auth:google-auth-library-oauth2-http:0.25.5' - - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:bases:base-java-s3') - implementation project(':airbyte-integrations:bases:base-typing-deduping') - implementation libs.airbyte.protocol - - implementation 'org.apache.commons:commons-lang3:3.11' - implementation 'org.apache.commons:commons-csv:1.4' - implementation 'com.github.alexmojaki:s3-stream-upload:2.2.2' - implementation 'com.fasterxml.jackson.core:jackson-databind' - implementation 'com.azure:azure-storage-blob:12.12.0' - -// A small utility library for working with units of digital information -// https://github.com/aesy/datasize - implementation "io.aesy:datasize:1.0.0" - - testImplementation libs.connectors.testcontainers.postgresql - testImplementation "org.mockito:mockito-inline:4.1.0" - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation libs.connectors.testcontainers.postgresql - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) -} diff --git a/airbyte-integrations/bases/bases-destination-jdbc/readme.md b/airbyte-integrations/bases/bases-destination-jdbc/readme.md deleted file mode 100644 index 90924191b4be..000000000000 --- a/airbyte-integrations/bases/bases-destination-jdbc/readme.md +++ /dev/null @@ -1,7 +0,0 @@ -# JDBC Destination - -We are not planning to expose this destination in the UI yet. It serves as a base upon which we can build all of our other JDBC-compliant destinations. - -The reasons we are not exposing this destination by itself are: -1. It is not terribly user-friendly (jdbc urls are hard for a human to parse) -1. Each JDBC-compliant db, we need to make sure the appropriate drivers are installed on the image. We don't want to frontload installing all possible drivers, and instead would like to be more methodical. Instead for each JDBC-compliant destination, we will extend this one and then install only the necessary JDBC drivers on that destination's image. diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java b/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java deleted file mode 100644 index 28079f9f50ea..000000000000 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/AbstractJdbcDestination.java +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.jdbc; - -import static io.airbyte.integrations.base.errors.messages.ErrorMessage.getErrorMessage; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.exceptions.ConnectionErrorException; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.map.MoreMaps; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.AirbyteTraceMessageUtility; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import java.sql.SQLException; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.UUID; -import java.util.function.Consumer; -import javax.sql.DataSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public abstract class AbstractJdbcDestination extends BaseConnector implements Destination { - - private static final Logger LOGGER = LoggerFactory.getLogger(AbstractJdbcDestination.class); - - private final String driverClass; - private final NamingConventionTransformer namingResolver; - private final SqlOperations sqlOperations; - - protected NamingConventionTransformer getNamingResolver() { - return namingResolver; - } - - protected SqlOperations getSqlOperations() { - return sqlOperations; - } - - public AbstractJdbcDestination(final String driverClass, - final NamingConventionTransformer namingResolver, - final SqlOperations sqlOperations) { - this.driverClass = driverClass; - this.namingResolver = namingResolver; - this.sqlOperations = sqlOperations; - } - - @Override - public AirbyteConnectionStatus check(final JsonNode config) { - final DataSource dataSource = getDataSource(config); - - try { - final JdbcDatabase database = getDatabase(dataSource); - final String outputSchema = namingResolver.getIdentifier(config.get(JdbcUtils.SCHEMA_KEY).asText()); - attemptSQLCreateAndDropTableOperations(outputSchema, database, namingResolver, sqlOperations); - return new AirbyteConnectionStatus().withStatus(Status.SUCCEEDED); - } catch (final ConnectionErrorException ex) { - final String message = getErrorMessage(ex.getStateCode(), ex.getErrorCode(), ex.getExceptionMessage(), ex); - AirbyteTraceMessageUtility.emitConfigErrorTrace(ex, message); - return new AirbyteConnectionStatus() - .withStatus(Status.FAILED) - .withMessage(message); - } catch (final Exception e) { - LOGGER.error("Exception while checking connection: ", e); - return new AirbyteConnectionStatus() - .withStatus(Status.FAILED) - .withMessage("Could not connect with provided configuration. \n" + e.getMessage()); - } finally { - try { - DataSourceFactory.close(dataSource); - } catch (final Exception e) { - LOGGER.warn("Unable to close data source.", e); - } - } - } - - /** - * This method is deprecated. It verifies table creation, but not insert right to a newly created - * table. Use attemptTableOperations with the attemptInsert argument instead. - */ - @Deprecated - public static void attemptSQLCreateAndDropTableOperations(final String outputSchema, - final JdbcDatabase database, - final NamingConventionTransformer namingResolver, - final SqlOperations sqlOps) - throws Exception { - attemptTableOperations(outputSchema, database, namingResolver, sqlOps, false); - } - - /** - * Verifies if provided creds has enough permissions. Steps are: 1. Create schema if not exists. 2. - * Create test table. 3. Insert dummy record to newly created table if "attemptInsert" set to true. - * 4. Delete table created on step 2. - * - * @param outputSchema - schema to tests against. - * @param database - database to tests against. - * @param namingResolver - naming resolver. - * @param sqlOps - SqlOperations object - * @param attemptInsert - set true if need to make attempt to insert dummy records to newly created - * table. Set false to skip insert step. - * @throws Exception - */ - public static void attemptTableOperations(final String outputSchema, - final JdbcDatabase database, - final NamingConventionTransformer namingResolver, - final SqlOperations sqlOps, - final boolean attemptInsert) - throws Exception { - // verify we have write permissions on the target schema by creating a table with a random name, - // then dropping that table - try { - // Get metadata from the database to see whether connection is possible - database.bufferedResultSetQuery(conn -> conn.getMetaData().getCatalogs(), JdbcUtils.getDefaultSourceOperations()::rowToJson); - - // verify we have write permissions on the target schema by creating a table with a random name, - // then dropping that table - final String outputTableName = namingResolver.getIdentifier("_airbyte_connection_test_" + UUID.randomUUID().toString().replaceAll("-", "")); - sqlOps.createSchemaIfNotExists(database, outputSchema); - sqlOps.createTableIfNotExists(database, outputSchema, outputTableName); - // verify if user has permission to make SQL INSERT queries - try { - if (attemptInsert) { - sqlOps.insertRecords(database, List.of(getDummyRecord()), outputSchema, outputTableName); - } - } finally { - sqlOps.dropTableIfExists(database, outputSchema, outputTableName); - } - } catch (final SQLException e) { - if (Objects.isNull(e.getCause()) || !(e.getCause() instanceof SQLException)) { - throw new ConnectionErrorException(e.getSQLState(), e.getErrorCode(), e.getMessage(), e); - } else { - final SQLException cause = (SQLException) e.getCause(); - throw new ConnectionErrorException(e.getSQLState(), cause.getErrorCode(), cause.getMessage(), e); - } - } catch (final Exception e) { - throw new Exception(e); - } - } - - /** - * Generates a dummy AirbyteRecordMessage with random values. - * - * @return AirbyteRecordMessage object with dummy values that may be used to test insert permission. - */ - private static AirbyteRecordMessage getDummyRecord() { - final JsonNode dummyDataToInsert = Jsons.deserialize("{ \"field1\": true }"); - return new AirbyteRecordMessage() - .withStream("stream1") - .withData(dummyDataToInsert) - .withEmittedAt(1602637589000L); - } - - protected DataSource getDataSource(final JsonNode config) { - final JsonNode jdbcConfig = toJdbcConfig(config); - return DataSourceFactory.create( - jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText(), - jdbcConfig.has(JdbcUtils.PASSWORD_KEY) ? jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, - driverClass, - jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), - getConnectionProperties(config)); - } - - protected JdbcDatabase getDatabase(final DataSource dataSource) { - return new DefaultJdbcDatabase(dataSource); - } - - protected Map getConnectionProperties(final JsonNode config) { - final Map customProperties = JdbcUtils.parseJdbcParameters(config, JdbcUtils.JDBC_URL_PARAMS_KEY); - final Map defaultProperties = getDefaultConnectionProperties(config); - assertCustomParametersDontOverwriteDefaultParameters(customProperties, defaultProperties); - return MoreMaps.merge(customProperties, defaultProperties); - } - - private void assertCustomParametersDontOverwriteDefaultParameters(final Map customParameters, - final Map defaultParameters) { - for (final String key : defaultParameters.keySet()) { - if (customParameters.containsKey(key) && !Objects.equals(customParameters.get(key), defaultParameters.get(key))) { - throw new IllegalArgumentException("Cannot overwrite default JDBC parameter " + key); - } - } - } - - protected abstract Map getDefaultConnectionProperties(final JsonNode config); - - public abstract JsonNode toJdbcConfig(JsonNode config); - - @Override - public AirbyteMessageConsumer getConsumer(final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final Consumer outputRecordCollector) { - return JdbcBufferedConsumerFactory.create(outputRecordCollector, getDatabase(getDataSource(config)), sqlOperations, namingResolver, config, - catalog); - } - -} diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/JdbcBufferedConsumerFactory.java b/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/JdbcBufferedConsumerFactory.java deleted file mode 100644 index 8be0130aeaaf..000000000000 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/JdbcBufferedConsumerFactory.java +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.jdbc; - -import static io.airbyte.integrations.destination.jdbc.constants.GlobalDataSizeConstants.DEFAULT_MAX_BATCH_SIZE_BYTES; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.base.Preconditions; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.buffered_stream_consumer.BufferedStreamConsumer; -import io.airbyte.integrations.destination.buffered_stream_consumer.OnCloseFunction; -import io.airbyte.integrations.destination.buffered_stream_consumer.OnStartFunction; -import io.airbyte.integrations.destination.buffered_stream_consumer.RecordWriter; -import io.airbyte.integrations.destination.record_buffer.InMemoryRecordBufferingStrategy; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import io.airbyte.protocol.models.v0.AirbyteStream; -import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Strategy: - *

      - * 1. Create a final table for each stream - *

      - * 2. Accumulate records in a buffer. One buffer per stream - *

      - * 3. As records accumulate write them in batch to the database. We set a minimum numbers of records - * before writing to avoid wasteful record-wise writes. In the case with slow syncs this will be - * superseded with a periodic record flush from {@link BufferedStreamConsumer#periodicBufferFlush()} - *

      - * 4. Once all records have been written to buffer, flush the buffer and write any remaining records - * to the database (regardless of how few are left) - */ -public class JdbcBufferedConsumerFactory { - - private static final Logger LOGGER = LoggerFactory.getLogger(JdbcBufferedConsumerFactory.class); - - public static AirbyteMessageConsumer create(final Consumer outputRecordCollector, - final JdbcDatabase database, - final SqlOperations sqlOperations, - final NamingConventionTransformer namingResolver, - final JsonNode config, - final ConfiguredAirbyteCatalog catalog) { - final List writeConfigs = createWriteConfigs(namingResolver, config, catalog, sqlOperations.isSchemaRequired()); - - return new BufferedStreamConsumer( - outputRecordCollector, - onStartFunction(database, sqlOperations, writeConfigs), - new InMemoryRecordBufferingStrategy(recordWriterFunction(database, sqlOperations, writeConfigs, catalog), DEFAULT_MAX_BATCH_SIZE_BYTES), - onCloseFunction(), - catalog, - sqlOperations::isValidData); - } - - private static List createWriteConfigs(final NamingConventionTransformer namingResolver, - final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final boolean schemaRequired) { - if (schemaRequired) { - Preconditions.checkState(config.has("schema"), "jdbc destinations must specify a schema."); - } - final Instant now = Instant.now(); - return catalog.getStreams().stream().map(toWriteConfig(namingResolver, config, now, schemaRequired)).collect(Collectors.toList()); - } - - private static Function toWriteConfig( - final NamingConventionTransformer namingResolver, - final JsonNode config, - final Instant now, - final boolean schemaRequired) { - return stream -> { - Preconditions.checkNotNull(stream.getDestinationSyncMode(), "Undefined destination sync mode"); - final AirbyteStream abStream = stream.getStream(); - - final String defaultSchemaName = schemaRequired ? namingResolver.getIdentifier(config.get("schema").asText()) - : namingResolver.getIdentifier(config.get(JdbcUtils.DATABASE_KEY).asText()); - final String outputSchema = getOutputSchema(abStream, defaultSchemaName, namingResolver); - - final String streamName = abStream.getName(); - final String tableName = namingResolver.getRawTableName(streamName); - final String tmpTableName = namingResolver.getTmpTableName(streamName); - final DestinationSyncMode syncMode = stream.getDestinationSyncMode(); - - final WriteConfig writeConfig = new WriteConfig(streamName, abStream.getNamespace(), outputSchema, tmpTableName, tableName, syncMode); - LOGGER.info("Write config: {}", writeConfig); - - return writeConfig; - }; - } - - /** - * Defer to the {@link AirbyteStream}'s namespace. If this is not set, use the destination's default - * schema. This namespace is source-provided, and can be potentially empty. - *

      - * The logic here matches the logic in the catalog_process.py for Normalization. Any modifications - * need to be reflected there and vice versa. - */ - private static String getOutputSchema(final AirbyteStream stream, - final String defaultDestSchema, - final NamingConventionTransformer namingResolver) { - return stream.getNamespace() != null - ? namingResolver.getNamespace(stream.getNamespace()) - : namingResolver.getNamespace(defaultDestSchema); - } - - /** - * Sets up destination storage through: - *

      - * 1. Creates Schema (if not exists) - *

      - * 2. Creates airybte_raw table (if not exists) - *

      - * 3. Truncates table if sync mode is in OVERWRITE - * - * @param database JDBC database to connect to - * @param sqlOperations interface for execution SQL queries - * @param writeConfigs settings for each stream - * @return - */ - private static OnStartFunction onStartFunction(final JdbcDatabase database, - final SqlOperations sqlOperations, - final List writeConfigs) { - return () -> { - LOGGER.info("Preparing raw tables in destination started for {} streams", writeConfigs.size()); - final List queryList = new ArrayList<>(); - for (final WriteConfig writeConfig : writeConfigs) { - final String schemaName = writeConfig.getOutputSchemaName(); - final String dstTableName = writeConfig.getOutputTableName(); - LOGGER.info("Preparing raw table in destination started for stream {}. schema: {}, table name: {}", - writeConfig.getStreamName(), - schemaName, - dstTableName); - sqlOperations.createSchemaIfNotExists(database, schemaName); - sqlOperations.createTableIfNotExists(database, schemaName, dstTableName); - switch (writeConfig.getSyncMode()) { - case OVERWRITE -> queryList.add(sqlOperations.truncateTableQuery(database, schemaName, dstTableName)); - case APPEND, APPEND_DEDUP -> {} - default -> throw new IllegalStateException("Unrecognized sync mode: " + writeConfig.getSyncMode()); - } - } - sqlOperations.executeTransaction(database, queryList); - LOGGER.info("Preparing raw tables in destination completed."); - }; - } - - /** - * Writes {@link AirbyteRecordMessage} to JDBC database's airbyte_raw table - * - * @param database JDBC database to connect to - * @param sqlOperations interface of SQL queries to execute - * @param writeConfigs settings for each stream - * @param catalog catalog of all streams to sync - * @return - */ - private static RecordWriter recordWriterFunction(final JdbcDatabase database, - final SqlOperations sqlOperations, - final List writeConfigs, - final ConfiguredAirbyteCatalog catalog) { - final Map pairToWriteConfig = writeConfigs.stream() - .collect(Collectors.toUnmodifiableMap(JdbcBufferedConsumerFactory::toNameNamespacePair, Function.identity())); - - return (pair, records) -> { - if (!pairToWriteConfig.containsKey(pair)) { - throw new IllegalArgumentException( - String.format("Message contained record from a stream that was not in the catalog. \ncatalog: %s", Jsons.serialize(catalog))); - } - - final WriteConfig writeConfig = pairToWriteConfig.get(pair); - sqlOperations.insertRecords(database, records, writeConfig.getOutputSchemaName(), writeConfig.getOutputTableName()); - }; - } - - /** - * Tear down functionality - * - * @return - */ - private static OnCloseFunction onCloseFunction() { - return (hasFailed) -> {}; - } - - private static AirbyteStreamNameNamespacePair toNameNamespacePair(final WriteConfig config) { - return new AirbyteStreamNameNamespacePair(config.getStreamName(), config.getNamespace()); - } - -} diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/JdbcSqlOperations.java b/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/JdbcSqlOperations.java deleted file mode 100644 index d9bd6e6212d2..000000000000 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/JdbcSqlOperations.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.jdbc; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.exceptions.ConfigErrorException; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import java.io.File; -import java.io.PrintWriter; -import java.nio.charset.StandardCharsets; -import java.sql.SQLException; -import java.sql.Timestamp; -import java.time.Instant; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.UUID; -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVPrinter; - -@SuppressWarnings("OptionalUsedAsFieldOrParameterType") -public abstract class JdbcSqlOperations implements SqlOperations { - - protected static final String SHOW_SCHEMAS = "show schemas;"; - protected static final String NAME = "name"; - - // this adapter modifies record message before inserting them to the destination - protected final Optional dataAdapter; - protected final Set schemaSet = new HashSet<>(); - - protected JdbcSqlOperations() { - this.dataAdapter = Optional.empty(); - } - - protected JdbcSqlOperations(final DataAdapter dataAdapter) { - this.dataAdapter = Optional.of(dataAdapter); - } - - @Override - public void createSchemaIfNotExists(final JdbcDatabase database, final String schemaName) throws Exception { - try { - if (!schemaSet.contains(schemaName) && !isSchemaExists(database, schemaName)) { - database.execute(String.format("CREATE SCHEMA IF NOT EXISTS %s;", schemaName)); - schemaSet.add(schemaName); - } - } catch (final Exception e) { - throw checkForKnownConfigExceptions(e).orElseThrow(() -> e); - } - } - - /** - * When an exception occurs, we may recognize it as an issue with the users permissions or other - * configuration options. In these cases, we can wrap the exception in a - * {@link ConfigErrorException} which will exclude the error from our on-call paging/reporting - * - * @param e the exception to check. - * @return A ConfigErrorException with a message with actionable feedback to the user. - */ - protected Optional checkForKnownConfigExceptions(final Exception e) { - return Optional.empty(); - } - - @Override - public void createTableIfNotExists(final JdbcDatabase database, final String schemaName, final String tableName) throws SQLException { - try { - database.execute(createTableQuery(database, schemaName, tableName)); - } catch (final SQLException e) { - throw checkForKnownConfigExceptions(e).orElseThrow(() -> e); - } - } - - @Override - public String createTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { - return String.format( - "CREATE TABLE IF NOT EXISTS %s.%s ( \n" - + "%s VARCHAR PRIMARY KEY,\n" - + "%s JSONB,\n" - + "%s TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP\n" - + ");\n", - schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_ID, JavaBaseConstants.COLUMN_NAME_DATA, JavaBaseConstants.COLUMN_NAME_EMITTED_AT); - } - - protected void writeBatchToFile(final File tmpFile, final List records) throws Exception { - try (final PrintWriter writer = new PrintWriter(tmpFile, StandardCharsets.UTF_8); - final CSVPrinter csvPrinter = new CSVPrinter(writer, CSVFormat.DEFAULT)) { - for (final AirbyteRecordMessage record : records) { - final var uuid = UUID.randomUUID().toString(); - final var jsonData = Jsons.serialize(formatData(record.getData())); - final var emittedAt = Timestamp.from(Instant.ofEpochMilli(record.getEmittedAt())); - csvPrinter.printRecord(uuid, jsonData, emittedAt); - } - } - } - - protected JsonNode formatData(final JsonNode data) { - return data; - } - - @Override - public String truncateTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { - return String.format("TRUNCATE TABLE %s.%s;\n", schemaName, tableName); - } - - @Override - public String insertTableQuery(final JdbcDatabase database, final String schemaName, final String srcTableName, final String dstTableName) { - return String.format("INSERT INTO %s.%s SELECT * FROM %s.%s;\n", schemaName, dstTableName, schemaName, srcTableName); - } - - @Override - public void executeTransaction(final JdbcDatabase database, final List queries) throws Exception { - final StringBuilder appendedQueries = new StringBuilder(); - appendedQueries.append("BEGIN;\n"); - for (final String query : queries) { - appendedQueries.append(query); - } - appendedQueries.append("COMMIT;"); - database.execute(appendedQueries.toString()); - } - - @Override - public void dropTableIfExists(final JdbcDatabase database, final String schemaName, final String tableName) throws SQLException { - try { - database.execute(dropTableIfExistsQuery(schemaName, tableName)); - } catch (final SQLException e) { - throw checkForKnownConfigExceptions(e).orElseThrow(() -> e); - } - } - - public String dropTableIfExistsQuery(final String schemaName, final String tableName) { - return String.format("DROP TABLE IF EXISTS %s.%s;\n", schemaName, tableName); - } - - @Override - public boolean isSchemaRequired() { - return true; - } - - @Override - public boolean isValidData(final JsonNode data) { - return true; - } - - @Override - public final void insertRecords(final JdbcDatabase database, - final List records, - final String schemaName, - final String tableName) - throws Exception { - dataAdapter.ifPresent(adapter -> records.forEach(airbyteRecordMessage -> adapter.adapt(airbyteRecordMessage.getData()))); - insertRecordsInternal(database, records, schemaName, tableName); - } - - protected abstract void insertRecordsInternal(JdbcDatabase database, - List records, - String schemaName, - String tableName) - throws Exception; - -} diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/WriteConfig.java b/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/WriteConfig.java deleted file mode 100644 index 5c3876eb601c..000000000000 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/WriteConfig.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.jdbc; - -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; - -/** - * Write configuration POJO (plain old java object) for all destinations extending - * {@link AbstractJdbcDestination}. - */ -public class WriteConfig { - - private final String streamName; - private final String namespace; - private final String outputSchemaName; - private final String tmpTableName; - private final String outputTableName; - private final DestinationSyncMode syncMode; - private final DateTime writeDatetime; - - public WriteConfig(final String streamName, - final String namespace, - final String outputSchemaName, - final String tmpTableName, - final String outputTableName, - final DestinationSyncMode syncMode) { - this(streamName, namespace, outputSchemaName, tmpTableName, outputTableName, syncMode, DateTime.now(DateTimeZone.UTC)); - } - - public WriteConfig(final String streamName, - final String namespace, - final String outputSchemaName, - final String tmpTableName, - final String outputTableName, - final DestinationSyncMode syncMode, - final DateTime writeDatetime) { - this.streamName = streamName; - this.namespace = namespace; - this.outputSchemaName = outputSchemaName; - this.tmpTableName = tmpTableName; - this.outputTableName = outputTableName; - this.syncMode = syncMode; - this.writeDatetime = writeDatetime; - } - - public String getStreamName() { - return streamName; - } - - public String getNamespace() { - return namespace; - } - - public String getTmpTableName() { - return tmpTableName; - } - - public String getOutputSchemaName() { - return outputSchemaName; - } - - public String getOutputTableName() { - return outputTableName; - } - - public DestinationSyncMode getSyncMode() { - return syncMode; - } - - public DateTime getWriteDatetime() { - return writeDatetime; - } - - @Override - public String toString() { - return "WriteConfig{" + - "streamName=" + streamName + - ", namespace=" + namespace + - ", outputSchemaName=" + outputSchemaName + - ", tmpTableName=" + tmpTableName + - ", outputTableName=" + outputTableName + - ", syncMode=" + syncMode + - '}'; - } - -} diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/azure/AzureBlobStorageConfig.java b/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/azure/AzureBlobStorageConfig.java deleted file mode 100644 index bc72328cd23b..000000000000 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/jdbc/copy/azure/AzureBlobStorageConfig.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.jdbc.copy.azure; - -import com.fasterxml.jackson.databind.JsonNode; -import java.util.Locale; - -public class AzureBlobStorageConfig { - - private static final String DEFAULT_STORAGE_ENDPOINT_DOMAIN_NAME = "blob.core.windows.net"; - - private final String endpointDomainName; - private final String accountName; - private final String containerName; - private final String sasToken; - - public AzureBlobStorageConfig( - String endpointDomainName, - String accountName, - String containerName, - String sasToken) { - this.endpointDomainName = endpointDomainName; - this.accountName = accountName; - this.containerName = containerName; - this.sasToken = sasToken; - } - - public String getEndpointDomainName() { - return endpointDomainName == null ? DEFAULT_STORAGE_ENDPOINT_DOMAIN_NAME : endpointDomainName; - } - - public String getAccountName() { - return accountName; - } - - public String getContainerName() { - return containerName; - } - - public String getSasToken() { - return sasToken; - } - - public String getEndpointUrl() { - return String.format(Locale.ROOT, "https://%s.%s", getAccountName(), getEndpointDomainName()); - } - - public static AzureBlobStorageConfig getAzureBlobConfig(JsonNode config) { - - return new AzureBlobStorageConfig( - config.get("azure_blob_storage_endpoint_domain_name") == null ? DEFAULT_STORAGE_ENDPOINT_DOMAIN_NAME - : config.get("azure_blob_storage_endpoint_domain_name").asText(), - config.get("azure_blob_storage_account_name").asText(), - config.get("azure_blob_storage_container_name").asText(), - config.get("azure_blob_storage_sas_token").asText()); - - } - -} diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/GeneralStagingFunctions.java b/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/GeneralStagingFunctions.java deleted file mode 100644 index 301ee7d649f2..000000000000 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/GeneralStagingFunctions.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.staging; - -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeOperationValve; -import io.airbyte.integrations.base.destination.typing_deduping.TyperDeduper; -import io.airbyte.integrations.destination.buffered_stream_consumer.OnCloseFunction; -import io.airbyte.integrations.destination.buffered_stream_consumer.OnStartFunction; -import io.airbyte.integrations.destination.jdbc.WriteConfig; -import java.util.ArrayList; -import java.util.List; -import lombok.extern.slf4j.Slf4j; - -/** - * Functions and logic common to all flushing strategies. - */ -@Slf4j -public class GeneralStagingFunctions { - - public static OnStartFunction onStartFunction(final JdbcDatabase database, - final StagingOperations stagingOperations, - final List writeConfigs, - final TyperDeduper typerDeduper) { - return () -> { - log.info("Preparing raw tables in destination started for {} streams", writeConfigs.size()); - typerDeduper.prepareTables(); - final List queryList = new ArrayList<>(); - for (final WriteConfig writeConfig : writeConfigs) { - final String schema = writeConfig.getOutputSchemaName(); - final String stream = writeConfig.getStreamName(); - final String dstTableName = writeConfig.getOutputTableName(); - final String stageName = stagingOperations.getStageName(schema, stream); - final String stagingPath = - stagingOperations.getStagingPath(StagingConsumerFactory.RANDOM_CONNECTION_ID, schema, stream, writeConfig.getWriteDatetime()); - - log.info("Preparing staging area in destination started for schema {} stream {}: target table: {}, stage: {}", - schema, stream, dstTableName, stagingPath); - - stagingOperations.createSchemaIfNotExists(database, schema); - stagingOperations.createTableIfNotExists(database, schema, dstTableName); - stagingOperations.createStageIfNotExists(database, stageName); - - /* - * When we're in OVERWRITE, clear out the table at the start of a sync, this is an expected side - * effect of checkpoint and the removal of temporary tables - */ - switch (writeConfig.getSyncMode()) { - case OVERWRITE -> queryList.add(stagingOperations.truncateTableQuery(database, schema, dstTableName)); - case APPEND, APPEND_DEDUP -> {} - default -> throw new IllegalStateException("Unrecognized sync mode: " + writeConfig.getSyncMode()); - } - - log.info("Preparing staging area in destination completed for schema {} stream {}", schema, stream); - } - log.info("Executing finalization of tables."); - stagingOperations.executeTransaction(database, queryList); - }; - } - - /** - * Handles copying data from staging area to destination table and clean up of staged files if - * upload was unsuccessful - */ - public static void copyIntoTableFromStage(final JdbcDatabase database, - final String stageName, - final String stagingPath, - final List stagedFiles, - final String tableName, - final String schemaName, - final StagingOperations stagingOperations, - final String streamNamespace, - final String streamName, - final TypeAndDedupeOperationValve typerDeduperValve, - final TyperDeduper typerDeduper) - throws Exception { - try { - stagingOperations.copyIntoTableFromStage(database, stageName, stagingPath, stagedFiles, - tableName, schemaName); - } catch (final Exception e) { - stagingOperations.cleanUpStage(database, stageName, stagedFiles); - log.info("Cleaning stage path {}", stagingPath); - throw new RuntimeException("Failed to upload data from stage " + stagingPath, e); - } - } - - /** - * Tear down process, will attempt to try to clean out any staging area - * - * @param database database used for syncing - * @param stagingOperations collection of SQL queries necessary for writing data into a staging area - * @param writeConfigs configuration settings for all destination connectors needed to write - * @param purgeStagingData drop staging area if true, keep otherwise - * @return - */ - public static OnCloseFunction onCloseFunction(final JdbcDatabase database, - final StagingOperations stagingOperations, - final List writeConfigs, - final boolean purgeStagingData, - final TyperDeduper typerDeduper) { - return (hasFailed) -> { - // After moving data from staging area to the target table (airybte_raw) clean up the staging - // area (if user configured) - log.info("Cleaning up destination started for {} streams", writeConfigs.size()); - for (final WriteConfig writeConfig : writeConfigs) { - final String schemaName = writeConfig.getOutputSchemaName(); - if (purgeStagingData) { - final String stageName = stagingOperations.getStageName(schemaName, writeConfig.getStreamName()); - log.info("Cleaning stage in destination started for stream {}. schema {}, stage: {}", writeConfig.getStreamName(), schemaName, - stageName); - stagingOperations.dropStageIfExists(database, stageName); - } - - typerDeduper.typeAndDedupe(writeConfig.getNamespace(), writeConfig.getStreamName()); - } - - typerDeduper.commitFinalTables(); - log.info("Cleaning up destination completed."); - }; - } - -} diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/StagingConsumerFactory.java b/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/StagingConsumerFactory.java deleted file mode 100644 index 4334e90a1d2d..000000000000 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/StagingConsumerFactory.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.staging; - -import static java.util.stream.Collectors.joining; -import static java.util.stream.Collectors.toList; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.base.Preconditions; -import io.airbyte.commons.exceptions.ConfigErrorException; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.SerializedAirbyteMessageConsumer; -import io.airbyte.integrations.base.TypingAndDedupingFlag; -import io.airbyte.integrations.base.destination.typing_deduping.ParsedCatalog; -import io.airbyte.integrations.base.destination.typing_deduping.StreamId; -import io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeOperationValve; -import io.airbyte.integrations.base.destination.typing_deduping.TyperDeduper; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.buffered_stream_consumer.BufferedStreamConsumer; -import io.airbyte.integrations.destination.jdbc.WriteConfig; -import io.airbyte.integrations.destination.record_buffer.BufferCreateFunction; -import io.airbyte.integrations.destination.record_buffer.SerializedBufferingStrategy; -import io.airbyte.integrations.destination_async.AsyncStreamConsumer; -import io.airbyte.integrations.destination_async.buffers.BufferManager; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteStream; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.StreamDescriptor; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.function.Consumer; -import java.util.function.Function; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Uses both Factory and Consumer design pattern to create a single point of creation for consuming - * {@link AirbyteMessage} for processing - */ -public class StagingConsumerFactory { - - private static final Logger LOGGER = LoggerFactory.getLogger(StagingConsumerFactory.class); - - // using a random string here as a placeholder for the moment. - // This would avoid mixing data in the staging area between different syncs (especially if they - // manipulate streams with similar names) - // if we replaced the random connection id by the actual connection_id, we'd gain the opportunity to - // leverage data that was uploaded to stage - // in a previous attempt but failed to load to the warehouse for some reason (interrupted?) instead. - // This would also allow other programs/scripts - // to load (or reload backups?) in the connection's staging area to be loaded at the next sync. - private static final DateTime SYNC_DATETIME = DateTime.now(DateTimeZone.UTC); - public static final UUID RANDOM_CONNECTION_ID = UUID.randomUUID(); - - public AirbyteMessageConsumer create(final Consumer outputRecordCollector, - final JdbcDatabase database, - final StagingOperations stagingOperations, - final NamingConventionTransformer namingResolver, - final BufferCreateFunction onCreateBuffer, - final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final boolean purgeStagingData, - final TypeAndDedupeOperationValve typerDeduperValve, - final TyperDeduper typerDeduper, - final ParsedCatalog parsedCatalog, - final String defaultNamespace) { - final List writeConfigs = createWriteConfigs(namingResolver, config, catalog, parsedCatalog); - return new BufferedStreamConsumer( - outputRecordCollector, - GeneralStagingFunctions.onStartFunction(database, stagingOperations, writeConfigs, typerDeduper), - new SerializedBufferingStrategy( - onCreateBuffer, - catalog, - SerialFlush.function(database, stagingOperations, writeConfigs, catalog, typerDeduperValve, typerDeduper)), - GeneralStagingFunctions.onCloseFunction(database, stagingOperations, writeConfigs, purgeStagingData, typerDeduper), - catalog, - stagingOperations::isValidData, - defaultNamespace); - } - - public SerializedAirbyteMessageConsumer createAsync(final Consumer outputRecordCollector, - final JdbcDatabase database, - final StagingOperations stagingOperations, - final NamingConventionTransformer namingResolver, - final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final boolean purgeStagingData, - final TypeAndDedupeOperationValve typerDeduperValve, - final TyperDeduper typerDeduper, - final ParsedCatalog parsedCatalog, - final String defaultNamespace) { - final List writeConfigs = createWriteConfigs(namingResolver, config, catalog, parsedCatalog); - final var streamDescToWriteConfig = streamDescToWriteConfig(writeConfigs); - final var flusher = new AsyncFlush(streamDescToWriteConfig, stagingOperations, database, catalog, typerDeduperValve, typerDeduper); - return new AsyncStreamConsumer( - outputRecordCollector, - GeneralStagingFunctions.onStartFunction(database, stagingOperations, writeConfigs, typerDeduper), - // todo (cgardens) - wrapping the old close function to avoid more code churn. - () -> GeneralStagingFunctions.onCloseFunction(database, stagingOperations, writeConfigs, purgeStagingData, typerDeduper).accept(false), - flusher, - catalog, - new BufferManager(), - defaultNamespace); - } - - private static Map streamDescToWriteConfig(final List writeConfigs) { - final Set conflictingStreams = new HashSet<>(); - final Map streamDescToWriteConfig = new HashMap<>(); - for (final WriteConfig config : writeConfigs) { - final StreamDescriptor streamIdentifier = toStreamDescriptor(config); - if (streamDescToWriteConfig.containsKey(streamIdentifier)) { - conflictingStreams.add(config); - final WriteConfig existingConfig = streamDescToWriteConfig.get(streamIdentifier); - // The first conflicting stream won't have any problems, so we need to explicitly add it here. - conflictingStreams.add(existingConfig); - } else { - streamDescToWriteConfig.put(streamIdentifier, config); - } - } - if (!conflictingStreams.isEmpty()) { - final String message = String.format( - "You are trying to write multiple streams to the same table. Consider switching to a custom namespace format using ${SOURCE_NAMESPACE}, or moving one of them into a separate connection with a different stream prefix. Affected streams: %s", - conflictingStreams.stream().map(config -> config.getNamespace() + "." + config.getStreamName()).collect(joining(", "))); - throw new ConfigErrorException(message); - } - return streamDescToWriteConfig; - } - - private static StreamDescriptor toStreamDescriptor(final WriteConfig config) { - return new StreamDescriptor().withName(config.getStreamName()).withNamespace(config.getNamespace()); - } - - /** - * Creates a list of all {@link WriteConfig} for each stream within a - * {@link ConfiguredAirbyteCatalog}. Each write config represents the configuration settings for - * writing to a destination connector - * - * @param namingResolver {@link NamingConventionTransformer} used to transform names that are - * acceptable by each destination connector - * @param config destination connector configuration parameters - * @param catalog {@link ConfiguredAirbyteCatalog} collection of configured - * {@link ConfiguredAirbyteStream} - * @return list of all write configs for each stream in a {@link ConfiguredAirbyteCatalog} - */ - private static List createWriteConfigs(final NamingConventionTransformer namingResolver, - final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final ParsedCatalog parsedCatalog) { - - return catalog.getStreams().stream().map(toWriteConfig(namingResolver, config, parsedCatalog)).collect(toList()); - } - - private static Function toWriteConfig(final NamingConventionTransformer namingResolver, - final JsonNode config, - final ParsedCatalog parsedCatalog) { - return stream -> { - Preconditions.checkNotNull(stream.getDestinationSyncMode(), "Undefined destination sync mode"); - final AirbyteStream abStream = stream.getStream(); - final String streamName = abStream.getName(); - - final String outputSchema; - final String tableName; - if (TypingAndDedupingFlag.isDestinationV2()) { - final StreamId streamId = parsedCatalog.getStream(abStream.getNamespace(), streamName).id(); - outputSchema = streamId.rawNamespace(); - tableName = streamId.rawName(); - } else { - outputSchema = getOutputSchema(abStream, config.get("schema").asText(), namingResolver); - tableName = namingResolver.getRawTableName(streamName); - } - final String tmpTableName = namingResolver.getTmpTableName(streamName); - final DestinationSyncMode syncMode = stream.getDestinationSyncMode(); - - final WriteConfig writeConfig = - new WriteConfig(streamName, abStream.getNamespace(), outputSchema, tmpTableName, tableName, syncMode, SYNC_DATETIME); - LOGGER.info("Write config: {}", writeConfig); - - return writeConfig; - }; - } - - private static String getOutputSchema(final AirbyteStream stream, - final String defaultDestSchema, - final NamingConventionTransformer namingResolver) { - return stream.getNamespace() != null - ? namingResolver.getNamespace(stream.getNamespace()) - : namingResolver.getNamespace(defaultDestSchema); - } - -} diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/StagingOperations.java b/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/StagingOperations.java deleted file mode 100644 index 4eae42d04e23..000000000000 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/main/java/io/airbyte/integrations/destination/staging/StagingOperations.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.staging; - -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.record_buffer.SerializableBuffer; -import java.util.List; -import java.util.UUID; -import org.joda.time.DateTime; - -/** - * Staging operations focuses on the SQL queries that are needed to success move data into a staging - * environment like GCS or S3. In general, the reference of staging is the usage of an object - * storage for the purposes of efficiently uploading bulk data to destinations - */ -public interface StagingOperations extends SqlOperations { - - /** - * Returns the staging environment's name - * - * @param namespace Name of schema - * @param streamName Name of the stream - * @return Fully qualified name of the staging environment - */ - String getStageName(String namespace, String streamName); - - String getStagingPath(UUID connectionId, String namespace, String streamName, DateTime writeDatetime); - - /** - * Create a staging folder where to upload temporary files before loading into the final destination - */ - void createStageIfNotExists(JdbcDatabase database, String stageName) throws Exception; - - /** - * Upload the data file into the stage area. - * - * @param database database used for syncing - * @param recordsData records stored in in-memory buffer - * @param schemaName name of schema - * @param stageName name of the staging area folder - * @param stagingPath path of staging folder to data files - * @return the name of the file that was uploaded. - */ - String uploadRecordsToStage(JdbcDatabase database, SerializableBuffer recordsData, String schemaName, String stageName, String stagingPath) - throws Exception; - - /** - * Load the data stored in the stage area into a temporary table in the destination - * - * @param database database interface - * @param stageName name of staging area folder - * @param stagingPath path to staging files - * @param stagedFiles collection of staged files - * @param tableName name of table to write staging files to - * @param schemaName name of schema - */ - void copyIntoTableFromStage(JdbcDatabase database, - String stageName, - String stagingPath, - List stagedFiles, - String tableName, - String schemaName) - throws Exception; - - /** - * Remove files that were just staged - * - * @param database database used for syncing - * @param stageName name of staging area folder - * @param stagedFiles collection of the staging files to remove - */ - void cleanUpStage(JdbcDatabase database, String stageName, List stagedFiles) throws Exception; - - /** - * Delete the stage area and all staged files that was in it - * - * @param database database used for syncing - * @param stageName Name of the staging area used to store files - */ - void dropStageIfExists(JdbcDatabase database, String stageName) throws Exception; - -} diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/TestJdbcSqlOperations.java b/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/TestJdbcSqlOperations.java deleted file mode 100644 index 9a27038cd698..000000000000 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/jdbc/TestJdbcSqlOperations.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.jdbc; - -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import java.sql.SQLException; -import java.util.List; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -public class TestJdbcSqlOperations extends JdbcSqlOperations { - - @Override - public void insertRecordsInternal(final JdbcDatabase database, - final List records, - final String schemaName, - final String tableName) - throws Exception { - // Not required for the testing - } - - @Test - public void testCreateSchemaIfNotExists() { - final JdbcDatabase db = Mockito.mock(JdbcDatabase.class); - final var schemaName = "foo"; - try { - Mockito.doThrow(new SQLException("TEST")).when(db).execute(Mockito.anyString()); - } catch (Exception e) { - // This would not be expected, but the `execute` method above will flag as an unhandled exception - assert false; - } - SQLException exception = Assertions.assertThrows(SQLException.class, () -> createSchemaIfNotExists(db, schemaName)); - Assertions.assertEquals(exception.getMessage(), "TEST"); - } - -} diff --git a/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/staging/StagingConsumerFactoryTest.java b/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/staging/StagingConsumerFactoryTest.java deleted file mode 100644 index 7d3f76f43119..000000000000 --- a/airbyte-integrations/bases/bases-destination-jdbc/src/test/java/io/airbyte/integrations/destination/staging/StagingConsumerFactoryTest.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.staging; - -import static org.junit.jupiter.api.Assertions.*; - -import io.airbyte.commons.exceptions.ConfigErrorException; -import io.airbyte.integrations.destination.jdbc.WriteConfig; -import java.util.List; -import org.junit.jupiter.api.Test; - -class StagingConsumerFactoryTest { - - @Test() - void detectConflictingStreams() { - final ConfigErrorException configErrorException = assertThrows( - ConfigErrorException.class, - () -> SerialFlush.function( - null, - null, - List.of( - new WriteConfig("example_stream", "source_schema", "destination_default_schema", null, null, null), - new WriteConfig("example_stream", "source_schema", "destination_default_schema", null, null, null)), - null, - null, - null)); - - assertEquals( - "You are trying to write multiple streams to the same table. Consider switching to a custom namespace format using ${SOURCE_NAMESPACE}, or moving one of them into a separate connection with a different stream prefix. Affected streams: source_schema.example_stream, source_schema.example_stream", - configErrorException.getMessage()); - } - -} diff --git a/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md b/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md index 993653cff454..407a1754e01c 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md +++ b/airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md @@ -1,5 +1,54 @@ # Changelog +## 3.0.1 +Upgrade to Dagger 0.9.6 + +## 3.0.0 +Upgrade to Dagger 0.9.5 + +## 2.2.0 +Add connector_attribute test suite and stream primary key validation + +## 2.1.4 + +Add check to ensure stream schemas are valid json schemas + +## 2.1.3 + +Remove dockerfile and migrations tools. We are now running CAT with `airbyte-ci` and new migration operation will be done by the `airbyte-ci` tool. + +## 2.1.2 + +Add check for duplicate stream names + +## 2.1.1 + +Centralize timeout duration declaration and increase timeout for discover. + +## 2.1.0 + +Make the container under test a sessions scoped fixture. +Support loading it from its Dagger container id for better performance. +Install pytest-xdist to support running tests in parallel. + +## 2.0.2 +Make `test_two_sequential_reads` handle namespace property in stream descriptor. + +## 2.0.1 +Changing `format` or `airbyte_type` in a field definition of a schema or specification is now a breaking change. + +## 2.0.0 +Update test_incremental.test_two_sequential_reads to be unaware of the contents of the state message. This is to support connectors that have a custom implementation of a cursor. + +## 1.0.4 +Fix edge case in skip_backward_compatibility_tests_fixture on discovery: if the current config structure is not compatible with the previous connector version, the discovery command failing and the previous connector version catalog could not be retrieved. + +## 1.0.3 +Add tests for display_type property + +## 1.0.2 +Fix bug in skip_backward_compatibility_tests_fixture, the previous connector version could not be retrieved. + ## 1.0.1 Pin airbyte-protocol-model to <1.0.0. @@ -323,4 +372,3 @@ Add test whether PKs present and not None if `source_defined_primary_key` define ## 0.1.5 Add configurable timeout for the acceptance tests: [#4296](https://github.com/airbytehq/airbyte/pull/4296) - diff --git a/airbyte-integrations/bases/connector-acceptance-test/Dockerfile b/airbyte-integrations/bases/connector-acceptance-test/Dockerfile index d70d45c824d7..34cfaa0b2689 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/Dockerfile +++ b/airbyte-integrations/bases/connector-acceptance-test/Dockerfile @@ -6,7 +6,7 @@ ENV DOCKER_VERSION = "24.0.2" RUN apt-get update \ && pip install --upgrade pip \ - && apt-get install tzdata bash curl + && apt-get -y install tzdata bash curl # Docker is required for the dagger in docker use case. RUN curl -fsSL https://get.docker.com | sh @@ -14,7 +14,7 @@ RUN curl -fsSL https://get.docker.com | sh RUN pip install poetry==1.5.1 -RUN poetry config virtualenvs.create false +RUN poetry config virtualenvs.create false RUN echo "Etc/UTC" > /etc/timezone WORKDIR /app @@ -24,7 +24,6 @@ RUN poetry install --no-root --only main --no-interaction --no-ansi COPY . /app RUN poetry install --only main --no-cache --no-interaction --no-ansi -LABEL io.airbyte.version=1.0.1 -LABEL io.airbyte.name=airbyte/connector-acceptance-test + WORKDIR /test_input ENTRYPOINT ["python", "-m", "pytest", "-p", "connector_acceptance_test.plugin", "-r", "fEsx", "--show-capture=log"] diff --git a/airbyte-integrations/bases/connector-acceptance-test/README.md b/airbyte-integrations/bases/connector-acceptance-test/README.md index ec0671e4dbf6..3b33b204b512 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/README.md +++ b/airbyte-integrations/bases/connector-acceptance-test/README.md @@ -37,7 +37,7 @@ To learn how to set up `ci_credentials` and your GSM Service account see [here]( export GCP_GSM_CREDENTIALS=`cat ` # Install the credentials tool -pipx install airbyte-ci/connectors/ci_credentials/ +pipx install airbyte-ci/connectors/ci_credentials/ --force --editable ``` **Retrieve a connectors sandbox secrets** @@ -66,28 +66,11 @@ poetry install poetry run pytest -p connector_acceptance_test.plugin --acceptance-test-config=../../connectors/source-faker --pdb ``` -### Running CAT via the production docker image (deprecated) -This is the old method and is not useful outside of helping third party connector developers run their tests. - -Ideally you should use `airbyte-ci` as described above. - -_Note: To use `FETCH_SECRETS=1` you must have `ci_credentials` and your GSM Service account setup see [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/ci_credentials/README.md)_ - - -```bash -# Navigate to the connectors folder -cd airbyte-integrations/connectors/source-faker - -# Run the tests -FETCH_SECRETS=1 ./acceptance-test-docker.sh -``` - -Note you can also use `LOCAL_CDK=1` to run tests against the local python CDK, if relevant. If not set, tests against the latest package published to pypi, or the version specified in the connector's setup.py. ### Manually 1. `cd` into your connector project (e.g. `airbyte-integrations/connectors/source-pokeapi`) 2. Edit `acceptance-test-config.yml` according to your need. Please refer to our [Connector Acceptance Test Reference](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference/) if you need details about the available options. -3. Build the connector docker image ( e.g.: `docker build . -t airbyte/source-pokeapi:dev`) +3. Build the connector docker image ( e.g.: `airbyte-ci connectors --name=source-pokeapi build`) 4. Use one of the following ways to run tests (**from your connector project directory**) @@ -96,21 +79,23 @@ You may want to iterate on the acceptance test project itself: adding new tests, These iterations are more conveniently achieved by remaining in the current directory. 1. Install dependencies via `poetry install` -3. Run the unit tests on the acceptance tests themselves: `poetry run python -m pytest unit_tests` (add the `--pdb` option if you want to enable the debugger on test failure) +2. Run the unit tests on the acceptance tests themselves: `poetry run pytest unit_tests` (add the `--pdb` option if you want to enable the debugger on test failure) +3. To run specific unit test(s), add `-k` to the above command, e.g. `poetry run python -m pytest unit_tests -k 'test_property_can_store_secret'`. You can use wildcards `*` here as well. 4. Make the changes you want: * Global pytest fixtures are defined in `./connector_acceptance_test/conftest.py` * Existing test modules are defined in `./connector_acceptance_test/tests` * `acceptance-test-config.yaml` structure is defined in `./connector_acceptance_test/config.py` 5. Unit test your changes by adding tests to `./unit_tests` 6. Run the unit tests on the acceptance tests again: `poetry run pytest unit_tests`, make sure the coverage did not decrease. You can bypass slow tests by using the `slow` marker: `poetry run pytest unit_tests -m "not slow"`. -7. Manually test the changes you made by running acceptance tests on a specific connector. e.g. `poetry run pytest -p connector_acceptance_test.plugin --acceptance-test-config=../../connectors/source-pokeapi` +7. Manually test the changes you made by running acceptance tests on a specific connector: + * First build the connector to ensure your local image is up-to-date: `airbyte-ci connectors --name=source-pokeapi build` + * Then run the acceptance tests on the connector: `poetry run pytest -p connector_acceptance_test.plugin --acceptance-test-config=../../connectors/source-pokeapi` 8. Make sure you updated `docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md` according to your changes -9. Bump the acceptance test docker image version in `airbyte-integrations/bases/connector-acceptance-test/Dockerfile` -10. Update the project changelog `airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md` -11. Open a PR on our GitHub repository -12. This [Github action workflow](https://github.com/airbytehq/airbyte/blob/master/.github/workflows/cat-tests.yml) will be triggered an run the unit tests on your branch. -13. Publish the new acceptance test version if your PR is approved by running `/legacy-publish connector=bases/connector-acceptance-test run-tests=false` in a GitHub comment -14. Merge your PR +9. Update the project changelog `airbyte-integrations/bases/connector-acceptance-test/CHANGELOG.md` +10. Open a PR on our GitHub repository +11. This [Github action workflow](https://github.com/airbytehq/airbyte/blob/master/.github/workflows/cat-tests.yml) will be triggered an run the unit tests on your branch. +12. Publish the new acceptance test version if your PR is approved by running `/legacy-publish connector=bases/connector-acceptance-test run-tests=false` in a GitHub comment +13. Merge your PR ## Migrating `acceptance-test-config.yml` to latest configuration format We introduced changes in the structure of `acceptance-test-config.yml` files in version 0.2.12. diff --git a/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh b/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh deleted file mode 100755 index e599e8549b5a..000000000000 --- a/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env sh - -set -e - -ROOT_DIR="$(git rev-parse --show-toplevel)" -source "$ROOT_DIR/airbyte-integrations/scripts/utils.sh" - -CONFIG_PATH="$(readlink_f acceptance-test-config.yml)" - -[ -n "$CONFIG_PATH" ] || die "Missing CONFIG_PATH" - -CONNECTOR_TAG_BASE="$(grep connector_image $CONFIG_PATH | head -n 1 | cut -d: -f2 | sed 's/^ *//')" -CONNECTOR_TAG="$CONNECTOR_TAG_BASE:dev" -CONNECTOR_NAME="$(echo $CONNECTOR_TAG_BASE | cut -d / -f 2)" -CONNECTOR_DIR="$ROOT_DIR/airbyte-integrations/connectors/$CONNECTOR_NAME" - -if [ -n "$FETCH_SECRETS" ]; then - cd $ROOT_DIR - pip install pipx - pipx ensurepath - pipx install airbyte-ci/connectors/ci_credentials - VERSION=dev ci_credentials $CONNECTOR_NAME write-to-storage || true - cd - -fi - -if [ -n "$LOCAL_CDK" ] && [ -f "$CONNECTOR_DIR/setup.py" ]; then - echo "Building Connector image with local CDK from $ROOT_DIR/airbyte-cdk" - echo "Building docker image $CONNECTOR_TAG." - CONNECTOR_NAME="$CONNECTOR_NAME" CONNECTOR_TAG="$CONNECTOR_TAG" QUIET_BUILD="$QUIET_BUILD" sh "$ROOT_DIR/airbyte-integrations/scripts/build-connector-image-with-local-cdk.sh" -else - # Build latest connector image - docker build -t "$CONNECTOR_TAG" . -fi - -# Pull latest acctest image -docker pull airbyte/connector-acceptance-test:latest - -# Run -docker run --rm \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /tmp:/tmp \ - -v "$CONNECTOR_DIR":/test_input \ - airbyte/connector-acceptance-test \ - --acceptance-test-config /test_input - diff --git a/airbyte-integrations/bases/connector-acceptance-test/build.gradle b/airbyte-integrations/bases/connector-acceptance-test/build.gradle deleted file mode 100644 index 55c378f71fb7..000000000000 --- a/airbyte-integrations/bases/connector-acceptance-test/build.gradle +++ /dev/null @@ -1,6 +0,0 @@ -plugins { - // airbyte-docker is kept to support /legacy-publish until airbyte-ci cat publish command exists. - id 'airbyte-docker' -} - - diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py index b79b939d958b..045e23671b74 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/config.py @@ -7,7 +7,7 @@ from copy import deepcopy from enum import Enum from pathlib import Path -from typing import Generic, List, Mapping, Optional, Set, TypeVar, Union +from typing import Generic, List, Mapping, Optional, Set, TypeVar from pydantic import BaseModel, Field, root_validator, validator from pydantic.generics import GenericModel @@ -19,6 +19,7 @@ ) configured_catalog_path: Optional[str] = Field(default=None, description="Path to configured catalog") timeout_seconds: int = Field(default=None, description="Test execution timeout_seconds", ge=0) +deployment_mode: Optional[str] = Field(default=None, description="Deployment mode to run the test in", regex=r"^(cloud|oss)$") SEMVER_REGEX = r"(0|(?:[1-9]\d*))(?:\.(0|(?:[1-9]\d*))(?:\.(0|(?:[1-9]\d*)))?(?:\-([\w][\w\.\-_]*))?)?" ALLOW_LEGACY_CONFIG = True @@ -45,6 +46,7 @@ class SpecTestConfig(BaseConfig): spec_path: str = spec_path config_path: str = config_path timeout_seconds: int = timeout_seconds + deployment_mode: Optional[str] = deployment_mode backward_compatibility_tests_config: BackwardCompatibilityTestsConfig = Field( description="Configuration for the backward compatibility tests.", default=BackwardCompatibilityTestsConfig() ) @@ -59,11 +61,13 @@ class Status(Enum): config_path: str = config_path status: Status = Field(Status.Succeed, description="Indicate if connection check should succeed with provided config") timeout_seconds: int = timeout_seconds + deployment_mode: Optional[str] = deployment_mode class DiscoveryTestConfig(BaseConfig): config_path: str = config_path timeout_seconds: int = timeout_seconds + deployment_mode: Optional[str] = deployment_mode backward_compatibility_tests_config: BackwardCompatibilityTestsConfig = Field( description="Configuration for the backward compatibility tests.", default=BackwardCompatibilityTestsConfig() ) @@ -120,8 +124,14 @@ class IgnoredFieldsConfiguration(BaseConfig): ) +class NoPrimaryKeyConfiguration(BaseConfig): + name: str + bypass_reason: Optional[str] = Field(default=None, description="Reason why this stream does not support a primary key") + + class BasicReadTestConfig(BaseConfig): config_path: str = config_path + deployment_mode: Optional[str] = deployment_mode configured_catalog_path: Optional[str] = configured_catalog_path empty_streams: Set[EmptyStreamConfiguration] = Field( default_factory=set, description="We validate that all streams has records. These are exceptions" @@ -148,6 +158,7 @@ class FullRefreshConfig(BaseConfig): config_path: str = config_path configured_catalog_path: Optional[str] = configured_catalog_path timeout_seconds: int = timeout_seconds + deployment_mode: Optional[str] = deployment_mode ignored_fields: Optional[Mapping[str, List[IgnoredFieldsConfiguration]]] = ignored_fields @@ -160,16 +171,9 @@ class FutureStateConfig(BaseConfig): class IncrementalConfig(BaseConfig): config_path: str = config_path configured_catalog_path: Optional[str] = configured_catalog_path - cursor_paths: Optional[Mapping[str, List[Union[int, str]]]] = Field( - description="For each stream, the path of its cursor field in the output state messages." - ) future_state: Optional[FutureStateConfig] = Field(description="Configuration for the future state.") timeout_seconds: int = timeout_seconds - threshold_days: int = Field( - description="Allow records to be emitted with a cursor value this number of days before the state cursor", - default=0, - ge=0, - ) + deployment_mode: Optional[str] = deployment_mode skip_comprehensive_incremental_tests: Optional[bool] = Field( description="Determines whether to skip more granular testing for incremental syncs", default=False ) @@ -178,6 +182,23 @@ class Config: smart_union = True +class ConnectorAttributesConfig(BaseConfig): + """ + Config that is used to verify that a connector and its streams uphold certain behavior and features that are + required to maintain enterprise-level standard of quality. + + Attributes: + streams_without_primary_key: A list of streams where a primary key is not available from the API or is not relevant to the record + """ + + timeout_seconds: int = timeout_seconds + config_path: str = config_path + + streams_without_primary_key: Optional[List[NoPrimaryKeyConfiguration]] = Field( + description="Streams that do not support a primary key such as reports streams" + ) + + class GenericTestConfig(GenericModel, Generic[TestConfigT]): bypass_reason: Optional[str] tests: Optional[List[TestConfigT]] @@ -196,6 +217,7 @@ class AcceptanceTestConfigurations(BaseConfig): basic_read: Optional[GenericTestConfig[BasicReadTestConfig]] full_refresh: Optional[GenericTestConfig[FullRefreshConfig]] incremental: Optional[GenericTestConfig[IncrementalConfig]] + connector_attributes: Optional[GenericTestConfig[ConnectorAttributesConfig]] class Config(BaseConfig): diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py index 2d25cba6fd29..f0df880650fa 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/conftest.py @@ -22,10 +22,10 @@ from connector_acceptance_test.config import Config, EmptyStreamConfiguration, ExpectedRecordsConfig, IgnoredFieldsConfiguration from connector_acceptance_test.tests import TestBasicRead from connector_acceptance_test.utils import ( - ConnectorRunner, SecretDict, build_configured_catalog_from_custom_catalog, build_configured_catalog_from_discovered_catalog_and_empty_streams, + connector_runner, filter_output, load_config, load_yaml_or_json_path, @@ -56,6 +56,11 @@ def custom_environment_variables_fixture(acceptance_test_config: Config) -> Mapp return acceptance_test_config.custom_environment_variables +@pytest.fixture(name="deployment_mode") +def deployment_mode_fixture(inputs) -> Optional[str]: + return getattr(inputs, "deployment_mode", None) + + @pytest.fixture(name="connector_config_path") def connector_config_path_fixture(inputs, base_path) -> Path: """Fixture with connector's config path. The path to the latest updated configurations will be returned if any.""" @@ -107,7 +112,7 @@ def configured_catalog_fixture( return build_configured_catalog_from_discovered_catalog_and_empty_streams(discovered_catalog, set()) -@pytest.fixture(name="image_tag") +@pytest.fixture(name="image_tag", scope="session") def image_tag_fixture(acceptance_test_config) -> str: return acceptance_test_config.connector_image @@ -161,16 +166,24 @@ async def dagger_client(anyio_backend): yield client +@pytest.fixture(scope="session") +async def connector_container(dagger_client, image_tag): + connector_container = await connector_runner.get_connector_container(dagger_client, image_tag) + if cachebuster := os.environ.get("CACHEBUSTER"): + connector_container = connector_container.with_env_variable("CACHEBUSTER", cachebuster) + return await connector_container + + @pytest.fixture(name="docker_runner", autouse=True) -async def docker_runner_fixture(image_tag, connector_config_path, custom_environment_variables, dagger_client) -> ConnectorRunner: - runner = ConnectorRunner( - image_tag, - dagger_client, +def docker_runner_fixture( + connector_container, connector_config_path, custom_environment_variables, deployment_mode +) -> connector_runner.ConnectorRunner: + return connector_runner.ConnectorRunner( + connector_container, connector_configuration_path=connector_config_path, custom_environment_variables=custom_environment_variables, + deployment_mode=deployment_mode, ) - await runner.load_container() - return runner @pytest.fixture(name="previous_connector_image_name") @@ -179,15 +192,26 @@ def previous_connector_image_name_fixture(image_tag, inputs) -> str: return f"{image_tag.split(':')[0]}:{inputs.backward_compatibility_tests_config.previous_connector_version}" +@pytest.fixture() +async def previous_version_connector_container( + dagger_client, + previous_connector_image_name, +): + connector_container = await connector_runner.get_connector_container(dagger_client, previous_connector_image_name) + if cachebuster := os.environ.get("CACHEBUSTER"): + connector_container = connector_container.with_env_variable("CACHEBUSTER", cachebuster) + return await connector_container + + @pytest.fixture(name="previous_connector_docker_runner") -async def previous_connector_docker_runner_fixture(previous_connector_image_name, dagger_client) -> ConnectorRunner: +async def previous_connector_docker_runner_fixture( + previous_version_connector_container, deployment_mode +) -> connector_runner.ConnectorRunner: """Fixture to create a connector runner with the previous connector docker image. Returns None if the latest image was not found, to skip downstream tests if the current connector is not yet published to the docker registry. Raise not found error if the previous connector image is not latest and expected to be published. """ - runner = ConnectorRunner(previous_connector_image_name, dagger_client) - await runner.load_container() - return runner + return connector_runner.ConnectorRunner(previous_version_connector_container, deployment_mode=deployment_mode) @pytest.fixture(name="empty_streams") @@ -269,7 +293,7 @@ def find_not_seeded_streams( @pytest.fixture(name="discovered_catalog") async def discovered_catalog_fixture( connector_config, - docker_runner: ConnectorRunner, + docker_runner: connector_runner.ConnectorRunner, ) -> MutableMapping[str, AirbyteStream]: """JSON schemas for each stream""" @@ -282,7 +306,7 @@ async def discovered_catalog_fixture( async def previous_discovered_catalog_fixture( connector_config, previous_connector_image_name, - previous_connector_docker_runner: ConnectorRunner, + previous_connector_docker_runner: connector_runner.ConnectorRunner, ) -> MutableMapping[str, AirbyteStream]: """JSON schemas for each stream""" if previous_connector_docker_runner is None: @@ -323,7 +347,7 @@ def detailed_logger() -> Logger: @pytest.fixture(name="actual_connector_spec") -async def actual_connector_spec_fixture(docker_runner: ConnectorRunner) -> ConnectorSpecification: +async def actual_connector_spec_fixture(docker_runner: connector_runner.ConnectorRunner) -> ConnectorSpecification: output = await docker_runner.call_spec() spec_messages = filter_output(output, Type.SPEC) assert len(spec_messages) == 1, "Spec message should be emitted exactly once" @@ -332,7 +356,7 @@ async def actual_connector_spec_fixture(docker_runner: ConnectorRunner) -> Conne @pytest.fixture(name="previous_connector_spec") async def previous_connector_spec_fixture( - request: BaseTest, previous_connector_docker_runner: ConnectorRunner + request: BaseTest, previous_connector_docker_runner: connector_runner.ConnectorRunner ) -> Optional[ConnectorSpecification]: if previous_connector_docker_runner is None: logging.warning( diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/__init__.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/__init__.py index cc93282d5660..be9bde14f32c 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/__init__.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/__init__.py @@ -2,8 +2,8 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # -from .test_core import TestBasicRead, TestConnection, TestDiscovery, TestSpec +from .test_core import TestBasicRead, TestConnection, TestConnectorAttributes, TestDiscovery, TestSpec from .test_full_refresh import TestFullRefresh from .test_incremental import TestIncremental -__all__ = ["TestSpec", "TestBasicRead", "TestConnection", "TestDiscovery", "TestFullRefresh", "TestIncremental"] +__all__ = ["TestSpec", "TestBasicRead", "TestConnection", "TestConnectorAttributes", "TestDiscovery", "TestFullRefresh", "TestIncremental"] diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py index b0fb44c41fdb..7ced702152db 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_core.py @@ -32,10 +32,12 @@ BasicReadTestConfig, Config, ConnectionTestConfig, + ConnectorAttributesConfig, DiscoveryTestConfig, EmptyStreamConfiguration, ExpectedRecordsConfig, IgnoredFieldsConfiguration, + NoPrimaryKeyConfiguration, SpecTestConfig, ) from connector_acceptance_test.utils import ConnectorRunner, SecretDict, delete_fields, filter_output, make_hashable, verify_records_schema @@ -54,6 +56,11 @@ get_object_structure, get_paths_in_connector_config, ) +from connector_acceptance_test.utils.timeouts import FIVE_MINUTES, ONE_MINUTE, TEN_MINUTES + +pytestmark = [ + pytest.mark.anyio, +] @pytest.fixture(name="connector_spec_dict") @@ -88,11 +95,12 @@ def secret_property_names_fixture(): DATETIME_PATTERN = "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2})?$" -# The connector fixture can be long to load, we have to increase the default timeout... -@pytest.mark.default_timeout(5 * 60) +# Running tests in parallel can sometime delay the execution of the tests if downstream services are not able to handle the load. +# This is why we set a timeout on tests that call command that should return quickly, like spec +@pytest.mark.default_timeout(ONE_MINUTE) class TestSpec(BaseTest): @pytest.fixture(name="skip_backward_compatibility_tests") - def skip_backward_compatibility_tests_fixture( + async def skip_backward_compatibility_tests_fixture( self, inputs: SpecTestConfig, previous_connector_docker_runner: ConnectorRunner, @@ -106,7 +114,7 @@ def skip_backward_compatibility_tests_fixture( pytest.skip("The previous connector image could not be retrieved.") # Get the real connector version in case 'latest' is used in the config: - previous_connector_version = previous_connector_docker_runner._image.labels.get("io.airbyte.version") + previous_connector_version = await previous_connector_docker_runner.get_container_label("io.airbyte.version") if previous_connector_version == inputs.backward_compatibility_tests_config.disable_for_version: pytest.skip(f"Backward compatibility tests are disabled for version {previous_connector_version}.") @@ -455,27 +463,26 @@ def test_nested_group(self, actual_connector_spec: ConnectorSpecification): errors.append(f"Groups can only be defined on top level, is defined at {group_path}") self._fail_on_errors(errors) - def test_required_always_show(self, actual_connector_spec: ConnectorSpecification): + def test_display_type(self, actual_connector_spec: ConnectorSpecification): """ - Fields with always_show are not allowed to be required fields because only optional fields can be hidden in the form in the first place. + The display_type property can only be set on fields which have a oneOf property, and must be either "dropdown" or "radio" """ errors = [] schema_helper = JsonSchemaHelper(actual_connector_spec.connectionSpecification) - for result in dpath.util.search(actual_connector_spec.connectionSpecification, "/properties/**/always_show", yielded=True): - always_show_path = result[0] - parent_path = schema_helper.get_parent_path(always_show_path) - is_property_named_always_show = parent_path.endswith("properties") - if is_property_named_always_show: + for result in dpath.util.search(actual_connector_spec.connectionSpecification, "/properties/**/display_type", yielded=True): + display_type_path = result[0] + parent_path = schema_helper.get_parent_path(display_type_path) + is_property_named_display_type = parent_path.endswith("properties") + if is_property_named_display_type: continue - property_name = parent_path.rsplit(sep="/", maxsplit=1)[1] - properties_path = schema_helper.get_parent_path(parent_path) - parent_object = schema_helper.get_parent(properties_path) - if ( - "required" in parent_object - and isinstance(parent_object.get("required"), List) - and property_name in parent_object.get("required") - ): - errors.append(f"always_show is only allowed on optional properties, but is set on {always_show_path}") + parent_object = schema_helper.get_parent(display_type_path) + if "oneOf" not in parent_object: + errors.append(f"display_type is only allowed on fields which have a oneOf property, but is set on {parent_path}") + display_type_value = parent_object.get("display_type") + if display_type_value != "dropdown" and display_type_value != "radio": + errors.append( + f"display_type must be either 'dropdown' or 'radio', but is set to '{display_type_value}' at {display_type_path}" + ) self._fail_on_errors(errors) def test_defined_refs_exist_in_json_spec_file(self, connector_spec_dict: dict): @@ -514,7 +521,7 @@ def test_oauth_flow_parameters(self, actual_connector_spec: ConnectorSpecificati diff = paths_to_validate - set(get_expected_schema_structure(spec_schema)) assert diff == set(), f"Specified oauth fields are missed from spec schema: {diff}" - @pytest.mark.default_timeout(60) + @pytest.mark.default_timeout(ONE_MINUTE) @pytest.mark.backward_compatibility def test_backward_compatibility( self, @@ -564,7 +571,7 @@ async def test_image_environment_variables(self, docker_runner: ConnectorRunner) assert await docker_runner.get_container_env_variable_value("AIRBYTE_ENTRYPOINT") == await docker_runner.get_container_entrypoint() -@pytest.mark.default_timeout(30) +@pytest.mark.default_timeout(ONE_MINUTE) class TestConnection(BaseTest): async def test_check(self, connector_config, inputs: ConnectionTestConfig, docker_runner: ConnectorRunner): if inputs.status == ConnectionTestConfig.Status.Succeed: @@ -589,9 +596,10 @@ async def test_check(self, connector_config, inputs: ConnectionTestConfig, docke assert trace.error.message is not None -@pytest.mark.default_timeout(30) +# Running tests in parallel can sometime delay the execution of the tests if downstream services are not able to handle the load. +# This is why we set a timeout on tests that call command that should return quickly, like discover +@pytest.mark.default_timeout(FIVE_MINUTES) class TestDiscovery(BaseTest): - VALID_TYPES = {"null", "string", "number", "integer", "boolean", "object", "array"} VALID_AIRBYTE_TYPES = {"timestamp_with_timezone", "timestamp_without_timezone", "integer"} VALID_FORMATS = {"date-time", "date"} @@ -611,10 +619,21 @@ class TestDiscovery(BaseTest): ({"number", "null"}, "integer"), ] + @pytest.fixture() + async def skip_backward_compatibility_tests_for_version( + self, inputs: DiscoveryTestConfig, previous_connector_docker_runner: ConnectorRunner + ): + # Get the real connector version in case 'latest' is used in the config: + previous_connector_version = await previous_connector_docker_runner.get_container_label("io.airbyte.version") + if previous_connector_version == inputs.backward_compatibility_tests_config.disable_for_version: + pytest.skip(f"Backward compatibility tests are disabled for version {previous_connector_version}.") + return False + @pytest.fixture(name="skip_backward_compatibility_tests") - def skip_backward_compatibility_tests_fixture( + async def skip_backward_compatibility_tests_fixture( self, - inputs: DiscoveryTestConfig, + # Even if unused, this fixture is required to make sure that the skip_backward_compatibility_tests_for_version fixture is called. + skip_backward_compatibility_tests_for_version: bool, previous_connector_docker_runner: ConnectorRunner, discovered_catalog: MutableMapping[str, AirbyteStream], previous_discovered_catalog: MutableMapping[str, AirbyteStream], @@ -625,21 +644,31 @@ def skip_backward_compatibility_tests_fixture( if previous_connector_docker_runner is None: pytest.skip("The previous connector image could not be retrieved.") - # Get the real connector version in case 'latest' is used in the config: - previous_connector_version = previous_connector_docker_runner._image.labels.get("io.airbyte.version") - - if previous_connector_version == inputs.backward_compatibility_tests_config.disable_for_version: - pytest.skip(f"Backward compatibility tests are disabled for version {previous_connector_version}.") return False async def test_discover(self, connector_config, docker_runner: ConnectorRunner): """Verify that discover produce correct schema.""" output = await docker_runner.call_discover(config=connector_config) catalog_messages = filter_output(output, Type.CATALOG) + duplicated_stream_names = self.duplicated_stream_names(catalog_messages[0].catalog.streams) assert len(catalog_messages) == 1, "Catalog message should be emitted exactly once" assert catalog_messages[0].catalog, "Message should have catalog" assert catalog_messages[0].catalog.streams, "Catalog should contain streams" + assert len(duplicated_stream_names) == 0, f"Catalog should have uniquely named streams, duplicates are: {duplicated_stream_names}" + + def duplicated_stream_names(self, streams) -> List[str]: + """Counts number of times a stream appears in the catalog""" + name_counts = dict() + for stream in streams: + count = name_counts.get(stream.name, 0) + name_counts[stream.name] = count + 1 + return [k for k, v in name_counts.items() if v > 1] + + def test_streams_have_valid_json_schemas(self, discovered_catalog: Mapping[str, Any]): + """Check if all stream schemas are valid json schemas.""" + for stream_name, stream in discovered_catalog.items(): + jsonschema.Draft7Validator.check_schema(stream.json_schema) def test_defined_cursors_exist_in_schema(self, discovered_catalog: Mapping[str, Any]): """Check if all of the source defined cursor fields are exists on stream's json schema.""" @@ -704,7 +733,7 @@ def test_additional_properties_is_true(self, discovered_catalog: Mapping[str, An [additional_properties_value is True for additional_properties_value in additional_properties_values] ), "When set, additionalProperties field value must be true for backward compatibility." - @pytest.mark.default_timeout(60) + @pytest.mark.default_timeout(ONE_MINUTE) @pytest.mark.backward_compatibility def test_backward_compatibility( self, @@ -777,7 +806,7 @@ def primary_keys_for_records(streams, records): yield pk_values, stream_record -@pytest.mark.default_timeout(10 * 60) +@pytest.mark.default_timeout(TEN_MINUTES) class TestBasicRead(BaseTest): @staticmethod def _validate_records_structure(records: List[AirbyteRecordMessage], configured_catalog: ConfiguredAirbyteCatalog): @@ -1085,9 +1114,17 @@ def compare_records( missing_expected = set(expected) - set(actual) if missing_expected: + extra = set(actual) - set(expected) msg = f"Stream {stream_name}: All expected records must be produced" detailed_logger.info(msg) - detailed_logger.log_json_list(missing_expected) + detailed_logger.info("missing:") + detailed_logger.log_json_list(sorted(missing_expected, key=lambda record: str(record.get("ID", "0")))) + detailed_logger.info("expected:") + detailed_logger.log_json_list(sorted(expected, key=lambda record: str(record.get("ID", "0")))) + detailed_logger.info("actual:") + detailed_logger.log_json_list(sorted(actual, key=lambda record: str(record.get("ID", "0")))) + detailed_logger.info("extra:") + detailed_logger.log_json_list(sorted(extra, key=lambda record: str(record.get("ID", "0")))) pytest.fail(msg) if not extra_records: @@ -1106,3 +1143,35 @@ def group_by_stream(records: List[AirbyteRecordMessage]) -> MutableMapping[str, result[record.stream].append(record.data) return result + + +@pytest.mark.default_timeout(TEN_MINUTES) +class TestConnectorAttributes(BaseTest): + MANDATORY_FOR_TEST_STRICTNESS_LEVELS = [] # Used so that this is not part of the mandatory high strictness test suite yet + + @pytest.fixture(name="operational_certification_test") + async def operational_certification_test_fixture(self, connector_metadata: dict) -> bool: + """ + Fixture that is used to skip a test that is reserved only for connectors that are supposed to be tested + against operational certification criteria + """ + + if connector_metadata.get("data", {}).get("ab_internal", {}).get("ql") < 400: + pytest.skip("Skipping operational connector certification test for uncertified connector") + return True + + @pytest.fixture(name="streams_without_primary_key") + def streams_without_primary_key_fixture(self, inputs: ConnectorAttributesConfig) -> List[NoPrimaryKeyConfiguration]: + return inputs.streams_without_primary_key or [] + + async def test_streams_define_primary_key( + self, operational_certification_test, streams_without_primary_key, connector_config, docker_runner: ConnectorRunner + ): + output = await docker_runner.call_discover(config=connector_config) + catalog_messages = filter_output(output, Type.CATALOG) + streams = catalog_messages[0].catalog.streams + discovered_streams_without_primary_key = {stream.name for stream in streams if not stream.source_defined_primary_key} + missing_primary_keys = discovered_streams_without_primary_key - {stream.name for stream in streams_without_primary_key} + + quoted_missing_primary_keys = {f"'{primary_key}'" for primary_key in missing_primary_keys} + assert not missing_primary_keys, f"The following streams {', '.join(quoted_missing_primary_keys)} do not define a primary_key" diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_full_refresh.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_full_refresh.py index d98e8d8e778c..103bd8b3aed8 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_full_refresh.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_full_refresh.py @@ -14,8 +14,7 @@ from connector_acceptance_test.config import IgnoredFieldsConfiguration from connector_acceptance_test.utils import ConnectorRunner, JsonSchemaHelper, SecretDict, full_refresh_only_catalog, make_hashable from connector_acceptance_test.utils.json_schema_helper import CatalogField - -# from airbyte_pr import ConfiguredAirbyteCatalog, Type +from connector_acceptance_test.utils.timeouts import TWENTY_MINUTES def primary_keys_by_stream(configured_catalog: ConfiguredAirbyteCatalog) -> Mapping[str, List[CatalogField]]: @@ -37,7 +36,7 @@ def primary_keys_only(record, pks): return ";".join([f"{pk.path}={pk.parse(record)}" for pk in pks]) -@pytest.mark.default_timeout(20 * 60) +@pytest.mark.default_timeout(TWENTY_MINUTES) class TestFullRefresh(BaseTest): def assert_emitted_at_increase_on_subsequent_runs(self, first_read_records, second_read_records): first_read_records_data = [record.data for record in first_read_records] @@ -100,8 +99,8 @@ async def test_sequential_reads( ) records_1 = [message.record for message in output_1 if message.type == Type.RECORD] - # sleep for 1 second to ensure that the emitted_at timestamp is different - time.sleep(1) + # sleep to ensure that the emitted_at timestamp is different + time.sleep(0.1) output_2 = await docker_runner.call_read(connector_config, configured_catalog, enable_caching=False) records_2 = [message.record for message in output_2 if message.type == Type.RECORD] diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_incremental.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_incremental.py index 4fc301ecb8cd..2f0231644228 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_incremental.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/tests/test_incremental.py @@ -3,16 +3,16 @@ # import json -from datetime import datetime from pathlib import Path -from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Tuple, Union +from typing import Any, Dict, List, Mapping, MutableMapping, Tuple, Union -import pendulum import pytest from airbyte_protocol.models import AirbyteMessage, AirbyteStateMessage, AirbyteStateType, ConfiguredAirbyteCatalog, SyncMode, Type from connector_acceptance_test import BaseTest from connector_acceptance_test.config import Config, EmptyStreamConfiguration, IncrementalConfig -from connector_acceptance_test.utils import ConnectorRunner, JsonSchemaHelper, SecretDict, filter_output, incremental_only_catalog +from connector_acceptance_test.utils import ConnectorRunner, SecretDict, filter_output, incremental_only_catalog +from connector_acceptance_test.utils.timeouts import TWENTY_MINUTES +from deepdiff import DeepDiff @pytest.fixture(name="future_state_configuration") @@ -53,18 +53,6 @@ def future_state_fixture(future_state_configuration, test_strictness_level, conf return states -@pytest.fixture(name="cursor_paths") -def cursor_paths_fixture(inputs, configured_catalog_for_incremental) -> Mapping[str, Any]: - cursor_paths = getattr(inputs, "cursor_paths") or {} - result = {} - - for stream in configured_catalog_for_incremental.streams: - path = cursor_paths.get(stream.stream.name, [stream.cursor_field[-1]]) - result[stream.stream.name] = path - - return result - - @pytest.fixture(name="configured_catalog_for_incremental") def configured_catalog_for_incremental_fixture(configured_catalog) -> ConfiguredAirbyteCatalog: catalog = incremental_only_catalog(configured_catalog) @@ -83,53 +71,6 @@ def configured_catalog_for_incremental_fixture(configured_catalog) -> Configured return catalog -def records_with_state(records, state, stream_mapping, state_cursor_paths) -> Iterable[Tuple[Any, Any, Any]]: - """Iterate over records and return cursor value with corresponding cursor value from state""" - for record in records: - stream_name = record.record.stream - stream = stream_mapping[stream_name] - helper = JsonSchemaHelper(schema=stream.stream.json_schema) - cursor_field = helper.field(stream.cursor_field) - record_value = cursor_field.parse(record=record.record.data) - try: - if state[stream_name] is None: - continue - - # first attempt to parse the state value assuming the state object is namespaced on stream names - state_value = cursor_field.parse(record=state[stream_name], path=state_cursor_paths[stream_name]) - except KeyError: - try: - # try second time as an absolute path in state file (i.e. bookmarks -> stream_name -> column -> value) - state_value = cursor_field.parse(record=state, path=state_cursor_paths[stream_name]) - except KeyError: - continue - yield record_value, state_value, stream_name - - -def compare_cursor_with_threshold(record_value, state_value, threshold_days: int) -> bool: - """ - Checks if the record's cursor value is older or equal to the state cursor value. - - If the threshold_days option is set, the values will be converted to dates so that the time-based offset can be applied. - :raises: pendulum.parsing.exceptions.ParserError: if threshold_days is passed with non-date cursor values. - """ - if threshold_days: - - def _parse_date_value(value) -> datetime: - if isinstance(value, datetime): - return value - if isinstance(value, (int, float)): - return pendulum.from_timestamp(value / 1000) - return pendulum.parse(value, strict=False) - - record_date_value = _parse_date_value(record_value) - state_date_value = _parse_date_value(state_value) - - return record_date_value >= (state_date_value - pendulum.duration(days=threshold_days)) - - return record_value >= state_value - - def is_per_stream_state(message: AirbyteMessage) -> bool: return message.state and isinstance(message.state, AirbyteStateMessage) and message.state.type == AirbyteStateType.STREAM @@ -148,59 +89,76 @@ def construct_latest_state_from_messages(messages: List[AirbyteMessage]) -> Dict return latest_per_stream_by_name -@pytest.mark.default_timeout(20 * 60) +def naive_diff_records(records_1: List[AirbyteMessage], records_2: List[AirbyteMessage]) -> DeepDiff: + """ + Naively diff two lists of records by comparing their data field. + """ + records_1_data = [record.record.data for record in records_1] + records_2_data = [record.record.data for record in records_2] + + # ignore_order=True because the order of records in the list is not guaranteed + diff = DeepDiff(records_1_data, records_2_data, ignore_order=True) + return diff + + +@pytest.mark.default_timeout(TWENTY_MINUTES) class TestIncremental(BaseTest): async def test_two_sequential_reads( self, - inputs: IncrementalConfig, connector_config: SecretDict, configured_catalog_for_incremental: ConfiguredAirbyteCatalog, - cursor_paths: dict[str, list[Union[int, str]]], docker_runner: ConnectorRunner, ): - threshold_days = getattr(inputs, "threshold_days") or 0 - stream_mapping = {stream.stream.name: stream for stream in configured_catalog_for_incremental.streams} + """ + This test makes two calls to the read method and verifies that the records returned are different. - output = await docker_runner.call_read(connector_config, configured_catalog_for_incremental) - records_1 = filter_output(output, type_=Type.RECORD) - states_1 = filter_output(output, type_=Type.STATE) + Important! - assert states_1, "Should produce at least one state" - assert records_1, "Should produce at least one record" + Assert only that the reads are different. Nothing else. + This is because there is only a small subset of assertions we can make + in the absense of enforcing that all connectors return 3 or more state messages + during the first read. + + To learn more: https://github.com/airbytehq/airbyte/issues/29926 + """ + output_1 = await docker_runner.call_read(connector_config, configured_catalog_for_incremental) + records_1 = filter_output(output_1, type_=Type.RECORD) + states_1 = filter_output(output_1, type_=Type.STATE) + + assert states_1, "First Read should produce at least one state" + assert records_1, "First Read should produce at least one record" # For legacy state format, the final state message contains the final state of all streams. For per-stream state format, # the complete final state of streams must be assembled by going through all prior state messages received - if is_per_stream_state(states_1[-1]): + is_per_stream = is_per_stream_state(states_1[-1]) + if is_per_stream: latest_state = construct_latest_state_from_messages(states_1) - state_input = list( - {"type": "STREAM", "stream": {"stream_descriptor": {"name": stream_name}, "stream_state": stream_state}} - for stream_name, stream_state in latest_state.items() - ) + state_input = [] + for stream_name, stream_state in latest_state.items(): + stream_descriptor = {"name": stream_name} + if "stream_namespace" in stream_state: + stream_descriptor["namespace"] = stream_state["stream_namespace"] + state_input.append( + { + "type": "STREAM", + "stream": {"stream_descriptor": stream_descriptor, "stream_state": stream_state}, + } + ) else: - latest_state = states_1[-1].state.data state_input = states_1[-1].state.data - parsed_records_1 = list(records_with_state(records_1, latest_state, stream_mapping, cursor_paths)) - - # This catches the case of a connector that emits an invalid state that is not compatible with the schema - # See https://github.com/airbytehq/airbyte/issues/21863 to understand more - assert parsed_records_1, "At least one valid state should be produced, given a cursor path" - - for record_value, state_value, stream_name in parsed_records_1: - assert ( - record_value <= state_value - ), f"First incremental sync should produce records younger or equal to cursor value from the state. Stream: {stream_name}" + # READ #2 - output = await docker_runner.call_read_with_state(connector_config, configured_catalog_for_incremental, state=state_input) - records_2 = filter_output(output, type_=Type.RECORD) + output_2 = await docker_runner.call_read_with_state(connector_config, configured_catalog_for_incremental, state=state_input) + records_2 = filter_output(output_2, type_=Type.RECORD) - for record_value, state_value, stream_name in records_with_state(records_2, latest_state, stream_mapping, cursor_paths): - assert compare_cursor_with_threshold( - record_value, state_value, threshold_days - ), f"Second incremental sync should produce records older or equal to cursor value from the state. Stream: {stream_name}" + diff = naive_diff_records(records_1, records_2) + assert ( + diff + ), f"Records should change between reads but did not.\n\n records_1: {records_1} \n\n state: {state_input} \n\n records_2: {records_2} \n\n diff: {diff}" async def test_read_sequential_slices( - self, inputs: IncrementalConfig, connector_config, configured_catalog_for_incremental, cursor_paths, docker_runner: ConnectorRunner + self, inputs: IncrementalConfig, connector_config, configured_catalog_for_incremental, docker_runner: ConnectorRunner ): """ Incremental test that makes calls to the read method without a state checkpoint. Then we partition the results by stream and @@ -212,52 +170,70 @@ async def test_read_sequential_slices( pytest.skip("Skipping new incremental test based on acceptance-test-config.yml") return - threshold_days = getattr(inputs, "threshold_days") or 0 - stream_mapping = {stream.stream.name: stream for stream in configured_catalog_for_incremental.streams} + output_1 = await docker_runner.call_read(connector_config, configured_catalog_for_incremental) + records_1 = filter_output(output_1, type_=Type.RECORD) + states_1 = filter_output(output_1, type_=Type.STATE) + + # We sometimes have duplicate identical state messages in a stream which we can filter out to speed things up + unique_state_messages = [message for index, message in enumerate(states_1) if message not in states_1[:index]] - output = await docker_runner.call_read(connector_config, configured_catalog_for_incremental) - records_1 = filter_output(output, type_=Type.RECORD) - states_1 = filter_output(output, type_=Type.STATE) + # Important! - assert states_1, "Should produce at least one state" - assert records_1, "Should produce at least one record" + # There is only a small subset of assertions we can make + # in the absense of enforcing that all connectors return 3 or more state messages + # during the first read. + + # To learn more: https://github.com/airbytehq/airbyte/issues/29926 + if len(unique_state_messages) < 3: + pytest.skip("Skipping test because there are not enough state messages to test with") + return + + assert records_1, "First Read should produce at least one record" # For legacy state format, the final state message contains the final state of all streams. For per-stream state format, # the complete final state of streams must be assembled by going through all prior state messages received is_per_stream = is_per_stream_state(states_1[-1]) - if is_per_stream: - latest_state = construct_latest_state_from_messages(states_1) - else: - latest_state = states_1[-1].state.data - for record_value, state_value, stream_name in records_with_state(records_1, latest_state, stream_mapping, cursor_paths): - assert ( - record_value <= state_value - ), f"First incremental sync should produce records younger or equal to cursor value from the state. Stream: {stream_name}" + # To avoid spamming APIs we only test a fraction of batches (10%) and enforce a minimum of 10 tested + min_batches_to_test = 5 + sample_rate = len(unique_state_messages) // min_batches_to_test - checkpoint_messages = filter_output(output, type_=Type.STATE) + mutating_stream_name_to_per_stream_state = dict() + for idx, state_message in enumerate(unique_state_messages): + assert state_message.type == Type.STATE - # We sometimes have duplicate identical state messages in a stream which we can filter out to speed things up - checkpoint_messages = [message for index, message in enumerate(checkpoint_messages) if message not in checkpoint_messages[:index]] + # if first state message, skip + # this is because we cannot assert if the first state message will result in new records + # as in this case it is possible for a connector to return an empty state message when it first starts. + # e.g. if the connector decides it wants to let the caller know that it has started with an empty state. + if idx == 0: + continue - # To avoid spamming APIs we only test a fraction of batches (10%) and enforce a minimum of 10 tested - min_batches_to_test = 10 - sample_rate = len(checkpoint_messages) // min_batches_to_test - stream_name_to_per_stream_state = dict() - for idx, state_message in enumerate(checkpoint_messages): - assert state_message.type == Type.STATE - state_input, complete_state = self.get_next_state_input(state_message, stream_name_to_per_stream_state, is_per_stream) + # if last state message, skip + # this is because we cannot assert if the last state message will result in new records + # as in this case it is possible for a connector to return a previous state message. + # e.g. if the connector is using pagination and the last page is only partially full + if idx == len(unique_state_messages) - 1: + continue - if len(checkpoint_messages) >= min_batches_to_test and idx % sample_rate != 0: + # if batching required, and not a sample, skip + if len(unique_state_messages) >= min_batches_to_test and idx % sample_rate != 0: continue - output = await docker_runner.call_read_with_state(connector_config, configured_catalog_for_incremental, state=state_input) - records = filter_output(output, type_=Type.RECORD) + state_input, mutating_stream_name_to_per_stream_state = self.get_next_state_input( + state_message, mutating_stream_name_to_per_stream_state, is_per_stream + ) - for record_value, state_value, stream_name in records_with_state(records, complete_state, stream_mapping, cursor_paths): - assert compare_cursor_with_threshold( - record_value, state_value, threshold_days - ), f"Second incremental sync should produce records older or equal to cursor value from the state. Stream: {stream_name}" + output_N = await docker_runner.call_read_with_state(connector_config, configured_catalog_for_incremental, state=state_input) + records_N = filter_output(output_N, type_=Type.RECORD) + assert ( + records_N + ), f"Read {idx + 2} of {len(unique_state_messages)} should produce at least one record.\n\n state: {state_input} \n\n records_{idx + 2}: {records_N}" + + diff = naive_diff_records(records_1, records_N) + assert ( + diff + ), f"Records for subsequent reads with new state should be different.\n\n records_1: {records_1} \n\n state: {state_input} \n\n records_{idx + 2}: {records_N} \n\n diff: {diff}" async def test_state_with_abnormally_large_values( self, connector_config, configured_catalog, future_state, docker_runner: ConnectorRunner diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/backward_compatibility.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/backward_compatibility.py index 13bcf9af0089..e91ee1935415 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/backward_compatibility.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/backward_compatibility.py @@ -51,14 +51,15 @@ def compute_diffs(self): # pragma: no cover def assert_is_backward_compatible(self): # pragma: no cover pass - def check_if_value_of_type_field_changed(self, diff: DeepDiff): - """Check if a type was changed on a property""" - # Detect type value change in case type field is declared as a string (e.g "str" -> "int"): - changes_on_property_type = [ - change for change in diff.get("values_changed", []) if {"properties", "type"}.issubset(change.path(output_format="list")) - ] - if changes_on_property_type: - self._raise_error("The'type' field value was changed.", diff) + def check_if_value_of_a_field_changed(self, diff: DeepDiff, field: str): + """ + Check if a type / airbyte_type / format was changed on a property. + Detect field value change: "str" -> "int" / "date-time" -> "date" / "timestamp_without_timezone" -> "timestamp_with_timezone" + """ + diffs = diff.get("values_changed", set()) | diff.get("dictionary_item_added", set()) | diff.get("dictionary_item_removed", set()) + field_value_changes = [change for change in diffs if {"properties", field}.issubset(change.path(output_format="list"))] + if field_value_changes: + self._raise_error(f"The '{field}' field value was changed.", diff) def check_if_new_type_was_added(self, diff: DeepDiff): # pragma: no cover """Detect type value added to type list if new type value is not None (e.g ["str"] -> ["str", "int"])""" @@ -129,7 +130,9 @@ def compute_diffs(self): def assert_is_backward_compatible(self): self.check_if_declared_new_required_field(self.connection_specification_diff) self.check_if_added_a_new_required_property(self.connection_specification_diff) - self.check_if_value_of_type_field_changed(self.connection_specification_diff) + self.check_if_value_of_a_field_changed(self.connection_specification_diff, "type") + self.check_if_value_of_a_field_changed(self.connection_specification_diff, "airbyte_type") + self.check_if_value_of_a_field_changed(self.connection_specification_diff, "format") # self.check_if_new_type_was_added(self.connection_specification_diff) We want to allow type expansion atm self.check_if_type_of_type_field_changed(self.connection_specification_diff, allow_type_widening=True) self.check_if_field_was_made_not_nullable(self.connection_specification_diff) @@ -257,8 +260,10 @@ def compute_diffs(self): def assert_is_backward_compatible(self): self.check_if_stream_was_removed(self.streams_json_schemas_diff) - self.check_if_value_of_type_field_changed(self.streams_json_schemas_diff) self.check_if_type_of_type_field_changed(self.streams_json_schemas_diff, allow_type_widening=False) + self.check_if_value_of_a_field_changed(self.streams_json_schemas_diff, "type") + self.check_if_value_of_a_field_changed(self.streams_json_schemas_diff, "format") + self.check_if_value_of_a_field_changed(self.streams_json_schemas_diff, "airbyte_type") self.check_if_field_removed(self.streams_json_schemas_diff) self.check_if_cursor_field_was_changed(self.streams_cursor_fields_diff) diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py index b54904ceeab4..0bf6810ea0ea 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/connector_runner.py @@ -8,12 +8,11 @@ import os import uuid from pathlib import Path -from typing import List, Mapping, Optional, Union +from typing import Any, List, Mapping, Optional, Union import dagger import docker import pytest -import yaml from airbyte_protocol.models import AirbyteMessage, ConfiguredAirbyteCatalog, OrchestratorType from airbyte_protocol.models import Type as AirbyteMessageType from anyio import Path as AnyioPath @@ -21,6 +20,106 @@ from pydantic import ValidationError +async def get_container_from_id(dagger_client: dagger.Client, container_id: str) -> dagger.Container: + """Get a dagger container from its id. + Please remind that container id are not persistent and can change between Dagger sessions. + + Args: + dagger_client (dagger.Client): The dagger client to use to import the connector image + """ + try: + return await dagger_client.container(id=dagger.ContainerID(container_id)) + except dagger.DaggerError as e: + pytest.exit(f"Failed to load connector container: {e}") + + +async def get_container_from_tarball_path(dagger_client: dagger.Client, tarball_path: Path): + if not tarball_path.exists(): + pytest.exit(f"Connector image tarball {tarball_path} does not exist") + container_under_test_tar_file = ( + dagger_client.host().directory(str(tarball_path.parent), include=tarball_path.name).file(tarball_path.name) + ) + try: + return await dagger_client.container().import_(container_under_test_tar_file) + except dagger.DaggerError as e: + pytest.exit(f"Failed to import connector image from tarball: {e}") + + +async def get_container_from_local_image(dagger_client: dagger.Client, local_image_name: str) -> Optional[dagger.Container]: + """Get a dagger container from a local image. + It will use Docker python client to export the image to a tarball and then import it into dagger. + + Args: + dagger_client (dagger.Client): The dagger client to use to import the connector image + local_image_name (str): The name of the local image to import + + Returns: + Optional[dagger.Container]: The dagger container for the local image or None if the image does not exist + """ + docker_client = docker.from_env() + + try: + image = docker_client.images.get(local_image_name) + except docker.errors.ImageNotFound: + return None + + image_digest = image.id.replace("sha256:", "") + tarball_path = Path(f"/tmp/{image_digest}.tar") + if not tarball_path.exists(): + logging.info(f"Exporting local connector image {local_image_name} to tarball {tarball_path}") + with open(tarball_path, "wb") as f: + for chunk in image.save(named=True): + f.write(chunk) + return await get_container_from_tarball_path(dagger_client, tarball_path) + + +async def get_container_from_dockerhub_image(dagger_client: dagger.Client, dockerhub_image_name: str) -> dagger.Container: + """Get a dagger container from a dockerhub image. + + Args: + dagger_client (dagger.Client): The dagger client to use to import the connector image + dockerhub_image_name (str): The name of the dockerhub image to import + + Returns: + dagger.Container: The dagger container for the dockerhub image + """ + try: + return await dagger_client.container().from_(dockerhub_image_name) + except dagger.DaggerError as e: + pytest.exit(f"Failed to import connector image from DockerHub: {e}") + + +async def get_connector_container(dagger_client: dagger.Client, image_name_with_tag: str) -> dagger.Container: + """Get a dagger container for the connector image to test. + + Args: + dagger_client (dagger.Client): The dagger client to use to import the connector image + image_name_with_tag (str): The docker image name and tag of the connector image to test + + Returns: + dagger.Container: The dagger container for the connector image to test + """ + # If a container_id.txt file is available, we'll use it to load the connector container + # We use a txt file as container ids can be too long to be passed as env vars + # It's used for dagger-in-dagger use case with airbyte-ci, when the connector container is built via an upstream dagger operation + connector_container_id_path = Path("/tmp/container_id.txt") + if connector_container_id_path.exists(): + # If the CONNECTOR_CONTAINER_ID env var is set, we'll use it to load the connector container + return await get_container_from_id(dagger_client, connector_container_id_path.read_text()) + + # If the CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH env var is set, we'll use it to import the connector image from the tarball + if connector_image_tarball_path := os.environ.get("CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH"): + tarball_path = Path(connector_image_tarball_path) + return await get_container_from_tarball_path(dagger_client, tarball_path) + + # Let's try to load the connector container from a local image + if connector_container := await get_container_from_local_image(dagger_client, image_name_with_tag): + return connector_container + + # If we get here, we'll try to pull the connector image from DockerHub + return await get_container_from_dockerhub_image(dagger_client, image_name_with_tag) + + class ConnectorRunner: IN_CONTAINER_CONFIG_PATH = "/data/config.json" IN_CONTAINER_CATALOG_PATH = "/data/catalog.json" @@ -29,33 +128,49 @@ class ConnectorRunner: def __init__( self, - image_tag: str, - dagger_client: dagger.Client, + connector_container: dagger.Container, connector_configuration_path: Optional[Path] = None, custom_environment_variables: Optional[Mapping] = {}, + deployment_mode: Optional[str] = None, ): - self._check_connector_under_test() - self.image_tag = image_tag - self.dagger_client = dagger_client + env_vars = ( + custom_environment_variables + if deployment_mode is None + else {**custom_environment_variables, "DEPLOYMENT_MODE": deployment_mode.upper()} + ) + self._connector_under_test_container = self.set_env_vars(connector_container, env_vars) self._connector_configuration_path = connector_configuration_path - self._custom_environment_variables = custom_environment_variables - connector_image_tarball_path = self._get_connector_image_tarball_path() - self._connector_under_test_container = self._get_connector_container(connector_image_tarball_path) - async def load_container(self): - """This is to pre-load the container following instantiation of the class. - This is useful to make sure that when using the connector runner fixture the costly _import is already done. + def set_env_vars(self, container: dagger.Container, env_vars: Mapping[str, Any]) -> dagger.Container: + """Set environment variables on a dagger container. + + Args: + container (dagger.Container): The dagger container to set the environment variables on. + env_vars (Mapping[str, str]): The environment variables to set. + + Returns: + dagger.Container: The dagger container with the environment variables set. """ - await self._connector_under_test_container.with_exec(["spec"]) + for k, v in env_vars.items(): + container = container.with_env_variable(k, str(v)) + return container async def call_spec(self, raise_container_error=False) -> List[AirbyteMessage]: return await self._run(["spec"], raise_container_error) async def call_check(self, config: SecretDict, raise_container_error: bool = False) -> List[AirbyteMessage]: - return await self._run(["check", "--config", self.IN_CONTAINER_CONFIG_PATH], raise_container_error, config=config) + return await self._run( + ["check", "--config", self.IN_CONTAINER_CONFIG_PATH], + raise_container_error, + config=config, + ) async def call_discover(self, config: SecretDict, raise_container_error: bool = False) -> List[AirbyteMessage]: - return await self._run(["discover", "--config", self.IN_CONTAINER_CONFIG_PATH], raise_container_error, config=config) + return await self._run( + ["discover", "--config", self.IN_CONTAINER_CONFIG_PATH], + raise_container_error, + config=config, + ) async def call_read( self, config: SecretDict, catalog: ConfiguredAirbyteCatalog, raise_container_error: bool = False, enable_caching: bool = True @@ -103,50 +218,6 @@ async def get_container_entrypoint(self): entrypoint = await self._connector_under_test_container.entrypoint() return " ".join(entrypoint) - def _get_connector_image_tarball_path(self) -> Optional[Path]: - if "CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH" not in os.environ and not self.image_tag.endswith(":dev"): - return None - if "CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH" in os.environ: - connector_under_test_image_tar_path = Path(os.environ["CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH"]) - elif self.image_tag.endswith(":dev"): - connector_under_test_image_tar_path = self._export_local_connector_image_to_tarball(self.image_tag) - assert connector_under_test_image_tar_path.exists(), "Connector image tarball does not exist" - return connector_under_test_image_tar_path - - def _export_local_connector_image_to_tarball(self, local_image_name: str) -> Optional[Path]: - tarball_path = Path("/tmp/connector_under_test_image.tar") - - docker_client = docker.from_env() - try: - image = docker_client.images.get(local_image_name) - with open(tarball_path, "wb") as f: - for chunk in image.save(named=True): - f.write(chunk) - - except docker.errors.ImageNotFound: - pytest.fail(f"Image {local_image_name} not found, please make sure to build or pull it before running the tests") - return tarball_path - - def _get_connector_container_from_tarball(self, tarball_path: Path) -> dagger.Container: - container_under_test_tar_file = ( - self.dagger_client.host().directory(str(tarball_path.parent), include=tarball_path.name).file(tarball_path.name) - ) - return self.dagger_client.container().import_(container_under_test_tar_file) - - def _get_connector_container(self, connector_image_tarball_path: Optional[Path]) -> dagger.Container: - if connector_image_tarball_path is not None: - container = self._get_connector_container_from_tarball(connector_image_tarball_path) - else: - # Try to pull the image from DockerHub - container = self.dagger_client.container().from_(self.image_tag) - # Client might pass a cachebuster env var to force recreation of the container - # We pass this env var to the container to ensure the cache is busted - if cachebuster_value := os.environ.get("CACHEBUSTER"): - container = container.with_env_variable("CACHEBUSTER", cachebuster_value) - for key, value in self._custom_environment_variables.items(): - container = container.with_env_variable(key, str(value)) - return container - async def _run( self, airbyte_command: List[str], @@ -156,7 +227,7 @@ async def _run( state: Union[dict, list] = None, enable_caching=True, ) -> List[AirbyteMessage]: - """_summary_ + """Run a command in the connector container and return the list of AirbyteMessages emitted by the connector. Args: airbyte_command (List[str]): The command to run in the connector container. @@ -166,23 +237,18 @@ async def _run( state (Union[dict, list], optional): The state to mount to the container. Defaults to None. enable_caching (bool, optional): Whether to enable command output caching. Defaults to True. - Raises: - e: _description_ - Returns: - List[AirbyteMessage]: _description_ + List[AirbyteMessage]: The list of AirbyteMessages emitted by the connector. """ container = self._connector_under_test_container if not enable_caching: container = container.with_env_variable("CAT_CACHEBUSTER", str(uuid.uuid4())) if config: - container = container.with_new_file(self.IN_CONTAINER_CONFIG_PATH, json.dumps(dict(config))) + container = container.with_new_file(self.IN_CONTAINER_CONFIG_PATH, contents=json.dumps(dict(config))) if state: - container = container.with_new_file(self.IN_CONTAINER_STATE_PATH, json.dumps(state)) + container = container.with_new_file(self.IN_CONTAINER_STATE_PATH, contents=json.dumps(state)) if catalog: - container = container.with_new_file(self.IN_CONTAINER_CATALOG_PATH, catalog.json()) - for key, value in self._custom_environment_variables.items(): - container = container.with_env_variable(key, str(value)) + container = container.with_new_file(self.IN_CONTAINER_CATALOG_PATH, contents=catalog.json()) try: output = await self._read_output_from_stdout(airbyte_command, container) except dagger.QueryError as e: @@ -259,16 +325,3 @@ def _persist_new_configuration(self, new_configuration: dict, configuration_emit json.dump(new_configuration, new_configuration_file) logging.info(f"Stored most recent configuration value to {new_configuration_file_path}") return new_configuration_file_path - - def _check_connector_under_test(self): - """ - As a safety measure, we check that the connector under test matches the connector being tested by comparing the content of the metadata.yaml file to the CONNECTOR_UNDER_TEST_TECHNICAL_NAME environment varialbe. - When running CAT from airbyte-ci we set this CONNECTOR_UNDER_TEST_TECHNICAL_NAME env var name, - This is a safety check to ensure the correct test inputs are mounted to the CAT container. - """ - if connector_under_test_technical_name := os.environ.get("CONNECTOR_UNDER_TEST_TECHNICAL_NAME"): - metadata = yaml.safe_load(Path("/test_input/metadata.yaml").read_text()) - assert metadata["data"]["dockerRepository"] == f"airbyte/{connector_under_test_technical_name}", ( - f"Connector under test env var {connector_under_test_technical_name} does not match the connector " - f"being tested {metadata['data']['dockerRepository']}" - ) diff --git a/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/timeouts.py b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/timeouts.py new file mode 100644 index 000000000000..dea43ddc0223 --- /dev/null +++ b/airbyte-integrations/bases/connector-acceptance-test/connector_acceptance_test/utils/timeouts.py @@ -0,0 +1,7 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +ONE_MINUTE = 60 +FIVE_MINUTES = 60 * 5 +TEN_MINUTES = 60 * 10 +TWENTY_MINUTES = 60 * 20 diff --git a/airbyte-integrations/bases/connector-acceptance-test/poetry.lock b/airbyte-integrations/bases/connector-acceptance-test/poetry.lock index 50a12f5ee023..fff2f1561b59 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/poetry.lock +++ b/airbyte-integrations/bases/connector-acceptance-test/poetry.lock @@ -1,14 +1,14 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "airbyte-protocol-models" -version = "0.4.1" +version = "0.5.3" description = "Declares the Airbyte Protocol." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte_protocol_models-0.4.1-py3-none-any.whl", hash = "sha256:95f1197c800d7867ba067f75770b83aeff4c2cec9b3d1def2dbf70261fee89ee"}, - {file = "airbyte_protocol_models-0.4.1.tar.gz", hash = "sha256:92602134eab4c921d1328fa4f24e9a810a679c117ccb352cf6b1521f95f0ed53"}, + {file = "airbyte_protocol_models-0.5.3-py3-none-any.whl", hash = "sha256:a913f1e86d5b2ae17d19e0135339e55fc25bb93bfc3f7ab38592677f29b56c57"}, + {file = "airbyte_protocol_models-0.5.3.tar.gz", hash = "sha256:a71bc0e98e0722d5cbd3122c40a59a7f9cbc91b6c934db7e768a57c40546f54b"}, ] [package.dependencies] @@ -47,21 +47,22 @@ files = [ [[package]] name = "attrs" -version = "23.1.0" +version = "23.2.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.7" files = [ - {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"}, - {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"}, + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, ] [package.extras] cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] -dev = ["attrs[docs,tests]", "pre-commit"] +dev = ["attrs[tests]", "pre-commit"] docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] tests = ["attrs[tests-no-zope]", "zope-interface"] -tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] [[package]] name = "backoff" @@ -76,13 +77,13 @@ files = [ [[package]] name = "beartype" -version = "0.15.0" +version = "0.16.4" description = "Unbearably fast runtime type checking in pure Python." optional = false python-versions = ">=3.8.0" files = [ - {file = "beartype-0.15.0-py3-none-any.whl", hash = "sha256:52cd2edea72fdd84e4e7f8011a9e3007bf0125c3d6d7219e937b9d8868169177"}, - {file = "beartype-0.15.0.tar.gz", hash = "sha256:2af6a8d8a7267ccf7d271e1a3bd908afbc025d2a09aa51123567d7d7b37438df"}, + {file = "beartype-0.16.4-py3-none-any.whl", hash = "sha256:64865952f9dff1e17f22684b3c7286fc79754553b47eaefeb1286224ae8c1bd9"}, + {file = "beartype-0.16.4.tar.gz", hash = "sha256:1ada89cf2d6eb30eb6e156eed2eb5493357782937910d74380918e53c2eae0bf"}, ] [package.extras] @@ -94,122 +95,137 @@ test-tox-coverage = ["coverage (>=5.5)"] [[package]] name = "cattrs" -version = "23.1.2" +version = "23.2.3" description = "Composable complex class support for attrs and dataclasses." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "cattrs-23.1.2-py3-none-any.whl", hash = "sha256:b2bb14311ac17bed0d58785e5a60f022e5431aca3932e3fc5cc8ed8639de50a4"}, - {file = "cattrs-23.1.2.tar.gz", hash = "sha256:db1c821b8c537382b2c7c66678c3790091ca0275ac486c76f3c8f3920e83c657"}, + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, ] [package.dependencies] -attrs = ">=20" -exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} -typing_extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} [package.extras] -bson = ["pymongo (>=4.2.0,<5.0.0)"] -cbor2 = ["cbor2 (>=5.4.6,<6.0.0)"] -msgpack = ["msgpack (>=1.0.2,<2.0.0)"] -orjson = ["orjson (>=3.5.2,<4.0.0)"] -pyyaml = ["PyYAML (>=6.0,<7.0)"] -tomlkit = ["tomlkit (>=0.11.4,<0.12.0)"] -ujson = ["ujson (>=5.4.0,<6.0.0)"] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] [[package]] name = "certifi" -version = "2023.7.22" +version = "2023.11.17" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"}, - {file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"}, + {file = "certifi-2023.11.17-py3-none-any.whl", hash = "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"}, + {file = "certifi-2023.11.17.tar.gz", hash = "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1"}, ] [[package]] name = "charset-normalizer" -version = "3.2.0" +version = "3.3.2" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7.0" files = [ - {file = "charset-normalizer-3.2.0.tar.gz", hash = "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-win32.whl", hash = "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96"}, - {file = "charset_normalizer-3.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-win32.whl", hash = "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1"}, - {file = "charset_normalizer-3.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-win32.whl", hash = "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1"}, - {file = "charset_normalizer-3.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-win32.whl", hash = "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706"}, - {file = "charset_normalizer-3.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-win32.whl", hash = "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9"}, - {file = "charset_normalizer-3.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80"}, - {file = "charset_normalizer-3.2.0-py3-none-any.whl", hash = "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6"}, + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, ] [[package]] @@ -225,71 +241,63 @@ files = [ [[package]] name = "coverage" -version = "7.2.7" +version = "7.4.0" description = "Code coverage measurement for Python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "coverage-7.2.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d39b5b4f2a66ccae8b7263ac3c8170994b65266797fb96cbbfd3fb5b23921db8"}, - {file = "coverage-7.2.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6d040ef7c9859bb11dfeb056ff5b3872436e3b5e401817d87a31e1750b9ae2fb"}, - {file = "coverage-7.2.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba90a9563ba44a72fda2e85302c3abc71c5589cea608ca16c22b9804262aaeb6"}, - {file = "coverage-7.2.7-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7d9405291c6928619403db1d10bd07888888ec1abcbd9748fdaa971d7d661b2"}, - {file = "coverage-7.2.7-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31563e97dae5598556600466ad9beea39fb04e0229e61c12eaa206e0aa202063"}, - {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ebba1cd308ef115925421d3e6a586e655ca5a77b5bf41e02eb0e4562a111f2d1"}, - {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cb017fd1b2603ef59e374ba2063f593abe0fc45f2ad9abdde5b4d83bd922a353"}, - {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62a5c7dad11015c66fbb9d881bc4caa5b12f16292f857842d9d1871595f4495"}, - {file = "coverage-7.2.7-cp310-cp310-win32.whl", hash = "sha256:ee57190f24fba796e36bb6d3aa8a8783c643d8fa9760c89f7a98ab5455fbf818"}, - {file = "coverage-7.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:f75f7168ab25dd93110c8a8117a22450c19976afbc44234cbf71481094c1b850"}, - {file = "coverage-7.2.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:06a9a2be0b5b576c3f18f1a241f0473575c4a26021b52b2a85263a00f034d51f"}, - {file = "coverage-7.2.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5baa06420f837184130752b7c5ea0808762083bf3487b5038d68b012e5937dbe"}, - {file = "coverage-7.2.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdec9e8cbf13a5bf63290fc6013d216a4c7232efb51548594ca3631a7f13c3a3"}, - {file = "coverage-7.2.7-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:52edc1a60c0d34afa421c9c37078817b2e67a392cab17d97283b64c5833f427f"}, - {file = "coverage-7.2.7-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63426706118b7f5cf6bb6c895dc215d8a418d5952544042c8a2d9fe87fcf09cb"}, - {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:afb17f84d56068a7c29f5fa37bfd38d5aba69e3304af08ee94da8ed5b0865833"}, - {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:48c19d2159d433ccc99e729ceae7d5293fbffa0bdb94952d3579983d1c8c9d97"}, - {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0e1f928eaf5469c11e886fe0885ad2bf1ec606434e79842a879277895a50942a"}, - {file = "coverage-7.2.7-cp311-cp311-win32.whl", hash = "sha256:33d6d3ea29d5b3a1a632b3c4e4f4ecae24ef170b0b9ee493883f2df10039959a"}, - {file = "coverage-7.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:5b7540161790b2f28143191f5f8ec02fb132660ff175b7747b95dcb77ac26562"}, - {file = "coverage-7.2.7-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f2f67fe12b22cd130d34d0ef79206061bfb5eda52feb6ce0dba0644e20a03cf4"}, - {file = "coverage-7.2.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a342242fe22407f3c17f4b499276a02b01e80f861f1682ad1d95b04018e0c0d4"}, - {file = "coverage-7.2.7-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:171717c7cb6b453aebac9a2ef603699da237f341b38eebfee9be75d27dc38e01"}, - {file = "coverage-7.2.7-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49969a9f7ffa086d973d91cec8d2e31080436ef0fb4a359cae927e742abfaaa6"}, - {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b46517c02ccd08092f4fa99f24c3b83d8f92f739b4657b0f146246a0ca6a831d"}, - {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:a3d33a6b3eae87ceaefa91ffdc130b5e8536182cd6dfdbfc1aa56b46ff8c86de"}, - {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:976b9c42fb2a43ebf304fa7d4a310e5f16cc99992f33eced91ef6f908bd8f33d"}, - {file = "coverage-7.2.7-cp312-cp312-win32.whl", hash = "sha256:8de8bb0e5ad103888d65abef8bca41ab93721647590a3f740100cd65c3b00511"}, - {file = "coverage-7.2.7-cp312-cp312-win_amd64.whl", hash = "sha256:9e31cb64d7de6b6f09702bb27c02d1904b3aebfca610c12772452c4e6c21a0d3"}, - {file = "coverage-7.2.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:58c2ccc2f00ecb51253cbe5d8d7122a34590fac9646a960d1430d5b15321d95f"}, - {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d22656368f0e6189e24722214ed8d66b8022db19d182927b9a248a2a8a2f67eb"}, - {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a895fcc7b15c3fc72beb43cdcbdf0ddb7d2ebc959edac9cef390b0d14f39f8a9"}, - {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e84606b74eb7de6ff581a7915e2dab7a28a0517fbe1c9239eb227e1354064dcd"}, - {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0a5f9e1dbd7fbe30196578ca36f3fba75376fb99888c395c5880b355e2875f8a"}, - {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:419bfd2caae268623dd469eff96d510a920c90928b60f2073d79f8fe2bbc5959"}, - {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2aee274c46590717f38ae5e4650988d1af340fe06167546cc32fe2f58ed05b02"}, - {file = "coverage-7.2.7-cp37-cp37m-win32.whl", hash = "sha256:61b9a528fb348373c433e8966535074b802c7a5d7f23c4f421e6c6e2f1697a6f"}, - {file = "coverage-7.2.7-cp37-cp37m-win_amd64.whl", hash = "sha256:b1c546aca0ca4d028901d825015dc8e4d56aac4b541877690eb76490f1dc8ed0"}, - {file = "coverage-7.2.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:54b896376ab563bd38453cecb813c295cf347cf5906e8b41d340b0321a5433e5"}, - {file = "coverage-7.2.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3d376df58cc111dc8e21e3b6e24606b5bb5dee6024f46a5abca99124b2229ef5"}, - {file = "coverage-7.2.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e330fc79bd7207e46c7d7fd2bb4af2963f5f635703925543a70b99574b0fea9"}, - {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e9d683426464e4a252bf70c3498756055016f99ddaec3774bf368e76bbe02b6"}, - {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d13c64ee2d33eccf7437961b6ea7ad8673e2be040b4f7fd4fd4d4d28d9ccb1e"}, - {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b7aa5f8a41217360e600da646004f878250a0d6738bcdc11a0a39928d7dc2050"}, - {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8fa03bce9bfbeeef9f3b160a8bed39a221d82308b4152b27d82d8daa7041fee5"}, - {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:245167dd26180ab4c91d5e1496a30be4cd721a5cf2abf52974f965f10f11419f"}, - {file = "coverage-7.2.7-cp38-cp38-win32.whl", hash = "sha256:d2c2db7fd82e9b72937969bceac4d6ca89660db0a0967614ce2481e81a0b771e"}, - {file = "coverage-7.2.7-cp38-cp38-win_amd64.whl", hash = "sha256:2e07b54284e381531c87f785f613b833569c14ecacdcb85d56b25c4622c16c3c"}, - {file = "coverage-7.2.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:537891ae8ce59ef63d0123f7ac9e2ae0fc8b72c7ccbe5296fec45fd68967b6c9"}, - {file = "coverage-7.2.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06fb182e69f33f6cd1d39a6c597294cff3143554b64b9825d1dc69d18cc2fff2"}, - {file = "coverage-7.2.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:201e7389591af40950a6480bd9edfa8ed04346ff80002cec1a66cac4549c1ad7"}, - {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f6951407391b639504e3b3be51b7ba5f3528adbf1a8ac3302b687ecababf929e"}, - {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f48351d66575f535669306aa7d6d6f71bc43372473b54a832222803eb956fd1"}, - {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b29019c76039dc3c0fd815c41392a044ce555d9bcdd38b0fb60fb4cd8e475ba9"}, - {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:81c13a1fc7468c40f13420732805a4c38a105d89848b7c10af65a90beff25250"}, - {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:975d70ab7e3c80a3fe86001d8751f6778905ec723f5b110aed1e450da9d4b7f2"}, - {file = "coverage-7.2.7-cp39-cp39-win32.whl", hash = "sha256:7ee7d9d4822c8acc74a5e26c50604dff824710bc8de424904c0982e25c39c6cb"}, - {file = "coverage-7.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:eb393e5ebc85245347950143969b241d08b52b88a3dc39479822e073a1a8eb27"}, - {file = "coverage-7.2.7-pp37.pp38.pp39-none-any.whl", hash = "sha256:b7b4c971f05e6ae490fef852c218b0e79d4e52f79ef0c8475566584a8fb3e01d"}, - {file = "coverage-7.2.7.tar.gz", hash = "sha256:924d94291ca674905fe9481f12294eb11f2d3d3fd1adb20314ba89e94f44ed59"}, + {file = "coverage-7.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:36b0ea8ab20d6a7564e89cb6135920bc9188fb5f1f7152e94e8300b7b189441a"}, + {file = "coverage-7.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0676cd0ba581e514b7f726495ea75aba3eb20899d824636c6f59b0ed2f88c471"}, + {file = "coverage-7.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d0ca5c71a5a1765a0f8f88022c52b6b8be740e512980362f7fdbb03725a0d6b9"}, + {file = "coverage-7.4.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a7c97726520f784239f6c62506bc70e48d01ae71e9da128259d61ca5e9788516"}, + {file = "coverage-7.4.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:815ac2d0f3398a14286dc2cea223a6f338109f9ecf39a71160cd1628786bc6f5"}, + {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:80b5ee39b7f0131ebec7968baa9b2309eddb35b8403d1869e08f024efd883566"}, + {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5b2ccb7548a0b65974860a78c9ffe1173cfb5877460e5a229238d985565574ae"}, + {file = "coverage-7.4.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:995ea5c48c4ebfd898eacb098164b3cc826ba273b3049e4a889658548e321b43"}, + {file = "coverage-7.4.0-cp310-cp310-win32.whl", hash = "sha256:79287fd95585ed36e83182794a57a46aeae0b64ca53929d1176db56aacc83451"}, + {file = "coverage-7.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:5b14b4f8760006bfdb6e08667af7bc2d8d9bfdb648351915315ea17645347137"}, + {file = "coverage-7.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:04387a4a6ecb330c1878907ce0dc04078ea72a869263e53c72a1ba5bbdf380ca"}, + {file = "coverage-7.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ea81d8f9691bb53f4fb4db603203029643caffc82bf998ab5b59ca05560f4c06"}, + {file = "coverage-7.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74775198b702868ec2d058cb92720a3c5a9177296f75bd97317c787daf711505"}, + {file = "coverage-7.4.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:76f03940f9973bfaee8cfba70ac991825611b9aac047e5c80d499a44079ec0bc"}, + {file = "coverage-7.4.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:485e9f897cf4856a65a57c7f6ea3dc0d4e6c076c87311d4bc003f82cfe199d25"}, + {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6ae8c9d301207e6856865867d762a4b6fd379c714fcc0607a84b92ee63feff70"}, + {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bf477c355274a72435ceb140dc42de0dc1e1e0bf6e97195be30487d8eaaf1a09"}, + {file = "coverage-7.4.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:83c2dda2666fe32332f8e87481eed056c8b4d163fe18ecc690b02802d36a4d26"}, + {file = "coverage-7.4.0-cp311-cp311-win32.whl", hash = "sha256:697d1317e5290a313ef0d369650cfee1a114abb6021fa239ca12b4849ebbd614"}, + {file = "coverage-7.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:26776ff6c711d9d835557ee453082025d871e30b3fd6c27fcef14733f67f0590"}, + {file = "coverage-7.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:13eaf476ec3e883fe3e5fe3707caeb88268a06284484a3daf8250259ef1ba143"}, + {file = "coverage-7.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846f52f46e212affb5bcf131c952fb4075b55aae6b61adc9856222df89cbe3e2"}, + {file = "coverage-7.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26f66da8695719ccf90e794ed567a1549bb2644a706b41e9f6eae6816b398c4a"}, + {file = "coverage-7.4.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:164fdcc3246c69a6526a59b744b62e303039a81e42cfbbdc171c91a8cc2f9446"}, + {file = "coverage-7.4.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:316543f71025a6565677d84bc4df2114e9b6a615aa39fb165d697dba06a54af9"}, + {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bb1de682da0b824411e00a0d4da5a784ec6496b6850fdf8c865c1d68c0e318dd"}, + {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:0e8d06778e8fbffccfe96331a3946237f87b1e1d359d7fbe8b06b96c95a5407a"}, + {file = "coverage-7.4.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a56de34db7b7ff77056a37aedded01b2b98b508227d2d0979d373a9b5d353daa"}, + {file = "coverage-7.4.0-cp312-cp312-win32.whl", hash = "sha256:51456e6fa099a8d9d91497202d9563a320513fcf59f33991b0661a4a6f2ad450"}, + {file = "coverage-7.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:cd3c1e4cb2ff0083758f09be0f77402e1bdf704adb7f89108007300a6da587d0"}, + {file = "coverage-7.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e9d1bf53c4c8de58d22e0e956a79a5b37f754ed1ffdbf1a260d9dcfa2d8a325e"}, + {file = "coverage-7.4.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:109f5985182b6b81fe33323ab4707011875198c41964f014579cf82cebf2bb85"}, + {file = "coverage-7.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3cc9d4bc55de8003663ec94c2f215d12d42ceea128da8f0f4036235a119c88ac"}, + {file = "coverage-7.4.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cc6d65b21c219ec2072c1293c505cf36e4e913a3f936d80028993dd73c7906b1"}, + {file = "coverage-7.4.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a10a4920def78bbfff4eff8a05c51be03e42f1c3735be42d851f199144897ba"}, + {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b8e99f06160602bc64da35158bb76c73522a4010f0649be44a4e167ff8555952"}, + {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7d360587e64d006402b7116623cebf9d48893329ef035278969fa3bbf75b697e"}, + {file = "coverage-7.4.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:29f3abe810930311c0b5d1a7140f6395369c3db1be68345638c33eec07535105"}, + {file = "coverage-7.4.0-cp38-cp38-win32.whl", hash = "sha256:5040148f4ec43644702e7b16ca864c5314ccb8ee0751ef617d49aa0e2d6bf4f2"}, + {file = "coverage-7.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:9864463c1c2f9cb3b5db2cf1ff475eed2f0b4285c2aaf4d357b69959941aa555"}, + {file = "coverage-7.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:936d38794044b26c99d3dd004d8af0035ac535b92090f7f2bb5aa9c8e2f5cd42"}, + {file = "coverage-7.4.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:799c8f873794a08cdf216aa5d0531c6a3747793b70c53f70e98259720a6fe2d7"}, + {file = "coverage-7.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7defbb9737274023e2d7af02cac77043c86ce88a907c58f42b580a97d5bcca9"}, + {file = "coverage-7.4.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a1526d265743fb49363974b7aa8d5899ff64ee07df47dd8d3e37dcc0818f09ed"}, + {file = "coverage-7.4.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf635a52fc1ea401baf88843ae8708591aa4adff875e5c23220de43b1ccf575c"}, + {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:756ded44f47f330666843b5781be126ab57bb57c22adbb07d83f6b519783b870"}, + {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:0eb3c2f32dabe3a4aaf6441dde94f35687224dfd7eb2a7f47f3fd9428e421058"}, + {file = "coverage-7.4.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bfd5db349d15c08311702611f3dccbef4b4e2ec148fcc636cf8739519b4a5c0f"}, + {file = "coverage-7.4.0-cp39-cp39-win32.whl", hash = "sha256:53d7d9158ee03956e0eadac38dfa1ec8068431ef8058fe6447043db1fb40d932"}, + {file = "coverage-7.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:cfd2a8b6b0d8e66e944d47cdec2f47c48fef2ba2f2dff5a9a75757f64172857e"}, + {file = "coverage-7.4.0-pp38.pp39.pp310-none-any.whl", hash = "sha256:c530833afc4707fe48524a44844493f36d8727f04dcce91fb978c414a8556cc6"}, + {file = "coverage-7.4.0.tar.gz", hash = "sha256:707c0f58cb1712b8809ece32b68996ee1e609f71bd14615bd8f87a1293cb610e"}, ] [package.dependencies] @@ -300,13 +308,13 @@ toml = ["tomli"] [[package]] name = "dagger-io" -version = "0.6.4" +version = "0.9.6" description = "A client package for running Dagger pipelines in Python." optional = false python-versions = ">=3.10" files = [ - {file = "dagger_io-0.6.4-py3-none-any.whl", hash = "sha256:b1bea624d1428a40228fffaa96407292cc3d18a7eca5bc036e6ceb9abd903d9a"}, - {file = "dagger_io-0.6.4.tar.gz", hash = "sha256:b754fd9820c41904e344377330ccca88f0a3409023eea8f0557db739b871e552"}, + {file = "dagger_io-0.9.6-py3-none-any.whl", hash = "sha256:e2f1e4bbc252071a314fa5b0bad11a910433a9ee043972b716f6fcc5f9fc8236"}, + {file = "dagger_io-0.9.6.tar.gz", hash = "sha256:147b5a33c44d17f602a4121679893655e91308beb8c46a466afed39cf40f789b"}, ] [package.dependencies] @@ -317,11 +325,8 @@ gql = ">=3.4.0" graphql-core = ">=3.2.3" httpx = ">=0.23.1" platformdirs = ">=2.6.2" -typing-extensions = ">=4.4.0" - -[package.extras] -cli = ["typer[all] (>=0.6.1)"] -server = ["strawberry-graphql (>=0.187.0)", "typer[all] (>=0.6.1)"] +rich = ">=10.11.0" +typing-extensions = ">=4.8.0" [[package]] name = "deepdiff" @@ -374,18 +379,32 @@ files = [ [[package]] name = "exceptiongroup" -version = "1.1.2" +version = "1.2.0" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"}, - {file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"}, + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, ] [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "execnet" +version = "2.0.2" +description = "execnet: rapid multi-Python deployment" +optional = false +python-versions = ">=3.7" +files = [ + {file = "execnet-2.0.2-py3-none-any.whl", hash = "sha256:88256416ae766bc9e8895c76a87928c0012183da3cc4fc18016e6f050e025f41"}, + {file = "execnet-2.0.2.tar.gz", hash = "sha256:cc59bc4423742fd71ad227122eb0dd44db51efb3dc4095b45ac9a08c770096af"}, +] + +[package.extras] +testing = ["hatch", "pre-commit", "pytest", "tox"] + [[package]] name = "fancycompleter" version = "0.9.1" @@ -403,29 +422,31 @@ pyrepl = ">=0.8.2" [[package]] name = "gql" -version = "3.4.1" +version = "3.5.0" description = "GraphQL client for Python" optional = false python-versions = "*" files = [ - {file = "gql-3.4.1-py2.py3-none-any.whl", hash = "sha256:315624ca0f4d571ef149d455033ebd35e45c1a13f18a059596aeddcea99135cf"}, - {file = "gql-3.4.1.tar.gz", hash = "sha256:11dc5d8715a827f2c2899593439a4f36449db4f0eafa5b1ea63948f8a2f8c545"}, + {file = "gql-3.5.0-py2.py3-none-any.whl", hash = "sha256:70dda5694a5b194a8441f077aa5fb70cc94e4ec08016117523f013680901ecb7"}, + {file = "gql-3.5.0.tar.gz", hash = "sha256:ccb9c5db543682b28f577069950488218ed65d4ac70bb03b6929aaadaf636de9"}, ] [package.dependencies] +anyio = ">=3.0,<5" backoff = ">=1.11.1,<3.0" graphql-core = ">=3.2,<3.3" yarl = ">=1.6,<2.0" [package.extras] -aiohttp = ["aiohttp (>=3.7.1,<3.9.0)"] -all = ["aiohttp (>=3.7.1,<3.9.0)", "botocore (>=1.21,<2)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] +aiohttp = ["aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)"] +all = ["aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "botocore (>=1.21,<2)", "httpx (>=0.23.1,<1)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "websockets (>=10,<12)"] botocore = ["botocore (>=1.21,<2)"] -dev = ["aiofiles", "aiohttp (>=3.7.1,<3.9.0)", "black (==22.3.0)", "botocore (>=1.21,<2)", "check-manifest (>=0.42,<1)", "flake8 (==3.8.1)", "isort (==4.3.21)", "mock (==4.0.2)", "mypy (==0.910)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "sphinx (>=3.0.0,<4)", "sphinx-argparse (==0.2.5)", "sphinx-rtd-theme (>=0.4,<1)", "types-aiofiles", "types-mock", "types-requests", "urllib3 (>=1.26,<2)", "vcrpy (==4.0.2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] -requests = ["requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)"] -test = ["aiofiles", "aiohttp (>=3.7.1,<3.9.0)", "botocore (>=1.21,<2)", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=0.9.1,<1)", "urllib3 (>=1.26,<2)", "vcrpy (==4.0.2)", "websockets (>=10,<11)", "websockets (>=9,<10)"] -test-no-transport = ["aiofiles", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==6.2.5)", "pytest-asyncio (==0.16.0)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "vcrpy (==4.0.2)"] -websockets = ["websockets (>=10,<11)", "websockets (>=9,<10)"] +dev = ["aiofiles", "aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "black (==22.3.0)", "botocore (>=1.21,<2)", "check-manifest (>=0.42,<1)", "flake8 (==3.8.1)", "httpx (>=0.23.1,<1)", "isort (==4.3.21)", "mock (==4.0.2)", "mypy (==0.910)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "sphinx (>=5.3.0,<6)", "sphinx-argparse (==0.2.5)", "sphinx-rtd-theme (>=0.4,<1)", "types-aiofiles", "types-mock", "types-requests", "vcrpy (==4.4.0)", "websockets (>=10,<12)"] +httpx = ["httpx (>=0.23.1,<1)"] +requests = ["requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)"] +test = ["aiofiles", "aiohttp (>=3.8.0,<4)", "aiohttp (>=3.9.0b0,<4)", "botocore (>=1.21,<2)", "httpx (>=0.23.1,<1)", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "requests (>=2.26,<3)", "requests-toolbelt (>=1.0.0,<2)", "vcrpy (==4.4.0)", "websockets (>=10,<12)"] +test-no-transport = ["aiofiles", "mock (==4.0.2)", "parse (==1.15.0)", "pytest (==7.4.2)", "pytest-asyncio (==0.21.1)", "pytest-console-scripts (==1.3.1)", "pytest-cov (==3.0.0)", "vcrpy (==4.4.0)"] +websockets = ["websockets (>=10,<12)"] [[package]] name = "graphql-core" @@ -451,39 +472,40 @@ files = [ [[package]] name = "httpcore" -version = "0.17.3" +version = "1.0.2" description = "A minimal low-level HTTP client." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "httpcore-0.17.3-py3-none-any.whl", hash = "sha256:c2789b767ddddfa2a5782e3199b2b7f6894540b17b16ec26b2c4d8e103510b87"}, - {file = "httpcore-0.17.3.tar.gz", hash = "sha256:a6f30213335e34c1ade7be6ec7c47f19f50c56db36abef1a9dfa3815b1cb3888"}, + {file = "httpcore-1.0.2-py3-none-any.whl", hash = "sha256:096cc05bca73b8e459a1fc3dcf585148f63e534eae4339559c9b8a8d6399acc7"}, + {file = "httpcore-1.0.2.tar.gz", hash = "sha256:9fc092e4799b26174648e54b74ed5f683132a464e95643b226e00c2ed2fa6535"}, ] [package.dependencies] -anyio = ">=3.0,<5.0" certifi = "*" h11 = ">=0.13,<0.15" -sniffio = "==1.*" [package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<0.23.0)"] [[package]] name = "httpx" -version = "0.24.1" +version = "0.26.0" description = "The next generation HTTP client." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "httpx-0.24.1-py3-none-any.whl", hash = "sha256:06781eb9ac53cde990577af654bd990a4949de37a28bdb4a230d434f3a30b9bd"}, - {file = "httpx-0.24.1.tar.gz", hash = "sha256:5853a43053df830c20f8110c5e69fe44d035d850b2dfe795e196f00fdb774bdd"}, + {file = "httpx-0.26.0-py3-none-any.whl", hash = "sha256:8915f5a3627c4d47b73e8202457cb28f1266982d1159bd5779d86a80c0eab1cd"}, + {file = "httpx-0.26.0.tar.gz", hash = "sha256:451b55c30d5185ea6b23c2c793abf9bb237d2a7dfb901ced6ff69ad37ec1dfaf"}, ] [package.dependencies] +anyio = "*" certifi = "*" -httpcore = ">=0.15.0,<0.18.0" +httpcore = "==1.*" idna = "*" sniffio = "*" @@ -495,21 +517,22 @@ socks = ["socksio (==1.*)"] [[package]] name = "hypothesis" -version = "6.82.3" +version = "6.96.0" description = "A library for property-based testing" optional = false python-versions = ">=3.8" files = [ - {file = "hypothesis-6.82.3-py3-none-any.whl", hash = "sha256:7ff0f6a12d3cd9372e30f84d300e2468c3923e813198a93b9e479dda91858460"}, + {file = "hypothesis-6.96.0-py3-none-any.whl", hash = "sha256:ec8e0348844e1a9368aeaf85dbea1d247f93f5f865fdf65801bc578b4608cc08"}, + {file = "hypothesis-6.96.0.tar.gz", hash = "sha256:fec50dcbc54ec5884a4199d723543ba9408bbab940cc3ab849a92fe1fab97625"}, ] [package.dependencies] -attrs = ">=19.2.0" +attrs = ">=22.2.0" exceptiongroup = {version = ">=1.0.0", markers = "python_version < \"3.11\""} sortedcontainers = ">=2.1.0,<3.0.0" [package.extras] -all = ["backports.zoneinfo (>=0.2.1)", "black (>=19.10b0)", "click (>=7.0)", "django (>=3.2)", "dpcontracts (>=0.4)", "lark (>=0.10.1)", "libcst (>=0.3.16)", "numpy (>=1.17.3)", "pandas (>=1.1)", "pytest (>=4.6)", "python-dateutil (>=1.4)", "pytz (>=2014.1)", "redis (>=3.0.0)", "rich (>=9.0.0)", "tzdata (>=2023.3)"] +all = ["backports.zoneinfo (>=0.2.1)", "black (>=19.10b0)", "click (>=7.0)", "django (>=3.2)", "dpcontracts (>=0.4)", "lark (>=0.10.1)", "libcst (>=0.3.16)", "numpy (>=1.17.3)", "pandas (>=1.1)", "pytest (>=4.6)", "python-dateutil (>=1.4)", "pytz (>=2014.1)", "redis (>=3.0.0)", "rich (>=9.0.0)", "tzdata (>=2023.4)"] cli = ["black (>=19.10b0)", "click (>=7.0)", "rich (>=9.0.0)"] codemods = ["libcst (>=0.3.16)"] dateutil = ["python-dateutil (>=1.4)"] @@ -522,22 +545,22 @@ pandas = ["pandas (>=1.1)"] pytest = ["pytest (>=4.6)"] pytz = ["pytz (>=2014.1)"] redis = ["redis (>=3.0.0)"] -zoneinfo = ["backports.zoneinfo (>=0.2.1)", "tzdata (>=2023.3)"] +zoneinfo = ["backports.zoneinfo (>=0.2.1)", "tzdata (>=2023.4)"] [[package]] name = "hypothesis-jsonschema" -version = "0.22.1" +version = "0.23.0" description = "Generate test data from JSON schemata with Hypothesis" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "hypothesis-jsonschema-0.22.1.tar.gz", hash = "sha256:5dd7449009f323e408a9aa64afb4d18bd1f60ea2eabf5bf152a510da728b34f2"}, - {file = "hypothesis_jsonschema-0.22.1-py3-none-any.whl", hash = "sha256:082968cb86a6aac2369627b08753cbf714c08054b1ebfce3588e3756e652cde6"}, + {file = "hypothesis-jsonschema-0.23.0.tar.gz", hash = "sha256:c3cc5ecddd78efcb5c10cc3fbcf06aa4d32d8300d0babb8c6f89485f7a503aef"}, + {file = "hypothesis_jsonschema-0.23.0-py3-none-any.whl", hash = "sha256:bbf13b49970216b69adfeab666e483bd83691573d9fee55f3c69adeefa978a09"}, ] [package.dependencies] -hypothesis = ">=6.31.6" -jsonschema = ">=4.0.0" +hypothesis = ">=6.84.3" +jsonschema = ">=4.18.0" [[package]] name = "icdiff" @@ -551,13 +574,13 @@ files = [ [[package]] name = "idna" -version = "3.4" +version = "3.6" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.5" files = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, ] [[package]] @@ -595,13 +618,13 @@ files = [ [[package]] name = "jsonschema" -version = "4.19.0" +version = "4.21.0" description = "An implementation of JSON Schema validation for Python" optional = false python-versions = ">=3.8" files = [ - {file = "jsonschema-4.19.0-py3-none-any.whl", hash = "sha256:043dc26a3845ff09d20e4420d6012a9c91c9aa8999fa184e7efcfeccb41e32cb"}, - {file = "jsonschema-4.19.0.tar.gz", hash = "sha256:6e1e7569ac13be8139b2dd2c21a55d350066ee3f80df06c608b398cdc6f30e8f"}, + {file = "jsonschema-4.21.0-py3-none-any.whl", hash = "sha256:70a09719d375c0a2874571b363c8a24be7df8071b80c9aa76bc4551e7297c63c"}, + {file = "jsonschema-4.21.0.tar.gz", hash = "sha256:3ba18e27f7491ea4a1b22edce00fb820eec968d397feb3f9cb61d5894bb38167"}, ] [package.dependencies] @@ -616,17 +639,52 @@ format-nongpl = ["fqdn", "idna", "isoduration", "jsonpointer (>1.13)", "rfc3339- [[package]] name = "jsonschema-specifications" -version = "2023.7.1" +version = "2023.12.1" description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" optional = false python-versions = ">=3.8" files = [ - {file = "jsonschema_specifications-2023.7.1-py3-none-any.whl", hash = "sha256:05adf340b659828a004220a9613be00fa3f223f2b82002e273dee62fd50524b1"}, - {file = "jsonschema_specifications-2023.7.1.tar.gz", hash = "sha256:c91a50404e88a1f6ba40636778e2ee08f6e24c5613fe4c53ac24578a5a7f72bb"}, + {file = "jsonschema_specifications-2023.12.1-py3-none-any.whl", hash = "sha256:87e4fdf3a94858b8a2ba2778d9ba57d8a9cafca7c7489c46ba0d30a8bc6a9c3c"}, + {file = "jsonschema_specifications-2023.12.1.tar.gz", hash = "sha256:48a76787b3e70f5ed53f1160d2b81f586e4ca6d1548c5de7085d1682674764cc"}, +] + +[package.dependencies] +referencing = ">=0.31.0" + +[[package]] +name = "markdown-it-py" +version = "3.0.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +optional = false +python-versions = ">=3.8" +files = [ + {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, + {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, ] [package.dependencies] -referencing = ">=0.28.0" +mdurl = ">=0.1,<1.0" + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +code-style = ["pre-commit (>=3.0,<4.0)"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +optional = false +python-versions = ">=3.7" +files = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] [[package]] name = "multidict" @@ -727,13 +785,13 @@ dev = ["black", "mypy", "pytest"] [[package]] name = "packaging" -version = "23.1" +version = "23.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, - {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, ] [[package]] @@ -758,47 +816,112 @@ testing = ["funcsigs", "pytest"] [[package]] name = "pendulum" -version = "2.1.2" +version = "3.0.0" description = "Python datetimes made easy" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +python-versions = ">=3.8" files = [ - {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, - {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, - {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, - {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, - {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, - {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, - {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, - {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, - {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, - {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, - {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, - {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, - {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, - {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, - {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, - {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, - {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, - {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, - {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, - {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, - {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, + {file = "pendulum-3.0.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2cf9e53ef11668e07f73190c805dbdf07a1939c3298b78d5a9203a86775d1bfd"}, + {file = "pendulum-3.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fb551b9b5e6059377889d2d878d940fd0bbb80ae4810543db18e6f77b02c5ef6"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c58227ac260d5b01fc1025176d7b31858c9f62595737f350d22124a9a3ad82d"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60fb6f415fea93a11c52578eaa10594568a6716602be8430b167eb0d730f3332"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b69f6b4dbcb86f2c2fe696ba991e67347bcf87fe601362a1aba6431454b46bde"}, + {file = "pendulum-3.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:138afa9c373ee450ede206db5a5e9004fd3011b3c6bbe1e57015395cd076a09f"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:83d9031f39c6da9677164241fd0d37fbfc9dc8ade7043b5d6d62f56e81af8ad2"}, + {file = "pendulum-3.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0c2308af4033fa534f089595bcd40a95a39988ce4059ccd3dc6acb9ef14ca44a"}, + {file = "pendulum-3.0.0-cp310-none-win_amd64.whl", hash = "sha256:9a59637cdb8462bdf2dbcb9d389518c0263799189d773ad5c11db6b13064fa79"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:3725245c0352c95d6ca297193192020d1b0c0f83d5ee6bb09964edc2b5a2d508"}, + {file = "pendulum-3.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6c035f03a3e565ed132927e2c1b691de0dbf4eb53b02a5a3c5a97e1a64e17bec"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:597e66e63cbd68dd6d58ac46cb7a92363d2088d37ccde2dae4332ef23e95cd00"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99a0f8172e19f3f0c0e4ace0ad1595134d5243cf75985dc2233e8f9e8de263ca"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:77d8839e20f54706aed425bec82a83b4aec74db07f26acd039905d1237a5e1d4"}, + {file = "pendulum-3.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afde30e8146292b059020fbc8b6f8fd4a60ae7c5e6f0afef937bbb24880bdf01"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:660434a6fcf6303c4efd36713ca9212c753140107ee169a3fc6c49c4711c2a05"}, + {file = "pendulum-3.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dee9e5a48c6999dc1106eb7eea3e3a50e98a50651b72c08a87ee2154e544b33e"}, + {file = "pendulum-3.0.0-cp311-none-win_amd64.whl", hash = "sha256:d4cdecde90aec2d67cebe4042fd2a87a4441cc02152ed7ed8fb3ebb110b94ec4"}, + {file = "pendulum-3.0.0-cp311-none-win_arm64.whl", hash = "sha256:773c3bc4ddda2dda9f1b9d51fe06762f9200f3293d75c4660c19b2614b991d83"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:409e64e41418c49f973d43a28afe5df1df4f1dd87c41c7c90f1a63f61ae0f1f7"}, + {file = "pendulum-3.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a38ad2121c5ec7c4c190c7334e789c3b4624798859156b138fcc4d92295835dc"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fde4d0b2024b9785f66b7f30ed59281bd60d63d9213cda0eb0910ead777f6d37"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b2c5675769fb6d4c11238132962939b960fcb365436b6d623c5864287faa319"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8af95e03e066826f0f4c65811cbee1b3123d4a45a1c3a2b4fc23c4b0dff893b5"}, + {file = "pendulum-3.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2165a8f33cb15e06c67070b8afc87a62b85c5a273e3aaa6bc9d15c93a4920d6f"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ad5e65b874b5e56bd942546ea7ba9dd1d6a25121db1c517700f1c9de91b28518"}, + {file = "pendulum-3.0.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:17fe4b2c844bbf5f0ece69cfd959fa02957c61317b2161763950d88fed8e13b9"}, + {file = "pendulum-3.0.0-cp312-none-win_amd64.whl", hash = "sha256:78f8f4e7efe5066aca24a7a57511b9c2119f5c2b5eb81c46ff9222ce11e0a7a5"}, + {file = "pendulum-3.0.0-cp312-none-win_arm64.whl", hash = "sha256:28f49d8d1e32aae9c284a90b6bb3873eee15ec6e1d9042edd611b22a94ac462f"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_10_12_x86_64.whl", hash = "sha256:d4e2512f4e1a4670284a153b214db9719eb5d14ac55ada5b76cbdb8c5c00399d"}, + {file = "pendulum-3.0.0-cp37-cp37m-macosx_11_0_arm64.whl", hash = "sha256:3d897eb50883cc58d9b92f6405245f84b9286cd2de6e8694cb9ea5cb15195a32"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e169cc2ca419517f397811bbe4589cf3cd13fca6dc38bb352ba15ea90739ebb"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f17c3084a4524ebefd9255513692f7e7360e23c8853dc6f10c64cc184e1217ab"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:826d6e258052715f64d05ae0fc9040c0151e6a87aae7c109ba9a0ed930ce4000"}, + {file = "pendulum-3.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2aae97087872ef152a0c40e06100b3665d8cb86b59bc8471ca7c26132fccd0f"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:ac65eeec2250d03106b5e81284ad47f0d417ca299a45e89ccc69e36130ca8bc7"}, + {file = "pendulum-3.0.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:a5346d08f3f4a6e9e672187faa179c7bf9227897081d7121866358af369f44f9"}, + {file = "pendulum-3.0.0-cp37-none-win_amd64.whl", hash = "sha256:235d64e87946d8f95c796af34818c76e0f88c94d624c268693c85b723b698aa9"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:6a881d9c2a7f85bc9adafcfe671df5207f51f5715ae61f5d838b77a1356e8b7b"}, + {file = "pendulum-3.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d7762d2076b9b1cb718a6631ad6c16c23fc3fac76cbb8c454e81e80be98daa34"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e8e36a8130819d97a479a0e7bf379b66b3b1b520e5dc46bd7eb14634338df8c"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7dc843253ac373358ffc0711960e2dd5b94ab67530a3e204d85c6e8cb2c5fa10"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0a78ad3635d609ceb1e97d6aedef6a6a6f93433ddb2312888e668365908c7120"}, + {file = "pendulum-3.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b30a137e9e0d1f751e60e67d11fc67781a572db76b2296f7b4d44554761049d6"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:c95984037987f4a457bb760455d9ca80467be792236b69d0084f228a8ada0162"}, + {file = "pendulum-3.0.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d29c6e578fe0f893766c0d286adbf0b3c726a4e2341eba0917ec79c50274ec16"}, + {file = "pendulum-3.0.0-cp38-none-win_amd64.whl", hash = "sha256:deaba8e16dbfcb3d7a6b5fabdd5a38b7c982809567479987b9c89572df62e027"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:b11aceea5b20b4b5382962b321dbc354af0defe35daa84e9ff3aae3c230df694"}, + {file = "pendulum-3.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a90d4d504e82ad236afac9adca4d6a19e4865f717034fc69bafb112c320dcc8f"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:825799c6b66e3734227756fa746cc34b3549c48693325b8b9f823cb7d21b19ac"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad769e98dc07972e24afe0cff8d365cb6f0ebc7e65620aa1976fcfbcadc4c6f3"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6fc26907eb5fb8cc6188cc620bc2075a6c534d981a2f045daa5f79dfe50d512"}, + {file = "pendulum-3.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c717eab1b6d898c00a3e0fa7781d615b5c5136bbd40abe82be100bb06df7a56"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3ddd1d66d1a714ce43acfe337190be055cdc221d911fc886d5a3aae28e14b76d"}, + {file = "pendulum-3.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:822172853d7a9cf6da95d7b66a16c7160cb99ae6df55d44373888181d7a06edc"}, + {file = "pendulum-3.0.0-cp39-none-win_amd64.whl", hash = "sha256:840de1b49cf1ec54c225a2a6f4f0784d50bd47f68e41dc005b7f67c7d5b5f3ae"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3b1f74d1e6ffe5d01d6023870e2ce5c2191486928823196f8575dcc786e107b1"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:729e9f93756a2cdfa77d0fc82068346e9731c7e884097160603872686e570f07"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e586acc0b450cd21cbf0db6bae386237011b75260a3adceddc4be15334689a9a"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22e7944ffc1f0099a79ff468ee9630c73f8c7835cd76fdb57ef7320e6a409df4"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:fa30af36bd8e50686846bdace37cf6707bdd044e5cb6e1109acbad3277232e04"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:440215347b11914ae707981b9a57ab9c7b6983ab0babde07063c6ee75c0dc6e7"}, + {file = "pendulum-3.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:314c4038dc5e6a52991570f50edb2f08c339debdf8cea68ac355b32c4174e820"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5acb1d386337415f74f4d1955c4ce8d0201978c162927d07df8eb0692b2d8533"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a789e12fbdefaffb7b8ac67f9d8f22ba17a3050ceaaa635cd1cc4645773a4b1e"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:860aa9b8a888e5913bd70d819306749e5eb488e6b99cd6c47beb701b22bdecf5"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:5ebc65ea033ef0281368217fbf59f5cb05b338ac4dd23d60959c7afcd79a60a0"}, + {file = "pendulum-3.0.0-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d9fef18ab0386ef6a9ac7bad7e43ded42c83ff7ad412f950633854f90d59afa8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1c134ba2f0571d0b68b83f6972e2307a55a5a849e7dac8505c715c531d2a8795"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:385680812e7e18af200bb9b4a49777418c32422d05ad5a8eb85144c4a285907b"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9eec91cd87c59fb32ec49eb722f375bd58f4be790cae11c1b70fac3ee4f00da0"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4386bffeca23c4b69ad50a36211f75b35a4deb6210bdca112ac3043deb7e494a"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:dfbcf1661d7146d7698da4b86e7f04814221081e9fe154183e34f4c5f5fa3bf8"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:04a1094a5aa1daa34a6b57c865b25f691848c61583fb22722a4df5699f6bf74c"}, + {file = "pendulum-3.0.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5b0ec85b9045bd49dd3a3493a5e7ddfd31c36a2a60da387c419fa04abcaecb23"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0a15b90129765b705eb2039062a6daf4d22c4e28d1a54fa260892e8c3ae6e157"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:bb8f6d7acd67a67d6fedd361ad2958ff0539445ef51cbe8cd288db4306503cd0"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd69b15374bef7e4b4440612915315cc42e8575fcda2a3d7586a0d88192d0c88"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc00f8110db6898360c53c812872662e077eaf9c75515d53ecc65d886eec209a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:83a44e8b40655d0ba565a5c3d1365d27e3e6778ae2a05b69124db9e471255c4a"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:1a3604e9fbc06b788041b2a8b78f75c243021e0f512447806a6d37ee5214905d"}, + {file = "pendulum-3.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:92c307ae7accebd06cbae4729f0ba9fa724df5f7d91a0964b1b972a22baa482b"}, + {file = "pendulum-3.0.0.tar.gz", hash = "sha256:5d034998dea404ec31fae27af6b22cff1708f830a1ed7353be4d1019bb9f584e"}, ] [package.dependencies] -python-dateutil = ">=2.6,<3.0" -pytzdata = ">=2020.1" +python-dateutil = ">=2.6" +tzdata = ">=2020.1" + +[package.extras] +test = ["time-machine (>=2.6.0)"] [[package]] name = "platformdirs" -version = "3.10.0" +version = "4.1.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "platformdirs-3.10.0-py3-none-any.whl", hash = "sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d"}, - {file = "platformdirs-3.10.0.tar.gz", hash = "sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d"}, + {file = "platformdirs-4.1.0-py3-none-any.whl", hash = "sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380"}, + {file = "platformdirs-4.1.0.tar.gz", hash = "sha256:906d548203468492d432bcb294d4bc2fff751bf84971fbb2c10918cc206ee420"}, ] [package.extras] @@ -807,13 +930,13 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-co [[package]] name = "pluggy" -version = "1.2.0" +version = "1.3.0" description = "plugin and hook calling mechanisms for python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, - {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, + {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, + {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, ] [package.extras] @@ -844,50 +967,51 @@ files = [ [[package]] name = "pydantic" -version = "1.9.2" +version = "1.10.13" description = "Data validation and settings management using python type hints" optional = false -python-versions = ">=3.6.1" -files = [ - {file = "pydantic-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9c9e04a6cdb7a363d7cb3ccf0efea51e0abb48e180c0d31dca8d247967d85c6e"}, - {file = "pydantic-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fafe841be1103f340a24977f61dee76172e4ae5f647ab9e7fd1e1fca51524f08"}, - {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afacf6d2a41ed91fc631bade88b1d319c51ab5418870802cedb590b709c5ae3c"}, - {file = "pydantic-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3ee0d69b2a5b341fc7927e92cae7ddcfd95e624dfc4870b32a85568bd65e6131"}, - {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:ff68fc85355532ea77559ede81f35fff79a6a5543477e168ab3a381887caea76"}, - {file = "pydantic-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c0f5e142ef8217019e3eef6ae1b6b55f09a7a15972958d44fbd228214cede567"}, - {file = "pydantic-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:615661bfc37e82ac677543704437ff737418e4ea04bef9cf11c6d27346606044"}, - {file = "pydantic-1.9.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:328558c9f2eed77bd8fffad3cef39dbbe3edc7044517f4625a769d45d4cf7555"}, - {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bd446bdb7755c3a94e56d7bdfd3ee92396070efa8ef3a34fab9579fe6aa1d84"}, - {file = "pydantic-1.9.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e0b214e57623a535936005797567231a12d0da0c29711eb3514bc2b3cd008d0f"}, - {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:d8ce3fb0841763a89322ea0432f1f59a2d3feae07a63ea2c958b2315e1ae8adb"}, - {file = "pydantic-1.9.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b34ba24f3e2d0b39b43f0ca62008f7ba962cff51efa56e64ee25c4af6eed987b"}, - {file = "pydantic-1.9.2-cp36-cp36m-win_amd64.whl", hash = "sha256:84d76ecc908d917f4684b354a39fd885d69dd0491be175f3465fe4b59811c001"}, - {file = "pydantic-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4de71c718c9756d679420c69f216776c2e977459f77e8f679a4a961dc7304a56"}, - {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5803ad846cdd1ed0d97eb00292b870c29c1f03732a010e66908ff48a762f20e4"}, - {file = "pydantic-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a8c5360a0297a713b4123608a7909e6869e1b56d0e96eb0d792c27585d40757f"}, - {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:cdb4272678db803ddf94caa4f94f8672e9a46bae4a44f167095e4d06fec12979"}, - {file = "pydantic-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:19b5686387ea0d1ea52ecc4cffb71abb21702c5e5b2ac626fd4dbaa0834aa49d"}, - {file = "pydantic-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:32e0b4fb13ad4db4058a7c3c80e2569adbd810c25e6ca3bbd8b2a9cc2cc871d7"}, - {file = "pydantic-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:91089b2e281713f3893cd01d8e576771cd5bfdfbff5d0ed95969f47ef6d676c3"}, - {file = "pydantic-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e631c70c9280e3129f071635b81207cad85e6c08e253539467e4ead0e5b219aa"}, - {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b3946f87e5cef3ba2e7bd3a4eb5a20385fe36521d6cc1ebf3c08a6697c6cfb3"}, - {file = "pydantic-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5565a49effe38d51882cb7bac18bda013cdb34d80ac336428e8908f0b72499b0"}, - {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:bd67cb2c2d9602ad159389c29e4ca964b86fa2f35c2faef54c3eb28b4efd36c8"}, - {file = "pydantic-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4aafd4e55e8ad5bd1b19572ea2df546ccace7945853832bb99422a79c70ce9b8"}, - {file = "pydantic-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:d70916235d478404a3fa8c997b003b5f33aeac4686ac1baa767234a0f8ac2326"}, - {file = "pydantic-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f0ca86b525264daa5f6b192f216a0d1e860b7383e3da1c65a1908f9c02f42801"}, - {file = "pydantic-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1061c6ee6204f4f5a27133126854948e3b3d51fcc16ead2e5d04378c199b2f44"}, - {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e78578f0c7481c850d1c969aca9a65405887003484d24f6110458fb02cca7747"}, - {file = "pydantic-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5da164119602212a3fe7e3bc08911a89db4710ae51444b4224c2382fd09ad453"}, - {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ead3cd020d526f75b4188e0a8d71c0dbbe1b4b6b5dc0ea775a93aca16256aeb"}, - {file = "pydantic-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7d0f183b305629765910eaad707800d2f47c6ac5bcfb8c6397abdc30b69eeb15"}, - {file = "pydantic-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:f1a68f4f65a9ee64b6ccccb5bf7e17db07caebd2730109cb8a95863cfa9c4e55"}, - {file = "pydantic-1.9.2-py3-none-any.whl", hash = "sha256:78a4d6bdfd116a559aeec9a4cfe77dda62acc6233f8b56a716edad2651023e5e"}, - {file = "pydantic-1.9.2.tar.gz", hash = "sha256:8cb0bc509bfb71305d7a59d00163d5f9fc4530f0881ea32c74ff4f74c85f3d3d"}, +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.13-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:efff03cc7a4f29d9009d1c96ceb1e7a70a65cfe86e89d34e4a5f2ab1e5693737"}, + {file = "pydantic-1.10.13-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3ecea2b9d80e5333303eeb77e180b90e95eea8f765d08c3d278cd56b00345d01"}, + {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1740068fd8e2ef6eb27a20e5651df000978edce6da6803c2bef0bc74540f9548"}, + {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84bafe2e60b5e78bc64a2941b4c071a4b7404c5c907f5f5a99b0139781e69ed8"}, + {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bc0898c12f8e9c97f6cd44c0ed70d55749eaf783716896960b4ecce2edfd2d69"}, + {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:654db58ae399fe6434e55325a2c3e959836bd17a6f6a0b6ca8107ea0571d2e17"}, + {file = "pydantic-1.10.13-cp310-cp310-win_amd64.whl", hash = "sha256:75ac15385a3534d887a99c713aa3da88a30fbd6204a5cd0dc4dab3d770b9bd2f"}, + {file = "pydantic-1.10.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c553f6a156deb868ba38a23cf0df886c63492e9257f60a79c0fd8e7173537653"}, + {file = "pydantic-1.10.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5e08865bc6464df8c7d61439ef4439829e3ab62ab1669cddea8dd00cd74b9ffe"}, + {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e31647d85a2013d926ce60b84f9dd5300d44535a9941fe825dc349ae1f760df9"}, + {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:210ce042e8f6f7c01168b2d84d4c9eb2b009fe7bf572c2266e235edf14bacd80"}, + {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8ae5dd6b721459bfa30805f4c25880e0dd78fc5b5879f9f7a692196ddcb5a580"}, + {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f8e81fc5fb17dae698f52bdd1c4f18b6ca674d7068242b2aff075f588301bbb0"}, + {file = "pydantic-1.10.13-cp311-cp311-win_amd64.whl", hash = "sha256:61d9dce220447fb74f45e73d7ff3b530e25db30192ad8d425166d43c5deb6df0"}, + {file = "pydantic-1.10.13-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4b03e42ec20286f052490423682016fd80fda830d8e4119f8ab13ec7464c0132"}, + {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59ef915cac80275245824e9d771ee939133be38215555e9dc90c6cb148aaeb5"}, + {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a1f9f747851338933942db7af7b6ee8268568ef2ed86c4185c6ef4402e80ba8"}, + {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:97cce3ae7341f7620a0ba5ef6cf043975cd9d2b81f3aa5f4ea37928269bc1b87"}, + {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854223752ba81e3abf663d685f105c64150873cc6f5d0c01d3e3220bcff7d36f"}, + {file = "pydantic-1.10.13-cp37-cp37m-win_amd64.whl", hash = "sha256:b97c1fac8c49be29486df85968682b0afa77e1b809aff74b83081cc115e52f33"}, + {file = "pydantic-1.10.13-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c958d053453a1c4b1c2062b05cd42d9d5c8eb67537b8d5a7e3c3032943ecd261"}, + {file = "pydantic-1.10.13-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c5370a7edaac06daee3af1c8b1192e305bc102abcbf2a92374b5bc793818599"}, + {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d6f6e7305244bddb4414ba7094ce910560c907bdfa3501e9db1a7fd7eaea127"}, + {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3a3c792a58e1622667a2837512099eac62490cdfd63bd407993aaf200a4cf1f"}, + {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c636925f38b8db208e09d344c7aa4f29a86bb9947495dd6b6d376ad10334fb78"}, + {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:678bcf5591b63cc917100dc50ab6caebe597ac67e8c9ccb75e698f66038ea953"}, + {file = "pydantic-1.10.13-cp38-cp38-win_amd64.whl", hash = "sha256:6cf25c1a65c27923a17b3da28a0bdb99f62ee04230c931d83e888012851f4e7f"}, + {file = "pydantic-1.10.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8ef467901d7a41fa0ca6db9ae3ec0021e3f657ce2c208e98cd511f3161c762c6"}, + {file = "pydantic-1.10.13-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:968ac42970f57b8344ee08837b62f6ee6f53c33f603547a55571c954a4225691"}, + {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9849f031cf8a2f0a928fe885e5a04b08006d6d41876b8bbd2fc68a18f9f2e3fd"}, + {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:56e3ff861c3b9c6857579de282ce8baabf443f42ffba355bf070770ed63e11e1"}, + {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f00790179497767aae6bcdc36355792c79e7bbb20b145ff449700eb076c5f96"}, + {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:75b297827b59bc229cac1a23a2f7a4ac0031068e5be0ce385be1462e7e17a35d"}, + {file = "pydantic-1.10.13-cp39-cp39-win_amd64.whl", hash = "sha256:e70ca129d2053fb8b728ee7d1af8e553a928d7e301a311094b8a0501adc8763d"}, + {file = "pydantic-1.10.13-py3-none-any.whl", hash = "sha256:b87326822e71bd5f313e7d3bfdc77ac3247035ac10b0c0618bd99dcf95b1e687"}, + {file = "pydantic-1.10.13.tar.gz", hash = "sha256:32c8b48dcd3b2ac4e78b0ba4af3a2c2eb6048cb75202f0ea7b34feb740efc340"}, ] [package.dependencies] -typing-extensions = ">=3.7.4.3" +typing-extensions = ">=4.2.0" [package.extras] dotenv = ["python-dotenv (>=0.10.4)"] @@ -895,17 +1019,18 @@ email = ["email-validator (>=1.0.3)"] [[package]] name = "pygments" -version = "2.16.1" +version = "2.17.2" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.7" files = [ - {file = "Pygments-2.16.1-py3-none-any.whl", hash = "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692"}, - {file = "Pygments-2.16.1.tar.gz", hash = "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"}, + {file = "pygments-2.17.2-py3-none-any.whl", hash = "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c"}, + {file = "pygments-2.17.2.tar.gz", hash = "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"}, ] [package.extras] plugins = ["importlib-metadata"] +windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pyreadline" @@ -1019,6 +1144,26 @@ files = [ [package.dependencies] pytest = ">=3.6.0" +[[package]] +name = "pytest-xdist" +version = "3.5.0" +description = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-xdist-3.5.0.tar.gz", hash = "sha256:cbb36f3d67e0c478baa57fa4edc8843887e0f6cfc42d677530a36d7472b32d8a"}, + {file = "pytest_xdist-3.5.0-py3-none-any.whl", hash = "sha256:d075629c7e00b611df89f490a5063944bee7a4362a5ff11c7cc7824a03dfce24"}, +] + +[package.dependencies] +execnet = ">=1.1" +pytest = ">=6.2.0" + +[package.extras] +psutil = ["psutil (>=3.0)"] +setproctitle = ["setproctitle"] +testing = ["filelock"] + [[package]] name = "python-dateutil" version = "2.8.2" @@ -1033,17 +1178,6 @@ files = [ [package.dependencies] six = ">=1.5" -[[package]] -name = "pytzdata" -version = "2020.1" -description = "The Olson timezone database for Python." -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [ - {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, - {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, -] - [[package]] name = "pywin32" version = "306" @@ -1079,6 +1213,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1086,8 +1221,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1104,6 +1246,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1111,6 +1254,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1118,13 +1262,13 @@ files = [ [[package]] name = "referencing" -version = "0.30.2" +version = "0.32.1" description = "JSON Referencing + Python" optional = false python-versions = ">=3.8" files = [ - {file = "referencing-0.30.2-py3-none-any.whl", hash = "sha256:449b6669b6121a9e96a7f9e410b245d471e8d48964c67113ce9afe50c8dd7bdf"}, - {file = "referencing-0.30.2.tar.gz", hash = "sha256:794ad8003c65938edcdbc027f1933215e0d0ccc0291e3ce20a4d87432b59efc0"}, + {file = "referencing-0.32.1-py3-none-any.whl", hash = "sha256:7e4dc12271d8e15612bfe35792f5ea1c40970dadf8624602e33db2758f7ee554"}, + {file = "referencing-0.32.1.tar.gz", hash = "sha256:3c57da0513e9563eb7e203ebe9bb3a1b509b042016433bd1e45a2853466c3dd3"}, ] [package.dependencies] @@ -1171,110 +1315,130 @@ six = "*" fixture = ["fixtures"] test = ["fixtures", "mock", "purl", "pytest", "sphinx", "testrepository (>=0.0.18)", "testtools"] +[[package]] +name = "rich" +version = "13.7.0" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "rich-13.7.0-py3-none-any.whl", hash = "sha256:6da14c108c4866ee9520bbffa71f6fe3962e193b7da68720583850cd4548e235"}, + {file = "rich-13.7.0.tar.gz", hash = "sha256:5cb5123b5cf9ee70584244246816e9114227e0b98ad9176eede6ad54bf5403fa"}, +] + +[package.dependencies] +markdown-it-py = ">=2.2.0" +pygments = ">=2.13.0,<3.0.0" + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<9)"] + [[package]] name = "rpds-py" -version = "0.9.2" +version = "0.17.1" description = "Python bindings to Rust's persistent data structures (rpds)" optional = false python-versions = ">=3.8" files = [ - {file = "rpds_py-0.9.2-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:ab6919a09c055c9b092798ce18c6c4adf49d24d4d9e43a92b257e3f2548231e7"}, - {file = "rpds_py-0.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d55777a80f78dd09410bd84ff8c95ee05519f41113b2df90a69622f5540c4f8b"}, - {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a216b26e5af0a8e265d4efd65d3bcec5fba6b26909014effe20cd302fd1138fa"}, - {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29cd8bfb2d716366a035913ced99188a79b623a3512292963d84d3e06e63b496"}, - {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:44659b1f326214950a8204a248ca6199535e73a694be8d3e0e869f820767f12f"}, - {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:745f5a43fdd7d6d25a53ab1a99979e7f8ea419dfefebcab0a5a1e9095490ee5e"}, - {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a987578ac5214f18b99d1f2a3851cba5b09f4a689818a106c23dbad0dfeb760f"}, - {file = "rpds_py-0.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bf4151acb541b6e895354f6ff9ac06995ad9e4175cbc6d30aaed08856558201f"}, - {file = "rpds_py-0.9.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:03421628f0dc10a4119d714a17f646e2837126a25ac7a256bdf7c3943400f67f"}, - {file = "rpds_py-0.9.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:13b602dc3e8dff3063734f02dcf05111e887f301fdda74151a93dbbc249930fe"}, - {file = "rpds_py-0.9.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:fae5cb554b604b3f9e2c608241b5d8d303e410d7dfb6d397c335f983495ce7f6"}, - {file = "rpds_py-0.9.2-cp310-none-win32.whl", hash = "sha256:47c5f58a8e0c2c920cc7783113df2fc4ff12bf3a411d985012f145e9242a2764"}, - {file = "rpds_py-0.9.2-cp310-none-win_amd64.whl", hash = "sha256:4ea6b73c22d8182dff91155af018b11aac9ff7eca085750455c5990cb1cfae6e"}, - {file = "rpds_py-0.9.2-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:e564d2238512c5ef5e9d79338ab77f1cbbda6c2d541ad41b2af445fb200385e3"}, - {file = "rpds_py-0.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f411330a6376fb50e5b7a3e66894e4a39e60ca2e17dce258d53768fea06a37bd"}, - {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e7521f5af0233e89939ad626b15278c71b69dc1dfccaa7b97bd4cdf96536bb7"}, - {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8d3335c03100a073883857e91db9f2e0ef8a1cf42dc0369cbb9151c149dbbc1b"}, - {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d25b1c1096ef0447355f7293fbe9ad740f7c47ae032c2884113f8e87660d8f6e"}, - {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6a5d3fbd02efd9cf6a8ffc2f17b53a33542f6b154e88dd7b42ef4a4c0700fdad"}, - {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5934e2833afeaf36bd1eadb57256239785f5af0220ed8d21c2896ec4d3a765f"}, - {file = "rpds_py-0.9.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:095b460e117685867d45548fbd8598a8d9999227e9061ee7f012d9d264e6048d"}, - {file = "rpds_py-0.9.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:91378d9f4151adc223d584489591dbb79f78814c0734a7c3bfa9c9e09978121c"}, - {file = "rpds_py-0.9.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:24a81c177379300220e907e9b864107614b144f6c2a15ed5c3450e19cf536fae"}, - {file = "rpds_py-0.9.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:de0b6eceb46141984671802d412568d22c6bacc9b230174f9e55fc72ef4f57de"}, - {file = "rpds_py-0.9.2-cp311-none-win32.whl", hash = "sha256:700375326ed641f3d9d32060a91513ad668bcb7e2cffb18415c399acb25de2ab"}, - {file = "rpds_py-0.9.2-cp311-none-win_amd64.whl", hash = "sha256:0766babfcf941db8607bdaf82569ec38107dbb03c7f0b72604a0b346b6eb3298"}, - {file = "rpds_py-0.9.2-cp312-cp312-macosx_10_7_x86_64.whl", hash = "sha256:b1440c291db3f98a914e1afd9d6541e8fc60b4c3aab1a9008d03da4651e67386"}, - {file = "rpds_py-0.9.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0f2996fbac8e0b77fd67102becb9229986396e051f33dbceada3debaacc7033f"}, - {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9f30d205755566a25f2ae0382944fcae2f350500ae4df4e795efa9e850821d82"}, - {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:159fba751a1e6b1c69244e23ba6c28f879a8758a3e992ed056d86d74a194a0f3"}, - {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1f044792e1adcea82468a72310c66a7f08728d72a244730d14880cd1dabe36b"}, - {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9251eb8aa82e6cf88510530b29eef4fac825a2b709baf5b94a6094894f252387"}, - {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01899794b654e616c8625b194ddd1e5b51ef5b60ed61baa7a2d9c2ad7b2a4238"}, - {file = "rpds_py-0.9.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b0c43f8ae8f6be1d605b0465671124aa8d6a0e40f1fb81dcea28b7e3d87ca1e1"}, - {file = "rpds_py-0.9.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:207f57c402d1f8712618f737356e4b6f35253b6d20a324d9a47cb9f38ee43a6b"}, - {file = "rpds_py-0.9.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b52e7c5ae35b00566d244ffefba0f46bb6bec749a50412acf42b1c3f402e2c90"}, - {file = "rpds_py-0.9.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:978fa96dbb005d599ec4fd9ed301b1cc45f1a8f7982d4793faf20b404b56677d"}, - {file = "rpds_py-0.9.2-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:6aa8326a4a608e1c28da191edd7c924dff445251b94653988efb059b16577a4d"}, - {file = "rpds_py-0.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:aad51239bee6bff6823bbbdc8ad85136c6125542bbc609e035ab98ca1e32a192"}, - {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4bd4dc3602370679c2dfb818d9c97b1137d4dd412230cfecd3c66a1bf388a196"}, - {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dd9da77c6ec1f258387957b754f0df60766ac23ed698b61941ba9acccd3284d1"}, - {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:190ca6f55042ea4649ed19c9093a9be9d63cd8a97880106747d7147f88a49d18"}, - {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:876bf9ed62323bc7dcfc261dbc5572c996ef26fe6406b0ff985cbcf460fc8a4c"}, - {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa2818759aba55df50592ecbc95ebcdc99917fa7b55cc6796235b04193eb3c55"}, - {file = "rpds_py-0.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9ea4d00850ef1e917815e59b078ecb338f6a8efda23369677c54a5825dbebb55"}, - {file = "rpds_py-0.9.2-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:5855c85eb8b8a968a74dc7fb014c9166a05e7e7a8377fb91d78512900aadd13d"}, - {file = "rpds_py-0.9.2-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:14c408e9d1a80dcb45c05a5149e5961aadb912fff42ca1dd9b68c0044904eb32"}, - {file = "rpds_py-0.9.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:65a0583c43d9f22cb2130c7b110e695fff834fd5e832a776a107197e59a1898e"}, - {file = "rpds_py-0.9.2-cp38-none-win32.whl", hash = "sha256:71f2f7715935a61fa3e4ae91d91b67e571aeb5cb5d10331ab681256bda2ad920"}, - {file = "rpds_py-0.9.2-cp38-none-win_amd64.whl", hash = "sha256:674c704605092e3ebbbd13687b09c9f78c362a4bc710343efe37a91457123044"}, - {file = "rpds_py-0.9.2-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:07e2c54bef6838fa44c48dfbc8234e8e2466d851124b551fc4e07a1cfeb37260"}, - {file = "rpds_py-0.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f7fdf55283ad38c33e35e2855565361f4bf0abd02470b8ab28d499c663bc5d7c"}, - {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:890ba852c16ace6ed9f90e8670f2c1c178d96510a21b06d2fa12d8783a905193"}, - {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:50025635ba8b629a86d9d5474e650da304cb46bbb4d18690532dd79341467846"}, - {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:517cbf6e67ae3623c5127206489d69eb2bdb27239a3c3cc559350ef52a3bbf0b"}, - {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0836d71ca19071090d524739420a61580f3f894618d10b666cf3d9a1688355b1"}, - {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c439fd54b2b9053717cca3de9583be6584b384d88d045f97d409f0ca867d80f"}, - {file = "rpds_py-0.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f68996a3b3dc9335037f82754f9cdbe3a95db42bde571d8c3be26cc6245f2324"}, - {file = "rpds_py-0.9.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:7d68dc8acded354c972116f59b5eb2e5864432948e098c19fe6994926d8e15c3"}, - {file = "rpds_py-0.9.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:f963c6b1218b96db85fc37a9f0851eaf8b9040aa46dec112611697a7023da535"}, - {file = "rpds_py-0.9.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5a46859d7f947061b4010e554ccd1791467d1b1759f2dc2ec9055fa239f1bc26"}, - {file = "rpds_py-0.9.2-cp39-none-win32.whl", hash = "sha256:e07e5dbf8a83c66783a9fe2d4566968ea8c161199680e8ad38d53e075df5f0d0"}, - {file = "rpds_py-0.9.2-cp39-none-win_amd64.whl", hash = "sha256:682726178138ea45a0766907957b60f3a1bf3acdf212436be9733f28b6c5af3c"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:196cb208825a8b9c8fc360dc0f87993b8b260038615230242bf18ec84447c08d"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:c7671d45530fcb6d5e22fd40c97e1e1e01965fc298cbda523bb640f3d923b387"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83b32f0940adec65099f3b1c215ef7f1d025d13ff947975a055989cb7fd019a4"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f67da97f5b9eac838b6980fc6da268622e91f8960e083a34533ca710bec8611"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:03975db5f103997904c37e804e5f340c8fdabbb5883f26ee50a255d664eed58c"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:987b06d1cdb28f88a42e4fb8a87f094e43f3c435ed8e486533aea0bf2e53d931"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c861a7e4aef15ff91233751619ce3a3d2b9e5877e0fcd76f9ea4f6847183aa16"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:02938432352359805b6da099c9c95c8a0547fe4b274ce8f1a91677401bb9a45f"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:ef1f08f2a924837e112cba2953e15aacfccbbfcd773b4b9b4723f8f2ddded08e"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:35da5cc5cb37c04c4ee03128ad59b8c3941a1e5cd398d78c37f716f32a9b7f67"}, - {file = "rpds_py-0.9.2-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:141acb9d4ccc04e704e5992d35472f78c35af047fa0cfae2923835d153f091be"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:79f594919d2c1a0cc17d1988a6adaf9a2f000d2e1048f71f298b056b1018e872"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:a06418fe1155e72e16dddc68bb3780ae44cebb2912fbd8bb6ff9161de56e1798"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b2eb034c94b0b96d5eddb290b7b5198460e2d5d0c421751713953a9c4e47d10"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8b08605d248b974eb02f40bdcd1a35d3924c83a2a5e8f5d0fa5af852c4d960af"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a0805911caedfe2736935250be5008b261f10a729a303f676d3d5fea6900c96a"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ab2299e3f92aa5417d5e16bb45bb4586171c1327568f638e8453c9f8d9e0f020"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c8d7594e38cf98d8a7df25b440f684b510cf4627fe038c297a87496d10a174f"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8b9ec12ad5f0a4625db34db7e0005be2632c1013b253a4a60e8302ad4d462afd"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:1fcdee18fea97238ed17ab6478c66b2095e4ae7177e35fb71fbe561a27adf620"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-musllinux_1_2_i686.whl", hash = "sha256:933a7d5cd4b84f959aedeb84f2030f0a01d63ae6cf256629af3081cf3e3426e8"}, - {file = "rpds_py-0.9.2-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:686ba516e02db6d6f8c279d1641f7067ebb5dc58b1d0536c4aaebb7bf01cdc5d"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:0173c0444bec0a3d7d848eaeca2d8bd32a1b43f3d3fde6617aac3731fa4be05f"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:d576c3ef8c7b2d560e301eb33891d1944d965a4d7a2eacb6332eee8a71827db6"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed89861ee8c8c47d6beb742a602f912b1bb64f598b1e2f3d758948721d44d468"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1054a08e818f8e18910f1bee731583fe8f899b0a0a5044c6e680ceea34f93876"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99e7c4bb27ff1aab90dcc3e9d37ee5af0231ed98d99cb6f5250de28889a3d502"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c545d9d14d47be716495076b659db179206e3fd997769bc01e2d550eeb685596"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9039a11bca3c41be5a58282ed81ae422fa680409022b996032a43badef2a3752"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fb39aca7a64ad0c9490adfa719dbeeb87d13be137ca189d2564e596f8ba32c07"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:2d8b3b3a2ce0eaa00c5bbbb60b6713e94e7e0becab7b3db6c5c77f979e8ed1f1"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:99b1c16f732b3a9971406fbfe18468592c5a3529585a45a35adbc1389a529a03"}, - {file = "rpds_py-0.9.2-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:c27ee01a6c3223025f4badd533bea5e87c988cb0ba2811b690395dfe16088cfe"}, - {file = "rpds_py-0.9.2.tar.gz", hash = "sha256:8d70e8f14900f2657c249ea4def963bed86a29b81f81f5b76b5a9215680de945"}, + {file = "rpds_py-0.17.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:4128980a14ed805e1b91a7ed551250282a8ddf8201a4e9f8f5b7e6225f54170d"}, + {file = "rpds_py-0.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ff1dcb8e8bc2261a088821b2595ef031c91d499a0c1b031c152d43fe0a6ecec8"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d65e6b4f1443048eb7e833c2accb4fa7ee67cc7d54f31b4f0555b474758bee55"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a71169d505af63bb4d20d23a8fbd4c6ce272e7bce6cc31f617152aa784436f29"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:436474f17733c7dca0fbf096d36ae65277e8645039df12a0fa52445ca494729d"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:10162fe3f5f47c37ebf6d8ff5a2368508fe22007e3077bf25b9c7d803454d921"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:720215373a280f78a1814becb1312d4e4d1077b1202a56d2b0815e95ccb99ce9"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:70fcc6c2906cfa5c6a552ba7ae2ce64b6c32f437d8f3f8eea49925b278a61453"}, + {file = "rpds_py-0.17.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:91e5a8200e65aaac342a791272c564dffcf1281abd635d304d6c4e6b495f29dc"}, + {file = "rpds_py-0.17.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:99f567dae93e10be2daaa896e07513dd4bf9c2ecf0576e0533ac36ba3b1d5394"}, + {file = "rpds_py-0.17.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:24e4900a6643f87058a27320f81336d527ccfe503984528edde4bb660c8c8d59"}, + {file = "rpds_py-0.17.1-cp310-none-win32.whl", hash = "sha256:0bfb09bf41fe7c51413f563373e5f537eaa653d7adc4830399d4e9bdc199959d"}, + {file = "rpds_py-0.17.1-cp310-none-win_amd64.whl", hash = "sha256:20de7b7179e2031a04042e85dc463a93a82bc177eeba5ddd13ff746325558aa6"}, + {file = "rpds_py-0.17.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:65dcf105c1943cba45d19207ef51b8bc46d232a381e94dd38719d52d3980015b"}, + {file = "rpds_py-0.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:01f58a7306b64e0a4fe042047dd2b7d411ee82e54240284bab63e325762c1147"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:071bc28c589b86bc6351a339114fb7a029f5cddbaca34103aa573eba7b482382"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ae35e8e6801c5ab071b992cb2da958eee76340e6926ec693b5ff7d6381441745"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:149c5cd24f729e3567b56e1795f74577aa3126c14c11e457bec1b1c90d212e38"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e796051f2070f47230c745d0a77a91088fbee2cc0502e9b796b9c6471983718c"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60e820ee1004327609b28db8307acc27f5f2e9a0b185b2064c5f23e815f248f8"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1957a2ab607f9added64478a6982742eb29f109d89d065fa44e01691a20fc20a"}, + {file = "rpds_py-0.17.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8587fd64c2a91c33cdc39d0cebdaf30e79491cc029a37fcd458ba863f8815383"}, + {file = "rpds_py-0.17.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4dc889a9d8a34758d0fcc9ac86adb97bab3fb7f0c4d29794357eb147536483fd"}, + {file = "rpds_py-0.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2953937f83820376b5979318840f3ee47477d94c17b940fe31d9458d79ae7eea"}, + {file = "rpds_py-0.17.1-cp311-none-win32.whl", hash = "sha256:1bfcad3109c1e5ba3cbe2f421614e70439f72897515a96c462ea657261b96518"}, + {file = "rpds_py-0.17.1-cp311-none-win_amd64.whl", hash = "sha256:99da0a4686ada4ed0f778120a0ea8d066de1a0a92ab0d13ae68492a437db78bf"}, + {file = "rpds_py-0.17.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:1dc29db3900cb1bb40353772417800f29c3d078dbc8024fd64655a04ee3c4bdf"}, + {file = "rpds_py-0.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:82ada4a8ed9e82e443fcef87e22a3eed3654dd3adf6e3b3a0deb70f03e86142a"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d36b2b59e8cc6e576f8f7b671e32f2ff43153f0ad6d0201250a7c07f25d570e"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3677fcca7fb728c86a78660c7fb1b07b69b281964673f486ae72860e13f512ad"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:516fb8c77805159e97a689e2f1c80655c7658f5af601c34ffdb916605598cda2"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df3b6f45ba4515632c5064e35ca7f31d51d13d1479673185ba8f9fefbbed58b9"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a967dd6afda7715d911c25a6ba1517975acd8d1092b2f326718725461a3d33f9"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dbbb95e6fc91ea3102505d111b327004d1c4ce98d56a4a02e82cd451f9f57140"}, + {file = "rpds_py-0.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:02866e060219514940342a1f84303a1ef7a1dad0ac311792fbbe19b521b489d2"}, + {file = "rpds_py-0.17.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:2528ff96d09f12e638695f3a2e0c609c7b84c6df7c5ae9bfeb9252b6fa686253"}, + {file = "rpds_py-0.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bd345a13ce06e94c753dab52f8e71e5252aec1e4f8022d24d56decd31e1b9b23"}, + {file = "rpds_py-0.17.1-cp312-none-win32.whl", hash = "sha256:2a792b2e1d3038daa83fa474d559acfd6dc1e3650ee93b2662ddc17dbff20ad1"}, + {file = "rpds_py-0.17.1-cp312-none-win_amd64.whl", hash = "sha256:292f7344a3301802e7c25c53792fae7d1593cb0e50964e7bcdcc5cf533d634e3"}, + {file = "rpds_py-0.17.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:8ffe53e1d8ef2520ebcf0c9fec15bb721da59e8ef283b6ff3079613b1e30513d"}, + {file = "rpds_py-0.17.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4341bd7579611cf50e7b20bb8c2e23512a3dc79de987a1f411cb458ab670eb90"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f4eb548daf4836e3b2c662033bfbfc551db58d30fd8fe660314f86bf8510b93"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b686f25377f9c006acbac63f61614416a6317133ab7fafe5de5f7dc8a06d42eb"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4e21b76075c01d65d0f0f34302b5a7457d95721d5e0667aea65e5bb3ab415c25"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b86b21b348f7e5485fae740d845c65a880f5d1eda1e063bc59bef92d1f7d0c55"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f175e95a197f6a4059b50757a3dca33b32b61691bdbd22c29e8a8d21d3914cae"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1701fc54460ae2e5efc1dd6350eafd7a760f516df8dbe51d4a1c79d69472fbd4"}, + {file = "rpds_py-0.17.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:9051e3d2af8f55b42061603e29e744724cb5f65b128a491446cc029b3e2ea896"}, + {file = "rpds_py-0.17.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:7450dbd659fed6dd41d1a7d47ed767e893ba402af8ae664c157c255ec6067fde"}, + {file = "rpds_py-0.17.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:5a024fa96d541fd7edaa0e9d904601c6445e95a729a2900c5aec6555fe921ed6"}, + {file = "rpds_py-0.17.1-cp38-none-win32.whl", hash = "sha256:da1ead63368c04a9bded7904757dfcae01eba0e0f9bc41d3d7f57ebf1c04015a"}, + {file = "rpds_py-0.17.1-cp38-none-win_amd64.whl", hash = "sha256:841320e1841bb53fada91c9725e766bb25009cfd4144e92298db296fb6c894fb"}, + {file = "rpds_py-0.17.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:f6c43b6f97209e370124baf2bf40bb1e8edc25311a158867eb1c3a5d449ebc7a"}, + {file = "rpds_py-0.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5e7d63ec01fe7c76c2dbb7e972fece45acbb8836e72682bde138e7e039906e2c"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81038ff87a4e04c22e1d81f947c6ac46f122e0c80460b9006e6517c4d842a6ec"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:810685321f4a304b2b55577c915bece4c4a06dfe38f6e62d9cc1d6ca8ee86b99"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:25f071737dae674ca8937a73d0f43f5a52e92c2d178330b4c0bb6ab05586ffa6"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aa5bfb13f1e89151ade0eb812f7b0d7a4d643406caaad65ce1cbabe0a66d695f"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dfe07308b311a8293a0d5ef4e61411c5c20f682db6b5e73de6c7c8824272c256"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a000133a90eea274a6f28adc3084643263b1e7c1a5a66eb0a0a7a36aa757ed74"}, + {file = "rpds_py-0.17.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5d0e8a6434a3fbf77d11448c9c25b2f25244226cfbec1a5159947cac5b8c5fa4"}, + {file = "rpds_py-0.17.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:efa767c220d94aa4ac3a6dd3aeb986e9f229eaf5bce92d8b1b3018d06bed3772"}, + {file = "rpds_py-0.17.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:dbc56680ecf585a384fbd93cd42bc82668b77cb525343170a2d86dafaed2a84b"}, + {file = "rpds_py-0.17.1-cp39-none-win32.whl", hash = "sha256:270987bc22e7e5a962b1094953ae901395e8c1e1e83ad016c5cfcfff75a15a3f"}, + {file = "rpds_py-0.17.1-cp39-none-win_amd64.whl", hash = "sha256:2a7b2f2f56a16a6d62e55354dd329d929560442bd92e87397b7a9586a32e3e76"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a3264e3e858de4fc601741498215835ff324ff2482fd4e4af61b46512dd7fc83"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:f2f3b28b40fddcb6c1f1f6c88c6f3769cd933fa493ceb79da45968a21dccc920"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9584f8f52010295a4a417221861df9bea4c72d9632562b6e59b3c7b87a1522b7"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c64602e8be701c6cfe42064b71c84ce62ce66ddc6422c15463fd8127db3d8066"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:060f412230d5f19fc8c8b75f315931b408d8ebf56aec33ef4168d1b9e54200b1"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b9412abdf0ba70faa6e2ee6c0cc62a8defb772e78860cef419865917d86c7342"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9737bdaa0ad33d34c0efc718741abaafce62fadae72c8b251df9b0c823c63b22"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9f0e4dc0f17dcea4ab9d13ac5c666b6b5337042b4d8f27e01b70fae41dd65c57"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:1db228102ab9d1ff4c64148c96320d0be7044fa28bd865a9ce628ce98da5973d"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:d8bbd8e56f3ba25a7d0cf980fc42b34028848a53a0e36c9918550e0280b9d0b6"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:be22ae34d68544df293152b7e50895ba70d2a833ad9566932d750d3625918b82"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:bf046179d011e6114daf12a534d874958b039342b347348a78b7cdf0dd9d6041"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:1a746a6d49665058a5896000e8d9d2f1a6acba8a03b389c1e4c06e11e0b7f40d"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0b8bf5b8db49d8fd40f54772a1dcf262e8be0ad2ab0206b5a2ec109c176c0a4"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f7f4cb1f173385e8a39c29510dd11a78bf44e360fb75610594973f5ea141028b"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7fbd70cb8b54fe745301921b0816c08b6d917593429dfc437fd024b5ba713c58"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9bdf1303df671179eaf2cb41e8515a07fc78d9d00f111eadbe3e14262f59c3d0"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fad059a4bd14c45776600d223ec194e77db6c20255578bb5bcdd7c18fd169361"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3664d126d3388a887db44c2e293f87d500c4184ec43d5d14d2d2babdb4c64cad"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:698ea95a60c8b16b58be9d854c9f993c639f5c214cf9ba782eca53a8789d6b19"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-musllinux_1_2_i686.whl", hash = "sha256:c3d2010656999b63e628a3c694f23020322b4178c450dc478558a2b6ef3cb9bb"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:938eab7323a736533f015e6069a7d53ef2dcc841e4e533b782c2bfb9fb12d84b"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1e626b365293a2142a62b9a614e1f8e331b28f3ca57b9f05ebbf4cf2a0f0bdc5"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:380e0df2e9d5d5d339803cfc6d183a5442ad7ab3c63c2a0982e8c824566c5ccc"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b760a56e080a826c2e5af09002c1a037382ed21d03134eb6294812dda268c811"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5576ee2f3a309d2bb403ec292d5958ce03953b0e57a11d224c1f134feaf8c40f"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f3c3461ebb4c4f1bbc70b15d20b565759f97a5aaf13af811fcefc892e9197ba"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:637b802f3f069a64436d432117a7e58fab414b4e27a7e81049817ae94de45d8d"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffee088ea9b593cc6160518ba9bd319b5475e5f3e578e4552d63818773c6f56a"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3ac732390d529d8469b831949c78085b034bff67f584559340008d0f6041a049"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:93432e747fb07fa567ad9cc7aaadd6e29710e515aabf939dfbed8046041346c6"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:7b7d9ca34542099b4e185b3c2a2b2eda2e318a7dbde0b0d83357a6d4421b5296"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:0387ce69ba06e43df54e43968090f3626e231e4bc9150e4c3246947567695f68"}, + {file = "rpds_py-0.17.1.tar.gz", hash = "sha256:0210b2668f24c078307260bf88bdac9d6f1093635df5123789bfee4d8d7fc8e7"}, ] [[package]] @@ -1312,13 +1476,13 @@ files = [ [[package]] name = "termcolor" -version = "2.3.0" +version = "2.4.0" description = "ANSI color formatting for output in terminal" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "termcolor-2.3.0-py3-none-any.whl", hash = "sha256:3afb05607b89aed0ffe25202399ee0867ad4d3cb4180d98aaf8eefa6a5f7d475"}, - {file = "termcolor-2.3.0.tar.gz", hash = "sha256:b5b08f68937f138fe92f6c089b99f1e2da0ae56c52b78bf7075fd95420fd9a5a"}, + {file = "termcolor-2.4.0-py3-none-any.whl", hash = "sha256:9297c0df9c99445c2412e832e882a7884038a25617c60cea2ad69488d4040d63"}, + {file = "termcolor-2.4.0.tar.gz", hash = "sha256:aab9e56047c8ac41ed798fa36d892a37aca6b3e9159f3e0c24bc64a9b3ac7b7a"}, ] [package.extras] @@ -1348,138 +1512,172 @@ files = [ [[package]] name = "typing-extensions" -version = "4.7.1" -description = "Backported and Experimental Type Hints for Python 3.7+" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, - {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "tzdata" +version = "2023.4" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2023.4-py2.py3-none-any.whl", hash = "sha256:aa3ace4329eeacda5b7beb7ea08ece826c28d761cda36e747cfbf97996d39bf3"}, + {file = "tzdata-2023.4.tar.gz", hash = "sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9"}, ] [[package]] name = "urllib3" -version = "1.26.16" +version = "1.26.18" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ - {file = "urllib3-1.26.16-py2.py3-none-any.whl", hash = "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f"}, - {file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"}, + {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, + {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "websocket-client" -version = "1.6.1" +version = "1.7.0" description = "WebSocket client for Python with low level API options" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "websocket-client-1.6.1.tar.gz", hash = "sha256:c951af98631d24f8df89ab1019fc365f2227c0892f12fd150e935607c79dd0dd"}, - {file = "websocket_client-1.6.1-py3-none-any.whl", hash = "sha256:f1f9f2ad5291f0225a49efad77abf9e700b6fef553900623060dad6e26503b9d"}, + {file = "websocket-client-1.7.0.tar.gz", hash = "sha256:10e511ea3a8c744631d3bd77e61eb17ed09304c413ad42cf6ddfa4c7787e8fe6"}, + {file = "websocket_client-1.7.0-py3-none-any.whl", hash = "sha256:f4c3d22fec12a2461427a29957ff07d35098ee2d976d3ba244e688b8b4057588"}, ] [package.extras] -docs = ["Sphinx (>=3.4)", "sphinx-rtd-theme (>=0.5)"] +docs = ["Sphinx (>=6.0)", "sphinx-rtd-theme (>=1.1.0)"] optional = ["python-socks", "wsaccel"] test = ["websockets"] [[package]] name = "wmctrl" -version = "0.4" +version = "0.5" description = "A tool to programmatically control windows inside X" optional = false -python-versions = "*" +python-versions = ">=2.7" files = [ - {file = "wmctrl-0.4.tar.gz", hash = "sha256:66cbff72b0ca06a22ec3883ac3a4d7c41078bdae4fb7310f52951769b10e14e0"}, + {file = "wmctrl-0.5-py2.py3-none-any.whl", hash = "sha256:ae695c1863a314c899e7cf113f07c0da02a394b968c4772e1936219d9234ddd7"}, + {file = "wmctrl-0.5.tar.gz", hash = "sha256:7839a36b6fe9e2d6fd22304e5dc372dbced2116ba41283ea938b2da57f53e962"}, ] +[package.dependencies] +attrs = "*" + +[package.extras] +test = ["pytest"] + [[package]] name = "yarl" -version = "1.9.2" +version = "1.9.4" description = "Yet another URL library" optional = false python-versions = ">=3.7" files = [ - {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c2ad583743d16ddbdf6bb14b5cd76bf43b0d0006e918809d5d4ddf7bde8dd82"}, - {file = "yarl-1.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:82aa6264b36c50acfb2424ad5ca537a2060ab6de158a5bd2a72a032cc75b9eb8"}, - {file = "yarl-1.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c0c77533b5ed4bcc38e943178ccae29b9bcf48ffd1063f5821192f23a1bd27b9"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ee4afac41415d52d53a9833ebae7e32b344be72835bbb589018c9e938045a560"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9bf345c3a4f5ba7f766430f97f9cc1320786f19584acc7086491f45524a551ac"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2a96c19c52ff442a808c105901d0bdfd2e28575b3d5f82e2f5fd67e20dc5f4ea"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:891c0e3ec5ec881541f6c5113d8df0315ce5440e244a716b95f2525b7b9f3608"}, - {file = "yarl-1.9.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c3a53ba34a636a256d767c086ceb111358876e1fb6b50dfc4d3f4951d40133d5"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:566185e8ebc0898b11f8026447eacd02e46226716229cea8db37496c8cdd26e0"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:2b0738fb871812722a0ac2154be1f049c6223b9f6f22eec352996b69775b36d4"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:32f1d071b3f362c80f1a7d322bfd7b2d11e33d2adf395cc1dd4df36c9c243095"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:e9fdc7ac0d42bc3ea78818557fab03af6181e076a2944f43c38684b4b6bed8e3"}, - {file = "yarl-1.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:56ff08ab5df8429901ebdc5d15941b59f6253393cb5da07b4170beefcf1b2528"}, - {file = "yarl-1.9.2-cp310-cp310-win32.whl", hash = "sha256:8ea48e0a2f931064469bdabca50c2f578b565fc446f302a79ba6cc0ee7f384d3"}, - {file = "yarl-1.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:50f33040f3836e912ed16d212f6cc1efb3231a8a60526a407aeb66c1c1956dde"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:646d663eb2232d7909e6601f1a9107e66f9791f290a1b3dc7057818fe44fc2b6"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:aff634b15beff8902d1f918012fc2a42e0dbae6f469fce134c8a0dc51ca423bb"}, - {file = "yarl-1.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a83503934c6273806aed765035716216cc9ab4e0364f7f066227e1aaea90b8d0"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b25322201585c69abc7b0e89e72790469f7dad90d26754717f3310bfe30331c2"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:22a94666751778629f1ec4280b08eb11815783c63f52092a5953faf73be24191"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ec53a0ea2a80c5cd1ab397925f94bff59222aa3cf9c6da938ce05c9ec20428d"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:159d81f22d7a43e6eabc36d7194cb53f2f15f498dbbfa8edc8a3239350f59fe7"}, - {file = "yarl-1.9.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:832b7e711027c114d79dffb92576acd1bd2decc467dec60e1cac96912602d0e6"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:95d2ecefbcf4e744ea952d073c6922e72ee650ffc79028eb1e320e732898d7e8"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d4e2c6d555e77b37288eaf45b8f60f0737c9efa3452c6c44626a5455aeb250b9"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:783185c75c12a017cc345015ea359cc801c3b29a2966c2655cd12b233bf5a2be"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:b8cc1863402472f16c600e3e93d542b7e7542a540f95c30afd472e8e549fc3f7"}, - {file = "yarl-1.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:822b30a0f22e588b32d3120f6d41e4ed021806418b4c9f0bc3048b8c8cb3f92a"}, - {file = "yarl-1.9.2-cp311-cp311-win32.whl", hash = "sha256:a60347f234c2212a9f0361955007fcf4033a75bf600a33c88a0a8e91af77c0e8"}, - {file = "yarl-1.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:be6b3fdec5c62f2a67cb3f8c6dbf56bbf3f61c0f046f84645cd1ca73532ea051"}, - {file = "yarl-1.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38a3928ae37558bc1b559f67410df446d1fbfa87318b124bf5032c31e3447b74"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac9bb4c5ce3975aeac288cfcb5061ce60e0d14d92209e780c93954076c7c4367"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3da8a678ca8b96c8606bbb8bfacd99a12ad5dd288bc6f7979baddd62f71c63ef"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13414591ff516e04fcdee8dc051c13fd3db13b673c7a4cb1350e6b2ad9639ad3"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf74d08542c3a9ea97bb8f343d4fcbd4d8f91bba5ec9d5d7f792dbe727f88938"}, - {file = "yarl-1.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e7221580dc1db478464cfeef9b03b95c5852cc22894e418562997df0d074ccc"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:494053246b119b041960ddcd20fd76224149cfea8ed8777b687358727911dd33"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:52a25809fcbecfc63ac9ba0c0fb586f90837f5425edfd1ec9f3372b119585e45"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:e65610c5792870d45d7b68c677681376fcf9cc1c289f23e8e8b39c1485384185"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:1b1bba902cba32cdec51fca038fd53f8beee88b77efc373968d1ed021024cc04"}, - {file = "yarl-1.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:662e6016409828ee910f5d9602a2729a8a57d74b163c89a837de3fea050c7582"}, - {file = "yarl-1.9.2-cp37-cp37m-win32.whl", hash = "sha256:f364d3480bffd3aa566e886587eaca7c8c04d74f6e8933f3f2c996b7f09bee1b"}, - {file = "yarl-1.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6a5883464143ab3ae9ba68daae8e7c5c95b969462bbe42e2464d60e7e2698368"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5610f80cf43b6202e2c33ba3ec2ee0a2884f8f423c8f4f62906731d876ef4fac"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b9a4e67ad7b646cd6f0938c7ebfd60e481b7410f574c560e455e938d2da8e0f4"}, - {file = "yarl-1.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:83fcc480d7549ccebe9415d96d9263e2d4226798c37ebd18c930fce43dfb9574"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fcd436ea16fee7d4207c045b1e340020e58a2597301cfbcfdbe5abd2356c2fb"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84e0b1599334b1e1478db01b756e55937d4614f8654311eb26012091be109d59"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3458a24e4ea3fd8930e934c129b676c27452e4ebda80fbe47b56d8c6c7a63a9e"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:838162460b3a08987546e881a2bfa573960bb559dfa739e7800ceeec92e64417"}, - {file = "yarl-1.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f4e2d08f07a3d7d3e12549052eb5ad3eab1c349c53ac51c209a0e5991bbada78"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:de119f56f3c5f0e2fb4dee508531a32b069a5f2c6e827b272d1e0ff5ac040333"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:149ddea5abf329752ea5051b61bd6c1d979e13fbf122d3a1f9f0c8be6cb6f63c"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:674ca19cbee4a82c9f54e0d1eee28116e63bc6fd1e96c43031d11cbab8b2afd5"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:9b3152f2f5677b997ae6c804b73da05a39daa6a9e85a512e0e6823d81cdad7cc"}, - {file = "yarl-1.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5415d5a4b080dc9612b1b63cba008db84e908b95848369aa1da3686ae27b6d2b"}, - {file = "yarl-1.9.2-cp38-cp38-win32.whl", hash = "sha256:f7a3d8146575e08c29ed1cd287068e6d02f1c7bdff8970db96683b9591b86ee7"}, - {file = "yarl-1.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:63c48f6cef34e6319a74c727376e95626f84ea091f92c0250a98e53e62c77c72"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:75df5ef94c3fdc393c6b19d80e6ef1ecc9ae2f4263c09cacb178d871c02a5ba9"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c027a6e96ef77d401d8d5a5c8d6bc478e8042f1e448272e8d9752cb0aff8b5c8"}, - {file = "yarl-1.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3b078dbe227f79be488ffcfc7a9edb3409d018e0952cf13f15fd6512847f3f7"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59723a029760079b7d991a401386390c4be5bfec1e7dd83e25a6a0881859e716"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b03917871bf859a81ccb180c9a2e6c1e04d2f6a51d953e6a5cdd70c93d4e5a2a"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c1012fa63eb6c032f3ce5d2171c267992ae0c00b9e164efe4d73db818465fac3"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a74dcbfe780e62f4b5a062714576f16c2f3493a0394e555ab141bf0d746bb955"}, - {file = "yarl-1.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8c56986609b057b4839968ba901944af91b8e92f1725d1a2d77cbac6972b9ed1"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2c315df3293cd521033533d242d15eab26583360b58f7ee5d9565f15fee1bef4"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b7232f8dfbd225d57340e441d8caf8652a6acd06b389ea2d3222b8bc89cbfca6"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:53338749febd28935d55b41bf0bcc79d634881195a39f6b2f767870b72514caf"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:066c163aec9d3d073dc9ffe5dd3ad05069bcb03fcaab8d221290ba99f9f69ee3"}, - {file = "yarl-1.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8288d7cd28f8119b07dd49b7230d6b4562f9b61ee9a4ab02221060d21136be80"}, - {file = "yarl-1.9.2-cp39-cp39-win32.whl", hash = "sha256:b124e2a6d223b65ba8768d5706d103280914d61f5cae3afbc50fc3dfcc016623"}, - {file = "yarl-1.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:61016e7d582bc46a5378ffdd02cd0314fb8ba52f40f9cf4d9a5e7dbef88dee18"}, - {file = "yarl-1.9.2.tar.gz", hash = "sha256:04ab9d4b9f587c06d801c2abfe9317b77cdf996c65a90d5e84ecc45010823571"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a8c1df72eb746f4136fe9a2e72b0c9dc1da1cbd23b5372f94b5820ff8ae30e0e"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a3a6ed1d525bfb91b3fc9b690c5a21bb52de28c018530ad85093cc488bee2dd2"}, + {file = "yarl-1.9.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c38c9ddb6103ceae4e4498f9c08fac9b590c5c71b0370f98714768e22ac6fa66"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9e09c9d74f4566e905a0b8fa668c58109f7624db96a2171f21747abc7524234"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8477c1ee4bd47c57d49621a062121c3023609f7a13b8a46953eb6c9716ca392"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5ff2c858f5f6a42c2a8e751100f237c5e869cbde669a724f2062d4c4ef93551"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:357495293086c5b6d34ca9616a43d329317feab7917518bc97a08f9e55648455"}, + {file = "yarl-1.9.4-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54525ae423d7b7a8ee81ba189f131054defdb122cde31ff17477951464c1691c"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:801e9264d19643548651b9db361ce3287176671fb0117f96b5ac0ee1c3530d53"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e516dc8baf7b380e6c1c26792610230f37147bb754d6426462ab115a02944385"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7d5aaac37d19b2904bb9dfe12cdb08c8443e7ba7d2852894ad448d4b8f442863"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:54beabb809ffcacbd9d28ac57b0db46e42a6e341a030293fb3185c409e626b8b"}, + {file = "yarl-1.9.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bac8d525a8dbc2a1507ec731d2867025d11ceadcb4dd421423a5d42c56818541"}, + {file = "yarl-1.9.4-cp310-cp310-win32.whl", hash = "sha256:7855426dfbddac81896b6e533ebefc0af2f132d4a47340cee6d22cac7190022d"}, + {file = "yarl-1.9.4-cp310-cp310-win_amd64.whl", hash = "sha256:848cd2a1df56ddbffeb375535fb62c9d1645dde33ca4d51341378b3f5954429b"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:35a2b9396879ce32754bd457d31a51ff0a9d426fd9e0e3c33394bf4b9036b099"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c7d56b293cc071e82532f70adcbd8b61909eec973ae9d2d1f9b233f3d943f2c"}, + {file = "yarl-1.9.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d8a1c6c0be645c745a081c192e747c5de06e944a0d21245f4cf7c05e457c36e0"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b3c1ffe10069f655ea2d731808e76e0f452fc6c749bea04781daf18e6039525"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:549d19c84c55d11687ddbd47eeb348a89df9cb30e1993f1b128f4685cd0ebbf8"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a7409f968456111140c1c95301cadf071bd30a81cbd7ab829169fb9e3d72eae9"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e23a6d84d9d1738dbc6e38167776107e63307dfc8ad108e580548d1f2c587f42"}, + {file = "yarl-1.9.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8b889777de69897406c9fb0b76cdf2fd0f31267861ae7501d93003d55f54fbe"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:03caa9507d3d3c83bca08650678e25364e1843b484f19986a527630ca376ecce"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4e9035df8d0880b2f1c7f5031f33f69e071dfe72ee9310cfc76f7b605958ceb9"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:c0ec0ed476f77db9fb29bca17f0a8fcc7bc97ad4c6c1d8959c507decb22e8572"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:ee04010f26d5102399bd17f8df8bc38dc7ccd7701dc77f4a68c5b8d733406958"}, + {file = "yarl-1.9.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49a180c2e0743d5d6e0b4d1a9e5f633c62eca3f8a86ba5dd3c471060e352ca98"}, + {file = "yarl-1.9.4-cp311-cp311-win32.whl", hash = "sha256:81eb57278deb6098a5b62e88ad8281b2ba09f2f1147c4767522353eaa6260b31"}, + {file = "yarl-1.9.4-cp311-cp311-win_amd64.whl", hash = "sha256:d1d2532b340b692880261c15aee4dc94dd22ca5d61b9db9a8a361953d36410b1"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0d2454f0aef65ea81037759be5ca9947539667eecebca092733b2eb43c965a81"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:44d8ffbb9c06e5a7f529f38f53eda23e50d1ed33c6c869e01481d3fafa6b8142"}, + {file = "yarl-1.9.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:aaaea1e536f98754a6e5c56091baa1b6ce2f2700cc4a00b0d49eca8dea471074"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3777ce5536d17989c91696db1d459574e9a9bd37660ea7ee4d3344579bb6f129"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9fc5fc1eeb029757349ad26bbc5880557389a03fa6ada41703db5e068881e5f2"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea65804b5dc88dacd4a40279af0cdadcfe74b3e5b4c897aa0d81cf86927fee78"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aa102d6d280a5455ad6a0f9e6d769989638718e938a6a0a2ff3f4a7ff8c62cc4"}, + {file = "yarl-1.9.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09efe4615ada057ba2d30df871d2f668af661e971dfeedf0c159927d48bbeff0"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:008d3e808d03ef28542372d01057fd09168419cdc8f848efe2804f894ae03e51"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6f5cb257bc2ec58f437da2b37a8cd48f666db96d47b8a3115c29f316313654ff"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:992f18e0ea248ee03b5a6e8b3b4738850ae7dbb172cc41c966462801cbf62cf7"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:0e9d124c191d5b881060a9e5060627694c3bdd1fe24c5eecc8d5d7d0eb6faabc"}, + {file = "yarl-1.9.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3986b6f41ad22988e53d5778f91855dc0399b043fc8946d4f2e68af22ee9ff10"}, + {file = "yarl-1.9.4-cp312-cp312-win32.whl", hash = "sha256:4b21516d181cd77ebd06ce160ef8cc2a5e9ad35fb1c5930882baff5ac865eee7"}, + {file = "yarl-1.9.4-cp312-cp312-win_amd64.whl", hash = "sha256:a9bd00dc3bc395a662900f33f74feb3e757429e545d831eef5bb280252631984"}, + {file = "yarl-1.9.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:63b20738b5aac74e239622d2fe30df4fca4942a86e31bf47a81a0e94c14df94f"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7d7f7de27b8944f1fee2c26a88b4dabc2409d2fea7a9ed3df79b67277644e17"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c74018551e31269d56fab81a728f683667e7c28c04e807ba08f8c9e3bba32f14"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ca06675212f94e7a610e85ca36948bb8fc023e458dd6c63ef71abfd482481aa5"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5aef935237d60a51a62b86249839b51345f47564208c6ee615ed2a40878dccdd"}, + {file = "yarl-1.9.4-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b134fd795e2322b7684155b7855cc99409d10b2e408056db2b93b51a52accc7"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d25039a474c4c72a5ad4b52495056f843a7ff07b632c1b92ea9043a3d9950f6e"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f7d6b36dd2e029b6bcb8a13cf19664c7b8e19ab3a58e0fefbb5b8461447ed5ec"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:957b4774373cf6f709359e5c8c4a0af9f6d7875db657adb0feaf8d6cb3c3964c"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:d7eeb6d22331e2fd42fce928a81c697c9ee2d51400bd1a28803965883e13cead"}, + {file = "yarl-1.9.4-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6a962e04b8f91f8c4e5917e518d17958e3bdee71fd1d8b88cdce74dd0ebbf434"}, + {file = "yarl-1.9.4-cp37-cp37m-win32.whl", hash = "sha256:f3bc6af6e2b8f92eced34ef6a96ffb248e863af20ef4fde9448cc8c9b858b749"}, + {file = "yarl-1.9.4-cp37-cp37m-win_amd64.whl", hash = "sha256:ad4d7a90a92e528aadf4965d685c17dacff3df282db1121136c382dc0b6014d2"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ec61d826d80fc293ed46c9dd26995921e3a82146feacd952ef0757236fc137be"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8be9e837ea9113676e5754b43b940b50cce76d9ed7d2461df1af39a8ee674d9f"}, + {file = "yarl-1.9.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bef596fdaa8f26e3d66af846bbe77057237cb6e8efff8cd7cc8dff9a62278bbf"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2d47552b6e52c3319fede1b60b3de120fe83bde9b7bddad11a69fb0af7db32f1"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:84fc30f71689d7fc9168b92788abc977dc8cefa806909565fc2951d02f6b7d57"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4aa9741085f635934f3a2583e16fcf62ba835719a8b2b28fb2917bb0537c1dfa"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:206a55215e6d05dbc6c98ce598a59e6fbd0c493e2de4ea6cc2f4934d5a18d130"}, + {file = "yarl-1.9.4-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07574b007ee20e5c375a8fe4a0789fad26db905f9813be0f9fef5a68080de559"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5a2e2433eb9344a163aced6a5f6c9222c0786e5a9e9cac2c89f0b28433f56e23"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6ad6d10ed9b67a382b45f29ea028f92d25bc0bc1daf6c5b801b90b5aa70fb9ec"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:6fe79f998a4052d79e1c30eeb7d6c1c1056ad33300f682465e1b4e9b5a188b78"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:a825ec844298c791fd28ed14ed1bffc56a98d15b8c58a20e0e08c1f5f2bea1be"}, + {file = "yarl-1.9.4-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8619d6915b3b0b34420cf9b2bb6d81ef59d984cb0fde7544e9ece32b4b3043c3"}, + {file = "yarl-1.9.4-cp38-cp38-win32.whl", hash = "sha256:686a0c2f85f83463272ddffd4deb5e591c98aac1897d65e92319f729c320eece"}, + {file = "yarl-1.9.4-cp38-cp38-win_amd64.whl", hash = "sha256:a00862fb23195b6b8322f7d781b0dc1d82cb3bcac346d1e38689370cc1cc398b"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:604f31d97fa493083ea21bd9b92c419012531c4e17ea6da0f65cacdcf5d0bd27"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8a854227cf581330ffa2c4824d96e52ee621dd571078a252c25e3a3b3d94a1b1"}, + {file = "yarl-1.9.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ba6f52cbc7809cd8d74604cce9c14868306ae4aa0282016b641c661f981a6e91"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6327976c7c2f4ee6816eff196e25385ccc02cb81427952414a64811037bbc8b"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8397a3817d7dcdd14bb266283cd1d6fc7264a48c186b986f32e86d86d35fbac5"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e0381b4ce23ff92f8170080c97678040fc5b08da85e9e292292aba67fdac6c34"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23d32a2594cb5d565d358a92e151315d1b2268bc10f4610d098f96b147370136"}, + {file = "yarl-1.9.4-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ddb2a5c08a4eaaba605340fdee8fc08e406c56617566d9643ad8bf6852778fc7"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:26a1dc6285e03f3cc9e839a2da83bcbf31dcb0d004c72d0730e755b33466c30e"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:18580f672e44ce1238b82f7fb87d727c4a131f3a9d33a5e0e82b793362bf18b4"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:29e0f83f37610f173eb7e7b5562dd71467993495e568e708d99e9d1944f561ec"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:1f23e4fe1e8794f74b6027d7cf19dc25f8b63af1483d91d595d4a07eca1fb26c"}, + {file = "yarl-1.9.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:db8e58b9d79200c76956cefd14d5c90af54416ff5353c5bfd7cbe58818e26ef0"}, + {file = "yarl-1.9.4-cp39-cp39-win32.whl", hash = "sha256:c7224cab95645c7ab53791022ae77a4509472613e839dab722a72abe5a684575"}, + {file = "yarl-1.9.4-cp39-cp39-win_amd64.whl", hash = "sha256:824d6c50492add5da9374875ce72db7a0733b29c2394890aef23d533106e2b15"}, + {file = "yarl-1.9.4-py3-none-any.whl", hash = "sha256:928cecb0ef9d5a7946eb6ff58417ad2fe9375762382f1bf5c55e61645f2c43ad"}, + {file = "yarl-1.9.4.tar.gz", hash = "sha256:566db86717cf8080b99b58b083b773a908ae40f06681e87e589a976faf8246bf"}, ] [package.dependencies] @@ -1489,4 +1687,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "d0bfe69918b1133a0ea31368570203e0413bccf22aabfca6b5339cca59e2df2d" +content-hash = "1c468b66c56cfccd5e5bff7d9c69f01c729d828132a8a56a7089447f5da0f534" diff --git a/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml b/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml index 141db6810674..12f3060287b3 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml +++ b/airbyte-integrations/bases/connector-acceptance-test/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "connector-acceptance-test" -version = "1.0.0" +version = "3.0.1" description = "Contains acceptance tests for connectors." authors = ["Airbyte "] license = "MIT" @@ -13,7 +13,7 @@ homepage = "https://github.com/airbytehq/airbyte" [tool.poetry.dependencies] python = "^3.10" airbyte-protocol-models = "<1.0.0" -dagger-io = "==0.6.4" +dagger-io = "==0.9.6" PyYAML = "~=6.0" icdiff = "~=1.9" inflection = "~=0.5" @@ -39,6 +39,6 @@ docker = ">=6,<7" # Related issue: https://github.com/docker/docker-py/issues/3113 urllib3 = "<2.0" requests = "<2.29.0" +pytest-xdist = "^3.3.1" [tool.poetry.dev-dependencies] - diff --git a/airbyte-integrations/bases/connector-acceptance-test/sample_files/acceptance-test-config.yml b/airbyte-integrations/bases/connector-acceptance-test/sample_files/acceptance-test-config.yml index efb1840ec4fc..dc6a5ea47839 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/sample_files/acceptance-test-config.yml +++ b/airbyte-integrations/bases/connector-acceptance-test/sample_files/acceptance-test-config.yml @@ -22,9 +22,6 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "sample_files/configured_catalog.json" future_state_path: "sample_files/abnormal_state.json" - cursor_paths: - subscription_changes: ["timestamp"] - email_events: ["timestamp"] full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "sample_files/configured_catalog.json" diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/README.md b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/README.md deleted file mode 100644 index 9883481f76f9..000000000000 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/README.md +++ /dev/null @@ -1,294 +0,0 @@ -# Tooling for automated migration of `acceptance-test-config.yml` files - -This directory contains scripts that can help us manage the migration of connectors' `acceptance-test-config.yml` files. - -## Setup -Before running these scripts you need to set up a local virtual environment in the **current directory**: -```bash -python -m venv .venv -source .venv/bin/activate -pip install -r requirements.txt -brew install gh -``` - -Then create a module to contain the information for your migration/issues etc.: -``` -mkdir migrations/ -touch migrations//__init__.py -touch migrations//config.py -``` - -Copy a config.py file from another migration and fill in the `MODULE_NAME` variable. The other variables -can be filled in when you use certain scripts. -```python -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from typing import Optional, List - -# SET THESE BEFORE USING THE SCRIPT -MODULE_NAME: str = "" -# GITHUB_PROJECT_NAME: Optional[str] = None -# COMMON_ISSUE_LABELS: List[str] = ["area/connectors", "team/connectors-python", "type/enhancement", "column-selection-sources"] -# ISSUE_TITLE: str = "Add undeclared columns to spec" -``` - -## Scripts - -The scripts perform operations on a set of connectors. - -### `run_tests.py`: Run CAT tests and save results by exit code - -#### What it does - -The script will run CAT on all connectors and report the results to `migrations//output` and classify the -results by exit code. - -TODO: Replace this process with Dagger - -#### Before running - -1. The tests will run on the `latest` version of CAT by default. To run the `dev` version of CAT, and select a specific -test, commit the hacky changes in [this commit](https://github.com/airbytehq/airbyte/pull/24377/commits/7d9fb1414911a512cd5d5ffafe2a384e8004fb1e). - -2. Give Docker a _lot_ of space to build all the connector images! - -3. Make sure you have the secrets downloaded from GSM for all of the connectors you want to run tests on. Please keep -in mind that secrets need to be re-uploaded for connectors with single-use Oauth tokens. - -#### How to run - -Typical usage: -``` -python run_tests.py -``` - -Full options: -``` -usage: run_tests.py [-h] --connectors [CONNECTORS ...] [--allow_community | --no-allow_community] [--max_concurrency MAX_CONCURRENCY] - -Run connector acceptance tests for a list of connectors. - -options: - -h, --help show this help message and exit - --connectors [CONNECTORS ...] - A list of connectors (separated by spaces) to run a script on. (default: all connectors) - --allow_alpha, --no-allow_alpha - Whether to apply the change to alpha connectors, if they are included in the list of connectors. (default: False) - --allow_community, --no-allow_community - Whether to apply the change to bets connectors, if they are included in the list of connectors. (default: False) - --max_concurrency MAX_CONCURRENCY - The maximum number of acceptance tests that should happen at once. -``` - -### `create_issues.py`: Create issues in bulk - -#### What it does: -For each connector: -1. Generate an issue content (title, body, labels, project), using an `issue.md.j2` template -2. Find an already existing issue with the same title, if one already exists -3. Create the issue and return its url if it does not exist. - -Issues get created with the title according to `ISSUE_TITLE`. Labels are added according to `COMMON_ISSUE_LABELS`. Issues are added to the `GITHUB_PROJECT_NAME` project, if one is provided. - -#### Before running - -1. Update your config file to define the following variables: - - ```python - # - # Copyright (c) 2023 Airbyte, Inc., all rights reserved. - # - - from typing import Optional, List - - # SET THESE BEFORE USING THE SCRIPT - MODULE_NAME: str = "" - GITHUB_PROJECT_NAME: Optional[str] = "" - COMMON_ISSUE_LABELS: List[str] = ["", "", "..."] - ISSUE_TITLE: str = "" - ``` - Note that `ISSUE_TITLE` will be prepended with `Source :` in the actual created issue. - -2. Create a template for your issue: - - ```bash - touch migrations//issue.md.j2 - ``` - - If you need to fill more variables than are currently defined in the call to `template.render()` - in `create_issues.py`, edit the script to allow filling of that variable and define how it should be - filled. Please keep in mind the other migrations when you do this. - -3. Update the following line in the script so that it points to the config file from your migration: - - ```python - ## Update this line before running the script - from migrations. import config - ``` - -#### How to run: - -Typical usage (dry run): -``` -python create_issues.py --connectors -``` - -Typical usage (real execution): -``` -python create_issues.py --connectors --no-dry -``` - -Full options: -``` -usage: create_issues.py [-h] [-d | --dry | --no-dry] --connectors [CONNECTORS ...] [--allow_community | --no-allow_community] [--allow_alpha | --no-allow_alpha] - -Create issues for a list of connectors from a template. - -options: - -h, --help show this help message and exit - -d, --dry, --no-dry Whether the action performed is a dry run. In the case of a dry run, no git actions will be pushed to the remote. (default: True) - --connectors [CONNECTORS ...] - A list of connectors (separated by spaces) to run a script on. (default: all connectors) - --allow_community, --no-allow_community - Whether to apply the change to bets connectors, if they are included in the list of connectors. (default: False) - --allow_alpha, --no-allow_alpha - Whether to apply the change to alpha connectors, if they are included in the list of connectors. (default: False) -``` - - -### `config_migration.py`: Perform migrations on `acceptance-test-config.yml` files - -#### What it does: -For each connector: -1. Load the connector's `acceptance-test-config.yml` -2. Migrate the connector to the new format if this option was chosen -3. Apply the given migration to the `acceptance-test-config.yml` file - -Note that all changes happen on the working branch. - -#### Before running - -1. Create a method in `config_migration.py` to perform your migration on a given config. For this, - You can take inspiration from the existing `set_high_strictness_level` and `set_ignore_extra_columns` - migration methods. - -2. Update the following line to point to your new migration: - ```python - # Update this before running the script - MIGRATION_TO_RUN = - ``` - -#### How to run: - -Typical usage: -``` -python config_migration.py --connectors -``` - -Full options: -``` -usage: config_migration.py [-h] --connectors [CONNECTORS ...] [--allow_community | --no-allow_community] [--migrate_from_legacy | --no-migrate_from_legacy] - -Migrate acceptance-test-config.yml files for a list of connectors. - -options: - -h, --help show this help message and exit - --connectors [CONNECTORS ...] - A list of connectors (separated by spaces) to run a script on. (default: all connectors) - --allow_community, --no-allow_community - Whether to apply the change to community connectors, if they are included in the list of connectors. (default: False) - --migrate_from_legacy, --no-migrate_from_legacy - Whether to migrate config files from the legacy format before applying the migration. (default: False) -``` - - -### `create_prs.py`: Create a PR per connector that performs a config migration and pushes it - -## Create migration PRs for Certified connectors (`create_prs.py`) - -#### What it does: -For each connector: -1. Create a branch and check it out -2. Locally apply the migration to `acceptance_test_config.yml` by calling `config_migration.py` -3. Commit and push the changes on this branch -4. Open a PR for this branch -5. Run a connector acceptance test on this branch by posting a `/test` comment on the PR - -An example of the PR it creates can be found [here](https://github.com/airbytehq/airbyte/pull/19136). - -PRs get created with the title according to `ISSUE_TITLE`. Labels are added according to `COMMON_ISSUE_LABELS`. PRs are added to the `GITHUB_PROJECT_NAME` project, if one is provided. - -#### Before running - -1. Update your config file to define the following variables: - - ```python - # - # Copyright (c) 2023 Airbyte, Inc., all rights reserved. - # - - from typing import Optional, List - - # SET THESE BEFORE USING THE SCRIPT - MODULE_NAME: str = "" - GITHUB_PROJECT_NAME: Optional[str] = "" - COMMON_ISSUE_LABELS: List[str] = ["", "", "..."] - ISSUE_TITLE: str = "" - ``` - Note that `ISSUE_TITLE` will be prepended with `Source :` in the actual created issue. - -2. Create a template for your PR description: - - ```bash - touch migrations//pr.md.j2 - ``` - - If you need to fill more variables than are currently defined in the call to `template.render()` - in `create_pr.py`, edit the script to allow filling of that variable and define how it should be - filled. Please keep in mind the other migrations when you do this. - -3. Update the following line in the `create_prs.py` so that it points to the config file from your migration: - - ```python - ## Update this line before running the script - from migrations. import config - ``` - -4. Ensure that your current git envronment is clean by (perhaps temorarily) committing changes. - - -#### How to run: - -Typical usage (dry run): -``` -python create_prs.py --connectors -``` - -Typical usage (real execution): -``` -python create_prs.py --connectors --no-dry -``` - -Full options: -``` -usage: create_prs.py [-h] [-d | --dry | --no-dry] --connectors [CONNECTORS ...] [--allow_community | --no-allow_community] - -Create PRs for a list of connectors from a template. - -options: - -h, --help show this help message and exit - -d, --dry, --no-dry Whether the action performed is a dry run. In the case of a dry run, no git actions will be pushed to the remote. (default: True) - --connectors [CONNECTORS ...] - A list of connectors (separated by spaces) to run a script on. (default: all connectors) - --allow_alpha, --no-allow_alpha - Whether to apply the change to alpha connectors, if they are included in the list of connectors. (default: False) - --allow_community, --no-allow_community - Whether to apply the change to community connectors, if they are included in the list of connectors. (default: False) -``` - -## Existing migrations -* `strictness_level_migration`: Migrates a connector from the old format to the new format, and adds enforcement of high strictness level. -* `fail_on_extra_columns`: Adds `fail_on_extra_columns: false` to connectors which fail the `Additional properties are not allowed` extra column validation. - Supports adding this parameter to configs in the old and new config format. \ No newline at end of file diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/config_migration.py b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/config_migration.py deleted file mode 100644 index e39d1bf02806..000000000000 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/config_migration.py +++ /dev/null @@ -1,88 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import argparse -import logging -from pathlib import Path -from typing import Callable - -import definitions -import utils -from connector_acceptance_test.config import Config -from ruamel.yaml import YAML - -yaml = YAML() -yaml.preserve_quotes = True -yaml.width = 150 - -parser = argparse.ArgumentParser(description="Migrate acceptance-test-config.yml files for a list of connectors.") -utils.add_connectors_param(parser) -utils.add_allow_alpha_param(parser) -utils.add_allow_beta_param(parser) -parser.add_argument( - "--migrate_from_legacy", - action=argparse.BooleanOptionalAction, - default=False, - help="Whether to migrate config files from the legacy format before applying the migration.", -) - - -def load_config(config_path: Path) -> Config: - with open(config_path, "r") as file: - config = yaml.load(file) - return config - - -def migrate_to_new_config_format(config: Config): - if Config.is_legacy(config): - return Config.migrate_legacy_to_current_config(config) - else: - logging.warning("The configuration is not in a legacy format.") - return config - - -def set_high_test_strictness_level(config): - if Config.is_legacy(config): - raise Exception("You can't set a strictness level on a legacy config. Please use the `--migrate_from_legacy` flag.") - config["test_strictness_level"] = "high" - for basic_read_test in config["acceptance_tests"].get("basic_read", {"tests": []})["tests"]: - basic_read_test.pop("configured_catalog_path", None) - return config - - -def set_ignore_extra_columns(config): - if Config.is_legacy(config): - for basic_read_test in config["tests"].get("basic_read"): - basic_read_test["fail_on_extra_columns"] = False - else: - for basic_read_test in config["acceptance_tests"].get("basic_read", {"tests": []})["tests"]: - basic_read_test["fail_on_extra_columns"] = False - return config - - -def write_new_config(new_config, output_path): - with open(output_path, "w") as output_file: - yaml.dump(new_config, output_file) - logging.info("Saved the configuration in its new format") - - -def update_configuration(config_path, migration: Callable, migrate_from_legacy: bool): - config_to_migrate = load_config(config_path) - if migrate_from_legacy: - config_to_migrate = migrate_to_new_config_format(config_to_migrate) - new_config = migration(config_to_migrate) - write_new_config(new_config, config_path) - logging.info(f"The configuration was successfully updated: {config_path}") - return config_path - - -if __name__ == "__main__": - args = parser.parse_args() - - # Update this before running the script - MIGRATION_TO_RUN = set_high_test_strictness_level - - for definition in utils.get_valid_definitions_from_args(args): - config_path = utils.acceptance_test_config_path(definitions.get_airbyte_connector_name_from_definition(definition)) - update_configuration(config_path, MIGRATION_TO_RUN, args.migrate_from_legacy) diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/create_issues.py b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/create_issues.py deleted file mode 100644 index afd93f83b5da..000000000000 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/create_issues.py +++ /dev/null @@ -1,116 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import argparse -import json -import logging -import os -import subprocess -import tempfile -from typing import Any, Dict, Optional, Text - -import definitions -import utils -from jinja2 import Environment, FileSystemLoader - -# Update this line before running the script -from migrations.strictness_level_migration import config - -logging.basicConfig(level=logging.DEBUG) -environment = Environment(loader=FileSystemLoader(utils.MIGRATIONS_FOLDER)) - -parser = argparse.ArgumentParser(description="Create issues for a list of connectors from a template.") -utils.add_dry_param(parser) -utils.add_connectors_param(parser) -utils.add_allow_beta_param(parser) -utils.add_allow_alpha_param(parser) - - -def get_test_failure_logs(definition): - test_failure_logs = "" - if config.MODULE_NAME == "fail_on_extra_columns": - connector_technical_name = definitions.get_airbyte_connector_name_from_definition(definition) - - try: - with open(f"{utils.MIGRATIONS_FOLDER}/{config.MODULE_NAME}/test_failure_logs/{connector_technical_name}", "r") as f: - for line in f: - test_failure_logs += line - except FileNotFoundError: - logging.warning(f"Skipping creating an issue for {definition['name']} -- could not find an output file for it.") - return - - return test_failure_logs - - -def get_issue_content(source_definition) -> Optional[Dict[Text, Any]]: - issue_title = f"Source {source_definition['name']}: {config.ISSUE_TITLE}" - - template = environment.get_template(f"{config.MODULE_NAME}/issue.md.j2") - - # TODO: Make list of variables to render, and how to render them, configurable - issue_body = template.render( - connector_name=source_definition["name"], - support_level=source_definition["supportLevel"], - test_failure_logs=get_test_failure_logs(source_definition), - ) - file_definition, issue_body_path = tempfile.mkstemp() - - with os.fdopen(file_definition, "w") as tmp: - tmp.write(issue_body) - - return {"title": issue_title, "body_file": issue_body_path, "labels": config.COMMON_ISSUE_LABELS, "project": config.GITHUB_PROJECT_NAME} - - -def get_existing_issues(issue_content): - list_command_arguments = ["gh", "issue", "list", "--state", "open", "--search", f"'{issue_content['title']}'", "--json", "url"] - list_existing_issue_process = subprocess.Popen(list_command_arguments, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = list_existing_issue_process.communicate() - existing_issues = json.loads(stdout.decode()) - return existing_issues - - -def create_command(issue_content): - create_command_arguments = [ - "gh", - "issue", - "create", - "--title", - issue_content["title"], - "--body-file", - issue_content["body_file"], - ] - if config.GITHUB_PROJECT_NAME: - create_command_arguments += ["--project", issue_content["project"]] - for label in issue_content["labels"]: - create_command_arguments += ["--label", label] - return create_command_arguments - - -def create_issue(source_definition, dry_run=True): - issue_content = get_issue_content(source_definition) - if not issue_content: - return - - existing_issues = get_existing_issues(issue_content) - if existing_issues: - logging.warning(f"An issue was already created for {source_definition['name']}: {existing_issues[0]}") - else: - if not dry_run: - process = subprocess.Popen(create_command(issue_content), stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = process.communicate() - if stderr: - logging.error(stderr.decode()) - else: - created_issue_url = stdout.decode() - logging.info(f"Created issue for {source_definition['name']}: {created_issue_url}") - else: - logging.info(f"[DRY RUN]: {' '.join(create_command(issue_content))}") - os.remove(issue_content["body_file"]) - - -if __name__ == "__main__": - args = parser.parse_args() - for definition in utils.get_valid_definitions_from_args(args): - create_issue(definition, dry_run=args.dry) diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/create_prs.py b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/create_prs.py deleted file mode 100644 index b9d441871c30..000000000000 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/create_prs.py +++ /dev/null @@ -1,142 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import argparse -import json -import logging -import os -import subprocess -import tempfile - -import definitions -import utils -from config_migration import set_high_test_strictness_level, update_configuration -from git import Repo -from jinja2 import Environment, FileSystemLoader - -# Update this before running the script -from migrations.strictness_level_migration import config - -REPO_ROOT = "../../../../../" -AIRBYTE_REPO = Repo(REPO_ROOT) -environment = Environment(loader=FileSystemLoader(utils.MIGRATIONS_FOLDER)) -PR_TEMPLATE = environment.get_template(f"{config.MODULE_NAME}/pr.md.j2") - -parser = argparse.ArgumentParser(description="Create PRs for a list of connectors from a template.") -utils.add_dry_param(parser) -utils.add_connectors_param(parser) -utils.add_allow_alpha_param(parser) -utils.add_allow_beta_param(parser) - -logging.basicConfig(level=logging.DEBUG) - - -def checkout_new_branch(connector_name): - AIRBYTE_REPO.heads.master.checkout() - new_branch_name = f"{connector_name}/{config.MODULE_NAME}" - new_branch = AIRBYTE_REPO.create_head(new_branch_name) - new_branch.checkout() - return new_branch - - -def commit_push_migrated_config(config_path, connector_name, new_branch, dry_run): - process = subprocess.Popen(["pre-commit", "run", "--files", config_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - process.communicate() - relative_config_path = f"airbyte-integrations/connectors/{connector_name}/acceptance-test-config.yml" - AIRBYTE_REPO.git.add(relative_config_path) - AIRBYTE_REPO.git.commit(m=f"Migrated config for {connector_name}") - logging.info(f"Committed migrated config on {new_branch}") - if not dry_run: - AIRBYTE_REPO.git.push("--set-upstream", "origin", new_branch) - logging.info(f"Pushed branch {new_branch} to origin") - - -def get_pr_content(definition): - pr_title = f"Source {definition['name']}: {config.ISSUE_TITLE}" - - pr_body = PR_TEMPLATE.render(connector_name=definition["name"], support_level=definition["supportLevel"]) - file_definition, pr_body_path = tempfile.mkstemp() - - with os.fdopen(file_definition, "w") as tmp: - tmp.write(pr_body) - - return {"title": pr_title, "body_file": pr_body_path, "labels": config.COMMON_ISSUE_LABELS} - - -def open_pr(definition, new_branch, dry_run): - pr_content = get_pr_content(definition) - list_command_arguments = ["gh", "pr", "list", "--state", "open", "--head", new_branch.name, "--json", "url"] - create_command_arguments = [ - "gh", - "pr", - "create", - "--draft", - "--title", - pr_content["title"], - "--body-file", - pr_content["body_file"], - ] - if config.GITHUB_PROJECT_NAME: - create_command_arguments += ["--project", config.GITHUB_PROJECT_NAME] - for label in pr_content["labels"]: - create_command_arguments += ["--label", label] - list_existing_pr_process = subprocess.Popen(list_command_arguments, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = list_existing_pr_process.communicate() - existing_prs = json.loads(stdout.decode()) - already_created = len(existing_prs) > 0 - if already_created: - logging.warning(f"A PR was already created for this definition: {existing_prs[0]}") - if not already_created: - if not dry_run: - process = subprocess.Popen(create_command_arguments, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = process.communicate() - if stderr: - logging.error(stderr.decode()) - else: - created_pr_url = stdout.decode() - logging.info(f"Created PR for {definition['name']}: {created_pr_url}") - else: - logging.info(f"[DRY RUN]: {' '.join(create_command_arguments)}") - os.remove(pr_content["body_file"]) - - -def add_test_comment(definition, new_branch, dry_run): - connector_name = definitions.get_airbyte_connector_name_from_definition(definition) - comment = f"/test connector=connectors/{connector_name}" - comment_command_arguments = ["gh", "pr", "comment", new_branch.name, "--body", comment] - if not dry_run: - process = subprocess.Popen(comment_command_arguments, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - _, stderr = process.communicate() - if stderr: - logging.error(stderr.decode()) - else: - logging.info("Added test comment") - else: - logging.info(f"[DRY RUN]: {' '.join(comment_command_arguments)}") - - -def migrate_config_on_new_branch(definition, dry_run): - AIRBYTE_REPO.heads.master.checkout() - connector_name = definitions.get_airbyte_connector_name_from_definition(definition) - new_branch = checkout_new_branch(connector_name) - config_path = utils.acceptance_test_config_path(connector_name) - update_configuration(config_path, migration=set_high_test_strictness_level, migrate_from_legacy=True) - commit_push_migrated_config(config_path, connector_name, new_branch, dry_run) - return new_branch - - -def migrate_definition_and_open_pr(definition, dry_run): - original_branch = AIRBYTE_REPO.active_branch - new_branch = migrate_config_on_new_branch(definition, dry_run) - open_pr(definition, new_branch, dry_run) - add_test_comment(definition, new_branch, dry_run) - original_branch.checkout() - AIRBYTE_REPO.git.branch(D=new_branch) - logging.info(f"Deleted branch {new_branch}") - - -if __name__ == "__main__": - args = parser.parse_args() - for definition in utils.get_valid_definitions_from_args(args): - migrate_definition_and_open_pr(definition, dry_run=args.dry) diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/definitions.py b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/definitions.py deleted file mode 100644 index 519bf5abb66f..000000000000 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/definitions.py +++ /dev/null @@ -1,52 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import logging -from typing import List - -import requests - -logging.basicConfig(level=logging.DEBUG) - -CONNECTOR_REGISTRY_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json" - - -def download_and_parse_registry_json(): - response = requests.get(CONNECTOR_REGISTRY_URL) - response.raise_for_status() - return response.json() - - -def read_source_definitions(): - return download_and_parse_registry_json()["sources"] - - -def find_by_support_level(source_definitions, support_level): - if support_level == "other": - return [definition for definition in source_definitions if definition.get("supportLevel", "") not in ["community", "certified"]] - else: - return [definition for definition in source_definitions if definition.get("supportLevel") == support_level] - - -def find_by_name(connector_names: List[str]): - definitions = [ - definition for definition in ALL_DEFINITIONS if get_airbyte_connector_name_from_definition(definition) in connector_names - ] - if len(definitions) != len(connector_names): - logging.warning(f"Looked for {len(connector_names)} items, got {len(definitions)} items. Did you misspell something?") - return definitions - - -def get_airbyte_connector_name_from_definition(connector_definition): - return connector_definition["dockerRepository"].replace("airbyte/", "") - - -def is_airbyte_connector(connector_definition): - return connector_definition["dockerRepository"].startswith("airbyte/") - - -ALL_DEFINITIONS = read_source_definitions() -CERTIFIED_DEFINITIONS = find_by_support_level(ALL_DEFINITIONS, "certified") -COMMUNITY_DEFINITIONS = find_by_support_level(ALL_DEFINITIONS, "community") -OTHER_DEFINITIONS = find_by_support_level(ALL_DEFINITIONS, "other") diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/fail_on_extra_columns/.gitignore b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/fail_on_extra_columns/.gitignore deleted file mode 100644 index fca46f75457f..000000000000 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/fail_on_extra_columns/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -output -test_failure_logs diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/fail_on_extra_columns/README.md b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/fail_on_extra_columns/README.md deleted file mode 100644 index 228226040f9c..000000000000 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/fail_on_extra_columns/README.md +++ /dev/null @@ -1,53 +0,0 @@ -# Bypassing column selection validation for sources which fail it -This migration adds `fail_on_extra_columns: false` to the `basic_read` test in the `acceptance-test-config.yml` -file for all community and certified connectors which fail the stricter validation added to the `basic_read` test. It creates -issues for each of the connectors whose configs were modified as a result. - -Before following this README, please reference the `acceptance_test_config_migraton` README for general -usage information for the given scripts. - -## Add bypass for connectors that fail the new CAT test - -### Run tests on all connectors -Run CAT on all community and certified connectors. - -``` -python run_tests.py --allow_community -``` - -### Collect output from connectors that fail due to `additionalProperties` -``` -cd migrations/fail_on_extra_columns -sh get_failures.sh -``` - -### Migrate configs for failed connectors -For the connectors that failed due to `Additional properties are not allowed:`, we want to add the new -`fail_on_extra_columns` input parameter to the basic read test. To do this, - -``` -python config_migration.py --connectors $(ls migrations/fail_on_extra_columns/test_failure_logs) --allow_community -``` - -Add these bypasses to the PR that adds the new CAT test! - - -## Create issues for failing connectors (`create_issues.py`) -Create one issue per certified connectors to add the missing columns to the spec and remove the `fail_on_extra_columns` bypass. - -Issues get created with the following labels: -* `area/connectors` -* `team/connectors-python` -* `type/enhancement` -* `column-selection-sources` - -### How to run: -**Dry run**: -``` -python create_issues.py --connectors $(ls migrations/fail_on_extra_columns/test_failure_logs) --allow_community -``` - -**Real execution**: -``` -python create_issues.py --connectors $(ls migrations/fail_on_extra_columns/test_failure_logs) --allow_community --no-dry -``` diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/fail_on_extra_columns/config.py b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/fail_on_extra_columns/config.py deleted file mode 100644 index e283a0b43d15..000000000000 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/fail_on_extra_columns/config.py +++ /dev/null @@ -1,11 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from typing import List, Optional - -# SET THESE BEFORE USING THE SCRIPT -MODULE_NAME: str = "fail_on_extra_columns" -GITHUB_PROJECT_NAME: Optional[str] = None -COMMON_ISSUE_LABELS: List[str] = ["area/connectors", "team/connectors-python", "type/enhancement", "column-selection-sources"] -ISSUE_TITLE: str = "Add undeclared columns to spec" diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/fail_on_extra_columns/get_failures.sh b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/fail_on_extra_columns/get_failures.sh deleted file mode 100644 index e7a572b90c48..000000000000 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/fail_on_extra_columns/get_failures.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# Outputs where the tests ran, but failed (exit code 1) -FULL_OUTPUT_DIR="./output/1" # TODO: make this work even when called from another dir - -# In test output look for lines with "Additional properties are not allowed" and collect them all into one file. -# Results from each output file will contain the name of the file in the lines output in the columns file -tmp_columns_file=$(mktemp) -grep -rnw "$FULL_OUTPUT_DIR" -e "Additional properties are not allowed" -B 1 > "$tmp_columns_file" - -# For each connector, grab the lines in the columns file associated with them and put them into a test_failure_logs -# file so that we can attach it to the issue created. -mkdir -p "./test_failure_logs" -for f in $(ls $FULL_OUTPUT_DIR); do - results=$(mktemp) - grep "$tmp_columns_file" -e "$f" > "$results" - - # If there weren't any 'additionalProperties are not allowed' logs, don't create test_failure_output for this connector - if [ -s "$results" ]; then - cat "$results" > "./test_failure_logs/$f" - fi -done \ No newline at end of file diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/fail_on_extra_columns/issue.md.j2 b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/fail_on_extra_columns/issue.md.j2 deleted file mode 100644 index fa50adc62ebc..000000000000 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/fail_on_extra_columns/issue.md.j2 +++ /dev/null @@ -1,31 +0,0 @@ -## What -In this [PR](https://github.com/airbytehq/airbyte/pull/23985), `BasicReadTest.test_read` test in the Connector Acceptance Tests was updated to fail if the connector produces -stream records which contain columns that haven't been declared in the spec. - -{{ connector_name }} currently fails the updated test. Its `acceptance-test-config.yml` was edited with the -`fail_on_extra_columns: false` parameter in order to avoid this change from making the connector fail CAT. - -We want to add the undeclared columns to the spec. - -**How this will help**: -- Column selection is currently blocked for connectors that don't declare all columns that they pass. This - is because turning on column selection will stop these undeclared columns from being sent, when they were - previously being sent. Doing this allows us to enable column selection for {{ connector_name }}! -- Users will better understand the shape of the data they'll receive, since the columns will match the spec. - -## How -The following descriptions of streams that pass undeclared columns come from results of the failed connector acceptance test: - -``` -{{ test_failure_logs }} -``` - -1. Add the missing properties indicated by the `Additional properties are not allowed ('', 'column' were unexpected)` logs to the connector's spec. -2. Remove `fail_on_extra_columns: false` from the `acceptance-test-config.yml` file. -3. Commit changes to spec and `acceptance-test-config.yml` and open a PR. -4. Run tests on the connector, either automatically via CI or manually via the `/test` command -5. Profit! - -Definition of done: {{ connector_name }} passes CAT without declaring `fail_on_extra_columns: false`. If the -API starts sending over extra columns in the future, we will catch and fix them as part of the #connector-health -movement. \ No newline at end of file diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/strictness_level_migration/README.md b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/strictness_level_migration/README.md deleted file mode 100644 index 38a946b1ef78..000000000000 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/strictness_level_migration/README.md +++ /dev/null @@ -1,48 +0,0 @@ -# Migrating connectors' acceptance test configs to high strictness -This migration sets connectors' `acceptance-test-config.yml` to `high` test strictness level. -In doing so, it migrates the config files from a previous format into our new format. - -Before following this README, please reference the `acceptance_test_config_migraton` README for general -usage information for the given scripts. - -## Create migration issue for GA connectors (`create_issues.py`) -Create one issue per GA connectors to migrate to `high` test strictness level. - -Issues get created with the following labels: -* `area/connectors` -* `team/connectors-python` -* `type/enhancement` -* `test-strictness-level` - -Issues are added to the following project: `SAT-high-test-strictness-level` - -### How to run: -**Dry run**: -``` -python create_issues.py -``` - -**Real execution**: -``` -python create_issues.py --no-dry -``` - -## Create migration PRs for GA connectors (`create_prs.py`) -Create one PR per GA connector to perform the migration to `high` test strictness level. - -An example of the PR it creates can be found [here](https://github.com/airbytehq/airbyte/pull/19136) - -PR get created with the following labels: -* `area/connectors` -* `team/connectors-python` -* `type/enhancement` -* `test-strictness-level` - -PR are added to the following project: `SAT-high-test-strictness-level` - -### How to run: -**Dry run**: -`python create_prs.py` - -**Real execution**: -`python create_prs.py --no-dry` \ No newline at end of file diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/strictness_level_migration/config.py b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/strictness_level_migration/config.py deleted file mode 100644 index 2ef9af30429c..000000000000 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/strictness_level_migration/config.py +++ /dev/null @@ -1,11 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from typing import List, Optional - -# SET THESE BEFORE USING THE SCRIPT -MODULE_NAME: str = "strictness_level_migration" -GITHUB_PROJECT_NAME: Optional[str] = "SAT-high-test-strictness-level" -COMMON_ISSUE_LABELS: List[str] = ["area/connectors", "team/connectors-python", "type/enhancement", "test-strictness-level"] -ISSUE_TITLE: str = "enable `high` test strictness level in connector acceptance test" diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/strictness_level_migration/issue.md.j2 b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/strictness_level_migration/issue.md.j2 deleted file mode 100644 index 395fc61376bd..000000000000 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/strictness_level_migration/issue.md.j2 +++ /dev/null @@ -1,15 +0,0 @@ -## What -A `test_strictness_level` field was introduced to Connector Acceptance Tests. -{{ connector_name }} is a {{ support_level }} connector, we want it to have a `high` test strictness level. - -**This will help**: -- maximize the acceptance test coverage on this connector. -- document its potential weaknesses in term of test coverage. - -## How -1. Migrate the existing `acceptance-test-config.yml` file to the latest configuration format. (See instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/bases/connector-acceptance-test/README.md#L61)) -2. Enable `high` test strictness level in `acceptance-test-config.yml`. (See instructions [here](https://github.com/airbytehq/airbyte/blob/master/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md#L240)) -3. Commit changes on `acceptance-test-config.yml` and open a PR. -4. Run the tests with the `/test` command on the branch. -5. If tests are failing please fix the failing test or use `bypass_reason` fields to explain why a specific test can't be run. - diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/strictness_level_migration/pr.md.j2 b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/strictness_level_migration/pr.md.j2 deleted file mode 100644 index c6f023e2b013..000000000000 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/migrations/strictness_level_migration/pr.md.j2 +++ /dev/null @@ -1,22 +0,0 @@ -## What -A `test_strictness_level` field was introduced to Connector Acceptance Tests. -{{ connector_name }} is a {{ support_level }} connector, we want it to have a `high` test strictness level. - -**This will help**: -- maximize the acceptance test coverage on this connector. -- document its potential weaknesses in term of test coverage. - -## How -1. Migrate the existing `acceptance-test-config.yml` file to the latest configuration format. (See instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/bases/connector-acceptance-test/README.md#L61)) -2. Enable `high` test strictness level in `acceptance-test-config.yml`. (See instructions [here](https://github.com/airbytehq/airbyte/blob/master/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md#L240)) - -⚠️ ⚠️ ⚠️ -**If tests are failing please fix the failing test by changing the `acceptance-test-config.yml` file or use `bypass_reason` fields to explain why a specific test can't be run.** - -Please open a new PR if the new enabled tests help discover a new bug. -Once this bug fix is merged please rebase this branch and run `/test` again. - -You can find more details about the rules enforced by `high` test strictness level [here](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference/). - -## Review process -Please ask the `connector-operations` teams for review. \ No newline at end of file diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/requirements.txt b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/requirements.txt deleted file mode 100644 index f286425918e9..000000000000 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/requirements.txt +++ /dev/null @@ -1,66 +0,0 @@ -airbyte-cdk==0.7.0 -appdirs==1.4.4 -attrs==22.1.0 -backoff==2.2.1 -cattrs==22.2.0 -certifi==2022.9.24 -charset-normalizer==2.1.1 -coverage==6.5.0 -dataclasses-jsonschema==2.15.1 -deepdiff==5.8.1 -Deprecated==1.2.13 -docker==5.0.3 -dpath==2.0.6 -exceptiongroup==1.0.1 -fancycompleter==0.9.1 -gitdb==4.0.9 -GitPython==3.1.29 -hypothesis==6.54.6 -hypothesis-jsonschema==0.20.1 -icdiff==1.9.1 -idna==3.4 -inflection==0.5.1 -iniconfig==1.1.1 -Jinja2==3.1.2 -jsonref==0.2 -jsonschema==3.2.0 -MarkupSafe==2.1.1 -ordered-set==4.1.0 -packaging==21.3 -pdbpp==0.10.3 -pendulum==2.1.2 -pluggy==1.0.0 -pre-commit==3.2.1 -pprintpp==0.4.0 -py==1.11.0 -pyaml==21.10.1 -pydantic==1.9.2 -Pygments==2.13.0 -pyparsing==3.0.9 -pyrepl==0.9.0 -pyrsistent==0.19.2 -pytest==6.2.5 -pytest-cov==3.0.0 -pytest-mock==3.6.1 -pytest-sugar==0.9.6 -pytest-timeout==1.4.2 -python-dateutil==2.8.2 -pytzdata==2020.1 -PyYAML==5.4.1 -requests==2.28.1 -requests-cache==0.9.7 -requests-mock==1.9.3 -ruamel.yaml==0.17.21 -six==1.16.0 -smmap==5.0.0 -sortedcontainers==2.4.0 --e ../.. # local `connector-acceptance-test` module -termcolor==2.1.0 -toml==0.10.2 -tomli==2.0.1 -typing_extensions==4.4.0 -url-normalize==1.4.3 -urllib3==1.26.12 -websocket-client==1.4.2 -wmctrl==0.4 -wrapt==1.14.1 diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/run_tests.py b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/run_tests.py deleted file mode 100644 index dd323c556e9b..000000000000 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/run_tests.py +++ /dev/null @@ -1,73 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import argparse -import asyncio -import logging -import os -from pathlib import Path - -import definitions -import utils -from migrations.fail_on_extra_columns import config - -parser = argparse.ArgumentParser(description="Run connector acceptance tests for a list of connectors.") -utils.add_connectors_param(parser) -utils.add_allow_alpha_param(parser) -utils.add_allow_beta_param(parser) -parser.add_argument("--max_concurrency", type=int, default=10, help="The maximum number of acceptance tests that should happen at once.") - - -async def run_tests(connector_name): - path_to_acceptance_test_runner = utils.acceptance_test_docker_sh_path(connector_name) - path_to_acceptance_test_config = utils.acceptance_test_config_path(connector_name) - - logging.info(f"Start running tests for {connector_name}.") - process = await asyncio.create_subprocess_exec( - "sh", - path_to_acceptance_test_runner, - env=dict(os.environ, CONFIG_PATH=path_to_acceptance_test_config), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - return_code = await process.wait() - - output_path = Path(utils.MIGRATIONS_FOLDER) / config.MODULE_NAME / "output" / str(return_code) - output_path.mkdir(parents=True, exist_ok=True) - - contents = await process.stdout.read() - with open(f"{output_path}/{connector_name}", "wb") as f: - f.write(contents) - - if return_code == 0: - logging.info(f"{connector_name} succeeded.") - else: - logging.info(f"{connector_name} tests failed with exit code {return_code}.") - - -async def semaphore_gather(coroutines, num_semaphores): - # Limit the amount of connectors we want to test at once - # To avoid crashing our docker by spinning up too many containers - # Or using too much CPU. How many you can run at once effectively - # will depend on the specs you've allocated to Docker - semaphore = asyncio.Semaphore(num_semaphores) - - async def _wrap_coroutine(coroutine): - async with semaphore: - return await coroutine - - return await asyncio.gather(*(_wrap_coroutine(coroutine) for coroutine in coroutines), return_exceptions=False) - - -async def main(args): - tasks = [] - for definition in utils.get_valid_definitions_from_args(args): - connector_name = definitions.get_airbyte_connector_name_from_definition(definition) - tasks.append(run_tests(connector_name)) - await asyncio.gather(semaphore_gather(tasks, num_semaphores=args.max_concurrency)) - - -if __name__ == "__main__": - args = parser.parse_args() - asyncio.run(main(args)) diff --git a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/utils.py b/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/utils.py deleted file mode 100644 index 044a3621d3ba..000000000000 --- a/airbyte-integrations/bases/connector-acceptance-test/tools/acceptance_test_config_migration/utils.py +++ /dev/null @@ -1,69 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import argparse -import logging -from pathlib import Path - -import definitions - -CONNECTORS_DIRECTORY = "../../../../connectors" -MIGRATIONS_FOLDER = "./migrations/" - - -def acceptance_test_config_path(connector_name): - """Returns the path to a given connector's acceptance-test-config.yml file.""" - return Path(CONNECTORS_DIRECTORY) / connector_name / "acceptance-test-config.yml" - - -def acceptance_test_docker_sh_path(connector_name): - return Path(CONNECTORS_DIRECTORY) / connector_name / "acceptance-test-docker.sh" - - -def add_dry_param(parser: argparse.ArgumentParser): - parser.add_argument( - "-d", - "--dry", - action=argparse.BooleanOptionalAction, - default=True, - help="Whether the action performed is a dry run. In the case of a dry run, no git actions will be pushed to the remote.", - ) - - -def add_allow_community_param(parser: argparse.ArgumentParser): - parser.add_argument( - "--allow_community", - action=argparse.BooleanOptionalAction, - default=False, - help="Whether to apply the change to bets connectors, if they are included in the list of connectors.", - ) - - -def add_connectors_param(parser: argparse.ArgumentParser): - parser.add_argument( - "--connectors", nargs="*", help="A list of connectors (separated by spaces) to run a script on. (default: all connectors)" - ) - - -def get_valid_definitions_from_args(args): - if not args.connectors: - requested_defintions = definitions.ALL_DEFINITIONS - else: - requested_defintions = definitions.find_by_name(args.connectors) - - valid_definitions = [] - for definition in requested_defintions: - connector_technical_name = definitions.get_airbyte_connector_name_from_definition(definition) - if not definitions.is_airbyte_connector(definition): - logging.warning(f"Skipping {connector_technical_name} since it's not an airbyte connector.") - elif not args.allow_community and definition in definitions.COMMUNITY_DEFINITIONS: - logging.warning( - f"Skipping {connector_technical_name} since it's a community connector. This is configurable via `--allow_community`" - ) - elif definition in definitions.OTHER_DEFINITIONS: - logging.warning(f"Skipping {connector_technical_name} since it doesn't have a support level.") - else: - valid_definitions.append(definition) - - return valid_definitions diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/conftest.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/conftest.py index 21184060b502..70aeefa7fc82 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/conftest.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/conftest.py @@ -36,3 +36,8 @@ def anyio_backend(): async def dagger_client(): async with dagger.Connection(dagger.Config(log_output=sys.stderr)) as client: yield client + + +@pytest.fixture(scope="module") +async def source_faker_container(dagger_client): + return await dagger_client.container().from_("airbyte/source-faker:latest") diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_backward_compatibility.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_backward_compatibility.py index d2fe626c25f8..306622325f2e 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_backward_compatibility.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_backward_compatibility.py @@ -488,6 +488,132 @@ def as_pytest_param(self): should_fail=True, is_valid_json_schema=False, ), + Transition( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": "string"}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": "string", "format": "date"}, + }, + } + ), + name="Adding a 'format' field should fail.", + should_fail=True, + is_valid_json_schema=False, + ), + Transition( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": "string", "format": "date"}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": "string"}, + }, + } + ), + name="Removing a 'format' field should fail.", + should_fail=True, + is_valid_json_schema=False, + ), + Transition( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": "string", "format": "date"}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": "string", "format": "date-time"}, + }, + } + ), + name="Changing a 'format' field value should fail.", + should_fail=True, + is_valid_json_schema=False, + ), + Transition( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": "string", "format": "time"}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": "string", "format": "time", "airbyte_type": "time_with_timezone"}, + }, + } + ), + name="Adding an 'airbyte_type' field should fail.", + should_fail=True, + is_valid_json_schema=False, + ), + Transition( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": "string", "format": "time", "airbyte_type": "time_with_timezone"}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": "string", "format": "time"}, + }, + } + ), + name="Removing an 'airbyte_type' field should fail.", + should_fail=True, + is_valid_json_schema=False, + ), + Transition( + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": "string", "format": "time", "airbyte_type": "time_with_timezone"}, + }, + } + ), + ConnectorSpecification( + connectionSpecification={ + "type": "object", + "properties": { + "my_string": {"type": "string", "format": "time", "airbyte_type": "time_without_timezone"}, + }, + } + ), + name="Changing an 'airbyte_type' field value should fail.", + should_fail=True, + is_valid_json_schema=False, + ), ] VALID_SPEC_TRANSITIONS = [ @@ -1042,6 +1168,178 @@ def test_validate_previous_configs(previous_connector_spec, actual_connector_spe ) }, ), + Transition( + name="Removing a field format should fail.", + should_fail=True, + previous={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": { + "properties": {"user": {"type": "object", "properties": {"created": {"type": "string", "format": "date"}}}} + }, + "supported_sync_modes": ["full_refresh"], + } + ) + }, + current={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"created": {"type": "string"}}}}}, + "supported_sync_modes": ["full_refresh"], + } + ) + }, + ), + Transition( + name="Adding a field format should fail.", + should_fail=True, + previous={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": {"properties": {"user": {"type": "object", "properties": {"created": {"type": "string"}}}}}, + "supported_sync_modes": ["full_refresh"], + } + ) + }, + current={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": { + "properties": {"user": {"type": "object", "properties": {"created": {"type": "string", "format": "date"}}}} + }, + "supported_sync_modes": ["full_refresh"], + } + ) + }, + ), + Transition( + name="Changing a field format should fail.", + should_fail=True, + previous={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": { + "properties": {"user": {"type": "object", "properties": {"created": {"type": "string", "format": "date-time"}}}} + }, + "supported_sync_modes": ["full_refresh"], + } + ) + }, + current={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": { + "properties": {"user": {"type": "object", "properties": {"created": {"type": "string", "format": "date"}}}} + }, + "supported_sync_modes": ["full_refresh"], + } + ) + }, + ), + Transition( + name="Removing a field airbyte type should fail.", + should_fail=True, + previous={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": { + "properties": { + "user": { + "type": "object", + "properties": {"created": {"type": "string", "format": "time", "airbyte_type": "type_with_timezone"}}, + } + } + }, + "supported_sync_modes": ["full_refresh"], + } + ) + }, + current={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": { + "properties": {"user": {"type": "object", "properties": {"created": {"type": "string", "format": "time"}}}} + }, + "supported_sync_modes": ["full_refresh"], + } + ) + }, + ), + Transition( + name="Adding a field airbyte type should fail.", + should_fail=True, + previous={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": { + "properties": {"user": {"type": "object", "properties": {"created": {"type": "string", "format": "time"}}}} + }, + "supported_sync_modes": ["full_refresh"], + } + ) + }, + current={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": { + "properties": { + "user": { + "type": "object", + "properties": {"created": {"type": "string", "format": "time", "airbyte_type": "time_with_timezone"}}, + } + } + }, + "supported_sync_modes": ["full_refresh"], + } + ) + }, + ), + Transition( + name="Changing a field airbyte type should fail.", + should_fail=True, + previous={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": { + "properties": { + "user": { + "type": "object", + "properties": {"created": {"type": "string", "format": "time", "airbyte_type": "time_with_timezone"}}, + } + } + }, + "supported_sync_modes": ["full_refresh"], + } + ) + }, + current={ + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": { + "properties": { + "user": { + "type": "object", + "properties": {"created": {"type": "string", "format": "time", "airbyte_type": "time_without_timezone"}}, + } + } + }, + "supported_sync_modes": ["full_refresh"], + } + ) + }, + ), Transition( name="Renaming a stream should fail.", should_fail=True, diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_config.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_config.py index 671129ceb922..2687bfaf5101 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_config.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_config.py @@ -100,33 +100,6 @@ def test_config_parsing(self, raw_config, expected_output_config, expected_error parsed_config = config.Config.parse_obj(raw_config) assert parsed_config == expected_output_config - def test_cursor_path_union_str(self): - parsed_config = config.Config.parse_obj(self._config_with_incremental_cursor_paths(["2331"])) - assert type(parsed_config.acceptance_tests.incremental.tests[0].cursor_paths["stream_name"][0]) == str - - def test_cursor_path_union_int(self): - parsed_config = config.Config.parse_obj(self._config_with_incremental_cursor_paths([2331])) - assert type(parsed_config.acceptance_tests.incremental.tests[0].cursor_paths["stream_name"][0]) == int - - @staticmethod - def _config_with_incremental_cursor_paths(cursor_paths): - return { - "connector_image": "foo", - "acceptance_tests": { - "incremental": { - "tests": [ - { - "config_path": "config_path.json", - "cursor_paths": { - "stream_name": cursor_paths - } - } - ] - } - }, - "test_strictness_level": "low" - } - @pytest.mark.parametrize( "legacy_config, expected_parsed_config", [ diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_attributes.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_attributes.py new file mode 100644 index 000000000000..1730c37a9aff --- /dev/null +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_attributes.py @@ -0,0 +1,77 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import pytest +from airbyte_protocol.models import AirbyteCatalog, AirbyteMessage, AirbyteStream, Type +from connector_acceptance_test.config import NoPrimaryKeyConfiguration +from connector_acceptance_test.tests import test_core + +pytestmark = pytest.mark.anyio + + +@pytest.mark.parametrize( + "stream_configs, excluded_streams, expected_error_streams", + [ + pytest.param([{"name": "stream_with_primary_key", "primary_key": [["id"]]}], [], None, id="test_stream_with_primary_key_succeeds"), + pytest.param([{"name": "stream_without_primary_key"}], [], ["stream_without_primary_key"], id="test_stream_without_primary_key_fails"), + pytest.param([{"name": "report_stream"}], ["report_stream"], None, id="test_primary_key_excluded_from_test"), + pytest.param( + [ + {"name": "freiren", "primary_key": [["mage"]]}, + {"name": "himmel"}, + {"name": "eisen", "primary_key": [["warrior"]]}, + {"name": "heiter"}, + ], [], ["himmel", "heiter"], id="test_multiple_streams_that_are_missing_primary_key"), + pytest.param( + [ + {"name": "freiren", "primary_key": [["mage"]]}, + {"name": "himmel"}, + {"name": "eisen", "primary_key": [["warrior"]]}, + {"name": "heiter"}, + ], ["himmel", "heiter"], None, id="test_multiple_streams_that_exclude_primary_key"), + pytest.param( + [ + {"name": "freiren", "primary_key": [["mage"]]}, + {"name": "himmel"}, + {"name": "eisen", "primary_key": [["warrior"]]}, + {"name": "heiter"}, + ], ["heiter"], ["himmel"], id="test_multiple_streams_missing_primary_key_or_excluded"), + ], +) +async def test_streams_define_primary_key(mocker, stream_configs, excluded_streams, expected_error_streams): + t = test_core.TestConnectorAttributes() + + streams = [AirbyteStream.parse_obj({ + "name": stream_config.get("name"), + "json_schema": {}, + "default_cursor_field": ["updated_at"], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": stream_config.get("primary_key"), + }) for stream_config in stream_configs] + + streams_without_primary_key = [NoPrimaryKeyConfiguration(name=stream, bypass_reason="") for stream in excluded_streams] + + docker_runner_mock = mocker.MagicMock( + call_discover=mocker.AsyncMock( + return_value=[AirbyteMessage(type=Type.CATALOG, catalog=AirbyteCatalog(streams=streams))] + ) + ) + + if expected_error_streams: + with pytest.raises(AssertionError) as e: + await t.test_streams_define_primary_key( + operational_certification_test=True, + streams_without_primary_key=streams_without_primary_key, + connector_config={}, + docker_runner=docker_runner_mock + ) + streams_in_error_message = [stream_name for stream_name in expected_error_streams if stream_name in e.value.args[0]] + assert streams_in_error_message == expected_error_streams + else: + await t.test_streams_define_primary_key( + operational_certification_test=True, + streams_without_primary_key=streams_without_primary_key, + connector_config={}, + docker_runner=docker_runner_mock + ) diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_runner.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_runner.py index 0ef84e81b0dd..c7399c1fbe73 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_runner.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_connector_runner.py @@ -5,6 +5,7 @@ import json import os +from pathlib import Path import pytest from airbyte_protocol.models import ( @@ -29,28 +30,15 @@ def dev_image_name(self): def released_image_name(self): return "airbyte/source-faker:latest" - @pytest.fixture() - async def local_tar_image(self, dagger_client, tmpdir, released_image_name): - local_image_tar_path = str(tmpdir / "local_image.tar") - await dagger_client.container().from_(released_image_name).export(local_image_tar_path) - os.environ["CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH"] = local_image_tar_path - yield local_image_tar_path - os.environ.pop("CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH") - - async def test_load_container_from_tar(self, dagger_client, dev_image_name, local_tar_image): - runner = connector_runner.ConnectorRunner(dev_image_name, dagger_client) - await runner.load_container() - assert await runner._connector_under_test_container.with_exec(["spec"]) - - async def test_load_container_from_released_connector(self, dagger_client, released_image_name): - runner = connector_runner.ConnectorRunner(released_image_name, dagger_client) - await runner.load_container() - assert await runner._connector_under_test_container.with_exec(["spec"]) - - async def test_get_container_env_variable_value(self, dagger_client, dev_image_name, local_tar_image): - runner = connector_runner.ConnectorRunner(dev_image_name, dagger_client, custom_environment_variables={"FOO": "BAR"}) + async def test_get_container_env_variable_value(self, source_faker_container): + runner = connector_runner.ConnectorRunner(source_faker_container, custom_environment_variables={"FOO": "BAR"}) assert await runner.get_container_env_variable_value("FOO") == "BAR" + @pytest.mark.parametrize("deployment_mode", ["oss", "cloud"]) + async def test_set_deployment_mode_env(self, source_faker_container, deployment_mode): + runner = connector_runner.ConnectorRunner(source_faker_container, deployment_mode=deployment_mode) + assert await runner.get_container_env_variable_value("DEPLOYMENT_MODE") == deployment_mode.upper() + def test_parse_airbyte_messages_from_command_output(self, mocker, tmp_path): old_configuration_path = tmp_path / "config.json" new_configuration = {"field_a": "new_value_a"} @@ -76,10 +64,8 @@ def test_parse_airbyte_messages_from_command_output(self, mocker, tmp_path): mocker.patch.object(connector_runner.ConnectorRunner, "_persist_new_configuration") runner = connector_runner.ConnectorRunner( - "source-test:dev", mocker.Mock(), connector_configuration_path=old_configuration_path, - custom_environment_variables={"foo": "bar"}, ) runner.parse_airbyte_messages_from_command_output(raw_command_output) runner._persist_new_configuration.assert_called_once_with(new_configuration, 1) @@ -125,10 +111,42 @@ def test_persist_new_configuration( json.dump(old_configuration, old_configuration_file) else: old_configuration_path = None - mocker.patch.object(connector_runner, "docker") - runner = connector_runner.ConnectorRunner("source-test:dev", mocker.MagicMock(), old_configuration_path) + + runner = connector_runner.ConnectorRunner(mocker.MagicMock(), connector_configuration_path=old_configuration_path) new_configuration_path = runner._persist_new_configuration(new_configuration, new_configuration_emitted_at) if not expect_new_configuration: assert new_configuration_path is None else: assert new_configuration_path == tmp_path / "updated_configurations" / f"config|{new_configuration_emitted_at}.json" + + +async def test_get_connector_container(mocker): + + dagger_client = mocker.AsyncMock() + os.environ["CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH"] = "test_tarball_path" + + # Mock the functions called within get_connector_container + mocker.patch.object(connector_runner, "get_container_from_id", new=mocker.AsyncMock()) + mocker.patch.object(connector_runner, "get_container_from_tarball_path", new=mocker.AsyncMock()) + mocker.patch.object(connector_runner, "get_container_from_local_image", new=mocker.AsyncMock()) + mocker.patch.object(connector_runner, "get_container_from_dockerhub_image", new=mocker.AsyncMock()) + + # Test the case when the CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH is set + await connector_runner.get_connector_container(dagger_client, "test_image:tag") + connector_runner.get_container_from_tarball_path.assert_called_with(dagger_client, Path("test_tarball_path")) + + # Test the case when the CONNECTOR_CONTAINER_ID is set + Path("/tmp/container_id.txt").write_text("test_container_id") + await connector_runner.get_connector_container(dagger_client, "test_image:tag") + connector_runner.get_container_from_id.assert_called_with(dagger_client, "test_container_id") + Path("/tmp/container_id.txt").unlink() + + # Test the case when none of the environment variables are set + os.environ.pop("CONNECTOR_UNDER_TEST_IMAGE_TAR_PATH") + await connector_runner.get_connector_container(dagger_client, "test_image:tag") + connector_runner.get_container_from_local_image.assert_called_with(dagger_client, "test_image:tag") + + # Test the case when all previous attempts fail + connector_runner.get_container_from_local_image.return_value = None + await connector_runner.get_connector_container(dagger_client, "test_image:tag") + connector_runner.get_container_from_dockerhub_image.assert_called_with(dagger_client, "test_image:tag") diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py index 7d894b0450f6..ee017ba36c18 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_core.py @@ -23,6 +23,7 @@ ) from connector_acceptance_test.config import BasicReadTestConfig, Config, ExpectedRecordsConfig, IgnoredFieldsConfiguration from connector_acceptance_test.tests import test_core +from jsonschema.exceptions import SchemaError from .conftest import does_not_raise @@ -59,6 +60,99 @@ def test_discovery(schema, cursors, should_fail): t.test_defined_cursors_exist_in_schema(discovered_catalog) +def test_discovery_uniquely_named_streams(): + t = test_core.TestDiscovery() + stream_a = AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": {"properties": {"created": {"type": "string"}}}, + "default_cursor_field": ["created"], + "supported_sync_modes": ["full_refresh", "incremental"], + } + ) + streams = [stream_a, stream_a] + assert t.duplicated_stream_names(streams) == ["test_stream"] + streams.pop() + assert len(t.duplicated_stream_names(streams)) == 0 + + +@pytest.mark.parametrize( + "schema, should_fail", + [ + ( + { + "$schema": "https://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "amount_details": { + "type": ["null", "object"], + "properties": { + "atm_fee": ["null", "integer"] + } + } + } + }, + True + ), + ( + { + "$schema": "https://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "properties": { + "amount": "integer", + "amount_details": { + "type": ["null", "object"], + "properties": { + "atm_fee": { + "type": ["null", "integer"] + } + } + } + } + }, + True + ), + ( + { + "$schema": "https://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "amount_details": { + "type": ["null", "object"], + "properties": { + "atm_fee": { + "type": ["null", "integer"] + } + } + } + } + }, + False + ) + ], +) +def test_streams_have_valid_json_schemas(schema, should_fail): + t = test_core.TestDiscovery() + discovered_catalog = { + "test_stream": AirbyteStream.parse_obj( + { + "name": "test_stream", + "json_schema": schema, + "supported_sync_modes": ["full_refresh", "incremental"], + } + ) + } + expectation = pytest.raises(SchemaError) if should_fail else does_not_raise() + with expectation: + t.test_streams_have_valid_json_schemas(discovered_catalog) + + @pytest.mark.parametrize( "schema, should_fail", [ diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py index 58256392d49f..a357a43f40b3 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_incremental.py @@ -29,11 +29,7 @@ from connector_acceptance_test.config import Config, EmptyStreamConfiguration, IncrementalConfig from connector_acceptance_test.tests import test_incremental from connector_acceptance_test.tests.test_incremental import TestIncremental as _TestIncremental -from connector_acceptance_test.tests.test_incremental import ( - compare_cursor_with_threshold, - future_state_configuration_fixture, - future_state_fixture, -) +from connector_acceptance_test.tests.test_incremental import future_state_configuration_fixture, future_state_fixture pytestmark = [ pytest.mark.anyio, @@ -66,93 +62,79 @@ def build_per_stream_state_message( ) -@pytest.mark.parametrize( - "record_value, state_value, threshold_days, expected_result", - [ - (datetime(2020, 10, 10), datetime(2020, 10, 9), 0, True), - (datetime(2020, 10, 10), datetime(2020, 10, 11), 0, False), - (datetime(2020, 10, 10), datetime(2020, 10, 11), 1, True), - (pendulum.parse("2020-10-10"), pendulum.parse("2020-10-09"), 0, True), - (pendulum.parse("2020-10-10"), pendulum.parse("2020-10-11"), 0, False), - (pendulum.parse("2020-10-10"), pendulum.parse("2020-10-11"), 1, True), - ("2020-10-10", "2020-10-09", 0, True), - ("2020-10-10", "2020-10-11", 0, False), - ("2020-10-10", "2020-10-11", 1, True), - (1602288000000, 1602201600000, 0, True), - (1602288000000, 1602374400000, 0, False), - (1602288000000, 1602374400000, 1, True), - (1602288000, 1602201600, 0, True), - (1602288000, 1602374400, 0, False), - (1602288000, 1602374400, 1, True), - ("aaa", "bbb", 0, False), - ("bbb", "aaa", 0, True), - ], -) -def test_compare_cursor_with_threshold(record_value, state_value, threshold_days, expected_result): - assert compare_cursor_with_threshold(record_value, state_value, threshold_days) == expected_result - - @pytest.mark.parametrize("cursor_type", ["date", "string"]) @pytest.mark.parametrize( - "records1, records2, latest_state, threshold_days, expected_error", + "records1, records2, latest_state, namespace, expected_error", [ - ([{"date": "2020-01-01"}, {"date": "2020-01-02"}], [], "2020-01-02", 0, does_not_raise()), + ([{"date": "2020-01-01"}, {"date": "2020-01-02"}], [], "2020-01-01", None, does_not_raise()), ( [{"date": "2020-01-02"}, {"date": "2020-01-03"}], [], "2020-01-02", - 0, - pytest.raises(AssertionError, match="First incremental sync should produce records younger"), + "public", + does_not_raise(), ), ( - [{"date": "2020-01-01"}, {"date": "2020-01-02"}], [{"date": "2020-01-02"}, {"date": "2020-01-03"}], + [], "2020-01-02", - 0, + None, does_not_raise(), ), ( + [{"date": "2020-01-01"}, {"date": "2020-01-02"}], + [{"date": "2020-01-02"}, {"date": "2020-01-03"}], + "2020-01-03", + None, + does_not_raise(), + ), + ( + [], [{"date": "2020-01-01"}], - [{"date": "2020-01-01"}], - "2020-01-02", - 0, - pytest.raises(AssertionError, match="Second incremental sync should produce records older"), + "2020-01-04", + None, + pytest.raises(AssertionError, match="First Read should produce at least one record"), ), ( [{"date": "2020-01-01"}, {"date": "2020-01-02"}], [{"date": "2020-01-01"}, {"date": "2020-01-02"}], - "2020-01-03", - 2, - does_not_raise(), + "2020-01-05", + None, + pytest.raises(AssertionError, match="Records should change between reads but did not."), ), ( [{"date": "2020-01-02"}, {"date": "2020-01-03"}], [], - "2020-01-02", - 2, - pytest.raises(AssertionError, match="First incremental sync should produce records younger"), + "2020-01-06", + None, + does_not_raise(), ), ( [{"date": "2020-01-01"}], [{"date": "2020-01-02"}], - "2020-01-06", - 3, - pytest.raises(AssertionError, match="Second incremental sync should produce records older"), + "2020-01-07", + "public", + does_not_raise(), + ), + ( + [{"date": "2020-01-01"}], + [{"date": "2020-01-02"}], + "someunparseablenonsensestate", + None, + does_not_raise(), ), ], ) @pytest.mark.parametrize( "run_per_stream_test", [ - # pytest.param(False, id="test_two_sequential_reads_using_a_mock_connector_emitting_legacy_state"), + pytest.param(False, id="test_two_sequential_reads_using_a_mock_connector_emitting_legacy_state"), pytest.param(True, id="test_two_sequential_reads_using_a_mock_connector_emitting_per_stream_state"), ], ) async def test_incremental_two_sequential_reads( - mocker, records1, records2, latest_state, threshold_days, cursor_type, expected_error, run_per_stream_test + mocker, records1, records2, latest_state, namespace, cursor_type, expected_error, run_per_stream_test ): - input_config = IncrementalConfig(threshold_days=threshold_days) - cursor_paths = {"test_stream": ["date"]} catalog = ConfiguredAirbyteCatalog( streams=[ ConfiguredAirbyteStream( @@ -169,9 +151,10 @@ async def test_incremental_two_sequential_reads( ) if run_per_stream_test: + stream_descriptor = StreamDescriptor(name="test_stream", namespace=namespace) call_read_output_messages = [ *build_messages_from_record_data("test_stream", records1), - build_per_stream_state_message(descriptor=StreamDescriptor(name="test_stream"), stream_state={"date": latest_state}), + build_per_stream_state_message(descriptor=stream_descriptor, stream_state={"date": latest_state}), ] call_read_with_state_output_messages = build_messages_from_record_data("test_stream", records2) else: @@ -188,96 +171,14 @@ async def test_incremental_two_sequential_reads( t = _TestIncremental() with expected_error: await t.test_two_sequential_reads( - inputs=input_config, - connector_config=MagicMock(), - configured_catalog_for_incremental=catalog, - cursor_paths=cursor_paths, - docker_runner=docker_runner_mock, - ) - - -@pytest.mark.parametrize( - "stream_name, cursor_type, cursor_paths, records1, records2, latest_state, expected_error", - [ - ( - "test_stream", - {"dateCreated": {"type": "string", "format": "date-time"}}, - {"test_stream": ["dateCreated"]}, - [{"dateCreated": "2020-01-01T01:01:01.000000Z"}, {"dateCreated": "2020-01-02T01:01:01.000000Z"}], - [], - {"dateCreated": "2020-01-02T01:01:01.000000Z"}, - does_not_raise(), - ), - ( - "test_stream", - {"dateCreated": {"type": "string", "format": "date-time"}}, - {"test_stream": ["dateCreated"]}, - [{"dateCreated": "2020-01-01T01:01:01.000000Z"}, {"dateCreated": "2020-01-02T01:01:01.000000Z"}], - [], - {}, - pytest.raises(AssertionError, match="At least one valid state should be produced, given a cursor path"), - ), - ], -) -@pytest.mark.parametrize( - "run_per_stream_test", - [ - pytest.param(False, id="test_two_sequential_reads_using_a_mock_connector_emitting_legacy_state"), - pytest.param(True, id="test_two_sequential_reads_using_a_mock_connector_emitting_per_stream_state"), - ], -) -async def test_incremental_two_sequential_reads_state_invalid( - mocker, stream_name, records1, records2, latest_state, cursor_type, cursor_paths, expected_error, run_per_stream_test -): - input_config = IncrementalConfig() - catalog = ConfiguredAirbyteCatalog( - streams=[ - ConfiguredAirbyteStream( - stream=AirbyteStream( - name=stream_name, - json_schema={"type": "object", "properties": cursor_type}, - supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental], - ), - sync_mode=SyncMode.incremental, - destination_sync_mode=DestinationSyncMode.overwrite, - default_cursor_field=["dateCreated"], - cursor_field=["dateCreated"], - ) - ] - ) - - if run_per_stream_test: - call_read_output_messages = [ - *build_messages_from_record_data(stream_name, records1), - build_per_stream_state_message(descriptor=StreamDescriptor(name=stream_name), stream_state=latest_state), - ] - else: - stream_state = dict() - stream_state[stream_name] = latest_state - call_read_output_messages = [ - *build_messages_from_record_data(stream_name, records1), - build_state_message(stream_state), - ] - - call_read_with_state_output_messages = build_messages_from_record_data(stream_name, records2) - - docker_runner_mock = MagicMock() - docker_runner_mock.call_read = mocker.AsyncMock(return_value=call_read_output_messages) - docker_runner_mock.call_read_with_state = mocker.AsyncMock(return_value=call_read_with_state_output_messages) - - t = _TestIncremental() - with expected_error: - await t.test_two_sequential_reads( - inputs=input_config, connector_config=MagicMock(), configured_catalog_for_incremental=catalog, - cursor_paths=cursor_paths, docker_runner=docker_runner_mock, ) @pytest.mark.parametrize( - "records, state_records, threshold_days, expected_error", + "first_records, subsequent_records, expected_error", [ pytest.param( [ @@ -294,9 +195,6 @@ async def test_incremental_two_sequential_reads_state_invalid( ], [ [ - {"type": Type.STATE, "name": "test_stream", "stream_state": {}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-07"}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-08"}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-09"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-09"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-10"}}, @@ -305,6 +203,27 @@ async def test_incremental_two_sequential_reads_state_invalid( {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-13"}}, ], + [ + {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-11"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-11"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, + {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-13"}}, + ], + [ + {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-13"}}, + ], + ], + does_not_raise(), + id="test_incremental_with_2_states", + ), + pytest.param( + [ + {"type": Type.STATE, "name": "test_stream", "stream_state": {}}, + {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-09"}}, + {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-11"}}, + {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-13"}}, + ], + [ [ {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-09"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-09"}}, @@ -324,43 +243,40 @@ async def test_incremental_two_sequential_reads_state_invalid( {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-13"}}, ], ], - 0, - does_not_raise(), - id="test_incremental_with_2_states", + pytest.raises(AssertionError, match="First Read should produce at least one record"), + id="test_incremental_no_record_on_first_read_raises_error", ), pytest.param( [ - {"type": Type.STATE, "name": "test_stream", "stream_state": {}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-07"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-08"}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-09"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-09"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-10"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-11"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-13"}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-11"}}, ], [ [ - {"type": Type.STATE, "name": "test_stream", "stream_state": {}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-07"}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-08"}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-09"}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-13"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-09"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-10"}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-11"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-11"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, + {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-13"}}, ], [ - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-09"}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-13"}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-11"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-11"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, + {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-13"}}, ], [ - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-11"}}, + {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-13"}}, ], ], - 0, - pytest.raises(AssertionError), - id="test_first_incremental_only_younger_records", + pytest.raises(AssertionError, match="First Read should produce at least one state"), + id="test_incremental_no_state_on_first_read_raises_error", ), pytest.param( [ @@ -368,33 +284,23 @@ async def test_incremental_two_sequential_reads_state_invalid( {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-07"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-08"}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-09"}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-07"}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-08"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-13"}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-11"}}, ], [ - [ - {"type": Type.STATE, "name": "test_stream", "stream_state": {}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-07"}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-08"}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-09"}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-07"}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-08"}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-11"}}, - ], [ {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-09"}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-07"}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-08"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-13"}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-11"}}, ], [ {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-11"}}, ], ], - 3, does_not_raise(), - id="test_incremental_with_threshold", + id="test_first_incremental_only_younger_records", ), pytest.param( [ @@ -421,29 +327,10 @@ async def test_incremental_two_sequential_reads_state_invalid( {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-11"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-13"}}, - ], - [ - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-09"}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-04"}}, # out of order - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-05"}}, # out of order - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-11"}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-11"}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-13"}}, - ], - [ - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-11"}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-11"}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-13"}}, - ], - [ - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-13"}}, - ], + ] ], - 0, - pytest.raises(AssertionError), - id="test_incremental_with_incorrect_messages", + pytest.raises(AssertionError, match="Records for subsequent reads with new state should be different"), + id="test_incremental_returns_identical", ), pytest.param( [ @@ -472,31 +359,6 @@ async def test_incremental_two_sequential_reads_state_invalid( }, ], [ - [ - {"type": Type.STATE, "name": "test_stream", "stream_state": {}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-07"}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-08"}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-09"}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-09"}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-10"}}, - {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-11"}}, - {"type": Type.RECORD, "name": "test_stream_2", "data": {"date": "2022-05-11"}}, - {"type": Type.RECORD, "name": "test_stream_2", "data": {"date": "2022-05-12"}}, - { - "type": Type.STATE, - "name": "test_stream_2", - "stream_state": {"date": "2022-05-13"}, - "data": {"test_stream": {"date": "2022-05-11"}, "test_stream_2": {"date": "2022-05-13"}}, - }, - {"type": Type.RECORD, "name": "test_stream_2", "data": {"date": "2022-05-13"}}, - {"type": Type.RECORD, "name": "test_stream_2", "data": {"date": "2022-05-14"}}, - { - "type": Type.STATE, - "name": "test_stream_2", - "stream_state": {"date": "2022-05-15"}, - "data": {"test_stream": {"date": "2022-05-11"}, "test_stream_2": {"date": "2022-05-15"}}, - }, - ], [ {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-09"}}, {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-09"}}, @@ -563,7 +425,6 @@ async def test_incremental_two_sequential_reads_state_invalid( }, ], ], - 0, does_not_raise(), id="test_incremental_with_multiple_streams", ), @@ -576,17 +437,31 @@ async def test_incremental_two_sequential_reads_state_invalid( ], [ [ - {"type": Type.STATE, "name": "test_stream", "stream_state": None}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-07"}}, - {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-08"}}, {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-09"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-09"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-10"}}, + {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-11"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-11"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, + {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-13"}}, ], - [], ], - 0, does_not_raise(), id="test_incremental_with_none_state", ), + pytest.param( + [ + {"type": Type.STATE, "name": "test_stream", "stream_state": {}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-07"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-08"}}, + {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-09"}}, + ], + [ + [], + ], + does_not_raise(), + id="test_incremental_with_empty_second_read", + ), ], ) @pytest.mark.parametrize( @@ -596,9 +471,7 @@ async def test_incremental_two_sequential_reads_state_invalid( pytest.param(True, id="test_read_with_multiple_states_using_a_mock_connector_emitting_per_stream_state"), ], ) -async def test_per_stream_read_with_multiple_states(mocker, records, state_records, threshold_days, expected_error, run_per_stream_test): - input_config = IncrementalConfig(threshold_days=threshold_days) - cursor_paths = {"test_stream": ["date"], "test_stream_2": ["date"]} +async def test_per_stream_read_with_multiple_states(mocker, first_records, subsequent_records, expected_error, run_per_stream_test): catalog = ConfiguredAirbyteCatalog( streams=[ ConfiguredAirbyteStream( @@ -631,7 +504,7 @@ async def test_per_stream_read_with_multiple_states(mocker, records, state_recor ) if record["type"] == Type.STATE else build_record_message(record["name"], record["data"]) - for record in list(records) + for record in list(first_records) ] call_read_with_state_output_messages = [ [ @@ -642,14 +515,14 @@ async def test_per_stream_read_with_multiple_states(mocker, records, state_recor else build_record_message(stream=record["name"], data=record["data"]) for record in state_records_group ] - for state_records_group in list(state_records) + for state_records_group in list(subsequent_records) ] else: call_read_output_messages = [ build_state_message(state=record.get("data") or {record["name"]: record["stream_state"]}) if record["type"] == Type.STATE else build_record_message(stream=record["name"], data=record["data"]) - for record in list(records) + for record in list(first_records) ] call_read_with_state_output_messages = [ [ @@ -658,7 +531,7 @@ async def test_per_stream_read_with_multiple_states(mocker, records, state_recor else build_record_message(stream=record["name"], data=record["data"]) for record in state_records_group ] - for state_records_group in list(state_records) + for state_records_group in list(subsequent_records) ] docker_runner_mock = MagicMock() @@ -666,22 +539,22 @@ async def test_per_stream_read_with_multiple_states(mocker, records, state_recor docker_runner_mock.call_read_with_state = mocker.AsyncMock(side_effect=call_read_with_state_output_messages) t = _TestIncremental() + # test if skipped with expected_error: await t.test_read_sequential_slices( - inputs=input_config, connector_config=MagicMock(), configured_catalog_for_incremental=catalog, - cursor_paths=cursor_paths, docker_runner=docker_runner_mock, + inputs=IncrementalConfig(), ) -def test_config_skip_test(): +async def test_config_skip_test(mocker): docker_runner_mock = MagicMock() - docker_runner_mock.call_read.return_value = [] + docker_runner_mock.call_read = mocker.AsyncMock(return_value=[]) t = _TestIncremental() with patch.object(pytest, "skip", return_value=None): - t.test_read_sequential_slices( + await t.test_read_sequential_slices( inputs=IncrementalConfig(skip_comprehensive_incremental_tests=True), connector_config=MagicMock(), configured_catalog_for_incremental=ConfiguredAirbyteCatalog( @@ -698,7 +571,6 @@ def test_config_skip_test(): ) ] ), - cursor_paths={}, docker_runner=docker_runner_mock, ) @@ -706,6 +578,59 @@ def test_config_skip_test(): docker_runner_mock.call_read.assert_not_called() +async def test_state_skip_test(mocker): + docker_runner_mock = MagicMock() + + first_records = [ + {"type": Type.STATE, "name": "test_stream", "stream_state": {}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-07"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-08"}}, + {"type": Type.STATE, "name": "test_stream", "stream_state": {"date": "2022-05-09"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-09"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-10"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-11"}}, + {"type": Type.RECORD, "name": "test_stream", "data": {"date": "2022-05-12"}}, + ] + + call_read_output_messages = [ + build_per_stream_state_message( + descriptor=StreamDescriptor(name=record["name"]), stream_state=record["stream_state"], data=record.get("data", None) + ) + if record["type"] == Type.STATE + else build_record_message(record["name"], record["data"]) + for record in list(first_records) + ] + + # There needs to be at least 3 state messages for the test to run + docker_runner_mock.call_read = mocker.AsyncMock(return_value=call_read_output_messages) + + t = _TestIncremental() + with patch.object(pytest, "skip", return_value=None): + await t.test_read_sequential_slices( + inputs=IncrementalConfig(), + connector_config=MagicMock(), + configured_catalog_for_incremental=ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream( + name="test_stream", + json_schema={"type": "object", "properties": {"date": {"type": "date"}}}, + supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental], + ), + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.overwrite, + cursor_field=["date"], + ) + ] + ), + docker_runner=docker_runner_mock, + ) + + # This is guaranteed to fail when the test gets executed + docker_runner_mock.call_read.assert_called() + docker_runner_mock.call_read_with_state.assert_not_called() + + @pytest.mark.parametrize( "read_output, expectation", [ diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_json_schema_helper.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_json_schema_helper.py index afb0ae67889c..a8f2f884d1f3 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_json_schema_helper.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_json_schema_helper.py @@ -3,7 +3,7 @@ # from enum import Enum -from typing import Any, List, Text, Union +from typing import Any, Iterable, List, Text, Tuple, Union import pendulum import pytest @@ -16,11 +16,34 @@ SyncMode, Type, ) -from connector_acceptance_test.tests.test_incremental import records_with_state from connector_acceptance_test.utils.json_schema_helper import JsonSchemaHelper, get_expected_schema_structure, get_object_structure from pydantic import BaseModel +def records_with_state(records, state, stream_mapping, state_cursor_paths) -> Iterable[Tuple[Any, Any, Any]]: + """Iterate over records and return cursor value with corresponding cursor value from state""" + + for record in records: + stream_name = record.record.stream + stream = stream_mapping[stream_name] + helper = JsonSchemaHelper(schema=stream.stream.json_schema) + cursor_field = helper.field(stream.cursor_field) + record_value = cursor_field.parse(record=record.record.data) + try: + if state[stream_name] is None: + continue + + # first attempt to parse the state value assuming the state object is namespaced on stream names + state_value = cursor_field.parse(record=state[stream_name], path=state_cursor_paths[stream_name]) + except KeyError: + try: + # try second time as an absolute path in state file (i.e. bookmarks -> stream_name -> column -> value) + state_value = cursor_field.parse(record=state, path=state_cursor_paths[stream_name]) + except KeyError: + continue + yield record_value, state_value, stream_name + + @pytest.fixture(name="simple_state") def simple_state_fixture(): return { diff --git a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py index fdca8ac03994..3303c2650d1e 100644 --- a/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py +++ b/airbyte-integrations/bases/connector-acceptance-test/unit_tests/test_spec.py @@ -1016,40 +1016,123 @@ def test_nested_group(mocker, connector_spec, should_fail): @pytest.mark.parametrize( "connector_spec, should_fail", ( - ({"type": "object", "properties": {"refresh_token": {"type": "boolean", "airbyte_secret": True}}}, False), - ({"type": "object", "properties": {"prop": {"type": "boolean", "airbyte_secret": True, "always_show": True}}}, False), + # SUCCESS: no display_type specified ( { "type": "object", - "required": ["prop"], - "properties": {"prop": {"type": "boolean", "airbyte_secret": True, "always_show": True}}, + "properties": { + "select_type": { + "type": "object", + "oneOf": [ + { + "type": "object", + "properties": { + "option_title": {"type": "string", "title": "Title", "const": "first option"}, + "something": {"type": "string"}, + }, + }, + { + "type": "object", + "properties": { + "option_title": {"type": "string", "title": "Title", "const": "second option"}, + "some_field": {"type": "boolean"}, + }, + }, + ], + }, + }, }, - True, + False, + ), + # SUCCESS: display_type is set to a valid value on a field with oneOf set + ( + { + "type": "object", + "properties": { + "select_type": { + "type": "object", + "display_type": "radio", + "oneOf": [ + { + "type": "object", + "properties": { + "option_title": {"type": "string", "title": "Title", "const": "first option"}, + "something": {"type": "string"}, + }, + }, + { + "type": "object", + "properties": { + "option_title": {"type": "string", "title": "Title", "const": "second option"}, + "some_field": {"type": "boolean"}, + }, + }, + ], + }, + }, + }, + False, ), + # SUCCESS: display_type is the name of the property ( - {"type": "object", "properties": {"jwt": {"type": "object", "properties": {"a": {"type": "string", "always_show": True}}}}}, + { + "type": "object", + "properties": { + "display_type": { + "type": "string", + }, + }, + }, False, ), + # FAILURE: display_type is set to an invalid value ( { "type": "object", - "properties": {"jwt": {"type": "object", "required": ["a"], "properties": {"a": {"type": "string", "always_show": True}}}}, + "properties": { + "select_type": { + "type": "object", + "display_type": "invalid", + "oneOf": [ + { + "type": "object", + "properties": { + "option_title": {"type": "string", "title": "Title", "const": "first option"}, + "something": {"type": "string"}, + }, + }, + { + "type": "object", + "properties": { + "option_title": {"type": "string", "title": "Title", "const": "second option"}, + "some_field": {"type": "boolean"}, + }, + }, + ], + }, + }, }, True, ), + # FAILURE: display_type is set on a non-oneOf field ( { "type": "object", - "properties": {"jwt": {"type": "object", "required": ["always_show"], "properties": {"always_show": {"type": "string"}}}}, + "properties": { + "select_type": { + "type": "string", + "display_type": "dropdown", + }, + }, }, - False, + True, ), ), ) -def test_required_always_show(mocker, connector_spec, should_fail): +def test_display_type(mocker, connector_spec, should_fail): mocker.patch.object(conftest.pytest, "fail") t = _TestSpec() - t.test_required_always_show(ConnectorSpecification(connectionSpecification=connector_spec)) + t.test_display_type(ConnectorSpecification(connectionSpecification=connector_spec)) if should_fail: conftest.pytest.fail.assert_called_once() else: diff --git a/airbyte-integrations/bases/debezium/build.gradle b/airbyte-integrations/bases/debezium/build.gradle deleted file mode 100644 index 5e515f0c6876..000000000000 --- a/airbyte-integrations/bases/debezium/build.gradle +++ /dev/null @@ -1,27 +0,0 @@ -plugins { - id "java-test-fixtures" -} - -project.configurations { - testFixturesImplementation.extendsFrom implementation -} -dependencies { - implementation libs.airbyte.protocol - implementation project(':airbyte-db:db-lib') - - implementation libs.bundles.debezium.bundle - implementation 'org.codehaus.plexus:plexus-utils:3.4.2' - - testFixturesImplementation project(':airbyte-db:db-lib') - testFixturesImplementation project(':airbyte-integrations:bases:base-java') - - testImplementation project(':airbyte-test-utils') - testImplementation libs.connectors.testcontainers.jdbc - testImplementation libs.connectors.testcontainers.mysql - testImplementation libs.connectors.testcontainers.postgresql - - testFixturesImplementation 'org.junit.jupiter:junit-jupiter-engine:5.4.2' - testFixturesImplementation 'org.junit.jupiter:junit-jupiter-api:5.4.2' - testFixturesImplementation 'org.junit.jupiter:junit-jupiter-params:5.4.2' - -} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/AirbyteDebeziumHandler.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/AirbyteDebeziumHandler.java deleted file mode 100644 index 432539edd373..000000000000 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/AirbyteDebeziumHandler.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.debezium; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.util.AutoCloseableIterator; -import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.commons.util.MoreIterators; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.debezium.internals.*; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.SyncMode; -import io.debezium.engine.ChangeEvent; -import java.time.Duration; -import java.time.Instant; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.Properties; -import java.util.concurrent.LinkedBlockingQueue; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class acts as the bridge between Airbyte DB connectors and debezium. If a DB connector wants - * to use debezium for CDC, it should use this class - */ -public class AirbyteDebeziumHandler { - - private static final Logger LOGGER = LoggerFactory.getLogger(AirbyteDebeziumHandler.class); - /** - * We use 10000 as capacity cause the default queue size and batch size of debezium is : - * {@link io.debezium.config.CommonConnectorConfig#DEFAULT_MAX_BATCH_SIZE}is 2048 - * {@link io.debezium.config.CommonConnectorConfig#DEFAULT_MAX_QUEUE_SIZE} is 8192 - */ - private static final int QUEUE_CAPACITY = 10000; - - private final JsonNode config; - private final CdcTargetPosition targetPosition; - private final boolean trackSchemaHistory; - private final Duration firstRecordWaitTime; - private final OptionalInt queueSize; - - public AirbyteDebeziumHandler(final JsonNode config, - final CdcTargetPosition targetPosition, - final boolean trackSchemaHistory, - final Duration firstRecordWaitTime, - final OptionalInt queueSize) { - this.config = config; - this.targetPosition = targetPosition; - this.trackSchemaHistory = trackSchemaHistory; - this.firstRecordWaitTime = firstRecordWaitTime; - this.queueSize = queueSize; - } - - public AutoCloseableIterator getSnapshotIterators( - final ConfiguredAirbyteCatalog catalogContainingStreamsToSnapshot, - final CdcMetadataInjector cdcMetadataInjector, - final Properties snapshotProperties, - final CdcStateHandler cdcStateHandler, - final Instant emittedAt) { - - LOGGER.info("Running snapshot for " + catalogContainingStreamsToSnapshot.getStreams().size() + " new tables"); - final LinkedBlockingQueue> queue = new LinkedBlockingQueue<>(queueSize.orElse(QUEUE_CAPACITY)); - - final AirbyteFileOffsetBackingStore offsetManager = AirbyteFileOffsetBackingStore.initializeDummyStateForSnapshotPurpose(); - final DebeziumRecordPublisher tableSnapshotPublisher = new DebeziumRecordPublisher(snapshotProperties, - config, - catalogContainingStreamsToSnapshot, - offsetManager, - schemaHistoryManager(new EmptySavedInfo())); - tableSnapshotPublisher.start(queue); - - final AutoCloseableIterator eventIterator = new DebeziumRecordIterator<>( - queue, - targetPosition, - tableSnapshotPublisher::hasClosed, - new DebeziumShutdownProcedure<>(queue, tableSnapshotPublisher::close, tableSnapshotPublisher::hasClosed), - firstRecordWaitTime); - - return AutoCloseableIterators.concatWithEagerClose(AutoCloseableIterators - .transform( - eventIterator, - (event) -> DebeziumEventUtils.toAirbyteMessage(event, cdcMetadataInjector, emittedAt)), - AutoCloseableIterators - .fromIterator(MoreIterators.singletonIteratorFromSupplier(cdcStateHandler::saveStateAfterCompletionOfSnapshotOfNewStreams))); - } - - public AutoCloseableIterator getIncrementalIterators(final ConfiguredAirbyteCatalog catalog, - final CdcSavedInfoFetcher cdcSavedInfoFetcher, - final CdcStateHandler cdcStateHandler, - final CdcMetadataInjector cdcMetadataInjector, - final Properties connectorProperties, - final Instant emittedAt, - final boolean addDbNameToState) { - LOGGER.info("Using CDC: {}", true); - final LinkedBlockingQueue> queue = new LinkedBlockingQueue<>(queueSize.orElse(QUEUE_CAPACITY)); - final AirbyteFileOffsetBackingStore offsetManager = AirbyteFileOffsetBackingStore.initializeState(cdcSavedInfoFetcher.getSavedOffset(), - addDbNameToState ? Optional.ofNullable(config.get(JdbcUtils.DATABASE_KEY).asText()) : Optional.empty()); - final Optional schemaHistoryManager = schemaHistoryManager(cdcSavedInfoFetcher); - final DebeziumRecordPublisher publisher = new DebeziumRecordPublisher(connectorProperties, config, catalog, offsetManager, - schemaHistoryManager); - publisher.start(queue); - - // handle state machine around pub/sub logic. - final AutoCloseableIterator eventIterator = new DebeziumRecordIterator<>( - queue, - targetPosition, - publisher::hasClosed, - new DebeziumShutdownProcedure<>(queue, publisher::close, publisher::hasClosed), - firstRecordWaitTime); - - final Duration syncCheckpointDuration = - config.get("sync_checkpoint_seconds") != null ? Duration.ofSeconds(config.get("sync_checkpoint_seconds").asLong()) - : DebeziumStateDecoratingIterator.SYNC_CHECKPOINT_DURATION; - final Long syncCheckpointRecords = config.get("sync_checkpoint_records") != null ? config.get("sync_checkpoint_records").asLong() - : DebeziumStateDecoratingIterator.SYNC_CHECKPOINT_RECORDS; - return AutoCloseableIterators.fromIterator(new DebeziumStateDecoratingIterator<>( - eventIterator, - cdcStateHandler, - targetPosition, - cdcMetadataInjector, - emittedAt, - offsetManager, - trackSchemaHistory, - schemaHistoryManager.orElse(null), - syncCheckpointDuration, - syncCheckpointRecords)); - } - - private Optional schemaHistoryManager(final CdcSavedInfoFetcher cdcSavedInfoFetcher) { - if (trackSchemaHistory) { - return Optional.of(AirbyteSchemaHistoryStorage.initializeDBHistory(cdcSavedInfoFetcher.getSavedSchemaHistory())); - } - - return Optional.empty(); - } - - public static boolean shouldUseCDC(final ConfiguredAirbyteCatalog catalog) { - return catalog.getStreams().stream().map(ConfiguredAirbyteStream::getSyncMode) - .anyMatch(syncMode -> syncMode == SyncMode.INCREMENTAL); - } - - private static class EmptySavedInfo implements CdcSavedInfoFetcher { - - @Override - public JsonNode getSavedOffset() { - return null; - } - - @Override - public Optional getSavedSchemaHistory() { - return Optional.empty(); - } - - } - -} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/CdcMetadataInjector.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/CdcMetadataInjector.java deleted file mode 100644 index 9b27dc3b5280..000000000000 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/CdcMetadataInjector.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.debezium; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; - -/** - * This interface is used to add metadata to the records fetched from the database. For instance, in - * Postgres we add the lsn to the records. In MySql we add the file name and position to the - * records. - */ -public interface CdcMetadataInjector { - - /** - * A debezium record contains multiple pieces. Ref : - * https://debezium.io/documentation/reference/1.9/connectors/mysql.html#mysql-create-events - * - * @param event is the actual record which contains data and would be written to the destination - * @param source contains the metadata about the record and we need to extract that metadata and add - * it to the event before writing it to destination - */ - void addMetaData(ObjectNode event, JsonNode source); - - default void addMetaDataToRowsFetchedOutsideDebezium(final ObjectNode record, final String transactionTimestamp, final T metadataToAdd) { - throw new RuntimeException("Not Supported"); - } - - /** - * As part of Airbyte record we need to add the namespace (schema name) - * - * @param source part of debezium record and contains the metadata about the record. We need to - * extract namespace out of this metadata and return Ref : - * https://debezium.io/documentation/reference/1.9/connectors/mysql.html#mysql-create-events - */ - String namespace(JsonNode source); - -} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/CdcSavedInfoFetcher.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/CdcSavedInfoFetcher.java deleted file mode 100644 index dcd96fd3feae..000000000000 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/CdcSavedInfoFetcher.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.debezium; - -import com.fasterxml.jackson.databind.JsonNode; -import java.util.Optional; - -/** - * This interface is used to fetch the saved info required for debezium to run incrementally. Each - * connector saves offset and schema history in different manner - */ -public interface CdcSavedInfoFetcher { - - JsonNode getSavedOffset(); - - Optional getSavedSchemaHistory(); - -} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/CdcStateHandler.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/CdcStateHandler.java deleted file mode 100644 index 16704ad135d5..000000000000 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/CdcStateHandler.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.debezium; - -import io.airbyte.protocol.models.v0.AirbyteMessage; -import java.util.Map; - -/** - * This interface is used to allow connectors to save the offset and schema history in the manner - * which suits them. Also, it adds some utils to verify CDC event status. - */ -public interface CdcStateHandler { - - AirbyteMessage saveState(Map offset, String dbHistory); - - AirbyteMessage saveStateAfterCompletionOfSnapshotOfNewStreams(); - - /** - * This function is used as feature flag for sending state messages as checkpoints in CDC syncs. - * - * @return Returns `true` if checkpoint state messages are enabled for CDC syncs. Otherwise, it - * returns `false` - */ - default boolean isCdcCheckpointEnabled() { - return false; - } - -} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/AirbyteSchemaHistoryStorage.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/AirbyteSchemaHistoryStorage.java deleted file mode 100644 index 1aa158ea5406..000000000000 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/AirbyteSchemaHistoryStorage.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.debezium.internals; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.Jsons; -import io.debezium.document.Document; -import io.debezium.document.DocumentReader; -import io.debezium.document.DocumentWriter; -import java.io.BufferedWriter; -import java.io.IOException; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.Optional; -import org.apache.commons.io.FileUtils; - -/** - * The purpose of this class is : to , 1. Read the contents of the file {@link #path} which contains - * the schema history at the end of the sync so that it can be saved in state for future syncs. - * Check {@link #read()} 2. Write the saved content back to the file {@link #path} at the beginning - * of the sync so that debezium can function smoothly. Check persist(Optional<JsonNode>). - */ -public class AirbyteSchemaHistoryStorage { - - private final Path path; - private static final Charset UTF8 = StandardCharsets.UTF_8; - private final DocumentReader reader = DocumentReader.defaultReader(); - private final DocumentWriter writer = DocumentWriter.defaultWriter(); - - public AirbyteSchemaHistoryStorage(final Path path) { - this.path = path; - } - - public Path getPath() { - return path; - } - - public String read() { - final StringBuilder fileAsString = new StringBuilder(); - try { - for (final String line : Files.readAllLines(path, UTF8)) { - if (line != null && !line.isEmpty()) { - final Document record = reader.read(line); - final String recordAsString = writer.write(record); - fileAsString.append(recordAsString); - fileAsString.append(System.lineSeparator()); - } - } - return fileAsString.toString(); - } catch (final IOException e) { - throw new RuntimeException(e); - } - } - - private void makeSureFileExists() { - try { - // Make sure the file exists ... - if (!Files.exists(path)) { - // Create parent directories if we have them ... - if (path.getParent() != null) { - Files.createDirectories(path.getParent()); - } - try { - Files.createFile(path); - } catch (final FileAlreadyExistsException e) { - // do nothing - } - } - } catch (final IOException e) { - throw new IllegalStateException( - "Unable to check or create history file at " + path + ": " + e.getMessage(), e); - } - } - - public void persist(final Optional schemaHistory) { - if (schemaHistory.isEmpty()) { - return; - } - final String fileAsString = Jsons.object(schemaHistory.get(), String.class); - - if (fileAsString == null || fileAsString.isEmpty()) { - return; - } - - FileUtils.deleteQuietly(path.toFile()); - makeSureFileExists(); - writeToFile(fileAsString); - } - - /** - * @param fileAsString Represents the contents of the file saved in state from previous syncs - */ - private void writeToFile(final String fileAsString) { - try { - final String[] split = fileAsString.split(System.lineSeparator()); - for (final String element : split) { - final Document read = reader.read(element); - final String line = writer.write(read); - - try (final BufferedWriter historyWriter = Files - .newBufferedWriter(path, StandardOpenOption.APPEND)) { - try { - historyWriter.append(line); - historyWriter.newLine(); - } catch (final IOException e) { - throw new RuntimeException(e); - } - } - } - } catch (final IOException e) { - throw new RuntimeException(e); - } - } - - public static AirbyteSchemaHistoryStorage initializeDBHistory(final Optional schemaHistory) { - final Path dbHistoryWorkingDir; - try { - dbHistoryWorkingDir = Files.createTempDirectory(Path.of("/tmp"), "cdc-db-history"); - } catch (final IOException e) { - throw new RuntimeException(e); - } - final Path dbHistoryFilePath = dbHistoryWorkingDir.resolve("dbhistory.dat"); - - final AirbyteSchemaHistoryStorage schemaHistoryManager = new AirbyteSchemaHistoryStorage(dbHistoryFilePath); - schemaHistoryManager.persist(schemaHistory); - return schemaHistoryManager; - } - -} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumEventUtils.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumEventUtils.java deleted file mode 100644 index 001820783c24..000000000000 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumEventUtils.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.debezium.internals; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.integrations.debezium.CdcMetadataInjector; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import java.sql.Timestamp; -import java.time.Instant; - -public class DebeziumEventUtils { - - public static final String CDC_LSN = "_ab_cdc_lsn"; - public static final String CDC_UPDATED_AT = "_ab_cdc_updated_at"; - public static final String CDC_DELETED_AT = "_ab_cdc_deleted_at"; - - public static AirbyteMessage toAirbyteMessage(final ChangeEventWithMetadata event, - final CdcMetadataInjector cdcMetadataInjector, - final Instant emittedAt) { - final JsonNode debeziumRecord = event.eventValueAsJson(); - final JsonNode before = debeziumRecord.get("before"); - final JsonNode after = debeziumRecord.get("after"); - final JsonNode source = debeziumRecord.get("source"); - - final JsonNode data = formatDebeziumData(before, after, source, cdcMetadataInjector); - final String schemaName = cdcMetadataInjector.namespace(source); - final String streamName = source.get("table").asText(); - - final AirbyteRecordMessage airbyteRecordMessage = new AirbyteRecordMessage() - .withStream(streamName) - .withNamespace(schemaName) - .withEmittedAt(emittedAt.toEpochMilli()) - .withData(data); - - return new AirbyteMessage() - .withType(AirbyteMessage.Type.RECORD) - .withRecord(airbyteRecordMessage); - } - - // warning mutates input args. - private static JsonNode formatDebeziumData(final JsonNode before, - final JsonNode after, - final JsonNode source, - final CdcMetadataInjector cdcMetadataInjector) { - final ObjectNode base = (ObjectNode) (after.isNull() ? before : after); - - final long transactionMillis = source.get("ts_ms").asLong(); - final String transactionTimestamp = new Timestamp(transactionMillis).toInstant().toString(); - - base.put(CDC_UPDATED_AT, transactionTimestamp); - cdcMetadataInjector.addMetaData(base, source); - - if (after.isNull()) { - base.put(CDC_DELETED_AT, transactionTimestamp); - } else { - base.put(CDC_DELETED_AT, (String) null); - } - - return base; - } - -} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumPropertiesManager.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumPropertiesManager.java deleted file mode 100644 index 7018cbed78f4..000000000000 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumPropertiesManager.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.debezium.internals; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.SyncMode; -import java.util.Iterator; -import java.util.Optional; -import java.util.Properties; -import java.util.regex.Pattern; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; -import org.codehaus.plexus.util.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class DebeziumPropertiesManager { - - private static final Logger LOGGER = LoggerFactory.getLogger(DebeziumPropertiesManager.class); - private static final String BYTE_VALUE_256_MB = Integer.toString(256 * 1024 * 1024); - private final JsonNode config; - private final AirbyteFileOffsetBackingStore offsetManager; - private final Optional schemaHistoryManager; - - private final Properties properties; - private final ConfiguredAirbyteCatalog catalog; - - public DebeziumPropertiesManager(final Properties properties, - final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final AirbyteFileOffsetBackingStore offsetManager, - final Optional schemaHistoryManager) { - this.properties = properties; - this.config = config; - this.catalog = catalog; - this.offsetManager = offsetManager; - this.schemaHistoryManager = schemaHistoryManager; - } - - public Properties getDebeziumProperties() { - final Properties props = new Properties(); - props.putAll(properties); - - // debezium engine configuration - // https://debezium.io/documentation/reference/2.2/development/engine.html#engine-properties - props.setProperty("offset.storage", "org.apache.kafka.connect.storage.FileOffsetBackingStore"); - props.setProperty("offset.storage.file.filename", offsetManager.getOffsetFilePath().toString()); - props.setProperty("offset.flush.interval.ms", "1000"); // todo: make this longer - // default values from debezium CommonConnectorConfig - props.setProperty("max.batch.size", "2048"); - props.setProperty("max.queue.size", "8192"); - - // Disabling retries because debezium startup time might exceed our 60-second wait limit - // The maximum number of retries on connection errors before failing (-1 = no limit, 0 = disabled, > - // 0 = num of retries). - props.setProperty("errors.max.retries", "0"); - // This property must be strictly less than errors.retry.delay.max.ms - // (https://github.com/debezium/debezium/blob/bcc7d49519a4f07d123c616cfa45cd6268def0b9/debezium-core/src/main/java/io/debezium/util/DelayStrategy.java#L135) - props.setProperty("errors.retry.delay.initial.ms", "299"); - props.setProperty("errors.retry.delay.max.ms", "300"); - - if (schemaHistoryManager.isPresent()) { - // https://debezium.io/documentation/reference/2.2/operations/debezium-server.html#debezium-source-database-history-class - // https://debezium.io/documentation/reference/development/engine.html#_in_the_code - // As mentioned in the documents above, debezium connector for MySQL needs to track the schema - // changes. If we don't do this, we can't fetch records for the table. - props.setProperty("schema.history.internal", "io.debezium.storage.file.history.FileSchemaHistory"); - props.setProperty("schema.history.internal.file.filename", schemaHistoryManager.get().getPath().toString()); - } - - // https://debezium.io/documentation/reference/2.2/configuration/avro.html - props.setProperty("key.converter.schemas.enable", "false"); - props.setProperty("value.converter.schemas.enable", "false"); - - // debezium names - props.setProperty("name", config.get(JdbcUtils.DATABASE_KEY).asText()); - - // db connection configuration - props.setProperty("database.hostname", config.get(JdbcUtils.HOST_KEY).asText()); - props.setProperty("database.port", config.get(JdbcUtils.PORT_KEY).asText()); - props.setProperty("database.user", config.get(JdbcUtils.USERNAME_KEY).asText()); - props.setProperty("database.dbname", config.get(JdbcUtils.DATABASE_KEY).asText()); - - if (config.has(JdbcUtils.PASSWORD_KEY)) { - props.setProperty("database.password", config.get(JdbcUtils.PASSWORD_KEY).asText()); - } - - // By default "decimal.handing.mode=precise" which's caused returning this value as a binary. - // The "double" type may cause a loss of precision, so set Debezium's config to store it as a String - // explicitly in its Kafka messages for more details see: - // https://debezium.io/documentation/reference/2.2/connectors/postgresql.html#postgresql-decimal-types - // https://debezium.io/documentation/faq/#how_to_retrieve_decimal_field_from_binary_representation - props.setProperty("decimal.handling.mode", "string"); - - // https://debezium.io/documentation/reference/2.2/connectors/postgresql.html#postgresql-property-max-queue-size-in-bytes - props.setProperty("max.queue.size.in.bytes", BYTE_VALUE_256_MB); - - // WARNING : Never change the value of this otherwise all the connectors would start syncing from - // scratch - props.setProperty("topic.prefix", config.get(JdbcUtils.DATABASE_KEY).asText()); - - // table selection - props.setProperty("table.include.list", getTableIncludelist(catalog)); - // column selection - props.setProperty("column.include.list", getColumnIncludeList(catalog)); - return props; - } - - public static String getTableIncludelist(final ConfiguredAirbyteCatalog catalog) { - // Turn "stream": { - // "namespace": "schema1" - // "name": "table1 - // }, - // "stream": { - // "namespace": "schema2" - // "name": "table2 - // } -------> info "schema1.table1, schema2.table2" - - return catalog.getStreams().stream() - .filter(s -> s.getSyncMode() == SyncMode.INCREMENTAL) - .map(ConfiguredAirbyteStream::getStream) - .map(stream -> stream.getNamespace() + "." + stream.getName()) - // debezium needs commas escaped to split properly - .map(x -> StringUtils.escape(Pattern.quote(x), ",".toCharArray(), "\\,")) - .collect(Collectors.joining(",")); - } - - public static String getColumnIncludeList(final ConfiguredAirbyteCatalog catalog) { - // Turn "stream": { - // "namespace": "schema1" - // "name": "table1" - // "jsonSchema": { - // "properties": { - // "column1": { - // }, - // "column2": { - // } - // } - // } - // } -------> info "schema1.table1.(column1 | column2)" - - return catalog.getStreams().stream() - .filter(s -> s.getSyncMode() == SyncMode.INCREMENTAL) - .map(ConfiguredAirbyteStream::getStream) - .map(s -> { - final String fields = parseFields(s.getJsonSchema().get("properties").fieldNames()); - // schema.table.(col1|col2) - return Pattern.quote(s.getNamespace() + "." + s.getName()) + (StringUtils.isNotBlank(fields) ? "\\." + fields : ""); - }) - .map(x -> StringUtils.escape(x, ",".toCharArray(), "\\,")) - .collect(Collectors.joining(",")); - } - - private static String parseFields(final Iterator fieldNames) { - if (fieldNames == null || !fieldNames.hasNext()) { - return ""; - } - final Iterable iter = () -> fieldNames; - return StreamSupport.stream(iter.spliterator(), false) - .map(f -> Pattern.quote(f)) - .collect(Collectors.joining("|", "(", ")")); - } - -} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumRecordPublisher.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumRecordPublisher.java deleted file mode 100644 index 2b1167db9b03..000000000000 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/DebeziumRecordPublisher.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.debezium.internals; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.debezium.engine.ChangeEvent; -import io.debezium.engine.DebeziumEngine; -import io.debezium.engine.format.Json; -import io.debezium.engine.spi.OffsetCommitPolicy; -import java.util.Optional; -import java.util.Properties; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * The purpose of this class is to initialize and spawn the debezium engine with the right - * properties to fetch records - */ -public class DebeziumRecordPublisher implements AutoCloseable { - - private static final Logger LOGGER = LoggerFactory.getLogger(DebeziumRecordPublisher.class); - private final ExecutorService executor; - private DebeziumEngine> engine; - private final AtomicBoolean hasClosed; - private final AtomicBoolean isClosing; - private final AtomicReference thrownError; - private final CountDownLatch engineLatch; - private final DebeziumPropertiesManager debeziumPropertiesManager; - - public DebeziumRecordPublisher(final Properties properties, - final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final AirbyteFileOffsetBackingStore offsetManager, - final Optional schemaHistoryManager) { - this.debeziumPropertiesManager = new DebeziumPropertiesManager(properties, config, catalog, offsetManager, - schemaHistoryManager); - this.hasClosed = new AtomicBoolean(false); - this.isClosing = new AtomicBoolean(false); - this.thrownError = new AtomicReference<>(); - this.executor = Executors.newSingleThreadExecutor(); - this.engineLatch = new CountDownLatch(1); - } - - public void start(final BlockingQueue> queue) { - engine = DebeziumEngine.create(Json.class) - .using(debeziumPropertiesManager.getDebeziumProperties()) - .using(new OffsetCommitPolicy.AlwaysCommitOffsetPolicy()) - .notifying(e -> { - // debezium outputs a tombstone event that has a value of null. this is an artifact of how it - // interacts with kafka. we want to ignore it. - // more on the tombstone: - // https://debezium.io/documentation/reference/2.2/transformations/event-flattening.html - if (e.value() != null) { - try { - queue.put(e); - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - throw new RuntimeException(ex); - } - } - }) - .using((success, message, error) -> { - LOGGER.info("Debezium engine shutdown."); - LOGGER.info(message); - thrownError.set(error); - engineLatch.countDown(); - }) - .build(); - - // Run the engine asynchronously ... - executor.execute(engine); - } - - public boolean hasClosed() { - return hasClosed.get(); - } - - public void close() throws Exception { - if (isClosing.compareAndSet(false, true)) { - // consumers should assume records can be produced until engine has closed. - if (engine != null) { - engine.close(); - } - - // wait for closure before shutting down executor service - engineLatch.await(5, TimeUnit.MINUTES); - - // shut down and await for thread to actually go down - executor.shutdown(); - executor.awaitTermination(5, TimeUnit.MINUTES); - - // after the engine is completely off, we can mark this as closed - hasClosed.set(true); - - if (thrownError.get() != null) { - throw new RuntimeException(thrownError.get()); - } - } - } - -} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/FirstRecordWaitTimeUtil.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/FirstRecordWaitTimeUtil.java deleted file mode 100644 index 10640d45883a..000000000000 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/FirstRecordWaitTimeUtil.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.debezium.internals; - -import com.fasterxml.jackson.databind.JsonNode; -import java.time.Duration; -import java.util.Optional; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class FirstRecordWaitTimeUtil { - - private static final Logger LOGGER = LoggerFactory.getLogger(FirstRecordWaitTimeUtil.class); - - public static final Duration MIN_FIRST_RECORD_WAIT_TIME = Duration.ofMinutes(2); - public static final Duration MAX_FIRST_RECORD_WAIT_TIME = Duration.ofMinutes(20); - public static final Duration DEFAULT_FIRST_RECORD_WAIT_TIME = Duration.ofMinutes(5); - - public static void checkFirstRecordWaitTime(final JsonNode config) { - // we need to skip the check because in tests, we set initial_waiting_seconds - // to 5 seconds for performance reasons, which is shorter than the minimum - // value allowed in production - if (config.has("is_test") && config.get("is_test").asBoolean()) { - return; - } - - final Optional firstRecordWaitSeconds = getFirstRecordWaitSeconds(config); - if (firstRecordWaitSeconds.isPresent()) { - final int seconds = firstRecordWaitSeconds.get(); - if (seconds < MIN_FIRST_RECORD_WAIT_TIME.getSeconds() || seconds > MAX_FIRST_RECORD_WAIT_TIME.getSeconds()) { - throw new IllegalArgumentException( - String.format("initial_waiting_seconds must be between %d and %d seconds", - MIN_FIRST_RECORD_WAIT_TIME.getSeconds(), MAX_FIRST_RECORD_WAIT_TIME.getSeconds())); - } - } - } - - public static Duration getFirstRecordWaitTime(final JsonNode config) { - final boolean isTest = config.has("is_test") && config.get("is_test").asBoolean(); - Duration firstRecordWaitTime = DEFAULT_FIRST_RECORD_WAIT_TIME; - - final Optional firstRecordWaitSeconds = getFirstRecordWaitSeconds(config); - if (firstRecordWaitSeconds.isPresent()) { - firstRecordWaitTime = Duration.ofSeconds(firstRecordWaitSeconds.get()); - if (!isTest && firstRecordWaitTime.compareTo(MIN_FIRST_RECORD_WAIT_TIME) < 0) { - LOGGER.warn("First record waiting time is overridden to {} minutes, which is the min time allowed for safety.", - MIN_FIRST_RECORD_WAIT_TIME.toMinutes()); - firstRecordWaitTime = MIN_FIRST_RECORD_WAIT_TIME; - } else if (!isTest && firstRecordWaitTime.compareTo(MAX_FIRST_RECORD_WAIT_TIME) > 0) { - LOGGER.warn("First record waiting time is overridden to {} minutes, which is the max time allowed for safety.", - MAX_FIRST_RECORD_WAIT_TIME.toMinutes()); - firstRecordWaitTime = MAX_FIRST_RECORD_WAIT_TIME; - } - } - - LOGGER.info("First record waiting time: {} seconds", firstRecordWaitTime.getSeconds()); - return firstRecordWaitTime; - } - - public static Optional getFirstRecordWaitSeconds(final JsonNode config) { - final JsonNode replicationMethod = config.get("replication_method"); - if (replicationMethod != null && replicationMethod.has("initial_waiting_seconds")) { - final int seconds = config.get("replication_method").get("initial_waiting_seconds").asInt(); - return Optional.of(seconds); - } - return Optional.empty(); - } - -} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbCdcTargetPosition.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbCdcTargetPosition.java deleted file mode 100644 index 92f6623d49e6..000000000000 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbCdcTargetPosition.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.debezium.internals.mongodb; - -import com.google.common.annotations.VisibleForTesting; -import com.mongodb.client.MongoClient; -import io.airbyte.integrations.debezium.CdcTargetPosition; -import io.airbyte.integrations.debezium.internals.ChangeEventWithMetadata; -import io.airbyte.integrations.debezium.internals.SnapshotMetadata; -import io.debezium.connector.mongodb.ResumeTokens; -import java.util.Map; -import java.util.Objects; -import org.bson.BsonDocument; -import org.bson.BsonTimestamp; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Implementation of the {@link CdcTargetPosition} interface that provides methods for determining - * when a sync has reached the target position of the CDC log for MongoDB. In this case, the target - * position is a resume token value from the MongoDB oplog. This implementation compares the - * timestamp present in the Debezium change event against the timestamp of the resume token recorded - * at the start of a sync. When the event timestamp exceeds the resume token timestamp, the sync - * should stop to prevent it from running forever. - */ -public class MongoDbCdcTargetPosition implements CdcTargetPosition { - - private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbCdcTargetPosition.class); - - private final BsonTimestamp resumeTokenTimestamp; - - public MongoDbCdcTargetPosition(final BsonDocument resumeToken) { - this.resumeTokenTimestamp = ResumeTokens.getTimestamp(resumeToken); - } - - /** - * Constructs a new {@link MongoDbCdcTargetPosition} by fetching the most recent resume token from - * the MongoDB database. - * - * @param mongoClient A {@link MongoClient} used to retrieve the resume token. - * @return The {@link MongoDbCdcTargetPosition} set to the most recent resume token present in the - * database. - */ - public static MongoDbCdcTargetPosition targetPosition(final MongoClient mongoClient) { - final BsonDocument resumeToken = MongoDbResumeTokenHelper.getResumeToken(mongoClient); - return new MongoDbCdcTargetPosition(resumeToken); - } - - @VisibleForTesting - BsonTimestamp getResumeTokenTimestamp() { - return resumeTokenTimestamp; - } - - @Override - public boolean isHeartbeatSupported() { - return true; - } - - @Override - public boolean reachedTargetPosition(final ChangeEventWithMetadata changeEventWithMetadata) { - if (changeEventWithMetadata.isSnapshotEvent()) { - return false; - } else if (SnapshotMetadata.LAST == changeEventWithMetadata.snapshotMetadata()) { - LOGGER.info("Signalling close because Snapshot is complete"); - return true; - } else { - final BsonTimestamp eventResumeTokenTimestamp = - MongoDbResumeTokenHelper.extractTimestamp(changeEventWithMetadata.eventValueAsJson()); - boolean isEventResumeTokenAfter = resumeTokenTimestamp.compareTo(eventResumeTokenTimestamp) <= 0; - if (isEventResumeTokenAfter) { - LOGGER.info("Signalling close because record's event timestamp {} is after target event timestamp {}.", - eventResumeTokenTimestamp, resumeTokenTimestamp); - } - return isEventResumeTokenAfter; - } - } - - @Override - public boolean reachedTargetPosition(final BsonTimestamp positionFromHeartbeat) { - return positionFromHeartbeat != null && positionFromHeartbeat.compareTo(resumeTokenTimestamp) >= 0; - } - - @Override - public BsonTimestamp extractPositionFromHeartbeatOffset(final Map sourceOffset) { - return ResumeTokens.getTimestamp( - ResumeTokens.fromData( - sourceOffset.get(MongoDbDebeziumConstants.ChangeEvent.SOURCE_RESUME_TOKEN).toString())); - } - - @Override - public boolean equals(final Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - MongoDbCdcTargetPosition that = (MongoDbCdcTargetPosition) o; - return Objects.equals(resumeTokenTimestamp, that.resumeTokenTimestamp); - } - - @Override - public int hashCode() { - return Objects.hash(resumeTokenTimestamp); - } - -} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbDebeziumConstants.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbDebeziumConstants.java deleted file mode 100644 index 198a5f9eb781..000000000000 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbDebeziumConstants.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.debezium.internals.mongodb; - -import io.debezium.connector.mongodb.SourceInfo; - -/** - * A collection of constants for use with the Debezium MongoDB Connector. - */ -public class MongoDbDebeziumConstants { - - /** - * Constants for Debezium Source Event data. - */ - public static class ChangeEvent { - - public static final String SOURCE = "source"; - - public static final String SOURCE_ORDER = SourceInfo.ORDER; - - public static final String SOURCE_RESUME_TOKEN = "resume_token"; - - public static final String SOURCE_SECONDS = SourceInfo.TIMESTAMP; - - public static final String SOURCE_TIMESTAMP_MS = "ts_ms"; - - } - - /** - * Constants for Debezium Offset State storage. - */ - public static class OffsetState { - - public static final String KEY_REPLICA_SET = SourceInfo.REPLICA_SET_NAME; - - public static final String KEY_SERVER_ID = SourceInfo.SERVER_ID_KEY; - - public static final String VALUE_INCREMENT = SourceInfo.ORDER; - - public static final String VALUE_RESUME_TOKEN = "resume_token"; - - public static final String VALUE_SECONDS = SourceInfo.TIMESTAMP; - - public static final String VALUE_TRANSACTION_ID = "transaction_id"; - - } - -} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtil.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtil.java deleted file mode 100644 index 5b8c6759dbca..000000000000 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtil.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.debezium.internals.mongodb; - -import com.fasterxml.jackson.databind.JsonNode; -import com.mongodb.client.MongoClient; -import io.airbyte.commons.json.Jsons; -import io.debezium.connector.mongodb.ResumeTokens; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.bson.BsonDocument; -import org.bson.BsonString; -import org.bson.BsonTimestamp; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Collection of utility methods related to the Debezium offset state. - */ -public class MongoDbDebeziumStateUtil { - - private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbDebeziumStateUtil.class); - - /** - * Constructs the initial Debezium offset state that will be used by the incremental CDC snapshot - * after an initial snapshot sync. - * - * @param mongoClient The {@link MongoClient} used to query the MongoDB server. - * @param database The database associated with the sync. - * @param replicaSet The replication set associated with the sync. - * @return The initial Debezium offset state storage document as a {@link JsonNode}. - */ - public JsonNode constructInitialDebeziumState(final MongoClient mongoClient, final String database, final String replicaSet) { - final BsonDocument resumeToken = MongoDbResumeTokenHelper.getResumeToken(mongoClient); - final String resumeTokenData = ((BsonString) ResumeTokens.getData(resumeToken)).getValue(); - final BsonTimestamp timestamp = ResumeTokens.getTimestamp(resumeToken); - - final List> key = List.of( - Map.of(MongoDbDebeziumConstants.OffsetState.KEY_REPLICA_SET, replicaSet, - MongoDbDebeziumConstants.OffsetState.KEY_SERVER_ID, database)); - - final Map value = new HashMap<>(); - value.put(MongoDbDebeziumConstants.OffsetState.VALUE_SECONDS, timestamp.getTime()); - value.put(MongoDbDebeziumConstants.OffsetState.VALUE_INCREMENT, timestamp.getInc()); - value.put(MongoDbDebeziumConstants.OffsetState.VALUE_TRANSACTION_ID, null); - value.put(MongoDbDebeziumConstants.OffsetState.VALUE_RESUME_TOKEN, resumeTokenData); - - final JsonNode state = Jsons.jsonNode(Map.of(key, value)); - LOGGER.info("Initial Debezium state constructed: {}", state); - return state; - } - -} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mssql/MSSQLConverter.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mssql/MSSQLConverter.java deleted file mode 100644 index 91b58cc306c2..000000000000 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mssql/MSSQLConverter.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.debezium.internals.mssql; - -import com.microsoft.sqlserver.jdbc.Geography; -import com.microsoft.sqlserver.jdbc.Geometry; -import com.microsoft.sqlserver.jdbc.SQLServerException; -import io.airbyte.db.DataTypeUtils; -import io.airbyte.db.jdbc.DateTimeConverter; -import io.airbyte.integrations.debezium.internals.DebeziumConverterUtils; -import io.debezium.spi.converter.CustomConverter; -import io.debezium.spi.converter.RelationalColumn; -import java.math.BigDecimal; -import java.nio.charset.Charset; -import java.sql.Timestamp; -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; -import java.util.Objects; -import java.util.Properties; -import java.util.Set; -import microsoft.sql.DateTimeOffset; -import org.apache.kafka.connect.data.SchemaBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class MSSQLConverter implements CustomConverter { - - private final Logger LOGGER = LoggerFactory.getLogger(MSSQLConverter.class); - - private final Set DATE_TYPES = Set.of("DATE", "DATETIME", "DATETIME2", "SMALLDATETIME"); - private final Set BINARY = Set.of("VARBINARY", "BINARY"); - private static final String DATETIMEOFFSET = "DATETIMEOFFSET"; - private static final String TIME_TYPE = "TIME"; - private static final String SMALLMONEY_TYPE = "SMALLMONEY"; - private static final String GEOMETRY = "GEOMETRY"; - private static final String GEOGRAPHY = "GEOGRAPHY"; - private static final String DEBEZIUM_DATETIMEOFFSET_FORMAT = "yyyy-MM-dd HH:mm:ss XXX"; - - @Override - public void configure(Properties props) {} - - @Override - public void converterFor(final RelationalColumn field, - final ConverterRegistration registration) { - if (DATE_TYPES.contains(field.typeName().toUpperCase())) { - registerDate(field, registration); - } else if (SMALLMONEY_TYPE.equalsIgnoreCase(field.typeName())) { - registerMoney(field, registration); - } else if (BINARY.contains(field.typeName().toUpperCase())) { - registerBinary(field, registration); - } else if (GEOMETRY.equalsIgnoreCase(field.typeName())) { - registerGeometry(field, registration); - } else if (GEOGRAPHY.equalsIgnoreCase(field.typeName())) { - registerGeography(field, registration); - } else if (TIME_TYPE.equalsIgnoreCase(field.typeName())) { - registerTime(field, registration); - } else if (DATETIMEOFFSET.equalsIgnoreCase(field.typeName())) { - registerDateTimeOffSet(field, registration); - } - } - - private void registerGeometry(final RelationalColumn field, - final ConverterRegistration registration) { - registration.register(SchemaBuilder.string(), input -> { - if (Objects.isNull(input)) { - return DebeziumConverterUtils.convertDefaultValue(field); - } - - if (input instanceof byte[]) { - try { - return Geometry.deserialize((byte[]) input).toString(); - } catch (SQLServerException e) { - LOGGER.error(e.getMessage()); - } - } - - LOGGER.warn("Uncovered Geometry class type '{}'. Use default converter", - input.getClass().getName()); - return input.toString(); - }); - } - - private void registerGeography(final RelationalColumn field, - final ConverterRegistration registration) { - registration.register(SchemaBuilder.string(), input -> { - if (Objects.isNull(input)) { - return DebeziumConverterUtils.convertDefaultValue(field); - } - - if (input instanceof byte[]) { - try { - return Geography.deserialize((byte[]) input).toString(); - } catch (SQLServerException e) { - LOGGER.error(e.getMessage()); - } - } - - LOGGER.warn("Uncovered Geography class type '{}'. Use default converter", - input.getClass().getName()); - return input.toString(); - }); - } - - private void registerDate(final RelationalColumn field, - final ConverterRegistration registration) { - registration.register(SchemaBuilder.string(), input -> { - if (Objects.isNull(input)) { - return DebeziumConverterUtils.convertDefaultValue(field); - } - if (field.typeName().equalsIgnoreCase("DATE")) { - return DateTimeConverter.convertToDate(input); - } - return DateTimeConverter.convertToTimestamp(input); - }); - } - - private void registerDateTimeOffSet(final RelationalColumn field, - final ConverterRegistration registration) { - registration.register(SchemaBuilder.string(), input -> { - if (Objects.isNull(input)) { - return DebeziumConverterUtils.convertDefaultValue(field); - } - - if (input instanceof DateTimeOffset) { - return DataTypeUtils.toISO8601String( - OffsetDateTime.parse(input.toString(), - DateTimeFormatter.ofPattern(DEBEZIUM_DATETIMEOFFSET_FORMAT))); - } - - LOGGER.warn("Uncovered DateTimeOffSet class type '{}'. Use default converter", - input.getClass().getName()); - return input.toString(); - }); - } - - private void registerTime(final RelationalColumn field, - final ConverterRegistration registration) { - registration.register(SchemaBuilder.string(), input -> { - if (Objects.isNull(input)) { - return DebeziumConverterUtils.convertDefaultValue(field); - } - - if (input instanceof Timestamp) { - return DataTypeUtils.toISOTimeString(((Timestamp) input).toLocalDateTime()); - } - - LOGGER.warn("Uncovered time class type '{}'. Use default converter", - input.getClass().getName()); - return input.toString(); - }); - } - - private void registerMoney(final RelationalColumn field, - final ConverterRegistration registration) { - registration.register(SchemaBuilder.float64(), input -> { - if (Objects.isNull(input)) { - return DebeziumConverterUtils.convertDefaultValue(field); - } - - if (input instanceof BigDecimal) { - return ((BigDecimal) input).doubleValue(); - } - - LOGGER.warn("Uncovered money class type '{}'. Use default converter", - input.getClass().getName()); - return input.toString(); - }); - } - - private void registerBinary(final RelationalColumn field, - final ConverterRegistration registration) { - registration.register(SchemaBuilder.string(), input -> { - if (Objects.isNull(input)) { - return DebeziumConverterUtils.convertDefaultValue(field); - } - - if (input instanceof byte[]) { - return new String((byte[]) input, Charset.defaultCharset()); - } - - LOGGER.warn("Uncovered binary class type '{}'. Use default converter", - input.getClass().getName()); - return input.toString(); - }); - } - -} diff --git a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mssql/MssqlCdcTargetPosition.java b/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mssql/MssqlCdcTargetPosition.java deleted file mode 100644 index 031224e32dd8..000000000000 --- a/airbyte-integrations/bases/debezium/src/main/java/io/airbyte/integrations/debezium/internals/mssql/MssqlCdcTargetPosition.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.debezium.internals.mssql; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.base.Preconditions; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.debezium.CdcTargetPosition; -import io.airbyte.integrations.debezium.internals.ChangeEventWithMetadata; -import io.airbyte.integrations.debezium.internals.SnapshotMetadata; -import io.debezium.connector.sqlserver.Lsn; -import java.io.IOException; -import java.sql.SQLException; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class MssqlCdcTargetPosition implements CdcTargetPosition { - - private static final Logger LOGGER = LoggerFactory.getLogger(MssqlCdcTargetPosition.class); - public final Lsn targetLsn; - - public MssqlCdcTargetPosition(final Lsn targetLsn) { - this.targetLsn = targetLsn; - } - - @Override - public boolean reachedTargetPosition(final ChangeEventWithMetadata changeEventWithMetadata) { - if (changeEventWithMetadata.isSnapshotEvent()) { - return false; - } else if (SnapshotMetadata.LAST == changeEventWithMetadata.snapshotMetadata()) { - LOGGER.info("Signalling close because Snapshot is complete"); - return true; - } else { - final Lsn recordLsn = extractLsn(changeEventWithMetadata.eventValueAsJson()); - final boolean isEventLSNAfter = targetLsn.compareTo(recordLsn) <= 0; - if (isEventLSNAfter) { - LOGGER.info("Signalling close because record's LSN : " + recordLsn + " is after target LSN : " + targetLsn); - } - return isEventLSNAfter; - } - } - - @Override - public Lsn extractPositionFromHeartbeatOffset(final Map sourceOffset) { - throw new RuntimeException("Heartbeat is not supported for MSSQL"); - } - - private Lsn extractLsn(final JsonNode valueAsJson) { - return Optional.ofNullable(valueAsJson.get("source")) - .flatMap(source -> Optional.ofNullable(source.get("commit_lsn").asText())) - .map(Lsn::valueOf) - .orElseThrow(() -> new IllegalStateException("Could not find LSN")); - } - - @Override - public boolean equals(final Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - final MssqlCdcTargetPosition that = (MssqlCdcTargetPosition) o; - return targetLsn.equals(that.targetLsn); - } - - @Override - public int hashCode() { - return targetLsn.hashCode(); - } - - public static MssqlCdcTargetPosition getTargetPosition(final JdbcDatabase database, final String dbName) { - try { - final List jsonNodes = database - .bufferedResultSetQuery(connection -> connection.createStatement().executeQuery( - "USE [" + dbName + "]; SELECT sys.fn_cdc_get_max_lsn() AS max_lsn;"), JdbcUtils.getDefaultSourceOperations()::rowToJson); - Preconditions.checkState(jsonNodes.size() == 1); - if (jsonNodes.get(0).get("max_lsn") != null) { - final Lsn maxLsn = Lsn.valueOf(jsonNodes.get(0).get("max_lsn").binaryValue()); - LOGGER.info("identified target lsn: " + maxLsn); - return new MssqlCdcTargetPosition(maxLsn); - } else { - throw new RuntimeException("SQL returned max LSN as null, this might be because the SQL Server Agent is not running. " + - "Please enable the Agent and try again (https://docs.microsoft.com/en-us/sql/ssms/agent/start-stop-or-pause-the-sql-server-agent-service?view=sql-server-ver15)"); - } - } catch (final SQLException | IOException e) { - throw new RuntimeException(e); - } - } - -} diff --git a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/DebeziumEventUtilsTest.java b/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/DebeziumEventUtilsTest.java deleted file mode 100644 index 4f40d6a106ac..000000000000 --- a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/DebeziumEventUtilsTest.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.debezium; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.debezium.internals.ChangeEventWithMetadata; -import io.airbyte.integrations.debezium.internals.DebeziumEventUtils; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import io.debezium.engine.ChangeEvent; -import java.io.IOException; -import java.time.Instant; -import org.junit.jupiter.api.Test; - -class DebeziumEventUtilsTest { - - @Test - public void testConvertChangeEvent() throws IOException { - final String stream = "names"; - final Instant emittedAt = Instant.now(); - final CdcMetadataInjector cdcMetadataInjector = new DummyMetadataInjector(); - final ChangeEventWithMetadata insertChangeEvent = mockChangeEvent("insert_change_event.json"); - final ChangeEventWithMetadata updateChangeEvent = mockChangeEvent("update_change_event.json"); - final ChangeEventWithMetadata deleteChangeEvent = mockChangeEvent("delete_change_event.json"); - - final AirbyteMessage actualInsert = DebeziumEventUtils.toAirbyteMessage(insertChangeEvent, cdcMetadataInjector, emittedAt); - final AirbyteMessage actualUpdate = DebeziumEventUtils.toAirbyteMessage(updateChangeEvent, cdcMetadataInjector, emittedAt); - final AirbyteMessage actualDelete = DebeziumEventUtils.toAirbyteMessage(deleteChangeEvent, cdcMetadataInjector, emittedAt); - - final AirbyteMessage expectedInsert = createAirbyteMessage(stream, emittedAt, "insert_message.json"); - final AirbyteMessage expectedUpdate = createAirbyteMessage(stream, emittedAt, "update_message.json"); - final AirbyteMessage expectedDelete = createAirbyteMessage(stream, emittedAt, "delete_message.json"); - - deepCompare(expectedInsert, actualInsert); - deepCompare(expectedUpdate, actualUpdate); - deepCompare(expectedDelete, actualDelete); - } - - private static ChangeEventWithMetadata mockChangeEvent(final String resourceName) throws IOException { - final ChangeEvent mocked = mock(ChangeEvent.class); - final String resource = MoreResources.readResource(resourceName); - when(mocked.value()).thenReturn(resource); - - return new ChangeEventWithMetadata(mocked); - } - - private static AirbyteMessage createAirbyteMessage(final String stream, final Instant emittedAt, final String resourceName) throws IOException { - final String data = MoreResources.readResource(resourceName); - - final AirbyteRecordMessage recordMessage = new AirbyteRecordMessage() - .withStream(stream) - .withNamespace("public") - .withData(Jsons.deserialize(data)) - .withEmittedAt(emittedAt.toEpochMilli()); - - return new AirbyteMessage() - .withType(AirbyteMessage.Type.RECORD) - .withRecord(recordMessage); - } - - private static void deepCompare(final Object expected, final Object actual) { - assertEquals(Jsons.deserialize(Jsons.serialize(expected)), Jsons.deserialize(Jsons.serialize(actual))); - } - - public static class DummyMetadataInjector implements CdcMetadataInjector { - - @Override - public void addMetaData(final ObjectNode event, final JsonNode source) { - final long lsn = source.get("lsn").asLong(); - event.put("_ab_cdc_lsn", lsn); - } - - @Override - public String namespace(final JsonNode source) { - return source.get("schema").asText(); - } - - } - -} diff --git a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/FirstRecordWaitTimeUtilTest.java b/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/FirstRecordWaitTimeUtilTest.java deleted file mode 100644 index e3cfd7a03a86..000000000000 --- a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/FirstRecordWaitTimeUtilTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.debezium.internals; - -import static io.airbyte.integrations.debezium.internals.FirstRecordWaitTimeUtil.MAX_FIRST_RECORD_WAIT_TIME; -import static io.airbyte.integrations.debezium.internals.FirstRecordWaitTimeUtil.MIN_FIRST_RECORD_WAIT_TIME; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.Jsons; -import java.time.Duration; -import java.util.Collections; -import java.util.Map; -import java.util.Optional; -import org.junit.jupiter.api.Test; - -public class FirstRecordWaitTimeUtilTest { - - @Test - void testGetFirstRecordWaitTime() { - final JsonNode emptyConfig = Jsons.jsonNode(Collections.emptyMap()); - assertDoesNotThrow(() -> FirstRecordWaitTimeUtil.checkFirstRecordWaitTime(emptyConfig)); - assertEquals(Optional.empty(), FirstRecordWaitTimeUtil.getFirstRecordWaitSeconds(emptyConfig)); - assertEquals(FirstRecordWaitTimeUtil.DEFAULT_FIRST_RECORD_WAIT_TIME, FirstRecordWaitTimeUtil.getFirstRecordWaitTime(emptyConfig)); - - final JsonNode normalConfig = Jsons.jsonNode(Map.of("replication_method", - Map.of("method", "CDC", "initial_waiting_seconds", 500))); - assertDoesNotThrow(() -> FirstRecordWaitTimeUtil.checkFirstRecordWaitTime(normalConfig)); - assertEquals(Optional.of(500), FirstRecordWaitTimeUtil.getFirstRecordWaitSeconds(normalConfig)); - assertEquals(Duration.ofSeconds(500), FirstRecordWaitTimeUtil.getFirstRecordWaitTime(normalConfig)); - - final int tooShortTimeout = (int) MIN_FIRST_RECORD_WAIT_TIME.getSeconds() - 1; - final JsonNode tooShortConfig = Jsons.jsonNode(Map.of("replication_method", - Map.of("method", "CDC", "initial_waiting_seconds", tooShortTimeout))); - assertThrows(IllegalArgumentException.class, () -> FirstRecordWaitTimeUtil.checkFirstRecordWaitTime(tooShortConfig)); - assertEquals(Optional.of(tooShortTimeout), FirstRecordWaitTimeUtil.getFirstRecordWaitSeconds(tooShortConfig)); - assertEquals(MIN_FIRST_RECORD_WAIT_TIME, FirstRecordWaitTimeUtil.getFirstRecordWaitTime(tooShortConfig)); - - final int tooLongTimeout = (int) MAX_FIRST_RECORD_WAIT_TIME.getSeconds() + 1; - final JsonNode tooLongConfig = Jsons.jsonNode(Map.of("replication_method", - Map.of("method", "CDC", "initial_waiting_seconds", tooLongTimeout))); - assertThrows(IllegalArgumentException.class, () -> FirstRecordWaitTimeUtil.checkFirstRecordWaitTime(tooLongConfig)); - assertEquals(Optional.of(tooLongTimeout), FirstRecordWaitTimeUtil.getFirstRecordWaitSeconds(tooLongConfig)); - assertEquals(MAX_FIRST_RECORD_WAIT_TIME, FirstRecordWaitTimeUtil.getFirstRecordWaitTime(tooLongConfig)); - } - -} diff --git a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbCdcTargetPositionTest.java b/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbCdcTargetPositionTest.java deleted file mode 100644 index b6a3346bde7a..000000000000 --- a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbCdcTargetPositionTest.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.debezium.internals.mongodb; - -import static com.mongodb.assertions.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.mongodb.client.ChangeStreamIterable; -import com.mongodb.client.MongoChangeStreamCursor; -import com.mongodb.client.MongoClient; -import com.mongodb.client.model.changestream.ChangeStreamDocument; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.debezium.internals.ChangeEventWithMetadata; -import io.debezium.connector.mongodb.ResumeTokens; -import io.debezium.engine.ChangeEvent; -import java.io.IOException; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import org.bson.BsonDocument; -import org.bson.BsonTimestamp; -import org.junit.jupiter.api.Test; - -class MongoDbCdcTargetPositionTest { - - private static final String RESUME_TOKEN = "8264BEB9F3000000012B0229296E04"; - - @Test - void testCreateTargetPosition() { - final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); - final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); - final MongoChangeStreamCursor> mongoChangeStreamCursor = - mock(MongoChangeStreamCursor.class); - final MongoClient mongoClient = mock(MongoClient.class); - - when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); - when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); - - final MongoDbCdcTargetPosition targetPosition = MongoDbCdcTargetPosition.targetPosition(mongoClient); - assertNotNull(targetPosition); - assertEquals(ResumeTokens.getTimestamp(resumeTokenDocument), targetPosition.getResumeTokenTimestamp()); - } - - @Test - void testReachedTargetPosition() throws IOException { - final String changeEventJson = MoreResources.readResource("mongodb/change_event.json"); - final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); - final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); - final MongoChangeStreamCursor> mongoChangeStreamCursor = - mock(MongoChangeStreamCursor.class); - final MongoClient mongoClient = mock(MongoClient.class); - final ChangeEvent changeEvent = mock(ChangeEvent.class); - - when(changeEvent.value()).thenReturn(changeEventJson); - when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); - when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); - - final ChangeEventWithMetadata changeEventWithMetadata = new ChangeEventWithMetadata(changeEvent); - final MongoDbCdcTargetPosition targetPosition = MongoDbCdcTargetPosition.targetPosition(mongoClient); - assertTrue(targetPosition.reachedTargetPosition(changeEventWithMetadata)); - - when(changeEvent.value()).thenReturn(changeEventJson.replaceAll("\"ts_ms\"\\: \\d+,", "\"ts_ms\": 1590221043000,")); - final ChangeEventWithMetadata changeEventWithMetadata2 = new ChangeEventWithMetadata(changeEvent); - assertFalse(targetPosition.reachedTargetPosition(changeEventWithMetadata2)); - } - - @Test - void testReachedTargetPositionSnapshotEvent() throws IOException { - final String changeEventJson = MoreResources.readResource("mongodb/change_event_snapshot.json"); - final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); - final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); - final MongoChangeStreamCursor> mongoChangeStreamCursor = - mock(MongoChangeStreamCursor.class); - final MongoClient mongoClient = mock(MongoClient.class); - final ChangeEvent changeEvent = mock(ChangeEvent.class); - - when(changeEvent.value()).thenReturn(changeEventJson); - when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); - when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); - - final ChangeEventWithMetadata changeEventWithMetadata = new ChangeEventWithMetadata(changeEvent); - final MongoDbCdcTargetPosition targetPosition = MongoDbCdcTargetPosition.targetPosition(mongoClient); - assertFalse(targetPosition.reachedTargetPosition(changeEventWithMetadata)); - } - - @Test - void testReachedTargetPositionSnapshotLastEvent() throws IOException { - final String changeEventJson = MoreResources.readResource("mongodb/change_event_snapshot_last.json"); - final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); - final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); - final MongoChangeStreamCursor> mongoChangeStreamCursor = - mock(MongoChangeStreamCursor.class); - final MongoClient mongoClient = mock(MongoClient.class); - final ChangeEvent changeEvent = mock(ChangeEvent.class); - - when(changeEvent.value()).thenReturn(changeEventJson); - when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); - when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); - - final ChangeEventWithMetadata changeEventWithMetadata = new ChangeEventWithMetadata(changeEvent); - final MongoDbCdcTargetPosition targetPosition = MongoDbCdcTargetPosition.targetPosition(mongoClient); - assertTrue(targetPosition.reachedTargetPosition(changeEventWithMetadata)); - } - - @Test - void testReachedTargetPositionFromHeartbeat() { - final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); - final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); - final MongoChangeStreamCursor> mongoChangeStreamCursor = - mock(MongoChangeStreamCursor.class); - final MongoClient mongoClient = mock(MongoClient.class); - - when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); - when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); - - final MongoDbCdcTargetPosition targetPosition = MongoDbCdcTargetPosition.targetPosition(mongoClient); - final BsonTimestamp heartbeatTimestamp = new BsonTimestamp( - Long.valueOf(ResumeTokens.getTimestamp(resumeTokenDocument).getTime() + TimeUnit.HOURS.toSeconds(1)).intValue(), - 0); - - assertTrue(targetPosition.reachedTargetPosition(heartbeatTimestamp)); - assertFalse(targetPosition.reachedTargetPosition((BsonTimestamp) null)); - } - - @Test - void testIsHeartbeatSupported() { - final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); - final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); - final MongoChangeStreamCursor> mongoChangeStreamCursor = - mock(MongoChangeStreamCursor.class); - final MongoClient mongoClient = mock(MongoClient.class); - - when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); - when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); - - final MongoDbCdcTargetPosition targetPosition = MongoDbCdcTargetPosition.targetPosition(mongoClient); - - assertTrue(targetPosition.isHeartbeatSupported()); - } - - @Test - void testExtractPositionFromHeartbeatOffset() { - final BsonDocument resumeTokenDocument = ResumeTokens.fromData(RESUME_TOKEN); - final BsonTimestamp resumeTokenTimestamp = ResumeTokens.getTimestamp(resumeTokenDocument); - final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); - final MongoChangeStreamCursor> mongoChangeStreamCursor = - mock(MongoChangeStreamCursor.class); - final MongoClient mongoClient = mock(MongoClient.class); - - when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); - when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); - - final MongoDbCdcTargetPosition targetPosition = MongoDbCdcTargetPosition.targetPosition(mongoClient); - - final Map sourceOffset = Map.of(MongoDbDebeziumConstants.ChangeEvent.SOURCE_SECONDS, resumeTokenTimestamp.getTime(), - MongoDbDebeziumConstants.ChangeEvent.SOURCE_ORDER, resumeTokenTimestamp.getInc(), - MongoDbDebeziumConstants.ChangeEvent.SOURCE_RESUME_TOKEN, RESUME_TOKEN); - - final BsonTimestamp timestamp = targetPosition.extractPositionFromHeartbeatOffset(sourceOffset); - assertEquals(resumeTokenTimestamp, timestamp); - } - -} diff --git a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtilTest.java b/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtilTest.java deleted file mode 100644 index 642a6e0ad601..000000000000 --- a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbDebeziumStateUtilTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.debezium.internals.mongodb; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.fasterxml.jackson.databind.JsonNode; -import com.mongodb.client.ChangeStreamIterable; -import com.mongodb.client.MongoChangeStreamCursor; -import com.mongodb.client.MongoClient; -import com.mongodb.client.model.changestream.ChangeStreamDocument; -import io.debezium.connector.mongodb.ResumeTokens; -import org.bson.BsonDocument; -import org.bson.BsonTimestamp; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class MongoDbDebeziumStateUtilTest { - - private MongoDbDebeziumStateUtil mongoDbDebeziumStateUtil; - - @BeforeEach - void setup() { - mongoDbDebeziumStateUtil = new MongoDbDebeziumStateUtil(); - } - - @Test - void testConstructInitialDebeziumState() { - final String database = "test"; - final String replicaSet = "test_rs"; - final String resumeToken = "8264BEB9F3000000012B0229296E04"; - final BsonDocument resumeTokenDocument = ResumeTokens.fromData(resumeToken); - final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); - final MongoChangeStreamCursor> mongoChangeStreamCursor = - mock(MongoChangeStreamCursor.class); - final MongoClient mongoClient = mock(MongoClient.class); - - when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); - when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); - - final JsonNode initialState = mongoDbDebeziumStateUtil.constructInitialDebeziumState(mongoClient, - database, replicaSet); - - assertNotNull(initialState); - assertEquals(1, initialState.size()); - final BsonTimestamp timestamp = ResumeTokens.getTimestamp(resumeTokenDocument); - final JsonNode offsetState = initialState.fields().next().getValue(); - assertEquals(resumeToken, offsetState.get(MongoDbDebeziumConstants.OffsetState.VALUE_RESUME_TOKEN).asText()); - assertEquals(timestamp.getTime(), offsetState.get(MongoDbDebeziumConstants.OffsetState.VALUE_SECONDS).asInt()); - assertEquals(timestamp.getInc(), offsetState.get(MongoDbDebeziumConstants.OffsetState.VALUE_INCREMENT).asInt()); - assertEquals("null", offsetState.get(MongoDbDebeziumConstants.OffsetState.VALUE_TRANSACTION_ID).asText()); - } - -} diff --git a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelperTest.java b/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelperTest.java deleted file mode 100644 index d2a3e10fd455..000000000000 --- a/airbyte-integrations/bases/debezium/src/test/java/io/airbyte/integrations/debezium/internals/mongodb/MongoDbResumeTokenHelperTest.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.debezium.internals.mongodb; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.fasterxml.jackson.databind.JsonNode; -import com.mongodb.client.ChangeStreamIterable; -import com.mongodb.client.MongoChangeStreamCursor; -import com.mongodb.client.MongoClient; -import com.mongodb.client.model.changestream.ChangeStreamDocument; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.debezium.connector.mongodb.ResumeTokens; -import java.io.IOException; -import java.util.concurrent.TimeUnit; -import org.bson.BsonDocument; -import org.bson.BsonTimestamp; -import org.junit.jupiter.api.Test; - -class MongoDbResumeTokenHelperTest { - - @Test - void testRetrievingResumeToken() { - final String resumeToken = "8264BEB9F3000000012B0229296E04"; - final BsonDocument resumeTokenDocument = ResumeTokens.fromData(resumeToken); - final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); - final MongoChangeStreamCursor> mongoChangeStreamCursor = - mock(MongoChangeStreamCursor.class); - final MongoClient mongoClient = mock(MongoClient.class); - - when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); - when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); - when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); - - final BsonDocument actualResumeToken = MongoDbResumeTokenHelper.getResumeToken(mongoClient); - assertEquals(resumeTokenDocument, actualResumeToken); - } - - @Test - void testTimestampExtraction() throws IOException { - final int timestampSec = Long.valueOf(TimeUnit.MILLISECONDS.toSeconds(1692651270000L)).intValue(); - final BsonTimestamp expectedTimestamp = new BsonTimestamp(timestampSec, 2); - final String changeEventJson = MoreResources.readResource("mongodb/change_event.json"); - final JsonNode changeEvent = Jsons.deserialize(changeEventJson); - - final BsonTimestamp timestamp = MongoDbResumeTokenHelper.extractTimestamp(changeEvent); - assertNotNull(timestamp); - assertEquals(expectedTimestamp, timestamp); - } - - @Test - void testTimestampExtractionSourceNotPresent() { - final JsonNode changeEvent = Jsons.deserialize("{}"); - assertThrows(IllegalStateException.class, () -> MongoDbResumeTokenHelper.extractTimestamp(changeEvent)); - } - - @Test - void testTimestampExtractionTimestampNotPresent() { - final JsonNode changeEvent = Jsons.deserialize("{\"source\":{}}"); - assertThrows(IllegalStateException.class, () -> MongoDbResumeTokenHelper.extractTimestamp(changeEvent)); - } - -} diff --git a/airbyte-integrations/bases/s3-destination-base-integration-test/build.gradle b/airbyte-integrations/bases/s3-destination-base-integration-test/build.gradle deleted file mode 100644 index 7e36afd5f292..000000000000 --- a/airbyte-integrations/bases/s3-destination-base-integration-test/build.gradle +++ /dev/null @@ -1,38 +0,0 @@ -plugins { - id 'java-library' -} - -dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:bases:base-java-s3') - - implementation(enforcedPlatform('org.junit:junit-bom:5.8.2')) - implementation 'org.junit.jupiter:junit-jupiter-api' - implementation 'org.junit.jupiter:junit-jupiter-params' - - // csv - implementation 'com.amazonaws:aws-java-sdk-s3:1.11.978' - implementation 'org.apache.commons:commons-csv:1.4' - implementation 'com.github.alexmojaki:s3-stream-upload:2.2.2' - - implementation 'org.apache.commons:commons-lang3:3.11' - implementation 'org.xerial.snappy:snappy-java:1.1.8.4' - implementation "org.mockito:mockito-inline:4.1.0" - - // parquet - implementation ('org.apache.hadoop:hadoop-common:3.3.3') { - exclude group: 'org.slf4j', module: 'slf4j-log4j12' - exclude group: 'org.slf4j', module: 'slf4j-reload4j' - } - implementation ('org.apache.hadoop:hadoop-aws:3.3.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} - implementation ('org.apache.hadoop:hadoop-mapreduce-client-core:3.3.3') { - exclude group: 'org.slf4j', module: 'slf4j-log4j12' - exclude group: 'org.slf4j', module: 'slf4j-reload4j' - } - implementation ('org.apache.parquet:parquet-avro:1.12.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} - implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} - - implementation project(':airbyte-integrations:bases:standard-destination-test') -} diff --git a/airbyte-integrations/bases/standard-destination-test/build.gradle b/airbyte-integrations/bases/standard-destination-test/build.gradle deleted file mode 100644 index 26823d934473..000000000000 --- a/airbyte-integrations/bases/standard-destination-test/build.gradle +++ /dev/null @@ -1,18 +0,0 @@ -plugins { - id 'java-library' -} -dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-connector-test-harnesses:acceptance-test-harness') - implementation project(':airbyte-config-oss:config-models-oss') - implementation project(':airbyte-config-oss:init-oss') - implementation project(':airbyte-json-validation') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - - implementation(enforcedPlatform('org.junit:junit-bom:5.8.2')) - implementation 'org.junit.jupiter:junit-jupiter-api' - implementation 'org.junit.jupiter:junit-jupiter-params' - implementation 'org.mockito:mockito-core:4.6.1' - -} diff --git a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/JdbcDestinationAcceptanceTest.java b/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/JdbcDestinationAcceptanceTest.java deleted file mode 100644 index ff6a0e09b10f..000000000000 --- a/airbyte-integrations/bases/standard-destination-test/src/main/java/io/airbyte/integrations/standardtest/destination/JdbcDestinationAcceptanceTest.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.standardtest.destination; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import java.util.Arrays; -import org.jooq.Record; - -public abstract class JdbcDestinationAcceptanceTest extends DestinationAcceptanceTest { - - protected final ObjectMapper mapper = new ObjectMapper(); - - protected JsonNode getJsonFromRecord(Record record) { - ObjectNode node = mapper.createObjectNode(); - - Arrays.stream(record.fields()).forEach(field -> { - var value = record.get(field); - - switch (field.getDataType().getTypeName()) { - case "varchar", "nvarchar", "jsonb", "json", "other": - var stringValue = (value != null ? value.toString() : null); - DestinationAcceptanceTestUtils.putStringIntoJson(stringValue, field.getName(), node); - break; - default: - node.put(field.getName(), (value != null ? value.toString() : null)); - } - }); - return node; - } - -} diff --git a/airbyte-integrations/bases/standard-source-test/.dockerignore b/airbyte-integrations/bases/standard-source-test/.dockerignore deleted file mode 100644 index 6145f27e93a0..000000000000 --- a/airbyte-integrations/bases/standard-source-test/.dockerignore +++ /dev/null @@ -1,4 +0,0 @@ -* -!Dockerfile -!build -!entrypoint.sh diff --git a/airbyte-integrations/bases/standard-source-test/Dockerfile b/airbyte-integrations/bases/standard-source-test/Dockerfile deleted file mode 100644 index eae2c7f1cf6d..000000000000 --- a/airbyte-integrations/bases/standard-source-test/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -ARG JDK_VERSION=17.0.4 -FROM amazoncorretto:${JDK_VERSION} - -ARG DOCKER_BUILD_ARCH=amd64 - -# Install Docker to launch worker images. Eventually should be replaced with Docker-java. -# See https://gitter.im/docker-java/docker-java?at=5f3eb87ba8c1780176603f4e for more information on why we are not currently using Docker-java -RUN amazon-linux-extras install -y docker -RUN yum install -y openssl jq tar && yum clean all - -ENV APPLICATION standard-source-test - -WORKDIR /app - -COPY entrypoint.sh . -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 - -ENTRYPOINT ["/app/entrypoint.sh"] - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/standard-source-test diff --git a/airbyte-integrations/bases/standard-source-test/build.gradle b/airbyte-integrations/bases/standard-source-test/build.gradle deleted file mode 100644 index 5d0b24036f02..000000000000 --- a/airbyte-integrations/bases/standard-source-test/build.gradle +++ /dev/null @@ -1,83 +0,0 @@ -buildscript { - dependencies { - classpath 'org.jsoup:jsoup:1.13.1' - } -} - -plugins { - id 'application' - id 'airbyte-docker' -} - -import org.jsoup.Jsoup; - -dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-api') - implementation project(':airbyte-connector-test-harnesses:acceptance-test-harness') - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation 'org.mockito:mockito-core:4.6.1' - - implementation 'net.sourceforge.argparse4j:argparse4j:0.8.1' - - runtimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.4.2' - implementation 'org.junit.platform:junit-platform-launcher:1.7.0' - implementation 'org.junit.jupiter:junit-jupiter-api:5.4.2' - implementation 'org.junit.jupiter:junit-jupiter-params:5.8.1' -} - -def getFullPath(String className) { - def matchingFiles = project.fileTree("src/main/java") - .filter { file -> file.getName().equals("${className}.java".toString()) }.asCollection() - if (matchingFiles.size() == 0) { - throw new IllegalArgumentException("Ambiguous class name ${className}: no file found.") - } - if (matchingFiles.size() > 1) { - throw new IllegalArgumentException("Ambiguous class name ${className}: more than one matching file was found. Files found: ${matchingFiles}") - } - def absoluteFilePath = matchingFiles[0].toString() - def pathInPackage = project.relativePath(absoluteFilePath.toString()).replaceAll("src/main/java/", "").replaceAll("\\.java", "") - return pathInPackage -} - -task generateSourceTestDocs(type: Javadoc) { - def javadocOutputDir = project.file("${project.buildDir}/docs/javadoc") - - options.addStringOption('Xdoclint:none', '-quiet') - classpath = sourceSets.main.runtimeClasspath - source = sourceSets.main.allJava - destinationDir = javadocOutputDir - - doLast { - def className = "SourceAcceptanceTest" - // this can be made into a list once we have multiple standard tests, and can also be used for destinations - def pathInPackage = getFullPath(className) - def stdSrcTest = project.file("${javadocOutputDir}/${pathInPackage}.html").readLines().join("\n") - def methodList = Jsoup.parse(stdSrcTest).body().select("section.methodDetails>ul>li>section") - def md = "" - for (methodInfo in methodList) { - def annotations = methodInfo.select(".memberSignature>.annotations").text() - if (!annotations.contains("@Test")) { - continue - } - def methodName = methodInfo.selectFirst("div>span.memberName").text() - def methodDocstring = methodInfo.selectFirst("div.block") - - md += "## ${methodName}\n\n" - md += "${methodDocstring != null ? methodDocstring.text().replaceAll(/([()])/, '\\\\$1') : 'No method description was provided'}\n\n" - } - def outputDoc = new File("${rootDir}/docs/connector-development/testing-connectors/standard-source-tests.md") - outputDoc.write "# Standard Source Test Suite\n\n" - outputDoc.append "Test methods start with `test`. Other methods are internal helpers in the java class implementing the test suite.\n\n" - outputDoc.append md - } - - outputs.upToDateWhen { false } -} - -project.build.dependsOn(generateSourceTestDocs) - -application { - mainClass = 'io.airbyte.integrations.standardtest.source.PythonSourceAcceptanceTest' -} diff --git a/airbyte-integrations/bases/standard-source-test/entrypoint.sh b/airbyte-integrations/bases/standard-source-test/entrypoint.sh deleted file mode 100755 index 2032df7db986..000000000000 --- a/airbyte-integrations/bases/standard-source-test/entrypoint.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -set -e - -/app/bin/${APPLICATION} "$@" diff --git a/airbyte-integrations/bases/standard-source-test/readme.md b/airbyte-integrations/bases/standard-source-test/readme.md deleted file mode 100644 index c55e734b6ac0..000000000000 --- a/airbyte-integrations/bases/standard-source-test/readme.md +++ /dev/null @@ -1,37 +0,0 @@ -# Standard Source Test - -## Overview - -### These tests are designed to do the following: - -1. Test basic functionality for any Airbyte source. Think of it as an "It works!" test. - 1. Each test should test functionality that is expected of all Airbyte sources. -1. Require minimum effort from the author of the integration. - 1. Think of these are "free" tests that Airbyte provides to make sure the integration works. -1. To be run for ALL Airbyte sources. - -### These tests are _not_ designed to do the following: - -1. Test any integration-specific cases. -1. Test corner cases for specific integrations. -1. Replace good unit testing and integration testing for the integration. - -We will _not_ merge sources that cannot pass these tests unless the author can provide a very good reason™. - -## What tests do the standard tests include? - -Check out each function in [SourceAcceptanceTest](src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java). Each function annotated with `@Test` is a single test. Each of these functions is proceeded by comments that document what they are testing. - -## How to run them from your integration? - -- If writing a source in Python, **don't use this.** - -## What do I need to provide as input to these tests? - -- The name of the image of your integration (with the `:dev` tag at the end). -- A handful of json inputs, e.g. a valid configuration file and a catalog that will be used to try to run the `read` operation on your integration. These are fully documented in [SourceAcceptanceTest](src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java). Each method that is marked as `abstract` are methods that the user needs to implement to provide the necessary information. Each of these methods are preceded by comments explaining what they need to return. -- Optionally you can run before and after methods before each test. - -## Do I have to write java to use these tests? - -No! Our goal is to allow you to write your integration _entirely_ in your language of choice. If you are writing an integration in Python, for instance, you should be able to interact with this test suite in python and not need to write java. _Right now, we only have a Python interface to reduce friction for interacting with these tests_, and with time, we intend to make more language-specific helpers available. In the meantime, however, you can still use your language of choice and leverage this standard test suite. diff --git a/airbyte-integrations/connector-templates/connector_acceptance_test_files/acceptance-test-docker.sh b/airbyte-integrations/connector-templates/connector_acceptance_test_files/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connector-templates/connector_acceptance_test_files/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connector-templates/destination-java/Destination.java.hbs b/airbyte-integrations/connector-templates/destination-java/Destination.java.hbs index 56ba3861303a..f5785a3be592 100644 --- a/airbyte-integrations/connector-templates/destination-java/Destination.java.hbs +++ b/airbyte-integrations/connector-templates/destination-java/Destination.java.hbs @@ -5,10 +5,10 @@ package io.airbyte.integrations.destination.{{snakeCase name}}; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/connector-templates/destination-java/DestinationAcceptanceTest.java.hbs b/airbyte-integrations/connector-templates/destination-java/DestinationAcceptanceTest.java.hbs index 061ddf1404b4..1663f1066429 100644 --- a/airbyte-integrations/connector-templates/destination-java/DestinationAcceptanceTest.java.hbs +++ b/airbyte-integrations/connector-templates/destination-java/DestinationAcceptanceTest.java.hbs @@ -5,7 +5,7 @@ package io.airbyte.integrations.destination.{{snakeCase name}}; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; import java.io.IOException; import java.util.List; import org.slf4j.Logger; diff --git a/airbyte-integrations/connector-templates/destination-java/Dockerfile.hbs b/airbyte-integrations/connector-templates/destination-java/Dockerfile.hbs deleted file mode 100644 index 3d187026e8ff..000000000000 --- a/airbyte-integrations/connector-templates/destination-java/Dockerfile.hbs +++ /dev/null @@ -1,18 +0,0 @@ -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte -ENV APPLICATION destination-{{dashCase name}} - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte -ENV APPLICATION destination-{{dashCase name}} - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/destination-{{dashCase name}} diff --git a/airbyte-integrations/connector-templates/destination-java/README.md.hbs b/airbyte-integrations/connector-templates/destination-java/README.md.hbs index 405c510368bb..390002dca65d 100644 --- a/airbyte-integrations/connector-templates/destination-java/README.md.hbs +++ b/airbyte-integrations/connector-templates/destination-java/README.md.hbs @@ -22,10 +22,10 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: ``` -./gradlew :airbyte-integrations:connectors:destination-{{dashCase name}}:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-{{dashCase name}}:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Once built, the docker image name and tag will be `airbyte/source-{{dashCase name}}:dev`. + #### Run Then run any of the connector commands as follows: @@ -50,7 +50,7 @@ Airbyte has a standard test suite that all destination connectors must pass. Imp All commands should be run from airbyte project root. To run unit tests: ``` -./gradlew :airbyte-integrations:connectors:destination-{{dashCase name}}:unitTest +./gradlew :airbyte-integrations:connectors:destination-{{dashCase name}}:check ``` To run acceptance and custom integration tests: ``` @@ -61,8 +61,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-{{dashCase name}} test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/{{dashCase name}}.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connector-templates/destination-java/build.gradle.hbs b/airbyte-integrations/connector-templates/destination-java/build.gradle.hbs index d3e7fa119c82..a999413ffe2a 100644 --- a/airbyte-integrations/connector-templates/destination-java/build.gradle.hbs +++ b/airbyte-integrations/connector-templates/destination-java/build.gradle.hbs @@ -1,19 +1,20 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = true +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.{{snakeCase name}}.{{properCase name}}Destination' } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-{{dashCase name}}') } diff --git a/airbyte-integrations/connector-templates/destination-python/.dockerignore b/airbyte-integrations/connector-templates/destination-python/.dockerignore deleted file mode 100644 index 76f2dace41f2..000000000000 --- a/airbyte-integrations/connector-templates/destination-python/.dockerignore +++ /dev/null @@ -1,5 +0,0 @@ -* -!Dockerfile -!main.py -!destination_{{snakeCase name}} -!setup.py diff --git a/airbyte-integrations/connector-templates/destination-python/Dockerfile b/airbyte-integrations/connector-templates/destination-python/Dockerfile deleted file mode 100644 index 0fc14e67b800..000000000000 --- a/airbyte-integrations/connector-templates/destination-python/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY destination_{{snakeCase name}} ./destination_{{snakeCase name}} - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/destination-{{dashCase name}} diff --git a/airbyte-integrations/connector-templates/destination-python/README.md b/airbyte-integrations/connector-templates/destination-python/README.md index a6e474742a66..a288a5693369 100644 --- a/airbyte-integrations/connector-templates/destination-python/README.md +++ b/airbyte-integrations/connector-templates/destination-python/README.md @@ -8,9 +8,9 @@ For information about how to use this connector within Airbyte, see [the documen ### Prerequisites **To iterate on this connector, make sure to complete this prerequisites section.** -#### Minimum Python version required `= 3.7.0` +#### Minimum Python version required `= 3.9.0` -#### Build & Activate Virtual Environment and install dependencies +#### Activate Virtual Environment and install dependencies From this connector directory, create a virtual environment: ``` python -m venv .venv @@ -29,11 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:destination-{{dashCase name}}:build -``` #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/destinations/{{dashCase name}}) @@ -54,19 +49,68 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/destination-{{dashCase name}}:dev +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name destination-{{dashCase name}} build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/destination-{{dashCase name}}:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") -You can also build the connector image via Gradle: +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:destination-{{dashCase name}}:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/destination-{{dashCase name}}:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/destination-{{dashCase name}}:dev . +# Running the spec command against your patched connector +docker run airbyte/destination-{{dashCase name}}:dev spec +```` #### Run Then run any of the connector commands as follows: ``` @@ -97,16 +141,8 @@ python -m pytest integration_tests #### Acceptance Tests Coming soon: -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:destination-{{dashCase name}}:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:destination-{{dashCase name}}:integrationTest -``` +### Using `airbyte-ci` to run tests +See [airbyte-ci documentation](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#connectors-test-command) ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. diff --git a/airbyte-integrations/connector-templates/destination-python/build.gradle b/airbyte-integrations/connector-templates/destination-python/build.gradle deleted file mode 100644 index 677f927afdb1..000000000000 --- a/airbyte-integrations/connector-templates/destination-python/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'destination_{{snakeCase name}}' -} diff --git a/airbyte-integrations/connector-templates/destination-python/metadata.yaml.hbs b/airbyte-integrations/connector-templates/destination-python/metadata.yaml.hbs index fdd5c3deb969..9f24ee5cdea5 100644 --- a/airbyte-integrations/connector-templates/destination-python/metadata.yaml.hbs +++ b/airbyte-integrations/connector-templates/destination-python/metadata.yaml.hbs @@ -7,6 +7,11 @@ data: enabled: false cloud: enabled: false + connectorBuildOptions: + # Please update to the latest version of the connector base image. + # Please use the full address with sha256 hash to guarantee build reproducibility. + # https://hub.docker.com/r/airbyte/python-connector-base + baseImage: docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27 connectorSubtype: database connectorType: destination definitionId: {{generateDefinitionId}} diff --git a/airbyte-integrations/connector-templates/generator/README.md b/airbyte-integrations/connector-templates/generator/README.md index c16b31d2f33a..38f79f4c1bc2 100644 --- a/airbyte-integrations/connector-templates/generator/README.md +++ b/airbyte-integrations/connector-templates/generator/README.md @@ -41,6 +41,6 @@ Together, these two invariants guarantee that the templates produce a valid modu The way this is performed is as follows: -1. [in CI ](https://github.com/airbytehq/airbyte/blob/master/.github/workflows/gradle.yml) we trigger the task `:airbyte-integrations:connector-templates:generator:testScaffoldTemplates`. This task deletes the checked in `java-jdbc-scaffolding`. Then the task generates a fresh instance of the module with the same name `java-jdbc-scaffolding`. +1. [in CI ](https://github.com/airbytehq/airbyte/blob/master/.github/workflows/gradle.yml) we trigger the task `:airbyte-integrations:connector-templates:generator:generateScaffolds`. This task deletes the checked in `java-jdbc-scaffolding`. Then the task generates a fresh instance of the module with the same name `java-jdbc-scaffolding`. 1. We run a `git diff`. If there is a diff, then fail the build (this means the latest version of the templates produce code which has not been manually reviewed by someone who checked them in intentionally). Steps 1 & 2 test the first invariant. 1. Separately, in `settings.gradle`, the `java-jdbc-scaffolding` module is registered as a java submodule. This causes it to be built as part of the normal build cycle triggered in CI. If the generated code does not compile for whatever reason, the build will fail on building the `java-jdbc-scaffolding` module. diff --git a/airbyte-integrations/connector-templates/generator/build.gradle b/airbyte-integrations/connector-templates/generator/build.gradle index 53df8f43d219..2a93a2f67a08 100644 --- a/airbyte-integrations/connector-templates/generator/build.gradle +++ b/airbyte-integrations/connector-templates/generator/build.gradle @@ -1,24 +1,10 @@ -plugins { - id "base" - id "com.github.node-gradle.node" version "3.5.1" -} - -def nodeVersion = System.getenv('NODE_VERSION') ?: '16.13.0' - -node { - download = true - version = nodeVersion -} - -assemble.dependsOn(npmInstall) - -task testScaffoldTemplates +def generateScaffolds = tasks.register('generateScaffolds') def addScaffoldTemplateTask(name, packageName, outputDirName, scaffoldParams=[]) { - def taskName = "testScaffoldTemplate_${name}" + def taskName = "generateScaffold_${name}" def outputDir = "airbyte-integrations/connectors/${outputDirName}" - def task = tasks.create(taskName) { + def task = tasks.register(taskName) { inputs.files rootProject.fileTree("airbyte-integrations/connector-templates/") doLast { @@ -28,7 +14,7 @@ def addScaffoldTemplateTask(name, packageName, outputDirName, scaffoldParams=[]) } exec { workingDir rootDir - def cmd = ['./tools/integrations/manage.sh', 'scaffold', name, packageName] + def cmd = [project.file('generate.sh'), name, packageName] cmd.addAll(scaffoldParams) commandLine cmd } @@ -37,9 +23,10 @@ def addScaffoldTemplateTask(name, packageName, outputDirName, scaffoldParams=[]) outputs.dir rootProject.file(outputDir) } - testScaffoldTemplates.dependsOn task + generateScaffolds.configure { dependsOn task } } + addScaffoldTemplateTask('Python Source', 'scaffold-source-python', 'source-scaffold-source-python') addScaffoldTemplateTask('Python HTTP API Source', 'scaffold-source-http', 'source-scaffold-source-http') addScaffoldTemplateTask('Java JDBC Source', 'scaffold-java-jdbc', 'source-scaffold-java-jdbc') diff --git a/airbyte-integrations/connector-templates/generator/plopfile.js b/airbyte-integrations/connector-templates/generator/plopfile.js index 8c5af59979f3..f1a97cc3f942 100644 --- a/airbyte-integrations/connector-templates/generator/plopfile.js +++ b/airbyte-integrations/connector-templates/generator/plopfile.js @@ -24,7 +24,6 @@ ${additionalMessage || ""} module.exports = function (plop) { const docRoot = '../../../docs/integrations'; - const definitionRoot = '../../../airbyte-config-oss/init-oss/src/main/resources'; const connectorAcceptanceTestFilesInputRoot = '../connector_acceptance_test_files'; @@ -105,13 +104,6 @@ module.exports = function (plop) { base: pythonDestinationInputRoot, templateFiles: `${pythonDestinationInputRoot}/**/**`, }, - // plop doesn't add dotfiles by default so we manually add them - { - type:'add', - abortOnFail: true, - templateFile: `${pythonDestinationInputRoot}/.dockerignore`, - path: `${pythonDestinationOutputRoot}/.dockerignore` - }, {type: 'emitSuccess', outputPath: pythonDestinationOutputRoot} ] }) @@ -137,13 +129,6 @@ module.exports = function (plop) { base: connectorAcceptanceTestFilesInputRoot, templateFiles: `${connectorAcceptanceTestFilesInputRoot}/**/**`, }, - // plop doesn't add dotfiles by default so we manually add them - { - type:'add', - abortOnFail: true, - templateFile: `${httpApiInputRoot}/.dockerignore.hbs`, - path: `${httpApiOutputRoot}/.dockerignore` - }, {type: 'emitSuccess', outputPath: httpApiOutputRoot} ] }); @@ -169,13 +154,6 @@ module.exports = function (plop) { base: connectorAcceptanceTestFilesInputRoot, templateFiles: `${connectorAcceptanceTestFilesInputRoot}/**/**`, }, - // plop doesn't add dotfiles by default so we manually add them - { - type:'add', - abortOnFail: true, - templateFile: `${lowCodeSourceInputRoot}/.dockerignore.hbs`, - path: `${pythonSourceOutputRoot}/.dockerignore` - }, {type: 'emitSuccess', outputPath: pythonSourceOutputRoot} ] }); @@ -245,12 +223,6 @@ module.exports = function (plop) { base: connectorAcceptanceTestFilesInputRoot, templateFiles: `${connectorAcceptanceTestFilesInputRoot}/**/**`, }, - { - type:'add', - abortOnFail: true, - templateFile: `${pythonSourceInputRoot}/.dockerignore.hbs`, - path: `${pythonSourceOutputRoot}/.dockerignore` - }, {type: 'emitSuccess', outputPath: pythonSourceOutputRoot, message: "For a checklist of what to do next go to https://docs.airbyte.com/connector-development/tutorials/building-a-python-source"}] }); @@ -323,12 +295,6 @@ module.exports = function (plop) { templateFile: `${javaDestinationInput}/.dockerignore.hbs`, path: `${javaDestinationOutputRoot}/.dockerignore` }, - { - type: 'add', - abortOnFail: true, - templateFile: `${javaDestinationInput}/Dockerfile.hbs`, - path: `${javaDestinationOutputRoot}/Dockerfile` - }, // Java { type: 'add', diff --git a/airbyte-integrations/connector-templates/source-configuration-based/.dockerignore.hbs b/airbyte-integrations/connector-templates/source-configuration-based/.dockerignore.hbs deleted file mode 100644 index bd2bfee8f431..000000000000 --- a/airbyte-integrations/connector-templates/source-configuration-based/.dockerignore.hbs +++ /dev/null @@ -1,6 +0,0 @@ -* -!Dockerfile -!main.py -!source_{{snakeCase name}} -!setup.py -!secrets diff --git a/airbyte-integrations/connector-templates/source-configuration-based/Dockerfile b/airbyte-integrations/connector-templates/source-configuration-based/Dockerfile deleted file mode 100644 index 66b0b426628b..000000000000 --- a/airbyte-integrations/connector-templates/source-configuration-based/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_{{snakeCase name}} ./source_{{snakeCase name}} - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-{{dashCase name}} diff --git a/airbyte-integrations/connector-templates/source-configuration-based/README.md.hbs b/airbyte-integrations/connector-templates/source-configuration-based/README.md.hbs index 76138fb834b9..ebfac13705c4 100644 --- a/airbyte-integrations/connector-templates/source-configuration-based/README.md.hbs +++ b/airbyte-integrations/connector-templates/source-configuration-based/README.md.hbs @@ -5,13 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:build -``` #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/{{dashCase name}}) @@ -24,18 +17,67 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-{{dashCase name}}:dev +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name source-{{dashCase name}} build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-{{dashCase name}}:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations -You can also build the connector image via Gradle: +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-{{dashCase name}}:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. + +2. Build your image: +```bash +docker build -t airbyte/source-{{dashCase name}}:dev . +# Running the spec command against your patched connector +docker run airbyte/source-{{dashCase name}}:dev spec #### Run Then run any of the connector commands as follows: @@ -47,24 +89,12 @@ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integrat ``` ## Testing -#### Acceptance Tests +### Acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. - -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:integrationTest +Please run acceptance tests via [airbyte-ci](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#connectors-test-command): +```bash +airbyte-ci connectors --name source-{{dashCase name}} test ``` ## Dependency Management @@ -75,8 +105,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-{{dashCase name}} test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/{{dashCase name}}.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connector-templates/source-configuration-based/build.gradle.hbs b/airbyte-integrations/connector-templates/source-configuration-based/build.gradle.hbs deleted file mode 100644 index 277ac58170ca..000000000000 --- a/airbyte-integrations/connector-templates/source-configuration-based/build.gradle.hbs +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_{{snakeCase name}}' -} diff --git a/airbyte-integrations/connector-templates/source-configuration-based/metadata.yaml.hbs b/airbyte-integrations/connector-templates/source-configuration-based/metadata.yaml.hbs index 6499f05d2d71..418482a0b246 100644 --- a/airbyte-integrations/connector-templates/source-configuration-based/metadata.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-configuration-based/metadata.yaml.hbs @@ -7,6 +7,11 @@ data: enabled: false cloud: enabled: false + connectorBuildOptions: + # Please update to the latest version of the connector base image. + # https://hub.docker.com/r/airbyte/python-connector-base + # Please use the full address with sha256 hash to guarantee build reproducibility. + baseImage: docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27 connectorSubtype: api connectorType: source definitionId: {{generateDefinitionId}} @@ -21,5 +26,5 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/{{dashCase name}} tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connector-templates/source-generic/build.gradle b/airbyte-integrations/connector-templates/source-generic/build.gradle deleted file mode 100644 index cd4877519bfc..000000000000 --- a/airbyte-integrations/connector-templates/source-generic/build.gradle +++ /dev/null @@ -1,6 +0,0 @@ -plugins { - // Makes building the docker image a dependency of Gradle's "build" command. This way you could run your entire build inside a docker image - // via ./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:build - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/.dockerignore b/airbyte-integrations/connector-templates/source-java-jdbc/.dockerignore index 65c7d0ad3e73..e4fbece78752 100644 --- a/airbyte-integrations/connector-templates/source-java-jdbc/.dockerignore +++ b/airbyte-integrations/connector-templates/source-java-jdbc/.dockerignore @@ -1,3 +1,3 @@ * !Dockerfile -!build +!build/distributions diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/Dockerfile b/airbyte-integrations/connector-templates/source-java-jdbc/Dockerfile deleted file mode 100644 index 1df524922dce..000000000000 --- a/airbyte-integrations/connector-templates/source-java-jdbc/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-{{dashCase name}} - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-{{dashCase name}} - -COPY --from=build /airbyte /airbyte - -# Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile. -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-{{dashCase name}} diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/README.md b/airbyte-integrations/connector-templates/source-java-jdbc/README.md index 785df78b1438..47e23b91d261 100644 --- a/airbyte-integrations/connector-templates/source-java-jdbc/README.md +++ b/airbyte-integrations/connector-templates/source-java-jdbc/README.md @@ -22,10 +22,10 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: ``` -./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:airbyteDocker +./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. + +Once built, the docker image name and tag will be `airbyte/source-{{dashCase name}}:dev`. #### Run Then run any of the connector commands as follows: @@ -62,8 +62,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-{{dashCase name}} test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/{{dashCase name}}.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/acceptance-test-docker.sh b/airbyte-integrations/connector-templates/source-java-jdbc/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connector-templates/source-java-jdbc/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/build.gradle b/airbyte-integrations/connector-templates/source-java-jdbc/build.gradle index cd68e6d2ad45..c1991bdadd16 100644 --- a/airbyte-integrations/connector-templates/source-java-jdbc/build.gradle +++ b/airbyte-integrations/connector-templates/source-java-jdbc/build.gradle @@ -1,30 +1,25 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' - id 'airbyte-connector-acceptance-test' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-sources'] + useLocalCdk = true +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.source.{{dashCase name}}.{{pascalCase name}}Source' } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation project(':airbyte-integrations:connectors:source-relational-db') //TODO Add jdbc driver import here. Ex: implementation 'com.microsoft.sqlserver:mssql-jdbc:8.4.1.jre14' - testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) - testImplementation 'org.apache.commons:commons-lang3:3.11' integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-{{dashCase name}}') - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/src/main/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}Source.java.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/src/main/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}Source.java.hbs index 5e5fdd537e59..a6b204586439 100644 --- a/airbyte-integrations/connector-templates/source-java-jdbc/src/main/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}Source.java.hbs +++ b/airbyte-integrations/connector-templates/source-java-jdbc/src/main/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}Source.java.hbs @@ -5,11 +5,11 @@ package io.airbyte.integrations.source.{{snakeCase name}}; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; import java.sql.JDBCType; import java.util.Set; import org.slf4j.Logger; diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceAcceptanceTest.java.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceAcceptanceTest.java.hbs index 3b120832769e..eba3f8c53e74 100644 --- a/airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceAcceptanceTest.java.hbs +++ b/airbyte-integrations/connector-templates/source-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceAcceptanceTest.java.hbs @@ -7,8 +7,8 @@ package io.airbyte.integrations.source.{{snakeCase name}}; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConnectorSpecification; import java.util.HashMap; diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}JdbcSourceAcceptanceTest.java.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}JdbcSourceAcceptanceTest.java.hbs index 6ea2c102d980..c2046c3a49da 100644 --- a/airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}JdbcSourceAcceptanceTest.java.hbs +++ b/airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}JdbcSourceAcceptanceTest.java.hbs @@ -5,8 +5,8 @@ package io.airbyte.integrations.source.{{snakeCase name}}; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import java.sql.JDBCType; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; diff --git a/airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceTests.java.hbs b/airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceTests.java.hbs index 9d537eb0df47..082504f2e4b8 100644 --- a/airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceTests.java.hbs +++ b/airbyte-integrations/connector-templates/source-java-jdbc/src/test/java/io/airbyte/integrations/source/{{snakeCase name}}/{{pascalCase name}}SourceTests.java.hbs @@ -5,7 +5,7 @@ package io.airbyte.integrations.source.{{snakeCase name}}; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.db.Database; +import io.airbyte.cdk.db.Database; import org.junit.jupiter.api.Test; public class {{pascalCase name}}SourceTests { diff --git a/airbyte-integrations/connector-templates/source-python-http-api/.dockerignore.hbs b/airbyte-integrations/connector-templates/source-python-http-api/.dockerignore.hbs deleted file mode 100644 index bd2bfee8f431..000000000000 --- a/airbyte-integrations/connector-templates/source-python-http-api/.dockerignore.hbs +++ /dev/null @@ -1,6 +0,0 @@ -* -!Dockerfile -!main.py -!source_{{snakeCase name}} -!setup.py -!secrets diff --git a/airbyte-integrations/connector-templates/source-python-http-api/Dockerfile b/airbyte-integrations/connector-templates/source-python-http-api/Dockerfile deleted file mode 100644 index 310591142568..000000000000 --- a/airbyte-integrations/connector-templates/source-python-http-api/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.13-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_{{snakeCase name}} ./source_{{snakeCase name}} - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-{{dashCase name}} diff --git a/airbyte-integrations/connector-templates/source-python-http-api/README.md.hbs b/airbyte-integrations/connector-templates/source-python-http-api/README.md.hbs index e7a8d2fb37c1..56e84e01802c 100644 --- a/airbyte-integrations/connector-templates/source-python-http-api/README.md.hbs +++ b/airbyte-integrations/connector-templates/source-python-http-api/README.md.hbs @@ -10,7 +10,7 @@ For information about how to use this connector within Airbyte, see [the documen #### Minimum Python version required `= 3.9.0` -#### Build & Activate Virtual Environment and install dependencies +#### Activate Virtual Environment and install dependencies From this connector directory, create a virtual environment: ``` python -m venv .venv @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/{{dashCase name}}) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_{{snakeCase name}}/spec.yaml` file. @@ -57,24 +49,68 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-{{dashCase name}}:dev -``` +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: -If you want to build the Docker image with the CDK on your local machine (rather than the most recent package published to pypi), from the airbyte base directory run: ```bash -CONNECTOR_TAG= CONNECTOR_NAME= sh airbyte-integrations/scripts/build-connector-image-with-local-cdk.sh +airbyte-ci connectors --name source-{{dashCase name}} build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-{{dashCase name}}:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations -You can also build the connector image via Gradle: +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-{{dashCase name}}:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. + +2. Build your image: +```bash +docker build -t airbyte/source-{{dashCase name}}:dev . +# Running the spec command against your patched connector +docker run airbyte/source-{{dashCase name}}:dev spec +```` #### Run Then run any of the connector commands as follows: @@ -103,24 +139,13 @@ Place custom tests inside `integration_tests/` folder, then, from the connector ``` python -m pytest integration_tests ``` -#### Acceptance Tests + +### Acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:integrationTest +Please run acceptance tests via [airbyte-ci](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#connectors-test-command): +```bash +airbyte-ci connectors --name source-{{dashCase name}} test ``` ## Dependency Management @@ -131,8 +156,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-{{dashCase name}} test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/{{dashCase name}}.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connector-templates/source-python-http-api/build.gradle.hbs b/airbyte-integrations/connector-templates/source-python-http-api/build.gradle.hbs deleted file mode 100644 index 277ac58170ca..000000000000 --- a/airbyte-integrations/connector-templates/source-python-http-api/build.gradle.hbs +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_{{snakeCase name}}' -} diff --git a/airbyte-integrations/connector-templates/source-python-http-api/main.py.hbs b/airbyte-integrations/connector-templates/source-python-http-api/main.py.hbs index dc8ed8df1dc9..202f3973567d 100644 --- a/airbyte-integrations/connector-templates/source-python-http-api/main.py.hbs +++ b/airbyte-integrations/connector-templates/source-python-http-api/main.py.hbs @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_{{snakeCase name}} import Source{{properCase name}} +from source_{{snakeCase name}}.run import run if __name__ == "__main__": - source = Source{{properCase name}}() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connector-templates/source-python-http-api/metadata.yaml.hbs b/airbyte-integrations/connector-templates/source-python-http-api/metadata.yaml.hbs index 629b17607a6b..fdc68039f864 100644 --- a/airbyte-integrations/connector-templates/source-python-http-api/metadata.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-python-http-api/metadata.yaml.hbs @@ -7,6 +7,11 @@ data: enabled: false cloud: enabled: false + connectorBuildOptions: + # Please update to the latest version of the connector base image. + # https://hub.docker.com/r/airbyte/python-connector-base + # Please use the full address with sha256 hash to guarantee build reproducibility. + baseImage: docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27 connectorSubtype: api connectorType: source definitionId: {{generateDefinitionId}} diff --git a/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs b/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs index 667a27713662..8f3eebe3cef1 100644 --- a/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs +++ b/airbyte-integrations/connector-templates/source-python-http-api/setup.py.hbs @@ -27,4 +27,9 @@ setup( extras_require={ "tests": TEST_REQUIREMENTS, }, + entry_points={ + "console_scripts": [ + "source-{{dashCase name}}=source_{{snakeCase name}}.run:run", + ], + }, ) diff --git a/airbyte-integrations/connector-templates/source-python-http-api/source_{{snakeCase name}}/run.py.hbs b/airbyte-integrations/connector-templates/source-python-http-api/source_{{snakeCase name}}/run.py.hbs new file mode 100644 index 000000000000..25c9400301f9 --- /dev/null +++ b/airbyte-integrations/connector-templates/source-python-http-api/source_{{snakeCase name}}/run.py.hbs @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from .source import Source{{properCase name}} + +def run(): + source = Source{{properCase name}}() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connector-templates/source-python/.dockerignore.hbs b/airbyte-integrations/connector-templates/source-python/.dockerignore.hbs deleted file mode 100644 index bd2bfee8f431..000000000000 --- a/airbyte-integrations/connector-templates/source-python/.dockerignore.hbs +++ /dev/null @@ -1,6 +0,0 @@ -* -!Dockerfile -!main.py -!source_{{snakeCase name}} -!setup.py -!secrets diff --git a/airbyte-integrations/connector-templates/source-python/Dockerfile b/airbyte-integrations/connector-templates/source-python/Dockerfile deleted file mode 100644 index 66b0b426628b..000000000000 --- a/airbyte-integrations/connector-templates/source-python/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_{{snakeCase name}} ./source_{{snakeCase name}} - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-{{dashCase name}} diff --git a/airbyte-integrations/connector-templates/source-python/README.md.hbs b/airbyte-integrations/connector-templates/source-python/README.md.hbs index a67e6e35e849..56e84e01802c 100644 --- a/airbyte-integrations/connector-templates/source-python/README.md.hbs +++ b/airbyte-integrations/connector-templates/source-python/README.md.hbs @@ -10,7 +10,7 @@ For information about how to use this connector within Airbyte, see [the documen #### Minimum Python version required `= 3.9.0` -#### Build & Activate Virtual Environment and install dependencies +#### Activate Virtual Environment and install dependencies From this connector directory, create a virtual environment: ``` python -m venv .venv @@ -21,6 +21,7 @@ development environment of choice. To activate it from the terminal, run: ``` source .venv/bin/activate pip install -r requirements.txt +pip install '.[tests]' ``` If you are in an IDE, follow your IDE's instructions to activate the virtualenv. @@ -29,16 +30,10 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/{{dashCase name}}) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_{{snakeCase name}}/spec.yaml` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source {{dashCase name}} test creds` @@ -54,23 +49,68 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-{{dashCase name}}:dev -``` +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: -If you want to build the Docker image with the CDK on your local machine (rather than the most recent package published to pypi), from the airbyte base directory run: ```bash -CONNECTOR_TAG= CONNECTOR_NAME= sh airbyte-integrations/scripts/build-connector-image-with-local-cdk.sh +airbyte-ci connectors --name source-{{dashCase name}} build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-{{dashCase name}}:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations -You can also build the connector image via Gradle: +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-{{dashCase name}}:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. + +2. Build your image: +```bash +docker build -t airbyte/source-{{dashCase name}}:dev . +# Running the spec command against your patched connector +docker run airbyte/source-{{dashCase name}}:dev spec +```` #### Run Then run any of the connector commands as follows: @@ -81,7 +121,7 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-{{dashCase name}}:dev docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-{{dashCase name}}:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. First install test dependencies into your virtual environment: ``` pip install .[tests] @@ -99,24 +139,13 @@ Place custom tests inside `integration_tests/` folder, then, from the connector ``` python -m pytest integration_tests ``` -#### Acceptance Tests + +### Acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}:integrationTest +Please run acceptance tests via [airbyte-ci](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#connectors-test-command): +```bash +airbyte-ci connectors --name source-{{dashCase name}} test ``` ## Dependency Management @@ -127,8 +156,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-{{dashCase name}} test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/{{dashCase name}}.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connector-templates/source-python/build.gradle.hbs b/airbyte-integrations/connector-templates/source-python/build.gradle.hbs deleted file mode 100644 index 942862d80dc9..000000000000 --- a/airbyte-integrations/connector-templates/source-python/build.gradle.hbs +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_{{snakeCase name}}_singer' -} diff --git a/airbyte-integrations/connector-templates/source-python/main.py.hbs b/airbyte-integrations/connector-templates/source-python/main.py.hbs index dc8ed8df1dc9..202f3973567d 100644 --- a/airbyte-integrations/connector-templates/source-python/main.py.hbs +++ b/airbyte-integrations/connector-templates/source-python/main.py.hbs @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_{{snakeCase name}} import Source{{properCase name}} +from source_{{snakeCase name}}.run import run if __name__ == "__main__": - source = Source{{properCase name}}() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connector-templates/source-python/metadata.yaml.hbs b/airbyte-integrations/connector-templates/source-python/metadata.yaml.hbs index 629b17607a6b..fdc68039f864 100644 --- a/airbyte-integrations/connector-templates/source-python/metadata.yaml.hbs +++ b/airbyte-integrations/connector-templates/source-python/metadata.yaml.hbs @@ -7,6 +7,11 @@ data: enabled: false cloud: enabled: false + connectorBuildOptions: + # Please update to the latest version of the connector base image. + # https://hub.docker.com/r/airbyte/python-connector-base + # Please use the full address with sha256 hash to guarantee build reproducibility. + baseImage: docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27 connectorSubtype: api connectorType: source definitionId: {{generateDefinitionId}} diff --git a/airbyte-integrations/connector-templates/source-python/setup.py.hbs b/airbyte-integrations/connector-templates/source-python/setup.py.hbs index 563d13c3708c..b16123258acb 100644 --- a/airbyte-integrations/connector-templates/source-python/setup.py.hbs +++ b/airbyte-integrations/connector-templates/source-python/setup.py.hbs @@ -27,4 +27,9 @@ setup( extras_require={ "tests": TEST_REQUIREMENTS, }, + entry_points={ + "console_scripts": [ + "source-{{dashCase name}}=source_{{snakeCase name}}.run:run", + ], + }, ) diff --git a/airbyte-integrations/connector-templates/source-python/source_{{snakeCase name}}/run.py.hbs b/airbyte-integrations/connector-templates/source-python/source_{{snakeCase name}}/run.py.hbs new file mode 100644 index 000000000000..25c9400301f9 --- /dev/null +++ b/airbyte-integrations/connector-templates/source-python/source_{{snakeCase name}}/run.py.hbs @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from .source import Source{{properCase name}} + +def run(): + source = Source{{properCase name}}() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connector-templates/source-singer/README.md.hbs b/airbyte-integrations/connector-templates/source-singer/README.md.hbs index 55398c85cc04..92f666a72d26 100644 --- a/airbyte-integrations/connector-templates/source-singer/README.md.hbs +++ b/airbyte-integrations/connector-templates/source-singer/README.md.hbs @@ -107,7 +107,7 @@ To run your integration tests with docker All commands should be run from airbyte project root. To run unit tests: ``` -./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}-singer:unitTest +./gradlew :airbyte-integrations:connectors:source-{{dashCase name}}-singer:check ``` To run acceptance and custom integration tests: ``` diff --git a/airbyte-integrations/connector-templates/source-singer/build.gradle.hbs b/airbyte-integrations/connector-templates/source-singer/build.gradle.hbs deleted file mode 100644 index 942862d80dc9..000000000000 --- a/airbyte-integrations/connector-templates/source-singer/build.gradle.hbs +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_{{snakeCase name}}_singer' -} diff --git a/airbyte-integrations/connectors-performance/destination-harness/.dockerignore b/airbyte-integrations/connectors-performance/destination-harness/.dockerignore index c8f982b06349..d538024ad446 100644 --- a/airbyte-integrations/connectors-performance/destination-harness/.dockerignore +++ b/airbyte-integrations/connectors-performance/destination-harness/.dockerignore @@ -1,4 +1,4 @@ * !Dockerfile -!build +!build/distributions !base.sh diff --git a/airbyte-integrations/connectors-performance/destination-harness/build.gradle b/airbyte-integrations/connectors-performance/destination-harness/build.gradle index 1d2335a175ef..25332d02ca27 100644 --- a/airbyte-integrations/connectors-performance/destination-harness/build.gradle +++ b/airbyte-integrations/connectors-performance/destination-harness/build.gradle @@ -1,31 +1,29 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' -} - -repositories { - maven { - url 'https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/' - } + id 'airbyte-docker-legacy' } application { mainClass = 'io.airbyte.integrations.destination_performance.Main' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } + dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol + implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + implementation 'io.fabric8:kubernetes-client:5.12.2' implementation 'org.apache.commons:commons-lang3:3.11' - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation 'io.airbyte:airbyte-commons-worker:0.42.0' implementation 'io.airbyte.airbyte-config:config-models:0.42.0' implementation 'junit:junit:4.13.1' - implementation 'junit:junit:4.13.1' implementation 'org.testng:testng:7.1.0' implementation 'org.junit.jupiter:junit-jupiter:5.8.1' } + +//This is only needed because we're using some very old libraries from airbyte-commons that were not packaged correctly +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} diff --git a/airbyte-integrations/connectors-performance/source-harness/.dockerignore b/airbyte-integrations/connectors-performance/source-harness/.dockerignore index c8f982b06349..d538024ad446 100644 --- a/airbyte-integrations/connectors-performance/source-harness/.dockerignore +++ b/airbyte-integrations/connectors-performance/source-harness/.dockerignore @@ -1,4 +1,4 @@ * !Dockerfile -!build +!build/distributions !base.sh diff --git a/airbyte-integrations/connectors-performance/source-harness/build.gradle b/airbyte-integrations/connectors-performance/source-harness/build.gradle index 652af6130bfa..2cdfcc461d3e 100644 --- a/airbyte-integrations/connectors-performance/source-harness/build.gradle +++ b/airbyte-integrations/connectors-performance/source-harness/build.gradle @@ -1,25 +1,26 @@ plugins { id 'application' - id 'airbyte-docker' -} - -repositories { - maven { - url 'https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/' - } + id 'airbyte-docker-legacy' } application { mainClass = 'io.airbyte.integrations.source_performance.Main' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } + dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol + implementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + implementation 'io.fabric8:kubernetes-client:5.12.2' implementation 'org.apache.commons:commons-lang3:3.11' - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation 'io.airbyte:airbyte-commons-worker:0.42.0' implementation 'io.airbyte.airbyte-config:config-models:0.42.0' + implementation 'com.datadoghq:datadog-api-client:2.16.0' +} + +//This is only needed because we're using some very old libraries from airbyte-commons that were not packaged correctly +java { + compileJava { + options.compilerArgs.remove("-Werror") + } } diff --git a/airbyte-integrations/connectors-performance/source-harness/src/main/java/io/airbyte/integrations/source_performance/Main.java b/airbyte-integrations/connectors-performance/source-harness/src/main/java/io/airbyte/integrations/source_performance/Main.java index 80d8094db2cf..56a0122e0593 100644 --- a/airbyte-integrations/connectors-performance/source-harness/src/main/java/io/airbyte/integrations/source_performance/Main.java +++ b/airbyte-integrations/connectors-performance/source-harness/src/main/java/io/airbyte/integrations/source_performance/Main.java @@ -8,8 +8,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; import java.io.IOException; -import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; @@ -28,6 +28,7 @@ public static void main(final String[] args) { // TODO: (ryankfu) add function parity with destination_performance int numOfParallelStreams = 1; String syncMode = "full_refresh"; + boolean reportToDatadog = false; // TODO: (ryankfu) Integrated something akin to {@link Clis} for parsing arguments. switch (args.length) { @@ -47,6 +48,13 @@ public static void main(final String[] args) { numOfParallelStreams = Integer.parseInt(args[2]); syncMode = args[3]; } + case 5 -> { + image = args[0]; + dataset = args[1]; + numOfParallelStreams = Integer.parseInt(args[2]); + syncMode = args[3]; + reportToDatadog = Boolean.parseBoolean(args[4]); + } default -> { log.info("unexpected arguments"); System.exit(1); @@ -65,7 +73,7 @@ public static void main(final String[] args) { final JsonNode catalog; try { - catalog = getCatalog(dataset, connector); + catalog = getCatalog(dataset, connector, syncMode); } catch (final IOException ex) { throw new IllegalStateException("Failed to read catalog", ex); } @@ -78,6 +86,9 @@ public static void main(final String[] args) { try { final PerformanceTest test = new PerformanceTest( image, + dataset, + syncMode, + reportToDatadog, config.toString(), catalog.toString()); test.runTest(); @@ -89,11 +100,11 @@ public static void main(final String[] args) { System.exit(0); } - static JsonNode getCatalog(final String dataset, final String connector) throws IOException { + static JsonNode getCatalog(final String dataset, final String connector, final String syncMode) throws IOException { final ObjectMapper objectMapper = new ObjectMapper(); final String catalogFilename = "catalogs/%s/%s_catalog.json".formatted(connector, dataset); - final InputStream is = Thread.currentThread().getContextClassLoader().getResourceAsStream(catalogFilename); - return objectMapper.readTree(is); + final String template = MoreResources.readResource(catalogFilename); + return objectMapper.readTree(String.format(template, syncMode)); } } diff --git a/airbyte-integrations/connectors-performance/source-harness/src/main/java/io/airbyte/integrations/source_performance/PerformanceTest.java b/airbyte-integrations/connectors-performance/source-harness/src/main/java/io/airbyte/integrations/source_performance/PerformanceTest.java index 14407818c932..4166aa28f80c 100644 --- a/airbyte-integrations/connectors-performance/source-harness/src/main/java/io/airbyte/integrations/source_performance/PerformanceTest.java +++ b/airbyte-integrations/connectors-performance/source-harness/src/main/java/io/airbyte/integrations/source_performance/PerformanceTest.java @@ -4,6 +4,15 @@ package io.airbyte.integrations.source_performance; +import com.datadog.api.client.ApiClient; +import com.datadog.api.client.ApiException; +import com.datadog.api.client.v2.api.MetricsApi; +import com.datadog.api.client.v2.model.IntakePayloadAccepted; +import com.datadog.api.client.v2.model.MetricIntakeType; +import com.datadog.api.client.v2.model.MetricPayload; +import com.datadog.api.client.v2.model.MetricPoint; +import com.datadog.api.client.v2.model.MetricResource; +import com.datadog.api.client.v2.model.MetricSeries; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -26,6 +35,8 @@ import java.net.InetAddress; import java.nio.file.Path; import java.time.Duration; +import java.time.OffsetDateTime; +import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.Set; @@ -43,17 +54,34 @@ public class PerformanceTest { public static final double MEGABYTE = Math.pow(1024, 2); private final String imageName; + private final String dataset; + private final String syncMode; + private final boolean reportToDatadog; private final JsonNode config; private final ConfiguredAirbyteCatalog catalog; - PerformanceTest(final String imageName, final String config, final String catalog) throws JsonProcessingException { + PerformanceTest(final String imageName, + final String dataset, + final String syncMode, + final Boolean reportToDatadog, + final String config, + final String catalog) + throws JsonProcessingException { final ObjectMapper mapper = new ObjectMapper(); this.imageName = imageName; + this.dataset = dataset; + this.syncMode = syncMode; + this.reportToDatadog = reportToDatadog; this.config = mapper.readTree(config); this.catalog = Jsons.deserialize(catalog, ConfiguredAirbyteCatalog.class); } void runTest() throws Exception { + + // Initialize datadog. + ApiClient defaultClient = ApiClient.getDefaultApiClient(); + MetricsApi apiInstance = new MetricsApi(defaultClient); + KubePortManagerSingleton.init(PORTS); final KubernetesClient fabricClient = new DefaultKubernetesClient(); @@ -100,13 +128,56 @@ void runTest() throws Exception { totalBytes / MEGABYTE); } } + if (source.getExitValue() > 0) { + throw new RuntimeException("Source failed with exit code: " + source.getExitValue()); + } log.info("Test ended successfully"); final var end = System.currentTimeMillis(); final var totalMB = totalBytes / MEGABYTE; final var totalTimeSecs = (end - start) / 1000.0; final var rps = counter / totalTimeSecs; - log.info("total secs: {}. total MB read: {}, rps: {}, throughput: {}", totalTimeSecs, totalMB, rps, totalMB / totalTimeSecs); + final var throughput = totalMB / totalTimeSecs; + log.info("total secs: {}. total MB read: {}, rps: {}, throughput: {}", totalTimeSecs, totalMB, rps, throughput); source.close(); + if (!reportToDatadog) { + return; + } + + final long reportingTimeInEpochSeconds = OffsetDateTime.now().toInstant().getEpochSecond(); + + List metricResources = List.of( + new MetricResource().name("github").type("runner"), + new MetricResource().name(imageName).type("image"), + new MetricResource().name(dataset).type("dataset"), + new MetricResource().name(syncMode).type("syncMode")); + MetricPayload body = + new MetricPayload() + .series( + List.of( + new MetricSeries() + .metric("connectors.performance.rps") + .type(MetricIntakeType.GAUGE) + .points( + Collections.singletonList( + new MetricPoint() + .timestamp(reportingTimeInEpochSeconds) + .value(rps))) + .resources(metricResources), + new MetricSeries() + .metric("connectors.performance.throughput") + .type(MetricIntakeType.GAUGE) + .points( + Collections.singletonList( + new MetricPoint() + .timestamp(reportingTimeInEpochSeconds) + .value(throughput))) + .resources(metricResources))); + try { + IntakePayloadAccepted result = apiInstance.submitMetrics(body); + System.out.println(result); + } catch (ApiException e) { + log.error("Exception when calling MetricsApi#submitMetrics.", e); + } } private static V0 convertProtocolObject(final V1 v1, final Class klass) { diff --git a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mongodb-v2/1m_catalog.json b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mongodb-v2/1m_catalog.json new file mode 100644 index 000000000000..bbc0682e9d01 --- /dev/null +++ b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mongodb-v2/1m_catalog.json @@ -0,0 +1,63 @@ +{ + "streams": [ + { + "stream": { + "name": "Cluster0", + "namespace": "perf_test_1m", + "json_schema": { + "type": "object", + "properties": { + "_id": { + "type": "string" + }, + "data": { + "type": "string" + }, + "name": { + "type": "string" + }, + "title": { + "type": "string" + }, + "intField": { + "type": "number" + }, + "paragraph": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "description": { + "type": "string" + }, + "doubleField": { + "type": "number" + }, + "objectField": { + "type": "object" + }, + "_ab_cdc_cursor": { + "type": "number", + "airbyte_type": "integer" + }, + "_ab_cdc_deleted_at": { + "type": "string" + }, + "_ab_cdc_updated_at": { + "type": "string" + } + } + }, + "default_cursor_field": ["_ab_cdc_cursor"], + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "source_defined_primary_key": [["_id"]] + }, + "sync_mode": "incremental", + "primary_key": [["_id"]], + "cursor_field": ["_ab_cdc_cursor"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/10m_catalog.json b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/10m_catalog.json index 75d4fc01be8f..48e4f23dfd5d 100644 --- a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/10m_catalog.json +++ b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/10m_catalog.json @@ -40,7 +40,7 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "full_refresh", + "sync_mode": "%1$s", "primary_key": [["id"]], "cursor_field": ["id"], "destination_sync_mode": "append" @@ -113,7 +113,7 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "full_refresh", + "sync_mode": "%1$s", "primary_key": [["id"]], "cursor_field": ["updated_at"], "destination_sync_mode": "append" @@ -152,7 +152,7 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "full_refresh", + "sync_mode": "%1$s", "primary_key": [["id"]], "cursor_field": ["created_at"], "destination_sync_mode": "append" diff --git a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/1m_catalog.json b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/1m_catalog.json index ad274523846a..f9ebc75dbb44 100644 --- a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/1m_catalog.json +++ b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/1m_catalog.json @@ -40,7 +40,7 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "full_refresh", + "sync_mode": "%1$s", "primary_key": [["id"]], "cursor_field": ["id"], "destination_sync_mode": "append" @@ -113,7 +113,7 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "full_refresh", + "sync_mode": "%1$s", "primary_key": [["id"]], "cursor_field": ["updated_at"], "destination_sync_mode": "append" @@ -152,7 +152,7 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "full_refresh", + "sync_mode": "%1$s", "primary_key": [["id"]], "cursor_field": ["created_at"], "destination_sync_mode": "append" diff --git a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/20m_catalog.json b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/20m_catalog.json index 55c38e69c72f..4f9baa60df95 100644 --- a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/20m_catalog.json +++ b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-mysql/20m_catalog.json @@ -40,7 +40,7 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "full_refresh", + "sync_mode": "%1$s", "primary_key": [["id"]], "cursor_field": ["id"], "destination_sync_mode": "append" @@ -113,7 +113,7 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "full_refresh", + "sync_mode": "%1$s", "primary_key": [["id"]], "cursor_field": ["updated_at"], "destination_sync_mode": "append" @@ -152,7 +152,7 @@ "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "full_refresh", + "sync_mode": "%1$s", "primary_key": [["id"]], "cursor_field": ["created_at"], "destination_sync_mode": "append" diff --git a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/10m_catalog.json b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/10m_catalog.json index 07b22645a2a9..a130885e5ef1 100644 --- a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/10m_catalog.json +++ b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/10m_catalog.json @@ -1,54 +1,9 @@ { "streams": [ - { - "stream": { - "name": "purchases", - "namespace": "10m_users", - "json_schema": { - "type": "object", - "properties": { - "id": { - "type": "number", - "airbyte_type": "integer" - }, - "user_id": { - "type": "number", - "airbyte_type": "integer" - }, - "product_id": { - "type": "number", - "airbyte_type": "integer" - }, - "returned_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "purchased_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "added_to_cart_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - } - } - }, - "default_cursor_field": [], - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "primary_key": [["id"]], - "cursor_field": ["id"], - "destination_sync_mode": "append" - }, { "stream": { "name": "users", - "namespace": "10m_users", + "namespace": "public", "json_schema": { "type": "object", "properties": { @@ -111,50 +66,12 @@ }, "default_cursor_field": [], "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, "source_defined_primary_key": [["id"]] }, - "sync_mode": "full_refresh", - "primary_key": [["id"]], - "cursor_field": ["updated_at"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "products", - "namespace": "10m_users", - "json_schema": { - "type": "object", - "properties": { - "id": { - "type": "number", - "airbyte_type": "integer" - }, - "make": { - "type": "string" - }, - "year": { - "type": "string" - }, - "model": { - "type": "string" - }, - "price": { - "type": "number" - }, - "created_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - } - } - }, - "default_cursor_field": [], - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", + "sync_mode": "%s", "primary_key": [["id"]], - "cursor_field": ["created_at"], + "cursor_field": [], "destination_sync_mode": "append" } ] diff --git a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/1m_catalog.json b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/1m_catalog.json index 5e46b7b5d5cb..a130885e5ef1 100644 --- a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/1m_catalog.json +++ b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/1m_catalog.json @@ -1,54 +1,9 @@ { "streams": [ - { - "stream": { - "name": "purchases", - "namespace": "1m_users", - "json_schema": { - "type": "object", - "properties": { - "id": { - "type": "number", - "airbyte_type": "integer" - }, - "user_id": { - "type": "number", - "airbyte_type": "integer" - }, - "product_id": { - "type": "number", - "airbyte_type": "integer" - }, - "returned_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "purchased_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "added_to_cart_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - } - } - }, - "default_cursor_field": [], - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "primary_key": [["id"]], - "cursor_field": ["id"], - "destination_sync_mode": "append" - }, { "stream": { "name": "users", - "namespace": "1m_users", + "namespace": "public", "json_schema": { "type": "object", "properties": { @@ -111,50 +66,12 @@ }, "default_cursor_field": [], "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, "source_defined_primary_key": [["id"]] }, - "sync_mode": "full_refresh", - "primary_key": [["id"]], - "cursor_field": ["updated_at"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "products", - "namespace": "1m_users", - "json_schema": { - "type": "object", - "properties": { - "id": { - "type": "number", - "airbyte_type": "integer" - }, - "make": { - "type": "string" - }, - "year": { - "type": "string" - }, - "model": { - "type": "string" - }, - "price": { - "type": "number" - }, - "created_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - } - } - }, - "default_cursor_field": [], - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", + "sync_mode": "%s", "primary_key": [["id"]], - "cursor_field": ["created_at"], + "cursor_field": [], "destination_sync_mode": "append" } ] diff --git a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/20m_catalog.json b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/20m_catalog.json index 93302af014f6..a130885e5ef1 100644 --- a/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/20m_catalog.json +++ b/airbyte-integrations/connectors-performance/source-harness/src/main/resources/catalogs/source-postgres/20m_catalog.json @@ -1,54 +1,9 @@ { "streams": [ - { - "stream": { - "name": "purchases", - "namespace": "20m_users", - "json_schema": { - "type": "object", - "properties": { - "id": { - "type": "number", - "airbyte_type": "integer" - }, - "user_id": { - "type": "number", - "airbyte_type": "integer" - }, - "product_id": { - "type": "number", - "airbyte_type": "integer" - }, - "returned_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "purchased_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - }, - "added_to_cart_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - } - } - }, - "default_cursor_field": [], - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "primary_key": [["id"]], - "cursor_field": ["id"], - "destination_sync_mode": "append" - }, { "stream": { "name": "users", - "namespace": "20m_users", + "namespace": "public", "json_schema": { "type": "object", "properties": { @@ -111,50 +66,12 @@ }, "default_cursor_field": [], "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, "source_defined_primary_key": [["id"]] }, - "sync_mode": "full_refresh", - "primary_key": [["id"]], - "cursor_field": ["updated_at"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "products", - "namespace": "20m_users", - "json_schema": { - "type": "object", - "properties": { - "id": { - "type": "number", - "airbyte_type": "integer" - }, - "make": { - "type": "string" - }, - "year": { - "type": "string" - }, - "model": { - "type": "string" - }, - "price": { - "type": "number" - }, - "created_at": { - "type": "string", - "format": "date-time", - "airbyte_type": "timestamp_with_timezone" - } - } - }, - "default_cursor_field": [], - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", + "sync_mode": "%s", "primary_key": [["id"]], - "cursor_field": ["created_at"], + "cursor_field": [], "destination_sync_mode": "append" } ] diff --git a/airbyte-integrations/connectors/destination-amazon-sqs/Dockerfile b/airbyte-integrations/connectors/destination-amazon-sqs/Dockerfile index 50e7598d2c3e..9861de2b6843 100644 --- a/airbyte-integrations/connectors/destination-amazon-sqs/Dockerfile +++ b/airbyte-integrations/connectors/destination-amazon-sqs/Dockerfile @@ -34,5 +34,5 @@ COPY destination_amazon_sqs ./destination_amazon_sqs ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/destination-amazon-sqs diff --git a/airbyte-integrations/connectors/destination-amazon-sqs/README.md b/airbyte-integrations/connectors/destination-amazon-sqs/README.md index ca8e2de7f0c6..2856f60b1ae7 100644 --- a/airbyte-integrations/connectors/destination-amazon-sqs/README.md +++ b/airbyte-integrations/connectors/destination-amazon-sqs/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:destination-amazon-sqs:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/destinations/amazon-sqs) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_amazon_sqs/spec.json` file. @@ -54,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/destination-amazon-sqs:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=destination-amazon-sqs build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:destination-amazon-sqs:airbyteDocker +An image will be built with the tag `airbyte/destination-amazon-sqs:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/destination-amazon-sqs:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,38 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-amazon-sqs:dev ch # messages.jsonl is a file containing line-separated JSON representing AirbyteMessages cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-amazon-sqs:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=destination-amazon-sqs test ``` -#### Acceptance Tests -Coming soon: -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:destination-amazon-sqs:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:destination-amazon-sqs:integrationTest -``` +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -116,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-amazon-sqs test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/amazon-sqs.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-amazon-sqs/bootstrap.md b/airbyte-integrations/connectors/destination-amazon-sqs/bootstrap.md index 37c5e934fad2..ce91ec1ef142 100644 --- a/airbyte-integrations/connectors/destination-amazon-sqs/bootstrap.md +++ b/airbyte-integrations/connectors/destination-amazon-sqs/bootstrap.md @@ -11,7 +11,7 @@ have performance implications if sending high volumes of messages. #### Message Body By default, the SQS Message body is built using the AirbyteMessageRecord's 'data' property. -If the **message_body_key** config item is set, we use the value as a key within the the AirbyteMessageRecord's 'data' property. This could be +If the **message_body_key** config item is set, we use the value as a key within the AirbyteMessageRecord's 'data' property. This could be improved to handle nested keys by using JSONPath syntax to lookup values. For example, given the input Record: @@ -56,4 +56,4 @@ to use as a dedupe ID. ### Credentials Requires an AWS IAM Access Key ID and Secret Key. -This could be improved to add support for configured AWS profiles, env vars etc. \ No newline at end of file +This could be improved to add support for configured AWS profiles, env vars etc. diff --git a/airbyte-integrations/connectors/destination-amazon-sqs/build.gradle b/airbyte-integrations/connectors/destination-amazon-sqs/build.gradle deleted file mode 100644 index c4c35b177b66..000000000000 --- a/airbyte-integrations/connectors/destination-amazon-sqs/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'destination_amazon_sqs' -} diff --git a/airbyte-integrations/connectors/destination-amazon-sqs/destination_amazon_sqs/spec.json b/airbyte-integrations/connectors/destination-amazon-sqs/destination_amazon_sqs/spec.json index ee0e2be338d5..f94d7d023e81 100644 --- a/airbyte-integrations/connectors/destination-amazon-sqs/destination_amazon_sqs/spec.json +++ b/airbyte-integrations/connectors/destination-amazon-sqs/destination_amazon_sqs/spec.json @@ -23,31 +23,39 @@ "description": "AWS Region of the SQS Queue", "type": "string", "enum": [ - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", "af-south-1", "ap-east-1", - "ap-south-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", + "ap-south-1", + "ap-south-2", "ap-southeast-1", "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", "ca-central-1", + "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", + "eu-central-2", "eu-north-1", "eu-south-1", + "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", - "sa-east-1", + "il-central-1", + "me-central-1", "me-south-1", + "sa-east-1", + "us-east-1", + "us-east-2", "us-gov-east-1", - "us-gov-west-1" + "us-gov-west-1", + "us-west-1", + "us-west-2" ], "order": 1 }, diff --git a/airbyte-integrations/connectors/destination-amazon-sqs/metadata.yaml b/airbyte-integrations/connectors/destination-amazon-sqs/metadata.yaml index 832eb8ca9454..8b6fa7635281 100644 --- a/airbyte-integrations/connectors/destination-amazon-sqs/metadata.yaml +++ b/airbyte-integrations/connectors/destination-amazon-sqs/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: destination definitionId: 0eeee7fb-518f-4045-bacc-9619e31c43ea - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 dockerRepository: airbyte/destination-amazon-sqs githubIssueLabel: destination-amazon-sqs icon: awssqs.svg diff --git a/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile b/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile index 5337033a942d..73d0a933e1c5 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile +++ b/airbyte-integrations/connectors/destination-aws-datalake/Dockerfile @@ -1,17 +1,17 @@ -FROM python:3.9-slim -# FROM python:3.9.11-alpine3.15 +FROM python:3.10-slim # Bash is installed for more convenient debugging. # RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* WORKDIR /airbyte/integration_code -COPY destination_aws_datalake ./destination_aws_datalake COPY main.py ./ COPY setup.py ./ RUN pip install . +COPY destination_aws_datalake ./destination_aws_datalake + ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.5 LABEL io.airbyte.name=airbyte/destination-aws-datalake diff --git a/airbyte-integrations/connectors/destination-aws-datalake/README.md b/airbyte-integrations/connectors/destination-aws-datalake/README.md index e4e7f1d858b2..72fe3deb31cf 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/README.md +++ b/airbyte-integrations/connectors/destination-aws-datalake/README.md @@ -34,14 +34,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle - -From the Airbyte repository root, run: - -```bash -./gradlew :airbyte-integrations:connectors:destination-aws-datalake:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/destinations/aws-datalake) @@ -63,23 +55,20 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build - -First, make sure you build the latest Docker image: +#### Build +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** ```bash -docker build . -t airbyte/destination-aws-datalake:dev +airbyte-ci connectors --name=destination-aws-datalake build ``` -You can also build the connector image via Gradle: +An image will be built with the tag `airbyte/destination-aws-datalake:dev`. +**Via `docker build`:** ```bash -./gradlew :airbyte-integrations:connectors:destination-aws-datalake:airbyteDocker +docker build -t airbyte/destination-aws-datalake:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. - #### Run Then run any of the connector commands as follows: @@ -91,86 +80,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-aws-datalake:dev cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-aws-datalake:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing - -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. - -First install test dependencies into your virtual environment: - -```bash -pip install .[tests] -``` - -### Unit Tests - -To run unit tests locally, from the connector directory run: - -```bash -python -m pytest unit_tests -``` - -### Integration Tests - -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). - -#### Custom Integration tests - -Place custom tests inside `integration_tests/` folder, then, from the connector root, run - -```bash -python -m pytest integration_tests -``` - -#### Acceptance Tests - -Coming soon: - -### Using gradle to run tests - -All commands should be run from airbyte project root. -To run unit tests: - -```bash -./gradlew :airbyte-integrations:connectors:destination-aws-datalake:unitTest -``` - -To run acceptance and custom integration tests: +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): ```bash -./gradlew :airbyte-integrations:connectors:destination-aws-datalake:integrationTest -``` - -#### Running the Destination Integration Tests - -To successfully run the Destination Acceptance Tests, you need a `secrets/config.json` file with appropriate information. For example: - -```json -{ - "aws_account_id": "111111111111", - "credentials": { - "credentials_title": "IAM User", - "aws_access_key_id": "aws_key_id", - "aws_secret_access_key": "aws_secret_key" - }, - "region": "us-east-1", - "bucket_name": "datalake-bucket", - "lakeformation_database_name": "test", - "format": { - "format_type": "Parquet", - "compression_codec": "SNAPPY" - }, - "partitioning": "NO PARTITIONING" -} - +airbyte-ci connectors --name=destination-aws-datalake test ``` -In the AWS account, you need to have the following elements in place: - -* An IAM user with appropriate IAM permissions (Notably S3 and Athena) -* A Lake Formation database pointing to the configured S3 location (See: [Creating a database](https://docs.aws.amazon.com/lake-formation/latest/dg/creating-database.html)) -* An Athena workspace named `AmazonAthenaLakeFormation` where the IAM user has proper authorizations -* The user must have appropriate permissions to the Lake Formation database to perform the tests (For example see: [Granting Database Permissions Using the Lake Formation Console and the Named Resource Method](https://docs.aws.amazon.com/lake-formation/latest/dg/granting-database-permissions.html)) - +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management @@ -182,11 +101,12 @@ We split dependencies between two groups, dependencies that are: * required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector - You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-aws-datalake test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/aws-datalake.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/destination-aws-datalake/build.gradle b/airbyte-integrations/connectors/destination-aws-datalake/build.gradle deleted file mode 100644 index 8a47d52ac042..000000000000 --- a/airbyte-integrations/connectors/destination-aws-datalake/build.gradle +++ /dev/null @@ -1,25 +0,0 @@ -plugins { - id 'application' - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-integration-test-java' -} - -airbytePython { - moduleDirectory 'destination_aws_datalake' -} - -dependencies { - // https://mvnrepository.com/artifact/com.google.guava/guava - implementation 'com.google.guava:guava:30.1.1-jre' - - // https://mvnrepository.com/artifact/software.amazon.awssdk/athena - implementation 'software.amazon.awssdk:athena:2.17.42' - - // https://mvnrepository.com/artifact/software.amazon.awssdk/glue - implementation 'software.amazon.awssdk:glue:2.17.42' - - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-aws-datalake') -} diff --git a/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/aws.py b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/aws.py index 1e1a679938d7..8458d01c9e1d 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/aws.py +++ b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/aws.py @@ -4,7 +4,7 @@ import logging from decimal import Decimal -from typing import Dict, Optional +from typing import Any, Dict, Optional import awswrangler as wr import boto3 @@ -15,11 +15,10 @@ from retrying import retry from .config_reader import CompressionCodec, ConnectorConfig, CredentialsType, OutputFormat +from .constants import BOOLEAN_VALUES, EMPTY_VALUES logger = logging.getLogger("airbyte") -null_values = ["", " ", "#N/A", "#N/A N/A", "#NA", "", "N/A", "NA", "NULL", "none", "None", "NaN", "n/a", "nan", "null"] - def _cast_pandas_column(df: pd.DataFrame, col: str, current_type: str, desired_type: str) -> pd.DataFrame: if desired_type == "datetime64": @@ -32,13 +31,13 @@ def _cast_pandas_column(df: pd.DataFrame, col: str, current_type: str, desired_t # First cast to string df = _cast_pandas_column(df=df, col=col, current_type=current_type, desired_type="string") # Then cast to decimal - df[col] = df[col].apply(lambda x: Decimal(str(x)) if str(x) not in null_values else None) + df[col] = df[col].apply(lambda x: Decimal(str(x)) if str(x) not in EMPTY_VALUES else None) elif desired_type.lower() in ["float64", "int64"]: df[col] = df[col].fillna("") df[col] = pd.to_numeric(df[col]) elif desired_type in ["boolean", "bool"]: if df[col].dtype in ["string", "O"]: - df[col] = df[col].fillna("false").apply(lambda x: str(x).lower() in ["true", "1", "1.0", "t", "y", "yes"]) + df[col] = df[col].fillna("false").apply(lambda x: str(x).lower() in BOOLEAN_VALUES) df[col] = df[col].astype(bool) else: @@ -53,7 +52,7 @@ def _cast_pandas_column(df: pd.DataFrame, col: str, current_type: str, desired_t "which may cause precision loss.", UserWarning, ) - df[col] = df[col].apply(lambda x: int(x) if str(x) not in null_values else None).astype(desired_type) + df[col] = df[col].apply(lambda x: int(x) if str(x) not in EMPTY_VALUES else None).astype(desired_type) return df @@ -66,7 +65,7 @@ def _cast_pandas_column(df: pd.DataFrame, col: str, current_type: str, desired_t class AwsHandler: - def __init__(self, connector_config: ConnectorConfig, destination: Destination): + def __init__(self, connector_config: ConnectorConfig, destination: Destination) -> None: self._config: ConnectorConfig = connector_config self._destination: Destination = destination self._session: boto3.Session = None @@ -79,7 +78,7 @@ def __init__(self, connector_config: ConnectorConfig, destination: Destination): self._table_type = "GOVERNED" if self._config.lakeformation_governed_tables else "EXTERNAL_TABLE" @retry(stop_max_attempt_number=10, wait_random_min=1000, wait_random_max=2000) - def create_session(self): + def create_session(self) -> None: if self._config.credentials_type == CredentialsType.IAM_USER: self._session = boto3.Session( aws_access_key_id=self._config.aws_access_key, @@ -108,7 +107,7 @@ def _get_s3_path(self, database: str, table: str) -> str: return f"{bucket}/{database}/{table}/" - def _get_compression_type(self, compression: CompressionCodec): + def _get_compression_type(self, compression: CompressionCodec) -> Optional[str]: if compression == CompressionCodec.GZIP: return "gzip" elif compression == CompressionCodec.SNAPPY: @@ -127,14 +126,16 @@ def _write_parquet( mode: str, dtype: Optional[Dict[str, str]], partition_cols: list = None, - ): + ) -> Any: return wr.s3.to_parquet( df=df, path=path, dataset=True, database=database, table=table, - table_type=self._table_type, + glue_table_settings={ + "table_type": self._table_type, + }, mode=mode, use_threads=False, # True causes s3 NoCredentialsError error catalog_versioning=True, @@ -153,14 +154,16 @@ def _write_json( mode: str, dtype: Optional[Dict[str, str]], partition_cols: list = None, - ): + ) -> Any: return wr.s3.to_json( df=df, path=path, dataset=True, database=database, table=table, - table_type=self._table_type, + glue_table_settings={ + "table_type": self._table_type, + }, mode=mode, use_threads=False, # True causes s3 NoCredentialsError error orient="records", @@ -172,7 +175,9 @@ def _write_json( compression=self._get_compression_type(self._config.compression_codec), ) - def _write(self, df: pd.DataFrame, path: str, database: str, table: str, mode: str, dtype: Dict[str, str], partition_cols: list = None): + def _write( + self, df: pd.DataFrame, path: str, database: str, table: str, mode: str, dtype: Dict[str, str], partition_cols: list = None + ) -> Any: self._create_database_if_not_exists(database) if self._config.format_type == OutputFormat.JSONL: @@ -184,7 +189,7 @@ def _write(self, df: pd.DataFrame, path: str, database: str, table: str, mode: s else: raise Exception(f"Unsupported output format: {self._config.format_type}") - def _create_database_if_not_exists(self, database: str): + def _create_database_if_not_exists(self, database: str) -> None: tag_key = self._config.lakeformation_database_default_tag_key tag_values = self._config.lakeformation_database_default_tag_values diff --git a/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/constants.py b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/constants.py new file mode 100644 index 000000000000..75ff5562c20f --- /dev/null +++ b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/constants.py @@ -0,0 +1,28 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +EMPTY_VALUES = ["", " ", "#N/A", "#N/A N/A", "#NA", "", "N/A", "NA", "NULL", "none", "None", "NaN", "n/a", "nan", "null", "[]", "{}"] +BOOLEAN_VALUES = ["true", "1", "1.0", "t", "y", "yes"] + +PANDAS_TYPE_MAPPING = { + "string": "string", + "integer": "Int64", + "number": "float64", + "boolean": "bool", + "object": "object", + "array": "object", +} + +GLUE_TYPE_MAPPING_DOUBLE = { + "string": "string", + "integer": "bigint", + "number": "double", + "boolean": "boolean", + "null": "string", +} + +GLUE_TYPE_MAPPING_DECIMAL = { + **GLUE_TYPE_MAPPING_DOUBLE, + "number": "decimal(38, 25)", +} diff --git a/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/spec.json b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/spec.json index cdf4a1de08c1..a868e1b3e5f1 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/spec.json +++ b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/spec.json @@ -90,31 +90,39 @@ "description": "The region of the S3 bucket. See here for all region codes.", "enum": [ "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", "af-south-1", "ap-east-1", - "ap-south-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", + "ap-south-1", + "ap-south-2", "ap-southeast-1", "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", "ca-central-1", + "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", + "eu-central-2", "eu-north-1", "eu-south-1", + "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", - "sa-east-1", + "il-central-1", + "me-central-1", "me-south-1", + "sa-east-1", + "us-east-1", + "us-east-2", "us-gov-east-1", - "us-gov-west-1" + "us-gov-west-1", + "us-west-1", + "us-west-2" ], "order": 3 }, diff --git a/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/stream_writer.py b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/stream_writer.py index ed110c35ef57..d7ee96e27688 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/stream_writer.py +++ b/airbyte-integrations/connectors/destination-aws-datalake/destination_aws_datalake/stream_writer.py @@ -4,19 +4,40 @@ import json import logging +from datetime import date, datetime +from decimal import Decimal, getcontext from typing import Any, Dict, List, Optional, Tuple, Union import pandas as pd from airbyte_cdk.models import ConfiguredAirbyteStream, DestinationSyncMode -from destination_aws_datalake.config_reader import ConnectorConfig, PartitionOptions from .aws import AwsHandler +from .config_reader import ConnectorConfig, PartitionOptions +from .constants import EMPTY_VALUES, GLUE_TYPE_MAPPING_DECIMAL, GLUE_TYPE_MAPPING_DOUBLE, PANDAS_TYPE_MAPPING +# By default we set glue decimal type to decimal(28,25) +# this setting matches that precision. +getcontext().prec = 25 logger = logging.getLogger("airbyte") +class DictEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, Decimal): + return str(obj) + + if isinstance(obj, (pd.Timestamp, datetime)): + # all timestamps and datetimes are converted to UTC + return obj.strftime("%Y-%m-%dT%H:%M:%SZ") + + if isinstance(obj, date): + return obj.strftime("%Y-%m-%d") + + return super(DictEncoder, self).default(obj) + + class StreamWriter: - def __init__(self, aws_handler: AwsHandler, config: ConnectorConfig, configured_stream: ConfiguredAirbyteStream): + def __init__(self, aws_handler: AwsHandler, config: ConnectorConfig, configured_stream: ConfiguredAirbyteStream) -> None: self._aws_handler: AwsHandler = aws_handler self._config: ConnectorConfig = config self._configured_stream: ConfiguredAirbyteStream = configured_stream @@ -32,17 +53,18 @@ def __init__(self, aws_handler: AwsHandler, config: ConnectorConfig, configured_ logger.info(f"Creating StreamWriter for {self._database}:{self._table}") - def _get_date_columns(self) -> list: + def _get_date_columns(self) -> List[str]: date_columns = [] for key, val in self._schema.items(): typ = val.get("type") - if (isinstance(typ, str) and typ == "string") or (isinstance(typ, list) and "string" in typ): + typ = self._get_json_schema_type(typ) + if isinstance(typ, str) and typ == "string": if val.get("format") in ["date-time", "date"]: date_columns.append(key) return date_columns - def _add_partition_column(self, col: str, df: pd.DataFrame) -> list: + def _add_partition_column(self, col: str, df: pd.DataFrame) -> Dict[str, str]: partitioning = self._config.partitioning if partitioning == PartitionOptions.NONE: @@ -91,36 +113,63 @@ def _drop_additional_top_level_properties(self, record: Dict[str, Any]) -> Dict[ return record - def _fix_obvious_type_violations(self, record: Dict[str, Any]) -> Dict[str, Any]: + def _json_schema_cast_value(self, value, schema_entry) -> Any: + typ = schema_entry.get("type") + typ = self._get_json_schema_type(typ) + props = schema_entry.get("properties") + items = schema_entry.get("items") + + if typ == "string": + format = schema_entry.get("format") + if format == "date-time": + return pd.to_datetime(value, errors="coerce", utc=True) + + return str(value) if value and value != "" else None + + elif typ == "integer": + return pd.to_numeric(value, errors="coerce") + + elif typ == "number": + if self._config.glue_catalog_float_as_decimal: + return Decimal(str(value)) if value else Decimal("0") + return pd.to_numeric(value, errors="coerce") + + elif typ == "boolean": + return bool(value) + + elif typ == "null": + return None + + elif typ == "object": + if value in EMPTY_VALUES: + return None + + if isinstance(value, dict) and props: + for key, val in value.items(): + if key in props: + value[key] = self._json_schema_cast_value(val, props[key]) + return value + + elif typ == "array" and items: + if value in EMPTY_VALUES: + return None + + if isinstance(value, list): + return [self._json_schema_cast_value(item, items) for item in value] + + return value + + def _json_schema_cast(self, record: Dict[str, Any]) -> Dict[str, Any]: """ Helper that fixes obvious type violations in a record's top level keys that may cause issues when casting data to pyarrow types. Such as: - Objects having empty strings or " " or "-" as value instead of null or {} - Arrays having empty strings or " " or "-" as value instead of null or [] """ - schema_keys = self._schema.keys() - for key in schema_keys: + for key, schema_type in self._schema.items(): typ = self._schema[key].get("type") typ = self._get_json_schema_type(typ) - if typ in ["object", "array"]: - if record.get(key) in ["", " ", "-", "/", "null"]: - record[key] = None - - return record - - def _add_missing_columns(self, record: Dict[str, Any]) -> Dict[str, Any]: - """ - Helper that adds missing columns to a record's top level keys. Required - for awswrangler to create the correct schema in glue, even with the explicit - schema passed in, awswrangler will remove those columns when not present - in the dataframe - """ - schema_keys = self._schema.keys() - records_keys = record.keys() - difference = list(set(schema_keys).difference(set(records_keys))) - - for key in difference: - record[key] = None + record[key] = self._json_schema_cast_value(record.get(key), schema_type) return record @@ -153,15 +202,6 @@ def _get_json_schema_type(self, types: Union[List[str], str]) -> str: return types[0] def _get_pandas_dtypes_from_json_schema(self, df: pd.DataFrame) -> Dict[str, str]: - type_mapper = { - "string": "string", - "integer": "Int64", - "number": "float64", - "boolean": "bool", - "object": "object", - "array": "object", - } - column_types = {} typ = "string" @@ -176,15 +216,16 @@ def _get_pandas_dtypes_from_json_schema(self, df: pd.DataFrame) -> Dict[str, str typ = self._get_json_schema_type(typ) - column_types[col] = type_mapper.get(typ, "string") + column_types[col] = PANDAS_TYPE_MAPPING.get(typ, "string") return column_types - def _get_json_schema_types(self): + def _get_json_schema_types(self) -> Dict[str, str]: types = {} for key, val in self._schema.items(): typ = val.get("type") types[key] = self._get_json_schema_type(typ) + return types def _is_invalid_struct_or_array(self, schema: Dict[str, Any]) -> bool: @@ -243,19 +284,11 @@ def _get_glue_dtypes_from_json_schema(self, schema: Dict[str, Any]) -> Tuple[Dic """ Helper that infers glue dtypes from a json schema. """ - - type_mapper = { - "string": "string", - "integer": "bigint", - "number": "decimal(38, 25)" if self._config.glue_catalog_float_as_decimal else "double", - "boolean": "boolean", - "null": "string", - } + type_mapper = GLUE_TYPE_MAPPING_DECIMAL if self._config.glue_catalog_float_as_decimal else GLUE_TYPE_MAPPING_DOUBLE column_types = {} json_columns = set() - for (col, definition) in schema.items(): - + for col, definition in schema.items(): result_typ = None col_typ = definition.get("type") airbyte_type = definition.get("airbyte_type") @@ -275,7 +308,7 @@ def _get_glue_dtypes_from_json_schema(self, schema: Dict[str, Any]) -> Tuple[Dic if col_typ == "object": properties = definition.get("properties") - allow_additional_properties = definition.get("additionalProperties") + allow_additional_properties = definition.get("additionalProperties", False) if properties and not allow_additional_properties and self._is_invalid_struct_or_array(properties): object_props, _ = self._get_glue_dtypes_from_json_schema(properties) result_typ = f"struct<{','.join([f'{k}:{v}' for k, v in object_props.items()])}>" @@ -336,8 +369,7 @@ def _cursor_fields(self) -> Optional[List[str]]: def append_message(self, message: Dict[str, Any]): clean_message = self._drop_additional_top_level_properties(message) - clean_message = self._fix_obvious_type_violations(clean_message) - clean_message = self._add_missing_columns(clean_message) + clean_message = self._json_schema_cast(clean_message) self._messages.append(clean_message) def reset(self): @@ -362,7 +394,7 @@ def flush(self, partial: bool = False): date_columns = self._get_date_columns() for col in date_columns: if col in df.columns: - df[col] = pd.to_datetime(df[col]) + df[col] = pd.to_datetime(df[col], format="mixed", utc=True) # Create date column for partitioning if self._cursor_fields and col in self._cursor_fields: @@ -378,7 +410,7 @@ def flush(self, partial: bool = False): # so they can be queried with json_extract for col in json_casts: if col in df.columns: - df[col] = df[col].apply(json.dumps) + df[col] = df[col].apply(lambda x: json.dumps(x, cls=DictEncoder)) if self._sync_mode == DestinationSyncMode.overwrite and self._partial_flush_count < 1: logger.debug(f"Overwriting {len(df)} records to {self._database}:{self._table}") diff --git a/airbyte-integrations/connectors/destination-aws-datalake/metadata.yaml b/airbyte-integrations/connectors/destination-aws-datalake/metadata.yaml index baf2cbf1ad98..032954e7f2b9 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/metadata.yaml +++ b/airbyte-integrations/connectors/destination-aws-datalake/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 99878c90-0fbd-46d3-9d98-ffde879d17fc - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.5 dockerRepository: airbyte/destination-aws-datalake githubIssueLabel: destination-aws-datalake icon: awsdatalake.svg diff --git a/airbyte-integrations/connectors/destination-aws-datalake/setup.py b/airbyte-integrations/connectors/destination-aws-datalake/setup.py index 4dccbefb7619..ca8628655464 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/setup.py +++ b/airbyte-integrations/connectors/destination-aws-datalake/setup.py @@ -8,8 +8,8 @@ MAIN_REQUIREMENTS = [ "airbyte-cdk~=0.1", "retrying", - "awswrangler==2.17.0", - "pandas==1.4.4", + "awswrangler==3.3.0", + "pandas==2.0.3", ] TEST_REQUIREMENTS = ["pytest~=6.1"] diff --git a/airbyte-integrations/connectors/destination-aws-datalake/unit_tests/stream_writer_test.py b/airbyte-integrations/connectors/destination-aws-datalake/unit_tests/stream_writer_test.py index b0a7ac314696..e981907cec40 100644 --- a/airbyte-integrations/connectors/destination-aws-datalake/unit_tests/stream_writer_test.py +++ b/airbyte-integrations/connectors/destination-aws-datalake/unit_tests/stream_writer_test.py @@ -4,14 +4,16 @@ import json from datetime import datetime +from decimal import Decimal from typing import Any, Dict, Mapping +import numpy as np import pandas as pd from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode from destination_aws_datalake import DestinationAwsDatalake from destination_aws_datalake.aws import AwsHandler from destination_aws_datalake.config_reader import ConnectorConfig -from destination_aws_datalake.stream_writer import StreamWriter +from destination_aws_datalake.stream_writer import DictEncoder, StreamWriter def get_config() -> Mapping[str, Any]: @@ -196,6 +198,109 @@ def get_big_schema_configured_stream(): ) +def get_camelcase_configured_stream(): + stream_name = "append_camelcase" + stream_schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "properties": { + "TaxRateRef": { + "properties": {"name": {"type": ["null", "string"]}, "value": {"type": ["null", "string"]}}, + "type": ["null", "object"], + }, + "DocNumber": {"type": ["null", "string"]}, + "CurrencyRef": { + "properties": {"name": {"type": ["null", "string"]}, "value": {"type": ["null", "string"]}}, + "type": ["null", "object"], + }, + "Id": {"type": ["null", "string"]}, + "domain": {"type": ["null", "string"]}, + "SyncToken": {"type": ["null", "string"]}, + "Line": { + "items": { + "properties": { + "Id": {"type": ["null", "string"]}, + "Amount": {"type": ["null", "number"]}, + "JournalEntryLineDetail": { + "properties": { + "AccountRef": { + "properties": {"name": {"type": ["null", "string"]}, "value": {"type": ["null", "string"]}}, + "type": ["null", "object"], + }, + "PostingType": {"type": ["null", "string"]}, + }, + "type": ["null", "object"], + }, + "DetailType": {"type": ["null", "string"]}, + "Description": {"type": ["null", "string"]}, + }, + "type": ["null", "object"], + }, + "type": ["null", "array"], + }, + "TxnDate": {"format": "date", "type": ["null", "string"]}, + "TxnTaxDetail": { + "type": ["null", "object"], + "properties": { + "TotalTax": {"type": ["null", "number"]}, + "TxnTaxCodeRef": { + "type": ["null", "object"], + "properties": {"value": {"type": ["null", "string"]}, "name": {"type": ["null", "string"]}}, + }, + "TaxLine": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "DetailType": {"type": ["null", "string"]}, + "Amount": {"type": ["null", "number"]}, + "TaxLineDetail": { + "type": ["null", "object"], + "properties": { + "TaxPercent": {"type": ["null", "number"]}, + "OverrideDeltaAmount": {"type": ["null", "number"]}, + "TaxInclusiveAmount": {"type": ["null", "number"]}, + "PercentBased": {"type": ["null", "boolean"]}, + "NetAmountTaxable": {"type": ["null", "number"]}, + "TaxRateRef": { + "type": ["null", "object"], + "properties": {"name": {"type": ["null", "string"]}, "value": {"type": ["null", "string"]}}, + }, + }, + }, + }, + }, + }, + }, + }, + "PrivateNote": {"type": ["null", "string"]}, + "ExchangeRate": {"type": ["null", "number"]}, + "MetaData": { + "properties": { + "CreateTime": {"format": "date-time", "type": ["null", "string"]}, + "LastUpdatedTime": {"format": "date-time", "type": ["null", "string"]}, + }, + "type": ["null", "object"], + }, + "Adjustment": {"type": ["null", "boolean"]}, + "sparse": {"type": ["null", "boolean"]}, + "airbyte_cursor": {"type": ["null", "string"]}, + }, + } + + return ConfiguredAirbyteStream( + stream=AirbyteStream( + name=stream_name, + json_schema=stream_schema, + default_cursor_field=["airbyte_cursor"], + supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh], + ), + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.append, + cursor_field=["airbyte_cursor"], + ) + + def get_big_schema_writer(config: Dict[str, Any]): connector_config = ConnectorConfig(**config) aws_handler = AwsHandler(connector_config, DestinationAwsDatalake()) @@ -291,6 +396,30 @@ def test_get_glue_dtypes_from_json_schema(): } +def test_get_glue_types_from_json_schema_camel_case(): + connector_config = ConnectorConfig(**get_config()) + aws_handler = AwsHandler(connector_config, DestinationAwsDatalake()) + writer = StreamWriter(aws_handler, connector_config, get_camelcase_configured_stream()) + result, _ = writer._get_glue_dtypes_from_json_schema(writer._schema) + assert result == { + "Adjustment": "boolean", + "CurrencyRef": "struct", + "DocNumber": "string", + "ExchangeRate": "double", + "Id": "string", + "Line": "array,PostingType:string>,DetailType:string,Description:string>>", + "MetaData": "struct", + "PrivateNote": "string", + "SyncToken": "string", + "TaxRateRef": "struct", + "TxnDate": "date", + "TxnTaxDetail": "struct,TaxLine:array>>>>", + "airbyte_cursor": "string", + "domain": "string", + "sparse": "boolean", + } + + def test_has_objects_with_no_properties_good(): writer = get_big_schema_writer(get_config()) assert writer._is_invalid_struct_or_array( @@ -328,3 +457,301 @@ def test_has_objects_with_no_properties_nested_bad(): } } ) + + +def test_json_schema_cast_value(): + writer = get_big_schema_writer(get_config()) + assert ( + writer._json_schema_cast_value( + "test", + { + "type": "string", + }, + ) + == "test" + ) + assert ( + writer._json_schema_cast_value( + "1", + { + "type": "integer", + }, + ) + == 1 + ) + + +def test_json_schema_cast_decimal(): + config = get_config() + config["glue_catalog_float_as_decimal"] = True + connector_config = ConnectorConfig(**config) + aws_handler = AwsHandler(connector_config, DestinationAwsDatalake()) + writer = StreamWriter(aws_handler, connector_config, get_camelcase_configured_stream()) + + assert writer._json_schema_cast( + { + "Adjustment": False, + "domain": "QBO", + "sparse": "true", + "Id": "147491", + "SyncToken": "2", + "MetaData": {"CreateTime": "2023-02-09T10:36:39-08:00", "LastUpdatedTime": "2023-06-15T16:08:39-07:00"}, + "DocNumber": "wt_JE001032", + "TxnDate": "2023-01-13", + "CurrencyRef": {"value": "USD", "name": "United States Dollar"}, + "Line": [ + { + "Id": "0", + "Description": "Payroll 01/13/23", + "Amount": "137973.66", + "DetailType": "JournalEntryLineDetail", + "JournalEntryLineDetail": { + "PostingType": "Debit", + "Entity": {"Type": "Vendor", "EntityRef": {"value": "1", "name": "Test"}}, + "AccountRef": {"value": "234", "name": "Expense"}, + "ClassRef": {"value": "14", "name": "Business"}, + }, + }, + ], + "airbyte_cursor": "2023-06-15T16:08:39-07:00", + } + ) == { + "Adjustment": False, + "CurrencyRef": {"name": "United States Dollar", "value": "USD"}, + "DocNumber": "wt_JE001032", + "Id": "147491", + "ExchangeRate": Decimal("0"), + "Line": [ + { + "Amount": Decimal("137973.66"), + "Description": "Payroll 01/13/23", + "DetailType": "JournalEntryLineDetail", + "Id": "0", + "JournalEntryLineDetail": { + "PostingType": "Debit", + "Entity": {"Type": "Vendor", "EntityRef": {"value": "1", "name": "Test"}}, + "AccountRef": {"value": "234", "name": "Expense"}, + "ClassRef": {"value": "14", "name": "Business"}, + }, + } + ], + "MetaData": { + "CreateTime": pd.to_datetime("2023-02-09T10:36:39-08:00", utc=True), + "LastUpdatedTime": pd.to_datetime("2023-06-15T16:08:39-07:00", utc=True), + }, + "PrivateNote": None, + "SyncToken": "2", + "TxnDate": "2023-01-13", + "TaxRateRef": None, + "TxnTaxDetail": None, + "airbyte_cursor": "2023-06-15T16:08:39-07:00", + "domain": "QBO", + "sparse": True, + } + + +def test_json_schema_cast(): + connector_config = ConnectorConfig(**get_config()) + aws_handler = AwsHandler(connector_config, DestinationAwsDatalake()) + writer = StreamWriter(aws_handler, connector_config, get_camelcase_configured_stream()) + + input = { + "Adjustment": False, + "domain": "QBO", + "sparse": False, + "Id": "147491", + "SyncToken": "2", + "ExchangeRate": "1.33", + "MetaData": {"CreateTime": "2023-02-09T10:36:39-08:00", "LastUpdatedTime": "2023-06-15T16:08:39-07:00"}, + "DocNumber": "wt_JE001032", + "TxnDate": "2023-01-13", + "CurrencyRef": {"value": "USD", "name": "United States Dollar"}, + "Line": [ + { + "Id": "0", + "Description": "Money", + "Amount": "137973.66", + "DetailType": "JournalEntryLineDetail", + "JournalEntryLineDetail": { + "PostingType": "Debit", + "Entity": {"Type": "Vendor", "EntityRef": {"value": "1", "name": "Test"}}, + "AccountRef": {"value": "234", "name": "Expense"}, + "ClassRef": {"value": "14", "name": "Business"}, + }, + }, + ], + "airbyte_cursor": "2023-06-15T16:08:39-07:00", + } + + expected = { + "Adjustment": False, + "ExchangeRate": 1.33, + "CurrencyRef": {"name": "United States Dollar", "value": "USD"}, + "DocNumber": "wt_JE001032", + "Id": "147491", + "Line": [ + { + "Amount": 137973.66, + "Description": "Money", + "DetailType": "JournalEntryLineDetail", + "Id": "0", + "JournalEntryLineDetail": { + "PostingType": "Debit", + "Entity": {"Type": "Vendor", "EntityRef": {"value": "1", "name": "Test"}}, + "AccountRef": {"value": "234", "name": "Expense"}, + "ClassRef": {"value": "14", "name": "Business"}, + }, + } + ], + "MetaData": { + "CreateTime": pd.to_datetime("2023-02-09T10:36:39-08:00", utc=True), + "LastUpdatedTime": pd.to_datetime("2023-06-15T16:08:39-07:00", utc=True), + }, + "PrivateNote": None, + "SyncToken": "2", + "TxnDate": "2023-01-13", + "TaxRateRef": None, + "TxnTaxDetail": None, + "airbyte_cursor": "2023-06-15T16:08:39-07:00", + "domain": "QBO", + "sparse": False, + } + + assert writer._json_schema_cast(input) == expected + + +def test_json_schema_cast_empty_values(): + connector_config = ConnectorConfig(**get_config()) + aws_handler = AwsHandler(connector_config, DestinationAwsDatalake()) + writer = StreamWriter(aws_handler, connector_config, get_camelcase_configured_stream()) + + input = { + "Line": [ + { + "Id": "0", + "Description": "Money", + "Amount": "", + "DetailType": "JournalEntryLineDetail", + "JournalEntryLineDetail": "", + }, + ], + "MetaData": {"CreateTime": "", "LastUpdatedTime": "2023-06-15"}, + } + + expected = { + "Adjustment": False, + "CurrencyRef": None, + "DocNumber": None, + "Id": None, + "Line": [ + { + "Description": "Money", + "DetailType": "JournalEntryLineDetail", + "Id": "0", + "JournalEntryLineDetail": None, + } + ], + "MetaData": {"LastUpdatedTime": pd.to_datetime("2023-06-15", utc=True)}, + "PrivateNote": None, + "SyncToken": None, + "TaxRateRef": None, + "TxnDate": None, + "TxnTaxDetail": None, + "airbyte_cursor": None, + "domain": None, + "sparse": False, + } + + result = writer._json_schema_cast(input) + exchange_rate = result.pop("ExchangeRate") + created_time = result["MetaData"].pop("CreateTime") + line_amount = result["Line"][0].pop("Amount") + + assert result == expected + assert np.isnan(exchange_rate) + assert np.isnan(line_amount) + assert pd.isna(created_time) + + +def test_json_schema_cast_bad_values(): + connector_config = ConnectorConfig(**get_config()) + aws_handler = AwsHandler(connector_config, DestinationAwsDatalake()) + writer = StreamWriter(aws_handler, connector_config, get_camelcase_configured_stream()) + + input = { + "domain": 12, + "sparse": "true", + "Adjustment": 0, + "Line": [ + { + "Id": "0", + "Description": "Money", + "Amount": "hello", + "DetailType": "JournalEntryLineDetail", + "JournalEntryLineDetail": "", + }, + ], + "MetaData": {"CreateTime": "hello", "LastUpdatedTime": "2023-06-15"}, + } + + expected = { + "Adjustment": False, + "CurrencyRef": None, + "DocNumber": None, + "Id": None, + "Line": [ + { + "Description": "Money", + "DetailType": "JournalEntryLineDetail", + "Id": "0", + "JournalEntryLineDetail": None, + } + ], + "MetaData": {"LastUpdatedTime": pd.to_datetime("2023-06-15", utc=True)}, + "PrivateNote": None, + "SyncToken": None, + "TaxRateRef": None, + "TxnDate": None, + "TxnTaxDetail": None, + "airbyte_cursor": None, + "domain": "12", + "sparse": True, + } + + result = writer._json_schema_cast(input) + exchange_rate = result.pop("ExchangeRate") + created_time = result["MetaData"].pop("CreateTime") + line_amount = result["Line"][0].pop("Amount") + + assert result == expected + assert np.isnan(exchange_rate) + assert np.isnan(line_amount) + assert pd.isna(created_time) + + +def test_json_dict_encoder(): + dt = "2023-08-01T23:32:11Z" + dt = pd.to_datetime(dt, utc=True) + + input = { + "boolean": False, + "integer": 1, + "float": 2.0, + "decimal": Decimal("13.232"), + "datetime": dt.to_pydatetime(), + "date": dt.date(), + "timestamp": dt, + "nested": { + "boolean": False, + "datetime": dt.to_pydatetime(), + "very_nested": { + "boolean": False, + "datetime": dt.to_pydatetime(), + }, + }, + } + + assert ( + json.dumps(input, cls=DictEncoder) + == '{"boolean": false, "integer": 1, "float": 2.0, "decimal": "13.232", "datetime": "2023-08-01T23:32:11Z", "date": "2023-08-01", "timestamp": "2023-08-01T23:32:11Z", "nested": {"boolean": false, "datetime": "2023-08-01T23:32:11Z", "very_nested": {"boolean": false, "datetime": "2023-08-01T23:32:11Z"}}}' + ) diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/.dockerignore b/airbyte-integrations/connectors/destination-azure-blob-storage/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/Dockerfile b/airbyte-integrations/connectors/destination-azure-blob-storage/Dockerfile deleted file mode 100644 index b848fec650d4..000000000000 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-azure-blob-storage - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-azure-blob-storage - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.2.0 -LABEL io.airbyte.name=airbyte/destination-azure-blob-storage diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/build.gradle b/airbyte-integrations/connectors/destination-azure-blob-storage/build.gradle index f42e9b1f456e..68dfe0c1cad3 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/build.gradle +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/build.gradle @@ -1,29 +1,33 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.azure_blob_storage.AzureBlobStorageDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:bases:bases-destination-jdbc') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation 'com.azure:azure-storage-blob:12.20.2' implementation 'org.apache.commons:commons-csv:1.4' - testImplementation project(':airbyte-integrations:bases:standard-destination-test') - testImplementation 'org.apache.commons:commons-lang3:3.11' testImplementation "org.testcontainers:junit-jupiter:1.17.5" - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-azure-blob-storage') } diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/metadata.yaml b/airbyte-integrations/connectors/destination-azure-blob-storage/metadata.yaml index 86283b46945f..70b5a54f39bb 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/metadata.yaml +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: file connectorType: destination definitionId: b4c5d105-31fd-4817-96b6-cb923bfc04cb - dockerImageTag: 0.2.0 + dockerImageTag: 0.2.1 dockerRepository: airbyte/destination-azure-blob-storage githubIssueLabel: destination-azure-blob-storage icon: azureblobstorage.svg diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageConnectionChecker.java b/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageConnectionChecker.java index 56043f31fd32..36716365e0b6 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageConnectionChecker.java +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageConnectionChecker.java @@ -10,7 +10,7 @@ import com.azure.storage.blob.specialized.AppendBlobClient; import com.azure.storage.blob.specialized.SpecializedBlobClientBuilder; import com.azure.storage.common.StorageSharedKeyCredential; -import io.airbyte.integrations.destination.jdbc.copy.azure.AzureBlobStorageConfig; +import io.airbyte.cdk.integrations.destination.jdbc.copy.azure.AzureBlobStorageConfig; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.nio.charset.StandardCharsets; diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageConsumer.java b/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageConsumer.java index 35e2c47c61cc..24d681b08ee0 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageConsumer.java +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageConsumer.java @@ -7,8 +7,8 @@ import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.specialized.AppendBlobClient; import com.azure.storage.blob.specialized.SpecializedBlobClientBuilder; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.integrations.destination.azure_blob_storage.writer.AzureBlobStorageWriter; import io.airbyte.integrations.destination.azure_blob_storage.writer.AzureBlobStorageWriterFactory; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageDestination.java b/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageDestination.java index 36f3219e9210..95bc88be2dfa 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageDestination.java +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageDestination.java @@ -5,10 +5,10 @@ package io.airbyte.integrations.destination.azure_blob_storage; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; import io.airbyte.integrations.destination.azure_blob_storage.writer.AzureBlobStorageWriterFactory; import io.airbyte.integrations.destination.azure_blob_storage.writer.ProductionWriterFactory; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/csv/NoFlatteningSheetGenerator.java b/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/csv/NoFlatteningSheetGenerator.java index b4ca5a572940..2a912ef6420e 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/csv/NoFlatteningSheetGenerator.java +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/csv/NoFlatteningSheetGenerator.java @@ -6,8 +6,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; import java.util.Collections; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/csv/RootLevelFlatteningSheetGenerator.java b/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/csv/RootLevelFlatteningSheetGenerator.java index 121f252a622c..b6c4cccff6af 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/csv/RootLevelFlatteningSheetGenerator.java +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/csv/RootLevelFlatteningSheetGenerator.java @@ -6,9 +6,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.integrations.base.JavaBaseConstants; import java.util.LinkedList; import java.util.List; import java.util.stream.Collectors; diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/jsonl/AzureBlobStorageJsonlWriter.java b/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/jsonl/AzureBlobStorageJsonlWriter.java index 9f6beb81187a..82600a6186ec 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/jsonl/AzureBlobStorageJsonlWriter.java +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/src/main/java/io/airbyte/integrations/destination/azure_blob_storage/jsonl/AzureBlobStorageJsonlWriter.java @@ -8,9 +8,9 @@ import com.azure.storage.blob.specialized.SpecializedBlobClientBuilder; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.azure_blob_storage.AzureBlobStorageDestinationConfig; import io.airbyte.integrations.destination.azure_blob_storage.writer.AzureBlobStorageWriter; import io.airbyte.integrations.destination.azure_blob_storage.writer.BaseAzureBlobStorageWriter; @@ -91,7 +91,7 @@ public void write(final UUID id, final AirbyteRecordMessage recordMessage) { printWriter.println(jsonRecord); replicatedBytes += recordSize; } - LOGGER.info("Replicated bytes to destination {}", replicatedBytes); + LOGGER.debug("Replicated bytes to destination {}", replicatedBytes); } @Override diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageCsvDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageCsvDestinationAcceptanceTest.java index 854bf2637171..e33324d7e5f4 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageCsvDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageCsvDestinationAcceptanceTest.java @@ -7,8 +7,8 @@ import com.azure.storage.blob.specialized.AppendBlobClient; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.Reader; diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageDestinationAcceptanceTest.java index e5799a6b9556..703312b49a60 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageDestinationAcceptanceTest.java @@ -13,12 +13,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.io.IOs; import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.nio.file.Path; import java.util.ArrayList; import java.util.HashSet; diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageJsonlDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageJsonlDestinationAcceptanceTest.java index 3fea2e266cbf..0bb73b1190e7 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageJsonlDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobStorageJsonlDestinationAcceptanceTest.java @@ -6,8 +6,8 @@ import com.azure.storage.blob.specialized.AppendBlobClient; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/src/test/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobRecordConsumerTest.java b/airbyte-integrations/connectors/destination-azure-blob-storage/src/test/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobRecordConsumerTest.java index 02c40698f74f..c5696ca63649 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/src/test/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobRecordConsumerTest.java +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/src/test/java/io/airbyte/integrations/destination/azure_blob_storage/AzureBlobRecordConsumerTest.java @@ -4,9 +4,9 @@ package io.airbyte.integrations.destination.azure_blob_storage; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.standardtest.destination.PerStreamStateMessageTest; import io.airbyte.integrations.destination.azure_blob_storage.writer.AzureBlobStorageWriterFactory; -import io.airbyte.integrations.standardtest.destination.PerStreamStateMessageTest; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import java.util.function.Consumer; diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/src/test/java/io/airbyte/integrations/destination/azure_blob_storage/csv/NoFlatteningSheetGeneratorTest.java b/airbyte-integrations/connectors/destination-azure-blob-storage/src/test/java/io/airbyte/integrations/destination/azure_blob_storage/csv/NoFlatteningSheetGeneratorTest.java index a13f798d522c..ba11f02a69e5 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/src/test/java/io/airbyte/integrations/destination/azure_blob_storage/csv/NoFlatteningSheetGeneratorTest.java +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/src/test/java/io/airbyte/integrations/destination/azure_blob_storage/csv/NoFlatteningSheetGeneratorTest.java @@ -9,8 +9,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.commons.jackson.MoreMappers; -import io.airbyte.integrations.base.JavaBaseConstants; import java.util.Collections; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/connectors/destination-azure-blob-storage/src/test/java/io/airbyte/integrations/destination/azure_blob_storage/csv/RootLevelFlatteningSheetGeneratorTest.java b/airbyte-integrations/connectors/destination-azure-blob-storage/src/test/java/io/airbyte/integrations/destination/azure_blob_storage/csv/RootLevelFlatteningSheetGeneratorTest.java index a9223db4313f..f5f96810b5bd 100644 --- a/airbyte-integrations/connectors/destination-azure-blob-storage/src/test/java/io/airbyte/integrations/destination/azure_blob_storage/csv/RootLevelFlatteningSheetGeneratorTest.java +++ b/airbyte-integrations/connectors/destination-azure-blob-storage/src/test/java/io/airbyte/integrations/destination/azure_blob_storage/csv/RootLevelFlatteningSheetGeneratorTest.java @@ -9,8 +9,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.commons.jackson.MoreMappers; -import io.airbyte.integrations.base.JavaBaseConstants; import java.util.Collections; import java.util.List; import org.junit.jupiter.api.BeforeEach; diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/.dockerignore b/airbyte-integrations/connectors/destination-bigquery-denormalized/.dockerignore index 65c7d0ad3e73..e4fbece78752 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/.dockerignore +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/.dockerignore @@ -1,3 +1,3 @@ * !Dockerfile -!build +!build/distributions diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/BOOTSTRAP.md b/airbyte-integrations/connectors/destination-bigquery-denormalized/BOOTSTRAP.md deleted file mode 100644 index edb26b327d2a..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/BOOTSTRAP.md +++ /dev/null @@ -1,5 +0,0 @@ -# BigQuery Denormalized Destination Connector Bootstrap - -Instead of splitting the final data into multiple tables, this destination leverages BigQuery capabilities with [Structured and Repeated fields](https://cloud.google.com/bigquery/docs/nested-repeated) to produce a single "big" table per stream. This does not write the `_airbyte_raw_*` tables in the destination and normalization from this connector is not supported at this time. - -See [this](https://docs.airbyte.io/integrations/destinations/databricks) link for the nuances about the connector. \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile b/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile deleted file mode 100644 index e0706ac44eae..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-bigquery-denormalized - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-bigquery-denormalized - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=1.5.3 -LABEL io.airbyte.name=airbyte/destination-bigquery-denormalized diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/README.md b/airbyte-integrations/connectors/destination-bigquery-denormalized/README.md deleted file mode 100644 index f4256e91594b..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# BigQuery vs BigQuery Denormalized - -This is the same destination as BigQuery but tables are created de-normalized (one big table) without the use of `_airbyte_raw_*` tables containing the JSON blob data. - -# BigQuery Test Configuration - -In order to test the BigQuery destination, you need a service account key file. - -## Community Contributor - -As a community contributor, you will need access to a GCP project and BigQuery to run tests. - -1. Go to the `Service Accounts` page on the GCP console -1. Click on `+ Create Service Account" button -1. Fill out a descriptive name/id/description -1. Click the edit icon next to the service account you created on the `IAM` page -1. Add the `BigQuery Data Editor` and `BigQuery User` role -1. Go back to the `Service Accounts` page and use the actions modal to `Create Key` -1. Download this key as a JSON file -1. Move and rename this file to `secrets/credentials.json` - -## Airbyte Employee - -1. Access the `BigQuery Integration Test User` secret on Rippling under the `Engineering` folder -1. Create a file with the contents at `secrets/credentials.json` diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle b/airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle deleted file mode 100644 index bab42c9c7c4a..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/build.gradle +++ /dev/null @@ -1,42 +0,0 @@ -plugins { - id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' -} - -application { - mainClass = 'io.airbyte.integrations.destination.bigquery.BigQueryDenormalizedDestination' - applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] -} - -dependencies { - implementation 'com.google.cloud:google-cloud-bigquery:1.122.2' - implementation 'org.apache.commons:commons-lang3:3.11' - - implementation project(':airbyte-config-oss:config-models-oss') - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:connectors:destination-bigquery') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java-s3') - implementation project(':airbyte-integrations:connectors:destination-gcs') - implementation group: 'org.apache.parquet', name: 'parquet-avro', version: '1.12.0' - - testImplementation 'org.hamcrest:hamcrest-all:1.3' - testImplementation 'org.mockito:mockito-inline:4.7.0' - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-bigquery-denormalized') - integrationTestJavaImplementation project(':airbyte-db:db-lib') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) -} - -configurations.all { - resolutionStrategy { - // at time of writing: deps.toml declares google-cloud-storage 2.17.2 - // which pulls in google-api-client:2.2.0 - // which conflicts with google-cloud-bigquery, which requires google-api-client:1.x - // google-cloud-storage is OK with downgrading to anything >=1.31.1. - force 'com.google.api-client:google-api-client:1.31.5' - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/metadata.yaml b/airbyte-integrations/connectors/destination-bigquery-denormalized/metadata.yaml index 6ce69ccbc34e..a651fabef37f 100644 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/metadata.yaml +++ b/airbyte-integrations/connectors/destination-bigquery-denormalized/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 079d5540-f236-4294-ba7c-ade8fd918496 - dockerImageTag: 1.5.3 + dockerImageTag: 2.0.0 dockerRepository: airbyte/destination-bigquery-denormalized githubIssueLabel: destination-bigquery-denormalized icon: bigquery.svg @@ -10,10 +10,15 @@ data: name: BigQuery (denormalized typed struct) registries: cloud: - enabled: true + enabled: false oss: - enabled: true - releaseStage: beta + enabled: false + releases: + breakingChanges: + 2.0.0: + message: "`destination-bigquery-denormalized` is being retired in favor of `destination-bigquery`, and is no longer maintained. Please switch to destination-bigquery, which will produce similar tables and contains many improvements. Learn more [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/)." + upgradeDeadline: "2023-11-01" + releaseStage: alpha resourceRequirements: jobSpecific: - jobType: sync diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java deleted file mode 100644 index 5545124a762c..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestination.java +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery; - -import static com.google.cloud.bigquery.Field.Mode.REPEATED; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.cloud.bigquery.Field; -import com.google.cloud.bigquery.Table; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.destination.bigquery.formatter.BigQueryRecordFormatter; -import io.airbyte.integrations.destination.bigquery.formatter.DefaultBigQueryDenormalizedRecordFormatter; -import io.airbyte.integrations.destination.bigquery.formatter.GcsBigQueryDenormalizedRecordFormatter; -import io.airbyte.integrations.destination.bigquery.formatter.arrayformater.LegacyArrayFormatter; -import io.airbyte.integrations.destination.bigquery.uploader.AbstractBigQueryUploader; -import io.airbyte.integrations.destination.bigquery.uploader.BigQueryUploaderFactory; -import io.airbyte.integrations.destination.bigquery.uploader.UploaderType; -import io.airbyte.integrations.destination.bigquery.uploader.config.UploaderConfig; -import io.airbyte.integrations.destination.s3.avro.JsonToAvroSchemaConverter; -import io.airbyte.protocol.models.v0.AirbyteStream; -import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; -import java.io.IOException; -import java.util.Map; -import java.util.function.BiFunction; -import java.util.function.Function; -import javax.annotation.Nullable; -import org.apache.avro.Schema; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class BigQueryDenormalizedDestination extends BigQueryDestination { - - private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryDenormalizedDestination.class); - - @Override - protected String getTargetTableName(final String streamName) { - // This BigQuery destination does not write to a staging "raw" table but directly to a normalized - // table - return namingResolver.getIdentifier(streamName); - } - - @Override - protected Map getFormatterMap(final JsonNode jsonSchema) { - return Map.of(UploaderType.STANDARD, new DefaultBigQueryDenormalizedRecordFormatter(jsonSchema, namingResolver), - UploaderType.AVRO, new GcsBigQueryDenormalizedRecordFormatter(jsonSchema, namingResolver)); - } - - /** - * BigQuery might have different structure of the Temporary table. If this method returns TRUE, - * temporary table will have only three common Airbyte attributes. In case of FALSE, temporary table - * structure will be in line with Airbyte message JsonSchema. - * - * @return use default AirbyteSchema or build using JsonSchema - */ - @Override - protected boolean isDefaultAirbyteTmpTableSchema() { - // Build temporary table structure based on incoming JsonSchema - return false; - } - - @Override - protected BiFunction getAvroSchemaCreator() { - // the json schema needs to be processed by the record former to denormalize - return (formatter, pair) -> new JsonToAvroSchemaConverter().getAvroSchema(formatter.getJsonSchema(), pair.getName(), - pair.getNamespace(), true, false, false, true); - } - - @Override - protected Function getRecordFormatterCreator(final BigQuerySQLNameTransformer namingResolver) { - return streamSchema -> new GcsBigQueryDenormalizedRecordFormatter(streamSchema, namingResolver); - } - - /** - * This BigQuery destination does not write to a staging "raw" table but directly to a normalized - * table. - */ - @Override - protected Function getTargetTableNameTransformer(final BigQuerySQLNameTransformer namingResolver) { - return namingResolver::getIdentifier; - } - - @Override - protected void putStreamIntoUploaderMap(AirbyteStream stream, - UploaderConfig uploaderConfig, - Map> uploaderMap) - throws IOException { - String datasetId = BigQueryUtils.sanitizeDatasetId(uploaderConfig.getConfigStream().getStream().getNamespace()); - Table existingTable = uploaderConfig.getBigQuery().getTable(datasetId, uploaderConfig.getTargetTableName()); - BigQueryRecordFormatter formatter = uploaderConfig.getFormatter(); - - if (existingTable != null) { - LOGGER.info("Target table already exists. Checking could we use the default destination processing."); - if (!compareSchemas((formatter.getBigQuerySchema()), existingTable.getDefinition().getSchema())) { - ((DefaultBigQueryDenormalizedRecordFormatter) formatter).setArrayFormatter(new LegacyArrayFormatter()); - LOGGER.warn("Existing target table has different structure with the new destination processing. Trying legacy implementation."); - } else { - LOGGER.info("Existing target table {} has equal structure with the destination schema. Using the default array processing.", - stream.getName()); - } - } else { - LOGGER.info("Target table is not created yet. The default destination processing will be used."); - } - - AbstractBigQueryUploader uploader = BigQueryUploaderFactory.getUploader(uploaderConfig); - uploaderMap.put( - AirbyteStreamNameNamespacePair.fromAirbyteStream(stream), - uploader); - } - - /** - * Compare calculated bigquery schema and existing schema of the table. Note! We compare only fields - * from the calculated schema to avoid manually created fields in the table. - * - * @param expectedSchema BigQuery schema of the table which we calculated using the stream schema - * config - * @param existingSchema BigQuery schema of the existing table (created by previous run) - * @return Are calculated fields same as we have in the existing table - */ - private boolean compareSchemas(com.google.cloud.bigquery.Schema expectedSchema, @Nullable com.google.cloud.bigquery.Schema existingSchema) { - if (expectedSchema != null && existingSchema == null) { - LOGGER.warn("Existing schema is null when we expect {}", expectedSchema); - return false; - } else if (expectedSchema == null && existingSchema == null) { - LOGGER.info("Existing and expected schemas are null."); - return true; - } else if (expectedSchema == null) { - LOGGER.warn("Expected schema is null when we have existing schema {}", existingSchema); - return false; - } - - var expectedFields = expectedSchema.getFields(); - var existingFields = existingSchema.getFields(); - - for (Field expectedField : expectedFields) { - var existingField = existingFields.get(expectedField.getName()); - if (isDifferenceBetweenFields(expectedField, existingField)) { - LOGGER.warn("Expected field {} is different from existing field {}", expectedField, existingField); - return false; - } - } - - LOGGER.info("Existing and expected schemas are equal."); - return true; - } - - private boolean isDifferenceBetweenFields(Field expectedField, Field existingField) { - if (existingField == null) { - return true; - } else { - return !expectedField.getType().equals(existingField.getType()) - || !compareRepeatedMode(expectedField, existingField) - || !compareSubFields(expectedField, existingField); - } - } - - /** - * Compare field modes. Field can have on of four modes: NULLABLE, REQUIRED, REPEATED, null. Only - * the REPEATED mode difference is critical. The method fails only if at least one is REPEATED and - * the second one is not. - * - * @param expectedField expected field structure - * @param existingField existing field structure - * @return is critical difference in the field modes - */ - private boolean compareRepeatedMode(Field expectedField, Field existingField) { - var expectedMode = expectedField.getMode(); - var existingMode = existingField.getMode(); - - if (expectedMode != null && expectedMode.equals(REPEATED) || existingMode != null && existingMode.equals(REPEATED)) { - return expectedMode != null && expectedMode.equals(existingMode); - } else { - return true; - } - } - - private boolean compareSubFields(Field expectedField, Field existingField) { - var expectedSubFields = expectedField.getSubFields(); - var existingSubFields = existingField.getSubFields(); - - if (expectedSubFields == null || expectedSubFields.isEmpty()) { - return true; - } else if (existingSubFields == null || existingSubFields.isEmpty()) { - return false; - } else { - for (Field expectedSubField : expectedSubFields) { - var existingSubField = existingSubFields.get(expectedSubField.getName()); - if (isDifferenceBetweenFields(expectedSubField, existingSubField)) { - return false; - } - } - return true; - } - } - - public static void main(final String[] args) throws Exception { - final Destination destination = new BigQueryDenormalizedDestination(); - new IntegrationRunner(destination).run(args); - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/JsonSchemaFormat.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/JsonSchemaFormat.java deleted file mode 100644 index b237e389199c..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/JsonSchemaFormat.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery; - -import com.google.cloud.bigquery.StandardSQLTypeName; -import java.util.Arrays; -import java.util.List; -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Mapping of JsonSchema formats to BigQuery Standard SQL types. - */ -public enum JsonSchemaFormat { - - DATE("date", null, StandardSQLTypeName.DATE), - DATETIME("date-time", null, StandardSQLTypeName.DATETIME), - DATETIME_WITH_TZ("date-time", "timestamp_with_timezone", StandardSQLTypeName.TIMESTAMP), - TIME("time", null, StandardSQLTypeName.TIME), - TIMESTAMP("timestamp-micros", null, StandardSQLTypeName.TIMESTAMP); - - private static final Logger LOGGER = LoggerFactory.getLogger(JsonSchemaFormat.class); - private final String jsonSchemaFormat; - private final String jsonSchemaAirbyteType; - private final StandardSQLTypeName bigQueryType; - - JsonSchemaFormat(final String jsonSchemaFormat, final String jsonSchemaAirbyteType, final StandardSQLTypeName bigQueryType) { - this.jsonSchemaAirbyteType = jsonSchemaAirbyteType; - this.jsonSchemaFormat = jsonSchemaFormat; - this.bigQueryType = bigQueryType; - } - - public static JsonSchemaFormat fromJsonSchemaFormat(final @Nonnull String jsonSchemaFormat, final @Nullable String jsonSchemaAirbyteType) { - List matchFormats = null; - // Match by Format + Type - if (jsonSchemaAirbyteType != null) { - matchFormats = Arrays.stream(values()) - .filter(format -> jsonSchemaFormat.equals(format.jsonSchemaFormat) && jsonSchemaAirbyteType.equals(format.jsonSchemaAirbyteType)).toList(); - } - - // Match by Format are no results already - if (matchFormats == null || matchFormats.isEmpty()) { - matchFormats = - Arrays.stream(values()).filter(format -> jsonSchemaFormat.equals(format.jsonSchemaFormat) && format.jsonSchemaAirbyteType == null).toList(); - } - - if (matchFormats.isEmpty()) { - return null; - } else if (matchFormats.size() > 1) { - throw new RuntimeException( - "Match with more than one json format! Matched formats : " + matchFormats + ", Inputs jsonSchemaFormat : " + jsonSchemaFormat - + ", jsonSchemaAirbyteType : " + jsonSchemaAirbyteType); - } else { - return matchFormats.get(0); - } - } - - public StandardSQLTypeName getBigQueryType() { - return bigQueryType; - } - - @Override - public String toString() { - return jsonSchemaFormat; - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/JsonSchemaType.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/JsonSchemaType.java deleted file mode 100644 index fa47e85d6f51..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/JsonSchemaType.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery; - -import com.google.cloud.bigquery.StandardSQLTypeName; - -/** - * Mapping of JsonSchema types to BigQuery Standard SQL types. - * - * The order field of the enum provides us the ability to sort union types (array of JsonSchemaType - * from narrow to wider scopes of types. For example, STRING takes precedence over NUMBER if both - * are included in the same type array. - */ -public enum JsonSchemaType { - - STRING(0, "string", StandardSQLTypeName.STRING), - NUMBER(1, "number", StandardSQLTypeName.FLOAT64), - INTEGER(2, "integer", StandardSQLTypeName.INT64), - BOOLEAN(3, "boolean", StandardSQLTypeName.BOOL), - OBJECT(4, "object", StandardSQLTypeName.STRUCT), - ARRAY(5, "array", StandardSQLTypeName.ARRAY), - NULL(6, "null", null); - - private final int order; - private final String jsonSchemaType; - private final StandardSQLTypeName bigQueryType; - - JsonSchemaType(final int order, final String jsonSchemaType, final StandardSQLTypeName bigQueryType) { - this.order = order; - this.jsonSchemaType = jsonSchemaType; - this.bigQueryType = bigQueryType; - } - - public static JsonSchemaType fromJsonSchemaType(final String value) { - for (final JsonSchemaType type : values()) { - if (value.equals(type.jsonSchemaType)) { - return type; - } - } - throw new IllegalArgumentException("Unexpected json schema type: " + value); - } - - public int getOrder() { - return order; - } - - public String getJsonSchemaType() { - return jsonSchemaType; - } - - public StandardSQLTypeName getBigQueryType() { - return bigQueryType; - } - - @Override - public String toString() { - return jsonSchemaType; - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryDenormalizedRecordFormatter.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryDenormalizedRecordFormatter.java deleted file mode 100644 index 7baeb4cc6857..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryDenormalizedRecordFormatter.java +++ /dev/null @@ -1,376 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery.formatter; - -import static io.airbyte.integrations.destination.bigquery.formatter.util.FormatterUtil.TYPE_FIELD; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectReader; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.TextNode; -import com.google.cloud.bigquery.Field; -import com.google.cloud.bigquery.Field.Builder; -import com.google.cloud.bigquery.Field.Mode; -import com.google.cloud.bigquery.FieldList; -import com.google.cloud.bigquery.LegacySQLTypeName; -import com.google.cloud.bigquery.QueryParameterValue; -import com.google.cloud.bigquery.Schema; -import com.google.cloud.bigquery.StandardSQLTypeName; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.util.MoreIterators; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.bigquery.BigQueryUtils; -import io.airbyte.integrations.destination.bigquery.JsonSchemaFormat; -import io.airbyte.integrations.destination.bigquery.JsonSchemaType; -import io.airbyte.integrations.destination.bigquery.formatter.arrayformater.ArrayFormatter; -import io.airbyte.integrations.destination.bigquery.formatter.arrayformater.DefaultArrayFormatter; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import java.io.IOException; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; -import java.util.stream.Collectors; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class DefaultBigQueryDenormalizedRecordFormatter extends DefaultBigQueryRecordFormatter { - - private static final Logger LOGGER = LoggerFactory.getLogger(DefaultBigQueryDenormalizedRecordFormatter.class); - - public static final String PROPERTIES_FIELD = "properties"; - private static final String ALL_OF_FIELD = "allOf"; - private static final String ANY_OF_FIELD = "anyOf"; - private static final String FORMAT_FIELD = "format"; - private static final String AIRBYTE_TYPE = "airbyte_type"; - private static final String REF_DEFINITION_KEY = "$ref"; - private static final ObjectMapper mapper = new ObjectMapper(); - - protected ArrayFormatter arrayFormatter; - - public DefaultBigQueryDenormalizedRecordFormatter(final JsonNode jsonSchema, final StandardNameTransformer namingResolver) { - super(jsonSchema, namingResolver); - } - - private ArrayFormatter getArrayFormatter() { - if (arrayFormatter == null) { - arrayFormatter = new DefaultArrayFormatter(); - } - return arrayFormatter; - } - - public void setArrayFormatter(final ArrayFormatter arrayFormatter) { - this.arrayFormatter = arrayFormatter; - this.jsonSchema = formatJsonSchema(this.originalJsonSchema.deepCopy()); - this.bigQuerySchema = getBigQuerySchema(jsonSchema); - } - - @Override - protected JsonNode formatJsonSchema(final JsonNode jsonSchema) { - final var modifiedJsonSchema = jsonSchema.deepCopy(); // Issue #5912 is reopened (PR #11166) formatAllOfAndAnyOfFields(namingResolver, - // jsonSchema); - getArrayFormatter().populateEmptyArrays(modifiedJsonSchema); - getArrayFormatter().surroundArraysByObjects(modifiedJsonSchema); - return modifiedJsonSchema; - } - - @Override - public JsonNode formatRecord(final AirbyteRecordMessage recordMessage) { - // Bigquery represents TIMESTAMP to the microsecond precision, so we convert to microseconds then - // use BQ helpers to string-format correctly. - Preconditions.checkArgument(recordMessage.getData().isObject()); - final ObjectNode data = (ObjectNode) formatData(getBigQuerySchema().getFields(), recordMessage.getData()); - // replace ObjectNode with TextNode for fields with $ref definition key - // Do not need to iterate through all JSON Object nodes, only first nesting object. - if (!fieldsContainRefDefinitionValue.isEmpty()) { - fieldsContainRefDefinitionValue.forEach(key -> { - if (data.get(key) != null && !data.get(key).isNull()) { - data.put(key, data.get(key).toString()); - } - }); - } - addAirbyteColumns(data, recordMessage); - - return data; - } - - protected void addAirbyteColumns(final ObjectNode data, final AirbyteRecordMessage recordMessage) { - // currently emittedAt time is in millis format from airbyte message - final long emittedAtMicroseconds = TimeUnit.MICROSECONDS.convert( - recordMessage.getEmittedAt(), TimeUnit.MILLISECONDS); - final String formattedEmittedAt = QueryParameterValue.timestamp(emittedAtMicroseconds).getValue(); - - data.put(JavaBaseConstants.COLUMN_NAME_AB_ID, UUID.randomUUID().toString()); - data.put(JavaBaseConstants.COLUMN_NAME_EMITTED_AT, formattedEmittedAt); - } - - private JsonNode formatData(final FieldList fields, final JsonNode root) { - // handles empty objects and arrays - if (fields == null) { - return root; - } - final JsonNode formattedData; - if (root.isObject()) { - formattedData = getObjectNode(fields, root); - } else if (root.isArray()) { - formattedData = getArrayNode(fields, root); - } else { - formattedData = root; - } - formatDateTimeFields(fields, formattedData); - - return formattedData; - } - - protected void formatDateTimeFields(final FieldList fields, final JsonNode root) { - final List dateTimeFields = BigQueryUtils.getDateTimeFieldsFromSchema(fields); - if (!dateTimeFields.isEmpty() && !root.isNull()) { - if (root.isArray()) { - root.forEach(jsonNode -> BigQueryUtils.transformJsonDateTimeToBigDataFormat(dateTimeFields, jsonNode)); - } else { - BigQueryUtils.transformJsonDateTimeToBigDataFormat(dateTimeFields, root); - } - } - } - - private JsonNode getArrayNode(final FieldList fields, final JsonNode root) { - // Arrays can have only one field - final Field arrayField = fields.get(0); - // If an array of records, we should use subfields - final FieldList subFields; - if (arrayField.getSubFields() == null || arrayField.getSubFields().isEmpty()) { - subFields = fields; - } else { - subFields = arrayField.getSubFields(); - } - final List arrayItems = MoreIterators.toList(root.elements()).stream() - .map(p -> formatData(subFields, p)) - .toList(); - - return getArrayFormatter().formatArrayItems(arrayItems); - } - - private JsonNode getObjectNode(final FieldList fields, final JsonNode root) { - final List fieldNames = fields.stream().map(Field::getName).collect(Collectors.toList()); - - fields.stream() - .filter(f -> f.getType().equals(LegacySQLTypeName.STRING)) - .filter(field -> root.get(field.getName()) != null) - .filter(f -> root.get(f.getName()).isObject()) - .forEach(f -> { - final String value = root.get(f.getName()).toString(); - ((ObjectNode) root).remove(f.getName()); - ((ObjectNode) root).put(f.getName(), new TextNode(value)); - }); - - return Jsons.jsonNode(Jsons.keys(root).stream() - .filter(key -> { - final boolean validKey = fieldNames.contains(namingResolver.getIdentifier(key)); - if (!validKey && !invalidKeys.contains(key)) { - logFieldFail("Ignoring field as it is not defined in catalog", key); - invalidKeys.add(key); - } - return validKey; - }) - .collect(Collectors.toMap(namingResolver::getIdentifier, - key -> formatData(fields.get(namingResolver.getIdentifier(key)).getSubFields(), root.get(key))))); - } - - @Override - public Schema getBigQuerySchema(final JsonNode jsonSchema) { - final List fieldList = getSchemaFields(namingResolver, jsonSchema); - if (fieldList.stream().noneMatch(f -> f.getName().equals(JavaBaseConstants.COLUMN_NAME_AB_ID))) { - fieldList.add(Field.of(JavaBaseConstants.COLUMN_NAME_AB_ID, StandardSQLTypeName.STRING)); - } - if (fieldList.stream().noneMatch(f -> f.getName().equals(JavaBaseConstants.COLUMN_NAME_EMITTED_AT))) { - fieldList.add(Field.of(JavaBaseConstants.COLUMN_NAME_EMITTED_AT, StandardSQLTypeName.TIMESTAMP)); - } - LOGGER.info("Airbyte Schema is transformed from {} to {}.", jsonSchema, fieldList); - return Schema.of(fieldList); - } - - private List getSchemaFields(final StandardNameTransformer namingResolver, final JsonNode jsonSchema) { - LOGGER.info("getSchemaFields : " + jsonSchema + " namingResolver " + namingResolver); - Preconditions.checkArgument(jsonSchema.isObject() && jsonSchema.has(PROPERTIES_FIELD)); - final ObjectNode properties = (ObjectNode) jsonSchema.get(PROPERTIES_FIELD); - final List tmpFields = Jsons.keys(properties).stream() - .peek(addToRefList(properties)) - .map(key -> getField(namingResolver, key, properties.get(key)) - .build()) - .collect(Collectors.toList()); - if (!fieldsContainRefDefinitionValue.isEmpty()) { - LOGGER.warn("Next fields contain \"$ref\" as Definition: {}. They are going to be saved as String Type column", - fieldsContainRefDefinitionValue); - } - return tmpFields; - } - - /** - * @param properties - JSON schema with properties - *

      - * The method is responsible for population of fieldsContainRefDefinitionValue set with keys - * contain $ref definition - *

      - * Currently, AirByte doesn't support parsing value by $ref key definition. The issue to - * track this 7725 - */ - private Consumer addToRefList(final ObjectNode properties) { - return key -> { - if (properties.get(key).has(REF_DEFINITION_KEY)) { - fieldsContainRefDefinitionValue.add(key); - } - }; - } - - private static JsonNode getFileDefinition(final JsonNode fieldDefinition) { - if (fieldDefinition.has(TYPE_FIELD)) { - return fieldDefinition; - } else { - if (fieldDefinition.has(ANY_OF_FIELD) && fieldDefinition.get(ANY_OF_FIELD).isArray()) { - return allOfAndAnyOfFieldProcessing(ANY_OF_FIELD, fieldDefinition); - } - if (fieldDefinition.has(ALL_OF_FIELD) && fieldDefinition.get(ALL_OF_FIELD).isArray()) { - return allOfAndAnyOfFieldProcessing(ALL_OF_FIELD, fieldDefinition); - } - } - return fieldDefinition; - } - - private static JsonNode allOfAndAnyOfFieldProcessing(final String fieldName, final JsonNode fieldDefinition) { - final ObjectReader reader = mapper.readerFor(new TypeReference>() {}); - final List list; - try { - list = reader.readValue(fieldDefinition.get(fieldName)); - } catch (final IOException e) { - throw new IllegalStateException( - String.format("Failed to read and process the following field - %s", fieldDefinition)); - } - final ObjectNode objectNode = mapper.createObjectNode(); - list.forEach(field -> { - objectNode.set("big_query_" + field.get("type").asText(), field); - }); - - return Jsons.jsonNode(ImmutableMap.builder() - .put("type", "object") - .put(PROPERTIES_FIELD, objectNode) - .put("additionalProperties", false) - .build()); - } - - private static Builder getField(final StandardNameTransformer namingResolver, final String key, final JsonNode fieldDefinition) { - final String fieldName = namingResolver.getIdentifier(key); - final Builder builder = Field.newBuilder(fieldName, StandardSQLTypeName.STRING); - final JsonNode updatedFileDefinition = getFileDefinition(fieldDefinition); - final JsonNode type = updatedFileDefinition.get(TYPE_FIELD); - final JsonNode airbyteType = updatedFileDefinition.get(AIRBYTE_TYPE); - final List fieldTypes = getTypes(fieldName, type); - for (int i = 0; i < fieldTypes.size(); i++) { - final JsonSchemaType fieldType = fieldTypes.get(i); - if (fieldType == JsonSchemaType.NULL) { - builder.setMode(Mode.NULLABLE); - } - if (i == 0) { - // Treat the first type in the list with the widest scope as the primary type - final JsonSchemaType primaryType = fieldTypes.get(i); - switch (primaryType) { - case NULL -> { - builder.setType(StandardSQLTypeName.STRING); - } - case STRING, INTEGER, BOOLEAN -> { - builder.setType(primaryType.getBigQueryType()); - } - case NUMBER -> { - if (airbyteType != null - && StringUtils.equalsAnyIgnoreCase(airbyteType.asText(), - "big_integer", "integer")) { - builder.setType(StandardSQLTypeName.INT64); - } else { - builder.setType(primaryType.getBigQueryType()); - } - } - case ARRAY -> { - final JsonNode items; - if (updatedFileDefinition.has("items")) { - items = updatedFileDefinition.get("items"); - } else { - LOGGER.warn("Source connector provided schema for ARRAY with missed \"items\", will assume that it's a String type"); - // this is handler for case when we get "array" without "items" - // (https://github.com/airbytehq/airbyte/issues/5486) - items = getTypeStringSchema(); - } - return getField(namingResolver, fieldName, items).setMode(Mode.REPEATED); - } - case OBJECT -> { - final JsonNode properties; - if (updatedFileDefinition.has(PROPERTIES_FIELD)) { - properties = updatedFileDefinition.get(PROPERTIES_FIELD); - } else { - properties = updatedFileDefinition; - } - final FieldList fieldList = FieldList.of(Jsons.keys(properties) - .stream() - .map(f -> getField(namingResolver, f, properties.get(f)).build()) - .collect(Collectors.toList())); - if (!fieldList.isEmpty()) { - builder.setType(StandardSQLTypeName.STRUCT, fieldList); - } else { - builder.setType(StandardSQLTypeName.STRING); - } - } - default -> { - throw new IllegalStateException( - String.format("Unexpected type for field %s: %s", fieldName, primaryType)); - } - } - } - } - - // If a specific format is defined, use their specific type instead of the JSON's one - final JsonNode fieldFormat = updatedFileDefinition.get(FORMAT_FIELD); - if (fieldFormat != null) { - final JsonSchemaFormat schemaFormat = JsonSchemaFormat.fromJsonSchemaFormat(fieldFormat.asText(), - (airbyteType != null ? airbyteType.asText() : null)); - if (schemaFormat != null) { - builder.setType(schemaFormat.getBigQueryType()); - } - } - - return builder; - } - - private static JsonNode getTypeStringSchema() { - return Jsons.deserialize("{\n" - + " \"type\": [\n" - + " \"string\"\n" - + " ]\n" - + " }"); - } - - private static List getTypes(final String fieldName, final JsonNode type) { - if (type == null) { - LOGGER.warn("Field {} has no type defined, defaulting to STRING", fieldName); - return List.of(JsonSchemaType.STRING); - } else if (type.isArray()) { - return MoreIterators.toList(type.elements()).stream() - .map(s -> JsonSchemaType.fromJsonSchemaType(s.asText())) - // re-order depending to make sure wider scope types are first - .sorted(Comparator.comparingInt(JsonSchemaType::getOrder)) - .collect(Collectors.toList()); - } else if (type.isTextual()) { - return Collections.singletonList(JsonSchemaType.fromJsonSchemaType(type.asText())); - } else { - throw new IllegalStateException("Unexpected type: " + type); - } - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/GcsBigQueryDenormalizedRecordFormatter.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/GcsBigQueryDenormalizedRecordFormatter.java deleted file mode 100644 index 2cbefde3bfb9..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/GcsBigQueryDenormalizedRecordFormatter.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery.formatter; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.cloud.bigquery.Schema; -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - -public class GcsBigQueryDenormalizedRecordFormatter extends DefaultBigQueryDenormalizedRecordFormatter { - - public GcsBigQueryDenormalizedRecordFormatter( - final JsonNode jsonSchema, - final StandardNameTransformer namingResolver) { - super(jsonSchema, namingResolver); - } - - @Override - protected JsonNode formatJsonSchema(final JsonNode jsonSchema) { - var textJson = Jsons.serialize(jsonSchema); - textJson = textJson.replace("{\"$ref\":\"", "{\"type\":[\"string\"], \"$ref\":\""); - return super.formatJsonSchema(Jsons.deserialize(textJson)); - } - - @Override - public Schema getBigQuerySchema(final JsonNode jsonSchema) { - final String schemaString = Jsons.serialize(jsonSchema) - // BigQuery avro file loader doesn't support date-time - // https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-avro#logical_types - // So we use timestamp for date-time - .replace("\"format\":\"date-time\"", "\"format\":\"timestamp-micros\""); - final JsonNode bigQuerySchema = Jsons.deserialize(schemaString); - return super.getBigQuerySchema(bigQuerySchema); - } - - @Override - protected void addAirbyteColumns(final ObjectNode data, final AirbyteRecordMessage recordMessage) { - final long emittedAtMicroseconds = TimeUnit.MILLISECONDS.convert(recordMessage.getEmittedAt(), TimeUnit.MILLISECONDS); - - data.put(JavaBaseConstants.COLUMN_NAME_AB_ID, UUID.randomUUID().toString()); - data.put(JavaBaseConstants.COLUMN_NAME_EMITTED_AT, emittedAtMicroseconds); - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/arrayformater/ArrayFormatter.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/arrayformater/ArrayFormatter.java deleted file mode 100644 index 85baf123349d..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/arrayformater/ArrayFormatter.java +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery.formatter.arrayformater; - -import com.fasterxml.jackson.databind.JsonNode; -import java.util.List; - -public interface ArrayFormatter { - - void populateEmptyArrays(final JsonNode node); - - void surroundArraysByObjects(final JsonNode node); - - JsonNode formatArrayItems(final List arrayItems); - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/arrayformater/DefaultArrayFormatter.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/arrayformater/DefaultArrayFormatter.java deleted file mode 100644 index 6d5eea2018b3..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/arrayformater/DefaultArrayFormatter.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery.formatter.arrayformater; - -import static io.airbyte.integrations.destination.bigquery.formatter.DefaultBigQueryDenormalizedRecordFormatter.PROPERTIES_FIELD; -import static io.airbyte.integrations.destination.bigquery.formatter.util.FormatterUtil.ARRAY_ITEMS_FIELD; -import static io.airbyte.integrations.destination.bigquery.formatter.util.FormatterUtil.NESTED_ARRAY_FIELD; -import static io.airbyte.integrations.destination.bigquery.formatter.util.FormatterUtil.TYPE_FIELD; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.bigquery.formatter.util.FormatterUtil; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - -public class DefaultArrayFormatter implements ArrayFormatter { - - @Override - public void populateEmptyArrays(final JsonNode node) { - findArrays(node).forEach(jsonNode -> { - if (!jsonNode.has(ARRAY_ITEMS_FIELD)) { - final ObjectNode nodeToChange = (ObjectNode) jsonNode; - nodeToChange.putObject(ARRAY_ITEMS_FIELD).putArray(TYPE_FIELD).add("string"); - } - }); - } - - @Override - public void surroundArraysByObjects(final JsonNode node) { - findArrays(node).forEach( - jsonNode -> { - if (FormatterUtil.isAirbyteArray(jsonNode.get(ARRAY_ITEMS_FIELD))) { - final ObjectNode arrayNode = jsonNode.get(ARRAY_ITEMS_FIELD).deepCopy(); - final ObjectNode originalNode = (ObjectNode) jsonNode; - - originalNode.remove(ARRAY_ITEMS_FIELD); - final ObjectNode itemsNode = originalNode.putObject(ARRAY_ITEMS_FIELD); - itemsNode.putArray(TYPE_FIELD).add("object"); - itemsNode.putObject(PROPERTIES_FIELD).putObject(NESTED_ARRAY_FIELD).setAll(arrayNode); - - surroundArraysByObjects(originalNode.get(ARRAY_ITEMS_FIELD)); - } - }); - } - - @Override - public JsonNode formatArrayItems(List arrayItems) { - return Jsons - .jsonNode(arrayItems.stream().map(node -> (node.isArray() ? Jsons.jsonNode(ImmutableMap.of(NESTED_ARRAY_FIELD, node)) : node)).toList()); - } - - protected List findArrays(final JsonNode node) { - if (node != null) { - return node.findParents(TYPE_FIELD).stream() - .filter(FormatterUtil::isAirbyteArray) - .collect(Collectors.toList()); - } else { - return Collections.emptyList(); - } - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/arrayformater/LegacyArrayFormatter.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/arrayformater/LegacyArrayFormatter.java deleted file mode 100644 index d99f8d88ab79..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/arrayformater/LegacyArrayFormatter.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery.formatter.arrayformater; - -import static io.airbyte.integrations.destination.bigquery.formatter.DefaultBigQueryDenormalizedRecordFormatter.PROPERTIES_FIELD; -import static io.airbyte.integrations.destination.bigquery.formatter.util.FormatterUtil.ARRAY_ITEMS_FIELD; -import static io.airbyte.integrations.destination.bigquery.formatter.util.FormatterUtil.NESTED_ARRAY_FIELD; -import static io.airbyte.integrations.destination.bigquery.formatter.util.FormatterUtil.TYPE_FIELD; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.bigquery.formatter.util.FormatterUtil; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - -public class LegacyArrayFormatter extends DefaultArrayFormatter { - - @Override - public void surroundArraysByObjects(final JsonNode node) { - findArrays(node).forEach( - jsonNode -> { - final JsonNode arrayNode = jsonNode.deepCopy(); - - final ObjectNode newNode = (ObjectNode) jsonNode; - newNode.removeAll(); - newNode.putArray(TYPE_FIELD).add("object"); - newNode.putObject(PROPERTIES_FIELD).set(NESTED_ARRAY_FIELD, arrayNode); - - surroundArraysByObjects(arrayNode.get(ARRAY_ITEMS_FIELD)); - }); - } - - @Override - protected List findArrays(final JsonNode node) { - if (node != null) { - return node.findParents(TYPE_FIELD).stream() - .filter(FormatterUtil::isAirbyteArray) - .collect(Collectors.toList()); - } else { - return Collections.emptyList(); - } - } - - @Override - public JsonNode formatArrayItems(List arrayItems) { - return Jsons.jsonNode(ImmutableMap.of(NESTED_ARRAY_FIELD, arrayItems)); - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/util/FormatterUtil.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/util/FormatterUtil.java deleted file mode 100644 index ca7bb2bfeb88..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/util/FormatterUtil.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery.formatter.util; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; - -public class FormatterUtil { - - public static final String NESTED_ARRAY_FIELD = "big_query_array"; - public static final String ARRAY_ITEMS_FIELD = "items"; - public static final String TYPE_FIELD = "type"; - - public static boolean isAirbyteArray(final JsonNode jsonSchemaNode) { - if (jsonSchemaNode == null || jsonSchemaNode.get("type") == null) { - return false; - } - final JsonNode type = jsonSchemaNode.get("type"); - if (type.isArray()) { - final ArrayNode typeNode = (ArrayNode) type; - for (final JsonNode arrayTypeNode : typeNode) { - if (arrayTypeNode.isTextual() && arrayTypeNode.textValue().equals("array")) { - return true; - } - } - } else if (type.isTextual()) { - return jsonSchemaNode.asText().equals("array"); - } - return false; - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/resources/spec.json deleted file mode 100644 index 74e8f48ba31b..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/main/resources/spec.json +++ /dev/null @@ -1,207 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/destinations/bigquery", - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["overwrite", "append"], - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "BigQuery Denormalized Typed Struct Destination Spec", - "type": "object", - "required": ["project_id", "dataset_id"], - "additionalProperties": true, - "properties": { - "project_id": { - "type": "string", - "description": "The GCP project ID for the project containing the target BigQuery dataset. Read more here.", - "title": "Project ID", - "order": 0 - }, - "dataset_id": { - "type": "string", - "description": "The default BigQuery Dataset ID that tables are replicated to if the source does not specify a namespace. Read more here.", - "title": "Default Dataset ID", - "order": 1 - }, - "loading_method": { - "type": "object", - "title": "Loading Method", - "description": "Loading method used to send select the way data will be uploaded to BigQuery.
      Standard Inserts - Direct uploading using SQL INSERT statements. This method is extremely inefficient and provided only for quick testing. In almost all cases, you should use staging.
      GCS Staging - Writes large batches of records to a file, uploads the file to GCS, then uses COPY INTO table to upload the file. Recommended for most workloads for better speed and scalability. Read more about GCS Staging here.", - "order": 2, - "oneOf": [ - { - "title": "Standard Inserts", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "Standard" - } - } - }, - { - "title": "GCS Staging", - "type": "object", - "required": [ - "method", - "gcs_bucket_name", - "gcs_bucket_path", - "credential" - ], - "properties": { - "method": { - "type": "string", - "const": "GCS Staging", - "order": 0 - }, - "credential": { - "title": "Credential", - "description": "An HMAC key is a type of credential and can be associated with a service account or a user account in Cloud Storage. Read more here.", - "type": "object", - "order": 1, - "oneOf": [ - { - "title": "HMAC key", - "order": 0, - "required": [ - "credential_type", - "hmac_key_access_id", - "hmac_key_secret" - ], - "properties": { - "credential_type": { - "type": "string", - "const": "HMAC_KEY", - "order": 0 - }, - "hmac_key_access_id": { - "type": "string", - "description": "HMAC key access ID. When linked to a service account, this ID is 61 characters long; when linked to a user account, it is 24 characters long.", - "title": "HMAC Key Access ID", - "airbyte_secret": true, - "examples": ["1234567890abcdefghij1234"], - "order": 1 - }, - "hmac_key_secret": { - "type": "string", - "description": "The corresponding secret for the access ID. It is a 40-character base-64 encoded string.", - "title": "HMAC Key Secret", - "airbyte_secret": true, - "examples": [ - "1234567890abcdefghij1234567890ABCDEFGHIJ" - ], - "order": 2 - } - } - } - ] - }, - "gcs_bucket_name": { - "title": "GCS Bucket Name", - "type": "string", - "description": "The name of the GCS bucket. Read more here.", - "examples": ["airbyte_sync"], - "order": 2 - }, - "gcs_bucket_path": { - "title": "GCS Bucket Path", - "description": "Directory under the GCS bucket where data will be written. Read more here.", - "type": "string", - "examples": ["data_sync/test"], - "order": 3 - }, - "keep_files_in_gcs-bucket": { - "type": "string", - "description": "This upload method is supposed to temporary store records in GCS bucket. By this select you can chose if these records should be removed from GCS when migration has finished. The default \"Delete all tmp files from GCS\" value is used if not set explicitly.", - "title": "GCS Tmp Files Afterward Processing", - "default": "Delete all tmp files from GCS", - "enum": [ - "Delete all tmp files from GCS", - "Keep all tmp files in GCS" - ], - "order": 4 - }, - "file_buffer_count": { - "title": "File Buffer Count", - "type": "integer", - "minimum": 10, - "maximum": 50, - "default": 10, - "description": "Number of file buffers allocated for writing data. Increasing this number is beneficial for connections using Change Data Capture (CDC) and up to the number of streams within a connection. Increasing the number of file buffers past the maximum number of streams has deteriorating effects", - "examples": ["10"], - "order": 5 - } - } - } - ] - }, - "credentials_json": { - "type": "string", - "description": "The contents of the JSON service account key. Check out the docs if you need help generating this key. Default credentials will be used if this field is left empty.", - "title": "Service Account Key JSON (Required for cloud, optional for open-source)", - "airbyte_secret": true, - "order": 3, - "always_show": true - }, - "dataset_location": { - "type": "string", - "description": "The location of the dataset. Warning: Changes made after creation will not be applied. The default \"US\" value is used if not set explicitly. Read more here.", - "title": "Dataset Location", - "default": "US", - "order": 4, - "enum": [ - "US", - "EU", - "asia-east1", - "asia-east2", - "asia-northeast1", - "asia-northeast2", - "asia-northeast3", - "asia-south1", - "asia-south2", - "asia-southeast1", - "asia-southeast2", - "australia-southeast1", - "australia-southeast2", - "europe-central1", - "europe-central2", - "europe-north1", - "europe-southwest1", - "europe-west1", - "europe-west2", - "europe-west3", - "europe-west4", - "europe-west6", - "europe-west7", - "europe-west8", - "europe-west9", - "me-west1", - "northamerica-northeast1", - "northamerica-northeast2", - "southamerica-east1", - "southamerica-west1", - "us-central1", - "us-east1", - "us-east2", - "us-east3", - "us-east4", - "us-east5", - "us-west1", - "us-west2", - "us-west3", - "us-west4" - ] - }, - "big_query_client_buffer_size_mb": { - "title": "Google BigQuery Client Chunk Size", - "description": "Google BigQuery client's chunk (buffer) size (MIN=1, MAX = 15) for each table. The size that will be written by a single RPC. Written data will be buffered and only flushed upon reaching this size or closing the channel. The default 15MB value is used if not set explicitly. Read more here.", - "type": "integer", - "minimum": 1, - "maximum": 15, - "default": 15, - "examples": ["15"], - "order": 5 - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestinationAcceptanceTest.java deleted file mode 100644 index 7a1abff0188a..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestinationAcceptanceTest.java +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery; - -import static io.airbyte.integrations.destination.bigquery.BigQueryDenormalizedTestConstants.AIRBYTE_COLUMNS; -import static io.airbyte.integrations.destination.bigquery.BigQueryDenormalizedTestConstants.CONFIG_PROJECT_ID; -import static io.airbyte.integrations.destination.bigquery.BigQueryDenormalizedTestConstants.NAME_TRANSFORMER; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.configureBigQuery; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.createCommonConfig; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getBigQueryDataSet; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.tearDownBigQuery; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.ConnectionProperty; -import com.google.cloud.bigquery.Dataset; -import com.google.cloud.bigquery.Field; -import com.google.cloud.bigquery.FieldList; -import com.google.cloud.bigquery.FieldValue; -import com.google.cloud.bigquery.FieldValueList; -import com.google.cloud.bigquery.Job; -import com.google.cloud.bigquery.JobId; -import com.google.cloud.bigquery.JobInfo; -import com.google.cloud.bigquery.QueryJobConfiguration; -import com.google.cloud.bigquery.TableResult; -import com.google.common.collect.Streams; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.db.bigquery.BigQueryResultSet; -import io.airbyte.db.bigquery.BigQuerySourceOperations; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.argproviders.DataArgumentsProvider; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; -import io.airbyte.protocol.models.v0.AirbyteCatalog; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import java.io.IOException; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.TimeZone; -import java.util.UUID; -import java.util.stream.Collectors; -import org.apache.commons.lang3.tuple.ImmutablePair; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ArgumentsSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class BigQueryDenormalizedDestinationAcceptanceTest extends DestinationAcceptanceTest { - - private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryDenormalizedDestinationAcceptanceTest.class); - - private BigQuery bigquery; - private Dataset dataset; - protected JsonNode config; - private final StandardNameTransformer namingResolver = new StandardNameTransformer(); - - @Override - protected String getImageName() { - return "airbyte/destination-bigquery-denormalized:dev"; - } - - @Override - protected JsonNode getConfig() { - return config; - } - - @Override - protected JsonNode getFailCheckConfig() { - ((ObjectNode) config).put(CONFIG_PROJECT_ID, "fake"); - return config; - } - - @Override - protected boolean implementsNamespaces() { - return true; - } - - @Override - protected boolean supportNamespaceTest() { - return true; - } - - @Override - protected Optional getNameTransformer() { - return Optional.of(NAME_TRANSFORMER); - } - - @Override - protected TestDataComparator getTestDataComparator() { - return new BigQueryDenormalizedTestDataComparator(); - } - - @Override - protected boolean supportBasicDataTypeTest() { - return true; - } - - // #13154 Normalization issue - @Override - protected boolean supportArrayDataTypeTest() { - return true; - } - - @Override - protected boolean supportObjectDataTypeTest() { - return true; - } - - @Override - protected void assertNamespaceNormalization(final String testCaseId, - final String expectedNormalizedNamespace, - final String actualNormalizedNamespace) { - final String message = String.format("Test case %s failed; if this is expected, please override assertNamespaceNormalization", testCaseId); - if (testCaseId.equals("S3A-1")) { - // bigquery allows namespace starting with a number, and prepending underscore - // will hide the dataset, so we don't do it as we do for other destinations - assertEquals("99namespace", actualNormalizedNamespace, message); - } else { - assertEquals(expectedNormalizedNamespace, actualNormalizedNamespace, message); - } - } - - @Override - protected String getDefaultSchema(final JsonNode config) { - return BigQueryUtils.getDatasetId(config); - } - - @Override - protected List retrieveNormalizedRecords(final TestDestinationEnv testEnv, final String streamName, final String namespace) - throws Exception { - final String tableName = namingResolver.getIdentifier(streamName); - final String schema = namingResolver.getIdentifier(namespace); - return retrieveRecordsFromTable(tableName, schema); - } - - @Override - protected List retrieveRecords(final TestDestinationEnv env, - final String streamName, - final String namespace, - final JsonNode streamSchema) - throws Exception { - final String tableName = namingResolver.getIdentifier(streamName); - final String schema = namingResolver.getIdentifier(namespace); - return retrieveRecordsFromTable(tableName, schema); - } - - private List retrieveRecordsFromTable(final String tableName, final String schema) throws InterruptedException { - TimeZone.setDefault(TimeZone.getTimeZone("UTC")); - - final QueryJobConfiguration queryConfig = - QueryJobConfiguration - .newBuilder( - String.format("SELECT * FROM `%s`.`%s` order by %s asc;", schema, tableName, - JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) - // .setUseLegacySql(false) - .setConnectionProperties(Collections.singletonList(ConnectionProperty.of("time_zone", "UTC"))) - .build(); - - final TableResult queryResults = executeQuery(bigquery, queryConfig).getLeft().getQueryResults(); - final FieldList fields = queryResults.getSchema().getFields(); - final BigQuerySourceOperations sourceOperations = new BigQuerySourceOperations(); - - return Streams.stream(queryResults.iterateAll()) - .map(fieldValues -> sourceOperations.rowToJson(new BigQueryResultSet(fieldValues, fields))).collect(Collectors.toList()); - } - - private boolean isAirbyteColumn(final String name) { - if (AIRBYTE_COLUMNS.contains(name)) { - return true; - } - return name.startsWith("_airbyte") && name.endsWith("_hashid"); - } - - private Object getTypedFieldValue(final FieldValueList row, final Field field) { - final FieldValue fieldValue = row.get(field.getName()); - if (fieldValue.getValue() != null) { - return switch (field.getType().getStandardType()) { - case FLOAT64, NUMERIC -> fieldValue.getDoubleValue(); - case INT64 -> fieldValue.getNumericValue().intValue(); - case STRING -> fieldValue.getStringValue(); - case BOOL -> fieldValue.getBooleanValue(); - case STRUCT -> fieldValue.getRecordValue().toString(); - default -> fieldValue.getValue(); - }; - } else { - return null; - } - } - - protected JsonNode createConfig() throws IOException { - return createCommonConfig(); - } - - @Override - protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws Exception { - config = createConfig(); - bigquery = configureBigQuery(config); - dataset = getBigQueryDataSet(config, bigquery); - } - - @Override - protected void tearDown(final TestDestinationEnv testEnv) { - tearDownBigQuery(dataset, bigquery); - } - - // todo (cgardens) - figure out how to share these helpers. they are currently copied from - // BigQueryDestination. - private static ImmutablePair executeQuery(final BigQuery bigquery, final QueryJobConfiguration queryConfig) { - final JobId jobId = JobId.of(UUID.randomUUID().toString()); - final Job queryJob = bigquery.create(JobInfo.newBuilder(queryConfig).setJobId(jobId).build()); - return executeQuery(queryJob); - } - - private static ImmutablePair executeQuery(final Job queryJob) { - final Job completedJob = waitForQuery(queryJob); - if (completedJob == null) { - throw new RuntimeException("Job no longer exists"); - } else if (completedJob.getStatus().getError() != null) { - // You can also look at queryJob.getStatus().getExecutionErrors() for all - // errors, not just the latest one. - return ImmutablePair.of(null, (completedJob.getStatus().getError().toString())); - } - - return ImmutablePair.of(completedJob, null); - } - - private static Job waitForQuery(final Job queryJob) { - try { - return queryJob.waitFor(); - } catch (final Exception e) { - throw new RuntimeException(e); - } - } - - /** - * Verify that the integration successfully writes normalized records successfully (without actually - * running the normalization module) Tests a wide variety of messages an schemas (aspirationally, - * anyway). - */ - @ParameterizedTest - @ArgumentsSource(DataArgumentsProvider.class) - public void testSyncNormalizedWithoutNormalization(final String messagesFilename, final String catalogFilename) throws Exception { - final AirbyteCatalog catalog = Jsons.deserialize(MoreResources.readResource(catalogFilename), AirbyteCatalog.class); - final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog(catalog); - final List messages = MoreResources.readResource(messagesFilename).lines() - .map(record -> Jsons.deserialize(record, AirbyteMessage.class)).collect(Collectors.toList()); - - final JsonNode config = getConfig(); - // don't run normalization though - runSyncAndVerifyStateOutput(config, messages, configuredCatalog, false); - - final String defaultSchema = getDefaultSchema(config); - final List actualMessages = retrieveNormalizedRecords(catalog, defaultSchema); - assertSameMessages(messages, actualMessages, true); - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestinationTest.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestinationTest.java deleted file mode 100644 index f1adeb47ff26..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestinationTest.java +++ /dev/null @@ -1,378 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery; - -import static io.airbyte.integrations.destination.bigquery.BigQueryDenormalizedTestConstants.AIRBYTE_COLUMNS; -import static io.airbyte.integrations.destination.bigquery.BigQueryDenormalizedTestConstants.BIGQUERY_DATETIME_FORMAT; -import static io.airbyte.integrations.destination.bigquery.BigQueryDenormalizedTestConstants.USERS_STREAM_NAME; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.configureBigQuery; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.createCommonConfig; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getAnyOfFormats; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getAnyOfFormatsWithEmptyList; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getAnyOfFormatsWithNull; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getAnyOfSchema; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getBigQueryDataSet; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getCommonCatalog; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getData; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getDataArrays; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getDataMaxNestedDepth; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getDataTooDeepNestedDepth; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getDataWithEmptyObjectAndArray; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getDataWithFormats; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getDataWithJSONDateTimeFormats; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getDataWithJSONWithReference; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getDataWithNestedDatetimeInsideNullObject; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getExpectedDataArrays; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getSchema; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getSchemaArrays; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getSchemaMaxNestedDepth; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getSchemaTooDeepNestedDepth; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getSchemaWithDateTime; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getSchemaWithFormats; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getSchemaWithInvalidArrayType; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getSchemaWithNestedDatetimeInsideNullObject; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.getSchemaWithReferenceDefinition; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.runDestinationWrite; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.tearDownBigQuery; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.params.provider.Arguments.arguments; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.Dataset; -import com.google.cloud.bigquery.Field; -import com.google.cloud.bigquery.QueryJobConfiguration; -import com.google.cloud.bigquery.Schema; -import com.google.cloud.bigquery.StandardSQLTypeName; -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import java.io.IOException; -import java.time.Instant; -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; -import org.assertj.core.util.Sets; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInfo; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -class BigQueryDenormalizedDestinationTest { - - private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryDenormalizedDestinationTest.class); - protected static final Instant NOW = Instant.now(); - protected static final AirbyteMessage MESSAGE_USERS1 = createRecordMessage(USERS_STREAM_NAME, getData()); - protected static final AirbyteMessage MESSAGE_USERS2 = createRecordMessage(USERS_STREAM_NAME, getDataWithEmptyObjectAndArray()); - protected static final AirbyteMessage MESSAGE_USERS3 = createRecordMessage(USERS_STREAM_NAME, getDataWithFormats()); - protected static final AirbyteMessage MESSAGE_USERS4 = createRecordMessage(USERS_STREAM_NAME, getDataWithJSONDateTimeFormats()); - protected static final AirbyteMessage MESSAGE_USERS5 = createRecordMessage(USERS_STREAM_NAME, getDataWithJSONWithReference()); - protected static final AirbyteMessage MESSAGE_USERS6 = createRecordMessage(USERS_STREAM_NAME, Jsons.deserialize("{\"users\":null}")); - protected static final AirbyteMessage MESSAGE_USERS7 = createRecordMessage(USERS_STREAM_NAME, getDataWithNestedDatetimeInsideNullObject()); - protected static final AirbyteMessage MESSAGE_USERS8 = createRecordMessage(USERS_STREAM_NAME, getAnyOfFormats()); - protected static final AirbyteMessage MESSAGE_USERS9 = createRecordMessage(USERS_STREAM_NAME, getAnyOfFormatsWithNull()); - protected static final AirbyteMessage MESSAGE_USERS10 = createRecordMessage(USERS_STREAM_NAME, getAnyOfFormatsWithEmptyList()); - protected static final AirbyteMessage MESSAGE_USERS11 = createRecordMessage(USERS_STREAM_NAME, getDataArrays()); - protected static final AirbyteMessage MESSAGE_USERS12 = createRecordMessage(USERS_STREAM_NAME, getDataTooDeepNestedDepth()); - protected static final AirbyteMessage MESSAGE_USERS13 = createRecordMessage(USERS_STREAM_NAME, getDataMaxNestedDepth()); - protected static final AirbyteMessage EMPTY_MESSAGE = createRecordMessage(USERS_STREAM_NAME, Jsons.deserialize("{}")); - - protected JsonNode config; - protected BigQuery bigquery; - protected Dataset dataset; - protected String datasetId; - - protected JsonNode createConfig() throws IOException { - return createCommonConfig(); - } - - @BeforeEach - void setup(final TestInfo info) throws IOException { - if (info.getDisplayName().equals("testSpec()")) { - return; - } - - config = createConfig(); - bigquery = configureBigQuery(config); - dataset = getBigQueryDataSet(config, bigquery); - datasetId = dataset.getDatasetId().getDataset(); - - MESSAGE_USERS1.getRecord().setNamespace(datasetId); - MESSAGE_USERS2.getRecord().setNamespace(datasetId); - MESSAGE_USERS3.getRecord().setNamespace(datasetId); - MESSAGE_USERS4.getRecord().setNamespace(datasetId); - MESSAGE_USERS5.getRecord().setNamespace(datasetId); - MESSAGE_USERS6.getRecord().setNamespace(datasetId); - MESSAGE_USERS7.getRecord().setNamespace(datasetId); - MESSAGE_USERS8.getRecord().setNamespace(datasetId); - MESSAGE_USERS9.getRecord().setNamespace(datasetId); - MESSAGE_USERS10.getRecord().setNamespace(datasetId); - MESSAGE_USERS11.getRecord().setNamespace(datasetId); - MESSAGE_USERS12.getRecord().setNamespace(datasetId); - MESSAGE_USERS13.getRecord().setNamespace(datasetId); - EMPTY_MESSAGE.getRecord().setNamespace(datasetId); - - } - - @AfterEach - void tearDown(final TestInfo info) { - if (info.getDisplayName().equals("testSpec()")) { - return; - } - - tearDownBigQuery(dataset, bigquery); - } - - @ParameterizedTest - @MethodSource("schemaAndDataProvider") - void testNestedWrite(final JsonNode schema, final AirbyteMessage message) throws Exception { - runDestinationWrite(getCommonCatalog(schema, datasetId), config, message); - - final List usersActual = retrieveRecordsAsJson(USERS_STREAM_NAME); - final JsonNode expectedUsersJson = message.getRecord().getData(); - assertEquals(usersActual.size(), 1); - final JsonNode resultJson = usersActual.get(0); - assertEquals(extractJsonValues(resultJson, "name"), extractJsonValues(expectedUsersJson, "name")); - assertEquals(extractJsonValues(resultJson, "grants"), extractJsonValues(expectedUsersJson, "grants")); - assertEquals(extractJsonValues(resultJson, "domain"), extractJsonValues(expectedUsersJson, "domain")); - } - - @Test - void testNestedDataTimeInsideNullObject() throws Exception { - runDestinationWrite(getCommonCatalog(getSchemaWithNestedDatetimeInsideNullObject(), datasetId), config, MESSAGE_USERS7); - - final List usersActual = retrieveRecordsAsJson(USERS_STREAM_NAME); - final JsonNode expectedUsersJson = MESSAGE_USERS7.getRecord().getData(); - assertEquals(usersActual.size(), 1); - final JsonNode resultJson = usersActual.get(0); - assertEquals(extractJsonValues(resultJson, "name"), extractJsonValues(expectedUsersJson, "name")); - assertEquals(extractJsonValues(resultJson, "appointment"), extractJsonValues(expectedUsersJson, "appointment")); - } - - protected Schema getExpectedSchemaForWriteWithFormatTest() { - return Schema.of( - Field.of("name", StandardSQLTypeName.STRING), - Field.of("date_of_birth", StandardSQLTypeName.DATE), - Field.of("updated_at", StandardSQLTypeName.DATETIME), - Field.of(JavaBaseConstants.COLUMN_NAME_AB_ID, StandardSQLTypeName.STRING), - Field.of(JavaBaseConstants.COLUMN_NAME_EMITTED_AT, StandardSQLTypeName.TIMESTAMP)); - } - - @Test - void testWriteWithFormat() throws Exception { - runDestinationWrite(getCommonCatalog(getSchemaWithFormats(), datasetId), config, MESSAGE_USERS3); - - final List usersActual = retrieveRecordsAsJson(USERS_STREAM_NAME); - final JsonNode expectedUsersJson = MESSAGE_USERS3.getRecord().getData(); - assertEquals(usersActual.size(), 1); - final JsonNode resultJson = usersActual.get(0); - assertEquals(extractJsonValues(resultJson, "name"), extractJsonValues(expectedUsersJson, "name")); - assertEquals(extractJsonValues(resultJson, "date_of_birth"), extractJsonValues(expectedUsersJson, "date_of_birth")); - - // Bigquery's datetime type accepts multiple input format but always outputs the same, so we can't - // expect to receive the value we sent. - var expectedValue = LocalDate.parse(extractJsonValues(expectedUsersJson, "updated_at").stream().findFirst().get(), - DateTimeFormatter.ofPattern(BIGQUERY_DATETIME_FORMAT)); - var actualValue = - LocalDate.parse(extractJsonValues(resultJson, "updated_at").stream().findFirst().get(), - DateTimeFormatter.ofPattern(BIGQUERY_DATETIME_FORMAT)); - assertEquals(expectedValue, actualValue); - - assertEquals(BigQueryUtils.getTableDefinition(bigquery, datasetId, USERS_STREAM_NAME).getSchema(), getExpectedSchemaForWriteWithFormatTest()); - } - - @Test - @Disabled // Issue #5912 is reopened - void testAnyOf() throws Exception { - runDestinationWrite(getCommonCatalog(getAnyOfSchema(), datasetId), config, MESSAGE_USERS8); - - final List usersActual = retrieveRecordsAsJson(USERS_STREAM_NAME); - final JsonNode expectedUsersJson = MESSAGE_USERS8.getRecord().getData(); - assertEquals(usersActual.size(), 1); - final JsonNode resultJson = usersActual.get(0); - assertEquals(extractJsonValues(resultJson, "id"), extractJsonValues(expectedUsersJson, "id")); - assertEquals(extractJsonValues(resultJson, "name"), extractJsonValues(expectedUsersJson, "name")); - assertEquals(extractJsonValues(resultJson, "type"), extractJsonValues(expectedUsersJson, "type")); - assertEquals(extractJsonValues(resultJson, "email"), extractJsonValues(expectedUsersJson, "email")); - assertEquals(extractJsonValues(resultJson, "avatar"), extractJsonValues(expectedUsersJson, "avatar")); - assertEquals(extractJsonValues(resultJson, "team_ids"), extractJsonValues(expectedUsersJson, "team_ids")); - assertEquals(extractJsonValues(resultJson, "admin_ids"), extractJsonValues(expectedUsersJson, "admin_ids")); - assertEquals(extractJsonValues(resultJson, "all_of_field"), extractJsonValues(expectedUsersJson, "all_of_field")); - assertEquals(extractJsonValues(resultJson, "job_title"), extractJsonValues(expectedUsersJson, "job_title")); - assertEquals(extractJsonValues(resultJson, "has_inbox_seat"), extractJsonValues(expectedUsersJson, "has_inbox_seat")); - assertEquals(extractJsonValues(resultJson, "away_mode_enabled"), extractJsonValues(expectedUsersJson, "away_mode_enabled")); - assertEquals(extractJsonValues(resultJson, "away_mode_reassign"), extractJsonValues(expectedUsersJson, "away_mode_reassign")); - } - - @Test - @Disabled // Issue #5912 is reopened - void testAnyOfWithNull() throws Exception { - runDestinationWrite(getCommonCatalog(getAnyOfSchema(), datasetId), config, MESSAGE_USERS9); - - final List usersActual = retrieveRecordsAsJson(USERS_STREAM_NAME); - final JsonNode expectedUsersJson = MESSAGE_USERS9.getRecord().getData(); - assertEquals(usersActual.size(), 1); - final JsonNode resultJson = usersActual.get(0); - assertEquals(extractJsonValues(resultJson, "name"), extractJsonValues(expectedUsersJson, "name")); - assertEquals(extractJsonValues(resultJson, "team_ids"), extractJsonValues(expectedUsersJson, "team_ids")); - assertEquals(extractJsonValues(resultJson, "all_of_field"), extractJsonValues(expectedUsersJson, "all_of_field")); - assertEquals(extractJsonValues(resultJson, "avatar"), extractJsonValues(expectedUsersJson, "avatar")); - } - - @Test - @Disabled // Issue #5912 is reopened - void testAnyOfWithEmptyList() throws Exception { - runDestinationWrite(getCommonCatalog(getAnyOfSchema(), datasetId), config, MESSAGE_USERS10); - - final List usersActual = retrieveRecordsAsJson(USERS_STREAM_NAME); - final JsonNode expectedUsersJson = MESSAGE_USERS10.getRecord().getData(); - assertEquals(usersActual.size(), 1); - final JsonNode resultJson = usersActual.get(0); - assertEquals(extractJsonValues(resultJson, "name"), extractJsonValues(expectedUsersJson, "name")); - assertEquals(extractJsonValues(resultJson, "team_ids"), extractJsonValues(expectedUsersJson, "team_ids")); - assertEquals(extractJsonValues(resultJson, "all_of_field"), extractJsonValues(expectedUsersJson, "all_of_field")); - } - - @Test - void testIfJSONDateTimeWasConvertedToBigQueryFormat() throws Exception { - runDestinationWrite(getCommonCatalog(getSchemaWithDateTime(), datasetId), config, MESSAGE_USERS4); - - final List usersActual = retrieveRecordsAsJson(USERS_STREAM_NAME); - assertEquals(usersActual.size(), 1); - final JsonNode resultJson = usersActual.get(0); - - // BigQuery Accepts "YYYY-MM-DD HH:MM:SS[.SSSSSS]" format - Set actualValues = extractJsonValues(resultJson, "updated_at"); - assertEquals(Set.of(new DateTime("2021-10-11T06:36:53+00:00").withZone(DateTimeZone.UTC).toString(BIGQUERY_DATETIME_FORMAT)), - actualValues); - - // check nested datetime - actualValues = extractJsonValues(resultJson.get("items"), "nested_datetime"); - assertEquals(Set.of(new DateTime("2021-11-11T06:36:53+00:00").withZone(DateTimeZone.UTC).toString(BIGQUERY_DATETIME_FORMAT)), - actualValues); - } - - @Test - void testJsonReferenceDefinition() throws Exception { - runDestinationWrite(getCommonCatalog(getSchemaWithReferenceDefinition(), datasetId), config, MESSAGE_USERS5, MESSAGE_USERS6, EMPTY_MESSAGE); - - final Set actual = - retrieveRecordsAsJson(USERS_STREAM_NAME).stream().flatMap(x -> extractJsonValues(x, "users").stream()).collect(Collectors.toSet()); - - final Set expected = Sets.set( - "\"{\\\"name\\\":\\\"John\\\",\\\"surname\\\":\\\"Adams\\\"}\"", - null // we expect one record to have not had the users field set - ); - - assertEquals(2, actual.size()); - assertEquals(expected, actual); - } - - @Test - void testArrays() throws Exception { - runDestinationWrite(getCommonCatalog(getSchemaArrays(), datasetId), config, MESSAGE_USERS11); - - assertEquals(getExpectedDataArrays(), retrieveRecordsAsJson(USERS_STREAM_NAME).get(0)); - } - - // Issue #14668 - @Test - void testTooDeepNestedDepth() { - try { - runDestinationWrite(getCommonCatalog(getSchemaTooDeepNestedDepth(), datasetId), config, MESSAGE_USERS12); - } catch (Exception e) { - assert (e.getCause().getMessage().contains("nested too deeply")); - } - } - - // Issue #14668 - @Test - void testMaxNestedDepth() throws Exception { - runDestinationWrite(getCommonCatalog(getSchemaMaxNestedDepth(), datasetId), config, MESSAGE_USERS13); - - assertEquals(getDataMaxNestedDepth().findValue("str_value").asText(), - retrieveRecordsAsJson(USERS_STREAM_NAME).get(0).findValue("str_value").asText()); - } - - private Set extractJsonValues(final JsonNode node, final String attributeName) { - final List valuesNode = node.findValues(attributeName); - final Set resultSet = new HashSet<>(); - valuesNode.forEach(jsonNode -> { - if (jsonNode.isArray()) { - jsonNode.forEach(arrayNodeValue -> resultSet.add(arrayNodeValue.textValue())); - } else if (jsonNode.isObject()) { - resultSet.addAll(extractJsonValues(jsonNode, "big_query_array")); - } else { - resultSet.add(jsonNode.textValue()); - } - }); - - return resultSet; - } - - private JsonNode removeAirbyteMetadataFields(final JsonNode record) { - for (final String airbyteMetadataField : AIRBYTE_COLUMNS) { - ((ObjectNode) record).remove(airbyteMetadataField); - } - return record; - } - - private List retrieveRecordsAsJson(final String tableName) throws Exception { - final QueryJobConfiguration queryConfig = - QueryJobConfiguration - .newBuilder( - String.format("select TO_JSON_STRING(t) as jsonValue from %s.%s t;", datasetId, tableName.toLowerCase())) - .setUseLegacySql(false).build(); - BigQueryUtils.executeQuery(bigquery, queryConfig); - - var valuesStream = StreamSupport - .stream(BigQueryUtils.executeQuery(bigquery, queryConfig).getLeft().getQueryResults().iterateAll().spliterator(), false) - .map(v -> v.get("jsonValue").getStringValue()); - return formatDateValues(valuesStream) - .map(Jsons::deserialize) - .map(this::removeAirbyteMetadataFields) - .collect(Collectors.toList()); - } - - /** - * BigQuery returns date values in a different format based on the column type. Datetime : - * YYYY-MM-DD'T'HH:MM:SS Timestamp : YYYY-MM-DD'T'HH:MM:SS'Z' - * - * This method formats all values as Airbite format to simplify test result validation. - */ - private Stream formatDateValues(Stream values) { - return values.map(s -> s.replaceAll("(\"\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2})(Z)(\")", "$1$3")); - } - - private static Stream schemaAndDataProvider() { - return Stream.of( - arguments(getSchema(), MESSAGE_USERS1), - arguments(getSchemaWithInvalidArrayType(), MESSAGE_USERS1), - arguments(getSchema(), MESSAGE_USERS2)); - } - - private static AirbyteMessage createRecordMessage(final String stream, final JsonNode data) { - return new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(stream) - .withData(data) - .withEmittedAt(NOW.toEpochMilli())); - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedGcsDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedGcsDestinationAcceptanceTest.java deleted file mode 100644 index 148331d8e92f..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedGcsDestinationAcceptanceTest.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery; - -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.createGcsConfig; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.record_buffer.FileBuffer; -import java.io.IOException; -import org.junit.jupiter.api.Test; - -public class BigQueryDenormalizedGcsDestinationAcceptanceTest extends BigQueryDenormalizedDestinationAcceptanceTest { - - @Override - protected JsonNode createConfig() throws IOException { - return createGcsConfig(); - } - - /* - * FileBuffer Default Tests - */ - @Test - public void testGetFileBufferDefault() { - final BigQueryDenormalizedDestination destination = new BigQueryDenormalizedDestination(); - assertEquals(destination.getNumberOfFileBuffers(config), - FileBuffer.DEFAULT_MAX_CONCURRENT_STREAM_IN_BUFFER); - } - - @Test - public void testGetFileBufferMaxLimited() { - final JsonNode defaultConfig = Jsons.clone(config); - ((ObjectNode) defaultConfig.get(BigQueryConsts.LOADING_METHOD)).put(FileBuffer.FILE_BUFFER_COUNT_KEY, 100); - final BigQueryDenormalizedDestination destination = new BigQueryDenormalizedDestination(); - assertEquals(FileBuffer.MAX_CONCURRENT_STREAM_IN_BUFFER, destination.getNumberOfFileBuffers(defaultConfig)); - } - - @Test - public void testGetMinimumFileBufferCount() { - final JsonNode defaultConfig = Jsons.clone(config); - ((ObjectNode) defaultConfig.get(BigQueryConsts.LOADING_METHOD)).put(FileBuffer.FILE_BUFFER_COUNT_KEY, 1); - final BigQueryDenormalizedDestination destination = new BigQueryDenormalizedDestination(); - // User cannot set number of file counts below the default file buffer count, which is existing - // behavior - assertEquals(FileBuffer.DEFAULT_MAX_CONCURRENT_STREAM_IN_BUFFER, destination.getNumberOfFileBuffers(defaultConfig)); - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedGcsDestinationTest.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedGcsDestinationTest.java deleted file mode 100644 index 180317737b63..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedGcsDestinationTest.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery; - -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestDataUtils.createGcsConfig; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.cloud.bigquery.Field; -import com.google.cloud.bigquery.Schema; -import com.google.cloud.bigquery.StandardSQLTypeName; -import io.airbyte.integrations.base.JavaBaseConstants; -import java.io.IOException; - -class BigQueryDenormalizedGcsDestinationTest extends BigQueryDenormalizedDestinationTest { - - @Override - protected JsonNode createConfig() throws IOException { - return createGcsConfig(); - } - - @Override - protected Schema getExpectedSchemaForWriteWithFormatTest() { - return Schema.of( - Field.of("name", StandardSQLTypeName.STRING), - Field.of("date_of_birth", StandardSQLTypeName.DATE), - Field.of("updated_at", StandardSQLTypeName.TIMESTAMP), - Field.of(JavaBaseConstants.COLUMN_NAME_AB_ID, StandardSQLTypeName.STRING), - Field.of(JavaBaseConstants.COLUMN_NAME_EMITTED_AT, StandardSQLTypeName.TIMESTAMP)); - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedTestConstants.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedTestConstants.java deleted file mode 100644 index a5a53efc058f..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedTestConstants.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery; - -import io.airbyte.integrations.base.JavaBaseConstants; -import java.nio.file.Path; -import java.util.List; - -public class BigQueryDenormalizedTestConstants { - - public static final BigQuerySQLNameTransformer NAME_TRANSFORMER = new BigQuerySQLNameTransformer(); - public static final Path CREDENTIALS_PATH = Path.of("secrets/credentials.json"); - public static final String CONFIG_DATASET_ID = "dataset_id"; - public static final String CONFIG_PROJECT_ID = "project_id"; - public static final String CONFIG_DATASET_LOCATION = "dataset_location"; - public static final String CONFIG_CREDS = "credentials_json"; - public static final List AIRBYTE_COLUMNS = List.of(JavaBaseConstants.COLUMN_NAME_AB_ID, JavaBaseConstants.COLUMN_NAME_EMITTED_AT); - public static final String USERS_STREAM_NAME = "users"; - - public static final String BIGQUERY_DATETIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss"; - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedTestDataComparator.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedTestDataComparator.java deleted file mode 100644 index cc7b6a2e50f9..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedTestDataComparator.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery; - -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.ZoneOffset; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.List; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class BigQueryDenormalizedTestDataComparator extends AdvancedTestDataComparator { - - private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryDenormalizedTestDataComparator.class); - private static final String BIGQUERY_DATETIME_FORMAT = "yyyy-MM-dd'T'HH:mm:ss'Z'"; - - private final StandardNameTransformer namingResolver = new StandardNameTransformer(); - - @Override - protected List resolveIdentifier(final String identifier) { - final List result = new ArrayList<>(); - result.add(identifier); - result.add(namingResolver.getIdentifier(identifier)); - return result; - } - - private LocalDate parseDate(String dateValue) { - if (dateValue != null) { - var format = (dateValue.matches(".+Z") ? BIGQUERY_DATETIME_FORMAT : AIRBYTE_DATE_FORMAT); - return LocalDate.parse(dateValue, DateTimeFormatter.ofPattern(format)); - } else { - return null; - } - } - - private LocalDateTime parseDateTime(String dateTimeValue) { - if (dateTimeValue != null) { - var format = (dateTimeValue.matches(".+Z") ? BIGQUERY_DATETIME_FORMAT : AIRBYTE_DATETIME_FORMAT); - return LocalDateTime.parse(dateTimeValue, DateTimeFormatter.ofPattern(format)); - } else { - return null; - } - } - - @Override - protected boolean compareDateTimeValues(String expectedValue, String actualValue) { - var destinationDate = parseDateTime(actualValue); - var expectedDate = LocalDateTime.parse(expectedValue, DateTimeFormatter.ofPattern(AIRBYTE_DATETIME_FORMAT)); - if (expectedDate.isBefore(getBrokenDate().toLocalDateTime())) { - LOGGER - .warn("Validation is skipped due to known Normalization issue. Values older then 1583 year and with time zone stored wrongly(lose days)."); - return true; - } else { - return expectedDate.equals(destinationDate); - } - } - - @Override - protected boolean compareDateValues(String expectedValue, String actualValue) { - var destinationDate = parseDate(actualValue); - var expectedDate = LocalDate.parse(expectedValue, DateTimeFormatter.ofPattern(AIRBYTE_DATE_FORMAT)); - return expectedDate.equals(destinationDate); - } - - @Override - protected ZonedDateTime parseDestinationDateWithTz(String destinationValue) { - return ZonedDateTime.of(LocalDateTime.parse(destinationValue, DateTimeFormatter.ofPattern(BIGQUERY_DATETIME_FORMAT)), ZoneOffset.UTC); - } - - @Override - protected boolean compareDateTimeWithTzValues(String airbyteMessageValue, String destinationValue) { - // #13123 Normalization issue - if (parseDestinationDateWithTz(destinationValue).isBefore(getBrokenDate())) { - LOGGER - .warn("Validation is skipped due to known Normalization issue. Values older then 1583 year and with time zone stored wrongly(lose days)."); - return true; - } else { - return super.compareDateTimeWithTzValues(airbyteMessageValue, destinationValue); - } - } - - // #13123 Normalization issue - private ZonedDateTime getBrokenDate() { - return ZonedDateTime.of(1583, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC); - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/util/BigQueryDenormalizedTestDataUtils.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/util/BigQueryDenormalizedTestDataUtils.java deleted file mode 100644 index c1dfa9230b87..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/java/io/airbyte/integrations/destination/bigquery/util/BigQueryDenormalizedTestDataUtils.java +++ /dev/null @@ -1,280 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery.util; - -import static io.airbyte.integrations.destination.bigquery.BigQueryDenormalizedTestConstants.CONFIG_CREDS; -import static io.airbyte.integrations.destination.bigquery.BigQueryDenormalizedTestConstants.CONFIG_DATASET_ID; -import static io.airbyte.integrations.destination.bigquery.BigQueryDenormalizedTestConstants.CONFIG_DATASET_LOCATION; -import static io.airbyte.integrations.destination.bigquery.BigQueryDenormalizedTestConstants.CONFIG_PROJECT_ID; -import static io.airbyte.integrations.destination.bigquery.BigQueryDenormalizedTestConstants.CREDENTIALS_PATH; -import static io.airbyte.integrations.destination.bigquery.BigQueryDenormalizedTestConstants.USERS_STREAM_NAME; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.auth.oauth2.ServiceAccountCredentials; -import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.BigQueryOptions; -import com.google.cloud.bigquery.Dataset; -import com.google.cloud.bigquery.DatasetInfo; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.string.Strings; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.destination.bigquery.BigQueryConsts; -import io.airbyte.integrations.destination.bigquery.BigQueryDenormalizedDestination; -import io.airbyte.integrations.destination.bigquery.BigQueryDenormalizedTestConstants; -import io.airbyte.integrations.destination.bigquery.BigQueryDestination; -import io.airbyte.integrations.destination.bigquery.BigQueryUtils; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteStream; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.SyncMode; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashSet; -import java.util.Set; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class BigQueryDenormalizedTestDataUtils { - - private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryDenormalizedTestDataUtils.class); - - private static final String JSON_FILES_BASE_LOCATION = "testdata/"; - - public static JsonNode getSchema() { - return getTestDataFromResourceJson("schema.json"); - } - - public static JsonNode getAnyOfSchema() { - return getTestDataFromResourceJson("schemaAnyOfAllOf.json"); - } - - public static JsonNode getSchemaWithFormats() { - return getTestDataFromResourceJson("schemaWithFormats.json"); - } - - public static JsonNode getSchemaWithDateTime() { - return getTestDataFromResourceJson("schemaWithDateTime.json"); - } - - public static JsonNode getSchemaWithInvalidArrayType() { - return getTestDataFromResourceJson("schemaWithInvalidArrayType.json"); - } - - public static JsonNode getSchemaArrays() { - return getTestDataFromResourceJson("schemaArrays.json"); - } - - public static JsonNode getDataArrays() { - return getTestDataFromResourceJson("dataArrays.json"); - } - - public static JsonNode getSchemaTooDeepNestedDepth() { - return getTestDataFromResourceJson("schemaTooDeepNestedDepth.json"); - } - - public static JsonNode getDataTooDeepNestedDepth() { - return getTestDataFromResourceJson("dataTooDeepNestedDepth.json"); - } - - public static JsonNode getSchemaMaxNestedDepth() { - return getTestDataFromResourceJson("schemaMaxNestedDepth.json"); - } - - public static JsonNode getDataMaxNestedDepth() { - return getTestDataFromResourceJson("dataMaxNestedDepth.json"); - } - - public static JsonNode getExpectedDataArrays() { - return getTestDataFromResourceJson("expectedDataArrays.json"); - } - - public static JsonNode getData() { - return getTestDataFromResourceJson("data.json"); - } - - public static JsonNode getDataWithFormats() { - return getTestDataFromResourceJson("dataWithFormats.json"); - } - - public static JsonNode getAnyOfFormats() { - return getTestDataFromResourceJson("dataAnyOfFormats.json"); - } - - public static JsonNode getAnyOfFormatsWithNull() { - return getTestDataFromResourceJson("dataAnyOfFormatsWithNull.json"); - } - - public static JsonNode getAnyOfFormatsWithEmptyList() { - return getTestDataFromResourceJson("dataAnyOfFormatsWithEmptyList.json"); - } - - public static JsonNode getDataWithJSONDateTimeFormats() { - return getTestDataFromResourceJson("dataWithJSONDateTimeFormats.json"); - } - - public static JsonNode getDataWithJSONWithReference() { - return getTestDataFromResourceJson("dataWithJSONWithReference.json"); - } - - public static JsonNode getSchemaWithReferenceDefinition() { - return getTestDataFromResourceJson("schemaWithReferenceDefinition.json"); - } - - public static JsonNode getSchemaWithNestedDatetimeInsideNullObject() { - return getTestDataFromResourceJson("schemaWithNestedDatetimeInsideNullObject.json"); - } - - public static JsonNode getDataWithEmptyObjectAndArray() { - return getTestDataFromResourceJson("dataWithEmptyObjectAndArray.json"); - } - - public static JsonNode getDataWithNestedDatetimeInsideNullObject() { - return getTestDataFromResourceJson("dataWithNestedDatetimeInsideNullObject.json"); - - } - - private static JsonNode getTestDataFromResourceJson(final String fileName) { - final String fileContent; - try { - fileContent = Files.readString(Path.of(BigQueryDenormalizedTestDataUtils.class.getClassLoader() - .getResource(JSON_FILES_BASE_LOCATION + fileName).getPath())); - } catch (final IOException e) { - throw new RuntimeException(e); - } - return Jsons.deserialize(fileContent); - } - - public static ConfiguredAirbyteCatalog getCommonCatalog(final JsonNode schema, final String datasetId) { - return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList(new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(USERS_STREAM_NAME).withNamespace(datasetId).withJsonSchema(schema) - .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH))) - .withSyncMode(SyncMode.FULL_REFRESH).withDestinationSyncMode(DestinationSyncMode.OVERWRITE))); - } - - public static void runDestinationWrite(ConfiguredAirbyteCatalog catalog, JsonNode config, AirbyteMessage... messages) throws Exception { - final BigQueryDestination destination = new BigQueryDenormalizedDestination(); - final AirbyteMessageConsumer consumer = destination.getConsumer(config, catalog, Destination::defaultOutputRecordCollector); - - consumer.start(); - for (AirbyteMessage message : messages) { - consumer.accept(message); - } - consumer.close(); - } - - private static void checkCredentialPath() { - if (!Files.exists(CREDENTIALS_PATH)) { - throw new IllegalStateException( - "Must provide path to a big query credentials file. By default {module-root}/" + CREDENTIALS_PATH - + ". Override by setting setting path with the CREDENTIALS_PATH constant."); - } - } - - public static JsonNode createCommonConfig() throws IOException { - checkCredentialPath(); - - final String credentialsJsonString = Files.readString(CREDENTIALS_PATH); - final JsonNode credentialsJson = Jsons.deserialize(credentialsJsonString).get(BigQueryConsts.BIGQUERY_BASIC_CONFIG); - final String projectId = credentialsJson.get(CONFIG_PROJECT_ID).asText(); - final String datasetLocation = "US"; - final String datasetId = Strings.addRandomSuffix("airbyte_tests", "_", 8); - - return Jsons.jsonNode(ImmutableMap.builder() - .put(CONFIG_PROJECT_ID, projectId) - .put(BigQueryDenormalizedTestConstants.CONFIG_CREDS, credentialsJson.toString()) - .put(CONFIG_DATASET_ID, datasetId) - .put(CONFIG_DATASET_LOCATION, datasetLocation) - .build()); - } - - public static JsonNode createGcsConfig() throws IOException { - checkCredentialPath(); - - final String credentialsJsonString = Files.readString(CREDENTIALS_PATH); - - final JsonNode fullConfigFromSecretFileJson = Jsons.deserialize(credentialsJsonString); - final JsonNode bigqueryConfigFromSecretFile = fullConfigFromSecretFileJson.get(BigQueryConsts.BIGQUERY_BASIC_CONFIG); - final JsonNode gcsConfigFromSecretFile = fullConfigFromSecretFileJson.get(BigQueryConsts.GCS_CONFIG); - - final String projectId = bigqueryConfigFromSecretFile.get(CONFIG_PROJECT_ID).asText(); - final String datasetLocation = "US"; - - final String datasetId = Strings.addRandomSuffix("airbyte_tests", "_", 8); - - final JsonNode gcsCredentialFromSecretFile = gcsConfigFromSecretFile.get(BigQueryConsts.CREDENTIAL); - final JsonNode credential = Jsons.jsonNode(ImmutableMap.builder() - .put(BigQueryConsts.CREDENTIAL_TYPE, gcsCredentialFromSecretFile.get(BigQueryConsts.CREDENTIAL_TYPE)) - .put(BigQueryConsts.HMAC_KEY_ACCESS_ID, gcsCredentialFromSecretFile.get(BigQueryConsts.HMAC_KEY_ACCESS_ID)) - .put(BigQueryConsts.HMAC_KEY_ACCESS_SECRET, gcsCredentialFromSecretFile.get(BigQueryConsts.HMAC_KEY_ACCESS_SECRET)) - .build()); - - final JsonNode loadingMethod = Jsons.jsonNode(ImmutableMap.builder() - .put(BigQueryConsts.METHOD, BigQueryConsts.GCS_STAGING) - .put(BigQueryConsts.GCS_BUCKET_NAME, gcsConfigFromSecretFile.get(BigQueryConsts.GCS_BUCKET_NAME)) - .put(BigQueryConsts.GCS_BUCKET_PATH, gcsConfigFromSecretFile.get(BigQueryConsts.GCS_BUCKET_PATH).asText() + System.currentTimeMillis()) - .put(BigQueryConsts.CREDENTIAL, credential) - .build()); - - return Jsons.jsonNode(ImmutableMap.builder() - .put(BigQueryConsts.CONFIG_PROJECT_ID, projectId) - .put(BigQueryConsts.CONFIG_CREDS, bigqueryConfigFromSecretFile.toString()) - .put(BigQueryConsts.CONFIG_DATASET_ID, datasetId) - .put(BigQueryConsts.CONFIG_DATASET_LOCATION, datasetLocation) - .put(BigQueryConsts.LOADING_METHOD, loadingMethod) - .build()); - } - - public static BigQuery configureBigQuery(final JsonNode config) throws IOException { - final ServiceAccountCredentials credentials = ServiceAccountCredentials - .fromStream(new ByteArrayInputStream(config.get(CONFIG_CREDS).asText().getBytes(StandardCharsets.UTF_8))); - - return BigQueryOptions.newBuilder() - .setProjectId(config.get(CONFIG_PROJECT_ID).asText()) - .setCredentials(credentials) - .build() - .getService(); - } - - public static Dataset getBigQueryDataSet(final JsonNode config, final BigQuery bigQuery) { - final DatasetInfo datasetInfo = - DatasetInfo.newBuilder(BigQueryUtils.getDatasetId(config)).setLocation(config.get(CONFIG_DATASET_LOCATION).asText()).build(); - Dataset dataset = bigQuery.create(datasetInfo); - trackTestDataSet(dataset, bigQuery); - return dataset; - } - - private static Set dataSetsForDrop = new HashSet<>(); - - public static void trackTestDataSet(final Dataset dataset, final BigQuery bigQuery) { - Runtime.getRuntime() - .addShutdownHook( - new Thread( - () -> tearDownBigQuery(dataset, bigQuery))); - } - - public static synchronized void tearDownBigQuery(final Dataset dataset, final BigQuery bigQuery) { - if (dataSetsForDrop.contains(dataset)) { - // allows deletion of a dataset that has contents - final BigQuery.DatasetDeleteOption option = BigQuery.DatasetDeleteOption.deleteContents(); - - final boolean success = bigQuery.delete(dataset.getDatasetId(), option); - if (success) { - LOGGER.info("BQ Dataset " + dataset + " deleted..."); - } else { - LOGGER.info("BQ Dataset cleanup for " + dataset + " failed!"); - } - dataSetsForDrop.remove(dataset); - } - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/data.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/data.json deleted file mode 100644 index cb0a7336a3cc..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/data.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "Andrii", - "accepts_marketing_updated_at": "2021-10-11T06:36:53-07:00", - "permission-list": [ - { - "domain": "abs", - "grants": ["admin"] - }, - { - "domain": "tools", - "grants": ["read", "write"] - } - ] -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataAnyOfFormats.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataAnyOfFormats.json deleted file mode 100644 index 631c91e6745f..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataAnyOfFormats.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "id": "ID", - "name": "Andrii", - "type": "some_type", - "email": "email@email.com", - "avatar": { - "image_url": "url_to_avatar.jpg" - }, - "team_ids": { - "big_query_array": [1, 2, 3], - "big_query_null": null - }, - "admin_ids": { - "big_query_array": [], - "big_query_null": null - }, - "all_of_field": { - "big_query_array": [4, 5, 6], - "big_query_string": "Some text", - "big_query_integer": 42 - }, - "job_title": "title", - "has_inbox_seat": true, - "away_mode_enabled": false, - "away_mode_reassign": false -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataAnyOfFormatsWithEmptyList.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataAnyOfFormatsWithEmptyList.json deleted file mode 100644 index d199a7ced4a4..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataAnyOfFormatsWithEmptyList.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "Sergii", - "team_ids": [], - "all_of_field": { - "big_query_array": [4, 5, 6], - "big_query_string": "Some text", - "big_query_integer": 42 - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataAnyOfFormatsWithNull.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataAnyOfFormatsWithNull.json deleted file mode 100644 index 31b3d5a86723..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataAnyOfFormatsWithNull.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "Mukola", - "team_ids": null, - "all_of_field": null, - "avatar": null -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataArrays.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataArrays.json deleted file mode 100644 index a06a3bb82198..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataArrays.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "object_with_arrays": { - "array_3": [1, 2, 3] - }, - "simple_string": "simple string", - "array_1": [ - [1, 2], - [2, 3] - ], - "array_4": [[[4]]], - "array_5": [[[[5]]]], - "array_6": [["2021-10-11T06:36:53+00:00", "2020-10-10T01:00:00+00:00"]], - "array_7": [[["2021-10-11T06:36:53+00:00", "2020-10-10T01:00:00+00:00"]]] -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataMaxNestedDepth.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataMaxNestedDepth.json deleted file mode 100644 index 206c7feb6315..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataMaxNestedDepth.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "rec_lvl_1": { - "rec_lvl_2": { - "rec_lvl_3": { - "rec_lvl_4": { - "rec_lvl_5": { - "rec_lvl_6": { - "rec_lvl_7": { - "rec_lvl_8": { - "rec_lvl_9": { - "rec_lvl_10": { - "rec_lvl_11": { - "rec_lvl_12": { - "rec_lvl_13": { - "rec_lvl_14": { - "str_value": "test_value" - } - } - } - } - } - } - } - } - } - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataTooDeepNestedDepth.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataTooDeepNestedDepth.json deleted file mode 100644 index 5bdc95d388aa..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataTooDeepNestedDepth.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "rec_lvl_1": { - "rec_lvl_2": { - "rec_lvl_3": { - "rec_lvl_4": { - "rec_lvl_5": { - "rec_lvl_6": { - "rec_lvl_7": { - "rec_lvl_8": { - "rec_lvl_9": { - "rec_lvl_10": { - "rec_lvl_11": { - "rec_lvl_12": { - "rec_lvl_13": { - "rec_lvl_14": { - "rec_lvl_15": { - "str_value": "test_value" - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataWithEmptyObjectAndArray.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataWithEmptyObjectAndArray.json deleted file mode 100644 index 5a10ec99288d..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataWithEmptyObjectAndArray.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "Andrii", - "permission-list": [ - { - "domain": "abs", - "items": {}, - "grants": ["admin"] - }, - { - "domain": "tools", - "grants": [], - "items": { - "object": {}, - "array": [] - } - } - ] -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataWithFormats.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataWithFormats.json deleted file mode 100644 index 702f9012ddd6..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataWithFormats.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "name": "Andrii", - "date_of_birth": "1996-01-25", - "updated_at": "2021-10-11T06:36:53" -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataWithJSONDateTimeFormats.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataWithJSONDateTimeFormats.json deleted file mode 100644 index 642f9b40f99f..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataWithJSONDateTimeFormats.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "updated_at": "2021-10-11T06:36:53+00:00", - "items": { - "nested_datetime": "2021-11-11T06:36:53+00:00" - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataWithJSONWithReference.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataWithJSONWithReference.json deleted file mode 100644 index 6870f8a72448..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataWithJSONWithReference.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "users": { - "name": "John", - "surname": "Adams" - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataWithNestedDatetimeInsideNullObject.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataWithNestedDatetimeInsideNullObject.json deleted file mode 100644 index 2b649fbe78eb..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/dataWithNestedDatetimeInsideNullObject.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "name": "Alice in Wonderland", - "appointment": null -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/expectedDataArrays.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/expectedDataArrays.json deleted file mode 100644 index ed63c2a27ce3..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/expectedDataArrays.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "object_with_arrays": { - "array_3": [1, 2, 3] - }, - "simple_string": "simple string", - "array_1": [ - { - "big_query_array": [1, 2] - }, - { - "big_query_array": [2, 3] - } - ], - "array_4": [ - { - "big_query_array": [ - { - "big_query_array": [4] - } - ] - } - ], - "array_5": [ - { - "big_query_array": [ - { - "big_query_array": [ - { - "big_query_array": [5] - } - ] - } - ] - } - ], - "array_6": [ - { - "big_query_array": ["2021-10-11T06:36:53", "2020-10-10T01:00:00"] - } - ], - "array_7": [ - { - "big_query_array": [ - { - "big_query_array": ["2021-10-11T06:36:53", "2020-10-10T01:00:00"] - } - ] - } - ] -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schema.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schema.json deleted file mode 100644 index 70cb8f7b8e1a..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schema.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "type": ["object"], - "properties": { - "accepts_marketing_updated_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "name": { - "type": ["string"] - }, - "permission-list": { - "type": ["array"], - "items": { - "type": ["object"], - "properties": { - "domain": { - "type": ["string"] - }, - "grants": { - "type": ["array"], - "items": { - "type": ["string"] - } - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaAnyOfAllOf.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaAnyOfAllOf.json deleted file mode 100644 index 422f173a6aae..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaAnyOfAllOf.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "avatar": { - "type": ["null", "object"], - "properties": { - "image_url": { - "type": ["null", "string"] - } - } - }, - "team_ids": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer" - } - }, - { - "type": "null" - } - ] - }, - "admin_ids": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "integer" - } - }, - { - "type": "null" - } - ] - }, - "all_of_field": { - "allOf": [ - { - "type": "array", - "items": { - "type": "integer" - } - }, - { - "type": "string" - }, - { - "type": "integer" - } - ] - }, - "job_title": { - "type": ["null", "string"] - }, - "has_inbox_seat": { - "type": ["null", "boolean"] - }, - "away_mode_enabled": { - "type": ["null", "boolean"] - }, - "away_mode_reassign": { - "type": ["null", "boolean"] - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaArrays.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaArrays.json deleted file mode 100644 index 90d0e379b667..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaArrays.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "type": ["object"], - "properties": { - "object_with_arrays": { - "type": ["object"], - "properties": { - "array_3": { - "type": ["array"], - "items": { - "type": "integer" - } - } - } - }, - "simple_string": { - "type": ["string"] - }, - "array_1": { - "type": ["array"], - "items": { - "type": ["array"], - "items": { - "type": "integer" - } - } - }, - "array_4": { - "type": ["array"], - "items": { - "type": ["array"], - "items": { - "type": ["array"], - "items": { - "type": "integer" - } - } - } - }, - "array_5": { - "type": ["array"], - "items": { - "type": ["array"], - "items": { - "type": ["array"], - "items": { - "type": ["array"], - "items": { - "type": "integer" - } - } - } - } - }, - "array_6": { - "type": ["array"], - "items": { - "type": ["array"], - "items": { - "type": ["string"], - "format": "date-time" - } - } - }, - "array_7": { - "type": ["array"], - "items": { - "type": ["array"], - "items": { - "type": ["array"], - "items": { - "type": ["string"], - "format": "date-time" - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaMaxNestedDepth.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaMaxNestedDepth.json deleted file mode 100644 index a6bf911b74e9..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaMaxNestedDepth.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "type": ["object"], - "properties": { - "rec_lvl_1": { - "type": ["object"], - "properties": { - "rec_lvl_2": { - "type": ["object"], - "properties": { - "rec_lvl_3": { - "type": ["object"], - "properties": { - "rec_lvl_4": { - "type": ["object"], - "properties": { - "rec_lvl_5": { - "type": ["object"], - "properties": { - "rec_lvl_6": { - "type": ["object"], - "properties": { - "rec_lvl_7": { - "type": ["object"], - "properties": { - "rec_lvl_8": { - "type": ["object"], - "properties": { - "rec_lvl_9": { - "type": ["object"], - "properties": { - "rec_lvl_10": { - "type": ["object"], - "properties": { - "rec_lvl_11": { - "type": ["object"], - "properties": { - "rec_lvl_12": { - "type": ["object"], - "properties": { - "rec_lvl_13": { - "type": ["object"], - "properties": { - "rec_lvl_14": { - "type": ["object"], - "properties": { - "str_value": { - "type": ["string"] - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaTooDeepNestedDepth.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaTooDeepNestedDepth.json deleted file mode 100644 index 2e6c71cdfbc8..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaTooDeepNestedDepth.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "type": ["object"], - "properties": { - "rec_lvl_1": { - "type": ["object"], - "properties": { - "rec_lvl_2": { - "type": ["object"], - "properties": { - "rec_lvl_3": { - "type": ["object"], - "properties": { - "rec_lvl_4": { - "type": ["object"], - "properties": { - "rec_lvl_5": { - "type": ["object"], - "properties": { - "rec_lvl_6": { - "type": ["object"], - "properties": { - "rec_lvl_7": { - "type": ["object"], - "properties": { - "rec_lvl_8": { - "type": ["object"], - "properties": { - "rec_lvl_9": { - "type": ["object"], - "properties": { - "rec_lvl_10": { - "type": ["object"], - "properties": { - "rec_lvl_11": { - "type": ["object"], - "properties": { - "rec_lvl_12": { - "type": ["object"], - "properties": { - "rec_lvl_13": { - "type": ["object"], - "properties": { - "rec_lvl_14": { - "type": ["object"], - "properties": { - "rec_lvl_15": { - "type": [ - "object" - ], - "properties": { - "str_value": { - "type": [ - "string" - ] - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaWithDateTime.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaWithDateTime.json deleted file mode 100644 index b9bb321012dc..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaWithDateTime.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "type": ["object"], - "properties": { - "updated_at": { - "type": ["string"], - "format": "date-time" - }, - "items": { - "type": ["object"], - "properties": { - "nested_datetime": { - "type": ["string"], - "format": "date-time" - } - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaWithFormats.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaWithFormats.json deleted file mode 100644 index 4f8c95bcdc16..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaWithFormats.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "type": ["object"], - "properties": { - "name": { - "type": ["string"] - }, - "date_of_birth": { - "type": ["string"], - "format": "date" - }, - "updated_at": { - "type": ["string"], - "format": "date-time" - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaWithInvalidArrayType.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaWithInvalidArrayType.json deleted file mode 100644 index 5517ffb855aa..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaWithInvalidArrayType.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "type": ["object"], - "properties": { - "name": { - "type": ["string"] - }, - "permission-list": { - "type": ["array"], - "items": { - "type": ["object"], - "properties": { - "domain": { - "type": ["string"] - }, - "grants": { - "type": ["array"] - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaWithNestedDatetimeInsideNullObject.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaWithNestedDatetimeInsideNullObject.json deleted file mode 100644 index 625d12295b79..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaWithNestedDatetimeInsideNullObject.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "type": ["object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "appointment": { - "type": ["null", "object"], - "properties": { - "street": { - "type": ["null", "string"] - }, - "expTime": { - "type": ["null", "string"], - "format": "date-time" - } - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaWithReferenceDefinition.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaWithReferenceDefinition.json deleted file mode 100644 index 64f5c21646f1..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test-integration/resources/testdata/schemaWithReferenceDefinition.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "type": ["null", "object"], - "properties": { - "users": { - "$ref": "#/definitions/users_" - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestinationTest.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestinationTest.java deleted file mode 100644 index b4e9235fdb1d..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryDenormalizedDestinationTest.java +++ /dev/null @@ -1,389 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery; - -import static com.google.cloud.bigquery.Field.Mode.REPEATED; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestSchemaUtils.getSchema; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.core.IsInstanceOf.instanceOf; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.Field; -import com.google.cloud.bigquery.FieldList; -import com.google.cloud.bigquery.LegacySQLTypeName; -import com.google.cloud.bigquery.Schema; -import com.google.cloud.bigquery.StandardSQLTypeName; -import com.google.cloud.bigquery.Table; -import com.google.cloud.bigquery.TableDefinition; -import io.airbyte.integrations.destination.bigquery.formatter.BigQueryRecordFormatter; -import io.airbyte.integrations.destination.bigquery.formatter.DefaultBigQueryDenormalizedRecordFormatter; -import io.airbyte.integrations.destination.bigquery.formatter.GcsBigQueryDenormalizedRecordFormatter; -import io.airbyte.integrations.destination.bigquery.formatter.arrayformater.LegacyArrayFormatter; -import io.airbyte.integrations.destination.bigquery.uploader.AbstractBigQueryUploader; -import io.airbyte.integrations.destination.bigquery.uploader.BigQueryDirectUploader; -import io.airbyte.integrations.destination.bigquery.uploader.BigQueryUploaderFactory; -import io.airbyte.integrations.destination.bigquery.uploader.UploaderType; -import io.airbyte.integrations.destination.bigquery.uploader.config.UploaderConfig; -import io.airbyte.protocol.models.v0.AirbyteStream; -import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; - -@ExtendWith(MockitoExtension.class) -class BigQueryDenormalizedDestinationTest { - - @Mock - UploaderConfig uploaderConfigMock; - @Mock - ConfiguredAirbyteStream configuredStreamMock; - @Mock - AirbyteStream airbyteStreamMock; - @Mock - DefaultBigQueryDenormalizedRecordFormatter bigQueryRecordFormatterMock; - @Mock - BigQuery bigQueryMock; - - MockedStatic uploaderFactoryMock; - - @InjectMocks - BigQueryDenormalizedDestination bqdd; - - final ObjectMapper mapper = new ObjectMapper(); - - @BeforeEach - void init() { - uploaderFactoryMock = Mockito.mockStatic(BigQueryUploaderFactory.class, Mockito.CALLS_REAL_METHODS); - uploaderFactoryMock.when(() -> BigQueryUploaderFactory.getUploader(any(UploaderConfig.class))).thenReturn(mock(BigQueryDirectUploader.class)); - } - - @AfterEach - public void teardown() { - uploaderFactoryMock.close(); - } - - @Test - void getFormatterMap() { - final JsonNode jsonNodeSchema = getSchema(); - final Map formatterMap = bqdd.getFormatterMap(jsonNodeSchema); - assertEquals(2, formatterMap.size()); - assertTrue(formatterMap.containsKey(UploaderType.AVRO)); - assertTrue(formatterMap.containsKey(UploaderType.STANDARD)); - assertThat(formatterMap.get(UploaderType.AVRO), instanceOf(GcsBigQueryDenormalizedRecordFormatter.class)); - assertThat(formatterMap.get(UploaderType.STANDARD), instanceOf(DefaultBigQueryDenormalizedRecordFormatter.class)); - } - - @Test - void isDefaultAirbyteTmpTableSchema() { - assertFalse(bqdd.isDefaultAirbyteTmpTableSchema()); - } - - @Test - void getRecordFormatterCreator() { - final BigQuerySQLNameTransformer nameTransformerMock = mock(BigQuerySQLNameTransformer.class); - final BigQueryRecordFormatter resultFormatter = bqdd.getRecordFormatterCreator(nameTransformerMock) - .apply(mapper.createObjectNode()); - - assertThat(resultFormatter, instanceOf(GcsBigQueryDenormalizedRecordFormatter.class)); - } - - @Test - void putStreamIntoUploaderMap_compareSchemas_expectedIsNotNullExistingIsNull() throws IOException { - final Map> uploaderMap = new HashMap<>(); - final String streamName = "stream_name"; - final String nameSpace = "name_space"; - final Table tableMock = mock(Table.class); - final Schema schemaMock = mock(Schema.class); - final TableDefinition tableDefinitionMock = mock(TableDefinition.class); - - mockBigqueryStream(); - when(tableMock.getDefinition()).thenReturn(tableDefinitionMock); - when(uploaderConfigMock.getTargetTableName()).thenReturn("target_table"); - when(airbyteStreamMock.getNamespace()).thenReturn(nameSpace); - when(airbyteStreamMock.getName()).thenReturn(streamName); - when(bigQueryMock.getTable(anyString(), anyString())).thenReturn(tableMock); - - // expected schema is not null - when(bigQueryRecordFormatterMock.getBigQuerySchema()).thenReturn(schemaMock); - // existing schema is null - when(tableDefinitionMock.getSchema()).thenReturn(null); - // run test - bqdd.putStreamIntoUploaderMap(airbyteStreamMock, uploaderConfigMock, uploaderMap); - // should use LegacyArrayFormatter - verify(bigQueryRecordFormatterMock, times(1)).setArrayFormatter(any(LegacyArrayFormatter.class)); - } - - @Test - void putStreamIntoUploaderMap_compareSchemas_existingAndExpectedAreNull() throws IOException { - final Map> uploaderMap = new HashMap<>(); - final Table tableMock = mock(Table.class); - final TableDefinition tableDefinitionMock = mock(TableDefinition.class); - - mockBigqueryStream(); - when(uploaderConfigMock.getTargetTableName()).thenReturn("target_table"); - when(airbyteStreamMock.getNamespace()).thenReturn("name_space"); - when(airbyteStreamMock.getName()).thenReturn("stream_name"); - when(bigQueryMock.getTable(anyString(), anyString())).thenReturn(tableMock); - when(tableMock.getDefinition()).thenReturn(tableDefinitionMock); - - // expected schema is null - when(bigQueryRecordFormatterMock.getBigQuerySchema()).thenReturn(null); - // existing schema is null - when(tableDefinitionMock.getSchema()).thenReturn(null); - // run test - bqdd.putStreamIntoUploaderMap(airbyteStreamMock, uploaderConfigMock, uploaderMap); - // should not use LegacyArrayFormatter - verify(bigQueryRecordFormatterMock, times(0)).setArrayFormatter(any(LegacyArrayFormatter.class)); - } - - @Test - void putStreamIntoUploaderMap_compareSchemas_expectedSchemaIsNull() throws IOException { - final Map> uploaderMap = new HashMap<>(); - final Table tableMock = mock(Table.class); - final Schema schemaMock = mock(Schema.class); - final TableDefinition tableDefinitionMock = mock(TableDefinition.class); - - mockBigqueryStream(); - when(tableMock.getDefinition()).thenReturn(tableDefinitionMock); - when(uploaderConfigMock.getTargetTableName()).thenReturn("target_table"); - when(bigQueryMock.getTable(anyString(), anyString())).thenReturn(tableMock); - when(airbyteStreamMock.getNamespace()).thenReturn("name_space"); - when(airbyteStreamMock.getName()).thenReturn("stream_name"); - when(bigQueryMock.getTable(anyString(), anyString())).thenReturn(tableMock); - - // expected schema is null - when(bigQueryRecordFormatterMock.getBigQuerySchema()).thenReturn(null); - // existing schema is not null - when(tableDefinitionMock.getSchema()).thenReturn(schemaMock); - // run test - bqdd.putStreamIntoUploaderMap(airbyteStreamMock, uploaderConfigMock, uploaderMap); - // should use LegacyArrayFormatter - verify(bigQueryRecordFormatterMock, times(1)).setArrayFormatter(any(LegacyArrayFormatter.class)); - } - - @Test - void putStreamIntoUploaderMap_isDifferenceBetweenFields_equalType() throws IOException { - final Map> uploaderMap = new HashMap<>(); - final Table tableMock = mock(Table.class); - final TableDefinition tableDefinitionMock = mock(TableDefinition.class); - final Schema existingSchemaMock = mock(Schema.class); - final Schema expectedSchemaMock = mock(Schema.class); - - mockBigqueryStream(); - when(tableMock.getDefinition()).thenReturn(tableDefinitionMock); - when(tableDefinitionMock.getSchema()).thenReturn(existingSchemaMock); - when(bigQueryRecordFormatterMock.getBigQuerySchema()).thenReturn(expectedSchemaMock); - when(uploaderConfigMock.getTargetTableName()).thenReturn("target_table"); - when(airbyteStreamMock.getNamespace()).thenReturn("name_space"); - when(bigQueryMock.getTable(anyString(), anyString())).thenReturn(tableMock); - - final FieldList existingFields = FieldList.of(Field.newBuilder("name", StandardSQLTypeName.STRING).build()); - final FieldList expectedFields = FieldList.of(Field.newBuilder("name", StandardSQLTypeName.STRING).build()); - when(existingSchemaMock.getFields()).thenReturn(existingFields); - when(expectedSchemaMock.getFields()).thenReturn(expectedFields); - - // run test - bqdd.putStreamIntoUploaderMap(airbyteStreamMock, uploaderConfigMock, uploaderMap); - // equal type should not use LegacyArrayFormatter - verify(bigQueryRecordFormatterMock, times(0)).setArrayFormatter(any(LegacyArrayFormatter.class)); - } - - @Test - void putStreamIntoUploaderMap_isDifferenceBetweenFields_notEqualType() throws IOException { - final Map> uploaderMap = new HashMap<>(); - final Table tableMock = mock(Table.class); - final TableDefinition tableDefinitionMock = mock(TableDefinition.class); - final Schema existingSchemaMock = mock(Schema.class); - final Schema expectedSchemaMock = mock(Schema.class); - - mockBigqueryStream(); - when(tableMock.getDefinition()).thenReturn(tableDefinitionMock); - when(tableDefinitionMock.getSchema()).thenReturn(existingSchemaMock); - when(bigQueryRecordFormatterMock.getBigQuerySchema()).thenReturn(expectedSchemaMock); - when(uploaderConfigMock.getTargetTableName()).thenReturn("target_table"); - when(airbyteStreamMock.getNamespace()).thenReturn("name_space"); - when(bigQueryMock.getTable(anyString(), anyString())).thenReturn(tableMock); - - final FieldList existingFields = FieldList.of(Field.newBuilder("name", StandardSQLTypeName.DATE).build()); - final FieldList expectedFields = FieldList.of(Field.newBuilder("name", StandardSQLTypeName.STRING).build()); - when(existingSchemaMock.getFields()).thenReturn(existingFields); - when(expectedSchemaMock.getFields()).thenReturn(expectedFields); - - // run test - bqdd.putStreamIntoUploaderMap(airbyteStreamMock, uploaderConfigMock, uploaderMap); - - // equal type should not use LegacyArrayFormatter - verify(bigQueryRecordFormatterMock, times(1)).setArrayFormatter(any(LegacyArrayFormatter.class)); - } - - @Test - void putStreamIntoUploaderMap_isDifferenceBetweenFields_existingFieldIsNull() throws IOException { - final Map> uploaderMap = new HashMap<>(); - final Table tableMock = mock(Table.class); - final TableDefinition tableDefinitionMock = mock(TableDefinition.class); - final Schema existingSchemaMock = mock(Schema.class); - final Schema expectedSchemaMock = mock(Schema.class); - - mockBigqueryStream(); - when(tableMock.getDefinition()).thenReturn(tableDefinitionMock); - when(tableDefinitionMock.getSchema()).thenReturn(existingSchemaMock); - when(bigQueryRecordFormatterMock.getBigQuerySchema()).thenReturn(expectedSchemaMock); - when(uploaderConfigMock.getTargetTableName()).thenReturn("target_table"); - when(airbyteStreamMock.getNamespace()).thenReturn("name_space"); - when(bigQueryMock.getTable(anyString(), anyString())).thenReturn(tableMock); - - final FieldList expectedFields = FieldList.of(Field.newBuilder("name", StandardSQLTypeName.STRING).build()); - final FieldList existingFields = mock(FieldList.class); - when(existingSchemaMock.getFields()).thenReturn(existingFields); - when(expectedSchemaMock.getFields()).thenReturn(expectedFields); - when(existingFields.get(anyString())).thenReturn(null); - - // run test - bqdd.putStreamIntoUploaderMap(airbyteStreamMock, uploaderConfigMock, uploaderMap); - - // equal type should not use LegacyArrayFormatter - verify(bigQueryRecordFormatterMock, times(1)).setArrayFormatter(any(LegacyArrayFormatter.class)); - } - - @Test - void putStreamIntoUploaderMap_compareRepeatedMode_isEqual() throws IOException { - final Map> uploaderMap = new HashMap<>(); - final Table tableMock = mock(Table.class); - final TableDefinition tableDefinitionMock = mock(TableDefinition.class); - final Schema existingSchemaMock = mock(Schema.class); - final Schema expectedSchemaMock = mock(Schema.class); - - when(tableMock.getDefinition()).thenReturn(tableDefinitionMock); - when(tableDefinitionMock.getSchema()).thenReturn(existingSchemaMock); - when(bigQueryRecordFormatterMock.getBigQuerySchema()).thenReturn(expectedSchemaMock); - mockBigqueryStream(); - when(uploaderConfigMock.getTargetTableName()).thenReturn("target_table"); - when(airbyteStreamMock.getNamespace()).thenReturn("name_space"); - when(airbyteStreamMock.getName()).thenReturn("stream_name"); - when(bigQueryMock.getTable(anyString(), anyString())).thenReturn(tableMock); - - final FieldList existingFields = FieldList.of(Field.newBuilder("name", StandardSQLTypeName.STRING).setMode(REPEATED).build()); - final FieldList expectedFields = FieldList.of(Field.newBuilder("name", StandardSQLTypeName.STRING).setMode(REPEATED).build()); - when(existingSchemaMock.getFields()).thenReturn(existingFields); - when(expectedSchemaMock.getFields()).thenReturn(expectedFields); - // run test - bqdd.putStreamIntoUploaderMap(airbyteStreamMock, uploaderConfigMock, uploaderMap); - // equal mode should not use LegacyArrayFormatter - verify(bigQueryRecordFormatterMock, times(0)).setArrayFormatter(any(LegacyArrayFormatter.class)); - } - - @Test - void putStreamIntoUploaderMap_compareSubFields_equalType() throws IOException { - final Map> uploaderMap = new HashMap<>(); - final String streamName = "stream_name"; - final String nameSpace = "name_space"; - final Table tableMock = mock(Table.class); - final TableDefinition tableDefinitionMock = mock(TableDefinition.class); - final Schema existingSchemaMock = mock(Schema.class); - final Schema expectedSchemaMock = mock(Schema.class); - - when(tableMock.getDefinition()).thenReturn(tableDefinitionMock); - when(tableDefinitionMock.getSchema()).thenReturn(existingSchemaMock); - when(bigQueryRecordFormatterMock.getBigQuerySchema()).thenReturn(expectedSchemaMock); - mockBigqueryStream(); - when(uploaderConfigMock.getTargetTableName()).thenReturn("target_table"); - when(airbyteStreamMock.getNamespace()).thenReturn(nameSpace); - when(airbyteStreamMock.getName()).thenReturn(streamName); - when(bigQueryMock.getTable(anyString(), anyString())).thenReturn(tableMock); - - final FieldList expectedSubField = FieldList.of(Field.newBuilder("sub_field_name", StandardSQLTypeName.STRING).build()); - final FieldList existingSubField = FieldList.of(Field.newBuilder("sub_field_name", StandardSQLTypeName.STRING).build()); - final Field existingField = Field.newBuilder("field_name", LegacySQLTypeName.RECORD, existingSubField).build(); - final Field expectedField = Field.newBuilder("field_name", LegacySQLTypeName.RECORD, expectedSubField).build(); - when(existingSchemaMock.getFields()).thenReturn(FieldList.of(existingField)); - when(expectedSchemaMock.getFields()).thenReturn(FieldList.of(expectedField)); - // run test - bqdd.putStreamIntoUploaderMap(airbyteStreamMock, uploaderConfigMock, uploaderMap); - // equal subfield type should not use LegacyArrayFormatter - verify(bigQueryRecordFormatterMock, times(0)).setArrayFormatter(any(LegacyArrayFormatter.class)); - } - - @Test - void putStreamIntoUploaderMap_compareSubFields_notEqualType() throws IOException { - final Map> uploaderMap = new HashMap<>(); - final String streamName = "stream_name"; - final String nameSpace = "name_space"; - final Table tableMock = mock(Table.class); - final TableDefinition tableDefinitionMock = mock(TableDefinition.class); - final Schema existingSchemaMock = mock(Schema.class); - final Schema expectedSchemaMock = mock(Schema.class); - - when(tableMock.getDefinition()).thenReturn(tableDefinitionMock); - when(tableDefinitionMock.getSchema()).thenReturn(existingSchemaMock); - when(bigQueryRecordFormatterMock.getBigQuerySchema()).thenReturn(expectedSchemaMock); - mockBigqueryStream(); - when(uploaderConfigMock.getTargetTableName()).thenReturn("target_table"); - when(airbyteStreamMock.getNamespace()).thenReturn(nameSpace); - when(airbyteStreamMock.getName()).thenReturn(streamName); - when(bigQueryMock.getTable(anyString(), anyString())).thenReturn(tableMock); - - final FieldList expectedSubField = FieldList.of(Field.newBuilder("sub_field_name", StandardSQLTypeName.DATE).build()); - final FieldList existingSubField = FieldList.of(Field.newBuilder("sub_field_name", StandardSQLTypeName.STRING).build()); - final Field existingField = Field.newBuilder("field_name", LegacySQLTypeName.RECORD, existingSubField).build(); - final Field expectedField = Field.newBuilder("field_name", LegacySQLTypeName.RECORD, expectedSubField).build(); - when(existingSchemaMock.getFields()).thenReturn(FieldList.of(existingField)); - when(expectedSchemaMock.getFields()).thenReturn(FieldList.of(expectedField)); - // run test - bqdd.putStreamIntoUploaderMap(airbyteStreamMock, uploaderConfigMock, uploaderMap); - // not equal subfield type should use LegacyArrayFormatter - verify(bigQueryRecordFormatterMock, times(1)).setArrayFormatter(any(LegacyArrayFormatter.class)); - } - - @Test - void putStreamIntoUploaderMap_existingTableIsNull() throws IOException { - final Map> uploaderMap = new HashMap<>(); - final String streamName = "stream_name"; - final String nameSpace = "name_space"; - final String targetTableName = "target_table"; - final AirbyteStreamNameNamespacePair expectedResult = new AirbyteStreamNameNamespacePair(streamName, nameSpace); - - mockBigqueryStream(); - when(uploaderConfigMock.getTargetTableName()).thenReturn(targetTableName); - when(airbyteStreamMock.getNamespace()).thenReturn(nameSpace); - when(airbyteStreamMock.getName()).thenReturn(streamName); - // existing table is null - when(bigQueryMock.getTable(anyString(), anyString())).thenReturn(null); - - // run test - bqdd.putStreamIntoUploaderMap(airbyteStreamMock, uploaderConfigMock, uploaderMap); - - verify(bigQueryRecordFormatterMock, times(0)).setArrayFormatter(any(LegacyArrayFormatter.class)); - assertTrue(uploaderMap.containsKey(expectedResult)); - } - - private void mockBigqueryStream() { - when(uploaderConfigMock.getConfigStream()).thenReturn(configuredStreamMock); - when(uploaderConfigMock.getBigQuery()).thenReturn(bigQueryMock); - when(uploaderConfigMock.getFormatter()).thenReturn(bigQueryRecordFormatterMock); - when(configuredStreamMock.getStream()).thenReturn(airbyteStreamMock); - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/JsonSchemaFormatTest.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/JsonSchemaFormatTest.java deleted file mode 100644 index cfcb04bd640a..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/JsonSchemaFormatTest.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery; - -import static org.junit.jupiter.api.Assertions.*; - -import org.junit.jupiter.api.Test; - -class JsonSchemaFormatTest { - - @Test - void fromJsonSchemaFormat_matchByFormatAndType() { - JsonSchemaFormat result = JsonSchemaFormat.fromJsonSchemaFormat("date-time", "timestamp_with_timezone"); - assertEquals(JsonSchemaFormat.DATETIME_WITH_TZ, result); - } - - @Test - void fromJsonSchemaFormat_matchByFormat() { - JsonSchemaFormat result = JsonSchemaFormat.fromJsonSchemaFormat("date", null); - assertEquals(JsonSchemaFormat.DATE, result); - } - - @Test - void fromJsonSchemaFormat_notExistingFormat() { - JsonSchemaFormat result = JsonSchemaFormat.fromJsonSchemaFormat("not_existing_format", null); - assertNull(result); - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/JsonSchemaTypeTest.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/JsonSchemaTypeTest.java deleted file mode 100644 index c705607124c5..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/JsonSchemaTypeTest.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import org.junit.jupiter.api.Test; - -class JsonSchemaTypeTest { - - @Test - void fromJsonSchemaType_notPresent() { - assertThrows(IllegalArgumentException.class, () -> JsonSchemaType.fromJsonSchemaType("not_existing_value")); - } - - @Test - void fromJsonSchemaType_getType() { - JsonSchemaType result = JsonSchemaType.fromJsonSchemaType("string"); - assertEquals(JsonSchemaType.STRING, result); - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryDenormalizedRecordFormatterTest.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryDenormalizedRecordFormatterTest.java deleted file mode 100644 index ccffa13cfdf2..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryDenormalizedRecordFormatterTest.java +++ /dev/null @@ -1,288 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery.formatter; - -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestSchemaUtils.getExpectedSchema; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestSchemaUtils.getExpectedSchemaArrays; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestSchemaUtils.getExpectedSchemaWithDateTime; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestSchemaUtils.getExpectedSchemaWithFormats; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestSchemaUtils.getExpectedSchemaWithInvalidArrayType; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestSchemaUtils.getExpectedSchemaWithNestedDatetimeInsideNullObject; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestSchemaUtils.getSchema; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestSchemaUtils.getSchemaArrays; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestSchemaUtils.getSchemaWithBigInteger; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestSchemaUtils.getSchemaWithDateTime; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestSchemaUtils.getSchemaWithFormats; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestSchemaUtils.getSchemaWithInvalidArrayType; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestSchemaUtils.getSchemaWithNestedDatetimeInsideNullObject; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestSchemaUtils.getSchemaWithReferenceDefinition; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.params.provider.Arguments.arguments; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.JsonNodeType; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.cloud.bigquery.Field; -import com.google.cloud.bigquery.Field.Mode; -import com.google.cloud.bigquery.LegacySQLTypeName; -import com.google.cloud.bigquery.Schema; -import com.google.cloud.bigquery.StandardSQLTypeName; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.bigquery.BigQuerySQLNameTransformer; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import java.util.stream.Stream; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; -import org.mockito.Mockito; - -class DefaultBigQueryDenormalizedRecordFormatterTest { - - private final ObjectMapper mapper = new ObjectMapper(); - - private static Stream actualAndExpectedSchemasProvider() { - return Stream.of( - arguments(getSchema(), getExpectedSchema()), - arguments(getSchemaWithFormats(), getExpectedSchemaWithFormats()), - arguments(getSchemaWithDateTime(), getExpectedSchemaWithDateTime()), - arguments(getSchemaWithInvalidArrayType(), getExpectedSchemaWithInvalidArrayType()), - arguments(getSchemaWithNestedDatetimeInsideNullObject(), - getExpectedSchemaWithNestedDatetimeInsideNullObject()), - arguments(getSchemaArrays(), getExpectedSchemaArrays())); - } - - @ParameterizedTest - @MethodSource("actualAndExpectedSchemasProvider") - void testDefaultSchema(final JsonNode schemaToProcess, final JsonNode expectedSchema) { - DefaultBigQueryDenormalizedRecordFormatter rf = new DefaultBigQueryDenormalizedRecordFormatter( - schemaToProcess, new BigQuerySQLNameTransformer()); - - assertEquals(expectedSchema, rf.formatJsonSchema(schemaToProcess)); - } - - @Test - void testSchema() { - final JsonNode jsonNodeSchema = getSchema(); - DefaultBigQueryDenormalizedRecordFormatter rf = new DefaultBigQueryDenormalizedRecordFormatter( - jsonNodeSchema, new BigQuerySQLNameTransformer()); - final Field subFields = Field.newBuilder("big_query_array", LegacySQLTypeName.RECORD, - Field.of("domain", LegacySQLTypeName.STRING), - Field.of("grants", LegacySQLTypeName.RECORD, - Field.newBuilder("big_query_array", StandardSQLTypeName.STRING).setMode(Mode.REPEATED).build())) - .setMode(Mode.REPEATED).build(); - final Schema expectedResult = Schema.of( - Field.newBuilder("accepts_marketing_updated_at", LegacySQLTypeName.DATETIME).setMode(Mode.NULLABLE).build(), - Field.of("name", LegacySQLTypeName.STRING), - Field.of("permission_list", LegacySQLTypeName.RECORD, subFields), - Field.of("_airbyte_ab_id", LegacySQLTypeName.STRING), - Field.of("_airbyte_emitted_at", LegacySQLTypeName.TIMESTAMP)); - - final Schema result = rf.getBigQuerySchema(jsonNodeSchema); - - assertEquals(expectedResult, result); - } - - @Test - void testSchemaWithFormats() { - final JsonNode jsonNodeSchema = getSchemaWithFormats(); - DefaultBigQueryDenormalizedRecordFormatter rf = new DefaultBigQueryDenormalizedRecordFormatter( - jsonNodeSchema, new BigQuerySQLNameTransformer()); - final Schema expectedResult = Schema.of( - Field.of("name", LegacySQLTypeName.STRING), - Field.of("date_of_birth", LegacySQLTypeName.DATE), - Field.of("updated_at", LegacySQLTypeName.DATETIME), - Field.of("_airbyte_ab_id", LegacySQLTypeName.STRING), - Field.of("_airbyte_emitted_at", LegacySQLTypeName.TIMESTAMP)); - - final Schema result = rf.getBigQuerySchema(jsonNodeSchema); - - assertEquals(expectedResult, result); - } - - @Test - void testSchemaWithBigInteger() { - final JsonNode jsonNodeSchema = getSchemaWithBigInteger(); - DefaultBigQueryDenormalizedRecordFormatter rf = new DefaultBigQueryDenormalizedRecordFormatter( - jsonNodeSchema, new BigQuerySQLNameTransformer()); - final Schema expectedResult = Schema.of( - Field.of("salary", LegacySQLTypeName.INTEGER), - Field.of("updated_at", LegacySQLTypeName.DATETIME), - Field.of("_airbyte_ab_id", LegacySQLTypeName.STRING), - Field.of("_airbyte_emitted_at", LegacySQLTypeName.TIMESTAMP)); - - final Schema result = rf.getBigQuerySchema(jsonNodeSchema); - - assertEquals(expectedResult, result); - } - - @Test - void testSchemaWithDateTime() { - final JsonNode jsonNodeSchema = getSchemaWithDateTime(); - DefaultBigQueryDenormalizedRecordFormatter rf = new DefaultBigQueryDenormalizedRecordFormatter( - jsonNodeSchema, new BigQuerySQLNameTransformer()); - final Schema expectedResult = Schema.of( - Field.of("updated_at", LegacySQLTypeName.DATETIME), - Field.of("items", LegacySQLTypeName.RECORD, Field.of("nested_datetime", LegacySQLTypeName.DATETIME)), - Field.of("_airbyte_ab_id", LegacySQLTypeName.STRING), - Field.of("_airbyte_emitted_at", LegacySQLTypeName.TIMESTAMP)); - - final Schema result = rf.getBigQuerySchema(jsonNodeSchema); - - assertEquals(expectedResult, result); - } - - @Test - void testSchemaWithInvalidArrayType() { - final JsonNode jsonNodeSchema = getSchemaWithInvalidArrayType(); - DefaultBigQueryDenormalizedRecordFormatter rf = new DefaultBigQueryDenormalizedRecordFormatter( - jsonNodeSchema, new BigQuerySQLNameTransformer()); - final Schema expectedResult = Schema.of( - Field.of("name", LegacySQLTypeName.STRING), - Field.newBuilder("permission_list", LegacySQLTypeName.RECORD, - Field.of("domain", LegacySQLTypeName.STRING), - Field.newBuilder("grants", LegacySQLTypeName.STRING).setMode(Mode.REPEATED).build()) - .setMode(Mode.REPEATED).build(), - Field.of("_airbyte_ab_id", LegacySQLTypeName.STRING), - Field.of("_airbyte_emitted_at", LegacySQLTypeName.TIMESTAMP)); - - final Schema result = rf.getBigQuerySchema(jsonNodeSchema); - - assertEquals(expectedResult, result); - } - - @Test - void testSchemaWithReferenceDefinition() { - final JsonNode jsonNodeSchema = getSchemaWithReferenceDefinition(); - DefaultBigQueryDenormalizedRecordFormatter rf = new DefaultBigQueryDenormalizedRecordFormatter( - jsonNodeSchema, new BigQuerySQLNameTransformer()); - final Schema expectedResult = Schema.of( - Field.of("users", LegacySQLTypeName.STRING), - Field.of("_airbyte_ab_id", LegacySQLTypeName.STRING), - Field.of("_airbyte_emitted_at", LegacySQLTypeName.TIMESTAMP)); - - final Schema result = rf.getBigQuerySchema(jsonNodeSchema); - - assertEquals(expectedResult, result); - } - - @Test - void testSchemaWithNestedDatetimeInsideNullObject() { - final JsonNode jsonNodeSchema = getSchemaWithNestedDatetimeInsideNullObject(); - DefaultBigQueryDenormalizedRecordFormatter rf = new DefaultBigQueryDenormalizedRecordFormatter( - jsonNodeSchema, new BigQuerySQLNameTransformer()); - final Schema expectedResult = Schema.of( - Field.newBuilder("name", LegacySQLTypeName.STRING).setMode(Mode.NULLABLE).build(), - Field.newBuilder("appointment", LegacySQLTypeName.RECORD, - Field.newBuilder("street", LegacySQLTypeName.STRING).setMode(Mode.NULLABLE).build(), - Field.newBuilder("expTime", LegacySQLTypeName.DATETIME).setMode(Mode.NULLABLE).build()) - .setMode(Mode.NULLABLE).build(), - Field.of("_airbyte_ab_id", LegacySQLTypeName.STRING), - Field.of("_airbyte_emitted_at", LegacySQLTypeName.TIMESTAMP)); - - final Schema result = rf.getBigQuerySchema(jsonNodeSchema); - - assertEquals(expectedResult, result); - } - - @Test - public void testEmittedAtTimeConversion() { - final DefaultBigQueryDenormalizedRecordFormatter mockedFormatter = Mockito.mock( - DefaultBigQueryDenormalizedRecordFormatter.class, Mockito.CALLS_REAL_METHODS); - - final ObjectNode objectNode = mapper.createObjectNode(); - - final AirbyteRecordMessage airbyteRecordMessage = new AirbyteRecordMessage(); - airbyteRecordMessage.setEmittedAt(1602637589000L); - mockedFormatter.addAirbyteColumns(objectNode, airbyteRecordMessage); - - assertEquals("2020-10-14 01:06:29.000000+00:00", - objectNode.get(JavaBaseConstants.COLUMN_NAME_EMITTED_AT).textValue()); - } - - @Test - void formatRecord_objectType() throws JsonProcessingException { - final JsonNode jsonNodeSchema = getSchema(); - final DefaultBigQueryDenormalizedRecordFormatter rf = new DefaultBigQueryDenormalizedRecordFormatter( - jsonNodeSchema, new BigQuerySQLNameTransformer()); - final JsonNode objectNode = mapper.readTree(""" - {"name":"data"} - """); - final AirbyteRecordMessage airbyteRecordMessage = new AirbyteRecordMessage(); - airbyteRecordMessage.setEmittedAt(1602637589000L); - airbyteRecordMessage.setData(objectNode); - - final JsonNode result = rf.formatRecord(airbyteRecordMessage); - - assertNotNull(result); - assertTrue(result.has("name")); - assertEquals("data", result.get("name").textValue()); - assertEquals(JsonNodeType.STRING, result.get("name").getNodeType()); - } - - @Test - void formatRecord_containsRefDefinition() throws JsonProcessingException { - final JsonNode jsonNodeSchema = getSchema(); - DefaultBigQueryDenormalizedRecordFormatter rf = new DefaultBigQueryDenormalizedRecordFormatter( - jsonNodeSchema, new BigQuerySQLNameTransformer()); - rf.fieldsContainRefDefinitionValue.add("name"); - final JsonNode objectNode = mapper.readTree(""" - {"name":"data"} - """); - final AirbyteRecordMessage airbyteRecordMessage = new AirbyteRecordMessage(); - airbyteRecordMessage.setEmittedAt(1602637589000L); - airbyteRecordMessage.setData(objectNode); - - final JsonNode result = rf.formatRecord(airbyteRecordMessage); - - assertNotNull(result); - assertTrue(result.has("name")); - assertEquals("\"data\"", result.get("name").textValue()); - assertEquals(JsonNodeType.STRING, result.get("name").getNodeType()); - } - - @Test - void formatRecord_objectWithArray() throws JsonProcessingException { - final JsonNode jsonNodeSchema = getSchemaArrays(); - DefaultBigQueryDenormalizedRecordFormatter rf = new DefaultBigQueryDenormalizedRecordFormatter( - jsonNodeSchema, new BigQuerySQLNameTransformer()); - final JsonNode objectNode = mapper.readTree(""" - {"object_with_arrays":["array_3"]} - """); - final AirbyteRecordMessage airbyteRecordMessage = new AirbyteRecordMessage(); - airbyteRecordMessage.setEmittedAt(1602637589000L); - airbyteRecordMessage.setData(objectNode); - - final JsonNode result = rf.formatRecord(airbyteRecordMessage); - - assertNotNull(result); - assertTrue(result.has("object_with_arrays")); - result.has("object_with_arrays"); - assertEquals(JsonNodeType.ARRAY, result.get("object_with_arrays").getNodeType()); - assertNotNull(result.get("object_with_arrays").get(0)); - assertEquals(JsonNodeType.STRING, result.get("object_with_arrays").get(0).getNodeType()); - } - - @Test - void formatRecordNotObject_thenThrowsError() throws JsonProcessingException { - final JsonNode jsonNodeSchema = getSchema(); - DefaultBigQueryDenormalizedRecordFormatter rf = new DefaultBigQueryDenormalizedRecordFormatter( - jsonNodeSchema, new BigQuerySQLNameTransformer()); - final JsonNode arrayNode = mapper.readTree(""" - ["one"]"""); - - final AirbyteRecordMessage airbyteRecordMessage = new AirbyteRecordMessage(); - airbyteRecordMessage.setEmittedAt(1602637589000L); - airbyteRecordMessage.setData(arrayNode); - - assertThrows(IllegalArgumentException.class, () -> rf.formatRecord(airbyteRecordMessage)); - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/formatter/GcsBigQueryDenormalizedRecordFormatterTest.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/formatter/GcsBigQueryDenormalizedRecordFormatterTest.java deleted file mode 100644 index 74fa0c8df4d2..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/formatter/GcsBigQueryDenormalizedRecordFormatterTest.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery.formatter; - -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestSchemaUtils.getExpectedSchemaWithReferenceDefinition; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestSchemaUtils.getSchemaWithDateTime; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestSchemaUtils.getSchemaWithReferenceDefinition; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.cloud.bigquery.Field; -import com.google.cloud.bigquery.LegacySQLTypeName; -import com.google.cloud.bigquery.Schema; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.bigquery.BigQuerySQLNameTransformer; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -class GcsBigQueryDenormalizedRecordFormatterTest { - - @Test - void refReplacement() { - final JsonNode jsonNodeSchema = getSchemaWithReferenceDefinition(); - final JsonNode expectedResult = getExpectedSchemaWithReferenceDefinition(); - final GcsBigQueryDenormalizedRecordFormatter rf = new GcsBigQueryDenormalizedRecordFormatter( - jsonNodeSchema, new BigQuerySQLNameTransformer()); - - final JsonNode result = rf.formatJsonSchema(jsonNodeSchema); - - assertEquals(expectedResult, result); - } - - @Test - void dataTimeReplacement() { - final JsonNode jsonNodeSchema = getSchemaWithDateTime(); - final GcsBigQueryDenormalizedRecordFormatter rf = new GcsBigQueryDenormalizedRecordFormatter( - jsonNodeSchema, new BigQuerySQLNameTransformer()); - final Schema expectedResult = Schema.of( - Field.of("updated_at", LegacySQLTypeName.TIMESTAMP), - Field.of("items", LegacySQLTypeName.RECORD, Field.of("nested_datetime", LegacySQLTypeName.TIMESTAMP)), - Field.of("_airbyte_ab_id", LegacySQLTypeName.STRING), - Field.of("_airbyte_emitted_at", LegacySQLTypeName.TIMESTAMP)); - - final Schema result = rf.getBigQuerySchema(jsonNodeSchema); - - assertEquals(expectedResult, result); - } - - @Test - public void testEmittedAtTimeConversion() { - final GcsBigQueryDenormalizedRecordFormatter mockedFormatter = Mockito.mock( - GcsBigQueryDenormalizedRecordFormatter.class, Mockito.CALLS_REAL_METHODS); - - final ObjectMapper mapper = new ObjectMapper(); - final ObjectNode objectNode = mapper.createObjectNode(); - - final AirbyteRecordMessage airbyteRecordMessage = new AirbyteRecordMessage(); - airbyteRecordMessage.setEmittedAt(1602637589000L); - mockedFormatter.addAirbyteColumns(objectNode, airbyteRecordMessage); - - assertEquals("1602637589000", - objectNode.get(JavaBaseConstants.COLUMN_NAME_EMITTED_AT).asText()); - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/formatter/arrayformater/DefaultArrayFormatterTest.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/formatter/arrayformater/DefaultArrayFormatterTest.java deleted file mode 100644 index 152630bd7e4a..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/formatter/arrayformater/DefaultArrayFormatterTest.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery.formatter.arrayformater; - -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestSchemaUtils.getExpectedSchemaArrays; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestSchemaUtils.getSchemaArrays; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; -import org.junit.jupiter.api.Test; - -class DefaultArrayFormatterTest { - - private final DefaultArrayFormatter formatter = new DefaultArrayFormatter(); - private final ObjectMapper mapper = new ObjectMapper(); - - @Test - void surroundArraysByObjects() { - final JsonNode schemaArrays = getSchemaArrays(); - final JsonNode expectedSchemaArrays = getExpectedSchemaArrays(); - formatter.surroundArraysByObjects(schemaArrays); - assertEquals(expectedSchemaArrays, schemaArrays); - } - - @Test - void formatArrayItems() throws JsonProcessingException { - final JsonNode expectedArrayNode = mapper.readTree( - """ - [ - {"big_query_array": ["one", "two"]}, - {"big_query_array": ["one", "two"]} - ] - """); - final List arrayNodes = List.of( - mapper.readTree(""" - ["one", "two"]"""), - mapper.readTree(""" - ["one", "two"]""")); - - final JsonNode result = formatter.formatArrayItems(arrayNodes); - - assertEquals(expectedArrayNode, result); - } - - @Test - void formatArrayItems_notArray() throws JsonProcessingException { - final JsonNode objectNodeInput = mapper.readTree(""" - {"type":"object","items":{"type":"integer"}}"""); - final JsonNode expectedResult = mapper.readTree(""" - [{"type":"object","items":{"type":"integer"}}]"""); - - final JsonNode result = formatter.formatArrayItems(List.of(objectNodeInput)); - - assertEquals(expectedResult, result); - } - - @Test - void findArrays() throws JsonProcessingException { - final JsonNode schemaArrays = getSchemaArrays(); - final List expectedResult = List.of( - mapper.readTree(""" - {"type":["array"],"items":{"type":"integer"}}"""), - mapper.readTree(""" - {"type":["array"],"items":{"type":["array"],"items":{"type":"integer"}}}"""), - mapper.readTree(""" - {"type":["array"],"items":{"type":"integer"}}"""), - mapper.readTree(""" - {"type":["array"],"items":{"type":["array"],"items":{"type":["array"],"items":{"type":"integer"}}}}"""), - mapper.readTree(""" - {"type":["array"],"items":{"type":["array"],"items":{"type":"integer"}}}"""), - mapper.readTree(""" - {"type":["array"],"items":{"type":"integer"}}"""), - mapper.readTree( - """ - {"type":["array"],"items":{"type":["array"],"items":{"type":["array"],"items":{"type":["array"],"items":{"type":"integer"}}}}}"""), - mapper.readTree(""" - {"type":["array"],"items":{"type":["array"],"items":{"type":["array"],"items":{"type":"integer"}}}}"""), - mapper.readTree(""" - {"type":["array"],"items":{"type":["array"],"items":{"type":"integer"}}}"""), - mapper.readTree(""" - {"type":["array"],"items":{"type":"integer"}}""")); - - final List result = formatter.findArrays(schemaArrays); - assertEquals(expectedResult, result); - } - - @Test - void findArrays_null() { - final List result = formatter.findArrays(null); - assertTrue(result.isEmpty()); - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/formatter/arrayformater/LegacyArrayFormatterTest.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/formatter/arrayformater/LegacyArrayFormatterTest.java deleted file mode 100644 index 71d528c392f6..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/formatter/arrayformater/LegacyArrayFormatterTest.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery.formatter.arrayformater; - -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestSchemaUtils.getExpectedSchemaArraysLegacy; -import static io.airbyte.integrations.destination.bigquery.util.BigQueryDenormalizedTestSchemaUtils.getSchemaArrays; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.List; -import org.junit.jupiter.api.Test; - -public class LegacyArrayFormatterTest { - - private final LegacyArrayFormatter formatter = new LegacyArrayFormatter(); - private final ObjectMapper mapper = new ObjectMapper(); - - @Test - void surroundArraysByObjects() { - final JsonNode schemaArrays = getSchemaArrays(); - final JsonNode expectedSchemaArrays = getExpectedSchemaArraysLegacy(); - - formatter.surroundArraysByObjects(schemaArrays); - assertEquals(expectedSchemaArrays, schemaArrays); - } - - @Test - void findArrays() throws JsonProcessingException { - final JsonNode schemaArrays = getSchemaArrays(); - final List expectedResult = List.of( - mapper.readTree(""" - {"type":["array"],"items":{"type":"integer"}}"""), - mapper.readTree(""" - {"type":["array"],"items":{"type":["array"],"items":{"type":"integer"}}}"""), - mapper.readTree(""" - {"type":["array"],"items":{"type":"integer"}}"""), - mapper.readTree(""" - {"type":["array"],"items":{"type":["array"],"items":{"type":["array"],"items":{"type":"integer"}}}}"""), - mapper.readTree(""" - {"type":["array"],"items":{"type":["array"],"items":{"type":"integer"}}}"""), - mapper.readTree(""" - {"type":["array"],"items":{"type":"integer"}}"""), - mapper.readTree( - """ - {"type":["array"],"items":{"type":["array"],"items":{"type":["array"],"items":{"type":["array"],"items":{"type":"integer"}}}}}"""), - mapper.readTree(""" - {"type":["array"],"items":{"type":["array"],"items":{"type":["array"],"items":{"type":"integer"}}}}"""), - mapper.readTree(""" - {"type":["array"],"items":{"type":["array"],"items":{"type":"integer"}}}"""), - mapper.readTree(""" - {"type":["array"],"items":{"type":"integer"}}""")); - - final List result = formatter.findArrays(schemaArrays); - - assertEquals(expectedResult, result); - } - - @Test - void findArrays_null() { - final List result = formatter.findArrays(null); - assertTrue(result.isEmpty()); - } - - @Test - void formatArrayItems() throws JsonProcessingException { - final JsonNode expectedArrayNode = mapper.readTree( - """ - {"big_query_array": [["one", "two"], ["one", "two"]]} - """); - final List arrayNodes = List.of( - mapper.readTree(""" - ["one", "two"]"""), - mapper.readTree(""" - ["one", "two"]""")); - - final JsonNode result = formatter.formatArrayItems(arrayNodes); - - assertEquals(expectedArrayNode, result); - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/formatter/util/FormatterUtilTest.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/formatter/util/FormatterUtilTest.java deleted file mode 100644 index 188c8ef4b6d4..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/formatter/util/FormatterUtilTest.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery.formatter.util; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.Test; - -class FormatterUtilTest { - - private final ObjectMapper mapper = new ObjectMapper(); - - @Test - void isAirbyteArray_typeIsNull() throws JsonProcessingException { - final JsonNode arrayNode = mapper.readTree( - """ - ["one", "two"]"""); - - final boolean result = FormatterUtil.isAirbyteArray(arrayNode); - assertFalse(result); - } - - @Test - void isAirbyteArray_typeFieldIsArray() throws JsonProcessingException { - final JsonNode arrayNode = mapper.readTree(""" - {"type":["array"],"items":{"type":"integer"}}"""); - - boolean result = FormatterUtil.isAirbyteArray(arrayNode); - assertTrue(result); - } - - @Test - void isAirbyteArray_typeFieldIsNotArray() throws JsonProcessingException { - final JsonNode objectNode = mapper.readTree(""" - {"type":"object"}"""); - final boolean result = FormatterUtil.isAirbyteArray(objectNode); - assertFalse(result); - } - - @Test - void isAirbyteArray_textIsNotArray() throws JsonProcessingException { - final JsonNode arrayNode = mapper.readTree(""" - {"type":["notArrayText"]}"""); - final boolean result = FormatterUtil.isAirbyteArray(arrayNode); - assertFalse(result); - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/util/BigQueryDenormalizedTestSchemaUtils.java b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/util/BigQueryDenormalizedTestSchemaUtils.java deleted file mode 100644 index f2f25253477d..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/java/io/airbyte/integrations/destination/bigquery/util/BigQueryDenormalizedTestSchemaUtils.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery.util; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.Jsons; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; - -public class BigQueryDenormalizedTestSchemaUtils { - - private static final String JSON_FILES_BASE_LOCATION = "schemas/"; - - public static JsonNode getSchema() { - return getTestDataFromResourceJson("schema.json"); - } - - public static JsonNode getSchemaWithFormats() { - return getTestDataFromResourceJson("schemaWithFormats.json"); - } - - public static JsonNode getSchemaWithDateTime() { - return getTestDataFromResourceJson("schemaWithDateTime.json"); - } - - public static JsonNode getSchemaWithBigInteger() { - return getTestDataFromResourceJson("schemaWithBigInteger.json"); - } - - public static JsonNode getSchemaWithInvalidArrayType() { - return getTestDataFromResourceJson("schemaWithInvalidArrayType.json"); - } - - public static JsonNode getSchemaWithReferenceDefinition() { - return getTestDataFromResourceJson("schemaWithReferenceDefinition.json"); - } - - public static JsonNode getSchemaWithNestedDatetimeInsideNullObject() { - return getTestDataFromResourceJson("schemaWithNestedDatetimeInsideNullObject.json"); - } - - public static JsonNode getSchemaArrays() { - return getTestDataFromResourceJson("schemaArrays.json"); - } - - public static JsonNode getExpectedSchema() { - return getTestDataFromResourceJson("expectedSchema.json"); - } - - public static JsonNode getExpectedSchemaWithFormats() { - return getTestDataFromResourceJson("expectedSchemaWithFormats.json"); - } - - public static JsonNode getExpectedSchemaWithDateTime() { - return getTestDataFromResourceJson("expectedSchemaWithDateTime.json"); - } - - public static JsonNode getExpectedSchemaWithInvalidArrayType() { - return getTestDataFromResourceJson("expectedSchemaWithInvalidArrayType.json"); - } - - public static JsonNode getExpectedSchemaWithReferenceDefinition() { - return getTestDataFromResourceJson("expectedSchemaWithReferenceDefinition.json"); - } - - public static JsonNode getExpectedSchemaWithNestedDatetimeInsideNullObject() { - return getTestDataFromResourceJson("expectedSchemaWithNestedDatetimeInsideNullObject.json"); - } - - public static JsonNode getExpectedSchemaArrays() { - return getTestDataFromResourceJson("expectedSchemaArrays.json"); - } - - public static JsonNode getExpectedSchemaArraysLegacy() { - return getTestDataFromResourceJson("expectedSchemaArraysLegacy.json"); - } - - public static JsonNode getSchemaWithAllOf() { - return getTestDataFromResourceJson("schemaAllOf.json"); - } - - private static JsonNode getTestDataFromResourceJson(final String fileName) { - final String fileContent; - try { - fileContent = Files.readString(Path.of(BigQueryDenormalizedTestSchemaUtils.class.getClassLoader() - .getResource(JSON_FILES_BASE_LOCATION + fileName).getPath())); - } catch (final IOException e) { - throw new RuntimeException(e); - } - return Jsons.deserialize(fileContent); - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchema.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchema.json deleted file mode 100644 index 6534fc7a3d0e..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchema.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "type": ["object"], - "properties": { - "accepts_marketing_updated_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "name": { "type": ["string"] }, - "permission-list": { - "type": ["object"], - "properties": { - "big_query_array": { - "type": ["array"], - "items": { - "type": ["object"], - "properties": { - "domain": { "type": ["string"] }, - "grants": { - "type": ["object"], - "properties": { - "big_query_array": { - "type": ["array"], - "items": { "type": ["string"] } - } - } - } - } - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchemaArrays.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchemaArrays.json deleted file mode 100644 index 3a1b9f624266..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchemaArrays.json +++ /dev/null @@ -1,85 +0,0 @@ -{ - "type": ["object"], - "properties": { - "object_with_arrays": { - "type": ["object"], - "properties": { - "array_3": { - "type": ["array"], - "items": { - "type": "integer" - } - } - } - }, - "simple_string": { - "type": ["string"] - }, - "array_1": { - "type": ["array"], - "items": { - "type": ["object"], - "properties": { - "big_query_array": { - "type": ["array"], - "items": { - "type": "integer" - } - } - } - } - }, - "array_4": { - "type": ["array"], - "items": { - "type": ["object"], - "properties": { - "big_query_array": { - "type": ["array"], - "items": { - "type": ["object"], - "properties": { - "big_query_array": { - "type": ["array"], - "items": { - "type": "integer" - } - } - } - } - } - } - } - }, - "array_5": { - "type": ["array"], - "items": { - "type": ["object"], - "properties": { - "big_query_array": { - "type": ["array"], - "items": { - "type": ["object"], - "properties": { - "big_query_array": { - "type": ["array"], - "items": { - "type": ["object"], - "properties": { - "big_query_array": { - "type": ["array"], - "items": { - "type": "integer" - } - } - } - } - } - } - } - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchemaArraysLegacy.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchemaArraysLegacy.json deleted file mode 100644 index 15f76827e99c..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchemaArraysLegacy.json +++ /dev/null @@ -1,105 +0,0 @@ -{ - "type": ["object"], - "properties": { - "object_with_arrays": { - "type": ["object"], - "properties": { - "array_3": { - "type": ["object"], - "properties": { - "big_query_array": { - "type": ["array"], - "items": { - "type": "integer" - } - } - } - } - } - }, - "simple_string": { - "type": ["string"] - }, - "array_1": { - "type": ["object"], - "properties": { - "big_query_array": { - "type": ["array"], - "items": { - "type": ["object"], - "properties": { - "big_query_array": { - "type": ["array"], - "items": { - "type": "integer" - } - } - } - } - } - } - }, - "array_4": { - "type": ["object"], - "properties": { - "big_query_array": { - "type": ["array"], - "items": { - "type": ["object"], - "properties": { - "big_query_array": { - "type": ["array"], - "items": { - "type": ["object"], - "properties": { - "big_query_array": { - "type": ["array"], - "items": { - "type": "integer" - } - } - } - } - } - } - } - } - } - }, - "array_5": { - "type": ["object"], - "properties": { - "big_query_array": { - "type": ["array"], - "items": { - "type": ["object"], - "properties": { - "big_query_array": { - "type": ["array"], - "items": { - "type": ["object"], - "properties": { - "big_query_array": { - "type": ["array"], - "items": { - "type": ["object"], - "properties": { - "big_query_array": { - "type": ["array"], - "items": { - "type": "integer" - } - } - } - } - } - } - } - } - } - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchemaWithDateTime.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchemaWithDateTime.json deleted file mode 100644 index b9bb321012dc..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchemaWithDateTime.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "type": ["object"], - "properties": { - "updated_at": { - "type": ["string"], - "format": "date-time" - }, - "items": { - "type": ["object"], - "properties": { - "nested_datetime": { - "type": ["string"], - "format": "date-time" - } - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchemaWithFormats.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchemaWithFormats.json deleted file mode 100644 index 4f8c95bcdc16..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchemaWithFormats.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "type": ["object"], - "properties": { - "name": { - "type": ["string"] - }, - "date_of_birth": { - "type": ["string"], - "format": "date" - }, - "updated_at": { - "type": ["string"], - "format": "date-time" - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchemaWithInvalidArrayType.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchemaWithInvalidArrayType.json deleted file mode 100644 index 4ecf67dd3f7d..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchemaWithInvalidArrayType.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "type": ["object"], - "properties": { - "name": { - "type": ["string"] - }, - "permission-list": { - "type": ["array"], - "items": { - "type": ["object"], - "properties": { - "domain": { - "type": ["string"] - }, - "grants": { - "type": ["array"], - "items": { - "type": ["string"] - } - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchemaWithNestedDatetimeInsideNullObject.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchemaWithNestedDatetimeInsideNullObject.json deleted file mode 100644 index 625d12295b79..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchemaWithNestedDatetimeInsideNullObject.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "type": ["object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "appointment": { - "type": ["null", "object"], - "properties": { - "street": { - "type": ["null", "string"] - }, - "expTime": { - "type": ["null", "string"], - "format": "date-time" - } - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchemaWithReferenceDefinition.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchemaWithReferenceDefinition.json deleted file mode 100644 index 6145af26303b..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/expectedSchemaWithReferenceDefinition.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "type": ["null", "object"], - "properties": { - "users": { - "type": ["string"], - "$ref": "#/definitions/users_" - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schema.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schema.json deleted file mode 100644 index 6534fc7a3d0e..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schema.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "type": ["object"], - "properties": { - "accepts_marketing_updated_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "name": { "type": ["string"] }, - "permission-list": { - "type": ["object"], - "properties": { - "big_query_array": { - "type": ["array"], - "items": { - "type": ["object"], - "properties": { - "domain": { "type": ["string"] }, - "grants": { - "type": ["object"], - "properties": { - "big_query_array": { - "type": ["array"], - "items": { "type": ["string"] } - } - } - } - } - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaAllOf.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaAllOf.json deleted file mode 100644 index 84fe8c6393c3..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaAllOf.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "definitions": { - "schemaArray": { - "type": "array", - "items": { "$ref": "#" } - } - }, - "type": ["object", "boolean"], - "properties": { - "title": { - "type": "string" - }, - - "allOf": { "$ref": "#/definitions/schemaArray" } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaArrays.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaArrays.json deleted file mode 100644 index 92de3f8afdb9..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaArrays.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "type": ["object"], - "properties": { - "object_with_arrays": { - "type": ["object"], - "properties": { - "array_3": { - "type": ["array"], - "items": { - "type": "integer" - } - } - } - }, - "simple_string": { - "type": ["string"] - }, - "array_1": { - "type": ["array"], - "items": { - "type": ["array"], - "items": { - "type": "integer" - } - } - }, - "array_4": { - "type": ["array"], - "items": { - "type": ["array"], - "items": { - "type": ["array"], - "items": { - "type": "integer" - } - } - } - }, - "array_5": { - "type": ["array"], - "items": { - "type": ["array"], - "items": { - "type": ["array"], - "items": { - "type": ["array"], - "items": { - "type": "integer" - } - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaWithBigInteger.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaWithBigInteger.json deleted file mode 100644 index 2b1c48571bab..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaWithBigInteger.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "type": ["object"], - "properties": { - "salary": { - "type": ["number"], - "airbyte_type": "big_integer" - }, - "updated_at": { - "type": ["string"], - "format": "date-time" - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaWithDateTime.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaWithDateTime.json deleted file mode 100644 index b9bb321012dc..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaWithDateTime.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "type": ["object"], - "properties": { - "updated_at": { - "type": ["string"], - "format": "date-time" - }, - "items": { - "type": ["object"], - "properties": { - "nested_datetime": { - "type": ["string"], - "format": "date-time" - } - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaWithFormats.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaWithFormats.json deleted file mode 100644 index 4f8c95bcdc16..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaWithFormats.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "type": ["object"], - "properties": { - "name": { - "type": ["string"] - }, - "date_of_birth": { - "type": ["string"], - "format": "date" - }, - "updated_at": { - "type": ["string"], - "format": "date-time" - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaWithInvalidArrayType.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaWithInvalidArrayType.json deleted file mode 100644 index 5517ffb855aa..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaWithInvalidArrayType.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "type": ["object"], - "properties": { - "name": { - "type": ["string"] - }, - "permission-list": { - "type": ["array"], - "items": { - "type": ["object"], - "properties": { - "domain": { - "type": ["string"] - }, - "grants": { - "type": ["array"] - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaWithNestedDatetimeInsideNullObject.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaWithNestedDatetimeInsideNullObject.json deleted file mode 100644 index 625d12295b79..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaWithNestedDatetimeInsideNullObject.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "type": ["object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "appointment": { - "type": ["null", "object"], - "properties": { - "street": { - "type": ["null", "string"] - }, - "expTime": { - "type": ["null", "string"], - "format": "date-time" - } - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaWithReferenceDefinition.json b/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaWithReferenceDefinition.json deleted file mode 100644 index 64f5c21646f1..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery-denormalized/src/test/resources/schemas/schemaWithReferenceDefinition.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "type": ["null", "object"], - "properties": { - "users": { - "$ref": "#/definitions/users_" - } - } -} diff --git a/airbyte-integrations/connectors/destination-bigquery/.dockerignore b/airbyte-integrations/connectors/destination-bigquery/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile deleted file mode 100644 index 3117f0af1636..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ /dev/null @@ -1,54 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-bigquery - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -RUN yum install -y python3 python3-devel jq sshpass git && yum clean all && \ - alternatives --install /usr/bin/python python /usr/bin/python3 60 && \ - python -m ensurepip --upgrade && \ - # these two lines are a workaround for https://github.com/yaml/pyyaml/issues/601 - pip3 install wheel && \ - pip3 install "Cython<3.0" "pyyaml==5.4" --no-build-isolation && \ - pip3 install dbt-bigquery==1.0.0 - -# Luckily, none of normalization's files conflict with destination-bigquery's files :) -# We don't enforce that in any way, but hopefully we're only living in this state for a short time. -COPY --from=airbyte/normalization:0.4.3 /airbyte /airbyte -# Install python dependencies -WORKDIR /airbyte/base_python_structs -RUN pip3 install . -WORKDIR /airbyte/normalization_code -RUN pip3 install . -WORKDIR /airbyte/normalization_code/dbt-template/ -# Download external dbt dependencies -# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15 -RUN pip3 install "urllib3<2" -RUN dbt deps - -WORKDIR /airbyte - -ENV APPLICATION destination-bigquery -ENV AIRBYTE_NORMALIZATION_INTEGRATION bigquery - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=1.10.2 -LABEL io.airbyte.name=airbyte/destination-bigquery - -ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh" -ENTRYPOINT ["/airbyte/run_with_normalization.sh"] diff --git a/airbyte-integrations/connectors/destination-bigquery/README.md b/airbyte-integrations/connectors/destination-bigquery/README.md index a1c5877eafcb..f911b3a45412 100644 --- a/airbyte-integrations/connectors/destination-bigquery/README.md +++ b/airbyte-integrations/connectors/destination-bigquery/README.md @@ -16,10 +16,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-bigquery:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-bigquery:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-bigquery:dev`. the Dockerfile. #### Run @@ -56,35 +57,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - -## Uploading options -There are 2 available options to upload data to bigquery `Standard` and `GCS Staging`. -- `Standard` is option to upload data directly from your source to BigQuery storage. This way is faster and requires less resources than GCS one. -Please be aware you may see some fails for big datasets and slow sources, i.e. if reading from source takes more than 10-12 hours. -It may happen if you have a slow connection to source and\or migrate a very big dataset. If that's a case, then select a GCS Uploading type. -This is caused by the Google BigQuery SDK client limitations. For more details please check https://github.com/airbytehq/airbyte/issues/3549 -- `GCS Uploading (CSV format)`. This approach has been implemented in order to avoid the issue for big datasets mentioned above. -At the first step all data is uploaded to GCS bucket and then all moved to BigQuery at one shot stream by stream. -The destination-gcs connector is partially used under the hood here, so you may check its documentation for more details. -There is no sense to use this uploading method if your migration doesn't take more than 10 hours and if you don't see the error like this in logs: - -"PUT https://www.googleapis.com/upload/bigquery/v2/projects/some-project-name/jobs?uploadType=resumable&upload_id=some_randomly_generated_upload_id". - - -# BigQuery Test Configuration - -In order to test the BigQuery destination, you need a service account key file. - -## Community Contributor - -Follow the setup guide in the [docs](https://docs.airbyte.io/integrations/destinations/bigquery) to obtain credentials. - -## Airbyte Employee +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-bigquery test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/bigquery.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. -1. Access the `BigQuery Integration Test User` secret on Lastpass under the `Engineering` folder -1. Create a file with the contents at `secrets/credentials.json` diff --git a/airbyte-integrations/connectors/destination-bigquery/build.gradle b/airbyte-integrations/connectors/destination-bigquery/build.gradle index 5b0004d9e808..9d4c49a4163d 100644 --- a/airbyte-integrations/connectors/destination-bigquery/build.gradle +++ b/airbyte-integrations/connectors/destination-bigquery/build.gradle @@ -1,42 +1,54 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.12.0' + features = ['db-destinations', 's3-destinations', 'typing-deduping'] + useLocalCdk = false +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.bigquery.BigQueryDestination' - applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] + applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0', + '-XX:NativeMemoryTracking=detail', '-XX:+UnlockDiagnosticVMOptions', + '-XX:GCLockerRetryAllocationCount=100', +// '-Djava.rmi.server.hostname=localhost', +// '-Dcom.sun.management.jmxremote=true', +// '-Dcom.sun.management.jmxremote.port=6000', +// '-Dcom.sun.management.jmxremote.rmi.port=6000', +// '-Dcom.sun.management.jmxremote.local.only=false', +// '-Dcom.sun.management.jmxremote.authenticate=false', +// '-Dcom.sun.management.jmxremote.ssl=false' + ] } +airbyteJavaConnector.addCdkDependencies() + dependencies { - implementation 'io.airbyte:airbyte-cdk:0.0.1' + // TODO: Pull out common classes into CDK instead of depending on another destination + implementation project(':airbyte-integrations:connectors:destination-gcs') - implementation 'com.google.cloud:google-cloud-bigquery:2.27.0' + implementation 'com.google.cloud:google-cloud-bigquery:2.31.1' implementation 'org.apache.commons:commons-lang3:3.11' implementation 'org.apache.commons:commons-csv:1.4' implementation 'org.apache.commons:commons-text:1.10.0' - implementation group: 'org.apache.parquet', name: 'parquet-avro', version: '1.12.0' implementation group: 'com.google.cloud', name: 'google-cloud-storage', version: '2.4.5' implementation group: 'com.codepoetics', name: 'protonpack', version: '1.13' - implementation project(':airbyte-config-oss:config-models-oss') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java-s3') - implementation project(':airbyte-integrations:bases:base-typing-deduping') - implementation project(':airbyte-integrations:connectors:destination-gcs') - implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} - - testImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:bases:base-typing-deduping-test') + implementation (libs.airbyte.protocol) { + exclude group: 'io.airbyte', module: 'airbyte-commons' + } + // implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-bigquery') - integrationTestJavaImplementation project(':airbyte-db:db-lib') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + // This dependency is required because GCSOperaitons is leaking S3Client interface to the BigQueryDestination. + implementation libs.aws.java.sdk.s3 } configurations.all { diff --git a/airbyte-integrations/connectors/destination-bigquery/gradle.properties b/airbyte-integrations/connectors/destination-bigquery/gradle.properties new file mode 100644 index 000000000000..4dbe8b8729df --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/gradle.properties @@ -0,0 +1 @@ +testExecutionConcurrency=-1 diff --git a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml index 0e191513ea61..13878dc43277 100644 --- a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml +++ b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml @@ -1,35 +1,50 @@ data: + ab_internal: + ql: 300 + sl: 300 connectorSubtype: database connectorType: destination definitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 - dockerImageTag: 1.10.2 + dockerImageTag: 2.3.30 dockerRepository: airbyte/destination-bigquery + documentationUrl: https://docs.airbyte.com/integrations/destinations/bigquery githubIssueLabel: destination-bigquery icon: bigquery.svg license: ELv2 name: BigQuery - normalizationConfig: - normalizationIntegrationType: bigquery - normalizationRepository: airbyte/normalization - normalizationTag: 0.4.3 registries: cloud: enabled: true oss: enabled: true releaseStage: generally_available + releases: + breakingChanges: + 2.0.0: + message: + "**Do not upgrade until you have run a test upgrade as outlined [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#testing-destinations-v2-for-a-single-connection)**. + + This version introduces [Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2), + which provides better error handling, incremental delivery of data for large + syncs, and improved final table structures. To review the breaking changes, + and how to upgrade, see [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#quick-start-to-upgrading). + These changes will likely require updates to downstream dbt / SQL models, + which we walk through [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#updating-downstream-transformations). + + Selecting `Upgrade` will upgrade **all** connections using this destination + at their next sync. You can manually sync existing connections prior to + the next scheduled sync to start the upgrade early. + + " + upgradeDeadline: "2023-11-07" resourceRequirements: jobSpecific: - jobType: sync resourceRequirements: memory_limit: 1Gi memory_request: 1Gi - documentationUrl: https://docs.airbyte.com/integrations/destinations/bigquery + supportLevel: certified supportsDbt: true tags: - language:java - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryAsyncFlush.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryAsyncFlush.java new file mode 100644 index 000000000000..b31b69170929 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryAsyncFlush.java @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.bigquery; + +import io.airbyte.cdk.integrations.destination.record_buffer.FileBuffer; +import io.airbyte.cdk.integrations.destination.record_buffer.SerializableBuffer; +import io.airbyte.cdk.integrations.destination.s3.csv.CsvSerializedBuffer; +import io.airbyte.cdk.integrations.destination.s3.csv.StagingDatabaseCsvSheetGenerator; +import io.airbyte.cdk.integrations.destination_async.DestinationFlushFunction; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; +import io.airbyte.commons.json.Jsons; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.Map; +import java.util.stream.Stream; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; + +/** + * Async flushing logic. Flushing async prevents backpressure and is the superior flushing strategy. + */ +@Slf4j +class BigQueryAsyncFlush implements DestinationFlushFunction { + + private final Map streamDescToWriteConfig; + private final BigQueryStagingOperations stagingOperations; + private final ConfiguredAirbyteCatalog catalog; + + public BigQueryAsyncFlush( + final Map streamDescToWriteConfig, + final BigQueryStagingOperations stagingOperations, + final ConfiguredAirbyteCatalog catalog) { + this.streamDescToWriteConfig = streamDescToWriteConfig; + this.stagingOperations = stagingOperations; + this.catalog = catalog; + } + + @Override + public void flush(final StreamDescriptor decs, final Stream stream) throws Exception { + final SerializableBuffer writer; + try { + writer = new CsvSerializedBuffer( + new FileBuffer(CsvSerializedBuffer.CSV_GZ_SUFFIX), + new StagingDatabaseCsvSheetGenerator(true), + true); + + stream.forEach(record -> { + try { + writer.accept(record.getSerialized(), record.getRecord().getEmittedAt()); + } catch (final Exception e) { + throw new RuntimeException(e); + } + }); + } catch (final Exception e) { + throw new RuntimeException(e); + } + + writer.flush(); + log.info("Flushing CSV buffer for stream {} ({}) to staging", decs.getName(), FileUtils.byteCountToDisplaySize(writer.getByteCount())); + if (!streamDescToWriteConfig.containsKey(decs)) { + throw new IllegalArgumentException( + String.format("Message contained record from a stream that was not in the catalog. \ncatalog: %s", Jsons.serialize(catalog))); + } + + final BigQueryWriteConfig writeConfig = streamDescToWriteConfig.get(decs); + try { + final String stagedFileName = stagingOperations.uploadRecordsToStage(writeConfig.datasetId(), writeConfig.streamName(), writer); + + stagingOperations.copyIntoTableFromStage( + writeConfig.datasetId(), + writeConfig.streamName(), + writeConfig.targetTableId(), + writeConfig.tableSchema(), + stagedFileName); + } catch (final Exception e) { + log.error("Failed to flush and commit buffer data into destination's raw table", e); + throw new RuntimeException("Failed to upload buffer to stage and commit to destination", e); + } + + writer.close(); + } + + @Override + public long getOptimalBatchSizeBytes() { + // Chosen arbitrarily (mostly to match legacy behavior). We have no reason to believe a larger + // number would be worse. + // This was previously set to 25MB, which ran into rate-limiting issues: + // https://cloud.google.com/bigquery/quotas#standard_tables + // > Your project can make up to 1,500 table modifications per table per day + return 200 * 1024 * 1024; + } + + @Override + public long getQueueFlushThresholdBytes() { + return 200 * 1024 * 1024; + } + +} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryAsyncStandardFlush.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryAsyncStandardFlush.java new file mode 100644 index 000000000000..3b719c687fc9 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryAsyncStandardFlush.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.bigquery; + +import com.google.cloud.bigquery.BigQuery; +import com.google.common.util.concurrent.RateLimiter; +import io.airbyte.cdk.integrations.destination_async.DestinationFlushFunction; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; +import io.airbyte.integrations.destination.bigquery.uploader.AbstractBigQueryUploader; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; +import java.util.stream.Stream; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class BigQueryAsyncStandardFlush implements DestinationFlushFunction { + + // TODO remove this once the async framework supports rate-limiting/backpressuring + private static final RateLimiter rateLimiter = RateLimiter.create(0.07); + + private final BigQuery bigQuery; + private final Supplier>> uploaderMap; + + public BigQueryAsyncStandardFlush(final BigQuery bigQuery, + final Supplier>> uploaderMap) { + this.bigQuery = bigQuery; + this.uploaderMap = uploaderMap; + } + + @Override + public void flush(final StreamDescriptor decs, final Stream stream) throws Exception { + rateLimiter.acquire(); + final ConcurrentMap> uploaderMapSupplied = uploaderMap.get(); + final AtomicInteger recordCount = new AtomicInteger(); + stream.forEach(aibyteMessage -> { + try { + final AirbyteStreamNameNamespacePair sd = new AirbyteStreamNameNamespacePair(aibyteMessage.getRecord().getStream(), + aibyteMessage.getRecord().getNamespace()); + uploaderMapSupplied.get(sd).upload(aibyteMessage); + recordCount.getAndIncrement(); + } catch (final Exception e) { + log.error("An error happened while trying to flush a record to big query", e); + throw e; + } + }); + uploaderMapSupplied.values().forEach(test -> test.closeAfterPush()); + } + + @Override + public long getOptimalBatchSizeBytes() { + // todo(ryankfu): this should be per-destination specific. currently this is for Snowflake. + // The size chosen is currently for improving the performance of low memory connectors. With 1 Gi of + // resource the connector will usually at most fill up around 150 MB in a single queue. By lowering + // the batch size, the AsyncFlusher will flush in smaller batches which allows for memory to be + // freed earlier similar to a sliding window effect + return Double.valueOf(Runtime.getRuntime().maxMemory() * 0.2).longValue(); + } + +} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryAvroSerializedBuffer.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryAvroSerializedBuffer.java deleted file mode 100644 index 9f68f48e70fd..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryAvroSerializedBuffer.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.destination.bigquery.formatter.BigQueryRecordFormatter; -import io.airbyte.integrations.destination.record_buffer.BufferCreateFunction; -import io.airbyte.integrations.destination.record_buffer.BufferStorage; -import io.airbyte.integrations.destination.s3.avro.AvroSerializedBuffer; -import io.airbyte.integrations.destination.s3.avro.S3AvroFormatConfig; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import io.airbyte.protocol.models.v0.AirbyteStream; -import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; -import java.io.IOException; -import java.util.concurrent.Callable; -import java.util.function.BiFunction; -import java.util.function.Function; -import org.apache.avro.Schema; -import org.apache.avro.file.CodecFactory; - -/** - * This class differs from {@link AvroSerializedBuffer} in that 1) the Avro schema can be customized - * by the caller, and 2) the message is formatted by {@link BigQueryRecordFormatter}. In this way, - * this buffer satisfies the needs of both the standard and the denormalized BigQuery destinations. - */ -public class BigQueryAvroSerializedBuffer extends AvroSerializedBuffer { - - private final BigQueryRecordFormatter recordFormatter; - - public BigQueryAvroSerializedBuffer(final BufferStorage bufferStorage, - final CodecFactory codecFactory, - final Schema schema, - final BigQueryRecordFormatter recordFormatter) - throws Exception { - super(bufferStorage, codecFactory, schema); - this.recordFormatter = recordFormatter; - } - - @Override - protected void writeRecord(final AirbyteRecordMessage record) throws IOException { - dataFileWriter.append(avroRecordFactory.getAvroRecord(recordFormatter.formatRecord(record))); - } - - public static BufferCreateFunction createBufferFunction(final S3AvroFormatConfig config, - final Function recordFormatterCreator, - final BiFunction schemaCreator, - final Callable createStorageFunction) { - final CodecFactory codecFactory = config.getCodecFactory(); - return (pair, catalog) -> { - final AirbyteStream stream = catalog.getStreams() - .stream() - .filter(s -> s.getStream().getName().equals(pair.getName())) - .findFirst() - .orElseThrow(() -> new RuntimeException(String.format("No such stream %s.%s", pair.getNamespace(), pair.getName()))) - .getStream(); - final BigQueryRecordFormatter recordFormatter = recordFormatterCreator.apply(stream.getJsonSchema()); - final Schema schema = schemaCreator.apply(recordFormatter, pair); - return new BigQueryAvroSerializedBuffer(createStorageFunction.call(), codecFactory, schema, recordFormatter); - }; - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryConsts.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryConsts.java index 2aeaf17cfabc..8cc29dd511fa 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryConsts.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryConsts.java @@ -24,6 +24,8 @@ public class BigQueryConsts { public static final String KEEP_GCS_FILES = "keep_files_in_gcs-bucket"; public static final String KEEP_GCS_FILES_VAL = "Keep all tmp files in GCS"; + public static final String DISABLE_TYPE_DEDUPE = "disable_type_dedupe"; + public static final String NAMESPACE_PREFIX = "n"; // tests diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java index 5ab5589e4abb..edec86e60991 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryDestination.java @@ -8,36 +8,38 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.BigQueryOptions; import com.google.cloud.bigquery.Dataset; import com.google.cloud.bigquery.Job; import com.google.cloud.bigquery.QueryJobConfiguration; +import com.google.cloud.bigquery.TableId; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Charsets; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteExceptionHandler; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.TypingAndDedupingFlag; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.TypingAndDedupingFlag; import io.airbyte.integrations.base.destination.typing_deduping.CatalogParser; import io.airbyte.integrations.base.destination.typing_deduping.DefaultTyperDeduper; -import io.airbyte.integrations.base.destination.typing_deduping.NoopTyperDeduper; +import io.airbyte.integrations.base.destination.typing_deduping.NoOpTyperDeduperWithV1V2Migrations; import io.airbyte.integrations.base.destination.typing_deduping.ParsedCatalog; import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; import io.airbyte.integrations.base.destination.typing_deduping.TyperDeduper; -import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.integrations.destination.bigquery.formatter.BigQueryRecordFormatter; import io.airbyte.integrations.destination.bigquery.formatter.DefaultBigQueryRecordFormatter; -import io.airbyte.integrations.destination.bigquery.formatter.GcsAvroBigQueryRecordFormatter; import io.airbyte.integrations.destination.bigquery.formatter.GcsCsvBigQueryRecordFormatter; import io.airbyte.integrations.destination.bigquery.typing_deduping.BigQueryDestinationHandler; import io.airbyte.integrations.destination.bigquery.typing_deduping.BigQuerySqlGenerator; import io.airbyte.integrations.destination.bigquery.typing_deduping.BigQueryV1V2Migrator; -import io.airbyte.integrations.destination.bigquery.typing_deduping.BigQueryV2RawTableMigrator; +import io.airbyte.integrations.destination.bigquery.typing_deduping.BigQueryV2TableMigrator; import io.airbyte.integrations.destination.bigquery.uploader.AbstractBigQueryUploader; import io.airbyte.integrations.destination.bigquery.uploader.BigQueryUploaderFactory; import io.airbyte.integrations.destination.bigquery.uploader.UploaderType; @@ -46,10 +48,6 @@ import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.integrations.destination.gcs.GcsNameTransformer; import io.airbyte.integrations.destination.gcs.GcsStorageOperations; -import io.airbyte.integrations.destination.gcs.util.GcsUtils; -import io.airbyte.integrations.destination.record_buffer.BufferCreateFunction; -import io.airbyte.integrations.destination.record_buffer.FileBuffer; -import io.airbyte.integrations.destination.s3.avro.S3AvroFormatConfig; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; @@ -57,17 +55,18 @@ import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.DestinationSyncMode; import java.io.ByteArrayInputStream; import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.function.BiFunction; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.function.Consumer; import java.util.function.Function; -import org.apache.avro.Schema; +import java.util.function.Supplier; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.joda.time.DateTime; @@ -191,16 +190,19 @@ public static BigQuery getBigQuery(final JsonNode config) { } public static GoogleCredentials getServiceAccountCredentials(final JsonNode config) throws IOException { - if (!BigQueryUtils.isUsingJsonCredentials(config)) { + final JsonNode serviceAccountKey = config.get(BigQueryConsts.CONFIG_CREDS); + // Follows this order of resolution: + // https://cloud.google.com/java/docs/reference/google-auth-library/latest/com.google.auth.oauth2.GoogleCredentials#com_google_auth_oauth2_GoogleCredentials_getApplicationDefault + if (serviceAccountKey == null) { LOGGER.info("No service account key json is provided. It is required if you are using Airbyte cloud."); LOGGER.info("Using the default service account credential from environment."); return GoogleCredentials.getApplicationDefault(); } // The JSON credential can either be a raw JSON object, or a serialized JSON object. - final String credentialsString = config.get(BigQueryConsts.CONFIG_CREDS).isObject() - ? Jsons.serialize(config.get(BigQueryConsts.CONFIG_CREDS)) - : config.get(BigQueryConsts.CONFIG_CREDS).asText(); + final String credentialsString = serviceAccountKey.isObject() + ? Jsons.serialize(serviceAccountKey) + : serviceAccountKey.asText(); return GoogleCredentials.fromStream( new ByteArrayInputStream(credentialsString.getBytes(Charsets.UTF_8))); } @@ -216,94 +218,113 @@ public static GoogleCredentials getServiceAccountCredentials(final JsonNode conf @Override public AirbyteMessageConsumer getConsumer(final JsonNode config, final ConfiguredAirbyteCatalog catalog, - final Consumer outputRecordCollector) - throws Exception { - // Set the default namespace on streams with null namespace. This means we don't need to repeat this - // logic in the rest of the connector. - // (record messages still need to handle null namespaces though, which currently happens in e.g. - // BigQueryRecordConsumer#acceptTracked) - // This probably should be shared logic amongst destinations eventually. - for (final ConfiguredAirbyteStream stream : catalog.getStreams()) { - if (StringUtils.isEmpty(stream.getStream().getNamespace())) { - stream.getStream().withNamespace(BigQueryUtils.getDatasetId(config)); - } - } + final Consumer outputRecordCollector) { + throw new UnsupportedOperationException("Should use getSerializedMessageConsumer"); + } + @Override + public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final Consumer outputRecordCollector) + throws Exception { + final UploadingMethod uploadingMethod = BigQueryUtils.getLoadingMethod(config); + final String defaultNamespace = BigQueryUtils.getDatasetId(config); + setDefaultStreamNamespace(catalog, defaultNamespace); + final boolean disableTypeDedupe = BigQueryUtils.getDisableTypeDedupFlag(config); final String datasetLocation = BigQueryUtils.getDatasetLocation(config); - final BigQuerySqlGenerator sqlGenerator = new BigQuerySqlGenerator(datasetLocation); - final CatalogParser catalogParser; - if (TypingAndDedupingFlag.getRawNamespaceOverride(RAW_DATA_DATASET).isPresent()) { - catalogParser = new CatalogParser(sqlGenerator, TypingAndDedupingFlag.getRawNamespaceOverride(RAW_DATA_DATASET).get()); - } else { - catalogParser = new CatalogParser(sqlGenerator); - } - final ParsedCatalog parsedCatalog; - + final BigQuerySqlGenerator sqlGenerator = new BigQuerySqlGenerator(config.get(BigQueryConsts.CONFIG_PROJECT_ID).asText(), datasetLocation); + final ParsedCatalog parsedCatalog = parseCatalog(config, catalog, datasetLocation); final BigQuery bigquery = getBigQuery(config); - final TyperDeduper typerDeduper; - if (TypingAndDedupingFlag.isDestinationV2()) { - parsedCatalog = catalogParser.parseCatalog(catalog); - final BigQueryV1V2Migrator migrator = new BigQueryV1V2Migrator(bigquery, namingResolver); - final BigQueryV2RawTableMigrator v2RawTableMigrator = new BigQueryV2RawTableMigrator(bigquery); - typerDeduper = new DefaultTyperDeduper<>( - sqlGenerator, - new BigQueryDestinationHandler(bigquery, datasetLocation), - parsedCatalog, - migrator, - v2RawTableMigrator); - } else { - parsedCatalog = null; - typerDeduper = new NoopTyperDeduper(); + final TyperDeduper typerDeduper = buildTyperDeduper(sqlGenerator, parsedCatalog, bigquery, datasetLocation, disableTypeDedupe); + + AirbyteExceptionHandler.addAllStringsInConfigForDeinterpolation(config); + final JsonNode serviceAccountKey = config.get(BigQueryConsts.CONFIG_CREDS); + if (serviceAccountKey != null) { + // If the service account key is a non-null string, we will try to + // deserialize it. Otherwise, we will let the Google library find it in + // the environment during the client initialization. + if (serviceAccountKey.isTextual()) { + // There are cases where we fail to deserialize the service account key. In these cases, we + // shouldn't do anything. + // Google's creds library is more lenient with JSON-parsing than Jackson, and I'd rather just let it + // go. + Jsons.tryDeserialize(serviceAccountKey.asText()) + .ifPresent(AirbyteExceptionHandler::addAllStringsInConfigForDeinterpolation); + } else { + AirbyteExceptionHandler.addAllStringsInConfigForDeinterpolation(serviceAccountKey); + } } - final UploadingMethod uploadingMethod = BigQueryUtils.getLoadingMethod(config); if (uploadingMethod == UploadingMethod.STANDARD) { LOGGER.warn("The \"standard\" upload mode is not performant, and is not recommended for production. " + "Please use the GCS upload mode if you are syncing a large amount of data."); return getStandardRecordConsumer(bigquery, config, catalog, parsedCatalog, outputRecordCollector, typerDeduper); - } else { - return getGcsRecordConsumer(bigquery, config, catalog, parsedCatalog, outputRecordCollector, typerDeduper); } + + final StandardNameTransformer gcsNameTransformer = new GcsNameTransformer(); + final GcsDestinationConfig gcsConfig = BigQueryUtils.getGcsCsvDestinationConfig(config); + final UUID stagingId = UUID.randomUUID(); + final DateTime syncDatetime = DateTime.now(DateTimeZone.UTC); + final boolean keepStagingFiles = BigQueryUtils.isKeepFilesInGcs(config); + final GcsStorageOperations gcsOperations = new GcsStorageOperations(gcsNameTransformer, gcsConfig.getS3Client(), gcsConfig); + final BigQueryStagingOperations bigQueryGcsOperations = new BigQueryGcsOperations( + bigquery, + gcsNameTransformer, + gcsConfig, + gcsOperations, + stagingId, + syncDatetime, + keepStagingFiles); + + return new BigQueryStagingConsumerFactory().createAsync( + config, + catalog, + outputRecordCollector, + bigQueryGcsOperations, + getCsvRecordFormatterCreator(namingResolver), + namingResolver::getTmpTableName, + typerDeduper, + parsedCatalog, + BigQueryUtils.getDatasetId(config)); } - protected Map> getUploaderMap( - final BigQuery bigquery, - final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final ParsedCatalog parsedCatalog, - final boolean use1s1t) + protected Supplier>> getUploaderMap( + final BigQuery bigquery, + final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final ParsedCatalog parsedCatalog) throws IOException { - final Map> uploaderMap = new HashMap<>(); - for (final ConfiguredAirbyteStream configStream : catalog.getStreams()) { - final AirbyteStream stream = configStream.getStream(); - final StreamConfig parsedStream; - - final String streamName = stream.getName(); - final String targetTableName; - if (use1s1t) { + return () -> { + final ConcurrentMap> uploaderMap = new ConcurrentHashMap<>(); + for (final ConfiguredAirbyteStream configStream : catalog.getStreams()) { + final AirbyteStream stream = configStream.getStream(); + final StreamConfig parsedStream; + + final String targetTableName; + parsedStream = parsedCatalog.getStream(stream.getNamespace(), stream.getName()); targetTableName = parsedStream.id().rawName(); - } else { - parsedStream = null; - targetTableName = getTargetTableName(streamName); - } - - final UploaderConfig uploaderConfig = UploaderConfig - .builder() - .bigQuery(bigquery) - .configStream(configStream) - .parsedStream(parsedStream) - .config(config) - .formatterMap(getFormatterMap(stream.getJsonSchema())) - .tmpTableName(namingResolver.getTmpTableName(streamName)) - .targetTableName(targetTableName) - // This refers to whether this is BQ denormalized or not - .isDefaultAirbyteTmpSchema(isDefaultAirbyteTmpTableSchema()) - .build(); - putStreamIntoUploaderMap(stream, uploaderConfig, uploaderMap); - } - return uploaderMap; + final UploaderConfig uploaderConfig = UploaderConfig + .builder() + .bigQuery(bigquery) + .configStream(configStream) + .parsedStream(parsedStream) + .config(config) + .formatterMap(getFormatterMap(stream.getJsonSchema())) + .targetTableName(targetTableName) + // This refers to whether this is BQ denormalized or not + .isDefaultAirbyteTmpSchema(isDefaultAirbyteTmpTableSchema()) + .build(); + + try { + putStreamIntoUploaderMap(stream, uploaderConfig, uploaderMap); + } catch (final IOException e) { + throw new RuntimeException(e); + } + } + return uploaderMap; + }; } protected void putStreamIntoUploaderMap(final AirbyteStream stream, @@ -327,137 +348,117 @@ protected boolean isDefaultAirbyteTmpTableSchema() { } protected Map getFormatterMap(final JsonNode jsonSchema) { - return Map.of(UploaderType.STANDARD, new DefaultBigQueryRecordFormatter(jsonSchema, namingResolver), - UploaderType.CSV, new GcsCsvBigQueryRecordFormatter(jsonSchema, namingResolver), - UploaderType.AVRO, new GcsAvroBigQueryRecordFormatter(jsonSchema, namingResolver)); + return Map.of( + UploaderType.STANDARD, new DefaultBigQueryRecordFormatter(jsonSchema, namingResolver), + UploaderType.CSV, new GcsCsvBigQueryRecordFormatter(jsonSchema, namingResolver)); } - protected String getTargetTableName(final String streamName) { - return namingResolver.getRawTableName(streamName); - } - - private AirbyteMessageConsumer getStandardRecordConsumer(final BigQuery bigquery, - final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final ParsedCatalog parsedCatalog, - final Consumer outputRecordCollector, - final TyperDeduper typerDeduper) + private SerializedAirbyteMessageConsumer getStandardRecordConsumer(final BigQuery bigquery, + final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final ParsedCatalog parsedCatalog, + final Consumer outputRecordCollector, + final TyperDeduper typerDeduper) throws Exception { typerDeduper.prepareTables(); - final Map> writeConfigs = getUploaderMap( + final Supplier>> writeConfigs = getUploaderMap( bigquery, config, catalog, - parsedCatalog, - TypingAndDedupingFlag.isDestinationV2()); - - return new BigQueryRecordConsumer( - bigquery, - writeConfigs, - outputRecordCollector, - BigQueryUtils.getDatasetId(config), - typerDeduper, parsedCatalog); - } - public AirbyteMessageConsumer getGcsRecordConsumer(final BigQuery bigQuery, - final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final ParsedCatalog parsedCatalog, - final Consumer outputRecordCollector, - final TyperDeduper typerDeduper) - throws Exception { - final StandardNameTransformer gcsNameTransformer = new GcsNameTransformer(); - final GcsDestinationConfig gcsConfig = BigQueryUtils.getGcsAvroDestinationConfig(config); - final UUID stagingId = UUID.randomUUID(); - final DateTime syncDatetime = DateTime.now(DateTimeZone.UTC); - final boolean keepStagingFiles = BigQueryUtils.isKeepFilesInGcs(config); - final GcsStorageOperations gcsOperations = new GcsStorageOperations(gcsNameTransformer, gcsConfig.getS3Client(), gcsConfig); - final BigQueryStagingOperations bigQueryGcsOperations = new BigQueryGcsOperations( - bigQuery, - gcsNameTransformer, - gcsConfig, - gcsOperations, - stagingId, - syncDatetime, - keepStagingFiles); - final S3AvroFormatConfig avroFormatConfig = (S3AvroFormatConfig) gcsConfig.getFormatConfig(); - final Function recordFormatterCreator = getRecordFormatterCreator(namingResolver); - final int numberOfFileBuffers = getNumberOfFileBuffers(config); - - if (numberOfFileBuffers > FileBuffer.SOFT_CAP_CONCURRENT_STREAM_IN_BUFFER) { - LOGGER.warn(""" - Increasing the number of file buffers past {} can lead to increased performance but - leads to increased memory usage. If the number of file buffers exceeds the number - of streams {} this will create more buffers than necessary, leading to nonexistent gains - """, FileBuffer.SOFT_CAP_CONCURRENT_STREAM_IN_BUFFER, catalog.getStreams().size()); - } + final String bqNamespace = BigQueryUtils.getDatasetId(config); - final BufferCreateFunction onCreateBuffer = - BigQueryAvroSerializedBuffer.createBufferFunction( - avroFormatConfig, - recordFormatterCreator, - getAvroSchemaCreator(), - () -> new FileBuffer(S3AvroFormatConfig.DEFAULT_SUFFIX, numberOfFileBuffers)); - - LOGGER.info("Creating BigQuery staging message consumer with staging ID {} at {}", stagingId, syncDatetime); - - return new BigQueryStagingConsumerFactory().create( - config, - catalog, + return new BigQueryRecordStandardConsumer( outputRecordCollector, - bigQueryGcsOperations, - onCreateBuffer, - recordFormatterCreator, - namingResolver::getTmpTableName, - getTargetTableNameTransformer(namingResolver), - typerDeduper, - parsedCatalog, - BigQueryUtils.getDatasetId(config)); + () -> { + // Set up our raw tables + writeConfigs.get().forEach((streamId, uploader) -> { + final StreamConfig stream = parsedCatalog.getStream(streamId); + if (stream.destinationSyncMode() == DestinationSyncMode.OVERWRITE) { + // For streams in overwrite mode, truncate the raw table. + // non-1s1t syncs actually overwrite the raw table at the end of the sync, so we only do this in + // 1s1t mode. + final TableId rawTableId = TableId.of(stream.id().rawNamespace(), stream.id().rawName()); + LOGGER.info("Deleting Raw table {}", rawTableId); + if (!bigquery.delete(rawTableId)) { + LOGGER.info("Raw table {} not found, continuing with creation", rawTableId); + } + LOGGER.info("Creating table {}", rawTableId); + BigQueryUtils.createPartitionedTableIfNotExists(bigquery, rawTableId, DefaultBigQueryRecordFormatter.SCHEMA_V2); + } else { + uploader.createRawTable(); + } + }); + }, + (hasFailed, streamSyncSummaries) -> { + try { + Thread.sleep(30 * 1000); + typerDeduper.typeAndDedupe(streamSyncSummaries); + typerDeduper.commitFinalTables(); + typerDeduper.cleanup(); + } catch (final Exception e) { + throw new RuntimeException(e); + } + }, + bigquery, + catalog, + bqNamespace, + writeConfigs); } - protected BiFunction getAvroSchemaCreator() { - return (formatter, pair) -> GcsUtils.getDefaultAvroSchema(pair.getName(), pair.getNamespace(), true); + protected Function getCsvRecordFormatterCreator(final BigQuerySQLNameTransformer namingResolver) { + return streamSchema -> new GcsCsvBigQueryRecordFormatter(streamSchema, namingResolver); } - protected Function getRecordFormatterCreator(final BigQuerySQLNameTransformer namingResolver) { - return streamSchema -> new GcsAvroBigQueryRecordFormatter(streamSchema, namingResolver); + private void setDefaultStreamNamespace(final ConfiguredAirbyteCatalog catalog, final String namespace) { + // Set the default namespace on streams with null namespace. This means we don't need to repeat this + // logic in the rest of the connector. + // (record messages still need to handle null namespaces though, which currently happens in e.g. + // AsyncStreamConsumer#accept) + // This probably should be shared logic amongst destinations eventually. + for (final ConfiguredAirbyteStream stream : catalog.getStreams()) { + if (StringUtils.isEmpty(stream.getStream().getNamespace())) { + stream.getStream().withNamespace(namespace); + } + } } - protected Function getTargetTableNameTransformer(final BigQuerySQLNameTransformer namingResolver) { - return namingResolver::getRawTableName; + private ParsedCatalog parseCatalog(final JsonNode config, final ConfiguredAirbyteCatalog catalog, final String datasetLocation) { + final BigQuerySqlGenerator sqlGenerator = new BigQuerySqlGenerator(config.get(BigQueryConsts.CONFIG_PROJECT_ID).asText(), datasetLocation); + final CatalogParser catalogParser = TypingAndDedupingFlag.getRawNamespaceOverride(RAW_DATA_DATASET).isPresent() + ? new CatalogParser(sqlGenerator, TypingAndDedupingFlag.getRawNamespaceOverride(RAW_DATA_DATASET).get()) + : new CatalogParser(sqlGenerator); + + return catalogParser.parseCatalog(catalog); } - /** - * Retrieves user configured file buffer amount so as long it doesn't exceed the maximum number of - * file buffers and sets the minimum number to the default - *

      - * NOTE: If Out Of Memory Exceptions (OOME) occur, this can be a likely cause as this hard limit has - * not been thoroughly load tested across all instance sizes - * - * @param config user configurations - * @return number of file buffers if configured otherwise default - */ - @VisibleForTesting - public int getNumberOfFileBuffers(final JsonNode config) { - // This null check is probably redundant, but I don't want to gamble on that right now - if (config.hasNonNull(BigQueryConsts.LOADING_METHOD)) { - final JsonNode loadingMethodConfig = config.get(BigQueryConsts.LOADING_METHOD); - final int numOfFileBuffers; - if (loadingMethodConfig.has(FileBuffer.FILE_BUFFER_COUNT_KEY)) { - numOfFileBuffers = Math.min(loadingMethodConfig.get(FileBuffer.FILE_BUFFER_COUNT_KEY).asInt(), FileBuffer.MAX_CONCURRENT_STREAM_IN_BUFFER); - } else { - numOfFileBuffers = FileBuffer.DEFAULT_MAX_CONCURRENT_STREAM_IN_BUFFER; - } - // Only allows for values 10 <= numOfFileBuffers <= 50 - return Math.max(numOfFileBuffers, FileBuffer.DEFAULT_MAX_CONCURRENT_STREAM_IN_BUFFER); + private TyperDeduper buildTyperDeduper(final BigQuerySqlGenerator sqlGenerator, + final ParsedCatalog parsedCatalog, + final BigQuery bigquery, + final String datasetLocation, + final boolean disableTypeDedupe) { + final BigQueryV1V2Migrator migrator = new BigQueryV1V2Migrator(bigquery, namingResolver); + final BigQueryV2TableMigrator v2RawTableMigrator = new BigQueryV2TableMigrator(bigquery); + final BigQueryDestinationHandler destinationHandler = new BigQueryDestinationHandler(bigquery, datasetLocation); + + if (disableTypeDedupe) { + return new NoOpTyperDeduperWithV1V2Migrations<>( + sqlGenerator, destinationHandler, parsedCatalog, migrator, v2RawTableMigrator, 8); } - // Return the default if we fail to parse the config - return FileBuffer.DEFAULT_MAX_CONCURRENT_STREAM_IN_BUFFER; + return new DefaultTyperDeduper<>( + sqlGenerator, + destinationHandler, + parsedCatalog, + migrator, + v2RawTableMigrator, + 8); + } public static void main(final String[] args) throws Exception { + AirbyteExceptionHandler.addThrowableForDeinterpolation(BigQueryException.class); final Destination destination = new BigQueryDestination(); new IntegrationRunner(destination).run(args); } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java index cfce93abf486..a6f4f9a8c0cf 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsOperations.java @@ -13,12 +13,12 @@ import com.google.cloud.bigquery.LoadJobConfiguration; import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.TableId; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.record_buffer.SerializableBuffer; +import io.airbyte.cdk.integrations.util.ConnectorExceptionUtil; import io.airbyte.commons.exceptions.ConfigErrorException; -import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.integrations.destination.gcs.GcsStorageOperations; -import io.airbyte.integrations.destination.record_buffer.SerializableBuffer; -import io.airbyte.integrations.util.ConnectorExceptionUtil; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -118,8 +118,9 @@ public String uploadRecordsToStage(final String datasetId, final String stream, } /** - * Similar to COPY INTO within {@link io.airbyte.integrations.destination.staging.StagingOperations} - * which loads the data stored in the stage area into a target table in the destination + * Similar to COPY INTO within + * {@link io.airbyte.cdk.integrations.destination.staging.StagingOperations} which loads the data + * stored in the stage area into a target table in the destination * * Reference * https://googleapis.dev/java/google-cloud-clients/latest/index.html?com/google/cloud/bigquery/package-summary.html @@ -129,38 +130,37 @@ public void copyIntoTableFromStage(final String datasetId, final String stream, final TableId tableId, final Schema tableSchema, - final List stagedFiles) { + final String stagedFileName) { LOGGER.info("Uploading records from staging files to target table {} (dataset {}): {}", - tableId, datasetId, stagedFiles); - - stagedFiles.parallelStream().forEach(stagedFile -> { - final String fullFilePath = String.format("gs://%s/%s%s", gcsConfig.getBucketName(), getStagingFullPath(datasetId, stream), stagedFile); - LOGGER.info("Uploading staged file: {}", fullFilePath); - final LoadJobConfiguration configuration = LoadJobConfiguration.builder(tableId, fullFilePath) - .setFormatOptions(FormatOptions.avro()) - .setSchema(tableSchema) - .setWriteDisposition(WriteDisposition.WRITE_APPEND) - .setUseAvroLogicalTypes(true) - .build(); - - final Job loadJob = this.bigQuery.create(JobInfo.of(configuration)); - LOGGER.info("[{}] Created a new job to upload record(s) to target table {} (dataset {}): {}", loadJob.getJobId(), - tableId, datasetId, loadJob); - - try { - BigQueryUtils.waitForJobFinish(loadJob); - LOGGER.info("[{}] Target table {} (dataset {}) is successfully appended with staging files", loadJob.getJobId(), - tableId, datasetId); - } catch (final BigQueryException | InterruptedException e) { - throw new RuntimeException( - String.format("[%s] Failed to upload staging files to destination table %s (%s)", loadJob.getJobId(), - tableId, datasetId), - e); - } - }); + tableId, datasetId, stagedFileName); + + final String fullFilePath = String.format("gs://%s/%s%s", gcsConfig.getBucketName(), getStagingFullPath(datasetId, stream), stagedFileName); + LOGGER.info("Uploading staged file: {}", fullFilePath); + final LoadJobConfiguration configuration = LoadJobConfiguration.builder(tableId, fullFilePath) + .setFormatOptions(FormatOptions.csv()) + .setSchema(tableSchema) + .setWriteDisposition(WriteDisposition.WRITE_APPEND) + .setJobTimeoutMs(600000L) // 10 min + .build(); + + final Job loadJob = this.bigQuery.create(JobInfo.of(configuration)); + LOGGER.info("[{}] Created a new job to upload record(s) to target table {} (dataset {}): {}", loadJob.getJobId(), + tableId, datasetId, loadJob); + + try { + BigQueryUtils.waitForJobFinish(loadJob); + LOGGER.info("[{}] Target table {} (dataset {}) is successfully appended with staging files", loadJob.getJobId(), + tableId, datasetId); + } catch (final BigQueryException | InterruptedException e) { + throw new RuntimeException( + String.format("[%s] Failed to upload staging files to destination table %s (%s)", loadJob.getJobId(), + tableId, datasetId), + e); + } } @Override + @Deprecated public void cleanUpStage(final String datasetId, final String stream, final List stagedFiles) { if (keepStagingFiles) { return; diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordConsumer.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordConsumer.java index 23d34d929057..c9e477afa917 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordConsumer.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordConsumer.java @@ -6,10 +6,9 @@ import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.TableId; -import io.airbyte.commons.string.Strings; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; -import io.airbyte.integrations.base.TypingAndDedupingFlag; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.util.ConnectorExceptionUtil; import io.airbyte.integrations.base.destination.typing_deduping.ParsedCatalog; import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; import io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeOperationValve; @@ -31,7 +30,8 @@ /** * Record Consumer used for STANDARD INSERTS */ -public class BigQueryRecordConsumer extends FailureTrackingAirbyteMessageConsumer implements AirbyteMessageConsumer { +@SuppressWarnings("try") +class BigQueryRecordConsumer extends FailureTrackingAirbyteMessageConsumer implements AirbyteMessageConsumer { private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryRecordConsumer.class); @@ -43,7 +43,6 @@ public class BigQueryRecordConsumer extends FailureTrackingAirbyteMessageConsume private final TypeAndDedupeOperationValve streamTDValve = new TypeAndDedupeOperationValve(); private final ParsedCatalog catalog; - private final boolean use1s1t; private final TyperDeduper typerDeduper; public BigQueryRecordConsumer(final BigQuery bigquery, @@ -58,7 +57,6 @@ public BigQueryRecordConsumer(final BigQuery bigquery, this.defaultDatasetId = defaultDatasetId; this.typerDeduper = typerDeduper; this.catalog = catalog; - this.use1s1t = TypingAndDedupingFlag.isDestinationV2(); LOGGER.info("Got parsed catalog {}", catalog); LOGGER.info("Got canonical stream IDs {}", uploaderMap.keySet()); @@ -67,22 +65,20 @@ public BigQueryRecordConsumer(final BigQuery bigquery, @Override protected void startTracked() { // todo (cgardens) - move contents of #write into this method. - if (use1s1t) { - // Set up our raw tables - uploaderMap.forEach((streamId, uploader) -> { - final StreamConfig stream = catalog.getStream(streamId); - if (stream.destinationSyncMode() == DestinationSyncMode.OVERWRITE) { - // For streams in overwrite mode, truncate the raw table. - // non-1s1t syncs actually overwrite the raw table at the end of the sync, so we only do this in - // 1s1t mode. - final TableId rawTableId = TableId.of(stream.id().rawNamespace(), stream.id().rawName()); - bigquery.delete(rawTableId); - BigQueryUtils.createPartitionedTableIfNotExists(bigquery, rawTableId, DefaultBigQueryRecordFormatter.SCHEMA_V2); - } else { - uploader.createRawTable(); - } - }); - } + // Set up our raw tables + uploaderMap.forEach((streamId, uploader) -> { + final StreamConfig stream = catalog.getStream(streamId); + if (stream.destinationSyncMode() == DestinationSyncMode.OVERWRITE) { + // For streams in overwrite mode, truncate the raw table. + // non-1s1t syncs actually overwrite the raw table at the end of the sync, so we only do this in + // 1s1t mode. + final TableId rawTableId = TableId.of(stream.id().rawNamespace(), stream.id().rawName()); + bigquery.delete(rawTableId); + BigQueryUtils.createPartitionedTableIfNotExists(bigquery, rawTableId, DefaultBigQueryRecordFormatter.SCHEMA_V2); + } else { + uploader.createRawTable(); + } + }); } /** @@ -129,16 +125,16 @@ public void close(final boolean hasFailed) throws Exception { uploaderMap.forEach((streamId, uploader) -> { try { uploader.close(hasFailed, outputRecordCollector, lastStateMessage); - typerDeduper.typeAndDedupe(streamId.getNamespace(), streamId.getName()); + typerDeduper.typeAndDedupe(streamId.getNamespace(), streamId.getName(), true); } catch (final Exception e) { exceptionsThrown.add(e); LOGGER.error("Exception while closing uploader {}", uploader, e); } }); typerDeduper.commitFinalTables(); - if (!exceptionsThrown.isEmpty()) { - throw new RuntimeException(String.format("Exceptions thrown while closing consumer: %s", Strings.join(exceptionsThrown, "\n"))); - } + typerDeduper.cleanup(); + + ConnectorExceptionUtil.logAllAndThrowFirst("Exceptions thrown while closing consumer: ", exceptionsThrown); } } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordStandardConsumer.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordStandardConsumer.java new file mode 100644 index 000000000000..c0cd460cfdfa --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordStandardConsumer.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.bigquery; + +import com.google.cloud.bigquery.BigQuery; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.OnStartFunction; +import io.airbyte.cdk.integrations.destination_async.AsyncStreamConsumer; +import io.airbyte.cdk.integrations.destination_async.OnCloseFunction; +import io.airbyte.cdk.integrations.destination_async.buffers.BufferManager; +import io.airbyte.integrations.destination.bigquery.uploader.AbstractBigQueryUploader; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.Executors; +import java.util.function.Consumer; +import java.util.function.Supplier; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@SuppressWarnings("try") +public class BigQueryRecordStandardConsumer extends AsyncStreamConsumer { + + public BigQueryRecordStandardConsumer(Consumer outputRecordCollector, + OnStartFunction onStart, + OnCloseFunction onClose, + BigQuery bigQuery, + ConfiguredAirbyteCatalog catalog, + String defaultNamespace, + Supplier>> uploaderMap) { + super(outputRecordCollector, + onStart, + onClose, + new BigQueryAsyncStandardFlush(bigQuery, uploaderMap), + catalog, + new BufferManager((long) (Runtime.getRuntime().maxMemory() * 0.5)), + defaultNamespace, + Executors.newFixedThreadPool(2)); + } + +} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQuerySQLNameTransformer.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQuerySQLNameTransformer.java index 30b44ac1b6ae..3102c1da1089 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQuerySQLNameTransformer.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQuerySQLNameTransformer.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.bigquery; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; public class BigQuerySQLNameTransformer extends StandardNameTransformer { @@ -41,4 +41,9 @@ public String getNamespace(final String input) { return normalizedName; } + @Deprecated + public String getTmpTableName(final String streamName, final String randomSuffix) { + return convertStreamName("_airbyte_tmp" + "_" + randomSuffix + "_" + streamName); + } + } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingConsumerFactory.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingConsumerFactory.java index 22e910857e7a..a929bfbf095f 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingConsumerFactory.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingConsumerFactory.java @@ -4,122 +4,102 @@ package io.airbyte.integrations.destination.bigquery; -import static io.airbyte.integrations.base.JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Functions; import com.google.common.base.Preconditions; -import io.airbyte.commons.functional.CheckedConsumer; -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.TypingAndDedupingFlag; +import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.BufferedStreamConsumer; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.OnCloseFunction; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.OnStartFunction; +import io.airbyte.cdk.integrations.destination_async.AsyncStreamConsumer; +import io.airbyte.cdk.integrations.destination_async.buffers.BufferManager; import io.airbyte.integrations.base.destination.typing_deduping.ParsedCatalog; import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; -import io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeOperationValve; import io.airbyte.integrations.base.destination.typing_deduping.TyperDeduper; import io.airbyte.integrations.destination.bigquery.formatter.BigQueryRecordFormatter; -import io.airbyte.integrations.destination.buffered_stream_consumer.BufferedStreamConsumer; -import io.airbyte.integrations.destination.buffered_stream_consumer.OnCloseFunction; -import io.airbyte.integrations.destination.buffered_stream_consumer.OnStartFunction; -import io.airbyte.integrations.destination.record_buffer.BufferCreateFunction; -import io.airbyte.integrations.destination.record_buffer.FlushBufferFunction; -import io.airbyte.integrations.destination.record_buffer.SerializedBufferingStrategy; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStream; -import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.DestinationSyncMode; -import java.util.List; +import io.airbyte.protocol.models.v0.StreamDescriptor; import java.util.Map; import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; -import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * This class mimics the same functionality as - * {@link io.airbyte.integrations.destination.staging.StagingConsumerFactory} which likely should be - * placed into a commons package to be utilized across all ConsumerFactories + * {@link io.airbyte.cdk.integrations.destination.staging.StagingConsumerFactory} which likely + * should be placed into a commons package to be utilized across all ConsumerFactories */ public class BigQueryStagingConsumerFactory { private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryStagingConsumerFactory.class); - public AirbyteMessageConsumer create(final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final Consumer outputRecordCollector, - final BigQueryStagingOperations bigQueryGcsOperations, - final BufferCreateFunction onCreateBuffer, - final Function recordFormatterCreator, - final Function tmpTableNameTransformer, - final Function targetTableNameTransformer, - final TyperDeduper typerDeduper, - final ParsedCatalog parsedCatalog, - final String defaultNamespace) - throws Exception { - final Map writeConfigs = createWriteConfigs( + public SerializedAirbyteMessageConsumer createAsync( + final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final Consumer outputRecordCollector, + final BigQueryStagingOperations bigQueryGcsOperations, + final Function recordFormatterCreator, + final Function tmpTableNameTransformer, + final TyperDeduper typerDeduper, + final ParsedCatalog parsedCatalog, + final String defaultNamespace) { + final Map writeConfigsByDescriptor = createWriteConfigs( config, catalog, parsedCatalog, recordFormatterCreator, - tmpTableNameTransformer, - targetTableNameTransformer); + tmpTableNameTransformer); - CheckedConsumer typeAndDedupeStreamFunction = - incrementalTypingAndDedupingStreamConsumer(typerDeduper); - - return new BufferedStreamConsumer( + final var flusher = new BigQueryAsyncFlush(writeConfigsByDescriptor, bigQueryGcsOperations, catalog); + return new AsyncStreamConsumer( outputRecordCollector, - onStartFunction(bigQueryGcsOperations, writeConfigs, typerDeduper), - new SerializedBufferingStrategy( - onCreateBuffer, - catalog, - flushBufferFunction(bigQueryGcsOperations, writeConfigs, catalog, typeAndDedupeStreamFunction)), - onCloseFunction(bigQueryGcsOperations, writeConfigs, typerDeduper), + onStartFunction(bigQueryGcsOperations, writeConfigsByDescriptor, typerDeduper), + (hasFailed, recordCounts) -> { + try { + onCloseFunction(bigQueryGcsOperations, writeConfigsByDescriptor, typerDeduper).accept(hasFailed, recordCounts); + } catch (final Exception e) { + throw new RuntimeException(e); + } + }, + flusher, catalog, - json -> true, + new BufferManager(getBigQueryBufferMemoryLimit()), defaultNamespace); } - private CheckedConsumer incrementalTypingAndDedupingStreamConsumer(final TyperDeduper typerDeduper) { - final TypeAndDedupeOperationValve valve = new TypeAndDedupeOperationValve(); - return (streamId) -> { - if (!valve.containsKey(streamId)) { - valve.addStream(streamId); - } - if (valve.readyToTypeAndDedupe(streamId)) { - typerDeduper.typeAndDedupe(streamId.getNamespace(), streamId.getName()); - valve.updateTimeAndIncreaseInterval(streamId); - } - }; + /** + * Out BigQuery's uploader threads use a fair amount of memory. We believe this is largely due to + * the sdk client we use. + * + * @return number of bytes to make available for message buffering. + */ + private long getBigQueryBufferMemoryLimit() { + return (long) (Runtime.getRuntime().maxMemory() * 0.4); } - private Map createWriteConfigs(final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final ParsedCatalog parsedCatalog, - final Function recordFormatterCreator, - final Function tmpTableNameTransformer, - final Function targetTableNameTransformer) { + private Map createWriteConfigs(final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final ParsedCatalog parsedCatalog, + final Function recordFormatterCreator, + final Function tmpTableNameTransformer) { return catalog.getStreams().stream() .map(configuredStream -> { Preconditions.checkNotNull(configuredStream.getDestinationSyncMode(), "Undefined destination sync mode"); final AirbyteStream stream = configuredStream.getStream(); - final StreamConfig streamConfig; - if (TypingAndDedupingFlag.isDestinationV2()) { - streamConfig = parsedCatalog.getStream(stream.getNamespace(), stream.getName()); - } else { - streamConfig = null; - } + final StreamConfig streamConfig = parsedCatalog.getStream(stream.getNamespace(), stream.getName()); final String streamName = stream.getName(); final BigQueryRecordFormatter recordFormatter = recordFormatterCreator.apply(stream.getJsonSchema()); - final var internalTableNamespace = - TypingAndDedupingFlag.isDestinationV2() ? streamConfig.id().rawNamespace() : BigQueryUtils.sanitizeDatasetId(stream.getNamespace()); - final var targetTableName = - TypingAndDedupingFlag.isDestinationV2() ? streamConfig.id().rawName() : targetTableNameTransformer.apply(streamName); + final var internalTableNamespace = streamConfig.id().rawNamespace(); + final var targetTableName = streamConfig.id().rawName(); final BigQueryWriteConfig writeConfig = new BigQueryWriteConfig( streamName, @@ -136,7 +116,7 @@ private Map createWriteConf return writeConfig; }) .collect(Collectors.toMap( - c -> new AirbyteStreamNameNamespacePair(c.streamName(), c.namespace()), + c -> new StreamDescriptor().withName(c.streamName()).withNamespace(c.namespace()), Functions.identity())); } @@ -151,18 +131,16 @@ private Map createWriteConf * @param writeConfigs configuration settings used to describe how to write data and where it exists */ private OnStartFunction onStartFunction(final BigQueryStagingOperations bigQueryGcsOperations, - final Map writeConfigs, + final Map writeConfigs, final TyperDeduper typerDeduper) { return () -> { LOGGER.info("Preparing airbyte_raw tables in destination started for {} streams", writeConfigs.size()); - if (TypingAndDedupingFlag.isDestinationV2()) { - typerDeduper.prepareTables(); - } + typerDeduper.prepareTables(); for (final BigQueryWriteConfig writeConfig : writeConfigs.values()) { LOGGER.info("Preparing staging are in destination for schema: {}, stream: {}, target table: {}, stage: {}", writeConfig.tableSchema(), writeConfig.streamName(), writeConfig.targetTableId(), writeConfig.streamName()); // In Destinations V2, we will always use the 'airbyte' schema/namespace for raw tables - final String rawDatasetId = TypingAndDedupingFlag.isDestinationV2() ? DEFAULT_AIRBYTE_INTERNAL_NAMESPACE : writeConfig.datasetId(); + final String rawDatasetId = DEFAULT_AIRBYTE_INTERNAL_NAMESPACE; // Regardless, ensure the schema the customer wants to write to exists bigQueryGcsOperations.createSchemaIfNotExists(writeConfig.datasetId(), writeConfig.datasetLocation()); // Schema used for raw and airbyte internal tables @@ -182,49 +160,6 @@ private OnStartFunction onStartFunction(final BigQueryStagingOperations bigQuery }; } - /** - * Flushes buffer data, writes to staging environment then proceeds to upload those same records to - * destination table - * - * @param bigQueryGcsOperations collection of utility SQL operations - * @param writeConfigs book keeping configurations for writing and storing state to write records - * @param catalog configured Airbyte catalog - */ - private FlushBufferFunction flushBufferFunction( - final BigQueryStagingOperations bigQueryGcsOperations, - final Map writeConfigs, - final ConfiguredAirbyteCatalog catalog, - final CheckedConsumer incrementalTypeAndDedupeConsumer) { - return (pair, writer) -> { - LOGGER.info("Flushing buffer for stream {} ({}) to staging", pair.getName(), FileUtils.byteCountToDisplaySize(writer.getByteCount())); - if (!writeConfigs.containsKey(pair)) { - throw new IllegalArgumentException( - String.format("Message contained record from a stream that was not in the catalog: %s.\nKeys: %s\ncatalog: %s", pair, - writeConfigs.keySet(), Jsons.serialize(catalog))); - } - - final BigQueryWriteConfig writeConfig = writeConfigs.get(pair); - final String datasetId = writeConfig.datasetId(); - final String stream = writeConfig.streamName(); - try (writer) { - writer.flush(); - final String stagedFile = bigQueryGcsOperations.uploadRecordsToStage(datasetId, stream, writer); - /* - * The primary reason for still adding staged files despite immediately uploading the staged file to - * the destination's raw table is because the cleanup for the staged files will occur at the end of - * the sync - */ - writeConfig.addStagedFile(stagedFile); - bigQueryGcsOperations.copyIntoTableFromStage(datasetId, stream, writeConfig.targetTableId(), writeConfig.tableSchema(), - List.of(stagedFile)); - incrementalTypeAndDedupeConsumer.accept(new AirbyteStreamNameNamespacePair(writeConfig.streamName(), writeConfig.namespace())); - } catch (final Exception e) { - LOGGER.error("Failed to flush and commit buffer data into destination's raw table:", e); - throw new RuntimeException("Failed to upload buffer to stage and commit to destination", e); - } - }; - } - /** * Tear down process, will attempt to clean out any staging area * @@ -232,21 +167,21 @@ private FlushBufferFunction flushBufferFunction( * @param writeConfigs configuration settings used to describe how to write data and where it exists */ private OnCloseFunction onCloseFunction(final BigQueryStagingOperations bigQueryGcsOperations, - final Map writeConfigs, + final Map writeConfigs, final TyperDeduper typerDeduper) { - return (hasFailed) -> { + return (hasFailed, streamSyncSummaries) -> { /* * Previously the hasFailed value was used to commit any remaining staged files into destination, * however, with the changes to checkpointing this will no longer be necessary since despite partial * successes, we'll be committing the target table (aka airbyte_raw) table throughout the sync */ - + typerDeduper.typeAndDedupe(streamSyncSummaries); LOGGER.info("Cleaning up destination started for {} streams", writeConfigs.size()); - for (final Map.Entry entry : writeConfigs.entrySet()) { - typerDeduper.typeAndDedupe(entry.getKey().getNamespace(), entry.getKey().getName()); + for (final Map.Entry entry : writeConfigs.entrySet()) { bigQueryGcsOperations.dropStageIfExists(entry.getValue().datasetId(), entry.getValue().streamName()); } typerDeduper.commitFinalTables(); + typerDeduper.cleanup(); LOGGER.info("Cleaning up destination completed."); }; } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingOperations.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingOperations.java index 65339ea10ffd..bd6b1e09ef47 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingOperations.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryStagingOperations.java @@ -6,7 +6,7 @@ import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.TableId; -import io.airbyte.integrations.destination.record_buffer.SerializableBuffer; +import io.airbyte.cdk.integrations.destination.record_buffer.SerializableBuffer; import java.util.List; /** @@ -14,7 +14,7 @@ * {@link io.airbyte.integrations.destination.s3.BlobStorageOperations}. * *

      - * Similar interface to {@link io.airbyte.integrations.destination.jdbc.SqlOperations} + * Similar interface to {@link io.airbyte.cdk.integrations.destination.jdbc.SqlOperations} *

      */ public interface BigQueryStagingOperations { @@ -55,14 +55,14 @@ public interface BigQueryStagingOperations { * @param stream Name of stream * @param tableId Name of destination's target table * @param schema Schema of the data being synced - * @param stagedFiles collection of staged files + * @param stagedFileName name of staged file * @throws Exception */ void copyIntoTableFromStage(final String datasetId, final String stream, final TableId tableId, final Schema schema, - final List stagedFiles) + final String stagedFileName) throws Exception; /** diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java index 57fe09be3282..3377acceb1da 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryUtils.java @@ -19,6 +19,7 @@ import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.FieldList; import com.google.cloud.bigquery.InsertAllRequest; +import com.google.cloud.bigquery.InsertAllRequest.RowToInsert; import com.google.cloud.bigquery.InsertAllResponse; import com.google.cloud.bigquery.Job; import com.google.cloud.bigquery.JobId; @@ -35,10 +36,10 @@ import com.google.cloud.bigquery.TimePartitioning; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.base.AirbyteExceptionHandler; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.base.TypingAndDedupingFlag; import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.protocol.models.v0.DestinationSyncMode; import java.time.Instant; @@ -85,7 +86,7 @@ public static ImmutablePair executeQuery(final Job queryJob) { throw new RuntimeException("Job no longer exists"); } else if (completedJob.getStatus().getError() != null) { // You can also look at queryJob.getStatus().getExecutionErrors() for all - // errors, not just the latest one. + // errors and not just the latest one. return ImmutablePair.of(null, (completedJob.getStatus().getError().toString())); } @@ -94,7 +95,9 @@ public static ImmutablePair executeQuery(final Job queryJob) { static Job waitForQuery(final Job queryJob) { try { - return queryJob.waitFor(); + final Job job = queryJob.waitFor(); + AirbyteExceptionHandler.addStringForDeinterpolation(job.getEtag()); + return job; } catch (final Exception e) { LOGGER.error("Failed to wait for a query job:" + queryJob); throw new RuntimeException(e); @@ -104,14 +107,12 @@ static Job waitForQuery(final Job queryJob) { public static void createSchemaAndTableIfNeeded(final BigQuery bigquery, final Set existingSchemas, final String schemaName, - final TableId tmpTableId, final String datasetLocation, final Schema schema) { if (!existingSchemas.contains(schemaName)) { getOrCreateDataset(bigquery, schemaName, datasetLocation); existingSchemas.add(schemaName); } - BigQueryUtils.createPartitionedTableIfNotExists(bigquery, tmpTableId, schema); } public static Dataset getOrCreateDataset(final BigQuery bigquery, final String datasetId, final String datasetLocation) { @@ -159,13 +160,15 @@ private static void attemptCreateTableAndTestInsert(final BigQuery bigquery, fin CHECK_TEST_TMP_TABLE_NAME, testTableSchema); // Try to make test (dummy records) insert to make sure that user has required permissions + // Use ids for BigQuery client to attempt idempotent retries. + // See https://github.com/airbytehq/airbyte/issues/33982 try { final InsertAllResponse response = bigquery.insertAll(InsertAllRequest .newBuilder(test_connection_table_name) - .addRow(Map.of("id", 1, "name", "James")) - .addRow(Map.of("id", 2, "name", "Eugene")) - .addRow(Map.of("id", 3, "name", "Angelina")) + .addRow(RowToInsert.of("1", ImmutableMap.of("id", 1, "name", "James"))) + .addRow(RowToInsert.of("2", ImmutableMap.of("id", 2, "name", "Eugene"))) + .addRow(RowToInsert.of("3", ImmutableMap.of("id", 3, "name", "Angelina"))) .build()); if (response.hasErrors()) { @@ -175,6 +178,7 @@ private static void attemptCreateTableAndTestInsert(final BigQuery bigquery, fin } } } catch (final BigQueryException e) { + LOGGER.error("Dummy inserts in check failed", e); throw new ConfigErrorException("Failed to check connection: \n" + e.getMessage()); } finally { test_connection_table_name.delete(); @@ -203,8 +207,7 @@ public static Table createTable(final BigQuery bigquery, final String datasetNam */ static void createPartitionedTableIfNotExists(final BigQuery bigquery, final TableId tableId, final Schema schema) { try { - final var chunkingColumn = - TypingAndDedupingFlag.isDestinationV2() ? JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT : JavaBaseConstants.COLUMN_NAME_EMITTED_AT; + final var chunkingColumn = JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT; final TimePartitioning partitioning = TimePartitioning.newBuilder(TimePartitioning.Type.DAY) .setField(chunkingColumn) .build(); @@ -248,7 +251,7 @@ public static JsonNode getGcsJsonNodeConfig(final JsonNode config) { + "}")) .build()); - LOGGER.debug("Composed GCS config is: \n" + gcsJsonNode.toPrettyString()); + // Do not log the gcsJsonNode to avoid accidentally emitting credentials (even at DEBUG/TRACE level) return gcsJsonNode; } @@ -256,6 +259,10 @@ public static GcsDestinationConfig getGcsAvroDestinationConfig(final JsonNode co return GcsDestinationConfig.getGcsDestinationConfig(getGcsAvroJsonNodeConfig(config)); } + public static GcsDestinationConfig getGcsCsvDestinationConfig(final JsonNode config) { + return GcsDestinationConfig.getGcsDestinationConfig(getGcsJsonNodeConfig(config)); + } + public static JsonNode getGcsAvroJsonNodeConfig(final JsonNode config) { final JsonNode loadingMethod = config.get(BigQueryConsts.LOADING_METHOD); final JsonNode gcsJsonNode = Jsons.jsonNode(ImmutableMap.builder() @@ -303,6 +310,14 @@ public static String getDatasetLocation(final JsonNode config) { } } + public static boolean getDisableTypeDedupFlag(final JsonNode config) { + if (config.has(BigQueryConsts.DISABLE_TYPE_DEDUPE)) { + return config.get(BigQueryConsts.DISABLE_TYPE_DEDUPE).asBoolean(false); + } + + return false; + } + static TableDefinition getTableDefinition(final BigQuery bigquery, final String datasetName, final String tableName) { final TableId tableId = TableId.of(datasetName, tableName); return bigquery.getTable(tableId).getDefinition(); @@ -381,18 +396,6 @@ public static JobInfo.WriteDisposition getWriteDisposition(final DestinationSync } } - public static boolean isUsingJsonCredentials(final JsonNode config) { - if (!config.has(BigQueryConsts.CONFIG_CREDS)) { - return false; - } - final JsonNode json = config.get(BigQueryConsts.CONFIG_CREDS); - if (json.isTextual()) { - return !json.asText().isEmpty(); - } else { - return !Jsons.serialize(json).isEmpty(); - } - } - // https://googleapis.dev/python/bigquery/latest/generated/google.cloud.bigquery.client.Client.html public static Integer getBigQueryClientChunkSize(final JsonNode config) { Integer chunkSizeFromConfig = null; @@ -433,6 +436,7 @@ public static boolean isKeepFilesInGcs(final JsonNode config) { public static void waitForJobFinish(final Job job) throws InterruptedException { if (job != null) { + AirbyteExceptionHandler.addStringForDeinterpolation(job.getEtag()); try { LOGGER.info("Waiting for job finish {}. Status: {}", job, job.getStatus()); job.waitFor(); diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryWriteConfig.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryWriteConfig.java index 965ac87980fb..7e51dcdfce74 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryWriteConfig.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryWriteConfig.java @@ -7,10 +7,6 @@ import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.TableId; import io.airbyte.protocol.models.v0.DestinationSyncMode; -import java.util.ArrayList; -import java.util.List; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * @param streamName output stream name @@ -21,7 +17,6 @@ * @param targetTableId BigQuery final raw table * @param tableSchema schema for the table * @param syncMode BigQuery's mapping of write modes to Airbyte's sync mode - * @param stagedFiles collection of staged files to copy data from */ public record BigQueryWriteConfig( String streamName, @@ -31,10 +26,7 @@ public record BigQueryWriteConfig( TableId tmpTableId, TableId targetTableId, Schema tableSchema, - DestinationSyncMode syncMode, - List stagedFiles) { - - private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryWriteConfig.class); + DestinationSyncMode syncMode) { public BigQueryWriteConfig(final String streamName, final String namespace, @@ -52,17 +44,7 @@ public BigQueryWriteConfig(final String streamName, TableId.of(datasetId, tmpTableName), TableId.of(datasetId, targetTableName), tableSchema, - syncMode, - new ArrayList<>()); - } - - public void addStagedFile(final String file) { - this.stagedFiles.add(file); - LOGGER.info("Added staged file: {}", file); - } - - public void clearStagedFiles() { - this.stagedFiles.clear(); + syncMode); } } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/BigQueryRecordFormatter.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/BigQueryRecordFormatter.java index 1d8048330313..703f24dca816 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/BigQueryRecordFormatter.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/BigQueryRecordFormatter.java @@ -6,7 +6,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.cloud.bigquery.Schema; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.util.HashMap; import java.util.HashSet; @@ -48,6 +49,10 @@ protected JsonNode formatJsonSchema(final JsonNode jsonSchema) { public abstract JsonNode formatRecord(AirbyteRecordMessage recordMessage); + public String formatRecord(PartialAirbyteMessage recordMessage) { + return ""; + } + public Schema getBigQuerySchema() { if (bigQuerySchema == null) { bigQuerySchema = getBigQuerySchema(jsonSchema); diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryRecordFormatter.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryRecordFormatter.java index 569a2772fdeb..2b3394aa5be3 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryRecordFormatter.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/DefaultBigQueryRecordFormatter.java @@ -9,13 +9,13 @@ import com.google.cloud.bigquery.QueryParameterValue; import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.StandardSQLTypeName; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteRecordMessage; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.base.TypingAndDedupingFlag; -import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.util.HashMap; -import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -25,11 +25,6 @@ */ public class DefaultBigQueryRecordFormatter extends BigQueryRecordFormatter { - public static final com.google.cloud.bigquery.Schema SCHEMA = com.google.cloud.bigquery.Schema.of( - Field.of(JavaBaseConstants.COLUMN_NAME_AB_ID, StandardSQLTypeName.STRING), - Field.of(JavaBaseConstants.COLUMN_NAME_EMITTED_AT, StandardSQLTypeName.TIMESTAMP), - Field.of(JavaBaseConstants.COLUMN_NAME_DATA, StandardSQLTypeName.STRING)); - public static final com.google.cloud.bigquery.Schema SCHEMA_V2 = com.google.cloud.bigquery.Schema.of( Field.of(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, StandardSQLTypeName.STRING), Field.of(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT, StandardSQLTypeName.TIMESTAMP), @@ -42,20 +37,31 @@ public DefaultBigQueryRecordFormatter(final JsonNode jsonSchema, final StandardN @Override public JsonNode formatRecord(final AirbyteRecordMessage recordMessage) { - if (TypingAndDedupingFlag.isDestinationV2()) { - // Map.of has a @NonNull requirement, so creating a new Hash map - final HashMap destinationV2record = new HashMap<>(); - destinationV2record.put(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, UUID.randomUUID().toString()); - destinationV2record.put(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT, getEmittedAtField(recordMessage)); - destinationV2record.put(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT, null); - destinationV2record.put(JavaBaseConstants.COLUMN_NAME_DATA, getData(recordMessage)); - return Jsons.jsonNode(destinationV2record); - } else { - return Jsons.jsonNode(Map.of( - JavaBaseConstants.COLUMN_NAME_AB_ID, UUID.randomUUID().toString(), - JavaBaseConstants.COLUMN_NAME_EMITTED_AT, getEmittedAtField(recordMessage), - JavaBaseConstants.COLUMN_NAME_DATA, getData(recordMessage))); - } + // Map.of has a @NonNull requirement, so creating a new Hash map + final HashMap destinationV2record = new HashMap<>(); + destinationV2record.put(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, UUID.randomUUID().toString()); + destinationV2record.put(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT, getEmittedAtField(recordMessage)); + destinationV2record.put(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT, null); + destinationV2record.put(JavaBaseConstants.COLUMN_NAME_DATA, getData(recordMessage)); + return Jsons.jsonNode(destinationV2record); + } + + @Override + public String formatRecord(PartialAirbyteMessage message) { + // Map.of has a @NonNull requirement, so creating a new Hash map + final HashMap destinationV2record = new HashMap<>(); + destinationV2record.put(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, UUID.randomUUID().toString()); + destinationV2record.put(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT, getEmittedAtField(message.getRecord())); + destinationV2record.put(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT, null); + destinationV2record.put(JavaBaseConstants.COLUMN_NAME_DATA, message.getSerialized()); + return Jsons.serialize(destinationV2record); + } + + protected Object getEmittedAtField(final PartialAirbyteRecordMessage recordMessage) { + // Bigquery represents TIMESTAMP to the microsecond precision, so we convert to microseconds then + // use BQ helpers to string-format correctly. + final long emittedAtMicroseconds = TimeUnit.MICROSECONDS.convert(recordMessage.getEmittedAt(), TimeUnit.MILLISECONDS); + return QueryParameterValue.timestamp(emittedAtMicroseconds).getValue(); } protected Object getEmittedAtField(final AirbyteRecordMessage recordMessage) { @@ -72,11 +78,7 @@ protected Object getData(final AirbyteRecordMessage recordMessage) { @Override public Schema getBigQuerySchema(final JsonNode jsonSchema) { - if (TypingAndDedupingFlag.isDestinationV2()) { - return SCHEMA_V2; - } else { - return SCHEMA; - } + return SCHEMA_V2; } } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/GcsAvroBigQueryRecordFormatter.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/GcsAvroBigQueryRecordFormatter.java index 8ee69576a1ce..111238c8d508 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/GcsAvroBigQueryRecordFormatter.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/GcsAvroBigQueryRecordFormatter.java @@ -5,7 +5,7 @@ package io.airbyte.integrations.destination.bigquery.formatter; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; /** diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/GcsCsvBigQueryRecordFormatter.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/GcsCsvBigQueryRecordFormatter.java index 18941e6796c8..8ac49af733c5 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/GcsCsvBigQueryRecordFormatter.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/formatter/GcsCsvBigQueryRecordFormatter.java @@ -5,7 +5,11 @@ package io.airbyte.integrations.destination.bigquery.formatter; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.destination.StandardNameTransformer; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardSQLTypeName; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; /** * Formatter for GCS CSV uploader. Contains specific filling of default Airbyte attributes. Note! @@ -13,8 +17,18 @@ */ public class GcsCsvBigQueryRecordFormatter extends DefaultBigQueryRecordFormatter { + public static final com.google.cloud.bigquery.Schema CSV_SCHEMA = com.google.cloud.bigquery.Schema.of( + Field.of(JavaBaseConstants.COLUMN_NAME_AB_ID, StandardSQLTypeName.STRING), + Field.of(JavaBaseConstants.COLUMN_NAME_DATA, StandardSQLTypeName.STRING), + Field.of(JavaBaseConstants.COLUMN_NAME_EMITTED_AT, StandardSQLTypeName.TIMESTAMP)); + public GcsCsvBigQueryRecordFormatter(JsonNode jsonSchema, StandardNameTransformer namingResolver) { super(jsonSchema, namingResolver); } + @Override + public Schema getBigQuerySchema(JsonNode jsonSchema) { + return SCHEMA_V2; + } + } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/helpers/LoggerHelper.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/helpers/LoggerHelper.java index f4ad416d89c9..4e0682865dd1 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/helpers/LoggerHelper.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/helpers/LoggerHelper.java @@ -30,11 +30,12 @@ public static void printHeapMemoryConsumption() { } public static String getJobErrorMessage(List errors, Job job) { - if (!errors.isEmpty()) { - return String.format("Error is happened during execution for job: %s, \n For more details see Big Query Error collection: %s:", job, - errors.stream().map(BigQueryError::toString).collect(Collectors.joining(",\n "))); + if (errors == null || errors.isEmpty()) { + return StringUtils.EMPTY; + } - return StringUtils.EMPTY; + return String.format("An error occurred during execution of job: %s, \n For more details see Big Query Error collection: %s:", job, + errors.stream().map(BigQueryError::toString).collect(Collectors.joining(",\n "))); } } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryDestinationHandler.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryDestinationHandler.java index a9c3a2949913..9b513819893b 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryDestinationHandler.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryDestinationHandler.java @@ -5,21 +5,31 @@ package io.airbyte.integrations.destination.bigquery.typing_deduping; import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryException; +import com.google.cloud.bigquery.FieldValue; import com.google.cloud.bigquery.Job; import com.google.cloud.bigquery.JobConfiguration; import com.google.cloud.bigquery.JobId; import com.google.cloud.bigquery.JobInfo; import com.google.cloud.bigquery.JobStatistics; +import com.google.cloud.bigquery.JobStatus; import com.google.cloud.bigquery.QueryJobConfiguration; import com.google.cloud.bigquery.Table; import com.google.cloud.bigquery.TableDefinition; import com.google.cloud.bigquery.TableId; +import com.google.common.collect.Streams; +import io.airbyte.cdk.integrations.base.AirbyteExceptionHandler; import io.airbyte.integrations.base.destination.typing_deduping.DestinationHandler; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import java.math.BigInteger; import java.util.Comparator; +import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.stream.Stream; +import org.apache.commons.text.StringSubstitutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,79 +41,131 @@ public class BigQueryDestinationHandler implements DestinationHandler findExistingTable(StreamId id) { + public Optional findExistingTable(final StreamId id) { final Table table = bq.getTable(id.finalNamespace(), id.finalName()); return Optional.ofNullable(table).map(Table::getDefinition); } @Override - public boolean isFinalTableEmpty(StreamId id) { + public boolean isFinalTableEmpty(final StreamId id) { return BigInteger.ZERO.equals(bq.getTable(TableId.of(id.finalNamespace(), id.finalName())).getNumRows()); } @Override - public void execute(final String sql) throws InterruptedException { - if ("".equals(sql)) { + public InitialRawTableState getInitialRawTableState(final StreamId id) throws Exception { + final Table rawTable = bq.getTable(TableId.of(id.rawNamespace(), id.rawName())); + if (rawTable == null) { + // Table doesn't exist. There are no unprocessed records, and no timestamp. + return new InitialRawTableState(false, Optional.empty()); + } + + final FieldValue unloadedRecordTimestamp = bq.query(QueryJobConfiguration.newBuilder(new StringSubstitutor(Map.of( + "raw_table", id.rawTableId(BigQuerySqlGenerator.QUOTE))).replace( + // bigquery timestamps have microsecond precision + """ + SELECT TIMESTAMP_SUB(MIN(_airbyte_extracted_at), INTERVAL 1 MICROSECOND) + FROM ${raw_table} + WHERE _airbyte_loaded_at IS NULL + """)) + .build()).iterateAll().iterator().next().get(0); + // If this value is null, then there are no records with null loaded_at. + // If it's not null, then we can return immediately - we've found some unprocessed records and their + // timestamp. + if (!unloadedRecordTimestamp.isNull()) { + return new InitialRawTableState(true, Optional.of(unloadedRecordTimestamp.getTimestampInstant())); + } + + final FieldValue loadedRecordTimestamp = bq.query(QueryJobConfiguration.newBuilder(new StringSubstitutor(Map.of( + "raw_table", id.rawTableId(BigQuerySqlGenerator.QUOTE))).replace( + """ + SELECT MAX(_airbyte_extracted_at) + FROM ${raw_table} + """)) + .build()).iterateAll().iterator().next().get(0); + // We know (from the previous query) that all records have been processed by T+D already. + // So we just need to get the timestamp of the most recent record. + if (loadedRecordTimestamp.isNull()) { + // Null timestamp because the table is empty. T+D can process the entire raw table during this sync. + return new InitialRawTableState(false, Optional.empty()); + } else { + // The raw table already has some records. T+D can skip all records with timestamp <= this value. + return new InitialRawTableState(false, Optional.of(loadedRecordTimestamp.getTimestampInstant())); + } + } + + @Override + public void execute(final Sql sql) throws InterruptedException { + final List transactions = sql.asSqlStrings("BEGIN TRANSACTION", "COMMIT TRANSACTION"); + if (transactions.isEmpty()) { return; } final UUID queryId = UUID.randomUUID(); - LOGGER.info("Executing sql {}: {}", queryId, sql); - - /* - * If you run a query like CREATE SCHEMA ... OPTIONS(location=foo); CREATE TABLE ...;, bigquery - * doesn't do a good job of inferring the query location. Pass it in explicitly. - */ - Job job = bq.create(JobInfo.of(JobId.newBuilder().setLocation(datasetLocation).build(), QueryJobConfiguration.newBuilder(sql).build())); - job = job.waitFor(); - // waitFor() seems to throw an exception if the query failed, but javadoc says we're supposed to - // handle this case - if (job.getStatus().getError() != null) { - throw new RuntimeException(job.getStatus().getError().toString()); - } + for (final String transaction : transactions) { + final UUID transactionId = UUID.randomUUID(); + LOGGER.debug("Executing sql {}-{}: {}", queryId, transactionId, transaction); + + /* + * If you run a query like CREATE SCHEMA ... OPTIONS(location=foo); CREATE TABLE ...;, bigquery + * doesn't do a good job of inferring the query location. Pass it in explicitly. + */ + Job job = bq.create(JobInfo.of(JobId.newBuilder().setLocation(datasetLocation).build(), QueryJobConfiguration.newBuilder(transaction).build())); + AirbyteExceptionHandler.addStringForDeinterpolation(job.getEtag()); + // job.waitFor() gets stuck forever in some failure cases, so manually poll the job instead. + while (!JobStatus.State.DONE.equals(job.getStatus().getState())) { + Thread.sleep(1000L); + job = job.reload(); + } + if (job.getStatus().getError() != null) { + throw new BigQueryException(Streams.concat( + Stream.of(job.getStatus().getError()), + job.getStatus().getExecutionErrors().stream()).toList()); + } + + final JobStatistics.QueryStatistics statistics = job.getStatistics(); + LOGGER.debug("Root-level job {}-{} completed in {} ms; processed {} bytes; billed for {} bytes", + queryId, + transactionId, + statistics.getEndTime() - statistics.getStartTime(), + statistics.getTotalBytesProcessed(), + statistics.getTotalBytesBilled()); - JobStatistics.QueryStatistics statistics = job.getStatistics(); - LOGGER.info("Root-level job {} completed in {} ms; processed {} bytes; billed for {} bytes", - queryId, - statistics.getEndTime() - statistics.getStartTime(), - statistics.getTotalBytesProcessed(), - statistics.getTotalBytesBilled()); - - // SQL transactions can spawn child jobs, which are billed individually. Log their stats too. - if (statistics.getNumChildJobs() != null) { - // There isn't (afaict) anything resembling job.getChildJobs(), so we have to ask bq for them - bq.listJobs(BigQuery.JobListOption.parentJobId(job.getJobId().getJob())).streamAll() - .sorted(Comparator.comparing(childJob -> childJob.getStatistics().getEndTime())) - .forEach(childJob -> { - JobConfiguration configuration = childJob.getConfiguration(); - if (configuration instanceof QueryJobConfiguration qc) { - JobStatistics.QueryStatistics childQueryStats = childJob.getStatistics(); - String truncatedQuery = qc.getQuery() - .replaceAll("\n", " ") - .replaceAll(" +", " ") - .substring(0, Math.min(100, qc.getQuery().length())); - if (!truncatedQuery.equals(qc.getQuery())) { - truncatedQuery += "..."; + // SQL transactions can spawn child jobs, which are billed individually. Log their stats too. + if (statistics.getNumChildJobs() != null) { + // There isn't (afaict) anything resembling job.getChildJobs(), so we have to ask bq for them + bq.listJobs(BigQuery.JobListOption.parentJobId(job.getJobId().getJob())).streamAll() + .sorted(Comparator.comparing(childJob -> childJob.getStatistics().getEndTime())) + .forEach(childJob -> { + final JobConfiguration configuration = childJob.getConfiguration(); + if (configuration instanceof final QueryJobConfiguration qc) { + final JobStatistics.QueryStatistics childQueryStats = childJob.getStatistics(); + String truncatedQuery = qc.getQuery() + .replaceAll("\n", " ") + .replaceAll(" +", " ") + .substring(0, Math.min(100, qc.getQuery().length())); + if (!truncatedQuery.equals(qc.getQuery())) { + truncatedQuery += "..."; + } + LOGGER.debug("Child sql {} completed in {} ms; processed {} bytes; billed for {} bytes", + truncatedQuery, + childQueryStats.getEndTime() - childQueryStats.getStartTime(), + childQueryStats.getTotalBytesProcessed(), + childQueryStats.getTotalBytesBilled()); + } else { + // other job types are extract/copy/load + // we're probably not using them, but handle just in case? + final JobStatistics childJobStats = childJob.getStatistics(); + LOGGER.debug("Non-query child job ({}) completed in {} ms", + configuration.getType(), + childJobStats.getEndTime() - childJobStats.getStartTime()); } - LOGGER.info("Child sql {} completed in {} ms; processed {} bytes; billed for {} bytes", - truncatedQuery, - childQueryStats.getEndTime() - childQueryStats.getStartTime(), - childQueryStats.getTotalBytesProcessed(), - childQueryStats.getTotalBytesBilled()); - } else { - // other job types are extract/copy/load - // we're probably not using them, but handle just in case? - JobStatistics childJobStats = childJob.getStatistics(); - LOGGER.info("Non-query child job ({}) completed in {} ms", - configuration.getType(), - childJobStats.getEndTime() - childJobStats.getStartTime()); - } - }); + }); + } } } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGenerator.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGenerator.java index 288bcd99d36b..c4370fc5dc0a 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGenerator.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGenerator.java @@ -7,19 +7,25 @@ import static io.airbyte.integrations.base.destination.typing_deduping.CollectionUtils.containsAllIgnoreCase; import static io.airbyte.integrations.base.destination.typing_deduping.CollectionUtils.containsIgnoreCase; import static io.airbyte.integrations.base.destination.typing_deduping.CollectionUtils.matchingKey; +import static io.airbyte.integrations.base.destination.typing_deduping.Sql.separately; +import static io.airbyte.integrations.base.destination.typing_deduping.Sql.transactionally; +import static io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeTransaction.SOFT_RESET_SUFFIX; import static java.util.stream.Collectors.joining; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.Field.Mode; import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.cloud.bigquery.StandardTableDefinition; import com.google.cloud.bigquery.TableDefinition; import com.google.cloud.bigquery.TimePartitioning; import com.google.common.annotations.VisibleForTesting; -import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType; import io.airbyte.integrations.base.destination.typing_deduping.AlterTableReport; import io.airbyte.integrations.base.destination.typing_deduping.Array; import io.airbyte.integrations.base.destination.typing_deduping.ColumnId; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; import io.airbyte.integrations.base.destination.typing_deduping.SqlGenerator; import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; import io.airbyte.integrations.base.destination.typing_deduping.StreamId; @@ -29,10 +35,11 @@ import io.airbyte.integrations.base.destination.typing_deduping.UnsupportedOneOf; import io.airbyte.integrations.destination.bigquery.BigQuerySQLNameTransformer; import io.airbyte.protocol.models.v0.DestinationSyncMode; +import java.time.Instant; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -52,14 +59,18 @@ public class BigQuerySqlGenerator implements SqlGenerator { private final ColumnId CDC_DELETED_AT_COLUMN = buildColumnId("_ab_cdc_deleted_at"); private final Logger LOGGER = LoggerFactory.getLogger(BigQuerySqlGenerator.class); + + private final String projectId; private final String datasetLocation; /** + * @param projectId * @param datasetLocation This is technically redundant with {@link BigQueryDestinationHandler} * setting the query execution location, but let's be explicit since this is typically a * compliance requirement. */ - public BigQuerySqlGenerator(final String datasetLocation) { + public BigQuerySqlGenerator(final String projectId, final String datasetLocation) { + this.projectId = projectId; this.datasetLocation = datasetLocation; } @@ -75,10 +86,13 @@ public StreamId buildStreamId(final String namespace, final String name, final S } @Override - public ColumnId buildColumnId(final String name) { - // Bigquery columns are case-insensitive, so do all our validation on the lowercased name - final String canonicalized = name.toLowerCase(); - return new ColumnId(nameTransformer.getIdentifier(name), name, canonicalized); + public ColumnId buildColumnId(final String name, final String suffix) { + final String nameWithSuffix = name + suffix; + return new ColumnId( + nameTransformer.getIdentifier(nameWithSuffix), + name, + // Bigquery columns are case-insensitive, so do all our validation on the lowercased name + nameTransformer.getIdentifier(nameWithSuffix.toLowerCase())); } public StandardSQLTypeName toDialectType(final AirbyteType type) { @@ -106,12 +120,14 @@ public StandardSQLTypeName toDialectType(final AirbyteType type) { throw new IllegalArgumentException("Unsupported AirbyteType: " + type); } - private String extractAndCast(final ColumnId column, final AirbyteType airbyteType) { + private String extractAndCast(final ColumnId column, final AirbyteType airbyteType, final boolean forceSafeCast) { if (airbyteType instanceof final Union u) { // This is guaranteed to not be a Union, so we won't recurse infinitely final AirbyteType chosenType = u.chooseType(); - return extractAndCast(column, chosenType); - } else if (airbyteType instanceof Struct) { + return extractAndCast(column, chosenType, forceSafeCast); + } + + if (airbyteType instanceof Struct) { // We need to validate that the struct is actually a struct. // Note that struct columns are actually nullable in two ways. For a column `foo`: // {foo: null} and {} are both valid, and are both written to the final table as a SQL NULL (_not_ a @@ -127,7 +143,9 @@ OR JSON_TYPE(PARSE_JSON(JSON_QUERY(`_airbyte_data`, '$."${column_name}"'), wide_ ELSE JSON_QUERY(`_airbyte_data`, '$."${column_name}"') END, wide_number_mode=>'round') """); - } else if (airbyteType instanceof Array) { + } + + if (airbyteType instanceof Array) { // Much like the Struct case above, arrays need special handling. return new StringSubstitutor(Map.of("column_name", escapeColumnNameForJsonPath(column.originalName()))).replace( """ @@ -138,7 +156,9 @@ OR JSON_TYPE(PARSE_JSON(JSON_QUERY(`_airbyte_data`, '$."${column_name}"'), wide_ ELSE JSON_QUERY(`_airbyte_data`, '$."${column_name}"') END, wide_number_mode=>'round') """); - } else if (airbyteType instanceof UnsupportedOneOf || airbyteType == AirbyteProtocolType.UNKNOWN) { + } + + if (airbyteType instanceof UnsupportedOneOf || airbyteType == AirbyteProtocolType.UNKNOWN) { // JSON_QUERY returns a SQL null if the field contains a JSON null, so we actually parse the // airbyte_data to json // and json_query it directly (which preserves nulls correctly). @@ -146,10 +166,32 @@ ELSE JSON_QUERY(`_airbyte_data`, '$."${column_name}"') """ JSON_QUERY(PARSE_JSON(`_airbyte_data`, wide_number_mode=>'round'), '$."${column_name}"') """); + } + + if (airbyteType == AirbyteProtocolType.STRING) { + // Special case String to only use json value for type string and parse the json for others + // Naive json_value returns NULL for object/array values and json_query adds escaped quotes to the + // string. + return new StringSubstitutor(Map.of("column_name", escapeColumnNameForJsonPath(column.originalName()))).replace( + """ + (CASE + WHEN JSON_QUERY(`_airbyte_data`, '$."${column_name}"') IS NULL + OR JSON_TYPE(PARSE_JSON(JSON_QUERY(`_airbyte_data`, '$."${column_name}"'), wide_number_mode=>'round')) != 'string' + THEN JSON_QUERY(`_airbyte_data`, '$."${column_name}"') + ELSE + JSON_VALUE(`_airbyte_data`, '$."${column_name}"') + END) + """); + } + + final StandardSQLTypeName dialectType = toDialectType(airbyteType); + final var baseTyping = "JSON_VALUE(`_airbyte_data`, '$.\"" + escapeColumnNameForJsonPath(column.originalName()) + "\"')"; + if (dialectType == StandardSQLTypeName.STRING) { + // json_value implicitly returns a string, so we don't need to cast it. + return baseTyping; } else { - final StandardSQLTypeName dialectType = toDialectType(airbyteType); - return "SAFE_CAST(JSON_VALUE(`_airbyte_data`, '$.\"" + escapeColumnNameForJsonPath(column.originalName()) + "\"') as " + dialectType.name() - + ")"; + // SAFE_CAST is actually a massive performance hit, so we should skip it if we can. + return cast(baseTyping, dialectType.name(), forceSafeCast); } } @@ -171,25 +213,22 @@ public StandardSQLTypeName toDialectType(final AirbyteProtocolType airbyteProtoc } @Override - public String createTable(final StreamConfig stream, final String suffix, final boolean force) { + public Sql createTable(final StreamConfig stream, final String suffix, final boolean force) { final String columnDeclarations = columnsAndTypes(stream); final String clusterConfig = clusteringColumns(stream).stream() .map(c -> StringUtils.wrap(c, QUOTE)) .collect(joining(", ")); final String forceCreateTable = force ? "OR REPLACE" : ""; - return new StringSubstitutor(Map.of( + return Sql.of(new StringSubstitutor(Map.of( + "project_id", '`' + projectId + '`', "final_namespace", stream.id().finalNamespace(QUOTE), - "dataset_location", datasetLocation, "force_create_table", forceCreateTable, "final_table_id", stream.id().finalTableId(QUOTE, suffix), "column_declarations", columnDeclarations, "cluster_config", clusterConfig)).replace( """ - CREATE SCHEMA IF NOT EXISTS ${final_namespace} - OPTIONS(location="${dataset_location}"); - - CREATE ${force_create_table} TABLE ${final_table_id} ( + CREATE ${force_create_table} TABLE ${project_id}.${final_table_id} ( _airbyte_raw_id STRING NOT NULL, _airbyte_extracted_at TIMESTAMP NOT NULL, _airbyte_meta JSON NOT NULL, @@ -197,7 +236,7 @@ public String createTable(final StreamConfig stream, final String suffix, final ) PARTITION BY (DATE_TRUNC(_airbyte_extracted_at, DAY)) CLUSTER BY ${cluster_config}; - """); + """)); } private List clusteringColumns(final StreamConfig stream) { @@ -259,6 +298,8 @@ public boolean partitioningMatches(final StandardTableDefinition existingTable) } public AlterTableReport buildAlterTableReport(final StreamConfig stream, final TableDefinition existingTable) { + final Set pks = getPks(stream); + final Map streamSchema = stream.columns().entrySet().stream() .collect(Collectors.toMap( entry -> entry.getKey().name(), @@ -281,17 +322,25 @@ public AlterTableReport buildAlterTableReport(final StreamConfig stream, final T .collect(Collectors.toSet()); // Columns that are typed differently than the StreamConfig - final Set columnsToChangeType = streamSchema.keySet().stream() - // If it's not in the existing schema, it should already be in the columnsToAdd Set - .filter(name -> { - // Big Query Columns are case-insensitive, first find the correctly cased key if it exists - return matchingKey(existingSchema.keySet(), name) - // if it does exist, only include it in this set if the type (the value in each respective map) - // is different between the stream and existing schemas - .map(key -> !existingSchema.get(key).equals(streamSchema.get(name))) - // if there is no matching key, then don't include it because it is probably already in columnsToAdd - .orElse(false); - }) + final Set columnsToChangeType = Stream.concat( + streamSchema.keySet().stream() + // If it's not in the existing schema, it should already be in the columnsToAdd Set + .filter(name -> { + // Big Query Columns are case-insensitive, first find the correctly cased key if it exists + return matchingKey(existingSchema.keySet(), name) + // if it does exist, only include it in this set if the type (the value in each respective map) + // is different between the stream and existing schemas + .map(key -> !existingSchema.get(key).equals(streamSchema.get(name))) + // if there is no matching key, then don't include it because it is probably already in columnsToAdd + .orElse(false); + }), + + // OR columns that used to have a non-null constraint and shouldn't + // (https://github.com/airbytehq/airbyte/pull/31082) + existingTable.getSchema().getFields().stream() + .filter(field -> pks.contains(field.getName())) + .filter(field -> field.getMode() == Mode.REQUIRED) + .map(Field::getName)) .collect(Collectors.toSet()); final boolean isDestinationV2Format = schemaContainAllFinalTableV2AirbyteColumns(existingSchema.keySet()); @@ -313,117 +362,194 @@ public static boolean schemaContainAllFinalTableV2AirbyteColumns(final Collectio } @Override - public String softReset(final StreamConfig stream) { - final String createTempTable = createTable(stream, SOFT_RESET_SUFFIX, true); - final String clearLoadedAt = clearLoadedAt(stream.id()); - final String rebuildInTempTable = updateTable(stream, SOFT_RESET_SUFFIX, false); - final String overwriteFinalTable = overwriteFinalTable(stream.id(), SOFT_RESET_SUFFIX); - return String.join("\n", createTempTable, clearLoadedAt, rebuildInTempTable, overwriteFinalTable); + public Sql prepareTablesForSoftReset(final StreamConfig stream) { + // Bigquery can't run DDL in a transaction, so these are separate transactions. + return Sql.concat( + // If a previous sync failed to delete the soft reset temp table (unclear why this happens), + // AND this sync is trying to change the clustering config, then we need to manually drop the soft + // reset temp table. + // Even though we're using CREATE OR REPLACE TABLE, bigquery will still complain about the + // clustering config being changed. + // So we explicitly drop the soft reset temp table first. + dropTableIfExists(stream, SOFT_RESET_SUFFIX), + createTable(stream, SOFT_RESET_SUFFIX, true), + clearLoadedAt(stream.id())); } - private String clearLoadedAt(final StreamId streamId) { - return new StringSubstitutor(Map.of("raw_table_id", streamId.rawTableId(QUOTE))) - .replace(""" - UPDATE ${raw_table_id} SET _airbyte_loaded_at = NULL WHERE 1=1; - """); + public Sql dropTableIfExists(final StreamConfig stream, final String suffix) { + return Sql.of(new StringSubstitutor(Map.of( + "project_id", '`' + projectId + '`', + "table_id", stream.id().finalTableId(QUOTE, suffix))) + .replace(""" + DROP TABLE IF EXISTS ${project_id}.${table_id}; + """)); } @Override - public String updateTable(final StreamConfig stream, final String finalSuffix) { - return updateTable(stream, finalSuffix, true); + public Sql clearLoadedAt(final StreamId streamId) { + return Sql.of(new StringSubstitutor(Map.of( + "project_id", '`' + projectId + '`', + "raw_table_id", streamId.rawTableId(QUOTE))) + .replace(""" + UPDATE ${project_id}.${raw_table_id} SET _airbyte_loaded_at = NULL WHERE 1=1; + """)); } - private String updateTable(final StreamConfig stream, final String finalSuffix, final boolean verifyPrimaryKeys) { - String pkVarDeclaration = ""; - String validatePrimaryKeys = ""; - if (verifyPrimaryKeys && stream.destinationSyncMode() == DestinationSyncMode.APPEND_DEDUP) { - pkVarDeclaration = "DECLARE missing_pk_count INT64;"; - validatePrimaryKeys = validatePrimaryKeys(stream.id(), stream.primaryKey(), stream.columns()); - } - final String insertNewRecords = insertNewRecords(stream, finalSuffix, stream.columns()); - String dedupFinalTable = ""; - String cdcDeletes = ""; - String dedupRawTable = ""; + @Override + public Sql updateTable(final StreamConfig stream, + final String finalSuffix, + final Optional minRawTimestamp, + final boolean useExpensiveSaferCasting) { + final String handleNewRecords; if (stream.destinationSyncMode() == DestinationSyncMode.APPEND_DEDUP) { - dedupRawTable = dedupRawTable(stream.id(), finalSuffix); - // If we're in dedup mode, then we must have a cursor - dedupFinalTable = dedupFinalTable(stream.id(), finalSuffix, stream.primaryKey(), stream.cursor()); - cdcDeletes = cdcDeletes(stream, finalSuffix, stream.columns()); + handleNewRecords = upsertNewRecords(stream, finalSuffix, useExpensiveSaferCasting, minRawTimestamp); + } else { + handleNewRecords = insertNewRecords(stream, finalSuffix, useExpensiveSaferCasting, minRawTimestamp); } - final String commitRawTable = commitRawTable(stream.id()); - - return new StringSubstitutor(Map.of( - "pk_var_declaration", pkVarDeclaration, - "validate_primary_keys", validatePrimaryKeys, - "insert_new_records", insertNewRecords, - "dedup_final_table", dedupFinalTable, - "cdc_deletes", cdcDeletes, - "dedupe_raw_table", dedupRawTable, - "commit_raw_table", commitRawTable)).replace( - """ - ${pk_var_declaration} + final String commitRawTable = commitRawTable(stream.id(), minRawTimestamp); - BEGIN TRANSACTION; - - ${validate_primary_keys} - - ${insert_new_records} - - ${dedup_final_table} + return transactionally(handleNewRecords, commitRawTable); + } - ${dedupe_raw_table} + private String insertNewRecords(final StreamConfig stream, + final String finalSuffix, + final boolean forceSafeCasting, + final Optional minRawTimestamp) { + final String columnList = stream.columns().keySet().stream().map(quotedColumnId -> quotedColumnId.name(QUOTE) + ",").collect(joining("\n")); + final String extractNewRawRecords = extractNewRawRecords(stream, forceSafeCasting, minRawTimestamp); - ${cdc_deletes} + return new StringSubstitutor(Map.of( + "project_id", '`' + projectId + '`', + "final_table_id", stream.id().finalTableId(QUOTE, finalSuffix), + "column_list", columnList, + "extractNewRawRecords", extractNewRawRecords)).replace( + """ + INSERT INTO ${project_id}.${final_table_id} + ( + ${column_list} + _airbyte_meta, + _airbyte_raw_id, + _airbyte_extracted_at + ) + ${extractNewRawRecords};"""); + } - ${commit_raw_table} + private String upsertNewRecords(final StreamConfig stream, + final String finalSuffix, + final boolean forceSafeCasting, + final Optional minRawTimestamp) { + final String pkEquivalent = stream.primaryKey().stream().map(pk -> { + final String quotedPk = pk.name(QUOTE); + // either the PKs are equal, or they're both NULL + return "(target_table." + quotedPk + " = new_record." + quotedPk + + " OR (target_table." + quotedPk + " IS NULL AND new_record." + quotedPk + " IS NULL))"; + }).collect(joining(" AND ")); + + final String columnList = stream.columns().keySet().stream() + .map(quotedColumnId -> quotedColumnId.name(QUOTE) + ",") + .collect(joining("\n")); + final String newRecordColumnList = stream.columns().keySet().stream() + .map(quotedColumnId -> "new_record." + quotedColumnId.name(QUOTE) + ",") + .collect(joining("\n")); + final String extractNewRawRecords = extractNewRawRecords(stream, forceSafeCasting, minRawTimestamp); + + final String cursorComparison; + if (stream.cursor().isPresent()) { + final String cursor = stream.cursor().get().name(QUOTE); + // Build a condition for "new_record is more recent than target_table": + cursorComparison = + // First, compare the cursors. + "(target_table." + cursor + " < new_record." + cursor + // Then, break ties with extracted_at. (also explicitly check for both new_record and final table + // having null cursor + // because NULL != NULL in SQL) + + " OR (target_table." + cursor + " = new_record." + cursor + + " AND target_table._airbyte_extracted_at < new_record._airbyte_extracted_at)" + + " OR (target_table." + cursor + " IS NULL AND new_record." + cursor + + " IS NULL AND target_table._airbyte_extracted_at < new_record._airbyte_extracted_at)" + // Or, if the final table has null cursor but new_record has non-null cursor, then take the new + // record. + + " OR (target_table." + cursor + " IS NULL AND new_record." + cursor + " IS NOT NULL))"; + } else { + // If there's no cursor, then we just take the most-recently-emitted record + cursorComparison = "target_table._airbyte_extracted_at < new_record._airbyte_extracted_at"; + } - COMMIT TRANSACTION; - """); - } + final String cdcDeleteClause; + final String cdcSkipInsertClause; + if (stream.columns().containsKey(CDC_DELETED_AT_COLUMN)) { + // Execute CDC deletions if there's already a record + cdcDeleteClause = "WHEN MATCHED AND new_record._ab_cdc_deleted_at IS NOT NULL AND " + cursorComparison + " THEN DELETE"; + // And skip insertion entirely if there's no matching record. + // (This is possible if a single T+D batch contains both an insertion and deletion for the same PK) + cdcSkipInsertClause = "AND new_record._ab_cdc_deleted_at IS NULL"; + } else { + cdcDeleteClause = ""; + cdcSkipInsertClause = ""; + } - @VisibleForTesting - String validatePrimaryKeys(final StreamId id, - final List primaryKeys, - final LinkedHashMap streamColumns) { - final String pkNullChecks = primaryKeys.stream().map( - pk -> { - final String jsonExtract = extractAndCast(pk, streamColumns.get(pk)); - return "AND " + jsonExtract + " IS NULL"; + final String columnAssignments = stream.columns().keySet().stream() + .map(airbyteType -> { + final String column = airbyteType.name(QUOTE); + return column + " = new_record." + column + ","; }).collect(joining("\n")); return new StringSubstitutor(Map.of( - "raw_table_id", id.rawTableId(QUOTE), - "pk_null_checks", pkNullChecks)).replace( + "project_id", '`' + projectId + '`', + "final_table_id", stream.id().finalTableId(QUOTE, finalSuffix), + "extractNewRawRecords", extractNewRawRecords, + "pkEquivalent", pkEquivalent, + "cdcDeleteClause", cdcDeleteClause, + "cursorComparison", cursorComparison, + "columnAssignments", columnAssignments, + "cdcSkipInsertClause", cdcSkipInsertClause, + "column_list", columnList, + "newRecordColumnList", newRecordColumnList)).replace( """ - SET missing_pk_count = ( - SELECT COUNT(1) - FROM ${raw_table_id} - WHERE - `_airbyte_loaded_at` IS NULL - ${pk_null_checks} - ); - - IF missing_pk_count > 0 THEN - RAISE USING message = FORMAT('Raw table has %s rows missing a primary key', CAST(missing_pk_count AS STRING)); - END IF - ;"""); + MERGE ${project_id}.${final_table_id} target_table + USING ( + ${extractNewRawRecords} + ) new_record + ON ${pkEquivalent} + ${cdcDeleteClause} + WHEN MATCHED AND ${cursorComparison} THEN UPDATE SET + ${columnAssignments} + _airbyte_meta = new_record._airbyte_meta, + _airbyte_raw_id = new_record._airbyte_raw_id, + _airbyte_extracted_at = new_record._airbyte_extracted_at + WHEN NOT MATCHED ${cdcSkipInsertClause} THEN INSERT ( + ${column_list} + _airbyte_meta, + _airbyte_raw_id, + _airbyte_extracted_at + ) VALUES ( + ${newRecordColumnList} + new_record._airbyte_meta, + new_record._airbyte_raw_id, + new_record._airbyte_extracted_at + );"""); } - @VisibleForTesting - String insertNewRecords(final StreamConfig stream, final String finalSuffix, final LinkedHashMap streamColumns) { - final String columnCasts = streamColumns.entrySet().stream().map( - col -> extractAndCast(col.getKey(), col.getValue()) + " as " + col.getKey().name(QUOTE) + ",") + /** + * A SQL SELECT statement that extracts new records from the raw table, casts their columns, and + * builds their airbyte_meta column. + *

      + * In dedup mode: Also extracts all raw CDC deletion records (for tombstoning purposes) and dedupes + * the records (since we only need the most-recent record to upsert). + */ + private String extractNewRawRecords(final StreamConfig stream, + final boolean forceSafeCasting, + final Optional minRawTimestamp) { + final String columnCasts = stream.columns().entrySet().stream().map( + col -> extractAndCast(col.getKey(), col.getValue(), forceSafeCasting) + " as " + col.getKey().name(QUOTE) + ",") .collect(joining("\n")); final String columnErrors; - if (streamColumns.isEmpty()) { - // ARRAY_CONCAT doesn't like having an empty argument list, so handle that case separately - columnErrors = "[]"; - } else { - columnErrors = "ARRAY_CONCAT(" + streamColumns.entrySet().stream().map( + if (forceSafeCasting) { + columnErrors = "[" + stream.columns().entrySet().stream().map( col -> new StringSubstitutor(Map.of( "raw_col_name", escapeColumnNameForJsonPath(col.getKey().originalName()), "col_type", toDialectType(col.getValue()).name(), - "json_extract", extractAndCast(col.getKey(), col.getValue()))).replace( + "json_extract", extractAndCast(col.getKey(), col.getValue(), true))).replace( // Explicitly parse json here. This is safe because we're not using the actual value anywhere, // and necessary because json_query """ @@ -431,161 +557,140 @@ String insertNewRecords(final StreamConfig stream, final String finalSuffix, fin WHEN (JSON_QUERY(PARSE_JSON(`_airbyte_data`, wide_number_mode=>'round'), '$."${raw_col_name}"') IS NOT NULL) AND (JSON_TYPE(JSON_QUERY(PARSE_JSON(`_airbyte_data`, wide_number_mode=>'round'), '$."${raw_col_name}"')) != 'null') AND (${json_extract} IS NULL) - THEN ['Problem with `${raw_col_name}`'] - ELSE [] + THEN 'Problem with `${raw_col_name}`' + ELSE NULL END""")) - .collect(joining(",\n")) + ")"; - } - final String columnList = streamColumns.keySet().stream().map(quotedColumnId -> quotedColumnId.name(QUOTE) + ",").collect(joining("\n")); - - String cdcConditionalOrIncludeStatement = ""; - if (stream.destinationSyncMode() == DestinationSyncMode.APPEND_DEDUP && streamColumns.containsKey(CDC_DELETED_AT_COLUMN)) { - cdcConditionalOrIncludeStatement = """ - OR ( - _airbyte_loaded_at IS NOT NULL - AND JSON_VALUE(`_airbyte_data`, '$._ab_cdc_deleted_at') IS NOT NULL - ) - """; + .collect(joining(",\n")) + "]"; + } else { + // We're not safe casting, so any error should throw an exception and trigger the safe cast logic + columnErrors = "[]"; } - return new StringSubstitutor(Map.of( - "raw_table_id", stream.id().rawTableId(QUOTE), - "final_table_id", stream.id().finalTableId(QUOTE, finalSuffix), - "column_casts", columnCasts, - "column_errors", columnErrors, - "cdcConditionalOrIncludeStatement", cdcConditionalOrIncludeStatement, - "column_list", columnList)).replace( - """ - INSERT INTO ${final_table_id} - ( - ${column_list} - _airbyte_meta, - _airbyte_raw_id, - _airbyte_extracted_at - ) - WITH intermediate_data AS ( - SELECT - ${column_casts} - ${column_errors} as _airbyte_cast_errors, - _airbyte_raw_id, - _airbyte_extracted_at - FROM ${raw_table_id} - WHERE - _airbyte_loaded_at IS NULL - ${cdcConditionalOrIncludeStatement} - ) - SELECT - ${column_list} - to_json(struct(_airbyte_cast_errors AS errors)) AS _airbyte_meta, - _airbyte_raw_id, - _airbyte_extracted_at - FROM intermediate_data;"""); - } + final String columnList = stream.columns().keySet().stream().map(quotedColumnId -> quotedColumnId.name(QUOTE) + ",").collect(joining("\n")); + final String extractedAtCondition = buildExtractedAtCondition(minRawTimestamp); - @VisibleForTesting - String dedupFinalTable(final StreamId id, - final String finalSuffix, - final List primaryKey, - final Optional cursor) { - final String pkList = primaryKey.stream().map(columnId -> columnId.name(QUOTE)).collect(joining(",")); - final String cursorOrderClause = cursor - .map(cursorId -> cursorId.name(QUOTE) + " DESC NULLS LAST,") - .orElse(""); + if (stream.destinationSyncMode() == DestinationSyncMode.APPEND_DEDUP) { + // When deduping, we need to dedup the raw records. Note the row_number() invocation in the SQL + // statement. Do the same extract+cast CTE + airbyte_meta construction as in non-dedup mode, but + // then add a row_number column so that we only take the most-recent raw record for each PK. + + // We also explicitly include old CDC deletion records, which act as tombstones to correctly delete + // out-of-order records. + String cdcConditionalOrIncludeStatement = ""; + if (stream.columns().containsKey(CDC_DELETED_AT_COLUMN)) { + cdcConditionalOrIncludeStatement = """ + OR ( + _airbyte_loaded_at IS NOT NULL + AND JSON_VALUE(`_airbyte_data`, '$._ab_cdc_deleted_at') IS NOT NULL + ) + """; + } - return new StringSubstitutor(Map.of( - "final_table_id", id.finalTableId(QUOTE, finalSuffix), - "pk_list", pkList, - "cursor_order_clause", cursorOrderClause)).replace( - """ - DELETE FROM ${final_table_id} - WHERE - `_airbyte_raw_id` IN ( - SELECT `_airbyte_raw_id` FROM ( - SELECT `_airbyte_raw_id`, row_number() OVER ( - PARTITION BY ${pk_list} ORDER BY ${cursor_order_clause} `_airbyte_extracted_at` DESC - ) as row_number FROM ${final_table_id} - ) - WHERE row_number != 1 + final String pkList = stream.primaryKey().stream().map(columnId -> columnId.name(QUOTE)).collect(joining(",")); + final String cursorOrderClause = stream.cursor() + .map(cursorId -> cursorId.name(QUOTE) + " DESC NULLS LAST,") + .orElse(""); + + return new StringSubstitutor(Map.of( + "project_id", '`' + projectId + '`', + "raw_table_id", stream.id().rawTableId(QUOTE), + "column_casts", columnCasts, + "column_errors", columnErrors, + "cdcConditionalOrIncludeStatement", cdcConditionalOrIncludeStatement, + "extractedAtCondition", extractedAtCondition, + "column_list", columnList, + "pk_list", pkList, + "cursor_order_clause", cursorOrderClause)).replace( + """ + WITH intermediate_data AS ( + SELECT + ${column_casts} + ${column_errors} AS column_errors, + _airbyte_raw_id, + _airbyte_extracted_at + FROM ${project_id}.${raw_table_id} + WHERE ( + _airbyte_loaded_at IS NULL + ${cdcConditionalOrIncludeStatement} + ) ${extractedAtCondition} + ), new_records AS ( + SELECT + ${column_list} + to_json(struct(COALESCE((SELECT ARRAY_AGG(unnested_column_errors IGNORE NULLS) FROM UNNEST(column_errors) unnested_column_errors), []) AS errors)) AS _airbyte_meta, + _airbyte_raw_id, + _airbyte_extracted_at + FROM intermediate_data + ), numbered_rows AS ( + SELECT *, row_number() OVER ( + PARTITION BY ${pk_list} ORDER BY ${cursor_order_clause} `_airbyte_extracted_at` DESC + ) AS row_number + FROM new_records ) - ;"""); - } - - @VisibleForTesting - String cdcDeletes(final StreamConfig stream, - final String finalSuffix, - final LinkedHashMap streamColumns) { - - if (stream.destinationSyncMode() != DestinationSyncMode.APPEND_DEDUP) { - return ""; - } - - if (!streamColumns.containsKey(CDC_DELETED_AT_COLUMN)) { - return ""; - } - - final String pkList = stream.primaryKey().stream().map(columnId -> columnId.name(QUOTE)).collect(joining(",")); - final String pkCasts = stream.primaryKey().stream().map(pk -> extractAndCast(pk, streamColumns.get(pk))).collect(joining(",\n")); - - // we want to grab IDs for deletion from the raw table (not the final table itself) to hand - // out-of-order record insertions after the delete has been registered - return new StringSubstitutor(Map.of( - "final_table_id", stream.id().finalTableId(QUOTE, finalSuffix), - "raw_table_id", stream.id().rawTableId(QUOTE), - "pk_list", pkList, - "pk_extracts", pkCasts, - "quoted_cdc_delete_column", QUOTE + "_ab_cdc_deleted_at" + QUOTE)).replace( - """ - DELETE FROM ${final_table_id} - WHERE - (${pk_list}) IN ( - SELECT ( - ${pk_extracts} - ) - FROM ${raw_table_id} - WHERE JSON_TYPE(PARSE_JSON(JSON_QUERY(`_airbyte_data`, '$._ab_cdc_deleted_at'), wide_number_mode=>'round')) != 'null' + SELECT ${column_list} _airbyte_meta, _airbyte_raw_id, _airbyte_extracted_at + FROM numbered_rows + WHERE row_number = 1"""); + } else { + // When not deduplicating, we just need to handle type casting. + // Extract+cast the not-yet-loaded records in a CTE, then select that CTE and build airbyte_meta. + + return new StringSubstitutor(Map.of( + "project_id", '`' + projectId + '`', + "raw_table_id", stream.id().rawTableId(QUOTE), + "column_casts", columnCasts, + "column_errors", columnErrors, + "extractedAtCondition", extractedAtCondition, + "column_list", columnList)).replace( + """ + WITH intermediate_data AS ( + SELECT + ${column_casts} + ${column_errors} AS column_errors, + _airbyte_raw_id, + _airbyte_extracted_at + FROM ${project_id}.${raw_table_id} + WHERE + _airbyte_loaded_at IS NULL + ${extractedAtCondition} ) - ;"""); + SELECT + ${column_list} + to_json(struct(COALESCE((SELECT ARRAY_AGG(unnested_column_errors IGNORE NULLS) FROM UNNEST(column_errors) unnested_column_errors), []) AS errors)) AS _airbyte_meta, + _airbyte_raw_id, + _airbyte_extracted_at + FROM intermediate_data"""); + } } - @VisibleForTesting - String dedupRawTable(final StreamId id, final String finalSuffix) { - return new StringSubstitutor(Map.of( - "raw_table_id", id.rawTableId(QUOTE), - "final_table_id", id.finalTableId(QUOTE, finalSuffix))).replace( - // Note that this leaves _all_ deletion records in the raw table. We _could_ clear them out, but it - // would be painful, - // and it only matters in a few edge cases. - """ - DELETE FROM - ${raw_table_id} - WHERE - `_airbyte_raw_id` NOT IN ( - SELECT `_airbyte_raw_id` FROM ${final_table_id} - ) - ;"""); + private static String buildExtractedAtCondition(final Optional minRawTimestamp) { + return minRawTimestamp + .map(ts -> " AND _airbyte_extracted_at > '" + ts + "'") + .orElse(""); } @VisibleForTesting - String commitRawTable(final StreamId id) { + String commitRawTable(final StreamId id, final Optional minRawTimestamp) { return new StringSubstitutor(Map.of( - "raw_table_id", id.rawTableId(QUOTE))).replace( + "project_id", '`' + projectId + '`', + "raw_table_id", id.rawTableId(QUOTE), + "extractedAtCondition", buildExtractedAtCondition(minRawTimestamp))).replace( """ - UPDATE ${raw_table_id} + UPDATE ${project_id}.${raw_table_id} SET `_airbyte_loaded_at` = CURRENT_TIMESTAMP() WHERE `_airbyte_loaded_at` IS NULL + ${extractedAtCondition} ;"""); } @Override - public String overwriteFinalTable(final StreamId streamId, final String finalSuffix) { - return new StringSubstitutor(Map.of( + public Sql overwriteFinalTable(final StreamId streamId, final String finalSuffix) { + final StringSubstitutor substitutor = new StringSubstitutor(Map.of( + "project_id", '`' + projectId + '`', "final_table_id", streamId.finalTableId(QUOTE), "tmp_final_table", streamId.finalTableId(QUOTE, finalSuffix), - "real_final_table", streamId.finalName(QUOTE))).replace( - """ - DROP TABLE IF EXISTS ${final_table_id}; - ALTER TABLE ${tmp_final_table} RENAME TO ${real_final_table}; - """); + "real_final_table", streamId.finalName(QUOTE))); + return separately( + substitutor.replace("DROP TABLE IF EXISTS ${project_id}.${final_table_id};"), + substitutor.replace("ALTER TABLE ${project_id}.${tmp_final_table} RENAME TO ${real_final_table};")); } private String wrapAndQuote(final String namespace, final String tableName) { @@ -595,17 +700,21 @@ private String wrapAndQuote(final String namespace, final String tableName) { } @Override - public String migrateFromV1toV2(final StreamId streamId, final String namespace, final String tableName) { - return new StringSubstitutor(Map.of( - "raw_namespace", StringUtils.wrap(streamId.rawNamespace(), QUOTE), - "dataset_location", datasetLocation, + public Sql createSchema(final String schema) { + return Sql.of(new StringSubstitutor(Map.of("schema", StringUtils.wrap(schema, QUOTE), + "project_id", StringUtils.wrap(projectId, QUOTE), + "dataset_location", datasetLocation)) + .replace("CREATE SCHEMA IF NOT EXISTS ${project_id}.${schema} OPTIONS(location=\"${dataset_location}\");")); + } + + @Override + public Sql migrateFromV1toV2(final StreamId streamId, final String namespace, final String tableName) { + return Sql.of(new StringSubstitutor(Map.of( + "project_id", '`' + projectId + '`', "v2_raw_table", streamId.rawTableId(QUOTE), "v1_raw_table", wrapAndQuote(namespace, tableName))).replace( """ - CREATE SCHEMA IF NOT EXISTS ${raw_namespace} - OPTIONS(location="${dataset_location}"); - - CREATE OR REPLACE TABLE ${v2_raw_table} ( + CREATE OR REPLACE TABLE ${project_id}.${v2_raw_table} ( _airbyte_raw_id STRING, _airbyte_data STRING, _airbyte_extracted_at TIMESTAMP, @@ -619,9 +728,9 @@ PARTITION BY DATE(_airbyte_extracted_at) _airbyte_data AS _airbyte_data, _airbyte_emitted_at AS _airbyte_extracted_at, CAST(NULL AS TIMESTAMP) AS _airbyte_loaded_at - FROM ${v1_raw_table} + FROM ${project_id}.${v1_raw_table} ); - """); + """)); } /** @@ -651,4 +760,17 @@ private String escapeColumnNameForJsonPath(final String stringContents) { .replace("'", "\\'"); } + private static String cast(final String content, final String asType, final boolean useSafeCast) { + final var open = useSafeCast ? "SAFE_CAST(" : "CAST("; + return wrap(open, content + " as " + asType, ")"); + } + + private static Set getPks(final StreamConfig stream) { + return stream.primaryKey() != null ? stream.primaryKey().stream().map(ColumnId::name).collect(Collectors.toSet()) : Collections.emptySet(); + } + + private static String wrap(final String open, final String content, final String close) { + return open + content + close; + } + } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryV2RawTableMigrator.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryV2RawTableMigrator.java deleted file mode 100644 index cbbf5e122f37..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryV2RawTableMigrator.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery.typing_deduping; - -import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.Field; -import com.google.cloud.bigquery.FieldList; -import com.google.cloud.bigquery.LegacySQLTypeName; -import com.google.cloud.bigquery.QueryJobConfiguration; -import com.google.cloud.bigquery.Schema; -import com.google.cloud.bigquery.Table; -import com.google.cloud.bigquery.TableDefinition; -import com.google.cloud.bigquery.TableId; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; -import io.airbyte.integrations.base.destination.typing_deduping.V2RawTableMigrator; -import java.util.Map; -import org.apache.commons.text.StringSubstitutor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class BigQueryV2RawTableMigrator implements V2RawTableMigrator { - - private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryV2RawTableMigrator.class); - - private final BigQuery bq; - - public BigQueryV2RawTableMigrator(final BigQuery bq) { - this.bq = bq; - } - - @Override - public void migrateIfNecessary(final StreamConfig streamConfig) throws InterruptedException { - final Table rawTable = bq.getTable(TableId.of(streamConfig.id().rawNamespace(), streamConfig.id().rawName())); - if (rawTable != null && rawTable.exists()) { - final Schema existingRawSchema = rawTable.getDefinition().getSchema(); - final FieldList fields = existingRawSchema.getFields(); - if (fields.stream().noneMatch(f -> JavaBaseConstants.COLUMN_NAME_DATA.equals(f.getName()))) { - throw new IllegalStateException( - "Table does not have a column named _airbyte_data. We are likely colliding with a completely different table."); - } - final Field dataColumn = fields.get(JavaBaseConstants.COLUMN_NAME_DATA); - if (dataColumn.getType() == LegacySQLTypeName.JSON) { - LOGGER.info("Raw table has _airbyte_data of type JSON. Migrating to STRING."); - final String tmpRawTableId = BigQuerySqlGenerator.QUOTE + streamConfig.id().rawNamespace() + BigQuerySqlGenerator.QUOTE + "." - + BigQuerySqlGenerator.QUOTE + streamConfig.id().rawName() + "_airbyte_tmp" + BigQuerySqlGenerator.QUOTE; - bq.query(QueryJobConfiguration.of( - new StringSubstitutor(Map.of( - "raw_table", streamConfig.id().rawTableId(BigQuerySqlGenerator.QUOTE), - "tmp_raw_table", tmpRawTableId, - "real_raw_table", BigQuerySqlGenerator.QUOTE + streamConfig.id().rawName() + BigQuerySqlGenerator.QUOTE)).replace( - // In full refresh / append mode, standard inserts is creating a non-partitioned raw table. - // (possibly also in overwrite mode?). - // We can't just CREATE OR REPLACE the table because bigquery will complain that we're trying to - // change the partitioning scheme. - // Do an explicit CREATE tmp + DROP + RENAME, similar to how we overwrite the final tables in - // OVERWRITE mode. - """ - CREATE TABLE ${tmp_raw_table} - PARTITION BY DATE(_airbyte_extracted_at) - CLUSTER BY _airbyte_extracted_at - AS ( - SELECT - _airbyte_raw_id, - _airbyte_extracted_at, - _airbyte_loaded_at, - to_json_string(_airbyte_data) as _airbyte_data - FROM ${raw_table} - ); - DROP TABLE IF EXISTS ${raw_table}; - ALTER TABLE ${tmp_raw_table} RENAME TO ${real_raw_table}; - """))); - LOGGER.info("Completed Data column Migration for stream {}", streamConfig.id().rawName()); - } else { - LOGGER.info("No Data column Migration Required for stream {}", streamConfig.id().rawName()); - } - } - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryV2TableMigrator.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryV2TableMigrator.java new file mode 100644 index 000000000000..15f9cb3411a9 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryV2TableMigrator.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.bigquery.typing_deduping; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.FieldList; +import com.google.cloud.bigquery.LegacySQLTypeName; +import com.google.cloud.bigquery.QueryJobConfiguration; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.Table; +import com.google.cloud.bigquery.TableId; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; +import io.airbyte.integrations.base.destination.typing_deduping.V2TableMigrator; +import java.util.Map; +import org.apache.commons.text.StringSubstitutor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BigQueryV2TableMigrator implements V2TableMigrator { + + private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryV2TableMigrator.class); + + private final BigQuery bq; + + public BigQueryV2TableMigrator(final BigQuery bq) { + this.bq = bq; + } + + @Override + public void migrateIfNecessary(final StreamConfig streamConfig) throws InterruptedException { + final Table rawTable = bq.getTable(TableId.of(streamConfig.id().rawNamespace(), streamConfig.id().rawName())); + if (rawTable != null && rawTable.exists()) { + final Schema existingRawSchema = rawTable.getDefinition().getSchema(); + final FieldList fields = existingRawSchema.getFields(); + if (fields.stream().noneMatch(f -> JavaBaseConstants.COLUMN_NAME_DATA.equals(f.getName()))) { + throw new IllegalStateException( + "Table does not have a column named _airbyte_data. We are likely colliding with a completely different table."); + } + final Field dataColumn = fields.get(JavaBaseConstants.COLUMN_NAME_DATA); + if (dataColumn.getType() == LegacySQLTypeName.JSON) { + LOGGER.info("Raw table has _airbyte_data of type JSON. Migrating to STRING."); + final String tmpRawTableId = BigQuerySqlGenerator.QUOTE + streamConfig.id().rawNamespace() + BigQuerySqlGenerator.QUOTE + "." + + BigQuerySqlGenerator.QUOTE + streamConfig.id().rawName() + "_airbyte_tmp" + BigQuerySqlGenerator.QUOTE; + bq.query(QueryJobConfiguration.of( + new StringSubstitutor(Map.of( + "raw_table", streamConfig.id().rawTableId(BigQuerySqlGenerator.QUOTE), + "tmp_raw_table", tmpRawTableId, + "real_raw_table", BigQuerySqlGenerator.QUOTE + streamConfig.id().rawName() + BigQuerySqlGenerator.QUOTE)).replace( + // In full refresh / append mode, standard inserts is creating a non-partitioned raw table. + // (possibly also in overwrite mode?). + // We can't just CREATE OR REPLACE the table because bigquery will complain that we're trying to + // change the partitioning scheme. + // Do an explicit CREATE tmp + DROP + RENAME, similar to how we overwrite the final tables in + // OVERWRITE mode. + """ + CREATE TABLE ${tmp_raw_table} + PARTITION BY DATE(_airbyte_extracted_at) + CLUSTER BY _airbyte_extracted_at + AS ( + SELECT + _airbyte_raw_id, + _airbyte_extracted_at, + _airbyte_loaded_at, + to_json_string(_airbyte_data) as _airbyte_data + FROM ${raw_table} + ); + DROP TABLE IF EXISTS ${raw_table}; + ALTER TABLE ${tmp_raw_table} RENAME TO ${real_raw_table}; + """))); + LOGGER.info("Completed Data column Migration for stream {}", streamConfig.id().rawName()); + } else { + LOGGER.info("No Data column Migration Required for stream {}", streamConfig.id().rawName()); + } + } + } + +} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/AbstractBigQueryUploader.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/AbstractBigQueryUploader.java index 390e2f7fdfd8..34b425cae7f5 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/AbstractBigQueryUploader.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/AbstractBigQueryUploader.java @@ -18,12 +18,13 @@ import com.google.cloud.bigquery.Table; import com.google.cloud.bigquery.TableId; import com.google.cloud.bigquery.TableInfo; +import io.airbyte.cdk.integrations.base.AirbyteExceptionHandler; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.s3.writer.DestinationWriter; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; import io.airbyte.commons.string.Strings; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.base.TypingAndDedupingFlag; import io.airbyte.integrations.destination.bigquery.BigQueryUtils; import io.airbyte.integrations.destination.bigquery.formatter.BigQueryRecordFormatter; -import io.airbyte.integrations.destination.s3.writer.DestinationWriter; import io.airbyte.protocol.models.v0.AirbyteMessage; import java.io.IOException; import java.util.function.Consumer; @@ -37,32 +38,23 @@ public abstract class AbstractBigQueryUploader { private static final Logger LOGGER = LoggerFactory.getLogger(AbstractBigQueryUploader.class); protected final TableId table; - protected final TableId tmpTable; protected final WriteDisposition syncMode; protected final T writer; protected final BigQuery bigQuery; protected final BigQueryRecordFormatter recordFormatter; - protected final boolean use1s1t; AbstractBigQueryUploader(final TableId table, - final TableId tmpTable, final T writer, final WriteDisposition syncMode, final BigQuery bigQuery, final BigQueryRecordFormatter recordFormatter) { - this.use1s1t = TypingAndDedupingFlag.isDestinationV2(); this.table = table; - this.tmpTable = tmpTable; this.writer = writer; this.syncMode = syncMode; this.bigQuery = bigQuery; this.recordFormatter = recordFormatter; } - public BigQueryRecordFormatter getRecordFormatter() { - return recordFormatter; - } - protected void postProcessAction(final boolean hasFailed) throws Exception { // Do nothing by default } @@ -73,9 +65,21 @@ public void upload(final AirbyteMessage airbyteMessage) { } catch (final IOException | RuntimeException e) { LOGGER.error("Got an error while writing message: {}", e.getMessage(), e); LOGGER.error(String.format( - "Failed to process a message for job: \n%s, \nAirbyteMessage: %s", - writer.toString(), - airbyteMessage.getRecord())); + "Failed to process a message for job: %s", + writer.toString())); + printHeapMemoryConsumption(); + throw new RuntimeException(e); + } + } + + public void upload(final PartialAirbyteMessage airbyteMessage) { + try { + writer.write(recordFormatter.formatRecord(airbyteMessage)); + } catch (final IOException | RuntimeException e) { + LOGGER.error("Got an error while writing message: {}", e.getMessage(), e); + LOGGER.error(String.format( + "Failed to process a message for job: %s", + writer.toString())); printHeapMemoryConsumption(); throw new RuntimeException(e); } @@ -85,14 +89,12 @@ public void close(final boolean hasFailed, final Consumer output try { recordFormatter.printAndCleanFieldFails(); - LOGGER.info("Closing connector: {}", this); this.writer.close(hasFailed); if (!hasFailed) { uploadData(outputRecordCollector, lastStateMessage); } this.postProcessAction(hasFailed); - LOGGER.info("Closed connector: {}", this); } catch (final Exception e) { LOGGER.error(String.format("Failed to close %s writer, \n details: %s", this, e.getMessage())); printHeapMemoryConsumption(); @@ -100,22 +102,21 @@ public void close(final boolean hasFailed, final Consumer output } } - protected void uploadData(final Consumer outputRecordCollector, final AirbyteMessage lastStateMessage) throws Exception { + public void closeAfterPush() { try { - if (!use1s1t) { - // This only needs to happen if we actually wrote to a tmp table. - LOGGER.info("Uploading data from the tmp table {} to the source table {}.", tmpTable.getTable(), table.getTable()); - uploadDataToTableFromTmpTable(); - LOGGER.info("Data is successfully loaded to the source table {}!", table.getTable()); - } + this.writer.close(false); + } catch (final IOException e) { + throw new RuntimeException(e); + } + } + protected void uploadData(final Consumer outputRecordCollector, final AirbyteMessage lastStateMessage) throws Exception { + try { outputRecordCollector.accept(lastStateMessage); LOGGER.info("Final state message is accepted."); } catch (final Exception e) { LOGGER.error("Upload data is failed!"); throw e; - } finally { - dropTmpTable(); } } @@ -131,25 +132,6 @@ public void createRawTable() { } } - protected void dropTmpTable() { - try { - // clean up tmp tables; - LOGGER.info("Removing tmp tables..."); - bigQuery.delete(tmpTable); - LOGGER.info("Finishing destination process...completed"); - } catch (final Exception e) { - LOGGER.error("Fail to tmp table drop table: " + e.getMessage()); - } - } - - protected void uploadDataToTableFromTmpTable() { - LOGGER.info("Replication finished with no explicit errors. Copying data from tmp tables to permanent"); - if (syncMode.equals(JobInfo.WriteDisposition.WRITE_APPEND)) { - partitionIfUnpartitioned(bigQuery, recordFormatter.getBigQuerySchema(), table); - } - copyTable(bigQuery, tmpTable, table, syncMode); - } - /** * Creates a partitioned table if the table previously was not partitioned * @@ -229,6 +211,7 @@ public static void copyTable(final BigQuery bigQuery, .build(); final Job job = bigQuery.create(JobInfo.of(configuration)); + AirbyteExceptionHandler.addStringForDeinterpolation(job.getEtag()); final ImmutablePair jobStringImmutablePair = BigQueryUtils.executeQuery(job); if (jobStringImmutablePair.getRight() != null) { LOGGER.error("Failed on copy tables with error:" + job.getStatus()); @@ -258,7 +241,6 @@ private static String getCreatePartitionedTableFromSelectQuery(final Schema sche public String toString() { return "AbstractBigQueryUploader{" + "table=" + table.getTable() + - ", tmpTable=" + tmpTable.getTable() + ", syncMode=" + syncMode + ", writer=" + writer.getClass() + ", recordFormatter=" + recordFormatter.getClass() + diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/AbstractGscBigQueryUploader.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/AbstractGscBigQueryUploader.java deleted file mode 100644 index 97cbda3763ad..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/AbstractGscBigQueryUploader.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery.uploader; - -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.BigQueryException; -import com.google.cloud.bigquery.Job; -import com.google.cloud.bigquery.JobInfo; -import com.google.cloud.bigquery.JobInfo.WriteDisposition; -import com.google.cloud.bigquery.LoadJobConfiguration; -import com.google.cloud.bigquery.TableId; -import io.airbyte.integrations.destination.bigquery.BigQueryUtils; -import io.airbyte.integrations.destination.bigquery.formatter.BigQueryRecordFormatter; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.s3.writer.DestinationFileWriter; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import java.util.List; -import java.util.function.Consumer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public abstract class AbstractGscBigQueryUploader extends AbstractBigQueryUploader { - - private static final Logger LOGGER = LoggerFactory.getLogger(AbstractGscBigQueryUploader.class); - - private final boolean isKeepFilesInGcs; - protected final GcsDestinationConfig gcsDestinationConfig; - - AbstractGscBigQueryUploader(final TableId table, - final TableId tmpTable, - final T writer, - final WriteDisposition syncMode, - final GcsDestinationConfig gcsDestinationConfig, - final BigQuery bigQuery, - final boolean isKeepFilesInGcs, - final BigQueryRecordFormatter recordFormatter) { - super(table, tmpTable, writer, syncMode, bigQuery, recordFormatter); - this.isKeepFilesInGcs = isKeepFilesInGcs; - this.gcsDestinationConfig = gcsDestinationConfig; - } - - @Override - public void postProcessAction(final boolean hasFailed) { - if (!isKeepFilesInGcs) { - deleteGcsFiles(); - } - } - - @Override - protected void uploadData(final Consumer outputRecordCollector, final AirbyteMessage lastStateMessage) throws Exception { - LOGGER.info("Uploading data to the tmp table {}.", tmpTable.getTable()); - uploadDataFromFileToTmpTable(); - super.uploadData(outputRecordCollector, lastStateMessage); - } - - protected void uploadDataFromFileToTmpTable() { - try { - final String fileLocation = this.writer.getFileLocation(); - - // Initialize client that will be used to send requests. This client only needs to be created - // once, and can be reused for multiple requests. - LOGGER.info(String.format("Started copying data from %s GCS " + getFileTypeName() + " file to %s tmp BigQuery table with schema: \n %s", - fileLocation, tmpTable, recordFormatter.getBigQuerySchema())); - - final LoadJobConfiguration configuration = getLoadConfiguration(); - - // For more information on Job see: - // https://googleapis.dev/java/google-cloud-clients/latest/index.html?com/google/cloud/bigquery/package-summary.html - // Load the table - final Job loadJob = this.bigQuery.create(JobInfo.of(configuration)); - LOGGER.info("Created a new job GCS " + getFileTypeName() + " file to tmp BigQuery table: " + loadJob); - - // Load data from a GCS parquet file into the table - // Blocks until this load table job completes its execution, either failing or succeeding. - BigQueryUtils.waitForJobFinish(loadJob); - - LOGGER.info("Table is successfully overwritten by file loaded from GCS: {}", getFileTypeName()); - } catch (final BigQueryException | InterruptedException e) { - LOGGER.error("Column not added during load append", e); - throw new RuntimeException("Column not added during load append \n" + e.toString()); - } - } - - abstract protected LoadJobConfiguration getLoadConfiguration(); - - private String getFileTypeName() { - return writer.getFileFormat().getFileExtension(); - } - - private void deleteGcsFiles() { - LOGGER.info("Deleting file {}", writer.getFileLocation()); - final GcsDestinationConfig gcsDestinationConfig = this.gcsDestinationConfig; - final AmazonS3 s3Client = gcsDestinationConfig.getS3Client(); - - final String gcsBucketName = gcsDestinationConfig.getBucketName(); - final String gcs_bucket_path = gcsDestinationConfig.getBucketPath(); - - final List objects = s3Client - .listObjects(gcsBucketName, gcs_bucket_path) - .getObjectSummaries(); - - objects.stream().filter(s3ObjectSummary -> s3ObjectSummary.getKey().equals(writer.getOutputPath())).forEach(s3ObjectSummary -> { - s3Client.deleteObject(gcsBucketName, new DeleteObjectsRequest.KeyVersion(s3ObjectSummary.getKey()).getKey()); - LOGGER.info("File is deleted : " + s3ObjectSummary.getKey()); - }); - s3Client.shutdown(); - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryDirectUploader.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryDirectUploader.java index 81a4641395ff..2a464e366645 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryDirectUploader.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryDirectUploader.java @@ -5,7 +5,7 @@ package io.airbyte.integrations.destination.bigquery.uploader; import com.google.cloud.bigquery.BigQuery; -import com.google.cloud.bigquery.JobInfo; +import com.google.cloud.bigquery.JobInfo.WriteDisposition; import com.google.cloud.bigquery.TableId; import io.airbyte.integrations.destination.bigquery.BigQueryUtils; import io.airbyte.integrations.destination.bigquery.formatter.BigQueryRecordFormatter; @@ -16,12 +16,11 @@ public class BigQueryDirectUploader extends AbstractBigQueryUploader { public BigQueryDirectUploader(final TableId table, - final TableId tmpTable, final BigQueryTableWriter writer, - final JobInfo.WriteDisposition syncMode, + final WriteDisposition syncMode, final BigQuery bigQuery, final BigQueryRecordFormatter recordFormatter) { - super(table, tmpTable, writer, syncMode, bigQuery, recordFormatter); + super(table, writer, syncMode, bigQuery, recordFormatter); } @Override diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryUploaderFactory.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryUploaderFactory.java index 3ee2fdafa23b..6eca8c9f947e 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryUploaderFactory.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/BigQueryUploaderFactory.java @@ -4,32 +4,23 @@ package io.airbyte.integrations.destination.bigquery.uploader; -import static io.airbyte.integrations.destination.s3.avro.AvroConstants.JSON_CONVERTER; -import static software.amazon.awssdk.http.HttpStatusCode.FORBIDDEN; -import static software.amazon.awssdk.http.HttpStatusCode.NOT_FOUND; - -import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.JsonNode; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryException; import com.google.cloud.bigquery.FormatOptions; import com.google.cloud.bigquery.JobId; import com.google.cloud.bigquery.JobInfo; +import com.google.cloud.bigquery.JobInfo.WriteDisposition; import com.google.cloud.bigquery.Schema; import com.google.cloud.bigquery.TableDataWriteChannel; import com.google.cloud.bigquery.TableId; import com.google.cloud.bigquery.WriteChannelConfiguration; import io.airbyte.commons.exceptions.ConfigErrorException; -import io.airbyte.integrations.base.TypingAndDedupingFlag; import io.airbyte.integrations.destination.bigquery.BigQueryUtils; import io.airbyte.integrations.destination.bigquery.formatter.BigQueryRecordFormatter; import io.airbyte.integrations.destination.bigquery.uploader.config.UploaderConfig; import io.airbyte.integrations.destination.bigquery.writer.BigQueryTableWriter; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.avro.GcsAvroWriter; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.io.IOException; -import java.sql.Timestamp; import java.util.HashSet; import java.util.Set; import org.slf4j.Logger; @@ -39,6 +30,9 @@ public class BigQueryUploaderFactory { private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryUploaderFactory.class); + private static final int HTTP_STATUS_CODE_FORBIDDEN = 403; + private static final int HTTP_STATUS_CODE_NOT_FOUND = 404; + private static final String CONFIG_ERROR_MSG = """ Failed to write to destination schema. @@ -51,18 +45,11 @@ public class BigQueryUploaderFactory { "Destination Default" option. More details: - """; + """; public static AbstractBigQueryUploader getUploader(final UploaderConfig uploaderConfig) throws IOException { - final String dataset; - if (TypingAndDedupingFlag.isDestinationV2()) { - dataset = uploaderConfig.getParsedStream().id().rawNamespace(); - } else { - // This previously needed to handle null namespaces. That's now happening at the top of the - // connector, so we can assume namespace is non-null here. - dataset = BigQueryUtils.sanitizeDatasetId(uploaderConfig.getConfigStream().getStream().getNamespace()); - } + final String dataset = uploaderConfig.getParsedStream().id().rawNamespace(); final String datasetLocation = BigQueryUtils.getDatasetLocation(uploaderConfig.getConfig()); final Set existingDatasets = new HashSet<>(); @@ -70,98 +57,37 @@ public static AbstractBigQueryUploader getUploader(final UploaderConfig uploa final Schema bigQuerySchema = recordFormatter.getBigQuerySchema(); final TableId targetTable = TableId.of(dataset, uploaderConfig.getTargetTableName()); - final TableId tmpTable = TableId.of(dataset, uploaderConfig.getTmpTableName()); BigQueryUtils.createSchemaAndTableIfNeeded( uploaderConfig.getBigQuery(), existingDatasets, dataset, - tmpTable, datasetLocation, bigQuerySchema); final JobInfo.WriteDisposition syncMode = BigQueryUtils.getWriteDisposition( uploaderConfig.getConfigStream().getDestinationSyncMode()); - return (uploaderConfig.isGcsUploadingMode() - ? getGcsBigQueryUploader( - uploaderConfig.getConfig(), - uploaderConfig.getConfigStream(), - targetTable, - tmpTable, - uploaderConfig.getBigQuery(), - syncMode, - recordFormatter, - uploaderConfig.isDefaultAirbyteTmpSchema()) - : getBigQueryDirectUploader( - uploaderConfig.getConfig(), - targetTable, - tmpTable, - uploaderConfig.getBigQuery(), - syncMode, - datasetLocation, - recordFormatter)); - } - - private static AbstractGscBigQueryUploader getGcsBigQueryUploader( - final JsonNode config, - final ConfiguredAirbyteStream configStream, - final TableId targetTable, - final TableId tmpTable, - final BigQuery bigQuery, - final JobInfo.WriteDisposition syncMode, - final BigQueryRecordFormatter formatter, - final boolean isDefaultAirbyteTmpSchema) - throws IOException { - - final GcsDestinationConfig gcsDestinationConfig = BigQueryUtils.getGcsAvroDestinationConfig(config); - final JsonNode tmpTableSchema = - (isDefaultAirbyteTmpSchema ? null : formatter.getJsonSchema()); - final GcsAvroWriter gcsCsvWriter = - initGcsWriter(gcsDestinationConfig, configStream, tmpTableSchema); - gcsCsvWriter.initialize(); - - return new GcsAvroBigQueryUploader( + return getBigQueryDirectUploader( + uploaderConfig.getConfig(), targetTable, - tmpTable, - gcsCsvWriter, + uploaderConfig.getBigQuery(), syncMode, - gcsDestinationConfig, - bigQuery, - BigQueryUtils.isKeepFilesInGcs(config), - formatter); - } - - private static GcsAvroWriter initGcsWriter( - final GcsDestinationConfig gcsDestinationConfig, - final ConfiguredAirbyteStream configuredStream, - final JsonNode jsonSchema) - throws IOException { - final Timestamp uploadTimestamp = new Timestamp(System.currentTimeMillis()); - - final AmazonS3 s3Client = gcsDestinationConfig.getS3Client(); - return new GcsAvroWriter( - gcsDestinationConfig, - s3Client, - configuredStream, - uploadTimestamp, - JSON_CONVERTER, - jsonSchema); + datasetLocation, + recordFormatter); } private static BigQueryDirectUploader getBigQueryDirectUploader( final JsonNode config, final TableId targetTable, - final TableId tmpTable, final BigQuery bigQuery, - final JobInfo.WriteDisposition syncMode, + final WriteDisposition syncMode, final String datasetLocation, final BigQueryRecordFormatter formatter) { // https://cloud.google.com/bigquery/docs/loading-data-local#loading_data_from_a_local_data_source - final TableId tableToWriteRawData = TypingAndDedupingFlag.isDestinationV2() ? targetTable : tmpTable; - LOGGER.info("Will write raw data to {} with schema {}", tableToWriteRawData, formatter.getBigQuerySchema()); + LOGGER.info("Will write raw data to {} with schema {}", targetTable, formatter.getBigQuerySchema()); final WriteChannelConfiguration writeChannelConfiguration = - WriteChannelConfiguration.newBuilder(tableToWriteRawData) + WriteChannelConfiguration.newBuilder(targetTable) .setCreateDisposition(JobInfo.CreateDisposition.CREATE_IF_NEEDED) .setSchema(formatter.getBigQuerySchema()) .setFormatOptions(FormatOptions.json()) @@ -178,7 +104,7 @@ private static BigQueryDirectUploader getBigQueryDirectUploader( try { writer = bigQuery.writer(job, writeChannelConfiguration); } catch (final BigQueryException e) { - if (e.getCode() == FORBIDDEN || e.getCode() == NOT_FOUND) { + if (e.getCode() == HTTP_STATUS_CODE_FORBIDDEN || e.getCode() == HTTP_STATUS_CODE_NOT_FOUND) { throw new ConfigErrorException(CONFIG_ERROR_MSG + e); } else { throw new BigQueryException(e.getCode(), e.getMessage()); @@ -194,7 +120,6 @@ private static BigQueryDirectUploader getBigQueryDirectUploader( return new BigQueryDirectUploader( targetTable, - tmpTable, new BigQueryTableWriter(writer), syncMode, bigQuery, diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/GcsAvroBigQueryUploader.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/GcsAvroBigQueryUploader.java deleted file mode 100644 index 023f5c9ef120..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/GcsAvroBigQueryUploader.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery.uploader; - -import com.google.cloud.bigquery.*; -import io.airbyte.integrations.destination.bigquery.formatter.BigQueryRecordFormatter; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.avro.GcsAvroWriter; - -public class GcsAvroBigQueryUploader extends AbstractGscBigQueryUploader { - - public GcsAvroBigQueryUploader(TableId table, - TableId tmpTable, - GcsAvroWriter writer, - JobInfo.WriteDisposition syncMode, - GcsDestinationConfig gcsDestinationConfig, - BigQuery bigQuery, - boolean isKeepFilesInGcs, - BigQueryRecordFormatter recordFormatter) { - super(table, tmpTable, writer, syncMode, gcsDestinationConfig, bigQuery, isKeepFilesInGcs, recordFormatter); - } - - @Override - protected LoadJobConfiguration getLoadConfiguration() { - return LoadJobConfiguration.builder(tmpTable, writer.getFileLocation()).setFormatOptions(FormatOptions.avro()) - .setSchema(recordFormatter.getBigQuerySchema()) - .setWriteDisposition(syncMode) - .setUseAvroLogicalTypes(true) - .build(); - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/GcsCsvBigQueryUploader.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/GcsCsvBigQueryUploader.java deleted file mode 100644 index 9abbe21565b2..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/GcsCsvBigQueryUploader.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery.uploader; - -import static com.amazonaws.util.StringUtils.UTF8; - -import com.google.cloud.bigquery.*; -import io.airbyte.integrations.destination.bigquery.formatter.BigQueryRecordFormatter; -import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.gcs.csv.GcsCsvWriter; - -public class GcsCsvBigQueryUploader extends AbstractGscBigQueryUploader { - - public GcsCsvBigQueryUploader(TableId table, - TableId tmpTable, - GcsCsvWriter writer, - JobInfo.WriteDisposition syncMode, - GcsDestinationConfig gcsDestinationConfig, - BigQuery bigQuery, - boolean isKeepFilesInGcs, - BigQueryRecordFormatter recordFormatter) { - super(table, tmpTable, writer, syncMode, gcsDestinationConfig, bigQuery, isKeepFilesInGcs, recordFormatter); - } - - @Override - protected LoadJobConfiguration getLoadConfiguration() { - final var csvOptions = CsvOptions.newBuilder().setEncoding(UTF8).setSkipLeadingRows(1).build(); - - return LoadJobConfiguration.builder(tmpTable, writer.getFileLocation()) - .setFormatOptions(csvOptions) - .setSchema(recordFormatter.getBigQuerySchema()) - .setWriteDisposition(syncMode) - .build(); - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/config/UploaderConfig.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/config/UploaderConfig.java index 6dad08ee4e11..28e2f0ea3f24 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/config/UploaderConfig.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/uploader/config/UploaderConfig.java @@ -31,7 +31,6 @@ public class UploaderConfig { */ private StreamConfig parsedStream; private String targetTableName; - private String tmpTableName; private BigQuery bigQuery; private Map formatterMap; private boolean isDefaultAirbyteTmpSchema; @@ -41,7 +40,7 @@ public boolean isGcsUploadingMode() { } public UploaderType getUploaderType() { - return (isGcsUploadingMode() ? UploaderType.AVRO : UploaderType.STANDARD); + return (isGcsUploadingMode() ? UploaderType.CSV : UploaderType.STANDARD); } public BigQueryRecordFormatter getFormatter() { diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/writer/BigQueryTableWriter.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/writer/BigQueryTableWriter.java index c7b695bcedf9..3127905d08b6 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/writer/BigQueryTableWriter.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/writer/BigQueryTableWriter.java @@ -5,10 +5,12 @@ package io.airbyte.integrations.destination.bigquery.writer; import com.fasterxml.jackson.databind.JsonNode; +import com.google.cloud.bigquery.Job; import com.google.cloud.bigquery.TableDataWriteChannel; import com.google.common.base.Charsets; +import io.airbyte.cdk.integrations.base.AirbyteExceptionHandler; +import io.airbyte.cdk.integrations.destination.s3.writer.DestinationWriter; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.s3.writer.DestinationWriter; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.io.IOException; import java.nio.ByteBuffer; @@ -22,7 +24,7 @@ public class BigQueryTableWriter implements DestinationWriter { private final TableDataWriteChannel writeChannel; - public BigQueryTableWriter(TableDataWriteChannel writeChannel) { + public BigQueryTableWriter(final TableDataWriteChannel writeChannel) { this.writeChannel = writeChannel; } @@ -30,18 +32,33 @@ public BigQueryTableWriter(TableDataWriteChannel writeChannel) { public void initialize() throws IOException {} @Override - public void write(UUID id, AirbyteRecordMessage recordMessage) { + public void write(final UUID id, final AirbyteRecordMessage recordMessage) { throw new RuntimeException("This write method is not used!"); } @Override - public void write(JsonNode formattedData) throws IOException { + public void write(final JsonNode formattedData) throws IOException { writeChannel.write(ByteBuffer.wrap((Jsons.serialize(formattedData) + "\n").getBytes(Charsets.UTF_8))); } @Override - public void close(boolean hasFailed) throws IOException { + public void write(final String formattedData) throws IOException { + writeChannel.write(ByteBuffer.wrap((formattedData + "\n").getBytes(Charsets.UTF_8))); + } + + @Override + public void close(final boolean hasFailed) throws IOException { this.writeChannel.close(); + try { + final Job job = writeChannel.getJob(); + if (job != null && job.getStatus().getError() != null) { + AirbyteExceptionHandler.addStringForDeinterpolation(job.getEtag()); + throw new RuntimeException("Fail to complete a load job in big query, Job id: " + writeChannel.getJob().getJobId() + + ", with error: " + writeChannel.getJob().getStatus().getError()); + } + } catch (final Exception e) { + throw new RuntimeException(e); + } } public TableDataWriteChannel getWriteChannel() { diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json index da8b9d83093b..41dafa21cc72 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-bigquery/src/main/resources/spec.json @@ -1,7 +1,6 @@ { "documentationUrl": "https://docs.airbyte.com/integrations/destinations/bigquery", "supportsIncremental": true, - "supportsNormalization": true, "supportsDBT": true, "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"], "connectionSpecification": { @@ -15,12 +14,14 @@ "type": "string", "description": "The GCP project ID for the project containing the target BigQuery dataset. Read more here.", "title": "Project ID", + "group": "connection", "order": 0 }, "dataset_location": { "type": "string", "description": "The location of the dataset. Warning: Changes made after creation will not be applied. Read more here.", "title": "Dataset Location", + "group": "connection", "order": 1, "enum": [ "US", @@ -48,6 +49,9 @@ "europe-west7", "europe-west8", "europe-west9", + "europe-west12", + "me-central1", + "me-central2", "me-west1", "northamerica-northeast1", "northamerica-northeast2", @@ -59,6 +63,7 @@ "us-east3", "us-east4", "us-east5", + "us-south1", "us-west1", "us-west2", "us-west3", @@ -69,26 +74,20 @@ "type": "string", "description": "The default BigQuery Dataset ID that tables are replicated to if the source does not specify a namespace. Read more here.", "title": "Default Dataset ID", + "group": "connection", "order": 2 }, "loading_method": { "type": "object", "title": "Loading Method", - "description": "Loading method used to send select the way data will be uploaded to BigQuery.
      Standard Inserts - Direct uploading using SQL INSERT statements. This method is extremely inefficient and provided only for quick testing. In almost all cases, you should use staging.
      GCS Staging - Writes large batches of records to a file, uploads the file to GCS, then uses COPY INTO table to upload the file. Recommended for most workloads for better speed and scalability. Read more about GCS Staging here.", + "description": "The way data will be uploaded to BigQuery.", + "display_type": "radio", + "group": "connection", "order": 3, "oneOf": [ - { - "title": "Standard Inserts", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "Standard" - } - } - }, { "title": "GCS Staging", + "description": "(recommended) Writes large batches of records to a file, uploads the file to GCS, then uses COPY INTO to load your data into BigQuery. Provides best-in-class speed, reliability and scalability. Read more about GCS Staging here.", "required": [ "method", "gcs_bucket_name", @@ -166,16 +165,17 @@ "Keep all tmp files in GCS" ], "order": 4 - }, - "file_buffer_count": { - "title": "File Buffer Count", - "type": "integer", - "minimum": 10, - "maximum": 50, - "default": 10, - "description": "Number of file buffers allocated for writing data. Increasing this number is beneficial for connections using Change Data Capture (CDC) and up to the number of streams within a connection. Increasing the number of file buffers past the maximum number of streams has deteriorating effects", - "examples": ["10"], - "order": 5 + } + } + }, + { + "title": "Standard Inserts", + "required": ["method"], + "description": "(not recommended) Direct loading using SQL INSERT statements. This method is extremely inefficient and provided only for quick testing. In all other cases, you should use GCS staging.", + "properties": { + "method": { + "type": "string", + "const": "Standard" } } } @@ -186,6 +186,7 @@ "description": "The contents of the JSON service account key. Check out the docs if you need help generating this key. Default credentials will be used if this field is left empty.", "title": "Service Account Key JSON (Required for cloud, optional for open-source)", "airbyte_secret": true, + "group": "connection", "order": 4, "always_show": true }, @@ -195,7 +196,8 @@ "title": "Transformation Query Run Type", "default": "interactive", "enum": ["interactive", "batch"], - "order": 5 + "order": 5, + "group": "advanced" }, "big_query_client_buffer_size_mb": { "title": "Google BigQuery Client Chunk Size", @@ -205,20 +207,34 @@ "maximum": 15, "default": 15, "examples": ["15"], - "order": 6 - }, - "use_1s1t_format": { - "type": "boolean", - "description": "(Early Access) Use Destinations V2.", - "title": "Use Destinations V2 (Early Access)", - "order": 7 + "order": 6, + "group": "advanced" }, "raw_data_dataset": { "type": "string", - "description": "(Early Access) The dataset to write raw tables into", - "title": "Destinations V2 Raw Table Dataset (Early Access)", - "order": 8 + "description": "The dataset to write raw tables into (default: airbyte_internal)", + "title": "Raw Table Dataset Name", + "order": 7, + "group": "advanced" + }, + "disable_type_dedupe": { + "type": "boolean", + "default": false, + "description": "Disable Writing Final Tables. WARNING! The data format in _airbyte_data is likely stable but there are no guarantees that other metadata columns will remain the same in future versions", + "title": "Disable Final Tables. (WARNING! Unstable option; Columns in raw table schema might change between versions)", + "order": 8, + "group": "advanced" + } + }, + "groups": [ + { + "id": "connection", + "title": "Connection" + }, + { + "id": "advanced", + "title": "Advanced" } - } + ] } } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/AbstractBigQueryDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/AbstractBigQueryDestinationAcceptanceTest.java index f15a5f30072e..0287efefe2ff 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/AbstractBigQueryDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/AbstractBigQueryDestinationAcceptanceTest.java @@ -18,16 +18,18 @@ import com.google.cloud.bigquery.QueryJobConfiguration; import com.google.cloud.bigquery.TableResult; import com.google.common.collect.Streams; +import io.airbyte.cdk.db.bigquery.BigQueryResultSet; +import io.airbyte.cdk.db.bigquery.BigQuerySourceOperations; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.TestingNamespaces; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.bigquery.BigQueryResultSet; -import io.airbyte.db.bigquery.BigQuerySourceOperations; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.TestingNamespaces; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import io.airbyte.integrations.destination.bigquery.typing_deduping.BigQuerySqlGenerator; import java.io.IOException; import java.nio.file.Path; import java.util.Collections; @@ -36,9 +38,11 @@ import java.util.TimeZone; import java.util.stream.Collectors; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@Disabled public abstract class AbstractBigQueryDestinationAcceptanceTest extends DestinationAcceptanceTest { private static final NamingConventionTransformer NAME_TRANSFORMER = new BigQuerySQLNameTransformer(); @@ -103,11 +107,6 @@ protected boolean supportIncrementalSchemaChanges() { return true; } - @Override - protected boolean supportsInDestinationNormalization() { - return true; - } - @Override protected Optional getNameTransformer() { return Optional.of(NAME_TRANSFORMER); @@ -141,21 +140,15 @@ protected String getDefaultSchema(final JsonNode config) { return BigQueryUtils.getDatasetId(config); } - @Override - protected List retrieveNormalizedRecords(final TestDestinationEnv testEnv, final String streamName, final String namespace) - throws Exception { - final String tableName = namingResolver.getIdentifier(streamName); - final String schema = namingResolver.getIdentifier(namespace); - return retrieveRecordsFromTable(tableName, schema); - } - @Override protected List retrieveRecords(final TestDestinationEnv env, final String streamName, final String namespace, final JsonNode streamSchema) throws Exception { - return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namingResolver.getIdentifier(namespace)) + final StreamId streamId = + new BigQuerySqlGenerator(null, null).buildStreamId(namespace, streamName, JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE); + return retrieveRecordsFromTable(streamId.rawName(), streamId.rawNamespace()) .stream() .map(node -> node.get(JavaBaseConstants.COLUMN_NAME_DATA).asText()) .map(Jsons::deserialize) @@ -169,7 +162,7 @@ protected List retrieveRecordsFromTable(final String tableName, final QueryJobConfiguration .newBuilder( String.format("SELECT * FROM `%s`.`%s` order by %s asc;", schema, tableName, - JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) + JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT)) .setUseLegacySql(false) .setConnectionProperties(Collections.singletonList(ConnectionProperty.of("time_zone", "UTC"))) .build(); diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java index de75a017a3d7..68f56c0fe41a 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryDestinationTest.java @@ -25,14 +25,16 @@ import com.google.cloud.bigquery.TableInfo; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.DestinationConfig; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.string.Strings; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.DestinationConfig; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.NamingConventionTransformer; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import io.airbyte.integrations.destination.bigquery.typing_deduping.BigQuerySqlGenerator; import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; @@ -63,6 +65,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.TestInstance; @@ -172,7 +175,7 @@ public static void beforeAll() throws IOException { } datasetId = Strings.addRandomSuffix(DATASET_NAME_PREFIX, "_", 8); - String stagingPath = Strings.addRandomSuffix("test_path", "_", 8); + final String stagingPath = Strings.addRandomSuffix("test_path", "_", 8); // Set up config objects for test scenarios // config - basic config for standard inserts that should succeed check and write tests // this config is also used for housekeeping (checking records, and cleaning up) @@ -287,6 +290,7 @@ void testCheckFailures(final String configName, final String error) { assertThat(ex.getMessage()).contains(error); } + @Disabled @ParameterizedTest @MethodSource("successTestConfigProvider") void testWriteSuccess(final String configName) throws Exception { @@ -345,6 +349,7 @@ void testCreateTableSuccessWhenTableAlreadyExists() throws Exception { }); } + @Disabled @ParameterizedTest @MethodSource("failWriteTestConfigProvider") void testWriteFailure(final String configName, final String error) throws Exception { @@ -412,14 +417,17 @@ private List retrieveRecords(final String tableName) throws Exception .collect(Collectors.toList()); } + @Disabled @ParameterizedTest @MethodSource("successTestConfigProviderBase") void testWritePartitionOverUnpartitioned(final String configName) throws Exception { final JsonNode testConfig = configs.get(configName); initBigQuery(config); - final String raw_table_name = String.format("_airbyte_raw_%s", USERS_STREAM_NAME); - createUnpartitionedTable(bigquery, dataset, raw_table_name); - assertFalse(isTablePartitioned(bigquery, dataset, raw_table_name)); + final StreamId streamId = + new BigQuerySqlGenerator(projectId, null).buildStreamId(datasetId, USERS_STREAM_NAME, JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE); + final Dataset dataset = BigQueryDestinationTestUtils.initDataSet(config, bigquery, streamId.rawNamespace()); + createUnpartitionedTable(bigquery, dataset, streamId.rawName()); + assertFalse(isTablePartitioned(bigquery, dataset, streamId.rawName())); final BigQueryDestination destination = new BigQueryDestination(); final AirbyteMessageConsumer consumer = destination.getConsumer(testConfig, catalog, Destination::defaultOutputRecordCollector); @@ -446,7 +454,7 @@ void testWritePartitionOverUnpartitioned(final String configName) throws Excepti .map(ConfiguredAirbyteStream::getStream) .map(AirbyteStream::getName) .collect(Collectors.toList())); - assertTrue(isTablePartitioned(bigquery, dataset, raw_table_name)); + assertTrue(isTablePartitioned(bigquery, dataset, streamId.rawName())); } private void createUnpartitionedTable(final BigQuery bigquery, final Dataset dataset, final String tableName) { diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java index eb60ef15aaa7..f17b12320adc 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryGcsDestinationAcceptanceTest.java @@ -4,24 +4,20 @@ package io.airbyte.integrations.destination.bigquery; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; import com.amazonaws.services.s3.AmazonS3; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.DestinationConfig; +import io.airbyte.cdk.integrations.base.DestinationConfig; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.record_buffer.FileBuffer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import java.nio.file.Path; import java.util.HashSet; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.TestInstance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@Disabled @TestInstance(PER_CLASS) public class BigQueryGcsDestinationAcceptanceTest extends AbstractBigQueryDestinationAcceptanceTest { @@ -69,32 +65,4 @@ protected void tearDownGcs() { BigQueryDestinationTestUtils.tearDownGcs(s3Client, config, LOGGER); } - /* - * FileBuffer Default Tests - */ - @Test - public void testGetFileBufferDefault() { - final BigQueryDestination destination = new BigQueryDestination(); - assertEquals(destination.getNumberOfFileBuffers(config), - FileBuffer.DEFAULT_MAX_CONCURRENT_STREAM_IN_BUFFER); - } - - @Test - public void testGetFileBufferMaxLimited() { - final JsonNode defaultConfig = Jsons.clone(config); - ((ObjectNode) defaultConfig.get(BigQueryConsts.LOADING_METHOD)).put(FileBuffer.FILE_BUFFER_COUNT_KEY, 100); - final BigQueryDestination destination = new BigQueryDestination(); - assertEquals(FileBuffer.MAX_CONCURRENT_STREAM_IN_BUFFER, destination.getNumberOfFileBuffers(defaultConfig)); - } - - @Test - public void testGetMinimumFileBufferCount() { - final JsonNode defaultConfig = Jsons.clone(config); - ((ObjectNode) defaultConfig.get(BigQueryConsts.LOADING_METHOD)).put(FileBuffer.FILE_BUFFER_COUNT_KEY, 1); - final BigQueryDestination destination = new BigQueryDestination(); - // User cannot set number of file counts below the default file buffer count, which is existing - // behavior - assertEquals(FileBuffer.DEFAULT_MAX_CONCURRENT_STREAM_IN_BUFFER, destination.getNumberOfFileBuffers(defaultConfig)); - } - } diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryStandardDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryStandardDestinationAcceptanceTest.java index 5e106e5b8caa..5287e439abc2 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryStandardDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryStandardDestinationAcceptanceTest.java @@ -6,13 +6,15 @@ import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; import java.nio.file.Path; import java.util.HashSet; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.TestInstance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@Disabled @TestInstance(PER_CLASS) public class BigQueryStandardDestinationAcceptanceTest extends AbstractBigQueryDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryTestDataComparator.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryTestDataComparator.java index 3044a83900ba..c9775de8bce8 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryTestDataComparator.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/BigQueryTestDataComparator.java @@ -5,9 +5,9 @@ package io.airbyte.integrations.destination.bigquery; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneOffset; diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/AbstractBigQueryTypingDedupingTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/AbstractBigQueryTypingDedupingTest.java index ffc5104f5cd7..cc9f499abdfe 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/AbstractBigQueryTypingDedupingTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/AbstractBigQueryTypingDedupingTest.java @@ -4,6 +4,9 @@ package io.airbyte.integrations.destination.bigquery.typing_deduping; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.cloud.bigquery.BigQuery; @@ -11,9 +14,12 @@ import com.google.cloud.bigquery.QueryJobConfiguration; import com.google.cloud.bigquery.TableId; import com.google.cloud.bigquery.TableResult; -import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.base.destination.typing_deduping.BaseTypingDedupingTest; +import io.airbyte.integrations.base.destination.typing_deduping.SqlGenerator; import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import io.airbyte.integrations.destination.bigquery.BigQueryConsts; import io.airbyte.integrations.destination.bigquery.BigQueryDestination; import io.airbyte.integrations.destination.bigquery.BigQueryDestinationTestUtils; import io.airbyte.integrations.destination.bigquery.BigQueryUtils; @@ -23,6 +29,7 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; import io.airbyte.protocol.models.v0.SyncMode; +import io.airbyte.workers.exception.TestHarnessException; import java.io.IOException; import java.nio.file.Path; import java.util.List; @@ -79,6 +86,11 @@ protected void teardownStreamAndNamespace(String streamNamespace, final String s bq.delete(DatasetId.of(streamNamespace), BigQuery.DatasetDeleteOption.deleteContents()); } + @Override + protected SqlGenerator getSqlGenerator() { + return new BigQuerySqlGenerator(getConfig().get(BigQueryConsts.CONFIG_PROJECT_ID).asText(), null); + } + /** * Run a sync using 1.9.0 (which is the highest version that still creates v2 raw tables with JSON * _airbyte_data). Then run a sync using our current version. @@ -97,23 +109,57 @@ public void testRawTableJsonToStringMigration() throws Exception { // First sync final List messages1 = readMessages("dat/sync1_messages.jsonl"); - runSync(catalog, messages1, "airbyte/destination-bigquery:1.9.0"); + runSync(catalog, messages1, "airbyte/destination-bigquery:1.9.0", config -> { + // Defensive to avoid weird behaviors or test failures if the original config is being altered by + // another thread, thanks jackson for a mutable JsonNode + JsonNode copiedConfig = Jsons.clone(config); + if (config instanceof ObjectNode) { + // Add opt-in T+D flag for older version. this is removed in newer version of the spec. + ((ObjectNode) copiedConfig).put("use_1s1t_format", true); + } + return copiedConfig; + }); // 1.9.0 is known-good, but we might as well check that we're in good shape before continuing. // If this starts erroring out because we added more test records and 1.9.0 had a latent bug, // just delete these three lines :P - final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_raw.jsonl"); + final List expectedRawRecords1 = readRecords("dat/sync1_expectedrecords_raw.jsonl"); final List expectedFinalRecords1 = readRecords("dat/sync1_expectedrecords_nondedup_final.jsonl"); - verifySyncResult(expectedRawRecords1, expectedFinalRecords1); + verifySyncResult(expectedRawRecords1, expectedFinalRecords1, disableFinalTableComparison()); // Second sync final List messages2 = readMessages("dat/sync2_messages.jsonl"); runSync(catalog, messages2); - final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_append_raw.jsonl"); + final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_raw.jsonl"); final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_append_final.jsonl"); - verifySyncResult(expectedRawRecords2, expectedFinalRecords2); + verifySyncResult(expectedRawRecords2, expectedFinalRecords2, disableFinalTableComparison()); + } + + @Test + public void testRemovingPKNonNullIndexes() throws Exception { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP) + .withPrimaryKey(List.of(List.of("id1"), List.of("id2"))) + .withStream(new AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(SCHEMA)))); + + // First sync + final List messages = readMessages("dat/sync_null_pk.jsonl"); + final TestHarnessException e = assertThrows( + TestHarnessException.class, + () -> runSync(catalog, messages, "airbyte/destination-bigquery:2.0.20")); // this version introduced non-null PKs to the final tables + // ideally we would assert on the logged content of the original exception within e, but that is + // proving to be tricky + + // Second sync + runSync(catalog, messages); // does not throw with latest version + assertEquals(1, dumpFinalTableRecords(streamNamespace, streamName).toArray().length); } /** diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryGcsRawOverrideDisableTypingDedupingTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryGcsRawOverrideDisableTypingDedupingTest.java new file mode 100644 index 000000000000..b2f5f7968f24 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryGcsRawOverrideDisableTypingDedupingTest.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.bigquery.typing_deduping; + +import org.junit.jupiter.api.Disabled; + +public class BigQueryGcsRawOverrideDisableTypingDedupingTest extends AbstractBigQueryTypingDedupingTest { + + @Override + public String getConfigPath() { + return "secrets/credentials-1s1t-disabletd-gcs-raw-override.json"; + } + + @Override + protected String getRawDataset() { + return "overridden_raw_dataset"; + } + + @Override + protected boolean disableFinalTableComparison() { + return true; + } + + @Override + @Disabled + public void testRemovingPKNonNullIndexes() throws Exception { + // Do nothing. + } + + @Override + @Disabled + public void identicalNameSimultaneousSync() throws Exception { + // TODO: create fixtures to verify how raw tables are affected. Base tests check for final tables. + } + +} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryGcsRawOverrideTypingDedupingTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryGcsRawOverrideTypingDedupingTest.java new file mode 100644 index 000000000000..66f665a81553 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryGcsRawOverrideTypingDedupingTest.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.bigquery.typing_deduping; + +public class BigQueryGcsRawOverrideTypingDedupingTest extends AbstractBigQueryTypingDedupingTest { + + @Override + public String getConfigPath() { + return "secrets/credentials-1s1t-gcs-raw-override.json"; + } + + @Override + protected String getRawDataset() { + return "overridden_raw_dataset"; + } + +} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGeneratorIntegrationTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGeneratorIntegrationTest.java index b9a30769fb9b..99ac8a8e75dd 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGeneratorIntegrationTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGeneratorIntegrationTest.java @@ -27,12 +27,14 @@ import com.google.cloud.bigquery.Table; import com.google.cloud.bigquery.TableDefinition; import com.google.cloud.bigquery.TableResult; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; import io.airbyte.integrations.base.destination.typing_deduping.BaseSqlGeneratorIntegrationTest; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import io.airbyte.integrations.destination.bigquery.BigQueryConsts; import io.airbyte.integrations.destination.bigquery.BigQueryDestination; import io.airbyte.protocol.models.v0.DestinationSyncMode; import io.airbyte.protocol.models.v0.SyncMode; @@ -60,17 +62,22 @@ public class BigQuerySqlGeneratorIntegrationTest extends BaseSqlGeneratorIntegra private static final Logger LOGGER = LoggerFactory.getLogger(BigQuerySqlGeneratorIntegrationTest.class); private static BigQuery bq; + private static String projectId; + private static String datasetLocation; @BeforeAll public static void setupBigquery() throws Exception { final String rawConfig = Files.readString(Path.of("secrets/credentials-gcs-staging.json")); final JsonNode config = Jsons.deserialize(rawConfig); bq = BigQueryDestination.getBigQuery(config); + + projectId = config.get(BigQueryConsts.CONFIG_PROJECT_ID).asText(); + datasetLocation = config.get(BigQueryConsts.CONFIG_DATASET_LOCATION).asText(); } @Override protected BigQuerySqlGenerator getSqlGenerator() { - return new BigQuerySqlGenerator("US"); + return new BigQuerySqlGenerator(projectId, datasetLocation); } @Override @@ -124,41 +131,6 @@ protected void createV1RawTable(final StreamId v1RawTable) throws Exception { .build()); } - @Override - protected void createFinalTable(final boolean includeCdcDeletedAt, final StreamId streamId, final String suffix) throws InterruptedException { - final String cdcDeletedAt = includeCdcDeletedAt ? "`_ab_cdc_deleted_at` TIMESTAMP," : ""; - bq.query(QueryJobConfiguration.newBuilder( - new StringSubstitutor(Map.of( - "final_table_id", streamId.finalTableId(BigQuerySqlGenerator.QUOTE, suffix), - "cdc_deleted_at", cdcDeletedAt)).replace( - """ - CREATE TABLE ${final_table_id} ( - _airbyte_raw_id STRING NOT NULL, - _airbyte_extracted_at TIMESTAMP NOT NULL, - _airbyte_meta JSON NOT NULL, - `id1` INT64, - `id2` INT64, - `updated_at` TIMESTAMP, - ${cdc_deleted_at} - `struct` JSON, - `array` JSON, - `string` STRING, - `number` NUMERIC, - `integer` INT64, - `boolean` BOOL, - `timestamp_with_timezone` TIMESTAMP, - `timestamp_without_timezone` DATETIME, - `time_with_timezone` STRING, - `time_without_timezone` TIME, - `date` DATE, - `unknown` JSON - ) - PARTITION BY (DATE_TRUNC(_airbyte_extracted_at, DAY)) - CLUSTER BY id1, id2, _airbyte_extracted_at; - """)) - .build()); - } - @Override protected void insertFinalTableRecords(final boolean includeCdcDeletedAt, final StreamId streamId, @@ -303,10 +275,7 @@ protected void insertRawTableRecords(final StreamId streamId, final List destinationHandler.execute(createTable)); diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryStandardInsertsRawOverrideDisableTypingDedupingTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryStandardInsertsRawOverrideDisableTypingDedupingTest.java new file mode 100644 index 000000000000..4f214569363d --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQueryStandardInsertsRawOverrideDisableTypingDedupingTest.java @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.bigquery.typing_deduping; + +import org.junit.jupiter.api.Disabled; + +public class BigQueryStandardInsertsRawOverrideDisableTypingDedupingTest extends AbstractBigQueryTypingDedupingTest { + + @Override + public String getConfigPath() { + return "secrets/credentials-1s1t-disabletd-standard-raw-override.json"; + } + + @Override + protected String getRawDataset() { + return "overridden_raw_dataset"; + } + + @Override + protected boolean disableFinalTableComparison() { + return true; + } + + @Override + @Disabled + public void testRemovingPKNonNullIndexes() throws Exception { + // Do nothing. + } + + @Override + @Disabled + public void identicalNameSimultaneousSync() throws Exception { + // TODO: create fixtures to verify how raw tables are affected. Base tests check for final tables. + } + +} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl index a9bf479e4e3e..916b0cb278b4 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl @@ -1,3 +1,4 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}} {"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} {"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} {"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl new file mode 100644 index 000000000000..12ae89a5f8ef --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl @@ -0,0 +1 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00Z", "name": "Someone completely different"} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_dedup_raw.jsonl deleted file mode 100644 index 88411c9e4de3..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_dedup_raw.jsonl +++ /dev/null @@ -1,4 +0,0 @@ -// Keep the Alice record with more recent updated_at -{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_raw.jsonl deleted file mode 100644 index 4b4db08115e5..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_raw.jsonl +++ /dev/null @@ -1,6 +0,0 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}} -// Note the duplicate record. In this sync mode, we don't dedup anything. -{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} -// Invalid data is still allowed in the raw table. -{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..569905e1f03d --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl @@ -0,0 +1,5 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +// Invalid data is still allowed in the raw table. +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl new file mode 100644 index 000000000000..051c914179d6 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl @@ -0,0 +1 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different"}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl index 4f3f04233ec1..62648ec30fa3 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl @@ -1,4 +1,7 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} {"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}} {"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} -// Charlie wasn't reemitted in sync2. This record still has an old_cursor value. -{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl new file mode 100644 index 000000000000..207364ef848a --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl @@ -0,0 +1 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2001-01-02T00:00:00Z", "name": "Someone completely different v2"} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_raw.jsonl deleted file mode 100644 index 5a3209db5e22..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_raw.jsonl +++ /dev/null @@ -1,5 +0,0 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} -// Keep the record that deleted Bob, but delete the other records associated with id=(1, 201) -{"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} -// And keep Charlie's record, even though it wasn't reemitted in sync2. -{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl similarity index 100% rename from airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_raw.jsonl rename to airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl new file mode 100644 index 000000000000..b8c566d38761 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl @@ -0,0 +1,2 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different v2"}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl index faf1bda26c1a..e83d33307523 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl @@ -1,7 +1,8 @@ {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}} {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "unknown": null, "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}} {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}} -{"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "unknown": null, "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": ["Problem with `struct`", "Problem with `array`", "Problem with `string`", "Problem with `number`", "Problem with `integer`", "Problem with `boolean`", "Problem with `timestamp_with_timezone`", "Problem with `timestamp_without_timezone`", "Problem with `time_with_timezone`", "Problem with `time_without_timezone`", "Problem with `date`"]}} +{"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "unknown": null, "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": ["Problem with `struct`", "Problem with `array`", "Problem with `number`", "Problem with `integer`", "Problem with `boolean`","Problem with `timestamp_with_timezone`", "Problem with `timestamp_without_timezone`", "Problem with `time_with_timezone`","Problem with `time_without_timezone`", "Problem with `date`"]}} // Note that for numbers where we parse the value to JSON (struct, array, unknown) we lose precision. // But for numbers where we create a NUMBER column, we do not lose precision (see the `number` column). {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.17411800000001}, "array": [67.17411800000001], "unknown": 67.17411800000001, "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}} +{"id1": 6, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "IamACaseSensitiveColumnName": "Case senstive value", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl index bc145f60abd3..aad52eb2e525 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl @@ -1,5 +1,6 @@ -{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}}' +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} {"_airbyte_raw_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": null, "struct": null, "string": null, "number": null, "integer": null, "boolean": null, "timestamp_with_timezone": null, "timestamp_without_timezone": null, "time_with_timezone": null, "time_without_timezone": null, "date": null, "unknown": null}} {"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z"}} -{"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": {}, "number": {}, "integer": {}, "boolean": {}, "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": {}, "unknown": null}} +{"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": null, "number": "foo", "integer": "bar", "boolean": "fizz", "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": "airbyte", "unknown": null}} {"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fce", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 6, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "IamACaseSensitiveColumnName": "Case senstive value"}} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl index e2c19ff210a9..c8291c59fc89 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl @@ -1,2 +1,3 @@ +{"_airbyte_raw_id": "d7b81af0-01da-4846-a650-cc398986bc99", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "string": "Alice", "struct": {"city": "San Francisco", "state": "CA"}, "integer": 42}} {"_airbyte_raw_id": "80c99b54-54b4-43bd-b51b-1f67dafa2c52", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00Z", "string": "Alice", "struct": {"city": "San Diego", "state": "CA"}, "integer": 84}} {"_airbyte_raw_id": "ad690bfb-c2c2-4172-bd73-a16c86ccbb67", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T03:00:00Z", "string": "Bob", "integer": "oops"}} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_final.jsonl new file mode 100644 index 000000000000..484f014c73ff --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_final.jsonl @@ -0,0 +1,5 @@ +{"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "[\"I\",\"am\",\"an\",\"array\"]", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}} +{"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "{\"I\":\"am\",\"an\":\"object\"}", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}} +{"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "true", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}} +{"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "3.14", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}} +{"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "I am a valid json string", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_meta": {"errors": []}} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..96ce4458e7af --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_raw.jsonl @@ -0,0 +1,5 @@ +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": ["I", "am", "an", "array"], "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": {"I": "am", "an": "object"}, "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": true, "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": 3.14, "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_extracted_at": "2023-01-01T00:00:00Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "I am a valid json string", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/reservedkeywords_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/reservedkeywords_expectedrecords_final.jsonl new file mode 100644 index 000000000000..faf284d27489 --- /dev/null +++ b/airbyte-integrations/connectors/destination-bigquery/src/test-integration/resources/sqlgenerator/reservedkeywords_expectedrecords_final.jsonl @@ -0,0 +1 @@ +{"_airbyte_raw_id":"b2e0efc4-38a8-47ba-970c-8103f09f08d5","_airbyte_extracted_at":"2023-01-01T00:00:00Z","_airbyte_meta":{"errors":[]}, "current_date": "foo", "join": "bar"} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordConsumerTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordConsumerTest.java index 881ee3973a7a..64f039215108 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordConsumerTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryRecordConsumerTest.java @@ -7,14 +7,14 @@ import static org.mockito.Mockito.mock; import com.google.cloud.bigquery.BigQuery; +import io.airbyte.cdk.integrations.base.DestinationConfig; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.standardtest.destination.PerStreamStateMessageTest; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.DestinationConfig; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.integrations.base.destination.typing_deduping.NoopTyperDeduper; import io.airbyte.integrations.base.destination.typing_deduping.ParsedCatalog; import io.airbyte.integrations.destination.bigquery.typing_deduping.BigQueryV1V2Migrator; import io.airbyte.integrations.destination.bigquery.uploader.AbstractBigQueryUploader; -import io.airbyte.integrations.standardtest.destination.PerStreamStateMessageTest; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import java.util.Collections; diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java index 8043726bda18..0f03515ee087 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/BigQueryUtilsTest.java @@ -5,18 +5,13 @@ package io.airbyte.integrations.destination.bigquery; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; -import java.util.Collections; -import java.util.Map; import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -58,27 +53,6 @@ public void testGetDatasetIdFail(final String projectId, final String datasetId, assertEquals(expected, exception.getMessage()); } - @Test - public void testIsUsingJsonCredentials() { - // empty - final JsonNode emptyConfig = Jsons.jsonNode(Collections.emptyMap()); - assertFalse(BigQueryUtils.isUsingJsonCredentials(emptyConfig)); - - // empty text - final JsonNode emptyTextConfig = Jsons.jsonNode(Map.of(BigQueryConsts.CONFIG_CREDS, "")); - assertFalse(BigQueryUtils.isUsingJsonCredentials(emptyTextConfig)); - - // non-empty text - final JsonNode nonEmptyTextConfig = Jsons.jsonNode( - Map.of(BigQueryConsts.CONFIG_CREDS, "{ \"service_account\": \"test@airbyte.io\" }")); - assertTrue(BigQueryUtils.isUsingJsonCredentials(nonEmptyTextConfig)); - - // object - final JsonNode objectConfig = Jsons.jsonNode(Map.of( - BigQueryConsts.CONFIG_CREDS, Jsons.jsonNode(Map.of("service_account", "test@airbyte.io")))); - assertTrue(BigQueryUtils.isUsingJsonCredentials(objectConfig)); - } - private static Stream validBigQueryIdProvider() { return Stream.of( Arguments.arguments("my-project", "my_dataset", "my_dataset"), diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/CdkImportTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/CdkImportTest.java deleted file mode 100644 index 2b8400ee05ca..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/CdkImportTest.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.bigquery; - -import static org.junit.jupiter.api.Assertions.*; - -import io.airbyte.cdk.CDKConstants; -import org.junit.jupiter.api.Test; - -class CdkImportTest { - - /** - * This test ensures that the CDK is able to be imported and that its version number matches the - * expected pinned version. - */ - @Test - void cdkVersionShouldMatch() { - assertEquals("0.0.1", CDKConstants.VERSION.replace("-SNAPSHOT", "")); - } - -} diff --git a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGeneratorTest.java b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGeneratorTest.java index f04008e47a1f..1fac62e2d681 100644 --- a/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGeneratorTest.java +++ b/airbyte-integrations/connectors/destination-bigquery/src/test/java/io/airbyte/integrations/destination/bigquery/typing_deduping/BigQuerySqlGeneratorTest.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.destination.bigquery.typing_deduping; +import static java.util.Collections.emptyList; import static org.junit.jupiter.api.Assertions.assertEquals; import com.google.cloud.bigquery.Clustering; @@ -11,18 +12,25 @@ import com.google.cloud.bigquery.StandardTableDefinition; import com.google.cloud.bigquery.TimePartitioning; import com.google.common.collect.ImmutableList; +import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType; import io.airbyte.integrations.base.destination.typing_deduping.Array; +import io.airbyte.integrations.base.destination.typing_deduping.CatalogParser; import io.airbyte.integrations.base.destination.typing_deduping.ColumnId; import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import io.airbyte.integrations.base.destination.typing_deduping.Struct; import io.airbyte.integrations.base.destination.typing_deduping.Union; import io.airbyte.integrations.base.destination.typing_deduping.UnsupportedOneOf; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.SyncMode; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -32,7 +40,7 @@ public class BigQuerySqlGeneratorTest { - private final BigQuerySqlGenerator generator = new BigQuerySqlGenerator("US"); + private final BigQuerySqlGenerator generator = new BigQuerySqlGenerator("foo", "US"); @Test public void testToDialectType() { @@ -139,4 +147,40 @@ public void testSchemaContainAllFinalTableV2AirbyteColumns() { BigQuerySqlGenerator.schemaContainAllFinalTableV2AirbyteColumns(Set.of("_AIRBYTE_META", "_AIRBYTE_EXTRACTED_AT", "_AIRBYTE_RAW_ID"))); } + @Test + void columnCollision() { + final CatalogParser parser = new CatalogParser(generator); + assertEquals( + new StreamConfig( + new StreamId("bar", "foo", "airbyte_internal", "bar_raw__stream_foo", "bar", "foo"), + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND, + emptyList(), + Optional.empty(), + new LinkedHashMap<>() { + + { + put(new ColumnId("CURRENT_DATE", "CURRENT_DATE", "current_date"), AirbyteProtocolType.STRING); + put(new ColumnId("current_date_1", "current_date", "current_date_1"), AirbyteProtocolType.INTEGER); + } + + }), + parser.toStreamConfig(new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(new AirbyteStream() + .withName("foo") + .withNamespace("bar") + .withJsonSchema(Jsons.deserialize( + """ + { + "type": "object", + "properties": { + "CURRENT_DATE": {"type": "string"}, + "current_date": {"type": "integer"} + } + } + """))))); + } + } diff --git a/airbyte-integrations/connectors/destination-cassandra/.dockerignore b/airbyte-integrations/connectors/destination-cassandra/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-cassandra/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-cassandra/Dockerfile b/airbyte-integrations/connectors/destination-cassandra/Dockerfile deleted file mode 100644 index 55aef547627c..000000000000 --- a/airbyte-integrations/connectors/destination-cassandra/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-cassandra - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-cassandra - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.4 -LABEL io.airbyte.name=airbyte/destination-cassandra diff --git a/airbyte-integrations/connectors/destination-cassandra/README.md b/airbyte-integrations/connectors/destination-cassandra/README.md index 5e5237291eab..21c6cde72284 100644 --- a/airbyte-integrations/connectors/destination-cassandra/README.md +++ b/airbyte-integrations/connectors/destination-cassandra/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-cassandra:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-cassandra:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-cassandra:dev`. the Dockerfile. #### Run @@ -61,8 +62,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-cassandra test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/cassandra.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-cassandra/build.gradle b/airbyte-integrations/connectors/destination-cassandra/build.gradle index 74361d3bb2cb..b9774a9b9c7f 100644 --- a/airbyte-integrations/connectors/destination-cassandra/build.gradle +++ b/airbyte-integrations/connectors/destination-cassandra/build.gradle @@ -1,9 +1,23 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.cassandra.CassandraDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] @@ -13,10 +27,6 @@ def cassandraDriver = '4.13.0' def assertVersion = '3.21.0' dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation "com.datastax.oss:java-driver-core:${cassandraDriver}" implementation "com.datastax.oss:java-driver-query-builder:${cassandraDriver}" @@ -25,10 +35,5 @@ dependencies { // https://mvnrepository.com/artifact/org.assertj/assertj-core testImplementation "org.assertj:assertj-core:${assertVersion}" - testImplementation libs.connectors.testcontainers.cassandra - testImplementation project(':airbyte-integrations:bases:standard-destination-test') - - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-cassandra') + testImplementation libs.testcontainers.cassandra } diff --git a/airbyte-integrations/connectors/destination-cassandra/docker-compose.yml b/airbyte-integrations/connectors/destination-cassandra/docker-compose.yml index 75090b3b59ca..a4786dda1b66 100644 --- a/airbyte-integrations/connectors/destination-cassandra/docker-compose.yml +++ b/airbyte-integrations/connectors/destination-cassandra/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3.7' +version: "3.7" services: cassandra1: @@ -9,7 +9,6 @@ services: - "MAX_HEAP_SIZE=2048M" - "HEAP_NEWSIZE=1024M" - "CASSANDRA_CLUSTER_NAME=cassandra_cluster" - # Uncomment if you want to run a Cassandra cluster # cassandra2: # image: cassandra:4.0 @@ -21,4 +20,4 @@ services: # - "CASSANDRA_SEEDS=cassandra1" # - "CASSANDRA_CLUSTER_NAME=cassandra_cluster" # depends_on: -# - cassandra1 \ No newline at end of file +# - cassandra1 diff --git a/airbyte-integrations/connectors/destination-cassandra/src/main/java/io/airbyte/integrations/destination/cassandra/CassandraCqlProvider.java b/airbyte-integrations/connectors/destination-cassandra/src/main/java/io/airbyte/integrations/destination/cassandra/CassandraCqlProvider.java index cc974541aacc..0e48b8d8aecc 100644 --- a/airbyte-integrations/connectors/destination-cassandra/src/main/java/io/airbyte/integrations/destination/cassandra/CassandraCqlProvider.java +++ b/airbyte-integrations/connectors/destination-cassandra/src/main/java/io/airbyte/integrations/destination/cassandra/CassandraCqlProvider.java @@ -14,7 +14,7 @@ import com.datastax.oss.driver.api.core.uuid.Uuids; import com.datastax.oss.driver.api.querybuilder.QueryBuilder; import com.datastax.oss.driver.api.querybuilder.SchemaBuilder; -import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import java.io.Closeable; import java.time.Instant; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-cassandra/src/main/java/io/airbyte/integrations/destination/cassandra/CassandraDestination.java b/airbyte-integrations/connectors/destination-cassandra/src/main/java/io/airbyte/integrations/destination/cassandra/CassandraDestination.java index 1aa7ce528f60..e2727ba734c2 100644 --- a/airbyte-integrations/connectors/destination-cassandra/src/main/java/io/airbyte/integrations/destination/cassandra/CassandraDestination.java +++ b/airbyte-integrations/connectors/destination-cassandra/src/main/java/io/airbyte/integrations/destination/cassandra/CassandraDestination.java @@ -5,10 +5,10 @@ package io.airbyte.integrations.destination.cassandra; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/connectors/destination-cassandra/src/main/java/io/airbyte/integrations/destination/cassandra/CassandraMessageConsumer.java b/airbyte-integrations/connectors/destination-cassandra/src/main/java/io/airbyte/integrations/destination/cassandra/CassandraMessageConsumer.java index 55de00889962..803cde8ffe34 100644 --- a/airbyte-integrations/connectors/destination-cassandra/src/main/java/io/airbyte/integrations/destination/cassandra/CassandraMessageConsumer.java +++ b/airbyte-integrations/connectors/destination-cassandra/src/main/java/io/airbyte/integrations/destination/cassandra/CassandraMessageConsumer.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.cassandra; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/connectors/destination-cassandra/src/main/java/io/airbyte/integrations/destination/cassandra/CassandraNameTransformer.java b/airbyte-integrations/connectors/destination-cassandra/src/main/java/io/airbyte/integrations/destination/cassandra/CassandraNameTransformer.java index 7248146a524c..da7f60bfba62 100644 --- a/airbyte-integrations/connectors/destination-cassandra/src/main/java/io/airbyte/integrations/destination/cassandra/CassandraNameTransformer.java +++ b/airbyte-integrations/connectors/destination-cassandra/src/main/java/io/airbyte/integrations/destination/cassandra/CassandraNameTransformer.java @@ -5,8 +5,8 @@ package io.airbyte.integrations.destination.cassandra; import com.google.common.base.CharMatcher; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.commons.text.Names; -import io.airbyte.integrations.destination.StandardNameTransformer; class CassandraNameTransformer extends StandardNameTransformer { diff --git a/airbyte-integrations/connectors/destination-cassandra/src/test-integration/java/io/airbyte/integrations/destination/cassandra/CassandraDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-cassandra/src/test-integration/java/io/airbyte/integrations/destination/cassandra/CassandraDestinationAcceptanceTest.java index c34561d2a9c6..44c7bf00b5bf 100644 --- a/airbyte-integrations/connectors/destination-cassandra/src/test-integration/java/io/airbyte/integrations/destination/cassandra/CassandraDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-cassandra/src/test-integration/java/io/airbyte/integrations/destination/cassandra/CassandraDestinationAcceptanceTest.java @@ -5,9 +5,9 @@ package io.airbyte.integrations.destination.cassandra; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.util.HostPortResolver; import java.util.Comparator; import java.util.HashSet; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-cassandra/src/test/java/io/airbyte/integrations/destination/cassandra/CassandraRecordConsumerTest.java b/airbyte-integrations/connectors/destination-cassandra/src/test/java/io/airbyte/integrations/destination/cassandra/CassandraRecordConsumerTest.java index 9be7650df2b5..dc35e4bffa02 100644 --- a/airbyte-integrations/connectors/destination-cassandra/src/test/java/io/airbyte/integrations/destination/cassandra/CassandraRecordConsumerTest.java +++ b/airbyte-integrations/connectors/destination-cassandra/src/test/java/io/airbyte/integrations/destination/cassandra/CassandraRecordConsumerTest.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.cassandra; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; -import io.airbyte.integrations.standardtest.destination.PerStreamStateMessageTest; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.standardtest.destination.PerStreamStateMessageTest; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import java.util.function.Consumer; diff --git a/airbyte-integrations/connectors/destination-chroma/.dockerignore b/airbyte-integrations/connectors/destination-chroma/.dockerignore new file mode 100644 index 000000000000..f89c3a5ca804 --- /dev/null +++ b/airbyte-integrations/connectors/destination-chroma/.dockerignore @@ -0,0 +1,5 @@ +* +!Dockerfile +!main.py +!destination_chroma +!setup.py diff --git a/airbyte-integrations/connectors/destination-chroma/Dockerfile b/airbyte-integrations/connectors/destination-chroma/Dockerfile new file mode 100644 index 000000000000..6eec4a792d2a --- /dev/null +++ b/airbyte-integrations/connectors/destination-chroma/Dockerfile @@ -0,0 +1,45 @@ +FROM python:3.10-slim as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +RUN apt-get update \ + && pip install --upgrade pip \ + && apt-get install -y build-essential cmake g++ libffi-dev libstdc++6 + +# upgrade pip to the latest version +COPY setup.py ./ + +RUN pip install --upgrade pip + +# This is required because the current connector dependency is not compatible with the CDK version +# An older CDK version will be used, which depends on pyYAML 5.4, for which we need to pin Cython to <3.0 +# As of today the CDK version that satisfies the main dependency requirements, is 0.1.80 ... +RUN pip install --prefix=/install "Cython<3.0" "pyyaml~=5.4" --no-build-isolation + +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apt-get install bash + +# copy payload code only +COPY main.py ./ +COPY destination_chroma ./destination_chroma + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.0.9 +LABEL io.airbyte.name=airbyte/destination-chroma diff --git a/airbyte-integrations/connectors/destination-chroma/README.md b/airbyte-integrations/connectors/destination-chroma/README.md new file mode 100644 index 000000000000..0a18a90bb5c9 --- /dev/null +++ b/airbyte-integrations/connectors/destination-chroma/README.md @@ -0,0 +1,99 @@ +# Chroma Destination + +This is the repository for the Chroma destination connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/destinations/chroma). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/destinations/chroma) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_chroma/spec.json` file. +Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `destination chroma test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + + +#### Build +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=destination-chroma build +``` + +An image will be built with the tag `airbyte/destination-chroma:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/destination-chroma:dev . +``` + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/destination-chroma:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-chroma:dev check --config /secrets/config.json +# messages.jsonl is a file containing line-separated JSON representing AirbyteMessages +cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-chroma:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=destination-chroma test +``` + +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-chroma test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/chroma.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-chroma/destination_chroma/__init__.py b/airbyte-integrations/connectors/destination-chroma/destination_chroma/__init__.py new file mode 100644 index 000000000000..3ebffc51e313 --- /dev/null +++ b/airbyte-integrations/connectors/destination-chroma/destination_chroma/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .destination import DestinationChroma + +__all__ = ["DestinationChroma"] diff --git a/airbyte-integrations/connectors/destination-chroma/destination_chroma/config.py b/airbyte-integrations/connectors/destination-chroma/destination_chroma/config.py new file mode 100644 index 000000000000..557d4f527986 --- /dev/null +++ b/airbyte-integrations/connectors/destination-chroma/destination_chroma/config.py @@ -0,0 +1,78 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Literal, Optional, Union + +from airbyte_cdk.destinations.vector_db_based.config import ( + AzureOpenAIEmbeddingConfigModel, + CohereEmbeddingConfigModel, + FakeEmbeddingConfigModel, + FromFieldEmbeddingConfigModel, + OpenAICompatibleEmbeddingConfigModel, + OpenAIEmbeddingConfigModel, + VectorDBConfigModel, +) +from pydantic import BaseModel, Field + + +class HttpMode(BaseModel): + mode: Literal["http_client"] = Field("http_client", const=True) + host: str = Field(..., title="Host", description="The URL to the chromadb instance", order=0) + port: int = Field(..., title="Port", description="The port to the chromadb instance", order=1) + ssl: bool = Field(..., title="SSL", description="Whether to use SSL to connect to the Chroma server", order=2) + username: Optional[str] = Field(default="", title="Username", description="Username used in server/client mode only", order=3) + password: Optional[str] = Field( + default="", title="Password", description="Password used in server/client mode only", airbyte_secret=True, order=4 + ) + + class Config: + title = "Client/Server Mode" + schema_extra = {"description": "Authenticate using username and password (suitable for self-managed Chroma clusters)"} + + +class PersistentMode(BaseModel): + mode: Literal["persistent_client"] = Field("persistent_client", const=True) + path: str = Field(..., title="Path", description="Where Chroma will store its database files on disk, and load them on start.") + + class Config: + title = "Persistent Client Mode" + schema_extra = {"description": "Configure Chroma to save and load from your local machine"} + + +class ChromaIndexingConfigModel(BaseModel): + + auth_method: Union[PersistentMode, HttpMode] = Field( + ..., title="Connection Mode", description="Mode how to connect to Chroma", discriminator="mode", type="object", order=0 + ) + collection_name: str = Field(..., title="Collection Name", description="The collection to load data into", order=3) + + class Config: + title = "Indexing" + schema_extra = { + "group": "indexing", + "description": "Indexing configuration", + } + + +class NoEmbeddingConfigModel(BaseModel): + mode: Literal["no_embedding"] = Field("no_embedding", const=True) + + class Config: + title = "Chroma Default Embedding Function" + schema_extra = { + "description": "Do not calculate embeddings. Chromadb uses the sentence transfomer (https://www.sbert.net/index.html) as a default if an embedding function is not defined. Note that depending on your hardware, calculating embeddings locally can be very slow and is mostly suited for prototypes." + } + + +class ConfigModel(VectorDBConfigModel): + indexing: ChromaIndexingConfigModel + embedding: Union[ + AzureOpenAIEmbeddingConfigModel, + OpenAIEmbeddingConfigModel, + CohereEmbeddingConfigModel, + FromFieldEmbeddingConfigModel, + FakeEmbeddingConfigModel, + OpenAICompatibleEmbeddingConfigModel, + NoEmbeddingConfigModel, + ] = Field(..., title="Embedding", description="Embedding configuration", discriminator="mode", group="embedding", type="object") diff --git a/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py b/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py new file mode 100644 index 000000000000..4245ecb7de61 --- /dev/null +++ b/airbyte-integrations/connectors/destination-chroma/destination_chroma/destination.py @@ -0,0 +1,95 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from typing import Any, Iterable, Mapping + +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.destinations import Destination +from airbyte_cdk.destinations.vector_db_based.document_processor import DocumentProcessor +from airbyte_cdk.destinations.vector_db_based.embedder import Embedder, create_from_config +from airbyte_cdk.destinations.vector_db_based.indexer import Indexer +from airbyte_cdk.destinations.vector_db_based.writer import Writer +from airbyte_cdk.models import ( + AirbyteConnectionStatus, + AirbyteMessage, + ConfiguredAirbyteCatalog, + ConnectorSpecification, + DestinationSyncMode, + Status, +) +from destination_chroma.config import ConfigModel +from destination_chroma.indexer import ChromaIndexer +from destination_chroma.no_embedder import NoEmbedder + +BATCH_SIZE = 128 + + +class DestinationChroma(Destination): + + indexer: Indexer + embedder: Embedder + + def _init_indexer(self, config: ConfigModel): + self.embedder = ( + create_from_config(config.embedding, config.processing) + if config.embedding.mode != "no_embedding" + else NoEmbedder(config.embedding) + ) + self.indexer = ChromaIndexer(config.indexing) + + def write( + self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] + ) -> Iterable[AirbyteMessage]: + + """ + Reads the input stream of messages, config, and catalog to write data to the destination. + + This method returns an iterable (typically a generator of AirbyteMessages via yield) containing state messages received + in the input message stream. Outputting a state message means that every AirbyteRecordMessage which came before it has been + successfully persisted to the destination. This is used to ensure fault tolerance in the case that a sync fails before fully completing, + then the source is given the last state message output from this method as the starting point of the next sync. + + :param config: dict of JSON configuration matching the configuration declared in spec.json + :param configured_catalog: The Configured Catalog describing the schema of the data being received and how it should be persisted in the + destination + :param input_messages: The stream of input messages received from the source + :return: Iterable of AirbyteStateMessages wrapped in AirbyteMessage structs + """ + + config_model = ConfigModel.parse_obj(config) + self._init_indexer(config_model) + writer = Writer( + config_model.processing, self.indexer, self.embedder, batch_size=BATCH_SIZE, omit_raw_text=config_model.omit_raw_text + ) + yield from writer.write(configured_catalog, input_messages) + + def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: + """ + Tests if the input configuration can be used to successfully connect to the destination with the needed permissions + e.g: if a provided API token or password can be used to connect and write to the destination. + + :param logger: Logging object to display debug/info/error to the logs + (logs will not be accessible via airbyte UI if they are not passed to this logger) + :param config: Json object containing the configuration of this destination, content of this json is as specified in + the properties of the spec.json file + + :return: AirbyteConnectionStatus indicating a Success or Failure + """ + parsed_config = ConfigModel.parse_obj(config) + self._init_indexer(parsed_config) + checks = [self.embedder.check(), self.indexer.check(), DocumentProcessor.check_config(parsed_config.processing)] + errors = [error for error in checks if error is not None] + if len(errors) > 0: + return AirbyteConnectionStatus(status=Status.FAILED, message="\n".join(errors)) + else: + return AirbyteConnectionStatus(status=Status.SUCCEEDED) + + def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: + return ConnectorSpecification( + documentationUrl="https://docs.airbyte.com/integrations/destinations/chroma", + supportsIncremental=True, + supported_destination_sync_modes=[DestinationSyncMode.overwrite, DestinationSyncMode.append, DestinationSyncMode.append_dedup], + connectionSpecification=ConfigModel.schema(), + ) diff --git a/airbyte-integrations/connectors/destination-chroma/destination_chroma/indexer.py b/airbyte-integrations/connectors/destination-chroma/destination_chroma/indexer.py new file mode 100644 index 000000000000..3b0d741d7feb --- /dev/null +++ b/airbyte-integrations/connectors/destination-chroma/destination_chroma/indexer.py @@ -0,0 +1,125 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import uuid + +import chromadb +from airbyte_cdk.destinations.vector_db_based.document_processor import METADATA_RECORD_ID_FIELD, METADATA_STREAM_FIELD +from airbyte_cdk.destinations.vector_db_based.indexer import Indexer +from airbyte_cdk.destinations.vector_db_based.utils import create_stream_identifier, format_exception +from airbyte_cdk.models import ConfiguredAirbyteCatalog +from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode +from chromadb.config import Settings +from destination_chroma.config import ChromaIndexingConfigModel +from destination_chroma.utils import is_valid_collection_name + + +class ChromaIndexer(Indexer): + def __init__(self, config: ChromaIndexingConfigModel): + super().__init__(config) + self.collection_name = config.collection_name + + def check(self): + collection_name_validation_error = is_valid_collection_name(self.collection_name) + if collection_name_validation_error: + return collection_name_validation_error + + auth_method = self.config.auth_method + if auth_method.mode == "persistent_client" and not auth_method.path.startswith("/local/"): + return "Path must be prefixed with /local" + + client = self._get_client() + try: + heartbeat = client.heartbeat() + if not heartbeat: + return "Chroma client server is not alive" + collection = client.get_or_create_collection(name=self.collection_name) + count = collection.count() + if count != 0 and not count: + return f"unable to get or create collection with name {self.collection_name}" + return + except Exception as e: + return format_exception(e) + finally: + del client + + def delete(self, delete_ids, namespace, stream): + if len(delete_ids) > 0: + self._delete_by_filter(field_name=METADATA_RECORD_ID_FIELD, field_values=delete_ids) + + def index(self, document_chunks, namespace, stream): + entities = [] + for i in range(len(document_chunks)): + chunk = document_chunks[i] + entities.append( + { + "id": str(uuid.uuid4()), + "embedding": chunk.embedding, + "metadata": self._normalize(chunk.metadata), + "document": chunk.page_content if chunk.page_content is not None else "", + } + ) + self._write_data(entities) + + def pre_sync(self, catalog: ConfiguredAirbyteCatalog) -> None: + self.client = self._get_client() + streams_to_overwrite = [ + create_stream_identifier(stream.stream) + for stream in catalog.streams + if stream.destination_sync_mode == DestinationSyncMode.overwrite + ] + if len(streams_to_overwrite): + self._delete_by_filter(field_name=METADATA_STREAM_FIELD, field_values=streams_to_overwrite) + + def _get_client(self): + auth_method = self.config.auth_method + if auth_method.mode == "persistent_client": + path = auth_method.path + client = chromadb.PersistentClient(path=path) + return client + + elif auth_method.mode == "http_client": + host = auth_method.host + port = auth_method.port + ssl = auth_method.ssl + username = auth_method.username + password = auth_method.password + + if username and password: + settings = Settings( + chroma_client_auth_provider="chromadb.auth.basic.BasicAuthClientProvider", + chroma_client_auth_credentials=f"{username}:{password}", + ) + client = chromadb.HttpClient(settings=settings, host=host, port=port, ssl=ssl) + else: + client = chromadb.HttpClient(host=host, port=port, ssl=ssl) + return client + return + + def _delete_by_filter(self, field_name, field_values): + collection = self.client.get_collection(name=self.collection_name) + where_filter = {field_name: {"$in": field_values}} + collection.delete(where=where_filter) + + def _normalize(self, metadata: dict) -> dict: + result = {} + for key, value in metadata.items(): + if isinstance(value, (str, int, float, bool)): + result[key] = value + else: + # JSON encode all other types + result[key] = json.dumps(value) + return result + + def _write_data(self, entities): + ids = [entity["id"] for entity in entities] + embeddings = [entity["embedding"] for entity in entities] + if not any(embeddings): + embeddings = None + metadatas = [entity["metadata"] for entity in entities] + documents = [entity["document"] for entity in entities] + + collection = self.client.get_collection(name=self.collection_name) + collection.add(ids=ids, embeddings=embeddings, metadatas=metadatas, documents=documents) diff --git a/airbyte-integrations/connectors/destination-chroma/destination_chroma/no_embedder.py b/airbyte-integrations/connectors/destination-chroma/destination_chroma/no_embedder.py new file mode 100644 index 000000000000..8cc1c7cac379 --- /dev/null +++ b/airbyte-integrations/connectors/destination-chroma/destination_chroma/no_embedder.py @@ -0,0 +1,24 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import List, Optional + +from airbyte_cdk.destinations.vector_db_based.document_processor import Chunk +from airbyte_cdk.destinations.vector_db_based.embedder import Embedder +from destination_chroma.config import NoEmbeddingConfigModel + + +class NoEmbedder(Embedder): + def __init__(self, config: NoEmbeddingConfigModel): + super().__init__() + + def check(self) -> Optional[str]: + return None + + def embed_chunks(self, chunks: List[Chunk]) -> List[None]: + return [None for _ in chunks] + + @property + def embedding_dimensions(self) -> int: + return None diff --git a/airbyte-integrations/connectors/destination-chroma/destination_chroma/utils.py b/airbyte-integrations/connectors/destination-chroma/destination_chroma/utils.py new file mode 100644 index 000000000000..1adefb606f9d --- /dev/null +++ b/airbyte-integrations/connectors/destination-chroma/destination_chroma/utils.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import re +from ipaddress import ip_address + + +def is_valid_collection_name(stream_name: str): + # Check length constraint + if len(stream_name) < 3 or len(stream_name) > 63: + return "The length of the collection name must be between 3 and 63 characters" + # Check lowercase letter or digit at start and end + if not (stream_name[0].islower() or stream_name[0].isdigit()) or not (stream_name[-1].islower() or stream_name[-1].isdigit()): + return "The collection name must start and end with a lowercase letter or a digit" + # Check allowed characters + if not re.match("^[a-z0-9][a-zA-Z0-9._-]*[a-z0-9]$", stream_name): + return "The collection name can only contain lower case alphanumerics, dots, dashes, and underscores" + # Check consecutive dots + if ".." in stream_name: + return "The collection name must not contain two consecutive dots" + # Check for valid IP address + try: + ip_address(stream_name) + return "The collection name must not be a valid IP address." + except ValueError: + if re.match("^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}$", stream_name): + return "The collection name must not be a valid IP address." + return diff --git a/airbyte-integrations/connectors/destination-chroma/icon.svg b/airbyte-integrations/connectors/destination-chroma/icon.svg new file mode 100644 index 000000000000..fc07764e4359 --- /dev/null +++ b/airbyte-integrations/connectors/destination-chroma/icon.svg @@ -0,0 +1,59 @@ + + + + + Chroma + + + + + + diff --git a/airbyte-integrations/connectors/destination-chroma/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-chroma/integration_tests/integration_test.py new file mode 100644 index 000000000000..d945ab6b09af --- /dev/null +++ b/airbyte-integrations/connectors/destination-chroma/integration_tests/integration_test.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +def integration_test(): + # TODO write integration tests + pass diff --git a/airbyte-integrations/connectors/destination-chroma/main.py b/airbyte-integrations/connectors/destination-chroma/main.py new file mode 100644 index 000000000000..88af98a9500c --- /dev/null +++ b/airbyte-integrations/connectors/destination-chroma/main.py @@ -0,0 +1,11 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from destination_chroma import DestinationChroma + +if __name__ == "__main__": + DestinationChroma().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-chroma/metadata.yaml b/airbyte-integrations/connectors/destination-chroma/metadata.yaml new file mode 100644 index 000000000000..8283e5453d0b --- /dev/null +++ b/airbyte-integrations/connectors/destination-chroma/metadata.yaml @@ -0,0 +1,22 @@ +data: + registries: + oss: + enabled: true + cloud: + enabled: false + connectorSubtype: vectorstore + connectorType: destination + definitionId: 0b75218b-f702-4a28-85ac-34d3d84c0fc2 + dockerImageTag: 0.0.9 + dockerRepository: airbyte/destination-chroma + githubIssueLabel: destination-chroma + icon: chroma.svg + license: MIT + name: Chroma + releaseDate: "2023-09-13" + releaseStage: alpha + supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/destinations/chroma + tags: + - language:python +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-duckdb/requirements.txt b/airbyte-integrations/connectors/destination-chroma/requirements.txt similarity index 100% rename from airbyte-integrations/connectors/destination-duckdb/requirements.txt rename to airbyte-integrations/connectors/destination-chroma/requirements.txt diff --git a/airbyte-integrations/connectors/destination-chroma/setup.py b/airbyte-integrations/connectors/destination-chroma/setup.py new file mode 100644 index 000000000000..ae2f70163452 --- /dev/null +++ b/airbyte-integrations/connectors/destination-chroma/setup.py @@ -0,0 +1,26 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk[vector-db-based]==0.57.0", + "chromadb", +] + +TEST_REQUIREMENTS = ["pytest~=6.2"] + +setup( + name="destination_chroma", + description="Destination implementation for Chroma.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/destination-chroma/unit_tests/__init__.py b/airbyte-integrations/connectors/destination-chroma/unit_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/destination-chroma/unit_tests/test_destination.py b/airbyte-integrations/connectors/destination-chroma/unit_tests/test_destination.py new file mode 100644 index 000000000000..f1c3c3682387 --- /dev/null +++ b/airbyte-integrations/connectors/destination-chroma/unit_tests/test_destination.py @@ -0,0 +1,95 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import unittest +from unittest.mock import MagicMock, Mock, patch + +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.models import ConnectorSpecification, Status +from destination_chroma.config import ConfigModel +from destination_chroma.destination import DestinationChroma + + +class TestDestinationChroma(unittest.TestCase): + def setUp(self): + self.config = { + "processing": {"text_fields": ["str_col"], "metadata_fields": [], "chunk_size": 1000}, + "embedding": {"mode": "openai", "openai_key": "mykey"}, + "indexing": { + "auth_method": {"mode": "persistent_client", "path": "./path"}, + "collection_name": "test2", + }, + } + self.config_model = ConfigModel.parse_obj(self.config) + self.logger = AirbyteLogger() + + @patch("destination_chroma.destination.ChromaIndexer") + @patch("destination_chroma.destination.create_from_config") + def test_check(self, MockedEmbedder, MockedChromaIndexer): + mock_embedder = Mock() + mock_indexer = Mock() + MockedChromaIndexer.return_value = mock_indexer + MockedEmbedder.return_value = mock_embedder + + mock_embedder.check.return_value = None + mock_indexer.check.return_value = None + + destination = DestinationChroma() + result = destination.check(self.logger, self.config) + + self.assertEqual(result.status, Status.SUCCEEDED) + mock_embedder.check.assert_called_once() + mock_indexer.check.assert_called_once() + + @patch("destination_chroma.destination.ChromaIndexer") + @patch("destination_chroma.destination.create_from_config") + def test_check_with_errors(self, MockedEmbedder, MockedChromaIndexer): + mock_embedder = Mock() + mock_indexer = Mock() + MockedChromaIndexer.return_value = mock_indexer + MockedEmbedder.return_value = mock_embedder + + embedder_error_message = "Embedder Error" + indexer_error_message = "Indexer Error" + + mock_embedder.check.return_value = embedder_error_message + mock_indexer.check.return_value = indexer_error_message + + destination = DestinationChroma() + result = destination.check(self.logger, self.config) + + self.assertEqual(result.status, Status.FAILED) + self.assertEqual(result.message, f"{embedder_error_message}\n{indexer_error_message}") + + mock_embedder.check.assert_called_once() + mock_indexer.check.assert_called_once() + + @patch("destination_chroma.destination.Writer") + @patch("destination_chroma.destination.ChromaIndexer") + @patch("destination_chroma.destination.create_from_config") + def test_write(self, MockedEmbedder, MockedChromaIndexer, MockedWriter): + mock_embedder = Mock() + mock_indexer = Mock() + mock_writer = Mock() + + MockedChromaIndexer.return_value = mock_indexer + MockedWriter.return_value = mock_writer + MockedEmbedder.return_value = mock_embedder + + mock_writer.write.return_value = [] + + configured_catalog = MagicMock() + input_messages = [] + + destination = DestinationChroma() + list(destination.write(self.config, configured_catalog, input_messages)) + + MockedWriter.assert_called_once_with(self.config_model.processing, mock_indexer, mock_embedder, batch_size=128, omit_raw_text=False) + mock_writer.write.assert_called_once_with(configured_catalog, input_messages) + + def test_spec(self): + destination = DestinationChroma() + result = destination.spec() + + self.assertIsInstance(result, ConnectorSpecification) diff --git a/airbyte-integrations/connectors/destination-chroma/unit_tests/test_indexer.py b/airbyte-integrations/connectors/destination-chroma/unit_tests/test_indexer.py new file mode 100644 index 000000000000..f1a9bf493d57 --- /dev/null +++ b/airbyte-integrations/connectors/destination-chroma/unit_tests/test_indexer.py @@ -0,0 +1,159 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import unittest +from unittest.mock import Mock + +from airbyte_cdk.models.airbyte_protocol import AirbyteStream, DestinationSyncMode, SyncMode +from destination_chroma.config import ChromaIndexingConfigModel +from destination_chroma.indexer import ChromaIndexer + + +class TestChromaIndexer(unittest.TestCase): + def setUp(self): + self.mock_config = ChromaIndexingConfigModel( + **{ + "collection_name": "dummy-collection", + "auth_method": { + "mode": "persistent_client", + "path": "/local/path", + }, + } + ) + self.chroma_indexer = ChromaIndexer(self.mock_config) + self.chroma_indexer._get_client = Mock() + self.mock_client = self.chroma_indexer._get_client() + self.mock_client.get_or_create_collection = Mock() + self.mock_collection = self.mock_client.get_or_create_collection() + self.chroma_indexer.client = self.mock_client + self.mock_client.get_collection = Mock() + + def test_valid_collection_name(self): + + test_configs = [ + ({"collection_name": "dummy-collection", "auth_method": {"mode": "persistent_client", "path": "/local/path"}}, None), + ( + {"collection_name": "du", "auth_method": {"mode": "persistent_client", "path": "/local/path"}}, + "The length of the collection name must be between 3 and 63 characters", + ), + ( + { + "collection_name": "dummy-collectionxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "auth_method": {"mode": "persistent_client", "path": "/local/path"}, + }, + "The length of the collection name must be between 3 and 63 characters", + ), + ( + {"collection_name": "1dummy-colle..ction4", "auth_method": {"mode": "persistent_client", "path": "/local/path"}}, + "The collection name must not contain two consecutive dots", + ), + ( + {"collection_name": "Dummy-coll...ectioN", "auth_method": {"mode": "persistent_client", "path": "/local/path"}}, + "The collection name must start and end with a lowercase letter or a digit", + ), + ( + {"collection_name": "-dum?my-collection-", "auth_method": {"mode": "persistent_client", "path": "/local/path"}}, + "The collection name must start and end with a lowercase letter or a digit", + ), + ( + {"collection_name": "dummy?collection", "auth_method": {"mode": "persistent_client", "path": "/local/path"}}, + "The collection name can only contain lower case alphanumerics, dots, dashes, and underscores", + ), + ( + {"collection_name": "345.4.23.12", "auth_method": {"mode": "persistent_client", "path": "/local/path"}}, + "The collection name must not be a valid IP address.", + ), + ] + + for config, expected_error in test_configs: + mock_config = ChromaIndexingConfigModel(**config) + chroma_indexer = ChromaIndexer(mock_config) + chroma_indexer._get_client = Mock() + + result = chroma_indexer.check() + self.assertEqual(result, expected_error) + + def test_valid_path(self): + test_configs = [ + ({"collection_name": "dummy-collection", "auth_method": {"mode": "persistent_client", "path": "/local/path"}}, None), + ( + {"collection_name": "dummy-collection", "auth_method": {"mode": "persistent_client", "path": "local/path"}}, + "Path must be prefixed with /local", + ), + ( + {"collection_name": "dummy-collection", "auth_method": {"mode": "persistent_client", "path": "/localpath"}}, + "Path must be prefixed with /local", + ), + ( + {"collection_name": "dummy-collection", "auth_method": {"mode": "persistent_client", "path": "./path"}}, + "Path must be prefixed with /local", + ), + ] + + for config, expected_error in test_configs: + mock_config = ChromaIndexingConfigModel(**config) + chroma_indexer = ChromaIndexer(mock_config) + chroma_indexer._get_client = Mock() + + result = chroma_indexer.check() + self.assertEqual(result, expected_error) + + def test_check_returns_expected_result(self): + check_result = self.chroma_indexer.check() + + self.assertIsNone(check_result) + + self.chroma_indexer._get_client.assert_called() + self.mock_client.heartbeat.assert_called() + self.mock_client.get_or_create_collection.assert_called() + self.mock_client.get_or_create_collection().count.assert_called() + + def test_check_handles_failure_conditions(self): + # Test 1: client heartbeat returns error + self.mock_client.heartbeat.side_effect = Exception("Random exception") + result = self.chroma_indexer.check() + self.assertTrue("Random exception" in result) + + # Test 2: client server is not alive + self.mock_client.heartbeat.side_effect = None + self.mock_client.heartbeat.return_value = None + result = self.chroma_indexer.check() + self.assertEqual(result, "Chroma client server is not alive") + + # Test 3: unable to get collection + self.mock_client.heartbeat.return_value = 45465 + self.mock_collection.count.return_value = None + result = self.chroma_indexer.check() + self.assertEqual(result, f"unable to get or create collection with name {self.chroma_indexer.collection_name}") + + def test_pre_sync_calls_delete(self): + self.chroma_indexer.pre_sync( + Mock( + streams=[ + Mock( + destination_sync_mode=DestinationSyncMode.overwrite, + stream=AirbyteStream(name="some_stream", json_schema={}, supported_sync_modes=[SyncMode.full_refresh]), + ) + ] + ) + ) + + self.mock_client.get_collection().delete.assert_called_with(where={"_ab_stream": {"$in": ["some_stream"]}}) + + def test_pre_sync_does_not_call_delete(self): + self.chroma_indexer.pre_sync( + Mock(streams=[Mock(destination_sync_mode=DestinationSyncMode.append, stream=Mock(name="some_stream"))]) + ) + + self.mock_client.get_collection().delete.assert_not_called() + + def test_index_calls_insert(self): + self.chroma_indexer.index([Mock(metadata={"key": "value"}, page_content="some content", embedding=[1, 2, 3])], None, "some_stream") + + self.mock_client.get_collection().add.assert_called_once() + + def test_index_calls_delete(self): + self.chroma_indexer.delete(["some_id"], None, "some_stream") + + self.mock_client.get_collection().delete.assert_called_with(where={"_ab_record_id": {"$in": ["some_id"]}}) diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/.dockerignore b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile deleted file mode 100644 index a86c49a6ce01..000000000000 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/Dockerfile +++ /dev/null @@ -1,51 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -RUN yum install -y python3 python3-devel jq sshpass git && yum clean all && \ - alternatives --install /usr/bin/python python /usr/bin/python3 60 && \ - python -m ensurepip --upgrade && \ - pip3 install dbt-clickhouse>=1.4.0 - -# Luckily, none of normalization's files conflict with destination-clickhouse's files :) -# We don't enforce that in any way, but hopefully we're only living in this state for a short time. -COPY --from=airbyte/normalization-clickhouse:dev /airbyte /airbyte -# Install python dependencies -WORKDIR /airbyte/base_python_structs -RUN pip3 install . -WORKDIR /airbyte/normalization_code -RUN pip3 install . -WORKDIR /airbyte/normalization_code/dbt-template/ -# Download external dbt dependencies -# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15 -RUN pip3 install "urllib3<2" -RUN dbt deps - -WORKDIR /airbyte - -ENV APPLICATION destination-clickhouse-strict-encrypt - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-clickhouse-strict-encrypt -ENV AIRBYTE_NORMALIZATION_INTEGRATION clickhouse - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.2.5 -LABEL io.airbyte.name=airbyte/destination-clickhouse-strict-encrypt - -ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh" -ENTRYPOINT ["/airbyte/run_with_normalization.sh"] diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle index bfdc74c9a53f..d1a316d740a4 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/build.gradle @@ -1,34 +1,36 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.clickhouse.ClickhouseDestinationStrictEncrypt' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:bases-destination-jdbc') implementation project(':airbyte-integrations:connectors:destination-clickhouse') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation 'com.clickhouse:clickhouse-jdbc:0.3.2-patch10:all' // https://mvnrepository.com/artifact/org.testcontainers/clickhouse - testImplementation libs.connectors.destination.testcontainers.clickhouse + testImplementation libs.testcontainers.clickhouse - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-clickhouse') // https://mvnrepository.com/artifact/org.testcontainers/clickhouse - integrationTestJavaImplementation libs.connectors.destination.testcontainers.clickhouse - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) -} - -tasks.named("airbyteDocker") { - dependsOn project(':airbyte-integrations:bases:base-normalization').airbyteDockerClickhouse + integrationTestJavaImplementation libs.testcontainers.clickhouse } diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncrypt.java b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncrypt.java index 20c6849f9331..98d998a9e3ef 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncrypt.java +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncrypt.java @@ -5,11 +5,11 @@ package io.airbyte.integrations.destination.clickhouse; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.spec_modification.SpecModifyingDestination; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.spec_modification.SpecModifyingDestination; import io.airbyte.protocol.models.v0.ConnectorSpecification; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java index d311d2a53f2f..6769060d4ff1 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationStrictEncryptAcceptanceTest.java @@ -9,18 +9,18 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; -import io.airbyte.integrations.util.HostPortResolver; import java.sql.SQLException; import java.time.Duration; import java.util.ArrayList; diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestDataComparator.java b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestDataComparator.java index 948bd8c97e72..16b0d5a390e0 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestDataComparator.java +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestDataComparator.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.clickhouse; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneOffset; diff --git a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestSourceOperations.java b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestSourceOperations.java index 3ef60b3f7205..afc76d3c5197 100644 --- a/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestSourceOperations.java +++ b/airbyte-integrations/connectors/destination-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestSourceOperations.java @@ -5,8 +5,8 @@ package io.airbyte.integrations.destination.clickhouse; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.db.DataTypeUtils; -import io.airbyte.db.jdbc.JdbcSourceOperations; +import io.airbyte.cdk.db.DataTypeUtils; +import io.airbyte.cdk.db.jdbc.JdbcSourceOperations; import java.sql.ResultSet; import java.sql.SQLException; import java.time.LocalDate; diff --git a/airbyte-integrations/connectors/destination-clickhouse/.dockerignore b/airbyte-integrations/connectors/destination-clickhouse/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-clickhouse/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile b/airbyte-integrations/connectors/destination-clickhouse/Dockerfile deleted file mode 100644 index 98560ddb4f04..000000000000 --- a/airbyte-integrations/connectors/destination-clickhouse/Dockerfile +++ /dev/null @@ -1,54 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -RUN yum install -y python3 python3-devel jq sshpass git gcc-c++ && yum clean all && \ - alternatives --install /usr/bin/python python /usr/bin/python3 60 && \ - python -m ensurepip --upgrade && \ - # these two lines are a workaround for https://github.com/yaml/pyyaml/issues/601 - pip3 install wheel && \ - pip3 install "Cython<3.0" "pyyaml==5.4" --no-build-isolation && \ - pip3 install dbt-clickhouse>=1.4.0 - -# Luckily, none of normalization's files conflict with destination-clickhouse's files :) -# We don't enforce that in any way, but hopefully we're only living in this state for a short time. -COPY --from=airbyte/normalization-clickhouse:dev /airbyte /airbyte -# Install python dependencies -WORKDIR /airbyte/base_python_structs -RUN pip3 install . -WORKDIR /airbyte/normalization_code -RUN pip3 install . -WORKDIR /airbyte/normalization_code/dbt-template/ -# Download external dbt dependencies -# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15 -RUN pip3 install "urllib3<2" -RUN dbt deps - -WORKDIR /airbyte - -ENV APPLICATION destination-clickhouse -ENV AIRBYTE_NORMALIZATION_INTEGRATION clickhouse - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-clickhouse - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.2.5 -LABEL io.airbyte.name=airbyte/destination-clickhouse - -ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh" -ENTRYPOINT ["/airbyte/run_with_normalization.sh"] diff --git a/airbyte-integrations/connectors/destination-clickhouse/README.md b/airbyte-integrations/connectors/destination-clickhouse/README.md index 38646535af5c..5ce61a36118f 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/README.md +++ b/airbyte-integrations/connectors/destination-clickhouse/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-clickhouse:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-clickhouse:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-clickhouse:dev`. the Dockerfile. #### Run @@ -61,8 +62,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-clickhouse test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/clickhouse.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-clickhouse/build.gradle b/airbyte-integrations/connectors/destination-clickhouse/build.gradle index b636c4258aed..0386841d5f45 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/build.gradle +++ b/airbyte-integrations/connectors/destination-clickhouse/build.gradle @@ -1,36 +1,35 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.clickhouse.ClickhouseDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:bases:bases-destination-jdbc') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation 'com.clickhouse:clickhouse-jdbc:0.3.2-patch10:all' // https://mvnrepository.com/artifact/org.testcontainers/clickhouse - testImplementation libs.connectors.destination.testcontainers.clickhouse - testImplementation project(":airbyte-json-validation") + testImplementation libs.testcontainers.clickhouse - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-clickhouse') - integrationTestJavaImplementation project(':airbyte-connector-test-harnesses:acceptance-test-harness') // https://mvnrepository.com/artifact/org.testcontainers/clickhouse - integrationTestJavaImplementation libs.connectors.destination.testcontainers.clickhouse - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) -} - -tasks.named("airbyteDocker") { - dependsOn project(':airbyte-integrations:bases:base-normalization').airbyteDockerClickhouse + integrationTestJavaImplementation libs.testcontainers.clickhouse } diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java index 4676d642ca3f..45a9d7cc8f08 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestination.java @@ -7,16 +7,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.ssh.SshWrappedDestination; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import java.util.Collections; diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSQLNameTransformer.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSQLNameTransformer.java index e529a9c4cf06..026fb353accb 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSQLNameTransformer.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSQLNameTransformer.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.clickhouse; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; public class ClickhouseSQLNameTransformer extends StandardNameTransformer { diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java index 9bb3570399fa..76d2fa56af89 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/main/java/io/airbyte/integrations/destination/clickhouse/ClickhouseSqlOperations.java @@ -7,9 +7,9 @@ import com.clickhouse.client.ClickHouseFormat; import com.clickhouse.jdbc.ClickHouseConnection; import com.clickhouse.jdbc.ClickHouseStatement; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.jdbc.JdbcSqlOperations; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.jdbc.JdbcSqlOperations; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.io.File; import java.io.IOException; diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java index 421a97bcd9b6..5f5c3ae948fa 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationAcceptanceTest.java @@ -9,18 +9,18 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; -import io.airbyte.integrations.util.HostPortResolver; import java.sql.SQLException; import java.time.Duration; import java.util.HashSet; diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestDataComparator.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestDataComparator.java index 948bd8c97e72..16b0d5a390e0 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestDataComparator.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestDataComparator.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.clickhouse; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneOffset; diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestSourceOperations.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestSourceOperations.java index 3ef60b3f7205..afc76d3c5197 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestSourceOperations.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/ClickhouseTestSourceOperations.java @@ -5,8 +5,8 @@ package io.airbyte.integrations.destination.clickhouse; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.db.DataTypeUtils; -import io.airbyte.db.jdbc.JdbcSourceOperations; +import io.airbyte.cdk.db.DataTypeUtils; +import io.airbyte.cdk.db.jdbc.JdbcSourceOperations; import java.sql.ResultSet; import java.sql.SQLException; import java.time.LocalDate; diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java index 0dd47c99402d..c82dfca207c1 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshClickhouseDestinationAcceptanceTest.java @@ -6,19 +6,19 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.base.ssh.SshBastionContainer; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.base.ssh.SshBastionContainer; -import io.airbyte.integrations.base.ssh.SshTunnel; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.util.ArrayList; import java.util.HashSet; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshKeyClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshKeyClickhouseDestinationAcceptanceTest.java index 10eef513b7f4..d5ebc6f26f99 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshKeyClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshKeyClickhouseDestinationAcceptanceTest.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.clickhouse; -import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; public class SshKeyClickhouseDestinationAcceptanceTest extends SshClickhouseDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshPasswordClickhouseDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshPasswordClickhouseDestinationAcceptanceTest.java index 222b72847745..56ccb4d81e1d 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshPasswordClickhouseDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test-integration/java/io/airbyte/integrations/destination/clickhouse/SshPasswordClickhouseDestinationAcceptanceTest.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.clickhouse; -import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; public class SshPasswordClickhouseDestinationAcceptanceTest extends SshClickhouseDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationSpecTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationSpecTest.java index 5f7074672a57..cf59c1712032 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationSpecTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationSpecTest.java @@ -10,10 +10,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.protocol.models.v0.ConnectorSpecification; import io.airbyte.validation.json.JsonSchemaValidator; import java.io.File; diff --git a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java index 99a84f3a7107..0b05cb932a8e 100644 --- a/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java +++ b/airbyte-integrations/connectors/destination-clickhouse/src/test/java/io/airbyte/integrations/destination/clickhouse/ClickhouseDestinationTest.java @@ -8,16 +8,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-convex/README.md b/airbyte-integrations/connectors/destination-convex/README.md index 96cb10f67685..dc91b1ed5119 100644 --- a/airbyte-integrations/connectors/destination-convex/README.md +++ b/airbyte-integrations/connectors/destination-convex/README.md @@ -34,14 +34,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle - -From the Airbyte repository root, run: - -``` -./gradlew :airbyte-integrations:connectors:destination-convex:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/destinations/convex) @@ -62,23 +54,20 @@ python main.py write --config secrets/config.json --catalog integration_tests/co ### Locally running the connector docker image -#### Build - -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/destination-convex:dev +#### Build +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=destination-convex build ``` -You can also build the connector image via Gradle: +An image will be built with the tag `airbyte/destination-convex:dev`. -``` -./gradlew :airbyte-integrations:connectors:destination-convex:airbyteDocker +**Via `docker build`:** +```bash +docker build -t airbyte/destination-convex:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. - #### Run Then run any of the connector commands as follows: @@ -90,53 +79,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-convex:dev check cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-convex:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing - -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: - -``` -pip install .[tests] -``` - -### Unit Tests - -To run unit tests locally, from the connector directory run: - -``` -python -m pytest unit_tests -``` - -### Integration Tests - -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). - -#### Custom Integration tests - -Place custom tests inside `integration_tests/` folder, then, from the connector root, run - -``` -python -m pytest integration_tests -``` - -#### Acceptance Tests -Coming soon: - -### Using gradle to run tests - -All commands should be run from airbyte project root. -To run unit tests: - -``` -./gradlew :airbyte-integrations:connectors:destination-convex:unitTest +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=destination-convex test ``` -To run acceptance and custom integration tests: - -``` -./gradlew :airbyte-integrations:connectors:destination-convex:integrationTest -``` +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management @@ -147,11 +99,12 @@ We split dependencies between two groups, dependencies that are: - required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector - You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-convex test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/convex.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/destination-convex/build.gradle b/airbyte-integrations/connectors/destination-convex/build.gradle deleted file mode 100644 index ff61bd60f9fe..000000000000 --- a/airbyte-integrations/connectors/destination-convex/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'destination_convex' -} diff --git a/airbyte-integrations/connectors/destination-convex/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-convex/unit_tests/unit_test.py index ee278a1d913e..5f0eccbb0a66 100644 --- a/airbyte-integrations/connectors/destination-convex/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/destination-convex/unit_tests/unit_test.py @@ -94,7 +94,12 @@ def setup_good_responses(config): def setup_bad_response(config): - responses.add(responses.PUT, f"{config['deployment_url']}/api/streaming_import/clear_tables", status=400, json={"code": "ErrorCode", "message": "error message"}) + responses.add( + responses.PUT, + f"{config['deployment_url']}/api/streaming_import/clear_tables", + status=400, + json={"code": "ErrorCode", "message": "error message"}, + ) @responses.activate diff --git a/airbyte-integrations/connectors/destination-csv/.dockerignore b/airbyte-integrations/connectors/destination-csv/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-csv/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-csv/Dockerfile b/airbyte-integrations/connectors/destination-csv/Dockerfile deleted file mode 100644 index 9c358e9d0c64..000000000000 --- a/airbyte-integrations/connectors/destination-csv/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-csv - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-csv - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=1.0.0 -LABEL io.airbyte.name=airbyte/destination-csv diff --git a/airbyte-integrations/connectors/destination-csv/build.gradle b/airbyte-integrations/connectors/destination-csv/build.gradle index 126d47b6e8a3..d4a73e5d7143 100644 --- a/airbyte-integrations/connectors/destination-csv/build.gradle +++ b/airbyte-integrations/connectors/destination-csv/build.gradle @@ -1,9 +1,23 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.csv.CsvDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] @@ -11,10 +25,4 @@ application { dependencies { implementation 'org.apache.commons:commons-csv:1.4' - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') } diff --git a/airbyte-integrations/connectors/destination-csv/src/main/java/io/airbyte/integrations/destination/csv/CsvDestination.java b/airbyte-integrations/connectors/destination-csv/src/main/java/io/airbyte/integrations/destination/csv/CsvDestination.java index 7528f4c9c0a7..9e04d2fdd366 100644 --- a/airbyte-integrations/connectors/destination-csv/src/main/java/io/airbyte/integrations/destination/csv/CsvDestination.java +++ b/airbyte-integrations/connectors/destination-csv/src/main/java/io/airbyte/integrations/destination/csv/CsvDestination.java @@ -6,14 +6,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Preconditions; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.CommitOnStateAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.CommitOnStateAirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-csv/src/test-integration/java/io/airbyte/integrations/destination/csv/CsvDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-csv/src/test-integration/java/io/airbyte/integrations/destination/csv/CsvDestinationAcceptanceTest.java index 30856da6aa84..550d5e1ff7bb 100644 --- a/airbyte-integrations/connectors/destination-csv/src/test-integration/java/io/airbyte/integrations/destination/csv/CsvDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-csv/src/test-integration/java/io/airbyte/integrations/destination/csv/CsvDestinationAcceptanceTest.java @@ -8,14 +8,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; +import io.airbyte.cdk.integrations.standardtest.destination.argproviders.DataArgumentsProvider; +import io.airbyte.cdk.integrations.standardtest.destination.argproviders.util.ArgumentProviderUtil; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.ProtocolVersion; -import io.airbyte.integrations.standardtest.destination.argproviders.DataArgumentsProvider; -import io.airbyte.integrations.standardtest.destination.argproviders.util.ArgumentProviderUtil; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.CatalogHelpers; diff --git a/airbyte-integrations/connectors/destination-csv/src/test/java/io/airbyte/integrations/destination/csv/CsvDestinationTest.java b/airbyte-integrations/connectors/destination-csv/src/test/java/io/airbyte/integrations/destination/csv/CsvDestinationTest.java index 00d800ee0cea..0d8144888b71 100644 --- a/airbyte-integrations/connectors/destination-csv/src/test/java/io/airbyte/integrations/destination/csv/CsvDestinationTest.java +++ b/airbyte-integrations/connectors/destination-csv/src/test/java/io/airbyte/integrations/destination/csv/CsvDestinationTest.java @@ -17,12 +17,12 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; diff --git a/airbyte-integrations/connectors/destination-cumulio/README.md b/airbyte-integrations/connectors/destination-cumulio/README.md index 7cebe3369192..62261106b05f 100644 --- a/airbyte-integrations/connectors/destination-cumulio/README.md +++ b/airbyte-integrations/connectors/destination-cumulio/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:destination-cumulio:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/destinations/cumulio) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_cumulio/spec.json` file. @@ -53,18 +47,19 @@ python main.py write --config secrets/config.json --catalog integration_tests/co ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/destination-cumulio:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=destination-cumulio build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:destination-cumulio:airbyteDocker +An image will be built with the tag `airbyte/destination-cumulio:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/destination-cumulio:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -74,38 +69,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-cumulio:dev check # messages.jsonl is a file containing line-separated JSON representing AirbyteMessages cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-cumulio:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=destination-cumulio test ``` -#### Acceptance Tests -Coming soon: -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:destination-cumulio:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:destination-cumulio:integrationTest -``` +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -115,8 +88,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-cumulio test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/cumulio.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-cumulio/build.gradle b/airbyte-integrations/connectors/destination-cumulio/build.gradle deleted file mode 100644 index 188e7dc95f70..000000000000 --- a/airbyte-integrations/connectors/destination-cumulio/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'destination_cumulio' -} diff --git a/airbyte-integrations/connectors/destination-cumulio/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-cumulio/integration_tests/integration_test.py index 0b7e1afcab63..545241d463e7 100644 --- a/airbyte-integrations/connectors/destination-cumulio/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-cumulio/integration_tests/integration_test.py @@ -71,9 +71,7 @@ def configured_catalog_fixture() -> ConfiguredAirbyteCatalog: @pytest.fixture(autouse=True) -def delete_datasets( - config: Mapping, configured_catalog: ConfiguredAirbyteCatalog, logger: Logger -): +def delete_datasets(config: Mapping, configured_catalog: ConfiguredAirbyteCatalog, logger: Logger): cumulio_client = CumulioClient(config, logger) for stream in configured_catalog.streams: dataset = cumulio_client.get_dataset_and_columns_from_stream_name(stream.stream.name) @@ -116,9 +114,7 @@ def _state(data: Dict[str, Any]) -> AirbyteMessage: return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data=data)) -def _record( - stream_name: str, str_value: str, int_value: int, obj_value: dict, arr_value: list -) -> AirbyteMessage: +def _record(stream_name: str, str_value: str, int_value: int, obj_value: dict, arr_value: list) -> AirbyteMessage: return AirbyteMessage( type=Type.RECORD, record=AirbyteRecordMessage( @@ -135,9 +131,7 @@ def _record( def _retrieve_all_records(cumulio_client, stream_name): - dataset_and_columns = cumulio_client.get_dataset_and_columns_from_stream_name( - stream_name - ) + dataset_and_columns = cumulio_client.get_dataset_and_columns_from_stream_name(stream_name) # Wait 5 seconds before trying to retrieve the data to ensure it can be properly retrieved time.sleep(5) if dataset_and_columns is not None: @@ -177,9 +171,7 @@ def _retrieve_all_records(cumulio_client, stream_name): airbyte_data_to_return.append( AirbyteMessage( type=Type.RECORD, - record=AirbyteRecordMessage( - stream=stream_name, data=airbyte_data_row, emitted_at=0 - ), + record=AirbyteRecordMessage(stream=stream_name, data=airbyte_data_row, emitted_at=0), ) ) return airbyte_data_to_return @@ -201,24 +193,14 @@ def test_write_append( destination = DestinationCumulio() state_message = _state({"state": "3"}) - record_chunk_1 = [ - _record(stream_name, "test-" + str(i), i, {"test": i}, ["test", i]) - for i in range(1, 3) - ] + record_chunk_1 = [_record(stream_name, "test-" + str(i), i, {"test": i}, ["test", i]) for i in range(1, 3)] - output_states_1 = list( - destination.write(config, configured_catalog, [*record_chunk_1, state_message]) - ) + output_states_1 = list(destination.write(config, configured_catalog, [*record_chunk_1, state_message])) assert [state_message] == output_states_1 - record_chunk_2 = [ - _record(stream_name, "test-" + str(i), i, {"test": i}, ["test", i]) - for i in range(3, 5) - ] + record_chunk_2 = [_record(stream_name, "test-" + str(i), i, {"test": i}, ["test", i]) for i in range(3, 5)] - output_states_2 = list( - destination.write(config, configured_catalog, [*record_chunk_2, state_message]) - ) + output_states_2 = list(destination.write(config, configured_catalog, [*record_chunk_2, state_message])) assert [state_message] == output_states_2 cumulio_client = CumulioClient(config, logger) @@ -260,24 +242,14 @@ def test_write_overwrite( destination = DestinationCumulio() state_message = _state({"state": "3"}) - record_chunk_1 = [ - _record(stream_name, "oldtest-" + str(i), i, {"oldtest": i}, ["oldtest", i]) - for i in range(1, 3) - ] + record_chunk_1 = [_record(stream_name, "oldtest-" + str(i), i, {"oldtest": i}, ["oldtest", i]) for i in range(1, 3)] - output_states_1 = list( - destination.write(config, configured_catalog, [*record_chunk_1, state_message]) - ) + output_states_1 = list(destination.write(config, configured_catalog, [*record_chunk_1, state_message])) assert [state_message] == output_states_1 - record_chunk_2 = [ - _record(stream_name, "newtest-" + str(i), i, {"newtest": i}, ["newtest", i]) - for i in range(1, 3) - ] + record_chunk_2 = [_record(stream_name, "newtest-" + str(i), i, {"newtest": i}, ["newtest", i]) for i in range(1, 3)] - output_states_2 = list( - destination.write(config, configured_catalog, [*record_chunk_2, state_message]) - ) + output_states_2 = list(destination.write(config, configured_catalog, [*record_chunk_2, state_message])) assert [state_message] == output_states_2 cumulio_client = CumulioClient(config, logger) diff --git a/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_client.py b/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_client.py index e29eb8cf6c20..258e8ff2a578 100644 --- a/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_client.py @@ -81,9 +81,7 @@ def test_batch_write_append_empty_write_buffer(cumulio_client: CumulioClient): cumulio_client._push_batch_to_existing_dataset.assert_not_called() -def test_batch_write_append_no_existing_dataset( - cumulio_client: CumulioClient, dummy_data: Mapping[str, Any] -): +def test_batch_write_append_no_existing_dataset(cumulio_client: CumulioClient, dummy_data: Mapping[str, Any]): cumulio_client.client.get = MagicMock(return_value={"count": 0, "Rows": []}) cumulio_client._push_batch_to_new_dataset = MagicMock() # type: ignore cumulio_client._push_batch_to_existing_dataset = MagicMock() # type: ignore @@ -116,14 +114,10 @@ def test_batch_write_append_no_existing_dataset( cumulio_client._push_batch_to_existing_dataset.assert_not_called() - cumulio_client._push_batch_to_new_dataset.assert_called_once_with( - stream_name, dummy_data["data"], dummy_data["columns"] - ) + cumulio_client._push_batch_to_new_dataset.assert_called_once_with(stream_name, dummy_data["data"], dummy_data["columns"]) -def test_batch_write_existing_dataset_no_first_batch_replace( - cumulio_client: CumulioClient, dummy_data: Mapping[str, Any] -): +def test_batch_write_existing_dataset_no_first_batch_replace(cumulio_client: CumulioClient, dummy_data: Mapping[str, Any]): cumulio_client._get_dataset_id_from_stream_name = MagicMock(return_value="dataset_id") # type: ignore cumulio_client._push_batch_to_new_dataset = MagicMock() # type: ignore cumulio_client._push_batch_to_existing_dataset = MagicMock() # type: ignore @@ -146,9 +140,7 @@ def test_batch_write_existing_dataset_no_first_batch_replace( ) -def test_batch_write_existing_dataset_first_batch_replace_overwrite_mode( - cumulio_client: CumulioClient, dummy_data: Mapping[str, Any] -): +def test_batch_write_existing_dataset_first_batch_replace_overwrite_mode(cumulio_client: CumulioClient, dummy_data: Mapping[str, Any]): cumulio_client._get_dataset_id_from_stream_name = MagicMock(return_value="dataset_id") # type: ignore cumulio_client._push_batch_to_new_dataset = MagicMock() # type: ignore cumulio_client._push_batch_to_existing_dataset = MagicMock() # type: ignore @@ -171,9 +163,7 @@ def test_batch_write_existing_dataset_first_batch_replace_overwrite_mode( ) -def test_batch_write_existing_dataset_first_batch_replace_tag( - cumulio_client: CumulioClient, dummy_data: Mapping[str, Any] -): +def test_batch_write_existing_dataset_first_batch_replace_tag(cumulio_client: CumulioClient, dummy_data: Mapping[str, Any]): cumulio_client._get_dataset_id_from_stream_name = MagicMock(return_value="dataset_id") # type: ignore cumulio_client._push_batch_to_new_dataset = MagicMock() # type: ignore cumulio_client._push_batch_to_existing_dataset = MagicMock() # type: ignore @@ -196,9 +186,7 @@ def test_batch_write_existing_dataset_first_batch_replace_tag( ) -def test_batch_write_existing_dataset_non_first_batch( - cumulio_client: CumulioClient, dummy_data: Mapping[str, Any] -): +def test_batch_write_existing_dataset_non_first_batch(cumulio_client: CumulioClient, dummy_data: Mapping[str, Any]): cumulio_client._get_dataset_id_from_stream_name = MagicMock(return_value="dataset_id") # type: ignore cumulio_client._push_batch_to_new_dataset = MagicMock() # type: ignore cumulio_client._push_batch_to_existing_dataset = MagicMock() # type: ignore @@ -235,23 +223,17 @@ def test_api_token_api_call(cumulio_client: CumulioClient): """ "Test that the test_api_token method makes an API request to the authorization endpoint""" cumulio_client.client.get = MagicMock(return_value={"count": 1}) cumulio_client.test_api_token() - cumulio_client.client.get.assert_called_with( - "authorization", {"where": {"type": "api"}} - ) + cumulio_client.client.get.assert_called_with("authorization", {"where": {"type": "api"}}) -def test_test_data_push_method( - cumulio_client: CumulioClient, dummy_data: Mapping[str, Any] -): +def test_test_data_push_method(cumulio_client: CumulioClient, dummy_data: Mapping[str, Any]): """ "Test that the test_data_push method deletes the dataset afterwards""" cumulio_client.batch_write = MagicMock() # type: ignore cumulio_client.delete_dataset = MagicMock() # type: ignore stream_name = "test-stream" - cumulio_client.test_data_push( - stream_name, dummy_data["data"], dummy_data["columns"] - ) + cumulio_client.test_data_push(stream_name, dummy_data["data"], dummy_data["columns"]) cumulio_client.delete_dataset.assert_called_once_with("test-stream") @@ -266,9 +248,7 @@ def test_delete_dataset_no_dataset_found(cumulio_client: CumulioClient): cumulio_client.delete_dataset("stream_name") # assert that the _get_dataset_id_from_stream_name method was called once with the correct arguments - cumulio_client._get_dataset_id_from_stream_name.assert_called_once_with( - "stream_name" - ) + cumulio_client._get_dataset_id_from_stream_name.assert_called_once_with("stream_name") # assert that the client.delete method is not called as no dataset was found cumulio_client.client.delete.assert_not_called() @@ -283,9 +263,7 @@ def test_delete_dataset_dataset_found(cumulio_client: CumulioClient): cumulio_client.delete_dataset("stream_name") # assert that the _get_dataset_id_from_stream_name method was called once with the correct arguments - cumulio_client._get_dataset_id_from_stream_name.assert_called_once_with( - "stream_name" - ) + cumulio_client._get_dataset_id_from_stream_name.assert_called_once_with("stream_name") # assert that the client.delete method was called once with the correct arguments cumulio_client.client.delete.assert_called_once_with("securable", "dataset_id") @@ -295,9 +273,7 @@ def test_delete_dataset_dataset_found(cumulio_client: CumulioClient): def test_get_ordered_columns_dataset_not_created(cumulio_client: CumulioClient): - cumulio_client.get_dataset_and_columns_from_stream_name = MagicMock( # type: ignore - return_value=None - ) + cumulio_client.get_dataset_and_columns_from_stream_name = MagicMock(return_value=None) # type: ignore result = cumulio_client.get_ordered_columns("stream_name") assert result == [] @@ -310,9 +286,7 @@ def test_get_ordered_columns_same_order(cumulio_client: CumulioClient): {"source_name": "column2", "order": 1}, ], } - cumulio_client.get_dataset_and_columns_from_stream_name = MagicMock( # type: ignore - return_value=cumulio_dataset_and_columns - ) + cumulio_client.get_dataset_and_columns_from_stream_name = MagicMock(return_value=cumulio_dataset_and_columns) # type: ignore result = cumulio_client.get_ordered_columns("stream_name") assert result == ["column2", "column1"] @@ -320,12 +294,8 @@ def test_get_ordered_columns_same_order(cumulio_client: CumulioClient): # tests for _push_batch_to_new_dataset method -def test_push_batch_to_new_dataset( - cumulio_client: CumulioClient, dummy_data: Mapping[str, Any] -): - cumulio_client.client.create = MagicMock( - return_value={"rows": [{"id": "new_dataset_id"}]} - ) +def test_push_batch_to_new_dataset(cumulio_client: CumulioClient, dummy_data: Mapping[str, Any]): + cumulio_client.client.create = MagicMock(return_value={"rows": [{"id": "new_dataset_id"}]}) cumulio_client._associate_tag_dataset_id = MagicMock() # type: ignore stream_name = "test_stream" @@ -339,35 +309,21 @@ def test_push_batch_to_new_dataset( "name": {"en": cumulio_client.INITIAL_DATASET_NAME_PREFIX + stream_name}, }, } - cumulio_client._push_batch_to_new_dataset( - stream_name, dummy_data["data"], dummy_data["columns"] - ) - cumulio_client.client.create.assert_called_once_with( - "data", expected_request_properties - ) - cumulio_client._associate_tag_dataset_id.assert_called_once_with( - stream_name, "new_dataset_id" - ) + cumulio_client._push_batch_to_new_dataset(stream_name, dummy_data["data"], dummy_data["columns"]) + cumulio_client.client.create.assert_called_once_with("data", expected_request_properties) + cumulio_client._associate_tag_dataset_id.assert_called_once_with(stream_name, "new_dataset_id") -def test_push_batch_to_new_dataset_all_retries_error( - cumulio_client: CumulioClient, dummy_data: Mapping[str, Any] -): - cumulio_client.client.create = MagicMock( - side_effect=RuntimeError("Internal Server Error") - ) +def test_push_batch_to_new_dataset_all_retries_error(cumulio_client: CumulioClient, dummy_data: Mapping[str, Any]): + cumulio_client.client.create = MagicMock(side_effect=RuntimeError("Internal Server Error")) stream_name = "test_stream" with patch("destination_cumulio.client.time", MagicMock()): with pytest.raises(Exception): - cumulio_client._push_batch_to_new_dataset( - stream_name, dummy_data["data"], dummy_data["columns"] - ) + cumulio_client._push_batch_to_new_dataset(stream_name, dummy_data["data"], dummy_data["columns"]) -def test_push_batch_to_new_dataset_first_try_fails( - cumulio_client: CumulioClient, dummy_data: Mapping[str, Any] -): +def test_push_batch_to_new_dataset_first_try_fails(cumulio_client: CumulioClient, dummy_data: Mapping[str, Any]): effects = iter([RuntimeError("Internal Server Error")]) def side_effect(*_): @@ -392,43 +348,29 @@ def side_effect(*_): } with patch("destination_cumulio.client.time", MagicMock()): - cumulio_client._push_batch_to_new_dataset( - stream_name, dummy_data["data"], dummy_data["columns"] - ) - cumulio_client.client.create.assert_called_with( - "data", expected_request_properties - ) + cumulio_client._push_batch_to_new_dataset(stream_name, dummy_data["data"], dummy_data["columns"]) + cumulio_client.client.create.assert_called_with("data", expected_request_properties) assert cumulio_client.client.create.call_count == 2 - cumulio_client._associate_tag_dataset_id.assert_called_once_with( - stream_name, "new_dataset_id" - ) + cumulio_client._associate_tag_dataset_id.assert_called_once_with(stream_name, "new_dataset_id") # tests for _push_batch_to_existing_dataset method -def test_push_batch_to_existing_dataset_all_retries_error( - cumulio_client: CumulioClient, dummy_data: Mapping[str, Any] -): - cumulio_client.client.create = MagicMock( - side_effect=RuntimeError("Internal Server Error") - ) +def test_push_batch_to_existing_dataset_all_retries_error(cumulio_client: CumulioClient, dummy_data: Mapping[str, Any]): + cumulio_client.client.create = MagicMock(side_effect=RuntimeError("Internal Server Error")) cumulio_client._remove_replace_tag_dataset_id_association = MagicMock() # type: ignore dataset_id = "dataset_id" with patch("destination_cumulio.client.time", MagicMock()): with pytest.raises(Exception): - cumulio_client._push_batch_to_existing_dataset( - dataset_id, dummy_data["data"], dummy_data["columns"], False, True - ) + cumulio_client._push_batch_to_existing_dataset(dataset_id, dummy_data["data"], dummy_data["columns"], False, True) -def test_push_batch_to_existing_dataset_first_try_fails( - cumulio_client: CumulioClient, dummy_data: Mapping[str, Any] -): +def test_push_batch_to_existing_dataset_first_try_fails(cumulio_client: CumulioClient, dummy_data: Mapping[str, Any]): effects = iter([RuntimeError("Internal Server Error")]) def side_effect(*_): @@ -453,21 +395,15 @@ def side_effect(*_): } with patch("destination_cumulio.client.time", MagicMock()): - cumulio_client._push_batch_to_existing_dataset( - dataset_id, dummy_data["data"], dummy_data["columns"], False, True - ) - cumulio_client.client.create.assert_called_with( - "data", expected_request_properties - ) + cumulio_client._push_batch_to_existing_dataset(dataset_id, dummy_data["data"], dummy_data["columns"], False, True) + cumulio_client.client.create.assert_called_with("data", expected_request_properties) assert cumulio_client.client.create.call_count == 2 cumulio_client._remove_replace_tag_dataset_id_association.assert_not_called() -def test_push_batch_to_existing_dataset_no_first_batch_replace( - cumulio_client: CumulioClient, dummy_data: Mapping[str, Any] -): +def test_push_batch_to_existing_dataset_no_first_batch_replace(cumulio_client: CumulioClient, dummy_data: Mapping[str, Any]): cumulio_client.client.create = MagicMock() cumulio_client._remove_replace_tag_dataset_id_association = MagicMock() # type: ignore @@ -483,18 +419,12 @@ def test_push_batch_to_existing_dataset_no_first_batch_replace( }, } - cumulio_client._push_batch_to_existing_dataset( - dataset_id, dummy_data["data"], dummy_data["columns"], False, True - ) - cumulio_client.client.create.assert_called_once_with( - "data", expected_request_properties - ) + cumulio_client._push_batch_to_existing_dataset(dataset_id, dummy_data["data"], dummy_data["columns"], False, True) + cumulio_client.client.create.assert_called_once_with("data", expected_request_properties) cumulio_client._remove_replace_tag_dataset_id_association.assert_not_called() -def test_push_batch_to_existing_dataset_first_batch_replace( - cumulio_client: CumulioClient, dummy_data: Mapping[str, Any] -): +def test_push_batch_to_existing_dataset_first_batch_replace(cumulio_client: CumulioClient, dummy_data: Mapping[str, Any]): cumulio_client.client.create = MagicMock() cumulio_client._remove_replace_tag_dataset_id_association = MagicMock() # type: ignore @@ -510,15 +440,9 @@ def test_push_batch_to_existing_dataset_first_batch_replace( }, } - cumulio_client._push_batch_to_existing_dataset( - dataset_id, dummy_data["data"], dummy_data["columns"], True, True - ) - cumulio_client.client.create.assert_called_once_with( - "data", expected_request_properties - ) - cumulio_client._remove_replace_tag_dataset_id_association.assert_called_once_with( - dataset_id - ) + cumulio_client._push_batch_to_existing_dataset(dataset_id, dummy_data["data"], dummy_data["columns"], True, True) + cumulio_client.client.create.assert_called_once_with("data", expected_request_properties) + cumulio_client._remove_replace_tag_dataset_id_association.assert_called_once_with(dataset_id) # tests for _dataset_contains_replace_tag method @@ -530,9 +454,7 @@ def test_get_dataset_and_columns_from_stream_name_no_dataset( cumulio_dataset_and_columns_result = {"count": 0, "rows": []} # Test when no dataset is found - cumulio_client.client.get = MagicMock( - return_value=cumulio_dataset_and_columns_result - ) + cumulio_client.client.get = MagicMock(return_value=cumulio_dataset_and_columns_result) result = cumulio_client.get_dataset_and_columns_from_stream_name("test_stream") assert result is None @@ -553,9 +475,7 @@ def test_get_dataset_and_columns_from_stream_name_single_existing_dataset( ], } # Test when dataset is found - cumulio_client.client.get = MagicMock( - return_value=cumulio_dataset_and_columns_result - ) + cumulio_client.client.get = MagicMock(return_value=cumulio_dataset_and_columns_result) result = cumulio_client.get_dataset_and_columns_from_stream_name("test_stream") assert result["id"] == cumulio_dataset_and_columns_result["rows"][0]["id"] assert result["columns"] == cumulio_dataset_and_columns_result["rows"][0]["columns"] @@ -585,9 +505,7 @@ def test_get_dataset_and_columns_from_stream_name_multiple_existing_datasets( ], } # Test when multiple datasets are found - cumulio_client.client.get = MagicMock( - return_value=cumulio_dataset_and_columns_result - ) + cumulio_client.client.get = MagicMock(return_value=cumulio_dataset_and_columns_result) with pytest.raises(Exception): cumulio_client.get_dataset_and_columns_from_stream_name("test_stream") @@ -601,26 +519,18 @@ def test_set_replace_tag_on_dataset_no_dataset_found(cumulio_client: CumulioClie cumulio_client.set_replace_tag_on_dataset("stream_name") - cumulio_client._get_dataset_id_from_stream_name.assert_called_once_with( - "stream_name" - ) + cumulio_client._get_dataset_id_from_stream_name.assert_called_once_with("stream_name") cumulio_client._associate_tag_dataset_id.assert_not_called() def test_set_replace_tag_on_dataset_existing_dataset(cumulio_client: CumulioClient): - cumulio_client._get_dataset_id_from_stream_name = MagicMock( # type: ignore - return_value="dataset_id" - ) + cumulio_client._get_dataset_id_from_stream_name = MagicMock(return_value="dataset_id") # type: ignore cumulio_client._associate_tag_dataset_id = MagicMock() # type: ignore cumulio_client.set_replace_tag_on_dataset("stream_name") - cumulio_client._get_dataset_id_from_stream_name.assert_called_once_with( - "stream_name" - ) - cumulio_client._associate_tag_dataset_id.assert_called_once_with( - cumulio_client.REPLACE_TAG, "dataset_id" - ) + cumulio_client._get_dataset_id_from_stream_name.assert_called_once_with("stream_name") + cumulio_client._associate_tag_dataset_id.assert_called_once_with(cumulio_client.REPLACE_TAG, "dataset_id") # tests for _dataset_contains_replace_tag method @@ -681,9 +591,7 @@ def test_associate_tag_dataset_id_no_tag_found(cumulio_client: CumulioClient): cumulio_client._associate_tag_dataset_id("test_stream", "test_dataset_id") - cumulio_client._create_and_associate_stream_name_tag_with_dataset_id.assert_called_once_with( - "test_stream", "test_dataset_id" - ) + cumulio_client._create_and_associate_stream_name_tag_with_dataset_id.assert_called_once_with("test_stream", "test_dataset_id") cumulio_client._associate_tag_with_dataset_id.assert_not_called() @@ -694,9 +602,7 @@ def test_associate_tag_dataset_id_tag_found(cumulio_client: CumulioClient): cumulio_client._associate_tag_dataset_id("test_stream", "test_dataset_id") - cumulio_client._associate_tag_with_dataset_id.assert_called_once_with( - "tag_id", "test_dataset_id" - ) + cumulio_client._associate_tag_with_dataset_id.assert_called_once_with("tag_id", "test_dataset_id") cumulio_client._create_and_associate_stream_name_tag_with_dataset_id.assert_not_called() diff --git a/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_destination.py b/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_destination.py index 3fbfb150c4f4..4805fb51ecf5 100644 --- a/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_destination.py +++ b/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_destination.py @@ -121,9 +121,7 @@ def test_write_no_input_messages( destination_cumulio = DestinationCumulio() input_messages = [airbyte_state_message] - result = list( - destination_cumulio.write(config, configured_catalog, input_messages) - ) + result = list(destination_cumulio.write(config, configured_catalog, input_messages)) assert result == [airbyte_state_message] assert cumulio_writer.mock_calls == [ @@ -145,19 +143,13 @@ def test_write( with patch("destination_cumulio.destination.CumulioWriter") as cumulio_writer: input_messages = [airbyte_message_1, airbyte_message_2, airbyte_state_message] destination_cumulio = DestinationCumulio() - result = list( - destination_cumulio.write(config, configured_catalog, input_messages) - ) + result = list(destination_cumulio.write(config, configured_catalog, input_messages)) assert result == [airbyte_state_message] assert cumulio_writer.mock_calls == [ call(config, configured_catalog, logger), call().delete_stream_entries("overwrite_stream"), - call().queue_write_operation( - "append_stream", {"string_column": "value_1", "int_column": 1} - ), - call().queue_write_operation( - "overwrite_stream", {"string_column": "value_2", "int_column": 2} - ), + call().queue_write_operation("append_stream", {"string_column": "value_1", "int_column": 1}), + call().queue_write_operation("overwrite_stream", {"string_column": "value_2", "int_column": 2}), call().flush_all(), # The first flush_all is called before yielding the state message call().flush_all(), # The second flush_all is called after going through all input messages ] diff --git a/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_writer.py b/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_writer.py index 93309375623a..ac921c7ef5c4 100644 --- a/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_writer.py +++ b/airbyte-integrations/connectors/destination-cumulio/unit_tests/test_writer.py @@ -61,9 +61,7 @@ def configured_catalog_fixture() -> ConfiguredAirbyteCatalog: destination_sync_mode=DestinationSyncMode.overwrite, ) - return ConfiguredAirbyteCatalog( - streams=[orders_append_stream, products_overwrite_stream] - ) + return ConfiguredAirbyteCatalog(streams=[orders_append_stream, products_overwrite_stream]) @pytest.fixture(name="writer") @@ -73,9 +71,7 @@ def writer_no_existing_cumulio_columns( logger: MagicMock, ) -> CumulioWriter: """Returns a CumulioWriter using MagicMock, and mocking the return_value of all used CumulioClient methods.""" - with patch( - "destination_cumulio.writer.CumulioClient", MagicMock() - ) as cumulio_client_mock: + with patch("destination_cumulio.writer.CumulioClient", MagicMock()) as cumulio_client_mock: # Mock get_ordered_columns to return no existing Cumul.io columns (dataset hasn't been created yet --> first sync) cumulio_client_mock.return_value.get_ordered_columns.return_value = [] # cumulio_client_mock.return_value.batch_write.return_value = None @@ -178,9 +174,7 @@ def test_queue_write_operation(writer: CumulioWriter): # Assert that write_buffer from the orders stream contains a single value assert len(writer.writers["orders"]["write_buffer"]) == 1 - case.assertCountEqual( - writer.writers["orders"]["write_buffer"][0], ["customer_1", 1, 100.0] - ) + case.assertCountEqual(writer.writers["orders"]["write_buffer"][0], ["customer_1", 1, 100.0]) def test_queue_write_operation_two_streams(writer: CumulioWriter): @@ -292,20 +286,15 @@ def writer_existing_cumulio_columns( existing_cumulio_columns = {} for configured_stream in configured_catalog.streams: existing_cumulio_columns[configured_stream.stream.name] = [ - column_name - for column_name in configured_stream.stream.json_schema["properties"] + column_name for column_name in configured_stream.stream.json_schema["properties"] ] def get_existing_cumulio_columns(stream_name): return existing_cumulio_columns[stream_name] - with patch( - "destination_cumulio.writer.CumulioClient", MagicMock() - ) as cumulio_client_mock: + with patch("destination_cumulio.writer.CumulioClient", MagicMock()) as cumulio_client_mock: # Mock get_ordered_columns to return existing_cumulio_columns - cumulio_client_mock.return_value.get_ordered_columns = MagicMock( - side_effect=get_existing_cumulio_columns - ) + cumulio_client_mock.return_value.get_ordered_columns = MagicMock(side_effect=get_existing_cumulio_columns) return CumulioWriter(config, configured_catalog, logger) @@ -314,9 +303,7 @@ def test_init_existing_cumulio_columns(writer_existing_cumulio_columns: CumulioW Due to identical columns in Cumul.io for this writer, both are False. """ assert writer_existing_cumulio_columns.writers["orders"]["update_metadata"] is False - assert ( - writer_existing_cumulio_columns.writers["products"]["update_metadata"] is False - ) + assert writer_existing_cumulio_columns.writers["products"]["update_metadata"] is False def test_equal_cumulio_and_merged_columns( @@ -344,10 +331,7 @@ def test_queue_write_operation_with_correct_data_order( expected_data = [] for column in result["orders"]["merged_columns"]: expected_data.append(order_data[column["name"]]) - assert ( - writer_existing_cumulio_columns.writers["orders"]["write_buffer"][0] - == expected_data - ) + assert writer_existing_cumulio_columns.writers["orders"]["write_buffer"][0] == expected_data @pytest.fixture(name="configured_catalog_with_new_column") @@ -384,9 +368,7 @@ def configured_catalog_with_new_column_fixture() -> ConfiguredAirbyteCatalog: destination_sync_mode=DestinationSyncMode.overwrite, ) - return ConfiguredAirbyteCatalog( - streams=[orders_append_stream, orders_overwrite_stream] - ) + return ConfiguredAirbyteCatalog(streams=[orders_append_stream, orders_overwrite_stream]) @pytest.fixture @@ -398,10 +380,7 @@ def writer_new_airbyte_column( """This will return a CumulioWriter that mocks airbyte stream catalogs that contains one column that does not exist in Cumul.io.""" existing_cumulio_columns = {} for configured_stream in configured_catalog_with_new_column.streams: - columns = [ - column_name - for column_name in configured_stream.stream.json_schema["properties"] - ] + columns = [column_name for column_name in configured_stream.stream.json_schema["properties"]] # get rid of the second element to mimic a new column being defined in configured_stream del columns[1] existing_cumulio_columns[configured_stream.stream.name] = columns @@ -409,13 +388,9 @@ def writer_new_airbyte_column( def get_existing_cumulio_columns(stream_name): return existing_cumulio_columns[stream_name] - with patch( - "destination_cumulio.writer.CumulioClient", MagicMock() - ) as cumulio_client_mock: + with patch("destination_cumulio.writer.CumulioClient", MagicMock()) as cumulio_client_mock: # Mock get_ordered_columns to return existing_cumulio_columns (which does not include one column defined in configured stream) - cumulio_client_mock.return_value.get_ordered_columns = MagicMock( - side_effect=get_existing_cumulio_columns - ) + cumulio_client_mock.return_value.get_ordered_columns = MagicMock(side_effect=get_existing_cumulio_columns) cumulio_client_mock.return_value.batch_writer.return_value = None cumulio_client_mock.return_value.set_replace_tag_on_dataset.return_value = None return CumulioWriter(config, configured_catalog_with_new_column, logger) @@ -424,9 +399,7 @@ def get_existing_cumulio_columns(stream_name): def test_init_new_airbyte_column(writer_new_airbyte_column: CumulioWriter): """Tests whether each stream is correctly initializing update_metadata (due to new Column in Airbyte for this writer, both are True)""" assert writer_new_airbyte_column.writers["orders_append"]["update_metadata"] is True - assert ( - writer_new_airbyte_column.writers["orders_overwrite"]["update_metadata"] is True - ) + assert writer_new_airbyte_column.writers["orders_overwrite"]["update_metadata"] is True def test_new_column_update_metadata(writer_new_airbyte_column: CumulioWriter): @@ -440,18 +413,13 @@ def test_new_column_appended(writer_new_airbyte_column: CumulioWriter): """Tests whether the Airbyte streams with one new column appends it at the end of the column list""" result = _get_cumulio_and_merged_columns(writer_new_airbyte_column) for stream_name in result: - assert ( - len(result[stream_name]["merged_columns"]) - == len(result[stream_name]["cumulio_columns"]) + 1 - ) + assert len(result[stream_name]["merged_columns"]) == len(result[stream_name]["cumulio_columns"]) + 1 for index, column in enumerate(result[stream_name]["cumulio_columns"]): # Assert that merged_columns are in same order as columns defined on Cumul.io's side. assert result[stream_name]["merged_columns"][index]["name"] == column with pytest.raises(Exception): # Test whether last element of merged_columns is the column that is not defined on Cumul.io's end. - result[stream_name]["cumulio_columns"].index( - result[stream_name]["merged_columns"][-1]["name"] - ) + result[stream_name]["cumulio_columns"].index(result[stream_name]["merged_columns"][-1]["name"]) @pytest.fixture(name="configured_catalog_with_deleted_column") @@ -482,9 +450,7 @@ def configured_catalog_with_deleted_column_fixture() -> ConfiguredAirbyteCatalog destination_sync_mode=DestinationSyncMode.overwrite, ) - return ConfiguredAirbyteCatalog( - streams=[orders_append_stream, orders_overwrite_stream] - ) + return ConfiguredAirbyteCatalog(streams=[orders_append_stream, orders_overwrite_stream]) @pytest.fixture @@ -496,10 +462,7 @@ def writer_deleted_airbyte_column( """This will return a CumulioWriter that mocks airbyte stream catalogs that doesn't contain one column that does exist in Cumul.io.""" existing_cumulio_columns = {} for configured_stream in configured_catalog_with_deleted_column.streams: - columns = [ - column_name - for column_name in configured_stream.stream.json_schema["properties"] - ] + columns = [column_name for column_name in configured_stream.stream.json_schema["properties"]] # Add customer_name column as second element to mimic a deleted column being defined in configured_stream columns.insert(1, "customer_name") existing_cumulio_columns[configured_stream.stream.name] = columns @@ -507,13 +470,9 @@ def writer_deleted_airbyte_column( def get_existing_cumulio_columns(stream_name): return existing_cumulio_columns[stream_name] - with patch( - "destination_cumulio.writer.CumulioClient", MagicMock() - ) as cumulio_client_mock: + with patch("destination_cumulio.writer.CumulioClient", MagicMock()) as cumulio_client_mock: # Mock get_ordered_columns to return existing_cumulio_columns (which does not include one column defined in configured stream) - cumulio_client_mock.return_value.get_ordered_columns = MagicMock( - side_effect=get_existing_cumulio_columns - ) + cumulio_client_mock.return_value.get_ordered_columns = MagicMock(side_effect=get_existing_cumulio_columns) cumulio_client_mock.return_value.batch_writer.return_value = None cumulio_client_mock.return_value.set_replace_tag_on_dataset.return_value = None return CumulioWriter(config, configured_catalog_with_deleted_column, logger) @@ -525,27 +484,15 @@ def test_init_deleted_airbyte_column(writer_deleted_airbyte_column: CumulioWrite - the update_metadata property for the orders dataset is set to False, as it's in append mode and thus should keep existing structure - the update_metadata property for the orders dataset is set to True, as it's in overwrite mode """ - assert ( - writer_deleted_airbyte_column.writers["orders_append"]["update_metadata"] - is False - ) - assert ( - writer_deleted_airbyte_column.writers["orders_overwrite"]["update_metadata"] - is True - ) + assert writer_deleted_airbyte_column.writers["orders_append"]["update_metadata"] is False + assert writer_deleted_airbyte_column.writers["orders_overwrite"]["update_metadata"] is True def test_deleted_column_update_metadata(writer_deleted_airbyte_column: CumulioWriter): """Tests whether Airbyte streams that do not contain a column defined on Cumul.io's side results in update_metadata for only overwrite streams (to inform Cumul.io about new column data being pushed)""" - assert ( - writer_deleted_airbyte_column.writers["orders_append"]["update_metadata"] - is False - ) - assert ( - writer_deleted_airbyte_column.writers["orders_overwrite"]["update_metadata"] - is True - ) + assert writer_deleted_airbyte_column.writers["orders_append"]["update_metadata"] is False + assert writer_deleted_airbyte_column.writers["orders_overwrite"]["update_metadata"] is True def test_merged_columns_order_for_deleted_column( @@ -556,17 +503,10 @@ def test_merged_columns_order_for_deleted_column( result = _get_cumulio_and_merged_columns(writer_deleted_airbyte_column) for stream_name in result: # Test whether merged_columns contains one less element - assert ( - len(result[stream_name]["merged_columns"]) - == len(result[stream_name]["cumulio_columns"]) - 1 - ) + assert len(result[stream_name]["merged_columns"]) == len(result[stream_name]["cumulio_columns"]) - 1 cumulio_columns_without_deleted = [ - column_name - for column_name in result[stream_name]["cumulio_columns"] - if column_name != "customer_name" + column_name for column_name in result[stream_name]["cumulio_columns"] if column_name != "customer_name" ] # Test whether elements, without deleted column, are equal and in the same position - assert cumulio_columns_without_deleted == [ - column["name"] for column in result[stream_name]["merged_columns"] - ] + assert cumulio_columns_without_deleted == [column["name"] for column in result[stream_name]["merged_columns"]] diff --git a/airbyte-integrations/connectors/destination-databend/README.md b/airbyte-integrations/connectors/destination-databend/README.md index 8ef9f9a85c16..9b50cd9ffbfe 100644 --- a/airbyte-integrations/connectors/destination-databend/README.md +++ b/airbyte-integrations/connectors/destination-databend/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:destination-databend:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/destinations/databend) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_databend/spec.json` file. @@ -54,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/destination-databend:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=destination-databend build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:destination-databend:airbyteDocker +An image will be built with the tag `airbyte/destination-databend:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/destination-databend:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,38 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-databend:dev chec # messages.jsonl is a file containing line-separated JSON representing AirbyteMessages cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-databend:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=destination-databend test ``` -#### Acceptance Tests -Coming soon: -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:destination-databend:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:destination-databend:integrationTest -``` +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -116,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-databend test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/databend.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-databend/build.gradle b/airbyte-integrations/connectors/destination-databend/build.gradle deleted file mode 100644 index dd8a2bfb94e1..000000000000 --- a/airbyte-integrations/connectors/destination-databend/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'destination_databend' -} diff --git a/airbyte-integrations/connectors/destination-databend/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-databend/integration_tests/integration_test.py index 906e0429c10c..a40494c4e048 100644 --- a/airbyte-integrations/connectors/destination-databend/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-databend/integration_tests/integration_test.py @@ -62,12 +62,12 @@ def client_fixture(databendConfig) -> DatabendClient: def test_check_valid_config(databendConfig: Mapping): - outcome = DestinationDatabend().check(logging.getLogger('airbyte'), databendConfig) + outcome = DestinationDatabend().check(logging.getLogger("airbyte"), databendConfig) assert outcome.status == Status.SUCCEEDED def test_check_invalid_config(): - outcome = DestinationDatabend().check(logging.getLogger('airbyte'), {"bucket_id": "not_a_real_id"}) + outcome = DestinationDatabend().check(logging.getLogger("airbyte"), {"bucket_id": "not_a_real_id"}) assert outcome.status == Status.FAILED diff --git a/airbyte-integrations/connectors/destination-databend/unit_tests/test_databend_destination.py b/airbyte-integrations/connectors/destination-databend/unit_tests/test_databend_destination.py index d8f221a9836e..e5a7c7e6d7d6 100644 --- a/airbyte-integrations/connectors/destination-databend/unit_tests/test_databend_destination.py +++ b/airbyte-integrations/connectors/destination-databend/unit_tests/test_databend_destination.py @@ -116,14 +116,14 @@ def test_connection(config: Dict[str, str], logger: MagicMock) -> None: @patch("destination_databend.writer.DatabendSQLWriter") @patch("destination_databend.client.DatabendClient") def test_sql_write_append( - mock_connection: MagicMock, - mock_writer: MagicMock, - config: Dict[str, str], - configured_stream1: ConfiguredAirbyteStream, - configured_stream2: ConfiguredAirbyteStream, - airbyte_message1: AirbyteMessage, - airbyte_message2: AirbyteMessage, - airbyte_state_message: AirbyteMessage, + mock_connection: MagicMock, + mock_writer: MagicMock, + config: Dict[str, str], + configured_stream1: ConfiguredAirbyteStream, + configured_stream2: ConfiguredAirbyteStream, + airbyte_message1: AirbyteMessage, + airbyte_message2: AirbyteMessage, + airbyte_state_message: AirbyteMessage, ) -> None: catalog = ConfiguredAirbyteCatalog(streams=[configured_stream1, configured_stream2]) @@ -140,14 +140,14 @@ def test_sql_write_append( @patch("destination_databend.writer.DatabendSQLWriter") @patch("destination_databend.client.DatabendClient") def test_sql_write_overwrite( - mock_connection: MagicMock, - mock_writer: MagicMock, - config: Dict[str, str], - configured_stream1: ConfiguredAirbyteStream, - configured_stream2: ConfiguredAirbyteStream, - airbyte_message1: AirbyteMessage, - airbyte_message2: AirbyteMessage, - airbyte_state_message: AirbyteMessage, + mock_connection: MagicMock, + mock_writer: MagicMock, + config: Dict[str, str], + configured_stream1: ConfiguredAirbyteStream, + configured_stream2: ConfiguredAirbyteStream, + airbyte_message1: AirbyteMessage, + airbyte_message2: AirbyteMessage, + airbyte_state_message: AirbyteMessage, ): # Overwrite triggers a delete configured_stream1.destination_sync_mode = DestinationSyncMode.overwrite diff --git a/airbyte-integrations/connectors/destination-databricks/.dockerignore b/airbyte-integrations/connectors/destination-databricks/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-databricks/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-databricks/Dockerfile b/airbyte-integrations/connectors/destination-databricks/Dockerfile deleted file mode 100644 index 450c77f83318..000000000000 --- a/airbyte-integrations/connectors/destination-databricks/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-databricks - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-databricks - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=1.1.0 -LABEL io.airbyte.name=airbyte/destination-databricks diff --git a/airbyte-integrations/connectors/destination-databricks/README.md b/airbyte-integrations/connectors/destination-databricks/README.md index dfd176be10c8..4f2162f728f2 100644 --- a/airbyte-integrations/connectors/destination-databricks/README.md +++ b/airbyte-integrations/connectors/destination-databricks/README.md @@ -35,10 +35,11 @@ From the Airbyte repository root, run: #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-databricks:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-databricks:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-databricks:dev`. the Dockerfile. #### Run @@ -75,8 +76,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-databricks test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/databricks.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-databricks/build.gradle b/airbyte-integrations/connectors/destination-databricks/build.gradle index 139a62525516..28be23138318 100644 --- a/airbyte-integrations/connectors/destination-databricks/build.gradle +++ b/airbyte-integrations/connectors/destination-databricks/build.gradle @@ -14,24 +14,31 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' id "de.undercouch.download" version "5.0.1" } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.databricks.DatabricksDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:bases:base-java-s3') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - implementation project(':airbyte-integrations:bases:bases-destination-jdbc') implementation project(':airbyte-integrations:connectors:destination-s3') implementation project(':airbyte-integrations:connectors:destination-azure-blob-storage') implementation group: 'com.databricks', name: 'databricks-jdbc', version: '2.6.25' @@ -48,10 +55,6 @@ dependencies { } implementation ('org.apache.parquet:parquet-avro:1.12.0') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} - - implementation 'com.azure:azure-storage-blob:12.18.0' - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-databricks') + implementation 'com.azure:azure-storage-blob:12.18.0' } diff --git a/airbyte-integrations/connectors/destination-databricks/metadata.yaml b/airbyte-integrations/connectors/destination-databricks/metadata.yaml index 0b707d0c110f..8d7eeeb33ee7 100644 --- a/airbyte-integrations/connectors/destination-databricks/metadata.yaml +++ b/airbyte-integrations/connectors/destination-databricks/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 072d5540-f236-4294-ba7c-ade8fd918496 - dockerImageTag: 1.1.0 + dockerImageTag: 1.1.1 dockerRepository: airbyte/destination-databricks githubIssueLabel: destination-databricks icon: databricks.svg @@ -11,8 +11,10 @@ data: registries: cloud: enabled: true + dockerImageTag: 1.1.0 # pinning due to CDK incompatibility, see https://github.com/airbytehq/alpha-beta-issues/issues/2596 oss: enabled: true + dockerImageTag: 1.1.0 # pinning due to CDK incompatibility, see https://github.com/airbytehq/alpha-beta-issues/issues/2596 releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/databricks tags: diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksDestination.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksDestination.java index d5ef1a105b12..f39b1745c579 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksDestination.java +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksDestination.java @@ -4,9 +4,9 @@ package io.airbyte.integrations.destination.databricks; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.destination.jdbc.copy.SwitchingDestination; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.destination.jdbc.copy.SwitchingDestination; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksDestinationResolver.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksDestinationResolver.java index d8c2c71a8966..d11321d838b7 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksDestinationResolver.java +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksDestinationResolver.java @@ -6,7 +6,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; -import io.airbyte.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.Destination; import io.airbyte.integrations.destination.databricks.azure.DatabricksAzureBlobStorageDestination; import io.airbyte.integrations.destination.databricks.s3.DatabricksS3Destination; import io.airbyte.integrations.destination.databricks.utils.DatabricksConstants; diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksExternalStorageBaseDestination.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksExternalStorageBaseDestination.java index 2a0ec84593a4..c193a803c037 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksExternalStorageBaseDestination.java +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksExternalStorageBaseDestination.java @@ -7,14 +7,14 @@ import static io.airbyte.integrations.destination.databricks.utils.DatabricksConstants.DATABRICKS_SCHEMA_KEY; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.copy.CopyConsumerFactory; +import io.airbyte.cdk.integrations.destination.jdbc.copy.CopyDestination; import io.airbyte.integrations.destination.databricks.utils.DatabricksDatabaseUtil; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.copy.CopyConsumerFactory; -import io.airbyte.integrations.destination.jdbc.copy.CopyDestination; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import java.util.function.Consumer; diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksManagedTablesDestination.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksManagedTablesDestination.java index 4658b42e8ffb..2169dcc9dcb9 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksManagedTablesDestination.java +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksManagedTablesDestination.java @@ -5,13 +5,13 @@ package io.airbyte.integrations.destination.databricks; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.destination.databricks.utils.DatabricksConstants; import io.airbyte.integrations.destination.databricks.utils.DatabricksDatabaseUtil; -import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; -import io.airbyte.integrations.destination.jdbc.SqlOperations; import java.util.Collections; import java.util.Map; import javax.sql.DataSource; diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksNameTransformer.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksNameTransformer.java index 1f9491e06353..dc3b0190cb08 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksNameTransformer.java +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksNameTransformer.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.databricks; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; public class DatabricksNameTransformer extends StandardNameTransformer { diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksSqlOperations.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksSqlOperations.java index bba434101a63..526fac52e77d 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksSqlOperations.java +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksSqlOperations.java @@ -4,10 +4,10 @@ package io.airbyte.integrations.destination.databricks; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.jdbc.JdbcSqlOperations; -import io.airbyte.integrations.destination.jdbc.SqlOperationsUtils; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.jdbc.JdbcSqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperationsUtils; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.sql.SQLException; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksStorageConfigProvider.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksStorageConfigProvider.java index a707c0c83dd4..d2e9cc386a9d 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksStorageConfigProvider.java +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksStorageConfigProvider.java @@ -5,11 +5,11 @@ package io.airbyte.integrations.destination.databricks; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.jdbc.copy.azure.AzureBlobStorageConfig; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; import io.airbyte.integrations.destination.databricks.azure.DatabricksAzureBlobStorageConfigProvider; import io.airbyte.integrations.destination.databricks.s3.DatabricksS3StorageConfigProvider; import io.airbyte.integrations.destination.databricks.utils.DatabricksConstants; -import io.airbyte.integrations.destination.jdbc.copy.azure.AzureBlobStorageConfig; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksStreamCopier.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksStreamCopier.java index bfc2278992fe..3f3867c33224 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksStreamCopier.java +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksStreamCopier.java @@ -4,13 +4,13 @@ package io.airbyte.integrations.destination.databricks; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.StagingFilenameGenerator; +import io.airbyte.cdk.integrations.destination.jdbc.constants.GlobalDataSizeConstants; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopier; import io.airbyte.integrations.destination.databricks.utils.DatabricksConstants; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.StagingFilenameGenerator; -import io.airbyte.integrations.destination.jdbc.constants.GlobalDataSizeConstants; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopier; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; import org.slf4j.Logger; diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksStreamCopierFactory.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksStreamCopierFactory.java index 2eba29b6fba7..e39abb862cf9 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksStreamCopierFactory.java +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/DatabricksStreamCopierFactory.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.databricks; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopierFactory; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopierFactory; public interface DatabricksStreamCopierFactory extends StreamCopierFactory { diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/azure/DatabricksAzureBlobStorageConfigProvider.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/azure/DatabricksAzureBlobStorageConfigProvider.java index 6c18103af264..d1ba14bb5056 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/azure/DatabricksAzureBlobStorageConfigProvider.java +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/azure/DatabricksAzureBlobStorageConfigProvider.java @@ -5,8 +5,8 @@ package io.airbyte.integrations.destination.databricks.azure; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.jdbc.copy.azure.AzureBlobStorageConfig; import io.airbyte.integrations.destination.databricks.DatabricksStorageConfigProvider; -import io.airbyte.integrations.destination.jdbc.copy.azure.AzureBlobStorageConfig; public class DatabricksAzureBlobStorageConfigProvider extends DatabricksStorageConfigProvider { diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/azure/DatabricksAzureBlobStorageDestination.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/azure/DatabricksAzureBlobStorageDestination.java index 35209fcdec66..06ae0586f74d 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/azure/DatabricksAzureBlobStorageDestination.java +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/azure/DatabricksAzureBlobStorageDestination.java @@ -4,11 +4,11 @@ package io.airbyte.integrations.destination.databricks.azure; +import io.airbyte.cdk.integrations.destination.jdbc.copy.azure.AzureBlobStorageConfig; import io.airbyte.integrations.destination.azure_blob_storage.AzureBlobStorageConnectionChecker; import io.airbyte.integrations.destination.databricks.DatabricksExternalStorageBaseDestination; import io.airbyte.integrations.destination.databricks.DatabricksStorageConfigProvider; import io.airbyte.integrations.destination.databricks.DatabricksStreamCopierFactory; -import io.airbyte.integrations.destination.jdbc.copy.azure.AzureBlobStorageConfig; public class DatabricksAzureBlobStorageDestination extends DatabricksExternalStorageBaseDestination { diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/azure/DatabricksAzureBlobStorageStreamCopier.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/azure/DatabricksAzureBlobStorageStreamCopier.java index 4614677d4660..9bb35acc33a2 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/azure/DatabricksAzureBlobStorageStreamCopier.java +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/azure/DatabricksAzureBlobStorageStreamCopier.java @@ -9,18 +9,18 @@ import com.azure.storage.blob.specialized.SpecializedBlobClientBuilder; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.copy.azure.AzureBlobStorageConfig; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.integrations.destination.azure_blob_storage.AzureBlobStorageDestinationConfig; import io.airbyte.integrations.destination.azure_blob_storage.AzureBlobStorageFormatConfig; import io.airbyte.integrations.destination.azure_blob_storage.csv.AzureBlobStorageCsvFormatConfig; import io.airbyte.integrations.destination.azure_blob_storage.csv.AzureBlobStorageCsvWriter; import io.airbyte.integrations.destination.databricks.DatabricksDestinationConfig; import io.airbyte.integrations.destination.databricks.DatabricksStreamCopier; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.copy.azure.AzureBlobStorageConfig; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/azure/DatabricksAzureBlobStorageStreamCopierFactory.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/azure/DatabricksAzureBlobStorageStreamCopierFactory.java index fb36b2d8bc95..11958c3f3168 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/azure/DatabricksAzureBlobStorageStreamCopierFactory.java +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/azure/DatabricksAzureBlobStorageStreamCopierFactory.java @@ -5,14 +5,14 @@ package io.airbyte.integrations.destination.databricks.azure; import com.azure.storage.blob.specialized.SpecializedBlobClientBuilder; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopier; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopierFactory; +import io.airbyte.cdk.integrations.destination.jdbc.copy.azure.AzureBlobStorageConfig; import io.airbyte.integrations.destination.databricks.DatabricksDestinationConfig; import io.airbyte.integrations.destination.databricks.DatabricksStreamCopierFactory; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopier; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopierFactory; -import io.airbyte.integrations.destination.jdbc.copy.azure.AzureBlobStorageConfig; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3Destination.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3Destination.java index fd11e3adb0f2..9e4611472084 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3Destination.java +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3Destination.java @@ -4,12 +4,12 @@ package io.airbyte.integrations.destination.databricks.s3; +import io.airbyte.cdk.integrations.destination.s3.S3BaseChecks; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3StorageOperations; import io.airbyte.integrations.destination.databricks.DatabricksExternalStorageBaseDestination; import io.airbyte.integrations.destination.databricks.DatabricksStorageConfigProvider; import io.airbyte.integrations.destination.databricks.DatabricksStreamCopierFactory; -import io.airbyte.integrations.destination.s3.S3BaseChecks; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.S3StorageOperations; public class DatabricksS3Destination extends DatabricksExternalStorageBaseDestination { diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3StorageConfigProvider.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3StorageConfigProvider.java index ad7f171a41d8..51288da8c35f 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3StorageConfigProvider.java +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3StorageConfigProvider.java @@ -6,10 +6,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.constant.S3Constants; +import io.airbyte.cdk.integrations.destination.s3.parquet.S3ParquetFormatConfig; import io.airbyte.integrations.destination.databricks.DatabricksStorageConfigProvider; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.constant.S3Constants; -import io.airbyte.integrations.destination.s3.parquet.S3ParquetFormatConfig; public class DatabricksS3StorageConfigProvider extends DatabricksStorageConfigProvider { diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3StreamCopier.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3StreamCopier.java index 3d4d4773d30c..691ab3b31c23 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3StreamCopier.java +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3StreamCopier.java @@ -8,16 +8,16 @@ import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.ObjectMapper; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopier; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.parquet.S3ParquetFormatConfig; +import io.airbyte.cdk.integrations.destination.s3.parquet.S3ParquetWriter; +import io.airbyte.cdk.integrations.destination.s3.writer.S3WriterFactory; import io.airbyte.integrations.destination.databricks.DatabricksDestinationConfig; import io.airbyte.integrations.destination.databricks.DatabricksStreamCopier; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopier; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.parquet.S3ParquetFormatConfig; -import io.airbyte.integrations.destination.s3.parquet.S3ParquetWriter; -import io.airbyte.integrations.destination.s3.writer.S3WriterFactory; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.sql.Timestamp; diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3StreamCopierFactory.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3StreamCopierFactory.java index cfec8529ea2c..fca486b627e1 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3StreamCopierFactory.java +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3StreamCopierFactory.java @@ -5,15 +5,15 @@ package io.airbyte.integrations.destination.databricks.s3; import com.amazonaws.services.s3.AmazonS3; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopier; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopierFactory; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.writer.ProductionWriterFactory; import io.airbyte.integrations.destination.databricks.DatabricksDestinationConfig; import io.airbyte.integrations.destination.databricks.DatabricksStreamCopierFactory; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopier; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopierFactory; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.writer.ProductionWriterFactory; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.sql.Timestamp; diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/utils/DatabricksConstants.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/utils/DatabricksConstants.java index 04226ddf801f..d5aa8786d7fa 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/utils/DatabricksConstants.java +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/utils/DatabricksConstants.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.databricks.utils; -import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.factory.DatabaseDriver; import java.util.Set; public class DatabricksConstants { diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/utils/DatabricksDatabaseUtil.java b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/utils/DatabricksDatabaseUtil.java index 41564d1820bf..dec6ed05bd38 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/utils/DatabricksDatabaseUtil.java +++ b/airbyte-integrations/connectors/destination-databricks/src/main/java/io/airbyte/integrations/destination/databricks/utils/DatabricksDatabaseUtil.java @@ -10,8 +10,8 @@ import static io.airbyte.integrations.destination.databricks.utils.DatabricksConstants.DATABRICKS_SCHEMA_KEY; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; import io.airbyte.integrations.destination.databricks.DatabricksDestinationConfig; import java.util.HashMap; import java.util.Map; diff --git a/airbyte-integrations/connectors/destination-databricks/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-databricks/src/main/resources/spec.json index 19b74c77a80f..5331b730b258 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-databricks/src/main/resources/spec.json @@ -129,31 +129,40 @@ "description": "The region of the S3 staging bucket to use if utilising a copy strategy.", "enum": [ "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", "af-south-1", "ap-east-1", - "ap-south-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", + "ap-south-1", + "ap-south-2", "ap-southeast-1", "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", "ca-central-1", + "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", + "eu-central-2", "eu-north-1", "eu-south-1", + "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", - "sa-east-1", + "il-central-1", + "me-central-1", "me-south-1", + "sa-east-1", + "sa-east-1", + "us-east-1", + "us-east-2", "us-gov-east-1", - "us-gov-west-1" + "us-gov-west-1", + "us-west-1", + "us-west-2" ], "order": 4 }, diff --git a/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksAzureBlobStorageDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksAzureBlobStorageDestinationAcceptanceTest.java index c7ab5cf11220..3b4943cf2a59 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksAzureBlobStorageDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksAzureBlobStorageDestinationAcceptanceTest.java @@ -13,9 +13,9 @@ import com.azure.storage.blob.specialized.SpecializedBlobClientBuilder; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.destination.jdbc.copy.azure.AzureBlobStorageConfig; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.jdbc.copy.azure.AzureBlobStorageConfig; import java.nio.file.Path; import java.sql.SQLException; import java.util.HashSet; diff --git a/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksDestinationAcceptanceTest.java index d8df20e1e116..d5074dca8ce8 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksDestinationAcceptanceTest.java @@ -8,15 +8,15 @@ import static org.jooq.impl.DSL.field; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopierFactory; +import io.airbyte.cdk.integrations.destination.s3.avro.JsonFieldNameUpdater; +import io.airbyte.cdk.integrations.destination.s3.util.AvroRecordHelper; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopierFactory; -import io.airbyte.integrations.destination.s3.avro.JsonFieldNameUpdater; -import io.airbyte.integrations.destination.s3.util.AvroRecordHelper; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import java.sql.SQLException; import java.util.List; import java.util.stream.Collectors; diff --git a/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksManagedTablesDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksManagedTablesDestinationAcceptanceTest.java index f51008bc7659..b64c9754be22 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksManagedTablesDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksManagedTablesDestinationAcceptanceTest.java @@ -12,13 +12,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopierFactory; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopierFactory; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import java.nio.file.Path; import java.sql.SQLException; import java.util.ArrayList; diff --git a/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksS3DestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksS3DestinationAcceptanceTest.java index 1b714915c222..ca852c7e3ee2 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksS3DestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksS3DestinationAcceptanceTest.java @@ -4,10 +4,10 @@ package io.airbyte.integrations.destination.databricks; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_ACCESS_KEY_ID; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_SECRET_ACCESS_KEY; import static io.airbyte.integrations.destination.databricks.utils.DatabricksConstants.DATABRICKS_DATA_SOURCE_KEY; import static io.airbyte.integrations.destination.databricks.utils.DatabricksConstants.DATABRICKS_SCHEMA_KEY; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_ACCESS_KEY_ID; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_SECRET_ACCESS_KEY; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.DeleteObjectsRequest; @@ -16,9 +16,9 @@ import com.amazonaws.services.s3.model.S3ObjectSummary; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; import java.nio.file.Path; import java.sql.SQLException; import java.util.HashSet; diff --git a/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksUtilTest.java b/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksUtilTest.java index 4aa7382a303b..9e434ef01fd2 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksUtilTest.java +++ b/airbyte-integrations/connectors/destination-databricks/src/test-integration/java/io/airbyte/integrations/destination/databricks/DatabricksUtilTest.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.databricks; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; import io.airbyte.integrations.destination.databricks.utils.DatabricksConstants; import io.airbyte.integrations.destination.databricks.utils.DatabricksDatabaseUtil; import java.sql.SQLException; diff --git a/airbyte-integrations/connectors/destination-databricks/src/test/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3StreamCopierTest.java b/airbyte-integrations/connectors/destination-databricks/src/test/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3StreamCopierTest.java index 147b3ae006b8..754f1a8b87ac 100644 --- a/airbyte-integrations/connectors/destination-databricks/src/test/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3StreamCopierTest.java +++ b/airbyte-integrations/connectors/destination-databricks/src/test/java/io/airbyte/integrations/destination/databricks/s3/DatabricksS3StreamCopierTest.java @@ -6,7 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; import java.util.UUID; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/connectors/destination-dev-null/.dockerignore b/airbyte-integrations/connectors/destination-dev-null/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-dev-null/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-dev-null/Dockerfile b/airbyte-integrations/connectors/destination-dev-null/Dockerfile deleted file mode 100644 index 51c5412192ef..000000000000 --- a/airbyte-integrations/connectors/destination-dev-null/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-dev-null - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-dev-null - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.3.0 -LABEL io.airbyte.name=airbyte/destination-dev-null diff --git a/airbyte-integrations/connectors/destination-dev-null/README.md b/airbyte-integrations/connectors/destination-dev-null/README.md index bfbf20a9cf2c..a0969564d326 100644 --- a/airbyte-integrations/connectors/destination-dev-null/README.md +++ b/airbyte-integrations/connectors/destination-dev-null/README.md @@ -14,10 +14,11 @@ From the Airbyte repository root, run: #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-dev-null:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-dev-null:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-dev-null:dev`. the Dockerfile. #### Run @@ -44,8 +45,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-dev-null test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/e2e-test.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-dev-null/build.gradle b/airbyte-integrations/connectors/destination-dev-null/build.gradle index acf9a2a93545..e167b803db50 100644 --- a/airbyte-integrations/connectors/destination-dev-null/build.gradle +++ b/airbyte-integrations/connectors/destination-dev-null/build.gradle @@ -1,21 +1,30 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.dev_null.DevNullDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:connectors:destination-e2e-test') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-dev-null') } diff --git a/airbyte-integrations/connectors/destination-dev-null/src/main/java/io/airbyte/integrations/destination/dev_null/DevNullDestination.java b/airbyte-integrations/connectors/destination-dev-null/src/main/java/io/airbyte/integrations/destination/dev_null/DevNullDestination.java index cb8ce5f475f9..ed077c41939b 100644 --- a/airbyte-integrations/connectors/destination-dev-null/src/main/java/io/airbyte/integrations/destination/dev_null/DevNullDestination.java +++ b/airbyte-integrations/connectors/destination-dev-null/src/main/java/io/airbyte/integrations/destination/dev_null/DevNullDestination.java @@ -7,10 +7,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.spec_modification.SpecModifyingDestination; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.spec_modification.SpecModifyingDestination; import io.airbyte.integrations.destination.e2e_test.TestingDestinations; import io.airbyte.protocol.models.v0.ConnectorSpecification; import java.util.Iterator; diff --git a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.java index 07df3387a066..2cc8a935d677 100644 --- a/airbyte-integrations/connectors/destination-dev-null/src/test-integration/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-dev-null/src/test-integration/java/io/airbyte/integrations/destination/dev_null/DevNullDestinationAcceptanceTest.java @@ -7,8 +7,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.util.Collections; diff --git a/airbyte-integrations/connectors/destination-doris/.dockerignore b/airbyte-integrations/connectors/destination-doris/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-doris/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-doris/Dockerfile b/airbyte-integrations/connectors/destination-doris/Dockerfile deleted file mode 100644 index b85292c6a93e..000000000000 --- a/airbyte-integrations/connectors/destination-doris/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte -ENV APPLICATION destination-doris - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte -ENV APPLICATION destination-doris - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/destination-doris diff --git a/airbyte-integrations/connectors/destination-doris/README.md b/airbyte-integrations/connectors/destination-doris/README.md index 2bb9f6c170b5..b67c3bd50347 100644 --- a/airbyte-integrations/connectors/destination-doris/README.md +++ b/airbyte-integrations/connectors/destination-doris/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-doris:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-doris:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-doris:dev`. the Dockerfile. #### Run @@ -61,8 +62,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-doris test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/doris.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-doris/build.gradle b/airbyte-integrations/connectors/destination-doris/build.gradle index c3d8bd520b8c..1fe67aaf8ff2 100644 --- a/airbyte-integrations/connectors/destination-doris/build.gradle +++ b/airbyte-integrations/connectors/destination-doris/build.gradle @@ -1,9 +1,23 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.doris.DorisDestination' } @@ -11,11 +25,4 @@ application { dependencies { implementation 'org.apache.commons:commons-csv:1.4' implementation group: 'mysql', name: 'mysql-connector-java', version: '8.0.16' - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-doris') } diff --git a/airbyte-integrations/connectors/destination-doris/src/main/java/io/airbyte/integrations/destination/doris/DorisConsumer.java b/airbyte-integrations/connectors/destination-doris/src/main/java/io/airbyte/integrations/destination/doris/DorisConsumer.java index 49230bb2a04b..db64c82b2a97 100644 --- a/airbyte-integrations/connectors/destination-doris/src/main/java/io/airbyte/integrations/destination/doris/DorisConsumer.java +++ b/airbyte-integrations/connectors/destination-doris/src/main/java/io/airbyte/integrations/destination/doris/DorisConsumer.java @@ -5,8 +5,8 @@ package io.airbyte.integrations.destination.doris; import com.fasterxml.jackson.core.io.JsonStringEncoder; +import io.airbyte.cdk.integrations.base.CommitOnStateAirbyteMessageConsumer; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.CommitOnStateAirbyteMessageConsumer; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/connectors/destination-doris/src/main/java/io/airbyte/integrations/destination/doris/DorisDestination.java b/airbyte-integrations/connectors/destination-doris/src/main/java/io/airbyte/integrations/destination/doris/DorisDestination.java index 659b0aef4802..12fd21b26134 100644 --- a/airbyte-integrations/connectors/destination-doris/src/main/java/io/airbyte/integrations/destination/doris/DorisDestination.java +++ b/airbyte-integrations/connectors/destination-doris/src/main/java/io/airbyte/integrations/destination/doris/DorisDestination.java @@ -8,12 +8,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Preconditions; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-doris/src/test-integration/java/io/airbyte/integrations/destination/doris/DorisDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-doris/src/test-integration/java/io/airbyte/integrations/destination/doris/DorisDestinationAcceptanceTest.java index 1221c9a7a4fe..b2e8dddf8037 100644 --- a/airbyte-integrations/connectors/destination-doris/src/test-integration/java/io/airbyte/integrations/destination/doris/DorisDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-doris/src/test-integration/java/io/airbyte/integrations/destination/doris/DorisDestinationAcceptanceTest.java @@ -5,11 +5,11 @@ package io.airbyte.integrations.destination.doris; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; diff --git a/airbyte-integrations/connectors/destination-doris/src/test/java/io/airbyte/integrations/destination/doris/DorisDestinationTest.java b/airbyte-integrations/connectors/destination-doris/src/test/java/io/airbyte/integrations/destination/doris/DorisDestinationTest.java index 6406cd6cf101..d98a37bf711f 100644 --- a/airbyte-integrations/connectors/destination-doris/src/test/java/io/airbyte/integrations/destination/doris/DorisDestinationTest.java +++ b/airbyte-integrations/connectors/destination-doris/src/test/java/io/airbyte/integrations/destination/doris/DorisDestinationTest.java @@ -11,12 +11,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; diff --git a/airbyte-integrations/connectors/destination-duckdb/.dockerignore b/airbyte-integrations/connectors/destination-duckdb/.dockerignore index 07bca5ba6cb9..eebfe30ec17a 100644 --- a/airbyte-integrations/connectors/destination-duckdb/.dockerignore +++ b/airbyte-integrations/connectors/destination-duckdb/.dockerignore @@ -3,3 +3,6 @@ !main.py !destination_duckdb !setup.py +!pyproject.toml +!poetry.lock +!README.md diff --git a/airbyte-integrations/connectors/destination-duckdb/.gitignore b/airbyte-integrations/connectors/destination-duckdb/.gitignore new file mode 100644 index 000000000000..fbcd3cd78b74 --- /dev/null +++ b/airbyte-integrations/connectors/destination-duckdb/.gitignore @@ -0,0 +1,2 @@ +# Ignore symlinks created within the dev container +.symlinks diff --git a/airbyte-integrations/connectors/destination-duckdb/Dockerfile b/airbyte-integrations/connectors/destination-duckdb/Dockerfile deleted file mode 100644 index 159e8de1de35..000000000000 --- a/airbyte-integrations/connectors/destination-duckdb/Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -FROM python:3.9.11 as base -# FROM python:3.9.11-alpine3.15 as base -# switched from alpine as there were tons of errors (in case you want to switch back to alpine) -# - https://stackoverflow.com/a/57485724/5246670 -# - numpy error: https://stackoverflow.com/a/22411624/5246670 -# - libstdc++ https://github.com/amancevice/docker-pandas/issues/12#issuecomment-717215043 -# - musl-dev linux-headers g++ because of: https://stackoverflow.com/a/40407099/5246670 - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apt-get update && apt-get -y upgrade \ - && pip install --upgrade pip - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . -# build a clean environment -FROM base -# RUN conda install -c conda-forge python-duckdb -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -#adding duckdb manually (outside of setup.py - lots of errors) -RUN pip install duckdb - -# copy payload code only -COPY main.py ./ -COPY destination_duckdb ./destination_duckdb - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/destination-duckdb diff --git a/airbyte-integrations/connectors/destination-duckdb/README.md b/airbyte-integrations/connectors/destination-duckdb/README.md index 71d69e52e39b..b2dfc61e352e 100644 --- a/airbyte-integrations/connectors/destination-duckdb/README.md +++ b/airbyte-integrations/connectors/destination-duckdb/README.md @@ -30,12 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:destination-duckdb:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/destinations/duckdb) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_duckdb/spec.json` file. @@ -56,18 +50,19 @@ cat integration_tests/messages.jsonl| python main.py write --config integration_ ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/destination-duckdb:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=destination-duckdb build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:destination-duckdb:airbyteDocker +An image will be built with the tag `airbyte/destination-duckdb:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/destination-duckdb:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,45 +73,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-duckdb:dev check cat integration_tests/messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-duckdb:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Coming soon: - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:destination-duckdb:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:destination-duckdb:integrationTest -``` - -To run normalization image: -``` -./gradlew :airbyte-integrations:bases:base-normalization:airbyteDockerDuckDb -docker tag airbyte/normalization-duckdb:dev airbyte/normalization-duckdb:0.2.22 +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=destination-duckdb test ``` +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -126,8 +92,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-duckdb test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/duckdb.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-duckdb/acceptance-test-config.yml b/airbyte-integrations/connectors/destination-duckdb/acceptance-test-config.yml new file mode 100644 index 000000000000..c55fdbceee42 --- /dev/null +++ b/airbyte-integrations/connectors/destination-duckdb/acceptance-test-config.yml @@ -0,0 +1,10 @@ +connector_image: airbyte/destination-duckdb:dev +acceptance_tests: + spec: + tests: + - spec_path: integration_tests/spec.json + config_path: "integration_tests/config.json" + connection: + tests: + - config_path: "integration_tests/config.json" + status: "succeed" diff --git a/airbyte-integrations/connectors/destination-duckdb/build.gradle b/airbyte-integrations/connectors/destination-duckdb/build.gradle index fef6aa0dbd77..b623727f32cd 100644 --- a/airbyte-integrations/connectors/destination-duckdb/build.gradle +++ b/airbyte-integrations/connectors/destination-duckdb/build.gradle @@ -1,8 +1,2 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'destination_duckdb' -} +// No-op Gradle build file. Required temporarily due to cross-project dependencies. +// TODO: Delete when no longer needed, per CI testing requirements. diff --git a/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py b/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py index 99a07284c20b..6e3d0882130f 100644 --- a/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py +++ b/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/destination.py @@ -5,6 +5,7 @@ import datetime import json import os +import re import uuid from collections import defaultdict from logging import getLogger @@ -17,6 +18,19 @@ logger = getLogger("airbyte") +CONFIG_MOTHERDUCK_API_KEY = "motherduck_api_key" +CONFIG_DEFAULT_SCHEMA = "main" + + +def validated_sql_name(sql_name: Any) -> str: + """Return the input if it is a valid SQL name, otherwise raise an exception.""" + pattern = r"^[a-zA-Z0-9_]*$" + result = str(sql_name) + if bool(re.match(pattern, result)): + return result + + raise ValueError(f"Invalid SQL name: {sql_name}") + class DestinationDuckdb(Destination): @staticmethod @@ -25,6 +39,9 @@ def _get_destination_path(destination_path: str) -> str: Get a normalized version of the destination path. Automatically append /local/ to the start of the path """ + if destination_path.startswith("md:") or destination_path.startswith("motherduck:"): + return destination_path + if not destination_path.startswith("/local"): destination_path = os.path.join("/local", destination_path) @@ -37,9 +54,11 @@ def _get_destination_path(destination_path: str) -> str: return destination_path def write( - self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] + self, + config: Mapping[str, Any], + configured_catalog: ConfiguredAirbyteCatalog, + input_messages: Iterable[AirbyteMessage], ) -> Iterable[AirbyteMessage]: - """ Reads the input stream of messages, config, and catalog to write data to the destination. @@ -56,33 +75,32 @@ def write( streams = {s.stream.name for s in configured_catalog.streams} logger.info(f"Starting write to DuckDB with {len(streams)} streams") - path = config.get("destination_path") + path = str(config.get("destination_path")) path = self._get_destination_path(path) - # check if file exists + schema_name = validated_sql_name(config.get("schema", CONFIG_DEFAULT_SCHEMA)) + + # Get and register auth token if applicable + motherduck_api_key = str(config.get(CONFIG_MOTHERDUCK_API_KEY, "")) + if motherduck_api_key: + os.environ["motherduck_token"] = motherduck_api_key - logger.info(f"Opening DuckDB file at {path}") con = duckdb.connect(database=path, read_only=False) - # create the tables if needed - # con.execute("BEGIN TRANSACTION") - for configured_stream in configured_catalog.streams: + con.execute(f"CREATE SCHEMA IF NOT EXISTS {schema_name}") + for configured_stream in configured_catalog.streams: name = configured_stream.stream.name table_name = f"_airbyte_raw_{name}" if configured_stream.destination_sync_mode == DestinationSyncMode.overwrite: # delete the tables logger.info(f"Dropping tables for overwrite: {table_name}") - query = """ - DROP TABLE IF EXISTS {} - """.format( - table_name - ) + query = f"DROP TABLE IF EXISTS {schema_name}.{table_name}" con.execute(query) # create the table if needed query = f""" - CREATE TABLE IF NOT EXISTS {table_name} ( + CREATE TABLE IF NOT EXISTS {schema_name}.{table_name} ( _airbyte_ab_id TEXT PRIMARY KEY, - _airbyte_emitted_at JSON, + _airbyte_emitted_at DATETIME, _airbyte_data JSON ) """ @@ -92,20 +110,16 @@ def write( buffer = defaultdict(list) for message in input_messages: - if message.type == Type.STATE: # flush the buffer for stream_name in buffer.keys(): - logger.info(f"flushing buffer for state: {message}") - query = """ - INSERT INTO {table_name} + table_name = f"_airbyte_raw_{stream_name}" + query = f""" + INSERT INTO {schema_name}.{table_name} + (_airbyte_ab_id, _airbyte_emitted_at, _airbyte_data) VALUES (?,?,?) - """.format( - table_name=f"_airbyte_raw_{stream_name}" - ) - logger.info(f"query: {query}") - + """ con.executemany(query, buffer[stream_name]) con.commit() @@ -120,19 +134,23 @@ def write( continue # add to buffer - buffer[stream].append((str(uuid.uuid4()), datetime.datetime.now().isoformat(), json.dumps(data))) + buffer[stream].append( + ( + str(uuid.uuid4()), + datetime.datetime.now().isoformat(), + json.dumps(data), + ) + ) else: logger.info(f"Message type {message.type} not supported, skipping") # flush any remaining messages for stream_name in buffer.keys(): - - query = """ - INSERT INTO {table_name} + table_name = f"_airbyte_raw_{stream_name}" + query = f""" + INSERT INTO {schema_name}.{table_name} VALUES (?,?,?) - """.format( - table_name=f"_airbyte_raw_{stream_name}" - ) + """ con.executemany(query, buffer[stream_name]) con.commit() @@ -150,11 +168,16 @@ def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConn :return: AirbyteConnectionStatus indicating a Success or Failure """ try: - # parse the destination path - param_path = config.get("destination_path") - path = self._get_destination_path(param_path) + path = config.get("destination_path") + path = self._get_destination_path(path) + + if path.startswith("/local"): + logger.info(f"Using DuckDB file at {path}") + os.makedirs(os.path.dirname(path), exist_ok=True) + + if CONFIG_MOTHERDUCK_API_KEY in config: + os.environ["motherduck_token"] = str(config[CONFIG_MOTHERDUCK_API_KEY]) - os.makedirs(os.path.dirname(path), exist_ok=True) con = duckdb.connect(database=path, read_only=False) con.execute("SELECT 1;") diff --git a/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/spec.json b/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/spec.json index 2517ac9ed772..511dd3aa4482 100644 --- a/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/spec.json +++ b/airbyte-integrations/connectors/destination-duckdb/destination_duckdb/spec.json @@ -1,9 +1,5 @@ { "documentationUrl": "https://docs.airbyte.io/integrations/destinations/duckdb", - "supported_destination_sync_modes": ["overwrite", "append"], - "supportsIncremental": true, - "supportsDBT": true, - "supportsNormalization": true, "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Destination Duckdb", @@ -11,16 +7,28 @@ "required": ["destination_path"], "additionalProperties": true, "properties": { + "motherduck_api_key": { + "title": "MotherDuck API Key", + "type": "string", + "description": "API key to use for authentication to a MotherDuck database.", + "airbyte_secret": true + }, "destination_path": { + "title": "Destination DB", "type": "string", - "description": "Path to the .duckdb file. The file will be placed inside that local mount. For more information check out our docs", - "example": "/local/destination.duckdb" + "description": "Path to the .duckdb file, or the text 'md:' to connect to MotherDuck. The file will be placed inside that local mount. For more information check out our docs", + "examples": ["/local/destination.duckdb", "md:", "motherduck:"] }, "schema": { + "title": "Destination Schema", "type": "string", - "description": "database schema, default for duckdb is main", + "description": "Database schema name, default for duckdb is 'main'.", "example": "main" } } - } + }, + "supportsIncremental": true, + "supportsNormalization": true, + "supportsDBT": true, + "supported_destination_sync_modes": ["overwrite", "append"] } diff --git a/airbyte-integrations/connectors/destination-duckdb/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-duckdb/integration_tests/integration_test.py index 255913dbd24a..e15a0b26136f 100644 --- a/airbyte-integrations/connectors/destination-duckdb/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-duckdb/integration_tests/integration_test.py @@ -2,12 +2,15 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from __future__ import annotations + import json import os import random import string import tempfile from datetime import datetime +from pathlib import Path from typing import Any, Dict from unittest.mock import MagicMock @@ -27,23 +30,58 @@ ) from destination_duckdb import DestinationDuckdb +CONFIG_PATH = "integration_tests/config.json" +SECRETS_CONFIG_PATH = "secrets/config.json" # Should contain a valid MotherDuck API token -@pytest.fixture(autouse=True) -def disable_destination_modification(monkeypatch, request): - if "disable_autouse" in request.keywords: + +def pytest_generate_tests(metafunc): + if "config" not in metafunc.fixturenames: return + + configs: list[str] = ["local_file_config"] + if Path(SECRETS_CONFIG_PATH).is_file(): + configs.append("motherduck_config") else: - monkeypatch.setattr(DestinationDuckdb, "_get_destination_path", lambda _, x: x) + print(f"Skipping MotherDuck tests because config file not found at: {SECRETS_CONFIG_PATH}") + + # for test_name in ["test_check_succeeds", "test_write"]: + metafunc.parametrize("config", configs, indirect=True) @pytest.fixture(scope="module") -def local_file_config() -> Dict[str, str]: - # create a file "myfile" in "mydir" in temp directory - tmp_dir = tempfile.TemporaryDirectory() - test = os.path.join(str(tmp_dir), "test.duckdb") +def test_schema_name() -> str: + letters = string.ascii_lowercase + rand_string = "".join(random.choice(letters) for _ in range(6)) + return f"test_schema_{rand_string}" + + +@pytest.fixture +def config(request, test_schema_name: str) -> Dict[str, str]: + if request.param == "local_file_config": + tmp_dir = tempfile.TemporaryDirectory() + test = os.path.join(str(tmp_dir.name), "test.duckdb") + yield {"destination_path": test, "schema": test_schema_name} + + elif request.param == "motherduck_config": + config_dict = json.loads(Path(SECRETS_CONFIG_PATH).read_text()) + config_dict["schema"] = test_schema_name + yield config_dict + + else: + raise ValueError(f"Unknown config type: {request.param}") + + +@pytest.fixture +def invalid_config() -> Dict[str, str]: + return {"destination_path": "/destination.duckdb"} + - # f1.write_text("text to myfile") - yield {"destination_path": test} +@pytest.fixture(autouse=True) +def disable_destination_modification(monkeypatch, request): + if "disable_autouse" in request.keywords: + return + else: + monkeypatch.setattr(DestinationDuckdb, "_get_destination_path", lambda _, x: x) @pytest.fixture(scope="module") @@ -63,7 +101,9 @@ def table_schema() -> str: def configured_catalogue(test_table_name: str, table_schema: str) -> ConfiguredAirbyteCatalog: append_stream = ConfiguredAirbyteStream( stream=AirbyteStream( - name=test_table_name, json_schema=table_schema, supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental] + name=test_table_name, + json_schema=table_schema, + supported_sync_modes=[SyncMode.full_refresh, SyncMode.incremental], ), sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.append, @@ -71,17 +111,14 @@ def configured_catalogue(test_table_name: str, table_schema: str) -> ConfiguredA return ConfiguredAirbyteCatalog(streams=[append_stream]) -@pytest.fixture -def invalid_config() -> Dict[str, str]: - return {"destination_path": "/destination.duckdb"} - - @pytest.fixture def airbyte_message1(test_table_name: str): return AirbyteMessage( type=Type.RECORD, record=AirbyteRecordMessage( - stream=test_table_name, data={"key1": "value1", "key2": 3}, emitted_at=int(datetime.now().timestamp()) * 1000 + stream=test_table_name, + data={"key1": "value1", "key2": 3}, + emitted_at=int(datetime.now().timestamp()) * 1000, ), ) @@ -91,7 +128,9 @@ def airbyte_message2(test_table_name: str): return AirbyteMessage( type=Type.RECORD, record=AirbyteRecordMessage( - stream=test_table_name, data={"key1": "value2", "key2": 2}, emitted_at=int(datetime.now().timestamp()) * 1000 + stream=test_table_name, + data={"key1": "value2", "key2": 2}, + emitted_at=int(datetime.now().timestamp()) * 1000, ), ) @@ -101,18 +140,17 @@ def airbyte_message3(): return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data={"state": "1"})) -@pytest.mark.parametrize("config", ["invalid_config"]) @pytest.mark.disable_autouse -def test_check_fails(config, request): - config = request.getfixturevalue(config) +def test_check_fails(invalid_config, request): destination = DestinationDuckdb() - status = destination.check(logger=MagicMock(), config=config) + status = destination.check(logger=MagicMock(), config=invalid_config) assert status.status == Status.FAILED -@pytest.mark.parametrize("config", ["local_file_config"]) -def test_check_succeeds(config, request): - config = request.getfixturevalue(config) +def test_check_succeeds( + config: dict[str, str], + request, +): destination = DestinationDuckdb() status = destination.check(logger=MagicMock(), config=config) assert status.status == Status.SUCCEEDED @@ -122,7 +160,6 @@ def _state(data: Dict[str, Any]) -> AirbyteMessage: return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data=data)) -@pytest.mark.parametrize("config", ["local_file_config"]) def test_write( config: Dict[str, str], request, @@ -131,10 +168,14 @@ def test_write( airbyte_message2: AirbyteMessage, airbyte_message3: AirbyteMessage, test_table_name: str, + test_schema_name: str, ): - config = request.getfixturevalue(config) destination = DestinationDuckdb() - generator = destination.write(config, configured_catalogue, [airbyte_message1, airbyte_message2, airbyte_message3]) + generator = destination.write( + config, + configured_catalogue, + [airbyte_message1, airbyte_message2, airbyte_message3], + ) result = list(generator) assert len(result) == 1 @@ -142,7 +183,8 @@ def test_write( con = duckdb.connect(database=config.get("destination_path"), read_only=False) with con: cursor = con.execute( - f"SELECT _airbyte_ab_id, _airbyte_emitted_at, _airbyte_data FROM _airbyte_raw_{test_table_name} ORDER BY _airbyte_data" + "SELECT _airbyte_ab_id, _airbyte_emitted_at, _airbyte_data " + f"FROM {test_schema_name}._airbyte_raw_{test_table_name} ORDER BY _airbyte_data" ) result = cursor.fetchall() diff --git a/airbyte-integrations/connectors/destination-duckdb/integration_tests/spec.json b/airbyte-integrations/connectors/destination-duckdb/integration_tests/spec.json new file mode 100644 index 000000000000..511dd3aa4482 --- /dev/null +++ b/airbyte-integrations/connectors/destination-duckdb/integration_tests/spec.json @@ -0,0 +1,34 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/destinations/duckdb", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Destination Duckdb", + "type": "object", + "required": ["destination_path"], + "additionalProperties": true, + "properties": { + "motherduck_api_key": { + "title": "MotherDuck API Key", + "type": "string", + "description": "API key to use for authentication to a MotherDuck database.", + "airbyte_secret": true + }, + "destination_path": { + "title": "Destination DB", + "type": "string", + "description": "Path to the .duckdb file, or the text 'md:' to connect to MotherDuck. The file will be placed inside that local mount. For more information check out our docs", + "examples": ["/local/destination.duckdb", "md:", "motherduck:"] + }, + "schema": { + "title": "Destination Schema", + "type": "string", + "description": "Database schema name, default for duckdb is 'main'.", + "example": "main" + } + } + }, + "supportsIncremental": true, + "supportsNormalization": true, + "supportsDBT": true, + "supported_destination_sync_modes": ["overwrite", "append"] +} diff --git a/airbyte-integrations/connectors/destination-duckdb/metadata.yaml b/airbyte-integrations/connectors/destination-duckdb/metadata.yaml index 3d751bfe1f80..1064593eb19f 100644 --- a/airbyte-integrations/connectors/destination-duckdb/metadata.yaml +++ b/airbyte-integrations/connectors/destination-duckdb/metadata.yaml @@ -1,8 +1,10 @@ data: + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: database connectorType: destination definitionId: 94bd199c-2ff0-4aa2-b98e-17f0acb72610 - dockerImageTag: 0.1.0 + dockerImageTag: 0.3.1 dockerRepository: airbyte/destination-duckdb githubIssueLabel: destination-duckdb icon: duckdb.svg @@ -10,10 +12,15 @@ data: name: DuckDB registries: cloud: - enabled: false + enabled: true oss: enabled: true releaseStage: alpha + releases: + breakingChanges: + 0.3.0: + message: "This version uses the DuckDB 0.9.1 database driver, which is not backwards compatible with prior versions. MotherDuck users can upgrade their database by visiting https://app.motherduck.com/ and accepting the upgrade. For more information, see the connector migration guide." + upgradeDeadline: "2023-10-31" documentationUrl: https://docs.airbyte.com/integrations/destinations/duckdb tags: - language:python diff --git a/airbyte-integrations/connectors/destination-duckdb/poetry.lock b/airbyte-integrations/connectors/destination-duckdb/poetry.lock new file mode 100644 index 000000000000..7562101a90b4 --- /dev/null +++ b/airbyte-integrations/connectors/destination-duckdb/poetry.lock @@ -0,0 +1,1134 @@ +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.51.44" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.51.44.tar.gz", hash = "sha256:faf3c28f19d7ad74d61355d4c289a10a4de3a2947b8330a8050ca48e4133a657"}, + {file = "airbyte_cdk-0.51.44-py3-none-any.whl", hash = "sha256:8e32e1d947057f9504994b5a591ab76a7dae4132c8b794cd82c5fc6b9c34364e"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.4.2" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "*" +pydantic = ">=1.10.8,<2.0.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pyarrow (==12.0.1)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "pyarrow (==12.0.1)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.4.2" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.4.2-py3-none-any.whl", hash = "sha256:d3bbb14d4af9483bd7b08f5eb06f87e7113553bf4baed3998af95be873a0d821"}, + {file = "airbyte_protocol_models-0.4.2.tar.gz", hash = "sha256:67b149d4812f8fdb88396b161274aa73cf0e16f22e35ce44f2bfc4d47e51915c"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "attrs" +version = "23.1.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.1.0-py3-none-any.whl", hash = "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04"}, + {file = "attrs-23.1.0.tar.gz", hash = "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[docs,tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "1.11.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "backoff-1.11.1-py2.py3-none-any.whl", hash = "sha256:61928f8fa48d52e4faa81875eecf308eccfb1016b018bb6bd21e05b5d90a96c5"}, + {file = "backoff-1.11.1.tar.gz", hash = "sha256:ccb962a2378418c667b3c979b504fdeb7d9e0d29c0579e3b13b86467177728cb"}, +] + +[[package]] +name = "black" +version = "23.11.0" +description = "The uncompromising code formatter." +optional = false +python-versions = ">=3.8" +files = [ + {file = "black-23.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dbea0bb8575c6b6303cc65017b46351dc5953eea5c0a59d7b7e3a2d2f433a911"}, + {file = "black-23.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:412f56bab20ac85927f3a959230331de5614aecda1ede14b373083f62ec24e6f"}, + {file = "black-23.11.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d136ef5b418c81660ad847efe0e55c58c8208b77a57a28a503a5f345ccf01394"}, + {file = "black-23.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:6c1cac07e64433f646a9a838cdc00c9768b3c362805afc3fce341af0e6a9ae9f"}, + {file = "black-23.11.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cf57719e581cfd48c4efe28543fea3d139c6b6f1238b3f0102a9c73992cbb479"}, + {file = "black-23.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:698c1e0d5c43354ec5d6f4d914d0d553a9ada56c85415700b81dc90125aac244"}, + {file = "black-23.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:760415ccc20f9e8747084169110ef75d545f3b0932ee21368f63ac0fee86b221"}, + {file = "black-23.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:58e5f4d08a205b11800332920e285bd25e1a75c54953e05502052738fe16b3b5"}, + {file = "black-23.11.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:45aa1d4675964946e53ab81aeec7a37613c1cb71647b5394779e6efb79d6d187"}, + {file = "black-23.11.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c44b7211a3a0570cc097e81135faa5f261264f4dfaa22bd5ee2875a4e773bd6"}, + {file = "black-23.11.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a9acad1451632021ee0d146c8765782a0c3846e0e0ea46659d7c4f89d9b212b"}, + {file = "black-23.11.0-cp38-cp38-win_amd64.whl", hash = "sha256:fc7f6a44d52747e65a02558e1d807c82df1d66ffa80a601862040a43ec2e3142"}, + {file = "black-23.11.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7f622b6822f02bfaf2a5cd31fdb7cd86fcf33dab6ced5185c35f5db98260b055"}, + {file = "black-23.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:250d7e60f323fcfc8ea6c800d5eba12f7967400eb6c2d21ae85ad31c204fb1f4"}, + {file = "black-23.11.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5133f5507007ba08d8b7b263c7aa0f931af5ba88a29beacc4b2dc23fcefe9c06"}, + {file = "black-23.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:421f3e44aa67138ab1b9bfbc22ee3780b22fa5b291e4db8ab7eee95200726b07"}, + {file = "black-23.11.0-py3-none-any.whl", hash = "sha256:54caaa703227c6e0c87b76326d0862184729a69b73d3b7305b6288e1d830067e"}, + {file = "black-23.11.0.tar.gz", hash = "sha256:4c68855825ff432d197229846f971bc4d6666ce90492e5b02013bcaca4d9ab05"}, +] + +[package.dependencies] +click = ">=8.0.0" +mypy-extensions = ">=0.4.3" +packaging = ">=22.0" +pathspec = ">=0.9.0" +platformdirs = ">=2" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.7.4)"] +jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] +uvloop = ["uvloop (>=0.15.2)"] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "certifi" +version = "2023.7.22" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2023.7.22-py3-none-any.whl", hash = "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"}, + {file = "certifi-2023.7.22.tar.gz", hash = "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "click" +version = "8.1.7" +description = "Composable command line interface toolkit" +optional = false +python-versions = ">=3.7" +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "duckdb" +version = "0.9.2" +description = "DuckDB embedded database" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "duckdb-0.9.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:aadcea5160c586704c03a8a796c06a8afffbefefb1986601104a60cb0bfdb5ab"}, + {file = "duckdb-0.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:08215f17147ed83cbec972175d9882387366de2ed36c21cbe4add04b39a5bcb4"}, + {file = "duckdb-0.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ee6c2a8aba6850abef5e1be9dbc04b8e72a5b2c2b67f77892317a21fae868fe7"}, + {file = "duckdb-0.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ff49f3da9399900fd58b5acd0bb8bfad22c5147584ad2427a78d937e11ec9d0"}, + {file = "duckdb-0.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd5ac5baf8597efd2bfa75f984654afcabcd698342d59b0e265a0bc6f267b3f0"}, + {file = "duckdb-0.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:81c6df905589a1023a27e9712edb5b724566587ef280a0c66a7ec07c8083623b"}, + {file = "duckdb-0.9.2-cp310-cp310-win32.whl", hash = "sha256:a298cd1d821c81d0dec8a60878c4b38c1adea04a9675fb6306c8f9083bbf314d"}, + {file = "duckdb-0.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:492a69cd60b6cb4f671b51893884cdc5efc4c3b2eb76057a007d2a2295427173"}, + {file = "duckdb-0.9.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:061a9ea809811d6e3025c5de31bc40e0302cfb08c08feefa574a6491e882e7e8"}, + {file = "duckdb-0.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a43f93be768af39f604b7b9b48891f9177c9282a408051209101ff80f7450d8f"}, + {file = "duckdb-0.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ac29c8c8f56fff5a681f7bf61711ccb9325c5329e64f23cb7ff31781d7b50773"}, + {file = "duckdb-0.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b14d98d26bab139114f62ade81350a5342f60a168d94b27ed2c706838f949eda"}, + {file = "duckdb-0.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:796a995299878913e765b28cc2b14c8e44fae2f54ab41a9ee668c18449f5f833"}, + {file = "duckdb-0.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6cb64ccfb72c11ec9c41b3cb6181b6fd33deccceda530e94e1c362af5f810ba1"}, + {file = "duckdb-0.9.2-cp311-cp311-win32.whl", hash = "sha256:930740cb7b2cd9e79946e1d3a8f66e15dc5849d4eaeff75c8788d0983b9256a5"}, + {file = "duckdb-0.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:c28f13c45006fd525001b2011cdf91fa216530e9751779651e66edc0e446be50"}, + {file = "duckdb-0.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fbce7bbcb4ba7d99fcec84cec08db40bc0dd9342c6c11930ce708817741faeeb"}, + {file = "duckdb-0.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:15a82109a9e69b1891f0999749f9e3265f550032470f51432f944a37cfdc908b"}, + {file = "duckdb-0.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9490fb9a35eb74af40db5569d90df8a04a6f09ed9a8c9caa024998c40e2506aa"}, + {file = "duckdb-0.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:696d5c6dee86c1a491ea15b74aafe34ad2b62dcd46ad7e03b1d00111ca1a8c68"}, + {file = "duckdb-0.9.2-cp37-cp37m-win32.whl", hash = "sha256:4f0935300bdf8b7631ddfc838f36a858c1323696d8c8a2cecbd416bddf6b0631"}, + {file = "duckdb-0.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:0aab900f7510e4d2613263865570203ddfa2631858c7eb8cbed091af6ceb597f"}, + {file = "duckdb-0.9.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:7d8130ed6a0c9421b135d0743705ea95b9a745852977717504e45722c112bf7a"}, + {file = "duckdb-0.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:974e5de0294f88a1a837378f1f83330395801e9246f4e88ed3bfc8ada65dcbee"}, + {file = "duckdb-0.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4fbc297b602ef17e579bb3190c94d19c5002422b55814421a0fc11299c0c1100"}, + {file = "duckdb-0.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1dd58a0d84a424924a35b3772419f8cd78a01c626be3147e4934d7a035a8ad68"}, + {file = "duckdb-0.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11a1194a582c80dfb57565daa06141727e415ff5d17e022dc5f31888a5423d33"}, + {file = "duckdb-0.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:be45d08541002a9338e568dca67ab4f20c0277f8f58a73dfc1435c5b4297c996"}, + {file = "duckdb-0.9.2-cp38-cp38-win32.whl", hash = "sha256:dd6f88aeb7fc0bfecaca633629ff5c986ac966fe3b7dcec0b2c48632fd550ba2"}, + {file = "duckdb-0.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:28100c4a6a04e69aa0f4a6670a6d3d67a65f0337246a0c1a429f3f28f3c40b9a"}, + {file = "duckdb-0.9.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7ae5bf0b6ad4278e46e933e51473b86b4b932dbc54ff097610e5b482dd125552"}, + {file = "duckdb-0.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e5d0bb845a80aa48ed1fd1d2d285dd352e96dc97f8efced2a7429437ccd1fe1f"}, + {file = "duckdb-0.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ce262d74a52500d10888110dfd6715989926ec936918c232dcbaddb78fc55b4"}, + {file = "duckdb-0.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6935240da090a7f7d2666f6d0a5e45ff85715244171ca4e6576060a7f4a1200e"}, + {file = "duckdb-0.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5cfb93e73911696a98b9479299d19cfbc21dd05bb7ab11a923a903f86b4d06e"}, + {file = "duckdb-0.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:64e3bc01751f31e7572d2716c3e8da8fe785f1cdc5be329100818d223002213f"}, + {file = "duckdb-0.9.2-cp39-cp39-win32.whl", hash = "sha256:6e5b80f46487636368e31b61461940e3999986359a78660a50dfdd17dd72017c"}, + {file = "duckdb-0.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:e6142a220180dbeea4f341708bd5f9501c5c962ce7ef47c1cadf5e8810b4cb13"}, + {file = "duckdb-0.9.2.tar.gz", hash = "sha256:3843afeab7c3fc4a4c0b53686a4cc1d9cdbdadcbb468d60fef910355ecafd447"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.1.3" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.1.3-py3-none-any.whl", hash = "sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3"}, + {file = "exceptiongroup-1.1.3.tar.gz", hash = "sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "itsdangerous" +version = "2.1.2" +description = "Safely pass data to untrusted environments and back." +optional = false +python-versions = ">=3.7" +files = [ + {file = "itsdangerous-2.1.2-py3-none-any.whl", hash = "sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44"}, + {file = "itsdangerous-2.1.2.tar.gz", hash = "sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a"}, +] + +[[package]] +name = "jinja2" +version = "3.1.2" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, + {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.2" +description = "An implementation of JSON Reference for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonref-0.2-py3-none-any.whl", hash = "sha256:b1e82fa0b62e2c2796a13e5401fe51790b248f6d9bf9d7212a3e31a3501b291f"}, + {file = "jsonref-0.2.tar.gz", hash = "sha256:f3c45b121cf6257eafabdc3a8008763aed1cd7da06dbabc59a9e4d2a5e4e6697"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.3" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win32.whl", hash = "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win32.whl", hash = "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win_amd64.whl", hash = "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win32.whl", hash = "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win_amd64.whl", hash = "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win32.whl", hash = "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba"}, + {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, +] + +[[package]] +name = "mypy" +version = "1.7.0" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mypy-1.7.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5da84d7bf257fd8f66b4f759a904fd2c5a765f70d8b52dde62b521972a0a2357"}, + {file = "mypy-1.7.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a3637c03f4025f6405737570d6cbfa4f1400eb3c649317634d273687a09ffc2f"}, + {file = "mypy-1.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b633f188fc5ae1b6edca39dae566974d7ef4e9aaaae00bc36efe1f855e5173ac"}, + {file = "mypy-1.7.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d6ed9a3997b90c6f891138e3f83fb8f475c74db4ccaa942a1c7bf99e83a989a1"}, + {file = "mypy-1.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:1fe46e96ae319df21359c8db77e1aecac8e5949da4773c0274c0ef3d8d1268a9"}, + {file = "mypy-1.7.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:df67fbeb666ee8828f675fee724cc2cbd2e4828cc3df56703e02fe6a421b7401"}, + {file = "mypy-1.7.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a79cdc12a02eb526d808a32a934c6fe6df07b05f3573d210e41808020aed8b5d"}, + {file = "mypy-1.7.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f65f385a6f43211effe8c682e8ec3f55d79391f70a201575def73d08db68ead1"}, + {file = "mypy-1.7.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0e81ffd120ee24959b449b647c4b2fbfcf8acf3465e082b8d58fd6c4c2b27e46"}, + {file = "mypy-1.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:f29386804c3577c83d76520abf18cfcd7d68264c7e431c5907d250ab502658ee"}, + {file = "mypy-1.7.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:87c076c174e2c7ef8ab416c4e252d94c08cd4980a10967754f91571070bf5fbe"}, + {file = "mypy-1.7.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6cb8d5f6d0fcd9e708bb190b224089e45902cacef6f6915481806b0c77f7786d"}, + {file = "mypy-1.7.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d93e76c2256aa50d9c82a88e2f569232e9862c9982095f6d54e13509f01222fc"}, + {file = "mypy-1.7.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cddee95dea7990e2215576fae95f6b78a8c12f4c089d7e4367564704e99118d3"}, + {file = "mypy-1.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:d01921dbd691c4061a3e2ecdbfbfad029410c5c2b1ee88946bf45c62c6c91210"}, + {file = "mypy-1.7.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:185cff9b9a7fec1f9f7d8352dff8a4c713b2e3eea9c6c4b5ff7f0edf46b91e41"}, + {file = "mypy-1.7.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7a7b1e399c47b18feb6f8ad4a3eef3813e28c1e871ea7d4ea5d444b2ac03c418"}, + {file = "mypy-1.7.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc9fe455ad58a20ec68599139ed1113b21f977b536a91b42bef3ffed5cce7391"}, + {file = "mypy-1.7.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d0fa29919d2e720c8dbaf07d5578f93d7b313c3e9954c8ec05b6d83da592e5d9"}, + {file = "mypy-1.7.0-cp38-cp38-win_amd64.whl", hash = "sha256:2b53655a295c1ed1af9e96b462a736bf083adba7b314ae775563e3fb4e6795f5"}, + {file = "mypy-1.7.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c1b06b4b109e342f7dccc9efda965fc3970a604db70f8560ddfdee7ef19afb05"}, + {file = "mypy-1.7.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:bf7a2f0a6907f231d5e41adba1a82d7d88cf1f61a70335889412dec99feeb0f8"}, + {file = "mypy-1.7.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:551d4a0cdcbd1d2cccdcc7cb516bb4ae888794929f5b040bb51aae1846062901"}, + {file = "mypy-1.7.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:55d28d7963bef00c330cb6461db80b0b72afe2f3c4e2963c99517cf06454e665"}, + {file = "mypy-1.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:870bd1ffc8a5862e593185a4c169804f2744112b4a7c55b93eb50f48e7a77010"}, + {file = "mypy-1.7.0-py3-none-any.whl", hash = "sha256:96650d9a4c651bc2a4991cf46f100973f656d69edc7faf91844e87fe627f7e96"}, + {file = "mypy-1.7.0.tar.gz", hash = "sha256:1e280b5697202efa698372d2f39e9a6713a0395a756b1c6bd48995f8d72690dc"}, +] + +[package.dependencies] +mypy-extensions = ">=1.0.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=4.1.0" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pathspec" +version = "0.11.2" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pathspec-0.11.2-py3-none-any.whl", hash = "sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20"}, + {file = "pathspec-0.11.2.tar.gz", hash = "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3"}, +] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "4.0.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.7" +files = [ + {file = "platformdirs-4.0.0-py3-none-any.whl", hash = "sha256:118c954d7e949b35437270383a3f2531e99dd93cf7ce4dc8340d3356d30f173b"}, + {file = "platformdirs-4.0.0.tar.gz", hash = "sha256:cb633b2bcf10c51af60beb0ab06d2f1d69064b43abf4c185ca6b28865f3f9731"}, +] + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] + +[[package]] +name = "pluggy" +version = "1.3.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, + {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pydantic" +version = "1.10.13" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.13-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:efff03cc7a4f29d9009d1c96ceb1e7a70a65cfe86e89d34e4a5f2ab1e5693737"}, + {file = "pydantic-1.10.13-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3ecea2b9d80e5333303eeb77e180b90e95eea8f765d08c3d278cd56b00345d01"}, + {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1740068fd8e2ef6eb27a20e5651df000978edce6da6803c2bef0bc74540f9548"}, + {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84bafe2e60b5e78bc64a2941b4c071a4b7404c5c907f5f5a99b0139781e69ed8"}, + {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bc0898c12f8e9c97f6cd44c0ed70d55749eaf783716896960b4ecce2edfd2d69"}, + {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:654db58ae399fe6434e55325a2c3e959836bd17a6f6a0b6ca8107ea0571d2e17"}, + {file = "pydantic-1.10.13-cp310-cp310-win_amd64.whl", hash = "sha256:75ac15385a3534d887a99c713aa3da88a30fbd6204a5cd0dc4dab3d770b9bd2f"}, + {file = "pydantic-1.10.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c553f6a156deb868ba38a23cf0df886c63492e9257f60a79c0fd8e7173537653"}, + {file = "pydantic-1.10.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5e08865bc6464df8c7d61439ef4439829e3ab62ab1669cddea8dd00cd74b9ffe"}, + {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e31647d85a2013d926ce60b84f9dd5300d44535a9941fe825dc349ae1f760df9"}, + {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:210ce042e8f6f7c01168b2d84d4c9eb2b009fe7bf572c2266e235edf14bacd80"}, + {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8ae5dd6b721459bfa30805f4c25880e0dd78fc5b5879f9f7a692196ddcb5a580"}, + {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f8e81fc5fb17dae698f52bdd1c4f18b6ca674d7068242b2aff075f588301bbb0"}, + {file = "pydantic-1.10.13-cp311-cp311-win_amd64.whl", hash = "sha256:61d9dce220447fb74f45e73d7ff3b530e25db30192ad8d425166d43c5deb6df0"}, + {file = "pydantic-1.10.13-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4b03e42ec20286f052490423682016fd80fda830d8e4119f8ab13ec7464c0132"}, + {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59ef915cac80275245824e9d771ee939133be38215555e9dc90c6cb148aaeb5"}, + {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a1f9f747851338933942db7af7b6ee8268568ef2ed86c4185c6ef4402e80ba8"}, + {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:97cce3ae7341f7620a0ba5ef6cf043975cd9d2b81f3aa5f4ea37928269bc1b87"}, + {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854223752ba81e3abf663d685f105c64150873cc6f5d0c01d3e3220bcff7d36f"}, + {file = "pydantic-1.10.13-cp37-cp37m-win_amd64.whl", hash = "sha256:b97c1fac8c49be29486df85968682b0afa77e1b809aff74b83081cc115e52f33"}, + {file = "pydantic-1.10.13-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c958d053453a1c4b1c2062b05cd42d9d5c8eb67537b8d5a7e3c3032943ecd261"}, + {file = "pydantic-1.10.13-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c5370a7edaac06daee3af1c8b1192e305bc102abcbf2a92374b5bc793818599"}, + {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d6f6e7305244bddb4414ba7094ce910560c907bdfa3501e9db1a7fd7eaea127"}, + {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3a3c792a58e1622667a2837512099eac62490cdfd63bd407993aaf200a4cf1f"}, + {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c636925f38b8db208e09d344c7aa4f29a86bb9947495dd6b6d376ad10334fb78"}, + {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:678bcf5591b63cc917100dc50ab6caebe597ac67e8c9ccb75e698f66038ea953"}, + {file = "pydantic-1.10.13-cp38-cp38-win_amd64.whl", hash = "sha256:6cf25c1a65c27923a17b3da28a0bdb99f62ee04230c931d83e888012851f4e7f"}, + {file = "pydantic-1.10.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8ef467901d7a41fa0ca6db9ae3ec0021e3f657ce2c208e98cd511f3161c762c6"}, + {file = "pydantic-1.10.13-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:968ac42970f57b8344ee08837b62f6ee6f53c33f603547a55571c954a4225691"}, + {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9849f031cf8a2f0a928fe885e5a04b08006d6d41876b8bbd2fc68a18f9f2e3fd"}, + {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:56e3ff861c3b9c6857579de282ce8baabf443f42ffba355bf070770ed63e11e1"}, + {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f00790179497767aae6bcdc36355792c79e7bbb20b145ff449700eb076c5f96"}, + {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:75b297827b59bc229cac1a23a2f7a4ac0031068e5be0ce385be1462e7e17a35d"}, + {file = "pydantic-1.10.13-cp39-cp39-win_amd64.whl", hash = "sha256:e70ca129d2053fb8b728ee7d1af8e553a928d7e301a311094b8a0501adc8763d"}, + {file = "pydantic-1.10.13-py3-none-any.whl", hash = "sha256:b87326822e71bd5f313e7d3bfdc77ac3247035ac10b0c0618bd99dcf95b1e687"}, + {file = "pydantic-1.10.13.tar.gz", hash = "sha256:32c8b48dcd3b2ac4e78b0ba4af3a2c2eb6048cb75202f0ea7b34feb740efc340"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "7.4.3" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.4.3-py3-none-any.whl", hash = "sha256:0d009c083ea859a71b76adf7c1d502e4bc170b80a8ef002da5806527b9591fac"}, + {file = "pytest-7.4.3.tar.gz", hash = "sha256:d989d136982de4e3b29dabcc838ad581c64e8ed52c11fbe86ddebd9da0818cd5"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "0.6.4" +description = "Persistent cache for requests library" +optional = false +python-versions = ">=3.6" +files = [ + {file = "requests-cache-0.6.4.tar.gz", hash = "sha256:dd9120a4ab7b8128cba9b6b120d8b5560d566a3cd0f828cced3d3fd60a42ec40"}, + {file = "requests_cache-0.6.4-py2.py3-none-any.whl", hash = "sha256:1102daa13a804abe23fad62d694e7dee58d6063a35d94bf6e8c9821e22e5a78b"}, +] + +[package.dependencies] +itsdangerous = "*" +requests = ">=2.0.0" +url-normalize = ">=1.4" + +[package.extras] +backends = ["boto3", "pymongo", "redis"] +build = ["coveralls", "twine", "wheel"] +dev = ["Sphinx (>=3.5.3,<3.6.0)", "black (==20.8b1)", "boto3", "coveralls", "flake8", "flake8-comprehensions", "flake8-polyfill", "isort", "m2r2", "pre-commit", "psutil", "pymongo", "pytest (>=5.0)", "pytest-cov (>=2.11)", "pytest-order (>=0.11.0,<0.12.0)", "pytest-xdist", "radon", "redis", "requests-mock (>=1.8)", "sphinx-autodoc-typehints", "sphinx-copybutton", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-apidoc", "timeout-decorator", "twine", "wheel"] +docs = ["Sphinx (>=3.5.3,<3.6.0)", "m2r2", "sphinx-autodoc-typehints", "sphinx-copybutton", "sphinx-rtd-theme (>=0.5.2,<0.6.0)", "sphinxcontrib-apidoc"] +test = ["black (==20.8b1)", "flake8", "flake8-comprehensions", "flake8-polyfill", "isort", "pre-commit", "psutil", "pytest (>=5.0)", "pytest-cov (>=2.11)", "pytest-order (>=0.11.0,<0.12.0)", "pytest-xdist", "radon", "requests-mock (>=1.8)", "timeout-decorator"] + +[[package]] +name = "ruff" +version = "0.0.286" +description = "An extremely fast Python linter, written in Rust." +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.0.286-py3-none-macosx_10_7_x86_64.whl", hash = "sha256:8e22cb557e7395893490e7f9cfea1073d19a5b1dd337f44fd81359b2767da4e9"}, + {file = "ruff-0.0.286-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:68ed8c99c883ae79a9133cb1a86d7130feee0397fdf5ba385abf2d53e178d3fa"}, + {file = "ruff-0.0.286-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8301f0bb4ec1a5b29cfaf15b83565136c47abefb771603241af9d6038f8981e8"}, + {file = "ruff-0.0.286-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:acc4598f810bbc465ce0ed84417ac687e392c993a84c7eaf3abf97638701c1ec"}, + {file = "ruff-0.0.286-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88c8e358b445eb66d47164fa38541cfcc267847d1e7a92dd186dddb1a0a9a17f"}, + {file = "ruff-0.0.286-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:0433683d0c5dbcf6162a4beb2356e820a593243f1fa714072fec15e2e4f4c939"}, + {file = "ruff-0.0.286-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ddb61a0c4454cbe4623f4a07fef03c5ae921fe04fede8d15c6e36703c0a73b07"}, + {file = "ruff-0.0.286-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:47549c7c0be24c8ae9f2bce6f1c49fbafea83bca80142d118306f08ec7414041"}, + {file = "ruff-0.0.286-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:559aa793149ac23dc4310f94f2c83209eedb16908a0343663be19bec42233d25"}, + {file = "ruff-0.0.286-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:d73cfb1c3352e7aa0ce6fb2321f36fa1d4a2c48d2ceac694cb03611ddf0e4db6"}, + {file = "ruff-0.0.286-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:3dad93b1f973c6d1db4b6a5da8690c5625a3fa32bdf38e543a6936e634b83dc3"}, + {file = "ruff-0.0.286-py3-none-musllinux_1_2_i686.whl", hash = "sha256:26afc0851f4fc3738afcf30f5f8b8612a31ac3455cb76e611deea80f5c0bf3ce"}, + {file = "ruff-0.0.286-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:9b6b116d1c4000de1b9bf027131dbc3b8a70507788f794c6b09509d28952c512"}, + {file = "ruff-0.0.286-py3-none-win32.whl", hash = "sha256:556e965ac07c1e8c1c2d759ac512e526ecff62c00fde1a046acb088d3cbc1a6c"}, + {file = "ruff-0.0.286-py3-none-win_amd64.whl", hash = "sha256:5d295c758961376c84aaa92d16e643d110be32add7465e197bfdaec5a431a107"}, + {file = "ruff-0.0.286-py3-none-win_arm64.whl", hash = "sha256:1d6142d53ab7f164204b3133d053c4958d4d11ec3a39abf23a40b13b0784e3f0"}, + {file = "ruff-0.0.286.tar.gz", hash = "sha256:f1e9d169cce81a384a26ee5bb8c919fe9ae88255f39a1a69fd1ebab233a85ed2"}, +] + +[[package]] +name = "setuptools" +version = "68.2.2" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-68.2.2-py3-none-any.whl", hash = "sha256:b454a35605876da60632df1a60f736524eb73cc47bbc9f3f1ef1b644de74fd2a"}, + {file = "setuptools-68.2.2.tar.gz", hash = "sha256:4ac1475276d2f1c48684874089fefcd83bd7162ddaafb81fac866ba0db282a87"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "typing-extensions" +version = "4.8.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.8.0-py3-none-any.whl", hash = "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0"}, + {file = "typing_extensions-4.8.0.tar.gz", hash = "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "2.1.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.1.0-py3-none-any.whl", hash = "sha256:55901e917a5896a349ff771be919f8bd99aff50b79fe58fec595eb37bbc56bb3"}, + {file = "urllib3-2.1.0.tar.gz", hash = "sha256:df7aa8afb0148fa78488e7899b2c59b5f4ffcfa82e6c54ccb9dd37c1d7b52d54"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = ">=3.8" +content-hash = "ac1f132517a569ab27f8b0e1af6a61fe748f1653d7c16b4e91af4d43d1ffe1f9" diff --git a/airbyte-integrations/connectors/destination-duckdb/pyproject.toml b/airbyte-integrations/connectors/destination-duckdb/pyproject.toml new file mode 100644 index 000000000000..1d47a9651f97 --- /dev/null +++ b/airbyte-integrations/connectors/destination-duckdb/pyproject.toml @@ -0,0 +1,22 @@ +[tool.poetry] +name = "destination-duckdb" +version = "0.3.0" +description = "Destination implementation for Duckdb." +authors = ["Simon Späti, Airbyte"] +license = "MIT" +readme = "README.md" + +[tool.poetry.dependencies] +python = ">=3.8" +airbyte-cdk = "^0.51.6" +duckdb = "0.9.2" + +[tool.poetry.group.dev.dependencies] +pytest = "^7.4.0" +ruff = "^0.0.286" +black = "^23.7.0" +mypy = "^1.5.1" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/airbyte-integrations/connectors/destination-duckdb/setup.py b/airbyte-integrations/connectors/destination-duckdb/setup.py deleted file mode 100644 index b5a2272c46d6..000000000000 --- a/airbyte-integrations/connectors/destination-duckdb/setup.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from setuptools import find_packages, setup - -MAIN_REQUIREMENTS = ["airbyte-cdk", "duckdb"] # duckdb added manually to dockerfile due to lots of errors - -TEST_REQUIREMENTS = ["pytest~=6.1"] - -setup( - name="destination_duckdb", - description="Destination implementation for Duckdb.", - author="Simon Späti", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/destination-duckdb/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-duckdb/unit_tests/unit_test.py index 4c83289a9b31..f7dcb2c361cc 100644 --- a/airbyte-integrations/connectors/destination-duckdb/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/destination-duckdb/unit_tests/unit_test.py @@ -3,13 +3,34 @@ # import pytest -from destination_duckdb import DestinationDuckdb +from destination_duckdb.destination import DestinationDuckdb, validated_sql_name def test_read_invalid_path(): - invalid_input = "/test.duckdb" with pytest.raises(ValueError): _ = DestinationDuckdb._get_destination_path(invalid_input) assert True + + +@pytest.mark.parametrize( + "input, expected", + [ + ("test", "test"), + ("test_123", "test_123"), + ("test;123", None), + ("test123;", None), + ("test-123", None), + ("test 123", None), + ("test.123", None), + ("test,123", None), + ("test!123", None), + ], +) +def test_validated_sql_name(input, expected): + if expected is None: + with pytest.raises(ValueError): + validated_sql_name(input) + else: + assert validated_sql_name(input) == expected diff --git a/airbyte-integrations/connectors/destination-dynamodb/.dockerignore b/airbyte-integrations/connectors/destination-dynamodb/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-dynamodb/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-dynamodb/Dockerfile b/airbyte-integrations/connectors/destination-dynamodb/Dockerfile deleted file mode 100644 index ab8505d1df6c..000000000000 --- a/airbyte-integrations/connectors/destination-dynamodb/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-dynamodb - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-dynamodb - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.7 -LABEL io.airbyte.name=airbyte/destination-dynamodb diff --git a/airbyte-integrations/connectors/destination-dynamodb/README.md b/airbyte-integrations/connectors/destination-dynamodb/README.md index 3677fb5347ee..4bc24d1abefa 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/README.md +++ b/airbyte-integrations/connectors/destination-dynamodb/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-dynamodb:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-dynamodb:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-dynamodb:dev`. the Dockerfile. #### Run @@ -61,8 +62,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-dynamodb test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/dynamodb.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-dynamodb/build.gradle b/airbyte-integrations/connectors/destination-dynamodb/build.gradle index a2388073580e..4ae5c529215b 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/build.gradle +++ b/airbyte-integrations/connectors/destination-dynamodb/build.gradle @@ -1,24 +1,28 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.dynamodb.DynamodbDestinationRunner' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:bases:bases-destination-jdbc') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation 'com.amazonaws:aws-java-sdk-dynamodb:1.12.47' - - testImplementation project(':airbyte-integrations:bases:standard-destination-test') - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-dynamodb') } diff --git a/airbyte-integrations/connectors/destination-dynamodb/metadata.yaml b/airbyte-integrations/connectors/destination-dynamodb/metadata.yaml index 961fa4be5cd3..c50ab11ae6b5 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/metadata.yaml +++ b/airbyte-integrations/connectors/destination-dynamodb/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 8ccd8909-4e99-4141-b48d-4984b70b2d89 - dockerImageTag: 0.1.7 + dockerImageTag: 0.1.8 dockerRepository: airbyte/destination-dynamodb githubIssueLabel: destination-dynamodb icon: dynamodb.svg diff --git a/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbChecker.java b/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbChecker.java index 140fc9116a4f..8497967643a3 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbChecker.java +++ b/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbChecker.java @@ -12,7 +12,7 @@ import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder; import com.amazonaws.services.dynamodbv2.document.*; import com.amazonaws.services.dynamodbv2.model.*; -import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import java.util.Arrays; import java.util.UUID; import org.slf4j.Logger; diff --git a/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbConsumer.java b/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbConsumer.java index 420a6dd73c5e..fa1e7a856bcf 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbConsumer.java +++ b/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbConsumer.java @@ -11,8 +11,8 @@ import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.dynamodbv2.AmazonDynamoDB; import com.amazonaws.services.dynamodbv2.AmazonDynamoDBClientBuilder; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStream; diff --git a/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestination.java b/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestination.java index 8c7181160d32..5560884ac08d 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestination.java +++ b/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestination.java @@ -5,10 +5,10 @@ package io.airbyte.integrations.destination.dynamodb; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestinationRunner.java b/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestinationRunner.java index c2895893fe17..16f3d029ebb0 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestinationRunner.java +++ b/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestinationRunner.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.dynamodb; -import io.airbyte.integrations.base.adaptive.AdaptiveDestinationRunner; +import io.airbyte.cdk.integrations.base.adaptive.AdaptiveDestinationRunner; public class DynamodbDestinationRunner { diff --git a/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbOutputTableHelper.java b/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbOutputTableHelper.java index b459ab2ed9d4..c38499755c76 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbOutputTableHelper.java +++ b/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbOutputTableHelper.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.dynamodb; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.protocol.models.v0.AirbyteStream; import java.util.LinkedList; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbWriter.java b/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbWriter.java index 29c48398c407..ed7e20b7d0aa 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbWriter.java +++ b/airbyte-integrations/connectors/destination-dynamodb/src/main/java/io/airbyte/integrations/destination/dynamodb/DynamodbWriter.java @@ -11,8 +11,8 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.commons.jackson.MoreMappers; -import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; diff --git a/airbyte-integrations/connectors/destination-dynamodb/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-dynamodb/src/main/resources/spec.json index c77cd537ff98..0da5853a1910 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-dynamodb/src/main/resources/spec.json @@ -36,31 +36,39 @@ "description": "The region of the DynamoDB.", "enum": [ "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", "af-south-1", "ap-east-1", - "ap-south-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", + "ap-south-1", + "ap-south-2", "ap-southeast-1", "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", "ca-central-1", + "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", + "eu-central-2", "eu-north-1", "eu-south-1", + "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", - "sa-east-1", + "il-central-1", + "me-central-1", "me-south-1", + "sa-east-1", + "us-east-1", + "us-east-2", "us-gov-east-1", - "us-gov-west-1" + "us-gov-west-1", + "us-west-1", + "us-west-2" ] }, "access_key_id": { diff --git a/airbyte-integrations/connectors/destination-dynamodb/src/test-integration/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-dynamodb/src/test-integration/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestinationAcceptanceTest.java index 56936761a4d6..e654bb6dce47 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/src/test-integration/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-dynamodb/src/test-integration/java/io/airbyte/integrations/destination/dynamodb/DynamodbDestinationAcceptanceTest.java @@ -14,10 +14,10 @@ import com.amazonaws.services.dynamodbv2.document.spec.ScanSpec; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import java.io.IOException; import java.math.BigDecimal; import java.nio.file.Path; diff --git a/airbyte-integrations/connectors/destination-dynamodb/src/test/java/io/airbyte/integrations/destination/dynamodb/DynamodbConsumerTest.java b/airbyte-integrations/connectors/destination-dynamodb/src/test/java/io/airbyte/integrations/destination/dynamodb/DynamodbConsumerTest.java index 7bb5fe7880cf..589f5cd37e62 100644 --- a/airbyte-integrations/connectors/destination-dynamodb/src/test/java/io/airbyte/integrations/destination/dynamodb/DynamodbConsumerTest.java +++ b/airbyte-integrations/connectors/destination-dynamodb/src/test/java/io/airbyte/integrations/destination/dynamodb/DynamodbConsumerTest.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.dynamodb; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; -import io.airbyte.integrations.standardtest.destination.PerStreamStateMessageTest; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.standardtest.destination.PerStreamStateMessageTest; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import java.util.function.Consumer; diff --git a/airbyte-integrations/connectors/destination-e2e-test/.dockerignore b/airbyte-integrations/connectors/destination-e2e-test/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-e2e-test/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-e2e-test/Dockerfile b/airbyte-integrations/connectors/destination-e2e-test/Dockerfile deleted file mode 100644 index 1fe676aa4ddd..000000000000 --- a/airbyte-integrations/connectors/destination-e2e-test/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-e2e-test - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-e2e-test - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.3.0 -LABEL io.airbyte.name=airbyte/destination-e2e-test diff --git a/airbyte-integrations/connectors/destination-e2e-test/README.md b/airbyte-integrations/connectors/destination-e2e-test/README.md index ee61061d7122..0303b6c0e58a 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/README.md +++ b/airbyte-integrations/connectors/destination-e2e-test/README.md @@ -17,10 +17,11 @@ No credential is needed for this connector. #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-e2e-test:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-e2e-test:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-e2e-test:dev`. the Dockerfile. #### Run @@ -60,8 +61,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -2. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -3. Create a Pull Request. -4. Pat yourself on the back for being an awesome contributor. -5. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-e2e-test test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/e2e-test.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-e2e-test/build.gradle b/airbyte-integrations/connectors/destination-e2e-test/build.gradle index 4253ce8ddd7b..c8d98e1ddc2b 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/build.gradle +++ b/airbyte-integrations/connectors/destination-e2e-test/build.gradle @@ -1,19 +1,26 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.e2e_test.TestingDestinations' } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-e2e-test') } diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/FailAfterNDestination.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/FailAfterNDestination.java index 988661322ced..8146b5159c23 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/FailAfterNDestination.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/FailAfterNDestination.java @@ -5,9 +5,9 @@ package io.airbyte.integrations.destination.e2e_test; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/LoggingDestination.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/LoggingDestination.java index bc5440265764..f608fc3258c6 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/LoggingDestination.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/LoggingDestination.java @@ -5,9 +5,9 @@ package io.airbyte.integrations.destination.e2e_test; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; import io.airbyte.integrations.destination.e2e_test.logging.LoggingConsumer; import io.airbyte.integrations.destination.e2e_test.logging.TestingLoggerFactory; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/SilentDestination.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/SilentDestination.java index e11f3403d124..bd041f53fafc 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/SilentDestination.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/SilentDestination.java @@ -5,9 +5,9 @@ package io.airbyte.integrations.destination.e2e_test; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/TestingDestinations.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/TestingDestinations.java index 48c0772030b4..ea5a31f84048 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/TestingDestinations.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/TestingDestinations.java @@ -6,10 +6,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/ThrottledDestination.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/ThrottledDestination.java index 54b894821c89..c027d460b55c 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/ThrottledDestination.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/ThrottledDestination.java @@ -7,9 +7,9 @@ import static java.lang.Thread.sleep; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/LoggingConsumer.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/LoggingConsumer.java index 7427fc965c0c..4977f2b763f7 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/LoggingConsumer.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/logging/LoggingConsumer.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.e2e_test.logging; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.AirbyteMessageConsumer; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.java index 25ad63f098bd..ba3921e87cda 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/test-integration/java/io/airbyte/integrations/destination/e2e_test/TestingSilentDestinationAcceptanceTest.java @@ -7,9 +7,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.destination.e2e_test.TestingDestinations.TestDestinationType; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.util.Collections; diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/test/java/io/airbyte/integrations/destination/e2e_test/ThrottledDestinationTest.java b/airbyte-integrations/connectors/destination-e2e-test/src/test/java/io/airbyte/integrations/destination/e2e_test/ThrottledDestinationTest.java index 411a28bc7c66..95abd4f94002 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/test/java/io/airbyte/integrations/destination/e2e_test/ThrottledDestinationTest.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/test/java/io/airbyte/integrations/destination/e2e_test/ThrottledDestinationTest.java @@ -8,8 +8,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.AirbyteMessageConsumer; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/.dockerignore b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/Dockerfile deleted file mode 100644 index 9d699b6b7f6f..000000000000 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-elasticsearch-strict-encrypt - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-elasticsearch-strict-encrypt - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.6 -LABEL io.airbyte.name=airbyte/destination-elasticsearch-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle index 875387ec2622..6cd2f88febbe 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/build.gradle @@ -1,19 +1,29 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.elasticsearch.ElasticsearchStrictEncryptDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation 'co.elastic.clients:elasticsearch-java:7.15.0' implementation 'com.fasterxml.jackson.core:jackson-databind:2.12.3' @@ -30,21 +40,6 @@ dependencies { // MIT // https://www.testcontainers.org/ - testImplementation libs.connectors.testcontainers.elasticsearch - integrationTestJavaImplementation libs.connectors.testcontainers.elasticsearch - - integrationTestJavaImplementation project(':airbyte-connector-test-harnesses:acceptance-test-harness') - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-elasticsearch') -} - -repositories { - maven { - name = "ESSnapshots" - url = "https://snapshots.elastic.co/maven/" - } - maven { - name = "ESJavaGithubPackages" - url = "https://maven.pkg.github.com/elastic/elasticsearch-java" - } + testImplementation libs.testcontainers.elasticsearch + integrationTestJavaImplementation libs.testcontainers.elasticsearch } diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestination.java b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestination.java index 8f6a73f700e7..e661204e5186 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestination.java +++ b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestination.java @@ -9,10 +9,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.spec_modification.SpecModifyingDestination; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.spec_modification.SpecModifyingDestination; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.ConnectorSpecification; import java.net.URL; diff --git a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java index 66cae14c5ed4..1ac24aa68294 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-elasticsearch-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchStrictEncryptDestinationAcceptanceTest.java @@ -9,11 +9,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.json.Jsons; import io.airbyte.configoss.StandardCheckConnectionOutput.Status; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; diff --git a/airbyte-integrations/connectors/destination-elasticsearch/.dockerignore b/airbyte-integrations/connectors/destination-elasticsearch/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-elasticsearch/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-elasticsearch/Dockerfile b/airbyte-integrations/connectors/destination-elasticsearch/Dockerfile deleted file mode 100644 index a458e479af2f..000000000000 --- a/airbyte-integrations/connectors/destination-elasticsearch/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-elasticsearch - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-elasticsearch - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.6 -LABEL io.airbyte.name=airbyte/destination-elasticsearch diff --git a/airbyte-integrations/connectors/destination-elasticsearch/README.md b/airbyte-integrations/connectors/destination-elasticsearch/README.md index 0d7468cc682b..b693a900f412 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/README.md +++ b/airbyte-integrations/connectors/destination-elasticsearch/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-elasticsearch:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-elasticsearch:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-elasticsearch:dev`. the Dockerfile. #### Run @@ -61,8 +62,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-elasticsearch test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/elasticsearch.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-elasticsearch/build.gradle b/airbyte-integrations/connectors/destination-elasticsearch/build.gradle index 0c6a3be8d24e..52c7536993c1 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/build.gradle +++ b/airbyte-integrations/connectors/destination-elasticsearch/build.gradle @@ -1,20 +1,29 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.elasticsearch.ElasticsearchDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation 'co.elastic.clients:elasticsearch-java:7.15.0' implementation 'com.fasterxml.jackson.core:jackson-databind:2.12.3' @@ -30,20 +39,6 @@ dependencies { // MIT // https://www.testcontainers.org/ - testImplementation libs.connectors.testcontainers.elasticsearch - integrationTestJavaImplementation libs.connectors.testcontainers.elasticsearch - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-elasticsearch') -} - -repositories { - maven { - name = "ESSnapshots" - url = "https://snapshots.elastic.co/maven/" - } - maven { - name = "ESJavaGithubPackages" - url = "https://maven.pkg.github.com/elastic/elasticsearch-java" - } + testImplementation libs.testcontainers.elasticsearch + integrationTestJavaImplementation libs.testcontainers.elasticsearch } diff --git a/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchAirbyteMessageConsumerFactory.java b/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchAirbyteMessageConsumerFactory.java index 32456160b17d..f5b22cec43b7 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchAirbyteMessageConsumerFactory.java +++ b/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchAirbyteMessageConsumerFactory.java @@ -9,13 +9,13 @@ import co.elastic.clients.elasticsearch._types.ErrorCause; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.BufferedStreamConsumer; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.OnCloseFunction; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.OnStartFunction; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.RecordWriter; +import io.airbyte.cdk.integrations.destination.record_buffer.InMemoryRecordBufferingStrategy; import io.airbyte.commons.functional.CheckedFunction; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.destination.buffered_stream_consumer.BufferedStreamConsumer; -import io.airbyte.integrations.destination.buffered_stream_consumer.OnCloseFunction; -import io.airbyte.integrations.destination.buffered_stream_consumer.OnStartFunction; -import io.airbyte.integrations.destination.buffered_stream_consumer.RecordWriter; -import io.airbyte.integrations.destination.record_buffer.InMemoryRecordBufferingStrategy; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchConnection.java b/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchConnection.java index 176398d6cdb7..2a6f095dfad0 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchConnection.java +++ b/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchConnection.java @@ -17,7 +17,7 @@ import com.fasterxml.jackson.core.JsonPointer; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import io.airbyte.db.util.SSLCertificateUtils; +import io.airbyte.cdk.db.util.SSLCertificateUtils; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import jakarta.json.JsonValue; import java.io.IOException; diff --git a/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchDestination.java b/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchDestination.java index 477aa1e31fc1..94eefe281fec 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchDestination.java +++ b/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchDestination.java @@ -6,11 +6,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.ssh.SshWrappedDestination; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchWriteConfig.java b/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchWriteConfig.java index 9289f183f4da..b9b2503f2b38 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchWriteConfig.java +++ b/airbyte-integrations/connectors/destination-elasticsearch/src/main/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchWriteConfig.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.elasticsearch; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.protocol.models.v0.DestinationSyncMode; import java.util.List; import java.util.Objects; diff --git a/airbyte-integrations/connectors/destination-elasticsearch/src/main/resources/log4j2-test.example.yml b/airbyte-integrations/connectors/destination-elasticsearch/src/main/resources/log4j2-test.example.yml index b09ce061fc86..bbff489a9242 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/src/main/resources/log4j2-test.example.yml +++ b/airbyte-integrations/connectors/destination-elasticsearch/src/main/resources/log4j2-test.example.yml @@ -30,4 +30,4 @@ Configuration: Root: level: info AppenderRef: - ref: STDOUT \ No newline at end of file + ref: STDOUT diff --git a/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchDestinationAcceptanceTest.java index e35d59ba56b7..9d50ced47a37 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchDestinationAcceptanceTest.java @@ -6,9 +6,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import java.time.Duration; import java.util.HashSet; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/SshElasticsearchDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/SshElasticsearchDestinationAcceptanceTest.java index d3f6e9e65de7..4e14249390ea 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/SshElasticsearchDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/SshElasticsearchDestinationAcceptanceTest.java @@ -7,9 +7,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.base.ssh.SshBastionContainer; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.ssh.SshBastionContainer; -import io.airbyte.integrations.base.ssh.SshTunnel; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.testcontainers.containers.Network; diff --git a/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/SshKeyElasticsearchDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/SshKeyElasticsearchDestinationAcceptanceTest.java index 15bcd5fa9bf8..30c247f72b0b 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/SshKeyElasticsearchDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/SshKeyElasticsearchDestinationAcceptanceTest.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.elasticsearch; -import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; public class SshKeyElasticsearchDestinationAcceptanceTest extends SshElasticsearchDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/SshPasswordElasticsearchDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/SshPasswordElasticsearchDestinationAcceptanceTest.java index cad0f60efb1b..9894a9678173 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/SshPasswordElasticsearchDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-elasticsearch/src/test-integration/java/io/airbyte/integrations/destination/elasticsearch/SshPasswordElasticsearchDestinationAcceptanceTest.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.elasticsearch; -import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; public class SshPasswordElasticsearchDestinationAcceptanceTest extends SshElasticsearchDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-elasticsearch/src/test/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchDestinationTest.java b/airbyte-integrations/connectors/destination-elasticsearch/src/test/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchDestinationTest.java index ae151671224c..0f25d5da091b 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/src/test/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchDestinationTest.java +++ b/airbyte-integrations/connectors/destination-elasticsearch/src/test/java/io/airbyte/integrations/destination/elasticsearch/ElasticsearchDestinationTest.java @@ -8,9 +8,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-elasticsearch/src/test/java/io/airbyte/integrations/destination/elasticsearch/elasticsearch.yml b/airbyte-integrations/connectors/destination-elasticsearch/src/test/java/io/airbyte/integrations/destination/elasticsearch/elasticsearch.yml index 65a930cda14b..9a834713585b 100644 --- a/airbyte-integrations/connectors/destination-elasticsearch/src/test/java/io/airbyte/integrations/destination/elasticsearch/elasticsearch.yml +++ b/airbyte-integrations/connectors/destination-elasticsearch/src/test/java/io/airbyte/integrations/destination/elasticsearch/elasticsearch.yml @@ -1,4 +1,4 @@ -node.data : true -network.host : 0.0.0.0 -discovery.seed_hosts : [] -cluster.initial_master_nodes : [] +node.data: true +network.host: 0.0.0.0 +discovery.seed_hosts: [] +cluster.initial_master_nodes: [] diff --git a/airbyte-integrations/connectors/destination-exasol/.dockerignore b/airbyte-integrations/connectors/destination-exasol/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-exasol/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-exasol/Dockerfile b/airbyte-integrations/connectors/destination-exasol/Dockerfile deleted file mode 100644 index 86ed795bf790..000000000000 --- a/airbyte-integrations/connectors/destination-exasol/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte -ENV APPLICATION destination-exasol - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte -ENV APPLICATION destination-exasol - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.1 -LABEL io.airbyte.name=airbyte/destination-exasol diff --git a/airbyte-integrations/connectors/destination-exasol/README.md b/airbyte-integrations/connectors/destination-exasol/README.md index 04a53edc78e6..8651db3ec762 100644 --- a/airbyte-integrations/connectors/destination-exasol/README.md +++ b/airbyte-integrations/connectors/destination-exasol/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-exasol:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-exasol:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-exasol:dev`. the Dockerfile. #### Run @@ -60,8 +61,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately according to [semantic versioning](https://semver.org/). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from the Airbyte team will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-exasol test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/exasol.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-exasol/build.gradle b/airbyte-integrations/connectors/destination-exasol/build.gradle index 73b46b771573..3380731e417d 100644 --- a/airbyte-integrations/connectors/destination-exasol/build.gradle +++ b/airbyte-integrations/connectors/destination-exasol/build.gradle @@ -1,19 +1,28 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.exasol.ExasolDestination' } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - implementation project(':airbyte-integrations:bases:bases-destination-jdbc') implementation 'com.exasol:exasol-jdbc:7.1.17' @@ -23,8 +32,6 @@ dependencies { // 'org.testcontainers.containers.GenericContainer com.exasol.containers.ExasolContainer.withCopyToContainer(org.testcontainers.images.builder.Transferable, java.lang.String)' testImplementation 'org.testcontainers:testcontainers:1.17.6' - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-exasol') integrationTestJavaImplementation 'com.exasol:exasol-testcontainers:6.5.0' integrationTestJavaImplementation 'org.testcontainers:testcontainers:1.17.6' } diff --git a/airbyte-integrations/connectors/destination-exasol/src/main/java/io/airbyte/integrations/destination/exasol/ExasolDestination.java b/airbyte-integrations/connectors/destination-exasol/src/main/java/io/airbyte/integrations/destination/exasol/ExasolDestination.java index 6179f817a0c8..8145c85c2444 100644 --- a/airbyte-integrations/connectors/destination-exasol/src/main/java/io/airbyte/integrations/destination/exasol/ExasolDestination.java +++ b/airbyte-integrations/connectors/destination-exasol/src/main/java/io/airbyte/integrations/destination/exasol/ExasolDestination.java @@ -6,12 +6,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import java.util.HashMap; import java.util.Map; diff --git a/airbyte-integrations/connectors/destination-exasol/src/main/java/io/airbyte/integrations/destination/exasol/ExasolSQLNameTransformer.java b/airbyte-integrations/connectors/destination-exasol/src/main/java/io/airbyte/integrations/destination/exasol/ExasolSQLNameTransformer.java index c79795e9ce4d..8fd3caf20a75 100644 --- a/airbyte-integrations/connectors/destination-exasol/src/main/java/io/airbyte/integrations/destination/exasol/ExasolSQLNameTransformer.java +++ b/airbyte-integrations/connectors/destination-exasol/src/main/java/io/airbyte/integrations/destination/exasol/ExasolSQLNameTransformer.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.exasol; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.commons.text.Names; -import io.airbyte.integrations.destination.StandardNameTransformer; public class ExasolSQLNameTransformer extends StandardNameTransformer { diff --git a/airbyte-integrations/connectors/destination-exasol/src/main/java/io/airbyte/integrations/destination/exasol/ExasolSqlOperations.java b/airbyte-integrations/connectors/destination-exasol/src/main/java/io/airbyte/integrations/destination/exasol/ExasolSqlOperations.java index bce985fd7404..e0353bd414b9 100644 --- a/airbyte-integrations/connectors/destination-exasol/src/main/java/io/airbyte/integrations/destination/exasol/ExasolSqlOperations.java +++ b/airbyte-integrations/connectors/destination-exasol/src/main/java/io/airbyte/integrations/destination/exasol/ExasolSqlOperations.java @@ -4,9 +4,9 @@ package io.airbyte.integrations.destination.exasol; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.jdbc.JdbcSqlOperations; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.jdbc.JdbcSqlOperations; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.nio.file.Files; import java.nio.file.Path; diff --git a/airbyte-integrations/connectors/destination-exasol/src/test-integration/java/io/airbyte/integrations/destination/exasol/ExasolDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-exasol/src/test-integration/java/io/airbyte/integrations/destination/exasol/ExasolDestinationAcceptanceTest.java index 23326d046f35..8fd01ec062dc 100644 --- a/airbyte-integrations/connectors/destination-exasol/src/test-integration/java/io/airbyte/integrations/destination/exasol/ExasolDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-exasol/src/test-integration/java/io/airbyte/integrations/destination/exasol/ExasolDestinationAcceptanceTest.java @@ -8,14 +8,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; import java.sql.SQLException; import java.util.ArrayList; import java.util.HashSet; diff --git a/airbyte-integrations/connectors/destination-exasol/src/test-integration/java/io/airbyte/integrations/destination/exasol/ExasolSqlOperationsAcceptanceTest.java b/airbyte-integrations/connectors/destination-exasol/src/test-integration/java/io/airbyte/integrations/destination/exasol/ExasolSqlOperationsAcceptanceTest.java index 8e37bb46de6d..dd32fea81ee6 100644 --- a/airbyte-integrations/connectors/destination-exasol/src/test-integration/java/io/airbyte/integrations/destination/exasol/ExasolSqlOperationsAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-exasol/src/test-integration/java/io/airbyte/integrations/destination/exasol/ExasolSqlOperationsAcceptanceTest.java @@ -10,9 +10,9 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import com.exasol.containers.ExasolContainer; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.SQLSyntaxErrorException; diff --git a/airbyte-integrations/connectors/destination-exasol/src/test/java/io/airbyte/integrations/destination/exasol/ExasolDestinationTest.java b/airbyte-integrations/connectors/destination-exasol/src/test/java/io/airbyte/integrations/destination/exasol/ExasolDestinationTest.java index 70ec951a5f94..79789c1232eb 100644 --- a/airbyte-integrations/connectors/destination-exasol/src/test/java/io/airbyte/integrations/destination/exasol/ExasolDestinationTest.java +++ b/airbyte-integrations/connectors/destination-exasol/src/test/java/io/airbyte/integrations/destination/exasol/ExasolDestinationTest.java @@ -9,9 +9,9 @@ import static org.junit.jupiter.api.Assertions.*; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; -import io.airbyte.db.jdbc.JdbcUtils; import java.util.HashMap; import java.util.Map; import org.junit.jupiter.api.BeforeEach; diff --git a/airbyte-integrations/connectors/destination-firebolt/README.md b/airbyte-integrations/connectors/destination-firebolt/README.md index 13e918af34b0..d19fb11dc8a0 100644 --- a/airbyte-integrations/connectors/destination-firebolt/README.md +++ b/airbyte-integrations/connectors/destination-firebolt/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:destination-firebolt:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/destinations/firebolt) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_firebolt/spec.json` file. @@ -54,18 +48,19 @@ cat integration_tests/messages.jsonl | python main.py write --config secrets/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/destination-firebolt:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=destination-firebolt build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:destination-firebolt:airbyteDocker +An image will be built with the tag `airbyte/destination-firebolt:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/destination-firebolt:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,38 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-firebolt:dev chec # messages.jsonl is a file containing line-separated JSON representing AirbyteMessages cat integration_tests/messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-firebolt:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=destination-firebolt test ``` -#### Acceptance Tests -Coming soon: -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:destination-firebolt:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:destination-firebolt:integrationTest -``` +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -116,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-firebolt test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/firebolt.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-firebolt/build.gradle b/airbyte-integrations/connectors/destination-firebolt/build.gradle deleted file mode 100644 index 08c1a70562ae..000000000000 --- a/airbyte-integrations/connectors/destination-firebolt/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'destination_firebolt' -} diff --git a/airbyte-integrations/connectors/destination-firestore/README.md b/airbyte-integrations/connectors/destination-firestore/README.md index 3d8b538708fc..448c941fe0a8 100644 --- a/airbyte-integrations/connectors/destination-firestore/README.md +++ b/airbyte-integrations/connectors/destination-firestore/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:destination-firestore:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://cloud.google.com/iam/docs/creating-managing-service-accounts) @@ -55,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/destination-firestore:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=destination-firestore build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:destination-firestore:airbyteDocker +An image will be built with the tag `airbyte/destination-firestore:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/destination-firestore:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -76,41 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-firestore:dev che # messages.jsonl is a file containing line-separated JSON representing AirbyteMessages cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-firestore:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` - -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). - -Integration tests will most likely delete all your data credentials has access to. Limit service account permissions in a separate project. -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=destination-firestore test ``` -#### Acceptance Tests -Coming soon: -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:destination-firestore:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:destination-firestore:integrationTest -``` +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -120,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-firestore test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/firestore.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-firestore/build.gradle b/airbyte-integrations/connectors/destination-firestore/build.gradle deleted file mode 100644 index 39aff196d032..000000000000 --- a/airbyte-integrations/connectors/destination-firestore/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'destinationfirestore' -} diff --git a/airbyte-integrations/connectors/destination-gcs/.dockerignore b/airbyte-integrations/connectors/destination-gcs/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-gcs/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-gcs/Dockerfile b/airbyte-integrations/connectors/destination-gcs/Dockerfile deleted file mode 100644 index df7f1d9ffd94..000000000000 --- a/airbyte-integrations/connectors/destination-gcs/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-gcs - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-gcs - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.4.4 -LABEL io.airbyte.name=airbyte/destination-gcs diff --git a/airbyte-integrations/connectors/destination-gcs/build.gradle b/airbyte-integrations/connectors/destination-gcs/build.gradle index fe7fa51f3eb9..a5cc5eb33d28 100644 --- a/airbyte-integrations/connectors/destination-gcs/build.gradle +++ b/airbyte-integrations/connectors/destination-gcs/build.gradle @@ -1,21 +1,29 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.gcs.GcsDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:bases:base-java-s3') - implementation project(':airbyte-integrations:bases:bases-destination-jdbc') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation platform('com.amazonaws:aws-java-sdk-bom:1.12.14') implementation 'com.google.cloud.bigdataoss:gcs-connector:hadoop3-2.2.1' @@ -40,17 +48,4 @@ dependencies { testImplementation 'org.apache.commons:commons-lang3:3.11' testImplementation 'org.xerial.snappy:snappy-java:1.1.8.4' - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-gcs') - integrationTestJavaImplementation project(':airbyte-connector-test-harnesses:acceptance-test-harness') } - - - - - - - - - diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestination.java b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestination.java index a5eb30974642..3b209f6fe7c7 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestination.java +++ b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestination.java @@ -4,22 +4,22 @@ package io.airbyte.integrations.destination.gcs; -import static io.airbyte.integrations.base.errors.messages.ErrorMessage.getErrorMessage; +import static io.airbyte.cdk.integrations.base.errors.messages.ErrorMessage.getErrorMessage; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.internal.SkipMd5CheckStrategy; import com.amazonaws.services.s3.model.AmazonS3Exception; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.AirbyteTraceMessageUtility; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.record_buffer.FileBuffer; -import io.airbyte.integrations.destination.s3.S3BaseChecks; -import io.airbyte.integrations.destination.s3.S3ConsumerFactory; -import io.airbyte.integrations.destination.s3.SerializedBufferFactory; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.record_buffer.FileBuffer; +import io.airbyte.cdk.integrations.destination.s3.S3BaseChecks; +import io.airbyte.cdk.integrations.destination.s3.S3ConsumerFactory; +import io.airbyte.cdk.integrations.destination.s3.SerializedBufferFactory; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestinationConfig.java b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestinationConfig.java index ff0ba045936a..23a72598e1cd 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestinationConfig.java +++ b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsDestinationConfig.java @@ -10,14 +10,14 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConstants; +import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; +import io.airbyte.cdk.integrations.destination.s3.S3FormatConfigs; +import io.airbyte.cdk.integrations.destination.s3.S3StorageOperations; import io.airbyte.integrations.destination.gcs.credential.GcsCredentialConfig; import io.airbyte.integrations.destination.gcs.credential.GcsCredentialConfigs; import io.airbyte.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.S3DestinationConstants; -import io.airbyte.integrations.destination.s3.S3FormatConfig; -import io.airbyte.integrations.destination.s3.S3FormatConfigs; -import io.airbyte.integrations.destination.s3.S3StorageOperations; /** * Currently we always reuse the S3 client for GCS. So the GCS config extends from the S3 config. diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsNameTransformer.java b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsNameTransformer.java index f38b1bfb218c..af2146bd742b 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsNameTransformer.java +++ b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsNameTransformer.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.gcs; -import io.airbyte.integrations.destination.s3.util.S3NameTransformer; +import io.airbyte.cdk.integrations.destination.s3.util.S3NameTransformer; public class GcsNameTransformer extends S3NameTransformer { diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsStorageOperations.java b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsStorageOperations.java index 22a1281d896f..453b4d60e7df 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsStorageOperations.java +++ b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/GcsStorageOperations.java @@ -6,9 +6,9 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.S3StorageOperations; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3StorageOperations; import java.util.HashMap; import java.util.List; import java.util.Map; diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriter.java b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriter.java index 9bf9ec3be3ae..e63b436a732f 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriter.java +++ b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriter.java @@ -4,21 +4,21 @@ package io.airbyte.integrations.destination.gcs.avro; -import static io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; +import static io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; import alex.mojaki.s3upload.MultiPartOutputStream; import alex.mojaki.s3upload.StreamTransferManager; import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.avro.AvroRecordFactory; +import io.airbyte.cdk.integrations.destination.s3.avro.JsonToAvroSchemaConverter; +import io.airbyte.cdk.integrations.destination.s3.avro.S3AvroFormatConfig; +import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; +import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter; import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.integrations.destination.gcs.util.GcsUtils; import io.airbyte.integrations.destination.gcs.writer.BaseGcsWriter; -import io.airbyte.integrations.destination.s3.S3Format; -import io.airbyte.integrations.destination.s3.avro.AvroRecordFactory; -import io.airbyte.integrations.destination.s3.avro.JsonToAvroSchemaConverter; -import io.airbyte.integrations.destination.s3.avro.S3AvroFormatConfig; -import io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory; -import io.airbyte.integrations.destination.s3.writer.DestinationFileWriter; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.io.IOException; @@ -63,7 +63,7 @@ public GcsAvroWriter(final GcsDestinationConfig config, super(config, s3Client, configuredStream); final Schema schema = jsonSchema == null - ? GcsUtils.getDefaultAvroSchema(stream.getName(), stream.getNamespace(), true) + ? GcsUtils.getDefaultAvroSchema(stream.getName(), stream.getNamespace(), true, false) : new JsonToAvroSchemaConverter().getAvroSchema(jsonSchema, stream.getName(), stream.getNamespace(), true, false, false, true); LOGGER.info("Avro schema for stream {}: {}", stream.getName(), schema.toString(false)); diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsCredentialConfig.java b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsCredentialConfig.java index 954374d59089..f8465486a9f8 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsCredentialConfig.java +++ b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsCredentialConfig.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.gcs.credential; -import io.airbyte.integrations.destination.s3.credential.BlobStorageCredentialConfig; -import io.airbyte.integrations.destination.s3.credential.S3CredentialConfig; +import io.airbyte.cdk.integrations.destination.s3.credential.BlobStorageCredentialConfig; +import io.airbyte.cdk.integrations.destination.s3.credential.S3CredentialConfig; import java.util.Optional; public interface GcsCredentialConfig extends BlobStorageCredentialConfig { diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsHmacKeyCredentialConfig.java b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsHmacKeyCredentialConfig.java index 2bf31555814f..e1521ad34bf0 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsHmacKeyCredentialConfig.java +++ b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/credential/GcsHmacKeyCredentialConfig.java @@ -5,8 +5,8 @@ package io.airbyte.integrations.destination.gcs.credential; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.destination.s3.credential.S3AccessKeyCredentialConfig; -import io.airbyte.integrations.destination.s3.credential.S3CredentialConfig; +import io.airbyte.cdk.integrations.destination.s3.credential.S3AccessKeyCredentialConfig; +import io.airbyte.cdk.integrations.destination.s3.credential.S3CredentialConfig; import java.util.Optional; public class GcsHmacKeyCredentialConfig implements GcsCredentialConfig { diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvWriter.java b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvWriter.java index b910731dad11..1e76838b8a85 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvWriter.java +++ b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvWriter.java @@ -4,19 +4,19 @@ package io.airbyte.integrations.destination.gcs.csv; -import static io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; +import static io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; import alex.mojaki.s3upload.MultiPartOutputStream; import alex.mojaki.s3upload.StreamTransferManager; import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.csv.CsvSheetGenerator; +import io.airbyte.cdk.integrations.destination.s3.csv.S3CsvFormatConfig; +import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; +import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter; import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.integrations.destination.gcs.writer.BaseGcsWriter; -import io.airbyte.integrations.destination.s3.S3Format; -import io.airbyte.integrations.destination.s3.csv.CsvSheetGenerator; -import io.airbyte.integrations.destination.s3.csv.S3CsvFormatConfig; -import io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory; -import io.airbyte.integrations.destination.s3.writer.DestinationFileWriter; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.io.IOException; diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlWriter.java b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlWriter.java index 473aa621b1e1..1d56bd357d1d 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlWriter.java +++ b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlWriter.java @@ -10,14 +10,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; +import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter; import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.integrations.destination.gcs.writer.BaseGcsWriter; -import io.airbyte.integrations.destination.s3.S3Format; -import io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory; -import io.airbyte.integrations.destination.s3.writer.DestinationFileWriter; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.io.IOException; diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/parquet/GcsParquetWriter.java b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/parquet/GcsParquetWriter.java index 205c82471029..b77aec84d515 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/parquet/GcsParquetWriter.java +++ b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/parquet/GcsParquetWriter.java @@ -7,13 +7,13 @@ import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.avro.AvroRecordFactory; +import io.airbyte.cdk.integrations.destination.s3.parquet.S3ParquetFormatConfig; +import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter; import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig; import io.airbyte.integrations.destination.gcs.writer.BaseGcsWriter; -import io.airbyte.integrations.destination.s3.S3Format; -import io.airbyte.integrations.destination.s3.avro.AvroRecordFactory; -import io.airbyte.integrations.destination.s3.parquet.S3ParquetFormatConfig; -import io.airbyte.integrations.destination.s3.writer.DestinationFileWriter; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.io.IOException; diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/util/GcsUtils.java b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/util/GcsUtils.java index 9ea223b318f6..6c305c177ec7 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/util/GcsUtils.java +++ b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/util/GcsUtils.java @@ -4,9 +4,8 @@ package io.airbyte.integrations.destination.gcs.util; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.base.TypingAndDedupingFlag; -import io.airbyte.integrations.destination.s3.avro.AvroConstants; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.s3.avro.AvroConstants; import javax.annotation.Nullable; import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; @@ -23,7 +22,8 @@ public class GcsUtils { public static Schema getDefaultAvroSchema(final String name, @Nullable final String namespace, - final boolean appendAirbyteFields) { + final boolean appendAirbyteFields, + final boolean useDestinationsV2Columns) { LOGGER.info("Default schema."); final String stdName = AvroConstants.NAME_TRANSFORMER.getIdentifier(name); final String stdNamespace = AvroConstants.NAME_TRANSFORMER.getNamespace(namespace); @@ -32,25 +32,24 @@ public static Schema getDefaultAvroSchema(final String name, if (stdNamespace != null) { builder = builder.namespace(stdNamespace); } - if (TypingAndDedupingFlag.isDestinationV2()) { + if (useDestinationsV2Columns) { builder.namespace("airbyte"); } SchemaBuilder.FieldAssembler assembler = builder.fields(); - if (TypingAndDedupingFlag.isDestinationV2()) { + if (useDestinationsV2Columns) { if (appendAirbyteFields) { assembler = assembler.name(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID).type(UUID_SCHEMA).noDefault(); assembler = assembler.name(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT).type(TIMESTAMP_MILLIS_SCHEMA).noDefault(); assembler = assembler.name(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT).type(NULLABLE_TIMESTAMP_MILLIS).withDefault(null); } - assembler = assembler.name(JavaBaseConstants.COLUMN_NAME_DATA).type().stringType().noDefault(); } else { if (appendAirbyteFields) { assembler = assembler.name(JavaBaseConstants.COLUMN_NAME_AB_ID).type(UUID_SCHEMA).noDefault(); assembler = assembler.name(JavaBaseConstants.COLUMN_NAME_EMITTED_AT).type(TIMESTAMP_MILLIS_SCHEMA).noDefault(); } - assembler = assembler.name(JavaBaseConstants.COLUMN_NAME_DATA).type().stringType().noDefault(); } + assembler = assembler.name(JavaBaseConstants.COLUMN_NAME_DATA).type().stringType().noDefault(); return assembler.endRecord(); } diff --git a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/writer/BaseGcsWriter.java b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/writer/BaseGcsWriter.java index ab5c6f40b139..225f85851fd7 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/writer/BaseGcsWriter.java +++ b/airbyte-integrations/connectors/destination-gcs/src/main/java/io/airbyte/integrations/destination/gcs/writer/BaseGcsWriter.java @@ -8,11 +8,11 @@ import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; import com.amazonaws.services.s3.model.HeadBucketRequest; import com.amazonaws.services.s3.model.S3ObjectSummary; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConstants; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.util.S3OutputPathHelper; +import io.airbyte.cdk.integrations.destination.s3.writer.DestinationFileWriter; import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; -import io.airbyte.integrations.destination.s3.S3DestinationConstants; -import io.airbyte.integrations.destination.s3.S3Format; -import io.airbyte.integrations.destination.s3.util.S3OutputPathHelper; -import io.airbyte.integrations.destination.s3.writer.DestinationFileWriter; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroDestinationAcceptanceTest.java index 7e1d519550ce..e19036bd5c47 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroDestinationAcceptanceTest.java @@ -8,13 +8,13 @@ import com.amazonaws.services.s3.model.S3ObjectSummary; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectReader; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.avro.AvroConstants; +import io.airbyte.cdk.integrations.destination.s3.avro.JsonFieldNameUpdater; +import io.airbyte.cdk.integrations.destination.s3.util.AvroRecordHelper; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.s3.S3Format; -import io.airbyte.integrations.destination.s3.avro.AvroConstants; -import io.airbyte.integrations.destination.s3.avro.JsonFieldNameUpdater; -import io.airbyte.integrations.destination.s3.util.AvroRecordHelper; -import io.airbyte.integrations.standardtest.destination.ProtocolVersion; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.util.HashMap; import java.util.LinkedList; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.java index d47802b40a83..f5178562d19b 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroParquetDestinationAcceptanceTest.java @@ -7,12 +7,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.avro.JsonSchemaType; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; +import io.airbyte.cdk.integrations.standardtest.destination.argproviders.NumberDataTypeTestArgumentProvider; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.destination.s3.S3Format; -import io.airbyte.integrations.destination.s3.avro.JsonSchemaType; -import io.airbyte.integrations.standardtest.destination.ProtocolVersion; -import io.airbyte.integrations.standardtest.destination.argproviders.NumberDataTypeTestArgumentProvider; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStream; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroTestDataComparator.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroTestDataComparator.java index a9b500ae50fc..2dcb585cd770 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroTestDataComparator.java +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsAvroTestDataComparator.java @@ -5,7 +5,7 @@ package io.airbyte.integrations.destination.gcs; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import java.nio.charset.StandardCharsets; import java.time.*; import java.time.format.DateTimeFormatter; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsCsvDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsCsvDestinationAcceptanceTest.java index 140bc69cd3f6..7c7985498e0e 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsCsvDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsCsvDestinationAcceptanceTest.java @@ -8,11 +8,11 @@ import com.amazonaws.services.s3.model.S3ObjectSummary; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.util.Flattening; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.s3.S3Format; -import io.airbyte.integrations.destination.s3.util.Flattening; -import io.airbyte.integrations.standardtest.destination.ProtocolVersion; import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsCsvGzipDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsCsvGzipDestinationAcceptanceTest.java index faf77ac8c836..075b9532a54f 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsCsvGzipDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsCsvGzipDestinationAcceptanceTest.java @@ -6,9 +6,9 @@ import com.amazonaws.services.s3.model.S3Object; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.s3.util.Flattening; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.s3.util.Flattening; -import io.airbyte.integrations.standardtest.destination.ProtocolVersion; import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsDestinationAcceptanceTest.java index 243e3c71005f..b78e0ef089d4 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsDestinationAcceptanceTest.java @@ -14,18 +14,18 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; +import io.airbyte.cdk.integrations.destination.s3.S3StorageOperations; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.io.IOs; import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; import io.airbyte.configoss.StandardCheckConnectionOutput.Status; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.s3.S3Format; -import io.airbyte.integrations.destination.s3.S3FormatConfig; -import io.airbyte.integrations.destination.s3.S3StorageOperations; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.ProtocolVersion; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import java.nio.file.Path; import java.util.Comparator; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsJsonlDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsJsonlDestinationAcceptanceTest.java index 28d88a539456..4d9fb0e21636 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsJsonlDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsJsonlDestinationAcceptanceTest.java @@ -7,10 +7,10 @@ import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.s3.S3Format; -import io.airbyte.integrations.standardtest.destination.ProtocolVersion; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsJsonlGzipDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsJsonlGzipDestinationAcceptanceTest.java index 8869dd2f72a3..152bf22d1535 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsJsonlGzipDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsJsonlGzipDestinationAcceptanceTest.java @@ -6,8 +6,8 @@ import com.amazonaws.services.s3.model.S3Object; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.standardtest.destination.ProtocolVersion; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsParquetDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsParquetDestinationAcceptanceTest.java index 11fcdbb0c5a1..65b2e68e8161 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsParquetDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-gcs/src/test-integration/java/io/airbyte/integrations/destination/gcs/GcsParquetDestinationAcceptanceTest.java @@ -8,15 +8,15 @@ import com.amazonaws.services.s3.model.S3ObjectSummary; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectReader; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.avro.AvroConstants; +import io.airbyte.cdk.integrations.destination.s3.avro.JsonFieldNameUpdater; +import io.airbyte.cdk.integrations.destination.s3.parquet.S3ParquetWriter; +import io.airbyte.cdk.integrations.destination.s3.util.AvroRecordHelper; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.destination.gcs.parquet.GcsParquetWriter; -import io.airbyte.integrations.destination.s3.S3Format; -import io.airbyte.integrations.destination.s3.avro.AvroConstants; -import io.airbyte.integrations.destination.s3.avro.JsonFieldNameUpdater; -import io.airbyte.integrations.destination.s3.parquet.S3ParquetWriter; -import io.airbyte.integrations.destination.s3.util.AvroRecordHelper; -import io.airbyte.integrations.standardtest.destination.ProtocolVersion; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/GcsDestinationConfigTest.java b/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/GcsDestinationConfigTest.java index 28ecc6b92a26..bbecfdd52e43 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/GcsDestinationConfigTest.java +++ b/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/GcsDestinationConfigTest.java @@ -8,12 +8,12 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; +import io.airbyte.cdk.integrations.destination.s3.avro.S3AvroFormatConfig; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.integrations.destination.gcs.credential.GcsCredentialConfig; import io.airbyte.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig; -import io.airbyte.integrations.destination.s3.S3FormatConfig; -import io.airbyte.integrations.destination.s3.avro.S3AvroFormatConfig; import java.io.IOException; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroFormatConfigTest.java b/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroFormatConfigTest.java index 84b81f4c7a09..ae91d60910be 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroFormatConfigTest.java +++ b/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroFormatConfigTest.java @@ -5,18 +5,18 @@ package io.airbyte.integrations.destination.gcs.avro; import static com.amazonaws.services.s3.internal.Constants.MB; -import static io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; +import static io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; import static org.junit.jupiter.api.Assertions.assertEquals; import alex.mojaki.s3upload.StreamTransferManager; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; +import io.airbyte.cdk.integrations.destination.s3.avro.S3AvroFormatConfig; +import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.integrations.destination.gcs.util.ConfigTestUtils; -import io.airbyte.integrations.destination.s3.S3FormatConfig; -import io.airbyte.integrations.destination.s3.avro.S3AvroFormatConfig; -import io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory; import java.util.List; import org.apache.avro.file.CodecFactory; import org.apache.avro.file.DataFileConstants; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriterTest.java b/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriterTest.java index fe1b3b30eb13..bec533af2b54 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriterTest.java +++ b/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/avro/GcsAvroWriterTest.java @@ -11,11 +11,11 @@ import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.base.DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.avro.S3AvroFormatConfig; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.DestinationConfig; import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.integrations.destination.gcs.credential.GcsHmacKeyCredentialConfig; -import io.airbyte.integrations.destination.s3.avro.S3AvroFormatConfig; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.SyncMode; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvFormatConfigTest.java b/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvFormatConfigTest.java index a3f14c1c83fc..5521f04a5dcf 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvFormatConfigTest.java +++ b/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/csv/GcsCsvFormatConfigTest.java @@ -5,18 +5,18 @@ package io.airbyte.integrations.destination.gcs.csv; import static com.amazonaws.services.s3.internal.Constants.MB; -import static io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; +import static io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import alex.mojaki.s3upload.StreamTransferManager; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; +import io.airbyte.cdk.integrations.destination.s3.util.Flattening; +import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.integrations.destination.gcs.util.ConfigTestUtils; -import io.airbyte.integrations.destination.s3.S3FormatConfig; -import io.airbyte.integrations.destination.s3.util.Flattening; -import io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory; import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlFormatConfigTest.java b/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlFormatConfigTest.java index c049c2bb1640..3db5d455daff 100644 --- a/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlFormatConfigTest.java +++ b/airbyte-integrations/connectors/destination-gcs/src/test/java/io/airbyte/integrations/destination/gcs/jsonl/GcsJsonlFormatConfigTest.java @@ -5,16 +5,16 @@ package io.airbyte.integrations.destination.gcs.jsonl; import static com.amazonaws.services.s3.internal.Constants.MB; -import static io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; +import static io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory.DEFAULT_PART_SIZE_MB; import static org.junit.jupiter.api.Assertions.assertEquals; import alex.mojaki.s3upload.StreamTransferManager; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.s3.S3FormatConfig; +import io.airbyte.cdk.integrations.destination.s3.util.StreamTransferManagerFactory; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.destination.gcs.GcsDestinationConfig; import io.airbyte.integrations.destination.gcs.util.ConfigTestUtils; -import io.airbyte.integrations.destination.s3.S3FormatConfig; -import io.airbyte.integrations.destination.s3.util.StreamTransferManagerFactory; import org.apache.commons.lang3.reflect.FieldUtils; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/connectors/destination-google-sheets/Dockerfile b/airbyte-integrations/connectors/destination-google-sheets/Dockerfile index 0d239b92faa6..e0f58d4a4541 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/Dockerfile +++ b/airbyte-integrations/connectors/destination-google-sheets/Dockerfile @@ -1,7 +1,8 @@ FROM python:3.9-slim # Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* +# Include socat binary in the connector image +RUN apt-get update && apt-get install -y bash && apt-get install -y socat && rm -rf /var/lib/apt/lists/* ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" @@ -13,5 +14,5 @@ COPY main.py ./ ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.2 +LABEL io.airbyte.version=0.2.3 LABEL io.airbyte.name=airbyte/destination-google-sheets diff --git a/airbyte-integrations/connectors/destination-google-sheets/README.md b/airbyte-integrations/connectors/destination-google-sheets/README.md index a39a52d52f30..419e1fcc6069 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/README.md +++ b/airbyte-integrations/connectors/destination-google-sheets/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:destination-google-sheets:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/destinations/google-sheets) to generate the necessary credentials. Then create a file `secrets/config_oauth.json` conforming to the `destination_google_sheets/spec.json` file. @@ -54,18 +48,19 @@ cat integration_tests/test_data/messages.txt | python main.py write --config sec ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/destination-google-sheets:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=destination-google-sheets build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:destination-google-sheets:airbyteDocker +An image will be built with the tag `airbyte/destination-google-sheets:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/destination-google-sheets:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,38 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-google-sheets:dev # messages.txt is a file containing line-separated JSON representing AirbyteMessages cat integration_tests/test_data/messages.txt | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-google-sheets:dev write --config /secrets/config_oauth.json --catalog /integration_tests/configured_catalog.json ``` -## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .'[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=destination-google-sheets test ``` -#### Acceptance Tests -Coming soon: -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:destination-google-sheets:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:destination-google-sheets:integrationTest -``` +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -116,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-google-sheets test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/google-sheets.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-google-sheets/build.gradle b/airbyte-integrations/connectors/destination-google-sheets/build.gradle deleted file mode 100644 index e4e62c47f2e2..000000000000 --- a/airbyte-integrations/connectors/destination-google-sheets/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'destination_google_sheets' -} diff --git a/airbyte-integrations/connectors/destination-google-sheets/metadata.yaml b/airbyte-integrations/connectors/destination-google-sheets/metadata.yaml index 223fda282f35..f8a2d30d507b 100644 --- a/airbyte-integrations/connectors/destination-google-sheets/metadata.yaml +++ b/airbyte-integrations/connectors/destination-google-sheets/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: destination definitionId: a4cbd2d1-8dbe-4818-b8bc-b90ad782d12a - dockerImageTag: 0.2.2 + dockerImageTag: 0.2.3 dockerRepository: airbyte/destination-google-sheets githubIssueLabel: destination-google-sheets icon: google-sheets.svg diff --git a/airbyte-integrations/connectors/destination-iceberg/.dockerignore b/airbyte-integrations/connectors/destination-iceberg/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-iceberg/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-iceberg/Dockerfile b/airbyte-integrations/connectors/destination-iceberg/Dockerfile deleted file mode 100644 index 283e51dd1f32..000000000000 --- a/airbyte-integrations/connectors/destination-iceberg/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte -ENV APPLICATION destination-iceberg - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte -ENV APPLICATION destination-iceberg - -ENV JAVA_OPTS="--add-opens java.base/java.lang=ALL-UNNAMED \ - --add-opens java.base/java.util=ALL-UNNAMED \ - --add-opens java.base/java.lang.reflect=ALL-UNNAMED \ - --add-opens java.base/java.text=ALL-UNNAMED \ - --add-opens java.base/sun.nio.ch=ALL-UNNAMED \ - --add-opens java.base/java.nio=ALL-UNNAMED " - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.4 -LABEL io.airbyte.name=airbyte/destination-iceberg diff --git a/airbyte-integrations/connectors/destination-iceberg/README.md b/airbyte-integrations/connectors/destination-iceberg/README.md index d995d3a0e083..be2a860e7823 100644 --- a/airbyte-integrations/connectors/destination-iceberg/README.md +++ b/airbyte-integrations/connectors/destination-iceberg/README.md @@ -32,11 +32,12 @@ the [instructions](https://docs.airbyte.io/connector-development#using-credentia Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-iceberg:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-iceberg:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` +Once built, the docker image name and tag on your host will be `airbyte/destination-iceberg:dev`. and `io.airbyte.version` `LABEL`s in the Dockerfile. @@ -82,13 +83,12 @@ To run acceptance and custom integration tests: ## Dependency Management ### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-iceberg test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/iceberg.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. -You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the -world. Now what? - -1. Make sure your changes are passing unit and integration tests. -2. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` - appropriately (we use [SemVer](https://semver.org/)). -3. Create a Pull Request. -4. Pat yourself on the back for being an awesome contributor. -5. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/destination-iceberg/build.gradle b/airbyte-integrations/connectors/destination-iceberg/build.gradle index a16bb53aa73f..37f06943d35d 100644 --- a/airbyte-integrations/connectors/destination-iceberg/build.gradle +++ b/airbyte-integrations/connectors/destination-iceberg/build.gradle @@ -1,18 +1,28 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.iceberg.IcebergDestination' } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation('org.apache.spark:spark-sql_2.13:3.3.2') { exclude(group: 'org.apache.hadoop', module: 'hadoop-common') @@ -39,11 +49,8 @@ dependencies { implementation "org.postgresql:postgresql:42.5.0" implementation "commons-collections:commons-collections:3.2.2" - testImplementation libs.connectors.testcontainers.postgresql - integrationTestJavaImplementation libs.connectors.testcontainers.postgresql - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-iceberg') + testImplementation libs.testcontainers.postgresql + integrationTestJavaImplementation libs.testcontainers.postgresql compileOnly 'org.projectlombok:lombok:1.18.24' annotationProcessor 'org.projectlombok:lombok:1.18.24' @@ -57,7 +64,3 @@ dependencies { test { jvmArgs = ['--add-opens=java.base/sun.nio.ch=ALL-UNNAMED', '--add-opens=java.base/java.nio=ALL-UNNAMED'] } - -task prepareKotlinBuildScriptModel { - -} diff --git a/airbyte-integrations/connectors/destination-iceberg/metadata.yaml b/airbyte-integrations/connectors/destination-iceberg/metadata.yaml index 659f3fcafe61..64e40d6491fd 100644 --- a/airbyte-integrations/connectors/destination-iceberg/metadata.yaml +++ b/airbyte-integrations/connectors/destination-iceberg/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: df65a8f3-9908-451b-aa9b-445462803560 - dockerImageTag: 0.1.4 + dockerImageTag: 0.1.5 dockerRepository: airbyte/destination-iceberg githubIssueLabel: destination-iceberg license: MIT diff --git a/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/IcebergConsumer.java b/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/IcebergConsumer.java index b67c277fef97..5f5ab3ae8830 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/IcebergConsumer.java +++ b/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/IcebergConsumer.java @@ -4,13 +4,13 @@ package io.airbyte.integrations.destination.iceberg; -import static io.airbyte.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_ID; -import static io.airbyte.integrations.base.JavaBaseConstants.COLUMN_NAME_DATA; -import static io.airbyte.integrations.base.JavaBaseConstants.COLUMN_NAME_EMITTED_AT; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_ID; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_DATA; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_EMITTED_AT; import static org.apache.logging.log4j.util.Strings.isNotBlank; +import io.airbyte.cdk.integrations.base.CommitOnStateAirbyteMessageConsumer; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.CommitOnStateAirbyteMessageConsumer; import io.airbyte.integrations.destination.iceberg.config.WriteConfig; import io.airbyte.integrations.destination.iceberg.config.catalog.IcebergCatalogConfig; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/IcebergDestination.java b/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/IcebergDestination.java index a6e61d149e40..37e6bbb71ce6 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/IcebergDestination.java +++ b/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/IcebergDestination.java @@ -6,10 +6,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; import io.airbyte.integrations.destination.iceberg.config.catalog.IcebergCatalogConfig; import io.airbyte.integrations.destination.iceberg.config.catalog.IcebergCatalogConfigFactory; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; diff --git a/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/WriteConfig.java b/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/WriteConfig.java index 4276d79f86e1..571cc651d8af 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/WriteConfig.java +++ b/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/WriteConfig.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.iceberg.config; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.integrations.destination.iceberg.IcebergConstants; import java.io.Serializable; import java.util.ArrayList; diff --git a/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/catalog/IcebergCatalogConfig.java b/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/catalog/IcebergCatalogConfig.java index 3fafd16d0f51..830599311d24 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/catalog/IcebergCatalogConfig.java +++ b/airbyte-integrations/connectors/destination-iceberg/src/main/java/io/airbyte/integrations/destination/iceberg/config/catalog/IcebergCatalogConfig.java @@ -6,7 +6,7 @@ import static org.apache.commons.lang3.StringUtils.isBlank; -import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.iceberg.IcebergConstants; import io.airbyte.integrations.destination.iceberg.config.format.FormatConfig; import io.airbyte.integrations.destination.iceberg.config.storage.StorageConfig; diff --git a/airbyte-integrations/connectors/destination-iceberg/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-iceberg/src/main/resources/spec.json index 503b24a6cf71..245874d890e0 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-iceberg/src/main/resources/spec.json @@ -214,31 +214,40 @@ "description": "The region of the S3 bucket. See here for all region codes.", "enum": [ "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", "af-south-1", "ap-east-1", - "ap-south-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", + "ap-south-1", + "ap-south-2", "ap-southeast-1", "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", "ca-central-1", + "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", + "eu-central-2", "eu-north-1", "eu-south-1", + "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", - "sa-east-1", + "il-central-1", + "me-central-1", "me-south-1", + "sa-east-1", + "sa-east-1", + "us-east-1", + "us-east-2", "us-gov-east-1", - "us-gov-west-1" + "us-gov-west-1", + "us-west-1", + "us-west-2" ], "order": 3 }, diff --git a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/IcebergIntegrationTestUtil.java b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/IcebergIntegrationTestUtil.java index efc18cc015f7..de48f264983a 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/IcebergIntegrationTestUtil.java +++ b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/IcebergIntegrationTestUtil.java @@ -12,10 +12,10 @@ import com.amazonaws.services.s3.model.Bucket; import com.fasterxml.jackson.databind.JsonNode; import com.github.dockerjava.api.model.ContainerNetwork; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.integrations.destination.iceberg.config.catalog.IcebergCatalogConfig; import io.airbyte.integrations.destination.iceberg.config.catalog.IcebergCatalogConfigFactory; import io.airbyte.integrations.destination.iceberg.config.storage.S3Config; diff --git a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hadoop/BaseIcebergHadoopCatalogS3IntegrationTest.java b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hadoop/BaseIcebergHadoopCatalogS3IntegrationTest.java index db297bb4cab8..b681c5629552 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hadoop/BaseIcebergHadoopCatalogS3IntegrationTest.java +++ b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hadoop/BaseIcebergHadoopCatalogS3IntegrationTest.java @@ -23,12 +23,12 @@ import static java.util.Map.ofEntries; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.destination.iceberg.IcebergIntegrationTestUtil; import io.airbyte.integrations.destination.iceberg.config.format.DataFileFormat; import io.airbyte.integrations.destination.iceberg.container.MinioContainer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.util.HostPortResolver; import java.util.HashSet; import java.util.List; import java.util.Map; diff --git a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hive/IcebergHiveCatalogS3AvroIntegrationTest.java b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hive/IcebergHiveCatalogS3AvroIntegrationTest.java index 734184a887a5..af0dcd91f826 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hive/IcebergHiveCatalogS3AvroIntegrationTest.java +++ b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hive/IcebergHiveCatalogS3AvroIntegrationTest.java @@ -7,10 +7,10 @@ import static io.airbyte.integrations.destination.iceberg.IcebergIntegrationTestUtil.ICEBERG_IMAGE_NAME; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.integrations.destination.iceberg.IcebergIntegrationTestUtil; import io.airbyte.integrations.destination.iceberg.config.format.DataFileFormat; import io.airbyte.integrations.destination.iceberg.container.HiveMetastoreS3PostgresCompose; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import java.util.HashSet; import java.util.List; import org.junit.jupiter.api.AfterAll; diff --git a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hive/IcebergHiveCatalogS3ParquetIntegrationTest.java b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hive/IcebergHiveCatalogS3ParquetIntegrationTest.java index 9fca8b9682aa..31776f7b9f6f 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hive/IcebergHiveCatalogS3ParquetIntegrationTest.java +++ b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/hive/IcebergHiveCatalogS3ParquetIntegrationTest.java @@ -7,10 +7,10 @@ import static io.airbyte.integrations.destination.iceberg.IcebergIntegrationTestUtil.ICEBERG_IMAGE_NAME; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.integrations.destination.iceberg.IcebergIntegrationTestUtil; import io.airbyte.integrations.destination.iceberg.config.format.DataFileFormat; import io.airbyte.integrations.destination.iceberg.container.HiveMetastoreS3PostgresCompose; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import java.util.HashSet; import java.util.List; import org.junit.jupiter.api.AfterAll; diff --git a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/jdbc/BaseIcebergJdbcCatalogS3IntegrationTest.java b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/jdbc/BaseIcebergJdbcCatalogS3IntegrationTest.java index 3839f0d3fb8f..b76279c6480c 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/jdbc/BaseIcebergJdbcCatalogS3IntegrationTest.java +++ b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/jdbc/BaseIcebergJdbcCatalogS3IntegrationTest.java @@ -28,12 +28,12 @@ import static java.util.Map.ofEntries; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.destination.iceberg.IcebergIntegrationTestUtil; import io.airbyte.integrations.destination.iceberg.config.format.DataFileFormat; import io.airbyte.integrations.destination.iceberg.container.MinioContainer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.util.HostPortResolver; import java.util.HashSet; import java.util.List; import java.util.Map; diff --git a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/rest/BaseIcebergRESTCatalogS3IntegrationTest.java b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/rest/BaseIcebergRESTCatalogS3IntegrationTest.java index 49c2f66986fe..c8f05d20c2b5 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/rest/BaseIcebergRESTCatalogS3IntegrationTest.java +++ b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/java/io/airbyte/integrations/destination/iceberg/rest/BaseIcebergRESTCatalogS3IntegrationTest.java @@ -7,10 +7,10 @@ import static io.airbyte.integrations.destination.iceberg.IcebergIntegrationTestUtil.ICEBERG_IMAGE_NAME; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.integrations.destination.iceberg.IcebergIntegrationTestUtil; import io.airbyte.integrations.destination.iceberg.config.format.DataFileFormat; import io.airbyte.integrations.destination.iceberg.container.RESTServerWithMinioCompose; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import java.util.HashSet; import java.util.List; import org.junit.jupiter.api.AfterAll; diff --git a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/resources/hive-metastore-compose.yml b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/resources/hive-metastore-compose.yml index 7f6e40e7ce84..cab85d023933 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/resources/hive-metastore-compose.yml +++ b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/resources/hive-metastore-compose.yml @@ -24,4 +24,4 @@ services: entrypoint: sh -c "bin/schematool -initSchema -dbType postgres && bin/hive --service metastore" depends_on: - postgres - - minio \ No newline at end of file + - minio diff --git a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/resources/rest-catalog-compose.yml b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/resources/rest-catalog-compose.yml index 9eb4ad04d010..e223618f3468 100644 --- a/airbyte-integrations/connectors/destination-iceberg/src/test-integration/resources/rest-catalog-compose.yml +++ b/airbyte-integrations/connectors/destination-iceberg/src/test-integration/resources/rest-catalog-compose.yml @@ -39,4 +39,4 @@ services: tail -f /dev/null " networks: - iceberg_net: \ No newline at end of file + iceberg_net: diff --git a/airbyte-integrations/connectors/destination-kafka/.dockerignore b/airbyte-integrations/connectors/destination-kafka/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-kafka/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-kafka/Dockerfile b/airbyte-integrations/connectors/destination-kafka/Dockerfile deleted file mode 100644 index 488e2fe3185f..000000000000 --- a/airbyte-integrations/connectors/destination-kafka/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-kafka - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-kafka - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.10 -LABEL io.airbyte.name=airbyte/destination-kafka diff --git a/airbyte-integrations/connectors/destination-kafka/README.md b/airbyte-integrations/connectors/destination-kafka/README.md index c5d7bfda9def..56aabe505cda 100644 --- a/airbyte-integrations/connectors/destination-kafka/README.md +++ b/airbyte-integrations/connectors/destination-kafka/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-kafka:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-kafka:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-kafka:dev`. the Dockerfile. #### Run @@ -61,8 +62,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-kafka test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/kafka.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-kafka/build.gradle b/airbyte-integrations/connectors/destination-kafka/build.gradle index 5865a6ac97e2..69da18f35960 100644 --- a/airbyte-integrations/connectors/destination-kafka/build.gradle +++ b/airbyte-integrations/connectors/destination-kafka/build.gradle @@ -1,27 +1,32 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.kafka.KafkaDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') implementation 'org.apache.kafka:kafka-clients:2.8.0' implementation 'org.apache.kafka:connect-json:2.8.0' - testImplementation project(':airbyte-integrations:bases:standard-destination-test') - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-kafka') - integrationTestJavaImplementation libs.connectors.testcontainers.kafka - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + integrationTestJavaImplementation libs.testcontainers.kafka } diff --git a/airbyte-integrations/connectors/destination-kafka/src/main/java/io/airbyte/integrations/destination/kafka/KafkaDestination.java b/airbyte-integrations/connectors/destination-kafka/src/main/java/io/airbyte/integrations/destination/kafka/KafkaDestination.java index dba32e7ed615..1e202b62c278 100644 --- a/airbyte-integrations/connectors/destination-kafka/src/main/java/io/airbyte/integrations/destination/kafka/KafkaDestination.java +++ b/airbyte-integrations/connectors/destination-kafka/src/main/java/io/airbyte/integrations/destination/kafka/KafkaDestination.java @@ -6,13 +6,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-kafka/src/main/java/io/airbyte/integrations/destination/kafka/KafkaRecordConsumer.java b/airbyte-integrations/connectors/destination-kafka/src/main/java/io/airbyte/integrations/destination/kafka/KafkaRecordConsumer.java index 838eaa1d7d01..0a1db503ee0e 100644 --- a/airbyte-integrations/connectors/destination-kafka/src/main/java/io/airbyte/integrations/destination/kafka/KafkaRecordConsumer.java +++ b/airbyte-integrations/connectors/destination-kafka/src/main/java/io/airbyte/integrations/destination/kafka/KafkaRecordConsumer.java @@ -6,9 +6,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; -import io.airbyte.integrations.destination.NamingConventionTransformer; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; diff --git a/airbyte-integrations/connectors/destination-kafka/src/test-integration/java/io/airbyte/integrations/destination/kafka/KafkaDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-kafka/src/test-integration/java/io/airbyte/integrations/destination/kafka/KafkaDestinationAcceptanceTest.java index 9de3ea4ab3d0..4f78cd3d8132 100644 --- a/airbyte-integrations/connectors/destination-kafka/src/test-integration/java/io/airbyte/integrations/destination/kafka/KafkaDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-kafka/src/test-integration/java/io/airbyte/integrations/destination/kafka/KafkaDestinationAcceptanceTest.java @@ -8,14 +8,14 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.time.Duration; import java.util.ArrayList; import java.util.Collections; diff --git a/airbyte-integrations/connectors/destination-kafka/src/test/java/io/airbyte/integrations/destination/kafka/KafkaRecordConsumerTest.java b/airbyte-integrations/connectors/destination-kafka/src/test/java/io/airbyte/integrations/destination/kafka/KafkaRecordConsumerTest.java index a1a4c6f89428..3075390cfaf9 100644 --- a/airbyte-integrations/connectors/destination-kafka/src/test/java/io/airbyte/integrations/destination/kafka/KafkaRecordConsumerTest.java +++ b/airbyte-integrations/connectors/destination-kafka/src/test/java/io/airbyte/integrations/destination/kafka/KafkaRecordConsumerTest.java @@ -11,11 +11,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.PerStreamStateMessageTest; import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.PerStreamStateMessageTest; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-keen/.dockerignore b/airbyte-integrations/connectors/destination-keen/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-keen/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-keen/Dockerfile b/airbyte-integrations/connectors/destination-keen/Dockerfile deleted file mode 100644 index 0f09ca580aa3..000000000000 --- a/airbyte-integrations/connectors/destination-keen/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-keen - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-keen - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.2.4 -LABEL io.airbyte.name=airbyte/destination-keen diff --git a/airbyte-integrations/connectors/destination-keen/build.gradle b/airbyte-integrations/connectors/destination-keen/build.gradle index eba011e214d8..777118dbb370 100644 --- a/airbyte-integrations/connectors/destination-keen/build.gradle +++ b/airbyte-integrations/connectors/destination-keen/build.gradle @@ -1,26 +1,33 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.keen.KeenDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') implementation 'org.apache.kafka:kafka-clients:2.8.0' implementation 'com.joestelmach:natty:0.11' - testImplementation project(':airbyte-integrations:bases:standard-destination-test') - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-keen') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + // TODO: remove this dependency + implementation libs.google.cloud.storage } diff --git a/airbyte-integrations/connectors/destination-keen/src/main/java/io/airbyte/integrations/destination/keen/KeenDestination.java b/airbyte-integrations/connectors/destination-keen/src/main/java/io/airbyte/integrations/destination/keen/KeenDestination.java index 79e60be08d63..6e9f94df646e 100644 --- a/airbyte-integrations/connectors/destination-keen/src/main/java/io/airbyte/integrations/destination/keen/KeenDestination.java +++ b/airbyte-integrations/connectors/destination-keen/src/main/java/io/airbyte/integrations/destination/keen/KeenDestination.java @@ -15,10 +15,10 @@ import static org.apache.kafka.common.security.plain.internals.PlainSaslServer.PLAIN_MECHANISM; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-keen/src/main/java/io/airbyte/integrations/destination/keen/KeenRecordsConsumer.java b/airbyte-integrations/connectors/destination-keen/src/main/java/io/airbyte/integrations/destination/keen/KeenRecordsConsumer.java index 1578009e5c87..62fef10b6141 100644 --- a/airbyte-integrations/connectors/destination-keen/src/main/java/io/airbyte/integrations/destination/keen/KeenRecordsConsumer.java +++ b/airbyte-integrations/connectors/destination-keen/src/main/java/io/airbyte/integrations/destination/keen/KeenRecordsConsumer.java @@ -9,8 +9,8 @@ import static io.airbyte.integrations.destination.keen.KeenDestination.INFER_TIMESTAMP; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; diff --git a/airbyte-integrations/connectors/destination-keen/src/test-integration/java/io/airbyte/integrations/destination/keen/KeenDestinationTest.java b/airbyte-integrations/connectors/destination-keen/src/test-integration/java/io/airbyte/integrations/destination/keen/KeenDestinationTest.java index 07c5d7eababa..42f0242110ff 100644 --- a/airbyte-integrations/connectors/destination-keen/src/test-integration/java/io/airbyte/integrations/destination/keen/KeenDestinationTest.java +++ b/airbyte-integrations/connectors/destination-keen/src/test-integration/java/io/airbyte/integrations/destination/keen/KeenDestinationTest.java @@ -11,11 +11,11 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.api.client.util.Lists; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import java.nio.file.Files; diff --git a/airbyte-integrations/connectors/destination-keen/src/test/java/io/airbyte/integrations/destination/keen/KeenRecordConsumerTest.java b/airbyte-integrations/connectors/destination-keen/src/test/java/io/airbyte/integrations/destination/keen/KeenRecordConsumerTest.java index b1295d0a5a5f..a6dd7852b1f6 100644 --- a/airbyte-integrations/connectors/destination-keen/src/test/java/io/airbyte/integrations/destination/keen/KeenRecordConsumerTest.java +++ b/airbyte-integrations/connectors/destination-keen/src/test/java/io/airbyte/integrations/destination/keen/KeenRecordConsumerTest.java @@ -9,9 +9,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.standardtest.destination.PerStreamStateMessageTest; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; -import io.airbyte.integrations.standardtest.destination.PerStreamStateMessageTest; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-kinesis/.dockerignore b/airbyte-integrations/connectors/destination-kinesis/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-kinesis/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-kinesis/Dockerfile b/airbyte-integrations/connectors/destination-kinesis/Dockerfile deleted file mode 100644 index dc881a2c96f9..000000000000 --- a/airbyte-integrations/connectors/destination-kinesis/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-kinesis - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-kinesis - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.5 -LABEL io.airbyte.name=airbyte/destination-kinesis diff --git a/airbyte-integrations/connectors/destination-kinesis/README.md b/airbyte-integrations/connectors/destination-kinesis/README.md index 5f2fd9df5b74..d0647dfb2ef8 100644 --- a/airbyte-integrations/connectors/destination-kinesis/README.md +++ b/airbyte-integrations/connectors/destination-kinesis/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-kinesis:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-kinesis:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-kinesis:dev`. the Dockerfile. #### Run @@ -61,8 +62,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-kinesis test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/kinesis.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-kinesis/build.gradle b/airbyte-integrations/connectors/destination-kinesis/build.gradle index 298dd90a9543..3abe284a89a8 100644 --- a/airbyte-integrations/connectors/destination-kinesis/build.gradle +++ b/airbyte-integrations/connectors/destination-kinesis/build.gradle @@ -1,9 +1,23 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.kinesis.KinesisDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] @@ -14,19 +28,10 @@ def testContainersVersion = '1.16.2' def assertVersion = '3.21.0' dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) // https://mvnrepository.com/artifact/software.amazon.awssdk/kinesis implementation "software.amazon.awssdk:kinesis:${kinesisVersion}" testImplementation "org.assertj:assertj-core:${assertVersion}" testImplementation "org.testcontainers:localstack:${testContainersVersion}" - testImplementation project(':airbyte-integrations:bases:standard-destination-test') - - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-kinesis') } diff --git a/airbyte-integrations/connectors/destination-kinesis/docker-compose.yml b/airbyte-integrations/connectors/destination-kinesis/docker-compose.yml index cb9e1c5e7236..64bafc3c5a95 100644 --- a/airbyte-integrations/connectors/destination-kinesis/docker-compose.yml +++ b/airbyte-integrations/connectors/destination-kinesis/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3.7' +version: "3.7" services: kinesis: diff --git a/airbyte-integrations/connectors/destination-kinesis/src/main/java/io/airbyte/integrations/destination/kinesis/KinesisDestination.java b/airbyte-integrations/connectors/destination-kinesis/src/main/java/io/airbyte/integrations/destination/kinesis/KinesisDestination.java index b64dba2e304a..1c4cdd16ff5f 100644 --- a/airbyte-integrations/connectors/destination-kinesis/src/main/java/io/airbyte/integrations/destination/kinesis/KinesisDestination.java +++ b/airbyte-integrations/connectors/destination-kinesis/src/main/java/io/airbyte/integrations/destination/kinesis/KinesisDestination.java @@ -5,10 +5,10 @@ package io.airbyte.integrations.destination.kinesis; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/connectors/destination-kinesis/src/main/java/io/airbyte/integrations/destination/kinesis/KinesisMessageConsumer.java b/airbyte-integrations/connectors/destination-kinesis/src/main/java/io/airbyte/integrations/destination/kinesis/KinesisMessageConsumer.java index b129a8c13da9..071db77d34d3 100644 --- a/airbyte-integrations/connectors/destination-kinesis/src/main/java/io/airbyte/integrations/destination/kinesis/KinesisMessageConsumer.java +++ b/airbyte-integrations/connectors/destination-kinesis/src/main/java/io/airbyte/integrations/destination/kinesis/KinesisMessageConsumer.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.kinesis; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/connectors/destination-kinesis/src/main/java/io/airbyte/integrations/destination/kinesis/KinesisNameTransformer.java b/airbyte-integrations/connectors/destination-kinesis/src/main/java/io/airbyte/integrations/destination/kinesis/KinesisNameTransformer.java index 419833f56fb4..f9d002099b85 100644 --- a/airbyte-integrations/connectors/destination-kinesis/src/main/java/io/airbyte/integrations/destination/kinesis/KinesisNameTransformer.java +++ b/airbyte-integrations/connectors/destination-kinesis/src/main/java/io/airbyte/integrations/destination/kinesis/KinesisNameTransformer.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.kinesis; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; /** * KinesisNameTransformer class for creating Kinesis stream names. diff --git a/airbyte-integrations/connectors/destination-kinesis/src/test-integration/java/io/airbyte/integrations/destination/kinesis/KinesisDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-kinesis/src/test-integration/java/io/airbyte/integrations/destination/kinesis/KinesisDestinationAcceptanceTest.java index 965f4d7c1d59..0a30d3555420 100644 --- a/airbyte-integrations/connectors/destination-kinesis/src/test-integration/java/io/airbyte/integrations/destination/kinesis/KinesisDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-kinesis/src/test-integration/java/io/airbyte/integrations/destination/kinesis/KinesisDestinationAcceptanceTest.java @@ -5,10 +5,10 @@ package io.airbyte.integrations.destination.kinesis; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.util.Comparator; import java.util.HashSet; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-kinesis/src/test/java/io/airbyte/integrations/destination/kinesis/KinesisRecordConsumerTest.java b/airbyte-integrations/connectors/destination-kinesis/src/test/java/io/airbyte/integrations/destination/kinesis/KinesisRecordConsumerTest.java index 7586c1be0cff..f2ac46a15cc3 100644 --- a/airbyte-integrations/connectors/destination-kinesis/src/test/java/io/airbyte/integrations/destination/kinesis/KinesisRecordConsumerTest.java +++ b/airbyte-integrations/connectors/destination-kinesis/src/test/java/io/airbyte/integrations/destination/kinesis/KinesisRecordConsumerTest.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.kinesis; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; -import io.airbyte.integrations.standardtest.destination.PerStreamStateMessageTest; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.standardtest.destination.PerStreamStateMessageTest; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import java.util.function.Consumer; diff --git a/airbyte-integrations/connectors/destination-kvdb/README.md b/airbyte-integrations/connectors/destination-kvdb/README.md index 79d3025318f2..b834894111b6 100644 --- a/airbyte-integrations/connectors/destination-kvdb/README.md +++ b/airbyte-integrations/connectors/destination-kvdb/README.md @@ -52,19 +52,20 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/destination-kvdb:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=destination-kvdb build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:destination-kvdb:airbyteDocker -``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +An image will be built with the tag `airbyte/destination-kvdb:dev`. +**Via `docker build`:** +```bash +docker build -t airbyte/destination-kvdb:dev . +``` #### Run Then run any of the connector commands as follows: ``` @@ -93,19 +94,12 @@ Place custom tests inside `integration_tests/` folder, then, from the connector python -m pytest integration_tests ``` #### Acceptance Tests -Coming soon: - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:destination-kvdb:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:destination-kvdb:integrationTest +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=destination-kvdb test ``` + ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -114,8 +108,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-kvdb test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/kvdb.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-kvdb/build.gradle b/airbyte-integrations/connectors/destination-kvdb/build.gradle deleted file mode 100644 index d2f41e640883..000000000000 --- a/airbyte-integrations/connectors/destination-kvdb/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'destination_kvdb' -} diff --git a/airbyte-integrations/connectors/destination-kvdb/setup.py b/airbyte-integrations/connectors/destination-kvdb/setup.py index 45450b3f7cce..dab5520718ab 100644 --- a/airbyte-integrations/connectors/destination-kvdb/setup.py +++ b/airbyte-integrations/connectors/destination-kvdb/setup.py @@ -5,7 +5,10 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk==0.1.6-rc1", "requests"] +MAIN_REQUIREMENTS = [ + "airbyte-cdk", + "requests", +] TEST_REQUIREMENTS = ["pytest~=6.1"] diff --git a/airbyte-integrations/connectors/destination-langchain/Dockerfile b/airbyte-integrations/connectors/destination-langchain/Dockerfile index 1472c50f57d4..30452c2628ac 100644 --- a/airbyte-integrations/connectors/destination-langchain/Dockerfile +++ b/airbyte-integrations/connectors/destination-langchain/Dockerfile @@ -42,5 +42,5 @@ COPY destination_langchain ./destination_langchain ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.0.8 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/destination-langchain diff --git a/airbyte-integrations/connectors/destination-langchain/README.md b/airbyte-integrations/connectors/destination-langchain/README.md index 5de181d487d8..76903c7373f9 100644 --- a/airbyte-integrations/connectors/destination-langchain/README.md +++ b/airbyte-integrations/connectors/destination-langchain/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:destination-langchain:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/destinations/langchain) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_langchain/spec.json` file. @@ -54,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/destination-langchain:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=destination-langchain build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:destination-langchain:airbyteDocker +An image will be built with the tag `airbyte/destination-langchain:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/destination-langchain:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,38 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-langchain:dev che # messages.jsonl is a file containing line-separated JSON representing AirbyteMessages cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-langchain:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=destination-langchain test ``` -#### Acceptance Tests -Coming soon: -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:destination-langchain:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:destination-langchain:integrationTest -``` +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -116,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-langchain test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/langchain.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-langchain/build.gradle b/airbyte-integrations/connectors/destination-langchain/build.gradle deleted file mode 100644 index 9ac8fa2191ac..000000000000 --- a/airbyte-integrations/connectors/destination-langchain/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'destination_langchain' -} diff --git a/airbyte-integrations/connectors/destination-langchain/destination_langchain/config.py b/airbyte-integrations/connectors/destination-langchain/destination_langchain/config.py index a5b00b0cfeaf..375595d2907d 100644 --- a/airbyte-integrations/connectors/destination-langchain/destination_langchain/config.py +++ b/airbyte-integrations/connectors/destination-langchain/destination_langchain/config.py @@ -7,6 +7,7 @@ from typing import List, Literal, Optional, Union import dpath.util +from airbyte_cdk.destinations.vector_db_based.embedder import FakeEmbeddingConfigModel, OpenAIEmbeddingConfigModel from jsonschema import RefResolver from pydantic import BaseModel, Field @@ -35,27 +36,6 @@ class Config: schema_extra = {"group": "processing"} -class OpenAIEmbeddingConfigModel(BaseModel): - mode: Literal["openai"] = Field("openai", const=True) - openai_key: str = Field(..., title="OpenAI API key", airbyte_secret=True) - - class Config: - title = "OpenAI" - schema_extra = { - "description": "Use the OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." - } - - -class FakeEmbeddingConfigModel(BaseModel): - mode: Literal["fake"] = Field("fake", const=True) - - class Config: - title = "Fake" - schema_extra = { - "description": "Use a fake embedding made out of random vectors with 1536 embedding dimensions. This is useful for testing the data pipeline without incurring any costs." - } - - class PineconeIndexingModel(BaseModel): mode: Literal["pinecone"] = Field("pinecone", const=True) pinecone_key: str = Field(..., title="Pinecone API key", airbyte_secret=True) diff --git a/airbyte-integrations/connectors/destination-langchain/destination_langchain/destination.py b/airbyte-integrations/connectors/destination-langchain/destination_langchain/destination.py index 94e3ade7b8d4..f6e3ea4446ed 100644 --- a/airbyte-integrations/connectors/destination-langchain/destination_langchain/destination.py +++ b/airbyte-integrations/connectors/destination-langchain/destination_langchain/destination.py @@ -7,6 +7,7 @@ from airbyte_cdk import AirbyteLogger from airbyte_cdk.destinations import Destination +from airbyte_cdk.destinations.vector_db_based import Embedder, FakeEmbedder, OpenAIEmbedder from airbyte_cdk.models import ( AirbyteConnectionStatus, AirbyteMessage, @@ -20,7 +21,6 @@ from destination_langchain.batcher import Batcher from destination_langchain.config import ConfigModel from destination_langchain.document_processor import DocumentProcessor -from destination_langchain.embedder import Embedder, FakeEmbedder, OpenAIEmbedder from destination_langchain.indexer import ChromaLocalIndexer, DocArrayHnswSearchIndexer, Indexer, PineconeIndexer from langchain.document_loaders.base import Document diff --git a/airbyte-integrations/connectors/destination-langchain/destination_langchain/document_processor.py b/airbyte-integrations/connectors/destination-langchain/destination_langchain/document_processor.py index 460a7614bf37..298de8c7dd28 100644 --- a/airbyte-integrations/connectors/destination-langchain/destination_langchain/document_processor.py +++ b/airbyte-integrations/connectors/destination-langchain/destination_langchain/document_processor.py @@ -84,7 +84,7 @@ def _extract_metadata(self, record: AirbyteRecordMessage) -> dict: metadata[METADATA_STREAM_FIELD] = stream_identifier # if the sync mode is deduping, use the primary key to upsert existing records instead of appending new ones if current_stream.primary_key and current_stream.destination_sync_mode == DestinationSyncMode.append_dedup: - metadata[METADATA_RECORD_ID_FIELD] = self._extract_primary_key(record, current_stream) + metadata[METADATA_RECORD_ID_FIELD] = f"{stream_identifier}_{self._extract_primary_key(record, current_stream)}" return metadata def _extract_primary_key(self, record: AirbyteRecordMessage, stream: ConfiguredAirbyteStream) -> dict: diff --git a/airbyte-integrations/connectors/destination-langchain/destination_langchain/embedder.py b/airbyte-integrations/connectors/destination-langchain/destination_langchain/embedder.py deleted file mode 100644 index 55e0674e88a3..000000000000 --- a/airbyte-integrations/connectors/destination-langchain/destination_langchain/embedder.py +++ /dev/null @@ -1,78 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from abc import ABC, abstractmethod -from typing import Optional - -from destination_langchain.config import FakeEmbeddingConfigModel, OpenAIEmbeddingConfigModel -from destination_langchain.utils import format_exception -from langchain.embeddings.base import Embeddings -from langchain.embeddings.fake import FakeEmbeddings -from langchain.embeddings.openai import OpenAIEmbeddings - - -class Embedder(ABC): - def __init__(self): - pass - - @abstractmethod - def check(self) -> Optional[str]: - pass - - @property - @abstractmethod - def langchain_embeddings(self) -> Embeddings: - pass - - @property - @abstractmethod - def embedding_dimensions(self) -> int: - pass - - -OPEN_AI_VECTOR_SIZE = 1536 - - -class OpenAIEmbedder(Embedder): - def __init__(self, config: OpenAIEmbeddingConfigModel): - super().__init__() - self.embeddings = OpenAIEmbeddings(openai_api_key=config.openai_key, chunk_size=8191) - - def check(self) -> Optional[str]: - try: - self.embeddings.embed_query("test") - except Exception as e: - return format_exception(e) - return None - - @property - def langchain_embeddings(self) -> Embeddings: - return self.embeddings - - @property - def embedding_dimensions(self) -> int: - # vector size produced by text-embedding-ada-002 model - return OPEN_AI_VECTOR_SIZE - - -class FakeEmbedder(Embedder): - def __init__(self, config: FakeEmbeddingConfigModel): - super().__init__() - self.embeddings = FakeEmbeddings(size=OPEN_AI_VECTOR_SIZE) - - def check(self) -> Optional[str]: - try: - self.embeddings.embed_query("test") - except Exception as e: - return format_exception(e) - return None - - @property - def langchain_embeddings(self) -> Embeddings: - return self.embeddings - - @property - def embedding_dimensions(self) -> int: - # use same vector size as for OpenAI embeddings to keep it realistic - return OPEN_AI_VECTOR_SIZE diff --git a/airbyte-integrations/connectors/destination-langchain/destination_langchain/indexer.py b/airbyte-integrations/connectors/destination-langchain/destination_langchain/indexer.py index 6f8fbaeafb0c..d14b39ea3599 100644 --- a/airbyte-integrations/connectors/destination-langchain/destination_langchain/indexer.py +++ b/airbyte-integrations/connectors/destination-langchain/destination_langchain/indexer.py @@ -9,11 +9,11 @@ from typing import Any, List, Optional import pinecone +from airbyte_cdk.destinations.vector_db_based import Embedder from airbyte_cdk.models import ConfiguredAirbyteCatalog from airbyte_cdk.models.airbyte_protocol import AirbyteLogMessage, AirbyteMessage, DestinationSyncMode, Level, Type from destination_langchain.config import ChromaLocalIndexingModel, DocArrayHnswSearchIndexingModel, PineconeIndexingModel from destination_langchain.document_processor import METADATA_RECORD_ID_FIELD, METADATA_STREAM_FIELD -from destination_langchain.embedder import Embedder from destination_langchain.measure_time import measure_time from destination_langchain.utils import format_exception from langchain.document_loaders.base import Document @@ -66,7 +66,7 @@ def __init__(self, config: PineconeIndexingModel, embedder: Embedder): super().__init__(config, embedder) pinecone.init(api_key=config.pinecone_key, environment=config.pinecone_environment, threaded=True) self.pinecone_index = pinecone.Index(config.index, pool_threads=10) - self.embed_fn = measure_time(self.embedder.langchain_embeddings.embed_documents) + self.embed_fn = measure_time(self.embedder.embeddings.embed_documents) def pre_sync(self, catalog: ConfiguredAirbyteCatalog): index_description = pinecone.describe_index(self.config.index) @@ -134,7 +134,7 @@ def __init__(self, config: DocArrayHnswSearchIndexingModel, embedder: Embedder): def _init_vectorstore(self): self.vectorstore = DocArrayHnswSearch.from_params( - embedding=self.embedder.langchain_embeddings, work_dir=self.config.destination_path, n_dim=self.embedder.embedding_dimensions + embedding=self.embedder.embeddings, work_dir=self.config.destination_path, n_dim=self.embedder.embedding_dimensions ) def pre_sync(self, catalog: ConfiguredAirbyteCatalog): @@ -172,7 +172,7 @@ def __init__(self, config: ChromaLocalIndexingModel, embedder: Embedder): def _init_vectorstore(self): self.vectorstore = Chroma( collection_name=self.config.collection_name, - embedding_function=self.embedder.langchain_embeddings, + embedding_function=self.embedder.embeddings, persist_directory=self.config.destination_path, ) diff --git a/airbyte-integrations/connectors/destination-langchain/integration_tests/chroma_integration_test.py b/airbyte-integrations/connectors/destination-langchain/integration_tests/chroma_integration_test.py index c5fa445add06..0add90e48e64 100644 --- a/airbyte-integrations/connectors/destination-langchain/integration_tests/chroma_integration_test.py +++ b/airbyte-integrations/connectors/destination-langchain/integration_tests/chroma_integration_test.py @@ -6,10 +6,10 @@ import logging import chromadb +from airbyte_cdk.destinations.vector_db_based.embedder import OPEN_AI_VECTOR_SIZE from airbyte_cdk.models import DestinationSyncMode, Status from chromadb.api.types import QueryResult from destination_langchain.destination import DestinationLangchain -from destination_langchain.embedder import OPEN_AI_VECTOR_SIZE from integration_tests.base_integration_test import LocalIntegrationTest from langchain.embeddings import OpenAIEmbeddings from langchain.vectorstores import Chroma @@ -44,7 +44,7 @@ def test_write(self): incremental_catalog = self._get_configured_catalog(DestinationSyncMode.append_dedup) list(destination.write(self.config, incremental_catalog, [self._record("mystream", "Cats are nice", 2), first_state_message])) chroma_result: QueryResult = self.chroma_client.get_collection("langchain").query( - query_embeddings=[0] * OPEN_AI_VECTOR_SIZE, n_results=10, where={"_record_id": "2"}, include=["documents"] + query_embeddings=[0] * OPEN_AI_VECTOR_SIZE, n_results=10, where={"_record_id": "mystream_2"}, include=["documents"] ) assert len(chroma_result["documents"][0]) == 1 assert chroma_result["documents"][0] == ["str_col: Cats are nice"] @@ -53,4 +53,4 @@ def test_write(self): embeddings = OpenAIEmbeddings(openai_api_key=self.config["embedding"]["openai_key"]) vector_store = Chroma(embedding_function=embeddings, persist_directory=self.temp_dir) result = vector_store.similarity_search("feline animals", 1) - assert result[0].metadata["_record_id"] == "2" + assert result[0].metadata["_record_id"] == "mystream_2" diff --git a/airbyte-integrations/connectors/destination-langchain/integration_tests/docarray_integration_test.py b/airbyte-integrations/connectors/destination-langchain/integration_tests/docarray_integration_test.py index ba93129e02bc..f3c27e15b83c 100644 --- a/airbyte-integrations/connectors/destination-langchain/integration_tests/docarray_integration_test.py +++ b/airbyte-integrations/connectors/destination-langchain/integration_tests/docarray_integration_test.py @@ -4,9 +4,9 @@ import logging +from airbyte_cdk.destinations.vector_db_based.embedder import OPEN_AI_VECTOR_SIZE from airbyte_cdk.models import DestinationSyncMode, Status from destination_langchain.destination import DestinationLangchain -from destination_langchain.embedder import OPEN_AI_VECTOR_SIZE from integration_tests.base_integration_test import LocalIntegrationTest from langchain.embeddings import FakeEmbeddings from langchain.vectorstores import DocArrayHnswSearch @@ -33,7 +33,9 @@ def test_write(self): destination = DestinationLangchain() list(destination.write(self.config, catalog, [*first_record_chunk, first_state_message])) - vector_store = DocArrayHnswSearch.from_params(embedding=FakeEmbeddings(size=OPEN_AI_VECTOR_SIZE), work_dir=self.temp_dir, n_dim=OPEN_AI_VECTOR_SIZE) + vector_store = DocArrayHnswSearch.from_params( + embedding=FakeEmbeddings(size=OPEN_AI_VECTOR_SIZE), work_dir=self.temp_dir, n_dim=OPEN_AI_VECTOR_SIZE + ) result = vector_store.similarity_search("does not match anyway", 10) assert len(result) == 5 diff --git a/airbyte-integrations/connectors/destination-langchain/integration_tests/pinecone_integration_test.py b/airbyte-integrations/connectors/destination-langchain/integration_tests/pinecone_integration_test.py index 55d6b01a98ee..d7a4b913702d 100644 --- a/airbyte-integrations/connectors/destination-langchain/integration_tests/pinecone_integration_test.py +++ b/airbyte-integrations/connectors/destination-langchain/integration_tests/pinecone_integration_test.py @@ -7,9 +7,9 @@ from time import sleep import pinecone +from airbyte_cdk.destinations.vector_db_based.embedder import OPEN_AI_VECTOR_SIZE from airbyte_cdk.models import DestinationSyncMode, Status from destination_langchain.destination import DestinationLangchain -from destination_langchain.embedder import OPEN_AI_VECTOR_SIZE from integration_tests.base_integration_test import BaseIntegrationTest from langchain.embeddings import OpenAIEmbeddings from langchain.vectorstores import Pinecone @@ -76,9 +76,7 @@ def test_write(self): if is_starter_pod: # Documents might not be available right away because Pinecone is handling them async sleep(20) - result = self._index.query( - vector=[0] * OPEN_AI_VECTOR_SIZE, top_k=10, filter={"_record_id": "2"}, include_metadata=True - ) + result = self._index.query(vector=[0] * OPEN_AI_VECTOR_SIZE, top_k=10, filter={"_record_id": "mystream_2"}, include_metadata=True) assert len(result.matches) == 1 assert result.matches[0].metadata["text"] == "str_col: Cats are nice" @@ -87,4 +85,4 @@ def test_write(self): pinecone.init(api_key=self.config["indexing"]["pinecone_key"], environment=self.config["indexing"]["pinecone_environment"]) vector_store = Pinecone(self._index, embeddings.embed_query, "text") result = vector_store.similarity_search("feline animals", 1) - assert result[0].metadata["_record_id"] == "2" + assert result[0].metadata["_record_id"] == "mystream_2" diff --git a/airbyte-integrations/connectors/destination-langchain/metadata.yaml b/airbyte-integrations/connectors/destination-langchain/metadata.yaml index 191f9d8c8f39..f8db27c1afe0 100644 --- a/airbyte-integrations/connectors/destination-langchain/metadata.yaml +++ b/airbyte-integrations/connectors/destination-langchain/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: database connectorType: destination definitionId: cf98d52c-ba5a-4dfd-8ada-c1baebfa6e73 - dockerImageTag: 0.0.8 + dockerImageTag: 0.1.2 dockerRepository: airbyte/destination-langchain githubIssueLabel: destination-langchain icon: langchain.svg @@ -22,4 +22,10 @@ data: sl: 100 ql: 100 supportLevel: community + releases: + breakingChanges: + 0.1.0: + message: > + This version changes the way record ids are tracked internally. If you are using a stream in **append-dedup** mode, you need to reset the connection after doing the upgrade to avoid duplicates. + upgradeDeadline: "2023-09-18" metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-langchain/setup.py b/airbyte-integrations/connectors/destination-langchain/setup.py index 80f25bd65f1e..5446952fc464 100644 --- a/airbyte-integrations/connectors/destination-langchain/setup.py +++ b/airbyte-integrations/connectors/destination-langchain/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", + "airbyte-cdk==0.51.10", "langchain", "openai", "requests", diff --git a/airbyte-integrations/connectors/destination-langchain/unit_tests/chroma_local_indexer_test.py b/airbyte-integrations/connectors/destination-langchain/unit_tests/chroma_local_indexer_test.py index 01452e3c74ef..3ccf57d015d0 100644 --- a/airbyte-integrations/connectors/destination-langchain/unit_tests/chroma_local_indexer_test.py +++ b/airbyte-integrations/connectors/destination-langchain/unit_tests/chroma_local_indexer_test.py @@ -51,9 +51,11 @@ def test_chroma_local_normalize_metadata(): docs, [], ) - indexer.vectorstore.add_documents.assert_called_with([ - Document(page_content="test", metadata={"_airbyte_stream": "abc", "a_boolean_value": "True"}), - ]) + indexer.vectorstore.add_documents.assert_called_with( + [ + Document(page_content="test", metadata={"_airbyte_stream": "abc", "a_boolean_value": "True"}), + ] + ) def test_chroma_local_index_empty_batch(): diff --git a/airbyte-integrations/connectors/destination-langchain/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-langchain/unit_tests/destination_test.py index 2b27d4d25ed6..6d60b3b6db07 100644 --- a/airbyte-integrations/connectors/destination-langchain/unit_tests/destination_test.py +++ b/airbyte-integrations/connectors/destination-langchain/unit_tests/destination_test.py @@ -18,7 +18,10 @@ def _generate_record_message(index: int): - return AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="example_stream", emitted_at=1234, data={"column_name": f"value {index}", "id": index})) + return AirbyteMessage( + type=Type.RECORD, + record=AirbyteRecordMessage(stream="example_stream", emitted_at=1234, data={"column_name": f"value {index}", "id": index}), + ) @patch.dict(embedder_map, {"openai": MagicMock()}) diff --git a/airbyte-integrations/connectors/destination-langchain/unit_tests/docarray_indexer_test.py b/airbyte-integrations/connectors/destination-langchain/unit_tests/docarray_indexer_test.py index d4486811d2dc..d2d8daf84edc 100644 --- a/airbyte-integrations/connectors/destination-langchain/unit_tests/docarray_indexer_test.py +++ b/airbyte-integrations/connectors/destination-langchain/unit_tests/docarray_indexer_test.py @@ -34,7 +34,44 @@ def test_docarray_index(self): def test_docarray_pre_sync_fail(self): try: - self.indexer.pre_sync(ConfiguredAirbyteCatalog.parse_obj( + self.indexer.pre_sync( + ConfiguredAirbyteCatalog.parse_obj( + { + "streams": [ + { + "stream": { + "name": "example_stream", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {}, + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": False, + "default_cursor_field": ["column_name"], + }, + "primary_key": [["id"]], + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup", + }, + ] + } + ) + ) + assert False, "Expected exception" + except Exception as e: + assert ( + str(e) + == "DocArrayHnswSearchIndexer only supports overwrite mode, got DestinationSyncMode.append_dedup for stream example_stream" + ) + + @patch("os.listdir") + @patch("os.remove") + def test_docarray_pre_sync_succeed(self, remove_mock, listdir_mock): + listdir_mock.return_value = ["file1", "file2"] + self.indexer._init_vectorstore = MagicMock() + self.indexer.pre_sync( + ConfiguredAirbyteCatalog.parse_obj( { "streams": [ { @@ -46,50 +83,24 @@ def test_docarray_pre_sync_fail(self): "default_cursor_field": ["column_name"], }, "primary_key": [["id"]], - "sync_mode": "incremental", - "destination_sync_mode": "append_dedup", + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + }, + { + "stream": { + "name": "example_stream2", + "json_schema": {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": {}}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": False, + "default_cursor_field": ["column_name"], + }, + "primary_key": [["id"]], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", }, ] } - )) - assert False, "Expected exception" - except Exception as e: - assert str(e) == "DocArrayHnswSearchIndexer only supports overwrite mode, got DestinationSyncMode.append_dedup for stream example_stream" - - @patch('os.listdir') - @patch('os.remove') - def test_docarray_pre_sync_succeed(self, remove_mock, listdir_mock): - listdir_mock.return_value = ["file1", "file2"] - self.indexer._init_vectorstore = MagicMock() - self.indexer.pre_sync(ConfiguredAirbyteCatalog.parse_obj( - { - "streams": [ - { - "stream": { - "name": "example_stream", - "json_schema": {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": {}}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": False, - "default_cursor_field": ["column_name"], - }, - "primary_key": [["id"]], - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - }, - { - "stream": { - "name": "example_stream2", - "json_schema": {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": {}}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": False, - "default_cursor_field": ["column_name"], - }, - "primary_key": [["id"]], - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - }, - ] - } - )) + ) + ) assert remove_mock.call_count == 2 assert self.indexer._init_vectorstore.call_count == 1 diff --git a/airbyte-integrations/connectors/destination-langchain/unit_tests/document_processor_test.py b/airbyte-integrations/connectors/destination-langchain/unit_tests/document_processor_test.py index b8076debe157..955e4df2af73 100644 --- a/airbyte-integrations/connectors/destination-langchain/unit_tests/document_processor_test.py +++ b/airbyte-integrations/connectors/destination-langchain/unit_tests/document_processor_test.py @@ -98,12 +98,7 @@ def test_complex_text_fields(): "non_text": "a", "non_text_2": 1, "text": "This is the regular text", - "other_nested": { - "non_text": { - "a": "xyz", - "b": "abc" - } - } + "other_nested": {"non_text": {"a": "xyz", "b": "abc"}}, }, emitted_at=1234, ) @@ -113,17 +108,15 @@ def test_complex_text_fields(): chunks, _ = processor.process(record) assert len(chunks) == 1 - assert chunks[0].page_content == """nested.texts.*.text: This is the text + assert ( + chunks[0].page_content + == """nested.texts.*.text: This is the text And another text: This is the regular text other_nested.non_text: \na: xyz b: abc""" - assert chunks[0].metadata == { - "id": 1, - "non_text": "a", - "non_text_2": 1, - "_airbyte_stream": "namespace1_stream1" - } + ) + assert chunks[0].metadata == {"id": 1, "non_text": "a", "non_text_2": 1, "_airbyte_stream": "namespace1_stream1"} def test_non_text_fields(): @@ -210,14 +203,16 @@ def test_process_multiple_chunks_with_relevant_fields(): @pytest.mark.parametrize( "primary_key_value, stringified_primary_key, primary_key", [ - ({"id": 99}, "99", [["id"]]), - ({"id": 99, "name": "John Doe"}, "99_John Doe", [["id"], ["name"]]), - ({"id": 99, "name": "John Doe", "age": 25}, "99_John Doe_25", [["id"], ["name"], ["age"]]), - ({"nested": {"id": "abc"}, "name": "John Doe"}, "abc_John Doe", [["nested", "id"], ["name"]]), - ({"nested": {"id": "abc"}}, "abc___not_found__", [["nested", "id"], ["name"]]), - ] + ({"id": 99}, "namespace1_stream1_99", [["id"]]), + ({"id": 99, "name": "John Doe"}, "namespace1_stream1_99_John Doe", [["id"], ["name"]]), + ({"id": 99, "name": "John Doe", "age": 25}, "namespace1_stream1_99_John Doe_25", [["id"], ["name"], ["age"]]), + ({"nested": {"id": "abc"}, "name": "John Doe"}, "namespace1_stream1_abc_John Doe", [["nested", "id"], ["name"]]), + ({"nested": {"id": "abc"}}, "namespace1_stream1_abc___not_found__", [["nested", "id"], ["name"]]), + ], ) -def test_process_multiple_chunks_with_dedupe_mode(primary_key_value: Mapping[str, Any], stringified_primary_key: str, primary_key: List[List[str]]): +def test_process_multiple_chunks_with_dedupe_mode( + primary_key_value: Mapping[str, Any], stringified_primary_key: str, primary_key: List[List[str]] +): processor = initialize_processor() record = AirbyteRecordMessage( @@ -226,7 +221,7 @@ def test_process_multiple_chunks_with_dedupe_mode(primary_key_value: Mapping[str data={ "text": "This is the text and it is long enough to be split into multiple chunks. This is the text and it is long enough to be split into multiple chunks. This is the text and it is long enough to be split into multiple chunks", "age": 25, - **primary_key_value + **primary_key_value, }, emitted_at=1234, ) diff --git a/airbyte-integrations/connectors/destination-langchain/unit_tests/pinecone_indexer_test.py b/airbyte-integrations/connectors/destination-langchain/unit_tests/pinecone_indexer_test.py index b7e430277dc2..2a2b50c5f605 100644 --- a/airbyte-integrations/connectors/destination-langchain/unit_tests/pinecone_indexer_test.py +++ b/airbyte-integrations/connectors/destination-langchain/unit_tests/pinecone_indexer_test.py @@ -42,7 +42,7 @@ def create_index_description(dimensions=3, pod_type="p1"): @pytest.fixture(scope="module", autouse=True) def mock_describe_index(): - with patch('pinecone.describe_index') as mock: + with patch("pinecone.describe_index") as mock: mock.return_value = create_index_description() yield mock @@ -79,7 +79,9 @@ def test_pinecone_index_upsert_and_delete_starter(mock_describe_index): ], ["delete_id1", "delete_id2"], ) - indexer.pinecone_index.query.assert_called_with(vector=[0,0,0],filter={"_record_id": {"$in": ["delete_id1", "delete_id2"]}}, top_k=10_000) + indexer.pinecone_index.query.assert_called_with( + vector=[0, 0, 0], filter={"_record_id": {"$in": ["delete_id1", "delete_id2"]}}, top_k=10_000 + ) indexer.pinecone_index.delete.assert_called_with(ids=["doc_id1", "doc_id2"]) indexer.pinecone_index.upsert.assert_called_with( vectors=( @@ -167,7 +169,7 @@ def test_pinecone_pre_sync_starter(mock_describe_index): indexer = create_pinecone_indexer() indexer.pinecone_index.query.return_value = MagicMock(matches=[MagicMock(id="doc_id1"), MagicMock(id="doc_id2")]) indexer.pre_sync(generate_catalog()) - indexer.pinecone_index.query.assert_called_with(vector=[0,0,0],filter={"_airbyte_stream": "example_stream2"}, top_k=10_000) + indexer.pinecone_index.query.assert_called_with(vector=[0, 0, 0], filter={"_airbyte_stream": "example_stream2"}, top_k=10_000) indexer.pinecone_index.delete.assert_called_with(ids=["doc_id1", "doc_id2"]) diff --git a/airbyte-integrations/connectors/destination-local-json/.dockerignore b/airbyte-integrations/connectors/destination-local-json/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-local-json/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-local-json/Dockerfile b/airbyte-integrations/connectors/destination-local-json/Dockerfile deleted file mode 100644 index e7c1832c055e..000000000000 --- a/airbyte-integrations/connectors/destination-local-json/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-local-json - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-local-json - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.2.11 -LABEL io.airbyte.name=airbyte/destination-local-json diff --git a/airbyte-integrations/connectors/destination-local-json/build.gradle b/airbyte-integrations/connectors/destination-local-json/build.gradle index c1794c616de9..84a09417b178 100644 --- a/airbyte-integrations/connectors/destination-local-json/build.gradle +++ b/airbyte-integrations/connectors/destination-local-json/build.gradle @@ -1,19 +1,27 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.local_json.LocalJsonDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') } diff --git a/airbyte-integrations/connectors/destination-local-json/src/main/java/io/airbyte/integrations/destination/local_json/LocalJsonDestination.java b/airbyte-integrations/connectors/destination-local-json/src/main/java/io/airbyte/integrations/destination/local_json/LocalJsonDestination.java index 776bc2b6371c..ba79a2fcb9b3 100644 --- a/airbyte-integrations/connectors/destination-local-json/src/main/java/io/airbyte/integrations/destination/local_json/LocalJsonDestination.java +++ b/airbyte-integrations/connectors/destination-local-json/src/main/java/io/airbyte/integrations/destination/local_json/LocalJsonDestination.java @@ -7,14 +7,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.CommitOnStateAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.CommitOnStateAirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-local-json/src/test-integration/java/io/airbyte/integrations/destination/local_json/LocalJsonDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-local-json/src/test-integration/java/io/airbyte/integrations/destination/local_json/LocalJsonDestinationAcceptanceTest.java index 77907527225b..0f19e9063709 100644 --- a/airbyte-integrations/connectors/destination-local-json/src/test-integration/java/io/airbyte/integrations/destination/local_json/LocalJsonDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-local-json/src/test-integration/java/io/airbyte/integrations/destination/local_json/LocalJsonDestinationAcceptanceTest.java @@ -8,12 +8,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashSet; diff --git a/airbyte-integrations/connectors/destination-local-json/src/test/java/io/airbyte/integrations/destination/local_json/LocalJsonDestinationTest.java b/airbyte-integrations/connectors/destination-local-json/src/test/java/io/airbyte/integrations/destination/local_json/LocalJsonDestinationTest.java index 63d26c27c5b1..4e74f07a1cf8 100644 --- a/airbyte-integrations/connectors/destination-local-json/src/test/java/io/airbyte/integrations/destination/local_json/LocalJsonDestinationTest.java +++ b/airbyte-integrations/connectors/destination-local-json/src/test/java/io/airbyte/integrations/destination/local_json/LocalJsonDestinationTest.java @@ -17,12 +17,12 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/.dockerignore b/airbyte-integrations/connectors/destination-mariadb-columnstore/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/Dockerfile b/airbyte-integrations/connectors/destination-mariadb-columnstore/Dockerfile deleted file mode 100644 index cb6bcd655fd2..000000000000 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-mariadb-columnstore - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-mariadb-columnstore - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.7 -LABEL io.airbyte.name=airbyte/destination-mariadb-columnstore diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/README.md b/airbyte-integrations/connectors/destination-mariadb-columnstore/README.md index 64e03f4c931f..1c1793ec10ee 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/README.md +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-mariadb-columnstore:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-mariadb-columnstore:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-mariadb-columnstore:dev`. the Dockerfile. #### Run @@ -61,8 +62,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-mariadb-columnstore test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/mariadb-columnstore.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/build.gradle b/airbyte-integrations/connectors/destination-mariadb-columnstore/build.gradle index 765322c42935..8d545e9b1813 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/build.gradle +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/build.gradle @@ -1,28 +1,32 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.mariadb_columnstore.MariadbColumnstoreDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:bases:bases-destination-jdbc') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation 'org.mariadb.jdbc:mariadb-java-client:2.7.4' implementation 'com.vdurmont:semver4j:3.1.0' - testImplementation project(":airbyte-json-validation") - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mariadb-columnstore') - integrationTestJavaImplementation libs.connectors.testcontainers.mariadb + integrationTestJavaImplementation libs.testcontainers.mariadb } diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java index 4eafdfd77868..af3dee3e9bd2 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestination.java @@ -6,15 +6,15 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination; +import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.ssh.SshWrappedDestination; -import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import io.airbyte.integrations.destination.mariadb_columnstore.MariadbColumnstoreSqlOperations.VersionCompatibility; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreNameTransformer.java b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreNameTransformer.java index b78b4b66d050..c2ac2540e171 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreNameTransformer.java +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreNameTransformer.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.mariadb_columnstore; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; public class MariadbColumnstoreNameTransformer extends StandardNameTransformer { diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreSqlOperations.java b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreSqlOperations.java index f4045ec71ec4..f1289ce6e480 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreSqlOperations.java +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/main/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreSqlOperations.java @@ -5,9 +5,9 @@ package io.airbyte.integrations.destination.mariadb_columnstore; import com.vdurmont.semver4j.Semver; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.jdbc.JdbcSqlOperations; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.jdbc.JdbcSqlOperations; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.io.File; import java.io.IOException; diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/MariaDbTestDataComparator.java b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/MariaDbTestDataComparator.java index 6f1175f95216..73da2d532ab5 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/MariaDbTestDataComparator.java +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/MariaDbTestDataComparator.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.mariadb_columnstore; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import java.util.ArrayList; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestinationAcceptanceTest.java index a75e4969e89b..1e07845b0ecb 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/MariadbColumnstoreDestinationAcceptanceTest.java @@ -7,16 +7,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.sql.SQLException; import java.util.HashSet; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/SshKeyMariadbColumnstoreDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/SshKeyMariadbColumnstoreDestinationAcceptanceTest.java index f009b94ca291..7d7b6232b8cc 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/SshKeyMariadbColumnstoreDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/SshKeyMariadbColumnstoreDestinationAcceptanceTest.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.mariadb_columnstore; -import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; public class SshKeyMariadbColumnstoreDestinationAcceptanceTest extends SshMariadbColumnstoreDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/SshMariadbColumnstoreDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/SshMariadbColumnstoreDestinationAcceptanceTest.java index 57b3b64db323..898aa505d932 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/SshMariadbColumnstoreDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/SshMariadbColumnstoreDestinationAcceptanceTest.java @@ -6,17 +6,17 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.base.ssh.SshBastionContainer; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.base.ssh.SshBastionContainer; -import io.airbyte.integrations.base.ssh.SshTunnel; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import java.util.ArrayList; import java.util.HashSet; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/SshPasswordMariadbColumnstoreDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/SshPasswordMariadbColumnstoreDestinationAcceptanceTest.java index 6b45c581e79f..89c7ca6d8910 100644 --- a/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/SshPasswordMariadbColumnstoreDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mariadb-columnstore/src/test-integration/java/io/airbyte/integrations/destination/mariadb_columnstore/SshPasswordMariadbColumnstoreDestinationAcceptanceTest.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.mariadb_columnstore; -import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; public class SshPasswordMariadbColumnstoreDestinationAcceptanceTest extends SshMariadbColumnstoreDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-meilisearch/Dockerfile b/airbyte-integrations/connectors/destination-meilisearch/Dockerfile index f573460c64a2..00bbfb9c4846 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/Dockerfile +++ b/airbyte-integrations/connectors/destination-meilisearch/Dockerfile @@ -34,5 +34,5 @@ COPY destination_meilisearch ./destination_meilisearch ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.0 +LABEL io.airbyte.version=1.0.1 LABEL io.airbyte.name=airbyte/destination-meilisearch diff --git a/airbyte-integrations/connectors/destination-meilisearch/README.md b/airbyte-integrations/connectors/destination-meilisearch/README.md index 560eabcc7a75..207e2898208e 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/README.md +++ b/airbyte-integrations/connectors/destination-meilisearch/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:destination-meilisearch:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/destinations/meilisearch) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_meilisearch/spec.json` file. @@ -54,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/destination-meilisearch:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=destination-meilisearch build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:destination-meilisearch:airbyteDocker +An image will be built with the tag `airbyte/destination-meilisearch:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/destination-meilisearch:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,38 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-meilisearch:dev c # messages.jsonl is a file containing line-separated JSON representing AirbyteMessages cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-meilisearch:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=destination-meilisearch test ``` -#### Acceptance Tests -Coming soon: -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:destination-meilisearch:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:destination-meilisearch:integrationTest -``` +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -116,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-meilisearch test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/meilisearch.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-meilisearch/build.gradle b/airbyte-integrations/connectors/destination-meilisearch/build.gradle deleted file mode 100644 index 7849c8cdc050..000000000000 --- a/airbyte-integrations/connectors/destination-meilisearch/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'destination_meilisearch' -} diff --git a/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/destination.py b/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/destination.py index d6a44c1d5f9c..32d08b787bf1 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/destination.py +++ b/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/destination.py @@ -3,14 +3,16 @@ # -from logging import Logger -from typing import Any, Iterable, Mapping +from logging import Logger, getLogger +from typing import Any, Dict, Iterable, Mapping from airbyte_cdk.destinations import Destination from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, DestinationSyncMode, Status, Type from destination_meilisearch.writer import MeiliWriter from meilisearch import Client +logger = getLogger("airbyte") + def get_client(config: Mapping[str, Any]) -> Client: host = config.get("host") @@ -21,36 +23,51 @@ def get_client(config: Mapping[str, Any]) -> Client: class DestinationMeilisearch(Destination): primary_key = "_ab_pk" + def _flush_streams(self, streams: Dict[str, MeiliWriter]) -> Iterable[AirbyteMessage]: + for stream in streams: + streams[stream].flush() + def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: client = get_client(config=config) + # Creating Meilisearch writers + writers = {s.stream.name: MeiliWriter(client, s.stream.name, self.primary_key) for s in configured_catalog.streams} for configured_stream in configured_catalog.streams: - steam_name = configured_stream.stream.name + stream_name = configured_stream.stream.name + # Deleting index in Meilisearch if sync mode is overwite if configured_stream.destination_sync_mode == DestinationSyncMode.overwrite: - client.delete_index(steam_name) - client.create_index(steam_name, {"primaryKey": self.primary_key}) - - writer = MeiliWriter(client, steam_name, self.primary_key) - for message in input_messages: - if message.type == Type.STATE: - writer.flush() - yield message - elif message.type == Type.RECORD: - writer.queue_write_operation(message.record.data) - else: + logger.debug(f"Deleting index: {stream_name}.") + client.delete_index(stream_name) + # Creating index in Meilisearch + client.create_index(stream_name, {"primaryKey": self.primary_key}) + logger.debug(f"Creating index: {stream_name}.") + + for message in input_messages: + if message.type == Type.STATE: + yield message + elif message.type == Type.RECORD: + data = message.record.data + stream = message.record.stream + # Skip unselected streams + if stream not in writers: + logger.debug(f"Stream {stream} was not present in configured streams, skipping") continue - writer.flush() + writers[stream].queue_write_operation(data) + else: + logger.info(f"Unhandled message type {message.type}: {message}") + + # Flush any leftover messages + self._flush_streams(writers) def check(self, logger: Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: try: client = get_client(config=config) - create_index_job = client.create_index("_airbyte", {"primaryKey": "id"}) - client.wait_for_task(create_index_job["taskUid"]) + client.create_index("_airbyte", {"primaryKey": "id"}) - add_documents_job = client.index("_airbyte").add_documents( + client.index("_airbyte").add_documents( [ { "id": 287947, @@ -59,9 +76,7 @@ def check(self, logger: Logger, config: Mapping[str, Any]) -> AirbyteConnectionS } ] ) - client.wait_for_task(add_documents_job.task_uid) - client.index("_airbyte").search("Shazam") client.delete_index("_airbyte") return AirbyteConnectionStatus(status=Status.SUCCEEDED) except Exception as e: diff --git a/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/writer.py b/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/writer.py index c2eca6a88ce9..e2450f825106 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/writer.py +++ b/airbyte-integrations/connectors/destination-meilisearch/destination_meilisearch/writer.py @@ -12,25 +12,28 @@ class MeiliWriter: - write_buffer = [] flush_interval = 50000 - def __init__(self, client: Client, steam_name: str, primary_key: str): + def __init__(self, client: Client, stream_name: str, primary_key: str): self.client = client - self.steam_name = steam_name self.primary_key = primary_key + self.stream_name: str = stream_name + self._write_buffer = [] + + logger.info(f"Creating MeiliWriter for {self.stream_name}") def queue_write_operation(self, data: Mapping): random_key = str(uuid4()) - self.write_buffer.append({**data, self.primary_key: random_key}) - if len(self.write_buffer) == self.flush_interval: + self._write_buffer.append({**data, self.primary_key: random_key}) + if len(self._write_buffer) == self.flush_interval: + logger.debug(f"Reached limit size: flushing records for {self.stream_name}") self.flush() def flush(self): - buffer_size = len(self.write_buffer) + buffer_size = len(self._write_buffer) if buffer_size == 0: return - logger.info(f"flushing {buffer_size} records") - response = self.client.index(self.steam_name).add_documents(self.write_buffer) + logger.info(f"Flushing {buffer_size} records") + response = self.client.index(self.stream_name).add_documents(self._write_buffer) self.client.wait_for_task(response.task_uid, 1800000, 1000) - self.write_buffer.clear() + self._write_buffer.clear() diff --git a/airbyte-integrations/connectors/destination-meilisearch/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-meilisearch/integration_tests/integration_test.py index 9e63e24dc87d..1d9687e97c7d 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-meilisearch/integration_tests/integration_test.py @@ -4,7 +4,6 @@ import json import logging -import time from typing import Any, Dict, Mapping import pytest @@ -56,12 +55,7 @@ def teardown(config: Mapping): def client_fixture(config) -> Client: client = get_client(config=config) resp = client.create_index("_airbyte", {"primaryKey": "_ab_pk"}) - while True: - time.sleep(0.2) - task = client.get_task(resp["taskUid"]) - status = task["status"] - if status == "succeeded" or status == "failed": - break + client.wait_for_task(_handle_breaking_wait_for_task(resp)) return client @@ -87,6 +81,13 @@ def _record(stream: str, str_value: str, int_value: int) -> AirbyteMessage: ) +def _handle_breaking_wait_for_task(task: Any) -> int: + if type(task) is dict: + return task["taskUid"] + else: + return task.task_uid + + def records_count(client: Client) -> int: documents_results = client.index("_airbyte").get_documents() return documents_results.total diff --git a/airbyte-integrations/connectors/destination-meilisearch/metadata.yaml b/airbyte-integrations/connectors/destination-meilisearch/metadata.yaml index 7826092b697b..79a5f5851984 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/metadata.yaml +++ b/airbyte-integrations/connectors/destination-meilisearch/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: destination definitionId: af7c921e-5892-4ff2-b6c1-4a5ab258fb7e - dockerImageTag: 1.0.0 + dockerImageTag: 1.0.1 dockerRepository: airbyte/destination-meilisearch githubIssueLabel: destination-meilisearch icon: meilisearch.svg diff --git a/airbyte-integrations/connectors/destination-meilisearch/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-meilisearch/unit_tests/unit_test.py index df1f503df180..c09a3f7d8744 100644 --- a/airbyte-integrations/connectors/destination-meilisearch/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/destination-meilisearch/unit_tests/unit_test.py @@ -9,15 +9,21 @@ @patch("meilisearch.Client") def test_queue_write_operation(client): - writer = MeiliWriter(client, "steam_name", "primary_key") + writer = MeiliWriter(client, "stream_name", "primary_key") writer.queue_write_operation({"a": "a"}) - assert len(writer.write_buffer) == 1 + assert len(writer._write_buffer) == 1 + writer.queue_write_operation({"b": "b"}) + assert len(writer._write_buffer) == 2 + writer2 = MeiliWriter(client, "stream_name2", "primary_key") + writer2.queue_write_operation({"a": "a"}) + assert len(writer2._write_buffer) == 1 + assert len(writer._write_buffer) == 2 @patch("meilisearch.Client") def test_flush(client): - writer = MeiliWriter(client, "steam_name", "primary_key") + writer = MeiliWriter(client, "stream_name", "primary_key") writer.queue_write_operation({"a": "a"}) writer.flush() - client.index.assert_called_once_with("steam_name") + client.index.assert_called_once_with("stream_name") client.wait_for_task.assert_called_once() diff --git a/airbyte-integrations/connectors/destination-milvus/.dockerignore b/airbyte-integrations/connectors/destination-milvus/.dockerignore new file mode 100644 index 000000000000..d18815658a44 --- /dev/null +++ b/airbyte-integrations/connectors/destination-milvus/.dockerignore @@ -0,0 +1,5 @@ +* +!Dockerfile +!main.py +!destination_milvus +!setup.py diff --git a/airbyte-integrations/connectors/destination-milvus/README.md b/airbyte-integrations/connectors/destination-milvus/README.md new file mode 100644 index 000000000000..be53efd50e02 --- /dev/null +++ b/airbyte-integrations/connectors/destination-milvus/README.md @@ -0,0 +1,149 @@ +# Milvus Destination + +This is the repository for the Milvus destination connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/destinations/milvus). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.10.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/destinations/milvus) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_langchain/spec.json` file. +Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `destination milvus test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=destination-milvus build +``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/destination-milvus:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") +``` + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/destination-milvus:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] +``` +Please use this as an example. This is not optimized. + +2. Build your image: +```bash +docker build -t airbyte/destination-milvus:dev . +# Running the spec command against your patched connector +docker run airbyte/destination-milvus:dev spec +``` +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/destination-langchain:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-langchain:dev check --config /secrets/config.json +# messages.jsonl is a file containing line-separated JSON representing AirbyteMessages +cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-langchain:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=destination-milvus test +``` + +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-milvus test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/milvus.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-milvus/acceptance-test-config.yml b/airbyte-integrations/connectors/destination-milvus/acceptance-test-config.yml new file mode 100644 index 000000000000..a217ebce9c76 --- /dev/null +++ b/airbyte-integrations/connectors/destination-milvus/acceptance-test-config.yml @@ -0,0 +1,7 @@ +acceptance_tests: + spec: + tests: + - spec_path: integration_tests/spec.json + backward_compatibility_tests_config: + disable_for_version: "0.0.1" +connector_image: airbyte/destination-milvus:dev diff --git a/airbyte-integrations/connectors/destination-milvus/bootstrap.md b/airbyte-integrations/connectors/destination-milvus/bootstrap.md new file mode 100644 index 000000000000..02a2cee7d2a2 --- /dev/null +++ b/airbyte-integrations/connectors/destination-milvus/bootstrap.md @@ -0,0 +1,12 @@ +# Milvus Destination Connector Bootstrap + +This destination does three things: +* Split records into chunks and separates metadata from text data +* Embeds text data into an embedding vector +* Stores the metadata and embedding vector in a vector database + +The record processing is using the text split components from https://python.langchain.com/docs/modules/data_connection/document_transformers/. + +There are various possible providers for generating embeddings, delivered as part of the CDK (`airbyte_cdk.destinations.vector_db_based`). + +Embedded documents are stored in the Milvus vector database. diff --git a/airbyte-integrations/connectors/destination-milvus/destination_milvus/__init__.py b/airbyte-integrations/connectors/destination-milvus/destination_milvus/__init__.py new file mode 100644 index 000000000000..c17e88fb8caf --- /dev/null +++ b/airbyte-integrations/connectors/destination-milvus/destination_milvus/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .destination import DestinationMilvus + +__all__ = ["DestinationMilvus"] diff --git a/airbyte-integrations/connectors/destination-milvus/destination_milvus/config.py b/airbyte-integrations/connectors/destination-milvus/destination_milvus/config.py new file mode 100644 index 000000000000..c1126786b3b6 --- /dev/null +++ b/airbyte-integrations/connectors/destination-milvus/destination_milvus/config.py @@ -0,0 +1,67 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Literal, Optional, Union + +from airbyte_cdk.destinations.vector_db_based.config import VectorDBConfigModel +from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig +from pydantic import BaseModel, Field + + +class UsernamePasswordAuth(BaseModel): + mode: Literal["username_password"] = Field("username_password", const=True) + username: str = Field(..., title="Username", description="Username for the Milvus instance", order=1) + password: str = Field(..., title="Password", description="Password for the Milvus instance", airbyte_secret=True, order=2) + + class Config(OneOfOptionConfig): + title = "Username/Password" + description = "Authenticate using username and password (suitable for self-managed Milvus clusters)" + discriminator = "mode" + + +class NoAuth(BaseModel): + mode: Literal["no_auth"] = Field("no_auth", const=True) + + class Config(OneOfOptionConfig): + title = "No auth" + description = "Do not authenticate (suitable for locally running test clusters, do not use for clusters with public IP addresses)" + discriminator = "mode" + + +class TokenAuth(BaseModel): + mode: Literal["token"] = Field("token", const=True) + token: str = Field(..., title="API Token", description="API Token for the Milvus instance", airbyte_secret=True) + + class Config(OneOfOptionConfig): + title = "API Token" + description = "Authenticate using an API token (suitable for Zilliz Cloud)" + discriminator = "mode" + + +class MilvusIndexingConfigModel(BaseModel): + host: str = Field( + ..., + title="Public Endpoint", + order=1, + description="The public endpoint of the Milvus instance. ", + examples=["https://my-instance.zone.zillizcloud.com", "tcp://host.docker.internal:19530", "tcp://my-local-milvus:19530"], + ) + db: Optional[str] = Field(title="Database Name", description="The database to connect to", default="") + collection: str = Field(..., title="Collection Name", description="The collection to load data into", order=3) + auth: Union[TokenAuth, UsernamePasswordAuth, NoAuth] = Field( + ..., title="Authentication", description="Authentication method", discriminator="mode", type="object", order=2 + ) + vector_field: str = Field(title="Vector Field", description="The field in the entity that contains the vector", default="vector") + text_field: str = Field(title="Text Field", description="The field in the entity that contains the embedded text", default="text") + + class Config: + title = "Indexing" + schema_extra = { + "group": "indexing", + "description": "Indexing configuration", + } + + +class ConfigModel(VectorDBConfigModel): + indexing: MilvusIndexingConfigModel diff --git a/airbyte-integrations/connectors/destination-milvus/destination_milvus/destination.py b/airbyte-integrations/connectors/destination-milvus/destination_milvus/destination.py new file mode 100644 index 000000000000..96919058e64c --- /dev/null +++ b/airbyte-integrations/connectors/destination-milvus/destination_milvus/destination.py @@ -0,0 +1,56 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from typing import Any, Iterable, Mapping + +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.destinations import Destination +from airbyte_cdk.destinations.vector_db_based.document_processor import DocumentProcessor +from airbyte_cdk.destinations.vector_db_based.embedder import Embedder, create_from_config +from airbyte_cdk.destinations.vector_db_based.indexer import Indexer +from airbyte_cdk.destinations.vector_db_based.writer import Writer +from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, ConnectorSpecification, Status +from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode +from destination_milvus.config import ConfigModel +from destination_milvus.indexer import MilvusIndexer + +BATCH_SIZE = 128 + + +class DestinationMilvus(Destination): + indexer: Indexer + embedder: Embedder + + def _init_indexer(self, config: ConfigModel): + self.embedder = create_from_config(config.embedding, config.processing) + self.indexer = MilvusIndexer(config.indexing, self.embedder.embedding_dimensions) + + def write( + self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] + ) -> Iterable[AirbyteMessage]: + config_model = ConfigModel.parse_obj(config) + self._init_indexer(config_model) + writer = Writer( + config_model.processing, self.indexer, self.embedder, batch_size=BATCH_SIZE, omit_raw_text=config_model.omit_raw_text + ) + yield from writer.write(configured_catalog, input_messages) + + def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: + parsed_config = ConfigModel.parse_obj(config) + self._init_indexer(parsed_config) + checks = [self.embedder.check(), self.indexer.check(), DocumentProcessor.check_config(parsed_config.processing)] + errors = [error for error in checks if error is not None] + if len(errors) > 0: + return AirbyteConnectionStatus(status=Status.FAILED, message="\n".join(errors)) + else: + return AirbyteConnectionStatus(status=Status.SUCCEEDED) + + def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: + return ConnectorSpecification( + documentationUrl="https://docs.airbyte.com/integrations/destinations/milvus", + supportsIncremental=True, + supported_destination_sync_modes=[DestinationSyncMode.overwrite, DestinationSyncMode.append, DestinationSyncMode.append_dedup], + connectionSpecification=ConfigModel.schema(), # type: ignore[attr-defined] + ) diff --git a/airbyte-integrations/connectors/destination-milvus/destination_milvus/indexer.py b/airbyte-integrations/connectors/destination-milvus/destination_milvus/indexer.py new file mode 100644 index 000000000000..bca9df695d8c --- /dev/null +++ b/airbyte-integrations/connectors/destination-milvus/destination_milvus/indexer.py @@ -0,0 +1,144 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import os +from multiprocessing import Process +from typing import Optional + +from airbyte_cdk.destinations.vector_db_based.document_processor import METADATA_RECORD_ID_FIELD, METADATA_STREAM_FIELD +from airbyte_cdk.destinations.vector_db_based.indexer import Indexer +from airbyte_cdk.destinations.vector_db_based.utils import create_stream_identifier, format_exception +from airbyte_cdk.models import ConfiguredAirbyteCatalog +from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode +from destination_milvus.config import MilvusIndexingConfigModel +from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, connections, utility + +CLOUD_DEPLOYMENT_MODE = "cloud" + + +class MilvusIndexer(Indexer): + config: MilvusIndexingConfigModel + + def __init__(self, config: MilvusIndexingConfigModel, embedder_dimensions: int): + super().__init__(config) + self.embedder_dimensions = embedder_dimensions + + def _connect(self): + connections.connect( + uri=self.config.host, + db_name=self.config.db if self.config.db else "", + user=self.config.auth.username if self.config.auth.mode == "username_password" else "", + password=self.config.auth.password if self.config.auth.mode == "username_password" else "", + token=self.config.auth.token if self.config.auth.mode == "token" else "", + ) + + def _connect_with_timeout(self): + # Run connect in a separate process as it will hang if the token is invalid. + proc = Process(target=self._connect) + proc.start() + proc.join(5) + if proc.is_alive(): + # If the process is still alive after 5 seconds, terminate it and raise an exception + proc.terminate() + proc.join() + raise Exception("Connection timed out, check your host and credentials") + + def _create_index(self, collection: Collection): + """ + Create an index on the vector field when auto-creating the collection. + + This uses an IVF_FLAT index with 1024 clusters. This is a good default for most use cases. If more control is needed, the index can be created manually (this is also stated in the documentation) + """ + collection.create_index( + field_name=self.config.vector_field, index_params={"metric_type": "L2", "index_type": "IVF_FLAT", "params": {"nlist": 1024}} + ) + + def _create_client(self): + self._connect_with_timeout() + # If the process exited within 5 seconds, it's safe to connect on the main process to execute the command + self._connect() + + if not utility.has_collection(self.config.collection): + pk = FieldSchema(name="pk", dtype=DataType.INT64, is_primary=True, auto_id=True) + vector = FieldSchema(name=self.config.vector_field, dtype=DataType.FLOAT_VECTOR, dim=self.embedder_dimensions) + schema = CollectionSchema(fields=[pk, vector], enable_dynamic_field=True) + collection = Collection(name=self.config.collection, schema=schema) + self._create_index(collection) + + self._collection = Collection(self.config.collection) + self._collection.load() + self._primary_key = self._collection.primary_field.name + + def check(self) -> Optional[str]: + deployment_mode = os.environ.get("DEPLOYMENT_MODE", "") + if deployment_mode.casefold() == CLOUD_DEPLOYMENT_MODE and not self._uses_safe_config(): + return "Host must start with https:// and authentication must be enabled on cloud deployment." + try: + self._create_client() + + description = self._collection.describe() + if not description["auto_id"]: + return "Only collections with auto_id are supported" + vector_field = next((field for field in description["fields"] if field["name"] == self.config.vector_field), None) + if vector_field is None: + return f"Vector field {self.config.vector_field} not found" + if vector_field["type"] != DataType.FLOAT_VECTOR: + return f"Vector field {self.config.vector_field} is not a vector" + if vector_field["params"]["dim"] != self.embedder_dimensions: + return f"Vector field {self.config.vector_field} is not a {self.embedder_dimensions}-dimensional vector" + except Exception as e: + return format_exception(e) + return None + + def _uses_safe_config(self) -> bool: + return self.config.host.startswith("https://") and not self.config.auth.mode == "no_auth" + + def pre_sync(self, catalog: ConfiguredAirbyteCatalog) -> None: + self._create_client() + for stream in catalog.streams: + if stream.destination_sync_mode == DestinationSyncMode.overwrite: + self._delete_for_filter(f'{METADATA_STREAM_FIELD} == "{create_stream_identifier(stream.stream)}"') + + def _delete_for_filter(self, expr: str) -> None: + iterator = self._collection.query_iterator(expr=expr) + page = iterator.next() + while len(page) > 0: + id_field = next(iter(page[0].keys())) + ids = [next(iter(entity.values())) for entity in page] + id_list_expr = ", ".join([str(id) for id in ids]) + self._collection.delete(expr=f"{id_field} in [{id_list_expr}]") + page = iterator.next() + + def _normalize(self, metadata: dict) -> dict: + result = {} + + for key, value in metadata.items(): + normalized_key = key + # the primary key can't be set directly with auto_id, so we prefix it with an underscore + if key == self._primary_key: + normalized_key = f"_{key}" + result[normalized_key] = value + + return result + + def index(self, document_chunks, namespace, stream): + entities = [] + for i in range(len(document_chunks)): + chunk = document_chunks[i] + entity = { + **self._normalize(chunk.metadata), + self.config.vector_field: chunk.embedding, + self.config.text_field: chunk.page_content, + } + if chunk.page_content is not None: + entity[self.config.text_field] = chunk.page_content + entities.append(entity) + self._collection.insert(entities) + + def delete(self, delete_ids, namespace, stream): + if len(delete_ids) > 0: + id_list_expr = ", ".join([f'"{id}"' for id in delete_ids]) + id_expr = f"{METADATA_RECORD_ID_FIELD} in [{id_list_expr}]" + self._delete_for_filter(id_expr) diff --git a/airbyte-integrations/connectors/destination-milvus/examples/configured_catalog.json b/airbyte-integrations/connectors/destination-milvus/examples/configured_catalog.json new file mode 100644 index 000000000000..acb931363d89 --- /dev/null +++ b/airbyte-integrations/connectors/destination-milvus/examples/configured_catalog.json @@ -0,0 +1,20 @@ +{ + "streams": [ + { + "stream": { + "name": "example_stream", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {} + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["column_name"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup", + "primary_key": [["pk"]] + } + ] +} diff --git a/airbyte-integrations/connectors/destination-milvus/examples/messages.jsonl b/airbyte-integrations/connectors/destination-milvus/examples/messages.jsonl new file mode 100644 index 000000000000..195d260c3ad6 --- /dev/null +++ b/airbyte-integrations/connectors/destination-milvus/examples/messages.jsonl @@ -0,0 +1 @@ +{"type": "RECORD", "record": {"stream": "example_stream", "data": { "title": "valueZXXXXXXXX1", "field2": "value2", "pk": "1" }, "emitted_at": 1625383200000}} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-milvus/icon.svg b/airbyte-integrations/connectors/destination-milvus/icon.svg new file mode 100644 index 000000000000..b4e13796df22 --- /dev/null +++ b/airbyte-integrations/connectors/destination-milvus/icon.svg @@ -0,0 +1,60 @@ + + + + + + + milvus-icon-color + + + + + + milvus-icon-color + + + + diff --git a/airbyte-integrations/connectors/destination-milvus/integration_tests/milvus_integration_test.py b/airbyte-integrations/connectors/destination-milvus/integration_tests/milvus_integration_test.py new file mode 100644 index 000000000000..731ba7edbe76 --- /dev/null +++ b/airbyte-integrations/connectors/destination-milvus/integration_tests/milvus_integration_test.py @@ -0,0 +1,122 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import logging + +from airbyte_cdk.destinations.vector_db_based.embedder import OPEN_AI_VECTOR_SIZE +from airbyte_cdk.destinations.vector_db_based.test_utils import BaseIntegrationTest +from airbyte_cdk.models import DestinationSyncMode, Status +from destination_milvus.destination import DestinationMilvus +from langchain.embeddings import OpenAIEmbeddings +from langchain.vectorstores import Milvus +from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, connections, utility + + +class MilvusIntegrationTest(BaseIntegrationTest): + """ + Zilliz call to create the collection: /v1/vector/collections/create + { + "collectionName": "test2", + "dimension": 1536, + "metricType": "L2", + "vectorField": "vector", + "primaryField": "pk" + } + """ + + def _init_milvus(self): + connections.connect(alias="test_driver", uri=self.config["indexing"]["host"], token=self.config["indexing"]["auth"]["token"]) + if utility.has_collection(self.config["indexing"]["collection"], using="test_driver"): + utility.drop_collection(self.config["indexing"]["collection"], using="test_driver") + + def setUp(self): + with open("secrets/config.json", "r") as f: + self.config = json.loads(f.read()) + self._init_milvus() + + def test_check_valid_config(self): + outcome = DestinationMilvus().check(logging.getLogger("airbyte"), self.config) + assert outcome.status == Status.SUCCEEDED + + def _create_collection(self, vector_dimensions=1536): + pk = FieldSchema(name="pk", dtype=DataType.INT64, is_primary=True, auto_id=True) + vector = FieldSchema(name="vector", dtype=DataType.FLOAT_VECTOR, dim=vector_dimensions) + schema = CollectionSchema(fields=[pk, vector], enable_dynamic_field=True) + collection = Collection(name=self.config["indexing"]["collection"], schema=schema, using="test_driver") + collection.create_index( + field_name="vector", index_params={"metric_type": "L2", "index_type": "IVF_FLAT", "params": {"nlist": 1024}} + ) + + def test_check_valid_config_pre_created_collection(self): + self._create_collection() + outcome = DestinationMilvus().check(logging.getLogger("airbyte"), self.config) + assert outcome.status == Status.SUCCEEDED + + def test_check_invalid_config_vector_dimension(self): + self._create_collection(vector_dimensions=666) + outcome = DestinationMilvus().check(logging.getLogger("airbyte"), self.config) + assert outcome.status == Status.FAILED + + def test_check_invalid_config(self): + outcome = DestinationMilvus().check( + logging.getLogger("airbyte"), + { + "processing": {"text_fields": ["str_col"], "metadata_fields": [], "chunk_size": 1000}, + "embedding": {"mode": "openai", "openai_key": "mykey"}, + "indexing": { + "host": "https://notmilvus.com", + "collection": "test2", + "auth": { + "mode": "token", + "token": "mytoken", + }, + "vector_field": "vector", + "text_field": "text", + }, + }, + ) + assert outcome.status == Status.FAILED + + def test_write(self): + self._init_milvus() + catalog = self._get_configured_catalog(DestinationSyncMode.overwrite) + first_state_message = self._state({"state": "1"}) + first_record_chunk = [self._record("mystream", f"Dogs are number {i}", i) for i in range(5)] + + # initial sync + destination = DestinationMilvus() + list(destination.write(self.config, catalog, [*first_record_chunk, first_state_message])) + collection = Collection(self.config["indexing"]["collection"], using="test_driver") + collection.flush() + assert len(collection.query(expr="pk != 0")) == 5 + + # incrementalally update a doc + incremental_catalog = self._get_configured_catalog(DestinationSyncMode.append_dedup) + list(destination.write(self.config, incremental_catalog, [self._record("mystream", "Cats are nice", 2), first_state_message])) + collection.flush() + result = collection.search( + anns_field=self.config["indexing"]["vector_field"], + param={}, + data=[[0] * OPEN_AI_VECTOR_SIZE], + limit=10, + expr='_ab_record_id == "mystream_2"', + output_fields=["text"], + ) + assert len(result[0]) == 1 + assert result[0][0].entity.get("text") == "str_col: Cats are nice" + + # test langchain integration + embeddings = OpenAIEmbeddings(openai_api_key=self.config["embedding"]["openai_key"]) + vs = Milvus( + embedding_function=embeddings, + collection_name=self.config["indexing"]["collection"], + connection_args={"uri": self.config["indexing"]["host"], "token": self.config["indexing"]["auth"]["token"]}, + ) + vs.fields.append("text") + vs.fields.append("_ab_record_id") + # call vs.fields.append() for all fields you need in the metadata + + result = vs.similarity_search("feline animals", 1) + assert result[0].metadata["_ab_record_id"] == "mystream_2" diff --git a/airbyte-integrations/connectors/destination-milvus/integration_tests/spec.json b/airbyte-integrations/connectors/destination-milvus/integration_tests/spec.json new file mode 100644 index 000000000000..445767f161c1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-milvus/integration_tests/spec.json @@ -0,0 +1,454 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/destinations/milvus", + "connectionSpecification": { + "title": "Destination Config", + "description": "The configuration model for the Vector DB based destinations. This model is used to generate the UI for the destination configuration,\nas well as to provide type safety for the configuration passed to the destination.\n\nThe configuration model is composed of four parts:\n* Processing configuration\n* Embedding configuration\n* Indexing configuration\n* Advanced configuration\n\nProcessing, embedding and advanced configuration are provided by this base class, while the indexing configuration is provided by the destination connector in the sub class.", + "type": "object", + "properties": { + "embedding": { + "title": "Embedding", + "description": "Embedding configuration", + "group": "embedding", + "type": "object", + "oneOf": [ + { + "title": "OpenAI", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "openai", + "const": "openai", + "enum": ["openai"], + "type": "string" + }, + "openai_key": { + "title": "OpenAI API key", + "airbyte_secret": true, + "type": "string" + } + }, + "required": ["openai_key", "mode"], + "description": "Use the OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." + }, + { + "title": "Cohere", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "cohere", + "const": "cohere", + "enum": ["cohere"], + "type": "string" + }, + "cohere_key": { + "title": "Cohere API key", + "airbyte_secret": true, + "type": "string" + } + }, + "required": ["cohere_key", "mode"], + "description": "Use the Cohere API to embed text." + }, + { + "title": "Fake", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "fake", + "const": "fake", + "enum": ["fake"], + "type": "string" + } + }, + "description": "Use a fake embedding made out of random vectors with 1536 embedding dimensions. This is useful for testing the data pipeline without incurring any costs.", + "required": ["mode"] + }, + { + "title": "Azure OpenAI", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "azure_openai", + "const": "azure_openai", + "enum": ["azure_openai"], + "type": "string" + }, + "openai_key": { + "title": "Azure OpenAI API key", + "description": "The API key for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", + "airbyte_secret": true, + "type": "string" + }, + "api_base": { + "title": "Resource base URL", + "description": "The base URL for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", + "examples": ["https://your-resource-name.openai.azure.com"], + "type": "string" + }, + "deployment": { + "title": "Deployment", + "description": "The deployment for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", + "examples": ["your-resource-name"], + "type": "string" + } + }, + "required": ["openai_key", "api_base", "deployment", "mode"], + "description": "Use the Azure-hosted OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." + }, + { + "title": "OpenAI-compatible", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "openai_compatible", + "const": "openai_compatible", + "enum": ["openai_compatible"], + "type": "string" + }, + "api_key": { + "title": "API key", + "default": "", + "airbyte_secret": true, + "type": "string" + }, + "base_url": { + "title": "Base URL", + "description": "The base URL for your OpenAI-compatible service", + "examples": ["https://your-service-name.com"], + "type": "string" + }, + "model_name": { + "title": "Model name", + "description": "The name of the model to use for embedding", + "default": "text-embedding-ada-002", + "examples": ["text-embedding-ada-002"], + "type": "string" + }, + "dimensions": { + "title": "Embedding dimensions", + "description": "The number of dimensions the embedding model is generating", + "examples": [1536, 384], + "type": "integer" + } + }, + "required": ["base_url", "dimensions", "mode"], + "description": "Use a service that's compatible with the OpenAI API to embed text." + } + ] + }, + "processing": { + "title": "ProcessingConfigModel", + "type": "object", + "properties": { + "chunk_size": { + "title": "Chunk size", + "description": "Size of chunks in tokens to store in vector store (make sure it is not too big for the context if your LLM)", + "maximum": 8191, + "minimum": 1, + "type": "integer" + }, + "chunk_overlap": { + "title": "Chunk overlap", + "description": "Size of overlap between chunks in tokens to store in vector store to better capture relevant context", + "default": 0, + "type": "integer" + }, + "text_fields": { + "title": "Text fields to embed", + "description": "List of fields in the record that should be used to calculate the embedding. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered text fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.", + "default": [], + "always_show": true, + "examples": ["text", "user.name", "users.*.name"], + "type": "array", + "items": { + "type": "string" + } + }, + "metadata_fields": { + "title": "Fields to store as metadata", + "description": "List of fields in the record that should be stored as metadata. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered metadata fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. When specifying nested paths, all matching values are flattened into an array set to a field named by the path.", + "default": [], + "always_show": true, + "examples": ["age", "user", "user.name"], + "type": "array", + "items": { + "type": "string" + } + }, + "text_splitter": { + "title": "Text splitter", + "description": "Split text fields into chunks based on the specified method.", + "type": "object", + "oneOf": [ + { + "title": "By Separator", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "separator", + "const": "separator", + "enum": ["separator"], + "type": "string" + }, + "separators": { + "title": "Separators", + "description": "List of separator strings to split text fields by. The separator itself needs to be wrapped in double quotes, e.g. to split by the dot character, use \".\". To split by a newline, use \"\\n\".", + "default": ["\"\\n\\n\"", "\"\\n\"", "\" \"", "\"\""], + "type": "array", + "items": { + "type": "string" + } + }, + "keep_separator": { + "title": "Keep separator", + "description": "Whether to keep the separator in the resulting chunks", + "default": false, + "type": "boolean" + } + }, + "description": "Split the text by the list of separators until the chunk size is reached, using the earlier mentioned separators where possible. This is useful for splitting text fields by paragraphs, sentences, words, etc.", + "required": ["mode"] + }, + { + "title": "By Markdown header", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "markdown", + "const": "markdown", + "enum": ["markdown"], + "type": "string" + }, + "split_level": { + "title": "Split level", + "description": "Level of markdown headers to split text fields by. Headings down to the specified level will be used as split points", + "default": 1, + "minimum": 1, + "maximum": 6, + "type": "integer" + } + }, + "description": "Split the text by Markdown headers down to the specified header level. If the chunk size fits multiple sections, they will be combined into a single chunk.", + "required": ["mode"] + }, + { + "title": "By Programming Language", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "code", + "const": "code", + "enum": ["code"], + "type": "string" + }, + "language": { + "title": "Language", + "description": "Split code in suitable places based on the programming language", + "enum": [ + "cpp", + "go", + "java", + "js", + "php", + "proto", + "python", + "rst", + "ruby", + "rust", + "scala", + "swift", + "markdown", + "latex", + "html", + "sol" + ], + "type": "string" + } + }, + "required": ["language", "mode"], + "description": "Split the text by suitable delimiters based on the programming language. This is useful for splitting code into chunks." + } + ] + }, + "field_name_mappings": { + "title": "Field name mappings", + "description": "List of fields to rename. Not applicable for nested fields, but can be used to rename fields already flattened via dot notation.", + "default": [], + "type": "array", + "items": { + "title": "FieldNameMappingConfigModel", + "type": "object", + "properties": { + "from_field": { + "title": "From field name", + "description": "The field name in the source", + "type": "string" + }, + "to_field": { + "title": "To field name", + "description": "The field name to use in the destination", + "type": "string" + } + }, + "required": ["from_field", "to_field"] + } + } + }, + "required": ["chunk_size"], + "group": "processing" + }, + "omit_raw_text": { + "title": "Do not store raw text", + "description": "Do not store the text that gets embedded along with the vector and the metadata in the destination. If set to true, only the vector and the metadata will be stored - in this case raw text for LLM use cases needs to be retrieved from another source.", + "default": false, + "group": "advanced", + "type": "boolean" + }, + "indexing": { + "title": "Indexing", + "type": "object", + "properties": { + "host": { + "title": "Public Endpoint", + "description": "The public endpoint of the Milvus instance. ", + "order": 1, + "examples": [ + "https://my-instance.zone.zillizcloud.com", + "tcp://host.docker.internal:19530", + "tcp://my-local-milvus:19530" + ], + "type": "string" + }, + "db": { + "title": "Database Name", + "description": "The database to connect to", + "default": "", + "type": "string" + }, + "collection": { + "title": "Collection Name", + "description": "The collection to load data into", + "order": 3, + "type": "string" + }, + "auth": { + "title": "Authentication", + "description": "Authentication method", + "type": "object", + "order": 2, + "oneOf": [ + { + "title": "API Token", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "token", + "const": "token", + "enum": ["token"], + "type": "string" + }, + "token": { + "title": "API Token", + "description": "API Token for the Milvus instance", + "airbyte_secret": true, + "type": "string" + } + }, + "required": ["token", "mode"], + "description": "Authenticate using an API token (suitable for Zilliz Cloud)" + }, + { + "title": "Username/Password", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "username_password", + "const": "username_password", + "enum": ["username_password"], + "type": "string" + }, + "username": { + "title": "Username", + "description": "Username for the Milvus instance", + "order": 1, + "type": "string" + }, + "password": { + "title": "Password", + "description": "Password for the Milvus instance", + "airbyte_secret": true, + "order": 2, + "type": "string" + } + }, + "required": ["username", "password", "mode"], + "description": "Authenticate using username and password (suitable for self-managed Milvus clusters)" + }, + { + "title": "No auth", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "no_auth", + "const": "no_auth", + "enum": ["no_auth"], + "type": "string" + } + }, + "description": "Do not authenticate (suitable for locally running test clusters, do not use for clusters with public IP addresses)", + "required": ["mode"] + } + ] + }, + "vector_field": { + "title": "Vector Field", + "description": "The field in the entity that contains the vector", + "default": "vector", + "type": "string" + }, + "text_field": { + "title": "Text Field", + "description": "The field in the entity that contains the embedded text", + "default": "text", + "type": "string" + } + }, + "required": ["host", "collection", "auth"], + "group": "indexing", + "description": "Indexing configuration" + } + }, + "required": ["embedding", "processing", "indexing"], + "groups": [ + { + "id": "processing", + "title": "Processing" + }, + { + "id": "embedding", + "title": "Embedding" + }, + { + "id": "indexing", + "title": "Indexing" + }, + { + "id": "advanced", + "title": "Advanced" + } + ] + }, + "supportsIncremental": true, + "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"] +} diff --git a/airbyte-integrations/connectors/destination-milvus/main.py b/airbyte-integrations/connectors/destination-milvus/main.py new file mode 100644 index 000000000000..ed23821ba6c3 --- /dev/null +++ b/airbyte-integrations/connectors/destination-milvus/main.py @@ -0,0 +1,11 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from destination_milvus import DestinationMilvus + +if __name__ == "__main__": + DestinationMilvus().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-milvus/metadata.yaml b/airbyte-integrations/connectors/destination-milvus/metadata.yaml new file mode 100644 index 000000000000..09cacd466702 --- /dev/null +++ b/airbyte-integrations/connectors/destination-milvus/metadata.yaml @@ -0,0 +1,40 @@ +data: + allowedHosts: + hosts: + - "${indexing.host}" + - api.openai.com + - api.cohere.ai + - "${embedding.api_base}" + registries: + cloud: + enabled: true + oss: + enabled: true + resourceRequirements: + jobSpecific: + - jobType: sync + # TODO: Remove once https://github.com/airbytehq/airbyte/issues/30611 is resolved + resourceRequirements: + memory_limit: 2Gi + memory_request: 2Gi + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + connectorSubtype: vectorstore + connectorType: destination + definitionId: 65de8962-48c9-11ee-be56-0242ac120002 + dockerImageTag: 0.0.12 + dockerRepository: airbyte/destination-milvus + githubIssueLabel: destination-milvus + icon: milvus.svg + license: MIT + name: Milvus + releaseDate: 2023-08-15 + releaseStage: alpha + documentationUrl: https://docs.airbyte.com/integrations/destinations/milvus + tags: + - language:python + ab_internal: + sl: 200 + ql: 300 + supportLevel: certified +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-file-secure/requirements.txt b/airbyte-integrations/connectors/destination-milvus/requirements.txt similarity index 100% rename from airbyte-integrations/connectors/source-file-secure/requirements.txt rename to airbyte-integrations/connectors/destination-milvus/requirements.txt diff --git a/airbyte-integrations/connectors/destination-milvus/setup.py b/airbyte-integrations/connectors/destination-milvus/setup.py new file mode 100644 index 000000000000..e5c0cf315a83 --- /dev/null +++ b/airbyte-integrations/connectors/destination-milvus/setup.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.57.0", "pymilvus==2.3.0"] + +TEST_REQUIREMENTS = ["pytest~=6.2"] + +setup( + name="destination_milvus", + description="Destination implementation for Milvus.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/destination-milvus/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-milvus/unit_tests/destination_test.py new file mode 100644 index 000000000000..eaf9b78a336d --- /dev/null +++ b/airbyte-integrations/connectors/destination-milvus/unit_tests/destination_test.py @@ -0,0 +1,101 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import unittest +from unittest.mock import MagicMock, Mock, patch + +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.models import ConnectorSpecification, Status +from destination_milvus.config import ConfigModel +from destination_milvus.destination import DestinationMilvus + + +class TestDestinationMilvus(unittest.TestCase): + def setUp(self): + self.config = { + "processing": {"text_fields": ["str_col"], "metadata_fields": [], "chunk_size": 1000}, + "embedding": {"mode": "openai", "openai_key": "mykey"}, + "indexing": { + "host": "https://notmilvus.com", + "collection": "test2", + "auth": { + "mode": "token", + "token": "mytoken", + }, + "vector_field": "vector", + "text_field": "text", + }, + } + self.config_model = ConfigModel.parse_obj(self.config) + self.logger = AirbyteLogger() + + @patch("destination_milvus.destination.MilvusIndexer") + @patch("destination_milvus.destination.create_from_config") + def test_check(self, MockedEmbedder, MockedMilvusIndexer): + mock_embedder = Mock() + mock_indexer = Mock() + MockedEmbedder.return_value = mock_embedder + MockedMilvusIndexer.return_value = mock_indexer + + mock_embedder.check.return_value = None + mock_indexer.check.return_value = None + + destination = DestinationMilvus() + result = destination.check(self.logger, self.config) + + self.assertEqual(result.status, Status.SUCCEEDED) + mock_embedder.check.assert_called_once() + mock_indexer.check.assert_called_once() + + @patch("destination_milvus.destination.MilvusIndexer") + @patch("destination_milvus.destination.create_from_config") + def test_check_with_errors(self, MockedEmbedder, MockedMilvusIndexer): + mock_embedder = Mock() + mock_indexer = Mock() + MockedEmbedder.return_value = mock_embedder + MockedMilvusIndexer.return_value = mock_indexer + + embedder_error_message = "Embedder Error" + indexer_error_message = "Indexer Error" + + mock_embedder.check.return_value = embedder_error_message + mock_indexer.check.return_value = indexer_error_message + + destination = DestinationMilvus() + result = destination.check(self.logger, self.config) + + self.assertEqual(result.status, Status.FAILED) + self.assertEqual(result.message, f"{embedder_error_message}\n{indexer_error_message}") + + mock_embedder.check.assert_called_once() + mock_indexer.check.assert_called_once() + + @patch("destination_milvus.destination.Writer") + @patch("destination_milvus.destination.MilvusIndexer") + @patch("destination_milvus.destination.create_from_config") + def test_write(self, MockedEmbedder, MockedMilvusIndexer, MockedWriter): + mock_embedder = Mock() + mock_indexer = Mock() + mock_writer = Mock() + + MockedEmbedder.return_value = mock_embedder + MockedMilvusIndexer.return_value = mock_indexer + MockedWriter.return_value = mock_writer + + mock_writer.write.return_value = [] + + configured_catalog = MagicMock() + input_messages = [] + + destination = DestinationMilvus() + list(destination.write(self.config, configured_catalog, input_messages)) + + MockedWriter.assert_called_once_with(self.config_model.processing, mock_indexer, mock_embedder, batch_size=128, omit_raw_text=False) + mock_writer.write.assert_called_once_with(configured_catalog, input_messages) + + def test_spec(self): + destination = DestinationMilvus() + result = destination.spec() + + self.assertIsInstance(result, ConnectorSpecification) diff --git a/airbyte-integrations/connectors/destination-milvus/unit_tests/indexer_test.py b/airbyte-integrations/connectors/destination-milvus/unit_tests/indexer_test.py new file mode 100644 index 000000000000..f88d064862c3 --- /dev/null +++ b/airbyte-integrations/connectors/destination-milvus/unit_tests/indexer_test.py @@ -0,0 +1,167 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import os +import unittest +from unittest.mock import Mock, call, patch + +from airbyte_cdk.models.airbyte_protocol import AirbyteStream, DestinationSyncMode, SyncMode +from destination_milvus.config import MilvusIndexingConfigModel, NoAuth, TokenAuth +from destination_milvus.indexer import MilvusIndexer +from pymilvus import DataType + + +@patch("destination_milvus.indexer.connections") +@patch("destination_milvus.indexer.utility") +@patch("destination_milvus.indexer.Collection") +class TestMilvusIndexer(unittest.TestCase): + def setUp(self): + self.mock_config = MilvusIndexingConfigModel( + **{ + "host": "https://notmilvus.com", + "collection": "test2", + "auth": { + "mode": "token", + "token": "mytoken", + }, + "vector_field": "vector", + "text_field": "text", + } + ) + self.milvus_indexer = MilvusIndexer(self.mock_config, 128) + self.milvus_indexer._connect_with_timeout = Mock() # Mocking this out to avoid testing multiprocessing + self.milvus_indexer._collection = Mock() + + def test_check_returns_expected_result(self, mock_Collection, mock_utility, mock_connections): + mock_Collection.return_value.describe.return_value = { + "auto_id": True, + "fields": [{"name": "vector", "type": DataType.FLOAT_VECTOR, "params": {"dim": 128}}], + } + + result = self.milvus_indexer.check() + + self.assertIsNone(result) + + mock_Collection.return_value.describe.assert_called() + + def test_check_secure_endpoint(self, mock_Collection, mock_utility, mock_connections): + mock_Collection.return_value.describe.return_value = { + "auto_id": True, + "fields": [{"name": "vector", "type": DataType.FLOAT_VECTOR, "params": {"dim": 128}}], + } + test_cases = [ + ( + "cloud", + "http://example.org", + TokenAuth(mode="token", token="abc"), + "Host must start with https:// and authentication must be enabled on cloud deployment.", + ), + ( + "cloud", + "https://example.org", + NoAuth(mode="no_auth"), + "Host must start with https:// and authentication must be enabled on cloud deployment.", + ), + ("cloud", "https://example.org", TokenAuth(mode="token", token="abc"), None), + ("", "http://example.org", TokenAuth(mode="token", token="abc"), None), + ("", "https://example.org", TokenAuth(mode="token", token="abc"), None), + ("", "https://example.org", NoAuth(mode="no_auth"), None), + ] + for deployment_mode, uri, auth, expected_error_message in test_cases: + os.environ["DEPLOYMENT_MODE"] = deployment_mode + self.milvus_indexer.config.host = uri + self.milvus_indexer.config.auth = auth + + result = self.milvus_indexer.check() + + self.assertEqual(result, expected_error_message) + + def test_check_handles_failure_conditions(self, mock_Collection, mock_utility, mock_connections): + # Test 1: General exception in describe + mock_Collection.return_value.describe.side_effect = Exception("Random exception") + result = self.milvus_indexer.check() + self.assertTrue("Random exception" in result) # Assuming format_exception includes the exception message + + # Test 2: auto_id is not True + mock_Collection.return_value.describe.return_value = {"auto_id": False} + mock_Collection.return_value.describe.side_effect = None + result = self.milvus_indexer.check() + self.assertEqual(result, "Only collections with auto_id are supported") + + # Test 3: Vector field not found + mock_Collection.return_value.describe.return_value = {"auto_id": True, "fields": [{"name": "wrong_vector_field"}]} + result = self.milvus_indexer.check() + self.assertEqual(result, f"Vector field {self.mock_config.vector_field} not found") + + # Test 4: Vector field is not a vector + mock_Collection.return_value.describe.return_value = { + "auto_id": True, + "fields": [{"name": self.mock_config.vector_field, "type": DataType.INT32}], + } + result = self.milvus_indexer.check() + self.assertEqual(result, f"Vector field {self.mock_config.vector_field} is not a vector") + + # Test 5: Vector field dimension mismatch + mock_Collection.return_value.describe.return_value = { + "auto_id": True, + "fields": [{"name": self.mock_config.vector_field, "type": DataType.FLOAT_VECTOR, "params": {"dim": 64}}], + } + result = self.milvus_indexer.check() + self.assertEqual(result, f"Vector field {self.mock_config.vector_field} is not a 128-dimensional vector") + + def test_pre_sync_creates_collection(self, mock_Collection, mock_utility, mock_connections): + self.milvus_indexer.config.collection = "ad_hoc" + self.milvus_indexer.config.vector_field = "my_vector_field" + mock_utility.has_collection.return_value = False + self.milvus_indexer.pre_sync( + Mock(streams=[Mock(destination_sync_mode=DestinationSyncMode.append, stream=Mock(name="some_stream"))]) + ) + mock_Collection.assert_has_calls([call("ad_hoc")]) + mock_Collection.return_value.create_index.assert_has_calls( + [call(field_name="my_vector_field", index_params={"metric_type": "L2", "index_type": "IVF_FLAT", "params": {"nlist": 1024}})] + ) + + def test_pre_sync_calls_delete(self, mock_Collection, mock_utility, mock_connections): + mock_iterator = Mock() + mock_iterator.next.side_effect = [[{"id": 1}], []] + mock_Collection.return_value.query_iterator.return_value = mock_iterator + + self.milvus_indexer.pre_sync( + Mock( + streams=[ + Mock( + destination_sync_mode=DestinationSyncMode.overwrite, + stream=AirbyteStream(name="some_stream", json_schema={}, supported_sync_modes=[SyncMode.full_refresh]), + ) + ] + ) + ) + + mock_Collection.return_value.query_iterator.assert_called_with(expr='_ab_stream == "some_stream"') + mock_Collection.return_value.delete.assert_called_with(expr="id in [1]") + + def test_pre_sync_does_not_call_delete(self, mock_Collection, mock_utility, mock_connections): + self.milvus_indexer.pre_sync( + Mock(streams=[Mock(destination_sync_mode=DestinationSyncMode.append, stream=Mock(name="some_stream"))]) + ) + + mock_Collection.return_value.delete.assert_not_called() + + def test_index_calls_insert(self, mock_Collection, mock_utility, mock_connections): + self.milvus_indexer._primary_key = "id" + self.milvus_indexer.index( + [Mock(metadata={"key": "value", "id": 5}, page_content="some content", embedding=[1, 2, 3])], None, "some_stream" + ) + + self.milvus_indexer._collection.insert.assert_called_with([{"key": "value", "vector": [1, 2, 3], "text": "some content", "_id": 5}]) + + def test_index_calls_delete(self, mock_Collection, mock_utility, mock_connections): + mock_iterator = Mock() + mock_iterator.next.side_effect = [[{"id": "123"}, {"id": "456"}], [{"id": "789"}], []] + self.milvus_indexer._collection.query_iterator.return_value = mock_iterator + + self.milvus_indexer.delete(["some_id"], None, "some_stream") + + self.milvus_indexer._collection.query_iterator.assert_called_with(expr='_ab_record_id in ["some_id"]') + self.milvus_indexer._collection.delete.assert_has_calls([call(expr="id in [123, 456]"), call(expr="id in [789]")], any_order=False) diff --git a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/.dockerignore b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/Dockerfile deleted file mode 100644 index a20abf6c4715..000000000000 --- a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-mongodb-strict-encrypt - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-mongodb-strict-encrypt - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.2.0 -LABEL io.airbyte.name=airbyte/destination-mongodb-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/README.md b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/README.md index be9c9f4026a6..7f0c78e98086 100644 --- a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/README.md +++ b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/README.md @@ -24,4 +24,4 @@ As a community contributor, you will need access to a MongoDB to run tests. ## Airbyte Employee 1. Access the `MONGODB_TEST_CREDS` secret on the LastPass -1. Create a file with the contents at `secrets/credentials.json` \ No newline at end of file +1. Create a file with the contents at `secrets/credentials.json` diff --git a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/build.gradle index b6f5a089a461..106b17a3248c 100644 --- a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/build.gradle @@ -1,27 +1,32 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.mongodb.MongodbDestinationStrictEncrypt' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-config-oss:config-models-oss') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:destination-mongodb') implementation 'org.mongodb:mongodb-driver-sync:4.3.0' - testImplementation libs.connectors.testcontainers.mongodb - - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mongodb-strict-encrypt') - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + testImplementation libs.testcontainers.mongodb } diff --git a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.destination.mongodb/MongodbDestinationStrictEncrypt.java b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.destination.mongodb/MongodbDestinationStrictEncrypt.java index 6da5a5828678..45a068cbf1f3 100644 --- a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.destination.mongodb/MongodbDestinationStrictEncrypt.java +++ b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.destination.mongodb/MongodbDestinationStrictEncrypt.java @@ -6,13 +6,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.spec_modification.SpecModifyingDestination; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.mongodb.MongoUtils; -import io.airbyte.db.mongodb.MongoUtils.MongoInstanceType; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.spec_modification.SpecModifyingDestination; +import io.airbyte.integrations.destination.mongodb.MongoUtils.MongoInstanceType; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.ConnectorSpecification; import org.slf4j.Logger; diff --git a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationStrictEncryptAcceptanceTest.java index 72cc7a8f830b..ec753b76370d 100644 --- a/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationStrictEncryptAcceptanceTest.java @@ -12,13 +12,11 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import com.mongodb.client.MongoCursor; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.mongodb.MongoDatabase; -import io.airbyte.db.mongodb.MongoUtils; -import io.airbyte.db.mongodb.MongoUtils.MongoInstanceType; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.integrations.destination.mongodb.MongoUtils.MongoInstanceType; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; diff --git a/airbyte-integrations/connectors/destination-mongodb/.dockerignore b/airbyte-integrations/connectors/destination-mongodb/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-mongodb/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-mongodb/Dockerfile b/airbyte-integrations/connectors/destination-mongodb/Dockerfile deleted file mode 100644 index 94afbd1579ad..000000000000 --- a/airbyte-integrations/connectors/destination-mongodb/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-mongodb - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-mongodb - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.2.0 -LABEL io.airbyte.name=airbyte/destination-mongodb diff --git a/airbyte-integrations/connectors/destination-mongodb/README.md b/airbyte-integrations/connectors/destination-mongodb/README.md index 4e4cd17ba335..aec60cd49bf8 100644 --- a/airbyte-integrations/connectors/destination-mongodb/README.md +++ b/airbyte-integrations/connectors/destination-mongodb/README.md @@ -14,4 +14,4 @@ As a community contributor, you will need access to a MongoDB to run tests. ## Airbyte Employee 1. Access the `MongoDB Integration Test User` secret on Rippling under the `Engineering` folder -1. Create a file with the contents at `secrets/credentials.json` \ No newline at end of file +1. Create a file with the contents at `secrets/credentials.json` diff --git a/airbyte-integrations/connectors/destination-mongodb/build.gradle b/airbyte-integrations/connectors/destination-mongodb/build.gradle index 413cc042e780..cad1ef429466 100644 --- a/airbyte-integrations/connectors/destination-mongodb/build.gradle +++ b/airbyte-integrations/connectors/destination-mongodb/build.gradle @@ -1,9 +1,23 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.mongodb.MongodbDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] @@ -11,18 +25,13 @@ application { dependencies { implementation 'org.apache.commons:commons-lang3:3.11' - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-config-oss:config-models-oss') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol implementation 'org.mongodb:mongodb-driver-sync:4.3.0' - testImplementation libs.connectors.testcontainers.mongodb - testImplementation project(':airbyte-integrations:bases:standard-destination-test') + // TODO: remove this dependency + implementation libs.google.cloud.storage - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mongodb') - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') + testImplementation libs.testcontainers.mongodb - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mongodb') } diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/mongodb/MongoDatabase.java b/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongoDatabase.java similarity index 98% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/mongodb/MongoDatabase.java rename to airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongoDatabase.java index 072d3abb4474..fe6cffd02290 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/mongodb/MongoDatabase.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongoDatabase.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.mongodb; +package io.airbyte.integrations.destination.mongodb; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; @@ -14,10 +14,10 @@ import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.MongoIterable; +import io.airbyte.cdk.db.AbstractDatabase; import io.airbyte.commons.exceptions.ConnectionErrorException; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.db.AbstractDatabase; import java.util.Collections; import java.util.List; import java.util.Optional; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java b/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongoUtils.java similarity index 99% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java rename to airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongoUtils.java index 982748d9503e..40a0df293b6b 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongoUtils.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.mongodb; +package io.airbyte.integrations.destination.mongodb; import static java.util.Arrays.asList; import static org.bson.BsonType.ARRAY; @@ -25,10 +25,10 @@ import com.mongodb.DBRefCodecProvider; import com.mongodb.client.AggregateIterable; import com.mongodb.client.MongoCollection; +import io.airbyte.cdk.db.DataTypeUtils; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.db.DataTypeUtils; -import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.protocol.models.CommonField; import io.airbyte.protocol.models.JsonSchemaType; import java.util.ArrayList; diff --git a/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbDestination.java b/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbDestination.java index 8497b1bbfabb..dacd5f500265 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbDestination.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbDestination.java @@ -5,7 +5,7 @@ package io.airbyte.integrations.destination.mongodb; import static com.mongodb.client.model.Projections.excludeId; -import static io.airbyte.integrations.base.errors.messages.ErrorMessage.getErrorMessage; +import static io.airbyte.cdk.integrations.base.errors.messages.ErrorMessage.getErrorMessage; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; @@ -14,18 +14,16 @@ import com.mongodb.MongoSecurityException; import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination; import io.airbyte.commons.exceptions.ConnectionErrorException; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.mongodb.MongoDatabase; -import io.airbyte.db.mongodb.MongoUtils; -import io.airbyte.db.mongodb.MongoUtils.MongoInstanceType; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.AirbyteTraceMessageUtility; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.ssh.SshWrappedDestination; +import io.airbyte.integrations.destination.mongodb.MongoUtils.MongoInstanceType; import io.airbyte.integrations.destination.mongodb.exception.MongodbDatabaseException; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbNameTransformer.java b/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbNameTransformer.java index 29ad2fddcb6a..604ddee868fe 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbNameTransformer.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbNameTransformer.java @@ -5,7 +5,7 @@ package io.airbyte.integrations.destination.mongodb; import com.google.common.annotations.VisibleForTesting; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; @VisibleForTesting public class MongodbNameTransformer extends StandardNameTransformer { diff --git a/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumer.java b/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumer.java index f2d8f684d4f7..cd4bb1aeafc9 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumer.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumer.java @@ -9,11 +9,10 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.mongodb.client.MongoCursor; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.lang.Exceptions; -import io.airbyte.db.mongodb.MongoDatabase; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/mongodb/TreeNode.java b/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/TreeNode.java similarity index 95% rename from airbyte-db/db-lib/src/main/java/io/airbyte/db/mongodb/TreeNode.java rename to airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/TreeNode.java index 37ff43b7b5cd..672030f750de 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/mongodb/TreeNode.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/main/java/io/airbyte/integrations/destination/mongodb/TreeNode.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db.mongodb; +package io.airbyte.integrations.destination.mongodb; import java.util.LinkedList; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java index a1e0798ea6f5..9e4d5a0b644a 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java @@ -12,12 +12,11 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import com.mongodb.client.MongoCursor; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.mongodb.MongoDatabase; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import java.io.IOException; import java.util.ArrayList; diff --git a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshKeyMongoDbDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshKeyMongoDbDestinationAcceptanceTest.java index 91629f5db5f8..72bdeffd1399 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshKeyMongoDbDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshKeyMongoDbDestinationAcceptanceTest.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.mongodb; -import io.airbyte.integrations.base.ssh.SshTunnel; -import io.airbyte.integrations.base.ssh.SshTunnel.TunnelMethod; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel.TunnelMethod; public class SshKeyMongoDbDestinationAcceptanceTest extends SshMongoDbDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshMongoDbDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshMongoDbDestinationAcceptanceTest.java index 84b2b992d71e..25885678d05a 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshMongoDbDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshMongoDbDestinationAcceptanceTest.java @@ -9,12 +9,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.mongodb.client.MongoCursor; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.ssh.SshBastionContainer; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.mongodb.MongoDatabase; -import io.airbyte.integrations.base.ssh.SshBastionContainer; -import io.airbyte.integrations.base.ssh.SshTunnel; -import io.airbyte.integrations.util.HostPortResolver; import java.util.ArrayList; import java.util.HashSet; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshPasswordMongoDbDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshPasswordMongoDbDestinationAcceptanceTest.java index 716e877c0001..775f7f4b17f6 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshPasswordMongoDbDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/SshPasswordMongoDbDestinationAcceptanceTest.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.mongodb; -import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; public class SshPasswordMongoDbDestinationAcceptanceTest extends SshMongoDbDestinationAcceptanceTest { diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/MongoUtilsTest.java b/airbyte-integrations/connectors/destination-mongodb/src/test/java/io/airbyte/integrations/destination/mongodb/MongoUtilsTest.java similarity index 93% rename from airbyte-db/db-lib/src/test/java/io/airbyte/db/MongoUtilsTest.java rename to airbyte-integrations/connectors/destination-mongodb/src/test/java/io/airbyte/integrations/destination/mongodb/MongoUtilsTest.java index 10c09fb6a434..46af33f9c3ab 100644 --- a/airbyte-db/db-lib/src/test/java/io/airbyte/db/MongoUtilsTest.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/test/java/io/airbyte/integrations/destination/mongodb/MongoUtilsTest.java @@ -2,9 +2,9 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.db; +package io.airbyte.integrations.destination.mongodb; -import static io.airbyte.db.mongodb.MongoUtils.AIRBYTE_SUFFIX; +import static io.airbyte.integrations.destination.mongodb.MongoUtils.AIRBYTE_SUFFIX; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -12,7 +12,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.mongodb.MongoUtils; import java.util.List; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/connectors/destination-mongodb/src/test/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationTest.java b/airbyte-integrations/connectors/destination-mongodb/src/test/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationTest.java index 9f1c2c9154a7..2364509f719e 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/test/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationTest.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/test/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationTest.java @@ -9,8 +9,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; import java.util.stream.Stream; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.params.ParameterizedTest; diff --git a/airbyte-integrations/connectors/destination-mongodb/src/test/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumerTest.java b/airbyte-integrations/connectors/destination-mongodb/src/test/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumerTest.java index 69482324d56a..8648eb32d48e 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/test/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumerTest.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/test/java/io/airbyte/integrations/destination/mongodb/MongodbRecordConsumerTest.java @@ -4,9 +4,8 @@ package io.airbyte.integrations.destination.mongodb; -import io.airbyte.db.mongodb.MongoDatabase; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; -import io.airbyte.integrations.standardtest.destination.PerStreamStateMessageTest; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.standardtest.destination.PerStreamStateMessageTest; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/connectors/destination-mqtt/.dockerignore b/airbyte-integrations/connectors/destination-mqtt/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-mqtt/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-mqtt/Dockerfile b/airbyte-integrations/connectors/destination-mqtt/Dockerfile deleted file mode 100644 index 478badef864e..000000000000 --- a/airbyte-integrations/connectors/destination-mqtt/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-mqtt - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-mqtt - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.3 -LABEL io.airbyte.name=airbyte/destination-mqtt diff --git a/airbyte-integrations/connectors/destination-mqtt/README.md b/airbyte-integrations/connectors/destination-mqtt/README.md index b3a7036e8832..f9d73b568550 100644 --- a/airbyte-integrations/connectors/destination-mqtt/README.md +++ b/airbyte-integrations/connectors/destination-mqtt/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-mqtt:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-mqtt:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-mqtt:dev`. the Dockerfile. #### Run @@ -61,8 +62,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-mqtt test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/mqtt.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-mqtt/build.gradle b/airbyte-integrations/connectors/destination-mqtt/build.gradle index 02bf1c442610..599b538f4ac2 100644 --- a/airbyte-integrations/connectors/destination-mqtt/build.gradle +++ b/airbyte-integrations/connectors/destination-mqtt/build.gradle @@ -1,25 +1,31 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.mqtt.MqttDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation 'org.eclipse.paho:org.eclipse.paho.client.mqttv3:1.2.5' testImplementation 'com.hivemq:hivemq-testcontainer-junit5:2.0.0' - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mqtt') } diff --git a/airbyte-integrations/connectors/destination-mqtt/src/main/java/io/airbyte/integrations/destination/mqtt/MqttDestination.java b/airbyte-integrations/connectors/destination-mqtt/src/main/java/io/airbyte/integrations/destination/mqtt/MqttDestination.java index fd34026ab901..179ddfb3a96f 100644 --- a/airbyte-integrations/connectors/destination-mqtt/src/main/java/io/airbyte/integrations/destination/mqtt/MqttDestination.java +++ b/airbyte-integrations/connectors/destination-mqtt/src/main/java/io/airbyte/integrations/destination/mqtt/MqttDestination.java @@ -7,12 +7,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Charsets; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-mqtt/src/main/java/io/airbyte/integrations/destination/mqtt/MqttRecordConsumer.java b/airbyte-integrations/connectors/destination-mqtt/src/main/java/io/airbyte/integrations/destination/mqtt/MqttRecordConsumer.java index f8a67146d8f3..5c4c3b0f12d3 100644 --- a/airbyte-integrations/connectors/destination-mqtt/src/main/java/io/airbyte/integrations/destination/mqtt/MqttRecordConsumer.java +++ b/airbyte-integrations/connectors/destination-mqtt/src/main/java/io/airbyte/integrations/destination/mqtt/MqttRecordConsumer.java @@ -7,9 +7,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Charsets; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.lang.Exceptions; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; diff --git a/airbyte-integrations/connectors/destination-mqtt/src/test-integration/java/io/airbyte/integrations/destination/mqtt/MqttDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mqtt/src/test-integration/java/io/airbyte/integrations/destination/mqtt/MqttDestinationAcceptanceTest.java index 67cf210e381e..6cb4ab0658a7 100644 --- a/airbyte-integrations/connectors/destination-mqtt/src/test-integration/java/io/airbyte/integrations/destination/mqtt/MqttDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mqtt/src/test-integration/java/io/airbyte/integrations/destination/mqtt/MqttDestinationAcceptanceTest.java @@ -11,10 +11,10 @@ import com.google.common.collect.Streams; import com.google.common.net.InetAddresses; import com.hivemq.testcontainer.junit5.HiveMQTestContainerExtension; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.net.InetAddress; import java.net.NetworkInterface; import java.net.SocketException; diff --git a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/.dockerignore b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile deleted file mode 100644 index d2d36689b8d1..000000000000 --- a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/Dockerfile +++ /dev/null @@ -1,51 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -RUN yum install -y python3 python3-devel jq sshpass git && yum clean all && \ - alternatives --install /usr/bin/python python /usr/bin/python3 60 && \ - python -m ensurepip --upgrade && \ - pip3 install dbt-sqlserver==1.0.0 - -# Luckily, none of normalization's files conflict with destination-mssql's files :) -# We don't enforce that in any way, but hopefully we're only living in this state for a short time. -COPY --from=airbyte/normalization-mssql:dev /airbyte /airbyte -# Install python dependencies -WORKDIR /airbyte/base_python_structs -RUN pip3 install . -WORKDIR /airbyte/normalization_code -RUN pip3 install . -WORKDIR /airbyte/normalization_code/dbt-template/ -# Download external dbt dependencies -# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15 -RUN pip3 install "urllib3<2" -RUN dbt deps - -WORKDIR /airbyte - -ENV APPLICATION destination-mssql-strict-encrypt -ENV AIRBYTE_NORMALIZATION_INTEGRATION mssql - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-mssql-strict-encrypt - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.2.0 -LABEL io.airbyte.name=airbyte/destination-mssql-strict-encrypt - -ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh" -ENTRYPOINT ["/airbyte/run_with_normalization.sh"] diff --git a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/build.gradle index 45378da4cf27..09e3a703a2b6 100644 --- a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/build.gradle @@ -1,35 +1,37 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = [ + 'db-sources', // required for tests + 'db-destinations', + ] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.mssql_strict_encrypt.MssqlStrictEncryptDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:bases:bases-destination-jdbc') - implementation libs.airbyte.protocol implementation project(':airbyte-integrations:connectors:destination-mssql') - implementation project(':airbyte-test-utils') implementation 'com.microsoft.sqlserver:mssql-jdbc:8.4.1.jre14' testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation libs.connectors.testcontainers.mssqlserver - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mssql-strict-encrypt') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) -} + testImplementation libs.testcontainers.mssqlserver -tasks.named("airbyteDocker") { - dependsOn project(':airbyte-integrations:bases:base-normalization').airbyteDockerMSSql } diff --git a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/gradle.properties b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/gradle.properties index 5d1adb3b55c1..2b147dcf7175 100644 --- a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/gradle.properties +++ b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/gradle.properties @@ -1,3 +1,3 @@ # currently limit the number of parallel threads until further investigation into the issues \ # where integration tests run into race conditions -numberThreads=1 +testExecutionConcurrency=1 diff --git a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/src/main/java/io/airbyte/integrations/destination/mssql_strict_encrypt/MssqlStrictEncryptDestination.java b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/src/main/java/io/airbyte/integrations/destination/mssql_strict_encrypt/MssqlStrictEncryptDestination.java index 9ba9c35bf221..d24ff4e1ee8b 100644 --- a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/src/main/java/io/airbyte/integrations/destination/mssql_strict_encrypt/MssqlStrictEncryptDestination.java +++ b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/src/main/java/io/airbyte/integrations/destination/mssql_strict_encrypt/MssqlStrictEncryptDestination.java @@ -5,10 +5,10 @@ package io.airbyte.integrations.destination.mssql_strict_encrypt; import com.fasterxml.jackson.databind.node.ArrayNode; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.spec_modification.SpecModifyingDestination; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.spec_modification.SpecModifyingDestination; import io.airbyte.integrations.destination.mssql.MSSQLDestination; import io.airbyte.protocol.models.v0.ConnectorSpecification; import org.slf4j.Logger; diff --git a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mssql_strict_encrypt/MssqlStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mssql_strict_encrypt/MssqlStrictEncryptDestinationAcceptanceTest.java index 2dff33999955..1be629003847 100644 --- a/airbyte-integrations/connectors/destination-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mssql_strict_encrypt/MssqlStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mssql_strict_encrypt/MssqlStrictEncryptDestinationAcceptanceTest.java @@ -9,19 +9,19 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.base.ssh.SshHelpers; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.testutils.DatabaseConnectionHelper; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.string.Strings; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.protocol.models.v0.ConnectorSpecification; -import io.airbyte.test.utils.DatabaseConnectionHelper; import java.sql.SQLException; import java.util.ArrayList; import java.util.HashSet; diff --git a/airbyte-integrations/connectors/destination-mssql/.dockerignore b/airbyte-integrations/connectors/destination-mssql/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-mssql/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-mssql/Dockerfile b/airbyte-integrations/connectors/destination-mssql/Dockerfile deleted file mode 100644 index d51950f7c0d5..000000000000 --- a/airbyte-integrations/connectors/destination-mssql/Dockerfile +++ /dev/null @@ -1,54 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -RUN yum install -y python3 python3-devel jq sshpass git && yum clean all && \ - alternatives --install /usr/bin/python python /usr/bin/python3 60 && \ - python -m ensurepip --upgrade && \ - # these two lines are a workaround for https://github.com/yaml/pyyaml/issues/601 - pip3 install wheel && \ - pip3 install "Cython<3.0" "pyyaml==5.4" --no-build-isolation && \ - pip3 install dbt-sqlserver==1.0.0 - -# Luckily, none of normalization's files conflict with destination-mssql's files :) -# We don't enforce that in any way, but hopefully we're only living in this state for a short time. -COPY --from=airbyte/normalization-mssql:dev /airbyte /airbyte -# Install python dependencies -WORKDIR /airbyte/base_python_structs -RUN pip3 install . -WORKDIR /airbyte/normalization_code -RUN pip3 install . -WORKDIR /airbyte/normalization_code/dbt-template/ -# Download external dbt dependencies -# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15 -RUN pip3 install "urllib3<2" -RUN dbt deps - -WORKDIR /airbyte - -ENV APPLICATION destination-mssql -ENV AIRBYTE_NORMALIZATION_INTEGRATION mssql - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-mssql - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.2.0 -LABEL io.airbyte.name=airbyte/destination-mssql - -ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh" -ENTRYPOINT ["/airbyte/run_with_normalization.sh"] diff --git a/airbyte-integrations/connectors/destination-mssql/build.gradle b/airbyte-integrations/connectors/destination-mssql/build.gradle index 300c78a44c21..ba588da10bce 100644 --- a/airbyte-integrations/connectors/destination-mssql/build.gradle +++ b/airbyte-integrations/connectors/destination-mssql/build.gradle @@ -1,33 +1,35 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = [ + 'db-sources', // required for tests + 'db-destinations', + ] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.mssql.MSSQLDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:bases-destination-jdbc') - implementation project(':airbyte-test-utils') implementation 'com.microsoft.sqlserver:mssql-jdbc:8.4.1.jre14' testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation libs.connectors.testcontainers.mssqlserver - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mssql') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) -} - -tasks.named("airbyteDocker") { - dependsOn project(':airbyte-integrations:bases:base-normalization').airbyteDockerMSSql + testImplementation libs.testcontainers.mssqlserver } diff --git a/airbyte-integrations/connectors/destination-mssql/gradle.properties b/airbyte-integrations/connectors/destination-mssql/gradle.properties index 5d1adb3b55c1..2b147dcf7175 100644 --- a/airbyte-integrations/connectors/destination-mssql/gradle.properties +++ b/airbyte-integrations/connectors/destination-mssql/gradle.properties @@ -1,3 +1,3 @@ # currently limit the number of parallel threads until further investigation into the issues \ # where integration tests run into race conditions -numberThreads=1 +testExecutionConcurrency=1 diff --git a/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLDestination.java b/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLDestination.java index ff937a9ba5d8..b23fd171cb78 100644 --- a/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLDestination.java +++ b/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLDestination.java @@ -6,13 +6,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination; +import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.ssh.SshWrappedDestination; -import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import java.io.File; import java.util.HashMap; import java.util.Map; diff --git a/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLNameTransformer.java b/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLNameTransformer.java index 98838535b022..fc070378dbe7 100644 --- a/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLNameTransformer.java +++ b/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/MSSQLNameTransformer.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.mssql; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; public class MSSQLNameTransformer extends StandardNameTransformer { diff --git a/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/SqlServerOperations.java b/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/SqlServerOperations.java index b73227fb1fd6..010793285c89 100644 --- a/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/SqlServerOperations.java +++ b/airbyte-integrations/connectors/destination-mssql/src/main/java/io/airbyte/integrations/destination/mssql/SqlServerOperations.java @@ -6,10 +6,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.SqlOperationsUtils; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperationsUtils; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.sql.SQLException; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTest.java index 584c6d8bacad..ca0b09115616 100644 --- a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTest.java @@ -7,18 +7,18 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.cdk.integrations.util.HostPortResolver; +import io.airbyte.cdk.testutils.DatabaseConnectionHelper; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; -import io.airbyte.integrations.util.HostPortResolver; -import io.airbyte.test.utils.DatabaseConnectionHelper; import java.sql.SQLException; import java.util.HashSet; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTestSSL.java b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTestSSL.java index eaeb3ffc16f2..4991ce4ff163 100644 --- a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTestSSL.java +++ b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationAcceptanceTestSSL.java @@ -7,18 +7,18 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.cdk.integrations.util.HostPortResolver; +import io.airbyte.cdk.testutils.DatabaseConnectionHelper; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; -import io.airbyte.integrations.util.HostPortResolver; -import io.airbyte.test.utils.DatabaseConnectionHelper; import java.sql.SQLException; import java.util.HashSet; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLTestDataComparator.java b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLTestDataComparator.java index 46fdc7c7330b..dbe5814078c1 100644 --- a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLTestDataComparator.java +++ b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/MSSQLTestDataComparator.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.mssql; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import java.time.LocalDate; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; diff --git a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshKeyMSSQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshKeyMSSQLDestinationAcceptanceTest.java index c987365e819b..561042d0f1cc 100644 --- a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshKeyMSSQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshKeyMSSQLDestinationAcceptanceTest.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.mssql; -import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; public class SshKeyMSSQLDestinationAcceptanceTest extends SshMSSQLDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshMSSQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshMSSQLDestinationAcceptanceTest.java index 1cc0ff6a8075..9a627746d5b9 100644 --- a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshMSSQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshMSSQLDestinationAcceptanceTest.java @@ -6,18 +6,18 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.base.ssh.SshBastionContainer; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.base.ssh.SshBastionContainer; -import io.airbyte.integrations.base.ssh.SshTunnel; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; diff --git a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshPasswordMSSQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshPasswordMSSQLDestinationAcceptanceTest.java index 014d48116687..b23963b6635b 100644 --- a/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshPasswordMSSQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mssql/src/test-integration/java/io/airbyte/integrations/destination/mssql/SshPasswordMSSQLDestinationAcceptanceTest.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.mssql; -import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; public class SshPasswordMSSQLDestinationAcceptanceTest extends SshMSSQLDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-mssql/src/test/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationTest.java b/airbyte-integrations/connectors/destination-mssql/src/test/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationTest.java index 38aaf472c56d..2e68640b9d36 100644 --- a/airbyte-integrations/connectors/destination-mssql/src/test/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationTest.java +++ b/airbyte-integrations/connectors/destination-mssql/src/test/java/io/airbyte/integrations/destination/mssql/MSSQLDestinationTest.java @@ -8,9 +8,9 @@ import static org.junit.jupiter.api.Assertions.*; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; -import io.airbyte.db.jdbc.JdbcUtils; import java.util.HashMap; import java.util.Map; import org.junit.jupiter.api.AfterEach; diff --git a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/.dockerignore b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile deleted file mode 100644 index 3488862613b2..000000000000 --- a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-mysql-strict-encrypt - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-mysql-strict-encrypt - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.2.0 -LABEL io.airbyte.name=airbyte/destination-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/README.md b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/README.md index c5cd3a791580..566b2e279c69 100644 --- a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/README.md +++ b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/README.md @@ -2,4 +2,4 @@ In order to test the MySql destination, you need to have the up and running MySql database that has SSL enabled. -This connector inherits the MySql destination, but support SSL connections only. \ No newline at end of file +This connector inherits the MySql destination, but support SSL connections only. diff --git a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/build.gradle index 489ad43d03b4..4f4da7b4aab9 100644 --- a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/build.gradle @@ -1,27 +1,43 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.8.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.mysql.MySQLDestinationStrictEncrypt' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:bases-destination-jdbc') implementation project(':airbyte-integrations:connectors:destination-mysql') implementation 'mysql:mysql-connector-java:8.0.22' - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') + // TODO: declare typing-deduping as a CDK feature instead of importing from source. + implementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') + integrationTestJavaImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:typing-deduping')) + integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mysql') - integrationTestJavaImplementation libs.connectors.testcontainers.mysql + integrationTestJavaImplementation libs.testcontainers.mysql +} - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) +configurations.all { + resolutionStrategy { + force libs.jooq + } } diff --git a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/metadata.yaml index d838a707043b..e03ea32c8f73 100644 --- a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/metadata.yaml @@ -1,13 +1,15 @@ data: registries: cloud: + dockerImageTag: 0.2.0 enabled: false # strict encrypt connectors are deployed to Cloud by their non strict encrypt sibling. oss: + dockerImageTag: 0.2.0 enabled: false # strict encrypt connectors are not used on OSS. connectorSubtype: database connectorType: destination definitionId: ca81ee7c-3163-4246-af40-094cc31e5e42 - dockerImageTag: 0.2.0 + dockerImageTag: 0.3.0 dockerRepository: airbyte/destination-mysql-strict-encrypt githubIssueLabel: destination-mysql icon: mysql.svg diff --git a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestinationStrictEncrypt.java b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestinationStrictEncrypt.java index 778f23512d1f..758ca56d2979 100644 --- a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestinationStrictEncrypt.java +++ b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestinationStrictEncrypt.java @@ -5,11 +5,11 @@ package io.airbyte.integrations.destination.mysql; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.spec_modification.SpecModifyingDestination; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.spec_modification.SpecModifyingDestination; import io.airbyte.protocol.models.v0.ConnectorSpecification; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLStrictEncryptDestinationAcceptanceTest.java index 77f88e227538..49e7776c7d11 100644 --- a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLStrictEncryptDestinationAcceptanceTest.java @@ -7,17 +7,19 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; @@ -73,23 +75,19 @@ protected boolean supportObjectDataTypeTest() { @Override protected JsonNode getConfig() { return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) + .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db)) .put(JdbcUtils.USERNAME_KEY, db.getUsername()) .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) + .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(db)) .build()); } @Override protected JsonNode getFailCheckConfig() { - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, "wrong password") - .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) - .build()); + final JsonNode clone = Jsons.clone(getConfig()); + ((ObjectNode) clone).put("password", "wrong password"); + return clone; } @Override @@ -141,7 +139,7 @@ protected List retrieveNormalizedRecords(final TestDestinationEnv test } @Override - protected void setup(final TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) { + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { db = new MySQLContainer<>("mysql:8.0"); db.start(); setLocalInFileToTrue(); diff --git a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySqlTestDataComparator.java b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySqlTestDataComparator.java index 7f516e9bd8fb..caa587109f6f 100644 --- a/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySqlTestDataComparator.java +++ b/airbyte-integrations/connectors/destination-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySqlTestDataComparator.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.mysql; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; diff --git a/airbyte-integrations/connectors/destination-mysql/.dockerignore b/airbyte-integrations/connectors/destination-mysql/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-mysql/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-mysql/Dockerfile b/airbyte-integrations/connectors/destination-mysql/Dockerfile deleted file mode 100644 index 8152df3ad141..000000000000 --- a/airbyte-integrations/connectors/destination-mysql/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-mysql - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-mysql - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.2.0 -LABEL io.airbyte.name=airbyte/destination-mysql diff --git a/airbyte-integrations/connectors/destination-mysql/build.gradle b/airbyte-integrations/connectors/destination-mysql/build.gradle index ff67bf5575cb..7c04b7df4447 100644 --- a/airbyte-integrations/connectors/destination-mysql/build.gradle +++ b/airbyte-integrations/connectors/destination-mysql/build.gradle @@ -1,26 +1,39 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.8.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.mysql.MySQLDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:bases-destination-jdbc') - implementation 'mysql:mysql-connector-java:8.0.22' + integrationTestJavaImplementation libs.testcontainers.mysql - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mysql') - integrationTestJavaImplementation libs.connectors.testcontainers.mysql + // TODO: declare typing-deduping as a CDK feature instead of importing from source. + implementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') + integrationTestJavaImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:typing-deduping')) +} - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) +configurations.all { + resolutionStrategy { + force libs.jooq + } } diff --git a/airbyte-integrations/connectors/destination-mysql/metadata.yaml b/airbyte-integrations/connectors/destination-mysql/metadata.yaml index ac3c702c6c58..9e6ee1de71e2 100644 --- a/airbyte-integrations/connectors/destination-mysql/metadata.yaml +++ b/airbyte-integrations/connectors/destination-mysql/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: ca81ee7c-3163-4246-af40-094cc31e5e42 - dockerImageTag: 0.2.0 + dockerImageTag: 0.3.0 dockerRepository: airbyte/destination-mysql githubIssueLabel: destination-mysql icon: mysql.svg @@ -14,9 +14,11 @@ data: normalizationTag: 0.4.3 registries: cloud: + dockerImageTag: 0.2.0 dockerRepository: airbyte/destination-mysql-strict-encrypt enabled: true oss: + dockerImageTag: 0.2.0 enabled: true releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/mysql diff --git a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java index bcd2aba7b47b..438086bd8b38 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java +++ b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLDestination.java @@ -4,22 +4,23 @@ package io.airbyte.integrations.destination.mysql; -import static io.airbyte.integrations.base.errors.messages.ErrorMessage.getErrorMessage; +import static io.airbyte.cdk.integrations.base.errors.messages.ErrorMessage.getErrorMessage; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination; +import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; import io.airbyte.commons.exceptions.ConnectionErrorException; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.AirbyteTraceMessageUtility; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.ssh.SshWrappedDestination; -import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import io.airbyte.integrations.destination.mysql.MySQLSqlOperations.VersionCompatibility; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; @@ -127,6 +128,11 @@ public JsonNode toJdbcConfig(final JsonNode config) { return Jsons.jsonNode(configBuilder.build()); } + @Override + protected JdbcSqlGenerator getSqlGenerator() { + throw new UnsupportedOperationException("mysql does not yet support DV2"); + } + public static void main(final String[] args) throws Exception { final Destination destination = MySQLDestination.sshWrappedDestination(); LOGGER.info("starting destination: {}", MySQLDestination.class); diff --git a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLNameTransformer.java b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLNameTransformer.java index 9e7cd8574929..c711f634eef2 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLNameTransformer.java +++ b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLNameTransformer.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.mysql; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; /** * Note that MySQL documentation discusses about identifiers case sensitivity using the diff --git a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLSqlOperations.java b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLSqlOperations.java index ef997318e8eb..2fa5ec8dd572 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLSqlOperations.java +++ b/airbyte-integrations/connectors/destination-mysql/src/main/java/io/airbyte/integrations/destination/mysql/MySQLSqlOperations.java @@ -6,11 +6,11 @@ import com.fasterxml.jackson.databind.JsonNode; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.JdbcSqlOperations; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.JdbcSqlOperations; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -32,7 +32,7 @@ public void executeTransaction(final JdbcDatabase database, final List q @Override public void insertRecordsInternal(final JdbcDatabase database, - final List records, + final List records, final String schemaName, final String tmpTableName) throws SQLException { @@ -52,8 +52,17 @@ public void insertRecordsInternal(final JdbcDatabase database, } } + @Override + protected void insertRecordsInternalV2(final JdbcDatabase database, + final List records, + final String schemaName, + final String tableName) + throws Exception { + throw new UnsupportedOperationException("mysql does not yet support DV2"); + } + private void loadDataIntoTable(final JdbcDatabase database, - final List records, + final List records, final String schemaName, final String tmpTableName, final File tmpFile) diff --git a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java index 9706f188dff8..b6d83448bf46 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySQLDestinationAcceptanceTest.java @@ -11,15 +11,17 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; @@ -35,6 +37,7 @@ import java.util.stream.Collectors; import org.jooq.DSLContext; import org.jooq.SQLDialect; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.testcontainers.containers.MySQLContainer; @@ -43,7 +46,7 @@ public class MySQLDestinationAcceptanceTest extends JdbcDestinationAcceptanceTes protected static final String USERNAME_WITHOUT_PERMISSION = "new_user"; protected static final String PASSWORD_WITHOUT_PERMISSION = "new_password"; - private MySQLContainer db; + protected MySQLContainer db; private final StandardNameTransformer namingResolver = new MySQLNameTransformer(); @Override @@ -79,25 +82,31 @@ protected boolean supportObjectDataTypeTest() { @Override protected JsonNode getConfig() { return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) + .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db)) .put(JdbcUtils.USERNAME_KEY, db.getUsername()) .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) + .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(db)) .put(JdbcUtils.SSL_KEY, false) .build()); } + /** + * {@link #getConfig()} returns a config with host/port set to the in-docker values. This works for + * running the destination-mysql container, but we have some tests which run the destination code + * directly from the JUnit process. These tests need to connect using the "normal" host/port. + */ + private JsonNode getConfigForBareMetalConnection() { + return ((ObjectNode) getConfig()) + .put(JdbcUtils.HOST_KEY, db.getHost()) + .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()); + } + @Override protected JsonNode getFailCheckConfig() { - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, "wrong password") - .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) - .put(JdbcUtils.SSL_KEY, false) - .build()); + final ObjectNode config = (ObjectNode) getConfig(); + config.put(JdbcUtils.PASSWORD_KEY, "wrong password"); + return config; } @Override @@ -260,58 +269,76 @@ protected void assertSameValue(final JsonNode expectedValue, final JsonNode actu @Test void testCheckIncorrectPasswordFailure() { - final JsonNode config = ((ObjectNode) getConfig()).put(JdbcUtils.PASSWORD_KEY, "fake"); + final JsonNode config = ((ObjectNode) getConfigForBareMetalConnection()).put(JdbcUtils.PASSWORD_KEY, "fake"); final MySQLDestination destination = new MySQLDestination(); final AirbyteConnectionStatus status = destination.check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 28000; Error code: 1045;")); + assertStringContains(status.getMessage(), "State code: 28000; Error code: 1045;"); } @Test public void testCheckIncorrectUsernameFailure() { - final JsonNode config = ((ObjectNode) getConfig()).put(JdbcUtils.USERNAME_KEY, "fake"); + final JsonNode config = ((ObjectNode) getConfigForBareMetalConnection()).put(JdbcUtils.USERNAME_KEY, "fake"); final MySQLDestination destination = new MySQLDestination(); final AirbyteConnectionStatus status = destination.check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 28000; Error code: 1045;")); + assertStringContains(status.getMessage(), "State code: 28000; Error code: 1045;"); } @Test public void testCheckIncorrectHostFailure() { - final JsonNode config = ((ObjectNode) getConfig()).put(JdbcUtils.HOST_KEY, "localhost2"); + final JsonNode config = ((ObjectNode) getConfigForBareMetalConnection()).put(JdbcUtils.HOST_KEY, "localhost2"); final MySQLDestination destination = new MySQLDestination(); final AirbyteConnectionStatus status = destination.check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 08S01;")); + assertStringContains(status.getMessage(), "State code: 08S01;"); } @Test public void testCheckIncorrectPortFailure() { - final JsonNode config = ((ObjectNode) getConfig()).put(JdbcUtils.PORT_KEY, "0000"); + final JsonNode config = ((ObjectNode) getConfigForBareMetalConnection()).put(JdbcUtils.PORT_KEY, "0000"); final MySQLDestination destination = new MySQLDestination(); final AirbyteConnectionStatus status = destination.check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 08S01;")); + assertStringContains(status.getMessage(), "State code: 08S01;"); } @Test public void testCheckIncorrectDataBaseFailure() { - final JsonNode config = ((ObjectNode) getConfig()).put(JdbcUtils.DATABASE_KEY, "wrongdatabase"); + final JsonNode config = ((ObjectNode) getConfigForBareMetalConnection()).put(JdbcUtils.DATABASE_KEY, "wrongdatabase"); final MySQLDestination destination = new MySQLDestination(); final AirbyteConnectionStatus status = destination.check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 42000; Error code: 1049;")); + assertStringContains(status.getMessage(), "State code: 42000; Error code: 1049;"); } @Test public void testUserHasNoPermissionToDataBase() { executeQuery("create user '" + USERNAME_WITHOUT_PERMISSION + "'@'%' IDENTIFIED BY '" + PASSWORD_WITHOUT_PERMISSION + "';\n"); - final JsonNode config = ((ObjectNode) getConfig()).put(JdbcUtils.USERNAME_KEY, USERNAME_WITHOUT_PERMISSION); + final JsonNode config = ((ObjectNode) getConfigForBareMetalConnection()).put(JdbcUtils.USERNAME_KEY, USERNAME_WITHOUT_PERMISSION); ((ObjectNode) config).put(JdbcUtils.PASSWORD_KEY, PASSWORD_WITHOUT_PERMISSION); final MySQLDestination destination = new MySQLDestination(); final AirbyteConnectionStatus status = destination.check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 42000; Error code: 1044;")); + assertStringContains(status.getMessage(), "State code: 42000; Error code: 1044;"); + } + + private static void assertStringContains(final String str, final String target) { + assertTrue(str.contains(target), "Expected message to contain \"" + target + "\" but got " + str); + } + + /** + * Legacy mysql normalization is broken, and uses the FLOAT type for numbers. This rounds off e.g. + * 12345.678 to 12345.7. We can fix this in DV2, but will not fix legacy normalization. As such, + * disabling the test case. + */ + @Override + @Disabled("MySQL normalization uses the wrong datatype for numbers. This will not be fixed, because we intend to replace normalization with DV2.") + public void testDataTypeTestWithNormalization(final String messagesFilename, + final String catalogFilename, + final DataTypeTestArgumentProvider.TestCompatibility testCompatibility) + throws Exception { + super.testDataTypeTestWithNormalization(messagesFilename, catalogFilename, testCompatibility); } } diff --git a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySqlTestDataComparator.java b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySqlTestDataComparator.java index 7f516e9bd8fb..caa587109f6f 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySqlTestDataComparator.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/MySqlTestDataComparator.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.mysql; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import java.time.LocalDate; import java.time.format.DateTimeFormatter; import java.util.ArrayList; diff --git a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshMySQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshMySQLDestinationAcceptanceTest.java index bceb783f701e..0750f3393ae3 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshMySQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshMySQLDestinationAcceptanceTest.java @@ -6,18 +6,19 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.base.ssh.SshTunnel; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.nio.file.Path; import java.util.HashSet; import java.util.List; @@ -25,6 +26,7 @@ import org.apache.commons.lang3.RandomStringUtils; import org.jooq.DSLContext; import org.jooq.SQLDialect; +import org.junit.jupiter.api.Disabled; /** * Abstract class that allows us to avoid duplicating testing logic for testing SSH with a key file @@ -159,4 +161,17 @@ protected void tearDown(final TestDestinationEnv testEnv) throws Exception { }); } + /** + * Disabled for the same reason as in {@link MySQLDestinationAcceptanceTest}. But for some reason, + * this class doesn't extend that one so we have to do it again. + */ + @Override + @Disabled("MySQL normalization uses the wrong datatype for numbers. This will not be fixed, because we intend to replace normalization with DV2.") + public void testDataTypeTestWithNormalization(final String messagesFilename, + final String catalogFilename, + final DataTypeTestArgumentProvider.TestCompatibility testCompatibility) + throws Exception { + super.testDataTypeTestWithNormalization(messagesFilename, catalogFilename, testCompatibility); + } + } diff --git a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshPasswordMySQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshPasswordMySQLDestinationAcceptanceTest.java index 59c63a0e2feb..2abd73408c03 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshPasswordMySQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SshPasswordMySQLDestinationAcceptanceTest.java @@ -5,6 +5,7 @@ package io.airbyte.integrations.destination.mysql; import java.nio.file.Path; +import org.junit.jupiter.api.Disabled; public class SshPasswordMySQLDestinationAcceptanceTest extends SshMySQLDestinationAcceptanceTest { @@ -13,4 +14,26 @@ public Path getConfigFilePath() { return Path.of("secrets/ssh-pwd-config.json"); } + /** + * Legacy normalization doesn't correctly parse the SSH password (or something). All tests involving + * the normalization container are broken. That's (mostly) fine; DV2 doesn't rely on that container. + */ + @Override + @Disabled("Our dbt interface doesn't correctly parse the SSH password. Won't fix this test, since DV2 will replace normalization.") + public void testSyncWithNormalization(final String messagesFilename, final String catalogFilename) + throws Exception { + super.testSyncWithNormalization(messagesFilename, catalogFilename); + } + + /** + * Similar to {@link #testSyncWithNormalization(String, String)}, disable the custom dbt test. + *

      + * TODO: get custom dbt transformations working https://github.com/airbytehq/airbyte/issues/33547 + */ + @Override + @Disabled("Our dbt interface doesn't correctly parse the SSH password. https://github.com/airbytehq/airbyte/issues/33547 to fix this.") + public void testCustomDbtTransformations() throws Exception { + super.testCustomDbtTransformations(); + } + } diff --git a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SslMySQLDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SslMySQLDestinationAcceptanceTest.java index 0d7636503e16..815e661bbbc1 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SslMySQLDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test-integration/java/io/airbyte/integrations/destination/mysql/SslMySQLDestinationAcceptanceTest.java @@ -9,12 +9,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import java.sql.SQLException; import java.util.HashSet; @@ -23,22 +24,20 @@ import org.jooq.DSLContext; import org.jooq.SQLDialect; import org.junit.jupiter.api.Test; -import org.testcontainers.containers.MySQLContainer; public class SslMySQLDestinationAcceptanceTest extends MySQLDestinationAcceptanceTest { - private MySQLContainer db; private DSLContext dslContext; private final StandardNameTransformer namingResolver = new MySQLNameTransformer(); @Override protected JsonNode getConfig() { return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) + .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db)) .put(JdbcUtils.USERNAME_KEY, db.getUsername()) .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) + .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(db)) .put(JdbcUtils.SSL_KEY, true) .build()); } @@ -46,11 +45,11 @@ protected JsonNode getConfig() { @Override protected JsonNode getFailCheckConfig() { return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) + .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db)) .put(JdbcUtils.USERNAME_KEY, db.getUsername()) .put(JdbcUtils.PASSWORD_KEY, "wrong password") .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) + .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(db)) .put(JdbcUtils.SSL_KEY, false) .build()); } @@ -86,8 +85,7 @@ public void testCustomDbtTransformations() { @Override protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { - db = new MySQLContainer<>("mysql:8.0"); - db.start(); + super.setup(testEnv, TEST_SCHEMAS); dslContext = DSLContextFactory.create( db.getUsername(), @@ -98,10 +96,6 @@ protected void setup(final TestDestinationEnv testEnv, final HashSet TES db.getFirstMappedPort(), db.getDatabaseName()), SQLDialect.DEFAULT); - - setLocalInFileToTrue(); - revokeAllPermissions(); - grantCorrectPermissions(); } @Override @@ -151,6 +145,7 @@ private void executeQuery(final String query) { } } + @Override @Test public void testUserHasNoPermissionToDataBase() { executeQuery("create user '" + USERNAME_WITHOUT_PERMISSION + "'@'%' IDENTIFIED BY '" + PASSWORD_WITHOUT_PERMISSION + "';\n"); diff --git a/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java b/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java index 6c1e194c3dea..ba80875e2a44 100644 --- a/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java +++ b/airbyte-integrations/connectors/destination-mysql/src/test/java/io/airbyte/integrations/destination/mysql/MySQLDestinationTest.java @@ -8,8 +8,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; import java.util.Map; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/.dockerignore b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile deleted file mode 100644 index 9ce70339438b..000000000000 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-oracle-strict-encrypt - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-oracle-strict-encrypt - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.2.0 -LABEL io.airbyte.name=airbyte/destination-oracle-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/README.md b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/README.md index 07193021227a..8caf97be3a55 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/README.md +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/README.md @@ -2,4 +2,4 @@ In order to test the Oracle destination, you need to have the up and running Oracle database that has SSL enabled. -This connector inherits the Oracle destination, but support SSL connections only. \ No newline at end of file +This connector inherits the Oracle destination, but support SSL connections only. diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle index 9292f9d99ca8..0e940345ab00 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/build.gradle @@ -1,9 +1,23 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.oracle_strict_encrypt.OracleStrictEncryptDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] @@ -14,22 +28,12 @@ dependencies { // required so that log4j uses a standard xml parser instead of an oracle one (that gets pulled in by the oracle driver) implementation group: 'xerces', name: 'xercesImpl', version: '2.12.1' - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:bases-destination-jdbc') implementation project(':airbyte-integrations:connectors:destination-oracle') implementation "com.oracle.database.jdbc:ojdbc8-production:19.7.0.0" - testImplementation project(':airbyte-test-utils') - testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation libs.connectors.destination.testcontainers.oracle.xe + testImplementation libs.testcontainers.oracle.xe - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-oracle') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/main/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestination.java b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/main/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestination.java index 5aba8a47d85a..dc9decce174f 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/main/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestination.java +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/main/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestination.java @@ -7,11 +7,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.spec_modification.SpecModifyingDestination; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.spec_modification.SpecModifyingDestination; import io.airbyte.integrations.destination.oracle.OracleDestination; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java index 68b1f7411622..72e2a11ce32f 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationAcceptanceTest.java @@ -10,19 +10,19 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.integrations.destination.oracle.OracleDestination; import io.airbyte.integrations.destination.oracle.OracleNameTransformer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import java.sql.SQLException; import java.util.ArrayList; import java.util.HashSet; diff --git a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationTest.java b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationTest.java index b1c65339db92..20a584c54134 100644 --- a/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationTest.java +++ b/airbyte-integrations/connectors/destination-oracle-strict-encrypt/src/test/java/io/airbyte/integrations/destination/oracle_strict_encrypt/OracleStrictEncryptDestinationTest.java @@ -4,9 +4,9 @@ package io.airbyte.integrations.destination.oracle_strict_encrypt; +import io.airbyte.cdk.integrations.base.ssh.SshHelpers; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.protocol.models.v0.ConnectorSpecification; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/connectors/destination-oracle/.dockerignore b/airbyte-integrations/connectors/destination-oracle/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-oracle/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-oracle/Dockerfile b/airbyte-integrations/connectors/destination-oracle/Dockerfile deleted file mode 100644 index 04cd94af0dc0..000000000000 --- a/airbyte-integrations/connectors/destination-oracle/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-oracle - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-oracle - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.2.0 -LABEL io.airbyte.name=airbyte/destination-oracle diff --git a/airbyte-integrations/connectors/destination-oracle/build.gradle b/airbyte-integrations/connectors/destination-oracle/build.gradle index f4676000e966..a192ee34744a 100644 --- a/airbyte-integrations/connectors/destination-oracle/build.gradle +++ b/airbyte-integrations/connectors/destination-oracle/build.gradle @@ -1,9 +1,23 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.oracle.OracleDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] @@ -14,19 +28,9 @@ dependencies { // required so that log4j uses a standard xml parser instead of an oracle one (that gets pulled in by the oracle driver) implementation group: 'xerces', name: 'xercesImpl', version: '2.12.1' - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:bases-destination-jdbc') implementation "com.oracle.database.jdbc:ojdbc8-production:19.7.0.0" testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation libs.connectors.destination.testcontainers.oracle.xe - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-oracle') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) + testImplementation libs.testcontainers.oracle.xe } diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java index 02c670bf39e0..9a515ef1f74e 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleDestination.java @@ -6,14 +6,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination; +import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.base.ssh.SshWrappedDestination; -import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import java.io.IOException; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleNameTransformer.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleNameTransformer.java index 729b4246a7c6..ace575355050 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleNameTransformer.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleNameTransformer.java @@ -5,7 +5,7 @@ package io.airbyte.integrations.destination.oracle; import com.google.common.annotations.VisibleForTesting; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import java.util.UUID; @VisibleForTesting diff --git a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java index 2d041d28b92f..468e33bd7345 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java +++ b/airbyte-integrations/connectors/destination-oracle/src/main/java/io/airbyte/integrations/destination/oracle/OracleOperations.java @@ -5,10 +5,10 @@ package io.airbyte.integrations.destination.oracle; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.SqlOperations; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.sql.PreparedStatement; import java.sql.SQLException; diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java index cb1ff56c3d89..fc946fdddae9 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/NneOracleDestinationAcceptanceTest.java @@ -12,11 +12,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; import java.sql.SQLException; import java.util.List; import java.util.Map; diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/OracleTestDataComparator.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/OracleTestDataComparator.java index 1cf97b092b9c..96237a13bf68 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/OracleTestDataComparator.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/OracleTestDataComparator.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.oracle; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import java.util.ArrayList; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshKeyOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshKeyOracleDestinationAcceptanceTest.java index daff3762d1b0..1bf5a766df52 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshKeyOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshKeyOracleDestinationAcceptanceTest.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.oracle; -import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; public class SshKeyOracleDestinationAcceptanceTest extends SshOracleDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java index 2e1e10b1bab7..10ce0fde6c7a 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshOracleDestinationAcceptanceTest.java @@ -7,18 +7,18 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.base.ssh.SshBastionContainer; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.base.ssh.SshBastionContainer; -import io.airbyte.integrations.base.ssh.SshTunnel; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.io.IOException; import java.util.HashSet; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshPasswordOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshPasswordOracleDestinationAcceptanceTest.java index 0e46d352bf59..44f607758dc9 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshPasswordOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/SshPasswordOracleDestinationAcceptanceTest.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.oracle; -import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; public class SshPasswordOracleDestinationAcceptanceTest extends SshOracleDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java index f492d9850437..d87d36041168 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test-integration/java/io/airbyte/integrations/destination/oracle/UnencryptedOracleDestinationAcceptanceTest.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.oracle; -import static io.airbyte.integrations.util.HostPortResolver.resolveHost; -import static io.airbyte.integrations.util.HostPortResolver.resolvePort; +import static io.airbyte.cdk.integrations.util.HostPortResolver.resolveHost; +import static io.airbyte.cdk.integrations.util.HostPortResolver.resolvePort; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; @@ -13,18 +13,18 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.sql.SQLException; import java.util.ArrayList; import java.util.HashSet; diff --git a/airbyte-integrations/connectors/destination-oracle/src/test/java/io/airbyte/integrations/destination/oracle/OracleDestinationTest.java b/airbyte-integrations/connectors/destination-oracle/src/test/java/io/airbyte/integrations/destination/oracle/OracleDestinationTest.java index 66d6d4f1da2a..51e172379c4a 100644 --- a/airbyte-integrations/connectors/destination-oracle/src/test/java/io/airbyte/integrations/destination/oracle/OracleDestinationTest.java +++ b/airbyte-integrations/connectors/destination-oracle/src/test/java/io/airbyte/integrations/destination/oracle/OracleDestinationTest.java @@ -10,9 +10,9 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; -import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.destination.oracle.OracleDestination.Protocol; import java.util.HashMap; import java.util.Map; diff --git a/airbyte-integrations/connectors/destination-pinecone/.dockerignore b/airbyte-integrations/connectors/destination-pinecone/.dockerignore new file mode 100644 index 000000000000..e494245da539 --- /dev/null +++ b/airbyte-integrations/connectors/destination-pinecone/.dockerignore @@ -0,0 +1,5 @@ +* +!Dockerfile +!main.py +!destination_pinecone +!setup.py diff --git a/airbyte-integrations/connectors/destination-pinecone/README.md b/airbyte-integrations/connectors/destination-pinecone/README.md new file mode 100644 index 000000000000..b4a52f0ff177 --- /dev/null +++ b/airbyte-integrations/connectors/destination-pinecone/README.md @@ -0,0 +1,148 @@ +# Pinecone Destination + +This is the repository for the Pinecone destination connector, written in Python. + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.10.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/destinations/pinecone) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_pinecone/spec.json` file. +Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `destination pinecone test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=destination-pinecone build +``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/destination-pinecone:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") +``` + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/destination-pinecone:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] +``` +Please use this as an example. This is not optimized. + +2. Build your image: +```bash +docker build -t airbyte/destination-pinecone:dev . +# Running the spec command against your patched connector +docker run airbyte/destination-pinecone:dev spec +``` +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/destination-pinecone:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-pinecone:dev check --config /secrets/config.json +# messages.jsonl is a file containing line-separated JSON representing AirbyteMessages +cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-pinecone:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=destination-pinecone test +``` + +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-pinecone test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/pinecone.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-pinecone/acceptance-test-config.yml b/airbyte-integrations/connectors/destination-pinecone/acceptance-test-config.yml new file mode 100644 index 000000000000..4a9a8ce0d17f --- /dev/null +++ b/airbyte-integrations/connectors/destination-pinecone/acceptance-test-config.yml @@ -0,0 +1,5 @@ +acceptance_tests: + spec: + tests: + - spec_path: integration_tests/spec.json +connector_image: airbyte/destination-pinecone:dev diff --git a/airbyte-integrations/connectors/destination-pinecone/bootstrap.md b/airbyte-integrations/connectors/destination-pinecone/bootstrap.md new file mode 100644 index 000000000000..cd6d535124d3 --- /dev/null +++ b/airbyte-integrations/connectors/destination-pinecone/bootstrap.md @@ -0,0 +1,8 @@ +# Pinecone Destination Connector Bootstrap + +This destination does three things: +* Split records into chunks and separates metadata from text data +* Embeds text data into an embedding vector +* Stores the metadata and embedding vector in Pinecone + +The record processing is using the text split components from https://python.langchain.com/docs/modules/data_connection/document_transformers/. \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/__init__.py b/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/__init__.py new file mode 100644 index 000000000000..855c38726121 --- /dev/null +++ b/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .destination import DestinationPinecone + +__all__ = ["DestinationPinecone"] diff --git a/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/config.py b/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/config.py new file mode 100644 index 000000000000..38ade8d0834a --- /dev/null +++ b/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/config.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.destinations.vector_db_based.config import VectorDBConfigModel +from pydantic import BaseModel, Field + + +class PineconeIndexingModel(BaseModel): + pinecone_key: str = Field( + ..., + title="Pinecone API key", + airbyte_secret=True, + description="The Pinecone API key to use matching the environment (copy from Pinecone console)", + ) + pinecone_environment: str = Field( + ..., title="Pinecone Environment", description="Pinecone Cloud environment to use", examples=["us-west1-gcp", "gcp-starter"] + ) + index: str = Field(..., title="Index", description="Pinecone index in your project to load data into") + + class Config: + title = "Indexing" + schema_extra = { + "description": "Pinecone is a popular vector store that can be used to store and retrieve embeddings.", + "group": "indexing", + } + + +class ConfigModel(VectorDBConfigModel): + indexing: PineconeIndexingModel diff --git a/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/destination.py b/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/destination.py new file mode 100644 index 000000000000..a8299e0e2710 --- /dev/null +++ b/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/destination.py @@ -0,0 +1,56 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from typing import Any, Iterable, Mapping + +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.destinations import Destination +from airbyte_cdk.destinations.vector_db_based.document_processor import DocumentProcessor +from airbyte_cdk.destinations.vector_db_based.embedder import Embedder, create_from_config +from airbyte_cdk.destinations.vector_db_based.indexer import Indexer +from airbyte_cdk.destinations.vector_db_based.writer import Writer +from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, ConnectorSpecification, Status +from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode +from destination_pinecone.config import ConfigModel +from destination_pinecone.indexer import PineconeIndexer + +BATCH_SIZE = 32 + + +class DestinationPinecone(Destination): + indexer: Indexer + embedder: Embedder + + def _init_indexer(self, config: ConfigModel): + self.embedder = create_from_config(config.embedding, config.processing) + self.indexer = PineconeIndexer(config.indexing, self.embedder.embedding_dimensions) + + def write( + self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] + ) -> Iterable[AirbyteMessage]: + config_model = ConfigModel.parse_obj(config) + self._init_indexer(config_model) + writer = Writer( + config_model.processing, self.indexer, self.embedder, batch_size=BATCH_SIZE, omit_raw_text=config_model.omit_raw_text + ) + yield from writer.write(configured_catalog, input_messages) + + def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: + parsed_config = ConfigModel.parse_obj(config) + self._init_indexer(parsed_config) + checks = [self.embedder.check(), self.indexer.check(), DocumentProcessor.check_config(parsed_config.processing)] + errors = [error for error in checks if error is not None] + if len(errors) > 0: + return AirbyteConnectionStatus(status=Status.FAILED, message="\n".join(errors)) + else: + return AirbyteConnectionStatus(status=Status.SUCCEEDED) + + def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: + return ConnectorSpecification( + documentationUrl="https://docs.airbyte.com/integrations/destinations/pinecone", + supportsIncremental=True, + supported_destination_sync_modes=[DestinationSyncMode.overwrite, DestinationSyncMode.append, DestinationSyncMode.append_dedup], + connectionSpecification=ConfigModel.schema(), # type: ignore[attr-defined] + ) diff --git a/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/indexer.py b/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/indexer.py new file mode 100644 index 000000000000..c09269f20268 --- /dev/null +++ b/airbyte-integrations/connectors/destination-pinecone/destination_pinecone/indexer.py @@ -0,0 +1,130 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import uuid +from typing import Optional + +import pinecone +import urllib3 +from airbyte_cdk.destinations.vector_db_based.document_processor import METADATA_RECORD_ID_FIELD, METADATA_STREAM_FIELD +from airbyte_cdk.destinations.vector_db_based.indexer import Indexer +from airbyte_cdk.destinations.vector_db_based.utils import create_chunks, create_stream_identifier, format_exception +from airbyte_cdk.models.airbyte_protocol import ConfiguredAirbyteCatalog, DestinationSyncMode +from destination_pinecone.config import PineconeIndexingModel + +# large enough to speed up processing, small enough to not hit pinecone request limits +PINECONE_BATCH_SIZE = 40 + +# do not flood the server with too many connections in parallel +PARALLELISM_LIMIT = 4 + +MAX_METADATA_SIZE = 40_960 - 10_000 + +MAX_IDS_PER_DELETE = 1000 + + +class PineconeIndexer(Indexer): + config: PineconeIndexingModel + + def __init__(self, config: PineconeIndexingModel, embedding_dimensions: int): + super().__init__(config) + pinecone.init(api_key=config.pinecone_key, environment=config.pinecone_environment, threaded=True) + + self.pinecone_index = pinecone.GRPCIndex(config.index) + self.embedding_dimensions = embedding_dimensions + + def pre_sync(self, catalog: ConfiguredAirbyteCatalog): + index_description = pinecone.describe_index(self.config.index) + self._pod_type = index_description.pod_type + for stream in catalog.streams: + if stream.destination_sync_mode == DestinationSyncMode.overwrite: + self.delete_vectors( + filter={METADATA_STREAM_FIELD: create_stream_identifier(stream.stream)}, namespace=stream.stream.namespace + ) + + def post_sync(self): + return [] + + def delete_vectors(self, filter, namespace=None): + if self._pod_type == "starter": + # Starter pod types have a maximum of 100000 rows + top_k = 10000 + self.delete_by_metadata(filter, top_k, namespace) + else: + self.pinecone_index.delete(filter=filter, namespace=namespace) + + def delete_by_metadata(self, filter, top_k, namespace=None): + zero_vector = [0.0] * self.embedding_dimensions + query_result = self.pinecone_index.query(vector=zero_vector, filter=filter, top_k=top_k, namespace=namespace) + while len(query_result.matches) > 0: + vector_ids = [doc.id for doc in query_result.matches] + if len(vector_ids) > 0: + # split into chunks of 1000 ids to avoid id limit + batches = create_chunks(vector_ids, batch_size=MAX_IDS_PER_DELETE) + for batch in batches: + self.pinecone_index.delete(ids=list(batch), namespace=namespace) + query_result = self.pinecone_index.query(vector=zero_vector, filter=filter, top_k=top_k, namespace=namespace) + + def _truncate_metadata(self, metadata: dict) -> dict: + """ + Normalize metadata to ensure it is within the size limit and doesn't contain complex objects. + """ + result = {} + current_size = 0 + + for key, value in metadata.items(): + if isinstance(value, (str, int, float, bool)) or (isinstance(value, list) and all(isinstance(item, str) for item in value)): + # Calculate the size of the key and value + item_size = len(str(key)) + len(str(value)) + + # Check if adding the item exceeds the size limit + if current_size + item_size <= MAX_METADATA_SIZE: + result[key] = value + current_size += item_size + + return result + + def index(self, document_chunks, namespace, stream): + pinecone_docs = [] + for i in range(len(document_chunks)): + chunk = document_chunks[i] + metadata = self._truncate_metadata(chunk.metadata) + if chunk.page_content is not None: + metadata["text"] = chunk.page_content + pinecone_docs.append((str(uuid.uuid4()), chunk.embedding, metadata)) + serial_batches = create_chunks(pinecone_docs, batch_size=PINECONE_BATCH_SIZE * PARALLELISM_LIMIT) + for batch in serial_batches: + async_results = [ + self.pinecone_index.upsert(vectors=ids_vectors_chunk, async_req=True, show_progress=False, namespace=namespace) + for ids_vectors_chunk in create_chunks(batch, batch_size=PINECONE_BATCH_SIZE) + ] + # Wait for and retrieve responses (this raises in case of error) + [async_result.result() for async_result in async_results] + + def delete(self, delete_ids, namespace, stream): + if len(delete_ids) > 0: + self.delete_vectors(filter={METADATA_RECORD_ID_FIELD: {"$in": delete_ids}}, namespace=namespace) + + def check(self) -> Optional[str]: + try: + indexes = pinecone.list_indexes() + if self.config.index not in indexes: + return f"Index {self.config.index} does not exist in environment {self.config.pinecone_environment}." + + description = pinecone.describe_index(self.config.index) + actual_dimension = int(description.dimension) + if actual_dimension != self.embedding_dimensions: + return f"Your embedding configuration will produce vectors with dimension {self.embedding_dimensions:d}, but your index is configured with dimension {actual_dimension:d}. Make sure embedding and indexing configurations match." + except Exception as e: + if isinstance(e, urllib3.exceptions.MaxRetryError): + if f"Failed to resolve 'controller.{self.config.pinecone_environment}.pinecone.io'" in str(e.reason): + return f"Failed to resolve environment, please check whether {self.config.pinecone_environment} is correct." + + if isinstance(e, pinecone.exceptions.UnauthorizedException): + if e.body: + return e.body + + formatted_exception = format_exception(e) + return formatted_exception + return None diff --git a/airbyte-integrations/connectors/destination-pinecone/examples/configured_catalog.json b/airbyte-integrations/connectors/destination-pinecone/examples/configured_catalog.json new file mode 100644 index 000000000000..fab8a309fdf6 --- /dev/null +++ b/airbyte-integrations/connectors/destination-pinecone/examples/configured_catalog.json @@ -0,0 +1,19 @@ +{ + "streams": [ + { + "stream": { + "name": "example_stream", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {} + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["column_name"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/destination-pinecone/examples/messages.jsonl b/airbyte-integrations/connectors/destination-pinecone/examples/messages.jsonl new file mode 100644 index 000000000000..80e15fb36948 --- /dev/null +++ b/airbyte-integrations/connectors/destination-pinecone/examples/messages.jsonl @@ -0,0 +1,2 @@ +{"type": "RECORD", "record": {"stream": "example_stream", "data": { "title": "value1", "field2": "value2" }, "emitted_at": 1625383200000}} +{"type": "RECORD", "record": {"stream": "example_stream", "data": { "title": "value2", "field2": "value2" }, "emitted_at": 1625383200000}} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-pinecone/icon.svg b/airbyte-integrations/connectors/destination-pinecone/icon.svg new file mode 100644 index 000000000000..14fcceb3e751 --- /dev/null +++ b/airbyte-integrations/connectors/destination-pinecone/icon.svg @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + diff --git a/airbyte-integrations/connectors/destination-pinecone/integration_tests/invalid_config.json b/airbyte-integrations/connectors/destination-pinecone/integration_tests/invalid_config.json new file mode 100644 index 000000000000..5e950be2eb2f --- /dev/null +++ b/airbyte-integrations/connectors/destination-pinecone/integration_tests/invalid_config.json @@ -0,0 +1,16 @@ +{ + "processing": { + "text_fields": ["str_col"], + "metadata_fields": ["int_col"], + "chunk_size": 1000 + }, + "embedding": { + "mode": "openai", + "openai_key": "mykey" + }, + "indexing": { + "pinecone_key": "mykey", + "index": "myindex", + "pinecone_environment": "us-east-1-aws" + } +} diff --git a/airbyte-integrations/connectors/destination-pinecone/integration_tests/pinecone_integration_test.py b/airbyte-integrations/connectors/destination-pinecone/integration_tests/pinecone_integration_test.py new file mode 100644 index 000000000000..b70232356dd8 --- /dev/null +++ b/airbyte-integrations/connectors/destination-pinecone/integration_tests/pinecone_integration_test.py @@ -0,0 +1,78 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import logging + +import pinecone +from airbyte_cdk.destinations.vector_db_based.embedder import OPEN_AI_VECTOR_SIZE +from airbyte_cdk.destinations.vector_db_based.test_utils import BaseIntegrationTest +from airbyte_cdk.models import DestinationSyncMode, Status +from destination_pinecone.destination import DestinationPinecone +from langchain.embeddings import OpenAIEmbeddings +from langchain.vectorstores import Pinecone + + +class PineconeIntegrationTest(BaseIntegrationTest): + def _init_pinecone(self): + pinecone.init(api_key=self.config["indexing"]["pinecone_key"], environment=self.config["indexing"]["pinecone_environment"]) + self.pinecone_index = pinecone.Index(self.config["indexing"]["index"]) + + def setUp(self): + with open("secrets/config.json", "r") as f: + self.config = json.loads(f.read()) + self._init_pinecone() + + def tearDown(self): + # make sure pinecone is initialized correctly before cleaning up + self._init_pinecone() + self.pinecone_index.delete(delete_all=True) + + def test_check_valid_config(self): + outcome = DestinationPinecone().check(logging.getLogger("airbyte"), self.config) + assert outcome.status == Status.SUCCEEDED + + def test_check_invalid_config(self): + outcome = DestinationPinecone().check( + logging.getLogger("airbyte"), + { + "processing": {"text_fields": ["str_col"], "chunk_size": 1000, "metadata_fields": ["int_col"]}, + "embedding": {"mode": "openai", "openai_key": "mykey"}, + "indexing": { + "mode": "pinecone", + "pinecone_key": "mykey", + "index": "testdata", + "pinecone_environment": "asia-southeast1-gcp-free", + }, + }, + ) + assert outcome.status == Status.FAILED + + def test_write(self): + catalog = self._get_configured_catalog(DestinationSyncMode.overwrite) + first_state_message = self._state({"state": "1"}) + first_record_chunk = [self._record("mystream", f"Dogs are number {i}", i) for i in range(5)] + + # initial sync + destination = DestinationPinecone() + list(destination.write(self.config, catalog, [*first_record_chunk, first_state_message])) + assert self.pinecone_index.describe_index_stats().total_vector_count == 5 + + # incrementalally update a doc + incremental_catalog = self._get_configured_catalog(DestinationSyncMode.append_dedup) + list(destination.write(self.config, incremental_catalog, [self._record("mystream", "Cats are nice", 2), first_state_message])) + result = self.pinecone_index.query( + vector=[0] * OPEN_AI_VECTOR_SIZE, top_k=10, filter={"_ab_record_id": "mystream_2"}, include_metadata=True + ) + assert len(result.matches) == 1 + assert ( + result.matches[0].metadata["text"] == "str_col: Cats are nice" + ), 'Ensure that "str_col" is included in the "text_fields" array under the "processing" section of /secrets/config.json.' + + # test langchain integration + embeddings = OpenAIEmbeddings(openai_api_key=self.config["embedding"]["openai_key"]) + self._init_pinecone() + vector_store = Pinecone(self.pinecone_index, embeddings.embed_query, "text") + result = vector_store.similarity_search("feline animals", 1) + assert result[0].metadata["_ab_record_id"] == "mystream_2" diff --git a/airbyte-integrations/connectors/destination-pinecone/integration_tests/sample_config.json b/airbyte-integrations/connectors/destination-pinecone/integration_tests/sample_config.json new file mode 100644 index 000000000000..8e8901ca5aac --- /dev/null +++ b/airbyte-integrations/connectors/destination-pinecone/integration_tests/sample_config.json @@ -0,0 +1,16 @@ +{ + "indexing": { + "pinecone_key": "mykey", + "pinecone_environment": "myenv", + "index": "myindex" + }, + "embedding": { + "mode": "openai", + "openai_key": "mykey" + }, + "processing": { + "chunk_size": 1000, + "text_fields": ["str_col"], + "metadata_fields": ["int_col"] + } +} diff --git a/airbyte-integrations/connectors/destination-pinecone/integration_tests/spec.json b/airbyte-integrations/connectors/destination-pinecone/integration_tests/spec.json new file mode 100644 index 000000000000..ce7176ff0467 --- /dev/null +++ b/airbyte-integrations/connectors/destination-pinecone/integration_tests/spec.json @@ -0,0 +1,346 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/destinations/pinecone", + "connectionSpecification": { + "title": "Destination Config", + "description": "The configuration model for the Vector DB based destinations. This model is used to generate the UI for the destination configuration,\nas well as to provide type safety for the configuration passed to the destination.\n\nThe configuration model is composed of four parts:\n* Processing configuration\n* Embedding configuration\n* Indexing configuration\n* Advanced configuration\n\nProcessing, embedding and advanced configuration are provided by this base class, while the indexing configuration is provided by the destination connector in the sub class.", + "type": "object", + "properties": { + "indexing": { + "title": "Indexing", + "type": "object", + "properties": { + "pinecone_key": { + "title": "Pinecone API key", + "description": "The Pinecone API key to use matching the environment (copy from Pinecone console)", + "airbyte_secret": true, + "type": "string" + }, + "pinecone_environment": { + "title": "Pinecone Environment", + "description": "Pinecone Cloud environment to use", + "examples": ["us-west1-gcp", "gcp-starter"], + "type": "string" + }, + "index": { + "title": "Index", + "description": "Pinecone index in your project to load data into", + "type": "string" + } + }, + "required": ["pinecone_key", "pinecone_environment", "index"], + "description": "Pinecone is a popular vector store that can be used to store and retrieve embeddings.", + "group": "indexing" + }, + "embedding": { + "title": "Embedding", + "description": "Embedding configuration", + "group": "embedding", + "type": "object", + "oneOf": [ + { + "title": "OpenAI", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "openai", + "const": "openai", + "enum": ["openai"], + "type": "string" + }, + "openai_key": { + "title": "OpenAI API key", + "airbyte_secret": true, + "type": "string" + } + }, + "required": ["openai_key", "mode"], + "description": "Use the OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." + }, + { + "title": "Cohere", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "cohere", + "const": "cohere", + "enum": ["cohere"], + "type": "string" + }, + "cohere_key": { + "title": "Cohere API key", + "airbyte_secret": true, + "type": "string" + } + }, + "required": ["cohere_key", "mode"], + "description": "Use the Cohere API to embed text." + }, + { + "title": "Fake", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "fake", + "const": "fake", + "enum": ["fake"], + "type": "string" + } + }, + "required": ["mode"], + "description": "Use a fake embedding made out of random vectors with 1536 embedding dimensions. This is useful for testing the data pipeline without incurring any costs." + }, + { + "title": "Azure OpenAI", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "azure_openai", + "const": "azure_openai", + "enum": ["azure_openai"], + "type": "string" + }, + "openai_key": { + "title": "Azure OpenAI API key", + "description": "The API key for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", + "airbyte_secret": true, + "type": "string" + }, + "api_base": { + "title": "Resource base URL", + "description": "The base URL for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", + "examples": ["https://your-resource-name.openai.azure.com"], + "type": "string" + }, + "deployment": { + "title": "Deployment", + "description": "The deployment for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", + "examples": ["your-resource-name"], + "type": "string" + } + }, + "required": ["openai_key", "api_base", "deployment", "mode"], + "description": "Use the Azure-hosted OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." + }, + { + "title": "OpenAI-compatible", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "openai_compatible", + "const": "openai_compatible", + "enum": ["openai_compatible"], + "type": "string" + }, + "api_key": { + "title": "API key", + "default": "", + "airbyte_secret": true, + "type": "string" + }, + "base_url": { + "title": "Base URL", + "description": "The base URL for your OpenAI-compatible service", + "examples": ["https://your-service-name.com"], + "type": "string" + }, + "model_name": { + "title": "Model name", + "description": "The name of the model to use for embedding", + "default": "text-embedding-ada-002", + "examples": ["text-embedding-ada-002"], + "type": "string" + }, + "dimensions": { + "title": "Embedding dimensions", + "description": "The number of dimensions the embedding model is generating", + "examples": [1536, 384], + "type": "integer" + } + }, + "required": ["base_url", "dimensions", "mode"], + "description": "Use a service that's compatible with the OpenAI API to embed text." + } + ] + }, + "processing": { + "title": "ProcessingConfigModel", + "type": "object", + "properties": { + "chunk_size": { + "title": "Chunk size", + "description": "Size of chunks in tokens to store in vector store (make sure it is not too big for the context if your LLM)", + "minimum": 1, + "maximum": 8191, + "type": "integer" + }, + "chunk_overlap": { + "title": "Chunk overlap", + "description": "Size of overlap between chunks in tokens to store in vector store to better capture relevant context", + "default": 0, + "type": "integer" + }, + "text_fields": { + "title": "Text fields to embed", + "description": "List of fields in the record that should be used to calculate the embedding. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered text fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.", + "default": [], + "always_show": true, + "examples": ["text", "user.name", "users.*.name"], + "type": "array", + "items": { "type": "string" } + }, + "metadata_fields": { + "title": "Fields to store as metadata", + "description": "List of fields in the record that should be stored as metadata. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered metadata fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. When specifying nested paths, all matching values are flattened into an array set to a field named by the path.", + "default": [], + "always_show": true, + "examples": ["age", "user", "user.name"], + "type": "array", + "items": { "type": "string" } + }, + "field_name_mappings": { + "title": "Field name mappings", + "description": "List of fields to rename. Not applicable for nested fields, but can be used to rename fields already flattened via dot notation.", + "default": [], + "type": "array", + "items": { + "title": "FieldNameMappingConfigModel", + "type": "object", + "properties": { + "from_field": { + "title": "From field name", + "description": "The field name in the source", + "type": "string" + }, + "to_field": { + "title": "To field name", + "description": "The field name to use in the destination", + "type": "string" + } + }, + "required": ["from_field", "to_field"] + } + }, + "text_splitter": { + "title": "Text splitter", + "description": "Split text fields into chunks based on the specified method.", + "type": "object", + "oneOf": [ + { + "title": "By Separator", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "separator", + "const": "separator", + "enum": ["separator"], + "type": "string" + }, + "separators": { + "title": "Separators", + "description": "List of separator strings to split text fields by. The separator itself needs to be wrapped in double quotes, e.g. to split by the dot character, use \".\". To split by a newline, use \"\\n\".", + "default": ["\"\\n\\n\"", "\"\\n\"", "\" \"", "\"\""], + "type": "array", + "items": { "type": "string" } + }, + "keep_separator": { + "title": "Keep separator", + "description": "Whether to keep the separator in the resulting chunks", + "default": false, + "type": "boolean" + } + }, + "required": ["mode"], + "description": "Split the text by the list of separators until the chunk size is reached, using the earlier mentioned separators where possible. This is useful for splitting text fields by paragraphs, sentences, words, etc." + }, + { + "title": "By Markdown header", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "markdown", + "const": "markdown", + "enum": ["markdown"], + "type": "string" + }, + "split_level": { + "title": "Split level", + "description": "Level of markdown headers to split text fields by. Headings down to the specified level will be used as split points", + "default": 1, + "minimum": 1, + "maximum": 6, + "type": "integer" + } + }, + "required": ["mode"], + "description": "Split the text by Markdown headers down to the specified header level. If the chunk size fits multiple sections, they will be combined into a single chunk." + }, + { + "title": "By Programming Language", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "code", + "const": "code", + "enum": ["code"], + "type": "string" + }, + "language": { + "title": "Language", + "description": "Split code in suitable places based on the programming language", + "enum": [ + "cpp", + "go", + "java", + "js", + "php", + "proto", + "python", + "rst", + "ruby", + "rust", + "scala", + "swift", + "markdown", + "latex", + "html", + "sol" + ], + "type": "string" + } + }, + "required": ["language", "mode"], + "description": "Split the text by suitable delimiters based on the programming language. This is useful for splitting code into chunks." + } + ] + } + }, + "required": ["chunk_size"], + "group": "processing" + }, + "omit_raw_text": { + "title": "Do not store raw text", + "description": "Do not store the text that gets embedded along with the vector and the metadata in the destination. If set to true, only the vector and the metadata will be stored - in this case raw text for LLM use cases needs to be retrieved from another source.", + "default": false, + "group": "advanced", + "type": "boolean" + } + }, + "required": ["embedding", "processing", "indexing"], + "groups": [ + { "id": "processing", "title": "Processing" }, + { "id": "embedding", "title": "Embedding" }, + { "id": "indexing", "title": "Indexing" }, + { "id": "advanced", "title": "Advanced" } + ] + }, + "supportsIncremental": true, + "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"] +} diff --git a/airbyte-integrations/connectors/destination-pinecone/main.py b/airbyte-integrations/connectors/destination-pinecone/main.py new file mode 100644 index 000000000000..9a12601a2cfa --- /dev/null +++ b/airbyte-integrations/connectors/destination-pinecone/main.py @@ -0,0 +1,11 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from destination_pinecone import DestinationPinecone + +if __name__ == "__main__": + DestinationPinecone().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-pinecone/metadata.yaml b/airbyte-integrations/connectors/destination-pinecone/metadata.yaml new file mode 100644 index 000000000000..a743a3609f40 --- /dev/null +++ b/airbyte-integrations/connectors/destination-pinecone/metadata.yaml @@ -0,0 +1,39 @@ +data: + ab_internal: + ql: 300 + sl: 200 + allowedHosts: + hosts: + - "*.${indexing.pinecone_environment}.pinecone.io" + - api.openai.com + - api.cohere.ai + - ${embedding.api_base} + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + connectorSubtype: vectorstore + connectorType: destination + definitionId: 3d2b6f84-7f0d-4e3f-a5e5-7c7d4b50eabd + dockerImageTag: 0.0.22 + dockerRepository: airbyte/destination-pinecone + documentationUrl: https://docs.airbyte.com/integrations/destinations/pinecone + githubIssueLabel: destination-pinecone + icon: pinecone.svg + license: MIT + name: Pinecone + registries: + cloud: + enabled: true + oss: + enabled: true + releaseDate: 2023-08-15 + releaseStage: beta + resourceRequirements: + jobSpecific: + - jobType: sync + resourceRequirements: + memory_limit: 2Gi + memory_request: 2Gi + supportLevel: certified + tags: + - language:python +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-pinecone/requirements.txt b/airbyte-integrations/connectors/destination-pinecone/requirements.txt new file mode 100644 index 000000000000..d6e1198b1ab1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-pinecone/requirements.txt @@ -0,0 +1 @@ +-e . diff --git a/airbyte-integrations/connectors/destination-pinecone/setup.py b/airbyte-integrations/connectors/destination-pinecone/setup.py new file mode 100644 index 000000000000..3e1fbd33d1dc --- /dev/null +++ b/airbyte-integrations/connectors/destination-pinecone/setup.py @@ -0,0 +1,26 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk[vector-db-based]==0.57.0", + "pinecone-client[grpc]", +] + +TEST_REQUIREMENTS = ["pytest~=6.2"] + +setup( + name="destination_pinecone", + description="Destination implementation for Pinecone.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/destination-pinecone/test_pinecone.py b/airbyte-integrations/connectors/destination-pinecone/test_pinecone.py new file mode 100644 index 000000000000..3145b0a66e8c --- /dev/null +++ b/airbyte-integrations/connectors/destination-pinecone/test_pinecone.py @@ -0,0 +1,43 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import os + +import pinecone +from langchain.chains import RetrievalQA +from langchain.embeddings import OpenAIEmbeddings +from langchain.llms import OpenAI +from langchain.vectorstores import Pinecone + +# Run with OPENAI_API_KEY, PINECONE_KEY and PINECONE_ENV set in the environment + +embeddings = OpenAIEmbeddings() +pinecone.init(api_key=os.environ["PINECONE_KEY"], environment=os.environ["PINECONE_ENV"]) +index = pinecone.Index("airbyte") +vector_store = Pinecone(index, embeddings.embed_query, "text") + + +# Playing with a Github issue search use case + +# prompt_template = """You are a question-answering bot operating on Github issues. Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer. In the end, state the issue number you based your answer on. + +# {context} + +# Question: {question} +# Helpful Answer:""" +# prompt = PromptTemplate( +# template=prompt_template, input_variables=["context", "question"] +# ) +# document_prompt = PromptTemplate(input_variables=["page_content", "number"], template="{page_content}, issue number: {number}") +# qa = RetrievalQA.from_chain_type(llm=OpenAI(temperature=0), chain_type="stuff", retriever=vector_store.as_retriever(), chain_type_kwargs={"prompt": prompt, "document_prompt": document_prompt}) + +qa = RetrievalQA.from_chain_type(llm=OpenAI(temperature=0), chain_type="stuff", retriever=vector_store.as_retriever()) + +print("Chat Langchain Demo") +print("Ask a question to begin:") +while True: + query = input("") + answer = qa.run(query) + print(answer) + print("\nWhat else can I help you with:") diff --git a/airbyte-integrations/connectors/destination-pinecone/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-pinecone/unit_tests/destination_test.py new file mode 100644 index 000000000000..884a69efabd1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-pinecone/unit_tests/destination_test.py @@ -0,0 +1,96 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import unittest +from unittest.mock import MagicMock, Mock, patch + +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.models import ConnectorSpecification, Status +from destination_pinecone.config import ConfigModel +from destination_pinecone.destination import DestinationPinecone + + +class TestDestinationPinecone(unittest.TestCase): + def setUp(self): + self.config = { + "processing": {"text_fields": ["str_col"], "metadata_fields": [], "chunk_size": 1000}, + "embedding": {"mode": "openai", "openai_key": "mykey"}, + "indexing": { + "pinecone_key": "mykey", + "pinecone_environment": "myenv", + "index": "myindex", + }, + } + self.config_model = ConfigModel.parse_obj(self.config) + self.logger = AirbyteLogger() + + @patch("destination_pinecone.destination.PineconeIndexer") + @patch("destination_pinecone.destination.create_from_config") + def test_check(self, MockedEmbedder, MockedPineconeIndexer): + mock_embedder = Mock() + mock_indexer = Mock() + MockedEmbedder.return_value = mock_embedder + MockedPineconeIndexer.return_value = mock_indexer + + mock_embedder.check.return_value = None + mock_indexer.check.return_value = None + + destination = DestinationPinecone() + result = destination.check(self.logger, self.config) + + self.assertEqual(result.status, Status.SUCCEEDED) + mock_embedder.check.assert_called_once() + mock_indexer.check.assert_called_once() + + @patch("destination_pinecone.destination.PineconeIndexer") + @patch("destination_pinecone.destination.create_from_config") + def test_check_with_errors(self, MockedEmbedder, MockedPineconeIndexer): + mock_embedder = Mock() + mock_indexer = Mock() + MockedEmbedder.return_value = mock_embedder + MockedPineconeIndexer.return_value = mock_indexer + + embedder_error_message = "Embedder Error" + indexer_error_message = "Indexer Error" + + mock_embedder.check.return_value = embedder_error_message + mock_indexer.check.return_value = indexer_error_message + + destination = DestinationPinecone() + result = destination.check(self.logger, self.config) + + self.assertEqual(result.status, Status.FAILED) + self.assertEqual(result.message, f"{embedder_error_message}\n{indexer_error_message}") + + mock_embedder.check.assert_called_once() + mock_indexer.check.assert_called_once() + + @patch("destination_pinecone.destination.Writer") + @patch("destination_pinecone.destination.PineconeIndexer") + @patch("destination_pinecone.destination.create_from_config") + def test_write(self, MockedEmbedder, MockedPineconeIndexer, MockedWriter): + mock_embedder = Mock() + mock_indexer = Mock() + MockedEmbedder.return_value = mock_embedder + mock_writer = Mock() + + MockedPineconeIndexer.return_value = mock_indexer + MockedWriter.return_value = mock_writer + + mock_writer.write.return_value = [] + + configured_catalog = MagicMock() + input_messages = [] + + destination = DestinationPinecone() + list(destination.write(self.config, configured_catalog, input_messages)) + + MockedWriter.assert_called_once_with(self.config_model.processing, mock_indexer, mock_embedder, batch_size=32, omit_raw_text=False) + mock_writer.write.assert_called_once_with(configured_catalog, input_messages) + + def test_spec(self): + destination = DestinationPinecone() + result = destination.spec() + + self.assertIsInstance(result, ConnectorSpecification) diff --git a/airbyte-integrations/connectors/destination-pinecone/unit_tests/pinecone_indexer_test.py b/airbyte-integrations/connectors/destination-pinecone/unit_tests/pinecone_indexer_test.py new file mode 100644 index 000000000000..ef21404a5bb0 --- /dev/null +++ b/airbyte-integrations/connectors/destination-pinecone/unit_tests/pinecone_indexer_test.py @@ -0,0 +1,266 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import ANY, MagicMock, Mock, call, patch + +import pytest +import urllib3 +from airbyte_cdk.models import ConfiguredAirbyteCatalog +from destination_pinecone.config import PineconeIndexingModel +from destination_pinecone.indexer import PineconeIndexer +from pinecone import IndexDescription, exceptions + + +def create_pinecone_indexer(): + config = PineconeIndexingModel(mode="pinecone", pinecone_environment="myenv", pinecone_key="mykey", index="myindex") + indexer = PineconeIndexer(config, 3) + + indexer.pinecone_index.delete = MagicMock() + indexer.pinecone_index.upsert = MagicMock() + indexer.pinecone_index.query = MagicMock() + return indexer + + +def create_index_description(dimensions=3, pod_type="p1"): + return IndexDescription( + name="", + metric="", + replicas=1, + dimension=dimensions, + shards=1, + pods=1, + pod_type=pod_type, + status=None, + metadata_config=None, + source_collection=None, + ) + + +@pytest.fixture(scope="module", autouse=True) +def mock_describe_index(): + with patch("pinecone.describe_index") as mock: + mock.return_value = create_index_description() + yield mock + + +def test_pinecone_index_upsert_and_delete(mock_describe_index): + indexer = create_pinecone_indexer() + indexer._pod_type = "p1" + indexer.index( + [ + Mock(page_content="test", metadata={"_ab_stream": "abc"}, embedding=[1, 2, 3]), + Mock(page_content="test2", metadata={"_ab_stream": "abc"}, embedding=[4, 5, 6]), + ], + "ns1", + "some_stream", + ) + indexer.delete(["delete_id1", "delete_id2"], "ns1", "some_stram") + indexer.pinecone_index.delete.assert_called_with(filter={"_ab_record_id": {"$in": ["delete_id1", "delete_id2"]}}, namespace="ns1") + indexer.pinecone_index.upsert.assert_called_with( + vectors=( + (ANY, [1, 2, 3], {"_ab_stream": "abc", "text": "test"}), + (ANY, [4, 5, 6], {"_ab_stream": "abc", "text": "test2"}), + ), + async_req=True, + show_progress=False, + namespace="ns1", + ) + + +def test_pinecone_index_upsert_and_delete_starter(mock_describe_index): + indexer = create_pinecone_indexer() + indexer._pod_type = "starter" + indexer.pinecone_index.query.side_effect = [ + MagicMock(matches=[MagicMock(id="doc_id1"), MagicMock(id="doc_id2")]), + MagicMock(matches=[MagicMock(id="doc_id3")]), + MagicMock(matches=[]), + ] + indexer.index( + [ + Mock(page_content="test", metadata={"_ab_stream": "abc"}, embedding=[1, 2, 3]), + Mock(page_content="test2", metadata={"_ab_stream": "abc"}, embedding=[4, 5, 6]), + ], + "ns1", + "some_stream", + ) + indexer.delete(["delete_id1", "delete_id2"], "ns1", "some_stram") + indexer.pinecone_index.query.assert_called_with( + vector=[0, 0, 0], filter={"_ab_record_id": {"$in": ["delete_id1", "delete_id2"]}}, top_k=10_000, namespace="ns1" + ) + indexer.pinecone_index.delete.assert_has_calls( + [call(ids=["doc_id1", "doc_id2"], namespace="ns1"), call(ids=["doc_id3"], namespace="ns1")] + ) + indexer.pinecone_index.upsert.assert_called_with( + vectors=( + (ANY, [1, 2, 3], {"_ab_stream": "abc", "text": "test"}), + (ANY, [4, 5, 6], {"_ab_stream": "abc", "text": "test2"}), + ), + async_req=True, + show_progress=False, + namespace="ns1", + ) + + +def test_pinecone_index_delete_1k_limit(mock_describe_index): + indexer = create_pinecone_indexer() + indexer._pod_type = "starter" + indexer.pinecone_index.query.side_effect = [ + MagicMock(matches=[MagicMock(id=f"doc_id_{str(i)}") for i in range(1300)]), + MagicMock(matches=[]), + ] + indexer.delete(["delete_id1"], "ns1", "some_stream") + indexer.pinecone_index.delete.assert_has_calls( + [ + call(ids=[f"doc_id_{str(i)}" for i in range(1000)], namespace="ns1"), + call(ids=[f"doc_id_{str(i+1000)}" for i in range(300)], namespace="ns1"), + ] + ) + + +def test_pinecone_index_empty_batch(): + indexer = create_pinecone_indexer() + indexer.index([], "ns1", "some_stream") + indexer.pinecone_index.delete.assert_not_called() + indexer.pinecone_index.upsert.assert_not_called() + + +def test_pinecone_index_upsert_batching(): + indexer = create_pinecone_indexer() + indexer.index( + [Mock(page_content=f"test {i}", metadata={"_ab_stream": "abc"}, embedding=[i, i, i]) for i in range(50)], + "ns1", + "some_stream", + ) + assert indexer.pinecone_index.upsert.call_count == 2 + for i in range(40): + assert indexer.pinecone_index.upsert.call_args_list[0].kwargs["vectors"][i] == ( + ANY, + [i, i, i], + {"_ab_stream": "abc", "text": f"test {i}"}, + ) + for i in range(40, 50): + assert indexer.pinecone_index.upsert.call_args_list[1].kwargs["vectors"][i - 40] == ( + ANY, + [i, i, i], + {"_ab_stream": "abc", "text": f"test {i}"}, + ) + + +def generate_catalog(): + return ConfiguredAirbyteCatalog.parse_obj( + { + "streams": [ + { + "stream": { + "name": "example_stream", + "json_schema": {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": {}}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": False, + "default_cursor_field": ["column_name"], + "namespace": "ns1", + }, + "primary_key": [["id"]], + "sync_mode": "incremental", + "destination_sync_mode": "append_dedup", + }, + { + "stream": { + "name": "example_stream2", + "json_schema": {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": {}}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": False, + "default_cursor_field": ["column_name"], + "namespace": "ns2", + }, + "primary_key": [["id"]], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + }, + ] + } + ) + + +def test_pinecone_pre_sync(mock_describe_index): + indexer = create_pinecone_indexer() + indexer.pre_sync(generate_catalog()) + indexer.pinecone_index.delete.assert_called_with(filter={"_ab_stream": "ns2_example_stream2"}, namespace="ns2") + + +def test_pinecone_pre_sync_starter(mock_describe_index): + mock_describe_index.return_value = create_index_description(pod_type="starter") + indexer = create_pinecone_indexer() + indexer.pinecone_index.query.side_effect = [ + MagicMock(matches=[MagicMock(id="doc_id1"), MagicMock(id="doc_id2")]), + MagicMock(matches=[]), + ] + indexer.pre_sync(generate_catalog()) + indexer.pinecone_index.query.assert_called_with( + vector=[0, 0, 0], filter={"_ab_stream": "ns2_example_stream2"}, top_k=10_000, namespace="ns2" + ) + indexer.pinecone_index.delete.assert_called_with(ids=["doc_id1", "doc_id2"], namespace="ns2") + + +@pytest.mark.parametrize( + "index_list, describe_throws,reported_dimensions,check_succeeds, error_message", + [ + (["myindex"], None, 3, True, None), + (["other_index"], None, 3, False, "Index myindex does not exist in environment"), + ( + ["myindex"], + urllib3.exceptions.MaxRetryError(None, "", reason=Exception("Failed to resolve 'controller.myenv.pinecone.io'")), + 3, + False, + "Failed to resolve environment", + ), + (["myindex"], exceptions.UnauthorizedException(http_resp=urllib3.HTTPResponse(body="No entry!")), 3, False, "No entry!"), + (["myindex"], None, 4, False, "Make sure embedding and indexing configurations match."), + (["myindex"], Exception("describe failed"), 3, False, "describe failed"), + (["myindex"], Exception("describe failed"), 4, False, "describe failed"), + ], +) +@patch("pinecone.describe_index") +@patch("pinecone.list_indexes") +def test_pinecone_check(list_mock, describe_mock, index_list, describe_throws, reported_dimensions, check_succeeds, error_message): + indexer = create_pinecone_indexer() + indexer.embedding_dimensions = 3 + if describe_throws: + describe_mock.side_effect = describe_throws + else: + describe_mock.return_value = create_index_description(dimensions=reported_dimensions) + list_mock.return_value = index_list + result = indexer.check() + if check_succeeds: + assert result is None + else: + assert error_message in result + + +def test_metadata_normalization(): + indexer = create_pinecone_indexer() + + indexer._pod_type = "p1" + indexer.index( + [ + Mock( + page_content="test", + embedding=[1, 2, 3], + metadata={ + "_ab_stream": "abc", + "id": 1, + "a_complex_field": {"a_nested_field": "a_nested_value"}, + "too_big": "a" * 40_000, + "small": "a", + }, + ), + ], + None, + "some_stream", + ) + indexer.pinecone_index.upsert.assert_called_with( + vectors=((ANY, [1, 2, 3], {"_ab_stream": "abc", "text": "test", "small": "a", "id": 1}),), + async_req=True, + show_progress=False, + namespace=None, + ) diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/.dockerignore b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile deleted file mode 100644 index 990198346407..000000000000 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-postgres-strict-encrypt - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-postgres-strict-encrypt - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.4.0 -LABEL io.airbyte.name=airbyte/destination-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle index 006fb163a269..8a0ce598b05f 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/build.gradle @@ -1,28 +1,48 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.13.1' + features = [ + 'db-sources', // required for tests + 'db-destinations' + ] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.postgres.PostgresDestinationStrictEncrypt' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:bases-destination-jdbc') implementation project(':airbyte-integrations:connectors:destination-postgres') + // TODO: declare typing-deduping as a CDK feature instead of importing from source. + implementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') + testImplementation libs.testcontainers.jdbc + testImplementation libs.testcontainers.postgresql - integrationTestJavaImplementation libs.connectors.testcontainers.postgresql + testFixturesImplementation libs.testcontainers.jdbc + testFixturesImplementation libs.testcontainers.postgresql - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) - integrationTestJavaImplementation project(':airbyte-config-oss:config-models-oss') - integrationTestJavaImplementation project(':airbyte-connector-test-harnesses:acceptance-test-harness') + integrationTestJavaImplementation testFixtures(project(':airbyte-integrations:connectors:destination-postgres')) + integrationTestJavaImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:typing-deduping')) +} +configurations.all { + resolutionStrategy { + force libs.jooq + } } diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/gradle.properties b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/gradle.properties index 5d1adb3b55c1..23da4989675e 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/gradle.properties +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/gradle.properties @@ -1,3 +1,3 @@ -# currently limit the number of parallel threads until further investigation into the issues \ -# where integration tests run into race conditions -numberThreads=1 +# our testcontainer has issues with too much concurrency. +# 4 threads seems to be the sweet spot. +testExecutionConcurrency=4 diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml index b24719b753e4..573dbcf99fd9 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/metadata.yaml @@ -1,14 +1,10 @@ data: - registries: - cloud: - enabled: false # strict encrypt connectors are deployed to Cloud by their non strict encrypt sibling. - oss: - enabled: false # strict encrypt connectors are not used on OSS. connectorSubtype: database connectorType: destination definitionId: 25c5221d-dce2-4163-ade9-739ef790f503 - dockerImageTag: 0.4.0 + dockerImageTag: 0.5.5 dockerRepository: airbyte/destination-postgres-strict-encrypt + documentationUrl: https://docs.airbyte.com/integrations/destinations/postgres githubIssueLabel: destination-postgres icon: postgresql.svg license: ELv2 @@ -17,8 +13,14 @@ data: normalizationIntegrationType: postgres normalizationRepository: airbyte/normalization normalizationTag: 0.4.1 + registries: + cloud: + dockerImageTag: 0.4.0 + enabled: false + oss: + dockerImageTag: 0.4.0 + enabled: false releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/destinations/postgres supportsDbt: true tags: - language:java diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncrypt.java b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncrypt.java index 2e01ab4b34f4..28aa74c8861f 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncrypt.java +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncrypt.java @@ -6,11 +6,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.spec_modification.SpecModifyingDestination; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.spec_modification.SpecModifyingDestination; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.ConnectorSpecification; diff --git a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.java index 37964dab269c..128f8903f0a7 100644 --- a/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationStrictEncryptAcceptanceTest.java @@ -4,43 +4,23 @@ package io.airbyte.integrations.destination.postgres; -import static io.airbyte.db.PostgresUtils.getCertificate; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel.TunnelMethod; import io.airbyte.configoss.StandardCheckConnectionOutput.Status; -import io.airbyte.db.Database; -import io.airbyte.db.PostgresUtils; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import java.sql.SQLException; -import java.util.ArrayList; +import io.airbyte.integrations.destination.postgres.PostgresTestDatabase.BaseImage; import java.util.HashSet; -import java.util.List; -import java.util.stream.Collectors; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.utility.DockerImageName; -// todo (cgardens) - DRY this up with PostgresDestinationAcceptanceTest -public class PostgresDestinationStrictEncryptAcceptanceTest extends DestinationAcceptanceTest { +public class PostgresDestinationStrictEncryptAcceptanceTest extends AbstractPostgresDestinationAcceptanceTest { - private PostgreSQLContainer db; - private final StandardNameTransformer namingResolver = new StandardNameTransformer(); + private PostgresTestDatabase testDb; protected static final String PASSWORD = "Passw0rd"; - protected static PostgresUtils.Certificate certs; - private static final String NORMALIZATION_VERSION = "dev"; // this is hacky. This test should extend or encapsulate - // PostgresDestinationAcceptanceTest @Override protected String getImageName() { @@ -49,128 +29,45 @@ protected String getImageName() { @Override protected JsonNode getConfig() { - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) - .put(JdbcUtils.SCHEMA_KEY, "public") - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.SSL_MODE_KEY, ImmutableMap.builder() - .put("mode", "verify-full") - .put("ca_certificate", certs.getCaCertificate()) - .put("client_certificate", certs.getClientCertificate()) - .put("client_key", certs.getClientKey()) - .put("client_key_password", PASSWORD) + return testDb.configBuilder() + .with("schema", "public") + .withDatabase() + .withResolvedHostAndPort() + .withCredentials() + .withSsl(ImmutableMap.builder() + .put("mode", "verify-ca") // verify-full will not work since the spawned container is only allowed for 127.0.0.1/32 CIDRs + .put("ca_certificate", testDb.getCertificates().caCertificate()) .build()) - .build()); + .build(); } @Override - protected JsonNode getFailCheckConfig() { - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, "wrong password") - .put(JdbcUtils.SCHEMA_KEY, "public") - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.SSL_KEY, false) - .build()); - } - - @Override - protected List retrieveRecords(final TestDestinationEnv env, - final String streamName, - final String namespace, - final JsonNode streamSchema) - throws Exception { - return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) - .stream() - .map(r -> Jsons.deserialize(r.get(JavaBaseConstants.COLUMN_NAME_DATA).asText())) - .collect(Collectors.toList()); - } - - @Override - protected boolean implementsNamespaces() { - return true; - } - - @Override - protected List retrieveNormalizedRecords(final TestDestinationEnv env, final String streamName, final String namespace) - throws Exception { - final String tableName = namingResolver.getIdentifier(streamName); - // Temporarily disabling the behavior of the StandardNameTransformer, see (issue #1785) so we don't - // use quoted names - // if (!tableName.startsWith("\"")) { - // // Currently, Normalization always quote tables identifiers - // //tableName = "\"" + tableName + "\""; - // } - return retrieveRecordsFromTable(tableName, namespace); - } - - @Override - protected List resolveIdentifier(final String identifier) { - final List result = new ArrayList<>(); - final String resolved = namingResolver.getIdentifier(identifier); - result.add(identifier); - result.add(resolved); - if (!resolved.startsWith("\"")) { - result.add(resolved.toLowerCase()); - result.add(resolved.toUpperCase()); - } - return result; - } - - private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { - try (final DSLContext dslContext = DSLContextFactory.create( - db.getUsername(), - db.getPassword(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - db.getJdbcUrl(), - SQLDialect.POSTGRES)) { - return new Database(dslContext) - .query( - ctx -> ctx - .fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) - .stream() - .map(r -> r.formatJSON(JdbcUtils.getDefaultJSONFormat())) - .map(Jsons::deserialize) - .collect(Collectors.toList())); - } + protected PostgresTestDatabase getTestDb() { + return testDb; } @Override protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws Exception { - db = new PostgreSQLContainer<>(DockerImageName.parse("postgres:bullseye") - .asCompatibleSubstituteFor("postgres")); - db.start(); - certs = getCertificate(db); + testDb = PostgresTestDatabase.in(BaseImage.POSTGRES_12, PostgresTestDatabase.ContainerModifier.CERT); } @Override protected void tearDown(final TestDestinationEnv testEnv) { - db.stop(); - db.close(); + testDb.close(); } @Test void testStrictSSLUnsecuredNoTunnel() throws Exception { - final JsonNode config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) - .put(JdbcUtils.SCHEMA_KEY, "public") - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.SSL_MODE_KEY, ImmutableMap.builder() + final JsonNode config = testDb.configBuilder() + .with("schema", "public") + .withDatabase() + .withResolvedHostAndPort() + .withCredentials() + .with("tunnel_method", ImmutableMap.builder().put("tunnel_method", TunnelMethod.NO_TUNNEL.toString()).build()) + .with("ssl_mode", ImmutableMap.builder() .put("mode", "prefer") .build()) - .put("tunnel_method", ImmutableMap.builder() - .put("tunnel_method", "NO_TUNNEL") - .build()) - .build()); - + .build(); final var actual = runCheck(config); assertEquals(Status.FAILED, actual.getStatus()); assertTrue(actual.getMessage().contains("Unsecured connection not allowed")); @@ -178,21 +75,16 @@ void testStrictSSLUnsecuredNoTunnel() throws Exception { @Test void testStrictSSLSecuredNoTunnel() throws Exception { - final JsonNode config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) - .put(JdbcUtils.SCHEMA_KEY, "public") - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.SSL_MODE_KEY, ImmutableMap.builder() + final JsonNode config = testDb.configBuilder() + .with("schema", "public") + .withDatabase() + .withResolvedHostAndPort() + .withCredentials() + .with("tunnel_method", ImmutableMap.builder().put("tunnel_method", TunnelMethod.NO_TUNNEL.toString()).build()) + .with("ssl_mode", ImmutableMap.builder() .put("mode", "require") .build()) - .put("tunnel_method", ImmutableMap.builder() - .put("tunnel_method", "NO_TUNNEL") - .build()) - .build()); - + .build(); final var actual = runCheck(config); assertEquals(Status.SUCCEEDED, actual.getStatus()); } @@ -212,4 +104,14 @@ protected String getDestinationDefinitionKey() { return "airbyte/destination-postgres"; } + @Override + protected boolean supportsInDestinationNormalization() { + return true; + } + + @Disabled("Custom DBT does not have root certificate created in the Postgres container.") + public void testCustomDbtTransformations() throws Exception { + super.testCustomDbtTransformations(); + } + } diff --git a/airbyte-integrations/connectors/destination-postgres/.dockerignore b/airbyte-integrations/connectors/destination-postgres/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-postgres/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-postgres/Dockerfile b/airbyte-integrations/connectors/destination-postgres/Dockerfile deleted file mode 100644 index 7d9f429bff5b..000000000000 --- a/airbyte-integrations/connectors/destination-postgres/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-postgres - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-postgres - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.4.0 -LABEL io.airbyte.name=airbyte/destination-postgres diff --git a/airbyte-integrations/connectors/destination-postgres/build.gradle b/airbyte-integrations/connectors/destination-postgres/build.gradle index bbf2ab1b1cef..da4dcd05868d 100644 --- a/airbyte-integrations/connectors/destination-postgres/build.gradle +++ b/airbyte-integrations/connectors/destination-postgres/build.gradle @@ -1,29 +1,50 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.13.1' + features = [ + 'db-sources', // required for tests + 'db-destinations', + ] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.postgres.PostgresDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:bases-destination-jdbc') + implementation libs.postgresql + // TODO: declare typing-deduping as a CDK feature instead of importing from source. + implementation project(':airbyte-cdk:java:airbyte-cdk:typing-deduping') + + testImplementation libs.testcontainers.jdbc + testImplementation libs.testcontainers.postgresql - testImplementation project(':airbyte-test-utils') + testFixturesImplementation libs.testcontainers.jdbc + testFixturesImplementation libs.testcontainers.postgresql - testImplementation libs.connectors.testcontainers.postgresql - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-postgres') + integrationTestJavaImplementation libs.testcontainers.postgresql - integrationTestJavaImplementation libs.connectors.testcontainers.postgresql + integrationTestJavaImplementation testFixtures(project(':airbyte-cdk:java:airbyte-cdk:typing-deduping')) +} - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) +configurations.all { + resolutionStrategy { + force libs.jooq + } } diff --git a/airbyte-integrations/connectors/destination-postgres/gradle.properties b/airbyte-integrations/connectors/destination-postgres/gradle.properties index 5d1adb3b55c1..23da4989675e 100644 --- a/airbyte-integrations/connectors/destination-postgres/gradle.properties +++ b/airbyte-integrations/connectors/destination-postgres/gradle.properties @@ -1,3 +1,3 @@ -# currently limit the number of parallel threads until further investigation into the issues \ -# where integration tests run into race conditions -numberThreads=1 +# our testcontainer has issues with too much concurrency. +# 4 threads seems to be the sweet spot. +testExecutionConcurrency=4 diff --git a/airbyte-integrations/connectors/destination-postgres/metadata.yaml b/airbyte-integrations/connectors/destination-postgres/metadata.yaml index 2b7a003fb13f..b0983ff2e291 100644 --- a/airbyte-integrations/connectors/destination-postgres/metadata.yaml +++ b/airbyte-integrations/connectors/destination-postgres/metadata.yaml @@ -1,9 +1,13 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: database connectorType: destination definitionId: 25c5221d-dce2-4163-ade9-739ef790f503 - dockerImageTag: 0.4.0 + dockerImageTag: 0.5.5 dockerRepository: airbyte/destination-postgres + documentationUrl: https://docs.airbyte.com/integrations/destinations/postgres githubIssueLabel: destination-postgres icon: postgresql.svg license: ELv2 @@ -19,12 +23,8 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/destinations/postgres + supportLevel: community supportsDbt: true tags: - language:java - ab_internal: - sl: 100 - ql: 200 - supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDataAdapter.java b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDataAdapter.java index 110c38dfdf54..2793af8cd1e6 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDataAdapter.java +++ b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDataAdapter.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.postgres; +import io.airbyte.cdk.integrations.destination.jdbc.DataAdapter; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.jdbc.DataAdapter; public class PostgresDataAdapter extends DataAdapter { diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java index e00fc425f5af..3a2a36ce446b 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java +++ b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresDestination.java @@ -4,21 +4,24 @@ package io.airbyte.integrations.destination.postgres; -import static io.airbyte.integrations.util.PostgresSslConnectionUtils.DISABLE; -import static io.airbyte.integrations.util.PostgresSslConnectionUtils.PARAM_MODE; -import static io.airbyte.integrations.util.PostgresSslConnectionUtils.PARAM_SSL; -import static io.airbyte.integrations.util.PostgresSslConnectionUtils.PARAM_SSL_MODE; -import static io.airbyte.integrations.util.PostgresSslConnectionUtils.obtainConnectionOptions; +import static io.airbyte.cdk.integrations.util.PostgresSslConnectionUtils.DISABLE; +import static io.airbyte.cdk.integrations.util.PostgresSslConnectionUtils.PARAM_MODE; +import static io.airbyte.cdk.integrations.util.PostgresSslConnectionUtils.PARAM_SSL; +import static io.airbyte.cdk.integrations.util.PostgresSslConnectionUtils.PARAM_SSL_MODE; +import static io.airbyte.cdk.integrations.util.PostgresSslConnectionUtils.obtainConnectionOptions; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination; +import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.ssh.SshWrappedDestination; -import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; +import io.airbyte.integrations.destination.postgres.typing_deduping.PostgresSqlGenerator; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.HashMap; @@ -41,6 +44,28 @@ public PostgresDestination() { super(DRIVER_CLASS, new PostgresSQLNameTransformer(), new PostgresSqlOperations()); } + @Override + protected DataSourceFactory.DataSourceBuilder modifyDataSourceBuilder(final DataSourceFactory.DataSourceBuilder builder) { + // Anything in the pg_temp schema is only visible to the connection that created it. + // So this creates an airbyte_safe_cast function that only exists for the duration of + // a single connection. + // This avoids issues with creating the same function concurrently (e.g. if multiple syncs run + // at the same time). + // Function definition copied from https://dba.stackexchange.com/a/203986 + return builder.withConnectionInitSql(""" + CREATE FUNCTION pg_temp.airbyte_safe_cast(_in text, INOUT _out ANYELEMENT) + LANGUAGE plpgsql AS + $func$ + BEGIN + EXECUTE format('SELECT %L::%s', $1, pg_typeof(_out)) + INTO _out; + EXCEPTION WHEN others THEN + -- do nothing: _out already carries default + END + $func$; + """); + } + @Override protected Map getDefaultConnectionProperties(final JsonNode config) { final Map additionalParameters = new HashMap<>(); @@ -67,7 +92,7 @@ public JsonNode toJdbcConfig(final JsonNode config) { if (encodedDatabase != null) { try { encodedDatabase = URLEncoder.encode(encodedDatabase, "UTF-8"); - } catch (UnsupportedEncodingException e) { + } catch (final UnsupportedEncodingException e) { // Should never happen e.printStackTrace(); } @@ -93,6 +118,11 @@ public JsonNode toJdbcConfig(final JsonNode config) { return Jsons.jsonNode(configBuilder.build()); } + @Override + protected JdbcSqlGenerator getSqlGenerator() { + return new PostgresSqlGenerator(new PostgresSQLNameTransformer()); + } + public static void main(final String[] args) throws Exception { final Destination destination = PostgresDestination.sshWrappedDestination(); LOGGER.info("starting destination: {}", PostgresDestination.class); diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresSQLNameTransformer.java b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresSQLNameTransformer.java index c5f276c9ebd2..e586f4047995 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresSQLNameTransformer.java +++ b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresSQLNameTransformer.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.postgres; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; public class PostgresSQLNameTransformer extends StandardNameTransformer { diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresSqlOperations.java b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresSqlOperations.java index 4198e00b6ced..01e4904b5684 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresSqlOperations.java +++ b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/PostgresSqlOperations.java @@ -4,9 +4,10 @@ package io.airbyte.integrations.destination.postgres; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.jdbc.JdbcSqlOperations; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.TypingAndDedupingFlag; +import io.airbyte.cdk.integrations.destination.jdbc.JdbcSqlOperations; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; @@ -14,6 +15,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.sql.SQLException; +import java.util.Collections; import java.util.List; import org.postgresql.copy.CopyManager; import org.postgresql.core.BaseConnection; @@ -24,9 +26,36 @@ public PostgresSqlOperations() { super(new PostgresDataAdapter()); } + @Override + protected List postCreateTableQueries(final String schemaName, final String tableName) { + if (TypingAndDedupingFlag.isDestinationV2()) { + return List.of( + // the raw_id index _could_ be unique (since raw_id is a UUID) + // but there's no reason to do that (because it's a UUID :P ) + // and it would just slow down inserts. + // also, intentionally don't specify the type of index (btree, hash, etc). Just use the default. + "CREATE INDEX IF NOT EXISTS " + tableName + "_raw_id" + " ON " + schemaName + "." + tableName + "(_airbyte_raw_id)", + "CREATE INDEX IF NOT EXISTS " + tableName + "_extracted_at" + " ON " + schemaName + "." + tableName + "(_airbyte_extracted_at)", + "CREATE INDEX IF NOT EXISTS " + tableName + "_loaded_at" + " ON " + schemaName + "." + tableName + + "(_airbyte_loaded_at, _airbyte_extracted_at)"); + } else { + return Collections.emptyList(); + } + } + + @Override + protected void insertRecordsInternalV2(final JdbcDatabase database, + final List records, + final String schemaName, + final String tableName) + throws Exception { + // idk apparently this just works + insertRecordsInternal(database, records, schemaName, tableName); + } + @Override public void insertRecordsInternal(final JdbcDatabase database, - final List records, + final List records, final String schemaName, final String tmpTableName) throws SQLException { diff --git a/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresSqlGenerator.java b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresSqlGenerator.java new file mode 100644 index 000000000000..342ab040fc79 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/main/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresSqlGenerator.java @@ -0,0 +1,319 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.postgres.typing_deduping; + +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_META; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_RAW_ID; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_DATA; +import static java.util.Collections.emptyList; +import static org.jooq.impl.DSL.array; +import static org.jooq.impl.DSL.case_; +import static org.jooq.impl.DSL.cast; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.function; +import static org.jooq.impl.DSL.name; +import static org.jooq.impl.DSL.quotedName; +import static org.jooq.impl.DSL.rowNumber; +import static org.jooq.impl.DSL.val; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.TableDefinition; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType; +import io.airbyte.integrations.base.destination.typing_deduping.Array; +import io.airbyte.integrations.base.destination.typing_deduping.ColumnId; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; +import io.airbyte.integrations.base.destination.typing_deduping.Struct; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.jooq.Condition; +import org.jooq.DataType; +import org.jooq.Field; +import org.jooq.Name; +import org.jooq.SQLDialect; +import org.jooq.impl.DefaultDataType; +import org.jooq.impl.SQLDataType; + +public class PostgresSqlGenerator extends JdbcSqlGenerator { + + public static final DataType JSONB_TYPE = new DefaultDataType<>(null, Object.class, "jsonb"); + + private static final Map POSTGRES_TYPE_NAME_TO_JDBC_TYPE = ImmutableMap.of( + "numeric", "decimal", + "int8", "bigint", + "bool", "boolean", + "timestamptz", "timestamp with time zone", + "timetz", "time with time zone"); + + public PostgresSqlGenerator(final NamingConventionTransformer namingTransformer) { + super(namingTransformer); + } + + @Override + protected DataType getStructType() { + return JSONB_TYPE; + } + + @Override + protected DataType getArrayType() { + return JSONB_TYPE; + } + + @Override + protected DataType getWidestType() { + return JSONB_TYPE; + } + + @Override + protected SQLDialect getDialect() { + return SQLDialect.POSTGRES; + } + + @Override + public Sql createTable(final StreamConfig stream, final String suffix, final boolean force) { + final List statements = new ArrayList<>(); + final Name finalTableName = name(stream.id().finalNamespace(), stream.id().finalName() + suffix); + + statements.add(super.createTable(stream, suffix, force)); + + if (stream.destinationSyncMode() == DestinationSyncMode.APPEND_DEDUP) { + // An index for our ROW_NUMBER() PARTITION BY pk ORDER BY cursor, extracted_at function + final List pkNames = stream.primaryKey().stream() + .map(pk -> quotedName(pk.name())) + .toList(); + statements.add(Sql.of(getDslContext().createIndex().on( + finalTableName, + Stream.of( + pkNames.stream(), + // if cursor is present, then a stream containing its name + // but if no cursor, then empty stream + stream.cursor().stream().map(cursor -> quotedName(cursor.name())), + Stream.of(name(COLUMN_NAME_AB_EXTRACTED_AT))).flatMap(Function.identity()).toList()) + .getSQL())); + } + statements.add(Sql.of(getDslContext().createIndex().on( + finalTableName, + name(COLUMN_NAME_AB_EXTRACTED_AT)) + .getSQL())); + + statements.add(Sql.of(getDslContext().createIndex().on( + finalTableName, + name(COLUMN_NAME_AB_RAW_ID)) + .getSQL())); + + return Sql.concat(statements); + } + + @Override + protected List createIndexSql(final StreamConfig stream, final String suffix) { + if (stream.destinationSyncMode() == DestinationSyncMode.APPEND_DEDUP && !stream.primaryKey().isEmpty()) { + return List.of( + getDslContext().createIndex().on( + name(stream.id().finalNamespace(), stream.id().finalName() + suffix), + stream.primaryKey().stream() + .map(pk -> quotedName(pk.name())) + .toList()) + .getSQL()); + } else { + return emptyList(); + } + } + + @Override + protected List> extractRawDataFields(final LinkedHashMap columns, final boolean useExpensiveSaferCasting) { + return columns + .entrySet() + .stream() + .map(column -> castedField( + extractColumnAsJson(column.getKey()), + column.getValue(), + column.getKey().name(), + useExpensiveSaferCasting)) + .collect(Collectors.toList()); + } + + @Override + protected Field castedField( + final Field field, + final AirbyteType type, + final String alias, + final boolean useExpensiveSaferCasting) { + return castedField(field, type, useExpensiveSaferCasting).as(quotedName(alias)); + } + + protected Field castedField( + final Field field, + final AirbyteType type, + final boolean useExpensiveSaferCasting) { + if (type instanceof Struct) { + // If this field is a struct, verify that the raw data is an object. + return cast( + case_() + .when(field.isNull().or(jsonTypeof(field).ne("object")), val((Object) null)) + .else_(field), + JSONB_TYPE); + } else if (type instanceof Array) { + // Do the same for arrays. + return cast( + case_() + .when(field.isNull().or(jsonTypeof(field).ne("array")), val((Object) null)) + .else_(field), + JSONB_TYPE); + } else if (type == AirbyteProtocolType.UNKNOWN) { + return cast(field, JSONB_TYPE); + } else if (type == AirbyteProtocolType.STRING) { + // we need to render the jsonb to a normal string. For strings, this is the difference between + // "\"foo\"" and "foo". + // postgres provides the #>> operator, which takes a json path and returns that extraction as a + // string. + // '{}' is an empty json path (it's an empty array literal), so it just stringifies the json value. + return field("{0} #>> '{}'", String.class, field); + } else { + final DataType dialectType = toDialectType(type); + // jsonb can't directly cast to most types, so convert to text first. + // also convert jsonb null to proper sql null. + final Field extractAsText = case_() + .when(field.isNull().or(jsonTypeof(field).eq("null")), val((String) null)) + .else_(cast(field, SQLDataType.VARCHAR)); + if (useExpensiveSaferCasting) { + return function(name("pg_temp", "airbyte_safe_cast"), dialectType, extractAsText, cast(val((Object) null), dialectType)); + } else { + return cast(extractAsText, dialectType); + } + } + } + + // TODO this isn't actually used right now... can we refactor this out? + // (redshift is doing something interesting with this method, so leaving it for now) + @Override + protected Field castedField(final Field field, final AirbyteProtocolType type, final boolean useExpensiveSaferCasting) { + return cast(field, toDialectType(type)); + } + + @Override + protected Field buildAirbyteMetaColumn(final LinkedHashMap columns) { + final Field[] dataFieldErrors = columns + .entrySet() + .stream() + .map(column -> toCastingErrorCaseStmt(column.getKey(), column.getValue())) + .toArray(Field[]::new); + return function( + "JSONB_BUILD_OBJECT", + JSONB_TYPE, + val("errors"), + function("ARRAY_REMOVE", JSONB_TYPE, array(dataFieldErrors), val((String) null))).as(COLUMN_NAME_AB_META); + } + + private Field toCastingErrorCaseStmt(final ColumnId column, final AirbyteType type) { + final Field extract = extractColumnAsJson(column); + if (type instanceof Struct) { + // If this field is a struct, verify that the raw data is an object or null. + return case_() + .when( + extract.isNotNull() + .and(jsonTypeof(extract).notIn("object", "null")), + val("Problem with `" + column.originalName() + "`")) + .else_(val((String) null)); + } else if (type instanceof Array) { + // Do the same for arrays. + return case_() + .when( + extract.isNotNull() + .and(jsonTypeof(extract).notIn("array", "null")), + val("Problem with `" + column.originalName() + "`")) + .else_(val((String) null)); + } else if (type == AirbyteProtocolType.UNKNOWN || type == AirbyteProtocolType.STRING) { + // Unknown types require no casting, so there's never an error. + // Similarly, everything can cast to string without error. + return val((String) null); + } else { + // For other type: If the raw data is not NULL or 'null', but the casted data is NULL, + // then we have a typing error. + return case_() + .when( + extract.isNotNull() + .and(jsonTypeof(extract).ne("null")) + .and(castedField(extract, type, true).isNull()), + val("Problem with `" + column.originalName() + "`")) + .else_(val((String) null)); + } + } + + @Override + protected Condition cdcDeletedAtNotNullCondition() { + return field(name(COLUMN_NAME_AB_LOADED_AT)).isNotNull() + .and(jsonTypeof(extractColumnAsJson(cdcDeletedAtColumn)).ne("null")); + } + + @Override + protected Field getRowNumber(final List primaryKeys, final Optional cursor) { + // literally identical to redshift's getRowNumber implementation, changes here probably should + // be reflected there + final List> primaryKeyFields = + primaryKeys != null ? primaryKeys.stream().map(columnId -> field(quotedName(columnId.name()))).collect(Collectors.toList()) + : new ArrayList<>(); + final List> orderedFields = new ArrayList<>(); + // We can still use Jooq's field to get the quoted name with raw sql templating. + // jooq's .desc returns SortField instead of Field and NULLS LAST doesn't work with it + cursor.ifPresent(columnId -> orderedFields.add(field("{0} desc NULLS LAST", field(quotedName(columnId.name()))))); + orderedFields.add(field("{0} desc", quotedName(COLUMN_NAME_AB_EXTRACTED_AT))); + return rowNumber() + .over() + .partitionBy(primaryKeyFields) + .orderBy(orderedFields).as(ROW_NUMBER_COLUMN_NAME); + } + + @Override + public boolean existingSchemaMatchesStreamConfig(final StreamConfig stream, final TableDefinition existingTable) { + // Check that the columns match, with special handling for the metadata columns. + // This is mostly identical to the redshift implementation, but swaps super to jsonb + final LinkedHashMap intendedColumns = stream.columns().entrySet().stream() + .collect(LinkedHashMap::new, + (map, column) -> map.put(column.getKey().name(), toDialectType(column.getValue()).getTypeName()), + LinkedHashMap::putAll); + final LinkedHashMap actualColumns = existingTable.columns().entrySet().stream() + .filter(column -> JavaBaseConstants.V2_FINAL_TABLE_METADATA_COLUMNS.stream() + .noneMatch(airbyteColumnName -> airbyteColumnName.equals(column.getKey()))) + .collect(LinkedHashMap::new, + (map, column) -> map.put(column.getKey(), jdbcTypeNameFromPostgresTypeName(column.getValue().type())), + LinkedHashMap::putAll); + + final boolean sameColumns = actualColumns.equals(intendedColumns) + && "varchar".equals(existingTable.columns().get(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID).type()) + && "timestamptz".equals(existingTable.columns().get(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT).type()) + && "jsonb".equals(existingTable.columns().get(JavaBaseConstants.COLUMN_NAME_AB_META).type()); + + return sameColumns; + } + + /** + * Extract a raw field, leaving it as jsonb + */ + private Field extractColumnAsJson(final ColumnId column) { + return field("{0} -> {1}", name(COLUMN_NAME_DATA), val(column.originalName())); + } + + private Field jsonTypeof(final Field field) { + return function("JSONB_TYPEOF", SQLDataType.VARCHAR, field); + } + + private static String jdbcTypeNameFromPostgresTypeName(final String redshiftType) { + return POSTGRES_TYPE_NAME_TO_JDBC_TYPE.getOrDefault(redshiftType, redshiftType); + } + +} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationAcceptanceTest.java index f70d06171432..369ecdd5caf2 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationAcceptanceTest.java @@ -5,138 +5,37 @@ package io.airbyte.integrations.destination.postgres; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; -import io.airbyte.integrations.util.HostPortResolver; -import java.sql.SQLException; +import io.airbyte.integrations.destination.postgres.PostgresTestDatabase.BaseImage; import java.util.HashSet; -import java.util.List; -import java.util.stream.Collectors; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.testcontainers.containers.PostgreSQLContainer; -public class PostgresDestinationAcceptanceTest extends JdbcDestinationAcceptanceTest { +public class PostgresDestinationAcceptanceTest extends AbstractPostgresDestinationAcceptanceTest { - private PostgreSQLContainer db; - private final StandardNameTransformer namingResolver = new StandardNameTransformer(); - - @Override - protected String getImageName() { - return "airbyte/destination-postgres:dev"; - } + private PostgresTestDatabase testDb; @Override protected JsonNode getConfig() { - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db)) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) - .put(JdbcUtils.SCHEMA_KEY, "public") - .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(db)) - .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.SSL_KEY, false) - .build()); - } - - @Override - protected JsonNode getFailCheckConfig() { - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, "wrong password") - .put(JdbcUtils.SCHEMA_KEY, "public") - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.SSL_KEY, false) - .build()); - } - - @Override - protected List retrieveRecords(final TestDestinationEnv env, - final String streamName, - final String namespace, - final JsonNode streamSchema) - throws Exception { - return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) - .stream() - .map(r -> r.get(JavaBaseConstants.COLUMN_NAME_DATA)) - .collect(Collectors.toList()); - } - - @Override - protected boolean implementsNamespaces() { - return true; - } - - @Override - protected TestDataComparator getTestDataComparator() { - return new PostgresTestDataComparator(); + return testDb.configBuilder() + .with("schema", "public") + .withDatabase() + .withResolvedHostAndPort() + .withCredentials() + .withoutSsl() + .build(); } @Override - protected boolean supportBasicDataTypeTest() { - return true; - } - - @Override - protected boolean supportArrayDataTypeTest() { - return true; - } - - @Override - protected boolean supportObjectDataTypeTest() { - return true; - } - - @Override - protected boolean supportIncrementalSchemaChanges() { - return true; - } - - @Override - protected List retrieveNormalizedRecords(final TestDestinationEnv env, final String streamName, final String namespace) - throws Exception { - final String tableName = namingResolver.getIdentifier(streamName); - return retrieveRecordsFromTable(tableName, namespace); - } - - private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { - try (final DSLContext dslContext = DSLContextFactory.create( - db.getUsername(), - db.getPassword(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - db.getJdbcUrl(), - SQLDialect.POSTGRES)) { - return new Database(dslContext) - .query(ctx -> { - ctx.execute("set time zone 'UTC';"); - return ctx.fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) - .stream() - .map(this::getJsonFromRecord) - .collect(Collectors.toList()); - }); - } + protected PostgresTestDatabase getTestDb() { + return testDb; } @Override protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) { - db = new PostgreSQLContainer<>("postgres:13-alpine"); - db.start(); + testDb = PostgresTestDatabase.in(BaseImage.POSTGRES_13); } @Override protected void tearDown(final TestDestinationEnv testEnv) { - db.stop(); - db.close(); + testDb.close(); } } diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationSSLFullCertificateAcceptanceTest.java b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationSSLFullCertificateAcceptanceTest.java index 942c53b2439e..37265484202d 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationSSLFullCertificateAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresDestinationSSLFullCertificateAcceptanceTest.java @@ -4,34 +4,15 @@ package io.airbyte.integrations.destination.postgres; -import static io.airbyte.db.PostgresUtils.getCertificate; - import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.PostgresUtils; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; -import java.sql.SQLException; +import io.airbyte.integrations.destination.postgres.PostgresTestDatabase.BaseImage; import java.util.HashSet; -import java.util.List; -import java.util.stream.Collectors; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.utility.DockerImageName; - -public class PostgresDestinationSSLFullCertificateAcceptanceTest extends JdbcDestinationAcceptanceTest { +import org.junit.jupiter.api.Disabled; - private PostgreSQLContainer db; +public class PostgresDestinationSSLFullCertificateAcceptanceTest extends AbstractPostgresDestinationAcceptanceTest { - protected static PostgresUtils.Certificate certs; - private final StandardNameTransformer namingResolver = new StandardNameTransformer(); + private PostgresTestDatabase testDb; @Override protected String getImageName() { @@ -40,111 +21,36 @@ protected String getImageName() { @Override protected JsonNode getConfig() { - return Jsons.jsonNode(ImmutableMap.builder() - .put("host", db.getHost()) - .put("username", "postgres") - .put("password", "postgres") - .put("schema", "public") - .put("port", db.getFirstMappedPort()) - .put("database", db.getDatabaseName()) - .put("ssl", true) - .put("ssl_mode", ImmutableMap.builder() - .put("mode", "verify-full") - .put("ca_certificate", certs.getCaCertificate()) - .put("client_certificate", certs.getClientCertificate()) - .put("client_key", certs.getClientKey()) - .put("client_key_password", "Passw0rd") + return testDb.configBuilder() + .with("schema", "public") + .withDatabase() + .withResolvedHostAndPort() + .withCredentials() + .withSsl(ImmutableMap.builder() + .put("mode", "verify-ca") // verify-full will not work since the spawned container is only allowed for 127.0.0.1/32 CIDRs + .put("ca_certificate", testDb.getCertificates().caCertificate()) .build()) - .build()); - } - - @Override - protected JsonNode getFailCheckConfig() { - return Jsons.jsonNode(ImmutableMap.builder() - .put("host", db.getHost()) - .put("username", db.getUsername()) - .put("password", "wrong password") - .put("schema", "public") - .put("port", db.getFirstMappedPort()) - .put("database", db.getDatabaseName()) - .put("ssl", false) - .build()); - } - - @Override - protected List retrieveRecords(final TestDestinationEnv env, - final String streamName, - final String namespace, - final JsonNode streamSchema) - throws Exception { - return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) - .stream() - .map(r -> r.get(JavaBaseConstants.COLUMN_NAME_DATA)) - .collect(Collectors.toList()); - } - - @Override - protected boolean implementsNamespaces() { - return true; - } - - @Override - protected TestDataComparator getTestDataComparator() { - return new PostgresTestDataComparator(); + .build(); } @Override - protected boolean supportBasicDataTypeTest() { - return true; - } - - @Override - protected boolean supportArrayDataTypeTest() { - return true; - } - - @Override - protected boolean supportObjectDataTypeTest() { - return true; - } - - @Override - protected List retrieveNormalizedRecords(final TestDestinationEnv env, final String streamName, final String namespace) - throws Exception { - final String tableName = namingResolver.getIdentifier(streamName); - return retrieveRecordsFromTable(tableName, namespace); - } - - private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { - try (final DSLContext dslContext = DSLContextFactory.create( - db.getUsername(), - db.getPassword(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - db.getJdbcUrl(), - SQLDialect.POSTGRES)) { - return new Database(dslContext) - .query(ctx -> { - ctx.execute("set time zone 'UTC';"); - return ctx.fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) - .stream() - .map(this::getJsonFromRecord) - .collect(Collectors.toList()); - }); - } + protected PostgresTestDatabase getTestDb() { + return testDb; } @Override protected void setup(final TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) throws Exception { - db = new PostgreSQLContainer<>(DockerImageName.parse("postgres:bullseye") - .asCompatibleSubstituteFor("postgres")); - db.start(); - certs = getCertificate(db); + testDb = PostgresTestDatabase.in(BaseImage.POSTGRES_12, PostgresTestDatabase.ContainerModifier.CERT); } @Override protected void tearDown(final TestDestinationEnv testEnv) throws Exception { - db.stop(); - db.close(); + testDb.close(); + } + + @Disabled("Custom DBT does not have root certificate created in the Postgres container.") + public void testCustomDbtTransformations() throws Exception { + super.testCustomDbtTransformations(); } } diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshKeyPostgresDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshKeyPostgresDestinationAcceptanceTest.java index 3460367c9f49..ce0e53ca1fdf 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshKeyPostgresDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshKeyPostgresDestinationAcceptanceTest.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.postgres; -import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; public class SshKeyPostgresDestinationAcceptanceTest extends SshPostgresDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPasswordPostgresDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPasswordPostgresDestinationAcceptanceTest.java index 2698271f7706..8404ee16f4fb 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPasswordPostgresDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPasswordPostgresDestinationAcceptanceTest.java @@ -4,7 +4,9 @@ package io.airbyte.integrations.destination.postgres; -import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider.TestCompatibility; +import org.junit.jupiter.api.Disabled; public class SshPasswordPostgresDestinationAcceptanceTest extends SshPostgresDestinationAcceptanceTest { @@ -13,4 +15,33 @@ public SshTunnel.TunnelMethod getTunnelMethod() { return SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH; } + @Disabled("sshpass tunnel is not working with DBT container. https://github.com/airbytehq/airbyte/issues/33547") + public void testIncrementalDedupeSync() throws Exception { + super.testIncrementalDedupeSync(); + } + + @Disabled("sshpass tunnel is not working with DBT container. https://github.com/airbytehq/airbyte/issues/33547") + @Override + public void testDataTypeTestWithNormalization(String messagesFilename, + String catalogFilename, + TestCompatibility testCompatibility) + throws Exception { + super.testDataTypeTestWithNormalization(messagesFilename, catalogFilename, testCompatibility); + } + + @Disabled("sshpass tunnel is not working with DBT container. https://github.com/airbytehq/airbyte/issues/33547") + @Override + public void testSyncWithNormalization(String messagesFilename, String catalogFilename) throws Exception { + super.testSyncWithNormalization(messagesFilename, catalogFilename); + } + + @Disabled("sshpass tunnel is not working with DBT container. https://github.com/airbytehq/airbyte/issues/33547") + @Override + public void testCustomDbtTransformations() throws Exception { + super.testCustomDbtTransformations(); + } + + // TODO: Although testCustomDbtTransformationsFailure is passing, the failure is for wrong reasons. + // See disabled tests. + } diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPostgresDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPostgresDestinationAcceptanceTest.java index 9755a3d60fb5..4412909f1539 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPostgresDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/SshPostgresDestinationAcceptanceTest.java @@ -4,104 +4,44 @@ package io.airbyte.integrations.destination.postgres; +import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.CONNECTION_OPTIONS_KEY; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.base.ssh.SshBastionContainer; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; import io.airbyte.commons.functional.CheckedFunction; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.base.ssh.SshBastionContainer; -import io.airbyte.integrations.base.ssh.SshTunnel; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.integrations.destination.postgres.PostgresTestDatabase.BaseImage; +import io.airbyte.integrations.destination.postgres.PostgresTestDatabase.ContainerModifier; import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; -import org.apache.commons.lang3.RandomStringUtils; import org.jooq.SQLDialect; -import org.testcontainers.containers.Network; -import org.testcontainers.containers.PostgreSQLContainer; - -// todo (cgardens) - likely some of this could be further de-duplicated with -// PostgresDestinationAcceptanceTest. /** * Abstract class that allows us to avoid duplicating testing logic for testing SSH with a key file * or with a password. */ -public abstract class SshPostgresDestinationAcceptanceTest extends JdbcDestinationAcceptanceTest { +public abstract class SshPostgresDestinationAcceptanceTest extends AbstractPostgresDestinationAcceptanceTest { - private final StandardNameTransformer namingResolver = new StandardNameTransformer(); - private static final String schemaName = RandomStringUtils.randomAlphabetic(8).toLowerCase(); - private static final Network network = Network.newNetwork(); - private static PostgreSQLContainer db; - private final SshBastionContainer bastion = new SshBastionContainer(); + private PostgresTestDatabase testdb; + private SshBastionContainer bastion; public abstract SshTunnel.TunnelMethod getTunnelMethod(); - @Override - protected String getImageName() { - return "airbyte/destination-postgres:dev"; - } - @Override protected JsonNode getConfig() throws Exception { - return bastion.getTunnelConfig(getTunnelMethod(), bastion.getBasicDbConfigBuider(db).put("schema", schemaName), false); - } - - @Override - protected JsonNode getFailCheckConfig() throws Exception { - final JsonNode clone = Jsons.clone(getConfig()); - ((ObjectNode) clone).put("password", "wrong password"); - return clone; - } - - @Override - protected List retrieveRecords(final TestDestinationEnv env, - final String streamName, - final String namespace, - final JsonNode streamSchema) - throws Exception { - return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) - .stream() - .map(r -> r.get(JavaBaseConstants.COLUMN_NAME_DATA)) - .collect(Collectors.toList()); - } - - @Override - protected boolean implementsNamespaces() { - return true; - } - - @Override - protected TestDataComparator getTestDataComparator() { - return new PostgresTestDataComparator(); - } - - @Override - protected boolean supportBasicDataTypeTest() { - return true; - } - - @Override - protected boolean supportArrayDataTypeTest() { - return true; - } - - @Override - protected boolean supportObjectDataTypeTest() { - return true; - } - - @Override - protected List retrieveNormalizedRecords(final TestDestinationEnv env, final String streamName, final String namespace) - throws Exception { - final String tableName = namingResolver.getIdentifier(streamName); - return retrieveRecordsFromTable(tableName, namespace); + // Here we use inner address because the tunnel is created inside the connector's container. + return testdb.integrationTestConfigBuilder() + .with("tunnel_method", bastion.getTunnelMethod(getTunnelMethod(), true)) + .with("schema", "public") + .withoutSsl() + .build(); } private static Database getDatabaseFromConfig(final JsonNode config) { @@ -117,8 +57,16 @@ private static Database getDatabaseFromConfig(final JsonNode config) { SQLDialect.POSTGRES)); } - private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws Exception { - final JsonNode config = getConfig(); + @Override + protected List retrieveRecordsFromTable(final String tableName, final String schemaName) throws Exception { + // Here we DO NOT use the inner address because the tunnel is created in the integration test's java + // process. + final JsonNode config = testdb.integrationTestConfigBuilder() + .with("tunnel_method", bastion.getTunnelMethod(getTunnelMethod(), false)) + .with("schema", "public") + .withoutSsl() + .build(); + ((ObjectNode) config).putObject(CONNECTION_OPTIONS_KEY); return SshTunnel.sshWrap( config, JdbcUtils.HOST_LIST_KEY, @@ -135,42 +83,20 @@ private List retrieveRecordsFromTable(final String tableName, final St @Override protected void setup(final TestDestinationEnv testEnv, HashSet TEST_SCHEMAS) throws Exception { - - startTestContainers(); - // do everything in a randomly generated schema so that we can wipe it out at the end. - SshTunnel.sshWrap( - getConfig(), - JdbcUtils.HOST_LIST_KEY, - JdbcUtils.PORT_LIST_KEY, - mangledConfig -> { - getDatabaseFromConfig(mangledConfig).query(ctx -> ctx.fetch(String.format("CREATE SCHEMA %s;", schemaName))); - TEST_SCHEMAS.add(schemaName); - }); - } - - private void startTestContainers() { - bastion.initAndStartBastion(network); - initAndStartJdbcContainer(); - } - - private void initAndStartJdbcContainer() { - db = new PostgreSQLContainer<>("postgres:13-alpine") - .withNetwork(network); - db.start(); + testdb = PostgresTestDatabase.in(BaseImage.POSTGRES_13, ContainerModifier.NETWORK); + bastion = new SshBastionContainer(); + bastion.initAndStartBastion(testdb.getContainer().getNetwork()); } @Override protected void tearDown(final TestDestinationEnv testEnv) throws Exception { - // blow away the test schema at the end. - SshTunnel.sshWrap( - getConfig(), - JdbcUtils.HOST_LIST_KEY, - JdbcUtils.PORT_LIST_KEY, - mangledConfig -> { - getDatabaseFromConfig(mangledConfig).query(ctx -> ctx.fetch(String.format("DROP SCHEMA %s CASCADE;", schemaName))); - }); + testdb.close(); + bastion.stopAndClose(); + } - bastion.stopAndCloseContainers(db); + @Override + protected PostgresTestDatabase getTestDb() { + return testdb; } } diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresRawOverrideTypingDedupingTest.java b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresRawOverrideTypingDedupingTest.java new file mode 100644 index 000000000000..f31c3325d226 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresRawOverrideTypingDedupingTest.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.postgres.typing_deduping; + +import com.fasterxml.jackson.databind.node.ObjectNode; + +public class PostgresRawOverrideTypingDedupingTest extends PostgresTypingDedupingTest { + + @Override + protected ObjectNode getBaseConfig() { + return super.getBaseConfig() + .put("raw_data_schema", "overridden_raw_dataset"); + } + + @Override + protected String getRawSchema() { + return "overridden_raw_dataset"; + } + +} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresSqlGeneratorIntegrationTest.java b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresSqlGeneratorIntegrationTest.java new file mode 100644 index 000000000000..ee80c3e12ab5 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresSqlGeneratorIntegrationTest.java @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.postgres.typing_deduping; + +import static io.airbyte.integrations.destination.postgres.typing_deduping.PostgresSqlGenerator.JSONB_TYPE; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcSourceOperations; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.destination.jdbc.TableDefinition; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcDestinationHandler; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; +import io.airbyte.cdk.integrations.standardtest.destination.typing_deduping.JdbcSqlGeneratorIntegrationTest; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.destination.typing_deduping.DestinationHandler; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; +import io.airbyte.integrations.destination.postgres.PostgresDestination; +import io.airbyte.integrations.destination.postgres.PostgresSQLNameTransformer; +import io.airbyte.integrations.destination.postgres.PostgresTestDatabase; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Optional; +import javax.sql.DataSource; +import org.jooq.DataType; +import org.jooq.Field; +import org.jooq.SQLDialect; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class PostgresSqlGeneratorIntegrationTest extends JdbcSqlGeneratorIntegrationTest { + + private static PostgresTestDatabase testContainer; + private static String databaseName; + private static JdbcDatabase database; + + /** + * See + * {@link io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftSqlGeneratorIntegrationTest.RedshiftSourceOperations}. + * Copied here to avoid weird dependencies. + */ + public static class PostgresSourceOperations extends JdbcSourceOperations { + + @Override + public void copyToJsonField(final ResultSet resultSet, final int colIndex, final ObjectNode json) throws SQLException { + final String columnName = resultSet.getMetaData().getColumnName(colIndex); + final String columnTypeName = resultSet.getMetaData().getColumnTypeName(colIndex).toLowerCase(); + + switch (columnTypeName) { + // JSONB has no equivalent in JDBCType + case "jsonb" -> json.set(columnName, Jsons.deserializeExact(resultSet.getString(colIndex))); + // For some reason, the driver maps these to their timezoneless equivalents (TIME and TIMESTAMP) + case "timetz" -> putTimeWithTimezone(json, columnName, resultSet, colIndex); + case "timestamptz" -> putTimestampWithTimezone(json, columnName, resultSet, colIndex); + default -> super.copyToJsonField(resultSet, colIndex, json); + } + } + + } + + @BeforeAll + public static void setupPostgres() { + testContainer = PostgresTestDatabase.in(PostgresTestDatabase.BaseImage.POSTGRES_13); + final JsonNode config = testContainer.configBuilder() + .with("schema", "public") + .withDatabase() + .withHostAndPort() + .withCredentials() + .withoutSsl() + .build(); + + databaseName = config.get(JdbcUtils.DATABASE_KEY).asText(); + final PostgresDestination postgresDestination = new PostgresDestination(); + final DataSource dataSource = postgresDestination.getDataSource(config); + database = new DefaultJdbcDatabase(dataSource, new PostgresSourceOperations()); + } + + @AfterAll + public static void teardownPostgres() { + testContainer.close(); + } + + @Override + protected JdbcDatabase getDatabase() { + return database; + } + + @Override + protected DataType getStructType() { + return JSONB_TYPE; + } + + @Override + protected JdbcSqlGenerator getSqlGenerator() { + return new PostgresSqlGenerator(new PostgresSQLNameTransformer()); + } + + @Override + protected DestinationHandler getDestinationHandler() { + return new JdbcDestinationHandler(databaseName, database); + } + + @Override + protected SQLDialect getSqlDialect() { + return SQLDialect.POSTGRES; + } + + @Override + protected Field toJsonValue(final String valueAsString) { + return DSL.cast(DSL.val(valueAsString), JSONB_TYPE); + } + + @Test + @Override + public void testCreateTableIncremental() throws Exception { + final Sql sql = generator.createTable(incrementalDedupStream, "", false); + destinationHandler.execute(sql); + + final Optional existingTable = destinationHandler.findExistingTable(incrementalDedupStream.id()); + + assertTrue(existingTable.isPresent()); + assertAll( + () -> assertEquals("varchar", existingTable.get().columns().get("_airbyte_raw_id").type()), + () -> assertEquals("timestamptz", existingTable.get().columns().get("_airbyte_extracted_at").type()), + () -> assertEquals("jsonb", existingTable.get().columns().get("_airbyte_meta").type()), + () -> assertEquals("int8", existingTable.get().columns().get("id1").type()), + () -> assertEquals("int8", existingTable.get().columns().get("id2").type()), + () -> assertEquals("timestamptz", existingTable.get().columns().get("updated_at").type()), + () -> assertEquals("jsonb", existingTable.get().columns().get("struct").type()), + () -> assertEquals("jsonb", existingTable.get().columns().get("array").type()), + () -> assertEquals("varchar", existingTable.get().columns().get("string").type()), + () -> assertEquals("numeric", existingTable.get().columns().get("number").type()), + () -> assertEquals("int8", existingTable.get().columns().get("integer").type()), + () -> assertEquals("bool", existingTable.get().columns().get("boolean").type()), + () -> assertEquals("timestamptz", existingTable.get().columns().get("timestamp_with_timezone").type()), + () -> assertEquals("timestamp", existingTable.get().columns().get("timestamp_without_timezone").type()), + () -> assertEquals("timetz", existingTable.get().columns().get("time_with_timezone").type()), + () -> assertEquals("time", existingTable.get().columns().get("time_without_timezone").type()), + () -> assertEquals("date", existingTable.get().columns().get("date").type()), + () -> assertEquals("jsonb", existingTable.get().columns().get("unknown").type())); + // TODO assert on table indexing, etc. + } + +} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresTypingDedupingTest.java b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresTypingDedupingTest.java new file mode 100644 index 000000000000..dbcb13a67781 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/typing_deduping/PostgresTypingDedupingTest.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.postgres.typing_deduping; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.JdbcCompatibleSourceOperations; +import io.airbyte.cdk.integrations.standardtest.destination.typing_deduping.JdbcTypingDedupingTest; +import io.airbyte.integrations.base.destination.typing_deduping.SqlGenerator; +import io.airbyte.integrations.destination.postgres.PostgresDestination; +import io.airbyte.integrations.destination.postgres.PostgresSQLNameTransformer; +import io.airbyte.integrations.destination.postgres.PostgresTestDatabase; +import javax.sql.DataSource; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; + +public class PostgresTypingDedupingTest extends JdbcTypingDedupingTest { + + protected static PostgresTestDatabase testContainer; + + @BeforeAll + public static void setupPostgres() { + testContainer = PostgresTestDatabase.in(PostgresTestDatabase.BaseImage.POSTGRES_13); + } + + @AfterAll + public static void teardownPostgres() { + testContainer.close(); + } + + @Override + protected ObjectNode getBaseConfig() { + final ObjectNode config = (ObjectNode) testContainer.configBuilder() + .with("schema", "public") + .withDatabase() + .withResolvedHostAndPort() + .withCredentials() + .withoutSsl() + .build(); + return config.put("use_1s1t_format", true); + } + + @Override + protected DataSource getDataSource(final JsonNode config) { + // Intentionally ignore the config and rebuild it. + // The config param has the resolved (i.e. in-docker) host/port. + // We need the unresolved host/port since the test wrapper code is running from the docker host + // rather than in a container. + return new PostgresDestination().getDataSource(testContainer.configBuilder() + .with("schema", "public") + .withDatabase() + .withHostAndPort() + .withCredentials() + .withoutSsl() + .build()); + } + + @Override + protected String getImageName() { + return "airbyte/destination-postgres:dev"; + } + + @Override + protected SqlGenerator getSqlGenerator() { + return new PostgresSqlGenerator(new PostgresSQLNameTransformer()); + } + + @Override + protected JdbcCompatibleSourceOperations getSourceOperations() { + return new PostgresSqlGeneratorIntegrationTest.PostgresSourceOperations(); + } + +} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl new file mode 100644 index 000000000000..9f11b2293a95 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "old_cursor": 1, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl new file mode 100644 index 000000000000..7f75f0f804e2 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl @@ -0,0 +1,4 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl new file mode 100644 index 000000000000..c805113dc6c2 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl @@ -0,0 +1,4 @@ +// Keep the Alice record with more recent updated_at +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl new file mode 100644 index 000000000000..b2bf47df66c1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl @@ -0,0 +1 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00.000000Z", "name": "Someone completely different"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl new file mode 100644 index 000000000000..8aa852183061 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl @@ -0,0 +1,5 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +// Invalid columns are nulled out (i.e. SQL null, not JSON null) +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..80fac124d28d --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl @@ -0,0 +1,5 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +// Invalid data is still allowed in the raw table. +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl new file mode 100644 index 000000000000..b489accda1bb --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl @@ -0,0 +1 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different"}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl new file mode 100644 index 000000000000..c26d4a49aacd --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +// Charlie wasn't reemitted with updated_at, so it still has a null cursor +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl new file mode 100644 index 000000000000..03f28e155af5 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl @@ -0,0 +1,7 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl new file mode 100644 index 000000000000..6e9258bab255 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl @@ -0,0 +1,8 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"} + +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000Z"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl new file mode 100644 index 000000000000..9d1f1499469f --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000Z"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl new file mode 100644 index 000000000000..33bc3280be27 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl new file mode 100644 index 000000000000..13c59b2f9912 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +// Delete Bob, keep Charlie +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl new file mode 100644 index 000000000000..53c304c89d31 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl @@ -0,0 +1 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2001-01-02T00:00:00.000000Z", "name": "Someone completely different v2"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..32a7e57b1c14 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl @@ -0,0 +1,9 @@ +// We keep the records from the first sync +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} +// And append the records from the second sync +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl new file mode 100644 index 000000000000..88b8ee7746c1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl @@ -0,0 +1,2 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different v2"}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl new file mode 100644 index 000000000000..76d0442ebe79 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl @@ -0,0 +1,8 @@ +{"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "unknown": null, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "unknown": null, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": ["Problem with `struct`", "Problem with `array`", "Problem with `number`", "Problem with `integer`", "Problem with `boolean`","Problem with `timestamp_with_timezone`", "Problem with `timestamp_without_timezone`", "Problem with `time_with_timezone`","Problem with `time_without_timezone`", "Problem with `date`"]}} +// Note that for numbers where we parse the value to JSON (struct, array, unknown) we lose precision. +// But for numbers where we create a NUMBER column, we do not lose precision (see the `number` column). +{"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 6, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "IamACaseSensitiveColumnName": "Case senstive value", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..6b99169ececf --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl @@ -0,0 +1,6 @@ +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": null, "struct": null, "string": null, "number": null, "integer": null, "boolean": null, "timestamp_with_timezone": null, "timestamp_without_timezone": null, "time_with_timezone": null, "time_without_timezone": null, "date": null, "unknown": null}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z"}} +{"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": null, "number": "foo", "integer": "bar", "boolean": "fizz", "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": "airbyte", "unknown": null}} +{"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fce", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 6, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "IamACaseSensitiveColumnName": "Case senstive value"}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_final.jsonl new file mode 100644 index 000000000000..5842f7b37e42 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_final.jsonl @@ -0,0 +1,2 @@ +{"_airbyte_raw_id": "80c99b54-54b4-43bd-b51b-1f67dafa2c52", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00.000000Z", "string": "Alice", "struct": {"city": "San Diego", "state": "CA"}, "integer": 84} +{"_airbyte_raw_id": "ad690bfb-c2c2-4172-bd73-a16c86ccbb67", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": ["Problem with `integer`"]}, "id1": 2, "id2": 100, "updated_at": "2023-01-01T03:00:00.000000Z", "string": "Bob"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..63569975abc2 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_raw_id": "d7b81af0-01da-4846-a650-cc398986bc99", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "string": "Alice", "struct": {"city": "San Francisco", "state": "CA"}, "integer": 42}} +{"_airbyte_raw_id": "80c99b54-54b4-43bd-b51b-1f67dafa2c52", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00Z", "string": "Alice", "struct": {"city": "San Diego", "state": "CA"}, "integer": 84}} +{"_airbyte_raw_id": "ad690bfb-c2c2-4172-bd73-a16c86ccbb67", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T03:00:00Z", "string": "Bob", "integer": "oops"}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_final.jsonl new file mode 100644 index 000000000000..edcc0cc462d6 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_final.jsonl @@ -0,0 +1,5 @@ +{"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "[\"I\", \"am\", \"an\", \"array\"]", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "{\"I\": \"am\", \"an\": \"object\"}", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "true", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "3.14", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "I am a valid json string", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..5c10203c7837 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_raw.jsonl @@ -0,0 +1,5 @@ +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": ["I", "am", "an", "array"], "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": {"I": "am", "an": "object"}, "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": true, "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": 3.14, "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "I am a valid json string", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_final.jsonl new file mode 100644 index 000000000000..4ecd95d83b63 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_final.jsonl @@ -0,0 +1 @@ +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..cd7c03aba677 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_raw.jsonl @@ -0,0 +1 @@ +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/reservedkeywords_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/reservedkeywords_expectedrecords_final.jsonl new file mode 100644 index 000000000000..b34ad054ab33 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/reservedkeywords_expectedrecords_final.jsonl @@ -0,0 +1 @@ +{"_airbyte_raw_id":"b2e0efc4-38a8-47ba-970c-8103f09f08d5","_airbyte_extracted_at":"2023-01-01T00:00:00.000000Z","_airbyte_meta":{"errors":[]}, "current_date": "foo", "join": "bar"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/timestampformats_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/timestampformats_expectedrecords_final.jsonl new file mode 100644 index 000000000000..78ded5f99d0e --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/timestampformats_expectedrecords_final.jsonl @@ -0,0 +1,16 @@ +// https://docs.aws.amazon.com/redshift/latest/dg/r_Datetime_types.html#r_Datetime_types-timetz +// TIME, TIMETZ, TIMESTAMP, TIMESTAMPTZ values are UTC in user tables. +// Note that redshift stores precision to microseconds. Java deserialization in tests preserves them only for non-zero values +// except for timestamp with time zone where Z is required at end for even zero values +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "time_with_timezone": "12:34:56Z"} +{"_airbyte_raw_id": "05028c5f-7813-4e9c-bd4b-387d1f8ba435", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T20:34:56.000000Z", "time_with_timezone": "12:34:56-08:00"} +{"_airbyte_raw_id": "95dfb0c6-6a67-4ba0-9935-643bebc90437", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T20:34:56.000000Z", "time_with_timezone": "12:34:56-08:00"} +{"_airbyte_raw_id": "f3d8abe2-bb0f-4caf-8ddc-0641df02f3a9", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T20:34:56.000000Z", "time_with_timezone": "12:34:56-08:00"} +{"_airbyte_raw_id": "a81ed40a-2a49-488d-9714-d53e8b052968", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T04:34:56.000000Z", "time_with_timezone": "12:34:56+08:00"} +{"_airbyte_raw_id": "c07763a0-89e6-4cb7-b7d0-7a34a7c9918a", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T04:34:56.000000Z", "time_with_timezone": "12:34:56+08:00"} +{"_airbyte_raw_id": "358d3b52-50ab-4e06-9094-039386f9bf0d", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T04:34:56.000000Z", "time_with_timezone": "12:34:56+08:00"} +{"_airbyte_raw_id": "db8200ac-b2b9-4b95-a053-8a0343042751", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.123000Z", "time_with_timezone": "12:34:56.123Z"} + +{"_airbyte_raw_id": "10ce5d93-6923-4217-a46f-103833837038", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_without_timezone": "2023-01-23T12:34:56", "time_without_timezone": "12:34:56", "date": "2023-01-23"} +// Bigquery returns 6 decimal places if there are any decimal places... but not for timestamp_with_timezone +{"_airbyte_raw_id": "a7a6e176-7464-4a0b-b55c-b4f936e8d5a1", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_without_timezone": "2023-01-23T12:34:56.123", "time_without_timezone": "12:34:56.123"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl new file mode 100644 index 000000000000..adfbd06d6a55 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl @@ -0,0 +1,9 @@ +// column renamings: +// * $starts_with_dollar_sign -> _starts_with_dollar_sign +// * includes"doublequote -> includes_doublequote +// * includes'singlequote -> includes_singlequote +// * includes`backtick -> includes_backtick +// * includes$$doubledollar -> includes__doubledollar +// * includes.period -> includes_period +// * endswithbackslash\ -> endswithbackslash_ +{"_airbyte_raw_id": "7e7330a1-42fb-41ec-a955-52f18bd61964", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00.000000Z", "_starts_with_dollar_sign": "foo", "includes_doublequote": "foo", "includes_singlequote": "foo", "includes_backtick": "foo", "includes_period": "foo", "includes__doubledollar": "foo", "endswithbackslash_": "foo"} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..2b602082a349 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_raw.jsonl @@ -0,0 +1 @@ +{"_airbyte_raw_id": "7e7330a1-42fb-41ec-a955-52f18bd61964", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00Z", "$starts_with_dollar_sign": "foo", "includes\"doublequote": "foo", "includes'singlequote": "foo", "includes`backtick": "foo", "includes.period": "foo", "includes$$doubledollar": "foo", "endswithbackslash\\": "foo"}} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java b/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java index 6d953bd462d7..84eb63509696 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java +++ b/airbyte-integrations/connectors/destination-postgres/src/test/java/io/airbyte/integrations/destination/postgres/PostgresDestinationTest.java @@ -10,16 +10,18 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.DestinationConfig; +import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer; +import io.airbyte.cdk.testutils.PostgreSQLContainerHelper; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; @@ -29,7 +31,7 @@ import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.CatalogHelpers; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.test.utils.PostgreSQLContainerHelper; +import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.HashMap; import java.util.List; @@ -240,20 +242,24 @@ public void testUserHasNoPermissionToDataBase() throws Exception { @Test void sanityTest() throws Exception { final Destination destination = new PostgresDestination(); - final AirbyteMessageConsumer consumer = destination.getConsumer(config, CATALOG, Destination::defaultOutputRecordCollector); + DestinationConfig.initialize(config); + final SerializedAirbyteMessageConsumer consumer = + destination.getSerializedMessageConsumer(config, CATALOG, Destination::defaultOutputRecordCollector); final List expectedRecords = getNRecords(10); consumer.start(); expectedRecords.forEach(m -> { try { - consumer.accept(m); + String message = Jsons.serialize(m); + consumer.accept(message, message.getBytes(StandardCharsets.UTF_8).length); } catch (final Exception e) { throw new RuntimeException(e); } }); - consumer.accept(new AirbyteMessage() + final String stateMessage = Jsons.serialize(new AirbyteMessage() .withType(Type.STATE) .withState(new AirbyteStateMessage().withData(Jsons.jsonNode(ImmutableMap.of(SCHEMA_NAME + "." + STREAM_NAME, 10))))); + consumer.accept(stateMessage, stateMessage.getBytes(StandardCharsets.UTF_8).length); consumer.close(); final JdbcDatabase database = getJdbcDatabaseFromConfig(getDataSourceFromConfig(config)); diff --git a/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/AbstractPostgresDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/AbstractPostgresDestinationAcceptanceTest.java new file mode 100644 index 000000000000..d5bb6d01fcb0 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/AbstractPostgresDestinationAcceptanceTest.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.postgres; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.commons.json.Jsons; +import java.util.List; +import java.util.stream.Collectors; + +public abstract class AbstractPostgresDestinationAcceptanceTest extends JdbcDestinationAcceptanceTest { + + public static final String DEFAULT_DEV_IMAGE = "airbyte/destination-postgres:dev"; + + private final StandardNameTransformer namingResolver = new StandardNameTransformer(); + + @Override + protected String getImageName() { + return DEFAULT_DEV_IMAGE; + } + + @Override + protected JsonNode getFailCheckConfig() throws Exception { + final JsonNode clone = Jsons.clone(getConfig()); + ((ObjectNode) clone).put("password", "wrong password"); + return clone; + } + + @Override + protected List retrieveNormalizedRecords(final TestDestinationEnv env, final String streamName, final String namespace) + throws Exception { + final String tableName = namingResolver.getIdentifier(streamName); + return retrieveRecordsFromTable(tableName, namespace); + } + + @Override + protected List retrieveRecords(final TestDestinationEnv env, + final String streamName, + final String namespace, + final JsonNode streamSchema) + throws Exception { + return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) + .stream() + .map(r -> r.get(JavaBaseConstants.COLUMN_NAME_DATA)) + .collect(Collectors.toList()); + } + + protected List retrieveRecordsFromTable(final String tableName, final String schemaName) throws Exception { + // TODO: Change emitted_at with DV2 + return getTestDb().query(ctx -> { + ctx.execute("set time zone 'UTC';"); + return ctx.fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) + .stream() + .map(this::getJsonFromRecord) + .collect(Collectors.toList()); + }); + } + + protected abstract PostgresTestDatabase getTestDb(); + + @Override + protected boolean implementsNamespaces() { + return true; + } + + @Override + protected TestDataComparator getTestDataComparator() { + return new PostgresTestDataComparator(); + } + + @Override + protected boolean supportBasicDataTypeTest() { + return true; + } + + @Override + protected boolean supportArrayDataTypeTest() { + return true; + } + + @Override + protected boolean supportObjectDataTypeTest() { + return true; + } + + @Override + protected boolean supportsInDestinationNormalization() { + return true; + } + +} diff --git a/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/PostgresContainerFactory.java b/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/PostgresContainerFactory.java new file mode 100644 index 000000000000..1a19f9eb50d5 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/PostgresContainerFactory.java @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.postgres; + +import io.airbyte.cdk.testutils.ContainerFactory; +import java.io.IOException; +import java.io.UncheckedIOException; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.MountableFile; + +/** + * TODO: This class is a copy from source-postgres:testFixtures. Eventually merge into a common + * fixtures module. + */ +public class PostgresContainerFactory implements ContainerFactory> { + + @Override + public PostgreSQLContainer createNewContainer(DockerImageName imageName) { + return new PostgreSQLContainer<>(imageName.asCompatibleSubstituteFor("postgres")); + + } + + @Override + public Class getContainerClass() { + return PostgreSQLContainer.class; + } + + /** + * Apply the postgresql.conf file that we've packaged as a resource. + */ + public void withConf(PostgreSQLContainer container) { + container + .withCopyFileToContainer( + MountableFile.forClasspathResource("postgresql.conf"), + "/etc/postgresql/postgresql.conf") + .withCommand("postgres -c config_file=/etc/postgresql/postgresql.conf"); + } + + /** + * Create a new network and bind it to the container. + */ + public void withNetwork(PostgreSQLContainer container) { + container.withNetwork(Network.newNetwork()); + } + + /** + * Configure postgres with wal_level=logical. + */ + public void withWalLevelLogical(PostgreSQLContainer container) { + container.withCommand("postgres -c wal_level=logical"); + } + + /** + * Generate SSL certificates and tell postgres to enable SSL and use them. + */ + public void withCert(PostgreSQLContainer container) { + container.start(); + String[] commands = { + "psql -U test -c \"CREATE USER postgres WITH PASSWORD 'postgres';\"", + "psql -U test -c \"GRANT CONNECT ON DATABASE \"test\" TO postgres;\"", + "psql -U test -c \"ALTER USER postgres WITH SUPERUSER;\"", + "openssl ecparam -name prime256v1 -genkey -noout -out ca.key", + "openssl req -new -x509 -sha256 -key ca.key -out ca.crt -subj \"/CN=127.0.0.1\"", + "openssl ecparam -name prime256v1 -genkey -noout -out server.key", + "openssl req -new -sha256 -key server.key -out server.csr -subj \"/CN=localhost\"", + "openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 365 -sha256", + "cp server.key /etc/ssl/private/", + "cp server.crt /etc/ssl/private/", + "cp ca.crt /etc/ssl/private/", + "chmod og-rwx /etc/ssl/private/server.* /etc/ssl/private/ca.*", + "chown postgres:postgres /etc/ssl/private/server.crt /etc/ssl/private/server.key /etc/ssl/private/ca.crt", + "echo \"ssl = on\" >> /var/lib/postgresql/data/postgresql.conf", + "echo \"ssl_cert_file = '/etc/ssl/private/server.crt'\" >> /var/lib/postgresql/data/postgresql.conf", + "echo \"ssl_key_file = '/etc/ssl/private/server.key'\" >> /var/lib/postgresql/data/postgresql.conf", + "echo \"ssl_ca_file = '/etc/ssl/private/ca.crt'\" >> /var/lib/postgresql/data/postgresql.conf", + "mkdir root/.postgresql", + "echo \"hostssl all all 127.0.0.1/32 cert clientcert=verify-full\" >> /var/lib/postgresql/data/pg_hba.conf", + "openssl ecparam -name prime256v1 -genkey -noout -out client.key", + "openssl req -new -sha256 -key client.key -out client.csr -subj \"/CN=postgres\"", + "openssl x509 -req -in client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out client.crt -days 365 -sha256", + "cp client.crt ~/.postgresql/postgresql.crt", + "cp client.key ~/.postgresql/postgresql.key", + "chmod 0600 ~/.postgresql/postgresql.crt ~/.postgresql/postgresql.key", + "cp ca.crt root/.postgresql/ca.crt", + "chown postgres:postgres ~/.postgresql/ca.crt", + "psql -U test -c \"SELECT pg_reload_conf();\"", + }; + for (String cmd : commands) { + try { + container.execInContainer("su", "-c", cmd); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + + /** + * Tell postgres to enable SSL. + */ + public void withSSL(PostgreSQLContainer container) { + container.withCommand("postgres " + + "-c ssl=on " + + "-c ssl_cert_file=/var/lib/postgresql/server.crt " + + "-c ssl_key_file=/var/lib/postgresql/server.key"); + } + + /** + * Configure postgres with client_encoding=sql_ascii. + */ + public void withASCII(PostgreSQLContainer container) { + container.withCommand("postgres -c client_encoding=sql_ascii"); + } + +} diff --git a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresTestDataComparator.java b/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/PostgresTestDataComparator.java similarity index 93% rename from airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresTestDataComparator.java rename to airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/PostgresTestDataComparator.java index 9093fc55f636..1775a17139b5 100644 --- a/airbyte-integrations/connectors/destination-postgres/src/test-integration/java/io/airbyte/integrations/destination/postgres/PostgresTestDataComparator.java +++ b/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/PostgresTestDataComparator.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.postgres; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneOffset; diff --git a/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/PostgresTestDatabase.java b/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/PostgresTestDatabase.java new file mode 100644 index 000000000000..31fb23b9fa79 --- /dev/null +++ b/airbyte-integrations/connectors/destination-postgres/src/testFixtures/java/io/airbyte/integrations/destination/postgres/PostgresTestDatabase.java @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.postgres; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.TestDatabase; +import io.airbyte.commons.json.Jsons; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.stream.Stream; +import org.jooq.SQLDialect; +import org.testcontainers.containers.PostgreSQLContainer; + +/** + * TODO: This class is a copy from source-postgres:testFixtures. Eventually merge into a common + * fixtures module. + */ +public class PostgresTestDatabase extends + TestDatabase, PostgresTestDatabase, PostgresTestDatabase.PostgresConfigBuilder> { + + public enum BaseImage { + + POSTGRES_16("postgres:16-bullseye"), + POSTGRES_12("postgres:12-bullseye"), + POSTGRES_13("postgres:13-alpine"), + POSTGRES_9("postgres:9-alpine"), + POSTGRES_SSL_DEV("marcosmarxm/postgres-ssl:dev"); + + private final String reference; + + private BaseImage(String reference) { + this.reference = reference; + }; + + } + + public static enum ContainerModifier { + + ASCII("withASCII"), + CONF("withConf"), + NETWORK("withNetwork"), + SSL("withSSL"), + WAL_LEVEL_LOGICAL("withWalLevelLogical"), + CERT("withCert"), + ; + + private String methodName; + + private ContainerModifier(String methodName) { + this.methodName = methodName; + } + + } + + static public PostgresTestDatabase in(BaseImage baseImage, ContainerModifier... modifiers) { + String[] methodNames = Stream.of(modifiers).map(im -> im.methodName).toList().toArray(new String[0]); + final var container = new PostgresContainerFactory().shared(baseImage.reference, methodNames); + return new PostgresTestDatabase(container).initialized(); + } + + public PostgresTestDatabase(PostgreSQLContainer container) { + super(container); + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.of(psqlCmd(Stream.of( + String.format("CREATE DATABASE %s", getDatabaseName()), + String.format("CREATE USER %s PASSWORD '%s'", getUserName(), getPassword()), + String.format("GRANT ALL PRIVILEGES ON DATABASE %s TO %s", getDatabaseName(), getUserName()), + String.format("ALTER USER %s WITH SUPERUSER", getUserName())))); + } + + /** + * Close resources held by this instance. This deliberately avoids dropping the database, which is + * really expensive in Postgres. This is because a DROP DATABASE in Postgres triggers a CHECKPOINT. + * Call {@link #dropDatabaseAndUser} to explicitly drop the database and the user. + */ + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + /** + * Drop the database owned by this instance. + */ + public void dropDatabaseAndUser() { + execInContainer(psqlCmd(Stream.of( + String.format("DROP DATABASE %s", getDatabaseName()), + String.format("DROP OWNED BY %s", getUserName()), + String.format("DROP USER %s", getUserName())))); + } + + public Stream psqlCmd(Stream sql) { + return Stream.concat( + Stream.of("psql", + "-d", getContainer().getDatabaseName(), + "-U", getContainer().getUsername(), + "-v", "ON_ERROR_STOP=1", + "-a"), + sql.flatMap(stmt -> Stream.of("-c", stmt))); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return DatabaseDriver.POSTGRESQL; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.POSTGRES; + } + + private Certificates cachedCerts; + + public synchronized Certificates getCertificates() { + if (cachedCerts == null) { + final String caCert, clientKey, clientCert; + try { + caCert = getContainer().execInContainer("su", "-c", "cat ca.crt").getStdout().trim(); + clientKey = getContainer().execInContainer("su", "-c", "cat client.key").getStdout().trim(); + clientCert = getContainer().execInContainer("su", "-c", "cat client.crt").getStdout().trim(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + cachedCerts = new Certificates(caCert, clientCert, clientKey); + } + return cachedCerts; + } + + public record Certificates(String caCertificate, String clientCertificate, String clientKey) {} + + @Override + public PostgresConfigBuilder configBuilder() { + return new PostgresConfigBuilder(this); + } + + public String getReplicationSlotName() { + return withNamespace("debezium_slot"); + } + + public String getPublicationName() { + return withNamespace("publication"); + } + + public PostgresTestDatabase withReplicationSlot() { + return this + .with("SELECT pg_create_logical_replication_slot('%s', 'pgoutput');", getReplicationSlotName()) + .onClose("SELECT pg_drop_replication_slot('%s');", getReplicationSlotName()); + } + + public PostgresTestDatabase withPublicationForAllTables() { + return this + .with("CREATE PUBLICATION %s FOR ALL TABLES;", getPublicationName()) + .onClose("DROP PUBLICATION %s CASCADE;", getPublicationName()); + } + + static public class PostgresConfigBuilder extends ConfigBuilder { + + protected PostgresConfigBuilder(PostgresTestDatabase testdb) { + super(testdb); + } + + public PostgresConfigBuilder withSchemas(String... schemas) { + return with(JdbcUtils.SCHEMAS_KEY, List.of(schemas)); + } + + public PostgresConfigBuilder withStandardReplication() { + return with("replication_method", ImmutableMap.builder().put("method", "Standard").build()); + } + + public PostgresConfigBuilder withCdcReplication() { + return withCdcReplication("While reading Data"); + } + + public PostgresConfigBuilder withCdcReplication(String LsnCommitBehaviour) { + return this + .with("is_test", true) + .with("replication_method", Jsons.jsonNode(ImmutableMap.builder() + .put("method", "CDC") + .put("replication_slot", testDatabase.getReplicationSlotName()) + .put("publication", testDatabase.getPublicationName()) + .put("initial_waiting_seconds", DEFAULT_CDC_REPLICATION_INITIAL_WAIT.getSeconds()) + .put("lsn_commit_behaviour", LsnCommitBehaviour) + .build())); + } + + public PostgresConfigBuilder withXminReplication() { + return this.with("replication_method", Jsons.jsonNode(ImmutableMap.builder().put("method", "Xmin").build())); + } + + } + +} diff --git a/airbyte-integrations/connectors/destination-pubsub/.dockerignore b/airbyte-integrations/connectors/destination-pubsub/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-pubsub/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-pubsub/Dockerfile b/airbyte-integrations/connectors/destination-pubsub/Dockerfile deleted file mode 100644 index eb148e9d0996..000000000000 --- a/airbyte-integrations/connectors/destination-pubsub/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-pubsub - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-pubsub - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.2.0 -LABEL io.airbyte.name=airbyte/destination-pubsub diff --git a/airbyte-integrations/connectors/destination-pubsub/build.gradle b/airbyte-integrations/connectors/destination-pubsub/build.gradle index 31f917dd0645..f0c8f25ea7f4 100644 --- a/airbyte-integrations/connectors/destination-pubsub/build.gradle +++ b/airbyte-integrations/connectors/destination-pubsub/build.gradle @@ -1,9 +1,23 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.pubsub.PubsubDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] @@ -12,13 +26,4 @@ application { dependencies { implementation group: 'com.google.cloud', name: 'google-cloud-pubsub', version: '1.114.7' - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - - testImplementation project(':airbyte-integrations:bases:standard-destination-test') - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-pubsub') } diff --git a/airbyte-integrations/connectors/destination-pubsub/src/main/java/io/airbyte/integrations/destination/pubsub/PubsubConsumer.java b/airbyte-integrations/connectors/destination-pubsub/src/main/java/io/airbyte/integrations/destination/pubsub/PubsubConsumer.java index 1be0c62ba4ff..48753e5cb7c7 100644 --- a/airbyte-integrations/connectors/destination-pubsub/src/main/java/io/airbyte/integrations/destination/pubsub/PubsubConsumer.java +++ b/airbyte-integrations/connectors/destination-pubsub/src/main/java/io/airbyte/integrations/destination/pubsub/PubsubConsumer.java @@ -12,9 +12,9 @@ import com.google.common.collect.Maps; import com.google.protobuf.ByteString; import com.google.pubsub.v1.PubsubMessage; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; -import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; diff --git a/airbyte-integrations/connectors/destination-pubsub/src/main/java/io/airbyte/integrations/destination/pubsub/PubsubDestination.java b/airbyte-integrations/connectors/destination-pubsub/src/main/java/io/airbyte/integrations/destination/pubsub/PubsubDestination.java index 4d20bda4dee1..b7dd40fa613c 100644 --- a/airbyte-integrations/connectors/destination-pubsub/src/main/java/io/airbyte/integrations/destination/pubsub/PubsubDestination.java +++ b/airbyte-integrations/connectors/destination-pubsub/src/main/java/io/airbyte/integrations/destination/pubsub/PubsubDestination.java @@ -11,10 +11,10 @@ import com.google.common.base.Preconditions; import com.google.iam.v1.TestIamPermissionsRequest; import com.google.iam.v1.TestIamPermissionsResponse; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-pubsub/src/test-integration/java/io/airbyte/integrations/destination/pubsub/PubsubDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-pubsub/src/test-integration/java/io/airbyte/integrations/destination/pubsub/PubsubDestinationAcceptanceTest.java index 9d3a7b8d774c..9be4b3adb2a9 100644 --- a/airbyte-integrations/connectors/destination-pubsub/src/test-integration/java/io/airbyte/integrations/destination/pubsub/PubsubDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-pubsub/src/test-integration/java/io/airbyte/integrations/destination/pubsub/PubsubDestinationAcceptanceTest.java @@ -35,12 +35,12 @@ import com.google.pubsub.v1.Subscription; import com.google.pubsub.v1.Topic; import com.google.pubsub.v1.TopicName; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import java.io.ByteArrayInputStream; import java.io.IOException; diff --git a/airbyte-integrations/connectors/destination-pubsub/src/test/java/io/airbyte/integration/destination/pubsub/PubsubConsumerTest.java b/airbyte-integrations/connectors/destination-pubsub/src/test/java/io/airbyte/integration/destination/pubsub/PubsubConsumerTest.java index 13b542d07c19..e1e3e8d5a858 100644 --- a/airbyte-integrations/connectors/destination-pubsub/src/test/java/io/airbyte/integration/destination/pubsub/PubsubConsumerTest.java +++ b/airbyte-integrations/connectors/destination-pubsub/src/test/java/io/airbyte/integration/destination/pubsub/PubsubConsumerTest.java @@ -4,10 +4,10 @@ package io.airbyte.integration.destination.pubsub; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.standardtest.destination.PerStreamStateMessageTest; import io.airbyte.integrations.destination.pubsub.PubsubConsumer; import io.airbyte.integrations.destination.pubsub.PubsubDestinationConfig; -import io.airbyte.integrations.standardtest.destination.PerStreamStateMessageTest; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import java.util.function.Consumer; diff --git a/airbyte-integrations/connectors/destination-pulsar/.dockerignore b/airbyte-integrations/connectors/destination-pulsar/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-pulsar/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-pulsar/Dockerfile b/airbyte-integrations/connectors/destination-pulsar/Dockerfile deleted file mode 100644 index 0b7107f149d1..000000000000 --- a/airbyte-integrations/connectors/destination-pulsar/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-pulsar - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-pulsar - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.3 -LABEL io.airbyte.name=airbyte/destination-pulsar diff --git a/airbyte-integrations/connectors/destination-pulsar/README.md b/airbyte-integrations/connectors/destination-pulsar/README.md index a291e2c6680d..f2c554ba1f67 100644 --- a/airbyte-integrations/connectors/destination-pulsar/README.md +++ b/airbyte-integrations/connectors/destination-pulsar/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-pulsar:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-pulsar:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-pulsar:dev`. the Dockerfile. #### Run @@ -61,8 +62,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-pulsar test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/pulsar.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-pulsar/build.gradle b/airbyte-integrations/connectors/destination-pulsar/build.gradle index 4b2d6f81a75c..fad585d1ed6b 100644 --- a/airbyte-integrations/connectors/destination-pulsar/build.gradle +++ b/airbyte-integrations/connectors/destination-pulsar/build.gradle @@ -1,26 +1,31 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.pulsar.PulsarDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation 'org.apache.pulsar:pulsar-client:2.8.1' - testImplementation libs.connectors.testcontainers.pulsar - testImplementation project(':airbyte-integrations:bases:standard-destination-test') - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-pulsar') + testImplementation libs.testcontainers.pulsar } diff --git a/airbyte-integrations/connectors/destination-pulsar/src/main/java/io/airbyte/integrations/destination/pulsar/PulsarDestination.java b/airbyte-integrations/connectors/destination-pulsar/src/main/java/io/airbyte/integrations/destination/pulsar/PulsarDestination.java index db83355e4912..79e5e7239bdc 100644 --- a/airbyte-integrations/connectors/destination-pulsar/src/main/java/io/airbyte/integrations/destination/pulsar/PulsarDestination.java +++ b/airbyte-integrations/connectors/destination-pulsar/src/main/java/io/airbyte/integrations/destination/pulsar/PulsarDestination.java @@ -6,13 +6,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-pulsar/src/main/java/io/airbyte/integrations/destination/pulsar/PulsarRecordConsumer.java b/airbyte-integrations/connectors/destination-pulsar/src/main/java/io/airbyte/integrations/destination/pulsar/PulsarRecordConsumer.java index 716f68e92da8..1ccd22c1437a 100644 --- a/airbyte-integrations/connectors/destination-pulsar/src/main/java/io/airbyte/integrations/destination/pulsar/PulsarRecordConsumer.java +++ b/airbyte-integrations/connectors/destination-pulsar/src/main/java/io/airbyte/integrations/destination/pulsar/PulsarRecordConsumer.java @@ -4,9 +4,9 @@ package io.airbyte.integrations.destination.pulsar; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; import io.airbyte.commons.lang.Exceptions; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; -import io.airbyte.integrations.destination.NamingConventionTransformer; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; diff --git a/airbyte-integrations/connectors/destination-pulsar/src/test-integration/java/io/airbyte/integrations/destination/pulsar/PulsarDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-pulsar/src/test-integration/java/io/airbyte/integrations/destination/pulsar/PulsarDestinationAcceptanceTest.java index 76a9fa933dfd..8b2dcd685acc 100644 --- a/airbyte-integrations/connectors/destination-pulsar/src/test-integration/java/io/airbyte/integrations/destination/pulsar/PulsarDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-pulsar/src/test-integration/java/io/airbyte/integrations/destination/pulsar/PulsarDestinationAcceptanceTest.java @@ -10,13 +10,13 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Streams; import com.google.common.net.InetAddresses; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.lang.Exceptions; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.io.IOException; import java.net.InetAddress; import java.net.NetworkInterface; diff --git a/airbyte-integrations/connectors/destination-pulsar/src/test/java/io/airbyte/integrations/destination/pulsar/PulsarRecordConsumerTest.java b/airbyte-integrations/connectors/destination-pulsar/src/test/java/io/airbyte/integrations/destination/pulsar/PulsarRecordConsumerTest.java index ea79e9082a8e..07750e293d99 100644 --- a/airbyte-integrations/connectors/destination-pulsar/src/test/java/io/airbyte/integrations/destination/pulsar/PulsarRecordConsumerTest.java +++ b/airbyte-integrations/connectors/destination-pulsar/src/test/java/io/airbyte/integrations/destination/pulsar/PulsarRecordConsumerTest.java @@ -13,10 +13,10 @@ import com.google.common.collect.Sets; import com.google.common.collect.Streams; import com.google.common.net.InetAddresses; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.PerStreamStateMessageTest; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.PerStreamStateMessageTest; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-qdrant/.dockerignore b/airbyte-integrations/connectors/destination-qdrant/.dockerignore new file mode 100644 index 000000000000..b423c7670a8c --- /dev/null +++ b/airbyte-integrations/connectors/destination-qdrant/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!destination_qdrant +!airbyte-cdk +!setup.py diff --git a/airbyte-integrations/connectors/destination-qdrant/.gitignore b/airbyte-integrations/connectors/destination-qdrant/.gitignore new file mode 100644 index 000000000000..ef80b1792edf --- /dev/null +++ b/airbyte-integrations/connectors/destination-qdrant/.gitignore @@ -0,0 +1 @@ +airbyte-cdk \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-qdrant/Dockerfile b/airbyte-integrations/connectors/destination-qdrant/Dockerfile new file mode 100644 index 000000000000..bbbdae6005c1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-qdrant/Dockerfile @@ -0,0 +1,45 @@ +FROM python:3.10-slim as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +RUN apt-get update \ + && pip install --upgrade pip \ + && apt-get install -y build-essential cmake g++ libffi-dev libstdc++6 + +# upgrade pip to the latest version +COPY setup.py ./ + +RUN pip install --upgrade pip + +# This is required because the current connector dependency is not compatible with the CDK version +# An older CDK version will be used, which depends on pyYAML 5.4, for which we need to pin Cython to <3.0 +# As of today the CDK version that satisfies the main dependency requirements, is 0.1.80 ... +RUN pip install --prefix=/install "Cython<3.0" "pyyaml~=5.4" --no-build-isolation + +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apt-get install bash + +# copy payload code only +COPY main.py ./ +COPY destination_qdrant ./destination_qdrant + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.0.10 +LABEL io.airbyte.name=airbyte/destination-qdrant diff --git a/airbyte-integrations/connectors/destination-qdrant/README.md b/airbyte-integrations/connectors/destination-qdrant/README.md new file mode 100644 index 000000000000..462eea0f76b7 --- /dev/null +++ b/airbyte-integrations/connectors/destination-qdrant/README.md @@ -0,0 +1,99 @@ +# Qdrant Destination + +This is the repository for the Qdrant destination connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/destinations/qdrant). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.10.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/destinations/qdrant) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_qdrant/spec.json` file. +Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `destination qdrant test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + + +#### Build +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=destination-qdrant build +``` + +An image will be built with the tag `airbyte/destination-qdrant:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/destination-qdrant:dev . +``` + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/destination-qdrant:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-qdrant:dev check --config /secrets/config.json +# messages.jsonl is a file containing line-separated JSON representing AirbyteMessages +cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-qdrant:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=destination-qdrant test +``` + +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-qdrant test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/qdrant.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/__init__.py b/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/__init__.py new file mode 100644 index 000000000000..b2fe4e95b0e2 --- /dev/null +++ b/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .destination import DestinationQdrant + +__all__ = ["DestinationQdrant"] diff --git a/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/config.py b/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/config.py new file mode 100644 index 000000000000..22de63c48f2b --- /dev/null +++ b/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/config.py @@ -0,0 +1,52 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from typing import Literal, Union + +from airbyte_cdk.destinations.vector_db_based.config import VectorDBConfigModel +from pydantic import BaseModel, Field + + +class NoAuth(BaseModel): + mode: Literal["no_auth"] = Field("no_auth", const=True) + + +class ApiKeyAuth(BaseModel): + mode: Literal["api_key_auth"] = Field("api_key_auth", const=True) + api_key: str = Field(..., title="API Key", description="API Key for the Qdrant instance", airbyte_secret=True) + + +class QdrantIndexingConfigModel(BaseModel): + url: str = Field(..., title="Public Endpoint", description="Public Endpoint of the Qdrant cluser", order=0) + auth_method: Union[ApiKeyAuth, NoAuth] = Field( + default="api_key_auth", + title="Authentication Method", + description="Method to authenticate with the Qdrant Instance", + discriminator="mode", + type="object", + order=1, + ) + prefer_grpc: bool = Field( + title="Prefer gRPC", description="Whether to prefer gRPC over HTTP. Set to true for Qdrant cloud clusters", default=True + ) + collection: str = Field(..., title="Collection Name", description="The collection to load data into", order=2) + distance_metric: str = Field( + default="cos", + title="Distance Metric", + enum=["dot", "cos", "euc"], + description="The Distance metric used to measure similarities among vectors. This field is only used if the collection defined in the does not exist yet and is created automatically by the connector.", + ) + text_field: str = Field(title="Text Field", description="The field in the payload that contains the embedded text", default="text") + + class Config: + title = "Indexing" + schema_extra = { + "group": "Indexing", + "description": "Indexing configuration", + } + + +class ConfigModel(VectorDBConfigModel): + indexing: QdrantIndexingConfigModel diff --git a/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/destination.py b/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/destination.py new file mode 100644 index 000000000000..371a03285fb0 --- /dev/null +++ b/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/destination.py @@ -0,0 +1,57 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Iterable, Mapping + +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.destinations import Destination +from airbyte_cdk.destinations.vector_db_based.document_processor import DocumentProcessor +from airbyte_cdk.destinations.vector_db_based.embedder import Embedder, create_from_config +from airbyte_cdk.destinations.vector_db_based.indexer import Indexer +from airbyte_cdk.destinations.vector_db_based.writer import Writer +from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, ConnectorSpecification, Status +from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode +from destination_qdrant.config import ConfigModel +from destination_qdrant.indexer import QdrantIndexer + +BATCH_SIZE = 256 + + +class DestinationQdrant(Destination): + indexer: Indexer + embedder: Embedder + + def _init_indexer(self, config: ConfigModel): + self.embedder = create_from_config(config.embedding, config.processing) + self.indexer = QdrantIndexer(config.indexing, self.embedder.embedding_dimensions) + + def write( + self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] + ) -> Iterable[AirbyteMessage]: + + config_model = ConfigModel.parse_obj(config) + self._init_indexer(config_model) + writer = Writer( + config_model.processing, self.indexer, self.embedder, batch_size=BATCH_SIZE, omit_raw_text=config_model.omit_raw_text + ) + yield from writer.write(configured_catalog, input_messages) + + def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: + parsed_config = ConfigModel.parse_obj(config) + self._init_indexer(parsed_config) + checks = [self.embedder.check(), self.indexer.check(), DocumentProcessor.check_config(parsed_config.processing)] + errors = [error for error in checks if error is not None] + if len(errors) > 0: + return AirbyteConnectionStatus(status=Status.FAILED, message="\n".join(errors)) + else: + return AirbyteConnectionStatus(status=Status.SUCCEEDED) + + def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: + + return ConnectorSpecification( + documentationUrl="https://docs.airbyte.com/integrations/destinations/qdrant", + supportsIncremental=True, + supported_destination_sync_modes=[DestinationSyncMode.overwrite, DestinationSyncMode.append, DestinationSyncMode.append_dedup], + connectionSpecification=ConfigModel.schema(), # type: ignore[attr-defined] + ) diff --git a/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/indexer.py b/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/indexer.py new file mode 100644 index 000000000000..1d78d8730e7a --- /dev/null +++ b/airbyte-integrations/connectors/destination-qdrant/destination_qdrant/indexer.py @@ -0,0 +1,141 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import uuid +from typing import List, Optional + +from airbyte_cdk.destinations.vector_db_based.document_processor import METADATA_RECORD_ID_FIELD, METADATA_STREAM_FIELD +from airbyte_cdk.destinations.vector_db_based.indexer import Indexer +from airbyte_cdk.destinations.vector_db_based.utils import create_stream_identifier, format_exception +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, ConfiguredAirbyteCatalog, Level, Type +from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode +from destination_qdrant.config import QdrantIndexingConfigModel +from qdrant_client import QdrantClient, models +from qdrant_client.conversions.common_types import PointsSelector +from qdrant_client.models import Distance, PayloadSchemaType, VectorParams + +DISTANCE_METRIC_MAP = { + "dot": Distance.DOT, + "cos": Distance.COSINE, + "euc": Distance.EUCLID, +} + + +class QdrantIndexer(Indexer): + config: QdrantIndexingConfigModel + + def __init__(self, config: QdrantIndexingConfigModel, embedding_dimensions: int): + super().__init__(config) + self.embedding_dimensions = embedding_dimensions + + def check(self) -> Optional[str]: + auth_method_mode = self.config.auth_method.mode + if auth_method_mode == "api_key_auth" and not self.config.url.startswith("https://"): + return "Host must start with https://" + + try: + self._create_client() + + if not self._client: + return "Qdrant client is not alive." + + available_collections = [collection.name for collection in self._client.get_collections().collections] + distance_metric = DISTANCE_METRIC_MAP[self.config.distance_metric] + + if self.config.collection in available_collections: + collection_info = self._client.get_collection(collection_name=self.config.collection) + assert ( + collection_info.config.params.vectors.size == self.embedding_dimensions + ), "The collection's vector's size must match the embedding dimensions" + assert ( + collection_info.config.params.vectors.distance == distance_metric + ), "The colection's vector's distance metric must match the selected distance metric option" + else: + self._client.recreate_collection( + collection_name=self.config.collection, + vectors_config=VectorParams(size=self.embedding_dimensions, distance=distance_metric), + ) + + except Exception as e: + return format_exception(e) + finally: + if self._client: + self._client.close() + + def pre_sync(self, catalog: ConfiguredAirbyteCatalog) -> None: + self._create_client() + streams_to_overwrite = [ + create_stream_identifier(stream.stream) + for stream in catalog.streams + if stream.destination_sync_mode == DestinationSyncMode.overwrite + ] + if streams_to_overwrite: + self._delete_for_filter( + models.FilterSelector( + filter=models.Filter( + should=[ + models.FieldCondition(key=METADATA_STREAM_FIELD, match=models.MatchValue(value=stream)) + for stream in streams_to_overwrite + ] + ) + ) + ) + for field in [METADATA_RECORD_ID_FIELD, METADATA_STREAM_FIELD]: + self._client.create_payload_index( + collection_name=self.config.collection, field_name=field, field_schema=PayloadSchemaType.KEYWORD + ) + + def delete(self, delete_ids, namespace, stream): + if len(delete_ids) > 0: + self._delete_for_filter( + models.FilterSelector( + filter=models.Filter( + should=[ + models.FieldCondition(key=METADATA_RECORD_ID_FIELD, match=models.MatchValue(value=_id)) for _id in delete_ids + ] + ) + ) + ) + + def index(self, document_chunks, namespace, stream): + entities = [] + for i in range(len(document_chunks)): + chunk = document_chunks[i] + payload = chunk.metadata + if chunk.page_content is not None: + payload[self.config.text_field] = chunk.page_content + entities.append( + models.Record( + id=str(uuid.uuid4()), + payload=payload, + vector=chunk.embedding, + ) + ) + self._client.upload_records(collection_name=self.config.collection, records=entities) + + def post_sync(self) -> List[AirbyteMessage]: + try: + self._client.close() + return [ + AirbyteMessage( + type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message="Qdrant Database Client has been closed successfully") + ) + ] + except Exception as e: + return [AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.ERROR, message=format_exception(e)))] + + def _create_client(self): + auth_method = self.config.auth_method + url = self.config.url + prefer_grpc = self.config.prefer_grpc + + if auth_method.mode == "no_auth": + self._client = QdrantClient(url=url, prefer_grpc=prefer_grpc) + elif auth_method.mode == "api_key_auth": + api_key = auth_method.api_key + self._client = QdrantClient(url=url, prefer_grpc=prefer_grpc, api_key=api_key) + + def _delete_for_filter(self, selector: PointsSelector) -> None: + self._client.delete(collection_name=self.config.collection, points_selector=selector) diff --git a/airbyte-integrations/connectors/destination-qdrant/examples/configured_catalog.json b/airbyte-integrations/connectors/destination-qdrant/examples/configured_catalog.json new file mode 100644 index 000000000000..acb931363d89 --- /dev/null +++ b/airbyte-integrations/connectors/destination-qdrant/examples/configured_catalog.json @@ -0,0 +1,20 @@ +{ + "streams": [ + { + "stream": { + "name": "example_stream", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {} + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["column_name"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup", + "primary_key": [["pk"]] + } + ] +} diff --git a/airbyte-integrations/connectors/destination-qdrant/examples/messages.jsonl b/airbyte-integrations/connectors/destination-qdrant/examples/messages.jsonl new file mode 100644 index 000000000000..195d260c3ad6 --- /dev/null +++ b/airbyte-integrations/connectors/destination-qdrant/examples/messages.jsonl @@ -0,0 +1 @@ +{"type": "RECORD", "record": {"stream": "example_stream", "data": { "title": "valueZXXXXXXXX1", "field2": "value2", "pk": "1" }, "emitted_at": 1625383200000}} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-qdrant/icon.svg b/airbyte-integrations/connectors/destination-qdrant/icon.svg new file mode 100644 index 000000000000..fbbf0f1d49fd --- /dev/null +++ b/airbyte-integrations/connectors/destination-qdrant/icon.svg @@ -0,0 +1,21 @@ + + + qdrant + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-qdrant/main.py b/airbyte-integrations/connectors/destination-qdrant/main.py new file mode 100644 index 000000000000..42c2e8492e9f --- /dev/null +++ b/airbyte-integrations/connectors/destination-qdrant/main.py @@ -0,0 +1,11 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from destination_qdrant import DestinationQdrant + +if __name__ == "__main__": + DestinationQdrant().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-qdrant/metadata.yaml b/airbyte-integrations/connectors/destination-qdrant/metadata.yaml new file mode 100644 index 000000000000..73c87125aef2 --- /dev/null +++ b/airbyte-integrations/connectors/destination-qdrant/metadata.yaml @@ -0,0 +1,38 @@ +data: + registries: + cloud: + enabled: true + oss: + enabled: true + allowedHosts: + hosts: + - "${indexing.url}" + - api.openai.com + - api.cohere.ai + - "${embedding.api_base}" + resourceRequirements: + jobSpecific: + - jobType: sync + # TODO: Remove once https://github.com/airbytehq/airbyte/issues/30611 is resolved + resourceRequirements: + memory_limit: 2Gi + memory_request: 2Gi + connectorSubtype: vectorstore + connectorType: destination + definitionId: 6eb1198a-6d38-43e5-aaaa-dccd8f71db2b + dockerImageTag: 0.0.10 + dockerRepository: airbyte/destination-qdrant + githubIssueLabel: destination-qdrant + icon: qdrant.svg + license: MIT + name: Qdrant + releaseDate: 2023-09-22 + releaseStage: alpha + documentationUrl: https://docs.airbyte.com/integrations/destinations/qdrant + tags: + - language:python + ab_internal: + sl: 100 + ql: 100 + supportLevel: community +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-qdrant/requirements.txt b/airbyte-integrations/connectors/destination-qdrant/requirements.txt new file mode 100644 index 000000000000..d6e1198b1ab1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-qdrant/requirements.txt @@ -0,0 +1 @@ +-e . diff --git a/airbyte-integrations/connectors/destination-qdrant/setup.py b/airbyte-integrations/connectors/destination-qdrant/setup.py new file mode 100644 index 000000000000..f30ca62213c4 --- /dev/null +++ b/airbyte-integrations/connectors/destination-qdrant/setup.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.57.0", "qdrant-client", "fastembed"] + +TEST_REQUIREMENTS = ["pytest~=6.2"] + +setup( + name="destination_qdrant", + description="Destination implementation for Qdrant.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/destination-qdrant/unit_tests/__init__.py b/airbyte-integrations/connectors/destination-qdrant/unit_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/destination-qdrant/unit_tests/test_destination.py b/airbyte-integrations/connectors/destination-qdrant/unit_tests/test_destination.py new file mode 100644 index 000000000000..52068475465b --- /dev/null +++ b/airbyte-integrations/connectors/destination-qdrant/unit_tests/test_destination.py @@ -0,0 +1,101 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import unittest +from unittest.mock import MagicMock, Mock, patch + +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.models import ConnectorSpecification, Status +from destination_qdrant.config import ConfigModel +from destination_qdrant.destination import DestinationQdrant + + +class TestDestinationQdrant(unittest.TestCase): + def setUp(self): + self.config = { + "processing": {"text_fields": ["str_col"], "metadata_fields": [], "chunk_size": 1000}, + "embedding": {"mode": "openai", "openai_key": "mykey"}, + "indexing": { + "url": "localhost:6333", + "auth_method": { + "mode": "no_auth", + }, + "prefer_grpc": False, + "collection": "dummy-collection", + "distance_metric": "dot", + "text_field": "text", + }, + } + self.config_model = ConfigModel.parse_obj(self.config) + self.logger = AirbyteLogger() + + @patch("destination_qdrant.destination.QdrantIndexer") + @patch("destination_qdrant.destination.create_from_config") + def test_check(self, MockedEmbedder, MockedQdrantIndexer): + mock_embedder = Mock() + mock_indexer = Mock() + MockedEmbedder.return_value = mock_embedder + MockedQdrantIndexer.return_value = mock_indexer + + mock_embedder.check.return_value = None + mock_indexer.check.return_value = None + + destination = DestinationQdrant() + result = destination.check(self.logger, self.config) + + self.assertEqual(result.status, Status.SUCCEEDED) + mock_embedder.check.assert_called_once() + mock_indexer.check.assert_called_once() + + @patch("destination_qdrant.destination.QdrantIndexer") + @patch("destination_qdrant.destination.create_from_config") + def test_check_with_errors(self, MockedEmbedder, MockedQdrantIndexer): + mock_embedder = Mock() + mock_indexer = Mock() + MockedEmbedder.return_value = mock_embedder + MockedQdrantIndexer.return_value = mock_indexer + + embedder_error_message = "Embedder Error" + indexer_error_message = "Indexer Error" + + mock_embedder.check.return_value = embedder_error_message + mock_indexer.check.return_value = indexer_error_message + + destination = DestinationQdrant() + result = destination.check(self.logger, self.config) + + self.assertEqual(result.status, Status.FAILED) + self.assertEqual(result.message, f"{embedder_error_message}\n{indexer_error_message}") + + mock_embedder.check.assert_called_once() + mock_indexer.check.assert_called_once() + + @patch("destination_qdrant.destination.Writer") + @patch("destination_qdrant.destination.QdrantIndexer") + @patch("destination_qdrant.destination.create_from_config") + def test_write(self, MockedEmbedder, MockedQdrantIndexer, MockedWriter): + mock_embedder = Mock() + mock_indexer = Mock() + mock_writer = Mock() + + MockedEmbedder.return_value = mock_embedder + MockedQdrantIndexer.return_value = mock_indexer + MockedWriter.return_value = mock_writer + + mock_writer.write.return_value = [] + + configured_catalog = MagicMock() + input_messages = [] + + destination = DestinationQdrant() + list(destination.write(self.config, configured_catalog, input_messages)) + + MockedWriter.assert_called_once_with(self.config_model.processing, mock_indexer, mock_embedder, batch_size=256, omit_raw_text=False) + mock_writer.write.assert_called_once_with(configured_catalog, input_messages) + + def test_spec(self): + destination = DestinationQdrant() + result = destination.spec() + + self.assertIsInstance(result, ConnectorSpecification) diff --git a/airbyte-integrations/connectors/destination-qdrant/unit_tests/test_indexer.py b/airbyte-integrations/connectors/destination-qdrant/unit_tests/test_indexer.py new file mode 100644 index 000000000000..eef6619302aa --- /dev/null +++ b/airbyte-integrations/connectors/destination-qdrant/unit_tests/test_indexer.py @@ -0,0 +1,194 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import unittest +from unittest.mock import Mock, call + +from airbyte_cdk.destinations.vector_db_based.utils import format_exception +from airbyte_cdk.models.airbyte_protocol import AirbyteLogMessage, AirbyteMessage, AirbyteStream, DestinationSyncMode, Level, SyncMode, Type +from destination_qdrant.config import QdrantIndexingConfigModel +from destination_qdrant.indexer import QdrantIndexer +from qdrant_client import models + + +class TestQdrantIndexer(unittest.TestCase): + def setUp(self): + self.mock_config = QdrantIndexingConfigModel( + **{ + "url": "https://client-url.io", + "auth_method": {"mode": "api_key_auth", "api_key": "api_key"}, + "prefer_grpc": False, + "collection": "dummy-collection", + "distance_metric": "dot", + "text_field": "text", + } + ) + self.qdrant_indexer = QdrantIndexer(self.mock_config, 100) + self.qdrant_indexer._create_client = Mock() + self.qdrant_indexer._client = Mock() + + def test_check_gets_existing_collection(self): + mock_collections = Mock(collections=[Mock()]) + mock_collections.collections[0].name = "dummy-collection" + self.qdrant_indexer._client.get_collections.return_value = mock_collections + + self.qdrant_indexer._client.get_collection.return_value = Mock( + config=Mock(params=Mock(vectors=Mock(size=100, distance=models.Distance.DOT))) + ) + + check_result = self.qdrant_indexer.check() + + self.assertIsNone(check_result) + + self.qdrant_indexer._create_client.assert_called() + self.qdrant_indexer._client.get_collections.assert_called() + self.qdrant_indexer._client.get_collection.assert_called() + self.qdrant_indexer._client.close.assert_called() + + def test_check_creates_new_collection_if_not_exists(self): + self.qdrant_indexer._client.get_collections.return_value = Mock(collections=[]) + check_result = self.qdrant_indexer.check() + + self.assertIsNone(check_result) + + self.qdrant_indexer._create_client.assert_called() + self.qdrant_indexer._client.get_collections.assert_called() + self.qdrant_indexer._client.recreate_collection.assert_called() + self.qdrant_indexer._client.close.assert_called() + + def test_check_handles_failure_conditions(self): + # Test 1: url starts with https:// + self.qdrant_indexer.config.url = "client-url.io" + result = self.qdrant_indexer.check() + self.assertEqual(result, "Host must start with https://") + + # Test 2: random exception + self.qdrant_indexer.config.url = "https://client-url.io" + self.qdrant_indexer._create_client.side_effect = Exception("Random exception") + result = self.qdrant_indexer.check() + self.assertTrue("Random exception" in result) + + # Test 3: client server is not alive + self.qdrant_indexer._create_client.side_effect = None + self.qdrant_indexer._client = None + result = self.qdrant_indexer.check() + self.assertEqual(result, "Qdrant client is not alive.") + + # Test 4: Test vector size does not match + mock_collections = Mock(collections=[Mock()]) + mock_collections.collections[0].name = "dummy-collection" + + self.qdrant_indexer._client = Mock() + self.qdrant_indexer._client.get_collections.return_value = mock_collections + self.qdrant_indexer._client.get_collection.return_value = Mock( + config=Mock(params=Mock(vectors=Mock(size=10, distance=models.Distance.DOT))) + ) + + result = self.qdrant_indexer.check() + self.assertTrue("The collection's vector's size must match the embedding dimensions" in result) + + # Test 5: Test distance metric does not match + self.qdrant_indexer._client.get_collection.return_value = Mock( + config=Mock(params=Mock(vectors=Mock(size=100, distance=models.Distance.COSINE))) + ) + result = self.qdrant_indexer.check() + self.assertTrue("The colection's vector's distance metric must match the selected distance metric option" in result) + + def test_pre_sync_calls_delete(self): + self.qdrant_indexer.pre_sync( + Mock( + streams=[ + Mock( + destination_sync_mode=DestinationSyncMode.overwrite, + stream=AirbyteStream(name="some_stream", json_schema={}, supported_sync_modes=[SyncMode.full_refresh]), + ), + Mock( + destination_sync_mode=DestinationSyncMode.overwrite, + stream=AirbyteStream(name="another_stream", json_schema={}, supported_sync_modes=[SyncMode.full_refresh]), + ), + Mock( + destination_sync_mode=DestinationSyncMode.append, + stream=AirbyteStream(name="incremental_stream", json_schema={}, supported_sync_modes=[SyncMode.full_refresh]), + ), + ] + ) + ) + + self.qdrant_indexer._client.delete.assert_called_with( + collection_name=self.mock_config.collection, + points_selector=models.FilterSelector( + filter=models.Filter( + should=[ + models.FieldCondition(key="_ab_stream", match=models.MatchValue(value="some_stream")), + models.FieldCondition(key="_ab_stream", match=models.MatchValue(value="another_stream")), + ] + ) + ), + ) + + def test_pre_sync_does_not_call_delete(self): + self.qdrant_indexer.pre_sync( + Mock(streams=[Mock(destination_sync_mode=DestinationSyncMode.append, stream=Mock(name="some_stream"))]) + ) + + self.qdrant_indexer._client.delete.assert_not_called() + + def test_pre_sync_calls_create_payload_index(self): + self.qdrant_indexer.pre_sync(Mock(streams=[])) + + calls = [ + call(collection_name=self.mock_config.collection, field_name="_ab_record_id", field_schema="keyword"), + call(collection_name=self.mock_config.collection, field_name="_ab_stream", field_schema="keyword"), + ] + + self.qdrant_indexer._client.create_payload_index.assert_has_calls(calls) + + def test_index_calls_insert(self): + self.qdrant_indexer.index( + [ + Mock(metadata={"key": "value1"}, page_content="some content", embedding=[1.0, 2.0, 3.0]), + Mock(metadata={"key": "value2"}, page_content="some other content", embedding=[4.0, 5.0, 6.0]), + ], + None, + "some_stream", + ) + + self.qdrant_indexer._client.upload_records.assert_called_once() + + def test_index_calls_delete(self): + self.qdrant_indexer.delete(["some_id", "another_id"], None, "some_stream") + + self.qdrant_indexer._client.delete.assert_called_with( + collection_name=self.mock_config.collection, + points_selector=models.FilterSelector( + filter=models.Filter( + should=[ + models.FieldCondition(key="_ab_record_id", match=models.MatchValue(value="some_id")), + models.FieldCondition(key="_ab_record_id", match=models.MatchValue(value="another_id")), + ] + ) + ), + ) + + def test_post_sync_calls_close(self): + result = self.qdrant_indexer.post_sync() + self.qdrant_indexer._client.close.assert_called_once() + self.assertEqual( + result, + [ + AirbyteMessage( + type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message="Qdrant Database Client has been closed successfully") + ) + ], + ) + + def test_post_sync_handles_failure(self): + exception = Exception("Random exception") + self.qdrant_indexer._client.close.side_effect = exception + result = self.qdrant_indexer.post_sync() + + self.qdrant_indexer._client.close.assert_called_once() + self.assertEqual( + result, [AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.ERROR, message=format_exception(exception)))] + ) diff --git a/airbyte-integrations/connectors/destination-r2/.dockerignore b/airbyte-integrations/connectors/destination-r2/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-r2/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-r2/Dockerfile b/airbyte-integrations/connectors/destination-r2/Dockerfile deleted file mode 100644 index d5814fc31d3c..000000000000 --- a/airbyte-integrations/connectors/destination-r2/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte -ENV APPLICATION destination-r2 - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte -ENV APPLICATION destination-r2 - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/destination-r2 diff --git a/airbyte-integrations/connectors/destination-r2/README.md b/airbyte-integrations/connectors/destination-r2/README.md index 3e560f7ec44b..229c1d0e2d21 100644 --- a/airbyte-integrations/connectors/destination-r2/README.md +++ b/airbyte-integrations/connectors/destination-r2/README.md @@ -23,10 +23,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-r2:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-r2:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-r2:dev`. the Dockerfile. #### Run @@ -63,8 +64,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-r2 test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/r2.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-r2/build.gradle b/airbyte-integrations/connectors/destination-r2/build.gradle index 92194d3b7fc9..94626b963c02 100644 --- a/airbyte-integrations/connectors/destination-r2/build.gradle +++ b/airbyte-integrations/connectors/destination-r2/build.gradle @@ -1,20 +1,29 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.r2.R2Destination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:bases:base-java-s3') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) // csv implementation 'com.amazonaws:aws-java-sdk-s3:1.11.978' @@ -37,8 +46,4 @@ dependencies { testImplementation 'org.apache.commons:commons-lang3:3.11' testImplementation 'org.xerial.snappy:snappy-java:1.1.8.4' testImplementation "org.mockito:mockito-inline:4.1.0" - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-r2') - integrationTestJavaImplementation project(':airbyte-integrations:bases:s3-destination-base-integration-test') } diff --git a/airbyte-integrations/connectors/destination-r2/src/main/java/io/airbyte/integrations/destination/r2/R2Destination.java b/airbyte-integrations/connectors/destination-r2/src/main/java/io/airbyte/integrations/destination/r2/R2Destination.java index 729a23096e7b..1c663ed2f54e 100644 --- a/airbyte-integrations/connectors/destination-r2/src/main/java/io/airbyte/integrations/destination/r2/R2Destination.java +++ b/airbyte-integrations/connectors/destination-r2/src/main/java/io/airbyte/integrations/destination/r2/R2Destination.java @@ -4,9 +4,9 @@ package io.airbyte.integrations.destination.r2; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.destination.s3.BaseS3Destination; -import io.airbyte.integrations.destination.s3.StorageProvider; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.destination.s3.BaseS3Destination; +import io.airbyte.cdk.integrations.destination.s3.StorageProvider; public class R2Destination extends BaseS3Destination { diff --git a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2AvroDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2AvroDestinationAcceptanceTest.java index 5129d57938ed..ac4b5267098a 100644 --- a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2AvroDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2AvroDestinationAcceptanceTest.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.r2; -import io.airbyte.integrations.destination.s3.S3BaseAvroDestinationAcceptanceTest; -import io.airbyte.integrations.destination.s3.StorageProvider; +import io.airbyte.cdk.integrations.destination.s3.S3BaseAvroDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.destination.s3.StorageProvider; public class R2AvroDestinationAcceptanceTest extends S3BaseAvroDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2CsvDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2CsvDestinationAcceptanceTest.java index 2caacc92819a..a9f2b72a5fd8 100644 --- a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2CsvDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2CsvDestinationAcceptanceTest.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.r2; -import io.airbyte.integrations.destination.s3.S3BaseCsvDestinationAcceptanceTest; -import io.airbyte.integrations.destination.s3.StorageProvider; +import io.airbyte.cdk.integrations.destination.s3.S3BaseCsvDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.destination.s3.StorageProvider; public class R2CsvDestinationAcceptanceTest extends S3BaseCsvDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2CsvGzipDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2CsvGzipDestinationAcceptanceTest.java index 63680636fad3..3b6df013b10e 100644 --- a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2CsvGzipDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2CsvGzipDestinationAcceptanceTest.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.r2; -import io.airbyte.integrations.destination.s3.S3BaseCsvGzipDestinationAcceptanceTest; -import io.airbyte.integrations.destination.s3.StorageProvider; +import io.airbyte.cdk.integrations.destination.s3.S3BaseCsvGzipDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.destination.s3.StorageProvider; public class R2CsvGzipDestinationAcceptanceTest extends S3BaseCsvGzipDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2JsonlDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2JsonlDestinationAcceptanceTest.java index 7afda89f9ddb..0385e1c7d008 100644 --- a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2JsonlDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2JsonlDestinationAcceptanceTest.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.r2; -import io.airbyte.integrations.destination.s3.S3BaseJsonlDestinationAcceptanceTest; -import io.airbyte.integrations.destination.s3.StorageProvider; +import io.airbyte.cdk.integrations.destination.s3.S3BaseJsonlDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.destination.s3.StorageProvider; public class R2JsonlDestinationAcceptanceTest extends S3BaseJsonlDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2JsonlGzipDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2JsonlGzipDestinationAcceptanceTest.java index 7c9ef53b508c..13506ab48dee 100644 --- a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2JsonlGzipDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2JsonlGzipDestinationAcceptanceTest.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.r2; -import io.airbyte.integrations.destination.s3.S3BaseJsonlGzipDestinationAcceptanceTest; -import io.airbyte.integrations.destination.s3.StorageProvider; +import io.airbyte.cdk.integrations.destination.s3.S3BaseJsonlGzipDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.destination.s3.StorageProvider; public class R2JsonlGzipDestinationAcceptanceTest extends S3BaseJsonlGzipDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2ParquetDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2ParquetDestinationAcceptanceTest.java index cf114b429e0c..6393a0e27502 100644 --- a/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2ParquetDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-r2/src/test-integration/java/io/airbyte/integrations/destination/r2/R2ParquetDestinationAcceptanceTest.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.r2; -import io.airbyte.integrations.destination.s3.S3BaseParquetDestinationAcceptanceTest; -import io.airbyte.integrations.destination.s3.StorageProvider; +import io.airbyte.cdk.integrations.destination.s3.S3BaseParquetDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.destination.s3.StorageProvider; import org.junit.jupiter.api.Disabled; /** diff --git a/airbyte-integrations/connectors/destination-rabbitmq/README.md b/airbyte-integrations/connectors/destination-rabbitmq/README.md index a31432ee33f3..f6952028a518 100644 --- a/airbyte-integrations/connectors/destination-rabbitmq/README.md +++ b/airbyte-integrations/connectors/destination-rabbitmq/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:destination-rabbitmq:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/destinations/rabbitmq) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_rabbitmq/spec.json` file. @@ -54,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/destination-rabbitmq:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=destination-rabbitmq build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:destination-rabbitmq:airbyteDocker +An image will be built with the tag `airbyte/destination-rabbitmq:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/destination-rabbitmq:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,38 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-rabbitmq:dev chec # messages.jsonl is a file containing line-separated JSON representing AirbyteMessages cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-rabbitmq:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=destination-rabbitmq test ``` -#### Acceptance Tests -Coming soon: -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:destination-rabbitmq:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:destination-rabbitmq:integrationTest -``` +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -116,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-rabbitmq test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/rabbitmq.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-rabbitmq/build.gradle b/airbyte-integrations/connectors/destination-rabbitmq/build.gradle deleted file mode 100644 index 1023d96b2913..000000000000 --- a/airbyte-integrations/connectors/destination-rabbitmq/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'destination_rabbitmq' -} diff --git a/airbyte-integrations/connectors/destination-redis/.dockerignore b/airbyte-integrations/connectors/destination-redis/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-redis/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-redis/Dockerfile b/airbyte-integrations/connectors/destination-redis/Dockerfile deleted file mode 100644 index 6bc988825006..000000000000 --- a/airbyte-integrations/connectors/destination-redis/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-redis - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-redis - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.4 -LABEL io.airbyte.name=airbyte/destination-redis diff --git a/airbyte-integrations/connectors/destination-redis/README.md b/airbyte-integrations/connectors/destination-redis/README.md index 11c35aaed89b..ab09827ef20e 100644 --- a/airbyte-integrations/connectors/destination-redis/README.md +++ b/airbyte-integrations/connectors/destination-redis/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-redis:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-redis:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-redis:dev`. the Dockerfile. #### Run @@ -61,8 +62,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-redis test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/redis.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-redis/build.gradle b/airbyte-integrations/connectors/destination-redis/build.gradle index 1307167e4181..83cf3a207e87 100644 --- a/airbyte-integrations/connectors/destination-redis/build.gradle +++ b/airbyte-integrations/connectors/destination-redis/build.gradle @@ -1,9 +1,23 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.redis.RedisDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] @@ -13,11 +27,6 @@ def redisDriver = '3.7.0' def assertVersion = '3.21.0' dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - // https://mvnrepository.com/artifact/redis.clients/jedis implementation "redis.clients:jedis:${redisDriver}" @@ -28,8 +37,5 @@ dependencies { // https://mvnrepository.com/artifact/org.assertj/assertj-core testImplementation "org.assertj:assertj-core:${assertVersion}" // https://mvnrepository.com/artifact/org.testcontainers/testcontainers - testImplementation libs.connectors.testcontainers - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-redis') + testImplementation libs.testcontainers } diff --git a/airbyte-integrations/connectors/destination-redis/docker-compose.yml b/airbyte-integrations/connectors/destination-redis/docker-compose.yml index 975475e3f5a1..a8ef9621116f 100644 --- a/airbyte-integrations/connectors/destination-redis/docker-compose.yml +++ b/airbyte-integrations/connectors/destination-redis/docker-compose.yml @@ -5,10 +5,10 @@ services: command: redis-server --requirepass pw ports: - "6379:6379" -# uncomment if you want to mount volumes for persistence -# volumes: -# - $PWD/redis-data:/var/lib/redis -# - $PWD/redis.conf:/usr/local/etc/redis/redis.conf + # uncomment if you want to mount volumes for persistence + # volumes: + # - $PWD/redis-data:/var/lib/redis + # - $PWD/redis.conf:/usr/local/etc/redis/redis.conf environment: - REDIS_REPLICATION_MODE=master networks: diff --git a/airbyte-integrations/connectors/destination-redis/src/main/java/io/airbyte/integrations/destination/redis/RedisDestination.java b/airbyte-integrations/connectors/destination-redis/src/main/java/io/airbyte/integrations/destination/redis/RedisDestination.java index ea577c968ed2..7ef1b6a5d04d 100644 --- a/airbyte-integrations/connectors/destination-redis/src/main/java/io/airbyte/integrations/destination/redis/RedisDestination.java +++ b/airbyte-integrations/connectors/destination-redis/src/main/java/io/airbyte/integrations/destination/redis/RedisDestination.java @@ -5,12 +5,12 @@ package io.airbyte.integrations.destination.redis; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.AirbyteTraceMessageUtility; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.ssh.SshWrappedDestination; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/connectors/destination-redis/src/main/java/io/airbyte/integrations/destination/redis/RedisMessageConsumer.java b/airbyte-integrations/connectors/destination-redis/src/main/java/io/airbyte/integrations/destination/redis/RedisMessageConsumer.java index ed158ff723d3..c4fd4c13cd43 100644 --- a/airbyte-integrations/connectors/destination-redis/src/main/java/io/airbyte/integrations/destination/redis/RedisMessageConsumer.java +++ b/airbyte-integrations/connectors/destination-redis/src/main/java/io/airbyte/integrations/destination/redis/RedisMessageConsumer.java @@ -5,8 +5,8 @@ package io.airbyte.integrations.destination.redis; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/RedisDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/RedisDestinationAcceptanceTest.java index 6321e2b2ab3e..c9194deb0c41 100644 --- a/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/RedisDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/RedisDestinationAcceptanceTest.java @@ -5,10 +5,10 @@ package io.airbyte.integrations.destination.redis; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.util.Comparator; import java.util.HashSet; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/SshKeyRedisDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/SshKeyRedisDestinationAcceptanceTest.java index 9764e82bf96a..cb0a1b876727 100644 --- a/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/SshKeyRedisDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/SshKeyRedisDestinationAcceptanceTest.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.redis; -import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; public class SshKeyRedisDestinationAcceptanceTest extends SshRedisDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/SshPasswordRedisDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/SshPasswordRedisDestinationAcceptanceTest.java index 3492ea08fe4a..a96ae4d2185b 100644 --- a/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/SshPasswordRedisDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/SshPasswordRedisDestinationAcceptanceTest.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.redis; -import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; public class SshPasswordRedisDestinationAcceptanceTest extends SshRedisDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/SshRedisDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/SshRedisDestinationAcceptanceTest.java index d8f0a539882c..4d713b3c29f6 100644 --- a/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/SshRedisDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redis/src/test-integration/java/io/airbyte/integrations/destination/redis/SshRedisDestinationAcceptanceTest.java @@ -6,14 +6,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.base.ssh.SshBastionContainer; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.ssh.SshBastionContainer; -import io.airbyte.integrations.base.ssh.SshTunnel; import io.airbyte.integrations.destination.redis.RedisContainerInitializr.RedisContainer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; -import io.airbyte.integrations.util.HostPortResolver; import java.util.Comparator; import java.util.HashSet; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-redpanda/.dockerignore b/airbyte-integrations/connectors/destination-redpanda/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-redpanda/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-redpanda/Dockerfile b/airbyte-integrations/connectors/destination-redpanda/Dockerfile deleted file mode 100644 index e363586fa18c..000000000000 --- a/airbyte-integrations/connectors/destination-redpanda/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte -ENV APPLICATION destination-redpanda - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte -ENV APPLICATION destination-redpanda - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/destination-redpanda diff --git a/airbyte-integrations/connectors/destination-redpanda/README.md b/airbyte-integrations/connectors/destination-redpanda/README.md index fed5a2daa34e..6f9f022a7f73 100644 --- a/airbyte-integrations/connectors/destination-redpanda/README.md +++ b/airbyte-integrations/connectors/destination-redpanda/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-redpanda:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-redpanda:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-redpanda:dev`. the Dockerfile. #### Run @@ -61,8 +62,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-redpanda test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/redpanda.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-redpanda/build.gradle b/airbyte-integrations/connectors/destination-redpanda/build.gradle index b6d222cc1681..a79982fe7c56 100644 --- a/airbyte-integrations/connectors/destination-redpanda/build.gradle +++ b/airbyte-integrations/connectors/destination-redpanda/build.gradle @@ -1,25 +1,32 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.redpanda.RedpandaDestination' } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) // https://mvnrepository.com/artifact/org.apache.kafka/kafka-clients implementation 'org.apache.kafka:kafka-clients:3.3.1' implementation 'org.apache.kafka:connect-json:3.3.1' testImplementation "org.testcontainers:redpanda:1.17.5" - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-redpanda') } diff --git a/airbyte-integrations/connectors/destination-redpanda/src/main/java/io/airbyte/integrations/destination/redpanda/RedpandaDestination.java b/airbyte-integrations/connectors/destination-redpanda/src/main/java/io/airbyte/integrations/destination/redpanda/RedpandaDestination.java index a8ed1e2dee85..93b6e3d13f92 100644 --- a/airbyte-integrations/connectors/destination-redpanda/src/main/java/io/airbyte/integrations/destination/redpanda/RedpandaDestination.java +++ b/airbyte-integrations/connectors/destination-redpanda/src/main/java/io/airbyte/integrations/destination/redpanda/RedpandaDestination.java @@ -5,11 +5,11 @@ package io.airbyte.integrations.destination.redpanda; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/connectors/destination-redpanda/src/main/java/io/airbyte/integrations/destination/redpanda/RedpandaMessageConsumer.java b/airbyte-integrations/connectors/destination-redpanda/src/main/java/io/airbyte/integrations/destination/redpanda/RedpandaMessageConsumer.java index f66201c4f4a4..4be72a2c3931 100644 --- a/airbyte-integrations/connectors/destination-redpanda/src/main/java/io/airbyte/integrations/destination/redpanda/RedpandaMessageConsumer.java +++ b/airbyte-integrations/connectors/destination-redpanda/src/main/java/io/airbyte/integrations/destination/redpanda/RedpandaMessageConsumer.java @@ -4,12 +4,12 @@ package io.airbyte.integrations.destination.redpanda; -import static io.airbyte.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_ID; -import static io.airbyte.integrations.base.JavaBaseConstants.COLUMN_NAME_DATA; -import static io.airbyte.integrations.base.JavaBaseConstants.COLUMN_NAME_EMITTED_AT; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_ID; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_DATA; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_EMITTED_AT; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/connectors/destination-redpanda/src/main/java/io/airbyte/integrations/destination/redpanda/RedpandaNameTransformer.java b/airbyte-integrations/connectors/destination-redpanda/src/main/java/io/airbyte/integrations/destination/redpanda/RedpandaNameTransformer.java index 394917f667b0..b368a4a137a8 100644 --- a/airbyte-integrations/connectors/destination-redpanda/src/main/java/io/airbyte/integrations/destination/redpanda/RedpandaNameTransformer.java +++ b/airbyte-integrations/connectors/destination-redpanda/src/main/java/io/airbyte/integrations/destination/redpanda/RedpandaNameTransformer.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.redpanda; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; public class RedpandaNameTransformer extends StandardNameTransformer { diff --git a/airbyte-integrations/connectors/destination-redpanda/src/test-integration/java/io/airbyte/integrations/destination/redpanda/RedpandaDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redpanda/src/test-integration/java/io/airbyte/integrations/destination/redpanda/RedpandaDestinationAcceptanceTest.java index 6ac4e241ab8e..9d276c456a33 100644 --- a/airbyte-integrations/connectors/destination-redpanda/src/test-integration/java/io/airbyte/integrations/destination/redpanda/RedpandaDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redpanda/src/test-integration/java/io/airbyte/integrations/destination/redpanda/RedpandaDestinationAcceptanceTest.java @@ -6,11 +6,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.time.Duration; import java.util.ArrayList; import java.util.Collections; diff --git a/airbyte-integrations/connectors/destination-redshift/.dockerignore b/airbyte-integrations/connectors/destination-redshift/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-redshift/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-redshift/Dockerfile b/airbyte-integrations/connectors/destination-redshift/Dockerfile deleted file mode 100644 index 85e41effc438..000000000000 --- a/airbyte-integrations/connectors/destination-redshift/Dockerfile +++ /dev/null @@ -1,53 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15 -RUN yum install -y python3 python3-devel jq sshpass git gcc-c++ && yum clean all && \ - alternatives --install /usr/bin/python python /usr/bin/python3 60 && \ - python -m ensurepip --upgrade && \ - # these two lines are a workaround for https://github.com/yaml/pyyaml/issues/601 - pip3 install wheel && \ - pip3 install "Cython<3.0" "pyyaml==5.4" --no-build-isolation && \ - pip3 install dbt-redshift==1.0.0 "urllib3<2" - -# Luckily, none of normalization's files conflict with destination-bigquery's files :) -# We don't enforce that in any way, but hopefully we're only living in this state for a short time. -COPY --from=airbyte/normalization-redshift:dev /airbyte /airbyte -# Install python dependencies -WORKDIR /airbyte/base_python_structs -RUN pip3 install . -WORKDIR /airbyte/normalization_code -RUN pip3 install . -WORKDIR /airbyte/normalization_code/dbt-template/ -# Download external dbt dependencies -RUN dbt deps - -WORKDIR /airbyte - -ENV APPLICATION destination-redshift -ENV AIRBYTE_NORMALIZATION_INTEGRATION redshift - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-redshift - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.6.5 -LABEL io.airbyte.name=airbyte/destination-redshift - -ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh" -ENTRYPOINT ["/airbyte/run_with_normalization.sh"] diff --git a/airbyte-integrations/connectors/destination-redshift/build.gradle b/airbyte-integrations/connectors/destination-redshift/build.gradle index 98bed326e017..aa75211ebf3e 100644 --- a/airbyte-integrations/connectors/destination-redshift/build.gradle +++ b/airbyte-integrations/connectors/destination-redshift/build.gradle @@ -1,50 +1,47 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } -application { - mainClass = 'io.airbyte.integrations.destination.redshift.RedshiftDestination' - applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] +airbyteJavaConnector { + cdkVersionRequired = '0.13.0' + features = ['db-destinations', 's3-destinations', 'typing-deduping'] + useLocalCdk = false } -repositories { - maven { - url "https://s3.amazonaws.com/redshift-maven-repository/release" +java { + compileJava { + options.compilerArgs.remove("-Werror") } } +airbyteJavaConnector.addCdkDependencies() + +application { + mainClass = 'io.airbyte.integrations.destination.redshift.RedshiftDestination' + applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0', + '-XX:NativeMemoryTracking=detail', '-XX:+UnlockDiagnosticVMOptions', + '-XX:GCLockerRetryAllocationCount=100',] +} + dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:bases-destination-jdbc') - implementation project(':airbyte-integrations:bases:base-java-s3') - implementation project(':airbyte-integrations:bases:base-typing-deduping') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation 'com.amazonaws:aws-java-sdk-s3:1.11.978' - implementation 'com.amazon.redshift:redshift-jdbc42-no-awssdk:1.2.51.1078' // use the no-sdk library to avoid aws classpath conflicts + // TODO: Verify no aws sdk code is pulled by this dependency causing classpath conflicts + // https://docs.aws.amazon.com/redshift/latest/mgmt/jdbc20-jdbc10-driver-differences.html + implementation 'com.amazon.redshift:redshift-jdbc42:2.1.0.23' implementation 'org.apache.commons:commons-csv:1.4' implementation 'com.github.alexmojaki:s3-stream-upload:2.2.2' - testImplementation project(':airbyte-test-utils') - testImplementation 'org.apache.commons:commons-text:1.10.0' testImplementation 'org.apache.commons:commons-lang3:3.11' testImplementation 'org.apache.commons:commons-dbcp2:2.7.0' testImplementation "org.mockito:mockito-inline:4.1.0" - testImplementation project(":airbyte-json-validation") - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-redshift') - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) } -tasks.named("airbyteDocker") { - // this is really inefficent (because base-normalization:airbyteDocker builds 9 docker images) - // but it's also just simple to implement. - // this also goes away once airbyte-ci becomes a reality. - dependsOn project(':airbyte-integrations:bases:base-normalization').airbyteDockerRedshift +configurations.all { + resolutionStrategy { + force libs.jooq + } } diff --git a/airbyte-integrations/connectors/destination-redshift/gradle.properties b/airbyte-integrations/connectors/destination-redshift/gradle.properties new file mode 100644 index 000000000000..8b137891791f --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/gradle.properties @@ -0,0 +1 @@ + diff --git a/airbyte-integrations/connectors/destination-redshift/metadata.yaml b/airbyte-integrations/connectors/destination-redshift/metadata.yaml index 0b7d33a37ced..b32972ffd65e 100644 --- a/airbyte-integrations/connectors/destination-redshift/metadata.yaml +++ b/airbyte-integrations/connectors/destination-redshift/metadata.yaml @@ -1,9 +1,13 @@ data: + ab_internal: + ql: 200 + sl: 100 connectorSubtype: database connectorType: destination definitionId: f7a7d195-377f-cf5b-70a5-be6b819019dc - dockerImageTag: 0.6.5 + dockerImageTag: 0.8.0 dockerRepository: airbyte/destination-redshift + documentationUrl: https://docs.airbyte.com/integrations/destinations/redshift githubIssueLabel: destination-redshift icon: redshift.svg license: MIT @@ -24,12 +28,8 @@ data: resourceRequirements: memory_limit: 1Gi memory_request: 1Gi - documentationUrl: https://docs.airbyte.com/integrations/destinations/redshift + supportLevel: community supportsDbt: true tags: - language:java - ab_internal: - sl: 100 - ql: 300 - supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftDestination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftDestination.java index 49018e253cb0..d10cabd45730 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftDestination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftDestination.java @@ -9,11 +9,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.destination.jdbc.copy.SwitchingDestination; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.destination.jdbc.copy.SwitchingDestination; import io.airbyte.protocol.models.v0.ConnectorSpecification; import java.util.Map; import org.slf4j.Logger; diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java index 08b9e21f3e27..a4ba7a669557 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestination.java @@ -7,16 +7,24 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap.Builder; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcSourceOperations; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination; +import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcDestinationHandler; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.ssh.SshWrappedDestination; -import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import io.airbyte.integrations.destination.redshift.operations.RedshiftSqlOperations; +import io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftDestinationHandler; +import io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftSqlGenerator; +import io.airbyte.integrations.destination.redshift.util.RedshiftUtil; +import java.time.Duration; +import java.util.HashMap; import java.util.Map; import java.util.Optional; import javax.sql.DataSource; @@ -49,7 +57,13 @@ public DataSource getDataSource(final JsonNode config) { jdbcConfig.has(JdbcUtils.PASSWORD_KEY) ? jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, RedshiftInsertDestination.DRIVER_CLASS, jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), - SSL_JDBC_PARAMETERS); + getDefaultConnectionProperties(config), + Duration.ofMinutes(2)); + } + + @Override + protected void destinationSpecificTableOperations(final JdbcDatabase database) throws Exception { + RedshiftUtil.checkSvvTableAccess(database); } @Override @@ -57,14 +71,29 @@ public JdbcDatabase getDatabase(final DataSource dataSource) { return new DefaultJdbcDatabase(dataSource); } + public JdbcDatabase getDatabase(final DataSource dataSource, final JdbcSourceOperations sourceOperations) { + return new DefaultJdbcDatabase(dataSource, sourceOperations); + } + @Override protected Map getDefaultConnectionProperties(final JsonNode config) { - return SSL_JDBC_PARAMETERS; + // The following properties can be overriden through jdbcUrlParameters in the config. + final Map connectionOptions = new HashMap<>(); + // Redshift properties + // https://docs.aws.amazon.com/redshift/latest/mgmt/jdbc20-configuration-options.html#jdbc20-connecttimeout-option + // connectTimeout is different from Hikari pool's connectionTimout, driver defaults to 10seconds so + // increase it to match hikari's default + connectionOptions.put("connectTimeout", "120"); + // HikariPool properties + // https://github.com/brettwooldridge/HikariCP?tab=readme-ov-file#frequently-used + // TODO: Change data source factory to configure these properties + connectionOptions.putAll(SSL_JDBC_PARAMETERS); + return connectionOptions; } public static JsonNode getJdbcConfig(final JsonNode redshiftConfig) { final String schema = Optional.ofNullable(redshiftConfig.get(JdbcUtils.SCHEMA_KEY)).map(JsonNode::asText).orElse("public"); - Builder configBuilder = ImmutableMap.builder() + final Builder configBuilder = ImmutableMap.builder() .put(JdbcUtils.USERNAME_KEY, redshiftConfig.get(JdbcUtils.USERNAME_KEY).asText()) .put(JdbcUtils.PASSWORD_KEY, redshiftConfig.get(JdbcUtils.PASSWORD_KEY).asText()) .put(JdbcUtils.JDBC_URL_KEY, String.format("jdbc:redshift://%s:%s/%s", @@ -80,4 +109,14 @@ public static JsonNode getJdbcConfig(final JsonNode redshiftConfig) { return Jsons.jsonNode(configBuilder.build()); } + @Override + protected JdbcSqlGenerator getSqlGenerator() { + return new RedshiftSqlGenerator(super.getNamingResolver()); + } + + @Override + protected JdbcDestinationHandler getDestinationHandler(final String databaseName, final JdbcDatabase database) { + return new RedshiftDestinationHandler(databaseName, database); + } + } diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftSQLNameTransformer.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftSQLNameTransformer.java index ef343405061e..7ec8f0471af3 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftSQLNameTransformer.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftSQLNameTransformer.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.redshift; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; public class RedshiftSQLNameTransformer extends StandardNameTransformer { diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java index 7d4250dd2f73..82af7555922b 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3Destination.java @@ -4,49 +4,66 @@ package io.airbyte.integrations.destination.redshift; -import static io.airbyte.integrations.base.errors.messages.ErrorMessage.getErrorMessage; +import static io.airbyte.cdk.integrations.base.errors.messages.ErrorMessage.getErrorMessage; +import static io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig.getS3DestinationConfig; import static io.airbyte.integrations.destination.redshift.RedshiftInsertDestination.SSL_JDBC_PARAMETERS; import static io.airbyte.integrations.destination.redshift.RedshiftInsertDestination.getJdbcConfig; import static io.airbyte.integrations.destination.redshift.constants.RedshiftDestinationConstants.UPLOADING_METHOD; import static io.airbyte.integrations.destination.redshift.util.RedshiftUtil.findS3Options; -import static io.airbyte.integrations.destination.s3.S3DestinationConfig.getS3DestinationConfig; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.TypingAndDedupingFlag; +import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcDestinationHandler; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcV1V2Migrator; +import io.airbyte.cdk.integrations.destination.record_buffer.FileBuffer; +import io.airbyte.cdk.integrations.destination.s3.AesCbcEnvelopeEncryption; +import io.airbyte.cdk.integrations.destination.s3.AesCbcEnvelopeEncryption.KeyType; +import io.airbyte.cdk.integrations.destination.s3.EncryptionConfig; +import io.airbyte.cdk.integrations.destination.s3.NoEncryption; +import io.airbyte.cdk.integrations.destination.s3.S3BaseChecks; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3StorageOperations; +import io.airbyte.cdk.integrations.destination.staging.StagingConsumerFactory; import io.airbyte.commons.exceptions.ConnectionErrorException; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.AirbyteTraceMessageUtility; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.SerializedAirbyteMessageConsumer; +import io.airbyte.integrations.base.destination.typing_deduping.CatalogParser; +import io.airbyte.integrations.base.destination.typing_deduping.DefaultTyperDeduper; +import io.airbyte.integrations.base.destination.typing_deduping.NoOpTyperDeduperWithV1V2Migrations; import io.airbyte.integrations.base.destination.typing_deduping.NoopTyperDeduper; +import io.airbyte.integrations.base.destination.typing_deduping.NoopV2TableMigrator; +import io.airbyte.integrations.base.destination.typing_deduping.ParsedCatalog; import io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeOperationValve; -import io.airbyte.integrations.base.ssh.SshWrappedDestination; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; -import io.airbyte.integrations.destination.record_buffer.FileBuffer; +import io.airbyte.integrations.base.destination.typing_deduping.TyperDeduper; import io.airbyte.integrations.destination.redshift.operations.RedshiftS3StagingSqlOperations; import io.airbyte.integrations.destination.redshift.operations.RedshiftSqlOperations; -import io.airbyte.integrations.destination.s3.AesCbcEnvelopeEncryption; -import io.airbyte.integrations.destination.s3.AesCbcEnvelopeEncryption.KeyType; -import io.airbyte.integrations.destination.s3.EncryptionConfig; -import io.airbyte.integrations.destination.s3.NoEncryption; -import io.airbyte.integrations.destination.s3.S3BaseChecks; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.S3StorageOperations; -import io.airbyte.integrations.destination.staging.StagingConsumerFactory; +import io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftDestinationHandler; +import io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftSqlGenerator; +import io.airbyte.integrations.destination.redshift.util.RedshiftUtil; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import java.time.Duration; +import java.util.HashMap; import java.util.Map; import java.util.function.Consumer; import javax.sql.DataSource; import org.apache.commons.lang3.NotImplementedException; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,7 +80,7 @@ public RedshiftStagingS3Destination() { } private boolean isEphemeralKeysAndPurgingStagingData(final JsonNode config, final EncryptionConfig encryptionConfig) { - return !isPurgeStagingData(config) && encryptionConfig instanceof AesCbcEnvelopeEncryption c && c.keyType() == KeyType.EPHEMERAL; + return !isPurgeStagingData(config) && encryptionConfig instanceof final AesCbcEnvelopeEncryption c && c.keyType() == KeyType.EPHEMERAL; } @Override @@ -87,7 +104,8 @@ public AirbyteConnectionStatus check(final JsonNode config) { try { final JdbcDatabase database = new DefaultJdbcDatabase(dataSource); final String outputSchema = super.getNamingResolver().getIdentifier(config.get(JdbcUtils.SCHEMA_KEY).asText()); - attemptSQLCreateAndDropTableOperations(outputSchema, database, nameTransformer, redshiftS3StagingSqlOperations); + attemptTableOperations(outputSchema, database, nameTransformer, redshiftS3StagingSqlOperations, false); + RedshiftUtil.checkSvvTableAccess(database); return new AirbyteConnectionStatus().withStatus(AirbyteConnectionStatus.Status.SUCCEEDED); } catch (final ConnectionErrorException e) { final String message = getErrorMessage(e.getStateCode(), e.getErrorCode(), e.getExceptionMessage(), e); @@ -117,7 +135,8 @@ public DataSource getDataSource(final JsonNode config) { jdbcConfig.has(JdbcUtils.PASSWORD_KEY) ? jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, RedshiftInsertDestination.DRIVER_CLASS, jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), - SSL_JDBC_PARAMETERS); + getDefaultConnectionProperties(config), + Duration.ofMinutes(2)); } @Override @@ -127,7 +146,23 @@ protected NamingConventionTransformer getNamingResolver() { @Override protected Map getDefaultConnectionProperties(final JsonNode config) { - return SSL_JDBC_PARAMETERS; + // TODO: Pull common code from RedshiftInsertDestination and RedshiftStagingS3Destination into a + // base class. + // The following properties can be overriden through jdbcUrlParameters in the config. + final Map connectionOptions = new HashMap<>(); + // Redshift properties + // https://docs.aws.amazon.com/redshift/latest/mgmt/jdbc20-configuration-options.html#jdbc20-connecttimeout-option + // connectTimeout is different from Hikari pool's connectionTimout, driver defaults to 10seconds so + // increase it to match hikari's default + connectionOptions.put("connectTimeout", "120"); + // HikariPool properties + // https://github.com/brettwooldridge/HikariCP?tab=readme-ov-file#frequently-used + // connectionTimeout is set explicitly to 2 minutes when creating data source. + // Do aggressive keepAlive with minimum allowed value, this only applies to connection sitting idle + // in the pool. + connectionOptions.put("keepaliveTime", Long.toString(Duration.ofSeconds(30).toMillis())); + connectionOptions.putAll(SSL_JDBC_PARAMETERS); + return connectionOptions; } // this is a no op since we override getDatabase. @@ -136,6 +171,16 @@ public JsonNode toJdbcConfig(final JsonNode config) { return Jsons.emptyObject(); } + @Override + protected JdbcSqlGenerator getSqlGenerator() { + return new RedshiftSqlGenerator(getNamingResolver()); + } + + @Override + protected JdbcDestinationHandler getDestinationHandler(final String databaseName, final JdbcDatabase database) { + return new RedshiftDestinationHandler(databaseName, database); + } + @Override @Deprecated public AirbyteMessageConsumer getConsumer(final JsonNode config, @@ -145,9 +190,9 @@ public AirbyteMessageConsumer getConsumer(final JsonNode config, } @Override - public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(JsonNode config, - ConfiguredAirbyteCatalog catalog, - Consumer outputRecordCollector) + public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final Consumer outputRecordCollector) throws Exception { final EncryptionConfig encryptionConfig = config.has(UPLOADING_METHOD) ? EncryptionConfig.fromJson(config.get(UPLOADING_METHOD).get(JdbcUtils.ENCRYPTION_KEY)) : new NoEncryption(); @@ -161,21 +206,69 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(JsonNode co of streams {} this will create more buffers than necessary, leading to nonexistent gains """, FileBuffer.SOFT_CAP_CONCURRENT_STREAM_IN_BUFFER, catalog.getStreams().size()); } + // Short circuit old way of running things during transition. + if (!TypingAndDedupingFlag.isDestinationV2()) { + return new StagingConsumerFactory().createAsync( + outputRecordCollector, + getDatabase(getDataSource(config)), + new RedshiftS3StagingSqlOperations(getNamingResolver(), s3Config.getS3Client(), s3Config, encryptionConfig), + getNamingResolver(), + config, + catalog, + isPurgeStagingData(s3Options), + new TypeAndDedupeOperationValve(), + new NoopTyperDeduper(), + // The parsedcatalog is only used in v2 mode, so just pass null for now + null, + // Overwriting null namespace with null is perfectly safe + null, + // still using v1 table format + false); + } + final String defaultNamespace = config.get("schema").asText(); + for (final ConfiguredAirbyteStream stream : catalog.getStreams()) { + if (StringUtils.isEmpty(stream.getStream().getNamespace())) { + stream.getStream().setNamespace(defaultNamespace); + } + } + final RedshiftSqlGenerator sqlGenerator = new RedshiftSqlGenerator(getNamingResolver()); + final ParsedCatalog parsedCatalog; + final TyperDeduper typerDeduper; + final JdbcDatabase database = getDatabase(getDataSource(config)); + final String databaseName = config.get(JdbcUtils.DATABASE_KEY).asText(); + final RedshiftDestinationHandler redshiftDestinationHandler = new RedshiftDestinationHandler(databaseName, database); + final CatalogParser catalogParser; + if (TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE).isPresent()) { + catalogParser = new CatalogParser(sqlGenerator, TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE).get()); + } else { + catalogParser = new CatalogParser(sqlGenerator); + } + parsedCatalog = catalogParser.parseCatalog(catalog); + final JdbcV1V2Migrator migrator = new JdbcV1V2Migrator(getNamingResolver(), database, databaseName); + final NoopV2TableMigrator v2TableMigrator = new NoopV2TableMigrator(); + final boolean disableTypeDedupe = config.has(DISABLE_TYPE_DEDUPE) && config.get(DISABLE_TYPE_DEDUPE).asBoolean(false); + final int defaultThreadCount = 8; + if (disableTypeDedupe) { + typerDeduper = new NoOpTyperDeduperWithV1V2Migrations<>(sqlGenerator, redshiftDestinationHandler, parsedCatalog, migrator, v2TableMigrator, + defaultThreadCount); + } else { + typerDeduper = + new DefaultTyperDeduper<>(sqlGenerator, redshiftDestinationHandler, parsedCatalog, migrator, v2TableMigrator, defaultThreadCount); + } return new StagingConsumerFactory().createAsync( outputRecordCollector, - getDatabase(getDataSource(config)), + database, new RedshiftS3StagingSqlOperations(getNamingResolver(), s3Config.getS3Client(), s3Config, encryptionConfig), getNamingResolver(), config, catalog, isPurgeStagingData(s3Options), new TypeAndDedupeOperationValve(), - new NoopTyperDeduper(), - // The parsedcatalog is only used in v2 mode, so just pass null for now - null, - // Overwriting null namespace with null is perfectly safe - null); + typerDeduper, + parsedCatalog, + defaultNamespace, + true); } /** diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopier.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopier.java index d96a93a256e6..3a8f801c4689 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopier.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopier.java @@ -7,16 +7,16 @@ import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.copy.s3.S3CopyConfig; +import io.airbyte.cdk.integrations.destination.jdbc.copy.s3.S3StreamCopier; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.credential.S3AccessKeyCredentialConfig; import io.airbyte.commons.lang.Exceptions; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.copy.s3.S3CopyConfig; -import io.airbyte.integrations.destination.jdbc.copy.s3.S3StreamCopier; import io.airbyte.integrations.destination.redshift.manifest.Entry; import io.airbyte.integrations.destination.redshift.manifest.Manifest; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.credential.S3AccessKeyCredentialConfig; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.sql.Timestamp; import java.time.Instant; diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopierFactory.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopierFactory.java index 20f821e354a5..5527002288bc 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopierFactory.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopierFactory.java @@ -5,12 +5,12 @@ package io.airbyte.integrations.destination.redshift.copiers; import com.amazonaws.services.s3.AmazonS3; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopier; -import io.airbyte.integrations.destination.jdbc.copy.s3.S3CopyConfig; -import io.airbyte.integrations.destination.jdbc.copy.s3.S3StreamCopierFactory; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopier; +import io.airbyte.cdk.integrations.destination.jdbc.copy.s3.S3CopyConfig; +import io.airbyte.cdk.integrations.destination.jdbc.copy.s3.S3StreamCopierFactory; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; /** diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java index e4037eb033fa..9cf38f7ce4af 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftS3StagingSqlOperations.java @@ -6,19 +6,19 @@ import com.amazonaws.services.s3.AmazonS3; import com.fasterxml.jackson.databind.ObjectMapper; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.record_buffer.SerializableBuffer; +import io.airbyte.cdk.integrations.destination.s3.AesCbcEnvelopeEncryption; +import io.airbyte.cdk.integrations.destination.s3.AesCbcEnvelopeEncryptionBlobDecorator; +import io.airbyte.cdk.integrations.destination.s3.EncryptionConfig; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3StorageOperations; +import io.airbyte.cdk.integrations.destination.s3.credential.S3AccessKeyCredentialConfig; +import io.airbyte.cdk.integrations.destination.staging.StagingOperations; import io.airbyte.commons.lang.Exceptions; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.record_buffer.SerializableBuffer; import io.airbyte.integrations.destination.redshift.manifest.Entry; import io.airbyte.integrations.destination.redshift.manifest.Manifest; -import io.airbyte.integrations.destination.s3.AesCbcEnvelopeEncryption; -import io.airbyte.integrations.destination.s3.AesCbcEnvelopeEncryptionBlobDecorator; -import io.airbyte.integrations.destination.s3.EncryptionConfig; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.S3StorageOperations; -import io.airbyte.integrations.destination.s3.credential.S3AccessKeyCredentialConfig; -import io.airbyte.integrations.destination.staging.StagingOperations; import java.util.Base64; import java.util.Base64.Encoder; import java.util.List; @@ -53,19 +53,16 @@ public RedshiftS3StagingSqlOperations(final NamingConventionTransformer nameTran } @Override - public String getStageName(final String namespace, final String streamName) { - return nameTransformer.applyDefaultCase(String.join("_", - nameTransformer.convertStreamName(namespace), - nameTransformer.convertStreamName(streamName))); - } - - @Override - public String getStagingPath(final UUID connectionId, final String namespace, final String streamName, final DateTime writeDatetime) { + public String getStagingPath(final UUID connectionId, + final String namespace, + final String streamName, + final String outputTableName, + final DateTime writeDatetime) { final String bucketPath = s3Config.getBucketPath(); final String prefix = bucketPath.isEmpty() ? "" : bucketPath + (bucketPath.endsWith("/") ? "" : "/"); return nameTransformer.applyDefaultCase(String.format("%s%s/%s_%02d_%02d_%02d_%s/", prefix, - getStageName(namespace, streamName), + nameTransformer.applyDefaultCase(nameTransformer.convertStreamName(outputTableName)), writeDatetime.year().get(), writeDatetime.monthOfYear().get(), writeDatetime.dayOfMonth().get(), @@ -74,9 +71,7 @@ public String getStagingPath(final UUID connectionId, final String namespace, fi } @Override - public void createStageIfNotExists(final JdbcDatabase database, final String stageName) throws Exception { - final String bucketPath = s3Config.getBucketPath(); - final String prefix = bucketPath.isEmpty() ? "" : bucketPath + (bucketPath.endsWith("/") ? "" : "/"); + public void createStageIfNotExists() throws Exception { s3StorageOperations.createBucketIfNotExists(); } @@ -84,10 +79,9 @@ public void createStageIfNotExists(final JdbcDatabase database, final String sta public String uploadRecordsToStage(final JdbcDatabase database, final SerializableBuffer recordsData, final String schemaName, - final String stageName, final String stagingPath) throws Exception { - return s3StorageOperations.uploadRecordsToBucket(recordsData, schemaName, stageName, stagingPath); + return s3StorageOperations.uploadRecordsToBucket(recordsData, schemaName, stagingPath); } private String putManifest(final String manifestContents, final String stagingPath) { @@ -98,7 +92,6 @@ private String putManifest(final String manifestContents, final String stagingPa @Override public void copyIntoTableFromStage(final JdbcDatabase database, - final String stageName, final String stagingPath, final List stagedFiles, final String tableName, @@ -166,18 +159,9 @@ private static String getManifestPath(final String s3BucketName, final String s3 return "s3://" + s3BucketName + "/" + stagingPath + s3StagingFile; } - @Override - public void cleanUpStage(final JdbcDatabase database, final String stageName, final List stagedFiles) throws Exception { - final String bucketPath = s3Config.getBucketPath(); - final String prefix = bucketPath.isEmpty() ? "" : bucketPath + (bucketPath.endsWith("/") ? "" : "/"); - s3StorageOperations.cleanUpBucketObject(prefix + stageName, stagedFiles); - } - @Override public void dropStageIfExists(final JdbcDatabase database, final String stageName) throws Exception { - final String bucketPath = s3Config.getBucketPath(); - final String prefix = bucketPath.isEmpty() ? "" : bucketPath + (bucketPath.endsWith("/") ? "" : "/"); - s3StorageOperations.dropBucketObject(prefix + stageName); + s3StorageOperations.dropBucketObject(stageName); } } diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java index d114e3e5e3c9..7b85a0d92706 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java @@ -4,13 +4,35 @@ package io.airbyte.integrations.destination.redshift.operations; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.jdbc.JdbcSqlOperations; -import io.airbyte.integrations.destination.jdbc.SqlOperationsUtils; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_RAW_ID; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_DATA; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.function; +import static org.jooq.impl.DSL.name; +import static org.jooq.impl.DSL.table; +import static org.jooq.impl.DSL.using; +import static org.jooq.impl.DSL.val; + +import com.google.common.collect.Iterables; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.jdbc.JdbcSqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperationsUtils; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; import java.sql.SQLException; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; import java.util.List; +import java.util.UUID; +import org.jooq.DSLContext; +import org.jooq.InsertValuesStep4; +import org.jooq.Record; +import org.jooq.SQLDialect; +import org.jooq.impl.DefaultDataType; +import org.jooq.impl.SQLDataType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -22,8 +44,12 @@ public class RedshiftSqlOperations extends JdbcSqlOperations { public RedshiftSqlOperations() {} + private DSLContext getDslContext() { + return using(SQLDialect.POSTGRES); + } + @Override - public String createTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { + protected String createTableQueryV1(final String schemaName, final String tableName) { return String.format(""" CREATE TABLE IF NOT EXISTS %s.%s ( %s VARCHAR PRIMARY KEY, @@ -35,9 +61,21 @@ public String createTableQuery(final JdbcDatabase database, final String schemaN JavaBaseConstants.COLUMN_NAME_EMITTED_AT); } + @Override + protected String createTableQueryV2(final String schemaName, final String tableName) { + final DSLContext dsl = getDslContext(); + return dsl.createTableIfNotExists(name(schemaName, tableName)) + .column(COLUMN_NAME_AB_RAW_ID, SQLDataType.VARCHAR(36).nullable(false)) + .column(COLUMN_NAME_AB_EXTRACTED_AT, + SQLDataType.TIMESTAMPWITHTIMEZONE.defaultValue(function("GETDATE", SQLDataType.TIMESTAMPWITHTIMEZONE))) + .column(COLUMN_NAME_AB_LOADED_AT, SQLDataType.TIMESTAMPWITHTIMEZONE) + .column(COLUMN_NAME_DATA, new DefaultDataType<>(null, String.class, "super").nullable(false)) + .getSQL(); + } + @Override public void insertRecordsInternal(final JdbcDatabase database, - final List records, + final List records, final String schemaName, final String tmpTableName) throws SQLException { @@ -58,4 +96,55 @@ public void insertRecordsInternal(final JdbcDatabase database, SqlOperationsUtils.insertRawRecordsInSingleQuery(insertQueryComponent, recordQueryComponent, database, records); } + @Override + protected void insertRecordsInternalV2(final JdbcDatabase database, + final List records, + final String schemaName, + final String tableName) { + try { + database.execute(connection -> { + LOGGER.info("Total records received to insert: {}", records.size()); + // This comment was copied from DV1 code (SqlOperationsUtils.insertRawRecordsInSingleQuery): + // > We also partition the query to run on 10k records at a time, since some DBs set a max limit on + // > how many records can be inserted at once + // > TODO(sherif) this should use a smarter, destination-aware partitioning scheme instead of 10k by + // > default + for (final List batch : Iterables.partition(records, 10_000)) { + LOGGER.info("Prepared batch size: {}, {}, {}", batch.size(), schemaName, tableName); + final DSLContext create = using(connection, SQLDialect.POSTGRES); + // JOOQ adds some overhead here. Building the InsertValuesStep object takes about 139ms for 5K + // records. + // That's a nontrivial execution speed loss when the actual statement execution takes 500ms. + // Hopefully we're executing these statements infrequently enough in a sync that it doesn't matter. + // But this is a potential optimization if we need to eke out a little more performance on standard + // inserts. + // ... which presumably we won't, because standard inserts is so inherently slow. + // See + // https://github.com/airbytehq/airbyte/blob/f73827eb43f62ee30093451c434ad5815053f32d/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/operations/RedshiftSqlOperations.java#L39 + // and + // https://github.com/airbytehq/airbyte/blob/f73827eb43f62ee30093451c434ad5815053f32d/airbyte-cdk/java/airbyte-cdk/db-destinations/src/main/java/io/airbyte/cdk/integrations/destination/jdbc/SqlOperationsUtils.java#L62 + // for how DV1 did this in pure JDBC. + InsertValuesStep4 insert = create + .insertInto(table(name(schemaName, tableName)), + field(COLUMN_NAME_AB_RAW_ID, SQLDataType.VARCHAR(36)), + field(COLUMN_NAME_DATA, new DefaultDataType<>(null, String.class, "super")), + field(COLUMN_NAME_AB_EXTRACTED_AT, SQLDataType.TIMESTAMPWITHTIMEZONE), + field(COLUMN_NAME_AB_LOADED_AT, SQLDataType.TIMESTAMPWITHTIMEZONE)); + for (final PartialAirbyteMessage record : batch) { + insert = insert.values( + val(UUID.randomUUID().toString()), + function("JSON_PARSE", String.class, val(record.getSerialized())), + val(Instant.ofEpochMilli(record.getRecord().getEmittedAt()).atOffset(ZoneOffset.UTC)), + val((OffsetDateTime) null)); + } + insert.execute(); + LOGGER.info("Executed batch size: {}, {}, {}", batch.size(), schemaName, tableName); + } + }); + } catch (final Exception e) { + LOGGER.error("Error while inserting records", e); + throw new RuntimeException(e); + } + } + } diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.java new file mode 100644 index 000000000000..3fe5e6ecf32d --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftDestinationHandler.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.redshift.typing_deduping; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcDestinationHandler; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class RedshiftDestinationHandler extends JdbcDestinationHandler { + + public RedshiftDestinationHandler(final String databaseName, final JdbcDatabase jdbcDatabase) { + super(databaseName, jdbcDatabase); + } + + @Override + public void execute(final Sql sql) throws Exception { + final List> transactions = sql.transactions(); + final UUID queryId = UUID.randomUUID(); + for (final List transaction : transactions) { + final UUID transactionId = UUID.randomUUID(); + log.info("Executing sql {}-{}: {}", queryId, transactionId, String.join("\n", transaction)); + final long startTime = System.currentTimeMillis(); + + try { + // Original list is immutable, so copying it into a different list. + final List modifiedStatements = new ArrayList<>(); + // This is required for Redshift to retrieve Json path query with upper case characters, even after + // specifying quotes. + // see https://github.com/airbytehq/airbyte/issues/33900 + modifiedStatements.add("SET enable_case_sensitive_identifier to TRUE;\n"); + modifiedStatements.addAll(transaction); + jdbcDatabase.executeWithinTransaction(modifiedStatements); + } catch (final SQLException e) { + log.error("Sql {}-{} failed", queryId, transactionId, e); + throw e; + } + + log.info("Sql {}-{} completed in {} ms", queryId, transactionId, System.currentTimeMillis() - startTime); + } + } + + @Override + public boolean isFinalTableEmpty(final StreamId id) throws Exception { + // Redshift doesn't have an information_schema.tables table, so we have to use SVV_TABLE_INFO. + // From https://docs.aws.amazon.com/redshift/latest/dg/r_SVV_TABLE_INFO.html: + // > The SVV_TABLE_INFO view doesn't return any information for empty tables. + // So we just query for our specific table, and if we get no rows back, + // then we assume the table is empty. + // Note that because the column names are reserved words (table, schema, database), + // we need to enquote them. + final List query = jdbcDatabase.queryJsons( + """ + SELECT 1 + FROM SVV_TABLE_INFO + WHERE "database" = ? + AND "schema" = ? + AND "table" = ? + """, + databaseName, + id.finalNamespace(), + id.finalName()); + return query.isEmpty(); + } + +} diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGenerator.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGenerator.java new file mode 100644 index 000000000000..4c110d7a20cd --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGenerator.java @@ -0,0 +1,272 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.redshift.typing_deduping; + +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_AB_META; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_DATA; +import static org.jooq.impl.DSL.cast; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.function; +import static org.jooq.impl.DSL.name; +import static org.jooq.impl.DSL.quotedName; +import static org.jooq.impl.DSL.rowNumber; +import static org.jooq.impl.DSL.val; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.TableDefinition; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType; +import io.airbyte.integrations.base.destination.typing_deduping.Array; +import io.airbyte.integrations.base.destination.typing_deduping.ColumnId; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; +import io.airbyte.integrations.base.destination.typing_deduping.Struct; +import io.airbyte.integrations.base.destination.typing_deduping.Union; +import io.airbyte.integrations.base.destination.typing_deduping.UnsupportedOneOf; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.jooq.Condition; +import org.jooq.DataType; +import org.jooq.Field; +import org.jooq.SQLDialect; +import org.jooq.impl.DefaultDataType; +import org.jooq.impl.SQLDataType; + +public class RedshiftSqlGenerator extends JdbcSqlGenerator { + + public static final String CASE_STATEMENT_SQL_TEMPLATE = "CASE WHEN {0} THEN {1} ELSE {2} END "; + public static final String CASE_STATEMENT_NO_ELSE_SQL_TEMPLATE = "CASE WHEN {0} THEN {1} END "; + private static final Map REDSHIFT_TYPE_NAME_TO_JDBC_TYPE = ImmutableMap.of( + "numeric", "decimal", + "int8", "bigint", + "bool", "boolean", + "timestamptz", "timestamp with time zone", + "timetz", "time with time zone"); + private static final String COLUMN_ERROR_MESSAGE_FORMAT = "Problem with `%s`"; + private static final String AIRBYTE_META_COLUMN_ERRORS_KEY = "errors"; + + public RedshiftSqlGenerator(final NamingConventionTransformer namingTransformer) { + super(namingTransformer); + } + + /** + * This method returns Jooq internal DataType, Ideally we need to implement DataType interface with + * all the required fields for Jooq typed query construction + * + * @return + */ + private DataType getSuperType() { + return new DefaultDataType<>(null, String.class, "super"); + } + + @Override + protected DataType getStructType() { + return getSuperType(); + } + + @Override + protected DataType getArrayType() { + return getSuperType(); + } + + @Override + protected DataType getWidestType() { + return getSuperType(); + } + + @Override + protected SQLDialect getDialect() { + return SQLDialect.POSTGRES; + } + + /** + * Notes about Redshift specific SQL * 16MB Limit on the total size of the SQL sent in a session * + * Default mode of casting within SUPER is lax mode, to enable strict use SET + * cast_super_null_on_error='OFF'; * * + * https://docs.aws.amazon.com/redshift/latest/dg/super-configurations.html * + * https://docs.aws.amazon.com/redshift/latest/dg/r_MERGE.html#r_MERGE_usage_notes * * (Cannot use + * WITH clause in MERGE statement). + * https://cloud.google.com/bigquery/docs/migration/redshift-sql#merge_statement * * + * https://docs.aws.amazon.com/redshift/latest/dg/r_WITH_clause.html#r_WITH_clause-usage-notes * + * Primary keys are informational only and not enforced + * (https://docs.aws.amazon.com/redshift/latest/dg/t_Defining_constraints.html) TODO: Look at SORT + * KEYS, DISTKEY in redshift for optimizing the query performance. + */ + + @Override + protected Field castedField(final Field field, final AirbyteType type, final String alias, final boolean useExpensiveSaferCasting) { + if (type instanceof final AirbyteProtocolType airbyteProtocolType) { + switch (airbyteProtocolType) { + case STRING -> { + return field(CASE_STATEMENT_SQL_TEMPLATE, + jsonTypeOf(field).ne("string").and(field.isNotNull()), + jsonSerialize(field), + castedField(field, airbyteProtocolType, useExpensiveSaferCasting)).as(quotedName(alias)); + } + default -> { + return castedField(field, airbyteProtocolType, useExpensiveSaferCasting).as(quotedName(alias)); + } + } + + } + // Redshift SUPER can silently cast an array type to struct and vice versa. + return switch (type.getTypeName()) { + case Struct.TYPE, UnsupportedOneOf.TYPE -> field(CASE_STATEMENT_NO_ELSE_SQL_TEMPLATE, + jsonTypeOf(field).eq("object"), + cast(field, getStructType())).as(quotedName(alias)); + case Array.TYPE -> field(CASE_STATEMENT_NO_ELSE_SQL_TEMPLATE, + jsonTypeOf(field).eq("array"), + cast(field, getArrayType())).as(quotedName(alias)); + // No nested Unions supported so this will definitely not result in infinite recursion. + case Union.TYPE -> castedField(field, ((Union) type).chooseType(), alias, useExpensiveSaferCasting); + default -> throw new IllegalArgumentException("Unsupported AirbyteType: " + type); + }; + } + + @Override + protected List> extractRawDataFields(final LinkedHashMap columns, final boolean useExpensiveSaferCasting) { + return columns + .entrySet() + .stream() + .map(column -> castedField( + field(quotedName(COLUMN_NAME_DATA, column.getKey().originalName())), + column.getValue(), + column.getKey().name(), + useExpensiveSaferCasting)) + .collect(Collectors.toList()); + } + + private Field jsonTypeOf(final Field field) { + return function("JSON_TYPEOF", SQLDataType.VARCHAR, field); + } + + private Field jsonSerialize(final Field field) { + return function("JSON_SERIALIZE", SQLDataType.VARCHAR, field); + } + + /** + * Redshift ARRAY_CONCAT supports only 2 arrays. Iteratively nest ARRAY_CONCAT to support more than + * 2 + * + * @param arrays + * @return + */ + Field arrayConcatStmt(final List> arrays) { + if (arrays.isEmpty()) { + return field("ARRAY()"); // Return an empty string if the list is empty + } + + Field result = arrays.get(0); + String renderedSql = getDslContext().render(result); + for (int i = 1; i < arrays.size(); i++) { + // We lose some nice indentation but thats ok. Queryparts + // are intentionally rendered here to avoid deep stack for function sql rendering. + result = field(getDslContext().renderNamedOrInlinedParams(function("ARRAY_CONCAT", getSuperType(), result, arrays.get(i)))); + } + return result; + } + + Field toCastingErrorCaseStmt(final ColumnId column, final AirbyteType type) { + final Field field = field(quotedName(COLUMN_NAME_DATA, column.originalName())); + // Just checks if data is not null but casted data is null. This also accounts for conditional + // casting result of array and struct. + // TODO: Timestamp format issues can result in null values when cast, add regex check if destination + // supports regex functions. + return field(CASE_STATEMENT_SQL_TEMPLATE, + field.isNotNull().and(castedField(field, type, column.name(), true).isNull()), + function("ARRAY", getSuperType(), val(COLUMN_ERROR_MESSAGE_FORMAT.formatted(column.name()))), field("ARRAY()")); + } + + @Override + protected Field buildAirbyteMetaColumn(final LinkedHashMap columns) { + final List> dataFields = columns + .entrySet() + .stream() + .map(column -> toCastingErrorCaseStmt(column.getKey(), column.getValue())) + .collect(Collectors.toList()); + return function("OBJECT", getSuperType(), val(AIRBYTE_META_COLUMN_ERRORS_KEY), arrayConcatStmt(dataFields)).as(COLUMN_NAME_AB_META); + + } + + @Override + public boolean existingSchemaMatchesStreamConfig(final StreamConfig stream, final TableDefinition existingTable) { + // Check that the columns match, with special handling for the metadata columns. + // This is mostly identical to the redshift implementation, but swaps jsonb to super + final LinkedHashMap intendedColumns = stream.columns().entrySet().stream() + .collect(LinkedHashMap::new, + (map, column) -> map.put(column.getKey().name(), toDialectType(column.getValue()).getTypeName()), + LinkedHashMap::putAll); + final LinkedHashMap actualColumns = existingTable.columns().entrySet().stream() + .filter(column -> JavaBaseConstants.V2_FINAL_TABLE_METADATA_COLUMNS.stream() + .noneMatch(airbyteColumnName -> airbyteColumnName.equals(column.getKey()))) + .collect(LinkedHashMap::new, + (map, column) -> map.put(column.getKey(), jdbcTypeNameFromRedshiftTypeName(column.getValue().type())), + LinkedHashMap::putAll); + + final boolean sameColumns = actualColumns.equals(intendedColumns) + && "varchar".equals(existingTable.columns().get(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID).type()) + && "timestamptz".equals(existingTable.columns().get(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT).type()) + && "super".equals(existingTable.columns().get(JavaBaseConstants.COLUMN_NAME_AB_META).type()); + + return sameColumns; + } + + /** + * Return ROW_NUMBER() OVER (PARTITION BY primaryKeys ORDER BY cursor DESC NULLS LAST, + * _airbyte_extracted_at DESC) + * + * @param primaryKeys + * @param cursor + * @return + */ + @Override + protected Field getRowNumber(final List primaryKeys, final Optional cursor) { + // literally identical to postgres's getRowNumber implementation, changes here probably should + // be reflected there + final List> primaryKeyFields = + primaryKeys != null ? primaryKeys.stream().map(columnId -> field(quotedName(columnId.name()))).collect(Collectors.toList()) + : new ArrayList<>(); + final List> orderedFields = new ArrayList<>(); + // We can still use Jooq's field to get the quoted name with raw sql templating. + // jooq's .desc returns SortField instead of Field and NULLS LAST doesn't work with it + cursor.ifPresent(columnId -> orderedFields.add(field("{0} desc NULLS LAST", field(quotedName(columnId.name()))))); + orderedFields.add(field("{0} desc", quotedName(COLUMN_NAME_AB_EXTRACTED_AT))); + return rowNumber() + .over() + .partitionBy(primaryKeyFields) + .orderBy(orderedFields).as(ROW_NUMBER_COLUMN_NAME); + } + + @Override + protected Condition cdcDeletedAtNotNullCondition() { + return field(name(COLUMN_NAME_AB_LOADED_AT)).isNotNull() + .and(function("JSON_TYPEOF", SQLDataType.VARCHAR, field(quotedName(COLUMN_NAME_DATA, cdcDeletedAtColumn.name()))) + .ne("null")); + } + + @Override + protected Field currentTimestamp() { + return function("GETDATE", SQLDataType.TIMESTAMP); + } + + @Override + public boolean shouldRetry(final Exception e) { + return false; + } + + private static String jdbcTypeNameFromRedshiftTypeName(final String redshiftType) { + return REDSHIFT_TYPE_NAME_TO_JDBC_TYPE.getOrDefault(redshiftType, redshiftType); + } + +} diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/util/RedshiftUtil.java b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/util/RedshiftUtil.java index c1433c4aa226..6551820a4831 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/util/RedshiftUtil.java +++ b/airbyte-integrations/connectors/destination-redshift/src/main/java/io/airbyte/integrations/destination/redshift/util/RedshiftUtil.java @@ -7,10 +7,13 @@ import static io.airbyte.integrations.destination.redshift.constants.RedshiftDestinationConstants.UPLOADING_METHOD; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import lombok.extern.log4j.Log4j2; /** * Helper class for Destination Redshift connector. */ +@Log4j2 public class RedshiftUtil { private RedshiftUtil() {} @@ -36,4 +39,9 @@ private static boolean isNullOrEmpty(final JsonNode jsonNode) { return null == jsonNode || "".equals(jsonNode.asText()); } + public static void checkSvvTableAccess(final JdbcDatabase database) throws Exception { + log.info("checking SVV_TABLE_INFO permissions"); + database.queryJsons("SELECT 1 FROM SVV_TABLE_INFO LIMIT 1;"); + } + } diff --git a/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json index 53c686b04f91..28a274642bbf 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-redshift/src/main/resources/spec.json @@ -15,6 +15,7 @@ "description": "Host Endpoint of the Redshift Cluster (must include the cluster-id, region and end with .redshift.amazonaws.com)", "type": "string", "title": "Host", + "group": "connection", "order": 1 }, "port": { @@ -25,12 +26,14 @@ "default": 5439, "examples": ["5439"], "title": "Port", + "group": "connection", "order": 2 }, "username": { "description": "Username to use to access the database.", "type": "string", "title": "Username", + "group": "connection", "order": 3 }, "password": { @@ -38,12 +41,14 @@ "type": "string", "airbyte_secret": true, "title": "Password", + "group": "connection", "order": 4 }, "database": { "description": "Name of the database.", "type": "string", "title": "Database", + "group": "connection", "order": 5 }, "schema": { @@ -51,6 +56,7 @@ "type": "string", "examples": ["public"], "default": "public", + "group": "connection", "title": "Default Schema", "order": 6 }, @@ -58,26 +64,20 @@ "title": "JDBC URL Params", "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", "type": "string", + "group": "connection", "order": 7 }, "uploading_method": { "title": "Uploading Method", "type": "object", - "description": "The method how the data will be uploaded to the database.", + "description": "The way data will be uploaded to Redshift.", + "group": "connection", "order": 8, + "display_type": "radio", "oneOf": [ { - "title": "Standard", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "Standard" - } - } - }, - { - "title": "S3 Staging", + "title": "AWS S3 Staging", + "description": "(recommended) Uploads data to S3 and then uses a COPY to insert the data into Redshift. COPY is recommended for production workloads for better speed and scalability. See AWS docs for more details.", "required": [ "method", "s3_bucket_name", @@ -93,46 +93,74 @@ "s3_bucket_name": { "title": "S3 Bucket Name", "type": "string", - "description": "The name of the staging S3 bucket to use if utilising a COPY strategy. COPY is recommended for production workloads for better speed and scalability. See AWS docs for more details.", - "examples": ["airbyte.staging"] + "description": "The name of the staging S3 bucket.", + "examples": ["airbyte.staging"], + "order": 0 }, "s3_bucket_path": { "title": "S3 Bucket Path", "type": "string", "description": "The directory under the S3 bucket where data will be written. If not provided, then defaults to the root directory. See path's name recommendations for more details.", - "examples": ["data_sync/test"] + "examples": ["data_sync/test"], + "order": 1 }, "s3_bucket_region": { "title": "S3 Bucket Region", "type": "string", "default": "", - "description": "The region of the S3 staging bucket to use if utilising a COPY strategy. See AWS docs for details.", + "description": "The region of the S3 staging bucket.", "enum": [ "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", "af-south-1", "ap-east-1", - "ap-south-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", + "ap-south-1", + "ap-south-2", "ap-southeast-1", "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", "ca-central-1", + "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", + "eu-central-2", "eu-north-1", "eu-south-1", + "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", + "il-central-1", + "me-central-1", + "me-south-1", + "sa-east-1", "sa-east-1", - "me-south-1" - ] + "us-east-1", + "us-east-2", + "us-gov-east-1", + "us-gov-west-1", + "us-west-1", + "us-west-2" + ], + "order": 2 + }, + "access_key_id": { + "type": "string", + "description": "This ID grants access to the above S3 staging bucket. Airbyte requires Read and Write permissions to the given bucket. See AWS docs on how to generate an access key ID and secret access key.", + "title": "S3 Access Key Id", + "airbyte_secret": true, + "order": 3 + }, + "secret_access_key": { + "type": "string", + "description": "The corresponding secret to the above access key id. See AWS docs on how to generate an access key ID and secret access key.", + "title": "S3 Secret Access Key", + "airbyte_secret": true, + "order": 4 }, "file_name_pattern": { "type": "string", @@ -145,25 +173,14 @@ "{part_number}", "{sync_id}" ], - "order": 8 - }, - "access_key_id": { - "type": "string", - "description": "This ID grants access to the above S3 staging bucket. Airbyte requires Read and Write permissions to the given bucket. See AWS docs on how to generate an access key ID and secret access key.", - "title": "S3 Key Id", - "airbyte_secret": true - }, - "secret_access_key": { - "type": "string", - "description": "The corresponding secret to the above access key id. See AWS docs on how to generate an access key ID and secret access key.", - "title": "S3 Access Key", - "airbyte_secret": true + "order": 5 }, "purge_staging_data": { "title": "Purge Staging Files and Tables", "type": "boolean", "description": "Whether to delete the staging files from S3 after completing the sync. See docs for details.", - "default": true + "default": true, + "order": 6 }, "encryption": { "title": "Encryption", @@ -205,7 +222,8 @@ } } } - ] + ], + "order": 7 }, "file_buffer_count": { "title": "File Buffer Count", @@ -217,9 +235,56 @@ "examples": ["10"] } } + }, + { + "title": "Standard", + "required": ["method"], + "description": "(not recommended) Direct loading using SQL INSERT statements. This method is extremely inefficient and provided only for quick testing. In all other cases, you should use S3 uploading.", + "properties": { + "method": { + "type": "string", + "const": "Standard" + } + } } ] + }, + "use_1s1t_format": { + "type": "boolean", + "description": "(Early Access) Use Destinations V2.", + "title": "Use Destinations V2 (Early Access)", + "order": 9, + "group": "connection" + }, + "raw_data_schema": { + "type": "string", + "description": "(Early Access) The schema to write raw tables into", + "title": "Destinations V2 Raw Table Schema (Early Access)", + "order": 10, + "group": "connection" + }, + "enable_incremental_final_table_updates": { + "type": "boolean", + "default": false, + "description": "When enabled your data will load into your final tables incrementally while your data is still being synced. When Disabled (the default), your data loads into your final tables once at the end of a sync. Note that this option only applies if you elect to create Final tables", + "title": "Enable Loading Data Incrementally to Final Tables (Early Access)", + "order": 11, + "group": "connection" + }, + "disable_type_dedupe": { + "type": "boolean", + "default": false, + "description": "Disable Writing Final Tables. WARNING! The data format in _airbyte_data is likely stable but there are no guarantees that other metadata columns will remain the same in future versions", + "title": "Disable Final Tables. (WARNING! Unstable option; Columns in raw table schema might change between versions) (Early Access)", + "order": 12, + "group": "connection" + } + }, + "groups": [ + { + "id": "connection", + "title": "Connection" } - } + ] } } diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftConnectionTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftConnectionTest.java new file mode 100644 index 000000000000..dfefbf0c0f10 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftConnectionTest.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.redshift; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; + +public class RedshiftConnectionTest { + + private final JsonNode config = Jsons.deserialize(IOs.readFile(Path.of("secrets/config.json"))); + private final RedshiftDestination destination = new RedshiftDestination(); + private AirbyteConnectionStatus status; + + @Test + void testCheckIncorrectPasswordFailure() throws Exception { + ((ObjectNode) config).put("password", "fake"); + status = destination.check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); + assertTrue(status.getMessage().contains("State code: 28000;")); + } + + @Test + public void testCheckIncorrectUsernameFailure() throws Exception { + ((ObjectNode) config).put("username", ""); + status = destination.check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); + assertTrue(status.getMessage().contains("State code: 28000;")); + } + + @Test + public void testCheckIncorrectHostFailure() throws Exception { + ((ObjectNode) config).put("host", "localhost2"); + status = destination.check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); + assertTrue(status.getMessage().contains("State code: 08001;")); + } + + @Test + public void testCheckIncorrectDataBaseFailure() throws Exception { + ((ObjectNode) config).put("database", "wrongdatabase"); + status = destination.check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); + assertTrue(status.getMessage().contains("State code: 3D000;")); + } + +} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftDestinationAcceptanceTest.java new file mode 100644 index 000000000000..b398c0b9e597 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftDestinationAcceptanceTest.java @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.redshift; + +import com.amazon.redshift.util.RedshiftTimestamp; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.ConnectionFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.TestingNamespaces; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.string.Strings; +import io.airbyte.integrations.destination.redshift.operations.RedshiftSqlOperations; +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +// these tests are not yet thread-safe, unlike the DV2 tests. +@Execution(ExecutionMode.SAME_THREAD) +public abstract class RedshiftDestinationAcceptanceTest extends JdbcDestinationAcceptanceTest { + + private static final Logger LOGGER = LoggerFactory.getLogger(RedshiftDestinationAcceptanceTest.class); + + // config from which to create / delete schemas. + private JsonNode baseConfig; + // config which refers to the schema that the test is being run in. + protected JsonNode config; + private final RedshiftSQLNameTransformer namingResolver = new RedshiftSQLNameTransformer(); + private final String USER_WITHOUT_CREDS = Strings.addRandomSuffix("test_user", "_", 5); + + private Database database; + private Connection connection; + protected TestDestinationEnv testDestinationEnv; + + @Override + protected String getImageName() { + return "airbyte/destination-redshift:dev"; + } + + @Override + protected JsonNode getConfig() { + return config; + } + + public abstract JsonNode getStaticConfig() throws IOException; + + @Override + protected JsonNode getFailCheckConfig() { + final JsonNode invalidConfig = Jsons.clone(config); + ((ObjectNode) invalidConfig).put("password", "wrong password"); + return invalidConfig; + } + + @Override + protected TestDataComparator getTestDataComparator() { + return new RedshiftTestDataComparator(); + } + + @Override + protected boolean supportBasicDataTypeTest() { + return true; + } + + @Override + protected boolean supportArrayDataTypeTest() { + return true; + } + + @Override + protected boolean supportObjectDataTypeTest() { + return true; + } + + @Override + protected boolean supportIncrementalSchemaChanges() { + return true; + } + + @Override + protected boolean supportsInDestinationNormalization() { + return true; + } + + @Override + protected List retrieveRecords(final TestDestinationEnv env, + final String streamName, + final String namespace, + final JsonNode streamSchema) + throws Exception { + return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) + .stream() + .map(j -> j.get(JavaBaseConstants.COLUMN_NAME_DATA)) + .collect(Collectors.toList()); + } + + @Override + protected boolean implementsNamespaces() { + return true; + } + + @Override + protected List retrieveNormalizedRecords(final TestDestinationEnv testEnv, final String streamName, final String namespace) + throws Exception { + String tableName = namingResolver.getIdentifier(streamName); + if (!tableName.startsWith("\"")) { + // Currently, Normalization always quote tables identifiers + tableName = "\"" + tableName + "\""; + } + return retrieveRecordsFromTable(tableName, namespace); + } + + private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { + return getDatabase().query( + ctx -> ctx + .fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) + .stream() + .map(record -> getJsonFromRecord( + record, + value -> { + if (value instanceof final RedshiftTimestamp rts) { + // We can't just use rts.toInstant().toString(), because that will mangle historical + // dates (e.g. 1504-02-28...) because toInstant() just converts to epoch millis, + // which works _very badly_ for for very old dates. + // Instead, convert to a string and then parse that string. + // We can't just rts.toString(), because that loses the timezone... + // so instead we use getPostgresqlString and parse that >.> + // Thanks, redshift. + return Optional.of( + ZonedDateTime.parse( + rts.getPostgresqlString(), + new DateTimeFormatterBuilder() + .appendPattern("yyyy-MM-dd HH:mm:ss") + .optionalStart() + .appendFraction(ChronoField.MILLI_OF_SECOND, 0, 9, true) + .optionalEnd() + .appendPattern("X") + .toFormatter()) + .withZoneSameInstant(ZoneOffset.UTC) + .toString()); + } else { + return Optional.empty(); + } + })) + .collect(Collectors.toList())); + } + + // for each test we create a new schema in the database. run the test in there and then remove it. + @Override + protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws Exception { + final String schemaName = TestingNamespaces.generate(); + final String createSchemaQuery = String.format("CREATE SCHEMA %s", schemaName); + baseConfig = getStaticConfig(); + database = createDatabase(); + removeOldNamespaces(); + getDatabase().query(ctx -> ctx.execute(createSchemaQuery)); + final String createUser = String.format("create user %s with password '%s' SESSION TIMEOUT 60;", + USER_WITHOUT_CREDS, baseConfig.get("password").asText()); + getDatabase().query(ctx -> ctx.execute(createUser)); + final JsonNode configForSchema = Jsons.clone(baseConfig); + ((ObjectNode) configForSchema).put("schema", schemaName); + TEST_SCHEMAS.add(schemaName); + config = configForSchema; + testDestinationEnv = testEnv; + } + + private void removeOldNamespaces() { + final List schemas; + try { + schemas = getDatabase().query(ctx -> ctx.fetch("SELECT schema_name FROM information_schema.schemata;")) + .stream() + .map(record -> record.get("schema_name").toString()) + .toList(); + } catch (final SQLException e) { + // if we can't fetch the schemas, just return. + return; + } + + int schemasDeletedCount = 0; + for (final String schema : schemas) { + if (TestingNamespaces.isOlderThan2Days(schema)) { + try { + getDatabase().query(ctx -> ctx.execute(String.format("DROP SCHEMA IF EXISTS %s CASCADE", schema))); + schemasDeletedCount++; + } catch (final SQLException e) { + LOGGER.error("Failed to delete old dataset: {}", schema, e); + } + } + } + LOGGER.info("Deleted {} old schemas.", schemasDeletedCount); + } + + @Override + protected void tearDown(final TestDestinationEnv testEnv) throws Exception { + System.out.println("TEARING_DOWN_SCHEMAS: " + TEST_SCHEMAS); + getDatabase().query(ctx -> ctx.execute(String.format("DROP SCHEMA IF EXISTS %s CASCADE", config.get("schema").asText()))); + for (final String schema : TEST_SCHEMAS) { + getDatabase().query(ctx -> ctx.execute(String.format("DROP SCHEMA IF EXISTS %s CASCADE", schema))); + } + getDatabase().query(ctx -> ctx.execute(String.format("drop user if exists %s;", USER_WITHOUT_CREDS))); + RedshiftConnectionHandler.close(connection); + } + + protected Database createDatabase() { + connection = ConnectionFactory.create(baseConfig.get(JdbcUtils.USERNAME_KEY).asText(), + baseConfig.get(JdbcUtils.PASSWORD_KEY).asText(), + RedshiftInsertDestination.SSL_JDBC_PARAMETERS, + String.format(DatabaseDriver.REDSHIFT.getUrlFormatString(), + baseConfig.get(JdbcUtils.HOST_KEY).asText(), + baseConfig.get(JdbcUtils.PORT_KEY).asInt(), + baseConfig.get(JdbcUtils.DATABASE_KEY).asText())); + + return new Database(DSL.using(connection)); + } + + protected Database getDatabase() { + return database; + } + + @Override + protected int getMaxRecordValueLimit() { + return RedshiftSqlOperations.REDSHIFT_VARCHAR_MAX_BYTE_SIZE; + } + + @Override + protected int getGenerateBigStringAddExtraCharacters() { + return 1; + } + +} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftFileBufferTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftFileBufferTest.java new file mode 100644 index 000000000000..bbeab71e6be0 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftFileBufferTest.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.redshift; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.destination.record_buffer.FileBuffer; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import java.nio.file.Path; +import org.junit.jupiter.api.Test; + +public class RedshiftFileBufferTest { + + private final JsonNode config = Jsons.deserialize(IOs.readFile(Path.of("secrets/config_staging.json"))); + private final RedshiftStagingS3Destination destination = new RedshiftStagingS3Destination(); + + @Test + public void testGetFileBufferDefault() { + assertEquals(destination.getNumberOfFileBuffers(config), FileBuffer.DEFAULT_MAX_CONCURRENT_STREAM_IN_BUFFER); + } + + @Test + public void testGetFileBufferMaxLimited() { + ((ObjectNode) config).put(FileBuffer.FILE_BUFFER_COUNT_KEY, 100); + assertEquals(destination.getNumberOfFileBuffers(config), FileBuffer.MAX_CONCURRENT_STREAM_IN_BUFFER); + } + + @Test + public void testGetMinimumFileBufferCount() { + ((ObjectNode) config).put(FileBuffer.FILE_BUFFER_COUNT_KEY, 1); + // User cannot set number of file counts below the default file buffer count, which is existing + // behavior + assertEquals(destination.getNumberOfFileBuffers(config), FileBuffer.DEFAULT_MAX_CONCURRENT_STREAM_IN_BUFFER); + } + +} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestinationAcceptanceTest.java index 57f61b4f39f2..6ca0a17fce3c 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftInsertDestinationAcceptanceTest.java @@ -13,7 +13,7 @@ /** * Integration test testing the {@link RedshiftInsertDestination}. */ -public class RedshiftInsertDestinationAcceptanceTest extends RedshiftStagingS3DestinationAcceptanceTest { +public class RedshiftInsertDestinationAcceptanceTest extends RedshiftDestinationAcceptanceTest { public JsonNode getStaticConfig() throws IOException { return Jsons.deserialize(Files.readString(Path.of("secrets/config.json"))); diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftS3StagingInsertDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftS3StagingInsertDestinationAcceptanceTest.java index 52cd07ce1748..fc054be51232 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftS3StagingInsertDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftS3StagingInsertDestinationAcceptanceTest.java @@ -9,7 +9,11 @@ import io.airbyte.commons.json.Jsons; import java.nio.file.Path; -public class RedshiftS3StagingInsertDestinationAcceptanceTest extends RedshiftStagingS3DestinationAcceptanceTest { +/** + * Integration test testing {@link RedshiftStagingS3Destination}. The default Redshift integration + * test credentials contain S3 credentials - this automatically causes COPY to be selected. + */ +public class RedshiftS3StagingInsertDestinationAcceptanceTest extends RedshiftDestinationAcceptanceTest { public JsonNode getStaticConfig() { return Jsons.deserialize(IOs.readFile(Path.of("secrets/config_staging.json"))); diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3DestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3DestinationAcceptanceTest.java deleted file mode 100644 index ad4f7dafd755..000000000000 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftStagingS3DestinationAcceptanceTest.java +++ /dev/null @@ -1,296 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.destination.redshift; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.commons.io.IOs; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.string.Strings; -import io.airbyte.db.Database; -import io.airbyte.db.factory.ConnectionFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.record_buffer.FileBuffer; -import io.airbyte.integrations.destination.redshift.operations.RedshiftSqlOperations; -import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.TestingNamespaces; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import java.io.IOException; -import java.nio.file.Path; -import java.sql.Connection; -import java.sql.SQLException; -import java.util.HashSet; -import java.util.List; -import java.util.stream.Collectors; -import org.jooq.impl.DSL; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Integration test testing {@link RedshiftStagingS3Destination}. The default Redshift integration - * test credentials contain S3 credentials - this automatically causes COPY to be selected. - */ -public abstract class RedshiftStagingS3DestinationAcceptanceTest extends JdbcDestinationAcceptanceTest { - - private static final Logger LOGGER = LoggerFactory.getLogger(RedshiftStagingS3DestinationAcceptanceTest.class); - - // config from which to create / delete schemas. - private JsonNode baseConfig; - // config which refers to the schema that the test is being run in. - protected JsonNode config; - private final RedshiftSQLNameTransformer namingResolver = new RedshiftSQLNameTransformer(); - private final String USER_WITHOUT_CREDS = Strings.addRandomSuffix("test_user", "_", 5); - - private Database database; - private Connection connection; - protected TestDestinationEnv testDestinationEnv; - - private final ObjectMapper mapper = new ObjectMapper(); - - @Override - protected String getImageName() { - return "airbyte/destination-redshift:dev"; - } - - @Override - protected JsonNode getConfig() { - return config; - } - - public JsonNode getStaticConfig() throws IOException { - return Jsons.deserialize(IOs.readFile(Path.of("secrets/config_staging.json"))); - } - - @Override - protected JsonNode getFailCheckConfig() { - final JsonNode invalidConfig = Jsons.clone(config); - ((ObjectNode) invalidConfig).put("password", "wrong password"); - return invalidConfig; - } - - @Test - void testCheckIncorrectPasswordFailure() throws Exception { - final JsonNode invalidConfig = Jsons.clone(config); - ((ObjectNode) invalidConfig).put("password", "fake"); - final RedshiftDestination destination = new RedshiftDestination(); - final AirbyteConnectionStatus status = destination.check(invalidConfig); - assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 28000; Error code: 500310;")); - } - - @Test - public void testCheckIncorrectUsernameFailure() throws Exception { - final JsonNode invalidConfig = Jsons.clone(config); - ((ObjectNode) invalidConfig).put("username", ""); - final RedshiftDestination destination = new RedshiftDestination(); - final AirbyteConnectionStatus status = destination.check(invalidConfig); - assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 28000; Error code: 500310;")); - } - - @Test - public void testCheckIncorrectHostFailure() throws Exception { - final JsonNode invalidConfig = Jsons.clone(config); - ((ObjectNode) invalidConfig).put("host", "localhost2"); - final RedshiftDestination destination = new RedshiftDestination(); - final AirbyteConnectionStatus status = destination.check(invalidConfig); - assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: HY000; Error code: 500150;")); - } - - @Test - public void testCheckIncorrectDataBaseFailure() throws Exception { - final JsonNode invalidConfig = Jsons.clone(config); - ((ObjectNode) invalidConfig).put("database", "wrongdatabase"); - final RedshiftDestination destination = new RedshiftDestination(); - final AirbyteConnectionStatus status = destination.check(invalidConfig); - assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 3D000; Error code: 500310;")); - } - - /* - * FileBuffer Default Tests - */ - @Test - public void testGetFileBufferDefault() { - final RedshiftStagingS3Destination destination = new RedshiftStagingS3Destination(); - assertEquals(destination.getNumberOfFileBuffers(config), FileBuffer.DEFAULT_MAX_CONCURRENT_STREAM_IN_BUFFER); - } - - @Test - public void testGetFileBufferMaxLimited() { - final JsonNode defaultConfig = Jsons.clone(config); - ((ObjectNode) defaultConfig).put(FileBuffer.FILE_BUFFER_COUNT_KEY, 100); - final RedshiftStagingS3Destination destination = new RedshiftStagingS3Destination(); - assertEquals(destination.getNumberOfFileBuffers(defaultConfig), FileBuffer.MAX_CONCURRENT_STREAM_IN_BUFFER); - } - - @Test - public void testGetMinimumFileBufferCount() { - final JsonNode defaultConfig = Jsons.clone(config); - ((ObjectNode) defaultConfig).put(FileBuffer.FILE_BUFFER_COUNT_KEY, 1); - final RedshiftStagingS3Destination destination = new RedshiftStagingS3Destination(); - // User cannot set number of file counts below the default file buffer count, which is existing - // behavior - assertEquals(destination.getNumberOfFileBuffers(defaultConfig), FileBuffer.DEFAULT_MAX_CONCURRENT_STREAM_IN_BUFFER); - } - - @Override - protected TestDataComparator getTestDataComparator() { - return new RedshiftTestDataComparator(); - } - - @Override - protected boolean supportBasicDataTypeTest() { - return true; - } - - @Override - protected boolean supportArrayDataTypeTest() { - return true; - } - - @Override - protected boolean supportObjectDataTypeTest() { - return true; - } - - @Override - protected boolean supportIncrementalSchemaChanges() { - return true; - } - - @Override - protected List retrieveRecords(final TestDestinationEnv env, - final String streamName, - final String namespace, - final JsonNode streamSchema) - throws Exception { - return retrieveRecordsFromTable(namingResolver.getRawTableName(streamName), namespace) - .stream() - .map(j -> j.get(JavaBaseConstants.COLUMN_NAME_DATA)) - .collect(Collectors.toList()); - } - - @Override - protected boolean implementsNamespaces() { - return true; - } - - @Override - protected List retrieveNormalizedRecords(final TestDestinationEnv testEnv, final String streamName, final String namespace) - throws Exception { - String tableName = namingResolver.getIdentifier(streamName); - if (!tableName.startsWith("\"")) { - // Currently, Normalization always quote tables identifiers - tableName = "\"" + tableName + "\""; - } - return retrieveRecordsFromTable(tableName, namespace); - } - - private List retrieveRecordsFromTable(final String tableName, final String schemaName) throws SQLException { - return getDatabase().query( - ctx -> ctx - .fetch(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schemaName, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT)) - .stream() - .map(this::getJsonFromRecord) - .collect(Collectors.toList())); - } - - // for each test we create a new schema in the database. run the test in there and then remove it. - @Override - protected void setup(final TestDestinationEnv testEnv, final HashSet TEST_SCHEMAS) throws Exception { - final String schemaName = TestingNamespaces.generate(); - final String createSchemaQuery = String.format("CREATE SCHEMA %s", schemaName); - baseConfig = getStaticConfig(); - database = createDatabase(); - removeOldNamespaces(); - getDatabase().query(ctx -> ctx.execute(createSchemaQuery)); - final String createUser = String.format("create user %s with password '%s' SESSION TIMEOUT 60;", - USER_WITHOUT_CREDS, baseConfig.get("password").asText()); - getDatabase().query(ctx -> ctx.execute(createUser)); - final JsonNode configForSchema = Jsons.clone(baseConfig); - ((ObjectNode) configForSchema).put("schema", schemaName); - TEST_SCHEMAS.add(schemaName); - config = configForSchema; - testDestinationEnv = testEnv; - } - - private void removeOldNamespaces() { - final List schemas; - try { - schemas = getDatabase().query(ctx -> ctx.fetch("SELECT schema_name FROM information_schema.schemata;")) - .stream() - .map(record -> record.get("schema_name").toString()) - .toList(); - } catch (final SQLException e) { - // if we can't fetch the schemas, just return. - return; - } - - int schemasDeletedCount = 0; - for (final String schema : schemas) { - if (TestingNamespaces.isOlderThan2Days(schema)) { - try { - getDatabase().query(ctx -> ctx.execute(String.format("DROP SCHEMA IF EXISTS %s CASCADE", schema))); - schemasDeletedCount++; - } catch (final SQLException e) { - LOGGER.error("Failed to delete old dataset: {}", schema, e); - } - } - } - LOGGER.info("Deleted {} old schemas.", schemasDeletedCount); - } - - @Override - protected void tearDown(final TestDestinationEnv testEnv) throws Exception { - System.out.println("TEARING_DOWN_SCHEMAS: " + TEST_SCHEMAS); - getDatabase().query(ctx -> ctx.execute(String.format("DROP SCHEMA IF EXISTS %s CASCADE", config.get("schema").asText()))); - for (final String schema : TEST_SCHEMAS) { - getDatabase().query(ctx -> ctx.execute(String.format("DROP SCHEMA IF EXISTS %s CASCADE", schema))); - } - getDatabase().query(ctx -> ctx.execute(String.format("drop user if exists %s;", USER_WITHOUT_CREDS))); - RedshiftConnectionHandler.close(connection); - } - - protected Database createDatabase() { - connection = ConnectionFactory.create(baseConfig.get(JdbcUtils.USERNAME_KEY).asText(), - baseConfig.get(JdbcUtils.PASSWORD_KEY).asText(), - RedshiftInsertDestination.SSL_JDBC_PARAMETERS, - String.format(DatabaseDriver.REDSHIFT.getUrlFormatString(), - baseConfig.get(JdbcUtils.HOST_KEY).asText(), - baseConfig.get(JdbcUtils.PORT_KEY).asInt(), - baseConfig.get(JdbcUtils.DATABASE_KEY).asText())); - - return new Database(DSL.using(connection)); - } - - protected Database getDatabase() { - return database; - } - - public RedshiftSQLNameTransformer getNamingResolver() { - return namingResolver; - } - - @Override - protected int getMaxRecordValueLimit() { - return RedshiftSqlOperations.REDSHIFT_VARCHAR_MAX_BYTE_SIZE; - } - - @Override - protected int getGenerateBigStringAddExtraCharacters() { - return 1; - } - -} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftTestDataComparator.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftTestDataComparator.java index 05a59cfda564..1a5e00c36567 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftTestDataComparator.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/RedshiftTestDataComparator.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.redshift; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import java.time.LocalDate; import java.time.ZoneOffset; import java.time.ZonedDateTime; @@ -40,14 +40,14 @@ protected List resolveIdentifier(final String identifier) { protected boolean compareDateTimeWithTzValues(final String airbyteMessageValue, final String destinationValue) { try { - final ZonedDateTime airbyteDate = ZonedDateTime.parse(airbyteMessageValue, - getAirbyteDateTimeWithTzFormatter()).withZoneSameInstant( - ZoneOffset.UTC); + final ZonedDateTime airbyteDate = ZonedDateTime.parse( + airbyteMessageValue, + getAirbyteDateTimeWithTzFormatter()).withZoneSameInstant(ZoneOffset.UTC); - final ZonedDateTime destinationDate = ZonedDateTime.parse(destinationValue, - DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ssX")); + final ZonedDateTime destinationDate = ZonedDateTime.parse(destinationValue) + .withZoneSameInstant(ZoneOffset.UTC); return airbyteDate.equals(destinationDate); - } catch (DateTimeParseException e) { + } catch (final DateTimeParseException e) { LOGGER.warn( "Fail to convert values to ZonedDateTime. Try to compare as text. Airbyte value({}), Destination value ({}). Exception: {}", airbyteMessageValue, destinationValue, e); @@ -56,14 +56,14 @@ protected boolean compareDateTimeWithTzValues(final String airbyteMessageValue, } @Override - protected boolean compareDateTimeValues(String expectedValue, String actualValue) { - var destinationDate = parseLocalDateTime(actualValue); - var expectedDate = LocalDate.parse(expectedValue, + protected boolean compareDateTimeValues(final String expectedValue, final String actualValue) { + final var destinationDate = parseLocalDateTime(actualValue); + final var expectedDate = LocalDate.parse(expectedValue, DateTimeFormatter.ofPattern(AIRBYTE_DATETIME_FORMAT)); return expectedDate.equals(destinationDate); } - private LocalDate parseLocalDateTime(String dateTimeValue) { + private LocalDate parseLocalDateTime(final String dateTimeValue) { if (dateTimeValue != null) { return LocalDate.parse(dateTimeValue, DateTimeFormatter.ofPattern(getFormat(dateTimeValue))); @@ -72,7 +72,7 @@ private LocalDate parseLocalDateTime(String dateTimeValue) { } } - private String getFormat(String dateTimeValue) { + private String getFormat(final String dateTimeValue) { if (dateTimeValue.contains("T")) { // MySql stores array of objects as a jsonb type, i.e. array of string for all cases return AIRBYTE_DATETIME_FORMAT; diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshKeyRedshiftInsertDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshKeyRedshiftInsertDestinationAcceptanceTest.java index 9a0a7f812e6b..1d80069433da 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshKeyRedshiftInsertDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshKeyRedshiftInsertDestinationAcceptanceTest.java @@ -5,9 +5,9 @@ package io.airbyte.integrations.destination.redshift; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel.TunnelMethod; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.ssh.SshTunnel.TunnelMethod; import java.io.IOException; import java.nio.file.Path; diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshPasswordRedshiftStagingDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshPasswordRedshiftStagingDestinationAcceptanceTest.java index 9747cec07c32..6f423e5e43d1 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshPasswordRedshiftStagingDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshPasswordRedshiftStagingDestinationAcceptanceTest.java @@ -5,9 +5,9 @@ package io.airbyte.integrations.destination.redshift; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel.TunnelMethod; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.ssh.SshTunnel.TunnelMethod; import java.io.IOException; import java.nio.file.Path; diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshRedshiftDestinationBaseAcceptanceTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshRedshiftDestinationBaseAcceptanceTest.java index bd0d9f6639c0..22ef33b5b098 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshRedshiftDestinationBaseAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/SshRedshiftDestinationBaseAcceptanceTest.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.redshift; -import static io.airbyte.integrations.base.ssh.SshTunnel.TunnelMethod.SSH_KEY_AUTH; -import static io.airbyte.integrations.base.ssh.SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH; +import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.TunnelMethod.SSH_KEY_AUTH; +import static io.airbyte.cdk.integrations.base.ssh.SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; @@ -13,19 +13,19 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap.Builder; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.ConnectionFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.TestingNamespaces; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.Database; -import io.airbyte.db.factory.ConnectionFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.base.ssh.SshTunnel; import io.airbyte.integrations.destination.redshift.operations.RedshiftSqlOperations; -import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.TestingNamespaces; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.io.IOException; import java.sql.Connection; import java.util.HashSet; diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/AbstractRedshiftTypingDedupingTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/AbstractRedshiftTypingDedupingTest.java new file mode 100644 index 000000000000..c5928ab534d6 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/AbstractRedshiftTypingDedupingTest.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.redshift.typing_deduping; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.JdbcCompatibleSourceOperations; +import io.airbyte.cdk.integrations.standardtest.destination.typing_deduping.JdbcTypingDedupingTest; +import io.airbyte.integrations.base.destination.typing_deduping.SqlGenerator; +import io.airbyte.integrations.destination.redshift.RedshiftInsertDestination; +import io.airbyte.integrations.destination.redshift.RedshiftSQLNameTransformer; +import io.airbyte.integrations.destination.redshift.typing_deduping.RedshiftSqlGeneratorIntegrationTest.RedshiftSourceOperations; +import javax.sql.DataSource; +import org.jooq.DSLContext; +import org.jooq.conf.Settings; +import org.jooq.impl.DSL; + +public abstract class AbstractRedshiftTypingDedupingTest extends JdbcTypingDedupingTest { + + @Override + protected String getImageName() { + return "airbyte/destination-redshift:dev"; + } + + @Override + protected DataSource getDataSource(final JsonNode config) { + return new RedshiftInsertDestination().getDataSource(config); + } + + @Override + protected JdbcCompatibleSourceOperations getSourceOperations() { + return new RedshiftSourceOperations(); + } + + @Override + protected SqlGenerator getSqlGenerator() { + return new RedshiftSqlGenerator(new RedshiftSQLNameTransformer()) { + + // Override only for tests to print formatted SQL. The actual implementation should use unformatted + // to save bytes. + @Override + protected DSLContext getDslContext() { + return DSL.using(getDialect(), new Settings().withRenderFormatted(true)); + } + + }; + } + +} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftS3StagingRawSchemaOverrideDisableTypingDedupingTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftS3StagingRawSchemaOverrideDisableTypingDedupingTest.java new file mode 100644 index 000000000000..972cde0a1a58 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftS3StagingRawSchemaOverrideDisableTypingDedupingTest.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.redshift.typing_deduping; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import java.nio.file.Path; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +public class RedshiftS3StagingRawSchemaOverrideDisableTypingDedupingTest extends AbstractRedshiftTypingDedupingTest { + + @Override + protected ObjectNode getBaseConfig() { + return (ObjectNode) Jsons.deserialize(IOs.readFile(Path.of("secrets/1s1t_config_staging_raw_schema_override.json"))); + } + + @Override + protected String getRawSchema() { + return "overridden_raw_dataset"; + } + + @Override + protected boolean disableFinalTableComparison() { + return true; + } + + @Disabled + @Test + @Override + public void identicalNameSimultaneousSync() {} + +} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftS3StagingTypingDedupingTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftS3StagingTypingDedupingTest.java new file mode 100644 index 000000000000..c38182ffa54a --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftS3StagingTypingDedupingTest.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.redshift.typing_deduping; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import java.nio.file.Path; + +public class RedshiftS3StagingTypingDedupingTest extends AbstractRedshiftTypingDedupingTest { + + @Override + protected ObjectNode getBaseConfig() { + return (ObjectNode) Jsons.deserialize(IOs.readFile(Path.of("secrets/1s1t_config_staging.json"))); + } + +} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java new file mode 100644 index 000000000000..4ac040794f26 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorIntegrationTest.java @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.redshift.typing_deduping; + +import static io.airbyte.cdk.db.jdbc.DateTimeConverter.putJavaSQLTime; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.DateTimeConverter; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcSourceOperations; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.destination.jdbc.TableDefinition; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; +import io.airbyte.cdk.integrations.standardtest.destination.typing_deduping.JdbcSqlGeneratorIntegrationTest; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.destination.typing_deduping.DestinationHandler; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; +import io.airbyte.integrations.destination.redshift.RedshiftInsertDestination; +import io.airbyte.integrations.destination.redshift.RedshiftSQLNameTransformer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.OffsetTime; +import java.time.ZoneOffset; +import java.util.Optional; +import javax.sql.DataSource; +import org.jooq.DSLContext; +import org.jooq.DataType; +import org.jooq.Field; +import org.jooq.SQLDialect; +import org.jooq.conf.Settings; +import org.jooq.impl.DSL; +import org.jooq.impl.DefaultDataType; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +public class RedshiftSqlGeneratorIntegrationTest extends JdbcSqlGeneratorIntegrationTest { + + /** + * Redshift's JDBC driver doesn't map certain data types onto {@link java.sql.JDBCType} usefully. + * This class adds special handling for those types. + */ + public static class RedshiftSourceOperations extends JdbcSourceOperations { + + @Override + public void copyToJsonField(final ResultSet resultSet, final int colIndex, final ObjectNode json) throws SQLException { + final String columnName = resultSet.getMetaData().getColumnName(colIndex); + final String columnTypeName = resultSet.getMetaData().getColumnTypeName(colIndex).toLowerCase(); + + switch (columnTypeName) { + // SUPER has no equivalent in JDBCType + case "super" -> json.set(columnName, Jsons.deserializeExact(resultSet.getString(colIndex))); + // For some reason, the driver maps these to their timezoneless equivalents (TIME and TIMESTAMP) + case "timetz" -> putTimeWithTimezone(json, columnName, resultSet, colIndex); + case "timestamptz" -> putTimestampWithTimezone(json, columnName, resultSet, colIndex); + default -> super.copyToJsonField(resultSet, colIndex, json); + } + } + + @Override + protected void putTimeWithTimezone(final ObjectNode node, + final String columnName, + final ResultSet resultSet, + final int index) + throws SQLException { + final OffsetTime offsetTime = resultSet.getTimestamp(index).toInstant().atOffset(ZoneOffset.UTC).toOffsetTime(); + node.put(columnName, DateTimeConverter.convertToTimeWithTimezone(offsetTime)); + } + + @Override + protected void putTime(final ObjectNode node, + final String columnName, + final ResultSet resultSet, + final int index) + throws SQLException { + putJavaSQLTime(node, columnName, resultSet, index); + } + + @Override + protected void putTimestampWithTimezone(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) + throws SQLException { + // The superclass implementation tries to fetch a OffsetDateTime, which fails. + try { + super.putTimestampWithTimezone(node, columnName, resultSet, index); + } catch (final Exception e) { + final Instant instant = resultSet.getTimestamp(index).toInstant(); + node.put(columnName, DateTimeConverter.convertToTimestampWithTimezone(instant)); + } + } + + // Base class is converting to Instant which assumes the base timezone is UTC and resolves the local + // value to system's timezone. + @Override + protected void putTimestamp(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { + try { + node.put(columnName, DateTimeConverter.convertToTimestamp(getObject(resultSet, index, LocalDateTime.class))); + } catch (final Exception e) { + final LocalDateTime localDateTime = resultSet.getTimestamp(index).toLocalDateTime(); + node.put(columnName, DateTimeConverter.convertToTimestamp(localDateTime)); + } + } + + } + + private static DataSource dataSource; + private static JdbcDatabase database; + private static String databaseName; + + @BeforeAll + public static void setupJdbcDatasource() throws Exception { + final String rawConfig = Files.readString(Path.of("secrets/1s1t_config.json")); + final JsonNode config = Jsons.deserialize(rawConfig); + // TODO: Existing in AbstractJdbcDestination, pull out to a util file + databaseName = config.get(JdbcUtils.DATABASE_KEY).asText(); + // TODO: Its sad to instantiate unneeded dependency to construct database and datsources. pull it to + // static methods. + final RedshiftInsertDestination insertDestination = new RedshiftInsertDestination(); + dataSource = insertDestination.getDataSource(config); + database = insertDestination.getDatabase(dataSource, new RedshiftSourceOperations()); + } + + @AfterAll + public static void teardownRedshift() throws Exception { + DataSourceFactory.close(dataSource); + } + + @Override + protected JdbcSqlGenerator getSqlGenerator() { + return new RedshiftSqlGenerator(new RedshiftSQLNameTransformer()) { + + // Override only for tests to print formatted SQL. The actual implementation should use unformatted + // to save bytes. + @Override + protected DSLContext getDslContext() { + return DSL.using(getDialect(), new Settings().withRenderFormatted(true)); + } + + }; + } + + @Override + protected DestinationHandler getDestinationHandler() { + return new RedshiftDestinationHandler(databaseName, database); + } + + @Override + protected JdbcDatabase getDatabase() { + return database; + } + + @Override + protected DataType getStructType() { + return new DefaultDataType<>(null, String.class, "super"); + } + + @Override + protected SQLDialect getSqlDialect() { + return SQLDialect.POSTGRES; + } + + @Override + protected Field toJsonValue(final String valueAsString) { + return DSL.function("JSON_PARSE", String.class, DSL.val(escapeStringLiteral(valueAsString))); + } + + @Override + @Test + public void testCreateTableIncremental() throws Exception { + final Sql sql = generator.createTable(incrementalDedupStream, "", false); + destinationHandler.execute(sql); + + final Optional existingTable = destinationHandler.findExistingTable(incrementalDedupStream.id()); + + assertTrue(existingTable.isPresent()); + assertAll( + () -> assertEquals("varchar", existingTable.get().columns().get("_airbyte_raw_id").type()), + () -> assertEquals("timestamptz", existingTable.get().columns().get("_airbyte_extracted_at").type()), + () -> assertEquals("super", existingTable.get().columns().get("_airbyte_meta").type()), + () -> assertEquals("int8", existingTable.get().columns().get("id1").type()), + () -> assertEquals("int8", existingTable.get().columns().get("id2").type()), + () -> assertEquals("timestamptz", existingTable.get().columns().get("updated_at").type()), + () -> assertEquals("super", existingTable.get().columns().get("struct").type()), + () -> assertEquals("super", existingTable.get().columns().get("array").type()), + () -> assertEquals("varchar", existingTable.get().columns().get("string").type()), + () -> assertEquals("numeric", existingTable.get().columns().get("number").type()), + () -> assertEquals("int8", existingTable.get().columns().get("integer").type()), + () -> assertEquals("bool", existingTable.get().columns().get("boolean").type()), + () -> assertEquals("timestamptz", existingTable.get().columns().get("timestamp_with_timezone").type()), + () -> assertEquals("timestamp", existingTable.get().columns().get("timestamp_without_timezone").type()), + () -> assertEquals("timetz", existingTable.get().columns().get("time_with_timezone").type()), + () -> assertEquals("time", existingTable.get().columns().get("time_without_timezone").type()), + () -> assertEquals("date", existingTable.get().columns().get("date").type()), + () -> assertEquals("super", existingTable.get().columns().get("unknown").type())); + // TODO assert on table clustering, etc. + } + + private static String escapeStringLiteral(final String str) { + if (str == null) { + return null; + } else { + // jooq handles most things + // but we need to manually escape backslashes for some reason + return str.replace("\\", "\\\\"); + } + } + +} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftStandardInsertsRawSchemaOverrideDisableTypingDedupingTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftStandardInsertsRawSchemaOverrideDisableTypingDedupingTest.java new file mode 100644 index 000000000000..b7c78a4cec8e --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftStandardInsertsRawSchemaOverrideDisableTypingDedupingTest.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.redshift.typing_deduping; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import java.nio.file.Path; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +public class RedshiftStandardInsertsRawSchemaOverrideDisableTypingDedupingTest extends AbstractRedshiftTypingDedupingTest { + + @Override + protected ObjectNode getBaseConfig() { + return (ObjectNode) Jsons.deserialize(IOs.readFile(Path.of("secrets/1s1t_config_raw_schema_override.json"))); + } + + @Override + protected String getRawSchema() { + return "overridden_raw_dataset"; + } + + @Override + protected boolean disableFinalTableComparison() { + return true; + } + + @Disabled + @Test + @Override + public void identicalNameSimultaneousSync() {} + +} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftStandardInsertsTypingDedupingTest.java b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftStandardInsertsTypingDedupingTest.java new file mode 100644 index 000000000000..d99d597e4510 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftStandardInsertsTypingDedupingTest.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.redshift.typing_deduping; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import java.nio.file.Path; + +public class RedshiftStandardInsertsTypingDedupingTest extends AbstractRedshiftTypingDedupingTest { + + @Override + protected ObjectNode getBaseConfig() { + return (ObjectNode) Jsons.deserialize(IOs.readFile(Path.of("secrets/1s1t_config.json"))); + } + +} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl new file mode 100644 index 000000000000..9f11b2293a95 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "old_cursor": 1, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl new file mode 100644 index 000000000000..7f75f0f804e2 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl @@ -0,0 +1,4 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl new file mode 100644 index 000000000000..c805113dc6c2 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl @@ -0,0 +1,4 @@ +// Keep the Alice record with more recent updated_at +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl new file mode 100644 index 000000000000..b2bf47df66c1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl @@ -0,0 +1 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00.000000Z", "name": "Someone completely different"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl new file mode 100644 index 000000000000..8aa852183061 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl @@ -0,0 +1,5 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +// Invalid columns are nulled out (i.e. SQL null, not JSON null) +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..80fac124d28d --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl @@ -0,0 +1,5 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +// Invalid data is still allowed in the raw table. +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl new file mode 100644 index 000000000000..b489accda1bb --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl @@ -0,0 +1 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different"}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl new file mode 100644 index 000000000000..c26d4a49aacd --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +// Charlie wasn't reemitted with updated_at, so it still has a null cursor +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl new file mode 100644 index 000000000000..03f28e155af5 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl @@ -0,0 +1,7 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl new file mode 100644 index 000000000000..6e9258bab255 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl @@ -0,0 +1,8 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"} + +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000Z"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl new file mode 100644 index 000000000000..9d1f1499469f --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000Z"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl new file mode 100644 index 000000000000..33bc3280be27 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl new file mode 100644 index 000000000000..13c59b2f9912 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +// Delete Bob, keep Charlie +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000Z", "name": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl new file mode 100644 index 000000000000..53c304c89d31 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl @@ -0,0 +1 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2001-01-02T00:00:00.000000Z", "name": "Someone completely different v2"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..32a7e57b1c14 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl @@ -0,0 +1,9 @@ +// We keep the records from the first sync +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} +// And append the records from the second sync +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl new file mode 100644 index 000000000000..88b8ee7746c1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl @@ -0,0 +1,2 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000Z", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different v2"}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl new file mode 100644 index 000000000000..f6441416658b --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl @@ -0,0 +1,9 @@ +{"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": ["Problem with `struct`", "Problem with `array`", "Problem with `number`", "Problem with `integer`", "Problem with `boolean`","Problem with `timestamp_with_timezone`", "Problem with `timestamp_without_timezone`", "Problem with `time_with_timezone`","Problem with `time_without_timezone`", "Problem with `date`"]}} +// Note that for numbers where we parse the value to JSON (struct, array, unknown) we lose precision. +// But for numbers where we create a NUMBER column, we do not lose precision (see the `number` column). +{"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +// Note that redshift downcases IAmACaseSensitiveColumnName to all lowercase +{"id1": 6, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "iamacasesensitivecolumnname": "Case senstive value", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..6b99169ececf --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl @@ -0,0 +1,6 @@ +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": null, "struct": null, "string": null, "number": null, "integer": null, "boolean": null, "timestamp_with_timezone": null, "timestamp_without_timezone": null, "time_with_timezone": null, "time_without_timezone": null, "date": null, "unknown": null}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z"}} +{"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": null, "number": "foo", "integer": "bar", "boolean": "fizz", "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": "airbyte", "unknown": null}} +{"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fce", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 6, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "IamACaseSensitiveColumnName": "Case senstive value"}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_final.jsonl new file mode 100644 index 000000000000..5842f7b37e42 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_final.jsonl @@ -0,0 +1,2 @@ +{"_airbyte_raw_id": "80c99b54-54b4-43bd-b51b-1f67dafa2c52", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00.000000Z", "string": "Alice", "struct": {"city": "San Diego", "state": "CA"}, "integer": 84} +{"_airbyte_raw_id": "ad690bfb-c2c2-4172-bd73-a16c86ccbb67", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": ["Problem with `integer`"]}, "id1": 2, "id2": 100, "updated_at": "2023-01-01T03:00:00.000000Z", "string": "Bob"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..63569975abc2 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl @@ -0,0 +1,3 @@ +{"_airbyte_raw_id": "d7b81af0-01da-4846-a650-cc398986bc99", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "string": "Alice", "struct": {"city": "San Francisco", "state": "CA"}, "integer": 42}} +{"_airbyte_raw_id": "80c99b54-54b4-43bd-b51b-1f67dafa2c52", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00Z", "string": "Alice", "struct": {"city": "San Diego", "state": "CA"}, "integer": 84}} +{"_airbyte_raw_id": "ad690bfb-c2c2-4172-bd73-a16c86ccbb67", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T03:00:00Z", "string": "Bob", "integer": "oops"}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_final.jsonl new file mode 100644 index 000000000000..52a9c10fcc47 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_final.jsonl @@ -0,0 +1,5 @@ +{"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "[\"I\",\"am\",\"an\",\"array\"]", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "{\"I\":\"am\",\"an\":\"object\"}", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "true", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "3.14", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} +{"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "I am a valid json string", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..5c10203c7837 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_raw.jsonl @@ -0,0 +1,5 @@ +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": ["I", "am", "an", "array"], "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": {"I": "am", "an": "object"}, "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": true, "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": 3.14, "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "I am a valid json string", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_final.jsonl new file mode 100644 index 000000000000..4ecd95d83b63 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_final.jsonl @@ -0,0 +1 @@ +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..cd7c03aba677 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_raw.jsonl @@ -0,0 +1 @@ +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/reservedkeywords_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/reservedkeywords_expectedrecords_final.jsonl new file mode 100644 index 000000000000..b34ad054ab33 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/reservedkeywords_expectedrecords_final.jsonl @@ -0,0 +1 @@ +{"_airbyte_raw_id":"b2e0efc4-38a8-47ba-970c-8103f09f08d5","_airbyte_extracted_at":"2023-01-01T00:00:00.000000Z","_airbyte_meta":{"errors":[]}, "current_date": "foo", "join": "bar"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/timestampformats_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/timestampformats_expectedrecords_final.jsonl new file mode 100644 index 000000000000..5a4bfc33d906 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/timestampformats_expectedrecords_final.jsonl @@ -0,0 +1,16 @@ +// https://docs.aws.amazon.com/redshift/latest/dg/r_Datetime_types.html#r_Datetime_types-timetz +// TIME, TIMETZ, TIMESTAMP, TIMESTAMPTZ values are UTC in user tables. +// Note that redshift stores precision to microseconds. Java deserialization in tests preserves them only for non-zero values +// except for timestamp with time zone where Z is required at end for even zero values +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.000000Z", "time_with_timezone": "12:34:56Z"} +{"_airbyte_raw_id": "05028c5f-7813-4e9c-bd4b-387d1f8ba435", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T20:34:56.000000Z", "time_with_timezone": "20:34:56Z"} +{"_airbyte_raw_id": "95dfb0c6-6a67-4ba0-9935-643bebc90437", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T20:34:56.000000Z", "time_with_timezone": "20:34:56Z"} +{"_airbyte_raw_id": "f3d8abe2-bb0f-4caf-8ddc-0641df02f3a9", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T20:34:56.000000Z", "time_with_timezone": "20:34:56Z"} +{"_airbyte_raw_id": "a81ed40a-2a49-488d-9714-d53e8b052968", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T04:34:56.000000Z", "time_with_timezone": "04:34:56Z"} +{"_airbyte_raw_id": "c07763a0-89e6-4cb7-b7d0-7a34a7c9918a", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T04:34:56.000000Z", "time_with_timezone": "04:34:56Z"} +{"_airbyte_raw_id": "358d3b52-50ab-4e06-9094-039386f9bf0d", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T04:34:56.000000Z", "time_with_timezone": "04:34:56Z"} +{"_airbyte_raw_id": "db8200ac-b2b9-4b95-a053-8a0343042751", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.123000Z", "time_with_timezone": "12:34:56.123Z"} + +{"_airbyte_raw_id": "10ce5d93-6923-4217-a46f-103833837038", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_without_timezone": "2023-01-23T12:34:56", "time_without_timezone": "12:34:56", "date": "2023-01-23"} +// Bigquery returns 6 decimal places if there are any decimal places... but not for timestamp_with_timezone +{"_airbyte_raw_id": "a7a6e176-7464-4a0b-b55c-b4f936e8d5a1", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "timestamp_without_timezone": "2023-01-23T12:34:56.123", "time_without_timezone": "12:34:56.123"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl new file mode 100644 index 000000000000..adfbd06d6a55 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl @@ -0,0 +1,9 @@ +// column renamings: +// * $starts_with_dollar_sign -> _starts_with_dollar_sign +// * includes"doublequote -> includes_doublequote +// * includes'singlequote -> includes_singlequote +// * includes`backtick -> includes_backtick +// * includes$$doubledollar -> includes__doubledollar +// * includes.period -> includes_period +// * endswithbackslash\ -> endswithbackslash_ +{"_airbyte_raw_id": "7e7330a1-42fb-41ec-a955-52f18bd61964", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_meta": {"errors": []}, "id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00.000000Z", "_starts_with_dollar_sign": "foo", "includes_doublequote": "foo", "includes_singlequote": "foo", "includes_backtick": "foo", "includes_period": "foo", "includes__doubledollar": "foo", "endswithbackslash_": "foo"} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..2b602082a349 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_raw.jsonl @@ -0,0 +1 @@ +{"_airbyte_raw_id": "7e7330a1-42fb-41ec-a955-52f18bd61964", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00Z", "$starts_with_dollar_sign": "foo", "includes\"doublequote": "foo", "includes'singlequote": "foo", "includes`backtick": "foo", "includes.period": "foo", "includes$$doubledollar": "foo", "endswithbackslash\\": "foo"}} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopierTest.java b/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopierTest.java index f1011abf42eb..5c029abc5e58 100644 --- a/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopierTest.java +++ b/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/copiers/RedshiftStreamCopierTest.java @@ -16,13 +16,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.DestinationConfig; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.copy.s3.S3CopyConfig; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.DestinationConfig; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.copy.s3.S3CopyConfig; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; diff --git a/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorTest.java b/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorTest.java new file mode 100644 index 000000000000..341c7df14ced --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test/java/io/airbyte/integrations/destination/redshift/typing_deduping/RedshiftSqlGeneratorTest.java @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.redshift.typing_deduping; + +import static org.junit.jupiter.api.Assertions.*; + +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType; +import io.airbyte.integrations.base.destination.typing_deduping.Array; +import io.airbyte.integrations.base.destination.typing_deduping.ColumnId; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import io.airbyte.integrations.base.destination.typing_deduping.Struct; +import io.airbyte.integrations.destination.redshift.RedshiftSQLNameTransformer; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.SyncMode; +import java.io.IOException; +import java.time.Instant; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Optional; +import java.util.Random; +import org.jooq.DSLContext; +import org.jooq.conf.Settings; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class RedshiftSqlGeneratorTest { + + private static final Random RANDOM = new Random(); + + private static final RedshiftSqlGenerator redshiftSqlGenerator = new RedshiftSqlGenerator(new RedshiftSQLNameTransformer()) { + + // Override only for tests to print formatted SQL. The actual implementation should use unformatted + // to save bytes. + @Override + protected DSLContext getDslContext() { + return DSL.using(getDialect(), new Settings().withRenderFormatted(true)); + } + + }; + + private StreamId streamId; + + private StreamConfig incrementalDedupStream; + + private StreamConfig incrementalAppendStream; + + @BeforeEach + public void setup() { + streamId = new StreamId("test_schema", "users_final", "test_schema", "users_raw", "test_schema", "users_final"); + final ColumnId id1 = redshiftSqlGenerator.buildColumnId("id1"); + final ColumnId id2 = redshiftSqlGenerator.buildColumnId("id2"); + final List primaryKey = List.of(id1, id2); + final ColumnId cursor = redshiftSqlGenerator.buildColumnId("updated_at"); + + final LinkedHashMap columns = new LinkedHashMap<>(); + columns.put(id1, AirbyteProtocolType.INTEGER); + columns.put(id2, AirbyteProtocolType.INTEGER); + columns.put(cursor, AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); + columns.put(redshiftSqlGenerator.buildColumnId("struct"), new Struct(new LinkedHashMap<>())); + columns.put(redshiftSqlGenerator.buildColumnId("array"), new Array(AirbyteProtocolType.UNKNOWN)); + columns.put(redshiftSqlGenerator.buildColumnId("string"), AirbyteProtocolType.STRING); + columns.put(redshiftSqlGenerator.buildColumnId("number"), AirbyteProtocolType.NUMBER); + columns.put(redshiftSqlGenerator.buildColumnId("integer"), AirbyteProtocolType.INTEGER); + columns.put(redshiftSqlGenerator.buildColumnId("boolean"), AirbyteProtocolType.BOOLEAN); + columns.put(redshiftSqlGenerator.buildColumnId("timestamp_with_timezone"), AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); + columns.put(redshiftSqlGenerator.buildColumnId("timestamp_without_timezone"), AirbyteProtocolType.TIMESTAMP_WITHOUT_TIMEZONE); + columns.put(redshiftSqlGenerator.buildColumnId("time_with_timezone"), AirbyteProtocolType.TIME_WITH_TIMEZONE); + columns.put(redshiftSqlGenerator.buildColumnId("time_without_timezone"), AirbyteProtocolType.TIME_WITHOUT_TIMEZONE); + columns.put(redshiftSqlGenerator.buildColumnId("date"), AirbyteProtocolType.DATE); + columns.put(redshiftSqlGenerator.buildColumnId("unknown"), AirbyteProtocolType.UNKNOWN); + columns.put(redshiftSqlGenerator.buildColumnId("_ab_cdc_deleted_at"), AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); + incrementalDedupStream = new StreamConfig( + streamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND_DEDUP, + primaryKey, + Optional.of(cursor), + columns); + incrementalAppendStream = new StreamConfig( + streamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND, + primaryKey, + Optional.of(cursor), + columns); + } + + @Test + public void testTypingAndDeduping() throws IOException { + final String expectedSql = MoreResources.readResource("typing_deduping_with_cdc.sql"); + final Sql generatedSql = + redshiftSqlGenerator.updateTable(incrementalDedupStream, "unittest", Optional.of(Instant.parse("2023-02-15T18:35:24.00Z")), false); + final List expectedSqlLines = Arrays.stream(expectedSql.split("\n")).map(String::trim).toList(); + final List generatedSqlLines = generatedSql.asSqlStrings("BEGIN", "COMMIT").stream() + .flatMap(statement -> Arrays.stream(statement.split("\n"))) + .map(String::trim) + .filter(line -> !line.isEmpty()) + .toList(); + System.out.println(generatedSql); + assertEquals(expectedSqlLines, generatedSqlLines); + } + + @Test + public void test2000ColumnSql() { + final ColumnId id1 = redshiftSqlGenerator.buildColumnId("id1"); + final ColumnId id2 = redshiftSqlGenerator.buildColumnId("id2"); + final List primaryKey = List.of(id1, id2); + final ColumnId cursor = redshiftSqlGenerator.buildColumnId("updated_at"); + + final LinkedHashMap columns = new LinkedHashMap<>(); + columns.put(id1, AirbyteProtocolType.INTEGER); + columns.put(id2, AirbyteProtocolType.INTEGER); + columns.put(cursor, AirbyteProtocolType.TIMESTAMP_WITH_TIMEZONE); + + for (int i = 0; i < 2000; i++) { + final String columnName = RANDOM + .ints('a', 'z' + 1) + .limit(15) + .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) + .toString(); + columns.put(redshiftSqlGenerator.buildColumnId(columnName), AirbyteProtocolType.STRING); + } + final Sql generatedSql = redshiftSqlGenerator.updateTable(new StreamConfig( + streamId, + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND_DEDUP, + primaryKey, + Optional.of(cursor), + columns), "unittest", Optional.of(Instant.parse("2023-02-15T18:35:24.00Z")), false); + // This should not throw an exception. + assertFalse(generatedSql.transactions().isEmpty()); + } + +} diff --git a/airbyte-integrations/connectors/destination-redshift/src/test/resources/typing_deduping_with_cdc.sql b/airbyte-integrations/connectors/destination-redshift/src/test/resources/typing_deduping_with_cdc.sql new file mode 100644 index 000000000000..bc107dd680e9 --- /dev/null +++ b/airbyte-integrations/connectors/destination-redshift/src/test/resources/typing_deduping_with_cdc.sql @@ -0,0 +1,215 @@ +BEGIN; +insert into "test_schema"."users_finalunittest" ( + "id1", + "id2", + "updated_at", + "struct", + "array", + "string", + "number", + "integer", + "boolean", + "timestamp_with_timezone", + "timestamp_without_timezone", + "time_with_timezone", + "time_without_timezone", + "date", + "unknown", + "_ab_cdc_deleted_at", + "_airbyte_raw_id", + "_airbyte_extracted_at", + "_airbyte_meta" +) +with + "intermediate_data" as ( + select + cast("_airbyte_data"."id1" as bigint) as "id1", + cast("_airbyte_data"."id2" as bigint) as "id2", + cast("_airbyte_data"."updated_at" as timestamp with time zone) as "updated_at", + CASE WHEN JSON_TYPEOF("_airbyte_data"."struct") = 'object' THEN cast("_airbyte_data"."struct" as super) END as "struct", + CASE WHEN JSON_TYPEOF("_airbyte_data"."array") = 'array' THEN cast("_airbyte_data"."array" as super) END as "array", + CASE WHEN ( + JSON_TYPEOF("_airbyte_data"."string") <> 'string' + and "_airbyte_data"."string" is not null + ) THEN JSON_SERIALIZE("_airbyte_data"."string") ELSE cast("_airbyte_data"."string" as varchar(65535)) END as "string", + cast("_airbyte_data"."number" as decimal(38, 9)) as "number", + cast("_airbyte_data"."integer" as bigint) as "integer", + cast("_airbyte_data"."boolean" as boolean) as "boolean", + cast("_airbyte_data"."timestamp_with_timezone" as timestamp with time zone) as "timestamp_with_timezone", + cast("_airbyte_data"."timestamp_without_timezone" as timestamp) as "timestamp_without_timezone", + cast("_airbyte_data"."time_with_timezone" as time with time zone) as "time_with_timezone", + cast("_airbyte_data"."time_without_timezone" as time) as "time_without_timezone", + cast("_airbyte_data"."date" as date) as "date", + cast("_airbyte_data"."unknown" as super) as "unknown", + cast("_airbyte_data"."_ab_cdc_deleted_at" as timestamp with time zone) as "_ab_cdc_deleted_at", + "_airbyte_raw_id", + "_airbyte_extracted_at", + OBJECT( + 'errors', + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + ARRAY_CONCAT( + CASE WHEN ( + "_airbyte_data"."id1" is not null + and "id1" is null + ) THEN ARRAY('Problem with `id1`') ELSE ARRAY() END , + CASE WHEN ( + "_airbyte_data"."id2" is not null + and "id2" is null + ) THEN ARRAY('Problem with `id2`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."updated_at" is not null + and "updated_at" is null + ) THEN ARRAY('Problem with `updated_at`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."struct" is not null + and "struct" is null + ) THEN ARRAY('Problem with `struct`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."array" is not null + and "array" is null + ) THEN ARRAY('Problem with `array`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."string" is not null + and "string" is null + ) THEN ARRAY('Problem with `string`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."number" is not null + and "number" is null + ) THEN ARRAY('Problem with `number`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."integer" is not null + and "integer" is null + ) THEN ARRAY('Problem with `integer`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."boolean" is not null + and "boolean" is null + ) THEN ARRAY('Problem with `boolean`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."timestamp_with_timezone" is not null + and "timestamp_with_timezone" is null + ) THEN ARRAY('Problem with `timestamp_with_timezone`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."timestamp_without_timezone" is not null + and "timestamp_without_timezone" is null + ) THEN ARRAY('Problem with `timestamp_without_timezone`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."time_with_timezone" is not null + and "time_with_timezone" is null + ) THEN ARRAY('Problem with `time_with_timezone`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."time_without_timezone" is not null + and "time_without_timezone" is null + ) THEN ARRAY('Problem with `time_without_timezone`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."date" is not null + and "date" is null + ) THEN ARRAY('Problem with `date`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."unknown" is not null + and "unknown" is null + ) THEN ARRAY('Problem with `unknown`') ELSE ARRAY() END + ), + CASE WHEN ( + "_airbyte_data"."_ab_cdc_deleted_at" is not null + and "_ab_cdc_deleted_at" is null + ) THEN ARRAY('Problem with `_ab_cdc_deleted_at`') ELSE ARRAY() END + ) + ) as "_airbyte_meta" + from "test_schema"."users_raw" + where ( + ( + "_airbyte_loaded_at" is null + or ( + "_airbyte_loaded_at" is not null + and JSON_TYPEOF("_airbyte_data"."_ab_cdc_deleted_at") <> 'null' + ) + ) + and "_airbyte_extracted_at" > '2023-02-15T18:35:24Z' + ) + ), + "numbered_rows" as ( + select + *, + row_number() over (partition by + "id1", + "id2" + order by + "updated_at" desc NULLS LAST, + "_airbyte_extracted_at" desc + ) as "row_number" + from "intermediate_data" + ) +select + "id1", + "id2", + "updated_at", + "struct", + "array", + "string", + "number", + "integer", + "boolean", + "timestamp_with_timezone", + "timestamp_without_timezone", + "time_with_timezone", + "time_without_timezone", + "date", + "unknown", + "_ab_cdc_deleted_at", + "_airbyte_raw_id", + "_airbyte_extracted_at", + "_airbyte_meta" +from "numbered_rows" +where "row_number" = 1; +delete from "test_schema"."users_finalunittest" +where "_airbyte_raw_id" in ( + select "_airbyte_raw_id" + from ( + select + "_airbyte_raw_id", + row_number() over (partition by + "id1", + "id2" + order by + "updated_at" desc NULLS LAST, + "_airbyte_extracted_at" desc + ) as "row_number" + from "test_schema"."users_finalunittest" + ) as "airbyte_ids" + where "row_number" <> 1 +); +delete from "test_schema"."users_finalunittest" +where "_ab_cdc_deleted_at" is not null; +update "test_schema"."users_raw" +set "_airbyte_loaded_at" = GETDATE() +where ( + "_airbyte_loaded_at" is null + and "_airbyte_extracted_at" > '2023-02-15T18:35:24Z' + ); +COMMIT; diff --git a/airbyte-integrations/connectors/destination-rockset/.dockerignore b/airbyte-integrations/connectors/destination-rockset/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-rockset/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-rockset/Dockerfile b/airbyte-integrations/connectors/destination-rockset/Dockerfile deleted file mode 100644 index 85b281c64662..000000000000 --- a/airbyte-integrations/connectors/destination-rockset/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-rockset - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-rockset - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.4 -LABEL io.airbyte.name=airbyte/destination-rockset diff --git a/airbyte-integrations/connectors/destination-rockset/README.md b/airbyte-integrations/connectors/destination-rockset/README.md index 3c46bc91603d..7eb71ac3ee7c 100644 --- a/airbyte-integrations/connectors/destination-rockset/README.md +++ b/airbyte-integrations/connectors/destination-rockset/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-rockset:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-rockset:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-rockset:dev`. the Dockerfile. #### Run @@ -61,8 +62,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-rockset test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/rockset.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-rockset/build.gradle b/airbyte-integrations/connectors/destination-rockset/build.gradle index 0618e24c9599..a5e64c058290 100644 --- a/airbyte-integrations/connectors/destination-rockset/build.gradle +++ b/airbyte-integrations/connectors/destination-rockset/build.gradle @@ -1,32 +1,30 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.rockset.RocksetDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } -repositories { - maven { url "https://mvnrepository.com/artifact/com.rockset/rockset-java" } - maven { url "https://mvnrepository.com/artifact/org.awaitility/awaitility" } -} - dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - implementation project(':airbyte-integrations:bases:bases-destination-jdbc') implementation group: 'com.rockset', name: 'rockset-java', version: '0.9.0' implementation group: 'org.awaitility', name: 'awaitility', version: '4.1.1' - - testImplementation project(':airbyte-integrations:bases:standard-destination-test') - - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-rockset') } diff --git a/airbyte-integrations/connectors/destination-rockset/src/main/java/io/airbyte/integrations/destination/rockset/RocksetDestination.java b/airbyte-integrations/connectors/destination-rockset/src/main/java/io/airbyte/integrations/destination/rockset/RocksetDestination.java index c71a18c96b8e..2c4b25c613dd 100644 --- a/airbyte-integrations/connectors/destination-rockset/src/main/java/io/airbyte/integrations/destination/rockset/RocksetDestination.java +++ b/airbyte-integrations/connectors/destination-rockset/src/main/java/io/airbyte/integrations/destination/rockset/RocksetDestination.java @@ -13,10 +13,10 @@ import com.rockset.client.ApiClient; import com.rockset.client.api.DocumentsApi; import com.rockset.client.model.AddDocumentsRequest; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/destination-rockset/src/main/java/io/airbyte/integrations/destination/rockset/RocksetSQLNameTransformer.java b/airbyte-integrations/connectors/destination-rockset/src/main/java/io/airbyte/integrations/destination/rockset/RocksetSQLNameTransformer.java index cf14fe393958..8faaef972fef 100644 --- a/airbyte-integrations/connectors/destination-rockset/src/main/java/io/airbyte/integrations/destination/rockset/RocksetSQLNameTransformer.java +++ b/airbyte-integrations/connectors/destination-rockset/src/main/java/io/airbyte/integrations/destination/rockset/RocksetSQLNameTransformer.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.rockset; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; public class RocksetSQLNameTransformer extends StandardNameTransformer { diff --git a/airbyte-integrations/connectors/destination-rockset/src/main/java/io/airbyte/integrations/destination/rockset/RocksetWriteApiConsumer.java b/airbyte-integrations/connectors/destination-rockset/src/main/java/io/airbyte/integrations/destination/rockset/RocksetWriteApiConsumer.java index 36e5c60b3314..1dd6687c0ea7 100644 --- a/airbyte-integrations/connectors/destination-rockset/src/main/java/io/airbyte/integrations/destination/rockset/RocksetWriteApiConsumer.java +++ b/airbyte-integrations/connectors/destination-rockset/src/main/java/io/airbyte/integrations/destination/rockset/RocksetWriteApiConsumer.java @@ -16,8 +16,8 @@ import com.rockset.client.model.AddDocumentsRequest; import com.rockset.client.model.AddDocumentsResponse; import com.rockset.client.model.DocumentStatus; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.commons.lang.Exceptions; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.DestinationSyncMode; diff --git a/airbyte-integrations/connectors/destination-rockset/src/test-integration/java/io/airbyte/integrations/destination/rockset/RocksetDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-rockset/src/test-integration/java/io/airbyte/integrations/destination/rockset/RocksetDestinationAcceptanceTest.java index 043a12b529bf..81c294eb7213 100644 --- a/airbyte-integrations/connectors/destination-rockset/src/test-integration/java/io/airbyte/integrations/destination/rockset/RocksetDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-rockset/src/test-integration/java/io/airbyte/integrations/destination/rockset/RocksetDestinationAcceptanceTest.java @@ -14,12 +14,12 @@ import com.rockset.client.model.QueryRequest; import com.rockset.client.model.QueryRequestSql; import com.squareup.okhttp.Response; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.lang.Exceptions; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; diff --git a/airbyte-integrations/connectors/destination-rockset/src/test/java/io/airbyte/integrations/destination/rockset/RocksetWriteApiConsumerTest.java b/airbyte-integrations/connectors/destination-rockset/src/test/java/io/airbyte/integrations/destination/rockset/RocksetWriteApiConsumerTest.java index 9b11bbbd6bef..6146f73fc59a 100644 --- a/airbyte-integrations/connectors/destination-rockset/src/test/java/io/airbyte/integrations/destination/rockset/RocksetWriteApiConsumerTest.java +++ b/airbyte-integrations/connectors/destination-rockset/src/test/java/io/airbyte/integrations/destination/rockset/RocksetWriteApiConsumerTest.java @@ -10,9 +10,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.standardtest.destination.PerStreamStateMessageTest; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; -import io.airbyte.integrations.standardtest.destination.PerStreamStateMessageTest; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import java.util.function.Consumer; diff --git a/airbyte-integrations/connectors/destination-s3-glue/.dockerignore b/airbyte-integrations/connectors/destination-s3-glue/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-s3-glue/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-s3-glue/Dockerfile b/airbyte-integrations/connectors/destination-s3-glue/Dockerfile deleted file mode 100644 index df4ecb366ea0..000000000000 --- a/airbyte-integrations/connectors/destination-s3-glue/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte -ENV APPLICATION destination-s3-glue - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte -ENV APPLICATION destination-s3-glue - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.7 -LABEL io.airbyte.name=airbyte/destination-s3-glue diff --git a/airbyte-integrations/connectors/destination-s3-glue/README.md b/airbyte-integrations/connectors/destination-s3-glue/README.md index 79550c6cf39b..7ac2f084fc6d 100644 --- a/airbyte-integrations/connectors/destination-s3-glue/README.md +++ b/airbyte-integrations/connectors/destination-s3-glue/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-s3-glue:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-s3-glue:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-s3-glue:dev`. the Dockerfile. #### Run @@ -61,8 +62,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-s3-glue test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/s3-glue.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-s3-glue/build.gradle b/airbyte-integrations/connectors/destination-s3-glue/build.gradle index bea00ad2fdcb..23f96e515697 100644 --- a/airbyte-integrations/connectors/destination-s3-glue/build.gradle +++ b/airbyte-integrations/connectors/destination-s3-glue/build.gradle @@ -1,25 +1,32 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.s3_glue.S3GlueDestination' } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:bases:base-java-s3') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) // https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk-glue implementation 'com.amazonaws:aws-java-sdk-glue:1.12.334' - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-s3-glue') - integrationTestJavaImplementation project(':airbyte-integrations:bases:s3-destination-base-integration-test') - + implementation libs.aws.java.sdk.s3 + implementation libs.s3 } diff --git a/airbyte-integrations/connectors/destination-s3-glue/metadata.yaml b/airbyte-integrations/connectors/destination-s3-glue/metadata.yaml index dc0a004c20bd..0ca7298c3f5b 100644 --- a/airbyte-integrations/connectors/destination-s3-glue/metadata.yaml +++ b/airbyte-integrations/connectors/destination-s3-glue/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: file connectorType: destination definitionId: 471e5cab-8ed1-49f3-ba11-79c687784737 - dockerImageTag: 0.1.7 + dockerImageTag: 0.1.8 dockerRepository: airbyte/destination-s3-glue githubIssueLabel: destination-s3-glue icon: s3-glue.svg diff --git a/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/GlueDestinationConfig.java b/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/GlueDestinationConfig.java index c47b583e4223..7bb11b04a106 100644 --- a/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/GlueDestinationConfig.java +++ b/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/GlueDestinationConfig.java @@ -4,9 +4,9 @@ package io.airbyte.integrations.destination.s3_glue; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.ACCESS_KEY_ID; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.SECRET_ACCESS_KEY; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_REGION; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.ACCESS_KEY_ID; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.SECRET_ACCESS_KEY; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_REGION; import static io.airbyte.integrations.destination.s3_glue.GlueConstants.GLUE_DATABASE; import static io.airbyte.integrations.destination.s3_glue.GlueConstants.SERIALIZATION_LIBRARY; diff --git a/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/S3GlueConsumerFactory.java b/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/S3GlueConsumerFactory.java index 3b4b8dbb0070..2993fa30dfed 100644 --- a/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/S3GlueConsumerFactory.java +++ b/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/S3GlueConsumerFactory.java @@ -7,19 +7,19 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Preconditions; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.BufferedStreamConsumer; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.OnCloseFunction; +import io.airbyte.cdk.integrations.destination.buffered_stream_consumer.OnStartFunction; +import io.airbyte.cdk.integrations.destination.record_buffer.BufferCreateFunction; +import io.airbyte.cdk.integrations.destination.record_buffer.FlushBufferFunction; +import io.airbyte.cdk.integrations.destination.record_buffer.SerializedBufferingStrategy; +import io.airbyte.cdk.integrations.destination.s3.BlobStorageOperations; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.WriteConfig; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.buffered_stream_consumer.BufferedStreamConsumer; -import io.airbyte.integrations.destination.buffered_stream_consumer.OnCloseFunction; -import io.airbyte.integrations.destination.buffered_stream_consumer.OnStartFunction; -import io.airbyte.integrations.destination.record_buffer.BufferCreateFunction; -import io.airbyte.integrations.destination.record_buffer.FlushBufferFunction; -import io.airbyte.integrations.destination.record_buffer.SerializedBufferingStrategy; -import io.airbyte.integrations.destination.s3.BlobStorageOperations; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.WriteConfig; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; diff --git a/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/S3GlueDestination.java b/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/S3GlueDestination.java index 902240a86435..0d8d988ea72a 100644 --- a/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/S3GlueDestination.java +++ b/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/S3GlueDestination.java @@ -5,17 +5,17 @@ package io.airbyte.integrations.destination.s3_glue; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.record_buffer.FileBuffer; +import io.airbyte.cdk.integrations.destination.s3.BaseS3Destination; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3StorageOperations; +import io.airbyte.cdk.integrations.destination.s3.SerializedBufferFactory; +import io.airbyte.cdk.integrations.destination.s3.StorageProvider; +import io.airbyte.cdk.integrations.destination.s3.util.S3NameTransformer; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.record_buffer.FileBuffer; -import io.airbyte.integrations.destination.s3.BaseS3Destination; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.S3StorageOperations; -import io.airbyte.integrations.destination.s3.SerializedBufferFactory; -import io.airbyte.integrations.destination.s3.StorageProvider; -import io.airbyte.integrations.destination.s3.util.S3NameTransformer; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/S3GlueWriteConfig.java b/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/S3GlueWriteConfig.java index c5eaa88f89c5..2ffa48090378 100644 --- a/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/S3GlueWriteConfig.java +++ b/airbyte-integrations/connectors/destination-s3-glue/src/main/java/io/airbyte/integrations/destination/s3_glue/S3GlueWriteConfig.java @@ -5,7 +5,7 @@ package io.airbyte.integrations.destination.s3_glue; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.destination.s3.WriteConfig; +import io.airbyte.cdk.integrations.destination.s3.WriteConfig; import io.airbyte.protocol.models.v0.DestinationSyncMode; public class S3GlueWriteConfig extends WriteConfig { diff --git a/airbyte-integrations/connectors/destination-s3-glue/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-s3-glue/src/main/resources/spec.json index d5571f31ceb1..896817109431 100644 --- a/airbyte-integrations/connectors/destination-s3-glue/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-s3-glue/src/main/resources/spec.json @@ -54,31 +54,39 @@ "description": "The region of the S3 bucket. See here for all region codes.", "enum": [ "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", "af-south-1", "ap-east-1", - "ap-south-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", + "ap-south-1", + "ap-south-2", "ap-southeast-1", "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", "ca-central-1", + "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", + "eu-central-2", "eu-north-1", "eu-south-1", + "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", - "sa-east-1", + "il-central-1", + "me-central-1", "me-south-1", + "sa-east-1", + "us-east-1", + "us-east-2", "us-gov-east-1", - "us-gov-west-1" + "us-gov-west-1", + "us-west-1", + "us-west-2" ], "order": 4 }, diff --git a/airbyte-integrations/connectors/destination-s3-glue/src/test-integration/java/io/airbyte/integrations/destination/s3_glue/S3GlueJsonlDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-s3-glue/src/test-integration/java/io/airbyte/integrations/destination/s3_glue/S3GlueJsonlDestinationAcceptanceTest.java index dc25fa123ccf..e58a417290fc 100644 --- a/airbyte-integrations/connectors/destination-s3-glue/src/test-integration/java/io/airbyte/integrations/destination/s3_glue/S3GlueJsonlDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-s3-glue/src/test-integration/java/io/airbyte/integrations/destination/s3_glue/S3GlueJsonlDestinationAcceptanceTest.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.s3_glue; -import io.airbyte.integrations.destination.s3.S3BaseJsonlDestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider; +import io.airbyte.cdk.integrations.destination.s3.S3BaseJsonlDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.argproviders.DataTypeTestArgumentProvider; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ArgumentsSource; diff --git a/airbyte-integrations/connectors/destination-s3-glue/src/test-integration/java/io/airbyte/integrations/destination/s3_glue/S3GlueJsonlGzipDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-s3-glue/src/test-integration/java/io/airbyte/integrations/destination/s3_glue/S3GlueJsonlGzipDestinationAcceptanceTest.java index e5d46921114f..d33f94d74ad7 100644 --- a/airbyte-integrations/connectors/destination-s3-glue/src/test-integration/java/io/airbyte/integrations/destination/s3_glue/S3GlueJsonlGzipDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-s3-glue/src/test-integration/java/io/airbyte/integrations/destination/s3_glue/S3GlueJsonlGzipDestinationAcceptanceTest.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.s3_glue; -import io.airbyte.integrations.destination.s3.S3BaseJsonlGzipDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.destination.s3.S3BaseJsonlGzipDestinationAcceptanceTest; public class S3GlueJsonlGzipDestinationAcceptanceTest extends S3BaseJsonlGzipDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-s3/.dockerignore b/airbyte-integrations/connectors/destination-s3/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-s3/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-s3/Dockerfile b/airbyte-integrations/connectors/destination-s3/Dockerfile deleted file mode 100644 index 18668204b383..000000000000 --- a/airbyte-integrations/connectors/destination-s3/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-s3 - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-s3 - -COPY --from=build /airbyte /airbyte -RUN /bin/bash -c 'set -e && \ - ARCH=`uname -m` && \ - if [ "$ARCH" == "x86_64" ] || [ "$ARCH" = "amd64" ]; then \ - echo "$ARCH" && \ - yum install lzop lzo lzo-dev -y; \ - fi' - -RUN yum clean all -LABEL io.airbyte.version=0.5.1 -LABEL io.airbyte.name=airbyte/destination-s3 diff --git a/airbyte-integrations/connectors/destination-s3/build.gradle b/airbyte-integrations/connectors/destination-s3/build.gradle index 928f1a0c6e5b..1e53b23dced0 100644 --- a/airbyte-integrations/connectors/destination-s3/build.gradle +++ b/airbyte-integrations/connectors/destination-s3/build.gradle @@ -1,20 +1,22 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.10.2' + features = ['db-destinations', 's3-destinations'] + useLocalCdk = false +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.s3.S3DestinationRunner' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:bases:base-java-s3') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) // csv implementation 'com.amazonaws:aws-java-sdk-s3:1.11.978' @@ -37,8 +39,4 @@ dependencies { testImplementation 'org.apache.commons:commons-lang3:3.11' testImplementation 'org.xerial.snappy:snappy-java:1.1.8.4' testImplementation "org.mockito:mockito-inline:4.1.0" - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-s3') - integrationTestJavaImplementation project(':airbyte-integrations:bases:s3-destination-base-integration-test') } diff --git a/airbyte-integrations/connectors/destination-s3/finalize_build.sh b/airbyte-integrations/connectors/destination-s3/finalize_build.sh index 256688b7f687..f6684932e999 100644 --- a/airbyte-integrations/connectors/destination-s3/finalize_build.sh +++ b/airbyte-integrations/connectors/destination-s3/finalize_build.sh @@ -7,7 +7,7 @@ ARCH=$(uname -m) if [ "$ARCH" == "x86_64" ] || [ "$ARCH" = "amd64" ]; then echo "$ARCH" - yum install lzop lzo lzo-dev -y + yum install lzop lzo lzo-devel -y fi yum clean all diff --git a/airbyte-integrations/connectors/destination-s3/metadata.yaml b/airbyte-integrations/connectors/destination-s3/metadata.yaml index 444ffc504893..aec7c1e0e295 100644 --- a/airbyte-integrations/connectors/destination-s3/metadata.yaml +++ b/airbyte-integrations/connectors/destination-s3/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: file connectorType: destination definitionId: 4816b78f-1489-44c1-9060-4b19d5fa9362 - dockerImageTag: 0.5.1 + dockerImageTag: 0.5.8 dockerRepository: airbyte/destination-s3 githubIssueLabel: destination-s3 icon: s3.svg @@ -25,6 +25,6 @@ data: - language:java ab_internal: sl: 300 - ql: 400 + ql: 300 supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Destination.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Destination.java index f12fcfcac960..9ebd5140b96b 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Destination.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3Destination.java @@ -5,7 +5,10 @@ package io.airbyte.integrations.destination.s3; import com.google.common.annotations.VisibleForTesting; -import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.destination.s3.BaseS3Destination; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfigFactory; +import io.airbyte.cdk.integrations.destination.s3.StorageProvider; public class S3Destination extends BaseS3Destination { diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationRunner.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationRunner.java index 127add6f3285..72734a3e18a3 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationRunner.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationRunner.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.s3; -import io.airbyte.integrations.base.adaptive.AdaptiveDestinationRunner; +import io.airbyte.cdk.integrations.base.adaptive.AdaptiveDestinationRunner; public class S3DestinationRunner { diff --git a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationStrictEncrypt.java b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationStrictEncrypt.java index 0eddfc89b80b..8258bbcb7d04 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationStrictEncrypt.java +++ b/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/S3DestinationStrictEncrypt.java @@ -6,6 +6,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; +import io.airbyte.cdk.integrations.destination.s3.S3BaseChecks; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfigFactory; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; diff --git a/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json index b8c2ce2b5e25..5e779c15eb6b 100644 --- a/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json @@ -20,6 +20,7 @@ "description": "The access key ID to access the S3 bucket. Airbyte requires Read and Write permissions to the given bucket. Read more here.", "title": "S3 Key ID", "airbyte_secret": true, + "always_show": true, "examples": ["A012345678910EXAMPLE"], "order": 0 }, @@ -28,6 +29,7 @@ "description": "The corresponding secret to the access key ID. Read more here", "title": "S3 Access Key", "airbyte_secret": true, + "always_show": true, "examples": ["a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"], "order": 1 }, @@ -52,31 +54,39 @@ "description": "The region of the S3 bucket. See here for all region codes.", "enum": [ "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", "af-south-1", "ap-east-1", - "ap-south-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", + "ap-south-1", + "ap-south-2", "ap-southeast-1", "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", "ca-central-1", + "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", + "eu-central-2", "eu-north-1", "eu-south-1", + "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", - "sa-east-1", + "il-central-1", + "me-central-1", "me-south-1", + "sa-east-1", + "us-east-1", + "us-east-2", "us-gov-east-1", - "us-gov-west-1" + "us-gov-west-1", + "us-west-1", + "us-west-2" ], "order": 4 }, @@ -85,6 +95,102 @@ "type": "object", "description": "Format of the data output. See here for more details", "oneOf": [ + { + "title": "CSV: Comma-Separated Values", + "required": ["format_type", "flattening"], + "properties": { + "format_type": { + "title": "Format Type", + "type": "string", + "enum": ["CSV"], + "default": "CSV" + }, + "flattening": { + "type": "string", + "title": "Flattening", + "description": "Whether the input json data should be normalized (flattened) in the output CSV. Please refer to docs for details.", + "default": "No flattening", + "enum": ["No flattening", "Root level flattening"] + }, + "compression": { + "title": "Compression", + "type": "object", + "description": "Whether the output files should be compressed. If compression is selected, the output filename will have an extra extension (GZIP: \".csv.gz\").", + "oneOf": [ + { + "title": "No Compression", + "requires": ["compression_type"], + "properties": { + "compression_type": { + "type": "string", + "enum": ["No Compression"], + "default": "No Compression" + } + } + }, + { + "title": "GZIP", + "requires": ["compression_type"], + "properties": { + "compression_type": { + "type": "string", + "enum": ["GZIP"], + "default": "GZIP" + } + } + } + ] + } + } + }, + { + "title": "JSON Lines: Newline-delimited JSON", + "required": ["format_type"], + "properties": { + "format_type": { + "title": "Format Type", + "type": "string", + "enum": ["JSONL"], + "default": "JSONL" + }, + "flattening": { + "type": "string", + "title": "Flattening", + "description": "Whether the input json data should be normalized (flattened) in the output JSON Lines. Please refer to docs for details.", + "default": "No flattening", + "enum": ["No flattening", "Root level flattening"] + }, + "compression": { + "title": "Compression", + "type": "object", + "description": "Whether the output files should be compressed. If compression is selected, the output filename will have an extra extension (GZIP: \".jsonl.gz\").", + "oneOf": [ + { + "title": "No Compression", + "requires": "compression_type", + "properties": { + "compression_type": { + "type": "string", + "enum": ["No Compression"], + "default": "No Compression" + } + } + }, + { + "title": "GZIP", + "requires": "compression_type", + "properties": { + "compression_type": { + "type": "string", + "enum": ["GZIP"], + "default": "GZIP" + } + } + } + ] + } + } + }, { "title": "Avro: Apache Avro", "required": ["format_type", "compression_codec"], @@ -202,102 +308,6 @@ } } }, - { - "title": "CSV: Comma-Separated Values", - "required": ["format_type", "flattening"], - "properties": { - "format_type": { - "title": "Format Type", - "type": "string", - "enum": ["CSV"], - "default": "CSV" - }, - "flattening": { - "type": "string", - "title": "Flattening", - "description": "Whether the input json data should be normalized (flattened) in the output CSV. Please refer to docs for details.", - "default": "No flattening", - "enum": ["No flattening", "Root level flattening"] - }, - "compression": { - "title": "Compression", - "type": "object", - "description": "Whether the output files should be compressed. If compression is selected, the output filename will have an extra extension (GZIP: \".csv.gz\").", - "oneOf": [ - { - "title": "No Compression", - "requires": ["compression_type"], - "properties": { - "compression_type": { - "type": "string", - "enum": ["No Compression"], - "default": "No Compression" - } - } - }, - { - "title": "GZIP", - "requires": ["compression_type"], - "properties": { - "compression_type": { - "type": "string", - "enum": ["GZIP"], - "default": "GZIP" - } - } - } - ] - } - } - }, - { - "title": "JSON Lines: Newline-delimited JSON", - "required": ["format_type"], - "properties": { - "format_type": { - "title": "Format Type", - "type": "string", - "enum": ["JSONL"], - "default": "JSONL" - }, - "flattening": { - "type": "string", - "title": "Flattening", - "description": "Whether the input json data should be normalized (flattened) in the output JSON Lines. Please refer to docs for details.", - "default": "No flattening", - "enum": ["No flattening", "Root level flattening"] - }, - "compression": { - "title": "Compression", - "type": "object", - "description": "Whether the output files should be compressed. If compression is selected, the output filename will have an extra extension (GZIP: \".jsonl.gz\").", - "oneOf": [ - { - "title": "No Compression", - "requires": "compression_type", - "properties": { - "compression_type": { - "type": "string", - "enum": ["No Compression"], - "default": "No Compression" - } - } - }, - { - "title": "GZIP", - "requires": "compression_type", - "properties": { - "compression_type": { - "type": "string", - "enum": ["GZIP"], - "default": "GZIP" - } - } - } - ] - } - } - }, { "title": "Parquet: Columnar Storage", "required": ["format_type"], diff --git a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3AvroDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3AvroDestinationAcceptanceTest.java index 9b80a8ada7ec..3daf5813f3aa 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3AvroDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3AvroDestinationAcceptanceTest.java @@ -5,8 +5,9 @@ package io.airbyte.integrations.destination.s3; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.standardtest.destination.ProtocolVersion; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.cdk.integrations.destination.s3.S3BaseAvroDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; public class S3AvroDestinationAcceptanceTest extends S3BaseAvroDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3AvroParquetTestDataComparator.java b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3AvroParquetTestDataComparator.java index 9cdcae6fde77..ee0bdcd99b51 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3AvroParquetTestDataComparator.java +++ b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3AvroParquetTestDataComparator.java @@ -5,7 +5,7 @@ package io.airbyte.integrations.destination.s3; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.LocalDate; diff --git a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java index 0ba14fd16455..d9e847d8678c 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvDestinationAcceptanceTest.java @@ -5,7 +5,8 @@ package io.airbyte.integrations.destination.s3; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.standardtest.destination.ProtocolVersion; +import io.airbyte.cdk.integrations.destination.s3.S3BaseCsvDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; public class S3CsvDestinationAcceptanceTest extends S3BaseCsvDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvGzipDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvGzipDestinationAcceptanceTest.java index 1fa19b5009a7..b2630cbcded5 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvGzipDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3CsvGzipDestinationAcceptanceTest.java @@ -5,7 +5,8 @@ package io.airbyte.integrations.destination.s3; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.standardtest.destination.ProtocolVersion; +import io.airbyte.cdk.integrations.destination.s3.S3BaseCsvGzipDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; public class S3CsvGzipDestinationAcceptanceTest extends S3BaseCsvGzipDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3JsonlDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3JsonlDestinationAcceptanceTest.java index ef877419bff0..78a75db98db3 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3JsonlDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3JsonlDestinationAcceptanceTest.java @@ -5,7 +5,8 @@ package io.airbyte.integrations.destination.s3; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.standardtest.destination.ProtocolVersion; +import io.airbyte.cdk.integrations.destination.s3.S3BaseJsonlDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; public class S3JsonlDestinationAcceptanceTest extends S3BaseJsonlDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3JsonlGzipDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3JsonlGzipDestinationAcceptanceTest.java index 341dea5927e2..0e11fbdde5d4 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3JsonlGzipDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3JsonlGzipDestinationAcceptanceTest.java @@ -5,7 +5,8 @@ package io.airbyte.integrations.destination.s3; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.standardtest.destination.ProtocolVersion; +import io.airbyte.cdk.integrations.destination.s3.S3BaseJsonlGzipDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; public class S3JsonlGzipDestinationAcceptanceTest extends S3BaseJsonlGzipDestinationAcceptanceTest { diff --git a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3ParquetDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3ParquetDestinationAcceptanceTest.java index 372f42747348..af365027d601 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3ParquetDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-s3/src/test-integration/java/io/airbyte/integrations/destination/s3/S3ParquetDestinationAcceptanceTest.java @@ -6,11 +6,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.destination.s3.S3BaseParquetDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.ProtocolVersion; +import io.airbyte.cdk.integrations.standardtest.destination.argproviders.DataArgumentsProvider; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.standardtest.destination.ProtocolVersion; -import io.airbyte.integrations.standardtest.destination.argproviders.DataArgumentsProvider; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.CatalogHelpers; diff --git a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3DestinationStrictEncryptTest.java b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3DestinationStrictEncryptTest.java index bb8d61e62d57..ee0b99fdbadb 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3DestinationStrictEncryptTest.java +++ b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3DestinationStrictEncryptTest.java @@ -15,6 +15,9 @@ import com.amazonaws.services.s3.model.UploadPartRequest; import com.amazonaws.services.s3.model.UploadPartResult; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfigFactory; +import io.airbyte.cdk.integrations.destination.s3.StorageProvider; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import org.junit.jupiter.api.BeforeEach; diff --git a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3DestinationTest.java b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3DestinationTest.java index 9e3269ce3a62..888e2a225f54 100644 --- a/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3DestinationTest.java +++ b/airbyte-integrations/connectors/destination-s3/src/test/java/io/airbyte/integrations/destination/s3/S3DestinationTest.java @@ -22,6 +22,11 @@ import com.amazonaws.services.s3.model.UploadPartRequest; import com.amazonaws.services.s3.model.UploadPartResult; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.destination.s3.S3BaseChecks; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfigFactory; +import io.airbyte.cdk.integrations.destination.s3.S3StorageOperations; +import io.airbyte.cdk.integrations.destination.s3.StorageProvider; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import org.junit.jupiter.api.BeforeEach; @@ -87,7 +92,7 @@ public void checksS3WithListObjectPermission() { @Test public void createsThenDeletesTestFile() { - S3BaseChecks.attemptS3WriteAndDelete(mock(S3StorageOperations.class), config, "fake-fileToWriteAndDelete", s3); + S3BaseChecks.attemptS3WriteAndDelete(mock(S3StorageOperations.class), config, "fake-fileToWriteAndDelete"); // We want to enforce that putObject happens before deleteObject, so use inOrder.verify() final InOrder inOrder = Mockito.inOrder(s3); diff --git a/airbyte-integrations/connectors/destination-scaffold-destination-python/.dockerignore b/airbyte-integrations/connectors/destination-scaffold-destination-python/.dockerignore deleted file mode 100644 index 2592e58c9261..000000000000 --- a/airbyte-integrations/connectors/destination-scaffold-destination-python/.dockerignore +++ /dev/null @@ -1,5 +0,0 @@ -* -!Dockerfile -!main.py -!destination_scaffold_destination_python -!setup.py diff --git a/airbyte-integrations/connectors/destination-scaffold-destination-python/Dockerfile b/airbyte-integrations/connectors/destination-scaffold-destination-python/Dockerfile deleted file mode 100644 index 7d70a8b79d08..000000000000 --- a/airbyte-integrations/connectors/destination-scaffold-destination-python/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY destination_scaffold_destination_python ./destination_scaffold_destination_python - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/destination-scaffold-destination-python diff --git a/airbyte-integrations/connectors/destination-scaffold-destination-python/README.md b/airbyte-integrations/connectors/destination-scaffold-destination-python/README.md index d0b3810f3948..f22d294346dd 100644 --- a/airbyte-integrations/connectors/destination-scaffold-destination-python/README.md +++ b/airbyte-integrations/connectors/destination-scaffold-destination-python/README.md @@ -8,9 +8,9 @@ For information about how to use this connector within Airbyte, see [the documen ### Prerequisites **To iterate on this connector, make sure to complete this prerequisites section.** -#### Minimum Python version required `= 3.7.0` +#### Minimum Python version required `= 3.9.0` -#### Build & Activate Virtual Environment and install dependencies +#### Activate Virtual Environment and install dependencies From this connector directory, create a virtual environment: ``` python -m venv .venv @@ -29,11 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:destination-scaffold-destination-python:build -``` #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/destinations/scaffold-destination-python) @@ -54,19 +49,68 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/destination-scaffold-destination-python:dev +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name destination-scaffold-destination-python build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/destination-scaffold-destination-python:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") -You can also build the connector image via Gradle: +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:destination-scaffold-destination-python:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/destination-scaffold-destination-python:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/destination-scaffold-destination-python:dev . +# Running the spec command against your patched connector +docker run airbyte/destination-scaffold-destination-python:dev spec +```` #### Run Then run any of the connector commands as follows: ``` @@ -97,16 +141,8 @@ python -m pytest integration_tests #### Acceptance Tests Coming soon: -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:destination-scaffold-destination-python:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:destination-scaffold-destination-python:integrationTest -``` +### Using `airbyte-ci` to run tests +See [airbyte-ci documentation](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#connectors-test-command) ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. diff --git a/airbyte-integrations/connectors/destination-scaffold-destination-python/build.gradle b/airbyte-integrations/connectors/destination-scaffold-destination-python/build.gradle deleted file mode 100644 index 3d537215bc9d..000000000000 --- a/airbyte-integrations/connectors/destination-scaffold-destination-python/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'destination_scaffold_destination_python' -} diff --git a/airbyte-integrations/connectors/destination-scaffold-destination-python/metadata.yaml b/airbyte-integrations/connectors/destination-scaffold-destination-python/metadata.yaml index 3b8c369a5312..20b876b5d86b 100644 --- a/airbyte-integrations/connectors/destination-scaffold-destination-python/metadata.yaml +++ b/airbyte-integrations/connectors/destination-scaffold-destination-python/metadata.yaml @@ -7,9 +7,14 @@ data: enabled: false cloud: enabled: false + connectorBuildOptions: + # Please update to the latest version of the connector base image. + # Please use the full address with sha256 hash to guarantee build reproducibility. + # https://hub.docker.com/r/airbyte/python-connector-base + baseImage: docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27 connectorSubtype: database connectorType: destination - definitionId: FAKE-UUID-0000-0000-000000000000 + definitionId: 1c342214-aad1-4344-8ee8-92c8c7e91c07 dockerImageTag: 0.1.0 dockerRepository: airbyte/destination-scaffold-destination-python githubIssueLabel: destination-scaffold-destination-python diff --git a/airbyte-integrations/connectors/destination-scylla/.dockerignore b/airbyte-integrations/connectors/destination-scylla/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-scylla/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-scylla/Dockerfile b/airbyte-integrations/connectors/destination-scylla/Dockerfile deleted file mode 100644 index cd2d7b2fb5a0..000000000000 --- a/airbyte-integrations/connectors/destination-scylla/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION destination-scylla - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION destination-scylla - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.3 -LABEL io.airbyte.name=airbyte/destination-scylla diff --git a/airbyte-integrations/connectors/destination-scylla/README.md b/airbyte-integrations/connectors/destination-scylla/README.md index 8e6e60e9f6b5..6fc6d930e4c3 100644 --- a/airbyte-integrations/connectors/destination-scylla/README.md +++ b/airbyte-integrations/connectors/destination-scylla/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-scylla:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-scylla:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-scylla:dev`. the Dockerfile. #### Run @@ -61,8 +62,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-scylla test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/scylla.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-scylla/build.gradle b/airbyte-integrations/connectors/destination-scylla/build.gradle index ef0fc27409c2..512279a1345e 100644 --- a/airbyte-integrations/connectors/destination-scylla/build.gradle +++ b/airbyte-integrations/connectors/destination-scylla/build.gradle @@ -1,9 +1,23 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.scylla.ScyllaDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] @@ -13,21 +27,11 @@ def scyllaDriver = '3.10.2-scylla-1' def assertVersion = '3.21.0' dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation "com.scylladb:scylla-driver-core:${scyllaDriver}" - testImplementation project(':airbyte-integrations:bases:standard-destination-test') // https://mvnrepository.com/artifact/org.assertj/assertj-core testImplementation "org.assertj:assertj-core:${assertVersion}" // https://mvnrepository.com/artifact/org.testcontainers/testcontainers - testImplementation libs.connectors.testcontainers.scylla - - - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-scylla') + testImplementation libs.testcontainers.scylla } diff --git a/airbyte-integrations/connectors/destination-scylla/docker-compose.yml b/airbyte-integrations/connectors/destination-scylla/docker-compose.yml index ad8561826411..af0ecac58303 100644 --- a/airbyte-integrations/connectors/destination-scylla/docker-compose.yml +++ b/airbyte-integrations/connectors/destination-scylla/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3' +version: "3" services: scylla1: @@ -7,7 +7,6 @@ services: - "9042:9042" container_name: scylla1 command: --smp 1 - # uncomment if you want to run a cluster of scylladb nodes # scylla2: # image: scylladb/scylla @@ -17,4 +16,4 @@ services: # scylla3: # image: scylladb/scylla # container_name: scylla3 -# command: --seeds=scylla1 \ No newline at end of file +# command: --seeds=scylla1 diff --git a/airbyte-integrations/connectors/destination-scylla/src/main/java/io/airbyte/integrations/destination/scylla/ScyllaCqlProvider.java b/airbyte-integrations/connectors/destination-scylla/src/main/java/io/airbyte/integrations/destination/scylla/ScyllaCqlProvider.java index b789e7d5d96d..d296bcc5faee 100644 --- a/airbyte-integrations/connectors/destination-scylla/src/main/java/io/airbyte/integrations/destination/scylla/ScyllaCqlProvider.java +++ b/airbyte-integrations/connectors/destination-scylla/src/main/java/io/airbyte/integrations/destination/scylla/ScyllaCqlProvider.java @@ -14,7 +14,7 @@ import com.datastax.driver.core.querybuilder.QueryBuilder; import com.datastax.driver.core.schemabuilder.SchemaBuilder; import com.datastax.driver.core.utils.UUIDs; -import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import java.io.Closeable; import java.time.Instant; import java.util.Date; diff --git a/airbyte-integrations/connectors/destination-scylla/src/main/java/io/airbyte/integrations/destination/scylla/ScyllaDestination.java b/airbyte-integrations/connectors/destination-scylla/src/main/java/io/airbyte/integrations/destination/scylla/ScyllaDestination.java index 0e154e6c5614..1a60831b9f15 100644 --- a/airbyte-integrations/connectors/destination-scylla/src/main/java/io/airbyte/integrations/destination/scylla/ScyllaDestination.java +++ b/airbyte-integrations/connectors/destination-scylla/src/main/java/io/airbyte/integrations/destination/scylla/ScyllaDestination.java @@ -5,10 +5,10 @@ package io.airbyte.integrations.destination.scylla; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/connectors/destination-scylla/src/main/java/io/airbyte/integrations/destination/scylla/ScyllaMessageConsumer.java b/airbyte-integrations/connectors/destination-scylla/src/main/java/io/airbyte/integrations/destination/scylla/ScyllaMessageConsumer.java index bb8ccd88b83e..2c9edfb4493a 100644 --- a/airbyte-integrations/connectors/destination-scylla/src/main/java/io/airbyte/integrations/destination/scylla/ScyllaMessageConsumer.java +++ b/airbyte-integrations/connectors/destination-scylla/src/main/java/io/airbyte/integrations/destination/scylla/ScyllaMessageConsumer.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.scylla; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/connectors/destination-scylla/src/main/java/io/airbyte/integrations/destination/scylla/ScyllaNameTransformer.java b/airbyte-integrations/connectors/destination-scylla/src/main/java/io/airbyte/integrations/destination/scylla/ScyllaNameTransformer.java index 4bcc0cd9b962..c45a9db07bd7 100644 --- a/airbyte-integrations/connectors/destination-scylla/src/main/java/io/airbyte/integrations/destination/scylla/ScyllaNameTransformer.java +++ b/airbyte-integrations/connectors/destination-scylla/src/main/java/io/airbyte/integrations/destination/scylla/ScyllaNameTransformer.java @@ -5,8 +5,8 @@ package io.airbyte.integrations.destination.scylla; import com.google.common.base.CharMatcher; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.commons.text.Names; -import io.airbyte.integrations.destination.StandardNameTransformer; class ScyllaNameTransformer extends StandardNameTransformer { diff --git a/airbyte-integrations/connectors/destination-scylla/src/test-integration/java/io/airbyte/integrations/destination/scylla/ScyllaCqlProviderTest.java b/airbyte-integrations/connectors/destination-scylla/src/test-integration/java/io/airbyte/integrations/destination/scylla/ScyllaCqlProviderTest.java index 852853927e0f..dea28fb7eb88 100644 --- a/airbyte-integrations/connectors/destination-scylla/src/test-integration/java/io/airbyte/integrations/destination/scylla/ScyllaCqlProviderTest.java +++ b/airbyte-integrations/connectors/destination-scylla/src/test-integration/java/io/airbyte/integrations/destination/scylla/ScyllaCqlProviderTest.java @@ -9,7 +9,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import com.datastax.driver.core.exceptions.InvalidQueryException; -import io.airbyte.integrations.util.HostPortResolver; +import io.airbyte.cdk.integrations.util.HostPortResolver; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/connectors/destination-scylla/src/test-integration/java/io/airbyte/integrations/destination/scylla/ScyllaDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-scylla/src/test-integration/java/io/airbyte/integrations/destination/scylla/ScyllaDestinationAcceptanceTest.java index 6ea7c8dca661..d2f9c7d7d1f5 100644 --- a/airbyte-integrations/connectors/destination-scylla/src/test-integration/java/io/airbyte/integrations/destination/scylla/ScyllaDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-scylla/src/test-integration/java/io/airbyte/integrations/destination/scylla/ScyllaDestinationAcceptanceTest.java @@ -5,12 +5,12 @@ package io.airbyte.integrations.destination.scylla; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.destination.scylla.ScyllaContainerInitializr.ScyllaContainer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; -import io.airbyte.integrations.util.HostPortResolver; import java.util.Comparator; import java.util.HashSet; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-scylla/src/test-integration/java/io/airbyte/integrations/destination/scylla/ScyllaDestinationTest.java b/airbyte-integrations/connectors/destination-scylla/src/test-integration/java/io/airbyte/integrations/destination/scylla/ScyllaDestinationTest.java index fff9e282e56a..53460b3f5714 100644 --- a/airbyte-integrations/connectors/destination-scylla/src/test-integration/java/io/airbyte/integrations/destination/scylla/ScyllaDestinationTest.java +++ b/airbyte-integrations/connectors/destination-scylla/src/test-integration/java/io/airbyte/integrations/destination/scylla/ScyllaDestinationTest.java @@ -6,8 +6,8 @@ import static org.assertj.core.api.Assertions.assertThat; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.integrations.destination.scylla.ScyllaContainerInitializr.ScyllaContainer; -import io.airbyte.integrations.util.HostPortResolver; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/connectors/destination-scylla/src/test/java/io/airbyte/integrations/destination/scylla/ScyllaRecordConsumerTest.java b/airbyte-integrations/connectors/destination-scylla/src/test/java/io/airbyte/integrations/destination/scylla/ScyllaRecordConsumerTest.java index ebcf531fc863..7dcf5275909d 100644 --- a/airbyte-integrations/connectors/destination-scylla/src/test/java/io/airbyte/integrations/destination/scylla/ScyllaRecordConsumerTest.java +++ b/airbyte-integrations/connectors/destination-scylla/src/test/java/io/airbyte/integrations/destination/scylla/ScyllaRecordConsumerTest.java @@ -5,9 +5,9 @@ package io.airbyte.integrations.destination.scylla; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.base.FailureTrackingAirbyteMessageConsumer; -import io.airbyte.integrations.standardtest.destination.PerStreamStateMessageTest; -import io.airbyte.integrations.util.HostPortResolver; +import io.airbyte.cdk.integrations.base.FailureTrackingAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.standardtest.destination.PerStreamStateMessageTest; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import java.util.function.Consumer; diff --git a/airbyte-integrations/connectors/destination-selectdb/.dockerignore b/airbyte-integrations/connectors/destination-selectdb/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-selectdb/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-selectdb/Dockerfile b/airbyte-integrations/connectors/destination-selectdb/Dockerfile deleted file mode 100644 index d1f387be1ac7..000000000000 --- a/airbyte-integrations/connectors/destination-selectdb/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte -ENV APPLICATION destination-selectdb - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte -ENV APPLICATION destination-selectdb - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/destination-selectdb diff --git a/airbyte-integrations/connectors/destination-selectdb/README.md b/airbyte-integrations/connectors/destination-selectdb/README.md index 076bba197398..cca3da005386 100644 --- a/airbyte-integrations/connectors/destination-selectdb/README.md +++ b/airbyte-integrations/connectors/destination-selectdb/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-selectdb:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-selectdb:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-selectdb:dev`. the Dockerfile. #### Run @@ -61,8 +62,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-selectdb test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/selectdb.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-selectdb/build.gradle b/airbyte-integrations/connectors/destination-selectdb/build.gradle index 1c8bf9e7adaf..0a654ec66f67 100644 --- a/airbyte-integrations/connectors/destination-selectdb/build.gradle +++ b/airbyte-integrations/connectors/destination-selectdb/build.gradle @@ -1,9 +1,23 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.selectdb.SelectdbDestination' } @@ -11,11 +25,4 @@ application { dependencies { implementation 'org.apache.commons:commons-csv:1.4' implementation group: 'mysql', name: 'mysql-connector-java', version: '8.0.16' - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-selectdb') } diff --git a/airbyte-integrations/connectors/destination-selectdb/src/main/java/io/airbyte/integrations/destination/selectdb/SelectdbConsumer.java b/airbyte-integrations/connectors/destination-selectdb/src/main/java/io/airbyte/integrations/destination/selectdb/SelectdbConsumer.java index 26789ef3d7b2..c14c7b2101d5 100644 --- a/airbyte-integrations/connectors/destination-selectdb/src/main/java/io/airbyte/integrations/destination/selectdb/SelectdbConsumer.java +++ b/airbyte-integrations/connectors/destination-selectdb/src/main/java/io/airbyte/integrations/destination/selectdb/SelectdbConsumer.java @@ -5,8 +5,8 @@ package io.airbyte.integrations.destination.selectdb; import com.fasterxml.jackson.core.io.JsonStringEncoder; +import io.airbyte.cdk.integrations.base.CommitOnStateAirbyteMessageConsumer; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.CommitOnStateAirbyteMessageConsumer; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/connectors/destination-selectdb/src/main/java/io/airbyte/integrations/destination/selectdb/SelectdbDestination.java b/airbyte-integrations/connectors/destination-selectdb/src/main/java/io/airbyte/integrations/destination/selectdb/SelectdbDestination.java index 240dede0efda..9619f3c19b8d 100644 --- a/airbyte-integrations/connectors/destination-selectdb/src/main/java/io/airbyte/integrations/destination/selectdb/SelectdbDestination.java +++ b/airbyte-integrations/connectors/destination-selectdb/src/main/java/io/airbyte/integrations/destination/selectdb/SelectdbDestination.java @@ -6,12 +6,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Preconditions; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.integrations.destination.selectdb.http.HttpUtil; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; diff --git a/airbyte-integrations/connectors/destination-selectdb/src/main/java/io/airbyte/integrations/destination/selectdb/SelectdbOperations.java b/airbyte-integrations/connectors/destination-selectdb/src/main/java/io/airbyte/integrations/destination/selectdb/SelectdbOperations.java index 1b6c7b2cbebc..05c322a8516d 100644 --- a/airbyte-integrations/connectors/destination-selectdb/src/main/java/io/airbyte/integrations/destination/selectdb/SelectdbOperations.java +++ b/airbyte-integrations/connectors/destination-selectdb/src/main/java/io/airbyte/integrations/destination/selectdb/SelectdbOperations.java @@ -5,7 +5,7 @@ package io.airbyte.integrations.destination.selectdb; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; diff --git a/airbyte-integrations/connectors/destination-selectdb/src/test-integration/java/io/airbyte/integrations/destination/selectdb/SelectdbDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-selectdb/src/test-integration/java/io/airbyte/integrations/destination/selectdb/SelectdbDestinationAcceptanceTest.java index 8ed3769edb9e..a3e097408da9 100644 --- a/airbyte-integrations/connectors/destination-selectdb/src/test-integration/java/io/airbyte/integrations/destination/selectdb/SelectdbDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-selectdb/src/test-integration/java/io/airbyte/integrations/destination/selectdb/SelectdbDestinationAcceptanceTest.java @@ -5,11 +5,11 @@ package io.airbyte.integrations.destination.selectdb; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; diff --git a/airbyte-integrations/connectors/destination-selectdb/src/test/java/io/airbyte/integrations/destination/selectdb/SelectdbDestinationTest.java b/airbyte-integrations/connectors/destination-selectdb/src/test/java/io/airbyte/integrations/destination/selectdb/SelectdbDestinationTest.java index e170e14dff5a..fdd249676d98 100644 --- a/airbyte-integrations/connectors/destination-selectdb/src/test/java/io/airbyte/integrations/destination/selectdb/SelectdbDestinationTest.java +++ b/airbyte-integrations/connectors/destination-selectdb/src/test/java/io/airbyte/integrations/destination/selectdb/SelectdbDestinationTest.java @@ -11,12 +11,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; diff --git a/airbyte-integrations/connectors/destination-sftp-json/README.md b/airbyte-integrations/connectors/destination-sftp-json/README.md index c4f46277a294..a584dd8a99b9 100644 --- a/airbyte-integrations/connectors/destination-sftp-json/README.md +++ b/airbyte-integrations/connectors/destination-sftp-json/README.md @@ -31,12 +31,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:destination-sftp-json:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/destinations/sftp-json) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_sftp_json/spec.json` file. @@ -56,18 +50,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/destination-sftp-json:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=destination-sftp-json build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:destination-sftp-json:airbyteDocker +An image will be built with the tag `airbyte/destination-sftp-json:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/destination-sftp-json:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -77,38 +72,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-sftp-json:dev che # messages.jsonl is a file containing line-separated JSON representing AirbyteMessages cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-sftp-json:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=destination-sftp-json test ``` -#### Acceptance Tests -Coming soon: -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:destination-sftp-json:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:destination-sftp-json:integrationTest -``` +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -118,8 +91,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-sftp-json test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/sftp-json.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-sftp-json/build.gradle b/airbyte-integrations/connectors/destination-sftp-json/build.gradle deleted file mode 100644 index e692847c50a7..000000000000 --- a/airbyte-integrations/connectors/destination-sftp-json/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'destination_sftp_json' -} diff --git a/airbyte-integrations/connectors/destination-snowflake/.dockerignore b/airbyte-integrations/connectors/destination-snowflake/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile deleted file mode 100644 index f5b15634eddd..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ /dev/null @@ -1,56 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev - -# uncomment to run Yourkit java profiling -#RUN yum install -y curl zip -# -#RUN curl -o /tmp/YourKit-JavaProfiler-2021.3-docker.zip https://www.yourkit.com/download/docker/YourKit-JavaProfiler-2021.3-docker.zip && \ -# unzip /tmp/YourKit-JavaProfiler-2021.3-docker.zip -d /usr/local && \ -# rm /tmp/YourKit-JavaProfiler-2021.3-docker.zip - -# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15 -RUN yum install -y python3 python3-devel jq sshpass git gcc-c++ && yum clean all && \ - alternatives --install /usr/bin/python python /usr/bin/python3 60 && \ - python -m ensurepip --upgrade && \ - # these two lines are a workaround for https://github.com/yaml/pyyaml/issues/601 - pip3 install wheel && \ - pip3 install "Cython<3.0" "pyyaml==5.4" --no-build-isolation && \ - pip3 install dbt-snowflake==1.0.0 "urllib3<2" - -COPY --from=airbyte/normalization-snowflake:dev /airbyte /airbyte -# Install python dependencies -WORKDIR /airbyte/base_python_structs -RUN pip3 install . -WORKDIR /airbyte/normalization_code -RUN pip3 install . -WORKDIR /airbyte/normalization_code/dbt-template/ -# Download external dbt dependencies -RUN dbt deps - -WORKDIR /airbyte - -ENV APPLICATION destination-snowflake -ENV AIRBYTE_NORMALIZATION_INTEGRATION snowflake - -# Needed for JDK17 (in turn, needed on M1 macs) - see https://github.com/snowflakedb/snowflake-jdbc/issues/589#issuecomment-983944767 -ENV DESTINATION_SNOWFLAKE_OPTS "--add-opens java.base/java.nio=ALL-UNNAMED" - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 - -ENV ENABLE_SENTRY true - - -LABEL io.airbyte.version=2.1.3 -LABEL io.airbyte.name=airbyte/destination-snowflake - -ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh" -ENTRYPOINT ["/airbyte/run_with_normalization.sh"] diff --git a/airbyte-integrations/connectors/destination-snowflake/build.gradle b/airbyte-integrations/connectors/destination-snowflake/build.gradle index 9059aeb8be09..4ce8698d75d8 100644 --- a/airbyte-integrations/connectors/destination-snowflake/build.gradle +++ b/airbyte-integrations/connectors/destination-snowflake/build.gradle @@ -1,70 +1,51 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.12.0' + features = ['db-destinations', 's3-destinations', 'typing-deduping'] + useLocalCdk = false +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.snowflake.SnowflakeDestinationRunner' // enable when profiling applicationDefaultJvmArgs = [ '-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0', -// '-Xmx2000m', + '-XX:NativeMemoryTracking=detail', + '-XX:+UnlockDiagnosticVMOptions', + '-XX:GCLockerRetryAllocationCount=100', // '-XX:NativeMemoryTracking=detail', // '-Djava.rmi.server.hostname=localhost', // '-Dcom.sun.management.jmxremote=true', // '-Dcom.sun.management.jmxremote.port=6000', // '-Dcom.sun.management.jmxremote.rmi.port=6000', -// '-Dcom.sun.management.jmxremote.local.only=false', +// '-Dcom.sun.management.jmxremote.local.only=false' // '-Dcom.sun.management.jmxremote.authenticate=false', // '-Dcom.sun.management.jmxremote.ssl=false', -// '-agentpath:/usr/local/YourKit-JavaProfiler-2021.3/bin/linux-x86-64/libyjpagent.so=port=10001,listen=all' ] } +integrationTestJava { + // This is needed to make the destination-snowflake tests succeed - https://github.com/snowflakedb/snowflake-jdbc/issues/589#issuecomment-983944767 + jvmArgs = ["--add-opens=java.base/java.nio=ALL-UNNAMED"] +} + dependencies { - implementation 'com.google.cloud:google-cloud-storage:1.113.16' - implementation 'com.google.auth:google-auth-library-oauth2-http:0.25.5' - // Updating to any newer version (e.g. 3.13.22) is causing a regression with normalization. - // See: https://github.com/airbytehq/airbyte/actions/runs/3078146312 - // implementation 'net.snowflake:snowflake-jdbc:3.13.19' - // Temporarily switch to a forked version of snowflake-jdbc to prevent infinitely-retried http requests - // the diff is to replace this while(true) with while(retryCount < 100) https://github.com/snowflakedb/snowflake-jdbc/blob/v3.13.19/src/main/java/net/snowflake/client/jdbc/RestRequest.java#L121 - // TODO (edgao) explain how you built this jar - implementation files('lib/snowflake-jdbc.jar') + implementation 'net.snowflake:snowflake-jdbc:3.14.1' implementation 'org.apache.commons:commons-csv:1.4' implementation 'org.apache.commons:commons-text:1.10.0' - implementation 'com.github.alexmojaki:s3-stream-upload:2.2.2' implementation "io.aesy:datasize:1.0.0" implementation 'com.zaxxer:HikariCP:5.0.1' - implementation project(':airbyte-config-oss:config-models-oss') - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:bases:bases-destination-jdbc') - implementation project(':airbyte-integrations:connectors:destination-gcs') - implementation project(':airbyte-integrations:bases:base-java-s3') - implementation project(':airbyte-integrations:bases:base-typing-deduping') - implementation libs.airbyte.protocol - // this is a configuration to make mockito work with final classes testImplementation 'org.mockito:mockito-inline:2.13.0' - integrationTestJavaImplementation project(':airbyte-connector-test-harnesses:acceptance-test-harness') - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:bases:base-typing-deduping-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-snowflake') integrationTestJavaImplementation 'org.apache.commons:commons-lang3:3.11' - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) -} - -tasks.named("airbyteDocker") { - // this is really inefficent (because base-normalization:airbyteDocker builds 9 docker images) - // but it's also just simple to implement. - // this also goes away once airbyte-ci becomes a reality. - dependsOn project(':airbyte-integrations:bases:base-normalization').airbyteDocker } diff --git a/airbyte-integrations/connectors/destination-snowflake/gradle.properties b/airbyte-integrations/connectors/destination-snowflake/gradle.properties index dcd58e02c0a5..3ce49dd31e29 100644 --- a/airbyte-integrations/connectors/destination-snowflake/gradle.properties +++ b/airbyte-integrations/connectors/destination-snowflake/gradle.properties @@ -1,3 +1,3 @@ # currently limit the number of parallel threads until further investigation into the issues \ # where Snowflake will fail to login using config credentials -numberThreads=4 +testExecutionConcurrency=4 diff --git a/airbyte-integrations/connectors/destination-snowflake/lib/snowflake-jdbc.jar b/airbyte-integrations/connectors/destination-snowflake/lib/snowflake-jdbc.jar deleted file mode 100644 index c8f782997abf..000000000000 Binary files a/airbyte-integrations/connectors/destination-snowflake/lib/snowflake-jdbc.jar and /dev/null differ diff --git a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml index 54b819007784..b5d7311d68df 100644 --- a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml +++ b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml @@ -1,40 +1,53 @@ data: + ab_internal: + ql: 300 + sl: 300 connectorSubtype: database connectorType: destination definitionId: 424892c4-daac-4491-b35d-c6688ba547ba - dockerImageTag: 2.1.3 + dockerImageTag: 3.4.22 dockerRepository: airbyte/destination-snowflake + documentationUrl: https://docs.airbyte.com/integrations/destinations/snowflake githubIssueLabel: destination-snowflake icon: snowflake.svg license: ELv2 name: Snowflake - normalizationConfig: - normalizationIntegrationType: snowflake - normalizationRepository: airbyte/normalization-snowflake - normalizationTag: 0.4.3 registries: cloud: enabled: true oss: enabled: true releaseStage: generally_available + releases: + breakingChanges: + 2.0.0: + message: Remove GCS/S3 loading method support. + upgradeDeadline: "2023-08-31" + 3.0.0: + message: + "**Do not upgrade until you have run a test upgrade as outlined [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#testing-destinations-v2-for-a-single-connection)**. + + This version introduces [Destinations V2](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2), + which provides better error handling, incremental delivery of data for large + syncs, and improved final table structures. To review the breaking changes, + and how to upgrade, see [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#quick-start-to-upgrading). + These changes will likely require updates to downstream dbt / SQL models, + which we walk through [here](https://docs.airbyte.com/release_notes/upgrading_to_destinations_v2/#updating-downstream-transformations). + + Selecting `Upgrade` will upgrade **all** connections using this destination + at their next sync. You can manually sync existing connections prior to + the next scheduled sync to start the upgrade early. + + " + upgradeDeadline: "2023-11-07" resourceRequirements: jobSpecific: - jobType: sync resourceRequirements: memory_limit: 2Gi memory_request: 2Gi - documentationUrl: https://docs.airbyte.com/integrations/destinations/snowflake + supportLevel: certified supportsDbt: true tags: - language:java - releases: - breakingChanges: - 2.0.0: - message: "Remove GCS/S3 loading method support." - upgradeDeadline: "2023-08-31" - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java index f35ae3adf14b..7c2bcdf8d2e7 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDatabase.java @@ -8,10 +8,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.zaxxer.hikari.HikariDataSource; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; import java.io.IOException; import java.io.PrintWriter; import java.net.URI; @@ -38,7 +38,7 @@ public class SnowflakeDatabase { private static final Logger LOGGER = LoggerFactory.getLogger(SnowflakeDatabase.class); - private static final int PAUSE_BETWEEN_TOKEN_REFRESH_MIN = 7; // snowflake access token's TTL is 10min and can't be modified + private static final int PAUSE_BETWEEN_TOKEN_REFRESH_MIN = 7; // snowflake access token TTL is 10min and can't be modified private static final Duration NETWORK_TIMEOUT = Duration.ofMinutes(1); private static final Duration QUERY_TIMEOUT = Duration.ofHours(3); @@ -134,6 +134,10 @@ public static HikariDataSource createDataSource(final JsonNode config, final Str // https://stackoverflow.com/questions/67409650/snowflake-jdbc-driver-internal-error-fail-to-retrieve-row-count-for-first-arrow properties.put("JDBC_QUERY_RESULT_FORMAT", "JSON"); + // https://docs.snowflake.com/sql-reference/parameters#abort-detached-query + // If the connector crashes, snowflake should abort in-flight queries. + properties.put("ABORT_DETACHED_QUERY", "true"); + // https://docs.snowflake.com/en/user-guide/jdbc-configure.html#jdbc-driver-connection-string if (config.has(JdbcUtils.JDBC_URL_PARAMS_KEY)) { jdbcUrl.append(config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText()); @@ -148,7 +152,7 @@ public static HikariDataSource createDataSource(final JsonNode config, final Str private static void createPrivateKeyFile(final String fileName, final String fileValue) { try (final PrintWriter out = new PrintWriter(fileName, StandardCharsets.UTF_8)) { out.print(fileValue); - } catch (IOException e) { + } catch (final IOException e) { throw new RuntimeException("Failed to create file for private key"); } } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java index f0c8244f15b0..217ed51abb78 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestination.java @@ -5,8 +5,9 @@ package io.airbyte.integrations.destination.snowflake; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.base.SerializedAirbyteMessageConsumer; -import io.airbyte.integrations.destination.jdbc.copy.SwitchingDestination; +import io.airbyte.cdk.integrations.base.AirbyteExceptionHandler; +import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.destination.jdbc.copy.SwitchingDestination; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import java.util.concurrent.Executors; @@ -35,6 +36,7 @@ public SnowflakeDestination(final String airbyteEnvironment) { public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonNode config, final ConfiguredAirbyteCatalog catalog, final Consumer outputRecordCollector) { + AirbyteExceptionHandler.addAllStringsInConfigForDeinterpolation(config); return new SnowflakeInternalStagingDestination(airbyteEnvironment).getSerializedMessageConsumer(config, catalog, outputRecordCollector); } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationResolver.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationResolver.java index 6f6044f3dd65..9ac2feb5f18d 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationResolver.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationResolver.java @@ -6,7 +6,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; -import io.airbyte.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.Destination; import io.airbyte.integrations.destination.snowflake.SnowflakeDestination.DestinationType; import java.util.Map; diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationRunner.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationRunner.java index ce252674d9af..9d6460dcb668 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationRunner.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationRunner.java @@ -6,12 +6,14 @@ import static io.airbyte.integrations.destination.snowflake.SnowflakeDestination.SCHEDULED_EXECUTOR_SERVICE; -import io.airbyte.integrations.base.adaptive.AdaptiveDestinationRunner; +import io.airbyte.cdk.integrations.base.AirbyteExceptionHandler; +import io.airbyte.cdk.integrations.base.adaptive.AdaptiveDestinationRunner; +import net.snowflake.client.jdbc.SnowflakeSQLException; public class SnowflakeDestinationRunner { public static void main(final String[] args) throws Exception { - + AirbyteExceptionHandler.addThrowableForDeinterpolation(SnowflakeSQLException.class); AdaptiveDestinationRunner.baseOnEnv() .withOssDestination(() -> new SnowflakeDestination(OssCloudEnvVarConsts.AIRBYTE_OSS)) .withCloudDestination(() -> new SnowflakeDestination(OssCloudEnvVarConsts.AIRBYTE_CLOUD)) diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java index 99f8f1200e4e..f0f2732518ad 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestination.java @@ -5,31 +5,34 @@ package io.airbyte.integrations.destination.snowflake; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.TypingAndDedupingFlag; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination; +import io.airbyte.cdk.integrations.destination.jdbc.typing_deduping.JdbcSqlGenerator; +import io.airbyte.cdk.integrations.destination.staging.StagingConsumerFactory; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.SerializedAirbyteMessageConsumer; -import io.airbyte.integrations.base.TypingAndDedupingFlag; import io.airbyte.integrations.base.destination.typing_deduping.CatalogParser; import io.airbyte.integrations.base.destination.typing_deduping.DefaultTyperDeduper; -import io.airbyte.integrations.base.destination.typing_deduping.NoopTyperDeduper; +import io.airbyte.integrations.base.destination.typing_deduping.NoOpTyperDeduperWithV1V2Migrations; import io.airbyte.integrations.base.destination.typing_deduping.ParsedCatalog; import io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeOperationValve; import io.airbyte.integrations.base.destination.typing_deduping.TyperDeduper; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import io.airbyte.integrations.destination.snowflake.typing_deduping.SnowflakeDestinationHandler; import io.airbyte.integrations.destination.snowflake.typing_deduping.SnowflakeSqlGenerator; import io.airbyte.integrations.destination.snowflake.typing_deduping.SnowflakeV1V2Migrator; -import io.airbyte.integrations.destination.staging.StagingConsumerFactory; +import io.airbyte.integrations.destination.snowflake.typing_deduping.SnowflakeV2TableMigrator; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.util.Collections; import java.util.Map; +import java.util.Optional; import java.util.UUID; import java.util.function.Consumer; import javax.sql.DataSource; @@ -40,7 +43,9 @@ public class SnowflakeInternalStagingDestination extends AbstractJdbcDestination implements Destination { private static final Logger LOGGER = LoggerFactory.getLogger(SnowflakeInternalStagingDestination.class); - private static final String RAW_SCHEMA_OVERRIDE = "raw_data_schema"; + public static final String RAW_SCHEMA_OVERRIDE = "raw_data_schema"; + + public static final String DISABLE_TYPE_DEDUPE = "disable_type_dedupe"; private final String airbyteEnvironment; public SnowflakeInternalStagingDestination(final String airbyteEnvironment) { @@ -99,12 +104,12 @@ private static void attemptStageOperations(final String outputSchema, } @Override - protected DataSource getDataSource(final JsonNode config) { + public DataSource getDataSource(final JsonNode config) { return SnowflakeDatabase.createDataSource(config, airbyteEnvironment); } @Override - protected JdbcDatabase getDatabase(final DataSource dataSource) { + public JdbcDatabase getDatabase(final DataSource dataSource) { return SnowflakeDatabase.getDatabase(dataSource); } @@ -119,6 +124,11 @@ public JsonNode toJdbcConfig(final JsonNode config) { return Jsons.emptyObject(); } + @Override + protected JdbcSqlGenerator getSqlGenerator() { + throw new UnsupportedOperationException("Snowflake does not yet use the native JDBC DV2 interface"); + } + @Override public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonNode config, final ConfiguredAirbyteCatalog catalog, @@ -134,21 +144,25 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN final ParsedCatalog parsedCatalog; final TyperDeduper typerDeduper; final JdbcDatabase database = getDatabase(getDataSource(config)); - if (TypingAndDedupingFlag.isDestinationV2()) { - final String databaseName = config.get(JdbcUtils.DATABASE_KEY).asText(); - final SnowflakeDestinationHandler snowflakeDestinationHandler = new SnowflakeDestinationHandler(databaseName, database); - final CatalogParser catalogParser; - if (TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE).isPresent()) { - catalogParser = new CatalogParser(sqlGenerator, TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE).get()); - } else { - catalogParser = new CatalogParser(sqlGenerator); - } - parsedCatalog = catalogParser.parseCatalog(catalog); - final SnowflakeV1V2Migrator migrator = new SnowflakeV1V2Migrator(getNamingResolver(), database, databaseName); - typerDeduper = new DefaultTyperDeduper<>(sqlGenerator, snowflakeDestinationHandler, parsedCatalog, migrator); + final String databaseName = config.get(JdbcUtils.DATABASE_KEY).asText(); + final SnowflakeDestinationHandler snowflakeDestinationHandler = new SnowflakeDestinationHandler(databaseName, database); + final CatalogParser catalogParser; + if (TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE).isPresent()) { + catalogParser = new CatalogParser(sqlGenerator, TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE).get()); } else { - parsedCatalog = null; - typerDeduper = new NoopTyperDeduper(); + catalogParser = new CatalogParser(sqlGenerator); + } + parsedCatalog = catalogParser.parseCatalog(catalog); + final SnowflakeV1V2Migrator migrator = new SnowflakeV1V2Migrator(getNamingResolver(), database, databaseName); + final SnowflakeV2TableMigrator v2TableMigrator = new SnowflakeV2TableMigrator(database, databaseName, sqlGenerator, snowflakeDestinationHandler); + final boolean disableTypeDedupe = config.has(DISABLE_TYPE_DEDUPE) && config.get(DISABLE_TYPE_DEDUPE).asBoolean(false); + final int defaultThreadCount = 8; + if (disableTypeDedupe) { + typerDeduper = new NoOpTyperDeduperWithV1V2Migrations<>(sqlGenerator, snowflakeDestinationHandler, parsedCatalog, migrator, v2TableMigrator, + defaultThreadCount); + } else { + typerDeduper = + new DefaultTyperDeduper<>(sqlGenerator, snowflakeDestinationHandler, parsedCatalog, migrator, v2TableMigrator, defaultThreadCount); } return new StagingConsumerFactory().createAsync( @@ -162,7 +176,13 @@ public SerializedAirbyteMessageConsumer getSerializedMessageConsumer(final JsonN new TypeAndDedupeOperationValve(), typerDeduper, parsedCatalog, - defaultNamespace); + defaultNamespace, + true, + Optional.of(getSnowflakeBufferMemoryLimit())); + } + + private static long getSnowflakeBufferMemoryLimit() { + return (long) (Runtime.getRuntime().maxMemory() * 0.5); } } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperations.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperations.java index 2391ef11ac5e..d789fe7e5c26 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperations.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperations.java @@ -5,11 +5,10 @@ package io.airbyte.integrations.destination.snowflake; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.record_buffer.SerializableBuffer; import io.airbyte.commons.string.Strings; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.TypingAndDedupingFlag; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.record_buffer.SerializableBuffer; import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; @@ -40,44 +39,30 @@ public class SnowflakeInternalStagingSqlOperations extends SnowflakeSqlStagingOp FIELD_OPTIONALLY_ENCLOSED_BY = '"' NULL_IF=('') )"""; - private static final String COPY_QUERY = """ - COPY INTO %s.%s FROM '@%s/%s' - file_format = ( - type = csv - compression = auto - field_delimiter = ',' - skip_header = 0 - FIELD_OPTIONALLY_ENCLOSED_BY = '"' - NULL_IF=('') - )"""; private static final String DROP_STAGE_QUERY = "DROP STAGE IF EXISTS %s;"; private static final String REMOVE_QUERY = "REMOVE @%s;"; private static final Logger LOGGER = LoggerFactory.getLogger(SnowflakeSqlOperations.class); private final NamingConventionTransformer nameTransformer; - private final boolean use1s1t; public SnowflakeInternalStagingSqlOperations(final NamingConventionTransformer nameTransformer) { this.nameTransformer = nameTransformer; - this.use1s1t = TypingAndDedupingFlag.isDestinationV2(); } @Override public String getStageName(final String namespace, final String streamName) { - if (use1s1t) { - return String.join(".", - '"' + nameTransformer.convertStreamName(namespace) + '"', - '"' + nameTransformer.convertStreamName(streamName) + '"'); - } else { - return nameTransformer.applyDefaultCase(String.join(".", - nameTransformer.convertStreamName(namespace), - nameTransformer.convertStreamName(streamName))); - } + return String.join(".", + '"' + nameTransformer.convertStreamName(namespace) + '"', + '"' + nameTransformer.convertStreamName(streamName) + '"'); } @Override - public String getStagingPath(final UUID connectionId, final String namespace, final String streamName, final DateTime writeDatetime) { + public String getStagingPath(final UUID connectionId, + final String namespace, + final String streamName, + final String outputTableName, + final DateTime writeDatetime) { // see https://docs.snowflake.com/en/user-guide/data-load-considerations-stage.html return nameTransformer.applyDefaultCase(String.format("%s/%02d/%02d/%02d/%s/", writeDatetime.year().get(), @@ -213,11 +198,7 @@ protected String getCopyQuery(final String stageName, final List stagedFiles, final String dstTableName, final String schemaName) { - if (use1s1t) { - return String.format(COPY_QUERY_1S1T + generateFilesList(stagedFiles) + ";", schemaName, dstTableName, stageName, stagingPath); - } else { - return String.format(COPY_QUERY + generateFilesList(stagedFiles) + ";", schemaName, dstTableName, stageName, stagingPath); - } + return String.format(COPY_QUERY_1S1T + generateFilesList(stagedFiles) + ";", schemaName, dstTableName, stageName, stagingPath); } @Override diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSQLNameTransformer.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSQLNameTransformer.java index f97e9bf45aeb..9fce5f3ebf69 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSQLNameTransformer.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSQLNameTransformer.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.snowflake; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; public class SnowflakeSQLNameTransformer extends StandardNameTransformer { diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperations.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperations.java index 5a517d17c715..6be94ea5032f 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperations.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperations.java @@ -5,14 +5,13 @@ package io.airbyte.integrations.destination.snowflake; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.jdbc.JdbcSqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperationsUtils; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; import io.airbyte.commons.exceptions.ConfigErrorException; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.base.TypingAndDedupingFlag; -import io.airbyte.integrations.destination.jdbc.JdbcSqlOperations; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.SqlOperationsUtils; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.sql.SQLException; import java.util.List; import java.util.Optional; @@ -32,22 +31,12 @@ class SnowflakeSqlOperations extends JdbcSqlOperations implements SqlOperations private static final String NO_PRIVILEGES_ERROR_MESSAGE = "but current role has no privileges on it"; private static final String IP_NOT_IN_WHITE_LIST_ERR_MSG = "not allowed to access Snowflake"; - private final boolean use1s1t; - - public SnowflakeSqlOperations() { - this.use1s1t = TypingAndDedupingFlag.isDestinationV2(); - } - @Override public void createSchemaIfNotExists(final JdbcDatabase database, final String schemaName) throws Exception { try { if (!schemaSet.contains(schemaName) && !isSchemaExists(database, schemaName)) { - if (use1s1t) { - // 1s1t is assuming a lowercase airbyte_internal schema name, so we need to quote it - database.execute(String.format("CREATE SCHEMA IF NOT EXISTS \"%s\";", schemaName)); - } else { - database.execute(String.format("CREATE SCHEMA IF NOT EXISTS %s;", schemaName)); - } + // 1s1t is assuming a lowercase airbyte_internal schema name, so we need to quote it + database.execute(String.format("CREATE SCHEMA IF NOT EXISTS \"%s\";", schemaName)); schemaSet.add(schemaName); } } catch (final Exception e) { @@ -57,41 +46,26 @@ public void createSchemaIfNotExists(final JdbcDatabase database, final String sc @Override public String createTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { - if (use1s1t) { - return String.format( - """ - CREATE TABLE IF NOT EXISTS "%s"."%s" ( - "%s" VARCHAR PRIMARY KEY, - "%s" TIMESTAMP WITH TIME ZONE DEFAULT current_timestamp(), - "%s" TIMESTAMP WITH TIME ZONE DEFAULT NULL, - "%s" VARIANT - ) data_retention_time_in_days = 0;""", - schemaName, - tableName, - JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, - JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT, - JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT, - JavaBaseConstants.COLUMN_NAME_DATA); - } else { - return String.format( - """ - CREATE TABLE IF NOT EXISTS %s.%s ( - %s VARCHAR PRIMARY KEY, - %s VARIANT, - %s TIMESTAMP WITH TIME ZONE DEFAULT current_timestamp() - ) data_retention_time_in_days = 0;""", - schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_ID, JavaBaseConstants.COLUMN_NAME_DATA, JavaBaseConstants.COLUMN_NAME_EMITTED_AT); - } + return String.format( + """ + CREATE TABLE IF NOT EXISTS "%s"."%s" ( + "%s" VARCHAR PRIMARY KEY, + "%s" TIMESTAMP WITH TIME ZONE DEFAULT current_timestamp(), + "%s" TIMESTAMP WITH TIME ZONE DEFAULT NULL, + "%s" VARIANT + ) data_retention_time_in_days = 0;""", + schemaName, + tableName, + JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, + JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT, + JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT, + JavaBaseConstants.COLUMN_NAME_DATA); } @Override public boolean isSchemaExists(final JdbcDatabase database, final String outputSchema) throws Exception { try (final Stream results = database.unsafeQuery(SHOW_SCHEMAS)) { - if (use1s1t) { - return results.map(schemas -> schemas.get(NAME).asText()).anyMatch(outputSchema::equals); - } else { - return results.map(schemas -> schemas.get(NAME).asText()).anyMatch(outputSchema::equalsIgnoreCase); - } + return results.map(schemas -> schemas.get(NAME).asText()).anyMatch(outputSchema::equals); } catch (final Exception e) { throw checkForKnownConfigExceptions(e).orElseThrow(() -> e); } @@ -99,25 +73,17 @@ public boolean isSchemaExists(final JdbcDatabase database, final String outputSc @Override public String truncateTableQuery(final JdbcDatabase database, final String schemaName, final String tableName) { - if (use1s1t) { - return String.format("TRUNCATE TABLE \"%s\".\"%s\";\n", schemaName, tableName); - } else { - return String.format("TRUNCATE TABLE %s.%s;\n", schemaName, tableName); - } + return String.format("TRUNCATE TABLE \"%s\".\"%s\";\n", schemaName, tableName); } @Override public String dropTableIfExistsQuery(final String schemaName, final String tableName) { - if (use1s1t) { - return String.format("DROP TABLE IF EXISTS \"%s\".\"%s\";\n", schemaName, tableName); - } else { - return String.format("DROP TABLE IF EXISTS %s.%s;\n", schemaName, tableName); - } + return String.format("DROP TABLE IF EXISTS \"%s\".\"%s\";\n", schemaName, tableName); } @Override public void insertRecordsInternal(final JdbcDatabase database, - final List records, + final List records, final String schemaName, final String tableName) throws SQLException { @@ -130,22 +96,23 @@ public void insertRecordsInternal(final JdbcDatabase database, // (?, ?, ?), // ... final String insertQuery; - if (use1s1t) { - // Note that the column order is weird here - that's intentional, to avoid needing to change - // SqlOperationsUtils.insertRawRecordsInSingleQuery to support a different column order. - insertQuery = String.format( - "INSERT INTO \"%s\".\"%s\" (\"%s\", \"%s\", \"%s\") SELECT column1, parse_json(column2), column3 FROM VALUES\n", - schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, JavaBaseConstants.COLUMN_NAME_DATA, - JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT); - } else { - insertQuery = String.format( - "INSERT INTO %s.%s (%s, %s, %s) SELECT column1, parse_json(column2), column3 FROM VALUES\n", - schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_ID, JavaBaseConstants.COLUMN_NAME_DATA, JavaBaseConstants.COLUMN_NAME_EMITTED_AT); - } + // Note that the column order is weird here - that's intentional, to avoid needing to change + // SqlOperationsUtils.insertRawRecordsInSingleQuery to support a different column order. + insertQuery = String.format( + "INSERT INTO \"%s\".\"%s\" (\"%s\", \"%s\", \"%s\") SELECT column1, parse_json(column2), column3 FROM VALUES\n", + schemaName, tableName, JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, JavaBaseConstants.COLUMN_NAME_DATA, + JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT); final String recordQuery = "(?, ?, ?),\n"; SqlOperationsUtils.insertRawRecordsInSingleQuery(insertQuery, recordQuery, database, records); } + @Override + protected void insertRecordsInternalV2(final JdbcDatabase jdbcDatabase, final List list, final String s, final String s1) + throws Exception { + // Snowflake doesn't have standard inserts... so we probably never want to do this + throw new UnsupportedOperationException("Snowflake does not use the native JDBC DV2 interface"); + } + protected String generateFilesList(final List files) { if (0 < files.size() && files.size() < MAX_FILES_IN_LOADING_QUERY_LIMIT) { // see https://docs.snowflake.com/en/user-guide/data-load-considerations-load.html#lists-of-files diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlStagingOperations.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlStagingOperations.java index fdddd805b61b..8d4a42e6de85 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlStagingOperations.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlStagingOperations.java @@ -4,12 +4,12 @@ package io.airbyte.integrations.destination.snowflake; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.record_buffer.FileBuffer; +import io.airbyte.cdk.integrations.destination.s3.csv.CsvSerializedBuffer; +import io.airbyte.cdk.integrations.destination.s3.csv.StagingDatabaseCsvSheetGenerator; +import io.airbyte.cdk.integrations.destination.staging.StagingOperations; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.record_buffer.FileBuffer; -import io.airbyte.integrations.destination.s3.csv.CsvSerializedBuffer; -import io.airbyte.integrations.destination.s3.csv.StagingDatabaseCsvSheetGenerator; -import io.airbyte.integrations.destination.staging.StagingOperations; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.util.Map; @@ -25,7 +25,7 @@ protected void attemptWriteToStage(final String outputSchema, final CsvSerializedBuffer csvSerializedBuffer = new CsvSerializedBuffer( new FileBuffer(CsvSerializedBuffer.CSV_GZ_SUFFIX), - new StagingDatabaseCsvSheetGenerator(), + new StagingDatabaseCsvSheetGenerator(true), true); // create a dummy stream\records that will bed used to test uploading diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeColumnDefinition.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeColumnDefinition.java new file mode 100644 index 000000000000..06be84ffe67f --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeColumnDefinition.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.snowflake.typing_deduping; + +/** + * isNullable is only used to execute a migration away from an older version of + * destination-snowflake, where we created PK columns as NOT NULL. This caused a lot of problems + * because many sources emit null PKs. We may want to remove this field eventually. + */ +public record SnowflakeColumnDefinition(String type, boolean isNullable) { + + @Deprecated + public boolean isNullable() { + return isNullable; + } + +} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.java index ba8794927743..3b6cca39daa2 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.java @@ -4,49 +4,57 @@ package io.airbyte.integrations.destination.snowflake.typing_deduping; -import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.integrations.base.destination.typing_deduping.DestinationHandler; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import java.sql.ResultSet; import java.sql.SQLException; +import java.time.Instant; import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.UUID; +import net.snowflake.client.jdbc.SnowflakeSQLException; +import org.apache.commons.text.StringSubstitutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class SnowflakeDestinationHandler implements DestinationHandler { private static final Logger LOGGER = LoggerFactory.getLogger(SnowflakeDestinationHandler.class); + public static final String EXCEPTION_COMMON_PREFIX = "JavaScript execution error: Uncaught Execution of multiple statements failed on statement"; private final String databaseName; private final JdbcDatabase database; - public SnowflakeDestinationHandler(String databaseName, JdbcDatabase database) { + public SnowflakeDestinationHandler(final String databaseName, final JdbcDatabase database) { this.databaseName = databaseName; this.database = database; } @Override - public Optional findExistingTable(StreamId id) throws SQLException { + public Optional findExistingTable(final StreamId id) throws SQLException { // The obvious database.getMetaData().getColumns() solution doesn't work, because JDBC translates // VARIANT as VARCHAR - LinkedHashMap columns = database.queryJsons( + final LinkedHashMap columns = database.queryJsons( """ - SELECT column_name, data_type + SELECT column_name, data_type, is_nullable FROM information_schema.columns WHERE table_catalog = ? AND table_schema = ? AND table_name = ? ORDER BY ordinal_position; """, - databaseName, - id.finalNamespace(), - id.finalName()).stream() + databaseName.toUpperCase(), + id.finalNamespace().toUpperCase(), + id.finalName().toUpperCase()).stream() .collect(LinkedHashMap::new, - (map, row) -> map.put(row.get("COLUMN_NAME").asText(), row.get("DATA_TYPE").asText()), + (map, row) -> map.put( + row.get("COLUMN_NAME").asText(), + new SnowflakeColumnDefinition(row.get("DATA_TYPE").asText(), fromSnowflakeBoolean(row.get("IS_NULLABLE").asText()))), LinkedHashMap::putAll); - // TODO query for indexes/partitioning/etc - if (columns.isEmpty()) { return Optional.empty(); } else { @@ -55,8 +63,8 @@ public Optional findExistingTable(StreamId id) throws } @Override - public boolean isFinalTableEmpty(StreamId id) throws SQLException { - int rowCount = database.queryInt( + public boolean isFinalTableEmpty(final StreamId id) throws SQLException { + final int rowCount = database.queryInt( """ SELECT row_count FROM information_schema.tables @@ -64,24 +72,93 @@ public boolean isFinalTableEmpty(StreamId id) throws SQLException { AND table_schema = ? AND table_name = ? """, - databaseName, - id.finalNamespace(), - id.finalName()); + databaseName.toUpperCase(), + id.finalNamespace().toUpperCase(), + id.finalName().toUpperCase()); return rowCount == 0; } @Override - public void execute(String sql) throws Exception { - if ("".equals(sql)) { - return; + public InitialRawTableState getInitialRawTableState(final StreamId id) throws Exception { + final ResultSet tables = database.getMetaData().getTables( + databaseName, + id.rawNamespace(), + id.rawName(), + null); + if (!tables.next()) { + return new InitialRawTableState(false, Optional.empty()); + } + // Snowflake timestamps have nanosecond precision, so decrement by 1ns + // And use two explicit queries because COALESCE doesn't short-circuit. + // This first query tries to find the oldest raw record with loaded_at = NULL + final Optional minUnloadedTimestamp = Optional.ofNullable(database.queryStrings( + conn -> conn.createStatement().executeQuery(new StringSubstitutor(Map.of( + "raw_table", id.rawTableId(SnowflakeSqlGenerator.QUOTE))).replace( + """ + SELECT to_varchar( + TIMESTAMPADD(NANOSECOND, -1, MIN("_airbyte_extracted_at")), + 'YYYY-MM-DDTHH24:MI:SS.FF9TZH:TZM' + ) AS MIN_TIMESTAMP + FROM ${raw_table} + WHERE "_airbyte_loaded_at" IS NULL + """)), + // The query will always return exactly one record, so use .get(0) + record -> record.getString("MIN_TIMESTAMP")).get(0)); + if (minUnloadedTimestamp.isPresent()) { + return new InitialRawTableState(true, minUnloadedTimestamp.map(Instant::parse)); } + + // If there are no unloaded raw records, then we can safely skip all existing raw records. + // This second query just finds the newest raw record. + final Optional maxTimestamp = Optional.ofNullable(database.queryStrings( + conn -> conn.createStatement().executeQuery(new StringSubstitutor(Map.of( + "raw_table", id.rawTableId(SnowflakeSqlGenerator.QUOTE))).replace( + """ + SELECT to_varchar( + MAX("_airbyte_extracted_at"), + 'YYYY-MM-DDTHH24:MI:SS.FF9TZH:TZM' + ) AS MIN_TIMESTAMP + FROM ${raw_table} + """)), + record -> record.getString("MIN_TIMESTAMP")).get(0)); + return new InitialRawTableState(false, maxTimestamp.map(Instant::parse)); + } + + @Override + public void execute(final Sql sql) throws Exception { + final List transactions = sql.asSqlStrings("BEGIN TRANSACTION", "COMMIT"); final UUID queryId = UUID.randomUUID(); - LOGGER.info("Executing sql {}: {}", queryId, sql); - long startTime = System.currentTimeMillis(); + for (final String transaction : transactions) { + final UUID transactionId = UUID.randomUUID(); + LOGGER.debug("Executing sql {}-{}: {}", queryId, transactionId, transaction); + final long startTime = System.currentTimeMillis(); - database.execute(sql); + try { + database.execute(transaction); + } catch (final SnowflakeSQLException e) { + LOGGER.error("Sql {} failed", queryId, e); + // Snowflake SQL exceptions by default may not be super helpful, so we try to extract the relevant + // part of the message. + final String trimmedMessage; + if (e.getMessage().startsWith(EXCEPTION_COMMON_PREFIX)) { + // The first line is a pretty generic message, so just remove it + trimmedMessage = e.getMessage().substring(e.getMessage().indexOf("\n") + 1); + } else { + trimmedMessage = e.getMessage(); + } + throw new RuntimeException(trimmedMessage, e); + } + + LOGGER.debug("Sql {}-{} completed in {} ms", queryId, transactionId, System.currentTimeMillis() - startTime); + } + } - LOGGER.info("Sql {} completed in {} ms", queryId, System.currentTimeMillis() - startTime); + /** + * In snowflake information_schema tables, booleans return "YES" and "NO", which DataBind doesn't + * know how to use + */ + private boolean fromSnowflakeBoolean(final String input) { + return input.equalsIgnoreCase("yes"); } } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java index 5f77e4445447..88733c74315d 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java @@ -4,14 +4,19 @@ package io.airbyte.integrations.destination.snowflake.typing_deduping; +import static io.airbyte.integrations.base.destination.typing_deduping.Sql.concat; +import static io.airbyte.integrations.base.destination.typing_deduping.Sql.transactionally; +import static io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeTransaction.SOFT_RESET_SUFFIX; import static java.util.stream.Collectors.joining; import com.google.common.annotations.VisibleForTesting; -import io.airbyte.integrations.base.JavaBaseConstants; +import com.google.common.collect.ImmutableList; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType; import io.airbyte.integrations.base.destination.typing_deduping.Array; import io.airbyte.integrations.base.destination.typing_deduping.ColumnId; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; import io.airbyte.integrations.base.destination.typing_deduping.SqlGenerator; import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; import io.airbyte.integrations.base.destination.typing_deduping.StreamId; @@ -20,10 +25,14 @@ import io.airbyte.integrations.base.destination.typing_deduping.Union; import io.airbyte.integrations.base.destination.typing_deduping.UnsupportedOneOf; import io.airbyte.protocol.models.v0.DestinationSyncMode; +import java.time.Instant; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.StringSubstitutor; @@ -33,22 +42,32 @@ public class SnowflakeSqlGenerator implements SqlGenerator RESERVED_COLUMN_NAMES = ImmutableList.of( + "CURRENT_DATE", + "CURRENT_TIME", + "CURRENT_TIMESTAMP", + "CURRENT_USER", + "LOCALTIME", + "LOCALTIMESTAMP"); + @Override public StreamId buildStreamId(final String namespace, final String name, final String rawNamespaceOverride) { - // No escaping needed, as far as I can tell. We quote all our identifier names. return new StreamId( - escapeIdentifier(namespace), - escapeIdentifier(name), - escapeIdentifier(rawNamespaceOverride), - escapeIdentifier(StreamId.concatenateRawTableName(namespace, name)), + escapeSqlIdentifier(namespace).toUpperCase(), + escapeSqlIdentifier(name).toUpperCase(), + escapeSqlIdentifier(rawNamespaceOverride), + escapeSqlIdentifier(StreamId.concatenateRawTableName(namespace, name)), namespace, name); } @Override - public ColumnId buildColumnId(final String name) { - // No escaping needed, as far as I can tell. We quote all our identifier names. - return new ColumnId(escapeIdentifier(name), name, name); + public ColumnId buildColumnId(final String name, final String suffix) { + final String escapedName = prefixReservedColumnName(escapeSqlIdentifier(name).toUpperCase()) + suffix.toUpperCase(); + return new ColumnId(escapedName, name, escapedName); } public String toDialectType(final AirbyteType type) { @@ -89,97 +108,89 @@ public String toDialectType(final AirbyteProtocolType airbyteProtocolType) { } @Override - public String createTable(final StreamConfig stream, final String suffix, final boolean force) { + public Sql createSchema(final String schema) { + return Sql.of(new StringSubstitutor(Map.of("schema", StringUtils.wrap(schema, QUOTE))) + .replace("CREATE SCHEMA IF NOT EXISTS ${schema};")); + } + + @Override + public Sql createTable(final StreamConfig stream, final String suffix, final boolean force) { final String columnDeclarations = stream.columns().entrySet().stream() .map(column -> "," + column.getKey().name(QUOTE) + " " + toDialectType(column.getValue())) .collect(joining("\n")); final String forceCreateTable = force ? "OR REPLACE" : ""; - return new StringSubstitutor(Map.of( - "final_namespace", stream.id().finalNamespace(QUOTE), - "final_table_id", stream.id().finalTableId(QUOTE, suffix), + return Sql.of(new StringSubstitutor(Map.of( + "final_table_id", stream.id().finalTableId(QUOTE, suffix.toUpperCase()), "force_create_table", forceCreateTable, "column_declarations", columnDeclarations)).replace( """ - CREATE SCHEMA IF NOT EXISTS ${final_namespace}; - CREATE ${force_create_table} TABLE ${final_table_id} ( - "_airbyte_raw_id" TEXT NOT NULL, - "_airbyte_extracted_at" TIMESTAMP_TZ NOT NULL, - "_airbyte_meta" VARIANT NOT NULL + "_AIRBYTE_RAW_ID" TEXT NOT NULL, + "_AIRBYTE_EXTRACTED_AT" TIMESTAMP_TZ NOT NULL, + "_AIRBYTE_META" VARIANT NOT NULL ${column_declarations} ); - """); + """)); } @Override public boolean existingSchemaMatchesStreamConfig(final StreamConfig stream, final SnowflakeTableDefinition existingTable) throws TableNotMigratedException { + final Set pks = getPks(stream); // Check that the columns match, with special handling for the metadata columns. - final LinkedHashMap intendedColumns = stream.columns().entrySet().stream() + final LinkedHashMap intendedColumns = stream.columns().entrySet().stream() .collect(LinkedHashMap::new, (map, column) -> map.put(column.getKey().name(), toDialectType(column.getValue())), LinkedHashMap::putAll); final LinkedHashMap actualColumns = existingTable.columns().entrySet().stream() - .filter(column -> !JavaBaseConstants.V2_FINAL_TABLE_METADATA_COLUMNS.contains(column.getKey())) + .filter(column -> JavaBaseConstants.V2_FINAL_TABLE_METADATA_COLUMNS.stream().map(String::toUpperCase) + .noneMatch(airbyteColumnName -> airbyteColumnName.equals(column.getKey()))) .collect(LinkedHashMap::new, - (map, column) -> map.put(column.getKey(), column.getValue()), + (map, column) -> map.put(column.getKey(), column.getValue().type()), LinkedHashMap::putAll); + // soft-resetting https://github.com/airbytehq/airbyte/pull/31082 + @SuppressWarnings("deprecation") + final boolean hasPksWithNonNullConstraint = existingTable.columns().entrySet().stream() + .anyMatch(c -> pks.contains(c.getKey()) && !c.getValue().isNullable()); + final boolean sameColumns = actualColumns.equals(intendedColumns) - && "TEXT".equals(existingTable.columns().get(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID)) - && "TIMESTAMP_TZ".equals(existingTable.columns().get(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT)) - && "VARIANT".equals(existingTable.columns().get(JavaBaseConstants.COLUMN_NAME_AB_META)); + && !hasPksWithNonNullConstraint + && "TEXT".equals(existingTable.columns().get(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID.toUpperCase()).type()) + && "TIMESTAMP_TZ".equals(existingTable.columns().get(JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT.toUpperCase()).type()) + && "VARIANT".equals(existingTable.columns().get(JavaBaseConstants.COLUMN_NAME_AB_META.toUpperCase()).type()); return sameColumns; } @Override - public String updateTable(final StreamConfig stream, final String finalSuffix) { - return updateTable(stream, finalSuffix, true); - } - - private String updateTable(final StreamConfig stream, final String finalSuffix, final boolean verifyPrimaryKeys) { - String validatePrimaryKeys = ""; - if (verifyPrimaryKeys && stream.destinationSyncMode() == DestinationSyncMode.APPEND_DEDUP) { - validatePrimaryKeys = validatePrimaryKeys(stream.id(), stream.primaryKey(), stream.columns()); - } - final String insertNewRecords = insertNewRecords(stream, finalSuffix, stream.columns()); + public Sql updateTable(final StreamConfig stream, + final String finalSuffix, + final Optional minRawTimestamp, + final boolean useExpensiveSaferCasting) { + final String insertNewRecords = insertNewRecords(stream, finalSuffix, stream.columns(), minRawTimestamp, useExpensiveSaferCasting); String dedupFinalTable = ""; String cdcDeletes = ""; - String dedupRawTable = ""; if (stream.destinationSyncMode() == DestinationSyncMode.APPEND_DEDUP) { - dedupRawTable = dedupRawTable(stream.id(), finalSuffix); - // If we're in dedup mode, then we must have a cursor dedupFinalTable = dedupFinalTable(stream.id(), finalSuffix, stream.primaryKey(), stream.cursor()); - cdcDeletes = cdcDeletes(stream, finalSuffix, stream.columns()); + cdcDeletes = cdcDeletes(stream, finalSuffix); } - final String commitRawTable = commitRawTable(stream.id()); + final String commitRawTable = commitRawTable(stream.id(), minRawTimestamp); - return new StringSubstitutor(Map.of( - "validate_primary_keys", validatePrimaryKeys, - "insert_new_records", insertNewRecords, - "dedup_final_table", dedupFinalTable, - "cdc_deletes", cdcDeletes, - "dedupe_raw_table", dedupRawTable, - "commit_raw_table", commitRawTable)).replace( - """ - BEGIN TRANSACTION; - ${validate_primary_keys} - ${insert_new_records} - ${dedup_final_table} - ${dedupe_raw_table} - ${cdc_deletes} - ${commit_raw_table} - COMMIT; - """); + return transactionally(insertNewRecords, dedupFinalTable, cdcDeletes, commitRawTable); } - private String extractAndCast(final ColumnId column, final AirbyteType airbyteType) { + private String extractAndCast(final ColumnId column, final AirbyteType airbyteType, final boolean useTryCast) { + return cast("\"_airbyte_data\":\"" + escapeJsonIdentifier(column.originalName()) + "\"", airbyteType, useTryCast); + } + + private String cast(final String sqlExpression, final AirbyteType airbyteType, final boolean useTryCast) { + final String castMethod = useTryCast ? "TRY_CAST" : "CAST"; if (airbyteType instanceof final Union u) { // This is guaranteed to not be a Union, so we won't recurse infinitely final AirbyteType chosenType = u.chooseType(); - return extractAndCast(column, chosenType); + return cast(sqlExpression, chosenType, useTryCast); } else if (airbyteType == AirbyteProtocolType.TIME_WITH_TIMEZONE) { // We're using TEXT for this type, so need to explicitly check the string format. // There's a bunch of ways we could do this; this regex is approximately correct and easy to @@ -189,18 +200,18 @@ private String extractAndCast(final ColumnId column, final AirbyteType airbyteTy // 12:34:56.7+08:00 // 12:34:56.7890123-0800 // 12:34:56-08 - return new StringSubstitutor(Map.of("column_name", escapeIdentifier(column.originalName()))).replace( + return new StringSubstitutor(Map.of("expression", sqlExpression)).replace( """ CASE - WHEN NOT ("_airbyte_data":"${column_name}"::TEXT REGEXP '\\\\d{1,2}:\\\\d{2}:\\\\d{2}(\\\\.\\\\d+)?(Z|[+\\\\-]\\\\d{1,2}(:?\\\\d{2})?)') + WHEN NOT ((${expression})::TEXT REGEXP '\\\\d{1,2}:\\\\d{2}:\\\\d{2}(\\\\.\\\\d+)?(Z|[+\\\\-]\\\\d{1,2}(:?\\\\d{2})?)') THEN NULL - ELSE "_airbyte_data":"${column_name}" + ELSE ${expression} END """); } else { final String dialectType = toDialectType(airbyteType); return switch (dialectType) { - case "TIMESTAMP_TZ" -> new StringSubstitutor(Map.of("column_name", escapeIdentifier(column.originalName()))).replace( + case "TIMESTAMP_TZ" -> new StringSubstitutor(Map.of("expression", sqlExpression, "cast", castMethod)).replace( // Handle offsets in +/-HHMM and +/-HH formats // The four cases, in order, match: // 2023-01-01T12:34:56-0800 @@ -210,151 +221,177 @@ WHEN NOT ("_airbyte_data":"${column_name}"::TEXT REGEXP '\\\\d{1,2}:\\\\d{2}:\\\ // And the ELSE will try to handle everything else. """ CASE - WHEN "_airbyte_data":"${column_name}"::TEXT REGEXP '\\\\d{4}-\\\\d{2}-\\\\d{2}T(\\\\d{2}:){2}\\\\d{2}(\\\\+|-)\\\\d{4}' - THEN TO_TIMESTAMP_TZ("_airbyte_data":"${column_name}"::TEXT, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') - WHEN "_airbyte_data":"${column_name}"::TEXT REGEXP '\\\\d{4}-\\\\d{2}-\\\\d{2}T(\\\\d{2}:){2}\\\\d{2}(\\\\+|-)\\\\d{2}' - THEN TO_TIMESTAMP_TZ("_airbyte_data":"${column_name}"::TEXT, 'YYYY-MM-DDTHH24:MI:SSTZH') - WHEN "_airbyte_data":"${column_name}"::TEXT REGEXP '\\\\d{4}-\\\\d{2}-\\\\d{2}T(\\\\d{2}:){2}\\\\d{2}\\\\.\\\\d{1,7}(\\\\+|-)\\\\d{4}' - THEN TO_TIMESTAMP_TZ("_airbyte_data":"${column_name}"::TEXT, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') - WHEN "_airbyte_data":"${column_name}"::TEXT REGEXP '\\\\d{4}-\\\\d{2}-\\\\d{2}T(\\\\d{2}:){2}\\\\d{2}\\\\.\\\\d{1,7}(\\\\+|-)\\\\d{2}' - THEN TO_TIMESTAMP_TZ("_airbyte_data":"${column_name}"::TEXT, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') - ELSE TRY_CAST("_airbyte_data":"${column_name}"::TEXT AS TIMESTAMP_TZ) + WHEN (${expression})::TEXT REGEXP '\\\\d{4}-\\\\d{2}-\\\\d{2}T(\\\\d{2}:){2}\\\\d{2}(\\\\+|-)\\\\d{4}' + THEN TO_TIMESTAMP_TZ((${expression})::TEXT, 'YYYY-MM-DDTHH24:MI:SSTZHTZM') + WHEN (${expression})::TEXT REGEXP '\\\\d{4}-\\\\d{2}-\\\\d{2}T(\\\\d{2}:){2}\\\\d{2}(\\\\+|-)\\\\d{2}' + THEN TO_TIMESTAMP_TZ((${expression})::TEXT, 'YYYY-MM-DDTHH24:MI:SSTZH') + WHEN (${expression})::TEXT REGEXP '\\\\d{4}-\\\\d{2}-\\\\d{2}T(\\\\d{2}:){2}\\\\d{2}\\\\.\\\\d{1,7}(\\\\+|-)\\\\d{4}' + THEN TO_TIMESTAMP_TZ((${expression})::TEXT, 'YYYY-MM-DDTHH24:MI:SS.FFTZHTZM') + WHEN (${expression})::TEXT REGEXP '\\\\d{4}-\\\\d{2}-\\\\d{2}T(\\\\d{2}:){2}\\\\d{2}\\\\.\\\\d{1,7}(\\\\+|-)\\\\d{2}' + THEN TO_TIMESTAMP_TZ((${expression})::TEXT, 'YYYY-MM-DDTHH24:MI:SS.FFTZH') + ELSE ${cast}((${expression})::TEXT AS TIMESTAMP_TZ) END """); // try_cast doesn't support variant/array/object, so handle them specially - case "VARIANT" -> "\"_airbyte_data\":\"" + escapeIdentifier(column.originalName()) + "\""; + case "VARIANT" -> sqlExpression; // We need to validate that the struct is actually a struct. // Note that struct columns are actually nullable in two ways. For a column `foo`: // {foo: null} and {} are both valid, and are both written to the final table as a SQL NULL (_not_ a // JSON null). - case "OBJECT" -> new StringSubstitutor(Map.of("column_name", escapeIdentifier(column.originalName()))).replace( + case "OBJECT" -> new StringSubstitutor(Map.of("expression", sqlExpression)).replace( """ CASE - WHEN TYPEOF("_airbyte_data":"${column_name}") != 'OBJECT' + WHEN TYPEOF(${expression}) != 'OBJECT' THEN NULL - ELSE "_airbyte_data":"${column_name}" + ELSE ${expression} END """); // Much like the object case, arrays need special handling. - case "ARRAY" -> new StringSubstitutor(Map.of("column_name", escapeIdentifier(column.originalName()))).replace( + case "ARRAY" -> new StringSubstitutor(Map.of("expression", sqlExpression)).replace( """ CASE - WHEN TYPEOF("_airbyte_data":"${column_name}") != 'ARRAY' + WHEN TYPEOF(${expression}) != 'ARRAY' THEN NULL - ELSE "_airbyte_data":"${column_name}" + ELSE ${expression} END """); - default -> "TRY_CAST(\"_airbyte_data\":\"" + escapeIdentifier(column.originalName()) + "\"::text as " + dialectType + ")"; + case "TEXT" -> "((" + sqlExpression + ")::text)"; // we don't need TRY_CAST on strings. + default -> castMethod + "((" + sqlExpression + ")::text as " + dialectType + ")"; }; } } @VisibleForTesting - String validatePrimaryKeys(final StreamId id, - final List primaryKeys, - final LinkedHashMap streamColumns) { - if (primaryKeys.stream().anyMatch(c -> c.originalName().contains("`"))) { - // TODO why is snowflake throwing a bizarre error when we try to use a column with a backtick in it? - // E.g. even this trivial procedure fails: (it should return the string `'foo`bar') - // execute immediate 'BEGIN RETURN \'foo`bar\'; END;' - return ""; - } - - final String pkNullChecks = primaryKeys.stream().map( - pk -> { - final String jsonExtract = extractAndCast(pk, streamColumns.get(pk)); - return "AND " + jsonExtract + " IS NULL"; - }).collect(joining("\n")); + String insertNewRecords(final StreamConfig stream, + final String finalSuffix, + final LinkedHashMap streamColumns, + final Optional minRawTimestamp, + final boolean useTryCast) { + final String columnList = streamColumns.keySet().stream().map(quotedColumnId -> quotedColumnId.name(QUOTE) + ",").collect(joining("\n")); + final String extractNewRawRecords = extractNewRawRecords(stream, minRawTimestamp, useTryCast); - final String script = new StringSubstitutor(Map.of( - "raw_table_id", id.rawTableId(QUOTE), - "pk_null_checks", pkNullChecks)).replace( - // Wrap this inside a script block so that we can use the scripting language + return new StringSubstitutor(Map.of( + "final_table_id", stream.id().finalTableId(QUOTE, finalSuffix.toUpperCase()), + "column_list", columnList, + "extractNewRawRecords", extractNewRawRecords)).replace( """ - BEGIN - LET missing_pk_count INTEGER := ( - SELECT COUNT(1) - FROM ${raw_table_id} - WHERE - "_airbyte_loaded_at" IS NULL - ${pk_null_checks} - ); - - IF (missing_pk_count > 0) THEN - RAISE STATEMENT_ERROR; - END IF; - RETURN 'SUCCESS'; - END; - """); - return "EXECUTE IMMEDIATE '" + escapeSingleQuotedString(script) + "';"; + INSERT INTO ${final_table_id} + ( + ${column_list} + "_AIRBYTE_META", + "_AIRBYTE_RAW_ID", + "_AIRBYTE_EXTRACTED_AT" + ) + ${extractNewRawRecords};"""); } - @VisibleForTesting - String insertNewRecords(final StreamConfig stream, final String finalSuffix, final LinkedHashMap streamColumns) { - final String columnCasts = streamColumns.entrySet().stream().map( - col -> extractAndCast(col.getKey(), col.getValue()) + " as " + col.getKey().name(QUOTE) + ",") + private String extractNewRawRecords(final StreamConfig stream, final Optional minRawTimestamp, final boolean useTryCast) { + final String columnCasts = stream.columns().entrySet().stream().map( + col -> extractAndCast(col.getKey(), col.getValue(), useTryCast) + " as " + col.getKey().name(QUOTE) + ",") .collect(joining("\n")); - final String columnErrors = streamColumns.entrySet().stream().map( + final String columnErrors = stream.columns().entrySet().stream().map( col -> new StringSubstitutor(Map.of( - "raw_col_name", escapeIdentifier(col.getKey().originalName()), + "raw_col_name", escapeJsonIdentifier(col.getKey().originalName()), "printable_col_name", escapeSingleQuotedString(col.getKey().originalName()), "col_type", toDialectType(col.getValue()), - "json_extract", extractAndCast(col.getKey(), col.getValue()))).replace( + "json_extract", extractAndCast(col.getKey(), col.getValue(), useTryCast))).replace( // TYPEOF returns "NULL_VALUE" for a JSON null and "NULL" for a SQL null """ CASE WHEN (TYPEOF("_airbyte_data":"${raw_col_name}") NOT IN ('NULL', 'NULL_VALUE')) AND (${json_extract} IS NULL) - THEN ['Problem with `${printable_col_name}`'] - ELSE [] + THEN 'Problem with `${printable_col_name}`' + ELSE NULL END""")) - .reduce( - "ARRAY_CONSTRUCT()", - (acc, col) -> "ARRAY_CAT(" + acc + ", " + col + ")"); - final String columnList = streamColumns.keySet().stream().map(quotedColumnId -> quotedColumnId.name(QUOTE) + ",").collect(joining("\n")); + .collect(joining(",\n")); + final String columnList = stream.columns().keySet().stream().map(quotedColumnId -> quotedColumnId.name(QUOTE) + ",").collect(joining("\n")); + final String extractedAtCondition = buildExtractedAtCondition(minRawTimestamp); - String cdcConditionalOrIncludeStatement = ""; - if (stream.destinationSyncMode() == DestinationSyncMode.APPEND_DEDUP && streamColumns.containsKey(CDC_DELETED_AT_COLUMN)) { - cdcConditionalOrIncludeStatement = """ - OR ( - "_airbyte_loaded_at" IS NOT NULL - AND TYPEOF("_airbyte_data":"_ab_cdc_deleted_at") NOT IN ('NULL', 'NULL_VALUE') - ) - """; + if (stream.destinationSyncMode() == DestinationSyncMode.APPEND_DEDUP) { + String cdcConditionalOrIncludeStatement = ""; + if (stream.columns().containsKey(CDC_DELETED_AT_COLUMN)) { + cdcConditionalOrIncludeStatement = """ + OR ( + "_airbyte_loaded_at" IS NOT NULL + AND TYPEOF("_airbyte_data":"_ab_cdc_deleted_at") NOT IN ('NULL', 'NULL_VALUE') + ) + """; + } + + final String pkList = stream.primaryKey().stream().map(columnId -> columnId.name(QUOTE)).collect(joining(",")); + final String cursorOrderClause = stream.cursor() + .map(cursorId -> cursorId.name(QUOTE) + " DESC NULLS LAST,") + .orElse(""); + + return new StringSubstitutor(Map.of( + "raw_table_id", stream.id().rawTableId(QUOTE), + "column_casts", columnCasts, + "column_errors", columnErrors, + "cdcConditionalOrIncludeStatement", cdcConditionalOrIncludeStatement, + "extractedAtCondition", extractedAtCondition, + "column_list", columnList, + "pk_list", pkList, + "cursor_order_clause", cursorOrderClause)).replace( + """ + WITH intermediate_data AS ( + SELECT + ${column_casts} + ARRAY_CONSTRUCT_COMPACT(${column_errors}) as "_airbyte_cast_errors", + "_airbyte_raw_id", + "_airbyte_extracted_at" + FROM ${raw_table_id} + WHERE ( + "_airbyte_loaded_at" IS NULL + ${cdcConditionalOrIncludeStatement} + ) ${extractedAtCondition} + ), new_records AS ( + SELECT + ${column_list} + OBJECT_CONSTRUCT('errors', "_airbyte_cast_errors") AS "_AIRBYTE_META", + "_airbyte_raw_id" AS "_AIRBYTE_RAW_ID", + "_airbyte_extracted_at" AS "_AIRBYTE_EXTRACTED_AT" + FROM intermediate_data + ), numbered_rows AS ( + SELECT *, row_number() OVER ( + PARTITION BY ${pk_list} ORDER BY ${cursor_order_clause} "_AIRBYTE_EXTRACTED_AT" DESC + ) AS row_number + FROM new_records + ) + SELECT ${column_list} "_AIRBYTE_META", "_AIRBYTE_RAW_ID", "_AIRBYTE_EXTRACTED_AT" + FROM numbered_rows + WHERE row_number = 1"""); + } else { + return new StringSubstitutor(Map.of( + "raw_table_id", stream.id().rawTableId(QUOTE), + "column_casts", columnCasts, + "column_errors", columnErrors, + "extractedAtCondition", extractedAtCondition, + "column_list", columnList)).replace( + """ + WITH intermediate_data AS ( + SELECT + ${column_casts} + ARRAY_CONSTRUCT_COMPACT(${column_errors}) as "_airbyte_cast_errors", + "_airbyte_raw_id", + "_airbyte_extracted_at" + FROM ${raw_table_id} + WHERE + "_airbyte_loaded_at" IS NULL + ${extractedAtCondition} + ) + SELECT + ${column_list} + OBJECT_CONSTRUCT('errors', "_airbyte_cast_errors") AS "_AIRBYTE_META", + "_airbyte_raw_id" AS "_AIRBYTE_RAW_ID", + "_airbyte_extracted_at" AS "_AIRBYTE_EXTRACTED_AT" + FROM intermediate_data"""); } + } - return new StringSubstitutor(Map.of( - "raw_table_id", stream.id().rawTableId(QUOTE), - "final_table_id", stream.id().finalTableId(QUOTE, finalSuffix), - "column_casts", columnCasts, - "column_errors", columnErrors, - "cdcConditionalOrIncludeStatement", cdcConditionalOrIncludeStatement, - "column_list", columnList)).replace( - """ - INSERT INTO ${final_table_id} - ( - ${column_list} - "_airbyte_meta", - "_airbyte_raw_id", - "_airbyte_extracted_at" - ) - WITH intermediate_data AS ( - SELECT - ${column_casts} - ${column_errors} as "_airbyte_cast_errors", - "_airbyte_raw_id", - "_airbyte_extracted_at" - FROM ${raw_table_id} - WHERE - "_airbyte_loaded_at" IS NULL - ${cdcConditionalOrIncludeStatement} - ) - SELECT - ${column_list} - OBJECT_CONSTRUCT('errors', "_airbyte_cast_errors") AS "_airbyte_meta", - "_airbyte_raw_id", - "_airbyte_extracted_at" - FROM intermediate_data;"""); + private static String buildExtractedAtCondition(final Optional minRawTimestamp) { + return minRawTimestamp + .map(ts -> " AND \"_airbyte_extracted_at\" > '" + ts + "'") + .orElse(""); } @VisibleForTesting @@ -368,15 +405,15 @@ String dedupFinalTable(final StreamId id, .orElse(""); return new StringSubstitutor(Map.of( - "final_table_id", id.finalTableId(QUOTE, finalSuffix), + "final_table_id", id.finalTableId(QUOTE, finalSuffix.toUpperCase()), "pk_list", pkList, "cursor_order_clause", cursorOrderClause)).replace( """ DELETE FROM ${final_table_id} - WHERE "_airbyte_raw_id" IN ( - SELECT "_airbyte_raw_id" FROM ( - SELECT "_airbyte_raw_id", row_number() OVER ( - PARTITION BY ${pk_list} ORDER BY ${cursor_order_clause} "_airbyte_extracted_at" DESC + WHERE "_AIRBYTE_RAW_ID" IN ( + SELECT "_AIRBYTE_RAW_ID" FROM ( + SELECT "_AIRBYTE_RAW_ID", row_number() OVER ( + PARTITION BY ${pk_list} ORDER BY ${cursor_order_clause} "_AIRBYTE_EXTRACTED_AT" DESC ) as row_number FROM ${final_table_id} ) WHERE row_number != 1 @@ -384,104 +421,67 @@ String dedupFinalTable(final StreamId id, """); } - @VisibleForTesting - String cdcDeletes(final StreamConfig stream, - final String finalSuffix, - final LinkedHashMap streamColumns) { - + private String cdcDeletes(final StreamConfig stream, final String finalSuffix) { if (stream.destinationSyncMode() != DestinationSyncMode.APPEND_DEDUP) { return ""; } - - if (!streamColumns.containsKey(CDC_DELETED_AT_COLUMN)) { + if (!stream.columns().containsKey(CDC_DELETED_AT_COLUMN)) { return ""; } - final String pkList = stream.primaryKey().stream().map(columnId -> columnId.name(QUOTE)).collect(joining(",")); - final String pkCasts = stream.primaryKey().stream().map(pk -> extractAndCast(pk, streamColumns.get(pk))).collect(joining(",\n")); - // we want to grab IDs for deletion from the raw table (not the final table itself) to hand // out-of-order record insertions after the delete has been registered return new StringSubstitutor(Map.of( - "final_table_id", stream.id().finalTableId(QUOTE, finalSuffix), - "raw_table_id", stream.id().rawTableId(QUOTE), - "pk_list", pkList, - "pk_extracts", pkCasts, - "quoted_cdc_delete_column", QUOTE + "_ab_cdc_deleted_at" + QUOTE)).replace( + "final_table_id", stream.id().finalTableId(QUOTE, finalSuffix.toUpperCase()))).replace( """ DELETE FROM ${final_table_id} - WHERE ARRAY_CONSTRUCT(${pk_list}) IN ( - SELECT ARRAY_CONSTRUCT( - ${pk_extracts} - ) - FROM ${raw_table_id} - WHERE "_airbyte_data":"_ab_cdc_deleted_at" != 'null' - ); + WHERE _AB_CDC_DELETED_AT IS NOT NULL; """); } @VisibleForTesting - String dedupRawTable(final StreamId id, final String finalSuffix) { + String commitRawTable(final StreamId id, final Optional minRawTimestamp) { return new StringSubstitutor(Map.of( "raw_table_id", id.rawTableId(QUOTE), - "final_table_id", id.finalTableId(QUOTE, finalSuffix))).replace( - // Note that this leaves _all_ deletion records in the raw table. We _could_ clear them out, but it - // would be painful, - // and it only matters in a few edge cases. - """ - DELETE FROM ${raw_table_id} - WHERE "_airbyte_raw_id" NOT IN ( - SELECT "_airbyte_raw_id" FROM ${final_table_id} - ); - """); - } - - @VisibleForTesting - String commitRawTable(final StreamId id) { - return new StringSubstitutor(Map.of( - "raw_table_id", id.rawTableId(QUOTE))).replace( + "extractedAtCondition", buildExtractedAtCondition(minRawTimestamp))).replace( """ UPDATE ${raw_table_id} SET "_airbyte_loaded_at" = CURRENT_TIMESTAMP() WHERE "_airbyte_loaded_at" IS NULL + ${extractedAtCondition} ;"""); } @Override - public String overwriteFinalTable(final StreamId stream, final String finalSuffix) { - return new StringSubstitutor(Map.of( + public Sql overwriteFinalTable(final StreamId stream, final String finalSuffix) { + final StringSubstitutor substitutor = new StringSubstitutor(Map.of( "final_table", stream.finalTableId(QUOTE), - "tmp_final_table", stream.finalTableId(QUOTE, finalSuffix))).replace( - """ - BEGIN TRANSACTION; - DROP TABLE IF EXISTS ${final_table}; - ALTER TABLE ${tmp_final_table} RENAME TO ${final_table}; - COMMIT; - """); + "tmp_final_table", stream.finalTableId(QUOTE, finalSuffix.toUpperCase()))); + return transactionally( + substitutor.replace("DROP TABLE IF EXISTS ${final_table};"), + substitutor.replace("ALTER TABLE ${tmp_final_table} RENAME TO ${final_table};")); } @Override - public String softReset(final StreamConfig stream) { - final String createTempTable = createTable(stream, SOFT_RESET_SUFFIX, true); - final String clearLoadedAt = clearLoadedAt(stream.id()); - final String rebuildInTempTable = updateTable(stream, SOFT_RESET_SUFFIX, false); - final String overwriteFinalTable = overwriteFinalTable(stream.id(), SOFT_RESET_SUFFIX); - return String.join("\n", createTempTable, clearLoadedAt, rebuildInTempTable, overwriteFinalTable); + public Sql prepareTablesForSoftReset(final StreamConfig stream) { + return concat( + createTable(stream, SOFT_RESET_SUFFIX.toUpperCase(), true), + clearLoadedAt(stream.id())); } - private String clearLoadedAt(final StreamId streamId) { - return new StringSubstitutor(Map.of("raw_table_id", streamId.rawTableId(QUOTE))) + @Override + public Sql clearLoadedAt(final StreamId streamId) { + return Sql.of(new StringSubstitutor(Map.of("raw_table_id", streamId.rawTableId(QUOTE))) .replace(""" UPDATE ${raw_table_id} SET "_airbyte_loaded_at" = NULL; - """); + """)); } @Override - public String migrateFromV1toV2(final StreamId streamId, final String namespace, final String tableName) { + public Sql migrateFromV1toV2(final StreamId streamId, final String namespace, final String tableName) { // In the SQL below, the v2 values are quoted to preserve their case while the v1 values are // intentionally _not_ quoted. This is to preserve the implicit upper-casing behavior in v1. - return new StringSubstitutor(Map.of( - "raw_namespace", StringUtils.wrap(streamId.rawNamespace(), QUOTE), + return Sql.of(new StringSubstitutor(Map.of( "raw_table_name", streamId.rawTableId(QUOTE), "raw_id", JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, "extracted_at", JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT, @@ -492,8 +492,6 @@ public String migrateFromV1toV2(final StreamId streamId, final String namespace, "v1_raw_table", String.join(".", namespace, tableName))) .replace( """ - CREATE SCHEMA IF NOT EXISTS ${raw_namespace}; - CREATE OR REPLACE TABLE ${raw_table_name} ( "${raw_id}" VARCHAR PRIMARY KEY, "${extracted_at}" TIMESTAMP WITH TIME ZONE DEFAULT current_timestamp(), @@ -510,23 +508,52 @@ public String migrateFromV1toV2(final StreamId streamId, final String namespace, FROM ${v1_raw_table} ) ; - """); + """)); } - public static String escapeIdentifier(final String identifier) { + /** + * Snowflake json object access is done using double-quoted strings, e.g. `SELECT + * "_airbyte_data":"foo"`. As such, we need to escape double-quotes in the field name. + */ + public static String escapeJsonIdentifier(final String identifier) { // Note that we don't need to escape backslashes here! // The only special character in an identifier is the double-quote, which needs to be doubled. return identifier.replace("\"", "\"\""); } + /** + * SQL identifiers are also double-quoted strings. They have slightly more stringent requirements + * than JSON field identifiers. + *

      + * This method is separate from {@link #escapeJsonIdentifier(String)} because we need to retain the + * original field name for JSON access, e.g. `SELECT "_airbyte_data":"${FOO" AS "__FOO"`. + */ + public static String escapeSqlIdentifier(String identifier) { + // Snowflake scripting language does something weird when the `${` bigram shows up in the script + // so replace these with something else. + // For completeness, if we trigger this, also replace closing curly braces with underscores. + if (identifier.contains("${")) { + identifier = identifier + .replace("$", "_") + .replace("{", "_") + .replace("}", "_"); + } + + return escapeJsonIdentifier(identifier); + } + + private static String prefixReservedColumnName(final String columnName) { + return RESERVED_COLUMN_NAMES.stream().anyMatch(k -> k.equalsIgnoreCase(columnName)) ? "_" + columnName : columnName; + } + public static String escapeSingleQuotedString(final String str) { return str .replace("\\", "\\\\") .replace("'", "\\'"); } - public static String escapeDollarString(final String str) { - return str.replace("$$", "\\$\\$"); + private static Set getPks(final StreamConfig stream) { + return stream.primaryKey() != null ? stream.primaryKey().stream().map(ColumnId::name).collect(Collectors.toSet()) : Collections.emptySet(); } } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeTableDefinition.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeTableDefinition.java index 8524222fcd35..2535d9004b13 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeTableDefinition.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeTableDefinition.java @@ -11,5 +11,4 @@ * {@link net.snowflake.client.jdbc.SnowflakeType} doesn't actually have all the types that * Snowflake supports. */ -// TODO fields for columns + indexes... or other stuff we want to set? -public record SnowflakeTableDefinition(LinkedHashMap columns) {} +public record SnowflakeTableDefinition(LinkedHashMap columns) {} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeV1V2Migrator.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeV1V2Migrator.java index cafb4ddb8e63..aa6eba7f7f96 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeV1V2Migrator.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeV1V2Migrator.java @@ -4,12 +4,12 @@ package io.airbyte.integrations.destination.snowflake.typing_deduping; -import io.airbyte.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; import io.airbyte.integrations.base.destination.typing_deduping.BaseDestinationV1V2Migrator; import io.airbyte.integrations.base.destination.typing_deduping.CollectionUtils; import io.airbyte.integrations.base.destination.typing_deduping.NamespacedTableName; import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; -import io.airbyte.integrations.destination.NamingConventionTransformer; import java.util.Collection; import java.util.LinkedHashMap; import java.util.Optional; @@ -33,7 +33,7 @@ public SnowflakeV1V2Migrator(final NamingConventionTransformer namingConventionT @SneakyThrows @Override - protected boolean doesAirbyteInternalNamespaceExist(final StreamConfig streamConfig) { + protected boolean doesAirbyteInternalNamespaceExist(final StreamConfig streamConfig) throws Exception { return !database .queryJsons( """ @@ -54,15 +54,15 @@ protected boolean schemaMatchesExpectation(final SnowflakeTableDefinition existi @SneakyThrows @Override - protected Optional getTableIfExists(final String namespace, final String tableName) { + protected Optional getTableIfExists(final String namespace, final String tableName) throws Exception { // TODO this is mostly copied from SnowflakeDestinationHandler#findExistingTable, we should probably // reuse this logic // The obvious database.getMetaData().getColumns() solution doesn't work, because JDBC translates // VARIANT as VARCHAR - LinkedHashMap columns = + final LinkedHashMap columns = database.queryJsons( """ - SELECT column_name, data_type + SELECT column_name, data_type, is_nullable FROM information_schema.columns WHERE table_catalog = ? AND table_schema = ? @@ -74,7 +74,8 @@ protected Optional getTableIfExists(final String names tableName) .stream() .collect(LinkedHashMap::new, - (map, row) -> map.put(row.get("COLUMN_NAME").asText(), row.get("DATA_TYPE").asText()), + (map, row) -> map.put(row.get("COLUMN_NAME").asText(), + new SnowflakeColumnDefinition(row.get("DATA_TYPE").asText(), fromSnowflakeBoolean(row.get("IS_NULLABLE").asText()))), LinkedHashMap::putAll); if (columns.isEmpty()) { return Optional.empty(); @@ -86,16 +87,26 @@ protected Optional getTableIfExists(final String names @Override protected NamespacedTableName convertToV1RawName(final StreamConfig streamConfig) { // The implicit upper-casing happens for this in the SqlGenerator + @SuppressWarnings("deprecation") + String tableName = this.namingConventionTransformer.getRawTableName(streamConfig.id().originalName()); return new NamespacedTableName( this.namingConventionTransformer.getIdentifier(streamConfig.id().originalNamespace()), - this.namingConventionTransformer.getRawTableName(streamConfig.id().originalName())); + tableName); } @Override - protected boolean doesValidV1RawTableExist(final String namespace, final String tableName) { + protected boolean doesValidV1RawTableExist(final String namespace, final String tableName) throws Exception { // Previously we were not quoting table names and they were being implicitly upper-cased. // In v2 we preserve cases return super.doesValidV1RawTableExist(namespace.toUpperCase(), tableName.toUpperCase()); } + /** + * In snowflake information_schema tables, booleans return "YES" and "NO", which DataBind doesn't + * know how to use + */ + private boolean fromSnowflakeBoolean(final String input) { + return input.equalsIgnoreCase("yes"); + } + } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeV2TableMigrator.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeV2TableMigrator.java new file mode 100644 index 000000000000..9e04ec3b6f22 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeV2TableMigrator.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.snowflake.typing_deduping; + +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE; +import static io.airbyte.integrations.destination.snowflake.SnowflakeInternalStagingDestination.RAW_SCHEMA_OVERRIDE; + +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.TypingAndDedupingFlag; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import io.airbyte.integrations.base.destination.typing_deduping.TypeAndDedupeTransaction; +import io.airbyte.integrations.base.destination.typing_deduping.V2TableMigrator; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import java.sql.SQLException; +import java.util.LinkedHashMap; +import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SnowflakeV2TableMigrator implements V2TableMigrator { + + private static final Logger LOGGER = LoggerFactory.getLogger(SnowflakeV2TableMigrator.class); + + private final JdbcDatabase database; + private final String rawNamespace; + private final String databaseName; + private final SnowflakeSqlGenerator generator; + private final SnowflakeDestinationHandler handler; + + public SnowflakeV2TableMigrator(final JdbcDatabase database, + final String databaseName, + final SnowflakeSqlGenerator generator, + final SnowflakeDestinationHandler handler) { + this.database = database; + this.databaseName = databaseName; + this.generator = generator; + this.handler = handler; + this.rawNamespace = TypingAndDedupingFlag.getRawNamespaceOverride(RAW_SCHEMA_OVERRIDE).orElse(DEFAULT_AIRBYTE_INTERNAL_NAMESPACE); + } + + @Override + public void migrateIfNecessary(final StreamConfig streamConfig) throws Exception { + final StreamId caseSensitiveStreamId = buildStreamId_caseSensitive( + streamConfig.id().originalNamespace(), + streamConfig.id().originalName(), + rawNamespace); + final boolean syncModeRequiresMigration = streamConfig.destinationSyncMode() != DestinationSyncMode.OVERWRITE; + final boolean existingTableCaseSensitiveExists = findExistingTable_caseSensitive(caseSensitiveStreamId).isPresent(); + final boolean existingTableUppercaseDoesNotExist = !handler.findExistingTable(streamConfig.id()).isPresent(); + LOGGER.info( + "Checking whether upcasing migration is necessary for {}.{}. Sync mode requires migration: {}; existing case-sensitive table exists: {}; existing uppercased table does not exist: {}", + streamConfig.id().originalNamespace(), + streamConfig.id().originalName(), + syncModeRequiresMigration, + existingTableCaseSensitiveExists, + existingTableUppercaseDoesNotExist); + if (syncModeRequiresMigration && existingTableCaseSensitiveExists && existingTableUppercaseDoesNotExist) { + LOGGER.info( + "Executing upcasing migration for {}.{}", + streamConfig.id().originalNamespace(), + streamConfig.id().originalName()); + TypeAndDedupeTransaction.executeSoftReset(generator, handler, streamConfig); + } + } + + // These methods were copied from + // https://github.com/airbytehq/airbyte/blob/d5fdb1b982d464f54941bf9a830b9684fb47d249/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java + // which is the highest version of destination-snowflake that still uses quoted+case-sensitive + // identifiers + private static StreamId buildStreamId_caseSensitive(final String namespace, final String name, final String rawNamespaceOverride) { + // No escaping needed, as far as I can tell. We quote all our identifier names. + return new StreamId( + escapeIdentifier_caseSensitive(namespace), + escapeIdentifier_caseSensitive(name), + escapeIdentifier_caseSensitive(rawNamespaceOverride), + escapeIdentifier_caseSensitive(StreamId.concatenateRawTableName(namespace, name)), + namespace, + name); + } + + private static String escapeIdentifier_caseSensitive(final String identifier) { + // Note that we don't need to escape backslashes here! + // The only special character in an identifier is the double-quote, which needs to be doubled. + return identifier.replace("\"", "\"\""); + } + + // And this was taken from + // https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeDestinationHandler.java + public Optional findExistingTable_caseSensitive(final StreamId id) throws SQLException { + // The obvious database.getMetaData().getColumns() solution doesn't work, because JDBC translates + // VARIANT as VARCHAR + final LinkedHashMap columns = database.queryJsons( + """ + SELECT column_name, data_type, is_nullable + FROM information_schema.columns + WHERE table_catalog = ? + AND table_schema = ? + AND table_name = ? + ORDER BY ordinal_position; + """, + databaseName.toUpperCase(), + id.finalNamespace(), + id.finalName()).stream() + .collect(LinkedHashMap::new, + (map, row) -> map.put( + row.get("COLUMN_NAME").asText(), + new SnowflakeColumnDefinition(row.get("DATA_TYPE").asText(), fromSnowflakeBoolean(row.get("IS_NULLABLE").asText()))), + LinkedHashMap::putAll); + if (columns.isEmpty()) { + return Optional.empty(); + } else { + return Optional.of(new SnowflakeTableDefinition(columns)); + } + } + + /** + * In snowflake information_schema tables, booleans return "YES" and "NO", which DataBind doesn't + * know how to use + */ + private boolean fromSnowflakeBoolean(String input) { + return input.equalsIgnoreCase("yes"); + } + +} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json index f59b63a692cc..5679a3f3fd31 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/resources/spec.json @@ -161,17 +161,25 @@ "type": "string", "order": 7 }, - "use_1s1t_format": { - "type": "boolean", - "description": "(Beta) Use Destinations V2. Contact Airbyte Support to participate in the beta program.", - "title": "Use Destinations V2 (Early Access)", - "order": 10 - }, "raw_data_schema": { "type": "string", - "description": "(Beta) The schema to write raw tables into", - "title": "Destinations V2 Raw Table Schema (Early Access)", + "description": "The schema to write raw tables into (default: airbyte_internal)", + "title": "Raw Table Schema Name", + "order": 10 + }, + "disable_type_dedupe": { + "type": "boolean", + "default": false, + "description": "Disable Writing Final Tables. WARNING! The data format in _airbyte_data is likely stable but there are no guarantees that other metadata columns will remain the same in future versions", + "title": "Disable Final Tables. (WARNING! Unstable option; Columns in raw table schema might change between versions)", "order": 11 + }, + "enable_incremental_final_table_updates": { + "type": "boolean", + "default": false, + "description": "When enabled your data will load into your final tables incrementally while your data is still being synced. When Disabled (the default), your data loads into your final tables once at the end of a sync. Note that this option only applies if you elect to create Final tables", + "title": "Enable Loading Data Incrementally to Final Tables", + "order": 12 } } }, diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationIntegrationTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationIntegrationTest.java index 8d7db2702655..bee7456a8b67 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationIntegrationTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationIntegrationTest.java @@ -10,11 +10,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.DestinationConfig; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.DestinationConfig; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import java.io.IOException; import java.nio.file.Files; diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestinationAcceptanceTest.java index 5cfd62d01da6..0fb53312d3d8 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInsertDestinationAcceptanceTest.java @@ -10,20 +10,22 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.DestinationConfig; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.argproviders.DataArgumentsProvider; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.string.Strings; import io.airbyte.configoss.StandardCheckConnectionOutput; import io.airbyte.configoss.StandardCheckConnectionOutput.Status; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.DestinationConfig; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.argproviders.DataArgumentsProvider; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import io.airbyte.integrations.destination.snowflake.typing_deduping.SnowflakeSqlGenerator; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; @@ -45,6 +47,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ArgumentsSource; +@Disabled public class SnowflakeInsertDestinationAcceptanceTest extends DestinationAcceptanceTest { private static final NamingConventionTransformer NAME_TRANSFORMER = new SnowflakeSQLNameTransformer(); @@ -54,7 +57,7 @@ public class SnowflakeInsertDestinationAcceptanceTest extends DestinationAccepta protected static final String NO_USER_PRIVILEGES_ERR_MSG = "Encountered Error with Snowflake Configuration: Current role does not have permissions on the target schema please verify your privileges"; - protected static final String IP_NOT_IN_WHITE_LIST_ERR_MSG = "is not allowed to access Snowflake. Contact your account administrator."; + protected static final String IP_NOT_IN_WHITE_LIST_ERR_MSG = "is not allowed to access Snowflake. Contact your local security administrator"; // this config is based on the static config, and it contains a random // schema name that is different for each test run @@ -119,9 +122,10 @@ protected List retrieveRecords(final TestDestinationEnv env, final String namespace, final JsonNode streamSchema) throws Exception { - return retrieveRecordsFromTable(NAME_TRANSFORMER.getRawTableName(streamName), NAME_TRANSFORMER.getNamespace(namespace)) + final StreamId streamId = new SnowflakeSqlGenerator().buildStreamId(namespace, streamName, JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE); + return retrieveRecordsFromTable(streamId.rawName(), streamId.rawNamespace()) .stream() - .map(r -> r.get(JavaBaseConstants.COLUMN_NAME_DATA.toUpperCase())) + .map(r -> r.get(JavaBaseConstants.COLUMN_NAME_DATA)) .collect(Collectors.toList()); } @@ -155,14 +159,15 @@ private List retrieveRecordsFromTable(final String tableName, final St return database.bufferedResultSetQuery( connection -> { try (final ResultSet tableInfo = connection.createStatement() - .executeQuery(String.format("SHOW TABLES LIKE '%s' IN SCHEMA %s;", tableName, schema))) { + .executeQuery(String.format("SHOW TABLES LIKE '%s' IN SCHEMA \"%s\";", tableName, schema))) { assertTrue(tableInfo.next()); // check that we're creating permanent tables. DBT defaults to transient tables, which have // `TRANSIENT` as the value for the `kind` column. assertEquals("TABLE", tableInfo.getString("kind")); connection.createStatement().execute("ALTER SESSION SET TIMEZONE = 'UTC';"); return connection.createStatement() - .executeQuery(String.format("SELECT * FROM %s.%s ORDER BY %s ASC;", schema, tableName, JavaBaseConstants.COLUMN_NAME_EMITTED_AT)); + .executeQuery(String.format("SELECT * FROM \"%s\".\"%s\" ORDER BY \"%s\" ASC;", schema, tableName, + JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT)); } }, new SnowflakeTestSourceOperations()::rowToJson); diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestinationAcceptanceTest.java index b03c8b8111f1..e8b22fab6d85 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingDestinationAcceptanceTest.java @@ -8,12 +8,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.standardtest.destination.argproviders.DataArgumentsProvider; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.configoss.StandardCheckConnectionOutput; import io.airbyte.configoss.StandardCheckConnectionOutput.Status; -import io.airbyte.integrations.standardtest.destination.argproviders.DataArgumentsProvider; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; @@ -27,6 +27,7 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ArgumentsSource; +@Disabled public class SnowflakeInternalStagingDestinationAcceptanceTest extends SnowflakeInsertDestinationAcceptanceTest { public JsonNode getStaticConfig() { diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeTestDataComparator.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeTestDataComparator.java index af58c38d2fc8..cff20511e99e 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeTestDataComparator.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeTestDataComparator.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.snowflake; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.ZoneOffset; diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeTestSourceOperations.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeTestSourceOperations.java index 37cea8bc1400..c25bcb6709d7 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeTestSourceOperations.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeTestSourceOperations.java @@ -4,12 +4,12 @@ package io.airbyte.integrations.destination.snowflake; -import static io.airbyte.db.jdbc.DateTimeConverter.putJavaSQLDate; -import static io.airbyte.db.jdbc.DateTimeConverter.putJavaSQLTime; +import static io.airbyte.cdk.db.jdbc.DateTimeConverter.putJavaSQLDate; +import static io.airbyte.cdk.db.jdbc.DateTimeConverter.putJavaSQLTime; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.jdbc.JdbcSourceOperations; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcSourceOperations; import java.sql.ResultSet; import java.sql.SQLException; diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeTestUtils.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeTestUtils.java index a4f0aadd1cd5..89039af021ee 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeTestUtils.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/SnowflakeTestUtils.java @@ -7,8 +7,8 @@ import static java.util.stream.Collectors.joining; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.integrations.destination.snowflake.typing_deduping.SnowflakeSqlGenerator; import java.sql.SQLException; import java.util.List; @@ -25,7 +25,9 @@ public static List dumpRawTable(final JdbcDatabase database, final Str timestampToString(quote(JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT)), quote(JavaBaseConstants.COLUMN_NAME_DATA)), database, - tableIdentifier); + tableIdentifier, + // Raw tables still have lowercase names + false); } public static List dumpFinalTable(final JdbcDatabase database, final String databaseName, final String schema, final String table) @@ -41,9 +43,9 @@ public static List dumpFinalTable(final JdbcDatabase database, final S AND table_name = ? ORDER BY ordinal_position; """, - unescapeIdentifier(databaseName), - unescapeIdentifier(schema), - unescapeIdentifier(table)).stream() + unescapeIdentifier(databaseName).toUpperCase(), + unescapeIdentifier(schema).toUpperCase(), + unescapeIdentifier(table).toUpperCase()).stream() .map(column -> { final String quotedName = quote(column.get("COLUMN_NAME").asText()); final String type = column.get("DATA_TYPE").asText(); @@ -59,7 +61,7 @@ public static List dumpFinalTable(final JdbcDatabase database, final S }; }) .toList(); - return dumpTable(columns, database, quote(schema) + "." + quote(table)); + return dumpTable(columns, database, quote(schema) + "." + quote(table), true); } /** @@ -71,18 +73,23 @@ public static List dumpFinalTable(final JdbcDatabase database, final S * * @param tableIdentifier Table identifier (e.g. "schema.table"), with quotes if necessary. */ - public static List dumpTable(final List columns, final JdbcDatabase database, final String tableIdentifier) throws SQLException { + public static List dumpTable(final List columns, + final JdbcDatabase database, + final String tableIdentifier, + final boolean upcaseExtractedAt) + throws SQLException { return database.bufferedResultSetQuery(connection -> connection.createStatement().executeQuery(new StringSubstitutor(Map.of( "columns", columns.stream().collect(joining(",")), - "table", tableIdentifier)).replace( + "table", tableIdentifier, + "extracted_at", upcaseExtractedAt ? "_AIRBYTE_EXTRACTED_AT" : "\"_airbyte_extracted_at\"")).replace( """ - SELECT ${columns} FROM ${table} ORDER BY "_airbyte_extracted_at" ASC + SELECT ${columns} FROM ${table} ORDER BY ${extracted_at} ASC """)), new SnowflakeTestSourceOperations()::rowToJson); } private static String quote(final String name) { - return '"' + SnowflakeSqlGenerator.escapeIdentifier(name) + '"'; + return '"' + SnowflakeSqlGenerator.escapeJsonIdentifier(name) + '"'; } public static String timestampToString(final String quotedName) { diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/AbstractSnowflakeTypingDedupingTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/AbstractSnowflakeTypingDedupingTest.java index 8fb2205f62da..a7aac9cef7cc 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/AbstractSnowflakeTypingDedupingTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/AbstractSnowflakeTypingDedupingTest.java @@ -4,25 +4,44 @@ package io.airbyte.integrations.destination.snowflake.typing_deduping; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.base.destination.typing_deduping.BaseTypingDedupingTest; +import io.airbyte.integrations.base.destination.typing_deduping.SqlGenerator; import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import io.airbyte.integrations.destination.snowflake.OssCloudEnvVarConsts; import io.airbyte.integrations.destination.snowflake.SnowflakeDatabase; import io.airbyte.integrations.destination.snowflake.SnowflakeTestUtils; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.SyncMode; +import io.airbyte.workers.exception.TestHarnessException; import java.nio.file.Path; import java.util.List; +import java.util.Map; import javax.sql.DataSource; +import org.junit.jupiter.api.Test; public abstract class AbstractSnowflakeTypingDedupingTest extends BaseTypingDedupingTest { + public static final Map FINAL_METADATA_COLUMN_NAMES = Map.of( + "_airbyte_raw_id", "_AIRBYTE_RAW_ID", + "_airbyte_extracted_at", "_AIRBYTE_EXTRACTED_AT", + "_airbyte_loaded_at", "_AIRBYTE_LOADED_AT", + "_airbyte_data", "_AIRBYTE_DATA", + "_airbyte_meta", "_AIRBYTE_META"); private String databaseName; private JdbcDatabase database; private DataSource dataSource; @@ -62,7 +81,7 @@ protected List dumpFinalTableRecords(String streamNamespace, final Str if (streamNamespace == null) { streamNamespace = getDefaultSchema(); } - return SnowflakeTestUtils.dumpFinalTable(database, databaseName, streamNamespace, streamName); + return SnowflakeTestUtils.dumpFinalTable(database, databaseName, streamNamespace.toUpperCase(), streamName.toUpperCase()); } @Override @@ -77,8 +96,9 @@ protected void teardownStreamAndNamespace(String streamNamespace, final String s DROP SCHEMA IF EXISTS "%s" CASCADE """, getRawSchema(), + // Raw table is still lowercase. StreamId.concatenateRawTableName(streamNamespace, streamName), - streamNamespace)); + streamNamespace.toUpperCase())); } @Override @@ -86,6 +106,16 @@ protected void globalTeardown() throws Exception { DataSourceFactory.close(dataSource); } + @Override + protected SqlGenerator getSqlGenerator() { + return new SnowflakeSqlGenerator(); + } + + @Override + protected Map getFinalMetadataColumnNames() { + return FINAL_METADATA_COLUMN_NAMES; + } + /** * Subclasses using a config with a nonstandard raw table schema should override this method. */ @@ -93,6 +123,101 @@ protected String getRawSchema() { return JavaBaseConstants.DEFAULT_AIRBYTE_INTERNAL_NAMESPACE; } + /** + * Run a sync using 3.0.0 (which is the highest version that still creates v2 final tables with + * lowercased+quoted names). Then run a sync using our current version. + */ + @Test + public void testFinalTableUppercasingMigration_append() throws Exception { + try { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.FULL_REFRESH) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(new AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(SCHEMA)))); + + // First sync + final List messages1 = readMessages("dat/sync1_messages.jsonl"); + runSync(catalog, messages1, "airbyte/destination-snowflake:3.0.0"); + // We no longer have the code to dump a lowercased table, so just move on directly to the new sync + + // Second sync + final List messages2 = readMessages("dat/sync2_messages.jsonl"); + + runSync(catalog, messages2); + + final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_raw.jsonl"); + final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_append_final.jsonl"); + verifySyncResult(expectedRawRecords2, expectedFinalRecords2, disableFinalTableComparison()); + } finally { + // manually drop the lowercased schema, since we no longer have the code to do it automatically + // (the raw table is still in lowercase "airbyte_internal"."whatever", so the auto-cleanup code + // handles it fine) + database.execute("DROP SCHEMA IF EXISTS \"" + streamNamespace + "\" CASCADE"); + } + } + + @Test + public void testFinalTableUppercasingMigration_overwrite() throws Exception { + try { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.FULL_REFRESH) + .withDestinationSyncMode(DestinationSyncMode.OVERWRITE) + .withStream(new AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(SCHEMA)))); + + // First sync + final List messages1 = readMessages("dat/sync1_messages.jsonl"); + runSync(catalog, messages1, "airbyte/destination-snowflake:3.0.0"); + // We no longer have the code to dump a lowercased table, so just move on directly to the new sync + + // Second sync + final List messages2 = readMessages("dat/sync2_messages.jsonl"); + + runSync(catalog, messages2); + + final List expectedRawRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_overwrite_raw.jsonl"); + final List expectedFinalRecords2 = readRecords("dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl"); + verifySyncResult(expectedRawRecords2, expectedFinalRecords2, disableFinalTableComparison()); + } finally { + // manually drop the lowercased schema, since we no longer have the code to do it automatically + // (the raw table is still in lowercase "airbyte_internal"."whatever", so the auto-cleanup code + // handles it fine) + database.execute("DROP SCHEMA IF EXISTS \"" + streamNamespace + "\" CASCADE"); + } + } + + @Test + public void testRemovingPKNonNullIndexes() throws Exception { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withDestinationSyncMode(DestinationSyncMode.APPEND_DEDUP) + .withPrimaryKey(List.of(List.of("id1"), List.of("id2"))) + .withStream(new AirbyteStream() + .withNamespace(streamNamespace) + .withName(streamName) + .withJsonSchema(SCHEMA)))); + + // First sync + final List messages = readMessages("dat/sync_null_pk.jsonl"); + final TestHarnessException e = assertThrows( + TestHarnessException.class, + () -> runSync(catalog, messages, "airbyte/destination-snowflake:3.1.18")); // this version introduced non-null PKs to the final tables + // ideally we would assert on the logged content of the original exception within e, but that is + // proving to be tricky + + // Second sync + runSync(catalog, messages); // does not throw with latest version + assertEquals(1, dumpFinalTableRecords(streamNamespace, streamName).toArray().length); + } + private String getDefaultSchema() { return getConfig().get("schema").asText(); } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeInternalStagingDisableTypingDedupingTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeInternalStagingDisableTypingDedupingTest.java new file mode 100644 index 000000000000..0fe354779bd6 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeInternalStagingDisableTypingDedupingTest.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.snowflake.typing_deduping; + +public class SnowflakeInternalStagingDisableTypingDedupingTest extends AbstractSnowflakeTypingDedupingTest { + + @Override + protected String getConfigPath() { + return "secrets/1s1t_disabletd_internal_staging_config.json"; + } + + @Override + protected boolean disableFinalTableComparison() { + return true; + } + + @Override + public void testRemovingPKNonNullIndexes() throws Exception { + // Do nothing. + } + + @Override + public void identicalNameSimultaneousSync() throws Exception { + // TODO: create fixtures to verify how raw tables are affected. Base tests check for final tables. + } + +} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeInternalStagingLowercaseDatabaseTypingDedupingTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeInternalStagingLowercaseDatabaseTypingDedupingTest.java new file mode 100644 index 000000000000..4411df398774 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeInternalStagingLowercaseDatabaseTypingDedupingTest.java @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.snowflake.typing_deduping; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.jdbc.JdbcUtils; + +public class SnowflakeInternalStagingLowercaseDatabaseTypingDedupingTest extends AbstractSnowflakeTypingDedupingTest { + + @Override + protected String getConfigPath() { + return "secrets/1s1t_internal_staging_config.json"; + } + + /** + * Verify that even if the config has a lowercase database name, we're able to run syncs + * successfully. This is a regression test for a bug where we were not upcasing the database name + * when checking for an existing final table. + */ + @Override + protected JsonNode generateConfig() { + final JsonNode config = super.generateConfig(); + ((ObjectNode) config).put(JdbcUtils.DATABASE_KEY, config.get(JdbcUtils.DATABASE_KEY).asText().toLowerCase()); + return config; + } + +} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeInternalStagingRawSchemaOverrideDisableTypingDedupingTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeInternalStagingRawSchemaOverrideDisableTypingDedupingTest.java new file mode 100644 index 000000000000..67785a0e2898 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeInternalStagingRawSchemaOverrideDisableTypingDedupingTest.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.snowflake.typing_deduping; + +public class SnowflakeInternalStagingRawSchemaOverrideDisableTypingDedupingTest extends AbstractSnowflakeTypingDedupingTest { + + @Override + protected String getConfigPath() { + return "secrets/1s1t_disabletd_internal_staging_config_raw_schema_override.json"; + } + + @Override + protected String getRawSchema() { + return "overridden_raw_dataset"; + } + + @Override + protected boolean disableFinalTableComparison() { + return true; + } + + @Override + public void testRemovingPKNonNullIndexes() throws Exception { + // Do nothing. + } + + @Override + public void identicalNameSimultaneousSync() throws Exception { + // TODO: create fixtures to verify how raw tables are affected. Base tests check for final tables. + } + +} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorIntegrationTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorIntegrationTest.java index eabe272d3e5e..13338a83a03e 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorIntegrationTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorIntegrationTest.java @@ -9,17 +9,20 @@ import static java.util.stream.Collectors.toMap; import static org.junit.jupiter.api.Assertions.assertAll; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; -import autovalue.shaded.com.google.common.collect.ImmutableMap; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.base.destination.typing_deduping.BaseSqlGeneratorIntegrationTest; +import io.airbyte.integrations.base.destination.typing_deduping.Sql; import io.airbyte.integrations.base.destination.typing_deduping.StreamId; import io.airbyte.integrations.destination.snowflake.OssCloudEnvVarConsts; import io.airbyte.integrations.destination.snowflake.SnowflakeDatabase; @@ -27,6 +30,7 @@ import io.airbyte.integrations.destination.snowflake.SnowflakeTestUtils; import java.nio.file.Path; import java.sql.SQLException; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Optional; @@ -69,9 +73,14 @@ protected SnowflakeDestinationHandler getDestinationHandler() { return new SnowflakeDestinationHandler(databaseName, database); } + @Override + protected StreamId buildStreamId(final String namespace, final String finalTableName, final String rawTableName) { + return new StreamId(namespace.toUpperCase(), finalTableName.toUpperCase(), namespace.toUpperCase(), rawTableName, namespace, finalTableName); + } + @Override protected void createNamespace(final String namespace) throws SQLException { - database.execute("CREATE SCHEMA IF NOT EXISTS \"" + namespace + '"'); + database.execute("CREATE SCHEMA IF NOT EXISTS \"" + namespace.toUpperCase() + '"'); } @Override @@ -88,37 +97,6 @@ protected void createRawTable(final StreamId streamId) throws Exception { """)); } - @Override - protected void createFinalTable(final boolean includeCdcDeletedAt, final StreamId streamId, final String suffix) throws Exception { - final String cdcDeletedAt = includeCdcDeletedAt ? "\"_ab_cdc_deleted_at\" TIMESTAMP_TZ," : ""; - database.execute(new StringSubstitutor(Map.of( - "final_table_id", streamId.finalTableId(SnowflakeSqlGenerator.QUOTE, suffix), - "cdc_deleted_at", cdcDeletedAt)).replace( - """ - CREATE TABLE ${final_table_id} ( - "_airbyte_raw_id" TEXT NOT NULL, - "_airbyte_extracted_at" TIMESTAMP_TZ NOT NULL, - "_airbyte_meta" VARIANT NOT NULL, - "id1" NUMBER, - "id2" NUMBER, - "updated_at" TIMESTAMP_TZ, - ${cdc_deleted_at} - "struct" OBJECT, - "array" ARRAY, - "string" TEXT, - "number" FLOAT, - "integer" NUMBER, - "boolean" BOOLEAN, - "timestamp_with_timezone" TIMESTAMP_TZ, - "timestamp_without_timezone" TIMESTAMP_NTZ, - "time_with_timezone" TEXT, - "time_without_timezone" TIME, - "date" DATE, - "unknown" VARIANT - ) - """)); - } - @Override protected List dumpRawTableRecords(final StreamId streamId) throws Exception { return SnowflakeTestUtils.dumpRawTable(database, streamId.rawTableId(SnowflakeSqlGenerator.QUOTE)); @@ -130,12 +108,12 @@ protected List dumpFinalTableRecords(final StreamId streamId, final St database, databaseName, streamId.finalNamespace(), - streamId.finalName() + suffix); + streamId.finalName() + suffix.toUpperCase()); } @Override protected void teardownNamespace(final String namespace) throws SQLException { - database.execute("DROP SCHEMA IF EXISTS \"" + namespace + '"'); + database.execute("DROP SCHEMA IF EXISTS \"" + namespace.toUpperCase() + '"'); } @Override @@ -145,7 +123,7 @@ protected void insertFinalTableRecords(final boolean includeCdcDeletedAt, final List records) throws Exception { final List columnNames = includeCdcDeletedAt ? FINAL_TABLE_COLUMN_NAMES_CDC : FINAL_TABLE_COLUMN_NAMES; - final String cdcDeletedAtName = includeCdcDeletedAt ? ",\"_ab_cdc_deleted_at\"" : ""; + final String cdcDeletedAtName = includeCdcDeletedAt ? ",\"_AB_CDC_DELETED_AT\"" : ""; final String cdcDeletedAtExtract = includeCdcDeletedAt ? ",column19" : ""; final String recordsText = records.stream() // For each record, convert it to a string like "(rawId, extractedAt, loadedAt, data)" @@ -158,7 +136,7 @@ protected void insertFinalTableRecords(final boolean includeCdcDeletedAt, database.execute(new StringSubstitutor( Map.of( - "final_table_id", streamId.finalTableId(SnowflakeSqlGenerator.QUOTE, suffix), + "final_table_id", streamId.finalTableId(SnowflakeSqlGenerator.QUOTE, suffix.toUpperCase()), "cdc_deleted_at_name", cdcDeletedAtName, "cdc_deleted_at_extract", cdcDeletedAtExtract, "records", recordsText), @@ -168,24 +146,24 @@ protected void insertFinalTableRecords(final boolean includeCdcDeletedAt, // parse_json(). """ INSERT INTO #{final_table_id} ( - "_airbyte_raw_id", - "_airbyte_extracted_at", - "_airbyte_meta", - "id1", - "id2", - "updated_at", - "struct", - "array", - "string", - "number", - "integer", - "boolean", - "timestamp_with_timezone", - "timestamp_without_timezone", - "time_with_timezone", - "time_without_timezone", - "date", - "unknown" + "_AIRBYTE_RAW_ID", + "_AIRBYTE_EXTRACTED_AT", + "_AIRBYTE_META", + "ID1", + "ID2", + "UPDATED_AT", + "STRUCT", + "ARRAY", + "STRING", + "NUMBER", + "INTEGER", + "BOOLEAN", + "TIMESTAMP_WITH_TIMEZONE", + "TIMESTAMP_WITHOUT_TIMEZONE", + "TIME_WITH_TIMEZONE", + "TIME_WITHOUT_TIMEZONE", + "DATE", + "UNKNOWN" #{cdc_deleted_at_name} ) SELECT @@ -258,15 +236,23 @@ protected void insertRawTableRecords(final StreamId streamId, final List getFinalMetadataColumnNames() { + return AbstractSnowflakeTypingDedupingTest.FINAL_METADATA_COLUMN_NAMES; + } + @Override @Test public void testCreateTableIncremental() throws Exception { - final String sql = generator.createTable(incrementalDedupStream, "", false); + final Sql sql = generator.createTable(incrementalDedupStream, "", false); destinationHandler.execute(sql); - final Optional tableKind = database.queryJsons(String.format("SHOW TABLES LIKE '%s' IN SCHEMA \"%s\";", "users_final", namespace)) - .stream().map(record -> record.get("kind").asText()) - .findFirst(); + // Note that USERS_FINAL is uppercased here. This is intentional, because snowflake upcases unquoted + // identifiers. + final Optional tableKind = + database.queryJsons(String.format("SHOW TABLES LIKE '%s' IN SCHEMA \"%s\";", "USERS_FINAL", namespace.toUpperCase())) + .stream().map(record -> record.get("kind").asText()) + .findFirst(); final Map columns = database.queryJsons( """ SELECT column_name, data_type, numeric_precision, numeric_scale @@ -277,8 +263,8 @@ public void testCreateTableIncremental() throws Exception { ORDER BY ordinal_position; """, databaseName, - namespace, - "users_final").stream() + namespace.toUpperCase(), + "USERS_FINAL").stream() .collect(toMap( record -> record.get("COLUMN_NAME").asText(), record -> { @@ -293,24 +279,24 @@ record -> { () -> assertEquals(Optional.of("TABLE"), tableKind, "Table should be permanent, not transient"), () -> assertEquals( ImmutableMap.builder() - .put("_airbyte_raw_id", "TEXT") - .put("_airbyte_extracted_at", "TIMESTAMP_TZ") - .put("_airbyte_meta", "VARIANT") - .put("id1", "NUMBER(38, 0)") - .put("id2", "NUMBER(38, 0)") - .put("updated_at", "TIMESTAMP_TZ") - .put("struct", "OBJECT") - .put("array", "ARRAY") - .put("string", "TEXT") - .put("number", "FLOAT") - .put("integer", "NUMBER(38, 0)") - .put("boolean", "BOOLEAN") - .put("timestamp_with_timezone", "TIMESTAMP_TZ") - .put("timestamp_without_timezone", "TIMESTAMP_NTZ") - .put("time_with_timezone", "TEXT") - .put("time_without_timezone", "TIME") - .put("date", "DATE") - .put("unknown", "VARIANT") + .put("_AIRBYTE_RAW_ID", "TEXT") + .put("_AIRBYTE_EXTRACTED_AT", "TIMESTAMP_TZ") + .put("_AIRBYTE_META", "VARIANT") + .put("ID1", "NUMBER(38, 0)") + .put("ID2", "NUMBER(38, 0)") + .put("UPDATED_AT", "TIMESTAMP_TZ") + .put("STRUCT", "OBJECT") + .put("ARRAY", "ARRAY") + .put("STRING", "TEXT") + .put("NUMBER", "FLOAT") + .put("INTEGER", "NUMBER(38, 0)") + .put("BOOLEAN", "BOOLEAN") + .put("TIMESTAMP_WITH_TIMEZONE", "TIMESTAMP_TZ") + .put("TIMESTAMP_WITHOUT_TIMEZONE", "TIMESTAMP_NTZ") + .put("TIME_WITH_TIMEZONE", "TEXT") + .put("TIME_WITHOUT_TIMEZONE", "TIME") + .put("DATE", "DATE") + .put("UNKNOWN", "VARIANT") .build(), columns)); } @@ -381,8 +367,8 @@ protected void migrationAssertions(final List v1RawRecords, final List record -> record.get(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID).asText(), Function.identity())); assertAll( - () -> assertEquals(5, v1RawRecords.size()), - () -> assertEquals(5, v2RawRecords.size())); + () -> assertEquals(6, v1RawRecords.size()), + () -> assertEquals(6, v2RawRecords.size())); v1RawRecords.forEach(v1Record -> { final var v1id = v1Record.get(JavaBaseConstants.COLUMN_NAME_AB_ID.toUpperCase()).asText(); assertAll( @@ -400,4 +386,46 @@ record -> record.get(JavaBaseConstants.COLUMN_NAME_AB_RAW_ID).asText(), }); } + /** + * Verify that the final table does not include NON-NULL PKs (after + * https://github.com/airbytehq/airbyte/pull/31082) + */ + @Test + public void ensurePKsAreIndexedUnique() throws Exception { + createRawTable(streamId); + insertRawTableRecords( + streamId, + List.of(Jsons.deserialize( + """ + { + "_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", + "_airbyte_extracted_at": "2023-01-01T00:00:00Z", + "_airbyte_data": { + "id1": 1, + "id2": 2 + } + } + """))); + + final Sql createTable = generator.createTable(incrementalDedupStream, "", false); + + // should be OK with new tables + destinationHandler.execute(createTable); + final Optional existingTableA = destinationHandler.findExistingTable(streamId); + assertTrue(generator.existingSchemaMatchesStreamConfig(incrementalDedupStream, existingTableA.get())); + destinationHandler.execute(Sql.of("DROP TABLE " + streamId.finalTableId(""))); + + // Hack the create query to add NOT NULLs to emulate the old behavior + List> createTableModified = createTable.transactions().stream().map(transaction -> transaction.stream() + .map(statement -> Arrays.stream(statement.split(System.lineSeparator())).map( + line -> !line.contains("CLUSTER") && (line.contains("id1") || line.contains("id2") || line.contains("ID1") || line.contains("ID2")) + ? line.replace(",", " NOT NULL,") + : line) + .collect(joining("\r\n"))) + .toList()).toList(); + destinationHandler.execute(new Sql(createTableModified)); + final Optional existingTableB = destinationHandler.findExistingTable(streamId); + assertFalse(generator.existingSchemaMatchesStreamConfig(incrementalDedupStream, existingTableB.get())); + } + } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl index bf928d688997..7c9e93b21705 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_final.jsonl @@ -1,3 +1,3 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "old_cursor": 1, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie"} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 200, "OLD_CURSOR": 1, "NAME": "Alice", "ADDRESS": {"city": "Los Angeles", "state": "CA"}} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 201, "OLD_CURSOR": 2, "NAME": "Bob", "ADDRESS": {"city": "Boston", "state": "MA"}} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "ID1": 2, "ID2": 200, "OLD_CURSOR": 3, "NAME": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl index 4a370ae69377..fcf596ac0380 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_cursorchange_expectedrecords_dedup_raw.jsonl @@ -1,3 +1,4 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}} {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} {"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl index f30261d6154c..813561b043bf 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final.jsonl @@ -1,4 +1,4 @@ -// Keep the Alice record with more recent updated_at -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000000Z", "name": "Charlie"} +// Keep the Alice record with more recent UPDATED_AT +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-01T00:01:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "Los Angeles", "state": "CA"}} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 201, "UPDATED_AT": "2000-01-01T00:02:00.000000000Z", "NAME": "Bob", "ADDRESS": {"city": "Boston", "state": "MA"}} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "ID1": 2, "ID2": 200, "UPDATED_AT": "2000-01-01T00:03:00.000000000Z", "NAME": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl new file mode 100644 index 000000000000..5f9395498870 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_dedup_final2.jsonl @@ -0,0 +1 @@ +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2001-01-01T00:00:00.000000000Z", "NAME": "Someone completely different"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_dedup_raw.jsonl deleted file mode 100644 index 7f6e7aa9438b..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_dedup_raw.jsonl +++ /dev/null @@ -1,6 +0,0 @@ -// Keep the Alice record with more recent updated_at -// Note that extracted_at uses microseconds precision (because it's parsed directly by snowflake) -// but updated_at is still using seconds precision (because Snowflake treats it as a normal string inside a JSON blob) -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl index a26cf1d5289d..d0c20a410997 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_final.jsonl @@ -1,5 +1,5 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-01T00:00:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "San Francisco", "state": "CA"}} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-01T00:01:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "Los Angeles", "state": "CA"}} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 201, "UPDATED_AT": "2000-01-01T00:02:00.000000000Z", "NAME": "Bob", "ADDRESS": {"city": "Boston", "state": "MA"}} // Invalid columns are nulled out (i.e. SQL null, not JSON null) -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000000Z", "name": "Charlie"} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "ID1": 2, "ID2": 200, "UPDATED_AT": "2000-01-01T00:03:00.000000000Z", "NAME": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl similarity index 100% rename from airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_nondedup_raw.jsonl rename to airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_raw.jsonl diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl new file mode 100644 index 000000000000..b0f0f8823c90 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync1_expectedrecords_raw2.jsonl @@ -0,0 +1 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different"}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl index 7ac3264abcc4..93e29eb904e4 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_final.jsonl @@ -1,3 +1,3 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} -// Charlie wasn't reemitted with updated_at, so it still has a null cursor -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "name": "Charlie"} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:02.000000000-08:00", "_AIRBYTE_META":{"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-02T00:00:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "Seattle", "state": "WA"}} +// Charlie wasn't reemitted with UPDATED_AT, so it still has a null cursor +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "ID1": 2, "ID2": 200, "NAME": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl index fc788ceeb4a5..347a9248d265 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_cursorchange_expectedrecords_incremental_dedup_raw.jsonl @@ -1,4 +1,7 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 0, "_ab_cdc_deleted_at": null, "name" :"Alice", "address": {"city": "San Francisco", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "old_cursor": 1, "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 201, "old_cursor": 2, "name": "Bob", "address": {"city": "Boston", "state": "MA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} {"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Bob", "address": {"city": "New York", "state": "NY"}}} {"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} -// Charlie wasn't reemitted in sync2. This record still has an old_cursor value. -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 2, "id2": 200, "old_cursor": 3, "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl index fd96bc516169..540aaf560412 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_final.jsonl @@ -1,8 +1,8 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:00:00.000000000Z", "name": "Alice", "address": {"city": "San Francisco", "state": "CA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-01T00:01:00.000000000Z", "name": "Alice", "address": {"city": "Los Angeles", "state": "CA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-01T00:02:00.000000000Z", "name": "Bob", "address": {"city": "Boston", "state": "MA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000000Z", "name": "Charlie"} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-01T00:00:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "San Francisco", "state": "CA"}} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-01T00:01:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "Los Angeles", "state": "CA"}} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":[]}, "ID1": 1, "ID2": 201, "UPDATED_AT": "2000-01-01T00:02:00.000000000Z", "NAME": "Bob", "ADDRESS": {"city": "Boston", "state": "MA"}} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "ID1": 2, "ID2": 200, "UPDATED_AT": "2000-01-01T00:03:00.000000000Z", "NAME": "Charlie"} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000000Z"} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:02.000000000-08:00", "_AIRBYTE_META":{"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-02T00:00:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "Seattle", "state": "WA"}} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:02.000000000-08:00", "_AIRBYTE_META":{"errors":[]}, "ID1": 1, "ID2": 201, "UPDATED_AT": "2000-01-02T00:00:00.000000000Z", "NAME": "Bob", "ADDRESS": {"city": "New York", "state": "NY"}} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:02.000000000-08:00", "_AIRBYTE_META":{"errors":[]}, "ID1": 1, "ID2": 201, "UPDATED_AT": "2000-01-02T00:01:00.000000000Z", "_AB_CDC_DELETED_AT": "1970-01-01T00:00:00.000000000Z"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl index da89677485f0..61366dee9ab4 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_overwrite_final.jsonl @@ -1,3 +1,3 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:00:00.000000000Z", "name": "Bob", "address": {"city": "New York", "state": "NY"}} -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00.000000000Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00.000000000Z"} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:02.000000000-08:00", "_AIRBYTE_META":{"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-02T00:00:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "Seattle", "state": "WA"}} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:02.000000000-08:00", "_AIRBYTE_META":{"errors":[]}, "ID1": 1, "ID2": 201, "UPDATED_AT": "2000-01-02T00:00:00.000000000Z", "NAME": "Bob", "ADDRESS": {"city": "New York", "state": "NY"}} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:02.000000000-08:00", "_AIRBYTE_META":{"errors":[]}, "ID1": 1, "ID2": 201, "UPDATED_AT": "2000-01-02T00:01:00.000000000Z", "_AB_CDC_DELETED_AT": "1970-01-01T00:00:00.000000000Z"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl index 96442a4e7fcd..9bd9f65927d8 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final.jsonl @@ -1,3 +1,3 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_meta":{"errors":[]}, "id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00.000000000Z", "name": "Alice", "address": {"city": "Seattle", "state": "WA"}} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:02.000000000-08:00", "_AIRBYTE_META":{"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2000-01-02T00:00:00.000000000Z", "NAME": "Alice", "ADDRESS": {"city": "Seattle", "state": "WA"}} // Delete Bob, keep Charlie -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_meta": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00.000000000Z", "name": "Charlie"} +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:01.000000000-08:00", "_AIRBYTE_META": {"errors":["Problem with `age`", "Problem with `registration_date`"]}, "ID1": 2, "ID2": 200, "UPDATED_AT": "2000-01-01T00:03:00.000000000Z", "NAME": "Charlie"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl new file mode 100644 index 000000000000..b86eb147ba89 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_final2.jsonl @@ -0,0 +1 @@ +{"_AIRBYTE_EXTRACTED_AT": "1970-01-01T00:00:02.000000000-08:00", "_AIRBYTE_META":{"errors":[]}, "ID1": 1, "ID2": 200, "UPDATED_AT": "2001-01-02T00:00:00.000000000Z", "NAME": "Someone completely different v2"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_raw.jsonl deleted file mode 100644 index fe2377ede753..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_incremental_dedup_raw.jsonl +++ /dev/null @@ -1,5 +0,0 @@ -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2000-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Alice", "address": {"city": "Seattle", "state": "WA"}}} -// Keep the record that deleted Bob, but delete the other records associated with id=(1, 201) -{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 201, "updated_at": "2000-01-02T00:01:00Z", "_ab_cdc_deleted_at": "1970-01-01T00:00:00Z"}} -// And keep Charlie's record, even though it wasn't reemitted in sync2. -{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 2, "id2": 200, "updated_at": "2000-01-01T00:03:00Z", "name": "Charlie", "age": "this is not an integer", "registration_date": "this is not a date"}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl similarity index 100% rename from airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_fullrefresh_append_raw.jsonl rename to airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_raw.jsonl diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl new file mode 100644 index 000000000000..4d2e3167888c --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/dat/sync2_expectedrecords_raw2.jsonl @@ -0,0 +1,2 @@ +{"_airbyte_extracted_at": "1970-01-01T00:00:01.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-01T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different"}} +{"_airbyte_extracted_at": "1970-01-01T00:00:02.000000000-08:00", "_airbyte_data": {"id1": 1, "id2": 200, "updated_at": "2001-01-02T00:00:00Z", "_ab_cdc_deleted_at": null, "name": "Someone completely different v2"}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl index c17a8134a49e..f7bffd258123 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_final.jsonl @@ -1,6 +1,7 @@ -{"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000000Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56.000000000Z", "timestamp_without_timezone": "2023-01-23T12:34:56.000000000", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56.000000000", "date": "2023-01-23", "unknown": {}, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}} -{"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000000Z", "unknown": null, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}} -{"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000000Z", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}} -{"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000000Z", "unknown": null, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": ["Problem with `struct`", "Problem with `array`", "Problem with `number`", "Problem with `integer`", "Problem with `boolean`", "Problem with `timestamp_with_timezone`", "Problem with `timestamp_without_timezone`", "Problem with `time_with_timezone`", "Problem with `time_without_timezone`", "Problem with `date`"]}, "string": "{}"} +{"ID1": 1, "ID2": 100, "UPDATED_AT": "2023-01-01T01:00:00.000000000Z", "ARRAY": ["foo"], "STRUCT": {"foo": "bar"}, "STRING": "foo", "NUMBER": 42.1, "INTEGER": 42, "BOOLEAN": true, "TIMESTAMP_WITH_TIMEZONE": "2023-01-23T12:34:56.000000000Z", "TIMESTAMP_WITHOUT_TIMEZONE": "2023-01-23T12:34:56.000000000", "TIME_WITH_TIMEZONE": "12:34:56Z", "TIME_WITHOUT_TIMEZONE": "12:34:56.000000000", "DATE": "2023-01-23", "UNKNOWN": {}, "_AIRBYTE_EXTRACTED_AT": "2023-01-01T00:00:00.000000000Z", "_AIRBYTE_META": {"errors": []}} +{"ID1": 2, "ID2": 100, "UPDATED_AT": "2023-01-01T01:00:00.000000000Z", "UNKNOWN": null, "_AIRBYTE_EXTRACTED_AT": "2023-01-01T00:00:00.000000000Z", "_AIRBYTE_META": {"errors": []}} +{"ID1": 3, "ID2": 100, "UPDATED_AT": "2023-01-01T01:00:00.000000000Z", "_AIRBYTE_EXTRACTED_AT": "2023-01-01T00:00:00.000000000Z", "_AIRBYTE_META": {"errors": []}} +{"ID1": 4, "ID2": 100, "UPDATED_AT": "2023-01-01T01:00:00.000000000Z", "UNKNOWN": null, "_AIRBYTE_EXTRACTED_AT": "2023-01-01T00:00:00.000000000Z", "_AIRBYTE_META": {"errors": ["Problem with `struct`", "Problem with `array`", "Problem with `number`", "Problem with `integer`", "Problem with `boolean`", "Problem with `timestamp_with_timezone`", "Problem with `timestamp_without_timezone`", "Problem with `time_with_timezone`", "Problem with `time_without_timezone`", "Problem with `date`"]}} // Note: no loss of precision on these numbers. A naive float64 conversion would yield 67.17411800000001. -{"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00.000000000Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118, "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}} +{"ID1": 5, "ID2": 100, "UPDATED_AT": "2023-01-01T01:00:00.000000000Z", "NUMBER": 67.174118, "STRUCT": {"nested_number": 67.174118}, "ARRAY": [67.174118], "UNKNOWN": 67.174118, "_AIRBYTE_EXTRACTED_AT": "2023-01-01T00:00:00.000000000Z", "_AIRBYTE_META": {"errors": []}} +{"ID1": 6, "ID2": 100, "UPDATED_AT": "2023-01-01T01:00:00.000000000Z", "IAMACASESENSITIVECOLUMNNAME": "Case senstive value", "_AIRBYTE_EXTRACTED_AT":"2023-01-01T00:00:00.000000000Z", "_AIRBYTE_META":{"errors":[]}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl index 45e13560e19f..e5909080bd83 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/alltypes_expectedrecords_raw.jsonl @@ -1,5 +1,6 @@ {"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "foo", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}}' {"_airbyte_raw_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": null, "struct": null, "string": null, "number": null, "integer": null, "boolean": null, "timestamp_with_timezone": null, "timestamp_without_timezone": null, "time_with_timezone": null, "time_without_timezone": null, "date": null, "unknown": null}} {"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z"}} -{"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": {}, "number": {}, "integer": {}, "boolean": {}, "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": {}, "unknown": null}} +{"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": {}, "struct": [], "string": null, "number": "foo", "integer": "bar", "boolean": "fizz", "timestamp_with_timezone": {}, "timestamp_without_timezone": {}, "time_with_timezone": {}, "time_without_timezone": {}, "date": "airbyte", "unknown": null}} {"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "number": 67.174118, "struct": {"nested_number": 67.174118}, "array": [67.174118], "unknown": 67.174118}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fce", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 6, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "IamACaseSensitiveColumnName": "Case senstive value"}} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_final.jsonl index 293be295e24d..f2c1e51fba67 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_final.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_final.jsonl @@ -1,2 +1,2 @@ -{"_airbyte_raw_id": "80c99b54-54b4-43bd-b51b-1f67dafa2c52", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00.000000000Z", "string": "Alice", "struct": {"city": "San Diego", "state": "CA"}, "integer": 84} -{"_airbyte_raw_id": "ad690bfb-c2c2-4172-bd73-a16c86ccbb67", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": ["Problem with `integer`"]}, "id1": 2, "id2": 100, "updated_at": "2023-01-01T03:00:00.000000000Z", "string": "Bob"} +{"_AIRBYTE_RAW_ID": "80c99b54-54b4-43bd-b51b-1f67dafa2c52", "_AIRBYTE_EXTRACTED_AT": "2023-01-01T00:00:00.000000000Z", "_AIRBYTE_META": {"errors": []}, "ID1": 1, "ID2": 100, "UPDATED_AT": "2023-01-01T02:00:00.000000000Z", "STRING": "Alice", "STRUCT": {"city": "San Diego", "state": "CA"}, "INTEGER": 84} +{"_AIRBYTE_RAW_ID": "ad690bfb-c2c2-4172-bd73-a16c86ccbb67", "_AIRBYTE_EXTRACTED_AT": "2023-01-01T00:00:00.000000000Z", "_AIRBYTE_META": {"errors": ["Problem with `integer`"]}, "ID1": 2, "ID2": 100, "UPDATED_AT": "2023-01-01T03:00:00.000000000Z", "STRING": "Bob"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl index 0cdce4a5e75e..cea4f178f80c 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/incrementaldedup_expectedrecords_raw.jsonl @@ -1,2 +1,3 @@ +{"_airbyte_raw_id": "d7b81af0-01da-4846-a650-cc398986bc99", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "string": "Alice", "struct": {"city": "San Francisco", "state": "CA"}, "integer": 42}} {"_airbyte_raw_id": "80c99b54-54b4-43bd-b51b-1f67dafa2c52", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00Z", "string": "Alice", "struct": {"city": "San Diego", "state": "CA"}, "integer": 84}} {"_airbyte_raw_id": "ad690bfb-c2c2-4172-bd73-a16c86ccbb67", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T03:00:00Z", "string": "Bob", "integer": "oops"}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_final.jsonl new file mode 100644 index 000000000000..7359f58de29f --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_final.jsonl @@ -0,0 +1,5 @@ +{"ID1":1,"ID2":100,"UPDATED_AT":"2023-01-01T01:00:00.000000000Z","ARRAY":["foo"],"STRUCT":{"foo":"bar"},"STRING":"[\"I\",\"am\",\"an\",\"array\"]","NUMBER":42.1,"INTEGER":42,"BOOLEAN":true,"TIMESTAMP_WITH_TIMEZONE":"2023-01-23T12:34:56.000000000Z","TIMESTAMP_WITHOUT_TIMEZONE":"2023-01-23T12:34:56.000000000","TIME_WITH_TIMEZONE":"12:34:56Z","TIME_WITHOUT_TIMEZONE":"12:34:56.000000000","DATE":"2023-01-23","UNKNOWN":{},"_AIRBYTE_EXTRACTED_AT":"2023-01-01T00:00:00.000000000Z","_AIRBYTE_META":{"errors":[]}} +{"ID1":2,"ID2":100,"UPDATED_AT":"2023-01-01T01:00:00.000000000Z","ARRAY":["foo"],"STRUCT":{"foo":"bar"},"STRING":"{\"I\":\"am\",\"an\":\"object\"}","NUMBER":42.1,"INTEGER":42,"BOOLEAN":true,"TIMESTAMP_WITH_TIMEZONE":"2023-01-23T12:34:56.000000000Z","TIMESTAMP_WITHOUT_TIMEZONE":"2023-01-23T12:34:56.000000000","TIME_WITH_TIMEZONE":"12:34:56Z","TIME_WITHOUT_TIMEZONE":"12:34:56.000000000","DATE":"2023-01-23","UNKNOWN":{},"_AIRBYTE_EXTRACTED_AT":"2023-01-01T00:00:00.000000000Z","_AIRBYTE_META":{"errors":[]}} +{"ID1":3,"ID2":100,"UPDATED_AT":"2023-01-01T01:00:00.000000000Z","ARRAY":["foo"],"STRUCT":{"foo":"bar"},"STRING":"true","NUMBER":42.1,"INTEGER":42,"BOOLEAN":true,"TIMESTAMP_WITH_TIMEZONE":"2023-01-23T12:34:56.000000000Z","TIMESTAMP_WITHOUT_TIMEZONE":"2023-01-23T12:34:56.000000000","TIME_WITH_TIMEZONE":"12:34:56Z","TIME_WITHOUT_TIMEZONE":"12:34:56.000000000","DATE":"2023-01-23","UNKNOWN":{},"_AIRBYTE_EXTRACTED_AT":"2023-01-01T00:00:00.000000000Z","_AIRBYTE_META":{"errors":[]}} +{"ID1":4,"ID2":100,"UPDATED_AT":"2023-01-01T01:00:00.000000000Z","ARRAY":["foo"],"STRUCT":{"foo":"bar"},"STRING":"3.14","NUMBER":42.1,"INTEGER":42,"BOOLEAN":true,"TIMESTAMP_WITH_TIMEZONE":"2023-01-23T12:34:56.000000000Z","TIMESTAMP_WITHOUT_TIMEZONE":"2023-01-23T12:34:56.000000000","TIME_WITH_TIMEZONE":"12:34:56Z","TIME_WITHOUT_TIMEZONE":"12:34:56.000000000","DATE":"2023-01-23","UNKNOWN":{},"_AIRBYTE_EXTRACTED_AT":"2023-01-01T00:00:00.000000000Z","_AIRBYTE_META":{"errors":[]}} +{"ID1":5,"ID2":100,"UPDATED_AT":"2023-01-01T01:00:00.000000000Z","ARRAY":["foo"],"STRUCT":{"foo":"bar"},"STRING":"I am a valid json string","NUMBER":42.1,"INTEGER":42,"BOOLEAN":true,"TIMESTAMP_WITH_TIMEZONE":"2023-01-23T12:34:56.000000000Z","TIMESTAMP_WITHOUT_TIMEZONE":"2023-01-23T12:34:56.000000000","TIME_WITH_TIMEZONE":"12:34:56Z","TIME_WITHOUT_TIMEZONE":"12:34:56.000000000","DATE":"2023-01-23","UNKNOWN":{},"_AIRBYTE_EXTRACTED_AT":"2023-01-01T00:00:00.000000000Z","_AIRBYTE_META":{"errors":[]}} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_raw.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_raw.jsonl new file mode 100644 index 000000000000..6a8cd4fac500 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/json_types_in_string_expectedrecords_raw.jsonl @@ -0,0 +1,5 @@ +{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 1, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": ["I", "am", "an", "array"], "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "53ce75a5-5bcc-47a3-b45c-96c2015cfe35", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 2, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": {"I": "am", "an": "object"}, "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "7e1fac0c-017e-4ad6-bc78-334a34d64fbe", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 3, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": true, "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "84242b60-3a34-4531-ad75-a26702960a9a", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 4, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": 3.14, "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} +{"_airbyte_raw_id": "a4a783b5-7729-4d0b-b659-48ceb08713f1", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_data": {"id1": 5, "id2": 100, "updated_at": "2023-01-01T01:00:00Z", "array": ["foo"], "struct": {"foo": "bar"}, "string": "I am a valid json string", "number": 42.1, "integer": 42, "boolean": true, "timestamp_with_timezone": "2023-01-23T12:34:56Z", "timestamp_without_timezone": "2023-01-23T12:34:56", "time_with_timezone": "12:34:56Z", "time_without_timezone": "12:34:56", "date": "2023-01-23", "unknown": {}}} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_final.jsonl index b36b5ea4b450..2c5abc7cd246 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_final.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/nocolumns_expectedrecords_final.jsonl @@ -1 +1 @@ -{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}} +{"_AIRBYTE_RAW_ID": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_AIRBYTE_EXTRACTED_AT": "2023-01-01T00:00:00.000000000Z", "_AIRBYTE_META": {"errors": []}} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/reservedkeywords_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/reservedkeywords_expectedrecords_final.jsonl new file mode 100644 index 000000000000..e939c3bec465 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/reservedkeywords_expectedrecords_final.jsonl @@ -0,0 +1,2 @@ +// Note that we prefix `current_date` with an underscore (to `_CURRENT_DATE`). This is because even if you quote `CURRENT_DATE`, Snowflake rejects it as a column name. +{"_AIRBYTE_RAW_ID":"b2e0efc4-38a8-47ba-970c-8103f09f08d5","_AIRBYTE_EXTRACTED_AT":"2023-01-01T00:00:00.000000000Z","_AIRBYTE_META":{"errors":[]}, "_CURRENT_DATE": "foo", "JOIN": "bar"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/timestampformats_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/timestampformats_expectedrecords_final.jsonl index 0064d5adf1b3..dd173acfb7e4 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/timestampformats_expectedrecords_final.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/timestampformats_expectedrecords_final.jsonl @@ -1,12 +1,12 @@ // snowflake/jdbc is returning 9 decimals for all timestamp/time types -{"_airbyte_raw_id": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.000000000Z", "time_with_timezone": "12:34:56Z"} -{"_airbyte_raw_id": "05028c5f-7813-4e9c-bd4b-387d1f8ba435", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.000000000-08:00", "time_with_timezone": "12:34:56-08:00"} -{"_airbyte_raw_id": "95dfb0c6-6a67-4ba0-9935-643bebc90437", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.000000000-08:00", "time_with_timezone": "12:34:56-0800"} -{"_airbyte_raw_id": "f3d8abe2-bb0f-4caf-8ddc-0641df02f3a9", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.000000000-08:00", "time_with_timezone": "12:34:56-08"} -{"_airbyte_raw_id": "a81ed40a-2a49-488d-9714-d53e8b052968", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.000000000+08:00", "time_with_timezone": "12:34:56+08:00"} -{"_airbyte_raw_id": "c07763a0-89e6-4cb7-b7d0-7a34a7c9918a", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.000000000+08:00", "time_with_timezone": "12:34:56+0800"} -{"_airbyte_raw_id": "358d3b52-50ab-4e06-9094-039386f9bf0d", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.000000000+08:00", "time_with_timezone": "12:34:56+08"} -{"_airbyte_raw_id": "db8200ac-b2b9-4b95-a053-8a0343042751", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "timestamp_with_timezone": "2023-01-23T12:34:56.123000000Z", "time_with_timezone": "12:34:56.123Z"} +{"_AIRBYTE_RAW_ID": "14ba7c7f-e398-4e69-ac22-28d578400dbc", "_AIRBYTE_EXTRACTED_AT": "2023-01-01T00:00:00.000000000Z", "_AIRBYTE_META": {"errors": []}, "TIMESTAMP_WITH_TIMEZONE": "2023-01-23T12:34:56.000000000Z", "TIME_WITH_TIMEZONE": "12:34:56Z"} +{"_AIRBYTE_RAW_ID": "05028c5f-7813-4e9c-bd4b-387d1f8ba435", "_AIRBYTE_EXTRACTED_AT": "2023-01-01T00:00:00.000000000Z", "_AIRBYTE_META": {"errors": []}, "TIMESTAMP_WITH_TIMEZONE": "2023-01-23T12:34:56.000000000-08:00", "TIME_WITH_TIMEZONE": "12:34:56-08:00"} +{"_AIRBYTE_RAW_ID": "95dfb0c6-6a67-4ba0-9935-643bebc90437", "_AIRBYTE_EXTRACTED_AT": "2023-01-01T00:00:00.000000000Z", "_AIRBYTE_META": {"errors": []}, "TIMESTAMP_WITH_TIMEZONE": "2023-01-23T12:34:56.000000000-08:00", "TIME_WITH_TIMEZONE": "12:34:56-0800"} +{"_AIRBYTE_RAW_ID": "f3d8abe2-bb0f-4caf-8ddc-0641df02f3a9", "_AIRBYTE_EXTRACTED_AT": "2023-01-01T00:00:00.000000000Z", "_AIRBYTE_META": {"errors": []}, "TIMESTAMP_WITH_TIMEZONE": "2023-01-23T12:34:56.000000000-08:00", "TIME_WITH_TIMEZONE": "12:34:56-08"} +{"_AIRBYTE_RAW_ID": "a81ed40a-2a49-488d-9714-d53e8b052968", "_AIRBYTE_EXTRACTED_AT": "2023-01-01T00:00:00.000000000Z", "_AIRBYTE_META": {"errors": []}, "TIMESTAMP_WITH_TIMEZONE": "2023-01-23T12:34:56.000000000+08:00", "TIME_WITH_TIMEZONE": "12:34:56+08:00"} +{"_AIRBYTE_RAW_ID": "c07763a0-89e6-4cb7-b7d0-7a34a7c9918a", "_AIRBYTE_EXTRACTED_AT": "2023-01-01T00:00:00.000000000Z", "_AIRBYTE_META": {"errors": []}, "TIMESTAMP_WITH_TIMEZONE": "2023-01-23T12:34:56.000000000+08:00", "TIME_WITH_TIMEZONE": "12:34:56+0800"} +{"_AIRBYTE_RAW_ID": "358d3b52-50ab-4e06-9094-039386f9bf0d", "_AIRBYTE_EXTRACTED_AT": "2023-01-01T00:00:00.000000000Z", "_AIRBYTE_META": {"errors": []}, "TIMESTAMP_WITH_TIMEZONE": "2023-01-23T12:34:56.000000000+08:00", "TIME_WITH_TIMEZONE": "12:34:56+08"} +{"_AIRBYTE_RAW_ID": "db8200ac-b2b9-4b95-a053-8a0343042751", "_AIRBYTE_EXTRACTED_AT": "2023-01-01T00:00:00.000000000Z", "_AIRBYTE_META": {"errors": []}, "TIMESTAMP_WITH_TIMEZONE": "2023-01-23T12:34:56.123000000Z", "TIME_WITH_TIMEZONE": "12:34:56.123Z"} -{"_airbyte_raw_id": "10ce5d93-6923-4217-a46f-103833837038", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "timestamp_without_timezone": "2023-01-23T12:34:56.000000000", "time_without_timezone": "12:34:56.000000000", "date": "2023-01-23"} -{"_airbyte_raw_id": "a7a6e176-7464-4a0b-b55c-b4f936e8d5a1", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "timestamp_without_timezone": "2023-01-23T12:34:56.123000000", "time_without_timezone": "12:34:56.123000000"} +{"_AIRBYTE_RAW_ID": "10ce5d93-6923-4217-a46f-103833837038", "_AIRBYTE_EXTRACTED_AT": "2023-01-01T00:00:00.000000000Z", "_AIRBYTE_META": {"errors": []}, "TIMESTAMP_WITHOUT_TIMEZONE": "2023-01-23T12:34:56.000000000", "TIME_WITHOUT_TIMEZONE": "12:34:56.000000000", "DATE": "2023-01-23"} +{"_AIRBYTE_RAW_ID": "a7a6e176-7464-4a0b-b55c-b4f936e8d5a1", "_AIRBYTE_EXTRACTED_AT": "2023-01-01T00:00:00.000000000Z", "_AIRBYTE_META": {"errors": []}, "TIMESTAMP_WITHOUT_TIMEZONE": "2023-01-23T12:34:56.123000000", "TIME_WITHOUT_TIMEZONE": "12:34:56.123000000"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl index b61de2de5f58..e003ad3c9282 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl +++ b/airbyte-integrations/connectors/destination-snowflake/src/test-integration/resources/sqlgenerator/weirdcolumnnames_expectedrecords_final.jsonl @@ -1,3 +1,3 @@ // columns with issues: -// * endswithbackslash\ -> written as null to the final table -{"_airbyte_raw_id": "7e7330a1-42fb-41ec-a955-52f18bd61964", "_airbyte_extracted_at": "2023-01-01T00:00:00.000000000Z", "_airbyte_meta": {"errors": []}, "id1": 1, "id2": 100, "updated_at": "2023-01-01T02:00:00.000000000Z", "$starts_with_dollar_sign": "foo", "includes\"doublequote": "foo", "includes'singlequote": "foo", "includes`backtick": "foo", "includes.period": "foo", "includes$$doubledollar": "foo", "endswithbackslash\\": "foo"} +// * ENDSWITHBACKSLASH\ -> written as null to the final table +{"_AIRBYTE_RAW_ID": "7e7330a1-42fb-41ec-a955-52f18bd61964", "_AIRBYTE_EXTRACTED_AT": "2023-01-01T00:00:00.000000000Z", "_AIRBYTE_META": {"errors": []}, "ID1": 1, "ID2": 100, "UPDATED_AT": "2023-01-01T02:00:00.000000000Z", "$STARTS_WITH_DOLLAR_SIGN": "foo", "INCLUDES\"DOUBLEQUOTE": "foo", "INCLUDES'SINGLEQUOTE": "foo", "INCLUDES`BACKTICK": "foo", "INCLUDES.PERIOD": "foo", "INCLUDES$$DOUBLEDOLLAR": "foo", "ENDSWITHBACKSLASH\\": "foo"} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationTest.java index 178933c9a653..8b15f5140900 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeDestinationTest.java @@ -8,12 +8,12 @@ import static org.junit.jupiter.params.provider.Arguments.arguments; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.base.DestinationConfig; +import io.airbyte.cdk.integrations.base.SerializedAirbyteMessageConsumer; +import io.airbyte.cdk.integrations.destination_async.AsyncStreamConsumer; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.base.DestinationConfig; -import io.airbyte.integrations.base.SerializedAirbyteMessageConsumer; import io.airbyte.integrations.destination.snowflake.SnowflakeDestination.DestinationType; -import io.airbyte.integrations.destination_async.AsyncStreamConsumer; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConnectorSpecification; import java.util.regex.Matcher; diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperationsTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperationsTest.java index 66f0ff677898..1d8f16da008d 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperationsTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeInternalStagingSqlOperationsTest.java @@ -7,8 +7,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import io.airbyte.cdk.integrations.base.DestinationConfig; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.DestinationConfig; import java.util.List; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -55,7 +55,7 @@ void listStage() { void copyIntoTmpTableFromStage() { final String expectedQuery = """ - COPY INTO schemaName.tableName FROM '@stageName/stagePath/2022/' + COPY INTO "schemaName"."tableName" FROM '@stageName/stagePath/2022/' file_format = ( type = csv compression = auto diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsTest.java index d02f71ec95c5..66fa8866f233 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsTest.java @@ -11,12 +11,12 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.DestinationConfig; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination_async.partial_messages.PartialAirbyteMessage; import io.airbyte.commons.functional.CheckedConsumer; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.DestinationConfig; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.sql.SQLException; import java.util.List; import org.junit.jupiter.api.BeforeEach; @@ -37,15 +37,21 @@ public void setup() { @Test void createTableQuery() { - String expectedQuery = String.format( + final String expectedQuery = String.format( """ - CREATE TABLE IF NOT EXISTS %s.%s ( - %s VARCHAR PRIMARY KEY, - %s VARIANT, - %s TIMESTAMP WITH TIME ZONE DEFAULT current_timestamp() + CREATE TABLE IF NOT EXISTS "%s"."%s" ( + "%s" VARCHAR PRIMARY KEY, + "%s" TIMESTAMP WITH TIME ZONE DEFAULT current_timestamp(), + "%s" TIMESTAMP WITH TIME ZONE DEFAULT NULL, + "%s" VARIANT ) data_retention_time_in_days = 0;""", - SCHEMA_NAME, TABLE_NAME, JavaBaseConstants.COLUMN_NAME_AB_ID, JavaBaseConstants.COLUMN_NAME_DATA, JavaBaseConstants.COLUMN_NAME_EMITTED_AT); - String actualQuery = snowflakeSqlOperations.createTableQuery(db, SCHEMA_NAME, TABLE_NAME); + SCHEMA_NAME, + TABLE_NAME, + JavaBaseConstants.COLUMN_NAME_AB_RAW_ID, + JavaBaseConstants.COLUMN_NAME_AB_EXTRACTED_AT, + JavaBaseConstants.COLUMN_NAME_AB_LOADED_AT, + JavaBaseConstants.COLUMN_NAME_DATA); + final String actualQuery = snowflakeSqlOperations.createTableQuery(db, SCHEMA_NAME, TABLE_NAME); assertEquals(expectedQuery, actualQuery); } @@ -57,7 +63,7 @@ void isSchemaExists() throws Exception { @Test void insertRecordsInternal() throws SQLException { - snowflakeSqlOperations.insertRecordsInternal(db, List.of(new AirbyteRecordMessage()), SCHEMA_NAME, TABLE_NAME); + snowflakeSqlOperations.insertRecordsInternal(db, List.of(new PartialAirbyteMessage()), SCHEMA_NAME, TABLE_NAME); verify(db, times(1)).execute(any(CheckedConsumer.class)); } diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsThrowConfigExceptionTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsThrowConfigExceptionTest.java index cb131b139189..06374e1fe613 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsThrowConfigExceptionTest.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/SnowflakeSqlOperationsThrowConfigExceptionTest.java @@ -7,10 +7,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.DestinationConfig; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.DestinationConfig; import java.sql.SQLException; import java.util.List; import java.util.stream.Stream; diff --git a/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorTest.java b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorTest.java new file mode 100644 index 000000000000..534cf31251e0 --- /dev/null +++ b/airbyte-integrations/connectors/destination-snowflake/src/test/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGeneratorTest.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.destination.snowflake.typing_deduping; + +import static java.util.Collections.emptyList; +import static org.junit.jupiter.api.Assertions.assertAll; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; +import io.airbyte.integrations.base.destination.typing_deduping.CatalogParser; +import io.airbyte.integrations.base.destination.typing_deduping.ColumnId; +import io.airbyte.integrations.base.destination.typing_deduping.StreamConfig; +import io.airbyte.integrations.base.destination.typing_deduping.StreamId; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.SyncMode; +import java.util.LinkedHashMap; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +public class SnowflakeSqlGeneratorTest { + + private final SnowflakeSqlGenerator generator = new SnowflakeSqlGenerator(); + + @Test + void columnNameSpecialCharacterHandling() { + assertAll( + // If a ${ is present, then we should replace all of $, {, and } with underscores + () -> assertEquals( + new ColumnId( + "__FOO_", + "${foo}", + "__FOO_"), + generator.buildColumnId("${foo}")), + // But normally, we should leave those characters untouched. + () -> assertEquals( + new ColumnId( + "{FO$O}", + "{fo$o}", + "{FO$O}"), + generator.buildColumnId("{fo$o}"))); + } + + /** + * Similar to {@link #columnNameSpecialCharacterHandling()}, but for stream name/namespace + */ + @Test + void streamNameSpecialCharacterHandling() { + assertAll( + () -> assertEquals( + new StreamId( + "__FOO_", + "__BAR_", + "airbyte_internal", + "__foo__raw__stream___bar_", + "${foo}", + "${bar}"), + generator.buildStreamId("${foo}", "${bar}", "airbyte_internal")), + () -> assertEquals( + new StreamId( + "{FO$O}", + "{BA$R}", + "airbyte_internal", + "{fo$o}_raw__stream_{ba$r}", + "{fo$o}", + "{ba$r}"), + generator.buildStreamId("{fo$o}", "{ba$r}", "airbyte_internal"))); + } + + @Test + void columnCollision() { + final CatalogParser parser = new CatalogParser(generator); + assertEquals( + new StreamConfig( + new StreamId("BAR", "FOO", "airbyte_internal", "bar_raw__stream_foo", "bar", "foo"), + SyncMode.INCREMENTAL, + DestinationSyncMode.APPEND, + emptyList(), + Optional.empty(), + new LinkedHashMap<>() { + + { + put(new ColumnId("_CURRENT_DATE", "CURRENT_DATE", "_CURRENT_DATE"), AirbyteProtocolType.STRING); + put(new ColumnId("_CURRENT_DATE_1", "current_date", "_CURRENT_DATE_1"), AirbyteProtocolType.INTEGER); + } + + }), + parser.toStreamConfig(new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(new AirbyteStream() + .withName("foo") + .withNamespace("bar") + .withJsonSchema(Jsons.deserialize( + """ + { + "type": "object", + "properties": { + "CURRENT_DATE": {"type": "string"}, + "current_date": {"type": "integer"} + } + } + """))))); + } + +} diff --git a/airbyte-integrations/connectors/destination-sqlite/README.md b/airbyte-integrations/connectors/destination-sqlite/README.md index 04813a44bcb6..18e9e61a6ca2 100644 --- a/airbyte-integrations/connectors/destination-sqlite/README.md +++ b/airbyte-integrations/connectors/destination-sqlite/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:destination-sqlite:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/destinations/sqlite) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_sqlite/spec.json` file. @@ -54,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/destination-sqlite:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=destination-sqlite build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:destination-sqlite:airbyteDocker +An image will be built with the tag `airbyte/destination-sqlite:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/destination-sqlite:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,38 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-sqlite:dev check # messages.jsonl is a file containing line-separated JSON representing AirbyteMessages cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-sqlite:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=destination-sqlite test ``` -#### Acceptance Tests -Coming soon: -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:destination-sqlite:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:destination-sqlite:integrationTest -``` +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -116,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-sqlite test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/sqlite.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-sqlite/build.gradle b/airbyte-integrations/connectors/destination-sqlite/build.gradle deleted file mode 100644 index 75500d0b42c7..000000000000 --- a/airbyte-integrations/connectors/destination-sqlite/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'destination_sqlite' -} diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/.dockerignore b/airbyte-integrations/connectors/destination-starburst-galaxy/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/Dockerfile b/airbyte-integrations/connectors/destination-starburst-galaxy/Dockerfile deleted file mode 100644 index 7df71dd5d613..000000000000 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte -ENV APPLICATION destination-starburst-galaxy - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte -ENV APPLICATION destination-starburst-galaxy - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.0.1 -LABEL io.airbyte.name=airbyte/destination-starburst-galaxy diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/README.md b/airbyte-integrations/connectors/destination-starburst-galaxy/README.md index d8ec77b405d3..2125775075a9 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/README.md +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/README.md @@ -25,7 +25,7 @@ If you are an Airbyte core member, you must follow the [instructions](https://do Build the connector image with Gradle: ``` -./gradlew :airbyte-integrations:connectors:destination-starburst-galaxy:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-starburst-galaxy:buildConnectorImage ``` When building with Gradle, the Docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` labels in the Dockerfile. @@ -62,4 +62,4 @@ After you have implemented a feature, bug fix or enhancement, you must do the fo 2. Update the connector version by incrementing the value of the `io.airbyte.version` label in the Dockerfile by following the [SemVer](https://semver.org/) versioning rules. 3. Create a Pull Request. -Airbyte will review your PR and request any changes necessary to merge it into master. \ No newline at end of file +Airbyte will review your PR and request any changes necessary to merge it into master. diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/build.gradle b/airbyte-integrations/connectors/destination-starburst-galaxy/build.gradle index 449997bb4e6f..ffe2bf71cf6b 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/build.gradle +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/build.gradle @@ -1,21 +1,28 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.starburst_galaxy.StarburstGalaxyDestination' } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - implementation project(':airbyte-integrations:bases:bases-destination-jdbc') - implementation project(path: ':airbyte-db:db-lib') - implementation project(path: ':airbyte-integrations:bases:base-java-s3') implementation project(path: ':airbyte-integrations:connectors:destination-s3') implementation ('io.trino:trino-iceberg:411') {exclude group: 'commons-cli', module: 'commons-cli'} @@ -37,8 +44,5 @@ dependencies { implementation ('com.github.airbytehq:json-avro-converter:1.1.0') { exclude group: 'ch.qos.logback', module: 'logback-classic'} - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-starburst-galaxy') - implementation ('org.apache.parquet:parquet-avro:1.12.3') { exclude group: 'org.slf4j', module: 'slf4j-log4j12'} } diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/HadoopCatalogIcebergS3ParquetWriter.java b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/HadoopCatalogIcebergS3ParquetWriter.java index fcc679572dfb..3a0765ca3a89 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/HadoopCatalogIcebergS3ParquetWriter.java +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/HadoopCatalogIcebergS3ParquetWriter.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.starburst_galaxy; -import static io.airbyte.integrations.destination.s3.writer.BaseS3Writer.determineOutputFilename; +import static io.airbyte.cdk.integrations.destination.s3.writer.BaseS3Writer.determineOutputFilename; import static org.apache.hadoop.fs.s3a.Constants.ACCESS_KEY; import static org.apache.hadoop.fs.s3a.Constants.AWS_CREDENTIALS_PROVIDER; import static org.apache.hadoop.fs.s3a.Constants.SECRET_KEY; @@ -15,10 +15,10 @@ import static org.apache.iceberg.aws.AwsProperties.S3FILEIO_SECRET_ACCESS_KEY; import com.amazonaws.services.s3.AmazonS3; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.S3Format; -import io.airbyte.integrations.destination.s3.credential.S3AccessKeyCredentialConfig; -import io.airbyte.integrations.destination.s3.template.S3FilenameTemplateParameterObject; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3Format; +import io.airbyte.cdk.integrations.destination.s3.credential.S3AccessKeyCredentialConfig; +import io.airbyte.cdk.integrations.destination.s3.template.S3FilenameTemplateParameterObject; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.io.IOException; diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyBaseDestination.java b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyBaseDestination.java index d0456364ee0f..2015e8f11450 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyBaseDestination.java +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyBaseDestination.java @@ -4,21 +4,21 @@ package io.airbyte.integrations.destination.starburst_galaxy; -import static io.airbyte.db.factory.DatabaseDriver.STARBURST; -import static io.airbyte.integrations.destination.jdbc.copy.CopyConsumerFactory.create; +import static io.airbyte.cdk.db.factory.DatabaseDriver.STARBURST; +import static io.airbyte.cdk.integrations.destination.jdbc.copy.CopyConsumerFactory.create; import static io.airbyte.integrations.destination.starburst_galaxy.StarburstGalaxyConstants.CATALOG_SCHEMA; import static io.airbyte.integrations.destination.starburst_galaxy.StarburstGalaxyConstants.STARBURST_GALAXY_DRIVER_CLASS; import static io.airbyte.integrations.destination.starburst_galaxy.StarburstGalaxyDestinationConfig.get; import static java.lang.String.format; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.copy.CopyDestination; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.copy.CopyDestination; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import java.util.function.Consumer; diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyConstants.java b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyConstants.java index 5e4d77c18f59..839e67afb03b 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyConstants.java +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyConstants.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.starburst_galaxy; -import static io.airbyte.db.factory.DatabaseDriver.STARBURST; +import static io.airbyte.cdk.db.factory.DatabaseDriver.STARBURST; public final class StarburstGalaxyConstants { diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestination.java b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestination.java index 63a65e08c6b8..139d04be0edf 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestination.java +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestination.java @@ -7,9 +7,9 @@ import static io.airbyte.integrations.destination.starburst_galaxy.StarburstGalaxyStagingStorageType.S3; import com.google.common.collect.ImmutableMap; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.destination.jdbc.copy.SwitchingDestination; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.destination.jdbc.copy.SwitchingDestination; import java.io.Closeable; import java.sql.DriverManager; diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestinationResolver.java b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestinationResolver.java index 9d823ec0eea9..aa7364a20511 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestinationResolver.java +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestinationResolver.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.starburst_galaxy; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_NAME; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_NAME; import static io.airbyte.integrations.destination.starburst_galaxy.StarburstGalaxyConstants.STAGING_OBJECT_STORE; import static io.airbyte.integrations.destination.starburst_galaxy.StarburstGalaxyStagingStorageType.S3; diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyNameTransformer.java b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyNameTransformer.java index 1337156046b3..d50133ac88f0 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyNameTransformer.java +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyNameTransformer.java @@ -6,7 +6,7 @@ import static java.util.Locale.ENGLISH; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; public class StarburstGalaxyNameTransformer extends StandardNameTransformer { diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3Destination.java b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3Destination.java index d6f0b535e43d..571e0b2e851a 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3Destination.java +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3Destination.java @@ -4,10 +4,10 @@ package io.airbyte.integrations.destination.starburst_galaxy; -import static io.airbyte.integrations.destination.s3.S3BaseChecks.attemptS3WriteAndDelete; +import static io.airbyte.cdk.integrations.destination.s3.S3BaseChecks.attemptS3WriteAndDelete; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.S3StorageOperations; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3StorageOperations; public class StarburstGalaxyS3Destination extends StarburstGalaxyBaseDestination { diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3StagingStorageConfig.java b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3StagingStorageConfig.java index cb338ef3bc69..3c403cce3d18 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3StagingStorageConfig.java +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3StagingStorageConfig.java @@ -4,16 +4,16 @@ package io.airbyte.integrations.destination.starburst_galaxy; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_ACCESS_KEY_ID; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_NAME; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_PATH; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_REGION; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_SECRET_ACCESS_KEY; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_ACCESS_KEY_ID; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_NAME; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_PATH; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_REGION; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_SECRET_ACCESS_KEY; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.parquet.S3ParquetFormatConfig; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.parquet.S3ParquetFormatConfig; public class StarburstGalaxyS3StagingStorageConfig extends StarburstGalaxyStagingStorageConfig { diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3StreamCopier.java b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3StreamCopier.java index 04742225219e..005316e76532 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3StreamCopier.java +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3StreamCopier.java @@ -15,14 +15,14 @@ import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.S3Object; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopier; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; -import io.airbyte.integrations.destination.s3.avro.AvroConstants; -import io.airbyte.integrations.destination.s3.avro.AvroRecordFactory; -import io.airbyte.integrations.destination.s3.avro.JsonToAvroSchemaConverter; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopier; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.avro.AvroConstants; +import io.airbyte.cdk.integrations.destination.s3.avro.AvroRecordFactory; +import io.airbyte.cdk.integrations.destination.s3.avro.JsonToAvroSchemaConverter; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.io.IOException; diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3StreamCopierFactory.java b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3StreamCopierFactory.java index 542a04381a83..43b17b4b06af 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3StreamCopierFactory.java +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3StreamCopierFactory.java @@ -4,14 +4,14 @@ package io.airbyte.integrations.destination.starburst_galaxy; -import static io.airbyte.integrations.destination.jdbc.copy.StreamCopierFactory.getSchema; +import static io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopierFactory.getSchema; import com.amazonaws.services.s3.AmazonS3; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopier; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopier; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.sql.Timestamp; diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxySqlOperations.java b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxySqlOperations.java index c43ae602f639..eb35eab5cb7b 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxySqlOperations.java +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxySqlOperations.java @@ -6,9 +6,9 @@ import static java.lang.String.format; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.jdbc.JdbcSqlOperations; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.jdbc.JdbcSqlOperations; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.sql.SQLException; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyStagingStorageConfig.java b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyStagingStorageConfig.java index 9dbeafad4c77..75144131b29a 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyStagingStorageConfig.java +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyStagingStorageConfig.java @@ -9,7 +9,7 @@ import static org.slf4j.LoggerFactory.getLogger; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; import org.slf4j.Logger; public abstract class StarburstGalaxyStagingStorageConfig { diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyStreamCopier.java b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyStreamCopier.java index b622ec8cfd7c..240e70061e3a 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyStreamCopier.java +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyStreamCopier.java @@ -12,10 +12,10 @@ import static java.util.Locale.ENGLISH; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.SqlOperations; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopier; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.SqlOperations; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopier; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; import java.io.IOException; diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyStreamCopierFactory.java b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyStreamCopierFactory.java index fe8959486775..d108d2e7c104 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyStreamCopierFactory.java +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/main/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyStreamCopierFactory.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.starburst_galaxy; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopierFactory; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopierFactory; public interface StarburstGalaxyStreamCopierFactory extends StreamCopierFactory { diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestinationAcceptanceTest.java index 8e3cf6fea561..4e90123091ef 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestinationAcceptanceTest.java @@ -4,11 +4,11 @@ package io.airbyte.integrations.destination.starburst_galaxy; +import static io.airbyte.cdk.db.factory.DSLContextFactory.create; +import static io.airbyte.cdk.db.jdbc.JdbcUtils.getDefaultJSONFormat; +import static io.airbyte.cdk.integrations.base.JavaBaseConstants.COLUMN_NAME_EMITTED_AT; +import static io.airbyte.cdk.integrations.destination.s3.util.AvroRecordHelper.pruneAirbyteJson; import static io.airbyte.commons.json.Jsons.deserialize; -import static io.airbyte.db.factory.DSLContextFactory.create; -import static io.airbyte.db.jdbc.JdbcUtils.getDefaultJSONFormat; -import static io.airbyte.integrations.base.JavaBaseConstants.COLUMN_NAME_EMITTED_AT; -import static io.airbyte.integrations.destination.s3.util.AvroRecordHelper.pruneAirbyteJson; import static io.airbyte.integrations.destination.starburst_galaxy.StarburstGalaxyBaseDestination.getGalaxyConnectionString; import static io.airbyte.integrations.destination.starburst_galaxy.StarburstGalaxyConstants.STARBURST_GALAXY_DRIVER_CLASS; import static io.airbyte.protocol.models.v0.AirbyteMessage.Type.RECORD; @@ -25,16 +25,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.ContextQueryFunction; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.copy.StreamCopierFactory; +import io.airbyte.cdk.integrations.destination.s3.avro.JsonFieldNameUpdater; +import io.airbyte.cdk.integrations.destination.s3.util.AvroRecordHelper; +import io.airbyte.cdk.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.ContextQueryFunction; -import io.airbyte.db.Database; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.copy.StreamCopierFactory; -import io.airbyte.integrations.destination.s3.avro.JsonFieldNameUpdater; -import io.airbyte.integrations.destination.s3.util.AvroRecordHelper; -import io.airbyte.integrations.standardtest.destination.DestinationAcceptanceTest; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStream; diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3DestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3DestinationAcceptanceTest.java index 459b2580eaab..fe1a9ad136a3 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3DestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test-integration/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyS3DestinationAcceptanceTest.java @@ -4,9 +4,9 @@ package io.airbyte.integrations.destination.starburst_galaxy; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_ACCESS_KEY_ID; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_PATH; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_SECRET_ACCESS_KEY; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_ACCESS_KEY_ID; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_PATH; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_SECRET_ACCESS_KEY; import static io.airbyte.integrations.destination.starburst_galaxy.StarburstGalaxyConstants.CATALOG_SCHEMA; import static io.airbyte.integrations.destination.starburst_galaxy.StarburstGalaxyConstants.STAGING_OBJECT_STORE; import static io.airbyte.integrations.destination.starburst_galaxy.StarburstGalaxyDestinationConfig.get; @@ -16,9 +16,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.destination.s3.S3DestinationConfig; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.destination.s3.S3DestinationConfig; import java.nio.file.Path; import java.util.HashSet; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestinationConfigTest.java b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestinationConfigTest.java index 97d9dbfeb027..8964fdd5456e 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestinationConfigTest.java +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestinationConfigTest.java @@ -4,12 +4,12 @@ package io.airbyte.integrations.destination.starburst_galaxy; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_ACCESS_KEY_ID; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_NAME; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_PATH; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_REGION; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_SECRET_ACCESS_KEY; import static io.airbyte.commons.jackson.MoreMappers.initMapper; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_ACCESS_KEY_ID; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_NAME; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_PATH; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_REGION; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_SECRET_ACCESS_KEY; import static io.airbyte.integrations.destination.starburst_galaxy.StarburstGalaxyConstants.ACCEPT_TERMS; import static io.airbyte.integrations.destination.starburst_galaxy.StarburstGalaxyConstants.CATALOG; import static io.airbyte.integrations.destination.starburst_galaxy.StarburstGalaxyConstants.CATALOG_SCHEMA; diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestinationResolverTest.java b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestinationResolverTest.java index 024c10fbdd52..8206c729f161 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestinationResolverTest.java +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyDestinationResolverTest.java @@ -4,9 +4,9 @@ package io.airbyte.integrations.destination.starburst_galaxy; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_NAME; import static io.airbyte.commons.jackson.MoreMappers.initMapper; import static io.airbyte.commons.resources.MoreResources.readResource; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_NAME; import static io.airbyte.integrations.destination.starburst_galaxy.StarburstGalaxyConstants.STAGING_OBJECT_STORE; import static io.airbyte.integrations.destination.starburst_galaxy.StarburstGalaxyDestinationResolver.getStagingStorageType; import static io.airbyte.integrations.destination.starburst_galaxy.StarburstGalaxyDestinationResolver.isS3StagingStore; diff --git a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyStagingStorageConfigTest.java b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyStagingStorageConfigTest.java index 298ed46b305b..46578d4919f9 100644 --- a/airbyte-integrations/connectors/destination-starburst-galaxy/src/test/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyStagingStorageConfigTest.java +++ b/airbyte-integrations/connectors/destination-starburst-galaxy/src/test/java/io/airbyte/integrations/destination/starburst_galaxy/StarburstGalaxyStagingStorageConfigTest.java @@ -4,12 +4,12 @@ package io.airbyte.integrations.destination.starburst_galaxy; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_ACCESS_KEY_ID; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_NAME; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_PATH; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_REGION; +import static io.airbyte.cdk.integrations.destination.s3.constant.S3Constants.S_3_SECRET_ACCESS_KEY; import static io.airbyte.commons.jackson.MoreMappers.initMapper; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_ACCESS_KEY_ID; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_NAME; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_PATH; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_BUCKET_REGION; -import static io.airbyte.integrations.destination.s3.constant.S3Constants.S_3_SECRET_ACCESS_KEY; import static io.airbyte.integrations.destination.starburst_galaxy.StarburstGalaxyConstants.OBJECT_STORE_TYPE; import static io.airbyte.integrations.destination.starburst_galaxy.StarburstGalaxyStagingStorageConfig.getStarburstGalaxyStagingStorageConfig; import static org.junit.jupiter.api.Assertions.assertNotNull; diff --git a/airbyte-integrations/connectors/destination-teradata/.dockerignore b/airbyte-integrations/connectors/destination-teradata/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-teradata/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-teradata/Dockerfile b/airbyte-integrations/connectors/destination-teradata/Dockerfile deleted file mode 100644 index 85b0198c5758..000000000000 --- a/airbyte-integrations/connectors/destination-teradata/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte -ENV APPLICATION destination-teradata - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte -ENV APPLICATION destination-teradata - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.2 -LABEL io.airbyte.name=airbyte/destination-teradata diff --git a/airbyte-integrations/connectors/destination-teradata/README.md b/airbyte-integrations/connectors/destination-teradata/README.md index 19688729c4d4..3bcb00e79722 100644 --- a/airbyte-integrations/connectors/destination-teradata/README.md +++ b/airbyte-integrations/connectors/destination-teradata/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-teradata:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-teradata:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-teradata:dev`. the Dockerfile. #### Run @@ -61,8 +62,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-teradata test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/teradata.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-teradata/build.gradle b/airbyte-integrations/connectors/destination-teradata/build.gradle index 13fab2b59424..0f1f2ebe89d4 100644 --- a/airbyte-integrations/connectors/destination-teradata/build.gradle +++ b/airbyte-integrations/connectors/destination-teradata/build.gradle @@ -1,25 +1,28 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.teradata.TeradataDestination' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:bases-destination-jdbc') implementation 'com.teradata.jdbc:terajdbc4:17.20.00.12' - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-teradata') } diff --git a/airbyte-integrations/connectors/destination-teradata/metadata.yaml b/airbyte-integrations/connectors/destination-teradata/metadata.yaml index 8df89e805032..91738294ff4e 100644 --- a/airbyte-integrations/connectors/destination-teradata/metadata.yaml +++ b/airbyte-integrations/connectors/destination-teradata/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 58e6f9da-904e-11ed-a1eb-0242ac120002 - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 dockerRepository: airbyte/destination-teradata githubIssueLabel: destination-teradata icon: teradata.svg @@ -15,6 +15,7 @@ data: enabled: true releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/destinations/teradata + supportsDbt: true tags: - language:java ab_internal: diff --git a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java index 3c9971d3a6a8..37fd84a973a6 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java +++ b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataDestination.java @@ -6,12 +6,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import java.io.IOException; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; diff --git a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java index 76df7bf8fbeb..85cf7dc27d53 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java +++ b/airbyte-integrations/connectors/destination-teradata/src/main/java/io/airbyte/integrations/destination/teradata/TeradataSqlOperations.java @@ -4,11 +4,11 @@ package io.airbyte.integrations.destination.teradata; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.jdbc.JdbcSqlOperations; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.AirbyteTraceMessageUtility; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.jdbc.JdbcSqlOperations; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.sql.PreparedStatement; import java.sql.SQLException; diff --git a/airbyte-integrations/connectors/destination-teradata/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-teradata/src/main/resources/spec.json index 25e54e8f2b0c..d1653e702a35 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-teradata/src/main/resources/spec.json @@ -2,7 +2,7 @@ "documentationUrl": "https://docs.airbyte.io/integrations/destinations/teradata", "supportsIncremental": false, "supportsNormalization": false, - "supportsDBT": false, + "supportsDBT": true, "supported_destination_sync_modes": ["overwrite", "append"], "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", diff --git a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java index 1386c172d7bd..c3fa9274ad98 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-teradata/src/test-integration/java/io/airbyte/integrations/destination/teradata/TeradataDestinationAcceptanceTest.java @@ -6,21 +6,21 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcSourceOperations; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; import io.airbyte.commons.string.Strings; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcSourceOperations; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.AirbyteTraceMessageUtility; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; import io.airbyte.integrations.destination.teradata.envclient.TeradataHttpClient; import io.airbyte.integrations.destination.teradata.envclient.dto.*; import io.airbyte.integrations.destination.teradata.envclient.exception.BaseException; -import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; import java.nio.file.Files; import java.nio.file.Paths; import java.sql.SQLException; diff --git a/airbyte-integrations/connectors/destination-teradata/src/test/java/io/airbyte/integrations/destination/teradata/TeradataDestinationTest.java b/airbyte-integrations/connectors/destination-teradata/src/test/java/io/airbyte/integrations/destination/teradata/TeradataDestinationTest.java index 2fb453e3162f..07bfc57c34c9 100644 --- a/airbyte-integrations/connectors/destination-teradata/src/test/java/io/airbyte/integrations/destination/teradata/TeradataDestinationTest.java +++ b/airbyte-integrations/connectors/destination-teradata/src/test/java/io/airbyte/integrations/destination/teradata/TeradataDestinationTest.java @@ -8,9 +8,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; -import io.airbyte.db.jdbc.JdbcUtils; import java.util.Map; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/airbyte-integrations/connectors/destination-tidb/.dockerignore b/airbyte-integrations/connectors/destination-tidb/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-tidb/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-tidb/Dockerfile b/airbyte-integrations/connectors/destination-tidb/Dockerfile deleted file mode 100644 index 35f6f8bd5d2f..000000000000 --- a/airbyte-integrations/connectors/destination-tidb/Dockerfile +++ /dev/null @@ -1,52 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -RUN yum install -y python3 python3-devel jq sshpass git gcc-c++ && yum clean all && \ - alternatives --install /usr/bin/python python /usr/bin/python3 60 && \ - python -m ensurepip --upgrade && \ - # these two lines are a workaround for https://github.com/yaml/pyyaml/issues/601 - pip3 install wheel && \ - pip3 install "Cython<3.0" "pyyaml==5.4" --no-build-isolation && \ - pip3 install dbt-tidb==1.0.1 - -# Luckily, none of normalization's files conflict with destination-tidb's files :) -# We don't enforce that in any way, but hopefully we're only living in this state for a short time. -COPY --from=airbyte/normalization-tidb:dev /airbyte /airbyte -# Install python dependencies -WORKDIR /airbyte/base_python_structs -RUN pip3 install . -WORKDIR /airbyte/normalization_code -RUN pip3 install . -WORKDIR /airbyte/normalization_code/dbt-template/ -# Download external dbt dependencies -# amazon linux 2 isn't compatible with urllib3 2.x, so force 1.26.15 -RUN pip3 install "urllib3<2" -RUN dbt deps - -WORKDIR /airbyte -ENV APPLICATION destination-tidb -ENV AIRBYTE_NORMALIZATION_INTEGRATION tidb - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte -ENV APPLICATION destination-tidb - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.4 -LABEL io.airbyte.name=airbyte/destination-tidb - -ENV AIRBYTE_ENTRYPOINT "/airbyte/run_with_normalization.sh" -ENTRYPOINT ["/airbyte/run_with_normalization.sh"] diff --git a/airbyte-integrations/connectors/destination-tidb/README.md b/airbyte-integrations/connectors/destination-tidb/README.md index 237ce55f63c4..0672e49025a0 100644 --- a/airbyte-integrations/connectors/destination-tidb/README.md +++ b/airbyte-integrations/connectors/destination-tidb/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-tidb:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-tidb:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-tidb:dev`. the Dockerfile. #### Run @@ -61,8 +62,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-tidb test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/tidb.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-tidb/build.gradle b/airbyte-integrations/connectors/destination-tidb/build.gradle index 674e4473783e..e0845bc0ee01 100644 --- a/airbyte-integrations/connectors/destination-tidb/build.gradle +++ b/airbyte-integrations/connectors/destination-tidb/build.gradle @@ -1,30 +1,31 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.tidb.TiDBDestination' } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:bases-destination-jdbc') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation 'mysql:mysql-connector-java:8.0.30' - implementation libs.connectors.testcontainers.tidb - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-tidb') - - integrationTestJavaImplementation libs.connectors.testcontainers.tidb - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-normalization').airbyteDocker.outputs) -} + testImplementation libs.testcontainers.tidb -tasks.named("airbyteDocker") { - dependsOn project(':airbyte-integrations:bases:base-normalization').airbyteDockerTiDB + integrationTestJavaImplementation libs.testcontainers.tidb } diff --git a/airbyte-integrations/connectors/destination-tidb/src/main/java/io/airbyte/integrations/destination/tidb/TiDBDestination.java b/airbyte-integrations/connectors/destination-tidb/src/main/java/io/airbyte/integrations/destination/tidb/TiDBDestination.java index 3a61b56a65b7..a4da8be25ac3 100644 --- a/airbyte-integrations/connectors/destination-tidb/src/main/java/io/airbyte/integrations/destination/tidb/TiDBDestination.java +++ b/airbyte-integrations/connectors/destination-tidb/src/main/java/io/airbyte/integrations/destination/tidb/TiDBDestination.java @@ -6,15 +6,15 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.Destination; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination; +import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.Destination; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.ssh.SshWrappedDestination; -import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import java.util.Map; import javax.sql.DataSource; diff --git a/airbyte-integrations/connectors/destination-tidb/src/main/java/io/airbyte/integrations/destination/tidb/TiDBSQLNameTransformer.java b/airbyte-integrations/connectors/destination-tidb/src/main/java/io/airbyte/integrations/destination/tidb/TiDBSQLNameTransformer.java index ab00efb728c5..5fd93204e645 100644 --- a/airbyte-integrations/connectors/destination-tidb/src/main/java/io/airbyte/integrations/destination/tidb/TiDBSQLNameTransformer.java +++ b/airbyte-integrations/connectors/destination-tidb/src/main/java/io/airbyte/integrations/destination/tidb/TiDBSQLNameTransformer.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.tidb; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; /** * TiDB has some limitations on identifier length. diff --git a/airbyte-integrations/connectors/destination-tidb/src/main/java/io/airbyte/integrations/destination/tidb/TiDBSqlOperations.java b/airbyte-integrations/connectors/destination-tidb/src/main/java/io/airbyte/integrations/destination/tidb/TiDBSqlOperations.java index 7b42f834dbb1..dc10af530a6c 100644 --- a/airbyte-integrations/connectors/destination-tidb/src/main/java/io/airbyte/integrations/destination/tidb/TiDBSqlOperations.java +++ b/airbyte-integrations/connectors/destination-tidb/src/main/java/io/airbyte/integrations/destination/tidb/TiDBSqlOperations.java @@ -6,10 +6,10 @@ import com.fasterxml.jackson.databind.JsonNode; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.destination.jdbc.JdbcSqlOperations; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.JdbcSqlOperations; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.io.File; import java.io.IOException; diff --git a/airbyte-integrations/connectors/destination-tidb/src/test-integration/java/io/airbyte/integrations/destination/tidb/TiDBDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-tidb/src/test-integration/java/io/airbyte/integrations/destination/tidb/TiDBDestinationAcceptanceTest.java index 608c2ddbd2a5..59af74411461 100644 --- a/airbyte-integrations/connectors/destination-tidb/src/test-integration/java/io/airbyte/integrations/destination/tidb/TiDBDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-tidb/src/test-integration/java/io/airbyte/integrations/destination/tidb/TiDBDestinationAcceptanceTest.java @@ -6,16 +6,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; -import io.airbyte.integrations.util.HostPortResolver; import java.sql.SQLException; import java.util.HashSet; import java.util.List; diff --git a/airbyte-integrations/connectors/destination-tidb/src/test-integration/java/io/airbyte/integrations/destination/tidb/TiDBTestDataComparator.java b/airbyte-integrations/connectors/destination-tidb/src/test-integration/java/io/airbyte/integrations/destination/tidb/TiDBTestDataComparator.java index b2d9bf0b6bbf..cf65de1e2899 100644 --- a/airbyte-integrations/connectors/destination-tidb/src/test-integration/java/io/airbyte/integrations/destination/tidb/TiDBTestDataComparator.java +++ b/airbyte-integrations/connectors/destination-tidb/src/test-integration/java/io/airbyte/integrations/destination/tidb/TiDBTestDataComparator.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.tidb; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; diff --git a/airbyte-integrations/connectors/destination-tidb/src/test/java/io/airbyte/integrations/destination/tidb/TiDBDestinationTest.java b/airbyte-integrations/connectors/destination-tidb/src/test/java/io/airbyte/integrations/destination/tidb/TiDBDestinationTest.java index 56a648d3f650..c1fa5e99dd46 100644 --- a/airbyte-integrations/connectors/destination-tidb/src/test/java/io/airbyte/integrations/destination/tidb/TiDBDestinationTest.java +++ b/airbyte-integrations/connectors/destination-tidb/src/test/java/io/airbyte/integrations/destination/tidb/TiDBDestinationTest.java @@ -9,14 +9,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.AirbyteMessageConsumer; +import io.airbyte.cdk.integrations.base.Destination; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.AirbyteMessageConsumer; -import io.airbyte.integrations.base.Destination; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; diff --git a/airbyte-integrations/connectors/destination-timeplus/README.md b/airbyte-integrations/connectors/destination-timeplus/README.md index bfd9cf3a6680..6ba14518f631 100755 --- a/airbyte-integrations/connectors/destination-timeplus/README.md +++ b/airbyte-integrations/connectors/destination-timeplus/README.md @@ -2,7 +2,6 @@ This is the repository for the Timeplus destination connector, written in Python. For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/destinations/timeplus). -This connector is built by following the [Building a Python Destination](https://docs.airbyte.com/connector-development/tutorials/building-a-python-destination) ## Local development @@ -11,17 +10,21 @@ This connector is built by following the [Building a Python Destination](https:/ #### Minimum Python version required `= 3.9.0` #### Build & Activate Virtual Environment and install dependencies + From this connector directory, create a virtual environment: + ``` python -m venv .venv ``` This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your development environment of choice. To activate it from the terminal, run: + ``` source .venv/bin/activate pip install -r requirements.txt ``` + If you are in an IDE, follow your IDE's instructions to activate the virtualenv. Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is @@ -29,13 +32,8 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:destination-timeplus:build -``` - #### Create credentials + **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/destinations/timeplus) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_timeplus/spec.json` file. Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. @@ -45,6 +43,7 @@ See `integration_tests/sample_config.json` for a sample config file. and place them into `secrets/config.json`. ### Locally running the connector + ``` python main.py spec python main.py check --config secrets/config.json @@ -53,70 +52,57 @@ cat integration_tests/messages.jsonl | python main.py write --config secrets/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/destination-timeplus:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=destination-timeplus build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:destination-timeplus:airbyteDocker +An image will be built with the tag `airbyte/destination-timeplus:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/destination-timeplus:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run + Then run any of the connector commands as follows: + ``` docker run --rm airbyte/destination-timeplus:dev spec docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-timeplus:dev check --config /secrets/config.json # messages.jsonl is a file containing line-separated JSON representing AirbyteMessages cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-timeplus:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install pytest -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Coming soon: -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:destination-timeplus:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:destination-timeplus:integrationTest +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=destination-timeplus test ``` +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + ## Dependency Management + All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list + +- required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +- required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-timeplus test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/timeplus.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-timeplus/build.gradle b/airbyte-integrations/connectors/destination-timeplus/build.gradle deleted file mode 100755 index 022593982902..000000000000 --- a/airbyte-integrations/connectors/destination-timeplus/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'destination_timeplus' -} diff --git a/airbyte-integrations/connectors/destination-typesense/Dockerfile b/airbyte-integrations/connectors/destination-typesense/Dockerfile index f5036e89ab0c..26fa70977d17 100644 --- a/airbyte-integrations/connectors/destination-typesense/Dockerfile +++ b/airbyte-integrations/connectors/destination-typesense/Dockerfile @@ -34,5 +34,5 @@ COPY destination_typesense ./destination_typesense ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/destination-typesense diff --git a/airbyte-integrations/connectors/destination-typesense/README.md b/airbyte-integrations/connectors/destination-typesense/README.md index 01c677ffcf05..a1b61228a321 100644 --- a/airbyte-integrations/connectors/destination-typesense/README.md +++ b/airbyte-integrations/connectors/destination-typesense/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:destination-typesense:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/destinations/typesense) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_typesense/spec.json` file. @@ -54,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/destination-typesense:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=destination-typesense build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:destination-typesense:airbyteDocker +An image will be built with the tag `airbyte/destination-typesense:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/destination-typesense:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,38 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-typesense:dev che # messages.jsonl is a file containing line-separated JSON representing AirbyteMessages cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-typesense:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=destination-typesense test ``` -#### Acceptance Tests -Coming soon: -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:destination-typesense:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:destination-typesense:integrationTest -``` +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -116,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-typesense test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/typesense.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-typesense/build.gradle b/airbyte-integrations/connectors/destination-typesense/build.gradle deleted file mode 100644 index 01ad66a130f7..000000000000 --- a/airbyte-integrations/connectors/destination-typesense/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'destination_typesense' -} diff --git a/airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py b/airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py index 5e4de404d2af..a0d5a00e91e7 100644 --- a/airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py +++ b/airbyte-integrations/connectors/destination-typesense/destination_typesense/destination.py @@ -3,7 +3,6 @@ # -from logging import Logger from typing import Any, Iterable, Mapping from airbyte_cdk.destinations import Destination @@ -38,18 +37,19 @@ def write( pass client.collections.create({"name": steam_name, "fields": [{"name": ".*", "type": "auto"}]}) - writer = TypesenseWriter(client, steam_name, config.get("batch_size")) - for message in input_messages: - if message.type == Type.STATE: - writer.flush() - yield message - elif message.type == Type.RECORD: - writer.queue_write_operation(message.record.data) - else: - continue - writer.flush() - - def check(self, logger: Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: + writer = TypesenseWriter(client, config.get("batch_size")) + for message in input_messages: + if message.type == Type.STATE: + writer.flush() + yield message + elif message.type == Type.RECORD: + record = message.record + writer.queue_write_operation(record.stream, record.data) + else: + continue + writer.flush() + + def check(self, config: Mapping[str, Any]) -> AirbyteConnectionStatus: try: client = get_client(config=config) client.collections.create({"name": "_airbyte", "fields": [{"name": "title", "type": "string"}]}) diff --git a/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py b/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py index fd9c0e3b5868..54e85d5512b7 100644 --- a/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py +++ b/airbyte-integrations/connectors/destination-typesense/destination_typesense/writer.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from collections import defaultdict from collections.abc import Mapping from logging import getLogger from uuid import uuid4 @@ -12,17 +13,15 @@ class TypesenseWriter: - write_buffer = [] + write_buffer: list[tuple[str, Mapping]] = [] - def __init__(self, client: Client, steam_name: str, batch_size: int = None): + def __init__(self, client: Client, batch_size: int = None): self.client = client - self.steam_name = steam_name self.batch_size = batch_size or 10000 - def queue_write_operation(self, data: Mapping): + def queue_write_operation(self, stream_name: str, data: Mapping): random_key = str(uuid4()) - data_with_id = data if "id" in data else {**data, "id": random_key} - self.write_buffer.append(data_with_id) + self.write_buffer.append((stream_name, {**data, "id": random_key})) if len(self.write_buffer) == self.batch_size: self.flush() @@ -31,5 +30,11 @@ def flush(self): if buffer_size == 0: return logger.info(f"flushing {buffer_size} records") - self.client.collections[self.steam_name].documents.import_(self.write_buffer) + + grouped_by_stream: defaultdict[str, list[Mapping]] = defaultdict(list) + for stream, data in self.write_buffer: + grouped_by_stream[stream].append(data) + + for (stream, data) in grouped_by_stream.items(): + self.client.collections[stream].documents.import_(data) self.write_buffer.clear() diff --git a/airbyte-integrations/connectors/destination-typesense/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-typesense/integration_tests/integration_test.py index cb0d5aae3145..0b159f3eb857 100644 --- a/airbyte-integrations/connectors/destination-typesense/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-typesense/integration_tests/integration_test.py @@ -3,7 +3,6 @@ # import json -from logging import getLogger from typing import Any, Dict, Mapping import pytest @@ -33,13 +32,13 @@ def config_fixture() -> Mapping[str, Any]: def configured_catalog_fixture() -> ConfiguredAirbyteCatalog: stream_schema = {"type": "object", "properties": {"col1": {"type": "str"}, "col2": {"type": "integer"}}} - overwrite_stream = ConfiguredAirbyteStream( - stream=AirbyteStream(name="_airbyte", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), + overwrite_stream = lambda n: ConfiguredAirbyteStream( + stream=AirbyteStream(name=f"_airbyte_{n}", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.overwrite, ) - return ConfiguredAirbyteCatalog(streams=[overwrite_stream]) + return ConfiguredAirbyteCatalog(streams=[overwrite_stream(i) for i in range(2)]) @pytest.fixture(autouse=True) @@ -60,12 +59,12 @@ def client_fixture(config) -> Client: def test_check_valid_config(config: Mapping): - outcome = DestinationTypesense().check(getLogger("airbyte"), config) + outcome = DestinationTypesense().check(config) assert outcome.status == Status.SUCCEEDED def test_check_invalid_config(): - outcome = DestinationTypesense().check(getLogger("airbyte"), {"api_key": "not_a_real_key", "host": "https://www.fake.com"}) + outcome = DestinationTypesense().check({"api_key": "not_a_real_key", "host": "https://www.fake.com"}) assert outcome.status == Status.FAILED @@ -79,17 +78,18 @@ def _record(stream: str, str_value: str, int_value: int) -> AirbyteMessage: ) -def records_count(client: Client) -> int: - documents_results = client.index("_airbyte").get_documents() - return documents_results.total +def collection_size(client: Client, stream: str) -> int: + collection = client.collections[stream].retrieve() + return collection["num_documents"] def test_write(config: Mapping, configured_catalog: ConfiguredAirbyteCatalog, client: Client): - overwrite_stream = configured_catalog.streams[0].stream.name + configured_streams = list(map(lambda s: s.stream.name, configured_catalog.streams)) first_state_message = _state({"state": "1"}) - first_record_chunk = [_record(overwrite_stream, str(i), i) for i in range(2)] + first_record_chunk = [_record(stream, str(i), i) for i, stream in enumerate(configured_streams)] destination = DestinationTypesense() list(destination.write(config, configured_catalog, [*first_record_chunk, first_state_message])) - collection = client.collections["_airbyte"].retrieve() - assert collection["num_documents"] == 2 + + for stream in configured_streams: + assert collection_size(client, stream) == 1 diff --git a/airbyte-integrations/connectors/destination-typesense/metadata.yaml b/airbyte-integrations/connectors/destination-typesense/metadata.yaml index b3ee53fe4fee..ac5d918b2c45 100644 --- a/airbyte-integrations/connectors/destination-typesense/metadata.yaml +++ b/airbyte-integrations/connectors/destination-typesense/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 36be8dc6-9851-49af-b776-9d4c30e4ab6a - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 dockerRepository: airbyte/destination-typesense githubIssueLabel: destination-typesense icon: typesense.svg diff --git a/airbyte-integrations/connectors/destination-typesense/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-typesense/unit_tests/unit_test.py index ba065cb9fc02..a14d7f5b2abf 100644 --- a/airbyte-integrations/connectors/destination-typesense/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/destination-typesense/unit_tests/unit_test.py @@ -9,32 +9,32 @@ @patch("typesense.Client") def test_default_batch_size(client): - writer = TypesenseWriter(client, "steam_name") + writer = TypesenseWriter(client) assert writer.batch_size == 10000 @patch("typesense.Client") def test_empty_batch_size(client): - writer = TypesenseWriter(client, "steam_name", "") + writer = TypesenseWriter(client, "") assert writer.batch_size == 10000 @patch("typesense.Client") def test_custom_batch_size(client): - writer = TypesenseWriter(client, "steam_name", 9000) + writer = TypesenseWriter(client, 9000) assert writer.batch_size == 9000 @patch("typesense.Client") def test_queue_write_operation(client): - writer = TypesenseWriter(client, "steam_name") - writer.queue_write_operation({"a": "a"}) + writer = TypesenseWriter(client) + writer.queue_write_operation("stream_name", {"a": "a"}) assert len(writer.write_buffer) == 1 @patch("typesense.Client") def test_flush(client): - writer = TypesenseWriter(client, "steam_name") - writer.queue_write_operation({"a": "a"}) + writer = TypesenseWriter(client) + writer.queue_write_operation("stream_name", {"a": "a"}) writer.flush() - client.collections.__getitem__.assert_called_once_with("steam_name") + client.collections.__getitem__.assert_called_once_with("stream_name") diff --git a/airbyte-integrations/connectors/destination-vectara/.dockerignore b/airbyte-integrations/connectors/destination-vectara/.dockerignore new file mode 100644 index 000000000000..f784000e19e2 --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/.dockerignore @@ -0,0 +1,5 @@ +* +!Dockerfile +!main.py +!destination_vectara +!setup.py diff --git a/airbyte-integrations/connectors/destination-vectara/Dockerfile b/airbyte-integrations/connectors/destination-vectara/Dockerfile new file mode 100644 index 000000000000..cf50f0758e22 --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY destination_vectara ./destination_vectara + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/destination-vectara diff --git a/airbyte-integrations/connectors/destination-vectara/README.md b/airbyte-integrations/connectors/destination-vectara/README.md new file mode 100644 index 000000000000..2c68229551bc --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/README.md @@ -0,0 +1,123 @@ +# Vectara Destination + +This is the repository for the Vectara destination connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/destinations/vectara). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +From the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:destination-vectara:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/destinations/vectara) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_vectara/spec.json` file. +Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `destination vectara test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/destination-vectara:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:destination-vectara:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/destination-vectara:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-vectara:dev check --config /secrets/config.json +# messages.jsonl is a file containing line-separated JSON representing AirbyteMessages +cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-vectara:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Coming soon: + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:destination-vectara:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:destination-vectara:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/destination-vectara/destination_vectara/__init__.py b/airbyte-integrations/connectors/destination-vectara/destination_vectara/__init__.py new file mode 100644 index 000000000000..1bc53911e4ef --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/destination_vectara/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .destination import DestinationVectara + +__all__ = ["DestinationVectara"] diff --git a/airbyte-integrations/connectors/destination-vectara/destination_vectara/client.py b/airbyte-integrations/connectors/destination-vectara/destination_vectara/client.py new file mode 100644 index 000000000000..2c0a641edec2 --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/destination_vectara/client.py @@ -0,0 +1,198 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import datetime +import json +import traceback +from concurrent.futures import ThreadPoolExecutor +from typing import Any, Mapping + +import backoff +import requests +from destination_vectara.config import VectaraConfig + +METADATA_STREAM_FIELD = "_ab_stream" + + +def user_error(e: Exception) -> bool: + """ + Return True if this exception is caused by user error, False otherwise. + """ + if not isinstance(e, requests.exceptions.RequestException): + return False + return bool(e.response and 400 <= e.response.status_code < 500) + + +class VectaraClient: + + BASE_URL = "https://api.vectara.io/v1" + + def __init__(self, config: VectaraConfig): + if isinstance(config, dict): + config = VectaraConfig.parse_obj(config) + self.customer_id = config.customer_id + self.corpus_name = config.corpus_name + self.client_id = config.oauth2.client_id + self.client_secret = config.oauth2.client_secret + self.parallelize = config.parallelize + self.check() + + def check(self): + """ + Check for an existing corpus in Vectara. + If more than one exists - then return a message + If exactly one exists with this name - ensure that the corpus has the correct metadata fields, and use it. + If not, create it. + """ + try: + jwt_token = self._get_jwt_token() + if not jwt_token: + return "Unable to get JWT Token. Confirm your Client ID and Client Secret." + + list_corpora_response = self._request(endpoint="list-corpora", data={"numResults": 100, "filter": self.corpus_name}) + possible_corpora_ids_names_map = { + corpus.get("id"): corpus.get("name") + for corpus in list_corpora_response.get("corpus") + if corpus.get("name") == self.corpus_name + } + if len(possible_corpora_ids_names_map) > 1: + return f"Multiple Corpora exist with name {self.corpus_name}" + if len(possible_corpora_ids_names_map) == 1: + self.corpus_id = list(possible_corpora_ids_names_map.keys())[0] + else: + data = { + "corpus": { + "name": self.corpus_name, + "filterAttributes": [ + { + "name": METADATA_STREAM_FIELD, + "indexed": True, + "type": "FILTER_ATTRIBUTE_TYPE__TEXT", + "level": "FILTER_ATTRIBUTE_LEVEL__DOCUMENT", + }, + ], + } + } + + create_corpus_response = self._request(endpoint="create-corpus", data=data) + self.corpus_id = create_corpus_response.get("corpusId") + + except Exception as e: + return str(e) + "\n" + "".join(traceback.TracebackException.from_exception(e).format()) + + def _get_jwt_token(self): + """Connect to the server and get a JWT token.""" + token_endpoint = f"https://vectara-prod-{self.customer_id}.auth.us-west-2.amazoncognito.com/oauth2/token" + headers = { + "Content-Type": "application/x-www-form-urlencoded", + } + data = {"grant_type": "client_credentials", "client_id": self.client_id, "client_secret": self.client_secret} + + request_time = datetime.datetime.now().timestamp() + response = requests.request(method="POST", url=token_endpoint, headers=headers, data=data) + response_json = response.json() + + self.jwt_token = response_json.get("access_token") + self.jwt_token_expires_ts = request_time + response_json.get("expires_in") + return self.jwt_token + + @backoff.on_exception(backoff.expo, requests.exceptions.RequestException, max_tries=5, giveup=user_error) + def _request(self, endpoint: str, http_method: str = "POST", params: Mapping[str, Any] = None, data: Mapping[str, Any] = None): + + url = f"{self.BASE_URL}/{endpoint}" + + current_ts = datetime.datetime.now().timestamp() + if self.jwt_token_expires_ts - current_ts <= 60: + self._get_jwt_token() + + headers = { + "Content-Type": "application/json", + "Accept": "application/json", + "Authorization": f"Bearer {self.jwt_token}", + "customer-id": self.customer_id, + "X-source": "airbyte", + } + + response = requests.request(method=http_method, url=url, headers=headers, params=params, data=json.dumps(data)) + response.raise_for_status() + return response.json() + + def delete_doc_by_metadata(self, metadata_field_name, metadata_field_values): + document_ids = [] + for value in metadata_field_values: + data = { + "query": [ + { + "query": "", + "numResults": 100, + "corpusKey": [ + { + "customerId": self.customer_id, + "corpusId": self.corpus_id, + "metadataFilter": f"doc.{metadata_field_name} = '{value}'", + } + ], + } + ] + } + query_documents_response = self._request(endpoint="query", data=data) + document_ids.extend([document.get("id") for document in query_documents_response.get("responseSet")[0].get("document")]) + self.delete_docs_by_id(document_ids=document_ids) + + def delete_docs_by_id(self, document_ids): + for document_id in document_ids: + self._request( + endpoint="delete-doc", data={"customerId": self.customer_id, "corpusId": self.corpus_id, "documentId": document_id} + ) + + def index_document(self, document): + document_section, document_metadata, document_id = document + if len(document_section) == 0: + return None # Document is empty, so skip it + document_metadata = self._normalize(document_metadata) + data = { + "customerId": self.customer_id, + "corpusId": self.corpus_id, + "document": { + "documentId": document_id, + "metadataJson": json.dumps(document_metadata), + "section": [ + {"text": f"{section_key}: {section_value}"} + for section_key, section_value in document_section.items() + if section_key != METADATA_STREAM_FIELD + ], + }, + } + index_document_response = self._request(endpoint="index", data=data) + return index_document_response + + def index_documents(self, documents): + if self.parallelize: + with ThreadPoolExecutor() as executor: ### DEBUG remove max_workers limit + futures = [executor.submit(self.index_document, doc) for doc in documents] + for future in futures: + try: + response = future.result() + if response is None: + continue + assert ( + response.get("status").get("code") == "OK" + or response.get("status").get("statusDetail") == "Document should have at least one part." + ) + except AssertionError as e: + # Handle the assertion error + pass + else: + for doc in documents: + self.index_document(doc) + + def _normalize(self, metadata: dict) -> dict: + result = {} + for key, value in metadata.items(): + if isinstance(value, (str, int, float, bool)): + result[key] = value + else: + # JSON encode all other types + result[key] = json.dumps(value) + return result diff --git a/airbyte-integrations/connectors/destination-vectara/destination_vectara/config.py b/airbyte-integrations/connectors/destination-vectara/destination_vectara/config.py new file mode 100644 index 000000000000..942a465e9cd5 --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/destination_vectara/config.py @@ -0,0 +1,68 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import List, Optional + +from airbyte_cdk.utils.spec_schema_transformations import resolve_refs +from pydantic import BaseModel, Field + + +class OAuth2(BaseModel): + client_id: str = Field(..., title="OAuth Client ID", description="OAuth2.0 client id", order=0) + client_secret: str = Field(..., title="OAuth Client Secret", description="OAuth2.0 client secret", airbyte_secret=True, order=1) + + class Config: + title = "OAuth2.0 Credentials" + schema_extra = { + "description": "OAuth2.0 credentials used to authenticate admin actions (creating/deleting corpora)", + "group": "auth", + } + + +class VectaraConfig(BaseModel): + oauth2: OAuth2 + customer_id: str = Field( + ..., title="Customer ID", description="Your customer id as it is in the authenticaion url", order=2, group="account" + ) + corpus_name: str = Field(..., title="Corpus Name", description="The Name of Corpus to load data into", order=3, group="account") + + parallelize: Optional[bool] = Field( + default=False, + title="Parallelize", + description="Parallelize indexing into Vectara with multiple threads", + always_show=True, + group="account", + ) + + text_fields: Optional[List[str]] = Field( + default=[], + title="Text fields to index with Vectara", + description="List of fields in the record that should be in the section of the document. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered text fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.", + always_show=True, + examples=["text", "user.name", "users.*.name"], + ) + metadata_fields: Optional[List[str]] = Field( + default=[], + title="Fields to store as metadata", + description="List of fields in the record that should be stored as metadata. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered metadata fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. When specifying nested paths, all matching values are flattened into an array set to a field named by the path.", + always_show=True, + examples=["age", "user"], + ) + + class Config: + title = "Vectara Config" + schema_extra = { + "description": "Configuration to connect to the Vectara instance", + "groups": [ + {"id": "account", "title": "Account"}, + {"id": "auth", "title": "Authentication"}, + ], + } + + @classmethod + def schema(cls): + """we're overriding the schema classmethod to enable some post-processing""" + schema = super().schema() + schema = resolve_refs(schema) + return schema diff --git a/airbyte-integrations/connectors/destination-vectara/destination_vectara/destination.py b/airbyte-integrations/connectors/destination-vectara/destination_vectara/destination.py new file mode 100644 index 000000000000..8235a8f36b52 --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/destination_vectara/destination.py @@ -0,0 +1,94 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Iterable, Mapping + +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.destinations import Destination +from airbyte_cdk.models import ( + AirbyteConnectionStatus, + AirbyteMessage, + ConfiguredAirbyteCatalog, + ConnectorSpecification, + DestinationSyncMode, + Status, + Type, +) +from destination_vectara.client import VectaraClient +from destination_vectara.config import VectaraConfig +from destination_vectara.writer import VectaraWriter + + +class DestinationVectara(Destination): + def write( + self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] + ) -> Iterable[AirbyteMessage]: + + """ + Reads the input stream of messages, config, and catalog to write data to the destination. + + This method returns an iterable (typically a generator of AirbyteMessages via yield) containing state messages received + in the input message stream. Outputting a state message means that every AirbyteRecordMessage which came before it has been + successfully persisted to the destination. This is used to ensure fault tolerance in the case that a sync fails before fully completing, + then the source is given the last state message output from this method as the starting point of the next sync. + + :param config: dict of JSON configuration matching the configuration declared in spec.json + :param configured_catalog: The Configured Catalog describing the schema of the data being received and how it should be persisted in the + destination + :param input_messages: The stream of input messages received from the source + :return: Iterable of AirbyteStateMessages wrapped in AirbyteMessage structs + """ + + config_model = VectaraConfig.parse_obj(config) + writer = VectaraWriter( + client=VectaraClient(config_model), + text_fields=config_model.text_fields, + metadata_fields=config_model.metadata_fields, + catalog=configured_catalog, + ) + + writer.delete_streams_to_overwrite(catalog=configured_catalog) + + for message in input_messages: + if message.type == Type.STATE: + # Emitting a state message indicates that all records which came before it have been written to the destination. So we flush + # the queue to ensure writes happen, then output the state message to indicate it's safe to checkpoint state + writer.flush() + yield message + elif message.type == Type.RECORD: + record = message.record + writer.queue_write_operation(record) + else: + # ignore other message types for now + continue + + # Make sure to flush any records still in the queue + writer.flush() + + def check(self, logger: AirbyteLogger, config: VectaraConfig) -> AirbyteConnectionStatus: + """ + Tests if the input configuration can be used to successfully connect to the destination with the needed permissions + e.g: if a provided API token or password can be used to connect and write to the destination. + + :param logger: Logging object to display debug/info/error to the logs + (logs will not be accessible via airbyte UI if they are not passed to this logger) + :param config: Json object containing the configuration of this destination, content of this json is as specified in + the properties of the spec.json file + + :return: AirbyteConnectionStatus indicating a Success or Failure + """ + client = VectaraClient(config=config) + client_error = client.check() + if client_error: + return AirbyteConnectionStatus(status=Status.FAILED, message="\n".join([client_error])) + else: + return AirbyteConnectionStatus(status=Status.SUCCEEDED) + + def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: + return ConnectorSpecification( + documentationUrl="https://docs.airbyte.com/integrations/destinations/vectara", + supportsIncremental=True, + supported_destination_sync_modes=[DestinationSyncMode.overwrite, DestinationSyncMode.append], + connectionSpecification=VectaraConfig.schema(), + ) diff --git a/airbyte-integrations/connectors/destination-vectara/destination_vectara/writer.py b/airbyte-integrations/connectors/destination-vectara/destination_vectara/writer.py new file mode 100644 index 000000000000..f93f8707cc55 --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/destination_vectara/writer.py @@ -0,0 +1,119 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import uuid +from typing import Any, Dict, List, Mapping, Optional + +import dpath.util +from airbyte_cdk.models import AirbyteRecordMessage, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode +from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode +from airbyte_cdk.utils.traced_exception import AirbyteTracedException, FailureType +from destination_vectara.client import VectaraClient + +METADATA_STREAM_FIELD = "_ab_stream" + + +class VectaraWriter: + + write_buffer: List[Mapping[str, Any]] = [] + flush_interval = 1000 + + def __init__( + self, + client: VectaraClient, + text_fields: Optional[List[str]], + metadata_fields: Optional[List[str]], + catalog: ConfiguredAirbyteCatalog, + ): + self.client = client + self.text_fields = text_fields + self.metadata_fields = metadata_fields + self.streams = {f"{stream.stream.namespace}_{stream.stream.name}": stream for stream in catalog.streams} + self.ids_to_delete: List[str] = [] + + def delete_streams_to_overwrite(self, catalog: ConfiguredAirbyteCatalog) -> None: + streams_to_overwrite = [ + f"{stream.stream.namespace}_{stream.stream.name}" + for stream in catalog.streams + if stream.destination_sync_mode == DestinationSyncMode.overwrite + ] + if len(streams_to_overwrite): + self.client.delete_doc_by_metadata(metadata_field_name=METADATA_STREAM_FIELD, metadata_field_values=streams_to_overwrite) + + def _delete_documents_to_dedupe(self): + if len(self.ids_to_delete) > 0: + self.client.delete_docs_by_id(document_ids=self.ids_to_delete) + + def queue_write_operation(self, record: AirbyteRecordMessage) -> None: + """Adds messages to the write queue and flushes if the buffer is full""" + + stream_identifier = self._get_stream_id(record=record) + document_section = self._get_document_section(record=record) + document_metadata = self._get_document_metadata(record=record) + primary_key = self._get_record_primary_key(record=record) + + if primary_key: + document_id = f"Stream_{stream_identifier}_Key_{primary_key}" + if self.streams[stream_identifier].destination_sync_mode == DestinationSyncMode.append_dedup: + self.ids_to_delete.append(document_id) + else: + document_id = str(uuid.uuid4().int) + + self.write_buffer.append((document_section, document_metadata, document_id)) + if len(self.write_buffer) == self.flush_interval: + self.flush() + + def flush(self) -> None: + """Flush all documents in Queue to Vectara""" + self._delete_documents_to_dedupe() + self.client.index_documents(self.write_buffer) + self.write_buffer.clear() + self.ids_to_delete.clear() + + def _get_document_section(self, record: AirbyteRecordMessage): + relevant_fields = self._extract_relevant_fields(record, self.text_fields) + if len(relevant_fields) == 0: + text_fields = ", ".join(self.text_fields) if self.text_fields else "all fields" + raise AirbyteTracedException( + internal_message="No text fields found in record", + message=f"Record {str(record.data)[:250]}... does not contain any of the configured text fields: {text_fields}. Please check your processing configuration, there has to be at least one text field set in each record.", + failure_type=FailureType.config_error, + ) + document_section = relevant_fields + return document_section + + def _extract_relevant_fields(self, record: AirbyteRecordMessage, fields: Optional[List[str]]) -> Dict[str, Any]: + relevant_fields = {} + if fields and len(fields) > 0: + for field in fields: + values = dpath.util.values(record.data, field, separator=".") + if values and len(values) > 0: + relevant_fields[field] = values if len(values) > 1 else values[0] + else: + relevant_fields = record.data + return relevant_fields + + def _get_document_metadata(self, record: AirbyteRecordMessage) -> Dict[str, Any]: + document_metadata = self._extract_relevant_fields(record, self.metadata_fields) + document_metadata[METADATA_STREAM_FIELD] = self._get_stream_id(record) + return document_metadata + + def _get_stream_id(self, record: AirbyteRecordMessage) -> str: + return f"{record.namespace}_{record.stream}" + + def _get_record_primary_key(self, record: AirbyteRecordMessage) -> Optional[str]: + stream_identifier = self._get_stream_id(record) + current_stream: ConfiguredAirbyteStream = self.streams[stream_identifier] + + if not current_stream.primary_key: + return None + + primary_key = [] + for key in current_stream.primary_key: + try: + primary_key.append(str(dpath.util.get(record.data, key))) + except KeyError: + primary_key.append("__not_found__") + stringified_primary_key = "_".join(primary_key) + return f"{stream_identifier}_{stringified_primary_key}" diff --git a/airbyte-integrations/connectors/destination-vectara/icon.svg b/airbyte-integrations/connectors/destination-vectara/icon.svg new file mode 100644 index 000000000000..70798dc5f55b --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/icon.svg @@ -0,0 +1,40 @@ + + + + + + diff --git a/airbyte-integrations/connectors/destination-vectara/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-vectara/integration_tests/integration_test.py new file mode 100644 index 000000000000..6bac4eba8dbc --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/integration_tests/integration_test.py @@ -0,0 +1,125 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import logging +import unittest +from typing import Any, Dict + +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStateMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + Status, + SyncMode, + Type, +) +from destination_vectara.client import VectaraClient +from destination_vectara.destination import DestinationVectara + + +class VectaraIntegrationTest(unittest.TestCase): + def _get_configured_catalog(self, destination_mode: DestinationSyncMode) -> ConfiguredAirbyteCatalog: + stream_schema = {"type": "object", "properties": {"str_col": {"type": "str"}, "int_col": {"type": "integer"}}} + + overwrite_stream = ConfiguredAirbyteStream( + stream=AirbyteStream( + name="mystream", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh] + ), + primary_key=[["int_col"]], + sync_mode=SyncMode.incremental, + destination_sync_mode=destination_mode, + ) + + return ConfiguredAirbyteCatalog(streams=[overwrite_stream]) + + def _state(self, data: Dict[str, Any]) -> AirbyteMessage: + return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data=data)) + + def _record(self, stream: str, str_value: str, int_value: int) -> AirbyteMessage: + return AirbyteMessage( + type=Type.RECORD, record=AirbyteRecordMessage(stream=stream, data={"str_col": str_value, "int_col": int_value}, emitted_at=0) + ) + def _clean(self): + self._client.delete_doc_by_metadata(metadata_field_name="_ab_stream", metadata_field_values=["None_mystream"]) + + def setUp(self): + with open("secrets/config.json", "r") as f: + self.config = json.loads(f.read()) + self._client = VectaraClient(self.config) + self._clean() + + def tearDown(self): + self._clean() + + def test_check_valid_config(self): + outcome = DestinationVectara().check(logging.getLogger("airbyte"), self.config) + assert outcome.status == Status.SUCCEEDED + + def test_check_invalid_config(self): + outcome = DestinationVectara().check( + logging.getLogger("airbyte"), + { + "oauth2": {"client_id": "myclientid", "client_secret": "myclientsecret"}, + "corpus_name": "teststore", + "customer_id": "123456", + "text_fields": [], + "metadata_fields": [], + }, + ) + assert outcome.status == Status.FAILED + + def _query_index(self, query="Everything", num_results=100): + return self._client._request( + "query", + data={ + "query": [ + { + "query": query, + "numResults": num_results, + "corpusKey": [ + { + "customerId": self._client.customer_id, + "corpusId": self._client.corpus_id, + } + ], + } + ] + }, + )["responseSet"][0] + + def test_write(self): + # validate corpus starts empty + initial_result = self._query_index()["document"] + assert len(initial_result) == 0 + + catalog = self._get_configured_catalog(DestinationSyncMode.overwrite) + first_state_message = self._state({"state": "1"}) + first_record_chunk = [self._record("mystream", f"Dogs are number {i}", i) for i in range(5)] + + # initial sync + destination = DestinationVectara() + list(destination.write(self.config, catalog, [*first_record_chunk, first_state_message])) + assert len(self._query_index()["document"]) == 5 + + # incrementalally update a doc + incremental_catalog = self._get_configured_catalog(DestinationSyncMode.append_dedup) + list(destination.write(self.config, incremental_catalog, [self._record("mystream", "Cats are nice", 2), first_state_message])) + assert len(self._query_index()["document"]) == 5 + + # use semantic search + result = self._query_index("Feline animals", 1) + assert result["document"] == [ + { + "id": "Stream_None_mystream_Key_None_mystream_2", + "metadata": [ + {"name": "int_col", "value": "2"}, + {"name": "_ab_stream", "value": "None_mystream"}, + ], + } + ] diff --git a/airbyte-integrations/connectors/destination-vectara/main.py b/airbyte-integrations/connectors/destination-vectara/main.py new file mode 100644 index 000000000000..289b411fb318 --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/main.py @@ -0,0 +1,11 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from destination_vectara import DestinationVectara + +if __name__ == "__main__": + DestinationVectara().run(sys.argv[1:]) diff --git a/airbyte-integrations/connectors/destination-vectara/metadata.yaml b/airbyte-integrations/connectors/destination-vectara/metadata.yaml new file mode 100644 index 000000000000..313188783aed --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/metadata.yaml @@ -0,0 +1,26 @@ +data: + allowedHosts: + hosts: + - api.vectara.io + - "vectara-prod-${self.customer_id}.auth.us-west-2.amazoncognito.com" + registries: + oss: + enabled: true + cloud: + enabled: true + connectorSubtype: database + connectorType: destination + definitionId: 102900e7-a236-4c94-83e4-a4189b99adc2 + dockerImageTag: 0.1.0 + dockerRepository: airbyte/destination-vectara + githubIssueLabel: destination-vectara + icon: vectara.svg + license: MIT + name: Vectara + releaseDate: 2023-12-16 + releaseStage: alpha + supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/destinations/vectara + tags: + - language:python +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-vectara/requirements.txt b/airbyte-integrations/connectors/destination-vectara/requirements.txt new file mode 100644 index 000000000000..d6e1198b1ab1 --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/requirements.txt @@ -0,0 +1 @@ +-e . diff --git a/airbyte-integrations/connectors/destination-vectara/setup.py b/airbyte-integrations/connectors/destination-vectara/setup.py new file mode 100644 index 000000000000..ab10a8c60fb9 --- /dev/null +++ b/airbyte-integrations/connectors/destination-vectara/setup.py @@ -0,0 +1,25 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk==0.57.8", +] + +TEST_REQUIREMENTS = ["pytest~=6.2"] + +setup( + name="destination_vectara", + description="Destination implementation for Vectara.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/destination-vectara/unit_tests/__init__.py b/airbyte-integrations/connectors/destination-vectara/unit_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/destination-vertica/.dockerignore b/airbyte-integrations/connectors/destination-vertica/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-vertica/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-vertica/Dockerfile b/airbyte-integrations/connectors/destination-vertica/Dockerfile deleted file mode 100644 index 06b60f67af53..000000000000 --- a/airbyte-integrations/connectors/destination-vertica/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte -ENV APPLICATION destination-vertica - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte -ENV APPLICATION destination-vertica - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/destination-vertica diff --git a/airbyte-integrations/connectors/destination-vertica/README.md b/airbyte-integrations/connectors/destination-vertica/README.md index c0df44ede3cd..18d51ba57fb0 100644 --- a/airbyte-integrations/connectors/destination-vertica/README.md +++ b/airbyte-integrations/connectors/destination-vertica/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-vertica:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-vertica:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-vertica:dev`. the Dockerfile. #### Run @@ -61,8 +62,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-vertica test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/vertica.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-vertica/build.gradle b/airbyte-integrations/connectors/destination-vertica/build.gradle index 2a9fc0fd4865..d5392f6c238b 100644 --- a/airbyte-integrations/connectors/destination-vertica/build.gradle +++ b/airbyte-integrations/connectors/destination-vertica/build.gradle @@ -1,22 +1,28 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.vertica.VerticaDestination' } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - implementation project(path: ':airbyte-integrations:bases:bases-destination-jdbc') implementation group: 'com.vertica.jdbc', name: 'vertica-jdbc', version: '12.0.3-0' - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-vertica') implementation group: 'org.testcontainers', name: 'jdbc', version: '1.18.0' } diff --git a/airbyte-integrations/connectors/destination-vertica/src/main/java/io/airbyte/integrations/destination/vertica/VerticaDestination.java b/airbyte-integrations/connectors/destination-vertica/src/main/java/io/airbyte/integrations/destination/vertica/VerticaDestination.java index c8cd6ca846f0..f9ec9851f449 100644 --- a/airbyte-integrations/connectors/destination-vertica/src/main/java/io/airbyte/integrations/destination/vertica/VerticaDestination.java +++ b/airbyte-integrations/connectors/destination-vertica/src/main/java/io/airbyte/integrations/destination/vertica/VerticaDestination.java @@ -4,22 +4,22 @@ package io.airbyte.integrations.destination.vertica; -import static io.airbyte.integrations.base.errors.messages.ErrorMessage.getErrorMessage; +import static io.airbyte.cdk.integrations.base.errors.messages.ErrorMessage.getErrorMessage; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.*; +import io.airbyte.cdk.integrations.base.ssh.SshWrappedDestination; +import io.airbyte.cdk.integrations.destination.NamingConventionTransformer; +import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination; +import io.airbyte.cdk.integrations.destination.jdbc.JdbcBufferedConsumerFactory; import io.airbyte.commons.exceptions.ConnectionErrorException; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.*; -import io.airbyte.integrations.base.ssh.SshWrappedDestination; -import io.airbyte.integrations.destination.NamingConventionTransformer; -import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; -import io.airbyte.integrations.destination.jdbc.JdbcBufferedConsumerFactory; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/connectors/destination-vertica/src/main/java/io/airbyte/integrations/destination/vertica/VerticaNameTransformer.java b/airbyte-integrations/connectors/destination-vertica/src/main/java/io/airbyte/integrations/destination/vertica/VerticaNameTransformer.java index ee752c6c4f5c..4e714d636021 100644 --- a/airbyte-integrations/connectors/destination-vertica/src/main/java/io/airbyte/integrations/destination/vertica/VerticaNameTransformer.java +++ b/airbyte-integrations/connectors/destination-vertica/src/main/java/io/airbyte/integrations/destination/vertica/VerticaNameTransformer.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.vertica; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; public class VerticaNameTransformer extends StandardNameTransformer { diff --git a/airbyte-integrations/connectors/destination-vertica/src/main/java/io/airbyte/integrations/destination/vertica/VerticaSqlOperations.java b/airbyte-integrations/connectors/destination-vertica/src/main/java/io/airbyte/integrations/destination/vertica/VerticaSqlOperations.java index 8161f91ed60f..398d09731d6b 100644 --- a/airbyte-integrations/connectors/destination-vertica/src/main/java/io/airbyte/integrations/destination/vertica/VerticaSqlOperations.java +++ b/airbyte-integrations/connectors/destination-vertica/src/main/java/io/airbyte/integrations/destination/vertica/VerticaSqlOperations.java @@ -4,9 +4,9 @@ package io.airbyte.integrations.destination.vertica; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.jdbc.JdbcSqlOperations; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.jdbc.JdbcSqlOperations; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.io.File; import java.io.FileWriter; diff --git a/airbyte-integrations/connectors/destination-vertica/src/test-integration/java/io/airbyte/integrations/destination/vertica/VerticaDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-vertica/src/test-integration/java/io/airbyte/integrations/destination/vertica/VerticaDestinationAcceptanceTest.java index 5ba85840d228..f098723501cd 100644 --- a/airbyte-integrations/connectors/destination-vertica/src/test-integration/java/io/airbyte/integrations/destination/vertica/VerticaDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-vertica/src/test-integration/java/io/airbyte/integrations/destination/vertica/VerticaDestinationAcceptanceTest.java @@ -6,14 +6,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; import java.io.IOException; import java.sql.SQLException; import java.util.HashSet; diff --git a/airbyte-integrations/connectors/destination-weaviate/Dockerfile b/airbyte-integrations/connectors/destination-weaviate/Dockerfile deleted file mode 100644 index 4d7641f5bb99..000000000000 --- a/airbyte-integrations/connectors/destination-weaviate/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base \ - && apk add libffi-dev - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY destination_weaviate ./destination_weaviate - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.1 -LABEL io.airbyte.name=airbyte/destination-weaviate diff --git a/airbyte-integrations/connectors/destination-weaviate/README.md b/airbyte-integrations/connectors/destination-weaviate/README.md index ff12caf64f32..c14faa824280 100644 --- a/airbyte-integrations/connectors/destination-weaviate/README.md +++ b/airbyte-integrations/connectors/destination-weaviate/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:destination-weaviate:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/destinations/weaviate) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_weaviate/spec.json` file. @@ -54,19 +48,70 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/destination-weaviate:dev + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=destination-weaviate build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/destination-weaviate:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. -You can also build the connector image via Gradle: +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:destination-weaviate:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/destination-weaviate:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/destination-weaviate:dev . +# Running the spec command against your patched connector +docker run airbyte/destination-weaviate:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -75,38 +120,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-weaviate:dev chec # messages.jsonl is a file containing line-separated JSON representing AirbyteMessages cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-weaviate:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=destination-weaviate test ``` -#### Acceptance Tests -Coming soon: -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:destination-weaviate:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:destination-weaviate:integrationTest -``` +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -116,8 +139,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-weaviate test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/weaviate.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-weaviate/acceptance-test-config.yml b/airbyte-integrations/connectors/destination-weaviate/acceptance-test-config.yml new file mode 100644 index 000000000000..1d9dca499c13 --- /dev/null +++ b/airbyte-integrations/connectors/destination-weaviate/acceptance-test-config.yml @@ -0,0 +1,7 @@ +acceptance_tests: + spec: + tests: + - spec_path: integration_tests/spec.json + backward_compatibility_tests_config: + disable_for_version: "0.2.0" +connector_image: airbyte/destination-weaviate:dev diff --git a/airbyte-integrations/connectors/destination-weaviate/build.gradle b/airbyte-integrations/connectors/destination-weaviate/build.gradle deleted file mode 100644 index b2101bb02449..000000000000 --- a/airbyte-integrations/connectors/destination-weaviate/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'destination_weaviate' -} diff --git a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/client.py b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/client.py deleted file mode 100644 index 3ba83b2a4a53..000000000000 --- a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/client.py +++ /dev/null @@ -1,141 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import json -import logging -import time -import uuid -from dataclasses import dataclass -from typing import Any, List, Mapping, MutableMapping - -import weaviate - -from .utils import generate_id, parse_id_schema, parse_vectors, stream_to_class_name - - -@dataclass -class BufferedObject: - id: str - properties: Mapping[str, Any] - vector: List[Any] - class_name: str - - -class WeaviatePartialBatchError(Exception): - pass - - -class Client: - def __init__(self, config: Mapping[str, Any], schema: Mapping[str, str]): - self.client = self.get_weaviate_client(config) - self.config = config - self.batch_size = int(config.get("batch_size", 100)) - self.schema = schema - self.vectors = parse_vectors(config.get("vectors")) - self.id_schema = parse_id_schema(config.get("id_schema")) - self.buffered_objects: MutableMapping[str, BufferedObject] = {} - self.objects_with_error: MutableMapping[str, BufferedObject] = {} - - def buffered_write_operation(self, stream_name: str, record: MutableMapping): - if self.id_schema.get(stream_name, "") in record: - id_field_name = self.id_schema.get(stream_name, "") - record_id = generate_id(record.get(id_field_name)) - del record[id_field_name] - else: - if "id" in record: - record_id = generate_id(record.get("id")) - del record["id"] - # Weaviate will throw an error if you try to store a field with name _id - elif "_id" in record: - record_id = generate_id(record.get("_id")) - del record["_id"] - else: - record_id = uuid.uuid4() - record_id = str(record_id) - - # TODO support nested objects instead of converting to json string when weaviate supports this - for k, v in record.items(): - if self.schema[stream_name].get(k, "") == "jsonify": - record[k] = json.dumps(v) - # Handling of empty list that's not part of defined schema otherwise Weaviate throws invalid string property - if isinstance(v, list) and len(v) == 0 and k not in self.schema[stream_name]: - record[k] = "" - - missing_properties = set(self.schema[stream_name].keys()).difference(record.keys()).discard("id") - for prop in missing_properties or []: - record[prop] = None - - additional_props = set(record.keys()).difference(self.schema[stream_name].keys()) - for prop in additional_props or []: - if isinstance(record[prop], dict): - record[prop] = json.dumps(record[prop]) - if isinstance(record[prop], list) and len(record[prop]) > 0 and isinstance(record[prop][0], dict): - record[prop] = json.dumps(record[prop]) - - # Property names in Weaviate have to start with lowercase letter - record = {k[0].lower() + k[1:]: v for k, v in record.items()} - vector = None - if stream_name in self.vectors: - vector_column_name = self.vectors.get(stream_name) - vector = record.get(vector_column_name) - del record[vector_column_name] - class_name = stream_to_class_name(stream_name) - self.client.batch.add_data_object(record, class_name, record_id, vector=vector) - self.buffered_objects[record_id] = BufferedObject(record_id, record, vector, class_name) - if self.client.batch.num_objects() >= self.batch_size: - self.flush() - - def flush(self, retries: int = 3): - if len(self.objects_with_error) > 0 and retries == 0: - error_msg = f"Objects had errors and retries failed as well. Object IDs: {self.objects_with_error.keys()}" - raise WeaviatePartialBatchError(error_msg) - - results = self.client.batch.create_objects() - self.objects_with_error.clear() - for result in results: - errors = result.get("result", {}).get("errors", []) - if errors: - obj_id = result.get("id") - self.objects_with_error[obj_id] = self.buffered_objects.get(obj_id) - logging.info(f"Object {obj_id} had errors: {errors}. Going to retry.") - - for buffered_object in self.objects_with_error.values(): - self.client.batch.add_data_object( - buffered_object.properties, buffered_object.class_name, buffered_object.id, buffered_object.vector - ) - - if len(self.objects_with_error) > 0 and retries > 0: - logging.info("sleeping 2 seconds before retrying batch again") - time.sleep(2) - self.flush(retries - 1) - - self.buffered_objects.clear() - - def delete_stream_entries(self, stream_name: str): - class_name = stream_to_class_name(stream_name) - try: - original_schema = self.client.schema.get(class_name=class_name) - self.client.schema.delete_class(class_name=class_name) - logging.info(f"Deleted class {class_name}") - self.client.schema.create_class(original_schema) - logging.info(f"Recreated class {class_name}") - except weaviate.exceptions.UnexpectedStatusCodeException as e: - if e.message.startswith("Get schema! Unexpected status code: 404"): - logging.info(f"Class {class_name} did not exist.") - else: - raise e - - @staticmethod - def get_weaviate_client(config: Mapping[str, Any]) -> weaviate.Client: - url, username, password = config.get("url"), config.get("username"), config.get("password") - - if username and not password: - raise Exception("Password is required when username is set") - if password and not username: - raise Exception("Username is required when password is set") - - if username and password: - credentials = weaviate.auth.AuthClientPassword(username, password) - return weaviate.Client(url=url, auth_client_secret=credentials) - return weaviate.Client(url=url, timeout_config=(2, 2)) diff --git a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/config.py b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/config.py new file mode 100644 index 000000000000..c4708d59ffc9 --- /dev/null +++ b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/config.py @@ -0,0 +1,118 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import List, Literal, Union + +from airbyte_cdk.destinations.vector_db_based.config import ( + AzureOpenAIEmbeddingConfigModel, + CohereEmbeddingConfigModel, + FakeEmbeddingConfigModel, + FromFieldEmbeddingConfigModel, + OpenAICompatibleEmbeddingConfigModel, + OpenAIEmbeddingConfigModel, + VectorDBConfigModel, +) +from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig +from pydantic import BaseModel, Field + + +class UsernamePasswordAuth(BaseModel): + mode: Literal["username_password"] = Field("username_password", const=True) + username: str = Field(..., title="Username", description="Username for the Weaviate cluster", order=1) + password: str = Field(..., title="Password", description="Password for the Weaviate cluster", airbyte_secret=True, order=2) + + class Config(OneOfOptionConfig): + title = "Username/Password" + description = "Authenticate using username and password (suitable for self-managed Weaviate clusters)" + discriminator = "mode" + + +class NoAuth(BaseModel): + mode: Literal["no_auth"] = Field("no_auth", const=True) + + class Config(OneOfOptionConfig): + title = "No Authentication" + description = "Do not authenticate (suitable for locally running test clusters, do not use for clusters with public IP addresses)" + discriminator = "mode" + + +class TokenAuth(BaseModel): + mode: Literal["token"] = Field("token", const=True) + token: str = Field(..., title="API Token", description="API Token for the Weaviate instance", airbyte_secret=True) + + class Config(OneOfOptionConfig): + title = "API Token" + description = "Authenticate using an API token (suitable for Weaviate Cloud)" + discriminator = "mode" + + +class Header(BaseModel): + header_key: str = Field(..., title="Header Key") + value: str = Field(..., title="Header Value", airbyte_secret=True) + + +class WeaviateIndexingConfigModel(BaseModel): + host: str = Field( + ..., + title="Public Endpoint", + order=1, + description="The public endpoint of the Weaviate cluster.", + examples=["https://my-cluster.weaviate.network"], + ) + auth: Union[TokenAuth, UsernamePasswordAuth, NoAuth] = Field( + ..., title="Authentication", description="Authentication method", discriminator="mode", type="object", order=2 + ) + batch_size: int = Field(title="Batch Size", description="The number of records to send to Weaviate in each batch", default=128) + text_field: str = Field(title="Text Field", description="The field in the object that contains the embedded text", default="text") + tenant_id: str = Field(title="Tenant ID", description="The tenant ID to use for multi tenancy", airbyte_secret=True, default="") + default_vectorizer: str = Field( + title="Default Vectorizer", + description="The vectorizer to use if new classes need to be created", + default="none", + enum=[ + "none", + "text2vec-cohere", + "text2vec-huggingface", + "text2vec-openai", + "text2vec-palm", + "text2vec-contextionary", + "text2vec-transformers", + "text2vec-gpt4all", + ], + ) + additional_headers: List[Header] = Field( + title="Additional headers", + description="Additional HTTP headers to send with every request.", + default=[], + examples=[{"header_key": "X-OpenAI-Api-Key", "value": "my-openai-api-key"}], + ) + + class Config: + title = "Indexing" + schema_extra = { + "group": "indexing", + "description": "Indexing configuration", + } + + +class NoEmbeddingConfigModel(BaseModel): + mode: Literal["no_embedding"] = Field("no_embedding", const=True) + + class Config(OneOfOptionConfig): + title = "No external embedding" + description = "Do not calculate and pass embeddings to Weaviate. Suitable for clusters with configured vectorizers to calculate embeddings within Weaviate or for classes that should only support regular text search." + discriminator = "mode" + + +class ConfigModel(VectorDBConfigModel): + indexing: WeaviateIndexingConfigModel + embedding: Union[ + NoEmbeddingConfigModel, + AzureOpenAIEmbeddingConfigModel, + OpenAIEmbeddingConfigModel, + CohereEmbeddingConfigModel, + FromFieldEmbeddingConfigModel, + FakeEmbeddingConfigModel, + OpenAICompatibleEmbeddingConfigModel, + ] = Field(..., title="Embedding", description="Embedding configuration", discriminator="mode", group="embedding", type="object") diff --git a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/destination.py b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/destination.py index 6c19dc8ac016..208e3610e998 100644 --- a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/destination.py +++ b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/destination.py @@ -2,78 +2,62 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import random -import string + from typing import Any, Iterable, Mapping from airbyte_cdk import AirbyteLogger from airbyte_cdk.destinations import Destination -from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, DestinationSyncMode, Status, Type - -from .client import Client -from .utils import get_schema_from_catalog +from airbyte_cdk.destinations.vector_db_based.document_processor import DocumentProcessor +from airbyte_cdk.destinations.vector_db_based.embedder import Embedder, create_from_config +from airbyte_cdk.destinations.vector_db_based.indexer import Indexer +from airbyte_cdk.destinations.vector_db_based.writer import Writer +from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, ConnectorSpecification, Status +from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode +from destination_weaviate.config import ConfigModel +from destination_weaviate.indexer import WeaviateIndexer +from destination_weaviate.no_embedder import NoEmbedder class DestinationWeaviate(Destination): + indexer: Indexer + embedder: Embedder + + def _init_indexer(self, config: ConfigModel): + self.embedder = ( + create_from_config(config.embedding, config.processing) + if config.embedding.mode != "no_embedding" + else NoEmbedder(config.embedding) + ) + self.indexer = WeaviateIndexer(config.indexing) + def write( self, config: Mapping[str, Any], configured_catalog: ConfiguredAirbyteCatalog, input_messages: Iterable[AirbyteMessage] ) -> Iterable[AirbyteMessage]: - """ - Reads the input stream of messages, config, and catalog to write data to the destination. - - This method returns an iterable (typically a generator of AirbyteMessages via yield) containing state messages received - in the input message stream. Outputting a state message means that every AirbyteRecordMessage which came before it has been - successfully persisted to the destination. This is used to ensure fault tolerance in the case that a sync fails before fully completing, - then the source is given the last state message output from this method as the starting point of the next sync. - - :param config: dict of JSON configuration matching the configuration declared in spec.json - :param configured_catalog: The Configured Catalog describing the schema of the data being received and how it should be persisted in the - destination - :param input_messages: The stream of input messages received from the source - :return: Iterable of AirbyteStateMessages wrapped in AirbyteMessage structs - """ - client = Client(config, get_schema_from_catalog(configured_catalog)) - for configured_stream in configured_catalog.streams: - if configured_stream.destination_sync_mode == DestinationSyncMode.overwrite: - client.delete_stream_entries(configured_stream.stream.name) - - for message in input_messages: - if message.type == Type.STATE: - # Emitting a state message indicates that all records which came before it have been written to the destination. So we flush - # the queue to ensure writes happen, then output the state message to indicate it's safe to checkpoint state - client.flush() - yield message - elif message.type == Type.RECORD: - record = message.record - client.buffered_write_operation(record.stream, record.data) - else: - # ignore other message types for now - continue - - # Make sure to flush any records still in the queue - client.flush() + config_model = ConfigModel.parse_obj(config) + self._init_indexer(config_model) + writer = Writer( + config_model.processing, + self.indexer, + self.embedder, + batch_size=config_model.indexing.batch_size, + omit_raw_text=config_model.omit_raw_text, + ) + yield from writer.write(configured_catalog, input_messages) def check(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: - """ - Tests if the input configuration can be used to successfully connect to the destination with the needed permissions - e.g: if a provided API token or password can be used to connect and write to the destination. - - :param logger: Logging object to display debug/info/error to the logs - (logs will not be accessible via airbyte UI if they are not passed to this logger) - :param config: Json object containing the configuration of this destination, content of this json is as specified in - the properties of the spec.json file - - :return: AirbyteConnectionStatus indicating a Success or Failure - """ - try: - client = Client.get_weaviate_client(config) - ready = client.is_ready() - if not ready: - return AirbyteConnectionStatus(status=Status.FAILED, message=f"Weaviate server {config.get('url')} not ready") - - class_name = "".join(random.choices(string.ascii_uppercase, k=10)) - client.schema.create_class({"class": class_name}) - client.schema.delete_class(class_name) + parsed_config = ConfigModel.parse_obj(config) + self._init_indexer(parsed_config) + checks = [self.embedder.check(), self.indexer.check(), DocumentProcessor.check_config(parsed_config.processing)] + errors = [error for error in checks if error is not None] + if len(errors) > 0: + return AirbyteConnectionStatus(status=Status.FAILED, message="\n".join(errors)) + else: return AirbyteConnectionStatus(status=Status.SUCCEEDED) - except Exception as e: - return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {repr(e)}") + + def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: + return ConnectorSpecification( + documentationUrl="https://docs.airbyte.com/integrations/destinations/weaviate", + supportsIncremental=True, + supported_destination_sync_modes=[DestinationSyncMode.overwrite, DestinationSyncMode.append, DestinationSyncMode.append_dedup], + connectionSpecification=ConfigModel.schema(), # type: ignore[attr-defined] + ) diff --git a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/indexer.py b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/indexer.py new file mode 100644 index 000000000000..93adb9d825a4 --- /dev/null +++ b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/indexer.py @@ -0,0 +1,208 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import json +import logging +import os +import re +import uuid +from collections import defaultdict +from typing import Optional + +import weaviate +from airbyte_cdk.destinations.vector_db_based.document_processor import METADATA_RECORD_ID_FIELD +from airbyte_cdk.destinations.vector_db_based.indexer import Indexer +from airbyte_cdk.destinations.vector_db_based.utils import create_chunks, format_exception +from airbyte_cdk.models import ConfiguredAirbyteCatalog +from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode +from destination_weaviate.config import WeaviateIndexingConfigModel + + +class WeaviatePartialBatchError(Exception): + pass + + +CLOUD_DEPLOYMENT_MODE = "cloud" + + +class WeaviateIndexer(Indexer): + config: WeaviateIndexingConfigModel + + def __init__(self, config: WeaviateIndexingConfigModel): + super().__init__(config) + + def _create_client(self): + headers = { + self.config.additional_headers[i].header_key: self.config.additional_headers[i].value + for i in range(len(self.config.additional_headers)) + } + if self.config.auth.mode == "username_password": + credentials = weaviate.auth.AuthClientPassword(self.config.auth.username, self.config.auth.password) + self.client = weaviate.Client(url=self.config.host, auth_client_secret=credentials, additional_headers=headers) + elif self.config.auth.mode == "token": + credentials = weaviate.auth.AuthApiKey(self.config.auth.token) + self.client = weaviate.Client(url=self.config.host, auth_client_secret=credentials, additional_headers=headers) + else: + self.client = weaviate.Client(url=self.config.host, additional_headers=headers) + + # disable dynamic batching because it's handled asynchroniously in the client + self.client.batch.configure( + batch_size=None, dynamic=False, weaviate_error_retries=weaviate.WeaviateErrorRetryConf(number_retries=5) + ) + + def _add_tenant_to_class_if_missing(self, class_name: str): + class_tenants = self.client.schema.get_class_tenants(class_name=class_name) + if class_tenants is not None and self.config.tenant_id not in [tenant.name for tenant in class_tenants]: + self.client.schema.add_class_tenants(class_name=class_name, tenants=[weaviate.Tenant(name=self.config.tenant_id)]) + logging.info(f"Added tenant {self.config.tenant_id} to class {class_name}") + else: + logging.info(f"Tenant {self.config.tenant_id} already exists in class {class_name}") + + def check(self) -> Optional[str]: + deployment_mode = os.environ.get("DEPLOYMENT_MODE", "") + if deployment_mode.casefold() == CLOUD_DEPLOYMENT_MODE and not self._uses_safe_config(): + return "Host must start with https:// and authentication must be enabled on cloud deployment." + try: + self._create_client() + except Exception as e: + return format_exception(e) + return None + + def _uses_safe_config(self) -> bool: + return self.config.host.startswith("https://") and not self.config.auth.mode == "no_auth" + + def pre_sync(self, catalog: ConfiguredAirbyteCatalog) -> None: + self._create_client() + classes = {c["class"]: c for c in self.client.schema.get().get("classes", [])} + self.has_record_id_metadata = defaultdict(lambda: False) + + if self.config.tenant_id.strip(): + for class_name in classes.keys(): + self._add_tenant_to_class_if_missing(class_name) + + for stream in catalog.streams: + class_name = self._stream_to_class_name(stream.stream.name) + schema = classes[class_name] if class_name in classes else None + if stream.destination_sync_mode == DestinationSyncMode.overwrite and schema is not None: + self.client.schema.delete_class(class_name=class_name) + logging.info(f"Deleted class {class_name}") + self.client.schema.create_class(schema) + logging.info(f"Recreated class {class_name}") + elif class_name not in classes: + config = { + "class": class_name, + "vectorizer": self.config.default_vectorizer, + "properties": [ + { + # Record ID is used for bookkeeping, not for searching + "name": METADATA_RECORD_ID_FIELD, + "dataType": ["text"], + "description": "Record ID, used for bookkeeping.", + "indexFilterable": True, + "indexSearchable": False, + "tokenization": "field", + } + ], + } + if self.config.tenant_id.strip(): + config["multiTenancyConfig"] = {"enabled": True} + + self.client.schema.create_class(config) + logging.info(f"Created class {class_name}") + + if self.config.tenant_id.strip(): + self._add_tenant_to_class_if_missing(class_name) + else: + self.has_record_id_metadata[class_name] = schema is not None and any( + prop.get("name") == METADATA_RECORD_ID_FIELD for prop in schema.get("properties", {}) + ) + + def delete(self, delete_ids, namespace, stream): + if len(delete_ids) > 0: + class_name = self._stream_to_class_name(stream) + if self.has_record_id_metadata[class_name]: + where_filter = {"path": [METADATA_RECORD_ID_FIELD], "operator": "ContainsAny", "valueStringArray": delete_ids} + if self.config.tenant_id.strip(): + self.client.batch.delete_objects( + class_name=class_name, + tenant=self.config.tenant_id, + where=where_filter, + ) + else: + self.client.batch.delete_objects( + class_name=class_name, + where=where_filter, + ) + + def index(self, document_chunks, namespace, stream): + if len(document_chunks) == 0: + return + + # As a single record can be split into lots of documents, break them into batches as configured to not overwhelm the cluster + batches = create_chunks(document_chunks, batch_size=self.config.batch_size) + for batch in batches: + for i in range(len(batch)): + chunk = batch[i] + weaviate_object = {**self._normalize(chunk.metadata)} + if chunk.page_content is not None: + weaviate_object[self.config.text_field] = chunk.page_content + object_id = str(uuid.uuid4()) + class_name = self._stream_to_class_name(chunk.record.stream) + if self.config.tenant_id.strip(): + self.client.batch.add_data_object( + weaviate_object, class_name, object_id, vector=chunk.embedding, tenant=self.config.tenant_id + ) + else: + self.client.batch.add_data_object(weaviate_object, class_name, object_id, vector=chunk.embedding) + self._flush() + + def _stream_to_class_name(self, stream_name: str) -> str: + pattern = "[^0-9A-Za-z_]+" + stream_name = re.sub(pattern, "", stream_name) + stream_name = stream_name.replace(" ", "") + return stream_name[0].upper() + stream_name[1:] + + def _normalize_property_name(self, field_name: str) -> str: + # Remove invalid characters and replace spaces with underscores + normalized = re.sub(r"[^0-9A-Za-z_]", "", field_name.replace(" ", "_")) + + # Ensure the name starts with a letter or underscore + if not re.match(r"^[_A-Za-z]", normalized): + normalized = "_" + normalized + + return normalized[0].lower() + normalized[1:] + + def _normalize(self, metadata: dict) -> dict: + result = {} + + for key, value in metadata.items(): + # Property names in Weaviate have to start with lowercase letter + normalized_key = self._normalize_property_name(key) + # "id" and "additional" are reserved properties in Weaviate, prefix to disambiguate + if key == "id" or key == "_id" or key == "_additional": + normalized_key = f"raw_{key}" + if isinstance(value, list) and len(value) == 0: + # Handling of empty list that's not part of defined schema otherwise Weaviate throws invalid string property + continue + if isinstance(value, (str, int, float, bool)) or (isinstance(value, list) and all(isinstance(item, str) for item in value)): + result[normalized_key] = value + else: + # JSON encode all other types + result[normalized_key] = json.dumps(value) + + return result + + def _flush(self, retries: int = 3): + results = self.client.batch.create_objects() + all_errors = [] + + for result in results: + errors = result.get("result", {}).get("errors", []) + if errors: + all_errors.extend(errors) + + if len(all_errors) > 0: + error_msg = "Errors while loading: " + ", ".join([str(error) for error in all_errors]) + raise WeaviatePartialBatchError(error_msg) diff --git a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/no_embedder.py b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/no_embedder.py new file mode 100644 index 000000000000..c46633633fca --- /dev/null +++ b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/no_embedder.py @@ -0,0 +1,24 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import List, Optional + +from airbyte_cdk.destinations.vector_db_based.document_processor import Chunk +from airbyte_cdk.destinations.vector_db_based.embedder import Embedder +from destination_weaviate.config import NoEmbeddingConfigModel + + +class NoEmbedder(Embedder): + def __init__(self, config: NoEmbeddingConfigModel): + super().__init__() + + def check(self) -> Optional[str]: + return None + + def embed_chunks(self, chunks: List[Chunk]) -> List[None]: + return [None for _ in chunks] + + @property + def embedding_dimensions(self) -> int: + return -1 diff --git a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/spec.json b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/spec.json deleted file mode 100644 index 8980482fa3c5..000000000000 --- a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/spec.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/destinations/weaviate", - "supported_destination_sync_modes": ["append", "overwrite"], - "supportsIncremental": true, - "supportsDBT": false, - "supportsNormalization": false, - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Destination Weaviate", - "type": "object", - "required": ["url"], - "additionalProperties": false, - "properties": { - "url": { - "type": "string", - "description": "The URL to the weaviate instance", - "examples": [ - "http://localhost:8080", - "https://your-instance.semi.network" - ] - }, - "username": { - "type": "string", - "description": "Username used with OIDC authentication", - "examples": ["xyz@weaviate.io"] - }, - "password": { - "type": "string", - "description": "Password used with OIDC authentication", - "airbyte_secret": true - }, - "batch_size": { - "type": "integer", - "description": "Batch size for writing to Weaviate", - "default": 100 - }, - "vectors": { - "type": "string", - "description": "Comma separated list of strings of `stream_name.vector_column_name` to specify which field holds the vectors.", - "examples": [ - "my_table.my_vector_column, another_table.vector", - "mytable.vector" - ] - }, - "id_schema": { - "type": "string", - "description": "Comma separated list of strings of `stream_name.id_column_name` to specify which field holds the ID of the record.", - "examples": ["my_table.my_id_column, another_table.id", "users.user_id"] - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/utils.py b/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/utils.py deleted file mode 100644 index db0eef139e78..000000000000 --- a/airbyte-integrations/connectors/destination-weaviate/destination_weaviate/utils.py +++ /dev/null @@ -1,83 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import hashlib -import re -import uuid -from typing import Any, Mapping - -from airbyte_cdk.models import ConfiguredAirbyteCatalog - - -def parse_vectors(vectors_config: str) -> Mapping[str, str]: - vectors = {} - if not vectors_config: - return vectors - - vectors_list = vectors_config.replace(" ", "").split(",") - for vector in vectors_list: - stream_name, vector_column_name = vector.split(".") - vectors[stream_name] = vector_column_name - return vectors - - -def parse_id_schema(id_schema_config: str) -> Mapping[str, str]: - id_schema = {} - if not id_schema_config: - return id_schema - - id_schema_list = id_schema_config.replace(" ", "").split(",") - for schema_id in id_schema_list: - stream_name, id_field_name = schema_id.split(".") - id_schema[stream_name] = id_field_name - return id_schema - - -def hex_to_int(hex_str: str) -> int: - try: - return int(hex_str, 16) - except ValueError: - return 0 - - -def is_uuid_string(uuid_string): - uuid_pattern = "^[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}$" - return re.match(uuid_pattern, uuid_string) - - -def generate_id(record_id: Any) -> uuid.UUID: - if isinstance(record_id, int): - return uuid.UUID(int=record_id) - - if isinstance(record_id, str): - if is_uuid_string(record_id): - return uuid.UUID(record_id) - id_int = hex_to_int(record_id) - if id_int > 0: - return uuid.UUID(int=id_int) - hex_string = hashlib.md5(record_id.encode("UTF-8")).hexdigest() - return uuid.UUID(hex=hex_string) - - -def get_schema_from_catalog(configured_catalog: ConfiguredAirbyteCatalog) -> Mapping[str, Mapping[str, str]]: - schema = {} - for stream in configured_catalog.streams: - stream_schema = {} - for k, v in stream.stream.json_schema.get("properties").items(): - stream_schema[k] = "default" - if "array" in v.get("type", []) and ( - "object" in v.get("items", {}).get("type", []) or "array" in v.get("items", {}).get("type", []) or v.get("items", {}) == {} - ): - stream_schema[k] = "jsonify" - if "object" in v.get("type", []): - stream_schema[k] = "jsonify" - schema[stream.stream.name] = stream_schema - return schema - - -def stream_to_class_name(stream_name: str) -> str: - pattern = "[^0-9A-Za-z_]+" - stream_name = re.sub(pattern, "", stream_name) - stream_name = stream_name.replace(" ", "") - return stream_name[0].upper() + stream_name[1:] diff --git a/airbyte-integrations/connectors/destination-weaviate/examples/config.json b/airbyte-integrations/connectors/destination-weaviate/examples/config.json new file mode 100644 index 000000000000..48731a797dd3 --- /dev/null +++ b/airbyte-integrations/connectors/destination-weaviate/examples/config.json @@ -0,0 +1,20 @@ +{ + "processing": { + "text_fields": ["str_col", "title"], + "metadata_fields": ["field2"], + "chunk_size": 1000 + }, + "embedding": { + "mode": "fake" + }, + "indexing": { + "host": "https://my-cluster.weaviate.network", + "class_name": "Test", + "auth": { + "mode": "token", + "token": "mytoken" + }, + "text_field": "text", + "batch_size": 128 + } +} diff --git a/airbyte-integrations/connectors/destination-weaviate/examples/configured_catalog.json b/airbyte-integrations/connectors/destination-weaviate/examples/configured_catalog.json new file mode 100644 index 000000000000..acb931363d89 --- /dev/null +++ b/airbyte-integrations/connectors/destination-weaviate/examples/configured_catalog.json @@ -0,0 +1,20 @@ +{ + "streams": [ + { + "stream": { + "name": "example_stream", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {} + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "default_cursor_field": ["column_name"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup", + "primary_key": [["pk"]] + } + ] +} diff --git a/airbyte-integrations/connectors/destination-weaviate/examples/messages.jsonl b/airbyte-integrations/connectors/destination-weaviate/examples/messages.jsonl new file mode 100644 index 000000000000..e274f981cc4a --- /dev/null +++ b/airbyte-integrations/connectors/destination-weaviate/examples/messages.jsonl @@ -0,0 +1,2 @@ +{"type": "RECORD", "record": {"stream": "example_stream", "data": { "title": "valueZXXXXXXXX1", "field2": "value2", "pk": "1", "vector": [1,2,3,4,5,6,7] }, "emitted_at": 1625383200000}} +{"type": "RECORD", "record": {"stream": "example_stream", "data": { "title": "a completely different value and so on", "field2": "value2", "pk": "1", "vector": [0,0,0,0,0,0,0] }, "emitted_at": 1625383200000}} \ No newline at end of file diff --git a/airbyte-integrations/connectors/destination-weaviate/icon.svg b/airbyte-integrations/connectors/destination-weaviate/icon.svg index 79b0033e12d8..ff31522d53bd 100644 --- a/airbyte-integrations/connectors/destination-weaviate/icon.svg +++ b/airbyte-integrations/connectors/destination-weaviate/icon.svg @@ -1,6 +1,197 @@ - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airbyte-integrations/connectors/destination-weaviate/integration_tests/create_objects_partial_error.json b/airbyte-integrations/connectors/destination-weaviate/integration_tests/create_objects_partial_error.json deleted file mode 100644 index 5d47610a805d..000000000000 --- a/airbyte-integrations/connectors/destination-weaviate/integration_tests/create_objects_partial_error.json +++ /dev/null @@ -1,27 +0,0 @@ -[ - { - "class": "NonExistingClass", - "creationTimeUnix": 1614852753747, - "id": "154cbccd-89f4-4b29-9c1b-001a3339d89a", - "properties": {}, - "deprecations": null, - "result": { - "errors": { - "error": [ - { - "message": "class 'NonExistingClass' not present in schema, class NonExistingClass not present" - } - ] - } - } - }, - { - "class": "ExistingClass", - "creationTimeUnix": 1614852753746, - "id": "b7b1cfbe-20da-496c-b932-008d35805f26", - "properties": {}, - "vector": [-0.05244319, 0.076136276], - "deprecations": null, - "result": {} - } -] diff --git a/airbyte-integrations/connectors/destination-weaviate/integration_tests/example-config.json b/airbyte-integrations/connectors/destination-weaviate/integration_tests/example-config.json deleted file mode 100644 index ca97fc7fc0ef..000000000000 --- a/airbyte-integrations/connectors/destination-weaviate/integration_tests/example-config.json +++ /dev/null @@ -1 +0,0 @@ -{ "url": "http://localhost:8081" } diff --git a/airbyte-integrations/connectors/destination-weaviate/integration_tests/exchange_rate_catalog.json b/airbyte-integrations/connectors/destination-weaviate/integration_tests/exchange_rate_catalog.json deleted file mode 100644 index 48d9ee0d8cb4..000000000000 --- a/airbyte-integrations/connectors/destination-weaviate/integration_tests/exchange_rate_catalog.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": "integer" - }, - "currency": { - "type": "string" - }, - "date": { - "type": "string" - }, - "HKD": { - "type": "number" - }, - "NZD": { - "type": "number" - }, - "USD": { - "type": "number" - } - } -} diff --git a/airbyte-integrations/connectors/destination-weaviate/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-weaviate/integration_tests/integration_test.py index fab211b55f90..e76a8afe4457 100644 --- a/airbyte-integrations/connectors/destination-weaviate/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-weaviate/integration_tests/integration_test.py @@ -4,660 +4,127 @@ import json import logging -import os import time -import uuid -from typing import Any, Dict, List, Mapping -from unittest.mock import Mock import docker -import pytest -from airbyte_cdk import AirbyteLogger -from airbyte_cdk.models import ( - AirbyteMessage, - AirbyteRecordMessage, - AirbyteStateMessage, - AirbyteStream, - ConfiguredAirbyteCatalog, - ConfiguredAirbyteStream, - DestinationSyncMode, - Status, - SyncMode, - Type, -) -from destination_weaviate import DestinationWeaviate -from destination_weaviate.client import Client, WeaviatePartialBatchError -from destination_weaviate.utils import get_schema_from_catalog, stream_to_class_name - - -@pytest.fixture(name="config") -def config_fixture() -> Mapping[str, Any]: - with open("integration_tests/example-config.json", "r") as f: - return json.loads(f.read()) - - -def create_catalog(stream_name: str, stream_schema: Mapping[str, Any], - sync_mode: DestinationSyncMode = DestinationSyncMode.append) -> ConfiguredAirbyteCatalog: - append_stream = ConfiguredAirbyteStream( - stream=AirbyteStream(name=stream_name, json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), - sync_mode=SyncMode.incremental, - destination_sync_mode=sync_mode, - ) - return ConfiguredAirbyteCatalog(streams=[append_stream]) - - -@pytest.fixture(name="article_catalog") -def article_catalog_fixture() -> ConfiguredAirbyteCatalog: - stream_schema = {"type": "object", "properties": {"title": {"type": "str"}, "wordCount": {"type": "integer"}}} - - append_stream = ConfiguredAirbyteStream( - stream=AirbyteStream(name="Article", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), - sync_mode=SyncMode.incremental, - destination_sync_mode=DestinationSyncMode.append, - ) - - return ConfiguredAirbyteCatalog(streams=[append_stream]) - - -def load_json_file(path: str) -> Mapping: - dirname = os.path.dirname(__file__) - file = open(os.path.join(dirname, path)) - return json.load(file) - - -@pytest.fixture(name="pokemon_catalog") -def pokemon_catalog_fixture() -> ConfiguredAirbyteCatalog: - stream_schema = load_json_file("pokemon-schema.json") - - append_stream = ConfiguredAirbyteStream( - stream=AirbyteStream(name="pokemon", json_schema=stream_schema, supported_sync_modes=[SyncMode.incremental]), - sync_mode=SyncMode.incremental, - destination_sync_mode=DestinationSyncMode.append, - ) - - return ConfiguredAirbyteCatalog(streams=[append_stream]) - - -@pytest.fixture(autouse=True) -def setup_teardown(config: Mapping): - env_vars = { - "QUERY_DEFAULTS_LIMIT": "25", - "AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED": "true", - "DEFAULT_VECTORIZER_MODULE": "none", - "CLUSTER_HOSTNAME": "node1", - "PERSISTENCE_DATA_PATH": "./data" - } - name = "weaviate-test-container-will-get-deleted" - docker_client = docker.from_env() - try: - docker_client.containers.get(name).remove(force=True) - except docker.errors.NotFound: - pass - - docker_client.containers.run( - "semitechnologies/weaviate:1.17.3", detach=True, environment=env_vars, name=name, - ports={8080: ('127.0.0.1', 8081)} - ) - time.sleep(0.5) - - retries = 3 - while retries > 0: +import weaviate +from airbyte_cdk.destinations.vector_db_based.embedder import OPEN_AI_VECTOR_SIZE +from airbyte_cdk.destinations.vector_db_based.test_utils import BaseIntegrationTest +from airbyte_cdk.models import DestinationSyncMode, Status +from destination_weaviate.destination import DestinationWeaviate +from langchain.embeddings import OpenAIEmbeddings +from langchain.vectorstores import Weaviate +from pytest_docker.plugin import get_docker_ip + +WEAVIATE_CONTAINER_NAME = "weaviate-test-container-will-get-deleted" + + +class WeaviateIntegrationTest(BaseIntegrationTest): + def _init_weaviate(self): + env_vars = { + "QUERY_DEFAULTS_LIMIT": "25", + "AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED": "true", + "DEFAULT_VECTORIZER_MODULE": "none", + "CLUSTER_HOSTNAME": "node1", + "PERSISTENCE_DATA_PATH": "./data", + } + self.docker_client = docker.from_env() try: - Client(config, {}) - break - except Exception as e: - logging.info(f"error connecting to weaviate with client. Retrying in 1 second. Exception: {e}") - time.sleep(1) - retries -= 1 - - yield - docker_client.containers.get(name).remove(force=True) - - -@pytest.fixture(name="client") -def client_fixture(config) -> Client: - return Client(config, {}) - - -def test_check_valid_config(config: Mapping): - outcome = DestinationWeaviate().check(AirbyteLogger(), config) - assert outcome.status == Status.SUCCEEDED - - -def test_check_invalid_config(): - outcome = DestinationWeaviate().check(AirbyteLogger(), {"url": "localhost:6666"}) - assert outcome.status == Status.FAILED - - -def _state(data: Dict[str, Any]) -> AirbyteMessage: - return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data=data)) - - -def _record(stream: str, data: Mapping[str, Any]) -> AirbyteMessage: - return AirbyteMessage( - type=Type.RECORD, record=AirbyteRecordMessage(stream=stream, data=data, emitted_at=0) - ) - - -def _article_record(stream: str, title: str, word_count: int) -> AirbyteMessage: - return AirbyteMessage( - type=Type.RECORD, record=AirbyteRecordMessage(stream=stream, data={"title": title, "wordCount": word_count}, emitted_at=0) - ) - - -def _pikachu_record(): - data = load_json_file("pokemon-pikachu.json") - return AirbyteMessage(type=Type.RECORD, record=AirbyteRecordMessage(stream="pokemon", data=data, emitted_at=0)) - - -def _record_with_id(stream: str, title: str, word_count: int, id: int) -> AirbyteMessage: - return AirbyteMessage( - type=Type.RECORD, record=AirbyteRecordMessage(stream=stream, data={ - "title": title, - "wordCount": word_count, - "id": id - }, emitted_at=0) - ) - - -def retrieve_all_articles(client: Client) -> List[AirbyteRecordMessage]: - """retrieves and formats all Articles as Airbyte messages""" - all_records = client.client.data_object.get(class_name="Article", ) - out = [] - for record in all_records.get("objects"): - props = record["properties"] - out.append(_article_record("Article", props["title"], props["wordCount"])) - out.sort(key=lambda x: x.record.data.get("title")) - return out - - -def get_objects(client: Client, class_name: str) -> List[Mapping[str, Any]]: - """retrieves and formats all Articles as Airbyte messages""" - all_records = client.client.data_object.get(class_name=class_name, with_vector=True) - return all_records.get("objects") - - -def count_objects(client: Client, class_name: str) -> int: - result = client.client.query.aggregate(class_name) \ - .with_fields('meta { count }') \ - .do() - return result["data"]["Aggregate"][class_name][0]["meta"]["count"] - - -def test_write(config: Mapping, article_catalog: ConfiguredAirbyteCatalog, client: Client): - append_stream = article_catalog.streams[0].stream.name - first_state_message = _state({"state": "1"}) - first_record_chunk = [_article_record(append_stream, str(i), i) for i in range(5)] - - destination = DestinationWeaviate() - - expected_states = [first_state_message] - output_states = list( - destination.write( - config, article_catalog, [*first_record_chunk, first_state_message] - ) - ) - assert expected_states == output_states, "Checkpoint state messages were expected from the destination" - - expected_records = [_article_record(append_stream, str(i), i) for i in range(5)] - records_in_destination = retrieve_all_articles(client) - assert expected_records == records_in_destination, "Records in destination should match records expected" - - -def test_write_large_batch(config: Mapping, article_catalog: ConfiguredAirbyteCatalog, client: Client): - append_stream = article_catalog.streams[0].stream.name - first_state_message = _state({"state": "1"}) - first_record_chunk = [_article_record(append_stream, str(i), i) for i in range(400)] - - destination = DestinationWeaviate() - - expected_states = [first_state_message] - output_states = list( - destination.write( - config, article_catalog, [*first_record_chunk, first_state_message] - ) - ) - assert expected_states == output_states, "Checkpoint state messages were expected from the destination" - assert count_objects(client, "Article") == 400, "There should be 400 records in weaviate" - - -def test_write_second_sync(config: Mapping, article_catalog: ConfiguredAirbyteCatalog, client: Client): - append_stream = article_catalog.streams[0].stream.name - first_state_message = _state({"state": "1"}) - second_state_message = _state({"state": "2"}) - first_record_chunk = [_article_record(append_stream, str(i), i) for i in range(5)] - - destination = DestinationWeaviate() - - expected_states = [first_state_message, second_state_message] - output_states = list( - destination.write( - config, article_catalog, [*first_record_chunk, first_state_message, *first_record_chunk, second_state_message] - ) - ) - assert expected_states == output_states, "Checkpoint state messages were expected from the destination" - assert count_objects(client, "Article") == 10, "First and second state should have flushed a total of 10 articles" - - -def test_line_break_characters(config: Mapping, client: Client): - stream_name = "currency" - stream_schema = load_json_file("exchange_rate_catalog.json") - catalog = create_catalog(stream_name, stream_schema) - first_state_message = _state({"state": "1"}) - data = {"id": 1, "currency": "USD\u2028", "date": "2020-03-\n31T00:00:00Z\r", "HKD": 10.1, "NZD": 700.1} - first_record_chunk = [_record(stream_name, data)] - - destination = DestinationWeaviate() - - expected_states = [first_state_message] - output_states = list( - destination.write( - config, catalog, [*first_record_chunk, first_state_message] - ) - ) - assert expected_states == output_states, "Checkpoint state messages were expected from the destination" - assert count_objects(client, "Currency") == 1, "There should be only 1 object of class currency in Weaviate" - actual = get_objects(client, "Currency")[0] - assert actual["properties"].get("date") == data.get("date"), "Dates with new line should match" - assert actual["properties"].get("hKD") == data.get("HKD"), "HKD should match hKD in Weaviate" - assert actual["properties"].get("nZD") == data.get("NZD") - assert actual["properties"].get("currency") == data.get("currency") - - -def test_write_id(config: Mapping, article_catalog: ConfiguredAirbyteCatalog, client: Client): - """ - This test verifies that records can have an ID that's an integer - """ - append_stream = article_catalog.streams[0].stream.name - first_state_message = _state({"state": "1"}) - first_record_chunk = [_record_with_id(append_stream, str(i), i, i) for i in range(1, 6)] - - destination = DestinationWeaviate() - - expected_states = [first_state_message] - output_states = list( - destination.write( - config, article_catalog, [*first_record_chunk, first_state_message] - ) - ) - assert expected_states == output_states, "Checkpoint state messages were expected from the destination" - - records_in_destination = retrieve_all_articles(client) - assert len(records_in_destination) == 5, "Expecting there should be 5 records" - - expected_records = [_article_record(append_stream, str(i), i) for i in range(1, 6)] - for expected, actual in zip(expected_records, records_in_destination): - assert expected.record.data.get("title") == actual.record.data.get("title"), "Titles should match" - assert expected.record.data.get("wordCount") == actual.record.data.get("wordCount"), "Titles should match" - - -def test_write_pokemon_source_pikachu(config: Mapping, pokemon_catalog: ConfiguredAirbyteCatalog, client: Client): - destination = DestinationWeaviate() - - first_state_message = _state({"state": "1"}) - pikachu = _pikachu_record() - output_states = list( - destination.write( - config, pokemon_catalog, [pikachu, first_state_message] - ) - ) - - expected_states = [first_state_message] - assert expected_states == output_states, "Checkpoint state messages were expected from the destination" - - records_in_destination = get_objects(client, "Pokemon") - assert len(records_in_destination) == 1, "Expecting there should be 1 record" - - actual = records_in_destination[0] - assert actual["properties"]["name"] == pikachu.record.data.get("name"), "Names should match" - - -def test_upload_vector(config: Mapping, client: Client): - stream_name = "article_with_vector" - stream_schema = {"type": "object", "properties": { - "title": {"type": "string"}, - "vector": {"type": "array", "items": {"type": "number}"}} - }} - catalog = create_catalog(stream_name, stream_schema) - first_state_message = _state({"state": "1"}) - data = {"title": "test1", "vector": [0.1, 0.2]} - first_record_chunk = [_record(stream_name, data)] - - destination = DestinationWeaviate() - config["vectors"] = "article_with_vector.vector" - - expected_states = [first_state_message] - output_states = list( - destination.write( - config, catalog, [*first_record_chunk, first_state_message] - ) - ) - assert expected_states == output_states, "Checkpoint state messages were expected from the destination" - - class_name = stream_to_class_name(stream_name) - assert count_objects(client, class_name) == 1, "There should be only 1 object of in Weaviate" - actual = get_objects(client, class_name)[0] - assert actual.get("vector") == data.get("vector"), "Vectors should match" - - -def test_weaviate_existing_class(config: Mapping, client: Client): - class_obj = { - "class": "Article", - "properties": [ - {"dataType": ["string"], "name": "title"}, - {"dataType": ["text"], "name": "content"} - ] - } - client.client.schema.create_class(class_obj) - stream_name = "article" - stream_schema = {"type": "object", "properties": { - "title": {"type": "string"}, - "text": {"type": "string"} - }} - catalog = create_catalog(stream_name, stream_schema) - first_state_message = _state({"state": "1"}) - data = {"title": "test1", "content": "test 1 content"} - first_record_chunk = [_record(stream_name, data)] - - destination = DestinationWeaviate() - expected_states = [first_state_message] - output_states = list( - destination.write( - config, catalog, [*first_record_chunk, first_state_message] - ) - ) - assert expected_states == output_states, "Checkpoint state messages were expected from the destination" - - class_name = stream_to_class_name(stream_name) - assert count_objects(client, class_name) == 1, "There should be only 1 object of in Weaviate" - actual = get_objects(client, class_name)[0] - assert actual["properties"].get("title") == data.get("title"), "Title should match" - assert actual["properties"].get("content") == data.get("content"), "Content should match" - - -def test_id_starting_with_underscore(config: Mapping, client: Client): - # This is common scenario from mongoDB - stream_name = "article" - stream_schema = {"type": "object", "properties": { - "_id": {"type": "integer"}, - "title": {"type": "string"} - }} - catalog = create_catalog(stream_name, stream_schema) - first_state_message = _state({"state": "1"}) - data = {"_id": "507f191e810c19729de860ea", "title": "test1"} - first_record_chunk = [_record(stream_name, data)] - - destination = DestinationWeaviate() - - expected_states = [first_state_message] - output_states = list( - destination.write( - config, catalog, [*first_record_chunk, first_state_message] - ) - ) - assert expected_states == output_states, "Checkpoint state messages were expected from the destination" - - class_name = stream_to_class_name(stream_name) - assert count_objects(client, class_name) == 1, "There should be only 1 object of in Weaviate" - actual = get_objects(client, class_name)[0] - assert actual.get("id") == str(uuid.UUID(int=int(data.get("_id"), 16))), "UUID should be created for _id field" - - -def test_id_with_text_string(config: Mapping, client: Client): - stream_name = "article" - stream_schema = {"type": "object", "properties": { - "title": {"type": "string"}, - "id": {"type": "string"} - }} - catalog = create_catalog(stream_name, stream_schema) - first_state_message = _state({"state": "1"}) - data = {"title": "test1", "id": "not a real id"} - first_record_chunk = [_record(stream_name, data)] - - destination = DestinationWeaviate() - - expected_states = [first_state_message] - output_states = list( - destination.write( - config, catalog, [*first_record_chunk, first_state_message] - ) - ) - assert expected_states == output_states, "Checkpoint state messages were expected from the destination" - - class_name = stream_to_class_name(stream_name) - assert count_objects(client, class_name) == 1, "There should be only 1 object of in Weaviate" - actual = get_objects(client, class_name)[0] - assert actual.get("id") - - -def test_array_no_item_type(config: Mapping, client: Client): - stream_name = "article" - stream_schema = {"type": "object", "properties": { - "arr": {"type": "array", "items": {}}, - }} - catalog = create_catalog(stream_name, stream_schema) - first_state_message = _state({"state": "1"}) - data = {"arr": {"test1": "test"}} - first_record_chunk = [_record(stream_name, data)] - - destination = DestinationWeaviate() - - expected_states = [first_state_message] - output_states = list( - destination.write( - config, catalog, [*first_record_chunk, first_state_message] - ) - ) - assert expected_states == output_states, "Checkpoint state messages were expected from the destination" - - class_name = stream_to_class_name(stream_name) - assert count_objects(client, class_name) == 1, "There should be only 1 object of in Weaviate" - actual = get_objects(client, class_name)[0] - assert actual["properties"].get("arr") == json.dumps(data["arr"]) - - -def test_array_of_objects_empty(config: Mapping, client: Client): - stream_name = "article" - stream_schema = {"type": "object", "properties": { - "arr": {"type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": {"name": {"type": ["null", "string"]}}} - }, - }} - catalog = create_catalog(stream_name, stream_schema) - first_state_message = _state({"state": "1"}) - data = {"arr": [{}]} - first_record_chunk = [_record(stream_name, data)] - - destination = DestinationWeaviate() - - expected_states = [first_state_message] - output_states = list( - destination.write( - config, catalog, [*first_record_chunk, first_state_message] - ) - ) - assert expected_states == output_states, "Checkpoint state messages were expected from the destination" - - class_name = stream_to_class_name(stream_name) - assert count_objects(client, class_name) == 1, "There should be only 1 object of in Weaviate" - actual = get_objects(client, class_name)[0] - assert actual["properties"].get("arr") == '[{}]' - - -def test_missing_fields(config: Mapping, client: Client): - stream_name = "article" - stream_schema = {"type": "object", "properties": { - "title": {"type": "string"}, - "arr": {"type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": {"name": {"type": ["null", "string"]}}} - }, - }} - catalog = create_catalog(stream_name, stream_schema) - first_state_message = _state({"state": "1"}) - data = {"title": "test-missing-array"} - first_record_chunk = [_record(stream_name, data)] - - destination = DestinationWeaviate() - - expected_states = [first_state_message] - output_states = list( - destination.write( - config, catalog, [*first_record_chunk, first_state_message] - ) - ) - assert expected_states == output_states, "Checkpoint state messages were expected from the destination" - - class_name = stream_to_class_name(stream_name) - assert count_objects(client, class_name) == 1, "There should be only 1 object of in Weaviate" - actual = get_objects(client, class_name)[0] - assert actual["properties"].get("arr") is None - - -def test_record_additional_properties(config: Mapping, client: Client): - stream_name = "article" - stream_schema = { - "type": "object", "additionalProperties": True, - "properties": {"title": {"type": "string"}} - } - catalog = create_catalog(stream_name, stream_schema) - first_state_message = _state({"state": "1"}) - first_record_chunk = [_record(stream_name, {"title": "a-first-record"})] - - destination = DestinationWeaviate() - - expected_states = [first_state_message] - output_states = list( - destination.write( - config, catalog, [*first_record_chunk, first_state_message] - ) - ) - assert expected_states == output_states, "Checkpoint state messages were expected from the destination" - class_name = stream_to_class_name(stream_name) - assert count_objects(client, class_name) == 1, "There should be only 1 object of in Weaviate" - - data = {"title": "with-add-props", "add_prop": "test", "add_prop2": ["test"], - "objArray": [{"title": "sam"}], "obj": {"title": "sam"}} - second_record_chunk = [_record(stream_name, data)] - second_state_message = _state({"state": 2}) - expected_states = [second_state_message] - output_states = list( - destination.write( - config, catalog, [*second_record_chunk, second_state_message] - ) - ) - assert expected_states == output_states, "Checkpoint state messages were expected from the destination" - - assert count_objects(client, class_name) == 2 - actual = sorted(get_objects(client, class_name), key=lambda x: x["properties"]["title"])[1] - assert actual["properties"].get("title") == data["title"] - assert actual["properties"].get("add_prop") == data["add_prop"] - - -def test_id_custom_field_name(config: Mapping, client: Client): - # This is common scenario from mongoDB - stream_name = "article" - stream_schema = {"type": "object", "properties": { - "my_id": {"type": "integer"}, - "title": {"type": "string"} - }} - catalog = create_catalog(stream_name, stream_schema) - first_state_message = _state({"state": "1"}) - data = {"my_id": "507f191e810c19729de860ea", "title": "test_id_schema"} - first_record_chunk = [_record(stream_name, data)] - - destination = DestinationWeaviate() - config["id_schema"] = "article.my_id" - - expected_states = [first_state_message] - output_states = list( - destination.write( - config, catalog, [*first_record_chunk, first_state_message] - ) - ) - assert expected_states == output_states, "Checkpoint state messages were expected from the destination" - - class_name = stream_to_class_name(stream_name) - assert count_objects(client, class_name) == 1, "There should be only 1 object of in Weaviate" - actual = get_objects(client, class_name)[0] - assert actual.get("id") == str(uuid.UUID(int=int(data.get("my_id"), 16))), "UUID should be created for my_id field" - - -def test_write_overwrite(config: Mapping, client: Client): - stream_name = "article" - stream_schema = {"type": "object", "properties": { - "title": {"type": "string"}, - "text": {"type": "string"} - }} - catalog = create_catalog(stream_name, stream_schema, sync_mode=DestinationSyncMode.overwrite) - first_state_message = _state({"state": "1"}) - data = {"title": "test1", "content": "test 1 content"} - first_record_chunk = [_record(stream_name, data), _record(stream_name, data)] - - destination = DestinationWeaviate() - expected_states = [first_state_message] - output_states = list( - destination.write( - config, catalog, [*first_record_chunk, first_state_message] - ) - ) - assert expected_states == output_states, "Checkpoint state messages were expected from the destination" - class_name = stream_to_class_name(stream_name) - assert count_objects(client, class_name) == 2 - - # After writing a 2nd time the existing 2 objects should be gone and there should only be 1 new object - second_state_message = _state({"state": "2"}) - second_record_chunk = [_record(stream_name, data)] - expected_states = [second_state_message] - output_states = list( - destination.write( - config, catalog, [*second_record_chunk, second_state_message] - ) - ) - - assert expected_states == output_states, "Checkpoint state messages were expected from the destination" - assert count_objects(client, class_name) == 1 - - -def test_client_delete_stream_entries(caplog, client: Client): - client.delete_stream_entries("doesnotexist") - assert "Class Doesnotexist did not exist." in caplog.text, "Should be a log entry that says class doesn't exist" - - class_obj = { - "class": "Article", - "properties": [ - {"dataType": ["string"], "name": "title", "moduleConfig": { - "text2vec-contextionary": { - "vectorizePropertyName": True - } - }}, - {"dataType": ["text"], "name": "content"} - ] - } - client.client.schema.create_class(class_obj) - client.client.data_object.create({"title": "test-deleted", "content": "test-deleted"}, "Article") - client.delete_stream_entries("article") - assert count_objects(client, "Article") == 0, "Ensure articles have been deleted however class was recreated" - actual_schema = client.client.schema.get("Article") - title_prop = next(filter(lambda x: x["name"] == "title", actual_schema["properties"])) - assert title_prop["moduleConfig"]["text2vec-contextionary"]["vectorizePropertyName"] is True, "Ensure moduleconfig is persisted" - - -def test_client_flush_partial_error(client: Client): - stream_name = "article" - stream_schema = {"type": "object", "properties": { - "id": {"type": "string"}, - "title": {"type": "string"}, - }} - catalog = create_catalog(stream_name, stream_schema, sync_mode=DestinationSyncMode.overwrite) - client.schema = get_schema_from_catalog(catalog) - partial_error_result = load_json_file("create_objects_partial_error.json") - client.client.batch.create_objects = Mock(return_value=partial_error_result) - time.sleep = Mock(return_value=None) - client.buffered_write_operation("article", {"id": "b7b1cfbe-20da-496c-b932-008d35805f26", "title": "test1"}) - client.buffered_write_operation("article", {"id": "154cbccd-89f4-4b29-9c1b-001a3339d89a", "title": "test2"}) - with pytest.raises(WeaviatePartialBatchError): - client.flush() + self.docker_client.containers.get(WEAVIATE_CONTAINER_NAME).remove(force=True) + except docker.errors.NotFound: + pass + + self.docker_client.containers.run( + "semitechnologies/weaviate:1.21.2", + detach=True, + environment=env_vars, + name=WEAVIATE_CONTAINER_NAME, + ports={8080: ("0.0.0.0", 8081)}, + ) + time.sleep(0.5) + docker_ip = get_docker_ip() + self.config["indexing"]["host"] = f"http://{docker_ip}:8081" + + retries = 10 + while retries > 0: + try: + self.client = weaviate.Client(url=self.config["indexing"]["host"]) + break + except Exception as e: + logging.info(f"error connecting to weaviate with indexer. Retrying in 1 second. Exception: {e}") + time.sleep(1) + retries -= 1 + + def setUp(self): + with open("secrets/config.json", "r") as f: + self.config = json.loads(f.read()) + self._init_weaviate() + + def tearDown(self) -> None: + self.docker_client.containers.get(WEAVIATE_CONTAINER_NAME).remove(force=True) + + def test_check_valid_config(self): + outcome = DestinationWeaviate().check(logging.getLogger("airbyte"), self.config) + assert outcome.status == Status.SUCCEEDED + + def test_check_invalid_config(self): + outcome = DestinationWeaviate().check( + logging.getLogger("airbyte"), + {**self.config, "indexing": {**self.config["indexing"], "host": "http://localhost:9999"}}, + ) + assert outcome.status == Status.FAILED + + def count_objects(self, class_name: str) -> int: + result = self.client.query.aggregate(class_name).with_fields("meta { count }").do() + return result["data"]["Aggregate"][class_name][0]["meta"]["count"] + + def test_write_overwrite(self): + catalog = self._get_configured_catalog(DestinationSyncMode.overwrite) + first_state_message = self._state({"state": "1"}) + first_record_chunk = [self._record("mystream", f"Dogs are number {i}", i) for i in range(5)] + + destination = DestinationWeaviate() + list(destination.write(self.config, catalog, [*first_record_chunk, first_state_message])) + assert self.count_objects("Mystream") == 5 + + second_record_chunk = [self._record("mystream", f"Dogs are number {i}", i + 1000) for i in range(2)] + list(destination.write(self.config, catalog, [*second_record_chunk, first_state_message])) + assert self.count_objects("Mystream") == 2 + + def test_write_incremental_dedup(self): + catalog = self._get_configured_catalog(DestinationSyncMode.append_dedup) + first_state_message = self._state({"state": "1"}) + first_record_chunk = [self._record("mystream", f"Dogs are number {i}", i) for i in range(5)] + + # initial sync + destination = DestinationWeaviate() + list(destination.write(self.config, catalog, [*first_record_chunk, first_state_message])) + assert self.count_objects("Mystream") == 5 + + # incrementalally update a doc + incremental_catalog = self._get_configured_catalog(DestinationSyncMode.append_dedup) + list(destination.write(self.config, incremental_catalog, [self._record("mystream", "Cats are nice", 2), first_state_message])) + result = ( + self.client.query.get("Mystream", ["text"]) + .with_near_vector({"vector": [0] * OPEN_AI_VECTOR_SIZE}) + .with_where({"path": ["_ab_record_id"], "operator": "Equal", "valueText": "mystream_2"}) + .do() + ) + + assert len(result["data"]["Get"]["Mystream"]) == 1 + assert self.count_objects("Mystream") == 5 + assert result["data"]["Get"]["Mystream"][0]["text"] == "str_col: Cats are nice" + + # test langchain integration + embeddings = OpenAIEmbeddings(openai_api_key=self.config["embedding"]["openai_key"]) + vs = Weaviate( + embedding=embeddings, + by_text=False, + client=self.client, + text_key="text", + index_name="Mystream", + attributes=["_ab_record_id"], + ) + + result = vs.similarity_search("feline animals", 1) + assert result[0].metadata["_ab_record_id"] == "mystream_2" diff --git a/airbyte-integrations/connectors/destination-weaviate/integration_tests/pokemon-pikachu.json b/airbyte-integrations/connectors/destination-weaviate/integration_tests/pokemon-pikachu.json deleted file mode 100644 index 264481d37723..000000000000 --- a/airbyte-integrations/connectors/destination-weaviate/integration_tests/pokemon-pikachu.json +++ /dev/null @@ -1,11125 +0,0 @@ -{ - "abilities": [ - { - "ability": { - "name": "static", - "url": "https://pokeapi.co/api/v2/ability/9/" - }, - "is_hidden": false, - "slot": 1 - }, - { - "ability": { - "name": "lightning-rod", - "url": "https://pokeapi.co/api/v2/ability/31/" - }, - "is_hidden": true, - "slot": 3 - } - ], - "base_experience": 112, - "forms": [ - { "name": "pikachu", "url": "https://pokeapi.co/api/v2/pokemon-form/25/" } - ], - "game_indices": [ - { - "game_index": 84, - "version": { - "name": "red", - "url": "https://pokeapi.co/api/v2/version/1/" - } - }, - { - "game_index": 84, - "version": { - "name": "blue", - "url": "https://pokeapi.co/api/v2/version/2/" - } - }, - { - "game_index": 84, - "version": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version/3/" - } - }, - { - "game_index": 25, - "version": { - "name": "gold", - "url": "https://pokeapi.co/api/v2/version/4/" - } - }, - { - "game_index": 25, - "version": { - "name": "silver", - "url": "https://pokeapi.co/api/v2/version/5/" - } - }, - { - "game_index": 25, - "version": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version/6/" - } - }, - { - "game_index": 25, - "version": { - "name": "ruby", - "url": "https://pokeapi.co/api/v2/version/7/" - } - }, - { - "game_index": 25, - "version": { - "name": "sapphire", - "url": "https://pokeapi.co/api/v2/version/8/" - } - }, - { - "game_index": 25, - "version": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version/9/" - } - }, - { - "game_index": 25, - "version": { - "name": "firered", - "url": "https://pokeapi.co/api/v2/version/10/" - } - }, - { - "game_index": 25, - "version": { - "name": "leafgreen", - "url": "https://pokeapi.co/api/v2/version/11/" - } - }, - { - "game_index": 25, - "version": { - "name": "diamond", - "url": "https://pokeapi.co/api/v2/version/12/" - } - }, - { - "game_index": 25, - "version": { - "name": "pearl", - "url": "https://pokeapi.co/api/v2/version/13/" - } - }, - { - "game_index": 25, - "version": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version/14/" - } - }, - { - "game_index": 25, - "version": { - "name": "heartgold", - "url": "https://pokeapi.co/api/v2/version/15/" - } - }, - { - "game_index": 25, - "version": { - "name": "soulsilver", - "url": "https://pokeapi.co/api/v2/version/16/" - } - }, - { - "game_index": 25, - "version": { - "name": "black", - "url": "https://pokeapi.co/api/v2/version/17/" - } - }, - { - "game_index": 25, - "version": { - "name": "white", - "url": "https://pokeapi.co/api/v2/version/18/" - } - }, - { - "game_index": 25, - "version": { - "name": "black-2", - "url": "https://pokeapi.co/api/v2/version/21/" - } - }, - { - "game_index": 25, - "version": { - "name": "white-2", - "url": "https://pokeapi.co/api/v2/version/22/" - } - } - ], - "height": 4, - "held_items": [ - { - "item": { - "name": "oran-berry", - "url": "https://pokeapi.co/api/v2/item/132/" - }, - "version_details": [ - { - "rarity": 50, - "version": { - "name": "ruby", - "url": "https://pokeapi.co/api/v2/version/7/" - } - }, - { - "rarity": 50, - "version": { - "name": "sapphire", - "url": "https://pokeapi.co/api/v2/version/8/" - } - }, - { - "rarity": 50, - "version": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version/9/" - } - }, - { - "rarity": 50, - "version": { - "name": "diamond", - "url": "https://pokeapi.co/api/v2/version/12/" - } - }, - { - "rarity": 50, - "version": { - "name": "pearl", - "url": "https://pokeapi.co/api/v2/version/13/" - } - }, - { - "rarity": 50, - "version": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version/14/" - } - }, - { - "rarity": 50, - "version": { - "name": "heartgold", - "url": "https://pokeapi.co/api/v2/version/15/" - } - }, - { - "rarity": 50, - "version": { - "name": "soulsilver", - "url": "https://pokeapi.co/api/v2/version/16/" - } - }, - { - "rarity": 50, - "version": { - "name": "black", - "url": "https://pokeapi.co/api/v2/version/17/" - } - }, - { - "rarity": 50, - "version": { - "name": "white", - "url": "https://pokeapi.co/api/v2/version/18/" - } - } - ] - }, - { - "item": { - "name": "light-ball", - "url": "https://pokeapi.co/api/v2/item/213/" - }, - "version_details": [ - { - "rarity": 5, - "version": { - "name": "ruby", - "url": "https://pokeapi.co/api/v2/version/7/" - } - }, - { - "rarity": 5, - "version": { - "name": "sapphire", - "url": "https://pokeapi.co/api/v2/version/8/" - } - }, - { - "rarity": 5, - "version": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version/9/" - } - }, - { - "rarity": 5, - "version": { - "name": "diamond", - "url": "https://pokeapi.co/api/v2/version/12/" - } - }, - { - "rarity": 5, - "version": { - "name": "pearl", - "url": "https://pokeapi.co/api/v2/version/13/" - } - }, - { - "rarity": 5, - "version": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version/14/" - } - }, - { - "rarity": 5, - "version": { - "name": "heartgold", - "url": "https://pokeapi.co/api/v2/version/15/" - } - }, - { - "rarity": 5, - "version": { - "name": "soulsilver", - "url": "https://pokeapi.co/api/v2/version/16/" - } - }, - { - "rarity": 1, - "version": { - "name": "black", - "url": "https://pokeapi.co/api/v2/version/17/" - } - }, - { - "rarity": 1, - "version": { - "name": "white", - "url": "https://pokeapi.co/api/v2/version/18/" - } - }, - { - "rarity": 5, - "version": { - "name": "black-2", - "url": "https://pokeapi.co/api/v2/version/21/" - } - }, - { - "rarity": 5, - "version": { - "name": "white-2", - "url": "https://pokeapi.co/api/v2/version/22/" - } - }, - { - "rarity": 5, - "version": { - "name": "x", - "url": "https://pokeapi.co/api/v2/version/23/" - } - }, - { - "rarity": 5, - "version": { - "name": "y", - "url": "https://pokeapi.co/api/v2/version/24/" - } - }, - { - "rarity": 5, - "version": { - "name": "omega-ruby", - "url": "https://pokeapi.co/api/v2/version/25/" - } - }, - { - "rarity": 5, - "version": { - "name": "alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version/26/" - } - }, - { - "rarity": 5, - "version": { - "name": "sun", - "url": "https://pokeapi.co/api/v2/version/27/" - } - }, - { - "rarity": 5, - "version": { - "name": "moon", - "url": "https://pokeapi.co/api/v2/version/28/" - } - }, - { - "rarity": 5, - "version": { - "name": "ultra-sun", - "url": "https://pokeapi.co/api/v2/version/29/" - } - }, - { - "rarity": 5, - "version": { - "name": "ultra-moon", - "url": "https://pokeapi.co/api/v2/version/30/" - } - } - ] - } - ], - "id": 25, - "is_default": true, - "location_area_encounters": "https://pokeapi.co/api/v2/pokemon/25/encounters", - "moves": [ - { - "move": { - "name": "mega-punch", - "url": "https://pokeapi.co/api/v2/move/5/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { "name": "pay-day", "url": "https://pokeapi.co/api/v2/move/6/" }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "thunder-punch", - "url": "https://pokeapi.co/api/v2/move/9/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { "name": "slam", "url": "https://pokeapi.co/api/v2/move/21/" }, - "version_group_details": [ - { - "level_learned_at": 20, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 20, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 20, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 20, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 20, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 20, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 21, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 21, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 21, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 26, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 20, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 20, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 26, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 26, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 37, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 37, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 37, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 24, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 28, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "double-kick", - "url": "https://pokeapi.co/api/v2/move/24/" - }, - "version_group_details": [ - { - "level_learned_at": 9, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - } - ] - }, - { - "move": { - "name": "mega-kick", - "url": "https://pokeapi.co/api/v2/move/25/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "headbutt", - "url": "https://pokeapi.co/api/v2/move/29/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - } - ] - }, - { - "move": { - "name": "body-slam", - "url": "https://pokeapi.co/api/v2/move/34/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "take-down", - "url": "https://pokeapi.co/api/v2/move/36/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - } - ] - }, - { - "move": { - "name": "double-edge", - "url": "https://pokeapi.co/api/v2/move/38/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - } - ] - }, - { - "move": { - "name": "tail-whip", - "url": "https://pokeapi.co/api/v2/move/39/" - }, - "version_group_details": [ - { - "level_learned_at": 6, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 6, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 6, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 6, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 6, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 6, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 5, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 5, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 5, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 5, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 6, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 6, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 5, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 3, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { "name": "growl", "url": "https://pokeapi.co/api/v2/move/45/" }, - "version_group_details": [ - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 5, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 5, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 5, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 5, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { "name": "surf", "url": "https://pokeapi.co/api/v2/move/57/" }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "stadium-surfing-pikachu", - "url": "https://pokeapi.co/api/v2/move-learn-method/5/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "stadium-surfing-pikachu", - "url": "https://pokeapi.co/api/v2/move-learn-method/5/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "submission", - "url": "https://pokeapi.co/api/v2/move/66/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - } - ] - }, - { - "move": { - "name": "counter", - "url": "https://pokeapi.co/api/v2/move/68/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - } - ] - }, - { - "move": { - "name": "seismic-toss", - "url": "https://pokeapi.co/api/v2/move/69/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - } - ] - }, - { - "move": { - "name": "strength", - "url": "https://pokeapi.co/api/v2/move/70/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - } - ] - }, - { - "move": { - "name": "thunder-shock", - "url": "https://pokeapi.co/api/v2/move/84/" - }, - "version_group_details": [ - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "thunderbolt", - "url": "https://pokeapi.co/api/v2/move/85/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 26, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 26, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 26, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 26, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 26, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 26, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 26, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 26, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 26, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 29, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 26, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 26, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 29, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 29, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 42, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 42, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 42, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 21, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 36, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "thunder-wave", - "url": "https://pokeapi.co/api/v2/move/86/" - }, - "version_group_details": [ - { - "level_learned_at": 9, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 8, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 8, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 8, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 8, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 8, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 8, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 10, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 10, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 10, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 10, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 8, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 8, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 10, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 13, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 18, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 18, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 18, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 15, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 4, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "thunder", - "url": "https://pokeapi.co/api/v2/move/87/" - }, - "version_group_details": [ - { - "level_learned_at": 43, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 41, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 41, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 41, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 41, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 41, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 41, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 45, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 45, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 45, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 50, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 41, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 41, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 50, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 50, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 58, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 58, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 58, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 30, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 44, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { "name": "dig", "url": "https://pokeapi.co/api/v2/move/91/" }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { "name": "toxic", "url": "https://pokeapi.co/api/v2/move/92/" }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - } - ] - }, - { - "move": { - "name": "agility", - "url": "https://pokeapi.co/api/v2/move/97/" - }, - "version_group_details": [ - { - "level_learned_at": 33, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 33, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 33, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 33, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 33, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 33, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 33, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 34, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 34, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 34, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 37, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 33, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 33, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 37, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 37, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 45, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 45, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 45, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 27, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 24, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "quick-attack", - "url": "https://pokeapi.co/api/v2/move/98/" - }, - "version_group_details": [ - { - "level_learned_at": 16, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 11, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 11, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 11, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 11, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 11, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 11, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 13, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 13, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 13, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 13, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 11, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 11, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 13, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 10, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 10, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 10, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 10, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 6, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { "name": "rage", "url": "https://pokeapi.co/api/v2/move/99/" }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - } - ] - }, - { - "move": { "name": "mimic", "url": "https://pokeapi.co/api/v2/move/102/" }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - } - ] - }, - { - "move": { - "name": "double-team", - "url": "https://pokeapi.co/api/v2/move/104/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 15, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 15, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 15, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 15, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 15, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 15, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 18, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 18, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 18, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 21, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 15, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 15, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 21, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 21, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 23, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 23, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 23, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 12, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 8, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "defense-curl", - "url": "https://pokeapi.co/api/v2/move/111/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - } - ] - }, - { - "move": { - "name": "light-screen", - "url": "https://pokeapi.co/api/v2/move/113/" - }, - "version_group_details": [ - { - "level_learned_at": 50, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 50, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 50, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 50, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 50, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 50, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 42, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 42, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 42, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 45, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 50, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 50, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 45, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 45, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 53, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 53, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 53, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 18, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 40, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "reflect", - "url": "https://pokeapi.co/api/v2/move/115/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { "name": "bide", "url": "https://pokeapi.co/api/v2/move/117/" }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - } - ] - }, - { - "move": { "name": "swift", "url": "https://pokeapi.co/api/v2/move/129/" }, - "version_group_details": [ - { - "level_learned_at": 26, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "skull-bash", - "url": "https://pokeapi.co/api/v2/move/130/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - } - ] - }, - { - "move": { "name": "flash", "url": "https://pokeapi.co/api/v2/move/148/" }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - } - ] - }, - { - "move": { "name": "rest", "url": "https://pokeapi.co/api/v2/move/156/" }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "substitute", - "url": "https://pokeapi.co/api/v2/move/164/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "red-blue", - "url": "https://pokeapi.co/api/v2/version-group/1/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "yellow", - "url": "https://pokeapi.co/api/v2/version-group/2/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { "name": "thief", "url": "https://pokeapi.co/api/v2/move/168/" }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { "name": "snore", "url": "https://pokeapi.co/api/v2/move/173/" }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { "name": "curse", "url": "https://pokeapi.co/api/v2/move/174/" }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - } - ] - }, - { - "move": { - "name": "reversal", - "url": "https://pokeapi.co/api/v2/move/179/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "protect", - "url": "https://pokeapi.co/api/v2/move/182/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "sweet-kiss", - "url": "https://pokeapi.co/api/v2/move/186/" - }, - "version_group_details": [ - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "mud-slap", - "url": "https://pokeapi.co/api/v2/move/189/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - } - ] - }, - { - "move": { - "name": "zap-cannon", - "url": "https://pokeapi.co/api/v2/move/192/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - } - ] - }, - { - "move": { - "name": "detect", - "url": "https://pokeapi.co/api/v2/move/197/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - } - ] - }, - { - "move": { - "name": "endure", - "url": "https://pokeapi.co/api/v2/move/203/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { "name": "charm", "url": "https://pokeapi.co/api/v2/move/204/" }, - "version_group_details": [ - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "rollout", - "url": "https://pokeapi.co/api/v2/move/205/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - } - ] - }, - { - "move": { - "name": "swagger", - "url": "https://pokeapi.co/api/v2/move/207/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - } - ] - }, - { - "move": { "name": "spark", "url": "https://pokeapi.co/api/v2/move/209/" }, - "version_group_details": [ - { - "level_learned_at": 26, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 26, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 26, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 20, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "attract", - "url": "https://pokeapi.co/api/v2/move/213/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "sleep-talk", - "url": "https://pokeapi.co/api/v2/move/214/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "return", - "url": "https://pokeapi.co/api/v2/move/216/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - } - ] - }, - { - "move": { - "name": "frustration", - "url": "https://pokeapi.co/api/v2/move/218/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - } - ] - }, - { - "move": { - "name": "dynamic-punch", - "url": "https://pokeapi.co/api/v2/move/223/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - } - ] - }, - { - "move": { - "name": "encore", - "url": "https://pokeapi.co/api/v2/move/227/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "iron-tail", - "url": "https://pokeapi.co/api/v2/move/231/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "hidden-power", - "url": "https://pokeapi.co/api/v2/move/237/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - } - ] - }, - { - "move": { - "name": "rain-dance", - "url": "https://pokeapi.co/api/v2/move/240/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "gold-silver", - "url": "https://pokeapi.co/api/v2/version-group/3/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "crystal", - "url": "https://pokeapi.co/api/v2/version-group/4/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "rock-smash", - "url": "https://pokeapi.co/api/v2/move/249/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - } - ] - }, - { - "move": { - "name": "uproar", - "url": "https://pokeapi.co/api/v2/move/253/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "facade", - "url": "https://pokeapi.co/api/v2/move/263/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "focus-punch", - "url": "https://pokeapi.co/api/v2/move/264/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - } - ] - }, - { - "move": { - "name": "helping-hand", - "url": "https://pokeapi.co/api/v2/move/270/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "brick-break", - "url": "https://pokeapi.co/api/v2/move/280/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "knock-off", - "url": "https://pokeapi.co/api/v2/move/282/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - } - ] - }, - { - "move": { - "name": "secret-power", - "url": "https://pokeapi.co/api/v2/move/290/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - } - ] - }, - { - "move": { - "name": "signal-beam", - "url": "https://pokeapi.co/api/v2/move/324/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - } - ] - }, - { - "move": { "name": "covet", "url": "https://pokeapi.co/api/v2/move/343/" }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - } - ] - }, - { - "move": { - "name": "volt-tackle", - "url": "https://pokeapi.co/api/v2/move/344/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - } - ] - }, - { - "move": { - "name": "calm-mind", - "url": "https://pokeapi.co/api/v2/move/347/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "lets-go-pikachu-lets-go-eevee", - "url": "https://pokeapi.co/api/v2/version-group/19/" - } - } - ] - }, - { - "move": { - "name": "shock-wave", - "url": "https://pokeapi.co/api/v2/move/351/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ruby-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/5/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "emerald", - "url": "https://pokeapi.co/api/v2/version-group/6/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "firered-leafgreen", - "url": "https://pokeapi.co/api/v2/version-group/7/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "colosseum", - "url": "https://pokeapi.co/api/v2/version-group/12/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "xd", - "url": "https://pokeapi.co/api/v2/version-group/13/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - } - ] - }, - { - "move": { - "name": "natural-gift", - "url": "https://pokeapi.co/api/v2/move/363/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - } - ] - }, - { - "move": { "name": "feint", "url": "https://pokeapi.co/api/v2/move/364/" }, - "version_group_details": [ - { - "level_learned_at": 29, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 29, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 29, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 34, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 34, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 34, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 21, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 21, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 21, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 16, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { "name": "fling", "url": "https://pokeapi.co/api/v2/move/374/" }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "magnet-rise", - "url": "https://pokeapi.co/api/v2/move/393/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - } - ] - }, - { - "move": { - "name": "nasty-plot", - "url": "https://pokeapi.co/api/v2/move/417/" - }, - "version_group_details": [ - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "discharge", - "url": "https://pokeapi.co/api/v2/move/435/" - }, - "version_group_details": [ - { - "level_learned_at": 37, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 37, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 37, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 42, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 42, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 42, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 34, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 34, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 34, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 32, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "captivate", - "url": "https://pokeapi.co/api/v2/move/445/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - } - ] - }, - { - "move": { - "name": "grass-knot", - "url": "https://pokeapi.co/api/v2/move/447/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "charge-beam", - "url": "https://pokeapi.co/api/v2/move/451/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "diamond-pearl", - "url": "https://pokeapi.co/api/v2/version-group/8/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "platinum", - "url": "https://pokeapi.co/api/v2/version-group/9/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "heartgold-soulsilver", - "url": "https://pokeapi.co/api/v2/version-group/10/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - } - ] - }, - { - "move": { - "name": "electro-ball", - "url": "https://pokeapi.co/api/v2/move/486/" - }, - "version_group_details": [ - { - "level_learned_at": 18, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 18, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 18, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 13, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 13, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 13, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 12, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { "name": "round", "url": "https://pokeapi.co/api/v2/move/496/" }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "echoed-voice", - "url": "https://pokeapi.co/api/v2/move/497/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - } - ] - }, - { - "move": { - "name": "volt-switch", - "url": "https://pokeapi.co/api/v2/move/521/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "electroweb", - "url": "https://pokeapi.co/api/v2/move/527/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "wild-charge", - "url": "https://pokeapi.co/api/v2/move/528/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-white", - "url": "https://pokeapi.co/api/v2/version-group/11/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "black-2-white-2", - "url": "https://pokeapi.co/api/v2/version-group/14/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 50, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 50, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 50, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "draining-kiss", - "url": "https://pokeapi.co/api/v2/move/577/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "play-rough", - "url": "https://pokeapi.co/api/v2/move/583/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "play-nice", - "url": "https://pokeapi.co/api/v2/move/589/" - }, - "version_group_details": [ - { - "level_learned_at": 7, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 7, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 7, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 7, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "confide", - "url": "https://pokeapi.co/api/v2/move/590/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - } - ] - }, - { - "move": { - "name": "electric-terrain", - "url": "https://pokeapi.co/api/v2/move/604/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "machine", - "url": "https://pokeapi.co/api/v2/move-learn-method/4/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "nuzzle", - "url": "https://pokeapi.co/api/v2/move/609/" - }, - "version_group_details": [ - { - "level_learned_at": 23, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "x-y", - "url": "https://pokeapi.co/api/v2/version-group/15/" - } - }, - { - "level_learned_at": 29, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "omega-ruby-alpha-sapphire", - "url": "https://pokeapi.co/api/v2/version-group/16/" - } - }, - { - "level_learned_at": 29, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sun-moon", - "url": "https://pokeapi.co/api/v2/version-group/17/" - } - }, - { - "level_learned_at": 29, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - }, - { - "level_learned_at": 1, - "move_learn_method": { - "name": "level-up", - "url": "https://pokeapi.co/api/v2/move-learn-method/1/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - }, - { - "move": { - "name": "laser-focus", - "url": "https://pokeapi.co/api/v2/move/673/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "ultra-sun-ultra-moon", - "url": "https://pokeapi.co/api/v2/version-group/18/" - } - } - ] - }, - { - "move": { - "name": "rising-voltage", - "url": "https://pokeapi.co/api/v2/move/804/" - }, - "version_group_details": [ - { - "level_learned_at": 0, - "move_learn_method": { - "name": "tutor", - "url": "https://pokeapi.co/api/v2/move-learn-method/3/" - }, - "version_group": { - "name": "sword-shield", - "url": "https://pokeapi.co/api/v2/version-group/20/" - } - } - ] - } - ], - "name": "pikachu", - "order": 35, - "past_types": [], - "species": { - "name": "pikachu", - "url": "https://pokeapi.co/api/v2/pokemon-species/25/" - }, - "sprites": { - "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/back/25.png", - "back_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/back/female/25.png", - "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/back/shiny/25.png", - "back_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/back/shiny/female/25.png", - "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/25.png", - "front_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/female/25.png", - "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/shiny/25.png", - "front_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/shiny/female/25.png", - "other": { - "dream_world": { - "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/dream-world/25.svg", - "front_female": null - }, - "home": { - "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/home/25.png", - "front_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/home/female/25.png", - "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/home/shiny/25.png", - "front_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/home/shiny/female/25.png" - }, - "official-artwork": { - "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/other/official-artwork/25.png" - } - }, - "versions": { - "generation-i": { - "red-blue": { - "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/back/25.png", - "back_gray": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/back/gray/25.png", - "back_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/transparent/back/25.png", - "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/25.png", - "front_gray": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/gray/25.png", - "front_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/red-blue/transparent/25.png" - }, - "yellow": { - "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/back/25.png", - "back_gray": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/back/gray/25.png", - "back_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/transparent/back/25.png", - "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/25.png", - "front_gray": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/gray/25.png", - "front_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-i/yellow/transparent/25.png" - } - }, - "generation-ii": { - "crystal": { - "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/back/25.png", - "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/back/shiny/25.png", - "back_shiny_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/transparent/back/shiny/25.png", - "back_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/transparent/back/25.png", - "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/25.png", - "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/shiny/25.png", - "front_shiny_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/transparent/shiny/25.png", - "front_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/crystal/transparent/25.png" - }, - "gold": { - "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/back/25.png", - "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/back/shiny/25.png", - "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/25.png", - "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/shiny/25.png", - "front_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/gold/transparent/25.png" - }, - "silver": { - "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/back/25.png", - "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/back/shiny/25.png", - "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/25.png", - "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/shiny/25.png", - "front_transparent": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-ii/silver/transparent/25.png" - } - }, - "generation-iii": { - "emerald": { - "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/emerald/25.png", - "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/emerald/shiny/25.png" - }, - "firered-leafgreen": { - "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/firered-leafgreen/back/25.png", - "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/firered-leafgreen/back/shiny/25.png", - "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/firered-leafgreen/25.png", - "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/firered-leafgreen/shiny/25.png" - }, - "ruby-sapphire": { - "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/ruby-sapphire/back/25.png", - "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/ruby-sapphire/back/shiny/25.png", - "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/ruby-sapphire/25.png", - "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iii/ruby-sapphire/shiny/25.png" - } - }, - "generation-iv": { - "diamond-pearl": { - "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/back/25.png", - "back_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/back/female/25.png", - "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/back/shiny/25.png", - "back_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/back/shiny/female/25.png", - "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/25.png", - "front_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/female/25.png", - "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/shiny/25.png", - "front_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/diamond-pearl/shiny/female/25.png" - }, - "heartgold-soulsilver": { - "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/back/25.png", - "back_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/back/female/25.png", - "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/back/shiny/25.png", - "back_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/back/shiny/female/25.png", - "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/25.png", - "front_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/female/25.png", - "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/shiny/25.png", - "front_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/heartgold-soulsilver/shiny/female/25.png" - }, - "platinum": { - "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/back/25.png", - "back_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/back/female/25.png", - "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/back/shiny/25.png", - "back_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/back/shiny/female/25.png", - "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/25.png", - "front_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/female/25.png", - "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/shiny/25.png", - "front_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-iv/platinum/shiny/female/25.png" - } - }, - "generation-v": { - "black-white": { - "animated": { - "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/back/25.gif", - "back_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/back/female/25.gif", - "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/back/shiny/25.gif", - "back_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/back/shiny/female/25.gif", - "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/25.gif", - "front_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/female/25.gif", - "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/shiny/25.gif", - "front_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/animated/shiny/female/25.gif" - }, - "back_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/back/25.png", - "back_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/back/female/25.png", - "back_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/back/shiny/25.png", - "back_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/back/shiny/female/25.png", - "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/25.png", - "front_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/female/25.png", - "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/shiny/25.png", - "front_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-v/black-white/shiny/female/25.png" - } - }, - "generation-vi": { - "omegaruby-alphasapphire": { - "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/omegaruby-alphasapphire/25.png", - "front_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/omegaruby-alphasapphire/female/25.png", - "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/omegaruby-alphasapphire/shiny/25.png", - "front_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/omegaruby-alphasapphire/shiny/female/25.png" - }, - "x-y": { - "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/x-y/25.png", - "front_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/x-y/female/25.png", - "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/x-y/shiny/25.png", - "front_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vi/x-y/shiny/female/25.png" - } - }, - "generation-vii": { - "icons": { - "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vii/icons/25.png", - "front_female": null - }, - "ultra-sun-ultra-moon": { - "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vii/ultra-sun-ultra-moon/25.png", - "front_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vii/ultra-sun-ultra-moon/female/25.png", - "front_shiny": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vii/ultra-sun-ultra-moon/shiny/25.png", - "front_shiny_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-vii/ultra-sun-ultra-moon/shiny/female/25.png" - } - }, - "generation-viii": { - "icons": { - "front_default": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-viii/icons/25.png", - "front_female": "https://raw.githubusercontent.com/PokeAPI/sprites/master/sprites/pokemon/versions/generation-viii/icons/female/25.png" - } - } - } - }, - "stats": [ - { - "base_stat": 35, - "effort": 0, - "stat": { "name": "hp", "url": "https://pokeapi.co/api/v2/stat/1/" } - }, - { - "base_stat": 55, - "effort": 0, - "stat": { "name": "attack", "url": "https://pokeapi.co/api/v2/stat/2/" } - }, - { - "base_stat": 40, - "effort": 0, - "stat": { "name": "defense", "url": "https://pokeapi.co/api/v2/stat/3/" } - }, - { - "base_stat": 50, - "effort": 0, - "stat": { - "name": "special-attack", - "url": "https://pokeapi.co/api/v2/stat/4/" - } - }, - { - "base_stat": 50, - "effort": 0, - "stat": { - "name": "special-defense", - "url": "https://pokeapi.co/api/v2/stat/5/" - } - }, - { - "base_stat": 90, - "effort": 2, - "stat": { "name": "speed", "url": "https://pokeapi.co/api/v2/stat/6/" } - } - ], - "types": [ - { - "slot": 1, - "type": { - "name": "electric", - "url": "https://pokeapi.co/api/v2/type/13/" - } - } - ], - "weight": 60 -} diff --git a/airbyte-integrations/connectors/destination-weaviate/integration_tests/pokemon-schema.json b/airbyte-integrations/connectors/destination-weaviate/integration_tests/pokemon-schema.json deleted file mode 100644 index 7c190d9cb6c4..000000000000 --- a/airbyte-integrations/connectors/destination-weaviate/integration_tests/pokemon-schema.json +++ /dev/null @@ -1,271 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "base_experience": { - "type": ["null", "integer"] - }, - "height": { - "type": ["null", "integer"] - }, - "is_default ": { - "type": ["null", "boolean"] - }, - "order": { - "type": ["null", "integer"] - }, - "weight": { - "type": ["null", "integer"] - }, - "abilities": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "is_hidden": { - "type": ["null", "boolean"] - }, - "slot": { - "type": ["null", "integer"] - }, - "ability": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - } - } - } - } - }, - "forms": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - } - } - }, - "game_indices": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "game_index": { - "type": ["null", "integer"] - }, - "version": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - } - } - } - } - }, - "held_items": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "item": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - } - }, - "version_details": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "version": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - } - }, - "rarity": { - "type": ["null", "integer"] - } - } - } - } - } - } - }, - "location_area_encounters": { - "type": ["null", "string"] - }, - "moves": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "move": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - } - }, - "version_group_details": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "move_learn_method": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - } - }, - "version_group": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - } - }, - "level_learned_at": { - "type": ["null", "integer"] - } - } - } - } - } - } - }, - "sprites": { - "type": ["null", "object"], - "properties": { - "front_default": { - "type": ["null", "string"] - }, - "front_shiny": { - "type": ["null", "string"] - }, - "front_female": { - "type": ["null", "string"] - }, - "front_shiny_female": { - "type": ["null", "string"] - }, - "back_default": { - "type": ["null", "string"] - }, - "back_shiny": { - "type": ["null", "string"] - }, - "back_female": { - "type": ["null", "string"] - }, - "back_shiny_female": { - "type": ["null", "string"] - } - } - }, - "species": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - } - }, - "stats": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "stat": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - } - }, - "effort": { - "type": ["null", "integer"] - }, - "base_stat": { - "type": ["null", "integer"] - } - } - } - }, - "types": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "slot": { - "type": ["null", "integer"] - }, - "type": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - } - } - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/destination-weaviate/integration_tests/spec.json b/airbyte-integrations/connectors/destination-weaviate/integration_tests/spec.json new file mode 100644 index 000000000000..a5db30c7213d --- /dev/null +++ b/airbyte-integrations/connectors/destination-weaviate/integration_tests/spec.json @@ -0,0 +1,531 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/destinations/weaviate", + "connectionSpecification": { + "title": "Destination Config", + "description": "The configuration model for the Vector DB based destinations. This model is used to generate the UI for the destination configuration,\nas well as to provide type safety for the configuration passed to the destination.\n\nThe configuration model is composed of four parts:\n* Processing configuration\n* Embedding configuration\n* Indexing configuration\n* Advanced configuration\n\nProcessing, embedding and advanced configuration are provided by this base class, while the indexing configuration is provided by the destination connector in the sub class.", + "type": "object", + "properties": { + "embedding": { + "title": "Embedding", + "description": "Embedding configuration", + "group": "embedding", + "type": "object", + "oneOf": [ + { + "title": "No external embedding", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "no_embedding", + "const": "no_embedding", + "enum": ["no_embedding"], + "type": "string" + } + }, + "description": "Do not calculate and pass embeddings to Weaviate. Suitable for clusters with configured vectorizers to calculate embeddings within Weaviate or for classes that should only support regular text search.", + "required": ["mode"] + }, + { + "title": "Azure OpenAI", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "azure_openai", + "const": "azure_openai", + "enum": ["azure_openai"], + "type": "string" + }, + "openai_key": { + "title": "Azure OpenAI API key", + "description": "The API key for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", + "airbyte_secret": true, + "type": "string" + }, + "api_base": { + "title": "Resource base URL", + "description": "The base URL for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", + "examples": ["https://your-resource-name.openai.azure.com"], + "type": "string" + }, + "deployment": { + "title": "Deployment", + "description": "The deployment for your Azure OpenAI resource. You can find this in the Azure portal under your Azure OpenAI resource", + "examples": ["your-resource-name"], + "type": "string" + } + }, + "required": ["openai_key", "api_base", "deployment", "mode"], + "description": "Use the Azure-hosted OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." + }, + { + "title": "OpenAI", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "openai", + "const": "openai", + "enum": ["openai"], + "type": "string" + }, + "openai_key": { + "title": "OpenAI API key", + "airbyte_secret": true, + "type": "string" + } + }, + "required": ["openai_key", "mode"], + "description": "Use the OpenAI API to embed text. This option is using the text-embedding-ada-002 model with 1536 embedding dimensions." + }, + { + "title": "Cohere", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "cohere", + "const": "cohere", + "enum": ["cohere"], + "type": "string" + }, + "cohere_key": { + "title": "Cohere API key", + "airbyte_secret": true, + "type": "string" + } + }, + "required": ["cohere_key", "mode"], + "description": "Use the Cohere API to embed text." + }, + { + "title": "From Field", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "from_field", + "const": "from_field", + "enum": ["from_field"], + "type": "string" + }, + "field_name": { + "title": "Field name", + "description": "Name of the field in the record that contains the embedding", + "examples": ["embedding", "vector"], + "type": "string" + }, + "dimensions": { + "title": "Embedding dimensions", + "description": "The number of dimensions the embedding model is generating", + "examples": [1536, 384], + "type": "integer" + } + }, + "required": ["field_name", "dimensions", "mode"], + "description": "Use a field in the record as the embedding. This is useful if you already have an embedding for your data and want to store it in the vector store." + }, + { + "title": "Fake", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "fake", + "const": "fake", + "enum": ["fake"], + "type": "string" + } + }, + "description": "Use a fake embedding made out of random vectors with 1536 embedding dimensions. This is useful for testing the data pipeline without incurring any costs.", + "required": ["mode"] + }, + { + "title": "OpenAI-compatible", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "openai_compatible", + "const": "openai_compatible", + "enum": ["openai_compatible"], + "type": "string" + }, + "api_key": { + "title": "API key", + "default": "", + "airbyte_secret": true, + "type": "string" + }, + "base_url": { + "title": "Base URL", + "description": "The base URL for your OpenAI-compatible service", + "examples": ["https://your-service-name.com"], + "type": "string" + }, + "model_name": { + "title": "Model name", + "description": "The name of the model to use for embedding", + "default": "text-embedding-ada-002", + "examples": ["text-embedding-ada-002"], + "type": "string" + }, + "dimensions": { + "title": "Embedding dimensions", + "description": "The number of dimensions the embedding model is generating", + "examples": [1536, 384], + "type": "integer" + } + }, + "required": ["base_url", "dimensions", "mode"], + "description": "Use a service that's compatible with the OpenAI API to embed text." + } + ] + }, + "processing": { + "title": "ProcessingConfigModel", + "type": "object", + "properties": { + "chunk_size": { + "title": "Chunk size", + "description": "Size of chunks in tokens to store in vector store (make sure it is not too big for the context if your LLM)", + "maximum": 8191, + "minimum": 1, + "type": "integer" + }, + "chunk_overlap": { + "title": "Chunk overlap", + "description": "Size of overlap between chunks in tokens to store in vector store to better capture relevant context", + "default": 0, + "type": "integer" + }, + "text_fields": { + "title": "Text fields to embed", + "description": "List of fields in the record that should be used to calculate the embedding. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered text fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array.", + "default": [], + "always_show": true, + "examples": ["text", "user.name", "users.*.name"], + "type": "array", + "items": { + "type": "string" + } + }, + "metadata_fields": { + "title": "Fields to store as metadata", + "description": "List of fields in the record that should be stored as metadata. The field list is applied to all streams in the same way and non-existing fields are ignored. If none are defined, all fields are considered metadata fields. When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. When specifying nested paths, all matching values are flattened into an array set to a field named by the path.", + "default": [], + "always_show": true, + "examples": ["age", "user", "user.name"], + "type": "array", + "items": { + "type": "string" + } + }, + "text_splitter": { + "title": "Text splitter", + "description": "Split text fields into chunks based on the specified method.", + "type": "object", + "oneOf": [ + { + "title": "By Separator", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "separator", + "const": "separator", + "enum": ["separator"], + "type": "string" + }, + "separators": { + "title": "Separators", + "description": "List of separator strings to split text fields by. The separator itself needs to be wrapped in double quotes, e.g. to split by the dot character, use \".\". To split by a newline, use \"\\n\".", + "default": ["\"\\n\\n\"", "\"\\n\"", "\" \"", "\"\""], + "type": "array", + "items": { + "type": "string" + } + }, + "keep_separator": { + "title": "Keep separator", + "description": "Whether to keep the separator in the resulting chunks", + "default": false, + "type": "boolean" + } + }, + "description": "Split the text by the list of separators until the chunk size is reached, using the earlier mentioned separators where possible. This is useful for splitting text fields by paragraphs, sentences, words, etc.", + "required": ["mode"] + }, + { + "title": "By Markdown header", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "markdown", + "const": "markdown", + "enum": ["markdown"], + "type": "string" + }, + "split_level": { + "title": "Split level", + "description": "Level of markdown headers to split text fields by. Headings down to the specified level will be used as split points", + "default": 1, + "minimum": 1, + "maximum": 6, + "type": "integer" + } + }, + "description": "Split the text by Markdown headers down to the specified header level. If the chunk size fits multiple sections, they will be combined into a single chunk.", + "required": ["mode"] + }, + { + "title": "By Programming Language", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "code", + "const": "code", + "enum": ["code"], + "type": "string" + }, + "language": { + "title": "Language", + "description": "Split code in suitable places based on the programming language", + "enum": [ + "cpp", + "go", + "java", + "js", + "php", + "proto", + "python", + "rst", + "ruby", + "rust", + "scala", + "swift", + "markdown", + "latex", + "html", + "sol" + ], + "type": "string" + } + }, + "required": ["language", "mode"], + "description": "Split the text by suitable delimiters based on the programming language. This is useful for splitting code into chunks." + } + ] + }, + "field_name_mappings": { + "title": "Field name mappings", + "description": "List of fields to rename. Not applicable for nested fields, but can be used to rename fields already flattened via dot notation.", + "default": [], + "type": "array", + "items": { + "title": "FieldNameMappingConfigModel", + "type": "object", + "properties": { + "from_field": { + "title": "From field name", + "description": "The field name in the source", + "type": "string" + }, + "to_field": { + "title": "To field name", + "description": "The field name to use in the destination", + "type": "string" + } + }, + "required": ["from_field", "to_field"] + } + } + }, + "required": ["chunk_size"], + "group": "processing" + }, + "omit_raw_text": { + "title": "Do not store raw text", + "description": "Do not store the text that gets embedded along with the vector and the metadata in the destination. If set to true, only the vector and the metadata will be stored - in this case raw text for LLM use cases needs to be retrieved from another source.", + "default": false, + "group": "advanced", + "type": "boolean" + }, + "indexing": { + "title": "Indexing", + "type": "object", + "properties": { + "host": { + "title": "Public Endpoint", + "description": "The public endpoint of the Weaviate cluster.", + "order": 1, + "examples": ["https://my-cluster.weaviate.network"], + "type": "string" + }, + "auth": { + "title": "Authentication", + "description": "Authentication method", + "type": "object", + "order": 2, + "oneOf": [ + { + "title": "API Token", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "token", + "const": "token", + "enum": ["token"], + "type": "string" + }, + "token": { + "title": "API Token", + "description": "API Token for the Weaviate instance", + "airbyte_secret": true, + "type": "string" + } + }, + "required": ["token", "mode"], + "description": "Authenticate using an API token (suitable for Weaviate Cloud)" + }, + { + "title": "Username/Password", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "username_password", + "const": "username_password", + "enum": ["username_password"], + "type": "string" + }, + "username": { + "title": "Username", + "description": "Username for the Weaviate cluster", + "order": 1, + "type": "string" + }, + "password": { + "title": "Password", + "description": "Password for the Weaviate cluster", + "airbyte_secret": true, + "order": 2, + "type": "string" + } + }, + "required": ["username", "password", "mode"], + "description": "Authenticate using username and password (suitable for self-managed Weaviate clusters)" + }, + { + "title": "No Authentication", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "no_auth", + "const": "no_auth", + "enum": ["no_auth"], + "type": "string" + } + }, + "description": "Do not authenticate (suitable for locally running test clusters, do not use for clusters with public IP addresses)", + "required": ["mode"] + } + ] + }, + "batch_size": { + "title": "Batch Size", + "description": "The number of records to send to Weaviate in each batch", + "default": 128, + "type": "integer" + }, + "text_field": { + "title": "Text Field", + "description": "The field in the object that contains the embedded text", + "default": "text", + "type": "string" + }, + "tenant_id": { + "title": "Tenant ID", + "description": "The tenant ID to use for multi tenancy", + "airbyte_secret": true, + "default": "", + "type": "string" + }, + "default_vectorizer": { + "title": "Default Vectorizer", + "description": "The vectorizer to use if new classes need to be created", + "default": "none", + "enum": [ + "none", + "text2vec-cohere", + "text2vec-huggingface", + "text2vec-openai", + "text2vec-palm", + "text2vec-contextionary", + "text2vec-transformers", + "text2vec-gpt4all" + ], + "type": "string" + }, + "additional_headers": { + "title": "Additional headers", + "description": "Additional HTTP headers to send with every request.", + "default": [], + "examples": [ + { + "header_key": "X-OpenAI-Api-Key", + "value": "my-openai-api-key" + } + ], + "type": "array", + "items": { + "title": "Header", + "type": "object", + "properties": { + "header_key": { + "title": "Header Key", + "type": "string" + }, + "value": { + "title": "Header Value", + "airbyte_secret": true, + "type": "string" + } + }, + "required": ["header_key", "value"] + } + } + }, + "required": ["host", "auth"], + "group": "indexing", + "description": "Indexing configuration" + } + }, + "required": ["embedding", "processing", "indexing"], + "groups": [ + { + "id": "processing", + "title": "Processing" + }, + { + "id": "embedding", + "title": "Embedding" + }, + { + "id": "indexing", + "title": "Indexing" + }, + { + "id": "advanced", + "title": "Advanced" + } + ] + }, + "supportsIncremental": true, + "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"] +} diff --git a/airbyte-integrations/connectors/destination-weaviate/metadata.yaml b/airbyte-integrations/connectors/destination-weaviate/metadata.yaml index c23934d934d0..29aeefaf0831 100644 --- a/airbyte-integrations/connectors/destination-weaviate/metadata.yaml +++ b/airbyte-integrations/connectors/destination-weaviate/metadata.yaml @@ -1,24 +1,47 @@ data: - connectorSubtype: database + ab_internal: + ql: 300 + sl: 200 + allowedHosts: + hosts: + - ${indexing.host} + - api.openai.com + - api.cohere.ai + - ${embedding.api_base} + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + connectorSubtype: vectorstore connectorType: destination definitionId: 7b7d7a0d-954c-45a0-bcfc-39a634b97736 - dockerImageTag: 0.1.1 + dockerImageTag: 0.2.14 dockerRepository: airbyte/destination-weaviate + documentationUrl: https://docs.airbyte.com/integrations/destinations/weaviate githubIssueLabel: destination-weaviate icon: weaviate.svg license: MIT name: Weaviate registries: cloud: - enabled: false + enabled: true oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/destinations/weaviate + releases: + breakingChanges: + 0.2.0: + message: + "After upgrading, you need to reconfigure the source. For more details + check out the migration guide. + + " + upgradeDeadline: "2023-10-01" + resourceRequirements: + jobSpecific: + - jobType: sync + resourceRequirements: + memory_limit: 2Gi + memory_request: 2Gi + supportLevel: certified tags: - language:python - ab_internal: - sl: 100 - ql: 100 - supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/destination-weaviate/setup.py b/airbyte-integrations/connectors/destination-weaviate/setup.py index b1a2ccba8938..ca4aa59495dd 100644 --- a/airbyte-integrations/connectors/destination-weaviate/setup.py +++ b/airbyte-integrations/connectors/destination-weaviate/setup.py @@ -5,9 +5,9 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk", "weaviate-client==3.11.0"] +MAIN_REQUIREMENTS = ["airbyte-cdk[vector-db-based]==0.57.0", "weaviate-client==3.25.2"] -TEST_REQUIREMENTS = ["pytest~=6.2", "docker"] +TEST_REQUIREMENTS = ["pytest~=6.2", "docker", "pytest-docker"] setup( name="destination_weaviate", diff --git a/airbyte-integrations/connectors/destination-weaviate/unit_tests/destination_test.py b/airbyte-integrations/connectors/destination-weaviate/unit_tests/destination_test.py new file mode 100644 index 000000000000..b1e329500714 --- /dev/null +++ b/airbyte-integrations/connectors/destination-weaviate/unit_tests/destination_test.py @@ -0,0 +1,92 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import unittest +from unittest.mock import MagicMock, Mock, patch + +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.models import ConnectorSpecification, Status +from destination_weaviate.config import ConfigModel +from destination_weaviate.destination import DestinationWeaviate + + +class TestDestinationWeaviate(unittest.TestCase): + def setUp(self): + self.config = { + "processing": {"text_fields": ["str_col"], "metadata_fields": [], "chunk_size": 1000}, + "embedding": {"mode": "openai", "openai_key": "mykey"}, + "indexing": {"host": "https://my-cluster.weaviate.network", "auth": {"mode": "no_auth"}}, + } + self.config_model = ConfigModel.parse_obj(self.config) + self.logger = AirbyteLogger() + + @patch("destination_weaviate.destination.WeaviateIndexer") + @patch("destination_weaviate.destination.create_from_config") + def test_check(self, MockedEmbedder, MockedWeaviateIndexer): + mock_embedder = Mock() + mock_indexer = Mock() + MockedEmbedder.return_value = mock_embedder + MockedWeaviateIndexer.return_value = mock_indexer + + mock_embedder.check.return_value = None + mock_indexer.check.return_value = None + + destination = DestinationWeaviate() + result = destination.check(self.logger, self.config) + + self.assertEqual(result.status, Status.SUCCEEDED) + mock_embedder.check.assert_called_once() + mock_indexer.check.assert_called_once() + + @patch("destination_weaviate.destination.WeaviateIndexer") + @patch("destination_weaviate.destination.create_from_config") + def test_check_with_errors(self, MockedEmbedder, MockedWeaviateIndexer): + mock_embedder = Mock() + mock_indexer = Mock() + MockedEmbedder.return_value = mock_embedder + MockedWeaviateIndexer.return_value = mock_indexer + + embedder_error_message = "Embedder Error" + indexer_error_message = "Indexer Error" + + mock_embedder.check.return_value = embedder_error_message + mock_indexer.check.return_value = indexer_error_message + + destination = DestinationWeaviate() + result = destination.check(self.logger, self.config) + + self.assertEqual(result.status, Status.FAILED) + self.assertEqual(result.message, f"{embedder_error_message}\n{indexer_error_message}") + + mock_embedder.check.assert_called_once() + mock_indexer.check.assert_called_once() + + @patch("destination_weaviate.destination.Writer") + @patch("destination_weaviate.destination.WeaviateIndexer") + @patch("destination_weaviate.destination.create_from_config") + def test_write(self, MockedEmbedder, MockedWeaviateIndexer, MockedWriter): + mock_embedder = Mock() + mock_indexer = Mock() + MockedEmbedder.return_value = mock_embedder + mock_writer = Mock() + + MockedWeaviateIndexer.return_value = mock_indexer + MockedWriter.return_value = mock_writer + + mock_writer.write.return_value = [] + + configured_catalog = MagicMock() + input_messages = [] + + destination = DestinationWeaviate() + list(destination.write(self.config, configured_catalog, input_messages)) + + MockedWriter.assert_called_once_with(self.config_model.processing, mock_indexer, mock_embedder, batch_size=128, omit_raw_text=False) + mock_writer.write.assert_called_once_with(configured_catalog, input_messages) + + def test_spec(self): + destination = DestinationWeaviate() + result = destination.spec() + + self.assertIsInstance(result, ConnectorSpecification) diff --git a/airbyte-integrations/connectors/destination-weaviate/unit_tests/indexer_test.py b/airbyte-integrations/connectors/destination-weaviate/unit_tests/indexer_test.py new file mode 100644 index 000000000000..a5b2526e392c --- /dev/null +++ b/airbyte-integrations/connectors/destination-weaviate/unit_tests/indexer_test.py @@ -0,0 +1,278 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import unittest +from collections import defaultdict +from unittest.mock import ANY, Mock, call, patch + +from airbyte_cdk.destinations.vector_db_based.document_processor import Chunk +from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage, DestinationSyncMode +from destination_weaviate.config import NoAuth, TokenAuth, WeaviateIndexingConfigModel +from destination_weaviate.indexer import WeaviateIndexer, WeaviatePartialBatchError + + +class TestWeaviateIndexer(unittest.TestCase): + def setUp(self): + self.config = WeaviateIndexingConfigModel( + host="https://test-host:12345", auth=TokenAuth(mode="token", token="abc") + ) # Setup your config here + self.indexer = WeaviateIndexer(self.config) + mock_catalog = Mock() + mock_stream = Mock() + mock_stream.stream.name = "test" + mock_stream.destination_sync_mode = DestinationSyncMode.append + self.mock_stream = mock_stream + mock_catalog.streams = [mock_stream] + self.mock_catalog = mock_catalog + + @patch("destination_weaviate.indexer.weaviate.Client") + def test_successful_check(self, MockClient): + self.assertIsNone(self.indexer.check()) + + @patch("destination_weaviate.indexer.weaviate.Client") + def test_failed_check_due_to_exception(self, MockClient): + MockClient.side_effect = Exception("Random exception") + self.assertIsNotNone(self.indexer.check()) + + @patch("destination_weaviate.indexer.os.environ") + def test_failed_check_due_to_cloud_env_and_no_https_host(self, mock_os_environ): + mock_os_environ.get.return_value = "cloud" + self.indexer.config.host = "http://example.com" + self.assertEqual(self.indexer.check(), "Host must start with https:// and authentication must be enabled on cloud deployment.") + + @patch("destination_weaviate.indexer.os.environ") + def test_failed_check_due_to_cloud_env_and_no_auth(self, mock_os_environ): + mock_os_environ.get.return_value = "cloud" + self.indexer.config.host = "http://example.com" + self.indexer.config.auth = NoAuth(mode="no_auth") + self.assertEqual(self.indexer.check(), "Host must start with https:// and authentication must be enabled on cloud deployment.") + + @patch("destination_weaviate.indexer.weaviate.Client") + def test_pre_sync_that_creates_class(self, MockClient): + mock_client = Mock() + mock_client.schema.get.return_value = {"classes": []} + MockClient.return_value = mock_client + self.indexer.pre_sync(self.mock_catalog) + mock_client.schema.create_class.assert_called_with( + { + "class": "Test", + "vectorizer": "none", + "properties": [ + { + "name": "_ab_record_id", + "dataType": ["text"], + "description": "Record ID, used for bookkeeping.", + "indexFilterable": True, + "indexSearchable": False, + "tokenization": "field", + } + ], + } + ) + + @patch("destination_weaviate.indexer.weaviate.Client") + def test_pre_sync_that_creates_class_with_multi_tenancy_enabled(self, MockClient): + mock_client = Mock() + self.config.tenant_id = "test_tenant" + mock_client.schema.get_class_tenants.return_value = [] + mock_client.schema.get.return_value = {"classes": []} + MockClient.return_value = mock_client + self.indexer.pre_sync(self.mock_catalog) + mock_client.schema.create_class.assert_called_with( + { + "class": "Test", + "multiTenancyConfig": {"enabled": True}, + "vectorizer": "none", + "properties": [ + { + "name": "_ab_record_id", + "dataType": ["text"], + "description": "Record ID, used for bookkeeping.", + "indexFilterable": True, + "indexSearchable": False, + "tokenization": "field", + } + ], + } + ) + + @patch("destination_weaviate.indexer.weaviate.Client") + def test_pre_sync_that_deletes(self, MockClient): + mock_client = Mock() + mock_client.schema.get.return_value = { + "classes": [{"class": "Test", "properties": [{"name": "_ab_stream"}, {"name": "_ab_record_id"}]}] + } + MockClient.return_value = mock_client + self.mock_stream.destination_sync_mode = DestinationSyncMode.overwrite + self.indexer.pre_sync(self.mock_catalog) + mock_client.schema.delete_class.assert_called_with(class_name="Test") + mock_client.schema.create_class.assert_called_with(mock_client.schema.get.return_value["classes"][0]) + + @patch("destination_weaviate.indexer.weaviate.Client") + def test_pre_sync_no_delete_no_overwrite_mode(self, MockClient): + mock_client = Mock() + mock_client.schema.get.return_value = { + "classes": [{"class": "Test", "properties": [{"name": "_ab_stream"}, {"name": "_ab_record_id"}]}] + } + MockClient.return_value = mock_client + self.indexer.pre_sync(self.mock_catalog) + mock_client.schema.delete_class.assert_not_called() + + def test_index_deletes_by_record_id(self): + mock_client = Mock() + self.indexer.client = mock_client + self.indexer.has_record_id_metadata = defaultdict(None) + self.indexer.has_record_id_metadata["Test"] = True + self.indexer.delete(["some_id", "some_other_id"], None, "test") + mock_client.batch.delete_objects.assert_called_with( + class_name="Test", + where={"path": ["_ab_record_id"], "operator": "ContainsAny", "valueStringArray": ["some_id", "some_other_id"]}, + ) + + def test_index_deletes_by_record_id_with_tenant_id(self): + mock_client = Mock() + self.config.tenant_id = "test_tenant" + self.indexer.client = mock_client + self.indexer.has_record_id_metadata = defaultdict(None) + self.indexer.has_record_id_metadata["Test"] = True + self.indexer.delete(["some_id", "some_other_id"], None, "test") + mock_client.batch.delete_objects.assert_called_with( + class_name="Test", + tenant="test_tenant", + where={"path": ["_ab_record_id"], "operator": "ContainsAny", "valueStringArray": ["some_id", "some_other_id"]}, + ) + + @patch("destination_weaviate.indexer.weaviate.Client") + def test_index_not_delete_no_metadata_field(self, MockClient): + mock_client = Mock() + MockClient.return_value = mock_client + self.indexer.has_record_id_metadata = defaultdict(None) + self.indexer.has_record_id_metadata["Test"] = False + self.indexer.delete(["some_id"], None, "test") + mock_client.batch.delete_objects.assert_not_called() + + def test_index_flushes_batch(self): + mock_client = Mock() + self.indexer.client = mock_client + mock_client.batch.create_objects.return_value = [] + mock_chunk1 = Chunk( + page_content="some_content", + embedding=[1, 2, 3], + metadata={"someField": "some_value"}, + record=AirbyteRecordMessage(stream="test", data={"someField": "some_value"}, emitted_at=0), + ) + mock_chunk2 = Chunk( + page_content="some_other_content", + embedding=[4, 5, 6], + metadata={"someField": "some_value2"}, + record=AirbyteRecordMessage(stream="test", data={"someField": "some_value"}, emitted_at=0), + ) + self.indexer.index([mock_chunk1, mock_chunk2], None, "test") + mock_client.batch.create_objects.assert_called() + chunk1_call = call({"someField": "some_value", "text": "some_content"}, "Test", ANY, vector=[1, 2, 3]) + chunk2_call = call({"someField": "some_value2", "text": "some_other_content"}, "Test", ANY, vector=[4, 5, 6]) + mock_client.batch.add_data_object.assert_has_calls([chunk1_call, chunk2_call], any_order=False) + + def test_index_splits_batch(self): + mock_client = Mock() + self.indexer.client = mock_client + mock_client.batch.create_objects.return_value = [] + self.indexer.config.batch_size = 2 + mock_chunk1 = Chunk( + page_content="some_content", + embedding=[1, 2, 3], + metadata={"someField": "some_value"}, + record=AirbyteRecordMessage(stream="test", data={"someField": "some_value"}, emitted_at=0), + ) + mock_chunk2 = Chunk( + page_content="some_other_content", + embedding=[4, 5, 6], + metadata={"someField": "some_value2"}, + record=AirbyteRecordMessage(stream="test", data={"someField": "some_value2"}, emitted_at=0), + ) + mock_chunk3 = Chunk( + page_content="third", + embedding=[7, 8, 9], + metadata={"someField": "some_value3"}, + record=AirbyteRecordMessage(stream="test", data={"someField": "some_value3"}, emitted_at=0), + ) + self.indexer.index([mock_chunk1, mock_chunk2, mock_chunk3], None, "test") + assert mock_client.batch.create_objects.call_count == 2 + + def test_index_on_empty_batch(self): + mock_client = Mock() + self.indexer.client = mock_client + self.indexer.index([], None, "test") + assert mock_client.batch.create_objects.call_count == 0 + + @patch("destination_weaviate.indexer.uuid.uuid4") + @patch("time.sleep", return_value=None) + def test_index_flushes_batch_and_propagates_error(self, MockTime, MockUUID): + mock_client = Mock() + self.indexer.client = mock_client + mock_client.batch.create_objects.return_value = [{"result": {"errors": ["some_error"]}, "id": "some_id"}] + MockUUID.side_effect = ["some_id", "some_id2"] + mock_chunk1 = Chunk( + page_content="some_content", + embedding=[1, 2, 3], + metadata={"someField": "some_value"}, + record=AirbyteRecordMessage(stream="test", data={"someField": "some_value"}, emitted_at=0), + ) + mock_chunk2 = Chunk( + page_content="some_other_content", + embedding=[4, 5, 6], + metadata={"someField": "some_value2"}, + record=AirbyteRecordMessage(stream="test", data={"someField": "some_value"}, emitted_at=0), + ) + with self.assertRaises(WeaviatePartialBatchError): + self.indexer.index([mock_chunk1, mock_chunk2], None, "test") + chunk1_call = call({"someField": "some_value", "text": "some_content"}, "Test", "some_id", vector=[1, 2, 3]) + self.assertEqual(mock_client.batch.create_objects.call_count, 1) + mock_client.batch.add_data_object.assert_has_calls([chunk1_call], any_order=False) + + def test_index_flushes_batch_and_normalizes(self): + mock_client = Mock() + self.indexer.client = mock_client + mock_client.batch.create_objects.return_value = [] + mock_chunk = Chunk( + page_content="some_content", + embedding=[1, 2, 3], + metadata={ + "someField": "some_value", + "complex": {"a": [1, 2, 3]}, + "UPPERCASE_NAME": "abc", + "id": 12, + "empty_list": [], + "referral Agency Name": "test1", + "123StartsWithNumber": "test2", + "special&*chars": "test3", + "with spaces": "test4", + "": "test5", + "_startsWithUnderscore": "test6", + "multiple spaces": "test7", + "SpecialCharacters!@#": "test8", + }, + record=AirbyteRecordMessage(stream="test", data={"someField": "some_value"}, emitted_at=0), + ) + self.indexer.index([mock_chunk], None, "test") + mock_client.batch.add_data_object.assert_called_with( + { + "someField": "some_value", + "complex": '{"a": [1, 2, 3]}', + "uPPERCASE_NAME": "abc", + "text": "some_content", + "raw_id": 12, + "referral_Agency_Name": "test1", + "_123StartsWithNumber": "test2", + "specialchars": "test3", + "with_spaces": "test4", + "_": "test5", + "_startsWithUnderscore": "test6", + "multiple__spaces": "test7", + "specialCharacters": "test8", + }, + "Test", + ANY, + vector=[1, 2, 3], + ) diff --git a/airbyte-integrations/connectors/destination-weaviate/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-weaviate/unit_tests/unit_test.py deleted file mode 100644 index dc007d7d6199..000000000000 --- a/airbyte-integrations/connectors/destination-weaviate/unit_tests/unit_test.py +++ /dev/null @@ -1,49 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import uuid -from unittest.mock import Mock - -from destination_weaviate.client import Client -from destination_weaviate.utils import generate_id, stream_to_class_name - - -def test_client_custom_vectors_config(): - mock_object = Client - mock_object.get_weaviate_client = Mock(return_value=None) - c = Client({"vectors": "my_table.test", "url": "http://test"}, schema={}) - assert c.vectors["my_table"] == "test", "Single vector should work" - - c = Client({"vectors": "case2.test, another_table.vector", "url": "http://test"}, schema={}) - assert c.vectors["case2"] == "test", "Multiple values case2 should work too" - assert c.vectors["another_table"] == "vector", "Multiple values another_table should work too" - - -def test_client_custom_id_schema_config(): - mock_object = Client - mock_object.get_weaviate_client = Mock(return_value=None) - c = Client({"id_schema": "my_table.my_id", "url": "http://test"}, schema={}) - assert c.id_schema["my_table"] == "my_id", "Single id_schema definition should work" - - c = Client({"id_schema": "my_table.my_id, another_table.my_id2", "url": "http://test"}, schema={}) - assert c.id_schema["my_table"] == "my_id", "Multiple values should work too" - assert c.id_schema["another_table"] == "my_id2", "Multiple values should work too" - - -def test_utils_stream_name_to_class_name(): - assert stream_to_class_name("s-a") == "Sa" - assert stream_to_class_name("s_a") == "S_a" - assert stream_to_class_name("s _ a") == "S_a" - assert stream_to_class_name("s{} _ a") == "S_a" - assert stream_to_class_name("s{} _ aA") == "S_aA" - - -def test_generate_id(): - assert generate_id("1") == uuid.UUID(int=1) - assert generate_id("0x1") == uuid.UUID(int=1) - assert generate_id(1) == uuid.UUID(int=1) - assert generate_id("123e4567-e89b-12d3-a456-426614174000") == uuid.UUID("123e4567-e89b-12d3-a456-426614174000") - assert generate_id("123e4567e89b12d3a456426614174000") == uuid.UUID("123e4567-e89b-12d3-a456-426614174000") - for i in range(10): - assert generate_id("this should be using md5") == uuid.UUID("802a479a-190e-92c8-8340-d687c860f53d") diff --git a/airbyte-integrations/connectors/destination-xata/README.md b/airbyte-integrations/connectors/destination-xata/README.md index 59d4bbd09b13..e6153ac20ba1 100644 --- a/airbyte-integrations/connectors/destination-xata/README.md +++ b/airbyte-integrations/connectors/destination-xata/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:destination-xata:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/destinations/xata) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_xata/spec.json` file. @@ -54,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/destination-xata:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=destination-xata build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:destination-xata:airbyteDocker +An image will be built with the tag `airbyte/destination-xata:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/destination-xata:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,38 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-xata:dev check -- # messages.jsonl is a file containing line-separated JSON representing AirbyteMessages cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-xata:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all destination connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=destination-xata test ``` -#### Acceptance Tests -Coming soon: -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:destination-xata:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:destination-xata:integrationTest -``` +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -116,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-xata test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/xata.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-xata/build.gradle b/airbyte-integrations/connectors/destination-xata/build.gradle deleted file mode 100644 index f52818d7ca2f..000000000000 --- a/airbyte-integrations/connectors/destination-xata/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'destination_xata' -} diff --git a/airbyte-integrations/connectors/destination-xata/integration_tests/integration_test.py b/airbyte-integrations/connectors/destination-xata/integration_tests/integration_test.py index 8aa903b715ba..b98d151d31d3 100644 --- a/airbyte-integrations/connectors/destination-xata/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/destination-xata/integration_tests/integration_test.py @@ -69,12 +69,19 @@ def test_write(config: Mapping): destination_sync_mode=DestinationSyncMode.append, ) - records = [AirbyteMessage( - type=Type.RECORD, record=AirbyteRecordMessage(stream="test_stream", data={ - "str_col": "example", - "int_col": 1, - }, emitted_at=0) - )] + records = [ + AirbyteMessage( + type=Type.RECORD, + record=AirbyteRecordMessage( + stream="test_stream", + data={ + "str_col": "example", + "int_col": 1, + }, + emitted_at=0, + ), + ) + ] # setup Xata workspace xata = XataClient(api_key=config["api_key"], db_url=config["db_url"]) @@ -82,19 +89,23 @@ def test_write(config: Mapping): # database exists ? assert xata.databases().getDatabaseMetadata(db_name).status_code == 200, f"database '{db_name}' does not exist." assert xata.table().createTable("test_stream").status_code == 201, "could not create table, if it already exists, please delete it." - assert xata.table().setTableSchema("test_stream", { - "columns": [ - {"name": "str_col", "type": "string"}, - {"name": "int_col", "type": "int"}, - ] - }).status_code == 200, "failed to set table schema" + assert ( + xata.table() + .setTableSchema( + "test_stream", + { + "columns": [ + {"name": "str_col", "type": "string"}, + {"name": "int_col", "type": "int"}, + ] + }, + ) + .status_code + == 200 + ), "failed to set table schema" dest = DestinationXata() - list(dest.write( - config=config, - configured_catalog=test_stream, - input_messages=records - )) + list(dest.write(config=config, configured_catalog=test_stream, input_messages=records)) # fetch record records = xata.data().queryTable("test_stream", {}) diff --git a/airbyte-integrations/connectors/destination-xata/unit_tests/unit_test.py b/airbyte-integrations/connectors/destination-xata/unit_tests/unit_test.py index 340fbf62c948..51726247685a 100644 --- a/airbyte-integrations/connectors/destination-xata/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/destination-xata/unit_tests/unit_test.py @@ -9,7 +9,6 @@ class DestinationConnectorXataTestCase(unittest.TestCase): - def test_request(self): xata = XataClient(db_url="https://unit_tests-mock.results-store.xata.sh/db/mock-db", api_key="mock-key") bp = BulkProcessor(xata, thread_pool_size=1, batch_size=2, flush_interval=1) @@ -25,5 +24,5 @@ def test_request(self): assert stats["failed_batches"] == 0 -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/airbyte-integrations/connectors/destination-yugabytedb/.dockerignore b/airbyte-integrations/connectors/destination-yugabytedb/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/destination-yugabytedb/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/destination-yugabytedb/Dockerfile b/airbyte-integrations/connectors/destination-yugabytedb/Dockerfile deleted file mode 100644 index c58506069918..000000000000 --- a/airbyte-integrations/connectors/destination-yugabytedb/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte -ENV APPLICATION destination-yugabytedb - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte -ENV APPLICATION destination-yugabytedb - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.1 -LABEL io.airbyte.name=airbyte/destination-yugabytedb diff --git a/airbyte-integrations/connectors/destination-yugabytedb/README.md b/airbyte-integrations/connectors/destination-yugabytedb/README.md index 7339896b055c..cf5c9b91fc61 100644 --- a/airbyte-integrations/connectors/destination-yugabytedb/README.md +++ b/airbyte-integrations/connectors/destination-yugabytedb/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:destination-yugabytedb:airbyteDocker +./gradlew :airbyte-integrations:connectors:destination-yugabytedb:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/destination-yugabytedb:dev`. the Dockerfile. #### Run @@ -61,8 +62,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=destination-yugabytedb test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/destinations/yugabytedb.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/destination-yugabytedb/build.gradle b/airbyte-integrations/connectors/destination-yugabytedb/build.gradle index 79d17bb0b387..2186a1b5d8ee 100644 --- a/airbyte-integrations/connectors/destination-yugabytedb/build.gradle +++ b/airbyte-integrations/connectors/destination-yugabytedb/build.gradle @@ -1,32 +1,33 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-destinations'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.destination.yugabytedb.YugabytedbDestination' } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - implementation project(':airbyte-integrations:bases:bases-destination-jdbc') - implementation project(':airbyte-db:db-lib') implementation 'com.yugabyte:jdbc-yugabytedb:42.3.5-yb-1' - testImplementation project(':airbyte-integrations:bases:standard-destination-test') - testImplementation "org.assertj:assertj-core:3.21.0" testImplementation "org.junit.jupiter:junit-jupiter:5.8.1" testImplementation "org.testcontainers:junit-jupiter:1.17.5" testImplementation "org.testcontainers:jdbc:1.17.5" - - - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-destination-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-yugabytedb') } diff --git a/airbyte-integrations/connectors/destination-yugabytedb/docker-compose.yml b/airbyte-integrations/connectors/destination-yugabytedb/docker-compose.yml index cbd967d1f4af..d8763350fa30 100644 --- a/airbyte-integrations/connectors/destination-yugabytedb/docker-compose.yml +++ b/airbyte-integrations/connectors/destination-yugabytedb/docker-compose.yml @@ -1,31 +1,36 @@ -version: '3' - +version: "3" # Note: add mount points at /mnt/master and /mnt/tserver for persistence services: yb-master: - image: yugabytedb/yugabyte:latest - container_name: yb-master-n1 - command: [ "/home/yugabyte/bin/yb-master", - "--fs_data_dirs=/mnt/master", - "--master_addresses=yb-master-n1:7100", - "--rpc_bind_addresses=yb-master-n1:7100", - "--replication_factor=1"] - ports: + image: yugabytedb/yugabyte:latest + container_name: yb-master-n1 + command: + [ + "/home/yugabyte/bin/yb-master", + "--fs_data_dirs=/mnt/master", + "--master_addresses=yb-master-n1:7100", + "--rpc_bind_addresses=yb-master-n1:7100", + "--replication_factor=1", + ] + ports: - "7000:7000" yb-tserver: - image: yugabytedb/yugabyte:latest - container_name: yb-tserver-n1 - command: [ "/home/yugabyte/bin/yb-tserver", - "--fs_data_dirs=/mnt/tserver", - "--start_pgsql_proxy", - "--rpc_bind_addresses=yb-tserver-n1:9100", - "--tserver_master_addrs=yb-master-n1:7100"] - ports: + image: yugabytedb/yugabyte:latest + container_name: yb-tserver-n1 + command: + [ + "/home/yugabyte/bin/yb-tserver", + "--fs_data_dirs=/mnt/tserver", + "--start_pgsql_proxy", + "--rpc_bind_addresses=yb-tserver-n1:9100", + "--tserver_master_addrs=yb-master-n1:7100", + ] + ports: - "9042:9042" - "5433:5433" - "9000:9000" - depends_on: + depends_on: - yb-master diff --git a/airbyte-integrations/connectors/destination-yugabytedb/src/main/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbDestination.java b/airbyte-integrations/connectors/destination-yugabytedb/src/main/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbDestination.java index 386725e84c21..2ae3fc7c423e 100644 --- a/airbyte-integrations/connectors/destination-yugabytedb/src/main/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbDestination.java +++ b/airbyte-integrations/connectors/destination-yugabytedb/src/main/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbDestination.java @@ -6,11 +6,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.destination.jdbc.AbstractJdbcDestination; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.destination.jdbc.AbstractJdbcDestination; import java.util.Collections; import java.util.Map; import java.util.Optional; diff --git a/airbyte-integrations/connectors/destination-yugabytedb/src/main/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbNamingTransformer.java b/airbyte-integrations/connectors/destination-yugabytedb/src/main/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbNamingTransformer.java index 681821ace8e2..2485c777308b 100644 --- a/airbyte-integrations/connectors/destination-yugabytedb/src/main/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbNamingTransformer.java +++ b/airbyte-integrations/connectors/destination-yugabytedb/src/main/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbNamingTransformer.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.destination.yugabytedb; -import io.airbyte.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; public class YugabytedbNamingTransformer extends StandardNameTransformer { diff --git a/airbyte-integrations/connectors/destination-yugabytedb/src/main/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbSqlOperations.java b/airbyte-integrations/connectors/destination-yugabytedb/src/main/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbSqlOperations.java index f88ec69bcd75..bb876f884d55 100644 --- a/airbyte-integrations/connectors/destination-yugabytedb/src/main/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbSqlOperations.java +++ b/airbyte-integrations/connectors/destination-yugabytedb/src/main/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbSqlOperations.java @@ -6,8 +6,8 @@ import com.yugabyte.copy.CopyManager; import com.yugabyte.core.BaseConnection; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.destination.jdbc.JdbcSqlOperations; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.destination.jdbc.JdbcSqlOperations; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import java.io.BufferedReader; import java.io.File; diff --git a/airbyte-integrations/connectors/destination-yugabytedb/src/test-integration/java/io/airbyte/integrations/destination/yugabytedb/YugabyteDataSource.java b/airbyte-integrations/connectors/destination-yugabytedb/src/test-integration/java/io/airbyte/integrations/destination/yugabytedb/YugabyteDataSource.java index 4b2dcbb20ceb..f7cea140311f 100644 --- a/airbyte-integrations/connectors/destination-yugabytedb/src/test-integration/java/io/airbyte/integrations/destination/yugabytedb/YugabyteDataSource.java +++ b/airbyte-integrations/connectors/destination-yugabytedb/src/test-integration/java/io/airbyte/integrations/destination/yugabytedb/YugabyteDataSource.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.destination.yugabytedb; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; import java.util.Collections; import javax.sql.DataSource; diff --git a/airbyte-integrations/connectors/destination-yugabytedb/src/test-integration/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-yugabytedb/src/test-integration/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbDestinationAcceptanceTest.java index 26b6a92d37f8..ef4e3b8a80da 100644 --- a/airbyte-integrations/connectors/destination-yugabytedb/src/test-integration/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-yugabytedb/src/test-integration/java/io/airbyte/integrations/destination/yugabytedb/YugabytedbDestinationAcceptanceTest.java @@ -6,14 +6,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.JavaBaseConstants; +import io.airbyte.cdk.integrations.destination.StandardNameTransformer; +import io.airbyte.cdk.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; +import io.airbyte.cdk.integrations.standardtest.destination.comparator.TestDataComparator; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.JavaBaseConstants; -import io.airbyte.integrations.destination.StandardNameTransformer; -import io.airbyte.integrations.standardtest.destination.JdbcDestinationAcceptanceTest; -import io.airbyte.integrations.standardtest.destination.comparator.AdvancedTestDataComparator; -import io.airbyte.integrations.standardtest.destination.comparator.TestDataComparator; import java.sql.SQLException; import java.util.HashSet; import java.util.List; diff --git a/airbyte-integrations/connectors/source-activecampaign/README.md b/airbyte-integrations/connectors/source-activecampaign/README.md index 703a9afe1025..51f5f7aadf2c 100644 --- a/airbyte-integrations/connectors/source-activecampaign/README.md +++ b/airbyte-integrations/connectors/source-activecampaign/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-activecampaign:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/activecampaign) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_activecampaign/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-activecampaign:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-activecampaign build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-activecampaign:airbyteDocker +An image will be built with the tag `airbyte/source-activecampaign:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-activecampaign:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-activecampaign:dev che docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-activecampaign:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-activecampaign:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-activecampaign test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-activecampaign:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-activecampaign:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-activecampaign test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/activecampaign.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-activecampaign/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-activecampaign/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-activecampaign/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-activecampaign/build.gradle b/airbyte-integrations/connectors/source-activecampaign/build.gradle deleted file mode 100644 index 16bfce7c325a..000000000000 --- a/airbyte-integrations/connectors/source-activecampaign/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_activecampaign' -} diff --git a/airbyte-integrations/connectors/source-adjust/README.md b/airbyte-integrations/connectors/source-adjust/README.md index aa4e984afd74..c624a57f43c7 100644 --- a/airbyte-integrations/connectors/source-adjust/README.md +++ b/airbyte-integrations/connectors/source-adjust/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-adjust:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/adjust) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_adjust/spec.yaml` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-adjust:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name source-adjust build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-adjust:airbyteDocker +An image will be built with the tag `airbyte/source-adjust:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-adjust:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-adjust:dev check --con docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-adjust:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-adjust:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-adjust test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-adjust:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-adjust:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. diff --git a/airbyte-integrations/connectors/source-adjust/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-adjust/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-adjust/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-adjust/build.gradle b/airbyte-integrations/connectors/source-adjust/build.gradle deleted file mode 100644 index 7d1a49483255..000000000000 --- a/airbyte-integrations/connectors/source-adjust/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_adjust' -} diff --git a/airbyte-integrations/connectors/source-aha/README.md b/airbyte-integrations/connectors/source-aha/README.md index 36241d02e172..aa43d70e16d0 100644 --- a/airbyte-integrations/connectors/source-aha/README.md +++ b/airbyte-integrations/connectors/source-aha/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-aha:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/aha) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_aha/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-aha:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-aha build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-aha:airbyteDocker +An image will be built with the tag `airbyte/source-aha:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-aha:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-aha:dev check --config docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-aha:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-aha:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-aha test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-aha:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-aha:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-aha test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/aha.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-aha/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-aha/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-aha/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-aha/build.gradle b/airbyte-integrations/connectors/source-aha/build.gradle deleted file mode 100644 index aeff63d6b86c..000000000000 --- a/airbyte-integrations/connectors/source-aha/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_aha' -} diff --git a/airbyte-integrations/connectors/source-aircall/README.md b/airbyte-integrations/connectors/source-aircall/README.md index 605e432fa276..750124c2a5a0 100644 --- a/airbyte-integrations/connectors/source-aircall/README.md +++ b/airbyte-integrations/connectors/source-aircall/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-aircall:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/aircall) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_aircall/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-aircall:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-aircall build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-aircall:airbyteDocker +An image will be built with the tag `airbyte/source-aircall:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-aircall:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-aircall:dev check --co docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-aircall:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-aircall:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-aircall test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-aircall:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-aircall:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-aircall test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/aircall.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-aircall/acceptance-test-config.yml b/airbyte-integrations/connectors/source-aircall/acceptance-test-config.yml index a9d65f94152b..c72d7490f58d 100644 --- a/airbyte-integrations/connectors/source-aircall/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-aircall/acceptance-test-config.yml @@ -18,7 +18,7 @@ acceptance_tests: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: + empty_streams: - name: "webhooks" bypass_reason: "Sandbox account cannot seed this stream" expect_records: @@ -26,12 +26,12 @@ acceptance_tests: extra_fields: no exact_order: no extra_records: yes - incremental: + incremental: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-aircall/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-aircall/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-aircall/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-aircall/build.gradle b/airbyte-integrations/connectors/source-aircall/build.gradle deleted file mode 100644 index 8bd32f352847..000000000000 --- a/airbyte-integrations/connectors/source-aircall/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_aircall' -} diff --git a/airbyte-integrations/connectors/source-airtable/Dockerfile b/airbyte-integrations/connectors/source-airtable/Dockerfile deleted file mode 100644 index 9fea0a49d386..000000000000 --- a/airbyte-integrations/connectors/source-airtable/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_airtable ./source_airtable - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=3.0.1 -LABEL io.airbyte.name=airbyte/source-airtable diff --git a/airbyte-integrations/connectors/source-airtable/README.md b/airbyte-integrations/connectors/source-airtable/README.md index a38d034e0287..d1118541f030 100644 --- a/airbyte-integrations/connectors/source-airtable/README.md +++ b/airbyte-integrations/connectors/source-airtable/README.md @@ -31,14 +31,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-airtable:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/airtable) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_airtable/spec.json` file. @@ -58,19 +50,70 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-airtable:dev + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-airtable build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-airtable:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. -You can also build the connector image via Gradle: +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-airtable:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-airtable:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-airtable:dev . +# Running the spec command against your patched connector +docker run airbyte/source-airtable:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -79,45 +122,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-airtable:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-airtable:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-airtable:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-airtable test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-airtable:dev \ -&& python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-airtable:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-airtable:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -127,8 +141,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-airtable test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/airtable.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-airtable/acceptance-test-config.yml b/airbyte-integrations/connectors/source-airtable/acceptance-test-config.yml index d19419e1ff17..19e267aea061 100644 --- a/airbyte-integrations/connectors/source-airtable/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-airtable/acceptance-test-config.yml @@ -34,12 +34,30 @@ acceptance_tests: extra_fields: true exact_order: true extra_records: false + ignored_fields: + users/field_type_test/tblFcp5mncufoYaR9: + - name: "attachment" + bypass_reason: "Attachments' preview links are changed frequently" + "users/50_columns/tbl01Hi93Tt6XJ0u5": + - name: "attachments" + bypass_reason: "Attachments' preview links are changed frequently" + - name: "attachments_2" + bypass_reason: "Attachments' preview links are changed frequently" - config_path: "secrets/config_oauth.json" expect_records: path: "integration_tests/expected_records.jsonl" extra_fields: true exact_order: true extra_records: false + ignored_fields: + users/field_type_test/tblFcp5mncufoYaR9: + - name: "attachment" + bypass_reason: "Attachments' preview links are changed frequently" + "users/50_columns/tbl01Hi93Tt6XJ0u5": + - name: "attachments" + bypass_reason: "Attachments' preview links are changed frequently" + - name: "attachments_2" + bypass_reason: "Attachments' preview links are changed frequently" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-airtable/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-airtable/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-airtable/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-airtable/build.gradle b/airbyte-integrations/connectors/source-airtable/build.gradle deleted file mode 100644 index 62d247a859f2..000000000000 --- a/airbyte-integrations/connectors/source-airtable/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_airtable' -} diff --git a/airbyte-integrations/connectors/source-airtable/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-airtable/integration_tests/expected_records.jsonl index 85ccd4a6c395..e5b5bf265227 100644 --- a/airbyte-integrations/connectors/source-airtable/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-airtable/integration_tests/expected_records.jsonl @@ -1,42 +1,43 @@ -{"stream": "users/table_1/tblmrNtgqio9IWVGx", "data": {"_airtable_id": "recgRMRMHxgcgJeDe", "_airtable_created_time": "2021-11-16T13:30:17.000Z", "name": "test2", "status": "test", "notes": "test_note2"}, "emitted_at": 1675276485893} -{"stream": "users/table_1/tblmrNtgqio9IWVGx", "data": {"_airtable_id": "recmJkSF51IKUGmlJ", "_airtable_created_time": "2022-12-22T20:58:05.000Z", "name": "test4_after_empty", "clo_with_empty_strings": "bla bla bla", "status": "In progress", "notes": "test_note4"}, "emitted_at": 1675276485893} -{"stream": "users/table_1/tblmrNtgqio9IWVGx", "data": {"_airtable_id": "recvp1qYYBlcOrzsc", "_airtable_created_time": "2021-11-16T13:30:17.000Z", "name": "test3", "clo_with_empty_strings": "test text here", "status": "test", "notes": "test-note3"}, "emitted_at": 1675276485894} -{"stream": "users/table_1/tblmrNtgqio9IWVGx", "data": {"_airtable_id": "reczEeQV9NrzFlVFF", "_airtable_created_time": "2021-11-16T13:30:17.000Z", "name": "test", "status": "test", "notes": "test_note"}, "emitted_at": 1675276485894} -{"stream": "users/table_2/tblCjIgm5yveWC4X5", "data": {"_airtable_id": "recB3upao4wpmeCEf", "_airtable_created_time": "2021-11-16T13:32:31.000Z", "status": "In progress", "name": "test2", "notes": "test2"}, "emitted_at": 1675276487209} -{"stream": "users/table_2/tblCjIgm5yveWC4X5", "data": {"_airtable_id": "recWeE6SiYeri4Duq", "_airtable_created_time": "2021-11-16T13:32:31.000Z", "status": "Todo", "name": "test", "notes": "test"}, "emitted_at": 1675276487209} -{"stream": "users/table_2/tblCjIgm5yveWC4X5", "data": {"_airtable_id": "recXn7kXbeEsJfDJQ", "_airtable_created_time": "2022-12-28T11:41:10.000Z", "name": " "}, "emitted_at": 1675276487210} -{"stream": "users/table_2/tblCjIgm5yveWC4X5", "data": {"_airtable_id": "recmMfFAec8plcOUY", "_airtable_created_time": "2021-11-16T13:32:31.000Z", "status": "Done", "name": "test 3", "notes": "test 3"}, "emitted_at": 1675276487210} -{"stream": "users/table_6/tblSXpxKHg0OiLxbI", "data": {"_airtable_id": "recHyy86oge9j5cYP", "_airtable_created_time": "2023-01-25T02:04:26.000Z", "name": "test_negative", "table_6": ["recHyy86oge9j5cYP", "recoY6ShPkGpav3Se"], "float": 0.3, "status": "In progress", "integer": -1.0, "assignee": [2.0], "assignee_(from_table_6)": [2.0, 2.0]}, "emitted_at": 1675276488139} -{"stream": "users/table_6/tblSXpxKHg0OiLxbI", "data": {"_airtable_id": "recLhKYa9btCCqmxs", "_airtable_created_time": "2023-01-25T02:04:26.000Z", "name": "test_attachment", "table_6": ["recLhKYa9btCCqmxs", "recoY6ShPkGpav3Se"], "float": 0.3, "status": "Todo", "integer": 1.0, "assignee": [2.0], "assignee_(from_table_6)": [2.0, 2.0]}, "emitted_at": 1675276488139} -{"stream": "users/table_6/tblSXpxKHg0OiLxbI", "data": {"_airtable_id": "recoY6ShPkGpav3Se", "_airtable_created_time": "2023-01-25T02:04:26.000Z", "name": "test_normal", "table_6": ["recoY6ShPkGpav3Se", "recHyy86oge9j5cYP", "recLhKYa9btCCqmxs"], "float": 0.7, "status": "Done", "integer": 2.0, "assignee": [2.0], "assignee_(from_table_6)": [2.0, 2.0, 2.0]}, "emitted_at": 1675276488140} -{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "rec0UJKftqCPj7zJu", "_airtable_created_time": "2022-12-01T20:27:46.000Z", "email": "integration-test-user@airbyte.io", "id": 17.0, "date": "2022-09-30", "multiple_select_(no_colors)": ["1", "2", "3"], "percent": 0.0, "rating": 5.0, "multiple_select_(colors)": ["1", "2", "3"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "checkbox": true, "number": 1.0, "duration": 0.0, "name": "John", "single_line": "Singe line", "phone": "(937) 999-999", "value": 1.0, "long_text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas vitae nisi nec justo laoreet tincidunt. Vivamus eget porttitor velit. Pellentesque gravida euismod massa, eget egestas sem facilisis gravida. Suspendisse quis mauris eget velit faucibus aliquam. Mauris porttitor urna lorem, eget tincidunt ex faucibus et. Nam viverra nibh quis turpis scelerisque, et condimentum nibh vulputate. Vivamus eu dolor posuere, finibus ligula vel, finibus erat. Sed hendrerit luctus erat, eu finibus ante blandit ut. Sed id mi ullamcorper, cursus urna nec, molestie lorem. In vulputate tempor nulla in laoreet. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Sed a risus risus. Praesent neque lorem, fringilla vitae rhoncus a, semper eu augue.", "single_select_(colors)": "1", "single_select_(no_colors)": "1", "url": "airbyte.io", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:46.000Z", "created_(w/o_time)": "2022-12-01", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1675276488924} -{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "rec3tSj3Yzi42uuS0", "_airtable_created_time": "2022-09-30T16:06:49.000Z", "id": 3.0, "100_columns": ["reccHYZ004lOuxLFH"], "name": "Blank", "created_(with_time)": "2022-09-30T16:06:49.000Z", "created_(w/o_time)": "2022-09-30", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1675276488925} -{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recAa5DUEhppIeeax", "_airtable_created_time": "2022-12-01T20:27:30.000Z", "email": "integration-test-user@airbyte.io", "id": 9.0, "date": "2022-09-30", "multiple_select_(no_colors)": ["1", "2", "3"], "percent": 0.0, "rating": 5.0, "multiple_select_(colors)": ["1", "2", "3"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "checkbox": true, "number": 1.0, "duration": 0.0, "name": "John", "single_line": "Singe line", "phone": "(937) 999-999", "value": 1.0, "long_text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas vitae nisi nec justo laoreet tincidunt. Vivamus eget porttitor velit. Pellentesque gravida euismod massa, eget egestas sem facilisis gravida. Suspendisse quis mauris eget velit faucibus aliquam. Mauris porttitor urna lorem, eget tincidunt ex faucibus et. Nam viverra nibh quis turpis scelerisque, et condimentum nibh vulputate. Vivamus eu dolor posuere, finibus ligula vel, finibus erat. Sed hendrerit luctus erat, eu finibus ante blandit ut. Sed id mi ullamcorper, cursus urna nec, molestie lorem. In vulputate tempor nulla in laoreet. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Sed a risus risus. Praesent neque lorem, fringilla vitae rhoncus a, semper eu augue.", "single_select_(colors)": "1", "single_select_(no_colors)": "1", "url": "airbyte.io", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:30.000Z", "created_(w/o_time)": "2022-12-01", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1675276488926} -{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recG989xsdfSxookl", "_airtable_created_time": "2022-12-01T20:27:46.000Z", "email": "integration-test-user@airbyte.io", "id": 23.0, "date": "2022-10-03", "multiple_select_(no_colors)": ["a", "b", "c"], "percent": 1.0, "rating": 3.0, "multiple_select_(colors)": ["a", "b", "c"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "number": 123123871234.33, "duration": 169409880.0, "name": "Jane", "single_line": "09-30-2022", "phone": "(937) 999-999", "value": 1128312.23, "long_text": "18923671273628376328495623874562981345982 234953249857239045798 342098523495723495732 23489532475\n\n23453245234532453245234523453245\n\n2345234532452345", "single_select_(colors)": "a", "single_select_(no_colors)": "a", "url": "www.google.com", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:46.000Z", "created_(w/o_time)": "2022-12-01", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1675276488927} -{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recGVqF9BIpDD7Gj1", "_airtable_created_time": "2022-12-01T20:27:46.000Z", "email": "integration-test-user@airbyte.io", "id": 22.0, "date": "2022-09-30", "multiple_select_(no_colors)": ["1", "2", "3"], "percent": 0.0, "rating": 5.0, "multiple_select_(colors)": ["1", "2", "3"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "checkbox": true, "number": 1.0, "duration": 0.0, "name": "John", "single_line": "Singe line", "phone": "(937) 999-999", "value": 1.0, "long_text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas vitae nisi nec justo laoreet tincidunt. Vivamus eget porttitor velit. Pellentesque gravida euismod massa, eget egestas sem facilisis gravida. Suspendisse quis mauris eget velit faucibus aliquam. Mauris porttitor urna lorem, eget tincidunt ex faucibus et. Nam viverra nibh quis turpis scelerisque, et condimentum nibh vulputate. Vivamus eu dolor posuere, finibus ligula vel, finibus erat. Sed hendrerit luctus erat, eu finibus ante blandit ut. Sed id mi ullamcorper, cursus urna nec, molestie lorem. In vulputate tempor nulla in laoreet. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Sed a risus risus. Praesent neque lorem, fringilla vitae rhoncus a, semper eu augue.", "single_select_(colors)": "1", "single_select_(no_colors)": "1", "url": "airbyte.io", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:46.000Z", "created_(w/o_time)": "2022-12-01", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1675276488928} -{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recHNlOoFvp6KjeZD", "_airtable_created_time": "2022-12-01T20:27:46.000Z", "id": 16.0, "name": "Blank", "created_(with_time)": "2022-12-01T20:27:46.000Z", "created_(w/o_time)": "2022-12-01", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1675276488929} -{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recL6sfQP0FyJr3Ua", "_airtable_created_time": "2022-12-01T20:27:46.000Z", "id": 21.0, "name": "Blank", "created_(with_time)": "2022-12-01T20:27:46.000Z", "created_(w/o_time)": "2022-12-01", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1675276488929} -{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recMtt4tMmdNbwV5R", "_airtable_created_time": "2022-12-01T20:27:20.000Z", "id": 8.0, "name": "Blank", "created_(with_time)": "2022-12-01T20:27:20.000Z", "created_(w/o_time)": "2022-12-01", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1675276488930} -{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recOz5BBo3Ey3ldb8", "_airtable_created_time": "2022-12-01T20:27:11.000Z", "email": "integration-test-user@airbyte.io", "id": 5.0, "date": "2022-10-03", "multiple_select_(no_colors)": ["a", "b", "c"], "percent": 1.0, "rating": 3.0, "multiple_select_(colors)": ["a", "b", "c"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "100_columns": ["recXVzpNmtsBcjrA2"], "number": 123123871234.33, "duration": 169409880.0, "name": "Jane", "single_line": "09-30-2022", "phone": "(937) 999-999", "value": 1128312.23, "long_text": "18923671273628376328495623874562981345982 234953249857239045798 342098523495723495732 23489532475\n\n23453245234532453245234523453245\n\n2345234532452345", "single_select_(colors)": "a", "single_select_(no_colors)": "a", "url": "www.google.com", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:11.000Z", "created_(w/o_time)": "2022-12-01", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1675276488930} -{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recSbSBPspxs22165", "_airtable_created_time": "2022-12-01T20:27:46.000Z", "email": "integration-test-user@airbyte.io", "id": 15.0, "date": "2022-10-03", "multiple_select_(no_colors)": ["a", "b", "c"], "percent": 1.0, "rating": 3.0, "multiple_select_(colors)": ["a", "b", "c"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "number": 123123871234.33, "duration": 169409880.0, "name": "Jane", "single_line": "09-30-2022", "phone": "(937) 999-999", "value": 1128312.23, "long_text": "18923671273628376328495623874562981345982 234953249857239045798 342098523495723495732 23489532475\n\n23453245234532453245234523453245\n\n2345234532452345", "single_select_(colors)": "a", "single_select_(no_colors)": "a", "url": "www.google.com", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:46.000Z", "created_(w/o_time)": "2022-12-01", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1675276488931} -{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recUzRSGAf5VYtXfZ", "_airtable_created_time": "2022-09-30T16:06:49.000Z", "email": "integration-test-user@airbyte.io", "id": 1.0, "date": "2022-09-30", "multiple_select_(no_colors)": ["1", "2", "3"], "percent": 0.0, "rating": 5.0, "multiple_select_(colors)": ["1", "2", "3"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "checkbox": true, "100_columns": ["rec2lCkcy9d0fZi8h"], "number": 1.0, "duration": 0.0, "name": "John", "single_line": "Singe line", "phone": "(937) 999-999", "value": 1.0, "long_text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas vitae nisi nec justo laoreet tincidunt. Vivamus eget porttitor velit. Pellentesque gravida euismod massa, eget egestas sem facilisis gravida. Suspendisse quis mauris eget velit faucibus aliquam. Mauris porttitor urna lorem, eget tincidunt ex faucibus et. Nam viverra nibh quis turpis scelerisque, et condimentum nibh vulputate. Vivamus eu dolor posuere, finibus ligula vel, finibus erat. Sed hendrerit luctus erat, eu finibus ante blandit ut. Sed id mi ullamcorper, cursus urna nec, molestie lorem. In vulputate tempor nulla in laoreet. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Sed a risus risus. Praesent neque lorem, fringilla vitae rhoncus a, semper eu augue.", "single_select_(colors)": "1", "single_select_(no_colors)": "1", "url": "airbyte.io", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-09-30T16:06:49.000Z", "created_(w/o_time)": "2022-09-30", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1675276488932} -{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recbQy9sb5BM1K3Z1", "_airtable_created_time": "2022-12-01T20:27:17.000Z", "email": "integration-test-user@airbyte.io", "id": 6.0, "date": "2022-09-30", "multiple_select_(no_colors)": ["1", "2", "3"], "percent": 0.0, "rating": 5.0, "multiple_select_(colors)": ["1", "2", "3"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "checkbox": true, "number": 1.0, "duration": 0.0, "name": "John", "single_line": "Singe line", "phone": "(937) 999-999", "value": 1.0, "long_text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas vitae nisi nec justo laoreet tincidunt. Vivamus eget porttitor velit. Pellentesque gravida euismod massa, eget egestas sem facilisis gravida. Suspendisse quis mauris eget velit faucibus aliquam. Mauris porttitor urna lorem, eget tincidunt ex faucibus et. Nam viverra nibh quis turpis scelerisque, et condimentum nibh vulputate. Vivamus eu dolor posuere, finibus ligula vel, finibus erat. Sed hendrerit luctus erat, eu finibus ante blandit ut. Sed id mi ullamcorper, cursus urna nec, molestie lorem. In vulputate tempor nulla in laoreet. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Sed a risus risus. Praesent neque lorem, fringilla vitae rhoncus a, semper eu augue.", "single_select_(colors)": "1", "single_select_(no_colors)": "1", "url": "airbyte.io", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:17.000Z", "created_(w/o_time)": "2022-12-01", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1675276488932} -{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recehwpjPyD1PLFkr", "_airtable_created_time": "2022-12-01T20:27:10.000Z", "email": "integration-test-user@airbyte.io", "id": 4.0, "date": "2022-09-30", "multiple_select_(no_colors)": ["1", "2", "3"], "percent": 0.0, "rating": 5.0, "multiple_select_(colors)": ["1", "2", "3"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "checkbox": true, "number": 1.0, "duration": 0.0, "name": "John", "single_line": "Singe line", "phone": "(937) 999-999", "value": 1.0, "long_text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas vitae nisi nec justo laoreet tincidunt. Vivamus eget porttitor velit. Pellentesque gravida euismod massa, eget egestas sem facilisis gravida. Suspendisse quis mauris eget velit faucibus aliquam. Mauris porttitor urna lorem, eget tincidunt ex faucibus et. Nam viverra nibh quis turpis scelerisque, et condimentum nibh vulputate. Vivamus eu dolor posuere, finibus ligula vel, finibus erat. Sed hendrerit luctus erat, eu finibus ante blandit ut. Sed id mi ullamcorper, cursus urna nec, molestie lorem. In vulputate tempor nulla in laoreet. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Sed a risus risus. Praesent neque lorem, fringilla vitae rhoncus a, semper eu augue.", "single_select_(colors)": "1", "single_select_(no_colors)": "1", "url": "airbyte.io", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:10.000Z", "created_(w/o_time)": "2022-12-01", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1675276488933} -{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recfdnbGNkUx1Rs5F", "_airtable_created_time": "2022-12-01T20:27:46.000Z", "email": "integration-test-user@airbyte.io", "id": 19.0, "date": "2022-09-30", "multiple_select_(no_colors)": ["1", "2", "3"], "percent": 0.0, "rating": 5.0, "multiple_select_(colors)": ["1", "2", "3"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "checkbox": true, "number": 1.0, "duration": 0.0, "name": "John", "single_line": "Singe line", "phone": "(937) 999-999", "value": 1.0, "long_text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas vitae nisi nec justo laoreet tincidunt. Vivamus eget porttitor velit. Pellentesque gravida euismod massa, eget egestas sem facilisis gravida. Suspendisse quis mauris eget velit faucibus aliquam. Mauris porttitor urna lorem, eget tincidunt ex faucibus et. Nam viverra nibh quis turpis scelerisque, et condimentum nibh vulputate. Vivamus eu dolor posuere, finibus ligula vel, finibus erat. Sed hendrerit luctus erat, eu finibus ante blandit ut. Sed id mi ullamcorper, cursus urna nec, molestie lorem. In vulputate tempor nulla in laoreet. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Sed a risus risus. Praesent neque lorem, fringilla vitae rhoncus a, semper eu augue.", "single_select_(colors)": "1", "single_select_(no_colors)": "1", "url": "airbyte.io", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:46.000Z", "created_(w/o_time)": "2022-12-01", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1675276488934} -{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recpkKhLwcFYbJLtJ", "_airtable_created_time": "2022-12-01T20:27:43.000Z", "email": "integration-test-user@airbyte.io", "id": 14.0, "date": "2022-09-30", "multiple_select_(no_colors)": ["1", "2", "3"], "percent": 0.0, "rating": 5.0, "multiple_select_(colors)": ["1", "2", "3"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "checkbox": true, "number": 1.0, "duration": 0.0, "name": "John", "single_line": "Singe line", "phone": "(937) 999-999", "value": 1.0, "long_text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas vitae nisi nec justo laoreet tincidunt. Vivamus eget porttitor velit. Pellentesque gravida euismod massa, eget egestas sem facilisis gravida. Suspendisse quis mauris eget velit faucibus aliquam. Mauris porttitor urna lorem, eget tincidunt ex faucibus et. Nam viverra nibh quis turpis scelerisque, et condimentum nibh vulputate. Vivamus eu dolor posuere, finibus ligula vel, finibus erat. Sed hendrerit luctus erat, eu finibus ante blandit ut. Sed id mi ullamcorper, cursus urna nec, molestie lorem. In vulputate tempor nulla in laoreet. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Sed a risus risus. Praesent neque lorem, fringilla vitae rhoncus a, semper eu augue.", "single_select_(colors)": "1", "single_select_(no_colors)": "1", "url": "airbyte.io", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:43.000Z", "created_(w/o_time)": "2022-12-01", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1675276488935} -{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recqurXMR6u1PO5wT", "_airtable_created_time": "2022-12-01T20:27:18.000Z", "email": "integration-test-user@airbyte.io", "id": 7.0, "date": "2022-10-03", "multiple_select_(no_colors)": ["a", "b", "c"], "percent": 1.0, "rating": 3.0, "multiple_select_(colors)": ["a", "b", "c"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "number": 123123871234.33, "duration": 169409880.0, "name": "Jane", "single_line": "09-30-2022", "phone": "(937) 999-999", "value": 1128312.23, "long_text": "18923671273628376328495623874562981345982 234953249857239045798 342098523495723495732 23489532475\n\n23453245234532453245234523453245\n\n2345234532452345", "single_select_(colors)": "a", "single_select_(no_colors)": "a", "url": "www.google.com", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:18.000Z", "created_(w/o_time)": "2022-12-01", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1675276488936} -{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recuYRu9z5dlkHxop", "_airtable_created_time": "2022-12-01T20:27:30.000Z", "email": "integration-test-user@airbyte.io", "id": 10.0, "date": "2022-10-03", "multiple_select_(no_colors)": ["a", "b", "c"], "percent": 1.0, "rating": 3.0, "multiple_select_(colors)": ["a", "b", "c"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "number": 123123871234.33, "duration": 169409880.0, "name": "Jane", "single_line": "09-30-2022", "phone": "(937) 999-999", "value": 1128312.23, "long_text": "18923671273628376328495623874562981345982 234953249857239045798 342098523495723495732 23489532475\n\n23453245234532453245234523453245\n\n2345234532452345", "single_select_(colors)": "a", "single_select_(no_colors)": "a", "url": "www.google.com", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:30.000Z", "created_(w/o_time)": "2022-12-01", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1675276488937} -{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recw3jDJFziB562hu", "_airtable_created_time": "2022-12-01T20:27:46.000Z", "email": "integration-test-user@airbyte.io", "id": 18.0, "date": "2022-10-03", "multiple_select_(no_colors)": ["a", "b", "c"], "percent": 1.0, "rating": 3.0, "multiple_select_(colors)": ["a", "b", "c"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "number": 123123871234.33, "duration": 169409880.0, "name": "Jane", "single_line": "09-30-2022", "phone": "(937) 999-999", "value": 1128312.23, "long_text": "18923671273628376328495623874562981345982 234953249857239045798 342098523495723495732 23489532475\n\n23453245234532453245234523453245\n\n2345234532452345", "single_select_(colors)": "a", "single_select_(no_colors)": "a", "url": "www.google.com", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:46.000Z", "created_(w/o_time)": "2022-12-01", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1675276488937} -{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recyJGdkZikJbLFRi", "_airtable_created_time": "2022-09-30T16:06:49.000Z", "email": "integration-test-user@airbyte.io", "id": 2.0, "date": "2022-10-03", "multiple_select_(no_colors)": ["a", "b", "c"], "percent": 1.0, "rating": 3.0, "multiple_select_(colors)": ["a", "b", "c"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "number": 123123871234.33, "duration": 169409880.0, "name": "Jane", "single_line": "09-30-2022", "phone": "(937) 999-999", "value": 1128312.23, "long_text": "18923671273628376328495623874562981345982 234953249857239045798 342098523495723495732 23489532475\n\n23453245234532453245234523453245\n\n2345234532452345", "single_select_(colors)": "a", "single_select_(no_colors)": "a", "url": "www.google.com", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-09-30T16:06:49.000Z", "created_(w/o_time)": "2022-09-30", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1675276488938} -{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "reczDi9vTuH3ezfJM", "_airtable_created_time": "2022-12-01T20:27:46.000Z", "email": "integration-test-user@airbyte.io", "id": 20.0, "date": "2022-10-03", "multiple_select_(no_colors)": ["a", "b", "c"], "percent": 1.0, "rating": 3.0, "multiple_select_(colors)": ["a", "b", "c"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "number": 123123871234.33, "duration": 169409880.0, "name": "Jane", "single_line": "09-30-2022", "phone": "(937) 999-999", "value": 1128312.23, "long_text": "18923671273628376328495623874562981345982 234953249857239045798 342098523495723495732 23489532475\n\n23453245234532453245234523453245\n\n2345234532452345", "single_select_(colors)": "a", "single_select_(no_colors)": "a", "url": "www.google.com", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:46.000Z", "created_(w/o_time)": "2022-12-01", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1675276488939} -{"stream": "users/50_columns/tbl01Hi93Tt6XJ0u5", "data": {"_airtable_id": "rec2lCkcy9d0fZi8h", "_airtable_created_time": "2022-12-02T17:29:41.000Z", "notes": "dasdafsdag", "phone_3": "(999) 999-9999", "tags_2": ["alpha", "issue"], "select_2": "top", "date_2": "2023-01-03", "euro": 136.46, "collaborator": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "number": 423.0, "assignee": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "percent_2": 0.45299999999999996, "url": "airbyte.com", "duration": 9900.0, "notes_3": " dfhsh f g dh", "status": "Done", "rating_2": 1.0, "barcode": "{'text': '312515435'}", "number_2": 3453.0, "rating": 4.0, "field_type_test": ["recUzRSGAf5VYtXfZ"], "usd": 143.64, "done": true, "id": 1.0, "notes_2": "1. fsafsf sfkjkl fsafs", "phone_2": "(222) 222-2222", "email": "email@airbyte.io", "percent_3": 0.24300000000000002, "select": "1st", "int": 65.0, "label_2": "Lorem", "date": "2022-12-02", "label_3": "line", "name": "Thing 1", "phone": "(999) 999-9999", "label": "Nulla quis lorem ut libero malesuada feugiat.", "duration_2": 20580.0, "tags": ["tag1"], "decimal": 5.8797, "percent": 0.2, "email_(from_field_type_test)": ["integration-test-user@airbyte.io"], "count_(field_type_test)": 1.0, "created_(w/o_time)_(from_field_type_test)": ["2022-09-30"], "created": "2022-12-02T17:29:41.000Z", "last_modified": "2023-01-24T11:12:27.000Z", "last_modified_2": "2023-01-24T11:12:27.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1675276490320} -{"stream": "users/50_columns/tbl01Hi93Tt6XJ0u5", "data": {"_airtable_id": "recXVzpNmtsBcjrA2", "_airtable_created_time": "2022-12-02T17:29:41.000Z", "notes": "opoiwepoirwe", "phone_3": "(999) 999-9999", "tags_2": ["ga"], "select_2": "bottom", "date_2": "2023-01-31", "euro": 279.62, "collaborator": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "number": 34535.5, "percent_2": 0.624, "duration": 10440.0, "notes_3": "hdfhe e et e true ttue ", "status": "Todo", "rating_2": 4.0, "barcode": "{'text': '351164554'}", "rating": 3.0, "field_type_test": ["recOz5BBo3Ey3ldb8"], "usd": 294.34, "done": true, "url_2": "airbyte.com", "id": 3.0, "notes_2": "3. flsflkj;flkjsf fskjlf lakjf; lskaj;klsjf", "phone_2": "(444) 444-4444", "percent_3": 0.13699999999999998, "select": "3rd", "int": 98.0, "date": "2022-12-14", "label_3": "line", "name": "Thing 3", "phone": "(999) 999-9999", "label": "Curabitur aliquet quam id dui posuere blandit.", "duration_2": 19740.0, "decimal": 6.6, "percent": 0.3, "email_(from_field_type_test)": ["integration-test-user@airbyte.io"], "count_(field_type_test)": 1.0, "created_(w/o_time)_(from_field_type_test)": ["2022-12-01"], "created": "2022-12-02T17:29:41.000Z", "last_modified": "2023-01-24T11:12:12.000Z", "last_modified_2": "2023-01-24T11:12:12.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1675276490322} -{"stream": "users/50_columns/tbl01Hi93Tt6XJ0u5", "data": {"_airtable_id": "reccHYZ004lOuxLFH", "_airtable_created_time": "2022-12-02T17:29:41.000Z", "notes": "hjyyfhgjjfgjr", "phone_3": "(999) 999-9999", "tags_2": ["beta", "no issue"], "select_2": "mid", "date_2": "2023-01-17", "euro": 325.13, "number": 22424.5, "assignee": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "percent_2": 0.562, "duration": 11520.0, "notes_3": "fhdfhdhgf hfh hdfgh", "status": "In progress", "barcode": "{'text': '5531515315'}", "number_2": 3452.0, "rating": 5.0, "field_type_test": ["rec3tSj3Yzi42uuS0"], "usd": 342.24, "url_2": "docs.airbyte.com", "id": 2.0, "notes_2": "2. fskldf f;sfkjk s;lkjfkls", "phone_2": "(333) 333-3333", "email": "what@airbyte.io", "percent_3": 0.532, "select": "2nd", "int": 53.0, "date": "2022-11-10", "label_3": "line", "name": "Thing 2", "phone": "(999) 999-9999", "label": "Quisque velit nisi, pretium ut lacinia in, elementum id enim.", "duration_2": 18180.0, "collaborator_2": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "tags": ["tag2", "tag3"], "done_2": true, "decimal": 4.134, "percent": 0.15, "count_(field_type_test)": 1.0, "created_(w/o_time)_(from_field_type_test)": ["2022-09-30"], "created": "2022-12-02T17:29:41.000Z", "last_modified": "2023-01-24T11:12:11.000Z", "last_modified_2": "2023-01-24T11:12:11.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1675276490323} -{"stream": "users/checkboxes/tbl81WIAUZg5nwGN8", "data": {"_airtable_id": "recLvpJ4k4mRG38My", "_airtable_created_time": "2022-12-02T19:50:00.000Z", "done_18": true, "done_21": true, "done_6": true, "done_11": true, "done_24": true, "done_2": true, "done_12": true, "done_16": true, "done_4": true, "done_10": true, "done_7": true, "done_22": true, "done_23": true, "done_20": true, "name": "Cloud"}, "emitted_at": 1675276491198} -{"stream": "users/checkboxes/tbl81WIAUZg5nwGN8", "data": {"_airtable_id": "reckszXRiFfg11IYD", "_airtable_created_time": "2022-12-02T19:50:00.000Z", "done_21": true, "done_6": true, "done_17": true, "done_5": true, "done_9": true, "done_11": true, "done_16": true, "done_25": true, "done_7": true, "done_22": true, "done_13": true, "done_3": true, "name": "Support"}, "emitted_at": 1675276491199} -{"stream": "users/checkboxes/tbl81WIAUZg5nwGN8", "data": {"_airtable_id": "recmaYwfPMvZtwTSJ", "_airtable_created_time": "2022-12-02T19:50:00.000Z", "done_5": true, "done_9": true, "done_19": true, "done_12": true, "done": true, "done_16": true, "done_14": true, "done_4": true, "done_25": true, "done_7": true, "done_23": true, "done_20": true, "name": "Airbyte"}, "emitted_at": 1675276491200} -{"stream": "untitled_base/table_1/tblT7mnwDS5TVtfOh", "data": {"_airtable_id": "recJ0l923fOFw6qbl", "_airtable_created_time": "2021-12-09T07:54:15.000Z", "name": "test2", "notes": "test2", "status": "In progress"}, "emitted_at": 1675276492018} -{"stream": "untitled_base/table_1/tblT7mnwDS5TVtfOh", "data": {"_airtable_id": "recNbrGzLJfOy6EjC", "_airtable_created_time": "2022-12-28T11:43:37.000Z", "name": "blank"}, "emitted_at": 1675276492018} -{"stream": "untitled_base/table_1/tblT7mnwDS5TVtfOh", "data": {"_airtable_id": "recZX1Je5k4nXhTi0", "_airtable_created_time": "2021-12-09T07:54:15.000Z", "name": "blank"}, "emitted_at": 1675276492019} -{"stream": "untitled_base/table_1/tblT7mnwDS5TVtfOh", "data": {"_airtable_id": "recxXAoQ0cC5yCMZ7", "_airtable_created_time": "2021-12-09T07:54:15.000Z", "name": "test1", "notes": "test", "status": "Todo"}, "emitted_at": 1675276492019} -{"stream": "untitled_base/table_1/tblT7mnwDS5TVtfOh", "data": {"_airtable_id": "reczJvdeo0b8KsM6K", "_airtable_created_time": "2022-12-28T11:43:38.000Z", "name": "test3", "notes": "test3", "status": "Done"}, "emitted_at": 1675276492019} +{"stream": "users/table_1/tblmrNtgqio9IWVGx", "data": {"_airtable_id": "recgRMRMHxgcgJeDe", "_airtable_created_time": "2021-11-16T13:30:17.000Z", "_airtable_table_name": "Table 1", "name": "test2", "status": "test", "notes": "test_note2"}, "emitted_at": 1696938630715} +{"stream": "users/table_1/tblmrNtgqio9IWVGx", "data": {"_airtable_id": "recmJkSF51IKUGmlJ", "_airtable_created_time": "2022-12-22T20:58:05.000Z", "_airtable_table_name": "Table 1", "name": "test4_after_empty", "clo_with_empty_strings": "bla bla bla", "status": "In progress", "notes": "test_note4"}, "emitted_at": 1696938630715} +{"stream": "users/table_1/tblmrNtgqio9IWVGx", "data": {"_airtable_id": "recvp1qYYBlcOrzsc", "_airtable_created_time": "2021-11-16T13:30:17.000Z", "_airtable_table_name": "Table 1", "name": "test3", "clo_with_empty_strings": "test text here", "status": "test", "notes": "test-note3"}, "emitted_at": 1696938630715} +{"stream": "users/table_1/tblmrNtgqio9IWVGx", "data": {"_airtable_id": "reczEeQV9NrzFlVFF", "_airtable_created_time": "2021-11-16T13:30:17.000Z", "_airtable_table_name": "Table 1", "name": "test", "status": "test", "notes": "test_note"}, "emitted_at": 1696938630716} +{"stream": "users/table_2/tblCjIgm5yveWC4X5", "data": {"_airtable_id": "recB3upao4wpmeCEf", "_airtable_created_time": "2021-11-16T13:32:31.000Z", "_airtable_table_name": "Table 2", "status": "In progress", "name": "test2", "notes": "test2"}, "emitted_at": 1696938631703} +{"stream": "users/table_2/tblCjIgm5yveWC4X5", "data": {"_airtable_id": "recWeE6SiYeri4Duq", "_airtable_created_time": "2021-11-16T13:32:31.000Z", "_airtable_table_name": "Table 2", "status": "Todo", "name": "test", "notes": "test"}, "emitted_at": 1696938631704} +{"stream": "users/table_2/tblCjIgm5yveWC4X5", "data": {"_airtable_id": "recXn7kXbeEsJfDJQ", "_airtable_created_time": "2022-12-28T11:41:10.000Z", "_airtable_table_name": "Table 2", "name": " "}, "emitted_at": 1696938631704} +{"stream": "users/table_2/tblCjIgm5yveWC4X5", "data": {"_airtable_id": "recmMfFAec8plcOUY", "_airtable_created_time": "2021-11-16T13:32:31.000Z", "_airtable_table_name": "Table 2", "status": "Done", "name": "test 3", "notes": "test 3"}, "emitted_at": 1696938631704} +{"stream": "users/table_6/tblSXpxKHg0OiLxbI", "data": {"_airtable_id": "recHyy86oge9j5cYP", "_airtable_created_time": "2023-01-25T02:04:26.000Z", "_airtable_table_name": "Table 6", "name": "test_negative", "table_6": ["recHyy86oge9j5cYP", "recoY6ShPkGpav3Se"], "float": 0.3, "status": "In progress", "integer": -1.0, "assignee": 2.0, "assignee_(from_table_6)": [2.0, 2.0]}, "emitted_at": 1696938632677} +{"stream": "users/table_6/tblSXpxKHg0OiLxbI", "data": {"_airtable_id": "recLhKYa9btCCqmxs", "_airtable_created_time": "2023-01-25T02:04:26.000Z", "_airtable_table_name": "Table 6", "name": "test_attachment", "table_6": ["recLhKYa9btCCqmxs", "recoY6ShPkGpav3Se"], "float": 0.3, "status": "Todo", "integer": 1.0, "assignee": 2.0, "assignee_(from_table_6)": [2.0, 2.0]}, "emitted_at": 1696938632677} +{"stream": "users/table_6/tblSXpxKHg0OiLxbI", "data": {"_airtable_id": "recoY6ShPkGpav3Se", "_airtable_created_time": "2023-01-25T02:04:26.000Z", "_airtable_table_name": "Table 6", "name": "test_normal", "table_6": ["recoY6ShPkGpav3Se", "recHyy86oge9j5cYP", "recLhKYa9btCCqmxs"], "float": 0.7, "status": "Done", "integer": 2.0, "assignee": 2.0, "assignee_(from_table_6)": [2.0, 2.0, 2.0]}, "emitted_at": 1696938632677} +{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "rec0UJKftqCPj7zJu", "_airtable_created_time": "2022-12-01T20:27:46.000Z", "_airtable_table_name": "Field Type Test", "email": "integration-test-user@airbyte.io", "id": 17.0, "date": "2022-09-30", "multiple_select_(no_colors)": ["1", "2", "3"], "percent": 0.0, "rating": 5.0, "multiple_select_(colors)": ["1", "2", "3"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "checkbox": true, "number": 1.0, "duration": 0.0, "name": "John", "single_line": "Singe line", "attachment": "[{'id': 'attSgnAhwJ2lZ8Kvp', 'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/Y8itnpzrb7hl71FKTu4YGg/_R8gsIdRqXUwHrJc6z9q6mjZ-Ni73bEHNBzpvUXbST_dpyLyL4Buonbe7eEb1SbcjYOkO1TauNU0YBr1-DKNQ_y7t_DfL6Q8hSp3_p9er7I0YFvx572HHw3C5-qWoMaD/xHCsSiGEA6MkydtHO6CU5uh1oucL4LP-iT5RCJHTerg', 'filename': 'mitre_logs_646490_txt.txt', 'size': 350181, 'type': 'text/plain'}]", "phone": "(937) 999-999", "value": 1.0, "long_text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas vitae nisi nec justo laoreet tincidunt. Vivamus eget porttitor velit. Pellentesque gravida euismod massa, eget egestas sem facilisis gravida. Suspendisse quis mauris eget velit faucibus aliquam. Mauris porttitor urna lorem, eget tincidunt ex faucibus et. Nam viverra nibh quis turpis scelerisque, et condimentum nibh vulputate. Vivamus eu dolor posuere, finibus ligula vel, finibus erat. Sed hendrerit luctus erat, eu finibus ante blandit ut. Sed id mi ullamcorper, cursus urna nec, molestie lorem. In vulputate tempor nulla in laoreet. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Sed a risus risus. Praesent neque lorem, fringilla vitae rhoncus a, semper eu augue.", "single_select_(colors)": "1", "single_select_(no_colors)": "1", "url": "airbyte.io", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:46.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "created_(w/o_time)": "2022-12-01"}, "emitted_at": 1696938633884} +{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "rec3tSj3Yzi42uuS0", "_airtable_created_time": "2022-09-30T16:06:49.000Z", "_airtable_table_name": "Field Type Test", "id": 3.0, "100_columns": ["reccHYZ004lOuxLFH"], "name": "Blank", "created_(with_time)": "2022-09-30T16:06:49.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "created_(w/o_time)": "2022-09-30"}, "emitted_at": 1696938633885} +{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recAa5DUEhppIeeax", "_airtable_created_time": "2022-12-01T20:27:30.000Z", "_airtable_table_name": "Field Type Test", "email": "integration-test-user@airbyte.io", "id": 9.0, "date": "2022-09-30", "multiple_select_(no_colors)": ["1", "2", "3"], "percent": 0.0, "rating": 5.0, "multiple_select_(colors)": ["1", "2", "3"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "checkbox": true, "number": 1.0, "duration": 0.0, "name": "John", "single_line": "Singe line", "attachment": "[{'id': 'attSgnAhwJ2lZ8Kvp', 'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/Y8itnpzrb7hl71FKTu4YGg/_R8gsIdRqXUwHrJc6z9q6mjZ-Ni73bEHNBzpvUXbST_dpyLyL4Buonbe7eEb1SbcjYOkO1TauNU0YBr1-DKNQ_y7t_DfL6Q8hSp3_p9er7I0YFvx572HHw3C5-qWoMaD/xHCsSiGEA6MkydtHO6CU5uh1oucL4LP-iT5RCJHTerg', 'filename': 'mitre_logs_646490_txt.txt', 'size': 350181, 'type': 'text/plain'}]", "phone": "(937) 999-999", "value": 1.0, "long_text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas vitae nisi nec justo laoreet tincidunt. Vivamus eget porttitor velit. Pellentesque gravida euismod massa, eget egestas sem facilisis gravida. Suspendisse quis mauris eget velit faucibus aliquam. Mauris porttitor urna lorem, eget tincidunt ex faucibus et. Nam viverra nibh quis turpis scelerisque, et condimentum nibh vulputate. Vivamus eu dolor posuere, finibus ligula vel, finibus erat. Sed hendrerit luctus erat, eu finibus ante blandit ut. Sed id mi ullamcorper, cursus urna nec, molestie lorem. In vulputate tempor nulla in laoreet. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Sed a risus risus. Praesent neque lorem, fringilla vitae rhoncus a, semper eu augue.", "single_select_(colors)": "1", "single_select_(no_colors)": "1", "url": "airbyte.io", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:30.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "created_(w/o_time)": "2022-12-01"}, "emitted_at": 1696938633885} +{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recG989xsdfSxookl", "_airtable_created_time": "2022-12-01T20:27:46.000Z", "_airtable_table_name": "Field Type Test", "email": "integration-test-user@airbyte.io", "id": 23.0, "date": "2022-10-03", "multiple_select_(no_colors)": ["a", "b", "c"], "percent": 1.0, "rating": 3.0, "multiple_select_(colors)": ["a", "b", "c"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "number": 123123871234.33, "duration": 169409880.0, "name": "Jane", "single_line": "09-30-2022", "attachment": "[{'id': 'attCx3wJdA0YahG34', 'width': 600, 'height': 201, 'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/GtCXXPc0drVyNXCGrHTKtQ/EpCnckqACDWilBmEQ813iNnUYsAIpFACzmaRyRM6MfIuJSCkSuE52JBoDif3Tv9VAgL5w8_cmUW8YksIridKKDQ5H2k8PF4cLMaPBIC6ETw/_hwChm4Y0VO637E-wzPTTBUbmWdQG7p3eCvtqlz1pxk', 'filename': 'email_signature.png', 'size': 112100, 'type': 'image/png', 'thumbnails': {'small': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/A3hZizS1p6ru5dRv94QFVQ/wqvVAY05wslLoJQRWdVKVXFTkK34efRjnVZfWO9ms6oB4DTuNuv9spB3O0tlp1u-tlSpU9pjYG2gw6AMWE9hqw/GglLpzscgR-QmuNgdkyw92fmefw17nYgYJMUIMko9Vg', 'width': 107, 'height': 36}, 'large': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/InjgprifvmFI-fSytEZ2Dg/yXLpCoZkT8erobAoDJxfZ18jR7XKEd71g04ZCoSo91WFYeoqAcPJPbGMIWvApVz8LV1zJZH3ED4fGq587Sbz1g/Y7daidsxdknj_ai8qlcb6ANySIFDhBH5k6ZoL7fPlLc', 'width': 600, 'height': 201}, 'full': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/ZQGMVCP7kSzt0A3Lw0I2FQ/HwqGkC2DaQ7Crnk7xXDs8h3y-C1H6NjmoqWd_3_xaagH80U1fmW2LA1BarNVFE6VOZWyGQIsm_bAZG6OO3kwWQ/5mIBdFM8bUOUfsEJP1FXf30OzYlGNbVt_59Sfc2NHbs', 'width': 3000, 'height': 3000}}}]", "phone": "(937) 999-999", "value": 1128312.23, "long_text": "18923671273628376328495623874562981345982 234953249857239045798 342098523495723495732 23489532475\n\n23453245234532453245234523453245\n\n2345234532452345", "single_select_(colors)": "a", "single_select_(no_colors)": "a", "url": "www.google.com", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:46.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "created_(w/o_time)": "2022-12-01"}, "emitted_at": 1696938633886} +{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recGVqF9BIpDD7Gj1", "_airtable_created_time": "2022-12-01T20:27:46.000Z", "_airtable_table_name": "Field Type Test", "email": "integration-test-user@airbyte.io", "id": 22.0, "date": "2022-09-30", "multiple_select_(no_colors)": ["1", "2", "3"], "percent": 0.0, "rating": 5.0, "multiple_select_(colors)": ["1", "2", "3"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "checkbox": true, "number": 1.0, "duration": 0.0, "name": "John", "single_line": "Singe line", "attachment": "[{'id': 'attSgnAhwJ2lZ8Kvp', 'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/Y8itnpzrb7hl71FKTu4YGg/_R8gsIdRqXUwHrJc6z9q6mjZ-Ni73bEHNBzpvUXbST_dpyLyL4Buonbe7eEb1SbcjYOkO1TauNU0YBr1-DKNQ_y7t_DfL6Q8hSp3_p9er7I0YFvx572HHw3C5-qWoMaD/xHCsSiGEA6MkydtHO6CU5uh1oucL4LP-iT5RCJHTerg', 'filename': 'mitre_logs_646490_txt.txt', 'size': 350181, 'type': 'text/plain'}]", "phone": "(937) 999-999", "value": 1.0, "long_text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas vitae nisi nec justo laoreet tincidunt. Vivamus eget porttitor velit. Pellentesque gravida euismod massa, eget egestas sem facilisis gravida. Suspendisse quis mauris eget velit faucibus aliquam. Mauris porttitor urna lorem, eget tincidunt ex faucibus et. Nam viverra nibh quis turpis scelerisque, et condimentum nibh vulputate. Vivamus eu dolor posuere, finibus ligula vel, finibus erat. Sed hendrerit luctus erat, eu finibus ante blandit ut. Sed id mi ullamcorper, cursus urna nec, molestie lorem. In vulputate tempor nulla in laoreet. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Sed a risus risus. Praesent neque lorem, fringilla vitae rhoncus a, semper eu augue.", "single_select_(colors)": "1", "single_select_(no_colors)": "1", "url": "airbyte.io", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:46.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "created_(w/o_time)": "2022-12-01"}, "emitted_at": 1696938633886} +{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recHNlOoFvp6KjeZD", "_airtable_created_time": "2022-12-01T20:27:46.000Z", "_airtable_table_name": "Field Type Test", "id": 16.0, "name": "Blank", "created_(with_time)": "2022-12-01T20:27:46.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "created_(w/o_time)": "2022-12-01"}, "emitted_at": 1696938633887} +{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recL6sfQP0FyJr3Ua", "_airtable_created_time": "2022-12-01T20:27:46.000Z", "_airtable_table_name": "Field Type Test", "id": 21.0, "name": "Blank", "created_(with_time)": "2022-12-01T20:27:46.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "created_(w/o_time)": "2022-12-01"}, "emitted_at": 1696938633887} +{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recMtt4tMmdNbwV5R", "_airtable_created_time": "2022-12-01T20:27:20.000Z", "_airtable_table_name": "Field Type Test", "id": 8.0, "name": "Blank", "created_(with_time)": "2022-12-01T20:27:20.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "created_(w/o_time)": "2022-12-01"}, "emitted_at": 1696938633887} +{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recOz5BBo3Ey3ldb8", "_airtable_created_time": "2022-12-01T20:27:11.000Z", "_airtable_table_name": "Field Type Test", "email": "integration-test-user@airbyte.io", "id": 5.0, "date": "2022-10-03", "multiple_select_(no_colors)": ["a", "b", "c"], "percent": 1.0, "rating": 3.0, "multiple_select_(colors)": ["a", "b", "c"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "100_columns": ["recXVzpNmtsBcjrA2"], "number": 123123871234.33, "duration": 169409880.0, "name": "Jane", "single_line": "09-30-2022", "attachment": "[{'id': 'attCx3wJdA0YahG34', 'width': 600, 'height': 201, 'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/GtCXXPc0drVyNXCGrHTKtQ/EpCnckqACDWilBmEQ813iNnUYsAIpFACzmaRyRM6MfIuJSCkSuE52JBoDif3Tv9VAgL5w8_cmUW8YksIridKKDQ5H2k8PF4cLMaPBIC6ETw/_hwChm4Y0VO637E-wzPTTBUbmWdQG7p3eCvtqlz1pxk', 'filename': 'email_signature.png', 'size': 112100, 'type': 'image/png', 'thumbnails': {'small': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/A3hZizS1p6ru5dRv94QFVQ/wqvVAY05wslLoJQRWdVKVXFTkK34efRjnVZfWO9ms6oB4DTuNuv9spB3O0tlp1u-tlSpU9pjYG2gw6AMWE9hqw/GglLpzscgR-QmuNgdkyw92fmefw17nYgYJMUIMko9Vg', 'width': 107, 'height': 36}, 'large': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/InjgprifvmFI-fSytEZ2Dg/yXLpCoZkT8erobAoDJxfZ18jR7XKEd71g04ZCoSo91WFYeoqAcPJPbGMIWvApVz8LV1zJZH3ED4fGq587Sbz1g/Y7daidsxdknj_ai8qlcb6ANySIFDhBH5k6ZoL7fPlLc', 'width': 600, 'height': 201}, 'full': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/ZQGMVCP7kSzt0A3Lw0I2FQ/HwqGkC2DaQ7Crnk7xXDs8h3y-C1H6NjmoqWd_3_xaagH80U1fmW2LA1BarNVFE6VOZWyGQIsm_bAZG6OO3kwWQ/5mIBdFM8bUOUfsEJP1FXf30OzYlGNbVt_59Sfc2NHbs', 'width': 3000, 'height': 3000}}}]", "phone": "(937) 999-999", "value": 1128312.23, "long_text": "18923671273628376328495623874562981345982 234953249857239045798 342098523495723495732 23489532475\n\n23453245234532453245234523453245\n\n2345234532452345", "single_select_(colors)": "a", "single_select_(no_colors)": "a", "url": "www.google.com", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:11.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "created_(w/o_time)": "2022-12-01"}, "emitted_at": 1696938633888} +{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recSbSBPspxs22165", "_airtable_created_time": "2022-12-01T20:27:46.000Z", "_airtable_table_name": "Field Type Test", "email": "integration-test-user@airbyte.io", "id": 15.0, "date": "2022-10-03", "multiple_select_(no_colors)": ["a", "b", "c"], "percent": 1.0, "rating": 3.0, "multiple_select_(colors)": ["a", "b", "c"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "number": 123123871234.33, "duration": 169409880.0, "name": "Jane", "single_line": "09-30-2022", "attachment": "[{'id': 'attCx3wJdA0YahG34', 'width': 600, 'height': 201, 'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/GtCXXPc0drVyNXCGrHTKtQ/EpCnckqACDWilBmEQ813iNnUYsAIpFACzmaRyRM6MfIuJSCkSuE52JBoDif3Tv9VAgL5w8_cmUW8YksIridKKDQ5H2k8PF4cLMaPBIC6ETw/_hwChm4Y0VO637E-wzPTTBUbmWdQG7p3eCvtqlz1pxk', 'filename': 'email_signature.png', 'size': 112100, 'type': 'image/png', 'thumbnails': {'small': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/A3hZizS1p6ru5dRv94QFVQ/wqvVAY05wslLoJQRWdVKVXFTkK34efRjnVZfWO9ms6oB4DTuNuv9spB3O0tlp1u-tlSpU9pjYG2gw6AMWE9hqw/GglLpzscgR-QmuNgdkyw92fmefw17nYgYJMUIMko9Vg', 'width': 107, 'height': 36}, 'large': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/InjgprifvmFI-fSytEZ2Dg/yXLpCoZkT8erobAoDJxfZ18jR7XKEd71g04ZCoSo91WFYeoqAcPJPbGMIWvApVz8LV1zJZH3ED4fGq587Sbz1g/Y7daidsxdknj_ai8qlcb6ANySIFDhBH5k6ZoL7fPlLc', 'width': 600, 'height': 201}, 'full': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/ZQGMVCP7kSzt0A3Lw0I2FQ/HwqGkC2DaQ7Crnk7xXDs8h3y-C1H6NjmoqWd_3_xaagH80U1fmW2LA1BarNVFE6VOZWyGQIsm_bAZG6OO3kwWQ/5mIBdFM8bUOUfsEJP1FXf30OzYlGNbVt_59Sfc2NHbs', 'width': 3000, 'height': 3000}}}]", "phone": "(937) 999-999", "value": 1128312.23, "long_text": "18923671273628376328495623874562981345982 234953249857239045798 342098523495723495732 23489532475\n\n23453245234532453245234523453245\n\n2345234532452345", "single_select_(colors)": "a", "single_select_(no_colors)": "a", "url": "www.google.com", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:46.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "created_(w/o_time)": "2022-12-01"}, "emitted_at": 1696938633888} +{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recUzRSGAf5VYtXfZ", "_airtable_created_time": "2022-09-30T16:06:49.000Z", "_airtable_table_name": "Field Type Test", "email": "integration-test-user@airbyte.io", "id": 1.0, "date": "2022-09-30", "multiple_select_(no_colors)": ["1", "2", "3"], "percent": 0.0, "rating": 5.0, "multiple_select_(colors)": ["1", "2", "3"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "checkbox": true, "100_columns": ["rec2lCkcy9d0fZi8h"], "number": 1.0, "duration": 0.0, "name": "John", "single_line": "Singe line", "attachment": "[{'id': 'attSgnAhwJ2lZ8Kvp', 'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/Y8itnpzrb7hl71FKTu4YGg/_R8gsIdRqXUwHrJc6z9q6mjZ-Ni73bEHNBzpvUXbST_dpyLyL4Buonbe7eEb1SbcjYOkO1TauNU0YBr1-DKNQ_y7t_DfL6Q8hSp3_p9er7I0YFvx572HHw3C5-qWoMaD/xHCsSiGEA6MkydtHO6CU5uh1oucL4LP-iT5RCJHTerg', 'filename': 'mitre_logs_646490_txt.txt', 'size': 350181, 'type': 'text/plain'}]", "phone": "(937) 999-999", "value": 1.0, "long_text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas vitae nisi nec justo laoreet tincidunt. Vivamus eget porttitor velit. Pellentesque gravida euismod massa, eget egestas sem facilisis gravida. Suspendisse quis mauris eget velit faucibus aliquam. Mauris porttitor urna lorem, eget tincidunt ex faucibus et. Nam viverra nibh quis turpis scelerisque, et condimentum nibh vulputate. Vivamus eu dolor posuere, finibus ligula vel, finibus erat. Sed hendrerit luctus erat, eu finibus ante blandit ut. Sed id mi ullamcorper, cursus urna nec, molestie lorem. In vulputate tempor nulla in laoreet. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Sed a risus risus. Praesent neque lorem, fringilla vitae rhoncus a, semper eu augue.", "single_select_(colors)": "1", "single_select_(no_colors)": "1", "url": "airbyte.io", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-09-30T16:06:49.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "created_(w/o_time)": "2022-09-30"}, "emitted_at": 1696938633889} +{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recbQy9sb5BM1K3Z1", "_airtable_created_time": "2022-12-01T20:27:17.000Z", "_airtable_table_name": "Field Type Test", "email": "integration-test-user@airbyte.io", "id": 6.0, "date": "2022-09-30", "multiple_select_(no_colors)": ["1", "2", "3"], "percent": 0.0, "rating": 5.0, "multiple_select_(colors)": ["1", "2", "3"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "checkbox": true, "number": 1.0, "duration": 0.0, "name": "John", "single_line": "Singe line", "attachment": "[{'id': 'attSgnAhwJ2lZ8Kvp', 'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/Y8itnpzrb7hl71FKTu4YGg/_R8gsIdRqXUwHrJc6z9q6mjZ-Ni73bEHNBzpvUXbST_dpyLyL4Buonbe7eEb1SbcjYOkO1TauNU0YBr1-DKNQ_y7t_DfL6Q8hSp3_p9er7I0YFvx572HHw3C5-qWoMaD/xHCsSiGEA6MkydtHO6CU5uh1oucL4LP-iT5RCJHTerg', 'filename': 'mitre_logs_646490_txt.txt', 'size': 350181, 'type': 'text/plain'}]", "phone": "(937) 999-999", "value": 1.0, "long_text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas vitae nisi nec justo laoreet tincidunt. Vivamus eget porttitor velit. Pellentesque gravida euismod massa, eget egestas sem facilisis gravida. Suspendisse quis mauris eget velit faucibus aliquam. Mauris porttitor urna lorem, eget tincidunt ex faucibus et. Nam viverra nibh quis turpis scelerisque, et condimentum nibh vulputate. Vivamus eu dolor posuere, finibus ligula vel, finibus erat. Sed hendrerit luctus erat, eu finibus ante blandit ut. Sed id mi ullamcorper, cursus urna nec, molestie lorem. In vulputate tempor nulla in laoreet. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Sed a risus risus. Praesent neque lorem, fringilla vitae rhoncus a, semper eu augue.", "single_select_(colors)": "1", "single_select_(no_colors)": "1", "url": "airbyte.io", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:17.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "created_(w/o_time)": "2022-12-01"}, "emitted_at": 1696938633889} +{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recehwpjPyD1PLFkr", "_airtable_created_time": "2022-12-01T20:27:10.000Z", "_airtable_table_name": "Field Type Test", "email": "integration-test-user@airbyte.io", "id": 4.0, "date": "2022-09-30", "multiple_select_(no_colors)": ["1", "2", "3"], "percent": 0.0, "rating": 5.0, "multiple_select_(colors)": ["1", "2", "3"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "checkbox": true, "number": 1.0, "duration": 0.0, "name": "John", "single_line": "Singe line", "attachment": "[{'id': 'attSgnAhwJ2lZ8Kvp', 'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/Y8itnpzrb7hl71FKTu4YGg/_R8gsIdRqXUwHrJc6z9q6mjZ-Ni73bEHNBzpvUXbST_dpyLyL4Buonbe7eEb1SbcjYOkO1TauNU0YBr1-DKNQ_y7t_DfL6Q8hSp3_p9er7I0YFvx572HHw3C5-qWoMaD/xHCsSiGEA6MkydtHO6CU5uh1oucL4LP-iT5RCJHTerg', 'filename': 'mitre_logs_646490_txt.txt', 'size': 350181, 'type': 'text/plain'}]", "phone": "(937) 999-999", "value": 1.0, "long_text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas vitae nisi nec justo laoreet tincidunt. Vivamus eget porttitor velit. Pellentesque gravida euismod massa, eget egestas sem facilisis gravida. Suspendisse quis mauris eget velit faucibus aliquam. Mauris porttitor urna lorem, eget tincidunt ex faucibus et. Nam viverra nibh quis turpis scelerisque, et condimentum nibh vulputate. Vivamus eu dolor posuere, finibus ligula vel, finibus erat. Sed hendrerit luctus erat, eu finibus ante blandit ut. Sed id mi ullamcorper, cursus urna nec, molestie lorem. In vulputate tempor nulla in laoreet. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Sed a risus risus. Praesent neque lorem, fringilla vitae rhoncus a, semper eu augue.", "single_select_(colors)": "1", "single_select_(no_colors)": "1", "url": "airbyte.io", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:10.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "created_(w/o_time)": "2022-12-01"}, "emitted_at": 1696938633890} +{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recfdnbGNkUx1Rs5F", "_airtable_created_time": "2022-12-01T20:27:46.000Z", "_airtable_table_name": "Field Type Test", "email": "integration-test-user@airbyte.io", "id": 19.0, "date": "2022-09-30", "multiple_select_(no_colors)": ["1", "2", "3"], "percent": 0.0, "rating": 5.0, "multiple_select_(colors)": ["1", "2", "3"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "checkbox": true, "number": 1.0, "duration": 0.0, "name": "John", "single_line": "Singe line", "attachment": "[{'id': 'attSgnAhwJ2lZ8Kvp', 'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/Y8itnpzrb7hl71FKTu4YGg/_R8gsIdRqXUwHrJc6z9q6mjZ-Ni73bEHNBzpvUXbST_dpyLyL4Buonbe7eEb1SbcjYOkO1TauNU0YBr1-DKNQ_y7t_DfL6Q8hSp3_p9er7I0YFvx572HHw3C5-qWoMaD/xHCsSiGEA6MkydtHO6CU5uh1oucL4LP-iT5RCJHTerg', 'filename': 'mitre_logs_646490_txt.txt', 'size': 350181, 'type': 'text/plain'}]", "phone": "(937) 999-999", "value": 1.0, "long_text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas vitae nisi nec justo laoreet tincidunt. Vivamus eget porttitor velit. Pellentesque gravida euismod massa, eget egestas sem facilisis gravida. Suspendisse quis mauris eget velit faucibus aliquam. Mauris porttitor urna lorem, eget tincidunt ex faucibus et. Nam viverra nibh quis turpis scelerisque, et condimentum nibh vulputate. Vivamus eu dolor posuere, finibus ligula vel, finibus erat. Sed hendrerit luctus erat, eu finibus ante blandit ut. Sed id mi ullamcorper, cursus urna nec, molestie lorem. In vulputate tempor nulla in laoreet. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Sed a risus risus. Praesent neque lorem, fringilla vitae rhoncus a, semper eu augue.", "single_select_(colors)": "1", "single_select_(no_colors)": "1", "url": "airbyte.io", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:46.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "created_(w/o_time)": "2022-12-01"}, "emitted_at": 1696938633890} +{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recpkKhLwcFYbJLtJ", "_airtable_created_time": "2022-12-01T20:27:43.000Z", "_airtable_table_name": "Field Type Test", "email": "integration-test-user@airbyte.io", "id": 14.0, "date": "2022-09-30", "multiple_select_(no_colors)": ["1", "2", "3"], "percent": 0.0, "rating": 5.0, "multiple_select_(colors)": ["1", "2", "3"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "checkbox": true, "number": 1.0, "duration": 0.0, "name": "John", "single_line": "Singe line", "attachment": "[{'id': 'attSgnAhwJ2lZ8Kvp', 'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/Y8itnpzrb7hl71FKTu4YGg/_R8gsIdRqXUwHrJc6z9q6mjZ-Ni73bEHNBzpvUXbST_dpyLyL4Buonbe7eEb1SbcjYOkO1TauNU0YBr1-DKNQ_y7t_DfL6Q8hSp3_p9er7I0YFvx572HHw3C5-qWoMaD/xHCsSiGEA6MkydtHO6CU5uh1oucL4LP-iT5RCJHTerg', 'filename': 'mitre_logs_646490_txt.txt', 'size': 350181, 'type': 'text/plain'}]", "phone": "(937) 999-999", "value": 1.0, "long_text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas vitae nisi nec justo laoreet tincidunt. Vivamus eget porttitor velit. Pellentesque gravida euismod massa, eget egestas sem facilisis gravida. Suspendisse quis mauris eget velit faucibus aliquam. Mauris porttitor urna lorem, eget tincidunt ex faucibus et. Nam viverra nibh quis turpis scelerisque, et condimentum nibh vulputate. Vivamus eu dolor posuere, finibus ligula vel, finibus erat. Sed hendrerit luctus erat, eu finibus ante blandit ut. Sed id mi ullamcorper, cursus urna nec, molestie lorem. In vulputate tempor nulla in laoreet. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Sed a risus risus. Praesent neque lorem, fringilla vitae rhoncus a, semper eu augue.", "single_select_(colors)": "1", "single_select_(no_colors)": "1", "url": "airbyte.io", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:43.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "created_(w/o_time)": "2022-12-01"}, "emitted_at": 1696938633891} +{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recqurXMR6u1PO5wT", "_airtable_created_time": "2022-12-01T20:27:18.000Z", "_airtable_table_name": "Field Type Test", "email": "integration-test-user@airbyte.io", "id": 7.0, "date": "2022-10-03", "multiple_select_(no_colors)": ["a", "b", "c"], "percent": 1.0, "rating": 3.0, "multiple_select_(colors)": ["a", "b", "c"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "number": 123123871234.33, "duration": 169409880.0, "name": "Jane", "single_line": "09-30-2022", "attachment": "[{'id': 'attCx3wJdA0YahG34', 'width': 600, 'height': 201, 'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/GtCXXPc0drVyNXCGrHTKtQ/EpCnckqACDWilBmEQ813iNnUYsAIpFACzmaRyRM6MfIuJSCkSuE52JBoDif3Tv9VAgL5w8_cmUW8YksIridKKDQ5H2k8PF4cLMaPBIC6ETw/_hwChm4Y0VO637E-wzPTTBUbmWdQG7p3eCvtqlz1pxk', 'filename': 'email_signature.png', 'size': 112100, 'type': 'image/png', 'thumbnails': {'small': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/A3hZizS1p6ru5dRv94QFVQ/wqvVAY05wslLoJQRWdVKVXFTkK34efRjnVZfWO9ms6oB4DTuNuv9spB3O0tlp1u-tlSpU9pjYG2gw6AMWE9hqw/GglLpzscgR-QmuNgdkyw92fmefw17nYgYJMUIMko9Vg', 'width': 107, 'height': 36}, 'large': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/InjgprifvmFI-fSytEZ2Dg/yXLpCoZkT8erobAoDJxfZ18jR7XKEd71g04ZCoSo91WFYeoqAcPJPbGMIWvApVz8LV1zJZH3ED4fGq587Sbz1g/Y7daidsxdknj_ai8qlcb6ANySIFDhBH5k6ZoL7fPlLc', 'width': 600, 'height': 201}, 'full': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/ZQGMVCP7kSzt0A3Lw0I2FQ/HwqGkC2DaQ7Crnk7xXDs8h3y-C1H6NjmoqWd_3_xaagH80U1fmW2LA1BarNVFE6VOZWyGQIsm_bAZG6OO3kwWQ/5mIBdFM8bUOUfsEJP1FXf30OzYlGNbVt_59Sfc2NHbs', 'width': 3000, 'height': 3000}}}]", "phone": "(937) 999-999", "value": 1128312.23, "long_text": "18923671273628376328495623874562981345982 234953249857239045798 342098523495723495732 23489532475\n\n23453245234532453245234523453245\n\n2345234532452345\n\n2345234532452345\n\n2345234532452345\n\n2345234532452345\n\n2345234532452345\n\n2345234532452345\n\n,2345234532452345,2345234532452345,2345234,2345234532452345532452345", "single_select_(colors)": "a", "single_select_(no_colors)": "a", "url": "www.google.com", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:18.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "created_(w/o_time)": "2022-12-01"}, "emitted_at": 1696938633891} +{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recuYRu9z5dlkHxop", "_airtable_created_time": "2022-12-01T20:27:30.000Z", "_airtable_table_name": "Field Type Test", "email": "integration-test-user@airbyte.io", "id": 10.0, "date": "2022-10-03", "multiple_select_(no_colors)": ["a", "b", "c"], "percent": 1.0, "rating": 3.0, "multiple_select_(colors)": ["a", "b", "c"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "number": 123123871234.33, "duration": 169409880.0, "name": "Jane", "single_line": "09-30-2022", "attachment": "[{'id': 'attCx3wJdA0YahG34', 'width': 600, 'height': 201, 'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/GtCXXPc0drVyNXCGrHTKtQ/EpCnckqACDWilBmEQ813iNnUYsAIpFACzmaRyRM6MfIuJSCkSuE52JBoDif3Tv9VAgL5w8_cmUW8YksIridKKDQ5H2k8PF4cLMaPBIC6ETw/_hwChm4Y0VO637E-wzPTTBUbmWdQG7p3eCvtqlz1pxk', 'filename': 'email_signature.png', 'size': 112100, 'type': 'image/png', 'thumbnails': {'small': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/A3hZizS1p6ru5dRv94QFVQ/wqvVAY05wslLoJQRWdVKVXFTkK34efRjnVZfWO9ms6oB4DTuNuv9spB3O0tlp1u-tlSpU9pjYG2gw6AMWE9hqw/GglLpzscgR-QmuNgdkyw92fmefw17nYgYJMUIMko9Vg', 'width': 107, 'height': 36}, 'large': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/InjgprifvmFI-fSytEZ2Dg/yXLpCoZkT8erobAoDJxfZ18jR7XKEd71g04ZCoSo91WFYeoqAcPJPbGMIWvApVz8LV1zJZH3ED4fGq587Sbz1g/Y7daidsxdknj_ai8qlcb6ANySIFDhBH5k6ZoL7fPlLc', 'width': 600, 'height': 201}, 'full': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/ZQGMVCP7kSzt0A3Lw0I2FQ/HwqGkC2DaQ7Crnk7xXDs8h3y-C1H6NjmoqWd_3_xaagH80U1fmW2LA1BarNVFE6VOZWyGQIsm_bAZG6OO3kwWQ/5mIBdFM8bUOUfsEJP1FXf30OzYlGNbVt_59Sfc2NHbs', 'width': 3000, 'height': 3000}}}]", "phone": "(937) 999-999", "value": 1128312.23, "long_text": "18923671273628376328495623874562981345982 234953249857239045798 342098523495723495732 23489532475\n\n23453245234532453245234523453245\n\n2345234532452345", "single_select_(colors)": "a", "single_select_(no_colors)": "a", "url": "www.google.com", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:30.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "created_(w/o_time)": "2022-12-01"}, "emitted_at": 1696938633891} +{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recw3jDJFziB562hu", "_airtable_created_time": "2022-12-01T20:27:46.000Z", "_airtable_table_name": "Field Type Test", "email": "integration-test-user@airbyte.io", "id": 18.0, "date": "2022-10-03", "multiple_select_(no_colors)": ["a", "b", "c"], "percent": 1.0, "rating": 3.0, "multiple_select_(colors)": ["a", "b", "c"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "number": 123123871234.33, "duration": 169409880.0, "name": "Jane", "single_line": "09-30-2022", "attachment": "[{'id': 'attCx3wJdA0YahG34', 'width': 600, 'height': 201, 'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/GtCXXPc0drVyNXCGrHTKtQ/EpCnckqACDWilBmEQ813iNnUYsAIpFACzmaRyRM6MfIuJSCkSuE52JBoDif3Tv9VAgL5w8_cmUW8YksIridKKDQ5H2k8PF4cLMaPBIC6ETw/_hwChm4Y0VO637E-wzPTTBUbmWdQG7p3eCvtqlz1pxk', 'filename': 'email_signature.png', 'size': 112100, 'type': 'image/png', 'thumbnails': {'small': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/A3hZizS1p6ru5dRv94QFVQ/wqvVAY05wslLoJQRWdVKVXFTkK34efRjnVZfWO9ms6oB4DTuNuv9spB3O0tlp1u-tlSpU9pjYG2gw6AMWE9hqw/GglLpzscgR-QmuNgdkyw92fmefw17nYgYJMUIMko9Vg', 'width': 107, 'height': 36}, 'large': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/InjgprifvmFI-fSytEZ2Dg/yXLpCoZkT8erobAoDJxfZ18jR7XKEd71g04ZCoSo91WFYeoqAcPJPbGMIWvApVz8LV1zJZH3ED4fGq587Sbz1g/Y7daidsxdknj_ai8qlcb6ANySIFDhBH5k6ZoL7fPlLc', 'width': 600, 'height': 201}, 'full': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/ZQGMVCP7kSzt0A3Lw0I2FQ/HwqGkC2DaQ7Crnk7xXDs8h3y-C1H6NjmoqWd_3_xaagH80U1fmW2LA1BarNVFE6VOZWyGQIsm_bAZG6OO3kwWQ/5mIBdFM8bUOUfsEJP1FXf30OzYlGNbVt_59Sfc2NHbs', 'width': 3000, 'height': 3000}}}]", "phone": "(937) 999-999", "value": 1128312.23, "long_text": "18923671273628376328495623874562981345982 234953249857239045798 342098523495723495732 23489532475\n\n23453245234532453245234523453245\n\n2345234532452345", "single_select_(colors)": "a", "single_select_(no_colors)": "a", "url": "www.google.com", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:46.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "created_(w/o_time)": "2022-12-01"}, "emitted_at": 1696938633892} +{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "recyJGdkZikJbLFRi", "_airtable_created_time": "2022-09-30T16:06:49.000Z", "_airtable_table_name": "Field Type Test", "email": "integration-test-user@airbyte.io", "id": 2.0, "date": "2022-10-03", "multiple_select_(no_colors)": ["a", "b", "c"], "percent": 1.0, "rating": 3.0, "multiple_select_(colors)": ["a", "b", "c"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "number": 123123871234.33, "duration": 169409880.0, "name": "Jane", "single_line": "09-30-2022", "attachment": "[{'id': 'attCx3wJdA0YahG34', 'width': 600, 'height': 201, 'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/GtCXXPc0drVyNXCGrHTKtQ/EpCnckqACDWilBmEQ813iNnUYsAIpFACzmaRyRM6MfIuJSCkSuE52JBoDif3Tv9VAgL5w8_cmUW8YksIridKKDQ5H2k8PF4cLMaPBIC6ETw/_hwChm4Y0VO637E-wzPTTBUbmWdQG7p3eCvtqlz1pxk', 'filename': 'email_signature.png', 'size': 112100, 'type': 'image/png', 'thumbnails': {'small': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/A3hZizS1p6ru5dRv94QFVQ/wqvVAY05wslLoJQRWdVKVXFTkK34efRjnVZfWO9ms6oB4DTuNuv9spB3O0tlp1u-tlSpU9pjYG2gw6AMWE9hqw/GglLpzscgR-QmuNgdkyw92fmefw17nYgYJMUIMko9Vg', 'width': 107, 'height': 36}, 'large': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/InjgprifvmFI-fSytEZ2Dg/yXLpCoZkT8erobAoDJxfZ18jR7XKEd71g04ZCoSo91WFYeoqAcPJPbGMIWvApVz8LV1zJZH3ED4fGq587Sbz1g/Y7daidsxdknj_ai8qlcb6ANySIFDhBH5k6ZoL7fPlLc', 'width': 600, 'height': 201}, 'full': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/ZQGMVCP7kSzt0A3Lw0I2FQ/HwqGkC2DaQ7Crnk7xXDs8h3y-C1H6NjmoqWd_3_xaagH80U1fmW2LA1BarNVFE6VOZWyGQIsm_bAZG6OO3kwWQ/5mIBdFM8bUOUfsEJP1FXf30OzYlGNbVt_59Sfc2NHbs', 'width': 3000, 'height': 3000}}}]", "phone": "(937) 999-999", "value": 1128312.23, "long_text": "18923671273628376328495623874562981345982 234953249857239045798 342098523495723495732 23489532475\n\n23453245234532453245234523453245\n\n2345234532452345", "single_select_(colors)": "a", "single_select_(no_colors)": "a", "url": "www.google.com", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-09-30T16:06:49.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "created_(w/o_time)": "2022-09-30"}, "emitted_at": 1696938633892} +{"stream": "users/field_type_test/tblFcp5mncufoYaR9", "data": {"_airtable_id": "reczDi9vTuH3ezfJM", "_airtable_created_time": "2022-12-01T20:27:46.000Z", "_airtable_table_name": "Field Type Test", "email": "integration-test-user@airbyte.io", "id": 20.0, "date": "2022-10-03", "multiple_select_(no_colors)": ["a", "b", "c"], "percent": 1.0, "rating": 3.0, "multiple_select_(colors)": ["a", "b", "c"], "user_(non-multiple)": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "number": 123123871234.33, "duration": 169409880.0, "name": "Jane", "single_line": "09-30-2022", "attachment": "[{'id': 'attCx3wJdA0YahG34', 'width': 600, 'height': 201, 'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/GtCXXPc0drVyNXCGrHTKtQ/EpCnckqACDWilBmEQ813iNnUYsAIpFACzmaRyRM6MfIuJSCkSuE52JBoDif3Tv9VAgL5w8_cmUW8YksIridKKDQ5H2k8PF4cLMaPBIC6ETw/_hwChm4Y0VO637E-wzPTTBUbmWdQG7p3eCvtqlz1pxk', 'filename': 'email_signature.png', 'size': 112100, 'type': 'image/png', 'thumbnails': {'small': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/A3hZizS1p6ru5dRv94QFVQ/wqvVAY05wslLoJQRWdVKVXFTkK34efRjnVZfWO9ms6oB4DTuNuv9spB3O0tlp1u-tlSpU9pjYG2gw6AMWE9hqw/GglLpzscgR-QmuNgdkyw92fmefw17nYgYJMUIMko9Vg', 'width': 107, 'height': 36}, 'large': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/InjgprifvmFI-fSytEZ2Dg/yXLpCoZkT8erobAoDJxfZ18jR7XKEd71g04ZCoSo91WFYeoqAcPJPbGMIWvApVz8LV1zJZH3ED4fGq587Sbz1g/Y7daidsxdknj_ai8qlcb6ANySIFDhBH5k6ZoL7fPlLc', 'width': 600, 'height': 201}, 'full': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/ZQGMVCP7kSzt0A3Lw0I2FQ/HwqGkC2DaQ7Crnk7xXDs8h3y-C1H6NjmoqWd_3_xaagH80U1fmW2LA1BarNVFE6VOZWyGQIsm_bAZG6OO3kwWQ/5mIBdFM8bUOUfsEJP1FXf30OzYlGNbVt_59Sfc2NHbs', 'width': 3000, 'height': 3000}}}]", "phone": "(937) 999-999", "value": 1128312.23, "long_text": "18923671273628376328495623874562981345982 234953249857239045798 342098523495723495732 23489532475\n\n23453245234532453245234523453245\n\n2345234532452345", "single_select_(colors)": "a", "single_select_(no_colors)": "a", "url": "www.google.com", "user_(multiple)": ["{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"], "created_(with_time)": "2022-12-01T20:27:46.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "created_(w/o_time)": "2022-12-01"}, "emitted_at": 1696938633893} +{"stream": "users/50_columns/tbl01Hi93Tt6XJ0u5", "data": {"_airtable_id": "rec2lCkcy9d0fZi8h", "_airtable_created_time": "2022-12-02T17:29:41.000Z", "_airtable_table_name": "50 Columns", "notes": "dasdafsdag", "phone_3": "(999) 999-9999", "tags_2": ["alpha", "issue"], "select_2": "top", "date_2": "2023-01-03", "euro": 136.46, "collaborator": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "number": 423.0, "assignee": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "percent_2": 0.45299999999999996, "url": "airbyte.com", "duration": 9900.0, "notes_3": " dfhsh f g dh", "status": "Done", "rating_2": 1.0, "barcode": "{'text': '312515435'}", "number_2": 3453.0, "rating": 4.0, "field_type_test": ["recUzRSGAf5VYtXfZ"], "usd": 143.64, "done": true, "id": 1.0, "notes_2": "1. fsafsf sfkjkl fsafs", "phone_2": "(222) 222-2222", "email": "email@airbyte.io", "percent_3": 0.24300000000000002, "select": "1st", "int": 65.0, "label_2": "Lorem", "date": "2022-12-02", "attachments": "[{'id': 'attX9RxylAGErJfMO', 'width': 973, 'height': 956, 'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/LYbNOJhFCbE3YIt3AC40pQ/8WIoUQBqzhD_iXjNzKzuI2u8UUJfal1bPp2exeEEeHOsMB3qgWk29-hiI7_D-bQ-vmfEfKebz2DVTUQ0kaMb7R2ncA_7HMkADX8FY3YgwZG6kxuipsUo-KZ6_wL6ED5i/2AhZIljLuOrh4RWj_FELvEovXwRB6b9HSW-FtyCZv3A', 'filename': 'octavia-hello-no-text.png', 'size': 248967, 'type': 'image/png', 'thumbnails': {'small': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/9YRq4yz0tWG8mc2qvFymWg/PqMhO_hx03gUpay3XPgCv_yU9noBM4PwjDMUmNLdZoFVKihAsvFOXmt_pP0CU_ssAm51K4pDxVovDmUGvNp9OQ/Kbq2ktuWzZ4X6X8oGk1YTkDcxoC9Ms6ohHbwZTFLdRU', 'width': 37, 'height': 36}, 'large': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/8SVA5qtF0SLiIOw2jlGZdQ/vcq1wQ-DM8IXYsagKln5wCGSd9uPy1Pc44wzdTnEBn8Xkp1zqEC2Un8Ovhe0m2bfufcPW8xQP8PpCp0W04aZrA/uLrD757BiMCeT-S1Vb2ljtitkUdRtpqMtMknyzXNOxI', 'width': 521, 'height': 512}, 'full': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/a_EVexsFKtYSgS20L_lBZA/AXmtleltix2zsVV7eAEEwYG9fVwXI6Kb4RXqqwppML7FwoEa9otYRiYgRVtq6uvAVIYazmUgVBXZm6EwW3VDLw/uUFc4xeoQ35vpPpvMNIvnoArgoaeoMVCXE5ocEMIAI4', 'width': 3000, 'height': 3000}}}]", "label_3": "line", "name": "Thing 1", "phone": "(999) 999-9999", "label": "Nulla quis lorem ut libero malesuada feugiat.", "duration_2": 20580.0, "tags": ["tag1"], "decimal": 5.8797, "percent": 0.2, "email_(from_field_type_test)": ["integration-test-user@airbyte.io"], "count_(field_type_test)": 1.0, "created_(w/o_time)_(from_field_type_test)": ["2022-09-30"], "created": "2022-12-02T17:29:41.000Z", "last_modified": "2023-01-24T11:12:27.000Z", "last_modified_2": "2023-01-24T11:12:27.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1696938634827} +{"stream": "users/50_columns/tbl01Hi93Tt6XJ0u5", "data": {"_airtable_id": "recCn7Z7bgfxWraJu", "_airtable_created_time": "2023-10-05T18:56:16.000Z", "_airtable_table_name": "50 Columns", "id": 4.0, "name": "email@airbyte.io", "count_(field_type_test)": 0.0, "created": "2023-10-05T18:56:16.000Z", "last_modified": "2023-10-05T18:56:16.000Z", "last_modified_2": "2023-10-05T18:56:16.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1696938634828} +{"stream": "users/50_columns/tbl01Hi93Tt6XJ0u5", "data": {"_airtable_id": "recXVzpNmtsBcjrA2", "_airtable_created_time": "2022-12-02T17:29:41.000Z", "_airtable_table_name": "50 Columns", "notes": "opoiwepoirwe", "phone_3": "(999) 999-9999", "tags_2": ["ga"], "select_2": "bottom", "date_2": "2023-01-31", "euro": 279.62, "collaborator": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "number": 34535.5, "percent_2": 0.624, "duration": 10440.0, "notes_3": "hdfhe e et e true ttue ", "status": "Todo", "rating_2": 4.0, "barcode": "{'text': '351164554'}", "rating": 3.0, "field_type_test": ["recOz5BBo3Ey3ldb8"], "usd": 294.34, "done": true, "url_2": "airbyte.com", "id": 3.0, "notes_2": "3. flsflkj;flkjsf fskjlf lakjf; lskaj;klsjf", "phone_2": "(444) 444-4444", "percent_3": 0.13699999999999998, "select": "3rd", "int": 98.0, "date": "2022-12-14", "label_3": "line", "name": "Thing 3", "phone": "(999) 999-9999", "label": "Curabitur aliquet quam id dui posuere blandit.", "duration_2": 19740.0, "decimal": 6.6, "percent": 0.3, "email_(from_field_type_test)": ["integration-test-user@airbyte.io"], "count_(field_type_test)": 1.0, "created_(w/o_time)_(from_field_type_test)": ["2022-12-01"], "created": "2022-12-02T17:29:41.000Z", "last_modified": "2023-01-24T11:12:12.000Z", "last_modified_2": "2023-01-24T11:12:12.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1696938634828} +{"stream": "users/50_columns/tbl01Hi93Tt6XJ0u5", "data": {"_airtable_id": "reccHYZ004lOuxLFH", "_airtable_created_time": "2022-12-02T17:29:41.000Z", "_airtable_table_name": "50 Columns", "notes": "hjyyfhgjjfgjr", "phone_3": "(999) 999-9999", "tags_2": ["beta", "no issue"], "attachments_2": "[{'id': 'att0nyUmkrLpgE1Aq', 'width': 973, 'height': 956, 'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/2x8ET5i91p4Ll2A4zNUWVw/vXrY6PAxyaLxUoJC-Ut5SThEbtjgC1--6WFDZXHG3RUbBZeNy1FAvGYZX5O1PIPqbSJng_PojxxroQb2WoOK3OgzY3nT5BjlW5afleGFjG4APPZ_AqM8P1j2ASNgCCRL/oXD_WPj-y1rsKSbSYRhVm3JyGa71QQZJnkXbqk-jVWA', 'filename': 'octavia-hello-no-text.png', 'size': 248967, 'type': 'image/png', 'thumbnails': {'small': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/02eZUnxhE_ebQRulYCqtiQ/xD3sqsLrJCRs-DGS3rmNllekXwyZr86oFrVtv5Ee3sT8MJ0lMBjqhA32tqFhM5jcIe8mSpx5MrAxwWF7BVlJAg/Fc4jrijErYUHv--PNrSDDxZi4wVSf3cRlMp6W9QPyLw', 'width': 37, 'height': 36}, 'large': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/AD9KHXD8hoPyUIBHuyLuYg/KP9uCskRiWL1lOuzqB_6Xqk9tcNPKWheE4JtAEBrOrjRhAp0LY4GKcRdQHkf_xD6ffrb34L31prh_4PVfXxdCg/TyE1QunwSfdw2DHF7IbQBmf6hUU0X0yUuUIt6AbOjwc', 'width': 521, 'height': 512}, 'full': {'url': 'https://v5.airtableusercontent.com/v1/21/21/1696946400000/3Qao-gbL6Lu54c9xJ4-2zg/bWGTH-4xKRO4P-9uyuHK5eGnCPQRYsckTdVqEcv6Wg7Z1evaH_uDx_vH8oaMRkPS_LaknGR-LtKyLWHOYlFs7g/IEdgquu8RG_1uuwg1RanLkHVQ-DqF_PiYBpxj0Hef4U', 'width': 3000, 'height': 3000}}}]", "select_2": "mid", "date_2": "2023-01-17", "euro": 325.13, "number": 22424.5, "assignee": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "percent_2": 0.562, "duration": 11520.0, "notes_3": "fhdfhdhgf hfh hdfgh", "status": "In progress", "barcode": "{'text': '5531515315'}", "number_2": 3452.0, "rating": 5.0, "field_type_test": ["rec3tSj3Yzi42uuS0"], "usd": 342.24, "url_2": "docs.airbyte.com", "id": 2.0, "notes_2": "2. fskldf f;sfkjk s;lkjfkls", "phone_2": "(333) 333-3333", "email": "what@airbyte.io", "percent_3": 0.532, "select": "2nd", "int": 53.0, "date": "2022-11-10", "label_3": "line", "name": "Thing 2", "phone": "(999) 999-9999", "label": "Quisque velit nisi, pretium ut lacinia in, elementum id enim.", "duration_2": 18180.0, "collaborator_2": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "tags": ["tag2", "tag3"], "done_2": true, "decimal": 4.134, "percent": 0.15, "count_(field_type_test)": 1.0, "created_(w/o_time)_(from_field_type_test)": ["2022-09-30"], "created": "2022-12-02T17:29:41.000Z", "last_modified": "2023-01-24T11:12:11.000Z", "last_modified_2": "2023-01-24T11:12:11.000Z", "created_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}", "last_modified_by": "{'id': 'usrr0cUCbVccLxU7x', 'email': 'integration-test@airbyte.io', 'name': 'Airbyte Team'}"}, "emitted_at": 1696938634829} +{"stream": "users/checkboxes/tbl81WIAUZg5nwGN8", "data": {"_airtable_id": "recLvpJ4k4mRG38My", "_airtable_created_time": "2022-12-02T19:50:00.000Z", "_airtable_table_name": "Checkboxes", "name": "Cloud", "done_2": true, "done_4": true, "done_6": true, "done_7": true, "done_10": true, "done_11": true, "done_12": true, "done_16": true, "done_18": true, "done_20": true, "done_21": true, "done_22": true, "done_23": true, "done_24": true}, "emitted_at": 1696938635946} +{"stream": "users/checkboxes/tbl81WIAUZg5nwGN8", "data": {"_airtable_id": "reckszXRiFfg11IYD", "_airtable_created_time": "2022-12-02T19:50:00.000Z", "_airtable_table_name": "Checkboxes", "name": "Support", "done_3": true, "done_5": true, "done_6": true, "done_7": true, "done_9": true, "done_11": true, "done_13": true, "done_16": true, "done_17": true, "done_21": true, "done_22": true, "done_25": true}, "emitted_at": 1696938635947} +{"stream": "users/checkboxes/tbl81WIAUZg5nwGN8", "data": {"_airtable_id": "recmaYwfPMvZtwTSJ", "_airtable_created_time": "2022-12-02T19:50:00.000Z", "_airtable_table_name": "Checkboxes", "name": "Airbyte", "done": true, "done_4": true, "done_5": true, "done_7": true, "done_9": true, "done_12": true, "done_14": true, "done_16": true, "done_19": true, "done_20": true, "done_23": true, "done_25": true}, "emitted_at": 1696938635947} +{"stream": "untitled_base/table_1/tblT7mnwDS5TVtfOh", "data": {"_airtable_id": "recJ0l923fOFw6qbl", "_airtable_created_time": "2021-12-09T07:54:15.000Z", "_airtable_table_name": "Table 1", "name": "test2", "notes": "test2", "status": "In progress"}, "emitted_at": 1696938637044} +{"stream": "untitled_base/table_1/tblT7mnwDS5TVtfOh", "data": {"_airtable_id": "recNbrGzLJfOy6EjC", "_airtable_created_time": "2022-12-28T11:43:37.000Z", "_airtable_table_name": "Table 1", "name": "blank"}, "emitted_at": 1696938637044} +{"stream": "untitled_base/table_1/tblT7mnwDS5TVtfOh", "data": {"_airtable_id": "recZX1Je5k4nXhTi0", "_airtable_created_time": "2021-12-09T07:54:15.000Z", "_airtable_table_name": "Table 1", "name": "blank"}, "emitted_at": 1696938637044} +{"stream": "untitled_base/table_1/tblT7mnwDS5TVtfOh", "data": {"_airtable_id": "recxXAoQ0cC5yCMZ7", "_airtable_created_time": "2021-12-09T07:54:15.000Z", "_airtable_table_name": "Table 1", "name": "test1", "notes": "test", "status": "Todo"}, "emitted_at": 1696938637044} +{"stream": "untitled_base/table_1/tblT7mnwDS5TVtfOh", "data": {"_airtable_id": "reczJvdeo0b8KsM6K", "_airtable_created_time": "2022-12-28T11:43:38.000Z", "_airtable_table_name": "Table 1", "name": "test3", "notes": "test3", "status": "Done"}, "emitted_at": 1696938637045} diff --git a/airbyte-integrations/connectors/source-airtable/metadata.yaml b/airbyte-integrations/connectors/source-airtable/metadata.yaml index ea6c51182a8c..c722087101dd 100644 --- a/airbyte-integrations/connectors/source-airtable/metadata.yaml +++ b/airbyte-integrations/connectors/source-airtable/metadata.yaml @@ -1,13 +1,19 @@ data: + ab_internal: + ql: 400 + sl: 200 allowedHosts: hosts: - api.airtable.com - airtable.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: api connectorType: source definitionId: 14c6e7ea-97ed-4f5e-a7b5-25e9a80b8212 - dockerImageTag: 3.0.1 + dockerImageTag: 4.1.5 dockerRepository: airbyte/source-airtable + documentationUrl: https://docs.airbyte.com/integrations/sources/airtable githubIssueLabel: source-airtable icon: airtable.svg license: MIT @@ -18,11 +24,16 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/airtable + releases: + breakingChanges: + 4.0.0: + message: + This release introduces changes to columns with formula to parse + values directly from `array` to `string` or `number` (where it is possible). + Users should refresh the source schema and reset affected streams after + upgrading to ensure uninterrupted syncs. + upgradeDeadline: "2023-10-23" + supportLevel: certified tags: - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/auth.py b/airbyte-integrations/connectors/source-airtable/source_airtable/auth.py index 3e49d0855e62..692e09ecae40 100644 --- a/airbyte-integrations/connectors/source-airtable/source_airtable/auth.py +++ b/airbyte-integrations/connectors/source-airtable/source_airtable/auth.py @@ -5,11 +5,13 @@ from typing import Any, Mapping, Union import requests +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.streams.http.requests_native_auth import ( BasicHttpAuthenticator, SingleUseRefreshTokenOauth2Authenticator, TokenAuthenticator, ) +from airbyte_cdk.utils import AirbyteTracedException class AirtableOAuth(SingleUseRefreshTokenOauth2Authenticator): @@ -42,8 +44,15 @@ def _get_refresh_access_token_response(self) -> Mapping[str, Any]: data=self.build_refresh_request_body(), headers=self.build_refresh_request_headers(), ) + content = response.json() + if response.status_code == 400 and content.get("error") == "invalid_grant": + raise AirbyteTracedException( + internal_message=content.get("error_description"), + message="Refresh token is invalid or expired. Please re-authenticate to restore access to Airtable.", + failure_type=FailureType.config_error, + ) response.raise_for_status() - return response.json() + return content class AirtableAuth: diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/schema_helpers.py b/airbyte-integrations/connectors/source-airtable/source_airtable/schema_helpers.py index a075118b7930..5ec6f022df9f 100644 --- a/airbyte-integrations/connectors/source-airtable/source_airtable/schema_helpers.py +++ b/airbyte-integrations/connectors/source-airtable/source_airtable/schema_helpers.py @@ -63,7 +63,7 @@ class SchemaTypes: "singleSelect": SchemaTypes.string, "externalSyncSource": SchemaTypes.string, "url": SchemaTypes.string, - # referal default type + # referral default type "simpleText": SchemaTypes.string, } @@ -76,6 +76,8 @@ class SchemaTypes: "rollup": SchemaTypes.array_with_any, } +ARRAY_FORMULAS = ("ARRAYCOMPACT", "ARRAYFLATTEN", "ARRAYUNIQUE", "ARRAYSLICE") + class SchemaHelpers: @staticmethod @@ -87,6 +89,7 @@ def get_json_schema(table: Dict[str, Any]) -> Dict[str, str]: properties: Dict = { "_airtable_id": SchemaTypes.string, "_airtable_created_time": SchemaTypes.string, + "_airtable_table_name": SchemaTypes.string, } fields: Dict = table.get("fields", {}) @@ -106,7 +109,11 @@ def get_json_schema(table: Dict[str, Any]) -> Dict[str, str]: # Other edge cases, if `field_type` not in SIMPLE_AIRTABLE_TYPES, fall back to "simpleText" == `string` # reference issue: https://github.com/airbytehq/oncall/issues/1432#issuecomment-1412743120 if complex_type == SchemaTypes.array_with_any: - if field_type in SIMPLE_AIRTABLE_TYPES: + if original_type == "formula" and field_type in ("number", "currency", "percent", "duration"): + complex_type = SchemaTypes.number + elif original_type == "formula" and not any((options.get("formula").startswith(x) for x in ARRAY_FORMULAS)): + complex_type = SchemaTypes.string + elif field_type in SIMPLE_AIRTABLE_TYPES: complex_type["items"] = deepcopy(SIMPLE_AIRTABLE_TYPES.get(field_type)) else: complex_type["items"] = SchemaTypes.string diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/source.py b/airbyte-integrations/connectors/source-airtable/source_airtable/source.py index 9c249e9d5659..e06960d3823a 100644 --- a/airbyte-integrations/connectors/source-airtable/source_airtable/source.py +++ b/airbyte-integrations/connectors/source-airtable/source_airtable/source.py @@ -11,6 +11,7 @@ from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.utils.schema_helpers import split_config +from airbyte_protocol.models import SyncMode from .auth import AirtableAuth from .schema_helpers import SchemaHelpers @@ -27,11 +28,11 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> auth = AirtableAuth(config) try: # try reading first table from each base, to check the connectivity, - for base in AirtableBases(authenticator=auth).read_records(sync_mode=None): + for base in AirtableBases(authenticator=auth).read_records(sync_mode=SyncMode.full_refresh): base_id = base.get("id") base_name = base.get("name") self.logger.info(f"Reading first table info for base: {base_name}") - next(AirtableTables(base_id=base_id, authenticator=auth).read_records(sync_mode=None)) + next(AirtableTables(base_id=base_id, authenticator=auth).read_records(sync_mode=SyncMode.full_refresh)) return True, None except Exception as e: return False, str(e) @@ -46,7 +47,7 @@ def _remove_missed_streams_from_catalog( if not stream_instance: table_id = configured_stream.stream.name.split("/")[2] similar_streams = [s for s in stream_instances if s.endswith(table_id)] - logger.warn( + logger.warning( f"The requested stream {configured_stream.stream.name} was not found in the source. Please check if this stream was renamed or removed previously and reset data, removing from catalog for this sync run. For more information please refer to documentation: https://docs.airbyte.com/integrations/sources/airtable/#note-on-changed-table-names-and-deleted-tables" f" Similar streams: {similar_streams}" f" Available streams: {stream_instances.keys()}" @@ -74,11 +75,11 @@ def discover(self, logger: AirbyteLogger, config) -> AirbyteCatalog: """ auth = self._auth or AirtableAuth(config) # list all bases available for authenticated account - for base in AirtableBases(authenticator=auth).read_records(sync_mode=None): + for base in AirtableBases(authenticator=auth).read_records(sync_mode=SyncMode.full_refresh): base_id = base.get("id") base_name = SchemaHelpers.clean_name(base.get("name")) # list and process each table under each base to generate the JSON Schema - for table in list(AirtableTables(base_id, authenticator=auth).read_records(sync_mode=None)): + for table in AirtableTables(base_id, authenticator=auth).read_records(sync_mode=SyncMode.full_refresh): self.streams_catalog.append( { "stream_path": f"{base_id}/{table.get('id')}", @@ -86,20 +87,24 @@ def discover(self, logger: AirbyteLogger, config) -> AirbyteCatalog: f"{base_name}/{SchemaHelpers.clean_name(table.get('name'))}/{table.get('id')}", SchemaHelpers.get_json_schema(table), ), + "table_name": table.get("name"), } ) return AirbyteCatalog(streams=[stream["stream"] for stream in self.streams_catalog]) def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + The Discover method is triggered during each synchronization to fetch all available streams (tables). + If a stream becomes unavailable, an ERROR message will be printed in the logs. + """ self._auth = AirtableAuth(config) - # trigger discovery to populate the streams_catalog if not self.streams_catalog: self.discover(None, config) - # build the stream class from prepared streams_catalog for stream in self.streams_catalog: yield AirtableStream( stream_path=stream["stream_path"], stream_name=stream["stream"].name, stream_schema=stream["stream"].json_schema, + table_name=stream["table_name"], authenticator=self._auth, ) diff --git a/airbyte-integrations/connectors/source-airtable/source_airtable/streams.py b/airbyte-integrations/connectors/source-airtable/source_airtable/streams.py index 94c99c59edd8..2d3ac6b3b2b4 100644 --- a/airbyte-integrations/connectors/source-airtable/source_airtable/streams.py +++ b/airbyte-integrations/connectors/source-airtable/source_airtable/streams.py @@ -8,7 +8,10 @@ import requests from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +from airbyte_cdk.utils import AirbyteTracedException +from airbyte_protocol.models import FailureType from source_airtable.schema_helpers import SchemaHelpers URL_BASE: str = "https://api.airtable.com/v0/" @@ -30,6 +33,15 @@ def path(self, **kwargs) -> str: return "meta/bases" def should_retry(self, response: requests.Response) -> bool: + if ( + response.status_code == requests.codes.FORBIDDEN + and response.json().get("error", {}).get("type") == "INVALID_PERMISSIONS_OR_MODEL_NOT_FOUND" + ): + if isinstance(self._session.auth, TokenAuthenticator): + error_message = "Personal Access Token has not enough permissions, please add all required permissions to existed one or create new PAT, see docs for more info: https://docs.airbyte.com/integrations/sources/airtable#step-1-set-up-airtable" + else: + error_message = "Access Token has not enough permissions, please reauthenticate" + raise AirbyteTracedException(message=error_message, failure_type=FailureType.config_error) if response.status_code == 403 or response.status_code == 422: self.logger.error(f"Stream {self.name}: permission denied or entity is unprocessable. Skipping.") setattr(self, "raise_on_http_errors", False) @@ -71,7 +83,17 @@ def parse_response(self, response: requests.Response, **kwargs) -> Mapping[str, } """ records = response.json().get(self.name) - yield from records + for base in records: + if base.get("permissionLevel") == "none": + if isinstance(self._session.auth, TokenAuthenticator): + additional_message = "if you'd like to see tables from this base, add base to the Access list for Personal Access Token, see Airtable docs for more info: https://support.airtable.com/docs/creating-and-using-api-keys-and-access-tokens#understanding-personal-access-token-basic-actions" + else: + additional_message = "reauthenticate and add this base to the Access list, see Airtable docs for more info: https://support.airtable.com/docs/third-party-integrations-via-oauth-overview#granting-access-to-airtable-workspaces-bases" + self.logger.warning( + f"Skipping base `{base.get('name')}` with id `{base.get('id')}`: Not enough permissions, {additional_message}" + ) + else: + yield base class AirtableTables(AirtableBases): @@ -89,11 +111,12 @@ def path(self, **kwargs) -> str: class AirtableStream(HttpStream, ABC): - def __init__(self, stream_path: str, stream_name: str, stream_schema, **kwargs): + def __init__(self, stream_path: str, stream_name: str, stream_schema, table_name: str, **kwargs): super().__init__(**kwargs) self.stream_path = stream_path self.stream_name = stream_name self.stream_schema = stream_schema + self.table_name = table_name url_base = URL_BASE primary_key = "id" @@ -146,6 +169,7 @@ def process_records(self, records) -> Iterable[Mapping[str, Any]]: yield { "_airtable_id": record.get("id"), "_airtable_created_time": record.get("createdTime"), + "_airtable_table_name": self.table_name, **{SchemaHelpers.clean_name(k): v for k, v in data.items()}, } diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/conftest.py b/airbyte-integrations/connectors/source-airtable/unit_tests/conftest.py index 20278981cd87..f2a42ae96efb 100644 --- a/airbyte-integrations/connectors/source-airtable/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/conftest.py @@ -65,11 +65,11 @@ def json_response(): "id": "abc", "fields": [ { - 'type': 'singleLineText', - 'id': '_fake_id_', - 'name': 'test', + "type": "singleLineText", + "id": "_fake_id_", + "name": "test", } - ] + ], } ] } @@ -80,23 +80,24 @@ def streams_json_response(): return { "records": [ { - 'id': 'some_id', - 'createdTime': '2022-12-02T19:50:00.000Z', - 'fields': {'field1': True, 'field2': "test", 'field3': 123}, + "id": "some_id", + "createdTime": "2022-12-02T19:50:00.000Z", + "fields": {"field1": True, "field2": "test", "field3": 123}, } ] } @pytest.fixture -def streams_processed_response(): +def streams_processed_response(table): return [ { - '_airtable_id': 'some_id', - '_airtable_created_time': '2022-12-02T19:50:00.000Z', - 'field1': True, - 'field2': 'test', - 'field3': 123, + "_airtable_id": "some_id", + "_airtable_created_time": "2022-12-02T19:50:00.000Z", + "_airtable_table_name": table, + "field1": True, + "field2": "test", + "field3": 123, } ] @@ -109,14 +110,15 @@ def expected_json_schema(): "properties": { "_airtable_created_time": {"type": ["null", "string"]}, "_airtable_id": {"type": ["null", "string"]}, + "_airtable_table_name": {"type": ["null", "string"]}, "test": {"type": ["null", "string"]}, }, "type": "object", } -@pytest.fixture(scope='function', autouse=True) -def prepared_stream(): +@pytest.fixture(scope="function", autouse=True) +def prepared_stream(table): return { "stream_path": "some_base_id/some_table_id", "stream": AirbyteStream( @@ -126,29 +128,16 @@ def prepared_stream(): "type": "object", "additionalProperties": True, "properties": { - "_airtable_id": { - "type": [ - "null", - "string" - ] - }, - "_airtable_created_time": { - "type": [ - "null", - "string" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - } - } + "_airtable_id": {"type": ["null", "string"]}, + "_airtable_created_time": {"type": ["null", "string"]}, + "_airtable_table_name": {"type": ["null", "string"]}, + "name": {"type": ["null", "string"]}, + }, }, supported_sync_modes=[SyncMode.full_refresh], supported_destination_sync_modes=[DestinationSyncMode.overwrite, DestinationSyncMode.append_dedup], - ) + ), + "table_name": table, } @@ -159,8 +148,10 @@ def make(name): stream_path=prepared_stream["stream_path"], stream_name=name, stream_schema=prepared_stream["stream"].json_schema, - authenticator=fake_auth + table_name=prepared_stream["table_name"], + authenticator=fake_auth, ) + return make @@ -176,8 +167,9 @@ def make(name): supported_destination_sync_modes=[DestinationSyncMode.overwrite, DestinationSyncMode.append_dedup], ), "sync_mode": SyncMode.full_refresh, - "destination_sync_mode": DestinationSyncMode.overwrite + "destination_sync_mode": DestinationSyncMode.overwrite, } + return make diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/expected_schema_for_sample_table.json b/airbyte-integrations/connectors/source-airtable/unit_tests/expected_schema_for_sample_table.json new file mode 100644 index 000000000000..209748dda06a --- /dev/null +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/expected_schema_for_sample_table.json @@ -0,0 +1,55 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "_airtable_created_time": { + "type": ["null", "string"] + }, + "_airtable_id": { + "type": ["null", "string"] + }, + "_airtable_table_name": { + "type": ["null", "string"] + }, + "assignee_(from_table_6)": { + "items": { + "type": ["null", "number"] + }, + "type": ["null", "array"] + }, + "barcode": { + "type": ["null", "string"] + }, + "float": { + "type": ["null", "number"] + }, + "formula_1": { + "type": ["null", "number"] + }, + "formula_2_(array)": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "formula_3_simple_text": { + "type": ["null", "string"] + }, + "integer": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "table_6": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + } + }, + "type": "object" +} diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/sample_table_with_formulas.json b/airbyte-integrations/connectors/source-airtable/unit_tests/sample_table_with_formulas.json new file mode 100644 index 000000000000..541684ae6911 --- /dev/null +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/sample_table_with_formulas.json @@ -0,0 +1,139 @@ +{ + "id": "tblSXpxKHg0OiLxbI", + "name": "Table 1", + "primaryFieldId": "some_id", + "fields": [ + { + "type": "singleLineText", + "id": "some_id", + "name": "Name" + }, + { + "type": "formula", + "options": { + "isValid": true, + "formula": "1+1", + "referencedFieldIds": [], + "result": { + "type": "number", + "options": { + "precision": 0 + } + } + }, + "id": "fldG2UVGl3hEglGJq", + "name": "Formula 1" + }, + { + "type": "formula", + "options": { + "isValid": true, + "formula": "ARRAYFLATTEN(1,2,3)", + "referencedFieldIds": [], + "result": { + "type": "string", + "options": { + "precision": 0 + } + } + }, + "id": "fldG2UVGl3hEg123Jq", + "name": "Formula 2 (array)" + }, + + { + "type": "formula", + "options": { + "isValid": true, + "formula": "CONCAT(1,2,3)", + "referencedFieldIds": [], + "result": { + "type": "string", + "options": { + "precision": 0 + } + } + }, + "id": "fldG2UVGl3hEg123Jq", + "name": "Formula 3 simple text" + }, + { + "type": "singleSelect", + "options": { + "choices": [ + { + "id": "seleFjJiXuyLDHNUM", + "name": "Todo", + "color": "redLight2" + }, + { + "id": "selskIXEljPBrKHLz", + "name": "In progress", + "color": "yellowLight2" + }, + { + "id": "selg4w4LeypED2gpW", + "name": "Done", + "color": "greenLight2" + } + ] + }, + "id": "fldpwaVKzdfcHe2YV", + "name": "Status" + }, + { + "type": "number", + "options": { + "precision": 1 + }, + "id": "fldZNvmdvMZymPxUc", + "name": "Float" + }, + { + "type": "number", + "options": { + "precision": 0 + }, + "id": "fldsjTjrIkKTv6KjG", + "name": "Integer" + }, + { + "type": "barcode", + "id": "fld899obV6ycadgWS", + "name": "Barcode" + }, + { + "type": "multipleRecordLinks", + "options": { + "linkedTableId": "tblSXpxKHg0OiLxbI", + "isReversed": false, + "prefersSingleRecordLink": false + }, + "id": "fldMkP8CfDgqc5r3j", + "name": "Table 6" + }, + { + "type": "multipleLookupValues", + "options": { + "isValid": true, + "recordLinkFieldId": "fldMkP8CfDgqc5r3j", + "fieldIdInLinkedTable": "fldG2UVGl3hEglGJq", + "result": { + "type": "number", + "options": { + "precision": 0 + } + } + }, + "id": "fldLQOO0P7mB4lm7x", + "name": "Assignee (from Table 6)" + } + ], + "views": [ + { + "id": "viwadxF7ds4lOBf3R", + "name": "Grid view", + "type": "grid" + } + ] +} diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/test_authenticator.py b/airbyte-integrations/connectors/source-airtable/unit_tests/test_authenticator.py new file mode 100644 index 000000000000..fbc0c6e9e49f --- /dev/null +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/test_authenticator.py @@ -0,0 +1,45 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +from airbyte_cdk.utils import AirbyteTracedException +from source_airtable.auth import AirtableAuth, AirtableOAuth + +CONFIG_OAUTH = {"credentials": {"auth_method": "oauth2.0", "client_id": "sample_client_id", "client_secret": "sample_client_secret"}} + + +@pytest.mark.parametrize( + "config, expected_auth_class", + [ + ({"api_key": "sample_api_key"}, TokenAuthenticator), + (CONFIG_OAUTH, AirtableOAuth), + ({"credentials": {"auth_method": "api_key", "api_key": "sample_api_key"}}, TokenAuthenticator), + ], + ids=["old_config_api_key", "oauth2.0", "api_key"], +) +def test_airtable_auth(config, expected_auth_class): + auth_instance = AirtableAuth(config) + assert isinstance(auth_instance, expected_auth_class) + + +def test_airtable_oauth(): + auth_instance = AirtableAuth(CONFIG_OAUTH) + assert isinstance(auth_instance, AirtableOAuth) + assert auth_instance.build_refresh_request_headers() == { + "Authorization": "Basic c2FtcGxlX2NsaWVudF9pZDpzYW1wbGVfY2xpZW50X3NlY3JldA==", + "Content-Type": "application/x-www-form-urlencoded", + } + assert auth_instance.build_refresh_request_body() == {"grant_type": "refresh_token", "refresh_token": ""} + + +def test_airtable_oauth_token_refresh_exception(requests_mock): + auth_instance = AirtableAuth(CONFIG_OAUTH) + requests_mock.post( + "https://airtable.com/oauth2/v1/token", status_code=400, json={"error": "invalid_grant", "error_description": "grant invalid"} + ) + with pytest.raises(AirbyteTracedException) as e: + auth_instance._get_refresh_access_token_response() + assert e.value.message == "Refresh token is invalid or expired. Please re-authenticate to restore access to Airtable." diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/test_helpers.py b/airbyte-integrations/connectors/source-airtable/unit_tests/test_helpers.py index 4cc7097e691d..5942592cffdb 100644 --- a/airbyte-integrations/connectors/source-airtable/unit_tests/test_helpers.py +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/test_helpers.py @@ -2,10 +2,19 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import json +import os +from typing import Any, Mapping from source_airtable.schema_helpers import SchemaHelpers +# HELPERS +def load_file(file_name: str) -> Mapping[str, Any]: + with open(f"{os.path.dirname(__file__)}/{file_name}", "r") as data: + return json.load(data) + + def test_clean_name(field_name_to_cleaned, expected_clean_name): assert expected_clean_name == SchemaHelpers.clean_name(field_name_to_cleaned) @@ -20,3 +29,12 @@ def test_get_airbyte_stream(table, expected_json_schema): assert stream assert stream.name == table assert stream.json_schema == expected_json_schema + + +def test_table_with_formulas(): + table = load_file("sample_table_with_formulas.json") + + stream_schema = SchemaHelpers.get_json_schema(table) + + expected_schema = load_file("expected_schema_for_sample_table.json") + assert stream_schema == expected_schema diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/test_source.py b/airbyte-integrations/connectors/source-airtable/unit_tests/test_source.py index 0fd7fc00c1ad..e4b6a6909de6 100644 --- a/airbyte-integrations/connectors/source-airtable/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/test_source.py @@ -7,24 +7,17 @@ from unittest.mock import MagicMock import pytest -from airbyte_cdk.models import AirbyteCatalog, ConnectorSpecification +from airbyte_cdk.models import AirbyteCatalog from source_airtable.source import SourceAirtable -def test_spec(config): - source = SourceAirtable() - logger_mock = MagicMock() - spec = source.spec(logger_mock) - assert isinstance(spec, ConnectorSpecification) - - @pytest.mark.parametrize( "status, check_passed", [ (200, (True, None)), - (401, (False, '401 Client Error: None for url: https://api.airtable.com/v0/meta/bases')), + (401, (False, "401 Client Error: None for url: https://api.airtable.com/v0/meta/bases")), ], - ids=["success", "fail"] + ids=["success", "fail"], ) def test_check_connection(config, status, check_passed, fake_bases_response, fake_tables_response, requests_mock): source = SourceAirtable() @@ -70,4 +63,6 @@ def test_remove_missed_streams_from_catalog(mocker, config, fake_catalog, fake_s assert streams_before - len(catalog.streams) == 1 assert len(caplog.messages) == 1 assert caplog.text.startswith("WARNING") + + # diff --git a/airbyte-integrations/connectors/source-airtable/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-airtable/unit_tests/test_streams.py index a276194a1ae6..c1d291030af3 100644 --- a/airbyte-integrations/connectors/source-airtable/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-airtable/unit_tests/test_streams.py @@ -88,12 +88,12 @@ def test_stream_name(self): class TestAirtableStream: - def stream_instance(self, prepared_stream): return AirtableStream( stream_path=prepared_stream["stream_path"], stream_name=prepared_stream["stream"].name, stream_schema=prepared_stream["stream"].json_schema, + table_name=prepared_stream["table_name"], authenticator=MagicMock(), ) @@ -104,7 +104,7 @@ def test_streams_primary_key(self, prepared_stream): assert self.stream_instance(prepared_stream).primary_key == "id" def test_streams_name(self, prepared_stream): - assert self.stream_instance(prepared_stream).name == 'test_base/test_table' + assert self.stream_instance(prepared_stream).name == "test_base/test_table" def test_streams_path(self, prepared_stream): assert self.stream_instance(prepared_stream).path() == "some_base_id/some_table_id" @@ -137,7 +137,7 @@ def test_streams_backoff_time(self, http_status, expected_backoff_time, prepared assert self.stream_instance(prepared_stream).backoff_time(response) == expected_backoff_time def test_streams_get_json_schema(self, prepared_stream): - assert self.stream_instance(prepared_stream).get_json_schema() == prepared_stream['stream'].json_schema + assert self.stream_instance(prepared_stream).get_json_schema() == prepared_stream["stream"].json_schema def test_streams_next_page(self, prepared_stream, requests_mock): url = "https://api.airtable.com/v0/meta/bases/" @@ -161,5 +161,3 @@ def test_streams_parse_response(self, prepared_stream, streams_json_response, st requests_mock.get(url, status_code=200, json=streams_json_response) response = requests.get(url) assert list(stream.parse_response(response)) == streams_processed_response - -# diff --git a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/.dockerignore b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile deleted file mode 100644 index 7af590ca55e7..000000000000 --- a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-alloydb-strict-encrypt - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-alloydb-strict-encrypt - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=3.1.5 -LABEL io.airbyte.name=airbyte/source-alloydb-strict-encrypt diff --git a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/acceptance-test-config.yml b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/acceptance-test-config.yml deleted file mode 100644 index f497d002a406..000000000000 --- a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/acceptance-test-config.yml +++ /dev/null @@ -1,6 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-postgres-strict-encrypt:dev -tests: - spec: - - spec_path: "src/test/resources/expected_spec.json" diff --git a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/build.gradle deleted file mode 100644 index 231bd7395f77..000000000000 --- a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/build.gradle +++ /dev/null @@ -1,32 +0,0 @@ -plugins { - id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' -} - -application { - mainClass = 'io.airbyte.integrations.source.alloydb.AlloyDbStrictEncryptSource' - applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] -} - -dependencies { - implementation project(':airbyte-db:db-lib') - - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:connectors:source-postgres') - implementation project(':airbyte-integrations:connectors:source-postgres-strict-encrypt') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation project(':airbyte-integrations:connectors:source-relational-db') - - - testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) - testImplementation project(':airbyte-test-utils') - - testImplementation libs.connectors.testcontainers.postgresql - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) -} diff --git a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/icon.svg b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/icon.svg deleted file mode 100644 index 03c290b6d3b6..000000000000 --- a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/icon.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/metadata.yaml deleted file mode 100644 index d3898532abe2..000000000000 --- a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/metadata.yaml +++ /dev/null @@ -1,24 +0,0 @@ -data: - allowedHosts: - hosts: - - ${host} - - ${tunnel_method.tunnel_host} - registries: - cloud: - enabled: false # strict encrypt connectors are deployed to Cloud by their non strict encrypt sibling. - oss: - enabled: false # strict encrypt connectors are not used on OSS. - connectorSubtype: database - connectorType: source - definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 - dockerImageTag: 3.1.5 - dockerRepository: airbyte/source-alloydb-strict-encrypt - githubIssueLabel: source-alloydb - icon: alloydb.svg - license: MIT - name: AlloyDB for PostgreSQL - releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb - tags: - - language:java -metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/src/main/java/io/airbyte/integrations/source/alloydb/AlloyDbStrictEncryptSource.java b/airbyte-integrations/connectors/source-alloydb-strict-encrypt/src/main/java/io/airbyte/integrations/source/alloydb/AlloyDbStrictEncryptSource.java deleted file mode 100644 index 72018a591097..000000000000 --- a/airbyte-integrations/connectors/source-alloydb-strict-encrypt/src/main/java/io/airbyte/integrations/source/alloydb/AlloyDbStrictEncryptSource.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.alloydb; - -import static io.airbyte.integrations.source.relationaldb.state.StateManager.LOGGER; - -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.source.postgres_strict_encrypt.PostgresSourceStrictEncrypt; - -public class AlloyDbStrictEncryptSource { - - public static void main(String[] args) throws Exception { - final Source source = new PostgresSourceStrictEncrypt(); - LOGGER.info("starting source: {}", PostgresSourceStrictEncrypt.class); - new IntegrationRunner(source).run(args); - LOGGER.info("completed source: {}", PostgresSourceStrictEncrypt.class); - } - -} diff --git a/airbyte-integrations/connectors/source-alloydb/.dockerignore b/airbyte-integrations/connectors/source-alloydb/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-alloydb/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-alloydb/Dockerfile b/airbyte-integrations/connectors/source-alloydb/Dockerfile deleted file mode 100644 index cf8e202ab39d..000000000000 --- a/airbyte-integrations/connectors/source-alloydb/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-alloydb - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-alloydb - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=3.1.5 -LABEL io.airbyte.name=airbyte/source-alloydb diff --git a/airbyte-integrations/connectors/source-alloydb/acceptance-test-config.yml b/airbyte-integrations/connectors/source-alloydb/acceptance-test-config.yml deleted file mode 100644 index 2838a5ca4273..000000000000 --- a/airbyte-integrations/connectors/source-alloydb/acceptance-test-config.yml +++ /dev/null @@ -1,6 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-alloydb:dev -tests: - spec: - - spec_path: "src/test/resources/expected_spec.json" diff --git a/airbyte-integrations/connectors/source-alloydb/build.gradle b/airbyte-integrations/connectors/source-alloydb/build.gradle deleted file mode 100644 index a92d92a5e8b9..000000000000 --- a/airbyte-integrations/connectors/source-alloydb/build.gradle +++ /dev/null @@ -1,33 +0,0 @@ -plugins { - id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' -} - -application { - mainClass = 'io.airbyte.integrations.source.alloydb.AlloyDbSource' - applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] -} - -dependencies { - implementation project(':airbyte-db:db-lib') - - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:connectors:source-postgres') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation project(':airbyte-integrations:connectors:source-relational-db') - - - testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) - testImplementation project(':airbyte-test-utils') - - testImplementation libs.connectors.testcontainers.postgresql - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - -} diff --git a/airbyte-integrations/connectors/source-alloydb/icon.svg b/airbyte-integrations/connectors/source-alloydb/icon.svg deleted file mode 100644 index 03c290b6d3b6..000000000000 --- a/airbyte-integrations/connectors/source-alloydb/icon.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-alloydb/metadata.yaml b/airbyte-integrations/connectors/source-alloydb/metadata.yaml deleted file mode 100644 index aed3d7766c62..000000000000 --- a/airbyte-integrations/connectors/source-alloydb/metadata.yaml +++ /dev/null @@ -1,29 +0,0 @@ -data: - allowedHosts: - hosts: - - ${host} - - ${tunnel_method.tunnel_host} - connectorSubtype: database - connectorType: source - definitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 - dockerImageTag: 3.1.5 - dockerRepository: airbyte/source-alloydb - githubIssueLabel: source-alloydb - icon: alloydb.svg - license: MIT - name: AlloyDB for PostgreSQL - registries: - cloud: - dockerRepository: airbyte/source-alloydb-strict-encrypt - enabled: true - oss: - enabled: true - releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/alloydb - tags: - - language:java - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified -metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-alloydb/src/main/java/io/airbyte/integrations/source/alloydb/AlloyDbSource.java b/airbyte-integrations/connectors/source-alloydb/src/main/java/io/airbyte/integrations/source/alloydb/AlloyDbSource.java deleted file mode 100644 index a03d265f0a82..000000000000 --- a/airbyte-integrations/connectors/source-alloydb/src/main/java/io/airbyte/integrations/source/alloydb/AlloyDbSource.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.alloydb; - -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.source.postgres.PostgresSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class AlloyDbSource { - - private static final Logger LOGGER = LoggerFactory.getLogger(AlloyDbSource.class); - - /** - * AlloyDB for PostgreSQL is a fully managed PostgreSQL-compatible database service. So the - * source-postgres connector is used under the hood. For more details please check the - * https://cloud.google.com/alloydb - */ - public static void main(final String[] args) throws Exception { - final Source source = PostgresSource.sshWrappedSource(); - LOGGER.info("starting source: AlloyDB for {}", PostgresSource.class); - new IntegrationRunner(source).run(args); - LOGGER.info("completed source: AlloyDB for {}", PostgresSource.class); - } - -} diff --git a/airbyte-integrations/connectors/source-alpha-vantage/README.md b/airbyte-integrations/connectors/source-alpha-vantage/README.md index 5d261772493e..6fd81b7208fa 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/README.md +++ b/airbyte-integrations/connectors/source-alpha-vantage/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-alpha-vantage:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/alpha-vantage) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_alpha_vantage/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-alpha-vantage:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-alpha-vantage build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-alpha-vantage:airbyteDocker +An image will be built with the tag `airbyte/source-alpha-vantage:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-alpha-vantage:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-alpha-vantage:dev chec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-alpha-vantage:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-alpha-vantage:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-alpha-vantage test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-alpha-vantage:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-alpha-vantage:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-alpha-vantage test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/alpha-vantage.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-alpha-vantage/acceptance-test-config.yml b/airbyte-integrations/connectors/source-alpha-vantage/acceptance-test-config.yml index b5ae66d30609..2e220351254f 100644 --- a/airbyte-integrations/connectors/source-alpha-vantage/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-alpha-vantage/acceptance-test-config.yml @@ -14,7 +14,7 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: + empty_streams: - "time_series_daily" - "time_series_monthly" - "quote" diff --git a/airbyte-integrations/connectors/source-alpha-vantage/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-alpha-vantage/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-alpha-vantage/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-alpha-vantage/build.gradle b/airbyte-integrations/connectors/source-alpha-vantage/build.gradle deleted file mode 100644 index c2c09821ebf8..000000000000 --- a/airbyte-integrations/connectors/source-alpha-vantage/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_alpha_vantage' -} diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile deleted file mode 100644 index 3ffd3401acb8..000000000000 --- a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY source_amazon_ads ./source_amazon_ads -COPY main.py ./ -COPY setup.py ./ -RUN pip install . - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - - -LABEL io.airbyte.version=3.1.0 -LABEL io.airbyte.name=airbyte/source-amazon-ads diff --git a/airbyte-integrations/connectors/source-amazon-ads/README.md b/airbyte-integrations/connectors/source-amazon-ads/README.md index acd73da4c15c..db784a8d9651 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/README.md +++ b/airbyte-integrations/connectors/source-amazon-ads/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-amazon-ads:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/amazon-ads) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_amazon_ads/spec.json` file. @@ -56,19 +48,70 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-amazon-ads:dev + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-amazon-ads build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-amazon-ads:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. -You can also build the connector image via Gradle: +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-amazon-ads:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-amazon-ads:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-amazon-ads:dev . +# Running the spec command against your patched connector +docker run airbyte/source-amazon-ads:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -77,44 +120,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-amazon-ads:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-amazon-ads:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-amazon-ads:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .'[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-amazon-ads test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-amazon-ads:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-amazon-ads:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -124,8 +139,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-amazon-ads test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/amazon-ads.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml index d0be492d3369..e3eb72c4b986 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml @@ -27,6 +27,8 @@ acceptance_tests: bypass_reason: "can't populate stream because it requires real ad campaign" - name: sponsored_products_report_stream bypass_reason: "can't populate stream because it requires real ad campaign" + - name: sponsored_display_creatives + bypass_reason: "can't populate stream because it requires real ad campaign" ignored_fields: sponsored_product_campaigns: - name: dailyBudget @@ -50,7 +52,7 @@ acceptance_tests: tests: - config_path: secrets/config.json backward_compatibility_tests_config: - disable_for_version: 2.3.1 + disable_for_version: 3.4.3 full_refresh: tests: - config_path: secrets/config.json @@ -61,5 +63,7 @@ acceptance_tests: spec: tests: - spec_path: integration_tests/spec.json + backward_compatibility_tests_config: + disable_for_version: 3.4.3 connector_image: airbyte/source-amazon-ads:dev test_strictness_level: high diff --git a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-amazon-ads/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-amazon-ads/build.gradle b/airbyte-integrations/connectors/source-amazon-ads/build.gradle deleted file mode 100644 index bc26868e475a..000000000000 --- a/airbyte-integrations/connectors/source-amazon-ads/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_amazon_ads' -} diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records.jsonl index 2c0ce04ab232..bf609dee5420 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/expected_records.jsonl @@ -47,7 +47,7 @@ {"stream":"sponsored_display_product_ads","data":{"adId":103527738992867,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNT386","state":"enabled"},"emitted_at":1659020219614} {"stream":"sponsored_display_product_ads","data":{"adId":195948665185008,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B000BNQBOA","state":"enabled"},"emitted_at":1659020219614} {"stream":"sponsored_display_product_ads","data":{"adId":130802512011075,"adGroupId":239470166910761,"campaignId":25934734632378,"asin":"B091G1HT4P","state":"enabled"},"emitted_at":1659020219614} -{"stream":"sponsored_display_targetings","data":{"adGroupId":239470166910761,"bid":0.4,"expression":[{"type":"similarProduct"}],"expressionType":"auto","resolvedExpression":[{"type":"similarProduct"}],"state":"enabled","targetId":124150067548052},"emitted_at":1659020220625} +{"stream":"sponsored_display_targetings","data":{"adGroupId":239470166910761,"bid":0.4,"expression":[{"type":"similarProduct"}],"expressionType":"auto","resolvedExpression":[{"type":"similarProduct"}],"state":"enabled","targetId":124150067548052,"campaignId": 25934734632378},"emitted_at":1659020220625} {"stream":"sponsored_product_campaigns","data":{"campaignId":39413387973397,"name":"Test campaging for profileId 1861552880916640","campaignType":"sponsoredProducts","targetingType":"manual","premiumBidAdjustment":true,"dailyBudget":10,"ruleBasedBudget":{"isProcessing":false},"startDate":"20220705","endDate":"20220712","state":"paused","bidding":{"strategy":"legacyForSales","adjustments":[{"predicate":"placementTop","percentage":50}]},"tags":{"PONumber":"examplePONumber","accountManager":"exampleAccountManager"}},"emitted_at":1687524797996} {"stream":"sponsored_product_campaigns","data":{"campaignId":135264288913079,"name":"Campaign - 7/5/2022 18:14:02","campaignType":"sponsoredProducts","targetingType":"auto","premiumBidAdjustment":false,"dailyBudget":10,"startDate":"20220705","state":"enabled","bidding":{"strategy":"legacyForSales","adjustments":[]},"portfolioId":270076898441727},"emitted_at":1687524798170} {"stream":"sponsored_product_campaigns","data":{"campaignId":191249325250025,"name":"Campaign - 7/8/2022 13:57:48","campaignType":"sponsoredProducts","targetingType":"auto","premiumBidAdjustment":true,"dailyBudget":50,"ruleBasedBudget":{"isProcessing":false},"startDate":"20220708","state":"enabled","bidding":{"strategy":"legacyForSales","adjustments":[{"predicate":"placementProductPage","percentage":100},{"predicate":"placementTop","percentage":100}]},"portfolioId":253945852845204},"emitted_at":1687524798171} diff --git a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json index 0e6137ff459f..96579cbe995e 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json @@ -13,6 +13,7 @@ "client_id": { "title": "Client ID", "description": "The client ID of your Amazon Ads developer application. See the docs for more information.", + "airbyte_secret": true, "order": 1, "type": "string" }, @@ -43,17 +44,28 @@ "description": "The Start date for collecting reports, should not be more than 60 days in the past. In YYYY-MM-DD format", "examples": ["2022-10-10", "2022-10-22"], "order": 5, - "type": "string" + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "format": "date" }, "profiles": { "title": "Profile IDs", - "description": "Profile IDs you want to fetch data for. See docs for more details.", + "description": "Profile IDs you want to fetch data for. See docs for more details. Note: If Marketplace IDs are also selected, profiles will be selected if they match the Profile ID OR the Marketplace ID.", "order": 6, "type": "array", "items": { "type": "integer" } }, + "marketplace_ids": { + "title": "Marketplace IDs", + "description": "Marketplace IDs you want to fetch data for. Note: If Profile IDs are also selected, profiles will be selected if they match the Profile ID OR the Marketplace ID.", + "order": 7, + "type": "array", + "items": { + "type": "string" + } + }, "state_filter": { "title": "State Filter", "description": "Reflects the state of the Display, Product, and Brand Campaign streams as enabled, paused, or archived. If you do not populate this field, it will be ignored completely.", @@ -63,14 +75,14 @@ }, "type": "array", "uniqueItems": true, - "order": 7 + "order": 8 }, "look_back_window": { "title": "Look Back Window", "description": "The amount of days to go back in time to get the updated data from Amazon Ads", "default": 3, "examples": [3, 10], - "order": 8, + "order": 9, "type": "integer" }, "report_record_types": { @@ -91,7 +103,7 @@ }, "type": "array", "uniqueItems": true, - "order": 9 + "order": 10 } }, "required": ["client_id", "client_secret", "refresh_token"], @@ -102,6 +114,16 @@ "predicate_key": ["auth_type"], "predicate_value": "oauth2.0", "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "region": { + "type": "string", + "path_in_connector_config": ["region"] + } + } + }, "complete_oauth_output_specification": { "type": "object", "additionalProperties": true, diff --git a/airbyte-integrations/connectors/source-amazon-ads/main.py b/airbyte-integrations/connectors/source-amazon-ads/main.py index fa309cc66e04..1c292d29e4ca 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/main.py +++ b/airbyte-integrations/connectors/source-amazon-ads/main.py @@ -7,7 +7,9 @@ from airbyte_cdk.entrypoint import launch from source_amazon_ads import SourceAmazonAds +from source_amazon_ads.config_migrations import MigrateStartDate if __name__ == "__main__": source = SourceAmazonAds() + MigrateStartDate.migrate(sys.argv[1:], source) launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml b/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml index 9881c233f8fc..0efe4b2b434d 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-ads/metadata.yaml @@ -1,15 +1,21 @@ data: + ab_internal: + ql: 400 + sl: 300 allowedHosts: hosts: - api.amazon.com - advertising-api.amazon.com - advertising-api-eu.amazon.com - advertising-api-fe.amazon.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246 - dockerImageTag: 3.1.0 + dockerImageTag: 4.0.1 dockerRepository: airbyte/source-amazon-ads + documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-ads githubIssueLabel: source-amazon-ads icon: amazonads.svg license: MIT @@ -20,16 +26,26 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-ads - tags: - - language:python releases: breakingChanges: + 4.0.0: + message: "Streams `SponsoredBrandsAdGroups` and `SponsoredBrandsKeywords` now have updated schemas." + upgradeDeadline: "2024-01-17" + scopedImpact: + - scopeType: stream + impactedScopes: + ["sponsored_brands_ad_groups", "sponsored_brands_keywords"] 3.0.0: - message: "Attribution report stream schemas fix." + message: Attribution report stream schemas fix. upgradeDeadline: "2023-07-24" - ab_internal: - sl: 300 - ql: 400 + suggestedStreams: + streams: + - profiles + - sponsored_brands_video_report_stream + - sponsored_display_report_stream + - sponsored_brands_report_stream + - sponsored_products_report_stream supportLevel: certified + tags: + - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-amazon-ads/setup.py b/airbyte-integrations/connectors/source-amazon-ads/setup.py index 13b317863d0e..7d612fffeaaa 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/setup.py +++ b/airbyte-integrations/connectors/source-amazon-ads/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.16", "requests_oauthlib~=1.3.1", "pendulum~=2.1.2"] +MAIN_REQUIREMENTS = ["airbyte-cdk", "requests_oauthlib~=1.3.1", "pendulum~=2.1.2"] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/config_migrations.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/config_migrations.py new file mode 100644 index 000000000000..dda920775164 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/config_migrations.py @@ -0,0 +1,104 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import logging +from typing import Any, List, Mapping + +from airbyte_cdk.config_observation import create_connector_config_control_message +from airbyte_cdk.entrypoint import AirbyteEntrypoint +from airbyte_cdk.sources import Source +from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + +logger = logging.getLogger("airbyte_logger") + + +class MigrateStartDate: + """ + This class stands for migrating the config at runtime, + while providing the backward compatibility when falling back to the previous source version. + + Delete start_date field if it set to None or an empty string (""). + """ + + message_repository: MessageRepository = InMemoryMessageRepository() + # Key used to identify start date in the configuration. + key: str = "start_date" + + @classmethod + def should_migrate(cls, config: Mapping[str, Any]) -> bool: + """ + Determines if a configuration requires migration. + + Args: + - config (Mapping[str, Any]): The configuration data to check. + + Returns: + - True: If the configuration requires migration. + - False: Otherwise. + """ + return not config.get(cls.key, "skip_if_start_date_in_config") + + @classmethod + def delete_from_config(cls, config: Mapping[str, Any], source: Source = None) -> Mapping[str, Any]: + """ + Removes the specified key from the configuration. + + Args: + - config (Mapping[str, Any]): The configuration from which the key should be removed. + - source (Source, optional): The data source. Defaults to None. + + Returns: + - Mapping[str, Any]: The configuration after removing the key. + """ + config.pop(cls.key, None) # Safely remove the key if it exists. + return config + + @classmethod + def modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Modifies the configuration and then saves it back to the source. + + Args: + - config_path (str): The path where the configuration is stored. + - source (Source): The data source. + - config (Mapping[str, Any]): The current configuration. + + Returns: + - Mapping[str, Any]: The updated configuration. + """ + migrated_config = cls.delete_from_config(config, source) + source.write_config(migrated_config, config_path) + return migrated_config + + @classmethod + def emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: + """ + Emits the control messages related to configuration migration. + + Args: + - migrated_config (Mapping[str, Any]): The migrated configuration. + """ + cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) + for message in cls.message_repository._message_queue: + print(message.json(exclude_unset=True)) + + @classmethod + def migrate(cls, args: List[str], source: Source) -> None: + """ + Orchestrates the configuration migration process. + + It first checks if the `--config` argument is provided, and if so, + determines whether migration is needed, and then performs the migration + if required. + + Args: + - args (List[str]): List of command-line arguments. + - source (Source): The data source. + """ + config_path = AirbyteEntrypoint(source).extract_config(args) + if config_path: + config = source.read_config(config_path) + if cls.should_migrate(config): + cls.emit_control_message(cls.modify_and_save(config_path, source, config)) diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/__init__.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/__init__.py index 8c3f0963a5ae..e9c8aae725fc 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/__init__.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/__init__.py @@ -5,7 +5,7 @@ from .common import CatalogModel, Keywords, MetricsReport, NegativeKeywords, Portfolio from .profile import Profile from .sponsored_brands import BrandsAdGroup, BrandsCampaign -from .sponsored_display import DisplayAdGroup, DisplayBudgetRules, DisplayCampaign, DisplayProductAds, DisplayTargeting +from .sponsored_display import DisplayAdGroup, DisplayBudgetRules, DisplayCampaign, DisplayCreatives, DisplayProductAds, DisplayTargeting from .sponsored_products import ( ProductAd, ProductAdGroupBidRecommendations, @@ -25,6 +25,7 @@ "DisplayTargeting", "DisplayBudgetRules", "Keywords", + "DisplayCreatives", "MetricsReport", "NegativeKeywords", "Portfolio", diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/attribution_report.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/attribution_report.py index 6aea054ea703..b7b3e89c5a79 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/attribution_report.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/attribution_report.py @@ -2,6 +2,8 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from typing import Optional + from .common import CatalogModel @@ -9,14 +11,14 @@ class AttributionReportModel(CatalogModel): date: str brandName: str marketplace: str - campaignId: str + campaignId: Optional[str] productAsin: str productConversionType: str advertiserName: str - adGroupId: str - creativeId: str + adGroupId: Optional[str] + creativeId: Optional[str] productName: str productCategory: str productSubcategory: str productGroup: str - publisher: str + publisher: Optional[str] diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_brands.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_brands.py index 510d2492d7b2..e7e2fa7cd07c 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_brands.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_brands.py @@ -3,11 +3,29 @@ # from decimal import Decimal -from typing import Dict +from typing import Dict, List, Optional from .common import CatalogModel +class LandingPage(CatalogModel): + pageType: str + url: str + + +class BidAdjustment(CatalogModel): + bidAdjustmentPredicate: str + bidAdjustmentPercent: int + + +class Creative(CatalogModel): + brandName: str + brandLogoAssetID: str + brandLogoUrl: str + asins: List[str] + shouldOptimizeAsins: bool + + class BrandsCampaign(CatalogModel): campaignId: Decimal name: str @@ -23,9 +41,19 @@ class BrandsCampaign(CatalogModel): bidOptimization: bool = None bidMultiplier: Decimal = None adFormat: str + bidAdjustments: Optional[List[BidAdjustment]] + creative: Optional[Creative] + landingPage: Optional[LandingPage] + supplySource: Optional[str] class BrandsAdGroup(CatalogModel): campaignId: Decimal adGroupId: Decimal name: str + bid: Decimal + keywordId: Decimal + keywordText: str + nativeLanguageKeyword: str + matchType: str + state: str diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_display.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_display.py index ffaf34b72e06..92f2c0ea4892 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_display.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_display.py @@ -3,7 +3,7 @@ # from decimal import Decimal -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional from .common import CatalogModel, Targeting @@ -43,10 +43,19 @@ class DisplayProductAds(CatalogModel): class DisplayTargeting(Targeting): + campaignId: Decimal expression: List[Dict[str, str]] resolvedExpression: List[Dict[str, str]] +class DisplayCreatives(CatalogModel): + adGroupId: Decimal + creativeId: Decimal + creativeType: str + properties: Dict[str, Any] + moderationStatus: str + + class DisplayBudgetRuleDetailsPerformanceMeasureCondition(CatalogModel): metricName: str comparisonOperator: str @@ -72,7 +81,7 @@ class DisplayBudgetRuleDetailsDurationEventTypeRuleDuration(CatalogModel): class DisplayBudgetRuleDetailsDurationDateRangeTypeRuleDuration(CatalogModel): - endDate: str + endDate: Optional[str] startDate: str @@ -94,7 +103,7 @@ class DisplayBudgetRules(CatalogModel): ruleId: str ruleStatus: str ruleState: str - lastUpdatedDate: Decimal + lastUpdatedDate: Optional[Decimal] createdDate: Decimal ruleDetails: DisplayBudgetRuleDetails = None ruleStatusDetails: Dict[str, str] = None diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_products.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_products.py index d60f1042592c..b5ca604e06b6 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_products.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/schemas/sponsored_products.py @@ -32,6 +32,7 @@ class ProductCampaign(CatalogModel): endDate: str = None premiumBidAdjustment: bool bidding: Bidding + networks: str class ProductAdGroups(CatalogModel): diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py index c521e04082a5..9d1852c33330 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py @@ -28,6 +28,7 @@ SponsoredDisplayAdGroups, SponsoredDisplayBudgetRules, SponsoredDisplayCampaigns, + SponsoredDisplayCreatives, SponsoredDisplayProductAds, SponsoredDisplayReportStream, SponsoredDisplayTargetings, @@ -79,7 +80,10 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> # in response body. # It doesnt support pagination so there is no sense of reading single # record, it would fetch all the data anyway. - Profiles(config, authenticator=self._make_authenticator(config)).get_all_profiles() + profiles_list = Profiles(config, authenticator=self._make_authenticator(config)).get_all_profiles() + filtered_profiles = self._choose_profiles(config, profiles_list) + if not filtered_profiles: + return False, "No profiles found after filtering by Profile ID and Marketplace ID" return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: @@ -101,6 +105,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: non_profile_stream_classes = [ SponsoredDisplayCampaigns, SponsoredDisplayAdGroups, + SponsoredDisplayCreatives, SponsoredDisplayProductAds, SponsoredDisplayTargetings, SponsoredDisplayReportStream, @@ -139,7 +144,13 @@ def _make_authenticator(config: Mapping[str, Any]): ) @staticmethod - def _choose_profiles(config: Mapping[str, Any], profiles: List[Profile]): - if not config.get("profiles"): - return profiles - return list(filter(lambda profile: profile.profileId in config["profiles"], profiles)) + def _choose_profiles(config: Mapping[str, Any], available_profiles: List[Profile]): + requested_profiles = config.get("profiles", []) + requested_marketplace_ids = config.get("marketplace_ids", []) + if requested_profiles or requested_marketplace_ids: + return [ + profile + for profile in available_profiles + if profile.profileId in requested_profiles or profile.accountInfo.marketplaceStringId in requested_marketplace_ids + ] + return available_profiles diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml index 795ca21589e7..0e703cb4ca3a 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml @@ -17,6 +17,7 @@ connectionSpecification: for more information. order: 1 type: string + airbyte_secret: true client_secret: title: Client Secret description: @@ -51,6 +52,8 @@ connectionSpecification: description: The Start date for collecting reports, should not be more than 60 days in the past. In YYYY-MM-DD format + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + format: date examples: - "2022-10-10" - "2022-10-22" @@ -58,13 +61,18 @@ connectionSpecification: type: string profiles: title: Profile IDs - description: - Profile IDs you want to fetch data for. See docs - for more details. + description: 'Profile IDs you want to fetch data for. See docs for more details. Note: If Marketplace IDs are also selected, profiles will be selected if they match the Profile ID OR the Marketplace ID.' order: 6 type: array items: type: integer + marketplace_ids: + title: Marketplace IDs + description: "Marketplace IDs you want to fetch data for. Note: If Profile IDs are also selected, profiles will be selected if they match the Profile ID OR the Marketplace ID." + order: 7 + type: array + items: + type: string state_filter: title: State Filter description: Reflects the state of the Display, Product, and Brand Campaign streams as enabled, paused, or archived. If you do not populate this field, it will be ignored completely. @@ -76,7 +84,7 @@ connectionSpecification: - archived type: array uniqueItems: true - order: 7 + order: 8 look_back_window: title: "Look Back Window" description: "The amount of days to go back in time to get the updated data from Amazon Ads" @@ -85,7 +93,7 @@ connectionSpecification: - 10 type: "integer" default: 3 - order: 8 + order: 9 report_record_types: title: Report Record Types description: @@ -107,7 +115,7 @@ connectionSpecification: - targets type: array uniqueItems: true - order: 9 + order: 10 required: - client_id - client_secret @@ -119,6 +127,14 @@ advanced_auth: - auth_type predicate_value: oauth2.0 oauth_config_specification: + oauth_user_input_from_connector_config_specification: + type: object + additionalProperties: false + properties: + region: + type: string + path_in_connector_config: + - region complete_oauth_output_specification: type: object additionalProperties: true diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/__init__.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/__init__.py index fb27c5d5c792..a449faafedb6 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/__init__.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/__init__.py @@ -21,6 +21,7 @@ SponsoredDisplayAdGroups, SponsoredDisplayBudgetRules, SponsoredDisplayCampaigns, + SponsoredDisplayCreatives, SponsoredDisplayProductAds, SponsoredDisplayTargetings, ) @@ -45,6 +46,7 @@ "SponsoredDisplayTargetings", "SponsoredDisplayBudgetRules", "SponsoredProductAdGroups", + "SponsoredDisplayCreatives", "SponsoredProductAdGroupBidRecommendations", "SponsoredProductAdGroupSuggestedKeywords", "SponsoredProductAds", diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py index 12f6fa0c7606..247122c1e9c2 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/common.py @@ -67,7 +67,7 @@ class to provide explanation why it had been done in this way. class ErrorResponse(BaseModel): code: str details: str - requestId: str + requestId: Optional[str] class BasicAmazonAdsStream(Stream, ABC): diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_display.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_display.py index 35520f22f1c2..9c2b0f24fb85 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_display.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_display.py @@ -4,7 +4,14 @@ from typing import Any, Mapping -from source_amazon_ads.schemas import DisplayAdGroup, DisplayBudgetRules, DisplayCampaign, DisplayProductAds, DisplayTargeting +from source_amazon_ads.schemas import ( + DisplayAdGroup, + DisplayBudgetRules, + DisplayCampaign, + DisplayCreatives, + DisplayProductAds, + DisplayTargeting, +) from source_amazon_ads.streams.common import SubProfilesStream @@ -71,6 +78,19 @@ def path(self, **kwargs) -> str: return "sd/targets" +class SponsoredDisplayCreatives(SubProfilesStream): + """ + This stream corresponds to Amazon Advertising API - Sponsored Displays Creatives + https://advertising.amazon.com/API/docs/en-us/sponsored-display/3-0/openapi#/Creatives/listCreatives + """ + + primary_key = "creativeId" + model = DisplayCreatives + + def path(self, **kwargs) -> str: + return "/sd/creatives" + + class SponsoredDisplayBudgetRules(SubProfilesStream): """ This stream corresponds to Amazon Advertising API - Sponsored Displays BudgetRules diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_products.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_products.py index 14b86f888737..c3b596ae0490 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_products.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/sponsored_products.py @@ -104,6 +104,13 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp f"Skip current AdGroup because it does not support request {response.request.url} for " f"{response.request.headers['Amazon-Advertising-API-Scope']} profile: {response.text}" ) + elif response.status_code == HTTPStatus.NOT_FOUND: + # 404 Either the specified ad group identifier was not found, + # or the specified ad group was found but no associated bid was found. + self.logger.warning( + f"Skip current AdGroup because the specified ad group has no associated bid {response.request.url} for " + f"{response.request.headers['Amazon-Advertising-API-Scope']} profile: {response.text}" + ) else: response.raise_for_status() diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py index a1335bfb5fb2..8ee9193d81a1 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py @@ -16,7 +16,7 @@ def config(): "refresh_token": "test_refresh", "region": "NA", "look_back_window": 3, - "report_record_types": [] + "report_record_types": [], } @@ -73,6 +73,13 @@ def targeting_response(): """ +@fixture +def creatives_response(): + return """ +[{"creativeId":0,"adGroupId":0,"creativeType":"IMAGE","properties":{"headline":"string"},"moderationStatus":"APPROVED"}] +""" + + @fixture def attribution_report_response(): def _internal(report_type: str): diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_attribution_report.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_attribution_report.py index cf24dd16ad7b..32203166e150 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_attribution_report.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_attribution_report.py @@ -138,22 +138,22 @@ def test_attribution_report_slices(config): slices = list(stream.stream_slices(sync_mode=SyncMode.full_refresh)) assert slices == [ - {'profileId': 1, 'startDate': '20220514', 'endDate': '20220515'}, - {'profileId': 2, 'startDate': '20220514', 'endDate': '20220515'} + {"profileId": 1, "startDate": "20220514", "endDate": "20220515"}, + {"profileId": 2, "startDate": "20220514", "endDate": "20220515"}, ] config["start_date"] = pendulum.from_format("2022-05-01", "YYYY-MM-DD").date() stream = AttributionReportProducts(config, profiles=profiles) slices = list(stream.stream_slices(sync_mode=SyncMode.full_refresh)) assert slices == [ - {'profileId': 1, 'startDate': '20220501', 'endDate': '20220515'}, - {'profileId': 2, 'startDate': '20220501', 'endDate': '20220515'} + {"profileId": 1, "startDate": "20220501", "endDate": "20220515"}, + {"profileId": 2, "startDate": "20220501", "endDate": "20220515"}, ] config["start_date"] = pendulum.from_format("2022-01-01", "YYYY-MM-DD").date() stream = AttributionReportProducts(config, profiles=profiles) slices = list(stream.stream_slices(sync_mode=SyncMode.full_refresh)) assert slices == [ - {'profileId': 1, 'startDate': '20220214', 'endDate': '20220515'}, - {'profileId': 2, 'startDate': '20220214', 'endDate': '20220515'} + {"profileId": 1, "startDate": "20220214", "endDate": "20220515"}, + {"profileId": 2, "startDate": "20220214", "endDate": "20220515"}, ] diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py index 9b4329f37031..13783837a56a 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py @@ -759,24 +759,7 @@ def test_streams_state_filter(mocker, config, state_filter, stream_class): @responses.activate @pytest.mark.parametrize( "custom_record_types, flag_match_error", - [ - ( - ["campaigns"], - True - ), - ( - ["campaigns", "adGroups"], - True - ), - ( - [], - False - ), - ( - ["invalid_record_type"], - True - ) - ] + [(["campaigns"], True), (["campaigns", "adGroups"], True), ([], False), (["invalid_record_type"], True)], ) def test_display_report_stream_with_custom_record_types(config_gen, custom_record_types, flag_match_error): setup_responses( @@ -791,7 +774,7 @@ def test_display_report_stream_with_custom_record_types(config_gen, custom_recor stream_slice = {"profile": profiles[0], "reportDate": "20210725"} records = list(stream.read_records(SyncMode.incremental, stream_slice=stream_slice)) for record in records: - if record['recordType'] not in custom_record_types: + if record["recordType"] not in custom_record_types: if flag_match_error: assert False @@ -800,37 +783,13 @@ def test_display_report_stream_with_custom_record_types(config_gen, custom_recor @pytest.mark.parametrize( "custom_record_types, expected_record_types, flag_match_error", [ - ( - ["campaigns"], - ["campaigns"], - True - ), - ( - ["asins_keywords"], - ["asins_keywords"], - True - ), - ( - ["asins_targets"], - ["asins_targets"], - True - ), - ( - ["campaigns", "adGroups"], - ["campaigns", "adGroups"], - True - ), - ( - [], - [], - False - ), - ( - ["invalid_record_type"], - [], - True - ) - ] + (["campaigns"], ["campaigns"], True), + (["asins_keywords"], ["asins_keywords"], True), + (["asins_targets"], ["asins_targets"], True), + (["campaigns", "adGroups"], ["campaigns", "adGroups"], True), + ([], [], False), + (["invalid_record_type"], [], True), + ], ) def test_products_report_stream_with_custom_record_types(config_gen, custom_record_types, expected_record_types, flag_match_error): setup_responses( @@ -846,7 +805,7 @@ def test_products_report_stream_with_custom_record_types(config_gen, custom_reco records = list(stream.read_records(SyncMode.incremental, stream_slice=stream_slice)) for record in records: print(record) - if record['recordType'] not in expected_record_types: + if record["recordType"] not in expected_record_types: if flag_match_error: assert False @@ -855,32 +814,12 @@ def test_products_report_stream_with_custom_record_types(config_gen, custom_reco @pytest.mark.parametrize( "custom_record_types, expected_record_types, flag_match_error", [ - ( - ["campaigns"], - ["campaigns"], - True - ), - ( - ["asins"], - ["asins"], - True - ), - ( - ["campaigns", "adGroups"], - ["campaigns", "adGroups"], - True - ), - ( - [], - [], - False - ), - ( - ["invalid_record_type"], - [], - True - ) - ] + (["campaigns"], ["campaigns"], True), + (["asins"], ["asins"], True), + (["campaigns", "adGroups"], ["campaigns", "adGroups"], True), + ([], [], False), + (["invalid_record_type"], [], True), + ], ) def test_brands_video_report_with_custom_record_types(config_gen, custom_record_types, expected_record_types, flag_match_error): setup_responses( @@ -896,7 +835,7 @@ def test_brands_video_report_with_custom_record_types(config_gen, custom_record_ records = list(stream.read_records(SyncMode.incremental, stream_slice=stream_slice)) for record in records: print(record) - if record['recordType'] not in expected_record_types: + if record["recordType"] not in expected_record_types: if flag_match_error: assert False @@ -906,8 +845,8 @@ def test_brands_video_report_with_custom_record_types(config_gen, custom_record_ [ ({"campaignId": "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"}, "campaigns"), ({"campaignId": ""}, "campaigns"), - ({"campaignId": None}, "campaigns") - ] + ({"campaignId": None}, "campaigns"), + ], ) def test_get_record_id_by_report_type(config, metric_object, record_type): """ diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py index 9126547b19ad..f3e8e9d93954 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py @@ -2,10 +2,12 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import responses from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConnectorSpecification, Status, Type from jsonschema import Draft4Validator from source_amazon_ads import SourceAmazonAds +from source_amazon_ads.schemas import Profile from .utils import command_check, url_strip_query @@ -19,7 +21,7 @@ def setup_responses(): responses.add( responses.GET, "https://advertising-api.amazon.com/v2/profiles", - json=[], + json=[{"profileId": 111, "timezone": "gtm", "accountInfo": {"marketplaceStringId": "mkt_id_1", "id": "111", "type": "vendor"}}], ) @@ -31,8 +33,7 @@ def ensure_additional_property_is_boolean(root): ensure_additional_property_is_boolean(prop) if "additionalProperties" in root: assert type(root["additionalProperties"]) == bool, ( - f"`additionalProperties` expected to be of 'bool' type. " - f"Got: {type(root['additionalProperties']).__name__}" + f"`additionalProperties` expected to be of 'bool' type. " f"Got: {type(root['additionalProperties']).__name__}" ) @@ -90,7 +91,7 @@ def test_source_streams(config): setup_responses() source = SourceAmazonAds() streams = source.streams(config) - assert len(streams) == 28 + assert len(streams) == 29 actual_stream_names = {stream.name for stream in streams} expected_stream_names = set( [ @@ -115,7 +116,48 @@ def test_source_streams(config): "attribution_report_performance_campaign", "attribution_report_performance_creative", "attribution_report_products", - "sponsored_display_budget_rules" + "sponsored_display_budget_rules", ] ) assert not expected_stream_names - actual_stream_names + + +def test_filter_profiles_exist(): + source = SourceAmazonAds() + mock_objs = [ + {"profileId": 111, "timezone": "gtm", "accountInfo": {"marketplaceStringId": "mkt_id_1", "id": "111", "type": "vendor"}}, + {"profileId": 222, "timezone": "gtm", "accountInfo": {"marketplaceStringId": "mkt_id_2", "id": "222", "type": "vendor"}}, + {"profileId": 333, "timezone": "gtm", "accountInfo": {"marketplaceStringId": "mkt_id_3", "id": "333", "type": "vendor"}}, + ] + + mock_profiles = [Profile.parse_obj(profile) for profile in mock_objs] + + filtered_profiles = source._choose_profiles({}, mock_profiles) + assert len(filtered_profiles) == 3 + + filtered_profiles = source._choose_profiles({"profiles": [111]}, mock_profiles) + assert len(filtered_profiles) == 1 + assert filtered_profiles[0].profileId == 111 + + filtered_profiles = source._choose_profiles({"profiles": [111, 333]}, mock_profiles) + assert len(filtered_profiles) == 2 + + filtered_profiles = source._choose_profiles({"profiles": [444]}, mock_profiles) + assert len(filtered_profiles) == 0 + + filtered_profiles = source._choose_profiles({"marketplace_ids": ["mkt_id_4"]}, mock_profiles) + assert len(filtered_profiles) == 0 + + filtered_profiles = source._choose_profiles({"marketplace_ids": ["mkt_id_1"]}, mock_profiles) + assert len(filtered_profiles) == 1 + assert filtered_profiles[0].accountInfo.marketplaceStringId == "mkt_id_1" + + filtered_profiles = source._choose_profiles({"marketplace_ids": ["mkt_id_1", "mkt_id_3"]}, mock_profiles) + assert len(filtered_profiles) == 2 + + filtered_profiles = source._choose_profiles({"profiles": [111], "marketplace_ids": ["mkt_id_2"]}, mock_profiles) + assert len(filtered_profiles) == 2 + + filtered_profiles = source._choose_profiles({"profiles": [111], "marketplace_ids": ["mkt_id_1"]}, mock_profiles) + assert len(filtered_profiles) == 1 + assert filtered_profiles[0].profileId == 111 diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_streams.py index 2a9abb1e4d16..1eb1a45d3ac1 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_streams.py @@ -22,6 +22,7 @@ def setup_responses( targeting_response=None, product_ads_response=None, generic_response=None, + creatives_response=None, ): responses.add( responses.POST, @@ -64,6 +65,12 @@ def setup_responses( "https://advertising-api.amazon.com/sd/productAds", body=product_ads_response, ) + if creatives_response: + responses.add( + responses.GET, + "https://advertising-api.amazon.com/sd/creatives", + body=creatives_response, + ) if generic_response: responses.add( responses.GET, @@ -72,8 +79,8 @@ def setup_responses( ) -def get_all_stream_records(stream): - records = stream.read_records(SyncMode.full_refresh) +def get_all_stream_records(stream, stream_slice=None): + records = stream.read_records(SyncMode.full_refresh, stream_slice=stream_slice) return [r for r in records] @@ -217,23 +224,19 @@ def test_streams_campaigns_pagination_403_error_expected(mocker, config, profile ("sponsored_display_ad_groups", "sd/adGroups"), ("sponsored_display_product_ads", "sd/productAds"), ("sponsored_display_targetings", "sd/targets"), + ("sponsored_display_creatives", "sd/creatives"), ], ) @responses.activate def test_streams_displays( - config, - stream_name, - endpoint, - profiles_response, - adgroups_response, - targeting_response, - product_ads_response, + config, stream_name, endpoint, profiles_response, adgroups_response, targeting_response, product_ads_response, creatives_response ): setup_responses( profiles_response=profiles_response, adgroups_response=adgroups_response, targeting_response=targeting_response, product_ads_response=product_ads_response, + creatives_response=creatives_response, ) source = SourceAmazonAds() @@ -273,3 +276,23 @@ def test_streams_brands_and_products(config, stream_name, endpoint, profiles_res records = get_all_stream_records(test_stream) assert records == [] assert any([endpoint in call.request.url for call in responses.calls]) + + +@responses.activate +def test_sponsored_product_ad_group_bid_recommendations_404_error(caplog, config, profiles_response): + setup_responses(profiles_response=profiles_response) + responses.add( + responses.GET, + "https://advertising-api.amazon.com/v2/sp/adGroups/xxx/bidRecommendations", + json={ + "code": "404", + "details": "404 Either the specified ad group identifier was not found or the specified ad group was found but no associated bid was found.", + }, + status=404, + ) + source = SourceAmazonAds() + streams = source.streams(config) + test_stream = get_stream_by_name(streams, "sponsored_product_ad_group_bid_recommendations") + records = get_all_stream_records(test_stream, stream_slice={"profileId": "1231", "adGroupId": "xxx"}) + assert records == [] + assert "Skip current AdGroup because the specified ad group has no associated bid" in caplog.text diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py index 2b69cd22e4e4..eb6cbb4dd93f 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py @@ -11,6 +11,7 @@ from airbyte_cdk.sources import Source from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit, split_config +from source_amazon_ads.config_migrations import MigrateStartDate def read_incremental(stream_instance: Stream, stream_state: MutableMapping[str, Any]) -> Iterator[dict]: @@ -41,6 +42,7 @@ def read_full_refresh(stream_instance: Stream): def command_check(source: Source, config): logger = mock.MagicMock() connector_config, _ = split_config(config) + connector_config = MigrateStartDate.modify_and_save("unit_tests/config.json", source, connector_config) if source.check_config_against_spec: source_spec: ConnectorSpecification = source.spec(logger) check_config_against_spec_or_exit(connector_config, source_spec) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/Dockerfile b/airbyte-integrations/connectors/source-amazon-seller-partner/Dockerfile deleted file mode 100644 index 3dd7d468d499..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY source_amazon_seller_partner ./source_amazon_seller_partner -COPY main.py ./ -COPY setup.py ./ -RUN pip install . - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - - -LABEL io.airbyte.version=1.5.1 -LABEL io.airbyte.name=airbyte/source-amazon-seller-partner diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/README.md b/airbyte-integrations/connectors/source-amazon-seller-partner/README.md index 5251d63fd92b..fc2ba61fd156 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/README.md +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/README.md @@ -27,12 +27,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-amazon-seller-partner:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/amazon-seller-partner) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_amazon_seller-partner/integration_tests/spec.json` file. @@ -52,18 +46,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-amazon-seller-partner:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-amazon-seller-partner build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-amazon-seller-partner:airbyteDocker +An image will be built with the tag `airbyte/source-amazon-seller-partner:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-amazon-seller-partner:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -73,44 +68,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-amazon-seller-partner: docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-amazon-seller-partner:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-amazon-seller-partner:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .'[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-amazon-seller-partner test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-amazon-seller-partner:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-amazon-seller-partner:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -120,8 +87,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-amazon-seller-partner test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/amazon-seller-partner.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml index 5b753c0a5ac1..e94239156b64 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-config.yml @@ -1,11 +1,11 @@ connector_image: airbyte/source-amazon-seller-partner:dev +test_strictness_level: high acceptance_tests: spec: tests: - - spec_path: "integration_tests/spec.json" - # massively refactored `spec` + - spec_path: "source_amazon_seller_partner/spec.json" backward_compatibility_tests_config: - disable_for_version: "1.2.0" + disable_for_version: "2.0.1" connection: tests: - config_path: "secrets/config.json" @@ -24,7 +24,38 @@ acceptance_tests: basic_read: tests: - config_path: "secrets/config.json" - timeout_seconds: 2400 + timeout_seconds: 3600 + ignored_fields: + GET_MERCHANT_LISTINGS_ALL_DATA: + - name: "dataEndTime" + bypass_reason: "This field is used as a cursor field and depends on today's date, so it changes every day" + GET_FLAT_FILE_OPEN_LISTINGS_DATA: + - name: "dataEndTime" + bypass_reason: "This field is used as a cursor field and depends on today's date, so it changes every day" + GET_MERCHANTS_LISTINGS_FYP_REPORT: + - name: "dataEndTime" + bypass_reason: "This field is used as a cursor field and depends on today's date, so it changes every day" + GET_MERCHANT_LISTINGS_DATA: + - name: "dataEndTime" + bypass_reason: "This field is used as a cursor field and depends on today's date, so it changes every day" + GET_MERCHANT_LISTINGS_INACTIVE_DATA: + - name: "dataEndTime" + bypass_reason: "This field is used as a cursor field and depends on today's date, so it changes every day" + GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT: + - name: "dataEndTime" + bypass_reason: "This field is used as a cursor field and depends on today's date, so it changes every day" + GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE: + - name: "dataEndTime" + bypass_reason: "This field is used as a cursor field and depends on today's date, so it changes every day" + GET_MERCHANT_LISTINGS_DATA_BACK_COMPAT: + - name: "dataEndTime" + bypass_reason: "This field is used as a cursor field and depends on today's date, so it changes every day" + GET_XML_BROWSE_TREE_DATA: + - name: "dataEndTime" + bypass_reason: "This field is used as a cursor field and depends on today's date, so it changes every day" + ListFinancialEvents: + - name: "PostedBefore" + bypass_reason: "This field is used as a cursor field and depends on today's date, so it changes every day" expect_records: path: "integration_tests/expected_records.jsonl" extra_fields: no @@ -36,8 +67,6 @@ acceptance_tests: bypass_reason: "no access and no data" - name: GET_ORDER_REPORT_DATA_SHIPPING bypass_reason: "no access and no data" - - name: GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL - bypass_reason: "no records" - name: GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL bypass_reason: "no records" - name: GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA @@ -48,16 +77,12 @@ acceptance_tests: bypass_reason: "no records" - name: GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA bypass_reason: "no records" - - name: GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT - bypass_reason: "no records" - name: GET_LEDGER_DETAIL_VIEW_DATA bypass_reason: "no records" - name: GET_AFN_INVENTORY_DATA_BY_COUNTRY bypass_reason: "no records" - name: GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE bypass_reason: "no records" - - name: GET_FBA_FULFILLMENT_MONTHLY_INVENTORY_DATA - bypass_reason: "no records" - name: GET_VENDOR_SALES_REPORT bypass_reason: "no records" - name: GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT @@ -66,12 +91,6 @@ acceptance_tests: bypass_reason: "no records" - name: GET_FBA_SNS_FORECAST_DATA bypass_reason: "no records" - - name: GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE - bypass_reason: "no records" - - name: GET_FBA_FULFILLMENT_INVENTORY_SUMMARY_DATA - bypass_reason: "no records" - - name: GET_BRAND_ANALYTICS_ITEM_COMPARISON_REPORT - bypass_reason: "no records" - name: GET_AFN_INVENTORY_DATA bypass_reason: "no records" - name: GET_MERCHANT_CANCELLED_LISTINGS_DATA @@ -80,14 +99,8 @@ acceptance_tests: bypass_reason: "no records" - name: GET_LEDGER_SUMMARY_VIEW_DATA bypass_reason: "no records" - - name: GET_BRAND_ANALYTICS_ALTERNATE_PURCHASE_REPORT - bypass_reason: "no records" - name: GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT bypass_reason: "no records" - - name: GET_FBA_FULFILLMENT_INVENTORY_ADJUSTMENTS_DATA - bypass_reason: "no records" - - name: GET_MERCHANT_LISTINGS_DATA_BACK_COMPAT - bypass_reason: "no records" - name: GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT bypass_reason: "no records" - name: VendorDirectFulfillmentShipping @@ -98,33 +111,25 @@ acceptance_tests: bypass_reason: "no records" - name: GET_FBA_SNS_PERFORMANCE_DATA bypass_reason: "no records" - - name: GET_FBA_FULFILLMENT_CURRENT_INVENTORY_DATA - bypass_reason: "no records" - name: GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA bypass_reason: "no records" - name: GET_FBA_INVENTORY_PLANNING_DATA bypass_reason: "no records" - name: GET_FBA_STORAGE_FEE_CHARGES_DATA bypass_reason: "no records" - - name: GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL - bypass_reason: "no records" - name: GET_SALES_AND_TRAFFIC_REPORT bypass_reason: "no records" - name: GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA bypass_reason: "no records" - name: GET_STRANDED_INVENTORY_UI_DATA bypass_reason: "no records" - - name: GET_FBA_FULFILLMENT_INVENTORY_RECEIPTS_DATA - bypass_reason: "no records" - - name: GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL - bypass_reason: "no records" - - name: ListFinancialEvents + - name: GET_FBA_REIMBURSEMENTS_DATA bypass_reason: "no records" - - name: ListFinancialEventGroups + - name: GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT bypass_reason: "no records" - - name: GET_FBA_REIMBURSEMENTS_DATA + - name: GET_VENDOR_REAL_TIME_INVENTORY_REPORT bypass_reason: "no records" - - name: GET_XML_BROWSE_TREE_DATA + - name: GET_VENDOR_TRAFFIC_REPORT bypass_reason: "no records" incremental: tests: diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/build.gradle b/airbyte-integrations/connectors/source-amazon-seller-partner/build.gradle deleted file mode 100644 index 3bbfc0b4a983..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_amazon_seller_partner' -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_brand_analytics_alternate_purchase.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_brand_analytics_alternate_purchase.json deleted file mode 100644 index 2ce8fbb81064..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_brand_analytics_alternate_purchase.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "GET_BRAND_ANALYTICS_ALTERNATE_PURCHASE_REPORT", - "json_schema": { - "title": "Brand Analytics Alternate Purchase Reports", - "description": "Brand Analytics Alternate Purchase Reports", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "startDate": { - "type": ["null", "string"], - "format": "date" - }, - "endDate": { - "type": ["null", "string"], - "format": "date" - }, - "asin": { - "type": ["null", "string"] - }, - "purchasedAsin": { - "type": ["null", "string"] - }, - "purchasedRank": { - "type": ["null", "integer"] - }, - "purchasedPct": { - "type": ["null", "number"] - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_brand_analytics_item_comparison.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_brand_analytics_item_comparison.json deleted file mode 100644 index 4d7300e63157..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_brand_analytics_item_comparison.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "GET_BRAND_ANALYTICS_ITEM_COMPARISON_REPORT", - "json_schema": { - "title": "Brand Analytics Item Comparison Reports", - "description": "Brand Analytics Item Comparison Reports", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "startDate": { - "type": ["null", "string"], - "format": "date" - }, - "endDate": { - "type": ["null", "string"], - "format": "date" - }, - "asin": { - "type": ["null", "string"] - }, - "comparedAsin": { - "type": ["null", "string"] - }, - "comparedRank": { - "type": ["null", "integer"] - }, - "comparedPct": { - "type": ["null", "number"] - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_current_inventory_data.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_current_inventory_data.json deleted file mode 100644 index 376c90214355..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_current_inventory_data.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "GET_FBA_FULFILLMENT_CURRENT_INVENTORY_DATA", - "json_schema": { - "title": "FBA Daily Inventory History Report", - "description": "", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "snapshot-date": { "type": ["null", "string"] }, - "fnsku": { "type": ["null", "string"] }, - "sku": { "type": ["null", "string"] }, - "product-name": { "type": ["null", "string"] }, - "quantity": { "type": ["null", "string"] }, - "fulfillment-center-id": { "type": ["null", "string"] }, - "detailed-disposition": { "type": ["null", "string"] }, - "country": { "type": ["null", "string"] } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_inventory_adjustments_data.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_inventory_adjustments_data.json deleted file mode 100644 index eb241d9e7e3e..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_inventory_adjustments_data.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "GET_FBA_FULFILLMENT_INVENTORY_ADJUSTMENTS_DATA", - "json_schema": { - "title": "FBA Inventory Adjustments Report", - "description": "", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "adjusted-date": { "type": ["null", "string"] }, - "transaction-item-id": { "type": ["null", "string"] }, - "fnsku": { "type": ["null", "string"] }, - "sku": { "type": ["null", "string"] }, - "product-name": { "type": ["null", "string"] }, - "fulfillment-center-id": { "type": ["null", "string"] }, - "quantity": { "type": ["null", "string"] }, - "reason": { "type": ["null", "string"] }, - "disposition": { "type": ["null", "string"] }, - "reconciled": { "type": ["null", "string"] }, - "unreconciled": { "type": ["null", "string"] } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_inventory_receipts_data.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_inventory_receipts_data.json deleted file mode 100644 index 92575cfb052e..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_inventory_receipts_data.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "GET_FBA_FULFILLMENT_INVENTORY_RECEIPTS_DATA", - "json_schema": { - "title": "FBA Received Inventory Report", - "description": "", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "received-date": { "type": ["null", "string"] }, - "fnsku": { "type": ["null", "string"] }, - "sku": { "type": ["null", "string"] }, - "product-name": { "type": ["null", "string"] }, - "quantity": { "type": ["null", "string"] }, - "fba-shipment-id": { "type": ["null", "string"] }, - "fulfillment-center-id": { "type": ["null", "string"] } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_inventory_summary_data.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_inventory_summary_data.json deleted file mode 100644 index b38e9b2849ef..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_inventory_summary_data.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "GET_FBA_FULFILLMENT_INVENTORY_SUMMARY_DATA", - "json_schema": { - "title": "FBA Inventory Event Detail Report", - "description": "", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "snapshot-date": { "type": ["null", "string"] }, - "transaction-type": { "type": ["null", "string"] }, - "fnsku": { "type": ["null", "string"] }, - "sku": { "type": ["null", "string"] }, - "product-name": { "type": ["null", "string"] }, - "fulfillment-center-id": { "type": ["null", "string"] }, - "quantity": { "type": ["null", "string"] }, - "disposition": { "type": ["null", "string"] } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_monthly_inventory_data.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_monthly_inventory_data.json deleted file mode 100644 index c695f887c910..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_get_fba_fulfillment_monthly_inventory_data.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "GET_FBA_FULFILLMENT_MONTHLY_INVENTORY_DATA", - "json_schema": { - "title": "FBA Monthly Inventory History Report", - "description": "", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "month": { "type": ["null", "string"] }, - "fnsku": { "type": ["null", "string"] }, - "sku": { "type": ["null", "string"] }, - "product-name": { "type": ["null", "string"] }, - "average-quantity": { "type": ["null", "string"] }, - "end-quantity": { "type": ["null", "string"] }, - "fulfillment-center-id": { "type": ["null", "string"] }, - "detailed-disposition": { "type": ["null", "string"] }, - "country": { "type": ["null", "string"] } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_incremental.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_incremental.json index 55c7437a3fc0..22d1634a68a8 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_incremental.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_incremental.json @@ -23,6 +23,42 @@ "sync_mode": "incremental", "destination_sync_mode": "append", "cursor_field": ["LastUpdateDate"] + }, + { + "stream": { + "name": "GET_AFN_INVENTORY_DATA", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["dataEndTime"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["dataEndTime"] + }, + { + "stream": { + "name": "GET_AFN_INVENTORY_DATA_BY_COUNTRY", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["dataEndTime"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["dataEndTime"] + }, + { + "stream": { + "name": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["dataEndTime"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["dataEndTime"] } ] } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_no_empty_streams.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_no_empty_streams.json index 3542eab16f4e..3d5607919cd4 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_no_empty_streams.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/configured_catalog_no_empty_streams.json @@ -9,15 +9,6 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, - { - "stream": { - "name": "GET_FBA_INVENTORY_AGED_DATA", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, { "stream": { "name": "GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL", diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/expected_records.jsonl index d3778d0803cc..c36b190c842b 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/expected_records.jsonl @@ -1,17 +1,87 @@ -{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "GiftBox", "item-description": "Monitor and optimize the GiftBox to reward your customers and increase the average order value", "listing-id": "0711ZJUYPNS", "seller-sku": "I0-RALD-N1UR", "price": "5", "quantity": "1000", "open-date": "2022-07-11 01:34:18 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Active"}, "emitted_at": 1690214254096} -{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "item-description": "", "listing-id": "0705Z8IQ8GS", "seller-sku": "0R-4KDA-Z2U8", "price": "5", "quantity": "983", "open-date": "2022-07-05 08:09:12 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHYM2E", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHYM2E", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive"}, "emitted_at": 1690214254097} -{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "Beyond Meat, Plant-Based Patties, Vegan, 8 Oz, 2 Patties", "item-description": "", "listing-id": "0708ZF4UYHW", "seller-sku": "2J-D6V7-C8XI", "price": "7", "quantity": "922", "open-date": "2022-07-08 03:50:23 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive"}, "emitted_at": 1690214254098} -{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "GiftBox", "item-description": "", "listing-id": "0711ZJWAW1J", "seller-sku": "G3-8N7Y-L93I", "price": "6", "quantity": "1000", "open-date": "2022-07-11 01:48:47 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive"}, "emitted_at": 1690214254098} -{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "Beyond Meat, Plant-Based Patties, Vegan, 8 Oz, 2 Patties", "item-description": "", "listing-id": "0711ZJW1CW7", "seller-sku": "M6-KYAA-V7O7", "price": "10", "quantity": "999999", "open-date": "2022-07-11 01:16:54 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive"}, "emitted_at": 1690214254098} -{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "House Foods, Organic Firm Tofu, 14 oz", "item-description": "", "listing-id": "0705Z8HWWAY", "seller-sku": "MP-V4RG-EDEY", "price": "5", "quantity": "1518", "open-date": "2022-07-05 08:00:10 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "1", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHRNUW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHRNUW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive"}, "emitted_at": 1690214254099} -{"stream": "GET_FLAT_FILE_OPEN_LISTINGS_DATA", "data": {"sku": "I0-RALD-N1UR", "asin": "B0B68NBQ1Y", "price": "5.00", "quantity": "1000", "Business Price": "6.0", "Quantity Price Type": "", "Quantity Lower Bound 1": "", "Quantity Price 1": "", "Quantity Lower Bound 2": "", "Quantity Price 2": "", "Quantity Lower Bound 3": "", "Quantity Price 3": "", "Quantity Lower Bound 4": "", "Quantity Price 4": "", "Quantity Lower Bound 5": "", "Quantity Price 5": "", "Progressive Price Type": "", "Progressive Lower Bound 1": "", "Progressive Price 1": "", "Progressive Lower Bound 2": "", "Progressive Price 2": "", "Progressive Lower Bound 3": "", "Progressive Price 3": ""}, "emitted_at": 1690217648401} -{"stream": "GET_MERCHANTS_LISTINGS_FYP_REPORT", "data": {"Status": "Search Suppressed", "Reason": "Missing info", "SKU": "G3-8N7Y-L93I", "ASIN": "B0B68NBQ1Y", "Product name": "GiftBox", "Condition": "11", "Status Change Date": "Jul 29, 2022", "Issue Description": "'[brand]' is required but not supplied."}, "emitted_at": 1690219384531} -{"stream": "GET_MERCHANTS_LISTINGS_FYP_REPORT", "data": {"Status": "Search Suppressed", "Reason": "Missing info", "SKU": "I0-RALD-N1UR", "ASIN": "B0B68NBQ1Y", "Product name": "GiftBox", "Condition": "11", "Status Change Date": "Jul 11, 2022", "Issue Description": "'[brand]' is required but not supplied."}, "emitted_at": 1690219384532} -{"stream": "GET_MERCHANT_LISTINGS_DATA", "data": {"item-name": "GiftBox", "item-description": "Monitor and optimize the GiftBox to reward your customers and increase the average order value", "listing-id": "0711ZJUYPNS", "seller-sku": "I0-RALD-N1UR", "price": "5", "quantity": "1000", "open-date": "2022-07-11 01:34:18 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "Business Price": "6.0", "Quantity Price Type": "", "Quantity Lower Bound 1": "", "Quantity Price 1": "", "Quantity Lower Bound 2": "", "Quantity Price 2": "", "Quantity Lower Bound 3": "", "Quantity Price 3": "", "Quantity Lower Bound 4": "", "Quantity Price 4": "", "Quantity Lower Bound 5": "", "Quantity Price 5": "", "merchant-shipping-group": "Migrated Template", "Progressive Price Type": "", "Progressive Lower Bound 1": "", "Progressive Price 1": "", "Progressive Lower Bound 2": "", "Progressive Price 2": "", "Progressive Lower Bound 3": "", "Progressive Price 3": ""}, "emitted_at": 1690220838938} -{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "item-description": "", "listing-id": "0705Z8IQ8GS", "seller-sku": "0R-4KDA-Z2U8", "price": "5", "quantity": "983", "open-date": "2022-07-05 08:09:12 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHYM2E", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHYM2E", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template"}, "emitted_at": 1690223127427} -{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "Beyond Meat, Plant-Based Patties, Vegan, 8 Oz, 2 Patties", "item-description": "", "listing-id": "0708ZF4UYHW", "seller-sku": "2J-D6V7-C8XI", "price": "7", "quantity": "922", "open-date": "2022-07-08 03:50:23 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template"}, "emitted_at": 1690223127429} -{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "GiftBox", "item-description": "", "listing-id": "0711ZJWAW1J", "seller-sku": "G3-8N7Y-L93I", "price": "6", "quantity": "1000", "open-date": "2022-07-11 01:48:47 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template"}, "emitted_at": 1690223127429} -{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "Beyond Meat, Plant-Based Patties, Vegan, 8 Oz, 2 Patties", "item-description": "", "listing-id": "0711ZJW1CW7", "seller-sku": "M6-KYAA-V7O7", "price": "10", "quantity": "999999", "open-date": "2022-07-11 01:16:54 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template"}, "emitted_at": 1690223127429} -{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "House Foods, Organic Firm Tofu, 14 oz", "item-description": "", "listing-id": "0705Z8HWWAY", "seller-sku": "MP-V4RG-EDEY", "price": "5", "quantity": "1518", "open-date": "2022-07-05 08:00:10 PDT", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "1", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHRNUW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHRNUW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template"}, "emitted_at": 1690223127429} -{"stream": "Orders", "data": {"BuyerInfo": {}, "AmazonOrderId": "112-4052057-4266618", "EarliestShipDate": "2022-07-25T07:00:00Z", "SalesChannel": "Amazon.com", "AutomatedShippingSettings": {"HasAutomatedShippingSettings": false}, "OrderStatus": "Canceled", "NumberOfItemsShipped": 0, "OrderType": "StandardOrder", "IsPremiumOrder": false, "IsPrime": false, "FulfillmentChannel": "MFN", "NumberOfItemsUnshipped": 0, "HasRegulatedItems": false, "IsReplacementOrder": "false", "IsSoldByAB": false, "LatestShipDate": "2022-07-26T06:59:59Z", "ShipServiceLevel": "Std US D2D Dom", "IsISPU": false, "MarketplaceId": "ATVPDKIKX0DER", "PurchaseDate": "2022-07-22T20:25:05Z", "IsAccessPointOrder": false, "IsBusinessOrder": false, "OrderTotal": {"CurrencyCode": "USD", "Amount": "7.00"}, "PaymentMethodDetails": ["Standard"], "IsGlobalExpressEnabled": false, "LastUpdateDate": "2022-09-01T13:16:42Z", "ShipmentServiceLevelCategory": "Standard"}, "emitted_at": 1691499338977} -{"stream": "OrderItems", "data": {"ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "7.00"}, "ASIN": "B074K5MDLW", "SellerSKU": "2J-D6V7-C8XI", "Title": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "00860509139506", "LastUpdateDate": "2022-09-01T13:16:42Z", "AmazonOrderId": "112-4052057-4266618"}, "emitted_at": 1691499343416} +{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "GiftBox", "item-description": "Monitor and optimize the GiftBox to reward your customers and increase the average order value", "listing-id": "0711ZJUYPNS", "seller-sku": "I0-RALD-N1UR", "price": "5", "quantity": "1000", "open-date": "2022-07-11T01:34:18-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Active", "dataEndTime": "2022-07-31"}, "emitted_at": 1701959478279} +{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "item-description": "", "listing-id": "0705Z8IQ8GS", "seller-sku": "0R-4KDA-Z2U8", "price": "5", "quantity": "983", "open-date": "2022-07-05T08:09:12-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHYM2E", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHYM2E", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive", "dataEndTime": "2022-07-31"}, "emitted_at": 1701959478281} +{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "Beyond Meat, Plant-Based Patties, Vegan, 8 Oz, 2 Patties", "item-description": "", "listing-id": "0708ZF4UYHW", "seller-sku": "2J-D6V7-C8XI", "price": "7", "quantity": "922", "open-date": "2022-07-08T03:50:23-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive", "dataEndTime": "2022-07-31"}, "emitted_at": 1701959478281} +{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "GiftBox", "item-description": "", "listing-id": "0711ZJWAW1J", "seller-sku": "G3-8N7Y-L93I", "price": "6", "quantity": "1000", "open-date": "2022-07-11T01:48:47-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive", "dataEndTime": "2022-07-31"}, "emitted_at": 1701959478281} +{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "Airbyte T-Shirt Black", "item-description": "Airbyte T-Shirt (Cotton)", "listing-id": "0915ADTXMIJ", "seller-sku": "IA-VREM-8L92", "price": "15", "quantity": "", "open-date": "2023-09-15T08:03:59-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0CJ5Q3NLP", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0CJ5Q3NLP", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "", "fulfillment-channel": "AMAZON_NA", "merchant-shipping-group": "Migrated Template", "status": "Inactive", "dataEndTime": "2022-07-31"}, "emitted_at": 1701959478281} +{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "Airbyte Merch White", "item-description": "Airbyte T-short", "listing-id": "0803A3SAML1", "seller-sku": "KW-J7BQ-WNKL", "price": "10", "quantity": "", "open-date": "2023-08-03T02:26:19-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0CDLLJ5VV", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0CDLLJ5VV", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "", "fulfillment-channel": "AMAZON_NA", "merchant-shipping-group": "Migrated Template", "status": "Inactive", "dataEndTime": "2022-07-31"}, "emitted_at": 1701959478282} +{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "Beyond Meat, Plant-Based Patties, Vegan, 8 Oz, 2 Patties", "item-description": "", "listing-id": "0711ZJW1CW7", "seller-sku": "M6-KYAA-V7O7", "price": "10", "quantity": "999999", "open-date": "2022-07-11T01:16:54-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive", "dataEndTime": "2022-07-31"}, "emitted_at": 1701959478282} +{"stream": "GET_MERCHANT_LISTINGS_ALL_DATA", "data": {"item-name": "House Foods, Organic Firm Tofu, 14 oz", "item-description": "", "listing-id": "0705Z8HWWAY", "seller-sku": "MP-V4RG-EDEY", "price": "5", "quantity": "1518", "open-date": "2022-07-05T08:00:10-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "1", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHRNUW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHRNUW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "status": "Inactive", "dataEndTime": "2022-07-31"}, "emitted_at": 1701959478282} +{"stream": "GET_FLAT_FILE_OPEN_LISTINGS_DATA", "data": {"sku": "I0-RALD-N1UR", "asin": "B0B68NBQ1Y", "price": "5.00", "quantity": "1000", "Business Price": "6.0", "Quantity Price Type": "", "Quantity Lower Bound 1": "", "Quantity Price 1": "", "Quantity Lower Bound 2": "", "Quantity Price 2": "", "Quantity Lower Bound 3": "", "Quantity Price 3": "", "Quantity Lower Bound 4": "", "Quantity Price 4": "", "Quantity Lower Bound 5": "", "Quantity Price 5": "", "Progressive Price Type": "", "Progressive Lower Bound 1": "", "Progressive Price 1": "", "Progressive Lower Bound 2": "", "Progressive Price 2": "", "Progressive Lower Bound 3": "", "Progressive Price 3": "", "dataEndTime": "2022-07-31"}, "emitted_at": 1701968460244} +{"stream": "GET_FLAT_FILE_OPEN_LISTINGS_DATA", "data": {"sku": "IA-VREM-8L92", "asin": "B0CJ5Q3NLP", "price": "15.00", "quantity": "", "Business Price": "", "Quantity Price Type": "", "Quantity Lower Bound 1": "", "Quantity Price 1": "", "Quantity Lower Bound 2": "", "Quantity Price 2": "", "Quantity Lower Bound 3": "", "Quantity Price 3": "", "Quantity Lower Bound 4": "", "Quantity Price 4": "", "Quantity Lower Bound 5": "", "Quantity Price 5": "", "Progressive Price Type": "", "Progressive Lower Bound 1": "", "Progressive Price 1": "", "Progressive Lower Bound 2": "", "Progressive Price 2": "", "Progressive Lower Bound 3": "", "Progressive Price 3": "", "dataEndTime": "2022-07-31"}, "emitted_at": 1701968460245} +{"stream": "GET_MERCHANTS_LISTINGS_FYP_REPORT", "data": {"Status": "Search Suppressed", "Reason": "Missing info", "SKU": "G3-8N7Y-L93I", "ASIN": "B0B68NBQ1Y", "Product name": "GiftBox", "Condition": "11", "Status Change Date": "2022-07-29", "Issue Description": "'[brand]' is required but not supplied.", "dataEndTime": "2022-07-31"}, "emitted_at": 1701968785470} +{"stream": "GET_MERCHANTS_LISTINGS_FYP_REPORT", "data": {"Status": "Search Suppressed", "Reason": "Missing info", "SKU": "I0-RALD-N1UR", "ASIN": "B0B68NBQ1Y", "Product name": "GiftBox", "Condition": "11", "Status Change Date": "2022-07-11", "Issue Description": "'[brand]' is required but not supplied.", "dataEndTime": "2022-07-31"}, "emitted_at": 1701968785473} +{"stream": "GET_MERCHANT_LISTINGS_DATA", "data": {"item-name": "GiftBox", "item-description": "Monitor and optimize the GiftBox to reward your customers and increase the average order value", "listing-id": "0711ZJUYPNS", "seller-sku": "I0-RALD-N1UR", "price": "5", "quantity": "1000", "open-date": "2022-07-11T01:34:18-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "Business Price": "6.0", "Quantity Price Type": "", "Quantity Lower Bound 1": "", "Quantity Price 1": "", "Quantity Lower Bound 2": "", "Quantity Price 2": "", "Quantity Lower Bound 3": "", "Quantity Price 3": "", "Quantity Lower Bound 4": "", "Quantity Price 4": "", "Quantity Lower Bound 5": "", "Quantity Price 5": "", "merchant-shipping-group": "Migrated Template", "Progressive Price Type": "", "Progressive Lower Bound 1": "", "Progressive Price 1": "", "Progressive Lower Bound 2": "", "Progressive Price 2": "", "Progressive Lower Bound 3": "", "Progressive Price 3": "", "dataEndTime": "2022-07-31"}, "emitted_at": 1701968964616} +{"stream": "GET_MERCHANT_LISTINGS_DATA", "data": {"item-name": "Airbyte T-Shirt Black", "item-description": "Airbyte T-Shirt (Cotton)", "listing-id": "0915ADTXMIJ", "seller-sku": "IA-VREM-8L92", "price": "15", "quantity": "", "open-date": "2023-09-15T08:03:59-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0CJ5Q3NLP", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0CJ5Q3NLP", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "", "fulfillment-channel": "AMAZON_NA", "Business Price": "", "Quantity Price Type": "", "Quantity Lower Bound 1": "", "Quantity Price 1": "", "Quantity Lower Bound 2": "", "Quantity Price 2": "", "Quantity Lower Bound 3": "", "Quantity Price 3": "", "Quantity Lower Bound 4": "", "Quantity Price 4": "", "Quantity Lower Bound 5": "", "Quantity Price 5": "", "merchant-shipping-group": "Migrated Template", "Progressive Price Type": "", "Progressive Lower Bound 1": "", "Progressive Price 1": "", "Progressive Lower Bound 2": "", "Progressive Price 2": "", "Progressive Lower Bound 3": "", "Progressive Price 3": "", "dataEndTime": "2022-07-31"}, "emitted_at": 1701968964618} +{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "item-description": "", "listing-id": "0705Z8IQ8GS", "seller-sku": "0R-4KDA-Z2U8", "price": "5", "quantity": "983", "open-date": "2022-07-05T08:09:12-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHYM2E", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHYM2E", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "dataEndTime": "2022-07-31"}, "emitted_at": 1701969137910} +{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "Beyond Meat, Plant-Based Patties, Vegan, 8 Oz, 2 Patties", "item-description": "", "listing-id": "0708ZF4UYHW", "seller-sku": "2J-D6V7-C8XI", "price": "7", "quantity": "922", "open-date": "2022-07-08T03:50:23-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "dataEndTime": "2022-07-31"}, "emitted_at": 1701969137911} +{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "GiftBox", "item-description": "", "listing-id": "0711ZJWAW1J", "seller-sku": "G3-8N7Y-L93I", "price": "6", "quantity": "1000", "open-date": "2022-07-11T01:48:47-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "dataEndTime": "2022-07-31"}, "emitted_at": 1701969137911} +{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "Airbyte T-Shirt Black", "item-description": "Airbyte T-Shirt (Cotton)", "listing-id": "0915ADTXMIJ", "seller-sku": "IA-VREM-8L92", "price": "15", "quantity": "", "open-date": "2023-09-15T08:03:59-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0CJ5Q3NLP", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0CJ5Q3NLP", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "", "fulfillment-channel": "AMAZON_NA", "merchant-shipping-group": "Migrated Template", "dataEndTime": "2022-07-31"}, "emitted_at": 1701969137911} +{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "Airbyte Merch White", "item-description": "Airbyte T-short", "listing-id": "0803A3SAML1", "seller-sku": "KW-J7BQ-WNKL", "price": "10", "quantity": "", "open-date": "2023-08-03T02:26:19-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0CDLLJ5VV", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0CDLLJ5VV", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "", "fulfillment-channel": "AMAZON_NA", "merchant-shipping-group": "Migrated Template", "dataEndTime": "2022-07-31"}, "emitted_at": 1701969137912} +{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "Beyond Meat, Plant-Based Patties, Vegan, 8 Oz, 2 Patties", "item-description": "", "listing-id": "0711ZJW1CW7", "seller-sku": "M6-KYAA-V7O7", "price": "10", "quantity": "999999", "open-date": "2022-07-11T01:16:54-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B074K5MDLW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B074K5MDLW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "dataEndTime": "2022-07-31"}, "emitted_at": 1701969137912} +{"stream": "GET_MERCHANT_LISTINGS_INACTIVE_DATA", "data": {"item-name": "House Foods, Organic Firm Tofu, 14 oz", "item-description": "", "listing-id": "0705Z8HWWAY", "seller-sku": "MP-V4RG-EDEY", "price": "5", "quantity": "1518", "open-date": "2022-07-05T08:00:10-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "1", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B000VHRNUW", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B000VHRNUW", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "fulfillment-channel": "DEFAULT", "merchant-shipping-group": "Migrated Template", "dataEndTime": "2022-07-31"}, "emitted_at": 1701969137912} +{"stream": "Orders", "data": {"BuyerInfo": {}, "AmazonOrderId": "111-1225255-7785053", "EarliestShipDate": "2022-07-18T07:00:00Z", "SalesChannel": "Amazon.com", "AutomatedShippingSettings": {"HasAutomatedShippingSettings": false}, "OrderStatus": "Canceled", "NumberOfItemsShipped": 0, "OrderType": "StandardOrder", "IsPremiumOrder": false, "IsPrime": false, "FulfillmentChannel": "MFN", "NumberOfItemsUnshipped": 0, "HasRegulatedItems": false, "IsReplacementOrder": "false", "IsSoldByAB": false, "LatestShipDate": "2022-07-19T06:59:59Z", "ShipServiceLevel": "Std US D2D Dom", "IsISPU": false, "MarketplaceId": "ATVPDKIKX0DER", "PurchaseDate": "2022-07-15T22:08:15Z", "IsAccessPointOrder": false, "IsBusinessOrder": false, "PaymentMethodDetails": ["Standard"], "IsGlobalExpressEnabled": false, "LastUpdateDate": "2022-07-18T22:54:07Z", "ShipmentServiceLevelCategory": "Standard"}, "emitted_at": 1701969184949} +{"stream": "Orders", "data": {"BuyerInfo": {}, "AmazonOrderId": "112-3632856-2922613", "EarliestShipDate": "2022-07-18T07:00:00Z", "SalesChannel": "Amazon.com", "AutomatedShippingSettings": {"HasAutomatedShippingSettings": false}, "OrderStatus": "Canceled", "NumberOfItemsShipped": 0, "OrderType": "StandardOrder", "IsPremiumOrder": false, "IsPrime": false, "FulfillmentChannel": "MFN", "NumberOfItemsUnshipped": 0, "HasRegulatedItems": false, "IsReplacementOrder": "false", "IsSoldByAB": false, "LatestShipDate": "2022-07-19T06:59:59Z", "ShipServiceLevel": "Std US D2D Dom", "IsISPU": false, "MarketplaceId": "ATVPDKIKX0DER", "PurchaseDate": "2022-07-17T07:44:26Z", "IsAccessPointOrder": false, "IsBusinessOrder": false, "PaymentMethodDetails": ["Standard"], "IsGlobalExpressEnabled": false, "LastUpdateDate": "2022-07-22T08:23:04Z", "ShipmentServiceLevelCategory": "Standard"}, "emitted_at": 1701969184949} +{"stream": "Orders", "data": {"BuyerInfo": {}, "AmazonOrderId": "113-8462063-1469066", "EarliestShipDate": "2022-07-25T07:00:00Z", "SalesChannel": "Amazon.com", "AutomatedShippingSettings": {"HasAutomatedShippingSettings": false}, "OrderStatus": "Canceled", "NumberOfItemsShipped": 0, "OrderType": "StandardOrder", "IsPremiumOrder": false, "IsPrime": false, "FulfillmentChannel": "MFN", "NumberOfItemsUnshipped": 0, "HasRegulatedItems": false, "IsReplacementOrder": "false", "IsSoldByAB": false, "LatestShipDate": "2022-07-26T06:59:59Z", "ShipServiceLevel": "Std US D2D Dom", "IsISPU": false, "MarketplaceId": "ATVPDKIKX0DER", "PurchaseDate": "2022-07-23T18:45:44Z", "IsAccessPointOrder": false, "IsBusinessOrder": false, "PaymentMethodDetails": ["Standard"], "IsGlobalExpressEnabled": false, "LastUpdateDate": "2022-07-23T18:46:16Z", "ShipmentServiceLevelCategory": "Standard"}, "emitted_at": 1701969184949} +{"stream": "Orders", "data": {"BuyerInfo": {}, "AmazonOrderId": "113-3281105-7707448", "EarliestShipDate": "2022-07-26T07:00:00Z", "SalesChannel": "Amazon.com", "AutomatedShippingSettings": {"HasAutomatedShippingSettings": false}, "OrderStatus": "Canceled", "NumberOfItemsShipped": 0, "OrderType": "StandardOrder", "IsPremiumOrder": false, "IsPrime": false, "FulfillmentChannel": "MFN", "NumberOfItemsUnshipped": 0, "HasRegulatedItems": false, "IsReplacementOrder": "false", "IsSoldByAB": false, "LatestShipDate": "2022-07-27T06:59:59Z", "ShipServiceLevel": "Std US D2D Dom", "IsISPU": false, "MarketplaceId": "ATVPDKIKX0DER", "PurchaseDate": "2022-07-25T16:07:42Z", "IsAccessPointOrder": false, "IsBusinessOrder": false, "PaymentMethodDetails": ["Standard"], "IsGlobalExpressEnabled": false, "LastUpdateDate": "2022-07-25T16:13:14Z", "ShipmentServiceLevelCategory": "Standard"}, "emitted_at": 1701969184949} +{"stream": "Orders", "data": {"BuyerInfo": {}, "AmazonOrderId": "112-3669120-1845053", "EarliestShipDate": "2022-07-15T07:00:00Z", "SalesChannel": "Amazon.com", "AutomatedShippingSettings": {"HasAutomatedShippingSettings": false}, "OrderStatus": "Canceled", "NumberOfItemsShipped": 0, "OrderType": "StandardOrder", "IsPremiumOrder": false, "IsPrime": false, "FulfillmentChannel": "MFN", "NumberOfItemsUnshipped": 0, "HasRegulatedItems": false, "IsReplacementOrder": "false", "IsSoldByAB": false, "LatestShipDate": "2022-07-16T06:59:59Z", "ShipServiceLevel": "Std US D2D Dom", "IsISPU": false, "MarketplaceId": "ATVPDKIKX0DER", "PurchaseDate": "2022-07-14T21:59:53Z", "IsAccessPointOrder": false, "IsBusinessOrder": false, "OrderTotal": {"CurrencyCode": "USD", "Amount": "10.00"}, "PaymentMethodDetails": ["Standard"], "IsGlobalExpressEnabled": false, "LastUpdateDate": "2022-07-26T07:16:14Z", "ShipmentServiceLevelCategory": "Standard"}, "emitted_at": 1701969184950} +{"stream": "Orders", "data": {"BuyerInfo": {}, "AmazonOrderId": "113-1507758-0081841", "EarliestShipDate": "2022-07-15T07:00:00Z", "SalesChannel": "Amazon.com", "AutomatedShippingSettings": {"HasAutomatedShippingSettings": false}, "OrderStatus": "Canceled", "NumberOfItemsShipped": 0, "OrderType": "StandardOrder", "IsPremiumOrder": false, "IsPrime": false, "FulfillmentChannel": "MFN", "NumberOfItemsUnshipped": 0, "HasRegulatedItems": false, "IsReplacementOrder": "false", "IsSoldByAB": false, "LatestShipDate": "2022-07-16T06:59:59Z", "ShipServiceLevel": "Std US D2D Dom", "IsISPU": false, "MarketplaceId": "ATVPDKIKX0DER", "PurchaseDate": "2022-07-14T20:22:16Z", "IsAccessPointOrder": false, "IsBusinessOrder": false, "OrderTotal": {"CurrencyCode": "USD", "Amount": "10.00"}, "PaymentMethodDetails": ["Standard"], "IsGlobalExpressEnabled": false, "LastUpdateDate": "2022-07-26T07:22:46Z", "ShipmentServiceLevelCategory": "Standard"}, "emitted_at": 1701969184950} +{"stream": "Orders", "data": {"BuyerInfo": {}, "AmazonOrderId": "113-8121041-0876267", "EarliestShipDate": "2022-07-18T07:00:00Z", "SalesChannel": "Amazon.com", "AutomatedShippingSettings": {"HasAutomatedShippingSettings": false}, "OrderStatus": "Canceled", "NumberOfItemsShipped": 0, "OrderType": "StandardOrder", "IsPremiumOrder": false, "IsPrime": false, "FulfillmentChannel": "MFN", "NumberOfItemsUnshipped": 0, "HasRegulatedItems": false, "IsReplacementOrder": "false", "IsSoldByAB": false, "LatestShipDate": "2022-07-19T06:59:59Z", "ShipServiceLevel": "Std US D2D Dom", "IsISPU": false, "MarketplaceId": "ATVPDKIKX0DER", "PurchaseDate": "2022-07-18T04:26:52Z", "IsAccessPointOrder": false, "IsBusinessOrder": false, "OrderTotal": {"CurrencyCode": "USD", "Amount": "14.00"}, "PaymentMethodDetails": ["Standard"], "IsGlobalExpressEnabled": false, "LastUpdateDate": "2022-07-28T07:22:14Z", "ShipmentServiceLevelCategory": "Standard"}, "emitted_at": 1701969184950} +{"stream": "OrderItems", "data": {"ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ASIN": "B000VHYM2E", "SellerSKU": "0R-4KDA-Z2U8", "Title": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "ConditionId": "New", "OrderItemId": "49158270219090", "LastUpdateDate": "2022-07-18T22:54:07Z", "AmazonOrderId": "111-1225255-7785053"}, "emitted_at": 1701969226265} +{"stream": "OrderItems", "data": {"ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ASIN": "B074K5MDLW", "SellerSKU": "2J-D6V7-C8XI", "Title": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "ConditionId": "New", "OrderItemId": "37736574199610", "LastUpdateDate": "2022-07-22T08:23:04Z", "AmazonOrderId": "112-3632856-2922613"}, "emitted_at": 1701969227457} +{"stream": "OrderItems", "data": {"ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ASIN": "B074K5MDLW", "SellerSKU": "2J-D6V7-C8XI", "Title": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "ConditionId": "New", "OrderItemId": "65706488326346", "LastUpdateDate": "2022-07-23T18:46:16Z", "AmazonOrderId": "113-8462063-1469066"}, "emitted_at": 1701969228659} +{"stream": "OrderItems", "data": {"ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ASIN": "B074K5MDLW", "SellerSKU": "2J-D6V7-C8XI", "Title": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "ConditionId": "New", "OrderItemId": "08960455780074", "LastUpdateDate": "2022-07-25T16:13:14Z", "AmazonOrderId": "113-3281105-7707448"}, "emitted_at": 1701969229850} +{"stream": "OrderItems", "data": {"TaxCollection": {"Model": "MarketplaceFacilitator", "ResponsibleParty": "Amazon Services, Inc."}, "ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "10.00"}, "ASIN": "B000VHYM2E", "SellerSKU": "0R-4KDA-Z2U8", "Title": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "33405118899762", "LastUpdateDate": "2022-07-26T07:16:14Z", "AmazonOrderId": "112-3669120-1845053"}, "emitted_at": 1701969231047} +{"stream": "OrderItems", "data": {"TaxCollection": {"Model": "MarketplaceFacilitator", "ResponsibleParty": "Amazon Services, Inc."}, "ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "10.00"}, "ASIN": "B000VHYM2E", "SellerSKU": "0R-4KDA-Z2U8", "Title": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "36800179130578", "LastUpdateDate": "2022-07-26T07:22:46Z", "AmazonOrderId": "113-1507758-0081841"}, "emitted_at": 1701969232250} +{"stream": "OrderItems", "data": {"TaxCollection": {"Model": "MarketplaceFacilitator", "ResponsibleParty": "Amazon Services, Inc."}, "ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "14.00"}, "ASIN": "B074K5MDLW", "SellerSKU": "2J-D6V7-C8XI", "Title": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "65043207929194", "LastUpdateDate": "2022-07-28T07:22:14Z", "AmazonOrderId": "113-8121041-0876267"}, "emitted_at": 1701969233471} +{"stream": "OrderItems", "data": {"TaxCollection": {"Model": "MarketplaceFacilitator", "ResponsibleParty": "Amazon Services, Inc."}, "ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.09"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "7.00"}, "ASIN": "B074K5MDLW", "SellerSKU": "2J-D6V7-C8XI", "Title": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "26165617935794", "LastUpdateDate": "2022-07-28T07:24:14Z", "AmazonOrderId": "114-3041148-1777835"}, "emitted_at": 1701969234665} +{"stream": "OrderItems", "data": {"ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "5.00"}, "ASIN": "B000VHYM2E", "SellerSKU": "0R-4KDA-Z2U8", "Title": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "49051239848578", "LastUpdateDate": "2022-07-28T07:42:43Z", "AmazonOrderId": "112-3720233-8146637"}, "emitted_at": 1701969235896} +{"stream": "OrderItems", "data": {"TaxCollection": {"Model": "MarketplaceFacilitator", "ResponsibleParty": "Amazon Services, Inc."}, "ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "10.00"}, "ASIN": "B000VHYM2E", "SellerSKU": "0R-4KDA-Z2U8", "Title": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "36347967018074", "LastUpdateDate": "2022-07-28T07:44:16Z", "AmazonOrderId": "111-9754278-6869864"}, "emitted_at": 1701969237108} +{"stream": "OrderItems", "data": {"TaxCollection": {"Model": "MarketplaceFacilitator", "ResponsibleParty": "Amazon Services, Inc."}, "ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "7.00"}, "ASIN": "B074K5MDLW", "SellerSKU": "2J-D6V7-C8XI", "Title": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "25578504674962", "LastUpdateDate": "2022-07-28T07:52:23Z", "AmazonOrderId": "114-4026932-3219457"}, "emitted_at": 1701969238326} +{"stream": "OrderItems", "data": {"TaxCollection": {"Model": "MarketplaceFacilitator", "ResponsibleParty": "Amazon Services, Inc."}, "ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "7.00"}, "ASIN": "B074K5MDLW", "SellerSKU": "2J-D6V7-C8XI", "Title": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "00770178005186", "LastUpdateDate": "2022-07-28T08:07:41Z", "AmazonOrderId": "112-1098428-3787449"}, "emitted_at": 1701969239527} +{"stream": "OrderItems", "data": {"TaxCollection": {"Model": "MarketplaceFacilitator", "ResponsibleParty": "Amazon Services, Inc."}, "ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "14.00"}, "ASIN": "B074K5MDLW", "SellerSKU": "2J-D6V7-C8XI", "Title": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "51815408701706", "LastUpdateDate": "2022-07-29T07:27:14Z", "AmazonOrderId": "112-8173974-4673832"}, "emitted_at": 1701969240733} +{"stream": "OrderItems", "data": {"TaxCollection": {"Model": "MarketplaceFacilitator", "ResponsibleParty": "Amazon Services, Inc."}, "ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "true", "BuyerCancelReason": "REASON_LEFT_UNSPECIFIED"}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "7.00"}, "ASIN": "B074K5MDLW", "SellerSKU": "2J-D6V7-C8XI", "Title": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "41300609058346", "LastUpdateDate": "2022-07-29T07:50:18Z", "AmazonOrderId": "114-5642155-9428269"}, "emitted_at": 1701969241937} +{"stream": "OrderItems", "data": {"TaxCollection": {"Model": "MarketplaceFacilitator", "ResponsibleParty": "Amazon Services, Inc."}, "ProductInfo": {"NumberOfItems": "1"}, "BuyerInfo": {}, "ItemTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "QuantityShipped": 0, "BuyerRequestedCancel": {"IsBuyerRequestedCancel": "false", "BuyerCancelReason": ""}, "ItemPrice": {"CurrencyCode": "USD", "Amount": "14.00"}, "ASIN": "B074K5MDLW", "SellerSKU": "2J-D6V7-C8XI", "Title": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "IsGift": "false", "ConditionSubtypeId": "New", "IsTransparency": false, "QuantityOrdered": 0, "PromotionDiscountTax": {"CurrencyCode": "USD", "Amount": "0.00"}, "ConditionId": "New", "PromotionDiscount": {"CurrencyCode": "USD", "Amount": "0.00"}, "OrderItemId": "64356568394218", "LastUpdateDate": "2022-07-29T08:19:16Z", "AmazonOrderId": "113-8871452-8288246"}, "emitted_at": 1701969243138} +{"stream": "GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT", "data": {"Country": "US", "Product Name": "Airbyte T-Shirt Black", "FNSKU": "X0041NMBPF", "Merchant SKU": "IA-VREM-8L92", "ASIN": "B0CJ5Q3NLP", "Condition": "New", "Supplier": "unassigned", "Supplier part no.": "", "Currency code": "USD", "Price": "15.00", "Sales last 30 days": "0.0", "Units Sold Last 30 Days": "0", "Total Units": "0", "Inbound": "0", "Available": "0", "FC transfer": "0", "FC Processing": "0", "Customer Order": "0", "Unfulfillable": "0", "Working": "0", "Shipped": "0", "Receiving": "0", "Fulfilled by": "Amazon", "Total Days of Supply (including units from open shipments)": "", "Days of Supply at Amazon Fulfillment Network": "", "Alert": "out_of_stock", "Recommended replenishment qty": "0", "Recommended ship date": "none", "Recommended action": "No action required", "Unit storage size": "", "dataEndTime": "2022-07-31"}, "emitted_at": 1701969512824} +{"stream": "GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT", "data": {"Country": "US", "Product Name": "Airbyte Merch White", "FNSKU": "X003X1FG67", "Merchant SKU": "KW-J7BQ-WNKL", "ASIN": "B0CDLLJ5VV", "Condition": "New", "Supplier": "unassigned", "Supplier part no.": "", "Currency code": "USD", "Price": "10.00", "Sales last 30 days": "0.0", "Units Sold Last 30 Days": "0", "Total Units": "0", "Inbound": "0", "Available": "0", "FC transfer": "0", "FC Processing": "0", "Customer Order": "0", "Unfulfillable": "0", "Working": "0", "Shipped": "0", "Receiving": "0", "Fulfilled by": "Amazon", "Total Days of Supply (including units from open shipments)": "", "Days of Supply at Amazon Fulfillment Network": "", "Alert": "out_of_stock", "Recommended replenishment qty": "0", "Recommended ship date": "none", "Recommended action": "No action required", "Unit storage size": "0.1736 ft3", "dataEndTime": "2022-07-31"}, "emitted_at": 1701969512826} +{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "19009771651", "settlement-start-date": "2023-11-13T22:51:31+00:00", "settlement-end-date": "2023-12-11T22:51:31+00:00", "deposit-date": "2023-12-13T22:51:31+00:00", "total-amount": "-39.99", "currency": "USD", "transaction-type": "", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": null, "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "", "dataEndTime": "2023-12-11"}, "emitted_at": 1705396604115} +{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "19009771651", "settlement-start-date": null, "settlement-end-date": null, "deposit-date": null, "total-amount": "", "currency": "", "transaction-type": "Payable to Amazon", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": "2023-11-13T22:51:31+00:00", "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "-39.99", "dataEndTime": "2023-12-11"}, "emitted_at": 1705396604117} +{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "19009771651", "settlement-start-date": null, "settlement-end-date": null, "deposit-date": null, "total-amount": "", "currency": "", "transaction-type": "Subscription Fee", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": "2023-12-09T20:02:53+00:00", "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "-39.99", "dataEndTime": "2023-12-11"}, "emitted_at": 1705396604118} +{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "19009771651", "settlement-start-date": null, "settlement-end-date": null, "deposit-date": null, "total-amount": "", "currency": "", "transaction-type": "Successful charge", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": "2023-11-13T23:51:01+00:00", "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "39.99 ", "dataEndTime": "2023-12-11"}, "emitted_at": 1705396604118} +{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18923842351", "settlement-start-date": "2023-10-16T22:51:31+00:00", "settlement-end-date": "2023-11-13T22:51:31+00:00", "deposit-date": "2023-11-15T22:51:31+00:00", "total-amount": "-39.99", "currency": "USD", "transaction-type": "", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": null, "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "", "dataEndTime": "2023-11-13"}, "emitted_at": 1705396605853} +{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18923842351", "settlement-start-date": null, "settlement-end-date": null, "deposit-date": null, "total-amount": "", "currency": "", "transaction-type": "Payable to Amazon", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": "2023-10-16T22:51:31+00:00", "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "-27.54", "dataEndTime": "2023-11-13"}, "emitted_at": 1705396605855} +{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18923842351", "settlement-start-date": null, "settlement-end-date": null, "deposit-date": null, "total-amount": "", "currency": "", "transaction-type": "Subscription Fee", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": "2023-11-09T18:44:35+00:00", "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "-39.99", "dataEndTime": "2023-11-13"}, "emitted_at": 1705396605856} +{"stream": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", "data": {"settlement-id": "18923842351", "settlement-start-date": null, "settlement-end-date": null, "deposit-date": null, "total-amount": "", "currency": "", "transaction-type": "Successful charge", "order-id": "", "merchant-order-id": "", "adjustment-id": "", "shipment-id": "", "marketplace-name": "", "shipment-fee-type": "", "shipment-fee-amount": "", "order-fee-type": "", "order-fee-amount": "", "fulfillment-id": "", "posted-date": "2023-10-17T00:01:09+00:00", "order-item-code": "", "merchant-order-item-id": "", "merchant-adjustment-item-id": "", "sku": "", "quantity-purchased": "", "price-type": "", "price-amount": "", "item-related-fee-type": "", "item-related-fee-amount": "", "misc-fee-amount": "", "other-fee-amount": "", "other-fee-reason-description": "", "direct-payment-type": "", "direct-payment-amount": "", "other-amount": "27.54 ", "dataEndTime": "2023-11-13"}, "emitted_at": 1705396605857} +{"stream": "GET_MERCHANT_LISTINGS_DATA_BACK_COMPAT", "data": {"item-name": "GiftBox", "item-description": "Monitor and optimize the GiftBox to reward your customers and increase the average order value", "listing-id": "0711ZJUYPNS", "seller-sku": "I0-RALD-N1UR", "price": "5", "quantity": "1000", "open-date": "2022-07-11T01:34:18-07:00", "image-url": "", "item-is-marketplace": "y", "product-id-type": "1", "zshop-shipping-fee": "", "item-note": "", "item-condition": "11", "zshop-category1": "", "zshop-browse-path": "", "zshop-storefront-feature": "", "asin1": "B0B68NBQ1Y", "asin2": "", "asin3": "", "will-ship-internationally": "", "expedited-shipping": "", "zshop-boldface": "", "product-id": "B0B68NBQ1Y", "bid-for-featured-placement": "", "add-delete": "", "pending-quantity": "0", "Business Price": "6.0", "Quantity Price Type": "", "Quantity Lower Bound 1": "", "Quantity Price 1": "", "Quantity Lower Bound 2": "", "Quantity Price 2": "", "Quantity Lower Bound 3": "", "Quantity Price 3": "", "Quantity Lower Bound 4": "", "Quantity Price 4": "", "Quantity Lower Bound 5": "", "Quantity Price 5": "", "merchant-shipping-group": "Migrated Template", "Progressive Price Type": "", "Progressive Lower Bound 1": "", "Progressive Price 1": "", "Progressive Lower Bound 2": "", "Progressive Price 2": "", "Progressive Lower Bound 3": "", "Progressive Price 3": "", "dataEndTime": "2022-07-31"}, "emitted_at": 1701976405556} +{"stream": "ListFinancialEvents", "data": {"ShipmentEventList": [], "ShipmentSettleEventList": [], "RefundEventList": [], "GuaranteeClaimEventList": [], "ChargebackEventList": [], "PayWithAmazonEventList": [], "ServiceProviderCreditEventList": [], "RetrochargeEventList": [], "RentalTransactionEventList": [], "PerformanceBondRefundEventList": [], "ProductAdsPaymentEventList": [{"postedDate": "2022-07-28T20:06:07Z", "transactionType": "Charge", "invoiceId": "TR1T7Z7DR-1", "baseValue": {"CurrencyCode": "USD", "CurrencyAmount": -9.08}, "taxValue": {"CurrencyCode": "USD", "CurrencyAmount": 0.0}, "transactionValue": {"CurrencyCode": "USD", "CurrencyAmount": -9.08}}], "ServiceFeeEventList": [], "SellerDealPaymentEventList": [], "DebtRecoveryEventList": [], "LoanServicingEventList": [], "AdjustmentEventList": [], "SAFETReimbursementEventList": [], "SellerReviewEnrollmentPaymentEventList": [], "FBALiquidationEventList": [], "CouponPaymentEventList": [], "ImagingServicesFeeEventList": [], "NetworkComminglingTransactionEventList": [], "AffordabilityExpenseEventList": [], "AffordabilityExpenseReversalEventList": [], "RemovalShipmentEventList": [], "RemovalShipmentAdjustmentEventList": [], "TrialShipmentEventList": [], "TDSReimbursementEventList": [], "AdhocDisbursementEventList": [], "TaxWithholdingEventList": [], "ChargeRefundEventList": [], "FailedAdhocDisbursementEventList": [], "ValueAddedServiceChargeEventList": [], "CapacityReservationBillingEventList": [], "PostedBefore": "2022-07-31T00:00:00Z"}, "emitted_at": 1701976465145} +{"stream": "ListFinancialEventGroups", "data": {"FinancialEventGroupId": "6uFLEEa3LQgyvcccMnVQ4Bj-I5zkOVNoM41q8leJzLk", "ProcessingStatus": "Closed", "FundTransferStatus": "Unknown", "OriginalTotal": {"CurrencyCode": "USD", "CurrencyAmount": -58.86}, "FundTransferDate": "2022-08-08T22:51:31Z", "BeginningBalance": {"CurrencyCode": "USD", "CurrencyAmount": -39.99}, "FinancialEventGroupStart": "2021-07-26T22:51:30Z", "FinancialEventGroupEnd": "2022-08-08T22:51:31Z"}, "emitted_at": 1701976502869} +{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20355628011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Vegetables - en_US", "browseNodeStoreContextName": "Vegetables - en_US", "browsePathById": "19162063011,19162064011,20355625011,20355628011", "browsePathByName": "Yggdrasil,Produce - en_US,Vegetables - en_US", "hasChildren": "true", "childNodes": {"count": "3", "id": ["20355644011", "20355643011", "20355645011"]}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}, "dataEndTime": "2022-07-31"}, "emitted_at": 1701976676487} +{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20355644011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Artichokes - en_US", "browseNodeStoreContextName": "Artichokes - en_US", "browsePathById": "19162063011,19162064011,20355625011,20355628011,20355644011", "browsePathByName": "Yggdrasil,Produce - en_US,Vegetables - en_US,Artichokes - en_US", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}, "dataEndTime": "2022-07-31"}, "emitted_at": 1701976676487} +{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20355643011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Celery - en_US", "browseNodeStoreContextName": "Celery - en_US", "browsePathById": "19162063011,19162064011,20355625011,20355628011,20355643011", "browsePathByName": "Yggdrasil,Produce - en_US,Vegetables - en_US,Celery - en_US", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}, "dataEndTime": "2022-07-31"}, "emitted_at": 1701976676487} +{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "20355645011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Eggplant - en_US", "browseNodeStoreContextName": "Eggplant - en_US", "browsePathById": "19162063011,19162064011,20355625011,20355628011,20355645011", "browsePathByName": "Yggdrasil,Produce - en_US,Vegetables - en_US,Eggplant - en_US", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}, "dataEndTime": "2022-07-31"}, "emitted_at": 1701976676487} +{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "21354445011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Test2", "browseNodeStoreContextName": "Test2", "browsePathById": "19162063011,19162064011,21354445011", "browsePathByName": "Yggdrasil,Test2", "hasChildren": "true", "childNodes": {"count": "1", "id": ["21354444011"]}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}, "dataEndTime": "2022-07-31"}, "emitted_at": 1701976676487} +{"stream": "GET_XML_BROWSE_TREE_DATA", "data": {"browseNodeId": "21354444011", "browseNodeAttributes": {"count": "0"}, "browseNodeName": "Test1", "browseNodeStoreContextName": "Test1", "browsePathById": "19162063011,19162064011,21354445011,21354444011", "browsePathByName": "Yggdrasil,Test2,Test1", "hasChildren": "false", "childNodes": {"count": "0"}, "productTypeDefinitions": null, "refinementsInformation": {"count": "0"}, "dataEndTime": "2022-07-31"}, "emitted_at": 1701976676488} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "113-8871452-8288246", "merchant-order-id": "", "purchase-date": "2022-07-18T18:52:47+00:00", "last-updated-date": "2022-07-29T08:19:16+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "14.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "KERRVILLE", "ship-state": "TX", "ship-postal-code": "78028-6411", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698682} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "114-5642155-9428269", "merchant-order-id": "", "purchase-date": "2022-07-18T17:15:20+00:00", "last-updated-date": "2022-07-29T07:50:18+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "7.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "SUNNY ISLES BEACH", "ship-state": "FL", "ship-postal-code": "33160-2404", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698685} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "112-8173974-4673832", "merchant-order-id": "", "purchase-date": "2022-07-18T19:42:56+00:00", "last-updated-date": "2022-07-29T07:27:14+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "14.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "MANSFIELD", "ship-state": "PA", "ship-postal-code": "16933-1252", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698686} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "112-1098428-3787449", "merchant-order-id": "", "purchase-date": "2022-07-17T17:49:26+00:00", "last-updated-date": "2022-07-28T08:07:41+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "7.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "Winnebago", "ship-state": "IL", "ship-postal-code": "61088", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698686} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "114-4026932-3219457", "merchant-order-id": "", "purchase-date": "2022-07-17T17:53:01+00:00", "last-updated-date": "2022-07-28T07:52:23+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "7.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "Clinton Township", "ship-state": "MI", "ship-postal-code": "48035", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698686} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "111-9754278-6869864", "merchant-order-id": "", "purchase-date": "2022-07-15T18:30:29+00:00", "last-updated-date": "2022-07-28T07:44:16+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "sku": "0R-4KDA-Z2U8", "asin": "B000VHYM2E", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "10.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "ROTONDA WEST", "ship-state": "FLORIDA", "ship-postal-code": "33947-1801", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698686} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "112-3720233-8146637", "merchant-order-id": "", "purchase-date": "2022-07-18T02:30:11+00:00", "last-updated-date": "2022-07-28T07:42:43+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "sku": "0R-4KDA-Z2U8", "asin": "B000VHYM2E", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "5.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "WINFIELD", "ship-state": "MO", "ship-postal-code": "63389-2051", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698687} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "114-3041148-1777835", "merchant-order-id": "", "purchase-date": "2022-07-18T00:32:07+00:00", "last-updated-date": "2022-07-28T07:24:14+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "7.0", "item-tax": "0.09", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "BURBANK", "ship-state": "IL", "ship-postal-code": "60459-3101", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698687} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "113-8121041-0876267", "merchant-order-id": "", "purchase-date": "2022-07-18T04:26:52+00:00", "last-updated-date": "2022-07-28T07:22:14+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "14.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "SEASIDE", "ship-state": "CA", "ship-postal-code": "93955-5450", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698687} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "113-1507758-0081841", "merchant-order-id": "", "purchase-date": "2022-07-14T20:22:16+00:00", "last-updated-date": "2022-07-26T07:22:46+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "sku": "0R-4KDA-Z2U8", "asin": "B000VHYM2E", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "10.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "PORT ARTHUR", "ship-state": "TX", "ship-postal-code": "77642-6487", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698687} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "112-3669120-1845053", "merchant-order-id": "", "purchase-date": "2022-07-14T21:59:53+00:00", "last-updated-date": "2022-07-26T07:16:14+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "sku": "0R-4KDA-Z2U8", "asin": "B000VHYM2E", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "10.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "North Andover", "ship-state": "MA", "ship-postal-code": "01845", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698688} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "113-3281105-7707448", "merchant-order-id": "", "purchase-date": "2022-07-25T16:07:42+00:00", "last-updated-date": "2022-07-25T16:13:14+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "", "item-price": "", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "NEW YORK", "ship-state": "NY", "ship-postal-code": "10023-7107", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698688} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "113-8462063-1469066", "merchant-order-id": "", "purchase-date": "2022-07-23T18:45:44+00:00", "last-updated-date": "2022-07-23T18:46:16+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "", "item-price": "", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "MARYSVILLE", "ship-state": "WA", "ship-postal-code": "98271-9030", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698688} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "112-3632856-2922613", "merchant-order-id": "", "purchase-date": "2022-07-17T07:44:26+00:00", "last-updated-date": "2022-07-22T08:23:04+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "", "item-price": "", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "BRONX", "ship-state": "NY", "ship-postal-code": "10475-4302", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698688} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", "data": {"amazon-order-id": "111-1225255-7785053", "merchant-order-id": "", "purchase-date": "2022-07-15T22:08:15+00:00", "last-updated-date": "2022-07-18T22:54:07+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "House Foods, Tofu Shirataki, Spaghetti Shaped Tofu, 8 oz", "sku": "0R-4KDA-Z2U8", "asin": "B000VHYM2E", "item-status": "", "quantity": "0", "currency": "", "item-price": "", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "TAMPA", "ship-state": "FL", "ship-postal-code": "33615-4914", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-30"}, "emitted_at": 1701956698689} +{"stream": "GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", "data": {"AmazonOrderID": "112-4470913-2725847", "PurchaseDate": "2022-07-29T08:14:41+00:00", "LastUpdatedDate": "2022-08-11T07:34:27+00:00", "OrderStatus": "Cancelled", "SalesChannel": "Amazon.com", "FulfillmentData": {"FulfillmentChannel": "Merchant", "ShipServiceLevel": "Standard", "Address": {"City": "BRONX", "State": "NY", "PostalCode": "10462-5935", "Country": "US"}}, "IsBusinessOrder": "false", "OrderItem": [{"AmazonOrderItemCode": "58620406098794", "ASIN": "B000VHRNUW", "SKU": "MP-V4RG-EDEY", "ProductName": "House Foods, Organic Firm Tofu, 14 oz", "Quantity": "0", "ItemPrice": {"Component": {"Type": "Principal", "Amount": {"currency": "USD", "value": "5.0"}}}, "SignatureConfirmationRecommended": "false"}], "dataEndTime": "2022-07-31"}, "emitted_at": 1701957188131} +{"stream": "GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", "data": {"AmazonOrderID": "112-9288908-5020240", "PurchaseDate": "2022-07-29T04:44:18+00:00", "LastUpdatedDate": "2022-08-09T07:54:14+00:00", "OrderStatus": "Cancelled", "SalesChannel": "Amazon.com", "FulfillmentData": {"FulfillmentChannel": "Merchant", "ShipServiceLevel": "Standard", "Address": {"City": "PERRY", "State": "UT", "PostalCode": "84302-4853", "Country": "US"}}, "IsBusinessOrder": "false", "OrderItem": [{"AmazonOrderItemCode": "02473315381338", "ASIN": "B074K5MDLW", "SKU": "2J-D6V7-C8XI", "ProductName": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "Quantity": "0", "ItemPrice": {"Component": [{"Type": "Principal", "Amount": {"currency": "USD", "value": "35.0"}}, {"Type": "Tax", "Amount": {"currency": "USD", "value": "1.05"}}]}, "SignatureConfirmationRecommended": "false"}], "dataEndTime": "2022-07-31"}, "emitted_at": 1701957188133} +{"stream": "GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", "data": {"AmazonOrderID": "114-6340460-9317849", "PurchaseDate": "2022-07-28T20:31:53+00:00", "LastUpdatedDate": "2022-08-09T07:44:43+00:00", "OrderStatus": "Cancelled", "SalesChannel": "Amazon.com", "FulfillmentData": {"FulfillmentChannel": "Merchant", "ShipServiceLevel": "Standard", "Address": {"City": "IRVINE", "State": "CA", "PostalCode": "92620-2213", "Country": "US"}}, "IsBusinessOrder": "false", "OrderItem": [{"AmazonOrderItemCode": "07455825901354", "ASIN": "B074K5MDLW", "SKU": "2J-D6V7-C8XI", "ProductName": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "Quantity": "0", "ItemPrice": {"Component": {"Type": "Principal", "Amount": {"currency": "USD", "value": "7.0"}}}, "SignatureConfirmationRecommended": "false"}], "dataEndTime": "2022-07-31"}, "emitted_at": 1701957188133} +{"stream": "GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", "data": {"AmazonOrderID": "114-9668619-2274637", "PurchaseDate": "2022-07-28T17:51:54+00:00", "LastUpdatedDate": "2022-08-09T07:12:17+00:00", "OrderStatus": "Cancelled", "SalesChannel": "Amazon.com", "FulfillmentData": {"FulfillmentChannel": "Merchant", "ShipServiceLevel": "Standard", "Address": {"City": "BROOKSVILLE", "State": "ME", "PostalCode": "04617-3551", "Country": "US"}}, "IsBusinessOrder": "false", "OrderItem": [{"AmazonOrderItemCode": "02835442928154", "ASIN": "B074K5MDLW", "SKU": "2J-D6V7-C8XI", "ProductName": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "Quantity": "0", "ItemPrice": {"Component": {"Type": "Principal", "Amount": {"currency": "USD", "value": "7.0"}}}, "SignatureConfirmationRecommended": "false"}], "dataEndTime": "2022-07-31"}, "emitted_at": 1701957188133} +{"stream": "GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", "data": {"AmazonOrderID": "112-1503149-6333038", "PurchaseDate": "2022-07-28T16:36:33+00:00", "LastUpdatedDate": "2022-08-09T07:36:17+00:00", "OrderStatus": "Cancelled", "SalesChannel": "Amazon.com", "FulfillmentData": {"FulfillmentChannel": "Merchant", "ShipServiceLevel": "Standard", "Address": {"City": "Woodbury", "State": "NJ", "PostalCode": "08096", "Country": "US"}}, "IsBusinessOrder": "false", "OrderItem": [{"AmazonOrderItemCode": "17238515541858", "ASIN": "B074K5MDLW", "SKU": "2J-D6V7-C8XI", "ProductName": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "Quantity": "0", "ItemPrice": {"Component": {"Type": "Principal", "Amount": {"currency": "USD", "value": "14.0"}}}, "SignatureConfirmationRecommended": "false"}], "dataEndTime": "2022-07-31"}, "emitted_at": 1701957188133} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", "data": {"amazon-order-id": "112-4470913-2725847", "merchant-order-id": "", "purchase-date": "2022-07-29T08:14:41+00:00", "last-updated-date": "2022-08-11T07:34:27+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "House Foods, Organic Firm Tofu, 14 oz", "sku": "MP-V4RG-EDEY", "asin": "B000VHRNUW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "5.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "BRONX", "ship-state": "NY", "ship-postal-code": "10462-5935", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-31"}, "emitted_at": 1701957513197} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", "data": {"amazon-order-id": "112-9288908-5020240", "merchant-order-id": "", "purchase-date": "2022-07-29T04:44:18+00:00", "last-updated-date": "2022-08-09T07:54:14+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "35.0", "item-tax": "1.05", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "PERRY", "ship-state": "UT", "ship-postal-code": "84302-4853", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-31"}, "emitted_at": 1701957513199} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", "data": {"amazon-order-id": "114-6340460-9317849", "merchant-order-id": "", "purchase-date": "2022-07-28T20:31:53+00:00", "last-updated-date": "2022-08-09T07:44:43+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "7.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "IRVINE", "ship-state": "CA", "ship-postal-code": "92620-2213", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-31"}, "emitted_at": 1701957513199} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", "data": {"amazon-order-id": "114-9668619-2274637", "merchant-order-id": "", "purchase-date": "2022-07-28T17:51:54+00:00", "last-updated-date": "2022-08-09T07:12:17+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "7.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "BROOKSVILLE", "ship-state": "ME", "ship-postal-code": "04617-3551", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-31"}, "emitted_at": 1701957513200} +{"stream": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", "data": {"amazon-order-id": "112-1503149-6333038", "merchant-order-id": "", "purchase-date": "2022-07-28T16:36:33+00:00", "last-updated-date": "2022-08-09T07:36:17+00:00", "order-status": "Cancelled", "fulfillment-channel": "Merchant", "sales-channel": "Amazon.com", "order-channel": "WebsiteOrderChannel", "ship-service-level": "Standard", "product-name": "Beyond Meat Beyond Burger Plant-Based Patties 2 pk, 8 oz (Frozen)", "sku": "2J-D6V7-C8XI", "asin": "B074K5MDLW", "item-status": "", "quantity": "0", "currency": "USD", "item-price": "14.0", "item-tax": "", "shipping-price": "", "shipping-tax": "", "gift-wrap-price": "", "gift-wrap-tax": "", "item-promotion-discount": "", "ship-promotion-discount": "", "ship-city": "Woodbury", "ship-state": "NJ", "ship-postal-code": "08096", "ship-country": "US", "promotion-ids": "", "cpf": "", "is-business-order": "false", "purchase-order-number": "", "price-designation": "", "signature-confirmation-recommended": "false", "dataEndTime": "2022-07-31"}, "emitted_at": 1701957513200} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/future_state.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/future_state.json index 7b295ecf757e..9bf0b19e4d41 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/future_state.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/future_state.json @@ -1,4 +1,521 @@ [ + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_AFN_INVENTORY_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_AFN_INVENTORY_DATA_BY_COUNTRY" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "queryEndDate": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "last-updated-date": "2121-07-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FBA_INVENTORY_PLANNING_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FBA_REIMBURSEMENTS_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FBA_SNS_FORECAST_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FBA_SNS_PERFORMANCE_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FBA_STORAGE_FEE_CHARGES_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "last-updated-date": "2121-07-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "last-updated-date": "2121-07-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "last-updated-date": "2121-07-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "GET_FLAT_FILE_ARCHIVED_ORDERS_DATA_BY_ORDER_DATE" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FLAT_FILE_OPEN_LISTINGS_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "Date": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_LEDGER_DETAIL_VIEW_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_LEDGER_SUMMARY_VIEW_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_MERCHANTS_LISTINGS_FYP_REPORT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_MERCHANT_CANCELLED_LISTINGS_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_MERCHANT_LISTINGS_ALL_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_MERCHANT_LISTINGS_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_MERCHANT_LISTINGS_DATA_BACK_COMPAT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_MERCHANT_LISTINGS_INACTIVE_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_ORDER_REPORT_DATA_SHIPPING" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "queryEndDate": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_SALES_AND_TRAFFIC_REPORT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "date": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_SELLER_FEEDBACK_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_STRANDED_INVENTORY_UI_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_VENDOR_INVENTORY_REPORT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_VENDOR_REAL_TIME_INVENTORY_REPORT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_VENDOR_SALES_REPORT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_VENDOR_TRAFFIC_REPORT" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "LastUpdatedDate": "2121-07-01T00:00:00+00:00" + }, + "stream_descriptor": { + "name": "GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "dataEndTime": "2121-07-01" + }, + "stream_descriptor": { + "name": "GET_XML_BROWSE_TREE_DATA" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "FinancialEventGroupStart": "2121-07-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "ListFinancialEventGroups" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "PostedBefore": "2121-07-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "ListFinancialEvents" + } + } + }, { "type": "STREAM", "stream": { @@ -20,5 +537,16 @@ "name": "Orders" } } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "createdBefore": "2121-07-01T00:00:00Z" + }, + "stream_descriptor": { + "name": "VendorDirectFulfillmentShipping" + } + } } ] diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/sample_state.json index bebd0fa00a49..fe62f0528bcf 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/sample_state.json @@ -11,9 +11,6 @@ "GET_MERCHANT_LISTINGS_ALL_DATA": { "createdTime": "2021-07-01T00:00:00Z" }, - "GET_FBA_INVENTORY_AGED_DATA": { - "createdTime": "2021-07-01T00:00:00Z" - }, "GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL": { "createdTime": "2021-07-01T00:00:00Z" }, @@ -41,24 +38,9 @@ "GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA": { "createdTime": "2021-07-01T00:00:00Z" }, - "GET_FBA_FULFILLMENT_CURRENT_INVENTORY_DATA": { - "createdTime": "2021-07-01T00:00:00Z" - }, "GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA": { "createdTime": "2021-07-01T00:00:00Z" }, - "GET_FBA_FULFILLMENT_INVENTORY_ADJUSTMENTS_DATA": { - "createdTime": "2021-07-01T00:00:00Z" - }, - "GET_FBA_FULFILLMENT_INVENTORY_RECEIPTS_DATA": { - "createdTime": "2021-07-01T00:00:00Z" - }, - "GET_FBA_FULFILLMENT_INVENTORY_SUMMARY_DATA": { - "createdTime": "2021-07-01T00:00:00Z" - }, - "GET_FBA_FULFILLMENT_MONTHLY_INVENTORY_DATA": { - "createdTime": "2021-07-01T00:00:00Z" - }, "GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA": { "createdTime": "2021-07-01T00:00:00Z" }, diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/spec.json b/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/spec.json deleted file mode 100644 index 9c8e32370a3e..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/integration_tests/spec.json +++ /dev/null @@ -1,200 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/amazon-seller-partner", - "changelogUrl": "https://docs.airbyte.com/integrations/sources/amazon-seller-partner", - "connectionSpecification": { - "title": "Amazon Seller Partner Spec", - "type": "object", - "required": [ - "aws_environment", - "region", - "lwa_app_id", - "lwa_client_secret", - "refresh_token", - "replication_start_date" - ], - "additionalProperties": true, - "properties": { - "auth_type": { - "title": "Auth Type", - "const": "oauth2.0", - "order": 0, - "type": "string" - }, - "aws_environment": { - "title": "AWS Environment", - "description": "Select the AWS Environment.", - "enum": ["PRODUCTION", "SANDBOX"], - "default": "PRODUCTION", - "type": "string", - "order": 1 - }, - "region": { - "title": "AWS Region", - "description": "Select the AWS Region.", - "enum": [ - "AE", - "AU", - "BE", - "BR", - "CA", - "DE", - "EG", - "ES", - "FR", - "GB", - "IN", - "IT", - "JP", - "MX", - "NL", - "PL", - "SA", - "SE", - "SG", - "TR", - "UK", - "US" - ], - "default": "US", - "type": "string", - "order": 2 - }, - "aws_access_key": { - "title": "AWS Access Key", - "description": "Specifies the AWS access key used as part of the credentials to authenticate the user.", - "airbyte_secret": true, - "order": 3, - "type": "string" - }, - "aws_secret_key": { - "title": "AWS Secret Access Key", - "description": "Specifies the AWS secret key used as part of the credentials to authenticate the user.", - "airbyte_secret": true, - "order": 4, - "type": "string" - }, - "role_arn": { - "title": "Role ARN", - "description": "Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations requested using this profile. (Needs permission to 'Assume Role' STS).", - "airbyte_secret": true, - "order": 5, - "type": "string" - }, - "lwa_app_id": { - "title": "LWA Client Id", - "description": "Your Login with Amazon Client ID.", - "order": 6, - "airbyte_secret": true, - "type": "string" - }, - "lwa_client_secret": { - "title": "LWA Client Secret", - "description": "Your Login with Amazon Client Secret.", - "airbyte_secret": true, - "order": 7, - "type": "string" - }, - "refresh_token": { - "title": "Refresh Token", - "description": "The Refresh Token obtained via OAuth flow authorization.", - "airbyte_secret": true, - "order": 8, - "type": "string" - }, - "replication_start_date": { - "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2017-01-25T00:00:00Z"], - "order": 9, - "type": "string" - }, - "replication_end_date": { - "title": "End Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data after this date will not be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$|^$", - "examples": ["2017-01-25T00:00:00Z"], - "order": 10, - "type": "string" - }, - "period_in_days": { - "title": "Period In Days", - "type": "integer", - "description": "Will be used for stream slicing for initial full_refresh sync when no updated state is present for reports that support sliced incremental sync.", - "default": 90, - "order": 11 - }, - "report_options": { - "title": "Report Options", - "description": "Additional information passed to reports. This varies by report type. Must be a valid json string.", - "examples": [ - "{\"GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT\": {\"reportPeriod\": \"WEEK\"}}", - "{\"GET_SOME_REPORT\": {\"custom\": \"true\"}}" - ], - "order": 12, - "type": "string" - }, - "max_wait_seconds": { - "title": "Max wait time for reports (in seconds)", - "description": "Sometimes report can take up to 30 minutes to generate. This will set the limit for how long to wait for a successful report.", - "default": 500, - "examples": ["500", "1980"], - "order": 13, - "type": "integer" - }, - "advanced_stream_options": { - "title": "Advanced Stream Options", - "description": "Additional information to configure report options. This varies by report type, not every report implement this kind of feature. Must be a valid json string.", - "examples": [ - "{\"GET_SALES_AND_TRAFFIC_REPORT\": {\"availability_sla_days\": 3}}", - "{\"GET_SOME_REPORT\": {\"custom\": \"true\"}}" - ], - "order": 14, - "type": "string" - } - } - }, - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "refresh_token": { - "type": "string", - "path_in_connector_config": ["refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "lwa_app_id": { - "type": "string" - }, - "lwa_client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "lwa_app_id": { - "type": "string", - "path_in_connector_config": ["lwa_app_id"] - }, - "lwa_client_secret": { - "type": "string", - "path_in_connector_config": ["lwa_client_secret"] - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/main.py b/airbyte-integrations/connectors/source-amazon-seller-partner/main.py index a09a9063026c..d53252191baf 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/main.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/main.py @@ -7,7 +7,10 @@ from airbyte_cdk.entrypoint import launch from source_amazon_seller_partner import SourceAmazonSellerPartner +from source_amazon_seller_partner.config_migrations import MigrateAccountType, MigrateReportOptions if __name__ == "__main__": source = SourceAmazonSellerPartner() + MigrateAccountType.migrate(sys.argv[1:], source) + MigrateReportOptions.migrate(sys.argv[1:], source) launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml index 1e981bf36a47..e6a6b883ff35 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/metadata.yaml @@ -1,11 +1,21 @@ data: + allowedHosts: + hosts: + - https://sellingpartnerapi-eu.amazon.com + - https://sellingpartnerapi-fe.amazon.com + - https://sellingpartnerapi-na.amazon.com + - https://sandbox.sellingpartnerapi-eu.amazon.com + - https://sandbox.sellingpartnerapi-fe.amazon.com + - https://sandbox.sellingpartnerapi-na.amazon.com ab_internal: - ql: 200 - sl: 100 + ql: 400 + sl: 200 + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: e55879a8-0ef8-4557-abcf-ab34c53ec460 - dockerImageTag: 1.5.1 + dockerImageTag: 3.1.0 dockerRepository: airbyte/source-amazon-seller-partner documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-seller-partner githubIssueLabel: source-amazon-seller-partner @@ -18,6 +28,24 @@ data: oss: enabled: true releaseStage: alpha + suggestedStreams: + streams: + - Orders + - OrderItems + - ListFinancialEvents + - ListFinancialEventGroups + releases: + breakingChanges: + 2.0.0: + message: "Deprecated FBA reports will be removed permanently from Cloud and Brand Analytics Reports will be removed temporarily. Updates on Brand Analytics Reports can be tracked here: [#32353](https://github.com/airbytehq/airbyte/issues/32353)" + upgradeDeadline: "2023-12-11" + 3.0.0: + message: + Streams 'GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL' and 'GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL' now have updated schemas. + Streams 'GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL', 'GET_LEDGER_DETAIL_VIEW_DATA', 'GET_MERCHANTS_LISTINGS_FYP_REPORT', + 'GET_STRANDED_INVENTORY_UI_DATA', and 'GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE' now have date-time formatted fields. + Users will need to refresh the source schemas and reset these streams after upgrading. + upgradeDeadline: "2024-01-12" supportLevel: community tags: - language:python diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py b/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py index 2e5ca2d1e6d5..8a171d3ee035 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/setup.py @@ -5,13 +5,9 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "boto3~=1.16", "pendulum~=2.1", "pycryptodome~=3.10", "xmltodict~=0.12"] +MAIN_REQUIREMENTS = ["airbyte-cdk", "xmltodict~=0.12", "dateparser==1.2.0"] -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest~=6.1", - "pytest-mock", -] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock"] setup( name="source_amazon_seller_partner", diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py index 26b0dd29ae58..4bdaab19df35 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/auth.py @@ -2,16 +2,11 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import hashlib -import hmac -import urllib.parse + from typing import Any, Mapping -from urllib.parse import urlparse import pendulum -import requests from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator -from requests.auth import AuthBase class AWSAuthenticator(Oauth2Authenticator): @@ -27,82 +22,3 @@ def get_auth_header(self) -> Mapping[str, Any]: "x-amz-access-token": self.get_access_token(), "x-amz-date": pendulum.now("utc").strftime("%Y%m%dT%H%M%SZ"), } - - -class AWSSignature(AuthBase): - """Source from https://github.com/saleweaver/python-amazon-sp-api/blob/master/sp_api/base/aws_sig_v4.py""" - - def __init__(self, service: str, aws_access_key_id: str, aws_secret_access_key: str, aws_session_token: str, region: str): - self.service = service - self.aws_access_key_id = aws_access_key_id - self.aws_secret_access_key = aws_secret_access_key - self.aws_session_token = aws_session_token - self.region = region - - @staticmethod - def _sign_msg(key: bytes, msg: str) -> bytes: - """Sign message using key""" - return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest() - - def _get_authorization_header(self, prepared_request: requests.PreparedRequest) -> str: - current_ts = pendulum.now("utc") - url_parsed = urlparse(prepared_request.url) - uri = urllib.parse.quote(url_parsed.path) - host = url_parsed.hostname - - amz_date = current_ts.strftime("%Y%m%dT%H%M%SZ") - datestamp = current_ts.strftime("%Y%m%d") - - # sort query parameters alphabetically - if len(url_parsed.query) > 0: - split_query_parameters = list(map(lambda param: param.split("="), url_parsed.query.split("&"))) - ordered_query_parameters = sorted(split_query_parameters, key=lambda param: (param[0], param[1])) - else: - ordered_query_parameters = list() - - canonical_querystring = "&".join(map(lambda param: "=".join(param), ordered_query_parameters)) - - headers_to_sign = {"host": host, "x-amz-date": amz_date} - if self.aws_session_token: - headers_to_sign["x-amz-security-token"] = self.aws_session_token - - ordered_headers = dict(sorted(headers_to_sign.items(), key=lambda h: h[0])) - canonical_headers = "".join(map(lambda h: ":".join(h) + "\n", ordered_headers.items())) - signed_headers = ";".join(ordered_headers.keys()) - - if prepared_request.method == "GET": - payload_hash = hashlib.sha256("".encode("utf-8")).hexdigest() - else: - if prepared_request.body: - payload_hash = hashlib.sha256(prepared_request.body.encode("utf-8")).hexdigest() - else: - payload_hash = hashlib.sha256("".encode("utf-8")).hexdigest() - - canonical_request = "\n".join( - [prepared_request.method, uri, canonical_querystring, canonical_headers, signed_headers, payload_hash] - ) - - credential_scope = "/".join([datestamp, self.region, self.service, "aws4_request"]) - string_to_sign = "\n".join( - ["AWS4-HMAC-SHA256", amz_date, credential_scope, hashlib.sha256(canonical_request.encode("utf-8")).hexdigest()] - ) - - datestamp_signed = self._sign_msg(("AWS4" + self.aws_secret_access_key).encode("utf-8"), datestamp) - region_signed = self._sign_msg(datestamp_signed, self.region) - service_signed = self._sign_msg(region_signed, self.service) - aws4_request_signed = self._sign_msg(service_signed, "aws4_request") - signature = hmac.new(aws4_request_signed, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest() - - authorization_header = "AWS4-HMAC-SHA256 Credential={}/{}, SignedHeaders={}, Signature={}".format( - self.aws_access_key_id, credential_scope, signed_headers, signature - ) - return authorization_header - - def __call__(self, prepared_request: requests.PreparedRequest) -> requests.PreparedRequest: - prepared_request.headers.update( - { - "authorization": self._get_authorization_header(prepared_request), - "x-amz-security-token": self.aws_session_token, - } - ) - return prepared_request diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/config_migrations.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/config_migrations.py new file mode 100644 index 000000000000..0267e3af07a1 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/config_migrations.py @@ -0,0 +1,150 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import json +import logging +from typing import Any, List, Mapping + +from airbyte_cdk.config_observation import create_connector_config_control_message +from airbyte_cdk.entrypoint import AirbyteEntrypoint +from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + +from .source import SourceAmazonSellerPartner + +logger = logging.getLogger("airbyte_logger") + + +class MigrateAccountType: + """ + This class stands for migrating the config at runtime, + while providing the backward compatibility when falling back to the previous source version. + + Specifically, starting from `2.0.1`, the `account_type` property becomes required. + For those connector configs that do not contain this key, the default value of `Seller` will be used. + Reverse operation is not needed as this field is ignored in previous versions of the connector. + """ + + message_repository: MessageRepository = InMemoryMessageRepository() + migration_key: str = "account_type" + + @classmethod + def _should_migrate(cls, config: Mapping[str, Any]) -> bool: + """ + This method determines whether config requires migration. + Returns: + > True, if the transformation is necessary + > False, otherwise. + """ + return cls.migration_key not in config + + @classmethod + def _populate_with_default_value(cls, config: Mapping[str, Any], source: SourceAmazonSellerPartner = None) -> Mapping[str, Any]: + config[cls.migration_key] = "Seller" + return config + + @classmethod + def _modify_and_save(cls, config_path: str, source: SourceAmazonSellerPartner, config: Mapping[str, Any]) -> Mapping[str, Any]: + # modify the config + migrated_config = cls._populate_with_default_value(config, source) + # save the config + source.write_config(migrated_config, config_path) + # return modified config + return migrated_config + + @classmethod + def _emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: + # add the Airbyte Control Message to message repo + cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) + # emit the Airbyte Control Message from message queue to stdout + for message in cls.message_repository.consume_queue(): + print(message.json(exclude_unset=True)) + + @classmethod + def migrate(cls, args: List[str], source: SourceAmazonSellerPartner) -> None: + """ + This method checks the input args, should the config be migrated, + transform if necessary and emit the CONTROL message. + """ + # get config path + config_path = AirbyteEntrypoint(source).extract_config(args) + # proceed only if `--config` arg is provided + if config_path: + # read the existing config + config = source.read_config(config_path) + # migration check + if cls._should_migrate(config): + cls._emit_control_message(cls._modify_and_save(config_path, source, config)) + + +class MigrateReportOptions: + """ + This class stands for migrating the config at runtime, + while providing the backward compatibility when falling back to the previous source version. + + Specifically, starting from `2.0.1`, the `account_type` property becomes required. + For those connector configs that do not contain this key, the default value of `Seller` will be used. + Reverse operation is not needed as this field is ignored in previous versions of the connector. + """ + + message_repository: MessageRepository = InMemoryMessageRepository() + migration_key: str = "report_options_list" + + @classmethod + def _should_migrate(cls, config: Mapping[str, Any]) -> bool: + """ + This method determines whether config requires migration. + Returns: + > True, if the transformation is necessary + > False, otherwise. + """ + return cls.migration_key not in config and (config.get("report_options") or config.get("advanced_stream_options")) + + @classmethod + def _transform_report_options(cls, config: Mapping[str, Any]) -> Mapping[str, Any]: + try: + report_options = json.loads(config.get("report_options", "{}") or "{}") + except json.JSONDecodeError: + report_options = {} + + report_options_list = [] + for stream_name, options in report_options.items(): + options_list = [{"option_name": name, "option_value": value} for name, value in options.items()] + report_options_list.append({"stream_name": stream_name, "options_list": options_list}) + + config[cls.migration_key] = report_options_list + return config + + @classmethod + def _modify_and_save(cls, config_path: str, source: SourceAmazonSellerPartner, config: Mapping[str, Any]) -> Mapping[str, Any]: + # modify the config + migrated_config = cls._transform_report_options(config) + # save the config + source.write_config(migrated_config, config_path) + # return modified config + return migrated_config + + @classmethod + def _emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: + # add the Airbyte Control Message to message repo + cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) + # emit the Airbyte Control Message from message queue to stdout + for message in cls.message_repository.consume_queue(): + print(message.json(exclude_unset=True)) + + @classmethod + def migrate(cls, args: List[str], source: SourceAmazonSellerPartner) -> None: + """ + This method checks the input args, should the config be migrated, + transform if necessary and emit the CONTROL message. + """ + # get config path + config_path = AirbyteEntrypoint(source).extract_config(args) + # proceed only if `--config` arg is provided + if config_path: + # read the existing config + config = source.read_config(config_path) + # migration check + if cls._should_migrate(config): + cls._emit_control_message(cls._modify_and_save(config_path, source, config)) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/constants.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/constants.py index e8dc0e56c1c8..4b0e1e99bbe5 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/constants.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/constants.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + """ Country marketplaceId Country code Canada A2EUQ1WTGCTBG2 CA diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AFN_INVENTORY_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AFN_INVENTORY_DATA.json index 4823ae4db8dc..5e771d793471 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AFN_INVENTORY_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AFN_INVENTORY_DATA.json @@ -9,6 +9,7 @@ "asin": { "type": ["null", "string"] }, "condition-type": { "type": ["null", "string"] }, "Warehouse-Condition-code": { "type": ["null", "string"] }, - "Quantity Available": { "type": ["null", "number"] } + "Quantity Available": { "type": ["null", "string"] }, + "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AFN_INVENTORY_DATA_BY_COUNTRY.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AFN_INVENTORY_DATA_BY_COUNTRY.json index 984f466bebc4..24c7bdabbfc9 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AFN_INVENTORY_DATA_BY_COUNTRY.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AFN_INVENTORY_DATA_BY_COUNTRY.json @@ -9,6 +9,7 @@ "asin": { "type": ["null", "string"] }, "condition-type": { "type": ["null", "string"] }, "country": { "type": ["null", "string"] }, - "quantity-for-local-fulfillment": { "type": ["null", "number"] } + "quantity-for-local-fulfillment": { "type": ["null", "string"] }, + "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL.json index d2e8b0e3c036..e04864960619 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL.json @@ -10,10 +10,10 @@ "shipment-item-id": { "type": ["null", "string"] }, "amazon-order-item-id": { "type": ["null", "string"] }, "merchant-order-item-id": { "type": ["null", "string"] }, - "purchase-date": { "type": ["null", "string"] }, - "payments-date": { "type": ["null", "string"] }, - "shipment-date": { "type": ["null", "string"] }, - "reporting-date": { "type": ["null", "string"] }, + "purchase-date": { "type": ["null", "string"], "format": "date-time" }, + "payments-date": { "type": ["null", "string"], "format": "date-time" }, + "shipment-date": { "type": ["null", "string"], "format": "date-time" }, + "reporting-date": { "type": ["null", "string"], "format": "date-time" }, "buyer-email": { "type": ["null", "string"] }, "buyer-name": { "type": ["null", "string"] }, "buyer-phone-number": { "type": ["null", "string"] }, @@ -48,9 +48,13 @@ "ship-promotion-discount": { "type": ["null", "string"] }, "carrier": { "type": ["null", "string"] }, "tracking-number": { "type": ["null", "string"] }, - "estimated-arrival-date": { "type": ["null", "string"] }, + "estimated-arrival-date": { + "type": ["null", "string"], + "format": "date-time" + }, "fulfillment-center-id": { "type": ["null", "string"] }, "fulfillment-channel": { "type": ["null", "string"] }, - "sales-channel": { "type": ["null", "string"] } + "sales-channel": { "type": ["null", "string"] }, + "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_ALTERNATE_PURCHASE_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_ALTERNATE_PURCHASE_REPORT.json deleted file mode 100644 index 84b78bc29385..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_ALTERNATE_PURCHASE_REPORT.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "title": "Brand Analytics Alternate Purchase Reports", - "description": "Brand Analytics Alternate Purchase Reports", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "startDate": { - "type": ["null", "string"], - "format": "date" - }, - "endDate": { - "type": ["null", "string"], - "format": "date" - }, - "asin": { - "type": ["null", "string"] - }, - "purchasedAsin": { - "type": ["null", "string"] - }, - "purchasedRank": { - "type": ["null", "integer"] - }, - "purchasedPct": { - "type": ["null", "number"] - } - } -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_ITEM_COMPARISON_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_ITEM_COMPARISON_REPORT.json deleted file mode 100644 index cae42a1150bb..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_ITEM_COMPARISON_REPORT.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "title": "Brand Analytics Item Comparison Reports", - "description": "Brand Analytics Item Comparison Reports", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "startDate": { - "type": ["null", "string"], - "format": "date" - }, - "endDate": { - "type": ["null", "string"], - "format": "date" - }, - "asin": { - "type": ["null", "string"] - }, - "comparedAsin": { - "type": ["null", "string"] - }, - "comparedRank": { - "type": ["null", "integer"] - }, - "comparedPct": { - "type": ["null", "number"] - } - } -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT.json index 88473f308b03..4ba25d9eefa1 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT.json @@ -23,6 +23,14 @@ }, "combinationPct": { "type": ["null", "number"] + }, + "dataEndTime": { + "type": ["null", "string"], + "format": "date" + }, + "queryEndDate": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT.json index 46da2d4f0307..06f26422fd69 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT.json @@ -37,6 +37,14 @@ }, "repeatPurchaseRevenuePctTotal": { "type": ["null", "number"] + }, + "dataEndTime": { + "type": ["null", "string"], + "format": "date" + }, + "queryEndDate": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT.json index 12ca9e762138..65880173e0dc 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT.json @@ -24,6 +24,14 @@ }, "conversionShare": { "type": ["null", "number"] + }, + "dataEndTime": { + "type": ["null", "string"], + "format": "date" + }, + "queryEndDate": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA.json index af433395dc9f..c0f077c37eaf 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA.json @@ -43,6 +43,10 @@ "estimated-future-referral-fee-per-unit": { "type": ["null", "string"] }, "current-fee-category": { "type": ["null", "string"] }, "future-fee-category": { "type": ["null", "string"] }, - "future-fee-category-effective-date": { "type": ["null", "string"] } + "future-fee-category-effective-date": { + "type": ["null", "string"], + "format": "date-time" + }, + "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_CURRENT_INVENTORY_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_CURRENT_INVENTORY_DATA.json deleted file mode 100644 index 401cbf484380..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_CURRENT_INVENTORY_DATA.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "title": "FBA Daily Inventory History Report", - "description": "", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "snapshot-date": { "type": ["null", "string"] }, - "fnsku": { "type": ["null", "string"] }, - "sku": { "type": ["null", "string"] }, - "product-name": { "type": ["null", "string"] }, - "quantity": { "type": ["null", "string"] }, - "fulfillment-center-id": { "type": ["null", "string"] }, - "detailed-disposition": { "type": ["null", "string"] }, - "country": { "type": ["null", "string"] } - } -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA.json index f00915ad3993..fa8a94fdbbe5 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA.json @@ -4,7 +4,7 @@ "type": "object", "$schema": "http://json-schema.org/draft-07/schema#", "properties": { - "return-date": { "type": ["null", "string"] }, + "return-date": { "type": ["null", "string"], "format": "date-time" }, "order-id": { "type": ["null", "string"] }, "sku": { "type": ["null", "string"] }, "asin": { "type": ["null", "string"] }, @@ -16,6 +16,7 @@ "reason": { "type": ["null", "string"] }, "status": { "type": ["null", "string"] }, "license-plate-number": { "type": ["null", "string"] }, - "customer-comments": { "type": ["null", "string"] } + "customer-comments": { "type": ["null", "string"] }, + "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA.json index 610e71ce9ac6..3e1a2056a456 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA.json @@ -4,7 +4,7 @@ "type": "object", "$schema": "http://json-schema.org/draft-07/schema#", "properties": { - "shipment-date": { "type": ["null", "string"] }, + "shipment-date": { "type": ["null", "string"], "format": "date-time" }, "currency": { "type": ["null", "string"] }, "item-promotion-discount": { "type": ["null", "string"] }, "item-promotion-id": { "type": ["null", "string"] }, @@ -12,6 +12,7 @@ "promotion-rule-value": { "type": ["null", "string"] }, "amazon-order-id": { "type": ["null", "string"] }, "shipment-id": { "type": ["null", "string"] }, - "shipment-item-id": { "type": ["null", "string"] } + "shipment-item-id": { "type": ["null", "string"] }, + "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA.json index 7dece42b15b7..f3471e9602af 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA.json @@ -49,6 +49,10 @@ "title": "Original Customer Order ID", "description": "Order ID of original shipment", "type": ["null", "string"] + }, + "dataEndTime": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_INVENTORY_ADJUSTMENTS_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_INVENTORY_ADJUSTMENTS_DATA.json deleted file mode 100644 index 916f932cc057..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_INVENTORY_ADJUSTMENTS_DATA.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "title": "FBA Inventory Adjustments Report", - "description": "", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "adjusted-date": { "type": ["null", "string"] }, - "transaction-item-id": { "type": ["null", "string"] }, - "fnsku": { "type": ["null", "string"] }, - "sku": { "type": ["null", "string"] }, - "product-name": { "type": ["null", "string"] }, - "fulfillment-center-id": { "type": ["null", "string"] }, - "quantity": { "type": ["null", "string"] }, - "reason": { "type": ["null", "string"] }, - "disposition": { "type": ["null", "string"] }, - "reconciled": { "type": ["null", "string"] }, - "unreconciled": { "type": ["null", "string"] } - } -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_INVENTORY_RECEIPTS_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_INVENTORY_RECEIPTS_DATA.json deleted file mode 100644 index 3d23369d51e1..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_INVENTORY_RECEIPTS_DATA.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "title": "FBA Received Inventory Report", - "description": "", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "received-date": { "type": ["null", "string"] }, - "fnsku": { "type": ["null", "string"] }, - "sku": { "type": ["null", "string"] }, - "product-name": { "type": ["null", "string"] }, - "quantity": { "type": ["null", "string"] }, - "fba-shipment-id": { "type": ["null", "string"] }, - "fulfillment-center-id": { "type": ["null", "string"] } - } -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_INVENTORY_SUMMARY_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_INVENTORY_SUMMARY_DATA.json deleted file mode 100644 index 1ddf4fceca59..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_INVENTORY_SUMMARY_DATA.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "title": "FBA Inventory Event Detail Report", - "description": "", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "snapshot-date": { "type": ["null", "string"] }, - "transaction-type": { "type": ["null", "string"] }, - "fnsku": { "type": ["null", "string"] }, - "sku": { "type": ["null", "string"] }, - "product-name": { "type": ["null", "string"] }, - "fulfillment-center-id": { "type": ["null", "string"] }, - "quantity": { "type": ["null", "string"] }, - "disposition": { "type": ["null", "string"] } - } -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_MONTHLY_INVENTORY_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_MONTHLY_INVENTORY_DATA.json deleted file mode 100644 index 796985c5210e..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_MONTHLY_INVENTORY_DATA.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "title": "FBA Monthly Inventory History Report", - "description": "", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "month": { "type": ["null", "string"] }, - "fnsku": { "type": ["null", "string"] }, - "sku": { "type": ["null", "string"] }, - "product-name": { "type": ["null", "string"] }, - "average-quantity": { "type": ["null", "string"] }, - "end-quantity": { "type": ["null", "string"] }, - "fulfillment-center-id": { "type": ["null", "string"] }, - "detailed-disposition": { "type": ["null", "string"] }, - "country": { "type": ["null", "string"] } - } -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA.json index 7da807eac263..05a3aa6d3b34 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA.json @@ -5,7 +5,8 @@ "$schema": "http://json-schema.org/draft-07/schema#", "properties": { "request-date": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "order-id": { "type": ["null", "string"] @@ -17,7 +18,8 @@ "type": ["null", "string"] }, "last-updated-date": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "sku": { "type": ["null", "string"] @@ -48,6 +50,10 @@ }, "currency": { "type": ["null", "string"] + }, + "dataEndTime": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA.json index f31a80bd0d1e..8dfea3ca7dc7 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA.json @@ -5,13 +5,15 @@ "$schema": "http://json-schema.org/draft-07/schema#", "properties": { "removal-date": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "order-id": { "type": ["null", "string"] }, "shipment-date": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "sku": { "type": ["null", "string"] @@ -30,6 +32,10 @@ }, "tracking-number": { "type": ["null", "string"] + }, + "dataEndTime": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_INVENTORY_AGED_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_INVENTORY_AGED_DATA.json deleted file mode 100644 index 430869075c75..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_INVENTORY_AGED_DATA.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "title": "FBA Inventory Aged Data Reports", - "description": "FBA Inventory Aged Data Reports", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "sku": { - "type": ["null", "string"] - }, - "fnsku": { - "type": ["null", "string"] - }, - "asin": { - "type": ["null", "string"] - }, - "product-name": { - "type": ["null", "string"] - }, - "condition": { - "type": ["null", "string"] - }, - "your-price": { - "type": ["null", "string"] - }, - "mfn-listing-exists": { - "type": ["null", "string"] - }, - "mfn-fulfillable-quantity": { - "type": ["null", "string"] - }, - "afn-listing-exists": { - "type": ["null", "string"] - }, - "afn-warehouse-quantity": { - "type": ["null", "string"] - }, - "afn-fulfillable-quantity": { - "type": ["null", "string"] - }, - "afn-unsellable-quantity": { - "type": ["null", "string"] - }, - "afn-reserved-quantity": { - "type": ["null", "string"] - }, - "afn-total-quantity": { - "type": ["null", "string"] - }, - "per-unit-volume": { - "type": ["null", "string"] - }, - "afn-inbound-working-quantity": { - "type": ["null", "string"] - }, - "afn-inbound-shipped-quantity": { - "type": ["null", "string"] - }, - "afn-inbound-receiving-quantity": { - "type": ["null", "string"] - }, - "afn-future-supply-buyable": { - "type": ["null", "string"] - }, - "afn-reserved-future-supply": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_INVENTORY_PLANNING_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_INVENTORY_PLANNING_DATA.json index 30ead73eab60..a2badbc91ccd 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_INVENTORY_PLANNING_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_INVENTORY_PLANNING_DATA.json @@ -4,7 +4,7 @@ "type": "object", "$schema": "http://json-schema.org/draft-07/schema#", "properties": { - "snapshot-date": { "type": ["null", "string"] }, + "snapshot-date": { "type": ["null", "string"], "format": "date-time" }, "sku": { "type": ["null", "string"] }, "fnsku": { "type": ["null", "string"] }, "asin": { "type": ["null", "string"] }, @@ -68,6 +68,21 @@ "inbound-received": { "type": ["null", "string"] }, "no-sale-last-6-months": { "type": ["null", "string"] }, "reserved-quantity": { "type": ["null", "string"] }, - "unfulfillable-quantity": { "type": ["null", "string"] } + "unfulfillable-quantity": { "type": ["null", "string"] }, + "estimated-ais-181-210-days": { "type": ["null", "number"] }, + "estimated-ais-211-240-days": { "type": ["null", "number"] }, + "estimated-ais-241-270-days": { "type": ["null", "number"] }, + "estimated-ais-271-300-days": { "type": ["null", "number"] }, + "estimated-ais-301-330-days": { "type": ["null", "number"] }, + "estimated-ais-331-365-days": { "type": ["null", "number"] }, + "estimated-ais-365-plus-days": { "type": ["null", "number"] }, + "quantity-to-be-charged-ais-181-210-days": { "type": ["null", "integer"] }, + "quantity-to-be-charged-ais-211-240-days": { "type": ["null", "integer"] }, + "quantity-to-be-charged-ais-241-270-days": { "type": ["null", "integer"] }, + "quantity-to-be-charged-ais-271-300-days": { "type": ["null", "integer"] }, + "quantity-to-be-charged-ais-301-330-days": { "type": ["null", "integer"] }, + "quantity-to-be-charged-ais-331-365-days": { "type": ["null", "integer"] }, + "quantity-to-be-charged-ais-365-plus-days": { "type": ["null", "integer"] }, + "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA.json index 77d1c2d6c9f0..3cb4214d7253 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA.json @@ -24,6 +24,7 @@ "afn-inbound-receiving-quantity": { "type": ["null", "string"] }, "afn-researching-quantity": { "type": ["null", "string"] }, "afn-reserved-future-supply": { "type": ["null", "string"] }, - "afn-future-supply-buyable": { "type": ["null", "string"] } + "afn-future-supply-buyable": { "type": ["null", "string"] }, + "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_REIMBURSEMENTS_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_REIMBURSEMENTS_DATA.json index 0c48c8651a11..8af9cf77362f 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_REIMBURSEMENTS_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_REIMBURSEMENTS_DATA.json @@ -6,7 +6,8 @@ "additionalProperties": true, "properties": { "approval-date": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "reimbursement-id": { "type": ["null", "string"] @@ -58,6 +59,10 @@ }, "original-reimbursement-type": { "type": ["null", "string"] + }, + "dataEndTime": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_SNS_FORECAST_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_SNS_FORECAST_DATA.json index d47ec26a178e..2a6e3e01602f 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_SNS_FORECAST_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_SNS_FORECAST_DATA.json @@ -5,7 +5,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "properties": { "offer-state": { "type": ["null", "string"] }, - "snapshot-date": { "type": ["null", "string"] }, + "snapshot-date": { "type": ["null", "string"], "format": "date-time" }, "sku": { "type": ["null", "string"] }, "fnsku": { "type": ["null", "string"] }, "asin": { "type": ["null", "string"] }, @@ -13,7 +13,7 @@ "product-name": { "type": ["null", "string"] }, "country": { "type": ["null", "string"] }, "active-subscriptions": { "type": ["null", "string"] }, - "week-1-start-date": { "type": ["null", "string"] }, + "week-1-start-date": { "type": ["null", "string"], "format": "date-time" }, "scheduled-sns-units-week-1": { "type": ["null", "string"] }, "scheduled-sns-units-week-2": { "type": ["null", "string"] }, "scheduled-sns-units-week-3": { "type": ["null", "string"] }, @@ -21,6 +21,7 @@ "scheduled-sns-units-week-5": { "type": ["null", "string"] }, "scheduled-sns-units-week-6": { "type": ["null", "string"] }, "scheduled-sns-units-week-7": { "type": ["null", "string"] }, - "scheduled-sns-units-week-8": { "type": ["null", "string"] } + "scheduled-sns-units-week-8": { "type": ["null", "string"] }, + "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_SNS_PERFORMANCE_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_SNS_PERFORMANCE_DATA.json index 16e1010a9387..c1cd3074d1ac 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_SNS_PERFORMANCE_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_SNS_PERFORMANCE_DATA.json @@ -5,13 +5,13 @@ "$schema": "http://json-schema.org/draft-07/schema#", "properties": { "offer-state": { "type": ["null", "string"] }, - "snapshot-date": { "type": ["null", "string"] }, + "snapshot-date": { "type": ["null", "string"], "format": "date-time" }, "sku": { "type": ["null", "string"] }, "fnsku": { "type": ["null", "string"] }, "asin": { "type": ["null", "string"] }, "product-name": { "type": ["null", "string"] }, "country": { "type": ["null", "string"] }, - "week-1-start-date": { "type": ["null", "string"] }, + "week-1-start-date": { "type": ["null", "string"], "format": "date-time" }, "sns-units-shipped-week-1": { "type": ["null", "string"] }, "oos-rate-week-1": { "type": ["null", "string"] }, "sns-sale-price-week-1": { "type": ["null", "string"] }, @@ -27,6 +27,7 @@ "sns-units-shipped-week-4": { "type": ["null", "string"] }, "oos-rate-week-4": { "type": ["null", "string"] }, "sns-sale-price-week-4": { "type": ["null", "string"] }, - "sns-discount-week-4": { "type": ["null", "string"] } + "sns-discount-week-4": { "type": ["null", "string"] }, + "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_STORAGE_FEE_CHARGES_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_STORAGE_FEE_CHARGES_DATA.json index 09b88ee57af7..33515d7bf222 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_STORAGE_FEE_CHARGES_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FBA_STORAGE_FEE_CHARGES_DATA.json @@ -66,6 +66,40 @@ }, "currency": { "type": ["null", "string"] + }, + "average-quantity-customer-orders": { + "type": ["null", "number"] + }, + "base-rate": { + "type": ["null", "number"] + }, + "breakdown-incentive-fee-amount": { + "type": ["null", "string"] + }, + "dangerous-goods-storage-type": { + "type": ["null", "string"] + }, + "eligible_for_inventory_discount": { + "type": ["null", "string"] + }, + "qualifies_for_inventory_discount": { + "type": ["null", "string"] + }, + "storage-utilization-ratio": { + "type": ["null", "number"] + }, + "storage-utilization-ratio-units": { + "type": ["null", "string"] + }, + "total-incentive-fee-amount": { + "type": ["null", "number"] + }, + "utilization-surcharge-rate": { + "type": ["null", "number"] + }, + "dataEndTime": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING.json index acaa95d38547..3113a3cc2c87 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING.json @@ -82,6 +82,10 @@ }, "sku": { "type": ["null", "string"] + }, + "dataEndTime": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL.json index f9fd0947e7ff..9b5dd9d9bb53 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL.json @@ -7,97 +7,107 @@ "amazon-order-id": { "type": "string" }, - "asin": { - "type": ["null", "string"] + "merchant-order-id": { + "type": "string" }, - "currency": { + "purchase-date": { + "type": ["null", "string"], + "format": "date-time" + }, + "last-updated-date": { + "type": "string", + "format": "date-time" + }, + "order-status": { "type": ["null", "string"] }, "fulfillment-channel": { "type": ["null", "string"] }, - "gift-wrap-price": { + "sales-channel": { "type": ["null", "string"] }, - "gift-wrap-tax": { + "order-channel": { "type": ["null", "string"] }, - "is-business-order": { + "ship-service-level": { "type": ["null", "string"] }, - "item-price": { + "product-name": { "type": ["null", "string"] }, - "item-promotion-discount": { + "sku": { "type": ["null", "string"] }, - "item-status": { + "asin": { "type": ["null", "string"] }, - "item-tax": { + "item-status": { "type": ["null", "string"] }, - "last-updated-date": { - "type": "string", - "format": "date-time" - }, - "merchant-order-id": { + "quantity": { "type": ["null", "string"] }, - "order-channel": { + "currency": { "type": ["null", "string"] }, - "order-status": { + "item-price": { "type": ["null", "string"] }, - "price-designation": { + "item-tax": { "type": ["null", "string"] }, - "product-name": { + "shipping-price": { "type": ["null", "string"] }, - "promotion-ids": { + "shipping-tax": { "type": ["null", "string"] }, - "purchase-date": { - "type": ["null", "string"], - "format": "date-time" + "gift-wrap-price": { + "type": ["null", "string"] }, - "purchase-order-number": { + "gift-wrap-tax": { "type": ["null", "string"] }, - "quantity": { + "item-promotion-discount": { "type": ["null", "string"] }, - "sales-channel": { + "ship-promotion-discount": { "type": ["null", "string"] }, "ship-city": { "type": ["null", "string"] }, - "ship-country": { + "ship-state": { "type": ["null", "string"] }, "ship-postal-code": { "type": ["null", "string"] }, - "ship-promotion-discount": { + "ship-country": { "type": ["null", "string"] }, - "ship-service-level": { + "promotion-ids": { "type": ["null", "string"] }, - "ship-state": { + "cpf": { "type": ["null", "string"] }, - "shipping-price": { + "is-business-order": { "type": ["null", "string"] }, - "shipping-tax": { + "purchase-order-number": { "type": ["null", "string"] }, - "sku": { + "price-designation": { "type": ["null", "string"] + }, + "signature-confirmation-recommended": { + "type": ["null", "string"] + }, + "dataEndTime": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL.json index 374434b39d80..d64b129e6afc 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL.json @@ -4,76 +4,63 @@ "type": "object", "$schema": "http://json-schema.org/draft-07/schema#", "properties": { - "order-id": { + "amazon-order-id": { "type": ["null", "string"] }, - "order-item-id": { + "merchant-order-id": { "type": ["null", "string"] }, "purchase-date": { - "type": ["null", "string"] - }, - "payments-date": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, - "buyer-email": { - "type": ["null", "string"] + "last-updated-date": { + "type": ["null", "string"], + "format": "date-time" }, - "buyer-name": { + "order-status": { "type": ["null", "string"] }, - "sku": { + "fulfillment-channel": { "type": ["null", "string"] }, - "product-name": { - "type": ["null", "string"] - }, - "quantity-purchased": { - "type": ["null", "string"] - }, - "currency": { + "sales-channel": { "type": ["null", "string"] }, - "item-price": { - "type": ["null", "string"] - }, - "shipping-price": { - "type": ["null", "string"] - }, - "item-tax": { + "order-channel": { "type": ["null", "string"] }, "ship-service-level": { "type": ["null", "string"] }, - "recipient-name": { + "product-name": { "type": ["null", "string"] }, - "ship-address-1": { + "sku": { "type": ["null", "string"] }, - "ship-address-2": { + "asin": { "type": ["null", "string"] }, - "ship-address-3": { + "item-status": { "type": ["null", "string"] }, - "ship-city": { + "quantity": { "type": ["null", "string"] }, - "ship-state": { + "currency": { "type": ["null", "string"] }, - "ship-postal-code": { + "item-price": { "type": ["null", "string"] }, - "ship-country": { + "item-tax": { "type": ["null", "string"] }, - "gift-wrap-type": { + "shipping-price": { "type": ["null", "string"] }, - "gift-message-text": { + "shipping-tax": { "type": ["null", "string"] }, "gift-wrap-price": { @@ -85,22 +72,25 @@ "item-promotion-discount": { "type": ["null", "string"] }, - "item-promotion-id": { + "ship-promotion-discount": { + "type": ["null", "string"] + }, + "ship-city": { "type": ["null", "string"] }, - "shipping-promotion-discount": { + "ship-state": { "type": ["null", "string"] }, - "shipping-promotion-id": { + "ship-postal-code": { "type": ["null", "string"] }, - "delivery-instructions": { + "ship-country": { "type": ["null", "string"] }, - "order-channel": { + "promotion-ids": { "type": ["null", "string"] }, - "order-channel-instance": { + "cpf": { "type": ["null", "string"] }, "is-business-order": { @@ -112,77 +102,12 @@ "price-designation": { "type": ["null", "string"] }, - "buyer-company-name": { + "signature-confirmation-recommended": { "type": ["null", "string"] }, - "licensee-name": { - "type": ["null", "string"] - }, - "license-number": { - "type": ["null", "string"] - }, - "license-state": { - "type": ["null", "string"] - }, - "license-expiration-date": { - "type": ["null", "string"] - }, - "Address-Type": { - "type": ["null", "string"] - }, - "Number-of-items": { - "type": ["null", "string"] - }, - "is-global-express": { - "type": ["null", "string"] - }, - "default-ship-from-address-name": { - "type": ["null", "string"] - }, - "default-ship-from-address-field-1": { - "type": ["null", "string"] - }, - "default-ship-from-address-field-2": { - "type": ["null", "string"] - }, - "default-ship-from-address-field-3": { - "type": ["null", "string"] - }, - "default-ship-from-address-city": { - "type": ["null", "string"] - }, - "default-ship-from-address-state": { - "type": ["null", "string"] - }, - "default-ship-from-address-country": { - "type": ["null", "string"] - }, - "default-ship-from-address-postal-code": { - "type": ["null", "string"] - }, - "actual-ship-from-address-name": { - "type": ["null", "string"] - }, - "actual-ship-from-address-1": { - "type": ["null", "string"] - }, - "actual-ship-from-address-field-2": { - "type": ["null", "string"] - }, - "actual-ship-from-address-field-3": { - "type": ["null", "string"] - }, - "actual-ship-from-address-city": { - "type": ["null", "string"] - }, - "actual-ship-from-address-state": { - "type": ["null", "string"] - }, - "actual-ship-from-address-country": { - "type": ["null", "string"] - }, - "actual-ship-from-address-postal-code": { - "type": ["null", "string"] + "dataEndTime": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ARCHIVED_ORDERS_DATA_BY_ORDER_DATE.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ARCHIVED_ORDERS_DATA_BY_ORDER_DATE.json index a2635187c16b..856e6b6757db 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ARCHIVED_ORDERS_DATA_BY_ORDER_DATE.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_ARCHIVED_ORDERS_DATA_BY_ORDER_DATE.json @@ -6,8 +6,8 @@ "properties": { "amazon-order-id": { "type": ["null", "string"] }, "merchant-order-id": { "type": ["null", "string"] }, - "purchase-date": { "type": ["null", "string"] }, - "last-updated-date": { "type": ["null", "string"] }, + "purchase-date": { "type": ["null", "string"], "format": "date-time" }, + "last-updated-date": { "type": ["null", "string"], "format": "date-time" }, "order-status": { "type": ["null", "string"] }, "fulfillment-channel": { "type": ["null", "string"] }, "sales-channel": { "type": ["null", "string"] }, @@ -34,6 +34,7 @@ "is-business-order": { "type": ["null", "string"] }, "purchase-order-number": { "type": ["null", "string"] }, "price-designation": { "type": ["null", "string"] }, - "is-replacement-order": { "type": ["null", "string"] } + "is-replacement-order": { "type": ["null", "string"] }, + "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_OPEN_LISTINGS_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_OPEN_LISTINGS_DATA.json index d3baf1147640..1ff32753268e 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_OPEN_LISTINGS_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_OPEN_LISTINGS_DATA.json @@ -72,6 +72,10 @@ }, "Progressive Price 3": { "type": ["null", "string"] + }, + "dataEndTime": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE.json index f2db5fb0b0a9..6c3d35a9f0cc 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE.json @@ -5,8 +5,11 @@ "$schema": "http://json-schema.org/draft-07/schema#", "properties": { "Order ID": { "type": ["null", "string"] }, - "Order date": { "type": ["null", "string"] }, - "Return request date": { "type": ["null", "string"] }, + "Order date": { "type": ["null", "string"], "format": "date-time" }, + "Return request date": { + "type": ["null", "string"], + "format": "date-time" + }, "Return request status": { "type": ["null", "string"] }, "Amazon RMA ID": { "type": ["null", "string"] }, "Merchant RMA ID": { "type": ["null", "string"] }, @@ -27,14 +30,21 @@ "Return type": { "type": ["null", "string"] }, "Resolution": { "type": ["null", "string"] }, "Invoice number": { "type": ["null", "string"] }, - "Return delivery date": { "type": ["null", "string"] }, + "Return delivery date": { + "type": ["null", "string"], + "format": "date-time" + }, "Order Amount": { "type": ["null", "string"] }, "Order quantity": { "type": ["null", "string"] }, "SafeT Action reason": { "type": ["null", "string"] }, "SafeT claim id": { "type": ["null", "string"] }, "SafeT claim state": { "type": ["null", "string"] }, - "SafeT claim creation time": { "type": ["null", "string"] }, + "SafeT claim creation time": { + "type": ["null", "string"], + "format": "date-time" + }, "SafeT claim reimbursement amount": { "type": ["null", "string"] }, - "Refunded Amount": { "type": ["null", "string"] } + "Refunded Amount": { "type": ["null", "string"] }, + "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_LEDGER_DETAIL_VIEW_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_LEDGER_DETAIL_VIEW_DATA.json index 1c3509889cf0..1fd4d4de1f8b 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_LEDGER_DETAIL_VIEW_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_LEDGER_DETAIL_VIEW_DATA.json @@ -17,6 +17,8 @@ "Reason": { "type": ["null", "string"] }, "Country": { "type": ["null", "string"] }, "Reconciled Quantity": { "type": ["null", "string"] }, - "Unreconciled Quantity": { "type": ["null", "string"] } + "Unreconciled Quantity": { "type": ["null", "string"] }, + "Date and Time": { "type": ["null", "string"], "format": "date-time" }, + "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_LEDGER_SUMMARY_VIEW_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_LEDGER_SUMMARY_VIEW_DATA.json index 288c4928a8a1..5b7d9aa7c908 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_LEDGER_SUMMARY_VIEW_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_LEDGER_SUMMARY_VIEW_DATA.json @@ -4,7 +4,7 @@ "type": "object", "$schema": "http://json-schema.org/draft-07/schema#", "properties": { - "Date": { "type": ["null", "string"] }, + "Date": { "type": ["null", "string"], "format": "date-time" }, "FNSKU": { "type": ["null", "string"] }, "ASIN": { "type": ["null", "string"] }, "MSKU": { "type": ["null", "string"] }, @@ -24,6 +24,7 @@ "Other Events": { "type": ["null", "string"] }, "Ending Warehouse Balance": { "type": ["null", "string"] }, "Unknown Events": { "type": ["null", "string"] }, - "Location": { "type": ["null", "string"] } + "Location": { "type": ["null", "string"] }, + "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANTS_LISTINGS_FYP_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANTS_LISTINGS_FYP_REPORT.json index f2f56e7d53b1..820f76581b69 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANTS_LISTINGS_FYP_REPORT.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANTS_LISTINGS_FYP_REPORT.json @@ -10,7 +10,8 @@ "ASIN": { "type": ["null", "string"] }, "Product name": { "type": ["null", "string"] }, "Condition": { "type": ["null", "string"] }, - "Status Change Date": { "type": ["null", "string"] }, - "Issue Description": { "type": ["null", "string"] } + "Status Change Date": { "type": ["null", "string"], "format": "date" }, + "Issue Description": { "type": ["null", "string"] }, + "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_CANCELLED_LISTINGS_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_CANCELLED_LISTINGS_DATA.json index 35c5cd32d74e..941dde8abc07 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_CANCELLED_LISTINGS_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_CANCELLED_LISTINGS_DATA.json @@ -45,6 +45,7 @@ "Progressive Lower Bound 2": { "type": ["null", "string"] }, "Progressive Price 2": { "type": ["null", "string"] }, "Progressive Lower Bound 3": { "type": ["null", "string"] }, - "Progressive Price 3": { "type": ["null", "string"] } + "Progressive Price 3": { "type": ["null", "string"] }, + "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_LISTINGS_ALL_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_LISTINGS_ALL_DATA.json index c4d4459eb099..8994c700d257 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_LISTINGS_ALL_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_LISTINGS_ALL_DATA.json @@ -23,7 +23,8 @@ "type": ["null", "string"] }, "open-date": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "image-url": { "type": ["null", "string"] @@ -90,6 +91,10 @@ }, "status": { "type": ["null", "string"] + }, + "dataEndTime": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_LISTINGS_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_LISTINGS_DATA.json index f4c243481e5d..4265fe343a91 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_LISTINGS_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_LISTINGS_DATA.json @@ -10,7 +10,7 @@ "seller-sku": { "type": ["null", "string"] }, "price": { "type": ["null", "string"] }, "quantity": { "type": ["null", "string"] }, - "open-date": { "type": ["null", "string"] }, + "open-date": { "type": ["null", "string"], "format": "date-time" }, "image-url": { "type": ["null", "string"] }, "item-is-marketplace": { "type": ["null", "string"] }, "product-id-type": { "type": ["null", "string"] }, @@ -50,6 +50,7 @@ "Progressive Lower Bound 2": { "type": ["null", "string"] }, "Progressive Price 2": { "type": ["null", "string"] }, "Progressive Lower Bound 3": { "type": ["null", "string"] }, - "Progressive Price 3": { "type": ["null", "string"] } + "Progressive Price 3": { "type": ["null", "string"] }, + "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_LISTINGS_DATA_BACK_COMPAT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_LISTINGS_DATA_BACK_COMPAT.json index af4b8c76d21e..046e364206a0 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_LISTINGS_DATA_BACK_COMPAT.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_LISTINGS_DATA_BACK_COMPAT.json @@ -10,7 +10,7 @@ "seller-sku": { "type": ["null", "string"] }, "price": { "type": ["null", "string"] }, "quantity": { "type": ["null", "string"] }, - "open-date": { "type": ["null", "string"] }, + "open-date": { "type": ["null", "string"], "format": "date-time" }, "image-url": { "type": ["null", "string"] }, "item-is-marketplace": { "type": ["null", "string"] }, "product-id-type": { "type": ["null", "string"] }, @@ -49,6 +49,7 @@ "Progressive Lower Bound 2": { "type": ["null", "string"] }, "Progressive Price 2": { "type": ["null", "string"] }, "Progressive Lower Bound 3": { "type": ["null", "string"] }, - "Progressive Price 3": { "type": ["null", "string"] } + "Progressive Price 3": { "type": ["null", "string"] }, + "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_LISTINGS_INACTIVE_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_LISTINGS_INACTIVE_DATA.json index 900d02802e1f..68e2b9676a5e 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_LISTINGS_INACTIVE_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_MERCHANT_LISTINGS_INACTIVE_DATA.json @@ -10,7 +10,7 @@ "seller-sku": { "type": ["null", "string"] }, "price": { "type": ["null", "string"] }, "quantity": { "type": ["null", "string"] }, - "open-date": { "type": ["null", "string"] }, + "open-date": { "type": ["null", "string"], "format": "date-time" }, "image-url": { "type": ["null", "string"] }, "item-is-marketplace": { "type": ["null", "string"] }, "product-id-type": { "type": ["null", "string"] }, @@ -31,6 +31,7 @@ "add-delete": { "type": ["null", "string"] }, "pending-quantity": { "type": ["null", "string"] }, "fulfillment-channel": { "type": ["null", "string"] }, - "merchant-shipping-group": { "type": ["null", "string"] } + "merchant-shipping-group": { "type": ["null", "string"] }, + "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_ORDER_REPORT_DATA_SHIPPING.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_ORDER_REPORT_DATA_SHIPPING.json index baa13fe62730..0c4f280ee54d 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_ORDER_REPORT_DATA_SHIPPING.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_ORDER_REPORT_DATA_SHIPPING.json @@ -187,6 +187,10 @@ "type": ["null", "string"] } } + }, + "dataEndTime": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT.json index b84b87085fd8..50d3d0994261 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT.json @@ -90,6 +90,13 @@ }, "Recommended action": { "type": ["null", "string"] + }, + "Unit storage size": { + "type": ["null", "string"] + }, + "dataEndTime": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_SALES_AND_TRAFFIC_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_SALES_AND_TRAFFIC_REPORT.json index 7df5452f9d1e..d41c4cb69df1 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_SALES_AND_TRAFFIC_REPORT.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_SALES_AND_TRAFFIC_REPORT.json @@ -5,7 +5,8 @@ "$schema": "http://json-schema.org/draft-07/schema#", "properties": { "queryEndDate": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "parentAsin": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_SALES_AND_TRAFFIC_REPORT_BY_DATE.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_SALES_AND_TRAFFIC_REPORT_BY_DATE.json deleted file mode 100644 index f19596b1f8d5..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_SALES_AND_TRAFFIC_REPORT_BY_DATE.json +++ /dev/null @@ -1,143 +0,0 @@ -{ - "title": "Seller Sales and Traffic Business Report - By Date", - "description": "Seller retail analytics reports - Sales and Traffic Business Report", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "date": { - "type": ["null", "string"], - "format": "date" - }, - "salesByDate": { - "type": "object", - "properties": { - "orderedProductSales": { - "type": "object", - "properties": { - "amount": { - "type": ["null", "number"] - }, - "currencyCode": { - "type": ["null", "string"] - } - } - }, - "unitsOrdered": { - "type": ["null", "number"] - }, - "totalOrderItems": { - "type": ["null", "number"] - }, - "averageSalesPerOrderItem": { - "type": "object", - "properties": { - "amount": { - "type": ["null", "number"] - }, - "currencyCode": { - "type": ["null", "string"] - } - } - }, - "averageUnitsPerOrderItem": { - "type": ["null", "number"] - }, - "averageSellingPrice": { - "type": "object", - "properties": { - "amount": { - "type": ["null", "number"] - }, - "currencyCode": { - "type": ["null", "string"] - } - } - }, - "unitsRefunded": { - "type": ["null", "number"] - }, - "refundRate": { - "type": ["null", "number"] - }, - "claimsGranted": { - "type": ["null", "number"] - }, - "claimsAmount": { - "type": "object", - "properties": { - "amount": { - "type": ["null", "number"] - }, - "currencyCode": { - "type": ["null", "string"] - } - } - }, - "shippedProductSales": { - "type": "object", - "properties": { - "amount": { - "type": ["null", "number"] - }, - "currencyCode": { - "type": ["null", "string"] - } - } - }, - "unitsShipped": { - "type": ["null", "number"] - }, - "ordersShipped": { - "type": ["null", "number"] - } - } - }, - "trafficByDate": { - "type": "object", - "properties": { - "browserPageViews": { - "type": ["null", "number"] - }, - "mobileAppPageViews": { - "type": ["null", "number"] - }, - "pageViews": { - "type": ["null", "number"] - }, - "browserSessions": { - "type": ["null", "number"] - }, - "mobileAppSessions": { - "type": ["null", "number"] - }, - "sessions": { - "type": ["null", "number"] - }, - "buyBoxPercentage": { - "type": ["null", "number"] - }, - "orderItemSessionPercentage": { - "type": ["null", "number"] - }, - "unitSessionPercentage": { - "type": ["null", "number"] - }, - "averageOfferCount": { - "type": ["null", "number"] - }, - "averageParentItems": { - "type": ["null", "number"] - }, - "feedbackReceived": { - "type": ["null", "number"] - }, - "negativeFeedbackReceived": { - "type": ["null", "number"] - }, - "receivedNegativeFeedbackRate": { - "type": ["null", "number"] - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_SELLER_FEEDBACK_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_SELLER_FEEDBACK_DATA.json index f03f3fea23c6..ad0272e5d046 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_SELLER_FEEDBACK_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_SELLER_FEEDBACK_DATA.json @@ -22,6 +22,10 @@ }, "rater_email": { "type": ["null", "string"] + }, + "dataEndTime": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_STRANDED_INVENTORY_UI_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_STRANDED_INVENTORY_UI_DATA.json index 3ebcb0d0db8b..c31445971147 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_STRANDED_INVENTORY_UI_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_STRANDED_INVENTORY_UI_DATA.json @@ -5,8 +5,11 @@ "$schema": "http://json-schema.org/draft-07/schema#", "properties": { "primary-action": { "type": ["null", "string"] }, - "date-stranded": { "type": ["null", "string"] }, - "Date-to-take-auto-removal": { "type": ["null", "string"] }, + "date-stranded": { "type": ["null", "string"], "format": "date-time" }, + "Date-to-take-auto-removal": { + "type": ["null", "string"], + "format": "date-time" + }, "status-primary": { "type": ["null", "string"] }, "status-secondary": { "type": ["null", "string"] }, "error-message": { "type": ["null", "string"] }, @@ -21,6 +24,7 @@ "your-price": { "type": ["null", "string"] }, "unfulfillable-qty": { "type": ["null", "string"] }, "reserved-quantity": { "type": ["null", "string"] }, - "inbound-shipped-qty": { "type": ["null", "string"] } + "inbound-shipped-qty": { "type": ["null", "string"] }, + "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE.json index 76470acdae3b..ffa44e5ee2a9 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE.json @@ -5,9 +5,15 @@ "$schema": "http://json-schema.org/draft-07/schema#", "properties": { "settlement-id": { "type": ["null", "string"] }, - "settlement-start-date": { "type": ["null", "string"] }, - "settlement-end-date": { "type": ["null", "string"] }, - "deposit-date": { "type": ["null", "string"] }, + "settlement-start-date": { + "type": ["null", "string"], + "format": "date-time" + }, + "settlement-end-date": { + "type": ["null", "string"], + "format": "date-time" + }, + "deposit-date": { "type": ["null", "string"], "format": "date-time" }, "total-amount": { "type": ["null", "string"] }, "currency": { "type": ["null", "string"] }, "transaction-type": { "type": ["null", "string"] }, @@ -21,7 +27,7 @@ "order-fee-type": { "type": ["null", "string"] }, "order-fee-amount": { "type": ["null", "string"] }, "fulfillment-id": { "type": ["null", "string"] }, - "posted-date": { "type": ["null", "string"] }, + "posted-date": { "type": ["null", "string"], "format": "date-time" }, "order-item-code": { "type": ["null", "string"] }, "merchant-order-item-id": { "type": ["null", "string"] }, "merchant-adjustment-item-id": { "type": ["null", "string"] }, @@ -40,6 +46,7 @@ "direct-payment-type": { "type": ["null", "string"] }, "direct-payment-amount": { "type": ["null", "string"] }, "other-amount": { "type": ["null", "string"] }, - "report_id": { "type": ["null", "string"] } + "report_id": { "type": ["null", "string"] }, + "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_INVENTORY_HEALTH_AND_PLANNING_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_INVENTORY_HEALTH_AND_PLANNING_REPORT.json deleted file mode 100644 index 5b48d11d5e10..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_INVENTORY_HEALTH_AND_PLANNING_REPORT.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "title": "Vendor Inventory Health and Planning Data", - "description": "Vendor Inventory Health and Planning Data Reports", - "type": "object", - "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "seller-sku": { - "type": ["null", "string"] - }, - "quantity": { - "type": ["null", "string"] - }, - "price": { - "type": ["null", "number"] - }, - "product ID": { - "type": ["null", "number"] - } - } -} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_INVENTORY_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_INVENTORY_REPORT.json index e6d68c2d6bd3..dc6a332fcea4 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_INVENTORY_REPORT.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_INVENTORY_REPORT.json @@ -5,10 +5,12 @@ "$schema": "http://json-schema.org/draft-07/schema#", "properties": { "startDate": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "endDate": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "asin": { "type": ["null", "string"] @@ -83,6 +85,14 @@ }, "unhealthyUnits": { "type": ["null", "number"] + }, + "dataEndTime": { + "type": ["null", "string"], + "format": "date" + }, + "queryEndDate": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT.json new file mode 100644 index 000000000000..079d303f3971 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT.json @@ -0,0 +1,53 @@ +{ + "title": "Net Pure Product Margin Report", + "description": "The Net Pure Product Margin report shares data with vendors on Amazon's profit margin selling the vendor's items both at an aggregated level (across the vendor's entire catalog of items) and at a per-ASIN level. Data is available at different date range aggregation levels: DAY, WEEK, MONTH, QUARTER, YEAR. Requests can span multiple date range periods.", + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "netPureProductMarginAggregate": { + "type": ["null", "array"], + "items": { + "type": ["object"] + }, + "properties": { + "startDate": { + "type": ["null", "string"], + "format": "date" + }, + "endDate": { + "type": ["null", "string"], + "format": "date" + }, + "netPureProductMargin": { + "type": ["null", "number"] + } + } + }, + "netPureProductMarginByAsin": { + "type": ["null", "array"], + "items": { + "type": ["object"] + }, + "properties": { + "startDate": { + "type": ["null", "string"], + "format": "date" + }, + "endDate": { + "type": ["null", "string"], + "format": "date" + }, + "asin": { + "type": ["null", "string"] + }, + "netPureProductMargin": { + "type": ["null", "number"] + } + } + }, + "dataEndTime": { + "type": ["null", "string"], + "format": "date" + } + } +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_REAL_TIME_INVENTORY_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_REAL_TIME_INVENTORY_REPORT.json new file mode 100644 index 000000000000..6dff239799bb --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_REAL_TIME_INVENTORY_REPORT.json @@ -0,0 +1,56 @@ +{ + "title": "Rapid Retail Analytics Inventory Report", + "description": "This report shares inventory data at an ASIN level, aggregated to an hourly granularity. Requests can span multiple date range periods, including the current day.", + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "reportSpecification": { + "type": ["null", "object"], + "properties": { + "reportType": { + "type": ["null", "string"] + }, + "dataStartTime": { + "type": ["null", "string"], + "format": "date-time" + }, + "dataEndTime": { + "type": ["null", "string"], + "format": "date-time" + }, + "marketplaceIds": { + "type": ["null", "array"], + "items": { + "type": ["string"] + } + } + } + }, + "reportData": { + "type": ["null", "array"], + "items": { + "type": ["object"] + }, + "properties": { + "startTime": { + "type": ["null", "string"], + "format": "date-time" + }, + "endTime": { + "type": ["null", "string"], + "format": "date-time" + }, + "asin": { + "type": ["null", "string"] + }, + "highlyAvailableInventory": { + "type": ["null", "integer"] + } + } + }, + "dataEndTime": { + "type": ["null", "string"], + "format": "date" + } + } +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_SALES_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_SALES_REPORT.json index e63716d0c668..61f753135d79 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_SALES_REPORT.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_SALES_REPORT.json @@ -5,10 +5,12 @@ "$schema": "http://json-schema.org/draft-07/schema#", "properties": { "startDate": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "endDate": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "asin": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_TRAFFIC_REPORT.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_TRAFFIC_REPORT.json new file mode 100644 index 000000000000..bdb05e8b1fe0 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_VENDOR_TRAFFIC_REPORT.json @@ -0,0 +1,26 @@ +{ + "title": "Vendor Traffic Report", + "description": "This report shares data on the customer traffic to the detail pages of the vendor's items both at an aggregated level (across the vendor's entire catalog of items) and at a per-ASIN level. Data is available for different date range aggregation levels: DAY, WEEK, MONTH, QUARTER, YEAR. Requests can span multiple date range periods.", + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "startDate": { + "type": ["null", "string"], + "format": "date" + }, + "endDate": { + "type": ["null", "string"], + "format": "date" + }, + "asin": { + "type": ["null", "string"] + }, + "glanceViews": { + "type": ["null", "integer"] + }, + "queryEndDate": { + "type": ["null", "string"], + "format": "date" + } + } +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL.json index e06c2dda083c..22c9b1eb93a5 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL.json @@ -4,11 +4,11 @@ "type": "object", "$schema": "http://json-schema.org/draft-07/schema#", "properties": { - "LastUpdatedDate": { "type": ["null", "string"] }, + "LastUpdatedDate": { "type": ["null", "string"], "format": "date-time" }, "SalesChannel": { "type": ["null", "string"] }, "OrderStatus": { "type": ["null", "string"] }, "AmazonOrderID": { "type": ["null", "string"] }, - "PurchaseDate": { "type": ["null", "string"] }, + "PurchaseDate": { "type": ["null", "string"], "format": "date-time" }, "OrderItem": { "type": ["array"], "items": { "type": ["null", "object"] }, @@ -57,6 +57,7 @@ } }, "IsBusinessOrder": { "type": ["null", "string"] }, - "MerchantOrderID": { "type": ["null", "string"] } + "MerchantOrderID": { "type": ["null", "string"] }, + "dataEndTime": { "type": ["null", "string"], "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_XML_BROWSE_TREE_DATA.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_XML_BROWSE_TREE_DATA.json index be41d2108a45..a8930bcda943 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_XML_BROWSE_TREE_DATA.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/GET_XML_BROWSE_TREE_DATA.json @@ -89,6 +89,10 @@ } } } + }, + "dataEndTime": { + "type": ["null", "string"], + "format": "date" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/ListFinancialEventGroups.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/ListFinancialEventGroups.json index 17833f54c1c2..d30d4a5ded69 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/ListFinancialEventGroups.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/ListFinancialEventGroups.json @@ -36,7 +36,8 @@ } }, "FundTransferDate": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "TraceId": { "type": ["null", "string"] @@ -56,10 +57,12 @@ } }, "FinancialEventGroupStart": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "FinancialEventGroupEnd": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/ListFinancialEvents.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/ListFinancialEvents.json index aaaf073bf2e1..dc194dbe53ff 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/ListFinancialEvents.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/ListFinancialEvents.json @@ -984,6 +984,10 @@ } } } + }, + "PostedBefore": { + "type": ["null", "string"], + "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/OrderItems.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/OrderItems.json index e8fa6b726250..cd882adfe0ca 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/OrderItems.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/OrderItems.json @@ -284,7 +284,8 @@ } }, "LastUpdateDate": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/Orders.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/Orders.json index 942eeb73a650..1dbd29c72433 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/Orders.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/Orders.json @@ -17,10 +17,12 @@ "additionalProperties": true }, "PurchaseDate": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "LastUpdateDate": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "OrderStatus": { "type": ["null", "string"] @@ -91,10 +93,12 @@ "type": ["null", "string"] }, "EarliestShipDate": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "LatestShipDate": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time" }, "IsBusinessOrder": { "type": ["null", "boolean"] @@ -113,6 +117,56 @@ }, "IsISPU": { "type": ["null", "boolean"] + }, + "DefaultShipFromLocationAddress": { + "type": ["null", "object"], + "properties": { + "AddressLine1": { + "type": ["null", "string"] + }, + "City": { + "type": ["null", "string"] + }, + "CountryCode": { + "type": ["null", "string"] + }, + "Name": { + "type": ["null", "string"] + }, + "PostalCode": { + "type": ["null", "string"] + }, + "StateOrRegion": { + "type": ["null", "string"] + } + } + }, + "EarliestDeliveryDate": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "LatestDeliveryDate": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "ShippingAddress": { + "type": ["null", "object"], + "properties": { + "City": { + "type": ["null", "string"] + }, + "CountryCode": { + "type": ["null", "string"] + }, + "PostalCode": { + "type": ["null", "string"] + }, + "StateOrRegion": { + "type": ["null", "string"] + } + } } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/VendorDirectFulfillmentShipping.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/VendorDirectFulfillmentShipping.json index 56bae0c6f47b..73c80399a196 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/VendorDirectFulfillmentShipping.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/schemas/VendorDirectFulfillmentShipping.json @@ -234,6 +234,10 @@ } } } + }, + "createdBefore": { + "type": ["null", "string"], + "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py index 5f9bb1c5b7b0..6f7c6a4b9c5f 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/source.py @@ -2,18 +2,19 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import Any, List, Mapping, Tuple -import boto3 +from os import getenv +from typing import Any, List, Mapping, Optional, Tuple + +import pendulum from airbyte_cdk.logger import AirbyteLogger from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream -from source_amazon_seller_partner.auth import AWSAuthenticator, AWSSignature +from requests import HTTPError +from source_amazon_seller_partner.auth import AWSAuthenticator from source_amazon_seller_partner.constants import get_marketplaces from source_amazon_seller_partner.streams import ( - BrandAnalyticsAlternatePurchaseReports, - BrandAnalyticsItemComparisonReports, BrandAnalyticsMarketBasketReports, BrandAnalyticsRepeatPurchaseReports, BrandAnalyticsSearchTermsReports, @@ -21,12 +22,7 @@ FbaAfnInventoryReports, FbaCustomerReturnsReports, FbaEstimatedFbaFeesTxtReport, - FbaFulfillmentCurrentInventoryReport, FbaFulfillmentCustomerShipmentPromotionReport, - FbaFulfillmentInventoryAdjustReport, - FbaFulfillmentInventoryReceiptsReport, - FbaFulfillmentInventorySummaryReport, - FbaFulfillmentMonthlyInventoryReport, FbaInventoryPlaningReport, FbaMyiUnsuppressedInventoryReport, FbaOrdersReports, @@ -55,9 +51,11 @@ MerchantListingsReport, MerchantListingsReportBackCompat, MerchantListingsReports, + NetPureProductMarginReport, OrderItems, OrderReportDataShipping, Orders, + RapidRetailAnalyticsInventoryReport, RestockInventoryReports, SellerAnalyticsSalesAndTrafficReports, SellerFeedbackReports, @@ -65,23 +63,16 @@ VendorDirectFulfillmentShipping, VendorInventoryReports, VendorSalesReports, + VendorTrafficReport, XmlAllOrdersDataByOrderDataGeneral, ) +from source_amazon_seller_partner.utils import AmazonConfigException class SourceAmazonSellerPartner(AbstractSource): - def _get_stream_kwargs(self, config: Mapping[str, Any]) -> Mapping[str, Any]: - endpoint, marketplace_id, region = get_marketplaces(config.get("aws_environment"))[config.get("region")] - - sts_credentials = self.get_sts_credentials(config) - role_creds = sts_credentials["Credentials"] - aws_signature = AWSSignature( - service="execute-api", - aws_access_key_id=role_creds.get("AccessKeyId"), - aws_secret_access_key=role_creds.get("SecretAccessKey"), - aws_session_token=role_creds.get("SessionToken"), - region=region, - ) + @staticmethod + def _get_stream_kwargs(config: Mapping[str, Any]) -> Mapping[str, Any]: + endpoint, marketplace_id, _ = get_marketplaces(config.get("aws_environment"))[config.get("region")] auth = AWSAuthenticator( token_refresh_endpoint="https://api.amazon.com/auth/o2/token", client_id=config.get("lwa_app_id"), @@ -90,45 +81,21 @@ def _get_stream_kwargs(self, config: Mapping[str, Any]) -> Mapping[str, Any]: host=endpoint.replace("https://", ""), refresh_access_token_headers={"Content-Type": "application/x-www-form-urlencoded"}, ) + start_date = ( + config.get("replication_start_date") + if config.get("replication_start_date") + else pendulum.now("utc").subtract(years=2).strftime("%Y-%m-%dT%H:%M:%SZ") + ) stream_kwargs = { "url_base": endpoint, "authenticator": auth, - "aws_signature": aws_signature, - "replication_start_date": config.get("replication_start_date"), + "replication_start_date": start_date, "marketplace_id": marketplace_id, - "period_in_days": config.get("period_in_days", 90), - "report_options": config.get("report_options"), - "max_wait_seconds": config.get("max_wait_seconds", 500), + "period_in_days": config.get("period_in_days", 30), "replication_end_date": config.get("replication_end_date"), - "advanced_stream_options": config.get("advanced_stream_options"), } return stream_kwargs - @staticmethod - def get_sts_credentials(config: Mapping[str, Any]) -> dict: - """ - We can only use a IAM User arn entity or a IAM Role entity. - If we use an IAM user arn entity in the connector configuration we need to get the credentials directly from the boto3 sts client - If we use an IAM role arn entity we need to invoke the assume_role from the boto3 sts client to get the credentials related to that role - - :param config: - """ - boto3_client = boto3.client( - "sts", aws_access_key_id=config.get("aws_access_key"), aws_secret_access_key=config.get("aws_secret_key") - ) - - if config.get("role_arn") is None: - return boto3_client.get_session_token() - - *_, arn_resource = config.get("role_arn").split(":") - if arn_resource.startswith("user"): - sts_credentials = boto3_client.get_session_token() - elif arn_resource.startswith("role"): - sts_credentials = boto3_client.assume_role(RoleArn=config.get("role_arn"), RoleSessionName="guid") - else: - raise ValueError("Invalid ARN, your ARN is not for a user or a role") - return sts_credentials - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: """ Check connection to Amazon SP API by requesting the Orders endpoint @@ -140,6 +107,8 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Show error message in case of request exception or unexpected response. """ try: + self.validate_replication_dates(config) + self.validate_stream_report_options(config) stream_kwargs = self._get_stream_kwargs(config) orders_stream = Orders(**stream_kwargs) next(orders_stream.read_records(sync_mode=SyncMode.full_refresh)) @@ -150,73 +119,109 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> if isinstance(e, StopIteration): return True, None - # Additional check, since Vendor-only accounts within Amazon Seller API - # will not pass the test without this exception + # Additional check, since Vendor-only accounts within Amazon Seller API will not pass the test without this exception if "403 Client Error" in str(e): stream_to_check = VendorSalesReports(**stream_kwargs) next(stream_to_check.read_records(sync_mode=SyncMode.full_refresh)) return True, None - return False, e + error_message = e.response.json().get("error_description") if isinstance(e, HTTPError) else e + return False, error_message def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ :param config: A Mapping of the user input configuration as defined in the connector spec. """ + self.validate_stream_report_options(config) + streams = [] stream_kwargs = self._get_stream_kwargs(config) - return [ - FbaCustomerReturnsReports(**stream_kwargs), - FbaAfnInventoryReports(**stream_kwargs), - FbaAfnInventoryByCountryReports(**stream_kwargs), - FbaOrdersReports(**stream_kwargs), - FbaShipmentsReports(**stream_kwargs), - FbaReplacementsReports(**stream_kwargs), - FbaStorageFeesReports(**stream_kwargs), - RestockInventoryReports(**stream_kwargs), - FlatFileActionableOrderDataShipping(**stream_kwargs), - FlatFileOpenListingsReports(**stream_kwargs), - FlatFileOrdersReports(**stream_kwargs), - FlatFileOrdersReportsByLastUpdate(**stream_kwargs), - FlatFileSettlementV2Reports(**stream_kwargs), - FulfilledShipmentsReports(**stream_kwargs), - MerchantListingsReports(**stream_kwargs), - VendorDirectFulfillmentShipping(**stream_kwargs), - VendorInventoryReports(**stream_kwargs), - VendorSalesReports(**stream_kwargs), - Orders(**stream_kwargs), - OrderItems(**stream_kwargs), - OrderReportDataShipping(**stream_kwargs), - SellerAnalyticsSalesAndTrafficReports(**stream_kwargs), - SellerFeedbackReports(**stream_kwargs), - BrandAnalyticsMarketBasketReports(**stream_kwargs), - BrandAnalyticsSearchTermsReports(**stream_kwargs), - BrandAnalyticsRepeatPurchaseReports(**stream_kwargs), - BrandAnalyticsAlternatePurchaseReports(**stream_kwargs), - BrandAnalyticsItemComparisonReports(**stream_kwargs), - GetXmlBrowseTreeData(**stream_kwargs), - ListFinancialEventGroups(**stream_kwargs), - ListFinancialEvents(**stream_kwargs), - LedgerDetailedViewReports(**stream_kwargs), - FbaEstimatedFbaFeesTxtReport(**stream_kwargs), - FbaFulfillmentCurrentInventoryReport(**stream_kwargs), - FbaFulfillmentCustomerShipmentPromotionReport(**stream_kwargs), - FbaFulfillmentInventoryAdjustReport(**stream_kwargs), - FbaFulfillmentInventoryReceiptsReport(**stream_kwargs), - FbaFulfillmentInventorySummaryReport(**stream_kwargs), - FbaMyiUnsuppressedInventoryReport(**stream_kwargs), - MerchantCancelledListingsReport(**stream_kwargs), - MerchantListingsReport(**stream_kwargs), - MerchantListingsReportBackCompat(**stream_kwargs), - MerchantListingsInactiveData(**stream_kwargs), - StrandedInventoryUiReport(**stream_kwargs), - XmlAllOrdersDataByOrderDataGeneral(**stream_kwargs), - FbaFulfillmentMonthlyInventoryReport(**stream_kwargs), - MerchantListingsFypReport(**stream_kwargs), - FbaSnsForecastReport(**stream_kwargs), - FbaSnsPerformanceReport(**stream_kwargs), - FlatFileArchivedOrdersDataByOrderDate(**stream_kwargs), - FlatFileReturnsDataByReturnDate(**stream_kwargs), - FbaInventoryPlaningReport(**stream_kwargs), - LedgerSummaryViewReport(**stream_kwargs), - FbaReimbursementsReports(**stream_kwargs), + stream_list = [ + FbaCustomerReturnsReports, + FbaAfnInventoryReports, + FbaAfnInventoryByCountryReports, + FbaOrdersReports, + FbaShipmentsReports, + FbaReplacementsReports, + FbaStorageFeesReports, + RestockInventoryReports, + FlatFileActionableOrderDataShipping, + FlatFileOpenListingsReports, + FlatFileOrdersReports, + FlatFileOrdersReportsByLastUpdate, + FlatFileSettlementV2Reports, + FulfilledShipmentsReports, + MerchantListingsReports, + VendorDirectFulfillmentShipping, + Orders, + OrderItems, + OrderReportDataShipping, + SellerFeedbackReports, + GetXmlBrowseTreeData, + ListFinancialEventGroups, + ListFinancialEvents, + LedgerDetailedViewReports, + FbaEstimatedFbaFeesTxtReport, + FbaFulfillmentCustomerShipmentPromotionReport, + FbaMyiUnsuppressedInventoryReport, + MerchantCancelledListingsReport, + MerchantListingsReport, + MerchantListingsReportBackCompat, + MerchantListingsInactiveData, + StrandedInventoryUiReport, + XmlAllOrdersDataByOrderDataGeneral, + MerchantListingsFypReport, + FbaSnsForecastReport, + FbaSnsPerformanceReport, + FlatFileArchivedOrdersDataByOrderDate, + FlatFileReturnsDataByReturnDate, + FbaInventoryPlaningReport, + LedgerSummaryViewReport, + FbaReimbursementsReports, ] + + # TODO: Remove after Brand Analytics will be enabled in CLOUD: https://github.com/airbytehq/airbyte/issues/32353 + if getenv("DEPLOYMENT_MODE", "").upper() != "CLOUD": + brand_analytics_reports = [ + BrandAnalyticsMarketBasketReports, + BrandAnalyticsSearchTermsReports, + BrandAnalyticsRepeatPurchaseReports, + SellerAnalyticsSalesAndTrafficReports, + VendorSalesReports, + VendorInventoryReports, + NetPureProductMarginReport, + RapidRetailAnalyticsInventoryReport, + VendorTrafficReport, + ] + stream_list += brand_analytics_reports + + for stream in stream_list: + streams.append(stream(**stream_kwargs, report_options=self.get_stream_report_options_list(stream.name, config))) + return streams + + @staticmethod + def validate_replication_dates(config: Mapping[str, Any]) -> None: + if ( + "replication_start_date" in config + and "replication_end_date" in config + and config["replication_end_date"] < config["replication_start_date"] + ): + raise AmazonConfigException(message="End Date should be greater than or equal to Start Date") + + @staticmethod + def validate_stream_report_options(config: Mapping[str, Any]) -> None: + if len([x.get("stream_name") for x in config.get("report_options_list", [])]) != len( + set(x.get("stream_name") for x in config.get("report_options_list", [])) + ): + raise AmazonConfigException(message="Stream name should be unique among all Report options list") + for stream_report_option in config.get("report_options_list", []): + if len([x.get("option_name") for x in stream_report_option.get("options_list")]) != len( + set(x.get("option_name") for x in stream_report_option.get("options_list")) + ): + raise AmazonConfigException( + message=f"Option names should be unique for `{stream_report_option.get('stream_name')}` report options" + ) + + @staticmethod + def get_stream_report_options_list(report_name: str, config: Mapping[str, Any]) -> Optional[List[Mapping[str, Any]]]: + if any(x for x in config.get("report_options_list", []) if x.get("stream_name") == report_name): + return [x.get("options_list") for x in config.get("report_options_list") if x.get("stream_name") == report_name][0] diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json index 9c8e32370a3e..9f84b550d8e7 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/spec.json @@ -7,10 +7,10 @@ "required": [ "aws_environment", "region", + "account_type", "lwa_app_id", "lwa_client_secret", - "refresh_token", - "replication_start_date" + "refresh_token" ], "additionalProperties": true, "properties": { @@ -59,31 +59,18 @@ "type": "string", "order": 2 }, - "aws_access_key": { - "title": "AWS Access Key", - "description": "Specifies the AWS access key used as part of the credentials to authenticate the user.", - "airbyte_secret": true, - "order": 3, - "type": "string" - }, - "aws_secret_key": { - "title": "AWS Secret Access Key", - "description": "Specifies the AWS secret key used as part of the credentials to authenticate the user.", - "airbyte_secret": true, - "order": 4, - "type": "string" - }, - "role_arn": { - "title": "Role ARN", - "description": "Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations requested using this profile. (Needs permission to 'Assume Role' STS).", - "airbyte_secret": true, - "order": 5, - "type": "string" + "account_type": { + "title": "AWS Seller Partner Account Type", + "description": "Type of the Account you're going to authorize the Airbyte application by", + "enum": ["Seller", "Vendor"], + "default": "Seller", + "type": "string", + "order": 3 }, "lwa_app_id": { "title": "LWA Client Id", "description": "Your Login with Amazon Client ID.", - "order": 6, + "order": 4, "airbyte_secret": true, "type": "string" }, @@ -91,66 +78,126 @@ "title": "LWA Client Secret", "description": "Your Login with Amazon Client Secret.", "airbyte_secret": true, - "order": 7, + "order": 5, "type": "string" }, "refresh_token": { "title": "Refresh Token", "description": "The Refresh Token obtained via OAuth flow authorization.", "airbyte_secret": true, - "order": 8, + "order": 6, "type": "string" }, "replication_start_date": { "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. If start date is not provided, the date 2 years ago from today will be used.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", "examples": ["2017-01-25T00:00:00Z"], - "order": 9, - "type": "string" + "order": 7, + "type": "string", + "format": "date-time" }, "replication_end_date": { "title": "End Date", "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data after this date will not be replicated.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$|^$", "examples": ["2017-01-25T00:00:00Z"], - "order": 10, - "type": "string" + "order": 8, + "type": "string", + "format": "date-time" }, "period_in_days": { "title": "Period In Days", "type": "integer", - "description": "Will be used for stream slicing for initial full_refresh sync when no updated state is present for reports that support sliced incremental sync.", + "description": "For syncs spanning a large date range, this option is used to request data in a smaller fixed window to improve sync reliability. This time window can be configured granularly by day.", "default": 90, - "order": 11 + "minimum": 1, + "order": 9 }, - "report_options": { + "report_options_list": { "title": "Report Options", - "description": "Additional information passed to reports. This varies by report type. Must be a valid json string.", - "examples": [ - "{\"GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT\": {\"reportPeriod\": \"WEEK\"}}", - "{\"GET_SOME_REPORT\": {\"custom\": \"true\"}}" - ], - "order": 12, - "type": "string" - }, - "max_wait_seconds": { - "title": "Max wait time for reports (in seconds)", - "description": "Sometimes report can take up to 30 minutes to generate. This will set the limit for how long to wait for a successful report.", - "default": 500, - "examples": ["500", "1980"], - "order": 13, - "type": "integer" - }, - "advanced_stream_options": { - "title": "Advanced Stream Options", - "description": "Additional information to configure report options. This varies by report type, not every report implement this kind of feature. Must be a valid json string.", - "examples": [ - "{\"GET_SALES_AND_TRAFFIC_REPORT\": {\"availability_sla_days\": 3}}", - "{\"GET_SOME_REPORT\": {\"custom\": \"true\"}}" - ], - "order": 14, - "type": "string" + "description": "Additional information passed to reports. This varies by report type.", + "order": 10, + "type": "array", + "items": { + "type": "object", + "title": "Report Options", + "required": ["stream_name", "options_list"], + "properties": { + "stream_name": { + "title": "Stream Name", + "type": "string", + "order": 0, + "enum": [ + "GET_AFN_INVENTORY_DATA", + "GET_AFN_INVENTORY_DATA_BY_COUNTRY", + "GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL", + "GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT", + "GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT", + "GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT", + "GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA", + "GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA", + "GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA", + "GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA", + "GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA", + "GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA", + "GET_FBA_INVENTORY_PLANNING_DATA", + "GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA", + "GET_FBA_REIMBURSEMENTS_DATA", + "GET_FBA_SNS_FORECAST_DATA", + "GET_FBA_SNS_PERFORMANCE_DATA", + "GET_FBA_STORAGE_FEE_CHARGES_DATA", + "GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING", + "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL", + "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", + "GET_FLAT_FILE_ARCHIVED_ORDERS_DATA_BY_ORDER_DATE", + "GET_FLAT_FILE_OPEN_LISTINGS_DATA", + "GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE", + "GET_LEDGER_DETAIL_VIEW_DATA", + "GET_LEDGER_SUMMARY_VIEW_DATA", + "GET_MERCHANT_CANCELLED_LISTINGS_DATA", + "GET_MERCHANT_LISTINGS_ALL_DATA", + "GET_MERCHANT_LISTINGS_DATA", + "GET_MERCHANT_LISTINGS_DATA_BACK_COMPAT", + "GET_MERCHANT_LISTINGS_INACTIVE_DATA", + "GET_MERCHANTS_LISTINGS_FYP_REPORT", + "GET_ORDER_REPORT_DATA_SHIPPING", + "GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT", + "GET_SALES_AND_TRAFFIC_REPORT", + "GET_SELLER_FEEDBACK_DATA", + "GET_STRANDED_INVENTORY_UI_DATA", + "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", + "GET_VENDOR_INVENTORY_REPORT", + "GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT", + "GET_VENDOR_TRAFFIC_REPORT", + "GET_VENDOR_SALES_REPORT", + "GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL", + "GET_XML_BROWSE_TREE_DATA" + ] + }, + "options_list": { + "title": "List of options", + "description": "List of options", + "type": "array", + "items": { + "type": "object", + "required": ["option_name", "option_value"], + "properties": { + "option_name": { + "title": "Name", + "type": "string", + "order": 0 + }, + "option_value": { + "title": "Value", + "type": "string", + "order": 1 + } + } + } + } + } + } } } }, @@ -159,6 +206,19 @@ "predicate_key": ["auth_type"], "predicate_value": "oauth2.0", "oauth_config_specification": { + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "properties": { + "region": { + "type": "string", + "path_in_connector_config": ["region"] + }, + "account_type": { + "type": "string", + "path_in_connector_config": ["account_type"] + } + } + }, "complete_oauth_output_specification": { "type": "object", "additionalProperties": false, diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py index d347fa185d87..c0d4afcb3c60 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/streams.py @@ -2,35 +2,35 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import csv +import gzip import json as json_lib import time -import zlib from abc import ABC, abstractmethod +from enum import Enum from io import StringIO from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Union -from urllib.parse import urljoin +import dateparser import pendulum import requests import xmltodict from airbyte_cdk.entrypoint import logger from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator -from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, RequestBodyException -from airbyte_cdk.sources.streams.http.http import BODY_REQUEST_METHODS +from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException from airbyte_cdk.sources.streams.http.rate_limiting import default_backoff_handler from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from source_amazon_seller_partner.auth import AWSSignature +from airbyte_cdk.utils.traced_exception import AirbyteTracedException -REPORTS_API_VERSION = "2021-06-30" # 2020-09-04 +REPORTS_API_VERSION = "2021-06-30" ORDERS_API_VERSION = "v0" VENDORS_API_VERSION = "v1" FINANCES_API_VERSION = "v0" DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +DATE_FORMAT = "%Y-%m-%d" class AmazonSPStream(HttpStream, ABC): @@ -39,14 +39,11 @@ class AmazonSPStream(HttpStream, ABC): def __init__( self, url_base: str, - aws_signature: AWSSignature, replication_start_date: str, marketplace_id: str, period_in_days: Optional[int], - report_options: Optional[str], - advanced_stream_options: Optional[str], - max_wait_seconds: Optional[int], replication_end_date: Optional[str], + report_options: Optional[List[Mapping[str, Any]]] = None, *args, **kwargs, ): @@ -56,7 +53,6 @@ def __init__( self._replication_start_date = replication_start_date self._replication_end_date = replication_end_date self.marketplace_id = marketplace_id - self._session.auth = aws_signature @property def url_base(self) -> str: @@ -103,14 +99,16 @@ def request_params( if next_page_token: return dict(next_page_token) - params = {self.replication_start_date_field: self._replication_start_date, self.page_size_field: self.page_size} + start_date = self._replication_start_date + params = {self.replication_start_date_field: start_date, self.page_size_field: self.page_size} - if self._replication_start_date and self.cursor_field: + if self.cursor_field: start_date = max(stream_state.get(self.cursor_field, self._replication_start_date), self._replication_start_date) - params.update({self.replication_start_date_field: start_date}) + start_date = min(start_date, pendulum.now("utc").to_date_string()) + params[self.replication_start_date_field] = start_date if self._replication_end_date: - params[self.replication_end_date_field] = self._replication_end_date + params[self.replication_end_date_field] = max(self._replication_end_date, start_date) return params @@ -121,7 +119,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, return {self.next_page_token_field: next_page_token} def parse_response( - self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, **kwargss + self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, **kwargs ) -> Iterable[Mapping]: """ :return an iterable containing each record in the response @@ -139,7 +137,16 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late return {self.cursor_field: latest_benchmark} -class ReportsAmazonSPStream(Stream, ABC): +class ReportProcessingStatus(str, Enum): + cancelled = "CANCELLED" + done = "DONE" + fatal = "FATAL" + in_progress = "IN_PROGRESS" + in_queue = "IN_QUEUE" + + +class ReportsAmazonSPStream(HttpStream, ABC): + max_wait_seconds = 3600 """ API docs: https://github.com/amzn/selling-partner-api-docs/blob/main/references/reports-api/reports_2020-09-04.md API model: https://github.com/amzn/selling-partner-api-models/blob/main/models/reports-api-model/reports_2020-09-04.json @@ -163,41 +170,42 @@ class ReportsAmazonSPStream(Stream, ABC): availability_sla_days = ( 1 # see data availability sla at https://developer-docs.amazon.com/sp-api/docs/report-type-values#vendor-retail-analytics-reports ) + availability_strategy = None def __init__( self, url_base: str, - aws_signature: AWSSignature, replication_start_date: str, marketplace_id: str, period_in_days: Optional[int], - report_options: Optional[str], - max_wait_seconds: Optional[int], replication_end_date: Optional[str], - advanced_stream_options: Optional[str], - authenticator: HttpAuthenticator = None, + report_options: Optional[List[Mapping[str, Any]]] = None, + *args, + **kwargs, ): - self._authenticator = authenticator - self._session = requests.Session() + super().__init__(*args, **kwargs) self._url_base = url_base.rstrip("/") + "/" - self._session.auth = aws_signature self._replication_start_date = replication_start_date self._replication_end_date = replication_end_date self.marketplace_id = marketplace_id self.period_in_days = max(period_in_days, self.replication_start_date_limit_in_days) # ensure old configs work as well - self._report_options = report_options or "{}" - self.max_wait_seconds = max_wait_seconds - self._advanced_stream_options = dict() - if advanced_stream_options is not None: - self._advanced_stream_options = json_lib.loads(advanced_stream_options) + self._report_options = report_options + self._http_method = "GET" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None @property - def url_base(self) -> str: - return self._url_base + def http_method(self) -> str: + return self._http_method + + @http_method.setter + def http_method(self, value: str): + self._http_method = value @property - def authenticator(self) -> HttpAuthenticator: - return self._authenticator + def url_base(self) -> str: + return self._url_base def request_params(self) -> MutableMapping[str, Any]: return {"MarketplaceIds": self.marketplace_id} @@ -208,37 +216,6 @@ def request_headers(self) -> Mapping[str, Any]: def path(self, document_id: str) -> str: return f"{self.path_prefix}/documents/{document_id}" - def should_retry(self, response: requests.Response) -> bool: - return response.status_code == 429 or 500 <= response.status_code < 600 - - @default_backoff_handler(max_tries=5, factor=5) - def _send_request(self, request: requests.PreparedRequest) -> requests.Response: - response: requests.Response = self._session.send(request) - if self.should_retry(response): - raise DefaultBackoffException(request=request, response=response) - else: - response.raise_for_status() - return response - - def _create_prepared_request( - self, path: str, http_method: str = "GET", headers: Mapping = None, params: Mapping = None, json: Any = None, data: Any = None - ) -> requests.PreparedRequest: - """ - Override to make http_method configurable per method call - """ - args = {"method": http_method, "url": urljoin(self.url_base, path), "headers": headers, "params": params} - if http_method.upper() in BODY_REQUEST_METHODS: - if json and data: - raise RequestBodyException( - "At the same time only one of the 'request_body_data' and 'request_body_json' functions can return data" - ) - elif json: - args["json"] = json - elif data: - args["data"] = data - - return self._session.prepare_request(requests.Request(**args)) - def _report_data( self, sync_mode: SyncMode, @@ -261,13 +238,14 @@ def _create_report( ) -> Mapping[str, Any]: request_headers = self.request_headers() report_data = self._report_data(sync_mode, cursor_field, stream_slice, stream_state) + self.http_method = "POST" create_report_request = self._create_prepared_request( - http_method="POST", path=f"{self.path_prefix}/reports", headers=dict(request_headers, **self.authenticator.get_auth_header()), data=json_lib.dumps(report_data), ) - report_response = self._send_request(create_report_request) + report_response = self._send_request(create_report_request, {}) + self.http_method = "GET" # rollback return report_response.json() def _retrieve_report(self, report_id: str) -> Mapping[str, Any]: @@ -276,26 +254,28 @@ def _retrieve_report(self, report_id: str) -> Mapping[str, Any]: path=f"{self.path_prefix}/reports/{report_id}", headers=dict(request_headers, **self.authenticator.get_auth_header()), ) - retrieve_report_response = self._send_request(retrieve_report_request) + retrieve_report_response = self._send_request(retrieve_report_request, {}) report_payload = retrieve_report_response.json() return report_payload - def decompress_report_document(self, url, payload): + @default_backoff_handler(factor=5, max_tries=5) + def download_and_decompress_report_document(self, payload: dict) -> str: """ Unpacks a report document """ - report = requests.get(url).content + report = requests.get(payload.get("url")) + report.raise_for_status() if "compressionAlgorithm" in payload: - return zlib.decompress(bytearray(report), 15 + 32).decode("iso-8859-1") - return report.decode("iso-8859-1") + return gzip.decompress(report.content).decode("iso-8859-1") + return report.content.decode("iso-8859-1") def parse_response( self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, **kwargs ) -> Iterable[Mapping]: payload = response.json() - document = self.decompress_report_document(payload.get("url"), payload) + document = self.download_and_decompress_report_document(payload) document_records = self.parse_document(document) yield from document_records @@ -303,23 +283,17 @@ def parse_response( def parse_document(self, document): return csv.DictReader(StringIO(document), delimiter="\t") - def report_options(self) -> Mapping[str, Any]: - if self._report_options is not None: - return json_lib.loads(self._report_options).get(self.name) - else: - return {} + def report_options(self) -> Optional[Mapping[str, Any]]: + return {option.get("option_name"): option.get("option_value") for option in self._report_options} if self._report_options else None def stream_slices( self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - start_date = max(pendulum.parse(self._replication_start_date), pendulum.now("utc").subtract(days=90)) - end_date = pendulum.now() - if self._replication_end_date and sync_mode == SyncMode.full_refresh: - # if replication_start_date is older than 90 days(from current date), we are overriding the value above. - # when replication_end_date is present, we should use the user provided replication_start_date. - # user may provide a date range which is older than 90 days. + now = pendulum.now("utc") + start_date = pendulum.parse(self._replication_start_date) + end_date = now + if self._replication_end_date: end_date = min(end_date, pendulum.parse(self._replication_end_date)) - start_date = pendulum.parse(self._replication_start_date) if stream_state: state = stream_state.get(self.cursor_field) @@ -346,23 +320,37 @@ def read_records( Decrypt and parse the report is its fully proceed, then yield the report document records. """ report_payload = {} - is_processed = False - is_done = False + stream_slice = stream_slice or {} start_time = pendulum.now("utc") seconds_waited = 0 - report_id = self._create_report(sync_mode, cursor_field, stream_slice, stream_state)["reportId"] + try: + report_id = self._create_report(sync_mode, cursor_field, stream_slice, stream_state)["reportId"] + except DefaultBackoffException as e: + logger.warning(f"The report for stream '{self.name}' was cancelled due to several failed retry attempts. {e}") + return [] + except requests.exceptions.HTTPError as e: + if e.response.status_code == requests.codes.FORBIDDEN: + logger.warning( + f"The endpoint {e.response.url} returned {e.response.status_code}: {e.response.reason}. " + "This is most likely due to insufficient permissions on the credentials in use. " + "Try to grant required permissions/scopes or re-authenticate." + ) + return [] + raise e # create and retrieve the report - while not is_processed and seconds_waited < self.max_wait_seconds: + processed = False + while not processed and seconds_waited < self.max_wait_seconds: report_payload = self._retrieve_report(report_id=report_id) seconds_waited = (pendulum.now("utc") - start_time).seconds - is_processed = report_payload.get("processingStatus") not in ["IN_QUEUE", "IN_PROGRESS"] - is_done = report_payload.get("processingStatus") == "DONE" - is_cancelled = report_payload.get("processingStatus") == "CANCELLED" - is_fatal = report_payload.get("processingStatus") == "FATAL" - time.sleep(self.sleep_seconds) + processed = report_payload.get("processingStatus") not in (ReportProcessingStatus.in_queue, ReportProcessingStatus.in_progress) + if not processed: + time.sleep(self.sleep_seconds) + + processing_status = report_payload.get("processingStatus") + report_end_date = pendulum.parse(report_payload.get("dataEndTime", stream_slice.get("dataEndTime"))) - if is_done: + if processing_status == ReportProcessingStatus.done: # retrieve and decrypt the report document document_id = report_payload["reportDocumentId"] request_headers = self.request_headers() @@ -371,47 +359,78 @@ def read_records( headers=dict(request_headers, **self.authenticator.get_auth_header()), params=self.request_params(), ) - response = self._send_request(request) - yield from self.parse_response(response, stream_state, stream_slice) - elif is_fatal: - raise Exception(f"The report for stream '{self.name}' was aborted due to a fatal error") - elif is_cancelled: - logger.warn(f"The report for stream '{self.name}' was cancelled or there is no data to return") + response = self._send_request(request, {}) + for record in self.parse_response(response, stream_state, stream_slice): + if report_end_date: + record["dataEndTime"] = report_end_date.strftime(DATE_FORMAT) + yield record + elif processing_status == ReportProcessingStatus.fatal: + raise AirbyteTracedException(message=f"The report for stream '{self.name}' was not created - skip reading") + elif processing_status == ReportProcessingStatus.cancelled: + logger.warning(f"The report for stream '{self.name}' was cancelled or there is no data to return") else: - raise Exception(f"Unknown response for stream `{self.name}`. Response body {report_payload}") + raise Exception(f"Unknown response for stream '{self.name}'. Response body {report_payload}") -class MerchantListingsReports(ReportsAmazonSPStream): - name = "GET_MERCHANT_LISTINGS_ALL_DATA" +class IncrementalReportsAmazonSPStream(ReportsAmazonSPStream): + @property + def cursor_field(self) -> Union[str, List[str]]: + return "dataEndTime" + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object + and returning an updated state object. + """ + latest_benchmark = latest_record[self.cursor_field] + if current_stream_state.get(self.cursor_field): + return {self.cursor_field: max(latest_benchmark, current_stream_state[self.cursor_field])} + return {self.cursor_field: latest_benchmark} -class FlatFileOrdersReports(ReportsAmazonSPStream): - """ - Field definitions: https://sellercentral.amazon.com/gp/help/help.html?itemID=201648780 - """ - name = "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL" +class MerchantReports(IncrementalReportsAmazonSPStream, ABC): + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.transformer.registerCustomTransform(self.get_transform_function()) -class FbaAfnInventoryReports(ReportsAmazonSPStream): - """ - Field definitions: https://developer-docs.amazon.com/sp-api/docs/report-type-values#inventory-reports - Report does seem to have an long-running issue (sometimes failing without a reason): https://github.com/amzn/selling-partner-api-docs/issues/2231 - """ + @staticmethod + def get_transform_function(): + def transform_function(original_value: Any, field_schema: Dict[str, Any]) -> Any: + if original_value and field_schema.get("format") == "date-time": + # open-date field is returned in format "2022-07-11 01:34:18 PDT" + transformed_value = dateparser.parse(original_value).isoformat() + return transformed_value + return original_value - name = "GET_AFN_INVENTORY_DATA" + return transform_function -class FbaAfnInventoryByCountryReports(ReportsAmazonSPStream): +class MerchantListingsReports(MerchantReports): + name = "GET_MERCHANT_LISTINGS_ALL_DATA" + primary_key = "listing-id" + + +class NetPureProductMarginReport(IncrementalReportsAmazonSPStream): + name = "GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT" + + +class RapidRetailAnalyticsInventoryReport(IncrementalReportsAmazonSPStream): + name = "GET_VENDOR_REAL_TIME_INVENTORY_REPORT" + + +class FlatFileOrdersReports(IncrementalReportsAmazonSPStream): """ - Field definitions: https://developer-docs.amazon.com/sp-api/docs/report-type-values#inventory-reports - Report does seem to have an long-running issue (sometimes failing without a reason): https://github.com/amzn/selling-partner-api-docs/issues/2231 + Field definitions: https://sellercentral.amazon.com/gp/help/help.html?itemID=201648780 """ - name = "GET_AFN_INVENTORY_DATA_BY_COUNTRY" + name = "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL" + primary_key = "amazon-order-id" + cursor_field = "last-updated-date" -class FbaStorageFeesReports(ReportsAmazonSPStream): +class FbaStorageFeesReports(IncrementalReportsAmazonSPStream): """ Field definitions: https://sellercentral.amazon.com/help/hub/reference/G202086720 """ @@ -419,7 +438,7 @@ class FbaStorageFeesReports(ReportsAmazonSPStream): name = "GET_FBA_STORAGE_FEE_CHARGES_DATA" -class FulfilledShipmentsReports(ReportsAmazonSPStream): +class FulfilledShipmentsReports(IncrementalReportsAmazonSPStream): """ Field definitions: https://sellercentral.amazon.com/gp/help/help.html?itemID=200453120 """ @@ -429,19 +448,20 @@ class FulfilledShipmentsReports(ReportsAmazonSPStream): replication_start_date_limit_in_days = 30 -class FlatFileOpenListingsReports(ReportsAmazonSPStream): +class FlatFileOpenListingsReports(IncrementalReportsAmazonSPStream): name = "GET_FLAT_FILE_OPEN_LISTINGS_DATA" -class FbaOrdersReports(ReportsAmazonSPStream): +class FbaOrdersReports(IncrementalReportsAmazonSPStream): """ Field definitions: https://sellercentral.amazon.com/gp/help/help.html?itemID=200989110 """ name = "GET_FBA_FULFILLMENT_REMOVAL_ORDER_DETAIL_DATA" + cursor_field = "last-updated-date" -class FlatFileActionableOrderDataShipping(ReportsAmazonSPStream): +class FlatFileActionableOrderDataShipping(IncrementalReportsAmazonSPStream): """ Field definitions: https://developer-docs.amazon.com/sp-api/docs/order-reports-attributes#get_flat_file_actionable_order_data_shipping """ @@ -449,7 +469,7 @@ class FlatFileActionableOrderDataShipping(ReportsAmazonSPStream): name = "GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING" -class OrderReportDataShipping(ReportsAmazonSPStream): +class OrderReportDataShipping(IncrementalReportsAmazonSPStream): """ Field definitions: https://developer-docs.amazon.com/sp-api/docs/order-reports-attributes#get_order_report_data_shipping """ @@ -457,7 +477,12 @@ class OrderReportDataShipping(ReportsAmazonSPStream): name = "GET_ORDER_REPORT_DATA_SHIPPING" def parse_document(self, document): - parsed = xmltodict.parse(document, attr_prefix="", cdata_key="value", force_list={"Message"}) + try: + parsed = xmltodict.parse(document, attr_prefix="", cdata_key="value", force_list={"Message"}) + except Exception as e: + self.logger.warning(f"Unable to parse the report for the stream {self.name}, error: {str(e)}") + return [] + reports = parsed.get("AmazonEnvelope", {}).get("Message", {}) result = [] for report in reports: @@ -466,7 +491,7 @@ def parse_document(self, document): return result -class FbaShipmentsReports(ReportsAmazonSPStream): +class FbaShipmentsReports(IncrementalReportsAmazonSPStream): """ Field definitions: https://sellercentral.amazon.com/gp/help/help.html?itemID=200989100 """ @@ -474,7 +499,7 @@ class FbaShipmentsReports(ReportsAmazonSPStream): name = "GET_FBA_FULFILLMENT_REMOVAL_SHIPMENT_DETAIL_DATA" -class FbaReplacementsReports(ReportsAmazonSPStream): +class FbaReplacementsReports(IncrementalReportsAmazonSPStream): """ Field definitions: https://sellercentral.amazon.com/help/hub/reference/200453300 """ @@ -482,7 +507,7 @@ class FbaReplacementsReports(ReportsAmazonSPStream): name = "GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_REPLACEMENT_DATA" -class RestockInventoryReports(ReportsAmazonSPStream): +class RestockInventoryReports(IncrementalReportsAmazonSPStream): """ Field definitions: https://sellercentral.amazon.com/help/hub/reference/202105670 """ @@ -490,59 +515,56 @@ class RestockInventoryReports(ReportsAmazonSPStream): name = "GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT" -class GetXmlBrowseTreeData(ReportsAmazonSPStream): +class GetXmlBrowseTreeData(IncrementalReportsAmazonSPStream): def parse_document(self, document): - parsed = xmltodict.parse( - document, dict_constructor=dict, attr_prefix="", cdata_key="text", force_list={"attribute", "id", "refinementField"} - ) + try: + parsed = xmltodict.parse( + document, dict_constructor=dict, attr_prefix="", cdata_key="text", force_list={"attribute", "id", "refinementField"} + ) + except Exception as e: + self.logger.warning(f"Unable to parse the report for the stream {self.name}, error: {str(e)}") + return [] + return parsed.get("Result", {}).get("Node", []) name = "GET_XML_BROWSE_TREE_DATA" + primary_key = "browseNodeId" -class FbaEstimatedFbaFeesTxtReport(ReportsAmazonSPStream): +class FbaEstimatedFbaFeesTxtReport(IncrementalReportsAmazonSPStream): name = "GET_FBA_ESTIMATED_FBA_FEES_TXT_DATA" -class FbaFulfillmentCurrentInventoryReport(ReportsAmazonSPStream): - name = "GET_FBA_FULFILLMENT_CURRENT_INVENTORY_DATA" - - -class FbaFulfillmentCustomerShipmentPromotionReport(ReportsAmazonSPStream): +class FbaFulfillmentCustomerShipmentPromotionReport(IncrementalReportsAmazonSPStream): name = "GET_FBA_FULFILLMENT_CUSTOMER_SHIPMENT_PROMOTION_DATA" -class FbaFulfillmentInventoryAdjustReport(ReportsAmazonSPStream): - name = "GET_FBA_FULFILLMENT_INVENTORY_ADJUSTMENTS_DATA" - - -class FbaFulfillmentInventoryReceiptsReport(ReportsAmazonSPStream): - name = "GET_FBA_FULFILLMENT_INVENTORY_RECEIPTS_DATA" - - -class FbaFulfillmentInventorySummaryReport(ReportsAmazonSPStream): - name = "GET_FBA_FULFILLMENT_INVENTORY_SUMMARY_DATA" - - -class FbaMyiUnsuppressedInventoryReport(ReportsAmazonSPStream): +class FbaMyiUnsuppressedInventoryReport(IncrementalReportsAmazonSPStream): name = "GET_FBA_MYI_UNSUPPRESSED_INVENTORY_DATA" -class MerchantListingsReport(ReportsAmazonSPStream): +class MerchantListingsReport(MerchantReports): name = "GET_MERCHANT_LISTINGS_DATA" + primary_key = "listing-id" -class MerchantListingsInactiveData(ReportsAmazonSPStream): +class MerchantListingsInactiveData(MerchantReports): name = "GET_MERCHANT_LISTINGS_INACTIVE_DATA" + primary_key = "listing-id" -class StrandedInventoryUiReport(ReportsAmazonSPStream): +class StrandedInventoryUiReport(IncrementalReportsAmazonSPStream): name = "GET_STRANDED_INVENTORY_UI_DATA" -class XmlAllOrdersDataByOrderDataGeneral(ReportsAmazonSPStream): +class XmlAllOrdersDataByOrderDataGeneral(IncrementalReportsAmazonSPStream): def parse_document(self, document): - parsed = xmltodict.parse(document, attr_prefix="", cdata_key="value", force_list={"Message", "OrderItem"}) + try: + parsed = xmltodict.parse(document, attr_prefix="", cdata_key="value", force_list={"Message", "OrderItem"}) + except Exception as e: + self.logger.warning(f"Unable to parse the report for the stream {self.name}, error: {str(e)}") + return [] + orders = parsed.get("AmazonEnvelope", {}).get("Message", []) result = [] if isinstance(orders, list): @@ -552,50 +574,64 @@ def parse_document(self, document): return result name = "GET_XML_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL" + primary_key = "AmazonOrderID" + cursor_field = "LastUpdatedDate" -class MerchantListingsReportBackCompat(ReportsAmazonSPStream): +class MerchantListingsReportBackCompat(MerchantReports): name = "GET_MERCHANT_LISTINGS_DATA_BACK_COMPAT" + primary_key = "listing-id" -class MerchantCancelledListingsReport(ReportsAmazonSPStream): +class MerchantCancelledListingsReport(IncrementalReportsAmazonSPStream): name = "GET_MERCHANT_CANCELLED_LISTINGS_DATA" -class FbaFulfillmentMonthlyInventoryReport(ReportsAmazonSPStream): - name = "GET_FBA_FULFILLMENT_MONTHLY_INVENTORY_DATA" +class MerchantListingsFypReport(IncrementalReportsAmazonSPStream): + name = "GET_MERCHANTS_LISTINGS_FYP_REPORT" + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.transformer.registerCustomTransform(self.get_transform_function()) -class MerchantListingsFypReport(ReportsAmazonSPStream): - name = "GET_MERCHANTS_LISTINGS_FYP_REPORT" + @staticmethod + def get_transform_function(): + def transform_function(original_value: Any, field_schema: Dict[str, Any]) -> Any: + if original_value and field_schema.get("format") == "date": + try: + transformed_value = pendulum.from_format(original_value, "MMM D[,] YYYY").to_date_string() + return transformed_value + except ValueError: + pass + return original_value + + return transform_function -class FbaSnsForecastReport(ReportsAmazonSPStream): +class FbaSnsForecastReport(IncrementalReportsAmazonSPStream): name = "GET_FBA_SNS_FORECAST_DATA" -class FbaSnsPerformanceReport(ReportsAmazonSPStream): +class FbaSnsPerformanceReport(IncrementalReportsAmazonSPStream): name = "GET_FBA_SNS_PERFORMANCE_DATA" -class FlatFileArchivedOrdersDataByOrderDate(ReportsAmazonSPStream): +class FlatFileArchivedOrdersDataByOrderDate(IncrementalReportsAmazonSPStream): name = "GET_FLAT_FILE_ARCHIVED_ORDERS_DATA_BY_ORDER_DATE" + cursor_field = "last-updated-date" -class FlatFileReturnsDataByReturnDate(ReportsAmazonSPStream): +class FlatFileReturnsDataByReturnDate(IncrementalReportsAmazonSPStream): name = "GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE" replication_start_date_limit_in_days = 60 -class FbaInventoryPlaningReport(ReportsAmazonSPStream): +class FbaInventoryPlaningReport(IncrementalReportsAmazonSPStream): name = "GET_FBA_INVENTORY_PLANNING_DATA" -class LedgerSummaryViewReport(ReportsAmazonSPStream): - name = "GET_LEDGER_SUMMARY_VIEW_DATA" - - class AnalyticsStream(ReportsAmazonSPStream): def parse_document(self, document): parsed = json_lib.loads(document) @@ -611,10 +647,9 @@ def _report_data( data = super()._report_data(sync_mode, cursor_field, stream_slice, stream_state) options = self.report_options() if options and options.get("reportPeriod") is not None: - data.update(self._augmented_data(self, options)) + data.update(self._augmented_data(options)) return data - @staticmethod def _augmented_data(self, report_options) -> Mapping[str, Any]: now = pendulum.now("utc") if report_options["reportPeriod"] == "DAY": @@ -648,36 +683,109 @@ def _augmented_data(self, report_options) -> Mapping[str, Any]: } -class BrandAnalyticsMarketBasketReports(AnalyticsStream): +class IncrementalAnalyticsStream(AnalyticsStream): + + fixed_period_in_days = 0 + + @property + def cursor_field(self) -> Union[str, List[str]]: + return "endDate" + + def _report_data( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Mapping[str, Any]: + data = super()._report_data(sync_mode, cursor_field, stream_slice, stream_state) + if stream_slice: + data_times = {} + if stream_slice.get("dataStartTime"): + data_times["dataStartTime"] = stream_slice["dataStartTime"] + if stream_slice.get("dataEndTime"): + data_times["dataEndTime"] = stream_slice["dataEndTime"] + data.update(data_times) + + return data + + def parse_response( + self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, **kwargs + ) -> Iterable[Mapping]: + + payload = response.json() + + document = self.download_and_decompress_report_document(payload) + document_records = self.parse_document(document) + + # Not all (partial) responses include the request date, so adding it manually here + for record in document_records: + if stream_slice.get("dataEndTime"): + record["queryEndDate"] = pendulum.parse(stream_slice["dataEndTime"]).strftime("%Y-%m-%d") + yield record + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object + and returning an updated state object. + """ + latest_benchmark = latest_record[self.cursor_field] + if current_stream_state.get(self.cursor_field): + return {self.cursor_field: max(latest_benchmark, current_stream_state[self.cursor_field])} + return {self.cursor_field: latest_benchmark} + + def stream_slices( + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + + start_date = pendulum.parse(self._replication_start_date) + end_date = pendulum.now("utc").subtract(days=self.availability_sla_days) + + if self._replication_end_date: + end_date = pendulum.parse(self._replication_end_date) + + if stream_state: + state = stream_state.get(self.cursor_field) + start_date = pendulum.parse(state) + + start_date = min(start_date, end_date) + + while start_date < end_date: + # If request only returns data on day level + if self.fixed_period_in_days != 0: + slice_range = self.fixed_period_in_days + else: + slice_range = self.period_in_days + + end_date_slice = start_date.add(days=slice_range) + yield { + "dataStartTime": start_date.strftime(DATE_TIME_FORMAT), + "dataEndTime": min(end_date_slice.subtract(seconds=1), end_date).strftime(DATE_TIME_FORMAT), + } + start_date = end_date_slice + + +class BrandAnalyticsMarketBasketReports(IncrementalAnalyticsStream): name = "GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT" result_key = "dataByAsin" -class BrandAnalyticsSearchTermsReports(AnalyticsStream): +class BrandAnalyticsSearchTermsReports(IncrementalAnalyticsStream): """ Field definitions: https://sellercentral.amazon.co.uk/help/hub/reference/G5NXWNY8HUD3VDCW """ name = "GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT" result_key = "dataByDepartmentAndSearchTerm" + cursor_field = "queryEndDate" -class BrandAnalyticsRepeatPurchaseReports(AnalyticsStream): +class BrandAnalyticsRepeatPurchaseReports(IncrementalAnalyticsStream): name = "GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT" result_key = "dataByAsin" -class BrandAnalyticsAlternatePurchaseReports(AnalyticsStream): - name = "GET_BRAND_ANALYTICS_ALTERNATE_PURCHASE_REPORT" - result_key = "dataByAsin" - - -class BrandAnalyticsItemComparisonReports(AnalyticsStream): - name = "GET_BRAND_ANALYTICS_ITEM_COMPARISON_REPORT" - result_key = "dataByAsin" - - -class VendorInventoryReports(AnalyticsStream): +class VendorInventoryReports(IncrementalAnalyticsStream): """ Field definitions: https://developer-docs.amazon.com/sp-api/docs/report-type-values#vendor-retail-analytics-reports """ @@ -687,21 +795,26 @@ class VendorInventoryReports(AnalyticsStream): availability_sla_days = 3 -class IncrementalReportsAmazonSPStream(ReportsAmazonSPStream): - @property - @abstractmethod - def cursor_field(self) -> Union[str, List[str]]: - pass +class VendorTrafficReport(IncrementalAnalyticsStream): + name = "GET_VENDOR_TRAFFIC_REPORT" + result_key = "trafficByAsin" - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object - and returning an updated state object. - """ - latest_benchmark = latest_record[self.cursor_field] - if current_stream_state.get(self.cursor_field): - return {self.cursor_field: max(latest_benchmark, current_stream_state[self.cursor_field])} - return {self.cursor_field: latest_benchmark} + +class SellerAnalyticsSalesAndTrafficReports(IncrementalAnalyticsStream): + """ + Field definitions: https://developer-docs.amazon.com/sp-api/docs/report-type-values#seller-retail-analytics-reports + """ + + name = "GET_SALES_AND_TRAFFIC_REPORT" + result_key = "salesAndTrafficByAsin" + cursor_field = "queryEndDate" + fixed_period_in_days = 1 + + +class VendorSalesReports(IncrementalAnalyticsStream): + name = "GET_VENDOR_SALES_REPORT" + result_key = "salesByAsin" + availability_sla_days = 4 # Data is only available after 4 days class SellerFeedbackReports(IncrementalReportsAmazonSPStream): @@ -761,8 +874,7 @@ def transform_function(original_value: Any, field_schema: Dict[str, Any]) -> Any return transform_function # csv header field names for this report differ per marketplace (are localized to marketplace language) - # but columns come in the same order - # so we set fieldnames to our custom ones + # but columns come in the same order, so we set fieldnames to our custom ones # and raise error if original and custom header field count does not match @staticmethod def parse_document(document): @@ -774,13 +886,33 @@ def parse_document(document): return reader +class FbaAfnInventoryReports(IncrementalReportsAmazonSPStream): + """ + Field definitions: https://developer-docs.amazon.com/sp-api/docs/report-type-values#inventory-reports + Report has a long-running issue (fails when requested frequently): https://github.com/amzn/selling-partner-api-docs/issues/2231 + """ + + name = "GET_AFN_INVENTORY_DATA" + + +class FbaAfnInventoryByCountryReports(IncrementalReportsAmazonSPStream): + """ + Field definitions: https://developer-docs.amazon.com/sp-api/docs/report-type-values#inventory-reports + Report has a long-running issue (fails when requested frequently): https://github.com/amzn/selling-partner-api-docs/issues/2231 + """ + + name = "GET_AFN_INVENTORY_DATA_BY_COUNTRY" + + class FlatFileOrdersReportsByLastUpdate(IncrementalReportsAmazonSPStream): """ Field definitions: https://sellercentral.amazon.com/gp/help/help.html?itemID=201648780 """ name = "GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL" + primary_key = "amazon-order-id" cursor_field = "last-updated-date" + replication_start_date_limit_in_days = 30 class Orders(IncrementalAmazonSPStream): @@ -806,7 +938,7 @@ def request_params( self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs ) -> MutableMapping[str, Any]: params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) - params.update({"MarketplaceIds": self.marketplace_id}) + params["MarketplaceIds"] = self.marketplace_id return params def parse_response( @@ -822,7 +954,7 @@ def backoff_time(self, response: requests.Response) -> Optional[float]: return self.default_backoff_time -class OrderItems(AmazonSPStream, ABC): +class OrderItems(IncrementalAmazonSPStream): """ API docs: https://developer-docs.amazon.com/sp-api/docs/orders-api-v0-reference#getorderitems API model: https://developer-docs.amazon.com/sp-api/docs/orders-api-v0-reference#orderitemslist @@ -834,6 +966,8 @@ class OrderItems(AmazonSPStream, ABC): parent_cursor_field = "LastUpdateDate" next_page_token_field = "NextToken" stream_slice_cursor_field = "AmazonOrderId" + replication_start_date_field = "LastUpdatedAfter" + replication_end_date_field = "LastUpdatedBefore" page_size_field = None default_backoff_time = 10 default_stream_slice_delay_time = 1 @@ -857,19 +991,12 @@ def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Ite orders = Orders(**self.stream_kwargs) for order_record in orders.read_records(sync_mode=SyncMode.incremental, stream_state=stream_state): self.cached_state[self.parent_cursor_field] = order_record[self.parent_cursor_field] - self.logger.info(f"OrderItems stream slice for order {order_record[self.stream_slice_cursor_field]}") time.sleep(self.default_stream_slice_delay_time) yield { self.stream_slice_cursor_field: order_record[self.stream_slice_cursor_field], self.parent_cursor_field: order_record[self.parent_cursor_field], } - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - latest_benchmark = self.cached_state[self.parent_cursor_field] - if current_stream_state.get(self.parent_cursor_field): - return {self.parent_cursor_field: max(latest_benchmark, current_stream_state[self.parent_cursor_field])} - return {self.parent_cursor_field: latest_benchmark} - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: stream_data = response.json() next_page_token = stream_data.get("payload").get(self.next_page_token_field) @@ -887,7 +1014,6 @@ def parse_response( self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, **kwargs ) -> Iterable[Mapping]: order_items_list = response.json().get(self.data_field, {}) - self.logger.info(f"order_items_list efim {order_items_list}") if order_items_list.get(self.next_page_token_field) is None: self.cached_state[self.parent_cursor_field] = stream_slice[self.parent_cursor_field] for order_item in order_items_list.get(self.name, []): @@ -909,135 +1035,25 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.transformer.registerCustomTransform(self.get_transform_function()) - def get_transform_function(self): + @staticmethod + def get_transform_function(): def transform_function(original_value: Any, field_schema: Dict[str, Any]) -> Any: if original_value and field_schema.get("format") == "date": - transformed_value = pendulum.from_format(original_value, "MM/DD/YYYY").to_date_string() - return transformed_value + try: + transformed_value = pendulum.from_format(original_value, "MM/DD/YYYY").to_date_string() + return transformed_value + except ValueError: + pass return original_value return transform_function -class IncrementalAnalyticsStream(AnalyticsStream): - - fixed_period_in_days = 0 - - @property - @abstractmethod - def cursor_field(self) -> Union[str, List[str]]: - pass - - def _report_data( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - ) -> Mapping[str, Any]: - data = super()._report_data(sync_mode, cursor_field, stream_slice, stream_state) - if stream_slice: - data_times = {} - if stream_slice.get("dataStartTime"): - data_times["dataStartTime"] = stream_slice["dataStartTime"] - if stream_slice.get("dataEndTime"): - data_times["dataEndTime"] = stream_slice["dataEndTime"] - data.update(data_times) - - return data - - def parse_response( - self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, **kwargs - ) -> Iterable[Mapping]: - - payload = response.json() - - document = self.decompress_report_document( - payload.get("url"), - payload, - ) - document_records = self.parse_document(document) - - # Not all (partial) responses include the request date, so adding it manually here - for record in document_records: - if stream_slice.get("dataEndTime"): - record["queryEndDate"] = pendulum.parse(stream_slice["dataEndTime"]).strftime("%Y-%m-%d") - yield record - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object - and returning an updated state object. - """ - latest_benchmark = latest_record[self.cursor_field] - if current_stream_state.get(self.cursor_field): - return {self.cursor_field: max(latest_benchmark, current_stream_state[self.cursor_field])} - return {self.cursor_field: latest_benchmark} - - def stream_slices( - self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: - - start_date = pendulum.parse(self._replication_start_date) - end_date = pendulum.now().subtract(days=self.availability_sla_days) - - if self._replication_end_date: - end_date = pendulum.parse(self._replication_end_date) - - if stream_state: - state = stream_state.get(self.cursor_field) - start_date = pendulum.parse(state) - - start_date = min(start_date, end_date) - slices = [] - - while start_date < end_date: - # If request only returns data on day level - if self.fixed_period_in_days != 0: - slice_range = self.fixed_period_in_days - else: - slice_range = self.period_in_days - - end_date_slice = start_date.add(days=slice_range) - slices.append( - { - "dataStartTime": start_date.strftime(DATE_TIME_FORMAT), - "dataEndTime": min(end_date_slice.subtract(seconds=1), end_date).strftime(DATE_TIME_FORMAT), - } - ) - start_date = end_date_slice - - return slices - - -class SellerAnalyticsSalesAndTrafficReports(IncrementalAnalyticsStream): - """ - Field definitions: https://developer-docs.amazon.com/sp-api/docs/report-type-values#seller-retail-analytics-reports - """ - - name = "GET_SALES_AND_TRAFFIC_REPORT" - result_key = "salesAndTrafficByAsin" - cursor_field = "queryEndDate" - fixed_period_in_days = 1 - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - if self.name in self._advanced_stream_options.keys(): - _options: dict = self._advanced_stream_options[self.name] - if isinstance(_options, dict): - for _option_attr, _option_val in _options.items(): - setattr(self, _option_attr, _option_val) - - -class VendorSalesReports(IncrementalAnalyticsStream): - name = "GET_VENDOR_SALES_REPORT" - result_key = "salesByAsin" - cursor_field = "endDate" - availability_sla_days = 4 # Data is only available after 4 days +class LedgerSummaryViewReport(LedgerDetailedViewReports): + name = "GET_LEDGER_SUMMARY_VIEW_DATA" -class VendorDirectFulfillmentShipping(AmazonSPStream): +class VendorDirectFulfillmentShipping(IncrementalAmazonSPStream): """ API docs: https://github.com/amzn/selling-partner-api-docs/blob/main/references/vendor-direct-fulfillment-shipping-api/vendorDirectFulfillmentShippingV1.md API model: https://github.com/amzn/selling-partner-api-models/blob/main/models/vendor-direct-fulfillment-shipping-api-model/vendorDirectFulfillmentShippingV1.json @@ -1048,12 +1064,13 @@ class VendorDirectFulfillmentShipping(AmazonSPStream): """ name = "VendorDirectFulfillmentShipping" - primary_key = None + primary_key = "purchaseOrderNumber" replication_start_date_field = "createdAfter" replication_end_date_field = "createdBefore" next_page_token_field = "nextToken" page_size_field = "limit" time_format = "%Y-%m-%dT%H:%M:%SZ" + cursor_field = "createdBefore" def path(self, **kwargs) -> str: return f"vendor/directFulfillment/shipping/{VENDORS_API_VERSION}/shippingLabels" @@ -1061,24 +1078,31 @@ def path(self, **kwargs) -> str: def request_params( self, stream_state: Mapping[str, Any], next_page_token: Mapping[str, Any] = None, **kwargs ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) - if not next_page_token: - end_date = pendulum.now("utc").strftime(self.time_format) - if self._replication_end_date: - end_date = self._replication_end_date - - start_date = max(pendulum.parse(self._replication_start_date), pendulum.parse(end_date).subtract(days=7, hours=1)).strftime( - self.time_format - ) + if next_page_token: + return dict(next_page_token) - params.update({self.replication_start_date_field: start_date, self.replication_end_date_field: end_date}) - return params + end_date = pendulum.now("utc").strftime(self.time_format) + if self._replication_end_date: + end_date = self._replication_end_date + # The date range to search must not be more than 7 days - see docs + # https://developer-docs.amazon.com/sp-api/docs/vendor-direct-fulfillment-shipping-api-v1-reference + start_date = max(pendulum.parse(self._replication_start_date), pendulum.parse(end_date).subtract(days=7, hours=1)).strftime( + self.time_format + ) + if stream_state_value := stream_state.get(self.cursor_field): + start_date = max(stream_state_value, start_date) + return {self.replication_start_date_field: start_date, self.replication_end_date_field: end_date} - def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: - yield from response.json().get(self.data_field, {}).get("shippingLabels", []) + def parse_response( + self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, **kwargs + ) -> Iterable[Mapping]: + params = self.request_params(stream_state) + for record in response.json().get(self.data_field, {}).get("shippingLabels", []): + record[self.replication_end_date_field] = params.get(self.replication_end_date_field) + yield record -class FinanceStream(AmazonSPStream, ABC): +class FinanceStream(IncrementalAmazonSPStream, ABC): next_page_token_field = "NextToken" page_size_field = "MaxResultsPerPage" page_size = 100 @@ -1111,6 +1135,10 @@ def request_params( DATE_TIME_FORMAT ) + stream_state = stream_state or {} + if stream_state_value := stream_state.get(self.cursor_field): + start_date = max(stream_state_value, start_date) + # logging to make sure user knows taken start date logger.info("start date used: %s", start_date) @@ -1142,13 +1170,17 @@ class ListFinancialEventGroups(FinanceStream): """ name = "ListFinancialEventGroups" + primary_key = "FinancialEventGroupId" replication_start_date_field = "FinancialEventGroupStartedAfter" replication_end_date_field = "FinancialEventGroupStartedBefore" + cursor_field = "FinancialEventGroupStart" def path(self, **kwargs) -> str: return f"finances/{FINANCES_API_VERSION}/financialEventGroups" - def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + def parse_response( + self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, **kwargs + ) -> Iterable[Mapping]: yield from response.json().get(self.data_field, {}).get("FinancialEventGroupList", []) @@ -1161,22 +1193,42 @@ class ListFinancialEvents(FinanceStream): name = "ListFinancialEvents" replication_start_date_field = "PostedAfter" replication_end_date_field = "PostedBefore" + cursor_field = "PostedBefore" def path(self, **kwargs) -> str: return f"finances/{FINANCES_API_VERSION}/financialEvents" - def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: - yield from [response.json().get(self.data_field, {}).get("FinancialEvents", {})] + def parse_response( + self, response: requests.Response, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, **kwargs + ) -> Iterable[Mapping]: + params = self.request_params(stream_state) + events = response.json().get(self.data_field, {}).get("FinancialEvents", {}) + events[self.replication_end_date_field] = params.get(self.replication_end_date_field) + yield from [events] -class FbaCustomerReturnsReports(ReportsAmazonSPStream): +class FbaCustomerReturnsReports(IncrementalReportsAmazonSPStream): name = "GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA" -class FlatFileSettlementV2Reports(ReportsAmazonSPStream): +class FlatFileSettlementV2Reports(IncrementalReportsAmazonSPStream): name = "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE" + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.transformer.registerCustomTransform(self.get_transform_function()) + + @staticmethod + def get_transform_function(): + def transform_function(original_value: Any, field_schema: Dict[str, Any]) -> Any: + if original_value == "" and field_schema.get("format") == "date-time": + return None + return original_value + + return transform_function def _create_report( self, @@ -1190,7 +1242,7 @@ def _create_report( return {"reportId": stream_slice.get("report_id")} def stream_slices( - self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: """ From https://developer-docs.amazon.com/sp-api/docs/report-type-values @@ -1202,9 +1254,14 @@ def stream_slices( """ strict_start_date = pendulum.now("utc").subtract(days=90) + utc_now = pendulum.now("utc").date().to_date_string() create_date = max(pendulum.parse(self._replication_start_date), strict_start_date) - end_date = pendulum.parse(self._replication_end_date or pendulum.now("utc").date().to_date_string()) + end_date = pendulum.parse(self._replication_end_date or utc_now) + + stream_state = stream_state or {} + if cursor_value := stream_state.get(self.cursor_field): + create_date = pendulum.parse(min(cursor_value, utc_now)) if end_date < strict_start_date: end_date = pendulum.now("utc") @@ -1219,15 +1276,13 @@ def stream_slices( complete = False while not complete: - request_headers = self.request_headers() get_reports = self._create_prepared_request( - http_method="GET", path=f"{self.path_prefix}/reports", headers=dict(request_headers, **self.authenticator.get_auth_header()), params=params, ) - report_response = self._send_request(get_reports) + report_response = self._send_request(get_reports, {}) response = report_response.json() data = response.get("reports", list()) records = [e.get("reportId") for e in data if e and e.get("reportId") not in unique_records] @@ -1242,7 +1297,7 @@ def stream_slices( complete = True -class FbaReimbursementsReports(ReportsAmazonSPStream): +class FbaReimbursementsReports(IncrementalReportsAmazonSPStream): """ Field definitions: https://sellercentral.amazon.com/help/hub/reference/G200732720 """ diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/utils.py b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/utils.py new file mode 100644 index 000000000000..fbc299d456e2 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/source_amazon_seller_partner/utils.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from airbyte_cdk.utils import AirbyteTracedException +from airbyte_protocol.models import FailureType + + +class AmazonConfigException(AirbyteTracedException): + def __init__(self, **kwargs): + failure_type: FailureType = FailureType.config_error + super(AmazonConfigException, self).__init__(failure_type=failure_type, **kwargs) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/conftest.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/conftest.py new file mode 100644 index 000000000000..1dcbb11a25fb --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/conftest.py @@ -0,0 +1,20 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from typing import Any, Dict + +import pytest + + +@pytest.fixture +def report_init_kwargs() -> Dict[str, Any]: + return { + "url_base": "https://test.url", + "replication_start_date": "2022-09-01T00:00:00Z", + "marketplace_id": "market", + "period_in_days": 90, + "report_options": None, + "replication_end_date": None, + } diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_analytics_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_analytics_streams.py new file mode 100644 index 000000000000..61905c59fd3a --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_analytics_streams.py @@ -0,0 +1,148 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from unittest.mock import patch + +import pendulum +import pytest +from airbyte_cdk.models import SyncMode +from source_amazon_seller_partner.streams import AnalyticsStream, IncrementalAnalyticsStream + + +class SomeAnalyticsStream(AnalyticsStream): + name = "GET_ANALYTICS_STREAM" + result_key = "result_key" + availability_sla_days = 3 + + +class SomeIncrementalAnalyticsStream(IncrementalAnalyticsStream): + name = "GET_INCREMENTAL_ANALYTICS_STREAM" + result_key = "result_key" + availability_sla_days = 3 + cursor_field = "endDate" + + +class TestAnalyticsStream: + @pytest.mark.parametrize( + ("input_document", "expected_value"), + ( + ('{"result_key": [{"some_key": "some_value"}]}', [{"some_key": "some_value"}]), + ('{"wrong_result_key": {"some_key": "some_value"}}', []), + ), + ) + def test_parse_document(self, report_init_kwargs, input_document, expected_value): + stream = SomeAnalyticsStream(**report_init_kwargs) + assert stream.parse_document(input_document) == expected_value + + @pytest.mark.parametrize( + ("report_options", "expected_result"), + ( + ({"reportPeriod": "DAY"}, {"dataStartTime": "2023-09-06T00:00:00Z", "dataEndTime": "2023-09-06T23:59:59Z"}), + ({"reportPeriod": "WEEK"}, {"dataStartTime": "2023-08-27T00:00:00Z", "dataEndTime": "2023-09-02T23:59:59Z"}), + ({"reportPeriod": "MONTH"}, {"dataStartTime": "2023-08-01T00:00:00Z", "dataEndTime": "2023-08-31T23:59:59Z"}), + ), + ) + def test_augmented_data(self, report_init_kwargs, report_options, expected_result): + stream = SomeAnalyticsStream(**report_init_kwargs) + expected_result["reportOptions"] = report_options + with patch("pendulum.now", return_value=pendulum.parse("2023-09-09T00:00:00Z")): + assert stream._augmented_data(report_options) == expected_result + + def test_augmented_data_incorrect_period(self, report_init_kwargs): + stream = SomeAnalyticsStream(**report_init_kwargs) + report_options = {"reportPeriod": "DAYS123"} + with pytest.raises(Exception) as e: + stream._augmented_data(report_options) + assert e.value.args[0] == [{'message': 'This reportPeriod is not implemented.'}] + + @pytest.mark.parametrize( + ("report_options", "report_option_dates"), + ( + ( + [{"option_name": "reportPeriod", "option_value": "DAY"}], + {"dataStartTime": "2023-09-06T00:00:00Z", "dataEndTime": "2023-09-06T23:59:59Z", "reportOptions": {"reportPeriod": "DAY"}}, + ), + ([], {}), + ), + ) + def test_report_data(self, report_init_kwargs, report_options, report_option_dates): + report_init_kwargs["report_options"] = report_options + stream = SomeAnalyticsStream(**report_init_kwargs) + expected_data = {"reportType": stream.name, "marketplaceIds": [report_init_kwargs["marketplace_id"]]} + expected_data.update(report_option_dates) + with patch("pendulum.now", return_value=pendulum.parse("2023-09-09T00:00:00Z")): + assert stream._report_data(sync_mode=SyncMode.full_refresh) == expected_data + + +class TestIncrementalAnalyticsStream: + @pytest.mark.parametrize( + "stream_slice", + ( + ({"dataStartTime": "2022-09-01T00:00:00Z", "dataEndTime": "2022-09-02T00:00:00Z"}), + ({"dataEndTime": "2022-09-02T00:00:00Z"}), + ({"dataStartTime": "2022-09-01T00:00:00Z"}), + ({}), + ), + ) + def test_report_data(self, report_init_kwargs, stream_slice): + stream = SomeIncrementalAnalyticsStream(**report_init_kwargs) + expected_data = {"reportType": stream.name, "marketplaceIds": [report_init_kwargs["marketplace_id"]]} + expected_data.update(stream_slice) + assert stream._report_data( + sync_mode=SyncMode.incremental, cursor_field=[stream.cursor_field], stream_slice=stream_slice + ) == expected_data + + @pytest.mark.parametrize( + ("current_stream_state", "latest_record", "expected_date"), + ( + ({"endDate": "2022-10-03T00:00:00Z"}, {"endDate": "2022-10-04T00:00:00Z"}, "2022-10-04T00:00:00Z"), + ({"endDate": "2022-10-04T00:00:00Z"}, {"endDate": "2022-10-03T00:00:00Z"}, "2022-10-04T00:00:00Z"), + ({}, {"endDate": "2022-10-03T00:00:00Z"}, "2022-10-03T00:00:00Z"), + ), + ) + def test_get_updated_state(self, report_init_kwargs, current_stream_state, latest_record, expected_date): + stream = SomeIncrementalAnalyticsStream(**report_init_kwargs) + expected_state = {stream.cursor_field: expected_date} + assert stream.get_updated_state(current_stream_state, latest_record) == expected_state + + @pytest.mark.parametrize( + ("start_date", "end_date", "stream_state", "fixed_period_in_days", "expected_slices"), + ( + ("2023-09-05T00:00:00Z", None, None, 1, [{"dataStartTime": "2023-09-05T00:00:00Z", "dataEndTime": "2023-09-05T23:59:59Z"}]), + ( + "2023-09-05T00:00:00Z", + "2023-09-06T00:00:00Z", + None, + 1, + [{"dataStartTime": "2023-09-05T00:00:00Z", "dataEndTime": "2023-09-05T23:59:59Z"}], + ), + ( + "2023-09-05T00:00:00Z", + "2023-09-07T00:00:00Z", + {"endDate": "2023-09-06T00:00:00Z"}, + 1, + [{"dataStartTime": "2023-09-06T00:00:00Z", "dataEndTime": "2023-09-06T23:59:59Z"}], + ), + ( + "2023-05-01T00:00:00Z", + "2023-09-07T00:00:00Z", + None, + 0, + [ + {"dataStartTime": "2023-05-01T00:00:00Z", "dataEndTime": "2023-07-29T23:59:59Z"}, + {"dataStartTime": "2023-07-30T00:00:00Z", "dataEndTime": "2023-09-07T00:00:00Z"}, + ], + ), + ), + ) + def test_stream_slices(self, report_init_kwargs, start_date, end_date, stream_state, fixed_period_in_days, expected_slices): + report_init_kwargs["replication_start_date"] = start_date + report_init_kwargs["replication_end_date"] = end_date + stream = SomeIncrementalAnalyticsStream(**report_init_kwargs) + stream.fixed_period_in_days = fixed_period_in_days + with patch("pendulum.now", return_value=pendulum.parse("2023-09-09T00:00:00Z")): + assert list( + stream.stream_slices(sync_mode=SyncMode.incremental, cursor_field=[stream.cursor_field], stream_state=stream_state) + ) == expected_slices diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py index bf0ad2cbcc0b..1d3ad8b7cb4f 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_finance_streams.py @@ -2,11 +2,14 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + +from unittest import mock + import pendulum import pytest import requests -from source_amazon_seller_partner.auth import AWSSignature -from source_amazon_seller_partner.streams import ListFinancialEventGroups, ListFinancialEvents +from airbyte_cdk.models import SyncMode +from source_amazon_seller_partner.streams import ListFinancialEventGroups, ListFinancialEvents, RestockInventoryReports list_financial_event_groups_data = { "payload": { @@ -96,24 +99,14 @@ @pytest.fixture def list_financial_event_groups_stream(): def _internal(start_date: str = START_DATE_1, end_date: str = END_DATE_1): - aws_signature = AWSSignature( - service="execute-api", - aws_access_key_id="AccessKeyId", - aws_secret_access_key="SecretAccessKey", - aws_session_token="SessionToken", - region="US", - ) stream = ListFinancialEventGroups( url_base="https://test.url", - aws_signature=aws_signature, replication_start_date=start_date, replication_end_date=end_date, marketplace_id="id", authenticator=None, period_in_days=0, report_options=None, - advanced_stream_options=None, - max_wait_seconds=500, ) return stream @@ -123,24 +116,14 @@ def _internal(start_date: str = START_DATE_1, end_date: str = END_DATE_1): @pytest.fixture def list_financial_events_stream(): def _internal(start_date: str = START_DATE_1, end_date: str = END_DATE_1): - aws_signature = AWSSignature( - service="execute-api", - aws_access_key_id="AccessKeyId", - aws_secret_access_key="SecretAccessKey", - aws_session_token="SessionToken", - region="US", - ) stream = ListFinancialEvents( url_base="https://test.url", - aws_signature=aws_signature, replication_start_date=start_date, replication_end_date=end_date, marketplace_id="id", authenticator=None, period_in_days=0, report_options=None, - advanced_stream_options=None, - max_wait_seconds=500, ) return stream @@ -219,3 +202,33 @@ def test_financial_events_stream_parse_response(mocker, list_financial_events_st assert list_financial_events_data.get("payload").get("FinancialEvents").get("AdjustmentEventList") == record.get( "AdjustmentEventList" ) + + +def test_reports_read_records_exit_on_backoff(mocker, requests_mock, caplog): + mocker.patch("time.sleep", lambda x: None) + requests_mock.post("https://test.url/reports/2021-06-30/reports", status_code=429) + + stream = RestockInventoryReports( + url_base="https://test.url", + replication_start_date=START_DATE_1, + replication_end_date=END_DATE_1, + marketplace_id="id", + authenticator=None, + period_in_days=0, + report_options=None, + ) + assert list(stream.read_records(sync_mode=SyncMode.full_refresh)) == [] + assert ( + "The report for stream 'GET_RESTOCK_INVENTORY_RECOMMENDATIONS_REPORT' was cancelled due to several failed retry attempts." + ) in caplog.messages[-1] + + +@pytest.mark.parametrize( + ("response_headers", "expected_backoff_time"), + (({"x-amzn-RateLimit-Limit": "2"}, 0.5), ({}, 60)), +) +def test_financial_events_stream_backoff_time(list_financial_events_stream, response_headers, expected_backoff_time): + stream = list_financial_events_stream() + response_mock = mock.MagicMock() + response_mock.headers = response_headers + assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py new file mode 100644 index 000000000000..7ff6c7fb1958 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations.py @@ -0,0 +1,84 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import json +from typing import Any, Mapping + +import pytest +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source +from source_amazon_seller_partner.config_migrations import MigrateAccountType, MigrateReportOptions +from source_amazon_seller_partner.source import SourceAmazonSellerPartner + +CMD = "check" +SOURCE: Source = SourceAmazonSellerPartner() + + +def load_config(config_path: str) -> Mapping[str, Any]: + with open(config_path, "r") as config: + return json.load(config) + + +class TestMigrateAccountType: + test_not_migrated_config_path = "unit_tests/test_migrations/account_type_migration/not_migrated_config.json" + test_migrated_config_path = "unit_tests/test_migrations/account_type_migration/migrated_config.json" + + def test_migrate_config(self, capsys): + config = load_config(self.test_not_migrated_config_path) + assert "account_type" not in config + migration_instance = MigrateAccountType() + migration_instance.migrate([CMD, "--config", self.test_not_migrated_config_path], SOURCE) + control_msg = json.loads(capsys.readouterr().out) + assert control_msg["type"] == Type.CONTROL.value + assert control_msg["control"]["type"] == OrchestratorType.CONNECTOR_CONFIG.value + migrated_config = control_msg["control"]["connectorConfig"]["config"] + assert migrated_config["account_type"] == "Seller" + + def test_should_not_migrate(self): + config = load_config(self.test_migrated_config_path) + assert config["account_type"] + migration_instance = MigrateAccountType() + assert not migration_instance._should_migrate(config) + + +class TestMigrateReportOptions: + test_not_migrated_config_path = "unit_tests/test_migrations/report_options_migration/not_migrated_config.json" + test_migrated_config_path = "unit_tests/test_migrations/report_options_migration/migrated_config.json" + + @pytest.mark.parametrize( + ("input_config", "expected_report_options_list"), + ( + ( + {"report_options": "{\"GET_REPORT\": {\"reportPeriod\": \"WEEK\"}}"}, + [{"stream_name": "GET_REPORT", "options_list": [{"option_name": "reportPeriod", "option_value": "WEEK"}]}], + ), + ({"report_options": None}, []), + ({"report_options": "{{}"}, []), + ({}, []), + ), + ) + def test_transform_report_options(self, input_config, expected_report_options_list): + expected_config = {**input_config, "report_options_list": expected_report_options_list} + assert MigrateReportOptions._transform_report_options(input_config) == expected_config + + def test_migrate_config(self, capsys): + config = load_config(self.test_not_migrated_config_path) + assert "report_options_list" not in config + migration_instance = MigrateReportOptions() + migration_instance.migrate([CMD, "--config", self.test_not_migrated_config_path], SOURCE) + control_msg = json.loads(capsys.readouterr().out) + assert control_msg["type"] == Type.CONTROL.value + assert control_msg["control"]["type"] == OrchestratorType.CONNECTOR_CONFIG.value + migrated_config = control_msg["control"]["connectorConfig"]["config"] + expected_report_options_list = [ + {"stream_name": "GET_REPORT", "options_list": [{"option_name": "reportPeriod", "option_value": "WEEK"}]}, + ] + assert migrated_config["report_options_list"] == expected_report_options_list + + def test_should_not_migrate(self): + config = load_config(self.test_migrated_config_path) + assert config["report_options_list"] + migration_instance = MigrateReportOptions() + assert not migration_instance._should_migrate(config) diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations/account_type_migration/migrated_config.json b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations/account_type_migration/migrated_config.json new file mode 100644 index 000000000000..f083c48104f6 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations/account_type_migration/migrated_config.json @@ -0,0 +1,9 @@ +{ + "refresh_token": "refresh_token", + "lwa_app_id": "amzn1.application-oa2-client.lwa_app_id", + "lwa_client_secret": "amzn1.oa2-cs.v1.lwa_client_secret", + "replication_start_date": "2022-09-01T00:00:00Z", + "aws_environment": "PRODUCTION", + "account_type": "Vendor", + "region": "US" +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations/account_type_migration/not_migrated_config.json b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations/account_type_migration/not_migrated_config.json new file mode 100644 index 000000000000..b5fed71e1417 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations/account_type_migration/not_migrated_config.json @@ -0,0 +1,8 @@ +{ + "refresh_token": "refresh_token", + "lwa_app_id": "amzn1.application-oa2-client.lwa_app_id", + "lwa_client_secret": "amzn1.oa2-cs.v1.lwa_client_secret", + "replication_start_date": "2022-09-01T00:00:00Z", + "aws_environment": "PRODUCTION", + "region": "US" +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations/report_options_migration/migrated_config.json b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations/report_options_migration/migrated_config.json new file mode 100644 index 000000000000..fdaa8d461eb6 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations/report_options_migration/migrated_config.json @@ -0,0 +1,15 @@ +{ + "refresh_token": "refresh_token", + "lwa_app_id": "amzn1.application-oa2-client.lwa_app_id", + "lwa_client_secret": "amzn1.oa2-cs.v1.lwa_client_secret", + "replication_start_date": "2022-09-01T00:00:00Z", + "aws_environment": "PRODUCTION", + "account_type": "Vendor", + "region": "US", + "report_options_list": [ + { + "stream_name": "GET_MERCHANT_LISTINGS_ALL_DATA", + "options_list": [{ "option_name": "custom", "option_value": "true" }] + } + ] +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations/report_options_migration/not_migrated_config.json b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations/report_options_migration/not_migrated_config.json new file mode 100644 index 000000000000..803752451583 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_migrations/report_options_migration/not_migrated_config.json @@ -0,0 +1,10 @@ +{ + "refresh_token": "refresh_token", + "lwa_app_id": "amzn1.application-oa2-client.lwa_app_id", + "lwa_client_secret": "amzn1.oa2-cs.v1.lwa_client_secret", + "replication_start_date": "2022-09-01T00:00:00Z", + "aws_environment": "PRODUCTION", + "region": "US", + "report_options": "{\"GET_REPORT\": {\"reportPeriod\": \"WEEK\"}}", + "advanced_stream_options": "{\"GET_REPORT_2\": {\"reportPeriod_2\": \"DAY\"}}" +} diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_order_items_stream.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_order_items_stream.py deleted file mode 100644 index 4e5d1364ed99..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_order_items_stream.py +++ /dev/null @@ -1,99 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pytest -import requests -from source_amazon_seller_partner.auth import AWSSignature -from source_amazon_seller_partner.streams import OrderItems - -list_order_items_payload_data = { - "payload": { - "OrderItems": [ - { - "ProductInfo": { - "NumberOfItems": "1" - }, - "IsGift": "false", - "BuyerInfo": {}, - "QuantityShipped": 0, - "IsTransparency": False, - "QuantityOrdered": 1, - "ASIN": "AKDDKDKD", - "SellerSKU": "AAA-VPx3-AMZ", - "Title": "Example product", - "OrderItemId": "88888888888" - } - ], - "AmazonOrderId": "111-0000000-2222222" - } -} - - -@pytest.fixture -def order_items_stream(): - def _internal(): - aws_signature = AWSSignature( - service="execute-api", - aws_access_key_id="AccessKeyId", - aws_secret_access_key="SecretAccessKey", - aws_session_token="SessionToken", - region="US", - ) - stream = OrderItems( - url_base="https://test.url", - aws_signature=aws_signature, - replication_start_date="2023-08-08T00:00:00Z", - replication_end_date=None, - marketplace_id="id", - authenticator=None, - period_in_days=0, - report_options=None, - advanced_stream_options=None, - max_wait_seconds=500, - ) - return stream - - return _internal - - -def test_order_items_stream_initialization(order_items_stream): - stream = order_items_stream() - assert stream._replication_start_date == "2023-08-08T00:00:00Z" - assert stream._replication_end_date is None - assert stream.marketplace_id == "id" - - -def test_order_items_stream_next_token(mocker, order_items_stream): - response = requests.Response() - token = "111111111" - expected = {"NextToken": token} - mocker.patch.object(response, "json", return_value={"payload": expected}) - assert expected == order_items_stream().next_page_token(response) - - mocker.patch.object(response, "json", return_value={"payload": {}}) - if order_items_stream().next_page_token(response) is not None: - assert False - - -def test_order_items_stream_parse_response(mocker, order_items_stream): - response = requests.Response() - mocker.patch.object(response, "json", return_value=list_order_items_payload_data) - - stream = order_items_stream() - stream.cached_state["LastUpdateDate"] = "2023-08-07T00:00:00Z" - parsed = stream.parse_response(response, stream_slice={"AmazonOrderId": "111-0000000-2222222", "LastUpdateDate": "2023-08-08T00:00:00Z"}) - - for record in parsed: - assert record["AmazonOrderId"] == "111-0000000-2222222" - assert record["OrderItemId"] == "88888888888" - assert record["SellerSKU"] == "AAA-VPx3-AMZ" - assert record["ASIN"] == "AKDDKDKD" - assert record["Title"] == "Example product" - assert record["QuantityOrdered"] == 1 - assert record["QuantityShipped"] == 0 - assert record["BuyerInfo"] == {} - assert record["IsGift"] == "false" - assert record["ProductInfo"] == {"NumberOfItems": "1"} - - assert stream.cached_state["LastUpdateDate"] == "2023-08-08T00:00:00Z" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_order_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_order_streams.py new file mode 100644 index 000000000000..a57deb47b748 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_order_streams.py @@ -0,0 +1,196 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from unittest import mock + +import pytest +import requests +from source_amazon_seller_partner.streams import OrderItems, Orders + + +@pytest.fixture +def orders_stream(): + def _internal(**kwargs): + stream = Orders( + url_base=kwargs.get("url_base", "https://test.url"), + replication_start_date=kwargs.get("replication_start_date", "2023-08-08T00:00:00Z"), + replication_end_date=kwargs.get("replication_end_date"), + marketplace_id=kwargs.get("marketplace_id", "id"), + authenticator=None, + period_in_days=kwargs.get("period_in_days", 0), + report_options=kwargs.get("report_options"), + ) + return stream + + return _internal + + +@pytest.fixture +def order_items_stream(): + def _internal(): + stream = OrderItems( + url_base="https://test.url", + replication_start_date="2023-08-08T00:00:00Z", + replication_end_date=None, + marketplace_id="id", + authenticator=None, + period_in_days=0, + report_options=None, + ) + return stream + + return _internal + + +class TestOrders: + def test_path(self, orders_stream): + stream = orders_stream() + assert stream.path() == "orders/v0/orders" + + @pytest.mark.parametrize( + ("start_date", "end_date", "stream_state", "next_page_token", "expected_params"), + ( + ("2022-09-01T00:00:00Z", "2022-10-01T00:00:00Z", {}, {"NextToken": "NextToken123"}, {"NextToken": "NextToken123"}), + ("2022-09-01T00:00:00Z", None, {}, None, {"LastUpdatedAfter": "2022-09-01T00:00:00Z", "MaxResultsPerPage": 100}), + ( + "2022-09-01T00:00:00Z", + "2022-10-01T00:00:00Z", + {}, + None, + {"LastUpdatedAfter": "2022-09-01T00:00:00Z", "MaxResultsPerPage": 100, "LastUpdatedBefore": "2022-10-01T00:00:00Z"}, + ), + ( + "2022-09-01T00:00:00Z", + "2022-11-01T00:00:00Z", + {"LastUpdateDate": "2022-10-01T00:00:00Z"}, + None, + {"LastUpdatedAfter": "2022-10-01T00:00:00Z", "MaxResultsPerPage": 100, "LastUpdatedBefore": "2022-11-01T00:00:00Z"}, + ), + ), + ) + def test_request_params(self, orders_stream, start_date, end_date, stream_state, next_page_token, expected_params): + marketplace_id = "market123" + stream = orders_stream(replication_start_date=start_date, replication_end_date=end_date, marketplace_id=marketplace_id) + expected_params.update({"MarketplaceIds": marketplace_id}) + assert stream.request_params(stream_state, next_page_token) == expected_params + + @pytest.mark.parametrize( + ("response_headers", "expected_backoff_time"), + (({"x-amzn-RateLimit-Limit": "2"}, 0.5), ({}, 60)), + ) + def test_backoff_time(self, orders_stream, response_headers, expected_backoff_time): + stream = orders_stream() + response_mock = mock.MagicMock() + response_mock.headers = response_headers + assert stream.backoff_time(response_mock) == expected_backoff_time + + @pytest.mark.parametrize( + ("payload", "expected_value"), + (({"NextToken": "NextToken123"}, {"NextToken": "NextToken123"}), ({}, None)), + ) + def test_next_page_token(self, mocker, orders_stream, payload, expected_value): + stream = orders_stream() + response_mock = requests.Response() + mocker.patch.object(response_mock, "json", return_value={"payload": payload}) + assert stream.next_page_token(response_mock) == expected_value + + @pytest.mark.parametrize( + ("current_stream_state", "latest_record", "expected_date"), + ( + ({"LastUpdateDate": "2022-10-03T00:00:00Z"}, {"LastUpdateDate": "2022-10-04T00:00:00Z"}, "2022-10-04T00:00:00Z"), + ({"LastUpdateDate": "2022-10-04T00:00:00Z"}, {"LastUpdateDate": "2022-10-03T00:00:00Z"}, "2022-10-04T00:00:00Z"), + ({}, {"LastUpdateDate": "2022-10-03T00:00:00Z"}, "2022-10-03T00:00:00Z"), + ), + ) + def test_get_updated_state(self, orders_stream, current_stream_state, latest_record, expected_date): + stream = orders_stream() + expected_state = {stream.cursor_field: expected_date} + assert stream.get_updated_state(current_stream_state, latest_record) == expected_state + + +class TestOrderItems: + list_order_items_payload_data = { + "payload": { + "OrderItems": [ + { + "ProductInfo": {"NumberOfItems": "1"}, + "IsGift": "false", + "BuyerInfo": {}, + "QuantityShipped": 0, + "IsTransparency": False, + "QuantityOrdered": 1, + "ASIN": "AKDDKDKD", + "SellerSKU": "AAA-VPx3-AMZ", + "Title": "Example product", + "OrderItemId": "88888888888", + } + ], + "AmazonOrderId": "111-0000000-2222222", + } + } + + def test_path(self, order_items_stream): + stream = order_items_stream() + stream_slice = {"AmazonOrderId": "AmazonOrderId123"} + assert stream.path(stream_slice) == "orders/v0/orders/AmazonOrderId123/orderItems" + + @pytest.mark.parametrize( + ("next_page_token", "expected_params"), + (({"NextToken": "NextToken123"}, {"NextToken": "NextToken123"}), (None, {})), + ) + def test_request_params(self, order_items_stream, next_page_token, expected_params): + stream = order_items_stream() + assert stream.request_params(stream_state={}, next_page_token=next_page_token) == expected_params + + @pytest.mark.parametrize( + ("response_headers", "expected_backoff_time"), + (({"x-amzn-RateLimit-Limit": "2"}, 0.5), ({}, 10)), + ) + def test_backoff_time(self, order_items_stream, response_headers, expected_backoff_time): + stream = order_items_stream() + response_mock = mock.MagicMock() + response_mock.headers = response_headers + assert stream.backoff_time(response_mock) == expected_backoff_time + + def test_stream_initialization(self, order_items_stream): + stream = order_items_stream() + assert stream._replication_start_date == "2023-08-08T00:00:00Z" + assert stream._replication_end_date is None + assert stream.marketplace_id == "id" + + def test_stream_next_token(self, mocker, order_items_stream): + response = requests.Response() + token = "111111111" + expected = {"NextToken": token} + mocker.patch.object(response, "json", return_value={"payload": expected}) + assert expected == order_items_stream().next_page_token(response) + + mocker.patch.object(response, "json", return_value={"payload": {}}) + if order_items_stream().next_page_token(response) is not None: + assert False + + def test_order_items_stream_parse_response(self, mocker, order_items_stream): + response = requests.Response() + mocker.patch.object(response, "json", return_value=self.list_order_items_payload_data) + + stream = order_items_stream() + stream.cached_state["LastUpdateDate"] = "2023-08-07T00:00:00Z" + parsed = stream.parse_response( + response, stream_slice={"AmazonOrderId": "111-0000000-2222222", "LastUpdateDate": "2023-08-08T00:00:00Z"} + ) + + for record in parsed: + assert record["AmazonOrderId"] == "111-0000000-2222222" + assert record["OrderItemId"] == "88888888888" + assert record["SellerSKU"] == "AAA-VPx3-AMZ" + assert record["ASIN"] == "AKDDKDKD" + assert record["Title"] == "Example product" + assert record["QuantityOrdered"] == 1 + assert record["QuantityShipped"] == 0 + assert record["BuyerInfo"] == {} + assert record["IsGift"] == "false" + assert record["ProductInfo"] == {"NumberOfItems": "1"} + + assert stream.cached_state["LastUpdateDate"] == "2023-08-08T00:00:00Z" diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_stream_sales_and_traffic.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_stream_sales_and_traffic.py deleted file mode 100644 index 9f2f5134ba3d..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_stream_sales_and_traffic.py +++ /dev/null @@ -1,33 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from source_amazon_seller_partner.auth import AWSSignature -from source_amazon_seller_partner.streams import SellerAnalyticsSalesAndTrafficReports - -START_DATE_1 = "2023-02-05T00:00:00Z" -END_DATE_1 = "2023-02-07T00:00:00Z" - - -def test_stream_uses_advanced_options(): - aws_signature = AWSSignature( - service="execute-api", - aws_access_key_id="AccessKeyId", - aws_secret_access_key="SecretAccessKey", - aws_session_token="SessionToken", - region="US", - ) - stream = SellerAnalyticsSalesAndTrafficReports( - url_base="https://test.url", - aws_signature=aws_signature, - replication_start_date=START_DATE_1, - replication_end_date=END_DATE_1, - marketplace_id="id", - authenticator=None, - period_in_days=0, - report_options=None, - advanced_stream_options='{"GET_SALES_AND_TRAFFIC_REPORT":{"availability_sla_days": 3}}', - max_wait_seconds=500, - ) - - assert stream.availability_sla_days == 3 diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_rate_limits.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_rate_limits.py deleted file mode 100644 index fec6e1b7326a..000000000000 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_rate_limits.py +++ /dev/null @@ -1,66 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pytest -import requests -from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException -from source_amazon_seller_partner.auth import AWSSignature -from source_amazon_seller_partner.streams import MerchantListingsReports - - -@pytest.fixture -def reports_stream(): - aws_signature = AWSSignature( - service="execute-api", - aws_access_key_id="AccessKeyId", - aws_secret_access_key="SecretAccessKey", - aws_session_token="SessionToken", - region="US", - ) - stream = MerchantListingsReports( - url_base="https://test.url", - aws_signature=aws_signature, - replication_start_date="2017-01-25T00:00:00Z", - replication_end_date="2017-02-25T00:00:00Z", - marketplace_id="id", - authenticator=None, - period_in_days=0, - report_options=None, - advanced_stream_options=None, - max_wait_seconds=500, - ) - return stream - - -def test_reports_stream_should_retry(mocker, reports_stream): - response = requests.Response() - response.status_code = 429 - mocker.patch.object(requests.Session, "send", return_value=response) - should_retry = reports_stream.should_retry(response=response) - - assert should_retry is True - - -def test_reports_stream_send_request(mocker, reports_stream): - response = requests.Response() - response.status_code = 200 - mocker.patch.object(requests.Session, "send", return_value=response) - - assert response == reports_stream._send_request(request=requests.PreparedRequest()) - - -def test_reports_stream_send_request_backoff_exception(mocker, caplog, reports_stream): - mocker.patch("time.sleep", lambda x: None) - response = requests.Response() - response.status_code = 429 - mocker.patch.object(requests.Session, "send", return_value=response) - - with pytest.raises(DefaultBackoffException): - reports_stream._send_request(request=requests.PreparedRequest()) - - assert "Backing off _send_request(...) for 5.0s" in caplog.text - assert "Backing off _send_request(...) for 10.0s" in caplog.text - assert "Backing off _send_request(...) for 20.0s" in caplog.text - assert "Backing off _send_request(...) for 40.0s" in caplog.text - assert "Giving up _send_request(...) after 5 tries" in caplog.text diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py index b94b2d30b381..0bdf83116fbc 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_reports_streams_settlement_report.py @@ -2,114 +2,45 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import pytest -import requests from airbyte_cdk.models import SyncMode -from source_amazon_seller_partner.auth import AWSSignature from source_amazon_seller_partner.streams import FlatFileSettlementV2Reports START_DATE_1 = "2022-05-25T00:00:00Z" END_DATE_1 = "2022-05-26T00:00:00Z" -generated_reports_from_amazon = { - "payload": [ - { - "createdTime": "2022-07-08T10:39:31+00:00", - "dataEndTime": "2022-07-08T09:59:21+00:00", - "dataStartTime": "2022-06-27T08:01:32+00:00", - "marketplaceIds": [ - "A1F83G8C2ARO7P", - "A1PA6795UKMFR9", - "A13V1IB3VIYZZH", - "AZMDEXL2RVFNN", - "A38D8NSA03LJTC", - "A1ZFFQZ3HTUKT9", - "APJ6JRA9NG5V4", - "A1RKKUPIHCS9HS", - "A62U237T8HV6N", - "AFQLKURYRPEL8", - "A1NYP31CE519TD", - "A1805IZSGTT6HS", - "A33AVAJ2PDY3EV", - "AMEN7PMS3EDWL", - "A2NODRKZP88ZB9", - "A1C3SOZRARQ6R3", - ], - "processingEndTime": "2022-07-08T10:39:31+00:00", - "processingStartTime": "2022-07-08T10:39:31+00:00", - "processingStatus": "DONE", - "reportDocumentId": "amzn1.spdoc.1.3.0fcde1b1-a35e-4fe1-b077-38e0e9f65d63.T1SN9707N5X5IQ.0000", - "reportId": "85968019100", - "reportType": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", - }, - { - "createdTime": "2022-07-06T09:12:07+00:00", - "dataEndTime": "2022-07-06T08:38:16+00:00", - "dataStartTime": "2022-06-22T08:38:16+00:00", - "marketplaceIds": [ - "A1F83G8C2ARO7P", - "A1PA6795UKMFR9", - "A13V1IB3VIYZZH", - "AZMDEXL2RVFNN", - "A38D8NSA03LJTC", - "A1ZFFQZ3HTUKT9", - "APJ6JRA9NG5V4", - "A1RKKUPIHCS9HS", - "A62U237T8HV6N", - "AFQLKURYRPEL8", - "A1NYP31CE519TD", - "A1805IZSGTT6HS", - "A33AVAJ2PDY3EV", - "AMEN7PMS3EDWL", - "A2NODRKZP88ZB9", - "A1C3SOZRARQ6R3", - ], - "processingEndTime": "2022-07-06T09:12:07+00:00", - "processingStartTime": "2022-07-06T09:12:07+00:00", - "processingStatus": "DONE", - "reportDocumentId": "amzn1.spdoc.1.3.f7f43990-7c58-40f2-a93f-565b79a88269.T3OS2416I1AAXM.0000", - "reportId": "85948019111", - "reportType": "GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE", - }, - ] -} - @pytest.fixture def settlement_reports_stream(): def _internal(start_date: str = START_DATE_1, end_date: str = END_DATE_1): - aws_signature = AWSSignature( - service="execute-api", - aws_access_key_id="AccessKeyId", - aws_secret_access_key="SecretAccessKey", - aws_session_token="SessionToken", - region="US", - ) stream = FlatFileSettlementV2Reports( url_base="https://test.url", - aws_signature=aws_signature, replication_start_date=start_date, replication_end_date=end_date, marketplace_id="id", authenticator=None, period_in_days=0, report_options=None, - advanced_stream_options=None, - max_wait_seconds=500, ) return stream return _internal -def test_stream_slices_method(mocker, settlement_reports_stream): - response = requests.Response() - mocker.patch.object(response, "json", return_value=generated_reports_from_amazon) - - data = response.json().get("payload", list()) - - slices = [{"report_id": e.get("reportId")} for e in data] - - for i in range(len(slices)): - report = settlement_reports_stream()._create_report(sync_mode=SyncMode.full_refresh, stream_slice=slices[i]) - assert report.get("reportId") == generated_reports_from_amazon.get("payload")[i].get("reportId") +def test_stream_slices(requests_mock, settlement_reports_stream): + requests_mock.register_uri( + "POST", + "https://api.amazon.com/auth/o2/token", + status_code=200, + json={"access_token": "access_token", "expires_in": "3600"}, + ) + requests_mock.register_uri( + "GET", + "https://test.url/reports/2021-06-30/reports", + status_code=200, + json={"reports": [{"reportId": "reportId 1"}, {"reportId": "reportId 2"}]}, + ) + + stream = settlement_reports_stream() + assert list(stream.stream_slices(sync_mode=SyncMode.full_refresh)) == [{"report_id": "reportId 1"}, {"report_id": "reportId 2"}] diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py index 4c24d0fccd65..b1721122d69b 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_source.py @@ -2,74 +2,160 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from unittest.mock import MagicMock + +import logging +from unittest.mock import patch import pytest from airbyte_cdk.sources.streams import Stream from source_amazon_seller_partner import SourceAmazonSellerPartner -from source_amazon_seller_partner.source import boto3 - +from source_amazon_seller_partner.streams import VendorSalesReports +from source_amazon_seller_partner.utils import AmazonConfigException -@pytest.fixture -def connector_source(): - return SourceAmazonSellerPartner() +logger = logging.getLogger("airbyte") @pytest.fixture -def connector_config(): +def connector_config_with_report_options(): return { "replication_start_date": "2017-01-25T00:00:00Z", "replication_end_date": "2017-02-25T00:00:00Z", "refresh_token": "Atzr|IwEBIP-abc123", - "app_id": "amzn1.sp.solution.2cfa6ca8-2c35-123-456-78910", "lwa_app_id": "amzn1.application-oa2-client.abc123", "lwa_client_secret": "abc123", - "aws_access_key": "aws_access_key", - "aws_secret_key": "aws_secret_key", - "role_arn": "arn:aws:iam::123456789098:role/some-role", "aws_environment": "SANDBOX", "region": "US", + "report_options_list": [ + { + "stream_name": "GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA", + "options_list": [ + {"option_name": "some_name_1", "option_value": "some_value_1"}, + {"option_name": "some_name_2", "option_value": "some_value_2"}, + ], + }, + ], } @pytest.fixture -def sts_credentials(): +def connector_config_without_start_date(): return { - "Credentials": { - "AccessKeyId": "foo", - "SecretAccessKey": "bar", - "SessionToken": "foobar", - } + "refresh_token": "Atzr|IwEBIP-abc123", + "lwa_app_id": "amzn1.application-oa2-client.abc123", + "lwa_client_secret": "abc123", + "aws_environment": "SANDBOX", + "region": "US", } -@pytest.fixture -def mock_boto_client(mocker, sts_credentials): - boto_client = MagicMock() - mocker.patch.object(boto3, "client", return_value=boto_client) - boto_client.assume_role.return_value = sts_credentials - boto_client.get_session_token.return_value = sts_credentials - return boto_client - - -def test_streams(connector_source, connector_config, mock_boto_client): - for stream in connector_source.streams(connector_config): +def test_check_connection_with_vendor_report(mocker, requests_mock, connector_config_with_report_options): + mocker.patch("time.sleep", lambda x: None) + requests_mock.register_uri( + "POST", + "https://api.amazon.com/auth/o2/token", + status_code=200, + json={"access_token": "access_token", "expires_in": "3600"}, + ) + requests_mock.register_uri( + "GET", + "https://sandbox.sellingpartnerapi-na.amazon.com/orders/v0/orders", + status_code=403, + json={"error": "forbidden"}, + ) + + with patch.object(VendorSalesReports, "read_records", return_value=iter([{"some_key": "some_value"}])): + assert SourceAmazonSellerPartner().check_connection(logger, connector_config_with_report_options) == (True, None) + + +def test_check_connection_with_orders_stop_iteration(requests_mock, connector_config_with_report_options): + requests_mock.register_uri( + "POST", + "https://api.amazon.com/auth/o2/token", + status_code=200, + json={"access_token": "access_token", "expires_in": "3600"}, + ) + requests_mock.register_uri( + "GET", + "https://sandbox.sellingpartnerapi-na.amazon.com/orders/v0/orders", + status_code=201, + json={"payload": {"Orders": []}}, + ) + assert SourceAmazonSellerPartner().check_connection(logger, connector_config_with_report_options) == (True, None) + + +def test_check_connection_with_orders(requests_mock, connector_config_with_report_options): + requests_mock.register_uri( + "POST", + "https://api.amazon.com/auth/o2/token", + status_code=200, + json={"access_token": "access_token", "expires_in": "3600"}, + ) + requests_mock.register_uri( + "GET", + "https://sandbox.sellingpartnerapi-na.amazon.com/orders/v0/orders", + status_code=200, + json={"payload": {"Orders": [{"some_key": "some_value"}]}}, + ) + assert SourceAmazonSellerPartner().check_connection(logger, connector_config_with_report_options) == (True, None) + + +@pytest.mark.parametrize( + ("report_name", "options_list"), + ( + ( + "GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA", + [ + {"option_name": "some_name_1", "option_value": "some_value_1"}, + {"option_name": "some_name_2", "option_value": "some_value_2"}, + ], + ), + ("SOME_OTHER_STREAM", None), + ), +) +def test_get_stream_report_options_list(connector_config_with_report_options, report_name, options_list): + assert SourceAmazonSellerPartner().get_stream_report_options_list(report_name, connector_config_with_report_options) == options_list + + +def test_config_report_options_validation_error_duplicated_streams(connector_config_with_report_options): + connector_config_with_report_options["report_options_list"].append(connector_config_with_report_options["report_options_list"][0]) + with pytest.raises(AmazonConfigException) as e: + SourceAmazonSellerPartner().validate_stream_report_options(connector_config_with_report_options) + assert e.value.message == "Stream name should be unique among all Report options list" + + +def test_config_report_options_validation_error_duplicated_options(connector_config_with_report_options): + connector_config_with_report_options["report_options_list"][0]["options_list"].append( + connector_config_with_report_options["report_options_list"][0]["options_list"][0] + ) + with pytest.raises(AmazonConfigException) as e: + SourceAmazonSellerPartner().validate_stream_report_options(connector_config_with_report_options) + assert e.value.message == "Option names should be unique for `GET_FBA_FULFILLMENT_CUSTOMER_RETURNS_DATA` report options" + + +def test_streams(connector_config_without_start_date): + for stream in SourceAmazonSellerPartner().streams(connector_config_without_start_date): assert isinstance(stream, Stream) -@pytest.mark.parametrize("arn", ("arn:aws:iam::123456789098:user/some-user", "arn:aws:iam::123456789098:role/some-role")) -def test_stream_with_good_iam_arn_value(mock_boto_client, connector_source, connector_config, arn): - connector_config["role_arn"] = arn - result = connector_source.get_sts_credentials(connector_config) - assert "Credentials" in result - if "user" in arn: - mock_boto_client.get_session_token.assert_called_once() - if "role" in arn: - mock_boto_client.assume_role.assert_called_once_with(RoleArn=arn, RoleSessionName="guid") +def test_streams_connector_config_without_start_date(connector_config_without_start_date): + for stream in SourceAmazonSellerPartner().streams(connector_config_without_start_date): + assert isinstance(stream, Stream) -def test_stream_with_bad_iam_arn_value(connector_source, connector_config, mock_boto_client): - connector_config["role_arn"] = "bad-arn" - with pytest.raises(ValueError) as e: - connector_source.get_sts_credentials(connector_config) - assert "Invalid" in e.message +@pytest.mark.parametrize( + ("config", "should_raise"), + ( + ({"replication_start_date": "2022-09-01T00:00:00Z", "replication_end_date": "2022-08-01T00:00:00Z"}, True), + ({"replication_start_date": "2022-09-01T00:00:00Z", "replication_end_date": "2022-10-01T00:00:00Z"}, False), + ({"replication_end_date": "2022-10-01T00:00:00Z"}, False), + ({"replication_start_date": "2022-09-01T00:00:00Z"}, False), + ({}, False), + ), +) +def test_replication_dates_validation(config, should_raise): + if should_raise: + with pytest.raises(AmazonConfigException) as e: + SourceAmazonSellerPartner().validate_replication_dates(config) + assert e.value.message == "End Date should be greater than or equal to Start Date" + else: + assert SourceAmazonSellerPartner().validate_replication_dates(config) is None diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py new file mode 100644 index 000000000000..3fe994016137 --- /dev/null +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_streams.py @@ -0,0 +1,241 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from unittest.mock import patch + +import pendulum +import pytest +import requests +from airbyte_cdk.models import SyncMode +from airbyte_cdk.utils import AirbyteTracedException +from source_amazon_seller_partner.streams import ( + IncrementalReportsAmazonSPStream, + ReportProcessingStatus, + ReportsAmazonSPStream, + VendorDirectFulfillmentShipping, +) + + +class SomeReportStream(ReportsAmazonSPStream): + name = "GET_TEST_REPORT" + + +class SomeIncrementalReportStream(IncrementalReportsAmazonSPStream): + name = "GET_TEST_INCREMENTAL_REPORT" + cursor_field = "dataEndTime" + + +class TestReportsAmazonSPStream: + def test_next_page_token(self, report_init_kwargs, mocker): + stream = SomeReportStream(**report_init_kwargs) + assert stream.next_page_token(mocker.Mock(spec=requests.Response)) is None + + def test_request_params(self, report_init_kwargs): + stream = SomeReportStream(**report_init_kwargs) + assert stream.request_params() == {"MarketplaceIds": report_init_kwargs["marketplace_id"]} + + def test_report_data(self, report_init_kwargs): + report_init_kwargs["report_options"] = [ + {"option_name": "some_name_1", "option_value": "some_value_1"}, + {"option_name": "some_name_2", "option_value": "some_value_2"}, + ] + stream = SomeReportStream(**report_init_kwargs) + expected_data = { + "reportType": stream.name, + "marketplaceIds": [report_init_kwargs["marketplace_id"]], + "reportOptions": {"some_name_1": "some_value_1", "some_name_2": "some_value_2"}, + } + + assert stream._report_data(sync_mode=SyncMode.full_refresh) == expected_data + + @pytest.mark.parametrize( + ("start_date", "end_date", "expected_slices"), + ( + ( + "2022-09-01T00:00:00Z", + "2022-10-01T00:00:00Z", + [{"dataStartTime": "2022-09-01T00:00:00Z", "dataEndTime": "2022-10-01T00:00:00Z"}], + ), + ( + "2022-09-01T00:00:00Z", + "2023-01-01T00:00:00Z", + [ + {"dataStartTime": "2022-09-01T00:00:00Z", "dataEndTime": "2022-11-29T23:59:59Z"}, + {"dataStartTime": "2022-11-30T00:00:00Z", "dataEndTime": "2023-01-01T00:00:00Z"}, + ], + ), + ( + "2022-10-01T00:00:00Z", + None, + [ + {"dataStartTime": "2022-10-01T00:00:00Z", "dataEndTime": "2022-12-29T23:59:59Z"}, + {"dataStartTime": "2022-12-30T00:00:00Z", "dataEndTime": "2023-01-01T00:00:00Z"} + ], + ), + ( + "2022-11-01T00:00:00Z", + None, + [{"dataStartTime": "2022-11-01T00:00:00Z", "dataEndTime": "2023-01-01T00:00:00Z"}], + ), + ), + ) + def test_stream_slices(self, report_init_kwargs, start_date, end_date, expected_slices): + report_init_kwargs["replication_start_date"] = start_date + report_init_kwargs["replication_end_date"] = end_date + + stream = SomeReportStream(**report_init_kwargs) + with patch("pendulum.now", return_value=pendulum.parse("2023-01-01T00:00:00Z")): + assert list(stream.stream_slices(sync_mode=SyncMode.full_refresh)) == expected_slices + + @pytest.mark.parametrize( + ("current_stream_state", "latest_record", "expected_date"), + ( + ({"dataEndTime": "2022-10-03"}, {"dataEndTime": "2022-10-04"}, "2022-10-04"), + ({"dataEndTime": "2022-10-04"}, {"dataEndTime": "2022-10-03"}, "2022-10-04"), + ({}, {"dataEndTime": "2022-10-03"}, "2022-10-03"), + ), + ) + def test_get_updated_state(self, report_init_kwargs, current_stream_state, latest_record, expected_date): + stream = SomeIncrementalReportStream(**report_init_kwargs) + expected_state = {stream.cursor_field: expected_date} + assert stream.get_updated_state(current_stream_state, latest_record) == expected_state + + def test_read_records_retrieve_fatal(self, report_init_kwargs, mocker, requests_mock): + mocker.patch("time.sleep", lambda x: None) + requests_mock.register_uri( + "POST", + "https://api.amazon.com/auth/o2/token", + status_code=200, + json={"access_token": "access_token", "expires_in": "3600"}, + ) + + report_id = "some_report_id" + requests_mock.register_uri( + "POST", + "https://test.url/reports/2021-06-30/reports", + status_code=201, + json={"reportId": report_id}, + ) + requests_mock.register_uri( + "GET", + f"https://test.url/reports/2021-06-30/reports/{report_id}", + status_code=200, + json={"processingStatus": ReportProcessingStatus.fatal, "dataEndTime": "2022-10-03T00:00:00Z"}, + ) + + stream = SomeReportStream(**report_init_kwargs) + with pytest.raises(AirbyteTracedException) as e: + list(stream.read_records(sync_mode=SyncMode.full_refresh)) + assert e.value.message == "The report for stream 'GET_TEST_REPORT' was not created - skip reading" + + def test_read_records_retrieve_cancelled(self, report_init_kwargs, mocker, requests_mock, caplog): + mocker.patch("time.sleep", lambda x: None) + requests_mock.register_uri( + "POST", + "https://api.amazon.com/auth/o2/token", + status_code=200, + json={"access_token": "access_token", "expires_in": "3600"}, + ) + + report_id = "some_report_id" + requests_mock.register_uri( + "POST", + "https://test.url/reports/2021-06-30/reports", + status_code=201, + json={"reportId": report_id}, + ) + requests_mock.register_uri( + "GET", + f"https://test.url/reports/2021-06-30/reports/{report_id}", + status_code=200, + json={"processingStatus": ReportProcessingStatus.cancelled, "dataEndTime": "2022-10-03T00:00:00Z"}, + ) + + stream = SomeReportStream(**report_init_kwargs) + list(stream.read_records(sync_mode=SyncMode.full_refresh)) + assert "The report for stream 'GET_TEST_REPORT' was cancelled or there is no data to return" in caplog.messages[-1] + + def test_read_records_retrieve_done(self, report_init_kwargs, mocker, requests_mock): + mocker.patch("time.sleep", lambda x: None) + requests_mock.register_uri( + "POST", + "https://api.amazon.com/auth/o2/token", + status_code=200, + json={"access_token": "access_token", "expires_in": "3600"}, + ) + + report_id = "some_report_id" + document_id = "some_document_id" + requests_mock.register_uri( + "POST", + "https://test.url/reports/2021-06-30/reports", + status_code=201, + json={"reportId": report_id}, + ) + requests_mock.register_uri( + "GET", + f"https://test.url/reports/2021-06-30/reports/{report_id}", + status_code=200, + json={"processingStatus": ReportProcessingStatus.done, "dataEndTime": "2022-10-03T00:00:00Z", "reportDocumentId": document_id}, + ) + requests_mock.register_uri( + "GET", + f"https://test.url/reports/2021-06-30/documents/{document_id}", + status_code=200, + json={"reportDocumentId": document_id}, + ) + + stream = SomeReportStream(**report_init_kwargs) + with patch.object(stream, "parse_response", return_value=[{"some_key": "some_value"}]): + records = list(stream.read_records(sync_mode=SyncMode.full_refresh)) + assert records[0] == {"some_key": "some_value", "dataEndTime": "2022-10-03"} + + def test_read_records_retrieve_forbidden(self, report_init_kwargs, mocker, requests_mock, caplog): + mocker.patch("time.sleep", lambda x: None) + requests_mock.register_uri( + "POST", + "https://api.amazon.com/auth/o2/token", + status_code=200, + json={"access_token": "access_token", "expires_in": "3600"}, + ) + + report_id = "some_report_id" + requests_mock.register_uri( + "POST", + "https://test.url/reports/2021-06-30/reports", + status_code=403, + json={"reportId": report_id}, + reason="Forbidden" + ) + + stream = SomeReportStream(**report_init_kwargs) + assert list(stream.read_records(sync_mode=SyncMode.full_refresh)) == [] + assert ( + "The endpoint https://test.url/reports/2021-06-30/reports returned 403: Forbidden. " + "This is most likely due to insufficient permissions on the credentials in use. " + "Try to grant required permissions/scopes or re-authenticate." + ) in caplog.messages[-1] + + +class TestVendorDirectFulfillmentShipping: + @pytest.mark.parametrize( + ("start_date", "end_date", "expected_params"), + ( + ("2022-09-01T00:00:00Z", None, {"createdAfter": "2022-09-01T00:00:00Z", "createdBefore": "2022-09-05T00:00:00Z"}), + ("2022-08-01T00:00:00Z", None, {"createdAfter": "2022-08-28T23:00:00Z", "createdBefore": "2022-09-05T00:00:00Z"}), + ( + "2022-09-01T00:00:00Z", + "2022-09-05T00:00:00Z", + {"createdAfter": "2022-09-01T00:00:00Z", "createdBefore": "2022-09-05T00:00:00Z"}, + ), + ), + ) + def test_request_params(self, report_init_kwargs, start_date, end_date, expected_params): + report_init_kwargs["replication_start_date"] = start_date + report_init_kwargs["replication_end_date"] = end_date + + stream = VendorDirectFulfillmentShipping(**report_init_kwargs) + with patch("pendulum.now", return_value=pendulum.parse("2022-09-05T00:00:00Z")): + assert stream.request_params(stream_state={}) == expected_params diff --git a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_transform_function.py b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_transform_function.py index 9f4df656aded..a2e343c6f077 100644 --- a/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_transform_function.py +++ b/airbyte-integrations/connectors/source-amazon-seller-partner/unit_tests/test_transform_function.py @@ -2,30 +2,26 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import pytest -from source_amazon_seller_partner.auth import AWSSignature -from source_amazon_seller_partner.streams import SellerFeedbackReports +from source_amazon_seller_partner.streams import ( + FlatFileSettlementV2Reports, + LedgerDetailedViewReports, + MerchantListingsFypReport, + MerchantListingsReports, + SellerFeedbackReports, +) def reports_stream(marketplace_id): - aws_signature = AWSSignature( - service="execute-api", - aws_access_key_id="AccessKeyId", - aws_secret_access_key="SecretAccessKey", - aws_session_token="SessionToken", - region="Mars", - ) stream = SellerFeedbackReports( url_base="https://test.url", - aws_signature=aws_signature, replication_start_date="2010-01-25T00:00:00Z", replication_end_date="2017-02-25T00:00:00Z", marketplace_id=marketplace_id, authenticator=None, period_in_days=0, report_options=None, - advanced_stream_options=None, - max_wait_seconds=0, ) return stream @@ -64,3 +60,78 @@ def test_transform_seller_feedback(marketplace_id, input_data, expected_data): transformer.transform(input_data, schema) assert input_data == expected_data + + +@pytest.mark.parametrize( + ("input_data", "expected_data"), + ( + ( + {"item-name": "GiftBox", "open-date": "2022-07-11 01:34:18 PDT", "dataEndTime": "2022-07-31"}, + {"item-name": "GiftBox", "open-date": "2022-07-11T01:34:18-07:00", "dataEndTime": "2022-07-31"}, + ), + ( + {"item-name": "GiftBox", "open-date": "", "dataEndTime": "2022-07-31"}, + {"item-name": "GiftBox", "open-date": "", "dataEndTime": "2022-07-31"}, + ), + ), +) +def test_transform_merchant_reports(report_init_kwargs, input_data, expected_data): + stream = MerchantListingsReports(**report_init_kwargs) + transformer = stream.transformer + schema = stream.get_json_schema() + transformer.transform(input_data, schema) + assert input_data == expected_data + + +@pytest.mark.parametrize( + ("input_data", "expected_data"), + ( + ( + {"Product name": "GiftBox", "Condition": "11", "Status Change Date": "Jul 29, 2022", "dataEndTime": "2022-07-31"}, + {"Product name": "GiftBox", "Condition": "11", "Status Change Date": "2022-07-29", "dataEndTime": "2022-07-31"}, + ), + ( + {"Product name": "GiftBox", "Condition": "11", "Status Change Date": "", "dataEndTime": "2022-07-31"}, + {"Product name": "GiftBox", "Condition": "11", "Status Change Date": "", "dataEndTime": "2022-07-31"}, + ), + ), +) +def test_transform_merchant_fyp_reports(report_init_kwargs, input_data, expected_data): + stream = MerchantListingsFypReport(**report_init_kwargs) + transformer = stream.transformer + schema = stream.get_json_schema() + transformer.transform(input_data, schema) + assert input_data == expected_data + + +@pytest.mark.parametrize( + ("input_data", "expected_data"), + ( + ({"Date": "7/29/2022", "dataEndTime": "2022-07-31"}, {"Date": "2022-07-29", "dataEndTime": "2022-07-31"}), + ({"Date": "", "dataEndTime": "2022-07-31"}, {"Date": "", "dataEndTime": "2022-07-31"}), + ), +) +def test_transform_ledger_reports(report_init_kwargs, input_data, expected_data): + stream = LedgerDetailedViewReports(**report_init_kwargs) + transformer = stream.transformer + schema = stream.get_json_schema() + transformer.transform(input_data, schema) + assert input_data == expected_data + + +@pytest.mark.parametrize( + ("input_data", "expected_data"), + ( + ( + {"posted-date": "2023-11-09T18:44:35+00:00", "dataEndTime": "2022-07-31"}, + {"posted-date": "2023-11-09T18:44:35+00:00", "dataEndTime": "2022-07-31"}, + ), + ({"posted-date": "", "dataEndTime": "2022-07-31"}, {"posted-date": None, "dataEndTime": "2022-07-31"}), + ), +) +def test_transform_settlement_reports(report_init_kwargs, input_data, expected_data): + stream = FlatFileSettlementV2Reports(**report_init_kwargs) + transformer = stream.transformer + schema = stream.get_json_schema() + transformer.transform(input_data, schema) + assert input_data == expected_data diff --git a/airbyte-integrations/connectors/source-amazon-sqs/Dockerfile b/airbyte-integrations/connectors/source-amazon-sqs/Dockerfile index f8020036fba8..ef097b4cbff5 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/Dockerfile +++ b/airbyte-integrations/connectors/source-amazon-sqs/Dockerfile @@ -34,5 +34,5 @@ COPY source_amazon_sqs ./source_amazon_sqs ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-amazon-sqs diff --git a/airbyte-integrations/connectors/source-amazon-sqs/README.md b/airbyte-integrations/connectors/source-amazon-sqs/README.md index 348214fff027..007a1acdf02d 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/README.md +++ b/airbyte-integrations/connectors/source-amazon-sqs/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-amazon-sqs:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/amazon-sqs) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_amazon_sqs/spec.json` file. @@ -54,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-amazon-sqs:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-amazon-sqs build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-amazon-sqs:airbyteDocker +An image will be built with the tag `airbyte/source-amazon-sqs:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-amazon-sqs:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,44 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-amazon-sqs:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-amazon-sqs:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-amazon-sqs:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-amazon-sqs test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-amazon-sqs:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-amazon-sqs:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -122,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-amazon-sqs test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/amazon-sqs.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-amazon-sqs/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-amazon-sqs/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-amazon-sqs/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-amazon-sqs/build.gradle b/airbyte-integrations/connectors/source-amazon-sqs/build.gradle deleted file mode 100644 index bf5c06f39a83..000000000000 --- a/airbyte-integrations/connectors/source-amazon-sqs/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_amazon_sqs_singer' -} diff --git a/airbyte-integrations/connectors/source-amazon-sqs/metadata.yaml b/airbyte-integrations/connectors/source-amazon-sqs/metadata.yaml index a62580bedfa1..457e4edafe0e 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/metadata.yaml +++ b/airbyte-integrations/connectors/source-amazon-sqs/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 983fd355-6bf3-4709-91b5-37afa391eeb6 - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 dockerRepository: airbyte/source-amazon-sqs documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-sqs githubIssueLabel: source-amazon-sqs diff --git a/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/spec.json b/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/spec.json index 0bb7d64eded0..4c71ef50c0b8 100644 --- a/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/spec.json +++ b/airbyte-integrations/connectors/source-amazon-sqs/source_amazon_sqs/spec.json @@ -21,31 +21,39 @@ "description": "AWS Region of the SQS Queue", "type": "string", "enum": [ - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", "af-south-1", "ap-east-1", - "ap-south-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", + "ap-south-1", + "ap-south-2", "ap-southeast-1", "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", "ca-central-1", + "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", + "eu-central-2", "eu-north-1", "eu-south-1", + "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", - "sa-east-1", + "il-central-1", + "me-central-1", "me-south-1", + "sa-east-1", + "us-east-1", + "us-east-2", "us-gov-east-1", - "us-gov-west-1" + "us-gov-west-1", + "us-west-1", + "us-west-2" ], "order": 1 }, diff --git a/airbyte-integrations/connectors/source-amplitude/Dockerfile b/airbyte-integrations/connectors/source-amplitude/Dockerfile deleted file mode 100644 index 5b7b21ada222..000000000000 --- a/airbyte-integrations/connectors/source-amplitude/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_amplitude ./source_amplitude - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.2.4 -LABEL io.airbyte.name=airbyte/source-amplitude diff --git a/airbyte-integrations/connectors/source-amplitude/README.md b/airbyte-integrations/connectors/source-amplitude/README.md index 17b153e7bbe2..6ace21da93d7 100644 --- a/airbyte-integrations/connectors/source-amplitude/README.md +++ b/airbyte-integrations/connectors/source-amplitude/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-amplitude:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/amplitude) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_amplitude/spec.yaml` file. @@ -24,19 +16,70 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-amplitude:dev + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-amplitude build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-amplitude:dev`. -You can also build the connector image via Gradle: +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-amplitude:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-amplitude:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-amplitude:dev . +# Running the spec command against your patched connector +docker run airbyte/source-amplitude:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -45,29 +88,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-amplitude:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-amplitude:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-amplitude:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-amplitude test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-amplitude:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-amplitude:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -77,8 +107,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-amplitude test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/amplitude.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml b/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml index f2d615d750d4..79eb3c22d680 100644 --- a/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-amplitude/acceptance-test-config.yml @@ -6,6 +6,8 @@ acceptance_tests: spec: tests: - spec_path: "source_amplitude/spec.yaml" + backward_compatibility_tests_config: + disable_for_version: 0.3.2 # `start_date` format changed to format: date-time connection: tests: - config_path: "secrets/config.json" @@ -16,7 +18,7 @@ acceptance_tests: tests: - config_path: "secrets/config.json" backward_compatibility_tests_config: - disable_for_version: 0.1.24 # cursor field for stream events has been changed + disable_for_version: 0.3.5 # `date` format changed to format: date-time in the AverageSessionLength stream basic_read: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-amplitude/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-amplitude/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-amplitude/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-amplitude/build.gradle b/airbyte-integrations/connectors/source-amplitude/build.gradle deleted file mode 100644 index 128e209c5984..000000000000 --- a/airbyte-integrations/connectors/source-amplitude/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_amplitude' -} diff --git a/airbyte-integrations/connectors/source-amplitude/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-amplitude/integration_tests/expected_records.jsonl index 412b5966b9bc..a23992c53d61 100644 --- a/airbyte-integrations/connectors/source-amplitude/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-amplitude/integration_tests/expected_records.jsonl @@ -1,116 +1,16 @@ -{"stream":"active_users","data":{"date":"2023-02-01","statistics":{"(none)":1,"Ukraine":0}},"emitted_at":1676633494310} -{"stream":"active_users","data":{"date":"2023-02-02","statistics":{"(none)":1,"Ukraine":0}},"emitted_at":1676633494310} -{"stream":"active_users","data":{"date":"2023-02-03","statistics":{"(none)":1,"Ukraine":0}},"emitted_at":1676633494310} -{"stream":"active_users","data":{"date":"2023-02-04","statistics":{"(none)":1,"Ukraine":0}},"emitted_at":1676633494310} -{"stream":"active_users","data":{"date":"2023-02-05","statistics":{"(none)":1,"Ukraine":0}},"emitted_at":1676633494310} -{"stream":"active_users","data":{"date":"2023-02-06","statistics":{"(none)":1,"Ukraine":0}},"emitted_at":1676633494311} -{"stream":"active_users","data":{"date":"2023-02-07","statistics":{"(none)":1,"Ukraine":0}},"emitted_at":1676633494311} -{"stream":"active_users","data":{"date":"2023-02-08","statistics":{"(none)":1,"Ukraine":0}},"emitted_at":1676633494311} -{"stream":"active_users","data":{"date":"2023-02-09","statistics":{"(none)":1,"Ukraine":1}},"emitted_at":1676633494311} -{"stream":"active_users","data":{"date":"2023-02-10","statistics":{"(none)":1,"Ukraine":0}},"emitted_at":1676633494311} -{"stream":"active_users","data":{"date":"2023-02-11","statistics":{"(none)":1,"Ukraine":0}},"emitted_at":1676633494312} -{"stream":"active_users","data":{"date":"2023-02-12","statistics":{"(none)":1,"Ukraine":0}},"emitted_at":1676633494312} -{"stream":"active_users","data":{"date":"2023-02-13","statistics":{"(none)":1,"Ukraine":0}},"emitted_at":1676633494312} -{"stream":"active_users","data":{"date":"2023-02-14","statistics":{"(none)":1,"Ukraine":0}},"emitted_at":1676633494312} -{"stream":"active_users","data":{"date":"2023-02-15","statistics":{"(none)":1,"Ukraine":0}},"emitted_at":1676633494312} -{"stream":"average_session_length","data":{"date":"2023-02-01","length":0},"emitted_at":1677160664431} -{"stream":"average_session_length","data":{"date":"2023-02-02","length":0},"emitted_at":1677160664432} -{"stream":"average_session_length","data":{"date":"2023-02-03","length":0},"emitted_at":1677160664434} -{"stream":"average_session_length","data":{"date":"2023-02-04","length":0},"emitted_at":1677160664435} -{"stream":"average_session_length","data":{"date":"2023-02-05","length":0},"emitted_at":1677160664436} -{"stream":"average_session_length","data":{"date":"2023-02-06","length":0},"emitted_at":1677160664438} -{"stream":"average_session_length","data":{"date":"2023-02-07","length":0},"emitted_at":1677160664439} -{"stream":"average_session_length","data":{"date":"2023-02-08","length":0},"emitted_at":1677160664441} -{"stream":"average_session_length","data":{"date":"2023-02-09","length":0},"emitted_at":1677160664442} -{"stream":"average_session_length","data":{"date":"2023-02-10","length":0},"emitted_at":1677160664443} -{"stream":"average_session_length","data":{"date":"2023-02-11","length":0},"emitted_at":1677160664445} -{"stream":"average_session_length","data":{"date":"2023-02-12","length":0},"emitted_at":1677160664446} -{"stream":"average_session_length","data":{"date":"2023-02-13","length":0},"emitted_at":1677160664447} -{"stream":"average_session_length","data":{"date":"2023-02-14","length":0},"emitted_at":1677160664449} -{"stream":"average_session_length","data":{"date":"2023-02-15","length":0},"emitted_at":1677160664450} -{"stream":"average_session_length","data":{"date":"2023-02-16","length":0},"emitted_at":1677160664803} -{"stream":"average_session_length","data":{"date":"2023-02-17","length":0},"emitted_at":1677160664805} -{"stream":"average_session_length","data":{"date":"2023-02-18","length":0},"emitted_at":1677160664806} -{"stream":"average_session_length","data":{"date":"2023-02-19","length":0},"emitted_at":1677160664812} -{"stream":"average_session_length","data":{"date":"2023-02-20","length":0},"emitted_at":1677160664814} -{"stream":"average_session_length","data":{"date":"2023-02-21","length":0},"emitted_at":1677160664815} -{"stream":"average_session_length","data":{"date":"2023-02-22","length":0},"emitted_at":1677160664816} -{"stream":"average_session_length","data":{"date":"2023-02-23","length":0},"emitted_at":1677160664818} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-DESKTOP-2023-01-31","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-01-31T08:00:00+00:00","client_upload_time":"2023-02-01T12:02:17.685000+00:00","country":null,"data":{"user_properties_updated":true,"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":82672889,"event_properties":{"ad_metrics.cost":0.055104,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"DESKTOP","ad_metrics.impressions":202,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-01-31T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-01T12:02:40.918000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-01T12:02:17.685000+00:00","server_upload_time":"2023-02-01T12:02:36.650000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"53e385c4-a228-11ed-8e51-171d53d06523","version_name":null},"emitted_at":1677160703462} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-TABLET-2023-01-31","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-01-31T08:00:00+00:00","client_upload_time":"2023-02-01T12:02:17.685000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":273196606,"event_properties":{"ad_metrics.cost":0.10283,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"TABLET","ad_metrics.impressions":188,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-01-31T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-01T12:02:40.918000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-01T12:02:17.685000+00:00","server_upload_time":"2023-02-01T12:02:36.650000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"53e39277-a228-11ed-9400-171d53d06523","version_name":null},"emitted_at":1677160703465} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-MOBILE-2023-01-31","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-01-31T08:00:00+00:00","client_upload_time":"2023-02-01T12:02:17.685000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":695784978,"event_properties":{"ad_metrics.cost":1.781222,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"MOBILE","ad_metrics.impressions":2406,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":3,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":3,"campaign_id":19410069806},"event_time":"2023-01-31T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-01T12:02:40.918000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-01T12:02:17.685000+00:00","server_upload_time":"2023-02-01T12:02:36.650000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"53e38ca8-a228-11ed-a707-171d53d06523","version_name":null},"emitted_at":1677160703491} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-DESKTOP-2023-02-01","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-01T08:00:00+00:00","client_upload_time":"2023-02-02T12:03:25.005000+00:00","country":null,"data":{"user_properties_updated":true,"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":579725194,"event_properties":{"ad_metrics.cost":0.087394,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"DESKTOP","ad_metrics.impressions":258,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-01T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-02T12:03:52.152000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-02T12:03:25.005000+00:00","server_upload_time":"2023-02-02T12:03:43.846000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"a8b6b648-a2f1-11ed-b981-774fa598444a","version_name":null},"emitted_at":1677160704054} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-TABLET-2023-02-01","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-01T08:00:00+00:00","client_upload_time":"2023-02-02T12:03:25.005000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":186292011,"event_properties":{"ad_metrics.cost":0.237483,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"TABLET","ad_metrics.impressions":268,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-01T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-02T12:03:52.152000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-02T12:03:25.005000+00:00","server_upload_time":"2023-02-02T12:03:43.846000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"a8b6c322-a2f1-11ed-9dbe-774fa598444a","version_name":null},"emitted_at":1677160704057} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-MOBILE-2023-02-01","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-01T08:00:00+00:00","client_upload_time":"2023-02-02T12:03:25.005000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":343534293,"event_properties":{"ad_metrics.cost":1.913081,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"MOBILE","ad_metrics.impressions":2380,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":4,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":4,"campaign_id":19410069806},"event_time":"2023-02-01T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-02T12:03:52.152000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-02T12:03:25.005000+00:00","server_upload_time":"2023-02-02T12:03:43.846000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"a8b6bd56-a2f1-11ed-84fd-774fa598444a","version_name":null},"emitted_at":1677160704059} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-DESKTOP-2023-02-02","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-02T08:00:00+00:00","client_upload_time":"2023-02-03T12:02:59.374000+00:00","country":null,"data":{"user_properties_updated":true,"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":444502983,"event_properties":{"ad_metrics.cost":0.098846,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"DESKTOP","ad_metrics.impressions":260,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-02T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-03T12:03:14.894000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-03T12:02:59.374000+00:00","server_upload_time":"2023-02-03T12:03:12.182000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"bcf76cb7-a3ba-11ed-bbc1-ef5367694578","version_name":null},"emitted_at":1677160704734} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-TABLET-2023-02-02","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-02T08:00:00+00:00","client_upload_time":"2023-02-03T12:02:59.374000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":540530303,"event_properties":{"ad_metrics.cost":0.255795,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"TABLET","ad_metrics.impressions":318,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-02T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-03T12:03:14.894000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-03T12:02:59.374000+00:00","server_upload_time":"2023-02-03T12:03:12.182000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"bcf77ba6-a3ba-11ed-b2f9-ef5367694578","version_name":null},"emitted_at":1677160704736} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-MOBILE-2023-02-02","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-02T08:00:00+00:00","client_upload_time":"2023-02-03T12:02:59.374000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":472999226,"event_properties":{"ad_metrics.cost":1.91728,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"MOBILE","ad_metrics.impressions":2473,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":3,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":3,"campaign_id":19410069806},"event_time":"2023-02-02T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-03T12:03:14.894000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-03T12:02:59.374000+00:00","server_upload_time":"2023-02-03T12:03:12.182000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"bcf77529-a3ba-11ed-9ee2-ef5367694578","version_name":null},"emitted_at":1677160704744} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-DESKTOP-2023-02-03","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-03T08:00:00+00:00","client_upload_time":"2023-02-04T12:02:25.436000+00:00","country":null,"data":{"user_properties_updated":true,"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":830727485,"event_properties":{"ad_metrics.cost":0.075498,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"DESKTOP","ad_metrics.impressions":229,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-03T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-04T12:02:41.144000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-04T12:02:25.436000+00:00","server_upload_time":"2023-02-04T12:02:37.836000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"d3388c42-a483-11ed-a241-a9a73855976a","version_name":null},"emitted_at":1677160705252} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-TABLET-2023-02-03","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-03T08:00:00+00:00","client_upload_time":"2023-02-04T12:02:25.436000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":738101948,"event_properties":{"ad_metrics.cost":0.109966,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"TABLET","ad_metrics.impressions":166,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-03T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-04T12:02:41.144000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-04T12:02:25.436000+00:00","server_upload_time":"2023-02-04T12:02:37.836000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"d3389906-a483-11ed-9a82-a9a73855976a","version_name":null},"emitted_at":1677160705254} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-MOBILE-2023-02-03","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-03T08:00:00+00:00","client_upload_time":"2023-02-04T12:02:25.436000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":181349706,"event_properties":{"ad_metrics.cost":2.031292,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"MOBILE","ad_metrics.impressions":2580,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":5,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":5,"campaign_id":19410069806},"event_time":"2023-02-03T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-04T12:02:41.144000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-04T12:02:25.436000+00:00","server_upload_time":"2023-02-04T12:02:37.836000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"d3389329-a483-11ed-90ec-a9a73855976a","version_name":null},"emitted_at":1677160705256} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-DESKTOP-2023-02-04","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-04T08:00:00+00:00","client_upload_time":"2023-02-05T12:04:06.941000+00:00","country":null,"data":{"user_properties_updated":true,"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":920042324,"event_properties":{"ad_metrics.cost":0.077221,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"DESKTOP","ad_metrics.impressions":272,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-04T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-05T12:17:06.057000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-05T12:04:06.941000+00:00","server_upload_time":"2023-02-05T12:15:52.509000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"01311b3f-a54f-11ed-bd6b-b9b1c4f00a9c","version_name":null},"emitted_at":1677160705685} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-CONNECTED_TV-2023-02-04","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-04T08:00:00+00:00","client_upload_time":"2023-02-05T12:04:06.941000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":323952360,"event_properties":{"ad_metrics.cost":0.000117,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"CONNECTED_TV","ad_metrics.impressions":6,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-04T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-05T12:17:06.057000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-05T12:04:06.941000+00:00","server_upload_time":"2023-02-05T12:15:52.509000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"01312d7f-a54f-11ed-b760-b9b1c4f00a9c","version_name":null},"emitted_at":1677160705687} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-TABLET-2023-02-04","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-04T08:00:00+00:00","client_upload_time":"2023-02-05T12:04:06.941000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":335634502,"event_properties":{"ad_metrics.cost":0.180594,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"TABLET","ad_metrics.impressions":238,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":2,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":2,"campaign_id":19410069806},"event_time":"2023-02-04T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-05T12:17:06.057000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-05T12:04:06.941000+00:00","server_upload_time":"2023-02-05T12:15:52.509000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"013127e4-a54f-11ed-94cb-b9b1c4f00a9c","version_name":null},"emitted_at":1677160705689} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-MOBILE-2023-02-04","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-04T08:00:00+00:00","client_upload_time":"2023-02-05T12:04:06.941000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":978891508,"event_properties":{"ad_metrics.cost":1.956155,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"MOBILE","ad_metrics.impressions":2409,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":5,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":5,"campaign_id":19410069806},"event_time":"2023-02-04T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-05T12:17:06.057000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-05T12:04:06.941000+00:00","server_upload_time":"2023-02-05T12:15:52.509000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"01312229-a54f-11ed-b0ff-b9b1c4f00a9c","version_name":null},"emitted_at":1677160705691} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-DESKTOP-2023-02-05","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-05T08:00:00+00:00","client_upload_time":"2023-02-06T12:01:06.017000+00:00","country":null,"data":{"user_properties_updated":true,"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":862973577,"event_properties":{"ad_metrics.cost":0.071026,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"DESKTOP","ad_metrics.impressions":195,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-05T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-06T12:02:08.452000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-06T12:01:06.017000+00:00","server_upload_time":"2023-02-06T12:01:17.335000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"14915d67-a616-11ed-b37b-1d24c3d1426b","version_name":null},"emitted_at":1677160706263} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-TABLET-2023-02-05","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-05T08:00:00+00:00","client_upload_time":"2023-02-06T12:01:06.017000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":183007151,"event_properties":{"ad_metrics.cost":0.144935,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"TABLET","ad_metrics.impressions":188,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-05T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-06T12:02:08.452000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-06T12:01:06.017000+00:00","server_upload_time":"2023-02-06T12:01:17.335000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"14916bdc-a616-11ed-abef-1d24c3d1426b","version_name":null},"emitted_at":1677160706329} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-MOBILE-2023-02-05","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-05T08:00:00+00:00","client_upload_time":"2023-02-06T12:01:06.017000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":681776226,"event_properties":{"ad_metrics.cost":1.997995,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"MOBILE","ad_metrics.impressions":2583,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":3,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":3,"campaign_id":19410069806},"event_time":"2023-02-05T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-06T12:02:08.452000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-06T12:01:06.017000+00:00","server_upload_time":"2023-02-06T12:01:17.335000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"149164c6-a616-11ed-b3e0-1d24c3d1426b","version_name":null},"emitted_at":1677160706331} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-DESKTOP-2023-02-06","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-06T08:00:00+00:00","client_upload_time":"2023-02-07T12:01:48.859000+00:00","country":null,"data":{"user_properties_updated":true,"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":970466740,"event_properties":{"ad_metrics.cost":0.078221,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"DESKTOP","ad_metrics.impressions":278,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-06T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-07T12:02:07.033000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-07T12:01:48.859000+00:00","server_upload_time":"2023-02-07T12:02:04.872000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"3e28db97-a6df-11ed-be64-5d7f35c3b01a","version_name":null},"emitted_at":1677160706837} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-CONNECTED_TV-2023-02-06","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-06T08:00:00+00:00","client_upload_time":"2023-02-07T12:01:48.859000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":567803347,"event_properties":{"ad_metrics.cost":0,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"CONNECTED_TV","ad_metrics.impressions":4,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-06T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-07T12:02:07.033000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-07T12:01:48.859000+00:00","server_upload_time":"2023-02-07T12:02:04.872000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"3e28ee9f-a6df-11ed-8c6f-5d7f35c3b01a","version_name":null},"emitted_at":1677160706848} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-TABLET-2023-02-06","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-06T08:00:00+00:00","client_upload_time":"2023-02-07T12:01:48.859000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":557070126,"event_properties":{"ad_metrics.cost":0.192373,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"TABLET","ad_metrics.impressions":213,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-06T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-07T12:02:07.033000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-07T12:01:48.859000+00:00","server_upload_time":"2023-02-07T12:02:04.872000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"3e28e905-a6df-11ed-b2b0-5d7f35c3b01a","version_name":null},"emitted_at":1677160706851} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-MOBILE-2023-02-06","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-06T08:00:00+00:00","client_upload_time":"2023-02-07T12:01:48.859000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":701875278,"event_properties":{"ad_metrics.cost":2.018299,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"MOBILE","ad_metrics.impressions":2595,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":4,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":4,"campaign_id":19410069806},"event_time":"2023-02-06T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-07T12:02:07.033000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-07T12:01:48.859000+00:00","server_upload_time":"2023-02-07T12:02:04.872000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"3e28e312-a6df-11ed-8dec-5d7f35c3b01a","version_name":null},"emitted_at":1677160706995} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-DESKTOP-2023-02-07","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-07T08:00:00+00:00","client_upload_time":"2023-02-08T12:02:25.066000+00:00","country":null,"data":{"user_properties_updated":true,"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":215560287,"event_properties":{"ad_metrics.cost":0.049814,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"DESKTOP","ad_metrics.impressions":208,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":1,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":1,"campaign_id":19410069806},"event_time":"2023-02-07T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-08T12:02:39.178000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-08T12:02:25.066000+00:00","server_upload_time":"2023-02-08T12:02:35.998000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"7bba250f-a7a8-11ed-9ce6-3718c73e0dcb","version_name":null},"emitted_at":1677160707555} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-TABLET-2023-02-07","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-07T08:00:00+00:00","client_upload_time":"2023-02-08T12:02:25.066000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":750045948,"event_properties":{"ad_metrics.cost":0.145436,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"TABLET","ad_metrics.impressions":180,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-07T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-08T12:02:39.178000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-08T12:02:25.066000+00:00","server_upload_time":"2023-02-08T12:02:35.998000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"7bba32f5-a7a8-11ed-a10d-3718c73e0dcb","version_name":null},"emitted_at":1677160707557} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-MOBILE-2023-02-07","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-07T08:00:00+00:00","client_upload_time":"2023-02-08T12:02:25.066000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":54427230,"event_properties":{"ad_metrics.cost":2.093716,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"MOBILE","ad_metrics.impressions":2694,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":3,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":3,"campaign_id":19410069806},"event_time":"2023-02-07T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-08T12:02:39.178000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-08T12:02:25.066000+00:00","server_upload_time":"2023-02-08T12:02:35.998000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{},"uuid":"7bba2cff-a7a8-11ed-baf0-3718c73e0dcb","version_name":null},"emitted_at":1677160707559} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-DESKTOP-2023-02-08","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-08T08:00:00+00:00","client_upload_time":"2023-02-09T12:02:17.656000+00:00","country":null,"data":{"user_properties_updated":true,"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":500159953,"event_properties":{"ad_metrics.cost":0.063632,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"DESKTOP","ad_metrics.impressions":275,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-08T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-09T12:02:40.173000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-09T12:02:17.656000+00:00","server_upload_time":"2023-02-09T12:02:37.115000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"a6b63b9b-a871-11ed-8e0e-556bceaee760","version_name":null},"emitted_at":1677160708339} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-CONNECTED_TV-2023-02-08","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-08T08:00:00+00:00","client_upload_time":"2023-02-09T12:02:17.656000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":91323750,"event_properties":{"ad_metrics.cost":2.4e-05,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"CONNECTED_TV","ad_metrics.impressions":2,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-08T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-09T12:02:40.173000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-09T12:02:17.656000+00:00","server_upload_time":"2023-02-09T12:02:37.115000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"a6b65793-a871-11ed-9158-556bceaee760","version_name":null},"emitted_at":1677160708341} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-TABLET-2023-02-08","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-08T08:00:00+00:00","client_upload_time":"2023-02-09T12:02:17.656000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":248713684,"event_properties":{"ad_metrics.cost":0.111586,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"TABLET","ad_metrics.impressions":188,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":1,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":1,"campaign_id":19410069806},"event_time":"2023-02-08T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-09T12:02:40.173000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-09T12:02:17.656000+00:00","server_upload_time":"2023-02-09T12:02:37.115000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"a6b6514c-a871-11ed-963a-556bceaee760","version_name":null},"emitted_at":1677160708344} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-MOBILE-2023-02-08","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-08T08:00:00+00:00","client_upload_time":"2023-02-09T12:02:17.656000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":771697037,"event_properties":{"ad_metrics.cost":2.075305,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"MOBILE","ad_metrics.impressions":2687,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":7,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":7,"campaign_id":19410069806},"event_time":"2023-02-08T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-09T12:02:40.173000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-09T12:02:17.656000+00:00","server_upload_time":"2023-02-09T12:02:37.115000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"a6b64994-a871-11ed-bdac-556bceaee760","version_name":null},"emitted_at":1677160708346} -{"stream":"events","data":{"$insert_id":"13648a56-a2bf-468d-89ce-af8148d5eb4b","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":568771836625,"app":434735,"city":"Kyiv","client_event_time":"2023-02-09T09:57:33+00:00","client_upload_time":"2023-02-09T09:57:33+00:00","country":"Ukraine","data":{"user_properties_updated":true,"group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"ff91a939-5a66-57f3-934c-6e58f7bd6da2","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":115597971,"event_properties":{"keyBool":true,"keyString":"valueString","keyInt":11},"event_time":"2023-02-09T09:57:33+00:00","event_type":"[Shopify] Connect Amplitude Test","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":"85.209.47.207","is_attribution_event":null,"language":null,"library":"amplitude-node/1.10.0","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-09T09:57:35.351000+00:00","region":"Kyiv City","sample_rate":null,"server_received_time":"2023-02-09T09:57:33+00:00","server_upload_time":"2023-02-09T09:57:33.014000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":"testuserEmail@gmail.com","user_properties":{},"uuid":"2d7ec4b1-a860-11ed-aae8-01da73a04a5b","version_name":null},"emitted_at":1677160708348} -{"stream":"events","data":{"$insert_id":"ca3fb836-163b-4898-9d59-1586fe103693","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":568771836625,"app":434735,"city":"Kyiv","client_event_time":"2023-02-09T09:57:39.118000+00:00","client_upload_time":"2023-02-09T09:57:39.118000+00:00","country":"Ukraine","data":{"group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"ff91a939-5a66-57f3-934c-6e58f7bd6da2","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":748377217,"event_properties":{"keyBool":true,"keyString":"valueString","keyInt":11},"event_time":"2023-02-09T09:57:39.118000+00:00","event_type":"[Shopify] Connect Amplitude Test","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":"85.209.47.207","is_attribution_event":null,"language":null,"library":"amplitude-node/1.10.0","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-09T09:57:40.295000+00:00","region":"Kyiv City","sample_rate":null,"server_received_time":"2023-02-09T09:57:39.118000+00:00","server_upload_time":"2023-02-09T09:57:39.123000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":"testuserEmail@gmail.com","user_properties":{},"uuid":"30869ad0-a860-11ed-a807-f949e6b8dfb8","version_name":null},"emitted_at":1677160708350} -{"stream":"events","data":{"$insert_id":"6bf42abd-5033-4ea6-9574-c1c674a4d32d","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":568771836625,"app":434735,"city":"Kyiv","client_event_time":"2023-02-09T09:57:44.390000+00:00","client_upload_time":"2023-02-09T09:57:44.390000+00:00","country":"Ukraine","data":{"group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"ff91a939-5a66-57f3-934c-6e58f7bd6da2","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":969823192,"event_properties":{"keyBool":true,"keyString":"valueString","keyInt":11},"event_time":"2023-02-09T09:57:44.390000+00:00","event_type":"[Shopify] Connect Amplitude Test","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":"85.209.47.207","is_attribution_event":null,"language":null,"library":"amplitude-node/1.10.0","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-09T09:57:45.721000+00:00","region":"Kyiv City","sample_rate":null,"server_received_time":"2023-02-09T09:57:44.390000+00:00","server_upload_time":"2023-02-09T09:57:44.394000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":"testuserEmail@gmail.com","user_properties":{},"uuid":"33c35984-a860-11ed-8460-f949e6b8dfb8","version_name":null},"emitted_at":1677160708352} -{"stream":"events","data":{"$insert_id":"afb8f963-cae5-4232-b20d-9047440d29c9","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":568771836625,"app":434735,"city":"Kyiv","client_event_time":"2023-02-09T09:58:07.981000+00:00","client_upload_time":"2023-02-09T09:58:07.981000+00:00","country":"Ukraine","data":{"group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"ff91a939-5a66-57f3-934c-6e58f7bd6da2","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":318819040,"event_properties":{"keyBool":true,"keyString":"valueString","keyInt":11},"event_time":"2023-02-09T09:58:07.981000+00:00","event_type":"[Shopify] Connect Amplitude Test","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":"85.209.47.207","is_attribution_event":null,"language":null,"library":"amplitude-node/1.10.0","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-09T09:58:10.307000+00:00","region":"Kyiv City","sample_rate":null,"server_received_time":"2023-02-09T09:58:07.981000+00:00","server_upload_time":"2023-02-09T09:58:07.987000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":"testuserEmail@gmail.com","user_properties":{},"uuid":"426b1ef9-a860-11ed-8868-f949e6b8dfb8","version_name":null},"emitted_at":1677160708354} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-DESKTOP-2023-02-09","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-09T08:00:00+00:00","client_upload_time":"2023-02-10T12:03:21.518000+00:00","country":null,"data":{"user_properties_updated":true,"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":549198577,"event_properties":{"ad_metrics.cost":0.091194,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"DESKTOP","ad_metrics.impressions":240,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-09T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-10T12:03:43.615000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-10T12:03:21.518000+00:00","server_upload_time":"2023-02-10T12:03:42.091000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"f6f54177-a93a-11ed-9aef-65efbbbc3a1e","version_name":null},"emitted_at":1677160708954} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-CONNECTED_TV-2023-02-09","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-09T08:00:00+00:00","client_upload_time":"2023-02-10T12:03:21.518000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":970373985,"event_properties":{"ad_metrics.cost":0.000757,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"CONNECTED_TV","ad_metrics.impressions":4,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-09T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-10T12:03:43.615000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-10T12:03:21.518000+00:00","server_upload_time":"2023-02-10T12:03:42.091000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"f6f553c2-a93a-11ed-bc53-65efbbbc3a1e","version_name":null},"emitted_at":1677160708956} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-TABLET-2023-02-09","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-09T08:00:00+00:00","client_upload_time":"2023-02-10T12:03:21.518000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":574382501,"event_properties":{"ad_metrics.cost":0.109206,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"TABLET","ad_metrics.impressions":158,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":2,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":2,"campaign_id":19410069806},"event_time":"2023-02-09T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-10T12:03:43.615000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-10T12:03:21.518000+00:00","server_upload_time":"2023-02-10T12:03:42.091000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"f6f54e4b-a93a-11ed-8085-65efbbbc3a1e","version_name":null},"emitted_at":1677160708958} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-MOBILE-2023-02-09","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-09T08:00:00+00:00","client_upload_time":"2023-02-10T12:03:21.518000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":557690644,"event_properties":{"ad_metrics.cost":2.006873,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"MOBILE","ad_metrics.impressions":2565,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":6,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":6,"campaign_id":19410069806},"event_time":"2023-02-09T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-10T12:03:43.615000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-10T12:03:21.518000+00:00","server_upload_time":"2023-02-10T12:03:42.091000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"f6f54886-a93a-11ed-b598-65efbbbc3a1e","version_name":null},"emitted_at":1677160708961} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-DESKTOP-2023-02-10","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-10T08:00:00+00:00","client_upload_time":"2023-02-11T12:01:50.248000+00:00","country":null,"data":{"user_properties_updated":true,"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":782291950,"event_properties":{"ad_metrics.cost":0.070298,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"DESKTOP","ad_metrics.impressions":250,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-10T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-11T12:02:19.794000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-11T12:01:50.248000+00:00","server_upload_time":"2023-02-11T12:02:04.360000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"ef62e73e-aa03-11ed-ab60-21041b90b30e","version_name":null},"emitted_at":1677160709463} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-TABLET-2023-02-10","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-10T08:00:00+00:00","client_upload_time":"2023-02-11T12:01:50.248000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":660413772,"event_properties":{"ad_metrics.cost":0.137032,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"TABLET","ad_metrics.impressions":184,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-10T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-11T12:02:19.794000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-11T12:01:50.248000+00:00","server_upload_time":"2023-02-11T12:02:04.360000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"ef62f428-aa03-11ed-9aea-21041b90b30e","version_name":null},"emitted_at":1677160709465} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-MOBILE-2023-02-10","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-10T08:00:00+00:00","client_upload_time":"2023-02-11T12:01:50.248000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":345846795,"event_properties":{"ad_metrics.cost":1.986076,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"MOBILE","ad_metrics.impressions":2366,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":3,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":3,"campaign_id":19410069806},"event_time":"2023-02-10T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-11T12:02:19.794000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-11T12:01:50.248000+00:00","server_upload_time":"2023-02-11T12:02:04.360000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"ef62ee58-aa03-11ed-9470-21041b90b30e","version_name":null},"emitted_at":1677160709468} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-DESKTOP-2023-02-11","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-11T08:00:00+00:00","client_upload_time":"2023-02-12T12:02:28.595000+00:00","country":null,"data":{"user_properties_updated":true,"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":589830428,"event_properties":{"ad_metrics.cost":0.069645,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"DESKTOP","ad_metrics.impressions":183,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-11T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-12T12:03:53.410000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-12T12:02:28.595000+00:00","server_upload_time":"2023-02-12T12:02:39.319000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"518af7fe-aacd-11ed-9a80-43c1bac2ac7b","version_name":null},"emitted_at":1677160709960} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-TABLET-2023-02-11","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-11T08:00:00+00:00","client_upload_time":"2023-02-12T12:02:28.595000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":661524851,"event_properties":{"ad_metrics.cost":0.100511,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"TABLET","ad_metrics.impressions":125,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-11T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-12T12:03:53.410000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-12T12:02:28.595000+00:00","server_upload_time":"2023-02-12T12:02:39.319000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"518b0451-aacd-11ed-bf81-43c1bac2ac7b","version_name":null},"emitted_at":1677160709962} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-MOBILE-2023-02-11","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-11T08:00:00+00:00","client_upload_time":"2023-02-12T12:02:28.595000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":608278585,"event_properties":{"ad_metrics.cost":2.019566,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"MOBILE","ad_metrics.impressions":2505,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":3,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":3,"campaign_id":19410069806},"event_time":"2023-02-11T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-12T12:03:53.410000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-12T12:02:28.595000+00:00","server_upload_time":"2023-02-12T12:02:39.319000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"518afebd-aacd-11ed-a1be-43c1bac2ac7b","version_name":null},"emitted_at":1677160709964} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-DESKTOP-2023-02-12","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-12T08:00:00+00:00","client_upload_time":"2023-02-13T12:02:12.416000+00:00","country":null,"data":{"user_properties_updated":true,"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":756118159,"event_properties":{"ad_metrics.cost":0.047594,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"DESKTOP","ad_metrics.impressions":228,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-12T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-13T12:02:26.981000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-13T12:02:12.416000+00:00","server_upload_time":"2023-02-13T12:02:22.901000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"488ce6ab-ab96-11ed-a18e-516670498ef9","version_name":null},"emitted_at":1677160710459} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-TABLET-2023-02-12","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-12T08:00:00+00:00","client_upload_time":"2023-02-13T12:02:12.416000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":726820846,"event_properties":{"ad_metrics.cost":0.094987,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"TABLET","ad_metrics.impressions":135,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-12T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-13T12:02:26.981000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-13T12:02:12.416000+00:00","server_upload_time":"2023-02-13T12:02:22.901000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"488cf2ca-ab96-11ed-baad-516670498ef9","version_name":null},"emitted_at":1677160710461} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-MOBILE-2023-02-12","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-12T08:00:00+00:00","client_upload_time":"2023-02-13T12:02:12.416000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":30838997,"event_properties":{"ad_metrics.cost":2.057797,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"MOBILE","ad_metrics.impressions":2555,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":7,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":7,"campaign_id":19410069806},"event_time":"2023-02-12T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-13T12:02:26.981000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-13T12:02:12.416000+00:00","server_upload_time":"2023-02-13T12:02:22.901000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"488ced3a-ab96-11ed-9cc6-516670498ef9","version_name":null},"emitted_at":1677160710463} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-DESKTOP-2023-02-13","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-13T08:00:00+00:00","client_upload_time":"2023-02-14T12:02:09.474000+00:00","country":null,"data":{"user_properties_updated":true,"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":602281810,"event_properties":{"ad_metrics.cost":0.084583,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"DESKTOP","ad_metrics.impressions":227,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-13T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-14T12:02:24.303000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-14T12:02:09.474000+00:00","server_upload_time":"2023-02-14T12:02:20.979000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"7152006b-ac5f-11ed-bb6d-b95ddc26bbd1","version_name":null},"emitted_at":1677160710878} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-TABLET-2023-02-13","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-13T08:00:00+00:00","client_upload_time":"2023-02-14T12:02:09.474000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":204201377,"event_properties":{"ad_metrics.cost":0.1489,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"TABLET","ad_metrics.impressions":191,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-13T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-14T12:02:24.303000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-14T12:02:09.474000+00:00","server_upload_time":"2023-02-14T12:02:20.979000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"71520cd4-ac5f-11ed-8538-b95ddc26bbd1","version_name":null},"emitted_at":1677160710880} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-MOBILE-2023-02-13","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-13T08:00:00+00:00","client_upload_time":"2023-02-14T12:02:09.474000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":96617686,"event_properties":{"ad_metrics.cost":2.033321,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"MOBILE","ad_metrics.impressions":2658,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":5,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":5,"campaign_id":19410069806},"event_time":"2023-02-13T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-14T12:02:24.303000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-14T12:02:09.474000+00:00","server_upload_time":"2023-02-14T12:02:20.979000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"71520734-ac5f-11ed-bc4a-b95ddc26bbd1","version_name":null},"emitted_at":1677160710882} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-DESKTOP-2023-02-14","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-14T08:00:00+00:00","client_upload_time":"2023-02-15T12:02:02.211000+00:00","country":null,"data":{"user_properties_updated":true,"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":325453045,"event_properties":{"ad_metrics.cost":0.065906,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"DESKTOP","ad_metrics.impressions":291,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-14T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-15T12:02:17.536000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-15T12:02:02.211000+00:00","server_upload_time":"2023-02-15T12:02:15.558000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"97bd311f-ad28-11ed-a272-9937469197fb","version_name":null},"emitted_at":1677160711338} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-TABLET-2023-02-14","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-14T08:00:00+00:00","client_upload_time":"2023-02-15T12:02:02.211000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":579050854,"event_properties":{"ad_metrics.cost":0.085808,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"TABLET","ad_metrics.impressions":166,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-14T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-15T12:02:17.536000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-15T12:02:02.211000+00:00","server_upload_time":"2023-02-15T12:02:15.558000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"97bd3d80-ad28-11ed-9543-9937469197fb","version_name":null},"emitted_at":1677160711340} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-MOBILE-2023-02-14","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-14T08:00:00+00:00","client_upload_time":"2023-02-15T12:02:02.211000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":153980233,"event_properties":{"ad_metrics.cost":2.036793,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"MOBILE","ad_metrics.impressions":2514,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":4,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":4,"campaign_id":19410069806},"event_time":"2023-02-14T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-15T12:02:17.536000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-15T12:02:02.211000+00:00","server_upload_time":"2023-02-15T12:02:15.558000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"97bd37e1-ad28-11ed-a554-9937469197fb","version_name":null},"emitted_at":1677160711342} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-DESKTOP-2023-02-15","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-15T08:00:00+00:00","client_upload_time":"2023-02-16T12:01:18.942000+00:00","country":null,"data":{"user_properties_updated":true,"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":230240803,"event_properties":{"ad_metrics.cost":0.043522,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"DESKTOP","ad_metrics.impressions":193,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-15T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-16T12:01:41.299000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-16T12:01:18.942000+00:00","server_upload_time":"2023-02-16T12:01:39.526000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"ac8bae8d-adf1-11ed-b15b-91b6ce29e523","version_name":null},"emitted_at":1677160711799} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-CONNECTED_TV-2023-02-15","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-15T08:00:00+00:00","client_upload_time":"2023-02-16T12:01:18.942000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":324823220,"event_properties":{"ad_metrics.cost":3e-05,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"CONNECTED_TV","ad_metrics.impressions":1,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-15T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-16T12:01:41.299000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-16T12:01:18.942000+00:00","server_upload_time":"2023-02-16T12:01:39.526000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"ac8bc13f-adf1-11ed-b15c-91b6ce29e523","version_name":null},"emitted_at":1677160711801} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-TABLET-2023-02-15","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-15T08:00:00+00:00","client_upload_time":"2023-02-16T12:01:18.942000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":427159891,"event_properties":{"ad_metrics.cost":0.127895,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"TABLET","ad_metrics.impressions":173,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":1,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":1,"campaign_id":19410069806},"event_time":"2023-02-15T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-16T12:01:41.299000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-16T12:01:18.942000+00:00","server_upload_time":"2023-02-16T12:01:39.526000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"ac8bbbc1-adf1-11ed-9548-91b6ce29e523","version_name":null},"emitted_at":1677160711803} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-MOBILE-2023-02-15","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-15T08:00:00+00:00","client_upload_time":"2023-02-16T12:01:18.942000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":26870094,"event_properties":{"ad_metrics.cost":2.069498,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"MOBILE","ad_metrics.impressions":2644,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":5,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":5,"campaign_id":19410069806},"event_time":"2023-02-15T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-16T12:01:41.299000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-16T12:01:18.942000+00:00","server_upload_time":"2023-02-16T12:01:39.526000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"ac8bb5e3-adf1-11ed-b665-91b6ce29e523","version_name":null},"emitted_at":1677160711805} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-DESKTOP-2023-02-16","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-16T08:00:00+00:00","client_upload_time":"2023-02-17T12:03:01.224000+00:00","country":null,"data":{"user_properties_updated":true,"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":285988082,"event_properties":{"ad_metrics.cost":0.032337,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"DESKTOP","ad_metrics.impressions":130,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-16T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-17T12:03:21.797000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-17T12:03:01.224000+00:00","server_upload_time":"2023-02-17T12:03:12.561000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"12ad685e-aebb-11ed-85a1-f5fef6fa6c3e","version_name":null},"emitted_at":1677160712406} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-TABLET-2023-02-16","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-16T08:00:00+00:00","client_upload_time":"2023-02-17T12:03:01.224000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":945641623,"event_properties":{"ad_metrics.cost":0.177236,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"TABLET","ad_metrics.impressions":215,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-16T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-17T12:03:21.797000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-17T12:03:01.224000+00:00","server_upload_time":"2023-02-17T12:03:12.561000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"12ad74f4-aebb-11ed-af5e-f5fef6fa6c3e","version_name":null},"emitted_at":1677160712408} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-MOBILE-2023-02-16","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-16T08:00:00+00:00","client_upload_time":"2023-02-17T12:03:01.224000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":684001950,"event_properties":{"ad_metrics.cost":2.002117,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"MOBILE","ad_metrics.impressions":2518,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":5,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":5,"campaign_id":19410069806},"event_time":"2023-02-16T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-17T12:03:21.797000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-17T12:03:01.224000+00:00","server_upload_time":"2023-02-17T12:03:12.561000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"12ad6f4f-aebb-11ed-849c-f5fef6fa6c3e","version_name":null},"emitted_at":1677160712411} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-DESKTOP-2023-02-18","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-18T08:00:00+00:00","client_upload_time":"2023-02-19T12:03:29.271000+00:00","country":null,"data":{"user_properties_updated":true,"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":131262759,"event_properties":{"ad_metrics.cost":0.029716,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"DESKTOP","ad_metrics.impressions":142,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-18T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-19T12:03:51.321000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-19T12:03:29.271000+00:00","server_upload_time":"2023-02-19T12:03:48.945000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"7948776d-b04d-11ed-9e25-0d3650f391bb","version_name":null},"emitted_at":1677160713566} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-TABLET-2023-02-18","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-18T08:00:00+00:00","client_upload_time":"2023-02-19T12:03:29.271000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":297590864,"event_properties":{"ad_metrics.cost":0.095908,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"TABLET","ad_metrics.impressions":105,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-18T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-19T12:03:51.321000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-19T12:03:29.271000+00:00","server_upload_time":"2023-02-19T12:03:48.945000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"794883a6-b04d-11ed-aa23-0d3650f391bb","version_name":null},"emitted_at":1677160713568} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-MOBILE-2023-02-18","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-18T08:00:00+00:00","client_upload_time":"2023-02-19T12:03:29.271000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":275214096,"event_properties":{"ad_metrics.cost":2.043589,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"MOBILE","ad_metrics.impressions":2421,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":5,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":5,"campaign_id":19410069806},"event_time":"2023-02-18T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-19T12:03:51.321000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-19T12:03:29.271000+00:00","server_upload_time":"2023-02-19T12:03:48.945000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"79487e0a-b04d-11ed-92e7-0d3650f391bb","version_name":null},"emitted_at":1677160713570} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-DESKTOP-2023-02-17","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-17T08:00:00+00:00","client_upload_time":"2023-02-19T13:01:25.998000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":592568898,"event_properties":{"ad_metrics.cost":0.070942,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"DESKTOP","ad_metrics.impressions":225,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-17T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-19T13:01:40.473000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-19T13:01:25.998000+00:00","server_upload_time":"2023-02-19T13:01:37.076000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"8d1b530d-b055-11ed-b3db-113a8bccd94d","version_name":null},"emitted_at":1677160713572} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-TABLET-2023-02-17","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-17T08:00:00+00:00","client_upload_time":"2023-02-19T13:01:25.998000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":921057594,"event_properties":{"ad_metrics.cost":0.082843,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"TABLET","ad_metrics.impressions":101,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-17T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-19T13:01:40.473000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-19T13:01:25.998000+00:00","server_upload_time":"2023-02-19T13:01:37.076000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"8d1b605e-b055-11ed-8e2a-113a8bccd94d","version_name":null},"emitted_at":1677160713574} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-MOBILE-2023-02-17","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-17T08:00:00+00:00","client_upload_time":"2023-02-19T13:01:25.998000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":771686719,"event_properties":{"ad_metrics.cost":2.034163,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"MOBILE","ad_metrics.impressions":2612,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":2,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":2,"campaign_id":19410069806},"event_time":"2023-02-17T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-19T13:01:40.473000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-19T13:01:25.998000+00:00","server_upload_time":"2023-02-19T13:01:37.076000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"8d1b59c4-b055-11ed-a1df-113a8bccd94d","version_name":null},"emitted_at":1677160713656} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-DESKTOP-2023-02-19","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-19T08:00:00+00:00","client_upload_time":"2023-02-20T12:03:35.170000+00:00","country":null,"data":{"user_properties_updated":true,"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":664632971,"event_properties":{"ad_metrics.cost":0.031298,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"DESKTOP","ad_metrics.impressions":139,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-19T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-20T12:04:00.393000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-20T12:03:35.170000+00:00","server_upload_time":"2023-02-20T12:03:46.062000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"a907cef2-b116-11ed-87dd-bdba20a14ceb","version_name":null},"emitted_at":1677160714816} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-TABLET-2023-02-19","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-19T08:00:00+00:00","client_upload_time":"2023-02-20T12:03:35.170000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":600181548,"event_properties":{"ad_metrics.cost":0.112333,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"TABLET","ad_metrics.impressions":154,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-19T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-20T12:04:00.393000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-20T12:03:35.170000+00:00","server_upload_time":"2023-02-20T12:03:46.062000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"a907db97-b116-11ed-9271-bdba20a14ceb","version_name":null},"emitted_at":1677160714819} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-MOBILE-2023-02-19","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-19T08:00:00+00:00","client_upload_time":"2023-02-20T12:03:35.170000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":780220200,"event_properties":{"ad_metrics.cost":2.061574,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"MOBILE","ad_metrics.impressions":2389,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":4,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":4,"campaign_id":19410069806},"event_time":"2023-02-19T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-20T12:04:00.393000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-20T12:03:35.170000+00:00","server_upload_time":"2023-02-20T12:03:46.062000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"a907d5dc-b116-11ed-8b8e-bdba20a14ceb","version_name":null},"emitted_at":1677160714821} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-DESKTOP-2023-02-20","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-20T08:00:00+00:00","client_upload_time":"2023-02-21T12:01:29.891000+00:00","country":null,"data":{"user_properties_updated":true,"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":801727177,"event_properties":{"ad_metrics.cost":0.045563,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"DESKTOP","ad_metrics.impressions":188,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-20T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-21T12:01:48.145000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-21T12:01:29.891000+00:00","server_upload_time":"2023-02-21T12:01:45.796000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"84ad0ece-b1df-11ed-8b09-bdc83361662a","version_name":null},"emitted_at":1677160715235} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-TABLET-2023-02-20","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-20T08:00:00+00:00","client_upload_time":"2023-02-21T12:01:29.891000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":847817114,"event_properties":{"ad_metrics.cost":0.153187,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"TABLET","ad_metrics.impressions":191,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-20T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-21T12:01:48.145000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-21T12:01:29.891000+00:00","server_upload_time":"2023-02-21T12:01:45.796000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"84ad1bde-b1df-11ed-ab01-bdc83361662a","version_name":null},"emitted_at":1677160715237} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-MOBILE-2023-02-20","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-20T08:00:00+00:00","client_upload_time":"2023-02-21T12:01:29.891000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":795787975,"event_properties":{"ad_metrics.cost":2.096516,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"MOBILE","ad_metrics.impressions":2720,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":3,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":3,"campaign_id":19410069806},"event_time":"2023-02-20T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-21T12:01:48.145000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-21T12:01:29.891000+00:00","server_upload_time":"2023-02-21T12:01:45.796000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"84ad15c9-b1df-11ed-b84e-bdc83361662a","version_name":null},"emitted_at":1677160715263} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-DESKTOP-2023-02-21","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-21T08:00:00+00:00","client_upload_time":"2023-02-22T12:02:15.518000+00:00","country":null,"data":{"user_properties_updated":true,"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":354116550,"event_properties":{"ad_metrics.cost":0.057382,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"DESKTOP","ad_metrics.impressions":186,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-21T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-22T12:02:37.973000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-22T12:02:15.518000+00:00","server_upload_time":"2023-02-22T12:02:34.242000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"ccc51943-b2a8-11ed-ac61-871718cb9747","version_name":null},"emitted_at":1677160715762} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-TABLET-2023-02-21","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-21T08:00:00+00:00","client_upload_time":"2023-02-22T12:02:15.518000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":541365877,"event_properties":{"ad_metrics.cost":0.118621,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"TABLET","ad_metrics.impressions":141,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-21T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-22T12:02:37.973000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-22T12:02:15.518000+00:00","server_upload_time":"2023-02-22T12:02:34.242000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"ccc525c9-b2a8-11ed-850f-871718cb9747","version_name":null},"emitted_at":1677160715764} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-MOBILE-2023-02-21","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-21T08:00:00+00:00","client_upload_time":"2023-02-22T12:02:15.518000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":562697441,"event_properties":{"ad_metrics.cost":2.088394,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"MOBILE","ad_metrics.impressions":2726,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":9,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":9,"campaign_id":19410069806},"event_time":"2023-02-21T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-22T12:02:37.973000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-22T12:02:15.518000+00:00","server_upload_time":"2023-02-22T12:02:34.242000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"ccc52020-b2a8-11ed-abd4-871718cb9747","version_name":null},"emitted_at":1677160715767} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-DESKTOP-2023-02-22","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-22T08:00:00+00:00","client_upload_time":"2023-02-23T12:04:28.073000+00:00","country":null,"data":{"user_properties_updated":true,"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":216489823,"event_properties":{"ad_metrics.cost":0.030981,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"DESKTOP","ad_metrics.impressions":137,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-22T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-23T12:04:44.116000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-23T12:04:28.073000+00:00","server_upload_time":"2023-02-23T12:04:40.311000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"426992d9-b372-11ed-8ac0-137abd2e6743","version_name":null},"emitted_at":1677160717689} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-TABLET-2023-02-22","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-22T08:00:00+00:00","client_upload_time":"2023-02-23T12:04:28.073000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":910022464,"event_properties":{"ad_metrics.cost":0.080812,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"TABLET","ad_metrics.impressions":118,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":0,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":0,"campaign_id":19410069806},"event_time":"2023-02-22T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-23T12:04:44.116000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-23T12:04:28.073000+00:00","server_upload_time":"2023-02-23T12:04:40.311000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"4269a22c-b372-11ed-84ce-137abd2e6743","version_name":null},"emitted_at":1677160717691} -{"stream":"events","data":{"$insert_id":"google-ad-4651612872-643022056303-MOBILE-2023-02-22","$insert_key":null,"$schema":null,"adid":null,"amplitude_attribution_ids":null,"amplitude_event_type":null,"amplitude_id":550106004607,"app":434735,"city":null,"client_event_time":"2023-02-22T08:00:00+00:00","client_upload_time":"2023-02-23T12:04:28.073000+00:00","country":null,"data":{"vacuum_source_id":"5955","group_first_event":{},"group_ids":{}},"data_type":"event","device_brand":null,"device_carrier":null,"device_family":null,"device_id":"google-ad-4651612872-643022056303","device_manufacturer":null,"device_model":null,"device_type":null,"dma":null,"event_id":119391830,"event_properties":{"ad_metrics.cost":2.106073,"campaign_advertising_channel_type":"DISPLAY","ad_segment_device":"MOBILE","ad_metrics.impressions":2741,"ad_group_type":"DISPLAY_STANDARD","campaign_name":"Brand awareness and reach-Display-1","ad_group_name":"Ad group 1","ad_id":643022056303,"campaign_start_date":"2022-12-28","final_url":"https://airbyte.com","ad_platform":"google","campaign_end_date":"2037-12-30","ad_metrics.clicks":1,"ad_group_id":144799120517,"ad_metrics.conversions":0,"ad_metrics.interactions":1,"campaign_id":19410069806},"event_time":"2023-02-22T08:00:00+00:00","event_type":"Daily Ad Metrics","global_user_properties":null,"group_properties":{},"groups":{},"idfa":null,"ip_address":null,"is_attribution_event":null,"language":null,"library":"google_ads","location_lat":null,"location_lng":null,"os_name":null,"os_version":null,"partner_id":null,"paying":null,"plan":{},"platform":null,"processed_time":"2023-02-23T12:04:44.116000+00:00","region":null,"sample_rate":null,"server_received_time":"2023-02-23T12:04:28.073000+00:00","server_upload_time":"2023-02-23T12:04:40.311000+00:00","session_id":-1,"source_id":null,"start_version":null,"user_creation_time":null,"user_id":null,"user_properties":{"country":"test","device_model":"test","city":"test","os_version":"test","City":"London","platform":"test","device_manufacturer":"test","carrier":"test","device_brand":"test","Region":"London","DMA":"London","Country":"UK","os_name":"test","region":"test"},"uuid":"42699ba5-b372-11ed-8487-137abd2e6743","version_name":null},"emitted_at":1677160717693} +{"stream": "active_users", "data": {"date": "2023-08-28", "statistics": {"(none)": 0}}, "emitted_at": 1694709513300} +{"stream": "active_users", "data": {"date": "2023-08-29", "statistics": {"(none)": 0}}, "emitted_at": 1694709513302} +{"stream": "active_users", "data": {"date": "2023-08-30", "statistics": {"(none)": 0}}, "emitted_at": 1694709513303} +{"stream": "active_users", "data": {"date": "2023-08-31", "statistics": {"(none)": 0}}, "emitted_at": 1694709513305} +{"stream": "average_session_length", "data": {"date": "2023-08-11T00:00:00", "length": 0.0}, "emitted_at": 1694709517092} +{"stream": "average_session_length", "data": {"date": "2023-08-18T00:00:00", "length": 0.0}, "emitted_at": 1694709517088} +{"stream": "average_session_length", "data": {"date": "2023-08-23T00:00:00", "length": 0.0}, "emitted_at": 1694709517090} +{"stream": "average_session_length", "data": {"date": "2023-08-27T00:00:00", "length": 0.0}, "emitted_at": 1694709517086} +{"stream": "events", "data": {"$insert_id": "google-ad-4651612872-643022056303-DESKTOP-2023-08-24", "$insert_key": null, "$schema": null, "adid": null, "amplitude_attribution_ids": null, "amplitude_event_type": null, "amplitude_id": 550106004607, "app": 434735, "city": null, "client_event_time": "2023-08-24T07:00:00+00:00", "client_upload_time": "2023-08-25T11:04:55.821000+00:00", "country": null, "data": {"path": "/batch", "user_properties_updated": true, "vacuum_source_id": "5955", "group_first_event": {}, "group_ids": {}}, "data_type": "event", "device_brand": null, "device_carrier": null, "device_family": null, "device_id": "google-ad-4651612872-643022056303", "device_manufacturer": null, "device_model": null, "device_type": null, "dma": null, "event_id": 355175889, "event_properties": {"ad_metrics.cost": 0.528957, "campaign_advertising_channel_type": "DISPLAY", "ad_segment_device": "DESKTOP", "ad_metrics.impressions": 1535, "ad_group_type": "DISPLAY_STANDARD", "campaign_name": "Brand awareness and reach-Display-1", "ad_group_name": "Ad group 1", "ad_id": 643022056303, "campaign_start_date": "2022-12-28", "final_url": "https://airbyte.com", "ad_platform": "google", "campaign_end_date": "2037-12-30", "ad_metrics.clicks": 0, "ad_group_id": 144799120517, "ad_metrics.conversions": 0.0, "ad_metrics.interactions": 0, "campaign_id": 19410069806}, "event_time": "2023-08-24T07:00:00+00:00", "event_type": "Daily Ad Metrics", "global_user_properties": null, "group_properties": {}, "groups": {}, "idfa": null, "ip_address": null, "is_attribution_event": null, "language": null, "library": "google_ads", "location_lat": null, "location_lng": null, "os_name": null, "os_version": null, "partner_id": null, "paying": null, "plan": {}, "platform": null, "processed_time": "2023-08-25T11:05:08.912000+00:00", "region": null, "sample_rate": null, "server_received_time": "2023-08-25T11:04:55.821000+00:00", "server_upload_time": "2023-08-25T11:05:08.013000+00:00", "session_id": -1, "source_id": null, "start_version": null, "user_creation_time": null, "user_id": null, "user_properties": {"country": "test", "device_model": "test", "city": "test", "os_version": "test", "City": "London", "platform": "test", "device_manufacturer": "test", "carrier": "test", "device_brand": "test", "Region": "London", "DMA": "London", "Country": "UK", "os_name": "test", "region": "test"}, "uuid": "37bcd2f0-2688-47d5-ba90-76b0bd13b0f8", "version_name": null}, "emitted_at": 1694709577929} +{"stream": "events", "data": {"$insert_id": "google-ad-4651612872-643022056303-MOBILE-2023-08-24", "$insert_key": null, "$schema": null, "adid": null, "amplitude_attribution_ids": null, "amplitude_event_type": null, "amplitude_id": 550106004607, "app": 434735, "city": null, "client_event_time": "2023-08-24T07:00:00+00:00", "client_upload_time": "2023-08-25T11:04:55.821000+00:00", "country": null, "data": {"path": "/batch", "vacuum_source_id": "5955", "group_first_event": {}, "group_ids": {}}, "data_type": "event", "device_brand": null, "device_carrier": null, "device_family": null, "device_id": "google-ad-4651612872-643022056303", "device_manufacturer": null, "device_model": null, "device_type": null, "dma": null, "event_id": 604299598, "event_properties": {"ad_metrics.cost": 11.398659, "campaign_advertising_channel_type": "DISPLAY", "ad_segment_device": "MOBILE", "ad_metrics.impressions": 15084, "ad_group_type": "DISPLAY_STANDARD", "campaign_name": "Brand awareness and reach-Display-1", "ad_group_name": "Ad group 1", "ad_id": 643022056303, "campaign_start_date": "2022-12-28", "final_url": "https://airbyte.com", "ad_platform": "google", "campaign_end_date": "2037-12-30", "ad_metrics.clicks": 28, "ad_group_id": 144799120517, "ad_metrics.conversions": 0.0, "ad_metrics.interactions": 28, "campaign_id": 19410069806}, "event_time": "2023-08-24T07:00:00+00:00", "event_type": "Daily Ad Metrics", "global_user_properties": null, "group_properties": {}, "groups": {}, "idfa": null, "ip_address": null, "is_attribution_event": null, "language": null, "library": "google_ads", "location_lat": null, "location_lng": null, "os_name": null, "os_version": null, "partner_id": null, "paying": null, "plan": {}, "platform": null, "processed_time": "2023-08-25T11:05:08.912000+00:00", "region": null, "sample_rate": null, "server_received_time": "2023-08-25T11:04:55.821000+00:00", "server_upload_time": "2023-08-25T11:05:08.013000+00:00", "session_id": -1, "source_id": null, "start_version": null, "user_creation_time": null, "user_id": null, "user_properties": {"country": "test", "device_model": "test", "city": "test", "os_version": "test", "City": "London", "platform": "test", "device_manufacturer": "test", "carrier": "test", "device_brand": "test", "Region": "London", "DMA": "London", "Country": "UK", "os_name": "test", "region": "test"}, "uuid": "60320805-6886-43d6-b2ea-d3a6eccefee2", "version_name": null}, "emitted_at": 1694709577931} +{"stream": "events", "data": {"$insert_id": "google-ad-4651612872-643022056303-TABLET-2023-08-24", "$insert_key": null, "$schema": null, "adid": null, "amplitude_attribution_ids": null, "amplitude_event_type": null, "amplitude_id": 550106004607, "app": 434735, "city": null, "client_event_time": "2023-08-24T07:00:00+00:00", "client_upload_time": "2023-08-25T11:04:55.821000+00:00", "country": null, "data": {"path": "/batch", "vacuum_source_id": "5955", "group_first_event": {}, "group_ids": {}}, "data_type": "event", "device_brand": null, "device_carrier": null, "device_family": null, "device_id": "google-ad-4651612872-643022056303", "device_manufacturer": null, "device_model": null, "device_type": null, "dma": null, "event_id": 798716893, "event_properties": {"ad_metrics.cost": 0.644529, "campaign_advertising_channel_type": "DISPLAY", "ad_segment_device": "TABLET", "ad_metrics.impressions": 931, "ad_group_type": "DISPLAY_STANDARD", "campaign_name": "Brand awareness and reach-Display-1", "ad_group_name": "Ad group 1", "ad_id": 643022056303, "campaign_start_date": "2022-12-28", "final_url": "https://airbyte.com", "ad_platform": "google", "campaign_end_date": "2037-12-30", "ad_metrics.clicks": 3, "ad_group_id": 144799120517, "ad_metrics.conversions": 0.0, "ad_metrics.interactions": 3, "campaign_id": 19410069806}, "event_time": "2023-08-24T07:00:00+00:00", "event_type": "Daily Ad Metrics", "global_user_properties": null, "group_properties": {}, "groups": {}, "idfa": null, "ip_address": null, "is_attribution_event": null, "language": null, "library": "google_ads", "location_lat": null, "location_lng": null, "os_name": null, "os_version": null, "partner_id": null, "paying": null, "plan": {}, "platform": null, "processed_time": "2023-08-25T11:05:08.912000+00:00", "region": null, "sample_rate": null, "server_received_time": "2023-08-25T11:04:55.821000+00:00", "server_upload_time": "2023-08-25T11:05:08.013000+00:00", "session_id": -1, "source_id": null, "start_version": null, "user_creation_time": null, "user_id": null, "user_properties": {"country": "test", "device_model": "test", "city": "test", "os_version": "test", "City": "London", "platform": "test", "device_manufacturer": "test", "carrier": "test", "device_brand": "test", "Region": "London", "DMA": "London", "Country": "UK", "os_name": "test", "region": "test"}, "uuid": "9ae0c1e5-3c39-4a4d-af8b-0902fe889410", "version_name": null}, "emitted_at": 1694709577932} +{"stream": "events", "data": {"$insert_id": "google-ad-4651612872-19959839954-DESKTOP-2023-08-24", "$insert_key": null, "$schema": null, "adid": null, "amplitude_attribution_ids": null, "amplitude_event_type": null, "amplitude_id": 617244169723, "app": 434735, "city": null, "client_event_time": "2023-08-24T07:00:00+00:00", "client_upload_time": "2023-08-25T11:04:55.821000+00:00", "country": null, "data": {"path": "/batch", "user_properties_updated": true, "vacuum_source_id": "5955", "group_first_event": {}, "group_ids": {}}, "data_type": "event", "device_brand": null, "device_carrier": null, "device_family": null, "device_id": "google-ad-4651612872-19959839954", "device_manufacturer": null, "device_model": null, "device_type": null, "dma": null, "event_id": 557213115, "event_properties": {"campaign_name": "Performance Max-5", "campaign_start_date": "2023-04-10", "ad_platform": "google", "campaign_end_date": "2037-12-30", "ad_metrics.cost": 0.12, "ad_metrics.clicks": 3, "campaign_advertising_channel_type": "PERFORMANCE_MAX", "ad_segment_device": "DESKTOP", "ad_metrics.conversions": 0.0, "ad_metrics.impressions": 3, "ad_metrics.interactions": 3, "campaign_id": 19959839954}, "event_time": "2023-08-24T07:00:00+00:00", "event_type": "Daily Ad Metrics", "global_user_properties": null, "group_properties": {}, "groups": {}, "idfa": null, "ip_address": null, "is_attribution_event": null, "language": null, "library": "google_ads", "location_lat": null, "location_lng": null, "os_name": null, "os_version": null, "partner_id": null, "paying": null, "plan": {}, "platform": null, "processed_time": "2023-08-25T11:05:08.968000+00:00", "region": null, "sample_rate": null, "server_received_time": "2023-08-25T11:04:55.821000+00:00", "server_upload_time": "2023-08-25T11:05:08.162000+00:00", "session_id": -1, "source_id": null, "start_version": null, "user_creation_time": null, "user_id": null, "user_properties": {}, "uuid": "b7bff8c4-382d-4438-af45-b289aecced89", "version_name": null}, "emitted_at": 1694709577933} +{"stream": "events", "data": {"$insert_id": "google-ad-4651612872-643022056303-DESKTOP-2023-08-25", "$insert_key": null, "$schema": null, "adid": null, "amplitude_attribution_ids": null, "amplitude_event_type": null, "amplitude_id": 550106004607, "app": 434735, "city": null, "client_event_time": "2023-08-25T07:00:00+00:00", "client_upload_time": "2023-08-26T11:03:07.417000+00:00", "country": null, "data": {"path": "/batch", "user_properties_updated": true, "vacuum_source_id": "5955", "group_first_event": {}, "group_ids": {}}, "data_type": "event", "device_brand": null, "device_carrier": null, "device_family": null, "device_id": "google-ad-4651612872-643022056303", "device_manufacturer": null, "device_model": null, "device_type": null, "dma": null, "event_id": 93897159, "event_properties": {"ad_metrics.cost": 0.632838, "campaign_advertising_channel_type": "DISPLAY", "ad_segment_device": "DESKTOP", "ad_metrics.impressions": 1797, "ad_group_type": "DISPLAY_STANDARD", "campaign_name": "Brand awareness and reach-Display-1", "ad_group_name": "Ad group 1", "ad_id": 643022056303, "campaign_start_date": "2022-12-28", "final_url": "https://airbyte.com", "ad_platform": "google", "campaign_end_date": "2037-12-30", "ad_metrics.clicks": 0, "ad_group_id": 144799120517, "ad_metrics.conversions": 0.0, "ad_metrics.interactions": 0, "campaign_id": 19410069806}, "event_time": "2023-08-25T07:00:00+00:00", "event_type": "Daily Ad Metrics", "global_user_properties": null, "group_properties": {}, "groups": {}, "idfa": null, "ip_address": null, "is_attribution_event": null, "language": null, "library": "google_ads", "location_lat": null, "location_lng": null, "os_name": null, "os_version": null, "partner_id": null, "paying": null, "plan": {}, "platform": null, "processed_time": "2023-08-26T11:03:24.953000+00:00", "region": null, "sample_rate": null, "server_received_time": "2023-08-26T11:03:07.417000+00:00", "server_upload_time": "2023-08-26T11:03:23.284000+00:00", "session_id": -1, "source_id": null, "start_version": null, "user_creation_time": null, "user_id": null, "user_properties": {"country": "test", "device_model": "test", "city": "test", "os_version": "test", "City": "London", "platform": "test", "device_manufacturer": "test", "carrier": "test", "device_brand": "test", "Region": "London", "DMA": "London", "Country": "UK", "os_name": "test", "region": "test"}, "uuid": "79192e37-2340-475f-8e96-f52cee76a2ab", "version_name": null}, "emitted_at": 1694709578891} +{"stream": "events", "data": {"$insert_id": "google-ad-4651612872-643022056303-TABLET-2023-08-25", "$insert_key": null, "$schema": null, "adid": null, "amplitude_attribution_ids": null, "amplitude_event_type": null, "amplitude_id": 550106004607, "app": 434735, "city": null, "client_event_time": "2023-08-25T07:00:00+00:00", "client_upload_time": "2023-08-26T11:03:07.417000+00:00", "country": null, "data": {"path": "/batch", "vacuum_source_id": "5955", "group_first_event": {}, "group_ids": {}}, "data_type": "event", "device_brand": null, "device_carrier": null, "device_family": null, "device_id": "google-ad-4651612872-643022056303", "device_manufacturer": null, "device_model": null, "device_type": null, "dma": null, "event_id": 552664481, "event_properties": {"ad_metrics.cost": 0.974837, "campaign_advertising_channel_type": "DISPLAY", "ad_segment_device": "TABLET", "ad_metrics.impressions": 1222, "ad_group_type": "DISPLAY_STANDARD", "campaign_name": "Brand awareness and reach-Display-1", "ad_group_name": "Ad group 1", "ad_id": 643022056303, "campaign_start_date": "2022-12-28", "final_url": "https://airbyte.com", "ad_platform": "google", "campaign_end_date": "2037-12-30", "ad_metrics.clicks": 2, "ad_group_id": 144799120517, "ad_metrics.conversions": 0.0, "ad_metrics.interactions": 2, "campaign_id": 19410069806}, "event_time": "2023-08-25T07:00:00+00:00", "event_type": "Daily Ad Metrics", "global_user_properties": null, "group_properties": {}, "groups": {}, "idfa": null, "ip_address": null, "is_attribution_event": null, "language": null, "library": "google_ads", "location_lat": null, "location_lng": null, "os_name": null, "os_version": null, "partner_id": null, "paying": null, "plan": {}, "platform": null, "processed_time": "2023-08-26T11:03:24.953000+00:00", "region": null, "sample_rate": null, "server_received_time": "2023-08-26T11:03:07.417000+00:00", "server_upload_time": "2023-08-26T11:03:23.284000+00:00", "session_id": -1, "source_id": null, "start_version": null, "user_creation_time": null, "user_id": null, "user_properties": {"country": "test", "device_model": "test", "city": "test", "os_version": "test", "City": "London", "platform": "test", "device_manufacturer": "test", "carrier": "test", "device_brand": "test", "Region": "London", "DMA": "London", "Country": "UK", "os_name": "test", "region": "test"}, "uuid": "efb44e1e-7818-4470-b5cd-7a77ceff7613", "version_name": null}, "emitted_at": 1694709578893} +{"stream": "events", "data": {"$insert_id": "google-ad-4651612872-643022056303-MOBILE-2023-08-25", "$insert_key": null, "$schema": null, "adid": null, "amplitude_attribution_ids": null, "amplitude_event_type": null, "amplitude_id": 550106004607, "app": 434735, "city": null, "client_event_time": "2023-08-25T07:00:00+00:00", "client_upload_time": "2023-08-26T11:03:07.417000+00:00", "country": null, "data": {"path": "/batch", "vacuum_source_id": "5955", "group_first_event": {}, "group_ids": {}}, "data_type": "event", "device_brand": null, "device_carrier": null, "device_family": null, "device_id": "google-ad-4651612872-643022056303", "device_manufacturer": null, "device_model": null, "device_type": null, "dma": null, "event_id": 908969629, "event_properties": {"ad_metrics.cost": 10.863868, "campaign_advertising_channel_type": "DISPLAY", "ad_segment_device": "MOBILE", "ad_metrics.impressions": 14025, "ad_group_type": "DISPLAY_STANDARD", "campaign_name": "Brand awareness and reach-Display-1", "ad_group_name": "Ad group 1", "ad_id": 643022056303, "campaign_start_date": "2022-12-28", "final_url": "https://airbyte.com", "ad_platform": "google", "campaign_end_date": "2037-12-30", "ad_metrics.clicks": 18, "ad_group_id": 144799120517, "ad_metrics.conversions": 0.0, "ad_metrics.interactions": 18, "campaign_id": 19410069806}, "event_time": "2023-08-25T07:00:00+00:00", "event_type": "Daily Ad Metrics", "global_user_properties": null, "group_properties": {}, "groups": {}, "idfa": null, "ip_address": null, "is_attribution_event": null, "language": null, "library": "google_ads", "location_lat": null, "location_lng": null, "os_name": null, "os_version": null, "partner_id": null, "paying": null, "plan": {}, "platform": null, "processed_time": "2023-08-26T11:03:24.953000+00:00", "region": null, "sample_rate": null, "server_received_time": "2023-08-26T11:03:07.417000+00:00", "server_upload_time": "2023-08-26T11:03:23.284000+00:00", "session_id": -1, "source_id": null, "start_version": null, "user_creation_time": null, "user_id": null, "user_properties": {"country": "test", "device_model": "test", "city": "test", "os_version": "test", "City": "London", "platform": "test", "device_manufacturer": "test", "carrier": "test", "device_brand": "test", "Region": "London", "DMA": "London", "Country": "UK", "os_name": "test", "region": "test"}, "uuid": "96b57950-0cd2-4647-b546-cb094e122982", "version_name": null}, "emitted_at": 1694709578894} +{"stream": "events", "data": {"$insert_id": "google-ad-4651612872-19959839954-DESKTOP-2023-08-25", "$insert_key": null, "$schema": null, "adid": null, "amplitude_attribution_ids": null, "amplitude_event_type": null, "amplitude_id": 617244169723, "app": 434735, "city": null, "client_event_time": "2023-08-25T07:00:00+00:00", "client_upload_time": "2023-08-26T11:03:07.417000+00:00", "country": null, "data": {"path": "/batch", "user_properties_updated": true, "vacuum_source_id": "5955", "group_first_event": {}, "group_ids": {}}, "data_type": "event", "device_brand": null, "device_carrier": null, "device_family": null, "device_id": "google-ad-4651612872-19959839954", "device_manufacturer": null, "device_model": null, "device_type": null, "dma": null, "event_id": 80976276, "event_properties": {"campaign_name": "Performance Max-5", "campaign_start_date": "2023-04-10", "ad_platform": "google", "campaign_end_date": "2037-12-30", "ad_metrics.cost": 0.02, "ad_metrics.clicks": 1, "campaign_advertising_channel_type": "PERFORMANCE_MAX", "ad_segment_device": "DESKTOP", "ad_metrics.conversions": 0.0, "ad_metrics.impressions": 2, "ad_metrics.interactions": 1, "campaign_id": 19959839954}, "event_time": "2023-08-25T07:00:00+00:00", "event_type": "Daily Ad Metrics", "global_user_properties": null, "group_properties": {}, "groups": {}, "idfa": null, "ip_address": null, "is_attribution_event": null, "language": null, "library": "google_ads", "location_lat": null, "location_lng": null, "os_name": null, "os_version": null, "partner_id": null, "paying": null, "plan": {}, "platform": null, "processed_time": "2023-08-26T11:03:21.833000+00:00", "region": null, "sample_rate": null, "server_received_time": "2023-08-26T11:03:07.417000+00:00", "server_upload_time": "2023-08-26T11:03:20.169000+00:00", "session_id": -1, "source_id": null, "start_version": null, "user_creation_time": null, "user_id": null, "user_properties": {}, "uuid": "6f7c67c1-14d4-4566-a18a-72fb8a010b6c", "version_name": null}, "emitted_at": 1694709578895} diff --git a/airbyte-integrations/connectors/source-amplitude/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-amplitude/integration_tests/integration_test.py new file mode 100644 index 000000000000..63ad0a9063ec --- /dev/null +++ b/airbyte-integrations/connectors/source-amplitude/integration_tests/integration_test.py @@ -0,0 +1,103 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import operator +from pathlib import Path + +import pytest +from airbyte_cdk.models import SyncMode +from source_amplitude.source import SourceAmplitude + + +@pytest.fixture(scope="module") +def config(): + with open(Path(__file__).parent.parent / "secrets/config.json", "r") as file: + return json.loads(file.read()) + + +@pytest.fixture(scope="module") +def streams(config): + return SourceAmplitude().streams(config=config) + + +@pytest.fixture(scope="module") +def annotations_stream(streams): + return next(filter(lambda s: s.name == "annotations", streams)) + + +@pytest.fixture(scope="module") +def cohorts_stream(streams): + return next(filter(lambda s: s.name == "cohorts", streams)) + + +@pytest.mark.parametrize( + "stream_fixture_name, url, expected_records", + [ + ( + "annotations_stream", + "https://amplitude.com/api/2/annotations", + [ + {"date": "2023-09-22", "details": "vacate et scire", "id": 1, "label": "veritas"}, + {"date": "2023-09-22", "details": "valenter volenter", "id": 2, "label": "veritas"}, + ], + ), + ( + "cohorts_stream", + "https://amplitude.com/api/3/cohorts", + [ + { + "appId": 1, + "archived": False, + "chart_id": "27f310c471e8409797a18f18fe2884fb", + "createdAt": 1695394830, + "definition": {}, + "description": "Arduus ad Solem", + "edit_id": "fab12bc14de641589630c2ceced1c197", + "finished": True, + "hidden": False, + "id": 1, + "is_official_content": True, + "is_predictive": True, + "last_viewed": 1695394946, + "lastComputed": 1695394830, + "lastMod": 1695394830, + "location_id": "517974113223461a8468400b6ce88383", + "metadata": ["me", "ta", "da", "ta"], + "name": "Solem", + "owners": ["me", "mom"], + "popularity": 100, + "published": True, + "shortcut_ids": ["solem"], + "size": 186, + "type": "one", + "view_count": 2, + "viewers": ["me", "mom"], + } + ], + ), + ], +) +def test_empty_streams(stream_fixture_name, url, expected_records, request, requests_mock): + """ + A test with synthetic data since we are not able to test `annotations_stream` and `cohorts_stream` streams + due to free subscription plan for the sandbox + """ + stream = request.getfixturevalue(stream_fixture_name) + records_reader = stream.read_records(sync_mode=SyncMode.full_refresh, cursor_field=None, stream_slice={}) + requests_mock.get(url, status_code=200, json={"data": expected_records}) + + # Sort actual and expected records by ID. + # Prepare pairs of the actual and expected versions of the same record. + pairs = zip(*[sorted(record, key=operator.itemgetter("id")) for record in (list(records_reader), expected_records)]) + + # Calculate unmatched records and return their key, actual value and expected value + unmatched = [ + [(key, _actual[key], _expected[key]) for key in _actual if _actual[key] != _expected[key]] + for _actual, _expected in pairs + if _actual != _expected + ] + + # Ensure we don't have any unmatched records + assert not any(unmatched) diff --git a/airbyte-integrations/connectors/source-amplitude/metadata.yaml b/airbyte-integrations/connectors/source-amplitude/metadata.yaml index be9ecbaf14e7..fac740931651 100644 --- a/airbyte-integrations/connectors/source-amplitude/metadata.yaml +++ b/airbyte-integrations/connectors/source-amplitude/metadata.yaml @@ -1,13 +1,19 @@ data: + ab_internal: + ql: 400 + sl: 200 allowedHosts: hosts: - amplitude.com - analytics.eu.amplitude.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: api connectorType: source definitionId: fa9f58c6-2d03-4237-aaa4-07d75e0c1396 - dockerImageTag: 0.2.4 + dockerImageTag: 0.3.6 dockerRepository: airbyte/source-amplitude + documentationUrl: https://docs.airbyte.com/integrations/sources/amplitude githubIssueLabel: source-amplitude icon: amplitude.svg license: MIT @@ -18,12 +24,14 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/amplitude + suggestedStreams: + streams: + - events + - active_users + - annotations + - cohorts + supportLevel: certified tags: - language:low-code - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-amplitude/setup.py b/airbyte-integrations/connectors/source-amplitude/setup.py index ad70b2ff395f..29cccb75e13f 100644 --- a/airbyte-integrations/connectors/source-amplitude/setup.py +++ b/airbyte-integrations/connectors/source-amplitude/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk==0.33.0", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/manifest.yaml b/airbyte-integrations/connectors/source-amplitude/source_amplitude/manifest.yaml index adabb7f021aa..e2a7dd1083f2 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/manifest.yaml +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/manifest.yaml @@ -1,4 +1,4 @@ -version: "0.29.0" +version: "0.51.13" definitions: selector: @@ -20,6 +20,9 @@ definitions: - http_codes: [400] action: FAIL error_message: The file size of the exported data is too large. Shorten the time ranges and try again. The limit size is 4GB. + - http_codes: [403] + action: FAIL + error_message: Access denied due to lack of permission or invalid API/Secret key or wrong data region. - http_codes: [404] action: IGNORE error_message: No data collected @@ -43,6 +46,9 @@ definitions: end_datetime: datetime: "{{ now_utc().strftime('%Y-%m-%d') }}" datetime_format: "%Y-%m-%d" + cursor_datetime_formats: + - "%Y-%m-%d" + - "%Y-%m-%dT%H:%M:%S" cursor_granularity: P1D step: P15D cursor_field: "{{ parameters['stream_cursor_field'] }}" diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/average_session_length.json b/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/average_session_length.json index 0f13484727d3..8cc4ceb15384 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/average_session_length.json +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/schemas/average_session_length.json @@ -4,7 +4,7 @@ "properties": { "date": { "type": ["null", "string"], - "format": "date" + "format": "date-time" }, "length": { "type": ["null", "number"] diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/source.py b/airbyte-integrations/connectors/source-amplitude/source_amplitude/source.py index 247482de5163..6ff2160da69c 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/source.py +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/source.py @@ -36,7 +36,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Events( authenticator=auth, start_date=config["start_date"], - data_region=config["data_region"], + data_region=config.get("data_region", "Standard Server"), event_time_interval={"size_unit": "hours", "size": config.get("request_time_range", 24)}, ) ) diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/spec.yaml b/airbyte-integrations/connectors/source-amplitude/source_amplitude/spec.yaml index ee950f5e6658..78cd3163a673 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/spec.yaml +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/spec.yaml @@ -33,6 +33,7 @@ connectionSpecification: airbyte_secret: true start_date: type: string + format: date-time title: Replication Start Date pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" description: diff --git a/airbyte-integrations/connectors/source-amplitude/source_amplitude/streams.py b/airbyte-integrations/connectors/source-amplitude/source_amplitude/streams.py index d4fd61949f6f..20ae757e0241 100644 --- a/airbyte-integrations/connectors/source-amplitude/source_amplitude/streams.py +++ b/airbyte-integrations/connectors/source-amplitude/source_amplitude/streams.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import datetime import gzip import io import json @@ -99,8 +100,21 @@ def _date_time_to_rfc3339(self, record: MutableMapping[str, Any]) -> MutableMapp record[item] = pendulum.parse(record[item]).to_rfc3339_string() return record + def get_most_recent_cursor(self, stream_state: Mapping[str, Any] = None) -> datetime.datetime: + """ + Use `start_time` instead of `cursor` in the case of more recent. + This can happen whenever a user simply finds that they are syncing to much data and would like to change `start_time` to be more recent. + See: https://github.com/airbytehq/airbyte/issues/25367 for more details + """ + cursor_date = ( + pendulum.parse(stream_state[self.cursor_field]) + if stream_state and self.cursor_field in stream_state + else datetime.datetime.min.replace(tzinfo=datetime.timezone.utc) + ) + return max(self._start_date, cursor_date) + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping]: - state_value = stream_state[self.cursor_field] if stream_state else self._start_date.strftime(self.compare_date_template) + most_recent_cursor = self.get_most_recent_cursor(stream_state).strftime(self.compare_date_template) try: zip_file = zipfile.ZipFile(io.BytesIO(response.content)) except zipfile.BadZipFile as e: @@ -114,7 +128,7 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str, for gzip_filename in zip_file.namelist(): with zip_file.open(gzip_filename) as file: for record in self._parse_zip_file(file): - if record[self.cursor_field] >= state_value: + if record[self.cursor_field] >= most_recent_cursor: yield self._date_time_to_rfc3339(record) # transform all `date-time` to RFC3339 def _parse_zip_file(self, zip_file: IO[bytes]) -> Iterable[MutableMapping]: @@ -124,7 +138,7 @@ def _parse_zip_file(self, zip_file: IO[bytes]) -> Iterable[MutableMapping]: def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: slices = [] - start = pendulum.parse(stream_state.get(self.cursor_field)) if stream_state else self._start_date + start = self.get_most_recent_cursor(stream_state=stream_state) end = pendulum.now() if start > end: self.logger.info("The data cannot be requested in the future. Skipping stream.") diff --git a/airbyte-integrations/connectors/source-amplitude/unit_tests/events_request_content.zip b/airbyte-integrations/connectors/source-amplitude/unit_tests/events_request_content.zip new file mode 100644 index 000000000000..71ee9d6eac94 Binary files /dev/null and b/airbyte-integrations/connectors/source-amplitude/unit_tests/events_request_content.zip differ diff --git a/airbyte-integrations/connectors/source-amplitude/unit_tests/test_custom_extractors.py b/airbyte-integrations/connectors/source-amplitude/unit_tests/test_custom_extractors.py index 13ebf865ef3a..7d2c07a2c604 100644 --- a/airbyte-integrations/connectors/source-amplitude/unit_tests/test_custom_extractors.py +++ b/airbyte-integrations/connectors/source-amplitude/unit_tests/test_custom_extractors.py @@ -3,59 +3,65 @@ # import os -from unittest.mock import MagicMock +import re +import types +from contextlib import nullcontext as does_not_raise +from unittest.mock import MagicMock, patch +import pendulum import pytest import requests +from airbyte_cdk.models import SyncMode from source_amplitude.components import ActiveUsersRecordExtractor, AverageSessionLengthRecordExtractor, EventsExtractor +from source_amplitude.streams import Events @pytest.mark.parametrize( "custom_extractor, data, expected", [ ( - ActiveUsersRecordExtractor, - { - "xValues": ["2021-01-01", "2021-01-02"], - "series": [[1, 5]], - "seriesCollapsed": [[0]], - "seriesLabels": [0], - "seriesMeta": [{"segmentIndex": 0}], - }, - [{"date": "2021-01-01", "statistics": {0: 1}}, {"date": "2021-01-02", "statistics": {0: 5}}], + ActiveUsersRecordExtractor, + { + "xValues": ["2021-01-01", "2021-01-02"], + "series": [[1, 5]], + "seriesCollapsed": [[0]], + "seriesLabels": [0], + "seriesMeta": [{"segmentIndex": 0}], + }, + [{"date": "2021-01-01", "statistics": {0: 1}}, {"date": "2021-01-02", "statistics": {0: 5}}], ), ( - ActiveUsersRecordExtractor, - { - "xValues": ["2021-01-01", "2021-01-02"], - "series": [], - "seriesCollapsed": [[0]], - "seriesLabels": [0], - "seriesMeta": [{"segmentIndex": 0}], - }, - [], + ActiveUsersRecordExtractor, + { + "xValues": ["2021-01-01", "2021-01-02"], + "series": [], + "seriesCollapsed": [[0]], + "seriesLabels": [0], + "seriesMeta": [{"segmentIndex": 0}], + }, + [], ), ( - AverageSessionLengthRecordExtractor, - { - "xValues": ["2019-05-23", "2019-05-24"], - "series": [[2, 6]], - "seriesCollapsed": [[0]], - "seriesLabels": [0], - "seriesMeta": [{"segmentIndex": 0}], - }, - [{"date": "2019-05-23", "length": 2}, {"date": "2019-05-24", "length": 6}], + AverageSessionLengthRecordExtractor, + { + "xValues": ["2019-05-23", "2019-05-24"], + "series": [[2, 6]], + "seriesCollapsed": [[0]], + "seriesLabels": [0], + "seriesMeta": [{"segmentIndex": 0}], + }, + [{"date": "2019-05-23", "length": 2}, {"date": "2019-05-24", "length": 6}], ), ( - AverageSessionLengthRecordExtractor, - { - "xValues": ["2019-05-23", "2019-05-24"], - "series": [], - "seriesCollapsed": [[0]], - "seriesLabels": [0], - "seriesMeta": [{"segmentIndex": 0}], - }, - [], + AverageSessionLengthRecordExtractor, + { + "xValues": ["2019-05-23", "2019-05-24"], + "series": [], + "seriesCollapsed": [[0]], + "seriesLabels": [0], + "seriesMeta": [{"segmentIndex": 0}], + }, + [], ), ], ids=["ActiveUsers", "EmptyActiveUsers", "AverageSessionLength", "EmptyAverageSessionLength"], @@ -63,13 +69,13 @@ def test_parse_response(custom_extractor, data, expected): extractor = custom_extractor() response = requests.Response() - response.json = MagicMock(return_value={'data': data}) + response.json = MagicMock(return_value={"data": data}) result = extractor.extract_records(response) assert result == expected class TestEventsExtractor: - extractor = EventsExtractor(config={}, parameters={'name': 'events'}) + extractor = EventsExtractor(config={}, parameters={"name": "events"}) def test_get_date_time_items_from_schema(self): expected = [ @@ -101,14 +107,166 @@ def test_date_time_to_rfc3339(self, record, expected): @pytest.mark.parametrize( "file_name, expected_records", [ - ('records.json.zip', [{"id": 123}]), - ('zipped.json.gz', []), + ("records.json.zip", [{"id": 123}]), + ("zipped.json.gz", []), ], ids=["normal_file", "wrong_file"], ) def test_parse_zip(self, requests_mock, file_name, expected_records): - with open(f"{os.path.dirname(__file__)}/{file_name}", 'rb') as zipped: + with open(f"{os.path.dirname(__file__)}/{file_name}", "rb") as zipped: url = "https://amplitude.com/" requests_mock.get(url, content=zipped.read()) response = requests.get(url) assert list(self.extractor.extract_records(response)) == expected_records + + def test_event_read(self, requests_mock): + stream = Events( + authenticator=MagicMock(), + start_date="2023-08-01T00:00:00Z", + data_region="Standard Server", + event_time_interval={"size_unit": "hours", "size": 24}, + ) + + with open(f"{os.path.dirname(__file__)}/events_request_content.zip", "rb") as zipped: + requests_mock.get("https://amplitude.com/api/2/export", content=zipped.read()) + + records = stream.read_records( + sync_mode=SyncMode.incremental, cursor_field="server_upload_time", stream_slice={"start": "20230701T00", "end": "20230701T23"} + ) + + assert len(list(records)) == 4 + + @pytest.mark.parametrize( + "error_code, expectation", + [ + (400, does_not_raise()), + (404, does_not_raise()), + (504, does_not_raise()), + (500, pytest.raises(requests.exceptions.HTTPError)), + ], + ) + def test_event_errors_read(self, mocker, requests_mock, error_code, expectation): + stream = Events( + authenticator=MagicMock(), + start_date="2023-08-01T00:00:00Z", + data_region="Standard Server", + event_time_interval={"size_unit": "hours", "size": 24}, + ) + + requests_mock.get("https://amplitude.com/api/2/export", status_code=error_code) + + mocker.patch("time.sleep", lambda x: None) + + with expectation: + records = stream.read_records( + sync_mode=SyncMode.incremental, + cursor_field="server_upload_time", + stream_slice={"start": "20230701T00", "end": "20230701T23"}, + ) + + assert list(records) == [] + + @pytest.mark.parametrize( + "file_name, content_is_valid, records_count", + [ + ("events_request_content.zip", True, 4), + ("zipped.json.gz", False, 0), + ], + ) + def test_events_parse_response(self, file_name, content_is_valid, records_count): + stream = Events( + authenticator=MagicMock(), + start_date="2023-08-01T00:00:00Z", + data_region="Standard Server", + event_time_interval={"size_unit": "hours", "size": 24}, + ) + + with open(f"{os.path.dirname(__file__)}/{file_name}", "rb") as zipped: + response = MagicMock() + response.content = zipped.read() + + # Ensure `.parse_response()` returns a types.GeneratorType. + # This was a main reason why the `Events` stream re-implemented as a separate python component + assert isinstance(stream.parse_response(response, stream_state={}), types.GeneratorType) + + parsed_response = list(stream.parse_response(response, stream_state={})) + + assert len(parsed_response) == records_count + + if content_is_valid: + + # RFC3339 pattern + pattern = r"^((?:(\d{4}-\d{2}-\d{2})T(\d{2}:\d{2}:\d{2}(?:\.\d+)?))(Z|[\+-]\d{2}:\d{2})?)$" + + # Check datetime fields match RFC3339 pattern after `.parse_response()` being applied to records + for record in parsed_response: + assert re.match(pattern, record["server_received_time"]) is not None + assert re.match(pattern, record["event_time"]) is not None + assert re.match(pattern, record["processed_time"]) is not None + assert re.match(pattern, record["client_upload_time"]) is not None + assert re.match(pattern, record["server_upload_time"]) is not None + assert re.match(pattern, record["client_event_time"]) is not None + + @pytest.mark.parametrize( + "start_date, end_date, expected_slices", + [ + ( + "2023-08-01T00:00:00Z", + "2023-08-05T00:00:00Z", + [ + {"start": "20230801T00", "end": "20230801T23"}, + {"start": "20230802T00", "end": "20230802T23"}, + {"start": "20230803T00", "end": "20230803T23"}, + {"start": "20230804T00", "end": "20230804T23"}, + {"start": "20230805T00", "end": "20230805T23"}, + ], + ), + ("2023-08-05T00:00:00Z", "2023-08-01T00:00:00Z", []), + ], + ) + @patch("source_amplitude.streams.pendulum.now") + def test_event_stream_slices(self, pendulum_now_mock, start_date, end_date, expected_slices): + stream = Events( + authenticator=MagicMock(), + start_date=start_date, + data_region="Standard Server", + event_time_interval={"size_unit": "hours", "size": 24}, + ) + + pendulum_now_mock.return_value = pendulum.parse(end_date) + + slices = stream.stream_slices(stream_state={}) + + assert slices == expected_slices + + def test_event_request_params(self): + stream = Events( + authenticator=MagicMock(), + start_date="2023-08-01T00:00:00Z", + data_region="Standard Server", + event_time_interval={"size_unit": "hours", "size": 24}, + ) + params = stream.request_params(stream_slice={"start": "20230801T00", "end": "20230801T23"}) + assert params == {"start": "20230801T00", "end": "20230801T23"} + + def test_updated_state(self): + stream = Events( + authenticator=MagicMock(), + start_date="2023-08-01T00:00:00Z", + data_region="Standard Server", + event_time_interval={"size_unit": "hours", "size": 24}, + ) + + # Sample is in unordered state on purpose. We need to ensure state allways keeps latest value + cursor_fields_smaple = [ + {"server_upload_time": "2023-08-29"}, + {"server_upload_time": "2023-08-28"}, + {"server_upload_time": "2023-08-31"}, + {"server_upload_time": "2023-08-30"}, + ] + + state = {"server_upload_time": "2023-01-01"} + for record in cursor_fields_smaple: + state = stream.get_updated_state(state, record) + + assert state["server_upload_time"] == "2023-08-31 00:00:00.000000" diff --git a/airbyte-integrations/connectors/source-apify-dataset/.dockerignore b/airbyte-integrations/connectors/source-apify-dataset/.dockerignore index 20d9889b51a0..e078e16af1ec 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/.dockerignore +++ b/airbyte-integrations/connectors/source-apify-dataset/.dockerignore @@ -1,6 +1,5 @@ * !Dockerfile -!Dockerfile.test !main.py !source_apify_dataset !setup.py diff --git a/airbyte-integrations/connectors/source-apify-dataset/Dockerfile b/airbyte-integrations/connectors/source-apify-dataset/Dockerfile index 31496bd68d70..6d3b4a5de1b7 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/Dockerfile +++ b/airbyte-integrations/connectors/source-apify-dataset/Dockerfile @@ -1,16 +1,38 @@ -FROM python:3.9-slim +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base WORKDIR /airbyte/integration_code -COPY source_apify_dataset ./source_apify_dataset + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only COPY main.py ./ -COPY setup.py ./ -RUN pip install . +COPY source_apify_dataset ./source_apify_dataset ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.version=2.1.1 LABEL io.airbyte.name=airbyte/source-apify-dataset diff --git a/airbyte-integrations/connectors/source-apify-dataset/README.md b/airbyte-integrations/connectors/source-apify-dataset/README.md index b9322f3f64d4..ef9656f7b339 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/README.md +++ b/airbyte-integrations/connectors/source-apify-dataset/README.md @@ -1,135 +1,105 @@ # Apify Dataset Source -This is the repository for the Apify Dataset source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/apify-dataset). - -# About connector -This connector allows you to download data from Apify [dataset](https://docs.apify.com/storage/dataset) to Airbyte. All you need -is Apify dataset ID. +This is the repository for the Apify Dataset configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/apify-dataset). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** +#### Building via Python -#### Minimum Python version required `= 3.7.0` +Create a Python virtual environment -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: ``` -python -m venv .venv +virtualenv --python $(which python3.10) .venv ``` -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: +Source it + ``` source .venv/bin/activate -pip install -r requirements.txt ``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. +Check connector specifications/definition -#### Building via Gradle -From the Airbyte repository root, run: ``` -./gradlew :airbyte-integrations:connectors:source-apify-dataset:build +python main.py spec ``` -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/apify-dataset) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_apify_dataset/spec.json` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +Basic check - check connection to the API -You can get your Apify credentials from Settings > Integration [section](https://my.apify.com/account#/integrations) of the Apify app +``` +python main.py check --config secrets/config.json +``` -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source apify-dataset test creds` -and place them into `secrets/config.json`. +Integration tests - read operation from the API -### Locally running the connector ``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json ``` +#### Create credentials + +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/apify-dataset) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_apify_dataset/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source apify-dataset test creds` +and place them into `secrets/config.json`. + ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-apify-dataset:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-apify-dataset build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-apify-dataset:airbyteDocker +An image will be built with the tag `airbyte/source-apify-dataset:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-apify-dataset:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run + Then run any of the connector commands as follows: + ``` docker run --rm airbyte/source-apify-dataset:dev spec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-apify-dataset:dev check --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-apify-dataset:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-apify-dataset:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-apify-dataset test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-apify-dataset:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-apify-dataset:integrationTest -``` ## Dependency Management + All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list + +- required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +- required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-apify-dataset test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/apify-dataset.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-apify-dataset/__init__.py b/airbyte-integrations/connectors/source-apify-dataset/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-apify-dataset/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-apify-dataset/acceptance-test-config.yml b/airbyte-integrations/connectors/source-apify-dataset/acceptance-test-config.yml index deaf98acb9c5..71a772f72426 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-apify-dataset/acceptance-test-config.yml @@ -1,19 +1,44 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests connector_image: airbyte/source-apify-dataset:dev -tests: +acceptance_tests: spec: - - spec_path: "source_apify_dataset/spec.json" + tests: + - spec_path: "source_apify_dataset/spec.yaml" + backward_compatibility_tests_config: + disable_for_version: 2.0.0 connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: 2.0.0 basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + incremental: + bypass_reason: Connector doesn't use incremental sync + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + ignored_fields: + dataset_collection: + - name: "accessedAt" + bypass_reason: "Change everytime" + - name: "stats/readCount" + bypass_reason: "Change everytime" + dataset: + - name: "accessedAt" + bypass_reason: "Change everytime" + - name: "stats/readCount" + bypass_reason: "Change everytime" diff --git a/airbyte-integrations/connectors/source-apify-dataset/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-apify-dataset/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-apify-dataset/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-apify-dataset/build.gradle b/airbyte-integrations/connectors/source-apify-dataset/build.gradle deleted file mode 100644 index ccdf56406c4a..000000000000 --- a/airbyte-integrations/connectors/source-apify-dataset/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_apify_dataset' -} diff --git a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/__init__.py b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..0a68a0900261 --- /dev/null +++ b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/abnormal_state.json @@ -0,0 +1,16 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "modifiedAt": "3021-09-08T07:04:28.000Z" }, + "stream_descriptor": { "name": "dataset" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "modifiedAt": "3021-09-08T07:04:28.000Z" }, + "stream_descriptor": { "name": "datasets" } + } + } +] diff --git a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/catalog.json b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/catalog.json deleted file mode 100644 index d736a2ebd156..000000000000 --- a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/catalog.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "streams": [ - { - "name": "DatasetItems", - "supported_sync_modes": ["full_refresh"], - "destination_sync_mode": "overwrite", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "data": { - "type": "object", - "additionalProperties": true - } - }, - "additionalProperties": true - } - } - ] -} diff --git a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/configured_catalog.json index 02e5fe573ea0..6a76c95e966e 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/configured_catalog.json @@ -1,24 +1,31 @@ { "streams": [ { + "stream": { + "name": "dataset_collection", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "dataset", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", + "destination_sync_mode": "overwrite" + }, + { "stream": { - "name": "DatasetItems", - "supported_sync_modes": ["full_refresh"], - "destination_sync_mode": "overwrite", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "data": { - "type": "object", - "additionalProperties": true - } - }, - "additionalProperties": true - } - } + "name": "item_collection_website_content_crawler", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/expected_records.jsonl new file mode 100644 index 000000000000..453feb2f631a --- /dev/null +++ b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/expected_records.jsonl @@ -0,0 +1,2 @@ +{"stream": "datasets", "data": {"id":"Mxnvcv4Rspg9P9aP0","name":"my-dataset-name","userId":"YnGtyk7naKpwpousW","username":"encouraging_cliff","createdAt":"2023-08-25T19:19:33.588Z","modifiedAt":"2023-08-25T19:19:33.588Z","accessedAt":"2023-08-25T19:19:43.646Z","itemCount":0,"cleanItemCount":0,"actId":null,"actRunId":null,"schema":null,"stats":{"inflatedBytes":0,"readCount":0,"writeCount":0}}, "emitted_at": 1692990238010} +{"stream": "dataset", "data": {"id":"Mxnvcv4Rspg9P9aP0","name":"my-dataset-name","userId":"YnGtyk7naKpwpousW","createdAt":"2023-08-25T19:19:33.588Z","modifiedAt":"2023-08-25T19:19:33.588Z","accessedAt":"2023-08-25T19:19:43.646Z","itemCount":0,"cleanItemCount":0,"actId":null,"actRunId":null,"schema":null,"stats":{"readCount":0,"writeCount":0,"storageBytes":0},"fields":[]}, "emitted_at": 1692990238010} diff --git a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/invalid_config.json index f24eacbcc74a..fe6cd5aa5b72 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/invalid_config.json @@ -1,4 +1,4 @@ { - "datasetId": "non_existent_dataset_id", - "clean": false + "token": "abc", + "start_date": "2099-08-25T00:00:59.244Z" } diff --git a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/sample_config.json new file mode 100644 index 000000000000..805d72ca8281 --- /dev/null +++ b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/sample_config.json @@ -0,0 +1,4 @@ +{ + "token": "apify_api_XXXXXXXXXXXXXXXXXXXX", + "start_date": "2023-08-25T00:00:59.244Z" +} diff --git a/airbyte-integrations/connectors/source-apify-dataset/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/sample_state.json new file mode 100644 index 000000000000..0208967714d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-apify-dataset/integration_tests/sample_state.json @@ -0,0 +1,9 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "modifiedAt": "3021-09-08T07:04:28.000Z" }, + "stream_descriptor": { "name": "example" } + } + } +] diff --git a/airbyte-integrations/connectors/source-apify-dataset/main.py b/airbyte-integrations/connectors/source-apify-dataset/main.py index d4b0ed78bc71..4ef9d72f02b4 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/main.py +++ b/airbyte-integrations/connectors/source-apify-dataset/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_apify_dataset import SourceApifyDataset +from source_apify_dataset.run import run if __name__ == "__main__": - source = SourceApifyDataset() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml b/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml index e3ab131aa173..0e4e0668f3d4 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml +++ b/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml @@ -1,24 +1,33 @@ data: + allowedHosts: + hosts: + - api.apify.com + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: 47f17145-fe20-4ef5-a548-e29b048adf84 - dockerImageTag: 0.2.0 + dockerImageTag: 2.1.1 dockerRepository: airbyte/source-apify-dataset githubIssueLabel: source-apify-dataset icon: apify.svg license: MIT name: Apify Dataset - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: 2023-08-25 releaseStage: alpha + releases: + breakingChanges: + 1.0.0: + upgradeDeadline: 2023-08-30 + message: "Update spec to use token and ingest all 3 streams correctly" + 2.0.0: + upgradeDeadline: 2023-09-18 + message: "This version introduces a new Item Collection (WCC) stream as a substitute of the now-removed Item Collection stream in order to retain data for Web-Content-Crawler datasets." + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/apify-dataset tags: - - language:python - ab_internal: - sl: 100 - ql: 100 - supportLevel: community + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-apify-dataset/requirements.txt b/airbyte-integrations/connectors/source-apify-dataset/requirements.txt index 7b9114ed5867..cc57334ef619 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/requirements.txt +++ b/airbyte-integrations/connectors/source-apify-dataset/requirements.txt @@ -1,2 +1,2 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. +-e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-apify-dataset/setup.py b/airbyte-integrations/connectors/source-apify-dataset/setup.py index ee122749a6ac..eb3a7db40ab2 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/setup.py +++ b/airbyte-integrations/connectors/source-apify-dataset/setup.py @@ -5,13 +5,9 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "apify-client~=0.0.1"] +MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1"] -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest-mock~=3.6.1", - "pytest~=6.1", -] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( name="source_apify_dataset", @@ -20,8 +16,13 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, + entry_points={ + "console_scripts": [ + "source-apify-dataset=source_apify_dataset.run:run", + ], + }, ) diff --git a/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/__init__.py b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/__init__.py index 478378a7bb48..146d6110de38 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/__init__.py +++ b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/__init__.py @@ -1,24 +1,6 @@ -# MIT License # -# Copyright (c) 2020 Airbyte +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. from .source import SourceApifyDataset diff --git a/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/manifest.yaml b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/manifest.yaml new file mode 100644 index 000000000000..1d2bd898809d --- /dev/null +++ b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/manifest.yaml @@ -0,0 +1,131 @@ +version: "0.51.11" +type: DeclarativeSource + +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/apify-dataset + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + title: Apify Dataset Spec + type: object + required: + - token + - dataset_id + properties: + token: + type: string + title: API token + description: >- + Personal API token of your Apify account. In Apify Console, you can find your API token in the + Settings section under the Integrations tab + after you login. See the Apify Docs + for more information. + examples: + - apify_api_PbVwb1cBbuvbfg2jRmAIHZKgx3NQyfEMG7uk + airbyte_secret: true + dataset_id: + type: string + title: Dataset ID + description: >- + ID of the dataset you would like to load to Airbyte. In Apify Console, you can view your datasets in the + Storage section under the Datasets tab + after you login. See the Apify Docs + for more information. + examples: + - rHuMdwm6xCFt6WiGU + additionalProperties: true + +definitions: + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: "https://api.apify.com/v2/" + http_method: "GET" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['token'] }}" + paginator: + type: "DefaultPaginator" + page_size_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "limit" + pagination_strategy: + type: "OffsetIncrement" + page_size: 50 + page_token_option: + type: "RequestOption" + field_name: "offset" + inject_into: "request_parameter" + +streams: + - type: DeclarativeStream + name: dataset_collection + primary_key: "id" + $parameters: + path: "datasets" + schema_loader: + type: JsonFileSchemaLoader + file_path: "./source_apify_dataset/schemas/dataset_collection.json" + retriever: + $ref: "#/definitions/retriever" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["data", "items"] + + - type: DeclarativeStream + name: dataset + primary_key: "id" + $parameters: + path: "datasets/{{ config['dataset_id'] }}" + schema_loader: + type: JsonFileSchemaLoader + file_path: "./source_apify_dataset/schemas/dataset.json" + retriever: + $ref: "#/definitions/retriever" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["data"] + + - type: DeclarativeStream + name: item_collection_website_content_crawler + $parameters: + path: "datasets/{{ config['dataset_id'] }}/items" + schema_loader: + type: JsonFileSchemaLoader + file_path: "./source_apify_dataset/schemas/item_collection_wcc.json" + retriever: + $ref: "#/definitions/retriever" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + + - type: DeclarativeStream + name: item_collection + $parameters: + path: "datasets/{{ config['dataset_id'] }}/items" + schema_loader: + type: JsonFileSchemaLoader + file_path: "./source_apify_dataset/schemas/item_collection.json" + retriever: + $ref: "#/definitions/retriever" + record_selector: + type: RecordSelector + extractor: + class_name: source_apify_dataset.wrapping_dpath_extractor.WrappingDpathExtractor + field_path: [] + +check: + type: CheckStream + stream_names: + - dataset_collection + - dataset + - item_collection_website_content_crawler + - item_collection diff --git a/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/run.py b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/run.py new file mode 100644 index 000000000000..c7488d02985e --- /dev/null +++ b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/run.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch + +from .source import SourceApifyDataset + + +def run(): + source = SourceApifyDataset() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/schemas/dataset.json b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/schemas/dataset.json new file mode 100644 index 000000000000..c98d9e2d81e4 --- /dev/null +++ b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/schemas/dataset.json @@ -0,0 +1,69 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Individual datasets schema", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "userId": { + "type": ["null", "string"] + }, + "createdAt": { + "type": ["null", "string"] + }, + "stats": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "readCount": { + "type": ["null", "number"] + }, + "storageBytes": { + "type": ["null", "number"] + }, + "writeCount": { + "type": ["null", "number"] + } + } + }, + "schema": { + "type": ["null", "string", "object"] + }, + "modifiedAt": { + "type": ["null", "string"] + }, + "accessedAt": { + "type": ["null", "string"] + }, + "itemCount": { + "type": ["null", "number"] + }, + "cleanItemCount": { + "type": ["null", "number"] + }, + "actId": { + "type": ["null", "string"] + }, + "actRunId": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "fields": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array" + } + ] + } + } +} diff --git a/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/schemas/dataset_collection.json b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/schemas/dataset_collection.json new file mode 100644 index 000000000000..ed494c694ff2 --- /dev/null +++ b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/schemas/dataset_collection.json @@ -0,0 +1,72 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Collection of datasets schema", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "userId": { + "type": ["null", "string"] + }, + "createdAt": { + "type": ["null", "string"] + }, + "modifiedAt": { + "type": ["null", "string"] + }, + "accessedAt": { + "type": ["null", "string"] + }, + "itemCount": { + "type": ["null", "number"] + }, + "username": { + "type": ["null", "string"] + }, + "stats": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "readCount": { + "type": ["null", "number"] + }, + "storageBytes": { + "type": ["null", "number"] + }, + "writeCount": { + "type": ["null", "number"] + } + } + }, + "schema": { + "type": ["null", "string"] + }, + "cleanItemCount": { + "type": ["null", "number"] + }, + "actId": { + "type": ["null", "string"] + }, + "actRunId": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "fields": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "array" + } + ] + } + } +} diff --git a/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/schemas/item_collection.json b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/schemas/item_collection.json new file mode 100644 index 000000000000..5ceff1848c55 --- /dev/null +++ b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/schemas/item_collection.json @@ -0,0 +1,12 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Item collection", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "data": { + "additionalProperties": true, + "type": ["null", "object"] + } + } +} diff --git a/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/schemas/item_collection_wcc.json b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/schemas/item_collection_wcc.json new file mode 100644 index 000000000000..dc7c8a68ab47 --- /dev/null +++ b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/schemas/item_collection_wcc.json @@ -0,0 +1,59 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Item collection - Website Content Crawler (WCC)", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "crawl": { + "additionalProperties": true, + "properties": { + "depth": { + "type": ["null", "number"] + }, + "httpStatusCode": { + "type": ["null", "number"] + }, + "loadedTime": { + "type": ["null", "string"] + }, + "loadedUrl": { + "type": ["null", "string"] + }, + "referrerUrl": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "markdown": { + "type": ["null", "string"] + }, + "metadata": { + "additionalProperties": true, + "properties": { + "canonicalUrl": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "languageCode": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "text": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "screenshotUrl": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/source.py b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/source.py index 68922a9f80a1..5b99be176ad1 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/source.py +++ b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/source.py @@ -2,143 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import concurrent.futures -import json -from datetime import datetime -from functools import partial -from typing import Dict, Generator +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import ( - AirbyteCatalog, - AirbyteConnectionStatus, - AirbyteMessage, - AirbyteRecordMessage, - AirbyteStream, - ConfiguredAirbyteCatalog, - Status, - Type, -) -from airbyte_cdk.models.airbyte_protocol import SyncMode -from airbyte_cdk.sources import Source -from apify_client import ApifyClient +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -DATASET_ITEMS_STREAM_NAME = "DatasetItems" +WARNING: Do not modify this file. +""" -# Batch size for downloading dataset items from Apify dataset -BATCH_SIZE = 50000 - -class SourceApifyDataset(Source): - def _apify_get_dataset_items(self, dataset_client, clean, offset): - """ - Wrapper around Apify dataset client that returns a single page with dataset items. - This function needs to be defined explicitly so it can be called in parallel in the main read function. - - :param dataset_client: Apify dataset client - :param clean: whether to fetch only clean items (clean are non-empty ones excluding hidden columns) - :param offset: page offset - - :return: dictionary where .items field contains the fetched dataset items - """ - return dataset_client.list_items(offset=offset, limit=BATCH_SIZE, clean=clean) - - def check(self, logger: AirbyteLogger, config: json) -> AirbyteConnectionStatus: - """ - Tests if the input configuration can be used to successfully connect to the Apify integration. - This is tested by trying to access the Apify user object with the provided userId and Apify token. - - :param logger: Logging object to display debug/info/error to the logs - (logs will not be accessible via airbyte UI if they are not passed to this logger) - :param config: Json object containing the configuration of this source, content of this json is as specified in - the properties of the spec.json file - - :return: AirbyteConnectionStatus indicating a Success or Failure - """ - - try: - dataset_id = config["datasetId"] - dataset = ApifyClient().dataset(dataset_id).get() - if dataset is None: - raise ValueError(f"Dataset {dataset_id} does not exist") - except Exception as e: - return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {str(e)}") - else: - return AirbyteConnectionStatus(status=Status.SUCCEEDED) - - def discover(self, logger: AirbyteLogger, config: json) -> AirbyteCatalog: - """ - Returns an AirbyteCatalog representing the available streams and fields in this integration. - For example, given valid credentials to a Postgres database, - returns an Airbyte catalog where each postgres table is a stream, and each table column is a field. - - :param logger: Logging object to display debug/info/error to the logs - (logs will not be accessible via airbyte UI if they are not passed to this logger) - :param config: Json object containing the configuration of this source, content of this json is as specified in - the properties of the spec.json file - - :return: AirbyteCatalog is an object describing a list of all available streams in this source. - A stream is an AirbyteStream object that includes: - - its stream name (or table name in the case of Postgres) - - json_schema providing the specifications of expected schema for this stream (a list of columns described - by their names and types) - """ - stream_name = DATASET_ITEMS_STREAM_NAME - json_schema = { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "data": { - "type": "object", - "additionalProperties": True, - } - }, - } - - return AirbyteCatalog( - streams=[AirbyteStream(name=stream_name, supported_sync_modes=[SyncMode.full_refresh], json_schema=json_schema)] - ) - - def read( - self, logger: AirbyteLogger, config: json, catalog: ConfiguredAirbyteCatalog, state: Dict[str, any] - ) -> Generator[AirbyteMessage, None, None]: - """ - Returns a generator of the AirbyteMessages generated by reading the source with the given configuration, - catalog, and state. - - :param logger: Logging object to display debug/info/error to the logs - (logs will not be accessible via airbyte UI if they are not passed to this logger) - :param config: Json object containing the configuration of this source, content of this json is as specified in - the properties of the spec.json file - :param catalog: The input catalog is a ConfiguredAirbyteCatalog which is almost the same as AirbyteCatalog - returned by discover(), but - in addition, it's been configured in the UI! For each particular stream and field, there may have been provided - with extra modifications such as: filtering streams and/or columns out, renaming some entities, etc - :param state: When a Airbyte reads data from a source, it might need to keep a checkpoint cursor to resume - replication in the future from that saved checkpoint. - This is the object that is provided with state from previous runs and avoid replicating the entire set of - data everytime. - - :return: A generator that produces a stream of AirbyteRecordMessage contained in AirbyteMessage object. - """ - logger.info("Reading data from Apify dataset") - - dataset_id = config["datasetId"] - clean = config.get("clean", False) - - client = ApifyClient() - dataset_client = client.dataset(dataset_id) - - # Get total number of items in dataset. This will be used in pagination - dataset = dataset_client.get() - num_items = dataset["itemCount"] - - with concurrent.futures.ThreadPoolExecutor() as executor: - for result in executor.map(partial(self._apify_get_dataset_items, dataset_client, clean), range(0, num_items, BATCH_SIZE)): - for data in result.items: - yield AirbyteMessage( - type=Type.RECORD, - record=AirbyteRecordMessage( - stream=DATASET_ITEMS_STREAM_NAME, data={"data": data}, emitted_at=int(datetime.now().timestamp()) * 1000 - ), - ) +# Declarative Source +class SourceApifyDataset(YamlDeclarativeSource): + def __init__(self): + super().__init__(path_to_yaml="manifest.yaml") diff --git a/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/spec.json b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/spec.json deleted file mode 100644 index 62e8578d611d..000000000000 --- a/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/spec.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/apify-dataset", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Apify Dataset Spec", - "type": "object", - "required": ["datasetId"], - "additionalProperties": true, - "properties": { - "datasetId": { - "type": "string", - "title": "Dataset ID", - "description": "ID of the dataset you would like to load to Airbyte." - }, - "clean": { - "type": "boolean", - "title": "Clean", - "description": "If set to true, only clean items will be downloaded from the dataset. See description of what clean means in Apify API docs. If not sure, set clean to false." - } - } - } -} diff --git a/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/wrapping_dpath_extractor.py b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/wrapping_dpath_extractor.py new file mode 100644 index 000000000000..0fea713e975a --- /dev/null +++ b/airbyte-integrations/connectors/source-apify-dataset/source_apify_dataset/wrapping_dpath_extractor.py @@ -0,0 +1,24 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import dataclass + +import requests +from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor +from airbyte_cdk.sources.declarative.types import Record + + +@dataclass +class WrappingDpathExtractor(DpathExtractor): + """ + Record extractor that wraps the extracted value into a dict, with the value being set to the key `data`. + This is done because the actual shape of the data is dynamic, so by wrapping everything into a `data` object + it can be specified as a generic object in the schema. + + Note that this will cause fields to not be normalized in the destination. + """ + + def extract_records(self, response: requests.Response) -> list[Record]: + records = super().extract_records(response) + return [{"data": record} for record in records] diff --git a/airbyte-integrations/connectors/source-apify-dataset/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-apify-dataset/unit_tests/unit_test.py deleted file mode 100644 index 219ae0142c72..000000000000 --- a/airbyte-integrations/connectors/source-apify-dataset/unit_tests/unit_test.py +++ /dev/null @@ -1,7 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -def test_example_method(): - assert True diff --git a/airbyte-integrations/connectors/source-appfollow/README.md b/airbyte-integrations/connectors/source-appfollow/README.md index 7d41040d954f..31306ce4b031 100644 --- a/airbyte-integrations/connectors/source-appfollow/README.md +++ b/airbyte-integrations/connectors/source-appfollow/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-appfollow:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/appfollow) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_appfollow/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-appfollow:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-appfollow build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-appfollow:airbyteDocker +An image will be built with the tag `airbyte/source-appfollow:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-appfollow:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-appfollow:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-appfollow:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-appfollow:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-appfollow test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-appfollow:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-appfollow:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-appfollow test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/appfollow.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-appfollow/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-appfollow/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-appfollow/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-appfollow/build.gradle b/airbyte-integrations/connectors/source-appfollow/build.gradle deleted file mode 100644 index 8362397b1ead..000000000000 --- a/airbyte-integrations/connectors/source-appfollow/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_appfollow' -} diff --git a/airbyte-integrations/connectors/source-appfollow/metadata.yaml b/airbyte-integrations/connectors/source-appfollow/metadata.yaml index e65309bd6e97..7e9cb43f0cac 100644 --- a/airbyte-integrations/connectors/source-appfollow/metadata.yaml +++ b/airbyte-integrations/connectors/source-appfollow/metadata.yaml @@ -20,7 +20,7 @@ data: releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/appfollow tags: - - language:lowcode + - language:low-code releases: breakingChanges: 1.0.0: diff --git a/airbyte-integrations/connectors/source-apple-search-ads/README.md b/airbyte-integrations/connectors/source-apple-search-ads/README.md index f6a1226115b2..1c8b95f9abaf 100644 --- a/airbyte-integrations/connectors/source-apple-search-ads/README.md +++ b/airbyte-integrations/connectors/source-apple-search-ads/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-apple-search-ads:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/apple-search-ads) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_apple_search_ads/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-apple-search-ads:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-apple-search-ads build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-apple-search-ads:airbyteDocker +An image will be built with the tag `airbyte/source-apple-search-ads:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-apple-search-ads:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-apple-search-ads:dev c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-apple-search-ads:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-apple-search-ads:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-apple-search-ads test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-apple-search-ads:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-apple-search-ads:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-apple-search-ads test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/apple-search-ads.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-apple-search-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-apple-search-ads/acceptance-test-config.yml index b169b167be72..fe09f6e5321d 100644 --- a/airbyte-integrations/connectors/source-apple-search-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-apple-search-ads/acceptance-test-config.yml @@ -27,7 +27,6 @@ acceptance_tests: future_state: future_state_path: "integration_tests/abnormal_state.json" timeout_seconds: 3600 - threshold_days: 30 # Reduce this if you have many data to retrieve full_refresh: tests: - config_path: "secrets/config.json" @@ -38,7 +37,7 @@ acceptance_tests: bypass_reason: "Can't be idempotent by nature" - name: metadata bypass_reason: "Can't be idempotent by nature" - campaigns_report_daily: + campaigns_report_daily: - name: granularity bypass_reason: "Can't be idempotent by nature" - name: metadata diff --git a/airbyte-integrations/connectors/source-apple-search-ads/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-apple-search-ads/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-apple-search-ads/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-apple-search-ads/build.gradle b/airbyte-integrations/connectors/source-apple-search-ads/build.gradle deleted file mode 100644 index 96c13217906d..000000000000 --- a/airbyte-integrations/connectors/source-apple-search-ads/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_apple_search_ads' -} diff --git a/airbyte-integrations/connectors/source-appsflyer/README.md b/airbyte-integrations/connectors/source-appsflyer/README.md index eda361dfe176..6acca4cd2e38 100644 --- a/airbyte-integrations/connectors/source-appsflyer/README.md +++ b/airbyte-integrations/connectors/source-appsflyer/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-appsflyer:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/appsflyer) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_appsflyer/spec.json` file. @@ -56,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-appsflyer:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-appsflyer build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-appsflyer:airbyteDocker +An image will be built with the tag `airbyte/source-appsflyer:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-appsflyer:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -77,44 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-appsflyer:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-appsflyer:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-appsflyer:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-appsflyer test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-appsflyer:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-appsflyer:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -124,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-appsflyer test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/appsflyer.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-appsflyer/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-appsflyer/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-appsflyer/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-appsflyer/build.gradle b/airbyte-integrations/connectors/source-appsflyer/build.gradle deleted file mode 100644 index a7ec6d65d5a4..000000000000 --- a/airbyte-integrations/connectors/source-appsflyer/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_appsflyer' -} diff --git a/airbyte-integrations/connectors/source-appstore-singer/README.md b/airbyte-integrations/connectors/source-appstore-singer/README.md index ef0299fa9aaa..1967ca8fb8f7 100644 --- a/airbyte-integrations/connectors/source-appstore-singer/README.md +++ b/airbyte-integrations/connectors/source-appstore-singer/README.md @@ -1,113 +1,67 @@ -# Source Appstore Singer +# Pendo Source -This is the repository for the Appstore source connector, based on a Singer tap. -For information about how to use this connector within Airbyte, see [the User Documentation](https://docs.airbyte.io/integrations/sources/appstore). +This is the repository for the Pendo configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/pendo). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-appstore:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/appstore) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_appstore_singer/spec.json` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/pendo) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_pendo/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source appstore test creds` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source pendo test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main_dev.py spec -python main_dev.py check --config secrets/config.json -python main_dev.py discover --config secrets/config.json -python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -### Unit Tests -To run unit tests locally, from the connector root run: -``` -pytest unit_tests -``` - -### Locally running the connector -``` -python main_dev.py spec -python main_dev.py check --config secrets/config.json -python main_dev.py discover --config secrets/config.json -python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-appstore-singer:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name source-pendo build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-appstore:airbyteDocker +An image will be built with the tag `airbyte/source-pendo:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-pendo:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-appstore-singer:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-appstore-singer:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-appstore-singer:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-appstore-singer:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm airbyte/source-pendo:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pendo:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pendo:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-pendo:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-appstore-singer:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, create a directory `integration_tests` which contain your tests and run them with `pytest integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-appstore-singer test +``` + +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-appstore-singer test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/appstore.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-appstore-singer/build.gradle b/airbyte-integrations/connectors/source-appstore-singer/build.gradle deleted file mode 100644 index e70dcfc6b644..000000000000 --- a/airbyte-integrations/connectors/source-appstore-singer/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'source_appstore_singer' -} diff --git a/airbyte-integrations/connectors/source-asana/BOOTSTRAP.md b/airbyte-integrations/connectors/source-asana/BOOTSTRAP.md index 2fc7a562ca16..4f31f66f078e 100644 --- a/airbyte-integrations/connectors/source-asana/BOOTSTRAP.md +++ b/airbyte-integrations/connectors/source-asana/BOOTSTRAP.md @@ -5,9 +5,11 @@ This connector adds ability to fetch projects, tasks, teams etc over REST API. Connector is implemented with [Airbyte CDK](https://docs.airbyte.io/connector-development/cdk-python). Some streams depend on: + - workspaces (Teams, Users, CustomFields, Projects, Tags, Users streams); -- projects (Sections, Tasks streams); -- tasks (Stories stream); +- projects (Events, SectionsCompact, Sections, Tasks streams); +- tasks (Events, StoriesCompact stream); +- storiescompact (Stories stream) - teams (TeamMemberships stream). Each record can be uniquely identified by a `gid` key. diff --git a/airbyte-integrations/connectors/source-asana/Dockerfile b/airbyte-integrations/connectors/source-asana/Dockerfile index 07ff709fff05..5c4f6e5908d0 100644 --- a/airbyte-integrations/connectors/source-asana/Dockerfile +++ b/airbyte-integrations/connectors/source-asana/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.7 +LABEL io.airbyte.version=0.6.1 LABEL io.airbyte.name=airbyte/source-asana diff --git a/airbyte-integrations/connectors/source-asana/README.md b/airbyte-integrations/connectors/source-asana/README.md index d035ea56ad2c..84a96fb4dbdb 100644 --- a/airbyte-integrations/connectors/source-asana/README.md +++ b/airbyte-integrations/connectors/source-asana/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-asana:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/asana) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_asana/spec.json` file. @@ -56,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-asana:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-asana build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-asana:airbyteDocker +An image will be built with the tag `airbyte/source-asana:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-asana:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -77,45 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-asana:dev check --conf docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-asana:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-asana:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .'[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-asana test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-asana:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-asana:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-asana:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-asana test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/asana.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-asana/acceptance-test-config.yml b/airbyte-integrations/connectors/source-asana/acceptance-test-config.yml index 39a5cacfd655..853b640e0559 100644 --- a/airbyte-integrations/connectors/source-asana/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-asana/acceptance-test-config.yml @@ -24,6 +24,14 @@ acceptance_tests: empty_streams: - name: custom_fields bypass_reason: "This stream is not available on the account we're currently using. Please follow https://github.com/airbytehq/airbyte/issues/19662." + - name: events + bypass_reason: "This stream is not available on our current account." + - name: portfolios_compact + bypass_reason: "This stream is not available on our current account, it requires a business/enterprise account" + - name: portfolios + bypass_reason: "This stream is not available on our current account, it requires a business/enterprise account" + - name: portfolios_memberships + bypass_reason: "This stream is not available on our current account, it requires a business/enterprise account" full_refresh: # tests: # - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-asana/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-asana/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-asana/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-asana/build.gradle b/airbyte-integrations/connectors/source-asana/build.gradle deleted file mode 100644 index b55683a5ba9c..000000000000 --- a/airbyte-integrations/connectors/source-asana/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_asana' -} diff --git a/airbyte-integrations/connectors/source-asana/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-asana/integration_tests/configured_catalog.json index 104d114b919a..f7076719ea81 100644 --- a/airbyte-integrations/connectors/source-asana/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-asana/integration_tests/configured_catalog.json @@ -1,5 +1,32 @@ { "streams": [ + { + "stream": { + "name": "attachments_compact", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "attachments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "organization_exports", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "projects", @@ -9,6 +36,15 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "sections_compact", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "sections", @@ -18,6 +54,15 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "stories_compact", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "stories", diff --git a/airbyte-integrations/connectors/source-asana/metadata.yaml b/airbyte-integrations/connectors/source-asana/metadata.yaml index 5a14041b24e4..d3a4dd21bda3 100644 --- a/airbyte-integrations/connectors/source-asana/metadata.yaml +++ b/airbyte-integrations/connectors/source-asana/metadata.yaml @@ -1,12 +1,16 @@ data: + ab_internal: + ql: 300 + sl: 100 allowedHosts: hosts: - app.asana.com connectorSubtype: api connectorType: source definitionId: d0243522-dccf-4978-8ba0-37ed47a0bdbf - dockerImageTag: 0.1.7 + dockerImageTag: 0.6.1 dockerRepository: airbyte/source-asana + documentationUrl: https://docs.airbyte.com/integrations/sources/asana githubIssueLabel: source-asana icon: asana.svg license: MIT @@ -17,11 +21,7 @@ data: oss: enabled: true releaseStage: beta - documentationUrl: https://docs.airbyte.com/integrations/sources/asana + supportLevel: community tags: - language:python - ab_internal: - sl: 200 - ql: 300 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-asana/source_asana/schemas/attachments.json b/airbyte-integrations/connectors/source-asana/source_asana/schemas/attachments.json new file mode 100644 index 000000000000..e12e779c518f --- /dev/null +++ b/airbyte-integrations/connectors/source-asana/source_asana/schemas/attachments.json @@ -0,0 +1,67 @@ +{ + "type": ["null", "object"], + "properties": { + "gid": { + "type": ["null", "string"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "resource_subtype": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "download_url": { + "type": ["null", "string"] + }, + "permanent_url": { + "type": ["null", "string"] + }, + "host": { + "type": ["null", "string"] + }, + "parent": { + "type": ["null", "object"], + "properties": { + "gid": { + "type": ["null", "string"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "resource_subtype": { + "type": ["null", "string"] + }, + "created_by": { + "type": ["null", "object"], + "properties": { + "gid": { + "type": ["null", "string"] + }, + "resource_type": { + "type": ["null", "string"] + } + } + } + } + }, + "size": { + "type": ["null", "integer"] + }, + "view_url": { + "type": ["null", "string"] + }, + "connected_to_app": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-asana/source_asana/schemas/attachments_compact.json b/airbyte-integrations/connectors/source-asana/source_asana/schemas/attachments_compact.json new file mode 100644 index 000000000000..cf9295d85c58 --- /dev/null +++ b/airbyte-integrations/connectors/source-asana/source_asana/schemas/attachments_compact.json @@ -0,0 +1,17 @@ +{ + "type": ["null", "object"], + "properties": { + "gid": { + "type": ["null", "string"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "resource_subtype": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-asana/source_asana/schemas/events.json b/airbyte-integrations/connectors/source-asana/source_asana/schemas/events.json new file mode 100644 index 000000000000..77a34c6c65e8 --- /dev/null +++ b/airbyte-integrations/connectors/source-asana/source_asana/schemas/events.json @@ -0,0 +1,60 @@ +{ + "type": ["null", "object"], + "properties": { + "user": { + "type": ["null", "object"], + "properties": { + "gid": { "type": ["null", "string"] }, + "resource_type": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] } + } + }, + "resource": { + "type": ["null", "object"], + "properties": { + "gid": { "type": ["null", "string"] }, + "resource_type": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] } + } + }, + "type": { "type": ["null", "string"] }, + "action": { "type": ["null", "string"] }, + "parent": { + "type": ["null", "object"], + "properties": { + "gid": { "type": ["null", "string"] }, + "resource_type": { "type": ["null", "string"] }, + "name": "Bug Task" + } + }, + "created_at": { "type": ["null", "string"], "format": "date-time" }, + "change": { + "type": ["null", "object"], + "properties": { + "field": { "type": ["null", "string"] }, + "action": { "type": ["null", "string"] }, + "new_value": { + "type": ["null", "object"], + "properties": { + "gid": { "type": ["null", "string"] }, + "resource_type": { "type": ["null", "string"] } + } + }, + "added_value": { + "type": ["null", "object"], + "properties": { + "gid": { "type": ["null", "string"] }, + "resource_type": { "type": ["null", "string"] } + } + }, + "removed_value": { + "type": ["null", "object"], + "properties": { + "gid": { "type": ["null", "string"] }, + "resource_type": { "type": ["null", "string"] } + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-asana/source_asana/schemas/organization_exports.json b/airbyte-integrations/connectors/source-asana/source_asana/schemas/organization_exports.json new file mode 100644 index 000000000000..a76f03149570 --- /dev/null +++ b/airbyte-integrations/connectors/source-asana/source_asana/schemas/organization_exports.json @@ -0,0 +1,35 @@ +{ + "type": ["null", "object"], + "properties": { + "gid": { + "type": ["null", "string"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "download_url": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "organization": { + "type": ["null", "object"], + "properties": { + "gid": { + "type": ["null", "string"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-asana/source_asana/schemas/portfolios.json b/airbyte-integrations/connectors/source-asana/source_asana/schemas/portfolios.json new file mode 100644 index 000000000000..be1bd6661391 --- /dev/null +++ b/airbyte-integrations/connectors/source-asana/source_asana/schemas/portfolios.json @@ -0,0 +1,63 @@ +{ + "type": ["null", "object"], + "properties": { + "gid": { "type": ["null", "string"] }, + "resource_type": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "color": { "type": ["null", "string"] }, + "created_at": { "type": ["null", "string"], "format": "date-time" }, + "created_by": { + "type": ["null", "object"], + "properties": { + "gid": { "type": ["null", "string"] }, + "resource_type": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] } + } + }, + "current_status_update": { + "type": ["null", "object"], + "properties": { + "gid": { "type": ["null", "string"] }, + "resource_type": { "type": ["null", "string"] }, + "title": { "type": ["null", "string"] }, + "resource_subtype": { "type": ["null", "string"] } + } + }, + "due_on": { "type": ["null", "string"], "format": "date-time" }, + "members": { + "type": ["null", "object"], + "properties": { + "gid": { "type": ["null", "string"] }, + "resource_type": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] } + } + }, + "owner": { + "type": ["null", "object"], + "properties": { + "gid": { "type": ["null", "string"] }, + "resource_type": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] } + } + }, + "start_on": { "type": ["null", "string"], "format": "date-time" }, + "workspace": { + "type": ["null", "object"], + "properties": { + "gid": { "type": ["null", "string"] }, + "resource_type": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] } + } + }, + "permalink_url": { "type": ["null", "string"] }, + "public": { "type": ["null", "boolean"] }, + "project_templates": { + "type": ["null", "object"], + "properties": { + "gid": { "type": ["null", "string"] }, + "resource_type": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-asana/source_asana/schemas/portfolios_compact.json b/airbyte-integrations/connectors/source-asana/source_asana/schemas/portfolios_compact.json new file mode 100644 index 000000000000..39d0b95b5bf8 --- /dev/null +++ b/airbyte-integrations/connectors/source-asana/source_asana/schemas/portfolios_compact.json @@ -0,0 +1,14 @@ +{ + "type": ["null", "object"], + "properties": { + "gid": { + "type": ["null", "string"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-asana/source_asana/schemas/portfolios_memberships.json b/airbyte-integrations/connectors/source-asana/source_asana/schemas/portfolios_memberships.json new file mode 100644 index 000000000000..49a548b16d27 --- /dev/null +++ b/airbyte-integrations/connectors/source-asana/source_asana/schemas/portfolios_memberships.json @@ -0,0 +1,42 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Portfolio Memberships Schema", + "additionalProperties": true, + "type": ["object", "null"], + "properties": { + "gid": { + "type": ["string", "null"] + }, + "resource_type": { + "type": ["string", "null"] + }, + "portfolio": { + "type": ["object", "null"], + "properties": { + "gid": { + "type": ["string", "null"] + }, + "resource_type": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + } + } + }, + "user": { + "type": ["object", "null"], + "properties": { + "gid": { + "type": ["string", "null"] + }, + "resource_type": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-asana/source_asana/schemas/sections_compact.json b/airbyte-integrations/connectors/source-asana/source_asana/schemas/sections_compact.json new file mode 100644 index 000000000000..39d0b95b5bf8 --- /dev/null +++ b/airbyte-integrations/connectors/source-asana/source_asana/schemas/sections_compact.json @@ -0,0 +1,14 @@ +{ + "type": ["null", "object"], + "properties": { + "gid": { + "type": ["null", "string"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-asana/source_asana/schemas/stories.json b/airbyte-integrations/connectors/source-asana/source_asana/schemas/stories.json index 238e48ed4704..fddd089f3091 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/schemas/stories.json +++ b/airbyte-integrations/connectors/source-asana/source_asana/schemas/stories.json @@ -1,38 +1,43 @@ { "type": ["null", "object"], "properties": { - "gid": { - "type": ["null", "string"] - }, - "resource_type": { - "type": ["null", "string"] - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time" - }, + "gid": { "type": ["null", "string"] }, + "resource_type": { "type": ["null", "string"] }, + "created_at": { "type": ["null", "string"], "format": "date-time" }, "created_by": { "type": ["null", "object"], "properties": { - "gid": { - "type": ["null", "string"] - }, - "resource_type": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - } + "gid": { "type": ["null", "string"] }, + "resource_type": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] } } }, - "resource_subtype": { - "type": ["null", "string"] - }, - "text": { - "type": ["null", "string"] + "resource_subtype": { "type": ["null", "string"] }, + "text": { "type": ["null", "string"] }, + "type": { "type": ["null", "string"] }, + "html_text": { "type": ["null", "string"] }, + "is_pinned": { "type": ["null", "boolean"] }, + "sticker_name": { "type": ["null", "string"] }, + "is_editable": { "type": ["null", "boolean"] }, + "is_edited": { "type": ["null", "boolean"] }, + "liked": { "type": ["null", "boolean"] }, + "likes": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "gid": { "type": ["null", "string"] }, + "user": { + "type": ["null", "object"], + "properties": { + "gid": { "type": ["null", "string"] }, + "resource_type": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] } + } + } + } + } }, - "type": { - "type": ["null", "string"] - } + "num_likes": { "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-asana/source_asana/schemas/stories_compact.json b/airbyte-integrations/connectors/source-asana/source_asana/schemas/stories_compact.json new file mode 100644 index 000000000000..238e48ed4704 --- /dev/null +++ b/airbyte-integrations/connectors/source-asana/source_asana/schemas/stories_compact.json @@ -0,0 +1,38 @@ +{ + "type": ["null", "object"], + "properties": { + "gid": { + "type": ["null", "string"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "created_by": { + "type": ["null", "object"], + "properties": { + "gid": { + "type": ["null", "string"] + }, + "resource_type": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "resource_subtype": { + "type": ["null", "string"] + }, + "text": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-asana/source_asana/source.py b/airbyte-integrations/connectors/source-asana/source_asana/source.py index 7c781fc1e0bf..6c177f21b623 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/source.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/source.py @@ -12,7 +12,27 @@ from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator from source_asana.oauth import AsanaOauth2Authenticator -from .streams import CustomFields, Projects, Sections, Stories, Tags, Tasks, TeamMemberships, Teams, Users, Workspaces +from .streams import ( + Attachments, + AttachmentsCompact, + CustomFields, + Events, + OrganizationExports, + Portfolios, + PortfoliosCompact, + PortfoliosMemberships, + Projects, + Sections, + SectionsCompact, + Stories, + StoriesCompact, + Tags, + Tasks, + TeamMemberships, + Teams, + Users, + Workspaces, +) class SourceAsana(AbstractSource): @@ -42,11 +62,19 @@ def _get_authenticator(config: dict) -> Union[TokenAuthenticator, AsanaOauth2Aut ) def streams(self, config: Mapping[str, Any]) -> List[Stream]: - args = {"authenticator": self._get_authenticator(config)} - return [ + args = {"authenticator": self._get_authenticator(config), "test_mode": config.get("test_mode", False)} + streams = [ + AttachmentsCompact(**args), + Attachments(**args), CustomFields(**args), + Events(**args), + PortfoliosCompact(**args), + Portfolios(**args), + PortfoliosMemberships(**args), Projects(**args), + SectionsCompact(**args), Sections(**args), + StoriesCompact(**args), Stories(**args), Tags(**args), Tasks(**args), @@ -55,3 +83,6 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Users(**args), Workspaces(**args), ] + if "organization_export_ids" in config: + streams.append(OrganizationExports(organization_export_ids=config.get("organization_export_ids"), **args)) + return streams diff --git a/airbyte-integrations/connectors/source-asana/source_asana/spec.json b/airbyte-integrations/connectors/source-asana/source_asana/spec.json index 28b4e3107f20..165744a559f3 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/spec.json +++ b/airbyte-integrations/connectors/source-asana/source_asana/spec.json @@ -62,6 +62,17 @@ } } ] + }, + "test_mode": { + "type": "boolean", + "title": "Test Mode", + "description": "This flag is used for testing purposes for certain streams that return a lot of data. This flag is not meant to be enabled for prod.", + "airbyte_hidden": true + }, + "organization_export_ids": { + "title": "Organization Export IDs", + "description": "Globally unique identifiers for the organization exports", + "type": "array" } } }, diff --git a/airbyte-integrations/connectors/source-asana/source_asana/streams.py b/airbyte-integrations/connectors/source-asana/source_asana/streams.py index 081d2a140ed9..64bbcdd46274 100644 --- a/airbyte-integrations/connectors/source-asana/source_asana/streams.py +++ b/airbyte-integrations/connectors/source-asana/source_asana/streams.py @@ -4,11 +4,14 @@ from abc import ABC -from typing import Any, Iterable, Mapping, MutableMapping, Optional, Type +from itertools import islice +from typing import Any, Iterable, Mapping, MutableMapping, Optional, Type, Union import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth.core import HttpAuthenticator +from requests.auth import AuthBase ASANA_ERRORS_MAPPING = { 402: "This stream is available to premium organizations and workspaces only", @@ -29,6 +32,10 @@ class AsanaStream(HttpStream, ABC): def AsanaStreamType(self) -> Type: return self.__class__ + def __init__(self, authenticator: Union[AuthBase, HttpAuthenticator] = None, test_mode: bool = False): + super().__init__(authenticator=authenticator) + self.test_mode = test_mode + def should_retry(self, response: requests.Response) -> bool: if response.status_code in ASANA_ERRORS_MAPPING.keys(): self.logger.error( @@ -91,7 +98,8 @@ def _handle_array_type(self, prop: str, value: MutableMapping[str, Any]) -> str: def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: response_json = response.json() - yield from response_json.get("data", []) # Asana puts records in a container array "data" + # Asana puts records in a container array "data" + yield from response_json.get("data", []) def read_slices_from_records(self, stream_class: AsanaStreamType, slice_field: str) -> Iterable[Optional[Mapping[str, Any]]]: """ @@ -100,6 +108,7 @@ def read_slices_from_records(self, stream_class: AsanaStreamType, slice_field: s """ stream = stream_class(authenticator=self.authenticator) stream_slices = stream.stream_slices(sync_mode=SyncMode.full_refresh) + for stream_slice in stream_slices: for record in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice): yield {slice_field: record["gid"]} @@ -132,7 +141,7 @@ def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> Mu class ProjectRelatedStream(AsanaStream, ABC): """ - Few streams (Sections and Tasks) depends on `project gid`: Sections as a part of url and Tasks as `projects` + Few streams (SectionsCompact and Tasks) depends on `project gid`: SectionsCompact as a part of url and Tasks as `projects` argument in request. """ @@ -140,12 +149,117 @@ def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: yield from self.read_slices_from_records(stream_class=Projects, slice_field="project_gid") +class AttachmentsCompact(AsanaStream): + use_cache = True + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return "attachments" + + def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + params = super().request_params(**kwargs) + params["parent"] = stream_slice["parent_gid"] + return params + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + yield from self.read_slices_from_records(stream_class=Projects, slice_field="parent_gid") + yield from self.read_slices_from_records(stream_class=Tasks, slice_field="parent_gid") + + +class Attachments(AsanaStream): + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + attachment_gid = stream_slice["attachment_gid"] + return f"attachments/{attachment_gid}" + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + yield from self.read_slices_from_records(stream_class=AttachmentsCompact, slice_field="attachment_gid") + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + response_json = response.json() + section_data = response_json.get("data", {}) + if isinstance(section_data, dict): # Check if section_data is a dictionary + yield section_data + elif isinstance(section_data, list): # Check if section_data is a list + yield from section_data + + class CustomFields(WorkspaceRelatedStream): def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: workspace_gid = stream_slice["workspace_gid"] return f"workspaces/{workspace_gid}/custom_fields" +class Events(AsanaStream): + primary_key = "created_at" + sync_token = None + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return "events" + + def read_records(self, *args, **kwargs): + # Check if sync token is available + if self.sync_token is not None: + # Pass the sync token as a request parameter + kwargs["next_page_token"] = {"sync": self.sync_token} + + yield from super().read_records(*args, **kwargs) + + # After reading records, update the sync token + self.sync_token = self.get_latest_sync_token() + + def get_latest_sync_token(self) -> str: + latest_sync_token = self.state.get("last_sync_token") # Get the previous sync token + + if latest_sync_token is None: + return None + + return latest_sync_token["sync"] # Extract the sync token value + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + if response.status_code == 412: # Check if response is a 412 error + response_json = response.json() + if "sync" in response_json: # Check if new sync token is available + self.sync_token = response_json["sync"] + else: + self.sync_token = None + self.logger.warning("Sync token expired. Fetch the full dataset for this query now.") + else: + response_json = response.json() + + # Check if response has new sync token + if "sync" in response_json: + self.sync_token = response_json["sync"] + + yield from response_json.get("data", []) + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + decoded_response = response.json() + last_sync = decoded_response.get("sync") + if last_sync: + return {"sync": last_sync} + + def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + params = super().request_params(**kwargs) + params["resource"] = stream_slice["resource_gid"] + return params + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + yield from self.read_slices_from_records(stream_class=Projects, slice_field="resource_gid") + yield from self.read_slices_from_records(stream_class=Tasks, slice_field="resource_gid") + + +class OrganizationExports(AsanaStream): + def __init__(self, organization_export_ids: str, **kwargs): + super().__init__(**kwargs) + self._organization_export_ids = organization_export_ids + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + organization_export_gid = stream_slice["organization_export_gid"] + return f"organization_exports/{organization_export_gid}" + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + yield from [{"organization_export_gid": organization_export_id for organization_export_id in self._organization_export_ids}] + + class Projects(WorkspaceRequestParamsRelatedStream): use_cache = True @@ -153,19 +267,100 @@ def path(self, **kwargs) -> str: return "projects" -class Sections(ProjectRelatedStream): +class PortfoliosCompact(WorkspaceRequestParamsRelatedStream): + def path(self, **kwargs) -> str: + return "portfolios" + + +class Portfolios(AsanaStream): + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + portfolio_gid = stream_slice["portfolio_gid"] + return f"portfolios/{portfolio_gid}" + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + yield from self.read_slices_from_records(stream_class=PortfoliosCompact, slice_field="portfolio_gid") + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + response_json = response.json() + section_data = response_json.get("data", {}) + if isinstance(section_data, dict): # Check if section_data is a dictionary + yield section_data + elif isinstance(section_data, list): # Check if section_data is a list + yield from section_data + + +class PortfoliosMemberships(AsanaStream): + def path(self, **kwargs) -> str: + return "portfolio_memberships" + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + yield from self.read_slices_from_records(stream_class=PortfoliosCompact, slice_field="portfolio_gid") + + def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + params = super().request_params(stream_slice=stream_slice, **kwargs) + params["portfolio"] = stream_slice["porfolio_gid"] + return params + + +class SectionsCompact(ProjectRelatedStream): + use_cache = True + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: project_gid = stream_slice["project_gid"] return f"projects/{project_gid}/sections" -class Stories(AsanaStream): +class Sections(AsanaStream): + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + section_gid = stream_slice["section_gid"] + return f"sections/{section_gid}" + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + yield from self.read_slices_from_records(stream_class=SectionsCompact, slice_field="section_gid") + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + response_json = response.json() + section_data = response_json.get("data", {}) + if isinstance(section_data, dict): # Check if section_data is a dictionary + yield section_data + elif isinstance(section_data, list): # Check if section_data is a list + yield from section_data + + +class StoriesCompact(AsanaStream): + use_cache = True + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: task_gid = stream_slice["task_gid"] return f"tasks/{task_gid}/stories" def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: - yield from self.read_slices_from_records(stream_class=Tasks, slice_field="task_gid") + # This streams causes tests to timeout (> 2hrs), so we limit stream slices to 100 to make tests less noisy + if self.test_mode: + yield from islice(self.read_slices_from_records(stream_class=Tasks, slice_field="task_gid"), 100) + else: + yield from self.read_slices_from_records(stream_class=Tasks, slice_field="task_gid") + + +class Stories(AsanaStream): + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + story_gid = stream_slice["story_gid"] + return f"stories/{story_gid}" + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + # This streams causes tests to timeout (> 2hrs), so we limit stream slices to 100 to make tests less noisy + if self.test_mode: + yield from islice(self.read_slices_from_records(stream_class=StoriesCompact, slice_field="story_gid"), 100) + else: + yield from self.read_slices_from_records(stream_class=StoriesCompact, slice_field="story_gid") + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + response_json = response.json() + section_data = response_json.get("data", {}) + if isinstance(section_data, dict): # Check if section_data is a dictionary + yield section_data + elif isinstance(section_data, list): # Check if section_data is a list + yield from section_data class Tags(WorkspaceRequestParamsRelatedStream): @@ -174,6 +369,8 @@ def path(self, **kwargs) -> str: class Tasks(ProjectRelatedStream): + use_cache = True + def path(self, **kwargs) -> str: return "tasks" diff --git a/airbyte-integrations/connectors/source-asana/unit_tests/test_source.py b/airbyte-integrations/connectors/source-asana/unit_tests/test_source.py deleted file mode 100644 index 7838cd23b8d5..000000000000 --- a/airbyte-integrations/connectors/source-asana/unit_tests/test_source.py +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import PropertyMock, patch - -from airbyte_cdk.logger import AirbyteLogger -from source_asana.source import SourceAsana - -logger = AirbyteLogger() - - -def test_check_connection_ok(config, mock_stream, mock_response): - mock_stream("workspaces", response=mock_response) - ok, error_msg = SourceAsana().check_connection(logger, config=config) - - assert ok - assert not error_msg - - -def test_check_connection_empty_config(config): - config = {} - - ok, error_msg = SourceAsana().check_connection(logger, config=config) - - assert not ok - assert error_msg - - -def test_check_connection_exception(config): - with patch("source_asana.streams.Workspaces.use_cache", new_callable=PropertyMock, return_value=False): - ok, error_msg = SourceAsana().check_connection(logger, config=config) - - assert not ok - assert error_msg - - -def test_streams(config): - streams = SourceAsana().streams(config) - - assert len(streams) == 10 diff --git a/airbyte-integrations/connectors/source-asana/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-asana/unit_tests/test_streams.py deleted file mode 100644 index 377a2e3f5181..000000000000 --- a/airbyte-integrations/connectors/source-asana/unit_tests/test_streams.py +++ /dev/null @@ -1,53 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock, patch - -import pytest -import requests_mock as req_mock -from airbyte_cdk.models import SyncMode -from source_asana.streams import AsanaStream, Sections, Stories, Tags, Tasks, TeamMemberships, Users - - -@pytest.mark.parametrize( - "stream", - [Tasks, Sections, Users, TeamMemberships, Tags, Stories], -) -def test_task_stream(requests_mock, stream, mock_response): - requests_mock.get(req_mock.ANY, json=mock_response) - instance = stream(authenticator=MagicMock()) - - stream_slice = next(instance.stream_slices(sync_mode=SyncMode.full_refresh)) - record = next(instance.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice)) - - assert record - - -@patch.multiple(AsanaStream, __abstractmethods__=set()) -def test_next_page_token(): - stream = AsanaStream() - inputs = {"response": MagicMock()} - expected = "offset" - assert expected in stream.next_page_token(**inputs) - - -@pytest.mark.parametrize( - ("http_status_code", "should_retry"), - [ - (402, False), - (403, False), - (404, False), - (451, False), - (429, True), - ], -) -def test_should_retry(http_status_code, should_retry): - """ - 402, 403, 404, 451 - should not retry. - 429 - should retry. - """ - response_mock = MagicMock() - response_mock.status_code = http_status_code - stream = Stories(MagicMock()) - assert stream.should_retry(response_mock) == should_retry diff --git a/airbyte-integrations/connectors/source-ashby/README.md b/airbyte-integrations/connectors/source-ashby/README.md index c0059c22759a..d19ec1c25c3f 100644 --- a/airbyte-integrations/connectors/source-ashby/README.md +++ b/airbyte-integrations/connectors/source-ashby/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-ashby:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/ashby) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_ashby/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-ashby:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-ashby build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-ashby:airbyteDocker +An image will be built with the tag `airbyte/source-ashby:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-ashby:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-ashby:dev check --conf docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-ashby:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-ashby:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-ashby test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-ashby:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-ashby:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-ashby test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/ashby.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-ashby/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-ashby/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-ashby/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-ashby/build.gradle b/airbyte-integrations/connectors/source-ashby/build.gradle deleted file mode 100644 index 396d5a509488..000000000000 --- a/airbyte-integrations/connectors/source-ashby/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_ashby' -} diff --git a/airbyte-integrations/connectors/source-auth0/Dockerfile b/airbyte-integrations/connectors/source-auth0/Dockerfile deleted file mode 100644 index 796a01c6fd39..000000000000 --- a/airbyte-integrations/connectors/source-auth0/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_auth0 ./source_auth0 - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.4.0 -LABEL io.airbyte.name=airbyte/source-auth0 diff --git a/airbyte-integrations/connectors/source-auth0/README.md b/airbyte-integrations/connectors/source-auth0/README.md index d6131a4feda3..8f341a4172ad 100644 --- a/airbyte-integrations/connectors/source-auth0/README.md +++ b/airbyte-integrations/connectors/source-auth0/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-auth0:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/auth0) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_auth0/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-auth0:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-auth0 build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-auth0:airbyteDocker +An image will be built with the tag `airbyte/source-auth0:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-auth0:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-auth0:dev check --conf docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-auth0:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-auth0:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-auth0 test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-auth0:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-auth0:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-auth0 test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/auth0.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-auth0/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-auth0/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-auth0/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-auth0/build.gradle b/airbyte-integrations/connectors/source-auth0/build.gradle deleted file mode 100644 index b1e0ab15c2d9..000000000000 --- a/airbyte-integrations/connectors/source-auth0/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_auth0' -} diff --git a/airbyte-integrations/connectors/source-auth0/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-auth0/integration_tests/expected_records.jsonl index fd248c8c1be1..c446f7f37561 100644 --- a/airbyte-integrations/connectors/source-auth0/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-auth0/integration_tests/expected_records.jsonl @@ -1 +1,12 @@ -{"stream": "users", "data": {"created_at":"2023-08-03T14:47:51.713Z","email":"admin@medusa-test.com","email_verified":false,"identities":[{"connection":"Username-Password-Authentication","user_id":"64cbbe17741f518beae16346","provider":"auth0","isSocial":false}],"name":"admin@medusa-test.com","nickname":"admin","picture":"https://s.gravatar.com/avatar/36ded5b8b1df85ba3f21bd1382c92bbb?s=480&r=pg&d=https%3A%2F%2Fcdn.auth0.com%2Favatars%2Fad.png","updated_at":"2023-08-23T01:40:56.928Z","user_id":"auth0|64cbbe17741f518beae16346","user_metadata":{"color":"blue"},"app_metadata":{}}, "emitted_at": 1691072031178} +{"stream": "users", "data": {"created_at": "2023-07-03T19:29:26.719Z", "email": "sajarin@airbyte.io", "email_verified": true, "identities": [{"connection": "Username-Password-Authentication", "user_id": "64a321962fcc0935eb98210c", "provider": "auth0", "isSocial": false}], "name": "sajarin@airbyte.io", "nickname": "sajarin", "picture": "https://s.gravatar.com/avatar/08d194f7ee3564047b3001bf280e62bc?s=480&r=pg&d=https%3A%2F%2Fcdn.auth0.com%2Favatars%2Fsa.png", "updated_at": "2023-07-06T17:34:58.222Z", "user_id": "auth0|64a321962fcc0935eb98210c"}, "emitted_at": 1695674203927} +{"stream": "users", "data": {"created_at": "2023-08-02T16:16:14.901Z", "email": "test@airbyte.io", "email_verified": false, "identities": [{"connection": "Username-Password-Authentication", "user_id": "64ca814ef109c23f604a2824", "provider": "auth0", "isSocial": false}], "name": "test@airbyte.io", "nickname": "test", "picture": "https://s.gravatar.com/avatar/7ce87483c751d651697bd644875429d9?s=480&r=pg&d=https%3A%2F%2Fcdn.auth0.com%2Favatars%2Fte.png", "updated_at": "2023-08-02T16:16:14.901Z", "user_id": "auth0|64ca814ef109c23f604a2824"}, "emitted_at": 1695674203927} +{"stream": "users", "data": {"created_at": "2023-08-02T16:17:18.658Z", "email": "integration-test@airbyte.io", "email_verified": true, "identities": [{"connection": "Username-Password-Authentication", "user_id": "64ca818ee66e52c43b36a694", "provider": "auth0", "isSocial": false}], "name": "integration-test@airbyte.io", "nickname": "integration-test", "picture": "https://s.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=480&r=pg&d=https%3A%2F%2Fcdn.auth0.com%2Favatars%2Fin.png", "updated_at": "2023-08-02T16:17:55.129Z", "user_id": "auth0|64ca818ee66e52c43b36a694", "user_metadata": {"some_metadata_example": "another"}, "app_metadata": {}}, "emitted_at": 1695674203927} +{"stream": "users", "data": {"created_at": "2023-08-02T16:18:33.640Z", "email": "marcos@airbyte.io", "email_verified": true, "identities": [{"connection": "Username-Password-Authentication", "user_id": "64ca81d999b54ea3177bf503", "provider": "auth0", "isSocial": false}], "name": "marcos@airbyte.io", "nickname": "marcos", "picture": "https://s.gravatar.com/avatar/a67e7e22e29d689fb1950ef8f9192977?s=480&r=pg&d=https%3A%2F%2Fcdn.auth0.com%2Favatars%2Fma.png", "updated_at": "2023-08-02T16:18:47.824Z", "user_id": "auth0|64ca81d999b54ea3177bf503"}, "emitted_at": 1695674203927} +{"stream": "clients", "data": {"tenant": "airbyte", "global": false, "is_token_endpoint_ip_header_trusted": false, "name": "Default App", "callbacks": [], "is_first_party": true, "oidc_conformant": true, "sso_disabled": false, "cross_origin_auth": false, "refresh_token": {"expiration_type": "non-expiring", "leeway": 0, "infinite_token_lifetime": true, "infinite_idle_token_lifetime": true, "token_lifetime": 2592000, "idle_token_lifetime": 1296000, "rotation_type": "non-rotating"}, "organization_require_behavior": "no_prompt", "organization_usage": "require", "allowed_clients": [], "native_social_login": {"apple": {"enabled": false}, "facebook": {"enabled": false}}, "signing_keys": [{"cert": "-----BEGIN CERTIFICATE-----\r\nMIIDAzCCAeugAwIBAgIJE+zYFKyf5PQ8MA0GCSqGSIb3DQEBCwUAMB8xHTAbBgNV\r\nBAMTFGFpcmJ5dGUudXMuYXV0aDAuY29tMB4XDTIyMDMyOTExNTkxMloXDTM1MTIw\r\nNjExNTkxMlowHzEdMBsGA1UEAxMUYWlyYnl0ZS51cy5hdXRoMC5jb20wggEiMA0G\r\nCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDFHg5L5E+8FYKDN/j1TwxQU7rD5pDW\r\ns7p2C047bnyZ0C/bNc1sag7NFEP007+zRIN4SbbDb4Wfv33OXkfCKDY5nHHod2qH\r\nY0JE1dW619Wz6vtsj6741Vm/BjlnHh6krYqGTUF5V65rvgBALzGZkmQDM2sjQEGH\r\nMJLAgB4pO8V5M2/77EOe2Akd+YLvzWGKhrdbjEMMo3727AwiRhCTJvucf9p+fSJ0\r\nmhQw72TrtAxCRex3ZuTNZKSa1MJ6gD70kpkD34QYY8WolCRc1U2OpEEdl1ouvqEu\r\nRaAXX4FLBTRKI49/YbFpcxpAfEQvtJkS4eJyUWn9LL8Bo0QzSx/Pihe3AgMBAAGj\r\nQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFJpmaoPV1muvA5kGGH/BGNAv\r\np/mLMA4GA1UdDwEB/wQEAwIChDANBgkqhkiG9w0BAQsFAAOCAQEAnjHa7fNkT7Xf\r\nWg4c3gArWpfCVhi74fyZL/BDiww8wfVZOIrshCpKAGBOWrukhCg1jJP70+0V4MIC\r\nKY2iRCYPq7l9NRbr8aVCB4KBrcY6bMDAvydOLqYuZEsgnwsh+ZOSdHFtcXH9BacK\r\nt9yYdXxZBN0B27wjXzuzwBPsGwMGvW2xLFLxxnGXA5om6ANOygTK6S+GhQbGxq/E\r\nHCA2rs07/IwHrhO4V9fcSs5K6gk11elHwRXK2ZyyQJlJzuCAtYx8IwibJUXlSyFf\r\nU6gPnEimjehwMIlOvWct7zEZFn5pFsNkpCod3sdKTT4L2oouSmpzim65gwIpe0Vi\r\nZt2LLXqikw==\r\n-----END CERTIFICATE-----", "pkcs7": "-----BEGIN PKCS7-----\r\nMIIDMgYJKoZIhvcNAQcCoIIDIzCCAx8CAQExADALBgkqhkiG9w0BBwGgggMHMIID\r\nAzCCAeugAwIBAgIJE+zYFKyf5PQ8MA0GCSqGSIb3DQEBCwUAMB8xHTAbBgNVBAMT\r\nFGFpcmJ5dGUudXMuYXV0aDAuY29tMB4XDTIyMDMyOTExNTkxMloXDTM1MTIwNjEx\r\nNTkxMlowHzEdMBsGA1UEAxMUYWlyYnl0ZS51cy5hdXRoMC5jb20wggEiMA0GCSqG\r\nSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDFHg5L5E+8FYKDN/j1TwxQU7rD5pDWs7p2\r\nC047bnyZ0C/bNc1sag7NFEP007+zRIN4SbbDb4Wfv33OXkfCKDY5nHHod2qHY0JE\r\n1dW619Wz6vtsj6741Vm/BjlnHh6krYqGTUF5V65rvgBALzGZkmQDM2sjQEGHMJLA\r\ngB4pO8V5M2/77EOe2Akd+YLvzWGKhrdbjEMMo3727AwiRhCTJvucf9p+fSJ0mhQw\r\n72TrtAxCRex3ZuTNZKSa1MJ6gD70kpkD34QYY8WolCRc1U2OpEEdl1ouvqEuRaAX\r\nX4FLBTRKI49/YbFpcxpAfEQvtJkS4eJyUWn9LL8Bo0QzSx/Pihe3AgMBAAGjQjBA\r\nMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFJpmaoPV1muvA5kGGH/BGNAvp/mL\r\nMA4GA1UdDwEB/wQEAwIChDANBgkqhkiG9w0BAQsFAAOCAQEAnjHa7fNkT7XfWg4c\r\n3gArWpfCVhi74fyZL/BDiww8wfVZOIrshCpKAGBOWrukhCg1jJP70+0V4MICKY2i\r\nRCYPq7l9NRbr8aVCB4KBrcY6bMDAvydOLqYuZEsgnwsh+ZOSdHFtcXH9BacKt9yY\r\ndXxZBN0B27wjXzuzwBPsGwMGvW2xLFLxxnGXA5om6ANOygTK6S+GhQbGxq/EHCA2\r\nrs07/IwHrhO4V9fcSs5K6gk11elHwRXK2ZyyQJlJzuCAtYx8IwibJUXlSyFfU6gP\r\nnEimjehwMIlOvWct7zEZFn5pFsNkpCod3sdKTT4L2oouSmpzim65gwIpe0ViZt2L\r\nLXqikzEA\r\n-----END PKCS7-----\r\n", "subject": "deprecated"}], "client_id": "Dkmh68B7i75cvisLmQkoBafyLJ7HVAqj", "callback_url_template": false, "client_secret": "jyByoS9-Xe0s6gNP4XVp22ivAfz_DXHgAh5bIEtb3P5m4538Naa7u_EDUZp4z0GT", "jwt_configuration": {"alg": "RS256", "lifetime_in_seconds": 36000, "secret_encoded": false}, "client_aliases": [], "token_endpoint_auth_method": "client_secret_basic", "grant_types": ["authorization_code", "implicit", "refresh_token"], "custom_login_page_on": true}, "emitted_at": 1695674204388} +{"stream": "clients", "data": {"tenant": "airbyte", "global": false, "is_token_endpoint_ip_header_trusted": false, "name": "Airbyte", "is_first_party": true, "oidc_conformant": true, "sso_disabled": false, "cross_origin_auth": false, "refresh_token": {"expiration_type": "expiring", "leeway": 0, "token_lifetime": 2592000, "idle_token_lifetime": 1296000, "infinite_token_lifetime": false, "infinite_idle_token_lifetime": false, "rotation_type": "rotating"}, "allowed_clients": [], "callbacks": ["http://localhost:3000/auth_flow", "http://localhost:8000/auth_flow", "https://dev-cloud.airbyte.com/auth_flow", "https://stage-cloud.airbyte.com/auth_flow", "https://cloud.airbyte.com/auth_flow"], "description": "Airbyte is the new open-source data integration platform that consolidates your data into your warehouses.", "initiate_login_uri": "https://airbyte.com", "logo_uri": "https://assets.website-files.com/605e01bc25f7e19a82e74788/60895f8d5f8d26bc5bc925f7_airbyte_dark_icon-p1myztvi9hmvoqwsgibgtssy34m5f8twxtgsor5vk0.png", "native_social_login": {"apple": {"enabled": false}, "facebook": {"enabled": false}}, "organization_require_behavior": "no_prompt", "organization_usage": "require", "allowed_logout_urls": ["http://localhost:3000/auth_flow", "http://localhost:8000/auth_flow", "https://dev-cloud.airbyte.com/auth_flow", "https://stage-cloud.airbyte.com/auth_flow", "https://cloud.airbyte.com/auth_flow"], "signing_keys": [{"cert": "-----BEGIN CERTIFICATE-----\r\nMIIDAzCCAeugAwIBAgIJE+zYFKyf5PQ8MA0GCSqGSIb3DQEBCwUAMB8xHTAbBgNV\r\nBAMTFGFpcmJ5dGUudXMuYXV0aDAuY29tMB4XDTIyMDMyOTExNTkxMloXDTM1MTIw\r\nNjExNTkxMlowHzEdMBsGA1UEAxMUYWlyYnl0ZS51cy5hdXRoMC5jb20wggEiMA0G\r\nCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDFHg5L5E+8FYKDN/j1TwxQU7rD5pDW\r\ns7p2C047bnyZ0C/bNc1sag7NFEP007+zRIN4SbbDb4Wfv33OXkfCKDY5nHHod2qH\r\nY0JE1dW619Wz6vtsj6741Vm/BjlnHh6krYqGTUF5V65rvgBALzGZkmQDM2sjQEGH\r\nMJLAgB4pO8V5M2/77EOe2Akd+YLvzWGKhrdbjEMMo3727AwiRhCTJvucf9p+fSJ0\r\nmhQw72TrtAxCRex3ZuTNZKSa1MJ6gD70kpkD34QYY8WolCRc1U2OpEEdl1ouvqEu\r\nRaAXX4FLBTRKI49/YbFpcxpAfEQvtJkS4eJyUWn9LL8Bo0QzSx/Pihe3AgMBAAGj\r\nQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFJpmaoPV1muvA5kGGH/BGNAv\r\np/mLMA4GA1UdDwEB/wQEAwIChDANBgkqhkiG9w0BAQsFAAOCAQEAnjHa7fNkT7Xf\r\nWg4c3gArWpfCVhi74fyZL/BDiww8wfVZOIrshCpKAGBOWrukhCg1jJP70+0V4MIC\r\nKY2iRCYPq7l9NRbr8aVCB4KBrcY6bMDAvydOLqYuZEsgnwsh+ZOSdHFtcXH9BacK\r\nt9yYdXxZBN0B27wjXzuzwBPsGwMGvW2xLFLxxnGXA5om6ANOygTK6S+GhQbGxq/E\r\nHCA2rs07/IwHrhO4V9fcSs5K6gk11elHwRXK2ZyyQJlJzuCAtYx8IwibJUXlSyFf\r\nU6gPnEimjehwMIlOvWct7zEZFn5pFsNkpCod3sdKTT4L2oouSmpzim65gwIpe0Vi\r\nZt2LLXqikw==\r\n-----END CERTIFICATE-----", "pkcs7": "-----BEGIN PKCS7-----\r\nMIIDMgYJKoZIhvcNAQcCoIIDIzCCAx8CAQExADALBgkqhkiG9w0BBwGgggMHMIID\r\nAzCCAeugAwIBAgIJE+zYFKyf5PQ8MA0GCSqGSIb3DQEBCwUAMB8xHTAbBgNVBAMT\r\nFGFpcmJ5dGUudXMuYXV0aDAuY29tMB4XDTIyMDMyOTExNTkxMloXDTM1MTIwNjEx\r\nNTkxMlowHzEdMBsGA1UEAxMUYWlyYnl0ZS51cy5hdXRoMC5jb20wggEiMA0GCSqG\r\nSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDFHg5L5E+8FYKDN/j1TwxQU7rD5pDWs7p2\r\nC047bnyZ0C/bNc1sag7NFEP007+zRIN4SbbDb4Wfv33OXkfCKDY5nHHod2qHY0JE\r\n1dW619Wz6vtsj6741Vm/BjlnHh6krYqGTUF5V65rvgBALzGZkmQDM2sjQEGHMJLA\r\ngB4pO8V5M2/77EOe2Akd+YLvzWGKhrdbjEMMo3727AwiRhCTJvucf9p+fSJ0mhQw\r\n72TrtAxCRex3ZuTNZKSa1MJ6gD70kpkD34QYY8WolCRc1U2OpEEdl1ouvqEuRaAX\r\nX4FLBTRKI49/YbFpcxpAfEQvtJkS4eJyUWn9LL8Bo0QzSx/Pihe3AgMBAAGjQjBA\r\nMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFJpmaoPV1muvA5kGGH/BGNAvp/mL\r\nMA4GA1UdDwEB/wQEAwIChDANBgkqhkiG9w0BAQsFAAOCAQEAnjHa7fNkT7XfWg4c\r\n3gArWpfCVhi74fyZL/BDiww8wfVZOIrshCpKAGBOWrukhCg1jJP70+0V4MICKY2i\r\nRCYPq7l9NRbr8aVCB4KBrcY6bMDAvydOLqYuZEsgnwsh+ZOSdHFtcXH9BacKt9yY\r\ndXxZBN0B27wjXzuzwBPsGwMGvW2xLFLxxnGXA5om6ANOygTK6S+GhQbGxq/EHCA2\r\nrs07/IwHrhO4V9fcSs5K6gk11elHwRXK2ZyyQJlJzuCAtYx8IwibJUXlSyFfU6gP\r\nnEimjehwMIlOvWct7zEZFn5pFsNkpCod3sdKTT4L2oouSmpzim65gwIpe0ViZt2L\r\nLXqikzEA\r\n-----END PKCS7-----\r\n", "subject": "deprecated"}], "client_id": "SZpaYND65ny0YbfWN3LfaR96FWzqnNC7", "callback_url_template": false, "client_secret": "z0YhRyWJ9545GSImwq-0zgaRGgWP9HhrxL_kfpoMxJs3m7zc5rwG1brFKEh2GHN5", "jwt_configuration": {"alg": "RS256", "lifetime_in_seconds": 36000, "secret_encoded": false}, "client_aliases": [], "token_endpoint_auth_method": "none", "app_type": "native", "grant_types": ["authorization_code", "implicit", "refresh_token"], "custom_login_page_on": true}, "emitted_at": 1695674204391} +{"stream": "clients", "data": {"tenant": "airbyte", "global": false, "is_token_endpoint_ip_header_trusted": false, "name": "API Explorer Application", "is_first_party": true, "oidc_conformant": true, "sso_disabled": false, "cross_origin_auth": false, "refresh_token": {"expiration_type": "non-expiring", "leeway": 0, "infinite_token_lifetime": true, "infinite_idle_token_lifetime": true, "token_lifetime": 31557600, "idle_token_lifetime": 2592000, "rotation_type": "non-rotating"}, "signing_keys": [{"cert": "-----BEGIN CERTIFICATE-----\r\nMIIDAzCCAeugAwIBAgIJE+zYFKyf5PQ8MA0GCSqGSIb3DQEBCwUAMB8xHTAbBgNV\r\nBAMTFGFpcmJ5dGUudXMuYXV0aDAuY29tMB4XDTIyMDMyOTExNTkxMloXDTM1MTIw\r\nNjExNTkxMlowHzEdMBsGA1UEAxMUYWlyYnl0ZS51cy5hdXRoMC5jb20wggEiMA0G\r\nCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDFHg5L5E+8FYKDN/j1TwxQU7rD5pDW\r\ns7p2C047bnyZ0C/bNc1sag7NFEP007+zRIN4SbbDb4Wfv33OXkfCKDY5nHHod2qH\r\nY0JE1dW619Wz6vtsj6741Vm/BjlnHh6krYqGTUF5V65rvgBALzGZkmQDM2sjQEGH\r\nMJLAgB4pO8V5M2/77EOe2Akd+YLvzWGKhrdbjEMMo3727AwiRhCTJvucf9p+fSJ0\r\nmhQw72TrtAxCRex3ZuTNZKSa1MJ6gD70kpkD34QYY8WolCRc1U2OpEEdl1ouvqEu\r\nRaAXX4FLBTRKI49/YbFpcxpAfEQvtJkS4eJyUWn9LL8Bo0QzSx/Pihe3AgMBAAGj\r\nQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFJpmaoPV1muvA5kGGH/BGNAv\r\np/mLMA4GA1UdDwEB/wQEAwIChDANBgkqhkiG9w0BAQsFAAOCAQEAnjHa7fNkT7Xf\r\nWg4c3gArWpfCVhi74fyZL/BDiww8wfVZOIrshCpKAGBOWrukhCg1jJP70+0V4MIC\r\nKY2iRCYPq7l9NRbr8aVCB4KBrcY6bMDAvydOLqYuZEsgnwsh+ZOSdHFtcXH9BacK\r\nt9yYdXxZBN0B27wjXzuzwBPsGwMGvW2xLFLxxnGXA5om6ANOygTK6S+GhQbGxq/E\r\nHCA2rs07/IwHrhO4V9fcSs5K6gk11elHwRXK2ZyyQJlJzuCAtYx8IwibJUXlSyFf\r\nU6gPnEimjehwMIlOvWct7zEZFn5pFsNkpCod3sdKTT4L2oouSmpzim65gwIpe0Vi\r\nZt2LLXqikw==\r\n-----END CERTIFICATE-----", "pkcs7": "-----BEGIN PKCS7-----\r\nMIIDMgYJKoZIhvcNAQcCoIIDIzCCAx8CAQExADALBgkqhkiG9w0BBwGgggMHMIID\r\nAzCCAeugAwIBAgIJE+zYFKyf5PQ8MA0GCSqGSIb3DQEBCwUAMB8xHTAbBgNVBAMT\r\nFGFpcmJ5dGUudXMuYXV0aDAuY29tMB4XDTIyMDMyOTExNTkxMloXDTM1MTIwNjEx\r\nNTkxMlowHzEdMBsGA1UEAxMUYWlyYnl0ZS51cy5hdXRoMC5jb20wggEiMA0GCSqG\r\nSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDFHg5L5E+8FYKDN/j1TwxQU7rD5pDWs7p2\r\nC047bnyZ0C/bNc1sag7NFEP007+zRIN4SbbDb4Wfv33OXkfCKDY5nHHod2qHY0JE\r\n1dW619Wz6vtsj6741Vm/BjlnHh6krYqGTUF5V65rvgBALzGZkmQDM2sjQEGHMJLA\r\ngB4pO8V5M2/77EOe2Akd+YLvzWGKhrdbjEMMo3727AwiRhCTJvucf9p+fSJ0mhQw\r\n72TrtAxCRex3ZuTNZKSa1MJ6gD70kpkD34QYY8WolCRc1U2OpEEdl1ouvqEuRaAX\r\nX4FLBTRKI49/YbFpcxpAfEQvtJkS4eJyUWn9LL8Bo0QzSx/Pihe3AgMBAAGjQjBA\r\nMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFJpmaoPV1muvA5kGGH/BGNAvp/mL\r\nMA4GA1UdDwEB/wQEAwIChDANBgkqhkiG9w0BAQsFAAOCAQEAnjHa7fNkT7XfWg4c\r\n3gArWpfCVhi74fyZL/BDiww8wfVZOIrshCpKAGBOWrukhCg1jJP70+0V4MICKY2i\r\nRCYPq7l9NRbr8aVCB4KBrcY6bMDAvydOLqYuZEsgnwsh+ZOSdHFtcXH9BacKt9yY\r\ndXxZBN0B27wjXzuzwBPsGwMGvW2xLFLxxnGXA5om6ANOygTK6S+GhQbGxq/EHCA2\r\nrs07/IwHrhO4V9fcSs5K6gk11elHwRXK2ZyyQJlJzuCAtYx8IwibJUXlSyFfU6gP\r\nnEimjehwMIlOvWct7zEZFn5pFsNkpCod3sdKTT4L2oouSmpzim65gwIpe0ViZt2L\r\nLXqikzEA\r\n-----END PKCS7-----\r\n", "subject": "deprecated"}], "client_id": "pbsK6cibj75LaLBcRiCWa40KxGm6cavh", "callback_url_template": false, "client_secret": "LnH74agIq9MBiDix1ISPBB2rWzuTmLDSkzBTyUuo5sfVTlxnscK1l3ZnHUlysz9c", "jwt_configuration": {"alg": "RS256", "lifetime_in_seconds": 36000, "secret_encoded": false}, "token_endpoint_auth_method": "client_secret_post", "app_type": "non_interactive", "grant_types": ["client_credentials"], "custom_login_page_on": true}, "emitted_at": 1695674204392} +{"stream": "clients", "data": {"tenant": "airbyte", "global": false, "is_token_endpoint_ip_header_trusted": false, "name": "My App", "is_first_party": true, "oidc_conformant": true, "sso_disabled": false, "cross_origin_auth": false, "refresh_token": {"expiration_type": "non-expiring", "leeway": 0, "infinite_token_lifetime": true, "infinite_idle_token_lifetime": true, "token_lifetime": 31557600, "idle_token_lifetime": 2592000, "rotation_type": "non-rotating"}, "signing_keys": [{"cert": "-----BEGIN CERTIFICATE-----\r\nMIIDAzCCAeugAwIBAgIJE+zYFKyf5PQ8MA0GCSqGSIb3DQEBCwUAMB8xHTAbBgNV\r\nBAMTFGFpcmJ5dGUudXMuYXV0aDAuY29tMB4XDTIyMDMyOTExNTkxMloXDTM1MTIw\r\nNjExNTkxMlowHzEdMBsGA1UEAxMUYWlyYnl0ZS51cy5hdXRoMC5jb20wggEiMA0G\r\nCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDFHg5L5E+8FYKDN/j1TwxQU7rD5pDW\r\ns7p2C047bnyZ0C/bNc1sag7NFEP007+zRIN4SbbDb4Wfv33OXkfCKDY5nHHod2qH\r\nY0JE1dW619Wz6vtsj6741Vm/BjlnHh6krYqGTUF5V65rvgBALzGZkmQDM2sjQEGH\r\nMJLAgB4pO8V5M2/77EOe2Akd+YLvzWGKhrdbjEMMo3727AwiRhCTJvucf9p+fSJ0\r\nmhQw72TrtAxCRex3ZuTNZKSa1MJ6gD70kpkD34QYY8WolCRc1U2OpEEdl1ouvqEu\r\nRaAXX4FLBTRKI49/YbFpcxpAfEQvtJkS4eJyUWn9LL8Bo0QzSx/Pihe3AgMBAAGj\r\nQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFJpmaoPV1muvA5kGGH/BGNAv\r\np/mLMA4GA1UdDwEB/wQEAwIChDANBgkqhkiG9w0BAQsFAAOCAQEAnjHa7fNkT7Xf\r\nWg4c3gArWpfCVhi74fyZL/BDiww8wfVZOIrshCpKAGBOWrukhCg1jJP70+0V4MIC\r\nKY2iRCYPq7l9NRbr8aVCB4KBrcY6bMDAvydOLqYuZEsgnwsh+ZOSdHFtcXH9BacK\r\nt9yYdXxZBN0B27wjXzuzwBPsGwMGvW2xLFLxxnGXA5om6ANOygTK6S+GhQbGxq/E\r\nHCA2rs07/IwHrhO4V9fcSs5K6gk11elHwRXK2ZyyQJlJzuCAtYx8IwibJUXlSyFf\r\nU6gPnEimjehwMIlOvWct7zEZFn5pFsNkpCod3sdKTT4L2oouSmpzim65gwIpe0Vi\r\nZt2LLXqikw==\r\n-----END CERTIFICATE-----", "pkcs7": "-----BEGIN PKCS7-----\r\nMIIDMgYJKoZIhvcNAQcCoIIDIzCCAx8CAQExADALBgkqhkiG9w0BBwGgggMHMIID\r\nAzCCAeugAwIBAgIJE+zYFKyf5PQ8MA0GCSqGSIb3DQEBCwUAMB8xHTAbBgNVBAMT\r\nFGFpcmJ5dGUudXMuYXV0aDAuY29tMB4XDTIyMDMyOTExNTkxMloXDTM1MTIwNjEx\r\nNTkxMlowHzEdMBsGA1UEAxMUYWlyYnl0ZS51cy5hdXRoMC5jb20wggEiMA0GCSqG\r\nSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDFHg5L5E+8FYKDN/j1TwxQU7rD5pDWs7p2\r\nC047bnyZ0C/bNc1sag7NFEP007+zRIN4SbbDb4Wfv33OXkfCKDY5nHHod2qHY0JE\r\n1dW619Wz6vtsj6741Vm/BjlnHh6krYqGTUF5V65rvgBALzGZkmQDM2sjQEGHMJLA\r\ngB4pO8V5M2/77EOe2Akd+YLvzWGKhrdbjEMMo3727AwiRhCTJvucf9p+fSJ0mhQw\r\n72TrtAxCRex3ZuTNZKSa1MJ6gD70kpkD34QYY8WolCRc1U2OpEEdl1ouvqEuRaAX\r\nX4FLBTRKI49/YbFpcxpAfEQvtJkS4eJyUWn9LL8Bo0QzSx/Pihe3AgMBAAGjQjBA\r\nMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFJpmaoPV1muvA5kGGH/BGNAvp/mL\r\nMA4GA1UdDwEB/wQEAwIChDANBgkqhkiG9w0BAQsFAAOCAQEAnjHa7fNkT7XfWg4c\r\n3gArWpfCVhi74fyZL/BDiww8wfVZOIrshCpKAGBOWrukhCg1jJP70+0V4MICKY2i\r\nRCYPq7l9NRbr8aVCB4KBrcY6bMDAvydOLqYuZEsgnwsh+ZOSdHFtcXH9BacKt9yY\r\ndXxZBN0B27wjXzuzwBPsGwMGvW2xLFLxxnGXA5om6ANOygTK6S+GhQbGxq/EHCA2\r\nrs07/IwHrhO4V9fcSs5K6gk11elHwRXK2ZyyQJlJzuCAtYx8IwibJUXlSyFfU6gP\r\nnEimjehwMIlOvWct7zEZFn5pFsNkpCod3sdKTT4L2oouSmpzim65gwIpe0ViZt2L\r\nLXqikzEA\r\n-----END PKCS7-----\r\n", "subject": "deprecated"}], "client_id": "RlAV1D7k8ENS2IDvEZfJwjCLW7e4pj7d", "callback_url_template": false, "client_secret": "VeN_8mLdIAy0cOGnsKcS47AHAXyRW8AfMUlNPKhAvHkBmbIpU9ev6d2BlowuvQ8g", "jwt_configuration": {"alg": "RS256", "lifetime_in_seconds": 36000, "secret_encoded": false}, "token_endpoint_auth_method": "client_secret_post", "app_type": "non_interactive", "grant_types": ["client_credentials"], "custom_login_page_on": true}, "emitted_at": 1695674204392} +{"stream": "clients", "data": {"tenant": "airbyte", "global": true, "callbacks": [], "is_first_party": true, "name": "All Applications", "refresh_token": {"expiration_type": "non-expiring", "leeway": 0, "infinite_token_lifetime": true, "infinite_idle_token_lifetime": true, "token_lifetime": 2592000, "idle_token_lifetime": 1296000, "rotation_type": "non-rotating"}, "owners": ["mr|auth0|6242f41de81d62006815e12a"], "signing_keys": [{"cert": "-----BEGIN CERTIFICATE-----\r\nMIIDAzCCAeugAwIBAgIJE+zYFKyf5PQ8MA0GCSqGSIb3DQEBCwUAMB8xHTAbBgNV\r\nBAMTFGFpcmJ5dGUudXMuYXV0aDAuY29tMB4XDTIyMDMyOTExNTkxMloXDTM1MTIw\r\nNjExNTkxMlowHzEdMBsGA1UEAxMUYWlyYnl0ZS51cy5hdXRoMC5jb20wggEiMA0G\r\nCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDFHg5L5E+8FYKDN/j1TwxQU7rD5pDW\r\ns7p2C047bnyZ0C/bNc1sag7NFEP007+zRIN4SbbDb4Wfv33OXkfCKDY5nHHod2qH\r\nY0JE1dW619Wz6vtsj6741Vm/BjlnHh6krYqGTUF5V65rvgBALzGZkmQDM2sjQEGH\r\nMJLAgB4pO8V5M2/77EOe2Akd+YLvzWGKhrdbjEMMo3727AwiRhCTJvucf9p+fSJ0\r\nmhQw72TrtAxCRex3ZuTNZKSa1MJ6gD70kpkD34QYY8WolCRc1U2OpEEdl1ouvqEu\r\nRaAXX4FLBTRKI49/YbFpcxpAfEQvtJkS4eJyUWn9LL8Bo0QzSx/Pihe3AgMBAAGj\r\nQjBAMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFJpmaoPV1muvA5kGGH/BGNAv\r\np/mLMA4GA1UdDwEB/wQEAwIChDANBgkqhkiG9w0BAQsFAAOCAQEAnjHa7fNkT7Xf\r\nWg4c3gArWpfCVhi74fyZL/BDiww8wfVZOIrshCpKAGBOWrukhCg1jJP70+0V4MIC\r\nKY2iRCYPq7l9NRbr8aVCB4KBrcY6bMDAvydOLqYuZEsgnwsh+ZOSdHFtcXH9BacK\r\nt9yYdXxZBN0B27wjXzuzwBPsGwMGvW2xLFLxxnGXA5om6ANOygTK6S+GhQbGxq/E\r\nHCA2rs07/IwHrhO4V9fcSs5K6gk11elHwRXK2ZyyQJlJzuCAtYx8IwibJUXlSyFf\r\nU6gPnEimjehwMIlOvWct7zEZFn5pFsNkpCod3sdKTT4L2oouSmpzim65gwIpe0Vi\r\nZt2LLXqikw==\r\n-----END CERTIFICATE-----", "pkcs7": "-----BEGIN PKCS7-----\r\nMIIDMgYJKoZIhvcNAQcCoIIDIzCCAx8CAQExADALBgkqhkiG9w0BBwGgggMHMIID\r\nAzCCAeugAwIBAgIJE+zYFKyf5PQ8MA0GCSqGSIb3DQEBCwUAMB8xHTAbBgNVBAMT\r\nFGFpcmJ5dGUudXMuYXV0aDAuY29tMB4XDTIyMDMyOTExNTkxMloXDTM1MTIwNjEx\r\nNTkxMlowHzEdMBsGA1UEAxMUYWlyYnl0ZS51cy5hdXRoMC5jb20wggEiMA0GCSqG\r\nSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDFHg5L5E+8FYKDN/j1TwxQU7rD5pDWs7p2\r\nC047bnyZ0C/bNc1sag7NFEP007+zRIN4SbbDb4Wfv33OXkfCKDY5nHHod2qHY0JE\r\n1dW619Wz6vtsj6741Vm/BjlnHh6krYqGTUF5V65rvgBALzGZkmQDM2sjQEGHMJLA\r\ngB4pO8V5M2/77EOe2Akd+YLvzWGKhrdbjEMMo3727AwiRhCTJvucf9p+fSJ0mhQw\r\n72TrtAxCRex3ZuTNZKSa1MJ6gD70kpkD34QYY8WolCRc1U2OpEEdl1ouvqEuRaAX\r\nX4FLBTRKI49/YbFpcxpAfEQvtJkS4eJyUWn9LL8Bo0QzSx/Pihe3AgMBAAGjQjBA\r\nMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFJpmaoPV1muvA5kGGH/BGNAvp/mL\r\nMA4GA1UdDwEB/wQEAwIChDANBgkqhkiG9w0BAQsFAAOCAQEAnjHa7fNkT7XfWg4c\r\n3gArWpfCVhi74fyZL/BDiww8wfVZOIrshCpKAGBOWrukhCg1jJP70+0V4MICKY2i\r\nRCYPq7l9NRbr8aVCB4KBrcY6bMDAvydOLqYuZEsgnwsh+ZOSdHFtcXH9BacKt9yY\r\ndXxZBN0B27wjXzuzwBPsGwMGvW2xLFLxxnGXA5om6ANOygTK6S+GhQbGxq/EHCA2\r\nrs07/IwHrhO4V9fcSs5K6gk11elHwRXK2ZyyQJlJzuCAtYx8IwibJUXlSyFfU6gP\r\nnEimjehwMIlOvWct7zEZFn5pFsNkpCod3sdKTT4L2oouSmpzim65gwIpe0ViZt2L\r\nLXqikzEA\r\n-----END PKCS7-----\r\n", "subject": "deprecated"}], "client_id": "ZECkc5C99GMpWxf5fs4ETyhvoAowhRqe", "client_secret": "rNmjIXJ4t2QxELN3z58GP7X8MwqkSD-kbbamvwC9up4TdqEmWWiDjIr7fmSmhwA0"}, "emitted_at": 1695674204393} +{"stream": "organizations", "data": {"id": "org_wdDJQpURZbhNjvyj", "name": "test", "display_name": "test"}, "emitted_at": 1695674204670} +{"stream": "organization_members", "data": {"user_id": "auth0|64a321962fcc0935eb98210c", "email": "sajarin@airbyte.io", "picture": "https://s.gravatar.com/avatar/08d194f7ee3564047b3001bf280e62bc?s=480&r=pg&d=https%3A%2F%2Fcdn.auth0.com%2Favatars%2Fsa.png", "name": "sajarin@airbyte.io"}, "emitted_at": 1695835577224} +{"stream": "organization_member_roles", "data": {"id": "rol_h1YXl8UUxOt5w4YL", "name": "Test", "description": "This is a test role"}, "emitted_at": 1695835577818} diff --git a/airbyte-integrations/connectors/source-auth0/metadata.yaml b/airbyte-integrations/connectors/source-auth0/metadata.yaml index b0087a7a72f9..9566ee9d7800 100644 --- a/airbyte-integrations/connectors/source-auth0/metadata.yaml +++ b/airbyte-integrations/connectors/source-auth0/metadata.yaml @@ -1,28 +1,30 @@ data: + ab_internal: + ql: 100 + sl: 100 allowedHosts: hosts: - "*.auth0.com" - registries: - oss: - enabled: true - cloud: - enabled: true + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: api connectorType: source definitionId: 6c504e48-14aa-4221-9a72-19cf5ff1ae78 - dockerImageTag: 0.4.0 + dockerImageTag: 0.5.1 dockerRepository: airbyte/source-auth0 + documentationUrl: https://docs.airbyte.com/integrations/sources/auth0 githubIssueLabel: source-auth0 icon: auth0.svg license: MIT name: Auth0 + registries: + cloud: + enabled: true + oss: + enabled: true releaseDate: 2023-08-10 releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/auth0 - tags: - - language:lowcode - ab_internal: - sl: 100 - ql: 100 supportLevel: community + tags: + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-auth0/setup.py b/airbyte-integrations/connectors/source-auth0/setup.py index c634b4b3fc2d..29b448c8c0de 100644 --- a/airbyte-integrations/connectors/source-auth0/setup.py +++ b/airbyte-integrations/connectors/source-auth0/setup.py @@ -12,7 +12,6 @@ TEST_REQUIREMENTS = [ "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/components.py b/airbyte-integrations/connectors/source-auth0/source_auth0/components.py new file mode 100644 index 000000000000..d363b3daa727 --- /dev/null +++ b/airbyte-integrations/connectors/source-auth0/source_auth0/components.py @@ -0,0 +1,26 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import dataclass +from typing import Any, Mapping + +from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator +from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator +from airbyte_cdk.sources.declarative.auth.token import BearerAuthenticator + + +@dataclass +class AuthenticatorAuth0(DeclarativeAuthenticator): + config: Mapping[str, Any] + bearer: BearerAuthenticator + oauth: DeclarativeOauth2Authenticator + + def __new__(cls, bearer, oauth, config, *args, **kwargs): + auth_type = config.get("credentials", {}).get("auth_type") + if auth_type == "oauth2_access_token": + return bearer + elif auth_type == "oauth2_confidential_application": + return oauth + else: + raise Exception("Not possible configure Auth method") diff --git a/airbyte-integrations/connectors/source-auth0/source_auth0/manifest.yaml b/airbyte-integrations/connectors/source-auth0/source_auth0/manifest.yaml index f5b944162e5f..d59aea47238a 100644 --- a/airbyte-integrations/connectors/source-auth0/source_auth0/manifest.yaml +++ b/airbyte-integrations/connectors/source-auth0/source_auth0/manifest.yaml @@ -7,31 +7,46 @@ definitions: type: DpathExtractor field_path: [] + oauth_authenticator: + type: OAuthAuthenticator + token_refresh_endpoint: "{{ config['base_url'] }}oauth/token" + client_id: "{{ config['credentials']['client_id'] }}" + client_secret: "{{ config['credentials']['client_secret'] }}" + refresh_token: "" + + bearer_authenticator: + type: BearerAuthenticator + api_token: "{{ config['credentials']['access_token'] }}" + requester: type: HttpRequester - url_base: "{{ config['base_url'] }}" + url_base: "{{ config['base_url'] }}/api/v2" http_method: "GET" authenticator: - type: BearerAuthenticator - api_token: "{{ config['credentials']['access_token'] }}" + class_name: source_auth0.components.AuthenticatorAuth0 + bearer: "#/definitions/bearer_authenticator" + oauth: "#/definitions/oauth_authenticator" + + paginator: + type: "DefaultPaginator" + page_size_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "per_page" + pagination_strategy: + type: "PageIncrement" + page_size: 50 + page_token_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "page" retriever: type: SimpleRetriever record_selector: $ref: "#/definitions/selector" paginator: - type: "DefaultPaginator" - page_size_option: - type: "RequestOption" - inject_into: "request_parameter" - field_name: "per_page" - pagination_strategy: - type: "PageIncrement" - page_size: 5 - page_token_option: - type: "RequestOption" - inject_into: "request_parameter" - field_name: "page" + $ref: "#/definitions/paginator" requester: $ref: "#/definitions/requester" @@ -48,23 +63,44 @@ definitions: path: "clients" users_stream: - $ref: "#/definitions/base_stream" + type: DeclarativeStream $parameters: name: "users" primary_key: "user_id" - path: "users" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/paginator" + requester: + type: HttpRequester + url_base: "{{ config['base_url'] }}/api/v2" + path: "users" + http_method: "GET" + authenticator: + class_name: source_auth0.components.AuthenticatorAuth0 + bearer: "#/definitions/bearer_authenticator" + oauth: "#/definitions/oauth_authenticator" + request_parameters: + sort: "updated_at:1" + include_totals: "false" + q: "updated_at:{{ '{' }}{{stream_interval.start_time ~ ' TO ' ~ stream_interval.end_time ~ ']' if stream_interval.start_time else config['start_date'] ~ ' TO *]'}}" + request_headers: {} incremental_sync: type: DatetimeBasedCursor cursor_field: "updated_at" - datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" - cursor_granularity: "PT0.000001S" + datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + cursor_granularity: "PT0.001S" start_datetime: - datetime: "{{ config['start_date'] }}" - datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" + type: MinMaxDatetime + datetime: "{{ config.get('start_date', '2013-01-01T00:00:00.000Z') }}" + datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + min_datetime: "2013-01-01T00:00:00.000Z" end_datetime: - datetime: "{{ today_utc() }}" - datetime_format: "%Y-%m-%d" - step: "P1M" + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%S.%fZ') }}" + step: "P1Y" organizations_stream: $ref: "#/definitions/base_stream" diff --git a/airbyte-integrations/connectors/source-aws-cloudtrail/README.md b/airbyte-integrations/connectors/source-aws-cloudtrail/README.md index 78397e586abc..fcc264a01c39 100644 --- a/airbyte-integrations/connectors/source-aws-cloudtrail/README.md +++ b/airbyte-integrations/connectors/source-aws-cloudtrail/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-aws-cloudtrail:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/aws-cloudtrail) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_aws_cloudtrail/spec.json` file. @@ -56,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-aws-cloudtrail:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-aws-cloudtrail build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-aws-cloudtrail:airbyteDocker +An image will be built with the tag `airbyte/source-aws-cloudtrail:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-aws-cloudtrail:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -77,44 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-aws-cloudtrail:dev che docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-aws-cloudtrail:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-aws-cloudtrail:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-aws-cloudtrail test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-aws-cloudtrail:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-aws-cloudtrail:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -124,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-aws-cloudtrail test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/aws-cloudtrail.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-aws-cloudtrail/acceptance-test-config.yml b/airbyte-integrations/connectors/source-aws-cloudtrail/acceptance-test-config.yml index a379e8274c2b..c387437fc8df 100644 --- a/airbyte-integrations/connectors/source-aws-cloudtrail/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-aws-cloudtrail/acceptance-test-config.yml @@ -15,8 +15,6 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" #future_state_path: "integration_tests/abnormal_state.json" - cursor_paths: - management_events: ["EventTime"] basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-aws-cloudtrail/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-aws-cloudtrail/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-aws-cloudtrail/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-aws-cloudtrail/build.gradle b/airbyte-integrations/connectors/source-aws-cloudtrail/build.gradle deleted file mode 100644 index 175c4816738a..000000000000 --- a/airbyte-integrations/connectors/source-aws-cloudtrail/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_aws_cloudtrail' -} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/.dockerignore b/airbyte-integrations/connectors/source-azure-blob-storage/.dockerignore new file mode 100644 index 000000000000..12815beba423 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_azure_blob_storage +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/Dockerfile b/airbyte-integrations/connectors/source-azure-blob-storage/Dockerfile deleted file mode 100644 index 7eec3ab1e3c8..000000000000 --- a/airbyte-integrations/connectors/source-azure-blob-storage/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-azure-blob-storage - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-azure-blob-storage - -COPY --from=build /airbyte /airbyte - -# Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile. -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-azure-blob-storage diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/README.md b/airbyte-integrations/connectors/source-azure-blob-storage/README.md index 855d13694a28..9e2f70b6ed52 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/README.md +++ b/airbyte-integrations/connectors/source-azure-blob-storage/README.md @@ -1,32 +1,117 @@ -# Source Azure Blob Storage +# Azure Blob Storage Source -This is the repository for the Azure Blob Storage source connector in Java. -For information about how to use this connector within Airbyte, see [the User Documentation](https://docs.airbyte.io/integrations/sources/azure-blob-storage). +This is the repository for the Azure Blob Storage source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/azure-blob-storage). ## Local development -#### Building via Gradle -From the Airbyte repository root, run: +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: ``` -./gradlew :airbyte-integrations:connectors:source-azure-blob-storage:build +source .venv/bin/activate +pip install -r requirements.txt ``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. #### Create credentials -**If you are a community contributor**, generate the necessary credentials and place them in `secrets/config.json` conforming to the spec file in `src/main/resources/spec.json`. -Note that the `secrets` directory is git-ignored by default, so there is no danger of accidentally checking in sensitive information. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/azure-blob-storage) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_azure_blob_storage/spec.yaml` file. +Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source azure-blob-storage test creds` +and place them into `secrets/config.json`. -**If you are an Airbyte core member**, follow the [instructions](https://docs.airbyte.io/connector-development#using-credentials-in-ci) to set up the credentials. +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` ### Locally running the connector docker image -#### Build -Build the connector image via Gradle: + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-azure-blob-storage build +``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-azure-blob-storage:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-azure-blob-storage:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-azure-blob-storage:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-azure-blob-storage:dev . +# Running the spec command against your patched connector +docker run airbyte/source-azure-blob-storage:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -37,33 +122,28 @@ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integrat ``` ## Testing -We use `JUnit` for Java tests. - -### Unit and Integration Tests -Place unit tests under `src/test/...` -Place integration tests in `src/test-integration/...` - -#### Acceptance Tests -Airbyte has a standard test suite that all source connectors must pass. Implement the `TODO`s in -`src/test-integration/java/io/airbyte/integrations/sources/AzureBlobStorageSourceAcceptanceTest.java`. - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-azure-blob-storage:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-azure-blob-storage:integrationTest +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-azure-blob-storage test ``` +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + ## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-azure-blob-storage test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/azure-blob-storage.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/acceptance-test-config.yml b/airbyte-integrations/connectors/source-azure-blob-storage/acceptance-test-config.yml index 80579ba60f35..71d40148b88f 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-azure-blob-storage/acceptance-test-config.yml @@ -1,7 +1,144 @@ -# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-azure-blob-storage:dev -acceptance-tests: +acceptance_tests: + basic_read: + tests: + - config_path: secrets/config.json + expect_records: + path: integration_tests/expected_records/csv.jsonl + exact_order: true + - config_path: secrets/csv_custom_encoding_config.json + expect_records: + path: integration_tests/expected_records/csv_custom_encoding.jsonl + exact_order: true + - config_path: secrets/csv_custom_format_config.json + expect_records: + path: integration_tests/expected_records/csv_custom_format.jsonl + exact_order: true + - config_path: secrets/csv_user_schema_config.json + expect_records: + path: integration_tests/expected_records/csv_user_schema.jsonl + exact_order: true + - config_path: secrets/csv_no_header_config.json + expect_records: + path: integration_tests/expected_records/csv_no_header.jsonl + exact_order: true + - config_path: secrets/csv_skip_rows_config.json + expect_records: + path: integration_tests/expected_records/csv_skip_rows.jsonl + exact_order: true + - config_path: secrets/csv_skip_rows_no_header_config.json + expect_records: + path: integration_tests/expected_records/csv_skip_rows_no_header.jsonl + exact_order: true + - config_path: secrets/csv_with_nulls_config.json + expect_records: + path: integration_tests/expected_records/csv_with_nulls.jsonl + exact_order: true + - config_path: secrets/csv_with_null_bools_config.json + expect_records: + path: integration_tests/expected_records/csv_with_null_bools.jsonl + exact_order: true + - config_path: secrets/parquet_config.json + expect_records: + path: integration_tests/expected_records/parquet.jsonl + exact_order: true + - config_path: secrets/avro_config.json + expect_records: + path: integration_tests/expected_records/avro.jsonl + exact_order: true + - config_path: secrets/jsonl_config.json + expect_records: + path: integration_tests/expected_records/jsonl.jsonl + exact_order: true + - config_path: secrets/jsonl_newlines_config.json + expect_records: + path: integration_tests/expected_records/jsonl_newlines.jsonl + exact_order: true + - config_path: secrets/unstructured_config.json + expect_records: + path: integration_tests/expected_records/unstructured.jsonl + exact_order: true + timeout_seconds: 1800 + connection: + tests: + - config_path: secrets/config.json + status: succeed + - config_path: secrets/csv_custom_encoding_config.json + status: succeed + - config_path: secrets/csv_custom_format_config.json + status: succeed + - config_path: secrets/csv_user_schema_config.json + status: succeed + - config_path: secrets/csv_no_header_config.json + status: succeed + - config_path: secrets/csv_skip_rows_config.json + status: succeed + - config_path: secrets/csv_skip_rows_no_header_config.json + status: succeed + - config_path: secrets/csv_with_nulls_config.json + status: succeed + - config_path: secrets/csv_with_null_bools_config.json + status: succeed + - config_path: secrets/parquet_config.json + status: succeed + - config_path: secrets/avro_config.json + status: succeed + - config_path: secrets/jsonl_config.json + status: succeed + - config_path: secrets/jsonl_newlines_config.json + status: succeed + - config_path: secrets/unstructured_config.json + status: succeed + discovery: + tests: + - config_path: secrets/config.json + - config_path: secrets/csv_custom_encoding_config.json + - config_path: secrets/csv_custom_format_config.json + - config_path: secrets/csv_user_schema_config.json + - config_path: secrets/csv_no_header_config.json + - config_path: secrets/csv_skip_rows_config.json + - config_path: secrets/csv_skip_rows_no_header_config.json + - config_path: secrets/csv_with_nulls_config.json + - config_path: secrets/csv_with_null_bools_config.json + - config_path: secrets/parquet_config.json + - config_path: secrets/avro_config.json + - config_path: secrets/jsonl_config.json + - config_path: secrets/jsonl_newlines_config.json + full_refresh: + tests: + - config_path: secrets/config.json + configured_catalog_path: integration_tests/configured_catalogs/csv.json + - config_path: secrets/parquet_config.json + configured_catalog_path: integration_tests/configured_catalogs/parquet.json + - config_path: secrets/avro_config.json + configured_catalog_path: integration_tests/configured_catalogs/avro.json + - config_path: secrets/jsonl_config.json + configured_catalog_path: integration_tests/configured_catalogs/jsonl.json + - config_path: secrets/jsonl_newlines_config.json + configured_catalog_path: integration_tests/configured_catalogs/jsonl.json + incremental: + tests: + - config_path: secrets/config.json + configured_catalog_path: integration_tests/configured_catalogs/csv.json + future_state: + future_state_path: integration_tests/abnormal_states/csv.json + - config_path: secrets/parquet_config.json + configured_catalog_path: integration_tests/configured_catalogs/parquet.json + future_state: + future_state_path: integration_tests/abnormal_states/parquet.json + - config_path: secrets/avro_config.json + configured_catalog_path: integration_tests/configured_catalogs/avro.json + future_state: + future_state_path: integration_tests/abnormal_states/avro.json + - config_path: secrets/jsonl_config.json + configured_catalog_path: integration_tests/configured_catalogs/jsonl.json + future_state: + future_state_path: integration_tests/abnormal_states/jsonl.json + - config_path: secrets/jsonl_newlines_config.json + configured_catalog_path: integration_tests/configured_catalogs/jsonl.json + future_state: + future_state_path: integration_tests/abnormal_states/jsonl_newlines.json spec: tests: - - spec_path: "main/resources/spec.json" + - spec_path: integration_tests/spec.json +connector_image: airbyte/source-azure-blob-storage:dev +test_strictness_level: low diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-azure-blob-storage/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-azure-blob-storage/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/build.gradle b/airbyte-integrations/connectors/source-azure-blob-storage/build.gradle deleted file mode 100644 index bc2e8f3730a7..000000000000 --- a/airbyte-integrations/connectors/source-azure-blob-storage/build.gradle +++ /dev/null @@ -1,30 +0,0 @@ -plugins { - id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' -} - -application { - mainClass = 'io.airbyte.integrations.source.azureblobstorage.AzureBlobStorageSource' -} - - -dependencies { - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:connectors:source-relational-db') - implementation project(':airbyte-config-oss:config-models-oss') - - implementation "com.azure:azure-storage-blob:12.20.2" - implementation "com.github.saasquatch:json-schema-inferrer:0.1.5" - - testImplementation "org.assertj:assertj-core:3.23.1" - testImplementation "org.testcontainers:junit-jupiter:1.17.5" - testImplementation 'org.skyscreamer:jsonassert:1.5.1' - - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-azure-blob-storage') - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) -} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/__init__.py b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/abnormal_states/avro.json b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/abnormal_states/avro.json new file mode 100644 index 000000000000..f9428f77d8c8 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/abnormal_states/avro.json @@ -0,0 +1,12 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "_ab_source_file_last_modified": "2999-01-01T00:00:00.000000Z_test_sample.avro", + "history": { "test_sample.avro": "2999-01-01T00:00:00.000000Z" } + }, + "stream_descriptor": { "name": "airbyte-source-azure-blob-storage-test" } + } + } +] diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/abnormal_states/csv.json b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/abnormal_states/csv.json new file mode 100644 index 000000000000..10347a3c9e7a --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/abnormal_states/csv.json @@ -0,0 +1,12 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "_ab_source_file_last_modified": "2999-01-01T00:00:00.000000Z_simple_test.csv", + "history": { "simple_test.csv": "2999-01-01T00:00:00.000000Z" } + }, + "stream_descriptor": { "name": "airbyte-source-azure-blob-storage-test" } + } + } +] diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/abnormal_states/jsonl.json b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/abnormal_states/jsonl.json new file mode 100644 index 000000000000..99aab040ba62 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/abnormal_states/jsonl.json @@ -0,0 +1,12 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "_ab_source_file_last_modified": "2999-01-01T00:00:00.000000Z_simple_test.jsonl", + "history": { "simple_test.jsonl": "2999-01-01T00:00:00.000000Z" } + }, + "stream_descriptor": { "name": "airbyte-source-azure-blob-storage-test" } + } + } +] diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/abnormal_states/jsonl_newlines.json b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/abnormal_states/jsonl_newlines.json new file mode 100644 index 000000000000..c645ceeab9f5 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/abnormal_states/jsonl_newlines.json @@ -0,0 +1,14 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "_ab_source_file_last_modified": "2999-01-01T00:00:00.000000Z_simple_test_newlines.jsonl", + "history": { + "simple_test_newlines.jsonl": "2999-01-01T00:00:00.000000Z" + } + }, + "stream_descriptor": { "name": "airbyte-source-azure-blob-storage-test" } + } + } +] diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/abnormal_states/parquet.json b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/abnormal_states/parquet.json new file mode 100644 index 000000000000..be24e222e3d3 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/abnormal_states/parquet.json @@ -0,0 +1,18 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "_ab_source_file_last_modified": "2999-01-01T00:00:00.000000Z_simple_test.csv", + "history": { + "test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet": "2999-01-01T00:00:00.000000Z", + "test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Hour/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet": "2999-01-01T00:00:00.000000Z", + "test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet": "2999-01-01T00:00:00.000000Z", + "test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet": "2999-01-01T00:00:00.000000Z", + "test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet": "2999-01-01T00:00:00.000000Z" + } + }, + "stream_descriptor": { "name": "airbyte-source-azure-blob-storage-test" } + } + } +] diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/acceptance.py new file mode 100644 index 000000000000..43ce950d77ca --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..d5ec74c6346f --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configured_catalog.json @@ -0,0 +1,14 @@ +{ + "streams": [ + { + "stream": { + "name": "airbyte-source-azure-blob-storage-test", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configured_catalogs/avro.json b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configured_catalogs/avro.json new file mode 100644 index 000000000000..85f11913e421 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configured_catalogs/avro.json @@ -0,0 +1,38 @@ +{ + "streams": [ + { + "stream": { + "name": "airbyte-source-azure-blob-storage-test", + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": ["integer", "null"] + }, + "fullname_and_valid": { + "type": ["object", "null"], + "fullname": { + "type": ["string", "null"] + }, + "valid": { + "type": ["boolean", "null"] + } + }, + "_ab_source_file_last_modified": { + "type": "string", + "format": "date-time" + }, + "_ab_source_file_url": { + "type": "string" + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["_ab_source_file_last_modified"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configured_catalogs/csv.json b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configured_catalogs/csv.json new file mode 100644 index 000000000000..009fe4584cc9 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configured_catalogs/csv.json @@ -0,0 +1,35 @@ +{ + "streams": [ + { + "stream": { + "name": "airbyte-source-azure-blob-storage-test", + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "valid": { + "type": ["null", "boolean"] + }, + "_ab_source_file_last_modified": { + "type": "string", + "format": "date-time" + }, + "_ab_source_file_url": { + "type": "string" + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["_ab_source_file_last_modified"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configured_catalogs/jsonl.json b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configured_catalogs/jsonl.json new file mode 100644 index 000000000000..102bebaf253e --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configured_catalogs/jsonl.json @@ -0,0 +1,41 @@ +{ + "streams": [ + { + "stream": { + "name": "airbyte-source-azure-blob-storage-test", + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "valid": { + "type": ["null", "boolean"] + }, + "value": { + "type": ["null", "number"] + }, + "event_date": { + "type": ["null", "string"] + }, + "_ab_source_file_last_modified": { + "type": "string", + "format": "date-time" + }, + "_ab_source_file_url": { + "type": "string" + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["_ab_source_file_last_modified"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configured_catalogs/parquet.json b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configured_catalogs/parquet.json new file mode 100644 index 000000000000..013465e64d42 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/configured_catalogs/parquet.json @@ -0,0 +1,74 @@ +{ + "streams": [ + { + "stream": { + "name": "airbyte-source-azure-blob-storage-test", + "json_schema": { + "type": "object", + "properties": { + "Payroll_Number": { + "type": ["null", "number"] + }, + "Last_Name": { + "type": ["null", "string"] + }, + "First_Name": { + "type": ["null", "string"] + }, + "Mid_Init": { + "type": ["null", "string"] + }, + "Agency_Start_Date": { + "type": ["null", "string"] + }, + "Work_Location_Borough": { + "type": ["null", "number"] + }, + "Title_Description": { + "type": ["null", "string"] + }, + "Base_Salary": { + "type": ["null", "number"] + }, + "Regular_Hours": { + "type": ["null", "number"] + }, + "Regular_Gross_Paid": { + "type": ["null", "number"] + }, + "OT_Hours": { + "type": ["null", "number"] + }, + "Total_OT_Paid": { + "type": ["null", "number"] + }, + "Total_Other_Pay": { + "type": ["null", "number"] + }, + "Fiscal_Year": { + "type": ["null", "string"] + }, + "Leave_Status_as_of_June_30": { + "type": ["null", "string"] + }, + "Pay_Basis": { + "type": ["null", "string"] + }, + "_ab_source_file_last_modified": { + "type": "string", + "format": "date-time" + }, + "_ab_source_file_url": { + "type": "string" + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["_ab_source_file_last_modified"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/avro.jsonl b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/avro.jsonl new file mode 100644 index 000000000000..6f7a2a884d1e --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/avro.jsonl @@ -0,0 +1,10 @@ +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 0, "fullname_and_valid": {"fullname": "cfjwIzCRTL", "valid": false}, "_ab_source_file_last_modified": "2023-10-12T15:27:33.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 1697137055360} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 1, "fullname_and_valid": {"fullname": "LYOnPyuTWw", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T15:27:33.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 1697137055363} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 2, "fullname_and_valid": {"fullname": "hyTFbsxlRB", "valid": false}, "_ab_source_file_last_modified": "2023-10-12T15:27:33.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 1697137055363} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 3, "fullname_and_valid": {"fullname": "ooEUiFcFqp", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T15:27:33.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 1697137055364} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 4, "fullname_and_valid": {"fullname": "pveENwAvOg", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T15:27:33.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 1697137055365} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 5, "fullname_and_valid": {"fullname": "pPhWgQgZFq", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T15:27:33.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 1697137055365} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 6, "fullname_and_valid": {"fullname": "MRNMXFkXZo", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T15:27:33.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 1697137055366} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 7, "fullname_and_valid": {"fullname": "MXvEWMgnIr", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T15:27:33.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 1697137055367} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 8, "fullname_and_valid": {"fullname": "rqmFGqZqdF", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T15:27:33.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 1697137055367} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 9, "fullname_and_valid": {"fullname": "lmPpQTcPFM", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T15:27:33.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 1697137055368} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv.jsonl b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv.jsonl new file mode 100644 index 000000000000..e0ce199ad2b4 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv.jsonl @@ -0,0 +1,8 @@ +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:28.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1697137323277} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-10-12T15:27:28.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1697137323280} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:28.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1697137323280} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:28.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1697137323281} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-10-12T15:27:28.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1697137323282} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-10-12T15:27:28.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1697137323282} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:28.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1697137323283} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:28.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1697137323283} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_custom_encoding.jsonl b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_custom_encoding.jsonl new file mode 100644 index 000000000000..6b2c285748b7 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_custom_encoding.jsonl @@ -0,0 +1,8 @@ +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 1, "name": "PVdhmjb1\u20ac", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1697137941724} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1697137941726} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1697137941727} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1697137941727} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1697137941728} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1697137941729} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1697137941729} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1697137941730} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_custom_format.jsonl b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_custom_format.jsonl new file mode 100644 index 000000000000..908ba5255dae --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_custom_format.jsonl @@ -0,0 +1,8 @@ +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 1, "name": "PVdhmj|b1", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:53.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1697137871167} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-10-12T15:27:53.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1697137871168} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:53.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1697137871168} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:53.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1697137871168} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-10-12T15:27:53.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1697137871168} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-10-12T15:27:53.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1697137871168} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:53.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1697137871168} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:53.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1697137871169} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_no_header.jsonl b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_no_header.jsonl new file mode 100644 index 000000000000..2814616406a3 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_no_header.jsonl @@ -0,0 +1,8 @@ +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"f0": 1, "f1": "PVdhmjb1", "f2": false, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1697190339868} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"f0": 2, "f1": "j4DyXTS7", "f2": true, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1697190339871} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"f0": 3, "f1": "v0w8fTME", "f2": false, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1697190339872} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"f0": 4, "f1": "1q6jD8Np", "f2": false, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1697190339873} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"f0": 5, "f1": "77h4aiMP", "f2": true, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1697190339873} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"f0": 6, "f1": "Le35Wyic", "f2": true, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1697190339874} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"f0": 7, "f1": "xZhh1Kyl", "f2": false, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1697190339875} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"f0": 8, "f1": "M2t286iJ", "f2": false, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1697190339876} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_skip_rows.jsonl b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_skip_rows.jsonl new file mode 100644 index 000000000000..26e6740250a8 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_skip_rows.jsonl @@ -0,0 +1,8 @@ +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:45.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1697138054160} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-10-12T15:27:45.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1697138054163} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:45.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1697138054163} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:45.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1697138054164} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-10-12T15:27:45.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1697138054165} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-10-12T15:27:45.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1697138054165} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:45.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1697138054166} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:45.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1697138054166} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_skip_rows_no_header.jsonl b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_skip_rows_no_header.jsonl new file mode 100644 index 000000000000..fdf068745cd4 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_skip_rows_no_header.jsonl @@ -0,0 +1,8 @@ +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"f0": 1, "f1": "PVdhmjb1", "f2": false, "_ab_source_file_last_modified": "2023-10-12T17:22:14.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1697190448512} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"f0": 2, "f1": "j4DyXTS7", "f2": true, "_ab_source_file_last_modified": "2023-10-12T17:22:14.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1697190448514} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"f0": 3, "f1": "v0w8fTME", "f2": false, "_ab_source_file_last_modified": "2023-10-12T17:22:14.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1697190448515} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"f0": 4, "f1": "1q6jD8Np", "f2": false, "_ab_source_file_last_modified": "2023-10-12T17:22:14.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1697190448516} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"f0": 5, "f1": "77h4aiMP", "f2": true, "_ab_source_file_last_modified": "2023-10-12T17:22:14.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1697190448516} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"f0": 6, "f1": "Le35Wyic", "f2": true, "_ab_source_file_last_modified": "2023-10-12T17:22:14.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1697190448517} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"f0": 7, "f1": "xZhh1Kyl", "f2": false, "_ab_source_file_last_modified": "2023-10-12T17:22:14.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1697190448517} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"f0": 8, "f1": "M2t286iJ", "f2": false, "_ab_source_file_last_modified": "2023-10-12T17:22:14.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1697190448518} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_user_schema.jsonl b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_user_schema.jsonl new file mode 100644 index 000000000000..4d4e8c269680 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_user_schema.jsonl @@ -0,0 +1,8 @@ +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 1.0, "name": "PVdhmjb1", "valid": false, "valid_string": "False", "array": "[\"a\", \"b\", \"c\"]", "dict": "{\"key\": \"value\"}", "_ab_source_file_last_modified": "2023-10-12T15:27:45.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1697190171210} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 2.0, "name": "j4DyXTS7", "valid": true, "valid_string": "True", "array": "[\"a\", \"b\"]", "dict": "{\"key\": \"value_with_comma\\,\"}", "_ab_source_file_last_modified": "2023-10-12T15:27:45.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1697190171213} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 3.0, "name": "v0w8fTME", "valid": false, "valid_string": "False", "array": "[\"a\"]", "dict": "{\"key\": \"value\"}", "_ab_source_file_last_modified": "2023-10-12T15:27:45.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1697190171214} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 4.0, "name": "1q6jD8Np", "valid": false, "valid_string": "False", "array": "[]", "dict": "{}", "_ab_source_file_last_modified": "2023-10-12T15:27:45.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1697190171214} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 5.0, "name": "77h4aiMP", "valid": true, "valid_string": "True", "array": "[\"b\", \"c\"]", "dict": "{}", "_ab_source_file_last_modified": "2023-10-12T15:27:45.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1697190171215} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 6.0, "name": "Le35Wyic", "valid": true, "valid_string": "True", "array": "[\"a\", \"c\"]", "dict": "{}", "_ab_source_file_last_modified": "2023-10-12T15:27:45.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1697190171216} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 7.0, "name": "xZhh1Kyl", "valid": false, "valid_string": "False", "array": "[\"b\"]", "dict": "{}", "_ab_source_file_last_modified": "2023-10-12T15:27:45.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1697190171216} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 8.0, "name": "M2t286iJ", "valid": false, "valid_string": "False", "array": "[\"c\"]", "dict": "{}", "_ab_source_file_last_modified": "2023-10-12T15:27:45.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1697190171217} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_with_null_bools.jsonl b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_with_null_bools.jsonl new file mode 100644 index 000000000000..5f67bca607c9 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_with_null_bools.jsonl @@ -0,0 +1,8 @@ +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 1, "name": "null", "valid": null, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1697138273346} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1697138273348} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 3, "name": "NULL", "valid": null, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1697138273349} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1697138273350} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 5, "name": "77h4aiMP", "valid": null, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1697138273351} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 6, "name": "", "valid": true, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1697138273352} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1697138273352} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:43.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1697138273353} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_with_nulls.jsonl b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_with_nulls.jsonl new file mode 100644 index 000000000000..c5e78a965126 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/csv_with_nulls.jsonl @@ -0,0 +1,8 @@ +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 1, "name": null, "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:53.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1697138317244} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-10-12T15:27:53.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1697138317246} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 3, "name": null, "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:53.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1697138317247} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:53.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1697138317247} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-10-12T15:27:53.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1697138317248} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-10-12T15:27:53.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1697138317248} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:53.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1697138317249} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-10-12T15:27:53.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1697138317250} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/jsonl.jsonl b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/jsonl.jsonl new file mode 100644 index 000000000000..363228ec8e7c --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/jsonl.jsonl @@ -0,0 +1,2 @@ +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false, "value": 1.2, "event_date": "2022-01-01T00:00:00Z", "_ab_source_file_last_modified": "2023-10-12T15:27:30.000000Z", "_ab_source_file_url": "simple_test.jsonl"}, "emitted_at": 1697137760681} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 2, "name": "ABCDEF", "valid": true, "value": 1, "event_date": "2023-01-01T00:00:00Z", "_ab_source_file_last_modified": "2023-10-12T15:27:30.000000Z", "_ab_source_file_url": "simple_test.jsonl"}, "emitted_at": 1697137760683} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/jsonl_newlines.jsonl b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/jsonl_newlines.jsonl new file mode 100644 index 000000000000..502cd837cc66 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/jsonl_newlines.jsonl @@ -0,0 +1,2 @@ +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false, "value": 1.2, "event_date": "2022-01-01T00:00:00Z", "_ab_source_file_last_modified": "2023-10-12T15:27:32.000000Z", "_ab_source_file_url": "simple_test_newlines.jsonl"}, "emitted_at": 1697137820278} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"id": 2, "name": "ABCDEF", "valid": true, "value": 1, "event_date": "2023-01-01T00:00:00Z", "_ab_source_file_last_modified": "2023-10-12T15:27:32.000000Z", "_ab_source_file_url": "simple_test_newlines.jsonl"}, "emitted_at": 1697137820280} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/parquet.jsonl b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/parquet.jsonl new file mode 100644 index 000000000000..75df4b4a4752 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/parquet.jsonl @@ -0,0 +1,15 @@ +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"Payroll_Number": 820.0, "Last_Name": "SCHWARTZ", "First_Name": "CHANA", "Mid_Init": "H", "Agency_Start_Date": "07/05/2010", "Work_Location_Borough": null, "Title_Description": "*ATTORNEY AT LAW", "Base_Salary": 77015.0, "Regular_Hours": 1046.25, "Regular_Gross_Paid": 47316.74, "OT_Hours": 0.0, "Total_OT_Paid": 0.0, "Total_Other_Pay": 8230.31, "Fiscal_Year": "2021", "Leave_Status_as_of_June_30": "ON LEAVE", "Pay_Basis": "per Annum", "_ab_source_file_last_modified": "2023-10-12T15:28:44.000000Z", "_ab_source_file_url": "test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"}, "emitted_at": 1697190604125} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"Payroll_Number": 820.0, "Last_Name": "WASHINGTON", "First_Name": "DOROTHY", "Mid_Init": null, "Agency_Start_Date": "07/05/2010", "Work_Location_Borough": null, "Title_Description": "ADM MANAGER-NON-MGRL FROM M1/M2", "Base_Salary": 53373.0, "Regular_Hours": 1825.0, "Regular_Gross_Paid": 47436.44, "OT_Hours": 0.0, "Total_OT_Paid": 0.0, "Total_Other_Pay": 1723.17, "Fiscal_Year": "2021", "Leave_Status_as_of_June_30": "ON LEAVE", "Pay_Basis": "per Annum", "_ab_source_file_last_modified": "2023-10-12T15:28:44.000000Z", "_ab_source_file_url": "test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"}, "emitted_at": 1697190604128} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"Payroll_Number": 820.0, "Last_Name": "DU", "First_Name": "MARK", "Mid_Init": null, "Agency_Start_Date": "03/24/2014", "Work_Location_Borough": null, "Title_Description": "HEARING OFFICER", "Base_Salary": 36.6, "Regular_Hours": 188.75, "Regular_Gross_Paid": 5334.45, "OT_Hours": 0.0, "Total_OT_Paid": 0.0, "Total_Other_Pay": 0.0, "Fiscal_Year": "2021", "Leave_Status_as_of_June_30": "ACTIVE", "Pay_Basis": "per Hour", "_ab_source_file_last_modified": "2023-10-12T15:28:45.000000Z", "_ab_source_file_url": "test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Hour/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"}, "emitted_at": 1697190608928} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"Payroll_Number": 820.0, "Last_Name": "BIEBEL", "First_Name": "ANN", "Mid_Init": "M", "Agency_Start_Date": "07/05/2010", "Work_Location_Borough": null, "Title_Description": "*ATTORNEY AT LAW", "Base_Salary": 77015.0, "Regular_Hours": 1825.0, "Regular_Gross_Paid": 76804.0, "OT_Hours": 0.0, "Total_OT_Paid": 0.0, "Total_Other_Pay": 13750.36, "Fiscal_Year": "2021", "Leave_Status_as_of_June_30": "ACTIVE", "Pay_Basis": "per Annum", "_ab_source_file_last_modified": "2023-10-12T15:28:46.000000Z", "_ab_source_file_url": "test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"}, "emitted_at": 1697190612045} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"Payroll_Number": 820.0, "Last_Name": "CARROLL", "First_Name": "FRAN", "Mid_Init": null, "Agency_Start_Date": "07/05/2010", "Work_Location_Borough": null, "Title_Description": "*ATTORNEY AT LAW", "Base_Salary": 77015.0, "Regular_Hours": 1825.0, "Regular_Gross_Paid": 76804.0, "OT_Hours": 0.0, "Total_OT_Paid": 0.0, "Total_Other_Pay": 13750.36, "Fiscal_Year": "2021", "Leave_Status_as_of_June_30": "ACTIVE", "Pay_Basis": "per Annum", "_ab_source_file_last_modified": "2023-10-12T15:28:46.000000Z", "_ab_source_file_url": "test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"}, "emitted_at": 1697190612046} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"Payroll_Number": 820.0, "Last_Name": "BROWNSTEIN", "First_Name": "ELFREDA", "Mid_Init": "G", "Agency_Start_Date": "07/05/2010", "Work_Location_Borough": null, "Title_Description": "*ATTORNEY AT LAW", "Base_Salary": 83504.0, "Regular_Hours": 1825.0, "Regular_Gross_Paid": 83275.15, "OT_Hours": 0.0, "Total_OT_Paid": 0.0, "Total_Other_Pay": 13750.36, "Fiscal_Year": "2021", "Leave_Status_as_of_June_30": "ACTIVE", "Pay_Basis": "per Annum", "_ab_source_file_last_modified": "2023-10-12T15:28:46.000000Z", "_ab_source_file_url": "test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"}, "emitted_at": 1697190612048} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"Payroll_Number": 820.0, "Last_Name": "WARD", "First_Name": "RENEE", "Mid_Init": null, "Agency_Start_Date": "07/05/2010", "Work_Location_Borough": null, "Title_Description": "ADM MANAGER-NON-MGRL FROM M1/M2", "Base_Salary": 53373.0, "Regular_Hours": 1825.0, "Regular_Gross_Paid": 46588.76, "OT_Hours": 0.0, "Total_OT_Paid": 0.0, "Total_Other_Pay": 3409.69, "Fiscal_Year": "2021", "Leave_Status_as_of_June_30": "ACTIVE", "Pay_Basis": "per Annum", "_ab_source_file_last_modified": "2023-10-12T15:28:46.000000Z", "_ab_source_file_url": "test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"}, "emitted_at": 1697190612050} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"Payroll_Number": 820.0, "Last_Name": "SPIVEY", "First_Name": "NATASHA", "Mid_Init": "L", "Agency_Start_Date": "07/05/2010", "Work_Location_Borough": null, "Title_Description": "ADM MANAGER-NON-MGRL FROM M1/M2", "Base_Salary": 53436.0, "Regular_Hours": 1825.0, "Regular_Gross_Paid": 53289.6, "OT_Hours": 0.0, "Total_OT_Paid": 0.0, "Total_Other_Pay": 0.0, "Fiscal_Year": "2021", "Leave_Status_as_of_June_30": "ACTIVE", "Pay_Basis": "per Annum", "_ab_source_file_last_modified": "2023-10-12T15:28:46.000000Z", "_ab_source_file_url": "test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"}, "emitted_at": 1697190612052} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"Payroll_Number": 820.0, "Last_Name": "SAMUEL", "First_Name": "GRACE", "Mid_Init": "Y", "Agency_Start_Date": "07/05/2010", "Work_Location_Borough": null, "Title_Description": "ADM MANAGER-NON-MGRL FROM M1/M2", "Base_Salary": 55337.0, "Regular_Hours": 1825.0, "Regular_Gross_Paid": 55185.52, "OT_Hours": 0.0, "Total_OT_Paid": 0.0, "Total_Other_Pay": 0.0, "Fiscal_Year": "2022", "Leave_Status_as_of_June_30": "ON LEAVE", "Pay_Basis": "per Annum", "_ab_source_file_last_modified": "2023-10-12T15:28:48.000000Z", "_ab_source_file_url": "test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"}, "emitted_at": 1697190614835} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"Payroll_Number": 820.0, "Last_Name": "THEIL", "First_Name": "JOANNE", "Mid_Init": "F", "Agency_Start_Date": "07/05/2010", "Work_Location_Borough": null, "Title_Description": "*ATTORNEY AT LAW", "Base_Salary": 80438.0, "Regular_Hours": 1825.0, "Regular_Gross_Paid": 80217.55, "OT_Hours": 0.0, "Total_OT_Paid": 0.0, "Total_Other_Pay": 13635.42, "Fiscal_Year": "2022", "Leave_Status_as_of_June_30": "ACTIVE", "Pay_Basis": "per Annum", "_ab_source_file_last_modified": "2023-10-12T15:28:49.000000Z", "_ab_source_file_url": "test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"}, "emitted_at": 1697190617470} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"Payroll_Number": 820.0, "Last_Name": "DEMAIO", "First_Name": "DEIRDRE", "Mid_Init": null, "Agency_Start_Date": "07/05/2010", "Work_Location_Borough": null, "Title_Description": "ADM MANAGER-NON-MGRL FROM M1/M2", "Base_Salary": 53512.0, "Regular_Hours": 1780.0, "Regular_Gross_Paid": 48727.47, "OT_Hours": 0.0, "Total_OT_Paid": 0.0, "Total_Other_Pay": 3318.35, "Fiscal_Year": "2022", "Leave_Status_as_of_June_30": "ACTIVE", "Pay_Basis": "per Annum", "_ab_source_file_last_modified": "2023-10-12T15:28:49.000000Z", "_ab_source_file_url": "test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"}, "emitted_at": 1697190617472} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"Payroll_Number": 820.0, "Last_Name": "MCLAURIN TRAPP", "First_Name": "CELESTINE", "Mid_Init": "T", "Agency_Start_Date": "07/05/2010", "Work_Location_Borough": null, "Title_Description": "ADM MANAGER-NON-MGRL FROM M1/M2", "Base_Salary": 58951.0, "Regular_Hours": 1818.0, "Regular_Gross_Paid": 58563.27, "OT_Hours": 0.0, "Total_OT_Paid": 0.0, "Total_Other_Pay": 8.25, "Fiscal_Year": "2022", "Leave_Status_as_of_June_30": "ACTIVE", "Pay_Basis": "per Annum", "_ab_source_file_last_modified": "2023-10-12T15:28:49.000000Z", "_ab_source_file_url": "test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"}, "emitted_at": 1697190617472} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"Payroll_Number": 820.0, "Last_Name": "BUNDRANT", "First_Name": "TROY", "Mid_Init": null, "Agency_Start_Date": "07/05/2010", "Work_Location_Borough": null, "Title_Description": "ADM MANAGER-NON-MGRL FROM M1/M2", "Base_Salary": 64769.0, "Regular_Hours": 1825.0, "Regular_Gross_Paid": 61817.94, "OT_Hours": 62.0, "Total_OT_Paid": 2576.58, "Total_Other_Pay": 106.68, "Fiscal_Year": "2022", "Leave_Status_as_of_June_30": "ACTIVE", "Pay_Basis": "per Annum", "_ab_source_file_last_modified": "2023-10-12T15:28:49.000000Z", "_ab_source_file_url": "test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"}, "emitted_at": 1697190617473} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"Payroll_Number": 820.0, "Last_Name": "CHASE JONES", "First_Name": "DIANA", "Mid_Init": null, "Agency_Start_Date": "07/05/2010", "Work_Location_Borough": null, "Title_Description": "ADM MANAGER-NON-MGRL FROM M1/M2", "Base_Salary": 66000.0, "Regular_Hours": 1825.0, "Regular_Gross_Paid": 65819.25, "OT_Hours": 0.0, "Total_OT_Paid": 0.0, "Total_Other_Pay": 0.0, "Fiscal_Year": "2022", "Leave_Status_as_of_June_30": "ACTIVE", "Pay_Basis": "per Annum", "_ab_source_file_last_modified": "2023-10-12T15:28:49.000000Z", "_ab_source_file_url": "test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"}, "emitted_at": 1697190617473} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"Payroll_Number": 820.0, "Last_Name": "JORDAN", "First_Name": "REGINALD", "Mid_Init": null, "Agency_Start_Date": "07/05/2010", "Work_Location_Borough": null, "Title_Description": "ADM MANAGER-NON-MGRL FROM M1/M2", "Base_Salary": 75000.0, "Regular_Hours": 1825.0, "Regular_Gross_Paid": 74794.46, "OT_Hours": 0.0, "Total_OT_Paid": 0.0, "Total_Other_Pay": 0.0, "Fiscal_Year": "2022", "Leave_Status_as_of_June_30": "ACTIVE", "Pay_Basis": "per Annum", "_ab_source_file_last_modified": "2023-10-12T15:28:49.000000Z", "_ab_source_file_url": "test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"}, "emitted_at": 1697190617474} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/unstructured.jsonl b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/unstructured.jsonl new file mode 100644 index 000000000000..8ac3010fd350 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/expected_records/unstructured.jsonl @@ -0,0 +1,2 @@ +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"content": "# Heading\n\nThis is the content which is not just a single word", "document_key": "Testdoc.pdf", "_ab_source_file_last_modified": "2023-10-30T11:38:48.000000Z", "_ab_source_file_url": "Testdoc.pdf", "_ab_source_file_parse_error": null}, "emitted_at": 1698666216334} +{"stream": "airbyte-source-azure-blob-storage-test", "data": {"content": "This is a test", "document_key": "Testdoc_OCR.pdf", "_ab_source_file_last_modified": "2023-10-30T11:38:48.000000Z", "_ab_source_file_url": "Testdoc_OCR.pdf", "_ab_source_file_parse_error": null}, "emitted_at": 1698666218048} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/spec.json b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/spec.json new file mode 100644 index 000000000000..81b04111ee81 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/integration_tests/spec.json @@ -0,0 +1,387 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/azure-blob-storage", + "connectionSpecification": { + "title": "Config", + "description": "NOTE: When this Spec is changed, legacy_config_transformer.py must also be modified to uptake the changes\nbecause it is responsible for converting legacy Azure Blob Storage v0 configs into v1 configs using the File-Based CDK.", + "type": "object", + "properties": { + "start_date": { + "title": "Start Date", + "description": "UTC date and time in the format 2017-01-25T00:00:00.000000Z. Any file modified before this date will not be replicated.", + "examples": ["2021-01-01T00:00:00.000000Z"], + "format": "date-time", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{6}Z$", + "pattern_descriptor": "YYYY-MM-DDTHH:mm:ss.SSSSSSZ", + "order": 1, + "type": "string" + }, + "streams": { + "title": "The list of streams to sync", + "description": "Each instance of this configuration defines a stream. Use this to define which files belong in the stream, their format, and how they should be parsed and validated. When sending data to warehouse destination such as Snowflake or BigQuery, each stream is a separate table.", + "order": 10, + "type": "array", + "items": { + "title": "FileBasedStreamConfig", + "type": "object", + "properties": { + "name": { + "title": "Name", + "description": "The name of the stream.", + "type": "string" + }, + "globs": { + "title": "Globs", + "default": ["**"], + "order": 1, + "description": "The pattern used to specify which files should be selected from the file system. For more information on glob pattern matching look here.", + "type": "array", + "items": { + "type": "string" + } + }, + "legacy_prefix": { + "title": "Legacy Prefix", + "description": "The path prefix configured in v3 versions of the S3 connector. This option is deprecated in favor of a single glob.", + "airbyte_hidden": true, + "type": "string" + }, + "validation_policy": { + "title": "Validation Policy", + "description": "The name of the validation policy that dictates sync behavior when a record does not adhere to the stream schema.", + "default": "Emit Record", + "enum": ["Emit Record", "Skip Record", "Wait for Discover"] + }, + "input_schema": { + "title": "Input Schema", + "description": "The schema that will be used to validate records extracted from the file. This will override the stream schema that is auto-detected from incoming files.", + "type": "string" + }, + "primary_key": { + "title": "Primary Key", + "description": "The column or columns (for a composite key) that serves as the unique identifier of a record. If empty, the primary key will default to the parser's default primary key.", + "type": "string", + "airbyte_hidden": true + }, + "days_to_sync_if_history_is_full": { + "title": "Days To Sync If History Is Full", + "description": "When the state history of the file store is full, syncs will only read files that were last modified in the provided day range.", + "default": 3, + "type": "integer" + }, + "format": { + "title": "Format", + "description": "The configuration options that are used to alter how to read incoming files that deviate from the standard formatting.", + "type": "object", + "oneOf": [ + { + "title": "Avro Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "avro", + "const": "avro", + "type": "string" + }, + "double_as_string": { + "title": "Convert Double Fields to Strings", + "description": "Whether to convert double fields to strings. This is recommended if you have decimal numbers with a high degree of precision because there can be a loss precision when handling floating point numbers.", + "default": false, + "type": "boolean" + } + }, + "required": ["filetype"] + }, + { + "title": "CSV Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "csv", + "const": "csv", + "type": "string" + }, + "delimiter": { + "title": "Delimiter", + "description": "The character delimiting individual cells in the CSV data. This may only be a 1-character string. For tab-delimited data enter '\\t'.", + "default": ",", + "type": "string" + }, + "quote_char": { + "title": "Quote Character", + "description": "The character used for quoting CSV values. To disallow quoting, make this field blank.", + "default": "\"", + "type": "string" + }, + "escape_char": { + "title": "Escape Character", + "description": "The character used for escaping special characters. To disallow escaping, leave this field blank.", + "type": "string" + }, + "encoding": { + "title": "Encoding", + "description": "The character encoding of the CSV data. Leave blank to default to UTF8. See list of python encodings for allowable options.", + "default": "utf8", + "type": "string" + }, + "double_quote": { + "title": "Double Quote", + "description": "Whether two quotes in a quoted CSV value denote a single quote in the data.", + "default": true, + "type": "boolean" + }, + "null_values": { + "title": "Null Values", + "description": "A set of case-sensitive strings that should be interpreted as null values. For example, if the value 'NA' should be interpreted as null, enter 'NA' in this field.", + "default": [], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "strings_can_be_null": { + "title": "Strings Can Be Null", + "description": "Whether strings can be interpreted as null values. If true, strings that match the null_values set will be interpreted as null. If false, strings that match the null_values set will be interpreted as the string itself.", + "default": true, + "type": "boolean" + }, + "skip_rows_before_header": { + "title": "Skip Rows Before Header", + "description": "The number of rows to skip before the header row. For example, if the header row is on the 3rd row, enter 2 in this field.", + "default": 0, + "type": "integer" + }, + "skip_rows_after_header": { + "title": "Skip Rows After Header", + "description": "The number of rows to skip after the header row.", + "default": 0, + "type": "integer" + }, + "header_definition": { + "title": "CSV Header Definition", + "description": "How headers will be defined. `User Provided` assumes the CSV does not have a header row and uses the headers provided and `Autogenerated` assumes the CSV does not have a header row and the CDK will generate headers using for `f{i}` where `i` is the index starting from 0. Else, the default behavior is to use the header from the CSV file. If a user wants to autogenerate or provide column names for a CSV having headers, they can skip rows.", + "default": { + "header_definition_type": "From CSV" + }, + "oneOf": [ + { + "title": "From CSV", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "From CSV", + "const": "From CSV", + "type": "string" + } + }, + "required": ["header_definition_type"] + }, + { + "title": "Autogenerated", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "Autogenerated", + "const": "Autogenerated", + "type": "string" + } + }, + "required": ["header_definition_type"] + }, + { + "title": "User Provided", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "User Provided", + "const": "User Provided", + "type": "string" + }, + "column_names": { + "title": "Column Names", + "description": "The column names that will be used while emitting the CSV records", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": ["column_names", "header_definition_type"] + } + ], + "type": "object" + }, + "true_values": { + "title": "True Values", + "description": "A set of case-sensitive strings that should be interpreted as true values.", + "default": ["y", "yes", "t", "true", "on", "1"], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "false_values": { + "title": "False Values", + "description": "A set of case-sensitive strings that should be interpreted as false values.", + "default": ["n", "no", "f", "false", "off", "0"], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "inference_type": { + "title": "Inference Type", + "description": "How to infer the types of the columns. If none, inference default to strings.", + "default": "None", + "airbyte_hidden": true, + "enum": ["None", "Primitive Types Only"] + } + }, + "required": ["filetype"] + }, + { + "title": "Jsonl Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "jsonl", + "const": "jsonl", + "type": "string" + } + }, + "required": ["filetype"] + }, + { + "title": "Parquet Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "parquet", + "const": "parquet", + "type": "string" + }, + "decimal_as_float": { + "title": "Convert Decimal Fields to Floats", + "description": "Whether to convert decimal fields to floats. There is a loss of precision when converting decimals to floats, so this is not recommended.", + "default": false, + "type": "boolean" + } + }, + "required": ["filetype"] + }, + { + "title": "Document File Type Format (Experimental)", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "unstructured", + "const": "unstructured", + "type": "string" + }, + "skip_unprocessable_files": { + "type": "boolean", + "default": true, + "title": "Skip Unprocessable Files", + "description": "If true, skip files that cannot be parsed and pass the error message along as the _ab_source_file_parse_error field. If false, fail the sync.", + "always_show": true + }, + "strategy": { + "type": "string", + "always_show": true, + "order": 0, + "default": "auto", + "title": "Parsing Strategy", + "enum": ["auto", "fast", "ocr_only", "hi_res"], + "description": "The strategy used to parse documents. `fast` extracts text directly from the document which doesn't work for all files. `ocr_only` is more reliable, but slower. `hi_res` is the most reliable, but requires an API key and a hosted instance of unstructured and can't be used with local mode. See the unstructured.io documentation for more details: https://unstructured-io.github.io/unstructured/core/partition.html#partition-pdf" + }, + "processing": { + "title": "Processing", + "description": "Processing configuration", + "default": { + "mode": "local" + }, + "type": "object", + "oneOf": [ + { + "title": "Local", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "local", + "const": "local", + "enum": ["local"], + "type": "string" + } + }, + "description": "Process files locally, supporting `fast` and `ocr` modes. This is the default option.", + "required": ["mode"] + } + ] + } + }, + "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file.", + "required": ["filetype"] + } + ] + }, + "schemaless": { + "title": "Schemaless", + "description": "When enabled, syncs will not validate or structure records against the stream's schema.", + "default": false, + "type": "boolean" + } + }, + "required": ["name", "format"] + } + }, + "azure_blob_storage_account_name": { + "title": "Azure Blob Storage account name", + "description": "The account's name of the Azure Blob Storage.", + "examples": ["airbyte5storage"], + "order": 2, + "type": "string" + }, + "azure_blob_storage_account_key": { + "title": "Azure Blob Storage account key", + "description": "The Azure blob storage account key.", + "airbyte_secret": true, + "examples": [ + "Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd==" + ], + "order": 3, + "type": "string" + }, + "azure_blob_storage_container_name": { + "title": "Azure blob storage container (Bucket) Name", + "description": "The name of the Azure blob storage container.", + "examples": ["airbytetescontainername"], + "order": 4, + "type": "string" + }, + "azure_blob_storage_endpoint": { + "title": "Endpoint Domain Name", + "description": "This is Azure Blob Storage endpoint domain name. Leave default value (or leave it empty if run container from command line) to use Microsoft native from example.", + "examples": ["blob.core.windows.net"], + "order": 11, + "type": "string" + } + }, + "required": [ + "streams", + "azure_blob_storage_account_name", + "azure_blob_storage_account_key", + "azure_blob_storage_container_name" + ] + } +} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/main.py b/airbyte-integrations/connectors/source-azure-blob-storage/main.py new file mode 100644 index 000000000000..b3361a6556d7 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/main.py @@ -0,0 +1,33 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import sys +import traceback +from datetime import datetime + +from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch +from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type +from source_azure_blob_storage import Config, SourceAzureBlobStorage, SourceAzureBlobStorageStreamReader + +if __name__ == "__main__": + args = sys.argv[1:] + catalog_path = AirbyteEntrypoint.extract_catalog(args) + try: + source = SourceAzureBlobStorage(SourceAzureBlobStorageStreamReader(), Config, catalog_path) + except Exception: + print( + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.ERROR, + emitted_at=int(datetime.now().timestamp() * 1000), + error=AirbyteErrorTraceMessage( + message="Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance.", + stack_trace=traceback.format_exc(), + ), + ), + ).json() + ) + else: + launch(source, args) diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml b/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml index 395b78ee078c..839510e2e159 100644 --- a/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml +++ b/airbyte-integrations/connectors/source-azure-blob-storage/metadata.yaml @@ -1,9 +1,15 @@ data: + ab_internal: + ql: 100 + sl: 100 + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: file connectorType: source definitionId: fdaaba68-4875-4ed9-8fcd-4ae1e0a25093 - dockerImageTag: 0.1.0 + dockerImageTag: 0.3.1 dockerRepository: airbyte/source-azure-blob-storage + documentationUrl: https://docs.airbyte.com/integrations/sources/azure-blob-storage githubIssueLabel: source-azure-blob-storage icon: azureblobstorage.svg license: MIT @@ -14,11 +20,7 @@ data: oss: enabled: true releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/azure-blob-storage - tags: - - language:java - ab_internal: - sl: 100 - ql: 100 supportLevel: community + tags: + - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/requirements.txt b/airbyte-integrations/connectors/source-azure-blob-storage/requirements.txt new file mode 100644 index 000000000000..7b9114ed5867 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/requirements.txt @@ -0,0 +1,2 @@ +# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. +-e . diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/setup.py b/airbyte-integrations/connectors/source-azure-blob-storage/setup.py new file mode 100644 index 000000000000..4246eb43364c --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/setup.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk[file-based]>=0.57.7", + "smart_open[azure]", + "pytz", +] + +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.2"] + +setup( + name="source_azure_blob_storage", + description="Source implementation for Azure Blob Storage.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/__init__.py b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/__init__.py new file mode 100644 index 000000000000..5ec5c4024c72 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/__init__.py @@ -0,0 +1,10 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .config import Config +from .source import SourceAzureBlobStorage +from .stream_reader import SourceAzureBlobStorageStreamReader + +__all__ = ["SourceAzureBlobStorage", "SourceAzureBlobStorageStreamReader", "Config"] diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/config.py b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/config.py new file mode 100644 index 000000000000..9955603ba74b --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/config.py @@ -0,0 +1,60 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Dict, Optional + +import dpath.util +from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec +from pydantic import AnyUrl, Field + + +class Config(AbstractFileBasedSpec): + """ + NOTE: When this Spec is changed, legacy_config_transformer.py must also be modified to uptake the changes + because it is responsible for converting legacy Azure Blob Storage v0 configs into v1 configs using the File-Based CDK. + """ + + @classmethod + def documentation_url(cls) -> AnyUrl: + return AnyUrl("https://docs.airbyte.com/integrations/sources/azure-blob-storage", scheme="https") + + azure_blob_storage_account_name: str = Field( + title="Azure Blob Storage account name", + description="The account's name of the Azure Blob Storage.", + examples=["airbyte5storage"], + order=2, + ) + azure_blob_storage_account_key: str = Field( + title="Azure Blob Storage account key", + description="The Azure blob storage account key.", + airbyte_secret=True, + examples=["Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd=="], + order=3, + ) + azure_blob_storage_container_name: str = Field( + title="Azure blob storage container (Bucket) Name", + description="The name of the Azure blob storage container.", + examples=["airbytetescontainername"], + order=4, + ) + azure_blob_storage_endpoint: Optional[str] = Field( + title="Endpoint Domain Name", + description="This is Azure Blob Storage endpoint domain name. Leave default value (or leave it empty if run container from " + "command line) to use Microsoft native from example.", + examples=["blob.core.windows.net"], + order=11, + ) + + @classmethod + def schema(cls, *args: Any, **kwargs: Any) -> Dict[str, Any]: + """ + Generates the mapping comprised of the config fields + """ + schema = super().schema(*args, **kwargs) + + # Hide API processing option until https://github.com/airbytehq/airbyte-platform-internal/issues/10354 is fixed + processing_options = dpath.util.get(schema, "properties/streams/items/properties/format/oneOf/4/properties/processing/oneOf") + dpath.util.set(schema, "properties/streams/items/properties/format/oneOf/4/properties/processing/oneOf", processing_options[:1]) + + return schema diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/legacy_config_transformer.py b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/legacy_config_transformer.py new file mode 100644 index 000000000000..e3c316d3ec0d --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/legacy_config_transformer.py @@ -0,0 +1,31 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Mapping, MutableMapping + + +class LegacyConfigTransformer: + """ + Class that takes in Azure Blob Storage source configs in the legacy format and transforms them into + configs that can be used by the new Azure Blob Storage source built with the file-based CDK. + """ + + @classmethod + def convert(cls, legacy_config: Mapping) -> MutableMapping[str, Any]: + azure_blob_storage_blobs_prefix = legacy_config.get("azure_blob_storage_blobs_prefix", "") + + return { + "azure_blob_storage_endpoint": legacy_config.get("azure_blob_storage_endpoint", None), + "azure_blob_storage_account_name": legacy_config["azure_blob_storage_account_name"], + "azure_blob_storage_account_key": legacy_config["azure_blob_storage_account_key"], + "azure_blob_storage_container_name": legacy_config["azure_blob_storage_container_name"], + "streams": [ + { + "name": legacy_config["azure_blob_storage_container_name"], + "legacy_prefix": azure_blob_storage_blobs_prefix, + "validation_policy": "Emit Record", + "format": {"filetype": "jsonl"}, + } + ], + } diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/source.py b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/source.py new file mode 100644 index 000000000000..419119bb3ef8 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/source.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Mapping + +from airbyte_cdk.config_observation import emit_configuration_as_airbyte_control_message +from airbyte_cdk.sources.file_based.file_based_source import FileBasedSource + +from .legacy_config_transformer import LegacyConfigTransformer + + +class SourceAzureBlobStorage(FileBasedSource): + def read_config(self, config_path: str) -> Mapping[str, Any]: + """ + Used to override the default read_config so that when the new file-based Azure Blob Storage connector processes a config + in the legacy format, it can be transformed into the new config. This happens in entrypoint before we + validate the config against the new spec. + """ + config = super().read_config(config_path) + if not self._is_v1_config(config): + converted_config = LegacyConfigTransformer.convert(config) + emit_configuration_as_airbyte_control_message(converted_config) + return converted_config + return config + + @staticmethod + def _is_v1_config(config: Mapping[str, Any]) -> bool: + return "streams" in config diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py new file mode 100644 index 000000000000..c751b72403bd --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/source_azure_blob_storage/stream_reader.py @@ -0,0 +1,75 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import logging +from contextlib import contextmanager +from io import IOBase +from typing import Iterable, List, Optional + +import pytz +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from azure.storage.blob import BlobServiceClient, ContainerClient +from smart_open import open + +from .config import Config + + +class SourceAzureBlobStorageStreamReader(AbstractFileBasedStreamReader): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._config = None + + @property + def config(self) -> Config: + return self._config + + @config.setter + def config(self, value: Config) -> None: + self._config = value + + @property + def account_url(self) -> str: + if not self.config.azure_blob_storage_endpoint: + return f"https://{self.config.azure_blob_storage_account_name}.blob.core.windows.net" + return self.config.azure_blob_storage_endpoint + + @property + def azure_container_client(self): + return ContainerClient( + self.account_url, + container_name=self.config.azure_blob_storage_container_name, + credential=self.config.azure_blob_storage_account_key, + ) + + @property + def azure_blob_service_client(self): + return BlobServiceClient(self.account_url, credential=self.config.azure_blob_storage_account_key) + + def get_matching_files( + self, + globs: List[str], + prefix: Optional[str], + logger: logging.Logger, + ) -> Iterable[RemoteFile]: + prefixes = [prefix] if prefix else self.get_prefixes_from_globs(globs) + prefixes = prefixes or [None] + for prefix in prefixes: + for blob in self.azure_container_client.list_blobs(name_starts_with=prefix): + remote_file = RemoteFile(uri=blob.name, last_modified=blob.last_modified.astimezone(pytz.utc).replace(tzinfo=None)) + if not globs or self.file_matches_globs(remote_file, globs): + yield remote_file + + def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger) -> IOBase: + try: + result = open( + f"azure://{self.config.azure_blob_storage_container_name}/{file.uri}", + transport_params={"client": self.azure_blob_service_client}, + mode=mode.value, + encoding=encoding, + ) + except OSError: + logger.warning( + f"We don't have access to {file.uri}. The file appears to have become unreachable during sync." + f"Check whether key {file.uri} exists in `{self.config.azure_blob_storage_container_name}` container and/or has proper ACL permissions" + ) + return result diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/src/main/java/io/airbyte/integrations/source/azureblobstorage/AzureBlob.java b/airbyte-integrations/connectors/source-azure-blob-storage/src/main/java/io/airbyte/integrations/source/azureblobstorage/AzureBlob.java deleted file mode 100644 index 3ae6c38ad433..000000000000 --- a/airbyte-integrations/connectors/source-azure-blob-storage/src/main/java/io/airbyte/integrations/source/azureblobstorage/AzureBlob.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.azureblobstorage; - -import java.time.OffsetDateTime; - -public record AzureBlob( - - String name, - - OffsetDateTime lastModified - -) { - - public static class Builder { - - private String name; - - private OffsetDateTime lastModified; - - public Builder withName(String name) { - this.name = name; - return this; - } - - public Builder withLastModified(OffsetDateTime lastModified) { - this.lastModified = lastModified; - return this; - } - - public AzureBlob build() { - return new AzureBlob(name, lastModified); - } - - } - -} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/src/main/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobAdditionalProperties.java b/airbyte-integrations/connectors/source-azure-blob-storage/src/main/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobAdditionalProperties.java deleted file mode 100644 index 7d2514ab5810..000000000000 --- a/airbyte-integrations/connectors/source-azure-blob-storage/src/main/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobAdditionalProperties.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.azureblobstorage; - -public class AzureBlobAdditionalProperties { - - private AzureBlobAdditionalProperties() { - - } - - public static final String LAST_MODIFIED = "_ab_source_file_last_modified"; - - public static final String BLOB_NAME = "_ab_source_blob_name"; - - public static final String ADDITIONAL_PROPERTIES = "_ab_additional_properties"; - -} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/src/main/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageConfig.java b/airbyte-integrations/connectors/source-azure-blob-storage/src/main/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageConfig.java deleted file mode 100644 index bfa058620f38..000000000000 --- a/airbyte-integrations/connectors/source-azure-blob-storage/src/main/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageConfig.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.azureblobstorage; - -import com.azure.storage.blob.BlobContainerClient; -import com.azure.storage.blob.BlobContainerClientBuilder; -import com.azure.storage.common.StorageSharedKeyCredential; -import com.fasterxml.jackson.databind.JsonNode; - -public record AzureBlobStorageConfig( - - String endpoint, - - String accountName, - - String accountKey, - - String containerName, - - String prefix, - - Long schemaInferenceLimit, - - FormatConfig formatConfig - -) { - - public record FormatConfig(Format format) { - - public enum Format { - - JSONL - - } - - } - - public static AzureBlobStorageConfig createAzureBlobStorageConfig(JsonNode jsonNode) { - return new AzureBlobStorageConfig( - jsonNode.has("azure_blob_storage_endpoint") ? jsonNode.get("azure_blob_storage_endpoint").asText() : null, - jsonNode.get("azure_blob_storage_account_name").asText(), - jsonNode.get("azure_blob_storage_account_key").asText(), - jsonNode.get("azure_blob_storage_container_name").asText(), - jsonNode.has("azure_blob_storage_blobs_prefix") ? jsonNode.get("azure_blob_storage_blobs_prefix").asText() : null, - jsonNode.has("azure_blob_storage_schema_inference_limit") ? jsonNode.get("azure_blob_storage_schema_inference_limit").asLong() : null, - formatConfig(jsonNode)); - } - - public BlobContainerClient createBlobContainerClient() { - StorageSharedKeyCredential credential = new StorageSharedKeyCredential( - this.accountName(), - this.accountKey()); - - var builder = new BlobContainerClientBuilder() - .credential(credential) - .containerName(this.containerName()); - - if (this.endpoint() != null) { - builder.endpoint(this.endpoint()); - } - - return builder.buildClient(); - } - - private static FormatConfig formatConfig(JsonNode config) { - JsonNode formatConfig = config.get("format"); - - FormatConfig.Format formatType = FormatConfig.Format - .valueOf(formatConfig.get("format_type").asText().toUpperCase()); - - return new FormatConfig(formatType); - } - -} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/src/main/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageOperations.java b/airbyte-integrations/connectors/source-azure-blob-storage/src/main/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageOperations.java deleted file mode 100644 index 705440a597a7..000000000000 --- a/airbyte-integrations/connectors/source-azure-blob-storage/src/main/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageOperations.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.azureblobstorage; - -import com.azure.storage.blob.BlobContainerClient; -import com.azure.storage.blob.models.BlobListDetails; -import com.azure.storage.blob.models.ListBlobsOptions; -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.functional.CheckedFunction; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.time.OffsetDateTime; -import java.util.ArrayList; -import java.util.List; -import org.apache.commons.lang3.StringUtils; - -public abstract class AzureBlobStorageOperations { - - protected final BlobContainerClient blobContainerClient; - - protected final AzureBlobStorageConfig azureBlobStorageConfig; - - protected AzureBlobStorageOperations(AzureBlobStorageConfig azureBlobStorageConfig) { - this.azureBlobStorageConfig = azureBlobStorageConfig; - this.blobContainerClient = azureBlobStorageConfig.createBlobContainerClient(); - } - - public abstract JsonNode inferSchema(); - - public abstract List readBlobs(OffsetDateTime offsetDateTime); - - public List listBlobs() { - - var listBlobsOptions = new ListBlobsOptions(); - listBlobsOptions.setDetails(new BlobListDetails() - .setRetrieveMetadata(true) - .setRetrieveDeletedBlobs(false)); - - if (!StringUtils.isBlank(azureBlobStorageConfig.prefix())) { - listBlobsOptions.setPrefix(azureBlobStorageConfig.prefix()); - } - - var pagedIterable = blobContainerClient.listBlobs(listBlobsOptions, null); - - List azureBlobs = new ArrayList<>(); - pagedIterable.forEach(blobItem -> azureBlobs.add(new AzureBlob.Builder() - .withName(blobItem.getName()) - .withLastModified(blobItem.getProperties().getLastModified()) - .build())); - return azureBlobs; - - } - - protected R handleCheckedIOException(CheckedFunction checkedFunction, T parameter) { - try { - return checkedFunction.apply(parameter); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - -} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/src/main/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageSource.java b/airbyte-integrations/connectors/source-azure-blob-storage/src/main/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageSource.java deleted file mode 100644 index 0b6fb64d4673..000000000000 --- a/airbyte-integrations/connectors/source-azure-blob-storage/src/main/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageSource.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.azureblobstorage; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.features.FeatureFlags; -import io.airbyte.commons.stream.AirbyteStreamUtils; -import io.airbyte.commons.util.AutoCloseableIterator; -import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteTraceMessageUtility; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.source.azureblobstorage.format.JsonlAzureBlobStorageOperations; -import io.airbyte.integrations.source.relationaldb.CursorInfo; -import io.airbyte.integrations.source.relationaldb.StateDecoratingIterator; -import io.airbyte.integrations.source.relationaldb.state.StateManager; -import io.airbyte.integrations.source.relationaldb.state.StateManagerFactory; -import io.airbyte.protocol.models.JsonSchemaPrimitiveUtil; -import io.airbyte.protocol.models.v0.AirbyteCatalog; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import io.airbyte.protocol.models.v0.AirbyteStream; -import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.SyncMode; -import java.time.Instant; -import java.time.OffsetDateTime; -import java.util.List; -import java.util.Optional; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class AzureBlobStorageSource extends BaseConnector implements Source { - - private static final Logger LOGGER = LoggerFactory.getLogger(AzureBlobStorageSource.class); - - private final FeatureFlags featureFlags = new EnvVariableFeatureFlags(); - - public static void main(final String[] args) throws Exception { - final Source source = new AzureBlobStorageSource(); - LOGGER.info("starting Source: {}", AzureBlobStorageSource.class); - new IntegrationRunner(source).run(args); - LOGGER.info("completed Source: {}", AzureBlobStorageSource.class); - } - - @Override - public AirbyteConnectionStatus check(JsonNode config) { - var azureBlobStorageConfig = AzureBlobStorageConfig.createAzureBlobStorageConfig(config); - try { - var azureBlobStorageOperations = switch (azureBlobStorageConfig.formatConfig().format()) { - case JSONL -> new JsonlAzureBlobStorageOperations(azureBlobStorageConfig); - }; - azureBlobStorageOperations.listBlobs(); - - return new AirbyteConnectionStatus() - .withStatus(AirbyteConnectionStatus.Status.SUCCEEDED); - } catch (Exception e) { - LOGGER.error("Error while listing Azure Blob Storage blobs with reason: ", e); - return new AirbyteConnectionStatus() - .withStatus(AirbyteConnectionStatus.Status.FAILED); - } - - } - - @Override - public AirbyteCatalog discover(JsonNode config) { - var azureBlobStorageConfig = AzureBlobStorageConfig.createAzureBlobStorageConfig(config); - - var azureBlobStorageOperations = switch (azureBlobStorageConfig.formatConfig().format()) { - case JSONL -> new JsonlAzureBlobStorageOperations(azureBlobStorageConfig); - }; - - JsonNode schema = azureBlobStorageOperations.inferSchema(); - - return new AirbyteCatalog() - .withStreams(List.of(new AirbyteStream() - .withName(azureBlobStorageConfig.containerName()) - .withJsonSchema(schema) - .withSourceDefinedCursor(true) - .withDefaultCursorField(List.of(AzureBlobAdditionalProperties.LAST_MODIFIED)) - .withSupportedSyncModes(List.of(SyncMode.INCREMENTAL, SyncMode.FULL_REFRESH)))); - } - - @Override - public AutoCloseableIterator read(JsonNode config, ConfiguredAirbyteCatalog catalog, JsonNode state) { - - final var streamState = - AzureBlobStorageStateManager.deserializeStreamState(state, featureFlags.useStreamCapableState()); - - final StateManager stateManager = StateManagerFactory - .createStateManager(streamState.airbyteStateType(), streamState.airbyteStateMessages(), catalog); - - var azureBlobStorageConfig = AzureBlobStorageConfig.createAzureBlobStorageConfig(config); - var azureBlobStorageOperations = switch (azureBlobStorageConfig.formatConfig().format()) { - case JSONL -> new JsonlAzureBlobStorageOperations(azureBlobStorageConfig); - }; - - // only one stream per connection - var streamIterators = catalog.getStreams().stream() - .map(cas -> switch (cas.getSyncMode()) { - case INCREMENTAL -> readIncremental(azureBlobStorageOperations, cas.getStream(), cas.getCursorField().get(0), - stateManager); - case FULL_REFRESH -> readFullRefresh(azureBlobStorageOperations, cas.getStream()); - }) - .toList(); - - return AutoCloseableIterators.concatWithEagerClose(streamIterators, AirbyteTraceMessageUtility::emitStreamStatusTrace); - - } - - private AutoCloseableIterator readIncremental(AzureBlobStorageOperations azureBlobStorageOperations, - AirbyteStream airbyteStream, - String cursorField, - StateManager stateManager) { - var streamPair = new AirbyteStreamNameNamespacePair(airbyteStream.getName(), airbyteStream.getNamespace()); - - Optional cursorInfo = stateManager.getCursorInfo(streamPair); - - var messageStream = cursorInfo - .map(cursor -> { - var offsetDateTime = cursor.getCursor() != null ? OffsetDateTime.parse(cursor.getCursor()) : null; - return azureBlobStorageOperations.readBlobs(offsetDateTime); - }) - .orElse(azureBlobStorageOperations.readBlobs(null)) - .stream() - .map(jn -> new AirbyteMessage() - .withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage() - .withStream(airbyteStream.getName()) - .withEmittedAt(Instant.now().toEpochMilli()) - .withData(jn))); - - final io.airbyte.protocol.models.AirbyteStreamNameNamespacePair airbyteStreamNameNamespacePair = - AirbyteStreamUtils.convertFromAirbyteStream(airbyteStream); - - return AutoCloseableIterators.transform(autoCloseableIterator -> new StateDecoratingIterator( - autoCloseableIterator, - stateManager, - streamPair, - cursorField, - cursorInfo.map(CursorInfo::getCursor).orElse(null), - JsonSchemaPrimitiveUtil.JsonSchemaPrimitive.TIMESTAMP_WITH_TIMEZONE_V1, - // TODO (itaseski) emit state after every record/blob since they can be sorted in increasing order - 0), - AutoCloseableIterators.fromStream(messageStream, airbyteStreamNameNamespacePair), - airbyteStreamNameNamespacePair); - } - - private AutoCloseableIterator readFullRefresh(AzureBlobStorageOperations azureBlobStorageOperations, - AirbyteStream airbyteStream) { - - final io.airbyte.protocol.models.AirbyteStreamNameNamespacePair airbyteStreamNameNamespacePair = - AirbyteStreamUtils.convertFromAirbyteStream(airbyteStream); - - var messageStream = azureBlobStorageOperations - .readBlobs(null) - .stream() - .map(jn -> new AirbyteMessage() - .withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage() - .withStream(airbyteStream.getName()) - .withEmittedAt(Instant.now().toEpochMilli()) - .withData(jn))); - - return AutoCloseableIterators.fromStream(messageStream, airbyteStreamNameNamespacePair); - } - -} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/src/main/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageStateManager.java b/airbyte-integrations/connectors/source-azure-blob-storage/src/main/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageStateManager.java deleted file mode 100644 index 78abcd9b032a..000000000000 --- a/airbyte-integrations/connectors/source-azure-blob-storage/src/main/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageStateManager.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.azureblobstorage; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.configoss.StateWrapper; -import io.airbyte.configoss.helpers.StateMessageHelper; -import io.airbyte.integrations.source.relationaldb.models.DbState; -import io.airbyte.protocol.models.v0.AirbyteStateMessage; -import io.airbyte.protocol.models.v0.AirbyteStreamState; -import java.util.List; -import java.util.Optional; - -public class AzureBlobStorageStateManager { - - private AzureBlobStorageStateManager() { - - } - - public static StreamState deserializeStreamState(final JsonNode state, final boolean useStreamCapableState) { - final Optional typedState = - StateMessageHelper.getTypedState(state, useStreamCapableState); - return typedState.map(stateWrapper -> switch (stateWrapper.getStateType()) { - case STREAM: - yield new StreamState(AirbyteStateMessage.AirbyteStateType.STREAM, stateWrapper.getStateMessages().stream() - .map(sm -> Jsons.object(Jsons.jsonNode(sm), AirbyteStateMessage.class)).toList()); - case LEGACY: - yield new StreamState(AirbyteStateMessage.AirbyteStateType.LEGACY, List.of( - new AirbyteStateMessage().withType(AirbyteStateMessage.AirbyteStateType.LEGACY) - .withData(stateWrapper.getLegacyState()))); - case GLOBAL: - throw new UnsupportedOperationException("Unsupported stream state"); - }).orElseGet(() -> { - // create empty initial state - if (useStreamCapableState) { - return new StreamState(AirbyteStateMessage.AirbyteStateType.STREAM, List.of( - new AirbyteStateMessage().withType(AirbyteStateMessage.AirbyteStateType.STREAM) - .withStream(new AirbyteStreamState()))); - } else { - // TODO (itaseski) remove support for DbState - return new StreamState(AirbyteStateMessage.AirbyteStateType.LEGACY, List.of( - new AirbyteStateMessage().withType(AirbyteStateMessage.AirbyteStateType.LEGACY) - .withData(Jsons.jsonNode(new DbState())))); - } - }); - } - - record StreamState( - - AirbyteStateMessage.AirbyteStateType airbyteStateType, - - List airbyteStateMessages) { - - } - -} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/src/main/java/io/airbyte/integrations/source/azureblobstorage/format/JsonlAzureBlobStorageOperations.java b/airbyte-integrations/connectors/source-azure-blob-storage/src/main/java/io/airbyte/integrations/source/azureblobstorage/format/JsonlAzureBlobStorageOperations.java deleted file mode 100644 index fcfe9b786b08..000000000000 --- a/airbyte-integrations/connectors/source-azure-blob-storage/src/main/java/io/airbyte/integrations/source/azureblobstorage/format/JsonlAzureBlobStorageOperations.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.azureblobstorage.format; - -import com.azure.storage.blob.BlobClient; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.saasquatch.jsonschemainferrer.AdditionalPropertiesPolicies; -import com.saasquatch.jsonschemainferrer.JsonSchemaInferrer; -import com.saasquatch.jsonschemainferrer.SpecVersion; -import io.airbyte.integrations.source.azureblobstorage.AzureBlob; -import io.airbyte.integrations.source.azureblobstorage.AzureBlobAdditionalProperties; -import io.airbyte.integrations.source.azureblobstorage.AzureBlobStorageConfig; -import io.airbyte.integrations.source.azureblobstorage.AzureBlobStorageOperations; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.UncheckedIOException; -import java.nio.charset.Charset; -import java.time.OffsetDateTime; -import java.util.List; -import java.util.Map; - -public class JsonlAzureBlobStorageOperations extends AzureBlobStorageOperations { - - private final ObjectMapper objectMapper; - - private final JsonSchemaInferrer jsonSchemaInferrer; - - public JsonlAzureBlobStorageOperations(AzureBlobStorageConfig azureBlobStorageConfig) { - super(azureBlobStorageConfig); - this.objectMapper = new ObjectMapper(); - this.jsonSchemaInferrer = JsonSchemaInferrer.newBuilder() - .setSpecVersion(SpecVersion.DRAFT_07) - .setAdditionalPropertiesPolicy(AdditionalPropertiesPolicies.allowed()) - .build(); - } - - @Override - public JsonNode inferSchema() { - var blobs = readBlobs(null, azureBlobStorageConfig.schemaInferenceLimit()); - - // create super schema inferred from all blobs in the container - var jsonSchema = jsonSchemaInferrer.inferForSamples(blobs); - - if (!jsonSchema.has("properties")) { - jsonSchema.putObject("properties"); - } - - ((ObjectNode) jsonSchema.get("properties")).putPOJO(AzureBlobAdditionalProperties.BLOB_NAME, - Map.of("type", "string")); - ((ObjectNode) jsonSchema.get("properties")).putPOJO(AzureBlobAdditionalProperties.LAST_MODIFIED, - Map.of("type", "string")); - return jsonSchema; - } - - @Override - public List readBlobs(OffsetDateTime offsetDateTime) { - return readBlobs(offsetDateTime, null); - } - - private List readBlobs(OffsetDateTime offsetDateTime, Long limit) { - record DecoratedAzureBlob(AzureBlob azureBlob, BlobClient blobClient) {} - - var blobsStream = limit == null ? listBlobs().stream() : listBlobs().stream().limit(limit); - - return blobsStream - .filter(ab -> { - if (offsetDateTime != null) { - return ab.lastModified().isAfter(offsetDateTime); - } else { - return true; - } - }) - .map(ab -> new DecoratedAzureBlob(ab, blobContainerClient.getBlobClient(ab.name()))) - .map(dab -> { - try ( - var br = new BufferedReader( - new InputStreamReader(dab.blobClient().downloadContent().toStream(), Charset.defaultCharset()))) { - return br.lines().map(line -> { - var jsonNode = - handleCheckedIOException(objectMapper::readTree, line); - ((ObjectNode) jsonNode).put(AzureBlobAdditionalProperties.BLOB_NAME, dab.azureBlob().name()); - ((ObjectNode) jsonNode).put(AzureBlobAdditionalProperties.LAST_MODIFIED, - dab.azureBlob().lastModified().toString()); - return jsonNode; - }) - // need to materialize stream otherwise reader gets closed on return - .toList(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }) - .flatMap(List::stream) - .toList(); - } - -} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/src/main/resources/spec.json b/airbyte-integrations/connectors/source-azure-blob-storage/src/main/resources/spec.json deleted file mode 100644 index 9e4d74450a36..000000000000 --- a/airbyte-integrations/connectors/source-azure-blob-storage/src/main/resources/spec.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/destinations/azureblobstorage", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "AzureBlobStorage Source Spec", - "type": "object", - "required": [ - "azure_blob_storage_account_name", - "azure_blob_storage_account_key", - "azure_blob_storage_container_name", - "format" - ], - "additionalProperties": true, - "properties": { - "azure_blob_storage_endpoint": { - "title": "Endpoint Domain Name", - "type": "string", - "default": "blob.core.windows.net", - "description": "This is Azure Blob Storage endpoint domain name. Leave default value (or leave it empty if run container from command line) to use Microsoft native from example.", - "examples": ["blob.core.windows.net"] - }, - "azure_blob_storage_container_name": { - "title": "Azure blob storage container (Bucket) Name", - "type": "string", - "description": "The name of the Azure blob storage container.", - "examples": ["airbytetescontainername"] - }, - "azure_blob_storage_account_name": { - "title": "Azure Blob Storage account name", - "type": "string", - "description": "The account's name of the Azure Blob Storage.", - "examples": ["airbyte5storage"] - }, - "azure_blob_storage_account_key": { - "title": "Azure Blob Storage account key", - "description": "The Azure blob storage account key.", - "airbyte_secret": true, - "type": "string", - "examples": [ - "Z8ZkZpteggFx394vm+PJHnGTvdRncaYS+JhLKdj789YNmD+iyGTnG+PV+POiuYNhBg/ACS+LKjd%4FG3FHGN12Nd==" - ] - }, - "azure_blob_storage_blobs_prefix": { - "title": "Azure Blob Storage blobs prefix", - "description": "The Azure blob storage prefix to be applied", - "type": "string", - "examples": ["FolderA/FolderB/"] - }, - "azure_blob_storage_schema_inference_limit": { - "title": "Azure Blob Storage schema inference limit", - "description": "The Azure blob storage blobs to scan for inferring the schema, useful on large amounts of data with consistent structure", - "type": "integer", - "examples": ["500"] - }, - "format": { - "title": "Input Format", - "type": "object", - "description": "Input data format", - "oneOf": [ - { - "title": "JSON Lines: newline-delimited JSON", - "required": ["format_type"], - "properties": { - "format_type": { - "type": "string", - "const": "JSONL" - } - } - } - ] - } - } - } -} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageContainer.java b/airbyte-integrations/connectors/source-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageContainer.java deleted file mode 100644 index 3acb639e2b8e..000000000000 --- a/airbyte-integrations/connectors/source-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageContainer.java +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.azureblobstorage; - -import org.testcontainers.containers.GenericContainer; - -// Azurite emulator for easier local azure storage development and testing -// https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=docker-hub -public class AzureBlobStorageContainer extends GenericContainer { - - public AzureBlobStorageContainer() { - super("mcr.microsoft.com/azure-storage/azurite"); - } - -} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageDataFactory.java b/airbyte-integrations/connectors/source-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageDataFactory.java deleted file mode 100644 index fd9126a32ad8..000000000000 --- a/airbyte-integrations/connectors/source-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageDataFactory.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.azureblobstorage; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.SyncMode; -import java.util.List; -import java.util.Map; - -public class AzureBlobStorageDataFactory { - - private AzureBlobStorageDataFactory() { - - } - - static JsonNode createAzureBlobStorageConfig(String host, String container) { - return Jsons.jsonNode(Map.of( - "azure_blob_storage_endpoint", host + "/devstoreaccount1", - "azure_blob_storage_account_name", "devstoreaccount1", - "azure_blob_storage_account_key", - "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", - "azure_blob_storage_container_name", container, - "azure_blob_storage_blobs_prefix", "FolderA/FolderB/", - "azure_blob_storage_schema_inference_limit", 10L, - "format", Jsons.deserialize(""" - { - "format_type": "JSONL" - }"""))); - } - - static ConfiguredAirbyteCatalog createConfiguredAirbyteCatalog(String streamName) { - return new ConfiguredAirbyteCatalog().withStreams(List.of( - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(List.of(AzureBlobAdditionalProperties.LAST_MODIFIED)) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withStream(CatalogHelpers.createAirbyteStream( - streamName, - Field.of("attr_1", JsonSchemaType.STRING), - Field.of("attr_2", JsonSchemaType.INTEGER), - Field.of(AzureBlobAdditionalProperties.LAST_MODIFIED, JsonSchemaType.TIMESTAMP_WITH_TIMEZONE_V1), - Field.of(AzureBlobAdditionalProperties.BLOB_NAME, JsonSchemaType.STRING)) - .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); - } - -} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageSourceAcceptanceTest.java deleted file mode 100644 index a892b684e61d..000000000000 --- a/airbyte-integrations/connectors/source-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageSourceAcceptanceTest.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.azureblobstorage; - -import com.azure.core.util.BinaryData; -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import java.util.HashMap; - -public class AzureBlobStorageSourceAcceptanceTest extends SourceAcceptanceTest { - - private static final String STREAM_NAME = "airbyte-container"; - - private AzureBlobStorageContainer azureBlobStorageContainer; - - private JsonNode jsonConfig; - - @Override - protected String getImageName() { - return "airbyte/source-azure-blob-storage:dev"; - } - - @Override - protected JsonNode getConfig() throws Exception { - return jsonConfig; - } - - @Override - protected void setupEnvironment(TestDestinationEnv environment) { - azureBlobStorageContainer = new AzureBlobStorageContainer().withExposedPorts(10000); - azureBlobStorageContainer.start(); - jsonConfig = AzureBlobStorageDataFactory.createAzureBlobStorageConfig( - "http://127.0.0.1:" + azureBlobStorageContainer.getMappedPort(10000), STREAM_NAME); - - var azureBlobStorageConfig = AzureBlobStorageConfig.createAzureBlobStorageConfig(jsonConfig); - var blobContainerClient = azureBlobStorageConfig.createBlobContainerClient(); - blobContainerClient.createIfNotExists(); - blobContainerClient.getBlobClient("FolderA/FolderB/blob1.json").upload(BinaryData.fromString("{\"attr1\":\"str_1\",\"attr2\":1}\n")); - } - - @Override - protected void tearDown(TestDestinationEnv testEnv) { - azureBlobStorageContainer.stop(); - azureBlobStorageContainer.close(); - } - - @Override - protected ConnectorSpecification getSpec() throws Exception { - return Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class); - } - - @Override - protected ConfiguredAirbyteCatalog getConfiguredCatalog() { - return AzureBlobStorageDataFactory.createConfiguredAirbyteCatalog(STREAM_NAME); - } - - @Override - protected JsonNode getState() { - return Jsons.jsonNode(new HashMap<>()); - } - -} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageSourceTest.java b/airbyte-integrations/connectors/source-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageSourceTest.java deleted file mode 100644 index 873c3f7e12a2..000000000000 --- a/airbyte-integrations/connectors/source-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageSourceTest.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.azureblobstorage; - -import static org.assertj.core.api.Assertions.assertThat; - -import com.azure.core.util.BinaryData; -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.SyncMode; -import java.util.Iterator; -import java.util.List; -import java.util.stream.Stream; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class AzureBlobStorageSourceTest { - - private AzureBlobStorageSource azureBlobStorageSource; - - private AzureBlobStorageContainer azureBlobStorageContainer; - - private JsonNode jsonConfig; - - private static final String STREAM_NAME = "airbyte-container"; - - @BeforeEach - void setup() { - azureBlobStorageContainer = new AzureBlobStorageContainer().withExposedPorts(10000); - azureBlobStorageContainer.start(); - azureBlobStorageSource = new AzureBlobStorageSource(); - jsonConfig = AzureBlobStorageDataFactory.createAzureBlobStorageConfig( - "http://127.0.0.1:" + azureBlobStorageContainer.getMappedPort(10000), STREAM_NAME); - - var azureBlobStorageConfig = AzureBlobStorageConfig.createAzureBlobStorageConfig(jsonConfig); - var blobContainerClient = azureBlobStorageConfig.createBlobContainerClient(); - blobContainerClient.createIfNotExists(); - blobContainerClient.getBlobClient("FolderA/FolderB/blob1.json") - .upload(BinaryData.fromString("{\"attr_1\":\"str_1\"}\n")); - blobContainerClient.getBlobClient("FolderA/FolderB/blob2.json") - .upload(BinaryData.fromString("{\"attr_2\":\"str_2\"}\n")); - // blob in ignored path - blobContainerClient.getBlobClient("FolderA/blob3.json").upload(BinaryData.fromString("{}")); - } - - @AfterEach - void tearDown() { - azureBlobStorageContainer.stop(); - azureBlobStorageContainer.close(); - } - - @Test - void testCheckConnectionWithSucceeded() { - var airbyteConnectionStatus = azureBlobStorageSource.check(jsonConfig); - - assertThat(airbyteConnectionStatus.getStatus()).isEqualTo(AirbyteConnectionStatus.Status.SUCCEEDED); - - } - - @Test - void testCheckConnectionWithFailed() { - - var failingConfig = AzureBlobStorageDataFactory.createAzureBlobStorageConfig( - "http://127.0.0.1:" + azureBlobStorageContainer.getMappedPort(10000), "missing-container"); - - var airbyteConnectionStatus = azureBlobStorageSource.check(failingConfig); - - assertThat(airbyteConnectionStatus.getStatus()).isEqualTo(AirbyteConnectionStatus.Status.FAILED); - - } - - @Test - void testDiscover() { - var airbyteCatalog = azureBlobStorageSource.discover(jsonConfig); - - assertThat(airbyteCatalog.getStreams()) - .hasSize(1) - .element(0) - .hasFieldOrPropertyWithValue("name", STREAM_NAME) - .hasFieldOrPropertyWithValue("sourceDefinedCursor", true) - .hasFieldOrPropertyWithValue("defaultCursorField", List.of(AzureBlobAdditionalProperties.LAST_MODIFIED)) - .hasFieldOrPropertyWithValue("supportedSyncModes", List.of(SyncMode.INCREMENTAL, SyncMode.FULL_REFRESH)) - .extracting("jsonSchema") - .isNotNull(); - - } - - @Test - void testRead() { - var configuredAirbyteCatalog = AzureBlobStorageDataFactory.createConfiguredAirbyteCatalog(STREAM_NAME); - - Iterator iterator = - azureBlobStorageSource.read(jsonConfig, configuredAirbyteCatalog, Jsons.emptyObject()); - - var airbyteRecordMessages = Stream.generate(() -> null) - .takeWhile(x -> iterator.hasNext()) - .map(n -> iterator.next()) - .filter(am -> am.getType() == AirbyteMessage.Type.RECORD) - .map(AirbyteMessage::getRecord) - .toList(); - - assertThat(airbyteRecordMessages) - .hasSize(2) - .anyMatch(arm -> arm.getStream().equals(STREAM_NAME) && - Jsons.serialize(arm.getData()).contains( - "\"attr_1\":\"str_1\"")) - .anyMatch(arm -> arm.getStream().equals(STREAM_NAME) && - Jsons.serialize(arm.getData()).contains( - "\"attr_2\":\"str_2\"")); - - } - -} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/source/azureblobstorage/JsonlAzureBlobStorageOperationsTest.java b/airbyte-integrations/connectors/source-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/source/azureblobstorage/JsonlAzureBlobStorageOperationsTest.java deleted file mode 100644 index 69e7fa9daf12..000000000000 --- a/airbyte-integrations/connectors/source-azure-blob-storage/src/test-integration/java/io/airbyte/integrations/source/azureblobstorage/JsonlAzureBlobStorageOperationsTest.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.azureblobstorage; - -import static org.assertj.core.api.Assertions.assertThat; - -import com.azure.core.util.BinaryData; -import com.azure.storage.blob.BlobContainerClient; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import io.airbyte.integrations.source.azureblobstorage.format.JsonlAzureBlobStorageOperations; -import java.time.OffsetDateTime; -import org.json.JSONException; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.skyscreamer.jsonassert.JSONAssert; - -class JsonlAzureBlobStorageOperationsTest { - - private AzureBlobStorageContainer azureBlobStorageContainer; - - private AzureBlobStorageOperations azureBlobStorageOperations; - - private BlobContainerClient blobContainerClient; - - private ObjectMapper objectMapper; - - private static final String STREAM_NAME = "airbyte-container"; - - @BeforeEach - void setup() { - azureBlobStorageContainer = new AzureBlobStorageContainer().withExposedPorts(10000); - azureBlobStorageContainer.start(); - JsonNode jsonConfig = AzureBlobStorageDataFactory.createAzureBlobStorageConfig( - "http://127.0.0.1:" + azureBlobStorageContainer.getMappedPort(10000), STREAM_NAME); - - var azureBlobStorageConfig = AzureBlobStorageConfig.createAzureBlobStorageConfig(jsonConfig); - blobContainerClient = azureBlobStorageConfig.createBlobContainerClient(); - blobContainerClient.createIfNotExists(); - blobContainerClient.getBlobClient("FolderA/FolderB/blob1.json").upload(BinaryData - .fromString(""" - {"name":"Molecule Man","age":29,"secretIdentity":"Dan Jukes","powers":["Radiation resistance","Turning tiny","Radiation blast"]} - {"name":"Bat Man","secretIdentity":"Bruce Wayne","powers":["Agility", "Detective skills", "Determination"]} - """)); - blobContainerClient.getBlobClient("FolderA/FolderB/blob2.json").upload(BinaryData.fromString( - "{\"name\":\"Molecule Man\",\"surname\":\"Powers\",\"powers\":[\"Radiation resistance\",\"Turning tiny\",\"Radiation blast\"]}\n")); - // should be ignored since its in ignored path - blobContainerClient.getBlobClient("FolderA/blob3.json").upload(BinaryData.fromString("{\"ignored\":true}\n")); - azureBlobStorageOperations = new JsonlAzureBlobStorageOperations(azureBlobStorageConfig); - objectMapper = new ObjectMapper(); - } - - @AfterEach - void tearDown() { - azureBlobStorageContainer.stop(); - azureBlobStorageContainer.close(); - } - - @Test - void testListBlobs() { - var azureBlobs = azureBlobStorageOperations.listBlobs(); - - assertThat(azureBlobs) - .hasSize(2) - .anyMatch(ab -> ab.name().equals("FolderA/FolderB/blob1.json")) - .anyMatch(ab -> ab.name().equals("FolderA/FolderB/blob2.json")); - } - - @Test - void testInferSchema() throws JsonProcessingException, JSONException { - - var jsonSchema = azureBlobStorageOperations.inferSchema(); - - JSONAssert.assertEquals(objectMapper.writeValueAsString(jsonSchema), """ - { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "age": { - "type": "integer" - }, - "secretIdentity": { - "type": "string" - }, - "powers": { - "type": "array", - "items": { - "type": "string" - } - }, - "surname": { - "type": "string" - }, - "_ab_source_blob_name": { - "type": "string" - }, - "_ab_source_file_last_modified": { - "type": "string" - } - }, - "additionalProperties": true - } - """, true); - - } - - @Test - void testReadBlobs() throws InterruptedException, JsonProcessingException, JSONException { - var now = OffsetDateTime.now(); - - Thread.sleep(1000); - - blobContainerClient.getBlobClient("FolderA/FolderB/blob1.json").upload(BinaryData.fromString( - "{\"name\":\"Super Man\",\"secretIdentity\":\"Clark Kent\",\"powers\":[\"Lightning fast\",\"Super strength\",\"Laser vision\"]}\n"), - true); - - var messages = azureBlobStorageOperations.readBlobs(now); - - var azureBlob = azureBlobStorageOperations.listBlobs().stream() - .filter(ab -> ab.name().equals("FolderA/FolderB/blob1.json")) - .findAny() - .orElseThrow(); - - assertThat(messages) - .hasSize(1); - - JSONAssert.assertEquals(objectMapper.writeValueAsString(messages.get(0)), String.format( - "{\"name\":\"Super Man\",\"secretIdentity\":\"Clark Kent\",\"powers\":[\"Lightning fast\",\"Super strength\",\"Laser vision\"],\"_ab_source_blob_name\":\"%s\",\"_ab_source_file_last_modified\":\"%s\"}\n", - azureBlob.name(), azureBlob.lastModified().toString()), true); - - } - -} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/src/test/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageConfigTest.java b/airbyte-integrations/connectors/source-azure-blob-storage/src/test/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageConfigTest.java deleted file mode 100644 index 003a395bbd6e..000000000000 --- a/airbyte-integrations/connectors/source-azure-blob-storage/src/test/java/io/airbyte/integrations/source/azureblobstorage/AzureBlobStorageConfigTest.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.azureblobstorage; - -import static org.assertj.core.api.Assertions.assertThat; - -import io.airbyte.commons.json.Jsons; -import java.util.Map; -import org.junit.jupiter.api.Test; - -class AzureBlobStorageConfigTest { - - @Test - void testAzureBlobStorageConfig() { - var jsonConfig = Jsons.jsonNode(Map.of( - "azure_blob_storage_endpoint", "http://127.0.0.1:10000/devstoreaccount1", - "azure_blob_storage_account_name", "devstoreaccount1", - "azure_blob_storage_account_key", - "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", - "azure_blob_storage_container_name", "airbyte-container", - "azure_blob_storage_blobs_prefix", "FolderA/FolderB/", - "azure_blob_storage_schema_inference_limit", 10L, - "format", Jsons.deserialize(""" - { - "format_type": "JSONL" - }"""))); - - var azureBlobStorageConfig = AzureBlobStorageConfig.createAzureBlobStorageConfig(jsonConfig); - - assertThat(azureBlobStorageConfig) - .hasFieldOrPropertyWithValue("endpoint", "http://127.0.0.1:10000/devstoreaccount1") - .hasFieldOrPropertyWithValue("accountName", "devstoreaccount1") - .hasFieldOrPropertyWithValue("accountKey", - "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==") - .hasFieldOrPropertyWithValue("containerName", "airbyte-container") - .hasFieldOrPropertyWithValue("prefix", "FolderA/FolderB/") - .hasFieldOrPropertyWithValue("schemaInferenceLimit", 10L) - .hasFieldOrPropertyWithValue("formatConfig", new AzureBlobStorageConfig.FormatConfig( - AzureBlobStorageConfig.FormatConfig.Format.JSONL)); - - } - -} diff --git a/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/unit_tests.py b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/unit_tests.py new file mode 100644 index 000000000000..88d003f81475 --- /dev/null +++ b/airbyte-integrations/connectors/source-azure-blob-storage/unit_tests/unit_tests.py @@ -0,0 +1,30 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from source_azure_blob_storage.legacy_config_transformer import LegacyConfigTransformer + + +def test_config_convertation(): + legacy_config = { + "azure_blob_storage_endpoint": "https://airbyteteststorage.blob.core.windows.net", + "azure_blob_storage_account_name": "airbyteteststorage", + "azure_blob_storage_account_key": "secret/key==", + "azure_blob_storage_container_name": "airbyte-source-azure-blob-storage-test", + "azure_blob_storage_blobs_prefix": "subfolder/", + "azure_blob_storage_schema_inference_limit": 500, + "format": "jsonl", + } + new_config = LegacyConfigTransformer.convert(legacy_config) + assert new_config == { + "azure_blob_storage_account_key": "secret/key==", + "azure_blob_storage_account_name": "airbyteteststorage", + "azure_blob_storage_container_name": "airbyte-source-azure-blob-storage-test", + "azure_blob_storage_endpoint": "https://airbyteteststorage.blob.core.windows.net", + "streams": [ + { + "format": {"filetype": "jsonl"}, + "legacy_prefix": "subfolder/", + "name": "airbyte-source-azure-blob-storage-test", + "validation_policy": "Emit Record", + } + ], + } diff --git a/airbyte-integrations/connectors/source-azure-table/README.md b/airbyte-integrations/connectors/source-azure-table/README.md index 4a7a3a78d43f..8fb2ae68d6bd 100644 --- a/airbyte-integrations/connectors/source-azure-table/README.md +++ b/airbyte-integrations/connectors/source-azure-table/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-azure-table:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/azure-table) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_azure_table/spec.json` file. @@ -54,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-azure-table:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-azure-table build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-azure-table:airbyteDocker +An image will be built with the tag `airbyte/source-azure-table:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-azure-table:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,44 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-azure-table:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-azure-table:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-azure-table:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json --state /integration_tests/state.json ``` + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-azure-table test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-azure-table:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-azure-table:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -122,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-azure-table test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/azure-table.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-azure-table/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-azure-table/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-azure-table/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-azure-table/build.gradle b/airbyte-integrations/connectors/source-azure-table/build.gradle deleted file mode 100644 index 1f0b4a464a92..000000000000 --- a/airbyte-integrations/connectors/source-azure-table/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_azure_table_singer' -} diff --git a/airbyte-integrations/connectors/source-babelforce/README.md b/airbyte-integrations/connectors/source-babelforce/README.md index f85b00f928f7..7ae9fd8b12d2 100644 --- a/airbyte-integrations/connectors/source-babelforce/README.md +++ b/airbyte-integrations/connectors/source-babelforce/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-babelforce:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/babelforce) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_babelforce/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-babelforce:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-babelforce build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-babelforce:airbyteDocker +An image will be built with the tag `airbyte/source-babelforce:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-babelforce:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-babelforce:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-babelforce:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-babelforce:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-babelforce test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-babelforce:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-babelforce:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-babelforce test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/babelforce.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-babelforce/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-babelforce/acceptance-test-docker.sh deleted file mode 100644 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-babelforce/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-babelforce/build.gradle b/airbyte-integrations/connectors/source-babelforce/build.gradle deleted file mode 100644 index dca68c46e194..000000000000 --- a/airbyte-integrations/connectors/source-babelforce/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_babelforce' -} diff --git a/airbyte-integrations/connectors/source-babelforce/metadata.yaml b/airbyte-integrations/connectors/source-babelforce/metadata.yaml index 9425eae3c84b..f4f4e35ea4ba 100644 --- a/airbyte-integrations/connectors/source-babelforce/metadata.yaml +++ b/airbyte-integrations/connectors/source-babelforce/metadata.yaml @@ -20,7 +20,7 @@ data: releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/babelforce tags: - - language:lowcode + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-bamboo-hr/README.md b/airbyte-integrations/connectors/source-bamboo-hr/README.md index ef90e2653050..6d76f40e69b2 100644 --- a/airbyte-integrations/connectors/source-bamboo-hr/README.md +++ b/airbyte-integrations/connectors/source-bamboo-hr/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-bamboo-hr:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/bamboo-hr) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_bamboo_hr/spec.json` file. @@ -54,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-bamboo-hr:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-bamboo-hr build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-bamboo-hr:airbyteDocker +An image will be built with the tag `airbyte/source-bamboo-hr:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-bamboo-hr:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,44 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-bamboo-hr:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-bamboo-hr:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-bamboo-hr:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-bamboo-hr test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-bamboo-hr:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-bamboo-hr:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -122,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-bamboo-hr test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/bamboo-hr.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-bamboo-hr/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-bamboo-hr/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-bamboo-hr/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-bamboo-hr/build.gradle b/airbyte-integrations/connectors/source-bamboo-hr/build.gradle deleted file mode 100644 index b0c10e34eac2..000000000000 --- a/airbyte-integrations/connectors/source-bamboo-hr/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_bamboo_hr_singer' -} diff --git a/airbyte-integrations/connectors/source-bigcommerce/.dockerignore b/airbyte-integrations/connectors/source-bigcommerce/.dockerignore index bf8edc6e2599..628d2f05f6ef 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/.dockerignore +++ b/airbyte-integrations/connectors/source-bigcommerce/.dockerignore @@ -1,6 +1,5 @@ * !Dockerfile -!Dockerfile.test !main.py !source_bigcommerce !setup.py diff --git a/airbyte-integrations/connectors/source-bigcommerce/Dockerfile b/airbyte-integrations/connectors/source-bigcommerce/Dockerfile index 0c14c34d26cb..756f514ab945 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/Dockerfile +++ b/airbyte-integrations/connectors/source-bigcommerce/Dockerfile @@ -1,16 +1,38 @@ -FROM python:3.9-slim +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base WORKDIR /airbyte/integration_code -COPY source_bigcommerce ./source_bigcommerce + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only COPY main.py ./ -COPY setup.py ./ -RUN pip install . +COPY source_bigcommerce ./source_bigcommerce ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.10 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-bigcommerce diff --git a/airbyte-integrations/connectors/source-bigcommerce/README.md b/airbyte-integrations/connectors/source-bigcommerce/README.md index b7eb80c54ccf..8ab2beb4e49b 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/README.md +++ b/airbyte-integrations/connectors/source-bigcommerce/README.md @@ -1,73 +1,34 @@ # Bigcommerce Source -This is the repository for the Bigcommerce source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/bigcommerce). +This is the repository for the Bigcommerce configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/bigcommerce). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-bigcommerce:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/bigcommerce) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_bigcommerce/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/bigcommerce) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_bigcommerce/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source bigcommerce test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-bigcommerce:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-bigcommerce build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-bigcommerce:airbyteDocker +An image will be built with the tag `airbyte/source-bigcommerce:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-bigcommerce:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -77,44 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-bigcommerce:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-bigcommerce:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-bigcommerce:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-bigcommerce test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-bigcommerce:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-bigcommerce:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -124,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-bigcommerce test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/bigcommerce.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-bigcommerce/__init__.py b/airbyte-integrations/connectors/source-bigcommerce/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-bigcommerce/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-bigcommerce/acceptance-test-config.yml b/airbyte-integrations/connectors/source-bigcommerce/acceptance-test-config.yml index 94d5f137867b..564a05cfa774 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-bigcommerce/acceptance-test-config.yml @@ -1,24 +1,47 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-bigcommerce:dev -tests: +acceptance_tests: spec: - - spec_path: "source_bigcommerce/spec.json" + tests: + - spec_path: "source_bigcommerce/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.10" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: ["orders", "transactions", "products", "customers", "channels", "order_products"] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: "orders" + bypass_reason: "sandbox account does not have records" + - name: "transactions" + bypass_reason: "sandbox account does not have records" + - name: "products" + bypass_reason: "sandbox account does not have records" + - name: "customers" + bypass_reason: "sandbox account does not have records" + - name: "channels" + bypass_reason: "sandbox account does not have records" + - name: "order_products" + bypass_reason: "sandbox account does not have records" + fail_on_extra_columns: false incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + skip_comprehensive_incremental_tests: true full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-bigcommerce/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-bigcommerce/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-bigcommerce/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-bigcommerce/build.gradle b/airbyte-integrations/connectors/source-bigcommerce/build.gradle deleted file mode 100644 index 4079177de232..000000000000 --- a/airbyte-integrations/connectors/source-bigcommerce/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_bigcommerce' -} diff --git a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/__init__.py b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/abnormal_state.json index 723a217dc775..eb1f5a064644 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/abnormal_state.json @@ -3,7 +3,7 @@ "type": "STREAM", "stream": { "stream_state": { - "date_modified": "2080-07-30T22:16:46" + "date_modified": "2080-07-30T22:16:46+00:00" }, "stream_descriptor": { "name": "customers" diff --git a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/catalog.json b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/catalog.json deleted file mode 100644 index ccb19696a04d..000000000000 --- a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/catalog.json +++ /dev/null @@ -1,1993 +0,0 @@ -{ - "streams": [ - { - "name": "customers", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "email": { - "type": ["null", "string"] - }, - "first_name": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "company": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "registration_ip_address": { - "type": ["null", "string"] - }, - "notes": { - "type": ["null", "string"] - }, - "tax_exempt_category": { - "type": ["null", "string"] - }, - "customer_group_id": { - "type": ["null", "number"] - }, - "id": { - "type": ["null", "number"] - }, - "date_modified": { - "type": ["null", "string"], - "format": "date-time" - }, - "date_created": { - "type": ["null", "string"], - "format": "date-time" - }, - "address_count": { - "type": ["null", "number"] - }, - "attribute_count": { - "type": ["null", "number"] - }, - "authentication": { - "type": ["null", "object"], - "properties": { - "force_password_reset": { - "type": ["null", "boolean"] - } - } - }, - "addresses": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "first_name": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "company": { - "type": ["null", "string"] - }, - "address1": { - "type": ["null", "string"] - }, - "address2": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "state_or_province": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "country_code": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "address_type": { - "type": ["null", "string"] - }, - "customer_id": { - "type": ["null", "number"] - }, - "id": { - "type": ["null", "number"] - }, - "country": { - "type": ["null", "string"] - }, - "form_fields": { - "type": ["null", "array"], - "oneOf": [ - { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["name", "string"] - }, - "value": { - "oneOf": [ - { - "type": ["null", "string"] - }, - { - "type": ["null", "number"] - }, - { - "type": ["null", "array"] - } - ] - }, - "customer_id": { - "type": ["null", "number"] - } - } - }, - { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["name", "string"] - }, - "value": { - "oneOf": [ - { - "type": ["null", "string"] - }, - { - "type": ["null", "number"] - }, - { - "type": ["null", "array"] - } - ] - }, - "address_id": { - "type": ["null", "number"] - } - } - } - ] - }, - "store_credit_amounts": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "number"] - } - } - } - }, - "accepts_product_review_abandoned_cart_emails": { - "type": ["null", "boolean"] - }, - "channel_ids": { - "type": ["null", "array"] - } - } - } - } - } - } - }, - { - "name": "orders", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "number"] - }, - "date_modified": { - "type": ["null", "string"], - "format": "date-time" - }, - "date_shipped": { - "type": ["null", "string"], - "format": "date-time" - }, - "cart_id": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "string"] - }, - "subtotal_tax": { - "type": ["null", "string"] - }, - "shipping_cost_tax": { - "type": ["null", "string"] - }, - "shipping_cost_tax_class_id": { - "type": ["null", "number"] - }, - "handling_cost_tax": { - "type": ["null", "string"] - }, - "handling_cost_tax_class_id": { - "type": ["null", "number"] - }, - "wrapping_cost_tax": { - "type": ["null", "string"] - }, - "wrapping_cost_tax_class_id": { - "type": ["null", "number"] - }, - "payment_status": { - "type": ["null", "string"] - }, - "store_credit_amount": { - "type": ["null", "string"] - }, - "gift_certificate_amount": { - "type": ["null", "string"] - }, - "currency_id": { - "type": ["null", "number"] - }, - "currency_code": { - "type": ["null", "string"] - }, - "currency_exchange_rate": { - "type": ["null", "string"] - }, - "default_currency_id": { - "type": ["null", "number"] - }, - "coupon_discount": { - "type": ["null", "string"] - }, - "shipping_address_count": { - "type": ["null", "number"] - }, - "is_email_opt_in": { - "type": ["null", "boolean"] - }, - "order_source": { - "type": ["null", "string"] - }, - "products": { - "type": ["null", "object"], - "properties": { - "url": { - "type": ["null", "string"] - }, - "resource": { - "type": ["null", "string"] - } - } - }, - "shipping_addresses": { - "type": ["null", "object"], - "properties": { - "url": { - "type": ["null", "string"] - }, - "resource": { - "type": ["null", "string"] - } - } - }, - "coupons": { - "type": ["null", "object"], - "properties": { - "url": { - "type": ["null", "string"] - }, - "resource": { - "type": ["null", "string"] - } - } - }, - "status_id": { - "type": ["null", "number"] - }, - "base_handling_cost": { - "type": ["null", "string"] - }, - "base_shipping_cost": { - "type": ["null", "string"] - }, - "base_wrapping_cost": { - "oneOf": [ - { - "type": ["null", "number"] - }, - { - "type": ["null", "string"] - } - ] - }, - "billing_address": { - "type": ["null", "object"], - "properties": { - "first_name": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "company": { - "type": ["null", "string"] - }, - "street_1": { - "type": ["null", "string"] - }, - "street_2": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "zip": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "country_iso2": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "form_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } - } - } - }, - "channel_id": { - "type": ["null", "number"] - }, - "customer_id": { - "type": ["null", "number"] - }, - "customer_message": { - "type": ["null", "string"] - }, - "date_created": { - "type": ["null", "string"], - "format": "date-time" - }, - "default_currency_code": { - "type": ["null", "string"] - }, - "discount_amount": { - "type": ["null", "string"] - }, - "ebay_order_id": { - "type": ["null", "string"] - }, - "external_id": { - "type": ["null", "string"] - }, - "external_source": { - "type": ["null", "string"] - }, - "geoip_country": { - "type": ["null", "string"] - }, - "geoip_country_iso2": { - "type": ["null", "string"] - }, - "handling_cost_ex_tax": { - "type": ["null", "string"] - }, - "handling_cost_inc_tax": { - "type": ["null", "string"] - }, - "ip_address": { - "type": ["null", "string"] - }, - "is_deleted": { - "type": ["null", "boolean"] - }, - "items_shipped": { - "type": ["null", "number"] - }, - "items_total": { - "type": ["null", "number"] - }, - "order_is_digital": { - "type": ["null", "boolean"] - }, - "payment_method": { - "type": ["null", "string"] - }, - "refunded_amount": { - "type": ["null", "string"] - }, - "shipping_cost_ex_tax": { - "type": ["null", "string"] - }, - "shipping_cost_inc_tax": { - "type": ["null", "string"] - }, - "staff_notes": { - "type": ["null", "string"] - }, - "subtotal_ex_tax": { - "type": ["null", "string"] - }, - "subtotal_inc_tax": { - "type": ["null", "string"] - }, - "tax_provider_id": { - "type": ["null", "string"] - }, - "customer_locale": { - "type": ["null", "string"] - }, - "total_ex_tax": { - "type": ["null", "string"] - }, - "total_inc_tax": { - "type": ["null", "string"] - }, - "wrapping_cost_ex_tax": { - "type": ["null", "string"] - }, - "wrapping_cost_inc_tax": { - "type": ["null", "string"] - } - } - } - }, - { - "name": "transactions", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "event": { - "type": ["null", "string"] - }, - "method": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "number"] - }, - "currency": { - "type": ["null", "string"] - }, - "gateway": { - "type": ["null", "string"] - }, - "gateway_transaction_id": { - "type": ["null", "string"] - }, - "date_created": { - "type": ["null", "string"] - }, - "test": { - "type": ["null", "boolean"] - }, - "status": { - "type": ["null", "string"] - }, - "fraud_review": { - "type": ["null", "boolean"] - }, - "reference_transaction_id": { - "type": ["null", "number"] - }, - "offline": { - "type": ["null", "object"], - "properties": { - "display_name": { - "type": ["null", "string"] - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "payment_method": { - "type": ["null", "string"] - } - } - }, - "payment_method_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "number"] - }, - "order_id": { - "type": ["null", "string"] - }, - "payment_instrument_token": { - "type": ["null", "string"] - }, - "avs_result": { - "type": ["null", "object"], - "properties": { - "code": { - "type": ["null", "string"] - }, - "message": { - "type": ["null", "string"] - }, - "street_match": { - "type": ["null", "string"] - }, - "postal_match": { - "type": ["null", "string"] - } - } - }, - "cvv_result": { - "type": ["null", "object"], - "properties": { - "code": { - "type": ["null", "string"] - }, - "message": { - "type": ["null", "string"] - } - } - }, - "credit_card": { - "type": ["null", "object"], - "properties": { - "card_type": { - "type": ["null", "string"] - }, - "card_iin": { - "type": ["null", "string"] - }, - "card_last4": { - "type": ["null", "string"] - }, - "card_expiry_month": { - "type": ["null", "number"] - }, - "card_expiry_year": { - "type": ["null", "number"] - } - } - }, - "gift_certificate": { - "type": ["null", "object"], - "properties": { - "code": { - "type": ["null", "string"] - }, - "original_balance": { - "type": ["null", "number"] - }, - "starting_balance": { - "type": ["null", "number"] - }, - "remaining_balance": { - "type": ["null", "number"] - }, - "status": { - "type": ["null", "string"] - } - } - }, - "store_credit": { - "type": ["null", "object"], - "properties": { - "remaining_balance": { - "type": ["null", "number"] - } - } - } - } - } - }, - { - "name": "order_products", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "integer"] - }, - "order_id": { - "type": ["null", "integer"] - }, - "product_id": { - "type": ["null", "integer"] - }, - "variant_id": { - "type": ["null", "integer"] - }, - "order_address_id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "name_customer": { - "type": ["null", "string"] - }, - "name_merchant": { - "type": ["null", "string"] - }, - "sku": { - "type": ["null", "string"] - }, - "upc": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "base_price": { - "type": ["null", "string"] - }, - "price_ex_tax": { - "type": ["null", "string"] - }, - "price_inc_tax": { - "type": ["null", "string"] - }, - "price_tax": { - "type": ["null", "string"] - }, - "base_total": { - "type": ["null", "string"] - }, - "total_ex_tax": { - "type": ["null", "string"] - }, - "total_inc_tax": { - "type": ["null", "string"] - }, - "total_tax": { - "type": ["null", "string"] - }, - "weight": { - "type": ["null", "string"] - }, - "width": { - "type": ["null", "string"] - }, - "height": { - "type": ["null", "string"] - }, - "depth": { - "type": ["null", "string"] - }, - "quantity": { - "type": ["null", "number"] - }, - "base_cost_price": { - "type": ["null", "string"] - }, - "cost_price_inc_tax": { - "type": ["null", "string"] - }, - "cost_price_ex_tax": { - "type": ["null", "string"] - }, - "cost_price_tax": { - "type": ["null", "string"] - }, - "is_refunded": { - "type": ["null", "boolean"] - }, - "quantity_refunded": { - "type": ["null", "number"] - }, - "refund_amount": { - "type": ["null", "string"] - }, - "return_id": { - "type": ["null", "number"] - }, - "wrapping_id": { - "type": ["null", "string"] - }, - "wrapping_name": { - "type": ["null", "string"] - }, - "base_wrapping_cost": { - "type": ["null", "string"] - }, - "wrapping_cost_ex_tax": { - "type": ["null", "string"] - }, - "wrapping_cost_inc_tax": { - "type": ["null", "string"] - }, - "wrapping_cost_tax": { - "type": ["null", "string"] - }, - "wrapping_message": { - "type": ["null", "string"] - }, - "quantity_shipped": { - "type": ["null", "number"] - }, - "event_name": { - "type": ["null", "string"] - }, - "event_date": { - "type": ["null", "string"] - }, - "fixed_shipping_cost": { - "type": ["null", "string"] - }, - "ebay_item_id": { - "type": ["null", "string"] - }, - "ebay_transaction_id": { - "type": ["null", "string"] - }, - "option_set_id": { - "type": ["null", "integer"] - }, - "parent_order_product_id": { - "type": ["null", "integer"] - }, - "is_bundled_product": { - "type": ["null", "boolean"] - }, - "bin_picking_number": { - "type": ["null", "string"] - }, - "external_id": { - "type": ["null", "string"] - }, - "fulfillment_source": { - "type": ["null", "string"] - }, - "brand": { - "type": ["null", "string"] - }, - "gift_certificate_id": { - "type": ["null", "integer"] - }, - "applied_discounts": { - "type": ["null", "array"] - }, - "applied_discounts": { - "type": ["null", "array"] - }, - "product_options": { - "type": ["null", "array"] - }, - "configurable_fields": { - "type": ["null", "array"] - } - } - } - }, - { - "name": "pages", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "email": { - "type": ["null", "string"] - }, - "meta_title": { - "type": ["null", "string"] - }, - "body": { - "type": ["null", "string"] - }, - "feed": { - "type": ["null", "string"] - }, - "link": { - "type": ["null", "string"] - }, - "contact_fields": { - "type": ["null", "string"] - }, - "meta_keywords": { - "type": ["null", "string"] - }, - "meta_description": { - "type": ["null", "string"] - }, - "search_keywords": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - }, - "channel_id": { - "type": ["null", "number"] - }, - "name": { - "type": ["null", "string"] - }, - "is_visible": { - "type": ["null", "boolean"] - }, - "parent_id": { - "type": ["null", "number"] - }, - "sort_order": { - "type": ["null", "number"] - }, - "type": { - "type": ["null", "string"] - }, - "is_homepage": { - "type": ["null", "boolean"] - }, - "is_customers_only": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "number"] - } - } - } - }, - { - "name": "brands", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "page_title": { - "type": ["null", "string"] - }, - "meta_keywords": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "meta_description": { - "type": ["null", "string"] - }, - "search_keywords": { - "type": ["null", "string"] - }, - "image_url": { - "type": ["null", "string"] - }, - "custom_url": { - "type": ["null", "object"], - "properties": { - "url": { - "type": ["null", "string"] - }, - "is_customized": { - "type": ["null", "boolean"] - } - } - } - } - } - }, - { - "name": "categories", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "integer"] - }, - "parent_id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "views": { - "type": ["null", "integer"] - }, - "sort_order": { - "type": ["null", "integer"] - }, - "page_title": { - "type": ["null", "string"] - }, - "meta_keywords": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "meta_description": { - "type": ["null", "string"] - }, - "layout_file": { - "type": ["null", "string"] - }, - "image_url": { - "type": ["null", "string"] - }, - "is_visible": { - "type": ["null", "boolean"] - }, - "search_keywords": { - "type": ["null", "string"] - }, - "default_product_sort": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "object"], - "properties": { - "path": { - "type": ["null", "string"] - }, - "is_customized": { - "type": ["null", "boolean"] - } - } - } - } - } - }, - { - "name": "products", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "name": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "sku": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "weight": { - "type": ["null", "number"], - "format": "float" - }, - "width": { - "type": ["null", "number"], - "format": "float" - }, - "depth": { - "type": ["null", "number"], - "format": "float" - }, - "height": { - "type": ["null", "number"], - "format": "float" - }, - "price": { - "type": ["null", "number"], - "format": "float" - }, - "cost_price": { - "type": ["null", "number"], - "format": "float" - }, - "retail_price": { - "type": ["null", "number"], - "format": "float" - }, - "sale_price": { - "type": ["null", "number"], - "format": "float" - }, - "map_price": { - "type": ["null", "number"] - }, - "tax_class_id": { - "type": ["null", "number"] - }, - "product_tax_code": { - "type": ["null", "string"] - }, - "categories": { - "type": ["null", "array"], - "items": { - "type": ["null", "number"] - } - }, - "brand_id": { - "type": ["null", "number"] - }, - "inventory_level": { - "type": ["null", "number"] - }, - "inventory_warning_level": { - "type": ["null", "number"] - }, - "inventory_tracking": { - "type": ["null", "string"] - }, - "fixed_cost_shipping_price": { - "type": ["null", "number"], - "format": "float" - }, - "is_free_shipping": { - "type": ["null", "boolean"] - }, - "is_visible": { - "type": ["null", "boolean"] - }, - "is_featured": { - "type": ["null", "boolean"] - }, - "related_products": { - "type": ["null", "array"], - "items": { - "type": ["null", "number"] - } - }, - "warranty": { - "type": ["null", "string"] - }, - "bin_picking_number": { - "type": ["null", "string"] - }, - "layout_file": { - "type": ["null", "string"] - }, - "upc": { - "type": ["null", "string"] - }, - "search_keywords": { - "type": ["null", "string"] - }, - "availability": { - "type": ["null", "string"] - }, - "availability_description": { - "type": ["null", "string"] - }, - "gift_wrapping_options_type": { - "type": ["null", "string"] - }, - "gift_wrapping_options_list": { - "type": ["null", "array"], - "items": { - "type": ["null", "number"] - } - }, - "sort_order": { - "type": ["null", "number"] - }, - "condition": { - "type": ["null", "string"] - }, - "is_condition_shown": { - "type": ["null", "boolean"] - }, - "order_quantity_minimum": { - "type": ["null", "number"] - }, - "order_quantity_maximum": { - "type": ["null", "number"] - }, - "page_title": { - "type": ["null", "string"] - }, - "meta_keywords": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "meta_description": { - "type": ["null", "string"] - }, - "view_count": { - "type": ["null", "number"] - }, - "preorder_release_date": { - "type": ["null", "string"], - "format": "date-time" - }, - "preorder_message": { - "type": ["null", "string"] - }, - "is_preorder_only": { - "type": ["null", "boolean"] - }, - "is_price_hidden": { - "type": ["null", "boolean"] - }, - "price_hidden_label": { - "type": ["null", "string"] - }, - "custom_url": { - "type": ["null", "object"], - "title": "customUrl_Full", - "properties": { - "url": { - "type": ["null", "string"] - }, - "is_customized": { - "type": ["null", "boolean"] - } - } - }, - "open_graph_type": { - "type": ["null", "string"] - }, - "open_graph_title": { - "type": ["null", "string"] - }, - "open_graph_description": { - "type": ["null", "string"] - }, - "open_graph_use_meta_description": { - "type": ["null", "boolean"] - }, - "open_graph_use_product_name": { - "type": ["null", "boolean"] - }, - "open_graph_use_image": { - "type": ["null", "boolean"] - }, - "brand_name or brand_id": { - "type": ["null", "string"] - }, - "gtin": { - "type": ["null", "string"] - }, - "mpn": { - "type": ["null", "string"] - }, - "reviews_rating_sum": { - "type": ["null", "number"] - }, - "reviews_count": { - "type": ["null", "number"] - }, - "total_sold": { - "type": ["null", "number"] - }, - "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "title": "productCustomField_Put", - "required": ["name", "value"], - "properties": { - "id": { - "type": ["null", "number"] - }, - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } - }, - "bulk_pricing_rules": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "title": "bulkPricingRule_Full", - "required": ["quantity_min", "quantity_max", "type", "amount"], - "properties": { - "id": { - "type": ["null", "number"] - }, - "quantity_min": { - "type": ["null", "number"] - }, - "quantity_max": { - "type": ["null", "number"] - }, - "type": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "number"] - } - } - } - }, - "images": { - "type": ["null", "array"], - "items": { - "title": "productImage_Full", - "type": ["null", "object"], - "properties": { - "image_file": { - "type": ["null", "string"] - }, - "is_thumbnail": { - "type": ["null", "boolean"] - }, - "sort_order": { - "type": ["null", "number"] - }, - "description": { - "type": ["null", "string"] - }, - "image_url": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "number"] - }, - "product_id": { - "type": ["null", "number"] - }, - "url_zoom": { - "type": ["null", "string"] - }, - "url_standard": { - "type": ["null", "string"] - }, - "url_thumbnail": { - "type": ["null", "string"] - }, - "url_tiny": { - "type": ["null", "string"] - }, - "date_modified": { - "format": "date-time", - "type": ["null", "string"] - } - } - } - }, - "videos": { - "type": ["null", "array"], - "items": { - "title": "productVideo_Full", - "type": ["null", "object"], - "properties": { - "title": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "sort_order": { - "type": ["null", "number"] - }, - "type": { - "type": ["null", "string"] - }, - "video_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "number"] - }, - "product_id": { - "type": ["null", "number"] - }, - "length": { - "type": ["null", "string"] - } - } - } - }, - "date_created": { - "type": ["null", "string"], - "format": "date-time" - }, - "date_modified": { - "type": ["null", "string"], - "format": "date-time" - }, - "id": { - "type": ["null", "number"] - }, - "base_variant_id": { - "type": ["null", "number"] - }, - "calculated_price": { - "type": ["null", "number"], - "format": "float" - }, - "options": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "title": "productOption_Base", - "properties": { - "id": { - "type": ["null", "number"] - }, - "product_id": { - "type": ["null", "number"] - }, - "display_name": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "config": { - "type": ["null", "object"], - "title": "productOptionConfig_Full", - "properties": { - "default_value": { - "type": ["null", "string"] - }, - "checked_by_default": { - "type": ["null", "boolean"] - }, - "checkbox_label": { - "type": ["null", "string"] - }, - "date_limited": { - "type": ["null", "boolean"] - }, - "date_limit_mode": { - "type": ["null", "string"] - }, - "date_earliest_value": { - "type": ["null", "string"], - "format": "date" - }, - "date_latest_value": { - "type": ["null", "string"], - "format": "date" - }, - "file_types_mode": { - "type": ["null", "string"] - }, - "file_types_supported": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "file_types_other": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "file_max_size": { - "type": ["null", "number"] - }, - "text_characters_limited": { - "type": ["null", "boolean"] - }, - "text_min_length": { - "type": ["null", "number"] - }, - "text_max_length": { - "type": ["null", "number"] - }, - "text_lines_limited": { - "type": ["null", "boolean"] - }, - "text_max_lines": { - "type": ["null", "number"] - }, - "number_limited": { - "type": ["null", "boolean"] - }, - "number_limit_mode": { - "type": ["null", "string"] - }, - "number_lowest_value": { - "type": ["null", "number"] - }, - "number_highest_value": { - "type": ["null", "number"] - }, - "number_numbers_only": { - "type": ["null", "boolean"] - }, - "product_list_adjusts_inventory": { - "type": ["null", "boolean"] - }, - "product_list_adjusts_pricing": { - "type": ["null", "boolean"] - }, - "product_list_shipping_calc": { - "type": ["null", "string"] - } - } - }, - "sort_order": { - "type": ["null", "number"] - }, - "option_values": { - "title": "productOptionOptionValue_Full", - "type": ["null", "object"], - "required": ["label", "sort_order"], - "properties": { - "is_default": { - "type": ["null", "boolean"] - }, - "label": { - "type": ["null", "string"] - }, - "sort_order": { - "type": ["null", "number"] - }, - "value_data": { - "type": ["object", "null"] - }, - "id": { - "type": ["null", "number"] - } - } - } - } - } - }, - "modifiers": { - "type": ["null", "array"], - "items": { - "title": "productModifier_Full", - "type": ["null", "object"], - "required": ["type", "required"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "required": { - "type": ["null", "boolean"] - }, - "sort_order": { - "type": ["null", "number"] - }, - "config": { - "type": ["null", "object"], - "title": "config_Full", - "properties": { - "default_value": { - "type": ["null", "string"] - }, - "checked_by_default": { - "type": ["null", "boolean"] - }, - "checkbox_label": { - "type": ["null", "string"] - }, - "date_limited": { - "type": ["null", "boolean"] - }, - "date_limit_mode": { - "type": ["null", "string"] - }, - "date_earliest_value": { - "type": ["null", "string"], - "format": "date" - }, - "date_latest_value": { - "type": ["null", "string"], - "format": "date" - }, - "file_types_mode": { - "type": ["null", "string"] - }, - "file_types_supported": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "file_types_other": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "file_max_size": { - "type": ["null", "number"] - }, - "text_characters_limited": { - "type": ["null", "boolean"] - }, - "text_min_length": { - "type": ["null", "number"] - }, - "text_max_length": { - "type": ["null", "number"] - }, - "text_lines_limited": { - "type": ["null", "boolean"] - }, - "text_max_lines": { - "type": ["null", "number"] - }, - "number_limited": { - "type": ["null", "boolean"] - }, - "number_limit_mode": { - "type": ["null", "string"] - }, - "number_lowest_value": { - "type": ["null", "number"] - }, - "number_highest_value": { - "type": ["null", "number"] - }, - "number_numbers_only": { - "type": ["null", "boolean"] - }, - "product_list_adjusts_inventory": { - "type": ["null", "boolean"] - }, - "product_list_adjusts_pricing": { - "type": ["null", "boolean"] - }, - "product_list_shipping_calc": { - "type": ["null", "string"] - } - } - }, - "display_name": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "number"] - }, - "product_id": { - "type": ["null", "number"] - }, - "name": { - "type": ["null", "string"] - }, - "option_values": { - "type": ["null", "array"], - "items": { - "title": "productModifierOptionValue_Full", - "type": ["null", "object"], - "required": ["label", "sort_order"], - "properties": { - "is_default": { - "type": ["null", "boolean"] - }, - "label": { - "type": ["null", "string"] - }, - "sort_order": { - "type": ["null", "number"] - }, - "value_data": { - "type": ["object", "null"] - }, - "adjusters": { - "type": ["null", "object"], - "title": "adjusters_Full", - "properties": { - "price": { - "type": ["null", "object"], - "title": "adjuster_Full", - "properties": { - "adjuster": { - "type": ["null", "string"] - }, - "adjuster_value": { - "type": ["null", "number"] - } - } - }, - "weight": { - "type": ["null", "object"], - "title": "adjuster_Full", - "properties": { - "adjuster": { - "type": ["null", "string"] - }, - "adjuster_value": { - "type": ["null", "number"] - } - } - }, - "image_url": { - "type": ["null", "string"] - }, - "purchasing_disabled": { - "type": ["null", "object"], - "properties": { - "status": { - "type": ["null", "boolean"] - }, - "message": { - "type": ["null", "string"] - } - } - } - } - }, - "id": { - "type": ["null", "number"] - }, - "option_id": { - "type": ["null", "number"] - } - } - } - } - } - } - }, - "option_set_id": { - "type": ["null", "number"] - }, - "option_set_display": { - "type": ["null", "string"] - }, - "variants": { - "type": ["null", "array"], - "items": { - "title": "productVariant_Full", - "type": ["null", "object"], - "properties": { - "cost_price": { - "type": ["null", "number"], - "format": "double" - }, - "price": { - "type": ["null", "number"], - "format": "double" - }, - "sale_price": { - "type": ["null", "number"], - "format": "double" - }, - "retail_price": { - "type": ["null", "number"], - "format": "double" - }, - "weight": { - "type": ["null", "number"], - "format": "double" - }, - "width": { - "type": ["null", "number"], - "format": "double" - }, - "height": { - "type": ["null", "number"], - "format": "double" - }, - "depth": { - "type": ["null", "number"], - "format": "double" - }, - "is_free_shipping": { - "type": ["null", "boolean"] - }, - "fixed_cost_shipping_price": { - "type": ["null", "number"], - "format": "double" - }, - "purchasing_disabled": { - "type": ["null", "boolean"] - }, - "purchasing_disabled_message": { - "type": ["null", "string"] - }, - "upc": { - "type": ["null", "string"] - }, - "inventory_level": { - "type": ["null", "number"] - }, - "inventory_warning_level": { - "type": ["null", "number"] - }, - "bin_picking_number": { - "type": ["null", "string"] - }, - "mpn": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "number"] - }, - "product_id": { - "type": ["null", "number"] - }, - "sku": { - "type": ["null", "string"] - }, - "sku_id": { - "type": ["null", "number"] - }, - "option_values": { - "type": ["null", "array"], - "items": { - "title": "productVariantOptionValue_Full", - "type": ["null", "object"], - "properties": { - "option_display_name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "number"] - }, - "option_id": { - "type": ["null", "number"] - } - } - } - }, - "calculated_price": { - "type": ["null", "number"], - "format": "double" - }, - "calculated_weight": { - "type": "number" - } - } - } - } - } - } - }, - { - "name": "channels", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "number"] - }, - "external_id": { - "type": ["null", "string"] - }, - "is_listable_from_ui": { - "type": ["null", "boolean"] - }, - "is_visible": { - "type": ["null", "boolean"] - }, - "status": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "platform": { - "type": ["null", "boolean"] - }, - "date_created": { - "type": ["null", "string"], - "format": "date-time" - }, - "date_modified": { - "type": ["null", "string"], - "format": "date-time" - }, - "icon_url": { - "type": ["null", "string"] - } - } - } - }, - { - "name": "store", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "account_uuid": { - "type": ["null", "string"] - }, - "domain": { - "type": ["null", "string"] - }, - "secure_url": { - "type": ["null", "string"] - }, - "control_panel_base_url": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "first_name": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "address": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "admin_email": { - "type": ["null", "string"] - }, - "order_email": { - "type": ["null", "string"] - }, - "favicon_url": { - "type": ["null", "string"] - }, - "timezone": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "raw_offset": { - "type": ["null", "number"] - }, - "dst_offset": { - "type": ["null", "number"] - }, - "date_format": { - "type": ["null", "object"], - "properties": { - "display": { - "type": ["null", "string"] - }, - "export": { - "type": ["null", "string"] - }, - "extended_display": { - "type": ["null", "string"] - } - } - } - } - }, - "language": { - "type": ["null", "string"] - }, - "currency": { - "type": ["null", "string"] - }, - "currency_symbol": { - "type": ["null", "string"] - }, - "decimal_separator": { - "type": ["null", "string"] - }, - "thousands_separator": { - "type": ["null", "string"] - }, - "decimal_places": { - "type": ["null", "string"] - }, - "currency_symbol_location": { - "type": ["null", "string"] - }, - "weight_units": { - "type": ["null", "string"] - }, - "dimension_units": { - "type": ["null", "string"] - }, - "dimension_decimal_places": { - "type": ["null", "string"] - }, - "dimension_decimal_token": { - "type": ["null", "string"] - }, - "dimension_thousands_token": { - "type": ["null", "string"] - }, - "plan_name": { - "type": ["null", "string"] - }, - "plan_level": { - "type": ["null", "string"] - }, - "industry": { - "type": ["null", "string"] - }, - "logo": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "url": { - "type": ["null", "string"] - } - } - } - }, - "is_price_entered_with_tax": { - "type": ["null", "boolean"] - }, - "store_id": { - "type": ["null", "number"] - }, - "default_site_id": { - "type": ["null", "number"] - }, - "default_channel_id": { - "type": ["null", "number"] - }, - "active_comparison_modules": { - "type": ["null", "array"] - }, - "features": { - "type": ["null", "object"], - "properties": { - "stencil_enabled": { - "type": ["null", "boolean"] - }, - "sitewidehttps_enabled": { - "type": ["null", "boolean"] - }, - "facebook_catalog_id": { - "type": ["null", "string"] - }, - "checkout_type": { - "type": ["null", "string"] - }, - "wishlists_enabled": { - "type": ["null", "boolean"] - }, - "graphql_storefront_api_enabled": { - "type": ["null", "boolean"] - }, - "shopper_consent_tracking_enabled": { - "type": ["null", "boolean"] - }, - "multi_storefront_enabled": { - "type": ["null", "boolean"] - } - } - } - } - } - } - ] -} diff --git a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/configured_catalog.json index 2c227e937328..2b0206c69c64 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-bigcommerce/integration_tests/configured_catalog.json @@ -3,184 +3,7 @@ { "stream": { "name": "customers", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "email": { - "type": ["null", "string"] - }, - "first_name": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "company": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "registration_ip_address": { - "type": ["null", "string"] - }, - "notes": { - "type": ["null", "string"] - }, - "tax_exempt_category": { - "type": ["null", "string"] - }, - "customer_group_id": { - "type": ["null", "number"] - }, - "id": { - "type": ["null", "number"] - }, - "date_modified": { - "type": ["null", "string"], - "format": "date-time" - }, - "date_created": { - "type": ["null", "string"], - "format": "date-time" - }, - "address_count": { - "type": ["null", "number"] - }, - "attribute_count": { - "type": ["null", "number"] - }, - "authentication": { - "type": ["null", "object"], - "properties": { - "force_password_reset": { - "type": ["null", "boolean"] - } - } - }, - "addresses": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "first_name": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "company": { - "type": ["null", "string"] - }, - "address1": { - "type": ["null", "string"] - }, - "address2": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "state_or_province": { - "type": ["null", "string"] - }, - "postal_code": { - "type": ["null", "string"] - }, - "country_code": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "address_type": { - "type": ["null", "string"] - }, - "customer_id": { - "type": ["null", "number"] - }, - "id": { - "type": ["null", "number"] - }, - "country": { - "type": ["null", "string"] - }, - "form_fields": { - "type": ["null", "array"], - "oneOf": [ - { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["name", "string"] - }, - "value": { - "oneOf": [ - { - "type": ["null", "string"] - }, - { - "type": ["null", "number"] - }, - { - "type": ["null", "array"] - } - ] - }, - "customer_id": { - "type": ["null", "number"] - } - } - }, - { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["name", "string"] - }, - "value": { - "oneOf": [ - { - "type": ["null", "string"] - }, - { - "type": ["null", "number"] - }, - { - "type": ["null", "array"] - } - ] - }, - "address_id": { - "type": ["null", "number"] - } - } - } - ] - }, - "store_credit_amounts": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "number"] - } - } - } - }, - "accepts_product_review_abandoned_cart_emails": { - "type": ["null", "boolean"] - }, - "channel_ids": { - "type": ["null", "array"] - } - } - } - } - } - }, + "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["date_modified"] @@ -192,284 +15,7 @@ { "stream": { "name": "orders", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "number"] - }, - "date_modified": { - "type": ["null", "string"], - "format": "date-time" - }, - "date_shipped": { - "type": ["null", "string"], - "format": "date-time" - }, - "cart_id": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "string"] - }, - "subtotal_tax": { - "type": ["null", "string"] - }, - "shipping_cost_tax": { - "type": ["null", "string"] - }, - "shipping_cost_tax_class_id": { - "type": ["null", "number"] - }, - "handling_cost_tax": { - "type": ["null", "string"] - }, - "handling_cost_tax_class_id": { - "type": ["null", "number"] - }, - "wrapping_cost_tax": { - "type": ["null", "string"] - }, - "wrapping_cost_tax_class_id": { - "type": ["null", "number"] - }, - "payment_status": { - "type": ["null", "string"] - }, - "store_credit_amount": { - "type": ["null", "string"] - }, - "gift_certificate_amount": { - "type": ["null", "string"] - }, - "currency_id": { - "type": ["null", "number"] - }, - "currency_code": { - "type": ["null", "string"] - }, - "currency_exchange_rate": { - "type": ["null", "string"] - }, - "default_currency_id": { - "type": ["null", "number"] - }, - "coupon_discount": { - "type": ["null", "string"] - }, - "shipping_address_count": { - "type": ["null", "number"] - }, - "is_email_opt_in": { - "type": ["null", "boolean"] - }, - "order_source": { - "type": ["null", "string"] - }, - "products": { - "type": ["null", "object"], - "properties": { - "url": { - "type": ["null", "string"] - }, - "resource": { - "type": ["null", "string"] - } - } - }, - "shipping_addresses": { - "type": ["null", "object"], - "properties": { - "url": { - "type": ["null", "string"] - }, - "resource": { - "type": ["null", "string"] - } - } - }, - "coupons": { - "type": ["null", "object"], - "properties": { - "url": { - "type": ["null", "string"] - }, - "resource": { - "type": ["null", "string"] - } - } - }, - "status_id": { - "type": ["null", "number"] - }, - "base_handling_cost": { - "type": ["null", "string"] - }, - "base_shipping_cost": { - "type": ["null", "string"] - }, - "base_wrapping_cost": { - "oneOf": [ - { - "type": ["null", "number"] - }, - { - "type": ["null", "string"] - } - ] - }, - "billing_address": { - "type": ["null", "object"], - "properties": { - "first_name": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "company": { - "type": ["null", "string"] - }, - "street_1": { - "type": ["null", "string"] - }, - "street_2": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "zip": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "country_iso2": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "form_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } - } - } - }, - "channel_id": { - "type": ["null", "number"] - }, - "customer_id": { - "type": ["null", "number"] - }, - "customer_message": { - "type": ["null", "string"] - }, - "date_created": { - "type": ["null", "string"], - "format": "date-time" - }, - "default_currency_code": { - "type": ["null", "string"] - }, - "discount_amount": { - "type": ["null", "string"] - }, - "ebay_order_id": { - "type": ["null", "string"] - }, - "external_id": { - "type": ["null", "string"] - }, - "external_source": { - "type": ["null", "string"] - }, - "geoip_country": { - "type": ["null", "string"] - }, - "geoip_country_iso2": { - "type": ["null", "string"] - }, - "handling_cost_ex_tax": { - "type": ["null", "string"] - }, - "handling_cost_inc_tax": { - "type": ["null", "string"] - }, - "ip_address": { - "type": ["null", "string"] - }, - "is_deleted": { - "type": ["null", "boolean"] - }, - "items_shipped": { - "type": ["null", "number"] - }, - "items_total": { - "type": ["null", "number"] - }, - "order_is_digital": { - "type": ["null", "boolean"] - }, - "payment_method": { - "type": ["null", "string"] - }, - "refunded_amount": { - "type": ["null", "string"] - }, - "shipping_cost_ex_tax": { - "type": ["null", "string"] - }, - "shipping_cost_inc_tax": { - "type": ["null", "string"] - }, - "staff_notes": { - "type": ["null", "string"] - }, - "subtotal_ex_tax": { - "type": ["null", "string"] - }, - "subtotal_inc_tax": { - "type": ["null", "string"] - }, - "tax_provider_id": { - "type": ["null", "string"] - }, - "customer_locale": { - "type": ["null", "string"] - }, - "total_ex_tax": { - "type": ["null", "string"] - }, - "total_inc_tax": { - "type": ["null", "string"] - }, - "wrapping_cost_ex_tax": { - "type": ["null", "string"] - }, - "wrapping_cost_inc_tax": { - "type": ["null", "string"] - } - } - }, + "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["date_modified"] @@ -481,149 +27,7 @@ { "stream": { "name": "transactions", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "event": { - "type": ["null", "string"] - }, - "method": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "number"] - }, - "currency": { - "type": ["null", "string"] - }, - "gateway": { - "type": ["null", "string"] - }, - "gateway_transaction_id": { - "type": ["null", "string"] - }, - "date_created": { - "type": ["null", "string"] - }, - "test": { - "type": ["null", "boolean"] - }, - "status": { - "type": ["null", "string"] - }, - "fraud_review": { - "type": ["null", "boolean"] - }, - "reference_transaction_id": { - "type": ["null", "number"] - }, - "offline": { - "type": ["null", "object"], - "properties": { - "display_name": { - "type": ["null", "string"] - } - } - }, - "custom": { - "type": ["null", "object"], - "properties": { - "payment_method": { - "type": ["null", "string"] - } - } - }, - "payment_method_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "number"] - }, - "order_id": { - "type": ["null", "string"] - }, - "payment_instrument_token": { - "type": ["null", "string"] - }, - "avs_result": { - "type": ["null", "object"], - "properties": { - "code": { - "type": ["null", "string"] - }, - "message": { - "type": ["null", "string"] - }, - "street_match": { - "type": ["null", "string"] - }, - "postal_match": { - "type": ["null", "string"] - } - } - }, - "cvv_result": { - "type": ["null", "object"], - "properties": { - "code": { - "type": ["null", "string"] - }, - "message": { - "type": ["null", "string"] - } - } - }, - "credit_card": { - "type": ["null", "object"], - "properties": { - "card_type": { - "type": ["null", "string"] - }, - "card_iin": { - "type": ["null", "string"] - }, - "card_last4": { - "type": ["null", "string"] - }, - "card_expiry_month": { - "type": ["null", "number"] - }, - "card_expiry_year": { - "type": ["null", "number"] - } - } - }, - "gift_certificate": { - "type": ["null", "object"], - "properties": { - "code": { - "type": ["null", "string"] - }, - "original_balance": { - "type": ["null", "number"] - }, - "starting_balance": { - "type": ["null", "number"] - }, - "remaining_balance": { - "type": ["null", "number"] - }, - "status": { - "type": ["null", "string"] - } - } - }, - "store_credit": { - "type": ["null", "object"], - "properties": { - "remaining_balance": { - "type": ["null", "number"] - } - } - } - } - }, + "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["id"] @@ -635,69 +39,7 @@ { "stream": { "name": "pages", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "email": { - "type": ["null", "string"] - }, - "meta_title": { - "type": ["null", "string"] - }, - "body": { - "type": ["null", "string"] - }, - "feed": { - "type": ["null", "string"] - }, - "link": { - "type": ["null", "string"] - }, - "contact_fields": { - "type": ["null", "string"] - }, - "meta_keywords": { - "type": ["null", "string"] - }, - "meta_description": { - "type": ["null", "string"] - }, - "search_keywords": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - }, - "channel_id": { - "type": ["null", "number"] - }, - "name": { - "type": ["null", "string"] - }, - "is_visible": { - "type": ["null", "boolean"] - }, - "parent_id": { - "type": ["null", "number"] - }, - "sort_order": { - "type": ["null", "number"] - }, - "type": { - "type": ["null", "string"] - }, - "is_homepage": { - "type": ["null", "boolean"] - }, - "is_customers_only": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "number"] - } - } - }, + "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["id"] @@ -709,47 +51,7 @@ { "stream": { "name": "brands", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "page_title": { - "type": ["null", "string"] - }, - "meta_keywords": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "meta_description": { - "type": ["null", "string"] - }, - "search_keywords": { - "type": ["null", "string"] - }, - "image_url": { - "type": ["null", "string"] - }, - "custom_url": { - "type": ["null", "object"], - "properties": { - "url": { - "type": ["null", "string"] - }, - "is_customized": { - "type": ["null", "boolean"] - } - } - } - } - }, + "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["id"] @@ -761,68 +63,7 @@ { "stream": { "name": "categories", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "integer"] - }, - "parent_id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "views": { - "type": ["null", "integer"] - }, - "sort_order": { - "type": ["null", "integer"] - }, - "page_title": { - "type": ["null", "string"] - }, - "meta_keywords": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "meta_description": { - "type": ["null", "string"] - }, - "layout_file": { - "type": ["null", "string"] - }, - "image_url": { - "type": ["null", "string"] - }, - "is_visible": { - "type": ["null", "boolean"] - }, - "search_keywords": { - "type": ["null", "string"] - }, - "default_product_sort": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "object"], - "properties": { - "path": { - "type": ["null", "string"] - }, - "is_customized": { - "type": ["null", "boolean"] - } - } - } - } - }, + "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["id"] @@ -834,803 +75,7 @@ { "stream": { "name": "products", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "name": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "sku": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "weight": { - "type": ["null", "number"], - "format": "float" - }, - "width": { - "type": ["null", "number"], - "format": "float" - }, - "depth": { - "type": ["null", "number"], - "format": "float" - }, - "height": { - "type": ["null", "number"], - "format": "float" - }, - "price": { - "type": ["null", "number"], - "format": "float" - }, - "cost_price": { - "type": ["null", "number"], - "format": "float" - }, - "retail_price": { - "type": ["null", "number"], - "format": "float" - }, - "sale_price": { - "type": ["null", "number"], - "format": "float" - }, - "map_price": { - "type": ["null", "number"] - }, - "tax_class_id": { - "type": ["null", "number"] - }, - "product_tax_code": { - "type": ["null", "string"] - }, - "categories": { - "type": ["null", "array"], - "items": { - "type": ["null", "number"] - } - }, - "brand_id": { - "type": ["null", "number"] - }, - "inventory_level": { - "type": ["null", "number"] - }, - "inventory_warning_level": { - "type": ["null", "number"] - }, - "inventory_tracking": { - "type": ["null", "string"] - }, - "fixed_cost_shipping_price": { - "type": ["null", "number"], - "format": "float" - }, - "is_free_shipping": { - "type": ["null", "boolean"] - }, - "is_visible": { - "type": ["null", "boolean"] - }, - "is_featured": { - "type": ["null", "boolean"] - }, - "related_products": { - "type": ["null", "array"], - "items": { - "type": ["null", "number"] - } - }, - "warranty": { - "type": ["null", "string"] - }, - "bin_picking_number": { - "type": ["null", "string"] - }, - "layout_file": { - "type": ["null", "string"] - }, - "upc": { - "type": ["null", "string"] - }, - "search_keywords": { - "type": ["null", "string"] - }, - "availability": { - "type": ["null", "string"] - }, - "availability_description": { - "type": ["null", "string"] - }, - "gift_wrapping_options_type": { - "type": ["null", "string"] - }, - "gift_wrapping_options_list": { - "type": ["null", "array"], - "items": { - "type": ["null", "number"] - } - }, - "sort_order": { - "type": ["null", "number"] - }, - "condition": { - "type": ["null", "string"] - }, - "is_condition_shown": { - "type": ["null", "boolean"] - }, - "order_quantity_minimum": { - "type": ["null", "number"] - }, - "order_quantity_maximum": { - "type": ["null", "number"] - }, - "page_title": { - "type": ["null", "string"] - }, - "meta_keywords": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "meta_description": { - "type": ["null", "string"] - }, - "view_count": { - "type": ["null", "number"] - }, - "preorder_release_date": { - "type": ["null", "string"], - "format": "date-time" - }, - "preorder_message": { - "type": ["null", "string"] - }, - "is_preorder_only": { - "type": ["null", "boolean"] - }, - "is_price_hidden": { - "type": ["null", "boolean"] - }, - "price_hidden_label": { - "type": ["null", "string"] - }, - "custom_url": { - "type": ["null", "object"], - "title": "customUrl_Full", - "properties": { - "url": { - "type": ["null", "string"] - }, - "is_customized": { - "type": ["null", "boolean"] - } - } - }, - "open_graph_type": { - "type": ["null", "string"] - }, - "open_graph_title": { - "type": ["null", "string"] - }, - "open_graph_description": { - "type": ["null", "string"] - }, - "open_graph_use_meta_description": { - "type": ["null", "boolean"] - }, - "open_graph_use_product_name": { - "type": ["null", "boolean"] - }, - "open_graph_use_image": { - "type": ["null", "boolean"] - }, - "brand_name or brand_id": { - "type": ["null", "string"] - }, - "gtin": { - "type": ["null", "string"] - }, - "mpn": { - "type": ["null", "string"] - }, - "reviews_rating_sum": { - "type": ["null", "number"] - }, - "reviews_count": { - "type": ["null", "number"] - }, - "total_sold": { - "type": ["null", "number"] - }, - "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "title": "productCustomField_Put", - "required": ["name", "value"], - "properties": { - "id": { - "type": ["null", "number"] - }, - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - } - }, - "bulk_pricing_rules": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "title": "bulkPricingRule_Full", - "required": ["quantity_min", "quantity_max", "type", "amount"], - "properties": { - "id": { - "type": ["null", "number"] - }, - "quantity_min": { - "type": ["null", "number"] - }, - "quantity_max": { - "type": ["null", "number"] - }, - "type": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "number"] - } - } - } - }, - "images": { - "type": ["null", "array"], - "items": { - "title": "productImage_Full", - "type": ["null", "object"], - "properties": { - "image_file": { - "type": ["null", "string"] - }, - "is_thumbnail": { - "type": ["null", "boolean"] - }, - "sort_order": { - "type": ["null", "number"] - }, - "description": { - "type": ["null", "string"] - }, - "image_url": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "number"] - }, - "product_id": { - "type": ["null", "number"] - }, - "url_zoom": { - "type": ["null", "string"] - }, - "url_standard": { - "type": ["null", "string"] - }, - "url_thumbnail": { - "type": ["null", "string"] - }, - "url_tiny": { - "type": ["null", "string"] - }, - "date_modified": { - "format": "date-time", - "type": ["null", "string"] - } - } - } - }, - "videos": { - "type": ["null", "array"], - "items": { - "title": "productVideo_Full", - "type": ["null", "object"], - "properties": { - "title": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "sort_order": { - "type": ["null", "number"] - }, - "type": { - "type": ["null", "string"] - }, - "video_id": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "number"] - }, - "product_id": { - "type": ["null", "number"] - }, - "length": { - "type": ["null", "string"] - } - } - } - }, - "date_created": { - "type": ["null", "string"], - "format": "date-time" - }, - "date_modified": { - "type": ["null", "string"], - "format": "date-time" - }, - "id": { - "type": ["null", "number"] - }, - "base_variant_id": { - "type": ["null", "number"] - }, - "calculated_price": { - "type": ["null", "number"], - "format": "float" - }, - "options": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "title": "productOption_Base", - "properties": { - "id": { - "type": ["null", "number"] - }, - "product_id": { - "type": ["null", "number"] - }, - "display_name": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "config": { - "type": ["null", "object"], - "title": "productOptionConfig_Full", - "properties": { - "default_value": { - "type": ["null", "string"] - }, - "checked_by_default": { - "type": ["null", "boolean"] - }, - "checkbox_label": { - "type": ["null", "string"] - }, - "date_limited": { - "type": ["null", "boolean"] - }, - "date_limit_mode": { - "type": ["null", "string"] - }, - "date_earliest_value": { - "type": ["null", "string"], - "format": "date" - }, - "date_latest_value": { - "type": ["null", "string"], - "format": "date" - }, - "file_types_mode": { - "type": ["null", "string"] - }, - "file_types_supported": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "file_types_other": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "file_max_size": { - "type": ["null", "number"] - }, - "text_characters_limited": { - "type": ["null", "boolean"] - }, - "text_min_length": { - "type": ["null", "number"] - }, - "text_max_length": { - "type": ["null", "number"] - }, - "text_lines_limited": { - "type": ["null", "boolean"] - }, - "text_max_lines": { - "type": ["null", "number"] - }, - "number_limited": { - "type": ["null", "boolean"] - }, - "number_limit_mode": { - "type": ["null", "string"] - }, - "number_lowest_value": { - "type": ["null", "number"] - }, - "number_highest_value": { - "type": ["null", "number"] - }, - "number_numbers_only": { - "type": ["null", "boolean"] - }, - "product_list_adjusts_inventory": { - "type": ["null", "boolean"] - }, - "product_list_adjusts_pricing": { - "type": ["null", "boolean"] - }, - "product_list_shipping_calc": { - "type": ["null", "string"] - } - } - }, - "sort_order": { - "type": ["null", "number"] - }, - "option_values": { - "title": "productOptionOptionValue_Full", - "type": ["null", "object"], - "required": ["label", "sort_order"], - "properties": { - "is_default": { - "type": ["null", "boolean"] - }, - "label": { - "type": ["null", "string"] - }, - "sort_order": { - "type": ["null", "number"] - }, - "value_data": { - "type": ["object", "null"] - }, - "id": { - "type": ["null", "number"] - } - } - } - } - } - }, - "modifiers": { - "type": ["null", "array"], - "items": { - "title": "productModifier_Full", - "type": ["null", "object"], - "required": ["type", "required"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "required": { - "type": ["null", "boolean"] - }, - "sort_order": { - "type": ["null", "number"] - }, - "config": { - "type": ["null", "object"], - "title": "config_Full", - "properties": { - "default_value": { - "type": ["null", "string"] - }, - "checked_by_default": { - "type": ["null", "boolean"] - }, - "checkbox_label": { - "type": ["null", "string"] - }, - "date_limited": { - "type": ["null", "boolean"] - }, - "date_limit_mode": { - "type": ["null", "string"] - }, - "date_earliest_value": { - "type": ["null", "string"], - "format": "date" - }, - "date_latest_value": { - "type": ["null", "string"], - "format": "date" - }, - "file_types_mode": { - "type": ["null", "string"] - }, - "file_types_supported": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "file_types_other": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "file_max_size": { - "type": ["null", "number"] - }, - "text_characters_limited": { - "type": ["null", "boolean"] - }, - "text_min_length": { - "type": ["null", "number"] - }, - "text_max_length": { - "type": ["null", "number"] - }, - "text_lines_limited": { - "type": ["null", "boolean"] - }, - "text_max_lines": { - "type": ["null", "number"] - }, - "number_limited": { - "type": ["null", "boolean"] - }, - "number_limit_mode": { - "type": ["null", "string"] - }, - "number_lowest_value": { - "type": ["null", "number"] - }, - "number_highest_value": { - "type": ["null", "number"] - }, - "number_numbers_only": { - "type": ["null", "boolean"] - }, - "product_list_adjusts_inventory": { - "type": ["null", "boolean"] - }, - "product_list_adjusts_pricing": { - "type": ["null", "boolean"] - }, - "product_list_shipping_calc": { - "type": ["null", "string"] - } - } - }, - "display_name": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "number"] - }, - "product_id": { - "type": ["null", "number"] - }, - "name": { - "type": ["null", "string"] - }, - "option_values": { - "type": ["null", "array"], - "items": { - "title": "productModifierOptionValue_Full", - "type": ["null", "object"], - "required": ["label", "sort_order"], - "properties": { - "is_default": { - "type": ["null", "boolean"] - }, - "label": { - "type": ["null", "string"] - }, - "sort_order": { - "type": ["null", "number"] - }, - "value_data": { - "type": ["object", "null"] - }, - "adjusters": { - "type": ["null", "object"], - "title": "adjusters_Full", - "properties": { - "price": { - "type": ["null", "object"], - "title": "adjuster_Full", - "properties": { - "adjuster": { - "type": ["null", "string"] - }, - "adjuster_value": { - "type": ["null", "number"] - } - } - }, - "weight": { - "type": ["null", "object"], - "title": "adjuster_Full", - "properties": { - "adjuster": { - "type": ["null", "string"] - }, - "adjuster_value": { - "type": ["null", "number"] - } - } - }, - "image_url": { - "type": ["null", "string"] - }, - "purchasing_disabled": { - "type": ["null", "object"], - "properties": { - "status": { - "type": ["null", "boolean"] - }, - "message": { - "type": ["null", "string"] - } - } - } - } - }, - "id": { - "type": ["null", "number"] - }, - "option_id": { - "type": ["null", "number"] - } - } - } - } - } - } - }, - "option_set_id": { - "type": ["null", "number"] - }, - "option_set_display": { - "type": ["null", "string"] - }, - "variants": { - "type": ["null", "array"], - "items": { - "title": "productVariant_Full", - "type": ["null", "object"], - "properties": { - "cost_price": { - "type": ["null", "number"], - "format": "double" - }, - "price": { - "type": ["null", "number"], - "format": "double" - }, - "sale_price": { - "type": ["null", "number"], - "format": "double" - }, - "retail_price": { - "type": ["null", "number"], - "format": "double" - }, - "weight": { - "type": ["null", "number"], - "format": "double" - }, - "width": { - "type": ["null", "number"], - "format": "double" - }, - "height": { - "type": ["null", "number"], - "format": "double" - }, - "depth": { - "type": ["null", "number"], - "format": "double" - }, - "is_free_shipping": { - "type": ["null", "boolean"] - }, - "fixed_cost_shipping_price": { - "type": ["null", "number"], - "format": "double" - }, - "purchasing_disabled": { - "type": ["null", "boolean"] - }, - "purchasing_disabled_message": { - "type": ["null", "string"] - }, - "upc": { - "type": ["null", "string"] - }, - "inventory_level": { - "type": ["null", "number"] - }, - "inventory_warning_level": { - "type": ["null", "number"] - }, - "bin_picking_number": { - "type": ["null", "string"] - }, - "mpn": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "number"] - }, - "product_id": { - "type": ["null", "number"] - }, - "sku": { - "type": ["null", "string"] - }, - "sku_id": { - "type": ["null", "number"] - }, - "option_values": { - "type": ["null", "array"], - "items": { - "title": "productVariantOptionValue_Full", - "type": ["null", "object"], - "properties": { - "option_display_name": { - "type": ["null", "string"] - }, - "label": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "number"] - }, - "option_id": { - "type": ["null", "number"] - } - } - } - }, - "calculated_price": { - "type": ["null", "number"], - "format": "double" - }, - "calculated_weight": { - "type": "number" - } - } - } - } - } - }, + "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["date_modified"] @@ -1642,47 +87,7 @@ { "stream": { "name": "channels", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "number"] - }, - "external_id": { - "type": ["null", "string"] - }, - "is_listable_from_ui": { - "type": ["null", "boolean"] - }, - "is_visible": { - "type": ["null", "boolean"] - }, - "status": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "platform": { - "type": ["null", "boolean"] - }, - "date_created": { - "type": ["null", "string"], - "format": "date-time" - }, - "date_modified": { - "type": ["null", "string"], - "format": "date-time" - }, - "icon_url": { - "type": ["null", "string"] - } - } - }, + "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["date_modified"] @@ -1694,185 +99,7 @@ { "stream": { "name": "store", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "account_uuid": { - "type": ["null", "string"] - }, - "domain": { - "type": ["null", "string"] - }, - "secure_url": { - "type": ["null", "string"] - }, - "control_panel_base_url": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "first_name": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "address": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "admin_email": { - "type": ["null", "string"] - }, - "order_email": { - "type": ["null", "string"] - }, - "favicon_url": { - "type": ["null", "string"] - }, - "timezone": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "raw_offset": { - "type": ["null", "number"] - }, - "dst_offset": { - "type": ["null", "number"] - }, - "date_format": { - "type": ["null", "object"], - "properties": { - "display": { - "type": ["null", "string"] - }, - "export": { - "type": ["null", "string"] - }, - "extended_display": { - "type": ["null", "string"] - } - } - } - } - }, - "language": { - "type": ["null", "string"] - }, - "currency": { - "type": ["null", "string"] - }, - "currency_symbol": { - "type": ["null", "string"] - }, - "decimal_separator": { - "type": ["null", "string"] - }, - "thousands_separator": { - "type": ["null", "string"] - }, - "decimal_places": { - "type": ["null", "string"] - }, - "currency_symbol_location": { - "type": ["null", "string"] - }, - "weight_units": { - "type": ["null", "string"] - }, - "dimension_units": { - "type": ["null", "string"] - }, - "dimension_decimal_places": { - "type": ["null", "string"] - }, - "dimension_decimal_token": { - "type": ["null", "string"] - }, - "dimension_thousands_token": { - "type": ["null", "string"] - }, - "plan_name": { - "type": ["null", "string"] - }, - "plan_level": { - "type": ["null", "string"] - }, - "industry": { - "type": ["null", "string"] - }, - "logo": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "url": { - "type": ["null", "string"] - } - } - } - }, - "is_price_entered_with_tax": { - "type": ["null", "boolean"] - }, - "store_id": { - "type": ["null", "number"] - }, - "default_site_id": { - "type": ["null", "number"] - }, - "default_channel_id": { - "type": ["null", "number"] - }, - "active_comparison_modules": { - "type": ["null", "array"] - }, - "features": { - "type": ["null", "object"], - "properties": { - "stencil_enabled": { - "type": ["null", "boolean"] - }, - "sitewidehttps_enabled": { - "type": ["null", "boolean"] - }, - "facebook_catalog_id": { - "type": ["null", "string"] - }, - "checkout_type": { - "type": ["null", "string"] - }, - "wishlists_enabled": { - "type": ["null", "boolean"] - }, - "graphql_storefront_api_enabled": { - "type": ["null", "boolean"] - }, - "shopper_consent_tracking_enabled": { - "type": ["null", "boolean"] - }, - "multi_storefront_enabled": { - "type": ["null", "boolean"] - } - } - } - } - }, + "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["store_id"] @@ -1884,180 +111,7 @@ { "stream": { "name": "order_products", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "integer"] - }, - "order_id": { - "type": ["null", "integer"] - }, - "product_id": { - "type": ["null", "integer"] - }, - "variant_id": { - "type": ["null", "integer"] - }, - "order_address_id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "name_customer": { - "type": ["null", "string"] - }, - "name_merchant": { - "type": ["null", "string"] - }, - "sku": { - "type": ["null", "string"] - }, - "upc": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "base_price": { - "type": ["null", "string"] - }, - "price_ex_tax": { - "type": ["null", "string"] - }, - "price_inc_tax": { - "type": ["null", "string"] - }, - "price_tax": { - "type": ["null", "string"] - }, - "base_total": { - "type": ["null", "string"] - }, - "total_ex_tax": { - "type": ["null", "string"] - }, - "total_inc_tax": { - "type": ["null", "string"] - }, - "total_tax": { - "type": ["null", "string"] - }, - "weight": { - "type": ["null", "string"] - }, - "width": { - "type": ["null", "string"] - }, - "height": { - "type": ["null", "string"] - }, - "depth": { - "type": ["null", "string"] - }, - "quantity": { - "type": ["null", "number"] - }, - "base_cost_price": { - "type": ["null", "string"] - }, - "cost_price_inc_tax": { - "type": ["null", "string"] - }, - "cost_price_ex_tax": { - "type": ["null", "string"] - }, - "cost_price_tax": { - "type": ["null", "string"] - }, - "is_refunded": { - "type": ["null", "boolean"] - }, - "quantity_refunded": { - "type": ["null", "number"] - }, - "refund_amount": { - "type": ["null", "string"] - }, - "return_id": { - "type": ["null", "number"] - }, - "wrapping_id": { - "type": ["null", "string"] - }, - "wrapping_name": { - "type": ["null", "string"] - }, - "base_wrapping_cost": { - "type": ["null", "string"] - }, - "wrapping_cost_ex_tax": { - "type": ["null", "string"] - }, - "wrapping_cost_inc_tax": { - "type": ["null", "string"] - }, - "wrapping_cost_tax": { - "type": ["null", "string"] - }, - "wrapping_message": { - "type": ["null", "string"] - }, - "quantity_shipped": { - "type": ["null", "number"] - }, - "event_name": { - "type": ["null", "string"] - }, - "event_date": { - "type": ["null", "string"] - }, - "fixed_shipping_cost": { - "type": ["null", "string"] - }, - "ebay_item_id": { - "type": ["null", "string"] - }, - "ebay_transaction_id": { - "type": ["null", "string"] - }, - "option_set_id": { - "type": ["null", "integer"] - }, - "parent_order_product_id": { - "type": ["null", "integer"] - }, - "is_bundled_product": { - "type": ["null", "boolean"] - }, - "bin_picking_number": { - "type": ["null", "string"] - }, - "external_id": { - "type": ["null", "string"] - }, - "fulfillment_source": { - "type": ["null", "string"] - }, - "brand": { - "type": ["null", "string"] - }, - "gift_certificate_id": { - "type": ["null", "integer"] - }, - "applied_discounts": { - "type": ["null", "array"] - }, - "product_options": { - "type": ["null", "array"] - }, - "configurable_fields": { - "type": ["null", "array"] - } - } - }, + "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, "default_cursor_field": ["id"] diff --git a/airbyte-integrations/connectors/source-bigcommerce/metadata.yaml b/airbyte-integrations/connectors/source-bigcommerce/metadata.yaml index 0ac3c370391c..fd54e9032d49 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/metadata.yaml +++ b/airbyte-integrations/connectors/source-bigcommerce/metadata.yaml @@ -1,24 +1,28 @@ data: - ab_internal: - ql: 200 - sl: 100 + allowedHosts: + hosts: + - api.bigcommerce.com + registries: + oss: + enabled: false + cloud: + enabled: false connectorSubtype: api connectorType: source definitionId: 59c5501b-9f95-411e-9269-7143c939adbd - dockerImageTag: 0.1.10 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-bigcommerce documentationUrl: https://docs.airbyte.com/integrations/sources/bigcommerce githubIssueLabel: source-bigcommerce icon: bigcommerce.svg license: MIT - name: BigCommerce - registries: - cloud: - enabled: true - oss: - enabled: true + name: Bigcommerce + releaseDate: 2021-08-19 releaseStage: alpha supportLevel: community tags: - - language:python + - language:low-code + ab_internal: + sl: 100 + ql: 200 metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-bigcommerce/setup.py b/airbyte-integrations/connectors/source-bigcommerce/setup.py index 7b6eed1b049c..7189d313888f 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/setup.py +++ b/airbyte-integrations/connectors/source-bigcommerce/setup.py @@ -6,13 +6,13 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", + "airbyte-cdk~=0.1", ] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", + "pytest~=6.2", "pytest-mock~=3.6.1", - "pytest~=6.1", ] setup( @@ -22,7 +22,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/__init__.py b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/__init__.py index 7b9287ea4c26..2a85736c808d 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/__init__.py +++ b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/__init__.py @@ -1,25 +1,5 @@ # -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/components.py b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/components.py new file mode 100644 index 000000000000..d9cdab67a57d --- /dev/null +++ b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/components.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import dataclass +from typing import Optional + +import dpath.util +import pendulum +from airbyte_cdk.sources.declarative.transformations.add_fields import AddFields +from airbyte_cdk.sources.declarative.types import Config, Record, StreamSlice, StreamState + + +@dataclass +class DateTimeTransformer(AddFields): + def transform( + self, + record: Record, + config: Optional[Config] = None, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + ) -> Record: + + kwargs = {"record": record, "stream_state": stream_state, "stream_slice": stream_slice} + for parsed_field in self._parsed_fields: + date_time = parsed_field.value.eval(config, **kwargs) + new_date_time = str(pendulum.from_format(date_time, "ddd, D MMM YYYY HH:mm:ss ZZ")) + dpath.util.new(record, parsed_field.path, new_date_time) + return record diff --git a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/manifest.yaml b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/manifest.yaml new file mode 100644 index 000000000000..4b06dd2f882f --- /dev/null +++ b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/manifest.yaml @@ -0,0 +1,338 @@ +version: 0.50.2 +type: DeclarativeSource + +check: + type: CheckStream + stream_names: + - customers + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + inject_into: request_parameter + field_name: limit + type: RequestOption + pagination_strategy: + type: PageIncrement + start_from_page: 1 + page_size: 250 + date_modified_incremental_sync: + type: DatetimeBasedCursor + cursor_field: date_modified + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%dT%H:%M:%S+00:00" + datetime_format: "%Y-%m-%d" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%d" + start_time_option: + inject_into: request_parameter + field_name: date_modified:min + type: RequestOption + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + base_id_incremental_sync: + type: DatetimeBasedCursor + cursor_field: id + cursor_datetime_formats: + - "%s" + datetime_format: "%Y-%m-%d" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%d" + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + id_incremental_sync: + $ref: "#/definitions/base_id_incremental_sync" + start_time_option: + type: RequestOption + field_name: date_modified:min + inject_into: request_parameter + order_id_partition_router: + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: order_id + stream: + $ref: "#/definitions/orders_stream" + requester: + type: HttpRequester + url_base: https://api.bigcommerce.com/stores/{{ config["store_hash"] }}/ + http_method: GET + request_headers: + Accept: application/json + Content-Type: application/json + authenticator: + type: ApiKeyAuthenticator + api_token: '{{ config["access_token"] }}' + inject_into: + type: RequestOption + inject_into: header + field_name: X-Auth-Token + request_body_json: {} + customers_stream: + type: DeclarativeStream + name: customers + primary_key: "id" + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: v3/customers + request_parameters: + sort: date_modified:asc + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/paginator" + incremental_sync: + $ref: "#/definitions/date_modified_incremental_sync" + orders_stream: + type: DeclarativeStream + name: orders + primary_key: "id" + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: v2/orders + request_parameters: + sort: date_modified:asc + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + $ref: "#/definitions/paginator" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: date_modified + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%SZ" + - "%Y-%m-%dT%H:%M:%S+00:00" + datetime_format: "%Y-%m-%d" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%d" + start_time_option: + inject_into: request_parameter + field_name: min_date_modified + type: RequestOption + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + transformations: + - type: CustomTransformation + class_name: source_bigcommerce.components.DateTimeTransformer + fields: + - path: + - date_modified + value: "{{ record.date_modified }}" + transactions_stream: + type: DeclarativeStream + name: transactions + primary_key: "id" + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: v2/orders/{{ stream_partition.order_id }}/transactions + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/paginator" + partition_router: + $ref: "#/definitions/order_id_partition_router" + incremental_sync: + $ref: "#/definitions/id_incremental_sync" + pages_stream: + type: DeclarativeStream + name: pages + primary_key: "id" + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: v3/content/pages + request_parameters: + sort: date_modified:asc + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/paginator" + incremental_sync: + $ref: "#/definitions/id_incremental_sync" + products_stream: + type: DeclarativeStream + name: products + primary_key: "id" + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: v3/catalog/products + request_parameters: + sort: date_modified + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/paginator" + incremental_sync: + $ref: "#/definitions/date_modified_incremental_sync" + channels_stream: + type: DeclarativeStream + name: channels + primary_key: "id" + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: /v3/channels + request_parameters: + sort: date_modified + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/paginator" + incremental_sync: + $ref: "#/definitions/date_modified_incremental_sync" + store_stream: + type: DeclarativeStream + name: store + primary_key: "id" + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: v2/store + request_parameters: + sort: date_modified:asc + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + $ref: "#/definitions/paginator" + order_products_stream: + type: DeclarativeStream + name: order_products + primary_key: "id" + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: v2/orders/{{ stream_partition.order_id }}/products + request_parameters: + sort: date_modified:asc + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + $ref: "#/definitions/paginator" + partition_router: + $ref: "#/definitions/order_id_partition_router" + incremental_sync: + $ref: "#/definitions/base_id_incremental_sync" + brands_stream: + type: DeclarativeStream + name: brands + primary_key: "id" + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: v3/catalog/brands + request_parameters: + sort: id + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/paginator" + incremental_sync: + $ref: "#/definitions/base_id_incremental_sync" + categories_stream: + type: DeclarativeStream + name: categories + primary_key: "id" + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: v3/catalog/categories + request_parameters: + sort: id + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/paginator" + incremental_sync: + $ref: "#/definitions/base_id_incremental_sync" + +streams: + - "#/definitions/customers_stream" + - "#/definitions/orders_stream" + - "#/definitions/transactions_stream" + - "#/definitions/pages_stream" + - "#/definitions/products_stream" + - "#/definitions/channels_stream" + - "#/definitions/store_stream" + - "#/definitions/order_products_stream" + - "#/definitions/brands_stream" + - "#/definitions/categories_stream" + +spec: + documentation_url: https://docs.airbyte.com/integrations/sources/bigcommerce + type: Spec + connection_specification: + additionalProperties: true + $schema: http://json-schema.org/draft-07/schema# + type: object + required: + - start_date + - access_token + - store_hash + properties: + start_date: + type: string + title: Start date + description: "The date you would like to replicate data. Format: YYYY-MM-DD." + examples: ["2021-01-01"] + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ + order: 0 + access_token: + type: string + title: Access Token + description: "Access Token for making authenticated requests." + airbyte_secret: true + order: 1 + store_hash: + title: Store Hash + description: >- + The hash code of the store. For https://api.bigcommerce.com/stores/HASH_CODE/v3/, The store's hash code is 'HASH_CODE'. + type: string + order: 2 diff --git a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/categories.json b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/categories.json index 83db91ea3cca..6b26ee372e55 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/categories.json +++ b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/categories.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] @@ -47,7 +48,7 @@ "default_product_sort": { "type": ["null", "string"] }, - "url": { + "custom_url": { "type": ["null", "object"], "properties": { "path": { diff --git a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/channels.json b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/channels.json index 1cc51924ea59..78e6edc8221e 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/channels.json +++ b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/channels.json @@ -36,6 +36,9 @@ }, "icon_url": { "type": ["null", "string"] + }, + "is_enabled": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/customers.json b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/customers.json index c5b367d15997..e00906cff851 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/customers.json +++ b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/customers.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "email": { "type": ["null", "string"] @@ -58,6 +59,7 @@ "type": ["null", "array"], "items": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "first_name": { "type": ["null", "string"] @@ -108,7 +110,7 @@ "type": ["null", "object"], "properties": { "name": { - "type": ["name", "string"] + "type": ["null", "string"] }, "value": { "oneOf": [ @@ -126,7 +128,7 @@ "type": ["null", "object"], "properties": { "name": { - "type": ["name", "string"] + "type": ["null", "string"] }, "value": { "oneOf": [ @@ -158,6 +160,9 @@ }, "channel_ids": { "type": ["null", "array"] + }, + "origin_channel_id": { + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/store.json b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/store.json index 66955e00e224..22a6188fa281 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/store.json +++ b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/schemas/store.json @@ -35,6 +35,9 @@ "country": { "type": ["null", "string"] }, + "country_code": { + "type": ["null", "string"] + }, "phone": { "type": ["null", "string"] }, @@ -91,7 +94,7 @@ "type": ["null", "string"] }, "decimal_places": { - "type": ["null", "string"] + "type": ["null", "integer"] }, "currency_symbol_location": { "type": ["null", "string"] @@ -103,7 +106,7 @@ "type": ["null", "string"] }, "dimension_decimal_places": { - "type": ["null", "string"] + "type": ["null", "integer"] }, "dimension_decimal_token": { "type": ["null", "string"] @@ -117,6 +120,9 @@ "plan_level": { "type": ["null", "string"] }, + "plan_is_trial": { + "type": ["null", "boolean"] + }, "industry": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/source.py b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/source.py index a31e25e705ca..997d0eb20418 100644 --- a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/source.py +++ b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/source.py @@ -2,361 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from abc import ABC -from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import pendulum -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator -from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +WARNING: Do not modify this file. +""" -class BigcommerceStream(HttpStream, ABC): - # Latest Stable Release - api_version = "v3" - # Page size - limit = 250 - # Define primary key as sort key for full_refresh, or very first sync for incremental_refresh - primary_key = "id" - order_field = "date_modified:asc" - filter_field = "date_modified:min" - data = "data" - - transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization) - - def __init__(self, start_date: str, store_hash: str, access_token: str, **kwargs): - super().__init__(**kwargs) - self.start_date = start_date - self.store_hash = store_hash - self.access_token = access_token - - @transformer.registerCustomTransform - def transform_function(original_value: Any, field_schema: Dict[str, Any]) -> Any: - """ - This functions tries to handle the various date-time formats BigCommerce API returns and normalize the values to isoformat. - """ - if "format" in field_schema and field_schema["format"] == "date-time": - if not original_value: # Some dates are empty strings: "". - return None - transformed_value = None - supported_formats = ["YYYY-MM-DD", "YYYY-MM-DDTHH:mm:ssZZ", "YYYY-MM-DDTHH:mm:ss[Z]", "ddd, D MMM YYYY HH:mm:ss ZZ"] - for format in supported_formats: - try: - transformed_value = str(pendulum.from_format(original_value, format)) # str() returns isoformat - except ValueError: - continue - if not transformed_value: - raise ValueError(f"Unsupported date-time format for {original_value}") - return transformed_value - return original_value - - @property - def url_base(self) -> str: - return f"https://api.bigcommerce.com/stores/{self.store_hash}/{self.api_version}/" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - json_response = response.json() - meta = json_response.get("meta", None) - if meta: - pagination = meta.get("pagination", None) - if pagination and pagination.get("current_page") < pagination.get("total_pages"): - return dict(page=pagination.get("current_page") + 1) - else: - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params = {"limit": self.limit} - params.update({"sort": self.order_field}) - if next_page_token: - params.update(**next_page_token) - else: - params[self.filter_field] = self.start_date - return params - - def request_headers( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> Mapping[str, Any]: - headers = super().request_headers(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - headers.update({"Accept": "application/json", "Content-Type": "application/json"}) - return headers - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - json_response = response.json() - records = json_response.get(self.data, []) if self.data is not None else json_response - yield from records - - -class IncrementalBigcommerceStream(BigcommerceStream, ABC): - # Getting page size as 'limit' from parent class - @property - def limit(self): - return super().limit - - # Setting the check point interval to the limit of the records output - state_checkpoint_interval = limit - # Setting the default cursor field for all streams - cursor_field = "date_modified" - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - return {self.cursor_field: max(latest_record.get(self.cursor_field, ""), current_stream_state.get(self.cursor_field, ""))} - - def request_params(self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs): - params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) - # If there is a next page token then we should only send pagination-related parameters. - if stream_state: - params[self.filter_field] = stream_state.get(self.cursor_field) - else: - params[self.filter_field] = self.start_date - return params - - def filter_records_newer_than_state(self, stream_state: Mapping[str, Any] = None, records_slice: Mapping[str, Any] = None) -> Iterable: - if stream_state: - for record in records_slice: - if record[self.cursor_field] >= stream_state.get(self.cursor_field): - yield record - else: - yield from records_slice - - -class OrderSubstream(IncrementalBigcommerceStream): - def read_records( - self, stream_state: Mapping[str, Any] = None, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs - ) -> Iterable[Mapping[str, Any]]: - orders_stream = Orders( - authenticator=self.authenticator, start_date=self.start_date, store_hash=self.store_hash, access_token=self.access_token - ) - for data in orders_stream.read_records(sync_mode=SyncMode.full_refresh): - slice = super().read_records(stream_slice={"order_id": data["id"]}, **kwargs) - yield from self.filter_records_newer_than_state(stream_state=stream_state, records_slice=slice) - - -class Customers(IncrementalBigcommerceStream): - data_field = "customers" - - def path(self, **kwargs) -> str: - return f"{self.data_field}" - - -class Products(IncrementalBigcommerceStream): - data_field = "products" - # Override `order_field` because Products API does not accept `asc` value - order_field = "date_modified" - - def path(self, **kwargs) -> str: - return f"catalog/{self.data_field}" - - -class Orders(IncrementalBigcommerceStream): - data_field = "orders" - api_version = "v2" - order_field = "date_modified:asc" - filter_field = "min_date_modified" - page = 1 - - def path(self, **kwargs) -> str: - return f"{self.data_field}" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - return response.json() if len(response.content) > 0 else [] - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - if len(response.content) > 0 and len(response.json()) == self.limit: - self.page = self.page + 1 - return dict(page=self.page) - else: - return None - - -class Pages(IncrementalBigcommerceStream): - data_field = "pages" - cursor_field = "id" - - def path(self, **kwargs) -> str: - return f"content/{self.data_field}" - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - return {self.cursor_field: max(latest_record.get(self.cursor_field, 0), current_stream_state.get(self.cursor_field, 0))} - - def read_records( - self, stream_state: Mapping[str, Any] = None, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs - ) -> Iterable[Mapping[str, Any]]: - slice = super().read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice, stream_state=stream_state) - yield from self.filter_records_newer_than_state(stream_state=stream_state, records_slice=slice) - - -class Brands(IncrementalBigcommerceStream): - data_field = "brands" - cursor_field = "id" - order_field = "id" - - def path(self, **kwargs) -> str: - return f"catalog/{self.data_field}" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params = {"limit": self.limit} - params.update({"sort": self.order_field}) - if next_page_token: - params.update(**next_page_token) - return params - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - return {self.cursor_field: max(latest_record.get(self.cursor_field, 0), current_stream_state.get(self.cursor_field, 0))} - - def read_records( - self, stream_state: Mapping[str, Any] = None, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs - ) -> Iterable[Mapping[str, Any]]: - slice = super().read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice, stream_state=stream_state) - yield from self.filter_records_newer_than_state(stream_state=stream_state, records_slice=slice) - - -class Categories(IncrementalBigcommerceStream): - data_field = "categories" - cursor_field = "id" - order_field = "id" - - def path(self, **kwargs) -> str: - return f"catalog/{self.data_field}" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params = {"limit": self.limit} - params.update({"sort": self.order_field}) - if next_page_token: - params.update(**next_page_token) - return params - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - return {self.cursor_field: max(latest_record.get(self.cursor_field, 0), current_stream_state.get(self.cursor_field, 0))} - - def read_records( - self, stream_state: Mapping[str, Any] = None, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs - ) -> Iterable[Mapping[str, Any]]: - slice = super().read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice, stream_state=stream_state) - yield from self.filter_records_newer_than_state(stream_state=stream_state, records_slice=slice) - - -class Transactions(OrderSubstream): - data_field = "transactions" - cursor_field = "id" - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - order_id = stream_slice["order_id"] - return f"orders/{order_id}/{self.data_field}" - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - return {self.cursor_field: max(latest_record.get(self.cursor_field, 0), current_stream_state.get(self.cursor_field, 0))} - - def request_params( - self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = {"limit": self.limit} - return params - - -class OrderProducts(OrderSubstream): - api_version = "v2" - data_field = "products" - cursor_field = "id" - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - order_id = stream_slice["order_id"] - return f"orders/{order_id}/{self.data_field}" - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - return {self.cursor_field: max(latest_record.get(self.cursor_field, 0), current_stream_state.get(self.cursor_field, 0))} - - def request_params( - self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = {"limit": self.limit} - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - return response.json() if len(response.content) > 0 else [] - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - if len(response.content) > 0 and len(response.json()) == self.limit: - self.page = self.page + 1 - return dict(page=self.page) - else: - return None - - -class Channels(IncrementalBigcommerceStream): - data_field = "channels" - # Override `order_field` bacause Channels API do not acept `asc` value - order_field = "date_modified" - - def path(self, **kwargs) -> str: - return f"{self.data_field}" - - -class Store(BigcommerceStream): - data_field = "store" - cursor_field = "store_id" - api_version = "v2" - data = None - - def path(self, **kwargs) -> str: - return f"{self.data_field}" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - json_response = response.json() - yield from [json_response] - - -class BigcommerceAuthenticator(HttpAuthenticator): - def __init__(self, token: str): - self.token = token - - def get_auth_header(self) -> Mapping[str, Any]: - return {"X-Auth-Token": f"{self.token}"} - - -class SourceBigcommerce(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - store_hash = config["store_hash"] - access_token = config["access_token"] - api_version = "v3" - - headers = {"X-Auth-Token": access_token, "Accept": "application/json", "Content-Type": "application/json"} - url = f"https://api.bigcommerce.com/stores/{store_hash}/{api_version}/channels" - - try: - session = requests.get(url, headers=headers) - session.raise_for_status() - return True, None - except requests.exceptions.RequestException as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - - auth = BigcommerceAuthenticator(token=config["access_token"]) - args = { - "authenticator": auth, - "start_date": config["start_date"], - "store_hash": config["store_hash"], - "access_token": config["access_token"], - } - return [ - Customers(**args), - Pages(**args), - Orders(**args), - Transactions(**args), - Products(**args), - Channels(**args), - Store(**args), - OrderProducts(**args), - Brands(**args), - Categories(**args), - ] +# Declarative Source +class SourceBigcommerce(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/spec.json b/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/spec.json deleted file mode 100644 index 02b6ddf1f69f..000000000000 --- a/airbyte-integrations/connectors/source-bigcommerce/source_bigcommerce/spec.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/bigcommerce", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "BigCommerce Source CDK Specifications", - "type": "object", - "required": ["start_date", "store_hash", "access_token"], - "additionalProperties": true, - "properties": { - "start_date": { - "type": "string", - "title": "Start Date", - "description": "The date you would like to replicate data. Format: YYYY-MM-DD.", - "examples": ["2021-01-01"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" - }, - "store_hash": { - "type": "string", - "title": "Store Hash", - "description": "The hash code of the store. For https://api.bigcommerce.com/stores/HASH_CODE/v3/, The store's hash code is 'HASH_CODE'." - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "Access Token for making authenticated requests.", - "airbyte_secret": true - } - } - } -} diff --git a/airbyte-integrations/connectors/source-bigcommerce/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-bigcommerce/unit_tests/unit_test.py deleted file mode 100644 index 219ae0142c72..000000000000 --- a/airbyte-integrations/connectors/source-bigcommerce/unit_tests/unit_test.py +++ /dev/null @@ -1,7 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -def test_example_method(): - assert True diff --git a/airbyte-integrations/connectors/source-bigquery/Dockerfile b/airbyte-integrations/connectors/source-bigquery/Dockerfile deleted file mode 100644 index f3db983b9d4e..000000000000 --- a/airbyte-integrations/connectors/source-bigquery/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-bigquery - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-bigquery - -COPY --from=build /airbyte /airbyte - -# Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile. -LABEL io.airbyte.version=0.3.0 -LABEL io.airbyte.name=airbyte/source-bigquery diff --git a/airbyte-integrations/connectors/source-bigquery/acceptance-test-config.yml b/airbyte-integrations/connectors/source-bigquery/acceptance-test-config.yml deleted file mode 100644 index db0e62843784..000000000000 --- a/airbyte-integrations/connectors/source-bigquery/acceptance-test-config.yml +++ /dev/null @@ -1,30 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-bigquery:dev -acceptance_tests: - spec: - tests: - - spec_path: "src/test-integration/resources/expected_spec.json" - config_path: "src/test-integration/resources/dummy_config.json" - connection: - tests: - - config_path: "secrets/sat-config.json" - status: "succeed" - discovery: - tests: - - config_path: "secrets/sat-config.json" - basic_read: - tests: - - config_path: "secrets/sat-config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - full_refresh: - tests: - - config_path: "secrets/sat-config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" -# DISABLED DUE TO ISSUE WITH DB STATES NOT MATCHING ACCEPTANCE TESTS EXPECTATIONS (wrong key-values) -# incremental: -# tests: -# - config_path: "secrets/sat-config.json" -# configured_catalog_path: "integration_tests/configured_catalog_inc.json" diff --git a/airbyte-integrations/connectors/source-bigquery/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-bigquery/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-bigquery/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-bigquery/build.gradle b/airbyte-integrations/connectors/source-bigquery/build.gradle index 485db19ebea5..1733086c17c5 100644 --- a/airbyte-integrations/connectors/source-bigquery/build.gradle +++ b/airbyte-integrations/connectors/source-bigquery/build.gradle @@ -1,10 +1,26 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' - id 'airbyte-connector-acceptance-test' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.7.7' + features = ['db-sources'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileTestJava { + options.compilerArgs.remove("-Werror") + } + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.source.bigquery.BigQuerySource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] @@ -13,20 +29,7 @@ application { dependencies { implementation 'com.google.cloud:google-cloud-bigquery:2.23.2' implementation 'org.apache.commons:commons-lang3:3.11' - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation project(':airbyte-integrations:connectors:source-relational-db') - - testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) testImplementation 'org.apache.commons:commons-lang3:3.11' - - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-bigquery') - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + // integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-bigquery') } - diff --git a/airbyte-integrations/connectors/source-bigquery/metadata.yaml b/airbyte-integrations/connectors/source-bigquery/metadata.yaml index f6e8623837bb..ac97ec4a3da8 100644 --- a/airbyte-integrations/connectors/source-bigquery/metadata.yaml +++ b/airbyte-integrations/connectors/source-bigquery/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: source definitionId: bfd1ddf8-ae8a-4620-b1d7-55597d2ba08c - dockerImageTag: 0.3.0 + dockerImageTag: 0.4.0 dockerRepository: airbyte/source-bigquery documentationUrl: https://docs.airbyte.com/integrations/sources/bigquery githubIssueLabel: source-bigquery diff --git a/airbyte-integrations/connectors/source-bigquery/src/main/java/io/airbyte/integrations/source/bigquery/BigQuerySource.java b/airbyte-integrations/connectors/source-bigquery/src/main/java/io/airbyte/integrations/source/bigquery/BigQuerySource.java index 32fdaab9c3c5..d50d679efe27 100644 --- a/airbyte-integrations/connectors/source-bigquery/src/main/java/io/airbyte/integrations/source/bigquery/BigQuerySource.java +++ b/airbyte-integrations/connectors/source-bigquery/src/main/java/io/airbyte/integrations/source/bigquery/BigQuerySource.java @@ -4,29 +4,29 @@ package io.airbyte.integrations.source.bigquery; -import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifierList; -import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; -import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.queryTable; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifierList; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.queryTable; import com.fasterxml.jackson.databind.JsonNode; import com.google.cloud.bigquery.QueryParameterValue; import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.cloud.bigquery.Table; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.SqlDatabase; +import io.airbyte.cdk.db.bigquery.BigQueryDatabase; +import io.airbyte.cdk.db.bigquery.BigQuerySourceOperations; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.source.relationaldb.AbstractDbSource; +import io.airbyte.cdk.integrations.source.relationaldb.CursorInfo; +import io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils; +import io.airbyte.cdk.integrations.source.relationaldb.TableInfo; import io.airbyte.commons.functional.CheckedConsumer; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.stream.AirbyteStreamUtils; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.db.SqlDatabase; -import io.airbyte.db.bigquery.BigQueryDatabase; -import io.airbyte.db.bigquery.BigQuerySourceOperations; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.source.relationaldb.AbstractDbSource; -import io.airbyte.integrations.source.relationaldb.CursorInfo; -import io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils; -import io.airbyte.integrations.source.relationaldb.TableInfo; import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.CommonField; import io.airbyte.protocol.models.JsonSchemaType; @@ -55,6 +55,10 @@ public class BigQuerySource extends AbstractDbSource Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-bing-ads:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-bing-ads:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-bing-ads:dev . +# Running the spec command against your patched connector +docker run airbyte/source-bing-ads:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -92,45 +135,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-bing-ads:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-bing-ads:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-bing-ads:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install ".[tests]" -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-bing-ads test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-bing-ads:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-bing-ads:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-bing-ads:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -140,8 +154,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-bing-ads test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/bing-ads.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml index e4ae31eea384..13c858d87148 100644 --- a/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-bing-ads/acceptance-test-config.yml @@ -4,22 +4,26 @@ acceptance_tests: spec: tests: - spec_path: source_bing_ads/spec.json + backward_compatibility_tests_config: + disable_for_version: "1.11.0" discovery: tests: - config_path: secrets/config.json + backward_compatibility_tests_config: + disable_for_version: "1.4.0" connection: tests: - - config_path: secrets/config_old.json - status: succeed - config_path: secrets/config.json status: succeed + - config_path: secrets/config_no_date.json + status: succeed - config_path: integration_tests/invalid_config.json status: failed basic_read: tests: - config_path: secrets/config.json expect_records: - path: "integration_tests/expected_records.txt" + path: "integration_tests/expected_records.jsonl" extra_records: yes empty_streams: - name: account_performance_report_hourly @@ -30,14 +34,751 @@ acceptance_tests: bypass_reason: "Hourly reports are disabled, because sync is too long" - name: campaign_performance_report_hourly bypass_reason: "Hourly reports are disabled, because sync is too long" + - name: campaign_impression_performance_report_hourly + bypass_reason: "Empty report; hourly data fetched is limited to 180 days" + - name: keyword_performance_report_hourly + bypass_reason: "Hourly reports are disabled, because sync is too long" + - name: geographic_performance_report_hourly + bypass_reason: "Hourly reports are disabled, because sync is too long" + - name: age_gender_audience_report_hourly + bypass_reason: "Empty report; hourly data fetched is limited to 180 days" + - name: search_query_performance_report_hourly + bypass_reason: "Empty report; hourly data fetched is limited to 180 days" + - name: user_location_performance_report_hourly + bypass_reason: "Empty report; hourly data fetched is limited to 180 days" + - name: account_impression_performance_report_hourly + bypass_reason: "Empty report; hourly data fetched is limited to 180 days" + - name: ad_group_impression_performance_report_hourly + bypass_reason: "Empty report; hourly data fetched is limited to 180 days" + - name: app_install_ads + bypass_reason: "Can not populate; new campaign with link to app needed; feature is not available yet" + - name: app_install_ad_labels + bypass_reason: "Can not populate; depends on stream app_install_ads" + #### TODO: remove *_report_monthly after all become populated on December, 1 + - name: account_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: ad_group_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: ad_group_impression_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: ad_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: campaign_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: campaign_impression_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: keyword_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: geographic_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: age_gender_audience_report_monthly + bypass_reason: "Campaign is still in progress" + - name: search_query_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: user_location_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: account_impression_performance_report_monthly + bypass_reason: "Campaign is still in progress" + ### For stream below start date in config is not relevant, empty data + - name: keyword_labels + bypass_reason: "This stream is tested without start date" + - name: keywords + bypass_reason: "This stream is tested without start date" + - name: ad_group_labels + bypass_reason: "This stream is tested without start date" + - name: labels + bypass_reason: "This stream is tested without start date" + - name: campaign_labels + bypass_reason: "This stream is tested without start date" + ignored_fields: + account_impression_performance_report_weekly: + - name: Ctr + bypass_reason: "dynamic field" + - name: Impressions + bypass_reason: "dynamic field" + - name: AverageCpm + bypass_reason: "dynamic field" + - name: LowQualityImpressionsPercent + bypass_reason: "dynamic field" + - name: Clicks + bypass_reason: "dynamic field" + - name: AverageCpc + bypass_reason: "dynamic field" + - name: Spend + bypass_reason: "dynamic field" + - name: ConversionRate + bypass_reason: "dynamic field" + - name: LowQualityClicksPercent + bypass_reason: "dynamic field" + - name: LowQualityImpressions + bypass_reason: "dynamic field" + - name: ReturnOnAdSpend + bypass_reason: "dynamic field" + - name: AllConversionRate + bypass_reason: "dynamic field" + - name: AllReturnOnAdSpend + bypass_reason: "dynamic field" + age_gender_audience_report_daily: + - name: Impressions + bypass_reason: "dynamic field" + age_gender_audience_report_weekly: + - name: Impressions + bypass_reason: "dynamic field" + keyword_performance_report_weekly: + - name: Mainline1Bid + bypass_reason: "dynamic field" + - name: MainlineBid + bypass_reason: "dynamic field" + - name: FirstPageBid + bypass_reason: "dynamic field" + campaign_impression_performance_report_weekly: + - name: Impressions + bypass_reason: "dynamic field" + - name: LowQualityImpressions + bypass_reason: "dynamic field" + - name: LowQualityImpressionsPercent + bypass_reason: "dynamic field" + - name: Clicks + bypass_reason: "dynamic field" + - name: Ctr + bypass_reason: "dynamic field" + - name: AverageCpc + bypass_reason: "dynamic field" + - name: Spend + bypass_reason: "dynamic field" + - name: LowQualityClicksPercent + bypass_reason: "dynamic field" + - name: ReturnOnAdSpend + bypass_reason: "dynamic field" + - name: AllReturnOnAdSpend + bypass_reason: "dynamic field" + - name: AverageCpm + bypass_reason: "dynamic field" + - name: LandingPageExperience + bypass_reason: "dynamic field" + campaign_performance_report_weekly: + - name: Impressions + bypass_reason: "dynamic field" + - name: DeviceType + bypass_reason: "dynamic field" + - name: DeviceOS + bypass_reason: "dynamic field" + - name: LowQualityImpressions + bypass_reason: "dynamic field" + - name: Clicks + bypass_reason: "dynamic field" + - name: Ctr + bypass_reason: "dynamic field" + - name: Spend + bypass_reason: "dynamic field" + - name: LandingPageExperience + bypass_reason: "dynamic field" + - name: ReturnOnAdSpend + bypass_reason: "dynamic field" + - name: AllReturnOnAdSpend + bypass_reason: "dynamic field" + - name: AverageCpc + bypass_reason: "dynamic field" + - name: AverageCpm + bypass_reason: "dynamic field" + - name: LowQualityClicksPercent + bypass_reason: "dynamic field" + ad_group_impression_performance_report_weekly: + - name: Impressions + bypass_reason: "dynamic field" + - name: DeviceType + bypass_reason: "dynamic field" + - name: HistoricalQualityScore + bypass_reason: "dynamic field" + - name: HistoricalExpectedCtr + bypass_reason: "dynamic field" + - name: HistoricalAdRelevance + bypass_reason: "dynamic field" + - name: HistoricalLandingPageExperience + bypass_reason: "dynamic field" + - name: Clicks + bypass_reason: "dynamic field" + - name: Ctr + bypass_reason: "dynamic field" + - name: AverageCpc + bypass_reason: "dynamic field" + - name: Spend + bypass_reason: "dynamic field" + - name: ConversionRate + bypass_reason: "dynamic field" + - name: ReturnOnAdSpend + bypass_reason: "dynamic field" + - name: AllConversionRate + bypass_reason: "dynamic field" + - name: AllReturnOnAdSpend + bypass_reason: "dynamic field" + - name: AverageCpm + bypass_reason: "dynamic field" + - name: LandingPageExperience + bypass_reason: "dynamic field" + ad_group_performance_report_weekly: + - name: Impressions + bypass_reason: "dynamic field" + - name: Ctr + bypass_reason: "dynamic field" + - name: Clicks + bypass_reason: "dynamic field" + - name: AverageCpm + bypass_reason: "dynamic field" + - name: Spend + bypass_reason: "dynamic field" + - name: AllReturnOnAdSpend + bypass_reason: "dynamic field" + - name: AllConversionRate + bypass_reason: "dynamic field" + - name: AverageCpc + bypass_reason: "dynamic field" + - name: ConversionRate + bypass_reason: "dynamic field" + - name: LandingPageExperience + bypass_reason: "dynamic field" + ad_performance_report_daily: + - name: TimePeriod + bypass_reason: "dynamic field" + - name: AbsoluteTopImpressionRatePercent + bypass_reason: "dynamic field" + - name: AdDistribution + bypass_reason: "dynamic field" + - name: DeviceType + bypass_reason: "dynamic field" + - name: Language + bypass_reason: "dynamic field" + - name: Network + bypass_reason: "dynamic field" + - name: DeviceOS + bypass_reason: "dynamic field" + - name: TopVsOther + bypass_reason: "dynamic field" + - name: Impressions + bypass_reason: "dynamic field" + - name: Ctr + bypass_reason: "dynamic field" + - name: Spend + bypass_reason: "dynamic field" + - name: AverageCpc + bypass_reason: "dynamic field" + - name: AverageCpm + bypass_reason: "dynamic field" + ad_performance_report_weekly: + - name: TimePeriod + bypass_reason: "dynamic field" + - name: AbsoluteTopImpressionRatePercent + bypass_reason: "dynamic field" + - name: AdDistribution + bypass_reason: "dynamic field" + - name: DeviceType + bypass_reason: "dynamic field" + - name: Language + bypass_reason: "dynamic field" + - name: Network + bypass_reason: "dynamic field" + - name: DeviceOS + bypass_reason: "dynamic field" + - name: TopVsOther + bypass_reason: "dynamic field" + - name: Impressions + bypass_reason: "dynamic field" + - name: Clicks + bypass_reason: "dynamic field" + - name: Ctr + bypass_reason: "dynamic field" + - name: Spent + bypass_reason: "dynamic field" + - name: ReturnOnAdSpend + bypass_reason: "dynamic field" + - name: AllReturnOnAdSpend + bypass_reason: "dynamic field" + - name: ConversionRate + bypass_reason: "dynamic field" + - name: AverageCpc + bypass_reason: "dynamic field" + - name: AverageCpm + bypass_reason: "dynamic field" + - name: AllConversions + bypass_reason: "dynamic field" + - name: AllConversionRate + bypass_reason: "dynamic field" + - name: AllRevenue + bypass_reason: "dynamic field" + ad_group_performance_report_daily: + - name: Impressions + bypass_reason: "dynamic field" + - name: Ctr + bypass_reason: "dynamic field" + - name: AverageCpm + bypass_reason: "dynamic field" + - name: HistoricalQualityScore + bypass_reason: "dynamic field" + - name: HistoricalExpectedCtr + bypass_reason: "dynamic field" + - name: HistoricalAdRelevance + bypass_reason: "dynamic field" + - name: HistoricalLandingPageExperience + bypass_reason: "dynamic field" + - name: LandingPageExperience + bypass_reason: "dynamic field" + budget_summary_report: + - name: Date + bypass_reason: "dynamic field" + - name: MonthlyBudget + bypass_reason: "dynamic field" + - name: DailySpend + bypass_reason: "dynamic field" + - name: MonthToDateSpend + bypass_reason: "dynamic field" + campaign_impression_performance_report_daily: + - name: AdDistribution + bypass_reason: "dynamic field" + - name: LowQualityImpressions + bypass_reason: "dynamic field" + - name: LowQualityImpressionsPercent + bypass_reason: "dynamic field" + - name: ImpressionSharePercent + bypass_reason: "dynamic field" + - name: ImpressionLostToBudgetPercent + bypass_reason: "dynamic field" + - name: ImpressionLostToRankAggPercent + bypass_reason: "dynamic field" + - name: HistoricalQualityScore + bypass_reason: "dynamic field" + - name: HistoricalExpectedCtr + bypass_reason: "dynamic field" + - name: HistoricalAdRelevance + bypass_reason: "dynamic field" + - name: HistoricalLandingPageExperience + bypass_reason: "dynamic field" + - name: ExactMatchImpressionSharePercent + bypass_reason: "dynamic field" + - name: AbsoluteTopImpressionSharePercent + bypass_reason: "dynamic field" + - name: TopImpressionShareLostToRankPercent + bypass_reason: "dynamic field" + - name: TopImpressionShareLostToBudgetPercent + bypass_reason: "dynamic field" + - name: AbsoluteTopImpressionShareLostToRankPercent + bypass_reason: "dynamic field" + - name: AbsoluteTopImpressionShareLostToBudgetPercent + bypass_reason: "dynamic field" + - name: TopImpressionSharePercent + bypass_reason: "dynamic field" + - name: AbsoluteTopImpressionRatePercent + bypass_reason: "dynamic field" + - name: TopImpressionRatePercent + bypass_reason: "dynamic field" + - name: Ctr + bypass_reason: "dynamic field" + - name: Impressions + bypass_reason: "dynamic field" + - name: AverageCpm + bypass_reason: "dynamic field" + - name: Clicks + bypass_reason: "dynamic field" + - name: AverageCpc + bypass_reason: "dynamic field" + - name: Spend + bypass_reason: "dynamic field" + - name: LowQualityClicksPercent + bypass_reason: "dynamic field" + - name: LowQualityClicks + bypass_reason: "dynamic field" + - name: LandingPageExperience + bypass_reason: "dynamic field" + - name: LowQualityGeneralClicks + bypass_reason: "dynamic field" + account_performance_report_daily: + - name: Ctr + bypass_reason: "dynamic field" + - name: Impressions + bypass_reason: "dynamic field" + - name: AverageCpm + bypass_reason: "dynamic field" + account_performance_report_weekly: + - name: Clicks + bypass_reason: "dynamic field" + - name: Spend + bypass_reason: "dynamic field" + - name: ReturnOnAdSpend + bypass_reason: "dynamic field" + - name: AverageCpc + bypass_reason: "dynamic field" + - name: ConversionRate + bypass_reason: "dynamic field" + - name: LowQualityClicksPercent + bypass_reason: "dynamic field" + - name: AverageCpm + bypass_reason: "dynamic field" + - name: Impressions + bypass_reason: "dynamic field" + - name: Ctr + bypass_reason: "dynamic field" + account_impression_performance_report_daily: + - name: Ctr + bypass_reason: "dynamic field" + - name: Impressions + bypass_reason: "dynamic field" + - name: AverageCpm + bypass_reason: "dynamic field" + - name: LowQualityImpressionsPercent + bypass_reason: "dynamic field" + - name: ExactMatchImpressionSharePercent + bypass_reason: "dynamic field" + - name: AbsoluteTopImpressionSharePercent + bypass_reason: "dynamic field" + - name: TopImpressionShareLostToRankPercent + bypass_reason: "dynamic field" + - name: TopImpressionShareLostToBudgetPercent + bypass_reason: "dynamic field" + - name: AbsoluteTopImpressionShareLostToRankPercent + bypass_reason: "dynamic field" + - name: AbsoluteTopImpressionShareLostToBudgetPercent + bypass_reason: "dynamic field" + - name: TopImpressionSharePercent + bypass_reason: "dynamic field" + - name: AbsoluteTopImpressionRatePercent + bypass_reason: "dynamic field" + - name: ImpressionSharePercent + bypass_reason: "dynamic field" + - name: ImpressionLostToBudgetPercent + bypass_reason: "dynamic field" + - name: ImpressionLostToRankAggPercent + bypass_reason: "dynamic field" + ad_group_impression_performance_report_daily: + - name: Ctr + bypass_reason: "dynamic field" + - name: Impressions + bypass_reason: "dynamic field" + - name: AverageCpm + bypass_reason: "dynamic field" + - name: HistoricalQualityScore + bypass_reason: "dynamic field" + - name: HistoricalExpectedCtr + bypass_reason: "dynamic field" + - name: HistoricalAdRelevance + bypass_reason: "dynamic field" + - name: HistoricalLandingPageExperience + bypass_reason: "dynamic field" + - name: LandingPageExperience + bypass_reason: "dynamic field" + - name: ImpressionSharePercent + bypass_reason: "dynamic field" + - name: ImpressionLostToBudgetPercent + bypass_reason: "dynamic field" + - name: ImpressionLostToRankAggPercent + bypass_reason: "dynamic field" + - name: ExactMatchImpressionSharePercent + bypass_reason: "dynamic field" + campaign_performance_report_daily: + - name: Ctr + bypass_reason: "dynamic field" + - name: Impressions + bypass_reason: "dynamic field" + - name: AverageCpm + bypass_reason: "dynamic field" + - name: HistoricalQualityScore + bypass_reason: "dynamic field" + - name: HistoricalExpectedCtr + bypass_reason: "dynamic field" + - name: HistoricalAdRelevance + bypass_reason: "dynamic field" + - name: HistoricalLandingPageExperience + bypass_reason: "dynamic field" + - name: LandingPageExperience + bypass_reason: "dynamic field" + keyword_performance_report_daily: + - name: Language + bypass_reason: "dynamic field" + - name: Network + bypass_reason: "dynamic field" + - name: DeviceOS + bypass_reason: "dynamic field" + - name: TopVsOther + bypass_reason: "dynamic field" + - name: Impressions + bypass_reason: "dynamic field" + - name: Clicks + bypass_reason: "dynamic field" + - name: Ctr + bypass_reason: "dynamic field" + - name: Spend + bypass_reason: "dynamic field" + - name: ReturnOnAdSpend + bypass_reason: "dynamic field" + - name: AllReturnOnAdSpend + bypass_reason: "dynamic field" + - name: ConversionRate + bypass_reason: "dynamic field" + - name: AverageCpc + bypass_reason: "dynamic field" + - name: AverageCpm + bypass_reason: "dynamic field" + - name: AllConversionRate + bypass_reason: "dynamic field" + - name: Mainline1Bid + bypass_reason: "dynamic field" + - name: MainlineBid + bypass_reason: "dynamic field" + - name: HistoricalExpectedCtr + bypass_reason: "dynamic field" + - name: HistoricalAdRelevance + bypass_reason: "dynamic field" + - name: HistoricalLandingPageExperience + bypass_reason: "dynamic field" + - name: HistoricalQualityScore + bypass_reason: "dynamic field" + - name: FirstPageBid + bypass_reason: "dynamic field" + timeout_seconds: 9000 + - config_path: secrets/config_no_date.json + expect_records: + path: "integration_tests/expected_records_no_start_date.jsonl" + extra_records: yes + empty_streams: + - name: app_install_ads + bypass_reason: "Can not populate; new campaign with link to app needed; feature is not available yet" + - name: app_install_ad_labels + bypass_reason: "Can not populate; depends on stream app_install_ads" + - name: age_gender_audience_report_hourly + bypass_reason: "Empty report; hourly data fetched is limited to 180 days" + - name: user_location_performance_report_hourly + bypass_reason: "Empty report; hourly data fetched is limited to 180 days" + - name: account_performance_report_hourly + bypass_reason: "Hourly reports are disabled, because sync is too long" + - name: account_impression_performance_report_hourly + bypass_reason: "Empty report; hourly data fetched is limited to 180 days" + - name: campaign_performance_report_hourly + bypass_reason: "Hourly reports are disabled, because sync is too long" + - name: ad_group_performance_report_hourly + bypass_reason: "Hourly reports are disabled, because sync is too long" + - name: ad_performance_report_hourly + bypass_reason: "Hourly reports are disabled, because sync is too long" + - name: ad_group_impression_performance_report_hourly + bypass_reason: "Empty report; hourly data fetched is limited to 180 days" - name: keyword_performance_report_hourly bypass_reason: "Hourly reports are disabled, because sync is too long" - name: geographic_performance_report_hourly bypass_reason: "Hourly reports are disabled, because sync is too long" + - name: campaign_impression_performance_report_hourly + bypass_reason: "Empty report; hourly data fetched is limited to 180 days" + - name: search_query_performance_report_hourly + bypass_reason: "Empty report; hourly data fetched is limited to 180 days" + #### TODO: remove *_report_monthly after all become populated on December, 1 + - name: account_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: ad_group_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: ad_group_impression_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: ad_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: campaign_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: campaign_impression_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: keyword_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: geographic_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: age_gender_audience_report_monthly + bypass_reason: "Campaign is still in progress" + - name: search_query_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: user_location_performance_report_monthly + bypass_reason: "Campaign is still in progress" + - name: account_impression_performance_report_monthly + bypass_reason: "Campaign is still in progress" + #### Streams below sync takes a lot of time if start date is not provided + - name: ad_groups + bypass_reason: "This stream is tested with config with start date" + - name: ads + bypass_reason: "This stream is tested with config with start date" + - name: campaigns + bypass_reason: "This stream is tested with config with start date" + - name: accounts + bypass_reason: "This stream is tested with config with start date" + - name: account_performance_report_daily + bypass_reason: "This stream is tested with config with start date" + - name: account_performance_report_weekly + bypass_reason: "This stream is tested with config with start date" + - name: ad_group_performance_report_daily + bypass_reason: "This stream is tested with config with start date" + - name: ad_group_performance_report_weekly + bypass_reason: "This stream is tested with config with start date" + - name: ad_group_impression_performance_report_daily + bypass_reason: "This stream is tested with config with start date" + - name: ad_group_impression_performance_report_weekly + bypass_reason: "This stream is tested with config with start date" + - name: ad_performance_report_daily + bypass_reason: "This stream is tested with config with start date" + - name: ad_performance_report_weekly + bypass_reason: "This stream is tested with config with start date" + - name: budget_summary_report + bypass_reason: "This stream is tested with config with start date" + - name: campaign_performance_report_daily + bypass_reason: "This stream is tested with config with start date" + - name: campaign_performance_report_weekly + bypass_reason: "This stream is tested with config with start date" + - name: campaign_impression_performance_report_daily + bypass_reason: "This stream is tested with config with start date" + - name: campaign_impression_performance_report_weekly + bypass_reason: "This stream is tested with config with start date" + - name: keyword_performance_report_daily + bypass_reason: "This stream is tested with config with start date" + - name: keyword_performance_report_weekly + bypass_reason: "This stream is tested with config with start date" + - name: geographic_performance_report_daily + bypass_reason: "This stream is tested with config with start date" + - name: geographic_performance_report_weekly + bypass_reason: "This stream is tested with config with start date" + - name: age_gender_audience_report_daily + bypass_reason: "This stream is tested with config with start date" + - name: age_gender_audience_report_weekly + bypass_reason: "This stream is tested with config with start date" + - name: search_query_performance_report_daily + bypass_reason: "This stream is tested with config with start date" + - name: search_query_performance_report_weekly + bypass_reason: "This stream is tested with config with start date" + - name: user_location_performance_report_daily + bypass_reason: "This stream is tested with config with start date" + - name: user_location_performance_report_weekly + bypass_reason: "This stream is tested with config with start date" + - name: account_impression_performance_report_daily + bypass_reason: "This stream is tested with config with start date" + - name: account_impression_performance_report_weekly + bypass_reason: "This stream is tested with config with start date" + ignored_fields: + campaign_labels: + - name: Modified Time + bypass_reason: "dynamic field" + keyword_labels: + - name: Modified Time + bypass_reason: "dynamic field" + ad_group_labels: + - name: Modified Time + bypass_reason: "dynamic field" + labels: + - name: Modified Time + bypass_reason: "dynamic field" + timeout_seconds: 9000 full_refresh: tests: - config_path: secrets/config.json - configured_catalog_path: integration_tests/configured_catalog.json + configured_catalog_path: integration_tests/configured_catalog_full_refresh.json + timeout_seconds: 9000 + ignored_fields: + account_performance_report_hourly: + - name: "AdDistribution" + bypass_reason: "dynamic field" + - name: "DeviceType" + bypass_reason: "dynamic field" + - name: "Network" + bypass_reason: "dynamic field" + - name: "DeliveredMatchType" + bypass_reason: "dynamic field" + - name: "DeviceOS" + bypass_reason: "dynamic field" + - name: "TopVsOther" + bypass_reason: "dynamic field" + - name: "Impressions" + bypass_reason: "dynamic field" + - name: "Clicks" + bypass_reason: "dynamic field" + - name: "Ctr" + bypass_reason: "dynamic field" + - name: "Spend" + bypass_reason: "dynamic field" + - name: "ReturnOnAdSpend" + bypass_reason: "dynamic field" + - name: "AverageCpc" + bypass_reason: "dynamic field" + - name: "AverageCpm" + bypass_reason: "dynamic field" + - name: "ConversionRate" + bypass_reason: "dynamic field" + - name: "LowQualityClicksPercent" + bypass_reason: "dynamic field" + - name: "LowQualityImpressions" + bypass_reason: "dynamic field" + account_performance_report_weekly: + - name: "AdDistribution" + bypass_reason: "dynamic field" + - name: "DeviceType" + bypass_reason: "dynamic field" + - name: "Network" + bypass_reason: "dynamic field" + - name: "DeliveredMatchType" + bypass_reason: "dynamic field" + - name: "DeviceOS" + bypass_reason: "dynamic field" + - name: "TopVsOther" + bypass_reason: "dynamic field" + - name: "Impressions" + bypass_reason: "dynamic field" + - name: "Clicks" + bypass_reason: "dynamic field" + - name: "Ctr" + bypass_reason: "dynamic field" + - name: "Spend" + bypass_reason: "dynamic field" + - name: "ReturnOnAdSpend" + bypass_reason: "dynamic field" + - name: "AverageCpc" + bypass_reason: "dynamic field" + - name: "AverageCpm" + bypass_reason: "dynamic field" + - name: "ConversionRate" + bypass_reason: "dynamic field" + - name: "LowQualityClicksPercent" + bypass_reason: "dynamic field" + - name: "LowQualityImpressions" + bypass_reason: "dynamic field" + account_performance_report_daily: + - name: "AdDistribution" + bypass_reason: "dynamic field" + - name: "DeviceType" + bypass_reason: "dynamic field" + - name: "Network" + bypass_reason: "dynamic field" + - name: "DeliveredMatchType" + bypass_reason: "dynamic field" + - name: "DeviceOS" + bypass_reason: "dynamic field" + - name: "TopVsOther" + bypass_reason: "dynamic field" + - name: "Impressions" + bypass_reason: "dynamic field" + - name: "Clicks" + bypass_reason: "dynamic field" + - name: "Ctr" + bypass_reason: "dynamic field" + - name: "Spend" + bypass_reason: "dynamic field" + - name: "ReturnOnAdSpend" + bypass_reason: "dynamic field" + - name: "AverageCpc" + bypass_reason: "dynamic field" + - name: "AverageCpm" + bypass_reason: "dynamic field" + - name: "ConversionRate" + bypass_reason: "dynamic field" + - name: "LowQualityClicksPercent" + bypass_reason: "dynamic field" + - name: "LowQualityImpressions" + bypass_reason: "dynamic field" + budget_summary_report: + - name: Date + bypass_reason: "dynamic field" + - name: MonthlyBudget + bypass_reason: "dynamic field" + - name: DailySpend + bypass_reason: "dynamic field" + - name: MonthToDateSpend + bypass_reason: "dynamic field" + incremental: tests: bypass_reason: "SAT doesn't support complex nested states used in incremental report streams" diff --git a/airbyte-integrations/connectors/source-bing-ads/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-bing-ads/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-bing-ads/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-bing-ads/build.gradle b/airbyte-integrations/connectors/source-bing-ads/build.gradle deleted file mode 100644 index e7415d618a7c..000000000000 --- a/airbyte-integrations/connectors/source-bing-ads/build.gradle +++ /dev/null @@ -1,23 +0,0 @@ -import ru.vyarus.gradle.plugin.python.task.PythonTask - -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_bing_ads' -} - -// setuptools 58.* removed support for use_2to3 which leads to the following issue: -// error in suds-jurko setup command: use_2to3 is invalid. -// https://setuptools.readthedocs.io/en/latest/history.html#v58-0-0 -// To be able to resolve this issue connector need to use 57.* version of setuptools -// TODO: Remove this step after resolution of this issue https://github.com/BingAds/BingAds-Python-SDK/issues/191 -task("customSetupToolsInstall", type: PythonTask, dependsOn: flakeCheck){ - module = "pip" - command = "install setuptools==57.5.0" -} - -installLocalReqs.dependsOn("customSetupToolsInstall") diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog.json index 501c98fb97e3..cdf122f10f7b 100644 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog.json @@ -236,6 +236,46 @@ "cursor_field": ["TimePeriod"], "destination_sync_mode": "append" }, + { + "stream": { + "name": "campaign_impression_performance_report_hourly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "campaign_impression_performance_report_daily", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "campaign_impression_performance_report_weekly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "campaign_impression_performance_report_monthly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, { "stream": { "name": "keyword_performance_report_hourly", @@ -275,6 +315,276 @@ "sync_mode": "incremental", "cursor_field": ["TimePeriod"], "destination_sync_mode": "append" + }, + { + "stream": { + "name": "age_gender_audience_report_daily", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "age_gender_audience_report_weekly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "age_gender_audience_report_hourly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "age_gender_audience_report_monthly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "search_query_performance_report_hourly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "search_query_performance_report_daily", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "search_query_performance_report_weekly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "search_query_performance_report_monthly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "user_location_performance_report_daily", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "user_location_performance_report_hourly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "user_location_performance_report_weekly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "user_location_performance_report_monthly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "account_impression_performance_report_daily", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "account_impression_performance_report_hourly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "account_impression_performance_report_weekly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "account_impression_performance_report_monthly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "labels", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["Modified Time"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "app_install_ads", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "app_install_ad_labels", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "campaign_labels", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "keyword_labels", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_group_labels", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "keywords", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_group_impression_performance_report_hourly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_group_impression_performance_report_daily", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_group_impression_performance_report_weekly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_group_impression_performance_report_monthly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog_full_refresh.json b/airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog_full_refresh.json new file mode 100644 index 000000000000..a5a62dab8de5 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/configured_catalog_full_refresh.json @@ -0,0 +1,150 @@ +{ + "streams": [ + { + "stream": { + "name": "ad_groups", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ads", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "campaigns", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "accounts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "account_performance_report_hourly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "account_performance_report_daily", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "account_performance_report_weekly", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "budget_summary_report", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "labels", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["Modified Time"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "app_install_ads", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "app_install_ad_labels", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "campaign_labels", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "keyword_labels", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "ad_group_labels", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "keywords", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"] + }, + "sync_mode": "incremental", + "cursor_field": ["TimePeriod"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl new file mode 100644 index 000000000000..c6373cec4d5b --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.jsonl @@ -0,0 +1,29 @@ +{"stream":"ad_groups","data":{"AdRotation":null,"AudienceAdsBidAdjustment":null,"BiddingScheme":{"Type":"InheritFromParent","InheritedBidStrategyType":"EnhancedCpc"},"CpcBid":{"Amount":2.27},"EndDate":null,"FinalUrlSuffix":null,"ForwardCompatibilityMap":null,"Id":1356799861840328,"Language":null,"Name":"keywords","Network":"OwnedAndOperatedAndSyndicatedSearch","PrivacyStatus":null,"Settings":null,"StartDate":{"Day":7,"Month":11,"Year":2023},"Status":"Active","TrackingUrlTemplate":null,"UrlCustomParameters":null,"AdScheduleUseSearcherTimeZone":false,"AdGroupType":"SearchStandard","CpvBid":{"Amount":null},"CpmBid":{"Amount":null},"CampaignId":531016227,"AccountId":180519267,"CustomerId":251186883},"emitted_at":1704833256596} +{"stream":"ads","data":{"AdFormatPreference":"All","DevicePreference":0,"EditorialStatus":"Active","FinalAppUrls":null,"FinalMobileUrls":null,"FinalUrlSuffix":null,"FinalUrls":{"string":["https://airbyte.com"]},"ForwardCompatibilityMap":null,"Id":84800390693061,"Status":"Active","TrackingUrlTemplate":null,"Type":"ResponsiveSearch","UrlCustomParameters":null,"Descriptions":{"AssetLink":[{"Asset":{"Id":10239363892977,"Name":null,"Type":"TextAsset","Text":"Connect, integrate, and sync data seamlessly with Airbyte's 800+ contributors and growing!"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363892976,"Name":null,"Type":"TextAsset","Text":"Move data like a pro with our powerful tool trusted by 40,000+ engineers worldwide!"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null}]},"Domain":"airbyte.com","Headlines":{"AssetLink":[{"Asset":{"Id":10239363892979,"Name":null,"Type":"TextAsset","Text":"Get synced with Airbyte"},"AssetPerformanceLabel":"Good","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363893384,"Name":null,"Type":"TextAsset","Text":"Data management made easy"},"AssetPerformanceLabel":"Best","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363892978,"Name":null,"Type":"TextAsset","Text":"Connectors for every need"},"AssetPerformanceLabel":"Low","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363892980,"Name":null,"Type":"TextAsset","Text":"Industry-leading connectors"},"AssetPerformanceLabel":"Low","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239363893383,"Name":null,"Type":"TextAsset","Text":"Try Airbyte now for free"},"AssetPerformanceLabel":"Low","EditorialStatus":"Active","PinnedField":null}]},"Path1":null,"Path2":null,"AdGroupId":1356799861840328,"AccountId":180519267,"CustomerId":251186883},"emitted_at":1704833266594} +{"stream": "campaigns", "data": {"AudienceAdsBidAdjustment": 0, "BiddingScheme": {"Type": "EnhancedCpc"}, "BudgetType": "DailyBudgetStandard", "DailyBudget": 2.0, "ExperimentId": null, "FinalUrlSuffix": null, "ForwardCompatibilityMap": null, "Id": 531016227, "MultimediaAdsBidAdjustment": 40, "Name": "Airbyte test", "Status": "Active", "SubType": null, "TimeZone": "CentralTimeUSCanada", "TrackingUrlTemplate": null, "UrlCustomParameters": null, "CampaignType": "Search", "Settings": {"Setting": [{"Type": "TargetSetting", "Details": {"TargetSettingDetail": [{"CriterionTypeGroup": "Audience", "TargetAndBid": false}]}}]}, "BudgetId": null, "Languages": {"string": ["English"]}, "AdScheduleUseSearcherTimeZone": false, "AccountId": 180519267, "CustomerId": 251186883}, "emitted_at": 1702903287209} +{"stream": "accounts", "data": {"BillToCustomerId": 251186883, "CurrencyCode": "USD", "AccountFinancialStatus": "ClearFinancialStatus", "Id": 180535609, "Language": "English", "LastModifiedByUserId": 0, "LastModifiedTime": "2023-08-11T08:24:26.603000", "Name": "DEMO-ACCOUNT", "Number": "F149W3B6", "ParentCustomerId": 251186883, "PaymentMethodId": null, "PaymentMethodType": null, "PrimaryUserId": 138225488, "AccountLifeCycleStatus": "Pause", "TimeStamp": "AAAAAH10c1A=", "TimeZone": "Santiago", "PauseReason": 2, "ForwardCompatibilityMap": null, "LinkedAgencies": null, "SalesHouseCustomerId": null, "TaxInformation": null, "BackUpPaymentInstrumentId": null, "BillingThresholdAmount": null, "BusinessAddress": {"City": "San Francisco", "CountryCode": "US", "Id": 149694999, "Line1": "350 29th avenue", "Line2": null, "Line3": null, "Line4": null, "PostalCode": "94121", "StateOrProvince": "CA", "TimeStamp": null, "BusinessName": "Daxtarity Inc."}, "AutoTagType": "Inactive", "SoldToPaymentInstrumentId": null, "AccountMode": "Expert"}, "emitted_at": 1702903290287} +{"stream":"account_performance_report_daily","data":{"AccountId":180519267,"TimePeriod":"2023-12-18","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Syndicated search partners","DeliveredMatchType":"Exact","DeviceOS":"Windows","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","AccountNumber":"F149MJ18","PhoneImpressions":0,"PhoneCalls":0,"Clicks":0,"Ctr":0.0,"Spend":0.0,"Impressions":1,"CostPerConversion":null,"Ptr":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionsQualified":0.0,"ConversionRate":null,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":0,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1704833285214} +{"stream":"account_performance_report_weekly","data":{"AccountId":180519267,"TimePeriod":"2023-12-17","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Syndicated search partners","DeliveredMatchType":"Exact","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","AccountNumber":"F149MJ18","PhoneImpressions":0,"PhoneCalls":0,"Clicks":0,"Ctr":0.0,"Spend":0.0,"Impressions":5,"CostPerConversion":null,"Ptr":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionsQualified":0.0,"ConversionRate":null,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":4,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1704833307364} +{"stream":"ad_group_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"TimePeriod":"2023-12-18","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Microsoft sites and select traffic","DeliveredMatchType":"Exact","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","Language":"Portuguese","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","AdGroupName":"keywords","AdGroupType":"Standard","Impressions":2,"Clicks":1,"Ctr":50.0,"Spend":0.01,"CostPerConversion":null,"QualityScore":7.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":2.0,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Assists":0,"CostPerAssist":null,"CustomParameters":null,"FinalUrlSuffix":null,"ViewThroughConversions":0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllConversions":0,"AllConversionRate":0.0,"AllRevenue":0.0,"AllRevenuePerConversion":null,"AverageCpc":0.01,"AveragePosition":0.0,"AverageCpm":5.0,"Conversions":0.0,"ConversionRate":0.0,"ConversionsQualified":0.0,"HistoricalQualityScore":6.0,"HistoricalExpectedCtr":2.0,"HistoricalAdRelevance":3.0,"HistoricalLandingPageExperience":2.0,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1704884363801} +{"stream":"ad_group_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"TimePeriod":"2023-12-17","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Syndicated search partners","DeliveredMatchType":"Exact","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","Language":"German","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","AdGroupName":"keywords","AdGroupType":"Standard","Impressions":1,"Clicks":0,"Ctr":0.0,"Spend":0.0,"CostPerConversion":null,"QualityScore":7.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":2.0,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Assists":0,"CostPerAssist":null,"CustomParameters":null,"FinalUrlSuffix":null,"ViewThroughConversions":0,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"HistoricalQualityScore":6.0,"HistoricalExpectedCtr":2.0,"HistoricalAdRelevance":3.0,"HistoricalLandingPageExperience":2.0,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1704833349472} +{"stream":"ad_group_impression_performance_report_daily","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-12-18","Status":"Active","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":1,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"DeviceType":"Computer","Language":"Czech","ImpressionSharePercent":null,"ImpressionLostToBudgetPercent":null,"ImpressionLostToRankAggPercent":null,"QualityScore":7,"ExpectedCtr":2.0,"AdRelevance":3,"LandingPageExperience":2,"HistoricalQualityScore":6,"HistoricalExpectedCtr":2,"HistoricalAdRelevance":3,"HistoricalLandingPageExperience":2,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Microsoft sites and select traffic","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","CampaignStatus":"Active","AdGroupLabels":null,"ExactMatchImpressionSharePercent":null,"ClickSharePercent":null,"AbsoluteTopImpressionSharePercent":null,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":null,"TopImpressionShareLostToBudgetPercent":null,"AbsoluteTopImpressionShareLostToRankPercent":null,"AbsoluteTopImpressionShareLostToBudgetPercent":null,"TopImpressionSharePercent":null,"AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"BaseCampaignId":531016227,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AdGroupType":"Standard","AverageCpm":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":0.0,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1704833929228} +{"stream":"ad_group_impression_performance_report_weekly","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-12-17","Status":"Active","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":3,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"DeviceType":"Computer","Language":"Bulgarian","ImpressionSharePercent":13.64,"ImpressionLostToBudgetPercent":9.09,"ImpressionLostToRankAggPercent":77.27,"QualityScore":7,"ExpectedCtr":2.0,"AdRelevance":3,"LandingPageExperience":2,"HistoricalQualityScore":6,"HistoricalExpectedCtr":2,"HistoricalAdRelevance":3,"HistoricalLandingPageExperience":2,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Microsoft sites and select traffic","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","CampaignStatus":"Active","AdGroupLabels":null,"ExactMatchImpressionSharePercent":null,"ClickSharePercent":null,"AbsoluteTopImpressionSharePercent":null,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":null,"TopImpressionShareLostToBudgetPercent":null,"AbsoluteTopImpressionShareLostToRankPercent":null,"AbsoluteTopImpressionShareLostToBudgetPercent":null,"TopImpressionSharePercent":null,"AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"BaseCampaignId":531016227,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AdGroupType":"Standard","AverageCpm":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":0.0,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1704833951765} +{"stream":"ad_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"AdId":84800390693061,"TimePeriod":"2023-12-18","AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Language":"Czech","Network":"Microsoft sites and select traffic","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","DeliveredMatchType":"Phrase","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","AdGroupName":"keywords","Impressions":1,"Clicks":0,"Ctr":0.0,"Spend":0.0,"CostPerConversion":null,"DestinationUrl":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"AdDescription":null,"AdDescription2":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1704833373752} +{"stream":"ad_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"AdId":84800390693061,"TimePeriod":"2023-12-17","AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Language":"Bulgarian","Network":"Microsoft sites and select traffic","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","DeliveredMatchType":"Phrase","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","AdGroupName":"keywords","Impressions":3,"Clicks":0,"Ctr":0.0,"Spend":0.0,"CostPerConversion":null,"DestinationUrl":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"AdDescription":null,"AdDescription2":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1704833394112} +{"stream":"budget_summary_report","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"CampaignId":531016227,"CampaignName":"Airbyte test","Date":"2023-12-18","MonthlyBudget":60.8,"DailySpend":2.06,"MonthToDateSpend":36.58},"emitted_at":1704833526694} +{"stream":"campaign_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"TimePeriod":"2023-12-18","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Syndicated search partners","DeliveredMatchType":"Exact","DeviceOS":"Windows","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","CampaignStatus":"Active","CampaignLabels":null,"Impressions":1,"Clicks":0,"Ctr":0.0,"Spend":0.0,"CostPerConversion":null,"QualityScore":7.0,"AdRelevance":3.0,"LandingPageExperience":2.0,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"ViewThroughConversions":0,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllConversions":0,"ConversionsQualified":0.0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionRate":null,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":0,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"HistoricalQualityScore":6.0,"HistoricalExpectedCtr":2.0,"HistoricalAdRelevance":3.0,"HistoricalLandingPageExperience":2.0,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null,"BudgetName":null,"BudgetStatus":null,"BudgetAssociationStatus":"Current"},"emitted_at":1704833545467} +{"stream":"campaign_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"TimePeriod":"2023-12-17","CurrencyCode":"USD","AdDistribution":"Search","DeviceType":"Computer","Network":"Syndicated search partners","DeliveredMatchType":"Exact","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","CampaignType":"Search & content","CampaignStatus":"Active","CampaignLabels":null,"Impressions":5,"Clicks":0,"Ctr":0.0,"Spend":0.0,"CostPerConversion":null,"QualityScore":7.0,"AdRelevance":3.0,"LandingPageExperience":2.0,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"ViewThroughConversions":0,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllConversions":0,"ConversionsQualified":0.0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"Conversions":0.0,"ConversionRate":null,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":4,"LowQualitySophisticatedClicks":0,"LowQualityConversions":0,"LowQualityConversionRate":null,"HistoricalQualityScore":6.0,"HistoricalExpectedCtr":2.0,"HistoricalAdRelevance":3.0,"HistoricalLandingPageExperience":2.0,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null,"BudgetName":null,"BudgetStatus":null,"BudgetAssociationStatus":"Current"},"emitted_at":1704833565296} +{"stream":"campaign_impression_performance_report_daily","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-12-18","CampaignStatus":"Active","CampaignName":"Airbyte test","CampaignId":531016227,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":22,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":6,"LowQualityImpressionsPercent":21.43,"LowQualityConversions":0,"LowQualityConversionRate":null,"DeviceType":"Computer","ImpressionSharePercent":34.92,"ImpressionLostToBudgetPercent":1.59,"ImpressionLostToRankAggPercent":63.49,"QualityScore":7.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":2.0,"HistoricalQualityScore":6,"HistoricalExpectedCtr":2,"HistoricalAdRelevance":3,"HistoricalLandingPageExperience":2,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","LowQualityGeneralClicks":0,"LowQualitySophisticatedClicks":0,"CampaignLabels":null,"ExactMatchImpressionSharePercent":5.26,"ClickSharePercent":null,"AbsoluteTopImpressionSharePercent":10.2,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":68.0,"TopImpressionShareLostToBudgetPercent":0.0,"AbsoluteTopImpressionShareLostToRankPercent":89.8,"AbsoluteTopImpressionShareLostToBudgetPercent":0.0,"TopImpressionSharePercent":32.0,"AbsoluteTopImpressionRatePercent":22.73,"TopImpressionRatePercent":72.73,"BaseCampaignId":531016227,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AverageCpm":0.0,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":0.0,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1704833589146} +{"stream":"campaign_impression_performance_report_weekly","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-12-17","CampaignStatus":"Active","CampaignName":"Airbyte test","CampaignId":531016227,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":639,"Clicks":14,"Ctr":2.19,"AverageCpc":0.12,"Spend":1.74,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"LowQualityClicks":6,"LowQualityClicksPercent":30.0,"LowQualityImpressions":53,"LowQualityImpressionsPercent":7.66,"LowQualityConversions":0,"LowQualityConversionRate":0.0,"DeviceType":"Computer","ImpressionSharePercent":13.57,"ImpressionLostToBudgetPercent":17.96,"ImpressionLostToRankAggPercent":68.47,"QualityScore":7.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":2.0,"HistoricalQualityScore":6,"HistoricalExpectedCtr":2,"HistoricalAdRelevance":3,"HistoricalLandingPageExperience":2,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"TrackingTemplate":null,"CustomParameters":null,"AccountStatus":"Active","LowQualityGeneralClicks":0,"LowQualitySophisticatedClicks":6,"CampaignLabels":null,"ExactMatchImpressionSharePercent":17.65,"ClickSharePercent":1.28,"AbsoluteTopImpressionSharePercent":3.2,"FinalUrlSuffix":null,"CampaignType":"Search & content","TopImpressionShareLostToRankPercent":74.15,"TopImpressionShareLostToBudgetPercent":18.25,"AbsoluteTopImpressionShareLostToRankPercent":78.51,"AbsoluteTopImpressionShareLostToBudgetPercent":18.29,"TopImpressionSharePercent":7.6,"AbsoluteTopImpressionRatePercent":22.69,"TopImpressionRatePercent":53.99,"BaseCampaignId":531016227,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"RelativeCtr":null,"AverageCpm":2.72,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":0.0,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1704833610948} +{"stream":"keyword_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"KeywordId":84801135055365,"Keyword":"connector","AdId":84800390693061,"TimePeriod":"2023-12-18","CurrencyCode":"USD","DeliveredMatchType":"Exact","AdDistribution":"Audience","DeviceType":"Computer","Language":"English","Network":"Audience","DeviceOS":"Unknown","TopVsOther":"Audience network","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","AdGroupName":"keywords","KeywordStatus":"Active","HistoricalExpectedCtr":2.0,"HistoricalAdRelevance":3.0,"HistoricalLandingPageExperience":1.0,"HistoricalQualityScore":5.0,"Impressions":6,"Clicks":0,"Ctr":0.0,"CurrentMaxCpc":2.27,"Spend":0.0,"CostPerConversion":null,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"QualityImpact":0.0,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"Mainline1Bid":null,"MainlineBid":0.66,"FirstPageBid":0.3,"FinalUrlSuffix":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1704833634746} +{"stream":"keyword_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"KeywordId":84801135055365,"Keyword":"connector","AdId":84800390693061,"TimePeriod":"2023-12-17","CurrencyCode":"USD","DeliveredMatchType":"Exact","AdDistribution":"Search","DeviceType":"Computer","Language":"Spanish","Network":"Microsoft sites and select traffic","DeviceOS":"Windows","TopVsOther":"Microsoft sites and select traffic - top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","AdGroupName":"keywords","KeywordStatus":"Active","Impressions":1,"Clicks":0,"Ctr":0.0,"CurrentMaxCpc":2.27,"Spend":0.0,"CostPerConversion":null,"QualityScore":5.0,"ExpectedCtr":"2","AdRelevance":3.0,"LandingPageExperience":1.0,"QualityImpact":0.0,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"CustomParameters":null,"FinalAppUrl":null,"Mainline1Bid":null,"MainlineBid":0.66,"FirstPageBid":0.3,"FinalUrlSuffix":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1704833656374} +{"stream":"geographic_performance_report_daily","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"TimePeriod":"2023-12-18","AccountNumber":"F149MJ18","Country":"Argentina","State":null,"MetroArea":null,"City":null,"ProximityTargetLocation":null,"Radius":"0","LocationType":"Physical location","MostSpecificLocation":"Argentina","AccountStatus":"Active","CampaignStatus":"Active","AdGroupStatus":"Active","County":null,"PostalCode":null,"LocationId":"8","BaseCampaignId":"531016227","Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":33.33,"TopImpressionRatePercent":"100.00","AllConversionsQualified":"0.00","Neighborhood":null,"ViewThroughRevenue":"0.00","CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null,"AssetGroupStatus":null,"CurrencyCode":"USD","DeliveredMatchType":"Phrase","AdDistribution":"Search","DeviceType":"Computer","Language":"Spanish","Network":"Syndicated search partners","DeviceOS":"Unknown","TopVsOther":"Syndicated search partners - Top","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","AdGroupName":"keywords","Impressions":3,"Clicks":0,"Ctr":0.0,"Spend":0.0,"CostPerConversion":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1704833416620} +{"stream":"geographic_performance_report_weekly","data":{"AccountId":180519267,"CampaignId":531016227,"AdGroupId":1356799861840328,"TimePeriod":"2023-12-17","AccountNumber":"F149MJ18","Country":"United Arab Emirates","State":"Dubai","MetroArea":null,"City":"Dubai","ProximityTargetLocation":null,"Radius":"0","LocationType":"Physical location","MostSpecificLocation":"Dubai","AccountStatus":"Active","CampaignStatus":"Active","AdGroupStatus":"Active","County":null,"PostalCode":null,"LocationId":"154645","BaseCampaignId":"531016227","Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":0.0,"TopImpressionRatePercent":"0.00","AllConversionsQualified":"0.00","Neighborhood":null,"ViewThroughRevenue":"0.00","CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null,"AssetGroupStatus":null,"CurrencyCode":"USD","DeliveredMatchType":"Exact","AdDistribution":"Audience","DeviceType":"Smartphone","Language":"English","Network":"Audience","DeviceOS":"Android","TopVsOther":"Audience network","BidMatchType":"Broad","AccountName":"Airbyte","CampaignName":"Airbyte test","AdGroupName":"keywords","Impressions":1,"Clicks":0,"Ctr":0.0,"Spend":0.0,"CostPerConversion":null,"Assists":0,"ReturnOnAdSpend":null,"CostPerAssist":null,"ViewThroughConversions":0,"ViewThroughConversionsQualified":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"Conversions":0.0,"ConversionRate":null,"ConversionsQualified":0.0,"AverageCpc":0.0,"AveragePosition":0.0,"AverageCpm":0.0,"AllConversions":0,"AllConversionRate":null,"AllRevenue":0.0,"AllRevenuePerConversion":null,"Revenue":0.0,"RevenuePerConversion":null,"RevenuePerAssist":null},"emitted_at":1704833479492} +{"stream":"age_gender_audience_report_daily","data":{"AccountId":180519267,"AgeGroup":"Unknown","Gender":"Unknown","TimePeriod":"2023-12-18","AllConversions":0,"AccountName":"Airbyte","AccountNumber":"F149MJ18","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"AdDistribution":"Search","Impressions":1,"Clicks":0,"Conversions":0.0,"Spend":0.0,"Revenue":0.0,"ExtendedCost":0.0,"Assists":0,"Language":"Czech","AccountStatus":"Active","CampaignStatus":"Active","AdGroupStatus":"Active","BaseCampaignId":"531016227","AllRevenue":0.0,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0},"emitted_at":1704833673872} +{"stream":"age_gender_audience_report_weekly","data":{"AccountId":180519267,"AgeGroup":"Unknown","Gender":"Unknown","TimePeriod":"2023-12-17","AllConversions":0,"AccountName":"Airbyte","AccountNumber":"F149MJ18","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"AdDistribution":"Search","Impressions":1,"Clicks":0,"Conversions":0.0,"Spend":0.0,"Revenue":0.0,"ExtendedCost":0.0,"Assists":0,"Language":"Bulgarian","AccountStatus":"Active","CampaignStatus":"Active","AdGroupStatus":"Active","BaseCampaignId":"531016227","AllRevenue":0.0,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0},"emitted_at":1704833693674} +{"stream":"search_query_performance_report_daily","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-12-18","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"AdId":84800390693061,"AdType":"Responsive search ad","DestinationUrl":null,"BidMatchType":"Broad","DeliveredMatchType":"Exact","CampaignStatus":"Active","AdStatus":"Active","Impressions":1,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"SearchQuery":"airbyte","Keyword":"Airbyte","AdGroupCriterionId":null,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"Language":"English","KeywordId":84801135055370,"Network":"Microsoft sites and select traffic","TopVsOther":"Microsoft sites and select traffic - top","DeviceType":"Computer","DeviceOS":"Windows","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"AccountStatus":"Active","AdGroupStatus":"Active","KeywordStatus":"Active","CampaignType":"Search & content","CustomerId":251186883,"CustomerName":"Daxtarity Inc.","AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"AverageCpm":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0},"emitted_at":1704833715419} +{"stream":"search_query_performance_report_weekly","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-12-17","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"AdId":84800390693061,"AdType":"Responsive search ad","DestinationUrl":null,"BidMatchType":"Broad","DeliveredMatchType":"Exact","CampaignStatus":"Active","AdStatus":"Active","Impressions":1,"Clicks":1,"Ctr":100.0,"AverageCpc":0.04,"Spend":0.04,"AveragePosition":0.0,"SearchQuery":"airbyte","Keyword":"Airbyte","AdGroupCriterionId":null,"Conversions":0,"ConversionRate":0.0,"CostPerConversion":null,"Language":"Czech","KeywordId":84801135055370,"Network":"Microsoft sites and select traffic","TopVsOther":"Microsoft sites and select traffic - top","DeviceType":"Computer","DeviceOS":"Unknown","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"AccountStatus":"Active","AdGroupStatus":"Active","KeywordStatus":"Active","CampaignType":"Search & content","CustomerId":251186883,"CustomerName":"Daxtarity Inc.","AllConversions":0,"AllRevenue":0.0,"AllConversionRate":0.0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":100.0,"TopImpressionRatePercent":100.0,"AverageCpm":40.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0},"emitted_at":1704833737157} +{"stream":"user_location_performance_report_daily","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-12-18","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"Country":"Argentina","State":null,"MetroArea":null,"CurrencyCode":"USD","AdDistribution":"Search","Impressions":3,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"ProximityTargetLocation":null,"Radius":0,"Language":"Spanish","City":null,"QueryIntentCountry":"Argentina","QueryIntentState":null,"QueryIntentCity":null,"QueryIntentDMA":null,"BidMatchType":"Broad","DeliveredMatchType":"Phrase","Network":"Syndicated search partners","TopVsOther":"Syndicated search partners - Top","DeviceType":"Computer","DeviceOS":"Unknown","Assists":0,"Conversions":0,"ConversionRate":null,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerConversion":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"County":null,"PostalCode":null,"QueryIntentCounty":null,"QueryIntentPostalCode":null,"LocationId":8,"QueryIntentLocationId":8,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":33.33,"TopImpressionRatePercent":100.0,"AverageCpm":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"Neighborhood":null,"QueryIntentNeighborhood":null,"ViewThroughRevenue":0.0,"CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null},"emitted_at":1704833762092} +{"stream":"user_location_performance_report_weekly","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-12-17","CampaignName":"Airbyte test","CampaignId":531016227,"AdGroupName":"keywords","AdGroupId":1356799861840328,"Country":"United Arab Emirates","State":"Dubai","MetroArea":null,"CurrencyCode":"USD","AdDistribution":"Audience","Impressions":1,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"ProximityTargetLocation":null,"Radius":0,"Language":"English","City":"Dubai","QueryIntentCountry":"United Arab Emirates","QueryIntentState":null,"QueryIntentCity":null,"QueryIntentDMA":null,"BidMatchType":"Broad","DeliveredMatchType":"Exact","Network":"Audience","TopVsOther":"Audience network","DeviceType":"Smartphone","DeviceOS":"Android","Assists":0,"Conversions":0,"ConversionRate":null,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerConversion":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"County":null,"PostalCode":null,"QueryIntentCounty":null,"QueryIntentPostalCode":null,"LocationId":154645,"QueryIntentLocationId":218,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"Goal":null,"GoalType":null,"AbsoluteTopImpressionRatePercent":0.0,"TopImpressionRatePercent":0.0,"AverageCpm":0.0,"ConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"Neighborhood":null,"QueryIntentNeighborhood":null,"ViewThroughRevenue":0.0,"CampaignType":"Search & content","AssetGroupId":null,"AssetGroupName":null},"emitted_at":1704833830043} +{"stream":"account_impression_performance_report_daily","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-12-18","CurrencyCode":"USD","AdDistribution":"Search","Impressions":22,"Clicks":0,"Ctr":0.0,"AverageCpc":0.0,"Spend":0.0,"AveragePosition":0.0,"Conversions":0,"ConversionRate":null,"CostPerConversion":null,"LowQualityClicks":0,"LowQualityClicksPercent":null,"LowQualityImpressions":6,"LowQualityImpressionsPercent":21.43,"LowQualityConversions":0,"LowQualityConversionRate":null,"DeviceType":"Computer","ImpressionSharePercent":34.92,"ImpressionLostToBudgetPercent":1.59,"ImpressionLostToRankAggPercent":63.49,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":null,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"AccountStatus":"Active","LowQualityGeneralClicks":0,"LowQualitySophisticatedClicks":0,"ExactMatchImpressionSharePercent":5.26,"ClickSharePercent":null,"AbsoluteTopImpressionSharePercent":10.2,"TopImpressionShareLostToRankPercent":68.0,"TopImpressionShareLostToBudgetPercent":0.0,"AbsoluteTopImpressionShareLostToRankPercent":89.8,"AbsoluteTopImpressionShareLostToBudgetPercent":0.0,"TopImpressionSharePercent":32.0,"AbsoluteTopImpressionRatePercent":22.73,"TopImpressionRatePercent":72.73,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":null,"AllCostPerConversion":null,"AllReturnOnAdSpend":null,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"AverageCpm":0.0,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":0.0,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1704833886551} +{"stream":"account_impression_performance_report_weekly","data":{"AccountName":"Airbyte","AccountNumber":"F149MJ18","AccountId":180519267,"TimePeriod":"2023-12-17","CurrencyCode":"USD","AdDistribution":"Search","Impressions":639,"Clicks":14,"Ctr":2.19,"AverageCpc":0.12,"Spend":1.74,"AveragePosition":0.0,"Conversions":0,"ConversionRate":0.0,"CostPerConversion":null,"LowQualityClicks":6,"LowQualityClicksPercent":30.0,"LowQualityImpressions":53,"LowQualityImpressionsPercent":7.66,"LowQualityConversions":0,"LowQualityConversionRate":0.0,"DeviceType":"Computer","ImpressionSharePercent":13.57,"ImpressionLostToBudgetPercent":17.96,"ImpressionLostToRankAggPercent":68.47,"PhoneImpressions":0,"PhoneCalls":0,"Ptr":null,"Network":"Syndicated search partners","Assists":0,"Revenue":0.0,"ReturnOnAdSpend":0.0,"CostPerAssist":null,"RevenuePerConversion":null,"RevenuePerAssist":null,"AccountStatus":"Active","LowQualityGeneralClicks":0,"LowQualitySophisticatedClicks":6,"ExactMatchImpressionSharePercent":17.65,"ClickSharePercent":1.28,"AbsoluteTopImpressionSharePercent":3.2,"TopImpressionShareLostToRankPercent":74.15,"TopImpressionShareLostToBudgetPercent":18.25,"AbsoluteTopImpressionShareLostToRankPercent":78.51,"AbsoluteTopImpressionShareLostToBudgetPercent":18.29,"TopImpressionSharePercent":7.6,"AbsoluteTopImpressionRatePercent":22.69,"TopImpressionRatePercent":53.99,"AllConversions":0,"AllRevenue":0.0,"AllConversionRate":0.0,"AllCostPerConversion":null,"AllReturnOnAdSpend":0.0,"AllRevenuePerConversion":null,"ViewThroughConversions":0,"AudienceImpressionSharePercent":null,"AudienceImpressionLostToRankPercent":null,"AudienceImpressionLostToBudgetPercent":null,"AverageCpm":2.72,"ConversionsQualified":0.0,"LowQualityConversionsQualified":0.0,"AllConversionsQualified":0.0,"ViewThroughConversionsQualified":null,"ViewThroughRevenue":0.0,"VideoViews":0,"ViewThroughRate":0.0,"AverageCPV":null,"VideoViewsAt25Percent":0,"VideoViewsAt50Percent":0,"VideoViewsAt75Percent":0,"CompletedVideoViews":0,"VideoCompletionRate":0.0,"TotalWatchTimeInMS":0,"AverageWatchTimePerVideoView":null,"AverageWatchTimePerImpression":0.0,"Sales":0,"CostPerSale":null,"RevenuePerSale":null,"Installs":0,"CostPerInstall":null,"RevenuePerInstall":null},"emitted_at":1704833908003} diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.txt deleted file mode 100644 index e7c35531066f..000000000000 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records.txt +++ /dev/null @@ -1,23 +0,0 @@ -{"stream":"ad_groups","data":{"AdRotation":null,"AudienceAdsBidAdjustment":null,"BiddingScheme":{"Type":"InheritFromParent","InheritedBidStrategyType":"EnhancedCpc"},"CpcBid":{"Amount":0.05},"EndDate":null,"FinalUrlSuffix":null,"ForwardCompatibilityMap":null,"Id":1357897480389129,"Language":null,"Name":"Data Integration Tool","Network":"OwnedAndOperatedAndSyndicatedSearch","PrivacyStatus":null,"Settings":null,"StartDate":{"Day":4,"Month":12,"Year":2020},"Status":"Paused","TrackingUrlTemplate":null,"UrlCustomParameters":null,"AdScheduleUseSearcherTimeZone":false,"AdGroupType":"SearchStandard","CpvBid":{"Amount":null},"CpmBid":{"Amount":null}},"emitted_at":1675189566179} -{"stream":"ads","data":{"AdFormatPreference":"All","DevicePreference":0,"EditorialStatus":"ActiveLimited","FinalAppUrls":null,"FinalMobileUrls":{"string":["https://airbyte.io"]},"FinalUrlSuffix":null,"FinalUrls":{"string":["https://airbyte.io"]},"ForwardCompatibilityMap":null,"Id":84525295496190,"Status":"Active","TrackingUrlTemplate":null,"Type":"ResponsiveSearch","UrlCustomParameters":null,"Descriptions":{"AssetLink":[{"Asset":{"Id":10239221964468,"Name":null,"Type":"TextAsset","Text":"Open data integration for modern data teams"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239221964466,"Name":null,"Type":"TextAsset","Text":"Get your data pipelines running in minutes. With pre-built or custom connectors"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null}]},"Domain":"airbyte.io","Headlines":{"AssetLink":[{"Asset":{"Id":10239221964471,"Name":null,"Type":"TextAsset","Text":"Data Integration Tool"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239221964469,"Name":null,"Type":"TextAsset","Text":"1,000+ Members"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null},{"Asset":{"Id":10239221964467,"Name":null,"Type":"TextAsset","Text":"Data Management Software"},"AssetPerformanceLabel":"Learning","EditorialStatus":"Active","PinnedField":null}]},"Path1":null,"Path2":null},"emitted_at":1675189577521} -{"stream":"campaigns","data":{"AudienceAdsBidAdjustment":0,"BiddingScheme":{"Type":"EnhancedCpc"},"BudgetType":"DailyBudgetStandard","DailyBudget":0.1,"ExperimentId":null,"FinalUrlSuffix":null,"ForwardCompatibilityMap":null,"Id":407519039,"MultimediaAdsBidAdjustment":40,"Name":"integration-test-campaign","Status":"Paused","SubType":null,"TimeZone":"Arizona","TrackingUrlTemplate":null,"UrlCustomParameters":null,"CampaignType":"Search","Settings":{"Setting":[{"Type":"TargetSetting","Details":{"TargetSettingDetail":[{"CriterionTypeGroup":"Audience","TargetAndBid":false}]}}]},"BudgetId":null,"Languages":{"string":["English"]},"AdScheduleUseSearcherTimeZone":false},"emitted_at":1675189585409} -{"stream": "accounts", "data": {"BillToCustomerId": 251186883, "CurrencyCode": "USD", "AccountFinancialStatus": "ClearFinancialStatus", "Id": 180278106, "Language": "English", "LastModifiedByUserId": 0, "LastModifiedTime": "2023-08-11T03:26:10.277000", "Name": "Daxtarity Inc.", "Number": "F149GKV5", "ParentCustomerId": 251186883, "PaymentMethodId": 138188746, "PaymentMethodType": "CreditCard", "PrimaryUserId": 138225488, "AccountLifeCycleStatus": "Active", "TimeStamp": "AAAAAH1yyMo=", "TimeZone": "Arizona", "PauseReason": null, "ForwardCompatibilityMap": null, "LinkedAgencies": null, "SalesHouseCustomerId": null, "TaxInformation": null, "BackUpPaymentInstrumentId": null, "BillingThresholdAmount": null, "BusinessAddress": {"City": "San Francisco", "CountryCode": "US", "Id": 149004358, "Line1": "350 29th avenue", "Line2": null, "Line3": null, "Line4": null, "PostalCode": "94121", "StateOrProvince": "CA", "TimeStamp": null, "BusinessName": "Daxtarity Inc."}, "AutoTagType": "Inactive", "SoldToPaymentInstrumentId": null, "AccountMode": "Expert"}, "emitted_at": 1692381691611} -{"stream":"account_performance_report_daily", "data":{"AccountId": 180278106, "TimePeriod": "2021-08-03", "CurrencyCode": "USD", "AdDistribution": "Search", "DeviceType": "Tablet", "Network": "Syndicated search partners", "DeliveredMatchType": "Exact", "DeviceOS": "Android", "TopVsOther": "Syndicated search partners - Other", "BidMatchType": "Broad", "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "PhoneImpressions": 0, "PhoneCalls": 0, "Clicks": 0, "Ctr": 0.0, "Spend": 0.0, "Impressions": 1, "CostPerConversion": null, "Ptr": null, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "LowQualityClicks": 0, "LowQualityClicksPercent": null, "LowQualityImpressions": 0, "LowQualitySophisticatedClicks": 0, "LowQualityConversions": 0, "LowQualityConversionRate": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1679944835117} -{"stream":"account_performance_report_weekly", "data": {"AccountId": 180278106, "TimePeriod": "2021-08-01", "CurrencyCode": "USD", "AdDistribution": "Search", "DeviceType": "Tablet", "Network": "Syndicated search partners", "DeliveredMatchType": "Broad", "DeviceOS": "Android", "TopVsOther": "Syndicated search partners - Top", "BidMatchType": "Broad", "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "PhoneImpressions": 0, "PhoneCalls": 0, "Clicks": 0, "Ctr": 0.0, "Spend": 0.0, "Impressions": 1, "CostPerConversion": null, "Ptr": null, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "LowQualityClicks": 0, "LowQualityClicksPercent": null, "LowQualityImpressions": 1, "LowQualitySophisticatedClicks": 0, "LowQualityConversions": 0, "LowQualityConversionRate": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1679944953111} -{"stream":"account_performance_report_monthly", "data": {"AccountId": 180278106, "TimePeriod": "2021-08-01", "CurrencyCode": "USD", "AdDistribution": "Search", "DeviceType": "Tablet", "Network": "Syndicated search partners", "DeliveredMatchType": "Broad", "DeviceOS": "Android", "TopVsOther": "Syndicated search partners - Top", "BidMatchType": "Broad", "AccountName": "Daxtarity Inc.", "AccountNumber": "F149GKV5", "PhoneImpressions": 0, "PhoneCalls": 0, "Clicks": 0, "Ctr": 0.0, "Spend": 0.0, "Impressions": 1, "CostPerConversion": null, "Ptr": null, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "LowQualityClicks": 0, "LowQualityClicksPercent": null, "LowQualityImpressions": 1, "LowQualitySophisticatedClicks": 0, "LowQualityConversions": 0, "LowQualityConversionRate": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1679945438344} -{"stream":"ad_group_performance_report_daily", "data": {"AccountId": 180278106, "CampaignId": 413444833, "AdGroupId": 1352400325389092, "TimePeriod": "2021-08-03", "CurrencyCode": "USD", "AdDistribution": "Search", "DeviceType": "Tablet", "Network": "Syndicated search partners", "DeliveredMatchType": "Exact", "DeviceOS": "Android", "TopVsOther": "Syndicated search partners - Other", "BidMatchType": "Broad", "Language": "English", "AccountName": "Daxtarity Inc.", "CampaignName": "Test 2", "CampaignType": "Search & content", "AdGroupName": "Airbyte", "AdGroupType": "Standard", "Impressions": 1, "Clicks": 0, "Ctr": 0.0, "Spend": 0.0, "CostPerConversion": null, "QualityScore": 0.0, "ExpectedCtr": "--", "AdRelevance": 0.0, "LandingPageExperience": 0.0, "PhoneImpressions": 0, "PhoneCalls": 0, "Ptr": null, "Assists": 0, "CostPerAssist": null, "CustomParameters": null, "FinalUrlSuffix": null, "ViewThroughConversions": 0, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null, "HistoricalQualityScore": 8.0, "HistoricalExpectedCtr": 2.0, "HistoricalAdRelevance": 3.0, "HistoricalLandingPageExperience": 3.0}, "emitted_at": 1679950558186} -{"stream":"ad_group_performance_report_weekly", "data": {"AccountId": 180278106, "CampaignId": 413444833, "AdGroupId": 1352400325389092, "TimePeriod": "2021-08-01", "CurrencyCode": "USD", "AdDistribution": "Search", "DeviceType": "Tablet", "Network": "Syndicated search partners", "DeliveredMatchType": "Broad", "DeviceOS": "Android", "TopVsOther": "Syndicated search partners - Top", "BidMatchType": "Broad", "Language": "English", "AccountName": "Daxtarity Inc.", "CampaignName": "Test 2", "CampaignType": "Search & content", "AdGroupName": "Airbyte", "AdGroupType": "Standard", "Impressions": 1, "Clicks": 0, "Ctr": 0.0, "Spend": 0.0, "CostPerConversion": null, "QualityScore": 0.0, "ExpectedCtr": "--", "AdRelevance": 0.0, "LandingPageExperience": 0.0, "PhoneImpressions": 0, "PhoneCalls": 0, "Ptr": null, "Assists": 0, "CostPerAssist": null, "CustomParameters": null, "FinalUrlSuffix": null, "ViewThroughConversions": 0, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null, "HistoricalQualityScore": 8.0, "HistoricalExpectedCtr": 2.0, "HistoricalAdRelevance": 3.0, "HistoricalLandingPageExperience": 3.0}, "emitted_at": 1679945714359} -{"stream":"ad_group_performance_report_monthly", "data": {"AccountId": 180278106, "CampaignId": 413444833, "AdGroupId": 1352400325389092, "TimePeriod": "2021-08-01", "CurrencyCode": "USD", "AdDistribution": "Search", "DeviceType": "Tablet", "Network": "Syndicated search partners", "DeliveredMatchType": "Broad", "DeviceOS": "Android", "TopVsOther": "Syndicated search partners - Top", "BidMatchType": "Broad", "Language": "English", "AccountName": "Daxtarity Inc.", "CampaignName": "Test 2", "CampaignType": "Search & content", "AdGroupName": "Airbyte", "AdGroupType": "Standard", "Impressions": 1, "Clicks": 0, "Ctr": 0.0, "Spend": 0.0, "CostPerConversion": null, "QualityScore": 0.0, "ExpectedCtr": "--", "AdRelevance": 0.0, "LandingPageExperience": 0.0, "PhoneImpressions": 0, "PhoneCalls": 0, "Ptr": null, "Assists": 0, "CostPerAssist": null, "CustomParameters": null, "FinalUrlSuffix": null, "ViewThroughConversions": 0, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null, "HistoricalQualityScore": 8.0, "HistoricalExpectedCtr": 2.0, "HistoricalAdRelevance": 3.0, "HistoricalLandingPageExperience": 3.0}, "emitted_at": 1679945656700} -{"stream": "ad_performance_report_daily", "data": {"AccountId": 180278106, "CampaignId": 413444833, "AdGroupId": 1352400325389092, "AdId": 84525295496190, "TimePeriod": "2021-08-03", "CurrencyCode": "USD", "AdDistribution": "Search", "DeviceType": "Tablet", "Language": "English", "Network": "Syndicated search partners", "DeviceOS": "Android", "TopVsOther": "Syndicated search partners - Other", "BidMatchType": "Broad", "DeliveredMatchType": "Exact", "AccountName": "Daxtarity Inc.", "CampaignName": "Test 2", "CampaignType": "Search & content", "AdGroupName": "Airbyte", "Impressions": 1, "Clicks": 0, "Ctr": 0.0, "Spend": 0.0, "CostPerConversion": null, "DestinationUrl": null, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "CustomParameters": null, "FinalAppUrl": null, "AdDescription": null, "AdDescription2": null, "ViewThroughConversions": 0, "ViewThroughConversionsQualified": null, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1679950716583} -{"stream": "ad_performance_report_weekly", "data": {"AccountId": 180278106, "CampaignId": 413444833, "AdGroupId": 1352400325389092, "AdId": 84525295496190, "TimePeriod": "2021-08-01", "CurrencyCode": "USD", "AdDistribution": "Search", "DeviceType": "Tablet", "Language": "English", "Network": "Syndicated search partners", "DeviceOS": "Android", "TopVsOther": "Syndicated search partners - Other", "BidMatchType": "Broad", "DeliveredMatchType": "Phrase", "AccountName": "Daxtarity Inc.", "CampaignName": "Test 2", "CampaignType": "Search & content", "AdGroupName": "Airbyte", "Impressions": 1, "Clicks": 0, "Ctr": 0.0, "Spend": 0.0, "CostPerConversion": null, "DestinationUrl": null, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "CustomParameters": null, "FinalAppUrl": null, "AdDescription": null, "AdDescription2": null, "ViewThroughConversions": 0, "ViewThroughConversionsQualified": null, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1679950767960} -{"stream": "ad_performance_report_monthly", "data": {"AccountId": 180278106, "CampaignId": 413444833, "AdGroupId": 1352400325389092, "AdId": 84525295496190, "TimePeriod": "2021-08-01", "CurrencyCode": "USD", "AdDistribution": "Search", "DeviceType": "Tablet", "Language": "English", "Network": "Syndicated search partners", "DeviceOS": "Android", "TopVsOther": "Syndicated search partners - Other", "BidMatchType": "Broad", "DeliveredMatchType": "Phrase", "AccountName": "Daxtarity Inc.", "CampaignName": "Test 2", "CampaignType": "Search & content", "AdGroupName": "Airbyte", "Impressions": 1, "Clicks": 0, "Ctr": 0.0, "Spend": 0.0, "CostPerConversion": null, "DestinationUrl": null, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "CustomParameters": null, "FinalAppUrl": null, "AdDescription": null, "AdDescription2": null, "ViewThroughConversions": 0, "ViewThroughConversionsQualified": null, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1679950813350} -{"stream":"budget_summary_report","data":{"AccountName":"Daxtarity Inc.","AccountNumber":"F149GKV5","AccountId":180278106,"CampaignName":"Test 2","CampaignId":413444833,"Date":"6/9/2021","MonthlyBudget":22.8,"DailySpend":0.71,"MonthToDateSpend":0.71},"emitted_at":1675189738185} -{"stream": "campaign_performance_report_daily", "data": {"AccountId": 180278106, "CampaignId": 413444833, "TimePeriod": "2021-06-09", "CurrencyCode": "USD", "AdDistribution": "Search", "DeviceType": "Computer", "Network": "Bing and Yahoo! search", "DeliveredMatchType": "Exact", "DeviceOS": "Windows", "TopVsOther": "Bing and Yahoo! search - Top", "BidMatchType": "Broad", "AccountName": "Daxtarity Inc.", "CampaignName": "Test 2", "CampaignType": "Search & content", "CampaignStatus": "Paused", "CampaignLabels": "integration-test-label;another label;what a new label", "Impressions": 1, "Clicks": 0, "Ctr": 0.0, "Spend": 0.0, "CostPerConversion": null, "QualityScore": 0.0, "AdRelevance": 0.0, "LandingPageExperience": 0.0, "PhoneImpressions": 0, "PhoneCalls": 0, "Ptr": null, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "CustomParameters": null, "ViewThroughConversions": 0, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "LowQualityClicks": 0, "LowQualityClicksPercent": null, "LowQualityImpressions": 2, "LowQualitySophisticatedClicks": 0, "LowQualityConversions": 0, "LowQualityConversionRate": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null, "BudgetName": null, "BudgetStatus": null, "BudgetAssociationStatus": "Current", "HistoricalQualityScore": 8.0, "HistoricalExpectedCtr": 2.0, "HistoricalAdRelevance": 3.0, "HistoricalLandingPageExperience": 3.0}, "emitted_at": 1683583446923} -{"stream": "campaign_performance_report_weekly", "data": {"AccountId": 180278106, "CampaignId": 413444833, "TimePeriod": "2021-07-04", "CurrencyCode": "USD", "AdDistribution": "Search", "DeviceType": "Smartphone", "Network": "Syndicated search partners", "DeliveredMatchType": "Broad", "DeviceOS": "iOS", "TopVsOther": "Syndicated search partners - Other", "BidMatchType": "Broad", "AccountName": "Daxtarity Inc.", "CampaignName": "Test 2", "CampaignType": "Search & content", "CampaignStatus": "Paused", "CampaignLabels": "integration-test-label;another label;what a new label", "Impressions": 5, "Clicks": 0, "Ctr": 0.0, "Spend": 0.0, "CostPerConversion": null, "QualityScore": 0.0, "AdRelevance": 0.0, "LandingPageExperience": 0.0, "PhoneImpressions": 0, "PhoneCalls": 0, "Ptr": null, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "CustomParameters": null, "ViewThroughConversions": 0, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "LowQualityClicks": 0, "LowQualityClicksPercent": null, "LowQualityImpressions": 0, "LowQualitySophisticatedClicks": 0, "LowQualityConversions": 0, "LowQualityConversionRate": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null, "BudgetName": null, "BudgetStatus": null, "BudgetAssociationStatus": "Current", "HistoricalQualityScore": 8.0, "HistoricalExpectedCtr": 2.0, "HistoricalAdRelevance": 3.0, "HistoricalLandingPageExperience": 3.0}, "emitted_at": 1683583674907} -{"stream": "campaign_performance_report_monthly", "data": {"AccountId": 180278106, "CampaignId": 413444833, "TimePeriod": "2021-06-01", "CurrencyCode": "USD", "AdDistribution": "Search", "DeviceType": "Computer", "Network": "Bing and Yahoo! search", "DeliveredMatchType": "Exact", "DeviceOS": "Windows", "TopVsOther": "Bing and Yahoo! search - Top", "BidMatchType": "Broad", "AccountName": "Daxtarity Inc.", "CampaignName": "Test 2", "CampaignType": "Search & content", "CampaignStatus": "Paused", "CampaignLabels": "integration-test-label;another label;what a new label", "Impressions": 11, "Clicks": 0, "Ctr": 0.0, "Spend": 0.0, "CostPerConversion": null, "QualityScore": 0.0, "AdRelevance": 0.0, "LandingPageExperience": 0.0, "PhoneImpressions": 0, "PhoneCalls": 0, "Ptr": null, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "CustomParameters": null, "ViewThroughConversions": 0, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "LowQualityClicks": 0, "LowQualityClicksPercent": null, "LowQualityImpressions": 30, "LowQualitySophisticatedClicks": 0, "LowQualityConversions": 0, "LowQualityConversionRate": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null, "BudgetName": null, "BudgetStatus": null, "BudgetAssociationStatus": "Current", "HistoricalQualityScore": 8.0, "HistoricalExpectedCtr": 2.0, "HistoricalAdRelevance": 3.0, "HistoricalLandingPageExperience": 3.0}, "emitted_at": 1683583743890} -{"stream": "keyword_performance_report_daily", "data": {"AccountId": 180278106, "CampaignId": 413444833, "AdGroupId": 1352400325389092, "KeywordId": 84525593559629, "AdId": 84525295496190, "TimePeriod": "2021-08-03", "CurrencyCode": "USD", "DeliveredMatchType": "Broad", "AdDistribution": "Search", "DeviceType": "Smartphone", "Language": "English", "Network": "Syndicated search partners", "DeviceOS": "Android", "TopVsOther": "Syndicated search partners - Top", "BidMatchType": "Broad", "AccountName": "Daxtarity Inc.", "CampaignName": "Test 2", "AdGroupName": "Airbyte", "Keyword": "data integration tools", "KeywordStatus": "Active", "Impressions": 1, "Clicks": 0, "Ctr": 0.0, "CurrentMaxCpc": 0.11, "Spend": 0.0, "CostPerConversion": null, "QualityScore": 0.0, "ExpectedCtr": "--", "AdRelevance": 0.0, "LandingPageExperience": 0.0, "QualityImpact": 0.0, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "CustomParameters": null, "FinalAppUrl": null, "Mainline1Bid": null, "MainlineBid": null, "FirstPageBid": null, "FinalUrlSuffix": null, "ViewThroughConversions": 0, "ViewThroughConversionsQualified": null, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null, "HistoricalQualityScore": 8.0, "HistoricalExpectedCtr": 2.0, "HistoricalAdRelevance": 3.0, "HistoricalLandingPageExperience": 3.0}, "emitted_at": 1679951505600} -{"stream": "keyword_performance_report_weekly", "data": {"AccountId": 180278106, "CampaignId": 413444833, "AdGroupId": 1352400325389092, "KeywordId": 84525593559629, "AdId": 84525295496190, "TimePeriod": "2021-08-01", "CurrencyCode": "USD", "DeliveredMatchType": "Broad", "AdDistribution": "Search", "DeviceType": "Tablet", "Language": "English", "Network": "Syndicated search partners", "DeviceOS": "Android", "TopVsOther": "Syndicated search partners - Top", "BidMatchType": "Broad", "AccountName": "Daxtarity Inc.", "CampaignName": "Test 2", "AdGroupName": "Airbyte", "Keyword": "data integration tools", "KeywordStatus": "Active", "Impressions": 1, "Clicks": 0, "Ctr": 0.0, "CurrentMaxCpc": 0.11, "Spend": 0.0, "CostPerConversion": null, "QualityScore": 0.0, "ExpectedCtr": "--", "AdRelevance": 0.0, "LandingPageExperience": 0.0, "QualityImpact": 0.0, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "CustomParameters": null, "FinalAppUrl": null, "Mainline1Bid": null, "MainlineBid": null, "FirstPageBid": null, "FinalUrlSuffix": null, "ViewThroughConversions": 0, "ViewThroughConversionsQualified": null, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1679951543951} -{"stream": "keyword_performance_report_monthly", "data": {"AccountId": 180278106, "CampaignId": 413444833, "AdGroupId": 1352400325389092, "KeywordId": 84525593559629, "AdId": 84525295496190, "TimePeriod": "2021-08-01", "CurrencyCode": "USD", "DeliveredMatchType": "Broad", "AdDistribution": "Search", "DeviceType": "Tablet", "Language": "English", "Network": "Syndicated search partners", "DeviceOS": "Android", "TopVsOther": "Syndicated search partners - Top", "BidMatchType": "Broad", "AccountName": "Daxtarity Inc.", "CampaignName": "Test 2", "AdGroupName": "Airbyte", "Keyword": "data integration tools", "KeywordStatus": "Active", "Impressions": 1, "Clicks": 0, "Ctr": 0.0, "CurrentMaxCpc": 0.11, "Spend": 0.0, "CostPerConversion": null, "QualityScore": 0.0, "ExpectedCtr": "--", "AdRelevance": 0.0, "LandingPageExperience": 0.0, "QualityImpact": 0.0, "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "CustomParameters": null, "FinalAppUrl": null, "Mainline1Bid": null, "MainlineBid": null, "FirstPageBid": null, "FinalUrlSuffix": null, "ViewThroughConversions": 0, "ViewThroughConversionsQualified": null, "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1679951588461} -{"stream": "geographic_performance_report_daily", "data": {"AccountId": 180278106, "CampaignId": 413444833, "AdGroupId": 1352400325389092, "TimePeriod": "2021-06-09", "Country": "Australia", "State": "New South Wales", "MetroArea": null, "City": null, "CurrencyCode": "USD", "DeliveredMatchType": "Broad", "AdDistribution": "Search", "DeviceType": "Computer", "Language": "English", "Network": "Syndicated search partners", "DeviceOS": "Windows", "TopVsOther": "Syndicated search partners - Top", "BidMatchType": "Broad", "AdGroupName": "Airbyte", "Ctr": 0.0, "ProximityTargetLocation": null, "Radius": "0", "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "LocationType": "Physical location", "MostSpecificLocation": "2000", "AccountStatus": "Active", "CampaignStatus": "Paused", "AdGroupStatus": "Active", "County": null, "PostalCode": "2000", "LocationId": "122395", "BaseCampaignId": "413444833", "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "ViewThroughConversions": 0, "Goal": null, "GoalType": null, "AbsoluteTopImpressionRatePercent": 0.0, "TopImpressionRatePercent": "100.00%", "AllConversionsQualified": "0.00", "ViewThroughConversionsQualified": null, "Neighborhood": null, "ViewThroughRevenue": "0.00", "CampaignType": "Search & content", "AssetGroupId": null, "AssetGroupName": null, "AssetGroupStatus": null, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1692275353788} -{"stream": "geographic_performance_report_weekly", "data": {"AccountId": 180278106, "CampaignId": 413444833, "AdGroupId": 1352400325389092, "TimePeriod": "2021-06-06", "Country": "Australia", "State": null, "MetroArea": null, "City": null, "CurrencyCode": "USD", "DeliveredMatchType": "Broad", "AdDistribution": "Search", "DeviceType": "Computer", "Language": "English", "Network": "Syndicated search partners", "DeviceOS": "Unknown", "TopVsOther": "Syndicated search partners - Top", "BidMatchType": "Broad", "AdGroupName": "Airbyte", "Ctr": 0.0, "ProximityTargetLocation": null, "Radius": "0", "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "LocationType": "Physical location", "MostSpecificLocation": "Australia", "AccountStatus": "Active", "CampaignStatus": "Paused", "AdGroupStatus": "Active", "County": null, "PostalCode": null, "LocationId": "9", "BaseCampaignId": "413444833", "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "ViewThroughConversions": 0, "Goal": null, "GoalType": null, "AbsoluteTopImpressionRatePercent": 0.0, "TopImpressionRatePercent": "100.00%", "AllConversionsQualified": "0.00", "ViewThroughConversionsQualified": null, "Neighborhood": null, "ViewThroughRevenue": "0.00", "CampaignType": "Search & content", "AssetGroupId": null, "AssetGroupName": null, "AssetGroupStatus": null, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1692275162937} -{"stream": "geographic_performance_report_monthly", "data": {"AccountId": 180278106, "CampaignId": 413444833, "AdGroupId": 1352400325389092, "TimePeriod": "2021-06-01", "Country": "United Arab Emirates", "State": null, "MetroArea": null, "City": null, "CurrencyCode": "USD", "DeliveredMatchType": "Broad", "AdDistribution": "Search", "DeviceType": "Computer", "Language": "English", "Network": "Bing and Yahoo! search", "DeviceOS": "Windows", "TopVsOther": "Bing and Yahoo! search - Top", "BidMatchType": "Broad", "AdGroupName": "Airbyte", "Ctr": 0.0, "ProximityTargetLocation": null, "Radius": "0", "Assists": 0, "ReturnOnAdSpend": null, "CostPerAssist": null, "LocationType": "Physical location", "MostSpecificLocation": "United Arab Emirates", "AccountStatus": "Active", "CampaignStatus": "Paused", "AdGroupStatus": "Active", "County": null, "PostalCode": null, "LocationId": "218", "BaseCampaignId": "413444833", "AllCostPerConversion": null, "AllReturnOnAdSpend": null, "ViewThroughConversions": 0, "Goal": null, "GoalType": null, "AbsoluteTopImpressionRatePercent": 0.0, "TopImpressionRatePercent": "100.00%", "AllConversionsQualified": "0.00", "ViewThroughConversionsQualified": null, "Neighborhood": null, "ViewThroughRevenue": "0.00", "CampaignType": "Search & content", "AssetGroupId": null, "AssetGroupName": null, "AssetGroupStatus": null, "Conversions": 0.0, "ConversionRate": null, "ConversionsQualified": 0.0, "AverageCpc": 0.0, "AveragePosition": 0.0, "AverageCpm": 0.0, "AllConversions": 0, "AllConversionRate": null, "AllRevenue": 0.0, "AllRevenuePerConversion": null, "Revenue": 0.0, "RevenuePerConversion": null, "RevenuePerAssist": null}, "emitted_at": 1692275633242} diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records_no_start_date.jsonl b/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records_no_start_date.jsonl new file mode 100644 index 000000000000..d9be9afa0293 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/expected_records_no_start_date.jsonl @@ -0,0 +1,5 @@ +{"stream": "keyword_labels", "data": {"Status": "Active", "Id": 10239203506495, "Parent Id": 84868925026027, "Client Id": null, "Modified Time": "2023-04-27T17:22:52.733+00:00", "Account Id": 180278106}, "emitted_at": 1701982357451} +{"stream": "keywords", "data": {"Status": "Active", "Id": 84801135055370, "Parent Id": "1356799861840328", "Campaign": "Airbyte test", "Ad Group": "keywords", "Client Id": null, "Modified Time": "2023-11-07T12:21:17.120+00:00", "Tracking Template": null, "Final Url Suffix": null, "Custom Parameter": null, "Final Url": null, "Mobile Final Url": null, "Bid Strategy Type": "InheritFromParent", "Inherited Bid Strategy Type": "EnhancedCpc", "Destination Url": null, "Editorial Status": "Active", "Editorial Location": null, "Editorial Term": null, "Editorial Reason Code": null, "Editorial Appeal Status": null, "Keyword": "Airbyte", "Match Type": "Broad", "Bid": null, "Param1": null, "Param2": null, "Param3": null, "Publisher Countries": null, "Quality Score": null, "Keyword Relevance": null, "Landing Page Relevance": null, "Landing Page User Experience": null, "Account Id": 180519267}, "emitted_at": 1701982417302} +{"stream": "ad_group_labels", "data": {"Status": "Active", "Id": 10239203506495, "Parent Id": 1350201453189474, "Campaign": null, "Ad Group": null, "Client Id": null, "Modified Time": "2023-04-27T18:00:14.970+00:00", "Account Id": 180278106}, "emitted_at": 1701982478843} +{"stream": "labels", "data": {"Status": "Active", "Id": 10239203506496, "Client Id": null, "Modified Time": "2023-04-27T17:16:53.430+00:00", "Description": null, "Label": "campaign label 2", "Color": "#D8558B", "Account Id": 180278106}, "emitted_at": 1701982532098} +{"stream": "campaign_labels", "data": {"Status": "Active", "Id": 10239203506495, "Parent Id": 413732450, "Campaign": null, "Client Id": null, "Modified Time": "2023-04-27T17:57:21.497+00:00", "Account Id": 180278106}, "emitted_at": 1701982600348} diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-bing-ads/integration_tests/sample_state.json new file mode 100644 index 000000000000..8b6447f78e3e --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/integration_tests/sample_state.json @@ -0,0 +1,46 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "180278106": { + "TimePeriod": "2021-08-01T14:00:00+00:00" + } + }, + "stream_descriptor": { "name": "keyword_performance_report_hourly" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "180278106": { + "Date": "2021-08-01" + } + }, + "stream_descriptor": { "name": "budget_summary_report" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "180278106": { + "TimePeriod": "2021-08-01" + } + }, + "stream_descriptor": { "name": "ad_performance_report_weekly" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "180278106": { + "TimePeriod": "2023-04-27T15:16:49.170+00:00" + } + }, + "stream_descriptor": { "name": "labels" } + } + } +] diff --git a/airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json b/airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json deleted file mode 100644 index ef99548a8bb2..000000000000 --- a/airbyte-integrations/connectors/source-bing-ads/integration_tests/state.json +++ /dev/null @@ -1,46 +0,0 @@ -[ - { - "type": "STREAM", - "stream": { - "stream_state": { - "180278106": { - "TimePeriod": 1627820152 - } - }, - "stream_descriptor": { "name": "keyword_performance_report_hourly" } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "180278106": { - "Date": 1627800152 - } - }, - "stream_descriptor": { "name": "budget_summary_report_hourly" } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "180278106": { - "TimePeriod": 1627795152 - } - }, - "stream_descriptor": { "name": "ad_performance_report_hourly" } - } - }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "180278106": { - "TimePeriod": 1727810152 - } - }, - "stream_descriptor": { "name": "campaign_performance_report_hourly" } - } - } -] diff --git a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml index 630df575f0a7..5420384149a1 100644 --- a/airbyte-integrations/connectors/source-bing-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-bing-ads/metadata.yaml @@ -1,4 +1,7 @@ data: + ab_internal: + ql: 400 + sl: 200 allowedHosts: hosts: - bingads.microsoft.com @@ -8,11 +11,14 @@ data: - ads.microsoft.com - api.ads.microsoft.com - clientcenter.api.bingads.microsoft.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: api connectorType: source definitionId: 47f25999-dd5e-4636-8c39-e7cea2453331 - dockerImageTag: 0.2.0 + dockerImageTag: 2.1.2 dockerRepository: airbyte/source-bing-ads + documentationUrl: https://docs.airbyte.com/integrations/sources/bing-ads githubIssueLabel: source-bing-ads icon: bingads.svg license: MIT @@ -23,11 +29,31 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/bing-ads + releases: + breakingChanges: + 1.0.0: + message: + Version 1.0.0 removes the primary keys from the geographic performance + report streams. This will prevent the connector from losing data in the + incremental append+dedup sync mode because of deduplication and incorrect + primary keys. A data reset and schema refresh of all the affected streams + is required for the changes to take effect. + upgradeDeadline: "2023-10-25" + 2.0.0: + message: > + Version 2.0.0 updates schemas for all hourly reports (end in report_hourly), and the following streams: Accounts, Campaigns, Search Query Performance Report, AppInstallAds, AppInstallAdLabels, Labels, Campaign Labels, Keyword Labels, Ad Group Labels, Keywords, and Budget Summary Report. + Users will need to refresh the source schema and reset affected streams after upgrading. + upgradeDeadline: "2023-12-11" + suggestedStreams: + streams: + - campaigns + - ad_performance_report_daily + - campaign_performance_report_daily + - account_performance_report_daily + - ad_group_performance_report_daily + - accounts + - ad_groups + supportLevel: certified tags: - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-bing-ads/setup.py b/airbyte-integrations/connectors/source-bing-ads/setup.py index c342863f078e..e586d0ea27ae 100644 --- a/airbyte-integrations/connectors/source-bing-ads/setup.py +++ b/airbyte-integrations/connectors/source-bing-ads/setup.py @@ -5,9 +5,10 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk", "bingads~=13.0.13", "vcrpy==4.1.1", "backoff==1.10.0", "pendulum==2.1.2", "urllib3<2.0"] +MAIN_REQUIREMENTS = ["airbyte-cdk", "cached_property~=1.5", "bingads~=13.0.17", "urllib3<2.0", "pandas"] TEST_REQUIREMENTS = [ + "freezegun", "requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1", diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/base_streams.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/base_streams.py new file mode 100644 index 000000000000..7294f453deb1 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/base_streams.py @@ -0,0 +1,351 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import ssl +import time +from abc import ABC, abstractmethod +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union +from urllib.error import URLError + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams import Stream +from bingads.service_client import ServiceClient +from bingads.v13.reporting.reporting_service_manager import ReportingServiceManager +from source_bing_ads.client import Client +from suds import sudsobject + + +class BingAdsBaseStream(Stream, ABC): + primary_key: Optional[Union[str, List[str], List[List[str]]]] = None + + def __init__(self, client: Client, config: Mapping[str, Any]) -> None: + super().__init__() + self.client = client + self.config = config + + +class BingAdsStream(BingAdsBaseStream, ABC): + @property + @abstractmethod + def operation_name(self) -> str: + """ + Specifies operation name to use for a current stream + """ + + @property + @abstractmethod + def service_name(self) -> str: + """ + Specifies bing ads service name for a current stream + """ + + @property + def parent_key_to_foreign_key_map(self) -> MutableMapping[str, str]: + """ + Specifies dict with field in record as kay and slice key as value to be inserted in record in transform method. + """ + return {} + + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + foreign_keys = {key: stream_slice.get(value) for key, value in self.parent_key_to_foreign_key_map.items()} + return record | foreign_keys + + @property + def _service(self) -> Union[ServiceClient, ReportingServiceManager]: + return self.client.get_service(service_name=self.service_name) + + @property + def _user_id(self) -> int: + return self._get_user_id() + + # TODO remove once Microsoft support confirm their SSL certificates are always valid... + def _get_user_id(self, number_of_retries=10): + """""" + try: + return self._service.GetUser().User.Id + except URLError as error: + if isinstance(error.reason, ssl.SSLError): + self.logger.warning("SSL certificate error, retrying...") + if number_of_retries > 0: + time.sleep(1) + return self._get_user_id(number_of_retries - 1) + else: + raise error + + def next_page_token(self, response: sudsobject.Object, **kwargs: Mapping[str, Any]) -> Optional[Mapping[str, Any]]: + """ + Default method for streams that don't support pagination + """ + return None + + def send_request(self, params: Mapping[str, Any], customer_id: str, account_id: str = None) -> Mapping[str, Any]: + request_kwargs = { + "service_name": self.service_name, + "customer_id": customer_id, + "account_id": account_id, + "operation_name": self.operation_name, + "params": params, + } + request = self.client.request(**request_kwargs) + return request + + def read_records( + self, + sync_mode: SyncMode, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + **kwargs: Mapping[str, Any], + ) -> Iterable[Mapping[str, Any]]: + stream_state = stream_state or {} + next_page_token = None + account_id = str(stream_slice.get("account_id")) if stream_slice else None + customer_id = str(stream_slice.get("customer_id")) if stream_slice else None + + while True: + params = self.request_params( + stream_state=stream_state, + stream_slice=stream_slice, + next_page_token=next_page_token, + account_id=account_id, + ) + response = self.send_request(params, customer_id=customer_id, account_id=account_id) + for record in self.parse_response(response): + yield self.transform(record, stream_slice) + + next_page_token = self.next_page_token(response, current_page_token=next_page_token) + if not next_page_token: + break + + def parse_response(self, response: sudsobject.Object, **kwargs) -> Iterable[Mapping]: + if response is not None and hasattr(response, self.data_field): + yield from self.client.asdict(response)[self.data_field] + + +class BingAdsCampaignManagementStream(BingAdsStream, ABC): + service_name: str = "CampaignManagement" + + @property + @abstractmethod + def data_field(self) -> str: + """ + Specifies root object name in a stream response + """ + + @property + @abstractmethod + def additional_fields(self) -> Optional[str]: + """ + Specifies which additional fields to fetch for a current stream. + Expected format: field names separated by space + """ + + def parse_response(self, response: sudsobject.Object, **kwargs) -> Iterable[Mapping]: + if response is not None and hasattr(response, self.data_field): + yield from self.client.asdict(response)[self.data_field] + + +class Accounts(BingAdsStream): + """ + Searches for accounts that the current authenticated user can access. + API doc: https://docs.microsoft.com/en-us/advertising/customer-management-service/searchaccounts?view=bingads-13 + Account schema: https://docs.microsoft.com/en-us/advertising/customer-management-service/advertiseraccount?view=bingads-13 + Stream caches incoming responses to be able to reuse this data in Campaigns stream + """ + + primary_key = "Id" + # Stream caches incoming responses to avoid duplicated http requests + use_cache: bool = True + data_field: str = "AdvertiserAccount" + service_name: str = "CustomerManagementService" + operation_name: str = "SearchAccounts" + additional_fields: str = "TaxCertificate AccountMode" + # maximum page size + page_size_limit: int = 1000 + + def __init__(self, client: Client, config: Mapping[str, Any]) -> None: + super().__init__(client, config) + self._account_names = config.get("account_names", []) + self._unique_account_ids = set() + + def next_page_token(self, response: sudsobject.Object, current_page_token: Optional[int]) -> Optional[Mapping[str, Any]]: + current_page_token = current_page_token or 0 + if response is not None and hasattr(response, self.data_field): + return None if self.page_size_limit > len(response[self.data_field]) else current_page_token + 1 + else: + return None + + def stream_slices( + self, + **kwargs: Mapping[str, Any], + ) -> Iterable[Optional[Mapping[str, Any]]]: + user_id_predicate = { + "Field": "UserId", + "Operator": "Equals", + "Value": self._user_id, + } + if self._account_names: + for account_config in self._account_names: + account_name_predicate = {"Field": "AccountName", "Operator": account_config["operator"], "Value": account_config["name"]} + + yield {"predicates": {"Predicate": [user_id_predicate, account_name_predicate]}} + else: + yield {"predicates": {"Predicate": [user_id_predicate]}} + + def request_params( + self, + next_page_token: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + **kwargs: Mapping[str, Any], + ) -> MutableMapping[str, Any]: + paging = self._service.factory.create("ns5:Paging") + paging.Index = next_page_token or 0 + paging.Size = self.page_size_limit + return { + "PageInfo": paging, + "Predicates": stream_slice["predicates"], + "ReturnAdditionalFields": self.additional_fields, + } + + def parse_response(self, response: sudsobject.Object, **kwargs) -> Iterable[Mapping]: + if response is not None and hasattr(response, self.data_field): + records = self.client.asdict(response)[self.data_field] + for record in records: + if record["Id"] not in self._unique_account_ids: + self._unique_account_ids.add(record["Id"]) + yield record + + +class Campaigns(BingAdsCampaignManagementStream): + """ + Gets the campaigns for all provided accounts. + API doc: https://docs.microsoft.com/en-us/advertising/campaign-management-service/getcampaignsbyaccountid?view=bingads-13 + Campaign schema: https://docs.microsoft.com/en-us/advertising/campaign-management-service/campaign?view=bingads-13 + Stream caches incoming responses to be able to reuse this data in AdGroups stream + """ + + primary_key = "Id" + # Stream caches incoming responses to avoid duplicated http requests + use_cache: bool = True + data_field: str = "Campaign" + operation_name: str = "GetCampaignsByAccountId" + additional_fields: Iterable[str] = [ + "AdScheduleUseSearcherTimeZone", + "BidStrategyId", + "CpvCpmBiddingScheme", + "DynamicDescriptionSetting", + "DynamicFeedSetting", + "MaxConversionValueBiddingScheme", + "MultimediaAdsBidAdjustment", + "TargetImpressionShareBiddingScheme", + "TargetSetting", + "VerifiedTrackingSetting", + ] + campaign_types: Iterable[str] = ["Audience", "DynamicSearchAds", "Search", "Shopping"] + + parent_key_to_foreign_key_map = { + "AccountId": "account_id", + "CustomerId": "customer_id", + } + + def request_params( + self, + stream_slice: Mapping[str, Any] = None, + **kwargs: Mapping[str, Any], + ) -> MutableMapping[str, Any]: + return { + "AccountId": stream_slice["account_id"], + "CampaignType": " ".join(self.campaign_types), + "ReturnAdditionalFields": " ".join(self.additional_fields), + } + + def stream_slices( + self, + **kwargs: Mapping[str, Any], + ) -> Iterable[Optional[Mapping[str, Any]]]: + accounts = Accounts(self.client, self.config) + for _slice in accounts.stream_slices(): + for account in accounts.read_records(SyncMode.full_refresh, _slice): + yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} + + +class AdGroups(BingAdsCampaignManagementStream): + """ + Gets the ad groups for all provided accounts. + API doc: https://docs.microsoft.com/en-us/advertising/campaign-management-service/getadgroupsbycampaignid?view=bingads-13 + AdGroup schema: https://docs.microsoft.com/en-us/advertising/campaign-management-service/adgroup?view=bingads-13 + Stream caches incoming responses to be able to reuse this data in Ads stream + """ + + primary_key = "Id" + # Stream caches incoming responses to avoid duplicated http requests + use_cache: bool = True + data_field: str = "AdGroup" + operation_name: str = "GetAdGroupsByCampaignId" + additional_fields: str = "AdGroupType AdScheduleUseSearcherTimeZone CpmBid CpvBid MultimediaAdsBidAdjustment" + + parent_key_to_foreign_key_map = {"CampaignId": "campaign_id", "AccountId": "account_id", "CustomerId": "customer_id"} + + def request_params( + self, + stream_slice: Mapping[str, Any] = None, + **kwargs: Mapping[str, Any], + ) -> MutableMapping[str, Any]: + return {"CampaignId": stream_slice["campaign_id"], "ReturnAdditionalFields": self.additional_fields} + + def stream_slices( + self, + **kwargs: Mapping[str, Any], + ) -> Iterable[Optional[Mapping[str, Any]]]: + campaigns = Campaigns(self.client, self.config) + accounts = Accounts(self.client, self.config) + for _slice in accounts.stream_slices(): + for account in accounts.read_records(SyncMode.full_refresh, _slice): + for campaign in campaigns.read_records( + sync_mode=SyncMode.full_refresh, stream_slice={"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} + ): + yield {"campaign_id": campaign["Id"], "account_id": account["Id"], "customer_id": account["ParentCustomerId"]} + + +class Ads(BingAdsCampaignManagementStream): + """ + Retrieves the ads for all provided accounts. + API doc: https://docs.microsoft.com/en-us/advertising/campaign-management-service/getadsbyadgroupid?view=bingads-13 + Ad schema: https://docs.microsoft.com/en-us/advertising/campaign-management-service/ad?view=bingads-13 + """ + + primary_key = "Id" + data_field: str = "Ad" + operation_name: str = "GetAdsByAdGroupId" + additional_fields: str = "ImpressionTrackingUrls Videos LongHeadlines" + ad_types: Iterable[str] = [ + "Text", + "Image", + "Product", + "AppInstall", + "ExpandedText", + "DynamicSearch", + "ResponsiveAd", + "ResponsiveSearch", + ] + + parent_key_to_foreign_key_map = {"AdGroupId": "ad_group_id", "AccountId": "account_id", "CustomerId": "customer_id"} + + def request_params( + self, + stream_slice: Mapping[str, Any] = None, + **kwargs: Mapping[str, Any], + ) -> MutableMapping[str, Any]: + return { + "AdGroupId": stream_slice["ad_group_id"], + "AdTypes": {"AdType": self.ad_types}, + "ReturnAdditionalFields": self.additional_fields, + } + + def stream_slices( + self, + **kwargs: Mapping[str, Any], + ) -> Iterable[Optional[Mapping[str, Any]]]: + ad_groups = AdGroups(self.client, self.config) + for slice in ad_groups.stream_slices(sync_mode=SyncMode.full_refresh): + for ad_group in ad_groups.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice): + yield {"ad_group_id": ad_group["Id"], "account_id": slice["account_id"], "customer_id": slice["customer_id"]} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/bulk_streams.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/bulk_streams.py new file mode 100644 index 000000000000..440b0a3607a0 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/bulk_streams.py @@ -0,0 +1,198 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import os +from abc import ABC, abstractmethod +from datetime import timezone +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import pandas as pd +import pendulum +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams import IncrementalMixin +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +from numpy import nan +from source_bing_ads.base_streams import Accounts, BingAdsBaseStream +from source_bing_ads.utils import transform_bulk_datetime_format_to_rfc_3339 + + +class BingAdsBulkStream(BingAdsBaseStream, IncrementalMixin, ABC): + + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization) + cursor_field = "Modified Time" + primary_key = "Id" + _state = {} + + @staticmethod + @transformer.registerCustomTransform + def custom_transform_date_rfc3339(original_value, field_schema): + if original_value and "format" in field_schema and field_schema["format"] == "date-time": + transformed_value = transform_bulk_datetime_format_to_rfc_3339(original_value) + return transformed_value + return original_value + + @property + @abstractmethod + def data_scope(self) -> List[str]: + """ + Defines scopes or types of data to download. Docs: https://learn.microsoft.com/en-us/advertising/bulk-service/datascope?view=bingads-13 + """ + + @property + @abstractmethod + def download_entities(self) -> List[str]: + """ + Defines the entities that should be downloaded. Docs: https://learn.microsoft.com/en-us/advertising/bulk-service/downloadentity?view=bingads-13 + """ + + def stream_slices( + self, + **kwargs: Mapping[str, Any], + ) -> Iterable[Optional[Mapping[str, Any]]]: + accounts = Accounts(self.client, self.config) + for _slice in accounts.stream_slices(): + for account in accounts.read_records(SyncMode.full_refresh, _slice): + yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} + + @property + def state(self) -> Mapping[str, Any]: + return self._state + + @state.setter + def state(self, value: Mapping[str, Any]): + # if key 'Account Id' exists, so we receive a record that should be parsed to state + # otherwise state object from connection state was received + account_id = value.get("Account Id") + + if account_id and value[self.cursor_field]: + current_state_value = self._state.get(str(value["Account Id"]), {}).get(self.cursor_field, "") + record_state_value = transform_bulk_datetime_format_to_rfc_3339(value[self.cursor_field]) + new_state_value = max(current_state_value, record_state_value) + self._state.update({str(value["Account Id"]): {self.cursor_field: new_state_value}}) + else: + self._state.update(value) + + def get_start_date(self, stream_state: Mapping[str, Any] = None, account_id: str = None) -> Optional[pendulum.DateTime]: + """ + The start_date in the query can only be specified if it is within a period of up to 30 days from today. + """ + min_available_date = pendulum.now().subtract(days=30).astimezone(tz=timezone.utc) + start_date = self.client.reports_start_date + if stream_state.get(account_id, {}).get(self.cursor_field): + start_date = pendulum.parse(stream_state[account_id][self.cursor_field]) + return start_date if start_date and start_date > min_available_date else None + + def read_records( + self, + sync_mode: SyncMode, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + **kwargs: Mapping[str, Any], + ) -> Iterable[Mapping[str, Any]]: + stream_state = stream_state or {} + account_id = str(stream_slice.get("account_id")) if stream_slice else None + customer_id = str(stream_slice.get("customer_id")) if stream_slice else None + + report_file_path = self.client.get_bulk_entity( + data_scope=self.data_scope, + download_entities=self.download_entities, + customer_id=customer_id, + account_id=account_id, + start_date=self.get_start_date(stream_state, account_id), + ) + for record in self.read_with_chunks(report_file_path): + record = self.transform(record, stream_slice) + yield record + self.state = record + + def read_with_chunks(self, path: str, chunk_size: int = 1024) -> Iterable[Tuple[int, Mapping[str, Any]]]: + try: + with open(path, "r") as data: + chunks = pd.read_csv(data, chunksize=chunk_size, iterator=True, dialect="unix", dtype=object) + for chunk in chunks: + chunk = chunk.replace({nan: None}).to_dict(orient="records") + for row in chunk: + if row.get("Type") not in ("Format Version", "Account"): + yield row + except pd.errors.EmptyDataError as e: + self.logger.info(f"Empty data received. {e}") + except IOError as ioe: + self.logger.fatal( + f"The IO/Error occurred while reading tmp data. Called: {path}. Stream: {self.name}", + ) + raise ioe + finally: + # remove binary tmp file, after data is read + os.remove(path) + + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + """ + Bing Ads Bulk API returns all available properties for all entities. + This method filter out only available properties. + """ + actual_record = {key: value for key, value in record.items() if key in self.get_json_schema()["properties"].keys()} + actual_record["Account Id"] = stream_slice.get("account_id") + return actual_record + + +class AppInstallAds(BingAdsBulkStream): + """ + https://learn.microsoft.com/en-us/advertising/bulk-service/app-install-ad?view=bingads-13 + """ + + data_scope = ["EntityData"] + download_entities = ["AppInstallAds"] + + +class AppInstallAdLabels(BingAdsBulkStream): + """ + https://learn.microsoft.com/en-us/advertising/bulk-service/app-install-ad-label?view=bingads-13 + """ + + data_scope = ["EntityData"] + download_entities = ["AppInstallAdLabels"] + + +class Labels(BingAdsBulkStream): + """ + https://learn.microsoft.com/en-us/advertising/bulk-service/label?view=bingads-13 + """ + + data_scope = ["EntityData"] + download_entities = ["Labels"] + + +class KeywordLabels(BingAdsBulkStream): + """ + https://learn.microsoft.com/en-us/advertising/bulk-service/keyword-label?view=bingads-13 + """ + + data_scope = ["EntityData"] + download_entities = ["KeywordLabels"] + + +class Keywords(BingAdsBulkStream): + """ + https://learn.microsoft.com/en-us/advertising/bulk-service/keyword?view=bingads-13 + """ + + data_scope = ["EntityData"] + download_entities = ["Keywords"] + + +class CampaignLabels(BingAdsBulkStream): + """ + https://learn.microsoft.com/en-us/advertising/bulk-service/campaign-label?view=bingads-13 + """ + + data_scope = ["EntityData"] + download_entities = ["CampaignLabels"] + + +class AdGroupLabels(BingAdsBulkStream): + """ + https://learn.microsoft.com/en-us/advertising/bulk-service/ad-group-label?view=bingads-13 + """ + + data_scope = ["EntityData"] + download_entities = ["AdGroupLabels"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/cache.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/cache.py deleted file mode 100644 index 52736e8ee945..000000000000 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/cache.py +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import contextlib -import tempfile - -import vcr - - -def _matcher(r1: vcr.request.Request, r2: vcr.request.Request) -> None: - """ - Defines algorithm to compare two bing ads requests. - Makes sure that uri, body and headers are equal in both requests - """ - assert r1.uri == r2.uri and r1.body == r2.body and r1.headers == r2.headers - - -class VcrCache: - """ - VcrPy wrapper to cache bing ads requests, and to be able to reuse results in other streams - """ - - def __init__(self) -> None: - self._vcr = vcr.VCR() - self._vcr.register_matcher("default", _matcher) - # Register default matcher - self._vcr.match_on = ["default"] - - self._cache_file = tempfile.NamedTemporaryFile() - # Init inmemory cache file with empty data - self._cache_file.write(b"interactions: []") - self._cache_file.flush() - self._cache_file.close() - - @contextlib.contextmanager - def use_cassette(self) -> None: - """ - Implements use_cassette method wrapper which uses in-memory temporary file for caching and yaml format for serialization - """ - with self._vcr.use_cassette(self._cache_file.name, record_mode="new_episodes", serializer="yaml"): - yield diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py index 0d9c7506493e..afd7862a93f9 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py @@ -1,24 +1,33 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - +import os import socket import ssl import sys +import uuid from datetime import datetime, timedelta, timezone from functools import lru_cache -from typing import Any, Iterator, Mapping, Optional, Union +from typing import Any, Iterator, List, Mapping, Optional, Union from urllib.error import URLError import backoff import pendulum from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.models import FailureType +from airbyte_cdk.utils import AirbyteTracedException from bingads.authorization import AuthorizationData, OAuthTokens, OAuthWebAuthCodeGrant +from bingads.exceptions import OAuthTokenRequestException from bingads.service_client import ServiceClient from bingads.util import errorcode_of_exception +from bingads.v13.bulk import BulkServiceManager, DownloadParameters +from bingads.v13.reporting.exceptions import ReportingDownloadException from bingads.v13.reporting.reporting_service_manager import ReportingServiceManager from suds import WebFault, sudsobject +FILE_TYPE = "Csv" +TIMEOUT_IN_MILLISECONDS = 3_600_000 + class Client: api_version: int = 13 @@ -28,7 +37,7 @@ class Client: # https://docs.microsoft.com/en-us/advertising/guides/services-protocol?view=bingads-13#throttling # https://docs.microsoft.com/en-us/advertising/guides/operation-error-codes?view=bingads-13 retry_on_codes: Iterator[str] = ["117", "207", "4204", "109", "0"] - max_retries: int = 10 + max_retries: int = 5 # A backoff factor to apply between attempts after the second try # {retry_factor} * (2 ** ({number of total retries} - 1)) retry_factor: int = 15 @@ -36,18 +45,22 @@ class Client: environment: str = "production" # The time interval in milliseconds between two status polling attempts. report_poll_interval: int = 15000 + # Timeout of downloading report + _download_timeout = 300000 + _max_download_timeout = 600000 + + reports_start_date = None def __init__( self, tenant_id: str, - reports_start_date: str, + reports_start_date: str = None, developer_token: str = None, client_id: str = None, client_secret: str = None, refresh_token: str = None, **kwargs: Mapping[str, Any], ) -> None: - self.authorization_data: Mapping[str, AuthorizationData] = {} self.refresh_token = refresh_token self.developer_token = developer_token @@ -56,7 +69,8 @@ def __init__( self.authentication = self._get_auth_client(client_id, tenant_id, client_secret) self.oauth: OAuthTokens = self._get_access_token() - self.reports_start_date = pendulum.parse(reports_start_date).astimezone(tz=timezone.utc) + if reports_start_date: + self.reports_start_date = pendulum.parse(reports_start_date).astimezone(tz=timezone.utc) def _get_auth_client(self, client_id: str, tenant_id: str, client_secret: str = None) -> OAuthWebAuthCodeGrant: # https://github.com/BingAds/BingAds-Python-SDK/blob/e7b5a618e87a43d0a5e2c79d9aa4626e208797bd/bingads/authorization.py#L390 @@ -86,7 +100,16 @@ def _get_access_token(self) -> OAuthTokens: # clear caches to be able to use new access token self.get_service.cache_clear() self._get_auth_data.cache_clear() - return self.authentication.request_oauth_tokens_by_refresh_token(self.refresh_token) + try: + tokens = self.authentication.request_oauth_tokens_by_refresh_token(self.refresh_token) + except OAuthTokenRequestException as e: + raise AirbyteTracedException( + message=str(e), + internal_message="Failed to get OAuth access token by refresh token. " + "The user could not be authenticated as the grant is expired. The user must sign in again.", + failure_type=FailureType.config_error, + ) + return tokens def is_token_expiring(self) -> bool: """ @@ -96,7 +119,7 @@ def is_token_expiring(self) -> bool: token_updated_expires_in: int = self.oauth.access_token_expires_in_seconds - token_total_lifetime.seconds return False if token_updated_expires_in > self.refresh_token_safe_delta else True - def should_give_up(self, error: Union[WebFault, URLError]) -> bool: + def should_give_up(self, error: Union[WebFault, URLError, ReportingDownloadException]) -> bool: if isinstance(error, URLError): if ( isinstance(error.reason, socket.timeout) @@ -104,6 +127,12 @@ def should_give_up(self, error: Union[WebFault, URLError]) -> bool: or isinstance(error.reason, socket.gaierror) # temporary failure in name resolution ): return False + if isinstance(error, ReportingDownloadException): + self.logger.info("Reporting file download tracking status timeout.") + if self._download_timeout < self._max_download_timeout: + self._download_timeout = self._download_timeout + 10000 + self.logger.info(f"Increasing time of timeout to {self._download_timeout}") + return False error_code = str(errorcode_of_exception(error)) give_up = error_code not in self.retry_on_codes @@ -123,7 +152,7 @@ def log_retry_attempt(self, details: Mapping[str, Any]) -> None: def request(self, **kwargs: Mapping[str, Any]) -> Mapping[str, Any]: return backoff.on_exception( backoff.expo, - (WebFault, URLError), + (WebFault, URLError, ReportingDownloadException), max_tries=self.max_retries, factor=self.retry_factor, jitter=None, @@ -150,6 +179,8 @@ def _request( service = self._get_reporting_service(customer_id=customer_id, account_id=account_id) else: service = self.get_service(service_name=service_name, customer_id=customer_id, account_id=account_id) + if operation_name == "download_report": + params["download_parameters"].timeout_in_milliseconds = self._download_timeout return getattr(service, operation_name)(**params) @lru_cache(maxsize=4) @@ -212,3 +243,35 @@ def asdict(cls, suds_object: sudsobject.Object) -> Mapping[str, Any]: else: result[field] = val return result + + def _bulk_service_manager(self, customer_id: Optional[str] = None, account_id: Optional[str] = None): + return BulkServiceManager( + authorization_data=self._get_auth_data(customer_id, account_id), + poll_interval_in_milliseconds=5000, + environment=self.environment, + ) + + def get_bulk_entity( + self, + download_entities: List[str], + data_scope: List[str], + customer_id: Optional[str] = None, + account_id: Optional[str] = None, + start_date: Optional[str] = None, + ) -> str: + """ + Return path with zipped csv archive + """ + download_parameters = DownloadParameters( + # campaign_ids=None, + data_scope=data_scope, + download_entities=download_entities, + file_type=FILE_TYPE, + last_sync_time_in_utc=start_date, + result_file_directory=os.getcwd(), + result_file_name=str(uuid.uuid4()), + overwrite_result_file=True, # Set this value true if you want to overwrite the same file. + timeout_in_milliseconds=TIMEOUT_IN_MILLISECONDS, # You may optionally cancel the download after a specified time interval. + ) + bulk_service_manager = self._bulk_service_manager(customer_id=customer_id, account_id=account_id) + return bulk_service_manager.download_file(download_parameters) diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/report_streams.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/report_streams.py new file mode 100644 index 000000000000..fafb43779e17 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/report_streams.py @@ -0,0 +1,754 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import re +import xml.etree.ElementTree as ET +from abc import ABC, abstractmethod +from datetime import datetime +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Set, Tuple, Union +from urllib.parse import urlparse + +import _csv +import pendulum +from airbyte_cdk.sources.streams.core import package_name_from_class +from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +from airbyte_protocol.models import SyncMode +from bingads import ServiceClient +from bingads.v13.internal.reporting.row_report import _RowReport +from bingads.v13.internal.reporting.row_report_iterator import _RowReportRecord +from bingads.v13.reporting import ReportingDownloadParameters +from cached_property import cached_property +from source_bing_ads.base_streams import Accounts, BingAdsStream +from source_bing_ads.utils import transform_date_format_to_rfc_3339, transform_report_hourly_datetime_format_to_rfc_3339 +from suds import WebFault, sudsobject + + +class HourlyReportTransformerMixin: + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization) + + @staticmethod + @transformer.registerCustomTransform + def custom_transform_datetime_rfc3339(original_value, field_schema): + if original_value and "format" in field_schema and field_schema["format"] == "date-time": + transformed_value = transform_report_hourly_datetime_format_to_rfc_3339(original_value) + return transformed_value + return original_value + + +class BingAdsReportingServiceStream(BingAdsStream, ABC): + # The directory where the file with report will be downloaded. + file_directory: str = "/tmp" + # timeout for reporting download operations in milliseconds + timeout: int = 300000 + report_file_format: str = "Csv" + + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + primary_key: List[str] = ["TimePeriod", "Network", "DeviceType"] + + cursor_field = "TimePeriod" + service_name: str = "ReportingService" + operation_name: str = "download_report" + + def get_json_schema(self) -> Mapping[str, Any]: + return ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema(self.report_schema_name) + + @property + @abstractmethod + def report_name(self) -> str: + """ + Specifies bing ads report naming + """ + + @property + @abstractmethod + def report_aggregation(self) -> Optional[str]: + """ + Specifies bing ads report aggregation type + Supported types: Hourly, Daily, Weekly, Monthly + """ + + @property + @abstractmethod + def report_schema_name(self) -> str: + """ + Specifies file name with schema + """ + + @property + def default_time_periods(self): + # used when reports start date is not provided + return ["LastYear", "ThisYear"] if self.report_aggregation not in ("DayOfWeek", "HourOfDay") else ["ThisYear"] + + @property + def report_columns(self) -> Iterable[str]: + return list(self.get_json_schema().get("properties", {}).keys()) + + def parse_response(self, response: sudsobject.Object, **kwargs: Mapping[str, Any]) -> Iterable[Mapping]: + if response is not None: + try: + for row in response.report_records: + yield {column: self.get_column_value(row, column) for column in self.report_columns} + except _csv.Error as e: + self.logger.warning(f"CSV report file for stream `{self.name}` is broken or cannot be read correctly: {e}, skipping ...") + + def get_column_value(self, row: _RowReportRecord, column: str) -> Union[str, None, int, float]: + """ + Reads field value from row and transforms: + 1. empty values to logical None + 2. Percent values to numeric string e.g. "12.25%" -> "12.25" + """ + value = row.value(column) + if not value or value == "--": + return None + if "%" in value: + value = value.replace("%", "") + if value and column in self._get_schema_numeric_properties: + value = value.replace(",", "") + return value + + @cached_property + def _get_schema_numeric_properties(self) -> Set[str]: + return set(k for k, v in self.get_json_schema()["properties"].items() if set(v.get("type")) & {"integer", "number"}) + + def get_request_date(self, reporting_service: ServiceClient, date: datetime) -> sudsobject.Object: + """ + Creates XML Date object based on datetime. + https://docs.microsoft.com/en-us/advertising/reporting-service/date?view=bingads-13 + The [suds.client.Factory-class.html factory] namespace provides a factory that may be used + to create instances of objects and types defined in the WSDL. + """ + request_date = reporting_service.factory.create("Date") + request_date.Day = date.day + request_date.Month = date.month + request_date.Year = date.year + return request_date + + def request_params( + self, stream_state: Mapping[str, Any] = None, account_id: str = None, **kwargs: Mapping[str, Any] + ) -> Mapping[str, Any]: + stream_slice = kwargs["stream_slice"] + start_date = self.get_start_date(stream_state, account_id) + + reporting_service = self.client.get_service("ReportingService") + request_time_zone = reporting_service.factory.create("ReportTimeZone") + + report_time = reporting_service.factory.create("ReportTime") + report_time.ReportTimeZone = request_time_zone.GreenwichMeanTimeDublinEdinburghLisbonLondon + if start_date: + report_time.CustomDateRangeStart = self.get_request_date(reporting_service, start_date) + report_time.CustomDateRangeEnd = self.get_request_date(reporting_service, datetime.utcnow()) + report_time.PredefinedTime = None + else: + report_time.CustomDateRangeStart = None + report_time.CustomDateRangeEnd = None + report_time.PredefinedTime = stream_slice["time_period"] + + report_request = self.get_report_request(account_id, False, False, False, self.report_file_format, False, report_time) + + return { + "report_request": report_request, + "result_file_directory": self.file_directory, + "result_file_name": self.report_name, + "overwrite_result_file": True, + "timeout_in_milliseconds": self.timeout, + } + + def get_start_date(self, stream_state: Mapping[str, Any] = None, account_id: str = None): + if stream_state and account_id: + if stream_state.get(account_id, {}).get(self.cursor_field): + return pendulum.parse(stream_state[account_id][self.cursor_field]) + + return self.client.reports_start_date + + def get_updated_state( + self, + current_stream_state: MutableMapping[str, Any], + latest_record: Mapping[str, Any], + ) -> Mapping[str, Any]: + account_id = str(latest_record["AccountId"]) + current_stream_state[account_id] = current_stream_state.get(account_id, {}) + current_stream_state[account_id][self.cursor_field] = max( + self.get_report_record_timestamp(latest_record[self.cursor_field]), + current_stream_state.get(account_id, {}).get(self.cursor_field, ""), + ) + return current_stream_state + + def send_request(self, params: Mapping[str, Any], customer_id: str, account_id: str) -> _RowReport: + request_kwargs = { + "service_name": None, + "customer_id": customer_id, + "account_id": account_id, + "operation_name": self.operation_name, + "is_report_service": True, + "params": {"download_parameters": ReportingDownloadParameters(**params)}, + } + return self.client.request(**request_kwargs) + + def get_report_request( + self, + account_id: str, + exclude_column_headers: bool, + exclude_report_footer: bool, + exclude_report_header: bool, + report_file_format: str, + return_only_complete_data: bool, + time: sudsobject.Object, + ) -> sudsobject.Object: + reporting_service = self.client.get_service(self.service_name) + report_request = reporting_service.factory.create(f"{self.report_name}Request") + if self.report_aggregation: + report_request.Aggregation = self.report_aggregation + + report_request.ExcludeColumnHeaders = exclude_column_headers + report_request.ExcludeReportFooter = exclude_report_footer + report_request.ExcludeReportHeader = exclude_report_header + report_request.Format = report_file_format + report_request.FormatVersion = "2.0" + report_request.ReturnOnlyCompleteData = return_only_complete_data + report_request.Time = time + report_request.ReportName = self.report_name + # Defines the set of accounts and campaigns to include in the report. + scope = reporting_service.factory.create("AccountThroughCampaignReportScope") + scope.AccountIds = {"long": [account_id]} + scope.Campaigns = None + report_request.Scope = scope + + columns = reporting_service.factory.create(f"ArrayOf{self.report_name}Column") + getattr(columns, f"{self.report_name}Column").append(self.report_columns) + report_request.Columns = columns + return report_request + + def get_report_record_timestamp(self, datestring: str) -> str: + """ + Parse report date field based on aggregation type + """ + return ( + self.transformer._custom_normalizer(datestring, self.get_json_schema()["properties"][self.cursor_field]) + if self.transformer._custom_normalizer + else datestring + ) + + def stream_slices( + self, + **kwargs: Mapping[str, Any], + ) -> Iterable[Optional[Mapping[str, Any]]]: + accounts = Accounts(self.client, self.config) + for _slice in accounts.stream_slices(): + for account in accounts.read_records(SyncMode.full_refresh, _slice): + for period in self.default_time_periods: + yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"], "time_period": period} + + +class BingAdsReportingServicePerformanceStream(BingAdsReportingServiceStream, ABC): + def get_start_date(self, stream_state: Mapping[str, Any] = None, account_id: str = None): + start_date = super().get_start_date(stream_state, account_id) + + if self.config.get("lookback_window") and start_date: + # Datetime subtract won't work with days = 0 + # it'll output an AirbyteError + return start_date.subtract(days=self.config["lookback_window"]) + else: + return start_date + + +class BudgetSummaryReport(BingAdsReportingServiceStream): + report_name: str = "BudgetSummaryReport" + report_aggregation = None + cursor_field = "Date" + report_schema_name = "budget_summary_report" + primary_key = "Date" + + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization | TransformConfig.CustomSchemaNormalization) + + @staticmethod + @transformer.registerCustomTransform + def custom_transform_date_rfc3339(original_value, field_schema): + if original_value and "format" in field_schema and field_schema["format"] == "date": + transformed_value = transform_date_format_to_rfc_3339(original_value) + return transformed_value + return original_value + + +class CampaignPerformanceReport(BingAdsReportingServicePerformanceStream, ABC): + report_name: str = "CampaignPerformanceReport" + + report_schema_name = "campaign_performance_report" + primary_key = [ + "AccountId", + "CampaignId", + "TimePeriod", + "CurrencyCode", + "AdDistribution", + "DeviceType", + "Network", + "DeliveredMatchType", + "DeviceOS", + "TopVsOther", + "BidMatchType", + ] + + +class CampaignPerformanceReportHourly(HourlyReportTransformerMixin, CampaignPerformanceReport): + report_aggregation = "Hourly" + + report_schema_name = "campaign_performance_report_hourly" + + +class CampaignPerformanceReportDaily(CampaignPerformanceReport): + report_aggregation = "Daily" + + +class CampaignPerformanceReportWeekly(CampaignPerformanceReport): + report_aggregation = "Weekly" + + +class CampaignPerformanceReportMonthly(CampaignPerformanceReport): + report_aggregation = "Monthly" + + +class CampaignImpressionPerformanceReport(BingAdsReportingServicePerformanceStream, ABC): + """ + https://learn.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13 + Primary key cannot be set: due to included `Impression Share Performance Statistics` some fields should be removed, + see https://learn.microsoft.com/en-us/advertising/guides/reports?view=bingads-13#columnrestrictions for more info. + """ + + report_name: str = "CampaignPerformanceReport" + + report_schema_name = "campaign_impression_performance_report" + + primary_key = None + + +class CampaignImpressionPerformanceReportHourly(HourlyReportTransformerMixin, CampaignImpressionPerformanceReport): + report_aggregation = "Hourly" + + report_schema_name = "campaign_impression_performance_report_hourly" + + +class CampaignImpressionPerformanceReportDaily(CampaignImpressionPerformanceReport): + report_aggregation = "Daily" + + +class CampaignImpressionPerformanceReportWeekly(CampaignImpressionPerformanceReport): + report_aggregation = "Weekly" + + +class CampaignImpressionPerformanceReportMonthly(CampaignImpressionPerformanceReport): + report_aggregation = "Monthly" + + +class AdPerformanceReport(BingAdsReportingServicePerformanceStream, ABC): + report_name: str = "AdPerformanceReport" + + report_schema_name = "ad_performance_report" + primary_key = [ + "AccountId", + "CampaignId", + "AdGroupId", + "AdId", + "TimePeriod", + "CurrencyCode", + "AdDistribution", + "DeviceType", + "Language", + "Network", + "DeviceOS", + "TopVsOther", + "BidMatchType", + "DeliveredMatchType", + ] + + +class AdPerformanceReportHourly(HourlyReportTransformerMixin, AdPerformanceReport): + report_aggregation = "Hourly" + report_schema_name = "ad_performance_report_hourly" + + +class AdPerformanceReportDaily(AdPerformanceReport): + report_aggregation = "Daily" + + +class AdPerformanceReportWeekly(AdPerformanceReport): + report_aggregation = "Weekly" + + +class AdPerformanceReportMonthly(AdPerformanceReport): + report_aggregation = "Monthly" + + +class AdGroupPerformanceReport(BingAdsReportingServicePerformanceStream, ABC): + report_name: str = "AdGroupPerformanceReport" + report_schema_name = "ad_group_performance_report" + + primary_key = [ + "AccountId", + "CampaignId", + "AdGroupId", + "TimePeriod", + "CurrencyCode", + "AdDistribution", + "DeviceType", + "Network", + "DeliveredMatchType", + "DeviceOS", + "TopVsOther", + "BidMatchType", + "Language", + ] + + +class AdGroupPerformanceReportHourly(HourlyReportTransformerMixin, AdGroupPerformanceReport): + report_aggregation = "Hourly" + report_schema_name = "ad_group_performance_report_hourly" + + +class AdGroupPerformanceReportDaily(AdGroupPerformanceReport): + report_aggregation = "Daily" + + +class AdGroupPerformanceReportWeekly(AdGroupPerformanceReport): + report_aggregation = "Weekly" + + +class AdGroupPerformanceReportMonthly(AdGroupPerformanceReport): + report_aggregation = "Monthly" + + +class AdGroupImpressionPerformanceReport(BingAdsReportingServicePerformanceStream, ABC): + """ + https://learn.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13 + Primary key cannot be set: due to included `Impression Share Performance Statistics` some fields should be removed, + see https://learn.microsoft.com/en-us/advertising/guides/reports?view=bingads-13#columnrestrictions for more info. + """ + + report_name: str = "AdGroupPerformanceReport" + report_schema_name = "ad_group_impression_performance_report" + + +class AdGroupImpressionPerformanceReportHourly(HourlyReportTransformerMixin, AdGroupImpressionPerformanceReport): + report_aggregation = "Hourly" + report_schema_name = "ad_group_impression_performance_report_hourly" + + +class AdGroupImpressionPerformanceReportDaily(AdGroupImpressionPerformanceReport): + report_aggregation = "Daily" + + +class AdGroupImpressionPerformanceReportWeekly(AdGroupImpressionPerformanceReport): + report_aggregation = "Weekly" + + +class AdGroupImpressionPerformanceReportMonthly(AdGroupImpressionPerformanceReport): + report_aggregation = "Monthly" + + +class KeywordPerformanceReport(BingAdsReportingServicePerformanceStream, ABC): + report_name: str = "KeywordPerformanceReport" + report_schema_name = "keyword_performance_report" + primary_key = [ + "AccountId", + "CampaignId", + "AdGroupId", + "KeywordId", + "AdId", + "TimePeriod", + "CurrencyCode", + "DeliveredMatchType", + "AdDistribution", + "DeviceType", + "Language", + "Network", + "DeviceOS", + "TopVsOther", + "BidMatchType", + ] + + +class KeywordPerformanceReportHourly(HourlyReportTransformerMixin, KeywordPerformanceReport): + report_aggregation = "Hourly" + report_schema_name = "keyword_performance_report_hourly" + + +class KeywordPerformanceReportDaily(KeywordPerformanceReport): + report_aggregation = "Daily" + report_schema_name = "keyword_performance_report_daily" + + +class KeywordPerformanceReportWeekly(KeywordPerformanceReport): + report_aggregation = "Weekly" + + +class KeywordPerformanceReportMonthly(KeywordPerformanceReport): + report_aggregation = "Monthly" + + +class GeographicPerformanceReport(BingAdsReportingServicePerformanceStream, ABC): + report_name: str = "GeographicPerformanceReport" + report_schema_name = "geographic_performance_report" + + # Need to override the primary key here because the one inherited from the PerformanceReportsMixin + # is incorrect for the geographic performance reports + primary_key = None + + +class GeographicPerformanceReportHourly(HourlyReportTransformerMixin, GeographicPerformanceReport): + report_aggregation = "Hourly" + report_schema_name = "geographic_performance_report_hourly" + + +class GeographicPerformanceReportDaily(GeographicPerformanceReport): + report_aggregation = "Daily" + + +class GeographicPerformanceReportWeekly(GeographicPerformanceReport): + report_aggregation = "Weekly" + + +class GeographicPerformanceReportMonthly(GeographicPerformanceReport): + report_aggregation = "Monthly" + + +class AccountPerformanceReport(BingAdsReportingServicePerformanceStream, ABC): + report_name: str = "AccountPerformanceReport" + report_schema_name = "account_performance_report" + primary_key = [ + "AccountId", + "TimePeriod", + "CurrencyCode", + "AdDistribution", + "DeviceType", + "Network", + "DeliveredMatchType", + "DeviceOS", + "TopVsOther", + "BidMatchType", + ] + + +class AccountPerformanceReportHourly(HourlyReportTransformerMixin, AccountPerformanceReport): + report_aggregation = "Hourly" + report_schema_name = "account_performance_report_hourly" + + +class AccountPerformanceReportDaily(AccountPerformanceReport): + report_aggregation = "Daily" + + +class AccountPerformanceReportWeekly(AccountPerformanceReport): + report_aggregation = "Weekly" + + +class AccountPerformanceReportMonthly(AccountPerformanceReport): + report_aggregation = "Monthly" + + +class AccountImpressionPerformanceReport(BingAdsReportingServicePerformanceStream, ABC): + """ + Report source: https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13 + Primary key cannot be set: due to included `Impression Share Performance Statistics` some fields should be removed, + see https://learn.microsoft.com/en-us/advertising/guides/reports?view=bingads-13#columnrestrictions for more info. + """ + + report_name: str = "AccountPerformanceReport" + report_schema_name = "account_impression_performance_report" + primary_key = None + + +class AccountImpressionPerformanceReportHourly(HourlyReportTransformerMixin, AccountImpressionPerformanceReport): + report_aggregation = "Hourly" + report_schema_name = "account_impression_performance_report_hourly" + + +class AccountImpressionPerformanceReportDaily(AccountImpressionPerformanceReport): + report_aggregation = "Daily" + + +class AccountImpressionPerformanceReportWeekly(AccountImpressionPerformanceReport): + report_aggregation = "Weekly" + + +class AccountImpressionPerformanceReportMonthly(AccountImpressionPerformanceReport): + report_aggregation = "Monthly" + + +class AgeGenderAudienceReport(BingAdsReportingServicePerformanceStream, ABC): + report_name: str = "AgeGenderAudienceReport" + + report_schema_name = "age_gender_audience_report" + primary_key = ["AgeGroup", "Gender", "TimePeriod", "AccountId", "CampaignId", "Language", "AdDistribution"] + + +class AgeGenderAudienceReportHourly(HourlyReportTransformerMixin, AgeGenderAudienceReport): + report_aggregation = "Hourly" + report_schema_name = "age_gender_audience_report_hourly" + + +class AgeGenderAudienceReportDaily(AgeGenderAudienceReport): + report_aggregation = "Daily" + + +class AgeGenderAudienceReportWeekly(AgeGenderAudienceReport): + report_aggregation = "Weekly" + + +class AgeGenderAudienceReportMonthly(AgeGenderAudienceReport): + report_aggregation = "Monthly" + + +class SearchQueryPerformanceReport(BingAdsReportingServicePerformanceStream, ABC): + report_name: str = "SearchQueryPerformanceReport" + report_schema_name = "search_query_performance_report" + + primary_key = [ + "SearchQuery", + "Keyword", + "TimePeriod", + "AccountId", + "CampaignId", + "Language", + "DeliveredMatchType", + "DeviceType", + "DeviceOS", + "TopVsOther", + ] + + +class SearchQueryPerformanceReportHourly(HourlyReportTransformerMixin, SearchQueryPerformanceReport): + report_aggregation = "Hourly" + report_schema_name = "search_query_performance_report_hourly" + + +class SearchQueryPerformanceReportDaily(SearchQueryPerformanceReport): + report_aggregation = "Daily" + + +class SearchQueryPerformanceReportWeekly(SearchQueryPerformanceReport): + report_aggregation = "Weekly" + + +class SearchQueryPerformanceReportMonthly(SearchQueryPerformanceReport): + report_aggregation = "Monthly" + + +class UserLocationPerformanceReport(BingAdsReportingServicePerformanceStream, ABC): + report_name: str = "UserLocationPerformanceReport" + report_schema_name = "user_location_performance_report" + primary_key = [ + "AccountId", + "AdGroupId", + "CampaignId", + "DeliveredMatchType", + "DeviceOS", + "DeviceType", + "Language", + "LocationId", + "QueryIntentLocationId", + "TimePeriod", + "TopVsOther", + ] + + +class UserLocationPerformanceReportHourly(HourlyReportTransformerMixin, UserLocationPerformanceReport): + report_aggregation = "Hourly" + report_schema_name = "user_location_performance_report_hourly" + + +class UserLocationPerformanceReportDaily(UserLocationPerformanceReport): + report_aggregation = "Daily" + + +class UserLocationPerformanceReportWeekly(UserLocationPerformanceReport): + report_aggregation = "Weekly" + + +class UserLocationPerformanceReportMonthly(UserLocationPerformanceReport): + report_aggregation = "Monthly" + + +class CustomReport(BingAdsReportingServicePerformanceStream, ABC): + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + custom_report_columns = [] + report_schema_name = None + primary_key = None + + @property + def cursor_field(self) -> Union[str, List[str]]: + # Summary aggregation doesn't include TimePeriod field + if self.report_aggregation not in ("Summary", "DayOfWeek", "HourOfDay"): + return "TimePeriod" + + @property + def report_columns(self): + # adding common and default columns + if "AccountId" not in self.custom_report_columns: + self.custom_report_columns.append("AccountId") + if self.cursor_field and self.cursor_field not in self.custom_report_columns: + self.custom_report_columns.append(self.cursor_field) + return list(frozenset(self.custom_report_columns)) + + def get_json_schema(self) -> Mapping[str, Any]: + columns_schema = {col: {"type": ["null", "string"]} for col in self.report_columns} + schema: Mapping[str, Any] = { + "$schema": "https://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "additionalProperties": True, + "properties": columns_schema, + } + return schema + + def validate_report_configuration(self) -> Tuple[bool, str]: + # gets /bingads/v13/proxies/production/reporting_service.xml + reporting_service_file = self.client.get_service(self.service_name)._get_service_info_dict(self.client.api_version)[ + ("reporting", self.client.environment) + ] + tree = ET.parse(urlparse(reporting_service_file).path) + request_object = tree.find(f".//{{*}}complexType[@name='{self.report_name}Request']") + + report_object_columns = self._get_object_columns(request_object, tree) + is_custom_cols_in_report_object_cols = all(x in report_object_columns for x in self.custom_report_columns) + + if not is_custom_cols_in_report_object_cols: + return False, ( + f"Reporting Columns are invalid. Columns that you provided don't belong to Reporting Data Object Columns:" + f" {self.custom_report_columns}. Please ensure it is correct in Bing Ads Docs." + ) + + return True, "" + + def _clear_namespace(self, type: str) -> str: + return re.sub(r"^[a-z]+:", "", type) + + def _get_object_columns(self, request_el: ET.Element, tree: ET.ElementTree) -> List[str]: + column_el = request_el.find(".//{*}element[@name='Columns']") + array_of_columns_name = self._clear_namespace(column_el.get("type")) + + array_of_columns_elements = tree.find(f".//{{*}}complexType[@name='{array_of_columns_name}']") + inner_array_of_columns_elements = array_of_columns_elements.find(".//{*}element") + column_el_name = self._clear_namespace(inner_array_of_columns_elements.get("type")) + + column_el = tree.find(f".//{{*}}simpleType[@name='{column_el_name}']") + column_enum_items = column_el.findall(".//{*}enumeration") + column_enum_items_values = [el.get("value") for el in column_enum_items] + return column_enum_items_values + + def get_report_record_timestamp(self, datestring: str) -> str: + """ + Parse report date field based on aggregation type + """ + if not self.report_aggregation or self.report_aggregation == "Summary": + datestring = transform_date_format_to_rfc_3339(datestring) + elif self.report_aggregation == "Hourly": + datestring = transform_report_hourly_datetime_format_to_rfc_3339(datestring) + return datestring + + def send_request(self, params: Mapping[str, Any], customer_id: str, account_id: str) -> _RowReport: + try: + return super().send_request(params, customer_id, account_id) + except WebFault as e: + self.logger.error( + f"Could not sync custom report {self.name}: Please validate your column and aggregation configuration. " + f"Error form server: [{e.fault.faultstring}]" + ) diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py deleted file mode 100644 index b3cd1c5bf47d..000000000000 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/reports.py +++ /dev/null @@ -1,354 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from abc import ABC, abstractmethod -from datetime import datetime -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union - -import pendulum -import source_bing_ads.source -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.core import package_name_from_class -from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader -from bingads.service_client import ServiceClient -from bingads.v13.internal.reporting.row_report import _RowReport -from bingads.v13.internal.reporting.row_report_iterator import _RowReportRecord -from bingads.v13.reporting import ReportingDownloadParameters -from suds import sudsobject - -AVERAGE_FIELD_TYPES = { - "AverageCpc": "number", - "AveragePosition": "number", - "AverageCpm": "number", -} -AVERAGE_FIELDS = list(AVERAGE_FIELD_TYPES.keys()) - -CONVERSION_FIELD_TYPES = { - "Conversions": "number", - "ConversionRate": "number", - "ConversionsQualified": "number", -} -CONVERSION_FIELDS = list(CONVERSION_FIELD_TYPES.keys()) - -ALL_CONVERSION_FIELD_TYPES = { - "AllConversions": "integer", - "AllConversionRate": "number", -} -ALL_CONVERSION_FIELDS = list(ALL_CONVERSION_FIELD_TYPES.keys()) - -LOW_QUALITY_FIELD_TYPES = { - "LowQualityClicks": "integer", - "LowQualityClicksPercent": "number", - "LowQualityImpressions": "integer", - "LowQualitySophisticatedClicks": "integer", - "LowQualityConversions": "integer", - "LowQualityConversionRate": "number", -} -LOW_QUALITY_FIELDS = list(LOW_QUALITY_FIELD_TYPES.keys()) - -REVENUE_FIELD_TYPES = { - "Revenue": "number", - "RevenuePerConversion": "number", - "RevenuePerAssist": "number", -} -REVENUE_FIELDS = list(REVENUE_FIELD_TYPES.keys()) - -ALL_REVENUE_FIELD_TYPES = { - "AllRevenue": "number", - "AllRevenuePerConversion": "number", -} -ALL_REVENUE_FIELDS = list(ALL_REVENUE_FIELD_TYPES.keys()) - -IMPRESSION_FIELD_TYPES = { - "ImpressionSharePercent": "number", - "ImpressionLostToBudgetPercent": "number", - "ImpressionLostToRankAggPercent": "number", -} -IMPRESSION_FIELDS = list(IMPRESSION_FIELD_TYPES.keys()) - - -HISTORICAL_FIELD_TYPES = { - "HistoricalQualityScore": "number", - "HistoricalExpectedCtr": "number", - "HistoricalAdRelevance": "number", - "HistoricalLandingPageExperience": "number", -} -HISTORICAL_FIELDS = list(HISTORICAL_FIELD_TYPES.keys()) - -BUDGET_FIELD_TYPES = { - "BudgetName": "string", - "BudgetStatus": "string", - "BudgetAssociationStatus": "string", -} -BUDGET_FIELDS = list(BUDGET_FIELD_TYPES.keys()) - -REPORT_FIELD_TYPES = { - "AccountId": "integer", - "AdId": "integer", - "AdGroupCriterionId": "integer", - "AdGroupId": "integer", - "AdRelevance": "number", - "Assists": "integer", - "AllCostPerConversion": "number", - "AllReturnOnAdSpend": "number", - "BusinessCategoryId": "integer", - "BusinessListingId": "integer", - "CampaignId": "integer", - "ClickCalls": "integer", - "Clicks": "integer", - "CostPerAssist": "number", - "CostPerConversion": "number", - "Ctr": "number", - "CurrentMaxCpc": "number", - "EstimatedClickPercent": "number", - "EstimatedClicks": "integer", - "EstimatedConversionRate": "number", - "EstimatedConversions": "integer", - "EstimatedCtr": "number", - "EstimatedImpressionPercent": "number", - "EstimatedImpressions": "integer", - "ExactMatchImpressionSharePercent": "number", - "Impressions": "integer", - "ImpressionSharePercent": "number", - "KeywordId": "integer", - "LandingPageExperience": "number", - "PhoneCalls": "integer", - "PhoneImpressions": "integer", - "Ptr": "number", - "QualityImpact": "number", - "QualityScore": "number", - "ReturnOnAdSpend": "number", - "SidebarBid": "number", - "Spend": "number", - "MonthlyBudget": "number", - "DailySpend": "number", - "MonthToDateSpend": "number", - "AbsoluteTopImpressionRatePercent": "number", - "ViewThroughConversions": "integer", - "ViewThroughConversionsQualified": "number", - "MainlineBid": "number", - "Mainline1Bid": "number", - "FirstPageBid": "number", - **AVERAGE_FIELD_TYPES, - **CONVERSION_FIELD_TYPES, - **ALL_CONVERSION_FIELD_TYPES, - **LOW_QUALITY_FIELD_TYPES, - **REVENUE_FIELD_TYPES, - **ALL_REVENUE_FIELD_TYPES, - **IMPRESSION_FIELD_TYPES, - **HISTORICAL_FIELD_TYPES, - **BUDGET_FIELD_TYPES, -} - - -class ReportsMixin(ABC): - # The directory where the file with report will be downloaded. - file_directory: str = "/tmp" - # timeout for reporting download operations in milliseconds - timeout: int = 300000 - report_file_format: str = "Csv" - - primary_key: List[str] = ["TimePeriod", "Network", "DeviceType"] - - @property - @abstractmethod - def report_name(self) -> str: - """ - Specifies bing ads report naming - """ - pass - - @property - @abstractmethod - def report_columns(self) -> Iterable[str]: - """ - Specifies bing ads report naming - """ - pass - - @property - @abstractmethod - def report_aggregation(self) -> Optional[str]: - """ - Specifies bing ads report aggregation type - Supported types: Hourly, Daily, Weekly, Monthly - """ - pass - - @property - @abstractmethod - def report_schema_name(self) -> str: - """ - Specifies file name with schema - """ - pass - - def get_json_schema(self) -> Mapping[str, Any]: - return ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema(self.report_schema_name) - - def get_request_date(self, reporting_service: ServiceClient, date: datetime) -> sudsobject.Object: - """ - Creates XML Date object based on datetime. - https://docs.microsoft.com/en-us/advertising/reporting-service/date?view=bingads-13 - The [suds.client.Factory-class.html factory] namespace provides a factory that may be used - to create instances of objects and types defined in the WSDL. - """ - request_date = reporting_service.factory.create("Date") - request_date.Day = date.day - request_date.Month = date.month - request_date.Year = date.year - return request_date - - def request_params( - self, stream_state: Mapping[str, Any] = None, account_id: str = None, **kwargs: Mapping[str, Any] - ) -> Mapping[str, Any]: - start_date = self.get_start_date(stream_state, account_id) - - reporting_service = self.client.get_service("ReportingService") - request_time_zone = reporting_service.factory.create("ReportTimeZone") - - report_time = reporting_service.factory.create("ReportTime") - report_time.CustomDateRangeStart = self.get_request_date(reporting_service, start_date) - report_time.CustomDateRangeEnd = self.get_request_date(reporting_service, datetime.utcnow()) - report_time.PredefinedTime = None - report_time.ReportTimeZone = request_time_zone.GreenwichMeanTimeDublinEdinburghLisbonLondon - - report_request = self.get_report_request(account_id, False, False, False, self.report_file_format, False, report_time) - - return { - "report_request": report_request, - "result_file_directory": self.file_directory, - "result_file_name": self.report_name, - "overwrite_result_file": True, - "timeout_in_milliseconds": self.timeout, - } - - def get_start_date(self, stream_state: Mapping[str, Any] = None, account_id: str = None): - if stream_state and account_id: - if stream_state.get(account_id, {}).get(self.cursor_field): - return pendulum.from_timestamp(stream_state[account_id][self.cursor_field]) - - return self.client.reports_start_date - - def get_updated_state( - self, - current_stream_state: MutableMapping[str, Any], - latest_record: Mapping[str, Any], - ) -> Mapping[str, Any]: - account_id = str(latest_record["AccountId"]) - current_stream_state[account_id] = current_stream_state.get(account_id, {}) - current_stream_state[account_id][self.cursor_field] = max( - self.get_report_record_timestamp(latest_record[self.cursor_field]), - current_stream_state.get(account_id, {}).get(self.cursor_field, 1), - ) - return current_stream_state - - def send_request(self, params: Mapping[str, Any], customer_id: str, account_id: str) -> _RowReport: - request_kwargs = { - "service_name": None, - "customer_id": customer_id, - "account_id": account_id, - "operation_name": self.operation_name, - "is_report_service": True, - "params": {"download_parameters": ReportingDownloadParameters(**params)}, - } - return self.client.request(**request_kwargs) - - def get_report_request( - self, - account_id: str, - exclude_column_headers: bool, - exclude_report_footer: bool, - exclude_report_header: bool, - report_file_format: str, - return_only_complete_data: bool, - time: sudsobject.Object, - ) -> sudsobject.Object: - reporting_service = self.client.get_service(self.service_name) - report_request = reporting_service.factory.create(f"{self.report_name}Request") - if self.report_aggregation: - report_request.Aggregation = self.report_aggregation - - report_request.ExcludeColumnHeaders = exclude_column_headers - report_request.ExcludeReportFooter = exclude_report_footer - report_request.ExcludeReportHeader = exclude_report_header - report_request.Format = report_file_format - report_request.FormatVersion = "2.0" - report_request.ReturnOnlyCompleteData = return_only_complete_data - report_request.Time = time - report_request.ReportName = self.report_name - # Defines the set of accounts and campaigns to include in the report. - scope = reporting_service.factory.create("AccountThroughCampaignReportScope") - scope.AccountIds = {"long": [account_id]} - scope.Campaigns = None - report_request.Scope = scope - - columns = reporting_service.factory.create(f"ArrayOf{self.report_name}Column") - getattr(columns, f"{self.report_name}Column").append(self.report_columns) - report_request.Columns = columns - return report_request - - def parse_response(self, response: sudsobject.Object, **kwargs: Mapping[str, Any]) -> Iterable[Mapping]: - if response is not None: - for row in response.report_records: - yield {column: self.get_column_value(row, column) for column in self.report_columns} - - yield from [] - - def get_column_value(self, row: _RowReportRecord, column: str) -> Union[str, None, int, float]: - """ - Reads field value from row and transforms string type field to numeric if possible - """ - value = row.value(column) - if value == "": - return None - - if value is not None and column in REPORT_FIELD_TYPES: - if REPORT_FIELD_TYPES[column] == "integer": - value = 0 if value == "--" else int(value.replace(",", "")) - elif REPORT_FIELD_TYPES[column] == "number": - if value == "--": - value = 0.0 - else: - if "%" in value: - value = float(value.replace("%", "").replace(",", "")) / 100 - else: - value = float(value.replace(",", "")) - - return value - - def get_report_record_timestamp(self, datestring: str) -> int: - """ - Parse report date field based on aggregation type - """ - if not self.report_aggregation: - date = pendulum.from_format(datestring, "M/D/YYYY") - else: - if self.report_aggregation == "Hourly": - date = pendulum.from_format(datestring, "YYYY-MM-DD|H") - else: - date = pendulum.parse(datestring) - - return date.int_timestamp - - def stream_slices( - self, - **kwargs: Mapping[str, Any], - ) -> Iterable[Optional[Mapping[str, Any]]]: - for account in source_bing_ads.source.Accounts(self.client, self.config).read_records(SyncMode.full_refresh): - yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} - - yield from [] - - -class PerformanceReportsMixin(ReportsMixin): - def get_start_date(self, stream_state: Mapping[str, Any] = None, account_id: str = None): - start_date = super().get_start_date(stream_state, account_id) - - if self.config.get("lookback_window"): - # Datetime subtract won't work with days = 0 - # it'll output an AirbuteError - return start_date.subtract(days=self.config["lookback_window"]) - else: - return start_date diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_impression_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_impression_performance_report.json new file mode 100644 index 000000000000..f2ce126eff0d --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_impression_performance_report.json @@ -0,0 +1,250 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountName": { + "type": ["null", "string"] + }, + "AccountNumber": { + "type": ["null", "string"] + }, + "AccountId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date" + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "Conversions": { + "type": ["null", "integer"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "LowQualityClicks": { + "type": ["null", "integer"] + }, + "LowQualityClicksPercent": { + "type": ["null", "number"] + }, + "LowQualityImpressions": { + "type": ["null", "integer"] + }, + "LowQualityImpressionsPercent": { + "type": ["null", "number"] + }, + "LowQualityConversions": { + "type": ["null", "integer"] + }, + "LowQualityConversionRate": { + "type": ["null", "number"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "ImpressionSharePercent": { + "type": ["null", "number"] + }, + "ImpressionLostToBudgetPercent": { + "type": ["null", "number"] + }, + "ImpressionLostToRankAggPercent": { + "type": ["null", "number"] + }, + "PhoneImpressions": { + "type": ["null", "integer"] + }, + "PhoneCalls": { + "type": ["null", "integer"] + }, + "Ptr": { + "type": ["null", "number"] + }, + "Network": { + "type": ["null", "string"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + }, + "AccountStatus": { + "type": ["null", "string"] + }, + "LowQualityGeneralClicks": { + "type": ["null", "integer"] + }, + "LowQualitySophisticatedClicks": { + "type": ["null", "integer"] + }, + "ExactMatchImpressionSharePercent": { + "type": ["null", "number"] + }, + "ClickSharePercent": { + "type": ["null", "number"] + }, + "AbsoluteTopImpressionSharePercent": { + "type": ["null", "number"] + }, + "TopImpressionShareLostToRankPercent": { + "type": ["null", "number"] + }, + "TopImpressionShareLostToBudgetPercent": { + "type": ["null", "number"] + }, + "AbsoluteTopImpressionShareLostToRankPercent": { + "type": ["null", "number"] + }, + "AbsoluteTopImpressionShareLostToBudgetPercent": { + "type": ["null", "number"] + }, + "TopImpressionSharePercent": { + "type": ["null", "number"] + }, + "AbsoluteTopImpressionRatePercent": { + "type": ["null", "number"] + }, + "TopImpressionRatePercent": { + "type": ["null", "number"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "AudienceImpressionSharePercent": { + "type": ["null", "number"] + }, + "AudienceImpressionLostToRankPercent": { + "type": ["null", "number"] + }, + "AudienceImpressionLostToBudgetPercent": { + "type": ["null", "number"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "LowQualityConversionsQualified": { + "type": ["null", "number"] + }, + "AllConversionsQualified": { + "type": ["null", "number"] + }, + "ViewThroughConversionsQualified": { + "type": ["null", "number"] + }, + "ViewThroughRevenue": { + "type": ["null", "number"] + }, + "VideoViews": { + "type": ["null", "integer"] + }, + "ViewThroughRate": { + "type": ["null", "number"] + }, + "AverageCPV": { + "type": ["null", "number"] + }, + "VideoViewsAt25Percent": { + "type": ["null", "integer"] + }, + "VideoViewsAt50Percent": { + "type": ["null", "integer"] + }, + "VideoViewsAt75Percent": { + "type": ["null", "integer"] + }, + "CompletedVideoViews": { + "type": ["null", "integer"] + }, + "VideoCompletionRate": { + "type": ["null", "number"] + }, + "TotalWatchTimeInMS": { + "type": ["null", "integer"] + }, + "AverageWatchTimePerVideoView": { + "type": ["null", "number"] + }, + "AverageWatchTimePerImpression": { + "type": ["null", "number"] + }, + "Sales": { + "type": ["null", "integer"] + }, + "CostPerSale": { + "type": ["null", "number"] + }, + "RevenuePerSale": { + "type": ["null", "number"] + }, + "Installs": { + "type": ["null", "integer"] + }, + "CostPerInstall": { + "type": ["null", "number"] + }, + "RevenuePerInstall": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_impression_performance_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_impression_performance_report_hourly.json new file mode 100644 index 000000000000..7c0cff82f779 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_impression_performance_report_hourly.json @@ -0,0 +1,206 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountName": { + "type": ["null", "string"] + }, + "AccountNumber": { + "type": ["null", "string"] + }, + "AccountId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "Conversions": { + "type": ["null", "integer"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "LowQualityClicks": { + "type": ["null", "integer"] + }, + "LowQualityClicksPercent": { + "type": ["null", "number"] + }, + "LowQualityImpressions": { + "type": ["null", "integer"] + }, + "LowQualityImpressionsPercent": { + "type": ["null", "number"] + }, + "LowQualityConversions": { + "type": ["null", "integer"] + }, + "LowQualityConversionRate": { + "type": ["null", "number"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "PhoneImpressions": { + "type": ["null", "integer"] + }, + "PhoneCalls": { + "type": ["null", "integer"] + }, + "Ptr": { + "type": ["null", "number"] + }, + "Network": { + "type": ["null", "string"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + }, + "AccountStatus": { + "type": ["null", "string"] + }, + "LowQualityGeneralClicks": { + "type": ["null", "integer"] + }, + "LowQualitySophisticatedClicks": { + "type": ["null", "integer"] + }, + "TopImpressionRatePercent": { + "type": ["null", "number"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "LowQualityConversionsQualified": { + "type": ["null", "number"] + }, + "AllConversionsQualified": { + "type": ["null", "number"] + }, + "ViewThroughConversionsQualified": { + "type": ["null", "number"] + }, + "ViewThroughRevenue": { + "type": ["null", "number"] + }, + "VideoViews": { + "type": ["null", "integer"] + }, + "ViewThroughRate": { + "type": ["null", "number"] + }, + "AverageCPV": { + "type": ["null", "number"] + }, + "VideoViewsAt25Percent": { + "type": ["null", "integer"] + }, + "VideoViewsAt50Percent": { + "type": ["null", "integer"] + }, + "VideoViewsAt75Percent": { + "type": ["null", "integer"] + }, + "CompletedVideoViews": { + "type": ["null", "integer"] + }, + "VideoCompletionRate": { + "type": ["null", "number"] + }, + "TotalWatchTimeInMS": { + "type": ["null", "integer"] + }, + "AverageWatchTimePerVideoView": { + "type": ["null", "number"] + }, + "AverageWatchTimePerImpression": { + "type": ["null", "number"] + }, + "Sales": { + "type": ["null", "integer"] + }, + "CostPerSale": { + "type": ["null", "number"] + }, + "RevenuePerSale": { + "type": ["null", "number"] + }, + "Installs": { + "type": ["null", "integer"] + }, + "CostPerInstall": { + "type": ["null", "number"] + }, + "RevenuePerInstall": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_performance_report.json index d72fd09dcb25..ab5c273877ab 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_performance_report.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_performance_report.json @@ -1,11 +1,13 @@ { - "type": ["null", "object"], + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", "properties": { "AccountId": { "type": ["null", "integer"] }, "TimePeriod": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" }, "CurrencyCode": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_performance_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_performance_report_hourly.json new file mode 100644 index 000000000000..4013dd037402 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/account_performance_report_hourly.json @@ -0,0 +1,122 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "Network": { + "type": ["null", "string"] + }, + "DeliveredMatchType": { + "type": ["null", "string"] + }, + "DeviceOS": { + "type": ["null", "string"] + }, + "TopVsOther": { + "type": ["null", "string"] + }, + "BidMatchType": { + "type": ["null", "string"] + }, + "AccountName": { + "type": ["null", "string"] + }, + "AccountNumber": { + "type": ["null", "string"] + }, + "PhoneImpressions": { + "type": ["null", "integer"] + }, + "PhoneCalls": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "Ptr": { + "type": ["null", "number"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "Conversions": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "LowQualityClicks": { + "type": ["null", "integer"] + }, + "LowQualityClicksPercent": { + "type": ["null", "number"] + }, + "LowQualityImpressions": { + "type": ["null", "integer"] + }, + "LowQualitySophisticatedClicks": { + "type": ["null", "integer"] + }, + "LowQualityConversions": { + "type": ["null", "integer"] + }, + "LowQualityConversionRate": { + "type": ["null", "number"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/accounts.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/accounts.json index 858ecfe2d9bc..63bf3d699add 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/accounts.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/accounts.json @@ -27,7 +27,15 @@ "type": ["null", "string"] }, "LinkedAgencies": { - "type": ["null", "string"] + "type": ["null", "object"], + "properties": { + "Id": { + "type": ["null", "integer"] + }, + "Name": { + "type": ["null", "string"] + } + } }, "TaxInformation": { "type": ["null", "string"] @@ -89,7 +97,9 @@ "type": ["null", "number"] }, "LastModifiedTime": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, "Name": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_impression_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_impression_performance_report.json new file mode 100644 index 000000000000..beee977875e8 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_impression_performance_report.json @@ -0,0 +1,292 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountName": { + "type": ["null", "string"] + }, + "AccountNumber": { + "type": ["null", "string"] + }, + "AccountId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date" + }, + "Status": { + "type": ["null", "string"] + }, + "CampaignName": { + "type": ["null", "string"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "AdGroupName": { + "type": ["null", "string"] + }, + "AdGroupId": { + "type": ["null", "integer"] + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "Conversions": { + "type": ["null", "integer"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "Language": { + "type": ["null", "string"] + }, + "ImpressionSharePercent": { + "type": ["null", "number"] + }, + "ImpressionLostToBudgetPercent": { + "type": ["null", "number"] + }, + "ImpressionLostToRankAggPercent": { + "type": ["null", "number"] + }, + "QualityScore": { + "type": ["null", "integer"] + }, + "ExpectedCtr": { + "type": ["null", "number"] + }, + "AdRelevance": { + "type": ["null", "integer"] + }, + "LandingPageExperience": { + "type": ["null", "integer"] + }, + "HistoricalQualityScore": { + "type": ["null", "integer"] + }, + "HistoricalExpectedCtr": { + "type": ["null", "integer"] + }, + "HistoricalAdRelevance": { + "type": ["null", "integer"] + }, + "HistoricalLandingPageExperience": { + "type": ["null", "integer"] + }, + "PhoneImpressions": { + "type": ["null", "integer"] + }, + "PhoneCalls": { + "type": ["null", "integer"] + }, + "Ptr": { + "type": ["null", "number"] + }, + "Network": { + "type": ["null", "string"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + }, + "TrackingTemplate": { + "type": ["null", "string"] + }, + "CustomParameters": { + "type": ["null", "string"] + }, + "AccountStatus": { + "type": ["null", "string"] + }, + "CampaignStatus": { + "type": ["null", "string"] + }, + "AdGroupLabels": { + "type": ["null", "string"] + }, + "ExactMatchImpressionSharePercent": { + "type": ["null", "number"] + }, + "ClickSharePercent": { + "type": ["null", "number"] + }, + "AbsoluteTopImpressionSharePercent": { + "type": ["null", "number"] + }, + "FinalUrlSuffix": { + "type": ["null", "string"] + }, + "CampaignType": { + "type": ["null", "string"] + }, + "TopImpressionShareLostToRankPercent": { + "type": ["null", "number"] + }, + "TopImpressionShareLostToBudgetPercent": { + "type": ["null", "number"] + }, + "AbsoluteTopImpressionShareLostToRankPercent": { + "type": ["null", "number"] + }, + "AbsoluteTopImpressionShareLostToBudgetPercent": { + "type": ["null", "number"] + }, + "TopImpressionSharePercent": { + "type": ["null", "number"] + }, + "AbsoluteTopImpressionRatePercent": { + "type": ["null", "number"] + }, + "TopImpressionRatePercent": { + "type": ["null", "number"] + }, + "BaseCampaignId": { + "type": ["null", "integer"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "AudienceImpressionSharePercent": { + "type": ["null", "number"] + }, + "AudienceImpressionLostToRankPercent": { + "type": ["null", "number"] + }, + "AudienceImpressionLostToBudgetPercent": { + "type": ["null", "number"] + }, + "RelativeCtr": { + "type": ["null", "number"] + }, + "AdGroupType": { + "type": ["null", "string"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "AllConversionsQualified": { + "type": ["null", "number"] + }, + "ViewThroughConversionsQualified": { + "type": ["null", "number"] + }, + "ViewThroughRevenue": { + "type": ["null", "number"] + }, + "VideoViews": { + "type": ["null", "integer"] + }, + "ViewThroughRate": { + "type": ["null", "number"] + }, + "AverageCPV": { + "type": ["null", "number"] + }, + "VideoViewsAt25Percent": { + "type": ["null", "integer"] + }, + "VideoViewsAt50Percent": { + "type": ["null", "integer"] + }, + "VideoViewsAt75Percent": { + "type": ["null", "integer"] + }, + "CompletedVideoViews": { + "type": ["null", "integer"] + }, + "VideoCompletionRate": { + "type": ["null", "number"] + }, + "TotalWatchTimeInMS": { + "type": ["null", "integer"] + }, + "AverageWatchTimePerVideoView": { + "type": ["null", "number"] + }, + "AverageWatchTimePerImpression": { + "type": ["null", "number"] + }, + "Sales": { + "type": ["null", "integer"] + }, + "CostPerSale": { + "type": ["null", "number"] + }, + "RevenuePerSale": { + "type": ["null", "number"] + }, + "Installs": { + "type": ["null", "integer"] + }, + "CostPerInstall": { + "type": ["null", "number"] + }, + "RevenuePerInstall": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_impression_performance_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_impression_performance_report_hourly.json new file mode 100644 index 000000000000..9ebbe2b60f2f --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_impression_performance_report_hourly.json @@ -0,0 +1,239 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountName": { + "type": ["null", "string"] + }, + "AccountNumber": { + "type": ["null", "string"] + }, + "AccountId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "Status": { + "type": ["null", "string"] + }, + "CampaignName": { + "type": ["null", "string"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "AdGroupName": { + "type": ["null", "string"] + }, + "AdGroupId": { + "type": ["null", "integer"] + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "Conversions": { + "type": ["null", "integer"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "Language": { + "type": ["null", "string"] + }, + "QualityScore": { + "type": ["null", "integer"] + }, + "ExpectedCtr": { + "type": ["null", "number"] + }, + "AdRelevance": { + "type": ["null", "integer"] + }, + "LandingPageExperience": { + "type": ["null", "integer"] + }, + "PhoneImpressions": { + "type": ["null", "integer"] + }, + "PhoneCalls": { + "type": ["null", "integer"] + }, + "Ptr": { + "type": ["null", "number"] + }, + "Network": { + "type": ["null", "string"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + }, + "TrackingTemplate": { + "type": ["null", "string"] + }, + "CustomParameters": { + "type": ["null", "string"] + }, + "AccountStatus": { + "type": ["null", "string"] + }, + "CampaignStatus": { + "type": ["null", "string"] + }, + "AdGroupLabels": { + "type": ["null", "string"] + }, + "FinalUrlSuffix": { + "type": ["null", "string"] + }, + "CampaignType": { + "type": ["null", "string"] + }, + "TopImpressionSharePercent": { + "type": ["null", "number"] + }, + "AbsoluteTopImpressionRatePercent": { + "type": ["null", "number"] + }, + "TopImpressionRatePercent": { + "type": ["null", "number"] + }, + "BaseCampaignId": { + "type": ["null", "integer"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "AdGroupType": { + "type": ["null", "string"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "AllConversionsQualified": { + "type": ["null", "number"] + }, + "ViewThroughConversionsQualified": { + "type": ["null", "number"] + }, + "ViewThroughRevenue": { + "type": ["null", "number"] + }, + "VideoViews": { + "type": ["null", "integer"] + }, + "ViewThroughRate": { + "type": ["null", "number"] + }, + "AverageCPV": { + "type": ["null", "number"] + }, + "VideoViewsAt25Percent": { + "type": ["null", "integer"] + }, + "VideoViewsAt50Percent": { + "type": ["null", "integer"] + }, + "VideoViewsAt75Percent": { + "type": ["null", "integer"] + }, + "CompletedVideoViews": { + "type": ["null", "integer"] + }, + "VideoCompletionRate": { + "type": ["null", "number"] + }, + "TotalWatchTimeInMS": { + "type": ["null", "integer"] + }, + "AverageWatchTimePerVideoView": { + "type": ["null", "number"] + }, + "AverageWatchTimePerImpression": { + "type": ["null", "number"] + }, + "Sales": { + "type": ["null", "integer"] + }, + "CostPerSale": { + "type": ["null", "number"] + }, + "RevenuePerSale": { + "type": ["null", "number"] + }, + "Installs": { + "type": ["null", "integer"] + }, + "CostPerInstall": { + "type": ["null", "number"] + }, + "RevenuePerInstall": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_labels.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_labels.json new file mode 100644 index 000000000000..5daab1862dbd --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_labels.json @@ -0,0 +1,32 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "Account Id": { + "type": ["null", "integer"] + }, + "Ad Group": { + "type": ["null", "string"] + }, + "Campaign": { + "type": ["null", "string"] + }, + "Client Id": { + "type": ["null", "string"] + }, + "Id": { + "type": ["null", "integer"] + }, + "Parent Id": { + "type": ["null", "integer"] + }, + "Modified Time": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "Status": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_performance_report.json index b0e952bb0aec..6b8d28e98867 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_performance_report.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_performance_report.json @@ -12,7 +12,8 @@ "type": ["null", "integer"] }, "TimePeriod": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" }, "CurrencyCode": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_performance_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_performance_report_hourly.json new file mode 100644 index 000000000000..a1300b327c69 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_group_performance_report_hourly.json @@ -0,0 +1,158 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountId": { + "type": ["null", "integer"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "AdGroupId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "Network": { + "type": ["null", "string"] + }, + "DeliveredMatchType": { + "type": ["null", "string"] + }, + "DeviceOS": { + "type": ["null", "string"] + }, + "TopVsOther": { + "type": ["null", "string"] + }, + "BidMatchType": { + "type": ["null", "string"] + }, + "Language": { + "type": ["null", "string"] + }, + "AccountName": { + "type": ["null", "string"] + }, + "CampaignName": { + "type": ["null", "string"] + }, + "CampaignType": { + "type": ["null", "string"] + }, + "AdGroupName": { + "type": ["null", "string"] + }, + "AdGroupType": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "QualityScore": { + "type": ["null", "number"] + }, + "ExpectedCtr": { + "type": ["null", "string"] + }, + "AdRelevance": { + "type": ["null", "number"] + }, + "LandingPageExperience": { + "type": ["null", "number"] + }, + "PhoneImpressions": { + "type": ["null", "integer"] + }, + "PhoneCalls": { + "type": ["null", "integer"] + }, + "Ptr": { + "type": ["null", "number"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "CustomParameters": { + "type": ["null", "string"] + }, + "FinalUrlSuffix": { + "type": ["null", "string"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "Conversions": { + "type": ["null", "number"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_groups.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_groups.json index adb0d3262869..72dd55e9416d 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_groups.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_groups.json @@ -2,6 +2,15 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { + "CampaignId": { + "type": ["null", "integer"] + }, + "AccountId": { + "type": ["null", "integer"] + }, + "CustomerId": { + "type": ["null", "integer"] + }, "AdRotation": { "type": ["null", "object"], "properties": { diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_performance_report.json index 44945b40758e..c884c8e5ffb3 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_performance_report.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_performance_report.json @@ -15,7 +15,14 @@ "type": ["null", "integer"] }, "TimePeriod": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" + }, + "AbsoluteTopImpressionRatePercent": { + "type": ["null", "number"] + }, + "TopImpressionRatePercent": { + "type": ["null", "number"] }, "CurrencyCode": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_performance_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_performance_report_hourly.json new file mode 100644 index 000000000000..93a690f08cd3 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ad_performance_report_hourly.json @@ -0,0 +1,152 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountId": { + "type": ["null", "integer"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "AdGroupId": { + "type": ["null", "integer"] + }, + "AdId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "Language": { + "type": ["null", "string"] + }, + "Network": { + "type": ["null", "string"] + }, + "DeviceOS": { + "type": ["null", "string"] + }, + "TopVsOther": { + "type": ["null", "string"] + }, + "BidMatchType": { + "type": ["null", "string"] + }, + "DeliveredMatchType": { + "type": ["null", "string"] + }, + "AccountName": { + "type": ["null", "string"] + }, + "CampaignName": { + "type": ["null", "string"] + }, + "CampaignType": { + "type": ["null", "string"] + }, + "AdGroupName": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "DestinationUrl": { + "type": ["null", "string"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "CustomParameters": { + "type": ["null", "string"] + }, + "FinalAppUrl": { + "type": ["null", "string"] + }, + "AdDescription": { + "type": ["null", "string"] + }, + "AdDescription2": { + "type": ["null", "string"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "ViewThroughConversionsQualified": { + "type": ["null", "number"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "Conversions": { + "type": ["null", "number"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ads.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ads.json index 810eeb047b97..a0e28296e072 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ads.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/ads.json @@ -2,6 +2,15 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { + "AdGroupId": { + "type": ["null", "integer"] + }, + "AccountId": { + "type": ["null", "integer"] + }, + "CustomerId": { + "type": ["null", "integer"] + }, "AdFormatPreference": { "type": ["null", "string"] }, @@ -11,6 +20,181 @@ "EditorialStatus": { "type": ["null", "string"] }, + "BusinessName": { + "type": ["null", "string"] + }, + "CallToAction": { + "type": ["null", "string"] + }, + "CallToActionLanguage": { + "type": ["null", "string"] + }, + "Headline": { + "type": ["null", "string"] + }, + "Images": { + "type": ["null", "object"], + "properties": { + "AssetLink": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "Asset": { + "type": ["null", "object"], + "properties": { + "Id": { + "type": ["null", "integer"] + }, + "Name": { + "type": ["null", "string"] + }, + "Type": { + "type": ["null", "string"] + }, + "Text": { + "type": ["null", "string"] + } + } + }, + "AssetPerformanceLabel": { + "type": ["null", "string"] + }, + "EditorialStatus": { + "type": ["null", "string"] + }, + "PinnedField": { + "type": ["null", "string"] + } + } + } + } + } + }, + "Videos": { + "type": ["null", "object"], + "properties": { + "AssetLink": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "Asset": { + "type": ["null", "object"], + "properties": { + "Id": { + "type": ["null", "integer"] + }, + "Name": { + "type": ["null", "string"] + }, + "Type": { + "type": ["null", "string"] + }, + "Text": { + "type": ["null", "string"] + } + } + }, + "AssetPerformanceLabel": { + "type": ["null", "string"] + }, + "EditorialStatus": { + "type": ["null", "string"] + }, + "PinnedField": { + "type": ["null", "string"] + } + } + } + } + } + }, + "LongHeadlines": { + "type": ["null", "object"], + "properties": { + "AssetLink": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "Asset": { + "type": ["null", "object"], + "properties": { + "Id": { + "type": ["null", "integer"] + }, + "Name": { + "type": ["null", "string"] + }, + "Type": { + "type": ["null", "string"] + }, + "Text": { + "type": ["null", "string"] + } + } + }, + "AssetPerformanceLabel": { + "type": ["null", "string"] + }, + "EditorialStatus": { + "type": ["null", "string"] + }, + "PinnedField": { + "type": ["null", "string"] + } + } + } + } + } + }, + "LongHeadline": { + "type": ["null", "object"], + "properties": { + "Asset": { + "type": ["null", "object"], + "properties": { + "Id": { + "type": ["null", "integer"] + }, + "Name": { + "type": ["null", "integer"] + }, + "Type": { + "type": ["null", "integer"] + } + } + }, + "AssetPerformanceLabel": { + "type": ["null", "string"] + }, + "EditorialStatus": { + "type": ["null", "string"] + }, + "PinnedField": { + "type": ["null", "string"] + } + } + }, + "LongHeadlineString": { + "type": ["null", "string"] + }, + "Text": { + "type": ["null", "string"] + }, + "TextPart2": { + "type": ["null", "string"] + }, + "TitlePart1": { + "type": ["null", "string"] + }, + "TitlePart2": { + "type": ["null", "string"] + }, + "TitlePart3": { + "type": ["null", "string"] + }, "FinalAppUrls": { "type": "null" }, diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/age_gender_audience_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/age_gender_audience_report.json new file mode 100644 index 000000000000..36285f85a5e2 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/age_gender_audience_report.json @@ -0,0 +1,109 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountId": { + "type": ["null", "integer"] + }, + "AgeGroup": { + "type": ["null", "string"] + }, + "Gender": { + "type": ["null", "string"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date" + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AccountName": { + "type": ["null", "string"] + }, + "AccountNumber": { + "type": ["null", "string"] + }, + "CampaignName": { + "type": ["null", "string"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "AdGroupName": { + "type": ["null", "string"] + }, + "AdGroupId": { + "type": ["null", "integer"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Conversions": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "ExtendedCost": { + "type": ["null", "number"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "Language": { + "type": ["null", "string"] + }, + "AccountStatus": { + "type": ["null", "string"] + }, + "CampaignStatus": { + "type": ["null", "string"] + }, + "AdGroupStatus": { + "type": ["null", "string"] + }, + "BaseCampaignId": { + "type": ["null", "string"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "Goal": { + "type": ["null", "string"] + }, + "GoalType": { + "type": ["null", "string"] + }, + "AbsoluteTopImpressionRatePercent": { + "type": ["null", "number"] + }, + "TopImpressionRatePercent": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "AllConversionsQualified": { + "type": ["null", "number"] + }, + "ViewThroughConversionsQualified": { + "type": ["null", "number"] + }, + "ViewThroughRevenue": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/age_gender_audience_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/age_gender_audience_report_hourly.json new file mode 100644 index 000000000000..544559e884d5 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/age_gender_audience_report_hourly.json @@ -0,0 +1,110 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountId": { + "type": ["null", "integer"] + }, + "AgeGroup": { + "type": ["null", "string"] + }, + "Gender": { + "type": ["null", "string"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AccountName": { + "type": ["null", "string"] + }, + "AccountNumber": { + "type": ["null", "string"] + }, + "CampaignName": { + "type": ["null", "string"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "AdGroupName": { + "type": ["null", "string"] + }, + "AdGroupId": { + "type": ["null", "integer"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Conversions": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "ExtendedCost": { + "type": ["null", "number"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "Language": { + "type": ["null", "string"] + }, + "AccountStatus": { + "type": ["null", "string"] + }, + "CampaignStatus": { + "type": ["null", "string"] + }, + "AdGroupStatus": { + "type": ["null", "string"] + }, + "BaseCampaignId": { + "type": ["null", "string"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "Goal": { + "type": ["null", "string"] + }, + "GoalType": { + "type": ["null", "string"] + }, + "AbsoluteTopImpressionRatePercent": { + "type": ["null", "number"] + }, + "TopImpressionRatePercent": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "AllConversionsQualified": { + "type": ["null", "number"] + }, + "ViewThroughConversionsQualified": { + "type": ["null", "number"] + }, + "ViewThroughRevenue": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/app_install_ad_labels.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/app_install_ad_labels.json new file mode 100644 index 000000000000..74ebe7d23dfe --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/app_install_ad_labels.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "Account Id": { + "type": ["null", "integer"] + }, + "Client Id": { + "type": ["null", "string"] + }, + "Id": { + "type": ["null", "integer"] + }, + "Parent Id": { + "type": ["null", "integer"] + }, + "Modified Time": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "Status": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/app_install_ads.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/app_install_ads.json new file mode 100644 index 000000000000..4db96b254020 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/app_install_ads.json @@ -0,0 +1,71 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "Ad Group": { + "type": ["null", "string"] + }, + "App Id": { + "type": ["null", "integer"] + }, + "Campaign": { + "type": ["null", "string"] + }, + "Client Id": { + "type": ["null", "integer"] + }, + "Custom Parameter": { + "type": ["null", "string"] + }, + "Device Preference": { + "type": ["null", "string"] + }, + "Editorial Appeal Status": { + "type": ["null", "string"] + }, + "Editorial Location": { + "type": ["null", "string"] + }, + "Editorial Reason Code": { + "type": ["null", "string"] + }, + "Editorial Status": { + "type": ["null", "string"] + }, + "Editorial Term": { + "type": ["null", "string"] + }, + "Final Url": { + "type": ["null", "string"] + }, + "Final Url Suffix": { + "type": ["null", "string"] + }, + "Id": { + "type": ["null", "integer"] + }, + "Modified Time": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "Parent Id": { + "type": ["null", "integer"] + }, + "Publisher Countries": { + "type": ["null", "string"] + }, + "Status": { + "type": ["null", "string"] + }, + "Text": { + "type": ["null", "string"] + }, + "Title": { + "type": ["null", "string"] + }, + "Tracking Template": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/budget_summary_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/budget_summary_report.json index f8366861a8c4..6c4cf7f5a939 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/budget_summary_report.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/budget_summary_report.json @@ -18,7 +18,8 @@ "type": ["null", "string"] }, "Date": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" }, "MonthlyBudget": { "type": ["null", "number"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_impression_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_impression_performance_report.json new file mode 100644 index 000000000000..4b2983bd3258 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_impression_performance_report.json @@ -0,0 +1,304 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountName": { + "type": ["null", "string"] + }, + "AccountNumber": { + "type": ["null", "string"] + }, + "AccountId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date" + }, + "CampaignStatus": { + "type": ["null", "string"] + }, + "CampaignName": { + "type": ["null", "string"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "Conversions": { + "type": ["null", "integer"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "LowQualityClicks": { + "type": ["null", "integer"] + }, + "LowQualityClicksPercent": { + "type": ["null", "number"] + }, + "LowQualityImpressions": { + "type": ["null", "integer"] + }, + "LowQualityImpressionsPercent": { + "type": ["null", "number"] + }, + "LowQualityConversions": { + "type": ["null", "integer"] + }, + "LowQualityConversionRate": { + "type": ["null", "number"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "ImpressionSharePercent": { + "type": ["null", "number"] + }, + "ImpressionLostToBudgetPercent": { + "type": ["null", "number"] + }, + "ImpressionLostToRankAggPercent": { + "type": ["null", "number"] + }, + "QualityScore": { + "type": ["null", "number"] + }, + "ExpectedCtr": { + "type": ["null", "string"] + }, + "AdRelevance": { + "type": ["null", "number"] + }, + "LandingPageExperience": { + "type": ["null", "number"] + }, + "HistoricalQualityScore": { + "type": ["null", "integer"] + }, + "HistoricalExpectedCtr": { + "type": ["null", "integer"] + }, + "HistoricalAdRelevance": { + "type": ["null", "integer"] + }, + "HistoricalLandingPageExperience": { + "type": ["null", "integer"] + }, + "PhoneImpressions": { + "type": ["null", "integer"] + }, + "PhoneCalls": { + "type": ["null", "integer"] + }, + "Ptr": { + "type": ["null", "number"] + }, + "Network": { + "type": ["null", "string"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + }, + "TrackingTemplate": { + "type": ["null", "string"] + }, + "CustomParameters": { + "type": ["null", "string"] + }, + "AccountStatus": { + "type": ["null", "string"] + }, + "LowQualityGeneralClicks": { + "type": ["null", "integer"] + }, + "LowQualitySophisticatedClicks": { + "type": ["null", "integer"] + }, + "CampaignLabels": { + "type": ["null", "string"] + }, + "ExactMatchImpressionSharePercent": { + "type": ["null", "number"] + }, + "ClickSharePercent": { + "type": ["null", "number"] + }, + "AbsoluteTopImpressionSharePercent": { + "type": ["null", "number"] + }, + "FinalUrlSuffix": { + "type": ["null", "string"] + }, + "CampaignType": { + "type": ["null", "string"] + }, + "TopImpressionShareLostToRankPercent": { + "type": ["null", "number"] + }, + "TopImpressionShareLostToBudgetPercent": { + "type": ["null", "number"] + }, + "AbsoluteTopImpressionShareLostToRankPercent": { + "type": ["null", "number"] + }, + "AbsoluteTopImpressionShareLostToBudgetPercent": { + "type": ["null", "number"] + }, + "TopImpressionSharePercent": { + "type": ["null", "number"] + }, + "AbsoluteTopImpressionRatePercent": { + "type": ["null", "number"] + }, + "TopImpressionRatePercent": { + "type": ["null", "number"] + }, + "BaseCampaignId": { + "type": ["null", "integer"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "AudienceImpressionSharePercent": { + "type": ["null", "number"] + }, + "AudienceImpressionLostToRankPercent": { + "type": ["null", "number"] + }, + "AudienceImpressionLostToBudgetPercent": { + "type": ["null", "number"] + }, + "RelativeCtr": { + "type": ["null", "number"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "LowQualityConversionsQualified": { + "type": ["null", "number"] + }, + "AllConversionsQualified": { + "type": ["null", "number"] + }, + "ViewThroughConversionsQualified": { + "type": ["null", "number"] + }, + "ViewThroughRevenue": { + "type": ["null", "number"] + }, + "VideoViews": { + "type": ["null", "integer"] + }, + "ViewThroughRate": { + "type": ["null", "number"] + }, + "AverageCPV": { + "type": ["null", "number"] + }, + "VideoViewsAt25Percent": { + "type": ["null", "integer"] + }, + "VideoViewsAt50Percent": { + "type": ["null", "integer"] + }, + "VideoViewsAt75Percent": { + "type": ["null", "integer"] + }, + "CompletedVideoViews": { + "type": ["null", "integer"] + }, + "VideoCompletionRate": { + "type": ["null", "number"] + }, + "TotalWatchTimeInMS": { + "type": ["null", "integer"] + }, + "AverageWatchTimePerVideoView": { + "type": ["null", "number"] + }, + "AverageWatchTimePerImpression": { + "type": ["null", "number"] + }, + "Sales": { + "type": ["null", "integer"] + }, + "CostPerSale": { + "type": ["null", "number"] + }, + "RevenuePerSale": { + "type": ["null", "number"] + }, + "Installs": { + "type": ["null", "integer"] + }, + "CostPerInstall": { + "type": ["null", "number"] + }, + "RevenuePerInstall": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_impression_performance_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_impression_performance_report_hourly.json new file mode 100644 index 000000000000..a5e48498383f --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_impression_performance_report_hourly.json @@ -0,0 +1,248 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountName": { + "type": ["null", "string"] + }, + "AccountNumber": { + "type": ["null", "string"] + }, + "AccountId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "CampaignStatus": { + "type": ["null", "string"] + }, + "CampaignName": { + "type": ["null", "string"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "Conversions": { + "type": ["null", "integer"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "LowQualityClicks": { + "type": ["null", "integer"] + }, + "LowQualityClicksPercent": { + "type": ["null", "number"] + }, + "LowQualityImpressions": { + "type": ["null", "integer"] + }, + "LowQualityImpressionsPercent": { + "type": ["null", "number"] + }, + "LowQualityConversions": { + "type": ["null", "integer"] + }, + "LowQualityConversionRate": { + "type": ["null", "number"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "QualityScore": { + "type": ["null", "number"] + }, + "ExpectedCtr": { + "type": ["null", "string"] + }, + "AdRelevance": { + "type": ["null", "number"] + }, + "LandingPageExperience": { + "type": ["null", "number"] + }, + "PhoneImpressions": { + "type": ["null", "integer"] + }, + "PhoneCalls": { + "type": ["null", "integer"] + }, + "Ptr": { + "type": ["null", "number"] + }, + "Network": { + "type": ["null", "string"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + }, + "TrackingTemplate": { + "type": ["null", "string"] + }, + "CustomParameters": { + "type": ["null", "string"] + }, + "AccountStatus": { + "type": ["null", "string"] + }, + "LowQualityGeneralClicks": { + "type": ["null", "integer"] + }, + "LowQualitySophisticatedClicks": { + "type": ["null", "integer"] + }, + "CampaignLabels": { + "type": ["null", "string"] + }, + "FinalUrlSuffix": { + "type": ["null", "string"] + }, + "CampaignType": { + "type": ["null", "string"] + }, + "AbsoluteTopImpressionRatePercent": { + "type": ["null", "number"] + }, + "TopImpressionRatePercent": { + "type": ["null", "number"] + }, + "BaseCampaignId": { + "type": ["null", "integer"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "LowQualityConversionsQualified": { + "type": ["null", "number"] + }, + "AllConversionsQualified": { + "type": ["null", "number"] + }, + "ViewThroughConversionsQualified": { + "type": ["null", "number"] + }, + "ViewThroughRevenue": { + "type": ["null", "number"] + }, + "VideoViews": { + "type": ["null", "integer"] + }, + "ViewThroughRate": { + "type": ["null", "number"] + }, + "AverageCPV": { + "type": ["null", "number"] + }, + "VideoViewsAt25Percent": { + "type": ["null", "integer"] + }, + "VideoViewsAt50Percent": { + "type": ["null", "integer"] + }, + "VideoViewsAt75Percent": { + "type": ["null", "integer"] + }, + "CompletedVideoViews": { + "type": ["null", "integer"] + }, + "VideoCompletionRate": { + "type": ["null", "number"] + }, + "TotalWatchTimeInMS": { + "type": ["null", "integer"] + }, + "AverageWatchTimePerVideoView": { + "type": ["null", "number"] + }, + "AverageWatchTimePerImpression": { + "type": ["null", "number"] + }, + "Sales": { + "type": ["null", "integer"] + }, + "CostPerSale": { + "type": ["null", "number"] + }, + "RevenuePerSale": { + "type": ["null", "number"] + }, + "Installs": { + "type": ["null", "integer"] + }, + "CostPerInstall": { + "type": ["null", "number"] + }, + "RevenuePerInstall": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_labels.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_labels.json new file mode 100644 index 000000000000..7db5d82599fa --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_labels.json @@ -0,0 +1,29 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "Account Id": { + "type": ["null", "integer"] + }, + "Campaign": { + "type": ["null", "string"] + }, + "Client Id": { + "type": ["null", "string"] + }, + "Id": { + "type": ["null", "integer"] + }, + "Parent Id": { + "type": ["null", "integer"] + }, + "Modified Time": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "Status": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_performance_report.json index eff183e144e9..6ac3320854d6 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_performance_report.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_performance_report.json @@ -9,7 +9,8 @@ "type": ["null", "integer"] }, "TimePeriod": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" }, "CurrencyCode": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_performance_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_performance_report_hourly.json new file mode 100644 index 000000000000..bda17310348f --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaign_performance_report_hourly.json @@ -0,0 +1,176 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountId": { + "type": ["null", "integer"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "Network": { + "type": ["null", "string"] + }, + "DeliveredMatchType": { + "type": ["null", "string"] + }, + "DeviceOS": { + "type": ["null", "string"] + }, + "TopVsOther": { + "type": ["null", "string"] + }, + "BidMatchType": { + "type": ["null", "string"] + }, + "AccountName": { + "type": ["null", "string"] + }, + "CampaignName": { + "type": ["null", "string"] + }, + "CampaignType": { + "type": ["null", "string"] + }, + "CampaignStatus": { + "type": ["null", "string"] + }, + "CampaignLabels": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "QualityScore": { + "type": ["null", "number"] + }, + "AdRelevance": { + "type": ["null", "number"] + }, + "LandingPageExperience": { + "type": ["null", "number"] + }, + "PhoneImpressions": { + "type": ["null", "integer"] + }, + "PhoneCalls": { + "type": ["null", "integer"] + }, + "Ptr": { + "type": ["null", "number"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "CustomParameters": { + "type": ["null", "string"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "Conversions": { + "type": ["null", "number"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "LowQualityClicks": { + "type": ["null", "integer"] + }, + "LowQualityClicksPercent": { + "type": ["null", "number"] + }, + "LowQualityImpressions": { + "type": ["null", "integer"] + }, + "LowQualitySophisticatedClicks": { + "type": ["null", "integer"] + }, + "LowQualityConversions": { + "type": ["null", "integer"] + }, + "LowQualityConversionRate": { + "type": ["null", "number"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + }, + "BudgetName": { + "type": ["null", "string"] + }, + "BudgetStatus": { + "type": ["null", "string"] + }, + "BudgetAssociationStatus": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaigns.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaigns.json index dfa4f6eb19c2..c5790cd8928f 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/campaigns.json @@ -2,6 +2,12 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { + "AccountId": { + "type": ["null", "integer"] + }, + "CustomerId": { + "type": ["null", "integer"] + }, "AudienceAdsBidAdjustment": { "type": ["null", "number"] }, @@ -15,7 +21,7 @@ "type": ["null", "object"], "properties": { "Amount": { - "type": ["null", "string"] + "type": ["null", "number"] } } } diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/geographic_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/geographic_performance_report.json index f2c65f08ff39..8e55c0a61121 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/geographic_performance_report.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/geographic_performance_report.json @@ -13,7 +13,8 @@ "type": ["null", "integer"] }, "TimePeriod": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" }, "AccountNumber": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/geographic_performance_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/geographic_performance_report_hourly.json new file mode 100644 index 000000000000..9b79a66cfe24 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/geographic_performance_report_hourly.json @@ -0,0 +1,213 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "AccountId": { + "type": ["null", "integer"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "AdGroupId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "AccountNumber": { + "type": ["null", "string"] + }, + "Country": { + "type": ["null", "string"] + }, + "State": { + "type": ["null", "string"] + }, + "MetroArea": { + "type": ["null", "string"] + }, + "City": { + "type": ["null", "string"] + }, + "ProximityTargetLocation": { + "type": ["null", "string"] + }, + "Radius": { + "type": ["null", "string"] + }, + "LocationType": { + "type": ["null", "string"] + }, + "MostSpecificLocation": { + "type": ["null", "string"] + }, + "AccountStatus": { + "type": ["null", "string"] + }, + "CampaignStatus": { + "type": ["null", "string"] + }, + "AdGroupStatus": { + "type": ["null", "string"] + }, + "County": { + "type": ["null", "string"] + }, + "PostalCode": { + "type": ["null", "string"] + }, + "LocationId": { + "type": ["null", "string"] + }, + "BaseCampaignId": { + "type": ["null", "string"] + }, + "Goal": { + "type": ["null", "string"] + }, + "GoalType": { + "type": ["null", "string"] + }, + "AbsoluteTopImpressionRatePercent": { + "type": ["null", "number"] + }, + "TopImpressionRatePercent": { + "type": ["null", "string"] + }, + "AllConversionsQualified": { + "type": ["null", "string"] + }, + "Neighborhood": { + "type": ["null", "string"] + }, + "ViewThroughRevenue": { + "type": ["null", "string"] + }, + "CampaignType": { + "type": ["null", "string"] + }, + "AssetGroupId": { + "type": ["null", "string"] + }, + "AssetGroupName": { + "type": ["null", "string"] + }, + "AssetGroupStatus": { + "type": ["null", "string"] + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "DeliveredMatchType": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "Language": { + "type": ["null", "string"] + }, + "Network": { + "type": ["null", "string"] + }, + "DeviceOS": { + "type": ["null", "string"] + }, + "TopVsOther": { + "type": ["null", "string"] + }, + "BidMatchType": { + "type": ["null", "string"] + }, + "AccountName": { + "type": ["null", "string"] + }, + "CampaignName": { + "type": ["null", "string"] + }, + "AdGroupName": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "ViewThroughConversionsQualified": { + "type": ["null", "number"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "Conversions": { + "type": ["null", "number"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_labels.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_labels.json new file mode 100644 index 000000000000..74ebe7d23dfe --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_labels.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "Account Id": { + "type": ["null", "integer"] + }, + "Client Id": { + "type": ["null", "string"] + }, + "Id": { + "type": ["null", "integer"] + }, + "Parent Id": { + "type": ["null", "integer"] + }, + "Modified Time": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "Status": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_performance_report.json index c6c6279fbde0..70ccc68a0b93 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_performance_report.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_performance_report.json @@ -22,7 +22,8 @@ "type": ["null", "integer"] }, "TimePeriod": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" }, "CurrencyCode": { "type": ["null", "string"] @@ -63,18 +64,6 @@ "KeywordStatus": { "type": ["null", "string"] }, - "HistoricalExpectedCtr": { - "type": ["null", "number"] - }, - "HistoricalAdRelevance": { - "type": ["null", "number"] - }, - "HistoricalLandingPageExperience": { - "type": ["null", "number"] - }, - "HistoricalQualityScore": { - "type": ["null", "number"] - }, "Impressions": { "type": ["null", "integer"] }, diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_performance_report_daily.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_performance_report_daily.json new file mode 100644 index 000000000000..48e35d9f3ce9 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_performance_report_daily.json @@ -0,0 +1,190 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountId": { + "type": ["null", "integer"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "AdGroupId": { + "type": ["null", "integer"] + }, + "KeywordId": { + "type": ["null", "integer"] + }, + "Keyword": { + "type": ["null", "string"] + }, + "AdId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date" + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "DeliveredMatchType": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "Language": { + "type": ["null", "string"] + }, + "Network": { + "type": ["null", "string"] + }, + "DeviceOS": { + "type": ["null", "string"] + }, + "TopVsOther": { + "type": ["null", "string"] + }, + "BidMatchType": { + "type": ["null", "string"] + }, + "AccountName": { + "type": ["null", "string"] + }, + "CampaignName": { + "type": ["null", "string"] + }, + "AdGroupName": { + "type": ["null", "string"] + }, + "KeywordStatus": { + "type": ["null", "string"] + }, + "HistoricalExpectedCtr": { + "type": ["null", "number"] + }, + "HistoricalAdRelevance": { + "type": ["null", "number"] + }, + "HistoricalLandingPageExperience": { + "type": ["null", "number"] + }, + "HistoricalQualityScore": { + "type": ["null", "number"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "CurrentMaxCpc": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "QualityScore": { + "type": ["null", "number"] + }, + "ExpectedCtr": { + "type": ["null", "string"] + }, + "AdRelevance": { + "type": ["null", "number"] + }, + "LandingPageExperience": { + "type": ["null", "number"] + }, + "QualityImpact": { + "type": ["null", "number"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "CustomParameters": { + "type": ["null", "string"] + }, + "FinalAppUrl": { + "type": ["null", "string"] + }, + "Mainline1Bid": { + "type": ["null", "number"] + }, + "MainlineBid": { + "type": ["null", "number"] + }, + "FirstPageBid": { + "type": ["null", "number"] + }, + "FinalUrlSuffix": { + "type": ["null", "string"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "ViewThroughConversionsQualified": { + "type": ["null", "number"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "Conversions": { + "type": ["null", "number"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_performance_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_performance_report_hourly.json new file mode 100644 index 000000000000..831c389d24a1 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keyword_performance_report_hourly.json @@ -0,0 +1,180 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "AccountId": { + "type": ["null", "integer"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "AdGroupId": { + "type": ["null", "integer"] + }, + "KeywordId": { + "type": ["null", "integer"] + }, + "Keyword": { + "type": ["null", "string"] + }, + "AdId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "DeliveredMatchType": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "Language": { + "type": ["null", "string"] + }, + "Network": { + "type": ["null", "string"] + }, + "DeviceOS": { + "type": ["null", "string"] + }, + "TopVsOther": { + "type": ["null", "string"] + }, + "BidMatchType": { + "type": ["null", "string"] + }, + "AccountName": { + "type": ["null", "string"] + }, + "CampaignName": { + "type": ["null", "string"] + }, + "AdGroupName": { + "type": ["null", "string"] + }, + "KeywordStatus": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "CurrentMaxCpc": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "QualityScore": { + "type": ["null", "number"] + }, + "ExpectedCtr": { + "type": ["null", "string"] + }, + "AdRelevance": { + "type": ["null", "number"] + }, + "LandingPageExperience": { + "type": ["null", "number"] + }, + "QualityImpact": { + "type": ["null", "number"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "CustomParameters": { + "type": ["null", "string"] + }, + "FinalAppUrl": { + "type": ["null", "string"] + }, + "Mainline1Bid": { + "type": ["null", "number"] + }, + "MainlineBid": { + "type": ["null", "number"] + }, + "FirstPageBid": { + "type": ["null", "number"] + }, + "FinalUrlSuffix": { + "type": ["null", "string"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "ViewThroughConversionsQualified": { + "type": ["null", "number"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "Conversions": { + "type": ["null", "number"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keywords.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keywords.json new file mode 100644 index 000000000000..4f25c1378753 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/keywords.json @@ -0,0 +1,104 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "Account Id": { + "type": ["null", "integer"] + }, + "Id": { + "type": ["null", "integer"] + }, + "Ad Group": { + "type": ["null", "string"] + }, + "Bid": { + "type": ["null", "string"] + }, + "Bid Strategy Type": { + "type": ["null", "string"] + }, + "Campaign": { + "type": ["null", "string"] + }, + "Client Id": { + "type": ["null", "integer"] + }, + "Custom Parameter": { + "type": ["null", "string"] + }, + "Destination Url": { + "type": ["null", "string"] + }, + "Modified Time": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "Editorial Appeal Status": { + "type": ["null", "string"] + }, + "Editorial Location": { + "type": ["null", "string"] + }, + "Editorial Reason Code": { + "type": ["null", "string"] + }, + "Editorial Status": { + "type": ["null", "string"] + }, + "Editorial Term": { + "type": ["null", "string"] + }, + "Final Url": { + "type": ["null", "string"] + }, + "Final Url Suffix": { + "type": ["null", "string"] + }, + "Inherited Bid Strategy Type": { + "type": ["null", "string"] + }, + "Keyword": { + "type": ["null", "string"] + }, + "Keyword Relevance": { + "type": ["null", "string"] + }, + "Landing Page Relevance": { + "type": ["null", "string"] + }, + "Landing Page User Experience": { + "type": ["null", "string"] + }, + "Match Type": { + "type": ["null", "string"] + }, + "Mobile Final Url": { + "type": ["null", "string"] + }, + "Param1": { + "type": ["null", "string"] + }, + "Param2": { + "type": ["null", "string"] + }, + "Param3": { + "type": ["null", "string"] + }, + "Parent Id": { + "type": ["null", "string"] + }, + "Publisher Countries": { + "type": ["null", "string"] + }, + "Quality Score": { + "type": ["null", "string"] + }, + "Status": { + "type": ["null", "string"] + }, + "Tracking Template": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/labels.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/labels.json new file mode 100644 index 000000000000..40845b0eb035 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/labels.json @@ -0,0 +1,32 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "Account Id": { + "type": ["null", "integer"] + }, + "Color": { + "type": ["null", "string"] + }, + "Client Id": { + "type": ["null", "string"] + }, + "Description": { + "type": ["null", "string"] + }, + "Id": { + "type": ["null", "integer"] + }, + "Label": { + "type": ["null", "string"] + }, + "Modified Time": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "Status": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/search_query_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/search_query_performance_report.json new file mode 100644 index 000000000000..57c50442e395 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/search_query_performance_report.json @@ -0,0 +1,181 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountName": { + "type": ["null", "string"] + }, + "AccountNumber": { + "type": ["null", "string"] + }, + "AccountId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date" + }, + "CampaignName": { + "type": ["null", "string"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "AdGroupName": { + "type": ["null", "string"] + }, + "AdGroupId": { + "type": ["null", "integer"] + }, + "AdId": { + "type": ["null", "integer"] + }, + "AdType": { + "type": ["null", "string"] + }, + "DestinationUrl": { + "type": ["null", "string"] + }, + "BidMatchType": { + "type": ["null", "string"] + }, + "DeliveredMatchType": { + "type": ["null", "string"] + }, + "CampaignStatus": { + "type": ["null", "string"] + }, + "AdStatus": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "SearchQuery": { + "type": ["null", "string"] + }, + "Keyword": { + "type": ["null", "string"] + }, + "AdGroupCriterionId": { + "type": ["null", "string"] + }, + "Conversions": { + "type": ["null", "integer"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "Language": { + "type": ["null", "string"] + }, + "KeywordId": { + "type": ["null", "integer"] + }, + "Network": { + "type": ["null", "string"] + }, + "TopVsOther": { + "type": ["null", "string"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "DeviceOS": { + "type": ["null", "string"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + }, + "AccountStatus": { + "type": ["null", "string"] + }, + "AdGroupStatus": { + "type": ["null", "string"] + }, + "KeywordStatus": { + "type": ["null", "string"] + }, + "CampaignType": { + "type": ["null", "string"] + }, + "CustomerId": { + "type": ["null", "integer"] + }, + "CustomerName": { + "type": ["null", "string"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "Goal": { + "type": ["null", "string"] + }, + "GoalType": { + "type": ["null", "string"] + }, + "AbsoluteTopImpressionRatePercent": { + "type": ["null", "number"] + }, + "TopImpressionRatePercent": { + "type": ["null", "number"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "AllConversionsQualified": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/search_query_performance_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/search_query_performance_report_hourly.json new file mode 100644 index 000000000000..27e35b2fc670 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/search_query_performance_report_hourly.json @@ -0,0 +1,182 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountName": { + "type": ["null", "string"] + }, + "AccountNumber": { + "type": ["null", "string"] + }, + "AccountId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "CampaignName": { + "type": ["null", "string"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "AdGroupName": { + "type": ["null", "string"] + }, + "AdGroupId": { + "type": ["null", "integer"] + }, + "AdId": { + "type": ["null", "integer"] + }, + "AdType": { + "type": ["null", "string"] + }, + "DestinationUrl": { + "type": ["null", "string"] + }, + "BidMatchType": { + "type": ["null", "string"] + }, + "DeliveredMatchType": { + "type": ["null", "string"] + }, + "CampaignStatus": { + "type": ["null", "string"] + }, + "AdStatus": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "SearchQuery": { + "type": ["null", "string"] + }, + "Keyword": { + "type": ["null", "string"] + }, + "AdGroupCriterionId": { + "type": ["null", "string"] + }, + "Conversions": { + "type": ["null", "integer"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "Language": { + "type": ["null", "string"] + }, + "KeywordId": { + "type": ["null", "integer"] + }, + "Network": { + "type": ["null", "string"] + }, + "TopVsOther": { + "type": ["null", "string"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "DeviceOS": { + "type": ["null", "string"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + }, + "AccountStatus": { + "type": ["null", "string"] + }, + "AdGroupStatus": { + "type": ["null", "string"] + }, + "KeywordStatus": { + "type": ["null", "string"] + }, + "CampaignType": { + "type": ["null", "string"] + }, + "CustomerId": { + "type": ["null", "integer"] + }, + "CustomerName": { + "type": ["null", "string"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "Goal": { + "type": ["null", "string"] + }, + "GoalType": { + "type": ["null", "string"] + }, + "AbsoluteTopImpressionRatePercent": { + "type": ["null", "number"] + }, + "TopImpressionRatePercent": { + "type": ["null", "number"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "AllConversionsQualified": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/user_location_performance_report.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/user_location_performance_report.json new file mode 100644 index 000000000000..8edbd095b605 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/user_location_performance_report.json @@ -0,0 +1,214 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountName": { + "type": ["null", "string"] + }, + "AccountNumber": { + "type": ["null", "string"] + }, + "AccountId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date" + }, + "CampaignName": { + "type": ["null", "string"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "AdGroupName": { + "type": ["null", "string"] + }, + "AdGroupId": { + "type": ["null", "integer"] + }, + "Country": { + "type": ["null", "string"] + }, + "State": { + "type": ["null", "string"] + }, + "MetroArea": { + "type": ["null", "string"] + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "ProximityTargetLocation": { + "type": ["null", "string"] + }, + "Radius": { + "type": ["null", "integer"] + }, + "Language": { + "type": ["null", "string"] + }, + "City": { + "type": ["null", "string"] + }, + "QueryIntentCountry": { + "type": ["null", "string"] + }, + "QueryIntentState": { + "type": ["null", "string"] + }, + "QueryIntentCity": { + "type": ["null", "string"] + }, + "QueryIntentDMA": { + "type": ["null", "string"] + }, + "BidMatchType": { + "type": ["null", "string"] + }, + "DeliveredMatchType": { + "type": ["null", "string"] + }, + "Network": { + "type": ["null", "string"] + }, + "TopVsOther": { + "type": ["null", "string"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "DeviceOS": { + "type": ["null", "string"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "Conversions": { + "type": ["null", "integer"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + }, + "County": { + "type": ["null", "string"] + }, + "PostalCode": { + "type": ["null", "string"] + }, + "QueryIntentCounty": { + "type": ["null", "string"] + }, + "QueryIntentPostalCode": { + "type": ["null", "string"] + }, + "LocationId": { + "type": ["null", "integer"] + }, + "QueryIntentLocationId": { + "type": ["null", "integer"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "Goal": { + "type": ["null", "string"] + }, + "GoalType": { + "type": ["null", "string"] + }, + "AbsoluteTopImpressionRatePercent": { + "type": ["null", "number"] + }, + "TopImpressionRatePercent": { + "type": ["null", "number"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "AllConversionsQualified": { + "type": ["null", "number"] + }, + "ViewThroughConversionsQualified": { + "type": ["null", "number"] + }, + "Neighborhood": { + "type": ["null", "string"] + }, + "QueryIntentNeighborhood": { + "type": ["null", "string"] + }, + "ViewThroughRevenue": { + "type": ["null", "number"] + }, + "CampaignType": { + "type": ["null", "string"] + }, + "AssetGroupId": { + "type": ["null", "integer"] + }, + "AssetGroupName": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/user_location_performance_report_hourly.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/user_location_performance_report_hourly.json new file mode 100644 index 000000000000..1bd42e6b8087 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/schemas/user_location_performance_report_hourly.json @@ -0,0 +1,215 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "AccountName": { + "type": ["null", "string"] + }, + "AccountNumber": { + "type": ["null", "string"] + }, + "AccountId": { + "type": ["null", "integer"] + }, + "TimePeriod": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "CampaignName": { + "type": ["null", "string"] + }, + "CampaignId": { + "type": ["null", "integer"] + }, + "AdGroupName": { + "type": ["null", "string"] + }, + "AdGroupId": { + "type": ["null", "integer"] + }, + "Country": { + "type": ["null", "string"] + }, + "State": { + "type": ["null", "string"] + }, + "MetroArea": { + "type": ["null", "string"] + }, + "CurrencyCode": { + "type": ["null", "string"] + }, + "AdDistribution": { + "type": ["null", "string"] + }, + "Impressions": { + "type": ["null", "integer"] + }, + "Clicks": { + "type": ["null", "integer"] + }, + "Ctr": { + "type": ["null", "number"] + }, + "AverageCpc": { + "type": ["null", "number"] + }, + "Spend": { + "type": ["null", "number"] + }, + "AveragePosition": { + "type": ["null", "number"] + }, + "ProximityTargetLocation": { + "type": ["null", "string"] + }, + "Radius": { + "type": ["null", "integer"] + }, + "Language": { + "type": ["null", "string"] + }, + "City": { + "type": ["null", "string"] + }, + "QueryIntentCountry": { + "type": ["null", "string"] + }, + "QueryIntentState": { + "type": ["null", "string"] + }, + "QueryIntentCity": { + "type": ["null", "string"] + }, + "QueryIntentDMA": { + "type": ["null", "string"] + }, + "BidMatchType": { + "type": ["null", "string"] + }, + "DeliveredMatchType": { + "type": ["null", "string"] + }, + "Network": { + "type": ["null", "string"] + }, + "TopVsOther": { + "type": ["null", "string"] + }, + "DeviceType": { + "type": ["null", "string"] + }, + "DeviceOS": { + "type": ["null", "string"] + }, + "Assists": { + "type": ["null", "integer"] + }, + "Conversions": { + "type": ["null", "integer"] + }, + "ConversionRate": { + "type": ["null", "number"] + }, + "Revenue": { + "type": ["null", "number"] + }, + "ReturnOnAdSpend": { + "type": ["null", "number"] + }, + "CostPerConversion": { + "type": ["null", "number"] + }, + "CostPerAssist": { + "type": ["null", "number"] + }, + "RevenuePerConversion": { + "type": ["null", "number"] + }, + "RevenuePerAssist": { + "type": ["null", "number"] + }, + "County": { + "type": ["null", "string"] + }, + "PostalCode": { + "type": ["null", "string"] + }, + "QueryIntentCounty": { + "type": ["null", "string"] + }, + "QueryIntentPostalCode": { + "type": ["null", "string"] + }, + "LocationId": { + "type": ["null", "integer"] + }, + "QueryIntentLocationId": { + "type": ["null", "integer"] + }, + "AllConversions": { + "type": ["null", "integer"] + }, + "AllRevenue": { + "type": ["null", "number"] + }, + "AllConversionRate": { + "type": ["null", "number"] + }, + "AllCostPerConversion": { + "type": ["null", "number"] + }, + "AllReturnOnAdSpend": { + "type": ["null", "number"] + }, + "AllRevenuePerConversion": { + "type": ["null", "number"] + }, + "ViewThroughConversions": { + "type": ["null", "integer"] + }, + "Goal": { + "type": ["null", "string"] + }, + "GoalType": { + "type": ["null", "string"] + }, + "AbsoluteTopImpressionRatePercent": { + "type": ["null", "number"] + }, + "TopImpressionRatePercent": { + "type": ["null", "number"] + }, + "AverageCpm": { + "type": ["null", "number"] + }, + "ConversionsQualified": { + "type": ["null", "number"] + }, + "AllConversionsQualified": { + "type": ["null", "number"] + }, + "ViewThroughConversionsQualified": { + "type": ["null", "number"] + }, + "Neighborhood": { + "type": ["null", "string"] + }, + "QueryIntentNeighborhood": { + "type": ["null", "string"] + }, + "ViewThroughRevenue": { + "type": ["null", "number"] + }, + "CampaignType": { + "type": ["null", "string"] + }, + "AssetGroupId": { + "type": ["null", "integer"] + }, + "AssetGroupName": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py index dd9ec06b0387..37c2b9bc5d2b 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/source.py @@ -1,837 +1,70 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - - -import ssl -import time -from abc import ABC, abstractmethod -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union -from urllib.error import URLError +from itertools import product +from typing import Any, List, Mapping, Optional, Tuple from airbyte_cdk import AirbyteLogger -from airbyte_cdk.models import SyncMode +from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream -from bingads.service_client import ServiceClient -from bingads.v13.reporting.reporting_service_manager import ReportingServiceManager -from source_bing_ads.cache import VcrCache +from airbyte_cdk.utils import AirbyteTracedException +from source_bing_ads.base_streams import Accounts, AdGroups, Ads, Campaigns +from source_bing_ads.bulk_streams import AdGroupLabels, AppInstallAdLabels, AppInstallAds, CampaignLabels, KeywordLabels, Keywords, Labels from source_bing_ads.client import Client -from source_bing_ads.reports import ( - ALL_CONVERSION_FIELDS, - ALL_REVENUE_FIELDS, - AVERAGE_FIELDS, - BUDGET_FIELDS, - CONVERSION_FIELDS, - HISTORICAL_FIELDS, - LOW_QUALITY_FIELDS, - REVENUE_FIELDS, - PerformanceReportsMixin, - ReportsMixin, +from source_bing_ads.report_streams import ( # noqa: F401 + AccountImpressionPerformanceReportDaily, + AccountImpressionPerformanceReportHourly, + AccountImpressionPerformanceReportMonthly, + AccountImpressionPerformanceReportWeekly, + AccountPerformanceReportDaily, + AccountPerformanceReportHourly, + AccountPerformanceReportMonthly, + AccountPerformanceReportWeekly, + AdGroupImpressionPerformanceReportDaily, + AdGroupImpressionPerformanceReportHourly, + AdGroupImpressionPerformanceReportMonthly, + AdGroupImpressionPerformanceReportWeekly, + AdGroupPerformanceReportDaily, + AdGroupPerformanceReportHourly, + AdGroupPerformanceReportMonthly, + AdGroupPerformanceReportWeekly, + AdPerformanceReportDaily, + AdPerformanceReportHourly, + AdPerformanceReportMonthly, + AdPerformanceReportWeekly, + AgeGenderAudienceReportDaily, + AgeGenderAudienceReportHourly, + AgeGenderAudienceReportMonthly, + AgeGenderAudienceReportWeekly, + BingAdsReportingServiceStream, + BudgetSummaryReport, + CampaignImpressionPerformanceReportDaily, + CampaignImpressionPerformanceReportHourly, + CampaignImpressionPerformanceReportMonthly, + CampaignImpressionPerformanceReportWeekly, + CampaignPerformanceReportDaily, + CampaignPerformanceReportHourly, + CampaignPerformanceReportMonthly, + CampaignPerformanceReportWeekly, + CustomReport, + GeographicPerformanceReportDaily, + GeographicPerformanceReportHourly, + GeographicPerformanceReportMonthly, + GeographicPerformanceReportWeekly, + KeywordPerformanceReportDaily, + KeywordPerformanceReportHourly, + KeywordPerformanceReportMonthly, + KeywordPerformanceReportWeekly, + SearchQueryPerformanceReportDaily, + SearchQueryPerformanceReportHourly, + SearchQueryPerformanceReportMonthly, + SearchQueryPerformanceReportWeekly, + UserLocationPerformanceReportDaily, + UserLocationPerformanceReportHourly, + UserLocationPerformanceReportMonthly, + UserLocationPerformanceReportWeekly, ) -from suds import sudsobject - -CACHE: VcrCache = VcrCache() - - -class BingAdsStream(Stream, ABC): - primary_key: Optional[Union[str, List[str], List[List[str]]]] = None - # indicates whether stream should cache incoming responses via VcrCache - use_cache: bool = False - - def __init__(self, client: Client, config: Mapping[str, Any]) -> None: - super().__init__() - self.client = client - self.config = config - - @property - @abstractmethod - def data_field(self) -> str: - """ - Specifies root object name in a stream response - """ - pass - - @property - @abstractmethod - def service_name(self) -> str: - """ - Specifies bing ads service name for a current stream - """ - pass - - @property - @abstractmethod - def operation_name(self) -> str: - """ - Specifies operation name to use for a current stream - """ - pass - - @property - @abstractmethod - def additional_fields(self) -> Optional[str]: - """ - Specifies which additional fields to fetch for a current stream. - Expected format: field names separated by space - """ - pass - - @property - def _service(self) -> Union[ServiceClient, ReportingServiceManager]: - return self.client.get_service(service_name=self.service_name) - - @property - def _user_id(self) -> int: - return self._get_user_id() - - # TODO remove once Microsoft support confirm their SSL certificates are always valid... - def _get_user_id(self, number_of_retries=10): - """""" - try: - return self._service.GetUser().User.Id - except URLError as error: - if isinstance(error.reason, ssl.SSLError): - self.logger.warn("SSL certificate error, retrying...") - if number_of_retries > 0: - time.sleep(1) - return self._get_user_id(number_of_retries - 1) - else: - raise error - - def next_page_token(self, response: sudsobject.Object, **kwargs: Mapping[str, Any]) -> Optional[Mapping[str, Any]]: - """ - Default method for streams that don't support pagination - """ - return None - - def parse_response(self, response: sudsobject.Object, **kwargs) -> Iterable[Mapping]: - if response is not None and hasattr(response, self.data_field): - yield from self.client.asdict(response)[self.data_field] - - yield from [] - - def send_request(self, params: Mapping[str, Any], customer_id: str, account_id: str = None) -> Mapping[str, Any]: - request_kwargs = { - "service_name": self.service_name, - "customer_id": customer_id, - "account_id": account_id, - "operation_name": self.operation_name, - "params": params, - } - request = self.client.request(**request_kwargs) - if self.use_cache: - with CACHE.use_cassette(): - return request - else: - return request - - def read_records( - self, - sync_mode: SyncMode, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - **kwargs: Mapping[str, Any], - ) -> Iterable[Mapping[str, Any]]: - stream_state = stream_state or {} - next_page_token = None - account_id = str(stream_slice.get("account_id")) if stream_slice else None - customer_id = str(stream_slice.get("customer_id")) if stream_slice else None - - while True: - params = self.request_params( - stream_state=stream_state, - stream_slice=stream_slice, - next_page_token=next_page_token, - account_id=account_id, - ) - response = self.send_request(params, customer_id=customer_id, account_id=account_id) - for record in self.parse_response(response): - yield record - - next_page_token = self.next_page_token(response, current_page_token=next_page_token) - if not next_page_token: - break - - yield from [] - - -class Accounts(BingAdsStream): - """ - Searches for accounts that the current authenticated user can access. - API doc: https://docs.microsoft.com/en-us/advertising/customer-management-service/searchaccounts?view=bingads-13 - Account schema: https://docs.microsoft.com/en-us/advertising/customer-management-service/advertiseraccount?view=bingads-13 - Stream caches incoming responses to be able to reuse this data in Campaigns stream - """ - - primary_key = "Id" - # Stream caches incoming responses to avoid duplicated http requests - use_cache: bool = True - data_field: str = "AdvertiserAccount" - service_name: str = "CustomerManagementService" - operation_name: str = "SearchAccounts" - additional_fields: str = "TaxCertificate AccountMode" - # maximum page size - page_size_limit: int = 1000 - - def next_page_token(self, response: sudsobject.Object, current_page_token: Optional[int]) -> Optional[Mapping[str, Any]]: - current_page_token = current_page_token or 0 - if response is not None and hasattr(response, self.data_field): - return None if self.page_size_limit > len(response[self.data_field]) else current_page_token + 1 - else: - return None - - def request_params( - self, - next_page_token: Mapping[str, Any] = None, - **kwargs: Mapping[str, Any], - ) -> MutableMapping[str, Any]: - predicates = { - "Predicate": [ - { - "Field": "UserId", - "Operator": "Equals", - "Value": self._user_id, - } - ] - } - - paging = self._service.factory.create("ns5:Paging") - paging.Index = next_page_token or 0 - paging.Size = self.page_size_limit - return { - "PageInfo": paging, - "Predicates": predicates, - "ReturnAdditionalFields": self.additional_fields, - } - - -class Campaigns(BingAdsStream): - """ - Gets the campaigns for all provided accounts. - API doc: https://docs.microsoft.com/en-us/advertising/campaign-management-service/getcampaignsbyaccountid?view=bingads-13 - Campaign schema: https://docs.microsoft.com/en-us/advertising/campaign-management-service/campaign?view=bingads-13 - Stream caches incoming responses to be able to reuse this data in AdGroups stream - """ - - primary_key = "Id" - # Stream caches incoming responses to avoid duplicated http requests - use_cache: bool = True - data_field: str = "Campaign" - service_name: str = "CampaignManagement" - operation_name: str = "GetCampaignsByAccountId" - additional_fields: Iterable[str] = [ - "AdScheduleUseSearcherTimeZone", - "BidStrategyId", - "CpvCpmBiddingScheme", - "DynamicDescriptionSetting", - "DynamicFeedSetting", - "MaxConversionValueBiddingScheme", - "MultimediaAdsBidAdjustment", - "TargetImpressionShareBiddingScheme", - "TargetSetting", - "VerifiedTrackingSetting", - ] - campaign_types: Iterable[str] = ["Audience", "DynamicSearchAds", "Search", "Shopping"] - - def request_params( - self, - stream_slice: Mapping[str, Any] = None, - **kwargs: Mapping[str, Any], - ) -> MutableMapping[str, Any]: - return { - "AccountId": stream_slice["account_id"], - "CampaignType": " ".join(self.campaign_types), - "ReturnAdditionalFields": " ".join(self.additional_fields), - } - - def stream_slices( - self, - **kwargs: Mapping[str, Any], - ) -> Iterable[Optional[Mapping[str, Any]]]: - for account in Accounts(self.client, self.config).read_records(SyncMode.full_refresh): - yield {"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} - - yield from [] - - -class AdGroups(BingAdsStream): - """ - Gets the ad groups for all provided accounts. - API doc: https://docs.microsoft.com/en-us/advertising/campaign-management-service/getadgroupsbycampaignid?view=bingads-13 - AdGroup schema: https://docs.microsoft.com/en-us/advertising/campaign-management-service/adgroup?view=bingads-13 - Stream caches incoming responses to be able to reuse this data in Ads stream - """ - - primary_key = "Id" - # Stream caches incoming responses to avoid duplicated http requests - use_cache: bool = True - data_field: str = "AdGroup" - service_name: str = "CampaignManagement" - operation_name: str = "GetAdGroupsByCampaignId" - additional_fields: str = "AdGroupType AdScheduleUseSearcherTimeZone CpmBid CpvBid MultimediaAdsBidAdjustment" - - def request_params( - self, - stream_slice: Mapping[str, Any] = None, - **kwargs: Mapping[str, Any], - ) -> MutableMapping[str, Any]: - return {"CampaignId": stream_slice["campaign_id"], "ReturnAdditionalFields": self.additional_fields} - - def stream_slices( - self, - **kwargs: Mapping[str, Any], - ) -> Iterable[Optional[Mapping[str, Any]]]: - campaigns = Campaigns(self.client, self.config) - for account in Accounts(self.client, self.config).read_records(SyncMode.full_refresh): - for campaign in campaigns.read_records( - sync_mode=SyncMode.full_refresh, stream_slice={"account_id": account["Id"], "customer_id": account["ParentCustomerId"]} - ): - yield {"campaign_id": campaign["Id"], "account_id": account["Id"], "customer_id": account["ParentCustomerId"]} - - yield from [] - - -class Ads(BingAdsStream): - """ - Retrieves the ads for all provided accounts. - API doc: https://docs.microsoft.com/en-us/advertising/campaign-management-service/getadsbyadgroupid?view=bingads-13 - Ad schema: https://docs.microsoft.com/en-us/advertising/campaign-management-service/ad?view=bingads-13 - """ - - primary_key = "Id" - data_field: str = "Ad" - service_name: str = "CampaignManagement" - operation_name: str = "GetAdsByAdGroupId" - additional_fields: str = "ImpressionTrackingUrls Videos LongHeadlines" - ad_types: Iterable[str] = [ - "Text", - "Image", - "Product", - "AppInstall", - "ExpandedText", - "DynamicSearch", - "ResponsiveAd", - "ResponsiveSearch", - ] - - def request_params( - self, - stream_slice: Mapping[str, Any] = None, - **kwargs: Mapping[str, Any], - ) -> MutableMapping[str, Any]: - return { - "AdGroupId": stream_slice["ad_group_id"], - "AdTypes": {"AdType": self.ad_types}, - "ReturnAdditionalFields": self.additional_fields, - } - - def stream_slices( - self, - **kwargs: Mapping[str, Any], - ) -> Iterable[Optional[Mapping[str, Any]]]: - ad_groups = AdGroups(self.client, self.config) - for slice in ad_groups.stream_slices(sync_mode=SyncMode.full_refresh): - for ad_group in ad_groups.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice): - yield {"ad_group_id": ad_group["Id"], "account_id": slice["account_id"], "customer_id": slice["customer_id"]} - yield from [] - - -class BudgetSummaryReport(ReportsMixin, BingAdsStream): - data_field: str = "" - service_name: str = "ReportingService" - report_name: str = "BudgetSummaryReport" - operation_name: str = "download_report" - additional_fields: str = "" - report_aggregation = None - cursor_field = "Date" - report_schema_name = "budget_summary_report" - primary_key = "Date" - - report_columns = [ - "AccountName", - "AccountNumber", - "AccountId", - "CampaignName", - "CampaignId", - "Date", - "MonthlyBudget", - "DailySpend", - "MonthToDateSpend", - ] - - -class CampaignPerformanceReport(PerformanceReportsMixin, BingAdsStream): - data_field: str = "" - service_name: str = "ReportingService" - report_name: str = "CampaignPerformanceReport" - operation_name: str = "download_report" - additional_fields: str = "" - cursor_field = "TimePeriod" - report_schema_name = "campaign_performance_report" - primary_key = [ - "AccountId", - "CampaignId", - "TimePeriod", - "CurrencyCode", - "AdDistribution", - "DeviceType", - "Network", - "DeliveredMatchType", - "DeviceOS", - "TopVsOther", - "BidMatchType", - ] - - report_columns = [ - *primary_key, - "AccountName", - "CampaignName", - "CampaignType", - "CampaignStatus", - "CampaignLabels", - "Impressions", - "Clicks", - "Ctr", - "Spend", - "CostPerConversion", - "QualityScore", - "AdRelevance", - "LandingPageExperience", - "PhoneImpressions", - "PhoneCalls", - "Ptr", - "Assists", - "ReturnOnAdSpend", - "CostPerAssist", - "CustomParameters", - "ViewThroughConversions", - "AllCostPerConversion", - "AllReturnOnAdSpend", - *ALL_CONVERSION_FIELDS, - *ALL_REVENUE_FIELDS, - *AVERAGE_FIELDS, - *CONVERSION_FIELDS, - *LOW_QUALITY_FIELDS, - *REVENUE_FIELDS, - *BUDGET_FIELDS, - ] - - -class CampaignPerformanceReportHourly(CampaignPerformanceReport): - report_aggregation = "Hourly" - - -class CampaignPerformanceReportDaily(CampaignPerformanceReport): - report_aggregation = "Daily" - report_columns = [ - *CampaignPerformanceReport.report_columns, - *HISTORICAL_FIELDS, - ] - - -class CampaignPerformanceReportWeekly(CampaignPerformanceReport): - report_aggregation = "Weekly" - report_columns = [ - *CampaignPerformanceReport.report_columns, - *HISTORICAL_FIELDS, - ] - - -class CampaignPerformanceReportMonthly(CampaignPerformanceReport): - report_aggregation = "Monthly" - report_columns = [ - *CampaignPerformanceReport.report_columns, - *HISTORICAL_FIELDS, - ] - - -class AdPerformanceReport(PerformanceReportsMixin, BingAdsStream): - data_field: str = "" - service_name: str = "ReportingService" - report_name: str = "AdPerformanceReport" - operation_name: str = "download_report" - additional_fields: str = "" - cursor_field = "TimePeriod" - report_schema_name = "ad_performance_report" - primary_key = [ - "AccountId", - "CampaignId", - "AdGroupId", - "AdId", - "TimePeriod", - "CurrencyCode", - "AdDistribution", - "DeviceType", - "Language", - "Network", - "DeviceOS", - "TopVsOther", - "BidMatchType", - "DeliveredMatchType", - ] - - report_columns = [ - *primary_key, - "AccountName", - "CampaignName", - "CampaignType", - "AdGroupName", - "Impressions", - "Clicks", - "Ctr", - "Spend", - "CostPerConversion", - "DestinationUrl", - "Assists", - "ReturnOnAdSpend", - "CostPerAssist", - "CustomParameters", - "FinalAppUrl", - "AdDescription", - "AdDescription2", - "ViewThroughConversions", - "ViewThroughConversionsQualified", - "AllCostPerConversion", - "AllReturnOnAdSpend", - *CONVERSION_FIELDS, - *AVERAGE_FIELDS, - *ALL_CONVERSION_FIELDS, - *ALL_REVENUE_FIELDS, - *REVENUE_FIELDS, - ] - - -class AdPerformanceReportHourly(AdPerformanceReport): - report_aggregation = "Hourly" - - -class AdPerformanceReportDaily(AdPerformanceReport): - report_aggregation = "Daily" - - -class AdPerformanceReportWeekly(AdPerformanceReport): - report_aggregation = "Weekly" - - -class AdPerformanceReportMonthly(AdPerformanceReport): - report_aggregation = "Monthly" - - -class AdGroupPerformanceReport(PerformanceReportsMixin, BingAdsStream): - data_field: str = "" - service_name: str = "ReportingService" - report_name: str = "AdGroupPerformanceReport" - operation_name: str = "download_report" - additional_fields: str = "" - cursor_field = "TimePeriod" - report_schema_name = "ad_group_performance_report" - - primary_key = [ - "AccountId", - "CampaignId", - "AdGroupId", - "TimePeriod", - "CurrencyCode", - "AdDistribution", - "DeviceType", - "Network", - "DeliveredMatchType", - "DeviceOS", - "TopVsOther", - "BidMatchType", - "Language", - ] - - report_columns = [ - *primary_key, - "AccountName", - "CampaignName", - "CampaignType", - "AdGroupName", - "AdGroupType", - "Impressions", - "Clicks", - "Ctr", - "Spend", - "CostPerConversion", - "QualityScore", - "ExpectedCtr", - "AdRelevance", - "LandingPageExperience", - "PhoneImpressions", - "PhoneCalls", - "Ptr", - "Assists", - "CostPerAssist", - "CustomParameters", - "FinalUrlSuffix", - "ViewThroughConversions", - "AllCostPerConversion", - "AllReturnOnAdSpend", - *ALL_CONVERSION_FIELDS, - *ALL_REVENUE_FIELDS, - *AVERAGE_FIELDS, - *CONVERSION_FIELDS, - *REVENUE_FIELDS, - ] - - -class AdGroupPerformanceReportHourly(AdGroupPerformanceReport): - report_aggregation = "Hourly" - - -class AdGroupPerformanceReportDaily(AdGroupPerformanceReport): - report_aggregation = "Daily" - report_columns = [ - *AdGroupPerformanceReport.report_columns, - *HISTORICAL_FIELDS, - ] - - -class AdGroupPerformanceReportWeekly(AdGroupPerformanceReport): - report_aggregation = "Weekly" - report_columns = [ - *AdGroupPerformanceReport.report_columns, - *HISTORICAL_FIELDS, - ] - - -class AdGroupPerformanceReportMonthly(AdGroupPerformanceReport): - report_aggregation = "Monthly" - report_columns = [ - *AdGroupPerformanceReport.report_columns, - *HISTORICAL_FIELDS, - ] - - -class KeywordPerformanceReport(PerformanceReportsMixin, BingAdsStream): - data_field: str = "" - service_name: str = "ReportingService" - report_name: str = "KeywordPerformanceReport" - operation_name: str = "download_report" - additional_fields: str = "" - cursor_field = "TimePeriod" - report_schema_name = "keyword_performance_report" - primary_key = [ - "AccountId", - "CampaignId", - "AdGroupId", - "KeywordId", - "AdId", - "TimePeriod", - "CurrencyCode", - "DeliveredMatchType", - "AdDistribution", - "DeviceType", - "Language", - "Network", - "DeviceOS", - "TopVsOther", - "BidMatchType", - ] - - report_columns = [ - *primary_key, - "AccountName", - "CampaignName", - "AdGroupName", - "Keyword", - "KeywordStatus", - "Impressions", - "Clicks", - "Ctr", - "CurrentMaxCpc", - "Spend", - "CostPerConversion", - "QualityScore", - "ExpectedCtr", - "AdRelevance", - "LandingPageExperience", - "QualityImpact", - "Assists", - "ReturnOnAdSpend", - "CostPerAssist", - "CustomParameters", - "FinalAppUrl", - "Mainline1Bid", - "MainlineBid", - "FirstPageBid", - "FinalUrlSuffix", - "ViewThroughConversions", - "ViewThroughConversionsQualified", - "AllCostPerConversion", - "AllReturnOnAdSpend", - *CONVERSION_FIELDS, - *AVERAGE_FIELDS, - *ALL_CONVERSION_FIELDS, - *ALL_REVENUE_FIELDS, - *REVENUE_FIELDS, - ] - - -class KeywordPerformanceReportHourly(KeywordPerformanceReport): - report_aggregation = "Hourly" - - -class KeywordPerformanceReportDaily(KeywordPerformanceReport): - report_aggregation = "Daily" - report_columns = [ - *KeywordPerformanceReport.report_columns, - *HISTORICAL_FIELDS, - ] - - -class KeywordPerformanceReportWeekly(KeywordPerformanceReport): - report_aggregation = "Weekly" - - -class KeywordPerformanceReportMonthly(KeywordPerformanceReport): - report_aggregation = "Monthly" - - -class GeographicPerformanceReport(PerformanceReportsMixin, BingAdsStream): - data_field: str = "" - service_name: str = "ReportingService" - report_name: str = "GeographicPerformanceReport" - operation_name: str = "download_report" - additional_fields: str = "" - cursor_field = "TimePeriod" - report_schema_name = "geographic_performance_report" - primary_key = [ - "AccountId", - "CampaignId", - "AdGroupId", - "TimePeriod", - "Country", - "CurrencyCode", - "DeliveredMatchType", - "AdDistribution", - "DeviceType", - "Language", - "Network", - "DeviceOS", - "TopVsOther", - "BidMatchType", - ] - - report_columns = [ - *primary_key, - "MetroArea", - "State", - "City", - "AdGroupName", - "Ctr", - "ProximityTargetLocation", - "Radius", - "Assists", - "ReturnOnAdSpend", - "CostPerAssist", - "LocationType", - "MostSpecificLocation", - "AccountStatus", - "CampaignStatus", - "AdGroupStatus", - "County", - "PostalCode", - "LocationId", - "BaseCampaignId", - "AllCostPerConversion", - "AllReturnOnAdSpend", - "ViewThroughConversions", - "Goal", - "GoalType", - "AbsoluteTopImpressionRatePercent", - "TopImpressionRatePercent", - "AllConversionsQualified", - "ViewThroughConversionsQualified", - "Neighborhood", - "ViewThroughRevenue", - "CampaignType", - "AssetGroupId", - "AssetGroupName", - "AssetGroupStatus", - *CONVERSION_FIELDS, - *AVERAGE_FIELDS, - *ALL_CONVERSION_FIELDS, - *ALL_REVENUE_FIELDS, - *REVENUE_FIELDS, - ] - - -class GeographicPerformanceReportHourly(GeographicPerformanceReport): - report_aggregation = "Hourly" - - -class GeographicPerformanceReportDaily(GeographicPerformanceReport): - report_aggregation = "Daily" - - -class GeographicPerformanceReportWeekly(GeographicPerformanceReport): - report_aggregation = "Weekly" - - -class GeographicPerformanceReportMonthly(GeographicPerformanceReport): - report_aggregation = "Monthly" - - -class AccountPerformanceReport(PerformanceReportsMixin, BingAdsStream): - data_field: str = "" - service_name: str = "ReportingService" - report_name: str = "AccountPerformanceReport" - operation_name: str = "download_report" - additional_fields: str = "" - cursor_field = "TimePeriod" - report_schema_name = "account_performance_report" - primary_key = [ - "AccountId", - "TimePeriod", - "CurrencyCode", - "AdDistribution", - "DeviceType", - "Network", - "DeliveredMatchType", - "DeviceOS", - "TopVsOther", - "BidMatchType", - ] - - report_columns = [ - *primary_key, - "AccountName", - "AccountNumber", - "PhoneImpressions", - "PhoneCalls", - "Clicks", - "Ctr", - "Spend", - "Impressions", - "CostPerConversion", - "Ptr", - "Assists", - "ReturnOnAdSpend", - "CostPerAssist", - *AVERAGE_FIELDS, - *CONVERSION_FIELDS, - *LOW_QUALITY_FIELDS, - *REVENUE_FIELDS, - ] - - -class AccountPerformanceReportHourly(AccountPerformanceReport): - report_aggregation = "Hourly" - - -class AccountPerformanceReportDaily(AccountPerformanceReport): - report_aggregation = "Daily" - - -class AccountPerformanceReportWeekly(AccountPerformanceReport): - report_aggregation = "Weekly" - - -class AccountPerformanceReportMonthly(AccountPerformanceReport): - report_aggregation = "Monthly" class SourceBingAds(AbstractSource): @@ -842,22 +75,51 @@ class SourceBingAds(AbstractSource): def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: try: client = Client(**config) - account_ids = {str(account["Id"]) for account in Accounts(client, config).read_records(SyncMode.full_refresh)} + accounts = Accounts(client, config) + account_ids = set() + for _slice in accounts.stream_slices(): + account_ids.update({str(account["Id"]) for account in accounts.read_records(SyncMode.full_refresh, _slice)}) + self.validate_custom_reposts(config, client) if account_ids: return True, None else: - raise Exception("You don't have accounts assigned to this user.") + raise AirbyteTracedException( + message="Config validation error: You don't have accounts assigned to this user. Please verify your developer token.", + internal_message="You don't have accounts assigned to this user.", + failure_type=FailureType.config_error, + ) except Exception as error: return False, error - def get_report_streams(self, aggregation_type: str) -> List[Stream]: + def validate_custom_reposts(self, config: Mapping[str, Any], client: Client): + custom_reports = self.get_custom_reports(config, client) + for custom_report in custom_reports: + is_valid, reason = custom_report.validate_report_configuration() + if not is_valid: + raise AirbyteTracedException( + message=f"Config validation error: {custom_report.name}: {reason}", + internal_message=f"{custom_report.name}: {reason}", + failure_type=FailureType.config_error, + ) + + def _clear_reporting_object_name(self, report_object: str) -> str: + # reporting mixin adds it + if report_object.endswith("Request"): + return report_object.replace("Request", "") + return report_object + + def get_custom_reports(self, config: Mapping[str, Any], client: Client) -> List[Optional[Stream]]: return [ - globals()[f"AccountPerformanceReport{aggregation_type}"], - globals()[f"KeywordPerformanceReport{aggregation_type}"], - globals()[f"AdGroupPerformanceReport{aggregation_type}"], - globals()[f"AdPerformanceReport{aggregation_type}"], - globals()[f"CampaignPerformanceReport{aggregation_type}"], - globals()[f"GeographicPerformanceReport{aggregation_type}"], + type( + report["name"], + (CustomReport,), + { + "report_name": self._clear_reporting_object_name(report["reporting_object"]), + "custom_report_columns": report["report_columns"], + "report_aggregation": report["report_aggregation"], + }, + )(client, config) + for report in config.get("custom_reports", []) ] def streams(self, config: Mapping[str, Any]) -> List[Stream]: @@ -865,15 +127,35 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: streams = [ Accounts(client, config), AdGroups(client, config), + AdGroupLabels(client, config), + AppInstallAds(client, config), + AppInstallAdLabels(client, config), Ads(client, config), Campaigns(client, config), + BudgetSummaryReport(client, config), + Labels(client, config), + KeywordLabels(client, config), + Keywords(client, config), + CampaignLabels(client, config), ] - streams.append(BudgetSummaryReport(client, config)) - - streams.extend([c(client, config) for c in self.get_report_streams("Hourly")]) - streams.extend([c(client, config) for c in self.get_report_streams("Daily")]) - streams.extend([c(client, config) for c in self.get_report_streams("Weekly")]) - streams.extend([c(client, config) for c in self.get_report_streams("Monthly")]) - + reports = ( + "AgeGenderAudienceReport", + "AccountImpressionPerformanceReport", + "AccountPerformanceReport", + "KeywordPerformanceReport", + "AdGroupPerformanceReport", + "AdPerformanceReport", + "AdGroupImpressionPerformanceReport", + "CampaignPerformanceReport", + "CampaignImpressionPerformanceReport", + "GeographicPerformanceReport", + "SearchQueryPerformanceReport", + "UserLocationPerformanceReport", + ) + report_aggregation = ("Hourly", "Daily", "Weekly", "Monthly") + streams.extend([eval(f"{report}{aggregation}")(client, config) for (report, aggregation) in product(reports, report_aggregation)]) + + custom_reports = self.get_custom_reports(config, client) + streams.extend(custom_reports) return streams diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json index 613e9d2b969d..f7f8e586223e 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/spec.json @@ -4,12 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Bing Ads Spec", "type": "object", - "required": [ - "developer_token", - "client_id", - "refresh_token", - "reports_start_date" - ], + "required": ["developer_token", "client_id", "refresh_token"], "additionalProperties": true, "properties": { "auth_method": { @@ -53,22 +48,144 @@ "airbyte_secret": true, "order": 4 }, + "account_names": { + "title": "Account Names Predicates", + "description": "Predicates that will be used to sync data by specific accounts.", + "type": "array", + "order": 5, + "items": { + "description": "Account Names Predicates Config.", + "type": "object", + "properties": { + "operator": { + "title": "Operator", + "description": "An Operator that will be used to filter accounts. The Contains predicate has features for matching words, matching inflectional forms of words, searching using wildcard characters, and searching using proximity. The Equals is used to return all rows where account name is equal(=) to the string that you provided", + "type": "string", + "enum": ["Contains", "Equals"] + }, + "name": { + "title": "Account Name", + "description": "Account Name is a string value for comparing with the specified predicate.", + "type": "string" + } + }, + "required": ["operator", "name"] + } + }, "reports_start_date": { "type": "string", "title": "Reports replication start date", "format": "date", - "default": "2020-01-01", - "description": "The start date from which to begin replicating report data. Any data generated before this date will not be replicated in reports. This is a UTC date in YYYY-MM-DD format.", - "order": 5 + "description": "The start date from which to begin replicating report data. Any data generated before this date will not be replicated in reports. This is a UTC date in YYYY-MM-DD format. If not set, data from previous and current calendar year will be replicated.", + "order": 6 }, "lookback_window": { "title": "Lookback window", - "description": "Also known as attribution or conversion window. How far into the past to look for records (in days). If your conversion window has an hours/minutes granularity, round it up to the number of days exceeding. Used only for performance report streams in incremental mode.", + "description": "Also known as attribution or conversion window. How far into the past to look for records (in days). If your conversion window has an hours/minutes granularity, round it up to the number of days exceeding. Used only for performance report streams in incremental mode without specified Reports Start Date.", "type": "integer", "default": 0, "minimum": 0, "maximum": 90, - "order": 6 + "order": 7 + }, + "custom_reports": { + "title": "Custom Reports", + "description": "You can add your Custom Bing Ads report by creating one.", + "order": 8, + "type": "array", + "items": { + "title": "Custom Report Config", + "type": "object", + "properties": { + "name": { + "title": "Report Name", + "description": "The name of the custom report, this name would be used as stream name", + "type": "string", + "examples": [ + "Account Performance", + "AdDynamicTextPerformanceReport", + "custom report" + ] + }, + "reporting_object": { + "title": "Reporting Data Object", + "description": "The name of the the object derives from the ReportRequest object. You can find it in Bing Ads Api docs - Reporting API - Reporting Data Objects.", + "type": "string", + "enum": [ + "AccountPerformanceReportRequest", + "AdDynamicTextPerformanceReportRequest", + "AdExtensionByAdReportRequest", + "AdExtensionByKeywordReportRequest", + "AdExtensionDetailReportRequest", + "AdGroupPerformanceReportRequest", + "AdPerformanceReportRequest", + "AgeGenderAudienceReportRequest", + "AudiencePerformanceReportRequest", + "CallDetailReportRequest", + "CampaignPerformanceReportRequest", + "ConversionPerformanceReportRequest", + "DestinationUrlPerformanceReportRequest", + "DSAAutoTargetPerformanceReportRequest", + "DSACategoryPerformanceReportRequest", + "DSASearchQueryPerformanceReportRequest", + "GeographicPerformanceReportRequest", + "GoalsAndFunnelsReportRequest", + "HotelDimensionPerformanceReportRequest", + "HotelGroupPerformanceReportRequest", + "KeywordPerformanceReportRequest", + "NegativeKeywordConflictReportRequest", + "ProductDimensionPerformanceReportRequest", + "ProductMatchCountReportRequest", + "ProductNegativeKeywordConflictReportRequest", + "ProductPartitionPerformanceReportRequest", + "ProductPartitionUnitPerformanceReportRequest", + "ProductSearchQueryPerformanceReportRequest", + "ProfessionalDemographicsAudienceReportRequest", + "PublisherUsagePerformanceReportRequest", + "SearchCampaignChangeHistoryReportRequest", + "SearchQueryPerformanceReportRequest", + "ShareOfVoiceReportRequest", + "UserLocationPerformanceReportRequest" + ] + }, + "report_columns": { + "title": "Columns", + "description": "A list of available report object columns. You can find it in description of reporting object that you want to add to custom report.", + "type": "array", + "items": { + "description": "Name of report column.", + "type": "string" + }, + "minItems": 1 + }, + "report_aggregation": { + "title": "Aggregation", + "description": "A list of available aggregations.", + "type": "string", + "items": { + "title": "ValidEnums", + "description": "An enumeration of aggregations.", + "enum": [ + "Hourly", + "Daily", + "Weekly", + "Monthly", + "DayOfWeek", + "HourOfDay", + "WeeklyStartingMonday", + "Summary" + ] + }, + "default": ["Hourly"] + } + }, + "required": [ + "name", + "reporting_object", + "report_columns", + "report_aggregation" + ] + } } } }, diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/utils.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/utils.py new file mode 100644 index 000000000000..785b1237452a --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/utils.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from datetime import datetime, timezone + + +def transform_bulk_datetime_format_to_rfc_3339(original_value: str) -> str: + """ + Bing Ads Bulk API provides datetime fields in custom format with milliseconds: "04/27/2023 18:00:14.970" + Return datetime in RFC3339 format: "2023-04-27T18:00:14.970+00:00" + """ + return datetime.strptime(original_value, "%m/%d/%Y %H:%M:%S.%f").replace(tzinfo=timezone.utc).isoformat(timespec="milliseconds") + + +def transform_date_format_to_rfc_3339(original_value: str) -> str: + """ + Bing Ads API provides date fields in custom format: "04/27/2023" + Return date in RFC3339 format: "2023-04-27" + """ + return datetime.strptime(original_value, "%m/%d/%Y").replace(tzinfo=timezone.utc).strftime("%Y-%m-%d") + + +def transform_report_hourly_datetime_format_to_rfc_3339(original_value: str) -> str: + """ + Bing Ads API reports with hourly aggregation provides date fields in custom format: "2023-11-04|11" + Return date in RFC3339 format: "2023-11-04T11:00:00+00:00" + """ + return datetime.strptime(original_value, "%Y-%m-%d|%H").replace(tzinfo=timezone.utc).isoformat(timespec="seconds") diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/app_install_ad_labels_base.csv b/airbyte-integrations/connectors/source-bing-ads/unit_tests/app_install_ad_labels_base.csv new file mode 100644 index 000000000000..48e937103719 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/app_install_ad_labels_base.csv @@ -0,0 +1,3 @@ +Type,Status,Id,Parent Id,Campaign,Ad Group,Client Id,Modified Time,Name,Description,Label,Color +Format Version,,,,,,,,6.0,,, +App Install Ad Label,,-22,-11112,,,ClientIdGoesHere,,,,, \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/app_install_ads_base.csv b/airbyte-integrations/connectors/source-bing-ads/unit_tests/app_install_ads_base.csv new file mode 100644 index 000000000000..58de2313d389 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/app_install_ads_base.csv @@ -0,0 +1,3 @@ +Type,Status,Id,Parent Id,Campaign,Ad Group,Client Id,Modified Time,Title,Text,Display Url,Destination Url,Promotion,Device Preference,Name,App Platform,App Id,Final Url,Mobile Final Url,Tracking Template,Final Url Suffix,Custom Parameter +Format Version,,,,,,,,,,,,,,6.0,,,,,,, +App Install Ad,Active,,-1111,ParentCampaignNameGoesHere,AdGroupNameGoesHere,ClientIdGoesHere,,Contoso Quick Setup,Find New Customers & Increase Sales!,,,,All,,Android,AppStoreIdGoesHere,FinalUrlGoesHere,,,,{_promoCode}=PROMO1; {_season}=summer \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/conftest.py new file mode 100644 index 000000000000..4ab93cea5958 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/conftest.py @@ -0,0 +1,75 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from unittest.mock import patch + +import pytest + + +@pytest.fixture(name="config") +def config_fixture(): + """Generates streams settings from a config file""" + return { + "tenant_id": "common", + "developer_token": "fake_developer_token", + "refresh_token": "fake_refresh_token", + "client_id": "fake_client_id", + "reports_start_date": "2020-01-01", + "lookback_window": 0, + } + +@pytest.fixture(name="config_with_account_names") +def config_with_account_names_fixture(): + """Generates streams settings from a config file""" + return { + "tenant_id": "common", + "developer_token": "fake_developer_token", + "refresh_token": "fake_refresh_token", + "client_id": "fake_client_id", + "reports_start_date": "2020-01-01", + "account_names": [{"operator": "Equals", "name": "airbyte"}, {"operator": "Contains", "name": "demo"}], + "lookback_window": 0, + } + + +@pytest.fixture(name="config_with_custom_reports") +def config_with_custom_reports_fixture(): + """Generates streams settings with custom reports from a config file""" + return { + "tenant_id": "common", + "developer_token": "fake_developer_token", + "refresh_token": "fake_refresh_token", + "client_id": "fake_client_id", + "reports_start_date": "2020-01-01", + "lookback_window": 0, + "custom_reports": [ + { + "name": "my test custom report", + "reporting_object": "DSAAutoTargetPerformanceReport", + "report_columns": [ + "AbsoluteTopImpressionRatePercent", + "AccountId", + "AccountName", + "AccountNumber", + "AccountStatus", + "AdDistribution", + "AdGroupId", + "AdGroupName", + "AdGroupStatus", + "AdId", + "AllConversionRate", + "AllConversions", + "AllConversionsQualified", + "AllCostPerConversion", + "AllReturnOnAdSpend", + "AllRevenue", + ], + "report_aggregation": "Weekly", + } + ], + } + + +@pytest.fixture(name="logger_mock") +def logger_mock_fixture(): + return patch("source_bing_ads.source.AirbyteLogger") diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/account_impression_performance.csv b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/account_impression_performance.csv new file mode 100644 index 000000000000..614529a0e45c --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/account_impression_performance.csv @@ -0,0 +1,2 @@ +AccountName,AccountNumber,AccountId,TimePeriod,CurrencyCode,AdDistribution,Impressions,Clicks,Ctr,AverageCpc,Spend,AveragePosition,Conversions,ConversionRate,CostPerConversion,LowQualityClicks,LowQualityClicksPercent,LowQualityImpressions,LowQualityImpressionsPercent,LowQualityConversions,LowQualityConversionRate,DeviceType,PhoneImpressions,PhoneCalls,Ptr,Network,Assists,Revenue,ReturnOnAdSpend,CostPerAssist,RevenuePerConversion,RevenuePerAssist,AccountStatus,LowQualityGeneralClicks,LowQualitySophisticatedClicks,TopImpressionRatePercent,AllConversions,AllRevenue,AllConversionRate,AllCostPerConversion,AllReturnOnAdSpend,AllRevenuePerConversion,ViewThroughConversions,AverageCpm,ConversionsQualified,LowQualityConversionsQualified,AllConversionsQualified,ViewThroughConversionsQualified,ViewThroughRevenue,VideoViews,ViewThroughRate,AverageCPV,VideoViewsAt25Percent,VideoViewsAt50Percent,VideoViewsAt75Percent,CompletedVideoViews,VideoCompletionRate,TotalWatchTimeInMS,AverageWatchTimePerVideoView,AverageWatchTimePerImpression,Sales,CostPerSale,RevenuePerSale,Installs,CostPerInstall,RevenuePerInstall +Airbyte,F149MJ18,180519267,2023-11-11T01:00:00+00:00,USD,Search,0,0,,0,0,0,0,,,0,,1,100,0,,Computer,0,0,,AOL search,0,0,,,,,Active,0,0,,0,0,,,,,0,0,0,0,0,,0,0,,,0,0,0,0,,0,,,0,,,0,, diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/account_impression_performance_records.json b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/account_impression_performance_records.json new file mode 100644 index 000000000000..ca1243eebf0b --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/account_impression_performance_records.json @@ -0,0 +1,70 @@ +[ + { + "AccountId": "180519267", + "AccountName": "Airbyte", + "AccountNumber": "F149MJ18", + "AccountStatus": "Active", + "AdDistribution": "Search", + "AllConversionRate": null, + "AllConversions": "0", + "AllConversionsQualified": "0", + "AllCostPerConversion": null, + "AllReturnOnAdSpend": null, + "AllRevenue": "0", + "AllRevenuePerConversion": null, + "Assists": "0", + "AverageCPV": null, + "AverageCpc": "0", + "AverageCpm": "0", + "AveragePosition": "0", + "AverageWatchTimePerImpression": null, + "AverageWatchTimePerVideoView": null, + "Clicks": "0", + "CompletedVideoViews": "0", + "ConversionRate": null, + "Conversions": "0", + "ConversionsQualified": "0", + "CostPerAssist": null, + "CostPerConversion": null, + "CostPerInstall": null, + "CostPerSale": null, + "Ctr": null, + "CurrencyCode": "USD", + "DeviceType": "Computer", + "Impressions": "0", + "Installs": "0", + "LowQualityClicks": "0", + "LowQualityClicksPercent": null, + "LowQualityConversionRate": null, + "LowQualityConversions": "0", + "LowQualityConversionsQualified": "0", + "LowQualityGeneralClicks": "0", + "LowQualityImpressions": "1", + "LowQualityImpressionsPercent": "100", + "LowQualitySophisticatedClicks": "0", + "Network": "AOL search", + "PhoneCalls": "0", + "PhoneImpressions": "0", + "Ptr": null, + "ReturnOnAdSpend": null, + "Revenue": "0", + "RevenuePerAssist": null, + "RevenuePerConversion": null, + "RevenuePerInstall": null, + "RevenuePerSale": null, + "Sales": "0", + "Spend": "0", + "TimePeriod": "2023-11-11T01:00:00+00:00", + "TopImpressionRatePercent": null, + "TotalWatchTimeInMS": "0", + "VideoCompletionRate": null, + "VideoViews": "0", + "VideoViewsAt25Percent": "0", + "VideoViewsAt50Percent": "0", + "VideoViewsAt75Percent": "0", + "ViewThroughConversions": "0", + "ViewThroughConversionsQualified": null, + "ViewThroughRate": null, + "ViewThroughRevenue": "0" + } +] diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/account_performance.csv b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/account_performance.csv new file mode 100644 index 000000000000..5f9456369f4b --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/account_performance.csv @@ -0,0 +1,2 @@ +AccountId,TimePeriod,CurrencyCode,AdDistribution,DeviceType,Network,DeliveredMatchType,DeviceOS,TopVsOther,BidMatchType,AccountName,AccountNumber,PhoneImpressions,PhoneCalls,Clicks,Ctr,Spend,Impressions,CostPerConversion,Ptr,Assists,ReturnOnAdSpend,CostPerAssist,AverageCpc,AveragePosition,AverageCpm,Conversions,ConversionsQualified,ConversionRate,LowQualityClicks,LowQualityClicksPercent,LowQualityImpressions,LowQualitySophisticatedClicks,LowQualityConversions,LowQualityConversionRate,Revenue,RevenuePerConversion,RevenuePerAssist +180519267,2023-11-08T00:00:00+00:00,USD,Search,Smartphone,Syndicated search partners,Phrase,Android,Syndicated search partners - Other,Broad,Airbyte,F149MJ18,0,0,0,,0,0,,,0,,,0,0,0,0,0,,0,,11,0,0,,0,, diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/account_performance_records.json b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/account_performance_records.json new file mode 100644 index 000000000000..306d42ab60db --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/account_performance_records.json @@ -0,0 +1,42 @@ +[ + { + "AccountId": "180519267", + "AccountName": "Airbyte", + "AccountNumber": "F149MJ18", + "AdDistribution": "Search", + "Assists": "0", + "AverageCpc": "0", + "AverageCpm": "0", + "AveragePosition": "0", + "BidMatchType": "Broad", + "Clicks": "0", + "ConversionRate": null, + "Conversions": "0", + "ConversionsQualified": "0", + "CostPerAssist": null, + "CostPerConversion": null, + "Ctr": null, + "CurrencyCode": "USD", + "DeliveredMatchType": "Phrase", + "DeviceOS": "Android", + "DeviceType": "Smartphone", + "Impressions": "0", + "LowQualityClicks": "0", + "LowQualityClicksPercent": null, + "LowQualityConversionRate": null, + "LowQualityConversions": "0", + "LowQualityImpressions": "11", + "LowQualitySophisticatedClicks": "0", + "Network": "Syndicated search partners", + "PhoneCalls": "0", + "PhoneImpressions": "0", + "Ptr": null, + "ReturnOnAdSpend": null, + "Revenue": "0", + "RevenuePerAssist": null, + "RevenuePerConversion": null, + "Spend": "0", + "TimePeriod": "2023-11-08T00:00:00+00:00", + "TopVsOther": "Syndicated search partners - Other" + } +] diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/ad_group_impression_performance.csv b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/ad_group_impression_performance.csv new file mode 100644 index 000000000000..3fc35b23a387 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/ad_group_impression_performance.csv @@ -0,0 +1,2 @@ +AccountName,AccountNumber,AccountId,TimePeriod,Status,CampaignName,CampaignId,AdGroupName,AdGroupId,CurrencyCode,AdDistribution,Impressions,Clicks,Ctr,AverageCpc,Spend,AveragePosition,Conversions,ConversionRate,CostPerConversion,DeviceType,Language,QualityScore,ExpectedCtr,AdRelevance,LandingPageExperience,PhoneImpressions,PhoneCalls,Ptr,Network,Assists,Revenue,ReturnOnAdSpend,CostPerAssist,RevenuePerConversion,RevenuePerAssist,TrackingTemplate,CustomParameters,AccountStatus,CampaignStatus,AdGroupLabels,FinalUrlSuffix,CampaignType,TopImpressionSharePercent,AbsoluteTopImpressionRatePercent,TopImpressionRatePercent,BaseCampaignId,AllConversions,AllRevenue,AllConversionRate,AllCostPerConversion,AllReturnOnAdSpend,AllRevenuePerConversion,ViewThroughConversions,AdGroupType,AverageCpm,ConversionsQualified,AllConversionsQualified,ViewThroughConversionsQualified,ViewThroughRevenue,VideoViews,ViewThroughRate,AverageCPV,VideoViewsAt25Percent,VideoViewsAt50Percent,VideoViewsAt75Percent,CompletedVideoViews,VideoCompletionRate,TotalWatchTimeInMS,AverageWatchTimePerVideoView,AverageWatchTimePerImpression,Sales,CostPerSale,RevenuePerSale,Installs,CostPerInstall,RevenuePerInstall +Airbyte,F149MJ18,180519267,2023-11-11T10:00:00+00:00,Active,Airbyte test,531016227,keywords,1356799861840328,USD,Search,2,0,0,0,0,0,0,,,Smartphone,Spanish,6,2,3,1,0,0,,Syndicated search partners,0,0,,,,,,,Active,Active,,,Search & content,,0,100,531016227,0,0,,,,,0,Standard,0,0,0,,0,0,0,,0,0,0,0,,0,,0,0,,,0,, diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/ad_group_impression_performance_records.json b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/ad_group_impression_performance_records.json new file mode 100644 index 000000000000..1fcc3ed4e7b4 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/ad_group_impression_performance_records.json @@ -0,0 +1,81 @@ +[ + { + "AbsoluteTopImpressionRatePercent": "0", + "AccountId": "180519267", + "AccountName": "Airbyte", + "AccountNumber": "F149MJ18", + "AccountStatus": "Active", + "AdDistribution": "Search", + "AdGroupId": "1356799861840328", + "AdGroupLabels": null, + "AdGroupName": "keywords", + "AdGroupType": "Standard", + "AdRelevance": "3", + "AllConversionRate": null, + "AllConversions": "0", + "AllConversionsQualified": "0", + "AllCostPerConversion": null, + "AllReturnOnAdSpend": null, + "AllRevenue": "0", + "AllRevenuePerConversion": null, + "Assists": "0", + "AverageCPV": null, + "AverageCpc": "0", + "AverageCpm": "0", + "AveragePosition": "0", + "AverageWatchTimePerImpression": "0", + "AverageWatchTimePerVideoView": null, + "BaseCampaignId": "531016227", + "CampaignId": "531016227", + "CampaignName": "Airbyte test", + "CampaignStatus": "Active", + "CampaignType": "Search & content", + "Clicks": "0", + "CompletedVideoViews": "0", + "ConversionRate": null, + "Conversions": "0", + "ConversionsQualified": "0", + "CostPerAssist": null, + "CostPerConversion": null, + "CostPerInstall": null, + "CostPerSale": null, + "Ctr": "0", + "CurrencyCode": "USD", + "CustomParameters": null, + "DeviceType": "Smartphone", + "ExpectedCtr": "2", + "FinalUrlSuffix": null, + "Impressions": "2", + "Installs": "0", + "LandingPageExperience": "1", + "Language": "Spanish", + "Network": "Syndicated search partners", + "PhoneCalls": "0", + "PhoneImpressions": "0", + "Ptr": null, + "QualityScore": "6", + "ReturnOnAdSpend": null, + "Revenue": "0", + "RevenuePerAssist": null, + "RevenuePerConversion": null, + "RevenuePerInstall": null, + "RevenuePerSale": null, + "Sales": "0", + "Spend": "0", + "Status": "Active", + "TimePeriod": "2023-11-11T10:00:00+00:00", + "TopImpressionRatePercent": "100", + "TopImpressionSharePercent": null, + "TotalWatchTimeInMS": "0", + "TrackingTemplate": null, + "VideoCompletionRate": null, + "VideoViews": "0", + "VideoViewsAt25Percent": "0", + "VideoViewsAt50Percent": "0", + "VideoViewsAt75Percent": "0", + "ViewThroughConversions": "0", + "ViewThroughConversionsQualified": null, + "ViewThroughRate": "0", + "ViewThroughRevenue": "0" + } +] diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/ad_group_performance.csv b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/ad_group_performance.csv new file mode 100644 index 000000000000..41dc72ca4d99 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/ad_group_performance.csv @@ -0,0 +1,2 @@ +AccountId,CampaignId,AdGroupId,TimePeriod,CurrencyCode,AdDistribution,DeviceType,Network,DeliveredMatchType,DeviceOS,TopVsOther,BidMatchType,Language,AccountName,CampaignName,CampaignType,AdGroupName,AdGroupType,Impressions,Clicks,Ctr,Spend,CostPerConversion,QualityScore,ExpectedCtr,AdRelevance,LandingPageExperience,PhoneImpressions,PhoneCalls,Ptr,Assists,CostPerAssist,CustomParameters,FinalUrlSuffix,ViewThroughConversions,AllCostPerConversion,AllReturnOnAdSpend,AllConversions,AllConversionRate,AllRevenue,AllRevenuePerConversion,AverageCpc,AveragePosition,AverageCpm,Conversions,ConversionRate,ConversionsQualified,Revenue,RevenuePerConversion,RevenuePerAssist +180519267,531016227,1356799861840328,2023-11-11T05:00:00+00:00,USD,Search,Computer,Syndicated search partners,Phrase,Unknown,Syndicated search partners - Top,Broad,Portuguese,Airbyte,Airbyte test,Search & content,keywords,Standard,1,0,0,0,,6,2,3,1,0,0,,0,,,,0,,,0,,0,,0,0,0,0,,0,0,, diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/ad_group_performance_records.json b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/ad_group_performance_records.json new file mode 100644 index 000000000000..af9834cffffd --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/ad_group_performance_records.json @@ -0,0 +1,54 @@ +[ + { + "AccountId": "180519267", + "AccountName": "Airbyte", + "AdDistribution": "Search", + "AdGroupId": "1356799861840328", + "AdGroupName": "keywords", + "AdGroupType": "Standard", + "AdRelevance": "3", + "AllConversionRate": null, + "AllConversions": "0", + "AllCostPerConversion": null, + "AllReturnOnAdSpend": null, + "AllRevenue": "0", + "AllRevenuePerConversion": null, + "Assists": "0", + "AverageCpc": "0", + "AverageCpm": "0", + "AveragePosition": "0", + "BidMatchType": "Broad", + "CampaignId": "531016227", + "CampaignName": "Airbyte test", + "CampaignType": "Search & content", + "Clicks": "0", + "ConversionRate": null, + "Conversions": "0", + "ConversionsQualified": "0", + "CostPerAssist": null, + "CostPerConversion": null, + "Ctr": "0", + "CurrencyCode": "USD", + "CustomParameters": null, + "DeliveredMatchType": "Phrase", + "DeviceOS": "Unknown", + "DeviceType": "Computer", + "ExpectedCtr": "2", + "FinalUrlSuffix": null, + "Impressions": "1", + "LandingPageExperience": "1", + "Language": "Portuguese", + "Network": "Syndicated search partners", + "PhoneCalls": "0", + "PhoneImpressions": "0", + "Ptr": null, + "QualityScore": "6", + "Revenue": "0", + "RevenuePerAssist": null, + "RevenuePerConversion": null, + "Spend": "0", + "TimePeriod": "2023-11-11T05:00:00+00:00", + "TopVsOther": "Syndicated search partners - Top", + "ViewThroughConversions": "0" + } +] diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/ad_performance.csv b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/ad_performance.csv new file mode 100644 index 000000000000..6027fb1bdfbc --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/ad_performance.csv @@ -0,0 +1,2 @@ +AccountId,CampaignId,AdGroupId,AdId,TimePeriod,CurrencyCode,AdDistribution,DeviceType,Language,Network,DeviceOS,TopVsOther,BidMatchType,DeliveredMatchType,AccountName,CampaignName,CampaignType,AdGroupName,Impressions,Clicks,Ctr,Spend,CostPerConversion,DestinationUrl,Assists,ReturnOnAdSpend,CostPerAssist,CustomParameters,FinalAppUrl,AdDescription,AdDescription2,ViewThroughConversions,ViewThroughConversionsQualified,AllCostPerConversion,AllReturnOnAdSpend,Conversions,ConversionRate,ConversionsQualified,AverageCpc,AveragePosition,AverageCpm,AllConversions,AllConversionRate,AllRevenue,AllRevenuePerConversion,Revenue,RevenuePerConversion,RevenuePerAssist +180519267,531016227,1356799861840328,84800390693061,2023-11-08T00:00:00+00:00,USD,Search,Tablet,English,Microsoft sites and select traffic,Android,Microsoft sites and select traffic - top,Broad,Phrase,Airbyte,Airbyte test,Search & content,keywords,2,0,0,0,,,0,,,,,,,0,,,,0,,0,0,0,0,0,,0,,0,, diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/ad_performance_records.json b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/ad_performance_records.json new file mode 100644 index 000000000000..89902ccdb270 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/ad_performance_records.json @@ -0,0 +1,52 @@ +[ + { + "AccountId": "180519267", + "AccountName": "Airbyte", + "AdDescription": null, + "AdDescription2": null, + "AdDistribution": "Search", + "AdGroupId": "1356799861840328", + "AdGroupName": "keywords", + "AdId": "84800390693061", + "AllConversionRate": null, + "AllConversions": "0", + "AllCostPerConversion": null, + "AllReturnOnAdSpend": null, + "AllRevenue": "0", + "AllRevenuePerConversion": null, + "Assists": "0", + "AverageCpc": "0", + "AverageCpm": "0", + "AveragePosition": "0", + "BidMatchType": "Broad", + "CampaignId": "531016227", + "CampaignName": "Airbyte test", + "CampaignType": "Search & content", + "Clicks": "0", + "ConversionRate": null, + "Conversions": "0", + "ConversionsQualified": "0", + "CostPerAssist": null, + "CostPerConversion": null, + "Ctr": "0", + "CurrencyCode": "USD", + "CustomParameters": null, + "DeliveredMatchType": "Phrase", + "DestinationUrl": null, + "DeviceOS": "Android", + "DeviceType": "Tablet", + "FinalAppUrl": null, + "Impressions": "2", + "Language": "English", + "Network": "Microsoft sites and select traffic", + "ReturnOnAdSpend": null, + "Revenue": "0", + "RevenuePerAssist": null, + "RevenuePerConversion": null, + "Spend": "0", + "TimePeriod": "2023-11-08T00:00:00+00:00", + "TopVsOther": "Microsoft sites and select traffic - top", + "ViewThroughConversions": "0", + "ViewThroughConversionsQualified": null + } +] diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/age_gender_audience.csv b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/age_gender_audience.csv new file mode 100644 index 000000000000..2b8a271cc54e --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/age_gender_audience.csv @@ -0,0 +1,2 @@ +AccountId,AgeGroup,Gender,TimePeriod,AllConversions,AccountName,AccountNumber,CampaignName,CampaignId,AdGroupName,AdGroupId,AdDistribution,Impressions,Clicks,Conversions,Spend,Revenue,ExtendedCost,Assists,Language,AccountStatus,CampaignStatus,AdGroupStatus,BaseCampaignId,AllRevenue,ViewThroughConversions,Goal,GoalType,AbsoluteTopImpressionRatePercent,TopImpressionRatePercent,ConversionsQualified,AllConversionsQualified,ViewThroughConversionsQualified,ViewThroughRevenue +180519267,Unknown,Unknown,2023-11-14T04:00:00+00:00,0,Airbyte,F149MJ18,Airbyte test,531016227,keywords,1356799861840328,Search,2,0,0,0,0,0,0,German,Active,Active,Active,531016227,0,0,,,100,100,0,0,,0 diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/age_gender_audience_records.json b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/age_gender_audience_records.json new file mode 100644 index 000000000000..95b460108d38 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/age_gender_audience_records.json @@ -0,0 +1,38 @@ +[ + { + "AbsoluteTopImpressionRatePercent": "100", + "AccountId": "180519267", + "AccountName": "Airbyte", + "AccountNumber": "F149MJ18", + "AccountStatus": "Active", + "AdDistribution": "Search", + "AdGroupId": "1356799861840328", + "AdGroupName": "keywords", + "AdGroupStatus": "Active", + "AgeGroup": "Unknown", + "AllConversions": "0", + "AllConversionsQualified": "0", + "AllRevenue": "0", + "Assists": "0", + "BaseCampaignId": "531016227", + "CampaignId": "531016227", + "CampaignName": "Airbyte test", + "CampaignStatus": "Active", + "Clicks": "0", + "Conversions": "0", + "ConversionsQualified": "0", + "ExtendedCost": "0", + "Gender": "Unknown", + "Goal": null, + "GoalType": null, + "Impressions": "2", + "Language": "German", + "Revenue": "0", + "Spend": "0", + "TimePeriod": "2023-11-14T04:00:00+00:00", + "TopImpressionRatePercent": "100", + "ViewThroughConversions": "0", + "ViewThroughConversionsQualified": null, + "ViewThroughRevenue": "0" + } +] diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/campaign_impression_performance.csv b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/campaign_impression_performance.csv new file mode 100644 index 000000000000..50815fcac5d2 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/campaign_impression_performance.csv @@ -0,0 +1,2 @@ +AccountName,AccountNumber,AccountId,TimePeriod,CampaignStatus,CampaignName,CampaignId,CurrencyCode,AdDistribution,Impressions,Clicks,Ctr,AverageCpc,Spend,AveragePosition,Conversions,ConversionRate,CostPerConversion,LowQualityClicks,LowQualityClicksPercent,LowQualityImpressions,LowQualityImpressionsPercent,LowQualityConversions,LowQualityConversionRate,DeviceType,QualityScore,ExpectedCtr,AdRelevance,LandingPageExperience,PhoneImpressions,PhoneCalls,Ptr,Network,Assists,Revenue,ReturnOnAdSpend,CostPerAssist,RevenuePerConversion,RevenuePerAssist,TrackingTemplate,CustomParameters,AccountStatus,LowQualityGeneralClicks,LowQualitySophisticatedClicks,CampaignLabels,FinalUrlSuffix,CampaignType,AbsoluteTopImpressionRatePercent,TopImpressionRatePercent,BaseCampaignId,AllConversions,AllRevenue,AllConversionRate,AllCostPerConversion,AllReturnOnAdSpend,AllRevenuePerConversion,ViewThroughConversions,AverageCpm,ConversionsQualified,LowQualityConversionsQualified,AllConversionsQualified,ViewThroughConversionsQualified,ViewThroughRevenue,VideoViews,ViewThroughRate,AverageCPV,VideoViewsAt25Percent,VideoViewsAt50Percent,VideoViewsAt75Percent,CompletedVideoViews,VideoCompletionRate,TotalWatchTimeInMS,AverageWatchTimePerVideoView,AverageWatchTimePerImpression,Sales,CostPerSale,RevenuePerSale,Installs,CostPerInstall,RevenuePerInstall +Airbyte,F149MJ18,180519267,2023-11-10T04:00:00+00:00,Active,Airbyte test,531016227,USD,Search,0,0,,0,0,0,0,,,0,,1,100,0,,Smartphone,6,2,3,1,0,0,,Syndicated search partners,0,0,,,,,,,Active,0,0,,,Search & content,,,531016227,0,0,,,,,0,0,0,0,0,,0,0,,,0,0,0,0,,0,,,0,,,0,, diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/campaign_impression_performance_records.json b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/campaign_impression_performance_records.json new file mode 100644 index 000000000000..8ffdae4ce8f7 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/campaign_impression_performance_records.json @@ -0,0 +1,84 @@ +[ + { + "AbsoluteTopImpressionRatePercent": null, + "AccountId": "180519267", + "AccountName": "Airbyte", + "AccountNumber": "F149MJ18", + "AccountStatus": "Active", + "AdDistribution": "Search", + "AdRelevance": "3", + "AllConversionRate": null, + "AllConversions": "0", + "AllConversionsQualified": "0", + "AllCostPerConversion": null, + "AllReturnOnAdSpend": null, + "AllRevenue": "0", + "AllRevenuePerConversion": null, + "Assists": "0", + "AverageCPV": null, + "AverageCpc": "0", + "AverageCpm": "0", + "AveragePosition": "0", + "AverageWatchTimePerImpression": null, + "AverageWatchTimePerVideoView": null, + "BaseCampaignId": "531016227", + "CampaignId": "531016227", + "CampaignLabels": null, + "CampaignName": "Airbyte test", + "CampaignStatus": "Active", + "CampaignType": "Search & content", + "Clicks": "0", + "CompletedVideoViews": "0", + "ConversionRate": null, + "Conversions": "0", + "ConversionsQualified": "0", + "CostPerAssist": null, + "CostPerConversion": null, + "CostPerInstall": null, + "CostPerSale": null, + "Ctr": null, + "CurrencyCode": "USD", + "CustomParameters": null, + "DeviceType": "Smartphone", + "ExpectedCtr": "2", + "FinalUrlSuffix": null, + "Impressions": "0", + "Installs": "0", + "LandingPageExperience": "1", + "LowQualityClicks": "0", + "LowQualityClicksPercent": null, + "LowQualityConversionRate": null, + "LowQualityConversions": "0", + "LowQualityConversionsQualified": "0", + "LowQualityGeneralClicks": "0", + "LowQualityImpressions": "1", + "LowQualityImpressionsPercent": "100", + "LowQualitySophisticatedClicks": "0", + "Network": "Syndicated search partners", + "PhoneCalls": "0", + "PhoneImpressions": "0", + "Ptr": null, + "QualityScore": "6", + "ReturnOnAdSpend": null, + "Revenue": "0", + "RevenuePerAssist": null, + "RevenuePerConversion": null, + "RevenuePerInstall": null, + "RevenuePerSale": null, + "Sales": "0", + "Spend": "0", + "TimePeriod": "2023-11-10T04:00:00+00:00", + "TopImpressionRatePercent": null, + "TotalWatchTimeInMS": "0", + "TrackingTemplate": null, + "VideoCompletionRate": null, + "VideoViews": "0", + "VideoViewsAt25Percent": "0", + "VideoViewsAt50Percent": "0", + "VideoViewsAt75Percent": "0", + "ViewThroughConversions": "0", + "ViewThroughConversionsQualified": null, + "ViewThroughRate": null, + "ViewThroughRevenue": "0" + } +] diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/campaign_performance.csv b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/campaign_performance.csv new file mode 100644 index 000000000000..511b8ea17d09 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/campaign_performance.csv @@ -0,0 +1,2 @@ +AccountId,CampaignId,TimePeriod,CurrencyCode,AdDistribution,DeviceType,Network,DeliveredMatchType,DeviceOS,TopVsOther,BidMatchType,AccountName,CampaignName,CampaignType,CampaignStatus,CampaignLabels,Impressions,Clicks,Ctr,Spend,CostPerConversion,QualityScore,AdRelevance,LandingPageExperience,PhoneImpressions,PhoneCalls,Ptr,Assists,ReturnOnAdSpend,CostPerAssist,CustomParameters,ViewThroughConversions,AllCostPerConversion,AllReturnOnAdSpend,AllConversions,ConversionsQualified,AllConversionRate,AllRevenue,AllRevenuePerConversion,AverageCpc,AveragePosition,AverageCpm,Conversions,ConversionRate,LowQualityClicks,LowQualityClicksPercent,LowQualityImpressions,LowQualitySophisticatedClicks,LowQualityConversions,LowQualityConversionRate,Revenue,RevenuePerConversion,RevenuePerAssist,BudgetName,BudgetStatus,BudgetAssociationStatus +180519267,531016227,2023-11-11T16:00:00+00:00,USD,Search,Computer,Microsoft sites and select traffic,Phrase,Windows,Microsoft sites and select traffic - other,Broad,Airbyte,Airbyte test,Search & content,Active,,1,0,0,0,,6,3,1,0,0,,0,,,,0,,,0,0,,0,,0,0,0,0,,0,,4,0,0,,0,,,,,Current diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/campaign_performance_records.json b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/campaign_performance_records.json new file mode 100644 index 000000000000..2d781b1185a8 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/campaign_performance_records.json @@ -0,0 +1,60 @@ +[ + { + "AccountId": "180519267", + "AccountName": "Airbyte", + "AdDistribution": "Search", + "AdRelevance": "3", + "AllConversionRate": null, + "AllConversions": "0", + "AllCostPerConversion": null, + "AllReturnOnAdSpend": null, + "AllRevenue": "0", + "AllRevenuePerConversion": null, + "Assists": "0", + "AverageCpc": "0", + "AverageCpm": "0", + "AveragePosition": "0", + "BidMatchType": "Broad", + "BudgetAssociationStatus": "Current", + "BudgetName": null, + "BudgetStatus": null, + "CampaignId": "531016227", + "CampaignLabels": null, + "CampaignName": "Airbyte test", + "CampaignStatus": "Active", + "CampaignType": "Search & content", + "Clicks": "0", + "ConversionRate": null, + "Conversions": "0", + "ConversionsQualified": "0", + "CostPerAssist": null, + "CostPerConversion": null, + "Ctr": "0", + "CurrencyCode": "USD", + "CustomParameters": null, + "DeliveredMatchType": "Phrase", + "DeviceOS": "Windows", + "DeviceType": "Computer", + "Impressions": "1", + "LandingPageExperience": "1", + "LowQualityClicks": "0", + "LowQualityClicksPercent": null, + "LowQualityConversionRate": null, + "LowQualityConversions": "0", + "LowQualityImpressions": "4", + "LowQualitySophisticatedClicks": "0", + "Network": "Microsoft sites and select traffic", + "PhoneCalls": "0", + "PhoneImpressions": "0", + "Ptr": null, + "QualityScore": "6", + "ReturnOnAdSpend": null, + "Revenue": "0", + "RevenuePerAssist": null, + "RevenuePerConversion": null, + "Spend": "0", + "TimePeriod": "2023-11-11T16:00:00+00:00", + "TopVsOther": "Microsoft sites and select traffic - other", + "ViewThroughConversions": "0" + } +] diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/geographic_performance.csv b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/geographic_performance.csv new file mode 100644 index 000000000000..8bf733bfff10 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/geographic_performance.csv @@ -0,0 +1,2 @@ +AccountId,CampaignId,AdGroupId,TimePeriod,AccountNumber,Country,State,MetroArea,City,ProximityTargetLocation,Radius,LocationType,MostSpecificLocation,AccountStatus,CampaignStatus,AdGroupStatus,County,PostalCode,LocationId,BaseCampaignId,Goal,GoalType,AbsoluteTopImpressionRatePercent,TopImpressionRatePercent,AllConversionsQualified,Neighborhood,ViewThroughRevenue,CampaignType,AssetGroupId,AssetGroupName,AssetGroupStatus,CurrencyCode,DeliveredMatchType,AdDistribution,DeviceType,Language,Network,DeviceOS,TopVsOther,BidMatchType,AccountName,CampaignName,AdGroupName,Impressions,Clicks,Ctr,Spend,CostPerConversion,Assists,ReturnOnAdSpend,CostPerAssist,ViewThroughConversions,ViewThroughConversionsQualified,AllCostPerConversion,AllReturnOnAdSpend,Conversions,ConversionRate,ConversionsQualified,AverageCpc,AveragePosition,AverageCpm,AllConversions,AllConversionRate,AllRevenue,AllRevenuePerConversion,Revenue,RevenuePerConversion,RevenuePerAssist +180519267,531016227,1356799861840328,2023-11-07T08:00:00+00:00,F149MJ18,United States,Illinois,"Champaign & Springfield-Decatur, IL",Riverton,,0,Physical location,62561,Active,Active,Active,Sangamon County,62561,94028,531016227,,,100,100.00,0.00,,0.00,Search & content,,,,USD,Phrase,Search,Computer,English,Microsoft sites and select traffic,Unknown,Microsoft sites and select traffic - top,Broad,Airbyte,Airbyte test,keywords,1,0,0,0,,0,,,0,,,,0,,0,0,0,0,0,,0,,0,, diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/geographic_performance_records.json b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/geographic_performance_records.json new file mode 100644 index 000000000000..8ade486df624 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/geographic_performance_records.json @@ -0,0 +1,72 @@ +[ + { + "AbsoluteTopImpressionRatePercent": "100", + "AccountId": "180519267", + "AccountName": "Airbyte", + "AccountNumber": "F149MJ18", + "AccountStatus": "Active", + "AdDistribution": "Search", + "AdGroupId": "1356799861840328", + "AdGroupName": "keywords", + "AdGroupStatus": "Active", + "AllConversionRate": null, + "AllConversions": "0", + "AllConversionsQualified": "0.00", + "AllCostPerConversion": null, + "AllReturnOnAdSpend": null, + "AllRevenue": "0", + "AllRevenuePerConversion": null, + "AssetGroupId": null, + "AssetGroupName": null, + "AssetGroupStatus": null, + "Assists": "0", + "AverageCpc": "0", + "AverageCpm": "0", + "AveragePosition": "0", + "BaseCampaignId": "531016227", + "BidMatchType": "Broad", + "CampaignId": "531016227", + "CampaignName": "Airbyte test", + "CampaignStatus": "Active", + "CampaignType": "Search & content", + "City": "Riverton", + "Clicks": "0", + "ConversionRate": null, + "Conversions": "0", + "ConversionsQualified": "0", + "CostPerAssist": null, + "CostPerConversion": null, + "Country": "United States", + "County": "Sangamon County", + "Ctr": "0", + "CurrencyCode": "USD", + "DeliveredMatchType": "Phrase", + "DeviceOS": "Unknown", + "DeviceType": "Computer", + "Goal": null, + "GoalType": null, + "Impressions": "1", + "Language": "English", + "LocationId": "94028", + "LocationType": "Physical location", + "MetroArea": "Champaign & Springfield-Decatur, IL", + "MostSpecificLocation": "62561", + "Neighborhood": null, + "Network": "Microsoft sites and select traffic", + "PostalCode": "62561", + "ProximityTargetLocation": null, + "Radius": "0", + "ReturnOnAdSpend": null, + "Revenue": "0", + "RevenuePerAssist": null, + "RevenuePerConversion": null, + "Spend": "0", + "State": "Illinois", + "TimePeriod": "2023-11-07T08:00:00+00:00", + "TopImpressionRatePercent": "100.00", + "TopVsOther": "Microsoft sites and select traffic - top", + "ViewThroughConversions": "0", + "ViewThroughConversionsQualified": null, + "ViewThroughRevenue": "0.00" + } +] diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/keyword_performance.csv b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/keyword_performance.csv new file mode 100644 index 000000000000..885d0b42770a --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/keyword_performance.csv @@ -0,0 +1,2 @@ +AccountId,CampaignId,AdGroupId,KeywordId,Keyword,AdId,TimePeriod,CurrencyCode,DeliveredMatchType,AdDistribution,DeviceType,Language,Network,DeviceOS,TopVsOther,BidMatchType,AccountName,CampaignName,AdGroupName,KeywordStatus,Impressions,Clicks,Ctr,CurrentMaxCpc,Spend,CostPerConversion,QualityScore,ExpectedCtr,AdRelevance,LandingPageExperience,QualityImpact,Assists,ReturnOnAdSpend,CostPerAssist,CustomParameters,FinalAppUrl,Mainline1Bid,MainlineBid,FirstPageBid,FinalUrlSuffix,ViewThroughConversions,ViewThroughConversionsQualified,AllCostPerConversion,AllReturnOnAdSpend,Conversions,ConversionRate,ConversionsQualified,AverageCpc,AveragePosition,AverageCpm,AllConversions,AllConversionRate,AllRevenue,AllRevenuePerConversion,Revenue,RevenuePerConversion,RevenuePerAssist +180519267,531016227,1356799861840328,84801135055365,connector,84800390693061,2023-11-10T00:00:00+00:00,USD,Phrase,Search,Smartphone,German,Syndicated search partners,Android,Syndicated search partners - Top,Broad,Airbyte,Airbyte test,keywords,Active,1,0,0,2.27,0,,5,2,3,1,0,0,,,,,,1.11,0.35,,0,,,,0,,0,0,0,0,0,,0,,0,, diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/keyword_performance_records.json b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/keyword_performance_records.json new file mode 100644 index 000000000000..a16a478b8354 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/keyword_performance_records.json @@ -0,0 +1,61 @@ +[ + { + "AccountId": "180519267", + "AccountName": "Airbyte", + "AdDistribution": "Search", + "AdGroupId": "1356799861840328", + "AdGroupName": "keywords", + "AdId": "84800390693061", + "AdRelevance": "3", + "AllConversionRate": null, + "AllConversions": "0", + "AllCostPerConversion": null, + "AllReturnOnAdSpend": null, + "AllRevenue": "0", + "AllRevenuePerConversion": null, + "Assists": "0", + "AverageCpc": "0", + "AverageCpm": "0", + "AveragePosition": "0", + "BidMatchType": "Broad", + "CampaignId": "531016227", + "CampaignName": "Airbyte test", + "Clicks": "0", + "ConversionRate": null, + "Conversions": "0", + "ConversionsQualified": "0", + "CostPerAssist": null, + "CostPerConversion": null, + "Ctr": "0", + "CurrencyCode": "USD", + "CurrentMaxCpc": "2.27", + "CustomParameters": null, + "DeliveredMatchType": "Phrase", + "DeviceOS": "Android", + "DeviceType": "Smartphone", + "ExpectedCtr": "2", + "FinalAppUrl": null, + "FinalUrlSuffix": null, + "FirstPageBid": "0.35", + "Impressions": "1", + "Keyword": "connector", + "KeywordId": "84801135055365", + "KeywordStatus": "Active", + "LandingPageExperience": "1", + "Language": "German", + "Mainline1Bid": null, + "MainlineBid": "1.11", + "Network": "Syndicated search partners", + "QualityImpact": "0", + "QualityScore": "5", + "ReturnOnAdSpend": null, + "Revenue": "0", + "RevenuePerAssist": null, + "RevenuePerConversion": null, + "Spend": "0", + "TimePeriod": "2023-11-10T00:00:00+00:00", + "TopVsOther": "Syndicated search partners - Top", + "ViewThroughConversions": "0", + "ViewThroughConversionsQualified": null + } +] diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/search_query_performance.csv b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/search_query_performance.csv new file mode 100644 index 000000000000..6319f41309d9 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/search_query_performance.csv @@ -0,0 +1,2 @@ +AccountName,AccountNumber,AccountId,TimePeriod,CampaignName,CampaignId,AdGroupName,AdGroupId,AdId,AdType,DestinationUrl,BidMatchType,DeliveredMatchType,CampaignStatus,AdStatus,Impressions,Clicks,Ctr,AverageCpc,Spend,AveragePosition,SearchQuery,Keyword,AdGroupCriterionId,Conversions,ConversionRate,CostPerConversion,Language,KeywordId,Network,TopVsOther,DeviceType,DeviceOS,Assists,Revenue,ReturnOnAdSpend,CostPerAssist,RevenuePerConversion,RevenuePerAssist,AccountStatus,AdGroupStatus,KeywordStatus,CampaignType,CustomerId,CustomerName,AllConversions,AllRevenue,AllConversionRate,AllCostPerConversion,AllReturnOnAdSpend,AllRevenuePerConversion,Goal,GoalType,AbsoluteTopImpressionRatePercent,TopImpressionRatePercent,AverageCpm,ConversionsQualified,AllConversionsQualified +Airbyte,F149MJ18,180519267,2023-11-07T08:00:00+00:00,Airbyte test,531016227,keywords,1356799861840328,84800390693061,Responsive search ad,,Broad,Phrase (close variant),Active,Active,1,0,0,0,0,0,halex screw clamp connectors,connector,,0,,,English,84801135055365,Microsoft sites and select traffic,Microsoft sites and select traffic - other,Computer,Windows,0,0,,,,,Active,Active,Active,Search & content,251186883,Daxtarity Inc.,0,0,,,,,,,0,0,0,0,0 diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/search_query_performance_records.json b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/search_query_performance_records.json new file mode 100644 index 000000000000..fda2521725d9 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/search_query_performance_records.json @@ -0,0 +1,62 @@ +[ + { + "AbsoluteTopImpressionRatePercent": "0", + "AccountId": "180519267", + "AccountName": "Airbyte", + "AccountNumber": "F149MJ18", + "AccountStatus": "Active", + "AdGroupCriterionId": null, + "AdGroupId": "1356799861840328", + "AdGroupName": "keywords", + "AdGroupStatus": "Active", + "AdId": "84800390693061", + "AdStatus": "Active", + "AdType": "Responsive search ad", + "AllConversionRate": null, + "AllConversions": "0", + "AllConversionsQualified": "0", + "AllCostPerConversion": null, + "AllReturnOnAdSpend": null, + "AllRevenue": "0", + "AllRevenuePerConversion": null, + "Assists": "0", + "AverageCpc": "0", + "AverageCpm": "0", + "AveragePosition": "0", + "BidMatchType": "Broad", + "CampaignId": "531016227", + "CampaignName": "Airbyte test", + "CampaignStatus": "Active", + "CampaignType": "Search & content", + "Clicks": "0", + "ConversionRate": null, + "Conversions": "0", + "ConversionsQualified": "0", + "CostPerAssist": null, + "CostPerConversion": null, + "Ctr": "0", + "CustomerId": "251186883", + "CustomerName": "Daxtarity Inc.", + "DeliveredMatchType": "Phrase (close variant)", + "DestinationUrl": null, + "DeviceOS": "Windows", + "DeviceType": "Computer", + "Goal": null, + "GoalType": null, + "Impressions": "1", + "Keyword": "connector", + "KeywordId": "84801135055365", + "KeywordStatus": "Active", + "Language": "English", + "Network": "Microsoft sites and select traffic", + "ReturnOnAdSpend": null, + "Revenue": "0", + "RevenuePerAssist": null, + "RevenuePerConversion": null, + "SearchQuery": "halex screw clamp connectors", + "Spend": "0", + "TimePeriod": "2023-11-07T08:00:00+00:00", + "TopImpressionRatePercent": "0", + "TopVsOther": "Microsoft sites and select traffic - other" + } +] diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/user_location_performance.csv b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/user_location_performance.csv new file mode 100644 index 000000000000..0e9812694d23 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/user_location_performance.csv @@ -0,0 +1,2 @@ +AccountName,AccountNumber,AccountId,TimePeriod,CampaignName,CampaignId,AdGroupName,AdGroupId,Country,State,MetroArea,CurrencyCode,AdDistribution,Impressions,Clicks,Ctr,AverageCpc,Spend,AveragePosition,ProximityTargetLocation,Radius,Language,City,QueryIntentCountry,QueryIntentState,QueryIntentCity,QueryIntentDMA,BidMatchType,DeliveredMatchType,Network,TopVsOther,DeviceType,DeviceOS,Assists,Conversions,ConversionRate,Revenue,ReturnOnAdSpend,CostPerConversion,CostPerAssist,RevenuePerConversion,RevenuePerAssist,County,PostalCode,QueryIntentCounty,QueryIntentPostalCode,LocationId,QueryIntentLocationId,AllConversions,AllRevenue,AllConversionRate,AllCostPerConversion,AllReturnOnAdSpend,AllRevenuePerConversion,ViewThroughConversions,Goal,GoalType,AbsoluteTopImpressionRatePercent,TopImpressionRatePercent,AverageCpm,ConversionsQualified,AllConversionsQualified,ViewThroughConversionsQualified,Neighborhood,QueryIntentNeighborhood,ViewThroughRevenue,CampaignType,AssetGroupId,AssetGroupName +Airbyte,F149MJ18,180519267,2023-11-07T08:00:00+00:00,Airbyte test,531016227,keywords,1356799861840328,United Kingdom,England,Warwickshire,USD,Search,1,0,0,0,0,0,,0,English,Rugby,United Kingdom,,,,Broad,Phrase,Syndicated search partners,Syndicated search partners - Top,Computer,Unknown,0,0,,0,,,,,,,,,,162523,188,0,0,,,,,0,,,100,100,0,0,0,,,,0,Search & content,, diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/user_location_performance_records.json b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/user_location_performance_records.json new file mode 100644 index 000000000000..0ac9344eacac --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/hourly_reports/user_location_performance_records.json @@ -0,0 +1,73 @@ +[ + { + "AbsoluteTopImpressionRatePercent": "100", + "AccountId": "180519267", + "AccountName": "Airbyte", + "AccountNumber": "F149MJ18", + "AdDistribution": "Search", + "AdGroupId": "1356799861840328", + "AdGroupName": "keywords", + "AllConversionRate": null, + "AllConversions": "0", + "AllConversionsQualified": "0", + "AllCostPerConversion": null, + "AllReturnOnAdSpend": null, + "AllRevenue": "0", + "AllRevenuePerConversion": null, + "AssetGroupId": null, + "AssetGroupName": null, + "Assists": "0", + "AverageCpc": "0", + "AverageCpm": "0", + "AveragePosition": "0", + "BidMatchType": "Broad", + "CampaignId": "531016227", + "CampaignName": "Airbyte test", + "CampaignType": "Search & content", + "City": "Rugby", + "Clicks": "0", + "ConversionRate": null, + "Conversions": "0", + "ConversionsQualified": "0", + "CostPerAssist": null, + "CostPerConversion": null, + "Country": "United Kingdom", + "County": null, + "Ctr": "0", + "CurrencyCode": "USD", + "DeliveredMatchType": "Phrase", + "DeviceOS": "Unknown", + "DeviceType": "Computer", + "Goal": null, + "GoalType": null, + "Impressions": "1", + "Language": "English", + "LocationId": "162523", + "MetroArea": "Warwickshire", + "Neighborhood": null, + "Network": "Syndicated search partners", + "PostalCode": null, + "ProximityTargetLocation": null, + "QueryIntentCity": null, + "QueryIntentCountry": "United Kingdom", + "QueryIntentCounty": null, + "QueryIntentDMA": null, + "QueryIntentLocationId": "188", + "QueryIntentNeighborhood": null, + "QueryIntentPostalCode": null, + "QueryIntentState": null, + "Radius": "0", + "ReturnOnAdSpend": null, + "Revenue": "0", + "RevenuePerAssist": null, + "RevenuePerConversion": null, + "Spend": "0", + "State": "England", + "TimePeriod": "2023-11-07T08:00:00+00:00", + "TopImpressionRatePercent": "100", + "TopVsOther": "Syndicated search partners - Top", + "ViewThroughConversions": "0", + "ViewThroughConversionsQualified": null, + "ViewThroughRevenue": "0" + } +] diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_bulk_streams.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_bulk_streams.py new file mode 100644 index 000000000000..c69f77e9bea2 --- /dev/null +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_bulk_streams.py @@ -0,0 +1,160 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from pathlib import Path +from unittest.mock import patch + +import pendulum +import pytest +import source_bing_ads +from freezegun import freeze_time +from pendulum import UTC, DateTime +from source_bing_ads.base_streams import Accounts +from source_bing_ads.bulk_streams import AppInstallAdLabels, AppInstallAds + + +@patch.object(source_bing_ads.source, "Client") +def test_bulk_stream_stream_slices(mocked_client, config): + slices = AppInstallAds(mocked_client, config).stream_slices() + assert list(slices) == [] + + app_install_ads = AppInstallAds(mocked_client, config) + accounts_read_records = iter([{"Id": 180519267, "ParentCustomerId": 100}, {"Id": 180278106, "ParentCustomerId": 200}]) + with patch.object(Accounts, "read_records", return_value=accounts_read_records): + slices = app_install_ads.stream_slices() + assert list(slices) == [{"account_id": 180519267, "customer_id": 100}, {"account_id": 180278106, "customer_id": 200}] + + +@patch.object(source_bing_ads.source, "Client") +def test_bulk_stream_transform(mocked_client, config): + record = {"Ad Group": "Ad Group", "App Id": "App Id", "Campaign": "Campaign", "Custom Parameter": "Custom Parameter"} + transformed_record = AppInstallAds(mocked_client, config).transform( + record=record, stream_slice={"account_id": 180519267, "customer_id": 100} + ) + assert transformed_record == { + "Account Id": 180519267, + "Ad Group": "Ad Group", + "App Id": "App Id", + "Campaign": "Campaign", + "Custom Parameter": "Custom Parameter", + } + + +@patch.object(source_bing_ads.source, "Client") +def test_bulk_stream_read_with_chunks(mocked_client, config): + path_to_file = Path(__file__).parent / "app_install_ads.csv" + path_to_file_base = Path(__file__).parent / "app_install_ads_base.csv" + with open(path_to_file_base, "r") as f1, open(path_to_file, "a") as f2: + for line in f1: + f2.write(line) + + app_install_ads = AppInstallAds(mocked_client, config) + result = app_install_ads.read_with_chunks(path=path_to_file) + assert next(result) == { + "Ad Group": "AdGroupNameGoesHere", + "App Id": "AppStoreIdGoesHere", + "App Platform": "Android", + "Campaign": "ParentCampaignNameGoesHere", + "Client Id": "ClientIdGoesHere", + "Custom Parameter": "{_promoCode}=PROMO1; {_season}=summer", + "Destination Url": None, + "Device Preference": "All", + "Display Url": None, + "Final Url": "FinalUrlGoesHere", + "Final Url Suffix": None, + "Id": None, + "Mobile Final Url": None, + "Modified Time": None, + "Name": None, + "Parent Id": "-1111", + "Promotion": None, + "Status": "Active", + "Text": "Find New Customers & Increase Sales!", + "Title": "Contoso Quick Setup", + "Tracking Template": None, + "Type": "App Install Ad", + } + + +@patch.object(source_bing_ads.source, "Client") +def test_bulk_stream_read_with_chunks_app_install_ad_labels(mocked_client, config): + path_to_file = Path(__file__).parent / "app_install_ad_labels.csv" + path_to_file_base = Path(__file__).parent / "app_install_ad_labels_base.csv" + with open(path_to_file_base, "r") as f1, open(path_to_file, "a") as f2: + for line in f1: + f2.write(line) + + app_install_ads = AppInstallAdLabels(mocked_client, config) + result = app_install_ads.read_with_chunks(path=path_to_file) + assert next(result) == { + 'Ad Group': None, + 'Campaign': None, + 'Client Id': 'ClientIdGoesHere', + 'Color': None, + 'Description': None, + 'Id': '-22', + 'Label': None, + 'Modified Time': None, + 'Name': None, + 'Parent Id': '-11112', + 'Status': None, + 'Type': 'App Install Ad Label' + } + + +@patch.object(source_bing_ads.source, "Client") +@freeze_time("2023-11-01T12:00:00.000+00:00") +@pytest.mark.parametrize( + "stream_state, config_start_date, expected_start_date", + [ + ({"some_account_id": {"Modified Time": "2023-10-15T12:00:00.000+00:00"}}, "2020-01-01", DateTime(2023, 10, 15, 12, 0, 0, tzinfo=UTC)), + ({"another_account_id": {"Modified Time": "2023-10-15T12:00:00.000+00:00"}}, "2020-01-01", None), + ({}, "2020-01-01", None), + ({}, "2023-10-21", DateTime(2023, 10, 21, 0, 0, 0, tzinfo=UTC)), + ], + ids=["state_within_30_days", "state_within_30_days_another_account_id", "empty_state", "empty_state_start_date_within_30"] +) +def test_bulk_stream_start_date(mocked_client, config, stream_state, config_start_date, expected_start_date): + mocked_client.reports_start_date = pendulum.parse(config_start_date) if config_start_date else None + stream = AppInstallAds(mocked_client, config) + assert expected_start_date == stream.get_start_date(stream_state, 'some_account_id') + + +@patch.object(source_bing_ads.source, "Client") +def test_bulk_stream_stream_state(mocked_client, config): + stream = AppInstallAds(mocked_client, config) + stream.state = {"Account Id": "some_account_id", "Modified Time": "04/27/2023 18:00:14.970"} + assert stream.state == {"some_account_id": {"Modified Time": "2023-04-27T18:00:14.970+00:00"}} + stream.state = {"Account Id": "some_account_id", "Modified Time": "05/27/2023 18:00:14.970"} + assert stream.state == {"some_account_id": {"Modified Time": "2023-05-27T18:00:14.970+00:00"}} + stream.state = {"Account Id": "some_account_id", "Modified Time": "05/25/2023 18:00:14.970"} + assert stream.state == {"some_account_id": {"Modified Time": "2023-05-27T18:00:14.970+00:00"}} + # stream state saved to connection state + stream.state = { + "120342748234": { + "Modified Time": "2022-11-05T12:07:29.360+00:00" + }, + "27364572345": { + "Modified Time": "2022-11-05T12:07:29.360+00:00" + }, + "732645723": { + "Modified Time": "2022-11-05T12:07:29.360+00:00" + }, + "837563864": { + "Modified Time": "2022-11-05T12:07:29.360+00:00" + } + } + assert stream.state == { + "120342748234": {"Modified Time": "2022-11-05T12:07:29.360+00:00"}, + "27364572345": {"Modified Time": "2022-11-05T12:07:29.360+00:00"}, + "732645723": {"Modified Time": "2022-11-05T12:07:29.360+00:00"}, + "837563864": {"Modified Time": "2022-11-05T12:07:29.360+00:00"}, + "some_account_id": {"Modified Time": "2023-05-27T18:00:14.970+00:00"}, + } + + +@patch.object(source_bing_ads.source, "Client") +def test_bulk_stream_custom_transform_date_rfc3339(mocked_client, config): + stream = AppInstallAds(mocked_client, config) + assert "2023-04-27T18:00:14.970+00:00" == stream.custom_transform_date_rfc3339("04/27/2023 18:00:14.970", stream.get_json_schema()["properties"][stream.cursor_field]) diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_client.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_client.py index 3e6df51fb1d7..96b2d5777f57 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_client.py @@ -2,12 +2,18 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import socket from datetime import datetime, timedelta from unittest import mock from unittest.mock import patch +from urllib.error import URLError +import pytest import source_bing_ads.client -from bingads.authorization import OAuthTokens +from airbyte_cdk.utils import AirbyteTracedException +from bingads.authorization import AuthorizationData, OAuthTokens +from bingads.v13.bulk import BulkServiceManager +from bingads.v13.reporting.exceptions import ReportingDownloadException from suds import sudsobject @@ -94,3 +100,92 @@ def test_get_auth_client(patched_request_tokens): client = source_bing_ads.client.Client("tenant_id", "2020-01-01", client_id="client_id", refresh_token="refresh_token") client._get_auth_client("client_id", "tenant_id") patched_request_tokens.assert_called_once_with("refresh_token") + + +@patch("bingads.authorization.OAuthWebAuthCodeGrant.request_oauth_tokens_by_refresh_token") +def test_get_auth_data(patched_request_tokens): + client = source_bing_ads.client.Client("tenant_id", "2020-01-01", client_id="client_id", refresh_token="refresh_token") + auth_data = client._get_auth_data() + assert isinstance(auth_data, AuthorizationData) + + +@patch("bingads.authorization.OAuthWebAuthCodeGrant.request_oauth_tokens_by_refresh_token") +def test_handling_ReportingDownloadException(patched_request_tokens): + client = source_bing_ads.client.Client("tenant_id", "2020-01-01", client_id="client_id", refresh_token="refresh_token") + give_up = client.should_give_up(ReportingDownloadException(message="test")) + assert False is give_up + assert client._download_timeout == 310000 + client._download_timeout = 600000 + client.should_give_up(ReportingDownloadException(message="test")) + assert client._download_timeout == 600000 + + +def test_get_access_token(requests_mock): + requests_mock.post( + "https://login.microsoftonline.com/tenant_id/oauth2/v2.0/token", + status_code=400, + json={ + "error": "invalid_grant", + "error_description": "AADSTS70000: The user could not be authenticated as the grant is expired. The user must sign in again.", + }, + ) + with pytest.raises( + AirbyteTracedException, + match="Failed to get OAuth access token by refresh token. The user could not be authenticated as the grant is expired. " + "The user must sign in again.", + ): + source_bing_ads.client.Client("tenant_id", "2020-01-01", client_id="client_id", refresh_token="refresh_token") + + +def test_get_access_token_success(requests_mock): + requests_mock.post( + "https://login.microsoftonline.com/tenant_id/oauth2/v2.0/token", + status_code=200, + json={"access_token": "test", "expires_in": "900", "refresh_token": "test"}, + ) + source_bing_ads.client.Client("tenant_id", "2020-01-01", client_id="client_id", refresh_token="refresh_token") + assert requests_mock.call_count == 1 + + +@patch("bingads.authorization.OAuthWebAuthCodeGrant.request_oauth_tokens_by_refresh_token") +def test_should_give_up(patched_request_tokens): + client = source_bing_ads.client.Client("tenant_id", "2020-01-01", client_id="client_id", refresh_token="refresh_token") + give_up = client.should_give_up(Exception()) + assert True is give_up + give_up = client.should_give_up(URLError(reason="test")) + assert True is give_up + give_up = client.should_give_up(URLError(reason=socket.timeout())) + assert False is give_up + + +@patch("bingads.authorization.OAuthWebAuthCodeGrant.request_oauth_tokens_by_refresh_token") +def test_get_service(patched_request_tokens): + client = source_bing_ads.client.Client("tenant_id", "2020-01-01", client_id="client_id", refresh_token="refresh_token") + service = client.get_service(service_name="CustomerManagementService") + assert "customermanagement_service.xml" in service.service_url + + +@patch("bingads.authorization.OAuthWebAuthCodeGrant.request_oauth_tokens_by_refresh_token") +def test_get_reporting_service(patched_request_tokens): + client = source_bing_ads.client.Client("tenant_id", "2020-01-01", client_id="client_id", refresh_token="refresh_token") + service = client._get_reporting_service() + assert (service._poll_interval_in_milliseconds, service._environment) == (client.report_poll_interval, client.environment) + + +@patch("bingads.authorization.OAuthWebAuthCodeGrant.request_oauth_tokens_by_refresh_token") +def test_bulk_service_manager(patched_request_tokens): + client = source_bing_ads.client.Client("tenant_id", "2020-01-01", client_id="client_id", refresh_token="refresh_token") + service = client._bulk_service_manager() + assert (service._poll_interval_in_milliseconds, service._environment) == (5000, client.environment) + + +def test_get_bulk_entity(requests_mock): + requests_mock.post( + "https://login.microsoftonline.com/tenant_id/oauth2/v2.0/token", + status_code=200, + json={"access_token": "test", "expires_in": "9000", "refresh_token": "test"}, + ) + client = source_bing_ads.client.Client("tenant_id", "2020-01-01", client_id="client_id", refresh_token="refresh_token") + with patch.object(BulkServiceManager, "download_file", return_value="file.csv"): + bulk_entity = client.get_bulk_entity(data_scope=["EntityData"], download_entities=["AppInstallAds"]) + assert bulk_entity == "file.csv" diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py index 4d82a559d91f..e00a6b67b6f7 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_reports.py @@ -3,18 +3,69 @@ # import copy +import json +import xml.etree.ElementTree as ET +from pathlib import Path +from unittest.mock import MagicMock, Mock, patch +from urllib.parse import urlparse +import _csv import pendulum +import pytest +import source_bing_ads +from airbyte_cdk.models import SyncMode +from bingads.service_info import SERVICE_INFO_DICT_V13 +from bingads.v13.internal.reporting.row_report import _RowReport from bingads.v13.internal.reporting.row_report_iterator import _RowReportRecord, _RowValues -from source_bing_ads.reports import PerformanceReportsMixin, ReportsMixin +from source_bing_ads.base_streams import Accounts +from source_bing_ads.report_streams import ( + AccountImpressionPerformanceReportDaily, + AccountImpressionPerformanceReportHourly, + AccountPerformanceReportDaily, + AccountPerformanceReportHourly, + AccountPerformanceReportMonthly, + AdGroupImpressionPerformanceReportDaily, + AdGroupImpressionPerformanceReportHourly, + AdGroupPerformanceReportDaily, + AdGroupPerformanceReportHourly, + AdPerformanceReportDaily, + AdPerformanceReportHourly, + AgeGenderAudienceReportDaily, + AgeGenderAudienceReportHourly, + BingAdsReportingServicePerformanceStream, + BingAdsReportingServiceStream, + BudgetSummaryReport, + CampaignImpressionPerformanceReportDaily, + CampaignImpressionPerformanceReportHourly, + CampaignPerformanceReportDaily, + CampaignPerformanceReportHourly, + GeographicPerformanceReportDaily, + GeographicPerformanceReportHourly, + GeographicPerformanceReportMonthly, + GeographicPerformanceReportWeekly, + KeywordPerformanceReportDaily, + KeywordPerformanceReportHourly, + SearchQueryPerformanceReportDaily, + SearchQueryPerformanceReportHourly, + UserLocationPerformanceReportDaily, + UserLocationPerformanceReportHourly, +) from source_bing_ads.source import SourceBingAds +from suds import WebFault + +TEST_CONFIG = { + "developer_token": "developer_token", + "client_id": "client_id", + "refresh_token": "refresh_token", + "reports_start_date": "2020-01-01T00:00:00Z", +} class TestClient: pass -class TestReport(ReportsMixin, SourceBingAds): +class TestReport(BingAdsReportingServiceStream, SourceBingAds): date_format, report_columns, report_name, cursor_field = "YYYY-MM-DD", None, None, "Time" report_aggregation = "Monthly" report_schema_name = "campaign_performance_report" @@ -23,7 +74,7 @@ def __init__(self) -> None: self.client = TestClient() -class TestPerformanceReport(PerformanceReportsMixin, SourceBingAds): +class TestPerformanceReport(BingAdsReportingServicePerformanceStream, SourceBingAds): date_format, report_columns, report_name, cursor_field = "YYYY-MM-DD", None, None, "Time" report_aggregation = "Monthly" report_schema_name = "campaign_performance_report" @@ -33,18 +84,40 @@ def __init__(self) -> None: def test_get_column_value(): + config = { + "developer_token": "developer_token", + "client_id": "client_id", + "refresh_token": "refresh_token", + "reports_start_date": "2020-01-01T00:00:00Z", + } + test_report = GeographicPerformanceReportDaily(client=Mock(), config=config) + row_values = _RowValues( - {"AccountId": 1, "AverageCpc": 3, "AdGroupId": 2, "AccountName": 5, "Spend": 4}, - {3: "11.5", 1: "33", 2: "--", 5: "123456789", 4: "120.3%"}, + {"AccountId": 1, "AverageCpc": 3, "AdGroupId": 2, "AccountName": 5, "Spend": 4, "AllRevenue": 6, "Assists": 7}, + {3: "11.5", 1: "33", 2: "--", 5: "123456789", 4: "120.3%", 6: "123,456,789.23", 7: "123,456,789"}, ) record = _RowReportRecord(row_values) - test_report = TestReport() - assert test_report.get_column_value(record, "AccountId") == 33 - assert test_report.get_column_value(record, "AverageCpc") == 11.5 - assert test_report.get_column_value(record, "AdGroupId") == 0 + assert test_report.get_column_value(record, "AccountId") == "33" + assert test_report.get_column_value(record, "AverageCpc") == "11.5" + assert test_report.get_column_value(record, "AdGroupId") is None assert test_report.get_column_value(record, "AccountName") == "123456789" - assert test_report.get_column_value(record, "Spend") == 1.203 + assert test_report.get_column_value(record, "Spend") == "120.3" + assert test_report.get_column_value(record, "AllRevenue") == "123456789.23" + assert test_report.get_column_value(record, "Assists") == "123456789" + + +@patch.object(source_bing_ads.source, "Client") +def test_AccountPerformanceReportMonthly_request_params(mocked_client, config): + accountperformancereportmonthly = AccountPerformanceReportMonthly(mocked_client, config) + request_params = accountperformancereportmonthly.request_params(account_id=180278106, stream_slice={"time_period": "ThisYear"}) + del request_params["report_request"] + assert request_params == { + "overwrite_result_file": True, + "result_file_directory": "/tmp", + "result_file_name": "AccountPerformanceReport", + "timeout_in_milliseconds": 300000, + } def test_get_updated_state_init_state(): @@ -52,20 +125,20 @@ def test_get_updated_state_init_state(): stream_state = {} latest_record = {"AccountId": 123, "Time": "2020-01-02"} new_state = test_report.get_updated_state(stream_state, latest_record) - assert new_state["123"]["Time"] == (pendulum.parse("2020-01-02")).timestamp() + assert new_state["123"]["Time"] == "2020-01-02" def test_get_updated_state_new_state(): test_report = TestReport() - stream_state = {"123": {"Time": pendulum.parse("2020-01-01").timestamp()}} + stream_state = {"123": {"Time": "2020-01-01"}} latest_record = {"AccountId": 123, "Time": "2020-01-02"} new_state = test_report.get_updated_state(stream_state, latest_record) - assert new_state["123"]["Time"] == pendulum.parse("2020-01-02").timestamp() + assert new_state["123"]["Time"] == "2020-01-02" def test_get_updated_state_state_unchanged(): test_report = TestReport() - stream_state = {"123": {"Time": pendulum.parse("2020-01-03").timestamp()}} + stream_state = {"123": {"Time": "2020-01-03"}} latest_record = {"AccountId": 123, "Time": "2020-01-02"} new_state = test_report.get_updated_state(copy.deepcopy(stream_state), latest_record) assert stream_state == new_state @@ -73,34 +146,65 @@ def test_get_updated_state_state_unchanged(): def test_get_updated_state_state_new_account(): test_report = TestReport() - stream_state = {"123": {"Time": pendulum.parse("2020-01-03").timestamp()}} + stream_state = {"123": {"Time": "2020-01-03"}} latest_record = {"AccountId": 234, "Time": "2020-01-02"} new_state = test_report.get_updated_state(stream_state, latest_record) assert "234" in new_state and "123" in new_state - assert new_state["234"]["Time"] == pendulum.parse("2020-01-02").timestamp() - - -def test_get_report_record_timestamp_daily(): - test_report = TestReport() - test_report.report_aggregation = "Daily" - assert pendulum.parse("2020-01-01").timestamp() == test_report.get_report_record_timestamp("2020-01-01") + assert new_state["234"]["Time"] == "2020-01-02" + + +@pytest.mark.parametrize( + "stream_report_daily_cls", + ( + AccountImpressionPerformanceReportDaily, + AccountPerformanceReportDaily, + AdGroupImpressionPerformanceReportDaily, + AdGroupPerformanceReportDaily, + AgeGenderAudienceReportDaily, + AdPerformanceReportDaily, + CampaignImpressionPerformanceReportDaily, + CampaignPerformanceReportDaily, + KeywordPerformanceReportDaily, + SearchQueryPerformanceReportDaily, + UserLocationPerformanceReportDaily, + GeographicPerformanceReportDaily, + ), +) +def test_get_report_record_timestamp_daily(stream_report_daily_cls): + stream_report = stream_report_daily_cls(client=Mock(), config=TEST_CONFIG) + assert "2020-01-01" == stream_report.get_report_record_timestamp("2020-01-01") def test_get_report_record_timestamp_without_aggregation(): - test_report = TestReport() - test_report.report_aggregation = None - assert pendulum.parse("2020-07-20").timestamp() == test_report.get_report_record_timestamp("7/20/2020") - - -def test_get_report_record_timestamp_hourly(): - test_report = TestReport() - test_report.report_aggregation = "Hourly" - assert pendulum.parse("2020-01-01T15:00:00").timestamp() == test_report.get_report_record_timestamp("2020-01-01|15") + stream_report = BudgetSummaryReport(client=Mock(), config=TEST_CONFIG) + assert "2020-07-20" == stream_report.get_report_record_timestamp("7/20/2020") + + +@pytest.mark.parametrize( + "stream_report_hourly_cls", + ( + AccountImpressionPerformanceReportHourly, + AccountPerformanceReportHourly, + AdGroupImpressionPerformanceReportHourly, + AdGroupPerformanceReportHourly, + AgeGenderAudienceReportHourly, + AdPerformanceReportHourly, + CampaignImpressionPerformanceReportHourly, + CampaignPerformanceReportHourly, + KeywordPerformanceReportHourly, + SearchQueryPerformanceReportHourly, + UserLocationPerformanceReportHourly, + GeographicPerformanceReportHourly, + ), +) +def test_get_report_record_timestamp_hourly(stream_report_hourly_cls): + stream_report = stream_report_hourly_cls(client=Mock(), config=TEST_CONFIG) + assert "2020-01-01T15:00:00+00:00" == stream_report.get_report_record_timestamp("2020-01-01|15") def test_report_get_start_date_wo_stream_state(): expected_start_date = "2020-01-01" - test_report = TestReport() + test_report = GeographicPerformanceReportDaily(client=Mock(), config=TEST_CONFIG) test_report.client.reports_start_date = "2020-01-01" stream_state = {} account_id = "123" @@ -109,20 +213,18 @@ def test_report_get_start_date_wo_stream_state(): def test_report_get_start_date_with_stream_state(): expected_start_date = pendulum.parse("2023-04-17T21:29:57") - test_report = TestReport() - test_report.cursor_field = "cursor_field" + test_report = GeographicPerformanceReportDaily(client=Mock(), config=TEST_CONFIG) test_report.client.reports_start_date = "2020-01-01" - stream_state = {"123": {"cursor_field": 1681766997}} + stream_state = {"123": {"TimePeriod": "2023-04-17T21:29:57+00:00"}} account_id = "123" assert expected_start_date == test_report.get_start_date(stream_state, account_id) def test_report_get_start_date_performance_report_with_stream_state(): expected_start_date = pendulum.parse("2023-04-07T21:29:57") - test_report = TestPerformanceReport() - test_report.cursor_field = "cursor_field" + test_report = GeographicPerformanceReportDaily(client=Mock(), config=TEST_CONFIG) test_report.config = {"lookback_window": 10} - stream_state = {"123": {"cursor_field": 1681766997}} + stream_state = {"123": {"TimePeriod": "2023-04-17T21:29:57+00:00"}} account_id = "123" assert expected_start_date == test_report.get_start_date(stream_state, account_id) @@ -130,10 +232,225 @@ def test_report_get_start_date_performance_report_with_stream_state(): def test_report_get_start_date_performance_report_wo_stream_state(): days_to_subtract = 10 reports_start_date = pendulum.parse("2021-04-07T00:00:00") - test_report = TestPerformanceReport() - test_report.cursor_field = "cursor_field" + test_report = GeographicPerformanceReportDaily(client=Mock(), config=TEST_CONFIG) test_report.client.reports_start_date = reports_start_date test_report.config = {"lookback_window": days_to_subtract} stream_state = {} account_id = "123" assert reports_start_date.subtract(days=days_to_subtract) == test_report.get_start_date(stream_state, account_id) + + +@pytest.mark.parametrize( + "performance_report_cls", + ( + GeographicPerformanceReportDaily, + GeographicPerformanceReportHourly, + GeographicPerformanceReportMonthly, + GeographicPerformanceReportWeekly, + ), +) +def test_geographic_performance_report_pk(performance_report_cls): + stream = performance_report_cls(client=Mock(), config=TEST_CONFIG) + assert stream.primary_key is None + + +def test_report_parse_response_csv_error(caplog): + stream_report = AccountPerformanceReportHourly(client=Mock(), config=TEST_CONFIG) + fake_response = MagicMock() + fake_response.report_records.__iter__ = MagicMock(side_effect=_csv.Error) + list(stream_report.parse_response(fake_response)) + assert "CSV report file for stream `account_performance_report_hourly` is broken or cannot be read correctly: , skipping ..." in caplog.messages + + +@patch.object(source_bing_ads.source, "Client") +def test_custom_report_clear_namespace(mocked_client, config_with_custom_reports, logger_mock): + custom_report = SourceBingAds().get_custom_reports(config_with_custom_reports, mocked_client)[0] + assert custom_report._clear_namespace("tns:ReportAggregation") == "ReportAggregation" + + +@patch.object(source_bing_ads.source, "Client") +def test_custom_report_get_object_columns(mocked_client, config_with_custom_reports, logger_mock): + reporting_service_mock = MagicMock() + reporting_service_mock._get_service_info_dict.return_value = SERVICE_INFO_DICT_V13 + mocked_client.get_service.return_value = reporting_service_mock + mocked_client.environment = "production" + + custom_report = SourceBingAds().get_custom_reports(config_with_custom_reports, mocked_client)[0] + + tree = ET.parse(urlparse(SERVICE_INFO_DICT_V13[("reporting", mocked_client.environment)]).path) + request_object = tree.find(f".//{{*}}complexType[@name='{custom_report.report_name}Request']") + + assert custom_report._get_object_columns(request_object, tree) == [ + "TimePeriod", + "AccountId", + "AccountName", + "AccountNumber", + "AccountStatus", + "CampaignId", + "CampaignName", + "CampaignStatus", + "AdGroupId", + "AdGroupName", + "AdGroupStatus", + "AdDistribution", + "Language", + "Network", + "TopVsOther", + "DeviceType", + "DeviceOS", + "BidStrategyType", + "TrackingTemplate", + "CustomParameters", + "DynamicAdTargetId", + "DynamicAdTarget", + "DynamicAdTargetStatus", + "WebsiteCoverage", + "Impressions", + "Clicks", + "Ctr", + "AverageCpc", + "Spend", + "AveragePosition", + "Conversions", + "ConversionRate", + "CostPerConversion", + "Assists", + "Revenue", + "ReturnOnAdSpend", + "CostPerAssist", + "RevenuePerConversion", + "RevenuePerAssist", + "AllConversions", + "AllRevenue", + "AllConversionRate", + "AllCostPerConversion", + "AllReturnOnAdSpend", + "AllRevenuePerConversion", + "ViewThroughConversions", + "Goal", + "GoalType", + "AbsoluteTopImpressionRatePercent", + "TopImpressionRatePercent", + "AverageCpm", + "ConversionsQualified", + "AllConversionsQualified", + "ViewThroughConversionsQualified", + "AdId", + "ViewThroughRevenue", + ] + + +@patch.object(source_bing_ads.source, "Client") +def test_custom_report_send_request(mocked_client, config_with_custom_reports, logger_mock, caplog): + class Fault: + faultstring = "Invalid Client Data" + + custom_report = SourceBingAds().get_custom_reports(config_with_custom_reports, mocked_client)[0] + with patch.object(BingAdsReportingServiceStream, "send_request", side_effect=WebFault(fault=Fault(), document=None)): + custom_report.send_request(params={}, customer_id="13131313", account_id="800800808") + assert ( + "Could not sync custom report my test custom report: Please validate your column and aggregation configuration. " + "Error form server: [Invalid Client Data]" + ) in caplog.text + + +@pytest.mark.parametrize( + "aggregation, datastring, expected", + [ + ( + "Summary", + "11/13/2023", + "2023-11-13", + ), + ( + "Hourly", + "2022-11-13|10", + "2022-11-13T10:00:00+00:00", + ), + ( + "Daily", + "2022-11-13", + "2022-11-13", + ), + ( + "Weekly", + "2022-11-13", + "2022-11-13", + ), + ( + "Monthly", + "2022-11-13", + "2022-11-13", + ), + ( + "WeeklyStartingMonday", + "2022-11-13", + "2022-11-13", + ), + ], +) +@patch.object(source_bing_ads.source, "Client") +def test_custom_report_get_report_record_timestamp(mocked_client, config_with_custom_reports, aggregation, datastring, expected): + custom_report = SourceBingAds().get_custom_reports(config_with_custom_reports, mocked_client)[0] + custom_report.report_aggregation = aggregation + assert custom_report.get_report_record_timestamp(datastring) == expected + + +@patch.object(source_bing_ads.source, "Client") +def test_account_performance_report_monthly_stream_slices(mocked_client, config): + account_performance_report_monthly = AccountPerformanceReportMonthly(mocked_client, config) + accounts_read_records = iter([{"Id": 180519267, "ParentCustomerId": 100}, {"Id": 180278106, "ParentCustomerId": 200}]) + with patch.object(Accounts, "read_records", return_value=accounts_read_records): + stream_slice = list(account_performance_report_monthly.stream_slices()) + assert stream_slice == [ + {'account_id': 180519267, 'customer_id': 100, 'time_period': 'LastYear'}, + {'account_id': 180519267, 'customer_id': 100, 'time_period': 'ThisYear'}, + {'account_id': 180278106, 'customer_id': 200, 'time_period': 'LastYear'}, + {'account_id': 180278106, 'customer_id': 200, 'time_period': 'ThisYear'} + ] + + +@pytest.mark.parametrize( + "aggregation", + [ + "DayOfWeek", + "HourOfDay", + ], +) +@patch.object(source_bing_ads.source, "Client") +def test_custom_performance_report_no_last_year_stream_slices(mocked_client, config_with_custom_reports, aggregation): + custom_report = SourceBingAds().get_custom_reports(config_with_custom_reports, mocked_client)[0] + custom_report.report_aggregation = aggregation + accounts_read_records = iter([{"Id": 180519267, "ParentCustomerId": 100}, {"Id": 180278106, "ParentCustomerId": 200}]) + with patch.object(Accounts, "read_records", return_value=accounts_read_records): + stream_slice = list(custom_report.stream_slices()) + assert stream_slice == [ + {"account_id": 180519267, "customer_id": 100, "time_period": "ThisYear"}, + {"account_id": 180278106, "customer_id": 200, "time_period": "ThisYear"}, + ] + + +@pytest.mark.parametrize( + "stream, response, records", + [ + (CampaignPerformanceReportHourly, "hourly_reports/campaign_performance.csv", "hourly_reports/campaign_performance_records.json"), + (AccountPerformanceReportHourly, "hourly_reports/account_performance.csv", "hourly_reports/account_performance_records.json"), + (AdGroupPerformanceReportHourly, "hourly_reports/ad_group_performance.csv", "hourly_reports/ad_group_performance_records.json"), + (AdPerformanceReportHourly, "hourly_reports/ad_performance.csv", "hourly_reports/ad_performance_records.json"), + (CampaignImpressionPerformanceReportHourly, "hourly_reports/campaign_impression_performance.csv", "hourly_reports/campaign_impression_performance_records.json"), + (KeywordPerformanceReportHourly, "hourly_reports/keyword_performance.csv", "hourly_reports/keyword_performance_records.json"), + (GeographicPerformanceReportHourly, "hourly_reports/geographic_performance.csv", "hourly_reports/geographic_performance_records.json"), + (AgeGenderAudienceReportHourly, "hourly_reports/age_gender_audience.csv", "hourly_reports/age_gender_audience_records.json"), + (SearchQueryPerformanceReportHourly, "hourly_reports/search_query_performance.csv", "hourly_reports/search_query_performance_records.json"), + (UserLocationPerformanceReportHourly, "hourly_reports/user_location_performance.csv", "hourly_reports/user_location_performance_records.json"), + (AccountImpressionPerformanceReportHourly, "hourly_reports/account_impression_performance.csv", "hourly_reports/account_impression_performance_records.json"), + (AdGroupImpressionPerformanceReportHourly, "hourly_reports/ad_group_impression_performance.csv", "hourly_reports/ad_group_impression_performance_records.json"), + ], +) +@patch.object(source_bing_ads.source, "Client") +def test_hourly_reports(mocked_client, config, stream, response, records): + stream_object = stream(mocked_client, config) + with patch.object(stream, "send_request", return_value=_RowReport(file=Path(__file__).parent / response)): + with open(Path(__file__).parent / records, "r") as file: + assert list(stream_object.read_records(sync_mode=SyncMode.full_refresh, stream_slice={}, stream_state={})) == json.load(file) + diff --git a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py index 23a53d6ac78e..210ccf1031be 100644 --- a/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-bing-ads/unit_tests/test_source.py @@ -2,32 +2,21 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import json -from unittest.mock import patch +from unittest.mock import MagicMock, patch import pytest import source_bing_ads from airbyte_cdk.models import SyncMode -from source_bing_ads.source import AccountPerformanceReportMonthly, Accounts, AdGroups, Ads, Campaigns, SourceBingAds - - -@pytest.fixture(name="config") -def config_fixture(): - """Generates streams settings from a config file""" - CONFIG_FILE = "secrets/config.json" - with open(CONFIG_FILE, "r") as f: - return json.loads(f.read()) - - -@pytest.fixture(name="logger_mock") -def logger_mock_fixture(): - return patch("source_bing_ads.source.AirbyteLogger") +from airbyte_cdk.utils import AirbyteTracedException +from bingads.service_info import SERVICE_INFO_DICT_V13 +from source_bing_ads.base_streams import Accounts, AdGroups, Ads, Campaigns +from source_bing_ads.source import SourceBingAds @patch.object(source_bing_ads.source, "Client") def test_streams_config_based(mocked_client, config): streams = SourceBingAds().streams(config) - assert len(streams) == 29 + assert len(streams) == 60 @patch.object(source_bing_ads.source, "Client") @@ -37,9 +26,73 @@ def test_source_check_connection_ok(mocked_client, config, logger_mock): @patch.object(source_bing_ads.source, "Client") -def test_source_check_connection_failed(mocked_client, config, logger_mock): - with patch.object(Accounts, "read_records", return_value=0): - assert SourceBingAds().check_connection(logger_mock, config=config)[0] is False +def test_source_check_connection_failed_user_do_not_have_accounts(mocked_client, config, logger_mock): + with patch.object(Accounts, "read_records", return_value=[]): + connected, reason = SourceBingAds().check_connection(logger_mock, config=config) + assert connected is False + assert ( + reason.message == "Config validation error: You don't have accounts assigned to this user. Please verify your developer token." + ) + + +def test_source_check_connection_failed_invalid_creds(config, logger_mock): + with patch.object(Accounts, "read_records", return_value=[]): + connected, reason = SourceBingAds().check_connection(logger_mock, config=config) + assert connected is False + assert ( + reason.internal_message + == "Failed to get OAuth access token by refresh token. The user could not be authenticated as the grant is expired. The user must sign in again." + ) + + +@patch.object(source_bing_ads.source, "Client") +def test_validate_custom_reposts(mocked_client, config_with_custom_reports, logger_mock): + reporting_service_mock = MagicMock() + reporting_service_mock._get_service_info_dict.return_value = SERVICE_INFO_DICT_V13 + mocked_client.get_service.return_value = reporting_service_mock + mocked_client.environment = "production" + res = SourceBingAds().validate_custom_reposts(config=config_with_custom_reports, client=mocked_client) + assert res is None + + +@patch.object(source_bing_ads.source, "Client") +def test_validate_custom_reposts_failed_invalid_report_columns(mocked_client, config_with_custom_reports, logger_mock): + reporting_service_mock = MagicMock() + reporting_service_mock._get_service_info_dict.return_value = SERVICE_INFO_DICT_V13 + mocked_client.get_service.return_value = reporting_service_mock + mocked_client.environment = "production" + config_with_custom_reports["custom_reports"][0]["report_columns"] = ["TimePeriod", "NonExistingColumn", "ConversionRate"] + + with pytest.raises(AirbyteTracedException) as e: + SourceBingAds().validate_custom_reposts(config=config_with_custom_reports, client=mocked_client) + assert e.value.internal_message == ( + "my test custom report: Reporting Columns are invalid. " + "Columns that you provided don't belong to Reporting Data Object Columns:" + " ['TimePeriod', 'NonExistingColumn', 'ConversionRate']. " + "Please ensure it is correct in Bing Ads Docs." + ) + assert ( + "Config validation error: my test custom report: Reporting Columns are " + "invalid. Columns that you provided don't belong to Reporting Data Object " + "Columns: ['TimePeriod', 'NonExistingColumn', 'ConversionRate']. Please " + "ensure it is correct in Bing Ads Docs." + ) in e.value.message + + +@patch.object(source_bing_ads.source, "Client") +def test_get_custom_reports(mocked_client, config_with_custom_reports): + custom_reports = SourceBingAds().get_custom_reports(config_with_custom_reports, mocked_client) + assert isinstance(custom_reports, list) + assert custom_reports[0].report_name == "DSAAutoTargetPerformanceReport" + assert custom_reports[0].report_aggregation == "Weekly" + assert "AccountId" in custom_reports[0].custom_report_columns + + +def test_clear_reporting_object_name(): + reporting_object = SourceBingAds()._clear_reporting_object_name("DSAAutoTargetPerformanceReportRequest") + assert reporting_object == "DSAAutoTargetPerformanceReport" + reporting_object = SourceBingAds()._clear_reporting_object_name("DSAAutoTargetPerformanceReport") + assert reporting_object == "DSAAutoTargetPerformanceReport" @patch.object(source_bing_ads.source, "Client") @@ -120,23 +173,40 @@ def test_ads_stream_slices(mocked_client, config): ] +@pytest.mark.parametrize( + ("stream", "stream_slice"), + ( + (Accounts, {"predicates": {"Predicate": [{"Field": "UserId", "Operator": "Equals", "Value": "131313131"},]}}), + (AdGroups, {"campaign_id": "campaign_id"}), + (Ads, {"ad_group_id": "ad_group_id"}), + (Campaigns, {"account_id": "account_id"}), + ), +) @patch.object(source_bing_ads.source, "Client") -def test_AccountPerformanceReportMonthly_request_params(mocked_client, config): +def test_streams_full_refresh(mocked_client, config, stream, stream_slice): + stream_instance = stream(mocked_client, config) + _ = list(stream_instance.read_records(SyncMode.full_refresh, stream_slice)) + mocked_client.request.assert_called_once() - accountperformancereportmonthly = AccountPerformanceReportMonthly(mocked_client, config) - request_params = accountperformancereportmonthly.request_params(account_id=180278106) - del request_params["report_request"] - assert request_params == { - "overwrite_result_file": True, - # 'report_request': , - "result_file_directory": "/tmp", - "result_file_name": "AccountPerformanceReport", - "timeout_in_milliseconds": 300000, + +@patch.object(source_bing_ads.source, "Client") +def test_transform(mocked_client, config): + record = {"AdFormatPreference": "All", "DevicePreference": 0, "EditorialStatus": "ActiveLimited", "FinalAppUrls": None} + transformed_record = Ads(mocked_client, config).transform( + record=record, stream_slice={"ad_group_id": 90909090, "account_id": 909090, "customer_id": 9090909} + ) + assert transformed_record == { + "AccountId": 909090, + "AdFormatPreference": "All", + "AdGroupId": 90909090, + "CustomerId": 9090909, + "DevicePreference": 0, + "EditorialStatus": "ActiveLimited", + "FinalAppUrls": None, } @patch.object(source_bing_ads.source, "Client") -def test_accounts(mocked_client, config): - accounts = Accounts(mocked_client, config) - _ = list(accounts.read_records(SyncMode.full_refresh)) - mocked_client.request.assert_called_once() +def test_check_connection_with_accounts_names_config(mocked_client, config_with_account_names, logger_mock): + with patch.object(Accounts, "read_records", return_value=iter([{"Id": 180519267}, {"Id": 180278106}])): + assert SourceBingAds().check_connection(logger_mock, config=config_with_account_names) == (True, None) diff --git a/airbyte-integrations/connectors/source-braintree/Dockerfile b/airbyte-integrations/connectors/source-braintree/Dockerfile index eaa0fafbec21..92572ab36332 100644 --- a/airbyte-integrations/connectors/source-braintree/Dockerfile +++ b/airbyte-integrations/connectors/source-braintree/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.version=0.2.1 LABEL io.airbyte.name=airbyte/source-braintree diff --git a/airbyte-integrations/connectors/source-braintree/README.md b/airbyte-integrations/connectors/source-braintree/README.md index b0b97315d229..5314c18967dd 100644 --- a/airbyte-integrations/connectors/source-braintree/README.md +++ b/airbyte-integrations/connectors/source-braintree/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-braintree:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/braintree) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_braintree/spec.json` file. @@ -54,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-braintree:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-braintree build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-braintree:airbyteDocker +An image will be built with the tag `airbyte/source-braintree:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-braintree:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,44 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-braintree:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-braintree:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-braintree:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-braintree test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-braintree:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-braintree:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -122,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-braintree test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/braintree.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-braintree/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-braintree/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-braintree/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-braintree/build.gradle b/airbyte-integrations/connectors/source-braintree/build.gradle deleted file mode 100644 index f61a6f058b5a..000000000000 --- a/airbyte-integrations/connectors/source-braintree/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_braintree_singer' -} diff --git a/airbyte-integrations/connectors/source-braintree/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-braintree/integration_tests/expected_records.jsonl index 68e96ef2b6b9..953c019b771a 100644 --- a/airbyte-integrations/connectors/source-braintree/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-braintree/integration_tests/expected_records.jsonl @@ -1,10 +1,11 @@ {"stream": "customer_stream", "data": {"addresses": [], "company": "airbyte", "created_at": "2020-12-02T06:21:40", "credit_cards": [{"billing_address": null, "bin": "411111", "card_type": "Visa", "cardholder_name": "test guy", "commercial": "Unknown", "country_of_issuance": "Unknown", "created_at": "2020-12-02T06:29:56", "customer_id": "896865626", "customer_location": "US", "debit": "Unknown", "default": true, "durbin_regulated": "Unknown", "expiration_month": "11", "expiration_year": "2022", "expired": true, "healthcare": "Unknown", "image_url": "https://assets.braintreegateway.com/payment_method_logo/visa.png?environment=sandbox", "issuing_bank": "Unknown", "last_4": "1111", "payroll": "Unknown", "prepaid": "Unknown", "product_id": "Unknown", "token": "7jt54nr", "unique_number_identifier": "7ae98b4956f624db53908bece7cb0a47", "updated_at": "2020-12-02T06:30:06"}], "custom_fields": "", "email": "customer@test.com", "fax": "", "first_name": "test", "graphql_id": "Y3VzdG9tZXJfODk2ODY1NjI2", "id": "896865626", "last_name": "test", "payment_methods": [{"billing_address": null, "bin": "411111", "card_type": "Visa", "cardholder_name": "test guy", "commercial": "Unknown", "country_of_issuance": "Unknown", "created_at": "2020-12-02T06:29:56", "customer_id": "896865626", "customer_location": "US", "debit": "Unknown", "default": true, "durbin_regulated": "Unknown", "expiration_month": "11", "expiration_year": "2022", "expired": true, "healthcare": "Unknown", "image_url": "https://assets.braintreegateway.com/payment_method_logo/visa.png?environment=sandbox", "issuing_bank": "Unknown", "last_4": "1111", "payroll": "Unknown", "prepaid": "Unknown", "product_id": "Unknown", "token": "7jt54nr", "unique_number_identifier": "7ae98b4956f624db53908bece7cb0a47", "updated_at": "2020-12-02T06:30:06"}], "phone": "1231231231", "updated_at": "2020-12-02T06:29:56", "website": "airbyte.io"}, "emitted_at": 1629119628000} {"stream": "discount_stream", "data": {"amount": 2.0, "description": "This is test discount", "id": "dissscount", "kind": "discount", "name": "test discount", "never_expires": true, "number_of_billing_cycles": null}, "emitted_at": 1629119628000} {"stream": "dispute_stream", "data": {"amount_disputed": 666.0, "amount_won": 0.0, "case_number": "CB173337554006", "chargeback_protection_level": null, "created_at": "2021-08-10T14:32:40+00:00", "currency_iso_code": "USD", "evidence": {}, "graphql_id": "ZGlzcHV0ZV95eG5jNzNtZnN2YzlienQz", "id": "yxnc73mfsvc9bzt3", "kind": "chargeback", "merchant_account_id": "airbyte", "original_dispute_id": null, "paypal_messages": [], "processor_comments": null, "reason": "fraud", "reason_code": "62", "reason_description": null, "received_date": "2021-08-10", "reference_number": null, "reply_by_date": "2021-08-16", "status": "lost", "updated_at": "2021-10-15T05:34:40+00:00"}, "emitted_at": 1635364766000} -{"stream": "transaction_stream", "data": {"acquirer_reference_number": null,"additional_processor_response": null,"amount": "666.00","authorization_expires_at": "2021-08-18T14:32:39","avs_error_response_code": null,"avs_postal_code_response_code": "I","avs_street_address_response_code": "I","billing_details": { "company": null, "country_code_alpha2": null, "country_code_alpha3": null, "country_code_numeric": null, "country_name": null, "extended_address": null, "first_name": null, "id": null, "last_name": null, "locality": null, "postal_code": null, "region": null, "street_address": null},"channel": null,"created_at": "2021-08-10T14:32:39","credit_card_details": { "billing_address": null, "bin": "402389", "card_type": "Visa", "cardholder_name": "", "commercial": "Unknown", "country_of_issuance": "Unknown", "customer_location": "US", "debit": "Unknown", "durbin_regulated": "Unknown", "expiration_month": "08", "expiration_year": "2120", "healthcare": "Unknown", "image_url": "https://assets.braintreegateway.com/payment_method_logo/visa.png?environment=sandbox", "issuing_bank": "Unknown", "last_4": "8028", "payroll": "Unknown", "prepaid": "Unknown", "product_id": "Unknown", "token": null, "unique_number_identifier": null},"currency_iso_code": "USD","custom_fields": "","customer_details": { "company": "", "email": "", "fax": "", "first_name": "", "id": null, "last_name": "", "payment_methods": [], "phone": "", "website": ""},"cvv_response_code": "I","disbursement_details": { "disbursement_date": "2021-08-11", "funds_held": false, "settlement_amount": 666.0, "settlement_base_currency_exchange_rate": 1, "settlement_currency_exchange_rate": 1, "settlement_currency_iso_code": "USD", "success": true},"discount_amount": null,"discounts": [],"disputes": [ { "amount_disputed": 666.0, "amount_won": 0.0, "case_number": "CB173337554006", "chargeback_protection_level": null, "created_at": "2021-08-10T14:32:40", "currency_iso_code": "USD", "evidence": {}, "graphql_id": "ZGlzcHV0ZV95eG5jNzNtZnN2YzlienQz", "id": "yxnc73mfsvc9bzt3", "kind": "chargeback", "merchant_account_id": "airbyte", "original_dispute_id": null, "paypal_messages": [], "processor_comments": null, "reason": "fraud", "reason_code": "62", "reason_description": null, "received_date": "2021-08-10", "reference_number": null, "reply_by_date": "2021-08-16", "status": "lost", "updated_at": "2021-10-15T05:34:40" }],"escrow_status": null,"gateway_rejection_reason": null,"global_id": "dHJhbnNhY3Rpb25fcmI0NzZ5MHg","graphql_id": "dHJhbnNhY3Rpb25fcmI0NzZ5MHg","id": "rb476y0x","installment_count": null,"merchant_account_id": "airbyte","merchant_address": { "locality": "Braintree", "postal_code": "02184", "region": "MA", "street_address": ""},"merchant_identification_number": "123456789012","merchant_name": "DESCRIPTORNAME","network_response_code": "XX","network_response_text": "sample network response text","network_transaction_id": "020210810143239","order_id": "","payment_instrument_type": "credit_card","pin_verified": false,"plan_id": null,"processed_with_network_token": false,"processor_authorization_code": "NJ3Q01","processor_response_code": "1000","processor_response_text": "Approved","processor_response_type": "approved","processor_settlement_response_code": "","processor_settlement_response_text": "","purchase_order_number": "","recurring": false,"refund_ids": [],"refund_global_ids": [],"refunded_transaction_id": null,"response_emv_data": null,"retrieval_reference_number": "1234567","sca_exemption_requested": null,"service_fee_amount": null,"settlement_batch_id": "2021-08-10_airbyte_1ma47zpg","shipping_amount": null,"shipping_details": { "company": null, "country_code_alpha2": null, "country_code_alpha3": null, "country_code_numeric": null, "country_name": null, "extended_address": null, "first_name": null, "id": null, "last_name": null, "locality": null, "postal_code": null, "region": null, "street_address": null},"ships_from_postal_code": null,"status": "settled","status_history": [ { "amount": 666.0, "status": "authorized", "timestamp": "2021-08-10T14:32:39", "transaction_source": "control_panel", "user": "sherifairbyte" }, { "amount": 666.0, "status": "submitted_for_settlement", "timestamp": "2021-08-10T14:32:39", "transaction_source": "control_panel", "user": "sherifairbyte" }, { "amount": 666.0, "status": "settled", "timestamp": "2021-08-10T15:05:35", "transaction_source": "" }],"subscription_details": { "billing_period_end_date": null, "billing_period_start_date": null},"subscription_id": null,"tax_amount": null,"tax_exempt": false,"terminal_identification_number": "00000001","type": "sale","updated_at": "2021-08-10T15:05:35","voice_referral_number": null},"emitted_at": 1635366055000} -{"stream": "transaction_stream", "data": {"acquirer_reference_number": null,"additional_processor_response": null,"amount": "2.00","authorization_expires_at": "2021-08-18T14:25:49","avs_error_response_code": null,"avs_postal_code_response_code": "I","avs_street_address_response_code": "I","billing_details": { "company": null, "country_code_alpha2": null, "country_code_alpha3": null, "country_code_numeric": null, "country_name": null, "extended_address": null, "first_name": null, "id": null, "last_name": null, "locality": null, "postal_code": null, "region": null, "street_address": null},"channel": null,"created_at": "2021-08-10T14:25:49","credit_card_details": { "billing_address": null, "bin": "411111", "card_type": "Visa", "cardholder_name": "sss", "commercial": "Unknown", "country_of_issuance": "Unknown", "customer_location": "US", "debit": "Unknown", "durbin_regulated": "Unknown", "expiration_month": "08", "expiration_year": "2100", "healthcare": "Unknown", "image_url": "https://assets.braintreegateway.com/payment_method_logo/visa.png?environment=sandbox", "issuing_bank": "Unknown", "last_4": "1111", "payroll": "Unknown", "prepaid": "Unknown", "product_id": "Unknown", "token": null, "unique_number_identifier": null},"currency_iso_code": "USD","custom_fields": "","customer_details": { "company": "", "email": "", "fax": "", "first_name": "", "id": null, "last_name": "", "payment_methods": [], "phone": "", "website": ""},"cvv_response_code": "I","disbursement_details": { "disbursement_date": "2021-08-11", "funds_held": false, "settlement_amount": 2.0, "settlement_base_currency_exchange_rate": 1, "settlement_currency_exchange_rate": 1, "settlement_currency_iso_code": "USD", "success": true},"discount_amount": null,"discounts": [],"disputes": [],"escrow_status": null,"gateway_rejection_reason": null,"global_id": "dHJhbnNhY3Rpb25fNTB0NnhxaGs","graphql_id": "dHJhbnNhY3Rpb25fNTB0NnhxaGs","id": "50t6xqhk","installment_count": null,"merchant_account_id": "airbyte","merchant_address": { "locality": "Braintree", "postal_code": "02184", "region": "MA", "street_address": ""},"merchant_identification_number": "123456789012","merchant_name": "DESCRIPTORNAME","network_response_code": "XX","network_response_text": "sample network response text","network_transaction_id": "020210810142550","order_id": "","payment_instrument_type": "credit_card","pin_verified": false,"plan_id": null,"processed_with_network_token": false,"processor_authorization_code": "T7RP3B","processor_response_code": "1000","processor_response_text": "Approved","processor_response_type": "approved","processor_settlement_response_code": "","processor_settlement_response_text": "","purchase_order_number": "","recurring": false,"refund_ids": [],"refund_global_ids": [],"refunded_transaction_id": null,"response_emv_data": null,"retrieval_reference_number": "1234567","sca_exemption_requested": null,"service_fee_amount": null,"settlement_batch_id": "2021-08-10_airbyte_fxs1hqvk","shipping_amount": null,"shipping_details": { "company": null, "country_code_alpha2": null, "country_code_alpha3": null, "country_code_numeric": null, "country_name": null, "extended_address": null, "first_name": null, "id": null, "last_name": null, "locality": null, "postal_code": null, "region": null, "street_address": null},"ships_from_postal_code": null,"status": "settled","status_history": [ { "amount": 2.0, "status": "authorized", "timestamp": "2021-08-10T14:25:50", "transaction_source": "control_panel", "user": "sherifairbyte" }, { "amount": 2.0, "status": "submitted_for_settlement", "timestamp": "2021-08-10T14:25:50", "transaction_source": "control_panel", "user": "sherifairbyte" }, { "amount": 2.0, "status": "settled", "timestamp": "2021-08-10T15:06:07", "transaction_source": "" }],"subscription_details": { "billing_period_end_date": null, "billing_period_start_date": null},"subscription_id": null,"tax_amount": null,"tax_exempt": false,"terminal_identification_number": "00000001","type": "sale","updated_at": "2021-08-10T15:06:07","voice_referral_number": null}, "emitted_at": 1635366055000} -{"stream": "transaction_stream", "data": {"acquirer_reference_number": null,"additional_processor_response": null,"amount": "2.00","authorization_expires_at": "2021-08-18T14:19:49","avs_error_response_code": null,"avs_postal_code_response_code": "I","avs_street_address_response_code": "I","billing_details": { "company": null, "country_code_alpha2": null, "country_code_alpha3": null, "country_code_numeric": null, "country_name": null, "extended_address": null, "first_name": null, "id": null, "last_name": null, "locality": null, "postal_code": null, "region": null, "street_address": null},"channel": null,"created_at": "2021-08-10T14:19:49","credit_card_details": { "billing_address": null, "bin": "411111", "card_type": "Visa", "cardholder_name": "sss", "commercial": "Unknown", "country_of_issuance": "Unknown", "customer_location": "US", "debit": "Unknown", "durbin_regulated": "Unknown", "expiration_month": "08", "expiration_year": "2100", "healthcare": "Unknown", "image_url": "https://assets.braintreegateway.com/payment_method_logo/visa.png?environment=sandbox", "issuing_bank": "Unknown", "last_4": "1111", "payroll": "Unknown", "prepaid": "Unknown", "product_id": "Unknown", "token": null, "unique_number_identifier": null},"currency_iso_code": "USD","custom_fields": "","customer_details": { "company": "", "email": "", "fax": "", "first_name": "", "id": null, "last_name": "", "payment_methods": [], "phone": "", "website": ""},"cvv_response_code": "I","disbursement_details": { "disbursement_date": "2021-08-11", "funds_held": false, "settlement_amount": 2.0, "settlement_base_currency_exchange_rate": 1, "settlement_currency_exchange_rate": 1, "settlement_currency_iso_code": "USD", "success": true},"discount_amount": null,"discounts": [],"disputes": [],"escrow_status": null,"gateway_rejection_reason": null,"global_id": "dHJhbnNhY3Rpb25fMmNkcW16c2c","graphql_id": "dHJhbnNhY3Rpb25fMmNkcW16c2c","id": "2cdqmzsg","installment_count": null,"merchant_account_id": "airbyte","merchant_address": { "locality": "Braintree", "postal_code": "02184", "region": "MA", "street_address": ""},"merchant_identification_number": "123456789012","merchant_name": "DESCRIPTORNAME","network_response_code": "XX","network_response_text": "sample network response text","network_transaction_id": "020210810141950","order_id": "","payment_instrument_type": "credit_card","pin_verified": false,"plan_id": null,"processed_with_network_token": false,"processor_authorization_code": "T8LSNZ","processor_response_code": "1000","processor_response_text": "Approved","processor_response_type": "approved","processor_settlement_response_code": "","processor_settlement_response_text": "","purchase_order_number": "","recurring": false,"refund_ids": [],"refund_global_ids": [],"refunded_transaction_id": null,"response_emv_data": null,"retrieval_reference_number": "1234567","sca_exemption_requested": null,"service_fee_amount": null,"settlement_batch_id": "2021-08-10_airbyte_1ma47zpg","shipping_amount": null,"shipping_details": { "company": null, "country_code_alpha2": null, "country_code_alpha3": null, "country_code_numeric": null, "country_name": null, "extended_address": null, "first_name": null, "id": null, "last_name": null, "locality": null, "postal_code": null, "region": null, "street_address": null},"ships_from_postal_code": null,"status": "settled","status_history": [ { "amount": 2.0, "status": "authorized", "timestamp": "2021-08-10T14:19:50", "transaction_source": "control_panel", "user": "sherifairbyte" }, { "amount": 2.0, "status": "submitted_for_settlement", "timestamp": "2021-08-10T14:19:50", "transaction_source": "control_panel", "user": "sherifairbyte" }, { "amount": 2.0, "status": "settled", "timestamp": "2021-08-10T15:05:35", "transaction_source": "" }],"subscription_details": { "billing_period_end_date": null, "billing_period_start_date": null},"subscription_id": null,"tax_amount": null,"tax_exempt": false,"terminal_identification_number": "00000001","type": "sale","updated_at": "2021-08-10T15:05:35","voice_referral_number": null}, "emitted_at": 1635366055000} -{"stream": "transaction_stream", "data": {"acquirer_reference_number": null,"additional_processor_response": null,"amount": "500.00","authorization_expires_at": "2020-12-09T06:30:06","avs_error_response_code": null,"avs_postal_code_response_code": "I","avs_street_address_response_code": "I","billing_details": { "company": null, "country_code_alpha2": null, "country_code_alpha3": null, "country_code_numeric": null, "country_name": null, "extended_address": null, "first_name": null, "id": null, "last_name": null, "locality": null, "postal_code": null, "region": null, "street_address": null},"channel": null,"created_at": "2020-12-02T06:30:06","credit_card_details": { "billing_address": null, "bin": "411111", "card_type": "Visa", "cardholder_name": "test guy", "commercial": "Unknown", "country_of_issuance": "Unknown", "customer_location": "US", "debit": "Unknown", "durbin_regulated": "Unknown", "expiration_month": "11", "expiration_year": "2022", "healthcare": "Unknown", "image_url": "https://assets.braintreegateway.com/payment_method_logo/visa.png?environment=sandbox", "issuing_bank": "Unknown", "last_4": "1111", "payroll": "Unknown", "prepaid": "Unknown", "product_id": "Unknown", "token": "7jt54nr", "unique_number_identifier": "7ae98b4956f624db53908bece7cb0a47"},"currency_iso_code": "USD","custom_fields": "","customer_details": { "company": "airbyte", "email": "customer@test.com", "fax": "", "first_name": "test", "graphql_id": "Y3VzdG9tZXJfODk2ODY1NjI2", "id": "896865626", "last_name": "test", "payment_methods": [], "phone": "1231231231", "website": "airbyte.io"},"cvv_response_code": "I","disbursement_details": { "disbursement_date": "2020-12-03", "funds_held": false, "settlement_amount": 500.0, "settlement_base_currency_exchange_rate": null, "settlement_currency_exchange_rate": 1, "settlement_currency_iso_code": "USD", "success": true},"discount_amount": null,"discounts": [],"disputes": [],"escrow_status": null,"gateway_rejection_reason": null,"global_id": "dHJhbnNhY3Rpb25faG5jbmtic3k","graphql_id": "dHJhbnNhY3Rpb25faG5jbmtic3k","id": "hncnkbsy","installment_count": null,"merchant_account_id": "airbyte","merchant_address": { "locality": "Braintree", "postal_code": "02184", "region": "MA", "street_address": ""},"merchant_identification_number": "123456789012","merchant_name": "DESCRIPTORNAME","network_response_code": "XX","network_response_text": "sample network response text","network_transaction_id": "020201202063006","order_id": "","payment_instrument_type": "credit_card","pin_verified": false,"plan_id": null,"processed_with_network_token": false,"processor_authorization_code": "MQ6H9W","processor_response_code": "1000","processor_response_text": "Approved","processor_response_type": "approved","processor_settlement_response_code": "","processor_settlement_response_text": "","purchase_order_number": "","recurring": false,"refund_ids": [],"refund_global_ids": [],"refunded_transaction_id": null,"response_emv_data": null,"retrieval_reference_number": "1234567","sca_exemption_requested": null,"service_fee_amount": null,"settlement_batch_id": "2020-12-02_airbyte_gcdq2ac6","shipping_amount": null,"shipping_details": { "company": null, "country_code_alpha2": null, "country_code_alpha3": null, "country_code_numeric": null, "country_name": null, "extended_address": null, "first_name": null, "id": null, "last_name": null, "locality": null, "postal_code": null, "region": null, "street_address": null},"ships_from_postal_code": null,"status": "settled","status_history": [ { "amount": 500.0, "status": "authorized", "timestamp": "2020-12-02T06:30:06", "transaction_source": "control_panel", "user": "sherifairbyte" }, { "amount": 500.0, "status": "submitted_for_settlement", "timestamp": "2020-12-02T06:30:06", "transaction_source": "control_panel", "user": "sherifairbyte" }, { "amount": 500.0, "status": "settled", "timestamp": "2020-12-02T08:20:40", "transaction_source": "" }],"subscription_details": { "billing_period_end_date": null, "billing_period_start_date": null},"subscription_id": null,"tax_amount": null,"tax_exempt": false,"terminal_identification_number": "00000001","type": "sale","updated_at": "2020-12-02T08:20:40","voice_referral_number": null}, "emitted_at": 1635366055000} +{"stream": "transaction_stream", "data": {"acquirer_reference_number": null, "additional_processor_response": null, "amount": "9.00", "authorization_expires_at": "2023-11-15T14:39:15", "avs_error_response_code": null, "avs_postal_code_response_code": "I", "avs_street_address_response_code": "I", "billing_details": {"company": null, "country_code_alpha2": null, "country_code_alpha3": null, "country_code_numeric": null, "country_name": null, "extended_address": null, "first_name": null, "id": null, "last_name": null, "locality": null, "postal_code": null, "region": null, "street_address": null}, "channel": null, "created_at": "2023-11-08T14:39:15", "credit_card_details": {"billing_address": null, "bin": "378282", "card_type": "American Express", "cardholder_name": "Jean LaFleur", "commercial": "Unknown", "country_of_issuance": "Unknown", "customer_location": "US", "debit": "Unknown", "durbin_regulated": "Unknown", "expiration_month": "04", "expiration_year": "2024", "healthcare": "Unknown", "image_url": "https://assets.braintreegateway.com/payment_method_logo/american_express.png?environment=sandbox", "issuing_bank": "Unknown", "last_4": "0005", "payroll": "Unknown", "prepaid": "Unknown", "product_id": "Unknown", "token": null, "unique_number_identifier": null}, "currency_iso_code": "UAH", "custom_fields": {"custom_field_test": "Custom Value store", "custom_field_example": "11111"}, "customer_details": {"company": "", "email": "", "fax": "", "first_name": "", "id": null, "last_name": "", "payment_methods": [], "phone": "", "website": ""}, "cvv_response_code": "M", "disbursement_details": {"disbursement_date": null, "funds_held": null, "settlement_amount": null, "settlement_base_currency_exchange_rate": null, "settlement_currency_exchange_rate": null, "settlement_currency_iso_code": null, "success": null}, "discount_amount": null, "discounts": [], "disputes": [], "escrow_status": null, "gateway_rejection_reason": null, "global_id": "dHJhbnNhY3Rpb25fY2NxeXRxNjI", "graphql_id": "dHJhbnNhY3Rpb25fY2NxeXRxNjI", "id": "ccqytq62", "installment_count": null, "merchant_account_id": "test_test", "merchant_address": {"locality": "Braintree", "postal_code": "02184", "region": "MA", "street_address": ""}, "merchant_identification_number": "123456789012", "merchant_name": "DESCRIPTORNAME", "network_response_code": "XX", "network_response_text": "sample network response text", "network_transaction_id": "020231108143915", "order_id": "1111", "payment_instrument_type": "credit_card", "pin_verified": false, "plan_id": null, "processed_with_network_token": false, "processor_authorization_code": "ZR5TGH", "processor_response_code": "1000", "processor_response_text": "Approved", "processor_response_type": "approved", "processor_settlement_response_code": "", "processor_settlement_response_text": "", "purchase_order_number": "1256715276182651", "recurring": false, "refund_ids": [], "refund_global_ids": [], "refunded_transaction_id": null, "response_emv_data": null, "retrieval_reference_number": "1234567", "sca_exemption_requested": null, "service_fee_amount": null, "settlement_batch_id": null, "shipping_amount": null, "shipping_details": {"company": null, "country_code_alpha2": null, "country_code_alpha3": null, "country_code_numeric": null, "country_name": null, "extended_address": null, "first_name": null, "id": null, "last_name": null, "locality": null, "postal_code": null, "region": null, "street_address": null}, "ships_from_postal_code": null, "status": "authorized", "status_history": [{"amount": 9.0, "status": "authorized", "timestamp": "2023-11-08T14:39:15", "transaction_source": "control_panel", "user": "sherifairbyte"}], "subscription_details": {"billing_period_end_date": null, "billing_period_start_date": null}, "subscription_id": null, "tax_amount": 2.0, "tax_exempt": false, "terminal_identification_number": "00000001", "type": "sale", "updated_at": "2023-11-08T14:39:15", "voice_referral_number": null}, "emitted_at": 1699467223896} +{"stream": "transaction_stream", "data": {"acquirer_reference_number": null, "additional_processor_response": null, "amount": "666.00", "authorization_expires_at": "2021-08-18T14:32:39", "avs_error_response_code": null, "avs_postal_code_response_code": "I", "avs_street_address_response_code": "I", "billing_details": {"company": null, "country_code_alpha2": null, "country_code_alpha3": null, "country_code_numeric": null, "country_name": null, "extended_address": null, "first_name": null, "id": null, "last_name": null, "locality": null, "postal_code": null, "region": null, "street_address": null}, "channel": null, "created_at": "2021-08-10T14:32:39", "credit_card_details": {"billing_address": null, "bin": "402389", "card_type": "Visa", "cardholder_name": "", "commercial": "Unknown", "country_of_issuance": "Unknown", "customer_location": "US", "debit": "Unknown", "durbin_regulated": "Unknown", "expiration_month": "08", "expiration_year": "2120", "healthcare": "Unknown", "image_url": "https://assets.braintreegateway.com/payment_method_logo/visa.png?environment=sandbox", "issuing_bank": "Unknown", "last_4": "8028", "payroll": "Unknown", "prepaid": "Unknown", "product_id": "Unknown", "token": null, "unique_number_identifier": null}, "currency_iso_code": "USD", "custom_fields": {}, "customer_details": {"company": "", "email": "", "fax": "", "first_name": "", "id": null, "last_name": "", "payment_methods": [], "phone": "", "website": ""}, "cvv_response_code": "I", "disbursement_details": {"disbursement_date": "2021-08-11", "funds_held": false, "settlement_amount": 666.0, "settlement_base_currency_exchange_rate": 1, "settlement_currency_exchange_rate": 1, "settlement_currency_iso_code": "USD", "success": true}, "discount_amount": null, "discounts": [], "disputes": [{"amount_disputed": 666.0, "amount_won": 0.0, "case_number": "CB173337554006", "chargeback_protection_level": null, "created_at": "2021-08-10T14:32:40", "currency_iso_code": "USD", "evidence": {}, "graphql_id": "ZGlzcHV0ZV95eG5jNzNtZnN2YzlienQz", "id": "yxnc73mfsvc9bzt3", "kind": "chargeback", "merchant_account_id": "airbyte", "original_dispute_id": null, "paypal_messages": [], "processor_comments": null, "reason": "fraud", "reason_code": "62", "reason_description": null, "received_date": "2021-08-10", "reference_number": null, "reply_by_date": "2021-08-16", "status": "lost", "updated_at": "2021-10-15T05:34:40"}], "escrow_status": null, "gateway_rejection_reason": null, "global_id": "dHJhbnNhY3Rpb25fcmI0NzZ5MHg", "graphql_id": "dHJhbnNhY3Rpb25fcmI0NzZ5MHg", "id": "rb476y0x", "installment_count": null, "merchant_account_id": "airbyte", "merchant_address": {"locality": "Braintree", "postal_code": "02184", "region": "MA", "street_address": ""}, "merchant_identification_number": "123456789012", "merchant_name": "DESCRIPTORNAME", "network_response_code": "XX", "network_response_text": "sample network response text", "network_transaction_id": "020210810143239", "order_id": "", "payment_instrument_type": "credit_card", "pin_verified": false, "plan_id": null, "processed_with_network_token": false, "processor_authorization_code": "NJ3Q01", "processor_response_code": "1000", "processor_response_text": "Approved", "processor_response_type": "approved", "processor_settlement_response_code": "", "processor_settlement_response_text": "", "purchase_order_number": "", "recurring": false, "refund_ids": [], "refund_global_ids": [], "refunded_transaction_id": null, "response_emv_data": null, "retrieval_reference_number": "1234567", "sca_exemption_requested": null, "service_fee_amount": null, "settlement_batch_id": "2021-08-10_airbyte_1ma47zpg", "shipping_amount": null, "shipping_details": {"company": null, "country_code_alpha2": null, "country_code_alpha3": null, "country_code_numeric": null, "country_name": null, "extended_address": null, "first_name": null, "id": null, "last_name": null, "locality": null, "postal_code": null, "region": null, "street_address": null}, "ships_from_postal_code": null, "status": "settled", "status_history": [{"amount": 666.0, "status": "authorized", "timestamp": "2021-08-10T14:32:39", "transaction_source": "control_panel", "user": "sherifairbyte"}, {"amount": 666.0, "status": "submitted_for_settlement", "timestamp": "2021-08-10T14:32:39", "transaction_source": "control_panel", "user": "sherifairbyte"}, {"amount": 666.0, "status": "settled", "timestamp": "2021-08-10T15:05:35", "transaction_source": ""}], "subscription_details": {"billing_period_end_date": null, "billing_period_start_date": null}, "subscription_id": null, "tax_amount": null, "tax_exempt": false, "terminal_identification_number": "00000001", "type": "sale", "updated_at": "2021-08-10T15:05:35", "voice_referral_number": null}, "emitted_at": 1699467223898} +{"stream": "transaction_stream", "data": {"acquirer_reference_number": null, "additional_processor_response": null, "amount": "2.00", "authorization_expires_at": "2021-08-18T14:25:49", "avs_error_response_code": null, "avs_postal_code_response_code": "I", "avs_street_address_response_code": "I", "billing_details": {"company": null, "country_code_alpha2": null, "country_code_alpha3": null, "country_code_numeric": null, "country_name": null, "extended_address": null, "first_name": null, "id": null, "last_name": null, "locality": null, "postal_code": null, "region": null, "street_address": null}, "channel": null, "created_at": "2021-08-10T14:25:49", "credit_card_details": {"billing_address": null, "bin": "411111", "card_type": "Visa", "cardholder_name": "sss", "commercial": "Unknown", "country_of_issuance": "Unknown", "customer_location": "US", "debit": "Unknown", "durbin_regulated": "Unknown", "expiration_month": "08", "expiration_year": "2100", "healthcare": "Unknown", "image_url": "https://assets.braintreegateway.com/payment_method_logo/visa.png?environment=sandbox", "issuing_bank": "Unknown", "last_4": "1111", "payroll": "Unknown", "prepaid": "Unknown", "product_id": "Unknown", "token": null, "unique_number_identifier": null}, "currency_iso_code": "USD", "custom_fields": {}, "customer_details": {"company": "", "email": "", "fax": "", "first_name": "", "id": null, "last_name": "", "payment_methods": [], "phone": "", "website": ""}, "cvv_response_code": "I", "disbursement_details": {"disbursement_date": "2021-08-11", "funds_held": false, "settlement_amount": 2.0, "settlement_base_currency_exchange_rate": 1, "settlement_currency_exchange_rate": 1, "settlement_currency_iso_code": "USD", "success": true}, "discount_amount": null, "discounts": [], "disputes": [], "escrow_status": null, "gateway_rejection_reason": null, "global_id": "dHJhbnNhY3Rpb25fNTB0NnhxaGs", "graphql_id": "dHJhbnNhY3Rpb25fNTB0NnhxaGs", "id": "50t6xqhk", "installment_count": null, "merchant_account_id": "airbyte", "merchant_address": {"locality": "Braintree", "postal_code": "02184", "region": "MA", "street_address": ""}, "merchant_identification_number": "123456789012", "merchant_name": "DESCRIPTORNAME", "network_response_code": "XX", "network_response_text": "sample network response text", "network_transaction_id": "020210810142550", "order_id": "", "payment_instrument_type": "credit_card", "pin_verified": false, "plan_id": null, "processed_with_network_token": false, "processor_authorization_code": "T7RP3B", "processor_response_code": "1000", "processor_response_text": "Approved", "processor_response_type": "approved", "processor_settlement_response_code": "", "processor_settlement_response_text": "", "purchase_order_number": "", "recurring": false, "refund_ids": [], "refund_global_ids": [], "refunded_transaction_id": null, "response_emv_data": null, "retrieval_reference_number": "1234567", "sca_exemption_requested": null, "service_fee_amount": null, "settlement_batch_id": "2021-08-10_airbyte_fxs1hqvk", "shipping_amount": null, "shipping_details": {"company": null, "country_code_alpha2": null, "country_code_alpha3": null, "country_code_numeric": null, "country_name": null, "extended_address": null, "first_name": null, "id": null, "last_name": null, "locality": null, "postal_code": null, "region": null, "street_address": null}, "ships_from_postal_code": null, "status": "settled", "status_history": [{"amount": 2.0, "status": "authorized", "timestamp": "2021-08-10T14:25:50", "transaction_source": "control_panel", "user": "sherifairbyte"}, {"amount": 2.0, "status": "submitted_for_settlement", "timestamp": "2021-08-10T14:25:50", "transaction_source": "control_panel", "user": "sherifairbyte"}, {"amount": 2.0, "status": "settled", "timestamp": "2021-08-10T15:06:07", "transaction_source": ""}], "subscription_details": {"billing_period_end_date": null, "billing_period_start_date": null}, "subscription_id": null, "tax_amount": null, "tax_exempt": false, "terminal_identification_number": "00000001", "type": "sale", "updated_at": "2021-08-10T15:06:07", "voice_referral_number": null}, "emitted_at": 1699467223899} +{"stream": "transaction_stream", "data": {"acquirer_reference_number": null, "additional_processor_response": null, "amount": "2.00", "authorization_expires_at": "2021-08-18T14:19:49", "avs_error_response_code": null, "avs_postal_code_response_code": "I", "avs_street_address_response_code": "I", "billing_details": {"company": null, "country_code_alpha2": null, "country_code_alpha3": null, "country_code_numeric": null, "country_name": null, "extended_address": null, "first_name": null, "id": null, "last_name": null, "locality": null, "postal_code": null, "region": null, "street_address": null}, "channel": null, "created_at": "2021-08-10T14:19:49", "credit_card_details": {"billing_address": null, "bin": "411111", "card_type": "Visa", "cardholder_name": "sss", "commercial": "Unknown", "country_of_issuance": "Unknown", "customer_location": "US", "debit": "Unknown", "durbin_regulated": "Unknown", "expiration_month": "08", "expiration_year": "2100", "healthcare": "Unknown", "image_url": "https://assets.braintreegateway.com/payment_method_logo/visa.png?environment=sandbox", "issuing_bank": "Unknown", "last_4": "1111", "payroll": "Unknown", "prepaid": "Unknown", "product_id": "Unknown", "token": null, "unique_number_identifier": null}, "currency_iso_code": "USD", "custom_fields": {}, "customer_details": {"company": "", "email": "", "fax": "", "first_name": "", "id": null, "last_name": "", "payment_methods": [], "phone": "", "website": ""}, "cvv_response_code": "I", "disbursement_details": {"disbursement_date": "2021-08-11", "funds_held": false, "settlement_amount": 2.0, "settlement_base_currency_exchange_rate": 1, "settlement_currency_exchange_rate": 1, "settlement_currency_iso_code": "USD", "success": true}, "discount_amount": null, "discounts": [], "disputes": [], "escrow_status": null, "gateway_rejection_reason": null, "global_id": "dHJhbnNhY3Rpb25fMmNkcW16c2c", "graphql_id": "dHJhbnNhY3Rpb25fMmNkcW16c2c", "id": "2cdqmzsg", "installment_count": null, "merchant_account_id": "airbyte", "merchant_address": {"locality": "Braintree", "postal_code": "02184", "region": "MA", "street_address": ""}, "merchant_identification_number": "123456789012", "merchant_name": "DESCRIPTORNAME", "network_response_code": "XX", "network_response_text": "sample network response text", "network_transaction_id": "020210810141950", "order_id": "", "payment_instrument_type": "credit_card", "pin_verified": false, "plan_id": null, "processed_with_network_token": false, "processor_authorization_code": "T8LSNZ", "processor_response_code": "1000", "processor_response_text": "Approved", "processor_response_type": "approved", "processor_settlement_response_code": "", "processor_settlement_response_text": "", "purchase_order_number": "", "recurring": false, "refund_ids": [], "refund_global_ids": [], "refunded_transaction_id": null, "response_emv_data": null, "retrieval_reference_number": "1234567", "sca_exemption_requested": null, "service_fee_amount": null, "settlement_batch_id": "2021-08-10_airbyte_1ma47zpg", "shipping_amount": null, "shipping_details": {"company": null, "country_code_alpha2": null, "country_code_alpha3": null, "country_code_numeric": null, "country_name": null, "extended_address": null, "first_name": null, "id": null, "last_name": null, "locality": null, "postal_code": null, "region": null, "street_address": null}, "ships_from_postal_code": null, "status": "settled", "status_history": [{"amount": 2.0, "status": "authorized", "timestamp": "2021-08-10T14:19:50", "transaction_source": "control_panel", "user": "sherifairbyte"}, {"amount": 2.0, "status": "submitted_for_settlement", "timestamp": "2021-08-10T14:19:50", "transaction_source": "control_panel", "user": "sherifairbyte"}, {"amount": 2.0, "status": "settled", "timestamp": "2021-08-10T15:05:35", "transaction_source": ""}], "subscription_details": {"billing_period_end_date": null, "billing_period_start_date": null}, "subscription_id": null, "tax_amount": null, "tax_exempt": false, "terminal_identification_number": "00000001", "type": "sale", "updated_at": "2021-08-10T15:05:35", "voice_referral_number": null}, "emitted_at": 1699467223901} +{"stream": "transaction_stream", "data": {"acquirer_reference_number": null, "additional_processor_response": null, "amount": "500.00", "authorization_expires_at": "2020-12-09T06:30:06", "avs_error_response_code": null, "avs_postal_code_response_code": "I", "avs_street_address_response_code": "I", "billing_details": {"company": null, "country_code_alpha2": null, "country_code_alpha3": null, "country_code_numeric": null, "country_name": null, "extended_address": null, "first_name": null, "id": null, "last_name": null, "locality": null, "postal_code": null, "region": null, "street_address": null}, "channel": null, "created_at": "2020-12-02T06:30:06", "credit_card_details": {"billing_address": null, "bin": "411111", "card_type": "Visa", "cardholder_name": "test guy", "commercial": "Unknown", "country_of_issuance": "Unknown", "customer_location": "US", "debit": "Unknown", "durbin_regulated": "Unknown", "expiration_month": "11", "expiration_year": "2022", "healthcare": "Unknown", "image_url": "https://assets.braintreegateway.com/payment_method_logo/visa.png?environment=sandbox", "issuing_bank": "Unknown", "last_4": "1111", "payroll": "Unknown", "prepaid": "Unknown", "product_id": "Unknown", "token": "7jt54nr", "unique_number_identifier": "7ae98b4956f624db53908bece7cb0a47"}, "currency_iso_code": "USD", "custom_fields": {}, "customer_details": {"company": "airbyte", "email": "customer@test.com", "fax": "", "first_name": "test", "graphql_id": "Y3VzdG9tZXJfODk2ODY1NjI2", "id": "896865626", "last_name": "test", "payment_methods": [], "phone": "1231231231", "website": "airbyte.io"}, "cvv_response_code": "I", "disbursement_details": {"disbursement_date": "2020-12-03", "funds_held": false, "settlement_amount": 500.0, "settlement_base_currency_exchange_rate": null, "settlement_currency_exchange_rate": 1, "settlement_currency_iso_code": "USD", "success": true}, "discount_amount": null, "discounts": [], "disputes": [], "escrow_status": null, "gateway_rejection_reason": null, "global_id": "dHJhbnNhY3Rpb25faG5jbmtic3k", "graphql_id": "dHJhbnNhY3Rpb25faG5jbmtic3k", "id": "hncnkbsy", "installment_count": null, "merchant_account_id": "airbyte", "merchant_address": {"locality": "Braintree", "postal_code": "02184", "region": "MA", "street_address": ""}, "merchant_identification_number": "123456789012", "merchant_name": "DESCRIPTORNAME", "network_response_code": "XX", "network_response_text": "sample network response text", "network_transaction_id": "020201202063006", "order_id": "", "payment_instrument_type": "credit_card", "pin_verified": false, "plan_id": null, "processed_with_network_token": false, "processor_authorization_code": "MQ6H9W", "processor_response_code": "1000", "processor_response_text": "Approved", "processor_response_type": "approved", "processor_settlement_response_code": "", "processor_settlement_response_text": "", "purchase_order_number": "", "recurring": false, "refund_ids": [], "refund_global_ids": [], "refunded_transaction_id": null, "response_emv_data": null, "retrieval_reference_number": "1234567", "sca_exemption_requested": null, "service_fee_amount": null, "settlement_batch_id": "2020-12-02_airbyte_gcdq2ac6", "shipping_amount": null, "shipping_details": {"company": null, "country_code_alpha2": null, "country_code_alpha3": null, "country_code_numeric": null, "country_name": null, "extended_address": null, "first_name": null, "id": null, "last_name": null, "locality": null, "postal_code": null, "region": null, "street_address": null}, "ships_from_postal_code": null, "status": "settled", "status_history": [{"amount": 500.0, "status": "authorized", "timestamp": "2020-12-02T06:30:06", "transaction_source": "control_panel", "user": "sherifairbyte"}, {"amount": 500.0, "status": "submitted_for_settlement", "timestamp": "2020-12-02T06:30:06", "transaction_source": "control_panel", "user": "sherifairbyte"}, {"amount": 500.0, "status": "settled", "timestamp": "2020-12-02T08:20:40", "transaction_source": ""}], "subscription_details": {"billing_period_end_date": null, "billing_period_start_date": null}, "subscription_id": null, "tax_amount": null, "tax_exempt": false, "terminal_identification_number": "00000001", "type": "sale", "updated_at": "2020-12-02T08:20:40", "voice_referral_number": null}, "emitted_at": 1699467223902} {"stream": "merchant_account_stream", "data": {"business_details": {"address_details": {}}, "currency_iso_code": "USD", "funding_details": {}, "id": "airbyte", "individual_details": {"address_details": {}}, "status": "active"}, "emitted_at": 1629119631000} {"stream": "merchant_account_stream", "data": {"business_details": {"address_details": {}}, "currency_iso_code": "UAH", "funding_details": {}, "id": "test_test", "individual_details": {"address_details": {}}, "status": "active"}, "emitted_at": 1629119631000} {"stream": "plan_stream", "data": {"add_ons": [{"amount": 2.0, "description": "this is test addon", "id": "test_addon", "kind": "add_on", "name": "test addon", "never_expires": false, "number_of_billing_cycles": 34}, {"amount": 1.0, "description": "", "id": "test_addon_empty", "kind": "add_on", "name": "test_addon", "never_expires": true, "number_of_billing_cycles": null}], "billing_day_of_month": null, "billing_frequency": 1, "created_at": "2021-08-10T12:43:15", "currency_iso_code": "USD", "description": "This is plan created for testing", "discounts": [{"amount": 2.0, "description": "This is test discount", "id": "dissscount", "kind": "discount", "name": "test discount", "never_expires": true, "number_of_billing_cycles": null}], "id": "eqweqw", "name": "test plan", "number_of_billing_cycles": 3, "price": 20.0, "trial_duration": 2, "trial_duration_unit": "month", "trial_period": true, "updated_at": "2021-08-10T12:43:15"}, "emitted_at": 1629119632000} diff --git a/airbyte-integrations/connectors/source-braintree/metadata.yaml b/airbyte-integrations/connectors/source-braintree/metadata.yaml index 330156721e05..d2f0ca52f2bb 100644 --- a/airbyte-integrations/connectors/source-braintree/metadata.yaml +++ b/airbyte-integrations/connectors/source-braintree/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 63cea06f-1c75-458d-88fe-ad48c7cb27fd - dockerImageTag: 0.2.0 + dockerImageTag: 0.2.1 dockerRepository: airbyte/source-braintree documentationUrl: https://docs.airbyte.com/integrations/sources/braintree githubIssueLabel: source-braintree diff --git a/airbyte-integrations/connectors/source-braintree/setup.py b/airbyte-integrations/connectors/source-braintree/setup.py index 1930cb0c4fdb..0a6f5d53752e 100644 --- a/airbyte-integrations/connectors/source-braintree/setup.py +++ b/airbyte-integrations/connectors/source-braintree/setup.py @@ -10,7 +10,6 @@ TEST_REQUIREMENTS = [ "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-braintree/source_braintree/manifest.yaml b/airbyte-integrations/connectors/source-braintree/source_braintree/manifest.yaml index ead085ac84e9..16403f6f4580 100644 --- a/airbyte-integrations/connectors/source-braintree/source_braintree/manifest.yaml +++ b/airbyte-integrations/connectors/source-braintree/source_braintree/manifest.yaml @@ -4720,7 +4720,8 @@ streams: custom_fields: type: - "null" - - string + - object + additionalProperties: true customer_details: oneOf: - type: "null" @@ -15897,7 +15898,7 @@ streams: incremental_sync: type: DatetimeBasedCursor cursor_field: received_date - datetime_format: "%Y-%m-%d %H:%M:%S" + datetime_format: "%Y-%m-%d" start_datetime: type: MinMaxDatetime datetime: "{{ config['start_date'] }}" diff --git a/airbyte-integrations/connectors/source-braintree/source_braintree/schemas/transaction.py b/airbyte-integrations/connectors/source-braintree/source_braintree/schemas/transaction.py index 659be4cd948d..d17dbbeb2e69 100644 --- a/airbyte-integrations/connectors/source-braintree/source_braintree/schemas/transaction.py +++ b/airbyte-integrations/connectors/source-braintree/source_braintree/schemas/transaction.py @@ -4,7 +4,7 @@ from datetime import date, datetime from decimal import Decimal -from typing import List, Optional +from typing import Dict, List, Optional from .cards import ( Address, @@ -61,7 +61,7 @@ class Transaction(BaseModel): created_at: datetime credit_card_details: CreditCard currency_iso_code: str - custom_fields: str + custom_fields: Dict[str, str] customer_details: Customer cvv_response_code: str disbursement_details: DisbursementDetails diff --git a/airbyte-integrations/connectors/source-braze/Dockerfile b/airbyte-integrations/connectors/source-braze/Dockerfile index 349a54f07f0b..09a925119c4e 100644 --- a/airbyte-integrations/connectors/source-braze/Dockerfile +++ b/airbyte-integrations/connectors/source-braze/Dockerfile @@ -34,5 +34,5 @@ COPY source_braze ./source_braze ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.3.0 LABEL io.airbyte.name=airbyte/source-braze diff --git a/airbyte-integrations/connectors/source-braze/README.md b/airbyte-integrations/connectors/source-braze/README.md index eb8215a5c015..b8010776cb7a 100644 --- a/airbyte-integrations/connectors/source-braze/README.md +++ b/airbyte-integrations/connectors/source-braze/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-braze:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/braze) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_braze/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-braze:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-braze build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-braze:airbyteDocker +An image will be built with the tag `airbyte/source-braze:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-braze:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-braze:dev check --conf docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-braze:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-braze:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-braze test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-braze:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-braze:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-braze test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/braze.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-braze/acceptance-test-config.yml b/airbyte-integrations/connectors/source-braze/acceptance-test-config.yml index 23de8af06889..c007cce98637 100644 --- a/airbyte-integrations/connectors/source-braze/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-braze/acceptance-test-config.yml @@ -29,18 +29,10 @@ acceptance_tests: bypass_reason: "no data" - name: kpi_daily_active_users bypass_reason: "no data" - expect_records: - # `cards_analytics` stream records are not included to `expected_records.jsonl` - # This stream returns summary by given `ending_at` date + `length`, and so - # as `ending_at` date depends on a day when tests run, we lose previous records in current response and tests fail. - path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: no - extra_records: yes incremental: tests: - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + configured_catalog_path: "integration_tests/configured_catalog_incremental.json" future_state: future_state_path: "integration_tests/abnormal_state.json" full_refresh: diff --git a/airbyte-integrations/connectors/source-braze/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-braze/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-braze/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-braze/build.gradle b/airbyte-integrations/connectors/source-braze/build.gradle deleted file mode 100644 index 953cadefb1f0..000000000000 --- a/airbyte-integrations/connectors/source-braze/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_braze' -} diff --git a/airbyte-integrations/connectors/source-braze/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-braze/integration_tests/abnormal_state.json index 73ca9a0e3a24..1cc0381a1541 100644 --- a/airbyte-integrations/connectors/source-braze/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-braze/integration_tests/abnormal_state.json @@ -1,22 +1,266 @@ -{ - "campaigns_analytics": { - "time": "2050-12-13", - "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a" +[ + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "campaigns_analytics" + }, + "stream_state": { + "states": [ + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "campaign_id": "8c846e45-a8ab-4afe-b315-04627504b526", + "parent_slice": {} + } + } + ] + } + } }, - "canvases_analytics": { - "time": "2050-12-13", - "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2" + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "canvases_analytics" + }, + "stream_state": { + "states": [ + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2", + "parent_slice": {} + } + } + ] + } + } }, - "events_analytics": {}, - "kpi_daily_new_users": {}, - "kpi_daily_active_users": {}, - "kpi_daily_app_uninstalls": {}, - "cards_analytics": { - "time": "2050-12-13", - "card_id": "609e4d4c-367b-4c87-a8ad-b1f903d058fd" + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "cards_analytics" + }, + "stream_state": { + "states": [ + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "card_id": "609e4d4c-367b-4c87-a8ad-b1f903d058fd", + "parent_slice": {} + } + } + ] + } + } }, - "segments_analytics": { - "time": "2050-12-13", - "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c" + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "segments_analytics" + }, + "stream_state": { + "states": [ + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2050-11-03" + }, + "partition": { + "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c", + "parent_slice": {} + } + } + ] + } + } } -} +] diff --git a/airbyte-integrations/connectors/source-braze/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-braze/integration_tests/configured_catalog.json index 27548a469018..f37c56a4206e 100644 --- a/airbyte-integrations/connectors/source-braze/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-braze/integration_tests/configured_catalog.json @@ -49,9 +49,9 @@ "stream": { "name": "events_analytics", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "incremental", + "sync_mode": "full_refresh", "destination_sync_mode": "append" }, { @@ -116,6 +116,33 @@ }, "sync_mode": "incremental", "destination_sync_mode": "append" + }, + { + "stream": { + "name": "campaigns_details", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "canvases_details", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "segments_details", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-braze/integration_tests/configured_catalog_incremental.json b/airbyte-integrations/connectors/source-braze/integration_tests/configured_catalog_incremental.json new file mode 100644 index 000000000000..d3477781bfba --- /dev/null +++ b/airbyte-integrations/connectors/source-braze/integration_tests/configured_catalog_incremental.json @@ -0,0 +1,49 @@ +{ + "streams": [ + { + "stream": { + "name": "campaigns_analytics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "canvases_analytics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "events_analytics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "cards_analytics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "segments_analytics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-braze/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-braze/integration_tests/expected_records.jsonl deleted file mode 100644 index baa7b69fbfc1..000000000000 --- a/airbyte-integrations/connectors/source-braze/integration_tests/expected_records.jsonl +++ /dev/null @@ -1,2336 +0,0 @@ -{"stream": "campaigns", "data": {"id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b", "name": "Test Campaign", "is_api_campaign": false, "tags": ["airbyte", "test"], "last_edited": "2022-11-29T14:59:02+00:00"}, "emitted_at": 1671100929032} -{"stream": "campaigns", "data": {"id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a", "name": "Test Push Campaign", "is_api_campaign": false, "tags": ["airbyte", "push"], "last_edited": "2022-12-14T11:24:15+00:00"}, "emitted_at": 1671100929033} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-01", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931646} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-02", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931649} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-03", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931651} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-04", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931653} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-05", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931655} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-06", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931658} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-07", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931660} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-08", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931662} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-09", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931664} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-10", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931667} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-11", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931669} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-12", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931671} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-13", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931674} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-14", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931676} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-15", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931678} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-16", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931681} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-17", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931683} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-18", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931687} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-19", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931690} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-20", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931693} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-21", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931695} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-22", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931697} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-23", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931700} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-24", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931702} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-25", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931704} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-26", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931706} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-27", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931709} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-28", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931711} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-29", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931713} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-30", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931715} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-01", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931718} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-02", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931720} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-03", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931722} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-04", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931725} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-05", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931727} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-06", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931729} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-07", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931731} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-08", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931734} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-09", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931736} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-10", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931738} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-11", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931740} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-12", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931742} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-13", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931745} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-14", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931747} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-15", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931749} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-16", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931751} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-17", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931754} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-18", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931756} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-19", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931758} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-20", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931760} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-21", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931762} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-22", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931765} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-23", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931767} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-24", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931769} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-25", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931771} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-26", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931774} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-27", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931776} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-28", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931778} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-29", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931780} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-30", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931782} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-31", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931785} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-01", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931787} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-02", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931789} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-03", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931791} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-04", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931793} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-05", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931796} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-06", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931798} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-07", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931800} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-08", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931802} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-09", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931804} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-10", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931807} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-11", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931809} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-12", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931811} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-13", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931813} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-14", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931816} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-15", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931818} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-16", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931820} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-17", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931822} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-18", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931824} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-19", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931827} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-20", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931829} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-21", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931831} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-22", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931833} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-23", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931836} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-24", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931838} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-25", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931841} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-26", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931844} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-27", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931846} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-28", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931849} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-29", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931852} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-30", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931856} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-01", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931859} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-02", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931862} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-03", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931864} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-04", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931866} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-05", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931869} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-06", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931871} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-07", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931873} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-08", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100931875} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-09", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100932503} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-10", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100932506} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-11", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100932509} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-12", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100932511} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-13", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100932513} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-14", "messages": {"email": [{"variation_api_id": "3e222e3c-c792-402e-a55b-705db18b63c4", "sent": 0, "opens": 0, "machine_open": 0, "unique_opens": 0, "machine_amp_open": 0, "clicks": 0, "unique_clicks": 0, "unsubscribes": 0, "bounces": 0, "delivered": 0, "reported_spam": 0}], "sms": []}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b"}, "emitted_at": 1671100932515} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-01", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933201} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-02", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933204} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-03", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933207} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-04", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933210} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-05", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933213} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-06", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933215} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-07", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933218} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-08", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933220} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-09", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933222} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-10", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933225} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-11", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933227} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-12", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933229} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-13", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933231} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-14", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933233} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-15", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933236} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-16", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933238} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-17", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933240} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-18", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933242} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-19", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933245} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-20", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933247} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-21", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933250} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-22", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933252} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-23", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933254} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-24", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933256} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-25", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933258} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-26", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933261} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-27", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933263} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-28", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933265} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-29", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933267} -{"stream": "campaigns_analytics", "data": {"time": "2022-09-30", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933269} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-01", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933272} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-02", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933274} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-03", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933276} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-04", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933278} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-05", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933281} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-06", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933283} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-07", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933285} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-08", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933287} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-09", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933290} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-10", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933292} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-11", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933294} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-12", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933296} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-13", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933298} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-14", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933301} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-15", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933303} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-16", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933307} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-17", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933310} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-18", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933314} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-19", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933317} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-20", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933337} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-21", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933339} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-22", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933341} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-23", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933344} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-24", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933346} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-25", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933348} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-26", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933350} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-27", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933353} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-28", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933355} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-29", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933357} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-30", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933359} -{"stream": "campaigns_analytics", "data": {"time": "2022-10-31", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933361} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-01", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933364} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-02", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933366} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-03", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933368} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-04", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933370} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-05", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933373} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-06", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933375} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-07", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933377} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-08", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933379} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-09", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933382} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-10", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933384} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-11", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933386} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-12", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933388} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-13", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933391} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-14", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933393} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-15", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933395} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-16", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933397} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-17", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933400} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-18", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933402} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-19", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933404} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-20", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933406} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-21", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933409} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-22", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933411} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-23", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933413} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-24", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933415} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-25", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933417} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-26", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933420} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-27", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933422} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-28", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933424} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-29", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933426} -{"stream": "campaigns_analytics", "data": {"time": "2022-11-30", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933428} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-01", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933431} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-02", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933433} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-03", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933435} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-04", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933437} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-05", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933440} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-06", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933442} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-07", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933444} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-08", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100933446} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-09", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100934088} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-10", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100934098} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-11", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100934104} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-12", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100934109} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-13", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100934114} -{"stream": "campaigns_analytics", "data": {"time": "2022-12-14", "messages": {"ios_push": [{"variation_name": "Variant 1", "variation_api_id": "668b8529-fcbd-4a29-9722-055942db7ab5", "sent": 0, "direct_opens": 0, "total_opens": 0, "bounces": 0, "body_clicks": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}, {"variation_name": "Control Group", "variation_api_id": "ee9003b5-2dda-4aa5-869a-84906b688f0f", "enrolled": 0, "revenue": 0.0, "unique_recipients": 0, "conversions": 0, "conversions_by_send_time": 0, "conversions1": 0, "conversions1_by_send_time": 0, "conversions2": 0, "conversions2_by_send_time": 0, "conversions3": 0, "conversions3_by_send_time": 0}]}, "conversions_by_send_time": 0, "conversions1_by_send_time": 0, "conversions2_by_send_time": 0, "conversions3_by_send_time": 0, "conversions": 0, "conversions1": 0, "conversions2": 0, "conversions3": 0, "unique_recipients": 0, "revenue": 0.0, "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a"}, "emitted_at": 1671100934117} -{"stream": "canvases", "data": {"id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2", "name": "Test Canvas", "tags": ["airbyte", "canvas"], "last_edited": "2022-11-29T15:07:49+00:00"}, "emitted_at": 1671100934567} -{"stream": "canvases_analytics", "data": {"time": "2022-09-01", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936195} -{"stream": "canvases_analytics", "data": {"time": "2022-09-02", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936204} -{"stream": "canvases_analytics", "data": {"time": "2022-09-03", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936211} -{"stream": "canvases_analytics", "data": {"time": "2022-09-04", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936216} -{"stream": "canvases_analytics", "data": {"time": "2022-09-05", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936220} -{"stream": "canvases_analytics", "data": {"time": "2022-09-06", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936224} -{"stream": "canvases_analytics", "data": {"time": "2022-09-07", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936227} -{"stream": "canvases_analytics", "data": {"time": "2022-09-08", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936230} -{"stream": "canvases_analytics", "data": {"time": "2022-09-09", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936232} -{"stream": "canvases_analytics", "data": {"time": "2022-09-10", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936235} -{"stream": "canvases_analytics", "data": {"time": "2022-09-11", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936237} -{"stream": "canvases_analytics", "data": {"time": "2022-09-12", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936240} -{"stream": "canvases_analytics", "data": {"time": "2022-09-13", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936242} -{"stream": "canvases_analytics", "data": {"time": "2022-09-14", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936856} -{"stream": "canvases_analytics", "data": {"time": "2022-09-15", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936866} -{"stream": "canvases_analytics", "data": {"time": "2022-09-16", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936873} -{"stream": "canvases_analytics", "data": {"time": "2022-09-17", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936878} -{"stream": "canvases_analytics", "data": {"time": "2022-09-18", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936882} -{"stream": "canvases_analytics", "data": {"time": "2022-09-19", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936886} -{"stream": "canvases_analytics", "data": {"time": "2022-09-20", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936889} -{"stream": "canvases_analytics", "data": {"time": "2022-09-21", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936892} -{"stream": "canvases_analytics", "data": {"time": "2022-09-22", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936895} -{"stream": "canvases_analytics", "data": {"time": "2022-09-23", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936897} -{"stream": "canvases_analytics", "data": {"time": "2022-09-24", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936900} -{"stream": "canvases_analytics", "data": {"time": "2022-09-25", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936902} -{"stream": "canvases_analytics", "data": {"time": "2022-09-26", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936904} -{"stream": "canvases_analytics", "data": {"time": "2022-09-27", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100936907} -{"stream": "canvases_analytics", "data": {"time": "2022-09-28", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937165} -{"stream": "canvases_analytics", "data": {"time": "2022-09-29", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937175} -{"stream": "canvases_analytics", "data": {"time": "2022-09-30", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937181} -{"stream": "canvases_analytics", "data": {"time": "2022-10-01", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937185} -{"stream": "canvases_analytics", "data": {"time": "2022-10-02", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937189} -{"stream": "canvases_analytics", "data": {"time": "2022-10-03", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937193} -{"stream": "canvases_analytics", "data": {"time": "2022-10-04", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937196} -{"stream": "canvases_analytics", "data": {"time": "2022-10-05", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937199} -{"stream": "canvases_analytics", "data": {"time": "2022-10-06", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937202} -{"stream": "canvases_analytics", "data": {"time": "2022-10-07", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937204} -{"stream": "canvases_analytics", "data": {"time": "2022-10-08", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937206} -{"stream": "canvases_analytics", "data": {"time": "2022-10-09", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937209} -{"stream": "canvases_analytics", "data": {"time": "2022-10-10", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937211} -{"stream": "canvases_analytics", "data": {"time": "2022-10-11", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937213} -{"stream": "canvases_analytics", "data": {"time": "2022-10-12", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937809} -{"stream": "canvases_analytics", "data": {"time": "2022-10-13", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937811} -{"stream": "canvases_analytics", "data": {"time": "2022-10-14", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937814} -{"stream": "canvases_analytics", "data": {"time": "2022-10-15", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937817} -{"stream": "canvases_analytics", "data": {"time": "2022-10-16", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937820} -{"stream": "canvases_analytics", "data": {"time": "2022-10-17", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937822} -{"stream": "canvases_analytics", "data": {"time": "2022-10-18", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937825} -{"stream": "canvases_analytics", "data": {"time": "2022-10-19", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937827} -{"stream": "canvases_analytics", "data": {"time": "2022-10-20", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937829} -{"stream": "canvases_analytics", "data": {"time": "2022-10-21", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937831} -{"stream": "canvases_analytics", "data": {"time": "2022-10-22", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937833} -{"stream": "canvases_analytics", "data": {"time": "2022-10-23", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937835} -{"stream": "canvases_analytics", "data": {"time": "2022-10-24", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937838} -{"stream": "canvases_analytics", "data": {"time": "2022-10-25", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100937840} -{"stream": "canvases_analytics", "data": {"time": "2022-10-26", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938433} -{"stream": "canvases_analytics", "data": {"time": "2022-10-27", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938437} -{"stream": "canvases_analytics", "data": {"time": "2022-10-28", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938441} -{"stream": "canvases_analytics", "data": {"time": "2022-10-29", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938444} -{"stream": "canvases_analytics", "data": {"time": "2022-10-30", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938448} -{"stream": "canvases_analytics", "data": {"time": "2022-10-31", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938452} -{"stream": "canvases_analytics", "data": {"time": "2022-11-01", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938456} -{"stream": "canvases_analytics", "data": {"time": "2022-11-02", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938460} -{"stream": "canvases_analytics", "data": {"time": "2022-11-03", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938464} -{"stream": "canvases_analytics", "data": {"time": "2022-11-04", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938468} -{"stream": "canvases_analytics", "data": {"time": "2022-11-05", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938472} -{"stream": "canvases_analytics", "data": {"time": "2022-11-06", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938476} -{"stream": "canvases_analytics", "data": {"time": "2022-11-07", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938480} -{"stream": "canvases_analytics", "data": {"time": "2022-11-08", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938484} -{"stream": "canvases_analytics", "data": {"time": "2022-11-09", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938797} -{"stream": "canvases_analytics", "data": {"time": "2022-11-10", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938799} -{"stream": "canvases_analytics", "data": {"time": "2022-11-11", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938802} -{"stream": "canvases_analytics", "data": {"time": "2022-11-12", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938804} -{"stream": "canvases_analytics", "data": {"time": "2022-11-13", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938806} -{"stream": "canvases_analytics", "data": {"time": "2022-11-14", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938808} -{"stream": "canvases_analytics", "data": {"time": "2022-11-15", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938811} -{"stream": "canvases_analytics", "data": {"time": "2022-11-16", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938813} -{"stream": "canvases_analytics", "data": {"time": "2022-11-17", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938815} -{"stream": "canvases_analytics", "data": {"time": "2022-11-18", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938817} -{"stream": "canvases_analytics", "data": {"time": "2022-11-19", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938820} -{"stream": "canvases_analytics", "data": {"time": "2022-11-20", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938822} -{"stream": "canvases_analytics", "data": {"time": "2022-11-21", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938824} -{"stream": "canvases_analytics", "data": {"time": "2022-11-22", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100938827} -{"stream": "canvases_analytics", "data": {"time": "2022-11-23", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100939069} -{"stream": "canvases_analytics", "data": {"time": "2022-11-24", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100939072} -{"stream": "canvases_analytics", "data": {"time": "2022-11-25", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100939077} -{"stream": "canvases_analytics", "data": {"time": "2022-11-26", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100939082} -{"stream": "canvases_analytics", "data": {"time": "2022-11-27", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100939086} -{"stream": "canvases_analytics", "data": {"time": "2022-11-28", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100939089} -{"stream": "canvases_analytics", "data": {"time": "2022-11-29", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100939092} -{"stream": "canvases_analytics", "data": {"time": "2022-11-30", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100939094} -{"stream": "canvases_analytics", "data": {"time": "2022-12-01", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100939096} -{"stream": "canvases_analytics", "data": {"time": "2022-12-02", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100939098} -{"stream": "canvases_analytics", "data": {"time": "2022-12-03", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100939101} -{"stream": "canvases_analytics", "data": {"time": "2022-12-04", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100939103} -{"stream": "canvases_analytics", "data": {"time": "2022-12-05", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100939105} -{"stream": "canvases_analytics", "data": {"time": "2022-12-06", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100939107} -{"stream": "canvases_analytics", "data": {"time": "2022-12-07", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100939689} -{"stream": "canvases_analytics", "data": {"time": "2022-12-08", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100939691} -{"stream": "canvases_analytics", "data": {"time": "2022-12-09", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100939693} -{"stream": "canvases_analytics", "data": {"time": "2022-12-10", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100939695} -{"stream": "canvases_analytics", "data": {"time": "2022-12-11", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100939698} -{"stream": "canvases_analytics", "data": {"time": "2022-12-12", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100939700} -{"stream": "canvases_analytics", "data": {"time": "2022-12-13", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100939702} -{"stream": "canvases_analytics", "data": {"time": "2022-12-14", "total_stats": {"revenue": 0.0, "entries": 0, "conversions": 0, "conversions_by_entry_time": 0}, "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2"}, "emitted_at": 1671100939704} -{"stream": "events", "data": {"event_name": "AirbyteInitialEvent"}, "emitted_at": 1671100940536} -{"stream": "events", "data": {"event_name": "Test Event"}, "emitted_at": 1671100940537} -{"stream": "events", "data": {"event_name": "Test Event 1"}, "emitted_at": 1671100940538} -{"stream": "cards", "data": {"id": "609e4d4c-367b-4c87-a8ad-b1f903d058fd", "type": "NewsItem", "title": "Test", "tags": ["News feed"]}, "emitted_at": 1671100949719} -{"stream": "segments", "data": {"id": "cd7a9141-3457-4c37-9336-52845dd5fe87", "name": "Lapsed Users - 7 days", "analytics_tracking_enabled": false, "tags": []}, "emitted_at": 1671100952425} -{"stream": "segments", "data": {"id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a", "name": "Lapsed Users - 30 days", "analytics_tracking_enabled": false, "tags": []}, "emitted_at": 1671100952428} -{"stream": "segments", "data": {"id": "ce4e667c-3180-45dd-a200-7f3a9489f306", "name": "User Onboarding - First Week", "analytics_tracking_enabled": false, "tags": []}, "emitted_at": 1671100952431} -{"stream": "segments", "data": {"id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4", "name": "User Onboarding - Second Week", "analytics_tracking_enabled": false, "tags": []}, "emitted_at": 1671100952433} -{"stream": "segments", "data": {"id": "bf9673e5-41ee-4df4-b528-e46829efd646", "name": "Engaged Recent Users", "analytics_tracking_enabled": false, "tags": []}, "emitted_at": 1671100952436} -{"stream": "segments", "data": {"id": "6a51c1cf-8997-4919-a973-38557dde18de", "name": "Lapsed Users - 7 days 1", "analytics_tracking_enabled": false, "tags": []}, "emitted_at": 1671100952438} -{"stream": "segments", "data": {"id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac", "name": "Lapsed Users - 30 days 1", "analytics_tracking_enabled": false, "tags": []}, "emitted_at": 1671100952440} -{"stream": "segments", "data": {"id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2", "name": "User Onboarding - First Week 1", "analytics_tracking_enabled": false, "tags": []}, "emitted_at": 1671100952442} -{"stream": "segments", "data": {"id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac", "name": "User Onboarding - Second Week 1", "analytics_tracking_enabled": false, "tags": []}, "emitted_at": 1671100952444} -{"stream": "segments", "data": {"id": "fb7e1079-f353-4887-a183-f197b30cf27e", "name": "Engaged Recent Users 1", "analytics_tracking_enabled": false, "tags": []}, "emitted_at": 1671100952446} -{"stream": "segments", "data": {"id": "7edf3784-c465-4ca8-89be-8a07c154fbf9", "name": "Lapsed Users - 7 days 2", "analytics_tracking_enabled": false, "tags": ["test"]}, "emitted_at": 1671100952447} -{"stream": "segments", "data": {"id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71", "name": "Lapsed Users - 30 days 2", "analytics_tracking_enabled": false, "tags": []}, "emitted_at": 1671100952449} -{"stream": "segments", "data": {"id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493", "name": "User Onboarding - First Week 2", "analytics_tracking_enabled": false, "tags": []}, "emitted_at": 1671100952450} -{"stream": "segments", "data": {"id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4", "name": "User Onboarding - Second Week 2", "analytics_tracking_enabled": false, "tags": ["card"]}, "emitted_at": 1671100952451} -{"stream": "segments", "data": {"id": "a94939de-135b-4e1a-9cdf-a5940747415e", "name": "Engaged Recent Users 2", "analytics_tracking_enabled": false, "tags": ["card"]}, "emitted_at": 1671100952453} -{"stream": "segments", "data": {"id": "8043672e-73fb-4731-854b-f9d315aa2b6e", "name": "All Users (test - Android)", "analytics_tracking_enabled": false, "tags": []}, "emitted_at": 1671100952454} -{"stream": "segments", "data": {"id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10", "name": "All Users (test - iOS)", "analytics_tracking_enabled": false, "tags": []}, "emitted_at": 1671100952455} -{"stream": "segments", "data": {"id": "4926071e-2f5a-44f8-98b8-0d9db823283e", "name": "All Users (Airbyte - Web)", "analytics_tracking_enabled": false, "tags": ["airbyte"]}, "emitted_at": 1671100952456} -{"stream": "segments", "data": {"id": "8b922b25-dcc6-4e4d-b345-f7b06106046c", "name": "All Users (iOS Test - iOS)", "analytics_tracking_enabled": false, "tags": []}, "emitted_at": 1671100952457} -{"stream": "segments_analytics", "data": {"time": "2022-09-01", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955312} -{"stream": "segments_analytics", "data": {"time": "2022-09-02", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955314} -{"stream": "segments_analytics", "data": {"time": "2022-09-03", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955316} -{"stream": "segments_analytics", "data": {"time": "2022-09-04", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955318} -{"stream": "segments_analytics", "data": {"time": "2022-09-05", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955321} -{"stream": "segments_analytics", "data": {"time": "2022-09-06", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955323} -{"stream": "segments_analytics", "data": {"time": "2022-09-07", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955325} -{"stream": "segments_analytics", "data": {"time": "2022-09-08", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955327} -{"stream": "segments_analytics", "data": {"time": "2022-09-09", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955329} -{"stream": "segments_analytics", "data": {"time": "2022-09-10", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955332} -{"stream": "segments_analytics", "data": {"time": "2022-09-11", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955334} -{"stream": "segments_analytics", "data": {"time": "2022-09-12", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955336} -{"stream": "segments_analytics", "data": {"time": "2022-09-13", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955338} -{"stream": "segments_analytics", "data": {"time": "2022-09-14", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955340} -{"stream": "segments_analytics", "data": {"time": "2022-09-15", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955342} -{"stream": "segments_analytics", "data": {"time": "2022-09-16", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955344} -{"stream": "segments_analytics", "data": {"time": "2022-09-17", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955347} -{"stream": "segments_analytics", "data": {"time": "2022-09-18", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955349} -{"stream": "segments_analytics", "data": {"time": "2022-09-19", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955351} -{"stream": "segments_analytics", "data": {"time": "2022-09-20", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955353} -{"stream": "segments_analytics", "data": {"time": "2022-09-21", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955355} -{"stream": "segments_analytics", "data": {"time": "2022-09-22", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955357} -{"stream": "segments_analytics", "data": {"time": "2022-09-23", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955359} -{"stream": "segments_analytics", "data": {"time": "2022-09-24", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955362} -{"stream": "segments_analytics", "data": {"time": "2022-09-25", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955364} -{"stream": "segments_analytics", "data": {"time": "2022-09-26", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955366} -{"stream": "segments_analytics", "data": {"time": "2022-09-27", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955368} -{"stream": "segments_analytics", "data": {"time": "2022-09-28", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955370} -{"stream": "segments_analytics", "data": {"time": "2022-09-29", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955372} -{"stream": "segments_analytics", "data": {"time": "2022-09-30", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955374} -{"stream": "segments_analytics", "data": {"time": "2022-10-01", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955376} -{"stream": "segments_analytics", "data": {"time": "2022-10-02", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955379} -{"stream": "segments_analytics", "data": {"time": "2022-10-03", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955381} -{"stream": "segments_analytics", "data": {"time": "2022-10-04", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955383} -{"stream": "segments_analytics", "data": {"time": "2022-10-05", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955385} -{"stream": "segments_analytics", "data": {"time": "2022-10-06", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955387} -{"stream": "segments_analytics", "data": {"time": "2022-10-07", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955389} -{"stream": "segments_analytics", "data": {"time": "2022-10-08", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955391} -{"stream": "segments_analytics", "data": {"time": "2022-10-09", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955393} -{"stream": "segments_analytics", "data": {"time": "2022-10-10", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955396} -{"stream": "segments_analytics", "data": {"time": "2022-10-11", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955398} -{"stream": "segments_analytics", "data": {"time": "2022-10-12", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955400} -{"stream": "segments_analytics", "data": {"time": "2022-10-13", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955402} -{"stream": "segments_analytics", "data": {"time": "2022-10-14", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955404} -{"stream": "segments_analytics", "data": {"time": "2022-10-15", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955406} -{"stream": "segments_analytics", "data": {"time": "2022-10-16", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955409} -{"stream": "segments_analytics", "data": {"time": "2022-10-17", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955411} -{"stream": "segments_analytics", "data": {"time": "2022-10-18", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955413} -{"stream": "segments_analytics", "data": {"time": "2022-10-19", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955415} -{"stream": "segments_analytics", "data": {"time": "2022-10-20", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955417} -{"stream": "segments_analytics", "data": {"time": "2022-10-21", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955420} -{"stream": "segments_analytics", "data": {"time": "2022-10-22", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955422} -{"stream": "segments_analytics", "data": {"time": "2022-10-23", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955424} -{"stream": "segments_analytics", "data": {"time": "2022-10-24", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955426} -{"stream": "segments_analytics", "data": {"time": "2022-10-25", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955428} -{"stream": "segments_analytics", "data": {"time": "2022-10-26", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955430} -{"stream": "segments_analytics", "data": {"time": "2022-10-27", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955432} -{"stream": "segments_analytics", "data": {"time": "2022-10-28", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955435} -{"stream": "segments_analytics", "data": {"time": "2022-10-29", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955437} -{"stream": "segments_analytics", "data": {"time": "2022-10-30", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955439} -{"stream": "segments_analytics", "data": {"time": "2022-10-31", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955441} -{"stream": "segments_analytics", "data": {"time": "2022-11-01", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955443} -{"stream": "segments_analytics", "data": {"time": "2022-11-02", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955445} -{"stream": "segments_analytics", "data": {"time": "2022-11-03", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955447} -{"stream": "segments_analytics", "data": {"time": "2022-11-04", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955450} -{"stream": "segments_analytics", "data": {"time": "2022-11-05", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955452} -{"stream": "segments_analytics", "data": {"time": "2022-11-06", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955454} -{"stream": "segments_analytics", "data": {"time": "2022-11-07", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955456} -{"stream": "segments_analytics", "data": {"time": "2022-11-08", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955458} -{"stream": "segments_analytics", "data": {"time": "2022-11-09", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955460} -{"stream": "segments_analytics", "data": {"time": "2022-11-10", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955462} -{"stream": "segments_analytics", "data": {"time": "2022-11-11", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955464} -{"stream": "segments_analytics", "data": {"time": "2022-11-12", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955467} -{"stream": "segments_analytics", "data": {"time": "2022-11-13", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955469} -{"stream": "segments_analytics", "data": {"time": "2022-11-14", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955471} -{"stream": "segments_analytics", "data": {"time": "2022-11-15", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955473} -{"stream": "segments_analytics", "data": {"time": "2022-11-16", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955476} -{"stream": "segments_analytics", "data": {"time": "2022-11-17", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955478} -{"stream": "segments_analytics", "data": {"time": "2022-11-18", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955480} -{"stream": "segments_analytics", "data": {"time": "2022-11-19", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955482} -{"stream": "segments_analytics", "data": {"time": "2022-11-20", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955484} -{"stream": "segments_analytics", "data": {"time": "2022-11-21", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955486} -{"stream": "segments_analytics", "data": {"time": "2022-11-22", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955489} -{"stream": "segments_analytics", "data": {"time": "2022-11-23", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955491} -{"stream": "segments_analytics", "data": {"time": "2022-11-24", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955493} -{"stream": "segments_analytics", "data": {"time": "2022-11-25", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955495} -{"stream": "segments_analytics", "data": {"time": "2022-11-26", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955497} -{"stream": "segments_analytics", "data": {"time": "2022-11-27", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955499} -{"stream": "segments_analytics", "data": {"time": "2022-11-28", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955501} -{"stream": "segments_analytics", "data": {"time": "2022-11-29", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955503} -{"stream": "segments_analytics", "data": {"time": "2022-11-30", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955506} -{"stream": "segments_analytics", "data": {"time": "2022-12-01", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955508} -{"stream": "segments_analytics", "data": {"time": "2022-12-02", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955510} -{"stream": "segments_analytics", "data": {"time": "2022-12-03", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955512} -{"stream": "segments_analytics", "data": {"time": "2022-12-04", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955514} -{"stream": "segments_analytics", "data": {"time": "2022-12-05", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955516} -{"stream": "segments_analytics", "data": {"time": "2022-12-06", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955518} -{"stream": "segments_analytics", "data": {"time": "2022-12-07", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955520} -{"stream": "segments_analytics", "data": {"time": "2022-12-08", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955523} -{"stream": "segments_analytics", "data": {"time": "2022-12-09", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955762} -{"stream": "segments_analytics", "data": {"time": "2022-12-10", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955771} -{"stream": "segments_analytics", "data": {"time": "2022-12-11", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955780} -{"stream": "segments_analytics", "data": {"time": "2022-12-12", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955783} -{"stream": "segments_analytics", "data": {"time": "2022-12-13", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955786} -{"stream": "segments_analytics", "data": {"time": "2022-12-14", "size": 0, "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87"}, "emitted_at": 1671100955788} -{"stream": "segments_analytics", "data": {"time": "2022-09-01", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956383} -{"stream": "segments_analytics", "data": {"time": "2022-09-02", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956385} -{"stream": "segments_analytics", "data": {"time": "2022-09-03", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956387} -{"stream": "segments_analytics", "data": {"time": "2022-09-04", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956389} -{"stream": "segments_analytics", "data": {"time": "2022-09-05", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956391} -{"stream": "segments_analytics", "data": {"time": "2022-09-06", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956393} -{"stream": "segments_analytics", "data": {"time": "2022-09-07", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956395} -{"stream": "segments_analytics", "data": {"time": "2022-09-08", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956397} -{"stream": "segments_analytics", "data": {"time": "2022-09-09", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956400} -{"stream": "segments_analytics", "data": {"time": "2022-09-10", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956402} -{"stream": "segments_analytics", "data": {"time": "2022-09-11", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956404} -{"stream": "segments_analytics", "data": {"time": "2022-09-12", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956406} -{"stream": "segments_analytics", "data": {"time": "2022-09-13", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956408} -{"stream": "segments_analytics", "data": {"time": "2022-09-14", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956410} -{"stream": "segments_analytics", "data": {"time": "2022-09-15", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956412} -{"stream": "segments_analytics", "data": {"time": "2022-09-16", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956414} -{"stream": "segments_analytics", "data": {"time": "2022-09-17", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956416} -{"stream": "segments_analytics", "data": {"time": "2022-09-18", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956419} -{"stream": "segments_analytics", "data": {"time": "2022-09-19", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956421} -{"stream": "segments_analytics", "data": {"time": "2022-09-20", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956423} -{"stream": "segments_analytics", "data": {"time": "2022-09-21", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956425} -{"stream": "segments_analytics", "data": {"time": "2022-09-22", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956427} -{"stream": "segments_analytics", "data": {"time": "2022-09-23", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956429} -{"stream": "segments_analytics", "data": {"time": "2022-09-24", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956431} -{"stream": "segments_analytics", "data": {"time": "2022-09-25", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956433} -{"stream": "segments_analytics", "data": {"time": "2022-09-26", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956435} -{"stream": "segments_analytics", "data": {"time": "2022-09-27", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956438} -{"stream": "segments_analytics", "data": {"time": "2022-09-28", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956440} -{"stream": "segments_analytics", "data": {"time": "2022-09-29", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956442} -{"stream": "segments_analytics", "data": {"time": "2022-09-30", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956444} -{"stream": "segments_analytics", "data": {"time": "2022-10-01", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956446} -{"stream": "segments_analytics", "data": {"time": "2022-10-02", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956448} -{"stream": "segments_analytics", "data": {"time": "2022-10-03", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956450} -{"stream": "segments_analytics", "data": {"time": "2022-10-04", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956452} -{"stream": "segments_analytics", "data": {"time": "2022-10-05", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956454} -{"stream": "segments_analytics", "data": {"time": "2022-10-06", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956456} -{"stream": "segments_analytics", "data": {"time": "2022-10-07", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956459} -{"stream": "segments_analytics", "data": {"time": "2022-10-08", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956461} -{"stream": "segments_analytics", "data": {"time": "2022-10-09", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956463} -{"stream": "segments_analytics", "data": {"time": "2022-10-10", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956465} -{"stream": "segments_analytics", "data": {"time": "2022-10-11", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956467} -{"stream": "segments_analytics", "data": {"time": "2022-10-12", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956469} -{"stream": "segments_analytics", "data": {"time": "2022-10-13", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956471} -{"stream": "segments_analytics", "data": {"time": "2022-10-14", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956473} -{"stream": "segments_analytics", "data": {"time": "2022-10-15", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956475} -{"stream": "segments_analytics", "data": {"time": "2022-10-16", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956478} -{"stream": "segments_analytics", "data": {"time": "2022-10-17", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956480} -{"stream": "segments_analytics", "data": {"time": "2022-10-18", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956482} -{"stream": "segments_analytics", "data": {"time": "2022-10-19", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956484} -{"stream": "segments_analytics", "data": {"time": "2022-10-20", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956486} -{"stream": "segments_analytics", "data": {"time": "2022-10-21", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956488} -{"stream": "segments_analytics", "data": {"time": "2022-10-22", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956490} -{"stream": "segments_analytics", "data": {"time": "2022-10-23", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956492} -{"stream": "segments_analytics", "data": {"time": "2022-10-24", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956494} -{"stream": "segments_analytics", "data": {"time": "2022-10-25", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956497} -{"stream": "segments_analytics", "data": {"time": "2022-10-26", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956499} -{"stream": "segments_analytics", "data": {"time": "2022-10-27", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956501} -{"stream": "segments_analytics", "data": {"time": "2022-10-28", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956503} -{"stream": "segments_analytics", "data": {"time": "2022-10-29", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956505} -{"stream": "segments_analytics", "data": {"time": "2022-10-30", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956507} -{"stream": "segments_analytics", "data": {"time": "2022-10-31", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956509} -{"stream": "segments_analytics", "data": {"time": "2022-11-01", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956511} -{"stream": "segments_analytics", "data": {"time": "2022-11-02", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956514} -{"stream": "segments_analytics", "data": {"time": "2022-11-03", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956516} -{"stream": "segments_analytics", "data": {"time": "2022-11-04", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956518} -{"stream": "segments_analytics", "data": {"time": "2022-11-05", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956520} -{"stream": "segments_analytics", "data": {"time": "2022-11-06", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956522} -{"stream": "segments_analytics", "data": {"time": "2022-11-07", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956524} -{"stream": "segments_analytics", "data": {"time": "2022-11-08", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956526} -{"stream": "segments_analytics", "data": {"time": "2022-11-09", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956528} -{"stream": "segments_analytics", "data": {"time": "2022-11-10", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956530} -{"stream": "segments_analytics", "data": {"time": "2022-11-11", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956533} -{"stream": "segments_analytics", "data": {"time": "2022-11-12", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956535} -{"stream": "segments_analytics", "data": {"time": "2022-11-13", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956537} -{"stream": "segments_analytics", "data": {"time": "2022-11-14", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956539} -{"stream": "segments_analytics", "data": {"time": "2022-11-15", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956541} -{"stream": "segments_analytics", "data": {"time": "2022-11-16", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956543} -{"stream": "segments_analytics", "data": {"time": "2022-11-17", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956545} -{"stream": "segments_analytics", "data": {"time": "2022-11-18", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956547} -{"stream": "segments_analytics", "data": {"time": "2022-11-19", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956549} -{"stream": "segments_analytics", "data": {"time": "2022-11-20", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956552} -{"stream": "segments_analytics", "data": {"time": "2022-11-21", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956554} -{"stream": "segments_analytics", "data": {"time": "2022-11-22", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956556} -{"stream": "segments_analytics", "data": {"time": "2022-11-23", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956558} -{"stream": "segments_analytics", "data": {"time": "2022-11-24", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956560} -{"stream": "segments_analytics", "data": {"time": "2022-11-25", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956562} -{"stream": "segments_analytics", "data": {"time": "2022-11-26", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956564} -{"stream": "segments_analytics", "data": {"time": "2022-11-27", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956566} -{"stream": "segments_analytics", "data": {"time": "2022-11-28", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956568} -{"stream": "segments_analytics", "data": {"time": "2022-11-29", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956570} -{"stream": "segments_analytics", "data": {"time": "2022-11-30", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956573} -{"stream": "segments_analytics", "data": {"time": "2022-12-01", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956575} -{"stream": "segments_analytics", "data": {"time": "2022-12-02", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956577} -{"stream": "segments_analytics", "data": {"time": "2022-12-03", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956579} -{"stream": "segments_analytics", "data": {"time": "2022-12-04", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956581} -{"stream": "segments_analytics", "data": {"time": "2022-12-05", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956583} -{"stream": "segments_analytics", "data": {"time": "2022-12-06", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956585} -{"stream": "segments_analytics", "data": {"time": "2022-12-07", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956587} -{"stream": "segments_analytics", "data": {"time": "2022-12-08", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100956589} -{"stream": "segments_analytics", "data": {"time": "2022-12-09", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100957063} -{"stream": "segments_analytics", "data": {"time": "2022-12-10", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100957072} -{"stream": "segments_analytics", "data": {"time": "2022-12-11", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100957080} -{"stream": "segments_analytics", "data": {"time": "2022-12-12", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100957085} -{"stream": "segments_analytics", "data": {"time": "2022-12-13", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100957090} -{"stream": "segments_analytics", "data": {"time": "2022-12-14", "size": 0, "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a"}, "emitted_at": 1671100957093} -{"stream": "segments_analytics", "data": {"time": "2022-09-01", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957744} -{"stream": "segments_analytics", "data": {"time": "2022-09-02", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957746} -{"stream": "segments_analytics", "data": {"time": "2022-09-03", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957748} -{"stream": "segments_analytics", "data": {"time": "2022-09-04", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957750} -{"stream": "segments_analytics", "data": {"time": "2022-09-05", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957753} -{"stream": "segments_analytics", "data": {"time": "2022-09-06", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957755} -{"stream": "segments_analytics", "data": {"time": "2022-09-07", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957757} -{"stream": "segments_analytics", "data": {"time": "2022-09-08", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957759} -{"stream": "segments_analytics", "data": {"time": "2022-09-09", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957761} -{"stream": "segments_analytics", "data": {"time": "2022-09-10", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957763} -{"stream": "segments_analytics", "data": {"time": "2022-09-11", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957765} -{"stream": "segments_analytics", "data": {"time": "2022-09-12", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957767} -{"stream": "segments_analytics", "data": {"time": "2022-09-13", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957769} -{"stream": "segments_analytics", "data": {"time": "2022-09-14", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957772} -{"stream": "segments_analytics", "data": {"time": "2022-09-15", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957774} -{"stream": "segments_analytics", "data": {"time": "2022-09-16", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957776} -{"stream": "segments_analytics", "data": {"time": "2022-09-17", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957778} -{"stream": "segments_analytics", "data": {"time": "2022-09-18", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957780} -{"stream": "segments_analytics", "data": {"time": "2022-09-19", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957782} -{"stream": "segments_analytics", "data": {"time": "2022-09-20", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957784} -{"stream": "segments_analytics", "data": {"time": "2022-09-21", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957786} -{"stream": "segments_analytics", "data": {"time": "2022-09-22", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957788} -{"stream": "segments_analytics", "data": {"time": "2022-09-23", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957791} -{"stream": "segments_analytics", "data": {"time": "2022-09-24", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957793} -{"stream": "segments_analytics", "data": {"time": "2022-09-25", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957795} -{"stream": "segments_analytics", "data": {"time": "2022-09-26", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957797} -{"stream": "segments_analytics", "data": {"time": "2022-09-27", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957799} -{"stream": "segments_analytics", "data": {"time": "2022-09-28", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957801} -{"stream": "segments_analytics", "data": {"time": "2022-09-29", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957803} -{"stream": "segments_analytics", "data": {"time": "2022-09-30", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957805} -{"stream": "segments_analytics", "data": {"time": "2022-10-01", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957808} -{"stream": "segments_analytics", "data": {"time": "2022-10-02", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957810} -{"stream": "segments_analytics", "data": {"time": "2022-10-03", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957812} -{"stream": "segments_analytics", "data": {"time": "2022-10-04", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957814} -{"stream": "segments_analytics", "data": {"time": "2022-10-05", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957817} -{"stream": "segments_analytics", "data": {"time": "2022-10-06", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957819} -{"stream": "segments_analytics", "data": {"time": "2022-10-07", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957821} -{"stream": "segments_analytics", "data": {"time": "2022-10-08", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957823} -{"stream": "segments_analytics", "data": {"time": "2022-10-09", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957825} -{"stream": "segments_analytics", "data": {"time": "2022-10-10", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957828} -{"stream": "segments_analytics", "data": {"time": "2022-10-11", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957830} -{"stream": "segments_analytics", "data": {"time": "2022-10-12", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957832} -{"stream": "segments_analytics", "data": {"time": "2022-10-13", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957834} -{"stream": "segments_analytics", "data": {"time": "2022-10-14", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957836} -{"stream": "segments_analytics", "data": {"time": "2022-10-15", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957838} -{"stream": "segments_analytics", "data": {"time": "2022-10-16", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957840} -{"stream": "segments_analytics", "data": {"time": "2022-10-17", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957842} -{"stream": "segments_analytics", "data": {"time": "2022-10-18", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957844} -{"stream": "segments_analytics", "data": {"time": "2022-10-19", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957847} -{"stream": "segments_analytics", "data": {"time": "2022-10-20", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957849} -{"stream": "segments_analytics", "data": {"time": "2022-10-21", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957851} -{"stream": "segments_analytics", "data": {"time": "2022-10-22", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957853} -{"stream": "segments_analytics", "data": {"time": "2022-10-23", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957855} -{"stream": "segments_analytics", "data": {"time": "2022-10-24", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957857} -{"stream": "segments_analytics", "data": {"time": "2022-10-25", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957859} -{"stream": "segments_analytics", "data": {"time": "2022-10-26", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957862} -{"stream": "segments_analytics", "data": {"time": "2022-10-27", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957864} -{"stream": "segments_analytics", "data": {"time": "2022-10-28", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957866} -{"stream": "segments_analytics", "data": {"time": "2022-10-29", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957868} -{"stream": "segments_analytics", "data": {"time": "2022-10-30", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957870} -{"stream": "segments_analytics", "data": {"time": "2022-10-31", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957872} -{"stream": "segments_analytics", "data": {"time": "2022-11-01", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957874} -{"stream": "segments_analytics", "data": {"time": "2022-11-02", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957876} -{"stream": "segments_analytics", "data": {"time": "2022-11-03", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957878} -{"stream": "segments_analytics", "data": {"time": "2022-11-04", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957881} -{"stream": "segments_analytics", "data": {"time": "2022-11-05", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957883} -{"stream": "segments_analytics", "data": {"time": "2022-11-06", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957885} -{"stream": "segments_analytics", "data": {"time": "2022-11-07", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957887} -{"stream": "segments_analytics", "data": {"time": "2022-11-08", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957889} -{"stream": "segments_analytics", "data": {"time": "2022-11-09", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957891} -{"stream": "segments_analytics", "data": {"time": "2022-11-10", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957894} -{"stream": "segments_analytics", "data": {"time": "2022-11-11", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957896} -{"stream": "segments_analytics", "data": {"time": "2022-11-12", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957898} -{"stream": "segments_analytics", "data": {"time": "2022-11-13", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957900} -{"stream": "segments_analytics", "data": {"time": "2022-11-14", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957902} -{"stream": "segments_analytics", "data": {"time": "2022-11-15", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957904} -{"stream": "segments_analytics", "data": {"time": "2022-11-16", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957906} -{"stream": "segments_analytics", "data": {"time": "2022-11-17", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957908} -{"stream": "segments_analytics", "data": {"time": "2022-11-18", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957911} -{"stream": "segments_analytics", "data": {"time": "2022-11-19", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957913} -{"stream": "segments_analytics", "data": {"time": "2022-11-20", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957915} -{"stream": "segments_analytics", "data": {"time": "2022-11-21", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957917} -{"stream": "segments_analytics", "data": {"time": "2022-11-22", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957919} -{"stream": "segments_analytics", "data": {"time": "2022-11-23", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957921} -{"stream": "segments_analytics", "data": {"time": "2022-11-24", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957923} -{"stream": "segments_analytics", "data": {"time": "2022-11-25", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957925} -{"stream": "segments_analytics", "data": {"time": "2022-11-26", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957928} -{"stream": "segments_analytics", "data": {"time": "2022-11-27", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957930} -{"stream": "segments_analytics", "data": {"time": "2022-11-28", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957932} -{"stream": "segments_analytics", "data": {"time": "2022-11-29", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957934} -{"stream": "segments_analytics", "data": {"time": "2022-11-30", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957936} -{"stream": "segments_analytics", "data": {"time": "2022-12-01", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957938} -{"stream": "segments_analytics", "data": {"time": "2022-12-02", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957940} -{"stream": "segments_analytics", "data": {"time": "2022-12-03", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957943} -{"stream": "segments_analytics", "data": {"time": "2022-12-04", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957945} -{"stream": "segments_analytics", "data": {"time": "2022-12-05", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957947} -{"stream": "segments_analytics", "data": {"time": "2022-12-06", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957950} -{"stream": "segments_analytics", "data": {"time": "2022-12-07", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957952} -{"stream": "segments_analytics", "data": {"time": "2022-12-08", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100957954} -{"stream": "segments_analytics", "data": {"time": "2022-12-09", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100958564} -{"stream": "segments_analytics", "data": {"time": "2022-12-10", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100958566} -{"stream": "segments_analytics", "data": {"time": "2022-12-11", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100958568} -{"stream": "segments_analytics", "data": {"time": "2022-12-12", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100958570} -{"stream": "segments_analytics", "data": {"time": "2022-12-13", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100958572} -{"stream": "segments_analytics", "data": {"time": "2022-12-14", "size": 0, "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306"}, "emitted_at": 1671100958574} -{"stream": "segments_analytics", "data": {"time": "2022-09-01", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959178} -{"stream": "segments_analytics", "data": {"time": "2022-09-02", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959180} -{"stream": "segments_analytics", "data": {"time": "2022-09-03", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959182} -{"stream": "segments_analytics", "data": {"time": "2022-09-04", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959184} -{"stream": "segments_analytics", "data": {"time": "2022-09-05", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959187} -{"stream": "segments_analytics", "data": {"time": "2022-09-06", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959189} -{"stream": "segments_analytics", "data": {"time": "2022-09-07", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959191} -{"stream": "segments_analytics", "data": {"time": "2022-09-08", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959193} -{"stream": "segments_analytics", "data": {"time": "2022-09-09", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959195} -{"stream": "segments_analytics", "data": {"time": "2022-09-10", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959197} -{"stream": "segments_analytics", "data": {"time": "2022-09-11", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959199} -{"stream": "segments_analytics", "data": {"time": "2022-09-12", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959202} -{"stream": "segments_analytics", "data": {"time": "2022-09-13", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959204} -{"stream": "segments_analytics", "data": {"time": "2022-09-14", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959206} -{"stream": "segments_analytics", "data": {"time": "2022-09-15", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959208} -{"stream": "segments_analytics", "data": {"time": "2022-09-16", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959210} -{"stream": "segments_analytics", "data": {"time": "2022-09-17", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959212} -{"stream": "segments_analytics", "data": {"time": "2022-09-18", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959214} -{"stream": "segments_analytics", "data": {"time": "2022-09-19", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959216} -{"stream": "segments_analytics", "data": {"time": "2022-09-20", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959219} -{"stream": "segments_analytics", "data": {"time": "2022-09-21", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959221} -{"stream": "segments_analytics", "data": {"time": "2022-09-22", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959223} -{"stream": "segments_analytics", "data": {"time": "2022-09-23", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959225} -{"stream": "segments_analytics", "data": {"time": "2022-09-24", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959227} -{"stream": "segments_analytics", "data": {"time": "2022-09-25", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959229} -{"stream": "segments_analytics", "data": {"time": "2022-09-26", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959231} -{"stream": "segments_analytics", "data": {"time": "2022-09-27", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959234} -{"stream": "segments_analytics", "data": {"time": "2022-09-28", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959236} -{"stream": "segments_analytics", "data": {"time": "2022-09-29", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959238} -{"stream": "segments_analytics", "data": {"time": "2022-09-30", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959240} -{"stream": "segments_analytics", "data": {"time": "2022-10-01", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959242} -{"stream": "segments_analytics", "data": {"time": "2022-10-02", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959245} -{"stream": "segments_analytics", "data": {"time": "2022-10-03", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959247} -{"stream": "segments_analytics", "data": {"time": "2022-10-04", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959249} -{"stream": "segments_analytics", "data": {"time": "2022-10-05", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959251} -{"stream": "segments_analytics", "data": {"time": "2022-10-06", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959253} -{"stream": "segments_analytics", "data": {"time": "2022-10-07", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959255} -{"stream": "segments_analytics", "data": {"time": "2022-10-08", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959257} -{"stream": "segments_analytics", "data": {"time": "2022-10-09", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959259} -{"stream": "segments_analytics", "data": {"time": "2022-10-10", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959262} -{"stream": "segments_analytics", "data": {"time": "2022-10-11", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959264} -{"stream": "segments_analytics", "data": {"time": "2022-10-12", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959266} -{"stream": "segments_analytics", "data": {"time": "2022-10-13", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959268} -{"stream": "segments_analytics", "data": {"time": "2022-10-14", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959270} -{"stream": "segments_analytics", "data": {"time": "2022-10-15", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959272} -{"stream": "segments_analytics", "data": {"time": "2022-10-16", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959274} -{"stream": "segments_analytics", "data": {"time": "2022-10-17", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959277} -{"stream": "segments_analytics", "data": {"time": "2022-10-18", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959279} -{"stream": "segments_analytics", "data": {"time": "2022-10-19", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959281} -{"stream": "segments_analytics", "data": {"time": "2022-10-20", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959283} -{"stream": "segments_analytics", "data": {"time": "2022-10-21", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959285} -{"stream": "segments_analytics", "data": {"time": "2022-10-22", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959287} -{"stream": "segments_analytics", "data": {"time": "2022-10-23", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959290} -{"stream": "segments_analytics", "data": {"time": "2022-10-24", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959292} -{"stream": "segments_analytics", "data": {"time": "2022-10-25", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959294} -{"stream": "segments_analytics", "data": {"time": "2022-10-26", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959296} -{"stream": "segments_analytics", "data": {"time": "2022-10-27", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959298} -{"stream": "segments_analytics", "data": {"time": "2022-10-28", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959301} -{"stream": "segments_analytics", "data": {"time": "2022-10-29", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959304} -{"stream": "segments_analytics", "data": {"time": "2022-10-30", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959307} -{"stream": "segments_analytics", "data": {"time": "2022-10-31", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959310} -{"stream": "segments_analytics", "data": {"time": "2022-11-01", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959313} -{"stream": "segments_analytics", "data": {"time": "2022-11-02", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959315} -{"stream": "segments_analytics", "data": {"time": "2022-11-03", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959317} -{"stream": "segments_analytics", "data": {"time": "2022-11-04", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959319} -{"stream": "segments_analytics", "data": {"time": "2022-11-05", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959321} -{"stream": "segments_analytics", "data": {"time": "2022-11-06", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959323} -{"stream": "segments_analytics", "data": {"time": "2022-11-07", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959326} -{"stream": "segments_analytics", "data": {"time": "2022-11-08", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959328} -{"stream": "segments_analytics", "data": {"time": "2022-11-09", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959330} -{"stream": "segments_analytics", "data": {"time": "2022-11-10", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959332} -{"stream": "segments_analytics", "data": {"time": "2022-11-11", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959334} -{"stream": "segments_analytics", "data": {"time": "2022-11-12", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959336} -{"stream": "segments_analytics", "data": {"time": "2022-11-13", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959338} -{"stream": "segments_analytics", "data": {"time": "2022-11-14", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959340} -{"stream": "segments_analytics", "data": {"time": "2022-11-15", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959343} -{"stream": "segments_analytics", "data": {"time": "2022-11-16", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959345} -{"stream": "segments_analytics", "data": {"time": "2022-11-17", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959347} -{"stream": "segments_analytics", "data": {"time": "2022-11-18", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959349} -{"stream": "segments_analytics", "data": {"time": "2022-11-19", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959351} -{"stream": "segments_analytics", "data": {"time": "2022-11-20", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959353} -{"stream": "segments_analytics", "data": {"time": "2022-11-21", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959356} -{"stream": "segments_analytics", "data": {"time": "2022-11-22", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959358} -{"stream": "segments_analytics", "data": {"time": "2022-11-23", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959360} -{"stream": "segments_analytics", "data": {"time": "2022-11-24", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959362} -{"stream": "segments_analytics", "data": {"time": "2022-11-25", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959364} -{"stream": "segments_analytics", "data": {"time": "2022-11-26", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959366} -{"stream": "segments_analytics", "data": {"time": "2022-11-27", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959368} -{"stream": "segments_analytics", "data": {"time": "2022-11-28", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959370} -{"stream": "segments_analytics", "data": {"time": "2022-11-29", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959373} -{"stream": "segments_analytics", "data": {"time": "2022-11-30", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959375} -{"stream": "segments_analytics", "data": {"time": "2022-12-01", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959377} -{"stream": "segments_analytics", "data": {"time": "2022-12-02", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959379} -{"stream": "segments_analytics", "data": {"time": "2022-12-03", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959381} -{"stream": "segments_analytics", "data": {"time": "2022-12-04", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959383} -{"stream": "segments_analytics", "data": {"time": "2022-12-05", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959385} -{"stream": "segments_analytics", "data": {"time": "2022-12-06", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959387} -{"stream": "segments_analytics", "data": {"time": "2022-12-07", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959389} -{"stream": "segments_analytics", "data": {"time": "2022-12-08", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959392} -{"stream": "segments_analytics", "data": {"time": "2022-12-09", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959892} -{"stream": "segments_analytics", "data": {"time": "2022-12-10", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959894} -{"stream": "segments_analytics", "data": {"time": "2022-12-11", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959896} -{"stream": "segments_analytics", "data": {"time": "2022-12-12", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959898} -{"stream": "segments_analytics", "data": {"time": "2022-12-13", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959900} -{"stream": "segments_analytics", "data": {"time": "2022-12-14", "size": 0, "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4"}, "emitted_at": 1671100959902} -{"stream": "segments_analytics", "data": {"time": "2022-09-01", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960509} -{"stream": "segments_analytics", "data": {"time": "2022-09-02", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960512} -{"stream": "segments_analytics", "data": {"time": "2022-09-03", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960514} -{"stream": "segments_analytics", "data": {"time": "2022-09-04", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960516} -{"stream": "segments_analytics", "data": {"time": "2022-09-05", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960518} -{"stream": "segments_analytics", "data": {"time": "2022-09-06", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960520} -{"stream": "segments_analytics", "data": {"time": "2022-09-07", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960522} -{"stream": "segments_analytics", "data": {"time": "2022-09-08", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960524} -{"stream": "segments_analytics", "data": {"time": "2022-09-09", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960526} -{"stream": "segments_analytics", "data": {"time": "2022-09-10", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960529} -{"stream": "segments_analytics", "data": {"time": "2022-09-11", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960531} -{"stream": "segments_analytics", "data": {"time": "2022-09-12", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960533} -{"stream": "segments_analytics", "data": {"time": "2022-09-13", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960535} -{"stream": "segments_analytics", "data": {"time": "2022-09-14", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960537} -{"stream": "segments_analytics", "data": {"time": "2022-09-15", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960539} -{"stream": "segments_analytics", "data": {"time": "2022-09-16", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960541} -{"stream": "segments_analytics", "data": {"time": "2022-09-17", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960543} -{"stream": "segments_analytics", "data": {"time": "2022-09-18", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960546} -{"stream": "segments_analytics", "data": {"time": "2022-09-19", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960548} -{"stream": "segments_analytics", "data": {"time": "2022-09-20", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960550} -{"stream": "segments_analytics", "data": {"time": "2022-09-21", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960552} -{"stream": "segments_analytics", "data": {"time": "2022-09-22", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960554} -{"stream": "segments_analytics", "data": {"time": "2022-09-23", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960556} -{"stream": "segments_analytics", "data": {"time": "2022-09-24", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960558} -{"stream": "segments_analytics", "data": {"time": "2022-09-25", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960560} -{"stream": "segments_analytics", "data": {"time": "2022-09-26", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960562} -{"stream": "segments_analytics", "data": {"time": "2022-09-27", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960565} -{"stream": "segments_analytics", "data": {"time": "2022-09-28", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960567} -{"stream": "segments_analytics", "data": {"time": "2022-09-29", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960569} -{"stream": "segments_analytics", "data": {"time": "2022-09-30", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960571} -{"stream": "segments_analytics", "data": {"time": "2022-10-01", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960573} -{"stream": "segments_analytics", "data": {"time": "2022-10-02", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960575} -{"stream": "segments_analytics", "data": {"time": "2022-10-03", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960577} -{"stream": "segments_analytics", "data": {"time": "2022-10-04", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960579} -{"stream": "segments_analytics", "data": {"time": "2022-10-05", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960582} -{"stream": "segments_analytics", "data": {"time": "2022-10-06", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960584} -{"stream": "segments_analytics", "data": {"time": "2022-10-07", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960586} -{"stream": "segments_analytics", "data": {"time": "2022-10-08", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960588} -{"stream": "segments_analytics", "data": {"time": "2022-10-09", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960590} -{"stream": "segments_analytics", "data": {"time": "2022-10-10", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960592} -{"stream": "segments_analytics", "data": {"time": "2022-10-11", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960594} -{"stream": "segments_analytics", "data": {"time": "2022-10-12", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960596} -{"stream": "segments_analytics", "data": {"time": "2022-10-13", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960598} -{"stream": "segments_analytics", "data": {"time": "2022-10-14", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960600} -{"stream": "segments_analytics", "data": {"time": "2022-10-15", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960603} -{"stream": "segments_analytics", "data": {"time": "2022-10-16", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960605} -{"stream": "segments_analytics", "data": {"time": "2022-10-17", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960607} -{"stream": "segments_analytics", "data": {"time": "2022-10-18", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960609} -{"stream": "segments_analytics", "data": {"time": "2022-10-19", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960611} -{"stream": "segments_analytics", "data": {"time": "2022-10-20", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960613} -{"stream": "segments_analytics", "data": {"time": "2022-10-21", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960615} -{"stream": "segments_analytics", "data": {"time": "2022-10-22", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960617} -{"stream": "segments_analytics", "data": {"time": "2022-10-23", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960619} -{"stream": "segments_analytics", "data": {"time": "2022-10-24", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960622} -{"stream": "segments_analytics", "data": {"time": "2022-10-25", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960624} -{"stream": "segments_analytics", "data": {"time": "2022-10-26", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960626} -{"stream": "segments_analytics", "data": {"time": "2022-10-27", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960628} -{"stream": "segments_analytics", "data": {"time": "2022-10-28", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960630} -{"stream": "segments_analytics", "data": {"time": "2022-10-29", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960632} -{"stream": "segments_analytics", "data": {"time": "2022-10-30", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960634} -{"stream": "segments_analytics", "data": {"time": "2022-10-31", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960637} -{"stream": "segments_analytics", "data": {"time": "2022-11-01", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960639} -{"stream": "segments_analytics", "data": {"time": "2022-11-02", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960641} -{"stream": "segments_analytics", "data": {"time": "2022-11-03", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960643} -{"stream": "segments_analytics", "data": {"time": "2022-11-04", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960645} -{"stream": "segments_analytics", "data": {"time": "2022-11-05", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960647} -{"stream": "segments_analytics", "data": {"time": "2022-11-06", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960649} -{"stream": "segments_analytics", "data": {"time": "2022-11-07", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960651} -{"stream": "segments_analytics", "data": {"time": "2022-11-08", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960653} -{"stream": "segments_analytics", "data": {"time": "2022-11-09", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960655} -{"stream": "segments_analytics", "data": {"time": "2022-11-10", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960658} -{"stream": "segments_analytics", "data": {"time": "2022-11-11", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960660} -{"stream": "segments_analytics", "data": {"time": "2022-11-12", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960662} -{"stream": "segments_analytics", "data": {"time": "2022-11-13", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960664} -{"stream": "segments_analytics", "data": {"time": "2022-11-14", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960666} -{"stream": "segments_analytics", "data": {"time": "2022-11-15", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960668} -{"stream": "segments_analytics", "data": {"time": "2022-11-16", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960670} -{"stream": "segments_analytics", "data": {"time": "2022-11-17", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960672} -{"stream": "segments_analytics", "data": {"time": "2022-11-18", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960674} -{"stream": "segments_analytics", "data": {"time": "2022-11-19", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960677} -{"stream": "segments_analytics", "data": {"time": "2022-11-20", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960679} -{"stream": "segments_analytics", "data": {"time": "2022-11-21", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960681} -{"stream": "segments_analytics", "data": {"time": "2022-11-22", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960683} -{"stream": "segments_analytics", "data": {"time": "2022-11-23", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960685} -{"stream": "segments_analytics", "data": {"time": "2022-11-24", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960687} -{"stream": "segments_analytics", "data": {"time": "2022-11-25", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960689} -{"stream": "segments_analytics", "data": {"time": "2022-11-26", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960691} -{"stream": "segments_analytics", "data": {"time": "2022-11-27", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960694} -{"stream": "segments_analytics", "data": {"time": "2022-11-28", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960696} -{"stream": "segments_analytics", "data": {"time": "2022-11-29", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960698} -{"stream": "segments_analytics", "data": {"time": "2022-11-30", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960700} -{"stream": "segments_analytics", "data": {"time": "2022-12-01", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960702} -{"stream": "segments_analytics", "data": {"time": "2022-12-02", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960704} -{"stream": "segments_analytics", "data": {"time": "2022-12-03", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960706} -{"stream": "segments_analytics", "data": {"time": "2022-12-04", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960709} -{"stream": "segments_analytics", "data": {"time": "2022-12-05", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960711} -{"stream": "segments_analytics", "data": {"time": "2022-12-06", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960713} -{"stream": "segments_analytics", "data": {"time": "2022-12-07", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960715} -{"stream": "segments_analytics", "data": {"time": "2022-12-08", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100960717} -{"stream": "segments_analytics", "data": {"time": "2022-12-09", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100961329} -{"stream": "segments_analytics", "data": {"time": "2022-12-10", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100961331} -{"stream": "segments_analytics", "data": {"time": "2022-12-11", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100961333} -{"stream": "segments_analytics", "data": {"time": "2022-12-12", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100961335} -{"stream": "segments_analytics", "data": {"time": "2022-12-13", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100961337} -{"stream": "segments_analytics", "data": {"time": "2022-12-14", "size": 0, "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646"}, "emitted_at": 1671100961339} -{"stream": "segments_analytics", "data": {"time": "2022-09-01", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961943} -{"stream": "segments_analytics", "data": {"time": "2022-09-02", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961945} -{"stream": "segments_analytics", "data": {"time": "2022-09-03", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961947} -{"stream": "segments_analytics", "data": {"time": "2022-09-04", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961949} -{"stream": "segments_analytics", "data": {"time": "2022-09-05", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961951} -{"stream": "segments_analytics", "data": {"time": "2022-09-06", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961953} -{"stream": "segments_analytics", "data": {"time": "2022-09-07", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961955} -{"stream": "segments_analytics", "data": {"time": "2022-09-08", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961958} -{"stream": "segments_analytics", "data": {"time": "2022-09-09", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961960} -{"stream": "segments_analytics", "data": {"time": "2022-09-10", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961962} -{"stream": "segments_analytics", "data": {"time": "2022-09-11", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961964} -{"stream": "segments_analytics", "data": {"time": "2022-09-12", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961966} -{"stream": "segments_analytics", "data": {"time": "2022-09-13", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961968} -{"stream": "segments_analytics", "data": {"time": "2022-09-14", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961970} -{"stream": "segments_analytics", "data": {"time": "2022-09-15", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961972} -{"stream": "segments_analytics", "data": {"time": "2022-09-16", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961974} -{"stream": "segments_analytics", "data": {"time": "2022-09-17", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961977} -{"stream": "segments_analytics", "data": {"time": "2022-09-18", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961979} -{"stream": "segments_analytics", "data": {"time": "2022-09-19", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961981} -{"stream": "segments_analytics", "data": {"time": "2022-09-20", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961983} -{"stream": "segments_analytics", "data": {"time": "2022-09-21", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961985} -{"stream": "segments_analytics", "data": {"time": "2022-09-22", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961987} -{"stream": "segments_analytics", "data": {"time": "2022-09-23", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961989} -{"stream": "segments_analytics", "data": {"time": "2022-09-24", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961991} -{"stream": "segments_analytics", "data": {"time": "2022-09-25", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961994} -{"stream": "segments_analytics", "data": {"time": "2022-09-26", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961996} -{"stream": "segments_analytics", "data": {"time": "2022-09-27", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100961998} -{"stream": "segments_analytics", "data": {"time": "2022-09-28", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962000} -{"stream": "segments_analytics", "data": {"time": "2022-09-29", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962002} -{"stream": "segments_analytics", "data": {"time": "2022-09-30", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962004} -{"stream": "segments_analytics", "data": {"time": "2022-10-01", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962006} -{"stream": "segments_analytics", "data": {"time": "2022-10-02", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962008} -{"stream": "segments_analytics", "data": {"time": "2022-10-03", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962011} -{"stream": "segments_analytics", "data": {"time": "2022-10-04", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962013} -{"stream": "segments_analytics", "data": {"time": "2022-10-05", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962015} -{"stream": "segments_analytics", "data": {"time": "2022-10-06", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962017} -{"stream": "segments_analytics", "data": {"time": "2022-10-07", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962019} -{"stream": "segments_analytics", "data": {"time": "2022-10-08", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962021} -{"stream": "segments_analytics", "data": {"time": "2022-10-09", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962023} -{"stream": "segments_analytics", "data": {"time": "2022-10-10", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962025} -{"stream": "segments_analytics", "data": {"time": "2022-10-11", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962027} -{"stream": "segments_analytics", "data": {"time": "2022-10-12", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962030} -{"stream": "segments_analytics", "data": {"time": "2022-10-13", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962032} -{"stream": "segments_analytics", "data": {"time": "2022-10-14", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962034} -{"stream": "segments_analytics", "data": {"time": "2022-10-15", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962036} -{"stream": "segments_analytics", "data": {"time": "2022-10-16", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962038} -{"stream": "segments_analytics", "data": {"time": "2022-10-17", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962040} -{"stream": "segments_analytics", "data": {"time": "2022-10-18", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962042} -{"stream": "segments_analytics", "data": {"time": "2022-10-19", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962044} -{"stream": "segments_analytics", "data": {"time": "2022-10-20", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962047} -{"stream": "segments_analytics", "data": {"time": "2022-10-21", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962049} -{"stream": "segments_analytics", "data": {"time": "2022-10-22", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962051} -{"stream": "segments_analytics", "data": {"time": "2022-10-23", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962053} -{"stream": "segments_analytics", "data": {"time": "2022-10-24", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962055} -{"stream": "segments_analytics", "data": {"time": "2022-10-25", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962057} -{"stream": "segments_analytics", "data": {"time": "2022-10-26", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962059} -{"stream": "segments_analytics", "data": {"time": "2022-10-27", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962061} -{"stream": "segments_analytics", "data": {"time": "2022-10-28", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962063} -{"stream": "segments_analytics", "data": {"time": "2022-10-29", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962066} -{"stream": "segments_analytics", "data": {"time": "2022-10-30", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962068} -{"stream": "segments_analytics", "data": {"time": "2022-10-31", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962070} -{"stream": "segments_analytics", "data": {"time": "2022-11-01", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962072} -{"stream": "segments_analytics", "data": {"time": "2022-11-02", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962074} -{"stream": "segments_analytics", "data": {"time": "2022-11-03", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962076} -{"stream": "segments_analytics", "data": {"time": "2022-11-04", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962078} -{"stream": "segments_analytics", "data": {"time": "2022-11-05", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962080} -{"stream": "segments_analytics", "data": {"time": "2022-11-06", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962083} -{"stream": "segments_analytics", "data": {"time": "2022-11-07", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962085} -{"stream": "segments_analytics", "data": {"time": "2022-11-08", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962087} -{"stream": "segments_analytics", "data": {"time": "2022-11-09", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962089} -{"stream": "segments_analytics", "data": {"time": "2022-11-10", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962091} -{"stream": "segments_analytics", "data": {"time": "2022-11-11", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962093} -{"stream": "segments_analytics", "data": {"time": "2022-11-12", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962095} -{"stream": "segments_analytics", "data": {"time": "2022-11-13", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962097} -{"stream": "segments_analytics", "data": {"time": "2022-11-14", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962099} -{"stream": "segments_analytics", "data": {"time": "2022-11-15", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962101} -{"stream": "segments_analytics", "data": {"time": "2022-11-16", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962104} -{"stream": "segments_analytics", "data": {"time": "2022-11-17", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962106} -{"stream": "segments_analytics", "data": {"time": "2022-11-18", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962108} -{"stream": "segments_analytics", "data": {"time": "2022-11-19", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962110} -{"stream": "segments_analytics", "data": {"time": "2022-11-20", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962112} -{"stream": "segments_analytics", "data": {"time": "2022-11-21", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962114} -{"stream": "segments_analytics", "data": {"time": "2022-11-22", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962116} -{"stream": "segments_analytics", "data": {"time": "2022-11-23", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962118} -{"stream": "segments_analytics", "data": {"time": "2022-11-24", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962121} -{"stream": "segments_analytics", "data": {"time": "2022-11-25", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962123} -{"stream": "segments_analytics", "data": {"time": "2022-11-26", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962125} -{"stream": "segments_analytics", "data": {"time": "2022-11-27", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962127} -{"stream": "segments_analytics", "data": {"time": "2022-11-28", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962129} -{"stream": "segments_analytics", "data": {"time": "2022-11-29", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962131} -{"stream": "segments_analytics", "data": {"time": "2022-11-30", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962133} -{"stream": "segments_analytics", "data": {"time": "2022-12-01", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962136} -{"stream": "segments_analytics", "data": {"time": "2022-12-02", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962138} -{"stream": "segments_analytics", "data": {"time": "2022-12-03", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962140} -{"stream": "segments_analytics", "data": {"time": "2022-12-04", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962142} -{"stream": "segments_analytics", "data": {"time": "2022-12-05", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962144} -{"stream": "segments_analytics", "data": {"time": "2022-12-06", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962146} -{"stream": "segments_analytics", "data": {"time": "2022-12-07", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962148} -{"stream": "segments_analytics", "data": {"time": "2022-12-08", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962150} -{"stream": "segments_analytics", "data": {"time": "2022-12-09", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962763} -{"stream": "segments_analytics", "data": {"time": "2022-12-10", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962765} -{"stream": "segments_analytics", "data": {"time": "2022-12-11", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962767} -{"stream": "segments_analytics", "data": {"time": "2022-12-12", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962769} -{"stream": "segments_analytics", "data": {"time": "2022-12-13", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962771} -{"stream": "segments_analytics", "data": {"time": "2022-12-14", "size": 0, "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de"}, "emitted_at": 1671100962773} -{"stream": "segments_analytics", "data": {"time": "2022-09-01", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963025} -{"stream": "segments_analytics", "data": {"time": "2022-09-02", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963033} -{"stream": "segments_analytics", "data": {"time": "2022-09-03", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963041} -{"stream": "segments_analytics", "data": {"time": "2022-09-04", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963046} -{"stream": "segments_analytics", "data": {"time": "2022-09-05", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963050} -{"stream": "segments_analytics", "data": {"time": "2022-09-06", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963054} -{"stream": "segments_analytics", "data": {"time": "2022-09-07", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963057} -{"stream": "segments_analytics", "data": {"time": "2022-09-08", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963060} -{"stream": "segments_analytics", "data": {"time": "2022-09-09", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963063} -{"stream": "segments_analytics", "data": {"time": "2022-09-10", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963065} -{"stream": "segments_analytics", "data": {"time": "2022-09-11", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963068} -{"stream": "segments_analytics", "data": {"time": "2022-09-12", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963070} -{"stream": "segments_analytics", "data": {"time": "2022-09-13", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963072} -{"stream": "segments_analytics", "data": {"time": "2022-09-14", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963074} -{"stream": "segments_analytics", "data": {"time": "2022-09-15", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963076} -{"stream": "segments_analytics", "data": {"time": "2022-09-16", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963079} -{"stream": "segments_analytics", "data": {"time": "2022-09-17", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963081} -{"stream": "segments_analytics", "data": {"time": "2022-09-18", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963083} -{"stream": "segments_analytics", "data": {"time": "2022-09-19", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963085} -{"stream": "segments_analytics", "data": {"time": "2022-09-20", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963087} -{"stream": "segments_analytics", "data": {"time": "2022-09-21", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963089} -{"stream": "segments_analytics", "data": {"time": "2022-09-22", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963091} -{"stream": "segments_analytics", "data": {"time": "2022-09-23", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963093} -{"stream": "segments_analytics", "data": {"time": "2022-09-24", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963095} -{"stream": "segments_analytics", "data": {"time": "2022-09-25", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963097} -{"stream": "segments_analytics", "data": {"time": "2022-09-26", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963100} -{"stream": "segments_analytics", "data": {"time": "2022-09-27", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963102} -{"stream": "segments_analytics", "data": {"time": "2022-09-28", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963104} -{"stream": "segments_analytics", "data": {"time": "2022-09-29", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963106} -{"stream": "segments_analytics", "data": {"time": "2022-09-30", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963108} -{"stream": "segments_analytics", "data": {"time": "2022-10-01", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963110} -{"stream": "segments_analytics", "data": {"time": "2022-10-02", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963113} -{"stream": "segments_analytics", "data": {"time": "2022-10-03", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963115} -{"stream": "segments_analytics", "data": {"time": "2022-10-04", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963117} -{"stream": "segments_analytics", "data": {"time": "2022-10-05", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963119} -{"stream": "segments_analytics", "data": {"time": "2022-10-06", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963121} -{"stream": "segments_analytics", "data": {"time": "2022-10-07", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963123} -{"stream": "segments_analytics", "data": {"time": "2022-10-08", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963125} -{"stream": "segments_analytics", "data": {"time": "2022-10-09", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963127} -{"stream": "segments_analytics", "data": {"time": "2022-10-10", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963130} -{"stream": "segments_analytics", "data": {"time": "2022-10-11", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963132} -{"stream": "segments_analytics", "data": {"time": "2022-10-12", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963134} -{"stream": "segments_analytics", "data": {"time": "2022-10-13", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963136} -{"stream": "segments_analytics", "data": {"time": "2022-10-14", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963138} -{"stream": "segments_analytics", "data": {"time": "2022-10-15", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963140} -{"stream": "segments_analytics", "data": {"time": "2022-10-16", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963143} -{"stream": "segments_analytics", "data": {"time": "2022-10-17", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963145} -{"stream": "segments_analytics", "data": {"time": "2022-10-18", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963147} -{"stream": "segments_analytics", "data": {"time": "2022-10-19", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963149} -{"stream": "segments_analytics", "data": {"time": "2022-10-20", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963151} -{"stream": "segments_analytics", "data": {"time": "2022-10-21", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963153} -{"stream": "segments_analytics", "data": {"time": "2022-10-22", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963155} -{"stream": "segments_analytics", "data": {"time": "2022-10-23", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963157} -{"stream": "segments_analytics", "data": {"time": "2022-10-24", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963160} -{"stream": "segments_analytics", "data": {"time": "2022-10-25", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963162} -{"stream": "segments_analytics", "data": {"time": "2022-10-26", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963164} -{"stream": "segments_analytics", "data": {"time": "2022-10-27", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963166} -{"stream": "segments_analytics", "data": {"time": "2022-10-28", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963168} -{"stream": "segments_analytics", "data": {"time": "2022-10-29", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963170} -{"stream": "segments_analytics", "data": {"time": "2022-10-30", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963172} -{"stream": "segments_analytics", "data": {"time": "2022-10-31", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963174} -{"stream": "segments_analytics", "data": {"time": "2022-11-01", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963176} -{"stream": "segments_analytics", "data": {"time": "2022-11-02", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963179} -{"stream": "segments_analytics", "data": {"time": "2022-11-03", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963181} -{"stream": "segments_analytics", "data": {"time": "2022-11-04", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963183} -{"stream": "segments_analytics", "data": {"time": "2022-11-05", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963185} -{"stream": "segments_analytics", "data": {"time": "2022-11-06", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963187} -{"stream": "segments_analytics", "data": {"time": "2022-11-07", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963189} -{"stream": "segments_analytics", "data": {"time": "2022-11-08", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963191} -{"stream": "segments_analytics", "data": {"time": "2022-11-09", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963193} -{"stream": "segments_analytics", "data": {"time": "2022-11-10", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963196} -{"stream": "segments_analytics", "data": {"time": "2022-11-11", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963198} -{"stream": "segments_analytics", "data": {"time": "2022-11-12", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963200} -{"stream": "segments_analytics", "data": {"time": "2022-11-13", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963202} -{"stream": "segments_analytics", "data": {"time": "2022-11-14", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963204} -{"stream": "segments_analytics", "data": {"time": "2022-11-15", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963206} -{"stream": "segments_analytics", "data": {"time": "2022-11-16", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963208} -{"stream": "segments_analytics", "data": {"time": "2022-11-17", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963210} -{"stream": "segments_analytics", "data": {"time": "2022-11-18", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963212} -{"stream": "segments_analytics", "data": {"time": "2022-11-19", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963214} -{"stream": "segments_analytics", "data": {"time": "2022-11-20", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963217} -{"stream": "segments_analytics", "data": {"time": "2022-11-21", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963219} -{"stream": "segments_analytics", "data": {"time": "2022-11-22", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963221} -{"stream": "segments_analytics", "data": {"time": "2022-11-23", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963223} -{"stream": "segments_analytics", "data": {"time": "2022-11-24", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963225} -{"stream": "segments_analytics", "data": {"time": "2022-11-25", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963227} -{"stream": "segments_analytics", "data": {"time": "2022-11-26", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963229} -{"stream": "segments_analytics", "data": {"time": "2022-11-27", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963231} -{"stream": "segments_analytics", "data": {"time": "2022-11-28", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963234} -{"stream": "segments_analytics", "data": {"time": "2022-11-29", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963236} -{"stream": "segments_analytics", "data": {"time": "2022-11-30", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963238} -{"stream": "segments_analytics", "data": {"time": "2022-12-01", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963240} -{"stream": "segments_analytics", "data": {"time": "2022-12-02", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963242} -{"stream": "segments_analytics", "data": {"time": "2022-12-03", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963244} -{"stream": "segments_analytics", "data": {"time": "2022-12-04", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963246} -{"stream": "segments_analytics", "data": {"time": "2022-12-05", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963248} -{"stream": "segments_analytics", "data": {"time": "2022-12-06", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963250} -{"stream": "segments_analytics", "data": {"time": "2022-12-07", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963253} -{"stream": "segments_analytics", "data": {"time": "2022-12-08", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963255} -{"stream": "segments_analytics", "data": {"time": "2022-12-09", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963592} -{"stream": "segments_analytics", "data": {"time": "2022-12-10", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963600} -{"stream": "segments_analytics", "data": {"time": "2022-12-11", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963608} -{"stream": "segments_analytics", "data": {"time": "2022-12-12", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963614} -{"stream": "segments_analytics", "data": {"time": "2022-12-13", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963619} -{"stream": "segments_analytics", "data": {"time": "2022-12-14", "size": 0, "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac"}, "emitted_at": 1671100963622} -{"stream": "segments_analytics", "data": {"time": "2022-09-01", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964299} -{"stream": "segments_analytics", "data": {"time": "2022-09-02", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964301} -{"stream": "segments_analytics", "data": {"time": "2022-09-03", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964303} -{"stream": "segments_analytics", "data": {"time": "2022-09-04", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964305} -{"stream": "segments_analytics", "data": {"time": "2022-09-05", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964307} -{"stream": "segments_analytics", "data": {"time": "2022-09-06", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964310} -{"stream": "segments_analytics", "data": {"time": "2022-09-07", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964312} -{"stream": "segments_analytics", "data": {"time": "2022-09-08", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964314} -{"stream": "segments_analytics", "data": {"time": "2022-09-09", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964316} -{"stream": "segments_analytics", "data": {"time": "2022-09-10", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964318} -{"stream": "segments_analytics", "data": {"time": "2022-09-11", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964320} -{"stream": "segments_analytics", "data": {"time": "2022-09-12", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964322} -{"stream": "segments_analytics", "data": {"time": "2022-09-13", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964324} -{"stream": "segments_analytics", "data": {"time": "2022-09-14", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964326} -{"stream": "segments_analytics", "data": {"time": "2022-09-15", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964329} -{"stream": "segments_analytics", "data": {"time": "2022-09-16", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964331} -{"stream": "segments_analytics", "data": {"time": "2022-09-17", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964333} -{"stream": "segments_analytics", "data": {"time": "2022-09-18", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964335} -{"stream": "segments_analytics", "data": {"time": "2022-09-19", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964338} -{"stream": "segments_analytics", "data": {"time": "2022-09-20", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964340} -{"stream": "segments_analytics", "data": {"time": "2022-09-21", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964342} -{"stream": "segments_analytics", "data": {"time": "2022-09-22", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964344} -{"stream": "segments_analytics", "data": {"time": "2022-09-23", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964346} -{"stream": "segments_analytics", "data": {"time": "2022-09-24", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964348} -{"stream": "segments_analytics", "data": {"time": "2022-09-25", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964350} -{"stream": "segments_analytics", "data": {"time": "2022-09-26", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964353} -{"stream": "segments_analytics", "data": {"time": "2022-09-27", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964355} -{"stream": "segments_analytics", "data": {"time": "2022-09-28", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964357} -{"stream": "segments_analytics", "data": {"time": "2022-09-29", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964359} -{"stream": "segments_analytics", "data": {"time": "2022-09-30", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964361} -{"stream": "segments_analytics", "data": {"time": "2022-10-01", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964363} -{"stream": "segments_analytics", "data": {"time": "2022-10-02", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964365} -{"stream": "segments_analytics", "data": {"time": "2022-10-03", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964367} -{"stream": "segments_analytics", "data": {"time": "2022-10-04", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964369} -{"stream": "segments_analytics", "data": {"time": "2022-10-05", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964372} -{"stream": "segments_analytics", "data": {"time": "2022-10-06", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964374} -{"stream": "segments_analytics", "data": {"time": "2022-10-07", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964376} -{"stream": "segments_analytics", "data": {"time": "2022-10-08", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964378} -{"stream": "segments_analytics", "data": {"time": "2022-10-09", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964380} -{"stream": "segments_analytics", "data": {"time": "2022-10-10", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964382} -{"stream": "segments_analytics", "data": {"time": "2022-10-11", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964384} -{"stream": "segments_analytics", "data": {"time": "2022-10-12", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964386} -{"stream": "segments_analytics", "data": {"time": "2022-10-13", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964389} -{"stream": "segments_analytics", "data": {"time": "2022-10-14", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964391} -{"stream": "segments_analytics", "data": {"time": "2022-10-15", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964393} -{"stream": "segments_analytics", "data": {"time": "2022-10-16", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964395} -{"stream": "segments_analytics", "data": {"time": "2022-10-17", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964397} -{"stream": "segments_analytics", "data": {"time": "2022-10-18", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964399} -{"stream": "segments_analytics", "data": {"time": "2022-10-19", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964401} -{"stream": "segments_analytics", "data": {"time": "2022-10-20", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964403} -{"stream": "segments_analytics", "data": {"time": "2022-10-21", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964405} -{"stream": "segments_analytics", "data": {"time": "2022-10-22", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964408} -{"stream": "segments_analytics", "data": {"time": "2022-10-23", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964410} -{"stream": "segments_analytics", "data": {"time": "2022-10-24", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964412} -{"stream": "segments_analytics", "data": {"time": "2022-10-25", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964414} -{"stream": "segments_analytics", "data": {"time": "2022-10-26", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964416} -{"stream": "segments_analytics", "data": {"time": "2022-10-27", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964418} -{"stream": "segments_analytics", "data": {"time": "2022-10-28", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964420} -{"stream": "segments_analytics", "data": {"time": "2022-10-29", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964422} -{"stream": "segments_analytics", "data": {"time": "2022-10-30", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964425} -{"stream": "segments_analytics", "data": {"time": "2022-10-31", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964427} -{"stream": "segments_analytics", "data": {"time": "2022-11-01", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964429} -{"stream": "segments_analytics", "data": {"time": "2022-11-02", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964431} -{"stream": "segments_analytics", "data": {"time": "2022-11-03", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964433} -{"stream": "segments_analytics", "data": {"time": "2022-11-04", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964435} -{"stream": "segments_analytics", "data": {"time": "2022-11-05", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964437} -{"stream": "segments_analytics", "data": {"time": "2022-11-06", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964439} -{"stream": "segments_analytics", "data": {"time": "2022-11-07", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964442} -{"stream": "segments_analytics", "data": {"time": "2022-11-08", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964444} -{"stream": "segments_analytics", "data": {"time": "2022-11-09", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964446} -{"stream": "segments_analytics", "data": {"time": "2022-11-10", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964448} -{"stream": "segments_analytics", "data": {"time": "2022-11-11", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964450} -{"stream": "segments_analytics", "data": {"time": "2022-11-12", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964452} -{"stream": "segments_analytics", "data": {"time": "2022-11-13", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964455} -{"stream": "segments_analytics", "data": {"time": "2022-11-14", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964457} -{"stream": "segments_analytics", "data": {"time": "2022-11-15", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964459} -{"stream": "segments_analytics", "data": {"time": "2022-11-16", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964461} -{"stream": "segments_analytics", "data": {"time": "2022-11-17", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964463} -{"stream": "segments_analytics", "data": {"time": "2022-11-18", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964465} -{"stream": "segments_analytics", "data": {"time": "2022-11-19", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964467} -{"stream": "segments_analytics", "data": {"time": "2022-11-20", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964469} -{"stream": "segments_analytics", "data": {"time": "2022-11-21", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964471} -{"stream": "segments_analytics", "data": {"time": "2022-11-22", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964474} -{"stream": "segments_analytics", "data": {"time": "2022-11-23", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964476} -{"stream": "segments_analytics", "data": {"time": "2022-11-24", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964478} -{"stream": "segments_analytics", "data": {"time": "2022-11-25", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964480} -{"stream": "segments_analytics", "data": {"time": "2022-11-26", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964482} -{"stream": "segments_analytics", "data": {"time": "2022-11-27", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964484} -{"stream": "segments_analytics", "data": {"time": "2022-11-28", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964486} -{"stream": "segments_analytics", "data": {"time": "2022-11-29", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964488} -{"stream": "segments_analytics", "data": {"time": "2022-11-30", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964490} -{"stream": "segments_analytics", "data": {"time": "2022-12-01", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964493} -{"stream": "segments_analytics", "data": {"time": "2022-12-02", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964495} -{"stream": "segments_analytics", "data": {"time": "2022-12-03", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964497} -{"stream": "segments_analytics", "data": {"time": "2022-12-04", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964499} -{"stream": "segments_analytics", "data": {"time": "2022-12-05", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964501} -{"stream": "segments_analytics", "data": {"time": "2022-12-06", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964503} -{"stream": "segments_analytics", "data": {"time": "2022-12-07", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964505} -{"stream": "segments_analytics", "data": {"time": "2022-12-08", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100964507} -{"stream": "segments_analytics", "data": {"time": "2022-12-09", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100965063} -{"stream": "segments_analytics", "data": {"time": "2022-12-10", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100965066} -{"stream": "segments_analytics", "data": {"time": "2022-12-11", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100965070} -{"stream": "segments_analytics", "data": {"time": "2022-12-12", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100965074} -{"stream": "segments_analytics", "data": {"time": "2022-12-13", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100965078} -{"stream": "segments_analytics", "data": {"time": "2022-12-14", "size": 0, "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2"}, "emitted_at": 1671100965082} -{"stream": "segments_analytics", "data": {"time": "2022-09-01", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965730} -{"stream": "segments_analytics", "data": {"time": "2022-09-02", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965733} -{"stream": "segments_analytics", "data": {"time": "2022-09-03", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965735} -{"stream": "segments_analytics", "data": {"time": "2022-09-04", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965737} -{"stream": "segments_analytics", "data": {"time": "2022-09-05", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965739} -{"stream": "segments_analytics", "data": {"time": "2022-09-06", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965741} -{"stream": "segments_analytics", "data": {"time": "2022-09-07", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965743} -{"stream": "segments_analytics", "data": {"time": "2022-09-08", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965746} -{"stream": "segments_analytics", "data": {"time": "2022-09-09", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965748} -{"stream": "segments_analytics", "data": {"time": "2022-09-10", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965750} -{"stream": "segments_analytics", "data": {"time": "2022-09-11", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965752} -{"stream": "segments_analytics", "data": {"time": "2022-09-12", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965754} -{"stream": "segments_analytics", "data": {"time": "2022-09-13", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965756} -{"stream": "segments_analytics", "data": {"time": "2022-09-14", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965758} -{"stream": "segments_analytics", "data": {"time": "2022-09-15", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965760} -{"stream": "segments_analytics", "data": {"time": "2022-09-16", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965762} -{"stream": "segments_analytics", "data": {"time": "2022-09-17", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965765} -{"stream": "segments_analytics", "data": {"time": "2022-09-18", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965767} -{"stream": "segments_analytics", "data": {"time": "2022-09-19", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965769} -{"stream": "segments_analytics", "data": {"time": "2022-09-20", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965771} -{"stream": "segments_analytics", "data": {"time": "2022-09-21", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965773} -{"stream": "segments_analytics", "data": {"time": "2022-09-22", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965775} -{"stream": "segments_analytics", "data": {"time": "2022-09-23", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965777} -{"stream": "segments_analytics", "data": {"time": "2022-09-24", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965779} -{"stream": "segments_analytics", "data": {"time": "2022-09-25", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965782} -{"stream": "segments_analytics", "data": {"time": "2022-09-26", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965784} -{"stream": "segments_analytics", "data": {"time": "2022-09-27", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965786} -{"stream": "segments_analytics", "data": {"time": "2022-09-28", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965788} -{"stream": "segments_analytics", "data": {"time": "2022-09-29", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965790} -{"stream": "segments_analytics", "data": {"time": "2022-09-30", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965792} -{"stream": "segments_analytics", "data": {"time": "2022-10-01", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965794} -{"stream": "segments_analytics", "data": {"time": "2022-10-02", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965796} -{"stream": "segments_analytics", "data": {"time": "2022-10-03", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965799} -{"stream": "segments_analytics", "data": {"time": "2022-10-04", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965801} -{"stream": "segments_analytics", "data": {"time": "2022-10-05", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965803} -{"stream": "segments_analytics", "data": {"time": "2022-10-06", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965805} -{"stream": "segments_analytics", "data": {"time": "2022-10-07", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965807} -{"stream": "segments_analytics", "data": {"time": "2022-10-08", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965809} -{"stream": "segments_analytics", "data": {"time": "2022-10-09", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965811} -{"stream": "segments_analytics", "data": {"time": "2022-10-10", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965813} -{"stream": "segments_analytics", "data": {"time": "2022-10-11", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965816} -{"stream": "segments_analytics", "data": {"time": "2022-10-12", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965818} -{"stream": "segments_analytics", "data": {"time": "2022-10-13", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965820} -{"stream": "segments_analytics", "data": {"time": "2022-10-14", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965822} -{"stream": "segments_analytics", "data": {"time": "2022-10-15", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965824} -{"stream": "segments_analytics", "data": {"time": "2022-10-16", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965826} -{"stream": "segments_analytics", "data": {"time": "2022-10-17", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965829} -{"stream": "segments_analytics", "data": {"time": "2022-10-18", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965831} -{"stream": "segments_analytics", "data": {"time": "2022-10-19", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965833} -{"stream": "segments_analytics", "data": {"time": "2022-10-20", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965835} -{"stream": "segments_analytics", "data": {"time": "2022-10-21", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965837} -{"stream": "segments_analytics", "data": {"time": "2022-10-22", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965839} -{"stream": "segments_analytics", "data": {"time": "2022-10-23", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965842} -{"stream": "segments_analytics", "data": {"time": "2022-10-24", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965844} -{"stream": "segments_analytics", "data": {"time": "2022-10-25", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965846} -{"stream": "segments_analytics", "data": {"time": "2022-10-26", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965848} -{"stream": "segments_analytics", "data": {"time": "2022-10-27", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965850} -{"stream": "segments_analytics", "data": {"time": "2022-10-28", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965852} -{"stream": "segments_analytics", "data": {"time": "2022-10-29", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965854} -{"stream": "segments_analytics", "data": {"time": "2022-10-30", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965856} -{"stream": "segments_analytics", "data": {"time": "2022-10-31", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965858} -{"stream": "segments_analytics", "data": {"time": "2022-11-01", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965861} -{"stream": "segments_analytics", "data": {"time": "2022-11-02", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965863} -{"stream": "segments_analytics", "data": {"time": "2022-11-03", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965865} -{"stream": "segments_analytics", "data": {"time": "2022-11-04", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965867} -{"stream": "segments_analytics", "data": {"time": "2022-11-05", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965869} -{"stream": "segments_analytics", "data": {"time": "2022-11-06", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965871} -{"stream": "segments_analytics", "data": {"time": "2022-11-07", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965873} -{"stream": "segments_analytics", "data": {"time": "2022-11-08", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965875} -{"stream": "segments_analytics", "data": {"time": "2022-11-09", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965878} -{"stream": "segments_analytics", "data": {"time": "2022-11-10", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965880} -{"stream": "segments_analytics", "data": {"time": "2022-11-11", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965882} -{"stream": "segments_analytics", "data": {"time": "2022-11-12", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965884} -{"stream": "segments_analytics", "data": {"time": "2022-11-13", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965886} -{"stream": "segments_analytics", "data": {"time": "2022-11-14", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965888} -{"stream": "segments_analytics", "data": {"time": "2022-11-15", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965890} -{"stream": "segments_analytics", "data": {"time": "2022-11-16", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965892} -{"stream": "segments_analytics", "data": {"time": "2022-11-17", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965894} -{"stream": "segments_analytics", "data": {"time": "2022-11-18", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965897} -{"stream": "segments_analytics", "data": {"time": "2022-11-19", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965899} -{"stream": "segments_analytics", "data": {"time": "2022-11-20", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965901} -{"stream": "segments_analytics", "data": {"time": "2022-11-21", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965903} -{"stream": "segments_analytics", "data": {"time": "2022-11-22", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965905} -{"stream": "segments_analytics", "data": {"time": "2022-11-23", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965907} -{"stream": "segments_analytics", "data": {"time": "2022-11-24", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965909} -{"stream": "segments_analytics", "data": {"time": "2022-11-25", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965911} -{"stream": "segments_analytics", "data": {"time": "2022-11-26", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965914} -{"stream": "segments_analytics", "data": {"time": "2022-11-27", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965916} -{"stream": "segments_analytics", "data": {"time": "2022-11-28", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965918} -{"stream": "segments_analytics", "data": {"time": "2022-11-29", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965920} -{"stream": "segments_analytics", "data": {"time": "2022-11-30", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965922} -{"stream": "segments_analytics", "data": {"time": "2022-12-01", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965924} -{"stream": "segments_analytics", "data": {"time": "2022-12-02", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965926} -{"stream": "segments_analytics", "data": {"time": "2022-12-03", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965929} -{"stream": "segments_analytics", "data": {"time": "2022-12-04", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965931} -{"stream": "segments_analytics", "data": {"time": "2022-12-05", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965933} -{"stream": "segments_analytics", "data": {"time": "2022-12-06", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965935} -{"stream": "segments_analytics", "data": {"time": "2022-12-07", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965937} -{"stream": "segments_analytics", "data": {"time": "2022-12-08", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100965939} -{"stream": "segments_analytics", "data": {"time": "2022-12-09", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100966449} -{"stream": "segments_analytics", "data": {"time": "2022-12-10", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100966451} -{"stream": "segments_analytics", "data": {"time": "2022-12-11", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100966453} -{"stream": "segments_analytics", "data": {"time": "2022-12-12", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100966456} -{"stream": "segments_analytics", "data": {"time": "2022-12-13", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100966458} -{"stream": "segments_analytics", "data": {"time": "2022-12-14", "size": 0, "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac"}, "emitted_at": 1671100966460} -{"stream": "segments_analytics", "data": {"time": "2022-09-01", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967178} -{"stream": "segments_analytics", "data": {"time": "2022-09-02", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967186} -{"stream": "segments_analytics", "data": {"time": "2022-09-03", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967192} -{"stream": "segments_analytics", "data": {"time": "2022-09-04", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967197} -{"stream": "segments_analytics", "data": {"time": "2022-09-05", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967201} -{"stream": "segments_analytics", "data": {"time": "2022-09-06", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967205} -{"stream": "segments_analytics", "data": {"time": "2022-09-07", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967208} -{"stream": "segments_analytics", "data": {"time": "2022-09-08", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967210} -{"stream": "segments_analytics", "data": {"time": "2022-09-09", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967213} -{"stream": "segments_analytics", "data": {"time": "2022-09-10", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967216} -{"stream": "segments_analytics", "data": {"time": "2022-09-11", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967218} -{"stream": "segments_analytics", "data": {"time": "2022-09-12", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967220} -{"stream": "segments_analytics", "data": {"time": "2022-09-13", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967222} -{"stream": "segments_analytics", "data": {"time": "2022-09-14", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967225} -{"stream": "segments_analytics", "data": {"time": "2022-09-15", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967227} -{"stream": "segments_analytics", "data": {"time": "2022-09-16", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967229} -{"stream": "segments_analytics", "data": {"time": "2022-09-17", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967231} -{"stream": "segments_analytics", "data": {"time": "2022-09-18", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967234} -{"stream": "segments_analytics", "data": {"time": "2022-09-19", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967236} -{"stream": "segments_analytics", "data": {"time": "2022-09-20", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967238} -{"stream": "segments_analytics", "data": {"time": "2022-09-21", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967240} -{"stream": "segments_analytics", "data": {"time": "2022-09-22", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967242} -{"stream": "segments_analytics", "data": {"time": "2022-09-23", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967244} -{"stream": "segments_analytics", "data": {"time": "2022-09-24", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967246} -{"stream": "segments_analytics", "data": {"time": "2022-09-25", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967249} -{"stream": "segments_analytics", "data": {"time": "2022-09-26", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967251} -{"stream": "segments_analytics", "data": {"time": "2022-09-27", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967253} -{"stream": "segments_analytics", "data": {"time": "2022-09-28", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967255} -{"stream": "segments_analytics", "data": {"time": "2022-09-29", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967257} -{"stream": "segments_analytics", "data": {"time": "2022-09-30", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967259} -{"stream": "segments_analytics", "data": {"time": "2022-10-01", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967261} -{"stream": "segments_analytics", "data": {"time": "2022-10-02", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967263} -{"stream": "segments_analytics", "data": {"time": "2022-10-03", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967266} -{"stream": "segments_analytics", "data": {"time": "2022-10-04", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967268} -{"stream": "segments_analytics", "data": {"time": "2022-10-05", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967270} -{"stream": "segments_analytics", "data": {"time": "2022-10-06", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967272} -{"stream": "segments_analytics", "data": {"time": "2022-10-07", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967274} -{"stream": "segments_analytics", "data": {"time": "2022-10-08", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967276} -{"stream": "segments_analytics", "data": {"time": "2022-10-09", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967279} -{"stream": "segments_analytics", "data": {"time": "2022-10-10", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967281} -{"stream": "segments_analytics", "data": {"time": "2022-10-11", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967283} -{"stream": "segments_analytics", "data": {"time": "2022-10-12", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967285} -{"stream": "segments_analytics", "data": {"time": "2022-10-13", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967287} -{"stream": "segments_analytics", "data": {"time": "2022-10-14", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967289} -{"stream": "segments_analytics", "data": {"time": "2022-10-15", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967291} -{"stream": "segments_analytics", "data": {"time": "2022-10-16", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967294} -{"stream": "segments_analytics", "data": {"time": "2022-10-17", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967296} -{"stream": "segments_analytics", "data": {"time": "2022-10-18", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967298} -{"stream": "segments_analytics", "data": {"time": "2022-10-19", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967300} -{"stream": "segments_analytics", "data": {"time": "2022-10-20", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967302} -{"stream": "segments_analytics", "data": {"time": "2022-10-21", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967304} -{"stream": "segments_analytics", "data": {"time": "2022-10-22", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967306} -{"stream": "segments_analytics", "data": {"time": "2022-10-23", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967309} -{"stream": "segments_analytics", "data": {"time": "2022-10-24", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967311} -{"stream": "segments_analytics", "data": {"time": "2022-10-25", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967313} -{"stream": "segments_analytics", "data": {"time": "2022-10-26", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967315} -{"stream": "segments_analytics", "data": {"time": "2022-10-27", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967317} -{"stream": "segments_analytics", "data": {"time": "2022-10-28", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967319} -{"stream": "segments_analytics", "data": {"time": "2022-10-29", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967321} -{"stream": "segments_analytics", "data": {"time": "2022-10-30", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967324} -{"stream": "segments_analytics", "data": {"time": "2022-10-31", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967326} -{"stream": "segments_analytics", "data": {"time": "2022-11-01", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967328} -{"stream": "segments_analytics", "data": {"time": "2022-11-02", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967330} -{"stream": "segments_analytics", "data": {"time": "2022-11-03", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967332} -{"stream": "segments_analytics", "data": {"time": "2022-11-04", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967334} -{"stream": "segments_analytics", "data": {"time": "2022-11-05", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967336} -{"stream": "segments_analytics", "data": {"time": "2022-11-06", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967339} -{"stream": "segments_analytics", "data": {"time": "2022-11-07", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967341} -{"stream": "segments_analytics", "data": {"time": "2022-11-08", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967343} -{"stream": "segments_analytics", "data": {"time": "2022-11-09", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967345} -{"stream": "segments_analytics", "data": {"time": "2022-11-10", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967347} -{"stream": "segments_analytics", "data": {"time": "2022-11-11", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967349} -{"stream": "segments_analytics", "data": {"time": "2022-11-12", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967351} -{"stream": "segments_analytics", "data": {"time": "2022-11-13", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967354} -{"stream": "segments_analytics", "data": {"time": "2022-11-14", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967356} -{"stream": "segments_analytics", "data": {"time": "2022-11-15", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967358} -{"stream": "segments_analytics", "data": {"time": "2022-11-16", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967360} -{"stream": "segments_analytics", "data": {"time": "2022-11-17", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967362} -{"stream": "segments_analytics", "data": {"time": "2022-11-18", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967364} -{"stream": "segments_analytics", "data": {"time": "2022-11-19", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967367} -{"stream": "segments_analytics", "data": {"time": "2022-11-20", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967369} -{"stream": "segments_analytics", "data": {"time": "2022-11-21", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967371} -{"stream": "segments_analytics", "data": {"time": "2022-11-22", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967373} -{"stream": "segments_analytics", "data": {"time": "2022-11-23", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967375} -{"stream": "segments_analytics", "data": {"time": "2022-11-24", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967377} -{"stream": "segments_analytics", "data": {"time": "2022-11-25", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967379} -{"stream": "segments_analytics", "data": {"time": "2022-11-26", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967382} -{"stream": "segments_analytics", "data": {"time": "2022-11-27", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967384} -{"stream": "segments_analytics", "data": {"time": "2022-11-28", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967386} -{"stream": "segments_analytics", "data": {"time": "2022-11-29", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967388} -{"stream": "segments_analytics", "data": {"time": "2022-11-30", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967390} -{"stream": "segments_analytics", "data": {"time": "2022-12-01", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967392} -{"stream": "segments_analytics", "data": {"time": "2022-12-02", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967394} -{"stream": "segments_analytics", "data": {"time": "2022-12-03", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967397} -{"stream": "segments_analytics", "data": {"time": "2022-12-04", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967399} -{"stream": "segments_analytics", "data": {"time": "2022-12-05", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967401} -{"stream": "segments_analytics", "data": {"time": "2022-12-06", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967403} -{"stream": "segments_analytics", "data": {"time": "2022-12-07", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967405} -{"stream": "segments_analytics", "data": {"time": "2022-12-08", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967407} -{"stream": "segments_analytics", "data": {"time": "2022-12-09", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967687} -{"stream": "segments_analytics", "data": {"time": "2022-12-10", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967695} -{"stream": "segments_analytics", "data": {"time": "2022-12-11", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967702} -{"stream": "segments_analytics", "data": {"time": "2022-12-12", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967707} -{"stream": "segments_analytics", "data": {"time": "2022-12-13", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967712} -{"stream": "segments_analytics", "data": {"time": "2022-12-14", "size": 0, "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e"}, "emitted_at": 1671100967715} -{"stream": "segments_analytics", "data": {"time": "2022-09-01", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100967996} -{"stream": "segments_analytics", "data": {"time": "2022-09-02", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968005} -{"stream": "segments_analytics", "data": {"time": "2022-09-03", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968012} -{"stream": "segments_analytics", "data": {"time": "2022-09-04", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968016} -{"stream": "segments_analytics", "data": {"time": "2022-09-05", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968020} -{"stream": "segments_analytics", "data": {"time": "2022-09-06", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968024} -{"stream": "segments_analytics", "data": {"time": "2022-09-07", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968027} -{"stream": "segments_analytics", "data": {"time": "2022-09-08", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968030} -{"stream": "segments_analytics", "data": {"time": "2022-09-09", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968032} -{"stream": "segments_analytics", "data": {"time": "2022-09-10", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968035} -{"stream": "segments_analytics", "data": {"time": "2022-09-11", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968037} -{"stream": "segments_analytics", "data": {"time": "2022-09-12", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968040} -{"stream": "segments_analytics", "data": {"time": "2022-09-13", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968042} -{"stream": "segments_analytics", "data": {"time": "2022-09-14", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968044} -{"stream": "segments_analytics", "data": {"time": "2022-09-15", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968046} -{"stream": "segments_analytics", "data": {"time": "2022-09-16", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968048} -{"stream": "segments_analytics", "data": {"time": "2022-09-17", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968050} -{"stream": "segments_analytics", "data": {"time": "2022-09-18", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968052} -{"stream": "segments_analytics", "data": {"time": "2022-09-19", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968055} -{"stream": "segments_analytics", "data": {"time": "2022-09-20", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968057} -{"stream": "segments_analytics", "data": {"time": "2022-09-21", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968059} -{"stream": "segments_analytics", "data": {"time": "2022-09-22", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968061} -{"stream": "segments_analytics", "data": {"time": "2022-09-23", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968063} -{"stream": "segments_analytics", "data": {"time": "2022-09-24", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968065} -{"stream": "segments_analytics", "data": {"time": "2022-09-25", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968067} -{"stream": "segments_analytics", "data": {"time": "2022-09-26", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968070} -{"stream": "segments_analytics", "data": {"time": "2022-09-27", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968072} -{"stream": "segments_analytics", "data": {"time": "2022-09-28", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968074} -{"stream": "segments_analytics", "data": {"time": "2022-09-29", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968076} -{"stream": "segments_analytics", "data": {"time": "2022-09-30", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968078} -{"stream": "segments_analytics", "data": {"time": "2022-10-01", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968080} -{"stream": "segments_analytics", "data": {"time": "2022-10-02", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968082} -{"stream": "segments_analytics", "data": {"time": "2022-10-03", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968085} -{"stream": "segments_analytics", "data": {"time": "2022-10-04", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968087} -{"stream": "segments_analytics", "data": {"time": "2022-10-05", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968089} -{"stream": "segments_analytics", "data": {"time": "2022-10-06", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968091} -{"stream": "segments_analytics", "data": {"time": "2022-10-07", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968093} -{"stream": "segments_analytics", "data": {"time": "2022-10-08", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968095} -{"stream": "segments_analytics", "data": {"time": "2022-10-09", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968097} -{"stream": "segments_analytics", "data": {"time": "2022-10-10", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968099} -{"stream": "segments_analytics", "data": {"time": "2022-10-11", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968101} -{"stream": "segments_analytics", "data": {"time": "2022-10-12", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968104} -{"stream": "segments_analytics", "data": {"time": "2022-10-13", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968106} -{"stream": "segments_analytics", "data": {"time": "2022-10-14", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968108} -{"stream": "segments_analytics", "data": {"time": "2022-10-15", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968110} -{"stream": "segments_analytics", "data": {"time": "2022-10-16", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968112} -{"stream": "segments_analytics", "data": {"time": "2022-10-17", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968114} -{"stream": "segments_analytics", "data": {"time": "2022-10-18", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968116} -{"stream": "segments_analytics", "data": {"time": "2022-10-19", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968118} -{"stream": "segments_analytics", "data": {"time": "2022-10-20", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968120} -{"stream": "segments_analytics", "data": {"time": "2022-10-21", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968123} -{"stream": "segments_analytics", "data": {"time": "2022-10-22", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968125} -{"stream": "segments_analytics", "data": {"time": "2022-10-23", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968127} -{"stream": "segments_analytics", "data": {"time": "2022-10-24", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968129} -{"stream": "segments_analytics", "data": {"time": "2022-10-25", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968131} -{"stream": "segments_analytics", "data": {"time": "2022-10-26", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968133} -{"stream": "segments_analytics", "data": {"time": "2022-10-27", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968136} -{"stream": "segments_analytics", "data": {"time": "2022-10-28", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968138} -{"stream": "segments_analytics", "data": {"time": "2022-10-29", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968140} -{"stream": "segments_analytics", "data": {"time": "2022-10-30", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968142} -{"stream": "segments_analytics", "data": {"time": "2022-10-31", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968144} -{"stream": "segments_analytics", "data": {"time": "2022-11-01", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968146} -{"stream": "segments_analytics", "data": {"time": "2022-11-02", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968149} -{"stream": "segments_analytics", "data": {"time": "2022-11-03", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968151} -{"stream": "segments_analytics", "data": {"time": "2022-11-04", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968153} -{"stream": "segments_analytics", "data": {"time": "2022-11-05", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968155} -{"stream": "segments_analytics", "data": {"time": "2022-11-06", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968157} -{"stream": "segments_analytics", "data": {"time": "2022-11-07", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968159} -{"stream": "segments_analytics", "data": {"time": "2022-11-08", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968161} -{"stream": "segments_analytics", "data": {"time": "2022-11-09", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968164} -{"stream": "segments_analytics", "data": {"time": "2022-11-10", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968166} -{"stream": "segments_analytics", "data": {"time": "2022-11-11", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968168} -{"stream": "segments_analytics", "data": {"time": "2022-11-12", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968170} -{"stream": "segments_analytics", "data": {"time": "2022-11-13", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968172} -{"stream": "segments_analytics", "data": {"time": "2022-11-14", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968174} -{"stream": "segments_analytics", "data": {"time": "2022-11-15", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968177} -{"stream": "segments_analytics", "data": {"time": "2022-11-16", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968179} -{"stream": "segments_analytics", "data": {"time": "2022-11-17", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968181} -{"stream": "segments_analytics", "data": {"time": "2022-11-18", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968183} -{"stream": "segments_analytics", "data": {"time": "2022-11-19", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968185} -{"stream": "segments_analytics", "data": {"time": "2022-11-20", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968187} -{"stream": "segments_analytics", "data": {"time": "2022-11-21", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968189} -{"stream": "segments_analytics", "data": {"time": "2022-11-22", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968192} -{"stream": "segments_analytics", "data": {"time": "2022-11-23", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968194} -{"stream": "segments_analytics", "data": {"time": "2022-11-24", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968196} -{"stream": "segments_analytics", "data": {"time": "2022-11-25", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968198} -{"stream": "segments_analytics", "data": {"time": "2022-11-26", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968200} -{"stream": "segments_analytics", "data": {"time": "2022-11-27", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968202} -{"stream": "segments_analytics", "data": {"time": "2022-11-28", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968204} -{"stream": "segments_analytics", "data": {"time": "2022-11-29", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968206} -{"stream": "segments_analytics", "data": {"time": "2022-11-30", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968209} -{"stream": "segments_analytics", "data": {"time": "2022-12-01", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968211} -{"stream": "segments_analytics", "data": {"time": "2022-12-02", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968213} -{"stream": "segments_analytics", "data": {"time": "2022-12-03", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968215} -{"stream": "segments_analytics", "data": {"time": "2022-12-04", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968217} -{"stream": "segments_analytics", "data": {"time": "2022-12-05", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968219} -{"stream": "segments_analytics", "data": {"time": "2022-12-06", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968221} -{"stream": "segments_analytics", "data": {"time": "2022-12-07", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968224} -{"stream": "segments_analytics", "data": {"time": "2022-12-08", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968226} -{"stream": "segments_analytics", "data": {"time": "2022-12-09", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968609} -{"stream": "segments_analytics", "data": {"time": "2022-12-10", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968618} -{"stream": "segments_analytics", "data": {"time": "2022-12-11", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968624} -{"stream": "segments_analytics", "data": {"time": "2022-12-12", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968630} -{"stream": "segments_analytics", "data": {"time": "2022-12-13", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968634} -{"stream": "segments_analytics", "data": {"time": "2022-12-14", "size": 0, "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9"}, "emitted_at": 1671100968638} -{"stream": "segments_analytics", "data": {"time": "2022-09-01", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969004} -{"stream": "segments_analytics", "data": {"time": "2022-09-02", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969006} -{"stream": "segments_analytics", "data": {"time": "2022-09-03", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969008} -{"stream": "segments_analytics", "data": {"time": "2022-09-04", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969011} -{"stream": "segments_analytics", "data": {"time": "2022-09-05", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969013} -{"stream": "segments_analytics", "data": {"time": "2022-09-06", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969015} -{"stream": "segments_analytics", "data": {"time": "2022-09-07", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969017} -{"stream": "segments_analytics", "data": {"time": "2022-09-08", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969019} -{"stream": "segments_analytics", "data": {"time": "2022-09-09", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969021} -{"stream": "segments_analytics", "data": {"time": "2022-09-10", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969023} -{"stream": "segments_analytics", "data": {"time": "2022-09-11", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969025} -{"stream": "segments_analytics", "data": {"time": "2022-09-12", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969027} -{"stream": "segments_analytics", "data": {"time": "2022-09-13", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969030} -{"stream": "segments_analytics", "data": {"time": "2022-09-14", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969032} -{"stream": "segments_analytics", "data": {"time": "2022-09-15", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969034} -{"stream": "segments_analytics", "data": {"time": "2022-09-16", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969036} -{"stream": "segments_analytics", "data": {"time": "2022-09-17", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969038} -{"stream": "segments_analytics", "data": {"time": "2022-09-18", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969040} -{"stream": "segments_analytics", "data": {"time": "2022-09-19", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969042} -{"stream": "segments_analytics", "data": {"time": "2022-09-20", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969045} -{"stream": "segments_analytics", "data": {"time": "2022-09-21", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969047} -{"stream": "segments_analytics", "data": {"time": "2022-09-22", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969049} -{"stream": "segments_analytics", "data": {"time": "2022-09-23", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969051} -{"stream": "segments_analytics", "data": {"time": "2022-09-24", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969053} -{"stream": "segments_analytics", "data": {"time": "2022-09-25", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969055} -{"stream": "segments_analytics", "data": {"time": "2022-09-26", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969057} -{"stream": "segments_analytics", "data": {"time": "2022-09-27", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969059} -{"stream": "segments_analytics", "data": {"time": "2022-09-28", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969061} -{"stream": "segments_analytics", "data": {"time": "2022-09-29", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969064} -{"stream": "segments_analytics", "data": {"time": "2022-09-30", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969066} -{"stream": "segments_analytics", "data": {"time": "2022-10-01", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969068} -{"stream": "segments_analytics", "data": {"time": "2022-10-02", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969070} -{"stream": "segments_analytics", "data": {"time": "2022-10-03", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969072} -{"stream": "segments_analytics", "data": {"time": "2022-10-04", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969075} -{"stream": "segments_analytics", "data": {"time": "2022-10-05", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969077} -{"stream": "segments_analytics", "data": {"time": "2022-10-06", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969079} -{"stream": "segments_analytics", "data": {"time": "2022-10-07", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969082} -{"stream": "segments_analytics", "data": {"time": "2022-10-08", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969085} -{"stream": "segments_analytics", "data": {"time": "2022-10-09", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969088} -{"stream": "segments_analytics", "data": {"time": "2022-10-10", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969090} -{"stream": "segments_analytics", "data": {"time": "2022-10-11", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969092} -{"stream": "segments_analytics", "data": {"time": "2022-10-12", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969095} -{"stream": "segments_analytics", "data": {"time": "2022-10-13", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969097} -{"stream": "segments_analytics", "data": {"time": "2022-10-14", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969099} -{"stream": "segments_analytics", "data": {"time": "2022-10-15", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969101} -{"stream": "segments_analytics", "data": {"time": "2022-10-16", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969103} -{"stream": "segments_analytics", "data": {"time": "2022-10-17", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969105} -{"stream": "segments_analytics", "data": {"time": "2022-10-18", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969107} -{"stream": "segments_analytics", "data": {"time": "2022-10-19", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969109} -{"stream": "segments_analytics", "data": {"time": "2022-10-20", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969112} -{"stream": "segments_analytics", "data": {"time": "2022-10-21", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969114} -{"stream": "segments_analytics", "data": {"time": "2022-10-22", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969116} -{"stream": "segments_analytics", "data": {"time": "2022-10-23", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969118} -{"stream": "segments_analytics", "data": {"time": "2022-10-24", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969120} -{"stream": "segments_analytics", "data": {"time": "2022-10-25", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969122} -{"stream": "segments_analytics", "data": {"time": "2022-10-26", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969125} -{"stream": "segments_analytics", "data": {"time": "2022-10-27", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969127} -{"stream": "segments_analytics", "data": {"time": "2022-10-28", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969129} -{"stream": "segments_analytics", "data": {"time": "2022-10-29", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969131} -{"stream": "segments_analytics", "data": {"time": "2022-10-30", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969133} -{"stream": "segments_analytics", "data": {"time": "2022-10-31", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969135} -{"stream": "segments_analytics", "data": {"time": "2022-11-01", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969138} -{"stream": "segments_analytics", "data": {"time": "2022-11-02", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969140} -{"stream": "segments_analytics", "data": {"time": "2022-11-03", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969142} -{"stream": "segments_analytics", "data": {"time": "2022-11-04", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969144} -{"stream": "segments_analytics", "data": {"time": "2022-11-05", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969146} -{"stream": "segments_analytics", "data": {"time": "2022-11-06", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969148} -{"stream": "segments_analytics", "data": {"time": "2022-11-07", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969151} -{"stream": "segments_analytics", "data": {"time": "2022-11-08", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969153} -{"stream": "segments_analytics", "data": {"time": "2022-11-09", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969155} -{"stream": "segments_analytics", "data": {"time": "2022-11-10", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969157} -{"stream": "segments_analytics", "data": {"time": "2022-11-11", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969159} -{"stream": "segments_analytics", "data": {"time": "2022-11-12", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969161} -{"stream": "segments_analytics", "data": {"time": "2022-11-13", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969163} -{"stream": "segments_analytics", "data": {"time": "2022-11-14", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969166} -{"stream": "segments_analytics", "data": {"time": "2022-11-15", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969168} -{"stream": "segments_analytics", "data": {"time": "2022-11-16", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969170} -{"stream": "segments_analytics", "data": {"time": "2022-11-17", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969172} -{"stream": "segments_analytics", "data": {"time": "2022-11-18", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969174} -{"stream": "segments_analytics", "data": {"time": "2022-11-19", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969177} -{"stream": "segments_analytics", "data": {"time": "2022-11-20", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969179} -{"stream": "segments_analytics", "data": {"time": "2022-11-21", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969181} -{"stream": "segments_analytics", "data": {"time": "2022-11-22", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969183} -{"stream": "segments_analytics", "data": {"time": "2022-11-23", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969185} -{"stream": "segments_analytics", "data": {"time": "2022-11-24", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969187} -{"stream": "segments_analytics", "data": {"time": "2022-11-25", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969189} -{"stream": "segments_analytics", "data": {"time": "2022-11-26", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969192} -{"stream": "segments_analytics", "data": {"time": "2022-11-27", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969194} -{"stream": "segments_analytics", "data": {"time": "2022-11-28", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969196} -{"stream": "segments_analytics", "data": {"time": "2022-11-29", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969198} -{"stream": "segments_analytics", "data": {"time": "2022-11-30", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969200} -{"stream": "segments_analytics", "data": {"time": "2022-12-01", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969202} -{"stream": "segments_analytics", "data": {"time": "2022-12-02", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969204} -{"stream": "segments_analytics", "data": {"time": "2022-12-03", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969207} -{"stream": "segments_analytics", "data": {"time": "2022-12-04", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969209} -{"stream": "segments_analytics", "data": {"time": "2022-12-05", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969211} -{"stream": "segments_analytics", "data": {"time": "2022-12-06", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969213} -{"stream": "segments_analytics", "data": {"time": "2022-12-07", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969215} -{"stream": "segments_analytics", "data": {"time": "2022-12-08", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969217} -{"stream": "segments_analytics", "data": {"time": "2022-12-09", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969782} -{"stream": "segments_analytics", "data": {"time": "2022-12-10", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969790} -{"stream": "segments_analytics", "data": {"time": "2022-12-11", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969797} -{"stream": "segments_analytics", "data": {"time": "2022-12-12", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969802} -{"stream": "segments_analytics", "data": {"time": "2022-12-13", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969806} -{"stream": "segments_analytics", "data": {"time": "2022-12-14", "size": 0, "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71"}, "emitted_at": 1671100969809} -{"stream": "segments_analytics", "data": {"time": "2022-09-01", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970134} -{"stream": "segments_analytics", "data": {"time": "2022-09-02", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970136} -{"stream": "segments_analytics", "data": {"time": "2022-09-03", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970138} -{"stream": "segments_analytics", "data": {"time": "2022-09-04", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970140} -{"stream": "segments_analytics", "data": {"time": "2022-09-05", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970142} -{"stream": "segments_analytics", "data": {"time": "2022-09-06", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970145} -{"stream": "segments_analytics", "data": {"time": "2022-09-07", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970147} -{"stream": "segments_analytics", "data": {"time": "2022-09-08", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970149} -{"stream": "segments_analytics", "data": {"time": "2022-09-09", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970151} -{"stream": "segments_analytics", "data": {"time": "2022-09-10", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970153} -{"stream": "segments_analytics", "data": {"time": "2022-09-11", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970156} -{"stream": "segments_analytics", "data": {"time": "2022-09-12", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970158} -{"stream": "segments_analytics", "data": {"time": "2022-09-13", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970160} -{"stream": "segments_analytics", "data": {"time": "2022-09-14", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970162} -{"stream": "segments_analytics", "data": {"time": "2022-09-15", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970164} -{"stream": "segments_analytics", "data": {"time": "2022-09-16", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970166} -{"stream": "segments_analytics", "data": {"time": "2022-09-17", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970168} -{"stream": "segments_analytics", "data": {"time": "2022-09-18", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970170} -{"stream": "segments_analytics", "data": {"time": "2022-09-19", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970173} -{"stream": "segments_analytics", "data": {"time": "2022-09-20", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970175} -{"stream": "segments_analytics", "data": {"time": "2022-09-21", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970177} -{"stream": "segments_analytics", "data": {"time": "2022-09-22", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970179} -{"stream": "segments_analytics", "data": {"time": "2022-09-23", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970181} -{"stream": "segments_analytics", "data": {"time": "2022-09-24", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970183} -{"stream": "segments_analytics", "data": {"time": "2022-09-25", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970186} -{"stream": "segments_analytics", "data": {"time": "2022-09-26", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970188} -{"stream": "segments_analytics", "data": {"time": "2022-09-27", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970190} -{"stream": "segments_analytics", "data": {"time": "2022-09-28", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970192} -{"stream": "segments_analytics", "data": {"time": "2022-09-29", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970194} -{"stream": "segments_analytics", "data": {"time": "2022-09-30", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970196} -{"stream": "segments_analytics", "data": {"time": "2022-10-01", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970198} -{"stream": "segments_analytics", "data": {"time": "2022-10-02", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970201} -{"stream": "segments_analytics", "data": {"time": "2022-10-03", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970203} -{"stream": "segments_analytics", "data": {"time": "2022-10-04", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970205} -{"stream": "segments_analytics", "data": {"time": "2022-10-05", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970207} -{"stream": "segments_analytics", "data": {"time": "2022-10-06", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970209} -{"stream": "segments_analytics", "data": {"time": "2022-10-07", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970211} -{"stream": "segments_analytics", "data": {"time": "2022-10-08", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970213} -{"stream": "segments_analytics", "data": {"time": "2022-10-09", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970216} -{"stream": "segments_analytics", "data": {"time": "2022-10-10", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970218} -{"stream": "segments_analytics", "data": {"time": "2022-10-11", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970220} -{"stream": "segments_analytics", "data": {"time": "2022-10-12", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970222} -{"stream": "segments_analytics", "data": {"time": "2022-10-13", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970224} -{"stream": "segments_analytics", "data": {"time": "2022-10-14", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970226} -{"stream": "segments_analytics", "data": {"time": "2022-10-15", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970228} -{"stream": "segments_analytics", "data": {"time": "2022-10-16", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970230} -{"stream": "segments_analytics", "data": {"time": "2022-10-17", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970233} -{"stream": "segments_analytics", "data": {"time": "2022-10-18", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970235} -{"stream": "segments_analytics", "data": {"time": "2022-10-19", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970237} -{"stream": "segments_analytics", "data": {"time": "2022-10-20", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970239} -{"stream": "segments_analytics", "data": {"time": "2022-10-21", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970241} -{"stream": "segments_analytics", "data": {"time": "2022-10-22", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970243} -{"stream": "segments_analytics", "data": {"time": "2022-10-23", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970245} -{"stream": "segments_analytics", "data": {"time": "2022-10-24", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970247} -{"stream": "segments_analytics", "data": {"time": "2022-10-25", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970249} -{"stream": "segments_analytics", "data": {"time": "2022-10-26", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970252} -{"stream": "segments_analytics", "data": {"time": "2022-10-27", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970255} -{"stream": "segments_analytics", "data": {"time": "2022-10-28", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970257} -{"stream": "segments_analytics", "data": {"time": "2022-10-29", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970259} -{"stream": "segments_analytics", "data": {"time": "2022-10-30", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970262} -{"stream": "segments_analytics", "data": {"time": "2022-10-31", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970264} -{"stream": "segments_analytics", "data": {"time": "2022-11-01", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970266} -{"stream": "segments_analytics", "data": {"time": "2022-11-02", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970268} -{"stream": "segments_analytics", "data": {"time": "2022-11-03", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970270} -{"stream": "segments_analytics", "data": {"time": "2022-11-04", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970272} -{"stream": "segments_analytics", "data": {"time": "2022-11-05", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970274} -{"stream": "segments_analytics", "data": {"time": "2022-11-06", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970276} -{"stream": "segments_analytics", "data": {"time": "2022-11-07", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970279} -{"stream": "segments_analytics", "data": {"time": "2022-11-08", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970281} -{"stream": "segments_analytics", "data": {"time": "2022-11-09", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970283} -{"stream": "segments_analytics", "data": {"time": "2022-11-10", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970285} -{"stream": "segments_analytics", "data": {"time": "2022-11-11", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970287} -{"stream": "segments_analytics", "data": {"time": "2022-11-12", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970289} -{"stream": "segments_analytics", "data": {"time": "2022-11-13", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970291} -{"stream": "segments_analytics", "data": {"time": "2022-11-14", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970293} -{"stream": "segments_analytics", "data": {"time": "2022-11-15", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970296} -{"stream": "segments_analytics", "data": {"time": "2022-11-16", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970298} -{"stream": "segments_analytics", "data": {"time": "2022-11-17", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970300} -{"stream": "segments_analytics", "data": {"time": "2022-11-18", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970302} -{"stream": "segments_analytics", "data": {"time": "2022-11-19", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970304} -{"stream": "segments_analytics", "data": {"time": "2022-11-20", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970306} -{"stream": "segments_analytics", "data": {"time": "2022-11-21", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970308} -{"stream": "segments_analytics", "data": {"time": "2022-11-22", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970310} -{"stream": "segments_analytics", "data": {"time": "2022-11-23", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970312} -{"stream": "segments_analytics", "data": {"time": "2022-11-24", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970315} -{"stream": "segments_analytics", "data": {"time": "2022-11-25", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970317} -{"stream": "segments_analytics", "data": {"time": "2022-11-26", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970319} -{"stream": "segments_analytics", "data": {"time": "2022-11-27", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970321} -{"stream": "segments_analytics", "data": {"time": "2022-11-28", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970323} -{"stream": "segments_analytics", "data": {"time": "2022-11-29", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970325} -{"stream": "segments_analytics", "data": {"time": "2022-11-30", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970327} -{"stream": "segments_analytics", "data": {"time": "2022-12-01", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970329} -{"stream": "segments_analytics", "data": {"time": "2022-12-02", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970331} -{"stream": "segments_analytics", "data": {"time": "2022-12-03", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970334} -{"stream": "segments_analytics", "data": {"time": "2022-12-04", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970336} -{"stream": "segments_analytics", "data": {"time": "2022-12-05", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970338} -{"stream": "segments_analytics", "data": {"time": "2022-12-06", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970340} -{"stream": "segments_analytics", "data": {"time": "2022-12-07", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970342} -{"stream": "segments_analytics", "data": {"time": "2022-12-08", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970344} -{"stream": "segments_analytics", "data": {"time": "2022-12-09", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970922} -{"stream": "segments_analytics", "data": {"time": "2022-12-10", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970925} -{"stream": "segments_analytics", "data": {"time": "2022-12-11", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970927} -{"stream": "segments_analytics", "data": {"time": "2022-12-12", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970929} -{"stream": "segments_analytics", "data": {"time": "2022-12-13", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970931} -{"stream": "segments_analytics", "data": {"time": "2022-12-14", "size": 0, "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493"}, "emitted_at": 1671100970933} -{"stream": "segments_analytics", "data": {"time": "2022-09-01", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971532} -{"stream": "segments_analytics", "data": {"time": "2022-09-02", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971534} -{"stream": "segments_analytics", "data": {"time": "2022-09-03", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971536} -{"stream": "segments_analytics", "data": {"time": "2022-09-04", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971538} -{"stream": "segments_analytics", "data": {"time": "2022-09-05", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971540} -{"stream": "segments_analytics", "data": {"time": "2022-09-06", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971543} -{"stream": "segments_analytics", "data": {"time": "2022-09-07", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971545} -{"stream": "segments_analytics", "data": {"time": "2022-09-08", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971547} -{"stream": "segments_analytics", "data": {"time": "2022-09-09", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971549} -{"stream": "segments_analytics", "data": {"time": "2022-09-10", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971551} -{"stream": "segments_analytics", "data": {"time": "2022-09-11", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971553} -{"stream": "segments_analytics", "data": {"time": "2022-09-12", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971555} -{"stream": "segments_analytics", "data": {"time": "2022-09-13", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971557} -{"stream": "segments_analytics", "data": {"time": "2022-09-14", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971559} -{"stream": "segments_analytics", "data": {"time": "2022-09-15", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971562} -{"stream": "segments_analytics", "data": {"time": "2022-09-16", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971564} -{"stream": "segments_analytics", "data": {"time": "2022-09-17", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971566} -{"stream": "segments_analytics", "data": {"time": "2022-09-18", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971568} -{"stream": "segments_analytics", "data": {"time": "2022-09-19", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971570} -{"stream": "segments_analytics", "data": {"time": "2022-09-20", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971572} -{"stream": "segments_analytics", "data": {"time": "2022-09-21", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971574} -{"stream": "segments_analytics", "data": {"time": "2022-09-22", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971577} -{"stream": "segments_analytics", "data": {"time": "2022-09-23", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971579} -{"stream": "segments_analytics", "data": {"time": "2022-09-24", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971581} -{"stream": "segments_analytics", "data": {"time": "2022-09-25", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971583} -{"stream": "segments_analytics", "data": {"time": "2022-09-26", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971585} -{"stream": "segments_analytics", "data": {"time": "2022-09-27", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971587} -{"stream": "segments_analytics", "data": {"time": "2022-09-28", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971589} -{"stream": "segments_analytics", "data": {"time": "2022-09-29", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971592} -{"stream": "segments_analytics", "data": {"time": "2022-09-30", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971594} -{"stream": "segments_analytics", "data": {"time": "2022-10-01", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971596} -{"stream": "segments_analytics", "data": {"time": "2022-10-02", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971598} -{"stream": "segments_analytics", "data": {"time": "2022-10-03", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971600} -{"stream": "segments_analytics", "data": {"time": "2022-10-04", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971602} -{"stream": "segments_analytics", "data": {"time": "2022-10-05", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971604} -{"stream": "segments_analytics", "data": {"time": "2022-10-06", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971607} -{"stream": "segments_analytics", "data": {"time": "2022-10-07", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971609} -{"stream": "segments_analytics", "data": {"time": "2022-10-08", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971611} -{"stream": "segments_analytics", "data": {"time": "2022-10-09", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971613} -{"stream": "segments_analytics", "data": {"time": "2022-10-10", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971615} -{"stream": "segments_analytics", "data": {"time": "2022-10-11", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971617} -{"stream": "segments_analytics", "data": {"time": "2022-10-12", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971619} -{"stream": "segments_analytics", "data": {"time": "2022-10-13", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971622} -{"stream": "segments_analytics", "data": {"time": "2022-10-14", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971624} -{"stream": "segments_analytics", "data": {"time": "2022-10-15", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971626} -{"stream": "segments_analytics", "data": {"time": "2022-10-16", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971628} -{"stream": "segments_analytics", "data": {"time": "2022-10-17", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971630} -{"stream": "segments_analytics", "data": {"time": "2022-10-18", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971632} -{"stream": "segments_analytics", "data": {"time": "2022-10-19", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971635} -{"stream": "segments_analytics", "data": {"time": "2022-10-20", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971637} -{"stream": "segments_analytics", "data": {"time": "2022-10-21", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971639} -{"stream": "segments_analytics", "data": {"time": "2022-10-22", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971641} -{"stream": "segments_analytics", "data": {"time": "2022-10-23", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971643} -{"stream": "segments_analytics", "data": {"time": "2022-10-24", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971645} -{"stream": "segments_analytics", "data": {"time": "2022-10-25", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971647} -{"stream": "segments_analytics", "data": {"time": "2022-10-26", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971650} -{"stream": "segments_analytics", "data": {"time": "2022-10-27", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971652} -{"stream": "segments_analytics", "data": {"time": "2022-10-28", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971654} -{"stream": "segments_analytics", "data": {"time": "2022-10-29", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971656} -{"stream": "segments_analytics", "data": {"time": "2022-10-30", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971658} -{"stream": "segments_analytics", "data": {"time": "2022-10-31", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971660} -{"stream": "segments_analytics", "data": {"time": "2022-11-01", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971663} -{"stream": "segments_analytics", "data": {"time": "2022-11-02", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971665} -{"stream": "segments_analytics", "data": {"time": "2022-11-03", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971667} -{"stream": "segments_analytics", "data": {"time": "2022-11-04", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971669} -{"stream": "segments_analytics", "data": {"time": "2022-11-05", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971671} -{"stream": "segments_analytics", "data": {"time": "2022-11-06", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971673} -{"stream": "segments_analytics", "data": {"time": "2022-11-07", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971675} -{"stream": "segments_analytics", "data": {"time": "2022-11-08", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971677} -{"stream": "segments_analytics", "data": {"time": "2022-11-09", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971679} -{"stream": "segments_analytics", "data": {"time": "2022-11-10", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971682} -{"stream": "segments_analytics", "data": {"time": "2022-11-11", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971684} -{"stream": "segments_analytics", "data": {"time": "2022-11-12", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971686} -{"stream": "segments_analytics", "data": {"time": "2022-11-13", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971688} -{"stream": "segments_analytics", "data": {"time": "2022-11-14", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971690} -{"stream": "segments_analytics", "data": {"time": "2022-11-15", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971692} -{"stream": "segments_analytics", "data": {"time": "2022-11-16", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971694} -{"stream": "segments_analytics", "data": {"time": "2022-11-17", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971697} -{"stream": "segments_analytics", "data": {"time": "2022-11-18", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971699} -{"stream": "segments_analytics", "data": {"time": "2022-11-19", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971701} -{"stream": "segments_analytics", "data": {"time": "2022-11-20", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971703} -{"stream": "segments_analytics", "data": {"time": "2022-11-21", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971705} -{"stream": "segments_analytics", "data": {"time": "2022-11-22", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971707} -{"stream": "segments_analytics", "data": {"time": "2022-11-23", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971709} -{"stream": "segments_analytics", "data": {"time": "2022-11-24", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971711} -{"stream": "segments_analytics", "data": {"time": "2022-11-25", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971714} -{"stream": "segments_analytics", "data": {"time": "2022-11-26", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971716} -{"stream": "segments_analytics", "data": {"time": "2022-11-27", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971718} -{"stream": "segments_analytics", "data": {"time": "2022-11-28", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971720} -{"stream": "segments_analytics", "data": {"time": "2022-11-29", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971722} -{"stream": "segments_analytics", "data": {"time": "2022-11-30", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971724} -{"stream": "segments_analytics", "data": {"time": "2022-12-01", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971726} -{"stream": "segments_analytics", "data": {"time": "2022-12-02", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971728} -{"stream": "segments_analytics", "data": {"time": "2022-12-03", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971730} -{"stream": "segments_analytics", "data": {"time": "2022-12-04", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971733} -{"stream": "segments_analytics", "data": {"time": "2022-12-05", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971735} -{"stream": "segments_analytics", "data": {"time": "2022-12-06", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971737} -{"stream": "segments_analytics", "data": {"time": "2022-12-07", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971739} -{"stream": "segments_analytics", "data": {"time": "2022-12-08", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971741} -{"stream": "segments_analytics", "data": {"time": "2022-12-09", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971962} -{"stream": "segments_analytics", "data": {"time": "2022-12-10", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971964} -{"stream": "segments_analytics", "data": {"time": "2022-12-11", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971966} -{"stream": "segments_analytics", "data": {"time": "2022-12-12", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971968} -{"stream": "segments_analytics", "data": {"time": "2022-12-13", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971970} -{"stream": "segments_analytics", "data": {"time": "2022-12-14", "size": 0, "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4"}, "emitted_at": 1671100971972} -{"stream": "segments_analytics", "data": {"time": "2022-09-01", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972591} -{"stream": "segments_analytics", "data": {"time": "2022-09-02", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972594} -{"stream": "segments_analytics", "data": {"time": "2022-09-03", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972596} -{"stream": "segments_analytics", "data": {"time": "2022-09-04", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972598} -{"stream": "segments_analytics", "data": {"time": "2022-09-05", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972600} -{"stream": "segments_analytics", "data": {"time": "2022-09-06", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972603} -{"stream": "segments_analytics", "data": {"time": "2022-09-07", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972607} -{"stream": "segments_analytics", "data": {"time": "2022-09-08", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972609} -{"stream": "segments_analytics", "data": {"time": "2022-09-09", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972611} -{"stream": "segments_analytics", "data": {"time": "2022-09-10", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972613} -{"stream": "segments_analytics", "data": {"time": "2022-09-11", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972615} -{"stream": "segments_analytics", "data": {"time": "2022-09-12", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972618} -{"stream": "segments_analytics", "data": {"time": "2022-09-13", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972620} -{"stream": "segments_analytics", "data": {"time": "2022-09-14", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972622} -{"stream": "segments_analytics", "data": {"time": "2022-09-15", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972624} -{"stream": "segments_analytics", "data": {"time": "2022-09-16", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972626} -{"stream": "segments_analytics", "data": {"time": "2022-09-17", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972628} -{"stream": "segments_analytics", "data": {"time": "2022-09-18", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972630} -{"stream": "segments_analytics", "data": {"time": "2022-09-19", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972632} -{"stream": "segments_analytics", "data": {"time": "2022-09-20", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972635} -{"stream": "segments_analytics", "data": {"time": "2022-09-21", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972637} -{"stream": "segments_analytics", "data": {"time": "2022-09-22", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972640} -{"stream": "segments_analytics", "data": {"time": "2022-09-23", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972643} -{"stream": "segments_analytics", "data": {"time": "2022-09-24", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972645} -{"stream": "segments_analytics", "data": {"time": "2022-09-25", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972647} -{"stream": "segments_analytics", "data": {"time": "2022-09-26", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972650} -{"stream": "segments_analytics", "data": {"time": "2022-09-27", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972652} -{"stream": "segments_analytics", "data": {"time": "2022-09-28", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972654} -{"stream": "segments_analytics", "data": {"time": "2022-09-29", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972656} -{"stream": "segments_analytics", "data": {"time": "2022-09-30", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972658} -{"stream": "segments_analytics", "data": {"time": "2022-10-01", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972660} -{"stream": "segments_analytics", "data": {"time": "2022-10-02", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972662} -{"stream": "segments_analytics", "data": {"time": "2022-10-03", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972665} -{"stream": "segments_analytics", "data": {"time": "2022-10-04", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972667} -{"stream": "segments_analytics", "data": {"time": "2022-10-05", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972669} -{"stream": "segments_analytics", "data": {"time": "2022-10-06", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972671} -{"stream": "segments_analytics", "data": {"time": "2022-10-07", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972673} -{"stream": "segments_analytics", "data": {"time": "2022-10-08", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972675} -{"stream": "segments_analytics", "data": {"time": "2022-10-09", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972677} -{"stream": "segments_analytics", "data": {"time": "2022-10-10", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972680} -{"stream": "segments_analytics", "data": {"time": "2022-10-11", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972682} -{"stream": "segments_analytics", "data": {"time": "2022-10-12", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972684} -{"stream": "segments_analytics", "data": {"time": "2022-10-13", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972686} -{"stream": "segments_analytics", "data": {"time": "2022-10-14", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972688} -{"stream": "segments_analytics", "data": {"time": "2022-10-15", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972690} -{"stream": "segments_analytics", "data": {"time": "2022-10-16", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972692} -{"stream": "segments_analytics", "data": {"time": "2022-10-17", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972694} -{"stream": "segments_analytics", "data": {"time": "2022-10-18", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972697} -{"stream": "segments_analytics", "data": {"time": "2022-10-19", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972699} -{"stream": "segments_analytics", "data": {"time": "2022-10-20", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972701} -{"stream": "segments_analytics", "data": {"time": "2022-10-21", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972703} -{"stream": "segments_analytics", "data": {"time": "2022-10-22", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972705} -{"stream": "segments_analytics", "data": {"time": "2022-10-23", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972707} -{"stream": "segments_analytics", "data": {"time": "2022-10-24", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972709} -{"stream": "segments_analytics", "data": {"time": "2022-10-25", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972712} -{"stream": "segments_analytics", "data": {"time": "2022-10-26", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972714} -{"stream": "segments_analytics", "data": {"time": "2022-10-27", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972716} -{"stream": "segments_analytics", "data": {"time": "2022-10-28", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972718} -{"stream": "segments_analytics", "data": {"time": "2022-10-29", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972720} -{"stream": "segments_analytics", "data": {"time": "2022-10-30", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972722} -{"stream": "segments_analytics", "data": {"time": "2022-10-31", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972724} -{"stream": "segments_analytics", "data": {"time": "2022-11-01", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972727} -{"stream": "segments_analytics", "data": {"time": "2022-11-02", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972729} -{"stream": "segments_analytics", "data": {"time": "2022-11-03", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972731} -{"stream": "segments_analytics", "data": {"time": "2022-11-04", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972733} -{"stream": "segments_analytics", "data": {"time": "2022-11-05", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972735} -{"stream": "segments_analytics", "data": {"time": "2022-11-06", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972737} -{"stream": "segments_analytics", "data": {"time": "2022-11-07", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972739} -{"stream": "segments_analytics", "data": {"time": "2022-11-08", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972742} -{"stream": "segments_analytics", "data": {"time": "2022-11-09", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972744} -{"stream": "segments_analytics", "data": {"time": "2022-11-10", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972746} -{"stream": "segments_analytics", "data": {"time": "2022-11-11", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972748} -{"stream": "segments_analytics", "data": {"time": "2022-11-12", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972750} -{"stream": "segments_analytics", "data": {"time": "2022-11-13", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972752} -{"stream": "segments_analytics", "data": {"time": "2022-11-14", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972754} -{"stream": "segments_analytics", "data": {"time": "2022-11-15", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972757} -{"stream": "segments_analytics", "data": {"time": "2022-11-16", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972759} -{"stream": "segments_analytics", "data": {"time": "2022-11-17", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972761} -{"stream": "segments_analytics", "data": {"time": "2022-11-18", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972763} -{"stream": "segments_analytics", "data": {"time": "2022-11-19", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972765} -{"stream": "segments_analytics", "data": {"time": "2022-11-20", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972767} -{"stream": "segments_analytics", "data": {"time": "2022-11-21", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972769} -{"stream": "segments_analytics", "data": {"time": "2022-11-22", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972772} -{"stream": "segments_analytics", "data": {"time": "2022-11-23", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972774} -{"stream": "segments_analytics", "data": {"time": "2022-11-24", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972776} -{"stream": "segments_analytics", "data": {"time": "2022-11-25", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972778} -{"stream": "segments_analytics", "data": {"time": "2022-11-26", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972780} -{"stream": "segments_analytics", "data": {"time": "2022-11-27", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972782} -{"stream": "segments_analytics", "data": {"time": "2022-11-28", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972784} -{"stream": "segments_analytics", "data": {"time": "2022-11-29", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972786} -{"stream": "segments_analytics", "data": {"time": "2022-11-30", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972789} -{"stream": "segments_analytics", "data": {"time": "2022-12-01", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972791} -{"stream": "segments_analytics", "data": {"time": "2022-12-02", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972793} -{"stream": "segments_analytics", "data": {"time": "2022-12-03", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972795} -{"stream": "segments_analytics", "data": {"time": "2022-12-04", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972797} -{"stream": "segments_analytics", "data": {"time": "2022-12-05", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972799} -{"stream": "segments_analytics", "data": {"time": "2022-12-06", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972802} -{"stream": "segments_analytics", "data": {"time": "2022-12-07", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972804} -{"stream": "segments_analytics", "data": {"time": "2022-12-08", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100972806} -{"stream": "segments_analytics", "data": {"time": "2022-12-09", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100973412} -{"stream": "segments_analytics", "data": {"time": "2022-12-10", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100973414} -{"stream": "segments_analytics", "data": {"time": "2022-12-11", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100973416} -{"stream": "segments_analytics", "data": {"time": "2022-12-12", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100973418} -{"stream": "segments_analytics", "data": {"time": "2022-12-13", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100973420} -{"stream": "segments_analytics", "data": {"time": "2022-12-14", "size": 0, "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e"}, "emitted_at": 1671100973423} -{"stream": "segments_analytics", "data": {"time": "2022-09-01", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974026} -{"stream": "segments_analytics", "data": {"time": "2022-09-02", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974028} -{"stream": "segments_analytics", "data": {"time": "2022-09-03", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974030} -{"stream": "segments_analytics", "data": {"time": "2022-09-04", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974032} -{"stream": "segments_analytics", "data": {"time": "2022-09-05", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974034} -{"stream": "segments_analytics", "data": {"time": "2022-09-06", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974036} -{"stream": "segments_analytics", "data": {"time": "2022-09-07", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974038} -{"stream": "segments_analytics", "data": {"time": "2022-09-08", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974041} -{"stream": "segments_analytics", "data": {"time": "2022-09-09", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974043} -{"stream": "segments_analytics", "data": {"time": "2022-09-10", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974045} -{"stream": "segments_analytics", "data": {"time": "2022-09-11", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974047} -{"stream": "segments_analytics", "data": {"time": "2022-09-12", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974049} -{"stream": "segments_analytics", "data": {"time": "2022-09-13", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974051} -{"stream": "segments_analytics", "data": {"time": "2022-09-14", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974054} -{"stream": "segments_analytics", "data": {"time": "2022-09-15", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974056} -{"stream": "segments_analytics", "data": {"time": "2022-09-16", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974058} -{"stream": "segments_analytics", "data": {"time": "2022-09-17", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974060} -{"stream": "segments_analytics", "data": {"time": "2022-09-18", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974062} -{"stream": "segments_analytics", "data": {"time": "2022-09-19", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974064} -{"stream": "segments_analytics", "data": {"time": "2022-09-20", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974066} -{"stream": "segments_analytics", "data": {"time": "2022-09-21", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974069} -{"stream": "segments_analytics", "data": {"time": "2022-09-22", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974071} -{"stream": "segments_analytics", "data": {"time": "2022-09-23", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974073} -{"stream": "segments_analytics", "data": {"time": "2022-09-24", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974076} -{"stream": "segments_analytics", "data": {"time": "2022-09-25", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974078} -{"stream": "segments_analytics", "data": {"time": "2022-09-26", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974080} -{"stream": "segments_analytics", "data": {"time": "2022-09-27", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974082} -{"stream": "segments_analytics", "data": {"time": "2022-09-28", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974084} -{"stream": "segments_analytics", "data": {"time": "2022-09-29", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974087} -{"stream": "segments_analytics", "data": {"time": "2022-09-30", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974089} -{"stream": "segments_analytics", "data": {"time": "2022-10-01", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974091} -{"stream": "segments_analytics", "data": {"time": "2022-10-02", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974093} -{"stream": "segments_analytics", "data": {"time": "2022-10-03", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974095} -{"stream": "segments_analytics", "data": {"time": "2022-10-04", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974097} -{"stream": "segments_analytics", "data": {"time": "2022-10-05", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974099} -{"stream": "segments_analytics", "data": {"time": "2022-10-06", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974102} -{"stream": "segments_analytics", "data": {"time": "2022-10-07", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974104} -{"stream": "segments_analytics", "data": {"time": "2022-10-08", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974106} -{"stream": "segments_analytics", "data": {"time": "2022-10-09", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974108} -{"stream": "segments_analytics", "data": {"time": "2022-10-10", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974110} -{"stream": "segments_analytics", "data": {"time": "2022-10-11", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974112} -{"stream": "segments_analytics", "data": {"time": "2022-10-12", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974114} -{"stream": "segments_analytics", "data": {"time": "2022-10-13", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974116} -{"stream": "segments_analytics", "data": {"time": "2022-10-14", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974119} -{"stream": "segments_analytics", "data": {"time": "2022-10-15", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974121} -{"stream": "segments_analytics", "data": {"time": "2022-10-16", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974123} -{"stream": "segments_analytics", "data": {"time": "2022-10-17", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974125} -{"stream": "segments_analytics", "data": {"time": "2022-10-18", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974127} -{"stream": "segments_analytics", "data": {"time": "2022-10-19", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974129} -{"stream": "segments_analytics", "data": {"time": "2022-10-20", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974131} -{"stream": "segments_analytics", "data": {"time": "2022-10-21", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974134} -{"stream": "segments_analytics", "data": {"time": "2022-10-22", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974136} -{"stream": "segments_analytics", "data": {"time": "2022-10-23", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974138} -{"stream": "segments_analytics", "data": {"time": "2022-10-24", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974140} -{"stream": "segments_analytics", "data": {"time": "2022-10-25", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974142} -{"stream": "segments_analytics", "data": {"time": "2022-10-26", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974144} -{"stream": "segments_analytics", "data": {"time": "2022-10-27", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974146} -{"stream": "segments_analytics", "data": {"time": "2022-10-28", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974149} -{"stream": "segments_analytics", "data": {"time": "2022-10-29", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974151} -{"stream": "segments_analytics", "data": {"time": "2022-10-30", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974153} -{"stream": "segments_analytics", "data": {"time": "2022-10-31", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974155} -{"stream": "segments_analytics", "data": {"time": "2022-11-01", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974157} -{"stream": "segments_analytics", "data": {"time": "2022-11-02", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974159} -{"stream": "segments_analytics", "data": {"time": "2022-11-03", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974162} -{"stream": "segments_analytics", "data": {"time": "2022-11-04", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974164} -{"stream": "segments_analytics", "data": {"time": "2022-11-05", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974166} -{"stream": "segments_analytics", "data": {"time": "2022-11-06", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974168} -{"stream": "segments_analytics", "data": {"time": "2022-11-07", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974170} -{"stream": "segments_analytics", "data": {"time": "2022-11-08", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974173} -{"stream": "segments_analytics", "data": {"time": "2022-11-09", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974175} -{"stream": "segments_analytics", "data": {"time": "2022-11-10", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974177} -{"stream": "segments_analytics", "data": {"time": "2022-11-11", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974179} -{"stream": "segments_analytics", "data": {"time": "2022-11-12", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974181} -{"stream": "segments_analytics", "data": {"time": "2022-11-13", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974183} -{"stream": "segments_analytics", "data": {"time": "2022-11-14", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974185} -{"stream": "segments_analytics", "data": {"time": "2022-11-15", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974188} -{"stream": "segments_analytics", "data": {"time": "2022-11-16", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974190} -{"stream": "segments_analytics", "data": {"time": "2022-11-17", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974192} -{"stream": "segments_analytics", "data": {"time": "2022-11-18", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974194} -{"stream": "segments_analytics", "data": {"time": "2022-11-19", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974196} -{"stream": "segments_analytics", "data": {"time": "2022-11-20", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974198} -{"stream": "segments_analytics", "data": {"time": "2022-11-21", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974200} -{"stream": "segments_analytics", "data": {"time": "2022-11-22", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974203} -{"stream": "segments_analytics", "data": {"time": "2022-11-23", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974205} -{"stream": "segments_analytics", "data": {"time": "2022-11-24", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974207} -{"stream": "segments_analytics", "data": {"time": "2022-11-25", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974209} -{"stream": "segments_analytics", "data": {"time": "2022-11-26", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974211} -{"stream": "segments_analytics", "data": {"time": "2022-11-27", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974214} -{"stream": "segments_analytics", "data": {"time": "2022-11-28", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974216} -{"stream": "segments_analytics", "data": {"time": "2022-11-29", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974218} -{"stream": "segments_analytics", "data": {"time": "2022-11-30", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974220} -{"stream": "segments_analytics", "data": {"time": "2022-12-01", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974222} -{"stream": "segments_analytics", "data": {"time": "2022-12-02", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974224} -{"stream": "segments_analytics", "data": {"time": "2022-12-03", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974226} -{"stream": "segments_analytics", "data": {"time": "2022-12-04", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974229} -{"stream": "segments_analytics", "data": {"time": "2022-12-05", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974231} -{"stream": "segments_analytics", "data": {"time": "2022-12-06", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974233} -{"stream": "segments_analytics", "data": {"time": "2022-12-07", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974235} -{"stream": "segments_analytics", "data": {"time": "2022-12-08", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974237} -{"stream": "segments_analytics", "data": {"time": "2022-12-09", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974550} -{"stream": "segments_analytics", "data": {"time": "2022-12-10", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974558} -{"stream": "segments_analytics", "data": {"time": "2022-12-11", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974565} -{"stream": "segments_analytics", "data": {"time": "2022-12-12", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974570} -{"stream": "segments_analytics", "data": {"time": "2022-12-13", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974575} -{"stream": "segments_analytics", "data": {"time": "2022-12-14", "size": 0, "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e"}, "emitted_at": 1671100974579} -{"stream": "segments_analytics", "data": {"time": "2022-09-01", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975153} -{"stream": "segments_analytics", "data": {"time": "2022-09-02", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975155} -{"stream": "segments_analytics", "data": {"time": "2022-09-03", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975157} -{"stream": "segments_analytics", "data": {"time": "2022-09-04", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975159} -{"stream": "segments_analytics", "data": {"time": "2022-09-05", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975161} -{"stream": "segments_analytics", "data": {"time": "2022-09-06", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975163} -{"stream": "segments_analytics", "data": {"time": "2022-09-07", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975165} -{"stream": "segments_analytics", "data": {"time": "2022-09-08", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975168} -{"stream": "segments_analytics", "data": {"time": "2022-09-09", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975170} -{"stream": "segments_analytics", "data": {"time": "2022-09-10", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975172} -{"stream": "segments_analytics", "data": {"time": "2022-09-11", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975174} -{"stream": "segments_analytics", "data": {"time": "2022-09-12", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975176} -{"stream": "segments_analytics", "data": {"time": "2022-09-13", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975178} -{"stream": "segments_analytics", "data": {"time": "2022-09-14", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975181} -{"stream": "segments_analytics", "data": {"time": "2022-09-15", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975183} -{"stream": "segments_analytics", "data": {"time": "2022-09-16", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975185} -{"stream": "segments_analytics", "data": {"time": "2022-09-17", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975187} -{"stream": "segments_analytics", "data": {"time": "2022-09-18", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975189} -{"stream": "segments_analytics", "data": {"time": "2022-09-19", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975191} -{"stream": "segments_analytics", "data": {"time": "2022-09-20", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975193} -{"stream": "segments_analytics", "data": {"time": "2022-09-21", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975196} -{"stream": "segments_analytics", "data": {"time": "2022-09-22", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975198} -{"stream": "segments_analytics", "data": {"time": "2022-09-23", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975200} -{"stream": "segments_analytics", "data": {"time": "2022-09-24", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975202} -{"stream": "segments_analytics", "data": {"time": "2022-09-25", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975204} -{"stream": "segments_analytics", "data": {"time": "2022-09-26", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975206} -{"stream": "segments_analytics", "data": {"time": "2022-09-27", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975209} -{"stream": "segments_analytics", "data": {"time": "2022-09-28", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975211} -{"stream": "segments_analytics", "data": {"time": "2022-09-29", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975213} -{"stream": "segments_analytics", "data": {"time": "2022-09-30", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975215} -{"stream": "segments_analytics", "data": {"time": "2022-10-01", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975217} -{"stream": "segments_analytics", "data": {"time": "2022-10-02", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975219} -{"stream": "segments_analytics", "data": {"time": "2022-10-03", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975222} -{"stream": "segments_analytics", "data": {"time": "2022-10-04", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975224} -{"stream": "segments_analytics", "data": {"time": "2022-10-05", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975226} -{"stream": "segments_analytics", "data": {"time": "2022-10-06", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975228} -{"stream": "segments_analytics", "data": {"time": "2022-10-07", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975230} -{"stream": "segments_analytics", "data": {"time": "2022-10-08", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975232} -{"stream": "segments_analytics", "data": {"time": "2022-10-09", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975234} -{"stream": "segments_analytics", "data": {"time": "2022-10-10", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975237} -{"stream": "segments_analytics", "data": {"time": "2022-10-11", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975239} -{"stream": "segments_analytics", "data": {"time": "2022-10-12", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975241} -{"stream": "segments_analytics", "data": {"time": "2022-10-13", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975243} -{"stream": "segments_analytics", "data": {"time": "2022-10-14", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975245} -{"stream": "segments_analytics", "data": {"time": "2022-10-15", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975247} -{"stream": "segments_analytics", "data": {"time": "2022-10-16", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975249} -{"stream": "segments_analytics", "data": {"time": "2022-10-17", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975252} -{"stream": "segments_analytics", "data": {"time": "2022-10-18", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975254} -{"stream": "segments_analytics", "data": {"time": "2022-10-19", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975256} -{"stream": "segments_analytics", "data": {"time": "2022-10-20", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975258} -{"stream": "segments_analytics", "data": {"time": "2022-10-21", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975260} -{"stream": "segments_analytics", "data": {"time": "2022-10-22", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975262} -{"stream": "segments_analytics", "data": {"time": "2022-10-23", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975264} -{"stream": "segments_analytics", "data": {"time": "2022-10-24", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975267} -{"stream": "segments_analytics", "data": {"time": "2022-10-25", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975269} -{"stream": "segments_analytics", "data": {"time": "2022-10-26", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975271} -{"stream": "segments_analytics", "data": {"time": "2022-10-27", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975273} -{"stream": "segments_analytics", "data": {"time": "2022-10-28", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975275} -{"stream": "segments_analytics", "data": {"time": "2022-10-29", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975277} -{"stream": "segments_analytics", "data": {"time": "2022-10-30", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975279} -{"stream": "segments_analytics", "data": {"time": "2022-10-31", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975282} -{"stream": "segments_analytics", "data": {"time": "2022-11-01", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975284} -{"stream": "segments_analytics", "data": {"time": "2022-11-02", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975286} -{"stream": "segments_analytics", "data": {"time": "2022-11-03", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975288} -{"stream": "segments_analytics", "data": {"time": "2022-11-04", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975290} -{"stream": "segments_analytics", "data": {"time": "2022-11-05", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975292} -{"stream": "segments_analytics", "data": {"time": "2022-11-06", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975294} -{"stream": "segments_analytics", "data": {"time": "2022-11-07", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975297} -{"stream": "segments_analytics", "data": {"time": "2022-11-08", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975299} -{"stream": "segments_analytics", "data": {"time": "2022-11-09", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975301} -{"stream": "segments_analytics", "data": {"time": "2022-11-10", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975303} -{"stream": "segments_analytics", "data": {"time": "2022-11-11", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975305} -{"stream": "segments_analytics", "data": {"time": "2022-11-12", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975307} -{"stream": "segments_analytics", "data": {"time": "2022-11-13", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975309} -{"stream": "segments_analytics", "data": {"time": "2022-11-14", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975312} -{"stream": "segments_analytics", "data": {"time": "2022-11-15", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975314} -{"stream": "segments_analytics", "data": {"time": "2022-11-16", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975316} -{"stream": "segments_analytics", "data": {"time": "2022-11-17", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975318} -{"stream": "segments_analytics", "data": {"time": "2022-11-18", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975320} -{"stream": "segments_analytics", "data": {"time": "2022-11-19", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975322} -{"stream": "segments_analytics", "data": {"time": "2022-11-20", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975324} -{"stream": "segments_analytics", "data": {"time": "2022-11-21", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975327} -{"stream": "segments_analytics", "data": {"time": "2022-11-22", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975329} -{"stream": "segments_analytics", "data": {"time": "2022-11-23", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975331} -{"stream": "segments_analytics", "data": {"time": "2022-11-24", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975333} -{"stream": "segments_analytics", "data": {"time": "2022-11-25", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975335} -{"stream": "segments_analytics", "data": {"time": "2022-11-26", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975337} -{"stream": "segments_analytics", "data": {"time": "2022-11-27", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975340} -{"stream": "segments_analytics", "data": {"time": "2022-11-28", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975342} -{"stream": "segments_analytics", "data": {"time": "2022-11-29", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975344} -{"stream": "segments_analytics", "data": {"time": "2022-11-30", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975346} -{"stream": "segments_analytics", "data": {"time": "2022-12-01", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975348} -{"stream": "segments_analytics", "data": {"time": "2022-12-02", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975350} -{"stream": "segments_analytics", "data": {"time": "2022-12-03", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975352} -{"stream": "segments_analytics", "data": {"time": "2022-12-04", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975355} -{"stream": "segments_analytics", "data": {"time": "2022-12-05", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975357} -{"stream": "segments_analytics", "data": {"time": "2022-12-06", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975359} -{"stream": "segments_analytics", "data": {"time": "2022-12-07", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975361} -{"stream": "segments_analytics", "data": {"time": "2022-12-08", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975363} -{"stream": "segments_analytics", "data": {"time": "2022-12-09", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975974} -{"stream": "segments_analytics", "data": {"time": "2022-12-10", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975976} -{"stream": "segments_analytics", "data": {"time": "2022-12-11", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975978} -{"stream": "segments_analytics", "data": {"time": "2022-12-12", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975980} -{"stream": "segments_analytics", "data": {"time": "2022-12-13", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975982} -{"stream": "segments_analytics", "data": {"time": "2022-12-14", "size": 0, "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10"}, "emitted_at": 1671100975985} -{"stream": "segments_analytics", "data": {"time": "2022-09-01", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976558} -{"stream": "segments_analytics", "data": {"time": "2022-09-02", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976561} -{"stream": "segments_analytics", "data": {"time": "2022-09-03", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976563} -{"stream": "segments_analytics", "data": {"time": "2022-09-04", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976565} -{"stream": "segments_analytics", "data": {"time": "2022-09-05", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976567} -{"stream": "segments_analytics", "data": {"time": "2022-09-06", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976570} -{"stream": "segments_analytics", "data": {"time": "2022-09-07", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976572} -{"stream": "segments_analytics", "data": {"time": "2022-09-08", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976574} -{"stream": "segments_analytics", "data": {"time": "2022-09-09", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976576} -{"stream": "segments_analytics", "data": {"time": "2022-09-10", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976578} -{"stream": "segments_analytics", "data": {"time": "2022-09-11", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976581} -{"stream": "segments_analytics", "data": {"time": "2022-09-12", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976583} -{"stream": "segments_analytics", "data": {"time": "2022-09-13", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976585} -{"stream": "segments_analytics", "data": {"time": "2022-09-14", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976587} -{"stream": "segments_analytics", "data": {"time": "2022-09-15", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976589} -{"stream": "segments_analytics", "data": {"time": "2022-09-16", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976591} -{"stream": "segments_analytics", "data": {"time": "2022-09-17", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976594} -{"stream": "segments_analytics", "data": {"time": "2022-09-18", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976596} -{"stream": "segments_analytics", "data": {"time": "2022-09-19", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976598} -{"stream": "segments_analytics", "data": {"time": "2022-09-20", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976600} -{"stream": "segments_analytics", "data": {"time": "2022-09-21", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976602} -{"stream": "segments_analytics", "data": {"time": "2022-09-22", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976605} -{"stream": "segments_analytics", "data": {"time": "2022-09-23", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976607} -{"stream": "segments_analytics", "data": {"time": "2022-09-24", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976609} -{"stream": "segments_analytics", "data": {"time": "2022-09-25", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976611} -{"stream": "segments_analytics", "data": {"time": "2022-09-26", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976613} -{"stream": "segments_analytics", "data": {"time": "2022-09-27", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976615} -{"stream": "segments_analytics", "data": {"time": "2022-09-28", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976617} -{"stream": "segments_analytics", "data": {"time": "2022-09-29", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976620} -{"stream": "segments_analytics", "data": {"time": "2022-09-30", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976622} -{"stream": "segments_analytics", "data": {"time": "2022-10-01", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976624} -{"stream": "segments_analytics", "data": {"time": "2022-10-02", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976626} -{"stream": "segments_analytics", "data": {"time": "2022-10-03", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976628} -{"stream": "segments_analytics", "data": {"time": "2022-10-04", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976630} -{"stream": "segments_analytics", "data": {"time": "2022-10-05", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976633} -{"stream": "segments_analytics", "data": {"time": "2022-10-06", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976635} -{"stream": "segments_analytics", "data": {"time": "2022-10-07", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976637} -{"stream": "segments_analytics", "data": {"time": "2022-10-08", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976639} -{"stream": "segments_analytics", "data": {"time": "2022-10-09", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976642} -{"stream": "segments_analytics", "data": {"time": "2022-10-10", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976644} -{"stream": "segments_analytics", "data": {"time": "2022-10-11", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976646} -{"stream": "segments_analytics", "data": {"time": "2022-10-12", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976648} -{"stream": "segments_analytics", "data": {"time": "2022-10-13", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976651} -{"stream": "segments_analytics", "data": {"time": "2022-10-14", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976653} -{"stream": "segments_analytics", "data": {"time": "2022-10-15", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976655} -{"stream": "segments_analytics", "data": {"time": "2022-10-16", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976657} -{"stream": "segments_analytics", "data": {"time": "2022-10-17", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976659} -{"stream": "segments_analytics", "data": {"time": "2022-10-18", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976662} -{"stream": "segments_analytics", "data": {"time": "2022-10-19", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976664} -{"stream": "segments_analytics", "data": {"time": "2022-10-20", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976666} -{"stream": "segments_analytics", "data": {"time": "2022-10-21", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976668} -{"stream": "segments_analytics", "data": {"time": "2022-10-22", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976670} -{"stream": "segments_analytics", "data": {"time": "2022-10-23", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976672} -{"stream": "segments_analytics", "data": {"time": "2022-10-24", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976675} -{"stream": "segments_analytics", "data": {"time": "2022-10-25", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976677} -{"stream": "segments_analytics", "data": {"time": "2022-10-26", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976679} -{"stream": "segments_analytics", "data": {"time": "2022-10-27", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976681} -{"stream": "segments_analytics", "data": {"time": "2022-10-28", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976683} -{"stream": "segments_analytics", "data": {"time": "2022-10-29", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976685} -{"stream": "segments_analytics", "data": {"time": "2022-10-30", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976687} -{"stream": "segments_analytics", "data": {"time": "2022-10-31", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976690} -{"stream": "segments_analytics", "data": {"time": "2022-11-01", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976692} -{"stream": "segments_analytics", "data": {"time": "2022-11-02", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976694} -{"stream": "segments_analytics", "data": {"time": "2022-11-03", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976696} -{"stream": "segments_analytics", "data": {"time": "2022-11-04", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976698} -{"stream": "segments_analytics", "data": {"time": "2022-11-05", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976700} -{"stream": "segments_analytics", "data": {"time": "2022-11-06", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976702} -{"stream": "segments_analytics", "data": {"time": "2022-11-07", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976705} -{"stream": "segments_analytics", "data": {"time": "2022-11-08", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976707} -{"stream": "segments_analytics", "data": {"time": "2022-11-09", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976709} -{"stream": "segments_analytics", "data": {"time": "2022-11-10", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976711} -{"stream": "segments_analytics", "data": {"time": "2022-11-11", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976713} -{"stream": "segments_analytics", "data": {"time": "2022-11-12", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976715} -{"stream": "segments_analytics", "data": {"time": "2022-11-13", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976717} -{"stream": "segments_analytics", "data": {"time": "2022-11-14", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976720} -{"stream": "segments_analytics", "data": {"time": "2022-11-15", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976722} -{"stream": "segments_analytics", "data": {"time": "2022-11-16", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976724} -{"stream": "segments_analytics", "data": {"time": "2022-11-17", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976726} -{"stream": "segments_analytics", "data": {"time": "2022-11-18", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976728} -{"stream": "segments_analytics", "data": {"time": "2022-11-19", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976730} -{"stream": "segments_analytics", "data": {"time": "2022-11-20", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976732} -{"stream": "segments_analytics", "data": {"time": "2022-11-21", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976734} -{"stream": "segments_analytics", "data": {"time": "2022-11-22", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976737} -{"stream": "segments_analytics", "data": {"time": "2022-11-23", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976739} -{"stream": "segments_analytics", "data": {"time": "2022-11-24", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976741} -{"stream": "segments_analytics", "data": {"time": "2022-11-25", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976743} -{"stream": "segments_analytics", "data": {"time": "2022-11-26", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976745} -{"stream": "segments_analytics", "data": {"time": "2022-11-27", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976747} -{"stream": "segments_analytics", "data": {"time": "2022-11-28", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976750} -{"stream": "segments_analytics", "data": {"time": "2022-11-29", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976752} -{"stream": "segments_analytics", "data": {"time": "2022-11-30", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976754} -{"stream": "segments_analytics", "data": {"time": "2022-12-01", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976756} -{"stream": "segments_analytics", "data": {"time": "2022-12-02", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976758} -{"stream": "segments_analytics", "data": {"time": "2022-12-03", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976761} -{"stream": "segments_analytics", "data": {"time": "2022-12-04", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976763} -{"stream": "segments_analytics", "data": {"time": "2022-12-05", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976765} -{"stream": "segments_analytics", "data": {"time": "2022-12-06", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976767} -{"stream": "segments_analytics", "data": {"time": "2022-12-07", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976769} -{"stream": "segments_analytics", "data": {"time": "2022-12-08", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100976771} -{"stream": "segments_analytics", "data": {"time": "2022-12-09", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100977053} -{"stream": "segments_analytics", "data": {"time": "2022-12-10", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100977062} -{"stream": "segments_analytics", "data": {"time": "2022-12-11", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100977069} -{"stream": "segments_analytics", "data": {"time": "2022-12-12", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100977073} -{"stream": "segments_analytics", "data": {"time": "2022-12-13", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100977077} -{"stream": "segments_analytics", "data": {"time": "2022-12-14", "size": 0, "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e"}, "emitted_at": 1671100977081} -{"stream": "segments_analytics", "data": {"time": "2022-09-01", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977675} -{"stream": "segments_analytics", "data": {"time": "2022-09-02", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977684} -{"stream": "segments_analytics", "data": {"time": "2022-09-03", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977690} -{"stream": "segments_analytics", "data": {"time": "2022-09-04", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977695} -{"stream": "segments_analytics", "data": {"time": "2022-09-05", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977699} -{"stream": "segments_analytics", "data": {"time": "2022-09-06", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977702} -{"stream": "segments_analytics", "data": {"time": "2022-09-07", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977705} -{"stream": "segments_analytics", "data": {"time": "2022-09-08", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977708} -{"stream": "segments_analytics", "data": {"time": "2022-09-09", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977711} -{"stream": "segments_analytics", "data": {"time": "2022-09-10", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977713} -{"stream": "segments_analytics", "data": {"time": "2022-09-11", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977715} -{"stream": "segments_analytics", "data": {"time": "2022-09-12", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977718} -{"stream": "segments_analytics", "data": {"time": "2022-09-13", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977720} -{"stream": "segments_analytics", "data": {"time": "2022-09-14", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977722} -{"stream": "segments_analytics", "data": {"time": "2022-09-15", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977724} -{"stream": "segments_analytics", "data": {"time": "2022-09-16", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977726} -{"stream": "segments_analytics", "data": {"time": "2022-09-17", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977728} -{"stream": "segments_analytics", "data": {"time": "2022-09-18", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977731} -{"stream": "segments_analytics", "data": {"time": "2022-09-19", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977733} -{"stream": "segments_analytics", "data": {"time": "2022-09-20", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977735} -{"stream": "segments_analytics", "data": {"time": "2022-09-21", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977737} -{"stream": "segments_analytics", "data": {"time": "2022-09-22", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977739} -{"stream": "segments_analytics", "data": {"time": "2022-09-23", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977741} -{"stream": "segments_analytics", "data": {"time": "2022-09-24", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977743} -{"stream": "segments_analytics", "data": {"time": "2022-09-25", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977746} -{"stream": "segments_analytics", "data": {"time": "2022-09-26", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977748} -{"stream": "segments_analytics", "data": {"time": "2022-09-27", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977750} -{"stream": "segments_analytics", "data": {"time": "2022-09-28", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977752} -{"stream": "segments_analytics", "data": {"time": "2022-09-29", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977754} -{"stream": "segments_analytics", "data": {"time": "2022-09-30", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977756} -{"stream": "segments_analytics", "data": {"time": "2022-10-01", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977758} -{"stream": "segments_analytics", "data": {"time": "2022-10-02", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977761} -{"stream": "segments_analytics", "data": {"time": "2022-10-03", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977763} -{"stream": "segments_analytics", "data": {"time": "2022-10-04", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977765} -{"stream": "segments_analytics", "data": {"time": "2022-10-05", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977767} -{"stream": "segments_analytics", "data": {"time": "2022-10-06", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977769} -{"stream": "segments_analytics", "data": {"time": "2022-10-07", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977771} -{"stream": "segments_analytics", "data": {"time": "2022-10-08", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977773} -{"stream": "segments_analytics", "data": {"time": "2022-10-09", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977776} -{"stream": "segments_analytics", "data": {"time": "2022-10-10", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977778} -{"stream": "segments_analytics", "data": {"time": "2022-10-11", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977780} -{"stream": "segments_analytics", "data": {"time": "2022-10-12", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977782} -{"stream": "segments_analytics", "data": {"time": "2022-10-13", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977784} -{"stream": "segments_analytics", "data": {"time": "2022-10-14", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977786} -{"stream": "segments_analytics", "data": {"time": "2022-10-15", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977788} -{"stream": "segments_analytics", "data": {"time": "2022-10-16", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977791} -{"stream": "segments_analytics", "data": {"time": "2022-10-17", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977793} -{"stream": "segments_analytics", "data": {"time": "2022-10-18", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977795} -{"stream": "segments_analytics", "data": {"time": "2022-10-19", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977797} -{"stream": "segments_analytics", "data": {"time": "2022-10-20", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977799} -{"stream": "segments_analytics", "data": {"time": "2022-10-21", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977801} -{"stream": "segments_analytics", "data": {"time": "2022-10-22", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977803} -{"stream": "segments_analytics", "data": {"time": "2022-10-23", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977805} -{"stream": "segments_analytics", "data": {"time": "2022-10-24", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977808} -{"stream": "segments_analytics", "data": {"time": "2022-10-25", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977810} -{"stream": "segments_analytics", "data": {"time": "2022-10-26", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977812} -{"stream": "segments_analytics", "data": {"time": "2022-10-27", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977814} -{"stream": "segments_analytics", "data": {"time": "2022-10-28", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977816} -{"stream": "segments_analytics", "data": {"time": "2022-10-29", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977818} -{"stream": "segments_analytics", "data": {"time": "2022-10-30", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977820} -{"stream": "segments_analytics", "data": {"time": "2022-10-31", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977822} -{"stream": "segments_analytics", "data": {"time": "2022-11-01", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977825} -{"stream": "segments_analytics", "data": {"time": "2022-11-02", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977827} -{"stream": "segments_analytics", "data": {"time": "2022-11-03", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977829} -{"stream": "segments_analytics", "data": {"time": "2022-11-04", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977831} -{"stream": "segments_analytics", "data": {"time": "2022-11-05", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977834} -{"stream": "segments_analytics", "data": {"time": "2022-11-06", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977836} -{"stream": "segments_analytics", "data": {"time": "2022-11-07", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977838} -{"stream": "segments_analytics", "data": {"time": "2022-11-08", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977840} -{"stream": "segments_analytics", "data": {"time": "2022-11-09", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977842} -{"stream": "segments_analytics", "data": {"time": "2022-11-10", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977844} -{"stream": "segments_analytics", "data": {"time": "2022-11-11", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977846} -{"stream": "segments_analytics", "data": {"time": "2022-11-12", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977849} -{"stream": "segments_analytics", "data": {"time": "2022-11-13", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977851} -{"stream": "segments_analytics", "data": {"time": "2022-11-14", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977853} -{"stream": "segments_analytics", "data": {"time": "2022-11-15", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977855} -{"stream": "segments_analytics", "data": {"time": "2022-11-16", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977857} -{"stream": "segments_analytics", "data": {"time": "2022-11-17", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977859} -{"stream": "segments_analytics", "data": {"time": "2022-11-18", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977861} -{"stream": "segments_analytics", "data": {"time": "2022-11-19", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977863} -{"stream": "segments_analytics", "data": {"time": "2022-11-20", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977866} -{"stream": "segments_analytics", "data": {"time": "2022-11-21", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977868} -{"stream": "segments_analytics", "data": {"time": "2022-11-22", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977870} -{"stream": "segments_analytics", "data": {"time": "2022-11-23", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977872} -{"stream": "segments_analytics", "data": {"time": "2022-11-24", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977874} -{"stream": "segments_analytics", "data": {"time": "2022-11-25", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977876} -{"stream": "segments_analytics", "data": {"time": "2022-11-26", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977878} -{"stream": "segments_analytics", "data": {"time": "2022-11-27", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977881} -{"stream": "segments_analytics", "data": {"time": "2022-11-28", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977883} -{"stream": "segments_analytics", "data": {"time": "2022-11-29", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977885} -{"stream": "segments_analytics", "data": {"time": "2022-11-30", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977887} -{"stream": "segments_analytics", "data": {"time": "2022-12-01", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977889} -{"stream": "segments_analytics", "data": {"time": "2022-12-02", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977891} -{"stream": "segments_analytics", "data": {"time": "2022-12-03", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977893} -{"stream": "segments_analytics", "data": {"time": "2022-12-04", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977896} -{"stream": "segments_analytics", "data": {"time": "2022-12-05", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977898} -{"stream": "segments_analytics", "data": {"time": "2022-12-06", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977900} -{"stream": "segments_analytics", "data": {"time": "2022-12-07", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977902} -{"stream": "segments_analytics", "data": {"time": "2022-12-08", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100977904} -{"stream": "segments_analytics", "data": {"time": "2022-12-09", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100978483} -{"stream": "segments_analytics", "data": {"time": "2022-12-10", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100978492} -{"stream": "segments_analytics", "data": {"time": "2022-12-11", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100978498} -{"stream": "segments_analytics", "data": {"time": "2022-12-12", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100978503} -{"stream": "segments_analytics", "data": {"time": "2022-12-13", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100978507} -{"stream": "segments_analytics", "data": {"time": "2022-12-14", "size": 0, "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c"}, "emitted_at": 1671100978510} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-braze/integration_tests/minimal.json b/airbyte-integrations/connectors/source-braze/integration_tests/minimal.json new file mode 100644 index 000000000000..1b20ba80105d --- /dev/null +++ b/airbyte-integrations/connectors/source-braze/integration_tests/minimal.json @@ -0,0 +1,31 @@ +{ + "streams": [ + { + "stream": { + "name": "campaigns_details", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "canvases_details", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "segments_details", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-braze/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-braze/integration_tests/sample_state.json index 40da794e3f38..5e838b8642c0 100644 --- a/airbyte-integrations/connectors/source-braze/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-braze/integration_tests/sample_state.json @@ -1,19 +1,266 @@ -{ - "campaigns_analytics": { - "time": "2022-09-01", - "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a" +[ + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "campaigns_analytics" + }, + "stream_state": { + "states": [ + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "campaign_id": "0c38a773-8e62-4dcb-a492-c973c8c6ab9b", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "campaign_id": "a9b2442f-ff68-49dd-96e0-b9318957cd9a", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "campaign_id": "8c846e45-a8ab-4afe-b315-04627504b526", + "parent_slice": {} + } + } + ] + } + } }, - "canvases_analytics": { - "time": "2022-09-01", - "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2" + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "canvases_analytics" + }, + "stream_state": { + "states": [ + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "canvas_id": "c73f8e7f-c6a5-441d-b227-46cb70f9f7d2", + "parent_slice": {} + } + } + ] + } + } }, - "events_analytics": {}, - "kpi_daily_new_users": {}, - "kpi_daily_active_users": {}, - "kpi_daily_app_uninstalls": {}, - "cards_analytics": {}, - "segments_analytics": { - "time": "2022-09-01", - "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c" + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "cards_analytics" + }, + "stream_state": { + "states": [ + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "card_id": "609e4d4c-367b-4c87-a8ad-b1f903d058fd", + "parent_slice": {} + } + } + ] + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "segments_analytics" + }, + "stream_state": { + "states": [ + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "segment_id": "cd7a9141-3457-4c37-9336-52845dd5fe87", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "segment_id": "2eaf781b-c4fb-4739-8ff1-080144e1aa1a", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "segment_id": "ce4e667c-3180-45dd-a200-7f3a9489f306", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "segment_id": "9d877c05-a4f6-4409-ae5a-eb7676d1e0b4", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "segment_id": "bf9673e5-41ee-4df4-b528-e46829efd646", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "segment_id": "6a51c1cf-8997-4919-a973-38557dde18de", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "segment_id": "18ae06cf-9f79-4b47-ab41-872caa0a2aac", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "segment_id": "d75a8f86-dd0d-4c60-b97e-855ca9f468e2", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "segment_id": "c693d38e-2fb6-4ef6-8e24-67b0977655ac", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "segment_id": "fb7e1079-f353-4887-a183-f197b30cf27e", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "segment_id": "7edf3784-c465-4ca8-89be-8a07c154fbf9", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "segment_id": "2ea0eeb1-26cf-4e63-a5ac-bec5622cdf71", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "segment_id": "5646a5f6-fc59-4e70-83e1-3e89a1c79493", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "segment_id": "b508cdb9-75cb-4122-8c6d-a7a6bfd24bf4", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "segment_id": "a94939de-135b-4e1a-9cdf-a5940747415e", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "segment_id": "8043672e-73fb-4731-854b-f9d315aa2b6e", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "segment_id": "a7a3a752-046e-4df4-bf3b-4e50d88d9a10", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "segment_id": "4926071e-2f5a-44f8-98b8-0d9db823283e", + "parent_slice": {} + } + }, + { + "cursor": { + "time": "2020-11-03" + }, + "partition": { + "segment_id": "8b922b25-dcc6-4e4d-b345-f7b06106046c", + "parent_slice": {} + } + } + ] + } + } } -} +] diff --git a/airbyte-integrations/connectors/source-braze/metadata.yaml b/airbyte-integrations/connectors/source-braze/metadata.yaml index 16aa19bfc0c8..bde782b0dc7f 100644 --- a/airbyte-integrations/connectors/source-braze/metadata.yaml +++ b/airbyte-integrations/connectors/source-braze/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 68b9c98e-0747-4c84-b05b-d30b47686725 - dockerImageTag: 0.1.3 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-braze documentationUrl: https://docs.airbyte.com/integrations/sources/braze githubIssueLabel: source-braze diff --git a/airbyte-integrations/connectors/source-braze/setup.py b/airbyte-integrations/connectors/source-braze/setup.py index 3bdf009a6ece..aade45358b41 100644 --- a/airbyte-integrations/connectors/source-braze/setup.py +++ b/airbyte-integrations/connectors/source-braze/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ diff --git a/airbyte-integrations/connectors/source-braze/source_braze/components.py b/airbyte-integrations/connectors/source-braze/source_braze/components.py new file mode 100644 index 000000000000..06e003ec51d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-braze/source_braze/components.py @@ -0,0 +1,20 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import dataclass + +import requests +from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor +from airbyte_cdk.sources.declarative.types import Record + + +@dataclass +class EventsRecordExtractor(DpathExtractor): + def extract_records(self, response: requests.Response) -> list[Record]: + response_body = self.decoder.decode(response) + events = response_body.get("events") + if events: + return [{"event_name": value} for value in events] + else: + return [] diff --git a/airbyte-integrations/connectors/source-braze/source_braze/datetime_incremental_sync.py b/airbyte-integrations/connectors/source-braze/source_braze/datetime_incremental_sync.py index 17a592226e7e..6662ac61868e 100644 --- a/airbyte-integrations/connectors/source-braze/source_braze/datetime_incremental_sync.py +++ b/airbyte-integrations/connectors/source-braze/source_braze/datetime_incremental_sync.py @@ -46,7 +46,7 @@ def _partition_daterange(self, start, end, step: datetime.timedelta): if get_start_time(dr) < get_end_time(dr) ] for i, _slice in enumerate(date_range): - start_time = self._parser.parse(get_start_time(_slice), self.start_datetime.datetime_format, self._timezone) - end_time = self._parser.parse(get_end_time(_slice), self.end_datetime.datetime_format, self._timezone) + start_time = self._parser.parse(get_start_time(_slice), self.start_datetime.datetime_format) + end_time = self._parser.parse(get_end_time(_slice), self.end_datetime.datetime_format) _slice[self.stream_slice_field_step.eval(self.config)] = (end_time + datetime.timedelta(days=int(bool(i))) - start_time).days return date_range diff --git a/airbyte-integrations/connectors/source-braze/source_braze/manifest.yaml b/airbyte-integrations/connectors/source-braze/source_braze/manifest.yaml index ae93a91b3929..d3940a9e8a79 100644 --- a/airbyte-integrations/connectors/source-braze/source_braze/manifest.yaml +++ b/airbyte-integrations/connectors/source-braze/source_braze/manifest.yaml @@ -8,6 +8,11 @@ definitions: list_selector: extractor: field_path: ["{{ parameters['name'] }}"] + event_selector: + extractor: + class_name: source_braze.components.EventsRecordExtractor + field_path: + - events # ----- Requester ----- requester: @@ -35,6 +40,21 @@ definitions: request_parameters: last_edit.time[gt]: "{{ config['start_date'] }}" + event_retriever: + record_selector: + $ref: "#/definitions/event_selector" + paginator: + type: DefaultPaginator + pagination_strategy: + type: PageIncrement + page_size: 2 + page_token_option: + type: RequestOption + inject_into: "request_parameter" + field_name: "page" + requester: + $ref: "#/definitions/requester" + # ----- Stream bases ----- list_stream: retriever: @@ -66,12 +86,8 @@ definitions: primary_key: "id" path: "/segments/list" custom_events_stream: - $ref: "#/definitions/list_stream" - transformations: - - class_name: "source_braze.TransformToRecordComponent" - fields: - - path: ["event_name"] - value: "{{ record }}" + retriever: + $ref: "#/definitions/event_retriever" $parameters: name: "events" path: "/events/list" @@ -164,6 +180,11 @@ definitions: $ref: "#/definitions/datetime_14d_incremental_sync" retriever: $ref: "#/definitions/non_paginated_retriever" + requester: + $ref: "#/definitions/requester" + request_parameters: + include_variant_breakdown: "true" + include_step_breakdown: "true" record_selector: extractor: field_path: ["data", "stats"] @@ -271,11 +292,93 @@ definitions: name: "events_analytics" path: "/events/data_series" + # -- Details streams + segments_details_stream: + retriever: + record_selector: + extractor: + field_path: [] + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester" + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/segments_stream" + parent_key: "id" + partition_field: "segment_id" + request_option: + field_name: "segment_id" + inject_into: "request_parameter" + transformations: + - type: AddFields + fields: + - path: ["segment_id"] + value: "{{ stream_slice.segment_id }}" + $parameters: + name: "segments_details" + path: "/segments/details" + campaigns_details_stream: + retriever: + record_selector: + extractor: + field_path: [] + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester" + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/campaigns_stream" + parent_key: "id" + partition_field: "campaign_id" + request_option: + field_name: "campaign_id" + inject_into: "request_parameter" + transformations: + - type: AddFields + fields: + - path: ["campaign_id"] + value: "{{ stream_slice.campaign_id }}" + $parameters: + name: "campaigns_details" + path: "/campaigns/details" + canvases_details_stream: + retriever: + record_selector: + extractor: + field_path: [] + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester" + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/canvases_stream" + parent_key: "id" + partition_field: "canvas_id" + request_option: + field_name: "canvas_id" + inject_into: "request_parameter" + transformations: + - type: AddFields + fields: + - path: ["canvas_id"] + value: "{{ stream_slice.canvas_id }}" + $parameters: + name: "canvases_details" + path: "/canvas/details" + streams: - "#/definitions/campaigns_stream" - "#/definitions/campaigns_analytics_stream" + - "#/definitions/campaigns_details_stream" - "#/definitions/canvases_stream" - "#/definitions/canvases_analytics_stream" + - "#/definitions/canvases_details_stream" - "#/definitions/custom_events_stream" - "#/definitions/events_analytics_stream" - "#/definitions/kpi_daily_new_users_stream" @@ -285,6 +388,7 @@ streams: - "#/definitions/cards_analytics_stream" - "#/definitions/segments_stream" - "#/definitions/segments_analytics_stream" + - "#/definitions/segments_details_stream" check: stream_names: diff --git a/airbyte-integrations/connectors/source-braze/source_braze/schemas/TODO.md b/airbyte-integrations/connectors/source-braze/source_braze/schemas/TODO.md deleted file mode 100644 index 775144471ec8..000000000000 --- a/airbyte-integrations/connectors/source-braze/source_braze/schemas/TODO.md +++ /dev/null @@ -1,16 +0,0 @@ -# TODO: Define your stream schemas -Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). - -You can describe the schema of your streams using one `.json` file per stream. - -## Static schemas -From the `braze.yaml` configuration file, you read the `.json` files in the `schemas/` directory. You can refer to a schema in your configuration file using the `schema_loader` component's `file_path` field. For example: -``` -schema_loader: - type: JsonSchema - file_path: "./source_braze/schemas/customers.json" -``` -Every stream specified in the configuration file should have a corresponding `.json` schema file. - -Delete this file once you're done. Or don't. Up to you :) - diff --git a/airbyte-integrations/connectors/source-braze/source_braze/schemas/campaigns_details.json b/airbyte-integrations/connectors/source-braze/source_braze/schemas/campaigns_details.json new file mode 100644 index 000000000000..14e253e3b616 --- /dev/null +++ b/airbyte-integrations/connectors/source-braze/source_braze/schemas/campaigns_details.json @@ -0,0 +1,72 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "message": { + "type": "string" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "archived": { + "type": ["null", "boolean"] + }, + "draft": { + "type": ["null", "boolean"] + }, + "name": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "schedule_type": { + "type": ["null", "string"] + }, + "first_sent": { + "type": ["null", "string"], + "format": "date-time" + }, + "last_sent": { + "type": ["null", "string"], + "format": "date-time" + }, + "campaign_id": { + "type": ["null", "string"] + }, + "channels": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "conversion_behaviors": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} + } + }, + "messages": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} + }, + "enabled": { + "type": ["null", "boolean"] + }, + "tags": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-braze/source_braze/schemas/canvases_analytics.json b/airbyte-integrations/connectors/source-braze/source_braze/schemas/canvases_analytics.json index c834118b67f0..92dc2437a92d 100644 --- a/airbyte-integrations/connectors/source-braze/source_braze/schemas/canvases_analytics.json +++ b/airbyte-integrations/connectors/source-braze/source_braze/schemas/canvases_analytics.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "canvas_id": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-braze/source_braze/schemas/canvases_details.json b/airbyte-integrations/connectors/source-braze/source_braze/schemas/canvases_details.json new file mode 100644 index 000000000000..ac6a58ef5ec8 --- /dev/null +++ b/airbyte-integrations/connectors/source-braze/source_braze/schemas/canvases_details.json @@ -0,0 +1,51 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "created_at": { "type": ["null", "string"], "format": "date-time" }, + "updated_at": { "type": ["null", "string"], "format": "date-time" }, + "name": { "type": ["null", "string"] }, + "description": { "type": ["null", "string"] }, + "archived": { "type": ["null", "boolean"] }, + "draft": { "type": ["null", "boolean"] }, + "schedule_type": { "type": ["null", "string"] }, + "first_entry": { "type": ["null", "string"], "format": "date-time" }, + "last_entry": { "type": ["null", "string"], "format": "date-time" }, + "message": { "type": "string" }, + "canvas_id": { + "type": ["null", "string"] + }, + "channels": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "steps": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} + } + }, + "variants": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} + } + }, + "enabled": { + "type": ["null", "boolean"] + }, + "tags": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-braze/source_braze/schemas/segments_analytics.json b/airbyte-integrations/connectors/source-braze/source_braze/schemas/segments_analytics.json index 52e8b896f3c7..4d7924755d1a 100644 --- a/airbyte-integrations/connectors/source-braze/source_braze/schemas/segments_analytics.json +++ b/airbyte-integrations/connectors/source-braze/source_braze/schemas/segments_analytics.json @@ -9,7 +9,7 @@ "type": ["null", "string"] }, "size": { - "type": ["null", "integer"] + "type": ["null", "number"] } } } diff --git a/airbyte-integrations/connectors/source-braze/source_braze/schemas/segments_details.json b/airbyte-integrations/connectors/source-braze/source_braze/schemas/segments_details.json new file mode 100644 index 000000000000..654bb0cb2f79 --- /dev/null +++ b/airbyte-integrations/connectors/source-braze/source_braze/schemas/segments_details.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "message": { "type": "string" }, + "created_at": { "type": ["null", "string"], "format": "date-time" }, + "updated_at": { "type": ["null", "string"], "format": "date-time" }, + "name": { "type": ["null", "string"] }, + "description": { "type": ["null", "string"] }, + "text_description": { "type": ["null", "string"] }, + "segment_id": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-braze/unit_tests/test_datetime_incremental_sync.py b/airbyte-integrations/connectors/source-braze/unit_tests/test_datetime_incremental_sync.py index 2992b692013c..d478ac686adc 100644 --- a/airbyte-integrations/connectors/source-braze/unit_tests/test_datetime_incremental_sync.py +++ b/airbyte-integrations/connectors/source-braze/unit_tests/test_datetime_incremental_sync.py @@ -2,7 +2,6 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.requesters import RequestOption from airbyte_cdk.sources.declarative.requesters.request_option import RequestOptionType from source_braze import DatetimeIncrementalSyncComponent @@ -31,4 +30,4 @@ def test_datetime_slicer(): {"start_time": "2022-12-04", "end_time": "2022-12-06", "step": 3}, {"start_time": "2022-12-07", "end_time": "2022-12-08", "step": 2}, ] - assert slicer.stream_slices(SyncMode.incremental, stream_state={}) == expected_slices + assert slicer.stream_slices() == expected_slices diff --git a/airbyte-integrations/connectors/source-braze/unit_tests/test_transformations.py b/airbyte-integrations/connectors/source-braze/unit_tests/test_transformations.py index b8dee3c42bd9..2ed02ec26eaa 100644 --- a/airbyte-integrations/connectors/source-braze/unit_tests/test_transformations.py +++ b/airbyte-integrations/connectors/source-braze/unit_tests/test_transformations.py @@ -10,7 +10,7 @@ def test_string_to_dict_transformation(): """ Test that given string record transforms to dict with given name and value as a record itself. """ - added_field = AddedFieldDefinition(path=["append_key"], value="{{ record }}", parameters={}) + added_field = AddedFieldDefinition(value_type=str, path=["append_key"], value="{{ record }}", parameters={}) transformation = TransformToRecordComponent(fields=[added_field], parameters={}) record = transformation.transform(record="StringRecord", config={}, stream_state={}, stream_slice={}) expected_record = {"append_key": "StringRecord"} diff --git a/airbyte-integrations/connectors/source-breezometer/README.md b/airbyte-integrations/connectors/source-breezometer/README.md index 817bee6aaaef..7da049dfa8d0 100644 --- a/airbyte-integrations/connectors/source-breezometer/README.md +++ b/airbyte-integrations/connectors/source-breezometer/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-breezometer:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/breezometer) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_breezometer/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-breezometer:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-breezometer build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-breezometer:airbyteDocker +An image will be built with the tag `airbyte/source-breezometer:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-breezometer:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-breezometer:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-breezometer:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-breezometer:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-breezometer test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-breezometer:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-breezometer:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-breezometer test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/breezometer.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-breezometer/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-breezometer/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-breezometer/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-breezometer/build.gradle b/airbyte-integrations/connectors/source-breezometer/build.gradle deleted file mode 100644 index 97e8fbb9d5d5..000000000000 --- a/airbyte-integrations/connectors/source-breezometer/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_breezometer' -} diff --git a/airbyte-integrations/connectors/source-callrail/README.md b/airbyte-integrations/connectors/source-callrail/README.md index 0dddd71e268e..199429bab103 100644 --- a/airbyte-integrations/connectors/source-callrail/README.md +++ b/airbyte-integrations/connectors/source-callrail/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-callrail:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/callrail) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_callrail/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-callrail:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-callrail build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-callrail:airbyteDocker +An image will be built with the tag `airbyte/source-callrail:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-callrail:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-callrail:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-callrail:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-callrail:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-callrail test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-callrail:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-callrail:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-callrail test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/callrail.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-callrail/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-callrail/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-callrail/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-callrail/build.gradle b/airbyte-integrations/connectors/source-callrail/build.gradle deleted file mode 100644 index 682d3b09eeda..000000000000 --- a/airbyte-integrations/connectors/source-callrail/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_callrail' -} diff --git a/airbyte-integrations/connectors/source-captain-data/README.md b/airbyte-integrations/connectors/source-captain-data/README.md index 0d93d706ca0c..4ac93829acbd 100644 --- a/airbyte-integrations/connectors/source-captain-data/README.md +++ b/airbyte-integrations/connectors/source-captain-data/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-captain-data:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/captain-data) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_captain_data/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-captain-data:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-captain-data build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-captain-data:airbyteDocker +An image will be built with the tag `airbyte/source-captain-data:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-captain-data:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-captain-data:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-captain-data:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-captain-data:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-captain-data test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-captain-data:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-captain-data:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-captain-data test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/captain-data.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-captain-data/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-captain-data/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-captain-data/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-captain-data/build.gradle b/airbyte-integrations/connectors/source-captain-data/build.gradle deleted file mode 100644 index d92bf8ed87a3..000000000000 --- a/airbyte-integrations/connectors/source-captain-data/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_captain_data' -} diff --git a/airbyte-integrations/connectors/source-cart/Dockerfile b/airbyte-integrations/connectors/source-cart/Dockerfile index 4323b70cdd86..d0175fec0c1a 100644 --- a/airbyte-integrations/connectors/source-cart/Dockerfile +++ b/airbyte-integrations/connectors/source-cart/Dockerfile @@ -21,5 +21,5 @@ COPY source_cart ./source_cart ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.1 +LABEL io.airbyte.version=0.3.1 LABEL io.airbyte.name=airbyte/source-cart diff --git a/airbyte-integrations/connectors/source-cart/README.md b/airbyte-integrations/connectors/source-cart/README.md index ab584dbf5f0c..90838c261bda 100644 --- a/airbyte-integrations/connectors/source-cart/README.md +++ b/airbyte-integrations/connectors/source-cart/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-cart:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/cart) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_cart/spec.json` file. @@ -56,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-cart:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-cart build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-cart:airbyteDocker +An image will be built with the tag `airbyte/source-cart:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-cart:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -77,44 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-cart:dev check --confi docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-cart:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-cart:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-cart test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-cart:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-cart:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -124,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-cart test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/cart.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-cart/acceptance-test-config.yml b/airbyte-integrations/connectors/source-cart/acceptance-test-config.yml index a730ef928a91..52803b05481a 100644 --- a/airbyte-integrations/connectors/source-cart/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-cart/acceptance-test-config.yml @@ -1,39 +1,51 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-cart:dev -tests: +test_strictness_level: low +acceptance_tests: spec: - - spec_path: "source_cart/spec.json" - backward_compatibility_tests_config: - disable_for_version: "0.1.6" + tests: + - spec_path: "source_cart/spec.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.6" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" - timeout_seconds: 180 + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + timeout_seconds: 180 discovery: - - config_path: "secrets/config.json" - backward_compatibility_tests_config: - disable_for_version: "0.1.6" + tests: + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.6" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 1800 + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 1800 + empty_streams: + - name: "order_payments" + bypass_reason: "no data" + - name: "products" + bypass_reason: "no data" incremental: - - config_path: "secrets/config_central_api_router.json" - configured_catalog_path: "integration_tests/configured_catalog_wo_order_statuses.json" - future_state_path: "integration_tests/abnormal_state.json" - timeout_seconds: 1800 - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" - timeout_seconds: 1800 + tests: + # - config_path: "secrets/config_central_api_router.json" + # configured_catalog_path: "integration_tests/configured_catalog_wo_order_statuses.json" + # future_state_path: "integration_tests/abnormal_state.json" + # timeout_seconds: 1800 + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + timeout_seconds: 1800 full_refresh: - - config_path: "secrets/config_central_api_router.json" - configured_catalog_path: "integration_tests/configured_catalog_wo_order_statuses.json" - timeout_seconds: 1800 - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 1800 - + tests: + - config_path: "secrets/config_central_api_router.json" + configured_catalog_path: "integration_tests/configured_catalog_wo_order_statuses.json" + timeout_seconds: 1800 + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 1800 diff --git a/airbyte-integrations/connectors/source-cart/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-cart/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-cart/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-cart/build.gradle b/airbyte-integrations/connectors/source-cart/build.gradle deleted file mode 100644 index 48e0a1cfa408..000000000000 --- a/airbyte-integrations/connectors/source-cart/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_cart' -} diff --git a/airbyte-integrations/connectors/source-cart/metadata.yaml b/airbyte-integrations/connectors/source-cart/metadata.yaml index 8f75a5db4ceb..d73ebd080df9 100644 --- a/airbyte-integrations/connectors/source-cart/metadata.yaml +++ b/airbyte-integrations/connectors/source-cart/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: bb1a6d31-6879-4819-a2bd-3eed299ea8e2 - dockerImageTag: 0.2.1 + dockerImageTag: 0.3.1 dockerRepository: airbyte/source-cart githubIssueLabel: source-cart icon: cart.svg @@ -10,7 +10,7 @@ data: name: Cart.com registries: cloud: - enabled: false + enabled: true oss: enabled: true releaseStage: alpha diff --git a/airbyte-integrations/connectors/source-cart/setup.py b/airbyte-integrations/connectors/source-cart/setup.py index d23fa4f8daf4..69df33c757d3 100644 --- a/airbyte-integrations/connectors/source-cart/setup.py +++ b/airbyte-integrations/connectors/source-cart/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ diff --git a/airbyte-integrations/connectors/source-cart/source_cart/schemas/addresses.json b/airbyte-integrations/connectors/source-cart/source_cart/schemas/addresses.json index 98fe097f1779..e7d377656a16 100644 --- a/airbyte-integrations/connectors/source-cart/source_cart/schemas/addresses.json +++ b/airbyte-integrations/connectors/source-cart/source_cart/schemas/addresses.json @@ -1,5 +1,7 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer" @@ -10,6 +12,9 @@ "address_line_1": { "type": ["string", "null"] }, + "address_type": { + "type": ["string", "null"] + }, "address_line_2": { "type": ["string", "null"] }, diff --git a/airbyte-integrations/connectors/source-cart/source_cart/schemas/customers_cart.json b/airbyte-integrations/connectors/source-cart/source_cart/schemas/customers_cart.json index 8520252b4485..23c4e341dce3 100644 --- a/airbyte-integrations/connectors/source-cart/source_cart/schemas/customers_cart.json +++ b/airbyte-integrations/connectors/source-cart/source_cart/schemas/customers_cart.json @@ -1,5 +1,7 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer" @@ -7,6 +9,12 @@ "customer_number": { "type": ["string", "null"] }, + "credit_limit": { + "type": ["string", "null"] + }, + "payment_net_term": { + "type": ["string", "null"] + }, "last_name": { "type": ["string", "null"] }, @@ -38,7 +46,13 @@ "type": ["integer", "null"] }, "is_no_tax_customer": { - "type": "boolean" + "type": ["boolean", "null"] + }, + "is_inactive": { + "type": ["boolean", "null"] + }, + "lock_default_address": { + "type": ["boolean", "null"] }, "comments": { "type": ["string", "null"] diff --git a/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_items.json b/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_items.json index e803c78c5ac1..b7223e79fd55 100644 --- a/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_items.json +++ b/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_items.json @@ -1,5 +1,7 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer" @@ -81,6 +83,69 @@ }, "warehouse_id": { "type": ["integer", "null"] + }, + "configuration": { + "type": ["string", "null"] + }, + "description": { + "type": ["string", "null"] + }, + "discount_amount": { + "type": ["number", "null"] + }, + "discount_percentage": { + "type": ["number", "null"] + }, + "fitment": { + "type": ["string", "null"] + }, + "is_non_shipping_item": { + "type": ["boolean", "null"] + }, + "item_number_full": { + "type": ["string", "null"] + }, + "order_shipping_address_id": { + "type": ["string", "null"] + }, + "personalizations": { + "type": ["array", "null"] + }, + "selected_shipping_method": { + "type": ["string", "null"] + }, + "selected_shipping_method_id": { + "type": ["string", "null"] + }, + "selected_shipping_provider_service": { + "type": ["string", "null"] + }, + "shipping_total": { + "type": ["string", "null"] + }, + "status": { + "type": ["string", "null"] + }, + "tax": { + "type": ["number", "null"] + }, + "tax_code": { + "type": ["string", "null"] + }, + "variant_inventory_id": { + "type": ["string", "null"] + }, + "shipping_classification_code": { + "type": ["string", "null"] + }, + "variants": { + "type": ["array", "null"] + }, + "vendor_store_id": { + "type": ["integer", "null"] + }, + "weight_unit": { + "type": ["string", "null"] } } } diff --git a/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_payments.json b/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_payments.json index f4dee9743008..ab2f2c844d71 100644 --- a/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_payments.json +++ b/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_payments.json @@ -1,5 +1,7 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer" diff --git a/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_statuses.json b/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_statuses.json index eb7182c2f368..b77422eb2f54 100644 --- a/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_statuses.json +++ b/airbyte-integrations/connectors/source-cart/source_cart/schemas/order_statuses.json @@ -1,5 +1,7 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer" }, "name": { "type": ["null", "string"] }, @@ -13,6 +15,7 @@ "created_at": { "type": ["null", "string"] }, "is_fully_refunded": { "type": ["null", "boolean"] }, "is_partially_refunded": { "type": ["null", "boolean"] }, - "is_quote_status": { "type": ["null", "boolean"] } + "is_quote_status": { "type": ["null", "boolean"] }, + "is_partially_shipped": { "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-cart/source_cart/schemas/orders.json b/airbyte-integrations/connectors/source-cart/source_cart/schemas/orders.json index e5e7091efda4..f1ebdb8b5b9d 100644 --- a/airbyte-integrations/connectors/source-cart/source_cart/schemas/orders.json +++ b/airbyte-integrations/connectors/source-cart/source_cart/schemas/orders.json @@ -1,5 +1,7 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer" @@ -7,6 +9,23 @@ "customer_id": { "type": ["integer", "null"] }, + "delivery_tax": { + "type": ["string", "null"] + }, + "entered_by_type": { + "type": ["string", "null"] + }, + "shipping_selections": { + "type": ["array", "null"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} + } + }, + "sales_agent_user_id": { + "type": ["string", "null"] + }, "customer_type_id": { "type": ["integer", "null"] }, diff --git a/airbyte-integrations/connectors/source-cart/source_cart/schemas/products.json b/airbyte-integrations/connectors/source-cart/source_cart/schemas/products.json index ed1473eb08a6..5d0ac08fa31a 100644 --- a/airbyte-integrations/connectors/source-cart/source_cart/schemas/products.json +++ b/airbyte-integrations/connectors/source-cart/source_cart/schemas/products.json @@ -1,5 +1,7 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer" diff --git a/airbyte-integrations/connectors/source-chargebee/Dockerfile b/airbyte-integrations/connectors/source-chargebee/Dockerfile deleted file mode 100644 index 3184e75a167b..000000000000 --- a/airbyte-integrations/connectors/source-chargebee/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_chargebee ./source_chargebee - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.2.4 -LABEL io.airbyte.name=airbyte/source-chargebee diff --git a/airbyte-integrations/connectors/source-chargebee/README.md b/airbyte-integrations/connectors/source-chargebee/README.md index 9a2752b810bc..4819ef2b1506 100644 --- a/airbyte-integrations/connectors/source-chargebee/README.md +++ b/airbyte-integrations/connectors/source-chargebee/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-chargebee:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/chargebee) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_chargebee/spec.yaml` file. @@ -24,19 +16,70 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-chargebee:dev + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-chargebee build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-chargebee:dev`. -You can also build the connector image via Gradle: +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-chargebee:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-chargebee:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-chargebee:dev . +# Running the spec command against your patched connector +docker run airbyte/source-chargebee:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -45,29 +88,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-chargebee:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-chargebee:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-chargebee:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-chargebee test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -docker build . --no-cache -t airbyte/source-chargebee:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-chargebee:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-chargebee:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -76,8 +107,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-chargebee test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/chargebee.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-chargebee/acceptance-test-config.yml b/airbyte-integrations/connectors/source-chargebee/acceptance-test-config.yml index 27941b91fb62..4a52973f0937 100644 --- a/airbyte-integrations/connectors/source-chargebee/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-chargebee/acceptance-test-config.yml @@ -5,68 +5,63 @@ test_strictness_level: high acceptance_tests: spec: tests: - - spec_path: "source_chargebee/spec.yaml" + - spec_path: "source_chargebee/spec.yaml" connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: tests: - - config_path: "secrets/config.json" - backward_compatibility_tests_config: - disable_for_version: "0.2.1" # New streams were added; fixed fields type + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.2.1" # New streams were added; fixed fields type basic_read: tests: - - config_path: "secrets/config.json" - timeout_seconds: 1200 - empty_streams: - - name: "addon" - bypass_reason: "Not permitted for this site" - - name: "plan" - bypass_reason: "Not permitted for this site" - - name: "virtual_bank_account" - bypass_reason: "Cannot populate with test data" - - name: "subscription" - bypass_reason: "Unstable data. Field current_term_start updated daily" - - name: "customer" - bypass_reason: "Unstable data. Depends on subscription" - - name: "invoice" - bypass_reason: "Unstable data. Depends on subscription" - - name: "credit_note" - bypass_reason: "Unstable data. Depends on subscription" - - name: "event" - bypass_reason: "Unstable data. Depends on subscription" - - name: "unbilled_charge" - bypass_reason: "Empty stream. Unstable data" - - name: "hosted_page" - bypass_reason: "Empty stream. Unstable data" - expect_records: - path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: no - extra_records: yes - fail_on_extra_columns: false + - config_path: "secrets/config.json" + timeout_seconds: 1200 + empty_streams: + - name: "addon" + bypass_reason: "Not available for Product Catalog 2.0 sites." + - name: "plan" + bypass_reason: "Not available for Product Catalog 2.0 sites." + - name: "virtual_bank_account" + bypass_reason: "Cannot populate with test data" + - name: "unbilled_charge" + bypass_reason: "Empty stream. Unstable data" + - name: "hosted_page" + bypass_reason: "Empty stream. Unstable data" + - name: "event" + bypass_reason: "Unstable data. Updated daily." + - name: "site_migration_detail" + bypass_reason: "Cannnot populate with test data." + ignored_fields: + subscription: + - name: "current_term_start" + bypass_reason: "Field updated daily." + expect_records: + path: "integration_tests/expected_records.jsonl" + extra_fields: no + exact_order: no + extra_records: yes + fail_on_extra_columns: false incremental: - # tests: - # - config_path: "secrets/config.json" - # timeout_seconds: 2400 - # configured_catalog_path: "integration_tests/configured_catalog.json" - # future_state: - # future_state_path: "integration_tests/future_state.json" - # missing_streams: - # - name: attached_item - # bypass_reason: "This stream is Full-Refresh only" - # - name: contact - # bypass_reason: "This stream is Full-Refresh only" - # - name: quote_line_group - # bypass_reason: "This stream is Full-Refresh only" - bypass_reason: > - "Incrremental tests are disabled until CAT works with cursor data-types directly, - relatated slack thread: https://airbyte-globallogic.slack.com/archives/C02U9R3AF37/p1690810513681859" + tests: + - config_path: "secrets/config.json" + timeout_seconds: 2400 + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/future_state.json" + missing_streams: + - name: attached_item + bypass_reason: "This stream is Full-Refresh only" + - name: contact + bypass_reason: "This stream is Full-Refresh only" + - name: quote_line_group + bypass_reason: "This stream is Full-Refresh only" full_refresh: tests: - - config_path: "secrets/config.json" - timeout_seconds: 2400 - configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config.json" + timeout_seconds: 2400 + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-chargebee/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-chargebee/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-chargebee/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-chargebee/build.gradle b/airbyte-integrations/connectors/source-chargebee/build.gradle deleted file mode 100644 index cb9358f89f6f..000000000000 --- a/airbyte-integrations/connectors/source-chargebee/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source-chargebee' -} diff --git a/airbyte-integrations/connectors/source-chargebee/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-chargebee/integration_tests/configured_catalog.json index 950c314639f2..b2ae2de6b192 100644 --- a/airbyte-integrations/connectors/source-chargebee/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-chargebee/integration_tests/configured_catalog.json @@ -243,6 +243,58 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "append" + }, + { + "stream": { + "name": "site_migration_detail", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["migrated_at"], + "source_defined_primary_key": [["entity_id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["migrated_at"] + }, + { + "stream": { + "name": "comment", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["created_at"] + }, + { + "stream": { + "name": "item_family", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "differential_price", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] } ] } diff --git a/airbyte-integrations/connectors/source-chargebee/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-chargebee/integration_tests/expected_records.jsonl index fd515d457d6c..9007c7adbdf3 100644 --- a/airbyte-integrations/connectors/source-chargebee/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-chargebee/integration_tests/expected_records.jsonl @@ -1,9 +1,9 @@ {"stream":"contact","data":{"id":"0000002","first_name":"User2","last_name":"Sample","email":"user2.sample.airbyte@gmail.com","phone":"+13335556789","label":"Tag2","enabled":true,"send_account_email":true,"send_billing_email":true,"object":"contact","custom_fields":[]},"emitted_at":1676569185767} {"stream":"contact","data":{"id":"Test 1","first_name":"Sample Name 1","last_name":"Sample Lastname 1","email":"name1@example.com","enabled":true,"send_account_email":false,"send_billing_email":false,"object":"contact","custom_fields":[]},"emitted_at":1676569186037} {"stream":"contact","data":{"id":"Test Contact 2","first_name":"Sample Name Two","last_name":"Sample Lastname 2","email":"name2@example.com","phone":"+13888433888","enabled":true,"send_account_email":false,"send_billing_email":false,"object":"contact","custom_fields":[]},"emitted_at":1676569186337} -{"stream":"order","data":{"id":"1","document_number":"lol1","invoice_id":"24","subscription_id":"6olOsTTHieWUY9","customer_id":"cbdemo_tyler","status":"queued","payment_status":"paid","order_type":"system_generated","price_type":"tax_exclusive","order_date":1674036524,"shipping_date":1674036524,"created_by":"Auto generated by system","tax":0,"amount_paid":1000,"amount_adjusted":0,"refundable_credits_issued":0,"refundable_credits":1000,"rounding_adjustement":0,"paid_on":1674036524,"exchange_rate":1,"created_at":1674036525,"updated_at":1674036525,"is_resent":false,"resource_version":1674036525755,"deleted":false,"object":"order","discount":0,"sub_total":1000,"order_line_items":[{"id":"o_li169lB6TTHiez02Fb4","invoice_id":"24","invoice_line_item_id":"li_6olOsTTHieX6YB","unit_price":1000,"amount":1000,"fulfillment_quantity":1,"fulfillment_amount":1000,"tax_amount":0,"amount_paid":1000,"amount_adjusted":0,"refundable_credits_issued":0,"refundable_credits":1000,"is_shippable":true,"status":"queued","object":"order_line_item","entity_id":"Test-Plan-1-USD-Daily","discount_amount":0,"item_level_discount_amount":0,"description":"Test Plan 1","entity_type":"plan_item_price"}],"total":1000,"currency_code":"USD","base_currency_code":"USD","is_gifted":false,"billing_address":{"first_name":"Tyler","last_name":"Durden","company":"Iselectrics","validation_status":"not_validated","object":"billing_address"},"linked_credit_notes":[],"resent_orders":[],"custom_fields":[]},"emitted_at":1677235847508} -{"stream":"order","data":{"id":"2","document_number":"lol2","invoice_id":"25","subscription_id":"AzZTZgTTHixMHV3","customer_id":"cbdemo_richard","status":"queued","payment_status":"paid","order_type":"system_generated","price_type":"tax_exclusive","order_date":1674036596,"shipping_date":1674036596,"created_by":"Auto generated by system","tax":0,"amount_paid":1000,"amount_adjusted":0,"refundable_credits_issued":0,"refundable_credits":1000,"rounding_adjustement":0,"paid_on":1674036596,"exchange_rate":1,"created_at":1674036599,"updated_at":1674036684,"is_resent":false,"resource_version":1674036684213,"deleted":false,"object":"order","discount":0,"sub_total":1000,"order_line_items":[{"id":"o_li16CQyCTTHiy9912Tu","invoice_id":"25","invoice_line_item_id":"li_AzZTZgTTHixMhV5","unit_price":1000,"amount":1000,"fulfillment_quantity":1,"fulfillment_amount":1000,"tax_amount":0,"amount_paid":1000,"amount_adjusted":0,"refundable_credits_issued":0,"refundable_credits":1000,"is_shippable":true,"status":"queued","object":"order_line_item","entity_id":"Test-Plan-1-USD-Daily","discount_amount":0,"item_level_discount_amount":0,"description":"Test Plan 1","entity_type":"plan_item_price"}],"total":1000,"currency_code":"USD","base_currency_code":"USD","is_gifted":false,"shipping_address":{"first_name":"Sample Name 1","last_name":"Sample Lastname 1","email":"name1@example.com","company":"Semiconductors","phone":"+1 382 846 3883","line1":"Ms Ninette Franck","line2":"4381","city":"San Francisco","state_code":"CA","state":"California","country":"US","zip":"94114","validation_status":"not_validated","object":"shipping_address"},"billing_address":{"first_name":"Richard","last_name":"Hendricks","company":"Zencorporation","validation_status":"not_validated","object":"billing_address"},"linked_credit_notes":[],"resent_orders":[],"custom_fields":[]},"emitted_at":1677235847512} -{"stream":"order","data":{"id":"3","document_number":"lol3","invoice_id":"26","subscription_id":"AzZTZgTTHmX8Gc1","customer_id":"cbdemo_simon","status":"queued","payment_status":"paid","order_type":"system_generated","price_type":"tax_exclusive","order_date":1674037448,"shipping_date":1674037448,"created_by":"Auto generated by system","tax":0,"amount_paid":700,"amount_adjusted":0,"refundable_credits_issued":0,"refundable_credits":700,"rounding_adjustement":0,"paid_on":1674037448,"exchange_rate":1,"created_at":1674037452,"updated_at":1674037452,"is_resent":false,"resource_version":1674037452271,"deleted":false,"object":"order","discount":300,"sub_total":700,"order_line_items":[{"id":"o_liAzZZMnTTHmY0s1O7g","invoice_id":"26","invoice_line_item_id":"li_AzZTZgTTHmX93c3","unit_price":1000,"amount":1000,"fulfillment_quantity":1,"fulfillment_amount":700,"tax_amount":0,"amount_paid":700,"amount_adjusted":0,"refundable_credits_issued":0,"refundable_credits":700,"is_shippable":true,"status":"queued","object":"order_line_item","entity_id":"Test-Plan-1-USD-Daily","discount_amount":300,"item_level_discount_amount":300,"description":"Test Plan 1","entity_type":"plan_item_price"}],"line_item_discounts":[{"object":"line_item_discount","line_item_id":"li_AzZTZgTTHmX93c3","discount_type":"item_level_coupon","discount_amount":300,"coupon_id":"cbdemo_launchoffer","entity_id":"cbdemo_launchoffer"}],"total":700,"currency_code":"USD","base_currency_code":"USD","is_gifted":false,"billing_address":{"first_name":"Simon","last_name":"Masrani","company":"Openlane Ltd","validation_status":"not_validated","object":"billing_address"},"linked_credit_notes":[],"resent_orders":[],"custom_fields":[]},"emitted_at":1677235847517} +{"stream": "order", "data": {"id": "1", "document_number": "lol1", "invoice_id": "24", "subscription_id": "6olOsTTHieWUY9", "customer_id": "cbdemo_tyler", "status": "queued", "payment_status": "paid", "order_type": "system_generated", "price_type": "tax_exclusive", "order_date": 1674036524, "shipping_date": 1674036524, "created_by": "Auto generated by system", "tax": 0, "amount_paid": 1000, "amount_adjusted": 0, "refundable_credits_issued": 0, "refundable_credits": 1000, "rounding_adjustement": 0, "paid_on": 1674036524, "exchange_rate": 1.0, "created_at": 1674036525, "updated_at": 1674036525, "is_resent": false, "resource_version": 1674036525755, "deleted": false, "object": "order", "discount": 0, "sub_total": 1000, "order_line_items": [{"id": "o_li169lB6TTHiez02Fb4", "invoice_id": "24", "invoice_line_item_id": "li_6olOsTTHieX6YB", "unit_price": 1000, "amount": 1000, "fulfillment_quantity": 1, "fulfillment_amount": 1000, "tax_amount": 0, "amount_paid": 1000, "amount_adjusted": 0, "refundable_credits_issued": 0, "refundable_credits": 1000, "is_shippable": true, "status": "queued", "object": "order_line_item", "entity_id": "Test-Plan-1-USD-Daily", "discount_amount": 0, "item_level_discount_amount": 0, "description": "Test Plan 1", "entity_type": "plan_item_price"}], "total": 1000, "currency_code": "USD", "base_currency_code": "USD", "is_gifted": false, "billing_address": {"first_name": "Tyler", "last_name": "Durden", "company": "Iselectrics", "validation_status": "not_validated", "object": "billing_address"}, "linked_credit_notes": [], "resent_orders": [], "custom_fields": []}, "emitted_at": 1703026216053} +{"stream": "order", "data": {"id": "2", "document_number": "lol2", "invoice_id": "25", "subscription_id": "AzZTZgTTHixMHV3", "customer_id": "cbdemo_richard", "status": "queued", "payment_status": "paid", "order_type": "system_generated", "price_type": "tax_exclusive", "order_date": 1674036596, "shipping_date": 1674036596, "created_by": "Auto generated by system", "tax": 0, "amount_paid": 1000, "amount_adjusted": 0, "refundable_credits_issued": 0, "refundable_credits": 1000, "rounding_adjustement": 0, "paid_on": 1674036596, "exchange_rate": 1.0, "created_at": 1674036599, "updated_at": 1674036684, "is_resent": false, "resource_version": 1674036684213, "deleted": false, "object": "order", "discount": 0, "sub_total": 1000, "order_line_items": [{"id": "o_li16CQyCTTHiy9912Tu", "invoice_id": "25", "invoice_line_item_id": "li_AzZTZgTTHixMhV5", "unit_price": 1000, "amount": 1000, "fulfillment_quantity": 1, "fulfillment_amount": 1000, "tax_amount": 0, "amount_paid": 1000, "amount_adjusted": 0, "refundable_credits_issued": 0, "refundable_credits": 1000, "is_shippable": true, "status": "queued", "object": "order_line_item", "entity_id": "Test-Plan-1-USD-Daily", "discount_amount": 0, "item_level_discount_amount": 0, "description": "Test Plan 1", "entity_type": "plan_item_price"}], "total": 1000, "currency_code": "USD", "base_currency_code": "USD", "is_gifted": false, "shipping_address": {"first_name": "Sample Name 1", "last_name": "Sample Lastname 1", "email": "name1@example.com", "company": "Semiconductors", "phone": "+1 382 846 3883", "line1": "Ms Ninette Franck", "line2": "4381", "city": "San Francisco", "state_code": "CA", "state": "California", "country": "US", "zip": "94114", "validation_status": "not_validated", "object": "shipping_address"}, "billing_address": {"first_name": "Richard", "last_name": "Hendricks", "company": "Zencorporation", "validation_status": "not_validated", "object": "billing_address"}, "linked_credit_notes": [], "resent_orders": [], "custom_fields": []}, "emitted_at": 1703026216060} +{"stream": "order", "data": {"id": "3", "document_number": "lol3", "invoice_id": "26", "subscription_id": "AzZTZgTTHmX8Gc1", "customer_id": "cbdemo_simon", "status": "queued", "payment_status": "paid", "order_type": "system_generated", "price_type": "tax_exclusive", "order_date": 1674037448, "shipping_date": 1674037448, "created_by": "Auto generated by system", "tax": 0, "amount_paid": 700, "amount_adjusted": 0, "refundable_credits_issued": 0, "refundable_credits": 700, "rounding_adjustement": 0, "paid_on": 1674037448, "exchange_rate": 1.0, "created_at": 1674037452, "updated_at": 1674037452, "is_resent": false, "resource_version": 1674037452271, "deleted": false, "object": "order", "discount": 300, "sub_total": 700, "order_line_items": [{"id": "o_liAzZZMnTTHmY0s1O7g", "invoice_id": "26", "invoice_line_item_id": "li_AzZTZgTTHmX93c3", "unit_price": 1000, "amount": 1000, "fulfillment_quantity": 1, "fulfillment_amount": 700, "tax_amount": 0, "amount_paid": 700, "amount_adjusted": 0, "refundable_credits_issued": 0, "refundable_credits": 700, "is_shippable": true, "status": "queued", "object": "order_line_item", "entity_id": "Test-Plan-1-USD-Daily", "discount_amount": 300, "item_level_discount_amount": 300, "description": "Test Plan 1", "entity_type": "plan_item_price"}], "line_item_discounts": [{"object": "line_item_discount", "line_item_id": "li_AzZTZgTTHmX93c3", "discount_type": "item_level_coupon", "discount_amount": 300, "coupon_id": "cbdemo_launchoffer", "entity_id": "cbdemo_launchoffer"}], "total": 700, "currency_code": "USD", "base_currency_code": "USD", "is_gifted": false, "billing_address": {"first_name": "Simon", "last_name": "Masrani", "company": "Openlane Ltd", "validation_status": "not_validated", "object": "billing_address"}, "linked_credit_notes": [], "resent_orders": [], "custom_fields": []}, "emitted_at": 1703026216066} {"stream": "item", "data": {"id": "cbdemo_advanced", "name": "Advanced", "external_name": "Advanced", "description": "Uncover hidden insights and carry out deeper analytics for your enterprise with this advanced plan.", "status": "active", "resource_version": 1674035640445, "updated_at": 1674035640, "item_family_id": "cbdemo_pf_analytics", "type": "plan", "is_shippable": true, "is_giftable": false, "enabled_for_checkout": true, "enabled_in_portal": true, "item_applicability": "all", "metered": false, "channel": "web", "metadata": {}, "object": "item", "custom_fields": []}, "emitted_at": 1678971136879} {"stream": "item", "data": {"id": "cbdemo_basic", "name": "Basic", "external_name": "Basic", "description": "Starter plan for all your basic reporting requirements.", "status": "active", "resource_version": 1674035673162, "updated_at": 1674035673, "item_family_id": "cbdemo_pf_analytics", "type": "plan", "is_shippable": true, "is_giftable": false, "enabled_for_checkout": true, "enabled_in_portal": true, "item_applicability": "all", "metered": false, "channel": "web", "metadata": {}, "object": "item", "custom_fields": []}, "emitted_at": 1678971136891} {"stream": "item", "data": {"id": "cbdemo_intermediary", "name": "Intermediary", "external_name": "Intermediary", "description": "Smart plan with the right mix of basic and slightly advanced reporting tools.", "status": "active", "resource_version": 1674035686971, "updated_at": 1674035686, "item_family_id": "cbdemo_pf_analytics", "type": "plan", "is_shippable": true, "is_giftable": false, "enabled_for_checkout": true, "enabled_in_portal": true, "item_applicability": "all", "metered": false, "channel": "web", "metadata": {}, "object": "item", "custom_fields": []}, "emitted_at": 1678971136900} @@ -31,3 +31,23 @@ {"stream":"quote_line_group","data":{"version":1,"id":"qlg_AzZTZgTTIx14p2aF","sub_total":1000,"total":0,"credits_applied":0,"amount_paid":0,"amount_due":0,"charge_event":"subscription_renewal","billing_cycle_number":1,"object":"quote_line_group","line_items":[{"id":"AzZTZgTTIx14r2aG","date_from":1674054723,"date_to":1674141123,"unit_amount":1000,"quantity":1,"amount":1000,"pricing_model":"flat_fee","is_taxed":false,"tax_amount":0,"object":"line_item","customer_id":"Test-Custome-1","description":"Test Plan 1","entity_type":"plan_item_price","entity_id":"Test-Plan-1-USD-Daily","entity_description":"Test","metered": false,"discount_amount":1000,"item_level_discount_amount":0}],"discounts":[{"object":"discount","entity_type":"promotional_credits","description":"Promotional Credits","amount":1000}],"line_item_discounts":[{"object":"line_item_discount","line_item_id":"AzZTZgTTIx14r2aG","discount_type":"promotional_credits","discount_amount":1000}],"taxes":[],"line_item_taxes":[],"custom_fields":[]},"emitted_at":1676569251063} {"stream":"quote_line_group","data":{"version":1,"id":"qlg_AzZTZgTTIxDoL2aq","sub_total":50000,"total":0,"credits_applied":0,"amount_paid":0,"amount_due":0,"charge_event":"immediate","object":"quote_line_group","line_items":[{"id":"AzZTZgTTIxDoN2ar","date_from":1674054772,"date_to":1674227572,"unit_amount":50000,"quantity":1,"amount":50000,"pricing_model":"flat_fee","is_taxed":false,"tax_amount":0,"object":"line_item","customer_id":"cbdemo_richard","description":"Implementation Charge","entity_type":"charge_item_price","entity_id":"cbdemo_implementation-charge-USD","metered": false,"discount_amount":50000,"item_level_discount_amount":0}],"discounts":[{"object":"discount","entity_type":"promotional_credits","description":"Promotional Credits","amount":50000}],"line_item_discounts":[{"object":"line_item_discount","line_item_id":"AzZTZgTTIxDoN2ar","discount_type":"promotional_credits","discount_amount":50000}],"taxes":[],"line_item_taxes":[],"custom_fields":[]},"emitted_at":1676569251257} {"stream":"quote_line_group","data":{"version":1,"id":"qlg_AzZTZgTTIxQzR2bG","sub_total":50000,"total":0,"credits_applied":0,"amount_paid":0,"amount_due":0,"charge_event":"immediate","object":"quote_line_group","line_items":[{"id":"AzZTZgTTIxQzS2bH","date_from":1674054823,"date_to":1674227623,"unit_amount":50000,"quantity":1,"amount":50000,"pricing_model":"flat_fee","is_taxed":false,"tax_amount":0,"object":"line_item","customer_id":"cbdemo_tyler","description":"Setup Charge","entity_type":"charge_item_price","entity_id":"cbdemo_setup-charge-USD","metered": false,"discount_amount":50000,"item_level_discount_amount":0}],"discounts":[{"object":"discount","entity_type":"promotional_credits","description":"Promotional Credits","amount":50000}],"line_item_discounts":[{"object":"line_item_discount","line_item_id":"AzZTZgTTIxQzS2bH","discount_type":"promotional_credits","discount_amount":50000}],"taxes":[],"line_item_taxes":[],"custom_fields":[]},"emitted_at":1676569251413} +{"stream": "subscription", "data": {"id": "cbdemo_cancelled_sub", "billing_period": 1, "billing_period_unit": "month", "auto_collection": "on", "customer_id": "cbdemo_carol", "status": "cancelled", "created_at": 1627671478, "started_at": 1624325278, "cancelled_at": 1626917278, "created_from_ip": "176.37.67.33", "updated_at": 1678806943, "has_scheduled_changes": false, "resource_version": 1678806943288, "deleted": false, "object": "subscription", "currency_code": "USD", "subscription_items": [{"item_price_id": "cbdemo_basic-USD-monthly", "item_type": "plan", "quantity": 1, "unit_price": 40000, "amount": 40000, "free_quantity": 0, "object": "subscription_item"}], "due_invoices_count": 0, "mrr": 0, "has_scheduled_advance_invoices": false, "auto_close_invoices": true, "channel": "web", "custom_fields": []}, "emitted_at": 1703096645549} +{"stream": "subscription", "data": {"id": "cbdemo_non_renewing_sub", "billing_period": 1, "billing_period_unit": "month", "auto_collection": "on", "customer_id": "cbdemo_tyler", "status": "cancelled", "current_term_start": 1626053278, "current_term_end": 1628731678, "created_at": 1627671479, "started_at": 1623461278, "activated_at": 1623461278, "cancelled_at": 1628731678, "created_from_ip": "176.37.67.33", "updated_at": 1678806943, "has_scheduled_changes": false, "cancel_schedule_created_at": 1627671479, "resource_version": 1678806943307, "deleted": false, "object": "subscription", "currency_code": "USD", "subscription_items": [{"item_price_id": "cbdemo_intermediary-USD-monthly", "item_type": "plan", "quantity": 1, "unit_price": 60000, "amount": 60000, "free_quantity": 0, "object": "subscription_item"}, {"item_price_id": "cbdemo_additional-analytics-USD-monthly", "item_type": "addon", "quantity": 1, "unit_price": 20000, "amount": 20000, "object": "subscription_item"}], "charged_items": [{"item_price_id": "cbdemo_setup-charge-USD", "last_charged_at": 1600153200, "object": "charged_item"}], "due_invoices_count": 0, "mrr": 0, "has_scheduled_advance_invoices": false, "auto_close_invoices": true, "channel": "web", "custom_fields": []}, "emitted_at": 1703096645556} +{"stream": "subscription", "data": {"id": "AzZTZgTTHdIU1NP", "billing_period": 1, "billing_period_unit": "month", "po_number": "0000002", "auto_collection": "on", "customer_id": "cbdemo_douglas", "status": "cancelled", "current_term_start": 1674035247, "current_term_end": 1676713649, "created_at": 1674035247, "started_at": 1674035247, "activated_at": 1674035247, "cancelled_at": 1676713649, "updated_at": 1678806943, "has_scheduled_changes": false, "cancel_schedule_created_at": 1674035247, "channel": "web", "resource_version": 1678806943363, "deleted": false, "object": "subscription", "currency_code": "USD", "subscription_items": [{"item_price_id": "cbdemo_advanced-USD-monthly", "item_type": "plan", "quantity": 1, "unit_price": 75000, "amount": 0, "free_quantity": 3, "object": "subscription_item"}, {"item_price_id": "cbdemo_additional-users-USD-monthly", "item_type": "addon", "quantity": 1, "unit_price": 50000, "amount": 50000, "object": "subscription_item"}, {"item_price_id": "cbdemo_setup-charge-USD", "item_type": "charge", "quantity": 1, "unit_price": 50000, "amount": 50000, "charge_on_event": "subscription_trial_start", "charge_once": false, "object": "subscription_item"}], "item_tiers": [{"item_price_id": "cbdemo_additional-users-USD-monthly", "starting_unit": 1, "ending_unit": 10, "price": 50000, "object": "item_tier"}, {"item_price_id": "cbdemo_additional-users-USD-monthly", "starting_unit": 11, "price": 100000, "object": "item_tier"}], "due_invoices_count": 0, "mrr": 0, "cf_my_custom_checkbox_field": "False", "has_scheduled_advance_invoices": false, "auto_close_invoices": true, "custom_fields": [{"name": "cf_my_custom_checkbox_field", "value": "False"}]}, "emitted_at": 1703096645562} +{"stream": "customer", "data": {"id": "Customer-5", "first_name": "Sample Name Five", "last_name": "Sample Lastname 5", "email": "name5@example.com", "phone": "+1 388 833 5555", "company": "Test Company Org 5", "auto_collection": "on", "net_term_days": 0, "allow_direct_debit": false, "created_at": 1674055210, "taxability": "taxable", "updated_at": 1678987385, "pii_cleared": "active", "channel": "web", "resource_version": 1678987385441, "deleted": false, "object": "customer", "card_status": "no_card", "promotional_credits": 0, "refundable_credits": 0, "excess_payments": 0, "unbilled_charges": 0, "preferred_currency_code": "USD", "mrr": 0, "tax_providers_fields": [], "auto_close_invoices": true, "custom_fields": []}, "emitted_at": 1703113529371} +{"stream": "customer", "data": {"id": "Test-Custome-1", "first_name": "Sample Name 2", "last_name": "Sample Lastname 2", "email": "name2@example.com", "phone": "+1 382 333 3883", "company": "Test Customer Org", "auto_collection": "on", "net_term_days": 0, "allow_direct_debit": false, "created_at": 1674038786, "taxability": "taxable", "updated_at": 1678987385, "locale": "en", "pii_cleared": "active", "channel": "web", "resource_version": 1678987385448, "deleted": false, "object": "customer", "billing_address": {"first_name": "Sample Name Two", "last_name": "Sample Lastname 2", "email": "name2@example.com", "company": "Test Company Org 2", "phone": "+1 388 846 3888", "line1": "Ms Ninette Franck", "line2": "4381", "city": "San Francisco", "state_code": "CA", "state": "California", "country": "US", "zip": "94114", "validation_status": "not_validated", "object": "billing_address"}, "card_status": "no_card", "contacts": [{"id": "Test Contact 2", "first_name": "Sample Name Two", "last_name": "Sample Lastname 2", "email": "name2@example.com", "phone": "+13888433888", "enabled": true, "send_account_email": false, "send_billing_email": false, "object": "contact"}, {"id": "Test Contact 3", "first_name": "Sample Name Three", "last_name": "Sample Lastname 3", "email": "name2@example.com", "phone": "+13888434888", "enabled": true, "send_account_email": false, "send_billing_email": false, "object": "contact"}, {"id": "Test Contact 4", "first_name": "Sample Name Four", "last_name": "Sample Lastname 4", "email": "name4@example.com", "phone": "+13888333888", "enabled": true, "send_account_email": false, "send_billing_email": false, "object": "contact"}], "balances": [{"promotional_credits": 120, "excess_payments": 0, "refundable_credits": 0, "unbilled_charges": 0, "object": "customer_balance", "currency_code": "USD", "balance_currency_code": "USD"}], "promotional_credits": 120, "refundable_credits": 0, "excess_payments": 0, "unbilled_charges": 0, "preferred_currency_code": "USD", "mrr": 0, "tax_providers_fields": [], "auto_close_invoices": true, "custom_fields": []}, "emitted_at": 1703113529374} +{"stream": "customer", "data": {"id": "cbdemo_simon", "first_name": "Simon", "last_name": "Masrani", "email": "simon_AT_test.com@example.com", "phone": "2909447832", "company": "Openlane Ltd", "auto_collection": "on", "net_term_days": 0, "allow_direct_debit": false, "created_at": 1627671474, "created_from_ip": "176.37.67.33", "taxability": "taxable", "updated_at": 1678987385, "pii_cleared": "active", "channel": "web", "resource_version": 1678987385451, "deleted": false, "object": "customer", "card_status": "valid", "promotional_credits": 0, "refundable_credits": 0, "excess_payments": 0, "unbilled_charges": 0, "preferred_currency_code": "USD", "mrr": 0, "primary_payment_source_id": "pm_AzZlweSefvdeFUxy", "payment_method": {"object": "payment_method", "type": "card", "reference_id": "tok_AzZlweSefvde2Uxx", "gateway": "chargebee", "gateway_account_id": "gw_16CKmRSb2oGddH4", "status": "valid"}, "tax_providers_fields": [], "auto_close_invoices": true, "custom_fields": []}, "emitted_at": 1703113529377} +{"stream": "invoice", "data": {"id": "cbdemo_inv_003", "customer_id": "cbdemo_tyler", "subscription_id": "cbdemo_non_renewing_sub", "recurring": true, "status": "paid", "price_type": "tax_exclusive", "date": 1626053281, "due_date": 1626053281, "net_term_days": 0, "exchange_rate": 1.0, "total": 80000, "amount_paid": 77000, "amount_adjusted": 3000, "write_off_amount": 0, "credits_applied": 0, "amount_due": 0, "paid_at": 1674070800, "updated_at": 1674034862, "resource_version": 1674034862390, "deleted": false, "object": "invoice", "first_invoice": true, "amount_to_collect": 0, "round_off_amount": 0, "new_sales_amount": 80000, "has_advance_charges": false, "currency_code": "USD", "base_currency_code": "USD", "generated_at": 1626053281, "is_gifted": false, "term_finalized": true, "channel": "web", "tax": 0, "line_items": [{"id": "li_AzZlweSefvesUUyx", "date_from": 1626053281, "date_to": 1628731680, "unit_amount": 60000, "quantity": 1, "amount": 60000, "pricing_model": "per_unit", "is_taxed": false, "tax_amount": 0, "object": "line_item", "subscription_id": "cbdemo_non_renewing_sub", "customer_id": "cbdemo_tyler", "description": "Intermediary - Monthly Plan", "entity_type": "plan_item_price", "entity_id": "cbdemo_intermediary-USD-monthly", "entity_description": "Intermediary Monthly Plan USD", "metered": false, "tax_exempt_reason": "tax_not_configured", "discount_amount": 0, "item_level_discount_amount": 0}, {"id": "li_AzZlweSefvesZUyy", "date_from": 1626053281, "date_to": 1628731680, "unit_amount": 20000, "quantity": 1, "amount": 20000, "pricing_model": "flat_fee", "is_taxed": false, "tax_amount": 0, "object": "line_item", "subscription_id": "cbdemo_non_renewing_sub", "customer_id": "cbdemo_tyler", "description": "Additional Analytics - Monthly Addon", "entity_type": "addon_item_price", "entity_id": "cbdemo_additional-analytics-USD-monthly", "metered": false, "tax_exempt_reason": "tax_not_configured", "discount_amount": 0, "item_level_discount_amount": 0}], "sub_total": 80000, "linked_payments": [{"txn_id": "txn_AzZlweSefvetOUyz", "applied_amount": 50000, "applied_at": 1626053281, "txn_status": "success", "txn_date": 1626053281, "txn_amount": 50000}, {"txn_id": "txn_AzZTZgTTHbgFlKJ", "applied_amount": 27000, "applied_at": 1674034862, "txn_status": "success", "txn_date": 1674070800, "txn_amount": 27000}], "applied_credits": [], "adjustment_credit_notes": [{"cn_id": "TEST-CN-4", "cn_reason_code": "waiver", "cn_create_reason_code": "Waiver", "cn_date": 1626917282, "cn_total": 3000, "cn_status": "adjusted"}], "issued_credit_notes": [], "linked_orders": [], "dunning_attempts": [], "billing_address": {"first_name": "Tyler", "last_name": "Durden", "company": "Iselectrics", "validation_status": "not_validated", "object": "billing_address"}, "custom_fields": []}, "emitted_at": 1703114360732} +{"stream": "invoice", "data": {"id": "23", "po_number": "0000002", "customer_id": "cbdemo_douglas", "subscription_id": "AzZTZgTTHdIU1NP", "recurring": true, "status": "paid", "price_type": "tax_exclusive", "date": 1674035390, "due_date": 1674035390, "net_term_days": 0, "exchange_rate": 1.0, "total": 50000, "amount_paid": 50000, "amount_adjusted": 0, "write_off_amount": 0, "credits_applied": 0, "amount_due": 0, "paid_at": 1674071372, "dunning_status": "stopped", "updated_at": 1674035428, "resource_version": 1674035428134, "deleted": false, "object": "invoice", "first_invoice": true, "amount_to_collect": 0, "round_off_amount": 0, "new_sales_amount": 50000, "has_advance_charges": false, "currency_code": "USD", "base_currency_code": "USD", "generated_at": 1674035390, "is_gifted": false, "term_finalized": true, "channel": "web", "tax": 0, "line_items": [{"id": "li_AzZTZgTTHdIUcNR", "date_from": 1674035247, "date_to": 1676713647, "unit_amount": 50000, "quantity": 1, "amount": 50000, "pricing_model": "tiered", "is_taxed": false, "tax_amount": 0, "object": "line_item", "subscription_id": "AzZTZgTTHdIU1NP", "customer_id": "cbdemo_douglas", "description": "Additional Users - Monthly Addon", "entity_type": "addon_item_price", "entity_id": "cbdemo_additional-users-USD-monthly", "metered": false, "tax_exempt_reason": "tax_not_configured", "discount_amount": 0, "item_level_discount_amount": 0}], "line_item_tiers": [{"starting_unit": 1, "ending_unit": 10, "quantity_used": 1, "unit_amount": 50000, "object": "line_item_tier", "line_item_id": "li_AzZTZgTTHdIUcNR"}], "sub_total": 50000, "linked_payments": [{"txn_id": "txn_16CQyCTTHdwAAwZq", "applied_amount": 50000, "applied_at": 1674035400, "txn_status": "failure", "txn_date": 1674035400, "txn_amount": 50000}, {"txn_id": "txn_16CLzOTTHe3QSDF", "applied_amount": 50000, "applied_at": 1674035428, "txn_status": "success", "txn_date": 1674071372, "txn_amount": 50000}], "applied_credits": [], "adjustment_credit_notes": [], "issued_credit_notes": [], "linked_orders": [], "dunning_attempts": [{"created_at": 1674035400, "attempt": 0, "dunning_type": "auto_collect", "transaction_id": "txn_16CQyCTTHdwAAwZq", "txn_status": "failure", "txn_amount": 50000}], "billing_address": {"first_name": "Douglas", "last_name": "Quaid", "company": "Greenplus Enterprises", "validation_status": "not_validated", "object": "billing_address"}, "custom_fields": []}, "emitted_at": 1703114360741} +{"stream": "invoice", "data": {"id": "24", "po_number": "0000003", "customer_id": "cbdemo_tyler", "subscription_id": "6olOsTTHieWUY9", "recurring": true, "status": "paid", "price_type": "tax_exclusive", "date": 1674036523, "due_date": 1674036523, "net_term_days": 0, "exchange_rate": 1.0, "total": 51000, "amount_paid": 51000, "amount_adjusted": 0, "write_off_amount": 0, "credits_applied": 0, "amount_due": 0, "paid_at": 1674036524, "updated_at": 1674036524, "resource_version": 1674036524204, "deleted": false, "object": "invoice", "first_invoice": true, "amount_to_collect": 0, "round_off_amount": 0, "new_sales_amount": 51000, "has_advance_charges": false, "currency_code": "USD", "base_currency_code": "USD", "generated_at": 1674036523, "is_gifted": false, "term_finalized": true, "channel": "web", "tax": 0, "line_items": [{"id": "li_6olOsTTHieX6YB", "date_from": 1674036523, "date_to": 1674122923, "unit_amount": 1000, "quantity": 1, "amount": 1000, "pricing_model": "flat_fee", "is_taxed": false, "tax_amount": 0, "object": "line_item", "subscription_id": "6olOsTTHieWUY9", "customer_id": "cbdemo_tyler", "description": "Test Plan 1", "entity_type": "plan_item_price", "entity_id": "Test-Plan-1-USD-Daily", "entity_description": "Test", "metered": false, "tax_exempt_reason": "tax_not_configured", "discount_amount": 0, "item_level_discount_amount": 0}, {"id": "li_6olOsTTHieXBYC", "date_from": 1674036523, "date_to": 1674900523, "unit_amount": 50000, "quantity": 1, "amount": 50000, "pricing_model": "flat_fee", "is_taxed": false, "tax_amount": 0, "object": "line_item", "subscription_id": "6olOsTTHieWUY9", "customer_id": "cbdemo_tyler", "description": "Setup Charge", "entity_type": "charge_item_price", "entity_id": "cbdemo_setup-charge-USD", "metered": false, "tax_exempt_reason": "tax_not_configured", "discount_amount": 0, "item_level_discount_amount": 0}], "sub_total": 51000, "linked_payments": [{"txn_id": "txn_6olOsTTHieYGYD", "applied_amount": 51000, "applied_at": 1674036524, "txn_status": "success", "txn_date": 1674036524, "txn_amount": 51000}], "applied_credits": [], "adjustment_credit_notes": [], "issued_credit_notes": [], "linked_orders": [{"id": "1", "status": "queued", "created_at": 1674036525}], "dunning_attempts": [], "billing_address": {"first_name": "Tyler", "last_name": "Durden", "company": "Iselectrics", "validation_status": "not_validated", "object": "billing_address"}, "notes": [{"note": "Test", "entity_type": "plan_item_price", "entity_id": "Test-Plan-1-USD-Daily"}], "custom_fields": []}, "emitted_at": 1703114360749} +{"stream": "credit_note", "data": {"id": "TEST-CN-5", "customer_id": "cbdemo_simon", "subscription_id": "cbdemo_future_sub", "reference_invoice_id": "19", "type": "refundable", "reason_code": "product_unsatisfactory", "status": "refunded", "date": 1674033113, "price_type": "tax_exclusive", "exchange_rate": 1.0, "total": 80000, "amount_allocated": 80000, "amount_refunded": 0, "amount_available": 0, "refunded_at": 1674872880, "generated_at": 1674033113, "updated_at": 1674872880, "channel": "web", "resource_version": 1674872880610, "deleted": false, "object": "credit_note", "create_reason_code": "Product Unsatisfactory", "currency_code": "USD", "round_off_amount": 0, "fractional_correction": 0, "base_currency_code": "USD", "sub_total": 80000, "line_items": [{"id": "li_16CM7mTTHULEa1hh", "date_from": 1674069063, "date_to": 1674069063, "unit_amount": 50000, "quantity": 1, "amount": 50000, "pricing_model": "per_unit", "is_taxed": false, "tax_amount": 0, "object": "line_item", "subscription_id": "cbdemo_future_sub", "customer_id": "cbdemo_simon", "description": "Lite - Monthly Plan", "entity_type": "plan_item_price", "entity_id": "cbdemo_lite-USD-monthly", "entity_description": "Lite Monthly Plan USD", "reference_line_item_id": "li_16CZgbTIgigQV7CGu", "metered": false, "tax_exempt_reason": "tax_not_configured", "discount_amount": 0, "item_level_discount_amount": 0}, {"id": "li_16CM7mTTHULEe1hi", "date_from": 1674069063, "date_to": 1674069063, "unit_amount": 30000, "quantity": 1, "amount": 30000, "pricing_model": "flat_fee", "is_taxed": false, "tax_amount": 0, "object": "line_item", "subscription_id": "cbdemo_future_sub", "customer_id": "cbdemo_simon", "description": "Concierge Support - Monthly Addon", "entity_type": "addon_item_price", "entity_id": "cbdemo_concierge-support-USD-monthly", "reference_line_item_id": "li_16CZgbTIgigQj7CGv", "metered": false, "tax_exempt_reason": "tax_not_configured", "discount_amount": 0, "item_level_discount_amount": 0}], "taxes": [], "line_item_taxes": [], "line_item_discounts": [], "linked_refunds": [], "allocations": [{"allocated_amount": 20300, "allocated_at": 1674872880, "invoice_id": "64", "invoice_date": 1674872877, "invoice_status": "paid"}, {"allocated_amount": 1000, "allocated_at": 1674815049, "invoice_id": "62", "invoice_date": 1674815048, "invoice_status": "paid"}, {"allocated_amount": 1000, "allocated_at": 1674728652, "invoice_id": "58", "invoice_date": 1674728648, "invoice_status": "paid"}, {"allocated_amount": 1000, "allocated_at": 1674642252, "invoice_id": "54", "invoice_date": 1674642248, "invoice_status": "paid"}, {"allocated_amount": 1000, "allocated_at": 1674555851, "invoice_id": "50", "invoice_date": 1674555848, "invoice_status": "paid"}, {"allocated_amount": 1000, "allocated_at": 1674469451, "invoice_id": "46", "invoice_date": 1674469448, "invoice_status": "paid"}, {"allocated_amount": 1000, "allocated_at": 1674383053, "invoice_id": "42", "invoice_date": 1674383048, "invoice_status": "paid"}, {"allocated_amount": 1000, "allocated_at": 1674296651, "invoice_id": "38", "invoice_date": 1674296648, "invoice_status": "paid"}, {"allocated_amount": 1000, "allocated_at": 1674210252, "invoice_id": "34", "invoice_date": 1674210248, "invoice_status": "paid"}, {"allocated_amount": 1000, "allocated_at": 1674123850, "invoice_id": "30", "invoice_date": 1674123848, "invoice_status": "paid"}, {"allocated_amount": 50700, "allocated_at": 1674037448, "invoice_id": "26", "invoice_date": 1674037448, "invoice_status": "paid"}], "billing_address": {"first_name": "Simon", "last_name": "Masrani", "company": "Openlane Ltd", "validation_status": "not_validated", "object": "billing_address"}, "customer_notes": "", "custom_fields": []}, "emitted_at": 1703612727121} +{"stream": "credit_note", "data": {"id": "TEST-CN-6", "customer_id": "cbdemo_simon", "subscription_id": "AzZTZgTTHmX8Gc1", "reference_invoice_id": "128", "type": "refundable", "reason_code": "product_unsatisfactory", "status": "refunded", "date": 1676371612, "price_type": "tax_exclusive", "exchange_rate": 1.0, "total": 200, "amount_allocated": 200, "amount_refunded": 0, "amount_available": 0, "refunded_at": 1676456650, "generated_at": 1676371612, "updated_at": 1676456650, "channel": "web", "resource_version": 1676456650511, "deleted": false, "object": "credit_note", "create_reason_code": "Product Unsatisfactory", "currency_code": "USD", "round_off_amount": 0, "fractional_correction": 0, "base_currency_code": "USD", "sub_total": 200, "line_items": [{"id": "li_16CM0pTVpkRhZ29v", "date_from": 1676371612, "date_to": 1676371612, "unit_amount": 200, "quantity": 1, "amount": 200, "pricing_model": "flat_fee", "is_taxed": false, "tax_amount": 0, "object": "line_item", "subscription_id": "AzZTZgTTHmX8Gc1", "customer_id": "cbdemo_simon", "description": "Test Plan 1", "entity_type": "plan_item_price", "entity_id": "Test-Plan-1-USD-Daily", "entity_description": "Test", "reference_line_item_id": "li_16CR6XTVdxgpUH1gi", "metered": false, "tax_exempt_reason": "tax_not_configured", "discount_amount": 0, "item_level_discount_amount": 0}], "taxes": [], "line_item_taxes": [], "line_item_discounts": [], "linked_refunds": [], "allocations": [{"allocated_amount": 200, "allocated_at": 1676456650, "invoice_id": "140", "invoice_date": 1676456648, "invoice_status": "paid"}], "billing_address": {"first_name": "Simon", "last_name": "Masrani", "company": "Openlane Ltd", "validation_status": "not_validated", "object": "billing_address"}, "customer_notes": "", "custom_fields": []}, "emitted_at": 1703612727285} +{"stream": "credit_note", "data": {"id": "TEST-CN-7", "customer_id": "cbdemo_simon", "subscription_id": "AzZTZgTTHmX8Gc1", "reference_invoice_id": "128", "type": "refundable", "reason_code": "product_unsatisfactory", "status": "refunded", "date": 1676371655, "price_type": "tax_exclusive", "exchange_rate": 1.0, "total": 100, "amount_allocated": 100, "amount_refunded": 0, "amount_available": 0, "refunded_at": 1676456650, "generated_at": 1676371655, "updated_at": 1676456650, "channel": "web", "resource_version": 1676456650518, "deleted": false, "object": "credit_note", "create_reason_code": "Product Unsatisfactory", "currency_code": "USD", "round_off_amount": 0, "fractional_correction": 0, "base_currency_code": "USD", "sub_total": 100, "line_items": [{"id": "li_AzZTODTVpkcu827U", "date_from": 1676371655, "date_to": 1676371655, "unit_amount": 100, "quantity": 1, "amount": 100, "pricing_model": "flat_fee", "is_taxed": false, "tax_amount": 0, "object": "line_item", "subscription_id": "AzZTZgTTHmX8Gc1", "customer_id": "cbdemo_simon", "description": "Test Plan 1", "entity_type": "plan_item_price", "entity_id": "Test-Plan-1-USD-Daily", "entity_description": "Test", "reference_line_item_id": "li_16CR6XTVdxgpUH1gi", "metered": false, "tax_exempt_reason": "tax_not_configured", "discount_amount": 0, "item_level_discount_amount": 0}], "taxes": [], "line_item_taxes": [], "line_item_discounts": [], "linked_refunds": [], "allocations": [{"allocated_amount": 100, "allocated_at": 1676456650, "invoice_id": "140", "invoice_date": 1676456648, "invoice_status": "paid"}], "billing_address": {"first_name": "Simon", "last_name": "Masrani", "company": "Openlane Ltd", "validation_status": "not_validated", "object": "billing_address"}, "customer_notes": "", "custom_fields": []}, "emitted_at": 1703612727293} +{"stream":"comment","data":{"id":"cmt_16CM7mTTHULGY1hj","entity_type":"credit_note","entity_id":"TEST-CN-5","notes":"Test","added_by":"integration-test@airbyte.io","created_at":1674033113,"type":"user","object":"comment","custom_fields":[]},"emitted_at":1703800968922} +{"stream":"comment","data":{"id":"cmt_AzZTZgTTHaxTnJV","entity_type":"subscription","entity_id":"cbdemo_trial_sub","notes":"Test PO 0000001","added_by":"integration-test@airbyte.io","created_at":1674034690,"type":"user","object":"comment","custom_fields":[]},"emitted_at":1703800968925} +{"stream":"comment","data":{"id":"cmt_AzZTZgTTHbgGdKK","entity_type":"transaction","entity_id":"txn_AzZTZgTTHbgFlKJ","notes":"Test cash payment","added_by":"integration-test@airbyte.io","created_at":1674034862,"type":"user","object":"comment","custom_fields":[]},"emitted_at":1703800968928} +{"stream":"item_family","data":{"id":"cbdemo_pf_analytics","name":"Analytics","description":"Reporting and analytics for your business.","status":"active","resource_version":1627671463249,"updated_at":1627671463,"object":"item_family","custom_fields":[]},"emitted_at":1703880848793} +{"stream":"item_family","data":{"id":"cbdemo_pf_crm","name":"CRM","description":"Turn your leads into paying customers.","status":"active","resource_version":1627671463158,"updated_at":1627671463,"object":"item_family","custom_fields":[]},"emitted_at":1703880848797} +{"stream":"differential_price","data":{"id":"7748afa6-fdbe-4304-ac23-d18a17f27715","item_price_id":"cbdemo_additional-analytics-USD-yearly","parent_item_id":"cbdemo_advanced","price":200000,"status":"active","resource_version":1674032542218,"updated_at":1674032542,"created_at":1674032542,"currency_code":"USD","object":"differential_price","custom_fields":[]},"emitted_at":1704223399378} +{"stream":"differential_price","data":{"id":"7748afa6-fdbe-4304-ac23-d18a17f27715","item_price_id":"cbdemo_additional-analytics-USD-yearly","parent_item_id":"cbdemo_advanced","price":200000,"status":"active","resource_version":1674032542218,"updated_at":1674032542,"created_at":1674032542,"currency_code":"USD","object":"differential_price","custom_fields":[]},"emitted_at":1704223399498} +{"stream":"differential_price","data":{"id":"7748afa6-fdbe-4304-ac23-d18a17f27715","item_price_id":"cbdemo_additional-analytics-USD-yearly","parent_item_id":"cbdemo_advanced","price":200000,"status":"active","resource_version":1674032542218,"updated_at":1674032542,"created_at":1674032542,"currency_code":"USD","object":"differential_price","custom_fields":[]},"emitted_at":1704223399634} diff --git a/airbyte-integrations/connectors/source-chargebee/integration_tests/future_state.json b/airbyte-integrations/connectors/source-chargebee/integration_tests/future_state.json index b7226a840311..32c87ffe2c6d 100644 --- a/airbyte-integrations/connectors/source-chargebee/integration_tests/future_state.json +++ b/airbyte-integrations/connectors/source-chargebee/integration_tests/future_state.json @@ -110,5 +110,33 @@ "stream_state": { "updated_at": 2147483647 }, "stream_descriptor": { "name": "transaction" } } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "migrated_at": 2147483647 }, + "stream_descriptor": { "name": "site_migration_detail" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created_at": 2147483647 }, + "stream_descriptor": { "name": "comment" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 2147483647 }, + "stream_descriptor": { "name": "item_family" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 2147483647 }, + "stream_descriptor": { "name": "differential_price" } + } } ] diff --git a/airbyte-integrations/connectors/source-chargebee/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-chargebee/integration_tests/sample_state.json index 468a1e167331..f8a43e7e1d9a 100644 --- a/airbyte-integrations/connectors/source-chargebee/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-chargebee/integration_tests/sample_state.json @@ -110,5 +110,33 @@ "stream_state": { "updated_at": 1625596058 }, "stream_descriptor": { "name": "transaction" } } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "migrated_at": 1625596058 }, + "stream_descriptor": { "name": "site_migration_detail" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "created_at": 1625596058 }, + "stream_descriptor": { "name": "comment" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 1625596058 }, + "stream_descriptor": { "name": "item_family" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": 1625596058 }, + "stream_descriptor": { "name": "differential_price" } + } } ] diff --git a/airbyte-integrations/connectors/source-chargebee/metadata.yaml b/airbyte-integrations/connectors/source-chargebee/metadata.yaml index 789788ab5d56..9fa19f17b91f 100644 --- a/airbyte-integrations/connectors/source-chargebee/metadata.yaml +++ b/airbyte-integrations/connectors/source-chargebee/metadata.yaml @@ -1,12 +1,18 @@ data: + ab_internal: + ql: 200 + sl: 200 allowedHosts: hosts: - "*.chargebee.com" + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: api connectorType: source definitionId: 686473f1-76d9-4994-9cc7-9b13da46147c - dockerImageTag: 0.2.4 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-chargebee + documentationUrl: https://docs.airbyte.com/integrations/sources/chargebee githubIssueLabel: source-chargebee icon: chargebee.svg license: MIT @@ -17,12 +23,8 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/chargebee + supportLevel: certified tags: - language:low-code - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/manifest.yaml b/airbyte-integrations/connectors/source-chargebee/source_chargebee/manifest.yaml index 5bfaee7de614..c47fa6d7dd11 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/manifest.yaml +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/manifest.yaml @@ -315,6 +315,54 @@ definitions: name: "quote_line_group" primary_key: "id" path: "/quotes/{{ stream_slice.id }}/quote_line_groups" + site_migration_detail_stream: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/retriever/requester" + request_parameters: + sort_by[asc]: migrated_at + include_deleted: "true" + migrated_at[between]: "'[{{stream_slice['start_time']}}, {{stream_slice['end_time']}}]'" + incremental_sync: + $ref: "#/definitions/date_stream_slicer" + $parameters: + name: "site_migration_detail" + primary_key: "entity_id" + path: "/site_migration_details" + stream_cursor_field: "migrated_at" + comment_stream: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/retriever/requester" + request_parameters: + sort_by[asc]: created_at + include_deleted: "true" + created_at[between]: "'[{{stream_slice['start_time']}}, {{stream_slice['end_time']}}]'" + incremental_sync: + $ref: "#/definitions/date_stream_slicer" + $parameters: + name: "comment" + primary_key: "id" + path: "/comments" + stream_cursor_field: "created_at" + item_family_stream: + $ref: "#/definitions/base_incremental_stream" + $parameters: + name: "item_family" + primary_key: "id" + path: "/item_families" + stream_cursor_field: "updated_at" + differential_price_stream: + $ref: "#/definitions/base_incremental_stream" + $parameters: + name: "differential_price" + primary_key: "id" + path: "/differential_prices" + stream_cursor_field: "updated_at" streams: - "#/definitions/addon_stream" @@ -339,6 +387,10 @@ streams: - "#/definitions/virtual_bank_account_stream" - "#/definitions/quote_stream" - "#/definitions/quote_line_group_stream" + - "#/definitions/site_migration_detail_stream" + - "#/definitions/comment_stream" + - "#/definitions/item_family_stream" + - "#/definitions/differential_price_stream" check: stream_names: diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/addon.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/addon.json index b92355f36862..0564ea8a6172 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/addon.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/addon.json @@ -138,6 +138,15 @@ "show_description_in_quotes": { "type": ["boolean", "null"] }, + "channel": { + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, "tiers": { "type": ["array", "null"], "items": { diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/comment.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/comment.json new file mode 100644 index 000000000000..1f5d920c7399 --- /dev/null +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/comment.json @@ -0,0 +1,34 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "name": "Comment", + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "entity_type": { + "type": ["string", "null"] + }, + "added_by": { + "type": ["string", "null"] + }, + "notes": { + "type": ["string", "null"] + }, + "created_at": { + "type": ["integer", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "entity_id": { + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] + }, + "custom_fields": { + "type": ["array", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/coupon.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/coupon.json index f6d484b980a1..4fbd0682d34b 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/coupon.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/coupon.json @@ -74,6 +74,9 @@ "type": ["string", "null"], "max-length": 2000 }, + "object": { + "type": ["string", "null"] + }, "item_constraints": { "type": ["array", "null"], "items": { diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/credit_note.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/credit_note.json index 3e3496697263..5bcc57427ca0 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/credit_note.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/credit_note.json @@ -106,6 +106,30 @@ "type": ["string", "null"], "maxLength": 10 }, + "base_currency_code": { + "type": ["string", "null"], + "maxLength": 3 + }, + "business_entity_id": { + "type": ["string", "null"], + "maxLength": 50 + }, + "channel": { + "type": ["string", "null"] + }, + "exchange_rate": { + "type": ["number", "null"] + }, + "is_digital": { + "type": ["boolean", "null"] + }, + "object": { + "type": ["string", "null"] + }, + "is_vat_moss_registered": { + "type": ["boolean", "null"], + "$comment": "Only available for accounts which have enabled taxes for EU Region for taxes." + }, "line_items": { "type": ["array", "null"], "items": { diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/customer.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/customer.json index 61f02e49f104..0d142b990644 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/customer.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/customer.json @@ -165,6 +165,16 @@ "type": ["string", "null"], "maxLength": 10 }, + "business_entity_id": { + "type": ["string", "null"], + "maxLength": 50 + }, + "channel": { + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] + }, "billing_address": { "type": ["object", "null"], "properties": { @@ -413,7 +423,6 @@ "card_status": { "type": ["string", "null"] }, - "meta_data": { "type": ["object", "null"], "properties": {} diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/differential_price.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/differential_price.json new file mode 100644 index 000000000000..5087f9f9bc99 --- /dev/null +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/differential_price.json @@ -0,0 +1,77 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "name": "Add-on", + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "item_price_id": { + "type": ["string", "null"] + }, + "parent_item_id": { + "type": ["string", "null"] + }, + "price": { + "type": ["integer", "null"] + }, + "price_in_decimal": { + "type": ["string", "null"] + }, + "status": { + "type": ["string", "null"] + }, + "resource_version": { + "type": ["integer", "null"] + }, + "updated_at": { + "type": ["integer", "null"] + }, + "created_at": { + "type": ["integer", "null"] + }, + "modified_at": { + "type": ["integer", "null"] + }, + "currency_code": { + "type": ["string", "null"] + }, + "tiers": { + "type": ["array", "null"], + "items": { + "type": "object", + "properties": { + "starting_unit": { + "type": ["integer", "null"] + }, + "ending_unit": { + "type": ["integer", "null"] + }, + "price": { + "type": ["integer", "null"] + } + } + } + }, + "parent_periods": { + "type": ["array", "null"], + "items": { + "type": "object", + "properties": { + "period_unit": { + "type": ["string", "null"] + }, + "period": { + "type": ["array", "null"], + "items": { + "type": ["integer", "null"] + } + } + } + } + }, + "object": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/event.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/event.json index f76bd77e4799..98fe0b6fdc13 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/event.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/event.json @@ -29,6 +29,9 @@ "content": { "type": ["object", "null"] }, + "object": { + "type": ["string", "null"] + }, "webhooks": { "type": ["array", "null"], "items": { diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/invoice.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/invoice.json index a3815605c08c..8c8bad1b3561 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/invoice.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/invoice.json @@ -2,6 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "name": "Invoice", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["string", "null"], @@ -109,6 +110,9 @@ "type": ["integer", "null"], "minimum": 0 }, + "local_currency_exchange_rate": { + "type": ["number", "null"] + }, "first_invoice": { "type": ["boolean", "null"] }, @@ -136,13 +140,11 @@ "minimum": 0 }, "round_off_amount": { - "type": ["integer", "null"], - "minimum": 0 + "type": ["integer", "null"] }, "payment_owner": { "type": ["string", "null"] }, - "void_reason_code": { "type": ["string", "null"], "maxLength": 100 @@ -150,10 +152,26 @@ "deleted": { "type": ["boolean", "null"] }, + "tax_category": { + "type": ["string", "null"] + }, "vat_number_prefix": { "type": ["string", "null"], "maxLength": 10 }, + "channel": { + "type": ["string", "null"] + }, + "business_entity_id": { + "type": ["string", "null"] + }, + "base_currency_code": { + "type": ["string", "null"], + "maxLength": 3 + }, + "object": { + "type": ["string", "null"] + }, "line_items": { "type": ["array", "null"], "items": { @@ -199,15 +217,15 @@ }, "unit_amount_in_decimal": { "type": ["string", "null"], - "maxLength": 33 + "maxLength": 39 }, "quantity_in_decimal": { "type": ["string", "null"], - "maxLength": 33 + "maxLength": 39 }, "amount_in_decimal": { "type": ["string", "null"], - "maxLength": 33 + "maxLength": 39 }, "discount_amount": { "type": ["integer", "null"], @@ -238,6 +256,9 @@ "customer_id": { "type": ["string", "null"], "maxLength": 100 + }, + "metered": { + "type": ["boolean", "null"] } } } @@ -261,6 +282,10 @@ "entity_id": { "type": ["string", "null"], "maxLength": 100 + }, + "coupon_set_code": { + "type": ["string", "null"], + "maxLength": 50 } } } @@ -281,7 +306,6 @@ "type": ["integer", "null"], "minimum": 0 }, - "entity_id": { "type": ["string", "null"], "maxLength": 100 @@ -325,6 +349,15 @@ "tax_rate": { "type": ["number", "null"] }, + "date_to": { + "type": ["integer", "null"] + }, + "date_from": { + "type": ["integer", "null"] + }, + "prorated_taxable_amount": { + "type": ["number", "null"] + }, "is_partial_tax_applied": { "type": ["boolean", "null"] }, @@ -354,7 +387,7 @@ "type": ["integer", "null"], "minimum": 0 }, - "local_currency-code": { + "local_currency_code": { "type": ["string", "null"] } } @@ -666,6 +699,23 @@ } } }, + "statement_descriptor": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string", "null"], + "maxLength": 40 + }, + "descriptor": { + "type": ["string", "null"], + "maxLength": 65000 + }, + "additional_info": { + "type": ["string", "null"], + "maxLength": 65000 + } + } + }, "billing_address": { "type": ["object", "null"], "properties": { @@ -726,6 +776,53 @@ } } }, + "einvoice": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string", "null"], + "maxLength": 50 + }, + "reference_number": { + "type": ["string", "null"], + "maxLength": 100 + }, + "status": { + "type": ["string", "null"] + }, + "message": { + "type": ["string", "null"], + "maxLength": 2500 + } + } + }, + "linked_taxes_withheld": { + "type": ["array", "null"], + "items": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string", "null"], + "maxLength": 40 + }, + "amount": { + "type": ["integer", "null"], + "minimum": 1 + }, + "description": { + "type": ["string", "null"], + "maxLength": 65000 + }, + "date": { + "type": ["integer", "null"] + }, + "reference_number": { + "type": ["string", "null"], + "maxLength": 100 + } + } + } + }, "custom_fields": { "type": ["null", "array"], "items": { diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item.json index 4607fb50c467..d3504eee807b 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item.json @@ -74,6 +74,10 @@ "type": ["object", "null"], "properties": {} }, + "external_name": { + "type": ["string", "null"], + "maxLength": 100 + }, "applicable_items": { "type": ["array", "null"], "items": { diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item_family.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item_family.json new file mode 100644 index 000000000000..e1e096e70d9c --- /dev/null +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/item_family.json @@ -0,0 +1,34 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "name": "Item Families", + "type": "object", + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "description": { + "type": ["string", "null"] + }, + "status": { + "type": ["string", "null"] + }, + "resource_version": { + "type": ["integer", "null"] + }, + "updated_at": { + "type": ["integer", "null"] + }, + "channel": { + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] + }, + "custom_fields": { + "type": ["array", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/order.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/order.json index 6b55529a9aab..34768cd9672f 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/order.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/order.json @@ -2,6 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "name": "Order", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["string", "null"], @@ -76,7 +77,6 @@ "type": ["string", "null"], "maxLength": 50 }, - "invoice_round_off_amount": { "type": ["integer", "null"], "minimum": 0 @@ -175,6 +175,20 @@ "type": ["string", "null"], "maxLength": 100 }, + "business_entity_id": { + "type": ["string", "null"], + "maxLength": 50 + }, + "base_currency_code": { + "type": ["string", "null"], + "maxLength": 3 + }, + "exchange_rate": { + "type": ["number", "null"] + }, + "object": { + "type": ["string", "null"] + }, "order_line_items": { "type": ["array", "null"], "items": { @@ -239,7 +253,6 @@ "type": ["string", "null"], "maxLength": 250 }, - "status": { "type": ["string", "null"] }, diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/payment_source.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/payment_source.json index 26095b14a47e..361df91555ee 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/payment_source.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/payment_source.json @@ -18,6 +18,9 @@ "customer_id": { "type": ["string", "null"] }, + "object": { + "type": ["string", "null"] + }, "type": { "type": ["string", "null"], "enum": [ diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/plan.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/plan.json index 6892dcc357ca..ecb0a70f78f1 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/plan.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/plan.json @@ -154,6 +154,15 @@ "type": ["string", "null"], "maxLength": 2000 }, + "channel": { + "type": ["string", "null"] + }, + "charge_model": { + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] + }, "taxable": { "type": ["boolean", "null"] }, diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/promotional_credit.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/promotional_credit.json index 7089ea9ef12c..4239abe82ae5 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/promotional_credit.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/promotional_credit.json @@ -41,6 +41,9 @@ "created_at": { "type": ["integer", "null"] }, + "object": { + "type": ["string", "null"] + }, "custom_fields": { "type": ["null", "array"], "items": { diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/site_migration_detail.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/site_migration_detail.json new file mode 100644 index 000000000000..1482bdcc657b --- /dev/null +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/site_migration_detail.json @@ -0,0 +1,41 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "name": "Site Migration Detail", + "type": "object", + "properties": { + "entity_id": { + "type": ["string", "null"], + "maxLength": 100 + }, + "other_site_name": { + "type": ["string", "null"], + "minLength": 4, + "maxLength": 100 + }, + "entity_id_at_other_site": { + "type": ["string", "null"], + "maxLength": 100 + }, + "migrated_at": { + "type": ["integer", "null"] + }, + "entity_type": { + "type": ["string", "null"], + "enum": [ + "customer", + "subscription", + "invoice", + "credit_note", + "transaction", + "order" + ] + }, + "status": { + "type": ["string", "null"], + "enum": ["moved_in", "moved_out", "moving_out"] + }, + "object": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/subscription.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/subscription.json index 4e7761b5d99e..e41fb562b6f8 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/subscription.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/subscription.json @@ -167,6 +167,13 @@ "auto_close_invoices": { "type": ["boolean", "null"] }, + "business_entity_id": { + "type": ["string", "null"], + "maxLength": 50 + }, + "channel": { + "type": ["string", "null"] + }, "coupons": { "type": ["array", "null"], "items": { @@ -347,7 +354,6 @@ } } }, - "subscription_items": { "type": ["array", "null"], "items": { @@ -466,7 +472,6 @@ } } }, - "plan_id": { "type": ["string", "null"], "maxLength": 100 diff --git a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/transaction.json b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/transaction.json index 4237fe31f793..87662abdf5c9 100644 --- a/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/transaction.json +++ b/airbyte-integrations/connectors/source-chargebee/source_chargebee/schemas/transaction.json @@ -132,6 +132,28 @@ "type": ["string", "null"], "max-length": 500 }, + "base_currency_code": { + "type": ["string", "null"], + "maxLength": 3 + }, + "business_entity_id": { + "type": ["string", "null"], + "maxLength": 50 + }, + "object": { + "type": ["string", "null"] + }, + "error_text": { + "type": ["string", "null"], + "maxLength": 65000 + }, + "payment_method_details": { + "type": ["string", "null"] + }, + "reference_number": { + "type": ["string", "null"], + "maxLength": 100 + }, "linked_invoices": { "type": ["array", "null"], "items": { diff --git a/airbyte-integrations/connectors/source-chargebee/unit_tests/test_component.py b/airbyte-integrations/connectors/source-chargebee/unit_tests/test_component.py index ce5f10bfde86..dcca547a7d16 100644 --- a/airbyte-integrations/connectors/source-chargebee/unit_tests/test_component.py +++ b/airbyte-integrations/connectors/source-chargebee/unit_tests/test_component.py @@ -9,13 +9,19 @@ @pytest.mark.parametrize( "record, expected_record", [ - ({'pk': 1, 'name': "example"}, {'pk': 1, 'name': "example", "custom_fields": []}), - ({'pk': 1, 'name': "example", "cf_field1": "val1", "cf_field2": "val2"}, - {'pk': 1, 'name': "example", "cf_field1": "val1", "cf_field2": "val2", - "custom_fields": [{"name": "cf_field1", "value": "val1"}, - {"name": "cf_field2", "value": "val2"}]}) + ({"pk": 1, "name": "example"}, {"pk": 1, "name": "example", "custom_fields": []}), + ( + {"pk": 1, "name": "example", "cf_field1": "val1", "cf_field2": "val2"}, + { + "pk": 1, + "name": "example", + "cf_field1": "val1", + "cf_field2": "val2", + "custom_fields": [{"name": "cf_field1", "value": "val1"}, {"name": "cf_field2", "value": "val2"}], + }, + ), ], - ids=["no_custom_field", "custom_field"] + ids=["no_custom_field", "custom_field"], ) def test_field_transformation(record, expected_record): transformer = CustomFieldTransformation() diff --git a/airbyte-integrations/connectors/source-chargify/Dockerfile b/airbyte-integrations/connectors/source-chargify/Dockerfile index 869d038aa4ce..274caf437671 100644 --- a/airbyte-integrations/connectors/source-chargify/Dockerfile +++ b/airbyte-integrations/connectors/source-chargify/Dockerfile @@ -34,5 +34,5 @@ COPY source_chargify ./source_chargify ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.0 +LABEL io.airbyte.version=0.4.0 LABEL io.airbyte.name=airbyte/source-chargify diff --git a/airbyte-integrations/connectors/source-chargify/README.md b/airbyte-integrations/connectors/source-chargify/README.md index 3f9b75013ad3..5d4ab397f1ea 100644 --- a/airbyte-integrations/connectors/source-chargify/README.md +++ b/airbyte-integrations/connectors/source-chargify/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-chargify:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/chargify) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_chargify/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-chargify:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-chargify build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-chargify:airbyteDocker +An image will be built with the tag `airbyte/source-chargify:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-chargify:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-chargify:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-chargify:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-chargify:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-chargify test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-chargify:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-chargify:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-chargify test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/chargify.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-chargify/acceptance-test-config.yml b/airbyte-integrations/connectors/source-chargify/acceptance-test-config.yml index 33cbcb687916..d25e058e4a93 100644 --- a/airbyte-integrations/connectors/source-chargify/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-chargify/acceptance-test-config.yml @@ -20,7 +20,9 @@ acceptance_tests: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] + empty_streams: + - name: transactions + bypass_reason: Not possible to retreive data using integration account # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file # expect_records: # path: "integration_tests/expected_records.jsonl" diff --git a/airbyte-integrations/connectors/source-chargify/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-chargify/acceptance-test-docker.sh deleted file mode 100644 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-chargify/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-chargify/build.gradle b/airbyte-integrations/connectors/source-chargify/build.gradle deleted file mode 100644 index e991e4fd1909..000000000000 --- a/airbyte-integrations/connectors/source-chargify/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_chargify' -} diff --git a/airbyte-integrations/connectors/source-chargify/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-chargify/integration_tests/configured_catalog.json index a97bddfc209e..4a1539233d96 100644 --- a/airbyte-integrations/connectors/source-chargify/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-chargify/integration_tests/configured_catalog.json @@ -17,6 +17,24 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "coupons", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "invoices", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-chargify/metadata.yaml b/airbyte-integrations/connectors/source-chargify/metadata.yaml index 873ecf5a0113..cc211d82e70c 100644 --- a/airbyte-integrations/connectors/source-chargify/metadata.yaml +++ b/airbyte-integrations/connectors/source-chargify/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: 9b2d3607-7222-4709-9fa2-c2abdebbdd88 - dockerImageTag: 0.3.0 + dockerImageTag: 0.4.0 dockerRepository: airbyte/source-chargify githubIssueLabel: source-chargify icon: chargify.svg @@ -21,7 +21,7 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/chargify tags: - - language:lowcode + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-chargify/source_chargify/manifest.yaml b/airbyte-integrations/connectors/source-chargify/source_chargify/manifest.yaml index 2b2f677e92b4..7aaa96a58261 100644 --- a/airbyte-integrations/connectors/source-chargify/source_chargify/manifest.yaml +++ b/airbyte-integrations/connectors/source-chargify/source_chargify/manifest.yaml @@ -64,9 +64,46 @@ definitions: path: "subscriptions.json" field_path: "subscription" + invoices_stream: + name: "invoices" + type: DeclarativeStream + primary_key: "uid" + retriever: + type: SimpleRetriever + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - "invoices" + paginator: + $ref: "#/definitions/paginator" + requester: + $ref: "#/definitions/requester" + $parameters: + path: "invoices.json" + field_path: "invoices" + + coupons_stream: + $ref: "#/definitions/base_stream" + name: "coupons" + $parameters: + path: "coupons.json" + field_path: "coupon" + + transactions_stream: + $ref: "#/definitions/base_stream" + name: "transactions" + $parameters: + path: "transactions.json" + field_path: "transaction" + streams: - "#/definitions/customers_stream" - "#/definitions/subscriptions_stream" + - "#/definitions/invoices_stream" + - "#/definitions/coupons_stream" + - "#/definitions/transactions_stream" check: type: CheckStream diff --git a/airbyte-integrations/connectors/source-chargify/source_chargify/schemas/coupons.json b/airbyte-integrations/connectors/source-chargify/source_chargify/schemas/coupons.json new file mode 100644 index 000000000000..067416b2d9d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-chargify/source_chargify/schemas/coupons.json @@ -0,0 +1,49 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "integer"] }, + "name": { "type": ["null", "string"] }, + "code": { "type": ["null", "string"] }, + "description": { "type": ["null", "string"] }, + "amount": { "type": ["null", "integer"] }, + "amount_in_cents": { "type": ["null", "integer"] }, + "product_family_id": { "type": ["null", "integer"] }, + "product_family_name": { "type": ["null", "string"] }, + "start_date": { "type": ["null", "string"] }, + "end_date": { "type": ["null", "string"] }, + "percentage": { "type": ["null", "integer"] }, + "recurring": { "type": ["null", "boolean"] }, + "recurring_scheme": { "type": ["null", "string"] }, + "duration_period_count": { "type": ["null", "integer"] }, + "duration_interval": { "type": ["null", "integer"] }, + "duration_interval_unit": { "type": ["null", "string"] }, + "duration_interval_span": { "type": ["null", "string"] }, + "allow_negative_balance": { "type": ["null", "boolean"] }, + "archived_at": { "type": ["null", "string"] }, + "conversion_limit": { "type": ["null", "string"] }, + "stackable": { "type": ["null", "boolean"] }, + "compounding_strategy": { "type": ["null", "string"] }, + "use_site_exchange_rate": { "type": ["null", "boolean"] }, + "created_at": { "type": ["null", "string"] }, + "updated_at": { "type": ["null", "string"] }, + "discount_type": { "type": ["null", "string"] }, + "exclude_mid_period_allocations": { "type": ["null", "boolean"] }, + "apply_on_cancel_at_end_of_period ": { "type": ["null", "boolean"] }, + "coupon_restrictions": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "string"] }, + "item_type": { "type": ["null", "string"] }, + "item_id": { "type": ["null", "integer"] }, + "name": { "type": ["null", "string"] }, + "handle": { "type": ["null", "string"] } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-chargify/source_chargify/schemas/invoices.json b/airbyte-integrations/connectors/source-chargify/source_chargify/schemas/invoices.json new file mode 100644 index 000000000000..7017fbc749cf --- /dev/null +++ b/airbyte-integrations/connectors/source-chargify/source_chargify/schemas/invoices.json @@ -0,0 +1,90 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "uid": { "type": ["null", "string"] }, + "site_id": { "type": ["null", "integer"] }, + "customer_id": { "type": ["null", "integer"] }, + "subscription_id": { "type": ["null", "integer"] }, + "number": { "type": ["null", "string"] }, + "sequence_number": { "type": ["null", "integer"] }, + "issue_date": { "type": ["null", "string"], "format": "date" }, + "due_date": { "type": ["null", "string"], "format": "date" }, + "paid_date": { "type": ["null", "string"], "format": "date" }, + "status": { "type": ["null", "string"] }, + "collection_method": { "type": ["null", "string"] }, + "payment_instructions": { "type": ["null", "string"] }, + "currency": { "type": ["null", "string"] }, + "consolidation_level": { "type": ["null", "string"] }, + "parent_invoice_uid": { "type": ["null", "integer"] }, + "parent_invoice_number": { "type": ["null", "integer"] }, + "group_primary_subscription_id": { "type": ["null", "integer"] }, + "product_name": { "type": ["null", "string"] }, + "product_family_name": { "type": ["null", "string"] }, + "seller": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "name": { "type": ["null", "string"] }, + "address": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "street": { "type": ["null", "string"] }, + "line2": { "type": ["null", "string"] }, + "city": { "type": ["null", "string"] }, + "state": { "type": ["null", "string"] }, + "zip": { "type": ["null", "string"] }, + "country": { "type": ["null", "string"] } + } + }, + "phone": { "type": ["null", "string"] } + } + }, + "customer": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "chargify_id": { "type": ["null", "integer"] }, + "first_name": { "type": ["null", "string"] }, + "last_name": { "type": ["null", "string"] }, + "organization": { "type": ["null", "string"] }, + "email": { "type": ["null", "string"] } + } + }, + "memo": { "type": ["null", "string"] }, + "billing_address": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "street": { "type": ["null", "string"] }, + "line2": { "type": ["null", "string"] }, + "city": { "type": ["null", "string"] }, + "state": { "type": ["null", "string"] }, + "zip": { "type": ["null", "string"] }, + "country": { "type": ["null", "string"] } + } + }, + "shipping_address": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "street": { "type": ["null", "string"] }, + "line2": { "type": ["null", "string"] }, + "city": { "type": ["null", "string"] }, + "state": { "type": ["null", "string"] }, + "zip": { "type": ["null", "string"] }, + "country": { "type": ["null", "string"] } + } + }, + "subtotal_amount": { "type": ["null", "string"] }, + "discount_amount": { "type": ["null", "string"] }, + "tax_amount": { "type": ["null", "string"] }, + "total_amount": { "type": ["null", "string"] }, + "credit_amount": { "type": ["null", "string"] }, + "paid_amount": { "type": ["null", "string"] }, + "refund_amount": { "type": ["null", "string"] }, + "due_amount": { "type": ["null", "string"] } + } +} diff --git a/airbyte-integrations/connectors/source-chargify/source_chargify/schemas/subscriptions.json b/airbyte-integrations/connectors/source-chargify/source_chargify/schemas/subscriptions.json index 0a3c44a610d3..683601fbfa58 100644 --- a/airbyte-integrations/connectors/source-chargify/source_chargify/schemas/subscriptions.json +++ b/airbyte-integrations/connectors/source-chargify/source_chargify/schemas/subscriptions.json @@ -30,6 +30,24 @@ "trial_started_at": { "type": ["null", "string"] }, + "credit_balance_in_cents": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "locale": { + "type": ["null", "string"] + }, + "prepayment_balance_in_cents": { + "type": ["null", "integer"] + }, + "receives_invoice_emails": { + "type": ["null", "boolean"] + }, + "scheduled_cancellation_at": { + "type": ["null", "string"] + }, "trial_ended_at": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-chargify/source_chargify/schemas/transactions.json b/airbyte-integrations/connectors/source-chargify/source_chargify/schemas/transactions.json new file mode 100644 index 000000000000..2dbd4518e142 --- /dev/null +++ b/airbyte-integrations/connectors/source-chargify/source_chargify/schemas/transactions.json @@ -0,0 +1,34 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "integer"] }, + "subscription_id": { "type": ["null", "integer"] }, + "type": { "type": ["null", "string"] }, + "kind": { "type": ["null", "string"] }, + "transaction_type": { "type": ["null", "string"] }, + "success": { "type": ["null", "boolean"] }, + "amount_in_cents": { "type": ["null", "integer"] }, + "memo": { "type": ["null", "string"] }, + "created_at": { "type": ["null", "string"] }, + "starting_balance_in_cents": { "type": ["null", "integer"] }, + "ending_balance_in_cents": { "type": ["null", "integer"] }, + "gateway_used": { "type": ["null", "string"] }, + "gateway_transaction_id": { "type": ["null", "string"] }, + "gateway_order_id": { "type": ["null", "integer"] }, + "payment_id": { "type": ["null", "integer"] }, + "product_id": { "type": ["null", "integer"] }, + "tax_id": { "type": ["null", "integer"] }, + "component_id": { "type": ["null", "integer"] }, + "statement_id": { "type": ["null", "integer"] }, + "customer_id": { "type": ["null", "integer"] }, + "item_name": { "type": ["null", "string"] }, + "parent_id": { "type": ["null", "integer"] }, + "role": { "type": ["null", "string"] }, + "card_number": { "type": ["null", "string"] }, + "card_expiration": { "type": ["null", "string"] }, + "card_type": { "type": ["null", "string"] }, + "refunded_amount_in_cents": { "type": ["null", "integer"] } + } +} diff --git a/airbyte-integrations/connectors/source-chartmogul/Dockerfile b/airbyte-integrations/connectors/source-chartmogul/Dockerfile index 6d424e6b531c..2893028afcdd 100644 --- a/airbyte-integrations/connectors/source-chartmogul/Dockerfile +++ b/airbyte-integrations/connectors/source-chartmogul/Dockerfile @@ -34,5 +34,5 @@ COPY source_chartmogul ./source_chartmogul ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.1 +LABEL io.airbyte.version=1.0.0 LABEL io.airbyte.name=airbyte/source-chartmogul diff --git a/airbyte-integrations/connectors/source-chartmogul/README.md b/airbyte-integrations/connectors/source-chartmogul/README.md index 4eebdb5830cc..5e1706e3c818 100644 --- a/airbyte-integrations/connectors/source-chartmogul/README.md +++ b/airbyte-integrations/connectors/source-chartmogul/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-chartmogul:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/chartmogul) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_chartmogul/spec.json` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-chartmogul:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-chartmogul build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-chartmogul:airbyteDocker +An image will be built with the tag `airbyte/source-chartmogul:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-chartmogul:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-chartmogul:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-chartmogul:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-chartmogul:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-chartmogul test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-chartmogul:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-chartmogul:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-chartmogul test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/chartmogul.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-chartmogul/acceptance-test-config.yml b/airbyte-integrations/connectors/source-chartmogul/acceptance-test-config.yml index fc3a3aa9948d..aff79bf554f2 100644 --- a/airbyte-integrations/connectors/source-chartmogul/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-chartmogul/acceptance-test-config.yml @@ -4,27 +4,27 @@ connector_image: airbyte/source-chartmogul:dev acceptance_tests: spec: tests: - - spec_path: "source_chartmogul/spec.yaml" + - spec_path: "source_chartmogul/spec.yaml" connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: tests: - - config_path: "secrets/config.json" + - config_path: "secrets/config.json" basic_read: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: no - extra_records: yes - fail_on_extra_columns: false + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + expect_records: + path: "integration_tests/expected_records.jsonl" + extra_fields: no + exact_order: no + extra_records: yes + fail_on_extra_columns: false full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-chartmogul/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-chartmogul/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-chartmogul/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-chartmogul/build.gradle b/airbyte-integrations/connectors/source-chartmogul/build.gradle deleted file mode 100644 index adb6d7d7d3f4..000000000000 --- a/airbyte-integrations/connectors/source-chartmogul/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_chartmogul' -} diff --git a/airbyte-integrations/connectors/source-chartmogul/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-chartmogul/integration_tests/configured_catalog.json index 5181874363d7..e2c258fb237e 100644 --- a/airbyte-integrations/connectors/source-chartmogul/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-chartmogul/integration_tests/configured_catalog.json @@ -22,7 +22,37 @@ }, { "stream": { - "name": "customer_count", + "name": "customer_daily_count", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["date"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "customer_weekly_count", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["date"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "customer_monthly_count", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["date"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "customer_quarterly_count", "json_schema": {}, "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["date"]] diff --git a/airbyte-integrations/connectors/source-chartmogul/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-chartmogul/integration_tests/expected_records.jsonl index 1e62444bbc8b..0142a30c8799 100644 --- a/airbyte-integrations/connectors/source-chartmogul/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-chartmogul/integration_tests/expected_records.jsonl @@ -1,3 +1,3 @@ {"stream":"customers","data":{"id":72038151,"uuid":"cus_e6e6b1e6-72d4-11ec-ac99-47c8cf90d52c","external_id":"cus_0001","name":"Adam Smith","email":"integration-test@airbyte.io","status":"Past Due","customer-since":"2015-11-01T00:00:00+00:00","attributes":{"custom":{},"clearbit":{},"stripe":{},"tags":[]},"data_source_uuid":"ds_2dbe3b80-72d4-11ec-bdef-0f585abe5bf3","data_source_uuids":["ds_2dbe3b80-72d4-11ec-bdef-0f585abe5bf3"],"external_ids":["cus_0001"],"company":"Airbyte","country":"US","state":"CA","city":"San Francisco","zip":"94121","lead_created_at":"2022-01-11T00:00:00.000Z","free_trial_started_at":"2022-01-11T00:00:00.000Z","address":{"country":"United States","state":"California","city":"San Francisco","address_zip":"94121"},"mrr":4100,"arr":49200,"billing-system-url":null,"chartmogul-url":"https://app.chartmogul.com/#/customers/72038151-Airbyte","billing-system-type":"Import API","currency":"USD","currency-sign":"$", "owner": null},"emitted_at":1668512244585} {"stream":"activities","data":{"description":"purchased the Bronze Plan plan with $10.00 discount applied","activity-mrr-movement":4100,"activity-mrr":4100,"activity-arr":49200,"date":"2015-11-01T00:00:00+00:00","type":"new_biz","currency":"USD","subscription-external-id":"sub_0001","plan-external-id":"bb8dcfe0-5505-013a-5d44-263d116b0774","customer-name":"Airbyte","customer-uuid":"cus_e6e6b1e6-72d4-11ec-ac99-47c8cf90d52c","customer-external-id":"cus_0001","billing-connector-uuid":"ds_2dbe3b80-72d4-11ec-bdef-0f585abe5bf3","uuid":"02a72371-72b1-440e-9816-d61435fd2e45"},"emitted_at":1668512245112} -{"stream":"customer_count","data":{"date":"2022-01-01","customers":1,"percentage-change":0},"emitted_at":1668512245914} +{"stream":"customer_daily_count","data":{"date":"2022-01-01","customers":1,"percentage-change":0},"emitted_at":1668512245914} diff --git a/airbyte-integrations/connectors/source-chartmogul/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-chartmogul/integration_tests/invalid_config.json index da1c2443aeb4..22e8f78489bd 100644 --- a/airbyte-integrations/connectors/source-chartmogul/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-chartmogul/integration_tests/invalid_config.json @@ -1,5 +1,4 @@ { "api_key": "", - "start_date": "2017-01-25T00:00:00Z", - "interval": "day" + "start_date": "2017-01-25T00:00:00Z" } diff --git a/airbyte-integrations/connectors/source-chartmogul/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-chartmogul/integration_tests/sample_config.json index 7d1a7b8f198f..c09fca36f467 100644 --- a/airbyte-integrations/connectors/source-chartmogul/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-chartmogul/integration_tests/sample_config.json @@ -1,5 +1,4 @@ { "api_key": "", - "start_date": "2022-01-05T12:09:00Z", - "interval": "day" + "start_date": "2022-01-05T12:09:00Z" } diff --git a/airbyte-integrations/connectors/source-chartmogul/metadata.yaml b/airbyte-integrations/connectors/source-chartmogul/metadata.yaml index 6f4af3668c87..42e6a35f9b9f 100644 --- a/airbyte-integrations/connectors/source-chartmogul/metadata.yaml +++ b/airbyte-integrations/connectors/source-chartmogul/metadata.yaml @@ -1,12 +1,21 @@ data: + ab_internal: + ql: 300 + sl: 100 allowedHosts: hosts: - api.chartmogul.com connectorSubtype: api connectorType: source definitionId: b6604cbd-1b12-4c08-8767-e140d0fb0877 - dockerImageTag: 0.2.1 + dockerImageTag: 1.0.0 + releases: + breakingChanges: + 1.0.0: + message: "This version separates the `customer_count` stream into multiple streams (daily, weekly, monthly, quarterly). Users previously using the `customer_count` stream will need to run a reset to enable the new streams and continue syncing." + upgradeDeadline: "2023-11-29" dockerRepository: airbyte/source-chartmogul + documentationUrl: https://docs.airbyte.com/integrations/sources/chartmogul githubIssueLabel: source-chartmogul icon: chartmogul.svg license: MIT @@ -17,12 +26,7 @@ data: oss: enabled: true releaseStage: beta - documentationUrl: https://docs.airbyte.com/integrations/sources/chartmogul + supportLevel: community tags: - language:low-code - - language:python - ab_internal: - sl: 200 - ql: 300 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-chartmogul/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-chartmogul/sample_files/configured_catalog.json index 9ba89f7766d2..b937284b0780 100644 --- a/airbyte-integrations/connectors/source-chartmogul/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-chartmogul/sample_files/configured_catalog.json @@ -20,7 +20,34 @@ }, { "stream": { - "name": "customer_count", + "name": "customer_count_daily", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "customer_count_weekly", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "customer_count_monthly", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "customer_count_quarterly", "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, diff --git a/airbyte-integrations/connectors/source-chartmogul/setup.py b/airbyte-integrations/connectors/source-chartmogul/setup.py index 76bc706ad50c..fa0d73f436c9 100644 --- a/airbyte-integrations/connectors/source-chartmogul/setup.py +++ b/airbyte-integrations/connectors/source-chartmogul/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.10", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ diff --git a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/manifest.yaml b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/manifest.yaml index ba8aa4dacf4a..6f172d789cf4 100644 --- a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/manifest.yaml +++ b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/manifest.yaml @@ -1,7 +1,7 @@ -version: "0.29.0" +version: "0.51.0" definitions: - selector: + entries_selector: extractor: field_path: ["entries"] requester: @@ -12,9 +12,10 @@ definitions: username: "{{ config['api_key'] }}" retriever: record_selector: - $ref: "#/definitions/selector" + $ref: "#/definitions/entries_selector" requester: $ref: "#/definitions/requester" + customers_stream: retriever: $ref: "#/definitions/retriever" @@ -35,6 +36,7 @@ definitions: name: "customers" primary_key: "id" path: "/v1/customers" + activities_stream: retriever: $ref: "#/definitions/retriever" @@ -60,7 +62,8 @@ definitions: name: "activities" primary_key: "uuid" path: "/v1/activities" - customer_count_stream: + + customer_daily_count_stream: retriever: $ref: "#/definitions/retriever" requester: @@ -68,18 +71,61 @@ definitions: request_body_data: start-date: "{{ format_datetime(config['start_date'], '%Y-%m-%d') }}" end-date: "{{ now_utc().strftime('%Y-%m-%d') }}" - interval: "{{ config['interval'] }}" - paginator: - type: NoPagination + interval: day + $parameters: + name: "customer_daily_count" + primary_key: "date" + path: "/v1/metrics/customer-count" + + customer_weekly_count_stream: + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + request_body_data: + start-date: "{{ format_datetime(config['start_date'], '%Y-%m-%d') }}" + end-date: "{{ now_utc().strftime('%Y-%m-%d') }}" + interval: week + $parameters: + name: "customer_weekly_count" + primary_key: "date" + path: "/v1/metrics/customer-count" + + customer_monthly_count_stream: + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + request_body_data: + start-date: "{{ format_datetime(config['start_date'], '%Y-%m-%d') }}" + end-date: "{{ now_utc().strftime('%Y-%m-%d') }}" + interval: month + $parameters: + name: "customer_monthly_count" + primary_key: "date" + path: "/v1/metrics/customer-count" + + customer_quarterly_count_stream: + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + request_body_data: + start-date: "{{ format_datetime(config['start_date'], '%Y-%m-%d') }}" + end-date: "{{ now_utc().strftime('%Y-%m-%d') }}" + interval: quarter $parameters: - name: "customer_count" + name: "customer_quarterly_count" primary_key: "date" path: "/v1/metrics/customer-count" streams: - "#/definitions/customers_stream" - "#/definitions/activities_stream" - - "#/definitions/customer_count_stream" + - "#/definitions/customer_daily_count_stream" + - "#/definitions/customer_weekly_count_stream" + - "#/definitions/customer_monthly_count_stream" + - "#/definitions/customer_quarterly_count_stream" check: stream_names: diff --git a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/activities.json b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/activities.json index 6cf82203be20..f4a3dc2294d7 100644 --- a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/activities.json +++ b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/activities.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "description": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/customer_count.json b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/customer_count.json deleted file mode 100644 index 64382e2ee087..000000000000 --- a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/customer_count.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "date": { - "type": ["string"] - }, - "customers": { - "type": ["integer"] - } - } -} diff --git a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/customer_daily_count.json b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/customer_daily_count.json new file mode 100644 index 000000000000..7da418c66cb0 --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/customer_daily_count.json @@ -0,0 +1,13 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "date": { + "type": ["string"] + }, + "customers": { + "type": ["integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/customer_monthly_count.json b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/customer_monthly_count.json new file mode 100644 index 000000000000..7da418c66cb0 --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/customer_monthly_count.json @@ -0,0 +1,13 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "date": { + "type": ["string"] + }, + "customers": { + "type": ["integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/customer_quarterly_count.json b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/customer_quarterly_count.json new file mode 100644 index 000000000000..7da418c66cb0 --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/customer_quarterly_count.json @@ -0,0 +1,13 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "date": { + "type": ["string"] + }, + "customers": { + "type": ["integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/customer_weekly_count.json b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/customer_weekly_count.json new file mode 100644 index 000000000000..7da418c66cb0 --- /dev/null +++ b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/schemas/customer_weekly_count.json @@ -0,0 +1,13 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "date": { + "type": ["string"] + }, + "customers": { + "type": ["integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/spec.yaml b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/spec.yaml index b6645d3e450d..1c1f5f38febf 100644 --- a/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/spec.yaml +++ b/airbyte-integrations/connectors/source-chartmogul/source_chartmogul/spec.yaml @@ -6,7 +6,6 @@ connectionSpecification: required: - api_key - start_date - - interval properties: api_key: type: string @@ -22,10 +21,3 @@ connectionSpecification: examples: ["2017-01-25T00:00:00Z"] order: 1 format: date-time - interval: - type: string - title: "Interval" - description: 'Some APIs such as Metrics require intervals to cluster data.' - enum: ["day", "week", "month", "quarter"] - default: "month" - order: 2 diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/.dockerignore b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile deleted file mode 100644 index 626c34f06ff1..000000000000 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-clickhouse-strict-encrypt - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-clickhouse-strict-encrypt - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.17 -LABEL io.airbyte.name=airbyte/source-clickhouse-strict-encrypt diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/ReadMe.md b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/ReadMe.md index f924484ee4b1..26ba470a99f5 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/ReadMe.md +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/ReadMe.md @@ -6,4 +6,4 @@ This connector inherits the Clickhouse source, but support SSL connections only. # Integration tests For ssl test custom image is used. To push it run this command under the tools\integration-tests-ssl dir: -*docker build -t your_user/clickhouse-with-ssl:dev -f Clickhouse.Dockerfile .* \ No newline at end of file +*docker build -t your_user/clickhouse-with-ssl:dev -f Clickhouse.Dockerfile .* diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/acceptance-test-config.yml b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/acceptance-test-config.yml deleted file mode 100644 index c4d71fd3e5b9..000000000000 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/acceptance-test-config.yml +++ /dev/null @@ -1,6 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-clickhouse-strict-encrypt:dev -tests: - spec: - - spec_path: "src/test-integration/resources/expected_spec.json" diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/build.gradle index f1d00512e126..504f7e638ae7 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/build.gradle @@ -1,28 +1,33 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.7.7' + features = ['db-sources'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.source.clickhouse.ClickHouseStrictEncryptSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation project(':airbyte-integrations:connectors:source-relational-db') implementation project(':airbyte-integrations:connectors:source-clickhouse') - implementation libs.airbyte.protocol - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation group: 'com.clickhouse', name: 'clickhouse-jdbc', version: '0.3.2-patch9' - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-clickhouse') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-clickhouse-strict-encrypt') - integrationTestJavaImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) - integrationTestJavaImplementation libs.connectors.source.testcontainers.clickhouse + integrationTestJavaImplementation libs.testcontainers.clickhouse } diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/metadata.yaml index beeb8dd90751..34e595a3063a 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/metadata.yaml @@ -11,7 +11,7 @@ data: connectorSubtype: database connectorType: source definitionId: bad83517-5e54-4a3d-9b53-63e85fbd4d7c - dockerImageTag: 0.1.17 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-clickhouse-strict-encrypt githubIssueLabel: source-clickhouse icon: clickhouse.svg diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseStrictEncryptSource.java b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseStrictEncryptSource.java index 57b42282bd82..9aebbe95c38c 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseStrictEncryptSource.java +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseStrictEncryptSource.java @@ -5,11 +5,11 @@ package io.airbyte.integrations.source.clickhouse; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.base.spec_modification.SpecModifyingSource; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.base.spec_modification.SpecModifyingSource; import io.airbyte.protocol.models.v0.ConnectorSpecification; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java index ae879635d4fa..63a644f1fd1a 100644 --- a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptJdbcSourceAcceptanceTest.java @@ -4,54 +4,50 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; +import static io.airbyte.integrations.io.airbyte.integration_tests.sources.ClickHouseStrictEncryptTestDatabase.DEFAULT_DB_NAME; +import static io.airbyte.integrations.io.airbyte.integration_tests.sources.ClickHouseStrictEncryptTestDatabase.HTTPS_PORT; import static java.time.temporal.ChronoUnit.SECONDS; import static org.junit.Assert.assertEquals; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.base.ssh.SshHelpers; +import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.commons.string.Strings; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.source.clickhouse.ClickHouseSource; import io.airbyte.integrations.source.clickhouse.ClickHouseStrictEncryptSource; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; -import io.airbyte.integrations.util.HostPortResolver; import io.airbyte.protocol.models.v0.ConnectorSpecification; -import java.sql.JDBCType; import java.time.Duration; import java.util.List; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +import org.testcontainers.clickhouse.ClickHouseContainer; import org.testcontainers.containers.BindMode; -import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.images.builder.ImageFromDockerfile; +import org.testcontainers.utility.MountableFile; -public class ClickHouseStrictEncryptJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { +@Disabled +public class ClickHouseStrictEncryptJdbcSourceAcceptanceTest + extends JdbcSourceAcceptanceTest { public static final Integer HTTP_PORT = 8123; public static final Integer NATIVE_PORT = 9000; - public static final Integer HTTPS_PORT = 8443; + public static final Integer NATIVE_SECURE_PORT = 9440; - private static final String DEFAULT_DB_NAME = "default"; - private static final String DEFAULT_USER_NAME = "default"; - private static GenericContainer container; - private static JdbcDatabase db; - private JsonNode config; - private String dbName; + @Override + protected ClickHouseStrictEncryptTestDatabase createTestDatabase() { + final ClickHouseContainer db = new ClickHouseContainer("clickhouse/clickhouse-server:22.5") + .withEnv("TZ", "UTC") + .withExposedPorts(HTTP_PORT, NATIVE_PORT, HTTPS_PORT, NATIVE_SECURE_PORT) + .withCopyFileToContainer(MountableFile.forClasspathResource("/docker/clickhouse_certs.sh"), + "/docker-entrypoint-initdb.d/clickhouse_certs.sh") + .withClasspathResourceMapping("ssl_ports.xml", "/etc/clickhouse-server/config.d/ssl_ports.xml", BindMode.READ_ONLY) + .waitingFor(Wait.forHttp("/ping").forPort(HTTP_PORT) + .forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS))); + db.start(); + return new ClickHouseStrictEncryptTestDatabase(db).initialized(); + } @Override public boolean supportsSchemas() { @@ -59,13 +55,13 @@ public boolean supportsSchemas() { } @Override - public JsonNode getConfig() { - return Jsons.clone(config); + public JsonNode config() { + return Jsons.clone(testdb.configBuilder().build()); } @Override - public String getDriverClass() { - return ClickHouseSource.DRIVER_CLASS; + protected ClickHouseStrictEncryptSource source() { + return new ClickHouseStrictEncryptSource(); } @Override @@ -75,7 +71,7 @@ public String createTableQuery(final String tableName, // ClickHouse requires Engine to be mentioned as part of create table query. // Refer : https://clickhouse.tech/docs/en/engines/table-engines/ for more information return String.format("CREATE TABLE %s(%s) %s", - dbName + "." + tableName, columnClause, primaryKeyClause.equals("") ? "Engine = TinyLog" + DEFAULT_DB_NAME + "." + tableName, columnClause, primaryKeyClause.equals("") ? "Engine = TinyLog" : "ENGINE = MergeTree() ORDER BY " + primaryKeyClause + " PRIMARY KEY " + primaryKeyClause); } @@ -83,60 +79,9 @@ public String createTableQuery(final String tableName, @BeforeAll static void init() { CREATE_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s Array(UInt32)) ENGINE = MergeTree ORDER BY tuple();"; - INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES([12, 13, 0, 1]);)"; + INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES([12, 13, 0, 1]);"; CREATE_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s Nullable(VARCHAR(20))) ENGINE = MergeTree ORDER BY tuple();"; INSERT_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES('Hello world :)');"; - - container = new GenericContainer<>(new ImageFromDockerfile("clickhouse-test") - .withFileFromClasspath("Dockerfile", "docker/Dockerfile") - .withFileFromClasspath("clickhouse_certs.sh", "docker/clickhouse_certs.sh")) - .withEnv("TZ", "UTC") - .withExposedPorts(HTTP_PORT, NATIVE_PORT, HTTPS_PORT, NATIVE_SECURE_PORT) - .withClasspathResourceMapping("ssl_ports.xml", "/etc/clickhouse-server/config.d/ssl_ports.xml", BindMode.READ_ONLY) - .waitingFor(Wait.forHttp("/ping").forPort(HTTP_PORT) - .forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS))); - container.start(); - } - - @BeforeEach - public void setup() throws Exception { - final JsonNode configWithoutDbName = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveIpAddress(container)) - .put(JdbcUtils.PORT_KEY, HTTPS_PORT) - .put(JdbcUtils.USERNAME_KEY, DEFAULT_USER_NAME) - .put("database", DEFAULT_DB_NAME) - .put(JdbcUtils.PASSWORD_KEY, "") - .build()); - - db = new DefaultJdbcDatabase( - DataSourceFactory.create( - configWithoutDbName.get(JdbcUtils.USERNAME_KEY).asText(), - configWithoutDbName.get(JdbcUtils.PASSWORD_KEY).asText(), - ClickHouseSource.DRIVER_CLASS, - String.format(DatabaseDriver.CLICKHOUSE.getUrlFormatString() + "?sslmode=none", - ClickHouseSource.HTTPS_PROTOCOL, - configWithoutDbName.get(JdbcUtils.HOST_KEY).asText(), - configWithoutDbName.get(JdbcUtils.PORT_KEY).asInt(), - configWithoutDbName.get("database").asText()))); - - dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); - - db.execute(ctx -> ctx.createStatement().execute(String.format("CREATE DATABASE %s;", dbName))); - config = Jsons.clone(configWithoutDbName); - ((ObjectNode) config).put(JdbcUtils.DATABASE_KEY, dbName); - - super.setup(); - } - - @AfterEach - public void tearDownMySql() throws Exception { - db.execute(ctx -> ctx.createStatement().execute(String.format("DROP DATABASE %s;", dbName))); - super.tearDown(); - } - - @AfterAll - public static void cleanUp() throws Exception { - container.close(); } @Override @@ -157,19 +102,9 @@ public String primaryKeyClause(final List columns) { return clause.toString(); } - @Override - public AbstractJdbcSource getJdbcSource() { - return new ClickHouseSource(); - } - - @Override - public Source getSource() { - return new ClickHouseStrictEncryptSource(); - } - @Test void testSpec() throws Exception { - final ConnectorSpecification actual = source.spec(); + final ConnectorSpecification actual = source().spec(); final ConnectorSpecification expected = SshHelpers.injectSshIntoSpec(Jsons.deserialize(MoreResources.readResource("expected_spec.json"), ConnectorSpecification.class)); diff --git a/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptTestDatabase.java b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptTestDatabase.java new file mode 100644 index 000000000000..ffadf1685ca7 --- /dev/null +++ b/airbyte-integrations/connectors/source-clickhouse-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseStrictEncryptTestDatabase.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.TestDatabase; +import java.util.stream.Stream; +import org.jooq.SQLDialect; +import org.testcontainers.clickhouse.ClickHouseContainer; + +public class ClickHouseStrictEncryptTestDatabase extends + TestDatabase { + + private static final String SCHEMA_NAME = "default"; + public static final Integer HTTPS_PORT = 8443; + public static final String DEFAULT_DB_NAME = "default"; + private static final String DEFAULT_USER_NAME = "default"; + private final ClickHouseContainer container; + + protected ClickHouseStrictEncryptTestDatabase(final ClickHouseContainer container) { + super(container); + this.container = container; + } + + @Override + public String getJdbcUrl() { + return container.getJdbcUrl(); + } + + @Override + public String getUserName() { + return container.getUsername(); + } + + @Override + public String getPassword() { + return container.getPassword(); + } + + @Override + public String getDatabaseName() { + return SCHEMA_NAME; + } + + @Override + public ClickHouseConfigBuilder configBuilder() { + return new ClickHouseConfigBuilder(this) + .with(JdbcUtils.HOST_KEY, container.getHost()) + .with(JdbcUtils.PORT_KEY, container.getMappedPort(HTTPS_PORT)) + .with(JdbcUtils.USERNAME_KEY, DEFAULT_USER_NAME) + .with(JdbcUtils.DATABASE_KEY, DEFAULT_DB_NAME) + .with(JdbcUtils.PASSWORD_KEY, ""); + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.empty(); + } + + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return DatabaseDriver.CLICKHOUSE; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.DEFAULT; + } + + @Override + public void close() { + container.close(); + } + + @Override + public ClickHouseConfigBuilder integrationTestConfigBuilder() { + return super.integrationTestConfigBuilder(); + } + + static public class ClickHouseConfigBuilder extends ConfigBuilder { + + protected ClickHouseConfigBuilder(final ClickHouseStrictEncryptTestDatabase testdb) { + super(testdb); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-clickhouse/.dockerignore b/airbyte-integrations/connectors/source-clickhouse/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-clickhouse/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-clickhouse/Dockerfile b/airbyte-integrations/connectors/source-clickhouse/Dockerfile deleted file mode 100644 index 00af2bdfb107..000000000000 --- a/airbyte-integrations/connectors/source-clickhouse/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-clickhouse - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-clickhouse - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.17 -LABEL io.airbyte.name=airbyte/source-clickhouse diff --git a/airbyte-integrations/connectors/source-clickhouse/ReadMe.md b/airbyte-integrations/connectors/source-clickhouse/ReadMe.md index 3c38c069c587..c0e976415f24 100644 --- a/airbyte-integrations/connectors/source-clickhouse/ReadMe.md +++ b/airbyte-integrations/connectors/source-clickhouse/ReadMe.md @@ -1,3 +1,3 @@ # Integration tests For ssl test custom image is used. To push it run this command under the tools\integration-tests-ssl dir: -*docker build -t your_user/clickhouse-with-ssl:dev -f Clickhouse.Dockerfile .* \ No newline at end of file +*docker build -t your_user/clickhouse-with-ssl:dev -f Clickhouse.Dockerfile .* diff --git a/airbyte-integrations/connectors/source-clickhouse/acceptance-test-config.yml b/airbyte-integrations/connectors/source-clickhouse/acceptance-test-config.yml deleted file mode 100644 index a329f4e6b0db..000000000000 --- a/airbyte-integrations/connectors/source-clickhouse/acceptance-test-config.yml +++ /dev/null @@ -1,7 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-clickhouse:dev -tests: - spec: - - spec_path: "src/test-integration/resources/expected_spec.json" - config_path: "src/test-integration/resources/dummy_config.json" diff --git a/airbyte-integrations/connectors/source-clickhouse/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-clickhouse/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-clickhouse/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-clickhouse/build.gradle b/airbyte-integrations/connectors/source-clickhouse/build.gradle index 5e9bb7f0f79c..314496458585 100644 --- a/airbyte-integrations/connectors/source-clickhouse/build.gradle +++ b/airbyte-integrations/connectors/source-clickhouse/build.gradle @@ -1,27 +1,31 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' - id 'airbyte-connector-acceptance-test' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.7.7' + features = ['db-sources'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.source.clickhouse.ClickHouseSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation project(':airbyte-integrations:connectors:source-relational-db') - implementation libs.airbyte.protocol - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation 'com.clickhouse:clickhouse-jdbc:0.3.2-patch10:all' - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-clickhouse') - integrationTestJavaImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) - integrationTestJavaImplementation libs.connectors.source.testcontainers.clickhouse + integrationTestJavaImplementation libs.testcontainers.clickhouse } diff --git a/airbyte-integrations/connectors/source-clickhouse/metadata.yaml b/airbyte-integrations/connectors/source-clickhouse/metadata.yaml index 07806c1237f3..ddf6f9121fc4 100644 --- a/airbyte-integrations/connectors/source-clickhouse/metadata.yaml +++ b/airbyte-integrations/connectors/source-clickhouse/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: bad83517-5e54-4a3d-9b53-63e85fbd4d7c - dockerImageTag: 0.1.17 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-clickhouse documentationUrl: https://docs.airbyte.com/integrations/sources/clickhouse githubIssueLabel: source-clickhouse @@ -18,7 +18,7 @@ data: name: ClickHouse registries: cloud: - dockerImageTag: 0.1.8 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-clickhouse-strict-encrypt enabled: true oss: diff --git a/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java b/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java index c89f66a1c509..1ba9be607e2f 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/main/java/io/airbyte/integrations/source/clickhouse/ClickHouseSource.java @@ -6,16 +6,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.jdbc.streaming.NoOpStreamingQueryConfig; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.base.ssh.SshWrappedSource; +import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.cdk.integrations.source.relationaldb.TableInfo; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.streaming.NoOpStreamingQueryConfig; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.base.ssh.SshWrappedSource; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.relationaldb.TableInfo; import io.airbyte.protocol.models.CommonField; import java.sql.JDBCType; import java.sql.PreparedStatement; diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java index ba86cd2e596a..9e7ad5a46345 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshClickHouseSourceAcceptanceTest.java @@ -8,18 +8,18 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.ssh.SshBastionContainer; +import io.airbyte.cdk.integrations.base.ssh.SshHelpers; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.ssh.SshBastionContainer; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.base.ssh.SshTunnel; import io.airbyte.integrations.source.clickhouse.ClickHouseSource; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.CatalogHelpers; @@ -31,10 +31,12 @@ import java.time.Duration; import java.util.HashMap; import javax.sql.DataSource; +import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.ClickHouseContainer; import org.testcontainers.containers.Network; import org.testcontainers.containers.wait.strategy.Wait; +@Disabled public abstract class AbstractSshClickHouseSourceAcceptanceTest extends SourceAcceptanceTest { private ClickHouseContainer db; diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java index 255f85e33f32..cee17bd1b6db 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcSourceAcceptanceTest.java @@ -4,36 +4,34 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; -import static io.airbyte.db.jdbc.JdbcUtils.JDBC_URL_KEY; +import static io.airbyte.cdk.db.jdbc.JdbcUtils.JDBC_URL_KEY; import static io.airbyte.integrations.source.clickhouse.ClickHouseSource.SSL_MODE; import static java.time.temporal.ChronoUnit.SECONDS; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.source.clickhouse.ClickHouseSource; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; -import io.airbyte.integrations.util.HostPortResolver; -import java.sql.JDBCType; -import java.sql.SQLException; import java.time.Duration; import java.util.List; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import org.testcontainers.containers.ClickHouseContainer; +import org.testcontainers.clickhouse.ClickHouseContainer; import org.testcontainers.containers.wait.strategy.Wait; -public class ClickHouseJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { +@Disabled +public class ClickHouseJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { - private static final String SCHEMA_NAME = "default"; - private ClickHouseContainer db; - private JsonNode config; + @BeforeAll + static void init() { + CREATE_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s Array(UInt32)) ENGINE = MergeTree ORDER BY tuple();"; + INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES([12, 13, 0, 1]);"; + CREATE_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s Nullable(VARCHAR(20))) ENGINE = MergeTree ORDER BY tuple();"; + INSERT_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES('Hello world :)');"; + } @Override public boolean supportsSchemas() { @@ -41,13 +39,17 @@ public boolean supportsSchemas() { } @Override - public JsonNode getConfig() { - return Jsons.clone(config); + protected JsonNode config() { + return Jsons.clone(testdb.configBuilder().build()); } @Override - public String getDriverClass() { - return ClickHouseSource.DRIVER_CLASS; + protected ClickHouseTestDatabase createTestDatabase() { + final ClickHouseContainer db = new ClickHouseContainer("clickhouse/clickhouse-server:22.5") + .waitingFor(Wait.forHttp("/ping").forPort(8123) + .forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS))); + db.start(); + return new ClickHouseTestDatabase(db).initialized(); } @Override @@ -60,22 +62,6 @@ public String createTableQuery(final String tableName, final String columnClause + primaryKeyClause); } - @BeforeAll - static void init() { - CREATE_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s Array(UInt32)) ENGINE = MergeTree ORDER BY tuple();"; - INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES([12, 13, 0, 1]);)"; - CREATE_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s Nullable(VARCHAR(20))) ENGINE = MergeTree ORDER BY tuple();"; - INSERT_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES('Hello world :)');"; - } - - @Override - @AfterEach - public void tearDown() throws SQLException { - db.close(); - db.stop(); - super.tearDown(); - } - @Override public String primaryKeyClause(final List columns) { if (columns.isEmpty()) { @@ -95,27 +81,7 @@ public String primaryKeyClause(final List columns) { } @Override - @BeforeEach - public void setup() throws Exception { - db = new ClickHouseContainer("clickhouse/clickhouse-server:22.5") - .waitingFor(Wait.forHttp("/ping").forPort(8123) - .forStatusCode(200).withStartupTimeout(Duration.of(60, SECONDS))); - db.start(); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(db)) - .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(db)) - .put(JdbcUtils.DATABASE_KEY, SCHEMA_NAME) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) - .put(JdbcUtils.SSL_KEY, false) - .build()); - - super.setup(); - } - - @Override - public AbstractJdbcSource getJdbcSource() { + protected ClickHouseSource source() { return new ClickHouseSource(); } diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcStressTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcStressTest.java index e3d74f620e97..27fa03ff7eb2 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcStressTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseJdbcStressTest.java @@ -8,12 +8,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.cdk.integrations.source.jdbc.test.JdbcStressTest; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.source.clickhouse.ClickHouseSource; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.test.JdbcStressTest; -import io.airbyte.integrations.util.HostPortResolver; import java.sql.JDBCType; import java.time.Duration; import java.util.Optional; diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java index 615be3c952b8..69950397b24d 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseSourceAcceptanceTest.java @@ -9,16 +9,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.ssh.SshHelpers; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.ssh.SshHelpers; import io.airbyte.integrations.source.clickhouse.ClickHouseSource; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.CatalogHelpers; @@ -30,9 +30,11 @@ import java.time.Duration; import java.util.HashMap; import javax.sql.DataSource; +import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.ClickHouseContainer; import org.testcontainers.containers.wait.strategy.Wait; +@Disabled public class ClickHouseSourceAcceptanceTest extends SourceAcceptanceTest { private ClickHouseContainer db; diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseTestDatabase.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseTestDatabase.java new file mode 100644 index 000000000000..35bc1595e6ff --- /dev/null +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/ClickHouseTestDatabase.java @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.TestDatabase; +import java.util.stream.Stream; +import org.jooq.SQLDialect; +import org.testcontainers.clickhouse.ClickHouseContainer; + +public class ClickHouseTestDatabase extends + TestDatabase { + + private static final String SCHEMA_NAME = "default"; + + private final ClickHouseContainer container; + + protected ClickHouseTestDatabase(final ClickHouseContainer container) { + super(container); + this.container = container; + } + + @Override + public String getJdbcUrl() { + return container.getJdbcUrl(); + } + + @Override + public String getUserName() { + return container.getUsername(); + } + + @Override + public String getPassword() { + return container.getPassword(); + } + + @Override + public String getDatabaseName() { + return SCHEMA_NAME; + } + + @Override + public ClickHouseConfigBuilder configBuilder() { + return new ClickHouseConfigBuilder(this) + .with(JdbcUtils.HOST_KEY, container.getHost()) + .with(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) + .with(JdbcUtils.DATABASE_KEY, SCHEMA_NAME) + .with(JdbcUtils.USERNAME_KEY, container.getUsername()) + .with(JdbcUtils.PASSWORD_KEY, container.getPassword()) + .with(JdbcUtils.SSL_KEY, false); + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.empty(); + } + + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return DatabaseDriver.CLICKHOUSE; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.DEFAULT; + } + + @Override + public void close() { + container.close(); + } + + @Override + public ClickHouseConfigBuilder integrationTestConfigBuilder() { + return super.integrationTestConfigBuilder(); + } + + static public class ClickHouseConfigBuilder extends TestDatabase.ConfigBuilder { + + protected ClickHouseConfigBuilder(final ClickHouseTestDatabase testdb) { + super(testdb); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshKeyClickhouseSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshKeyClickhouseSourceAcceptanceTest.java index 1a92423e9e14..ebcb3f632c67 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshKeyClickhouseSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshKeyClickhouseSourceAcceptanceTest.java @@ -4,8 +4,10 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; -import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import org.junit.jupiter.api.Disabled; +@Disabled public class SshKeyClickhouseSourceAcceptanceTest extends AbstractSshClickHouseSourceAcceptanceTest { diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordClickhouseSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordClickhouseSourceAcceptanceTest.java index 08ca2475a364..7223031d1735 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordClickhouseSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordClickhouseSourceAcceptanceTest.java @@ -4,8 +4,10 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; -import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import org.junit.jupiter.api.Disabled; +@Disabled public class SshPasswordClickhouseSourceAcceptanceTest extends AbstractSshClickHouseSourceAcceptanceTest { diff --git a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java index 3fbfcb12bc10..1359111fca8a 100644 --- a/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-clickhouse/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SslClickHouseJdbcSourceAcceptanceTest.java @@ -7,20 +7,22 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.source.clickhouse.ClickHouseSource; import javax.sql.DataSource; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.GenericContainer; +@Disabled public class SslClickHouseJdbcSourceAcceptanceTest extends ClickHouseJdbcSourceAcceptanceTest { private static GenericContainer container; diff --git a/airbyte-integrations/connectors/source-clickup-api/README.md b/airbyte-integrations/connectors/source-clickup-api/README.md index c02b885876d3..155ef6c59a2c 100644 --- a/airbyte-integrations/connectors/source-clickup-api/README.md +++ b/airbyte-integrations/connectors/source-clickup-api/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-clickup-api:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/clickup-api) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_clickup_api/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-clickup-api:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-clickup-api build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-clickup-api:airbyteDocker +An image will be built with the tag `airbyte/source-clickup-api:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-clickup-api:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-clickup-api:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-clickup-api:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-clickup-api:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-clickup-api test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-clickup-api:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-clickup-api:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-clickup-api test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/clickup-api.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-clickup-api/acceptance-test-config.yml b/airbyte-integrations/connectors/source-clickup-api/acceptance-test-config.yml index 05e39a97c4e1..1ef5060a67b1 100644 --- a/airbyte-integrations/connectors/source-clickup-api/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-clickup-api/acceptance-test-config.yml @@ -1,31 +1,31 @@ acceptance_tests: basic_read: tests: - - config_path: secrets/config.json - empty_streams: [] + - config_path: secrets/config.json + empty_streams: [] connection: tests: - - config_path: secrets/config.json - status: succeed - - config_path: integration_tests/invalid_config.json - status: failed + - config_path: secrets/config.json + status: succeed + - config_path: integration_tests/invalid_config.json + status: failed discovery: tests: - - config_path: secrets/config.json + - config_path: secrets/config.json full_refresh: tests: - - config_path: secrets/config.json - configured_catalog_path: integration_tests/configured_catalog.json - ignored_fields: - team: - - name: members - bypass_reason: ignore changing value in full_refresh of acceptance-tests - - name: last_active - bypass_reason: ignore changing value in full_refresh of acceptance-tests + - config_path: secrets/config.json + configured_catalog_path: integration_tests/configured_catalog.json + ignored_fields: + team: + - name: members + bypass_reason: ignore changing value in full_refresh of acceptance-tests + - name: last_active + bypass_reason: ignore changing value in full_refresh of acceptance-tests spec: tests: - - spec_path: source_clickup_api/spec.yaml - timeout_seconds: 1200 + - spec_path: source_clickup_api/spec.yaml + timeout_seconds: 1200 incremental: bypass_reason: "Incremental syncs are not supported on this connector." connector_image: airbyte/source-clickup-api:dev diff --git a/airbyte-integrations/connectors/source-clickup-api/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-clickup-api/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-clickup-api/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-clickup-api/build.gradle b/airbyte-integrations/connectors/source-clickup-api/build.gradle deleted file mode 100644 index 211412e41195..000000000000 --- a/airbyte-integrations/connectors/source-clickup-api/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_clickup_api' -} diff --git a/airbyte-integrations/connectors/source-clockify/.gitignore b/airbyte-integrations/connectors/source-clockify/.gitignore deleted file mode 100644 index 945168c8f81b..000000000000 --- a/airbyte-integrations/connectors/source-clockify/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -users.yml -projects.yml -schemas diff --git a/airbyte-integrations/connectors/source-clockify/Dockerfile b/airbyte-integrations/connectors/source-clockify/Dockerfile index 4a69b4505941..3c34733940c1 100644 --- a/airbyte-integrations/connectors/source-clockify/Dockerfile +++ b/airbyte-integrations/connectors/source-clockify/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.13-alpine3.15 as base +FROM python:3.9.11-alpine3.15 as base # build and load all requirements FROM base as builder @@ -34,5 +34,5 @@ COPY source_clockify ./source_clockify ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.1 +LABEL io.airbyte.version=0.3.0 LABEL io.airbyte.name=airbyte/source-clockify diff --git a/airbyte-integrations/connectors/source-clockify/README.md b/airbyte-integrations/connectors/source-clockify/README.md index 1a464e4dd95f..d3581477fc51 100644 --- a/airbyte-integrations/connectors/source-clockify/README.md +++ b/airbyte-integrations/connectors/source-clockify/README.md @@ -1,45 +1,12 @@ # Clockify Source -This is the repository for the Clockify source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/clockify). +This is the repository for the Clockify configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/clockify). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-clockify:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/clockify) +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/clockify) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_clockify/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. @@ -47,28 +14,21 @@ See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source clockify test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-clockify:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-clockify build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-clockify:airbyteDocker +An image will be built with the tag `airbyte/source-clockify:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-clockify:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-clockify:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-clockify:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-clockify:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-clockify test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-clockify:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-clockify:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-clockify test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/clockify.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-clockify/__init__.py b/airbyte-integrations/connectors/source-clockify/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-clockify/acceptance-test-config.yml b/airbyte-integrations/connectors/source-clockify/acceptance-test-config.yml index b1555d69f8e3..326def29cf01 100644 --- a/airbyte-integrations/connectors/source-clockify/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-clockify/acceptance-test-config.yml @@ -1,10 +1,8 @@ -# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) -# for more information about how to configure these tests connector_image: airbyte/source-clockify:dev acceptance_tests: spec: tests: - - spec_path: "source_clockify/spec.json" + - spec_path: "source_clockify/spec.yaml" connection: tests: - config_path: "secrets/config.json" @@ -15,12 +13,27 @@ acceptance_tests: tests: - config_path: "secrets/config.json" backward_compatibility_tests_config: - previous_connector_version: "0.2.0" - disable_for_version: "0.2.0" + previous_connector_version: "0.2.1" + disable_for_version: "0.2.1" basic_read: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.jsonl" + # extra_fields: no + # exact_order: no + # extra_records: yes + incremental: + bypass_reason: "This connector does not implement incremental sync" + # TODO uncomment this block this block if your connector implements incremental sync: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-clockify/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-clockify/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-clockify/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-clockify/build.gradle b/airbyte-integrations/connectors/source-clockify/build.gradle deleted file mode 100644 index cbfa98aee8a7..000000000000 --- a/airbyte-integrations/connectors/source-clockify/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_clockify' -} diff --git a/airbyte-integrations/connectors/source-clockify/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-clockify/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-clockify/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-clockify/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-clockify/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-clockify/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-clockify/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-clockify/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-clockify/metadata.yaml b/airbyte-integrations/connectors/source-clockify/metadata.yaml index f0425dc400cb..90baae2689d8 100644 --- a/airbyte-integrations/connectors/source-clockify/metadata.yaml +++ b/airbyte-integrations/connectors/source-clockify/metadata.yaml @@ -1,24 +1,28 @@ data: + allowedHosts: + hosts: + - api.clockify.me + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: e71aae8a-5143-11ed-bdc3-0242ac120002 - dockerImageTag: 0.2.1 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-clockify githubIssueLabel: source-clockify icon: clockify.svg license: MIT name: Clockify - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: 2023-08-27 releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/clockify tags: - - language:python + - language:low-code ab_internal: sl: 100 ql: 100 - supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-clockify/requirements.txt b/airbyte-integrations/connectors/source-clockify/requirements.txt index d6e1198b1ab1..cc57334ef619 100644 --- a/airbyte-integrations/connectors/source-clockify/requirements.txt +++ b/airbyte-integrations/connectors/source-clockify/requirements.txt @@ -1 +1,2 @@ +-e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-clockify/setup.py b/airbyte-integrations/connectors/source-clockify/setup.py index 5836e4cbb175..bcd38b28c29a 100644 --- a/airbyte-integrations/connectors/source-clockify/setup.py +++ b/airbyte-integrations/connectors/source-clockify/setup.py @@ -6,10 +6,15 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", + "airbyte-cdk~=0.1", ] -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", "responses"] +TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest~=6.2", + "pytest-mock~=3.6.1", + "connector-acceptance-test", +] setup( name="source_clockify", diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/manifest.yaml b/airbyte-integrations/connectors/source-clockify/source_clockify/manifest.yaml new file mode 100644 index 000000000000..eb8c29b1674a --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/manifest.yaml @@ -0,0 +1,133 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + + requester: + type: HttpRequester + url_base: "https://api.clockify.me/api/v1/" + http_method: "GET" + authenticator: + type: "ApiKeyAuthenticator" + header: "X-Api-Key" + api_token: "{{ config['api_key'] }}" + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: "DefaultPaginator" + page_size_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "page-size" + pagination_strategy: + type: "PageIncrement" + page_size: 50 + page_token_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "page" + requester: + $ref: "#/definitions/requester" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + users_stream: + $ref: "#/definitions/base_stream" + name: "users" + primary_key: "id" + $parameters: + path: "workspaces/{{ config['workspace_id'] }}/users" + + projects_stream: + $ref: "#/definitions/base_stream" + name: "projects" + primary_key: "id" + $parameters: + path: "workspaces/{{ config['workspace_id'] }}/projects" + + clients_stream: + $ref: "#/definitions/base_stream" + name: "clients" + primary_key: "id" + $parameters: + path: "workspaces/{{ config['workspace_id'] }}/clients" + + tags_stream: + $ref: "#/definitions/base_stream" + name: "tags" + primary_key: "id" + $parameters: + path: "workspaces/{{ config['workspace_id'] }}/tags" + + user_groups_stream: + $ref: "#/definitions/base_stream" + name: "user_groups" + primary_key: "id" + $parameters: + path: "workspaces/{{ config['workspace_id'] }}/user-groups" + + users_partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/users_stream" + parent_key: "id" + partition_field: "user_id" + + time_entries_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "time_entries" + primary_key: "id" + path: "workspaces/{{ config['workspace_id'] }}/user/{{ stream_partition.user_id }}/time-entries" + retriever: + $ref: "#/definitions/retriever" + partition_router: + $ref: "#/definitions/users_partition_router" + + projects_partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/projects_stream" + parent_key: "id" + partition_field: "project_id" + + tasks_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "tasks" + primary_key: "id" + path: "workspaces/{{ config['workspace_id'] }}/projects/{{ stream_partition.project_id }}/tasks" + retriever: + $ref: "#/definitions/retriever" + partition_router: + $ref: "#/definitions/projects_partition_router" + +streams: + - "#/definitions/users_stream" + - "#/definitions/projects_stream" + - "#/definitions/clients_stream" + - "#/definitions/tags_stream" + - "#/definitions/user_groups_stream" + - "#/definitions/time_entries_stream" + - "#/definitions/tasks_stream" + +check: + type: CheckStream + stream_names: + - "users" + - "projects" + - "clients" + - "tags" + - "user_groups" + - "time_entries" + - "tasks" diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tasks.json b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tasks.json index df6eef85a5e3..2693e4c6ad56 100644 --- a/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tasks.json +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/schemas/tasks.json @@ -2,6 +2,9 @@ "$schema": "http://json-schema.org/draft-07/schema#", "additionalProperties": true, "properties": { + "budgetEstimate": { + "type": ["null", "number"] + }, "assigneeId": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/source.py b/airbyte-integrations/connectors/source-clockify/source_clockify/source.py index 37547b049a2f..e19d7f20ca07 100644 --- a/airbyte-integrations/connectors/source-clockify/source_clockify/source.py +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/source.py @@ -2,33 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import Any, List, Mapping, Tuple +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.requests_native_auth.token import TokenAuthenticator +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -from .streams import Clients, Projects, Tags, Tasks, TimeEntries, UserGroups, Users +WARNING: Do not modify this file. +""" -# Source -class SourceClockify(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - try: - workspace_stream = Users( - authenticator=TokenAuthenticator(token=config["api_key"], auth_header="X-Api-Key", auth_method=""), - workspace_id=config["workspace_id"], - api_url=config["api_url"], - ) - next(workspace_stream.read_records(sync_mode=SyncMode.full_refresh)) - return True, None - except Exception as e: - return False, f"Please check that your API key and workspace id are entered correctly: {repr(e)}" - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - authenticator = TokenAuthenticator(token=config["api_key"], auth_header="X-Api-Key", auth_method="") - - args = {"authenticator": authenticator, "workspace_id": config["workspace_id"], "api_url": config["api_url"]} - - return [Users(**args), Projects(**args), Clients(**args), Tags(**args), UserGroups(**args), TimeEntries(**args), Tasks(**args)] +# Declarative Source +class SourceClockify(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/spec.json b/airbyte-integrations/connectors/source-clockify/source_clockify/spec.json deleted file mode 100644 index 42756964f11a..000000000000 --- a/airbyte-integrations/connectors/source-clockify/source_clockify/spec.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/clockify", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Clockify Spec", - "type": "object", - "required": ["workspace_id", "api_key"], - "additionalProperties": true, - "properties": { - "workspace_id": { - "title": "Workspace Id", - "description": "WorkSpace Id", - "type": "string" - }, - "api_key": { - "title": "API Key", - "description": "You can get your api access_key here This API is Case Sensitive.", - "type": "string", - "airbyte_secret": true - }, - "api_url": { - "title": "API Url", - "description": "The URL for the Clockify API. This should only need to be modified if connecting to an enterprise version of Clockify.", - "type": "string", - "default": "https://api.clockify.me" - } - } - } -} diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/spec.yaml b/airbyte-integrations/connectors/source-clockify/source_clockify/spec.yaml new file mode 100644 index 000000000000..21305f888294 --- /dev/null +++ b/airbyte-integrations/connectors/source-clockify/source_clockify/spec.yaml @@ -0,0 +1,28 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/clockify +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Clockify Spec + type: object + required: + - workspace_id + - api_key + additionalProperties: true + properties: + workspace_id: + title: Workspace Id + description: WorkSpace Id + type: string + api_key: + title: API Key + description: + You can get your api access_key here + This API is Case Sensitive. + type: string + airbyte_secret: true + api_url: + title: API Url + description: + The URL for the Clockify API. This should only need to be modified + if connecting to an enterprise version of Clockify. + type: string + default: https://api.clockify.me diff --git a/airbyte-integrations/connectors/source-clockify/source_clockify/streams.py b/airbyte-integrations/connectors/source-clockify/source_clockify/streams.py deleted file mode 100644 index f20462415943..000000000000 --- a/airbyte-integrations/connectors/source-clockify/source_clockify/streams.py +++ /dev/null @@ -1,131 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from abc import ABC -from typing import Any, Iterable, Mapping, MutableMapping, Optional - -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream -from requests.auth import AuthBase - - -class ClockifyStream(HttpStream, ABC): - url_base = "" - api_url = "" - api_path = "/api/v1/" - page_size = 50 - page = 1 - primary_key = None - - def __init__(self, workspace_id: str, api_url: str, **kwargs): - super().__init__(**kwargs) - self.api_url = api_url - self.url_base = self.api_url + self.api_path - self.workspace_id = workspace_id - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - next_page = response.json() - self.page = self.page + 1 - if next_page: - return {"page": self.page} - else: - self.page = 1 - - def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: - params = { - "page-size": self.page_size, - } - - if next_page_token: - params.update(next_page_token) - - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield from response.json() - - -class Users(ClockifyStream): - @property - def use_cache(self) -> bool: - return True - - def path(self, **kwargs) -> str: - return f"workspaces/{self.workspace_id}/users" - - -class Projects(ClockifyStream): - @property - def use_cache(self) -> bool: - return True - - def path(self, **kwargs) -> str: - return f"workspaces/{self.workspace_id}/projects" - - -class Clients(ClockifyStream): - def path(self, **kwargs) -> str: - return f"workspaces/{self.workspace_id}/clients" - - -class Tags(ClockifyStream): - def path(self, **kwargs) -> str: - return f"workspaces/{self.workspace_id}/tags" - - -class UserGroups(ClockifyStream): - def path(self, **kwargs) -> str: - return f"workspaces/{self.workspace_id}/user-groups" - - -class TimeEntries(HttpSubStream, ClockifyStream): - def __init__(self, authenticator: AuthBase, workspace_id: Mapping[str, Any], api_url: str, **kwargs): - super().__init__( - authenticator=authenticator, - workspace_id=workspace_id, - api_url=api_url, - parent=Users(authenticator=authenticator, workspace_id=workspace_id, api_url=api_url, **kwargs), - ) - - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: - """ - self.authenticator (which should be used as the - authenticator for Users) is object of NoAuth() - - so self._session.auth is used instead - """ - users_stream = Users(authenticator=self._session.auth, workspace_id=self.workspace_id, api_url=self.api_url) - for user in users_stream.read_records(sync_mode=SyncMode.full_refresh): - yield {"user_id": user["id"]} - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - user_id = stream_slice["user_id"] - return f"workspaces/{self.workspace_id}/user/{user_id}/time-entries" - - -class Tasks(HttpSubStream, ClockifyStream): - def __init__(self, authenticator: AuthBase, workspace_id: Mapping[str, Any], api_url: str, **kwargs): - super().__init__( - authenticator=authenticator, - workspace_id=workspace_id, - api_url=api_url, - parent=Projects(authenticator=authenticator, workspace_id=workspace_id, api_url=api_url, **kwargs), - ) - - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: - """ - self.authenticator (which should be used as the - authenticator for Projects) is object of NoAuth() - - so self._session.auth is used instead - """ - projects_stream = Projects(authenticator=self._session.auth, workspace_id=self.workspace_id, api_url=self.api_url) - for project in projects_stream.read_records(sync_mode=SyncMode.full_refresh): - yield {"project_id": project["id"]} - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - project_id = stream_slice["project_id"] - return f"workspaces/{self.workspace_id}/projects/{project_id}/tasks" diff --git a/airbyte-integrations/connectors/source-clockify/unit_tests/conftest.py b/airbyte-integrations/connectors/source-clockify/unit_tests/conftest.py deleted file mode 100644 index f712b6c15dd9..000000000000 --- a/airbyte-integrations/connectors/source-clockify/unit_tests/conftest.py +++ /dev/null @@ -1,10 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pytest - - -@pytest.fixture(scope="session", name="config") -def config_fixture(): - return {"api_key": "test_api_key", "workspace_id": "workspace_id", "api_url": "http://some.test.url"} diff --git a/airbyte-integrations/connectors/source-clockify/unit_tests/test_source.py b/airbyte-integrations/connectors/source-clockify/unit_tests/test_source.py deleted file mode 100644 index 3cca00a0c4a4..000000000000 --- a/airbyte-integrations/connectors/source-clockify/unit_tests/test_source.py +++ /dev/null @@ -1,33 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -import responses -from source_clockify.source import SourceClockify - - -def setup_responses(): - responses.add( - responses.GET, - "http://some.test.url/api/v1/workspaces/workspace_id/users", - json={"access_token": "test_api_key", "expires_in": 3600}, - ) - - -@responses.activate -def test_check_connection(config): - setup_responses() - source = SourceClockify() - logger_mock = MagicMock() - assert source.check_connection(logger_mock, config) == (True, None) - - -def test_streams(mocker): - source = SourceClockify() - config_mock = MagicMock() - streams = source.streams(config_mock) - - expected_streams_number = 7 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-clockify/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-clockify/unit_tests/test_streams.py deleted file mode 100644 index debe32e0d4ac..000000000000 --- a/airbyte-integrations/connectors/source-clockify/unit_tests/test_streams.py +++ /dev/null @@ -1,49 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -import pytest -from airbyte_cdk.models import SyncMode -from source_clockify.streams import ClockifyStream - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(ClockifyStream, "path", "v0/example_endpoint") - mocker.patch.object(ClockifyStream, "primary_key", "test_primary_key") - mocker.patch.object(ClockifyStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = ClockifyStream(workspace_id=MagicMock(), api_url=MagicMock()) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {"page-size": 50} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = ClockifyStream(workspace_id=MagicMock(), api_url=MagicMock()) - inputs = {"response": MagicMock()} - expected_token = {"page": 2} - assert stream.next_page_token(**inputs) == expected_token - - -def test_read_records(patch_base_class): - stream = ClockifyStream(workspace_id=MagicMock(), api_url=MagicMock()) - assert stream.read_records(sync_mode=SyncMode.full_refresh) - - -def test_request_headers(patch_base_class): - stream = ClockifyStream(workspace_id=MagicMock(), api_url=MagicMock()) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = ClockifyStream(workspace_id=MagicMock(), api_url=MagicMock()) - expected_method = "GET" - assert stream.http_method == expected_method diff --git a/airbyte-integrations/connectors/source-close-com/Dockerfile b/airbyte-integrations/connectors/source-close-com/Dockerfile index 4deb69dd321d..44603bb80be5 100644 --- a/airbyte-integrations/connectors/source-close-com/Dockerfile +++ b/airbyte-integrations/connectors/source-close-com/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.4.2 +LABEL io.airbyte.version=0.5.0 LABEL io.airbyte.name=airbyte/source-close-com diff --git a/airbyte-integrations/connectors/source-close-com/README.md b/airbyte-integrations/connectors/source-close-com/README.md index 61300aa46a02..8dbd979e6b6e 100644 --- a/airbyte-integrations/connectors/source-close-com/README.md +++ b/airbyte-integrations/connectors/source-close-com/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew clean :airbyte-integrations:connectors:source-close-com:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/close-com) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_close_com/spec.json` file. @@ -54,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-close-com:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-close-com build ``` -You can also build the connector image via Gradle: -``` -./gradlew clean :airbyte-integrations:connectors:source-close-com:airbyteDocker +An image will be built with the tag `airbyte/source-close-com:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-close-com:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,44 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-close-com:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-close-com:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-close-com:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-close-com test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew clean :airbyte-integrations:connectors:source-close-com:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew clean :airbyte-integrations:connectors:source-close-com:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -122,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-close-com test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/close-com.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-close-com/acceptance-test-config.yml b/airbyte-integrations/connectors/source-close-com/acceptance-test-config.yml index c1382737b9f5..41c2635332b7 100644 --- a/airbyte-integrations/connectors/source-close-com/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-close-com/acceptance-test-config.yml @@ -4,63 +4,61 @@ connector_image: airbyte/source-close-com:dev acceptance_tests: spec: tests: - - spec_path: "source_close_com/spec.json" + - spec_path: "source_close_com/spec.json" connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: tests: - - config_path: "secrets/config.json" + - config_path: "secrets/config.json" basic_read: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - empty_streams: - - name: missed_call_tasks - bypass_reason: "unable to populate" - - name: answered_detached_call_tasks - bypass_reason: "unable to populate" - - name: incoming_sms_tasks - bypass_reason: "unable to populate" - - name: send_as - bypass_reason: "unable to populate" - - name: voicemail_tasks - bypass_reason: "unable to populate" - - name: leads - bypass_reason: "unable to test due to fast-changing data" - - name: events - bypass_reason: "unable to test due to fast-changing data" - - name: users - bypass_reason: "unable to test due to fast-changing data" - - name: contacts - bypass_reason: "unable to test due to fast-changing data" - - name: google_connected_accounts - bypass_reason: "unable to test due to fast-changing data" - - name: custom_email_connected_accounts - bypass_reason: "unable to test due to fast-changing data" - - name: zoom_connected_accounts - bypass_reason: "unable to test due to fast-changing data" - - name: email_bulk_actions - bypass_reason: "unable to test due to fast-changing data" - - name: incoming_email_tasks - bypass_reason: "unable to test due to fast-changing data" - - name: created_activities - bypass_reason: "return records randomly" - - name: opportunity_status_change_activities - bypass_reason: "return records randomly" - fail_on_extra_columns: false + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: missed_call_tasks + bypass_reason: "unable to populate" + - name: answered_detached_call_tasks + bypass_reason: "unable to populate" + - name: incoming_sms_tasks + bypass_reason: "unable to populate" + - name: send_as + bypass_reason: "unable to populate" + - name: voicemail_tasks + bypass_reason: "unable to populate" + - name: leads + bypass_reason: "unable to test due to fast-changing data" + - name: events + bypass_reason: "unable to test due to fast-changing data" + - name: users + bypass_reason: "unable to test due to fast-changing data" + - name: contacts + bypass_reason: "unable to test due to fast-changing data" + - name: google_connected_accounts + bypass_reason: "unable to test due to fast-changing data" + - name: custom_email_connected_accounts + bypass_reason: "unable to test due to fast-changing data" + - name: zoom_connected_accounts + bypass_reason: "unable to test due to fast-changing data" + - name: email_bulk_actions + bypass_reason: "unable to test due to fast-changing data" + - name: incoming_email_tasks + bypass_reason: "unable to test due to fast-changing data" + - name: created_activities + bypass_reason: "return records randomly" + - name: opportunity_status_change_activities + bypass_reason: "return records randomly" + fail_on_extra_columns: false incremental: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-close-com/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-close-com/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-close-com/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-close-com/build.gradle b/airbyte-integrations/connectors/source-close-com/build.gradle deleted file mode 100644 index 6297a043adcf..000000000000 --- a/airbyte-integrations/connectors/source-close-com/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_close_com' -} diff --git a/airbyte-integrations/connectors/source-close-com/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-close-com/integration_tests/expected_records.jsonl index bfcc3040ef80..072d820d710f 100644 --- a/airbyte-integrations/connectors/source-close-com/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-close-com/integration_tests/expected_records.jsonl @@ -13,10 +13,15 @@ {"stream": "sms_activities", "data": {"local_country_iso": "US", "activity_at": "2021-08-11T18:14:32.750000+00:00", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_phone_formatted": "+1 202-555-0186", "sequence_subscription_id": null, "date_scheduled": null, "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "_type": "SMS", "updated_by_name": "Airbyte Team", "local_phone": "+14154445555", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "id": "acti_GIVSys3F0wFeA519lDa5QKRfYOPgskyqKj2aXiCMSEO", "attachments": [], "sequence_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_sent": null, "local_phone_formatted": "+1 415-444-5555", "created_by_name": "Airbyte Team", "date_created": "2021-08-11T18:14:32.750000+00:00", "cost": null, "text": "Hi! This is a reminder that we have a call scheduled for 12pm PT today.", "direction": "outbound", "error_message": null, "date_updated": "2021-08-11T18:14:32.750000+00:00", "source": "Close.io", "template_id": null, "sequence_id": null, "remote_country_iso": "US", "status": "draft", "remote_phone": "+12025550186", "user_name": "Airbyte Team"}, "emitted_at": 1691417093006} {"stream": "sms_activities", "data": {"lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "attachments": [], "remote_phone": "+14156236785", "sequence_id": null, "remote_country_iso": "US", "sequence_subscription_id": null, "cost": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_phone_formatted": "+1 415-623-6785", "created_by_name": "Airbyte Team", "date_updated": "2022-11-09T14:02:24.943000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "direction": "outbound", "local_phone_formatted": "+1 415-625-1293", "text": "Hi!", "template_id": "smstmpl_6zpaSGDsZyhBhrQH0jmZK9", "date_created": "2022-11-09T14:02:18.756000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "local_phone": "+14156251293", "source": "Close.io", "_type": "SMS", "local_country_iso": "US", "updated_by_name": "Airbyte Team", "date_scheduled": null, "error_message": null, "id": "acti_CXrlrlHc8QpP5MBXqCfcJKpctDqBREuG3Whj0JQODu4", "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "activity_at": "2022-11-09T14:02:18.756000+00:00", "user_name": "Airbyte Team", "sequence_name": null, "status": "draft", "date_sent": null, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417093912} {"stream": "sms_activities", "data": {"lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "attachments": [{"media_id": "media_580CfGelcIvP5BzvIYTlhX", "url": "https://app.close.com/go/sms/acti_eQ2xwn1RN5sdEXkGpln8Vag8rwPsYfU1qbyeN7dBPsP/media/media_580CfGelcIvP5BzvIYTlhX/", "thumbnail_url": "https://app.close.com/go/sms/acti_eQ2xwn1RN5sdEXkGpln8Vag8rwPsYfU1qbyeN7dBPsP/media/media_580CfGelcIvP5BzvIYTlhX/thumbnail/", "content_type": "image/png", "size": 6132, "filename": "Airbyte_logo_75x75.png"}], "remote_phone": "+14156236785", "sequence_id": null, "remote_country_iso": "US", "sequence_subscription_id": null, "cost": "3", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_phone_formatted": "+1 415-623-6785", "created_by_name": "Airbyte Team", "date_updated": "2022-11-09T12:54:44.405000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "direction": "outbound", "local_phone_formatted": "+1 415-625-1293", "text": "Hi!", "template_id": "smstmpl_6zpaSGDsZyhBhrQH0jmZK9", "date_created": "2022-11-09T12:54:15.456000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "local_phone": "+14156251293", "source": "Close.io", "_type": "SMS", "local_country_iso": "US", "updated_by_name": "Airbyte Team", "date_scheduled": null, "error_message": null, "id": "acti_eQ2xwn1RN5sdEXkGpln8Vag8rwPsYfU1qbyeN7dBPsP", "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "activity_at": "2022-11-09T12:54:44.404000+00:00", "user_name": "Airbyte Team", "sequence_name": null, "status": "sent", "date_sent": "2022-11-09T12:54:44.404000+00:00", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417093915} -{"stream": "call_activities", "data": {"local_phone_formatted": null, "voicemail_url": "https://api.close.com/call/acti_NsDheeFBzEAmjRfpBmclxdzKWqLnGZaIvU3ZvvyrDJt/voicemail/", "transferred_to_user_id": null, "forwarded_to": null, "remote_country_iso": "US", "date_answered": null, "voicemail_duration": 28, "_type": "Call", "disposition": "vm-left", "status": "completed", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "updated_by_name": null, "date_created": "2021-07-16T00:00:12.646000+00:00", "recording_expires_at": null, "remote_phone_formatted": "+1 650-517-6539", "cost": null, "phone": "+16505176539", "is_joinable": false, "created_by_name": null, "note": null, "sequence_id": null, "recording_url": null, "local_phone": null, "sequence_subscription_id": null, "transferred_from_user_id": null, "has_recording": false, "source": "Close.io", "transferred_from": null, "direction": "inbound", "sequence_name": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "dialer_saved_search_id": null, "created_by": null, "remote_phone": "+16505176539", "updated_by": null, "duration": 0, "call_method": "regular", "id": "acti_NsDheeFBzEAmjRfpBmclxdzKWqLnGZaIvU3ZvvyrDJt", "coach_legs": [], "is_to_group_number": false, "date_updated": "2021-07-16T00:00:12.646000+00:00", "contact_id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "users": [], "user_name": null, "note_html": null, "is_forwarded": false, "transferred_to": null, "dialer_id": null, "local_country_iso": "", "user_id": null, "activity_at": "2021-07-16T00:00:12.646000+00:00"}, "emitted_at": 1691417095367} -{"stream": "call_activities", "data": {"local_phone_formatted": null, "voicemail_url": null, "transferred_to_user_id": null, "forwarded_to": null, "remote_country_iso": "US", "date_answered": null, "voicemail_duration": 0, "_type": "Call", "disposition": null, "status": "no-answer", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "updated_by_name": "Airbyte Team", "date_created": "2021-07-05T11:39:00.850000+00:00", "recording_expires_at": null, "remote_phone_formatted": "+1 202-555-0186", "cost": null, "phone": "+12025550186", "is_joinable": false, "created_by_name": "Airbyte Team", "note": "Gob never answered.", "sequence_id": null, "recording_url": null, "local_phone": null, "sequence_subscription_id": null, "transferred_from_user_id": null, "has_recording": false, "source": "Close.io", "transferred_from": null, "direction": "outbound", "sequence_name": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "dialer_saved_search_id": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_phone": "+12025550186", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "duration": 0, "call_method": "regular", "id": "acti_wszWUd92D7wNYSbn5gKWXCf55NeyU0jc1Vguv7DfUiH", "coach_legs": [], "is_to_group_number": false, "date_updated": "2021-07-13T11:39:04.536000+00:00", "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "users": [], "user_name": "Airbyte Team", "note_html": "

      Gob never answered.

      ", "is_forwarded": false, "transferred_to": null, "dialer_id": null, "local_country_iso": "", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "activity_at": "2021-07-05T11:39:00.850000+00:00"}, "emitted_at": 1691417095372} -{"stream": "call_activities", "data": {"activity_at": "2022-11-09T13:57:14.751000+00:00", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "sequence_name": null, "users": [], "recording_url": null, "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "transferred_to": null, "remote_phone": "+14156236785", "remote_country_iso": "US", "cost": "2", "_type": "Call", "local_country_iso": "US", "duration": 17, "created_by_name": "Airbyte Team", "direction": "outbound", "has_recording": false, "voicemail_url": null, "transferred_to_user_id": null, "date_answered": "2022-11-09T13:57:22.063000+00:00", "transferred_from": null, "coach_legs": [], "phone": "+14156236785", "dialer_saved_search_id": null, "voicemail_duration": 0, "updated_by_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "dialer_id": null, "is_forwarded": false, "is_joinable": false, "transferred_from_user_id": null, "local_phone": "+14156251293", "date_updated": "2022-11-09T13:57:40.167000+00:00", "recording_expires_at": null, "forwarded_to": null, "sequence_subscription_id": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status": "completed", "source": "Close.io", "date_created": "2022-11-09T13:57:14.751000+00:00", "local_phone_formatted": "+1 415-625-1293", "note_html": "

      ", "is_to_group_number": false, "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "disposition": "answered", "remote_phone_formatted": "+1 415-623-6785", "user_name": "Airbyte Team", "note": "", "call_method": "regular", "id": "acti_ZgqHg31m0XXDZPwaxUVUNOhnEoLSG3fY8rMn9iIS3cN", "sequence_id": null}, "emitted_at": 1691417095961} -{"stream": "meeting_activities", "data": {"starts_at": "2022-11-12T18:00:00+00:00", "user_note": null, "updated_by_name": "Airbyte Team", "notetaker_id": null, "_type": "Meeting", "id": "acti_AlRbqNk15jdt7Eq2phHXuV49qSpkLBB7af3mwvJkk1z", "duration": 3600, "ends_at": "2022-11-12T19:00:00+00:00", "title": "Test meeting 2", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status": "completed", "activity_at": "2022-11-12T18:00:00+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "connected_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "user_name": "Airbyte Team", "user_note_html": null, "date_updated": "2022-11-12T19:00:03.639000+00:00", "is_recurring": false, "contact_id": null, "attendees": [{"contact_id": null, "is_organizer": false, "email": "irina.grankova@gmail.com", "name": null, "status": "yes", "user_id": null}, {"contact_id": null, "is_organizer": true, "email": "iryna.grankova@airbyte.io", "name": null, "status": "yes", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}], "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "users": ["user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"], "calendar_event_link": "https://www.google.com/calendar/event?eid=MWRlcDU1cDZtamY0MGxnYnJ2OXI5ajlocG8gaXJ5bmEuZ3JhbmtvdmFAYWlyYnl0ZS5pbw", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "location": null, "integrations": [], "created_by_name": "Airbyte Team", "source": "calendar", "note": "", "date_created": "2022-11-12T18:00:00+00:00", "calendar_event_uids": ["1dep55p6mjf40lgbrv9r9j9hpo"]}, "emitted_at": 1691417097896} +{"stream": "call_activities", "data": {"has_recording": false, "note": null, "remote_phone_formatted": "+1 650-517-6539", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "is_forwarded": false, "sequence_subscription_id": null, "dialer_id": null, "date_answered": null, "transferred_from_user_id": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by_name": null, "transferred_to": null, "updated_by_name": null, "dialer_saved_search_id": null, "call_method": "regular", "parent_meeting_id": null, "local_phone_formatted": null, "updated_by": null, "remote_country_iso": "US", "is_to_group_number": false, "date_created": "2021-07-16T00:00:12.646000+00:00", "voicemail_url": "https://api.close.com/call/acti_NsDheeFBzEAmjRfpBmclxdzKWqLnGZaIvU3ZvvyrDJt/voicemail/", "duration": 0, "forwarded_to": null, "recording_duration": null, "phone": "+16505176539", "voicemail_duration": 28, "cost": null, "sequence_name": null, "created_by": null, "note_html": null, "local_country_iso": "", "transferred_from": null, "source": "Close.io", "users": [], "transferred_to_user_id": null, "recording_url": null, "id": "acti_NsDheeFBzEAmjRfpBmclxdzKWqLnGZaIvU3ZvvyrDJt", "status": "completed", "contact_id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "_type": "Call", "disposition": "vm-left", "local_phone": null, "direction": "inbound", "is_joinable": false, "coach_legs": [], "date_updated": "2021-07-16T00:00:12.646000+00:00", "recording_expires_at": null, "user_id": null, "sequence_id": null, "user_name": null, "activity_at": "2021-07-16T00:00:12.646000+00:00", "remote_phone": "+16505176539"}, "emitted_at": 1699627822986} +{"stream": "call_activities", "data": {"has_recording": false, "note": "Gob never answered.", "remote_phone_formatted": "+1 202-555-0186", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "is_forwarded": false, "sequence_subscription_id": null, "dialer_id": null, "date_answered": null, "transferred_from_user_id": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by_name": "Airbyte Team", "transferred_to": null, "updated_by_name": "Airbyte Team", "dialer_saved_search_id": null, "call_method": "regular", "parent_meeting_id": null, "local_phone_formatted": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_country_iso": "US", "is_to_group_number": false, "date_created": "2021-07-05T11:39:00.850000+00:00", "voicemail_url": null, "duration": 0, "forwarded_to": null, "recording_duration": null, "phone": "+12025550186", "voicemail_duration": 0, "cost": null, "sequence_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "note_html": "

      Gob never answered.

      ", "local_country_iso": "", "transferred_from": null, "source": "Close.io", "users": [], "transferred_to_user_id": null, "recording_url": null, "id": "acti_wszWUd92D7wNYSbn5gKWXCf55NeyU0jc1Vguv7DfUiH", "status": "no-answer", "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "_type": "Call", "disposition": null, "local_phone": null, "direction": "outbound", "is_joinable": false, "coach_legs": [], "date_updated": "2021-07-13T11:39:04.536000+00:00", "recording_expires_at": null, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "sequence_id": null, "user_name": "Airbyte Team", "activity_at": "2021-07-05T11:39:00.850000+00:00", "remote_phone": "+12025550186"}, "emitted_at": 1699627822992} +{"stream": "call_activities", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_country_iso": "US", "date_created": "2022-11-09T13:57:14.751000+00:00", "has_recording": false, "voicemail_url": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "duration": 17, "parent_meeting_id": null, "recording_duration": null, "activity_at": "2022-11-09T13:57:14.751000+00:00", "dialer_saved_search_id": null, "dialer_id": null, "sequence_name": null, "voicemail_duration": 0, "direction": "outbound", "updated_by_name": "Airbyte Team", "recording_url": null, "call_method": "regular", "date_updated": "2022-11-09T13:57:40.167000+00:00", "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "transferred_to_user_id": null, "recording_expires_at": null, "note": "", "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "status": "completed", "user_name": "Airbyte Team", "local_country_iso": "US", "transferred_from": null, "cost": "2", "transferred_to": null, "sequence_id": null, "source": "Close.io", "local_phone": "+14156251293", "disposition": "answered", "coach_legs": [], "phone": "+14156236785", "transferred_from_user_id": null, "created_by_name": "Airbyte Team", "sequence_subscription_id": null, "is_forwarded": false, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "note_html": "

      ", "is_to_group_number": false, "forwarded_to": null, "remote_phone_formatted": "+1 415-623-6785", "local_phone_formatted": "+1 415-625-1293", "id": "acti_ZgqHg31m0XXDZPwaxUVUNOhnEoLSG3fY8rMn9iIS3cN", "remote_phone": "+14156236785", "date_answered": "2022-11-09T13:57:22.063000+00:00", "is_joinable": false, "users": [], "_type": "Call"}, "emitted_at": 1699627823579} +{"stream": "call_activities", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_country_iso": "US", "date_created": "2022-11-09T12:55:26.751000+00:00", "has_recording": false, "voicemail_url": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "duration": 0, "parent_meeting_id": null, "recording_duration": null, "activity_at": "2022-11-09T12:55:26.751000+00:00", "dialer_saved_search_id": null, "dialer_id": null, "sequence_name": null, "voicemail_duration": 0, "direction": "outbound", "updated_by_name": "Airbyte Team", "recording_url": null, "call_method": "regular", "date_updated": "2022-11-09T12:57:27.648000+00:00", "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "transferred_to_user_id": null, "recording_expires_at": null, "note": "", "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "status": "completed", "user_name": "Airbyte Team", "local_country_iso": "US", "transferred_from": null, "cost": null, "transferred_to": null, "sequence_id": null, "source": "Close.io", "local_phone": "+14156251293", "disposition": "answered", "coach_legs": [], "phone": "+14156236785", "transferred_from_user_id": null, "created_by_name": "Airbyte Team", "sequence_subscription_id": null, "is_forwarded": false, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "note_html": "

      ", "is_to_group_number": false, "forwarded_to": null, "remote_phone_formatted": "+1 415-623-6785", "local_phone_formatted": "+1 415-625-1293", "id": "acti_694PvtEQICHUY5umTjEM8nl9bLU0MmWWV6HhMEcqAEF", "remote_phone": "+14156236785", "date_answered": null, "is_joinable": false, "users": [], "_type": "Call"}, "emitted_at": 1699627823583} +{"stream": "call_activities", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_country_iso": "US", "date_created": "2022-11-09T12:55:03.314000+00:00", "has_recording": false, "voicemail_url": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "duration": 10, "parent_meeting_id": null, "recording_duration": null, "activity_at": "2022-11-09T12:55:03.314000+00:00", "dialer_saved_search_id": null, "dialer_id": null, "sequence_name": null, "voicemail_duration": 0, "direction": "outbound", "updated_by_name": "Airbyte Team", "recording_url": null, "call_method": "regular", "date_updated": "2022-11-09T12:55:23.356000+00:00", "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "transferred_to_user_id": null, "recording_expires_at": null, "note": "", "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "status": "completed", "user_name": "Airbyte Team", "local_country_iso": "US", "transferred_from": null, "cost": "2", "transferred_to": null, "sequence_id": null, "source": "Close.io", "local_phone": "+14156251293", "disposition": "answered", "coach_legs": [], "phone": "+14156236785", "transferred_from_user_id": null, "created_by_name": "Airbyte Team", "sequence_subscription_id": null, "is_forwarded": false, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "note_html": "

      ", "is_to_group_number": false, "forwarded_to": null, "remote_phone_formatted": "+1 415-623-6785", "local_phone_formatted": "+1 415-625-1293", "id": "acti_xzjEYrejim9F9IWz4mrs16ehPCcnUcit43wJTZkwuhv", "remote_phone": "+14156236785", "date_answered": "2022-11-09T12:55:11.703000+00:00", "is_joinable": false, "users": [], "_type": "Call"}, "emitted_at": 1699627823587} +{"stream": "call_activities", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_country_iso": "US", "date_created": "2022-11-09T11:20:32.524000+00:00", "has_recording": false, "voicemail_url": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "duration": 3, "parent_meeting_id": null, "recording_duration": null, "activity_at": "2022-11-09T11:20:32.524000+00:00", "dialer_saved_search_id": null, "dialer_id": null, "sequence_name": null, "voicemail_duration": 0, "direction": "outbound", "updated_by_name": "Airbyte Team", "recording_url": null, "call_method": "regular", "date_updated": "2022-11-09T11:20:42.502000+00:00", "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "transferred_to_user_id": null, "recording_expires_at": null, "note": "", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "status": "completed", "user_name": "Airbyte Team", "local_country_iso": "US", "transferred_from": null, "cost": "2", "transferred_to": null, "sequence_id": null, "source": "Close.io", "local_phone": "+14156251293", "disposition": "answered", "coach_legs": [], "phone": "+14156236785", "transferred_from_user_id": null, "created_by_name": "Airbyte Team", "sequence_subscription_id": null, "is_forwarded": false, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "note_html": "

      ", "is_to_group_number": false, "forwarded_to": null, "remote_phone_formatted": "+1 415-623-6785", "local_phone_formatted": "+1 415-625-1293", "id": "acti_2PDiIs2kT5NQaipTXd1lyavwVLS5a6wipeheGVWOO54", "remote_phone": "+14156236785", "date_answered": "2022-11-09T11:20:38.986000+00:00", "is_joinable": false, "users": [], "_type": "Call"}, "emitted_at": 1699627823592} +{"stream": "call_activities", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_country_iso": "UA", "date_created": "2022-11-08T15:55:39.770000+00:00", "has_recording": false, "voicemail_url": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "duration": 0, "parent_meeting_id": null, "recording_duration": null, "activity_at": "2022-11-08T15:55:39.770000+00:00", "dialer_saved_search_id": null, "dialer_id": null, "sequence_name": null, "voicemail_duration": 0, "direction": "outbound", "updated_by_name": "Airbyte Team", "recording_url": null, "call_method": "regular", "date_updated": "2022-11-09T12:56:13.631000+00:00", "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "transferred_to_user_id": null, "recording_expires_at": null, "note": "Test test", "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "status": "failed", "user_name": "Airbyte Team", "local_country_iso": "", "transferred_from": null, "cost": null, "transferred_to": null, "sequence_id": null, "source": "Close.io", "local_phone": null, "disposition": "blocked", "coach_legs": [], "phone": "+380636306253", "transferred_from_user_id": null, "created_by_name": "Airbyte Team", "sequence_subscription_id": null, "is_forwarded": false, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "note_html": "

      Test test

      ", "is_to_group_number": false, "forwarded_to": null, "remote_phone_formatted": "+380 63 630 6253", "local_phone_formatted": null, "id": "acti_cBMvILyTlk57YSm7c8xfb9qNjgTX1z2OyjnPPsTXpRG", "remote_phone": "+380636306253", "date_answered": null, "is_joinable": false, "users": [], "_type": "Call"}, "emitted_at": 1699627823596} +{"stream": "call_activities", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "remote_country_iso": "UA", "date_created": "2022-11-08T15:53:39.388000+00:00", "has_recording": false, "voicemail_url": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "duration": 0, "parent_meeting_id": null, "recording_duration": null, "activity_at": "2022-11-08T15:53:39.388000+00:00", "dialer_saved_search_id": null, "dialer_id": null, "sequence_name": null, "voicemail_duration": 0, "direction": "outbound", "updated_by_name": "Airbyte Team", "recording_url": null, "call_method": "regular", "date_updated": "2022-11-08T15:54:58.677000+00:00", "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "transferred_to_user_id": null, "recording_expires_at": null, "note": "", "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "status": "failed", "user_name": "Airbyte Team", "local_country_iso": "", "transferred_from": null, "cost": null, "transferred_to": null, "sequence_id": null, "source": "Close.io", "local_phone": null, "disposition": "blocked", "coach_legs": [], "phone": "+380636306253", "transferred_from_user_id": null, "created_by_name": "Airbyte Team", "sequence_subscription_id": null, "is_forwarded": false, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "note_html": "

      ", "is_to_group_number": false, "forwarded_to": null, "remote_phone_formatted": "+380 63 630 6253", "local_phone_formatted": null, "id": "acti_HpPgUhPNiZeEkjdpfW6jmK11uXJyTZHHiubYmU0uoFZ", "remote_phone": "+380636306253", "date_answered": null, "is_joinable": false, "users": [], "_type": "Call"}, "emitted_at": 1699627823601} +{"stream": "meeting_activities", "data": {"calendar_event_link": "https://www.google.com/calendar/event?eid=MWRlcDU1cDZtamY0MGxnYnJ2OXI5ajlocG8gaXJ5bmEuZ3JhbmtvdmFAYWlyYnl0ZS5pbw", "notetaker_id": null, "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "integrations": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "summary": null, "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "created_by_name": "Airbyte Team", "note": "", "user_note": null, "duration": 3600, "location": null, "date_updated": "2022-11-12T19:00:03.639000+00:00", "provider_calendar_ids": ["iryna.grankova@airbyte.io"], "users": ["user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"], "title": "Test meeting 2", "is_recurring": false, "id": "acti_AlRbqNk15jdt7Eq2phHXuV49qSpkLBB7af3mwvJkk1z", "activity_at": "2022-11-12T18:00:00+00:00", "provider_calendar_type": "google", "status": "completed", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "conference_links": [], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_note_date_updated": null, "date_created": "2022-11-12T18:00:00+00:00", "user_note_html": null, "contact_id": null, "source": "calendar", "calendar_event_uids": ["1dep55p6mjf40lgbrv9r9j9hpo"], "attached_call_ids": [], "provider_calendar_event_id": "1dep55p6mjf40lgbrv9r9j9hpo", "_type": "Meeting", "starts_at": "2022-11-12T18:00:00+00:00", "attendees": [{"email": "irina.grankova@gmail.com", "name": null, "status": "yes", "user_id": null, "contact_id": null, "is_organizer": false}, {"email": "iryna.grankova@airbyte.io", "name": null, "status": "yes", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "contact_id": null, "is_organizer": true}], "ends_at": "2022-11-12T19:00:00+00:00", "connected_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "user_name": "Airbyte Team", "updated_by_name": "Airbyte Team"}, "emitted_at": 1699628913615} {"stream": "lead_status_change_activities", "data": {"new_status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "_type": "LeadStatusChange", "old_status_label": "Potential", "users": [], "activity_at": "2021-08-25T21:15:35.163000+00:00", "date_created": "2021-08-25T21:15:35.163000+00:00", "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "user_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "acti_wSO4ltT4XHGI6wkN2oCZpRdCtw1ALhsRttt8osqVjll", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "date_updated": "2021-08-25T21:15:35.163000+00:00", "old_status_id": "stat_HrZ1aYkkxRORQSxdBcNPT31HkqxkK2w2uWGiK6yjkmK", "new_status_label": "Interested", "contact_id": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417099274} {"stream": "lead_status_change_activities", "data": {"new_status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "_type": "LeadStatusChange", "old_status_label": "Qualified", "users": [], "activity_at": "2021-08-25T21:15:34.607000+00:00", "date_created": "2021-08-25T21:15:34.607000+00:00", "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "user_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "acti_T4XA2LBQYvrqAimflFjU2KXY0I2g1BPbOUn2jHZ59Dj", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "date_updated": "2021-08-25T21:15:34.607000+00:00", "old_status_id": "stat_y6v7svdpj3v1ZHd1GoiJFcKrUGrA0jl2Af53jfGbkN9", "new_status_label": "Interested", "contact_id": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417099279} {"stream": "lead_status_change_activities", "data": {"new_status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "_type": "LeadStatusChange", "old_status_label": "Qualified", "users": [], "activity_at": "2021-08-25T21:15:34.044000+00:00", "date_created": "2021-08-25T21:15:34.044000+00:00", "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "user_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "id": "acti_udWvk5RJ6SFuO9Ra6d1pAJg200Q0Zyq43kEq0FpceIx", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "date_updated": "2021-08-25T21:15:34.044000+00:00", "old_status_id": "stat_y6v7svdpj3v1ZHd1GoiJFcKrUGrA0jl2Af53jfGbkN9", "new_status_label": "Interested", "contact_id": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417099283} @@ -32,7 +37,7 @@ {"stream": "activity_custom_fields", "data": {"choices": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "editable_with_roles": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "cf_UftGNTS2rq9XMG9hOHkALp5cgn4Xl8nZqN7gReax3lc", "referenced_custom_type_id": null, "date_created": "2021-07-13T11:39:02.212000+00:00", "name": "Contact", "back_reference_is_visible": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "accepts_multiple_values": false, "type": "contact", "date_updated": "2021-07-13T11:39:02.212000+00:00", "is_shared": false, "description": null, "custom_activity_type_id": "actitype_0J9YvrOw4opjiYI4aDY6wj", "required": false}, "emitted_at": 1691417109700} {"stream": "activity_custom_fields", "data": {"choices": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "editable_with_roles": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "cf_3W4n175LyZ3QMfr665jUS19nt2QATHyM7QaxXQC4QqA", "referenced_custom_type_id": null, "date_created": "2021-07-13T11:39:02.744000+00:00", "name": "Current Vendor: Other (if applicable)", "back_reference_is_visible": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "accepts_multiple_values": false, "type": "text", "date_updated": "2021-07-13T11:39:02.744000+00:00", "is_shared": false, "description": null, "custom_activity_type_id": "actitype_0J9YvrOw4opjiYI4aDY6wj", "required": false}, "emitted_at": 1691417109703} {"stream": "activity_custom_fields", "data": {"choices": null, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "editable_with_roles": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "id": "cf_bp93vNo2vxbmM2QhQ7JvdytgcozCCGmOklTG9E8rDYa", "referenced_custom_type_id": null, "date_created": "2021-07-13T11:39:02.478000+00:00", "name": "Industry: Other (if applicable)", "back_reference_is_visible": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "accepts_multiple_values": false, "type": "text", "date_updated": "2021-07-13T11:39:02.478000+00:00", "is_shared": false, "description": null, "custom_activity_type_id": "actitype_0J9YvrOw4opjiYI4aDY6wj", "required": false}, "emitted_at": 1691417109705} -{"stream": "users", "data": {"last_used_timezone": "America/New_York", "date_created": "2021-07-13T11:36:04.905000+00:00", "date_updated": "2023-06-29T19:39:02.749000+00:00", "first_name": "Airbyte", "last_name": "Team", "google_profile_image_url": null, "image": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef", "organizations": ["orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi"], "email": "integration-test@airbyte.io", "email_verified_at": "2021-07-13T11:37:23.175000+00:00", "id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417111009} +{"stream": "users", "data": {"google_profile_image_url": null, "date_created": "2021-07-13T11:36:04.905000+00:00", "date_updated": "2023-10-21T04:56:24.365000+00:00", "id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organizations": ["orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi"], "last_name": "Team", "email_verified_at": "2021-07-13T11:37:23.175000+00:00", "last_used_timezone": "America/New_York", "image": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef", "first_name": "Airbyte", "email": "integration-test@airbyte.io"}, "emitted_at": 1699629347222} {"stream": "contacts", "data": {"id": "cont_b4h4BcmWn7rKbnsHQ0JfADwXGgndbpc5JlQEdHHyv78", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=User1"}], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "display_name": "User1", "date_created": "2022-11-11T09:00:52.289000+00:00", "emails": [], "lead_id": "lead_AUtZm7EBlaSbYqOrDjIZEuC4tfhLxTDtaK9jlEPMb3y", "date_updated": "2023-01-30T16:02:41.174000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "phones": [{"phone_formatted": "+1 600-000-0001", "country": null, "type": "office", "phone": "+16000000001"}], "urls": [], "title": "Product Manager", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "User1"}, "emitted_at": 1691417111908} {"stream": "contacts", "data": {"id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=User2"}], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "display_name": "User2", "date_created": "2022-11-08T15:54:42.381000+00:00", "emails": [{"type": "office", "email": "user2.sample@gmail.com"}], "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "date_updated": "2023-01-30T16:03:28.787000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "phones": [{"phone_formatted": "+1 415-623-6785", "country": "US", "type": "office", "phone": "+14156236785"}], "urls": [], "title": "Test Lead", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "User2", "custom.cf_Qj3b4cWxvmvqTtZ0U5TaprRDah8g7jRsavfVh8NCPcu": "2022-12-01"}, "emitted_at": 1691417111912} {"stream": "contacts", "data": {"id": "cont_Dsi7AGMRelIZ2I6DIKicaGJU7mwxPZElLIHy33xbjCM", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Cooper"}], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "display_name": "Cooper", "date_created": "2022-07-05T21:01:25.612000+00:00", "emails": [], "lead_id": "lead_Eohw2Vf6WOKZHQ97nS1UTL3iV62pAJFX3ROgJ5WT4cY", "date_updated": "2022-07-05T21:01:25.612000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "phones": [], "urls": [], "title": "", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Cooper"}, "emitted_at": 1691417111916} @@ -48,10 +53,38 @@ {"stream": "pipelines", "data": {"created_by": null, "date_created": "2021-07-13T11:36:04.983404", "date_updated": "2021-07-13T11:36:04.983404", "id": "pipe_0IAl41rGk9OPls9CdxFpHy", "name": "Sales", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "statuses": [{"id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "label": "Demo Completed", "type": "active"}, {"id": "stat_AWXzFJkvkHVyJQPFulY0wM7LrQRiMEtQChGumG035bH", "label": "Proposal Sent", "type": "active"}, {"id": "stat_ObYTUqjVZW0nTZXjvHhzMGyK999e42WZdIhkaNq12En", "label": "Contract Sent", "type": "active"}, {"id": "stat_gaqEGSVHIFzrofTfzzg5UfjyBZ1B6KERccIy2MOp8FG", "label": "Won", "type": "won"}, {"id": "stat_CZr5826cyG8wqIg4tD6bbjaqePP4HAYOMSLgOhi1Xbf", "label": "Lost", "type": "lost"}], "updated_by": null}, "emitted_at": 1691417119534} {"stream": "email_templates", "data": {"organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "attachments": [], "body": "Hi {{ contact.first_name }},

      I'm {{ user.first_name }} with {{ organization.name }}. We help companies in the {{ lead.custom.[\"Industry\"] }} space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at {{ lead.display_name }} and show you what we're working on.

      Are you available for a quick call tomorrow afternoon?
      ", "name": "Email 1 - Intro", "id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "is_shared": true, "is_archived": false, "date_created": "2021-07-13T11:39:00.497000+00:00", "subject": "{{ lead.display_name }} + {{ organization.name }}", "date_updated": "2022-08-11T18:11:08.966000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417120444} {"stream": "email_templates", "data": {"organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "attachments": [], "body": "Hi {{ contact.first_name }},

      Friendly follow-up.

      I wanted to show you how {{ organization.name }} can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

      - Wed @ 11AM
      - Thur @ 2PM
      - Fri @ 3PM
      ", "name": "Email 2 - Follow-up #1", "id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "is_shared": true, "is_archived": false, "date_created": "2021-07-13T11:39:00.503000+00:00", "subject": "{{ organization.name }} Follow-up", "date_updated": "2022-08-11T18:11:08.972000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417120447} -{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "next_billing_on"], "data": {"address_id": null, "bundle_id": null, "carrier": "twilio", "country": "US", "date_created": "2022-11-08T12:35:29.464000+00:00", "date_updated": "2023-08-07T10:04:12.414000+00:00", "forward_to": null, "forward_to_enabled": false, "forward_to_formatted": null, "id": "phon_jhMWlB6anhT8vcsGNEFukaVl806zfxCgbSAAkvtNBpN", "is_group_number": false, "is_verified": false, "label": null, "last_billed_price": "1.15", "mms_enabled": true, "next_billing_on": "2023-09-07", "number": "+14156251293", "number_formatted": "+1 415-625-1293", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "press_1_to_accept": true, "sms_enabled": true, "supports_mms_to_countries": ["CA", "US"], "supports_sms_to_countries": ["CA", "PR", "US"], "type": "internal", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "voicemail_greeting_url": "https://closeio-voicemail-greetings.s3.amazonaws.com/14bzVblrIaekmXSGQEKM11/undefined.mp3"}, "date_created": "2023-08-07T10:04:12.416000", "date_updated": "2023-08-07T10:04:12.416000", "id": "ev_3o6F0cl3A3CkYyoc9vrVjM", "lead_id": null, "meta": {}, "oauth_client_id": null, "oauth_scope": null, "object_id": "phon_jhMWlB6anhT8vcsGNEFukaVl806zfxCgbSAAkvtNBpN", "object_type": "phone_number", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-07-07T10:03:30.945000+00:00", "next_billing_on": "2023-08-07"}, "request_id": null, "user_id": null}, "emitted_at": 1691417122379} -{"stream": "leads", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_label": "Potential", "contacts": [{"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Cooper", "phones": [], "urls": [], "display_name": "Cooper", "id": "cont_Dsi7AGMRelIZ2I6DIKicaGJU7mwxPZElLIHy33xbjCM", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2022-07-05T21:01:25.612000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "emails": [], "title": "", "date_updated": "2022-07-05T21:01:25.612000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Cooper"}], "lead_id": "lead_Eohw2Vf6WOKZHQ97nS1UTL3iV62pAJFX3ROgJ5WT4cY"}], "status_id": "stat_HrZ1aYkkxRORQSxdBcNPT31HkqxkK2w2uWGiK6yjkmK", "id": "lead_Eohw2Vf6WOKZHQ97nS1UTL3iV62pAJFX3ROgJ5WT4cY", "addresses": [], "date_created": "2022-07-05T21:01:25.608000+00:00", "custom": {}, "html_url": "https://app.close.com/lead/lead_Eohw2Vf6WOKZHQ97nS1UTL3iV62pAJFX3ROgJ5WT4cY/", "date_updated": "2022-07-05T21:01:25.658000+00:00", "tasks": [], "name": "Alex", "created_by_name": "Airbyte Team", "display_name": "Alex", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Alex"}], "updated_by_name": "Airbyte Team", "description": "", "url": null, "opportunities": []}, "emitted_at": 1691417125238} -{"stream": "leads", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_label": "Interested", "contacts": [{"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Gob Bluth", "phones": [{"phone": "+12025550186", "phone_formatted": "+1 202-555-0186", "country": "US", "type": "office"}], "urls": [], "display_name": "Gob Bluth", "id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2021-07-13T11:39:04.430000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "emails": [{"email": "bluth@close.com", "type": "office"}], "title": "Magician", "date_updated": "2021-07-13T11:39:04.430000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Gob%20Bluth"}], "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Gatekeeper"]}, {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Tobias F\u00fcnke", "phones": [], "urls": [], "display_name": "Tobias F\u00fcnke", "id": "cont_at5uglNbyasFp2KsoWQpjQmLp4lmmqX2p1nhvmPYytq", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2021-07-13T11:39:04.441000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "emails": [{"email": "tobiasfunke@close.com", "type": "office"}], "title": "Blue Man Group (Understudy)", "date_updated": "2021-07-13T11:39:04.441000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Tobias%20F%C3%BCnke"}], "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Point of Contact"]}], "status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "addresses": [{"address_1": "100 Bluth Drive", "label": "business", "country": "US", "city": "Los Angeles", "zipcode": "90210", "state": "CA", "address_2": null}], "date_created": "2021-07-05T11:39:00.850000+00:00", "custom": {"Current Vendor/Software": "BiffCo", "Industry": "Real estate", "Lead Owner": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "Referral Source": "Google Search"}, "html_url": "https://app.close.com/lead/lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2/", "date_updated": "2022-11-08T16:28:54.174000+00:00", "tasks": [], "name": "Bluth Company (Example\u00a0Lead)", "created_by_name": "Airbyte Team", "display_name": "Bluth Company (Example\u00a0Lead)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Bluth%20Company%20%28Example%C2%A0Lead%29"}], "updated_by_name": "Airbyte Team", "description": null, "url": null, "opportunities": [{"annualized_expected_value": 225000, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_display_name": "Demo Completed", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "status_type": "active", "status_id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "id": "oppo_NkKuhUWfDoErNArh44hw7jkkx7sCl5bhlamtezmX7BE", "date_created": "2021-07-13T11:39:04.817000+00:00", "expected_value": 225000, "lead_name": "Bluth Company (Example\u00a0Lead)", "date_lost": null, "date_updated": "2021-07-13T11:39:04.817000+00:00", "note": "Gob's ready to buy a $3,000 suit.", "value_formatted": "$3,000", "status_label": "Demo Completed", "contact_name": null, "created_by_name": "Airbyte Team", "confidence": 75, "contact_id": null, "value_period": "one_time", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "value": 300000, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_won": "2021-07-16", "value_currency": "USD", "updated_by_name": "Airbyte Team", "annualized_value": 300000, "integration_links": [], "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2"}], "custom.cf_1exVDDcGOEiIdhBhBv2VEGqnpIcJZZqiWkk4O7hbU3D": "BiffCo", "custom.cf_ZuP9X9UjiQzjptNHlT7DxzRATaFil2Ysoz0aGMq0Kim": "Real estate", "custom.cf_mhBoQeiuwFRlz7zqyi4kJgzUreEoUp0hUsLwrnUTEgh": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_05o22yQYHMrFh4cCYCMSQJdSpODdabCbGQ8il5Do7X4": "Google Search"}, "emitted_at": 1691417125245} -{"stream": "leads", "data": {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_label": "Interested", "contacts": [{"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Steli Efti", "phones": [{"phone": "+16505176539", "phone_formatted": "+1 650-517-6539", "country": "US", "type": "office"}], "urls": [], "display_name": "Steli Efti", "id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2021-07-13T11:39:03.354000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "emails": [{"email": "sales@close.com", "type": "office"}], "title": "CEO & Co-Founder", "date_updated": "2021-07-13T11:39:03.354000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Steli%20Efti"}], "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Decision Maker"]}, {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Nick Persico", "phones": [{"phone": "+18334625673", "phone_formatted": "+1 833-462-5673", "country": "US", "type": "office"}], "urls": [], "display_name": "Nick Persico", "id": "cont_FY5ws8upMQQyD9vKg4jzwRb6V3MLctQeNTc2NaUmAyo", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2021-07-13T11:39:03.366000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "emails": [{"email": "nick@close.com", "type": "office"}], "title": "Director of Revenue", "date_updated": "2021-07-13T11:39:03.366000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Nick%20Persico"}], "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Gatekeeper", "Point of Contact"]}, {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Customer Success Team", "phones": [{"phone": "+18334625673", "phone_formatted": "+1 833-462-5673", "country": "US", "type": "office"}], "urls": [], "display_name": "Customer Success Team", "id": "cont_4cmimyQMTMi61kc72mLAV9XdHdddw6LR1LqzvoNdSuV", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2021-07-13T11:39:03.374000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "emails": [{"email": "success@close.com", "type": "office"}], "title": null, "date_updated": "2021-07-13T11:39:03.374000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Customer%20Success%20Team"}], "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz"}, {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "name": "Support", "phones": [], "urls": [], "display_name": "Support", "id": "cont_CI5c6Ekew0cyhSgoFIXeaz2PJVmmtigF2XNIGUDXKnu", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "date_created": "2021-07-13T11:39:03.380000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "emails": [{"email": "support@close.com", "type": "office"}], "title": null, "date_updated": "2021-07-13T11:39:03.380000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Support"}], "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz"}], "status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "addresses": [{"address_1": "PO Box 7775 #69574", "label": "mailing", "country": "US", "city": "San Francisco", "zipcode": "94120", "state": "CA", "address_2": null}], "date_created": "2021-07-13T11:39:03.315000+00:00", "custom": {"Current Vendor/Software": "Stark Industries", "Industry": "Software", "Lead Owner": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "Referral Source": "Website"}, "html_url": "https://app.close.com/lead/lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz/", "date_updated": "2022-11-09T02:40:26.489000+00:00", "tasks": [{"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "view": "archive", "id": "task_bboTdYSlGqTBSXF0FEwhBOywDCV6iKDyMWiEfNFi7sW", "date_created": "2021-07-13T11:39:03.520000+00:00", "lead_name": "Close (Example\u00a0Lead)", "is_dateless": null, "date_updated": "2022-11-08T12:21:15.730000+00:00", "text": "Call Steli", "is_complete": true, "contact_name": null, "created_by_name": "Airbyte Team", "date": "2021-07-18", "contact_id": null, "due_date": "2021-07-18", "assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "object_id": "acti_POPD3uA7lSfdf4xLO7PgsxKeoJS1dCRthzQRb13Xbyw", "_type": "lead", "updated_by_name": "Airbyte Team", "object_type": "taskcompleted", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz"}, {"updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "view": "archive", "id": "task_7kpMfXIPms858l9GKZTo3BCtVflvcIsOYPXL8mZgzyp", "date_created": "2021-07-13T11:39:03.463000+00:00", "lead_name": "Close (Example\u00a0Lead)", "is_dateless": null, "date_updated": "2021-08-18T10:31:52.081000+00:00", "text": "Send Steli an email", "is_complete": true, "contact_name": null, "created_by_name": "Airbyte Team", "date": "2021-07-16", "contact_id": null, "due_date": "2021-07-16", "assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "object_id": "acti_gdh0Iw30XYfKhKctrqPuxAbNghsxlDBDFaN8k35ZAxf", "_type": "lead", "updated_by_name": "Airbyte Team", "object_type": "taskcompleted", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz"}], "name": "Close (Example\u00a0Lead)", "created_by_name": "Airbyte Team", "display_name": "Close (Example\u00a0Lead)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Close%20%28Example%C2%A0Lead%29"}], "updated_by_name": "Airbyte Team", "description": "Visit our blog for high quality sales content, blog.close.com!", "url": "https://close.com", "opportunities": [{"annualized_expected_value": 37500, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_display_name": "Proposal Sent", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "status_type": "active", "status_id": "stat_AWXzFJkvkHVyJQPFulY0wM7LrQRiMEtQChGumG035bH", "id": "oppo_1TfmaSLuECcdSQIBnjUOBj1gAXdyrp4SFaogvcEmtbk", "date_created": "2021-07-13T11:39:04.284000+00:00", "expected_value": 37500, "lead_name": "Close (Example\u00a0Lead)", "date_lost": null, "date_updated": "2021-08-18T10:26:44.306000+00:00", "note": "Use opportunities to track which stage of the pipeline your deals are in and the revenue associated with them.", "value_formatted": "$500", "status_label": "Proposal Sent", "contact_name": "Steli Efti", "created_by_name": "Airbyte Team", "confidence": 75, "contact_id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "value_period": "one_time", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "value": 50000, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_won": "2021-07-15", "value_currency": "USD", "updated_by_name": "Airbyte Team", "annualized_value": 50000, "integration_links": [], "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz"}], "custom.cf_1exVDDcGOEiIdhBhBv2VEGqnpIcJZZqiWkk4O7hbU3D": "Stark Industries", "custom.cf_ZuP9X9UjiQzjptNHlT7DxzRATaFil2Ysoz0aGMq0Kim": "Software", "custom.cf_mhBoQeiuwFRlz7zqyi4kJgzUreEoUp0hUsLwrnUTEgh": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_05o22yQYHMrFh4cCYCMSQJdSpODdabCbGQ8il5Do7X4": "Website"}, "emitted_at": 1691417125251} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T12:58:44.679000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Gob,

      Friendly follow-up.

      I wanted to show you how Airbyte can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

      - Wed @ 11AM
      - Thur @ 2PM
      - Fri @ 3PM
      ", "body_preview": "Hi Gob, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur ", "body_text": "Hi Gob, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur @ 2PM \n\\- Fri @ 3PM", "bulk_email_action_id": "bulkemail_bCJdPbYpJ2xfUuj8Dif3sBMHlxtbOKuEZSCdNcTgsWB", "cc": [], "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T12:58:44.579000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T12:58:44.679000+00:00", "date_updated": "2023-11-10T15:30:43.534000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 12:58:44 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166791232468.5302.9789676641910298968@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Airbyte Follow-up", "to": [{"email": "bluth@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166791232468.5302.9789676641910298968@smtpgw.close.com>", "<166791232468.5302.12366928756676498962@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "66.249.92.10", "opened_at": "2022-11-08T12:58:47.565000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.087000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.539000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.838000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.336000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:30:43.534000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by bluth@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "Airbyte Follow-up", "template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "template_name": "Email 2 - Follow-up #1", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["bluth@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:30:43.537000", "date_updated": "2023-11-10T15:30:43.537000", "id": "ev_0RUlfL6sHJ5SEBgJn9CHf1", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x/M0vtraSSWF.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T15:14:45.336000+00:00", "opens": [{"ip_address": "66.249.92.10", "opened_at": "2022-11-08T12:58:47.565000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.087000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.539000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.838000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.336000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_7O5Zj8MPaQeNzSpDaq1m5N", "user_id": null}, "emitted_at": 1699630289703} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T15:21:48.934000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Gob,

      I'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth Company (Example\u00a0Lead) and show you what we're working on.

      Are you available for a quick call tomorrow afternoon?
      ", "body_preview": "Hi Gob, \n \nI'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR\nPRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth\nCompany (Example Lead) and show y", "body_text": "Hi Gob, \n \nI'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR\nPRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth\nCompany (Example Lead) and show you what we're working on. \n \nAre you available for a quick call tomorrow afternoon?", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "created_by": null, "created_by_name": null, "date_created": "2022-11-08T15:21:49.243000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T15:21:48.934000+00:00", "date_updated": "2023-11-10T15:30:43.185000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 15:21:50 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792090941.10645.3407832652098417673@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Bluth Company (Example\u00a0Lead) + Airbyte", "to": [{"email": "bluth@close.com", "name": "Gob Bluth"}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166792090941.10645.3407832652098417673@smtpgw.close.com>", "<166792090941.10645.8722656406065573871@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "66.249.92.23", "opened_at": "2022-11-08T15:21:52.108000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.461000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.208000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.120000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.534000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:20.249000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.338000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:30:43.185000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Gob Bluth (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": "seq_17gkOeJEV1QPdPaOMJ9mTX", "sequence_name": "Sequence 1", "sequence_subscription_id": "sub_77gWyXwSd44zuQaciQYW0O", "status": "sent", "subject": "Bluth Company (Example\u00a0Lead) + Airbyte", "template_id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "template_name": "Email 1 - Intro", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["Gob Bluth "], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:30:43.187000", "date_updated": "2023-11-10T15:30:43.187000", "id": "ev_2KbeIQIHqBOfD0z0btKUxB", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp/DIwaIsCbZM.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T15:14:45.338000+00:00", "opens": [{"ip_address": "66.249.92.23", "opened_at": "2022-11-08T15:21:52.108000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.461000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.208000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.120000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.534000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:20.249000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.338000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_1WktI0e4spbk65BgvBZPf4", "user_id": null}, "emitted_at": 1699630289709} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T12:56:52.538000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Bruce,

      I'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at Wayne Enterprises (Example Lead) and show you what we're working on.

      Are you available for a quick call tomorrow afternoon?", "body_preview": "Hi Bruce, \n \nI'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT\nYOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at\nWayne Enterprises (Example Lead) an", "body_text": "Hi Bruce, \n \nI'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT\nYOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at\nWayne Enterprises (Example Lead) and show you what we're working on. \n \nAre you available for a quick call tomorrow afternoon?", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T12:54:51.103000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T12:56:52.538000+00:00", "date_updated": "2023-11-10T15:30:43.106000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 12:56:52 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166791221256.1235.11589873218010129427@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Wayne Enterprises (Example\u00a0Lead) + Airbyte", "to": [{"email": "thedarkknight@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe", "in_reply_to_id": null, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "message_ids": ["<166791221256.1235.11589873218010129427@smtpgw.close.com>", "<166791221256.1235.14063741572606059919@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:25:56.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:48:01.421000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:00.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:10.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:28.676000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:24.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:34.029000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:07:41.553000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:10:30.843000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:13:05.125000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:15:47.492000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "85.209.47.207", "opened_at": "2022-11-08T15:43:38.471000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.479000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.770000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.340000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.086000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.538000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.276000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:58.767000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.335000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:30:43.106000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "Wayne Enterprises (Example\u00a0Lead) + Airbyte", "template_id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "template_name": "Email 1 - Intro", "thread_id": "acti_wK6I4m4SqRMvvm0VHR8DhdUL35HRfjIlBkrMVCKWNaU", "to": ["thedarkknight@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:30:43.109000", "date_updated": "2023-11-10T15:30:43.109000", "id": "ev_3tACWCm4grjX9YAlnhQ5xA", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "meta": {"request_method": "GET", "request_path": "/t/ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe/LKMjsg1Cwf.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T15:14:45.335000+00:00", "opens": [{"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:25:56.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:48:01.421000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:00.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:10.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:28.676000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:24.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:34.029000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:07:41.553000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:10:30.843000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:13:05.125000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:15:47.492000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "85.209.47.207", "opened_at": "2022-11-08T15:43:38.471000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.479000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.770000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.340000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.086000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.538000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.276000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:58.767000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.335000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_4wMd9PSmUsCUSfgGimvrqL", "user_id": null}, "emitted_at": 1699630289715} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T15:44:17.803000+00:00", "attachments": [], "bcc": [], "body_html": "Test", "body_preview": "Test", "body_text": "Test", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T15:43:48.819000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T15:44:17.803000+00:00", "date_updated": "2023-11-10T15:30:43.104000+00:00", "direction": "outgoing", "email_account_id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 15:44:18 +0000", "from": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792225784.2629.13990873364570888607@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "subject": "(no subject)", "to": [{"email": "thedarkknight@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs", "in_reply_to_id": null, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "message_ids": ["<166792225784.2629.13990873364570888607@smtpgw.close.com>", "<166792225784.2629.12825104744076060180@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "85.209.47.207", "opened_at": "2022-11-09T11:12:59.685000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.460000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.207000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.084000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.536000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.339000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:30:43.104000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "(no subject)", "template_id": null, "template_name": null, "thread_id": "acti_wK6I4m4SqRMvvm0VHR8DhdUL35HRfjIlBkrMVCKWNaU", "to": ["thedarkknight@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:30:43.106000", "date_updated": "2023-11-10T15:30:43.106000", "id": "ev_5BDijLS75lw6znax3SZyAa", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "meta": {"request_method": "GET", "request_path": "/t/tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs/LKMjsg1Cwf.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T15:14:45.339000+00:00", "opens": [{"ip_address": "85.209.47.207", "opened_at": "2022-11-09T11:12:59.685000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.460000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.207000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.084000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.536000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.339000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_2Nx6LPRvRgu3mdrpqIp5xd", "user_id": null}, "emitted_at": 1699630289720} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T16:09:03.809000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Tobias,

      Friendly follow-up.

      I wanted to show you how Airbyte can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

      - Wed @ 11AM
      - Thur @ 2PM
      - Fri @ 3PM
      ", "body_preview": "Hi Tobias, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Th", "body_text": "Hi Tobias, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur @ 2PM \n\\- Fri @ 3PM", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_at5uglNbyasFp2KsoWQpjQmLp4lmmqX2p1nhvmPYytq", "created_by": null, "created_by_name": null, "date_created": "2022-11-08T16:09:04.170000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T16:09:03.809000+00:00", "date_updated": "2023-11-10T15:30:43.104000+00:00", "direction": "outgoing", "email_account_id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 16:09:04 +0000", "from": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792374433.22448.16558415745556798661@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "subject": "Airbyte Follow-up", "to": [{"email": "tobiasfunke@close.com", "name": "Tobias F\u00fcnke"}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166792374433.22448.16558415745556798661@smtpgw.close.com>", "<166792374433.22448.17907165931371313011@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.922000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.205000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.242000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.540000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.274000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.339000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:30:43.104000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Tobias F\u00fcnke (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": "seq_7gEZ4ByvvLv1rI6szKolhR", "sequence_name": "Sequence 2", "sequence_subscription_id": "sub_1rgSspOohLocAaucazbFbB", "status": "sent", "subject": "Airbyte Follow-up", "template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "template_name": "Email 2 - Follow-up #1", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["Tobias F\u00fcnke "], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:30:43.105000", "date_updated": "2023-11-10T15:30:43.105000", "id": "ev_69tSzwIVfGiBvN8k4y6XlM", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp/lnZoSPgicp.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T15:14:45.339000+00:00", "opens": [{"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.922000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.205000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.242000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.540000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.274000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.339000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_7Ox9Zm8ChxH4nU9ctwgvKj", "user_id": null}, "emitted_at": 1699630289725} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T16:09:03.809000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Tobias,

      Friendly follow-up.

      I wanted to show you how Airbyte can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

      - Wed @ 11AM
      - Thur @ 2PM
      - Fri @ 3PM
      ", "body_preview": "Hi Tobias, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Th", "body_text": "Hi Tobias, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur @ 2PM \n\\- Fri @ 3PM", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_at5uglNbyasFp2KsoWQpjQmLp4lmmqX2p1nhvmPYytq", "created_by": null, "created_by_name": null, "date_created": "2022-11-08T16:09:04.170000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T16:09:03.809000+00:00", "date_updated": "2023-11-10T15:14:45.339000+00:00", "direction": "outgoing", "email_account_id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 16:09:04 +0000", "from": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792374433.22448.16558415745556798661@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "subject": "Airbyte Follow-up", "to": [{"email": "tobiasfunke@close.com", "name": "Tobias F\u00fcnke"}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166792374433.22448.16558415745556798661@smtpgw.close.com>", "<166792374433.22448.17907165931371313011@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.922000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.205000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.242000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.540000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.274000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.339000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Tobias F\u00fcnke (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": "seq_7gEZ4ByvvLv1rI6szKolhR", "sequence_name": "Sequence 2", "sequence_subscription_id": "sub_1rgSspOohLocAaucazbFbB", "status": "sent", "subject": "Airbyte Follow-up", "template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "template_name": "Email 2 - Follow-up #1", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["Tobias F\u00fcnke "], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:14:45.341000", "date_updated": "2023-11-10T15:14:45.341000", "id": "ev_2b2ycyLs8ry3UeIcfbwyhO", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp/lnZoSPgicp.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T15:06:57.840000+00:00", "opens": [{"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.922000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.205000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.242000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.540000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.274000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_60PxPjrZKKbbBHR0Ys3X7s", "user_id": null}, "emitted_at": 1699630289731} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T15:44:17.803000+00:00", "attachments": [], "bcc": [], "body_html": "Test", "body_preview": "Test", "body_text": "Test", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T15:43:48.819000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T15:44:17.803000+00:00", "date_updated": "2023-11-10T15:14:45.339000+00:00", "direction": "outgoing", "email_account_id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 15:44:18 +0000", "from": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792225784.2629.13990873364570888607@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "subject": "(no subject)", "to": [{"email": "thedarkknight@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs", "in_reply_to_id": null, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "message_ids": ["<166792225784.2629.13990873364570888607@smtpgw.close.com>", "<166792225784.2629.12825104744076060180@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "85.209.47.207", "opened_at": "2022-11-09T11:12:59.685000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.460000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.207000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.084000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.536000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.339000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "(no subject)", "template_id": null, "template_name": null, "thread_id": "acti_wK6I4m4SqRMvvm0VHR8DhdUL35HRfjIlBkrMVCKWNaU", "to": ["thedarkknight@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:14:45.340000", "date_updated": "2023-11-10T15:14:45.340000", "id": "ev_2qHLh4rzCWZj4hSW4BYIJD", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "meta": {"request_method": "GET", "request_path": "/t/tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs/LKMjsg1Cwf.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T15:06:57.840000+00:00", "opens": [{"ip_address": "85.209.47.207", "opened_at": "2022-11-09T11:12:59.685000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.460000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.207000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.084000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.536000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_1lvmCiZBTALUDLAXw0LaM8", "user_id": null}, "emitted_at": 1699630289737} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T15:21:48.934000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Gob,

      I'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth Company (Example\u00a0Lead) and show you what we're working on.

      Are you available for a quick call tomorrow afternoon?
      ", "body_preview": "Hi Gob, \n \nI'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR\nPRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth\nCompany (Example Lead) and show y", "body_text": "Hi Gob, \n \nI'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR\nPRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth\nCompany (Example Lead) and show you what we're working on. \n \nAre you available for a quick call tomorrow afternoon?", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "created_by": null, "created_by_name": null, "date_created": "2022-11-08T15:21:49.243000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T15:21:48.934000+00:00", "date_updated": "2023-11-10T15:14:45.338000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 15:21:50 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792090941.10645.3407832652098417673@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Bluth Company (Example\u00a0Lead) + Airbyte", "to": [{"email": "bluth@close.com", "name": "Gob Bluth"}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166792090941.10645.3407832652098417673@smtpgw.close.com>", "<166792090941.10645.8722656406065573871@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "66.249.92.23", "opened_at": "2022-11-08T15:21:52.108000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.461000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.208000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.120000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.534000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:20.249000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.338000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Gob Bluth (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": "seq_17gkOeJEV1QPdPaOMJ9mTX", "sequence_name": "Sequence 1", "sequence_subscription_id": "sub_77gWyXwSd44zuQaciQYW0O", "status": "sent", "subject": "Bluth Company (Example\u00a0Lead) + Airbyte", "template_id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "template_name": "Email 1 - Intro", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["Gob Bluth "], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:14:45.340000", "date_updated": "2023-11-10T15:14:45.340000", "id": "ev_29E1EPZK1xaOxSmT3BBU6Y", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp/DIwaIsCbZM.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T15:06:57.840000+00:00", "opens": [{"ip_address": "66.249.92.23", "opened_at": "2022-11-08T15:21:52.108000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.461000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.208000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.120000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.534000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:20.249000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_79TlUClmIFBrdx1OaUbMz9", "user_id": null}, "emitted_at": 1699630289742} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T12:58:44.679000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Gob,

      Friendly follow-up.

      I wanted to show you how Airbyte can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

      - Wed @ 11AM
      - Thur @ 2PM
      - Fri @ 3PM
      ", "body_preview": "Hi Gob, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur ", "body_text": "Hi Gob, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur @ 2PM \n\\- Fri @ 3PM", "bulk_email_action_id": "bulkemail_bCJdPbYpJ2xfUuj8Dif3sBMHlxtbOKuEZSCdNcTgsWB", "cc": [], "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T12:58:44.579000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T12:58:44.679000+00:00", "date_updated": "2023-11-10T15:14:45.336000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 12:58:44 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166791232468.5302.9789676641910298968@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Airbyte Follow-up", "to": [{"email": "bluth@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166791232468.5302.9789676641910298968@smtpgw.close.com>", "<166791232468.5302.12366928756676498962@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "66.249.92.10", "opened_at": "2022-11-08T12:58:47.565000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.087000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.539000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.838000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.336000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by bluth@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "Airbyte Follow-up", "template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "template_name": "Email 2 - Follow-up #1", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["bluth@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:14:45.338000", "date_updated": "2023-11-10T15:14:45.338000", "id": "ev_6PyTh6KVyKMePoEEm0jwdZ", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x/M0vtraSSWF.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T15:06:57.838000+00:00", "opens": [{"ip_address": "66.249.92.10", "opened_at": "2022-11-08T12:58:47.565000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.087000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.539000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.838000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_2JxdqbgZNqNO4mxjnS9yTp", "user_id": null}, "emitted_at": 1699630289747} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T12:56:52.538000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Bruce,

      I'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at Wayne Enterprises (Example Lead) and show you what we're working on.

      Are you available for a quick call tomorrow afternoon?", "body_preview": "Hi Bruce, \n \nI'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT\nYOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at\nWayne Enterprises (Example Lead) an", "body_text": "Hi Bruce, \n \nI'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT\nYOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at\nWayne Enterprises (Example Lead) and show you what we're working on. \n \nAre you available for a quick call tomorrow afternoon?", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T12:54:51.103000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T12:56:52.538000+00:00", "date_updated": "2023-11-10T15:14:45.335000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 12:56:52 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166791221256.1235.11589873218010129427@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Wayne Enterprises (Example\u00a0Lead) + Airbyte", "to": [{"email": "thedarkknight@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe", "in_reply_to_id": null, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "message_ids": ["<166791221256.1235.11589873218010129427@smtpgw.close.com>", "<166791221256.1235.14063741572606059919@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:25:56.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:48:01.421000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:00.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:10.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:28.676000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:24.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:34.029000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:07:41.553000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:10:30.843000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:13:05.125000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:15:47.492000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "85.209.47.207", "opened_at": "2022-11-08T15:43:38.471000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.479000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.770000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.340000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.086000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.538000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.276000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:58.767000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:14:45.335000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "Wayne Enterprises (Example\u00a0Lead) + Airbyte", "template_id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "template_name": "Email 1 - Intro", "thread_id": "acti_wK6I4m4SqRMvvm0VHR8DhdUL35HRfjIlBkrMVCKWNaU", "to": ["thedarkknight@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:14:45.336000", "date_updated": "2023-11-10T15:14:45.336000", "id": "ev_0d8wShZMOSQeoUTw6w9nyX", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "meta": {"request_method": "GET", "request_path": "/t/ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe/LKMjsg1Cwf.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T15:06:58.767000+00:00", "opens": [{"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:25:56.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:48:01.421000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:00.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:10.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:28.676000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:24.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:34.029000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:07:41.553000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:10:30.843000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:13:05.125000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:15:47.492000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "85.209.47.207", "opened_at": "2022-11-08T15:43:38.471000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.479000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.770000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.340000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.086000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.538000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.276000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:58.767000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_0iKLh9PrBVh2nUq62pYqQL", "user_id": null}, "emitted_at": 1699630289752} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T12:56:52.538000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Bruce,

      I'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at Wayne Enterprises (Example Lead) and show you what we're working on.

      Are you available for a quick call tomorrow afternoon?", "body_preview": "Hi Bruce, \n \nI'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT\nYOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at\nWayne Enterprises (Example Lead) an", "body_text": "Hi Bruce, \n \nI'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT\nYOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at\nWayne Enterprises (Example Lead) and show you what we're working on. \n \nAre you available for a quick call tomorrow afternoon?", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T12:54:51.103000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T12:56:52.538000+00:00", "date_updated": "2023-11-10T15:06:58.767000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 12:56:52 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166791221256.1235.11589873218010129427@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Wayne Enterprises (Example\u00a0Lead) + Airbyte", "to": [{"email": "thedarkknight@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe", "in_reply_to_id": null, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "message_ids": ["<166791221256.1235.11589873218010129427@smtpgw.close.com>", "<166791221256.1235.14063741572606059919@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:25:56.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:48:01.421000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:00.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:10.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:28.676000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:24.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:34.029000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:07:41.553000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:10:30.843000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:13:05.125000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:15:47.492000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "85.209.47.207", "opened_at": "2022-11-08T15:43:38.471000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.479000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.770000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.340000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.086000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.538000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.276000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:58.767000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "Wayne Enterprises (Example\u00a0Lead) + Airbyte", "template_id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "template_name": "Email 1 - Intro", "thread_id": "acti_wK6I4m4SqRMvvm0VHR8DhdUL35HRfjIlBkrMVCKWNaU", "to": ["thedarkknight@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:06:58.770000", "date_updated": "2023-11-10T15:06:58.770000", "id": "ev_55owS3VVOSN5ym6nGjJYnp", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "meta": {"request_method": "GET", "request_path": "/t/ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe/LKMjsg1Cwf.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T14:41:19.276000+00:00", "opens": [{"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:25:56.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:48:01.421000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:00.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:10.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:28.676000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:24.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:34.029000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:07:41.553000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:10:30.843000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:13:05.125000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:15:47.492000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "85.209.47.207", "opened_at": "2022-11-08T15:43:38.471000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.479000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.770000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.340000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.086000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.538000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.276000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_5s2fFhSgvg58QClPZQdkxe", "user_id": null}, "emitted_at": 1699630289758} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T15:21:48.934000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Gob,

      I'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth Company (Example\u00a0Lead) and show you what we're working on.

      Are you available for a quick call tomorrow afternoon?
      ", "body_preview": "Hi Gob, \n \nI'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR\nPRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth\nCompany (Example Lead) and show y", "body_text": "Hi Gob, \n \nI'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR\nPRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth\nCompany (Example Lead) and show you what we're working on. \n \nAre you available for a quick call tomorrow afternoon?", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "created_by": null, "created_by_name": null, "date_created": "2022-11-08T15:21:49.243000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T15:21:48.934000+00:00", "date_updated": "2023-11-10T15:06:57.840000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 15:21:50 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792090941.10645.3407832652098417673@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Bluth Company (Example\u00a0Lead) + Airbyte", "to": [{"email": "bluth@close.com", "name": "Gob Bluth"}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166792090941.10645.3407832652098417673@smtpgw.close.com>", "<166792090941.10645.8722656406065573871@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "66.249.92.23", "opened_at": "2022-11-08T15:21:52.108000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.461000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.208000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.120000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.534000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:20.249000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Gob Bluth (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": "seq_17gkOeJEV1QPdPaOMJ9mTX", "sequence_name": "Sequence 1", "sequence_subscription_id": "sub_77gWyXwSd44zuQaciQYW0O", "status": "sent", "subject": "Bluth Company (Example\u00a0Lead) + Airbyte", "template_id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "template_name": "Email 1 - Intro", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["Gob Bluth "], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:06:57.842000", "date_updated": "2023-11-10T15:06:57.842000", "id": "ev_4d4oWnQeD4olrMOlNcH30C", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp/DIwaIsCbZM.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T14:41:20.249000+00:00", "opens": [{"ip_address": "66.249.92.23", "opened_at": "2022-11-08T15:21:52.108000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.461000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.208000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.120000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.534000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:20.249000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_6mJCuOuq0mvtvmWdyDCcwk", "user_id": null}, "emitted_at": 1699630289763} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T15:44:17.803000+00:00", "attachments": [], "bcc": [], "body_html": "Test", "body_preview": "Test", "body_text": "Test", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T15:43:48.819000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T15:44:17.803000+00:00", "date_updated": "2023-11-10T15:06:57.840000+00:00", "direction": "outgoing", "email_account_id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 15:44:18 +0000", "from": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792225784.2629.13990873364570888607@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "subject": "(no subject)", "to": [{"email": "thedarkknight@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs", "in_reply_to_id": null, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "message_ids": ["<166792225784.2629.13990873364570888607@smtpgw.close.com>", "<166792225784.2629.12825104744076060180@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "85.209.47.207", "opened_at": "2022-11-09T11:12:59.685000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.460000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.207000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.084000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.536000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "(no subject)", "template_id": null, "template_name": null, "thread_id": "acti_wK6I4m4SqRMvvm0VHR8DhdUL35HRfjIlBkrMVCKWNaU", "to": ["thedarkknight@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:06:57.842000", "date_updated": "2023-11-10T15:06:57.842000", "id": "ev_3NPqRa3FAbQINcjzP9OvnO", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "meta": {"request_method": "GET", "request_path": "/t/tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs/LKMjsg1Cwf.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T14:41:19.278000+00:00", "opens": [{"ip_address": "85.209.47.207", "opened_at": "2022-11-09T11:12:59.685000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.460000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.207000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.084000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.536000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_4ZNBg8yzqoRKPsqTdPquZS", "user_id": null}, "emitted_at": 1699630289768} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T16:09:03.809000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Tobias,

      Friendly follow-up.

      I wanted to show you how Airbyte can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

      - Wed @ 11AM
      - Thur @ 2PM
      - Fri @ 3PM
      ", "body_preview": "Hi Tobias, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Th", "body_text": "Hi Tobias, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur @ 2PM \n\\- Fri @ 3PM", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_at5uglNbyasFp2KsoWQpjQmLp4lmmqX2p1nhvmPYytq", "created_by": null, "created_by_name": null, "date_created": "2022-11-08T16:09:04.170000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T16:09:03.809000+00:00", "date_updated": "2023-11-10T15:06:57.840000+00:00", "direction": "outgoing", "email_account_id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 16:09:04 +0000", "from": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792374433.22448.16558415745556798661@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "subject": "Airbyte Follow-up", "to": [{"email": "tobiasfunke@close.com", "name": "Tobias F\u00fcnke"}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166792374433.22448.16558415745556798661@smtpgw.close.com>", "<166792374433.22448.17907165931371313011@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.922000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.205000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.242000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.540000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.274000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.840000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Tobias F\u00fcnke (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": "seq_7gEZ4ByvvLv1rI6szKolhR", "sequence_name": "Sequence 2", "sequence_subscription_id": "sub_1rgSspOohLocAaucazbFbB", "status": "sent", "subject": "Airbyte Follow-up", "template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "template_name": "Email 2 - Follow-up #1", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["Tobias F\u00fcnke "], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:06:57.841000", "date_updated": "2023-11-10T15:06:57.841000", "id": "ev_5eJEEFysXUZesjkFWky3h4", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp/lnZoSPgicp.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T14:41:19.274000+00:00", "opens": [{"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.922000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.205000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.242000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.540000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.274000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_5AGdFTB2Wtd3QunmnoPpCM", "user_id": null}, "emitted_at": 1699630289773} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens"], "data": {"_type": "Email", "activity_at": "2022-11-08T12:58:44.679000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Gob,

      Friendly follow-up.

      I wanted to show you how Airbyte can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

      - Wed @ 11AM
      - Thur @ 2PM
      - Fri @ 3PM
      ", "body_preview": "Hi Gob, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur ", "body_text": "Hi Gob, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur @ 2PM \n\\- Fri @ 3PM", "bulk_email_action_id": "bulkemail_bCJdPbYpJ2xfUuj8Dif3sBMHlxtbOKuEZSCdNcTgsWB", "cc": [], "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T12:58:44.579000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T12:58:44.679000+00:00", "date_updated": "2023-11-10T15:06:57.838000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 12:58:44 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166791232468.5302.9789676641910298968@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Airbyte Follow-up", "to": [{"email": "bluth@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166791232468.5302.9789676641910298968@smtpgw.close.com>", "<166791232468.5302.12366928756676498962@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "66.249.92.10", "opened_at": "2022-11-08T12:58:47.565000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.087000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.539000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T15:06:57.838000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by bluth@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "Airbyte Follow-up", "template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "template_name": "Email 2 - Follow-up #1", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["bluth@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T15:06:57.840000", "date_updated": "2023-11-10T15:06:57.840000", "id": "ev_7Pcuc4cBZ0sJpICFbiXdM2", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x/M0vtraSSWF.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-10T14:41:19.279000+00:00", "opens": [{"ip_address": "66.249.92.10", "opened_at": "2022-11-08T12:58:47.565000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.087000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.539000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}]}, "request_id": "req_2f6VqlA9IdvNnjzNibe5WW", "user_id": null}, "emitted_at": 1699630289777} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens", "opens_summary"], "data": {"_type": "Email", "activity_at": "2022-11-08T15:21:48.934000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Gob,

      I'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth Company (Example\u00a0Lead) and show you what we're working on.

      Are you available for a quick call tomorrow afternoon?
      ", "body_preview": "Hi Gob, \n \nI'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR\nPRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth\nCompany (Example Lead) and show y", "body_text": "Hi Gob, \n \nI'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR\nPRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth\nCompany (Example Lead) and show you what we're working on. \n \nAre you available for a quick call tomorrow afternoon?", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "created_by": null, "created_by_name": null, "date_created": "2022-11-08T15:21:49.243000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T15:21:48.934000+00:00", "date_updated": "2023-11-10T14:41:20.249000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 15:21:50 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792090941.10645.3407832652098417673@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Bluth Company (Example\u00a0Lead) + Airbyte", "to": [{"email": "bluth@close.com", "name": "Gob Bluth"}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166792090941.10645.3407832652098417673@smtpgw.close.com>", "<166792090941.10645.8722656406065573871@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "66.249.92.23", "opened_at": "2022-11-08T15:21:52.108000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.461000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.208000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.120000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.534000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:20.249000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Gob Bluth (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": "seq_17gkOeJEV1QPdPaOMJ9mTX", "sequence_name": "Sequence 1", "sequence_subscription_id": "sub_77gWyXwSd44zuQaciQYW0O", "status": "sent", "subject": "Bluth Company (Example\u00a0Lead) + Airbyte", "template_id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "template_name": "Email 1 - Intro", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["Gob Bluth "], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T14:41:20.253000", "date_updated": "2023-11-10T14:41:20.253000", "id": "ev_6dudv39N6yamql3fp3n1mZ", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp/DIwaIsCbZM.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-07T17:16:57.534000+00:00", "opens": [{"ip_address": "66.249.92.23", "opened_at": "2022-11-08T15:21:52.108000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.461000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.208000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.120000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.534000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Gob Bluth (5+ times, latest 2023-11-07)"}, "request_id": "req_2HPaptmDh9wUootbcVVmAU", "user_id": null}, "emitted_at": 1699630289781} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens", "opens_summary"], "data": {"_type": "Email", "activity_at": "2022-11-08T12:58:44.679000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Gob,

      Friendly follow-up.

      I wanted to show you how Airbyte can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

      - Wed @ 11AM
      - Thur @ 2PM
      - Fri @ 3PM
      ", "body_preview": "Hi Gob, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur ", "body_text": "Hi Gob, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur @ 2PM \n\\- Fri @ 3PM", "bulk_email_action_id": "bulkemail_bCJdPbYpJ2xfUuj8Dif3sBMHlxtbOKuEZSCdNcTgsWB", "cc": [], "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T12:58:44.579000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T12:58:44.679000+00:00", "date_updated": "2023-11-10T14:41:19.279000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 12:58:44 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166791232468.5302.9789676641910298968@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Airbyte Follow-up", "to": [{"email": "bluth@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166791232468.5302.9789676641910298968@smtpgw.close.com>", "<166791232468.5302.12366928756676498962@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "66.249.92.10", "opened_at": "2022-11-08T12:58:47.565000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.087000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.539000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by bluth@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "Airbyte Follow-up", "template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "template_name": "Email 2 - Follow-up #1", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["bluth@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T14:41:19.283000", "date_updated": "2023-11-10T14:41:19.283000", "id": "ev_7Ju4WMcrHkfBGyXZhpuZIu", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x/M0vtraSSWF.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-07T17:16:57.539000+00:00", "opens": [{"ip_address": "66.249.92.10", "opened_at": "2022-11-08T12:58:47.565000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.087000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.539000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by bluth@close.com (5+ times, latest 2023-11-07)"}, "request_id": "req_1j1zj4MDzVzRdrQLO9KioF", "user_id": null}, "emitted_at": 1699630289787} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens", "opens_summary"], "data": {"_type": "Email", "activity_at": "2022-11-08T15:44:17.803000+00:00", "attachments": [], "bcc": [], "body_html": "Test", "body_preview": "Test", "body_text": "Test", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T15:43:48.819000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T15:44:17.803000+00:00", "date_updated": "2023-11-10T14:41:19.278000+00:00", "direction": "outgoing", "email_account_id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 15:44:18 +0000", "from": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792225784.2629.13990873364570888607@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "subject": "(no subject)", "to": [{"email": "thedarkknight@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs", "in_reply_to_id": null, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "message_ids": ["<166792225784.2629.13990873364570888607@smtpgw.close.com>", "<166792225784.2629.12825104744076060180@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "85.209.47.207", "opened_at": "2022-11-09T11:12:59.685000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.460000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.207000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.084000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.536000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.278000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "(no subject)", "template_id": null, "template_name": null, "thread_id": "acti_wK6I4m4SqRMvvm0VHR8DhdUL35HRfjIlBkrMVCKWNaU", "to": ["thedarkknight@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T14:41:19.281000", "date_updated": "2023-11-10T14:41:19.281000", "id": "ev_1srZ3pOsXl7lK7GkGZKY8h", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "meta": {"request_method": "GET", "request_path": "/t/tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs/LKMjsg1Cwf.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-07T17:16:57.536000+00:00", "opens": [{"ip_address": "85.209.47.207", "opened_at": "2022-11-09T11:12:59.685000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.460000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.207000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.084000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.536000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-07)"}, "request_id": "req_22HCK1MouFKeXP7DWW88UW", "user_id": null}, "emitted_at": 1699630289792} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens", "opens_summary"], "data": {"_type": "Email", "activity_at": "2022-11-08T12:56:52.538000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Bruce,

      I'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at Wayne Enterprises (Example Lead) and show you what we're working on.

      Are you available for a quick call tomorrow afternoon?", "body_preview": "Hi Bruce, \n \nI'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT\nYOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at\nWayne Enterprises (Example Lead) an", "body_text": "Hi Bruce, \n \nI'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT\nYOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at\nWayne Enterprises (Example Lead) and show you what we're working on. \n \nAre you available for a quick call tomorrow afternoon?", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T12:54:51.103000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T12:56:52.538000+00:00", "date_updated": "2023-11-10T14:41:19.276000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 12:56:52 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166791221256.1235.11589873218010129427@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Wayne Enterprises (Example\u00a0Lead) + Airbyte", "to": [{"email": "thedarkknight@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe", "in_reply_to_id": null, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "message_ids": ["<166791221256.1235.11589873218010129427@smtpgw.close.com>", "<166791221256.1235.14063741572606059919@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:25:56.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:48:01.421000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:00.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:10.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:28.676000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:24.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:34.029000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:07:41.553000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:10:30.843000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:13:05.125000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:15:47.492000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "85.209.47.207", "opened_at": "2022-11-08T15:43:38.471000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.479000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.770000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.340000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.086000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.538000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.276000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "Wayne Enterprises (Example\u00a0Lead) + Airbyte", "template_id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "template_name": "Email 1 - Intro", "thread_id": "acti_wK6I4m4SqRMvvm0VHR8DhdUL35HRfjIlBkrMVCKWNaU", "to": ["thedarkknight@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T14:41:19.278000", "date_updated": "2023-11-10T14:41:19.278000", "id": "ev_0j9qPXyGEdfiSdhiCn1nGw", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "meta": {"request_method": "GET", "request_path": "/t/ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe/LKMjsg1Cwf.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-07T17:16:57.538000+00:00", "opens": [{"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:25:56.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:48:01.421000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:00.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:10.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:28.676000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:24.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:34.029000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:07:41.553000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:10:30.843000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:13:05.125000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:15:47.492000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "85.209.47.207", "opened_at": "2022-11-08T15:43:38.471000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.479000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.770000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.340000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.086000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.538000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-07)"}, "request_id": "req_2qQadxDEP4DZZiDMWB9USz", "user_id": null}, "emitted_at": 1699630289796} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens", "opens_summary"], "data": {"_type": "Email", "activity_at": "2022-11-08T16:09:03.809000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Tobias,

      Friendly follow-up.

      I wanted to show you how Airbyte can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

      - Wed @ 11AM
      - Thur @ 2PM
      - Fri @ 3PM
      ", "body_preview": "Hi Tobias, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Th", "body_text": "Hi Tobias, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur @ 2PM \n\\- Fri @ 3PM", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_at5uglNbyasFp2KsoWQpjQmLp4lmmqX2p1nhvmPYytq", "created_by": null, "created_by_name": null, "date_created": "2022-11-08T16:09:04.170000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T16:09:03.809000+00:00", "date_updated": "2023-11-10T14:41:19.274000+00:00", "direction": "outgoing", "email_account_id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 16:09:04 +0000", "from": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792374433.22448.16558415745556798661@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "subject": "Airbyte Follow-up", "to": [{"email": "tobiasfunke@close.com", "name": "Tobias F\u00fcnke"}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166792374433.22448.16558415745556798661@smtpgw.close.com>", "<166792374433.22448.17907165931371313011@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.922000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.205000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.242000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.540000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-10T14:41:19.274000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Tobias F\u00fcnke (5+ times, latest 2023-11-10)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": "seq_7gEZ4ByvvLv1rI6szKolhR", "sequence_name": "Sequence 2", "sequence_subscription_id": "sub_1rgSspOohLocAaucazbFbB", "status": "sent", "subject": "Airbyte Follow-up", "template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "template_name": "Email 2 - Follow-up #1", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["Tobias F\u00fcnke "], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-10T14:41:19.277000", "date_updated": "2023-11-10T14:41:19.277000", "id": "ev_5j7Pk6YNQU9E9nZjwCLe6t", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp/lnZoSPgicp.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-11-07T17:16:57.540000+00:00", "opens": [{"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.922000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.205000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.242000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.540000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Tobias F\u00fcnke (5+ times, latest 2023-11-07)"}, "request_id": "req_2c5eao433qyUjb3hiqes6f", "user_id": null}, "emitted_at": 1699630289801} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens", "opens_summary"], "data": {"_type": "Email", "activity_at": "2022-11-08T16:09:03.809000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Tobias,

      Friendly follow-up.

      I wanted to show you how Airbyte can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

      - Wed @ 11AM
      - Thur @ 2PM
      - Fri @ 3PM
      ", "body_preview": "Hi Tobias, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Th", "body_text": "Hi Tobias, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur @ 2PM \n\\- Fri @ 3PM", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_at5uglNbyasFp2KsoWQpjQmLp4lmmqX2p1nhvmPYytq", "created_by": null, "created_by_name": null, "date_created": "2022-11-08T16:09:04.170000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T16:09:03.809000+00:00", "date_updated": "2023-11-07T17:16:57.540000+00:00", "direction": "outgoing", "email_account_id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 16:09:04 +0000", "from": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792374433.22448.16558415745556798661@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "subject": "Airbyte Follow-up", "to": [{"email": "tobiasfunke@close.com", "name": "Tobias F\u00fcnke"}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166792374433.22448.16558415745556798661@smtpgw.close.com>", "<166792374433.22448.17907165931371313011@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.922000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.205000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.242000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.540000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Tobias F\u00fcnke (5+ times, latest 2023-11-07)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": "seq_7gEZ4ByvvLv1rI6szKolhR", "sequence_name": "Sequence 2", "sequence_subscription_id": "sub_1rgSspOohLocAaucazbFbB", "status": "sent", "subject": "Airbyte Follow-up", "template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "template_name": "Email 2 - Follow-up #1", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["Tobias F\u00fcnke "], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-07T17:16:57.542000", "date_updated": "2023-11-07T17:16:57.542000", "id": "ev_3yPOiNAqhOB5tzHLQlaPXq", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp/lnZoSPgicp.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_3XzsaHLl42CAYwO98zCzxA5h5SScscYmxqg856bFvdp", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-08-15T13:46:12.242000+00:00", "opens": [{"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.922000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.205000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.242000+00:00", "opened_by": "Tobias F\u00fcnke ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Tobias F\u00fcnke (5 times, latest 2023-08-15)"}, "request_id": "req_59kkiOFtRVDLGEHmetdbZh", "user_id": null}, "emitted_at": 1699630289805} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens", "opens_summary"], "data": {"_type": "Email", "activity_at": "2022-11-08T12:58:44.679000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Gob,

      Friendly follow-up.

      I wanted to show you how Airbyte can help you with [INSERT YOUR PRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week?

      - Wed @ 11AM
      - Thur @ 2PM
      - Fri @ 3PM
      ", "body_preview": "Hi Gob, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur ", "body_text": "Hi Gob, \n \nFriendly follow-up. \n \nI wanted to show you how Airbyte can help you with [INSERT YOUR\nPRODUCT/SERVICE]. Do you have 15 minutes for a quick call this week? \n \n\\- Wed @ 11AM \n\\- Thur @ 2PM \n\\- Fri @ 3PM", "bulk_email_action_id": "bulkemail_bCJdPbYpJ2xfUuj8Dif3sBMHlxtbOKuEZSCdNcTgsWB", "cc": [], "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T12:58:44.579000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T12:58:44.679000+00:00", "date_updated": "2023-11-07T17:16:57.539000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 12:58:44 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166791232468.5302.9789676641910298968@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Airbyte Follow-up", "to": [{"email": "bluth@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166791232468.5302.9789676641910298968@smtpgw.close.com>", "<166791232468.5302.12366928756676498962@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "66.249.92.10", "opened_at": "2022-11-08T12:58:47.565000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.087000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.539000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by bluth@close.com (5+ times, latest 2023-11-07)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "Airbyte Follow-up", "template_id": "tmpl_Cilxd4yapDRweK6caKOTGzSHqyP9ddiEz4G0DLg0nwt", "template_name": "Email 2 - Follow-up #1", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["bluth@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-07T17:16:57.541000", "date_updated": "2023-11-07T17:16:57.541000", "id": "ev_45XKOg0O0KTHiJF0cQAH34", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x/M0vtraSSWF.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_uTYUIkVvkt5gjJjn5D3RtEIveSqwuP6pCVwC6vNA69x", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-08-15T13:46:12.087000+00:00", "opens": [{"ip_address": "66.249.92.10", "opened_at": "2022-11-08T12:58:47.565000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.459000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.087000+00:00", "opened_by": "bluth@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}], "opens_summary": "Opened by bluth@close.com (5+ times, latest 2023-08-15)"}, "request_id": "req_1l0ppWWlua2dq9abxQBEQK", "user_id": null}, "emitted_at": 1699630289810} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens", "opens_summary"], "data": {"_type": "Email", "activity_at": "2022-11-08T12:56:52.538000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Bruce,

      I'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at Wayne Enterprises (Example Lead) and show you what we're working on.

      Are you available for a quick call tomorrow afternoon?", "body_preview": "Hi Bruce, \n \nI'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT\nYOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at\nWayne Enterprises (Example Lead) an", "body_text": "Hi Bruce, \n \nI'm Jean with Airbyte. We help companies in the Manufacturing space [INSERT\nYOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at\nWayne Enterprises (Example Lead) and show you what we're working on. \n \nAre you available for a quick call tomorrow afternoon?", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T12:54:51.103000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T12:56:52.538000+00:00", "date_updated": "2023-11-07T17:16:57.538000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 12:56:52 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166791221256.1235.11589873218010129427@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Wayne Enterprises (Example\u00a0Lead) + Airbyte", "to": [{"email": "thedarkknight@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe", "in_reply_to_id": null, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "message_ids": ["<166791221256.1235.11589873218010129427@smtpgw.close.com>", "<166791221256.1235.14063741572606059919@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:25:56.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:48:01.421000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:00.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:10.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:28.676000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:24.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:34.029000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:07:41.553000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:10:30.843000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:13:05.125000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:15:47.492000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "85.209.47.207", "opened_at": "2022-11-08T15:43:38.471000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.479000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.770000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.340000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.086000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.538000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-07)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "Wayne Enterprises (Example\u00a0Lead) + Airbyte", "template_id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "template_name": "Email 1 - Intro", "thread_id": "acti_wK6I4m4SqRMvvm0VHR8DhdUL35HRfjIlBkrMVCKWNaU", "to": ["thedarkknight@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-07T17:16:57.540000", "date_updated": "2023-11-07T17:16:57.540000", "id": "ev_3TtqVtgYKK8DFWvPVDLgy9", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "meta": {"request_method": "GET", "request_path": "/t/ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe/LKMjsg1Cwf.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_ZU8MvgvFu21A7QMkYFD5SAI8EqrITmfJuu4f5FJDnCe", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-08-15T13:46:12.086000+00:00", "opens": [{"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:25:56.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:48:01.421000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:00.139000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:10.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T14:59:28.676000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:24.078000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:00:34.029000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:07:41.553000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:10:30.843000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:13:05.125000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "185.143.147.236", "opened_at": "2022-11-08T15:15:47.492000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "85.209.47.207", "opened_at": "2022-11-08T15:43:38.471000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.479000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.770000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.340000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.206000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.086000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-08-15)"}, "request_id": "req_6129LX7KFu95RZWKBShau3", "user_id": null}, "emitted_at": 1699630289814} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens", "opens_summary"], "data": {"_type": "Email", "activity_at": "2022-11-08T15:44:17.803000+00:00", "attachments": [], "bcc": [], "body_html": "Test", "body_preview": "Test", "body_text": "Test", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date_created": "2022-11-08T15:43:48.819000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T15:44:17.803000+00:00", "date_updated": "2023-11-07T17:16:57.536000+00:00", "direction": "outgoing", "email_account_id": "emailacct_chcWvlCbL58B28cadf8XzEjQymaAMyJW6uqGjdiqmHK", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 15:44:18 +0000", "from": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792225784.2629.13990873364570888607@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "integration-test@airbyte.io", "name": "Jean Lafleur"}], "subject": "(no subject)", "to": [{"email": "thedarkknight@close.com", "name": ""}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs", "in_reply_to_id": null, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "message_ids": ["<166792225784.2629.13990873364570888607@smtpgw.close.com>", "<166792225784.2629.12825104744076060180@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "85.209.47.207", "opened_at": "2022-11-09T11:12:59.685000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.460000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.207000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.084000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.536000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-11-07)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": null, "sequence_name": null, "sequence_subscription_id": null, "status": "sent", "subject": "(no subject)", "template_id": null, "template_name": null, "thread_id": "acti_wK6I4m4SqRMvvm0VHR8DhdUL35HRfjIlBkrMVCKWNaU", "to": ["thedarkknight@close.com"], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-07T17:16:57.538000", "date_updated": "2023-11-07T17:16:57.538000", "id": "ev_4fYnwSM4MPvqo3VmdFLyEz", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "meta": {"request_method": "GET", "request_path": "/t/tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs/LKMjsg1Cwf.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_tDbd3J7HQmX2I5dBWJSXrlwCdKpZedpsTdAWLsYPzUs", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-08-15T13:46:12.084000+00:00", "opens": [{"ip_address": "85.209.47.207", "opened_at": "2022-11-09T11:12:59.685000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36 Edg/106.0.1370.42"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.460000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.342000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.207000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.084000+00:00", "opened_by": "thedarkknight@close.com", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}], "opens_summary": "Opened by thedarkknight@close.com (5+ times, latest 2023-08-15)"}, "request_id": "req_2o2D7qw5scrSNtN097kCp1", "user_id": null}, "emitted_at": 1699630289819} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "opens", "opens_summary"], "data": {"_type": "Email", "activity_at": "2022-11-08T15:21:48.934000+00:00", "attachments": [], "bcc": [], "body_html": "Hi Gob,

      I'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR PRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth Company (Example\u00a0Lead) and show you what we're working on.

      Are you available for a quick call tomorrow afternoon?
      ", "body_preview": "Hi Gob, \n \nI'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR\nPRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth\nCompany (Example Lead) and show y", "body_text": "Hi Gob, \n \nI'm Jean with Airbyte. We help companies in the Real estate space [INSERT YOUR\nPRODUCT/SERVICE]. I wanted to learn how you handle this currently at Bluth\nCompany (Example Lead) and show you what we're working on. \n \nAre you available for a quick call tomorrow afternoon?", "bulk_email_action_id": null, "cc": [], "contact_id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "created_by": null, "created_by_name": null, "date_created": "2022-11-08T15:21:49.243000+00:00", "date_scheduled": null, "date_sent": "2022-11-08T15:21:48.934000+00:00", "date_updated": "2023-11-07T17:16:57.534000+00:00", "direction": "outgoing", "email_account_id": "emailacct_QeGtVE7epttFYuPJqtqVqaKgW7BEq5q7jHx8M2IHxwe", "envelope": {"bcc": [], "cc": [], "date": "Tue, 08 Nov 2022 15:21:50 +0000", "from": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "in_reply_to": null, "is_autoreply": false, "message_id": "<166792090941.10645.3407832652098417673@smtpgw.close.com>", "reply_to": [], "sender": [{"email": "iryna.grankova@airbyte.io", "name": "Jean Lafleur"}], "subject": "Bluth Company (Example\u00a0Lead) + Airbyte", "to": [{"email": "bluth@close.com", "name": "Gob Bluth"}]}, "followup_sequence_delay": null, "followup_sequence_id": null, "has_reply": false, "id": "acti_AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp", "in_reply_to_id": null, "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "message_ids": ["<166792090941.10645.3407832652098417673@smtpgw.close.com>", "<166792090941.10645.8722656406065573871@smtpgw.close.com>"], "need_smtp_credentials": false, "opens": [{"ip_address": "66.249.92.23", "opened_at": "2022-11-08T15:21:52.108000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.461000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.208000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.120000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "186.215.116.19", "opened_at": "2023-11-07T17:16:57.534000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Gob Bluth (5+ times, latest 2023-11-07)", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "references": [], "send_as_id": null, "send_attempts": [], "sender": "Jean Lafleur ", "sequence_id": "seq_17gkOeJEV1QPdPaOMJ9mTX", "sequence_name": "Sequence 1", "sequence_subscription_id": "sub_77gWyXwSd44zuQaciQYW0O", "status": "sent", "subject": "Bluth Company (Example\u00a0Lead) + Airbyte", "template_id": "tmpl_qbI7mmvEPJla4qdCeBygtzgZ7twup69mdEr2CEVNHIM", "template_name": "Email 1 - Intro", "thread_id": "acti_4XmjyRgIPIH5OSEa18MH6FFVPf0JnS65X8AX8KucQwJ", "to": ["Gob Bluth "], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "user_name": "Airbyte Team", "users": []}, "date_created": "2023-11-07T17:16:57.536000", "date_updated": "2023-11-07T17:16:57.536000", "id": "ev_2ZSra6V4WKPGYvErNty0Ms", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "meta": {"request_method": "GET", "request_path": "/t/AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp/DIwaIsCbZM.png"}, "oauth_client_id": null, "oauth_scope": null, "object_id": "acti_AiN0xVV0ZqsaXF2lowGGjtav6fPvWMVvuqCd5ihHLtp", "object_type": "activity.email", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-08-15T13:46:12.120000+00:00", "opens": [{"ip_address": "66.249.92.23", "opened_at": "2022-11-08T15:21:52.108000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0"}, {"ip_address": "213.110.150.76", "opened_at": "2023-08-14T06:39:56.461000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-14T07:32:52.771000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "89.105.242.251", "opened_at": "2023-08-14T11:02:57.341000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "45.89.89.227", "opened_at": "2023-08-15T11:01:54.208000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}, {"ip_address": "88.142.221.221", "opened_at": "2023-08-15T13:46:12.120000+00:00", "opened_by": "Gob Bluth ", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36"}], "opens_summary": "Opened by Gob Bluth (5+ times, latest 2023-08-15)"}, "request_id": "req_4AquNjQSZfA3wuLQ129LCR", "user_id": null}, "emitted_at": 1699630289823} +{"stream": "events", "data": {"action": "updated", "api_key_id": null, "changed_fields": ["date_updated", "next_billing_on"], "data": {"address_id": null, "bundle_id": null, "carrier": "twilio", "carrier_type": "local", "country": "US", "date_created": "2022-11-08T12:35:29.464000+00:00", "date_updated": "2023-11-07T10:04:24.797000+00:00", "forward_to": null, "forward_to_enabled": false, "forward_to_formatted": null, "id": "phon_jhMWlB6anhT8vcsGNEFukaVl806zfxCgbSAAkvtNBpN", "is_group_number": false, "is_verified": false, "label": null, "last_billed_price": "1.15", "mms_enabled": true, "next_billing_on": "2023-12-07", "number": "+14156251293", "number_formatted": "+1 415-625-1293", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "press_1_to_accept": true, "sms_enabled": true, "supports_mms_to_countries": ["CA", "US"], "supports_sms_to_countries": ["CA", "PR", "US"], "type": "internal", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "voicemail_greeting_url": "https://closeio-voicemail-greetings.s3.amazonaws.com/14bzVblrIaekmXSGQEKM11/undefined.mp3"}, "date_created": "2023-11-07T10:04:24.799000", "date_updated": "2023-11-07T10:04:24.799000", "id": "ev_741AbDOXqemOl5abyZTut4", "lead_id": null, "meta": {}, "oauth_client_id": null, "oauth_scope": null, "object_id": "phon_jhMWlB6anhT8vcsGNEFukaVl806zfxCgbSAAkvtNBpN", "object_type": "phone_number", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "previous_data": {"date_updated": "2023-10-07T10:03:53.255000+00:00", "next_billing_on": "2023-11-07"}, "request_id": null, "user_id": null}, "emitted_at": 1699630289827} +{"stream": "leads", "data": {"contacts": [{"created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "urls": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "display_name": "Cooper", "id": "cont_Dsi7AGMRelIZ2I6DIKicaGJU7mwxPZElLIHy33xbjCM", "title": "", "date_created": "2022-07-05T21:01:25.612000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Cooper"}], "name": "Cooper", "lead_id": "lead_Eohw2Vf6WOKZHQ97nS1UTL3iV62pAJFX3ROgJ5WT4cY", "emails": [], "phones": [], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_updated": "2022-07-05T21:01:25.612000+00:00"}], "description": "", "date_created": "2022-07-05T21:01:25.608000+00:00", "opportunities": [], "date_updated": "2022-07-05T21:01:25.658000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "url": null, "custom": {}, "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "status_id": "stat_HrZ1aYkkxRORQSxdBcNPT31HkqxkK2w2uWGiK6yjkmK", "tasks": [], "display_name": "Alex", "html_url": "https://app.close.com/lead/lead_Eohw2Vf6WOKZHQ97nS1UTL3iV62pAJFX3ROgJ5WT4cY/", "addresses": [], "status_label": "Potential", "id": "lead_Eohw2Vf6WOKZHQ97nS1UTL3iV62pAJFX3ROgJ5WT4cY", "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Alex"}], "name": "Alex", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1699641295267} +{"stream": "leads", "data": {"contacts": [{"created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "urls": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "display_name": "Steli Efti", "id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "title": "CEO & Co-Founder", "date_created": "2021-07-13T11:39:03.354000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Steli%20Efti"}], "name": "Steli Efti", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "emails": [{"type": "office", "email": "sales@close.com"}], "phones": [{"phone": "+16505176539", "phone_formatted": "+1 650-517-6539", "type": "office", "country": "US"}], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_updated": "2021-07-13T11:39:03.354000+00:00", "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Decision Maker"]}, {"created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "urls": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "display_name": "Nick Persico", "id": "cont_FY5ws8upMQQyD9vKg4jzwRb6V3MLctQeNTc2NaUmAyo", "title": "Director of Revenue", "date_created": "2021-07-13T11:39:03.366000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Nick%20Persico"}], "name": "Nick Persico", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "emails": [{"type": "office", "email": "nick@close.com"}], "phones": [{"phone": "+18334625673", "phone_formatted": "+1 833-462-5673", "type": "office", "country": "US"}], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_updated": "2021-07-13T11:39:03.366000+00:00", "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Gatekeeper", "Point of Contact"]}, {"created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "urls": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "display_name": "Customer Success Team", "id": "cont_4cmimyQMTMi61kc72mLAV9XdHdddw6LR1LqzvoNdSuV", "title": null, "date_created": "2021-07-13T11:39:03.374000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Customer%20Success%20Team"}], "name": "Customer Success Team", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "emails": [{"type": "office", "email": "success@close.com"}], "phones": [{"phone": "+18334625673", "phone_formatted": "+1 833-462-5673", "type": "office", "country": "US"}], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_updated": "2021-07-13T11:39:03.374000+00:00"}, {"created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "urls": [], "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "display_name": "Support", "id": "cont_CI5c6Ekew0cyhSgoFIXeaz2PJVmmtigF2XNIGUDXKnu", "title": null, "date_created": "2021-07-13T11:39:03.380000+00:00", "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Support"}], "name": "Support", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "emails": [{"type": "office", "email": "support@close.com"}], "phones": [], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_updated": "2021-07-13T11:39:03.380000+00:00"}], "description": "Visit our blog for high quality sales content, blog.close.com!", "date_created": "2021-07-13T11:39:03.315000+00:00", "opportunities": [{"expected_value": 37500, "confidence": 75, "date_created": "2021-07-13T11:39:04.284000+00:00", "user_name": "Airbyte Team", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "date_updated": "2021-08-18T10:26:44.306000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "contact_name": "Steli Efti", "updated_by_name": "Airbyte Team", "lead_name": "Close (Example\u00a0Lead)", "date_won": "2021-07-15", "annualized_expected_value": 37500, "value_formatted": "$500", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "status_id": "stat_AWXzFJkvkHVyJQPFulY0wM7LrQRiMEtQChGumG035bH", "contact_id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "date_lost": null, "value": 50000, "value_period": "one_time", "id": "oppo_1TfmaSLuECcdSQIBnjUOBj1gAXdyrp4SFaogvcEmtbk", "value_currency": "USD", "note": "Use opportunities to track which stage of the pipeline your deals are in and the revenue associated with them.", "status_label": "Proposal Sent", "integration_links": [], "status_display_name": "Proposal Sent", "annualized_value": 50000, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_type": "active", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}], "date_updated": "2022-11-09T02:40:26.489000+00:00", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "url": "https://close.com", "custom": {"Current Vendor/Software": "Stark Industries", "Industry": "Software", "Lead Owner": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "Referral Source": "Website"}, "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "tasks": [{"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "contact_id": null, "contact_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date": "2021-07-18", "date_created": "2021-07-13T11:39:03.520000+00:00", "date_updated": "2022-11-08T12:21:15.730000+00:00", "due_date": "2021-07-18", "_type": "lead", "view": "archive", "id": "task_bboTdYSlGqTBSXF0FEwhBOywDCV6iKDyMWiEfNFi7sW", "is_complete": true, "is_dateless": null, "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "lead_name": "Close (Example\u00a0Lead)", "object_id": "acti_POPD3uA7lSfdf4xLO7PgsxKeoJS1dCRthzQRb13Xbyw", "object_type": "taskcompleted", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "text": "Call Steli", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team"}, {"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "contact_id": null, "contact_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date": "2021-07-16", "date_created": "2021-07-13T11:39:03.463000+00:00", "date_updated": "2021-08-18T10:31:52.081000+00:00", "due_date": "2021-07-16", "_type": "lead", "view": "archive", "id": "task_7kpMfXIPms858l9GKZTo3BCtVflvcIsOYPXL8mZgzyp", "is_complete": true, "is_dateless": null, "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "lead_name": "Close (Example\u00a0Lead)", "object_id": "acti_gdh0Iw30XYfKhKctrqPuxAbNghsxlDBDFaN8k35ZAxf", "object_type": "taskcompleted", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "text": "Send Steli an email", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team"}], "display_name": "Close (Example\u00a0Lead)", "html_url": "https://app.close.com/lead/lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz/", "addresses": [{"city": "San Francisco", "address_2": null, "label": "mailing", "zipcode": "94120", "state": "CA", "country": "US", "address_1": "PO Box 7775 #69574"}], "status_label": "Interested", "id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Close%20%28Example%C2%A0Lead%29"}], "name": "Close (Example\u00a0Lead)", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_1exVDDcGOEiIdhBhBv2VEGqnpIcJZZqiWkk4O7hbU3D": "Stark Industries", "custom.cf_ZuP9X9UjiQzjptNHlT7DxzRATaFil2Ysoz0aGMq0Kim": "Software", "custom.cf_mhBoQeiuwFRlz7zqyi4kJgzUreEoUp0hUsLwrnUTEgh": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_05o22yQYHMrFh4cCYCMSQJdSpODdabCbGQ8il5Do7X4": "Website"}, "emitted_at": 1699641295282} +{"stream": "leads", "data": {"tasks": [{"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "contact_id": null, "contact_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date": "2022-11-30T09:00:00+00:00", "date_created": "2022-11-11T09:01:05.650000+00:00", "date_updated": "2022-11-11T09:01:24.622000+00:00", "due_date": "2022-11-30T09:00:00+00:00", "_type": "lead", "view": "inbox", "id": "task_1uqKlzwO0qLGYgBMT5DcLtklNmgtrUNVobSNEiPpoU6", "is_complete": false, "is_dateless": false, "lead_id": "lead_AUtZm7EBlaSbYqOrDjIZEuC4tfhLxTDtaK9jlEPMb3y", "lead_name": "Airbyte", "object_id": null, "object_type": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "text": "Follow up", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team"}], "url": null, "date_updated": "2023-01-30T16:02:41.186000+00:00", "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Airbyte"}], "name": "Airbyte", "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "status_id": "stat_HrZ1aYkkxRORQSxdBcNPT31HkqxkK2w2uWGiK6yjkmK", "contacts": [{"lead_id": "lead_AUtZm7EBlaSbYqOrDjIZEuC4tfhLxTDtaK9jlEPMb3y", "id": "cont_b4h4BcmWn7rKbnsHQ0JfADwXGgndbpc5JlQEdHHyv78", "date_updated": "2023-01-30T16:02:41.174000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-11T09:00:52.289000+00:00", "name": "User1", "emails": [], "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=User1"}], "urls": [], "display_name": "User1", "title": "Product Manager", "phones": [{"phone_formatted": "+1 600-000-0001", "phone": "+16000000001", "type": "office", "country": null}], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}], "addresses": [], "status_label": "Potential", "description": "", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "html_url": "https://app.close.com/lead/lead_AUtZm7EBlaSbYqOrDjIZEuC4tfhLxTDtaK9jlEPMb3y/", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-11T09:00:52.285000+00:00", "opportunities": [], "custom": {}, "display_name": "Airbyte", "id": "lead_AUtZm7EBlaSbYqOrDjIZEuC4tfhLxTDtaK9jlEPMb3y", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1699641295991} +{"stream": "leads", "data": {"tasks": [{"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "contact_id": null, "contact_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date": "2022-11-17T08:30:00+00:00", "date_created": "2022-11-08T15:55:06.275000+00:00", "date_updated": "2022-11-08T15:55:06.275000+00:00", "due_date": "2022-11-17T08:30:00+00:00", "_type": "lead", "view": "inbox", "id": "task_Va4jRQkIhZrUwzejqF3lZ4VaJeqR0cAvzzuaN0IaE5r", "is_complete": false, "is_dateless": false, "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "lead_name": "Test Lead", "object_id": null, "object_type": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "text": "Follow up", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team"}, {"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "contact_id": null, "contact_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date": "2022-11-23T09:00:00+00:00", "date_created": "2022-11-09T12:37:29.827000+00:00", "date_updated": "2022-11-09T12:44:26.200000+00:00", "due_date": "2022-11-23T09:00:00+00:00", "_type": "lead", "view": "archive", "id": "task_KAzmd4tqcQ1dYypqOjeZqmpKVlFZf3LWJfxCSoX29GY", "is_complete": true, "is_dateless": false, "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "lead_name": "Test Lead", "object_id": "acti_tEFyhPH4AKZ3YHaqqFEdm9Lot1oEI4yEyKzEf0ncbGE", "object_type": "taskcompleted", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "text": "Follow up", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team"}], "url": null, "date_updated": "2023-01-30T16:03:28.802000+00:00", "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Test%20Lead"}], "name": "Test Lead", "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "contacts": [{"lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "date_updated": "2023-01-30T16:03:28.787000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-08T15:54:42.381000+00:00", "name": "User2", "emails": [{"email": "user2.sample@gmail.com", "type": "office"}], "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=User2"}], "urls": [], "display_name": "User2", "title": "Test Lead", "phones": [{"phone_formatted": "+1 415-623-6785", "phone": "+14156236785", "type": "office", "country": "US"}], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_Qj3b4cWxvmvqTtZ0U5TaprRDah8g7jRsavfVh8NCPcu": "2022-12-01"}], "addresses": [], "status_label": "Interested", "description": "", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "html_url": "https://app.close.com/lead/lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j/", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2022-11-08T15:54:42.377000+00:00", "opportunities": [{"user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "contact_id": "cont_OH7f9TYVgcDMqiSmL6Jawba9bxOIumKXD3NYtmClWAP", "date_updated": "2022-11-08T15:55:27.750000+00:00", "integration_links": [], "note": "Test", "value": 10000, "user_name": "Airbyte Team", "annualized_expected_value": 5000, "value_formatted": "$100", "annualized_value": 10000, "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "date_won": "2022-11-16", "status_id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "status_type": "active", "lead_id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "value_period": "one_time", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_display_name": "Demo Completed", "date_created": "2022-11-08T15:55:27.750000+00:00", "contact_name": "User2", "date_lost": null, "lead_name": "Test Lead", "expected_value": 5000, "value_currency": "USD", "id": "oppo_QtfwXdJFxmOLRzd3mirP7S5vWWYTEho2uWtHYNSiOUE", "confidence": 50, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_label": "Demo Completed"}], "custom": {}, "display_name": "Test Lead", "id": "lead_aVZGHXTPH0GfOguQ9vMQBZI6PpISOEzmwp8GBMmWZ3j", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1699641295999} +{"stream": "leads", "data": {"tasks": [], "url": null, "date_updated": "2023-11-10T17:59:45.408000+00:00", "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Bluth%20Company%20%28Example%C2%A0Lead%29"}], "name": "Bluth Company (Example\u00a0Lead)", "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "contacts": [{"lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "id": "cont_fcD6Y7PO1v6Olb4gGs36mLtLhOjyRA9SjuXEpwBVyhI", "date_updated": "2021-07-13T11:39:04.430000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-07-13T11:39:04.430000+00:00", "name": "Gob Bluth", "emails": [{"email": "bluth@close.com", "type": "office"}], "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Gob%20Bluth"}], "urls": [], "display_name": "Gob Bluth", "title": "Magician", "phones": [{"phone_formatted": "+1 202-555-0186", "phone": "+12025550186", "type": "office", "country": "US"}], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Gatekeeper"]}, {"lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "id": "cont_at5uglNbyasFp2KsoWQpjQmLp4lmmqX2p1nhvmPYytq", "date_updated": "2021-07-13T11:39:04.441000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-07-13T11:39:04.441000+00:00", "name": "Tobias F\u00fcnke", "emails": [{"email": "tobiasfunke@close.com", "type": "office"}], "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Tobias%20F%C3%BCnke"}], "urls": [], "display_name": "Tobias F\u00fcnke", "title": "Blue Man Group (Understudy)", "phones": [], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Point of Contact"]}], "addresses": [{"city": "Los Angeles", "zipcode": "90210", "address_1": "100 Bluth Drive", "address_2": null, "label": "business", "state": "CA", "country": "US"}], "status_label": "Interested", "description": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "html_url": "https://app.close.com/lead/lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2/", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-07-05T11:39:00.850000+00:00", "opportunities": [{"user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "contact_id": null, "date_updated": "2021-07-13T11:39:04.817000+00:00", "integration_links": [], "note": "Gob's ready to buy a $3,000 suit.", "value": 300000, "user_name": "Airbyte Team", "annualized_expected_value": 225000, "value_formatted": "$3,000", "annualized_value": 300000, "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "date_won": "2021-07-16", "status_id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "status_type": "active", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "value_period": "one_time", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_display_name": "Demo Completed", "date_created": "2021-07-13T11:39:04.817000+00:00", "contact_name": null, "date_lost": null, "lead_name": "Bluth Company (Example\u00a0Lead)", "expected_value": 225000, "value_currency": "USD", "id": "oppo_NkKuhUWfDoErNArh44hw7jkkx7sCl5bhlamtezmX7BE", "confidence": 75, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_label": "Demo Completed"}], "custom": {"Current Vendor/Software": "BiffCo", "Industry": "Real estate", "Lead Owner": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "Referral Source": "Google Search"}, "display_name": "Bluth Company (Example\u00a0Lead)", "id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_1exVDDcGOEiIdhBhBv2VEGqnpIcJZZqiWkk4O7hbU3D": "BiffCo", "custom.cf_ZuP9X9UjiQzjptNHlT7DxzRATaFil2Ysoz0aGMq0Kim": "Real estate", "custom.cf_mhBoQeiuwFRlz7zqyi4kJgzUreEoUp0hUsLwrnUTEgh": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_05o22yQYHMrFh4cCYCMSQJdSpODdabCbGQ8il5Do7X4": "Google Search"}, "emitted_at": 1699641296006} +{"stream": "leads", "data": {"tasks": [{"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "contact_id": null, "contact_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date": "2022-11-12", "date_created": "2022-11-08T15:10:57.220000+00:00", "date_updated": "2022-11-09T11:06:06.145000+00:00", "due_date": "2022-11-12", "_type": "lead", "view": "inbox", "id": "task_Wgzt6t0ZGRlvZTnfhSfxHoFRWm6zXBMuuecKxRkfYmZ", "is_complete": false, "is_dateless": false, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "lead_name": "Wayne Enterprises (Example\u00a0Lead)", "object_id": null, "object_type": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "text": "Follow up", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team"}, {"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "contact_id": null, "contact_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date": "2022-11-12", "date_created": "2022-11-09T11:05:42.844000+00:00", "date_updated": "2022-11-09T11:06:06.147000+00:00", "due_date": "2022-11-12", "_type": "lead", "view": "inbox", "id": "task_7NzjQt8zXoqAUMJsb58EGBP4rqTTbm5iR8jRuOY3MAJ", "is_complete": false, "is_dateless": false, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "lead_name": "Wayne Enterprises (Example\u00a0Lead)", "object_id": null, "object_type": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "text": "Follow up", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team"}, {"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "contact_id": null, "contact_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date": "2022-11-19T09:00:00+00:00", "date_created": "2022-11-08T15:13:22.159000+00:00", "date_updated": "2022-11-08T15:13:22.159000+00:00", "due_date": "2022-11-19T09:00:00+00:00", "_type": "lead", "view": "inbox", "id": "task_pIYI4An2qA84qea5mCLggCL0dZ8QIk2c2YrzoqSV8fd", "is_complete": false, "is_dateless": false, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "lead_name": "Wayne Enterprises (Example\u00a0Lead)", "object_id": null, "object_type": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "text": "Follow up", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team"}, {"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "contact_id": null, "contact_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date": "2022-11-30T08:00:00+00:00", "date_created": "2022-11-08T15:13:13.470000+00:00", "date_updated": "2022-11-09T10:57:24.756000+00:00", "due_date": "2022-11-30T08:00:00+00:00", "_type": "lead", "view": "archive", "id": "task_hMgl18LN4kAUM7XWO4fYlu3amtKadZahsb7OV9J5qWD", "is_complete": true, "is_dateless": false, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "lead_name": "Wayne Enterprises (Example\u00a0Lead)", "object_id": "acti_SFQmP02YDdCHlkwAWoEtIfZXEOvhoenTmKVdEw7BtJB", "object_type": "taskcompleted", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "text": "Follow up", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team"}, {"assigned_to": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "assigned_to_name": "Airbyte Team", "contact_id": null, "contact_name": null, "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "created_by_name": "Airbyte Team", "date": "2022-11-09T07:30:00+00:00", "date_created": "2022-11-09T11:05:46.682000+00:00", "date_updated": "2022-11-09T11:06:11.243000+00:00", "due_date": "2022-11-09T07:30:00+00:00", "_type": "lead", "view": "archive", "id": "task_iHHSk7GuvunATuloc0hM2k2btiuJpasL12GkwHnIVjP", "is_complete": true, "is_dateless": false, "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "lead_name": "Wayne Enterprises (Example\u00a0Lead)", "object_id": "acti_DJOFYNqKoXSmPB4yJHVRafxAqhnvk5olHmmihw4cqND", "object_type": "taskcompleted", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "text": "Follow up", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "updated_by_name": "Airbyte Team"}], "url": null, "date_updated": "2023-11-10T17:59:45.437000+00:00", "integration_links": [{"name": "Google Search", "url": "https://google.com/search?q=Wayne%20Enterprises%20%28Example%C2%A0Lead%29"}], "name": "Wayne Enterprises (Example\u00a0Lead)", "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "status_id": "stat_nzPGZ5qJXdpP2GSFqzbPdyHgZWXkRfx6BjQih76ss0q", "contacts": [{"lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "id": "cont_ubIO1eBUVw3iFJ1Ot4LY12R7oADqn7bsLUF7NU2fJO6", "date_updated": "2022-11-08T15:01:17.915000+00:00", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-07-13T11:39:04.977000+00:00", "name": "Bruce Wayne", "emails": [{"email": "thedarkknight@close.com", "type": "office"}], "integration_links": [{"name": "LinkedIn Search", "url": "https://www.linkedin.com/search/results/people/?keywords=Bruce%20Wayne"}], "urls": [], "display_name": "Bruce Wayne", "title": "The Dark Knight", "phones": [{"phone_formatted": "+1 415-623-6785", "phone": "+14156236785", "type": "office", "country": "US"}], "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_oYaaZ3ikZjy6qc7htLdSWJxSxEZTova9HHLLLj67cyi": ["Decision Maker", "Point of Contact"]}], "addresses": [], "status_label": "Interested", "description": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "html_url": "https://app.close.com/lead/lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN/", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "date_created": "2021-07-13T11:39:04.960000+00:00", "opportunities": [{"user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "contact_id": null, "date_updated": "2021-07-13T11:39:05.061000+00:00", "integration_links": [], "note": "Bruce needs new software for the Bat Cave.", "value": 50000, "user_name": "Airbyte Team", "annualized_expected_value": 37500, "value_formatted": "$500", "annualized_value": 50000, "created_by_name": "Airbyte Team", "updated_by_name": "Airbyte Team", "date_won": "2021-07-15", "status_id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "status_type": "active", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "value_period": "one_time", "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_display_name": "Demo Completed", "date_created": "2021-07-13T11:39:05.061000+00:00", "contact_name": null, "date_lost": null, "lead_name": "Wayne Enterprises (Example\u00a0Lead)", "expected_value": 37500, "value_currency": "USD", "id": "oppo_jXatpqaQ3HK50yBO9vMWg2wFyHoqbCJtgbRYAMuEGor", "confidence": 75, "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "status_label": "Demo Completed"}], "custom": {"Current Vendor/Software": "Initech", "Industry": "Manufacturing", "Lead Owner": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "Referral Source": "Facebook"}, "display_name": "Wayne Enterprises (Example\u00a0Lead)", "id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_1exVDDcGOEiIdhBhBv2VEGqnpIcJZZqiWkk4O7hbU3D": "Initech", "custom.cf_ZuP9X9UjiQzjptNHlT7DxzRATaFil2Ysoz0aGMq0Kim": "Manufacturing", "custom.cf_mhBoQeiuwFRlz7zqyi4kJgzUreEoUp0hUsLwrnUTEgh": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "custom.cf_05o22yQYHMrFh4cCYCMSQJdSpODdabCbGQ8il5Do7X4": "Facebook"}, "emitted_at": 1699641296013} {"stream": "opportunities", "data": {"user_name": "Airbyte Team", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "value": 300000, "status_display_name": "Demo Completed", "created_by_name": "Airbyte Team", "status_type": "active", "status_label": "Demo Completed", "value_currency": "USD", "id": "oppo_NkKuhUWfDoErNArh44hw7jkkx7sCl5bhlamtezmX7BE", "lead_name": "Bluth Company (Example\u00a0Lead)", "date_updated": "2021-07-13T11:39:04.817000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "value_period": "one_time", "integration_links": [], "status_id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "expected_value": 225000, "date_created": "2021-07-13T11:39:04.817000+00:00", "date_won": "2021-07-16", "note": "Gob's ready to buy a $3,000 suit.", "lead_id": "lead_p7HyK4BZKAZJH2m8AAhFpcla2E0fDa6TACgU8KWv7o2", "contact_id": null, "contact_name": null, "annualized_expected_value": 225000, "confidence": 75, "annualized_value": 300000, "updated_by_name": "Airbyte Team", "value_formatted": "$3,000", "date_lost": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417127685} {"stream": "opportunities", "data": {"user_name": "Airbyte Team", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "value": 50000, "status_display_name": "Demo Completed", "created_by_name": "Airbyte Team", "status_type": "active", "status_label": "Demo Completed", "value_currency": "USD", "id": "oppo_jXatpqaQ3HK50yBO9vMWg2wFyHoqbCJtgbRYAMuEGor", "lead_name": "Wayne Enterprises (Example\u00a0Lead)", "date_updated": "2021-07-13T11:39:05.061000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "value_period": "one_time", "integration_links": [], "status_id": "stat_pI63Ohv8ByAaIFsguWoGCOP8FPV9vL9YJ8VbxTXgSe6", "expected_value": 37500, "date_created": "2021-07-13T11:39:05.061000+00:00", "date_won": "2021-07-15", "note": "Bruce needs new software for the Bat Cave.", "lead_id": "lead_MH9KHM5OqPHgTyGj5liiN3LTPIyuGBqzBS4uzkTtpeN", "contact_id": null, "contact_name": null, "annualized_expected_value": 37500, "confidence": 75, "annualized_value": 50000, "updated_by_name": "Airbyte Team", "value_formatted": "$500", "date_lost": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417127689} {"stream": "opportunities", "data": {"user_name": "Airbyte Team", "created_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "value": 50000, "status_display_name": "Proposal Sent", "created_by_name": "Airbyte Team", "status_type": "active", "status_label": "Proposal Sent", "value_currency": "USD", "id": "oppo_1TfmaSLuECcdSQIBnjUOBj1gAXdyrp4SFaogvcEmtbk", "lead_name": "Close (Example\u00a0Lead)", "date_updated": "2021-08-18T10:26:44.306000+00:00", "updated_by": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg", "value_period": "one_time", "integration_links": [], "status_id": "stat_AWXzFJkvkHVyJQPFulY0wM7LrQRiMEtQChGumG035bH", "expected_value": 37500, "date_created": "2021-07-13T11:39:04.284000+00:00", "date_won": "2021-07-15", "note": "Use opportunities to track which stage of the pipeline your deals are in and the revenue associated with them.", "lead_id": "lead_87BPw5opGPqBjSpz70L28NzDyBmJZHHdc3bvVt6JRlz", "contact_id": "cont_2ZhjI4qVESIBNDPJTeQF5avXJoMJ65TZoIelDXaswCI", "contact_name": "Steli Efti", "annualized_expected_value": 37500, "confidence": 75, "annualized_value": 50000, "updated_by_name": "Airbyte Team", "value_formatted": "$500", "date_lost": null, "organization_id": "orga_ya3w9oMjeLtWe7zFGZr63Dz8ruBbjybG0EIUdUXaESi", "user_id": "user_SOwJFVMqtgZCL6QrMDLujSXbQabQnNfOwjFX1mdOulg"}, "emitted_at": 1691417127692} diff --git a/airbyte-integrations/connectors/source-close-com/metadata.yaml b/airbyte-integrations/connectors/source-close-com/metadata.yaml index 6cd112e206e1..ccb09a7823a9 100644 --- a/airbyte-integrations/connectors/source-close-com/metadata.yaml +++ b/airbyte-integrations/connectors/source-close-com/metadata.yaml @@ -1,12 +1,16 @@ data: + ab_internal: + ql: 300 + sl: 100 allowedHosts: hosts: - api.close.com connectorSubtype: api connectorType: source definitionId: dfffecb7-9a13-43e9-acdc-b92af7997ca9 - dockerImageTag: 0.4.2 + dockerImageTag: 0.5.0 dockerRepository: airbyte/source-close-com + documentationUrl: https://docs.airbyte.com/integrations/sources/close-com githubIssueLabel: source-close-com icon: close.svg license: MIT @@ -17,12 +21,8 @@ data: oss: enabled: true releaseStage: beta - documentationUrl: https://docs.airbyte.com/integrations/sources/close-com + supportLevel: community tags: - language:low-code - language:python - ab_internal: - sl: 200 - ql: 300 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/__init__.py b/airbyte-integrations/connectors/source-close-com/source_close_com/__init__.py index 290f7d5f74e0..26f244576b38 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/__init__.py +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/__init__.py @@ -22,6 +22,6 @@ from .datetime_incremental_sync import CustomDatetimeIncrementalSync -from .source_lc import SourceCloseCom +from .source import SourceCloseCom __all__ = ["SourceCloseCom", "CustomDatetimeIncrementalSync"] diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/manifest.yaml b/airbyte-integrations/connectors/source-close-com/source_close_com/manifest.yaml index f7768c310240..f4f6ba4d0a12 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/manifest.yaml +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/manifest.yaml @@ -194,9 +194,10 @@ definitions: name: "email_thread_activities" path: "activity/emailthread" email_activities_stream: - $ref: "#/definitions/activities_base_stream" + $ref: "#/definitions/base_stream" + incremental_sync: + $ref: "#/definitions/incremental_sync__cursor_date_created" $parameters: - $ref: "#/definitions/activities_base_stream/$parameters" name: "email_activities" path: "activity/email" sms_activities_stream: diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/activity_custom_fields.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/activity_custom_fields.json index cafcf348dd9a..1675b0e43dee 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/activity_custom_fields.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/activity_custom_fields.json @@ -6,6 +6,18 @@ "description": { "date_updated": ["null", "string"] }, + "back_reference_is_visible": { + "date_updated": ["null", "boolean"] + }, + "choices": { + "date_updated": ["null", "array"] + }, + "is_shared": { + "date_updated": ["null", "boolean"] + }, + "referenced_custom_type_id": { + "date_updated": ["null", "string"] + }, "name": { "date_updated": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/call_activities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/call_activities.json index 886d0d164101..b6b4df319d36 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/call_activities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/call_activities.json @@ -6,6 +6,30 @@ "source": { "type": ["null", "string"] }, + "forwarded_to": { + "type": ["null", "string"] + }, + "is_forwarded": { + "type": ["null", "boolean"] + }, + "note_html": { + "type": ["null", "string"] + }, + "parent_meeting_id": { + "type": ["null", "string"] + }, + "recording_duration": { + "type": ["null", "string"] + }, + "sequence_id": { + "type": ["null", "string"] + }, + "sequence_name": { + "type": ["null", "string"] + }, + "sequence_subscription_id": { + "type": ["null", "string"] + }, "remote_phone_formatted": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/contact_custom_fields.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/contact_custom_fields.json index 97ed85cb4ca3..b3f26fa90b46 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/contact_custom_fields.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/contact_custom_fields.json @@ -9,6 +9,15 @@ "name": { "date_updated": ["null", "string"] }, + "back_reference_is_visible": { + "date_updated": ["null", "string"] + }, + "is_shared": { + "date_updated": ["null", "boolean"] + }, + "referenced_custom_type_id": { + "date_updated": ["null", "string"] + }, "choices": { "date_updated": ["null", "array"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/created_activities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/created_activities.json index e092624b7a2c..ddec9f66cfd9 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/created_activities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/created_activities.json @@ -6,6 +6,9 @@ "date_updated": { "type": ["null", "string"] }, + "activity_at": { + "type": ["null", "string"] + }, "date_created": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_followup_tasks.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_followup_tasks.json index 35ab577ebc36..c33a632c08c0 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_followup_tasks.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/email_followup_tasks.json @@ -9,6 +9,12 @@ "view": { "type": ["null", "string"] }, + "total_emails_count_in_thread": { + "type": ["null", "integer"] + }, + "updated_by_name": { + "type": ["null", "string"] + }, "assigned_to": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/events.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/events.json index 0deeddc49d12..290446df8def 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/events.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/events.json @@ -9,6 +9,12 @@ "api_key_id": { "type": ["null", "string"] }, + "oauth_client_id": { + "type": ["null", "string"] + }, + "oauth_scope": { + "type": ["null", "string"] + }, "changed_fields": { "type": ["null", "array"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_custom_fields.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_custom_fields.json index 19bc94eed623..776000912839 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_custom_fields.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/lead_custom_fields.json @@ -9,6 +9,15 @@ "description": { "date_updated": ["null", "string"] }, + "back_reference_is_visible": { + "date_updated": ["null", "string"] + }, + "date_updated": { + "date_updated": ["null", "string"] + }, + "referenced_custom_type_id": { + "date_updated": ["null", "string"] + }, "name": { "date_updated": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/meeting_activities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/meeting_activities.json index 66cb927a51e9..05b86653ceff 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/meeting_activities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/meeting_activities.json @@ -6,6 +6,57 @@ "date_updated": { "type": ["null", "string"] }, + "activity_at": { + "type": ["null", "string"] + }, + "contact_id": { + "type": ["null", "string"] + }, + "notetaker_id": { + "type": ["null", "string"] + }, + "provider_calendar_event_id": { + "type": ["null", "string"] + }, + "provider_calendar_type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "user_note": { + "type": ["null", "string"] + }, + "user_note_date_updated": { + "type": ["null", "string"] + }, + "user_note_html": { + "type": ["null", "string"] + }, + "provider_calendar_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "integrations": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "calendar_event_uids": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "conference_links": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, "created_by_name": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_custom_fields.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_custom_fields.json index 97ed85cb4ca3..b3f26fa90b46 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_custom_fields.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_custom_fields.json @@ -9,6 +9,15 @@ "name": { "date_updated": ["null", "string"] }, + "back_reference_is_visible": { + "date_updated": ["null", "string"] + }, + "is_shared": { + "date_updated": ["null", "boolean"] + }, + "referenced_custom_type_id": { + "date_updated": ["null", "string"] + }, "choices": { "date_updated": ["null", "array"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_status_change_activities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_status_change_activities.json index 46a659113aa3..389307edcbcb 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_status_change_activities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/opportunity_status_change_activities.json @@ -21,6 +21,27 @@ "date_created": { "type": ["null", "string"] }, + "activity_at": { + "type": ["null", "string"] + }, + "new_pipeline_name": { + "type": ["null", "string"] + }, + "users": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "opportunity_value_period": { + "type": ["null", "string"] + }, + "old_pipeline_name": { + "type": ["null", "string"] + }, + "opportunity_confidence": { + "type": ["null", "integer"] + }, "date_updated": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/pipelines.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/pipelines.json index 46ec2b161dbd..bafc4f534fa6 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/pipelines.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/pipelines.json @@ -6,6 +6,9 @@ "created_by": { "type": ["null", "string"] }, + "updated_by": { + "type": ["null", "string"] + }, "date_updated": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/sms_activities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/sms_activities.json index a31ed621a3df..ee7841862e38 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/sms_activities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/sms_activities.json @@ -6,6 +6,36 @@ "date_updated": { "type": ["null", "string"] }, + "activity_at": { + "type": ["null", "string"] + }, + "attachments": { + "type": ["null", "array"] + }, + "date_scheduled": { + "type": ["null", "string"] + }, + "error_message": { + "type": ["null", "string"] + }, + "local_phone_formatted": { + "type": ["null", "string"] + }, + "sequence_id": { + "type": ["null", "string"] + }, + "sequence_name": { + "type": ["null", "string"] + }, + "sequence_subscription_id": { + "type": ["null", "string"] + }, + "source": { + "type": ["null", "string"] + }, + "template_id": { + "type": ["null", "string"] + }, "date_created": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/task_completed_activities.json b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/task_completed_activities.json index 3c8e3434563d..8f753d39e125 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/task_completed_activities.json +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/schemas/task_completed_activities.json @@ -9,6 +9,21 @@ "id": { "type": ["null", "string"] }, + "activity_at": { + "type": ["null", "string"] + }, + "task_assigned_to": { + "type": ["null", "string"] + }, + "task_assigned_to_name": { + "type": ["null", "string"] + }, + "user_name": { + "type": ["null", "string"] + }, + "users": { + "type": ["null", "array"] + }, "task_id": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-close-com/source_close_com/source.py b/airbyte-integrations/connectors/source-close-com/source_close_com/source.py index 2fe9c2f85ab8..9754c114bbd8 100644 --- a/airbyte-integrations/connectors/source-close-com/source_close_com/source.py +++ b/airbyte-integrations/connectors/source-close-com/source_close_com/source.py @@ -58,7 +58,6 @@ def request_params( stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - params = {} if self.number_of_items_per_page: params.update({"_limit": self.number_of_items_per_page}) @@ -87,8 +86,24 @@ def backoff_time(self, response: requests.Response) -> Optional[float]: return backoff_time -class IncrementalCloseComStream(CloseComStream): +class CloseComStreamCustomFields(CloseComStream): + """Class to get custom fields for close objects that support them.""" + + def get_custom_field_schema(self) -> Mapping[str, Any]: + """Get custom field schema if it exists.""" + resp = requests.request("GET", url=f"{self.url_base}/custom_field/{self.path()}/", headers=self.authenticator.get_auth_header()) + resp.raise_for_status() + resp_json: Mapping[str, Any] = resp.json()["data"] + return {f"custom.{data['id']}": {"type": ["null", "string", "number", "boolean"]} for data in resp_json} + + def get_json_schema(self): + """Override default get_json_schema method to add custom fields to schema.""" + schema = super().get_json_schema() + schema["properties"].update(self.get_custom_field_schema()) + return schema + +class IncrementalCloseComStream(CloseComStream): cursor_field = "date_updated" def get_updated_state( @@ -105,6 +120,10 @@ def get_updated_state( return {self.cursor_field: max(latest_record.get(self.cursor_field, ""), current_stream_state.get(self.cursor_field, ""))} +class IncrementalCloseComStreamCustomFields(CloseComStreamCustomFields, IncrementalCloseComStream): + """Class to get custom fields for close objects using incremental stream.""" + + class CloseComActivitiesStream(IncrementalCloseComStream): """ General class for activities. Define request params based on cursor_field value. @@ -233,7 +252,7 @@ def request_params(self, stream_state=None, **kwargs): return params -class Leads(IncrementalCloseComStream): +class Leads(IncrementalCloseComStreamCustomFields): """ Get leads on a specific date API Docs: https://developer.close.com/#leads @@ -404,7 +423,7 @@ def path(self, **kwargs) -> str: return "user" -class Contacts(CloseComStream): +class Contacts(CloseComStreamCustomFields): """ Get contacts for Close.com account organization API Docs: https://developer.close.com/#contacts @@ -416,7 +435,7 @@ def path(self, **kwargs) -> str: return "contact" -class Opportunities(IncrementalCloseComStream): +class Opportunities(IncrementalCloseComStreamCustomFields): """ Get opportunities on a specific date API Docs: https://developer.close.com/#opportunities diff --git a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile deleted file mode 100644 index b521bb75a979..000000000000 --- a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-cockroachdb-strict-encrypt - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-cockroachdb-strict-encrypt - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.22 -LABEL io.airbyte.name=airbyte/source-cockroachdb-strict-encrypt diff --git a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/acceptance-test-config.yml b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/acceptance-test-config.yml deleted file mode 100644 index c1e7ae01c0be..000000000000 --- a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/acceptance-test-config.yml +++ /dev/null @@ -1,6 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-cockroachdb-strict-encrypt:dev -tests: - spec: - - spec_path: "src/test/resources/expected_spec.json" diff --git a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/build.gradle deleted file mode 100644 index f651236698cc..000000000000 --- a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/build.gradle +++ /dev/null @@ -1,31 +0,0 @@ -plugins { - id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' -} - -application { - mainClass = 'io.airbyte.integrations.source.cockroachdb.CockroachDbSourceStrictEncrypt' - applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] -} - -dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation project(':airbyte-integrations:connectors:source-relational-db') - implementation project(':airbyte-integrations:connectors:source-cockroachdb') - - implementation libs.connectors.testcontainers - implementation libs.connectors.testcontainers.jdbc - implementation libs.connectors.testcontainers.cockroachdb - implementation libs.postgresql - - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-cockroachdb') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-cockroachdb-strict-encrypt') - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) -} diff --git a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/icon.svg b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/icon.svg deleted file mode 100644 index f03198714fc5..000000000000 --- a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/icon.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/metadata.yaml deleted file mode 100644 index 1c19f743e3b4..000000000000 --- a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/metadata.yaml +++ /dev/null @@ -1,23 +0,0 @@ -data: - allowedHosts: - hosts: - - ${host} - registries: - cloud: - enabled: false # strict encrypt connectors are deployed to Cloud by their non strict encrypt sibling. - oss: - enabled: false # strict encrypt connectors are not used on OSS. - connectorSubtype: database - connectorType: source - definitionId: 9fa5862c-da7c-11eb-8d19-0242ac130003 - dockerImageTag: 0.1.22 - dockerRepository: airbyte/source-cockroachdb-strict-encrypt - githubIssueLabel: source-cockroachdb - icon: cockroachdb.svg - license: MIT - name: Cockroachdb - releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/cockroachdb - tags: - - language:java -metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceStrictEncrypt.java b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceStrictEncrypt.java deleted file mode 100644 index 25da8068e57f..000000000000 --- a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceStrictEncrypt.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.cockroachdb; - -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.base.spec_modification.SpecModifyingSource; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class CockroachDbSourceStrictEncrypt extends SpecModifyingSource implements Source { - - private static final Logger LOGGER = LoggerFactory.getLogger(CockroachDbSourceStrictEncrypt.class); - - public CockroachDbSourceStrictEncrypt() { - super(CockroachDbSource.sshWrappedSource()); - } - - @Override - public ConnectorSpecification modifySpec(final ConnectorSpecification originalSpec) throws Exception { - final ConnectorSpecification spec = Jsons.clone(originalSpec); - ((ObjectNode) spec.getConnectionSpecification().get("properties")).remove(JdbcUtils.SSL_KEY); - return spec; - } - - public static void main(final String[] args) throws Exception { - final Source source = new CockroachDbSourceStrictEncrypt(); - LOGGER.info("starting source: {}", CockroachDbSourceStrictEncrypt.class); - new IntegrationRunner(source).run(args); - LOGGER.info("completed source: {}", CockroachDbSourceStrictEncrypt.class); - } - -} diff --git a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSslTestContainer.java b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSslTestContainer.java deleted file mode 100644 index 5af15f6eeb24..000000000000 --- a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSslTestContainer.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.cockroachdb; - -import java.util.concurrent.TimeUnit; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.Network; -import org.testcontainers.images.builder.ImageFromDockerfile; - -public class CockroachDbSslTestContainer { - - private GenericContainer cockroachSslDbContainer; - - public void start() throws Exception { - if (cockroachSslDbContainer != null) - cockroachSslDbContainer.stop(); - - Network network = Network.newNetwork(); - cockroachSslDbContainer = new GenericContainer( - new ImageFromDockerfile("cockroach-test") - .withFileFromClasspath("Dockerfile", "docker/Dockerfile") - .withFileFromClasspath("cockroachdb_init.sh", "docker/cockroachdb_init.sh") - .withFileFromClasspath("cockroachdb_test_user.sh", "docker/cockroachdb_test_user.sh")) - .withNetwork(network) - .withExposedPorts(26257); - cockroachSslDbContainer.start(); - - // Wait till test user is created - TimeUnit.SECONDS.sleep(5); - } - - public void close() { - cockroachSslDbContainer.stop(); - cockroachSslDbContainer = null; - } - - public GenericContainer getCockroachSslDbContainer() { - return cockroachSslDbContainer; - } - -} diff --git a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/resources/docker/Dockerfile b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/resources/docker/Dockerfile deleted file mode 100644 index b82881c50a8e..000000000000 --- a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/resources/docker/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM cockroachdb/cockroach:v20.2.18 - -## -# Test CockroachDB container with enabled SSL -# Database: defaultdb -# User: test_user -# Build command: docker build -f Dockerfile -t cockroachdb-test-ssl:latest . -# Run command: docker run -td -p 26257:26257 -p 8080:8080 --name cockroach-test-cont cockroachdb-test-ssl:latest -## - -ENV COCKROACH_HOST localhost:26257 -ENV COCKROACH_CERTS_DIR certs - -EXPOSE 8080 -EXPOSE 26257 - -COPY cockroachdb_init.sh . -COPY cockroachdb_test_user.sh . - -RUN chmod +x cockroachdb_init.sh - -ENTRYPOINT cockroachdb_init.sh diff --git a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/resources/docker/cockroachdb_init.sh b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/resources/docker/cockroachdb_init.sh deleted file mode 100644 index 78573c2815bc..000000000000 --- a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/resources/docker/cockroachdb_init.sh +++ /dev/null @@ -1,12 +0,0 @@ -echo "Preparing certs" -mkdir "certs" -mkdir "safe-place" -cockroach cert create-ca --certs-dir=certs --ca-key=safe-place/ca.key -cockroach cert create-node localhost "$HOSTNAME" --certs-dir=certs --ca-key=safe-place/ca.key -cockroach cert create-client root --certs-dir=certs --ca-key=safe-place/ca.key -cockroach cert create-client test_user --certs-dir=certs --ca-key=safe-place/ca.key -echo "Finished preparing certs" - -echo "Starting CockroachDB" -nohup sh cockroachdb_test_user.sh & -cockroach start-single-node --certs-dir=certs diff --git a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/resources/docker/cockroachdb_test_user.sh b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/resources/docker/cockroachdb_test_user.sh deleted file mode 100644 index 68ebd4f2328a..000000000000 --- a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/resources/docker/cockroachdb_test_user.sh +++ /dev/null @@ -1,13 +0,0 @@ -countAttempt=0 -maxAttempt=10 -while : ; do - cockroach sql -e "create user test_user with password 'test_user'" &> user_creation.log && cockroach sql -e "GRANT ALL ON DATABASE defaultdb TO test_user" &>> user_creation.log - ((countAttempt++)) - - if [ "$countAttempt" = "$maxAttempt" ] || [ "$(grep -c 'ERROR' user_creation.log)" = 0 ]; then - echo "User test_user is created with grants to defaultdb! Login=Pass" - break - fi - sleep 2 - echo "Attempt #$countAttempt to create a user is failed!" -done \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/resources/expected_spec.json b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/resources/expected_spec.json deleted file mode 100644 index c24e2c7a25f0..000000000000 --- a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/resources/expected_spec.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/cockroachdb", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Cockroach Source Spec", - "type": "object", - "required": ["host", "port", "database", "username"], - "properties": { - "host": { - "title": "Host", - "description": "Hostname of the database.", - "type": "string", - "order": 0 - }, - "port": { - "title": "Port", - "description": "Port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 5432, - "examples": ["5432"], - "order": 1 - }, - "database": { - "title": "DB Name", - "description": "Name of the database.", - "type": "string", - "order": 2 - }, - "username": { - "title": "User", - "description": "Username to use to access the database.", - "type": "string", - "order": 3 - }, - "password": { - "title": "Password", - "description": "Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 4 - }, - "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", - "title": "JDBC URL Parameters (Advanced)", - "type": "string", - "order": 5 - } - } - } -} diff --git a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbEncryptSourceAcceptanceTest.java deleted file mode 100644 index 1342a701b0a6..000000000000 --- a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbEncryptSourceAcceptanceTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.cockroachdb; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.SyncMode; -import java.util.HashMap; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; - -public class CockroachDbEncryptSourceAcceptanceTest extends SourceAcceptanceTest { - - private static final String STREAM_NAME = "public.id_and_name"; - private static final String STREAM_NAME2 = "public.starships"; - - private CockroachDbSslTestContainer container; - private JsonNode config; - - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - container = new CockroachDbSslTestContainer(); - container.start(); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getCockroachSslDbContainer().getHost()) - .put(JdbcUtils.PORT_KEY, container.getCockroachSslDbContainer().getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, "defaultdb") - .put(JdbcUtils.USERNAME_KEY, "test_user") - .put(JdbcUtils.PASSWORD_KEY, "test_user") - .build()); - - try (final DSLContext dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asInt(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.POSTGRES)) { - final Database database = new Database(dslContext); - - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch( - "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch( - "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - return null; - }); - } - - } - - @Override - protected void tearDown(final TestDestinationEnv testEnv) { - container.close(); - } - - @Override - protected String getImageName() { - return "airbyte/source-cockroachdb-strict-encrypt:dev"; - } - - @Override - protected ConnectorSpecification getSpec() throws Exception { - return SshHelpers.injectSshIntoSpec(Jsons.deserialize(MoreResources.readResource("expected_spec.json"), ConnectorSpecification.class)); - } - - @Override - protected JsonNode getConfig() { - return config; - } - - @Override - protected ConfiguredAirbyteCatalog getConfiguredCatalog() { - return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(Lists.newArrayList("id")) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withStream(CatalogHelpers.createAirbyteStream( - STREAM_NAME, - Field.of("id", JsonSchemaType.NUMBER), - Field.of("name", JsonSchemaType.STRING)) - .withSupportedSyncModes( - Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))), - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(Lists.newArrayList("id")) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withStream(CatalogHelpers.createAirbyteStream( - STREAM_NAME2, - Field.of("id", JsonSchemaType.NUMBER), - Field.of("name", JsonSchemaType.STRING)) - .withSupportedSyncModes( - Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); - } - - @Override - protected JsonNode getState() { - return Jsons.jsonNode(new HashMap<>()); - } - -} diff --git a/airbyte-integrations/connectors/source-cockroachdb/Dockerfile b/airbyte-integrations/connectors/source-cockroachdb/Dockerfile deleted file mode 100644 index c7f103645019..000000000000 --- a/airbyte-integrations/connectors/source-cockroachdb/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-cockroachdb - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-cockroachdb - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.22 -LABEL io.airbyte.name=airbyte/source-cockroachdb diff --git a/airbyte-integrations/connectors/source-cockroachdb/acceptance-test-config.yml b/airbyte-integrations/connectors/source-cockroachdb/acceptance-test-config.yml deleted file mode 100644 index 56c47e4288c9..000000000000 --- a/airbyte-integrations/connectors/source-cockroachdb/acceptance-test-config.yml +++ /dev/null @@ -1,7 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-cockroachdb:dev -tests: - spec: - - spec_path: "src/test-integration/resources/expected_spec.json" - config_path: "src/test-integration/resources/dummy_config.json" diff --git a/airbyte-integrations/connectors/source-cockroachdb/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-cockroachdb/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-cockroachdb/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-cockroachdb/build.gradle b/airbyte-integrations/connectors/source-cockroachdb/build.gradle index 172d69b97a31..544ce6b475d3 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/build.gradle +++ b/airbyte-integrations/connectors/source-cockroachdb/build.gradle @@ -1,37 +1,36 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' - id 'airbyte-connector-acceptance-test' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.7.7' + features = ['db-sources'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileTestJava { + options.compilerArgs.remove("-Werror") + } + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.source.cockroachdb.CockroachDbSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation project(':airbyte-integrations:connectors:source-relational-db') implementation 'org.apache.commons:commons-lang3:3.11' implementation libs.postgresql - testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) - - testImplementation libs.connectors.testcontainers.cockroachdb + testImplementation libs.testcontainers.cockroachdb testImplementation 'org.apache.commons:commons-lang3:3.11' - - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-cockroachdb') - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - - testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) - testImplementation project(":airbyte-json-validation") - testImplementation project(':airbyte-test-utils') } diff --git a/airbyte-integrations/connectors/source-cockroachdb/metadata.yaml b/airbyte-integrations/connectors/source-cockroachdb/metadata.yaml index 9b2a4df22be9..608f1fa8718d 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/metadata.yaml +++ b/airbyte-integrations/connectors/source-cockroachdb/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: source definitionId: 9fa5862c-da7c-11eb-8d19-0242ac130003 - dockerImageTag: 0.1.22 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-cockroachdb githubIssueLabel: source-cockroachdb icon: cockroachdb.svg @@ -13,7 +13,7 @@ data: name: Cockroachdb registries: cloud: - enabled: false + enabled: false # Can not be in cloud until security is handled via DEPLOYMENT_MODE=cloud. oss: enabled: true releaseStage: alpha diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java b/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java index b95e57c3d12f..b41bcfcf9728 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java @@ -4,29 +4,30 @@ package io.airbyte.integrations.source.cockroachdb; -import static io.airbyte.db.jdbc.JdbcUtils.AMPERSAND; +import static io.airbyte.cdk.db.jdbc.JdbcUtils.AMPERSAND; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.base.ssh.SshWrappedSource; +import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.cdk.integrations.source.jdbc.dto.JdbcPrivilegeDto; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.base.ssh.SshWrappedSource; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.dto.JdbcPrivilegeDto; import java.sql.Connection; import java.sql.JDBCType; import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -105,14 +106,15 @@ protected boolean isNotInternalSchema(final JsonNode jsonNode, final Set @Override public JdbcDatabase createDatabase(final JsonNode sourceConfig) throws SQLException { final JsonNode jdbcConfig = toDatabaseConfig(sourceConfig); - + final Map connectionProperties = JdbcUtils.parseJdbcParameters(jdbcConfig, JdbcUtils.CONNECTION_PROPERTIES_KEY); // Create the JDBC data source final DataSource dataSource = DataSourceFactory.create( jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText(), jdbcConfig.has(JdbcUtils.PASSWORD_KEY) ? jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, - driverClass, + driverClassName, jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), - JdbcUtils.parseJdbcParameters(jdbcConfig, JdbcUtils.CONNECTION_PROPERTIES_KEY)); + connectionProperties, + getConnectionTimeout(connectionProperties, driverClassName)); dataSources.add(dataSource); final JdbcDatabase database = new DefaultJdbcDatabase(dataSource, sourceOperations); diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachJdbcDatabase.java b/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachJdbcDatabase.java index a5676d0181f8..5036469dadc7 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachJdbcDatabase.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachJdbcDatabase.java @@ -5,10 +5,10 @@ package io.airbyte.integrations.source.cockroachdb; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.JdbcCompatibleSourceOperations; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.commons.functional.CheckedConsumer; import io.airbyte.commons.functional.CheckedFunction; -import io.airbyte.db.JdbcCompatibleSourceOperations; -import io.airbyte.db.jdbc.JdbcDatabase; import java.sql.Connection; import java.sql.DatabaseMetaData; import java.sql.PreparedStatement; diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachJdbcSourceOperations.java b/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachJdbcSourceOperations.java index 8c1913cdd981..9005f0c43696 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachJdbcSourceOperations.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachJdbcSourceOperations.java @@ -4,12 +4,12 @@ package io.airbyte.integrations.source.cockroachdb; -import static io.airbyte.db.DataTypeUtils.TIMETZ_FORMATTER; +import static io.airbyte.cdk.db.DataTypeUtils.TIMETZ_FORMATTER; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.jdbc.JdbcSourceOperations; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcSourceOperations; import java.sql.ResultSet; import java.sql.SQLException; import java.time.OffsetTime; diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceAcceptanceTest.java index a0f8cecaf9b3..1e3f9f47eb7d 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceAcceptanceTest.java @@ -7,14 +7,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.CatalogHelpers; @@ -27,8 +27,10 @@ import java.util.Objects; import org.jooq.DSLContext; import org.jooq.SQLDialect; +import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.CockroachContainer; +@Disabled public class CockroachDbSourceAcceptanceTest extends SourceAcceptanceTest { private static final String STREAM_NAME = "public.id_and_name"; diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java index e1c1bb66ddbc..04b5cf2f50cf 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test-integration/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceDatatypeTest.java @@ -6,24 +6,26 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDataHolder; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; -import io.airbyte.integrations.standardtest.source.TestDataHolder; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.JsonSchemaType; import java.sql.SQLException; import java.util.Objects; import java.util.Set; import org.jooq.DSLContext; import org.jooq.SQLDialect; +import org.junit.jupiter.api.Disabled; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testcontainers.containers.CockroachContainer; +@Disabled public class CockroachDbSourceDatatypeTest extends AbstractSourceDatabaseTypeTest { private CockroachContainer container; diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbJdbcSourceAcceptanceTest.java index 7a3270322f1a..932d7fb1cace 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbJdbcSourceAcceptanceTest.java @@ -12,43 +12,26 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbState; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.string.Strings; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; -import io.airbyte.integrations.source.relationaldb.models.DbState; -import io.airbyte.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.AirbyteCatalog; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import io.airbyte.protocol.models.v0.*; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; -import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import io.airbyte.protocol.models.v0.AirbyteStateMessage; -import io.airbyte.protocol.models.v0.AirbyteStream; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.SyncMode; -import java.sql.JDBCType; import java.util.*; import java.util.stream.Collectors; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.testcontainers.containers.CockroachContainer; -class CockroachDbJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { +@Disabled +class CockroachDbJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { - private static CockroachContainer PSQL_DB; public static String COL_ROW_ID = "rowid"; public static Long ID_VALUE_1 = 1L; @@ -57,77 +40,34 @@ class CockroachDbJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { public static Long ID_VALUE_4 = 4L; public static Long ID_VALUE_5 = 5L; - private JsonNode config; - private String dbName; - - @BeforeAll - static void init() { - PSQL_DB = new CockroachContainer("cockroachdb/cockroach:v20.2.18"); - PSQL_DB.start(); - } - - @BeforeEach - public void setup() throws Exception { - dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, Objects.requireNonNull(PSQL_DB.getContainerInfo() - .getNetworkSettings() - .getNetworks() - .entrySet().stream() - .findFirst() - .get().getValue().getIpAddress())) - .put(JdbcUtils.PORT_KEY, PSQL_DB.getExposedPorts().get(1)) - .put(JdbcUtils.DATABASE_KEY, dbName) - .put(JdbcUtils.USERNAME_KEY, PSQL_DB.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, PSQL_DB.getPassword()) - .put(JdbcUtils.SSL_KEY, false) - .build()); - - final JsonNode clone = Jsons.clone(config); - ((ObjectNode) clone).put("database", PSQL_DB.getDatabaseName()); - final JsonNode jdbcConfig = getToDatabaseConfigFunction().apply(clone); - - database = new DefaultJdbcDatabase( - DataSourceFactory.create( - jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText(), - jdbcConfig.has(JdbcUtils.PASSWORD_KEY) ? jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, - getDriverClass(), - jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), - JdbcUtils.parseJdbcParameters(jdbcConfig, JdbcUtils.CONNECTION_PROPERTIES_KEY))); - database.execute(connection -> connection.createStatement().execute("CREATE DATABASE " + dbName + ";")); - super.setup(); - } + static final String DB_NAME = "postgres"; @Override protected String createTableQuery(final String tableName, final String columnClause, final String primaryKeyClause) { - return String.format("CREATE TABLE " + dbName + ".%s(%s %s %s)", + return String.format("CREATE TABLE " + DB_NAME + ".%s(%s %s %s)", tableName, columnClause, primaryKeyClause.equals("") ? "" : ",", primaryKeyClause); } @Override - public boolean supportsSchemas() { - return true; + protected CockroachDbTestDatabase createTestDatabase() { + final CockroachContainer cockroachContainer = new CockroachContainer("cockroachdb/cockroach:v20.2.18"); + cockroachContainer.start(); + return new CockroachDbTestDatabase(cockroachContainer).initialized(); } @Override - public AbstractJdbcSource getJdbcSource() { - return new CockroachDbSource(); + public boolean supportsSchemas() { + return true; } @Override - public JsonNode getConfig() { - return config; + protected CockroachDbSource source() { + return new CockroachDbSource(); } @Override - public String getDriverClass() { - return CockroachDbSource.DRIVER_CLASS; - } - - @AfterAll - static void cleanUp() { - PSQL_DB.close(); + public JsonNode config() { + return Jsons.clone(testdb.configBuilder().build()); } @Override @@ -163,32 +103,33 @@ protected AirbyteCatalog getCatalog(final String defaultNamespace) { @Override protected List getTestMessages() { - return Lists.newArrayList( + return List.of( new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName) + .withRecord(new AirbyteRecordMessage().withStream(streamName()) .withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(ImmutableMap + .withData(Jsons.jsonNode(Map .of(COL_ID, ID_VALUE_1, COL_NAME, "picard", COL_UPDATED_AT, "2004-10-19")))), new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName) + .withRecord(new AirbyteRecordMessage().withStream(streamName()) .withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(ImmutableMap + .withData(Jsons.jsonNode(Map .of(COL_ID, ID_VALUE_2, COL_NAME, "crusher", COL_UPDATED_AT, "2005-10-19")))), new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName) + .withRecord(new AirbyteRecordMessage().withStream(streamName()) .withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(ImmutableMap + .withData(Jsons.jsonNode(Map .of(COL_ID, ID_VALUE_3, COL_NAME, "vash", COL_UPDATED_AT, "2006-10-19"))))); } @Test + @Override protected void testDiscoverWithNonCursorFields() throws Exception { /* * this test is not valid for cockroach db, when table has no introduced PK it will add a hidden @@ -199,6 +140,7 @@ protected void testDiscoverWithNonCursorFields() throws Exception { } @Test + @Override protected void testDiscoverWithNullableCursorFields() throws Exception { /* * this test is not valid for cockroach db, when table has no introduced PK it will add a hidden @@ -209,20 +151,23 @@ protected void testDiscoverWithNullableCursorFields() throws Exception { } @Test - void testCheckFailure() throws Exception { + @Override + protected void testCheckFailure() throws Exception { + final JsonNode config = config(); ((ObjectNode) config).put(JdbcUtils.PASSWORD_KEY, "fake"); ((ObjectNode) config).put(JdbcUtils.USERNAME_KEY, "fake"); - final AirbyteConnectionStatus actual = source.check(config); + final AirbyteConnectionStatus actual = source().check(config); assertEquals(Status.FAILED, actual.getStatus()); } @Test - void testReadOneColumn() throws Exception { + @Override + protected void testReadOneColumn() throws Exception { final ConfiguredAirbyteCatalog catalog = CatalogHelpers - .createConfiguredAirbyteCatalog(streamName, getDefaultNamespace(), + .createConfiguredAirbyteCatalog(streamName(), getDefaultNamespace(), Field.of(COL_ID, JsonSchemaType.NUMBER)); final List actualMessages = MoreIterators - .toList(source.read(config, catalog, null)); + .toList(source().read(config(), catalog, null)); setEmittedAtToNull(actualMessages); @@ -241,7 +186,8 @@ void testReadOneColumn() throws Exception { } @Test - void testTablesWithQuoting() throws Exception { + @Override + protected void testTablesWithQuoting() throws Exception { final ConfiguredAirbyteStream streamForTableWithSpaces = createTableWithSpaces(); final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() @@ -249,7 +195,7 @@ void testTablesWithQuoting() throws Exception { getConfiguredCatalogWithOneStream(getDefaultNamespace()).getStreams().get(0), streamForTableWithSpaces)); final List actualMessages = MoreIterators - .toList(source.read(config, catalog, null)); + .toList(source().read(config(), catalog, null)); setEmittedAtToNull(actualMessages); @@ -274,7 +220,8 @@ void testTablesWithQuoting() throws Exception { } @Test - void testReadOneTableIncrementallyTwice() throws Exception { + @Override + protected void testReadOneTableIncrementallyTwice() throws Exception { final String namespace = getDefaultNamespace(); final ConfiguredAirbyteCatalog configuredCatalog = getConfiguredCatalogWithOneStream(namespace); configuredCatalog.getStreams().forEach(airbyteStream -> { @@ -285,38 +232,34 @@ void testReadOneTableIncrementallyTwice() throws Exception { final DbState state = new DbState() .withStreams(Lists.newArrayList( - new DbStreamState().withStreamName(streamName).withStreamNamespace(namespace))); + new DbStreamState().withStreamName(streamName()).withStreamNamespace(namespace))); final List actualMessagesFirstSync = MoreIterators - .toList(source.read(config, configuredCatalog, Jsons.jsonNode(state))); + .toList(source().read(config(), configuredCatalog, Jsons.jsonNode(state))); final Optional stateAfterFirstSyncOptional = actualMessagesFirstSync.stream() .filter(r -> r.getType() == Type.STATE).findFirst(); assertTrue(stateAfterFirstSyncOptional.isPresent()); - database.execute(connection -> { - connection.createStatement().execute( - String.format("INSERT INTO " + dbName + ".%s(id, name, updated_at) VALUES (4,'riker', '2006-10-19')", - getFullyQualifiedTableName(TABLE_NAME))); - connection.createStatement().execute( - String.format("INSERT INTO " + dbName + ".%s(id, name, updated_at) VALUES (5, 'data', '2006-10-19')", - getFullyQualifiedTableName(TABLE_NAME))); - }); + testdb.with(String.format("INSERT INTO " + DB_NAME + ".%s(id, name, updated_at) VALUES (4,'riker', '2006-10-19')", + getFullyQualifiedTableName(TABLE_NAME))) + .with(String.format("INSERT INTO " + DB_NAME + ".%s(id, name, updated_at) VALUES (5, 'data', '2006-10-19')", + getFullyQualifiedTableName(TABLE_NAME))); final List actualMessagesSecondSync = MoreIterators - .toList(source.read(config, configuredCatalog, + .toList(source().read(config(), configuredCatalog, stateAfterFirstSyncOptional.get().getState().getData())); assertEquals(2, (int) actualMessagesSecondSync.stream().filter(r -> r.getType() == Type.RECORD).count()); final List expectedMessages = new ArrayList<>(); expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + .withRecord(new AirbyteRecordMessage().withStream(streamName()).withNamespace(namespace) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_4, COL_NAME, "riker", COL_UPDATED_AT, "2006-10-19"))))); expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + .withRecord(new AirbyteRecordMessage().withStream(streamName()).withNamespace(namespace) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_5, COL_NAME, "data", @@ -324,11 +267,19 @@ void testReadOneTableIncrementallyTwice() throws Exception { expectedMessages.add(new AirbyteMessage() .withType(Type.STATE) .withState(new AirbyteStateMessage() - .withType(AirbyteStateMessage.AirbyteStateType.LEGACY) + .withType(AirbyteStateMessage.AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withName(streamName()).withNamespace(namespace)) + .withStreamState(Jsons.jsonNode(new DbStreamState() + .withStreamName(streamName()) + .withStreamNamespace(namespace) + .withCursor("5") + .withCursorRecordCount(1L) + .withCursorField(Collections.singletonList(COL_ID))))) .withData(Jsons.jsonNode(new DbState() .withCdc(false) .withStreams(Lists.newArrayList(new DbStreamState() - .withStreamName(streamName) + .withStreamName(streamName()) .withStreamNamespace(namespace) .withCursorField(ImmutableList.of(COL_ID)) .withCursor("5") @@ -342,29 +293,24 @@ void testReadOneTableIncrementallyTwice() throws Exception { } @Test - void testReadMultipleTables() throws Exception { + @Override + protected void testReadMultipleTables() throws Exception { final ConfiguredAirbyteCatalog catalog = getConfiguredCatalogWithOneStream( getDefaultNamespace()); final List expectedMessages = new ArrayList<>(getTestMessages()); for (int i = 2; i < 10; i++) { final int iFinal = i; - final String streamName2 = streamName + i; - database.execute(connection -> { - connection.createStatement() - .execute( - createTableQuery(getFullyQualifiedTableName(TABLE_NAME + iFinal), - "id INTEGER, name VARCHAR(200)", "")); - connection.createStatement() - .execute(String.format("INSERT INTO " + dbName + ".%s(id, name) VALUES (1,'picard')", - getFullyQualifiedTableName(TABLE_NAME + iFinal))); - connection.createStatement() - .execute(String.format("INSERT INTO " + dbName + ".%s(id, name) VALUES (2, 'crusher')", - getFullyQualifiedTableName(TABLE_NAME + iFinal))); - connection.createStatement() - .execute(String.format("INSERT INTO " + dbName + ".%s(id, name) VALUES (3, 'vash')", - getFullyQualifiedTableName(TABLE_NAME + iFinal))); - }); + final String streamName2 = streamName() + i; + testdb.with(createTableQuery(getFullyQualifiedTableName(TABLE_NAME + iFinal), + "id INTEGER, name VARCHAR(200)", "")) + .with(String.format("INSERT INTO " + DB_NAME + ".%s(id, name) VALUES (1,'picard')", + getFullyQualifiedTableName(TABLE_NAME + iFinal))) + .with(String.format("INSERT INTO " + DB_NAME + ".%s(id, name) VALUES (2, 'crusher')", + getFullyQualifiedTableName(TABLE_NAME + iFinal))) + .with(String.format("INSERT INTO " + DB_NAME + ".%s(id, name) VALUES (3, 'vash')", + getFullyQualifiedTableName(TABLE_NAME + iFinal))); + catalog.getStreams().add(CatalogHelpers.createConfiguredAirbyteStream( streamName2, getDefaultNamespace(), @@ -386,7 +332,7 @@ void testReadMultipleTables() throws Exception { } final List actualMessages = MoreIterators - .toList(source.read(config, catalog, null)); + .toList(source().read(config(), catalog, null)); setEmittedAtToNull(actualMessages); @@ -396,23 +342,18 @@ void testReadMultipleTables() throws Exception { } @Test - void testReadMultipleTablesIncrementally() throws Exception { + @Override + protected void testReadMultipleTablesIncrementally() throws Exception { final String tableName2 = TABLE_NAME + 2; - final String streamName2 = streamName + 2; - database.execute(ctx -> { - ctx.createStatement().execute( - createTableQuery(getFullyQualifiedTableName(tableName2), "id INTEGER, name VARCHAR(200)", - "")); - ctx.createStatement().execute( - String.format("INSERT INTO " + dbName + ".%s(id, name) VALUES (1,'picard')", - getFullyQualifiedTableName(tableName2))); - ctx.createStatement().execute( - String.format("INSERT INTO " + dbName + ".%s(id, name) VALUES (2, 'crusher')", - getFullyQualifiedTableName(tableName2))); - ctx.createStatement().execute( - String.format("INSERT INTO " + dbName + ".%s(id, name) VALUES (3, 'vash')", - getFullyQualifiedTableName(tableName2))); - }); + final String streamName2 = streamName() + 2; + testdb.with(createTableQuery(getFullyQualifiedTableName(tableName2), "id INTEGER, name VARCHAR(200)", + "")) + .with(String.format("INSERT INTO " + DB_NAME + ".%s(id, name) VALUES (1,'picard')", + getFullyQualifiedTableName(tableName2))) + .with(String.format("INSERT INTO " + DB_NAME + ".%s(id, name) VALUES (2, 'crusher')", + getFullyQualifiedTableName(tableName2))) + .with(String.format("INSERT INTO " + DB_NAME + ".%s(id, name) VALUES (3, 'vash')", + getFullyQualifiedTableName(tableName2))); final String namespace = getDefaultNamespace(); final ConfiguredAirbyteCatalog configuredCatalog = getConfiguredCatalogWithOneStream( @@ -430,9 +371,9 @@ void testReadMultipleTablesIncrementally() throws Exception { final DbState state = new DbState() .withStreams(Lists.newArrayList( - new DbStreamState().withStreamName(streamName).withStreamNamespace(namespace))); + new DbStreamState().withStreamName(streamName()).withStreamNamespace(namespace))); final List actualMessagesFirstSync = MoreIterators - .toList(source.read(config, configuredCatalog, Jsons.jsonNode(state))); + .toList(source().read(config(), configuredCatalog, Jsons.jsonNode(state))); // get last state message. final Optional stateAfterFirstSyncOptional = actualMessagesFirstSync.stream() @@ -456,12 +397,20 @@ void testReadMultipleTablesIncrementally() throws Exception { expectedMessagesFirstSync.add(new AirbyteMessage() .withType(Type.STATE) .withState(new AirbyteStateMessage() - .withType(AirbyteStateMessage.AirbyteStateType.LEGACY) + .withType(AirbyteStateMessage.AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withNamespace(namespace).withName(streamName())) + .withStreamState(Jsons.jsonNode(new DbStreamState() + .withStreamName(streamName()) + .withStreamNamespace(namespace) + .withCursor("3") + .withCursorRecordCount(1L) + .withCursorField(Collections.singletonList(COL_ID))))) .withData(Jsons.jsonNode(new DbState() .withCdc(false) .withStreams(Lists.newArrayList( new DbStreamState() - .withStreamName(streamName) + .withStreamName(streamName()) .withStreamNamespace(namespace) .withCursorField(ImmutableList.of(COL_ID)) .withCursor("3") @@ -475,12 +424,20 @@ void testReadMultipleTablesIncrementally() throws Exception { expectedMessagesFirstSync.add(new AirbyteMessage() .withType(Type.STATE) .withState(new AirbyteStateMessage() - .withType(AirbyteStateMessage.AirbyteStateType.LEGACY) + .withType(AirbyteStateMessage.AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withNamespace(namespace).withName(streamName2)) + .withStreamState(Jsons.jsonNode(new DbStreamState() + .withStreamName(streamName2) + .withStreamNamespace(namespace) + .withCursor("3") + .withCursorRecordCount(1L) + .withCursorField(Collections.singletonList(COL_ID))))) .withData(Jsons.jsonNode(new DbState() .withCdc(false) .withStreams(Lists.newArrayList( new DbStreamState() - .withStreamName(streamName) + .withStreamName(streamName()) .withStreamNamespace(namespace) .withCursorField(ImmutableList.of(COL_ID)) .withCursor("3") @@ -500,30 +457,19 @@ void testReadMultipleTablesIncrementally() throws Exception { } @Test - void testDiscoverWithMultipleSchemas() throws Exception { - // clickhouse and mysql do not have a concept of schemas, so this test does not make sense for them. - if (getDriverClass().toLowerCase().contains("mysql") || getDriverClass().toLowerCase() - .contains("clickhouse")) { - return; - } - + @Override + protected void testDiscoverWithMultipleSchemas() throws Exception { // add table and data to a separate schema. - database.execute(connection -> { - connection.createStatement().execute( - String.format("CREATE TABLE " + dbName + ".%s(id VARCHAR(200), name VARCHAR(200))", - JdbcUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))); - connection.createStatement() - .execute(String.format("INSERT INTO " + dbName + ".%s(id, name) VALUES ('1','picard')", - JdbcUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))); - connection.createStatement() - .execute(String.format("INSERT INTO " + dbName + ".%s(id, name) VALUES ('2', 'crusher')", - JdbcUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))); - connection.createStatement() - .execute(String.format("INSERT INTO " + dbName + ".%s(id, name) VALUES ('3', 'vash')", - JdbcUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))); - }); - - final AirbyteCatalog actual = source.discover(config); + testdb.with(String.format("CREATE TABLE " + DB_NAME + ".%s(id VARCHAR(200), name VARCHAR(200))", + JdbcUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))) + .with(String.format("INSERT INTO " + DB_NAME + ".%s(id, name) VALUES ('1','picard')", + JdbcUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))) + .with(String.format("INSERT INTO " + DB_NAME + ".%s(id, name) VALUES ('2', 'crusher')", + JdbcUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))) + .with(String.format("INSERT INTO " + DB_NAME + ".%s(id, name) VALUES ('3', 'vash')", + JdbcUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))); + + final AirbyteCatalog actual = source().discover(config()); final AirbyteCatalog expected = getCatalog(getDefaultNamespace()); expected.getStreams().add(CatalogHelpers diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceTest.java index f01f6082f0e2..713c7ff27327 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSourceTest.java @@ -11,13 +11,13 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; @@ -36,9 +36,11 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.testcontainers.containers.CockroachContainer; +@Disabled class CockroachDbSourceTest { private static final String SCHEMA_NAME = "public"; diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSpecTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSpecTest.java index 68a889fbcb07..92037e322f92 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSpecTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSpecTest.java @@ -10,10 +10,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.protocol.models.v0.ConnectorSpecification; import io.airbyte.validation.json.JsonSchemaValidator; import java.io.File; @@ -21,12 +21,14 @@ import java.nio.file.Files; import java.nio.file.Path; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; /** * Tests that the postgres spec passes JsonSchema validation. While this may seem like overkill, we * are doing it because there are some gotchas in correctly configuring the oneOf. */ +@Disabled public class CockroachDbSpecTest { private static final String CONFIGURATION = "{ " diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbTestDatabase.java b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbTestDatabase.java new file mode 100644 index 000000000000..0f1f87622a96 --- /dev/null +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbTestDatabase.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.cockroachdb; + +import static io.airbyte.integrations.source.cockroachdb.CockroachDbJdbcSourceAcceptanceTest.DB_NAME; + +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.TestDatabase; +import java.util.stream.Stream; +import org.jooq.SQLDialect; +import org.testcontainers.containers.CockroachContainer; + +public class CockroachDbTestDatabase extends + TestDatabase { + + private final CockroachContainer container; + + protected CockroachDbTestDatabase(final CockroachContainer container) { + super(container); + this.container = container; + } + + @Override + public String getJdbcUrl() { + return container.getJdbcUrl(); + } + + @Override + public String getUserName() { + return container.getUsername(); + } + + @Override + public String getPassword() { + return container.getPassword(); + } + + @Override + public String getDatabaseName() { + return DB_NAME; + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.empty(); + } + + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return DatabaseDriver.POSTGRESQL; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.POSTGRES; + } + + @Override + public void close() { + container.close(); + } + + @Override + public CockroachDbConfigBuilder configBuilder() { + return new CockroachDbConfigBuilder(this) + .with(JdbcUtils.HOST_KEY, container.getHost()) + .with(JdbcUtils.PORT_KEY, container.getMappedPort(26257)) + .with(JdbcUtils.DATABASE_KEY, DB_NAME) + .with(JdbcUtils.USERNAME_KEY, container.getUsername()) + .with(JdbcUtils.PASSWORD_KEY, container.getPassword()) + .with(JdbcUtils.SSL_KEY, false); + } + + static public class CockroachDbConfigBuilder extends TestDatabase.ConfigBuilder { + + protected CockroachDbConfigBuilder(final CockroachDbTestDatabase testdb) { + super(testdb); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-coda/README.md b/airbyte-integrations/connectors/source-coda/README.md index cb17f9a23641..a38c1310f4d2 100644 --- a/airbyte-integrations/connectors/source-coda/README.md +++ b/airbyte-integrations/connectors/source-coda/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-coda:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/coda) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_coda/spec.yaml` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-coda:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-coda build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-coda:airbyteDocker +An image will be built with the tag `airbyte/source-coda:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-coda:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-coda:dev check --confi docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-coda:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-coda:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-coda test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-coda:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-coda:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-coda test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/coda.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-coda/acceptance-test-config.yml b/airbyte-integrations/connectors/source-coda/acceptance-test-config.yml index d31c73aef6b8..e278e2d567bc 100644 --- a/airbyte-integrations/connectors/source-coda/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-coda/acceptance-test-config.yml @@ -24,7 +24,7 @@ acceptance_tests: - name: formulas bypass_reason: "Sandbox account can't seed the stream" - name: permissions - incremental: + incremental: bypass_reason: "This connector does not implement incremental sync" full_refresh: tests: diff --git a/airbyte-integrations/connectors/source-coda/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-coda/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-coda/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-coda/build.gradle b/airbyte-integrations/connectors/source-coda/build.gradle deleted file mode 100644 index 591f4084a04e..000000000000 --- a/airbyte-integrations/connectors/source-coda/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_coda' -} diff --git a/airbyte-integrations/connectors/source-coin-api/README.md b/airbyte-integrations/connectors/source-coin-api/README.md index e6066eb02c93..7f0f00c31d73 100644 --- a/airbyte-integrations/connectors/source-coin-api/README.md +++ b/airbyte-integrations/connectors/source-coin-api/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-coin-api:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/coin-api) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_coin_api/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-coin-api:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-coin-api build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-coin-api:airbyteDocker +An image will be built with the tag `airbyte/source-coin-api:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-coin-api:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-coin-api:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-coin-api:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-coin-api:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-coin-api test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-coin-api:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-coin-api:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-coin-api test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/coin-api.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-coin-api/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-coin-api/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-coin-api/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-coin-api/build.gradle b/airbyte-integrations/connectors/source-coin-api/build.gradle deleted file mode 100644 index 635bf0d2385f..000000000000 --- a/airbyte-integrations/connectors/source-coin-api/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_coin_api' -} diff --git a/airbyte-integrations/connectors/source-coingecko-coins/README.md b/airbyte-integrations/connectors/source-coingecko-coins/README.md index 090b4bd32dab..c35987386b92 100644 --- a/airbyte-integrations/connectors/source-coingecko-coins/README.md +++ b/airbyte-integrations/connectors/source-coingecko-coins/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-coingecko-coins:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/coingecko-coins) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_coingecko_coins/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-coingecko-coins:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-coingecko-coins build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-coingecko-coins:airbyteDocker +An image will be built with the tag `airbyte/source-coingecko-coins:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-coingecko-coins:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-coingecko-coins:dev ch docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-coingecko-coins:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-coingecko-coins:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-coingecko-coins test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-coingecko-coins:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-coingecko-coins:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-coingecko-coins test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/coingecko-coins.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-coingecko-coins/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-coingecko-coins/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-coingecko-coins/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-coingecko-coins/build.gradle b/airbyte-integrations/connectors/source-coingecko-coins/build.gradle deleted file mode 100644 index 04085cbbc265..000000000000 --- a/airbyte-integrations/connectors/source-coingecko-coins/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_coingecko_coins' -} diff --git a/airbyte-integrations/connectors/source-coinmarketcap/README.md b/airbyte-integrations/connectors/source-coinmarketcap/README.md index 6858e55fe36e..85a5701114c4 100644 --- a/airbyte-integrations/connectors/source-coinmarketcap/README.md +++ b/airbyte-integrations/connectors/source-coinmarketcap/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-coinmarketcap:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/coinmarketcap) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_coinmarketcap/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-coinmarketcap:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-coinmarketcap build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-coinmarketcap:airbyteDocker +An image will be built with the tag `airbyte/source-coinmarketcap:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-coinmarketcap:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-coinmarketcap:dev chec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-coinmarketcap:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-coinmarketcap:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-coinmarketcap test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-coinmarketcap:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-coinmarketcap:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-coinmarketcap test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/coinmarketcap.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-coinmarketcap/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-coinmarketcap/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-coinmarketcap/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-coinmarketcap/build.gradle b/airbyte-integrations/connectors/source-coinmarketcap/build.gradle deleted file mode 100644 index 4aa435500679..000000000000 --- a/airbyte-integrations/connectors/source-coinmarketcap/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_coinmarketcap' -} diff --git a/airbyte-integrations/connectors/source-commcare/README.md b/airbyte-integrations/connectors/source-commcare/README.md index 659dd3037c48..af931d5d8e5e 100644 --- a/airbyte-integrations/connectors/source-commcare/README.md +++ b/airbyte-integrations/connectors/source-commcare/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-commcare:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/commcare) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_commcare/spec.yaml` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-commcare:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-commcare build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-commcare:airbyteDocker +An image will be built with the tag `airbyte/source-commcare:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-commcare:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-commcare:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-commcare:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-commcare:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-commcare test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-commcare:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-commcare:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-commcare test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/commcare.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-commcare/acceptance-test-config.yml b/airbyte-integrations/connectors/source-commcare/acceptance-test-config.yml index 205c4d88126f..86ddfe670764 100644 --- a/airbyte-integrations/connectors/source-commcare/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-commcare/acceptance-test-config.yml @@ -20,7 +20,7 @@ tests: full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - incremental: + incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" \ No newline at end of file + future_state_path: "integration_tests/abnormal_state.json" diff --git a/airbyte-integrations/connectors/source-commcare/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-commcare/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-commcare/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-commcare/build.gradle b/airbyte-integrations/connectors/source-commcare/build.gradle deleted file mode 100644 index 78c86bc364a5..000000000000 --- a/airbyte-integrations/connectors/source-commcare/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_commcare' -} diff --git a/airbyte-integrations/connectors/source-commcare/unit_tests/test_source.py b/airbyte-integrations/connectors/source-commcare/unit_tests/test_source.py index dac61860c0c4..b38e196d2561 100644 --- a/airbyte-integrations/connectors/source-commcare/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-commcare/unit_tests/test_source.py @@ -8,9 +8,9 @@ from source_commcare.source import SourceCommcare -@pytest.fixture(name='config') +@pytest.fixture(name="config") def config_fixture(): - return {'api_key': 'apikey', 'app_id': 'appid', 'start_date': '2022-01-01T00:00:00Z'} + return {"api_key": "apikey", "app_id": "appid", "start_date": "2022-01-01T00:00:00Z"} def test_check_connection_ok(mocker, config): diff --git a/airbyte-integrations/connectors/source-commercetools/README.md b/airbyte-integrations/connectors/source-commercetools/README.md index d79ac18a3c12..20b53f7ec0d5 100644 --- a/airbyte-integrations/connectors/source-commercetools/README.md +++ b/airbyte-integrations/connectors/source-commercetools/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-commercetools:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/commercetools) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_commercetools/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-commercetools:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-commercetools build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-commercetools:airbyteDocker +An image will be built with the tag `airbyte/source-commercetools:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-commercetools:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-commercetools:dev chec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-commercetools:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-commercetools:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-commercetools test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-commercetools:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-commercetools:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-commercetools test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/commercetools.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-commercetools/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-commercetools/acceptance-test-docker.sh deleted file mode 100644 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-commercetools/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-commercetools/build.gradle b/airbyte-integrations/connectors/source-commercetools/build.gradle deleted file mode 100644 index 8b7d69253918..000000000000 --- a/airbyte-integrations/connectors/source-commercetools/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_commercetools' -} diff --git a/airbyte-integrations/connectors/source-commercetools/metadata.yaml b/airbyte-integrations/connectors/source-commercetools/metadata.yaml index dcbb88d3fafe..5af0802c2c0e 100644 --- a/airbyte-integrations/connectors/source-commercetools/metadata.yaml +++ b/airbyte-integrations/connectors/source-commercetools/metadata.yaml @@ -20,7 +20,7 @@ data: releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/commercetools tags: - - language:lowcode + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-configcat/README.md b/airbyte-integrations/connectors/source-configcat/README.md index 1b15f5857321..9de06ba31ce1 100644 --- a/airbyte-integrations/connectors/source-configcat/README.md +++ b/airbyte-integrations/connectors/source-configcat/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-configcat:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/configcat) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_configcat/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-configcat:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-configcat build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-configcat:airbyteDocker +An image will be built with the tag `airbyte/source-configcat:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-configcat:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-configcat:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-configcat:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-configcat:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-configcat test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-configcat:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-configcat:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-configcat test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/configcat.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-configcat/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-configcat/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-configcat/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-configcat/build.gradle b/airbyte-integrations/connectors/source-configcat/build.gradle deleted file mode 100644 index 73167f5abb79..000000000000 --- a/airbyte-integrations/connectors/source-configcat/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_configcat' -} diff --git a/airbyte-integrations/connectors/source-confluence/README.md b/airbyte-integrations/connectors/source-confluence/README.md index a884f5a32a49..179ecdce34b6 100644 --- a/airbyte-integrations/connectors/source-confluence/README.md +++ b/airbyte-integrations/connectors/source-confluence/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-confluence:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/confluence) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_confluence/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-confluence:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-confluence build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-confluence:airbyteDocker +An image will be built with the tag `airbyte/source-confluence:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-confluence:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-confluence:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-confluence:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-confluence:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-confluence test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-confluence:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-confluence:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-confluence test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/confluence.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-confluence/acceptance-test-config.yml b/airbyte-integrations/connectors/source-confluence/acceptance-test-config.yml index 581340ee6eab..6ff3ec48adc8 100644 --- a/airbyte-integrations/connectors/source-confluence/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-confluence/acceptance-test-config.yml @@ -4,31 +4,31 @@ connector_image: airbyte/source-confluence:dev acceptance_tests: spec: tests: - - spec_path: "source_confluence/spec.yaml" + - spec_path: "source_confluence/spec.yaml" connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: tests: - - config_path: "secrets/config.json" + - config_path: "secrets/config.json" basic_read: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - fail_on_extra_columns: false - ignored_fields: - pages: - - name: body/view/value - bypass_reason: "Different class order" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + expect_records: + path: "integration_tests/expected_records.jsonl" + fail_on_extra_columns: false + ignored_fields: + pages: + - name: body/view/value + bypass_reason: "Different class order" full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - ignored_fields: - pages: - - name: "body/view" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + ignored_fields: + pages: + - name: "body/view" diff --git a/airbyte-integrations/connectors/source-confluence/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-confluence/acceptance-test-docker.sh deleted file mode 100644 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-confluence/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-confluence/build.gradle b/airbyte-integrations/connectors/source-confluence/build.gradle deleted file mode 100644 index 7e0a2354b838..000000000000 --- a/airbyte-integrations/connectors/source-confluence/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_confluence' -} diff --git a/airbyte-integrations/connectors/source-confluence/metadata.yaml b/airbyte-integrations/connectors/source-confluence/metadata.yaml index f45628d976b2..87d324b5ebec 100644 --- a/airbyte-integrations/connectors/source-confluence/metadata.yaml +++ b/airbyte-integrations/connectors/source-confluence/metadata.yaml @@ -1,4 +1,7 @@ data: + ab_internal: + ql: 300 + sl: 100 allowedHosts: hosts: - ${subdomain}.atlassian.net @@ -12,17 +15,14 @@ data: definitionId: cf40a7f8-71f8-45ce-a7fa-fca053e4028c dockerImageTag: 0.2.0 dockerRepository: airbyte/source-confluence + documentationUrl: https://docs.airbyte.com/integrations/sources/confluence githubIssueLabel: source-confluence icon: confluence.svg license: MIT name: Confluence releaseDate: 2021-11-05 releaseStage: beta - supportLevel: certified - documentationUrl: https://docs.airbyte.com/integrations/sources/confluence + supportLevel: community tags: - language:low-code - ab_internal: - sl: 200 - ql: 300 metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-convertkit/README.md b/airbyte-integrations/connectors/source-convertkit/README.md index 8ea831dba592..d9e9cf2ac883 100644 --- a/airbyte-integrations/connectors/source-convertkit/README.md +++ b/airbyte-integrations/connectors/source-convertkit/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-convertkit:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/convertkit) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_convertkit/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-convertkit:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-convertkit build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-convertkit:airbyteDocker +An image will be built with the tag `airbyte/source-convertkit:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-convertkit:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-convertkit:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-convertkit:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-convertkit:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-convertkit test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-convertkit:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-convertkit:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-convertkit test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/convertkit.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-convertkit/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-convertkit/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-convertkit/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-convertkit/build.gradle b/airbyte-integrations/connectors/source-convertkit/build.gradle deleted file mode 100644 index f62e24d99570..000000000000 --- a/airbyte-integrations/connectors/source-convertkit/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_convertkit' -} diff --git a/airbyte-integrations/connectors/source-convex/Dockerfile b/airbyte-integrations/connectors/source-convex/Dockerfile index 472b313e8dd7..3b46ff759568 100644 --- a/airbyte-integrations/connectors/source-convex/Dockerfile +++ b/airbyte-integrations/connectors/source-convex/Dockerfile @@ -34,5 +34,5 @@ COPY source_convex ./source_convex ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.version=0.4.0 LABEL io.airbyte.name=airbyte/source-convex diff --git a/airbyte-integrations/connectors/source-convex/README.md b/airbyte-integrations/connectors/source-convex/README.md index 0998e63ec879..50d5c8f19770 100644 --- a/airbyte-integrations/connectors/source-convex/README.md +++ b/airbyte-integrations/connectors/source-convex/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-convex:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/convex) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_convex/spec.yaml` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-convex:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-convex build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-convex:airbyteDocker +An image will be built with the tag `airbyte/source-convex:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-convex:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-convex:dev check --con docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-convex:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-convex:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-convex test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-convex:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-convex:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-convex test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/convex.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-convex/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-convex/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-convex/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-convex/build.gradle b/airbyte-integrations/connectors/source-convex/build.gradle deleted file mode 100644 index 4b9986cd278c..000000000000 --- a/airbyte-integrations/connectors/source-convex/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_convex' -} diff --git a/airbyte-integrations/connectors/source-convex/metadata.yaml b/airbyte-integrations/connectors/source-convex/metadata.yaml index 911f0ab12cf9..b58c7a7b40dd 100644 --- a/airbyte-integrations/connectors/source-convex/metadata.yaml +++ b/airbyte-integrations/connectors/source-convex/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: c332628c-f55c-4017-8222-378cfafda9b2 - dockerImageTag: 0.2.0 + dockerImageTag: 0.4.0 dockerRepository: airbyte/source-convex githubIssueLabel: source-convex icon: convex.svg diff --git a/airbyte-integrations/connectors/source-convex/source_convex/source.py b/airbyte-integrations/connectors/source-convex/source_convex/source.py index ecf094ff9c01..664f5bf3ca16 100644 --- a/airbyte-integrations/connectors/source-convex/source_convex/source.py +++ b/airbyte-integrations/connectors/source-convex/source_convex/source.py @@ -5,7 +5,7 @@ from datetime import datetime from json import JSONDecodeError -from typing import Any, Dict, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Tuple, TypedDict +from typing import Any, Dict, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Tuple, TypedDict, cast import requests from airbyte_cdk.models import SyncMode @@ -31,7 +31,7 @@ }, ) -CONVEX_CLIENT_VERSION = "0.2.0" +CONVEX_CLIENT_VERSION = "0.4.0" # Source @@ -39,14 +39,14 @@ class SourceConvex(AbstractSource): def _json_schemas(self, config: ConvexConfig) -> requests.Response: deployment_url = config["deployment_url"] access_key = config["access_key"] - url = f"{deployment_url}/api/json_schemas?deltaSchema=true&format=convex_json" + url = f"{deployment_url}/api/json_schemas?deltaSchema=true&format=json" headers = { "Authorization": f"Convex {access_key}", "Convex-Client": f"airbyte-export-{CONVEX_CLIENT_VERSION}", } return requests.get(url, headers=headers) - def check_connection(self, logger: Any, config: ConvexConfig) -> Tuple[bool, Any]: + def check_connection(self, logger: Any, config: Mapping[str, Any]) -> Tuple[bool, Any]: """ Connection check to validate that the user-provided config can be used to connect to the underlying API @@ -54,16 +54,18 @@ def check_connection(self, logger: Any, config: ConvexConfig) -> Tuple[bool, Any :param logger: logger object :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ + config = cast(ConvexConfig, config) resp = self._json_schemas(config) if resp.status_code == 200: return True, None else: return False, format_http_error("Connection to Convex via json_schemas endpoint failed", resp) - def streams(self, config: ConvexConfig) -> List[Stream]: + def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ :param config: A Mapping of the user input configuration as defined in the connector spec. """ + config = cast(ConvexConfig, config) resp = self._json_schemas(config) if resp.status_code != 200: raise Exception(format_http_error("Failed request to json_schemas", resp)) @@ -73,6 +75,7 @@ def streams(self, config: ConvexConfig) -> List[Stream]: ConvexStream( config["deployment_url"], config["access_key"], + "json", # Use `json` export format table_name, json_schemas[table_name], ) @@ -81,8 +84,16 @@ def streams(self, config: ConvexConfig) -> List[Stream]: class ConvexStream(HttpStream, IncrementalMixin): - def __init__(self, deployment_url: str, access_key: str, table_name: str, json_schema: Mapping[str, Any]): + def __init__( + self, + deployment_url: str, + access_key: str, + fmt: str, + table_name: str, + json_schema: Dict[str, Any], + ): self.deployment_url = deployment_url + self.fmt = fmt self.table_name = table_name if json_schema: json_schema["additionalProperties"] = True @@ -106,7 +117,7 @@ def name(self) -> str: def url_base(self) -> str: return self.deployment_url - def get_json_schema(self) -> Mapping[str, Any]: + def get_json_schema(self) -> Mapping[str, Any]: # type: ignore[override] return self.json_schema primary_key = "_id" @@ -116,18 +127,20 @@ def get_json_schema(self) -> Mapping[str, Any]: state_checkpoint_interval = 128 @property - def state(self) -> ConvexState: - return { + def state(self) -> MutableMapping[str, Any]: + value: ConvexState = { "snapshot_cursor": self._snapshot_cursor_value, "snapshot_has_more": self._snapshot_has_more, "delta_cursor": self._delta_cursor_value, } + return cast(MutableMapping[str, Any], value) @state.setter - def state(self, value: ConvexState) -> None: - self._snapshot_cursor_value = value["snapshot_cursor"] - self._snapshot_has_more = value["snapshot_has_more"] - self._delta_cursor_value = value["delta_cursor"] + def state(self, value: MutableMapping[str, Any]) -> None: + state = cast(ConvexState, value) + self._snapshot_cursor_value = state["snapshot_cursor"] + self._snapshot_has_more = state["snapshot_has_more"] + self._delta_cursor_value = state["delta_cursor"] def next_page_token(self, response: requests.Response) -> Optional[ConvexState]: if response.status_code != 200: @@ -140,13 +153,14 @@ def next_page_token(self, response: requests.Response) -> Optional[ConvexState]: else: self._delta_cursor_value = resp_json["cursor"] self._delta_has_more = resp_json["hasMore"] - return self.state if self._delta_has_more else None + has_more = self._snapshot_has_more or self._delta_has_more + return cast(ConvexState, self.state) if has_more else None def path( self, - stream_state: Optional[ConvexState] = None, + stream_state: Optional[Mapping[str, Any]] = None, stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[ConvexState] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> str: # https://docs.convex.dev/http-api/#sync if self._snapshot_has_more: @@ -157,10 +171,10 @@ def path( def parse_response( self, response: requests.Response, - stream_state: ConvexState, + stream_state: Mapping[str, Any], stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[ConvexState] = None, - ) -> Iterable[Any]: + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Iterable[Mapping[str, Any]]: if response.status_code != 200: raise Exception(format_http_error("Failed request", response)) resp_json = response.json() @@ -168,11 +182,11 @@ def parse_response( def request_params( self, - stream_state: ConvexState, + stream_state: Optional[Mapping[str, Any]], stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[ConvexState] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> MutableMapping[str, Any]: - params: Dict[str, Any] = {"tableName": self.table_name, "format": "convex_json"} + params: Dict[str, Any] = {"tableName": self.table_name, "format": self.fmt} if self._snapshot_has_more: if self._snapshot_cursor_value: params["cursor"] = self._snapshot_cursor_value @@ -185,9 +199,9 @@ def request_params( def request_headers( self, - stream_state: ConvexState, + stream_state: Optional[Mapping[str, Any]], stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[ConvexState] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> Dict[str, str]: """ Custom headers for each HTTP request, not including Authorization. @@ -200,11 +214,12 @@ def get_updated_state(self, current_stream_state: ConvexState, latest_record: Ma """ This (deprecated) method is still used by AbstractSource to update state between calls to `read_records`. """ - return self.state + return cast(ConvexState, self.state) def read_records(self, sync_mode: SyncMode, *args: Any, **kwargs: Any) -> Iterator[Any]: self._delta_has_more = sync_mode == SyncMode.incremental - for record in super().read_records(sync_mode, *args, **kwargs): + for read_record in super().read_records(sync_mode, *args, **kwargs): + record = dict(read_record) ts_ns = record["_ts"] ts_seconds = ts_ns / 1e9 # convert from nanoseconds. # equivalent of java's `new Timestamp(transactionMillis).toInstant().toString()` diff --git a/airbyte-integrations/connectors/source-convex/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-convex/unit_tests/test_incremental_streams.py index 6f7a56c5577c..c1006b9f6167 100644 --- a/airbyte-integrations/connectors/source-convex/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-convex/unit_tests/test_incremental_streams.py @@ -19,13 +19,13 @@ def patch_incremental_base_class(mocker): def test_cursor_field(patch_incremental_base_class): - stream = ConvexStream("murky-swan-635", "accesskey", "messages", None) + stream = ConvexStream("murky-swan-635", "accesskey", "json", "messages", None) expected_cursor_field = "_ts" assert stream.cursor_field == expected_cursor_field def test_get_updated_state(patch_incremental_base_class): - stream = ConvexStream("murky-swan-635", "accesskey", "messages", None) + stream = ConvexStream("murky-swan-635", "accesskey", "json", "messages", None) resp = MagicMock() resp.json = lambda: {"values": [{"_id": "my_id", "field": "f", "_ts": 123}], "cursor": 1234, "snapshot": 3000, "hasMore": True} resp.status_code = 200 @@ -65,7 +65,7 @@ def test_get_updated_state(patch_incremental_base_class): def test_stream_slices(patch_incremental_base_class): - stream = ConvexStream("murky-swan-635", "accesskey", "messages", None) + stream = ConvexStream("murky-swan-635", "accesskey", "json", "messages", None) inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} expected_stream_slice = [None] assert stream.stream_slices(**inputs) == expected_stream_slice @@ -73,16 +73,16 @@ def test_stream_slices(patch_incremental_base_class): def test_supports_incremental(patch_incremental_base_class, mocker): mocker.patch.object(ConvexStream, "cursor_field", "dummy_field") - stream = ConvexStream("murky-swan-635", "accesskey", "messages", None) + stream = ConvexStream("murky-swan-635", "accesskey", "json", "messages", None) assert stream.supports_incremental def test_source_defined_cursor(patch_incremental_base_class): - stream = ConvexStream("murky-swan-635", "accesskey", "messages", None) + stream = ConvexStream("murky-swan-635", "accesskey", "json", "messages", None) assert stream.source_defined_cursor def test_stream_checkpoint_interval(patch_incremental_base_class): - stream = ConvexStream("murky-swan-635", "accesskey", "messages", None) + stream = ConvexStream("murky-swan-635", "accesskey", "json", "messages", None) expected_checkpoint_interval = 128 assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-convex/unit_tests/test_source.py b/airbyte-integrations/connectors/source-convex/unit_tests/test_source.py index 156ebd7f7cb2..134b908698c3 100644 --- a/airbyte-integrations/connectors/source-convex/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-convex/unit_tests/test_source.py @@ -37,14 +37,14 @@ def setup_responses(): } responses.add( responses.GET, - "https://murky-swan-635.convex.cloud/api/json_schemas?deltaSchema=true&format=convex_json", + "https://murky-swan-635.convex.cloud/api/json_schemas?deltaSchema=true&format=json", json=sample_shapes_resp, ) responses.add( responses.GET, - "https://curious-giraffe-964.convex.cloud/api/json_schemas?deltaSchema=true&format=convex_json", - json={'code': "Error code", "message": "Error message"}, - status=400 + "https://curious-giraffe-964.convex.cloud/api/json_schemas?deltaSchema=true&format=json", + json={"code": "Error code", "message": "Error message"}, + status=400, ) diff --git a/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py index 267126670024..17512d01cf07 100644 --- a/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-convex/unit_tests/test_streams.py @@ -6,33 +6,35 @@ from unittest.mock import MagicMock import pytest +import requests +import responses +from airbyte_cdk.models import SyncMode from source_convex.source import ConvexStream @pytest.fixture def patch_base_class(mocker): # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(ConvexStream, "path", "v0/example_endpoint") mocker.patch.object(ConvexStream, "primary_key", "test_primary_key") mocker.patch.object(ConvexStream, "__abstractmethods__", set()) def test_request_params(patch_base_class): - stream = ConvexStream("murky-swan-635", "accesskey", "messages", None) + stream = ConvexStream("murky-swan-635", "accesskey", "json", "messages", None) inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {"tableName": "messages", "format": "convex_json"} + expected_params = {"tableName": "messages", "format": "json"} assert stream.request_params(**inputs) == expected_params stream._snapshot_cursor_value = 1234 - expected_params = {"tableName": "messages", "format": "convex_json", "cursor": 1234} + expected_params = {"tableName": "messages", "format": "json", "cursor": 1234} assert stream.request_params(**inputs) == expected_params stream._snapshot_has_more = False stream._delta_cursor_value = 2345 - expected_params = {"tableName": "messages", "format": "convex_json", "cursor": 2345} + expected_params = {"tableName": "messages", "format": "json", "cursor": 2345} assert stream.request_params(**inputs) == expected_params def test_next_page_token(patch_base_class): - stream = ConvexStream("murky-swan-635", "accesskey", "messages", None) + stream = ConvexStream("murky-swan-635", "accesskey", "json", "messages", None) resp = MagicMock() resp.json = lambda: {"values": [{"_id": "my_id", "field": "f", "_ts": 123}], "cursor": 1234, "snapshot": 5000, "hasMore": True} resp.status_code = 200 @@ -62,8 +64,68 @@ def test_next_page_token(patch_base_class): assert stream.state == {"snapshot_cursor": 1235, "snapshot_has_more": False, "delta_cursor": 7000} +@responses.activate +def test_read_records_full_refresh(patch_base_class): + stream = ConvexStream("http://mocked_base_url:8080", "accesskey", "json", "messages", None) + snapshot0_resp = {"values": [{"_id": "my_id", "field": "f", "_ts": 123}], "cursor": 1234, "snapshot": 5000, "hasMore": True} + responses.add( + responses.GET, + "http://mocked_base_url:8080/api/list_snapshot?tableName=messages&format=json", + json=snapshot0_resp, + ) + snapshot1_resp = {"values": [{"_id": "an_id", "field": "b", "_ts": 100}], "cursor": 2345, "snapshot": 5000, "hasMore": True} + responses.add( + responses.GET, + "http://mocked_base_url:8080/api/list_snapshot?tableName=messages&format=json&cursor=1234&snapshot=5000", + json=snapshot1_resp, + ) + snapshot2_resp = {"values": [{"_id": "a_id", "field": "x", "_ts": 300}], "cursor": 3456, "snapshot": 5000, "hasMore": False} + responses.add( + responses.GET, + "http://mocked_base_url:8080/api/list_snapshot?tableName=messages&format=json&cursor=2345&snapshot=5000", + json=snapshot2_resp, + ) + records = list(stream.read_records(SyncMode.full_refresh)) + assert len(records) == 3 + assert [record["field"] for record in records] == ["f", "b", "x"] + assert stream.state == {"delta_cursor": 5000, "snapshot_cursor": 3456, "snapshot_has_more": False} + + +@responses.activate +def test_read_records_incremental(patch_base_class): + stream = ConvexStream("http://mocked_base_url:8080", "accesskey", "json", "messages", None) + snapshot0_resp = {"values": [{"_id": "my_id", "field": "f", "_ts": 123}], "cursor": 1234, "snapshot": 5000, "hasMore": True} + responses.add( + responses.GET, + "http://mocked_base_url:8080/api/list_snapshot?tableName=messages&format=json", + json=snapshot0_resp, + ) + snapshot1_resp = {"values": [{"_id": "an_id", "field": "b", "_ts": 100}], "cursor": 2345, "snapshot": 5000, "hasMore": False} + responses.add( + responses.GET, + "http://mocked_base_url:8080/api/list_snapshot?tableName=messages&format=json&cursor=1234&snapshot=5000", + json=snapshot1_resp, + ) + delta0_resp = {"values": [{"_id": "a_id", "field": "x", "_ts": 300}], "cursor": 6000, "hasMore": True} + responses.add( + responses.GET, + "http://mocked_base_url:8080/api/document_deltas?tableName=messages&format=json&cursor=5000", + json=delta0_resp, + ) + delta1_resp = {"values": [{"_id": "a_id", "field": "x", "_ts": 400}], "cursor": 7000, "hasMore": False} + responses.add( + responses.GET, + "http://mocked_base_url:8080/api/document_deltas?tableName=messages&format=json&cursor=6000", + json=delta1_resp, + ) + records = list(stream.read_records(SyncMode.incremental)) + assert len(records) == 4 + assert [record["field"] for record in records] == ["f", "b", "x", "x"] + assert stream.state == {"delta_cursor": 7000, "snapshot_cursor": 2345, "snapshot_has_more": False} + + def test_parse_response(patch_base_class): - stream = ConvexStream("murky-swan-635", "accesskey", "messages", None) + stream = ConvexStream("murky-swan-635", "accesskey", "json", "messages", None) resp = MagicMock() resp.json = lambda: {"values": [{"_id": "my_id", "field": "f", "_ts": 1234}], "cursor": 1234, "snapshot": 2000, "hasMore": True} resp.status_code = 200 @@ -73,13 +135,13 @@ def test_parse_response(patch_base_class): def test_request_headers(patch_base_class): - stream = ConvexStream("murky-swan-635", "accesskey", "messages", None) + stream = ConvexStream("murky-swan-635", "accesskey", "json", "messages", None) inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - assert stream.request_headers(**inputs) == {"Convex-Client": "airbyte-export-0.2.0"} + assert stream.request_headers(**inputs) == {"Convex-Client": "airbyte-export-0.4.0"} def test_http_method(patch_base_class): - stream = ConvexStream("murky-swan-635", "accesskey", "messages", None) + stream = ConvexStream("murky-swan-635", "accesskey", "json", "messages", None) expected_method = "GET" assert stream.http_method == expected_method @@ -96,12 +158,12 @@ def test_http_method(patch_base_class): def test_should_retry(patch_base_class, http_status, should_retry): response_mock = MagicMock() response_mock.status_code = http_status - stream = ConvexStream("murky-swan-635", "accesskey", "messages", None) + stream = ConvexStream("murky-swan-635", "accesskey", "json", "messages", None) assert stream.should_retry(response_mock) == should_retry def test_backoff_time(patch_base_class): response_mock = MagicMock() - stream = ConvexStream("murky-swan-635", "accesskey", "messages", None) + stream = ConvexStream("murky-swan-635", "accesskey", "json", "messages", None) expected_backoff_time = None assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-copper/Dockerfile b/airbyte-integrations/connectors/source-copper/Dockerfile index 85e6e9425e6c..6e7bdbce34f3 100644 --- a/airbyte-integrations/connectors/source-copper/Dockerfile +++ b/airbyte-integrations/connectors/source-copper/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.13-alpine3.15 as base +FROM python:3.9.11-alpine3.15 as base # build and load all requirements FROM base as builder @@ -34,5 +34,6 @@ COPY source_copper ./source_copper ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.1 + +LABEL io.airbyte.version=0.3.0 LABEL io.airbyte.name=airbyte/source-copper diff --git a/airbyte-integrations/connectors/source-copper/README.md b/airbyte-integrations/connectors/source-copper/README.md index b6e7d25f1328..738008f1139d 100644 --- a/airbyte-integrations/connectors/source-copper/README.md +++ b/airbyte-integrations/connectors/source-copper/README.md @@ -1,45 +1,12 @@ # Copper Source -This is the repository for the Copper source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/copper). +This is the repository for the Copper configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/copper). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-copper:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/copper) +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/copper) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_copper/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. @@ -47,28 +14,21 @@ See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source copper test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-copper:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-copper build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-copper:airbyteDocker +An image will be built with the tag `airbyte/source-copper:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-copper:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-copper:dev check --con docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-copper:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-copper:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-copper test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-copper:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-copper:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-copper test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/copper.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-copper/__init__.py b/airbyte-integrations/connectors/source-copper/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-copper/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-copper/acceptance-test-config.yml b/airbyte-integrations/connectors/source-copper/acceptance-test-config.yml index 9d1a24bc8d47..42544e846a39 100644 --- a/airbyte-integrations/connectors/source-copper/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-copper/acceptance-test-config.yml @@ -1,5 +1,3 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests connector_image: airbyte/source-copper:dev acceptance_tests: spec: @@ -16,12 +14,12 @@ acceptance_tests: - config_path: "secrets/config.json" backward_compatibility_tests_config: disable_for_version: "0.2.0" + basic_read: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file # expect_records: # path: "integration_tests/expected_records.jsonl" # extra_fields: no @@ -29,11 +27,11 @@ acceptance_tests: # extra_records: yes incremental: bypass_reason: "This connector does not implement incremental sync" - # TODO uncomment this block this block if your connector implements incremental sync: # tests: # - config_path: "secrets/config.json" # configured_catalog_path: "integration_tests/configured_catalog.json" - # future_state_path: "integration_tests/abnormal_state.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-copper/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-copper/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-copper/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-copper/build.gradle b/airbyte-integrations/connectors/source-copper/build.gradle deleted file mode 100644 index 652f68b862ba..000000000000 --- a/airbyte-integrations/connectors/source-copper/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_copper' -} diff --git a/airbyte-integrations/connectors/source-copper/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-copper/integration_tests/acceptance.py index 9e6409236281..82823254d266 100644 --- a/airbyte-integrations/connectors/source-copper/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-copper/integration_tests/acceptance.py @@ -11,6 +11,4 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" - # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield - # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-copper/metadata.yaml b/airbyte-integrations/connectors/source-copper/metadata.yaml index 884ebdff532d..283dd98e3a15 100644 --- a/airbyte-integrations/connectors/source-copper/metadata.yaml +++ b/airbyte-integrations/connectors/source-copper/metadata.yaml @@ -1,24 +1,24 @@ data: + allowedHosts: + hosts: + - https://api.copper.com/ + registries: + oss: + enabled: true + cloud: + enabled: false connectorSubtype: api connectorType: source definitionId: 44f3002f-2df9-4f6d-b21c-02cd3b47d0dc - dockerImageTag: 0.2.1 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-copper githubIssueLabel: source-copper icon: copper.svg license: MIT name: Copper - registries: - cloud: - enabled: false - oss: - enabled: true releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/copper tags: - - language:python - ab_internal: - sl: 100 - ql: 100 - supportLevel: community + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-copper/setup.py b/airbyte-integrations/connectors/source-copper/setup.py index 0ede569ed344..15fe92381858 100644 --- a/airbyte-integrations/connectors/source-copper/setup.py +++ b/airbyte-integrations/connectors/source-copper/setup.py @@ -5,16 +5,9 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", -] +MAIN_REQUIREMENTS = ["airbyte-cdk"] -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest~=6.1", - "pytest-mock~=3.6.1", - "responses~=0.21.0", -] +TEST_REQUIREMENTS = ["pytest~=6.2", "pytest-mock~=3.6.1"] setup( name="source_copper", diff --git a/airbyte-integrations/connectors/source-copper/source_copper/manifest.yaml b/airbyte-integrations/connectors/source-copper/source_copper/manifest.yaml new file mode 100644 index 000000000000..82235198f3a7 --- /dev/null +++ b/airbyte-integrations/connectors/source-copper/source_copper/manifest.yaml @@ -0,0 +1,86 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + requester: + type: HttpRequester + url_base: "https://api.copper.com/developer_api/v1/" + http_method: "POST" + authenticator: + type: "ApiKeyAuthenticator" + header: "X-PW-AccessToken" + api_token: "{{ config['api_key'] }}" + request_headers: + X-PW-UserEmail: "{{ config['user_email'] }}" + X-PW-Application: "developer_api" + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: "DefaultPaginator" + page_size_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "page_size" + pagination_strategy: + type: "PageIncrement" + page_size: 200 + page_token_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "page_number" + requester: + $ref: "#/definitions/requester" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + people_stream: + $ref: "#/definitions/base_stream" + name: "people" + primary_key: "id" + $parameters: + path: "people/search" + + projects_stream: + $ref: "#/definitions/base_stream" + name: "projects" + primary_key: "id" + $parameters: + path: "projects/search" + + companies_stream: + $ref: "#/definitions/base_stream" + name: "companies" + primary_key: "id" + $parameters: + path: "companies/search" + + opportunities_stream: + $ref: "#/definitions/base_stream" + name: "opportunities" + primary_key: "id" + $parameters: + path: "opportunities/search" + +streams: + - "#/definitions/people_stream" + - "#/definitions/projects_stream" + - "#/definitions/companies_stream" + - "#/definitions/opportunities_stream" + +check: + type: CheckStream + stream_names: + - "people" + - "projects" + - "companies" + - "opportunities" diff --git a/airbyte-integrations/connectors/source-copper/source_copper/schemas/companies.json b/airbyte-integrations/connectors/source-copper/source_copper/schemas/companies.json index e2f1947e515c..b5f5d1266207 100644 --- a/airbyte-integrations/connectors/source-copper/source_copper/schemas/companies.json +++ b/airbyte-integrations/connectors/source-copper/source_copper/schemas/companies.json @@ -6,12 +6,30 @@ "id": { "type": ["null", "integer"] }, + "phone_numbers": { + "type": ["null", "array"], + "properties": { + "category": { + "type": ["null", "string"] + }, + "number": { + "type": ["null", "string"] + } + } + }, + "custom_fields": { + "type": ["null", "array"], + "properties": { + "mpc": { + "type": ["null", "string"] + } + } + }, "name": { "type": ["null", "string"] }, "address": { - "type": ["null", "object"], - "additionalProperties": true, + "type": "object", "properties": { "street": { "type": ["null", "string"] @@ -45,8 +63,7 @@ "socials": { "type": ["null", "array"], "items": { - "type": ["null", "object"], - "additionalProperties": true, + "type": "object", "properties": { "url": { "type": ["null", "string"] @@ -59,15 +76,12 @@ }, "tags": { "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } + "items": {} }, "websites": { "type": ["null", "array"], "items": { "type": ["object", "null"], - "additionalProperties": true, "properties": { "url": { "type": ["null", "string"] @@ -86,20 +100,6 @@ }, "date_modified": { "type": ["null", "integer"] - }, - "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "additionalProperties": true - } - }, - "phone_numbers": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "additionalProperties": true - } } } } diff --git a/airbyte-integrations/connectors/source-copper/source_copper/schemas/people.json b/airbyte-integrations/connectors/source-copper/source_copper/schemas/people.json index f7f25b051543..3cceeebfeebe 100644 --- a/airbyte-integrations/connectors/source-copper/source_copper/schemas/people.json +++ b/airbyte-integrations/connectors/source-copper/source_copper/schemas/people.json @@ -9,6 +9,38 @@ "name": { "type": ["null", "string"] }, + "socials": { + "type": ["null", "array"], + "properties": { + "note": { + "type": ["null", "string"] + } + } + }, + "leads_converted_from": { + "type": ["null", "array"], + "properties": { + "leads": { + "type": ["null", "string"] + } + } + }, + "tags": { + "type": ["null", "array"], + "properties": { + "sticker": { + "type": ["null", "string"] + } + } + }, + "custom_fields": { + "type": ["null", "array"], + "properties": { + "mprc": { + "type": ["null", "string"] + } + } + }, "prefix": { "type": ["null", "string"] }, @@ -26,10 +58,9 @@ }, "address": { "type": ["null", "object"], - "additionalProperties": true, "properties": { "street": { - "type": ["string", "null"] + "type": ["null", "string"] }, "city": { "type": ["string", "null"] @@ -54,44 +85,16 @@ "company_name": { "type": ["null", "string"] }, - "tags": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "socials": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "additionalProperties": true - } - }, - "leads_converted_from": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "additionalProperties": true - } - }, - "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "additionalProperties": true - } - }, "contact_type_id": { "type": ["null", "integer"] }, "details": { - "type": ["null", "string"] + "type": "null" }, "emails": { "type": ["null", "array"], "items": { "type": ["null", "object"], - "additionalProperties": true, "properties": { "category": { "type": ["null", "string"] @@ -106,7 +109,6 @@ "type": ["null", "array"], "items": { "type": ["null", "object"], - "additionalProperties": true, "properties": { "number": { "type": ["null", "string"] @@ -124,13 +126,12 @@ "type": ["null", "array"], "items": { "type": ["null", "object"], - "additionalProperties": true, "properties": { "url": { - "type": "string" + "type": ["null", "string"] }, "category": { - "type": "string" + "type": ["null", "string"] } } } @@ -139,16 +140,17 @@ "type": ["null", "integer"] }, "date_modified": { - "type": ["integer", "null"] + "type": ["null", "integer"] }, "date_last_contacted": { - "type": ["integer", "null"] + "type": ["null", "integer"] }, "interaction_count": { "type": ["null", "integer"] }, "date_lead_created": { - "type": ["integer", "null"] + "type": ["null", "integer"] } - } + }, + "required": ["id"] } diff --git a/airbyte-integrations/connectors/source-copper/source_copper/schemas/projects.json b/airbyte-integrations/connectors/source-copper/source_copper/schemas/projects.json index 611f3091f803..cbb925ff712a 100644 --- a/airbyte-integrations/connectors/source-copper/source_copper/schemas/projects.json +++ b/airbyte-integrations/connectors/source-copper/source_copper/schemas/projects.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", + "type": ["null", "object"], "additionalProperties": true, "properties": { "id": { @@ -9,8 +9,24 @@ "name": { "type": ["null", "string"] }, + "tags": { + "type": ["null", "array"], + "properties": { + "sticker": { + "type": ["null", "string"] + } + } + }, + "custom_fields": { + "type": ["null", "array"], + "properties": { + "mprc": { + "type": ["null", "string"] + } + } + }, "related_resource": { - "type": ["string", "null"] + "type": ["null", "string"] }, "assignee_id": { "type": ["null", "integer"] @@ -26,19 +42,6 @@ }, "date_modified": { "type": ["null", "integer"] - }, - "custom_fields": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "additionalProperties": true - } - }, - "tags": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } } } } diff --git a/airbyte-integrations/connectors/source-copper/source_copper/source.py b/airbyte-integrations/connectors/source-copper/source_copper/source.py index a7cd15a28739..6225c6754410 100644 --- a/airbyte-integrations/connectors/source-copper/source_copper/source.py +++ b/airbyte-integrations/connectors/source-copper/source_copper/source.py @@ -2,113 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import json -from abc import ABC -from typing import Any, Iterable, List, Mapping, Optional, Tuple +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. +WARNING: Do not modify this file. +""" -# Basic full refresh stream -class CopperStream(HttpStream, ABC): - def __init__(self, *args, api_key: str = None, user_email: str = None, **kwargs): - super().__init__(*args, **kwargs) - self._user_email = user_email - self._api_key = api_key - url_base = "https://api.copper.com/developer_api/v1/" - - @property - def http_method(self) -> str: - return "POST" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - body = json.loads(response.request.body) - result = response.json() - if body and result: - page_number = body.get("page_number") - return {"page_number": page_number + 1, "page_size": 200} - else: - return None - - def request_body_json( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Optional[Mapping]: - - if next_page_token: - return next_page_token - - return {"page_number": 1, "page_size": 200} - - def request_headers( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> Mapping[str, Any]: - return { - "X-PW-AccessToken": self._api_key, - "X-PW-UserEmail": self._user_email, - "X-PW-Application": "developer_api", - "Content-type": "application/json", - } - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - response_result = response.json() - if response_result: - yield from response_result - return - - -class People(CopperStream): - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "people/search" - - -class Projects(CopperStream): - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "projects/search" - - -class Companies(CopperStream): - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "companies/search" - - -class Opportunities(CopperStream): - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "opportunities/search" - - -# Source -class SourceCopper(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - try: - records = People(**config).read_records(sync_mode=SyncMode.full_refresh) - next(records, None) - return True, None - except Exception as error: - return False, f"Unable to connect to Copper API with the provided credentials - {repr(error)}" - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - return [People(**config), Companies(**config), Projects(**config), Opportunities(**config)] +# Declarative Source +class SourceCopper(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-copper/unit_tests/test_source.py b/airbyte-integrations/connectors/source-copper/unit_tests/test_source.py deleted file mode 100644 index 178f99562f88..000000000000 --- a/airbyte-integrations/connectors/source-copper/unit_tests/test_source.py +++ /dev/null @@ -1,26 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -import responses -from source_copper.source import SourceCopper - - -@responses.activate -def test_check_connection(mocker): - source = SourceCopper() - logger_mock, config_mock = MagicMock(), MagicMock() - url = "https://api.copper.com/developer_api/v1/people/search" - responses.add(responses.POST, url, json={}) - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -def test_streams(mocker): - source = SourceCopper() - config_mock = MagicMock() - streams = source.streams(config_mock) - # TODO: replace this with your streams number - expected_streams_number = 4 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-copper/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-copper/unit_tests/test_streams.py deleted file mode 100644 index 7e08f1765485..000000000000 --- a/airbyte-integrations/connectors/source-copper/unit_tests/test_streams.py +++ /dev/null @@ -1,70 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_copper.source import CopperStream - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(CopperStream, "path", "v0/example_endpoint") - mocker.patch.object(CopperStream, "primary_key", "test_primary_key") - mocker.patch.object(CopperStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = CopperStream() - # TODO: replace this with your input parameters - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request parameters - expected_params = {} - assert stream.request_params(**inputs) == expected_params - - -def test_request_headers(patch_base_class): - stream = CopperStream() - # TODO: replace this with your input parameters - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request headers - expected_headers = { - "Content-type": "application/json", - "X-PW-AccessToken": None, - "X-PW-Application": "developer_api", - "X-PW-UserEmail": None, - } - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = CopperStream() - # TODO: replace this with your expected http request method - expected_method = "POST" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = CopperStream() - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = CopperStream() - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-courier/README.md b/airbyte-integrations/connectors/source-courier/README.md index 74294b4c464f..a6678343f478 100644 --- a/airbyte-integrations/connectors/source-courier/README.md +++ b/airbyte-integrations/connectors/source-courier/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-courier:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/courier) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_courier/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-courier:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-courier build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-courier:airbyteDocker +An image will be built with the tag `airbyte/source-courier:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-courier:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-courier:dev check --co docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-courier:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-courier:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-courier test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-courier:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-courier:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-courier test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/courier.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-courier/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-courier/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-courier/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-courier/build.gradle b/airbyte-integrations/connectors/source-courier/build.gradle deleted file mode 100644 index f02f8618cbc0..000000000000 --- a/airbyte-integrations/connectors/source-courier/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_courier' -} diff --git a/airbyte-integrations/connectors/source-customer-io/.dockerignore b/airbyte-integrations/connectors/source-customer-io/.dockerignore new file mode 100644 index 000000000000..70db04ede412 --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_customer_io +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-customer-io/Dockerfile b/airbyte-integrations/connectors/source-customer-io/Dockerfile new file mode 100644 index 000000000000..7a5aa6a0ee5e --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_customer_io ./source_customer_io + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.name=airbyte/source-customer-io diff --git a/airbyte-integrations/connectors/source-customer-io/README.md b/airbyte-integrations/connectors/source-customer-io/README.md new file mode 100644 index 000000000000..0f0790855f06 --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/README.md @@ -0,0 +1,67 @@ +# Customer Io Source + +This is the repository for the Customer Io configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/customer-io). + +## Local development + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/customer-io) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_customer_io/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source customer-io test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + + +#### Build +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-customer-io build +``` + +An image will be built with the tag `airbyte/source-customer-io:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-customer-io:dev . +``` + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-customer-io:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-customer-io:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-customer-io:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-customer-io:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-customer-io test +``` + +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-customer-io test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/customer-io.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-customer-io/__init__.py b/airbyte-integrations/connectors/source-customer-io/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-customer-io/acceptance-test-config.yml b/airbyte-integrations/connectors/source-customer-io/acceptance-test-config.yml new file mode 100644 index 000000000000..6d56394853a4 --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/acceptance-test-config.yml @@ -0,0 +1,35 @@ +connector_image: airbyte/source-customer-io:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_customer_io/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + expect_records: + path: "integration_tests/expected_records.jsonl" + extra_fields: no + exact_order: no + extra_records: yes + # incremental: + # # bypass_reason: "This connector does not implement incremental sync" + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-customer-io/icon.svg b/airbyte-integrations/connectors/source-customer-io/icon.svg new file mode 100644 index 000000000000..d26ace2490f3 --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/icon.svg @@ -0,0 +1,27 @@ + + + + + diff --git a/airbyte-integrations/connectors/source-customer-io/integration_tests/__init__.py b/airbyte-integrations/connectors/source-customer-io/integration_tests/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-customer-io/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-customer-io/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..cb7c9a0d288c --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/integration_tests/abnormal_state.json @@ -0,0 +1,23 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "updated": 2545332027 }, + "stream_descriptor": { "name": "campaigns" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated": 2545332027 }, + "stream_descriptor": { "name": "campaigns_action" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated": 2545332027 }, + "stream_descriptor": { "name": "newsletters" } + } + } +] diff --git a/airbyte-integrations/connectors/source-file-secure/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-customer-io/integration_tests/acceptance.py similarity index 100% rename from airbyte-integrations/connectors/source-file-secure/integration_tests/acceptance.py rename to airbyte-integrations/connectors/source-customer-io/integration_tests/acceptance.py diff --git a/airbyte-integrations/connectors/source-customer-io/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-customer-io/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..4610d52ec4c0 --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/integration_tests/configured_catalog.json @@ -0,0 +1,31 @@ +{ + "streams": [ + { + "stream": { + "name": "campaigns", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "campaigns_actions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "newsletters", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-customer-io/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-customer-io/integration_tests/expected_records.jsonl new file mode 100644 index 000000000000..e4ad156186d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/integration_tests/expected_records.jsonl @@ -0,0 +1,3 @@ +{"stream": "campaigns", "data": {"id":4,"deduplicate_id":"4:1692284670","name":"Order Confirmation","type":"event","created":1692284573,"updated":1692284670,"active":false,"state":"draft","actions":[{"id":25,"type":"attribute_update"},{"id":26,"type":"email"},{"id":29,"type":"email"}],"first_started":0,"tags":["Sample"],"event_name":"purchase"}, "emitted_at": 1691072031178} +{"stream": "campaigns_actions", "data": {"id":"25","campaign_id":4,"deduplicate_id":"11:1692284573","name":"Update order count","layout":"","body":"","created":1692284573,"updated":1692284573,"type":"attribute_update","language":"","editor":"","recipient":"","recipient_environment_id":144601}, "emitted_at": 1691072031198} +{"stream": "newsletters", "data": {"id":1,"deduplicate_id":"1:1692285687","content_ids":[18],"name":"weekly","sent_at":null,"created":1692285669,"updated":1692285687,"type":"email","tags":[]}, "emitted_at": 1691072031198} diff --git a/airbyte-integrations/connectors/source-customer-io/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-customer-io/integration_tests/invalid_config.json new file mode 100644 index 000000000000..3fc3880ca1e0 --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/integration_tests/invalid_config.json @@ -0,0 +1,4 @@ +{ + "app_api_key": "Invalid", + "start_date": "2099-08-01T00:00:00Z" +} diff --git a/airbyte-integrations/connectors/source-customer-io/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-customer-io/integration_tests/sample_config.json new file mode 100644 index 000000000000..2638c95de566 --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "app_api_key": "Sample" +} diff --git a/airbyte-integrations/connectors/source-customer-io/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-customer-io/integration_tests/sample_state.json new file mode 100644 index 000000000000..29b1a39ad612 --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/integration_tests/sample_state.json @@ -0,0 +1,23 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "updated": 1692304734 }, + "stream_descriptor": { "name": "campaigns" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated": 1692304734 }, + "stream_descriptor": { "name": "campaigns_action" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated": 1692304734 }, + "stream_descriptor": { "name": "newsletters" } + } + } +] diff --git a/airbyte-integrations/connectors/source-customer-io/main.py b/airbyte-integrations/connectors/source-customer-io/main.py new file mode 100644 index 000000000000..835bc1df92ba --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_customer_io import SourceCustomerIo + +if __name__ == "__main__": + source = SourceCustomerIo() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-customer-io/metadata.yaml b/airbyte-integrations/connectors/source-customer-io/metadata.yaml new file mode 100644 index 000000000000..3cf0252a2512 --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/metadata.yaml @@ -0,0 +1,25 @@ +data: + allowedHosts: + hosts: + - https://api.customer.io/v1/ + registries: + oss: + enabled: false + cloud: + enabled: false + connectorSubtype: api + connectorType: source + definitionId: 34f697bc-b989-4eda-b06f-d0f39b88825b + dockerImageTag: 0.2.0 + dockerRepository: airbyte/source-customer-io + githubIssueLabel: source-customer-io + icon: customer-io.svg + license: MIT + name: Customer Io + releaseDate: 2023-08-20 + releaseStage: alpha + supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/sources/customer-io + tags: + - language:low-code +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-customer-io/requirements.txt b/airbyte-integrations/connectors/source-customer-io/requirements.txt new file mode 100644 index 000000000000..cc57334ef619 --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/connector-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-customer-io/setup.py b/airbyte-integrations/connectors/source-customer-io/setup.py new file mode 100644 index 000000000000..04cd8664c8a1 --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.2", + "pytest-mock~=3.6.1", + "connector-acceptance-test", +] + +setup( + name="source_customer_io", + description="Source implementation for Customer Io.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-customer-io/source_customer_io/__init__.py b/airbyte-integrations/connectors/source-customer-io/source_customer_io/__init__.py new file mode 100644 index 000000000000..529f0341c4f2 --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/source_customer_io/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceCustomerIo + +__all__ = ["SourceCustomerIo"] diff --git a/airbyte-integrations/connectors/source-customer-io/source_customer_io/manifest.yaml b/airbyte-integrations/connectors/source-customer-io/source_customer_io/manifest.yaml new file mode 100644 index 000000000000..9403b6878b0f --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/source_customer_io/manifest.yaml @@ -0,0 +1,77 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["{{ parameters.extractor_path }}"] + + requester: + type: HttpRequester + url_base: "https://api.customer.io/v1" + http_method: "GET" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['app_api_key'] }}" + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + campaigns_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "campaigns" + primary_key: "id" + path: "/campaigns" + extractor_path: "campaigns" + + campaigns_actions_partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/campaigns_stream" + parent_key: "id" + partition_field: "parent_id" + + campaigns_actions_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "campaigns_actions" + primary_key: "id" + path: "/campaigns/{{ stream_partition.parent_id }}/actions" + extractor_path: "actions" + retriever: + $ref: "#/definitions/retriever" + partition_router: + $ref: "#/definitions/campaigns_actions_partition_router" + + newsletters_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "newsletters" + primary_key: "id" + path: "/newsletters" + extractor_path: "newsletters" + +streams: + - "#/definitions/campaigns_stream" + - "#/definitions/campaigns_actions_stream" + - "#/definitions/newsletters_stream" + +check: + type: CheckStream + stream_names: + - "campaigns" + - "campaigns_actions" + - "newsletters" diff --git a/airbyte-integrations/connectors/source-customer-io/source_customer_io/schemas/campaigns.json b/airbyte-integrations/connectors/source-customer-io/source_customer_io/schemas/campaigns.json new file mode 100644 index 000000000000..9c6d93b6a2cc --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/source_customer_io/schemas/campaigns.json @@ -0,0 +1,104 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "start_hour": { + "type": ["null", "integer"] + }, + "timezone": { + "type": ["null", "string"] + }, + "date_attribute": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "array"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + } + } + }, + "event_name": { + "type": ["null", "string"] + }, + "trigger_segment_ids": { + "type": ["null", "array"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + } + } + }, + "first_started": { + "type": ["null", "integer"] + }, + "frequency": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "start_minutes": { + "type": ["null", "integer"] + }, + "use_customer_timezone": { + "type": ["null", "boolean"] + }, + "updated": { + "type": ["null", "integer"] + }, + "deduplicate_id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "msg_templates": { + "type": "array", + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + } + } + } + }, + "actions": { + "type": "array", + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + } + } + } + }, + "created_by": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-customer-io/source_customer_io/schemas/campaigns_actions.json b/airbyte-integrations/connectors/source-customer-io/source_customer_io/schemas/campaigns_actions.json new file mode 100644 index 000000000000..8d817fbfceae --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/source_customer_io/schemas/campaigns_actions.json @@ -0,0 +1,88 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["object", "null"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "string"] + }, + "recipient_environment_id": { + "type": ["null", "integer"] + }, + "language": { + "type": ["null", "string"] + }, + "layout": { + "type": ["null", "string"] + }, + "recipient": { + "type": ["null", "string"] + }, + "campaign_id": { + "type": ["null", "integer"] + }, + "editor": { + "type": ["null", "string"] + }, + "preheader_text": { + "type": ["null", "string"] + }, + "from_id": { + "type": ["null", "string"] + }, + "bcc": { + "type": ["null", "string"] + }, + "subject": { + "type": ["null", "string"] + }, + "reply_to": { + "type": ["null", "string"] + }, + "reply_to_id": { + "type": ["null", "string"] + }, + "sending_state": { + "type": ["null", "string"] + }, + "fake_bcc": { + "type": ["null", "boolean"] + }, + "request_method": { + "type": ["null", "string"] + }, + "headers": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "from": { + "type": ["null", "string"] + }, + "preprocessor": { + "type": ["null", "string"] + }, + "parent_action_id": { + "type": ["null", "integer"] + }, + "deduplicate_id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "integer"] + }, + "updated": { + "type": ["null", "integer"] + }, + "body": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-customer-io/source_customer_io/schemas/newsletters.json b/airbyte-integrations/connectors/source-customer-io/source_customer_io/schemas/newsletters.json new file mode 100644 index 000000000000..2390c9b0971d --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/source_customer_io/schemas/newsletters.json @@ -0,0 +1,52 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "content_ids": { + "type": ["null", "array"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + } + } + }, + "sent_at": { + "type": ["null", "array"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "string"] + } + } + }, + "tags": { + "type": ["null", "array"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + } + } + }, + "deduplicate_id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "updated": { + "type": ["null", "integer"] + }, + "created": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-customer-io/source_customer_io/source.py b/airbyte-integrations/connectors/source-customer-io/source_customer_io/source.py new file mode 100644 index 000000000000..b524ee906a84 --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/source_customer_io/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceCustomerIo(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-customer-io/source_customer_io/spec.yaml b/airbyte-integrations/connectors/source-customer-io/source_customer_io/spec.yaml new file mode 100644 index 000000000000..fcc6b9f81994 --- /dev/null +++ b/airbyte-integrations/connectors/source-customer-io/source_customer_io/spec.yaml @@ -0,0 +1,13 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/customer-io/ +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Customer.io Spec + type: object + required: + - app_api_key + additionalProperties: true + properties: + app_api_key: + type: string + title: Customer.io App API Key + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-datadog/Dockerfile b/airbyte-integrations/connectors/source-datadog/Dockerfile index 37ad70df367c..cd4d7a7f1b01 100644 --- a/airbyte-integrations/connectors/source-datadog/Dockerfile +++ b/airbyte-integrations/connectors/source-datadog/Dockerfile @@ -1,16 +1,38 @@ -FROM python:3.9-slim +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base WORKDIR /airbyte/integration_code -COPY source_datadog ./source_datadog + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only COPY main.py ./ -COPY setup.py ./ -RUN pip install . +COPY source_datadog ./source_datadog ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.2 +LABEL io.airbyte.version=0.4.0 LABEL io.airbyte.name=airbyte/source-datadog diff --git a/airbyte-integrations/connectors/source-datadog/README.md b/airbyte-integrations/connectors/source-datadog/README.md index f2c4e7889bb1..1cad0882c4f3 100644 --- a/airbyte-integrations/connectors/source-datadog/README.md +++ b/airbyte-integrations/connectors/source-datadog/README.md @@ -1,72 +1,34 @@ -# Datadog Source +# Datadog Source -This is the repository for the Datadog source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/datadog). +This is the repository for the Datadog configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/datadog). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-datadog:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/datadog) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_datadog/spec.json` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/datadog) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_datadog/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source datadog test creds` and place them into `secrets/config.json`. - -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-datadog:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-datadog build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-datadog:airbyteDocker +An image will be built with the tag `airbyte/source-datadog:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-datadog:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -77,33 +39,29 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-datadog:dev discover - docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-datadog:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. -If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-datadog:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` - -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-datadog test ``` -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-datadog:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-datadog test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/datadog.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-datadog/__init__.py b/airbyte-integrations/connectors/source-datadog/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-datadog/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-datadog/acceptance-test-config.yml b/airbyte-integrations/connectors/source-datadog/acceptance-test-config.yml index b9b95f0cb687..191fa7e68daa 100644 --- a/airbyte-integrations/connectors/source-datadog/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-datadog/acceptance-test-config.yml @@ -19,18 +19,33 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: - - name: "incident_teams" - bypass_reason: "Test account does not have data for this stream" - - name: "logs" - bypass_reason: "Test account does not have data for this stream" - # TODO: logs stream currently has no data. This will need to be added to test incremental. - # incremental: - # tests: - # - config_path: "secrets/config.json" - # configured_catalog_path: "integration_tests/inc_configured_catalog.json" - # future_state: - # future_state_path: "integration_tests/abnormal_state.json" + - name: audit_logs + bypass_reason: Sandbox account can't seed this stream + - name: incident_teams + bypass_reason: Sandbox account can't seed this stream + - name: service_level_objectives + bypass_reason: Sandbox account can't seed this stream + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.jsonl" + # extra_fields: no + # exact_order: no + # extra_records: yes + incremental: + bypass_reason: "This connector does not implement incremental sync" + # TODO uncomment this block this block if your connector implements incremental sync: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + ignored_fields: + logs: + - name: "attributes.timestamp" + bypass_reason: "Change everytime" + - name: "id" + bypass_reason: "Change everytime" diff --git a/airbyte-integrations/connectors/source-datadog/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-datadog/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-datadog/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-datadog/build.gradle b/airbyte-integrations/connectors/source-datadog/build.gradle deleted file mode 100644 index e776cf72d784..000000000000 --- a/airbyte-integrations/connectors/source-datadog/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_datadog' -} diff --git a/airbyte-integrations/connectors/source-datadog/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-datadog/integration_tests/abnormal_state.json index b93902698218..d5628188894c 100644 --- a/airbyte-integrations/connectors/source-datadog/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-datadog/integration_tests/abnormal_state.json @@ -1,5 +1,9 @@ -{ - "logs": { - "sync_date": "2024-10-10T00:10:00Z" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "sync_date": "9999-04-12T18:13:36Z" }, + "stream_descriptor": { "name": "logs" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-datadog/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-datadog/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-datadog/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-datadog/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-datadog/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-datadog/integration_tests/configured_catalog.json index b8a33996b8d4..c555994aa46b 100644 --- a/airbyte-integrations/connectors/source-datadog/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-datadog/integration_tests/configured_catalog.json @@ -50,6 +50,16 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "monitors", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "synthetic_tests", diff --git a/airbyte-integrations/connectors/source-datadog/integration_tests/inc_configured_catalog.json b/airbyte-integrations/connectors/source-datadog/integration_tests/inc_configured_catalog.json deleted file mode 100644 index ee26bd107c2e..000000000000 --- a/airbyte-integrations/connectors/source-datadog/integration_tests/inc_configured_catalog.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "logs", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["sync_date"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "cursor_field": ["sync_date"], - "destination_sync_mode": "append" - } - ] -} diff --git a/airbyte-integrations/connectors/source-datadog/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-datadog/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-datadog/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-datadog/metadata.yaml b/airbyte-integrations/connectors/source-datadog/metadata.yaml index 734c40ef9f90..88bef85da8ba 100644 --- a/airbyte-integrations/connectors/source-datadog/metadata.yaml +++ b/airbyte-integrations/connectors/source-datadog/metadata.yaml @@ -1,24 +1,32 @@ data: + allowedHosts: + hosts: + - datadoghq.com + - us3.datadoghq.com + - us5.datadoghq.com + - datadoghq.eu + - ddog-gov.com + registries: + oss: + enabled: true + cloud: + enabled: false connectorSubtype: api connectorType: source definitionId: 1cfc30c7-82db-43f4-9fd7-ac1b42312cda - dockerImageTag: 0.2.2 + dockerImageTag: 0.4.0 dockerRepository: airbyte/source-datadog githubIssueLabel: source-datadog icon: datadog.svg license: MIT name: Datadog - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: 2023-08-27 releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/datadog tags: - - language:python + - language:low-code ab_internal: sl: 100 ql: 100 - supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-datadog/requirements.txt b/airbyte-integrations/connectors/source-datadog/requirements.txt index 7b9114ed5867..cc57334ef619 100644 --- a/airbyte-integrations/connectors/source-datadog/requirements.txt +++ b/airbyte-integrations/connectors/source-datadog/requirements.txt @@ -1,2 +1,2 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. +-e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-datadog/setup.py b/airbyte-integrations/connectors/source-datadog/setup.py index 8d2a7f0da611..432515ebd8f0 100644 --- a/airbyte-integrations/connectors/source-datadog/setup.py +++ b/airbyte-integrations/connectors/source-datadog/setup.py @@ -5,12 +5,14 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "requests==2.25.1"] +MAIN_REQUIREMENTS = [ + "airbyte-cdk", +] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", + "pytest~=6.2", "pytest-mock~=3.6.1", - "pytest", ] setup( @@ -20,7 +22,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-datadog/source_datadog/__init__.py b/airbyte-integrations/connectors/source-datadog/source_datadog/__init__.py index e7b9d6ba7582..283ad17afffd 100644 --- a/airbyte-integrations/connectors/source-datadog/source_datadog/__init__.py +++ b/airbyte-integrations/connectors/source-datadog/source_datadog/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-datadog/source_datadog/manifest.yaml b/airbyte-integrations/connectors/source-datadog/source_datadog/manifest.yaml new file mode 100644 index 000000000000..f4d0c397be67 --- /dev/null +++ b/airbyte-integrations/connectors/source-datadog/source_datadog/manifest.yaml @@ -0,0 +1,262 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["{{ parameters.name }}"] + + data_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["data"] + + requester_v1: + type: HttpRequester + url_base: "https://api.{{ config['site'] }}/api/v1/" + http_method: "GET" + authenticator: + type: "ApiKeyAuthenticator" + header: "DD-API-KEY" + api_token: "{{ config['api_key'] }}" + request_headers: + DD-APPLICATION-KEY: "{{ config['application_key'] }}" + + requester_v2: + $ref: "#/definitions/requester_v1" + url_base: "https://api.{{ config['site'] }}/api/v2/" + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester_v1" + + retriever_v2: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/data_selector" + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester_v2" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + base_paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ last_records['meta', 'pagination', 'next_offset'] }}" + page_token_option: + type: "RequestPath" + field_name: "page[offset]" + inject_into: "request_parameter" + + base_after_paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ last_records['meta', 'page', 'after'] }}" + page_token_option: + type: "RequestPath" + field_name: "page[cursor]" + inject_into: "request_parameter" + + incremental_sync_base: + type: DatetimeBasedCursor + cursor_field: "{{ parameters.incremental_cursor }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + cursor_granularity: "PT0.000001S" + lookback_window: "P31D" + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + datetime: "{{ config['end_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: filter[from] + inject_into: request_parameter + end_time_option: + type: RequestOption + field_name: filter[to] + inject_into: request_parameter + step: "P1M" + + dashboards_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "dashboards" + primary_key: "id" + path: "dashboard" + + downtimes_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "downtimes" + primary_key: "id" + path: "downtime" + retriever: + $ref: "#/definitions/retriever" + record_selector: + $ref: "#/definitions/selector" + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + + synthetic_tests_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "synthetic_tests" + path: "synthetics/tests" + retriever: + $ref: "#/definitions/retriever" + record_selector: + $ref: "#/definitions/selector" + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["tests"] + + monitors_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "monitors" + primary_key: "id" + path: "monitor" + retriever: + $ref: "#/definitions/retriever" + record_selector: + $ref: "#/definitions/selector" + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + + service_level_objectives_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "service_level_objectives" + primary_key: "id" + path: "slo" + retriever: + $ref: "#/definitions/retriever" + record_selector: + $ref: "#/definitions/selector" + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["data"] + + audit_logs_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "audit_logs" + primary_key: "id" + path: "audit/events" + incremental_cursor: "sync_date" + retriever: + $ref: "#/definitions/retriever_v2" + paginator: + $ref: "#/definitions/base_after_paginator" + + logs_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "logs" + primary_key: "id" + path: "logs/events" + incremental_cursor: "sync_date" + retriever: + $ref: "#/definitions/retriever_v2" + requester: + $ref: "#/definitions/requester_v2" + request_parameters: + filter[query]: "{{ config['query'] }}" + paginator: + $ref: "#/definitions/base_after_paginator" + incremental_sync: + $ref: "#/definitions/incremental_sync_base" + + metrics_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "metrics" + primary_key: "id" + path: "metrics" + retriever: + $ref: "#/definitions/retriever_v2" + paginator: + type: NoPagination + + incidents_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "incidents" + primary_key: "id" + path: "incidents" + retriever: + $ref: "#/definitions/retriever_v2" + paginator: + $ref: "#/definitions/base_paginator" + + incident_teams_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "incident_teams" + primary_key: "id" + path: "teams" + retriever: + $ref: "#/definitions/retriever_v2" + paginator: + $ref: "#/definitions/base_paginator" + + users_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "users" + primary_key: "id" + path: "users" + retriever: + $ref: "#/definitions/retriever_v2" + paginator: + $ref: "#/definitions/base_paginator" + +streams: + - "#/definitions/audit_logs_stream" + - "#/definitions/dashboards_stream" + - "#/definitions/downtimes_stream" + - "#/definitions/incident_teams_stream" + - "#/definitions/incidents_stream" + - "#/definitions/logs_stream" + - "#/definitions/metrics_stream" + - "#/definitions/monitors_stream" + - "#/definitions/service_level_objectives_stream" + - "#/definitions/synthetic_tests_stream" + - "#/definitions/users_stream" + +check: + type: CheckStream + stream_names: + - "audit_logs" + - "dashboards" + - "downtimes" + - "incident_teams" + - "incidents" + - "logs" + - "metrics" + - "monitors" + - "service_level_objectives" + - "synthetic_tests" + - "users" diff --git a/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/monitors.json b/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/monitors.json new file mode 100644 index 000000000000..7905aa2bc7fc --- /dev/null +++ b/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/monitors.json @@ -0,0 +1,374 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "created": { + "type": ["string", "null"] + }, + "created_at": { + "type": ["integer", "null"] + }, + "org_id": { + "type": ["integer", "null"] + }, + "overall_state_modified": { + "type": ["string", "null"] + }, + "creator": { + "additionalProperties": true, + "properties": { + "email": { + "type": ["string", "null"] + }, + "handle": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "deleted": { + "type": ["string", "null"] + }, + "id": { + "type": ["integer", "null"] + }, + "matching_downtimes": { + "items": { + "additionalProperties": true, + "properties": { + "end": { + "type": ["integer", "null"] + }, + "id": { + "type": ["integer", "null"] + }, + "scope": { + "items": { + "type": ["string", "null"] + }, + "type": ["array", "null"] + }, + "start": { + "type": ["integer", "null"] + } + }, + "type": ["object", "null"] + }, + "type": ["array", "null"] + }, + "message": { + "type": ["string", "null"] + }, + "modified": { + "type": ["string", "null"] + }, + "multi": { + "type": ["boolean", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "options": { + "additionalProperties": true, + "properties": { + "aggregation": { + "additionalProperties": true, + "properties": { + "groupBy": { + "items": { + "type": ["string", "null"] + }, + "type": ["array", "null"] + }, + "metric": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "device_ids": { + "items": { + "type": ["string", "null"] + }, + "type": ["array", "null"] + }, + "enable_logs_sample": { + "type": ["boolean", "null"] + }, + "enable_samples": { + "type": ["boolean", "null"] + }, + "escalation_message": { + "type": ["string", "null"] + }, + "evaluation_delay": { + "type": ["integer", "null"] + }, + "group_retention_duration": { + "type": ["string", "null"] + }, + "groupby_simple_monitor": { + "type": ["boolean", "null"] + }, + "include_tags": { + "type": ["boolean", "null"] + }, + "locked": { + "type": ["boolean", "null"] + }, + "new_group_delay": { + "type": ["integer", "null"] + }, + "new_host_delay": { + "type": ["integer", "null"] + }, + "no_data_timeframe": { + "type": ["integer", "null"] + }, + "notification_preset_name": { + "type": ["string", "null"] + }, + "notify_audit": { + "type": ["boolean", "null"] + }, + "notify_by": { + "items": { + "type": ["string", "null"] + }, + "type": ["array", "null"] + }, + "notify_no_data": { + "type": ["boolean", "null"] + }, + "on_missing_data": { + "type": ["string", "null"] + }, + "renotify_interval": { + "type": ["integer", "null"] + }, + "renotify_occurrences": { + "type": ["integer", "null"] + }, + "renotify_statuses": { + "items": { + "type": ["string", "null"] + }, + "type": ["array", "null"] + }, + "require_full_window": { + "type": ["boolean", "null"] + }, + "scheduling_options": { + "additionalProperties": true, + "properties": { + "evaluation_window": { + "additionalProperties": true, + "properties": { + "day_starts": { + "type": ["string", "null"] + }, + "hour_starts": { + "type": ["string", "null"] + }, + "month_starts": { + "type": ["integer", "null"] + } + }, + "type": ["object", "null"] + } + }, + "type": ["object", "null"] + }, + "silenced": { + "patternProperties": { + ".+": { + "type": ["integer", "null"] + } + } + }, + "synthetics_check_id": { + "type": ["string", "null"] + }, + "threshold_windows": { + "additionalProperties": true, + "properties": { + "recovery_window": { + "type": ["string", "null"] + }, + "trigger_window": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "thresholds": { + "additionalProperties": true, + "properties": { + "critical": { + "type": ["number", "null"] + }, + "critical_recovery": { + "type": ["number", "null"] + }, + "ok": { + "type": ["number", "null"] + }, + "unknown": { + "type": ["number", "null"] + }, + "warning": { + "type": ["number", "null"] + }, + "warning_recovery": { + "type": ["number", "null"] + } + }, + "type": ["object", "null"] + }, + "timeout_h": { + "type": ["integer", "null"] + }, + "variables": { + "items": { + "additionalProperties": true, + "properties": { + "compute": { + "additionalProperties": true, + "properties": { + "aggregation": { + "type": ["string", "null"] + }, + "interval": { + "type": ["integer", "null"] + }, + "metric": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "data_source": { + "type": ["string", "null"] + }, + "group_by": { + "items": { + "additionalProperties": true, + "properties": { + "facet": { + "type": ["string", "null"] + }, + "limit": { + "type": ["integer", "null"] + }, + "sort": { + "additionalProperties": true, + "properties": { + "aggregation": { + "type": ["string", "null"] + }, + "metric": { + "type": ["string", "null"] + }, + "order": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + } + }, + "type": ["object", "null"] + }, + "type": ["array", "null"] + }, + "indexes": { + "items": { + "type": ["string", "null"] + }, + "type": ["array", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "search": { + "additionalProperties": true, + "properties": { + "query": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + } + }, + "type": ["object", "null"] + }, + "type": ["array", "null"] + } + }, + "type": ["object", "null"] + }, + "overall_state": { + "type": ["string", "null"] + }, + "priority": { + "type": ["integer", "null"] + }, + "query": { + "type": ["string", "null"] + }, + "restricted_roles": { + "items": { + "type": ["string", "null"] + }, + "type": ["array", "null"] + }, + "state": { + "properties": { + "groups": { + "patternProperties": { + ".+": { + "properties": { + "last_nodata_ts": { + "type": ["integer", "null"] + }, + "last_notified_ts": { + "type": ["integer", "null"] + }, + "last_resolved_ts": { + "type": ["integer", "null"] + }, + "last_triggered_ts": { + "type": ["integer", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "status": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + } + }, + "type": ["object", "null"] + } + }, + "type": ["object", "null"] + }, + "tags": { + "items": { + "type": ["string", "null"] + }, + "type": ["array", "null"] + }, + "type": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] +} diff --git a/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/service_level_objectives.json b/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/service_level_objectives.json new file mode 100644 index 000000000000..f387885cf940 --- /dev/null +++ b/airbyte-integrations/connectors/source-datadog/source_datadog/schemas/service_level_objectives.json @@ -0,0 +1,112 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "created_at": { + "type": ["integer", "null"] + }, + "creator": { + "additionalProperties": true, + "properties": { + "email": { + "type": ["string", "null"] + }, + "handle": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "description": { + "type": ["string", "null"] + }, + "groups": { + "items": { + "type": ["string", "null"] + }, + "type": ["array", "null"] + }, + "id": { + "type": ["string", "null"] + }, + "modified_at": { + "type": ["integer", "null"] + }, + "monitor_ids": { + "items": { + "type": ["integer", "null"] + }, + "type": ["array", "null"] + }, + "monitor_tags": { + "items": { + "type": ["string", "null"] + }, + "type": ["array", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "query": { + "additionalProperties": true, + "properties": { + "denominator": { + "type": ["string", "null"] + }, + "numerator": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "tags": { + "items": { + "type": ["string", "null"] + }, + "type": ["array", "null"] + }, + "target_threshold": { + "type": ["number", "null"] + }, + "thresholds": { + "items": { + "additionalProperties": true, + "properties": { + "target": { + "type": ["number", "null"] + }, + "target_display": { + "type": ["string", "null"] + }, + "timeframe": { + "type": ["string", "null"] + }, + "warning": { + "type": ["number", "null"] + }, + "warning_display": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "type": ["array", "null"] + }, + "timeframe": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "type_id": { + "type": ["integer", "null"] + }, + "warning_threshold": { + "type": ["number", "null"] + } + }, + "type": ["object", "null"] +} diff --git a/airbyte-integrations/connectors/source-datadog/source_datadog/source.py b/airbyte-integrations/connectors/source-datadog/source_datadog/source.py index 90dcba7fb305..36418d6a9669 100644 --- a/airbyte-integrations/connectors/source-datadog/source_datadog/source.py +++ b/airbyte-integrations/connectors/source-datadog/source_datadog/source.py @@ -2,102 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import logging -from datetime import datetime -from typing import Any, List, Mapping, Optional, Tuple +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import requests -from airbyte_cdk import AirbyteLogger -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from pydantic.datetime_parse import timedelta -from source_datadog.streams import ( - AuditLogs, - Dashboards, - Downtimes, - Incidents, - IncidentTeams, - Logs, - Metrics, - SeriesStream, - SyntheticTests, - Users, -) +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. +WARNING: Do not modify this file. +""" -class SourceDatadog(AbstractSource): - @staticmethod - def _get_authenticator(config: Mapping[str, Any]): - return DatadogAuthenticator(api_key=config["api_key"], application_key=config["application_key"]) - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: - try: - args = self.connector_config(config) - dashboards_stream = Dashboards(**args) - records = dashboards_stream.read_records(sync_mode=SyncMode.full_refresh) - next(records, None) - return True, None - except Exception as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - args = self.connector_config(config) - base_streams = [ - AuditLogs(**args), - Dashboards(**args), - Downtimes(**args), - Incidents(**args), - IncidentTeams(**args), - Logs(**args), - Metrics(**args), - SyntheticTests(**args), - Users(**args), - ] - queries = config.get("queries", []) - - # Create a stream for each query in the list - query_streams = [] - for query in queries: - if all(field in query and query[field] for field in ["name", "data_source", "query"]): - name = query["name"] - data_source = query["data_source"] - query_string = query["query"] - - # Create a new stream using the query name, data source, and query string - new_stream = SeriesStream( - name=name, - data_source=data_source, - query_string=query_string, - **args, - ) - query_streams.append(new_stream) - else: - logging.info("Query fields are missing, Streams not created") - - # Combine the base streams and query streams - return base_streams + query_streams - - def connector_config(self, config: Mapping[str, Any]) -> Mapping[str, Any]: - return { - "site": config.get("site", "datadoghq.com"), - "authenticator": self._get_authenticator(config), - "query": config.get("query", ""), - "max_records_per_request": config.get("max_records_per_request", 5000), - "start_date": config.get("start_date", datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")), - "end_date": config.get("end_date", (datetime.now() + timedelta(seconds=1)).strftime("%Y-%m-%dT%H:%M:%SZ")), - "query_start_date": config.get("start_date", ""), - "query_end_date": config.get("end_date", ""), - "queries": config.get("queries", []), - } - - -class DatadogAuthenticator(requests.auth.AuthBase): - def __init__(self, api_key: str, application_key: str): - self.api_key = api_key - self.application_key = application_key - - def __call__(self, r): - r.headers["DD-API-KEY"] = self.api_key - r.headers["DD-APPLICATION-KEY"] = self.application_key - return r +# Declarative Source +class SourceDatadog(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-datadog/source_datadog/streams.py b/airbyte-integrations/connectors/source-datadog/source_datadog/streams.py deleted file mode 100644 index 44e17453678b..000000000000 --- a/airbyte-integrations/connectors/source-datadog/source_datadog/streams.py +++ /dev/null @@ -1,403 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from abc import ABC, abstractmethod -from datetime import datetime -from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Union - -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams import IncrementalMixin -from airbyte_cdk.sources.streams.http import HttpStream -from dateutil.parser import parse -from pydantic.datetime_parse import timedelta - - -class DatadogStream(HttpStream, ABC): - """ - Datadog API Reference: https://docs.datadoghq.com/api/latest/ - """ - - primary_key: Optional[str] = None - parse_response_root: Optional[str] = None - - def __init__( - self, - site: str, - query: str, - max_records_per_request: int, - start_date: str, - end_date: str, - query_start_date: str, - query_end_date: str, - queries: List[Dict[str, str]] = None, - **kwargs, - ): - super().__init__(**kwargs) - self.site = site - self.query = query - self.max_records_per_request = max_records_per_request - self.start_date = start_date - self.end_date = end_date - self.query_start_date = query_start_date - self.query_end_date = query_end_date - self.queries = queries or [] - self._cursor_value = None - - @property - def url_base(self) -> str: - return f"https://api.{self.site}/api" - - def request_headers(self, **kwargs) -> Mapping[str, Any]: - return { - "Accept": "application/json", - "Content-Type": "application/json", - } - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params: Dict[str, str] = {} - - if next_page_token: - params.update(next_page_token) - - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - response_json = response.json() - records = response_json if not self.parse_response_root else response_json.get(self.parse_response_root, []) - for record in records: - yield self.transform(record=record, **kwargs) - - def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - return record - - -class V1ApiDatadogStream(DatadogStream, ABC): - @property - def url_base(self) -> str: - return f"{super().url_base}/v1/" - - @property - def http_method(self) -> str: - return "GET" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - -class Dashboards(V1ApiDatadogStream): - """ - https://docs.datadoghq.com/api/latest/dashboards/#get-all-dashboards - """ - - parse_response_root: Optional[str] = "dashboards" - - def path(self, **kwargs) -> str: - return "dashboard" - - -class Downtimes(V1ApiDatadogStream): - """ - https://docs.datadoghq.com/api/latest/downtimes/#get-all-downtimes - """ - - def path(self, **kwargs) -> str: - return "downtime" - - -class SyntheticTests(V1ApiDatadogStream): - """ - https://docs.datadoghq.com/api/latest/synthetics/#get-the-list-of-all-tests - """ - - parse_response_root: Optional[str] = "tests" - - def path(self, **kwargs) -> str: - return "synthetics/tests" - - -class V2ApiDatadogStream(DatadogStream, ABC): - @property - def url_base(self) -> str: - return f"{super().url_base}/v2/" - - -class IncrementalSearchableStream(V2ApiDatadogStream, IncrementalMixin, ABC): - primary_key: Optional[str] = "id" - parse_response_root: Optional[str] = "data" - - def __init__(self, site: str, query: str, max_records_per_request: int, start_date: str, end_date: str, **kwargs): - super().__init__(site, query, max_records_per_request, start_date, end_date, **kwargs) - self._cursor_value = "" - - @property - def http_method(self) -> str: - return "POST" - - @property - def state(self) -> Mapping[str, Any]: - if self._cursor_value: - return {self.cursor_field: self._cursor_value} - else: - return {self.cursor_field: self.start_date} - - @state.setter - def state(self, value: Mapping[str, Any]): - self._cursor_value = value[self.cursor_field] - - @property - def cursor_field(self) -> Union[str, List[str]]: - return "sync_date" - - def request_body_json( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Optional[Mapping]: - cursor = None - if next_page_token: - cursor = next_page_token.get("page", {}).get("cursor", {}) - return self.get_payload(cursor) - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - response_json = response.json() - cursor = response_json.get("meta", {}).get("page", {}).get("after", {}) - if not cursor: - self._cursor_value = self.end_date - else: - return self.get_payload(cursor) - - def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: - record[self.cursor_field] = self._cursor_value if self._cursor_value else self.end_date - return record - - def read_records( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Mapping[str, Any]]: - if self.start_date >= self.end_date or self.end_date <= self._cursor_value: - return [] - return super().read_records(sync_mode, cursor_field, stream_slice, stream_state) - - def get_payload(self, cursor: Optional[str]) -> Mapping[str, Any]: - payload = { - "filter": {"query": self.query, "from": self._cursor_value if self._cursor_value else self.start_date, "to": self.end_date}, - "page": {"limit": self.max_records_per_request}, - } - if cursor: - payload["page"]["cursor"] = cursor - - return payload - - -class AuditLogs(IncrementalSearchableStream): - """ - https://docs.datadoghq.com/api/latest/audit/#search-audit-logs-events - """ - - def path(self, **kwargs) -> str: - return "audit/events/search" - - -class Logs(IncrementalSearchableStream): - """ - https://docs.datadoghq.com/api/latest/logs/#search-logs - """ - - def path(self, **kwargs) -> str: - return "logs/events/search" - - -class BasedListStream(V2ApiDatadogStream, ABC): - parse_response_root: Optional[str] = "data" - - @property - def http_method(self) -> str: - return "GET" - - -class Metrics(BasedListStream): - """ - https://docs.datadoghq.com/api/latest/metrics/#get-a-list-of-metrics - """ - - def path(self, **kwargs) -> str: - return "metrics?window[seconds]=1209600" # max value allowed (2 weeks) - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - -class PaginatedBasedListStream(BasedListStream, ABC): - primary_key: Optional[str] = "id" - - def path( - self, - *, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> str: - offset = None - if next_page_token: - offset = next_page_token.get("offset") - return self.get_url_path(offset) - - @abstractmethod - def get_url_path(self, offset: Optional[str]) -> str: - """ - Returns the relative URL with the corresponding offset - """ - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - response_json = response.json() - next_offset = response_json.get("meta", {}).get("pagination", {}).get("next_offset", -1) - current_offset = response_json.get("meta", {}).get("pagination", {}).get("offset", -1) - next_page_token = None - if next_offset != current_offset: - next_page_token = {"offset": next_offset} - return next_page_token - - -class Incidents(PaginatedBasedListStream): - """ - https://docs.datadoghq.com/api/latest/incidents/#get-a-list-of-incidents - """ - - def get_url_path(self, offset: Optional[str]) -> str: - params = f"&page[offset]={offset}" if offset else "" - return f"incidents?page[size]={self.max_records_per_request}{params}" - - -class IncidentTeams(PaginatedBasedListStream): - """ - https://docs.datadoghq.com/api/latest/incident-teams/#get-a-list-of-all-incident-teams - """ - - def get_url_path(self, offset: Optional[str]) -> str: - params = f"&page[offset]={offset}" if offset else "" - return f"teams?page[size]={self.max_records_per_request}{params}" - - -class Users(PaginatedBasedListStream): - """ - https://docs.datadoghq.com/api/latest/users/#list-all-users - """ - - current_page = 0 - - def get_url_path(self, offset: Optional[int]) -> str: - params = f"&page[number]={offset}" if offset else "" - return f"users?page[size]={self.max_records_per_request}{params}" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - response_json = response.json() - next_page_token = None - if len(response_json.get("data", [])) > 0: - self.current_page += 1 - next_page_token = {"offset": self.current_page} - return next_page_token - - -class SeriesStream(IncrementalSearchableStream, ABC): - """ - https://docs.datadoghq.com/api/latest/metrics/?code-lang=curl#query-timeseries-data-across-multiple-products - """ - - primary_key: Optional[str] = None - parse_response_root: Optional[str] = "data" - - def __init__(self, name, data_source, query_string, **kwargs): - super().__init__(**kwargs) - self.name = name - self.data_source = data_source - self.query_string = query_string - - @property - def http_method(self) -> str: - return "POST" - - def path(self, **kwargs) -> str: - return "query/timeseries" - - @property - def name(self) -> str: - return self._name - - @name.setter - def name(self, value): - self._name = value - - @property - def cursor_field(self) -> Union[str, List[str]]: - return "sync_date" - - def request_body_json( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Optional[Mapping]: - - if self.query_end_date: - end_date = int(parse(self.query_end_date).timestamp() * 1000) - else: - end_date = int(datetime.now().timestamp()) * 1000 - - if self.query_start_date: - start_date = int(parse(self.query_start_date).timestamp() * 1000) - elif self._cursor_value: - start_date = int(parse(self._cursor_value).timestamp() * 1000) - else: - start_date = int((datetime.now() - timedelta(hours=24)).timestamp()) * 1000 - - payload = { - "data": { - "type": "timeseries_request", - "attributes": { - "to": end_date, - "from": start_date, - "queries": [ - { - "data_source": self.data_source, - "name": self.name, - } - ], - }, - } - } - - if self.data_source in ["metrics", "cloud_cost"]: - payload["data"]["attributes"]["queries"][0]["query"] = self.query_string - elif self.data_source in ["logs", "rum"]: - payload["data"]["attributes"]["queries"][0]["search"] = {"query": self.query_string} - payload["data"]["attributes"]["queries"][0]["compute"] = {"aggregation": "count"} - print(payload) - return payload - - def get_json_schema(self) -> Mapping[str, Any]: - local_json_schema = { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": {}, - "additionalProperties": True, - } - return local_json_schema - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - data = response.json() - data["stream"] = self.name - data["query"] = self.query_string - data["data_source"] = self.data_source - return [data] - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - self._cursor_value = self.end_date - return None diff --git a/airbyte-integrations/connectors/source-datadog/unit_tests/conftest.py b/airbyte-integrations/connectors/source-datadog/unit_tests/conftest.py deleted file mode 100644 index 6526981ddf26..000000000000 --- a/airbyte-integrations/connectors/source-datadog/unit_tests/conftest.py +++ /dev/null @@ -1,457 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from pytest import fixture - - -@fixture(name="config") -def config_fixture(): - return { - "site": "datadoghq.com", - "api_key": "test_api_key", - "application_key": "test_application_key", - "query": "", - "max_records_per_request": 5000, - "start_date": "2022-10-10T00:00:00Z", - "end_date": "2022-10-10T00:10:00Z", - } - - -@fixture(name="config_eu") -def config_fixture_eu(): - return { - "site": "datadoghq.eu", - "api_key": "test_api_key", - "application_key": "test_application_key", - "query": "", - "max_records_per_request": 5000, - "start_date": "2022-10-10T00:00:00Z", - "end_date": "2022-10-10T00:10:00Z", - } - - -@fixture(name="mock_responses") -def mock_responses(): - return { - "Dashboards": { - "dashboards": [ - { - "author_handle": "string", - "created_at": "2019-09-19T10:00:00.000Z", - "description": "string", - "id": "string", - "is_read_only": False, - "layout_type": "ordered", - "modified_at": "2019-09-19T10:00:00.000Z", - "title": "string", - "url": "string", - } - ], - }, - "Downtimes": { - "active": True, - "active_child": { - "active": True, - "canceled": 1412799983, - "creator_id": 123456, - "disabled": False, - "downtime_type": 2, - "end": 1412793983, - "id": 1626, - "message": "Message on the downtime", - "monitor_id": 123456, - "monitor_tags": ["*"], - "mute_first_recovery_notification": False, - "parent_id": 123, - "recurrence": { - "period": 1, - "rrule": "FREQ=MONTHLY;BYSETPOS=3;BYDAY=WE;INTERVAL=1", - "type": "weeks", - "until_date": 1447786293, - "until_occurrences": 2, - "week_days": ["Mon", "Tue"], - }, - "scope": ["env:staging"], - "start": 1412792983, - "timezone": "America/New_York", - "updater_id": 123456, - }, - "canceled": 1412799983, - "creator_id": 123456, - "disabled": False, - "downtime_type": 2, - "end": 1412793983, - "id": 1625, - "message": "Message on the downtime", - "monitor_id": 123456, - "monitor_tags": ["*"], - "mute_first_recovery_notification": False, - "parent_id": 123, - "recurrence": { - "period": 1, - "rrule": "FREQ=MONTHLY;BYSETPOS=3;BYDAY=WE;INTERVAL=1", - "type": "weeks", - "until_date": 1447786293, - "until_occurrences": 2, - "week_days": ["Mon", "Tue"], - }, - "scope": ["env:staging"], - "start": 1412792983, - "timezone": "America/New_York", - "updater_id": 123456, - }, - "SyntheticTests": { - "tests": [ - { - "config": { - "assertions": [{"operator": "contains", "property": "string", "target": 123456, "type": "statusCode"}], - "configVariables": [ - {"example": "string", "id": "string", "name": "VARIABLE_NAME", "pattern": "string", "type": "text"} - ], - "request": { - "allow_insecure": False, - "basicAuth": {"password": "PaSSw0RD!", "type": "web", "username": "my_username"}, - "body": "string", - "certificate": { - "cert": {"content": "string", "filename": "string", "updatedAt": "string"}, - "key": {"content": "string", "filename": "string", "updatedAt": "string"}, - }, - "certificateDomains": [], - "dnsServer": "string", - "dnsServerPort": "integer", - "follow_redirects": False, - "headers": {"": "string"}, - "host": "string", - "message": "string", - "metadata": {"": "string"}, - "method": "GET", - "noSavingResponseBody": False, - "numberOfPackets": "integer", - "port": "integer", - "proxy": {"headers": {"": "string"}, "url": "https://example.com"}, - "query": {}, - "servername": "string", - "service": "string", - "shouldTrackHops": False, - "timeout": "number", - "url": "https://example.com", - }, - "variables": [{"example": "string", "id": "string", "name": "VARIABLE_NAME", "pattern": "string", "type": "text"}], - }, - "creator": {"email": "string", "handle": "string", "name": "string"}, - "locations": ["aws:eu-west-3"], - "message": "string", - "monitor_id": "integer", - "name": "string", - "options": { - "accept_self_signed": False, - "allow_insecure": False, - "checkCertificateRevocation": False, - "ci": {"executionRule": "string"}, - "device_ids": ["laptop_large"], - "disableCors": False, - "disableCsp": False, - "follow_redirects": False, - "ignoreServerCertificateError": False, - "initialNavigationTimeout": "integer", - "min_failure_duration": "integer", - "min_location_failed": "integer", - "monitor_name": "string", - "monitor_options": {"renotify_interval": "integer"}, - "monitor_priority": "integer", - "noScreenshot": False, - "restricted_roles": ["xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"], - "retry": {"count": "integer", "interval": "number"}, - "rumSettings": {"applicationId": "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx", "clientTokenId": 12345, "isEnabled": True}, - "tick_every": "integer", - }, - "public_id": "string", - "status": "live", - "steps": [ - { - "allowFailure": False, - "isCritical": False, - "name": "string", - "params": {}, - "timeout": "integer", - "type": "assertElementContent", - } - ], - "subtype": "http", - "tags": [], - "type": "string", - } - ] - }, - "Metrics": {"data": [{"id": "test.metric.latency", "type": "metrics"}]}, - "Incidents": { - "data": [ - { - "attributes": { - "created": "2019-09-19T10:00:00.000Z", - "customer_impact_duration": "integer", - "customer_impact_end": "2019-09-19T10:00:00.000Z", - "customer_impact_scope": "An example customer impact scope", - "customer_impact_start": "2019-09-19T10:00:00.000Z", - "customer_impacted": False, - "detected": "2019-09-19T10:00:00.000Z", - "fields": {"": {}}, - "modified": "2019-09-19T10:00:00.000Z", - "notification_handles": [{"display_name": "Jane Doe", "handle": "@test.user@test.com"}], - "public_id": 1, - "resolved": "2019-09-19T10:00:00.000Z", - "time_to_detect": "integer", - "time_to_internal_response": "integer", - "time_to_repair": "integer", - "time_to_resolve": "integer", - "title": "A test incident title", - }, - "id": "00000000-0000-0000-1234-000000000000", - "relationships": { - "attachments": {"data": [{"id": "00000000-0000-abcd-1000-000000000000", "type": "incident_attachments"}]}, - "commander_user": {"data": {"id": "00000000-0000-0000-0000-000000000000", "type": "users"}}, - "created_by_user": {"data": {"id": "00000000-0000-0000-2345-000000000000", "type": "users"}}, - "integrations": {"data": [{"id": "00000000-abcd-0001-0000-000000000000", "type": "incident_integrations"}]}, - "last_modified_by_user": {"data": {"id": "00000000-0000-0000-2345-000000000000", "type": "users"}}, - }, - "type": "incidents", - } - ], - "included": [ - { - "attributes": { - "created_at": "2019-09-19T10:00:00.000Z", - "disabled": False, - "email": "string", - "handle": "string", - "icon": "string", - "modified_at": "2019-09-19T10:00:00.000Z", - "name": "string", - "service_account": False, - "status": "string", - "title": "string", - "verified": False, - }, - "id": "string", - "relationships": { - "org": {"data": {"id": "00000000-0000-beef-0000-000000000000", "type": "orgs"}}, - "other_orgs": {"data": [{"id": "00000000-0000-beef-0000-000000000000", "type": "orgs"}]}, - "other_users": {"data": [{"id": "00000000-0000-0000-2345-000000000000", "type": "users"}]}, - "roles": {"data": [{"id": "3653d3c6-0c75-11ea-ad28-fb5701eabc7d", "type": "roles"}]}, - }, - "type": "users", - } - ], - "meta": {"pagination": {"next_offset": 1000, "offset": 10, "size": 1000}}, - }, - "IncidentTeams": { - "data": [ - { - "attributes": {"created": "2019-09-19T10:00:00.000Z", "modified": "2019-09-19T10:00:00.000Z", "name": "team name"}, - "id": "00000000-7ea3-0000-000a-000000000000", - "relationships": { - "created_by": {"data": {"id": "00000000-0000-0000-2345-000000000000", "type": "users"}}, - "last_modified_by": {"data": {"id": "00000000-0000-0000-2345-000000000000", "type": "users"}}, - }, - "type": "teams", - } - ], - "included": [ - { - "attributes": { - "created_at": "2019-09-19T10:00:00.000Z", - "disabled": False, - "email": "string", - "handle": "string", - "icon": "string", - "modified_at": "2019-09-19T10:00:00.000Z", - "name": "string", - "service_account": False, - "status": "string", - "title": "string", - "verified": False, - }, - "id": "string", - "relationships": { - "org": {"data": {"id": "00000000-0000-beef-0000-000000000000", "type": "orgs"}}, - "other_orgs": {"data": [{"id": "00000000-0000-beef-0000-000000000000", "type": "orgs"}]}, - "other_users": {"data": [{"id": "00000000-0000-0000-2345-000000000000", "type": "users"}]}, - "roles": {"data": [{"id": "3653d3c6-0c75-11ea-ad28-fb5701eabc7d", "type": "roles"}]}, - }, - "type": "users", - } - ], - "meta": {"pagination": {"next_offset": 1000, "offset": 10, "size": 1000}}, - }, - "Users": { - "data": [ - { - "attributes": { - "created_at": "2019-09-19T10:00:00.000Z", - "disabled": False, - "email": "string", - "handle": "string", - "icon": "string", - "modified_at": "2019-09-19T10:00:00.000Z", - "name": "string", - "service_account": False, - "status": "string", - "title": "string", - "verified": False, - }, - "id": "string", - "relationships": { - "org": {"data": {"id": "00000000-0000-beef-0000-000000000000", "type": "orgs"}}, - "other_orgs": {"data": [{"id": "00000000-0000-beef-0000-000000000000", "type": "orgs"}]}, - "other_users": {"data": [{"id": "00000000-0000-0000-2345-000000000000", "type": "users"}]}, - "roles": {"data": [{"id": "3653d3c6-0c75-11ea-ad28-fb5701eabc7d", "type": "roles"}]}, - }, - "type": "users", - } - ], - "included": [ - { - "attributes": { - "created_at": "2019-09-19T10:00:00.000Z", - "description": "string", - "disabled": False, - "modified_at": "2019-09-19T10:00:00.000Z", - "name": "string", - "public_id": "string", - "sharing": "string", - "url": "string", - }, - "id": "string", - "type": "orgs", - } - ], - "meta": {"page": {"total_count": "integer", "total_filtered_count": "integer"}}, - }, - "Logs": { - "data": [ - { - "attributes": { - "attributes": {"customAttribute": 123, "duration": 2345}, - "host": "i-0123", - "message": "Host connected to remote", - "service": "agent", - "status": "INFO", - "tags": ["team:A"], - "timestamp": "2019-01-02T09:42:36.320Z", - }, - "id": "AAAAAWgN8Xwgr1vKDQAAAABBV2dOOFh3ZzZobm1mWXJFYTR0OA", - "type": "log", - } - ], - "links": { - "next": "https://app.datadoghq.com/api/v2/logs/event?filter[query]=foo\u0026page[cursor]=eyJzdGFydEF0IjoiQVFBQUFYS2tMS3pPbm40NGV3QUFBQUJCV0V0clRFdDZVbG8zY3pCRmNsbHJiVmxDWlEifQ==" - }, - "meta": { - "elapsed": 132, - "page": {"after": "eyJzdGFydEF0IjoiQVFBQUFYS2tMS3pPbm40NGV3QUFBQUJCV0V0clRFdDZVbG8zY3pCRmNsbHJiVmxDWlEifQ=="}, - "request_id": "MWlFUjVaWGZTTTZPYzM0VXp1OXU2d3xLSVpEMjZKQ0VKUTI0dEYtM3RSOFVR", - "status": "done", - "warnings": [ - { - "code": "unknown_index", - "detail": "indexes: foo, bar", - "title": "One or several indexes are missing or invalid, results hold data from the other indexes", - } - ], - }, - }, - "AuditLogs": { - "data": [ - { - "attributes": { - "attributes": {"customAttribute": 123, "duration": 2345}, - "service": "web-app", - "tags": ["team:A"], - "timestamp": "2019-01-02T09:42:36.320Z", - }, - "id": "AAAAAWgN8Xwgr1vKDQAAAABBV2dOOFh3ZzZobm1mWXJFYTR0OA", - "type": "audit", - } - ], - "links": { - "next": "https://app.datadoghq.com/api/v2/audit/event?filter[query]=foo\u0026page[cursor]=eyJzdGFydEF0IjoiQVFBQUFYS2tMS3pPbm40NGV3QUFBQUJCV0V0clRFdDZVbG8zY3pCRmNsbHJiVmxDWlEifQ==" - }, - "meta": { - "elapsed": 132, - "page": {"after": "eyJzdGFydEF0IjoiQVFBQUFYS2tMS3pPbm40NGV3QUFBQUJCV0V0clRFdDZVbG8zY3pCRmNsbHJiVmxDWlEifQ=="}, - "request_id": "MWlFUjVaWGZTTTZPYzM0VXp1OXU2d3xLSVpEMjZKQ0VKUTI0dEYtM3RSOFVR", - "status": "done", - "warnings": [ - { - "code": "unknown_index", - "detail": "indexes: foo, bar", - "title": "One or several indexes are missing or invalid, results hold data from the other indexes", - } - ], - }, - }, - } - - -@fixture(name="mock_stream") -def mock_stream_fixture(requests_mock): - def _mock_stream(path, response=None): - if response is None: - response = {} - - url = f"https://api.datadoghq.com/api/v1/{path}" - requests_mock.get(url, json=response) - - return _mock_stream - - -@fixture(name="config_timeseries") -def config_timeseries_fixture(): - return { - "site": "datadoghq.eu", - "api_key": "test_api_key", - "application_key": "test_application_key", - "query": "", - "max_records_per_request": 5000, - "start_date": "2022-10-10T00:00:00Z", - "end_date": "2022-10-10T00:10:00Z", - "queries": [ - { - "name": "NodeCount", - "data_source": "metrics", - "query": "kubernetes_state.node.count{*}" - }, - { - "name": "Resource", - "data_source": "rum", - "query": "@type:resource @resource.status_code:>=400 @resource.type:(xhr OR fetch)" - } - ] - } - - -@fixture(name="config_timeseries_invalid") -def config_timeseries_invalid_fixture(): - return { - "site": "datadoghq.eu", - "api_key": "test_api_key", - "application_key": "test_application_key", - "query": "", - "max_records_per_request": 5000, - "start_date": "2022-10-10T00:00:00Z", - "end_date": "2022-10-10T00:10:00Z", - "queries": [ - { - "data_source": "metrics", - "query": "missing_name_query_string", - }, - { - "query": "missing_name_and_data_source_query_string", - }, - { - "name": "MissingQuery", - "data_source": "metrics", - } - ] - } diff --git a/airbyte-integrations/connectors/source-datadog/unit_tests/test_source.py b/airbyte-integrations/connectors/source-datadog/unit_tests/test_source.py deleted file mode 100644 index b8c36d194397..000000000000 --- a/airbyte-integrations/connectors/source-datadog/unit_tests/test_source.py +++ /dev/null @@ -1,86 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import logging -from unittest.mock import MagicMock - -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import ConnectorSpecification -from source_datadog.source import SourceDatadog -from source_datadog.streams import AuditLogs, SeriesStream - -logger = AirbyteLogger() - - -def test_check_connection_ok(config, mock_stream, mock_responses): - mock_stream("dashboard", response=mock_responses.get("Dashboards")) - ok, error_msg = SourceDatadog().check_connection(logger, config=config) - - assert ok - assert not error_msg - - -def test_check_connection_exception(config, mock_stream, mock_responses): - mock_stream("invalid_path", response=mock_responses.get("Dashboards")) - ok, error_msg = SourceDatadog().check_connection(logger, config=config) - - assert not ok - assert error_msg - - -def test_check_connection_empty_config(config): - config = {} - - ok, error_msg = SourceDatadog().check_connection(logger, config=config) - - assert not ok - assert error_msg - - -def test_check_connection_invalid_config(config): - config.pop("api_key") - - ok, error_msg = SourceDatadog().check_connection(logger, config=config) - - assert not ok - assert error_msg - - -def test_streams(config): - streams = SourceDatadog().streams(config) - - assert len(streams) == 9 - - -def test_spec(): - logger_mock = MagicMock() - spec = SourceDatadog().spec(logger_mock) - - assert isinstance(spec, ConnectorSpecification) - - -def test_streams_with_valid_queries(config_timeseries): - streams = SourceDatadog().streams(config_timeseries) - - assert len(streams) == 11 - assert isinstance(streams[0], AuditLogs) - assert isinstance(streams[-1], SeriesStream) - assert streams[-1].name == "Resource" - assert streams[-2].name == "NodeCount" - - -def test_streams_with_invalid_queries(config_timeseries_invalid, caplog): - with caplog.at_level(logging.INFO): - streams = SourceDatadog().streams(config_timeseries_invalid) - - assert len(streams) == 9 - assert isinstance(streams[0], AuditLogs) - - invalid_query_names = ["", "MissingQuery"] - invalid_queries_exist = any(isinstance(stream, SeriesStream) and stream.name in invalid_query_names for stream in streams) - assert not invalid_queries_exist - - missing_query_logs = "Query fields are missing, Streams not created" - assert missing_query_logs in caplog.messages - assert len(caplog.records) == 3 diff --git a/airbyte-integrations/connectors/source-datadog/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-datadog/unit_tests/test_streams.py deleted file mode 100644 index 45d65c35abc2..000000000000 --- a/airbyte-integrations/connectors/source-datadog/unit_tests/test_streams.py +++ /dev/null @@ -1,205 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import json -from unittest.mock import MagicMock, patch - -import pytest -import requests -import requests_mock as req_mock -from airbyte_cdk.models import SyncMode -from source_datadog.source import SourceDatadog -from source_datadog.streams import ( - AuditLogs, - Dashboards, - DatadogStream, - Downtimes, - Incidents, - IncidentTeams, - Logs, - Metrics, - SeriesStream, - SyntheticTests, - Users, -) - - -@pytest.mark.parametrize( - "stream", - [AuditLogs, Dashboards, Downtimes, Incidents, IncidentTeams, Logs, Metrics, SyntheticTests, Users], -) -def test_task_stream(requests_mock, stream, config, mock_responses): - requests_mock.get(req_mock.ANY, json=mock_responses.get(stream.__name__)) - requests_mock.post(req_mock.ANY, json=mock_responses.get(stream.__name__)) - args = SourceDatadog().connector_config(config) - instance = stream(**args) - - stream_slice = instance.stream_slices(sync_mode=SyncMode.full_refresh) - record = next(instance.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice)) - - assert record - - -@patch.multiple(DatadogStream, __abstractmethods__=set()) -def test_next_page_token(config): - stream = DatadogStream( - site=config["site"], - query=config["query"], - max_records_per_request=config["max_records_per_request"], - start_date=config["start_date"], - end_date=config["end_date"], - query_start_date=config["start_date"], - query_end_date=config["end_date"], - ) - inputs = {"response": MagicMock()} - assert stream.next_page_token(**inputs) is None - - -def test_site_config(config): - assert config['site'] == 'datadoghq.com' - - -def test_site_config_eu(config_eu): - assert config_eu['site'] == 'datadoghq.eu' - - -@pytest.mark.parametrize( - "stream", - [AuditLogs, Dashboards, Downtimes, Incidents, IncidentTeams, Logs, Metrics, SyntheticTests, Users], -) -def test_next_page_token_empty_response(stream, config): - expected_token = None - args = SourceDatadog().connector_config(config) - instance = stream(**args) - response = requests.Response() - response._content = json.dumps({}).encode("utf-8") - assert instance.next_page_token(response=response) == expected_token - - -@pytest.mark.parametrize( - "stream", - [AuditLogs, Logs], -) -def test_next_page_token_inc(stream, config): - args = SourceDatadog().connector_config(config) - instance = stream(**args) - response = requests.Response() - body_content = {"meta": {"page": {"after": "test_cursor"}}} - response._content = json.dumps(body_content).encode("utf-8") - result = instance.next_page_token(response=response) - assert result.get("page").get("cursor") == "test_cursor" - - -@pytest.mark.parametrize( - "stream", - [Incidents, IncidentTeams], -) -def test_next_page_token_paginated(stream, config): - args = SourceDatadog().connector_config(config) - instance = stream(**args) - response = requests.Response() - body_content = { - "meta": { - "pagination": { - "offset": 998, - "next_offset": 999, - } - } - } - response._content = json.dumps(body_content).encode("utf-8") - result = instance.next_page_token(response=response) - assert result.get("offset") == 999 - - -@patch.multiple(DatadogStream, __abstractmethods__=set()) -def test_site_parameter_is_set(config): - site = "example.com" - stream = DatadogStream( - site=site, - query=config["query"], - max_records_per_request=config["max_records_per_request"], - start_date=config["start_date"], - end_date=config["end_date"], - query_start_date=config["start_date"], - query_end_date=config["end_date"], - ) - url_base = stream.url_base - expected_url_base = f"https://api.{site}/api" - assert url_base == expected_url_base - - -@patch.multiple(DatadogStream, __abstractmethods__=set()) -def test_site_parameter_is_not_set(config): - stream = DatadogStream( - site=config["site"], - query=config["query"], - max_records_per_request=config["max_records_per_request"], - start_date=config["start_date"], - end_date=config["end_date"], - query_start_date=config["start_date"], - query_end_date=config["end_date"], - ) - url_base = stream.url_base - expected_url_base = "https://api.datadoghq.com/api" - assert url_base == expected_url_base - - -@patch.multiple(SeriesStream, __abstractmethods__=set()) -def test_request_body_json(config): - stream = SeriesStream( - site=config["site"], - query=config["query"], - max_records_per_request=config["max_records_per_request"], - start_date=config["start_date"], - end_date=config["end_date"], - query_start_date="2023-01-01T00:00:00Z", - query_end_date="2023-02-01T00:00:00Z", - name="test_stream", - data_source="metrics", - query_string="test_query" - ) - stream_state = { - "stream_state_key": "value" - } - expected_payload = { - "data": { - "type": "timeseries_request", - "attributes": { - "to": 1675209600000, - "from": 1672531200000, - "queries": [ - { - "data_source": "metrics", - "query": "test_query", - "name": "test_stream" - } - ] - } - } - } - payload = stream.request_body_json(stream_state) - assert payload == expected_payload - - -@patch.multiple(SeriesStream, __abstractmethods__=set()) -def test_get_json_schema(config): - stream = SeriesStream( - site=config["site"], - query=config["query"], - max_records_per_request=config["max_records_per_request"], - start_date=config["start_date"], - end_date=config["end_date"], - query_start_date=config["start_date"], - query_end_date=config["end_date"], - name="test_stream", - data_source="metrics", - query_string="test_query" - ) - expected_schema = { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": {}, - "additionalProperties": True - } - assert stream.get_json_schema() == expected_schema diff --git a/airbyte-integrations/connectors/source-datascope/README.md b/airbyte-integrations/connectors/source-datascope/README.md index 272e0b581afe..226989dc0f6d 100644 --- a/airbyte-integrations/connectors/source-datascope/README.md +++ b/airbyte-integrations/connectors/source-datascope/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-datascope:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/datascope) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_datascope/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-datascope:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-datascope build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-datascope:airbyteDocker +An image will be built with the tag `airbyte/source-datascope:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-datascope:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-datascope:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-datascope:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-datascope:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-datascope test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-datascope:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-datascope:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-datascope test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/datascope.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-datascope/acceptance-test-config.yml b/airbyte-integrations/connectors/source-datascope/acceptance-test-config.yml index 147e648cf984..ae30da05cc76 100644 --- a/airbyte-integrations/connectors/source-datascope/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-datascope/acceptance-test-config.yml @@ -19,7 +19,6 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" - threshold_days: 1 full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-datascope/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-datascope/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-datascope/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-datascope/build.gradle b/airbyte-integrations/connectors/source-datascope/build.gradle deleted file mode 100644 index 034b286839b8..000000000000 --- a/airbyte-integrations/connectors/source-datascope/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_datascope' -} diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/.gitignore b/airbyte-integrations/connectors/source-db2-strict-encrypt/.gitignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile deleted file mode 100644 index 89480b74edbc..000000000000 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-db2-strict-encrypt - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-db2-strict-encrypt - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.19 -LABEL io.airbyte.name=airbyte/source-db2-strict-encrypt diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/acceptance-test-config.yml b/airbyte-integrations/connectors/source-db2-strict-encrypt/acceptance-test-config.yml deleted file mode 100644 index 46b9d792ff44..000000000000 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/acceptance-test-config.yml +++ /dev/null @@ -1,6 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-db2-strict-encrypt:dev -tests: - spec: - - spec_path: "src/test/resources/expected_spec.json" diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-db2-strict-encrypt/build.gradle deleted file mode 100644 index 62cedc738ebe..000000000000 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/build.gradle +++ /dev/null @@ -1,31 +0,0 @@ -plugins { - id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' -} - -application { - mainClass = 'io.airbyte.integrations.source.db2_strict_encrypt.Db2StrictEncryptSource' - applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] -} - -dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:connectors:source-db2') - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation project(':airbyte-integrations:connectors:source-relational-db') - implementation libs.airbyte.protocol - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - - implementation group: 'com.ibm.db2', name: 'jcc', version: '11.5.5.0' - - testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) - testImplementation project(':airbyte-test-utils') - testImplementation libs.connectors.testcontainers.db2 - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-db2') - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation 'org.apache.commons:commons-lang3:3.11' -} diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/main/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2StrictEncryptSource.java b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/main/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2StrictEncryptSource.java deleted file mode 100644 index ac2323f4f401..000000000000 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/main/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2StrictEncryptSource.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.db2_strict_encrypt; - -import com.fasterxml.jackson.databind.node.ArrayNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.base.spec_modification.SpecModifyingSource; -import io.airbyte.integrations.source.db2.Db2Source; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class Db2StrictEncryptSource extends SpecModifyingSource implements Source { - - private static final Logger LOGGER = LoggerFactory.getLogger(Db2StrictEncryptSource.class); - public static final String DRIVER_CLASS = Db2Source.DRIVER_CLASS; - - public Db2StrictEncryptSource() { - super(new Db2Source()); - } - - @Override - public ConnectorSpecification modifySpec(final ConnectorSpecification originalSpec) { - final ConnectorSpecification spec = Jsons.clone(originalSpec); - // We need to remove the first item from one Of, which is responsible for connecting to the source - // without encrypted. - ((ArrayNode) spec.getConnectionSpecification().get("properties").get("encryption").get("oneOf")).remove(0); - return spec; - } - - public static void main(final String[] args) throws Exception { - final Source source = new Db2StrictEncryptSource(); - LOGGER.info("starting source: {}", Db2StrictEncryptSource.class); - new IntegrationRunner(source).run(args); - LOGGER.info("completed source: {}", Db2StrictEncryptSource.class); - } - -} diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java deleted file mode 100644 index f437607e17ce..000000000000 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2StrictEncryptSourceCertificateAcceptanceTest.java +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.io.airbyte.integration_tests.sources; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.source.db2.Db2Source; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.SyncMode; -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.nio.charset.StandardCharsets; -import java.util.HashMap; -import java.util.concurrent.TimeUnit; -import javax.sql.DataSource; -import org.testcontainers.containers.Db2Container; - -public class Db2StrictEncryptSourceCertificateAcceptanceTest extends SourceAcceptanceTest { - - private static final String SCHEMA_NAME = "SOURCE_INTEGRATION_TEST"; - private static final String STREAM_NAME1 = "ID_AND_NAME1"; - private static final String STREAM_NAME2 = "ID_AND_NAME2"; - - private static final String TEST_KEY_STORE_PASS = "Passw0rd"; - private static final String KEY_STORE_FILE_PATH = "clientkeystore.jks"; - private static final String SSL_CONFIG = ":sslConnection=true;sslTrustStoreLocation=" + KEY_STORE_FILE_PATH + - ";sslTrustStorePassword=" + TEST_KEY_STORE_PASS + ";"; - - private Db2Container db; - private JsonNode config; - - @Override - protected String getImageName() { - return "airbyte/source-db2-strict-encrypt:dev"; - } - - @Override - protected ConnectorSpecification getSpec() throws Exception { - return Jsons.deserialize(MoreResources.readResource("expected_spec.json"), ConnectorSpecification.class); - } - - @Override - protected JsonNode getConfig() { - return config; - } - - @Override - protected ConfiguredAirbyteCatalog getConfiguredCatalog() { - return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(Lists.newArrayList("ID")) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withStream(CatalogHelpers.createAirbyteStream( - String.format("%s.%s", SCHEMA_NAME, STREAM_NAME1), - Field.of("ID", JsonSchemaType.NUMBER), - Field.of("NAME", JsonSchemaType.STRING)) - .withSupportedSyncModes( - Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))), - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.FULL_REFRESH) - .withDestinationSyncMode(DestinationSyncMode.OVERWRITE) - .withStream(CatalogHelpers.createAirbyteStream( - String.format("%s.%s", SCHEMA_NAME, STREAM_NAME2), - Field.of("ID", JsonSchemaType.NUMBER), - Field.of("NAME", JsonSchemaType.STRING)) - .withSupportedSyncModes( - Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); - } - - @Override - protected JsonNode getState() { - return Jsons.jsonNode(new HashMap<>()); - } - - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - db = new Db2Container("ibmcom/db2:11.5.5.0").withCommand().acceptLicense() - .withExposedPorts(50000); - db.start(); - - final var certificate = getCertificate(); - try { - convertAndImportCertificate(certificate); - } catch (final IOException | InterruptedException e) { - throw new RuntimeException("Failed to import certificate into Java Keystore"); - } - - config = Jsons.jsonNode(ImmutableMap.builder() - .put("host", db.getHost()) - .put("port", db.getMappedPort(50000)) - .put("db", db.getDatabaseName()) - .put("username", db.getUsername()) - .put("password", db.getPassword()) - .put("encryption", Jsons.jsonNode(ImmutableMap.builder() - .put("encryption_method", "encrypted_verify_certificate") - .put("ssl_certificate", certificate) - .put("key_store_password", TEST_KEY_STORE_PASS) - .build())) - .build()); - - final String jdbcUrl = String.format("jdbc:db2://%s:%s/%s", - config.get("host").asText(), - db.getMappedPort(50000), - config.get("db").asText()) + SSL_CONFIG; - - final DataSource dataSource = DataSourceFactory.create( - config.get("username").asText(), - config.get("password").asText(), - Db2Source.DRIVER_CLASS, - jdbcUrl); - - try { - final JdbcDatabase database = new DefaultJdbcDatabase(dataSource); - - final String createSchemaQuery = String.format("CREATE SCHEMA %s", SCHEMA_NAME); - final String createTableQuery1 = String - .format("CREATE TABLE %s.%s (ID INTEGER, NAME VARCHAR(200))", SCHEMA_NAME, STREAM_NAME1); - final String createTableQuery2 = String - .format("CREATE TABLE %s.%s (ID INTEGER, NAME VARCHAR(200))", SCHEMA_NAME, STREAM_NAME2); - final String insertIntoTableQuery1 = String - .format("INSERT INTO %s.%s (ID, NAME) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash')", - SCHEMA_NAME, STREAM_NAME1); - final String insertIntoTableQuery2 = String - .format("INSERT INTO %s.%s (ID, NAME) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash')", - SCHEMA_NAME, STREAM_NAME2); - - database.execute(createSchemaQuery); - database.execute(createTableQuery1); - database.execute(createTableQuery2); - database.execute(insertIntoTableQuery1); - database.execute(insertIntoTableQuery2); - } finally { - DataSourceFactory.close(dataSource); - } - } - - @Override - protected void tearDown(final TestDestinationEnv testEnv) { - new File("certificate.pem").delete(); - new File("certificate.der").delete(); - new File(KEY_STORE_FILE_PATH).delete(); - db.close(); - } - - /* Helpers */ - - private String getCertificate() throws IOException, InterruptedException { - // To enable SSL connection on the server, we need to generate self-signed certificates for the - // server and add them to the configuration. - // Then you need to enable SSL connection and specify on which port it will work. These changes will - // take effect after restart. - // The certificate for generating a user certificate has the extension *.arm. - db.execInContainer("su", "-", "db2inst1", "-c", "gsk8capicmd_64 -keydb -create -db \"server.kdb\" -pw \"" + TEST_KEY_STORE_PASS + "\" -stash"); - db.execInContainer("su", "-", "db2inst1", "-c", "gsk8capicmd_64 -cert -create -db \"server.kdb\" -pw \"" + TEST_KEY_STORE_PASS - + "\" -label \"mylabel\" -dn \"CN=testcompany\" -size 2048 -sigalg SHA256_WITH_RSA"); - db.execInContainer("su", "-", "db2inst1", "-c", "gsk8capicmd_64 -cert -extract -db \"server.kdb\" -pw \"" + TEST_KEY_STORE_PASS - + "\" -label \"mylabel\" -target \"server.arm\" -format ascii -fips"); - - db.execInContainer("su", "-", "db2inst1", "-c", "db2 update dbm cfg using SSL_SVR_KEYDB /database/config/db2inst1/server.kdb"); - db.execInContainer("su", "-", "db2inst1", "-c", "db2 update dbm cfg using SSL_SVR_STASH /database/config/db2inst1/server.sth"); - db.execInContainer("su", "-", "db2inst1", "-c", "db2 update dbm cfg using SSL_SVR_LABEL mylabel"); - db.execInContainer("su", "-", "db2inst1", "-c", "db2 update dbm cfg using SSL_VERSIONS TLSV12"); - db.execInContainer("su", "-", "db2inst1", "-c", "db2 update dbm cfg using SSL_SVCENAME 50000"); - db.execInContainer("su", "-", "db2inst1", "-c", "db2set -i db2inst1 DB2COMM=SSL"); - db.execInContainer("su", "-", "db2inst1", "-c", "db2stop force"); - db.execInContainer("su", "-", "db2inst1", "-c", "db2start"); - return db.execInContainer("su", "-", "db2inst1", "-c", "cat server.arm").getStdout(); - } - - private static void convertAndImportCertificate(final String certificate) throws IOException, InterruptedException { - final Runtime run = Runtime.getRuntime(); - try (final PrintWriter out = new PrintWriter("certificate.pem", StandardCharsets.UTF_8)) { - out.print(certificate); - } - runProcess("openssl x509 -outform der -in certificate.pem -out certificate.der", run); - runProcess( - "keytool -import -alias rds-root -keystore " + KEY_STORE_FILE_PATH + " -file certificate.der -storepass " + TEST_KEY_STORE_PASS - + " -noprompt", - run); - } - - private static void runProcess(final String cmd, final Runtime run) throws IOException, InterruptedException { - final Process pr = run.exec(cmd); - if (!pr.waitFor(30, TimeUnit.SECONDS)) { - pr.destroy(); - throw new RuntimeException("Timeout while executing: " + cmd); - } - } - -} diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2JdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2JdbcSourceAcceptanceTest.java deleted file mode 100644 index c6b8ca0f8c2f..000000000000 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/java/io/airbyte/integrations/source/db2_strict_encrypt/Db2JdbcSourceAcceptanceTest.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.db2_strict_encrypt; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.source.db2.Db2Source; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; -import io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import java.io.File; -import java.io.IOException; -import java.io.PrintWriter; -import java.nio.charset.StandardCharsets; -import java.sql.JDBCType; -import java.util.Collections; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.Db2Container; - -class Db2JdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { - - private static final String TEST_KEY_STORE_PASS = "Passw0rd"; - private static final String KEY_STORE_FILE_PATH = "clientkeystore.jks"; - - private static Set TEST_TABLES = Collections.emptySet(); - private static String certificate; - private static Db2Container db; - private JsonNode config; - - @BeforeAll - static void init() throws IOException, InterruptedException { - db = new Db2Container("ibmcom/db2:11.5.5.0").acceptLicense(); - db.start(); - - certificate = getCertificate(); - try { - convertAndImportCertificate(certificate); - } catch (final IOException | InterruptedException e) { - throw new RuntimeException("Failed to import certificate into Java Keystore"); - } - - // Db2 transforms names to upper case, so we need to use upper case name to retrieve data later. - SCHEMA_NAME = "JDBC_INTEGRATION_TEST1"; - SCHEMA_NAME2 = "JDBC_INTEGRATION_TEST2"; - TEST_SCHEMAS = ImmutableSet.of(SCHEMA_NAME, SCHEMA_NAME2); - TABLE_NAME = "ID_AND_NAME"; - TABLE_NAME_WITH_SPACES = "ID AND NAME"; - TABLE_NAME_WITHOUT_PK = "ID_AND_NAME_WITHOUT_PK"; - TABLE_NAME_COMPOSITE_PK = "FULL_NAME_COMPOSITE_PK"; - TABLE_NAME_WITHOUT_CURSOR_TYPE = "TABLE_NAME_WITHOUT_CURSOR_TYPE"; - TABLE_NAME_WITH_NULLABLE_CURSOR_TYPE = "TABLE_NAME_WITH_NULLABLE_CURSOR_TYPE"; - TABLE_NAME_AND_TIMESTAMP = "NAME_AND_TIMESTAMP"; - TEST_TABLES = ImmutableSet - .of(TABLE_NAME, TABLE_NAME_WITHOUT_PK, TABLE_NAME_COMPOSITE_PK, TABLE_NAME_AND_TIMESTAMP); - COL_ID = "ID"; - COL_NAME = "NAME"; - COL_UPDATED_AT = "UPDATED_AT"; - COL_FIRST_NAME = "FIRST_NAME"; - COL_LAST_NAME = "LAST_NAME"; - COL_LAST_NAME_WITH_SPACE = "LAST NAME"; - COL_TIMESTAMP = "TIMESTAMP"; - // In Db2 PK columns must be declared with NOT NULL statement. - COLUMN_CLAUSE_WITH_PK = "id INTEGER NOT NULL, name VARCHAR(200), updated_at DATE"; - COLUMN_CLAUSE_WITH_COMPOSITE_PK = "first_name VARCHAR(200) NOT NULL, last_name VARCHAR(200) NOT NULL, updated_at DATE"; - // There is no IF EXISTS statement for a schema in Db2. - // The schema name must be in the catalog when attempting the DROP statement; otherwise an error is - // returned. - DROP_SCHEMA_QUERY = "DROP SCHEMA %s RESTRICT"; - CREATE_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s boolean)"; - INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES(true)"; - } - - @BeforeEach - public void setup() throws Exception { - config = Jsons.jsonNode(ImmutableMap.builder() - .put("host", db.getHost()) - .put("port", db.getFirstMappedPort()) - .put("db", db.getDatabaseName()) - .put("username", db.getUsername()) - .put("password", db.getPassword()) - .put("encryption", Jsons.jsonNode(ImmutableMap.builder() - .put("encryption_method", "encrypted_verify_certificate") - .put("ssl_certificate", certificate) - .put("key_store_password", TEST_KEY_STORE_PASS) - .build())) - .build()); - - super.setup(); - } - - @AfterEach - public void clean() throws Exception { - // In Db2 before dropping a schema, all objects that were in that schema must be dropped or moved to - // another schema. - for (final String tableName : TEST_TABLES) { - final String dropTableQuery = String - .format("DROP TABLE IF EXISTS %s.%s", SCHEMA_NAME, tableName); - super.database.execute(connection -> connection.createStatement().execute(dropTableQuery)); - } - for (int i = 2; i < 10; i++) { - final String dropTableQuery = String - .format("DROP TABLE IF EXISTS %s.%s%s", SCHEMA_NAME, TABLE_NAME, i); - super.database.execute(connection -> connection.createStatement().execute(dropTableQuery)); - } - super.database.execute(connection -> connection.createStatement().execute(String - .format("DROP TABLE IF EXISTS %s.%s", SCHEMA_NAME, - RelationalDbQueryUtils.enquoteIdentifier(TABLE_NAME_WITH_SPACES, connection.getMetaData().getIdentifierQuoteString())))); - super.database.execute(connection -> connection.createStatement().execute(String - .format("DROP TABLE IF EXISTS %s.%s", SCHEMA_NAME, - RelationalDbQueryUtils.enquoteIdentifier(TABLE_NAME_WITH_SPACES + 2, connection.getMetaData().getIdentifierQuoteString())))); - super.database.execute(connection -> connection.createStatement().execute(String - .format("DROP TABLE IF EXISTS %s.%s", SCHEMA_NAME2, - RelationalDbQueryUtils.enquoteIdentifier(TABLE_NAME, connection.getMetaData().getIdentifierQuoteString())))); - super.database.execute(connection -> connection.createStatement().execute(String - .format("DROP TABLE IF EXISTS %s.%s", SCHEMA_NAME, - RelationalDbQueryUtils.enquoteIdentifier(TABLE_NAME_WITHOUT_CURSOR_TYPE, connection.getMetaData().getIdentifierQuoteString())))); - super.database.execute(connection -> connection.createStatement().execute(String - .format("DROP TABLE IF EXISTS %s.%s", SCHEMA_NAME, - RelationalDbQueryUtils.enquoteIdentifier(TABLE_NAME_WITH_NULLABLE_CURSOR_TYPE, connection.getMetaData().getIdentifierQuoteString())))); - super.tearDown(); - } - - @AfterAll - static void cleanUp() { - new File("certificate.pem").delete(); - new File("certificate.der").delete(); - new File(KEY_STORE_FILE_PATH).delete(); - db.close(); - } - - @Override - public boolean supportsSchemas() { - return true; - } - - @Override - public JsonNode getConfig() { - return Jsons.clone(config); - } - - @Override - public String getDriverClass() { - return Db2StrictEncryptSource.DRIVER_CLASS; - } - - @Override - public AbstractJdbcSource getJdbcSource() { - return new Db2Source(); - } - - @Override - public Source getSource() { - return new Db2StrictEncryptSource(); - } - - @Test - void testSpec() throws Exception { - final ConnectorSpecification actual = source.spec(); - final ConnectorSpecification expected = - Jsons.deserialize(MoreResources.readResource("expected_spec.json"), ConnectorSpecification.class); - - assertEquals(expected, actual); - } - - /* Helpers */ - - private static String getCertificate() throws IOException, InterruptedException { - db.execInContainer("su", "-", "db2inst1", "-c", "gsk8capicmd_64 -keydb -create -db \"server.kdb\" -pw \"" + TEST_KEY_STORE_PASS + "\" -stash"); - db.execInContainer("su", "-", "db2inst1", "-c", "gsk8capicmd_64 -cert -create -db \"server.kdb\" -pw \"" + TEST_KEY_STORE_PASS - + "\" -label \"mylabel\" -dn \"CN=testcompany\" -size 2048 -sigalg SHA256_WITH_RSA"); - db.execInContainer("su", "-", "db2inst1", "-c", "gsk8capicmd_64 -cert -extract -db \"server.kdb\" -pw \"" + TEST_KEY_STORE_PASS - + "\" -label \"mylabel\" -target \"server.arm\" -format ascii -fips"); - - db.execInContainer("su", "-", "db2inst1", "-c", "db2 update dbm cfg using SSL_SVR_KEYDB /database/config/db2inst1/server.kdb"); - db.execInContainer("su", "-", "db2inst1", "-c", "db2 update dbm cfg using SSL_SVR_STASH /database/config/db2inst1/server.sth"); - db.execInContainer("su", "-", "db2inst1", "-c", "db2 update dbm cfg using SSL_SVR_LABEL mylabel"); - db.execInContainer("su", "-", "db2inst1", "-c", "db2 update dbm cfg using SSL_VERSIONS TLSV12"); - db.execInContainer("su", "-", "db2inst1", "-c", "db2 update dbm cfg using SSL_SVCENAME 50000"); - db.execInContainer("su", "-", "db2inst1", "-c", "db2set -i db2inst1 DB2COMM=SSL"); - db.execInContainer("su", "-", "db2inst1", "-c", "db2stop force"); - db.execInContainer("su", "-", "db2inst1", "-c", "db2start"); - return db.execInContainer("su", "-", "db2inst1", "-c", "cat server.arm").getStdout(); - } - - private static void convertAndImportCertificate(final String certificate) throws IOException, InterruptedException { - final Runtime run = Runtime.getRuntime(); - try (final PrintWriter out = new PrintWriter("certificate.pem", StandardCharsets.UTF_8)) { - out.print(certificate); - } - runProcess("openssl x509 -outform der -in certificate.pem -out certificate.der", run); - runProcess( - "keytool -import -alias rds-root -keystore " + KEY_STORE_FILE_PATH + " -file certificate.der -storepass " + TEST_KEY_STORE_PASS - + " -noprompt", - run); - } - - private static void runProcess(final String cmd, final Runtime run) throws IOException, InterruptedException { - final Process pr = run.exec(cmd); - if (!pr.waitFor(30, TimeUnit.SECONDS)) { - pr.destroy(); - throw new RuntimeException("Timeout while executing: " + cmd); - } - } - -} diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/resources/expected_spec.json deleted file mode 100644 index 3fe1ab780525..000000000000 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/resources/expected_spec.json +++ /dev/null @@ -1,80 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/db2", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "IBM Db2 Source Spec", - "type": "object", - "required": ["host", "port", "db", "username", "password", "encryption"], - "properties": { - "host": { - "description": "Host of the Db2.", - "type": "string", - "order": 0 - }, - "port": { - "description": "Port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 8123, - "examples": ["8123"], - "order": 1 - }, - "db": { - "description": "Name of the database.", - "type": "string", - "examples": ["default"], - "order": 2 - }, - "username": { - "description": "Username to use to access the database.", - "type": "string", - "order": 3 - }, - "password": { - "description": "Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 4 - }, - "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "title": "JDBC URL Params", - "type": "string", - "order": 5 - }, - "encryption": { - "title": "Encryption", - "type": "object", - "description": "Encryption method to use when communicating with the database", - "order": 6, - "oneOf": [ - { - "title": "TLS Encrypted (verify certificate)", - "description": "Verify and use the cert provided by the server.", - "required": ["encryption_method", "ssl_certificate"], - "properties": { - "encryption_method": { - "type": "string", - "const": "encrypted_verify_certificate" - }, - "ssl_certificate": { - "title": "SSL PEM file", - "description": "Privacy Enhanced Mail (PEM) files are concatenated certificate containers frequently used in certificate installations", - "type": "string", - "airbyte_secret": true, - "multiline": true - }, - "key_store_password": { - "title": "Key Store Password. This field is optional. If you do not fill in this field, the password will be randomly generated.", - "description": "Key Store Password", - "type": "string", - "airbyte_secret": true - } - } - } - ] - } - } - } -} diff --git a/airbyte-integrations/connectors/source-db2/.dockerignore b/airbyte-integrations/connectors/source-db2/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-db2/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-db2/Dockerfile b/airbyte-integrations/connectors/source-db2/Dockerfile deleted file mode 100644 index 1a70ab09d31a..000000000000 --- a/airbyte-integrations/connectors/source-db2/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-db2 - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-db2 - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.20 -LABEL io.airbyte.name=airbyte/source-db2 diff --git a/airbyte-integrations/connectors/source-db2/README.md b/airbyte-integrations/connectors/source-db2/README.md index 8192be33c499..d4606b29c326 100644 --- a/airbyte-integrations/connectors/source-db2/README.md +++ b/airbyte-integrations/connectors/source-db2/README.md @@ -7,4 +7,4 @@ ## Integration tests For acceptance tests run -`./gradlew :airbyte-integrations:connectors:db2:integrationTest` \ No newline at end of file +`./gradlew :airbyte-integrations:connectors:db2:integrationTest` diff --git a/airbyte-integrations/connectors/source-db2/acceptance-test-config.yml b/airbyte-integrations/connectors/source-db2/acceptance-test-config.yml deleted file mode 100644 index 37095f7ef91f..000000000000 --- a/airbyte-integrations/connectors/source-db2/acceptance-test-config.yml +++ /dev/null @@ -1,7 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-db2:dev -tests: - spec: - - spec_path: "src/test-integration/resources/expected_spec.json" - config_path: "src/test-integration/resources/dummy_config.json" diff --git a/airbyte-integrations/connectors/source-db2/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-db2/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-db2/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-db2/build.gradle b/airbyte-integrations/connectors/source-db2/build.gradle index 0cb0e59c15fa..e8fbadae109e 100644 --- a/airbyte-integrations/connectors/source-db2/build.gradle +++ b/airbyte-integrations/connectors/source-db2/build.gradle @@ -1,33 +1,36 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' - id 'airbyte-connector-acceptance-test' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.7.7' + features = ['db-sources'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileTestJava { + options.compilerArgs.remove("-Werror") + } + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.source.db2.Db2Source' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation project(':airbyte-integrations:connectors:source-relational-db') - implementation libs.airbyte.protocol - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation group: 'com.ibm.db2', name: 'jcc', version: '11.5.5.0' - testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) - testImplementation project(':airbyte-test-utils') - testImplementation libs.connectors.testcontainers.db2 - testImplementation project(":airbyte-json-validation") + testImplementation libs.testcontainers.db2 - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-db2') - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) integrationTestJavaImplementation 'org.apache.commons:commons-lang3:3.11' } - diff --git a/airbyte-integrations/connectors/source-db2/metadata.yaml b/airbyte-integrations/connectors/source-db2/metadata.yaml index 69cda6038a2f..84c13bc16e24 100644 --- a/airbyte-integrations/connectors/source-db2/metadata.yaml +++ b/airbyte-integrations/connectors/source-db2/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: source definitionId: 447e0381-3780-4b46-bb62-00a4e3c8b8e2 - dockerImageTag: 0.1.20 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-db2 githubIssueLabel: source-db2 icon: db2.svg @@ -13,7 +13,7 @@ data: name: IBM Db2 registries: cloud: - enabled: false # Requires a strict-encrypt variant + enabled: false # Would require DEPLOYMENT_MODE=cloud handling to release to cloud again oss: enabled: true releaseStage: alpha diff --git a/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java b/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java index aa98e99b46a5..9437232493e3 100644 --- a/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java +++ b/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java @@ -7,16 +7,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.cdk.integrations.source.jdbc.dto.JdbcPrivilegeDto; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.dto.JdbcPrivilegeDto; import java.io.IOException; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; diff --git a/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2SourceOperations.java b/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2SourceOperations.java index bd5cc47b5132..049d52ffcd90 100644 --- a/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2SourceOperations.java +++ b/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2SourceOperations.java @@ -4,13 +4,13 @@ package io.airbyte.integrations.source.db2; -import static io.airbyte.db.jdbc.DateTimeConverter.putJavaSQLTime; +import static io.airbyte.cdk.db.jdbc.DateTimeConverter.putJavaSQLTime; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.jdbc.DateTimeConverter; +import io.airbyte.cdk.db.jdbc.JdbcSourceOperations; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.DateTimeConverter; -import io.airbyte.db.jdbc.JdbcSourceOperations; import java.sql.Date; import java.sql.PreparedStatement; import java.sql.ResultSet; diff --git a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceAcceptanceTest.java b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceAcceptanceTest.java index 41621650b23c..48230d65505a 100644 --- a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceAcceptanceTest.java @@ -9,16 +9,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.source.db2.Db2Source; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; @@ -34,9 +34,11 @@ import java.util.HashMap; import java.util.List; import javax.sql.DataSource; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.testcontainers.containers.Db2Container; +@Disabled public class Db2SourceAcceptanceTest extends SourceAcceptanceTest { private static final String SCHEMA_NAME = "SOURCE_INTEGRATION_TEST"; diff --git a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceCertificateAcceptanceTest.java b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceCertificateAcceptanceTest.java index 310157fa8ad3..011033264271 100644 --- a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceCertificateAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceCertificateAcceptanceTest.java @@ -7,15 +7,15 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.source.db2.Db2Source; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.CatalogHelpers; @@ -31,8 +31,10 @@ import java.util.HashMap; import java.util.concurrent.TimeUnit; import javax.sql.DataSource; +import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.Db2Container; +@Disabled public class Db2SourceCertificateAcceptanceTest extends SourceAcceptanceTest { private static final String SCHEMA_NAME = "SOURCE_INTEGRATION_TEST"; diff --git a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java index 0e4dbcfde53e..d4f70d9d2119 100644 --- a/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-db2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/Db2SourceDatatypeTest.java @@ -6,20 +6,22 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDataHolder; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.source.db2.Db2Source; -import io.airbyte.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; -import io.airbyte.integrations.standardtest.source.TestDataHolder; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.JsonSchemaType; import org.jooq.DSLContext; import org.jooq.SQLDialect; +import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.Db2Container; +@Disabled public class Db2SourceDatatypeTest extends AbstractSourceDatabaseTypeTest { private static final String CREATE_TABLE_SQL = "CREATE TABLE %1$s(%2$s INTEGER NOT NULL PRIMARY KEY, %3$s %4$s)"; diff --git a/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2JdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2JdbcSourceAcceptanceTest.java index 0b0486459211..3d53bc09582e 100644 --- a/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2JdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2JdbcSourceAcceptanceTest.java @@ -4,35 +4,28 @@ package io.airbyte.integrations.source.db2; -import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifier; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifier; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; -import java.sql.JDBCType; import java.util.Collections; import java.util.Set; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.Db2Container; -class Db2JdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { +@Disabled +class Db2JdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { + private final static Db2Container DB_2_CONTAINER = new Db2Container("ibmcom/db2:11.5.5.0").acceptLicense(); + private static final String QUOTE_STRING = "\""; private static Set TEST_TABLES = Collections.emptySet(); - private static Db2Container db; - private JsonNode config; @BeforeAll static void init() { - db = new Db2Container("ibmcom/db2:11.5.5.0").acceptLicense(); - db.start(); - // Db2 transforms names to upper case, so we need to use upper case name to retrieve data later. SCHEMA_NAME = "JDBC_INTEGRATION_TEST1"; SCHEMA_NAME2 = "JDBC_INTEGRATION_TEST2"; @@ -64,76 +57,63 @@ static void init() { INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES(true)"; } - @BeforeEach - public void setup() throws Exception { - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) - .put("db", db.getDatabaseName()) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) - .put(JdbcUtils.ENCRYPTION_KEY, Jsons.jsonNode(ImmutableMap.builder() - .put("encryption_method", "unencrypted") - .build())) - .build()); - - super.setup(); + @AfterAll + static void cleanUp() { + DB_2_CONTAINER.close(); } - @AfterEach - public void clean() throws Exception { + static void deleteTablesAndSchema(final Db2TestDatabase testdb) { // In Db2 before dropping a schema, all objects that were in that schema must be dropped or moved to // another schema. for (final String tableName : TEST_TABLES) { final String dropTableQuery = String .format("DROP TABLE IF EXISTS %s.%s", SCHEMA_NAME, tableName); - super.database.execute(connection -> connection.createStatement().execute(dropTableQuery)); + testdb.with(dropTableQuery); } for (int i = 2; i < 10; i++) { final String dropTableQuery = String .format("DROP TABLE IF EXISTS %s.%s%s", SCHEMA_NAME, TABLE_NAME, i); - super.database.execute(connection -> connection.createStatement().execute(dropTableQuery)); + testdb.with(dropTableQuery); } - super.database.execute(connection -> connection.createStatement().execute(String + testdb.with(String .format("DROP TABLE IF EXISTS %s.%s", SCHEMA_NAME, - enquoteIdentifier(TABLE_NAME_WITH_SPACES, connection.getMetaData().getIdentifierQuoteString())))); - super.database.execute(connection -> connection.createStatement().execute(String + enquoteIdentifier(TABLE_NAME_WITH_SPACES, QUOTE_STRING))); + testdb.with(String .format("DROP TABLE IF EXISTS %s.%s", SCHEMA_NAME, - enquoteIdentifier(TABLE_NAME_WITH_SPACES + 2, connection.getMetaData().getIdentifierQuoteString())))); - super.database.execute(connection -> connection.createStatement().execute(String + enquoteIdentifier(TABLE_NAME_WITH_SPACES + 2, QUOTE_STRING))); + testdb.with(String .format("DROP TABLE IF EXISTS %s.%s", SCHEMA_NAME2, - enquoteIdentifier(TABLE_NAME, connection.getMetaData().getIdentifierQuoteString())))); - super.database.execute(connection -> connection.createStatement().execute(String + enquoteIdentifier(TABLE_NAME, QUOTE_STRING))); + testdb.with(String .format("DROP TABLE IF EXISTS %s.%s", SCHEMA_NAME, - enquoteIdentifier(TABLE_NAME_WITHOUT_CURSOR_TYPE, connection.getMetaData().getIdentifierQuoteString())))); - super.database.execute(connection -> connection.createStatement().execute(String + enquoteIdentifier(TABLE_NAME_WITHOUT_CURSOR_TYPE, QUOTE_STRING))); + testdb.with(String .format("DROP TABLE IF EXISTS %s.%s", SCHEMA_NAME, - enquoteIdentifier(TABLE_NAME_WITH_NULLABLE_CURSOR_TYPE, connection.getMetaData().getIdentifierQuoteString())))); - super.tearDown(); - } + enquoteIdentifier(TABLE_NAME_WITH_NULLABLE_CURSOR_TYPE, QUOTE_STRING))); + for (final String schemaName : TEST_SCHEMAS) { + testdb.with(DROP_SCHEMA_QUERY, schemaName); + } - @AfterAll - static void cleanUp() { - db.close(); } @Override - public boolean supportsSchemas() { - return true; + protected Db2TestDatabase createTestDatabase() { + DB_2_CONTAINER.start(); + return new Db2TestDatabase(DB_2_CONTAINER).initialized(); } @Override - public JsonNode getConfig() { - return Jsons.clone(config); + public boolean supportsSchemas() { + return true; } @Override - public String getDriverClass() { - return Db2Source.DRIVER_CLASS; + public JsonNode config() { + return Jsons.clone(testdb.configBuilder().build()); } @Override - public AbstractJdbcSource getJdbcSource() { + protected Db2Source source() { return new Db2Source(); } diff --git a/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2SpecTest.java b/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2SpecTest.java index f3a985c4cd01..c11d7736a9ef 100644 --- a/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2SpecTest.java +++ b/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2SpecTest.java @@ -20,8 +20,10 @@ import java.nio.file.Path; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +@Disabled public class Db2SpecTest { private static JsonNode schema; diff --git a/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2TestDatabase.java b/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2TestDatabase.java new file mode 100644 index 000000000000..216ed0ee5319 --- /dev/null +++ b/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2TestDatabase.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.db2; + +import static io.airbyte.integrations.source.db2.Db2JdbcSourceAcceptanceTest.deleteTablesAndSchema; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.TestDatabase; +import io.airbyte.commons.json.Jsons; +import java.util.stream.Stream; +import org.jooq.SQLDialect; +import org.testcontainers.containers.Db2Container; + +public class Db2TestDatabase extends + TestDatabase { + + private final Db2Container container; + + protected Db2TestDatabase(final Db2Container container) { + super(container); + this.container = container; + } + + @Override + public String getJdbcUrl() { + return container.getJdbcUrl(); + } + + @Override + public String getUserName() { + return container.getUsername(); + } + + @Override + public String getPassword() { + return container.getPassword(); + } + + @Override + public String getDatabaseName() { + return container.getDatabaseName(); + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.empty(); + } + + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return DatabaseDriver.DB2; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.DEFAULT; + } + + @Override + public void close() { + deleteTablesAndSchema(this); + } + + @Override + public Db2DbConfigBuilder configBuilder() { + return new Db2DbConfigBuilder(this) + .with(JdbcUtils.HOST_KEY, container.getHost()) + .with(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) + .with("db", container.getDatabaseName()) + .with(JdbcUtils.USERNAME_KEY, container.getUsername()) + .with(JdbcUtils.PASSWORD_KEY, container.getPassword()) + .with(JdbcUtils.ENCRYPTION_KEY, Jsons.jsonNode(ImmutableMap.builder() + .put("encryption_method", "unencrypted") + .build())); + } + + static public class Db2DbConfigBuilder extends TestDatabase.ConfigBuilder { + + protected Db2DbConfigBuilder(final Db2TestDatabase testdb) { + super(testdb); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-delighted/Dockerfile b/airbyte-integrations/connectors/source-delighted/Dockerfile index 823aa5756ed8..8b3dc98fe629 100644 --- a/airbyte-integrations/connectors/source-delighted/Dockerfile +++ b/airbyte-integrations/connectors/source-delighted/Dockerfile @@ -12,5 +12,5 @@ COPY main.py ./. ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.2 +LABEL io.airbyte.version=0.2.3 LABEL io.airbyte.name=airbyte/source-delighted diff --git a/airbyte-integrations/connectors/source-delighted/README.md b/airbyte-integrations/connectors/source-delighted/README.md index c608e669d31d..30870b5fe37e 100644 --- a/airbyte-integrations/connectors/source-delighted/README.md +++ b/airbyte-integrations/connectors/source-delighted/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-delighted:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/delighted) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_delighted/spec.json` file. @@ -56,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-delighted:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-delighted build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-delighted:airbyteDocker +An image will be built with the tag `airbyte/source-delighted:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-delighted:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -77,44 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-delighted:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-delighted:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-delighted:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-delighted test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](connector-acceptance-tests.md) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-delighted:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-delighted:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -124,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-delighted test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/delighted.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-delighted/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-delighted/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-delighted/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-delighted/build.gradle b/airbyte-integrations/connectors/source-delighted/build.gradle deleted file mode 100644 index 90911256fb30..000000000000 --- a/airbyte-integrations/connectors/source-delighted/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_delighted' -} diff --git a/airbyte-integrations/connectors/source-delighted/metadata.yaml b/airbyte-integrations/connectors/source-delighted/metadata.yaml index 9b27d60b5d52..2a050c2b4814 100644 --- a/airbyte-integrations/connectors/source-delighted/metadata.yaml +++ b/airbyte-integrations/connectors/source-delighted/metadata.yaml @@ -1,12 +1,16 @@ data: + ab_internal: + ql: 300 + sl: 100 allowedHosts: hosts: - api.delighted.com connectorSubtype: api connectorType: source definitionId: cc88c43f-6f53-4e8a-8c4d-b284baaf9635 - dockerImageTag: 0.2.2 + dockerImageTag: 0.2.3 dockerRepository: airbyte/source-delighted + documentationUrl: https://docs.airbyte.com/integrations/sources/delighted githubIssueLabel: source-delighted icon: delighted.svg license: MIT @@ -17,12 +21,8 @@ data: oss: enabled: true releaseStage: beta - documentationUrl: https://docs.airbyte.com/integrations/sources/delighted + supportLevel: community tags: - language:low-code - language:python - ab_internal: - sl: 200 - ql: 300 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-delighted/source_delighted/spec.json b/airbyte-integrations/connectors/source-delighted/source_delighted/spec.json index 7de4d10499d6..c10655a32376 100644 --- a/airbyte-integrations/connectors/source-delighted/source_delighted/spec.json +++ b/airbyte-integrations/connectors/source-delighted/source_delighted/spec.json @@ -15,7 +15,7 @@ "order": 0 }, "since": { - "title": "Date Since", + "title": "Replication Start Date", "type": "string", "description": "The date from which you'd like to replicate the data", "examples": ["2022-05-30T04:50:23Z", "2022-05-30 04:50:23"], diff --git a/airbyte-integrations/connectors/source-dixa/.dockerignore b/airbyte-integrations/connectors/source-dixa/.dockerignore index 3c42828e18ee..d93da11b6063 100644 --- a/airbyte-integrations/connectors/source-dixa/.dockerignore +++ b/airbyte-integrations/connectors/source-dixa/.dockerignore @@ -1,6 +1,5 @@ * !Dockerfile -!Dockerfile.test !main.py !source_dixa !setup.py diff --git a/airbyte-integrations/connectors/source-dixa/Dockerfile b/airbyte-integrations/connectors/source-dixa/Dockerfile index e902f7d6f7d6..a1ef26aa947f 100644 --- a/airbyte-integrations/connectors/source-dixa/Dockerfile +++ b/airbyte-integrations/connectors/source-dixa/Dockerfile @@ -1,16 +1,38 @@ -FROM python:3.9-slim +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base WORKDIR /airbyte/integration_code -COPY source_dixa ./source_dixa + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only COPY main.py ./ -COPY setup.py ./ -RUN pip install . +COPY source_dixa ./source_dixa ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.version=0.3.0 LABEL io.airbyte.name=airbyte/source-dixa diff --git a/airbyte-integrations/connectors/source-dixa/README.md b/airbyte-integrations/connectors/source-dixa/README.md index dc11a762c554..a1ad889ba936 100644 --- a/airbyte-integrations/connectors/source-dixa/README.md +++ b/airbyte-integrations/connectors/source-dixa/README.md @@ -1,78 +1,34 @@ # Dixa Source -### DISCLAIMER - -This source is currently not running CI pending the creation of a sandbox account, tracked [here](https://github.com/airbytehq/airbyte/issues/4667). -### END DISCLAIMER - -This is the repository for the Dixa source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/dixa). +This is the repository for the Dixa configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/dixa). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-dixa:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/dixa) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_dixa/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/dixa) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_dixa/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source dixa test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-dixa:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-dixa build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-dixa:airbyteDocker +An image will be built with the tag `airbyte/source-dixa:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-dixa:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -82,44 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-dixa:dev check --confi docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-dixa:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-dixa:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-dixa test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-dixa:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-dixa:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -129,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-dixa test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/dixa.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-dixa/__init__.py b/airbyte-integrations/connectors/source-dixa/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-dixa/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-dixa/acceptance-test-config.yml b/airbyte-integrations/connectors/source-dixa/acceptance-test-config.yml index 30ff974f5d74..74ae95d9eb2f 100644 --- a/airbyte-integrations/connectors/source-dixa/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-dixa/acceptance-test-config.yml @@ -1,26 +1,34 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-dixa:dev -tests: +test_strictness_level: low +acceptance_tests: spec: - - spec_path: "source_dixa/spec.json" + tests: + - spec_path: "source_dixa/spec.json" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 1200 + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 1200 incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" - timeout_seconds: 1200 + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + timeout_seconds: 1200 full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 1200 + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 1200 diff --git a/airbyte-integrations/connectors/source-dixa/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-dixa/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-dixa/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-dixa/build.gradle b/airbyte-integrations/connectors/source-dixa/build.gradle deleted file mode 100644 index b8b497e5e99c..000000000000 --- a/airbyte-integrations/connectors/source-dixa/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_dixa' -} diff --git a/airbyte-integrations/connectors/source-dixa/integration_tests/__init__.py b/airbyte-integrations/connectors/source-dixa/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-dixa/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-dixa/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-dixa/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-dixa/integration_tests/acceptance.py index d49b55882333..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-dixa/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-dixa/integration_tests/acceptance.py @@ -10,4 +10,7 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-dixa/metadata.yaml b/airbyte-integrations/connectors/source-dixa/metadata.yaml index d26ebbd01c91..9d45d0bb45f7 100644 --- a/airbyte-integrations/connectors/source-dixa/metadata.yaml +++ b/airbyte-integrations/connectors/source-dixa/metadata.yaml @@ -1,24 +1,28 @@ data: + allowedHosts: + hosts: + - exports.dixa.io + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: 0b5c867e-1b12-4d02-ab74-97b2184ff6d7 - dockerImageTag: 0.2.0 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-dixa githubIssueLabel: source-dixa icon: dixa.svg license: MIT name: Dixa - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: 2021-07-07 releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/dixa tags: - - language:python + - language:low-code ab_internal: sl: 100 ql: 100 - supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-dixa/setup.py b/airbyte-integrations/connectors/source-dixa/setup.py index 65b250fac3ca..be6537c6a3cd 100644 --- a/airbyte-integrations/connectors/source-dixa/setup.py +++ b/airbyte-integrations/connectors/source-dixa/setup.py @@ -11,18 +11,18 @@ TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", + "pytest~=6.2", "pytest-mock~=3.6.1", - "pytest~=6.1", ] setup( name="source_dixa", description="Source implementation for Dixa.", - author="Oliver Meyer, Airbyte", + author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-dixa/source_dixa/__init__.py b/airbyte-integrations/connectors/source-dixa/source_dixa/__init__.py index cb020c8e70d1..54de1e3b103f 100644 --- a/airbyte-integrations/connectors/source-dixa/source_dixa/__init__.py +++ b/airbyte-integrations/connectors/source-dixa/source_dixa/__init__.py @@ -1,26 +1,7 @@ -""" -MIT License +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" from .source import SourceDixa diff --git a/airbyte-integrations/connectors/source-dixa/source_dixa/manifest.yaml b/airbyte-integrations/connectors/source-dixa/source_dixa/manifest.yaml new file mode 100644 index 000000000000..2c61e206d12f --- /dev/null +++ b/airbyte-integrations/connectors/source-dixa/source_dixa/manifest.yaml @@ -0,0 +1,95 @@ +version: "0.29.0" + +type: DeclarativeSource + +check: + type: CheckStream + stream_names: + - conversation_export + +streams: + - type: DeclarativeStream + name: conversation_export + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://exports.dixa.io/v1/ + path: conversation_export + http_method: GET + authenticator: + type: BearerAuthenticator + api_token: "{{ config['api_token'] }}" + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + backoff_strategies: + - type: ConstantBackoffStrategy + backoff_time_in_seconds: 60 + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: NoPagination + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updated_at + cursor_datetime_formats: + - "%ms" + datetime_format: "%ms" + step: P{{ config.batch_size }}D + cursor_granularity: P1D + start_datetime: + datetime: "{{ format_datetime(config['start_date'], '%Y-%m-%d') }}" + datetime_format: "%Y-%m-%d" + start_time_option: + type: RequestOption + inject_into: request_parameter + field_name: updated_after + end_datetime: + datetime: "{{ now_utc().strftime('%Y-%m-%d') }}" + datetime_format: "%Y-%m-%d" + end_time_option: + type: RequestOption + inject_into: request_parameter + field_name: updated_before + +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/dixa + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + type: object + additionalProperties: true + required: + - api_token + - start_date + properties: + api_token: + type: string + description: Dixa API token + airbyte_secret: true + order: 1 + batch_size: + type: integer + description: Number of days to batch into one request. Max 31. + pattern: ^[0-9]{1,2}$ + examples: + - 1 + - 31 + default: 31 + order: 2 + start_date: + type: string + title: Start date + format: date-time + description: The connector pulls records updated from this date onwards. + examples: + - YYYY-MM-DD + order: 3 diff --git a/airbyte-integrations/connectors/source-dixa/source_dixa/schemas/conversation_export.json b/airbyte-integrations/connectors/source-dixa/source_dixa/schemas/conversation_export.json index 6ee94ca78602..2381498b5db6 100644 --- a/airbyte-integrations/connectors/source-dixa/source_dixa/schemas/conversation_export.json +++ b/airbyte-integrations/connectors/source-dixa/source_dixa/schemas/conversation_export.json @@ -13,6 +13,18 @@ "initial_channel": { "type": ["null", "string"] }, + "requester_additional_emails": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "requester_additional_phone_numbers": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, "requester_id": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-dixa/source_dixa/source.py b/airbyte-integrations/connectors/source-dixa/source_dixa/source.py index 8f0689672881..e0458651420a 100644 --- a/airbyte-integrations/connectors/source-dixa/source_dixa/source.py +++ b/airbyte-integrations/connectors/source-dixa/source_dixa/source.py @@ -2,127 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from abc import ABC -from datetime import datetime -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import requests -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +WARNING: Do not modify this file. +""" -from . import utils - -class DixaStream(HttpStream, ABC): - - primary_key = "id" - url_base = "https://exports.dixa.io/v1/" - - backoff_sleep = 60 # seconds - - def __init__(self, config: Mapping[str, Any]) -> None: - super().__init__(authenticator=config["authenticator"]) - self.start_date = datetime.strptime(config["start_date"], "%Y-%m-%d") - self.start_timestamp = utils.datetime_to_ms_timestamp(self.start_date) - self.end_timestamp = utils.datetime_to_ms_timestamp(datetime.now()) + 1 - self.batch_size = config["batch_size"] - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: - return stream_slice - - def backoff_time(self, response: requests.Response): - """ - The rate limit is 10 requests per minute, so we sleep - for defined backoff_sleep time (default is 60 sec) before we continue. - - See https://support.dixa.help/en/articles/174-export-conversations-via-api - """ - return self.backoff_sleep - - -class IncrementalDixaStream(DixaStream): - - cursor_field = "updated_at" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield from response.json() - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: - """ - Uses the `updated_at` field, which is a Unix timestamp with millisecond precision. - """ - current_stream_state = current_stream_state or {} - return { - self.cursor_field: max( - current_stream_state.get(self.cursor_field, self.start_timestamp), - latest_record.get(self.cursor_field, self.start_timestamp), - ) - } - - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs): - """ - Returns slices of size self.batch_size. - """ - slices = [] - - stream_state = stream_state or {} - # If stream_state contains the cursor field and the value of the cursor - # field is higher than start_timestamp, then start at the cursor field - # value. Otherwise, start at start_timestamp. - updated_after = max(stream_state.get(self.cursor_field, 0), self.start_timestamp) - updated_before = min(utils.add_days_to_ms_timestamp(days=self.batch_size, ms_timestamp=updated_after), self.end_timestamp) - - # When we have abnormaly_large start_date, start_date > Now(), - # assign updated_before to the value of updated_after + batch_size, - # return single slice - if updated_after > updated_before: - updated_before = utils.add_days_to_ms_timestamp(days=self.batch_size, ms_timestamp=updated_after) - return [{"updated_after": updated_after, "updated_before": updated_before}] - else: - while updated_after < self.end_timestamp: - updated_before = min(utils.add_days_to_ms_timestamp(days=self.batch_size, ms_timestamp=updated_after), self.end_timestamp) - slices.append({"updated_after": updated_after, "updated_before": updated_before}) - updated_after = updated_before - - return slices - - -class ConversationExport(IncrementalDixaStream): - """ - https://support.dixa.help/en/articles/174-export-conversations-via-api - """ - - def path(self, **kwargs) -> str: - return "conversation_export" - - -class SourceDixa(AbstractSource): - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: - """ - Check connectivity using one day's worth of data. - """ - try: - config["authenticator"] = TokenAuthenticator(token=config["api_token"]) - stream = ConversationExport(config) - # using 1 day batch size for slices. - stream.batch_size = 1 - # use the first slice from stream_slices list - stream_slice = stream.stream_slices()[0] - list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice)) - return True, None - except Exception as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - config["authenticator"] = TokenAuthenticator(token=config["api_token"]) - return [ - ConversationExport(config), - ] +# Declarative Source +class SourceDixa(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-dixa/source_dixa/spec.json b/airbyte-integrations/connectors/source-dixa/source_dixa/spec.json deleted file mode 100644 index d4bfe2b03417..000000000000 --- a/airbyte-integrations/connectors/source-dixa/source_dixa/spec.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/dixa", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Dixa Spec", - "type": "object", - "required": ["api_token", "start_date"], - "additionalProperties": true, - "properties": { - "api_token": { - "type": "string", - "description": "Dixa API token", - "airbyte_secret": true - }, - "start_date": { - "type": "string", - "description": "The connector pulls records updated from this date onwards.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "examples": ["YYYY-MM-DD"] - }, - "batch_size": { - "type": "integer", - "description": "Number of days to batch into one request. Max 31.", - "pattern": "^[0-9]{1,2}$", - "examples": [1, 31], - "default": 31 - } - } - } -} diff --git a/airbyte-integrations/connectors/source-dixa/source_dixa/utils.py b/airbyte-integrations/connectors/source-dixa/source_dixa/utils.py deleted file mode 100644 index 91687b34f0b2..000000000000 --- a/airbyte-integrations/connectors/source-dixa/source_dixa/utils.py +++ /dev/null @@ -1,30 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from datetime import datetime, timedelta, timezone - - -def validate_ms_timestamp(ms_timestamp: int) -> int: - if not type(ms_timestamp) == int or not len(str(ms_timestamp)) == 13: - raise ValueError(f"Not a millisecond-precision timestamp: {ms_timestamp}") - return ms_timestamp - - -def ms_timestamp_to_datetime(ms_timestamp: int) -> datetime: - """ - Converts a millisecond-precision timestamp to a datetime object. - """ - return datetime.fromtimestamp(validate_ms_timestamp(ms_timestamp) / 1000, tz=timezone.utc) - - -def datetime_to_ms_timestamp(dt: datetime) -> int: - """ - Converts a datetime object to a millisecond-precision timestamp. - """ - return int(dt.timestamp() * 1000) - - -def add_days_to_ms_timestamp(days: int, ms_timestamp: int) -> int: - return datetime_to_ms_timestamp(ms_timestamp_to_datetime(validate_ms_timestamp(ms_timestamp)) + timedelta(days=days)) diff --git a/airbyte-integrations/connectors/source-dixa/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-dixa/unit_tests/unit_test.py deleted file mode 100644 index cbd78e67e68c..000000000000 --- a/airbyte-integrations/connectors/source-dixa/unit_tests/unit_test.py +++ /dev/null @@ -1,117 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from datetime import datetime, timezone - -import pytest -from source_dixa import utils -from source_dixa.source import ConversationExport - -config = {"authenticator": "", "start_date": "2021-07-01", "api_token": "TOKEN", "batch_size": 1} - - -@pytest.fixture -def conversation_export(): - return ConversationExport(config) - - -def test_validate_ms_timestamp_with_valid_input(): - assert utils.validate_ms_timestamp(1234567890123) == 1234567890123 - - -def test_validate_ms_timestamp_with_invalid_input_type(): - with pytest.raises(ValueError): - assert utils.validate_ms_timestamp(1.2) - - -def test_validate_ms_timestamp_with_invalid_input_length(): - with pytest.raises(ValueError): - assert utils.validate_ms_timestamp(1) - - -def test_ms_timestamp_to_datetime(): - assert utils.ms_timestamp_to_datetime(1625312980123) == datetime( - year=2021, month=7, day=3, hour=11, minute=49, second=40, microsecond=123000, tzinfo=timezone.utc - ) - - -def test_datetime_to_ms_timestamp(): - assert ( - utils.datetime_to_ms_timestamp( - datetime(year=2021, month=7, day=3, hour=11, minute=49, second=40, microsecond=123000, tzinfo=timezone.utc) - ) - == 1625312980123 - ) - - -def test_add_days_to_ms_timestamp(): - assert utils.add_days_to_ms_timestamp(days=1, ms_timestamp=1625312980123) == 1625399380123 - - -def test_stream_slices_without_state(conversation_export): - conversation_export.end_timestamp = 1625259600000 # 2021-07-03 00:00:00 + 1 ms - - expected_slices = [ - {'updated_after': 1625097600000, 'updated_before': 1625184000000}, - {'updated_after': 1625184000000, 'updated_before': 1625259600000}, - ] - - actual_slices = conversation_export.stream_slices() - assert actual_slices == expected_slices - - -def test_stream_slices_without_state_large_batch(): - - updated_config = config - updated_config["batch_size"] = 31 - - conversation_export = ConversationExport(updated_config) - conversation_export.end_timestamp = 1625259600000 # 2021-07-03 00:00:00 + 1 ms - expected_slices = [{"updated_after": 1625097600000, "updated_before": 1625259600000}] # 2021-07-01 12:00:00 """ - actual_slices = conversation_export.stream_slices() - assert actual_slices == expected_slices - - -def test_stream_slices_with_state(conversation_export): - conversation_export.end_timestamp = 1625259600001 # 2021-07-03 00:00:00 + 1 ms - expected_slices = [{"updated_after": 1625220000000, "updated_before": 1625259600001}] # 2021-07-01 12:00:00 - actual_slices = conversation_export.stream_slices(stream_state={"updated_at": 1625220000000}) # # 2021-07-02 12:00:00 - assert actual_slices == expected_slices - - -def test_stream_slices_with_start_timestamp_larger_than_state(): - # - # Test that if start_timestamp is larger than state, then start at start_timestamp. - # - updated_config = config - updated_config["start_date"] = "2021-12-01" - updated_config["batch_size"] = 31 - - conversation_export = ConversationExport(updated_config) - conversation_export.end_timestamp = 1638352800001 # 2021-12-01 12:00:00 + 1 ms - expected_slices = [{"updated_after": 1638316800000, "updated_before": 1638352800001}] # 2021-07-01 12:00:00 """ - actual_slices = conversation_export.stream_slices(stream_state={"updated_at": 1625216400000}) # # 2021-07-02 12:00:00 - assert actual_slices == expected_slices - - -def test_get_updated_state_without_state(conversation_export): - expected = {"updated_at": 1638316800000} - actual = conversation_export.get_updated_state(current_stream_state=None, latest_record={"updated_at": 1625259600001}) - assert actual == expected - - -def test_get_updated_state_with_bigger_state(conversation_export): - expected = {"updated_at": 1625263200000} - actual = conversation_export.get_updated_state( - current_stream_state={"updated_at": 1625263200000}, latest_record={"updated_at": 1625220000000} - ) - assert actual == expected - - -def test_get_updated_state_with_smaller_state(conversation_export): - expected = {"updated_at": 1625263200000} - actual = conversation_export.get_updated_state( - current_stream_state={"updated_at": 1625220000000}, latest_record={"updated_at": 1625263200000} - ) - assert actual == expected diff --git a/airbyte-integrations/connectors/source-dockerhub/README.md b/airbyte-integrations/connectors/source-dockerhub/README.md index 73055dd9d3ea..1ea11091ef9c 100644 --- a/airbyte-integrations/connectors/source-dockerhub/README.md +++ b/airbyte-integrations/connectors/source-dockerhub/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-dockerhub:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/dockerhub) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_dockerhub/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-dockerhub:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-dockerhub build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-dockerhub:airbyteDocker +An image will be built with the tag `airbyte/source-dockerhub:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-dockerhub:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-dockerhub:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-dockerhub:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-dockerhub:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-dockerhub test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-dockerhub:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-dockerhub:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-dockerhub test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/dockerhub.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-dockerhub/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-dockerhub/acceptance-test-docker.sh deleted file mode 100644 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-dockerhub/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-dockerhub/build.gradle b/airbyte-integrations/connectors/source-dockerhub/build.gradle deleted file mode 100644 index 80198b4ea6a1..000000000000 --- a/airbyte-integrations/connectors/source-dockerhub/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_dockerhub' -} diff --git a/airbyte-integrations/connectors/source-dockerhub/metadata.yaml b/airbyte-integrations/connectors/source-dockerhub/metadata.yaml index 3afe52544412..e0b403736301 100644 --- a/airbyte-integrations/connectors/source-dockerhub/metadata.yaml +++ b/airbyte-integrations/connectors/source-dockerhub/metadata.yaml @@ -21,7 +21,7 @@ data: releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/dockerhub tags: - - language:lowcode + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-dremio/README.md b/airbyte-integrations/connectors/source-dremio/README.md index f6940f75d578..4e6fe4090673 100644 --- a/airbyte-integrations/connectors/source-dremio/README.md +++ b/airbyte-integrations/connectors/source-dremio/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-dremio:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/dremio) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_dremio/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-dremio:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-dremio build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-dremio:airbyteDocker +An image will be built with the tag `airbyte/source-dremio:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-dremio:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-dremio:dev check --con docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-dremio:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-dremio:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-dremio test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-dremio:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-dremio:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-dremio test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/dremio.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-dremio/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-dremio/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-dremio/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-dremio/build.gradle b/airbyte-integrations/connectors/source-dremio/build.gradle deleted file mode 100644 index dc98548e5e0d..000000000000 --- a/airbyte-integrations/connectors/source-dremio/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_dremio' -} diff --git a/airbyte-integrations/connectors/source-drift/README.md b/airbyte-integrations/connectors/source-drift/README.md index 7bc078e5f95d..cf6d5b59aad7 100644 --- a/airbyte-integrations/connectors/source-drift/README.md +++ b/airbyte-integrations/connectors/source-drift/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-drift:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/drift) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_drift/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-drift:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-drift build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-drift:airbyteDocker +An image will be built with the tag `airbyte/source-drift:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-drift:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-drift:dev check --conf docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-drift:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-drift:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-drift test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-drift:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-drift:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-drift test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/drift.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-drift/acceptance-test-config.yml b/airbyte-integrations/connectors/source-drift/acceptance-test-config.yml index 1a7dd7340b9d..08662b2c94b3 100644 --- a/airbyte-integrations/connectors/source-drift/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-drift/acceptance-test-config.yml @@ -23,19 +23,19 @@ acceptance_tests: bypass_reason: "Sandbox account can't seed this stream" - name: accounts bypass_reason: "Sandbox account can't seed this stream" -# expect_records: -# path: "integration_tests/expected_records.jsonl" -# extra_fields: no -# exact_order: no -# extra_records: yes - incremental: + # expect_records: + # path: "integration_tests/expected_records.jsonl" + # extra_fields: no + # exact_order: no + # extra_records: yes + incremental: bypass_reason: "This connector does not implement incremental sync" -# TODO uncomment this block this block if your connector implements incremental sync: -# tests: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# future_state: -# future_state_path: "integration_tests/abnormal_state.json" + # TODO uncomment this block this block if your connector implements incremental sync: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-drift/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-drift/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-drift/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-drift/build.gradle b/airbyte-integrations/connectors/source-drift/build.gradle deleted file mode 100644 index af5fa27a38bb..000000000000 --- a/airbyte-integrations/connectors/source-drift/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_drift' -} diff --git a/airbyte-integrations/connectors/source-drift/metadata.yaml b/airbyte-integrations/connectors/source-drift/metadata.yaml index f17ce487e236..12e3531ac2c3 100644 --- a/airbyte-integrations/connectors/source-drift/metadata.yaml +++ b/airbyte-integrations/connectors/source-drift/metadata.yaml @@ -21,7 +21,7 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/drift tags: - - language:lowcode + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-dv-360/README.md b/airbyte-integrations/connectors/source-dv-360/README.md index edf18e77fd34..024e835dfa63 100644 --- a/airbyte-integrations/connectors/source-dv-360/README.md +++ b/airbyte-integrations/connectors/source-dv-360/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-dv-360:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/dv360) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_dv_360/spec.json` file. @@ -54,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-dv-360:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-dv-360 build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-dv-360:airbyteDocker +An image will be built with the tag `airbyte/source-dv-360:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-dv-360:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,44 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-dv-360:dev check --con docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-dv-360:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-dv-360:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-dv-360 test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-dv-360:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-dv-360:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -122,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-dv-360 test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/dv-360.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-dv-360/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-dv-360/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-dv-360/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-dv-360/build.gradle b/airbyte-integrations/connectors/source-dv-360/build.gradle deleted file mode 100644 index df00e8c6842c..000000000000 --- a/airbyte-integrations/connectors/source-dv-360/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_dv_360_singer' -} diff --git a/airbyte-integrations/connectors/source-dynamodb/Dockerfile b/airbyte-integrations/connectors/source-dynamodb/Dockerfile deleted file mode 100644 index c41dd8ca95f2..000000000000 --- a/airbyte-integrations/connectors/source-dynamodb/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-dynamodb - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-dynamodb - -COPY --from=build /airbyte /airbyte - -# Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile. -LABEL io.airbyte.version=0.1.2 -LABEL io.airbyte.name=airbyte/source-dynamodb diff --git a/airbyte-integrations/connectors/source-dynamodb/README.md b/airbyte-integrations/connectors/source-dynamodb/README.md index 56b62b4d9a31..9923e01a6d0d 100644 --- a/airbyte-integrations/connectors/source-dynamodb/README.md +++ b/airbyte-integrations/connectors/source-dynamodb/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:source-dynamodb:airbyteDocker +./gradlew :airbyte-integrations:connectors:source-dynamodb:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/source-dynamodb:dev`. the Dockerfile. #### Run @@ -62,8 +63,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-dynamodb test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/dynamodb.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-dynamodb/acceptance-test-config.yml b/airbyte-integrations/connectors/source-dynamodb/acceptance-test-config.yml deleted file mode 100644 index db5ee6a4a9cd..000000000000 --- a/airbyte-integrations/connectors/source-dynamodb/acceptance-test-config.yml +++ /dev/null @@ -1,8 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-dynamodb:dev -acceptance-tests: - spec: - tests: - - spec_path: "main/resources/spec.json" - \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-dynamodb/build.gradle b/airbyte-integrations/connectors/source-dynamodb/build.gradle index 5940aa61b252..3d741a844137 100644 --- a/airbyte-integrations/connectors/source-dynamodb/build.gradle +++ b/airbyte-integrations/connectors/source-dynamodb/build.gradle @@ -1,9 +1,23 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.7.7' + features = ['db-sources'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.source.dynamodb.DynamodbSource' } @@ -12,11 +26,6 @@ def testContainersVersion = '1.17.5' def assertVersion = '3.23.1' dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:connectors:source-relational-db') - implementation project(':airbyte-config-oss:config-models-oss') implementation platform('software.amazon.awssdk:bom:2.18.1') // https://mvnrepository.com/artifact/software.amazon.awssdk/dynamodb @@ -33,11 +42,6 @@ dependencies { testImplementation "org.assertj:assertj-core:${assertVersion}" testImplementation "org.testcontainers:localstack:${testContainersVersion}" + integrationTestJavaImplementation 'com.amazonaws:aws-java-sdk:1.12.610' - - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-dynamodb') - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-dynamodb/metadata.yaml b/airbyte-integrations/connectors/source-dynamodb/metadata.yaml index e71ffe623c56..43079b9fde0a 100644 --- a/airbyte-integrations/connectors/source-dynamodb/metadata.yaml +++ b/airbyte-integrations/connectors/source-dynamodb/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 50401137-8871-4c5a-abb7-1f5fda35545a - dockerImageTag: 0.1.2 + dockerImageTag: 0.2.1 dockerRepository: airbyte/source-dynamodb documentationUrl: https://docs.airbyte.com/integrations/sources/dynamodb githubIssueLabel: source-dynamodb diff --git a/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbOperations.java b/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbOperations.java index cc30adfb8af6..27b1fbb3fa8d 100644 --- a/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbOperations.java +++ b/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbOperations.java @@ -7,7 +7,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.module.SimpleModule; -import io.airbyte.db.AbstractDatabase; +import io.airbyte.cdk.db.AbstractDatabase; import java.io.Closeable; import java.time.LocalDate; import java.time.format.DateTimeParseException; diff --git a/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbSource.java b/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbSource.java index 987aedd4a7d7..f3ffab950c66 100644 --- a/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbSource.java +++ b/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbSource.java @@ -8,19 +8,18 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.features.FeatureFlags; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.source.relationaldb.CursorInfo; +import io.airbyte.cdk.integrations.source.relationaldb.StateDecoratingIterator; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManagerFactory; import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.stream.AirbyteStreamUtils; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteTraceMessageUtility; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.source.relationaldb.CursorInfo; -import io.airbyte.integrations.source.relationaldb.StateDecoratingIterator; -import io.airbyte.integrations.source.relationaldb.state.StateManager; -import io.airbyte.integrations.source.relationaldb.state.StateManagerFactory; import io.airbyte.protocol.models.JsonSchemaPrimitiveUtil.JsonSchemaPrimitive; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; @@ -41,8 +40,6 @@ public class DynamodbSource extends BaseConnector implements Source { private static final Logger LOGGER = LoggerFactory.getLogger(DynamodbSource.class); - private final FeatureFlags featureFlags = new EnvVariableFeatureFlags(); - private final ObjectMapper objectMapper = new ObjectMapper(); public static void main(final String[] args) throws Exception { @@ -97,7 +94,7 @@ public AutoCloseableIterator read(final JsonNode config, final ConfiguredAirbyteCatalog catalog, final JsonNode state) { - final var streamState = DynamodbUtils.deserializeStreamState(state, featureFlags.useStreamCapableState()); + final var streamState = DynamodbUtils.deserializeStreamState(state); final StateManager stateManager = StateManagerFactory .createStateManager(streamState.airbyteStateType(), streamState.airbyteStateMessages(), catalog); @@ -124,7 +121,6 @@ private AutoCloseableIterator scanIncremental(final DynamodbOper final StateManager stateManager) { final var streamPair = new AirbyteStreamNameNamespacePair(airbyteStream.getName(), airbyteStream.getNamespace()); - final Optional cursorInfo = stateManager.getCursorInfo(streamPair); final Map properties = objectMapper.convertValue(airbyteStream.getJsonSchema().get("properties"), new TypeReference<>() {}); @@ -164,17 +160,18 @@ private AutoCloseableIterator scanIncremental(final DynamodbOper .map(jn -> DynamodbUtils.mapAirbyteMessage(airbyteStream.getName(), jn)); // wrap stream in state emission iterator - return AutoCloseableIterators.transform(autoCloseableIterator -> new StateDecoratingIterator( - autoCloseableIterator, - stateManager, - streamPair, - cursorField, - cursorInfo.map(CursorInfo::getCursor).orElse(null), - JsonSchemaPrimitive.valueOf(cursorType.toUpperCase()), - // emit state after full stream has been processed - 0), - AutoCloseableIterators.fromStream(messageStream)); - + return AutoCloseableIterators.fromIterator( + new StateDecoratingIterator( + AutoCloseableIterators.fromStream( + messageStream, + AirbyteStreamUtils.convertFromNameAndNamespace(airbyteStream.getName(), airbyteStream.getNamespace())), + stateManager, + streamPair, + cursorField, + cursorInfo.map(CursorInfo::getCursor).orElse(null), + JsonSchemaPrimitive.valueOf(cursorType.toUpperCase()), + // emit state after full stream has been processed + 0)); } private AutoCloseableIterator scanFullRefresh(final DynamodbOperations dynamodbOperations, @@ -187,7 +184,9 @@ private AutoCloseableIterator scanFullRefresh(final DynamodbOper .stream() .map(jn -> DynamodbUtils.mapAirbyteMessage(airbyteStream.getName(), jn)); - return AutoCloseableIterators.fromStream(messageStream); + return AutoCloseableIterators.fromStream( + messageStream, + AirbyteStreamUtils.convertFromNameAndNamespace(airbyteStream.getName(), airbyteStream.getNamespace())); } } diff --git a/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbUtils.java b/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbUtils.java index 31f2dd3c1376..e79dc1833af0 100644 --- a/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbUtils.java +++ b/airbyte-integrations/connectors/source-dynamodb/src/main/java/io/airbyte/integrations/source/dynamodb/DynamodbUtils.java @@ -8,7 +8,6 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.configoss.StateWrapper; import io.airbyte.configoss.helpers.StateMessageHelper; -import io.airbyte.integrations.source.relationaldb.models.DbState; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; @@ -53,9 +52,9 @@ public static AirbyteMessage mapAirbyteMessage(final String stream, final JsonNo .withData(data)); } - public static StreamState deserializeStreamState(final JsonNode state, final boolean useStreamCapableState) { + public static StreamState deserializeStreamState(final JsonNode state) { final Optional typedState = - StateMessageHelper.getTypedState(state, useStreamCapableState); + StateMessageHelper.getTypedState(state); return typedState.map(stateWrapper -> switch (stateWrapper.getStateType()) { case STREAM: yield new StreamState(AirbyteStateMessage.AirbyteStateType.STREAM, @@ -68,15 +67,10 @@ yield new StreamState(AirbyteStateMessage.AirbyteStateType.LEGACY, List.of( throw new UnsupportedOperationException("Unsupported stream state"); }).orElseGet(() -> { // create empty initial state - if (useStreamCapableState) { - return new StreamState(AirbyteStateMessage.AirbyteStateType.STREAM, List.of( - new AirbyteStateMessage().withType(AirbyteStateMessage.AirbyteStateType.STREAM) - .withStream(new AirbyteStreamState()))); - } else { - return new StreamState(AirbyteStateMessage.AirbyteStateType.LEGACY, List.of( - new AirbyteStateMessage().withType(AirbyteStateMessage.AirbyteStateType.LEGACY) - .withData(Jsons.jsonNode(new DbState())))); - } + return new StreamState(AirbyteStateMessage.AirbyteStateType.STREAM, List.of( + new AirbyteStateMessage().withType(AirbyteStateMessage.AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState()))); + }); } diff --git a/airbyte-integrations/connectors/source-dynamodb/src/main/resources/spec.json b/airbyte-integrations/connectors/source-dynamodb/src/main/resources/spec.json index 6b6a09f075fb..25745616e501 100644 --- a/airbyte-integrations/connectors/source-dynamodb/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-dynamodb/src/main/resources/spec.json @@ -21,31 +21,39 @@ "description": "The region of the Dynamodb database", "enum": [ "", - "us-east-1", - "us-east-2", - "us-west-1", - "us-west-2", "af-south-1", "ap-east-1", - "ap-south-1", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", + "ap-south-1", + "ap-south-2", "ap-southeast-1", "ap-southeast-2", + "ap-southeast-3", + "ap-southeast-4", "ca-central-1", + "ca-west-1", "cn-north-1", "cn-northwest-1", "eu-central-1", + "eu-central-2", "eu-north-1", "eu-south-1", + "eu-south-2", "eu-west-1", "eu-west-2", "eu-west-3", - "sa-east-1", + "il-central-1", + "me-central-1", "me-south-1", + "sa-east-1", + "us-east-1", + "us-east-2", "us-gov-east-1", - "us-gov-west-1" + "us-gov-west-1", + "us-west-1", + "us-west-2" ] }, "access_key_id": { diff --git a/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbOperationsTest.java b/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbOperationsTest.java index d2784fbeddce..814dbcff8458 100644 --- a/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbOperationsTest.java +++ b/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbOperationsTest.java @@ -15,12 +15,14 @@ import org.json.JSONException; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.skyscreamer.jsonassert.JSONAssert; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; +@Disabled public class DynamodbOperationsTest { private static final String TABLE_NAME = "airbyte_table"; diff --git a/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbSourceAcceptanceTest.java index f5e0fcfccef0..86c86fae53be 100644 --- a/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbSourceAcceptanceTest.java @@ -5,18 +5,20 @@ package io.airbyte.integrations.source.dynamodb; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConnectorSpecification; import java.util.HashMap; import java.util.Map; +import org.junit.jupiter.api.Disabled; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; +@Disabled public class DynamodbSourceAcceptanceTest extends SourceAcceptanceTest { private static final String TABLE_NAME = "airbyte_table"; diff --git a/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbSourceTest.java b/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbSourceTest.java index 7e37aa47e338..713e84ed53b6 100644 --- a/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbSourceTest.java +++ b/airbyte-integrations/connectors/source-dynamodb/src/test-integration/java/io/airbyte/integrations/source/dynamodb/DynamodbSourceTest.java @@ -17,11 +17,13 @@ import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.dynamodb.model.AttributeValue; import software.amazon.awssdk.services.dynamodb.model.PutItemRequest; +@Disabled public class DynamodbSourceTest { private static final String TABLE_NAME = "airbyte_table"; diff --git a/airbyte-integrations/connectors/source-e2e-test-cloud/.dockerignore b/airbyte-integrations/connectors/source-e2e-test-cloud/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-e2e-test-cloud/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-e2e-test-cloud/Dockerfile b/airbyte-integrations/connectors/source-e2e-test-cloud/Dockerfile deleted file mode 100644 index 9bce1d6e9969..000000000000 --- a/airbyte-integrations/connectors/source-e2e-test-cloud/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-e2e-test-cloud - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-e2e-test-cloud - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=2.1.4 -LABEL io.airbyte.name=airbyte/source-e2e-test-cloud diff --git a/airbyte-integrations/connectors/source-e2e-test-cloud/README.md b/airbyte-integrations/connectors/source-e2e-test-cloud/README.md index a64de4a14072..bf16bd7df3ab 100644 --- a/airbyte-integrations/connectors/source-e2e-test-cloud/README.md +++ b/airbyte-integrations/connectors/source-e2e-test-cloud/README.md @@ -17,10 +17,11 @@ No credential is needed for this connector. #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:source-e2e-test-cloud:airbyteDocker +./gradlew :airbyte-integrations:connectors:source-e2e-test-cloud:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/source-e2e-test-cloud:dev`. the Dockerfile. #### Run @@ -60,8 +61,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -2. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -3. Create a Pull Request. -4. Pat yourself on the back for being an awesome contributor. -5. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-e2e-test-cloud test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/e2e-test.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-e2e-test-cloud/build.gradle b/airbyte-integrations/connectors/source-e2e-test-cloud/build.gradle index b843c8624dc8..e7f6abbb0d0b 100644 --- a/airbyte-integrations/connectors/source-e2e-test-cloud/build.gradle +++ b/airbyte-integrations/connectors/source-e2e-test-cloud/build.gradle @@ -1,22 +1,29 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.7.7' + features = ['db-sources'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.source.e2e_test.CloudTestingSources' } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') implementation project(':airbyte-integrations:connectors:source-e2e-test') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - - testImplementation project(":airbyte-json-validation") - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-e2e-test-cloud') } diff --git a/airbyte-integrations/connectors/source-e2e-test-cloud/metadata.yaml b/airbyte-integrations/connectors/source-e2e-test-cloud/metadata.yaml index c6eeee25e73c..9c9322556330 100644 --- a/airbyte-integrations/connectors/source-e2e-test-cloud/metadata.yaml +++ b/airbyte-integrations/connectors/source-e2e-test-cloud/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: 50bd8338-7c4e-46f1-8c7f-3ef95de19fdd - dockerImageTag: 2.1.0 + dockerImageTag: 2.2.0 dockerRepository: airbyte/source-e2e-test-cloud githubIssueLabel: source-e2e-test-cloud icon: airbyte.svg diff --git a/airbyte-integrations/connectors/source-e2e-test-cloud/src/main/java/io/airbyte/integrations/source/e2e_test/CloudTestingSources.java b/airbyte-integrations/connectors/source-e2e-test-cloud/src/main/java/io/airbyte/integrations/source/e2e_test/CloudTestingSources.java index 4f02bd8d4a67..e1a9afcfae83 100644 --- a/airbyte-integrations/connectors/source-e2e-test-cloud/src/main/java/io/airbyte/integrations/source/e2e_test/CloudTestingSources.java +++ b/airbyte-integrations/connectors/source-e2e-test-cloud/src/main/java/io/airbyte/integrations/source/e2e_test/CloudTestingSources.java @@ -7,10 +7,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.base.spec_modification.SpecModifyingSource; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.base.spec_modification.SpecModifyingSource; import io.airbyte.protocol.models.v0.ConnectorSpecification; /** diff --git a/airbyte-integrations/connectors/source-e2e-test-cloud/src/test-integration/java/io/airbyte/integrations/source/e2e_test/CloudTestingSourcesAcceptanceTest.java b/airbyte-integrations/connectors/source-e2e-test-cloud/src/test-integration/java/io/airbyte/integrations/source/e2e_test/CloudTestingSourcesAcceptanceTest.java index 4858487cf9c1..70e85032a0d4 100644 --- a/airbyte-integrations/connectors/source-e2e-test-cloud/src/test-integration/java/io/airbyte/integrations/source/e2e_test/CloudTestingSourcesAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-e2e-test-cloud/src/test-integration/java/io/airbyte/integrations/source/e2e_test/CloudTestingSourcesAcceptanceTest.java @@ -9,12 +9,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.integrations.source.e2e_test.ContinuousFeedConfig.MockCatalogType; import io.airbyte.integrations.source.e2e_test.TestingSources.TestingSourceType; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.CatalogHelpers; diff --git a/airbyte-integrations/connectors/source-e2e-test/.dockerignore b/airbyte-integrations/connectors/source-e2e-test/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-e2e-test/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-e2e-test/Dockerfile b/airbyte-integrations/connectors/source-e2e-test/Dockerfile deleted file mode 100644 index 6f8292891dc8..000000000000 --- a/airbyte-integrations/connectors/source-e2e-test/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-e2e-test - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-e2e-test - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=2.1.4 -LABEL io.airbyte.name=airbyte/source-e2e-test diff --git a/airbyte-integrations/connectors/source-e2e-test/README.md b/airbyte-integrations/connectors/source-e2e-test/README.md index 045bde10a19f..a75d586bbcf9 100644 --- a/airbyte-integrations/connectors/source-e2e-test/README.md +++ b/airbyte-integrations/connectors/source-e2e-test/README.md @@ -24,10 +24,11 @@ No credential is needed for this connector. #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:source-e2e-test:airbyteDocker +./gradlew :airbyte-integrations:connectors:source-e2e-test:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/source-e2e-test:dev`. the Dockerfile. #### Run @@ -67,8 +68,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -2. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -3. Create a Pull Request. -4. Pat yourself on the back for being an awesome contributor. -5. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-e2e-test test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/e2e-test.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-e2e-test/build.gradle b/airbyte-integrations/connectors/source-e2e-test/build.gradle index ba548f2a568d..d3f5e8df59d5 100644 --- a/airbyte-integrations/connectors/source-e2e-test/build.gradle +++ b/airbyte-integrations/connectors/source-e2e-test/build.gradle @@ -1,31 +1,39 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.7.7' + features = ['db-sources'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileTestJava { + options.compilerArgs.remove("-Werror") + } + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.source.e2e_test.TestingSources' } dependencies { - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-json-validation') implementation 'org.apache.commons:commons-lang3:3.11' implementation 'com.networknt:json-schema-validator:1.0.72' - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) // random Json object generation from Json schema // https://github.com/airbytehq/jsongenerator - implementation 'net.jimblackler.jsonschemafriend:core:0.11.2' + implementation 'net.jimblackler.jsonschemafriend:core:0.12.1' implementation 'org.mozilla:rhino-engine:1.7.14' - implementation group: 'com.github.airbytehq', name: 'jsongenerator', version: '1.0.1' - - testImplementation project(":airbyte-json-validation") - testImplementation project(':airbyte-test-utils') + implementation group: 'com.github.airbytehq', name: 'jsongenerator', version: '1.0.2' - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-e2e-test') - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-e2e-test/metadata.yaml b/airbyte-integrations/connectors/source-e2e-test/metadata.yaml index c695e3a3052f..eebd858fa99e 100644 --- a/airbyte-integrations/connectors/source-e2e-test/metadata.yaml +++ b/airbyte-integrations/connectors/source-e2e-test/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: d53f9084-fa6b-4a5a-976c-5b8392f4ad8a - dockerImageTag: 2.1.4 + dockerImageTag: 2.2.0 dockerRepository: airbyte/source-e2e-test githubIssueLabel: source-e2e-test icon: airbyte.svg diff --git a/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedSource.java b/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedSource.java index 176c7f795982..d5e1379e92ce 100644 --- a/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedSource.java +++ b/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedSource.java @@ -7,11 +7,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.AbstractIterator; import com.google.common.collect.Iterators; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.Source; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.Source; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; diff --git a/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/LegacyExceptionAfterNSource.java b/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/LegacyExceptionAfterNSource.java index 0c067e09653f..a2775f18a121 100644 --- a/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/LegacyExceptionAfterNSource.java +++ b/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/LegacyExceptionAfterNSource.java @@ -7,20 +7,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.AbstractIterator; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.Source; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.Source; -import io.airbyte.protocol.models.v0.AirbyteCatalog; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import io.airbyte.protocol.models.v0.*; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; -import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import io.airbyte.protocol.models.v0.AirbyteStateMessage; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.SyncMode; import java.time.Instant; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; @@ -78,7 +72,12 @@ protected AirbyteMessage computeNext() { hasEmittedStateAtCount.set(true); return new AirbyteMessage() .withType(Type.STATE) - .withState(new AirbyteStateMessage().withData(Jsons.jsonNode(ImmutableMap.of(LegacyConstants.DEFAULT_COLUMN, recordValue.get())))); + .withState(new AirbyteStateMessage() + .withType(AirbyteStateMessage.AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withName(LegacyConstants.DEFAULT_STREAM)) + .withStreamState(Jsons.jsonNode(ImmutableMap.of(LegacyConstants.DEFAULT_COLUMN, recordValue.get())))) + .withData(Jsons.jsonNode(ImmutableMap.of(LegacyConstants.DEFAULT_COLUMN, recordValue.get())))); } else if (throwAfterNRecords > recordsEmitted.get()) { recordsEmitted.incrementAndGet(); recordValue.incrementAndGet(); diff --git a/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/LegacyInfiniteFeedSource.java b/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/LegacyInfiniteFeedSource.java index 65ce7480c7b4..6d05f3719c51 100644 --- a/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/LegacyInfiniteFeedSource.java +++ b/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/LegacyInfiniteFeedSource.java @@ -9,11 +9,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.AbstractIterator; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.Source; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.Source; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; diff --git a/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/SpeedBenchmarkSource.java b/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/SpeedBenchmarkSource.java index d606980ce464..6098728851ab 100644 --- a/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/SpeedBenchmarkSource.java +++ b/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/SpeedBenchmarkSource.java @@ -5,10 +5,10 @@ package io.airbyte.integrations.source.e2e_test; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.Source; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.Source; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; diff --git a/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/TestingSources.java b/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/TestingSources.java index 42eac4bc7dcd..c3339c311984 100644 --- a/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/TestingSources.java +++ b/airbyte-integrations/connectors/source-e2e-test/src/main/java/io/airbyte/integrations/source/e2e_test/TestingSources.java @@ -6,10 +6,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; import io.airbyte.commons.util.AutoCloseableIterator; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/source-e2e-test/src/test-integration/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-e2e-test/src/test-integration/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedSourceAcceptanceTest.java index 257474d16742..a0aeeb393a30 100644 --- a/airbyte-integrations/connectors/source-e2e-test/src/test-integration/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-e2e-test/src/test-integration/java/io/airbyte/integrations/source/e2e_test/ContinuousFeedSourceAcceptanceTest.java @@ -9,12 +9,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.integrations.source.e2e_test.ContinuousFeedConfig.MockCatalogType; import io.airbyte.integrations.source.e2e_test.TestingSources.TestingSourceType; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.CatalogHelpers; diff --git a/airbyte-integrations/connectors/source-elasticsearch/.dockerignore b/airbyte-integrations/connectors/source-elasticsearch/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-elasticsearch/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-elasticsearch/Dockerfile b/airbyte-integrations/connectors/source-elasticsearch/Dockerfile deleted file mode 100644 index 4f8da9d4e7b7..000000000000 --- a/airbyte-integrations/connectors/source-elasticsearch/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-elasticsearch - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-elasticsearch - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.1 -LABEL io.airbyte.name=airbyte/source-elasticsearch diff --git a/airbyte-integrations/connectors/source-elasticsearch/README.md b/airbyte-integrations/connectors/source-elasticsearch/README.md index fbd9835d8656..5b12fbcb1a60 100644 --- a/airbyte-integrations/connectors/source-elasticsearch/README.md +++ b/airbyte-integrations/connectors/source-elasticsearch/README.md @@ -20,10 +20,11 @@ Credentials can be provided in three ways: #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:source-elasticsearch:airbyteDocker +./gradlew :airbyte-integrations:connectors:source-elasticsearch:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/source-elasticsearch:dev`. the Dockerfile. #### Run @@ -63,8 +64,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -2. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -3. Create a Pull Request. -4. Pat yourself on the back for being an awesome contributor. -5. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-elasticsearch test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/elasticsearch.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-elasticsearch/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-elasticsearch/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-elasticsearch/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-elasticsearch/build.gradle b/airbyte-integrations/connectors/source-elasticsearch/build.gradle index 8c6e81ef6d28..31f1cfca5fcc 100644 --- a/airbyte-integrations/connectors/source-elasticsearch/build.gradle +++ b/airbyte-integrations/connectors/source-elasticsearch/build.gradle @@ -1,20 +1,29 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' - id 'airbyte-connector-acceptance-test' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-sources'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.source.elasticsearch.ElasticsearchSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation 'co.elastic.clients:elasticsearch-java:7.15.0' implementation 'com.fasterxml.jackson.core:jackson-databind:2.12.3' @@ -30,20 +39,6 @@ dependencies { // MIT // https://www.testcontainers.org/ - testImplementation libs.connectors.testcontainers.elasticsearch - integrationTestJavaImplementation libs.connectors.testcontainers.elasticsearch - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-elasticsearch') -} - -repositories { - maven { - name = "ESSnapshots" - url = "https://snapshots.elastic.co/maven/" - } - maven { - name = "ESJavaGithubPackages" - url = "https://maven.pkg.github.com/elastic/elasticsearch-java" - } + testImplementation libs.testcontainers.elasticsearch + integrationTestJavaImplementation libs.testcontainers.elasticsearch } diff --git a/airbyte-integrations/connectors/source-elasticsearch/src/main/java/io/airbyte/integrations/source/elasticsearch/ElasticsearchSource.java b/airbyte-integrations/connectors/source-elasticsearch/src/main/java/io/airbyte/integrations/source/elasticsearch/ElasticsearchSource.java index 286c62c17f2c..62d49706f5fe 100644 --- a/airbyte-integrations/connectors/source-elasticsearch/src/main/java/io/airbyte/integrations/source/elasticsearch/ElasticsearchSource.java +++ b/airbyte-integrations/connectors/source-elasticsearch/src/main/java/io/airbyte/integrations/source/elasticsearch/ElasticsearchSource.java @@ -8,12 +8,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteTraceMessageUtility; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/source-elasticsearch/src/test-integration/java/io/airbyte/integrations/source/elasticsearch/ElasticsearchSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-elasticsearch/src/test-integration/java/io/airbyte/integrations/source/elasticsearch/ElasticsearchSourceAcceptanceTest.java index bbb021c7d6e7..980a71a9c016 100644 --- a/airbyte-integrations/connectors/source-elasticsearch/src/test-integration/java/io/airbyte/integrations/source/elasticsearch/ElasticsearchSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-elasticsearch/src/test-integration/java/io/airbyte/integrations/source/elasticsearch/ElasticsearchSourceAcceptanceTest.java @@ -6,11 +6,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConnectorSpecification; import java.io.IOException; diff --git a/airbyte-integrations/connectors/source-emailoctopus/README.md b/airbyte-integrations/connectors/source-emailoctopus/README.md index 67ae2b7a7180..505de7529779 100644 --- a/airbyte-integrations/connectors/source-emailoctopus/README.md +++ b/airbyte-integrations/connectors/source-emailoctopus/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-emailoctopus:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/emailoctopus) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_emailoctopus/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-emailoctopus:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-emailoctopus build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-emailoctopus:airbyteDocker +An image will be built with the tag `airbyte/source-emailoctopus:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-emailoctopus:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-emailoctopus:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-emailoctopus:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-emailoctopus:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-emailoctopus test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-emailoctopus:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-emailoctopus:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. \ No newline at end of file +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-emailoctopus test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/emailoctopus.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-emailoctopus/acceptance-test-config.yml b/airbyte-integrations/connectors/source-emailoctopus/acceptance-test-config.yml index fe9c29c8fd88..171dae669138 100644 --- a/airbyte-integrations/connectors/source-emailoctopus/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-emailoctopus/acceptance-test-config.yml @@ -19,9 +19,9 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - incremental: + incremental: bypass_reason: "This connector does not implement incremental sync" full_refresh: tests: - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" \ No newline at end of file + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-emailoctopus/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-emailoctopus/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-emailoctopus/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-emailoctopus/build.gradle b/airbyte-integrations/connectors/source-emailoctopus/build.gradle deleted file mode 100644 index 81ac7a95f987..000000000000 --- a/airbyte-integrations/connectors/source-emailoctopus/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_emailoctopus' -} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-everhour/README.md b/airbyte-integrations/connectors/source-everhour/README.md index 2a03a5f160a0..c33ef8dcf44a 100644 --- a/airbyte-integrations/connectors/source-everhour/README.md +++ b/airbyte-integrations/connectors/source-everhour/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-everhour:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/everhour) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_everhour/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-everhour:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-everhour build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-everhour:airbyteDocker +An image will be built with the tag `airbyte/source-everhour:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-everhour:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-everhour:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-everhour:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-everhour:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-everhour test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-everhour:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-everhour:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-everhour test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/everhour.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-everhour/acceptance-test-config.yml b/airbyte-integrations/connectors/source-everhour/acceptance-test-config.yml index aa77e3c2697c..b98116dc2e8a 100644 --- a/airbyte-integrations/connectors/source-everhour/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-everhour/acceptance-test-config.yml @@ -19,7 +19,7 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - incremental: + incremental: bypass_reason: "This connector does not implement incremental sync" full_refresh: tests: diff --git a/airbyte-integrations/connectors/source-everhour/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-everhour/acceptance-test-docker.sh deleted file mode 100644 index c51577d10690..000000000000 --- a/airbyte-integrations/connectors/source-everhour/acceptance-test-docker.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env sh - -# Build latest connector image -docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) - -# Pull latest acctest image -docker pull airbyte/source-acceptance-test:latest - -# Run -docker run --rm -it \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /tmp:/tmp \ - -v $(pwd):/test_input \ - airbyte/source-acceptance-test \ - --acceptance-test-config /test_input - diff --git a/airbyte-integrations/connectors/source-everhour/build.gradle b/airbyte-integrations/connectors/source-everhour/build.gradle deleted file mode 100644 index 2698346a9377..000000000000 --- a/airbyte-integrations/connectors/source-everhour/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_everhour' -} diff --git a/airbyte-integrations/connectors/source-exchange-rates/README.md b/airbyte-integrations/connectors/source-exchange-rates/README.md index 78969eda4723..522c69925267 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/README.md +++ b/airbyte-integrations/connectors/source-exchange-rates/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-exchange-rates:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/exchange-rates) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_exchange_rates/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-exchange-rates:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-exchange-rates build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-exchange-rates:airbyteDocker +An image will be built with the tag `airbyte/source-exchange-rates:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-exchange-rates:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-exchange-rates:dev che docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-exchange-rates:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-exchange-rates:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-exchange-rates test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-exchange-rates:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-exchange-rates:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-exchange-rates test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/exchange-rates.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-exchange-rates/acceptance-test-config.yml b/airbyte-integrations/connectors/source-exchange-rates/acceptance-test-config.yml index 325b1b1ea837..e0005bd9749a 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-exchange-rates/acceptance-test-config.yml @@ -21,13 +21,13 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - incremental: + incremental: # bypass_reason: "This connector does not implement incremental sync" - tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-exchange-rates/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-exchange-rates/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-exchange-rates/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-exchange-rates/build.gradle b/airbyte-integrations/connectors/source-exchange-rates/build.gradle deleted file mode 100644 index 25b4d6b6a1b6..000000000000 --- a/airbyte-integrations/connectors/source-exchange-rates/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_exchange_rates' -} diff --git a/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml b/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml index dc7062891b8f..20d3fa38cb7c 100644 --- a/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml +++ b/airbyte-integrations/connectors/source-exchange-rates/metadata.yaml @@ -22,5 +22,5 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/exchange-rates tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile deleted file mode 100644 index a7647a8e0db0..000000000000 --- a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY source_facebook_marketing ./source_facebook_marketing -COPY main.py ./ -COPY setup.py ./ -RUN pip install . - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - - -LABEL io.airbyte.version=1.1.7 -LABEL io.airbyte.name=airbyte/source-facebook-marketing diff --git a/airbyte-integrations/connectors/source-facebook-marketing/README.md b/airbyte-integrations/connectors/source-facebook-marketing/README.md index a4cb3969608e..1e31681a3fb2 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/README.md +++ b/airbyte-integrations/connectors/source-facebook-marketing/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-facebook-marketing:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/facebook-marketing) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_facebook_marketing/spec.json` file. @@ -54,19 +48,70 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-facebook-marketing:dev + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-facebook-marketing build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-facebook-marketing:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations -You can also build the connector image via Gradle: +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-facebook-marketing:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-facebook-marketing:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-facebook-marketing:dev . +# Running the spec command against your patched connector +docker run airbyte/source-facebook-marketing:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -75,45 +120,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-facebook-marketing:dev docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-facebook-marketing:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-facebook-marketing:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .'[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-facebook-marketing test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-facebook-marketing:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unittest run: -``` -./gradlew :airbyte-integrations:connectors:source-facebook-marketing:unitTest -``` -To run acceptance and custom integration tests run: -``` -./gradlew :airbyte-integrations:connectors:source-facebook-marketing:IntegrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -123,8 +139,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-facebook-marketing test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/facebook-marketing.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml b/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml index 6a93f0f9ecbc..5015ff90f970 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-config.yml @@ -7,18 +7,22 @@ acceptance_tests: tests: - spec_path: "integration_tests/spec.json" backward_compatibility_tests_config: - disable_for_version: "0.5.0" + disable_for_version: "1.2.2" + previous_connector_version: "1.2.1" connection: tests: - config_path: "secrets/config.json" status: "succeed" + - config_path: "secrets/config_no_date.json" + status: "succeed" - config_path: "integration_tests/invalid_config.json" status: "failed" discovery: tests: - config_path: "secrets/config.json" backward_compatibility_tests_config: - disable_for_version: "0.5.0" + disable_for_version: "1.2.2" + previous_connector_version: "1.2.1" basic_read: tests: - config_path: "secrets/config.json" @@ -46,6 +50,35 @@ acceptance_tests: ads_insights_dma: - name: cost_per_estimated_ad_recallers bypass_reason: can be missing + ads_insights_age_and_gender: + - name: cost_per_estimated_ad_recallers + bypass_reason: can be missing + ads_insights_delivery_device: + - name: cost_per_estimated_ad_recallers + bypass_reason: can be missing + ads_insights_delivery_platform_and_device_platform: + - name: cost_per_estimated_ad_recallers + bypass_reason: can be missing + ads_insights_demographics_age: + - name: cost_per_estimated_ad_recallers + bypass_reason: can be missing + ads_insights_demographics_country: + - name: cost_per_estimated_ad_recallers + bypass_reason: can be missing + ads_insights_demographics_gender: + - name: cost_per_estimated_ad_recallers + bypass_reason: can be missing + ads_insights_platform_and_device: + - name: cost_per_estimated_ad_recallers + bypass_reason: can be missing + ads_insights_region: + - name: cost_per_estimated_ad_recallers + bypass_reason: can be missing + custom_audiences: + - name: approximate_count_lower_bound + bypass_reason: is changeable + - name: approximate_count_upper_bound + bypass_reason: is changeable empty_streams: - name: "ads_insights_action_product_id" bypass_reason: "Data not permanent" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-facebook-marketing/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/build.gradle b/airbyte-integrations/connectors/source-facebook-marketing/build.gradle deleted file mode 100644 index 1587459504ea..000000000000 --- a/airbyte-integrations/connectors/source-facebook-marketing/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_facebook_marketing' -} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/conftest.py b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/conftest.py index f58ee7224089..b2e2c416bc3c 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/conftest.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/conftest.py @@ -6,12 +6,15 @@ import json import pytest +from source_facebook_marketing.config_migrations import MigrateAccountIdToArray @pytest.fixture(scope="session", name="config") def config_fixture(): with open("secrets/config.json", "r") as config_file: - return json.load(config_file) + config = json.load(config_file) + migrated_config = MigrateAccountIdToArray.transform(config) + return migrated_config @pytest.fixture(scope="session", name="config_with_wrong_token") @@ -21,7 +24,7 @@ def config_with_wrong_token_fixture(config): @pytest.fixture(scope="session", name="config_with_wrong_account") def config_with_wrong_account_fixture(config): - return {**config, "account_id": "WRONG_ACCOUNT"} + return {**config, "account_ids": ["WRONG_ACCOUNT"]} @pytest.fixture(scope="session", name="config_with_include_deleted") diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/expected_records.jsonl index a75f785fadce..92fc12d2f710 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/expected_records.jsonl @@ -1,28 +1,29 @@ -{"stream":"ad_account","data":{"id":"act_212551616838260","account_id":"212551616838260","account_status":1,"age":1219.3692361111,"amount_spent":"39125","balance":"0","business":{"id":"1506473679510495","name":"Airbyte"},"business_city":"","business_country_code":"US","business_name":"","business_street":"","business_street2":"","can_create_brand_lift_study":false,"capabilities":["CAN_CREATE_CALL_ADS","CAN_SEE_GROWTH_OPPORTUNITY_DATA","ENABLE_IA_RECIRC_AD_DISPLAY_FORMAT","CAN_USE_MOBILE_EXTERNAL_PAGE_TYPE","CAN_USE_FB_FEED_POSITION_IN_VIDEO_VIEW_15S","ENABLE_BIZ_DISCO_ADS","ENABLE_BRAND_OBJECTIVES_FOR_BIZ_DISCO_ADS","ENABLE_DIRECT_REACH_FOR_BIZ_DISCO_ADS","ENABLE_DYNAMIC_ADS_ON_IG_STORIES_ADS","ENABLE_IG_STORIES_ADS_PPE_OBJECTIVE","ENABLE_IG_STORIES_ADS_MESSENGER_DESTINATION","ENABLE_PAC_FOR_BIZ_DISCO_ADS","CAN_USE_FB_INSTREAM_POSITION_IN_VIDEO_VIEW_15S","CAN_USE_FB_STORY_POSITION_IN_VIDEO_VIEW_15S","CAN_USE_AN_INSTREAM_POSITION_IN_VIDEO_VIEW_15S","CAN_USE_IG_STORY_POSITION_IN_VIDEO_VIEW_15S","CAN_USE_FB_IA_POSITION_IN_VIDEO_VIEW_15S","CAN_USE_FB_SUG_VIDEO_POSITION_IN_VIDEO_VIEW_15S","CAN_USE_FB_MKT_PLACE_POSITION_IN_VIDEO_VIEW_15S","CAN_USE_IG_FEED_POSITION_IN_VIDEO_VIEW_15S","CAN_USE_IG_EXPLORE_POSITION_IN_VIDEO_VIEW_15S","CAN_USE_AN_CLASSIC_POSITION_IN_VIDEO_VIEW_15S","CAN_USE_AN_REWARD_VIDEO_POSITION_IN_VIDEO_VIEW_15S","CAN_USE_RECURRING_BUDGET","HAS_VALID_PAYMENT_METHODS","CAN_USE_LINK_CLICK_BILLING_EVENT","CAN_USE_CPA_BILLING_EVENT","CAN_SEE_NEW_CONVERSION_WINDOW_NUX","ADS_INSTREAM_INTERFACE_INTEGRITY","ADS_INSTREAM_LINK_CLICK","ADS_INSTREAM_LINK_CLICK_IMAGE","ADS_IN_OBJECTIVES_DEPRECATION","MESSENGER_INBOX_ADS_PRODUCT_CATALOG_SALES","CAN_SHOW_MESSENGER_DUPLICSTION_UPSELL","ALLOW_INSTREAM_ONLY_FOR_REACH","ADS_INSTREAM_VIDEO_PLACEMENT_CONVERSIONS","CAN_CREATE_INSTAGRAM_EXPLORE_ADS","ALLOW_INSTREAM_VIDEOS_PLACEMENT_ONLY","ALLOW_INSTREAM_NON_INTERRUPTIVE_LEADGEN","INSTREAM_VIDEO_AD_DESKTOP_CONVERSION_AD_PREVIEW","ALLOW_INSTREAM_ONLY_FOR_BRAND_AWARENESS_AUCTION","ALLOW_SUGGESTED_VIDEOS_PLACEMENT_ONLY","WHATSAPP_DESTINATION_ADS","CTM_ADS_CREATION_CLICK_TO_DIRECT","CTW_ADS_ENABLE_IG_FEED_PLACEMENT","CTW_ADS_FOR_NON_MESSAGES_OBJECTIVE","CTW_ADS_TRUSTED_TIER_2_PLUS_ADVERTISER","CTW_ADS_TRUSTED_TIER_ADVERTISER","ADS_PLACEMENT_MARKETPLACE","CAN_CHANGE_BILLING_THRESHOLD","CAN_USE_APP_EVENT_AVERAGE_COST_BIDDING","CAN_USE_LEAD_GEN_AVERAGE_COST_BIDDING","ADS_VALUE_OPTIMIZATION_DYNAMIC_ADS_1D","ADS_DELIVERY_INSIGHTS_IN_BIDDING_PRESET_EXPERIMENT","ADS_DELIVERY_INSIGHTS_OPTIMIZATION_PRESET","CAN_SEE_APP_AD_EVENTS","CAN_SEE_NEW_STANDARD_EVENTS_BETA","CAN_SEE_VCK_HOLIDAY_TEMPLATES","ENABLE_DCO_FOR_FB_STORY_ADS","CAN_USE_IG_EXPLORE_GRID_HOME_PLACEMENT","CAN_USE_IG_EXPLORE_HOME_IN_REACH_AND_FREQUENCY","CAN_USE_IG_EXPLORE_HOME_POST_ENGAGEMENT_MESSAGES","CAN_USE_IG_SEARCH_PLACEMENT","CAN_USE_IG_SEARCH_GRID_ADS","CAN_USE_IG_REELS_PAC_CAROUSEL","CAN_USE_IG_REELS_POSITION","CAN_SEE_CONVERSION_LIFT_SUMMARY","CAN_USE_IG_PROFILE_FEED_POSITION","CAN_USE_IG_PROFILE_FEED_AUTO_PLACEMENT","CAN_USE_IG_PROFILE_FEED_DPA_CREATION","CAN_USE_IG_REELS_AUTO_PLACEMENT","CAN_USE_IG_REELS_REACH_AND_FREQUENCY","CAN_USE_IG_REELS_OVERLAY_POSITION","CAN_USE_IG_REELS_OVERLAY_AUTO_PLACEMENT","CAN_USE_IG_REELS_OVERLAY_PAC","CAN_USE_IG_SHOP_TAB_PAC","CAN_SEE_LEARNING_STAGE","ENABLE_WEBSITE_CONVERSIONS_FOR_FB_STORY_ADS","ENABLE_MESSENGER_INBOX_VIDEO_ADS","ENABLE_VIDEO_VIEWS_FOR_FB_STORY_ADS","ENABLE_LINK_CLICKS_FOR_FB_STORY_ADS","ENABLE_REACH_FOR_FB_STORY_ADS","CAN_USE_CALL_TO_ACTION_LINK_IMPORT_EXPORT","ADS_INSTREAM_VIDEO_ENABLE_SLIDE_SHOW","ALLOW_INSTREAM_VIDEOS_PLACEMENT_ONLY_IN_VV_REACH_AND_FREQUENCY","ENABLE_MOBILE_APP_INSTALLS_FOR_FB_STORY_ADS","ENABLE_LEAD_GEN_FOR_FB_STORY_ADS","CAN_USE_FB_MKT_PLACE_POSITION_IN_REACH","CAN_USE_FB_MKT_PLACE_POSITION_IN_VIDEO_VIEW","CAN_USE_FB_MKT_PLACE_POSITION_IN_STORE_VISIT","ENABLE_MOBILE_APP_ENGAGEMENT_FOR_FB_STORY_ADS","CAN_USE_FB_MKT_PLACE_POSITION_IN_BRAND_AWARENESS","CAN_USE_FB_MKT_PLACE_POSITION_IN_APP_INSTALLS","CAN_USE_FB_MKT_PLACE_POSITION_IN_LEAD_GENERATION","CAN_USE_FB_MKT_PLACE_POSITION_IN_MESSAGE","CAN_USE_FB_MKT_PLACE_POSITION_IN_PAGE_LIKE","CAN_USE_FB_MKT_PLACE_POSITION_IN_POST_ENGAGEMENT","RF_ALLOW_MARKETPLACE_ACCOUNT","RF_ALLOW_SEARCH_ACCOUNT","VERTICAL_VIDEO_PAC_INSTREAM_UPSELL","IX_COLLECTION_ENABLED_FOR_BAO_AND_REACH","ADS_BM_REQUIREMENTS_OCT_15_RELEASE","ENABLE_POST_ENGAGEMENT_FOR_FB_STORY","ENBABLE_CATALOG_SALES_FOR_FB_STORY","CAN_USE_WHATSAPP_DESTINATION_ON_LINK_CLICKS_AND_CONVERSIONS","CAN_USE_WHATSAPP_DESTINATION_ON_CONVERSIONS","IS_NON_TAIL_AD_ACCOUNT","IS_IN_DSA_GK","IS_IN_IG_EXISTING_POST_CTA_DEFAULTING_EXPERIMENT","IS_IN_SHORT_WA_LINK_CTWA_UNCONV_TRAFFIC_EXPERIMENT","IS_IN_ODAX_EXPERIENCE","IS_IN_REACH_BRAND_AWARENESS_WHATSAPP_L1_DESTINATION_EXPERIMENT","IS_IN_VIDEO_VIEWS_WHATSAPP_L1_DESTINATION_EXPERIMENT","IS_IN_WHATSAPP_DESTINATION_DEFAULTING_EXPERIMENT","CAN_USE_MARKETPLACE_DESKTOP","ADS_MERCHANT_OVERLAYS_DEPRECATION","CONNECTIONS_DEPRECATION_V2","CAN_USE_LIVE_VIDEO_FOR_THRUPLAY","CAN_SEE_HEC_AM_FLOW","CAN_SEE_POLITICAL_FLOW","ADS_INSTREAM_PLACEMENT_CATALOG_SALES","ENABLE_CONVERSIONS_FOR_FB_GROUP_TAB_ADS","ENABLE_LINK_CLICK_FOR_FB_GROUP_TAB_ADS","ENABLE_REACH_FOR_FB_GROUP_TAB_ADS","CAN_USE_CONVERSATIONS_OPTIMIZATION","ENABLE_THRUPLAY_OPTIMIZATION_MESSENGER_STORY_ADS","CAN_USE_IG_STORY_POLLS_PAC_CREATION","IOS14_CEO_CAMPAIGN_CREATION","ENABLE_VIDEO_CHANNEL_PLACEMENT_FOR_RSVP_ADS","DIGITAL_CIRCULAR_ADS","CAN_SEE_SAFR_V3_FLOW","CAN_USE_FB_REELS_POSITION","CAN_USE_ADS_ON_FB_REELS_POSITION","CAN_USE_FB_REELS_AUTO_PLACEMENT","ENABLE_FB_REELS_CREATION_PAC_ADS","ENABLE_FB_REELS_CREATION_DCO_ADS","ENABLE_FB_REELS_POSTLOOP_CREATION_DCO_ADS","ENABLE_FB_REELS_POSTLOOP_CREATION_PAC_ADS","RF_CPA_BILLING_DEPRECATION_PHASE_2","ENABLE_APP_INSTALL_CUSTOM_PRODUCT_PAGES","ENABLE_ADS_ON_FB_REELS_PLACEMENT_UNIFICATION","ENABLE_ADS_ON_FB_REELS_PLACEMENT_UNIFICATION_L2_NUX","ENABLE_ADS_ON_IG_SHOP_TAB_DEPRECATION_L2_NUX","ADS_RF_FB_REELS_PLACEMENT","ENABLE_ADS_ON_FB_INSTANT_ARTICLE_DEPRECATION_L2_NUX","REELS_DM_ADS_ENABLE_REACH_AND_FREQUENCY"],"created_time":"2020-04-13T18:04:59-0700","currency":"USD","disable_reason":0,"end_advertiser":1506473679510495,"end_advertiser_name":"Airbyte","fb_entity":85,"has_migrated_permissions":true,"is_attribution_spec_system_default":true,"is_direct_deals_enabled":false,"is_in_3ds_authorization_enabled_market":false,"is_notifications_enabled":true,"is_personal":0,"is_tax_id_required":false,"min_campaign_group_spend_cap":10000,"min_daily_budget":100,"name":"Airbyte","offsite_pixels_tos_accepted":true,"owner":1506473679510495,"rf_spec":{"min_reach_limits":{"US":200000,"CA":200000,"GB":200000,"AR":200000,"AU":200000,"AT":200000,"BE":200000,"BR":200000,"CL":200000,"CN":200000,"CO":200000,"HR":200000,"DK":200000,"DO":200000,"EG":200000,"FI":200000,"FR":200000,"DE":200000,"GR":200000,"HK":200000,"IN":200000,"ID":200000,"IE":200000,"IL":200000,"IT":200000,"JP":200000,"JO":200000,"KW":200000,"LB":200000,"MY":200000,"MX":200000,"NL":200000,"NZ":200000,"NG":200000,"NO":200000,"PK":200000,"PA":200000,"PE":200000,"PH":200000,"PL":200000,"RU":200000,"SA":200000,"RS":200000,"SG":200000,"ZA":200000,"KR":200000,"ES":200000,"SE":200000,"CH":200000,"TW":200000,"TH":200000,"TR":200000,"AE":200000,"VE":200000,"PT":200000,"LU":200000,"BG":200000,"CZ":200000,"SI":200000,"IS":200000,"SK":200000,"LT":200000,"TT":200000,"BD":200000,"LK":200000,"KE":200000,"HU":200000,"MA":200000,"CY":200000,"JM":200000,"EC":200000,"RO":200000,"BO":200000,"GT":200000,"CR":200000,"QA":200000,"SV":200000,"HN":200000,"NI":200000,"PY":200000,"UY":200000,"PR":200000,"BA":200000,"PS":200000,"TN":200000,"BH":200000,"VN":200000,"GH":200000,"MU":200000,"UA":200000,"MT":200000,"BS":200000,"MV":200000,"OM":200000,"MK":200000,"LV":200000,"EE":200000,"IQ":200000,"DZ":200000,"AL":200000,"NP":200000,"MO":200000,"ME":200000,"SN":200000,"GE":200000,"BN":200000,"UG":200000,"GP":200000,"BB":200000,"AZ":200000,"TZ":200000,"LY":200000,"MQ":200000,"CM":200000,"BW":200000,"ET":200000,"KZ":200000,"NA":200000,"MG":200000,"NC":200000,"MD":200000,"FJ":200000,"BY":200000,"JE":200000,"GU":200000,"YE":200000,"ZM":200000,"IM":200000,"HT":200000,"KH":200000,"AW":200000,"PF":200000,"AF":200000,"BM":200000,"GY":200000,"AM":200000,"MW":200000,"AG":200000,"RW":200000,"GG":200000,"GM":200000,"FO":200000,"LC":200000,"KY":200000,"BJ":200000,"AD":200000,"GD":200000,"VI":200000,"BZ":200000,"VC":200000,"MN":200000,"MZ":200000,"ML":200000,"AO":200000,"GF":200000,"UZ":200000,"DJ":200000,"BF":200000,"MC":200000,"TG":200000,"GL":200000,"GA":200000,"GI":200000,"CD":200000,"KG":200000,"PG":200000,"BT":200000,"KN":200000,"SZ":200000,"LS":200000,"LA":200000,"LI":200000,"MP":200000,"SR":200000,"SC":200000,"VG":200000,"TC":200000,"DM":200000,"MR":200000,"AX":200000,"SM":200000,"SL":200000,"NE":200000,"CG":200000,"AI":200000,"YT":200000,"CV":200000,"GN":200000,"TM":200000,"BI":200000,"TJ":200000,"VU":200000,"SB":200000,"ER":200000,"WS":200000,"AS":200000,"FK":200000,"GQ":200000,"TO":200000,"KM":200000,"PW":200000,"FM":200000,"CF":200000,"SO":200000,"MH":200000,"VA":200000,"TD":200000,"KI":200000,"ST":200000,"TV":200000,"NR":200000,"RE":200000,"LR":200000,"ZW":200000,"CI":200000,"MM":200000,"AN":200000,"AQ":200000,"BQ":200000,"BV":200000,"IO":200000,"CX":200000,"CC":200000,"CK":200000,"CW":200000,"TF":200000,"GW":200000,"HM":200000,"XK":200000,"MS":200000,"NU":200000,"NF":200000,"PN":200000,"BL":200000,"SH":200000,"MF":200000,"PM":200000,"SX":200000,"GS":200000,"SS":200000,"SJ":200000,"TL":200000,"TK":200000,"UM":200000,"WF":200000,"EH":200000},"countries":["US","CA","GB","AR","AU","AT","BE","BR","CL","CN","CO","HR","DK","DO","EG","FI","FR","DE","GR","HK","IN","ID","IE","IL","IT","JP","JO","KW","LB","MY","MX","NL","NZ","NG","NO","PK","PA","PE","PH","PL","RU","SA","RS","SG","ZA","KR","ES","SE","CH","TW","TH","TR","AE","VE","PT","LU","BG","CZ","SI","IS","SK","LT","TT","BD","LK","KE","HU","MA","CY","JM","EC","RO","BO","GT","CR","QA","SV","HN","NI","PY","UY","PR","BA","PS","TN","BH","VN","GH","MU","UA","MT","BS","MV","OM","MK","EE","LV","IQ","DZ","AL","NP","MO","ME","SN","GE","BN","UG","GP","BB","ZW","CI","AZ","TZ","LY","MQ","MM","CM","BW","ET","KZ","NA","MG","NC","MD","FJ","BY","JE","GU","YE","ZM","IM","HT","KH","AW","PF","AF","BM","GY","AM","MW","AG","RW","GG","GM","FO","LC","KY","BJ","AD","GD","VI","BZ","VC","MN","MZ","ML","AO","GF","UZ","DJ","BF","MC","TG","GL","GA","GI","CD","KG","PG","BT","KN","SZ","LS","LA","LI","MP","SR","SC","VG","TC","DM","MR","AX","SM","SL","NE","CG","AI","YT","LR","CV","GN","TM","BI","TJ","VU","SB","ER","WS","AS","FK","GQ","TO","KM","PW","FM","CF","SO","MH","VA","TD","KI","ST","TV","NR","RE","AN","AQ","BQ","BV","IO","CX","CC","CK","CW","TF","GW","HM","XK","MS","NU","NF","PN","BL","SH","MF","PM","SX","GS","SS","SJ","TL","TK","UM","WF","EH"],"min_campaign_duration":{"US":1,"CA":1,"GB":1,"AR":1,"AU":1,"AT":1,"BE":1,"BR":1,"CL":1,"CN":1,"CO":1,"HR":1,"DK":1,"DO":1,"EG":1,"FI":1,"FR":1,"DE":1,"GR":1,"HK":1,"IN":1,"ID":1,"IE":1,"IL":1,"IT":1,"JP":1,"JO":1,"KW":1,"LB":1,"MY":1,"MX":1,"NL":1,"NZ":1,"NG":1,"NO":1,"PK":1,"PA":1,"PE":1,"PH":1,"PL":1,"RU":1,"SA":1,"RS":1,"SG":1,"ZA":1,"KR":1,"ES":1,"SE":1,"CH":1,"TW":1,"TH":1,"TR":1,"AE":1,"VE":1,"PT":1,"LU":1,"BG":1,"CZ":1,"SI":1,"IS":1,"SK":1,"LT":1,"TT":1,"BD":1,"LK":1,"KE":1,"HU":1,"MA":1,"CY":1,"JM":1,"EC":1,"RO":1,"BO":1,"GT":1,"CR":1,"QA":1,"SV":1,"HN":1,"NI":1,"PY":1,"UY":1,"PR":1,"BA":1,"PS":1,"TN":1,"BH":1,"VN":1,"GH":1,"MU":1,"UA":1,"MT":1,"BS":1,"MV":1,"OM":1,"MK":1,"LV":1,"EE":1,"IQ":1,"DZ":1,"AL":1,"NP":1,"MO":1,"ME":1,"SN":1,"GE":1,"BN":1,"UG":1,"GP":1,"BB":1,"AZ":1,"TZ":1,"LY":1,"MQ":1,"CM":1,"BW":1,"ET":1,"KZ":1,"NA":1,"MG":1,"NC":1,"MD":1,"FJ":1,"BY":1,"JE":1,"GU":1,"YE":1,"ZM":1,"IM":1,"HT":1,"KH":1,"AW":1,"PF":1,"AF":1,"BM":1,"GY":1,"AM":1,"MW":1,"AG":1,"RW":1,"GG":1,"GM":1,"FO":1,"LC":1,"KY":1,"BJ":1,"AD":1,"GD":1,"VI":1,"BZ":1,"VC":1,"MN":1,"MZ":1,"ML":1,"AO":1,"GF":1,"UZ":1,"DJ":1,"BF":1,"MC":1,"TG":1,"GL":1,"GA":1,"GI":1,"CD":1,"KG":1,"PG":1,"BT":1,"KN":1,"SZ":1,"LS":1,"LA":1,"LI":1,"MP":1,"SR":1,"SC":1,"VG":1,"TC":1,"DM":1,"MR":1,"AX":1,"SM":1,"SL":1,"NE":1,"CG":1,"AI":1,"YT":1,"CV":1,"GN":1,"TM":1,"BI":1,"TJ":1,"VU":1,"SB":1,"ER":1,"WS":1,"AS":1,"FK":1,"GQ":1,"TO":1,"KM":1,"PW":1,"FM":1,"CF":1,"SO":1,"MH":1,"VA":1,"TD":1,"KI":1,"ST":1,"TV":1,"NR":1,"RE":1,"LR":1,"ZW":1,"CI":1,"MM":1,"AN":1,"AQ":1,"BQ":1,"BV":1,"IO":1,"CX":1,"CC":1,"CK":1,"CW":1,"TF":1,"GW":1,"HM":1,"XK":1,"MS":1,"NU":1,"NF":1,"PN":1,"BL":1,"SH":1,"MF":1,"PM":1,"SX":1,"GS":1,"SS":1,"SJ":1,"TL":1,"TK":1,"UM":1,"WF":1,"EH":1},"max_campaign_duration":{"US":90,"CA":90,"GB":90,"AR":90,"AU":90,"AT":90,"BE":90,"BR":90,"CL":90,"CN":90,"CO":90,"HR":90,"DK":90,"DO":90,"EG":90,"FI":90,"FR":90,"DE":90,"GR":90,"HK":90,"IN":90,"ID":90,"IE":90,"IL":90,"IT":90,"JP":90,"JO":90,"KW":90,"LB":90,"MY":90,"MX":90,"NL":90,"NZ":90,"NG":90,"NO":90,"PK":90,"PA":90,"PE":90,"PH":90,"PL":90,"RU":90,"SA":90,"RS":90,"SG":90,"ZA":90,"KR":90,"ES":90,"SE":90,"CH":90,"TW":90,"TH":90,"TR":90,"AE":90,"VE":90,"PT":90,"LU":90,"BG":90,"CZ":90,"SI":90,"IS":90,"SK":90,"LT":90,"TT":90,"BD":90,"LK":90,"KE":90,"HU":90,"MA":90,"CY":90,"JM":90,"EC":90,"RO":90,"BO":90,"GT":90,"CR":90,"QA":90,"SV":90,"HN":90,"NI":90,"PY":90,"UY":90,"PR":90,"BA":90,"PS":90,"TN":90,"BH":90,"VN":90,"GH":90,"MU":90,"UA":90,"MT":90,"BS":90,"MV":90,"OM":90,"MK":90,"LV":90,"EE":90,"IQ":90,"DZ":90,"AL":90,"NP":90,"MO":90,"ME":90,"SN":90,"GE":90,"BN":90,"UG":90,"GP":90,"BB":90,"AZ":90,"TZ":90,"LY":90,"MQ":90,"CM":90,"BW":90,"ET":90,"KZ":90,"NA":90,"MG":90,"NC":90,"MD":90,"FJ":90,"BY":90,"JE":90,"GU":90,"YE":90,"ZM":90,"IM":90,"HT":90,"KH":90,"AW":90,"PF":90,"AF":90,"BM":90,"GY":90,"AM":90,"MW":90,"AG":90,"RW":90,"GG":90,"GM":90,"FO":90,"LC":90,"KY":90,"BJ":90,"AD":90,"GD":90,"VI":90,"BZ":90,"VC":90,"MN":90,"MZ":90,"ML":90,"AO":90,"GF":90,"UZ":90,"DJ":90,"BF":90,"MC":90,"TG":90,"GL":90,"GA":90,"GI":90,"CD":90,"KG":90,"PG":90,"BT":90,"KN":90,"SZ":90,"LS":90,"LA":90,"LI":90,"MP":90,"SR":90,"SC":90,"VG":90,"TC":90,"DM":90,"MR":90,"AX":90,"SM":90,"SL":90,"NE":90,"CG":90,"AI":90,"YT":90,"CV":90,"GN":90,"TM":90,"BI":90,"TJ":90,"VU":90,"SB":90,"ER":90,"WS":90,"AS":90,"FK":90,"GQ":90,"TO":90,"KM":90,"PW":90,"FM":90,"CF":90,"SO":90,"MH":90,"VA":90,"TD":90,"KI":90,"ST":90,"TV":90,"NR":90,"RE":90,"LR":90,"ZW":90,"CI":90,"MM":90,"AN":90,"AQ":90,"BQ":90,"BV":90,"IO":90,"CX":90,"CC":90,"CK":90,"CW":90,"TF":90,"GW":90,"HM":90,"XK":90,"MS":90,"NU":90,"NF":90,"PN":90,"BL":90,"SH":90,"MF":90,"PM":90,"SX":90,"GS":90,"SS":90,"SJ":90,"TL":90,"TK":90,"UM":90,"WF":90,"EH":90},"max_days_to_finish":{"US":180,"CA":180,"GB":180,"AR":180,"AU":180,"AT":180,"BE":180,"BR":180,"CL":180,"CN":180,"CO":180,"HR":180,"DK":180,"DO":180,"EG":180,"FI":180,"FR":180,"DE":180,"GR":180,"HK":180,"IN":180,"ID":180,"IE":180,"IL":180,"IT":180,"JP":180,"JO":180,"KW":180,"LB":180,"MY":180,"MX":180,"NL":180,"NZ":180,"NG":180,"NO":180,"PK":180,"PA":180,"PE":180,"PH":180,"PL":180,"RU":180,"SA":180,"RS":180,"SG":180,"ZA":180,"KR":180,"ES":180,"SE":180,"CH":180,"TW":180,"TH":180,"TR":180,"AE":180,"VE":180,"PT":180,"LU":180,"BG":180,"CZ":180,"SI":180,"IS":180,"SK":180,"LT":180,"TT":180,"BD":180,"LK":180,"KE":180,"HU":180,"MA":180,"CY":180,"JM":180,"EC":180,"RO":180,"BO":180,"GT":180,"CR":180,"QA":180,"SV":180,"HN":180,"NI":180,"PY":180,"UY":180,"PR":180,"BA":180,"PS":180,"TN":180,"BH":180,"VN":180,"GH":180,"MU":180,"UA":180,"MT":180,"BS":180,"MV":180,"OM":180,"MK":180,"LV":180,"EE":180,"IQ":180,"DZ":180,"AL":180,"NP":180,"MO":180,"ME":180,"SN":180,"GE":180,"BN":180,"UG":180,"GP":180,"BB":180,"AZ":180,"TZ":180,"LY":180,"MQ":180,"CM":180,"BW":180,"ET":180,"KZ":180,"NA":180,"MG":180,"NC":180,"MD":180,"FJ":180,"BY":180,"JE":180,"GU":180,"YE":180,"ZM":180,"IM":180,"HT":180,"KH":180,"AW":180,"PF":180,"AF":180,"BM":180,"GY":180,"AM":180,"MW":180,"AG":180,"RW":180,"GG":180,"GM":180,"FO":180,"LC":180,"KY":180,"BJ":180,"AD":180,"GD":180,"VI":180,"BZ":180,"VC":180,"MN":180,"MZ":180,"ML":180,"AO":180,"GF":180,"UZ":180,"DJ":180,"BF":180,"MC":180,"TG":180,"GL":180,"GA":180,"GI":180,"CD":180,"KG":180,"PG":180,"BT":180,"KN":180,"SZ":180,"LS":180,"LA":180,"LI":180,"MP":180,"SR":180,"SC":180,"VG":180,"TC":180,"DM":180,"MR":180,"AX":180,"SM":180,"SL":180,"NE":180,"CG":180,"AI":180,"YT":180,"CV":180,"GN":180,"TM":180,"BI":180,"TJ":180,"VU":180,"SB":180,"ER":180,"WS":180,"AS":180,"FK":180,"GQ":180,"TO":180,"KM":180,"PW":180,"FM":180,"CF":180,"SO":180,"MH":180,"VA":180,"TD":180,"KI":180,"ST":180,"TV":180,"NR":180,"RE":180,"LR":180,"ZW":180,"CI":180,"MM":180,"AN":180,"AQ":180,"BQ":180,"BV":180,"IO":180,"CX":180,"CC":180,"CK":180,"CW":180,"TF":180,"GW":180,"HM":180,"XK":180,"MS":180,"NU":180,"NF":180,"PN":180,"BL":180,"SH":180,"MF":180,"PM":180,"SX":180,"GS":180,"SS":180,"SJ":180,"TL":180,"TK":180,"UM":180,"WF":180,"EH":180},"global_io_max_campaign_duration":100},"spend_cap":"0","tax_id_status":0,"timezone_id":1,"timezone_name":"America/Los_Angeles","timezone_offset_hours_utc":-7,"tos_accepted":{"web_custom_audience_tos":1},"user_tasks":["DRAFT","ANALYZE"]},"emitted_at":1692180820858} -{"stream":"ads", "data": {"bid_type": "ABSOLUTE_OCPM", "account_id": "212551616838260", "campaign_id": "23853619670350398", "adset_id": "23853619670380398", "status": "ACTIVE", "creative": {"id": "23853666125630398"}, "id": "23853620198790398", "updated_time": "2023-03-21T22:33:56-0700", "created_time": "2023-03-17T08:04:29-0700", "name": "Don't Compromise Between Cost/Relaibility", "targeting": {"age_max": 60, "age_min": 18, "custom_audiences": [{"id": "23853630753300398", "name": "Lookalike (US, 10%) - Airbyte Cloud Users"}, {"id": "23853683587660398", "name": "Web Traffic [ALL] - _copy"}], "geo_locations": {"countries": ["US"], "location_types": ["home", "recent"]}, "brand_safety_content_filter_levels": ["FACEBOOK_STANDARD", "AN_STANDARD"], "targeting_relaxation_types": {"lookalike": 1, "custom_audience": 1}, "publisher_platforms": ["facebook", "instagram", "audience_network", "messenger"], "facebook_positions": ["feed", "biz_disco_feed", "facebook_reels", "facebook_reels_overlay", "right_hand_column", "video_feeds", "instant_article", "instream_video", "marketplace", "story", "search"], "instagram_positions": ["stream", "story", "explore", "reels", "shop", "explore_home", "profile_feed"], "device_platforms": ["mobile", "desktop"], "messenger_positions": ["story"], "audience_network_positions": ["classic", "instream_video", "rewarded_video"]}, "effective_status": "ACTIVE", "last_updated_by_app_id": "119211728144504", "source_ad_id": "0", "tracking_specs": [{"action.type": ["offsite_conversion"], "fb_pixel": ["917042523049733"]}, {"action.type": ["post_engagement"], "page": ["112704783733939"], "post": ["660122622785523", "662226992575086"]}, {"action.type": ["link_click"], "post": ["660122622785523", "662226992575086"], "post.wall": ["112704783733939"]}], "conversion_specs": [{"action.type": ["offsite_conversion"], "conversion_id": ["6015304265216283"]}]}, "emitted_at": 1682686047377} +{"stream": "ad_account", "data": {"id": "act_212551616838260", "account_id": "212551616838260", "account_status": 1, "age": 1305.7507638889, "amount_spent": "39125", "balance": "0", "business": {"id": "1506473679510495", "name": "Airbyte"}, "business_city": "", "business_country_code": "US", "business_name": "", "business_street": "", "business_street2": "", "can_create_brand_lift_study": false, "capabilities": ["CAN_CREATE_CALL_ADS", "CAN_SEE_GROWTH_OPPORTUNITY_DATA", "ENABLE_IA_RECIRC_AD_DISPLAY_FORMAT", "CAN_USE_MOBILE_EXTERNAL_PAGE_TYPE", "CAN_USE_FB_FEED_POSITION_IN_VIDEO_VIEW_15S", "ENABLE_BIZ_DISCO_ADS", "ENABLE_BRAND_OBJECTIVES_FOR_BIZ_DISCO_ADS", "ENABLE_DIRECT_REACH_FOR_BIZ_DISCO_ADS", "ENABLE_DYNAMIC_ADS_ON_IG_STORIES_ADS", "ENABLE_IG_STORIES_ADS_PPE_OBJECTIVE", "ENABLE_IG_STORIES_ADS_MESSENGER_DESTINATION", "ENABLE_PAC_FOR_BIZ_DISCO_ADS", "CAN_USE_FB_INSTREAM_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_FB_STORY_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_AN_INSTREAM_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_IG_STORY_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_FB_IA_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_FB_SUG_VIDEO_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_FB_MKT_PLACE_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_IG_FEED_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_IG_EXPLORE_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_AN_CLASSIC_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_AN_REWARD_VIDEO_POSITION_IN_VIDEO_VIEW_15S", "CAN_USE_REACH_AND_FREQUENCY", "CAN_USE_RECURRING_BUDGET", "HAS_VALID_PAYMENT_METHODS", "CAN_USE_LINK_CLICK_BILLING_EVENT", "CAN_USE_CPA_BILLING_EVENT", "CAN_SEE_NEW_CONVERSION_WINDOW_NUX", "ADS_INSTREAM_INTERFACE_INTEGRITY", "ADS_INSTREAM_LINK_CLICK", "ADS_INSTREAM_LINK_CLICK_IMAGE", "ADS_IN_OBJECTIVES_DEPRECATION", "MESSENGER_INBOX_ADS_PRODUCT_CATALOG_SALES", "CAN_SHOW_MESSENGER_DUPLICSTION_UPSELL", "ALLOW_INSTREAM_ONLY_FOR_REACH", "ADS_INSTREAM_VIDEO_PLACEMENT_CONVERSIONS", "CAN_CREATE_INSTAGRAM_EXPLORE_ADS", "ALLOW_INSTREAM_VIDEOS_PLACEMENT_ONLY", "ALLOW_INSTREAM_NON_INTERRUPTIVE_LEADGEN", "INSTREAM_VIDEO_AD_DESKTOP_CONVERSION_AD_PREVIEW", "ALLOW_INSTREAM_ONLY_FOR_BRAND_AWARENESS_AUCTION", "ALLOW_SUGGESTED_VIDEOS_PLACEMENT_ONLY", "WHATSAPP_DESTINATION_ADS", "CTM_ADS_CREATION_CLICK_TO_DIRECT", "CTW_ADS_ENABLE_IG_FEED_PLACEMENT", "CTW_ADS_FOR_NON_MESSAGES_OBJECTIVE", "CTW_ADS_TRUSTED_TIER_2_PLUS_ADVERTISER", "CTW_ADS_TRUSTED_TIER_ADVERTISER", "ADS_PLACEMENT_MARKETPLACE", "ADNW_DISABLE_INSTREAM_AND_WEB_PLACEMENT", "CAN_CHANGE_BILLING_THRESHOLD", "CAN_USE_APP_EVENT_AVERAGE_COST_BIDDING", "CAN_USE_LEAD_GEN_AVERAGE_COST_BIDDING", "ADS_VALUE_OPTIMIZATION_DYNAMIC_ADS_1D", "ADS_DELIVERY_INSIGHTS_IN_BIDDING_PRESET_EXPERIMENT", "ADS_DELIVERY_INSIGHTS_OPTIMIZATION_PRESET", "CAN_SEE_APP_AD_EVENTS", "CAN_SEE_NEW_STANDARD_EVENTS_BETA", "CAN_SEE_VCK_HOLIDAY_TEMPLATES", "ENABLE_DCO_FOR_FB_STORY_ADS", "CAN_USE_IG_EXPLORE_GRID_HOME_PLACEMENT", "CAN_USE_IG_EXPLORE_HOME_IN_REACH_AND_FREQUENCY", "CAN_USE_IG_EXPLORE_HOME_POST_ENGAGEMENT_MESSAGES", "CAN_USE_IG_SEARCH_PLACEMENT", "CAN_USE_IG_SEARCH_GRID_ADS", "CAN_USE_IG_SEARCH_RESULTS_AUTO_PLACEMENT", "CAN_USE_IG_REELS_PAC_CAROUSEL", "CAN_USE_IG_REELS_POSITION", "CAN_SEE_CONVERSION_LIFT_SUMMARY", "CAN_USE_IG_PROFILE_FEED_POSITION", "CAN_USE_IG_PROFILE_FEED_AUTO_PLACEMENT", "CAN_USE_IG_PROFILE_FEED_ADDITIONAL_OBJECTIVES", "CAN_USE_IG_REELS_REACH_AND_FREQUENCY", "CAN_USE_IG_REELS_OVERLAY_POSITION", "CAN_USE_IG_REELS_OVERLAY_AUTO_PLACEMENT", "CAN_USE_IG_REELS_OVERLAY_PAC", "CAN_USE_IG_SHOP_TAB_PAC", "CAN_SEE_LEARNING_STAGE", "ENABLE_WEBSITE_CONVERSIONS_FOR_FB_STORY_ADS", "ENABLE_MESSENGER_INBOX_VIDEO_ADS", "ENABLE_VIDEO_VIEWS_FOR_FB_STORY_ADS", "ENABLE_LINK_CLICKS_FOR_FB_STORY_ADS", "ENABLE_REACH_FOR_FB_STORY_ADS", "CAN_USE_CALL_TO_ACTION_LINK_IMPORT_EXPORT", "ADS_INSTREAM_VIDEO_ENABLE_SLIDE_SHOW", "ALLOW_INSTREAM_VIDEOS_PLACEMENT_ONLY_IN_VV_REACH_AND_FREQUENCY", "ENABLE_MOBILE_APP_INSTALLS_FOR_FB_STORY_ADS", "ENABLE_LEAD_GEN_FOR_FB_STORY_ADS", "CAN_USE_FB_MKT_PLACE_POSITION_IN_REACH", "CAN_USE_FB_MKT_PLACE_POSITION_IN_VIDEO_VIEW", "CAN_USE_FB_MKT_PLACE_POSITION_IN_STORE_VISIT", "ENABLE_MOBILE_APP_ENGAGEMENT_FOR_FB_STORY_ADS", "CAN_USE_FB_MKT_PLACE_POSITION_IN_BRAND_AWARENESS", "CAN_USE_FB_MKT_PLACE_POSITION_IN_APP_INSTALLS", "CAN_USE_FB_MKT_PLACE_POSITION_IN_LEAD_GENERATION", "CAN_USE_FB_MKT_PLACE_POSITION_IN_MESSAGE", "CAN_USE_FB_MKT_PLACE_POSITION_IN_PAGE_LIKE", "CAN_USE_FB_MKT_PLACE_POSITION_IN_POST_ENGAGEMENT", "RF_ALLOW_MARKETPLACE_ACCOUNT", "RF_ALLOW_SEARCH_ACCOUNT", "VERTICAL_VIDEO_PAC_INSTREAM_UPSELL", "IX_COLLECTION_ENABLED_FOR_BAO_AND_REACH", "ADS_BM_REQUIREMENTS_OCT_15_RELEASE", "ENABLE_POST_ENGAGEMENT_FOR_FB_STORY", "ENBABLE_CATALOG_SALES_FOR_FB_STORY", "CAN_USE_WHATSAPP_DESTINATION_ON_LINK_CLICKS_AND_CONVERSIONS", "CAN_USE_WHATSAPP_DESTINATION_ON_CONVERSIONS", "IS_NON_TAIL_AD_ACCOUNT", "IS_IN_DSA_GK", "IS_IN_IG_EXISTING_POST_CTA_DEFAULTING_EXPERIMENT", "IS_IN_SHORT_WA_LINK_CTWA_UNCONV_TRAFFIC_EXPERIMENT", "IS_IN_ODAX_EXPERIENCE", "IS_IN_REACH_BRAND_AWARENESS_WHATSAPP_L1_DESTINATION_EXPERIMENT", "IS_IN_VIDEO_VIEWS_WHATSAPP_L1_DESTINATION_EXPERIMENT", "IS_IN_WHATSAPP_DESTINATION_DEFAULTING_EXPERIMENT", "CAN_USE_MARKETPLACE_DESKTOP", "ADS_MERCHANT_OVERLAYS_DEPRECATION", "CONNECTIONS_DEPRECATION_V2", "CAN_USE_LIVE_VIDEO_FOR_THRUPLAY", "CAN_SEE_HEC_AM_FLOW", "CAN_SEE_POLITICAL_FLOW", "ADS_INSTREAM_PLACEMENT_CATALOG_SALES", "ENABLE_CONVERSIONS_FOR_FB_GROUP_TAB_ADS", "ENABLE_LINK_CLICK_FOR_FB_GROUP_TAB_ADS", "ENABLE_REACH_FOR_FB_GROUP_TAB_ADS", "CAN_USE_CONVERSATIONS_OPTIMIZATION", "ENABLE_THRUPLAY_OPTIMIZATION_MESSENGER_STORY_ADS", "CAN_USE_IG_STORY_POLLS_PAC_CREATION", "IOS14_CEO_CAMPAIGN_CREATION", "ENABLE_VIDEO_CHANNEL_PLACEMENT_FOR_RSVP_ADS", "DIGITAL_CIRCULAR_ADS", "CAN_SEE_SAFR_V3_FLOW", "CAN_USE_FB_REELS_POSITION", "CAN_USE_ADS_ON_FB_REELS_POSITION", "CAN_USE_FB_REELS_AUTO_PLACEMENT", "ENABLE_FB_REELS_CREATION_PAC_ADS", "ENABLE_FB_REELS_CREATION_DCO_ADS", "ENABLE_FB_REELS_POSTLOOP_CREATION_DCO_ADS", "ENABLE_FB_REELS_POSTLOOP_CREATION_PAC_ADS", "RF_CPA_BILLING_DEPRECATION_PHASE_2", "ENABLE_APP_INSTALL_CUSTOM_PRODUCT_PAGES", "ENABLE_ADS_ON_FB_REELS_PLACEMENT_UNIFICATION", "ENABLE_ADS_ON_IG_SHOP_TAB_DEPRECATION_L2_NUX", "ADS_RF_FB_REELS_PLACEMENT", "ENABLE_ADS_ON_FB_INSTANT_ARTICLE_DEPRECATION_L2_NUX", "REELS_DM_ADS_ENABLE_REACH_AND_FREQUENCY", "ADS_AEMV2_HAS_LAUNCHED", "ELIGIBLE_FOR_TEXT_GEN"], "created_time": "2020-04-13T18:04:59-0700", "currency": "USD", "disable_reason": 0.0, "end_advertiser": 1506473679510495.0, "end_advertiser_name": "Airbyte", "fb_entity": 85.0, "funding_source": 2825262454257003.0, "funding_source_details": {"id": "2825262454257003", "type": 1}, "has_migrated_permissions": true, "is_attribution_spec_system_default": true, "is_direct_deals_enabled": false, "is_in_3ds_authorization_enabled_market": false, "is_notifications_enabled": true, "is_personal": 0.0, "is_prepay_account": false, "is_tax_id_required": false, "min_campaign_group_spend_cap": 10000.0, "min_daily_budget": 100.0, "name": "Airbyte", "offsite_pixels_tos_accepted": true, "owner": 1506473679510495.0, "rf_spec": {"min_reach_limits": {"US": 200000, "CA": 200000, "GB": 200000, "AR": 200000, "AU": 200000, "AT": 200000, "BE": 200000, "BR": 200000, "CL": 200000, "CN": 200000, "CO": 200000, "HR": 200000, "DK": 200000, "DO": 200000, "EG": 200000, "FI": 200000, "FR": 200000, "DE": 200000, "GR": 200000, "HK": 200000, "IN": 200000, "ID": 200000, "IE": 200000, "IL": 200000, "IT": 200000, "JP": 200000, "JO": 200000, "KW": 200000, "LB": 200000, "MY": 200000, "MX": 200000, "NL": 200000, "NZ": 200000, "NG": 200000, "NO": 200000, "PK": 200000, "PA": 200000, "PE": 200000, "PH": 200000, "PL": 200000, "RU": 200000, "SA": 200000, "RS": 200000, "SG": 200000, "ZA": 200000, "KR": 200000, "ES": 200000, "SE": 200000, "CH": 200000, "TW": 200000, "TH": 200000, "TR": 200000, "AE": 200000, "VE": 200000, "PT": 200000, "LU": 200000, "BG": 200000, "CZ": 200000, "SI": 200000, "IS": 200000, "SK": 200000, "LT": 200000, "TT": 200000, "BD": 200000, "LK": 200000, "KE": 200000, "HU": 200000, "MA": 200000, "CY": 200000, "JM": 200000, "EC": 200000, "RO": 200000, "BO": 200000, "GT": 200000, "CR": 200000, "QA": 200000, "SV": 200000, "HN": 200000, "NI": 200000, "PY": 200000, "UY": 200000, "PR": 200000, "BA": 200000, "PS": 200000, "TN": 200000, "BH": 200000, "VN": 200000, "GH": 200000, "MU": 200000, "UA": 200000, "MT": 200000, "BS": 200000, "MV": 200000, "OM": 200000, "MK": 200000, "LV": 200000, "EE": 200000, "IQ": 200000, "DZ": 200000, "AL": 200000, "NP": 200000, "MO": 200000, "ME": 200000, "SN": 200000, "GE": 200000, "BN": 200000, "UG": 200000, "GP": 200000, "BB": 200000, "AZ": 200000, "TZ": 200000, "LY": 200000, "MQ": 200000, "CM": 200000, "BW": 200000, "ET": 200000, "KZ": 200000, "NA": 200000, "MG": 200000, "NC": 200000, "MD": 200000, "FJ": 200000, "BY": 200000, "JE": 200000, "GU": 200000, "YE": 200000, "ZM": 200000, "IM": 200000, "HT": 200000, "KH": 200000, "AW": 200000, "PF": 200000, "AF": 200000, "BM": 200000, "GY": 200000, "AM": 200000, "MW": 200000, "AG": 200000, "RW": 200000, "GG": 200000, "GM": 200000, "FO": 200000, "LC": 200000, "KY": 200000, "BJ": 200000, "AD": 200000, "GD": 200000, "VI": 200000, "BZ": 200000, "VC": 200000, "MN": 200000, "MZ": 200000, "ML": 200000, "AO": 200000, "GF": 200000, "UZ": 200000, "DJ": 200000, "BF": 200000, "MC": 200000, "TG": 200000, "GL": 200000, "GA": 200000, "GI": 200000, "CD": 200000, "KG": 200000, "PG": 200000, "BT": 200000, "KN": 200000, "SZ": 200000, "LS": 200000, "LA": 200000, "LI": 200000, "MP": 200000, "SR": 200000, "SC": 200000, "VG": 200000, "TC": 200000, "DM": 200000, "MR": 200000, "AX": 200000, "SM": 200000, "SL": 200000, "NE": 200000, "CG": 200000, "AI": 200000, "YT": 200000, "CV": 200000, "GN": 200000, "TM": 200000, "BI": 200000, "TJ": 200000, "VU": 200000, "SB": 200000, "ER": 200000, "WS": 200000, "AS": 200000, "FK": 200000, "GQ": 200000, "TO": 200000, "KM": 200000, "PW": 200000, "FM": 200000, "CF": 200000, "SO": 200000, "MH": 200000, "VA": 200000, "TD": 200000, "KI": 200000, "ST": 200000, "TV": 200000, "NR": 200000, "RE": 200000, "LR": 200000, "ZW": 200000, "CI": 200000, "MM": 200000, "AN": 200000, "AQ": 200000, "BQ": 200000, "BV": 200000, "IO": 200000, "CX": 200000, "CC": 200000, "CK": 200000, "CW": 200000, "TF": 200000, "GW": 200000, "HM": 200000, "XK": 200000, "MS": 200000, "NU": 200000, "NF": 200000, "PN": 200000, "BL": 200000, "SH": 200000, "MF": 200000, "PM": 200000, "SX": 200000, "GS": 200000, "SS": 200000, "SJ": 200000, "TL": 200000, "TK": 200000, "UM": 200000, "WF": 200000, "EH": 200000}, "countries": ["US", "CA", "GB", "AR", "AU", "AT", "BE", "BR", "CL", "CN", "CO", "HR", "DK", "DO", "EG", "FI", "FR", "DE", "GR", "HK", "IN", "ID", "IE", "IL", "IT", "JP", "JO", "KW", "LB", "MY", "MX", "NL", "NZ", "NG", "NO", "PK", "PA", "PE", "PH", "PL", "RU", "SA", "RS", "SG", "ZA", "KR", "ES", "SE", "CH", "TW", "TH", "TR", "AE", "VE", "PT", "LU", "BG", "CZ", "SI", "IS", "SK", "LT", "TT", "BD", "LK", "KE", "HU", "MA", "CY", "JM", "EC", "RO", "BO", "GT", "CR", "QA", "SV", "HN", "NI", "PY", "UY", "PR", "BA", "PS", "TN", "BH", "VN", "GH", "MU", "UA", "MT", "BS", "MV", "OM", "MK", "EE", "LV", "IQ", "DZ", "AL", "NP", "MO", "ME", "SN", "GE", "BN", "UG", "GP", "BB", "ZW", "CI", "AZ", "TZ", "LY", "MQ", "MM", "CM", "BW", "ET", "KZ", "NA", "MG", "NC", "MD", "FJ", "BY", "JE", "GU", "YE", "ZM", "IM", "HT", "KH", "AW", "PF", "AF", "BM", "GY", "AM", "MW", "AG", "RW", "GG", "GM", "FO", "LC", "KY", "BJ", "AD", "GD", "VI", "BZ", "VC", "MN", "MZ", "ML", "AO", "GF", "UZ", "DJ", "BF", "MC", "TG", "GL", "GA", "GI", "CD", "KG", "PG", "BT", "KN", "SZ", "LS", "LA", "LI", "MP", "SR", "SC", "VG", "TC", "DM", "MR", "AX", "SM", "SL", "NE", "CG", "AI", "YT", "LR", "CV", "GN", "TM", "BI", "TJ", "VU", "SB", "ER", "WS", "AS", "FK", "GQ", "TO", "KM", "PW", "FM", "CF", "SO", "MH", "VA", "TD", "KI", "ST", "TV", "NR", "RE", "AN", "AQ", "BQ", "BV", "IO", "CX", "CC", "CK", "CW", "TF", "GW", "HM", "XK", "MS", "NU", "NF", "PN", "BL", "SH", "MF", "PM", "SX", "GS", "SS", "SJ", "TL", "TK", "UM", "WF", "EH"], "min_campaign_duration": {"US": 1, "CA": 1, "GB": 1, "AR": 1, "AU": 1, "AT": 1, "BE": 1, "BR": 1, "CL": 1, "CN": 1, "CO": 1, "HR": 1, "DK": 1, "DO": 1, "EG": 1, "FI": 1, "FR": 1, "DE": 1, "GR": 1, "HK": 1, "IN": 1, "ID": 1, "IE": 1, "IL": 1, "IT": 1, "JP": 1, "JO": 1, "KW": 1, "LB": 1, "MY": 1, "MX": 1, "NL": 1, "NZ": 1, "NG": 1, "NO": 1, "PK": 1, "PA": 1, "PE": 1, "PH": 1, "PL": 1, "RU": 1, "SA": 1, "RS": 1, "SG": 1, "ZA": 1, "KR": 1, "ES": 1, "SE": 1, "CH": 1, "TW": 1, "TH": 1, "TR": 1, "AE": 1, "VE": 1, "PT": 1, "LU": 1, "BG": 1, "CZ": 1, "SI": 1, "IS": 1, "SK": 1, "LT": 1, "TT": 1, "BD": 1, "LK": 1, "KE": 1, "HU": 1, "MA": 1, "CY": 1, "JM": 1, "EC": 1, "RO": 1, "BO": 1, "GT": 1, "CR": 1, "QA": 1, "SV": 1, "HN": 1, "NI": 1, "PY": 1, "UY": 1, "PR": 1, "BA": 1, "PS": 1, "TN": 1, "BH": 1, "VN": 1, "GH": 1, "MU": 1, "UA": 1, "MT": 1, "BS": 1, "MV": 1, "OM": 1, "MK": 1, "LV": 1, "EE": 1, "IQ": 1, "DZ": 1, "AL": 1, "NP": 1, "MO": 1, "ME": 1, "SN": 1, "GE": 1, "BN": 1, "UG": 1, "GP": 1, "BB": 1, "AZ": 1, "TZ": 1, "LY": 1, "MQ": 1, "CM": 1, "BW": 1, "ET": 1, "KZ": 1, "NA": 1, "MG": 1, "NC": 1, "MD": 1, "FJ": 1, "BY": 1, "JE": 1, "GU": 1, "YE": 1, "ZM": 1, "IM": 1, "HT": 1, "KH": 1, "AW": 1, "PF": 1, "AF": 1, "BM": 1, "GY": 1, "AM": 1, "MW": 1, "AG": 1, "RW": 1, "GG": 1, "GM": 1, "FO": 1, "LC": 1, "KY": 1, "BJ": 1, "AD": 1, "GD": 1, "VI": 1, "BZ": 1, "VC": 1, "MN": 1, "MZ": 1, "ML": 1, "AO": 1, "GF": 1, "UZ": 1, "DJ": 1, "BF": 1, "MC": 1, "TG": 1, "GL": 1, "GA": 1, "GI": 1, "CD": 1, "KG": 1, "PG": 1, "BT": 1, "KN": 1, "SZ": 1, "LS": 1, "LA": 1, "LI": 1, "MP": 1, "SR": 1, "SC": 1, "VG": 1, "TC": 1, "DM": 1, "MR": 1, "AX": 1, "SM": 1, "SL": 1, "NE": 1, "CG": 1, "AI": 1, "YT": 1, "CV": 1, "GN": 1, "TM": 1, "BI": 1, "TJ": 1, "VU": 1, "SB": 1, "ER": 1, "WS": 1, "AS": 1, "FK": 1, "GQ": 1, "TO": 1, "KM": 1, "PW": 1, "FM": 1, "CF": 1, "SO": 1, "MH": 1, "VA": 1, "TD": 1, "KI": 1, "ST": 1, "TV": 1, "NR": 1, "RE": 1, "LR": 1, "ZW": 1, "CI": 1, "MM": 1, "AN": 1, "AQ": 1, "BQ": 1, "BV": 1, "IO": 1, "CX": 1, "CC": 1, "CK": 1, "CW": 1, "TF": 1, "GW": 1, "HM": 1, "XK": 1, "MS": 1, "NU": 1, "NF": 1, "PN": 1, "BL": 1, "SH": 1, "MF": 1, "PM": 1, "SX": 1, "GS": 1, "SS": 1, "SJ": 1, "TL": 1, "TK": 1, "UM": 1, "WF": 1, "EH": 1}, "max_campaign_duration": {"US": 90, "CA": 90, "GB": 90, "AR": 90, "AU": 90, "AT": 90, "BE": 90, "BR": 90, "CL": 90, "CN": 90, "CO": 90, "HR": 90, "DK": 90, "DO": 90, "EG": 90, "FI": 90, "FR": 90, "DE": 90, "GR": 90, "HK": 90, "IN": 90, "ID": 90, "IE": 90, "IL": 90, "IT": 90, "JP": 90, "JO": 90, "KW": 90, "LB": 90, "MY": 90, "MX": 90, "NL": 90, "NZ": 90, "NG": 90, "NO": 90, "PK": 90, "PA": 90, "PE": 90, "PH": 90, "PL": 90, "RU": 90, "SA": 90, "RS": 90, "SG": 90, "ZA": 90, "KR": 90, "ES": 90, "SE": 90, "CH": 90, "TW": 90, "TH": 90, "TR": 90, "AE": 90, "VE": 90, "PT": 90, "LU": 90, "BG": 90, "CZ": 90, "SI": 90, "IS": 90, "SK": 90, "LT": 90, "TT": 90, "BD": 90, "LK": 90, "KE": 90, "HU": 90, "MA": 90, "CY": 90, "JM": 90, "EC": 90, "RO": 90, "BO": 90, "GT": 90, "CR": 90, "QA": 90, "SV": 90, "HN": 90, "NI": 90, "PY": 90, "UY": 90, "PR": 90, "BA": 90, "PS": 90, "TN": 90, "BH": 90, "VN": 90, "GH": 90, "MU": 90, "UA": 90, "MT": 90, "BS": 90, "MV": 90, "OM": 90, "MK": 90, "LV": 90, "EE": 90, "IQ": 90, "DZ": 90, "AL": 90, "NP": 90, "MO": 90, "ME": 90, "SN": 90, "GE": 90, "BN": 90, "UG": 90, "GP": 90, "BB": 90, "AZ": 90, "TZ": 90, "LY": 90, "MQ": 90, "CM": 90, "BW": 90, "ET": 90, "KZ": 90, "NA": 90, "MG": 90, "NC": 90, "MD": 90, "FJ": 90, "BY": 90, "JE": 90, "GU": 90, "YE": 90, "ZM": 90, "IM": 90, "HT": 90, "KH": 90, "AW": 90, "PF": 90, "AF": 90, "BM": 90, "GY": 90, "AM": 90, "MW": 90, "AG": 90, "RW": 90, "GG": 90, "GM": 90, "FO": 90, "LC": 90, "KY": 90, "BJ": 90, "AD": 90, "GD": 90, "VI": 90, "BZ": 90, "VC": 90, "MN": 90, "MZ": 90, "ML": 90, "AO": 90, "GF": 90, "UZ": 90, "DJ": 90, "BF": 90, "MC": 90, "TG": 90, "GL": 90, "GA": 90, "GI": 90, "CD": 90, "KG": 90, "PG": 90, "BT": 90, "KN": 90, "SZ": 90, "LS": 90, "LA": 90, "LI": 90, "MP": 90, "SR": 90, "SC": 90, "VG": 90, "TC": 90, "DM": 90, "MR": 90, "AX": 90, "SM": 90, "SL": 90, "NE": 90, "CG": 90, "AI": 90, "YT": 90, "CV": 90, "GN": 90, "TM": 90, "BI": 90, "TJ": 90, "VU": 90, "SB": 90, "ER": 90, "WS": 90, "AS": 90, "FK": 90, "GQ": 90, "TO": 90, "KM": 90, "PW": 90, "FM": 90, "CF": 90, "SO": 90, "MH": 90, "VA": 90, "TD": 90, "KI": 90, "ST": 90, "TV": 90, "NR": 90, "RE": 90, "LR": 90, "ZW": 90, "CI": 90, "MM": 90, "AN": 90, "AQ": 90, "BQ": 90, "BV": 90, "IO": 90, "CX": 90, "CC": 90, "CK": 90, "CW": 90, "TF": 90, "GW": 90, "HM": 90, "XK": 90, "MS": 90, "NU": 90, "NF": 90, "PN": 90, "BL": 90, "SH": 90, "MF": 90, "PM": 90, "SX": 90, "GS": 90, "SS": 90, "SJ": 90, "TL": 90, "TK": 90, "UM": 90, "WF": 90, "EH": 90}, "max_days_to_finish": {"US": 180, "CA": 180, "GB": 180, "AR": 180, "AU": 180, "AT": 180, "BE": 180, "BR": 180, "CL": 180, "CN": 180, "CO": 180, "HR": 180, "DK": 180, "DO": 180, "EG": 180, "FI": 180, "FR": 180, "DE": 180, "GR": 180, "HK": 180, "IN": 180, "ID": 180, "IE": 180, "IL": 180, "IT": 180, "JP": 180, "JO": 180, "KW": 180, "LB": 180, "MY": 180, "MX": 180, "NL": 180, "NZ": 180, "NG": 180, "NO": 180, "PK": 180, "PA": 180, "PE": 180, "PH": 180, "PL": 180, "RU": 180, "SA": 180, "RS": 180, "SG": 180, "ZA": 180, "KR": 180, "ES": 180, "SE": 180, "CH": 180, "TW": 180, "TH": 180, "TR": 180, "AE": 180, "VE": 180, "PT": 180, "LU": 180, "BG": 180, "CZ": 180, "SI": 180, "IS": 180, "SK": 180, "LT": 180, "TT": 180, "BD": 180, "LK": 180, "KE": 180, "HU": 180, "MA": 180, "CY": 180, "JM": 180, "EC": 180, "RO": 180, "BO": 180, "GT": 180, "CR": 180, "QA": 180, "SV": 180, "HN": 180, "NI": 180, "PY": 180, "UY": 180, "PR": 180, "BA": 180, "PS": 180, "TN": 180, "BH": 180, "VN": 180, "GH": 180, "MU": 180, "UA": 180, "MT": 180, "BS": 180, "MV": 180, "OM": 180, "MK": 180, "LV": 180, "EE": 180, "IQ": 180, "DZ": 180, "AL": 180, "NP": 180, "MO": 180, "ME": 180, "SN": 180, "GE": 180, "BN": 180, "UG": 180, "GP": 180, "BB": 180, "AZ": 180, "TZ": 180, "LY": 180, "MQ": 180, "CM": 180, "BW": 180, "ET": 180, "KZ": 180, "NA": 180, "MG": 180, "NC": 180, "MD": 180, "FJ": 180, "BY": 180, "JE": 180, "GU": 180, "YE": 180, "ZM": 180, "IM": 180, "HT": 180, "KH": 180, "AW": 180, "PF": 180, "AF": 180, "BM": 180, "GY": 180, "AM": 180, "MW": 180, "AG": 180, "RW": 180, "GG": 180, "GM": 180, "FO": 180, "LC": 180, "KY": 180, "BJ": 180, "AD": 180, "GD": 180, "VI": 180, "BZ": 180, "VC": 180, "MN": 180, "MZ": 180, "ML": 180, "AO": 180, "GF": 180, "UZ": 180, "DJ": 180, "BF": 180, "MC": 180, "TG": 180, "GL": 180, "GA": 180, "GI": 180, "CD": 180, "KG": 180, "PG": 180, "BT": 180, "KN": 180, "SZ": 180, "LS": 180, "LA": 180, "LI": 180, "MP": 180, "SR": 180, "SC": 180, "VG": 180, "TC": 180, "DM": 180, "MR": 180, "AX": 180, "SM": 180, "SL": 180, "NE": 180, "CG": 180, "AI": 180, "YT": 180, "CV": 180, "GN": 180, "TM": 180, "BI": 180, "TJ": 180, "VU": 180, "SB": 180, "ER": 180, "WS": 180, "AS": 180, "FK": 180, "GQ": 180, "TO": 180, "KM": 180, "PW": 180, "FM": 180, "CF": 180, "SO": 180, "MH": 180, "VA": 180, "TD": 180, "KI": 180, "ST": 180, "TV": 180, "NR": 180, "RE": 180, "LR": 180, "ZW": 180, "CI": 180, "MM": 180, "AN": 180, "AQ": 180, "BQ": 180, "BV": 180, "IO": 180, "CX": 180, "CC": 180, "CK": 180, "CW": 180, "TF": 180, "GW": 180, "HM": 180, "XK": 180, "MS": 180, "NU": 180, "NF": 180, "PN": 180, "BL": 180, "SH": 180, "MF": 180, "PM": 180, "SX": 180, "GS": 180, "SS": 180, "SJ": 180, "TL": 180, "TK": 180, "UM": 180, "WF": 180, "EH": 180}, "global_io_max_campaign_duration": 100}, "spend_cap": "0", "tax_id_status": 0.0, "tax_id_type": "0", "timezone_id": 1.0, "timezone_name": "America/Los_Angeles", "timezone_offset_hours_utc": -8.0, "tos_accepted": {"web_custom_audience_tos": 1}, "user_tasks": ["DRAFT", "ANALYZE", "ADVERTISE", "MANAGE"]}, "emitted_at": 1699644186066} +{"stream":"ads","data":{"bid_type":"ABSOLUTE_OCPM","account_id":"212551616838260","campaign_id":"23853619670350398","adset_id":"23853619670380398","status":"ACTIVE","creative":{"id":"23853666125630398"},"id":"23853620198790398","updated_time":"2023-03-21T22:33:56-0700","created_time":"2023-03-17T08:04:29-0700","name":"Don't Compromise Between Cost/Relaibility","targeting":{"age_max":60,"age_min":18,"custom_audiences":[{"id":"23853630753300398","name":"Lookalike (US, 10%) - Airbyte Cloud Users"},{"id":"23853683587660398","name":"Web Traffic [ALL] - _copy"}],"geo_locations":{"countries":["US"],"location_types":["home","recent"]},"brand_safety_content_filter_levels":["FACEBOOK_STANDARD","AN_STANDARD"],"targeting_relaxation_types":{"lookalike":1,"custom_audience":1},"publisher_platforms":["facebook","instagram","audience_network","messenger"],"facebook_positions":["feed","biz_disco_feed","facebook_reels","facebook_reels_overlay","right_hand_column","video_feeds","instant_article","instream_video","marketplace","story","search"],"instagram_positions":["stream","story","explore","reels","shop","explore_home","profile_feed"],"device_platforms":["mobile","desktop"],"messenger_positions":["story"],"audience_network_positions":["classic","instream_video","rewarded_video"]},"effective_status":"ACTIVE","last_updated_by_app_id":"119211728144504","source_ad_id":"0","tracking_specs":[{"action.type":["offsite_conversion"],"fb_pixel":["917042523049733"]},{"action.type":["post_engagement"],"page":["112704783733939"],"post":["660122622785523","662226992575086"]},{"action.type":["link_click"],"post":["660122622785523","662226992575086"],"post.wall":["112704783733939"]}],"conversion_specs":[{"action.type":["offsite_conversion"],"conversion_id":["6015304265216283"]}]},"emitted_at":1682686047377} {"stream":"ad_sets","data":{"name":"Lookalike audience_Free Connector Program","promoted_object":{"pixel_id":"917042523049733","custom_event_type":"COMPLETE_REGISTRATION"},"id":"23853619670380398","account_id":"212551616838260","updated_time":"2023-03-21T14:20:51-0700","daily_budget":2000,"budget_remaining":2000,"effective_status":"ACTIVE","campaign_id":"23853619670350398","created_time":"2023-03-17T08:04:28-0700","start_time":"2023-03-17T08:04:28-0700","lifetime_budget":0,"targeting":{"age_max":60,"age_min":18,"custom_audiences":[{"id":"23853630753300398","name":"Lookalike (US, 10%) - Airbyte Cloud Users"},{"id":"23853683587660398","name":"Web Traffic [ALL] - _copy"}],"geo_locations":{"countries":["US"],"location_types":["home","recent"]},"brand_safety_content_filter_levels":["FACEBOOK_STANDARD","AN_STANDARD"],"targeting_relaxation_types":{"lookalike":1,"custom_audience":1},"publisher_platforms":["facebook","instagram","audience_network","messenger"],"facebook_positions":["feed","biz_disco_feed","facebook_reels","facebook_reels_overlay","right_hand_column","video_feeds","instant_article","instream_video","marketplace","story","search"],"instagram_positions":["stream","story","explore","reels","shop","explore_home","profile_feed"],"device_platforms":["mobile","desktop"],"messenger_positions":["story"],"audience_network_positions":["classic","instream_video","rewarded_video"]},"bid_strategy":"LOWEST_COST_WITHOUT_CAP"},"emitted_at":1692180821847} -{"stream":"campaigns", "data": {"account_id": "212551616838260", "budget_rebalance_flag": false, "budget_remaining": 0.0, "buying_type": "AUCTION", "created_time": "2021-01-18T21:36:42-0800", "effective_status": "PAUSED", "id": "23846542053890398", "name": "Fake Campaign 0", "objective": "MESSAGES", "smart_promotion_type": "GUIDED_CREATION", "source_campaign_id": 0.0, "special_ad_category": "NONE", "start_time": "1969-12-31T15:59:59-0800", "updated_time": "2021-02-18T01:00:02-0800"}, "emitted_at": 1682686106887} -{"stream":"custom_audiences","data":{"id":"23853683587660398","account_id":"212551616838260","approximate_count_lower_bound":4700,"approximate_count_upper_bound":5600,"customer_file_source":"PARTNER_PROVIDED_ONLY","data_source":{"type":"UNKNOWN","sub_type":"ANYTHING","creation_params":"[]"},"delivery_status":{"code":200,"description":"This audience is ready for use."},"description":"Custom Audience-Web Traffic [ALL] - _copy","is_value_based":false,"name":"Web Traffic [ALL] - _copy","operation_status":{"code":200,"description":"Normal"},"permission_for_actions":{"can_edit":false,"can_see_insight":"True","can_share":"False","subtype_supports_lookalike":"True","supports_recipient_lookalike":"False"},"retention_days":0,"subtype":"CUSTOM","time_content_updated":1679433484,"time_created":1679433479,"time_updated":1679433484},"emitted_at":1692028917200} -{"stream":"ad_creatives","data":{"id":"23844568440620398","account_id":"212551616838260","actor_id":"112704783733939","asset_feed_spec":{"images":[{"adlabels":[{"name":"placement_asset_fb19ee1baacc68_1586830094862","id":"23844521781280398"}],"hash":"7394ffb578c53e8761b6498d3008725b","image_crops":{"191x100":[[0,411],[589,719]]}},{"adlabels":[{"name":"placement_asset_f1f518506ae7e68_1586830094842","id":"23844521781340398"}],"hash":"7394ffb578c53e8761b6498d3008725b","image_crops":{"100x100":[[12,282],[574,844]]}},{"adlabels":[{"name":"placement_asset_f311b79c14a30c_1586830094845","id":"23844521781330398"}],"hash":"7394ffb578c53e8761b6498d3008725b","image_crops":{"90x160":[[14,72],[562,1046]]}},{"adlabels":[{"name":"placement_asset_f2c2fe4f20af66c_1586830157386","id":"23844521783780398"}],"hash":"7394ffb578c53e8761b6498d3008725b","image_crops":{"90x160":[[0,0],[589,1047]]}}],"bodies":[{"adlabels":[{"name":"placement_asset_f2d65f15340e594_1586830094852","id":"23844521781260398"},{"name":"placement_asset_f1f97c3e3a63d74_1586830094858","id":"23844521781300398"},{"name":"placement_asset_f14cee2ab5d786_1586830094863","id":"23844521781370398"},{"name":"placement_asset_f14877915fb5acc_1586830157387","id":"23844521783760398"}],"text":""}],"call_to_action_types":["LEARN_MORE"],"descriptions":[{"text":"Unmatched attribution, ad performances, and lead conversion, by unlocking your ad-blocked traffic across all your tools."}],"link_urls":[{"adlabels":[{"name":"placement_asset_f309294689f2c6c_1586830094864","id":"23844521781290398"},{"name":"placement_asset_f136a02466f2bc_1586830094856","id":"23844521781310398"},{"name":"placement_asset_fa79b032b68274_1586830094860","id":"23844521781320398"},{"name":"placement_asset_f28a128696c7428_1586830157387","id":"23844521783790398"}],"website_url":"http://dataline.io/","display_url":""}],"titles":[{"adlabels":[{"name":"placement_asset_f1013e29f89c38_1586830094864","id":"23844521781350398"},{"name":"placement_asset_fcb53b78a11574_1586830094859","id":"23844521781360398"},{"name":"placement_asset_f1a3b3d525f4998_1586830094854","id":"23844521781380398"},{"name":"placement_asset_f890656071c9ac_1586830157387","id":"23844521783770398"}],"text":"Unblock all your adblocked traffic"}],"ad_formats":["AUTOMATIC_FORMAT"],"asset_customization_rules":[{"customization_spec":{"age_max":65,"age_min":13,"publisher_platforms":["instagram","audience_network","messenger"],"instagram_positions":["story"],"messenger_positions":["story"],"audience_network_positions":["classic"]},"image_label":{"name":"placement_asset_f311b79c14a30c_1586830094845","id":"23844521781330398"},"body_label":{"name":"placement_asset_f1f97c3e3a63d74_1586830094858","id":"23844521781300398"},"link_url_label":{"name":"placement_asset_fa79b032b68274_1586830094860","id":"23844521781320398"},"title_label":{"name":"placement_asset_fcb53b78a11574_1586830094859","id":"23844521781360398"},"priority":1},{"customization_spec":{"age_max":65,"age_min":13,"publisher_platforms":["facebook"],"facebook_positions":["right_hand_column","instant_article","search"]},"image_label":{"name":"placement_asset_fb19ee1baacc68_1586830094862","id":"23844521781280398"},"body_label":{"name":"placement_asset_f14cee2ab5d786_1586830094863","id":"23844521781370398"},"link_url_label":{"name":"placement_asset_f309294689f2c6c_1586830094864","id":"23844521781290398"},"title_label":{"name":"placement_asset_f1013e29f89c38_1586830094864","id":"23844521781350398"},"priority":2},{"customization_spec":{"age_max":65,"age_min":13,"publisher_platforms":["facebook"],"facebook_positions":["story"]},"image_label":{"name":"placement_asset_f2c2fe4f20af66c_1586830157386","id":"23844521783780398"},"body_label":{"name":"placement_asset_f14877915fb5acc_1586830157387","id":"23844521783760398"},"link_url_label":{"name":"placement_asset_f28a128696c7428_1586830157387","id":"23844521783790398"},"title_label":{"name":"placement_asset_f890656071c9ac_1586830157387","id":"23844521783770398"},"priority":3},{"customization_spec":{"age_max":65,"age_min":13},"image_label":{"name":"placement_asset_f1f518506ae7e68_1586830094842","id":"23844521781340398"},"body_label":{"name":"placement_asset_f2d65f15340e594_1586830094852","id":"23844521781260398"},"link_url_label":{"name":"placement_asset_f136a02466f2bc_1586830094856","id":"23844521781310398"},"title_label":{"name":"placement_asset_f1a3b3d525f4998_1586830094854","id":"23844521781380398"},"priority":4}],"optimization_type":"PLACEMENT","additional_data":{"multi_share_end_card":false,"is_click_to_message":false}},"effective_object_story_id":"112704783733939_117519556585795","name":"{{product.name}} 2020-04-21-49cbe5bd90ed9861ea68bb38f7d6fc7c","instagram_actor_id":"3437258706290825","object_story_spec":{"page_id":"112704783733939","instagram_actor_id":"3437258706290825"},"object_type":"SHARE","status":"ACTIVE","thumbnail_url":"https://scontent-dus1-1.xx.fbcdn.net/v/t45.1600-4/93287504_23844521781140398_125048020067680256_n.jpg?_nc_cat=108&ccb=1-7&_nc_sid=a3999f&_nc_ohc=-TT4Z0FkPeYAX97qejq&_nc_ht=scontent-dus1-1.xx&edm=AAT1rw8EAAAA&stp=c0.5000x0.5000f_dst-emg0_p64x64_q75&ur=58080a&oh=00_AfBjMrayWFyOLmIgVt8Owtv2fBSJVyCmtNuPLpCQyggdpg&oe=64E18154"},"emitted_at":1692180825964} -{"stream":"activities","data":{"actor_id":"10167035656105444","actor_name":"Ilana Enoukov","application_id":"119211728144504","application_name":"Power Editor","date_time_in_timezone":"03/21/2023 at 2:20 PM","event_time":"2023-03-21T21:20:39+0000","event_type":"update_ad_set_target_spec","extra_data":"{\"old_value\":[{\"content\":\"Custom audience:\",\"children\":[\"Web Traffic [ALL] - _copy\"]},{\"content\":\"Location:\",\"children\":[\"United States\"]},{\"content\":\"Age:\",\"children\":[\"18 - 60\"]},{\"content\":\"Placements:\",\"children\":[\"on pages: Feed on desktop computers, Video feeds on desktop computers, Instagram feed, Instagram Stories, Instagram Profile Feed, Instagram Explore, Instagram Explore home, Instagram Reels, Instagram Shop, Third-party apps and websites on mobile devices, Feed on mobile devices, Video feeds on mobile devices, Right column on desktop computers, Instream video on mobile devices, Instream video on desktop computers, Instant Article, Marketplace on desktop computers, Marketplace on mobile devices, Facebook Stories on mobile devices, Messenger Stories, Marketplace search on desktop devices, Marketplace search on mobile devices, Search on mobile devices, Search on desktop devices, Facebook Reels overlay on mobile devices, Facebook Business Explore on mobile devices, Facebook Reels or Video search on mobile devices\"]},{\"content\":\"Advantage custom audience:\",\"children\":[\"Off\"]}],\"new_value\":[{\"content\":\"Custom audience:\",\"children\":[\"Lookalike (US, 10\\u0025) - Airbyte Cloud Users or Web Traffic [ALL] - _copy\"]},{\"content\":\"Location:\",\"children\":[\"United States\"]},{\"content\":\"Age:\",\"children\":[\"18 - 60\"]},{\"content\":\"Placements:\",\"children\":[\"on pages: Feed on desktop computers, Video feeds on desktop computers, Instagram feed, Instagram Stories, Instagram Profile Feed, Instagram Explore, Instagram Explore home, Instagram Reels, Instagram Shop, Third-party apps and websites on mobile devices, Feed on mobile devices, Video feeds on mobile devices, Right column on desktop computers, Instream video on mobile devices, Instream video on desktop computers, Instant Article, Marketplace on desktop computers, Marketplace on mobile devices, Facebook Stories on mobile devices, Messenger Stories, Marketplace search on desktop devices, Marketplace search on mobile devices, Search on mobile devices, Search on desktop devices, Facebook Reels overlay on mobile devices, Facebook Business Explore on mobile devices, Facebook Reels or Video search on mobile devices\"]},{\"content\":\"Advantage custom audience:\",\"children\":[\"On\"]}],\"type\":\"targets_spec\"}","object_id":"23853619670380398","object_name":"Lookalike audience_Free Connector Program","object_type":"CAMPAIGN","translated_event_type":"Ad set targeting updated"},"emitted_at":1692180829460} +{"stream":"campaigns","data":{"id":"23846542053890398","account_id":"212551616838260","budget_rebalance_flag":false,"budget_remaining":0.0,"buying_type":"AUCTION","created_time":"2021-01-18T21:36:42-0800","configured_status":"PAUSED","effective_status":"PAUSED","name":"Fake Campaign 0","objective":"MESSAGES","smart_promotion_type":"GUIDED_CREATION","source_campaign_id":0.0,"special_ad_category":"NONE","start_time":"1969-12-31T15:59:59-0800","status":"PAUSED","updated_time":"2021-02-18T01:00:02-0800"},"emitted_at":1694795155769} +{"stream": "custom_audiences", "data": {"id": "23853683587660398", "account_id": "212551616838260", "approximate_count_lower_bound": 4700, "approximate_count_upper_bound": 5500, "customer_file_source": "PARTNER_PROVIDED_ONLY", "data_source": {"type": "UNKNOWN", "sub_type": "ANYTHING", "creation_params": "[]"}, "delivery_status": {"code": 200, "description": "This audience is ready for use."}, "description": "Custom Audience-Web Traffic [ALL] - _copy", "is_value_based": false, "name": "Web Traffic [ALL] - _copy", "operation_status": {"code": 200, "description": "Normal"}, "permission_for_actions": {"can_edit": true, "can_see_insight": "True", "can_share": "True", "subtype_supports_lookalike": "True", "supports_recipient_lookalike": "False"}, "retention_days": 0, "subtype": "CUSTOM", "time_content_updated": 1679433484, "time_created": 1679433479, "time_updated": 1679433484}, "emitted_at": 1698925454024} +{"stream":"ad_creatives","data":{"id":"23844568440620398","account_id":"212551616838260","actor_id":"112704783733939","asset_feed_spec":{"images":[{"adlabels":[{"name":"placement_asset_fb19ee1baacc68_1586830094862","id":"23844521781280398"}],"hash":"7394ffb578c53e8761b6498d3008725b","image_crops":{"191x100":[[0,411],[589,719]]}},{"adlabels":[{"name":"placement_asset_f1f518506ae7e68_1586830094842","id":"23844521781340398"}],"hash":"7394ffb578c53e8761b6498d3008725b","image_crops":{"100x100":[[12,282],[574,844]]}},{"adlabels":[{"name":"placement_asset_f311b79c14a30c_1586830094845","id":"23844521781330398"}],"hash":"7394ffb578c53e8761b6498d3008725b","image_crops":{"90x160":[[14,72],[562,1046]]}},{"adlabels":[{"name":"placement_asset_f2c2fe4f20af66c_1586830157386","id":"23844521783780398"}],"hash":"7394ffb578c53e8761b6498d3008725b","image_crops":{"90x160":[[0,0],[589,1047]]}}],"bodies":[{"adlabels":[{"name":"placement_asset_f2d65f15340e594_1586830094852","id":"23844521781260398"},{"name":"placement_asset_f1f97c3e3a63d74_1586830094858","id":"23844521781300398"},{"name":"placement_asset_f14cee2ab5d786_1586830094863","id":"23844521781370398"},{"name":"placement_asset_f14877915fb5acc_1586830157387","id":"23844521783760398"}],"text":""}],"call_to_action_types":["LEARN_MORE"],"descriptions":[{"text":"Unmatched attribution, ad performances, and lead conversion, by unlocking your ad-blocked traffic across all your tools."}],"link_urls":[{"adlabels":[{"name":"placement_asset_f309294689f2c6c_1586830094864","id":"23844521781290398"},{"name":"placement_asset_f136a02466f2bc_1586830094856","id":"23844521781310398"},{"name":"placement_asset_fa79b032b68274_1586830094860","id":"23844521781320398"},{"name":"placement_asset_f28a128696c7428_1586830157387","id":"23844521783790398"}],"website_url":"http://dataline.io/","display_url":""}],"titles":[{"adlabels":[{"name":"placement_asset_f1013e29f89c38_1586830094864","id":"23844521781350398"},{"name":"placement_asset_fcb53b78a11574_1586830094859","id":"23844521781360398"},{"name":"placement_asset_f1a3b3d525f4998_1586830094854","id":"23844521781380398"},{"name":"placement_asset_f890656071c9ac_1586830157387","id":"23844521783770398"}],"text":"Unblock all your adblocked traffic"}],"ad_formats":["AUTOMATIC_FORMAT"],"asset_customization_rules":[{"customization_spec":{"age_max":65,"age_min":13,"publisher_platforms":["instagram","audience_network","messenger"],"instagram_positions":["story"],"messenger_positions":["story"],"audience_network_positions":["classic"]},"image_label":{"name":"placement_asset_f311b79c14a30c_1586830094845","id":"23844521781330398"},"body_label":{"name":"placement_asset_f1f97c3e3a63d74_1586830094858","id":"23844521781300398"},"link_url_label":{"name":"placement_asset_fa79b032b68274_1586830094860","id":"23844521781320398"},"title_label":{"name":"placement_asset_fcb53b78a11574_1586830094859","id":"23844521781360398"},"priority":1},{"customization_spec":{"age_max":65,"age_min":13,"publisher_platforms":["facebook"],"facebook_positions":["right_hand_column","instant_article","search"]},"image_label":{"name":"placement_asset_fb19ee1baacc68_1586830094862","id":"23844521781280398"},"body_label":{"name":"placement_asset_f14cee2ab5d786_1586830094863","id":"23844521781370398"},"link_url_label":{"name":"placement_asset_f309294689f2c6c_1586830094864","id":"23844521781290398"},"title_label":{"name":"placement_asset_f1013e29f89c38_1586830094864","id":"23844521781350398"},"priority":2},{"customization_spec":{"age_max":65,"age_min":13,"publisher_platforms":["facebook"],"facebook_positions":["story"]},"image_label":{"name":"placement_asset_f2c2fe4f20af66c_1586830157386","id":"23844521783780398"},"body_label":{"name":"placement_asset_f14877915fb5acc_1586830157387","id":"23844521783760398"},"link_url_label":{"name":"placement_asset_f28a128696c7428_1586830157387","id":"23844521783790398"},"title_label":{"name":"placement_asset_f890656071c9ac_1586830157387","id":"23844521783770398"},"priority":3},{"customization_spec":{"age_max":65,"age_min":13},"image_label":{"name":"placement_asset_f1f518506ae7e68_1586830094842","id":"23844521781340398"},"body_label":{"name":"placement_asset_f2d65f15340e594_1586830094852","id":"23844521781260398"},"link_url_label":{"name":"placement_asset_f136a02466f2bc_1586830094856","id":"23844521781310398"},"title_label":{"name":"placement_asset_f1a3b3d525f4998_1586830094854","id":"23844521781380398"},"priority":4}],"optimization_type":"PLACEMENT","reasons_to_shop":false,"shops_bundle":false,"additional_data":{"multi_share_end_card":false,"is_click_to_message":false}},"effective_object_story_id":"112704783733939_117519556585795","name":"{{product.name}} 2020-04-21-49cbe5bd90ed9861ea68bb38f7d6fc7c","instagram_actor_id":"3437258706290825","object_story_spec":{"page_id":"112704783733939","instagram_actor_id":"3437258706290825"},"object_type":"SHARE","status":"ACTIVE","thumbnail_url":"https://scontent-dus1-1.xx.fbcdn.net/v/t45.1600-4/93287504_23844521781140398_125048020067680256_n.jpg?_nc_cat=108&ccb=1-7&_nc_sid=a3999f&_nc_ohc=-TT4Z0FkPeYAX97qejq&_nc_ht=scontent-dus1-1.xx&edm=AAT1rw8EAAAA&stp=c0.5000x0.5000f_dst-emg0_p64x64_q75&ur=58080a&oh=00_AfBjMrayWFyOLmIgVt8Owtv2fBSJVyCmtNuPLpCQyggdpg&oe=64E18154"},"emitted_at":1692180825964} +{"stream":"activities","data":{"account_id":"212551616838260","actor_id":"122043039268043192","actor_name":"Payments RTU Processor","application_id":"0","date_time_in_timezone":"03/13/2023 at 6:30 AM","event_time":"2023-03-13T13:30:47+0000","event_type":"ad_account_billing_charge","extra_data":"{\"currency\":\"USD\",\"new_value\":1188,\"transaction_id\":\"5885578541558696-11785530\",\"action\":67,\"type\":\"payment_amount\"}","object_id":"212551616838260","object_name":"Airbyte","object_type":"ACCOUNT","translated_event_type":"Account billed"},"emitted_at":1696931251153} {"stream":"custom_conversions","data":{"id":"694166388077667","account_id":"212551616838260","creation_time":"2020-04-22T01:36:00+0000","custom_event_type":"CONTACT","data_sources":[{"id":"2667253716886462","source_type":"PIXEL","name":"Dataline's Pixel"}],"default_conversion_value":0,"event_source_type":"pixel","is_archived":true,"is_unavailable":false,"name":"SubscribedButtonClick","retention_days":0,"rule":"{\"and\":[{\"event\":{\"eq\":\"PageView\"}},{\"or\":[{\"URL\":{\"i_contains\":\"SubscribedButtonClick\"}}]}]}"},"emitted_at":1692180839174} {"stream":"images","data":{"id":"212551616838260:c1e94a8768a405f0f212d71fe8336647","account_id":"212551616838260","name":"Audience_1_Ad_3_1200x1200_blue_CTA_arrow.png_105","creatives":["23853630775340398","23853630871360398","23853666124200398"],"original_height":1200,"original_width":1200,"permalink_url":"https://www.facebook.com/ads/image/?d=AQIDNjjLb7VzVJ26jXb_HpudCEUJqbV_lLF2JVsdruDcBxnXQEKfzzd21VVJnkm0B-JLosUXNNg1BH78y7FxnK3AH-0D_lnk7kn39_bIcOMK7Z9HYyFInfsVY__adup3A5zGTIcHC9Y98Je5qK-yD8F6","status":"ACTIVE","url":"https://scontent-dus1-1.xx.fbcdn.net/v/t45.1600-4/335907140_23853620220420398_4375584095210967511_n.png?_nc_cat=104&ccb=1-7&_nc_sid=2aac32&_nc_ohc=xdjrPpbRGNAAX8Dck01&_nc_ht=scontent-dus1-1.xx&edm=AJcBmwoEAAAA&oh=00_AfDCqQ6viqrgLcfbO3O5-n030Usq7Zyt2c1TmsatqnYf7Q&oe=64E2779A","created_time":"2023-03-16T13:13:17-0700","hash":"c1e94a8768a405f0f212d71fe8336647","url_128":"https://scontent-dus1-1.xx.fbcdn.net/v/t45.1600-4/335907140_23853620220420398_4375584095210967511_n.png?stp=dst-png_s128x128&_nc_cat=104&ccb=1-7&_nc_sid=2aac32&_nc_ohc=xdjrPpbRGNAAX8Dck01&_nc_ht=scontent-dus1-1.xx&edm=AJcBmwoEAAAA&oh=00_AfAY50CMpox2s4w_f18IVx7sZuXlg4quF6YNIJJ8D4PZew&oe=64E2779A","is_associated_creatives_in_adgroups":true,"updated_time":"2023-03-17T08:09:56-0700","height":1200,"width":1200},"emitted_at":1692180839582} -{"stream":"ads_insights", "data": {"account_currency": "USD", "account_id": "212551616838260", "account_name": "Airbyte", "actions": [{"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "link_click", "value": 3.0, "1d_click": 3.0, "7d_click": 3.0, "28d_click": 3.0}, {"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "page_engagement", "value": 3.0, "1d_click": 3.0, "7d_click": 3.0, "28d_click": 3.0}, {"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "post_engagement", "value": 3.0, "1d_click": 3.0, "7d_click": 3.0, "28d_click": 3.0}], "ad_id": "23846765228310398", "ad_name": "Airbyte Ad", "adset_id": "23846765228280398", "adset_name": "Vanilla awareness ad set", "buying_type": "AUCTION", "campaign_id": "23846765228240398", "campaign_name": "Airbyte Awareness Campaign 1 (sherif)", "catalog_segment_value_mobile_purchase_roas": [{"value": 0.0}], "clicks": 3, "conversion_rate_ranking": "UNKNOWN", "cost_per_estimated_ad_recallers": 0.007, "cost_per_inline_link_click": 0.396667, "cost_per_inline_post_engagement": 0.396667, "cost_per_unique_click": 0.396667, "cost_per_unique_inline_link_click": 0.396667, "cpc": 0.396667, "cpm": 0.902199, "cpp": 0.948207, "created_time": "2021-02-09", "ctr": 0.227445, "date_start": "2021-02-15", "date_stop": "2021-02-15", "engagement_rate_ranking": "UNKNOWN", "estimated_ad_recall_rate": 13.545817, "estimated_ad_recallers": 170.0, "frequency": 1.050996, "impressions": 1319, "inline_link_click_ctr": 0.227445, "inline_link_clicks": 3, "inline_post_engagement": 3, "mobile_app_purchase_roas": [{"value": 0.0}], "objective": "BRAND_AWARENESS", "optimization_goal": "AD_RECALL_LIFT", "outbound_clicks": [{"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "outbound_click", "value": 3.0}], "quality_ranking": "UNKNOWN", "reach": 1255, "social_spend": 0.0, "spend": 1.19, "unique_actions": [{"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "link_click", "value": 3.0, "1d_click": 3.0, "7d_click": 3.0, "28d_click": 3.0}, {"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "page_engagement", "value": 3.0, "1d_click": 3.0, "7d_click": 3.0, "28d_click": 3.0}, {"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "post_engagement", "value": 3.0, "1d_click": 3.0, "7d_click": 3.0, "28d_click": 3.0}], "unique_clicks": 3, "unique_ctr": 0.239044, "unique_inline_link_click_ctr": 0.239044, "unique_inline_link_clicks": 3, "unique_link_clicks_ctr": 0.239044, "unique_outbound_clicks": [{"action_destination": "244953057175777", "action_target_id": "244953057175777", "action_type": "outbound_click", "value": 3.0}], "updated_time": "2021-08-27", "website_ctr": [{"action_type": "link_click", "value": 0.227445}], "website_purchase_roas": [{"value": 0.0}], "wish_bid": 0.0}, "emitted_at": 1682686057366} -{"stream":"ads_insights_action_carousel_card","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":3,"conversion_rate_ranking":"UNKNOWN","cost_per_estimated_ad_recallers":0.007,"cost_per_inline_link_click":0.396667,"cost_per_inline_post_engagement":0.396667,"cost_per_unique_click":0.396667,"cost_per_unique_inline_link_click":0.396667,"cpc":0.396667,"cpm":0.902199,"cpp":0.948207,"created_time":"2021-02-09","ctr":0.227445,"date_start":"2021-02-15","date_stop":"2021-02-15","engagement_rate_ranking":"UNKNOWN","estimated_ad_recall_rate":13.545817,"estimated_ad_recallers":170,"frequency":1.050996,"impressions":1319,"inline_link_click_ctr":0.227445,"inline_link_clicks":3,"inline_post_engagement":3,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","quality_ranking":"UNKNOWN","reach":1255,"social_spend":0,"spend":1.19,"unique_actions":[{"value":3,"1d_click":3,"7d_click":3,"28d_click":3}],"unique_clicks":3,"unique_ctr":0.239044,"unique_inline_link_click_ctr":0.239044,"unique_inline_link_clicks":3,"updated_time":"2021-08-27","website_ctr":[{"action_type":"link_click","value":0.227445}],"website_purchase_roas":[{"value":0}],"wish_bid":0},"emitted_at":1692180857757} -{"stream":"ads_insights_action_conversion_device","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":0,"cost_per_estimated_ad_recallers":0.01,"cpm":1.470588,"cpp":1.492537,"created_time":"2021-02-09","ctr":0,"date_start":"2021-02-11","date_stop":"2021-02-11","estimated_ad_recall_rate":14.925373,"estimated_ad_recallers":10,"frequency":1.014925,"impressions":68,"inline_link_clicks":0,"inline_post_engagement":0,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":67,"spend":0.1,"unique_clicks":0,"unique_ctr":0,"unique_inline_link_clicks":0,"unique_link_clicks_ctr":0,"updated_time":"2021-08-27","website_purchase_roas":[{"value":0}],"device_platform":"desktop"},"emitted_at":1692180864186} -{"stream":"ads_insights_action_reaction","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"conversion_rate_ranking":"UNKNOWN","created_time":"2021-02-09","date_start":"2021-02-08","date_stop":"2021-02-08","engagement_rate_ranking":"UNKNOWN","frequency":0,"impressions":0,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","quality_ranking":"UNKNOWN","reach":0,"social_spend":0,"spend":0,"unique_clicks":0,"updated_time":"2021-08-27","wish_bid":0},"emitted_at":1692180909192} -{"stream":"ads_insights_action_type","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","actions":[{"action_type":"link_click","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"page_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"post_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2}],"ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":2,"conversion_rate_ranking":"UNKNOWN","cost_per_action_type":[{"action_type":"link_click","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23},{"action_type":"page_engagement","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23},{"action_type":"post_engagement","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23}],"cost_per_estimated_ad_recallers":0.006571,"cost_per_inline_link_click":0.23,"cost_per_inline_post_engagement":0.23,"cost_per_outbound_click":[{"action_type":"outbound_click","value":0.23}],"cost_per_unique_action_type":[{"action_type":"link_click","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23},{"action_type":"page_engagement","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23},{"action_type":"post_engagement","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23}],"cost_per_unique_click":0.23,"cost_per_unique_inline_link_click":0.23,"cost_per_unique_outbound_click":[{"action_type":"outbound_click","value":0.23}],"cpc":0.23,"cpm":0.946502,"cpp":0.964361,"created_time":"2021-02-09","ctr":0.411523,"date_start":"2021-02-11","date_stop":"2021-02-11","engagement_rate_ranking":"UNKNOWN","estimated_ad_recall_rate":14.675052,"estimated_ad_recallers":70,"frequency":1.018868,"impressions":486,"inline_link_click_ctr":0.411523,"inline_link_clicks":2,"inline_post_engagement":2,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","outbound_clicks":[{"action_type":"outbound_click","value":2}],"outbound_clicks_ctr":[{"action_type":"outbound_click","value":0.411523}],"quality_ranking":"UNKNOWN","reach":477,"social_spend":0,"spend":0.46,"unique_actions":[{"action_type":"link_click","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"page_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"post_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2}],"unique_clicks":2,"unique_ctr":0.419287,"unique_inline_link_click_ctr":0.419287,"unique_inline_link_clicks":2,"unique_link_clicks_ctr":0.419287,"unique_outbound_clicks":[{"action_type":"outbound_click","value":2}],"unique_outbound_clicks_ctr":[{"action_type":"outbound_click","value":0.419287}],"updated_time":"2021-08-27","website_ctr":[{"action_type":"link_click","value":0.411523}],"website_purchase_roas":[{"value":0}],"wish_bid":0},"emitted_at":1692180948892} -{"stream":"ads_insights_action_video_sound","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"conversion_rate_ranking":"UNKNOWN","created_time":"2021-02-09","date_start":"2021-02-08","date_stop":"2021-02-08","engagement_rate_ranking":"UNKNOWN","frequency":0,"impressions":0,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","quality_ranking":"UNKNOWN","reach":0,"social_spend":0,"spend":0,"unique_clicks":0,"updated_time":"2021-08-27","wish_bid":0},"emitted_at":1692180958086} -{"stream":"ads_insights_action_video_type","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"conversion_rate_ranking":"UNKNOWN","created_time":"2021-02-09","date_start":"2021-02-08","date_stop":"2021-02-08","engagement_rate_ranking":"UNKNOWN","frequency":0,"impressions":0,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","quality_ranking":"UNKNOWN","reach":0,"social_spend":0,"spend":0,"unique_clicks":0,"updated_time":"2021-08-27","wish_bid":0},"emitted_at":1692180966403} -{"stream":"ads_insights_age_and_gender","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":0,"cost_per_estimated_ad_recallers":0.01,"cpm":0.714286,"cpp":0.769231,"created_time":"2021-02-09","ctr":0,"date_start":"2021-02-11","date_stop":"2021-02-11","estimated_ad_recall_rate":7.692308,"estimated_ad_recallers":1,"frequency":1.076923,"gender_targeting":"female","impressions":14,"inline_link_clicks":0,"inline_post_engagement":0,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":13,"spend":0.01,"unique_clicks":0,"unique_ctr":0,"unique_inline_link_clicks":0,"unique_link_clicks_ctr":0,"updated_time":"2021-08-27","website_purchase_roas":[{"value":0}],"age":"18-24","gender":"female"},"emitted_at":1692180975689} -{"stream":"ads_insights_country","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":0,"cost_per_estimated_ad_recallers":0.008667,"cpm":1.293532,"cpp":1.381142,"created_time":"2021-02-09","ctr":0,"date_start":"2021-02-12","date_stop":"2021-02-12","estimated_ad_recall_rate":15.936255,"estimated_ad_recallers":120,"frequency":1.067729,"impressions":804,"inline_link_clicks":0,"inline_post_engagement":0,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":753,"spend":1.04,"unique_clicks":0,"unique_ctr":0,"unique_inline_link_clicks":0,"unique_link_clicks_ctr":0,"updated_time":"2021-08-27","website_purchase_roas":[{"value":0}],"country":"US"},"emitted_at":1692180985386} -{"stream":"ads_insights_delivery_device","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":0,"cost_per_estimated_ad_recallers":0.01,"cpm":1.470588,"cpp":1.492537,"created_time":"2021-02-09","ctr":0,"date_start":"2021-02-11","date_stop":"2021-02-11","estimated_ad_recall_rate":14.925373,"estimated_ad_recallers":10,"frequency":1.014925,"impressions":68,"inline_link_clicks":0,"inline_post_engagement":0,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":67,"spend":0.1,"unique_clicks":0,"unique_ctr":0,"unique_inline_link_clicks":0,"unique_link_clicks_ctr":0,"updated_time":"2021-08-27","website_purchase_roas":[{"value":0}],"device_platform":"desktop"},"emitted_at":1692180993524} -{"stream":"ads_insights_delivery_platform","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","actions":[{"action_type":"link_click","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"page_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"post_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2}],"ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":2,"cost_per_action_type":[{"action_type":"link_click","value":0.225,"1d_click":0.225,"7d_click":0.225,"28d_click":0.225},{"action_type":"page_engagement","value":0.225,"1d_click":0.225,"7d_click":0.225,"28d_click":0.225},{"action_type":"post_engagement","value":0.225,"1d_click":0.225,"7d_click":0.225,"28d_click":0.225}],"cost_per_estimated_ad_recallers":0.0075,"cost_per_inline_link_click":0.225,"cost_per_inline_post_engagement":0.225,"cost_per_outbound_click":[{"action_type":"outbound_click","value":0.225}],"cost_per_unique_action_type":[{"action_type":"link_click","value":0.225,"1d_click":0.225,"7d_click":0.225,"28d_click":0.225},{"action_type":"page_engagement","value":0.225,"1d_click":0.225,"7d_click":0.225,"28d_click":0.225},{"action_type":"post_engagement","value":0.225,"1d_click":0.225,"7d_click":0.225,"28d_click":0.225}],"cost_per_unique_click":0.225,"cost_per_unique_inline_link_click":0.225,"cost_per_unique_outbound_click":[{"action_type":"outbound_click","value":0.225}],"cpc":0.225,"cpm":1.034483,"cpp":1.056338,"created_time":"2021-02-09","ctr":0.45977,"date_start":"2021-02-11","date_stop":"2021-02-11","estimated_ad_recall_rate":14.084507,"estimated_ad_recallers":60,"frequency":1.021127,"impressions":435,"inline_link_click_ctr":0.45977,"inline_link_clicks":2,"inline_post_engagement":2,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","outbound_clicks":[{"action_type":"outbound_click","value":2}],"outbound_clicks_ctr":[{"action_type":"outbound_click","value":0.45977}],"reach":426,"spend":0.45,"unique_actions":[{"action_type":"link_click","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"page_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"post_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2}],"unique_clicks":2,"unique_ctr":0.469484,"unique_inline_link_click_ctr":0.469484,"unique_inline_link_clicks":2,"unique_link_clicks_ctr":0.469484,"unique_outbound_clicks":[{"action_type":"outbound_click","value":2}],"unique_outbound_clicks_ctr":[{"action_type":"outbound_click","value":0.469484}],"updated_time":"2021-08-27","website_ctr":[{"action_type":"link_click","value":0.45977}],"website_purchase_roas":[{"value":0}],"publisher_platform":"facebook"},"emitted_at":1692181004988} -{"stream":"ads_insights_delivery_platform_and_device_platform","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":0,"cost_per_estimated_ad_recallers":0.01,"cpm":1.470588,"cpp":1.492537,"created_time":"2021-02-09","ctr":0,"date_start":"2021-02-11","date_stop":"2021-02-11","estimated_ad_recall_rate":14.925373,"estimated_ad_recallers":10,"frequency":1.014925,"impressions":68,"inline_link_clicks":0,"inline_post_engagement":0,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":67,"spend":0.1,"unique_clicks":0,"unique_ctr":0,"unique_inline_link_clicks":0,"unique_link_clicks_ctr":0,"updated_time":"2021-08-27","website_purchase_roas":[{"value":0}],"publisher_platform":"facebook","device_platform":"desktop"},"emitted_at":1692181043593} -{"stream":"ads_insights_demographics_age","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":0,"cost_per_estimated_ad_recallers":0.0045,"cpm":0.633803,"cpp":0.656934,"created_time":"2021-02-09","ctr":0,"date_start":"2021-02-11","date_stop":"2021-02-11","estimated_ad_recall_rate":14.59854,"estimated_ad_recallers":20,"frequency":1.036496,"impressions":142,"inline_link_clicks":0,"inline_post_engagement":0,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":137,"spend":0.09,"unique_clicks":0,"unique_ctr":0,"unique_inline_link_clicks":0,"unique_link_clicks_ctr":0,"updated_time":"2021-08-27","website_purchase_roas":[{"value":0}],"age":"18-24"},"emitted_at":1692181051722} -{"stream":"ads_insights_demographics_country","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","actions":[{"action_type":"link_click","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"page_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"post_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2}],"ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":2,"cost_per_action_type":[{"action_type":"link_click","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23},{"action_type":"page_engagement","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23},{"action_type":"post_engagement","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23}],"cost_per_estimated_ad_recallers":0.006571,"cost_per_inline_link_click":0.23,"cost_per_inline_post_engagement":0.23,"cost_per_outbound_click":[{"action_type":"outbound_click","value":0.23}],"cost_per_unique_action_type":[{"action_type":"link_click","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23},{"action_type":"page_engagement","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23},{"action_type":"post_engagement","value":0.23,"1d_click":0.23,"7d_click":0.23,"28d_click":0.23}],"cost_per_unique_click":0.23,"cost_per_unique_inline_link_click":0.23,"cost_per_unique_outbound_click":[{"action_type":"outbound_click","value":0.23}],"cpc":0.23,"cpm":0.946502,"cpp":0.964361,"created_time":"2021-02-09","ctr":0.411523,"date_start":"2021-02-11","date_stop":"2021-02-11","estimated_ad_recall_rate":14.675052,"estimated_ad_recallers":70,"frequency":1.018868,"impressions":486,"inline_link_click_ctr":0.411523,"inline_link_clicks":2,"inline_post_engagement":2,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","outbound_clicks":[{"action_type":"outbound_click","value":2}],"outbound_clicks_ctr":[{"action_type":"outbound_click","value":0.411523}],"reach":477,"spend":0.46,"unique_actions":[{"action_type":"link_click","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"page_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2},{"action_type":"post_engagement","value":2,"1d_click":2,"7d_click":2,"28d_click":2}],"unique_clicks":2,"unique_ctr":0.419287,"unique_inline_link_click_ctr":0.419287,"unique_inline_link_clicks":2,"unique_link_clicks_ctr":0.419287,"unique_outbound_clicks":[{"action_type":"outbound_click","value":2}],"unique_outbound_clicks_ctr":[{"action_type":"outbound_click","value":0.419287}],"updated_time":"2021-08-27","website_ctr":[{"action_type":"link_click","value":0.411523}],"website_purchase_roas":[{"value":0}],"country":"US"},"emitted_at":1692181061009} -{"stream":"ads_insights_demographics_dma_region","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"cost_per_estimated_ad_recallers":0,"cpm":0,"created_time":"2021-02-09","ctr":0,"date_start":"2021-02-12","date_stop":"2021-02-12","estimated_ad_recallers":1,"frequency":1.333333,"impressions":4,"inline_link_clicks":0,"inline_post_engagement":0,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","outbound_clicks_ctr":[{"value":0}],"reach":3,"spend":0,"unique_clicks":0,"unique_outbound_clicks_ctr":[{"value":0}],"updated_time":"2021-08-27","dma":"Abilene-Sweetwater"},"emitted_at":1692181100592} -{"stream":"ads_insights_demographics_gender","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":0,"cost_per_estimated_ad_recallers":0.008,"cpm":1.333333,"cpp":1.37931,"created_time":"2021-02-09","ctr":0,"date_start":"2021-02-11","date_stop":"2021-02-11","estimated_ad_recall_rate":17.241379,"estimated_ad_recallers":10,"frequency":1.034483,"gender_targeting":"female","impressions":60,"inline_link_clicks":0,"inline_post_engagement":0,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":58,"spend":0.08,"unique_clicks":0,"unique_ctr":0,"unique_inline_link_clicks":0,"unique_link_clicks_ctr":0,"updated_time":"2021-08-27","website_purchase_roas":[{"value":0}],"gender":"female"},"emitted_at":1692181125390} -{"stream":"ads_insights_dma","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"cost_per_estimated_ad_recallers":0,"cpm":0,"created_time":"2021-02-09","ctr":0,"date_start":"2021-02-12","date_stop":"2021-02-12","estimated_ad_recallers":1,"frequency":1.333333,"impressions":4,"inline_link_clicks":0,"inline_post_engagement":0,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":3,"spend":0,"unique_clicks":0,"updated_time":"2021-08-27","dma":"Abilene-Sweetwater"},"emitted_at":1692181165207} -{"stream":"ads_insights_platform_and_device","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","catalog_segment_value_mobile_purchase_roas":[{"value":0}],"clicks":0,"cost_per_estimated_ad_recallers":0.007,"cpm":1.627907,"cpp":1.627907,"created_time":"2021-02-09","ctr":0,"date_start":"2021-02-11","date_stop":"2021-02-11","estimated_ad_recall_rate":23.255814,"estimated_ad_recallers":10,"frequency":1,"impressions":43,"inline_link_clicks":0,"inline_post_engagement":0,"mobile_app_purchase_roas":[{"value":0}],"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":43,"spend":0.07,"unique_clicks":0,"unique_ctr":0,"unique_inline_link_clicks":0,"unique_link_clicks_ctr":0,"updated_time":"2021-08-27","website_purchase_roas":[{"value":0}],"publisher_platform":"facebook","platform_position":"feed","impression_device":"android_smartphone"},"emitted_at":1692181188895} -{"stream":"ads_insights_region","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"cost_per_estimated_ad_recallers":0,"cpm":0,"cpp":0,"created_time":"2021-02-09","ctr":0,"date_start":"2021-02-11","date_stop":"2021-02-11","estimated_ad_recall_rate":8.333333,"estimated_ad_recallers":1,"frequency":1,"impressions":12,"inline_link_clicks":0,"inline_post_engagement":0,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":12,"spend":0,"unique_clicks":0,"unique_ctr":0,"unique_inline_link_clicks":0,"unique_link_clicks_ctr":0,"updated_time":"2021-08-27","region":"Alabama"},"emitted_at":1692181230021} +{"stream":"ads_insights","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","actions":[{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"page_engagement","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0},{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"post_engagement","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0},{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"link_click","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0}],"ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":3,"conversion_rate_ranking":"UNKNOWN","cost_per_estimated_ad_recallers":0.007,"cost_per_inline_link_click":0.396667,"cost_per_inline_post_engagement":0.396667,"cost_per_unique_click":0.396667,"cost_per_unique_inline_link_click":0.396667,"cpc":0.396667,"cpm":0.902199,"cpp":0.948207,"created_time":"2021-02-09","ctr":0.227445,"date_start":"2021-02-15","date_stop":"2021-02-15","engagement_rate_ranking":"UNKNOWN","estimated_ad_recall_rate":13.545817,"estimated_ad_recallers":170.0,"frequency":1.050996,"impressions":1319,"inline_link_click_ctr":0.227445,"inline_link_clicks":3,"inline_post_engagement":3,"instant_experience_clicks_to_open":1.0,"instant_experience_clicks_to_start":1.0,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","outbound_clicks":[{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"outbound_click","value":3.0}],"quality_ranking":"UNKNOWN","reach":1255,"social_spend":0.0,"spend":1.19,"unique_actions":[{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"page_engagement","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0},{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"post_engagement","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0},{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"link_click","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0}],"unique_clicks":3,"unique_ctr":0.239044,"unique_inline_link_click_ctr":0.239044,"unique_inline_link_clicks":3,"unique_link_clicks_ctr":0.239044,"unique_outbound_clicks":[{"action_destination":"244953057175777","action_target_id":"244953057175777","action_type":"outbound_click","value":3.0}],"updated_time":"2021-08-27","video_play_curve_actions":[{"action_type":"video_view"}],"website_ctr":[{"action_type":"link_click","value":0.227445}],"wish_bid":0.0},"emitted_at":1682686057366} +{"stream":"ads_insights_action_carousel_card","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":3,"conversion_rate_ranking":"UNKNOWN","cost_per_estimated_ad_recallers":0.007,"cost_per_inline_link_click":0.396667,"cost_per_inline_post_engagement":0.396667,"cost_per_unique_click":0.396667,"cost_per_unique_inline_link_click":0.396667,"cpc":0.396667,"cpm":0.902199,"cpp":0.948207,"created_time":"2021-02-09","ctr":0.227445,"date_start":"2021-02-15","date_stop":"2021-02-15","engagement_rate_ranking":"UNKNOWN","estimated_ad_recall_rate":13.545817,"estimated_ad_recallers":170.0,"frequency":1.050996,"impressions":1319,"inline_link_click_ctr":0.227445,"inline_post_engagement":3,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","quality_ranking":"UNKNOWN","reach":1255,"social_spend":0.0,"spend":1.19,"unique_actions":[{"action_type":"page_engagement","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0},{"action_type":"post_engagement","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0},{"action_type":"link_click","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0}],"unique_clicks":3,"unique_ctr":0.239044,"unique_inline_link_click_ctr":0.239044,"unique_inline_link_clicks":3,"updated_time":"2021-08-27","video_play_curve_actions":[{"action_type":"video_view"}],"website_ctr":[{"action_type":"link_click","value":0.227445}],"wish_bid":0.0},"emitted_at":1692180857757} +{"stream":"ads_insights_action_conversion_device","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846784938030398","ad_name":"Stock photo ad 2","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"cost_per_estimated_ad_recallers":0.004,"cpm":0.754717,"cpp":0.784314,"created_time":"2021-02-11","ctr":0.0,"date_start":"2021-02-15","date_stop":"2021-02-15","estimated_ad_recall_rate":19.607843,"estimated_ad_recallers":10.0,"frequency":1.039216,"impressions":53,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":51,"spend":0.04,"unique_clicks":0,"updated_time":"2021-08-27","device_platform":"desktop"},"emitted_at":1696936270620} +{"stream":"ads_insights_action_reaction","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846784938030398","ad_name":"Stock photo ad 2","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":1,"conversion_rate_ranking":"UNKNOWN","cost_per_estimated_ad_recallers":0.008889,"cost_per_unique_click":0.8,"cpc":0.8,"cpm":1.255887,"cpp":1.296596,"created_time":"2021-02-11","ctr":0.156986,"date_start":"2021-02-14","date_stop":"2021-02-14","engagement_rate_ranking":"UNKNOWN","estimated_ad_recall_rate":14.58671,"estimated_ad_recallers":90.0,"frequency":1.032415,"impressions":637,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","quality_ranking":"UNKNOWN","reach":617,"social_spend":0.0,"spend":0.8,"unique_clicks":1,"unique_ctr":0.162075,"updated_time":"2021-08-27","video_play_curve_actions":[{"action_type":"video_view"}],"wish_bid":0.0},"emitted_at":1696936287351} +{"stream":"ads_insights_action_type","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846784938030398","ad_name":"Stock photo ad 2","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":1,"conversion_rate_ranking":"UNKNOWN","cost_per_estimated_ad_recallers":0.008889,"cost_per_unique_click":0.8,"cpc":0.8,"cpm":1.255887,"cpp":1.296596,"created_time":"2021-02-11","ctr":0.156986,"date_start":"2021-02-14","date_stop":"2021-02-14","engagement_rate_ranking":"UNKNOWN","estimated_ad_recall_rate":14.58671,"estimated_ad_recallers":90.0,"frequency":1.032415,"impressions":637,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","quality_ranking":"UNKNOWN","reach":617,"social_spend":0.0,"spend":0.8,"unique_clicks":1,"unique_ctr":0.162075,"updated_time":"2021-08-27","video_play_curve_actions":[{"action_type":"video_view"}],"wish_bid":0.0},"emitted_at":1696936315908} +{"stream":"ads_insights_action_video_sound","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846784938030398","ad_name":"Stock photo ad 2","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":1,"conversion_rate_ranking":"UNKNOWN","cost_per_estimated_ad_recallers":0.008889,"cost_per_unique_click":0.8,"cpc":0.8,"cpm":1.255887,"cpp":1.296596,"created_time":"2021-02-11","ctr":0.156986,"date_start":"2021-02-14","date_stop":"2021-02-14","engagement_rate_ranking":"UNKNOWN","estimated_ad_recall_rate":14.58671,"estimated_ad_recallers":90.0,"frequency":1.032415,"impressions":637,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","quality_ranking":"UNKNOWN","reach":617,"social_spend":0.0,"spend":0.8,"unique_clicks":1,"unique_ctr":0.162075,"updated_time":"2021-08-27","video_play_curve_actions": [{"action_type": "video_view"}],"wish_bid":0.0},"emitted_at":1696936296894} +{"stream":"ads_insights_action_video_type","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846784938030398","ad_name":"Stock photo ad 2","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":1,"conversion_rate_ranking":"UNKNOWN","cost_per_estimated_ad_recallers":0.008889,"cost_per_unique_click":0.8,"cpc":0.8,"cpm":1.255887,"cpp":1.296596,"created_time":"2021-02-11","ctr":0.156986,"date_start":"2021-02-14","date_stop":"2021-02-14","engagement_rate_ranking":"UNKNOWN","estimated_ad_recall_rate":14.58671,"estimated_ad_recallers":90.0,"frequency":1.032415,"impressions":637,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","quality_ranking":"UNKNOWN","reach":617,"social_spend":0.0,"spend":0.8,"unique_clicks":1,"unique_ctr":0.162075,"updated_time":"2021-08-27","video_play_curve_actions":[{"action_type":"video_view"}],"wish_bid":0.0},"emitted_at":1696936306631} +{"stream":"ads_insights_age_and_gender","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846784938030398","ad_name":"Stock photo ad 2","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"cost_per_estimated_ad_recallers":0.02,"cpm":0.869565,"cpp":0.952381,"created_time":"2021-02-11","ctr":0.0,"date_start":"2021-02-15","date_stop":"2021-02-15","estimated_ad_recall_rate":4.761905,"estimated_ad_recallers":1.0,"frequency":1.095238,"gender_targeting":"female","impressions":23,"instant_experience_clicks_to_open":1.0,"instant_experience_clicks_to_start":1.0,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":21,"spend":0.02,"unique_clicks":0,"updated_time":"2021-08-27","age":"55-64","gender":"female"},"emitted_at":1696939548058} +{"stream":"ads_insights_country","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846784938030398","ad_name":"Stock photo ad 2","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":1,"cost_per_estimated_ad_recallers":0.008889,"cost_per_unique_click":0.8,"cpc":0.8,"cpm":1.255887,"cpp":1.296596,"created_time":"2021-02-11","ctr":0.156986,"date_start":"2021-02-14","date_stop":"2021-02-14","estimated_ad_recall_rate":14.58671,"estimated_ad_recallers":90.0,"frequency":1.032415,"impressions":637,"instant_experience_clicks_to_open":1.0,"instant_experience_clicks_to_start":1.0,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":617,"spend":0.8,"unique_clicks":1,"unique_ctr":0.162075,"updated_time":"2021-08-27","country":"US"},"emitted_at":1696936565587} +{"stream":"ads_insights_delivery_device","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"cost_per_estimated_ad_recallers":0.0075,"cpm":1.630435,"cpp":1.744186,"created_time":"2021-02-09","ctr":0.0,"date_start":"2021-02-15","date_stop":"2021-02-15","estimated_ad_recall_rate":23.255814,"estimated_ad_recallers":20.0,"frequency":1.069767,"impressions":92,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":86,"spend":0.15,"unique_clicks":0,"updated_time":"2021-08-27","device_platform":"desktop"},"emitted_at":1696936327621} +{"stream":"ads_insights_delivery_platform","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","actions":[{"action_type":"page_engagement","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0},{"action_type":"post_engagement","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0},{"action_type":"link_click","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0}],"ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":3,"cost_per_action_type":[{"action_type":"post_engagement","value":0.39,"1d_click":0.39,"7d_click":0.39,"28d_click":0.39},{"action_type":"page_engagement","value":0.39,"1d_click":0.39,"7d_click":0.39,"28d_click":0.39},{"action_type":"link_click","value":0.39,"1d_click":0.39,"7d_click":0.39,"28d_click":0.39}],"cost_per_estimated_ad_recallers":0.006882,"cost_per_inline_link_click":0.39,"cost_per_inline_post_engagement":0.39,"cost_per_outbound_click":[{"action_type":"outbound_click","value":0.39}],"cost_per_unique_action_type":[{"action_type":"post_engagement","value":0.39,"1d_click":0.39,"7d_click":0.39,"28d_click":0.39},{"action_type":"page_engagement","value":0.39,"1d_click":0.39,"7d_click":0.39,"28d_click":0.39},{"action_type":"link_click","value":0.39,"1d_click":0.39,"7d_click":0.39,"28d_click":0.39}],"cost_per_unique_click":0.39,"cost_per_unique_inline_link_click":0.39,"cost_per_unique_outbound_click":[{"action_type":"outbound_click","value":0.39}],"cpc":0.39,"cpm":0.922713,"cpp":0.971761,"created_time":"2021-02-09","ctr":0.236593,"date_start":"2021-02-15","date_stop":"2021-02-15","estimated_ad_recall_rate":14.119601,"estimated_ad_recallers":170.0,"frequency":1.053156,"impressions":1268,"inline_link_click_ctr":0.236593,"inline_link_clicks":3,"inline_post_engagement":3,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","outbound_clicks":[{"action_type":"outbound_click","value":3.0}],"outbound_clicks_ctr":[{"action_type":"outbound_click","value":0.236593}],"reach":1204,"spend":1.17,"unique_actions":[{"action_type":"page_engagement","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0},{"action_type":"post_engagement","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0},{"action_type":"link_click","value":3.0,"1d_click":3.0,"7d_click":3.0,"28d_click":3.0}],"unique_clicks":3,"unique_ctr":0.249169,"unique_inline_link_click_ctr":0.249169,"unique_inline_link_clicks":3,"unique_link_clicks_ctr":0.249169,"unique_outbound_clicks":[{"action_type":"outbound_click","value":3.0}],"unique_outbound_clicks_ctr":[{"action_type":"outbound_click","value":0.249169}],"updated_time":"2021-08-27","website_ctr":[{"action_type":"link_click","value":0.236593}],"publisher_platform":"facebook"},"emitted_at":1696936337306} +{"stream":"ads_insights_delivery_platform_and_device_platform","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846765228310398","ad_name":"Airbyte Ad","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"cost_per_estimated_ad_recallers":0.002,"cpm":0.392157,"cpp":0.392157,"created_time":"2021-02-09","ctr":0.0,"date_start":"2021-02-15","date_stop":"2021-02-15","estimated_ad_recall_rate":19.607843,"estimated_ad_recallers":10.0,"frequency":1.0,"impressions":51,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":51,"spend":0.02,"unique_clicks":0,"updated_time":"2021-08-27","publisher_platform":"instagram","device_platform":"mobile_app"},"emitted_at":1696967644628} +{"stream":"ads_insights_demographics_age","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846784938030398","ad_name":"Stock photo ad 2","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"cost_per_estimated_ad_recallers":0.0085,"cpm":1.14094,"cpp":1.188811,"created_time":"2021-02-11","ctr":0.0,"date_start":"2021-02-15","date_stop":"2021-02-15","estimated_ad_recall_rate":13.986014,"estimated_ad_recallers":20.0,"frequency":1.041958,"impressions":149,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":143,"spend":0.17,"unique_clicks":0,"updated_time":"2021-08-27","age":"25-34"},"emitted_at":1696936389857} +{"stream":"ads_insights_demographics_country","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846784938030398","ad_name":"Stock photo ad 2","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":1,"cost_per_estimated_ad_recallers":0.008889,"cost_per_unique_click":0.8,"cpc":0.8,"cpm":1.255887,"cpp":1.296596,"created_time":"2021-02-11","ctr":0.156986,"date_start":"2021-02-14","date_stop":"2021-02-14","estimated_ad_recall_rate":14.58671,"estimated_ad_recallers":90.0,"frequency":1.032415,"impressions":637,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":617,"spend":0.8,"unique_clicks":1,"unique_ctr":0.162075,"updated_time":"2021-08-27","country":"US"},"emitted_at":1696936440731} +{"stream":"ads_insights_demographics_dma_region","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846784938030398","ad_name":"Stock photo ad 2","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"cost_per_estimated_ad_recallers":0.0,"cpm":0.0,"created_time":"2021-02-11","ctr":0.0,"date_start":"2021-02-15","date_stop":"2021-02-15","estimated_ad_recallers":1.0,"frequency":1.0,"impressions":1,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":1,"spend":0.0,"unique_clicks":0,"updated_time":"2021-08-27","dma":"Anchorage"},"emitted_at":1696936491393} +{"stream":"ads_insights_demographics_gender","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846784938030398","ad_name":"Stock photo ad 2","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"cost_per_estimated_ad_recallers":0.0085,"cpm":1.268657,"cpp":1.338583,"created_time":"2021-02-11","ctr":0.0,"date_start":"2021-02-14","date_stop":"2021-02-14","estimated_ad_recall_rate":15.748032,"estimated_ad_recallers":20.0,"frequency":1.055118,"gender_targeting":"female","impressions":134,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":127,"spend":0.17,"unique_clicks":0,"updated_time":"2021-08-27","gender":"female"},"emitted_at":1696967753477} +{"stream":"ads_insights_dma","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846784938030398","ad_name":"Stock photo ad 2","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"cost_per_estimated_ad_recallers":0.0,"cpm":0.0,"created_time":"2021-02-11","ctr":0.0,"date_start":"2021-02-15","date_stop":"2021-02-15","estimated_ad_recallers":1.0,"frequency":1.0,"impressions":1,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":1,"spend":0.0,"unique_clicks":0,"updated_time":"2021-08-27","dma":"West Palm Beach-Ft. Pierce"},"emitted_at":1696936556045} +{"stream":"ads_insights_platform_and_device","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846784938030398","ad_name":"Stock photo ad 2","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"cost_per_estimated_ad_recallers":0.0,"cpm":0.0,"cpp":0.0,"created_time":"2021-02-11","ctr":0.0,"date_start":"2021-02-15","date_stop":"2021-02-15","estimated_ad_recall_rate":12.5,"estimated_ad_recallers":1.0,"frequency":1.0,"impressions":8,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":8,"spend":0.0,"unique_clicks":0,"updated_time":"2021-08-27","publisher_platform":"instagram","platform_position":"feed","impression_device":"android_smartphone"},"emitted_at":1696936579028} +{"stream":"ads_insights_region","data":{"account_currency":"USD","account_id":"212551616838260","account_name":"Airbyte","ad_id":"23846784938030398","ad_name":"Stock photo ad 2","adset_id":"23846765228280398","adset_name":"Vanilla awareness ad set","buying_type":"AUCTION","campaign_id":"23846765228240398","campaign_name":"Airbyte Awareness Campaign 1 (sherif)","clicks":0,"cost_per_estimated_ad_recallers":0.02,"cpm":1.111111,"cpp":1.111111,"created_time":"2021-02-11","ctr":0.0,"date_start":"2021-02-15","date_stop":"2021-02-15","estimated_ad_recall_rate":5.555556,"estimated_ad_recallers":1.0,"frequency":1.0,"impressions":18,"instant_experience_clicks_to_open":1.0,"instant_experience_clicks_to_start":1.0,"objective":"BRAND_AWARENESS","optimization_goal":"AD_RECALL_LIFT","reach":18,"spend":0.02,"unique_clicks":0,"updated_time":"2021-08-27","region":"New York"},"emitted_at":1696936621899} +{"stream":"customcustom_insight_stream","data":{"account_id":"212551616838260","cpc":0.27,"ad_id":"23846765228310398","clicks":1,"account_name":"Airbyte","date_start":"2021-02-15","date_stop":"2021-02-15","gender":"female"},"emitted_at":1695385890508} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/future_state.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/future_state.json index 35ac2f8aa38b..1cc425ce9f5d 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/future_state.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/future_state.json @@ -335,5 +335,17 @@ "name": "ads_insights_demographics_gender" } } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "date_start": "2121-07-25T13:34:26Z", + "include_deleted": true + }, + "stream_descriptor": { + "name": "customcustom_insight_stream" + } + } } ] diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json index cea425df779e..1657aaeda2d0 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json @@ -5,19 +5,31 @@ "title": "Source Facebook Marketing", "type": "object", "properties": { - "account_id": { - "title": "Account ID", - "description": "The Facebook Ad account ID to use when pulling data from the Facebook Marketing API. Open your Meta Ads Manager. The Ad account ID number is in the account dropdown menu or in your browser's address bar. See the docs for more information.", + "account_ids": { + "title": "Ad Account ID(s)", + "description": "The Facebook Ad account ID(s) to pull data from. The Ad account ID number is in the account dropdown menu or in your browser's address bar of your Meta Ads Manager. See the docs for more information.", "order": 0, - "pattern": "^[0-9]+$", - "pattern_descriptor": "1234567890", + "pattern_descriptor": "The Ad Account ID must be a number.", "examples": ["111111111111111"], + "type": "array", + "minItems": 1, + "items": { + "pattern": "^[0-9]+$", + "type": "string" + }, + "uniqueItems": true + }, + "access_token": { + "title": "Access Token", + "description": "The value of the generated access token. From your App\u2019s Dashboard, click on \"Marketing API\" then \"Tools\". Select permissions ads_management, ads_read, read_insights, business_management. Then click on \"Get token\". See the docs for more information.", + "order": 1, + "airbyte_secret": true, "type": "string" }, "start_date": { "title": "Start Date", - "description": "The date from which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", - "order": 1, + "description": "The date from which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DDT00:00:00Z. If not set then all data will be replicated for usual streams and only last 2 years for insight streams.", + "order": 2, "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", "examples": ["2017-01-25T00:00:00Z"], "type": "string", @@ -26,19 +38,12 @@ "end_date": { "title": "End Date", "description": "The date until which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DDT00:00:00Z. All data generated between the start date and this end date will be replicated. Not setting this option will result in always syncing the latest data.", - "order": 2, + "order": 3, "pattern": "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", "examples": ["2017-01-26T00:00:00Z"], "type": "string", "format": "date-time" }, - "access_token": { - "title": "Access Token", - "description": "The value of the generated access token. From your App\u2019s Dashboard, click on \"Marketing API\" then \"Tools\". Select permissions ads_management, ads_read, read_insights, business_management. Then click on \"Get token\". See the docs for more information.", - "order": 3, - "airbyte_secret": true, - "type": "string" - }, "include_deleted": { "title": "Include Deleted Campaigns, Ads, and AdSets", "description": "Set to active if you want to include data from deleted Campaigns, Ads, and AdSets.", @@ -324,6 +329,15 @@ "mininum": 1, "exclusiveMinimum": 0, "type": "integer" + }, + "insights_job_timeout": { + "title": "Custom Insights Job Timeout", + "description": "The insights job timeout", + "default": 60, + "maximum": 60, + "mininum": 10, + "exclusiveMinimum": 0, + "type": "integer" } }, "required": ["name"] @@ -347,13 +361,14 @@ "exclusiveMinimum": 0, "type": "integer" }, - "max_batch_size": { - "title": "Maximum size of Batched Requests", - "description": "Maximum batch size used when sending batch requests to Facebook API. Most users do not need to set this field unless they specifically need to tune the connector to address specific issues or use cases.", - "default": 50, + "insights_job_timeout": { + "title": "Insights Job Timeout", + "description": "Insights Job Timeout establishes the maximum amount of time (in minutes) of waiting for the report job to complete. When timeout is reached the job is considered failed and we are trying to request smaller amount of data by breaking the job to few smaller ones. If you definitely know that 60 minutes is not enough for your report to be processed then you can decrease the timeout value, so we start breaking job to smaller parts faster.", + "default": 60, "order": 9, + "maximum": 60, + "mininum": 10, "exclusiveMinimum": 0, - "maximum": 50, "type": "integer" }, "action_breakdowns_allow_empty": { @@ -378,7 +393,7 @@ "type": "string" } }, - "required": ["account_id", "start_date", "access_token"] + "required": ["account_ids", "access_token"] }, "supportsIncremental": true, "supported_destination_sync_modes": ["append"], diff --git a/airbyte-integrations/connectors/source-facebook-marketing/main.py b/airbyte-integrations/connectors/source-facebook-marketing/main.py index 64be48a5343e..fc25c7149e93 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/main.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/main.py @@ -3,11 +3,7 @@ # -import sys - -from airbyte_cdk.entrypoint import launch -from source_facebook_marketing import SourceFacebookMarketing +from source_facebook_marketing.run import run if __name__ == "__main__": - source = SourceFacebookMarketing() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml b/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml index 57c1f108f263..0bac6b20fc1d 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml +++ b/airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml @@ -1,12 +1,18 @@ data: + ab_internal: + ql: 400 + sl: 300 allowedHosts: hosts: - graph.facebook.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: api connectorType: source definitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c - dockerImageTag: 1.1.7 + dockerImageTag: 1.3.0 dockerRepository: airbyte/source-facebook-marketing + documentationUrl: https://docs.airbyte.com/integrations/sources/facebook-marketing githubIssueLabel: source-facebook-marketing icon: facebook.svg license: ELv2 @@ -17,11 +23,23 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/facebook-marketing + suggestedStreams: + streams: + - ads_insights + - campaigns + - ads + - ad_sets + - ad_creatives + - ads_insights_age_and_gender + - ads_insights_action_type + - custom_conversions + - images + - ads_insights_country + - ads_insights_platform_and_device + - ads_insights_region + - ads_insights_dma + - activities + supportLevel: certified tags: - language:python - ab_internal: - sl: 300 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/setup.py b/airbyte-integrations/connectors/source-facebook-marketing/setup.py index 144e8b73abc7..44f12e25a0d1 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/setup.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/setup.py @@ -25,4 +25,9 @@ extras_require={ "tests": TEST_REQUIREMENTS, }, + entry_points={ + "console_scripts": [ + "source-facebook-marketing=source_facebook_marketing.run:run", + ], + }, ) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py index 92a85fb7dd8b..61a171b9659d 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/api.py @@ -6,12 +6,10 @@ import logging from dataclasses import dataclass from time import sleep +from typing import List import backoff import pendulum -from airbyte_cdk.models import FailureType -from airbyte_cdk.utils import AirbyteTracedException -from cached_property import cached_property from facebook_business import FacebookAdsApi from facebook_business.adobjects.adaccount import AdAccount from facebook_business.api import FacebookResponse @@ -77,7 +75,6 @@ def _parse_call_rate_header(headers): ) if usage_header_business: - usage_header_business_loaded = json.loads(usage_header_business) for business_object_id in usage_header_business_loaded: usage_limits = usage_header_business_loaded.get(business_object_id)[0] @@ -176,8 +173,8 @@ def call( class API: """Simple wrapper around Facebook API""" - def __init__(self, account_id: str, access_token: str, page_size: int = 100): - self._account_id = account_id + def __init__(self, access_token: str, page_size: int = 100): + self._accounts = {} # design flaw in MyFacebookAdsApi requires such strange set of new default api instance self.api = MyFacebookAdsApi.init(access_token=access_token, crash_log=False) # adding the default page size from config to the api base class @@ -186,23 +183,14 @@ def __init__(self, account_id: str, access_token: str, page_size: int = 100): # set the default API client to Facebook lib. FacebookAdsApi.set_default_api(self.api) - @cached_property - def account(self) -> AdAccount: - """Find current account""" - return self._find_account(self._account_id) + def get_account(self, account_id: str) -> AdAccount: + """Get AdAccount object by id""" + if account_id in self._accounts: + return self._accounts[account_id] + self._accounts[account_id] = self._find_account(account_id) + return self._accounts[account_id] @staticmethod def _find_account(account_id: str) -> AdAccount: """Actual implementation of find account""" - try: - return AdAccount(f"act_{account_id}").api_get() - except FacebookRequestError as exc: - message = ( - f"Error: {exc.api_error_code()}, {exc.api_error_message()}. " - f"Please also verify your Account ID: " - f"See the https://www.facebook.com/business/help/1492627900875762 for more information." - ) - raise AirbyteTracedException( - message=message, - failure_type=FailureType.config_error, - ) from exc + return AdAccount(f"act_{account_id}").api_get() diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py new file mode 100644 index 000000000000..c8b6c7e109a2 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/config_migrations.py @@ -0,0 +1,82 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import logging +from typing import Any, List, Mapping + +from airbyte_cdk.config_observation import create_connector_config_control_message +from airbyte_cdk.entrypoint import AirbyteEntrypoint +from airbyte_cdk.sources import Source +from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + +logger = logging.getLogger("airbyte_logger") + + +class MigrateAccountIdToArray: + """ + This class stands for migrating the config at runtime. + This migration is backwards compatible with the previous version, as new property will be created. + When falling back to the previous source version connector will use old property `account_id`. + + Starting from `1.3.0`, the `account_id` property is replaced with `account_ids` property, which is a list of strings. + """ + + message_repository: MessageRepository = InMemoryMessageRepository() + migrate_from_key: str = "account_id" + migrate_to_key: str = "account_ids" + + @classmethod + def should_migrate(cls, config: Mapping[str, Any]) -> bool: + """ + This method determines whether the config should be migrated to have the new structure for the `custom_reports`, + based on the source spec. + Returns: + > True, if the transformation is necessary + > False, otherwise. + > Raises the Exception if the structure could not be migrated. + """ + return False if config.get(cls.migrate_to_key) else True + + @classmethod + def transform(cls, config: Mapping[str, Any]) -> Mapping[str, Any]: + # transform the config + config[cls.migrate_to_key] = [config[cls.migrate_from_key]] + # return transformed config + return config + + @classmethod + def modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, Any]) -> Mapping[str, Any]: + # modify the config + migrated_config = cls.transform(config) + # save the config + source.write_config(migrated_config, config_path) + # return modified config + return migrated_config + + @classmethod + def emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: + # add the Airbyte Control Message to message repo + cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) + # emit the Airbyte Control Message from message queue to stdout + for message in cls.message_repository._message_queue: + print(message.json(exclude_unset=True)) + + @classmethod + def migrate(cls, args: List[str], source: Source) -> None: + """ + This method checks the input args, should the config be migrated, + transform if neccessary and emit the CONTROL message. + """ + # get config path + config_path = AirbyteEntrypoint(source).extract_config(args) + # proceed only if `--config` arg is provided + if config_path: + # read the existing config + config = source.read_config(config_path) + # migration check + if cls.should_migrate(config): + cls.emit_control_message( + cls.modify_and_save(config_path, source, config), + ) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/run.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/run.py new file mode 100644 index 000000000000..2e92663e42fd --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/run.py @@ -0,0 +1,17 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch + +from .config_migrations import MigrateAccountIdToArray +from .source import SourceFacebookMarketing + + +def run(): + source = SourceFacebookMarketing() + MigrateAccountIdToArray.migrate(sys.argv[1:], source) + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/activities.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/activities.json index 4b5c729e0686..69a31b5f8b55 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/activities.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/activities.json @@ -1,5 +1,8 @@ { "properties": { + "account_id": { + "type": ["null", "string"] + }, "actor_id": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json index f6e8d8c04eba..cd0b5ec85f1c 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/ad_creatives.json @@ -496,6 +496,12 @@ } ] }, + "reasons_to_shop": { + "type": ["null", "boolean"] + }, + "shops_bundle": { + "type": ["null", "boolean"] + }, "titles": { "type": ["null", "array"], "items": { diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json index e0a2d8c0c33b..ce96ff0a6a99 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/campaigns.json @@ -28,6 +28,9 @@ "bid_strategy": { "type": ["null", "string"] }, + "boosted_object_id": { + "type": ["null", "string"] + }, "budget_rebalance_flag": { "type": ["null", "boolean"] }, @@ -44,6 +47,9 @@ "type": "string", "format": "date-time" }, + "configured_status": { + "type": ["null", "string"] + }, "effective_status": { "type": ["null", "string"] }, @@ -104,6 +110,9 @@ "type": "string", "format": "date-time" }, + "status": { + "type": ["null", "string"] + }, "stop_time": { "type": "string", "format": "date-time" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/videos.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/videos.json index 9aa51674dd10..3a146978ada6 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/videos.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/videos.json @@ -1,5 +1,8 @@ { "properties": { + "account_id": { + "type": ["null", "string"] + }, "id": { "type": "string" }, diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index 16db2e4f6245..65e8c057852a 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -7,7 +7,6 @@ import facebook_business import pendulum -import requests from airbyte_cdk.models import ( AdvancedAuth, AuthFlowType, @@ -15,12 +14,12 @@ DestinationSyncMode, FailureType, OAuthConfigSpecification, + SyncMode, ) from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.utils import AirbyteTracedException -from pydantic.error_wrappers import ValidationError -from source_facebook_marketing.api import API, FacebookAPIException +from source_facebook_marketing.api import API from source_facebook_marketing.spec import ConnectorConfig from source_facebook_marketing.streams import ( Activities, @@ -54,7 +53,6 @@ Images, Videos, ) -from source_facebook_marketing.streams.common import AccountTypeException from .utils import validate_end_date, validate_start_date @@ -63,13 +61,24 @@ class SourceFacebookMarketing(AbstractSource): + # Skip exceptions on missing streams + raise_exception_on_missing_stream = False + def _validate_and_transform(self, config: Mapping[str, Any]): config.setdefault("action_breakdowns_allow_empty", False) if config.get("end_date") == "": config.pop("end_date") + config = ConnectorConfig.parse_obj(config) - config.start_date = pendulum.instance(config.start_date) - config.end_date = pendulum.instance(config.end_date) + + if config.start_date: + config.start_date = pendulum.instance(config.start_date) + + if config.end_date: + config.end_date = pendulum.instance(config.end_date) + + config.account_ids = list(config.account_ids) + return config def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: @@ -84,30 +93,30 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> if config.end_date > pendulum.now(): return False, "Date range can not be in the future." - if config.end_date < config.start_date: - return False, "end_date must be equal or after start_date." + if config.start_date and config.end_date < config.start_date: + return False, "End date must be equal or after start date." - api = API(account_id=config.account_id, access_token=config.access_token, page_size=config.page_size) - logger.info(f"Select account {api.account}") + api = API(access_token=config.access_token, page_size=config.page_size) - account_info = api.account.api_get(fields=["is_personal"]) + for account_id in config.account_ids: + # Get Ad Account to check creds + logger.info(f"Attempting to retrieve information for account with ID: {account_id}") + ad_account = api.get_account(account_id=account_id) + logger.info(f"Successfully retrieved account information for account: {ad_account}") - if account_info.get("is_personal"): - message = ( - "The personal ad account you're currently using is not eligible " - "for this operation. Please switch to a business ad account." - ) - raise AccountTypeException(message) + # make sure that we have valid combination of "action_breakdowns" and "breakdowns" parameters + for stream in self.get_custom_insights_streams(api, config): + stream.check_breakdowns(account_id=account_id) + + except facebook_business.exceptions.FacebookRequestError as e: + return False, e._api_error_message - except (requests.exceptions.RequestException, ValidationError, FacebookAPIException, AccountTypeException) as e: - return False, e + except AirbyteTracedException as e: + return False, f"{e.message}. Full error: {e.internal_message}" + + except Exception as e: + return False, f"Unexpected error: {repr(e)}" - # make sure that we have valid combination of "action_breakdowns" and "breakdowns" parameters - for stream in self.get_custom_insights_streams(api, config): - try: - stream.check_breakdowns() - except facebook_business.exceptions.FacebookRequestError as e: - return False, e._api_error_message return True, None def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: @@ -117,101 +126,110 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: :return: list of the stream instances """ config = self._validate_and_transform(config) - config.start_date = validate_start_date(config.start_date) - config.end_date = validate_end_date(config.start_date, config.end_date) + if config.start_date: + config.start_date = validate_start_date(config.start_date) + config.end_date = validate_end_date(config.start_date, config.end_date) + + api = API(access_token=config.access_token, page_size=config.page_size) - api = API(account_id=config.account_id, access_token=config.access_token, page_size=config.page_size) + # if start_date not specified then set default start_date for report streams to 2 years ago + report_start_date = config.start_date or pendulum.now().add(years=-2) insights_args = dict( - api=api, start_date=config.start_date, end_date=config.end_date, insights_lookback_window=config.insights_lookback_window + api=api, + account_ids=config.account_ids, + start_date=report_start_date, + end_date=config.end_date, + insights_lookback_window=config.insights_lookback_window, + insights_job_timeout=config.insights_job_timeout, ) streams = [ - AdAccount(api=api), + AdAccount(api=api, account_ids=config.account_ids), AdSets( api=api, + account_ids=config.account_ids, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted, page_size=config.page_size, - max_batch_size=config.max_batch_size, ), Ads( api=api, + account_ids=config.account_ids, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted, page_size=config.page_size, - max_batch_size=config.max_batch_size, ), AdCreatives( api=api, + account_ids=config.account_ids, fetch_thumbnail_images=config.fetch_thumbnail_images, page_size=config.page_size, - max_batch_size=config.max_batch_size, ), - AdsInsights(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), - AdsInsightsAgeAndGender(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), - AdsInsightsCountry(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), - AdsInsightsRegion(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), - AdsInsightsDma(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), - AdsInsightsPlatformAndDevice(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), - AdsInsightsActionType(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), - AdsInsightsActionCarouselCard(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), - AdsInsightsActionConversionDevice(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), - AdsInsightsActionProductID(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), - AdsInsightsActionReaction(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), - AdsInsightsActionVideoSound(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), - AdsInsightsActionVideoType(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), - AdsInsightsDeliveryDevice(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), - AdsInsightsDeliveryPlatform(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), - AdsInsightsDeliveryPlatformAndDevicePlatform(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), - AdsInsightsDemographicsAge(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), - AdsInsightsDemographicsCountry(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), - AdsInsightsDemographicsDMARegion(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), - AdsInsightsDemographicsGender(page_size=config.page_size, max_batch_size=config.max_batch_size, **insights_args), + AdsInsights(page_size=config.page_size, **insights_args), + AdsInsightsAgeAndGender(page_size=config.page_size, **insights_args), + AdsInsightsCountry(page_size=config.page_size, **insights_args), + AdsInsightsRegion(page_size=config.page_size, **insights_args), + AdsInsightsDma(page_size=config.page_size, **insights_args), + AdsInsightsPlatformAndDevice(page_size=config.page_size, **insights_args), + AdsInsightsActionType(page_size=config.page_size, **insights_args), + AdsInsightsActionCarouselCard(page_size=config.page_size, **insights_args), + AdsInsightsActionConversionDevice(page_size=config.page_size, **insights_args), + AdsInsightsActionProductID(page_size=config.page_size, **insights_args), + AdsInsightsActionReaction(page_size=config.page_size, **insights_args), + AdsInsightsActionVideoSound(page_size=config.page_size, **insights_args), + AdsInsightsActionVideoType(page_size=config.page_size, **insights_args), + AdsInsightsDeliveryDevice(page_size=config.page_size, **insights_args), + AdsInsightsDeliveryPlatform(page_size=config.page_size, **insights_args), + AdsInsightsDeliveryPlatformAndDevicePlatform(page_size=config.page_size, **insights_args), + AdsInsightsDemographicsAge(page_size=config.page_size, **insights_args), + AdsInsightsDemographicsCountry(page_size=config.page_size, **insights_args), + AdsInsightsDemographicsDMARegion(page_size=config.page_size, **insights_args), + AdsInsightsDemographicsGender(page_size=config.page_size, **insights_args), Campaigns( api=api, + account_ids=config.account_ids, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted, page_size=config.page_size, - max_batch_size=config.max_batch_size, ), CustomConversions( api=api, + account_ids=config.account_ids, include_deleted=config.include_deleted, page_size=config.page_size, - max_batch_size=config.max_batch_size, ), CustomAudiences( api=api, + account_ids=config.account_ids, include_deleted=config.include_deleted, page_size=config.page_size, - max_batch_size=config.max_batch_size, ), Images( api=api, + account_ids=config.account_ids, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted, page_size=config.page_size, - max_batch_size=config.max_batch_size, ), Videos( api=api, + account_ids=config.account_ids, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted, page_size=config.page_size, - max_batch_size=config.max_batch_size, ), Activities( api=api, + account_ids=config.account_ids, start_date=config.start_date, end_date=config.end_date, include_deleted=config.include_deleted, page_size=config.page_size, - max_batch_size=config.max_batch_size, ), ] @@ -273,6 +291,7 @@ def get_custom_insights_streams(self, api: API, config: ConnectorConfig) -> List ) stream = AdsInsights( api=api, + account_ids=config.account_ids, name=f"Custom{insight.name}", fields=list(insight_fields), breakdowns=list(set(insight.breakdowns)), @@ -280,9 +299,10 @@ def get_custom_insights_streams(self, api: API, config: ConnectorConfig) -> List action_breakdowns_allow_empty=config.action_breakdowns_allow_empty, action_report_time=insight.action_report_time, time_increment=insight.time_increment, - start_date=insight.start_date or config.start_date, + start_date=insight.start_date or config.start_date or pendulum.now().add(years=-2), end_date=insight.end_date or config.end_date, insights_lookback_window=insight.insights_lookback_window or config.insights_lookback_window, + insights_job_timeout=insight.insights_job_timeout or config.insights_job_timeout, level=insight.level, ) streams.append(stream) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py index 99340caf954d..951ce0a2a63c 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py @@ -5,11 +5,11 @@ import logging from datetime import datetime, timezone from enum import Enum -from typing import List, Optional +from typing import List, Optional, Set from airbyte_cdk.sources.config import BaseConfig from facebook_business.adobjects.adsinsights import AdsInsights -from pydantic import BaseModel, Field, PositiveInt +from pydantic import BaseModel, Field, PositiveInt, constr logger = logging.getLogger("airbyte") @@ -97,6 +97,13 @@ class Config: mininum=1, default=28, ) + insights_job_timeout: Optional[PositiveInt] = Field( + title="Custom Insights Job Timeout", + description="The insights job timeout", + maximum=60, + mininum=10, + default=60, + ) class ConnectorConfig(BaseConfig): @@ -105,25 +112,38 @@ class ConnectorConfig(BaseConfig): class Config: title = "Source Facebook Marketing" - account_id: str = Field( - title="Account ID", + account_ids: Set[constr(regex="^[0-9]+$")] = Field( + title="Ad Account ID(s)", order=0, description=( - "The Facebook Ad account ID to use when pulling data from the Facebook Marketing API." - " Open your Meta Ads Manager. The Ad account ID number is in the account dropdown menu or in your browser's address bar. " + "The Facebook Ad account ID(s) to pull data from. " + "The Ad account ID number is in the account dropdown menu or in your browser's address " + 'bar of your Meta Ads Manager. ' 'See the docs for more information.' ), - pattern="^[0-9]+$", - pattern_descriptor="1234567890", + pattern_descriptor="The Ad Account ID must be a number.", examples=["111111111111111"], + min_items=1, ) - start_date: datetime = Field( - title="Start Date", + access_token: str = Field( + title="Access Token", order=1, + description=( + "The value of the generated access token. " + 'From your App’s Dashboard, click on "Marketing API" then "Tools". ' + 'Select permissions ads_management, ads_read, read_insights, business_management. Then click on "Get token". ' + 'See the docs for more information.' + ), + airbyte_secret=True, + ) + + start_date: Optional[datetime] = Field( + title="Start Date", + order=2, description=( "The date from which you'd like to replicate data for all incremental streams, " - "in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated." + "in the format YYYY-MM-DDT00:00:00Z. If not set then all data will be replicated for usual streams and only last 2 years for insight streams." ), pattern=DATE_TIME_PATTERN, examples=["2017-01-25T00:00:00Z"], @@ -131,7 +151,7 @@ class Config: end_date: Optional[datetime] = Field( title="End Date", - order=2, + order=3, description=( "The date until which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DDT00:00:00Z." " All data generated between the start date and this end date will be replicated. " @@ -142,18 +162,6 @@ class Config: default_factory=lambda: datetime.now(tz=timezone.utc), ) - access_token: str = Field( - title="Access Token", - order=3, - description=( - "The value of the generated access token. " - 'From your App’s Dashboard, click on "Marketing API" then "Tools". ' - 'Select permissions ads_management, ads_read, read_insights, business_management. Then click on "Get token". ' - 'See the docs for more information.' - ), - airbyte_secret=True, - ) - include_deleted: bool = Field( title="Include Deleted Campaigns, Ads, and AdSets", order=4, @@ -201,16 +209,18 @@ class Config: default=28, ) - max_batch_size: Optional[int] = Field( - title="Maximum size of Batched Requests", + insights_job_timeout: Optional[PositiveInt] = Field( + title="Insights Job Timeout", order=9, description=( - "Maximum batch size used when sending batch requests to Facebook API. " - "Most users do not need to set this field unless they specifically need to tune the connector to address specific issues or use cases." + "Insights Job Timeout establishes the maximum amount of time (in minutes) of waiting for the report job to complete. " + "When timeout is reached the job is considered failed and we are trying to request smaller amount of data by breaking the job to few smaller ones. " + "If you definitely know that 60 minutes is not enough for your report to be processed then you can decrease the timeout value, " + "so we start breaking job to smaller parts faster." ), - default=50, - gt=0, - le=50, + maximum=60, + mininum=10, + default=60, ) action_breakdowns_allow_empty: bool = Field( diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py index fe44c18223e8..317be6673410 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job.py @@ -17,6 +17,7 @@ from facebook_business.adobjects.campaign import Campaign from facebook_business.adobjects.objectparser import ObjectParser from facebook_business.api import FacebookAdsApi, FacebookAdsApiBatch, FacebookBadObjectError, FacebookResponse +from pendulum.duration import Duration from source_facebook_marketing.streams.common import retry_pattern from ..utils import validate_start_date @@ -189,10 +190,9 @@ def __str__(self) -> str: class InsightAsyncJob(AsyncJob): """AsyncJob wraps FB AdReport class and provides interface to restart/retry the async job""" - job_timeout = pendulum.duration(hours=1) page_size = 100 - def __init__(self, edge_object: Union[AdAccount, Campaign, AdSet, Ad], params: Mapping[str, Any], **kwargs): + def __init__(self, edge_object: Union[AdAccount, Campaign, AdSet, Ad], params: Mapping[str, Any], job_timeout: Duration, **kwargs): """Initialize :param api: FB API @@ -205,6 +205,7 @@ def __init__(self, edge_object: Union[AdAccount, Campaign, AdSet, Ad], params: M "since": self._interval.start.to_date_string(), "until": self._interval.end.to_date_string(), } + self._job_timeout = job_timeout self._edge_object = edge_object self._job: Optional[AdReportRun] = None @@ -251,7 +252,12 @@ def _split_by_edge_class(self, edge_class: Union[Type[Campaign], Type[AdSet], Ty ids = set(row[pk_name] for row in result) logger.info(f"Got {len(ids)} {pk_name}s for period {self._interval}: {ids}") - jobs = [InsightAsyncJob(api=self._api, edge_object=edge_class(pk), params=self._params, interval=self._interval) for pk in ids] + jobs = [ + InsightAsyncJob( + api=self._api, edge_object=edge_class(pk), params=self._params, interval=self._interval, job_timeout=self._job_timeout + ) + for pk in ids + ] return jobs def start(self): @@ -335,8 +341,8 @@ def _check_status(self) -> bool: percent = self._job["async_percent_completion"] logger.info(f"{self}: is {percent} complete ({job_status})") - if self.elapsed_time > self.job_timeout: - logger.info(f"{self}: run more than maximum allowed time {self.job_timeout}.") + if self.elapsed_time > self._job_timeout: + logger.info(f"{self}: run more than maximum allowed time {self._job_timeout}.") self._finish_time = pendulum.now() self._failed = True return True diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py index 8bfcc6fe74af..738507e4408b 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/async_job_manager.py @@ -32,13 +32,14 @@ class InsightAsyncJobManager: # limit is not reliable indicator of async workload capability we still have to use this parameter. MAX_JOBS_IN_QUEUE = 100 - def __init__(self, api: "API", jobs: Iterator[AsyncJob]): + def __init__(self, api: "API", jobs: Iterator[AsyncJob], account_id: str): """Init :param api: :param jobs: """ self._api = api + self._account_id = account_id self._jobs = iter(jobs) self._running_jobs = [] @@ -147,4 +148,4 @@ def _update_api_throttle_limit(self): respond with empty list of data so api use "x-fb-ads-insights-throttle" header to update current insights throttle limit. """ - self._api.account.get_insights() + self._api.get_account(account_id=self._account_id).get_insights() diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py index 5913b8a78eb4..c671a4b9b917 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_insight_streams.py @@ -11,10 +11,10 @@ from airbyte_cdk.sources.streams.core import package_name_from_class from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader from airbyte_cdk.utils import AirbyteTracedException -from cached_property import cached_property -from facebook_business.exceptions import FacebookBadObjectError +from facebook_business.exceptions import FacebookBadObjectError, FacebookRequestError from source_facebook_marketing.streams.async_job import AsyncJob, InsightAsyncJob from source_facebook_marketing.streams.async_job_manager import InsightAsyncJobManager +from source_facebook_marketing.streams.common import traced_exception from .base_streams import FBMarketingIncrementalStream @@ -25,7 +25,6 @@ class AdsInsights(FBMarketingIncrementalStream): """doc: https://developers.facebook.com/docs/marketing-api/insights""" cursor_field = "date_start" - use_batch = False enable_deleted = False ALL_ACTION_ATTRIBUTION_WINDOWS = [ @@ -63,13 +62,14 @@ def __init__( action_report_time: str = "mixed", time_increment: Optional[int] = None, insights_lookback_window: int = None, + insights_job_timeout: int = 60, level: str = "ad", **kwargs, ): super().__init__(**kwargs) self._start_date = self._start_date.date() self._end_date = self._end_date.date() - self._fields = fields + self._custom_fields = fields if action_breakdowns_allow_empty: if action_breakdowns is not None: self.action_breakdowns = action_breakdowns @@ -82,12 +82,13 @@ def __init__( self.action_report_time = action_report_time self._new_class_name = name self._insights_lookback_window = insights_lookback_window + self._insights_job_timeout = insights_job_timeout self.level = level # state - self._cursor_value: Optional[pendulum.Date] = None # latest period that was read - self._next_cursor_value = self._get_start_date() - self._completed_slices = set() + self._cursor_values: Optional[Mapping[str, pendulum.Date]] = None # latest period that was read for each account + self._next_cursor_values = self._get_start_date() + self._completed_slices = {account_id: set() for account_id in self._account_ids} @property def name(self) -> str: @@ -110,6 +111,10 @@ def insights_lookback_period(self): """ return pendulum.duration(days=self._insights_lookback_window) + @property + def insights_job_timeout(self): + return pendulum.duration(minutes=self._insights_job_timeout) + def list_objects(self, params: Mapping[str, Any]) -> Iterable: """Because insights has very different read_records we don't need this method anymore""" @@ -122,35 +127,46 @@ def read_records( ) -> Iterable[Mapping[str, Any]]: """Waits for current job to finish (slice) and yield its result""" job = stream_slice["insight_job"] + account_id = stream_slice["account_id"] + try: for obj in job.get_result(): - yield obj.export_all_data() + data = obj.export_all_data() + if self._response_data_is_valid(data): + yield data except FacebookBadObjectError as e: raise AirbyteTracedException( message=f"API error occurs on Facebook side during job: {job}, wrong (empty) response received with errors: {e} " f"Please try again later", failure_type=FailureType.system_error, ) from e + except FacebookRequestError as exc: + raise traced_exception(exc) - self._completed_slices.add(job.interval.start) - if job.interval.start == self._next_cursor_value: - self._advance_cursor() + self._completed_slices[account_id].add(job.interval.start) + if job.interval.start == self._next_cursor_values[account_id]: + self._advance_cursor(account_id) @property def state(self) -> MutableMapping[str, Any]: """State getter, the result can be stored by the source""" - if self._cursor_value: - return { - self.cursor_field: self._cursor_value.isoformat(), - "slices": [d.isoformat() for d in self._completed_slices], - "time_increment": self.time_increment, - } + new_state = {account_id: {} for account_id in self._account_ids} + + if self._cursor_values: + for account_id in self._account_ids: + if account_id in self._cursor_values and self._cursor_values[account_id]: + new_state[account_id] = {self.cursor_field: self._cursor_values[account_id].isoformat()} + + new_state[account_id]["slices"] = {d.isoformat() for d in self._completed_slices[account_id]} + new_state["time_increment"] = self.time_increment + return new_state if self._completed_slices: - return { - "slices": [d.isoformat() for d in self._completed_slices], - "time_increment": self.time_increment, - } + for account_id in self._account_ids: + new_state[account_id]["slices"] = {d.isoformat() for d in self._completed_slices[account_id]} + + new_state["time_increment"] = self.time_increment + return new_state return {} @@ -160,13 +176,23 @@ def state(self, value: Mapping[str, Any]): # if the time increment configured for this stream is different from the one in the previous state # then the previous state object is invalid and we should start replicating data from scratch # to achieve this, we skip setting the state - if value.get("time_increment", 1) != self.time_increment: + transformed_state = self._transform_state_from_old_format(value, ["time_increment"]) + if transformed_state.get("time_increment", 1) != self.time_increment: logger.info(f"Ignoring bookmark for {self.name} because of different `time_increment` option.") return - self._cursor_value = pendulum.parse(value[self.cursor_field]).date() if value.get(self.cursor_field) else None - self._completed_slices = set(pendulum.parse(v).date() for v in value.get("slices", [])) - self._next_cursor_value = self._get_start_date() + self._cursor_values = { + account_id: pendulum.parse(transformed_state[account_id][self.cursor_field]).date() + if transformed_state.get(account_id, {}).get(self.cursor_field) + else None + for account_id in self._account_ids + } + self._completed_slices = { + account_id: set(pendulum.parse(v).date() for v in transformed_state.get(account_id, {}).get("slices", [])) + for account_id in self._account_ids + } + + self._next_cursor_values = self._get_start_date() def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): """Update stream state from latest record @@ -176,38 +202,47 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late """ return self.state - def _date_intervals(self) -> Iterator[pendulum.Date]: + def _date_intervals(self, account_id: str) -> Iterator[pendulum.Date]: """Get date period to sync""" - if self._end_date < self._next_cursor_value: + if self._end_date < self._next_cursor_values[account_id]: return - date_range = self._end_date - self._next_cursor_value + date_range = self._end_date - self._next_cursor_values[account_id] yield from date_range.range("days", self.time_increment) - def _advance_cursor(self): + def _advance_cursor(self, account_id: str): """Iterate over state, find continuing sequence of slices. Get last value, advance cursor there and remove slices from state""" - for ts_start in self._date_intervals(): - if ts_start not in self._completed_slices: - self._next_cursor_value = ts_start + for ts_start in self._date_intervals(account_id): + if ts_start not in self._completed_slices[account_id]: + self._next_cursor_values[account_id] = ts_start break - self._completed_slices.remove(ts_start) - self._cursor_value = ts_start + self._completed_slices[account_id].remove(ts_start) + if self._cursor_values: + self._cursor_values[account_id] = ts_start + else: + self._cursor_values = {account_id: ts_start} - def _generate_async_jobs(self, params: Mapping) -> Iterator[AsyncJob]: + def _generate_async_jobs(self, params: Mapping, account_id: str) -> Iterator[AsyncJob]: """Generator of async jobs :param params: :return: """ - self._next_cursor_value = self._get_start_date() - for ts_start in self._date_intervals(): - if ts_start in self._completed_slices: + self._next_cursor_values = self._get_start_date() + for ts_start in self._date_intervals(account_id): + if ts_start in self._completed_slices.get(account_id, []): continue ts_end = ts_start + pendulum.duration(days=self.time_increment - 1) interval = pendulum.Period(ts_start, ts_end) - yield InsightAsyncJob(api=self._api.api, edge_object=self._api.account, interval=interval, params=params) - - def check_breakdowns(self): + yield InsightAsyncJob( + api=self._api.api, + edge_object=self._api.get_account(account_id=account_id), + interval=interval, + params=params, + job_timeout=self.insights_job_timeout, + ) + + def check_breakdowns(self, account_id: str): """ Making call to check "action_breakdowns" and "breakdowns" combinations https://developers.facebook.com/docs/marketing-api/insights/breakdowns#combiningbreakdowns @@ -217,12 +252,17 @@ def check_breakdowns(self): "breakdowns": self.breakdowns, "fields": ["account_id"], } - self._api.account.get_insights(params=params, is_async=False) + self._api.get_account(account_id=account_id).get_insights(params=params, is_async=False) + + def _response_data_is_valid(self, data: Iterable[Mapping[str, Any]]) -> bool: + """ + Ensure data contains all the fields specified in self.breakdowns + """ + return all([breakdown in data for breakdown in self.breakdowns]) def stream_slices( self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - """Slice by date periods and schedule async job for each period, run at most MAX_ASYNC_JOBS jobs at the same time. This solution for Async was chosen because: 1. we should commit state after each successful job @@ -238,11 +278,19 @@ def stream_slices( if stream_state: self.state = stream_state - manager = InsightAsyncJobManager(api=self._api, jobs=self._generate_async_jobs(params=self.request_params())) - for job in manager.completed_jobs(): - yield {"insight_job": job} + for account_id in self._account_ids: + try: + manager = InsightAsyncJobManager( + api=self._api, + jobs=self._generate_async_jobs(params=self.request_params(), account_id=account_id), + account_id=account_id, + ) + for job in manager.completed_jobs(): + yield {"insight_job": job, "account_id": account_id} + except FacebookRequestError as exc: + raise traced_exception(exc) - def _get_start_date(self) -> pendulum.Date: + def _get_start_date(self) -> Mapping[str, pendulum.Date]: """Get start date to begin sync with. It is not that trivial as it might seem. There are few rules: - don't read data older than start_date @@ -257,33 +305,42 @@ def _get_start_date(self) -> pendulum.Date: today = pendulum.today().date() oldest_date = today - self.INSIGHTS_RETENTION_PERIOD refresh_date = today - self.insights_lookback_period - if self._cursor_value: - start_date = self._cursor_value + pendulum.duration(days=self.time_increment) - if start_date > refresh_date: - logger.info( - f"The cursor value within refresh period ({self.insights_lookback_period}), start sync from {refresh_date} instead." + + start_dates_for_account = {} + for account_id in self._account_ids: + cursor_value = self._cursor_values.get(account_id) if self._cursor_values else None + if cursor_value: + start_date = cursor_value + pendulum.duration(days=self.time_increment) + if start_date > refresh_date: + logger.info( + f"The cursor value within refresh period ({self.insights_lookback_period}), start sync from {refresh_date} instead." + ) + start_date = min(start_date, refresh_date) + + if start_date < self._start_date: + logger.warning(f"Ignore provided state and start sync from start_date ({self._start_date}).") + start_date = max(start_date, self._start_date) + else: + start_date = self._start_date + if start_date < oldest_date: + logger.warning( + f"Loading insights older then {self.INSIGHTS_RETENTION_PERIOD} is not possible. Start sync from {oldest_date}." ) - start_date = min(start_date, refresh_date) + start_dates_for_account[account_id] = max(oldest_date, start_date) - if start_date < self._start_date: - logger.warning(f"Ignore provided state and start sync from start_date ({self._start_date}).") - start_date = max(start_date, self._start_date) - else: - start_date = self._start_date - if start_date < oldest_date: - logger.warning(f"Loading insights older then {self.INSIGHTS_RETENTION_PERIOD} is not possible. Start sync from {oldest_date}.") - return max(oldest_date, start_date) + return start_dates_for_account def request_params(self, **kwargs) -> MutableMapping[str, Any]: - return { + req_params = { "level": self.level, "action_breakdowns": self.action_breakdowns, "action_report_time": self.action_report_time, "breakdowns": self.breakdowns, - "fields": self.fields, + "fields": self.fields(), "time_increment": self.time_increment, "action_attribution_windows": self.action_attribution_windows, } + return req_params def _state_filter(self, stream_state: Mapping[str, Any]) -> Mapping[str, Any]: """Works differently for insights, so remove it""" @@ -295,17 +352,23 @@ def get_json_schema(self) -> Mapping[str, Any]: """ loader = ResourceSchemaLoader(package_name_from_class(self.__class__)) schema = loader.get_schema("ads_insights") - if self._fields: - schema["properties"] = {k: v for k, v in schema["properties"].items() if k in self._fields + [self.cursor_field]} + if self._custom_fields: + # 'date_stop' and 'account_id' are also returned by default, even if they are not requested + custom_fields = set(self._custom_fields + [self.cursor_field, "date_stop", "account_id", "ad_id"]) + schema["properties"] = {k: v for k, v in schema["properties"].items() if k in custom_fields} if self.breakdowns: breakdowns_properties = loader.get_schema("ads_insights_breakdowns")["properties"] schema["properties"].update({prop: breakdowns_properties[prop] for prop in self.breakdowns}) return schema - @cached_property - def fields(self) -> List[str]: + def fields(self, **kwargs) -> List[str]: """List of fields that we want to query, for now just all properties from stream's schema""" + if self._custom_fields: + return self._custom_fields + if self._fields: return self._fields + schema = ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("ads_insights") - return list(schema.get("properties", {}).keys()) + self._fields = list(schema.get("properties", {}).keys()) + return self._fields diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py index 27011e7e1640..9f396077df8a 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/base_streams.py @@ -5,9 +5,6 @@ import logging from abc import ABC, abstractmethod from datetime import datetime -from functools import partial -from math import ceil -from queue import Queue from typing import TYPE_CHECKING, Any, Iterable, List, Mapping, MutableMapping, Optional import pendulum @@ -15,18 +12,17 @@ from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from cached_property import cached_property from facebook_business.adobjects.abstractobject import AbstractObject -from facebook_business.api import FacebookAdsApiBatch, FacebookRequest, FacebookResponse +from facebook_business.exceptions import FacebookRequestError +from source_facebook_marketing.streams.common import traced_exception from .common import deep_merge if TYPE_CHECKING: # pragma: no cover from source_facebook_marketing.api import API -logger = logging.getLogger("airbyte") -FACEBOOK_BATCH_ERROR_CODE = 960 +logger = logging.getLogger("airbyte") class FBMarketingStream(Stream, ABC): @@ -35,8 +31,6 @@ class FBMarketingStream(Stream, ABC): primary_key = "id" transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - # use batch API to retrieve details for each record in a stream - use_batch = True # this flag will override `include_deleted` option for streams that does not support it enable_deleted = True # entity prefix for `include_deleted` filter, it usually matches singular version of stream name @@ -48,72 +42,108 @@ class FBMarketingStream(Stream, ABC): def availability_strategy(self) -> Optional["AvailabilityStrategy"]: return None - def __init__(self, api: "API", include_deleted: bool = False, page_size: int = 100, max_batch_size: int = 50, **kwargs): + def __init__(self, api: "API", account_ids: List[str], include_deleted: bool = False, page_size: int = 100, **kwargs): super().__init__(**kwargs) self._api = api + self._account_ids = account_ids self.page_size = page_size if page_size is not None else 100 self._include_deleted = include_deleted if self.enable_deleted else False - self.max_batch_size = self._initial_max_batch_size = max_batch_size if max_batch_size is not None else 50 + self._fields = None - @cached_property - def fields(self) -> List[str]: + def fields(self, **kwargs) -> List[str]: """List of fields that we want to query, for now just all properties from stream's schema""" - return list(self.get_json_schema().get("properties", {}).keys()) - - def _execute_batch(self, batch: FacebookAdsApiBatch) -> None: - """Execute batch, retry in case of failures""" - while batch: - batch = batch.execute() - if batch: - logger.info("Retry failed requests in batch") - - def execute_in_batch(self, pending_requests: Iterable[FacebookRequest]) -> Iterable[MutableMapping[str, Any]]: - """Execute list of requests in batches""" - requests_q = Queue() - records = [] - for r in pending_requests: - requests_q.put(r) - - def success(response: FacebookResponse): - self.max_batch_size = self._initial_max_batch_size - records.append(response.json()) - - def reduce_batch_size(request: FacebookRequest): - if self.max_batch_size == 1 and set(self.fields_exceptions) & set(request._fields): - logger.warning( - f"Removing fields from object {self.name} with id={request._node_id} : {set(self.fields_exceptions) & set(request._fields)}" - ) - request._fields = [x for x in request._fields if x not in self.fields_exceptions] - elif self.max_batch_size == 1: - raise RuntimeError("Batch request failed with only 1 request in it") - self.max_batch_size = ceil(self.max_batch_size / 2) - logger.warning(f"Caught retryable error: Too much data was requested in batch. Reducing batch size to {self.max_batch_size}") - - def failure(response: FacebookResponse, request: Optional[FacebookRequest] = None): - # although it is Optional in the signature for compatibility, we need it always - assert request, "Missing a request object" - resp_body = response.json() - if not isinstance(resp_body, dict): - raise RuntimeError(f"Batch request failed with response: {resp_body}") - elif resp_body.get("error", {}).get("message") == "Please reduce the amount of data you're asking for, then retry your request": - reduce_batch_size(request) - elif resp_body.get("error", {}).get("code") != FACEBOOK_BATCH_ERROR_CODE: - raise RuntimeError(f"Batch request failed with response: {resp_body}; unknown error code") - requests_q.put(request) - - api_batch: FacebookAdsApiBatch = self._api.api.new_batch() - - while not requests_q.empty(): - request = requests_q.get() - api_batch.add_request(request, success=success, failure=partial(failure, request=request)) - if len(api_batch) == self.max_batch_size or requests_q.empty(): - # make a call for every max_batch_size items or less if it is the last call - self._execute_batch(api_batch) - yield from records - records = [] - api_batch: FacebookAdsApiBatch = self._api.api.new_batch() - - yield from records + if self._fields: + return self._fields + self._saved_fields = list(self.get_json_schema().get("properties", {}).keys()) + return self._saved_fields + + @classmethod + def fix_date_time(cls, record): + date_time_fields = ( + "created_time", + "creation_time", + "updated_time", + "event_time", + "start_time", + "first_fired_time", + "last_fired_time", + ) + + if isinstance(record, dict): + for field, value in record.items(): + if isinstance(value, str): + if field in date_time_fields: + record[field] = value.replace("t", "T").replace(" 0000", "+0000") + else: + cls.fix_date_time(value) + + elif isinstance(record, list): + for entry in record: + cls.fix_date_time(entry) + + @staticmethod + def add_account_id(record, account_id: str): + if "account_id" not in record: + record["account_id"] = account_id + + def get_account_state(self, account_id: str, stream_state: Mapping[str, Any] = None) -> MutableMapping[str, Any]: + """ + Retrieve the state for a specific account. + + If multiple account IDs are present, the state for the specific account ID + is returned if it exists in the stream state. If only one account ID is + present, the entire stream state is returned. + + :param account_id: The account ID for which to retrieve the state. + :param stream_state: The current stream state, optional. + :return: The state information for the specified account as a MutableMapping. + """ + if stream_state and account_id and account_id in stream_state: + account_state = stream_state.get(account_id) + + # copy `include_deleted` from general stream state + if "include_deleted" in stream_state: + account_state["include_deleted"] = stream_state["include_deleted"] + return account_state + elif len(self._account_ids) == 1: + return stream_state + else: + return {} + + def _transform_state_from_old_format(self, state: Mapping[str, Any], move_fields: List[str] = None) -> Mapping[str, Any]: + """ + Transforms the state from an old format to a new format based on account IDs. + + This method transforms the old state to be a dictionary where the keys are account IDs. + If the state is in the old format (not keyed by account IDs), it will transform the state + by nesting it under the account ID. + + :param state: The original state dictionary to transform. + :param move_fields: A list of field names whose values should be moved to the top level of the new state dictionary. + :return: The transformed state dictionary. + """ + + # If the state already contains any of the account IDs, return the state as is. + for account_id in self._account_ids: + if account_id in state: + return state + + # Handle the case where there is only one account ID. + # Transform the state by nesting it under the account ID. + if state and len(self._account_ids) == 1: + account_id = self._account_ids[0] + new_state = {account_id: state} + + # Move specified fields to the top level of the new state. + if move_fields: + for move_field in move_fields: + if move_field in state: + new_state[move_field] = state.pop(move_field) + + return new_state + + # If the state is empty or there are multiple account IDs, return an empty dictionary. + return {} def read_records( self, @@ -123,16 +153,23 @@ def read_records( stream_state: Mapping[str, Any] = None, ) -> Iterable[Mapping[str, Any]]: """Main read method used by CDK""" - records_iter = self.list_objects(params=self.request_params(stream_state=stream_state)) - loaded_records_iter = (record.api_get(fields=self.fields, pending=self.use_batch) for record in records_iter) - if self.use_batch: - loaded_records_iter = self.execute_in_batch(loaded_records_iter) - - for record in loaded_records_iter: - if isinstance(record, AbstractObject): - yield record.export_all_data() # convert FB object to dict - else: - yield record # execute_in_batch will emmit dicts + account_id = stream_slice["account_id"] + account_state = stream_slice.get("stream_state", {}) + + try: + for record in self.list_objects(params=self.request_params(stream_state=account_state), account_id=account_id): + if isinstance(record, AbstractObject): + record = record.export_all_data() # convert FB object to dict + self.fix_date_time(record) + self.add_account_id(record, stream_slice["account_id"]) + yield record + except FacebookRequestError as exc: + raise traced_exception(exc) + + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + for account_id in self._account_ids: + account_state = self.get_account_state(account_id, stream_state) + yield {"account_id": account_id, "stream_state": account_state} @abstractmethod def list_objects(self, params: Mapping[str, Any]) -> Iterable: @@ -182,27 +219,28 @@ class FBMarketingIncrementalStream(FBMarketingStream, ABC): cursor_field = "updated_time" - def __init__(self, start_date: datetime, end_date: datetime, **kwargs): + def __init__(self, start_date: Optional[datetime], end_date: Optional[datetime], **kwargs): super().__init__(**kwargs) - self._start_date = pendulum.instance(start_date) - self._end_date = pendulum.instance(end_date) - - if self._end_date < self._start_date: - logger.error("The end_date must be after start_date.") + self._start_date = pendulum.instance(start_date) if start_date else None + self._end_date = pendulum.instance(end_date) if end_date else None def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): """Update stream state from latest record""" - potentially_new_records_in_the_past = self._include_deleted and not current_stream_state.get("include_deleted", False) + account_id = latest_record["account_id"] + state_for_accounts = self._transform_state_from_old_format(current_stream_state, ["include_deleted"]) + account_state = self.get_account_state(account_id, state_for_accounts) + + potentially_new_records_in_the_past = self._include_deleted and not account_state.get("include_deleted", False) record_value = latest_record[self.cursor_field] - state_value = current_stream_state.get(self.cursor_field) or record_value + state_value = account_state.get(self.cursor_field) or record_value max_cursor = max(pendulum.parse(state_value), pendulum.parse(record_value)) if potentially_new_records_in_the_past: max_cursor = record_value - return { - self.cursor_field: str(max_cursor), - "include_deleted": self._include_deleted, - } + state_for_accounts.setdefault(account_id, {})[self.cursor_field] = str(max_cursor) + + state_for_accounts["include_deleted"] = self._include_deleted + return state_for_accounts def request_params(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: """Include state filter""" @@ -212,13 +250,24 @@ def request_params(self, stream_state: Mapping[str, Any], **kwargs) -> MutableMa def _state_filter(self, stream_state: Mapping[str, Any]) -> Mapping[str, Any]: """Additional filters associated with state if any set""" + state_value = stream_state.get(self.cursor_field) - filter_value = self._start_date if not state_value else pendulum.parse(state_value) + if stream_state: + filter_value = pendulum.parse(state_value) + elif self._start_date: + filter_value = self._start_date + else: + # if start_date is not specified then do not use date filters + return {} potentially_new_records_in_the_past = self._include_deleted and not stream_state.get("include_deleted", False) if potentially_new_records_in_the_past: self.logger.info(f"Ignoring bookmark for {self.name} because of enabled `include_deleted` option") - filter_value = self._start_date + if self._start_date: + filter_value = self._start_date + else: + # if start_date is not specified then do not use date filters + return {} return { "filtering": [ @@ -238,28 +287,31 @@ class FBMarketingReversedIncrementalStream(FBMarketingIncrementalStream, ABC): def __init__(self, **kwargs): super().__init__(**kwargs) - self._cursor_value = None - self._max_cursor_value = None + self._cursor_values = {} @property def state(self) -> Mapping[str, Any]: """State getter, get current state and serialize it to emmit Airbyte STATE message""" - if self._cursor_value: - return { - self.cursor_field: self._cursor_value, - "include_deleted": self._include_deleted, - } + if self._cursor_values: + result_state = {account_id: {self.cursor_field: cursor_value} for account_id, cursor_value in self._cursor_values.items()} + result_state["include_deleted"] = self._include_deleted + return result_state return {} @state.setter def state(self, value: Mapping[str, Any]): """State setter, ignore state if current settings mismatch saved state""" - if self._include_deleted and not value.get("include_deleted"): + transformed_state = self._transform_state_from_old_format(value, ["include_deleted"]) + if self._include_deleted and not transformed_state.get("include_deleted"): logger.info(f"Ignoring bookmark for {self.name} because of enabled `include_deleted` option") return - self._cursor_value = pendulum.parse(value[self.cursor_field]) + self._cursor_values = {} + for account_id in self._account_ids: + cursor_value = transformed_state.get(account_id, {}).get(self.cursor_field) + if cursor_value is not None: + self._cursor_values[account_id] = pendulum.parse(cursor_value) def _state_filter(self, stream_state: Mapping[str, Any]) -> Mapping[str, Any]: """Don't have classic cursor filtering""" @@ -281,16 +333,27 @@ def read_records( - update state only when we reach the end - stop reading when we reached the end """ - records_iter = self.list_objects(params=self.request_params(stream_state=stream_state)) - for record in records_iter: - record_cursor_value = pendulum.parse(record[self.cursor_field]) - if self._cursor_value and record_cursor_value < self._cursor_value: - break - if not self._include_deleted and self.get_record_deleted_status(record): - continue - - self._max_cursor_value = self._max_cursor_value or record_cursor_value - self._max_cursor_value = max(self._max_cursor_value, record_cursor_value) - yield record.export_all_data() - - self._cursor_value = self._max_cursor_value + account_id = stream_slice["account_id"] + account_state = stream_slice.get("stream_state") + + try: + records_iter = self.list_objects(params=self.request_params(stream_state=account_state), account_id=account_id) + account_cursor = self._cursor_values.get(account_id) + + max_cursor_value = None + for record in records_iter: + record_cursor_value = pendulum.parse(record[self.cursor_field]) + if account_cursor and record_cursor_value < account_cursor: + break + if not self._include_deleted and self.get_record_deleted_status(record): + continue + + max_cursor_value = max(max_cursor_value, record_cursor_value) if max_cursor_value else record_cursor_value + record = record.export_all_data() + self.fix_date_time(record) + self.add_account_id(record, stream_slice["account_id"]) + yield record + + self._cursor_values[account_id] = max_cursor_value + except FacebookRequestError as exc: + raise traced_exception(exc) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py index db1913ffc90e..3947689761b3 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py @@ -9,6 +9,8 @@ import backoff import pendulum +from airbyte_cdk.models import FailureType +from airbyte_cdk.utils import AirbyteTracedException from facebook_business.exceptions import FacebookRequestError # The Facebook API error codes indicating rate-limiting are listed at @@ -40,7 +42,7 @@ def log_retry_attempt(details): def reduce_request_record_limit(details): _, exc, _ = sys.exc_info() # the list of error patterns to track, - # in order to reduce the requestt page size and retry + # in order to reduce the request page size and retry error_patterns = [ "Please reduce the amount of data you're asking for, then retry your request", "An unknown error occurred", @@ -114,3 +116,46 @@ def deep_merge(a: Any, b: Any) -> Any: return a | b else: return a if b is None else b + + +def traced_exception(fb_exception: FacebookRequestError): + """Add user-friendly message for FacebookRequestError + + Please see ../unit_tests/test_errors.py for full error examples + Please add new errors to the tests + """ + msg = fb_exception.api_error_message() + + if "Error validating access token" in msg: + failure_type = FailureType.config_error + friendly_msg = "Invalid access token. Re-authenticate if FB oauth is used or refresh access token with all required permissions" + + elif "(#100) Missing permissions" in msg: + failure_type = FailureType.config_error + friendly_msg = ( + "Credentials don't have enough permissions. Check if correct Ad Account Id is used (as in Ads Manager), " + "re-authenticate if FB oauth is used or refresh access token with all required permissions" + ) + + elif "permission" in msg: + failure_type = FailureType.config_error + friendly_msg = ( + "Credentials don't have enough permissions. Re-authenticate if FB oauth is used or refresh access token " + "with all required permissions." + ) + + elif "An unknown error occurred" in msg and "error_user_title" in fb_exception._error: + msg = fb_exception._error["error_user_title"] + if "profile is not linked to delegate page" in msg or "el perfil no est" in msg: + failure_type = FailureType.config_error + friendly_msg = ( + "Current profile is not linked to delegate page. Check if correct business (not personal) " + "Ad Account Id is used (as in Ads Manager), re-authenticate if FB oauth is used or refresh " + "access token with all required permissions." + ) + + else: + failure_type = FailureType.system_error + friendly_msg = f"Error: {fb_exception.api_error_code()}, {fb_exception.api_error_message()}." + + return AirbyteTracedException(message=friendly_msg or msg, internal_message=msg, failure_type=failure_type, exception=fb_exception) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py index 2f447af8630e..c7fd0237963b 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py @@ -9,11 +9,10 @@ import pendulum import requests from airbyte_cdk.models import SyncMode -from cached_property import cached_property -from facebook_business.adobjects.abstractobject import AbstractObject from facebook_business.adobjects.adaccount import AdAccount as FBAdAccount from facebook_business.adobjects.adimage import AdImage from facebook_business.adobjects.user import User +from facebook_business.exceptions import FacebookRequestError from .base_insight_streams import AdsInsights from .base_streams import FBMarketingIncrementalStream, FBMarketingReversedIncrementalStream, FBMarketingStream @@ -31,8 +30,8 @@ def fetch_thumbnail_data_url(url: str) -> Optional[str]: return f"data:{_type};base64,{data.decode('ascii')}" else: logger.warning(f"Got {repr(response)} while requesting thumbnail image.") - except requests.exceptions.RequestException as exc: - logger.warning(f"Got {str(exc)} while requesting thumbnail image.") + except Exception as exc: + logger.warning(f"Got {str(exc)} while requesting thumbnail image: {url}.") return None @@ -48,10 +47,13 @@ def __init__(self, fetch_thumbnail_images: bool = False, **kwargs): super().__init__(**kwargs) self._fetch_thumbnail_images = fetch_thumbnail_images - @cached_property - def fields(self) -> List[str]: + def fields(self, **kwargs) -> List[str]: """Remove "thumbnail_data_url" field because it is computed field and it's not a field that we can request from Facebook""" - return [f for f in super().fields if f != "thumbnail_data_url"] + if self._fields: + return self._fields + + self._fields = [f for f in super().fields(**kwargs) if f != "thumbnail_data_url"] + return self._fields def read_records( self, @@ -68,8 +70,8 @@ def read_records( record["thumbnail_data_url"] = fetch_thumbnail_data_url(thumbnail_url) yield record - def list_objects(self, params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_ad_creatives(params=params) + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_ad_creatives(params=params, fields=self.fields()) class CustomConversions(FBMarketingStream): @@ -78,12 +80,12 @@ class CustomConversions(FBMarketingStream): entity_prefix = "customconversion" enable_deleted = False - def list_objects(self, params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_custom_conversions(params=params) + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_custom_conversions(params=params, fields=self.fields()) class CustomAudiences(FBMarketingStream): - """doc: https://developers.facebook.com/docs/marketing-api/reference/custom-conversion""" + """doc: https://developers.facebook.com/docs/marketing-api/reference/custom-audience""" entity_prefix = "customaudience" enable_deleted = False @@ -91,8 +93,8 @@ class CustomAudiences(FBMarketingStream): # https://github.com/airbytehq/oncall/issues/2765 fields_exceptions = ["rule"] - def list_objects(self, params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_custom_audiences(params=params) + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_custom_audiences(params=params, fields=self.fields()) class Ads(FBMarketingIncrementalStream): @@ -100,8 +102,8 @@ class Ads(FBMarketingIncrementalStream): entity_prefix = "ad" - def list_objects(self, params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_ads(params=params) + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_ads(params=params, fields=self.fields()) class AdSets(FBMarketingIncrementalStream): @@ -109,8 +111,8 @@ class AdSets(FBMarketingIncrementalStream): entity_prefix = "adset" - def list_objects(self, params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_ad_sets(params=params) + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_ad_sets(params=params, fields=self.fields()) class Campaigns(FBMarketingIncrementalStream): @@ -118,8 +120,8 @@ class Campaigns(FBMarketingIncrementalStream): entity_prefix = "campaign" - def list_objects(self, params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_campaigns(params=params) + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_campaigns(params=params, fields=self.fields()) class Activities(FBMarketingIncrementalStream): @@ -129,34 +131,36 @@ class Activities(FBMarketingIncrementalStream): cursor_field = "event_time" primary_key = None - def list_objects(self, fields: List[str], params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_activities(fields=fields, params=params) + def fields(self, **kwargs) -> List[str]: + """Remove account_id from fields as cannot be requested, but it is part of schema as foreign key, will be added during processing""" + if self._fields: + return self._fields - def read_records( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Mapping[str, Any]]: - """Main read method used by CDK""" - loaded_records_iter = self.list_objects(fields=self.fields, params=self.request_params(stream_state=stream_state)) + self._fields = [f for f in super().fields(**kwargs) if f != "account_id"] + return self._fields - for record in loaded_records_iter: - if isinstance(record, AbstractObject): - yield record.export_all_data() # convert FB object to dict - else: - yield record # execute_in_batch will emmit dicts + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_activities(fields=self.fields(), params=params) def _state_filter(self, stream_state: Mapping[str, Any]) -> Mapping[str, Any]: """Additional filters associated with state if any set""" state_value = stream_state.get(self.cursor_field) - since = self._start_date if not state_value else pendulum.parse(state_value) + if stream_state: + since = pendulum.parse(state_value) + elif self._start_date: + since = self._start_date + else: + # if start_date is not specified then do not use date filters + return {} potentially_new_records_in_the_past = self._include_deleted and not stream_state.get("include_deleted", False) if potentially_new_records_in_the_past: self.logger.info(f"Ignoring bookmark for {self.name} because of enabled `include_deleted` option") - since = self._start_date + if self._start_date: + since = self._start_date + else: + # if start_date is not specified then do not use date filters + return {} return {"since": since.int_timestamp} @@ -166,9 +170,17 @@ class Videos(FBMarketingReversedIncrementalStream): entity_prefix = "video" - def list_objects(self, params: Mapping[str, Any]) -> Iterable: + def fields(self, **kwargs) -> List[str]: + """Remove account_id from fields as cannot be requested, but it is part of schema as foreign key, will be added during processing""" + if self._fields: + return self._fields + + self._fields = [f for f in super().fields() if f != "account_id"] + return self._fields + + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: # Remove filtering as it is not working for this stream since 2023-01-13 - return self._api.account.get_ad_videos(params=params, fields=self.fields) + return self._api.get_account(account_id=account_id).get_ad_videos(params=params, fields=self.fields()) class AdAccount(FBMarketingStream): @@ -177,39 +189,66 @@ class AdAccount(FBMarketingStream): use_batch = False enable_deleted = False - def get_task_permissions(self) -> Set[str]: + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._fields_dict = {} + + def get_task_permissions(self, account_id: str) -> Set[str]: """https://developers.facebook.com/docs/marketing-api/reference/ad-account/assigned_users/""" res = set() me = User(fbid="me", api=self._api.api) for business_user in me.get_business_users(): - assigned_users = self._api.account.get_assigned_users(params={"business": business_user["business"].get_id()}) + assigned_users = self._api.get_account(account_id=account_id).get_assigned_users( + params={"business": business_user["business"].get_id()} + ) for assigned_user in assigned_users: if business_user.get_id() == assigned_user.get_id(): res.update(set(assigned_user["tasks"])) return res - @cached_property - def fields(self) -> List[str]: - properties = super().fields + def fields(self, account_id: str, **kwargs) -> List[str]: + if self._fields_dict.get(account_id): + return self._fields_dict.get(account_id) + + properties = super().fields(**kwargs) # https://developers.facebook.com/docs/marketing-apis/guides/javascript-ads-dialog-for-payments/ # To access "funding_source_details", the user making the API call must have a MANAGE task permission for # that specific ad account. - if "funding_source_details" in properties and "MANAGE" not in self.get_task_permissions(): + permissions = self.get_task_permissions(account_id=account_id) + if "funding_source_details" in properties and "MANAGE" not in permissions: properties.remove("funding_source_details") - if "is_prepay_account" in properties and "MANAGE" not in self.get_task_permissions(): + if "is_prepay_account" in properties and "MANAGE" not in permissions: properties.remove("is_prepay_account") + + self._fields_dict[account_id] = properties return properties - def list_objects(self, params: Mapping[str, Any]) -> Iterable: + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: """noop in case of AdAccount""" - return [FBAdAccount(self._api.account.get_id())] + fields = self.fields(account_id=account_id) + try: + print(f"{self._api.get_account(account_id=account_id).get_id()=} {account_id=}") + return [FBAdAccount(self._api.get_account(account_id=account_id).get_id()).api_get(fields=fields)] + except FacebookRequestError as e: + # This is a workaround for cases when account seem to have all the required permissions + # but despite of that is not allowed to get `owner` field. See (https://github.com/airbytehq/oncall/issues/3167) + if e.api_error_code() == 200 and e.api_error_message() == "(#200) Requires business_management permission to manage the object": + fields.remove("owner") + return [FBAdAccount(self._api.get_account(account_id=account_id).get_id()).api_get(fields=fields)] + # FB api returns a non-obvious error when accessing the `funding_source_details` field + # even though user is granted all the required permissions (`MANAGE`) + # https://github.com/airbytehq/oncall/issues/3031 + if e.api_error_code() == 100 and e.api_error_message() == "Unsupported request - method type: get": + fields.remove("funding_source_details") + return [FBAdAccount(self._api.get_account(account_id=account_id).get_id()).api_get(fields=fields)] + raise e class Images(FBMarketingReversedIncrementalStream): """See: https://developers.facebook.com/docs/marketing-api/reference/ad-image""" - def list_objects(self, params: Mapping[str, Any]) -> Iterable: - return self._api.account.get_ad_images(params=params, fields=self.fields) + def list_objects(self, params: Mapping[str, Any], account_id: str) -> Iterable: + return self._api.get_account(account_id=account_id).get_ad_images(params=params, fields=self.fields(account_id=account_id)) def get_record_deleted_status(self, record) -> bool: return record[AdImage.Field.status] == AdImage.Status.deleted diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py index ad2454b02ea4..a7574ce206f9 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/conftest.py @@ -23,7 +23,7 @@ def account_id_fixture(): @fixture(scope="session", name="some_config") def some_config_fixture(account_id): - return {"start_date": "2021-01-23T00:00:00Z", "account_id": f"{account_id}", "access_token": "unknown_token"} + return {"start_date": "2021-01-23T00:00:00Z", "account_ids": [f"{account_id}"], "access_token": "unknown_token"} @fixture(autouse=True) @@ -49,8 +49,10 @@ def fb_account_response_fixture(account_id): @fixture(name="api") def api_fixture(some_config, requests_mock, fb_account_response): - api = API(account_id=some_config["account_id"], access_token=some_config["access_token"], page_size=100) + api = API(access_token=some_config["access_token"], page_size=100) requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/me/adaccounts", [fb_account_response]) - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{some_config['account_id']}/", [fb_account_response]) + requests_mock.register_uri( + "GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{some_config['account_ids'][0]}/", [fb_account_response] + ) return api diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_api.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_api.py index 3bc8a37c2db8..29b2ccbfaaff 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_api.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_api.py @@ -136,6 +136,6 @@ def test__handle_call_rate_limit(self, mocker, fb_api, params, min_rate, usage, def test_find_account(self, api, account_id, requests_mock): requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/", [{"json": {"id": "act_test"}}]) - account = api._find_account(account_id) + account = api.get_account(account_id) assert isinstance(account, AdAccount) assert account.get_id() == "act_test" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py index 69941a42501d..a969999ee383 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job.py @@ -6,6 +6,7 @@ import time from typing import Iterator +import freezegun import pendulum import pytest from facebook_business.adobjects.ad import Ad @@ -48,7 +49,7 @@ def job_fixture(api, account): } interval = pendulum.Period(pendulum.Date(2019, 1, 1), pendulum.Date(2019, 1, 1)) - return InsightAsyncJob(edge_object=account, api=api, interval=interval, params=params) + return InsightAsyncJob(edge_object=account, api=api, interval=interval, params=params, job_timeout= pendulum.duration(minutes=60)) @pytest.fixture(name="grouped_jobs") @@ -206,7 +207,7 @@ def test_update_job(self, started_job, adreport): adreport.api_get.assert_called_once() def test_update_job_expired(self, started_job, adreport, mocker): - mocker.patch.object(started_job, "job_timeout", new=pendulum.Duration()) + mocker.patch.object(started_job, "_job_timeout", new=pendulum.Duration()) started_job.update_job() assert started_job.failed @@ -284,6 +285,7 @@ def test_str(self, api, account): api=api, params={"breakdowns": [10, 20]}, interval=interval, + job_timeout=pendulum.duration(minutes=60) ) assert str(job) == f"InsightAsyncJob(id=, {account}, time_range= 2011-01-01]>, breakdowns=[10, 20])" @@ -328,12 +330,13 @@ def test_get_result_when_job_is_failed(self, failed_job): (AdSet, Ad, "ad_id"), ], ) + @freezegun.freeze_time("2023-10-29") def test_split_job(self, mocker, api, edge_class, next_edge_class, id_field): """Test that split will correctly downsize edge_object""" today = pendulum.today().date() start, end = today - pendulum.duration(days=365 * 3 + 20), today - pendulum.duration(days=365 * 3 + 10) params = {"time_increment": 1, "breakdowns": []} - job = InsightAsyncJob(api=api, edge_object=edge_class(1), interval=pendulum.Period(start, end), params=params) + job = InsightAsyncJob(api=api, edge_object=edge_class(1), interval=pendulum.Period(start, end), params=params, job_timeout=pendulum.duration(minutes=60)) mocker.patch.object(edge_class, "get_insights", return_value=[{id_field: 1}, {id_field: 2}, {id_field: 3}]) small_jobs = job.split_job() @@ -344,8 +347,12 @@ def test_split_job(self, mocker, api, edge_class, next_edge_class, id_field): "fields": [id_field], "level": next_edge_class.__name__.lower(), "time_range": { + # This time range is valid for dates that share the same day of the month + # with the one 37 months ago, that's why current date is frozen. + # For a different date the since date would be also different. + # See facebook_marketing.utils.validate_start_date for reference "since": (today - pendulum.duration(months=37) + pendulum.duration(days=1)).to_date_string(), - "until": end.to_date_string() + "until": end.to_date_string(), }, } ) @@ -359,7 +366,7 @@ def test_split_job_smallest(self, mocker, api): """Test that split will correctly downsize edge_object""" interval = pendulum.Period(pendulum.Date(2010, 1, 1), pendulum.Date(2010, 1, 10)) params = {"time_increment": 1, "breakdowns": []} - job = InsightAsyncJob(api=api, edge_object=Ad(1), interval=interval, params=params) + job = InsightAsyncJob(api=api, edge_object=Ad(1), interval=interval, params=params, job_timeout=pendulum.duration(minutes=60)) with pytest.raises(ValueError, match="The job is already splitted to the smallest size."): job.split_job() diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job_manager.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job_manager.py index a9234fc31465..cb0cffffeabb 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job_manager.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_async_job_manager.py @@ -29,24 +29,24 @@ def update_job_mock_fixture(mocker): class TestInsightAsyncManager: - def test_jobs_empty(self, api): + def test_jobs_empty(self, api, some_config): """Should work event without jobs""" - manager = InsightAsyncJobManager(api=api, jobs=[]) + manager = InsightAsyncJobManager(api=api, jobs=[], account_id=some_config["account_ids"][0]) jobs = list(manager.completed_jobs()) assert not jobs - def test_jobs_completed_immediately(self, api, mocker, time_mock): + def test_jobs_completed_immediately(self, api, mocker, time_mock, some_config): """Manager should emmit jobs without waiting if they completed""" jobs = [ mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False), mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False), ] - manager = InsightAsyncJobManager(api=api, jobs=jobs) + manager = InsightAsyncJobManager(api=api, jobs=jobs, account_id=some_config["account_ids"][0]) completed_jobs = list(manager.completed_jobs()) assert jobs == completed_jobs time_mock.sleep.assert_not_called() - def test_jobs_wait(self, api, mocker, time_mock, update_job_mock): + def test_jobs_wait(self, api, mocker, time_mock, update_job_mock, some_config): """Manager should return completed jobs and wait for others""" def update_job_behaviour(): @@ -61,7 +61,7 @@ def update_job_behaviour(): mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=False), mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=False), ] - manager = InsightAsyncJobManager(api=api, jobs=jobs) + manager = InsightAsyncJobManager(api=api, jobs=jobs, account_id=some_config["account_ids"][0]) job = next(manager.completed_jobs(), None) assert job == jobs[1] @@ -74,7 +74,7 @@ def update_job_behaviour(): job = next(manager.completed_jobs(), None) assert job is None - def test_job_restarted(self, api, mocker, time_mock, update_job_mock): + def test_job_restarted(self, api, mocker, time_mock, update_job_mock, some_config): """Manager should restart failed jobs""" def update_job_behaviour(): @@ -89,7 +89,7 @@ def update_job_behaviour(): mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=True), mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=False), ] - manager = InsightAsyncJobManager(api=api, jobs=jobs) + manager = InsightAsyncJobManager(api=api, jobs=jobs, account_id=some_config["account_ids"][0]) job = next(manager.completed_jobs(), None) assert job == jobs[0] @@ -101,7 +101,7 @@ def update_job_behaviour(): job = next(manager.completed_jobs(), None) assert job is None - def test_job_split(self, api, mocker, time_mock, update_job_mock): + def test_job_split(self, api, mocker, time_mock, update_job_mock, some_config): """Manager should split failed jobs when they fail second time""" def update_job_behaviour(): @@ -121,7 +121,7 @@ def update_job_behaviour(): sub_jobs[0].get_result.return_value = [1, 2] sub_jobs[1].get_result.return_value = [3, 4] jobs[1].split_job.return_value = sub_jobs - manager = InsightAsyncJobManager(api=api, jobs=jobs) + manager = InsightAsyncJobManager(api=api, jobs=jobs, account_id=some_config["account_ids"][0]) job = next(manager.completed_jobs(), None) assert job == jobs[0] @@ -134,7 +134,7 @@ def update_job_behaviour(): job = next(manager.completed_jobs(), None) assert job is None - def test_job_failed_too_many_times(self, api, mocker, time_mock, update_job_mock): + def test_job_failed_too_many_times(self, api, mocker, time_mock, update_job_mock, some_config): """Manager should fail when job failed too many times""" def update_job_behaviour(): @@ -147,12 +147,12 @@ def update_job_behaviour(): mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=True), mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=False), ] - manager = InsightAsyncJobManager(api=api, jobs=jobs) + manager = InsightAsyncJobManager(api=api, jobs=jobs, account_id=some_config["account_ids"][0]) with pytest.raises(JobException, match=f"{jobs[1]}: failed more than {InsightAsyncJobManager.MAX_NUMBER_OF_ATTEMPTS} times."): next(manager.completed_jobs(), None) - def test_nested_job_failed_too_many_times(self, api, mocker, time_mock, update_job_mock): + def test_nested_job_failed_too_many_times(self, api, mocker, time_mock, update_job_mock, some_config): """Manager should fail when a nested job within a ParentAsyncJob failed too many times""" def update_job_behaviour(): @@ -170,7 +170,7 @@ def update_job_behaviour(): mocker.Mock(spec=InsightAsyncJob, attempt_number=1, failed=False, completed=True), mocker.Mock(spec=ParentAsyncJob, _jobs=sub_jobs, attempt_number=1, failed=False, completed=False), ] - manager = InsightAsyncJobManager(api=api, jobs=jobs) + manager = InsightAsyncJobManager(api=api, jobs=jobs, account_id=some_config["account_ids"][0]) with pytest.raises(JobException): next(manager.completed_jobs(), None) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py index c773ad505dec..3d6ef2aaa5e6 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_insight_streams.py @@ -48,8 +48,14 @@ def async_job_mock_fixture(mocker): class TestBaseInsightsStream: - def test_init(self, api): - stream = AdsInsights(api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28) + def test_init(self, api, some_config): + stream = AdsInsights( + api=api, + account_ids=some_config["account_ids"], + start_date=datetime(2010, 1, 1), + end_date=datetime(2011, 1, 1), + insights_lookback_window=28, + ) assert not stream.breakdowns assert stream.action_breakdowns == ["action_type", "action_target_id", "action_destination"] @@ -57,9 +63,10 @@ def test_init(self, api): assert stream.primary_key == ["date_start", "account_id", "ad_id"] assert stream.action_report_time == "mixed" - def test_init_override(self, api): + def test_init_override(self, api, some_config): stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), name="CustomName", @@ -73,7 +80,7 @@ def test_init_override(self, api): assert stream.name == "custom_name" assert stream.primary_key == ["date_start", "account_id", "ad_id", "test1", "test2"] - def test_read_records_all(self, mocker, api): + def test_read_records_all(self, mocker, api, some_config): """1. yield all from mock 2. if read slice 2, 3 state not changed if read slice 2, 3, 1 state changed to 3 @@ -83,6 +90,7 @@ def test_read_records_all(self, mocker, api): job.interval = pendulum.Period(pendulum.date(2010, 1, 1), pendulum.date(2010, 1, 1)) stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28, @@ -91,13 +99,13 @@ def test_read_records_all(self, mocker, api): records = list( stream.read_records( sync_mode=SyncMode.incremental, - stream_slice={"insight_job": job}, + stream_slice={"insight_job": job, "account_id": some_config["account_ids"][0]}, ) ) assert len(records) == 3 - def test_read_records_random_order(self, mocker, api): + def test_read_records_random_order(self, mocker, api, some_config): """1. yield all from mock 2. if read slice 2, 3 state not changed if read slice 2, 3, 1 state changed to 3 @@ -105,62 +113,144 @@ def test_read_records_random_order(self, mocker, api): job = mocker.Mock(spec=AsyncJob) job.get_result.return_value = [mocker.Mock(), mocker.Mock(), mocker.Mock()] job.interval = pendulum.Period(pendulum.date(2010, 1, 1), pendulum.date(2010, 1, 1)) - stream = AdsInsights(api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28) + stream = AdsInsights( + api=api, + account_ids=some_config["account_ids"], + start_date=datetime(2010, 1, 1), + end_date=datetime(2011, 1, 1), + insights_lookback_window=28, + ) records = list( stream.read_records( sync_mode=SyncMode.incremental, - stream_slice={"insight_job": job}, + stream_slice={"insight_job": job, "account_id": some_config["account_ids"][0]}, ) ) assert len(records) == 3 @pytest.mark.parametrize( - "state", + "state,result_state", [ - { - AdsInsights.cursor_field: "2010-10-03", - "slices": [ - "2010-01-01", - "2010-01-02", - ], - "time_increment": 1, - }, - { - AdsInsights.cursor_field: "2010-10-03", - }, - { - "slices": [ - "2010-01-01", - "2010-01-02", - ] - }, + # Old format + ( + { + AdsInsights.cursor_field: "2010-10-03", + "slices": [ + "2010-01-01", + "2010-01-02", + ], + "time_increment": 1, + }, + { + "unknown_account": { + AdsInsights.cursor_field: "2010-10-03", + "slices": { + "2010-01-01", + "2010-01-02", + }, + }, + "time_increment": 1, + }, + ), + ( + { + AdsInsights.cursor_field: "2010-10-03", + }, + { + "unknown_account": { + AdsInsights.cursor_field: "2010-10-03", + } + }, + ), + ( + { + "slices": [ + "2010-01-01", + "2010-01-02", + ] + }, + { + "unknown_account": { + "slices": { + "2010-01-01", + "2010-01-02", + } + } + }, + ), + # New format - nested with account_id + ( + { + "unknown_account": { + AdsInsights.cursor_field: "2010-10-03", + "slices": { + "2010-01-01", + "2010-01-02", + }, + }, + "time_increment": 1, + }, + None, + ), + ( + { + "unknown_account": { + AdsInsights.cursor_field: "2010-10-03", + } + }, + None, + ), + ( + { + "unknown_account": { + "slices": { + "2010-01-01", + "2010-01-02", + } + } + }, + None, + ), ], ) - def test_state(self, api, state): + def test_state(self, api, state, result_state, some_config): """State setter/getter should work with all combinations""" - stream = AdsInsights(api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28) + stream = AdsInsights( + api=api, + account_ids=some_config["account_ids"], + start_date=datetime(2010, 1, 1), + end_date=datetime(2011, 1, 1), + insights_lookback_window=28, + ) - assert stream.state == {} + assert stream.state == {"time_increment": 1, "unknown_account": {"slices": set()}} stream.state = state actual_state = stream.state - actual_state["slices"] = sorted(actual_state.get("slices", [])) - state["slices"] = sorted(state.get("slices", [])) - state["time_increment"] = 1 - assert actual_state == state + result_state = state if not result_state else result_state + result_state[some_config["account_ids"][0]]["slices"] = result_state[some_config["account_ids"][0]].get("slices", set()) + result_state["time_increment"] = 1 - def test_stream_slices_no_state(self, api, async_manager_mock, start_date): + assert actual_state == result_state + + def test_stream_slices_no_state(self, api, async_manager_mock, start_date, some_config): """Stream will use start_date when there is not state""" end_date = start_date + duration(weeks=2) - stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=28) + stream = AdsInsights( + api=api, account_ids=some_config["account_ids"], start_date=start_date, end_date=end_date, insights_lookback_window=28 + ) async_manager_mock.completed_jobs.return_value = [1, 2, 3] slices = list(stream.stream_slices(stream_state=None, sync_mode=SyncMode.incremental)) - assert slices == [{"insight_job": 1}, {"insight_job": 2}, {"insight_job": 3}] + assert slices == [ + {"account_id": "unknown_account", "insight_job": 1}, + {"account_id": "unknown_account", "insight_job": 2}, + {"account_id": "unknown_account", "insight_job": 3}, + ] async_manager_mock.assert_called_once() args, kwargs = async_manager_mock.call_args generated_jobs = list(kwargs["jobs"]) @@ -168,16 +258,22 @@ def test_stream_slices_no_state(self, api, async_manager_mock, start_date): assert generated_jobs[0].interval.start == start_date.date() assert generated_jobs[1].interval.start == start_date.date() + duration(days=1) - def test_stream_slices_no_state_close_to_now(self, api, async_manager_mock, recent_start_date): + def test_stream_slices_no_state_close_to_now(self, api, async_manager_mock, recent_start_date, some_config): """Stream will use start_date when there is not state and start_date within 28d from now""" start_date = recent_start_date end_date = pendulum.now() - stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=28) + stream = AdsInsights( + api=api, account_ids=some_config["account_ids"], start_date=start_date, end_date=end_date, insights_lookback_window=28 + ) async_manager_mock.completed_jobs.return_value = [1, 2, 3] slices = list(stream.stream_slices(stream_state=None, sync_mode=SyncMode.incremental)) - assert slices == [{"insight_job": 1}, {"insight_job": 2}, {"insight_job": 3}] + assert slices == [ + {"account_id": "unknown_account", "insight_job": 1}, + {"account_id": "unknown_account", "insight_job": 2}, + {"account_id": "unknown_account", "insight_job": 3}, + ] async_manager_mock.assert_called_once() args, kwargs = async_manager_mock.call_args generated_jobs = list(kwargs["jobs"]) @@ -185,17 +281,23 @@ def test_stream_slices_no_state_close_to_now(self, api, async_manager_mock, rece assert generated_jobs[0].interval.start == start_date.date() assert generated_jobs[1].interval.start == start_date.date() + duration(days=1) - def test_stream_slices_with_state(self, api, async_manager_mock, start_date): + def test_stream_slices_with_state(self, api, async_manager_mock, start_date, some_config): """Stream will use cursor_value from state when there is state""" end_date = start_date + duration(days=10) cursor_value = start_date + duration(days=5) state = {AdsInsights.cursor_field: cursor_value.date().isoformat()} - stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=28) + stream = AdsInsights( + api=api, account_ids=some_config["account_ids"], start_date=start_date, end_date=end_date, insights_lookback_window=28 + ) async_manager_mock.completed_jobs.return_value = [1, 2, 3] slices = list(stream.stream_slices(stream_state=state, sync_mode=SyncMode.incremental)) - assert slices == [{"insight_job": 1}, {"insight_job": 2}, {"insight_job": 3}] + assert slices == [ + {"account_id": "unknown_account", "insight_job": 1}, + {"account_id": "unknown_account", "insight_job": 2}, + {"account_id": "unknown_account", "insight_job": 3}, + ] async_manager_mock.assert_called_once() args, kwargs = async_manager_mock.call_args generated_jobs = list(kwargs["jobs"]) @@ -203,18 +305,24 @@ def test_stream_slices_with_state(self, api, async_manager_mock, start_date): assert generated_jobs[0].interval.start == cursor_value.date() + duration(days=1) assert generated_jobs[1].interval.start == cursor_value.date() + duration(days=2) - def test_stream_slices_with_state_close_to_now(self, api, async_manager_mock, recent_start_date): + def test_stream_slices_with_state_close_to_now(self, api, async_manager_mock, recent_start_date, some_config): """Stream will use start_date when close to now and start_date close to now""" start_date = recent_start_date end_date = pendulum.now() cursor_value = end_date - duration(days=1) state = {AdsInsights.cursor_field: cursor_value.date().isoformat()} - stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=28) + stream = AdsInsights( + api=api, account_ids=some_config["account_ids"], start_date=start_date, end_date=end_date, insights_lookback_window=28 + ) async_manager_mock.completed_jobs.return_value = [1, 2, 3] slices = list(stream.stream_slices(stream_state=state, sync_mode=SyncMode.incremental)) - assert slices == [{"insight_job": 1}, {"insight_job": 2}, {"insight_job": 3}] + assert slices == [ + {"account_id": "unknown_account", "insight_job": 1}, + {"account_id": "unknown_account", "insight_job": 2}, + {"account_id": "unknown_account", "insight_job": 3}, + ] async_manager_mock.assert_called_once() args, kwargs = async_manager_mock.call_args generated_jobs = list(kwargs["jobs"]) @@ -222,20 +330,36 @@ def test_stream_slices_with_state_close_to_now(self, api, async_manager_mock, re assert generated_jobs[0].interval.start == start_date.date() assert generated_jobs[1].interval.start == start_date.date() + duration(days=1) - def test_stream_slices_with_state_and_slices(self, api, async_manager_mock, start_date): + @pytest.mark.parametrize("state_format", ["old_format", "new_format"]) + def test_stream_slices_with_state_and_slices(self, api, async_manager_mock, start_date, some_config, state_format): """Stream will use cursor_value from state, but will skip saved slices""" end_date = start_date + duration(days=10) cursor_value = start_date + duration(days=5) - state = { - AdsInsights.cursor_field: cursor_value.date().isoformat(), - "slices": [(cursor_value + duration(days=1)).date().isoformat(), (cursor_value + duration(days=3)).date().isoformat()], - } - stream = AdsInsights(api=api, start_date=start_date, end_date=end_date, insights_lookback_window=28) + + if state_format == "old_format": + state = { + AdsInsights.cursor_field: cursor_value.date().isoformat(), + "slices": [(cursor_value + duration(days=1)).date().isoformat(), (cursor_value + duration(days=3)).date().isoformat()], + } + else: + state = { + "unknown_account": { + AdsInsights.cursor_field: cursor_value.date().isoformat(), + "slices": [(cursor_value + duration(days=1)).date().isoformat(), (cursor_value + duration(days=3)).date().isoformat()], + } + } + stream = AdsInsights( + api=api, account_ids=some_config["account_ids"], start_date=start_date, end_date=end_date, insights_lookback_window=28 + ) async_manager_mock.completed_jobs.return_value = [1, 2, 3] slices = list(stream.stream_slices(stream_state=state, sync_mode=SyncMode.incremental)) - assert slices == [{"insight_job": 1}, {"insight_job": 2}, {"insight_job": 3}] + assert slices == [ + {"account_id": "unknown_account", "insight_job": 1}, + {"account_id": "unknown_account", "insight_job": 2}, + {"account_id": "unknown_account", "insight_job": 3}, + ] async_manager_mock.assert_called_once() args, kwargs = async_manager_mock.call_args generated_jobs = list(kwargs["jobs"]) @@ -243,18 +367,25 @@ def test_stream_slices_with_state_and_slices(self, api, async_manager_mock, star assert generated_jobs[0].interval.start == cursor_value.date() + duration(days=2) assert generated_jobs[1].interval.start == cursor_value.date() + duration(days=4) - def test_get_json_schema(self, api): - stream = AdsInsights(api=api, start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28) + def test_get_json_schema(self, api, some_config): + stream = AdsInsights( + api=api, + account_ids=some_config["account_ids"], + start_date=datetime(2010, 1, 1), + end_date=datetime(2011, 1, 1), + insights_lookback_window=28, + ) schema = stream.get_json_schema() assert "device_platform" not in schema["properties"] assert "country" not in schema["properties"] - assert not (set(stream.fields) - set(schema["properties"].keys())), "all fields present in schema" + assert not (set(stream.fields()) - set(schema["properties"].keys())), "all fields present in schema" - def test_get_json_schema_custom(self, api): + def test_get_json_schema_custom(self, api, some_config): stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), breakdowns=["device_platform", "country"], @@ -265,43 +396,64 @@ def test_get_json_schema_custom(self, api): assert "device_platform" in schema["properties"] assert "country" in schema["properties"] - assert not (set(stream.fields) - set(schema["properties"].keys())), "all fields present in schema" + assert not (set(stream.fields()) - set(schema["properties"].keys())), "all fields present in schema" - def test_fields(self, api): + def test_fields(self, api, some_config): stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), insights_lookback_window=28, ) - fields = stream.fields + fields = stream.fields() assert "account_id" in fields assert "account_currency" in fields assert "actions" in fields - def test_fields_custom(self, api): + def test_fields_custom(self, api, some_config): stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), fields=["account_id", "account_currency"], insights_lookback_window=28, ) - assert stream.fields == ["account_id", "account_currency"] + assert stream.fields() == ["account_id", "account_currency"] schema = stream.get_json_schema() - assert schema["properties"].keys() == set(["account_currency", "account_id", stream.cursor_field]) + assert schema["properties"].keys() == set(["account_currency", "account_id", stream.cursor_field, "date_stop", "ad_id"]) - def test_level_custom(self, api): + def test_level_custom(self, api, some_config): stream = AdsInsights( api=api, + account_ids=some_config["account_ids"], start_date=datetime(2010, 1, 1), end_date=datetime(2011, 1, 1), fields=["account_id", "account_currency"], insights_lookback_window=28, - level="adset" + level="adset", ) assert stream.level == "adset" + + def test_breackdowns_fields_present_in_response_data(self, api, some_config): + stream = AdsInsights( + api=api, + account_ids=some_config["account_ids"], + start_date=datetime(2010, 1, 1), + end_date=datetime(2011, 1, 1), + breakdowns=["age", "gender"], + insights_lookback_window=28, + ) + + data = {"age": "0-100", "gender": "male"} + + assert stream._response_data_is_valid(data) + + data = {"id": "0000001", "name": "Pipenpodl Absakopalis"} + + assert not stream._response_data_is_valid(data) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_streams.py index c64f1c778ba8..66604660645f 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_base_streams.py @@ -2,15 +2,14 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import json from functools import partial from typing import Any, Iterable, Mapping import pytest from facebook_business import FacebookSession -from facebook_business.api import FacebookAdsApi, FacebookAdsApiBatch, FacebookRequest +from facebook_business.api import FacebookAdsApi, FacebookAdsApiBatch from source_facebook_marketing.api import MyFacebookAdsApi -from source_facebook_marketing.streams.base_streams import FBMarketingStream +from source_facebook_marketing.streams.base_streams import FBMarketingIncrementalStream, FBMarketingStream @pytest.fixture(name="mock_batch_responses") @@ -32,163 +31,119 @@ def list_objects(self, params: Mapping[str, Any]) -> Iterable: yield from [] -class TestBaseStream: - def test_execute_in_batch_with_few_requests(self, api, batch, mock_batch_responses): - """Should execute single batch if number of requests less than MAX_BATCH_SIZE.""" - mock_batch_responses( - [ +class TestDateTimeValue: + def test_date_time_value(self): + record = { + "bla": "2023-01-19t20:38:59 0000", + "created_time": "2023-01-19t20:38:59 0000", + "creation_time": "2023-01-19t20:38:59 0000", + "updated_time": "2023-01-19t20:38:59 0000", + "event_time": "2023-01-19t20:38:59 0000", + "first_fired_time": "2023-01-19t20:38:59 0000", + "last_fired_time": "2023-01-19t20:38:59 0000", + "sub_list": [ { - "json": [{"body": json.dumps({"name": "creative 1"}), "code": 200, "headers": {}}] * 3, + "bla": "2023-01-19t20:38:59 0000", + "created_time": "2023-01-19t20:38:59 0000", + "creation_time": "2023-01-19t20:38:59 0000", + "updated_time": "2023-01-19t20:38:59 0000", + "event_time": "2023-01-19t20:38:59 0000", + "first_fired_time": "2023-01-19t20:38:59 0000", + "last_fired_time": "2023-01-19t20:38:59 0000", } - ] - ) + ], + "sub_entries1": { + "sub_entries2": { + "bla": "2023-01-19t20:38:59 0000", + "created_time": "2023-01-19t20:38:59 0000", + "creation_time": "2023-01-19t20:38:59 0000", + "updated_time": "2023-01-19t20:38:59 0000", + "event_time": "2023-01-19t20:38:59 0000", + "first_fired_time": "2023-01-19t20:38:59 0000", + "last_fired_time": "2023-01-19t20:38:59 0000", + } + }, + } + FBMarketingStream.fix_date_time(record) + assert { + "bla": "2023-01-19t20:38:59 0000", + "created_time": "2023-01-19T20:38:59+0000", + "creation_time": "2023-01-19T20:38:59+0000", + "updated_time": "2023-01-19T20:38:59+0000", + "event_time": "2023-01-19T20:38:59+0000", + "first_fired_time": "2023-01-19T20:38:59+0000", + "last_fired_time": "2023-01-19T20:38:59+0000", + "sub_list": [ + { + "bla": "2023-01-19t20:38:59 0000", + "created_time": "2023-01-19T20:38:59+0000", + "creation_time": "2023-01-19T20:38:59+0000", + "updated_time": "2023-01-19T20:38:59+0000", + "event_time": "2023-01-19T20:38:59+0000", + "first_fired_time": "2023-01-19T20:38:59+0000", + "last_fired_time": "2023-01-19T20:38:59+0000", + } + ], + "sub_entries1": { + "sub_entries2": { + "bla": "2023-01-19t20:38:59 0000", + "created_time": "2023-01-19T20:38:59+0000", + "creation_time": "2023-01-19T20:38:59+0000", + "updated_time": "2023-01-19T20:38:59+0000", + "event_time": "2023-01-19T20:38:59+0000", + "first_fired_time": "2023-01-19T20:38:59+0000", + "last_fired_time": "2023-01-19T20:38:59+0000", + } + }, + } == record - stream = SomeTestStream(api=api) - requests = [FacebookRequest("node", "GET", "endpoint") for _ in range(5)] - result = list(stream.execute_in_batch(requests)) +class ConcreteFBMarketingIncrementalStream(FBMarketingIncrementalStream): + cursor_field = "date" - assert batch.add_request.call_count == len(requests) - batch.execute.assert_called_once() - assert len(result) == 3 + def list_objects(self, **kwargs): + return [] - def test_execute_in_batch_with_many_requests(self, api, batch, mock_batch_responses): - """Should execute as many batches as needed if number of requests bigger than MAX_BATCH_SIZE.""" - mock_batch_responses( - [ - { - "json": [{"body": json.dumps({"name": "creative 1"}), "code": 200, "headers": {}}] * 5, - } - ] - ) - stream = SomeTestStream(api=api) - requests = [FacebookRequest("node", "GET", "endpoint") for _ in range(50 + 1)] +@pytest.fixture +def incremental_class_instance(api): + return ConcreteFBMarketingIncrementalStream(api=api, account_ids=["123", "456", "789"], start_date=None, end_date=None) - result = list(stream.execute_in_batch(requests)) - assert batch.add_request.call_count == len(requests) - assert batch.execute.call_count == 2 - assert len(result) == 5 * 2 +class TestFBMarketingIncrementalStreamSliceAndState: + def test_stream_slices_multiple_accounts_with_state(self, incremental_class_instance): + stream_state = {"123": {"state_key": "state_value"}, "456": {"state_key": "another_state_value"}} + expected_slices = [ + {"account_id": "123", "stream_state": {"state_key": "state_value"}}, + {"account_id": "456", "stream_state": {"state_key": "another_state_value"}}, + {"account_id": "789", "stream_state": {}}, + ] + assert list(incremental_class_instance.stream_slices(stream_state)) == expected_slices - def test_execute_in_batch_with_retries(self, api, batch, mock_batch_responses): - """Should retry batch execution until succeed""" - # batch.execute.side_effect = [batch, batch, None] - mock_batch_responses( - [ - { - "json": [ - {}, - {}, - {"body": json.dumps({"name": "creative 1"}), "code": 200, "headers": {}}, - ], - }, - { - "json": [ - {}, - {"body": json.dumps({"name": "creative 1"}), "code": 200, "headers": {}}, - ], - }, - { - "json": [ - {"body": json.dumps({"name": "creative 1"}), "code": 200, "headers": {}}, - ], - }, - ] - ) - - stream = SomeTestStream(api=api) - requests = [FacebookRequest("node", "GET", "endpoint") for _ in range(3)] - - result = list(stream.execute_in_batch(requests)) - - assert batch.add_request.call_count == len(requests) - assert batch.execute.call_count == 1 - assert len(result) == 3 - - def test_execute_in_batch_with_fails(self, api, batch, mock_batch_responses): - """Should fail with exception when any request returns error""" - mock_batch_responses( - [ - { - "json": [ - {"body": "{}", "code": 500, "headers": {}}, - {"body": json.dumps({"name": "creative 1"}), "code": 200, "headers": {}}, - ], - } - ] - ) + def test_stream_slices_multiple_accounts_empty_state(self, incremental_class_instance): + expected_slices = [ + {"account_id": "123", "stream_state": {}}, + {"account_id": "456", "stream_state": {}}, + {"account_id": "789", "stream_state": {}}, + ] + assert list(incremental_class_instance.stream_slices()) == expected_slices - stream = SomeTestStream(api=api) - requests = [FacebookRequest("node", "GET", "endpoint") for _ in range(5)] + def test_stream_slices_single_account_with_state(self, incremental_class_instance): + incremental_class_instance._account_ids = ["123"] + stream_state = {"state_key": "state_value"} + expected_slices = [{"account_id": "123", "stream_state": stream_state}] + assert list(incremental_class_instance.stream_slices(stream_state)) == expected_slices - with pytest.raises(RuntimeError, match="Batch request failed with response:"): - list(stream.execute_in_batch(requests)) + def test_stream_slices_single_account_empty_state(self, incremental_class_instance): + incremental_class_instance._account_ids = ["123"] + expected_slices = [{"account_id": "123", "stream_state": None}] + assert list(incremental_class_instance.stream_slices()) == expected_slices - assert batch.add_request.call_count == len(requests) - assert batch.execute.call_count == 1 + def test_get_updated_state(self, incremental_class_instance): + current_stream_state = {"123": {"date": "2021-01-15T00:00:00+00:00"}, "include_deleted": False} + latest_record = {"account_id": "123", "date": "2021-01-20T00:00:00+00:00"} - def test_batch_reduce_amount(self, api, batch, mock_batch_responses, caplog): - """Reduce batch size to 1 and finally fail with message""" + expected_state = {"123": {"date": "2021-01-20T00:00:00+00:00", "include_deleted": False}, "include_deleted": False} - retryable_message = "Please reduce the amount of data you're asking for, then retry your request" - mock_batch_responses( - [ - { - "json": [ - {"body": {"error": {"message": retryable_message}}, "code": 500, "headers": {}}, - ], - } - ] - ) - - stream = SomeTestStream(api=api) - requests = [FacebookRequest("node", "GET", "endpoint")] - with pytest.raises(RuntimeError, match="Batch request failed with only 1 request in..."): - list(stream.execute_in_batch(requests)) - - assert batch.add_request.call_count == 7 - assert batch.execute.call_count == 7 - assert stream.max_batch_size == 1 - for index, expected_batch_size in enumerate(["25", "13", "7", "4", "2", "1"]): - assert expected_batch_size in caplog.messages[index] - - def test_execute_in_batch_retry_batch_error(self, api, batch, mock_batch_responses): - """Should retry without exception when any request returns 960 error code""" - mock_batch_responses( - [ - { - "json": [ - {"body": json.dumps({"name": "creative 1"}), "code": 200, "headers": {}}, - { - "body": json.dumps( - { - "error": { - "message": "Request aborted. This could happen if a dependent request failed or the entire request timed out.", - "type": "FacebookApiException", - "code": 960, - "fbtrace_id": "AWuyQlmgct0a_n64b-D1AFQ", - } - } - ), - "code": 500, - "headers": {}, - }, - {"body": json.dumps({"name": "creative 3"}), "code": 200, "headers": {}}, - ], - }, - { - "json": [ - {"body": json.dumps({"name": "creative 2"}), "code": 200, "headers": {}}, - ], - }, - ] - ) - - stream = SomeTestStream(api=api) - requests = [FacebookRequest("node", "GET", "endpoint") for _ in range(3)] - result = list(stream.execute_in_batch(requests)) - - assert batch.add_request.call_count == len(requests) + 1 - assert batch.execute.call_count == 2 - assert len(result) == len(requests) + new_state = incremental_class_instance.get_updated_state(current_stream_state, latest_record) + assert new_state == expected_state diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py index 310546d5ec19..0d862aab6f31 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py @@ -7,6 +7,7 @@ import pendulum import pytest from airbyte_cdk.models import SyncMode +from airbyte_cdk.utils import AirbyteTracedException from facebook_business import FacebookAdsApi, FacebookSession from facebook_business.exceptions import FacebookRequestError from source_facebook_marketing.streams import Activities, AdAccount, AdCreatives, Campaigns, Videos @@ -51,10 +52,9 @@ def fb_call_amount_data_response_fixture(): class TestBackoff: - def test_limit_reached(self, mocker, requests_mock, api, fb_call_rate_response, account_id): + def test_limit_reached(self, mocker, requests_mock, api, fb_call_rate_response, account_id, some_config): """Error once, check that we retry and not fail""" # turn Campaigns into non batch mode to test non batch logic - mocker.patch.object(Campaigns, "use_batch", new_callable=mocker.PropertyMock, return_value=False) campaign_responses = [ fb_call_rate_response, { @@ -67,9 +67,9 @@ def test_limit_reached(self, mocker, requests_mock, api, fb_call_rate_response, requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/1/", [{"status_code": 200}]) requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/2/", [{"status_code": 200}]) - stream = Campaigns(api=api, start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False) + stream = Campaigns(api=api, account_ids=[account_id], start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False) try: - records = list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + records = list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={}, stream_slice={"account_id": account_id})) assert records except FacebookRequestError: pytest.fail("Call rate error has not being handled") @@ -111,10 +111,13 @@ def test_batch_limit_reached(self, requests_mock, api, fb_call_rate_response, ac requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/", responses) requests_mock.register_uri("POST", FacebookSession.GRAPH + f"/{FB_API_VERSION}/", batch_responses) - stream = AdCreatives(api=api, include_deleted=False) - records = list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + stream = AdCreatives(api=api, account_ids=[account_id], include_deleted=False) + records = list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={}, stream_slice={"account_id": account_id})) - assert records == [{"name": "creative 1"}, {"name": "creative 2"}] + assert records == [ + {"account_id": "unknown_account", "id": "123", "object_type": "SHARE", "status": "ACTIVE"}, + {"account_id": "unknown_account", "id": "1234", "object_type": "SHARE", "status": "ACTIVE"}, + ] @pytest.mark.parametrize( "error_response", @@ -127,7 +130,7 @@ def test_batch_limit_reached(self, requests_mock, api, fb_call_rate_response, ac ) def test_common_error_retry(self, error_response, requests_mock, api, account_id): """Error once, check that we retry and not fail""" - account_data = {"id": 1, "updated_time": "2020-09-25T00:00:00Z", "name": "Some name"} + account_data = {"account_id": "unknown_account", "id": 1, "updated_time": "2020-09-25T00:00:00Z", "name": "Some name"} responses = [ error_response, { @@ -140,8 +143,8 @@ def test_common_error_retry(self, error_response, requests_mock, api, account_id requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/", responses) requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{account_data['id']}/", responses) - stream = AdAccount(api=api) - accounts = list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + stream = AdAccount(api=api, account_ids=[account_id]) + accounts = list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={}, stream_slice={"account_id": account_id})) assert accounts == [account_data] @@ -152,10 +155,12 @@ def test_limit_error_retry(self, fb_call_amount_data_response, requests_mock, ap "GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/campaigns", [fb_call_amount_data_response] ) - stream = Campaigns(api=api, start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False, page_size=100) + stream = Campaigns( + api=api, account_ids=[account_id], start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False, page_size=100 + ) try: - list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) - except FacebookRequestError: + list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={}, stream_slice={"account_id": account_id})) + except AirbyteTracedException: assert [x.qs.get("limit")[0] for x in res.request_history] == ["100", "50", "25", "12", "6"] def test_limit_error_retry_revert_page_size(self, requests_mock, api, account_id): @@ -172,13 +177,13 @@ def test_limit_error_retry_revert_page_size(self, requests_mock, api, account_id } success = { "json": { - 'data': [], + "data": [], "paging": { "cursors": { "after": "test", }, - "next": f"https://graph.facebook.com/{FB_API_VERSION}/act_{account_id}/activities?limit=31&after=test" - } + "next": f"https://graph.facebook.com/{FB_API_VERSION}/act_{account_id}/activities?limit=31&after=test", + }, }, "status_code": 200, } @@ -189,32 +194,60 @@ def test_limit_error_retry_revert_page_size(self, requests_mock, api, account_id [error, success, error, success], ) - stream = Activities(api=api, start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False, page_size=100) + stream = Activities( + api=api, account_ids=[account_id], start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False, page_size=100 + ) try: - list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) + list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={}, stream_slice={"account_id": account_id})) except FacebookRequestError: - assert [x.qs.get("limit")[0] for x in res.request_history] == ['100', '50', '100', '50'] + assert [x.qs.get("limit")[0] for x in res.request_history] == ["100", "50", "100", "50"] + + def test_start_date_not_provided(self, requests_mock, api, account_id): + success = { + "json": { + "data": [], + "paging": { + "cursors": { + "after": "test", + }, + "next": f"https://graph.facebook.com/{FB_API_VERSION}/act_{account_id}/activities?limit=31&after=test", + }, + }, + "status_code": 200, + } + + requests_mock.register_uri( + "GET", + FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/activities", + [success], + ) + + stream = Activities(api=api, account_ids=[account_id], start_date=None, end_date=None, include_deleted=False, page_size=100) + list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={}, stream_slice={"account_id": account_id})) def test_limit_error_retry_next_page(self, fb_call_amount_data_response, requests_mock, api, account_id): """Unlike the previous test, this one tests the API call fail on the second or more page of a request.""" base_url = FacebookSession.GRAPH + f"/{FB_API_VERSION}/act_{account_id}/advideos" res = requests_mock.register_uri( - "GET", base_url, + "GET", + base_url, [ { "json": { "data": [{"id": 1, "updated_time": "2020-09-25T00:00:00Z"}, {"id": 2, "updated_time": "2020-09-25T00:00:00Z"}], - "paging": {"next": f"{base_url}?after=after_page_1&limit=100"} + "paging": {"next": f"{base_url}?after=after_page_1&limit=100"}, }, - "status_code": 200 + "status_code": 200, }, - fb_call_amount_data_response - ] + fb_call_amount_data_response, + ], ) - stream = Videos(api=api, start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False, page_size=100) + stream = Videos( + api=api, account_ids=[account_id], start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False, page_size=100 + ) try: - list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={})) - except FacebookRequestError: + list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={}, stream_slice={"account_id": account_id})) + except AirbyteTracedException: assert [x.qs.get("limit")[0] for x in res.request_history] == ["100", "100", "50", "25", "12", "6"] diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py new file mode 100644 index 000000000000..092b855396c1 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_config_migrations.py @@ -0,0 +1,87 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import json +from typing import Any, Mapping + +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source +from source_facebook_marketing.config_migrations import MigrateAccountIdToArray +from source_facebook_marketing.source import SourceFacebookMarketing + +# BASE ARGS +CMD = "check" +TEST_CONFIG_PATH = "unit_tests/test_migrations/test_old_config.json" +NEW_TEST_CONFIG_PATH = "unit_tests/test_migrations/test_new_config.json" +UPGRADED_TEST_CONFIG_PATH = "unit_tests/test_migrations/test_upgraded_config.json" +SOURCE_INPUT_ARGS = [CMD, "--config", TEST_CONFIG_PATH] +SOURCE: Source = SourceFacebookMarketing() + + +# HELPERS +def load_config(config_path: str = TEST_CONFIG_PATH) -> Mapping[str, Any]: + with open(config_path, "r") as config: + return json.load(config) + + +def revert_migration(config_path: str = TEST_CONFIG_PATH) -> None: + with open(config_path, "r") as test_config: + config = json.load(test_config) + config.pop("account_ids") + with open(config_path, "w") as updated_config: + config = json.dumps(config) + updated_config.write(config) + + +def test_migrate_config(): + migration_instance = MigrateAccountIdToArray() + original_config = load_config() + # migrate the test_config + migration_instance.migrate(SOURCE_INPUT_ARGS, SOURCE) + # load the updated config + test_migrated_config = load_config() + # check migrated property + assert "account_ids" in test_migrated_config + assert isinstance(test_migrated_config["account_ids"], list) + # check the old property is in place + assert "account_id" in test_migrated_config + assert isinstance(test_migrated_config["account_id"], str) + # check the migration should be skipped, once already done + assert not migration_instance.should_migrate(test_migrated_config) + # load the old custom reports VS migrated + assert [original_config["account_id"]] == test_migrated_config["account_ids"] + # test CONTROL MESSAGE was emitted + control_msg = migration_instance.message_repository._message_queue[0] + assert control_msg.type == Type.CONTROL + assert control_msg.control.type == OrchestratorType.CONNECTOR_CONFIG + # old custom_reports are stil type(str) + assert isinstance(control_msg.control.connectorConfig.config["account_id"], str) + # new custom_reports are type(list) + assert isinstance(control_msg.control.connectorConfig.config["account_ids"], list) + # check the migrated values + assert control_msg.control.connectorConfig.config["account_ids"] == ["01234567890"] + # revert the test_config to the starting point + revert_migration() + + +def test_config_is_reverted(): + # check the test_config state, it has to be the same as before tests + test_config = load_config() + # check the config no longer has the migarted property + assert "account_ids" not in test_config + # check the old property is still there + assert "account_id" in test_config + assert isinstance(test_config["account_id"], str) + + +def test_should_not_migrate_new_config(): + new_config = load_config(NEW_TEST_CONFIG_PATH) + migration_instance = MigrateAccountIdToArray() + assert not migration_instance.should_migrate(new_config) + +def test_should_not_migrate_upgraded_config(): + new_config = load_config(UPGRADED_TEST_CONFIG_PATH) + migration_instance = MigrateAccountIdToArray() + assert not migration_instance.should_migrate(new_config) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py new file mode 100644 index 000000000000..105306b25f55 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py @@ -0,0 +1,434 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +from datetime import datetime + +import pytest +from airbyte_cdk.models import FailureType, SyncMode +from airbyte_cdk.utils.traced_exception import AirbyteTracedException +from facebook_business import FacebookAdsApi, FacebookSession +from source_facebook_marketing.api import API +from source_facebook_marketing.streams import AdAccount, AdCreatives, AdsInsights + +FB_API_VERSION = FacebookAdsApi.API_VERSION + +account_id = "unknown_account" +some_config = {"start_date": "2021-01-23T00:00:00Z", "account_ids": [account_id], "access_token": "unknown_token"} +base_url = f"{FacebookSession.GRAPH}/{FB_API_VERSION}/" +act_url = f"{base_url}act_{account_id}/" + +ad_account_response = { + "json": { + "data": [{"account_id": account_id, "id": f"act_{account_id}"}], + "status_code": 200, + } +} +ad_creative_data = [ + {"account_id": account_id, "id": "111111", "name": "ad creative 1", "updated_time": "2023-03-21T22:33:56-0700"}, + {"account_id": account_id, "id": "222222", "name": "ad creative 2", "updated_time": "2023-03-22T22:33:56-0700"}, +] +ad_creative_response = { + "json": { + "data": ad_creative_data, + "status_code": 200, + } +} + +# "name, friendly_msg, config_error_response", +CONFIG_ERRORS = [ + ( + "error_400_validating_access_token_session_expired", + "Invalid access token. Re-authenticate if FB oauth is used or refresh access token with all required permissions", + { + "status_code": 400, + "json": { + "error": { + "message": "Error validating access token: Session has expired on Friday, 18-Aug", + "type": "OAuthException", + "code": 190, + "error_subcode": 463, + } + }, + }, + ), + ( + "error_400_validating_access_token_user_changed_their_password", + "Invalid access token. Re-authenticate if FB oauth is used or refresh access token with all required permissions", + { + "status_code": 400, + "json": { + "error": { + "message": "Error validating access token: The session has been invalidated because the user changed their password or Facebook has changed the session for security reasons", + "type": "OAuthException", + "code": 190, + "error_subcode": 460, + } + }, + }, + ), + ( + "error_400_validating_access_token_not_authorized_application", + "Invalid access token. Re-authenticate if FB oauth is used or refresh access token with all required permissions", + { + "status_code": 400, + "json": { + "error": { + "message": "Error validating access token: The user has not authorized application 2586347315015828.", + "type": "OAuthException", + "code": 190, + "error_subcode": 458, + "fbtrace_id": "A3pz5DCfhBg3mGCS6Z9z9zY", + } + }, + }, + ), + ( + "error_400_missing_permission", + "Credentials don't have enough permissions. Check if correct Ad Account Id is used (as in Ads Manager), re-authenticate if FB oauth is used or refresh access token with all required permissions", + { + "status_code": 400, + "json": { + "error": { + "message": "(#100) Missing permissions", + "type": "OAuthException", + "code": 100, + } + }, + } + # Error randomly happens for different connections. + # Can be reproduced on https://developers.facebook.com/tools/explorer/?method=GET&path=act_&version=v17.0 + # 1st reason: incorrect ad account id is used + # 2nd reason: access_token does not have permissions: + # remove all permissions + # re-generate access token + # Re-authenticate (for cloud) or refresh access token (for oss) and check if all required permissions are granted + ), + ( + # One of possible reasons why this error happen is an attempt to access `owner` field: + # GET /act_?fields=,owner,... + "error_403_requires_permission", + "Credentials don't have enough permissions. Re-authenticate if FB oauth is used or refresh access token with all required permissions.", + { + "status_code": 403, + "json": { + "error": { + "code": 200, + "message": "(#200) Requires business_management permission to manage the object", + } + }, + }, + ), + ( + "error_400_permission_must_be_granted", + "Credentials don't have enough permissions. Re-authenticate if FB oauth is used or refresh access token with all required permissions.", + { + "status_code": 400, + "json": { + "error": { + "message": "Any of the pages_read_engagement, pages_manage_metadata,\n pages_read_user_content, pages_manage_ads, pages_show_list or\n pages_messaging permission(s) must be granted before impersonating a\n user's page.", + "type": "OAuthException", + "code": 190, + } + }, + }, + ), + ( + "error_unsupported_get_request", + "Credentials don't have enough permissions. Re-authenticate if FB oauth is used or refresh access token with all required permissions.", + { + "json": { + "error": { + "message": "Unsupported get request. Object with ID 'xxx' does not exist, cannot be loaded due to missing permissions, or does not support this operation. Please read the Graph API documentation at https://developers.facebook.com/docs/graph-api", + "type": "GraphMethodException", + "code": 100, + "error_subcode": 33, + "fbtrace_id": "A7qVRrTcBm8Pt6iUvnBrxwf", + } + }, + "status_code": 400, + }, + ), + ( + "error_400_unknown_profile_is_no_linked", + "Current profile is not linked to delegate page. Check if correct business (not personal) Ad Account Id is used (as in Ads Manager), re-authenticate if FB oauth is used or refresh access token with all required permissions.", + { + "status_code": 400, + "json": { + "error": { + "message": "An unknown error occurred", + "type": "OAuthException", + "code": 1, + "error_subcode": 2853001, + "is_transient": False, + "error_user_title": "profile is not linked to delegate page", + "error_user_msg": "profile should always be linked to delegate page", + } + }, + } + # Error happens on Video stream: https://graph.facebook.com/v17.0/act_XXXXXXXXXXXXXXXX/advideos + # Recommendations says that the problem can be fixed by switching to Business Ad Account Id + ), + ( + "error_400_unknown_profile_is_no_linked_es", + "Current profile is not linked to delegate page. Check if correct business (not personal) Ad Account Id is used (as in Ads Manager), re-authenticate if FB oauth is used or refresh access token with all required permissions.", + { + "status_code": 400, + "json": { + "error": { + "message": "An unknown error occurred", + "type": "OAuthException", + "code": 1, + "error_subcode": 2853001, + "is_transient": False, + "error_user_title": "el perfil no est\u00e1 vinculado a la p\u00e1gina del delegado", + "error_user_msg": "el perfil deber\u00eda estar siempre vinculado a la p\u00e1gina del delegado", + } + }, + }, + ), + # ("error_400_unsupported request", + # "Re-authenticate because current credential missing permissions", + # { + # "status_code": 400, + # "json": { + # "error": { + # "message": "Unsupported request - method type: get", + # "type": "GraphMethodException", + # "code": 100, + # } + # } + # } + # # for 'ad_account' stream, endpoint: https://graph.facebook.com/v17.0/act_1231630184301950/, + # # further attempts failed as well + # # previous sync of 'activities' stream was successfull + # # It seems like random problem: + # # - https://stackoverflow.com/questions/71195844/unsupported-request-method-type-get + # # "Same issue, but it turned out to be caused by Facebook (confirmed by their employee). A few hours later, the Graph API returned to normal without any action taken." + # # - https://developers.facebook.com/community/threads/805349521160054/ + # # "following, I've bein getting this error too, since last week, randomly." + # + # + # # https://developers.facebook.com/community/threads/1232870724022634/ + # # I observed that if I remove preview_shareable_link field from the request, the code is working properly. + # # Update (Denys Davydov): same for me, but removing the `funding_source_details` field helps, so + # # we do remove it and do not raise errors; this is tested by a different unit test - see `test_adaccount_list_objects_retry`. + # + # ), +] + + +class TestRealErrors: + @pytest.mark.parametrize( + "name, retryable_error_response", + [ + ( + "error_400_too_many_calls", + { + "json": { + "error": { + "message": ( + "(#80000) There have been too many calls from this ad-account. Wait a bit and try again. " + "For more info, please refer to https://developers.facebook.com/docs/graph-api/overview/rate-limiting." + ), + "type": "OAuthException", + "code": 80000, + "error_subcode": 2446079, + "fbtrace_id": "this_is_fake_response", + }, + }, + "status_code": 400, + "headers": {"x-app-usage": json.dumps({"call_count": 28, "total_time": 25, "total_cputime": 25})}, + }, + ), + ( + "error_500_unknown", + { + "json": {"error": {"code": 1, "message": "An unknown error occurred", "error_subcode": 99}}, + "status_code": 500, + }, + ), + ( + "error_400_service_temporarily_unavailable", + { + "status_code": 400, + "json": { + "error": { + "message": "(#2) Service temporarily unavailable", + "type": "OAuthException", + "is_transient": True, + "code": 2, + "fbtrace_id": "AnUyGZoFqN2m50GHVpOQEqr", + } + }, + }, + ), + ( + "error_500_reduce_the_amount_of_data", + { + "status_code": 500, + "json": { + "error": { + "message": "Please reduce the amount of data you're asking for, then retry your request", + "code": 1, + } + }, + } + # It can be a temporal problem: + # Happened during 'ad_account' stream sync which always returns only 1 record. + # Potentially could be caused by some particular field (list of requested fields is constant). + # But since sync was successful on next attempt, then conclusion is that this is a temporal problem. + ), + ], + ) + def test_retryable_error(self, some_config, requests_mock, name, retryable_error_response): + """Error once, check that we retry and not fail""" + requests_mock.reset_mock() + requests_mock.register_uri("GET", f"{act_url}", [retryable_error_response, ad_account_response]) + requests_mock.register_uri("GET", f"{act_url}adcreatives", [retryable_error_response, ad_creative_response]) + + api = API(access_token=some_config["access_token"], page_size=100) + stream = AdCreatives(api=api, account_ids=some_config["account_ids"], include_deleted=False) + ad_creative_records = list( + stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={}, stream_slice={"account_id": account_id}) + ) + + assert ad_creative_records == ad_creative_data + + # requests_mock.register_uri("GET", f"{self.act_url}advideos", [error_400_service_temporarily_unavailable, ad_creative_response]) + # stream = Videos(api=api, start_date=pendulum.now(), end_date=pendulum.now(), include_deleted=False, page_size=100) + + @pytest.mark.parametrize("name, friendly_msg, config_error_response", CONFIG_ERRORS) + def test_config_error_during_account_info_read(self, requests_mock, name, friendly_msg, config_error_response): + """Error raised during account info read""" + + api = API(access_token=some_config["access_token"], page_size=100) + stream = AdCreatives(api=api, account_ids=some_config["account_ids"], include_deleted=False) + + requests_mock.register_uri("GET", f"{act_url}", [config_error_response, ad_account_response]) + try: + list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={}, stream_slice={"account_id": account_id})) + assert False + except Exception as error: + assert isinstance(error, AirbyteTracedException) + assert error.failure_type == FailureType.config_error + assert friendly_msg in error.message + + # @pytest.mark.parametrize("name, friendly_msg, config_error_response", [CONFIG_ERRORS[-1]]) + @pytest.mark.parametrize("name, friendly_msg, config_error_response", CONFIG_ERRORS) + def test_config_error_during_actual_nodes_read(self, requests_mock, name, friendly_msg, config_error_response): + """Error raised during actual nodes read""" + + api = API(access_token=some_config["access_token"], page_size=100) + stream = AdCreatives(api=api, account_ids=some_config["account_ids"], include_deleted=False) + + requests_mock.register_uri("GET", f"{act_url}", [ad_account_response]) + requests_mock.register_uri("GET", f"{act_url}adcreatives", [config_error_response, ad_creative_response]) + try: + list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_state={}, stream_slice={"account_id": account_id})) + assert False + except Exception as error: + assert isinstance(error, AirbyteTracedException) + assert error.failure_type == FailureType.config_error + assert friendly_msg in error.message + + @pytest.mark.parametrize("name, friendly_msg, config_error_response", CONFIG_ERRORS) + def test_config_error_insights_account_info_read(self, requests_mock, name, friendly_msg, config_error_response): + """Error raised during actual nodes read""" + + api = API(access_token=some_config["access_token"], page_size=100) + stream = AdsInsights( + api=api, + account_ids=some_config["account_ids"], + start_date=datetime(2010, 1, 1), + end_date=datetime(2011, 1, 1), + fields=["account_id", "account_currency"], + insights_lookback_window=28, + ) + requests_mock.register_uri("GET", f"{act_url}", [config_error_response, ad_account_response]) + try: + slice = list(stream.stream_slices(sync_mode=SyncMode.full_refresh, stream_state={}))[0] + list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice, stream_state={})) + assert False + except Exception as error: + assert isinstance(error, AirbyteTracedException) + assert error.failure_type == FailureType.config_error + assert friendly_msg in error.message + + @pytest.mark.parametrize("name, friendly_msg, config_error_response", [CONFIG_ERRORS[0]]) + def test_config_error_insights_during_actual_nodes_read(self, requests_mock, name, friendly_msg, config_error_response): + """Error raised during actual nodes read""" + + api = API(access_token=some_config["access_token"], page_size=100) + stream = AdsInsights( + api=api, + account_ids=some_config["account_ids"], + start_date=datetime(2010, 1, 1), + end_date=datetime(2011, 1, 1), + fields=["account_id", "account_currency"], + insights_lookback_window=28, + ) + requests_mock.register_uri("GET", f"{act_url}", [ad_account_response]) + requests_mock.register_uri("GET", f"{act_url}insights", [config_error_response, ad_creative_response]) + + try: + slice = list(stream.stream_slices(sync_mode=SyncMode.full_refresh, stream_state={}))[0] + list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice, stream_state={})) + assert False + except Exception as error: + assert isinstance(error, AirbyteTracedException) + assert error.failure_type == FailureType.config_error + assert friendly_msg in error.message + + @pytest.mark.parametrize( + "failure_response", + ( + { + "status_code": 403, + "json": { + "message": "(#200) Requires business_management permission to manage the object", + "type": "OAuthException", + "code": 200, + "fbtrace_id": "AOm48i-YaiRlzqnNEnECcW8", + }, + }, + { + "status_code": 400, + "json": { + "message": "Unsupported request - method type: get", + "type": "GraphMethodException", + "code": 100, + "fbtrace_id": "AOm48i-YaiRlzqnNEnECcW8", + }, + }, + ), + ) + def test_adaccount_list_objects_retry(self, requests_mock, failure_response): + """ + Sometimes we get an error: "Requires business_management permission to manage the object" when account has all the required permissions: + [ + 'ads_management', + 'ads_read', + 'business_management', + 'public_profile' + ] + As a workaround for this case we can retry the API call excluding `owner` from `?fields=` GET query param. + """ + api = API(access_token=some_config["access_token"], page_size=100) + stream = AdAccount( + api=api, + account_ids=some_config["account_ids"], + ) + + business_user = {"account_id": account_id, "business": {"id": "1", "name": "TEST"}} + requests_mock.register_uri("GET", f"{base_url}me/business_users", status_code=200, json=business_user) + + assigend_users = {"account_id": account_id, "tasks": ["TASK"]} + requests_mock.register_uri("GET", f"{act_url}assigned_users", status_code=200, json=assigend_users) + + success_response = {"status_code": 200, "json": {"account_id": account_id}} + requests_mock.register_uri("GET", f"{act_url}", [failure_response, success_response]) + + record_gen = stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"account_id": account_id}, stream_state={}) + assert list(record_gen) == [{"account_id": "unknown_account", "id": "act_unknown_account"}] diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/test_new_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/test_new_config.json new file mode 100644 index 000000000000..489ff3fd68fb --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/test_new_config.json @@ -0,0 +1,14 @@ +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "account_ids": ["01234567890"], + "access_token": "access_token" +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/test_old_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/test_old_config.json new file mode 100644 index 000000000000..a04560eb7710 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/test_old_config.json @@ -0,0 +1,14 @@ +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "account_id": "01234567890", + "access_token": "access_token" +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/test_upgraded_config.json b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/test_upgraded_config.json new file mode 100644 index 000000000000..648b4e2c390b --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_migrations/test_upgraded_config.json @@ -0,0 +1,15 @@ +{ + "start_date": "2021-02-08T00:00:00Z", + "end_date": "2021-02-15T00:00:00Z", + "custom_insights": [ + { + "name": "custom_insight_stream", + "fields": ["account_name", "clicks", "cpc", "account_id", "ad_id"], + "breakdowns": ["gender"], + "action_breakdowns": [] + } + ], + "account_id": "01234567890", + "account_ids": ["01234567890"], + "access_token": "access_token" +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py index 0f614bb6ef02..7b96c5ced3b9 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py @@ -4,13 +4,22 @@ from copy import deepcopy +from unittest.mock import call import pytest -from airbyte_cdk.models import AirbyteConnectionStatus, ConnectorSpecification, Status +from airbyte_cdk.models import ( + AirbyteConnectionStatus, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + ConnectorSpecification, + DestinationSyncMode, + Status, + SyncMode, +) from facebook_business import FacebookAdsApi, FacebookSession from source_facebook_marketing import SourceFacebookMarketing from source_facebook_marketing.spec import ConnectorConfig -from source_facebook_marketing.streams.common import AccountTypeException from .utils import command_check @@ -18,12 +27,13 @@ @pytest.fixture(name="config") def config_fixture(requests_mock): config = { - "account_id": "123", + "account_ids": ["123"], "access_token": "TOKEN", "start_date": "2019-10-10T00:00:00Z", "end_date": "2020-10-10T00:00:00Z", } - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FacebookAdsApi.API_VERSION}/act_123/", {}) + requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FacebookAdsApi.API_VERSION}/me/business_users", json={"data": []}) + requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FacebookAdsApi.API_VERSION}/act_123/", json={"account": 123}) return config @@ -41,7 +51,14 @@ def inner(**kwargs): @pytest.fixture(name="api") def api_fixture(mocker): api_mock = mocker.patch("source_facebook_marketing.source.API") - api_mock.return_value = mocker.Mock(account=123) + api_mock.return_value = mocker.Mock(account=mocker.Mock(return_value=123)) + return api_mock + + +@pytest.fixture(name="api_find_account") +def api_fixture_find_account(mocker): + api_mock = mocker.patch("source_facebook_marketing.source.API._find_account") + api_mock.return_value = "1234" return api_mock @@ -62,6 +79,20 @@ def test_check_connection_ok(self, config, logger_mock, fb_marketing): assert ok assert not error_msg + def test_check_connection_find_account_was_called(self, api_find_account, config, logger_mock, fb_marketing): + """Check if _find_account was called to validate credentials""" + ok, error_msg = fb_marketing.check_connection(logger_mock, config=config) + + api_find_account.assert_called_once_with(config["account_ids"][0]) + logger_mock.info.assert_has_calls( + [ + call("Attempting to retrieve information for account with ID: 123"), + call("Successfully retrieved account information for account: 1234"), + ] + ) + assert ok + assert not error_msg + def test_check_connection_future_date_range(self, api, config, logger_mock, fb_marketing): config["start_date"] = "2219-10-10T00:00:00" config["end_date"] = "2219-10-11T00:00:00" @@ -75,7 +106,7 @@ def test_check_connection_end_date_before_start_date(self, api, config, logger_m config["end_date"] = "2019-10-09T00:00:00" assert fb_marketing.check_connection(logger_mock, config=config) == ( False, - "end_date must be equal or after start_date.", + "End date must be equal or after start date.", ) def test_check_connection_empty_config(self, api, logger_mock, fb_marketing): @@ -85,18 +116,20 @@ def test_check_connection_empty_config(self, api, logger_mock, fb_marketing): assert not ok assert error_msg - def test_check_connection_invalid_config(self, api, config, logger_mock, fb_marketing): + def test_check_connection_config_no_start_date(self, api, config, logger_mock, fb_marketing): config.pop("start_date") ok, error_msg = fb_marketing.check_connection(logger_mock, config=config) - assert not ok - assert error_msg + assert ok + assert not error_msg def test_check_connection_exception(self, api, config, logger_mock, fb_marketing): api.side_effect = RuntimeError("Something went wrong!") - with pytest.raises(RuntimeError, match="Something went wrong!"): - fb_marketing.check_connection(logger_mock, config=config) + ok, error_msg = fb_marketing.check_connection(logger_mock, config=config) + + assert not ok + assert error_msg == "Unexpected error: RuntimeError('Something went wrong!')" def test_streams(self, config, api, fb_marketing): streams = fb_marketing.streams(config) @@ -132,6 +165,26 @@ def test_get_custom_insights_action_breakdowns_allow_empty(self, api, config, fb assert streams[0].breakdowns == ["ad_format_asset"] assert streams[0].action_breakdowns == [] + def test_read_missing_stream(self, config, api, logger_mock, fb_marketing): + catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream( + name="fake_stream", + json_schema={}, + supported_sync_modes=[SyncMode.full_refresh], + ), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + ] + ) + + try: + list(fb_marketing.read(logger_mock, config=config, catalog=catalog)) + except KeyError as error: + pytest.fail(str(error)) + def test_check_config(config_gen, requests_mock, fb_marketing): requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FacebookAdsApi.API_VERSION}/act_123/", {}) @@ -144,19 +197,8 @@ def test_check_config(config_gen, requests_mock, fb_marketing): status = command_check(fb_marketing, config_gen(end_date="2019-99-10T00:00:00Z")) assert status.status == Status.FAILED - with pytest.raises(Exception): - assert command_check(fb_marketing, config_gen(start_date=...)) + status = command_check(fb_marketing, config_gen(start_date=...)) + assert status.status == Status.SUCCEEDED assert command_check(fb_marketing, config_gen(end_date=...)) == AirbyteConnectionStatus(status=Status.SUCCEEDED, message=None) assert command_check(fb_marketing, config_gen(end_date="")) == AirbyteConnectionStatus(status=Status.SUCCEEDED, message=None) - - -def test_check_connection_account_type_exception(mocker, fb_marketing, config, logger_mock): - api_mock = mocker.Mock() - api_mock.account.api_get.return_value = {"account": 123, "is_personal": 1} - mocker.patch('source_facebook_marketing.source.API', return_value=api_mock) - - result, error = fb_marketing.check_connection(logger=logger_mock, config=config) - - assert not result - assert isinstance(error, AccountTypeException) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py index 13638dbfef97..a2b03c52e67c 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_streams.py @@ -19,7 +19,7 @@ from source_facebook_marketing.streams.streams import fetch_thumbnail_data_url -def test_filter_all_statuses(api, mocker): +def test_filter_all_statuses(api, mocker, some_config): mocker.patch.multiple(FBMarketingStream, __abstractmethods__=set()) expected = { "filtering": [ @@ -45,16 +45,14 @@ def test_filter_all_statuses(api, mocker): } ] } - assert FBMarketingStream(api=api)._filter_all_statuses() == expected + assert FBMarketingStream(api=api, account_ids=some_config["account_ids"])._filter_all_statuses() == expected @pytest.mark.parametrize( - "url", ["https://graph.facebook.com", - "https://graph.facebook.com?test=123%23%24%25%2A&test2=456", "https://graph.facebook.com?"] + "url", ["https://graph.facebook.com", "https://graph.facebook.com?test=123%23%24%25%2A&test2=456", "https://graph.facebook.com?"] ) def test_fetch_thumbnail_data_url(url, requests_mock): - requests_mock.get(url, status_code=200, headers={ - "content-type": "content-type"}, content=b"") + requests_mock.get(url, status_code=200, headers={"content-type": "content-type"}, content=b"") assert fetch_thumbnail_data_url(url) == "data:content-type;base64," @@ -63,8 +61,7 @@ def test_parse_call_rate_header(): "x-business-use-case-usage": '{"test":[{"type":"ads_management","call_count":1,"total_cputime":1,' '"total_time":1,"estimated_time_to_regain_access":1}]}' } - assert MyFacebookAdsApi._parse_call_rate_header( - headers) == (1, duration(minutes=1)) + assert MyFacebookAdsApi._parse_call_rate_header(headers) == (1, duration(minutes=1)) @pytest.mark.parametrize( @@ -72,48 +69,57 @@ def test_parse_call_rate_header(): [ [AdsInsights, [], ["action_type", "action_target_id", "action_destination"]], [AdsInsightsActionType, [], ["action_type"]], - [AdsInsightsAgeAndGender, ["age", "gender"], [ - "action_type", "action_target_id", "action_destination"]], - [AdsInsightsCountry, ["country"], ["action_type", - "action_target_id", "action_destination"]], - [AdsInsightsDma, ["dma"], ["action_type", - "action_target_id", "action_destination"]], - [AdsInsightsPlatformAndDevice, ["publisher_platform", - "platform_position", "impression_device"], ["action_type"]], - [AdsInsightsRegion, ["region"], ["action_type", - "action_target_id", "action_destination"]], + [AdsInsightsAgeAndGender, ["age", "gender"], ["action_type", "action_target_id", "action_destination"]], + [AdsInsightsCountry, ["country"], ["action_type", "action_target_id", "action_destination"]], + [AdsInsightsDma, ["dma"], ["action_type", "action_target_id", "action_destination"]], + [AdsInsightsPlatformAndDevice, ["publisher_platform", "platform_position", "impression_device"], ["action_type"]], + [AdsInsightsRegion, ["region"], ["action_type", "action_target_id", "action_destination"]], ], ) -def test_ads_insights_breakdowns(class_name, breakdowns, action_breakdowns): - kwargs = {"api": None, "start_date": pendulum.now( - ), "end_date": pendulum.now(), "insights_lookback_window": 1} +def test_ads_insights_breakdowns(class_name, breakdowns, action_breakdowns, some_config): + kwargs = { + "api": None, + "account_ids": some_config["account_ids"], + "start_date": pendulum.now(), + "end_date": pendulum.now(), + "insights_lookback_window": 1, + } stream = class_name(**kwargs) assert stream.breakdowns == breakdowns assert stream.action_breakdowns == action_breakdowns -def test_custom_ads_insights_breakdowns(): - kwargs = {"api": None, "start_date": pendulum.now( - ), "end_date": pendulum.now(), "insights_lookback_window": 1} - stream = AdsInsights(breakdowns=["mmm"], action_breakdowns=[ - "action_destination"], **kwargs) +def test_custom_ads_insights_breakdowns(some_config): + kwargs = { + "api": None, + "account_ids": some_config["account_ids"], + "start_date": pendulum.now(), + "end_date": pendulum.now(), + "insights_lookback_window": 1, + } + stream = AdsInsights(breakdowns=["mmm"], action_breakdowns=["action_destination"], **kwargs) assert stream.breakdowns == ["mmm"] assert stream.action_breakdowns == ["action_destination"] stream = AdsInsights(breakdowns=[], action_breakdowns=[], **kwargs) assert stream.breakdowns == [] - assert stream.action_breakdowns == [ - "action_type", "action_target_id", "action_destination"] + assert stream.action_breakdowns == ["action_type", "action_target_id", "action_destination"] - stream = AdsInsights(breakdowns=[], action_breakdowns=[], - action_breakdowns_allow_empty=True, **kwargs) + stream = AdsInsights(breakdowns=[], action_breakdowns=[], action_breakdowns_allow_empty=True, **kwargs) assert stream.breakdowns == [] assert stream.action_breakdowns == [] -def test_custom_ads_insights_action_report_times(): - kwargs = {"api": None, "start_date": pendulum.now(), "end_date": pendulum.now( - ), "insights_lookback_window": 1, "action_breakdowns": ["action_destination"], "breakdowns": []} +def test_custom_ads_insights_action_report_times(some_config): + kwargs = { + "api": None, + "account_ids": some_config["account_ids"], + "start_date": pendulum.now(), + "end_date": pendulum.now(), + "insights_lookback_window": 1, + "action_breakdowns": ["action_destination"], + "breakdowns": [], + } stream = AdsInsights(**kwargs) assert stream.action_report_time == "mixed" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_utils.py index d50f9f48f5bc..ccde2ee1fcba 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_utils.py @@ -23,10 +23,7 @@ "start_date", pendulum.local(2019, 1, 1), pendulum.local(2020, 3, 2), - [ - f"The start date cannot be beyond 37 months from the current date. " - f"Set start date to {pendulum.local(2020, 3, 2)}." - ] + [f"The start date cannot be beyond 37 months from the current date. " f"Set start date to {pendulum.local(2020, 3, 2)}."], ), ( "start_date", diff --git a/airbyte-integrations/connectors/source-facebook-pages/README.md b/airbyte-integrations/connectors/source-facebook-pages/README.md index 728fcd0e5d33..98a52a206125 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/README.md +++ b/airbyte-integrations/connectors/source-facebook-pages/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-facebook-pages:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/facebook-pages) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_facebook_pages/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-facebook-pages:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-facebook-pages build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-facebook-pages:airbyteDocker +An image will be built with the tag `airbyte/source-facebook-pages:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-facebook-pages:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-facebook-pages:dev che docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-facebook-pages:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-facebook-pages:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-facebook-pages test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-facebook-pages:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-facebook-pages:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-facebook-pages test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/facebook-pages.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-facebook-pages/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-facebook-pages/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-facebook-pages/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-facebook-pages/build.gradle b/airbyte-integrations/connectors/source-facebook-pages/build.gradle deleted file mode 100644 index 6ecc25e59b25..000000000000 --- a/airbyte-integrations/connectors/source-facebook-pages/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_facebook_pages' -} diff --git a/airbyte-integrations/connectors/source-facebook-pages/metadata.yaml b/airbyte-integrations/connectors/source-facebook-pages/metadata.yaml index 510bf9e10a33..d8dd267fcc57 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/metadata.yaml +++ b/airbyte-integrations/connectors/source-facebook-pages/metadata.yaml @@ -1,4 +1,7 @@ data: + ab_internal: + ql: 300 + sl: 100 allowedHosts: hosts: - graph.facebook.com @@ -7,22 +10,19 @@ data: definitionId: 010eb12f-837b-4685-892d-0a39f76a98f5 dockerImageTag: 0.3.0 dockerRepository: airbyte/source-facebook-pages + documentationUrl: https://docs.airbyte.com/integrations/sources/facebook-pages githubIssueLabel: source-facebook-pages icon: facebook.svg license: ELv2 name: Facebook Pages registries: cloud: - enabled: true + enabled: false # hide from cloud until https://github.com/airbytehq/airbyte/issues/25515 is finished oss: enabled: true releaseStage: beta - documentationUrl: https://docs.airbyte.com/integrations/sources/facebook-pages + supportLevel: community tags: - language:low-code - language:python - ab_internal: - sl: 200 - ql: 300 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-facebook-pages/unit_tests/test_custom_field_transformation.py b/airbyte-integrations/connectors/source-facebook-pages/unit_tests/test_custom_field_transformation.py index 267d44f08869..36fe896ca5d8 100644 --- a/airbyte-integrations/connectors/source-facebook-pages/unit_tests/test_custom_field_transformation.py +++ b/airbyte-integrations/connectors/source-facebook-pages/unit_tests/test_custom_field_transformation.py @@ -9,8 +9,9 @@ def test_field_transformation(): - with open(f"{os.path.dirname(__file__)}/initial_record.json", "r") as initial_record, \ - open(f"{os.path.dirname(__file__)}/transformed_record.json", "r") as transformed_record: + with open(f"{os.path.dirname(__file__)}/initial_record.json", "r") as initial_record, open( + f"{os.path.dirname(__file__)}/transformed_record.json", "r" + ) as transformed_record: initial_record = json.loads(initial_record.read()) transformed_record = json.loads(transformed_record.read()) record_transformation = CustomFieldTransformation(config={}, parameters={"name": "page"}) diff --git a/airbyte-integrations/connectors/source-facebook-pages/unit_tests/test_facebook_authenticator.py b/airbyte-integrations/connectors/source-facebook-pages/unit_tests/test_facebook_authenticator.py index 499ac8a05458..846dbf896550 100755 --- a/airbyte-integrations/connectors/source-facebook-pages/unit_tests/test_facebook_authenticator.py +++ b/airbyte-integrations/connectors/source-facebook-pages/unit_tests/test_facebook_authenticator.py @@ -15,17 +15,13 @@ def req_mock(): def test_facebook_url_params(req_mock): - config = { - "access_token": "initial_token", - "page_id": "pageID" - } + config = {"access_token": "initial_token", "page_id": "pageID"} parameters = config req_mock.get("https://graph.facebook.com/pageID", json={"access_token": "page_access_token"}) - authenticator = AuthenticatorFacebookPageAccessToken(config=config, - page_id=config.get("page_id"), - access_token=config.get("access_token"), - parameters=parameters) + authenticator = AuthenticatorFacebookPageAccessToken( + config=config, page_id=config.get("page_id"), access_token=config.get("access_token"), parameters=parameters + ) page_token = authenticator.generate_page_access_token() assert page_token == "page_access_token" prepared_request = requests.PreparedRequest() diff --git a/airbyte-integrations/connectors/source-faker/Dockerfile b/airbyte-integrations/connectors/source-faker/Dockerfile index d0648a0212e1..9db110142dbc 100644 --- a/airbyte-integrations/connectors/source-faker/Dockerfile +++ b/airbyte-integrations/connectors/source-faker/Dockerfile @@ -34,5 +34,5 @@ COPY source_faker ./source_faker ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=5.0.0 +LABEL io.airbyte.version=5.0.2 LABEL io.airbyte.name=airbyte/source-faker diff --git a/airbyte-integrations/connectors/source-faker/README.md b/airbyte-integrations/connectors/source-faker/README.md index 7864a4b45d53..b8415263d9a6 100644 --- a/airbyte-integrations/connectors/source-faker/README.md +++ b/airbyte-integrations/connectors/source-faker/README.md @@ -34,14 +34,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle - -From the Airbyte repository root, run: - -``` -./gradlew :airbyte-integrations:connectors:source-faker:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/faker) @@ -63,22 +55,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: - -``` -docker build . -t airbyte/source-faker:dev +#### Build +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-faker build ``` -You can also build the connector image via Gradle: +An image will be built with the tag `airbyte/source-faker:dev`. +**Via `docker build`:** +```bash +docker build -t airbyte/source-faker:dev . ``` -./gradlew :airbyte-integrations:connectors:source-faker:airbyteDocker -``` - -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run @@ -91,61 +80,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-faker:dev discover --c docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-faker:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing - -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: - -``` -pip install ".[tests]" -``` - -### Unit Tests - -To run unit tests locally, from the connector directory run: - -``` -python -m pytest unit_tests -``` - -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). - -#### Custom Integration tests - -Place custom tests inside `integration_tests/` folder, then, from the connector root, run - -``` -python -m pytest integration_tests +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-faker test ``` -#### Acceptance Tests - -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run - -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` - -To run your integration tests with docker - -### Using gradle to run tests - -All commands should be run from airbyte project root. -To run unit tests: - -``` -./gradlew :airbyte-integrations:connectors:source-faker:unitTest -``` - -To run acceptance and custom integration tests: - -``` -./gradlew :airbyte-integrations:connectors:source-faker:integrationTest -``` ## Dependency Management @@ -156,13 +100,12 @@ We split dependencies between two groups, dependencies that are: - required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector - You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-faker test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/faker.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - -The end diff --git a/airbyte-integrations/connectors/source-faker/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-faker/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-faker/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-faker/build.gradle b/airbyte-integrations/connectors/source-faker/build.gradle deleted file mode 100644 index 7a2559a214e3..000000000000 --- a/airbyte-integrations/connectors/source-faker/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_faker' -} diff --git a/airbyte-integrations/connectors/source-faker/main.py b/airbyte-integrations/connectors/source-faker/main.py index 782659c7a6fb..9df2974ae7bd 100644 --- a/airbyte-integrations/connectors/source-faker/main.py +++ b/airbyte-integrations/connectors/source-faker/main.py @@ -3,11 +3,7 @@ # -import sys - -from airbyte_cdk.entrypoint import launch -from source_faker import SourceFaker +from source_faker.run import run if __name__ == "__main__": - source = SourceFaker() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-faker/metadata.yaml b/airbyte-integrations/connectors/source-faker/metadata.yaml index 1d1a3bfffae2..e228708b816e 100644 --- a/airbyte-integrations/connectors/source-faker/metadata.yaml +++ b/airbyte-integrations/connectors/source-faker/metadata.yaml @@ -1,11 +1,15 @@ data: + ab_internal: + ql: 300 + sl: 100 allowedHosts: hosts: [] connectorSubtype: api connectorType: source definitionId: dfd88b22-b603-4c3d-aad7-3701784586b1 - dockerImageTag: 5.0.0 + dockerImageTag: 5.0.2 dockerRepository: airbyte/source-faker + documentationUrl: https://docs.airbyte.com/integrations/sources/faker githubIssueLabel: source-faker icon: faker.svg license: MIT @@ -16,6 +20,16 @@ data: oss: enabled: true releaseStage: beta + releases: + breakingChanges: + 4.0.0: + message: This is a breaking change message + upgradeDeadline: "2023-07-19" + 5.0.0: + message: + ID and products.year fields are changing to be integers instead of + floats. + upgradeDeadline: "2023-08-31" resourceRequirements: jobSpecific: - jobType: sync @@ -27,21 +41,11 @@ data: - users - products - purchases - documentationUrl: https://docs.airbyte.com/integrations/sources/faker + supportLevel: community + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-faker tags: - language:python - releases: - breakingChanges: - 5.0.0: - message: - "ID and products.year fields are changing to be integers instead - of floats." - upgradeDeadline: "2023-08-31" - 4.0.0: - message: "This is a breaking change message" - upgradeDeadline: "2023-07-19" - ab_internal: - sl: 200 - ql: 300 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-faker/setup.py b/airbyte-integrations/connectors/source-faker/setup.py index 1a16ba5ea485..ab39ea239037 100644 --- a/airbyte-integrations/connectors/source-faker/setup.py +++ b/airbyte-integrations/connectors/source-faker/setup.py @@ -24,4 +24,10 @@ extras_require={ "tests": TEST_REQUIREMENTS, }, + # register console entry points + entry_points={ + "console_scripts": [ + "source-faker=source_faker.run:run", + ], + }, ) diff --git a/airbyte-integrations/connectors/source-faker/source_faker/run.py b/airbyte-integrations/connectors/source-faker/source_faker/run.py new file mode 100644 index 000000000000..5bf64ce0d724 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/source_faker/run.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_faker import SourceFaker + + +def run(): + source = SourceFaker() + launch(source, sys.argv[1:]) + + +if __name__ == "__main__": + run() diff --git a/airbyte-integrations/connectors/source-faker/source_faker/streams.py b/airbyte-integrations/connectors/source-faker/source_faker/streams.py index ba7d70b7dd2c..002866ba7c54 100644 --- a/airbyte-integrations/connectors/source-faker/source_faker/streams.py +++ b/airbyte-integrations/connectors/source-faker/source_faker/streams.py @@ -119,9 +119,9 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: if records_remaining_this_loop == 0: break - self.state = {"seed": self.seed, "updated_at": updated_at} + self.state = {"seed": self.seed, "updated_at": updated_at, "loop_offset": loop_offset} - self.state = {"seed": self.seed, "updated_at": updated_at} + self.state = {"seed": self.seed, "updated_at": updated_at, "loop_offset": loop_offset} class Purchases(Stream, IncrementalMixin): @@ -180,6 +180,6 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: if records_remaining_this_loop == 0: break - self.state = {"seed": self.seed, "updated_at": updated_at} + self.state = {"seed": self.seed, "updated_at": updated_at, "loop_offset": loop_offset} - self.state = {"seed": self.seed, "updated_at": updated_at} + self.state = {"seed": self.seed, "updated_at": updated_at, "loop_offset": loop_offset} diff --git a/airbyte-integrations/connectors/source-fastbill/README.md b/airbyte-integrations/connectors/source-fastbill/README.md index a6cbffd7fe59..9fba400b29bd 100644 --- a/airbyte-integrations/connectors/source-fastbill/README.md +++ b/airbyte-integrations/connectors/source-fastbill/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-fastbill:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/fastbill) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_fastbill/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-fastbill:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-fastbill build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-fastbill:airbyteDocker +An image will be built with the tag `airbyte/source-fastbill:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-fastbill:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-fastbill:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-fastbill:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-fastbill:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-fastbill test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-fastbill:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-fastbill:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-fastbill test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/fastbill.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-fastbill/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-fastbill/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-fastbill/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-fastbill/build.gradle b/airbyte-integrations/connectors/source-fastbill/build.gradle deleted file mode 100644 index b0b71f4f7511..000000000000 --- a/airbyte-integrations/connectors/source-fastbill/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_fastbill' -} diff --git a/airbyte-integrations/connectors/source-fastbill/metadata.yaml b/airbyte-integrations/connectors/source-fastbill/metadata.yaml index 805d8dd61b3c..9a6e795f561d 100644 --- a/airbyte-integrations/connectors/source-fastbill/metadata.yaml +++ b/airbyte-integrations/connectors/source-fastbill/metadata.yaml @@ -21,5 +21,5 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/fastbill tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-fauna/README.md b/airbyte-integrations/connectors/source-fauna/README.md index 56353b6fc5d0..5086b5130879 100644 --- a/airbyte-integrations/connectors/source-fauna/README.md +++ b/airbyte-integrations/connectors/source-fauna/README.md @@ -88,12 +88,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-fauna:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/fauna) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_fauna/spec.yaml` file. @@ -113,18 +107,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-fauna:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-fauna build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-fauna:airbyteDocker +An image will be built with the tag `airbyte/source-fauna:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-fauna:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -134,44 +129,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-fauna:dev check --conf docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-fauna:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-fauna:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-fauna test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-fauna:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-fauna:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -181,8 +148,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-fauna test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/fauna.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-fauna/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-fauna/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-fauna/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-fauna/build.gradle b/airbyte-integrations/connectors/source-fauna/build.gradle deleted file mode 100644 index ae661a5c3654..000000000000 --- a/airbyte-integrations/connectors/source-fauna/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_fauna_singer' -} diff --git a/airbyte-integrations/connectors/source-fauna/unit_tests/database_test.py b/airbyte-integrations/connectors/source-fauna/unit_tests/database_test.py index 0b30733c5638..5137d192adaf 100644 --- a/airbyte-integrations/connectors/source-fauna/unit_tests/database_test.py +++ b/airbyte-integrations/connectors/source-fauna/unit_tests/database_test.py @@ -172,18 +172,21 @@ def setup_container(): def run_discover_test(source: SourceFauna, logger): # See `test_util.py` for these values - catalog = source.discover(logger, { - "secret": "secret", - "domain": "localhost", - "port": 9000, - "scheme": "http", - "collection": { - "page_size": 64, - "deletions": { - "deletion_mode": "ignore", - } - } - }) + catalog = source.discover( + logger, + { + "secret": "secret", + "domain": "localhost", + "port": 9000, + "scheme": "http", + "collection": { + "page_size": 64, + "deletions": { + "deletion_mode": "ignore", + }, + }, + }, + ) assert len(catalog.streams) == 1 stream = catalog.streams[0] assert stream.name == "foo" diff --git a/airbyte-integrations/connectors/source-fauna/unit_tests/incremental_test.py b/airbyte-integrations/connectors/source-fauna/unit_tests/incremental_test.py index afcc8d236ec9..9a955244a5d4 100644 --- a/airbyte-integrations/connectors/source-fauna/unit_tests/incremental_test.py +++ b/airbyte-integrations/connectors/source-fauna/unit_tests/incremental_test.py @@ -113,9 +113,7 @@ def read_removes_hardcoded( sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.append_dedup, stream=AirbyteStream( - name="my_stream_name", - json_schema={}, - supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh] + name="my_stream_name", json_schema={}, supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh] ), ) ] @@ -225,9 +223,7 @@ def read_removes_hardcoded( sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.append_dedup, stream=AirbyteStream( - name="my_stream_name", - json_schema={}, - supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh] + name="my_stream_name", json_schema={}, supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh] ), ) ] @@ -683,9 +679,7 @@ def query_hardcoded(expr): sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.append_dedup, stream=AirbyteStream( - name="my_stream_name", - json_schema={}, - supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh] + name="my_stream_name", json_schema={}, supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh] ), ), CollectionConfig(page_size=PAGE_SIZE), @@ -720,9 +714,7 @@ def query_hardcoded(expr): sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.append_dedup, stream=AirbyteStream( - name="my_stream_name", - json_schema={}, - supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh] + name="my_stream_name", json_schema={}, supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh] ), ), CollectionConfig(page_size=PAGE_SIZE), @@ -743,9 +735,7 @@ def query_hardcoded(expr): sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.append_dedup, stream=AirbyteStream( - name="my_stream_name", - json_schema={}, - supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh] + name="my_stream_name", json_schema={}, supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh] ), ), CollectionConfig(page_size=PAGE_SIZE), @@ -857,9 +847,7 @@ def query_hardcoded(expr): sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.append_dedup, stream=AirbyteStream( - name="my_stream_name", - json_schema={}, - supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh] + name="my_stream_name", json_schema={}, supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh] ), ), CollectionConfig(page_size=PAGE_SIZE), @@ -889,9 +877,7 @@ def query_hardcoded(expr): sync_mode=SyncMode.incremental, destination_sync_mode=DestinationSyncMode.append_dedup, stream=AirbyteStream( - name="my_stream_name", - json_schema={}, - supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh] + name="my_stream_name", json_schema={}, supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh] ), ), CollectionConfig(page_size=PAGE_SIZE), diff --git a/airbyte-integrations/connectors/source-file-secure/.dockerignore b/airbyte-integrations/connectors/source-file-secure/.dockerignore deleted file mode 100644 index 9ef96044faba..000000000000 --- a/airbyte-integrations/connectors/source-file-secure/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -build - diff --git a/airbyte-integrations/connectors/source-file-secure/Dockerfile b/airbyte-integrations/connectors/source-file-secure/Dockerfile deleted file mode 100644 index fa3b8daf6d7e..000000000000 --- a/airbyte-integrations/connectors/source-file-secure/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -### WARNING ### -# This Dockerfile will soon be deprecated. -# It is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L771 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/source-file:0.3.11 - -WORKDIR /airbyte/integration_code -COPY source_file_secure ./source_file_secure -COPY main.py ./ -COPY setup.py ./ -ENV DOCKER_BUILD=True -RUN pip install . - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.3.11 -LABEL io.airbyte.name=airbyte/source-file-secure diff --git a/airbyte-integrations/connectors/source-file-secure/README.md b/airbyte-integrations/connectors/source-file-secure/README.md deleted file mode 100644 index a46a3d4bb0a7..000000000000 --- a/airbyte-integrations/connectors/source-file-secure/README.md +++ /dev/null @@ -1,96 +0,0 @@ -# File Source Secure -This is the repository for the File source connector, written in Python. -This is modificaion of another connector Source File. This version has only one difference with the origin version is this one doesn't support local file storages and is orientated for cloud and cluster platforms. -More details about dependencies and requirement are available [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-file/README.md) - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-file-secure:build -``` - -#### Create credentials -Details are explained [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-file/README.md#create-credentials) - -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. - - - -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source file test creds` -and place them into `secrets/config.json`. - - -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` - -### Locally running the connector docker image - -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-file-secure:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` - -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-file-secure:airbyteDocker -``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. - -#### Run -Then run any of the connector commands as follows: -``` -docker run --rm airbyte/source-file-secure:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-file-secure:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-file-secure:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-file-secure:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json -``` - -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-file-secure:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s ../source-file/integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. - -## Dependency Management -All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. - -### Publishing a new version of the connector -You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master diff --git a/airbyte-integrations/connectors/source-file-secure/acceptance-test-config.yml b/airbyte-integrations/connectors/source-file-secure/acceptance-test-config.yml deleted file mode 100644 index 280e3619823b..000000000000 --- a/airbyte-integrations/connectors/source-file-secure/acceptance-test-config.yml +++ /dev/null @@ -1,27 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests - -# Here we tries to test a basic tests only. -# The main part of tests should be executed for the source-file connector -connector_image: airbyte/source-file-secure:dev -tests: - spec: - - spec_path: "integration_tests/spec.json" - connection: - - config_path: "integration_tests/invalid_config.json" - status: "failed" - # for https - - config_path: "integration_tests/config.json" - status: "succeed" - # for local should be failed - - config_path: "integration_tests/local_config.json" - status: "failed" - - discovery: - # for https - - config_path: "integration_tests/config.json" - - basic_read: - # for https - - config_path: "integration_tests/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-file-secure/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-file-secure/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-file-secure/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-file-secure/build.gradle b/airbyte-integrations/connectors/source-file-secure/build.gradle deleted file mode 100644 index 5b42a5a6b866..000000000000 --- a/airbyte-integrations/connectors/source-file-secure/build.gradle +++ /dev/null @@ -1,10 +0,0 @@ - -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_file_secure' -} diff --git a/airbyte-integrations/connectors/source-file-secure/icon.svg b/airbyte-integrations/connectors/source-file-secure/icon.svg deleted file mode 100644 index dfb63026b6d2..000000000000 --- a/airbyte-integrations/connectors/source-file-secure/icon.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-file-secure/integration_tests/config.json b/airbyte-integrations/connectors/source-file-secure/integration_tests/config.json deleted file mode 100644 index 2a659d71b1f7..000000000000 --- a/airbyte-integrations/connectors/source-file-secure/integration_tests/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "dataset_name": "test", - "format": "csv", - "reader_options": "{\"sep\": \",\", \"nrows\": 20}", - "url": "https://www.stats.govt.nz/assets/Uploads/Business-price-indexes/Business-price-indexes-September-2020-quarter/Download-data/business-price-indexes-september-2020-quarter-corrections-to-previously-published-statistics.csv", - "provider": { - "storage": "HTTPS", - "reader_impl": "gcsfs" - } -} diff --git a/airbyte-integrations/connectors/source-file-secure/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-file-secure/integration_tests/configured_catalog.json deleted file mode 100644 index d46f2b60bde3..000000000000 --- a/airbyte-integrations/connectors/source-file-secure/integration_tests/configured_catalog.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "test", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-file-secure/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-file-secure/integration_tests/invalid_config.json deleted file mode 100644 index fd1448b39352..000000000000 --- a/airbyte-integrations/connectors/source-file-secure/integration_tests/invalid_config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "dataset_name": "fake_name", - "format": "csv", - "reader_options": "{\"bla\": \",\", \"nrows\": 20}", - "url": "https://fake-fake.com", - "provider": { - "storage": "HTTPS", - "reader_impl": "fake" - } -} diff --git a/airbyte-integrations/connectors/source-file-secure/main.py b/airbyte-integrations/connectors/source-file-secure/main.py deleted file mode 100644 index c6eef7f6b135..000000000000 --- a/airbyte-integrations/connectors/source-file-secure/main.py +++ /dev/null @@ -1,12 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import sys - -from airbyte_cdk.entrypoint import launch -from source_file_secure import SourceFileSecure - -if __name__ == "__main__": - launch(SourceFileSecure(), sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-file-secure/metadata.yaml b/airbyte-integrations/connectors/source-file-secure/metadata.yaml deleted file mode 100644 index 9bccb5d2ef88..000000000000 --- a/airbyte-integrations/connectors/source-file-secure/metadata.yaml +++ /dev/null @@ -1,23 +0,0 @@ -data: - allowedHosts: - hosts: - - "*" - connectorSubtype: file - connectorType: source - definitionId: 778daa7c-feaf-4db6-96f3-70fd645acc77 - dockerImageTag: 0.3.11 - dockerRepository: airbyte/source-file-secure - githubIssueLabel: source-file - icon: file.svg - license: MIT - name: File (CSV, JSON, Excel, Feather, Parquet) - registries: - cloud: - enabled: false # strict encrypt connectors are deployed to Cloud by their non strict encrypt sibling. - oss: - enabled: false # strict encrypt connectors are not used on OSS. - releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/file - tags: - - language:python -metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-file-secure/setup.cfg b/airbyte-integrations/connectors/source-file-secure/setup.cfg deleted file mode 100644 index 672fb95c5976..000000000000 --- a/airbyte-integrations/connectors/source-file-secure/setup.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[aliases] -test='pytest' - -[tool:pytest] -testpaths = unit_tests diff --git a/airbyte-integrations/connectors/source-file-secure/setup.py b/airbyte-integrations/connectors/source-file-secure/setup.py deleted file mode 100644 index f2fb71c1899a..000000000000 --- a/airbyte-integrations/connectors/source-file-secure/setup.py +++ /dev/null @@ -1,53 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import os -from pathlib import Path - -from setuptools import find_packages, setup - - -def local_dependency(name: str) -> str: - """Returns a path to a local package.""" - if os.environ.get("DAGGER_BUILD"): - return f"{name} @ file:///local_dependencies/{name}" - else: - return f"{name} @ file://{Path.cwd().parent / name}" - - -MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", - "gcsfs==2022.7.1", - "genson==1.2.2", - "google-cloud-storage==2.5.0", - "pandas==1.4.3", - "paramiko==2.11.0", - "s3fs==2022.7.1", - "smart-open[all]==6.0.0", - "lxml==4.9.1", - "html5lib==1.1", - "beautifulsoup4==4.11.1", - "pyarrow==9.0.0", - "xlrd==2.0.1", - "openpyxl==3.0.10", - "pyxlsb==1.0.9", -] - -if not os.environ.get("DOCKER_BUILD"): - MAIN_REQUIREMENTS.append(local_dependency("source-file")) - -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "boto3==1.21.21", "pytest==7.1.2", "pytest-docker==1.0.0", "pytest-mock~=3.8.2"] - -setup( - name="source_file_secure", - description="Source implementation for File", - author="Airbyte", - author_email="contact@airbyte.io", - packages=find_packages(), - install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json"]}, - extras_require={ - "tests": TEST_REQUIREMENTS, - }, -) diff --git a/airbyte-integrations/connectors/source-file-secure/source_file_secure/__init__.py b/airbyte-integrations/connectors/source-file-secure/source_file_secure/__init__.py deleted file mode 100644 index 2107707a1cb1..000000000000 --- a/airbyte-integrations/connectors/source-file-secure/source_file_secure/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .source import SourceFileSecure - -__all__ = ["SourceFileSecure"] diff --git a/airbyte-integrations/connectors/source-file-secure/source_file_secure/source.py b/airbyte-integrations/connectors/source-file-secure/source_file_secure/source.py deleted file mode 100644 index efc59cfbdceb..000000000000 --- a/airbyte-integrations/connectors/source-file-secure/source_file_secure/source.py +++ /dev/null @@ -1,65 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import json -import os - -import source_file - -# some integration tests doesn't setup dependences from -# requirements.txt file and Python can return a exception. -# Thus we should to import this parent module manually -from airbyte_cdk import AirbyteLogger -from airbyte_cdk.models import ConnectorSpecification - -LOCAL_STORAGE_NAME = "local" - - -class URLFileSecure(source_file.client.URLFile): - """Updating of default logic: - This connector shouldn't work with local files. - """ - - def __init__(self, url: str, provider: dict, binary=None, encoding=None): - storage_name = provider["storage"].lower() - if url.startswith("file://") or storage_name == LOCAL_STORAGE_NAME: - raise RuntimeError("the local file storage is not supported by this connector.") - super().__init__(url, provider, binary, encoding) - - -class SourceFileSecure(source_file.SourceFile): - """Updating of default source logic - This connector shouldn't work with local files. - The base logic of this connector are implemented in the "source-file" connector. - """ - - @property - def client_class(self): - # replace a standard class variable to the new one - class ClientSecure(source_file.client.Client): - reader_class = URLFileSecure - - return ClientSecure - - def spec(self, logger: AirbyteLogger) -> ConnectorSpecification: - """Tries to find and remove a spec data about local storage settings""" - - parent_code_dir = os.path.dirname(source_file.source.__file__) - parent_spec_file = os.path.join(parent_code_dir, "spec.json") - with open(parent_spec_file, "r") as f: - spec = ConnectorSpecification.parse_obj(json.load(f)) - - # correction of the "storage" property to const type - for provider in spec.connectionSpecification["properties"]["provider"]["oneOf"]: - storage = provider["properties"]["storage"] - - if "enum" in storage: - storage.pop("enum") - storage["const"] = storage.pop("default") - - for i in range(len(spec.connectionSpecification["properties"]["provider"]["oneOf"])): - provider = spec.connectionSpecification["properties"]["provider"]["oneOf"][i] - if provider["properties"]["storage"]["const"] == LOCAL_STORAGE_NAME: - spec.connectionSpecification["properties"]["provider"]["oneOf"].pop(i) - return spec diff --git a/airbyte-integrations/connectors/source-file-secure/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-file-secure/unit_tests/unit_test.py deleted file mode 100644 index bc714e8ecaeb..000000000000 --- a/airbyte-integrations/connectors/source-file-secure/unit_tests/unit_test.py +++ /dev/null @@ -1,34 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pytest -from airbyte_cdk import AirbyteLogger -from source_file_secure import SourceFileSecure -from source_file_secure.source import LOCAL_STORAGE_NAME - -local_storage_config = { - "dataset_name": "test", - "format": "csv", - "reader_options": '{"sep": ",", "nrows": 20}', - "url": "file:///tmp/fake_file.csv", - "provider": { - "storage": LOCAL_STORAGE_NAME.upper(), - }, -} - - -def test_local_storage_spec(): - """Checks spec properties""" - source = SourceFileSecure() - spec = source.spec(logger=AirbyteLogger()) - for provider in spec.connectionSpecification["properties"]["provider"]["oneOf"]: - assert provider["properties"]["storage"]["const"] != LOCAL_STORAGE_NAME, "This connector shouldn't work with local files." - - -def test_local_storage_check(): - """Checks working with a local options""" - source = SourceFileSecure() - with pytest.raises(RuntimeError) as exc: - source.check(logger=AirbyteLogger(), config=local_storage_config) - assert "not supported" in str(exc.value) diff --git a/airbyte-integrations/connectors/source-file/Dockerfile b/airbyte-integrations/connectors/source-file/Dockerfile deleted file mode 100644 index 5755c3e6505d..000000000000 --- a/airbyte-integrations/connectors/source-file/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM python:3.9-slim as base -FROM base as builder - -RUN apt-get update -WORKDIR /airbyte/integration_code -COPY setup.py ./ -RUN pip install --prefix=/install . - -FROM base -WORKDIR /airbyte/integration_code -COPY --from=builder /install /usr/local - -COPY main.py ./ -COPY source_file ./source_file - - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.3.11 -LABEL io.airbyte.name=airbyte/source-file diff --git a/airbyte-integrations/connectors/source-file/README.md b/airbyte-integrations/connectors/source-file/README.md index dc50ea398d38..2bfd494c401f 100644 --- a/airbyte-integrations/connectors/source-file/README.md +++ b/airbyte-integrations/connectors/source-file/README.md @@ -1,122 +1,34 @@ -# File Source +# File Source -This is the repository for the File source connector, written in Python. +This is the repository for the File source connector. For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/file). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Connector-Specific Dependencies - -For this connector, you will need Rust, as it is a prerequisite for running `pip install cryptography`. You can do this with the recommended installation pattern noted [here](https://www.rust-lang.org/tools/install) on the Rust website. - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-file:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/file) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_file/spec.json` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. - -#### Necessary Credentials for tests - -In order to run integrations tests in this connector, you need: -1. Testing Google Cloud Service Storage - 1. Download and store your Google [Service Account](https://console.cloud.google.com/iam-admin/serviceaccounts) JSON file in `secrets/gcs.json`, it should look something like this: - ``` - { - "type": "service_account", - "project_id": "XXXXXXX", - "private_key_id": "XXXXXXXX", - "private_key": "-----BEGIN PRIVATE KEY-----\nXXXXXXXXXX\n-----END PRIVATE KEY-----\n", - "client_email": "XXXXX@XXXXXX.iam.gserviceaccount.com", - "client_id": "XXXXXXXXX", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/XXXXXXX0XXXXXX.iam.gserviceaccount.com" - } - ``` - 2. Your Service Account should have [Storage Admin Rights](https://console.cloud.google.com/iam-admin/iam) (to create Buckets, read and store files in GCS) - -2. Testing Amazon S3 - 1. Create a file at `secrets/aws.json` - ``` - { - "aws_access_key_id": "XXXXXXX", - "aws_secret_access_key": "XXXXXXX" - } - ``` - -3. Testing Azure Blob Storage - 1. Create a file at `secrets/azblob.json` - ``` - { - "storage_account": "XXXXXXX", - "shared_key": "XXXXXXX", - "sas_token": "XXXXXXX" - } - ``` +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_file/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source file test creds` and place them into `secrets/config.json`. - -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-file:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name source-file build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-file:airbyteDocker +An image will be built with the tag `airbyte/source-file:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-file:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -124,32 +36,32 @@ Then run any of the connector commands as follows: docker run --rm airbyte/source-file:dev spec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-file:dev check --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-file:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-file:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-file:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-file:integrationTest` to run the standard integration test suite. -2. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-file test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-file:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` -To run your integration tests with docker ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -2. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -3. In addition to bumping the connector version of `source-file`, you must also increment the version of `source-file-secure` which depends on this source. The versions of these connectors should always remain in sync. Depending on the changes to `source-file`, you may also need to make changes to `source-file-secure` to retain compatibility. -4. Create a Pull Request -5. Pat yourself on the back for being an awesome contributor -6. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-file test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/file.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-file/acceptance-test-config.yml b/airbyte-integrations/connectors/source-file/acceptance-test-config.yml index b747ed70a605..34d9327e1a9e 100644 --- a/airbyte-integrations/connectors/source-file/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-file/acceptance-test-config.yml @@ -6,12 +6,17 @@ acceptance_tests: spec: tests: - spec_path: "source_file/spec.json" + - spec_path: "integration_tests/cloud_spec.json" + deployment_mode: "cloud" connection: tests: - config_path: "integration_tests/config.json" status: "succeed" - config_path: "integration_tests/invalid_config.json" status: "failed" + - config_path: "integration_tests/local_config.json" + deployment_mode: "cloud" + status: "failed" discovery: tests: - config_path: "integration_tests/config.json" diff --git a/airbyte-integrations/connectors/source-file/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-file/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-file/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-file/build.gradle b/airbyte-integrations/connectors/source-file/build.gradle deleted file mode 100644 index a023a720abe7..000000000000 --- a/airbyte-integrations/connectors/source-file/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_file' -} diff --git a/airbyte-integrations/connectors/source-file/build_customization.py b/airbyte-integrations/connectors/source-file/build_customization.py new file mode 100644 index 000000000000..595240afc694 --- /dev/null +++ b/airbyte-integrations/connectors/source-file/build_customization.py @@ -0,0 +1,24 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + """ + Docker compose is required to run the integration tests so we install Docker on top of the base image. + """ + return ( + base_image_container.with_exec(["sh", "-c", "apt-get update && apt-get install -y curl"]) + # Download install-docker.sh script + .with_exec(["curl", "-fsSL", "https://get.docker.com", "-o", "/tmp/install-docker.sh"]) + # Run the install-docker.sh script with a pinned Docker version + .with_exec(["sh", "/tmp/install-docker.sh", "--version", "23.0"]) + # Remove the install-docker.sh script + .with_exec(["rm", "/tmp/install-docker.sh"]) + ) diff --git a/airbyte-integrations/connectors/source-file-secure/integration_tests/spec.json b/airbyte-integrations/connectors/source-file/integration_tests/cloud_spec.json similarity index 100% rename from airbyte-integrations/connectors/source-file-secure/integration_tests/spec.json rename to airbyte-integrations/connectors/source-file/integration_tests/cloud_spec.json diff --git a/airbyte-integrations/connectors/source-file/integration_tests/file_formats_test.py b/airbyte-integrations/connectors/source-file/integration_tests/file_formats_test.py index bdf0a835d2a5..2f9b195df3c7 100644 --- a/airbyte-integrations/connectors/source-file/integration_tests/file_formats_test.py +++ b/airbyte-integrations/connectors/source-file/integration_tests/file_formats_test.py @@ -7,8 +7,9 @@ import pytest from airbyte_cdk import AirbyteLogger +from airbyte_cdk.utils import AirbyteTracedException from source_file import SourceFile -from source_file.client import Client, ConfigurationError +from source_file.client import Client SAMPLE_DIRECTORY = Path(__file__).resolve().parent.joinpath("sample_files/formats") @@ -59,7 +60,7 @@ def test_raises_file_wrong_format(file_format, extension, wrong_format, filename file_path = str(file_directory.joinpath(f"{filename}.{extension}")) configs = {"dataset_name": "test", "format": wrong_format, "url": file_path, "provider": {"storage": "local"}} client = Client(**configs) - with pytest.raises((TypeError, ValueError, ConfigurationError)): + with pytest.raises((TypeError, ValueError, AirbyteTracedException)): list(client.read()) diff --git a/airbyte-integrations/connectors/source-file-secure/integration_tests/local_config.json b/airbyte-integrations/connectors/source-file/integration_tests/local_config.json similarity index 100% rename from airbyte-integrations/connectors/source-file-secure/integration_tests/local_config.json rename to airbyte-integrations/connectors/source-file/integration_tests/local_config.json diff --git a/airbyte-integrations/connectors/source-file/metadata.yaml b/airbyte-integrations/connectors/source-file/metadata.yaml index a6a868e7b8c3..a2c748e050d3 100644 --- a/airbyte-integrations/connectors/source-file/metadata.yaml +++ b/airbyte-integrations/connectors/source-file/metadata.yaml @@ -1,29 +1,29 @@ data: + ab_internal: + ql: 400 + sl: 200 allowedHosts: hosts: - "*" + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: file connectorType: source definitionId: 778daa7c-feaf-4db6-96f3-70fd645acc77 - dockerImageTag: 0.3.11 + dockerImageTag: 0.3.15 dockerRepository: airbyte/source-file + documentationUrl: https://docs.airbyte.com/integrations/sources/file githubIssueLabel: source-file icon: file.svg license: MIT name: File (CSV, JSON, Excel, Feather, Parquet) registries: cloud: - dockerRepository: airbyte/source-file-secure - dockerImageTag: 0.3.11 # Dont forget to publish source-file-secure as well when updating this. enabled: true oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/file + supportLevel: certified tags: - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-file/setup.py b/airbyte-integrations/connectors/source-file/setup.py index d3ae04ed60c6..513527062877 100644 --- a/airbyte-integrations/connectors/source-file/setup.py +++ b/airbyte-integrations/connectors/source-file/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", + "airbyte-cdk~=0.51.25", "gcsfs==2022.7.1", "genson==1.2.2", "google-cloud-storage==2.5.0", @@ -24,7 +24,7 @@ "pyxlsb==1.0.9", ] -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-docker~=1.0.0", "pytest-mock~=3.6.1", "docker-compose"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-docker~=2.0.1", "pytest-mock~=3.6.1"] setup( name="source_file", diff --git a/airbyte-integrations/connectors/source-file/source_file/client.py b/airbyte-integrations/connectors/source-file/source_file/client.py index 1c6dd8cb7c0a..6c8dd424dda8 100644 --- a/airbyte-integrations/connectors/source-file/source_file/client.py +++ b/airbyte-integrations/connectors/source-file/source_file/client.py @@ -24,7 +24,7 @@ import smart_open.ssh from airbyte_cdk.entrypoint import logger from airbyte_cdk.models import AirbyteStream, FailureType, SyncMode -from airbyte_cdk.utils import AirbyteTracedException +from airbyte_cdk.utils import AirbyteTracedException, is_cloud_environment from azure.storage.blob import BlobServiceClient from genson import SchemaBuilder from google.cloud.storage import Client as GCSClient @@ -36,7 +36,7 @@ from urllib3.exceptions import ProtocolError from yaml import safe_load -from .utils import backoff_handler +from .utils import LOCAL_STORAGE_NAME, backoff_handler SSH_TIMEOUT = 60 @@ -44,14 +44,6 @@ logging.getLogger("smart_open").setLevel(logging.ERROR) -class ConfigurationError(Exception): - """Client mis-configured""" - - -class PermissionsError(Exception): - """User don't have enough permissions""" - - class URLFile: """Class to manage read from file located at different providers @@ -211,7 +203,7 @@ def _open_gcs_url(self) -> object: except json.decoder.JSONDecodeError as err: error_msg = f"Failed to parse gcs service account json: {repr(err)}" logger.error(f"{error_msg}\n{traceback.format_exc()}") - raise ConfigurationError(error_msg) from err + raise AirbyteTracedException(message=error_msg, internal_message=error_msg, failure_type=FailureType.config_error) from err if credentials: credentials = service_account.Credentials.from_service_account_info(credentials) @@ -260,7 +252,6 @@ class Client: """Class that manages reading and parsing data from streams""" CSV_CHUNK_SIZE = 10_000 - reader_class = URLFile binary_formats = {"excel", "excel_binary", "feather", "parquet", "orc", "pickle"} def __init__(self, dataset_name: str, url: str, provider: dict, format: str = None, reader_options: dict = None): @@ -272,6 +263,13 @@ def __init__(self, dataset_name: str, url: str, provider: dict, format: str = No self.binary_source = self._reader_format in self.binary_formats self.encoding = self._reader_options.get("encoding") + @property + def reader_class(self): + if is_cloud_environment(): + return URLFileSecure + + return URLFile + @property def stream_name(self) -> str: if self._dataset_name: @@ -341,7 +339,7 @@ def load_dataframes(self, fp, skip_data=False, read_sample_chunk: bool = False) except KeyError as err: error_msg = f"Reader {self._reader_format} is not supported." logger.error(f"{error_msg}\n{traceback.format_exc()}") - raise ConfigurationError(error_msg) from err + raise AirbyteTracedException(message=error_msg, internal_message=error_msg, failure_type=FailureType.config_error) from err reader_options = {**self._reader_options} try: @@ -367,13 +365,17 @@ def load_dataframes(self, fp, skip_data=False, read_sample_chunk: bool = False) yield reader(fp, **reader_options) else: yield reader(fp, **reader_options) + except ParserError as err: + error_msg = f"File {fp} can not be parsed. Please check your reader_options. https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html" + logger.error(f"{error_msg}\n{traceback.format_exc()}") + raise AirbyteTracedException(message=error_msg, internal_message=error_msg, failure_type=FailureType.config_error) from err except UnicodeDecodeError as err: error_msg = ( f"File {fp} can't be parsed with reader of chosen type ({self._reader_format}). " f"Please check provided Format and Reader Options. {repr(err)}." ) logger.error(f"{error_msg}\n{traceback.format_exc()}") - raise ConfigurationError(error_msg) from err + raise AirbyteTracedException(message=error_msg, internal_message=error_msg, failure_type=FailureType.config_error) from err @staticmethod def dtype_to_json_type(current_type: str, dtype) -> str: @@ -430,11 +432,7 @@ def read(self, fields: Iterable = None) -> Iterable[dict]: f"File {fp} can not be opened due to connection issues on provider side. Please check provided links and options" ) logger.error(f"{error_msg}\n{traceback.format_exc()}") - raise ConfigurationError(error_msg) from err - except ParserError as err: - error_msg = f"File {fp} can not be parsed. Please check your reader_options. https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html" - logger.error(f"{error_msg}\n{traceback.format_exc()}") - raise ConfigurationError(error_msg) from err + raise AirbyteTracedException(message=error_msg, internal_message=error_msg, failure_type=FailureType.config_error) from err def _cache_stream(self, fp): """cache stream to file""" @@ -505,3 +503,15 @@ def openpyxl_chunk_reader(self, file, **kwargs): df = pd.DataFrame(data=(next(data) for _ in range(start, min(start + step, end))), columns=cols) yield df start += step + + +class URLFileSecure(URLFile): + """Updating of default logic: + This connector shouldn't work with local files. + """ + + def __init__(self, url: str, provider: dict, binary=None, encoding=None): + storage_name = provider["storage"].lower() + if url.startswith("file://") or storage_name == LOCAL_STORAGE_NAME: + raise RuntimeError("the local file storage is not supported by this connector.") + super().__init__(url, provider, binary, encoding) diff --git a/airbyte-integrations/connectors/source-file/source_file/source.py b/airbyte-integrations/connectors/source-file/source_file/source.py index aa8af0a4af6f..9293ac5dc1e1 100644 --- a/airbyte-integrations/connectors/source-file/source_file/source.py +++ b/airbyte-integrations/connectors/source-file/source_file/source.py @@ -17,13 +17,16 @@ AirbyteMessage, AirbyteRecordMessage, ConfiguredAirbyteCatalog, + ConnectorSpecification, + FailureType, Status, Type, ) from airbyte_cdk.sources import Source +from airbyte_cdk.utils import AirbyteTracedException, is_cloud_environment -from .client import Client, ConfigurationError -from .utils import dropbox_force_download +from .client import Client +from .utils import LOCAL_STORAGE_NAME, dropbox_force_download class SourceFile(Source): @@ -85,41 +88,52 @@ def _validate_and_transform(config: Mapping[str, Any]): try: config["reader_options"] = json.loads(config["reader_options"]) if not isinstance(config["reader_options"], dict): - raise ConfigurationError( + message = ( "Field 'reader_options' is not a valid JSON object. " "Please provide key-value pairs, See field description for examples." ) + raise AirbyteTracedException(message=message, internal_message=message, failure_type=FailureType.config_error) except ValueError: - raise ConfigurationError("Field 'reader_options' is not valid JSON object. https://www.json.org/") + message = "Field 'reader_options' is not valid JSON object. https://www.json.org/" + raise AirbyteTracedException(message=message, internal_message=message, failure_type=FailureType.config_error) else: config["reader_options"] = {} config["url"] = dropbox_force_download(config["url"]) parse_result = urlparse(config["url"]) if parse_result.netloc == "docs.google.com" and parse_result.path.lower().startswith("/spreadsheets/"): - raise ConfigurationError(f'Failed to load {config["url"]}: please use the Official Google Sheets Source connector') + message = f'Failed to load {config["url"]}: please use the Official Google Sheets Source connector' + raise AirbyteTracedException(message=message, internal_message=message, failure_type=FailureType.config_error) return config + def spec(self, logger: AirbyteLogger) -> ConnectorSpecification: + """Returns the json schema for the spec""" + spec = super().spec(logger) + + # override cloud spec to remove local file support + if is_cloud_environment(): + for i in range(len(spec.connectionSpecification["properties"]["provider"]["oneOf"])): + provider = spec.connectionSpecification["properties"]["provider"]["oneOf"][i] + if provider["properties"]["storage"]["const"] == LOCAL_STORAGE_NAME: + spec.connectionSpecification["properties"]["provider"]["oneOf"].pop(i) + + return spec + def check(self, logger, config: Mapping) -> AirbyteConnectionStatus: """ Check involves verifying that the specified file is reachable with our credentials. """ - try: - config = self._validate_and_transform(config) - except ConfigurationError as e: - logger.error(str(e)) - return AirbyteConnectionStatus(status=Status.FAILED, message=str(e)) - + config = self._validate_and_transform(config) client = self._get_client(config) source_url = client.reader.full_url try: list(client.streams(empty_schema=True)) return AirbyteConnectionStatus(status=Status.SUCCEEDED) - except (TypeError, ValueError, ConfigurationError) as err: + except (TypeError, ValueError, AirbyteTracedException) as err: reason = f"Failed to load {source_url}. Please check File Format and Reader Options are set correctly." logger.error(f"{reason}\n{repr(err)}") - return AirbyteConnectionStatus(status=Status.FAILED, message=reason) + raise AirbyteTracedException(message=reason, internal_message=reason, failure_type=FailureType.config_error) except Exception as err: reason = f"Failed to load {source_url}. You could have provided an invalid URL, please verify it: {repr(err)}." logger.error(reason) diff --git a/airbyte-integrations/connectors/source-file/source_file/utils.py b/airbyte-integrations/connectors/source-file/source_file/utils.py index 5da97cb27054..ef8c258b9179 100644 --- a/airbyte-integrations/connectors/source-file/source_file/utils.py +++ b/airbyte-integrations/connectors/source-file/source_file/utils.py @@ -8,6 +8,8 @@ # default logger logger = logging.getLogger("airbyte") +LOCAL_STORAGE_NAME = "local" + def dropbox_force_download(url): """ diff --git a/airbyte-integrations/connectors/source-file/unit_tests/test_client.py b/airbyte-integrations/connectors/source-file/unit_tests/test_client.py index 4f23d96bd885..d4cae3d46d41 100644 --- a/airbyte-integrations/connectors/source-file/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-file/unit_tests/test_client.py @@ -6,9 +6,10 @@ from unittest.mock import patch, sentinel import pytest +from airbyte_cdk.utils import AirbyteTracedException from pandas import read_csv, read_excel from paramiko import SSHException -from source_file.client import Client, ConfigurationError, URLFile +from source_file.client import Client, URLFile from urllib3.exceptions import ProtocolError @@ -57,7 +58,7 @@ def test_load_dataframes(client, wrong_format_client, absolute_path, test_files) expected = read_csv(f) assert read_file.equals(expected) - with pytest.raises(ConfigurationError): + with pytest.raises(AirbyteTracedException): next(wrong_format_client.load_dataframes(fp=f)) with pytest.raises(StopIteration): @@ -66,7 +67,7 @@ def test_load_dataframes(client, wrong_format_client, absolute_path, test_files) def test_raises_configuration_error_with_incorrect_file_type(csv_format_client, absolute_path, test_files): f = f"{absolute_path}/{test_files}/archive_with_test_xlsx.zip" - with pytest.raises(ConfigurationError): + with pytest.raises(AirbyteTracedException): next(csv_format_client.load_dataframes(fp=f)) @@ -139,7 +140,7 @@ def test_open_gcs_url(): assert URLFile(url="", provider=provider)._open_gcs_url() provider.update({"service_account_json": '{service_account_json": "service_account_json"}'}) - with pytest.raises(ConfigurationError): + with pytest.raises(AirbyteTracedException): assert URLFile(url="", provider=provider)._open_gcs_url() @@ -155,10 +156,10 @@ def test_read(test_read_config): def test_read_network_issues(test_read_config): - test_read_config.update(format='excel') + test_read_config.update(format="excel") client = Client(**test_read_config) client.sleep_on_retry_sec = 0 # just for test - with patch.object(client, "_cache_stream", side_effect=ProtocolError), pytest.raises(ConfigurationError): + with patch.object(client, "_cache_stream", side_effect=ProtocolError), pytest.raises(AirbyteTracedException): next(client.read(["date", "key"])) @@ -176,8 +177,8 @@ def patched_open(self): sleep_mock = mocker.patch("time.sleep") monkeypatch.setattr(URLFile, "_open", patched_open) - provider = {'storage': 'SFTP', 'user': 'user', 'password': 'password', 'host': 'sftp.domain.com', 'port': 22} - reader = URLFile(url='/DISTDA.CSV', provider=provider, binary=False) + provider = {"storage": "SFTP", "user": "user", "password": "password", "host": "sftp.domain.com", "port": 22} + reader = URLFile(url="/DISTDA.CSV", provider=provider, binary=False) with pytest.raises(SSHException): reader.open() assert reader._file is None diff --git a/airbyte-integrations/connectors/source-file/unit_tests/test_source.py b/airbyte-integrations/connectors/source-file/unit_tests/test_source.py index b1704e6673d9..22638f1fbb1a 100644 --- a/airbyte-integrations/connectors/source-file/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-file/unit_tests/test_source.py @@ -5,7 +5,6 @@ import json import logging from copy import deepcopy -from unittest.mock import PropertyMock import jsonschema import pytest @@ -21,7 +20,7 @@ SyncMode, Type, ) -from source_file.client import ConfigurationError +from airbyte_cdk.utils import AirbyteTracedException from source_file.source import SourceFile logger = logging.getLogger("airbyte") @@ -43,7 +42,7 @@ def test_csv_with_utf16_encoding(absolute_path, test_files): config_local_csv_utf16 = { "dataset_name": "AAA", "format": "csv", - "reader_options": '{"encoding":"utf_16", "parse_dates": [\"header5\"]}', + "reader_options": '{"encoding":"utf_16", "parse_dates": ["header5"]}', "url": f"{absolute_path}/{test_files}/test_utf16.csv", "provider": {"storage": "local"}, } @@ -133,9 +132,8 @@ def test_check_invalid_config(source, invalid_config): def test_check_invalid_reader_options(source, invalid_reader_options_config): - expected = AirbyteConnectionStatus(status=Status.FAILED) - actual = source.check(logger=logger, config=invalid_reader_options_config) - assert actual.status == expected.status + with pytest.raises(AirbyteTracedException, match="Field 'reader_options' is not a valid JSON object. Please provide key-value pairs"): + source.check(logger=logger, config=invalid_reader_options_config) def test_discover_dropbox_link(source, config_dropbox_link): @@ -149,25 +147,20 @@ def test_discover(source, config, client): for schema in schemas: jsonschema.Draft7Validator.check_schema(schema) - type(client).streams = PropertyMock(side_effect=Exception) - - with pytest.raises(Exception): - source.discover(logger=logger, config=config) - def test_check_wrong_reader_options(source, config): config["reader_options"] = '{encoding":"utf_16"}' - assert source.check(logger=logger, config=config) == AirbyteConnectionStatus( - status=Status.FAILED, message="Field 'reader_options' is not valid JSON object. https://www.json.org/" - ) + with pytest.raises(AirbyteTracedException, match="Field 'reader_options' is not valid JSON object. https://www.json.org/"): + source.check(logger=logger, config=config) def test_check_google_spreadsheets_url(source, config): config["url"] = "https://docs.google.com/spreadsheets/d/" - assert source.check(logger=logger, config=config) == AirbyteConnectionStatus( - status=Status.FAILED, - message="Failed to load https://docs.google.com/spreadsheets/d/: please use the Official Google Sheets Source connector", - ) + with pytest.raises( + AirbyteTracedException, + match="Failed to load https://docs.google.com/spreadsheets/d/: please use the Official Google Sheets Source connector", + ): + source.check(logger=logger, config=config) def test_pandas_header_not_none(absolute_path, test_files): @@ -218,9 +211,17 @@ def test_incorrect_reader_options(absolute_path, test_files): "provider": {"storage": "local"}, } - catalog = get_catalog({"0": {"type": ["string", "null"]}, "1": {"type": ["string", "null"]}}) source = SourceFile() - with pytest.raises(ConfigurationError) as e: + with pytest.raises( + AirbyteTracedException, + match="can not be parsed. Please check your reader_options. https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html", + ): + _ = source.discover(logger=logger, config=deepcopy(config)) + + with pytest.raises( + AirbyteTracedException, + match="can not be parsed. Please check your reader_options. https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html", + ): + catalog = get_catalog({"0": {"type": ["string", "null"]}, "1": {"type": ["string", "null"]}}) records = source.read(logger=logger, config=deepcopy(config), catalog=catalog) records = [r.record.data for r in records] - assert "can not be parsed. Please check your reader_options. https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html" in str(e.value) diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/README.md b/airbyte-integrations/connectors/source-firebase-realtime-database/README.md index 9f9f61363db6..f4be44977da9 100644 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/README.md +++ b/airbyte-integrations/connectors/source-firebase-realtime-database/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-firebase-realtime-database:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/firebase-realtime-database) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_firebase_realtime_database/spec.yaml` file. @@ -54,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-firebase-realtime-database:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-firebase-realtime-database build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-firebase-realtime-database:airbyteDocker +An image will be built with the tag `airbyte/source-firebase-realtime-database:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-firebase-realtime-database:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,44 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-firebase-realtime-data docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-firebase-realtime-database:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-firebase-realtime-database:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-firebase-realtime-database test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-firebase-realtime-database:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-firebase-realtime-database:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -122,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-firebase-realtime-database test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/firebase-realtime-database.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-firebase-realtime-database/acceptance-test-docker.sh deleted file mode 100644 index c51577d10690..000000000000 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/acceptance-test-docker.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env sh - -# Build latest connector image -docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) - -# Pull latest acctest image -docker pull airbyte/source-acceptance-test:latest - -# Run -docker run --rm -it \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /tmp:/tmp \ - -v $(pwd):/test_input \ - airbyte/source-acceptance-test \ - --acceptance-test-config /test_input - diff --git a/airbyte-integrations/connectors/source-firebase-realtime-database/build.gradle b/airbyte-integrations/connectors/source-firebase-realtime-database/build.gradle deleted file mode 100644 index 585fdcb0ad2e..000000000000 --- a/airbyte-integrations/connectors/source-firebase-realtime-database/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_firebase_realtime_database' -} diff --git a/airbyte-integrations/connectors/source-firebolt/README.md b/airbyte-integrations/connectors/source-firebolt/README.md index 3b60e7094091..6a517492a9e9 100644 --- a/airbyte-integrations/connectors/source-firebolt/README.md +++ b/airbyte-integrations/connectors/source-firebolt/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-firebolt:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/firebolt) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_firebolt/spec.json` file. @@ -54,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-firebolt:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-firebolt build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-firebolt:airbyteDocker +An image will be built with the tag `airbyte/source-firebolt:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-firebolt:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,47 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-firebolt:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-firebolt:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-firebolt:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-firebolt test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-firebolt:unitTest -``` -To run acceptance and custom integration tests: - -Make sure you have a running Firebolt engine that was specified in the config.json. It is needed to run the test queries. - -``` -./gradlew :airbyte-integrations:connectors:source-firebolt:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-firebolt test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/firebolt.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-firebolt/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-firebolt/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-firebolt/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-firebolt/build.gradle b/airbyte-integrations/connectors/source-firebolt/build.gradle deleted file mode 100644 index 2bf29b2581c4..000000000000 --- a/airbyte-integrations/connectors/source-firebolt/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_firebolt_singer' -} diff --git a/airbyte-integrations/connectors/source-flexport/README.md b/airbyte-integrations/connectors/source-flexport/README.md index c93479ce2a32..1ebd5343400b 100644 --- a/airbyte-integrations/connectors/source-flexport/README.md +++ b/airbyte-integrations/connectors/source-flexport/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-flexport:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/flexport) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_flexport/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-flexport:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-flexport build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-flexport:airbyteDocker +An image will be built with the tag `airbyte/source-flexport:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-flexport:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-flexport:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-flexport:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-flexport:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-flexport test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-flexport:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-flexport:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-flexport test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/flexport.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-flexport/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-flexport/acceptance-test-docker.sh deleted file mode 100644 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-flexport/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-flexport/build.gradle b/airbyte-integrations/connectors/source-flexport/build.gradle deleted file mode 100644 index 8945e87680dc..000000000000 --- a/airbyte-integrations/connectors/source-flexport/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_flexport' -} diff --git a/airbyte-integrations/connectors/source-freshcaller/.dockerignore b/airbyte-integrations/connectors/source-freshcaller/.dockerignore index 9dcc6f34cdf5..58468c817897 100644 --- a/airbyte-integrations/connectors/source-freshcaller/.dockerignore +++ b/airbyte-integrations/connectors/source-freshcaller/.dockerignore @@ -1,6 +1,5 @@ * !Dockerfile -!Dockerfile.test !main.py !source_freshcaller !setup.py diff --git a/airbyte-integrations/connectors/source-freshcaller/Dockerfile b/airbyte-integrations/connectors/source-freshcaller/Dockerfile index 546d084535f1..0b83d2b1718f 100644 --- a/airbyte-integrations/connectors/source-freshcaller/Dockerfile +++ b/airbyte-integrations/connectors/source-freshcaller/Dockerfile @@ -31,9 +31,8 @@ RUN apk --no-cache add bash COPY main.py ./ COPY source_freshcaller ./source_freshcaller - ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.version=0.3.1 LABEL io.airbyte.name=airbyte/source-freshcaller diff --git a/airbyte-integrations/connectors/source-freshcaller/README.md b/airbyte-integrations/connectors/source-freshcaller/README.md index 1cd93030fc58..f50f7789a1c5 100644 --- a/airbyte-integrations/connectors/source-freshcaller/README.md +++ b/airbyte-integrations/connectors/source-freshcaller/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-freshcaller:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/freshcaller) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_freshcaller/spec.json` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-freshcaller:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-freshcaller build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-freshcaller:airbyteDocker +An image will be built with the tag `airbyte/source-freshcaller:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-freshcaller:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,45 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-freshcaller:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-freshcaller:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-freshcaller:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-freshcaller test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-freshcaller:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-freshcaller:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-freshcaller:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -126,8 +90,10 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-freshcaller test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/freshcaller.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-freshcaller/__init__.py b/airbyte-integrations/connectors/source-freshcaller/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-freshcaller/acceptance-test-config.yml b/airbyte-integrations/connectors/source-freshcaller/acceptance-test-config.yml index c9b4d6e711d7..912f1a29efac 100644 --- a/airbyte-integrations/connectors/source-freshcaller/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-freshcaller/acceptance-test-config.yml @@ -1,24 +1,37 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests connector_image: airbyte/source-freshcaller:dev -tests: +test_strictness_level: low +acceptance_tests: spec: - - spec_path: "source_freshcaller/spec.json" - backward_compatibility_tests_config: - disable_for_version: 0.1.0 # Fix start_date type + tests: + - spec_path: "source_freshcaller/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_full_refresh.json" - empty_streams: ["teams"] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_full_refresh.json" + empty_streams: + - name: teams incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_incremental.json" - future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_incremental.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_incremental_metrics.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_full_refresh.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_full_refresh.json" diff --git a/airbyte-integrations/connectors/source-freshcaller/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-freshcaller/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-freshcaller/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-freshcaller/build.gradle b/airbyte-integrations/connectors/source-freshcaller/build.gradle deleted file mode 100644 index 8d7cb806a436..000000000000 --- a/airbyte-integrations/connectors/source-freshcaller/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_freshcaller' -} diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/__init__.py b/airbyte-integrations/connectors/source-freshcaller/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-freshcaller/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-freshcaller/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-freshcaller/integration_tests/abnormal_state.json index dbc7bff5fef0..0515b081aab8 100644 --- a/airbyte-integrations/connectors/source-freshcaller/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-freshcaller/integration_tests/abnormal_state.json @@ -1,8 +1,24 @@ -{ - "call_metrics": { - "created_time": "2099-03-16T05:33:40.987000+00:00" +[ + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "call_metrics" + }, + "stream_state": { + "created_time": "2050-11-07 18:19:48.909179+00:00" + } + } }, - "calls": { - "created_time": "2099-03-16T05:33:40.823000+00:00" + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "calls" + }, + "stream_state": { + "created_time": "2050-11-07 18:19:42.687520+00:00" + } + } } -} +] diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/catalog.json b/airbyte-integrations/connectors/source-freshcaller/integration_tests/catalog.json deleted file mode 100644 index e634015fb51f..000000000000 --- a/airbyte-integrations/connectors/source-freshcaller/integration_tests/catalog.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "users", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "teams", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "calls", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"] - }, - "destination_sync_mode": "overwrite", - "sync_mode": "full_refresh" - }, - { - "stream": { - "name": "call_metrics", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"] - }, - "destination_sync_mode": "overwrite", - "sync_mode": "full_refresh" - } - ] -} diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/configured_catalog_incremental.json b/airbyte-integrations/connectors/source-freshcaller/integration_tests/configured_catalog_incremental.json index 0d038391a8c1..7b3fdd9d00a5 100644 --- a/airbyte-integrations/connectors/source-freshcaller/integration_tests/configured_catalog_incremental.json +++ b/airbyte-integrations/connectors/source-freshcaller/integration_tests/configured_catalog_incremental.json @@ -12,19 +12,6 @@ "destination_sync_mode": "append", "cursor_field": ["created_time"], "sync_mode": "incremental" - }, - { - "stream": { - "name": "call_metrics", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created_time"], - "source_defined_primary_key": [["id"]] - }, - "destination_sync_mode": "append", - "cursor_field": ["created_time"], - "sync_mode": "incremental" } ] } diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/configured_catalog_incremental_metrics.json b/airbyte-integrations/connectors/source-freshcaller/integration_tests/configured_catalog_incremental_metrics.json new file mode 100644 index 000000000000..a8cacfd0e3bb --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/integration_tests/configured_catalog_incremental_metrics.json @@ -0,0 +1,17 @@ +{ + "streams": [ + { + "stream": { + "name": "call_metrics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created_time"], + "source_defined_primary_key": [["id"]] + }, + "destination_sync_mode": "append", + "cursor_field": ["created_time"], + "sync_mode": "incremental" + } + ] +} diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-freshcaller/integration_tests/invalid_config.json index 3e0559353558..ca728caefaa1 100644 --- a/airbyte-integrations/connectors/source-freshcaller/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-freshcaller/integration_tests/invalid_config.json @@ -1,5 +1,5 @@ { "api_key": "incorrect_key", "start_date": "2021-08-02T00:00:00Z", - "domain": "incorrect_domain" + "domain": "incorrect-domain" } diff --git a/airbyte-integrations/connectors/source-freshcaller/sample_files/config.json b/airbyte-integrations/connectors/source-freshcaller/integration_tests/sample_config.json similarity index 100% rename from airbyte-integrations/connectors/source-freshcaller/sample_files/config.json rename to airbyte-integrations/connectors/source-freshcaller/integration_tests/sample_config.json diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-freshcaller/integration_tests/sample_state.json new file mode 100644 index 000000000000..6787adf72137 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/integration_tests/sample_state.json @@ -0,0 +1,24 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "call_metrics" + }, + "stream_state": { + "created_time": "2021-11-07 18:19:48.909179+00:00" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "calls" + }, + "stream_state": { + "created_time": "2021-11-07 18:19:42.687520+00:00" + } + } + } +] diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-freshcaller/integration_tests/test_incremental_streams.py deleted file mode 100644 index c285e8e23d41..000000000000 --- a/airbyte-integrations/connectors/source-freshcaller/integration_tests/test_incremental_streams.py +++ /dev/null @@ -1,126 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -import pendulum -import pytest -from airbyte_cdk.models import SyncMode -from source_freshcaller.streams import APIIncrementalFreshcallerStream, CallMetrics, Calls - - -@pytest.fixture -def patch_incremental_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(APIIncrementalFreshcallerStream, "cursor_field", "created_time") - mocker.patch.object(APIIncrementalFreshcallerStream, "__abstractmethods__", set()) - - -@pytest.fixture -def args(): - return {"authenticator": None, "config": {"api_key": "", "domain": "airbyte", "start_date": "2021-01-01T00:00:00.000Z"}} - - -@pytest.fixture -def stream(patch_incremental_base_class, args): - return APIIncrementalFreshcallerStream(**args) - - -@pytest.fixture -def call_metrics_stream(args): - return CallMetrics(**args) - - -@pytest.fixture -def calls_stream(args): - return Calls(**args) - - -@pytest.fixture -def streams_dict(calls_stream, call_metrics_stream): - return {"calls_stream": calls_stream, "call_metrics_stream": call_metrics_stream} - - -@pytest.mark.parametrize("fixture_name, expected", [("calls_stream", "created_time"), ("call_metrics_stream", "created_time")]) -def test_cursor_field(streams_dict, fixture_name, expected): - stream = streams_dict[fixture_name] - assert stream.cursor_field == expected - - -@pytest.mark.parametrize("fixture_name", [("calls_stream"), ("call_metrics_stream")]) -def test_get_updated_state(streams_dict, fixture_name): - stream = streams_dict[fixture_name] - inputs = { - "current_stream_state": {"created_time": "2021-10-10T00:00:00.00Z"}, - "latest_record": {"created_time": "2021-10-20T00:00:00.00Z"}, - } - state = stream.get_updated_state(**inputs) - assert state["created_time"] == pendulum.parse("2021-10-20T00:00:00.00Z") - - inputs = {"current_stream_state": state, "latest_record": {"created_time": "2021-10-30T00:00:00.00Z"}} - state = stream.get_updated_state(**inputs) - assert state["created_time"] == pendulum.parse("2021-10-30T00:00:00.00Z") - - -@pytest.mark.parametrize("fixture_name", [("calls_stream"), ("call_metrics_stream")]) -def test_get_updated_state_2(streams_dict, fixture_name): - stream = streams_dict[fixture_name] - current_stream_state = {"created_time": pendulum.now().add(days=-40)} - inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": current_stream_state} - # The number of slices should be total lookback days by window_in_days i.e., 40 / 5 = 8 - assert len(stream.stream_slices(**inputs)) == 8 - - -def test_end_of_stream_state(calls_stream, requests_mock): - stream = calls_stream - requests_mock.get( - "https://airbyte.freshcaller.com/api/v1/calls?per_page=1000", - json={ - "calls": [{"created_time": "2021-10-30T00:00:00.00Z"}, {"created_time": "2021-10-29T00:00:00.00Z"}], - "meta": {"total_pages": 40, "current": 40}, - }, - ) - - state = {"created_time": "2021-10-01T00:00:00.00Z"} - sync_mode = SyncMode.incremental - last_state = None - for idx, app_slice in enumerate(stream.stream_slices(state, **MagicMock())): - for record in stream.read_records(sync_mode=sync_mode, stream_slice=app_slice): - state = stream.get_updated_state(state, record) - last_state = state["created_time"] - assert last_state == pendulum.parse("2021-10-30T00:00:00.00Z") - - -@pytest.mark.parametrize("fixture_name", [("calls_stream"), ("call_metrics_stream")]) -def test_supports_incremental(mocker, streams_dict, fixture_name): - stream = streams_dict[fixture_name] - mocker.patch.object(APIIncrementalFreshcallerStream, "cursor_field", "dummy_field") - assert stream.supports_incremental - - -@pytest.mark.parametrize("fixture_name", [("calls_stream"), ("call_metrics_stream")]) -def test_source_defined_cursor(mocker, streams_dict, fixture_name): - stream = streams_dict[fixture_name] - assert stream.source_defined_cursor - - -@pytest.mark.parametrize("fixture_name", [("calls_stream"), ("call_metrics_stream")]) -def test_stream_checkpoint_interval(mocker, streams_dict, fixture_name): - stream = streams_dict[fixture_name] - expected_checkpoint_interval = None - assert stream.state_checkpoint_interval == expected_checkpoint_interval - - -@pytest.mark.parametrize("fixture_name", [("calls_stream"), ("call_metrics_stream")]) -def test_request_params(mocker, streams_dict, fixture_name): - stream = streams_dict[fixture_name] - inputs = { - "stream_state": {}, - "next_page_token": {"page": "5"}, - "stream_slice": {"by_time[from]": "2022-03-04 18:27:40", "by_time[to]": "2022-03-09 18:27:39"}, - } - expected_request_params = {"per_page": 1000, "page": "5", "by_time[from]": "2022-03-04 18:27:40", "by_time[to]": "2022-03-09 18:27:39"} - if stream.path() == "calls": - expected_request_params.update({"has_ancestry": "true"}) - assert stream.request_params(**inputs) == expected_request_params diff --git a/airbyte-integrations/connectors/source-freshcaller/metadata.yaml b/airbyte-integrations/connectors/source-freshcaller/metadata.yaml index 29164f4d589e..ac2d4fb29261 100644 --- a/airbyte-integrations/connectors/source-freshcaller/metadata.yaml +++ b/airbyte-integrations/connectors/source-freshcaller/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: 8a5d48f6-03bb-4038-a942-a8d3f175cca3 - dockerImageTag: 0.2.0 + dockerImageTag: 0.3.1 dockerRepository: airbyte/source-freshcaller githubIssueLabel: source-freshcaller icon: freshcaller.svg @@ -14,11 +14,11 @@ data: oss: enabled: true releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/freshcaller tags: - - language:python + - language:low-code ab_internal: sl: 100 ql: 100 - supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog.json deleted file mode 100644 index c9226aa5bdc0..000000000000 --- a/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog.json +++ /dev/null @@ -1,407 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "users", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "id": { - "type": ["integer", "null"] - }, - "name": { - "type": ["string", "null"] - }, - "email": { - "type": ["string", "null"] - }, - "phone": { - "type": ["string", "null"] - }, - "status": { - "type": ["integer", "null"] - }, - "preference": { - "type": ["integer", "null"] - }, - "mobile_app_preference": { - "type": ["integer", "null"] - }, - "last_call_time": { - "format": "date-time", - "type": ["string", "null"] - }, - "last_seen_time": { - "format": "date-time", - "type": ["string", "null"] - }, - "confirmed": { - "type": ["boolean", "null"] - }, - "language": { - "type": ["string", "null"] - }, - "time_zone": { - "type": ["string", "null"] - }, - "deleted": { - "type": ["boolean", "null"] - }, - "role": { - "type": ["string", "null"] - }, - "teams": { - "items": { - "properties": { - "id": { - "type": ["integer", "null"] - } - }, - "type": "object" - }, - "type": ["array", "null"] - } - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]], - "destination_sync_mode": "overwrite", - "sync_mode": "full_refresh" - }, - { - "stream": { - "name": "teams", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "id": { - "type": ["integer", "null"] - }, - "name": { - "type": ["string", "null"] - }, - "description": { - "type": ["string", "null"] - }, - "users": { - "items": { - "properties": { - "id": { - "type": ["integer", "null"] - } - }, - "type": "object" - }, - "type": ["array", "null"] - }, - "omni_channel": { - "type": ["boolean", "null"] - } - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]], - "destination_sync_mode": "overwrite", - "sync_mode": "full_refresh" - }, - { - "stream": { - "name": "calls", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "id": { - "type": ["integer", "null"] - }, - "direction": { - "type": ["string", "null"] - }, - "parent_call_id": { - "type": ["integer", "null"] - }, - "root_call_id": { - "type": ["integer", "null"] - }, - "phone_number_id": { - "type": ["integer", "null"] - }, - "phone_number": { - "type": ["string", "null"] - }, - "assigned_agent_id": { - "type": ["integer", "null"] - }, - "assigned_agent_name": { - "type": ["string", "null"] - }, - "assigned_team_id": { - "type": ["integer", "null"] - }, - "assigned_team_name": { - "type": ["string", "null"] - }, - "assigned_call_queue_id": { - "type": ["integer", "null"] - }, - "assigned_call_queue_name": { - "type": ["string", "null"] - }, - "assigned_ivr_id": { - "type": ["integer", "null"] - }, - "assigned_ivr_name": { - "type": ["string", "null"] - }, - "bill_duration": { - "type": ["number", "null"] - }, - "bill_duration_unit": { - "type": ["string", "null"] - }, - "created_time": { - "format": "date-time", - "type": ["string", "null"] - }, - "updated_time": { - "format": "date-time", - "type": ["string", "null"] - }, - "call_notes": { - "type": ["string", "null"] - }, - "integrated_resources": { - "items": { - "properties": { - "integration_name": { - "type": ["string", "null"] - }, - "type": { - "type": ["string", "null"] - }, - "id": { - "type": ["string", "null"] - } - }, - "type": "object" - }, - "type": ["array", "null"] - }, - "recording": { - "properties": { - "id": { - "type": ["integer", "null"] - }, - "url": { - "type": ["string", "null"] - }, - "transcription_url": { - "type": ["string", "null"] - }, - "duration": { - "type": ["number", "null"] - }, - "duration_unit": { - "type": ["string", "null"] - } - }, - "type": ["object", "null"] - }, - - "participants": { - "items": { - "properties": { - "id": { - "type": ["integer", "null"] - }, - "call_id": { - "type": ["integer", "null"] - }, - "caller_id": { - "type": ["integer", "null"] - }, - "caller_number": { - "type": ["string", "null"] - }, - "caller_name": { - "type": ["string", "null"] - }, - "participant_id": { - "type": ["integer", "null"] - }, - "participant_type": { - "type": ["string", "null"] - }, - "connection_type": { - "type": ["number", "null"] - }, - "call_status": { - "type": ["integer", "null"] - }, - "duration": { - "type": ["integer", "null"] - }, - "duration_unit": { - "type": ["string", "null"] - }, - "cost": { - "type": ["number", "null"] - }, - "cost_unit": { - "type": ["string", "null"] - }, - "enqueued_time": { - "type": ["string", "null"] - }, - "created_time": { - "format": "date-time", - "type": ["string", "null"] - }, - "updated_time": { - "format": "date-time", - "type": ["string", "null"] - } - }, - "type": "object" - }, - "type": ["array", "null"] - } - } - } - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["updated_time"], - "source_defined_primary_key": [["id"]], - "destination_sync_mode": "overwrite", - "sync_mode": "incremental" - }, - { - "stream": { - "name": "call_metrics", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "id": { - "type": ["integer", "null"] - }, - "call_id": { - "type": ["integer", "null"] - }, - "ivr_time": { - "type": ["integer", "null"] - }, - "ivr_time_unit": { - "type": ["string", "null"] - }, - "hold_duration": { - "type": ["number", "null"] - }, - "hold_duration_unit": { - "type": ["string", "null"] - }, - "call_work_time": { - "type": ["number", "null"] - }, - "call_work_time_unit": { - "type": ["string", "null"] - }, - "total_ringing_time": { - "type": ["number", "null"] - }, - "total_ringing_time_unit": { - "type": ["string", "null"] - }, - "talk_time": { - "type": ["number", "null"] - }, - "talk_time_unit": { - "type": ["string", "null"] - }, - "answering_speed": { - "type": ["number", "null"] - }, - "answering_speed_unit": { - "type": ["string", "null"] - }, - "recording_duration": { - "type": ["number", "null"] - }, - "recording_duration_unit": { - "type": ["string", "null"] - }, - "bill_duration": { - "type": ["number", "null"] - }, - "bill_duration_unit": { - "type": ["string", "null"] - }, - "cost": { - "type": ["number", "null"] - }, - "cost_unit": { - "type": ["string", "null"] - }, - "csat": { - "properties": { - "transfer_made": { - "type": ["boolean", "null"] - }, - "outcome": { - "type": ["string", "null"] - }, - "time": { - "type": ["number", "null"] - }, - "time_unit": { - "type": ["string", "null"] - } - }, - "type": ["object", "null"] - }, - "created_time": { - "format": "date-time", - "type": ["string", "null"] - }, - "updated_time": { - "format": "date-time", - "type": ["string", "null"] - }, - "tags": { - "items": { - "properties": { - "id": { - "type": ["integer", "null"] - }, - "name": { - "type": ["string", "null"] - }, - "default": { - "type": ["boolean", "null"] - } - }, - "type": "object" - }, - "type": ["array", "null"] - } - } - } - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["updated_time"], - "source_defined_primary_key": [["id"]], - "destination_sync_mode": "overwrite", - "sync_mode": "incremental" - } - ] -} diff --git a/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_calls.json b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_calls.json deleted file mode 100644 index 743c16e299e6..000000000000 --- a/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_calls.json +++ /dev/null @@ -1,177 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "calls", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "id": { - "type": ["integer", "null"] - }, - "direction": { - "type": ["string", "null"] - }, - "parent_call_id": { - "type": ["integer", "null"] - }, - "root_call_id": { - "type": ["integer", "null"] - }, - "phone_number_id": { - "type": ["integer", "null"] - }, - "phone_number": { - "type": ["string", "null"] - }, - "assigned_agent_id": { - "type": ["integer", "null"] - }, - "assigned_agent_name": { - "type": ["string", "null"] - }, - "assigned_team_id": { - "type": ["integer", "null"] - }, - "assigned_team_name": { - "type": ["string", "null"] - }, - "assigned_call_queue_id": { - "type": ["integer", "null"] - }, - "assigned_call_queue_name": { - "type": ["string", "null"] - }, - "assigned_ivr_id": { - "type": ["integer", "null"] - }, - "assigned_ivr_name": { - "type": ["string", "null"] - }, - "bill_duration": { - "type": ["number", "null"] - }, - "bill_duration_unit": { - "type": ["string", "null"] - }, - "created_time": { - "format": "date-time", - "type": ["string", "null"] - }, - "updated_time": { - "format": "date-time", - "type": ["string", "null"] - }, - "call_notes": { - "type": ["string", "null"] - }, - "integrated_resources": { - "items": { - "properties": { - "integration_name": { - "type": ["string", "null"] - }, - "type": { - "type": ["string", "null"] - }, - "id": { - "type": ["string", "null"] - } - }, - "type": "object" - }, - "type": ["array", "null"] - }, - "recording": { - "properties": { - "id": { - "type": ["integer", "null"] - }, - "url": { - "type": ["string", "null"] - }, - "transcription_url": { - "type": ["string", "null"] - }, - "duration": { - "type": ["number", "null"] - }, - "duration_unit": { - "type": ["string", "null"] - } - }, - "type": ["object", "null"] - }, - - "participants": { - "items": { - "properties": { - "id": { - "type": ["integer", "null"] - }, - "call_id": { - "type": ["integer", "null"] - }, - "caller_id": { - "type": ["integer", "null"] - }, - "caller_number": { - "type": ["string", "null"] - }, - "caller_name": { - "type": ["string", "null"] - }, - "participant_id": { - "type": ["integer", "null"] - }, - "participant_type": { - "type": ["string", "null"] - }, - "connection_type": { - "type": ["number", "null"] - }, - "call_status": { - "type": ["integer", "null"] - }, - "duration": { - "type": ["integer", "null"] - }, - "duration_unit": { - "type": ["string", "null"] - }, - "cost": { - "type": ["number", "null"] - }, - "cost_unit": { - "type": ["string", "null"] - }, - "enqueued_time": { - "type": ["string", "null"] - }, - "created_time": { - "format": "date-time", - "type": ["string", "null"] - }, - "updated_time": { - "format": "date-time", - "type": ["string", "null"] - } - }, - "type": "object" - }, - "type": ["array", "null"] - } - } - } - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["updated_time"], - "source_defined_primary_key": [["id"]], - "destination_sync_mode": "overwrite", - "sync_mode": "incremental" - } - ] -} diff --git a/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_full_refresh.json b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_full_refresh.json deleted file mode 100644 index e634015fb51f..000000000000 --- a/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_full_refresh.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "users", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "teams", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "calls", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"] - }, - "destination_sync_mode": "overwrite", - "sync_mode": "full_refresh" - }, - { - "stream": { - "name": "call_metrics", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"] - }, - "destination_sync_mode": "overwrite", - "sync_mode": "full_refresh" - } - ] -} diff --git a/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_incremental.json b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_incremental.json deleted file mode 100644 index 6ba515640165..000000000000 --- a/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_incremental.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "calls", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["updated_time"], - "source_defined_primary_key": [["id"]] - }, - "destination_sync_mode": "append", - "cursor_field": ["updated_time"], - "sync_mode": "incremental" - }, - { - "stream": { - "name": "call_metrics", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["updated_time"], - "source_defined_primary_key": [["id"]] - }, - "destination_sync_mode": "append", - "cursor_field": ["updated_time"], - "sync_mode": "incremental" - } - ] -} diff --git a/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_metrics.json b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_metrics.json deleted file mode 100644 index 2ac580a06e50..000000000000 --- a/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_metrics.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "call_metrics", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "id": { - "type": ["integer", "null"] - }, - "call_id": { - "type": ["integer", "null"] - }, - "ivr_time": { - "type": ["integer", "null"] - }, - "ivr_time_unit": { - "type": ["string", "null"] - }, - "hold_duration": { - "type": ["number", "null"] - }, - "hold_duration_unit": { - "type": ["string", "null"] - }, - "call_work_time": { - "type": ["number", "null"] - }, - "call_work_time_unit": { - "type": ["string", "null"] - }, - "total_ringing_time": { - "type": ["number", "null"] - }, - "total_ringing_time_unit": { - "type": ["string", "null"] - }, - "talk_time": { - "type": ["number", "null"] - }, - "talk_time_unit": { - "type": ["string", "null"] - }, - "answering_speed": { - "type": ["number", "null"] - }, - "answering_speed_unit": { - "type": ["string", "null"] - }, - "recording_duration": { - "type": ["number", "null"] - }, - "recording_duration_unit": { - "type": ["string", "null"] - }, - "bill_duration": { - "type": ["number", "null"] - }, - "bill_duration_unit": { - "type": ["string", "null"] - }, - "cost": { - "type": ["number", "null"] - }, - "cost_unit": { - "type": ["string", "null"] - }, - "csat": { - "properties": { - "transfer_made": { - "type": ["boolean", "null"] - }, - "outcome": { - "type": ["string", "null"] - }, - "time": { - "type": ["number", "null"] - }, - "time_unit": { - "type": ["string", "null"] - } - }, - "type": ["object", "null"] - }, - "created_time": { - "format": "date-time", - "type": ["string", "null"] - }, - "updated_time": { - "format": "date-time", - "type": ["string", "null"] - }, - "tags": { - "items": { - "properties": { - "id": { - "type": ["integer", "null"] - }, - "name": { - "type": ["string", "null"] - }, - "default": { - "type": ["boolean", "null"] - } - }, - "type": "object" - }, - "type": ["array", "null"] - } - } - } - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["updated_time"], - "source_defined_primary_key": [["id"]], - "destination_sync_mode": "overwrite", - "sync_mode": "incremental" - } - ] -} diff --git a/airbyte-integrations/connectors/source-freshcaller/setup.py b/airbyte-integrations/connectors/source-freshcaller/setup.py index ec2fdf7dba17..27b47c30913f 100644 --- a/airbyte-integrations/connectors/source-freshcaller/setup.py +++ b/airbyte-integrations/connectors/source-freshcaller/setup.py @@ -6,11 +6,11 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "pytest~=6.2", "pytest-mock~=3.6.1", "requests-mock~=1.9.3", ] @@ -18,11 +18,11 @@ setup( name="source_freshcaller", description="Source implementation for Freshcaller.", - author="Jay Bujala (Snapcommerce)", - author_email="jay.bujala@snapcommerce.com", + author="Airbyte", + author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/__init__.py b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/__init__.py index c4276956372e..4bdab86bedb7 100644 --- a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/__init__.py +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/__init__.py @@ -1,3 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + from .source import SourceFreshcaller __all__ = ["SourceFreshcaller"] diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/manifest.yaml b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/manifest.yaml new file mode 100644 index 000000000000..4d3e95eafcb8 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/manifest.yaml @@ -0,0 +1,180 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + requester: + type: HttpRequester + url_base: "https://{{ config['domain'] }}.freshcaller.com/api/v1" + http_method: "GET" + request_headers: + Accept: "application/json" + authenticator: + type: ApiKeyAuthenticator + header: "X-Api-Auth" + api_token: "{{ config['api_key'] }}" + datetime_cursor: + type: "DatetimeBasedCursor" + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + datetime: "{{ now_utc() }}" + datetime_format: "%Y-%m-%d %H:%M:%S.%f+00:00" + step: "P1Y" + datetime_format: "%Y-%m-%d %H:%M:%S.%f+00:00" + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%f%z" + cursor_granularity: "P1D" + cursor_field: "created_time" + start_time_option: + type: RequestOption + field_name: "by_time[from]" + inject_into: "request_parameter" + end_time_option: + type: RequestOption + field_name: "by_time[to]" + inject_into: "request_parameter" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: "DefaultPaginator" + page_size_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "per_page" + pagination_strategy: + type: "PageIncrement" + page_size: 1000 + start_from_page: 1 + page_token_option: + type: "RequestOption" + field_name: "page" + inject_into: "request_parameter" + requester: + $ref: "#/definitions/requester" + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + users_stream: + $ref: "#/definitions/base_stream" + name: "users" + primary_key: "id" + $parameters: + path: "/users" + retriever: + $ref: "#/definitions/retriever" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["users"] + teams_stream: + $ref: "#/definitions/base_stream" + name: "teams" + primary_key: "id" + $parameters: + path: "/teams" + retriever: + $ref: "#/definitions/retriever" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["teams"] + calls_stream: + $ref: "#/definitions/base_stream" + incremental_sync: + $ref: "#/definitions/datetime_cursor" + name: "calls" + $parameters: + path: "/calls" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + request_parameters: + has_ancestry: "true" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["calls"] + call_metrics_stream: + $ref: "#/definitions/base_stream" + incremental_sync: + $ref: "#/definitions/datetime_cursor" + name: "call_metrics" + $parameters: + path: "/call_metrics" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + request_parameters: + has_ancestry: "true" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["call_metrics"] + +streams: + - "#/definitions/users_stream" + - "#/definitions/teams_stream" + - "#/definitions/calls_stream" + - "#/definitions/call_metrics_stream" + +check: + type: CheckStream + stream_names: + - "users" + - "teams" + - "calls" + - "call_metrics" + +spec: + type: Spec + documentationUrl: https://docs.airbyte.com/integrations/sources/freshcaller + connection_specification: + "$schema": https://json-schema.org/draft-07/schema# + title: Freshcaller Spec + type: object + required: + - domain + - api_key + additionalProperties: true + properties: + domain: + type: string + title: Domain for Freshcaller account + description: Used to construct Base URL for the Freshcaller APIs + examples: + - snaptravel + api_key: + type: string + title: API Key + description: Freshcaller API Key. See the docs for more information on how to obtain this key. + airbyte_secret: true + requests_per_minute: + title: Requests per minute + type: integer + description: The number of requests per minute that this source allowed to use. There is a rate limit of 50 requests per minute per app per account. + start_date: + title: Start Date + description: UTC date and time. Any data created after this date will be replicated. + format: date-time + type: string + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + examples: + - "2022-01-01T12:00:00Z" + sync_lag_minutes: + title: Lag in minutes for each sync + type: integer + description: Lag in minutes for each sync, i.e., at time T, data for the time range [prev_sync_time, T-30] will be fetched diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/source.py b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/source.py index b6997a6e9329..61eb6c017422 100644 --- a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/source.py +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/source.py @@ -2,45 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import logging -from typing import Any, List, Mapping, Tuple +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import requests -from airbyte_cdk import AirbyteLogger -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -from source_freshcaller.streams import CallMetrics, Calls, Teams, Users +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -logger = logging.getLogger("airbyte") +WARNING: Do not modify this file. +""" -class FreshcallerTokenAuthenticator(TokenAuthenticator): - def get_auth_header(self) -> Mapping[str, Any]: - return {"X-Api-Auth": self._token} - - -class SourceFreshcaller(AbstractSource): - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: - api_url = f"https://{config['domain']}.freshcaller.com/api/v1" - auth = FreshcallerTokenAuthenticator(token=config["api_key"]).get_auth_header() - url = "{api_url}/users".format(api_url=api_url) - auth.update({"Accept": "application/json"}) - auth.update({"Content-Type": "application/json"}) - - try: - session = requests.get(url, headers=auth) - session.raise_for_status() - return True, None - except Exception as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - authenticator = FreshcallerTokenAuthenticator(token=config["api_key"]) - args = {"authenticator": authenticator, "config": config} - return [ - Users(**args), - Teams(**args), - Calls(**args), - CallMetrics(**args), - ] +# Declarative Source +class SourceFreshcaller(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/spec.json b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/spec.json deleted file mode 100644 index 6e2fd360dd3f..000000000000 --- a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/spec.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/freshcaller", - "connectionSpecification": { - "$schema": "https://json-schema.org/draft-07/schema#", - "title": "Freshcaller Spec", - "type": "object", - "required": ["domain", "api_key", "start_date"], - "additionalProperties": true, - "properties": { - "domain": { - "type": "string", - "title": "Domain for Freshcaller account", - "description": "Used to construct Base URL for the Freshcaller APIs", - "examples": ["snaptravel"] - }, - "api_key": { - "type": "string", - "title": "API Key", - "description": "Freshcaller API Key. See the docs for more information on how to obtain this key.", - "airbyte_secret": true - }, - "requests_per_minute": { - "title": "Requests per minute", - "type": "integer", - "description": "The number of requests per minute that this source allowed to use. There is a rate limit of 50 requests per minute per app per account." - }, - "start_date": { - "title": "Start Date", - "description": "UTC date and time. Any data created after this date will be replicated.", - "format": "date-time", - "type": "string", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2022-01-01T12:00:00Z"] - }, - "sync_lag_minutes": { - "title": "Lag in minutes for each sync", - "type": "integer", - "description": "Lag in minutes for each sync, i.e., at time T, data for the time range [prev_sync_time, T-30] will be fetched" - } - } - } -} diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/streams.py b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/streams.py deleted file mode 100644 index 2ffee1474369..000000000000 --- a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/streams.py +++ /dev/null @@ -1,202 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from abc import ABC, abstractmethod -from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional, Union - -import pendulum -import requests -from airbyte_cdk.sources.streams.http import HttpStream - - -class FreshcallerStream(HttpStream, ABC): - """Abstract class curated for Freshcaller""" - - primary_key = "id" - data_field = "" - start = 1 - page_limit = 1000 - api_version = 2 - curr_page_param = "page" - - def __init__(self, config: Dict, **kwargs): - super().__init__(**kwargs) - self.config = config - - @property - def url_base(self) -> str: - return f"https://{self.config['domain']}.freshcaller.com/api/v1/" - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - - params = {"per_page": self.page_limit, self.curr_page_param: self.start} - - # Handle pagination by inserting the next page's token in the request parameters - if next_page_token: - self.logger.debug(f"The next page is: {next_page_token}") - params.update(next_page_token) - return params - - def request_headers(self, **kwargs) -> Mapping[str, Any]: - return {"Accept": "application/json", "User-Agent": "PostmanRuntime/7.28.0", "Content-Type": "application/json"} - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - decoded_response = response.json() - meta_data = decoded_response.get("meta") - if meta_data: - total_pages = meta_data["total_pages"] - current_page = meta_data["current"] - if current_page < total_pages: - current_page += 1 - return {self.curr_page_param: current_page} - return None - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - response_json = response.json() - yield from response_json.get(self.data_field, []) - - @property - def max_retries(self) -> Union[int, None]: - return 10 - - -class APIIncrementalFreshcallerStream(FreshcallerStream): - """ - Base abstract class for a "true" incremental stream, i.e., for an endpoint that supports - filtering by date or time - """ - - start_param = "by_time[from]" - end_param = "by_time[to]" - - def __init__(self, config: Dict, **kwargs): - super().__init__(config, **kwargs) - self.config = config - self.start_date = config["start_date"] - self.window_in_days = config.get("window_in_days", 5) - self.sync_lag_minutes = config.get("sync_lag_minutes", 30) - - @property - @abstractmethod - def cursor_field(self) -> str: - """ - Defining a cursor field indicates that a stream is incremental, so any incremental stream must extend this class - and define a cursor field. - """ - - def get_updated_state( - self, - current_stream_state: MutableMapping[str, Any], - latest_record: Mapping[str, Any], - ) -> Mapping[str, Any]: - """ - Override default get_updated_state CDK method to return the latest state by comparing the cursor value in the latest record with the stream's most recent state object - and returning an updated state object. - """ - last_synced_at = current_stream_state.get(self.cursor_field, self.start_date) - last_synced_at = pendulum.parse(last_synced_at) if isinstance(last_synced_at, str) else last_synced_at - return {self.cursor_field: max(pendulum.parse(latest_record.get(self.cursor_field)).in_tz("UTC"), last_synced_at)} - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state, stream_slice, next_page_token) - params[self.start_param] = stream_slice[self.start_param] - params[self.end_param] = stream_slice[self.end_param] - self.logger.info(f"Endpoint[{self.path()}] - Request params: {params}") - return params - - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: - """ - Override default stream_slices CDK method to provide date_slices as page chunks for data fetch. - Returns list of dict, example: [{ - "by_time[from]": "2022-03-07 15:00:01", - "by_time[to]": "2022-03-07 18:00:00" - }, - { - "by_time[from]": "2022-03-07 18:00:01", - "by_time[to]": "2022-03-07 21:00:00" - }, - ...] - """ - start_date = pendulum.parse(self.start_date).in_timezone("UTC") - end_date = pendulum.now("UTC").subtract(minutes=self.sync_lag_minutes) # have a safe lag - - # Determine stream_state, if no stream_state we use start_date - if stream_state: - start_date = stream_state.get(self.cursor_field) - start_date = pendulum.parse(start_date) if isinstance(start_date, str) else start_date - start_date = start_date.in_tz("UTC") - # use the lowest date between start_date and self.end_date, otherwise API fails if start_date is in future - start_date: pendulum.DateTime = min(start_date, end_date) - date_slices = [] - - while start_date <= end_date: - end_date_slice = start_date.add(days=self.window_in_days) - # add 1 second for start next slice to not duplicate data from previous slice end date. - stream_slice = { - self.start_param: start_date.add(seconds=1).to_datetime_string(), - self.end_param: min(end_date_slice, end_date).to_datetime_string(), - } - date_slices.append(stream_slice) - start_date = end_date_slice - - return date_slices - - -class Users(FreshcallerStream): - """ - API docs: https://developers.freshcaller.com/api/#users - """ - - data_field = "users" - - def path(self, **kwargs) -> str: - return "users" - - -class Teams(FreshcallerStream): - """ - API docs: https://developers.freshcaller.com/api/#teams - """ - - data_field = "teams" - - def path(self, **kwargs) -> str: - return "teams" - - -class Calls(APIIncrementalFreshcallerStream): - """ - API docs: https://developers.freshcaller.com/api/#calls - """ - - data_field = "calls" - cursor_field = "created_time" - - def path(self, **kwargs) -> str: - return "calls" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params = {**super().request_params(stream_state, stream_slice, next_page_token), "has_ancestry": "true"} - return params - - -class CallMetrics(APIIncrementalFreshcallerStream): - """ - API docs: https://developers.freshcaller.com/api/#call-metrics - """ - - data_field = "call_metrics" - cursor_field = "created_time" - - def path(self, **kwargs) -> str: - return "call_metrics" diff --git a/airbyte-integrations/connectors/source-freshcaller/unit_tests/test_source.py b/airbyte-integrations/connectors/source-freshcaller/unit_tests/test_source.py deleted file mode 100644 index 05dce26412b9..000000000000 --- a/airbyte-integrations/connectors/source-freshcaller/unit_tests/test_source.py +++ /dev/null @@ -1,28 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pendulum -from source_freshcaller.source import FreshcallerTokenAuthenticator, SourceFreshcaller - -now_dt = pendulum.now() - - -def test_authenticator(requests_mock): - URL = "https://example.com/" - TOKEN = "test_token" - config = { - "domain": "https://example.com", - "api_key": "test_token", - } - requests_mock.post(URL, json={"token": TOKEN}) - a = FreshcallerTokenAuthenticator(config["api_key"]) - auth_headers = a.get_auth_header() - assert auth_headers["X-Api-Auth"] == TOKEN - - -def test_count_streams(mocker): - source = SourceFreshcaller() - config_mock = mocker.MagicMock() - streams = source.streams(config_mock) - assert len(streams) == 4 diff --git a/airbyte-integrations/connectors/source-freshdesk/Dockerfile b/airbyte-integrations/connectors/source-freshdesk/Dockerfile deleted file mode 100644 index 12a7dad4e74b..000000000000 --- a/airbyte-integrations/connectors/source-freshdesk/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_freshdesk ./source_freshdesk - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=3.0.4 -LABEL io.airbyte.name=airbyte/source-freshdesk diff --git a/airbyte-integrations/connectors/source-freshdesk/README.md b/airbyte-integrations/connectors/source-freshdesk/README.md index b7adde01d06d..b96f514b3d95 100644 --- a/airbyte-integrations/connectors/source-freshdesk/README.md +++ b/airbyte-integrations/connectors/source-freshdesk/README.md @@ -1,19 +1,19 @@ -# Freshdesk Source +# Freshdesk Source -This is the repository for the Freshdesk source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/freshdesk). +This is the repository for the Freshdesk source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/freshdesk). ## Local development ### Prerequisites **To iterate on this connector, make sure to complete this prerequisites section.** -#### Minimum Python version required `= 3.7.0` +#### Minimum Python version required `= 3.9.0` #### Build & Activate Virtual Environment and install dependencies From this connector directory, create a virtual environment: ``` -python3 -m venv .venv +python -m venv .venv ``` This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your @@ -29,22 +29,15 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-freshdesk:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/freshdesk) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_freshdesk/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/freshdesk) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_freshdesk/spec.yaml` file. Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. +See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source freshdesk test creds` and place them into `secrets/config.json`. - ### Locally running the connector ``` python main.py spec @@ -53,59 +46,104 @@ python main.py discover --config secrets/config.json python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json ``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` - ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-freshdesk:dev + + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name source-freshdesk build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-freshdesk:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations -You can also build the connector image via Gradle: +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-freshdesk:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-freshdesk:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-freshdesk:dev . +# Running the spec command against your patched connector +docker run airbyte/source-freshdesk:dev spec +``` #### Run Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-freshdesk:dev spec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-freshdesk:dev check --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-freshdesk:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-freshdesk:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-freshdesk:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-freshdesk:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. - -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](../../../docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md) for more information. -If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-freshdesk:dev \ -&& python -m pytest -p integration_tests.acceptance +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-freshdesk test ``` -To run your integration tests with docker +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-freshdesk test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/freshdesk.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-freshdesk/acceptance-test-config.yml b/airbyte-integrations/connectors/source-freshdesk/acceptance-test-config.yml index 7026134da21f..199aeae9461b 100644 --- a/airbyte-integrations/connectors/source-freshdesk/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-freshdesk/acceptance-test-config.yml @@ -5,41 +5,42 @@ test_strictness_level: high acceptance_tests: spec: tests: - - spec_path: "source_freshdesk/spec.json" + - spec_path: "source_freshdesk/spec.json" connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: tests: - - config_path: "secrets/config.json" - # please remove the backward_capability checks bypass, once updated to the newer version - backward_compatibility_tests_config: - disable_for_version: "1.0.0" + - config_path: + "secrets/config.json" + # please remove the backward_capability checks bypass, once updated to the newer version + backward_compatibility_tests_config: + disable_for_version: "1.0.0" basic_read: tests: - - config_path: "secrets/config.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: no - extra_records: yes - timeout_seconds: 600 - empty_streams: - - name: skills - bypass_reason: "no records" - - name: products - bypass_reason: "no records" - fail_on_extra_columns: false + - config_path: "secrets/config.json" + expect_records: + path: "integration_tests/expected_records.jsonl" + extra_fields: no + exact_order: no + extra_records: yes + timeout_seconds: 600 + empty_streams: + - name: skills + bypass_reason: "no records" + - name: products + bypass_reason: "no records" + fail_on_extra_columns: false incremental: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/incremental_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/incremental_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-freshdesk/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-freshdesk/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-freshdesk/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-freshdesk/build.gradle b/airbyte-integrations/connectors/source-freshdesk/build.gradle deleted file mode 100644 index 82e2a7cccc57..000000000000 --- a/airbyte-integrations/connectors/source-freshdesk/build.gradle +++ /dev/null @@ -1,10 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_freshdesk' -} - diff --git a/airbyte-integrations/connectors/source-freshdesk/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-freshdesk/integration_tests/expected_records.jsonl index a12306162437..05d5dc687b2a 100644 --- a/airbyte-integrations/connectors/source-freshdesk/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-freshdesk/integration_tests/expected_records.jsonl @@ -1,10 +1,10 @@ -{"stream": "agents", "data": {"available": false, "occasional": true, "id": 67013469930, "ticket_scope": 1, "created_at": "2020-10-22T02:37:15Z", "updated_at": "2020-10-22T02:37:15Z", "last_active_at": null, "available_since": null, "type": "support_agent", "contact": {"active": false, "email": "custserv@freshdesk.com", "job_title": null, "language": "en", "last_login_at": null, "mobile": null, "name": "Customer Service", "phone": null, "time_zone": "Magadan", "created_at": "2020-10-22T02:37:15Z", "updated_at": "2020-10-22T02:37:15Z"}, "signature": null}, "emitted_at": 1682938529795} -{"stream": "agents", "data": {"available": false, "occasional": false, "id": 67013469919, "ticket_scope": 1, "created_at": "2020-10-22T02:37:13Z", "updated_at": "2023-01-16T15:57:48Z", "last_active_at": "2023-01-16T15:57:48Z", "available_since": null, "type": "support_agent", "contact": {"active": true, "email": "integration-test@airbyte.io", "job_title": null, "language": "en", "last_login_at": "2022-12-29T13:41:44Z", "mobile": null, "name": "Team Airbyte", "phone": null, "time_zone": "Magadan", "created_at": "2020-10-22T02:37:13Z", "updated_at": "2020-10-22T03:30:13Z"}, "signature": null}, "emitted_at": 1682938529797} -{"stream": "business_hours", "data": {"id": 67000039282, "name": "Default", "is_default": true, "description": "Default Business Calendar", "time_zone": "Magadan", "created_at": "2020-10-22T02:37:13Z", "updated_at": "2020-10-22T03:27:48Z", "working_hours": {"monday": {"start_time": "8:00 am", "end_time": "5:00 pm"}, "tuesday": {"start_time": "8:00 am", "end_time": "5:00 pm"}, "wednesday": {"start_time": "8:00 am", "end_time": "5:00 pm"}, "thursday": {"start_time": "8:00 am", "end_time": "5:00 pm"}, "friday": {"start_time": "8:00 am", "end_time": "5:00 pm"}}}, "emitted_at": 1682938530749} +{"stream": "agents", "data": {"available": false, "occasional": false, "id": 67013469919, "ticket_scope": 1, "created_at": "2020-10-22T02:37:13Z", "updated_at": "2023-01-16T15:57:48Z", "last_active_at": "2023-01-16T15:57:48Z", "available_since": null, "type": "support_agent", "contact": {"active": true, "email": "integration-test@airbyte.io", "job_title": null, "language": "en", "last_login_at": "2022-12-29T13:41:44Z", "mobile": null, "name": "Team Airbyte", "phone": null, "time_zone": "Magadan", "created_at": "2020-10-22T02:37:13Z", "updated_at": "2020-10-22T03:30:13Z"}, "deactivated": false, "signature": null}, "emitted_at": 1701377411005} +{"stream": "agents", "data": {"available": false, "occasional": true, "id": 67021120644, "ticket_scope": 1, "created_at": "2021-02-11T22:20:50Z", "updated_at": "2021-12-01T00:09:07Z", "last_active_at": null, "available_since": null, "type": "support_agent", "contact": {"active": false, "email": "test_agent_1@test.com", "job_title": null, "language": "en", "last_login_at": null, "mobile": null, "name": "Test Agent 1", "phone": null, "time_zone": "Magadan", "created_at": "2021-02-11T22:20:50Z", "updated_at": "2021-02-11T22:20:50Z"}, "deactivated": false, "signature": "


      \n
      "}, "emitted_at": 1701377411006} +{"stream": "business_hours", "data": {"id": 67000039282, "name": "Default", "is_default": true, "description": "Default Business Calendar", "time_zone": "Magadan", "created_at": "2020-10-22T02:37:13Z", "updated_at": "2020-10-22T03:27:48Z", "working_hours": {"monday": {"start_time": "8:00 am", "end_time": "5:00 pm"}, "tuesday": {"start_time": "8:00 am", "end_time": "5:00 pm"}, "wednesday": {"start_time": "8:00 am", "end_time": "5:00 pm"}, "thursday": {"start_time": "8:00 am", "end_time": "5:00 pm"}, "friday": {"start_time": "8:00 am", "end_time": "5:00 pm"}}}, "emitted_at": 1701377411526} {"stream": "canned_response_folders", "data": {"id": 67000078780, "name": "Personal", "created_at": "2020-10-22T02:37:14Z", "updated_at": "2020-10-22T02:37:14Z", "personal": true, "responses_count": 1}, "emitted_at": 1682938531688} {"stream": "canned_response_folders", "data": {"id": 67000174510, "name": "Test Folder 2", "created_at": "2022-04-27T02:19:10Z", "updated_at": "2022-04-27T02:19:10Z", "personal": false, "responses_count": 1}, "emitted_at": 1682938531689} -{"stream": "canned_responses", "data": {"id": 67000143821, "title": "Another canned response", "folder_id": 67000078780, "content": "This is another sample canned response", "content_html": "
      This is another sample canned response
      ", "attachments": [], "created_at": "2022-04-27T03:04:45Z", "updated_at": "2022-04-27T03:04:45Z", "group_ids": [], "visibility": 1}, "emitted_at": 1682938533427} -{"stream": "canned_responses", "data": {"id": 67000143819, "title": "Sample canned response", "folder_id": 67000174510, "content": "This is a sample canned response", "content_html": "
      This is a sample canned response
      ", "attachments": [], "created_at": "2022-04-27T02:25:33Z", "updated_at": "2022-04-27T02:25:33Z", "group_ids": [], "visibility": 0}, "emitted_at": 1682938533665} +{"stream": "canned_responses", "data": {"id": 67000143821, "title": "Another canned response", "folder_id": 67000078780, "content": "This is another sample canned response", "content_html": "
      This is another sample canned response
      ", "attachments": [], "created_at": "2022-04-27T03:04:45Z", "updated_at": "2022-04-27T03:04:45Z", "group_ids": [], "visibility": 1}, "emitted_at": 1701377413382} +{"stream": "canned_responses", "data": {"id": 67000143819, "title": "Sample canned response", "folder_id": 67000174510, "content": "This is a sample canned response", "content_html": "
      This is a sample canned response
      ", "attachments": [], "created_at": "2022-04-27T02:25:33Z", "updated_at": "2022-04-27T02:25:33Z", "group_ids": [], "visibility": 0}, "emitted_at": 1701377413951} {"stream": "companies", "data": {"id": 67000893975, "name": "Abbott Group", "description": null, "note": null, "domains": [], "created_at": "2021-11-16T14:48:31Z", "updated_at": "2021-11-16T14:48:31Z", "custom_fields": {}, "health_score": null, "account_tier": null, "renewal_date": null, "industry": null}, "emitted_at": 1682938535264} {"stream": "companies", "data": {"id": 67000888235, "name": "Abbott Inc", "description": null, "note": null, "domains": [], "created_at": "2021-11-16T14:17:49Z", "updated_at": "2021-11-16T14:17:49Z", "custom_fields": {}, "health_score": null, "account_tier": null, "renewal_date": null, "industry": null}, "emitted_at": 1682938535265} {"stream": "companies", "data": {"id": 67000881973, "name": "Abbott-Davis", "description": null, "note": null, "domains": [], "created_at": "2021-11-16T13:35:18Z", "updated_at": "2021-11-16T13:35:18Z", "custom_fields": {}, "health_score": null, "account_tier": null, "renewal_date": null, "industry": null}, "emitted_at": 1682938535265} @@ -54,10 +54,10 @@ {"stream": "tickets", "data": {"cc_emails": null, "fwd_emails": null, "reply_cc_emails": null, "ticket_cc_emails": null, "fr_escalated": true, "spam": false, "email_config_id": null, "group_id": 67000259997, "priority": 2, "requester_id": 67038833126, "responder_id": null, "source": 1, "company_id": null, "status": 2, "subject": "Payment failed", "association_type": null, "support_email": null, "to_emails": null, "product_id": null, "id": 18, "type": "Question", "due_by": "2022-05-26T21:25:35Z", "fr_due_by": "2022-05-26T05:25:35Z", "is_escalated": true, "custom_fields": {}, "created_at": "2020-10-22T02:37:13Z", "updated_at": "2022-05-26T21:26:52Z", "associated_tickets_count": null, "tags": [], "description": "I was trying to make a payment on your site and got a \"Your payment failed\" error. However, my card was charged. Can you let me know if the order is processed or what I need to do? Please see the attached screenshot.

      Thanks,
      Matt", "description_text": "I was trying to make a payment on your site and got a \"Your payment failed\" error. However, my card was charged. Can you let me know if the order is processed or what I need to do? Please see the attached screenshot.\n\nThanks,\nMatt", "requester": {"id": 67038833126, "name": "Matt Rogers", "email": "matt.rogers@freshdesk.com", "mobile": null, "phone": "+61 1800 861 302"}, "stats": {"agent_responded_at": null, "requester_responded_at": null, "first_responded_at": null, "status_updated_at": "2020-10-22T02:37:13Z", "reopened_at": null, "resolved_at": null, "closed_at": null, "pending_since": null}, "nr_due_by": null, "nr_escalated": false}, "emitted_at": 1682938733367} {"stream": "tickets", "data": {"cc_emails": null, "fwd_emails": null, "reply_cc_emails": null, "ticket_cc_emails": null, "fr_escalated": true, "spam": false, "email_config_id": null, "group_id": 67000259997, "priority": 1, "requester_id": 67038833129, "responder_id": null, "source": 1, "company_id": null, "status": 2, "subject": "Received a broken TV", "association_type": null, "support_email": null, "to_emails": null, "product_id": null, "id": 19, "type": "Question", "due_by": "2022-05-28T21:25:35Z", "fr_due_by": "2022-05-26T21:25:35Z", "is_escalated": true, "custom_fields": {}, "created_at": "2020-10-22T02:37:13Z", "updated_at": "2022-05-28T21:26:13Z", "associated_tickets_count": null, "tags": [], "description": "Hi,

      The television I ordered from your site was delivered with a cracked screen. I need some help with a refund or a replacement.

      Here is the order number FD07062010

      Thanks,
      Sarah", "description_text": "Hi,\n\nThe television I ordered from your site was delivered with a cracked screen. I need some help with a refund or a replacement.\n\nHere is the order number FD07062010\n\nThanks,\nSarah", "requester": {"id": 67038833129, "name": "Sarah James", "email": "sarah.james@freshdesk.com", "mobile": null, "phone": "+1 (855) 747 676"}, "stats": {"agent_responded_at": null, "requester_responded_at": null, "first_responded_at": null, "status_updated_at": "2020-10-22T02:37:13Z", "reopened_at": null, "resolved_at": null, "closed_at": null, "pending_since": null}, "nr_due_by": null, "nr_escalated": false}, "emitted_at": 1682938733370} {"stream": "tickets", "data": {"cc_emails": [], "fwd_emails": [], "reply_cc_emails": [], "ticket_cc_emails": [], "fr_escalated": false, "spam": false, "email_config_id": 67000041667, "group_id": null, "priority": 1, "requester_id": 67042965499, "responder_id": 67013469919, "source": 10, "company_id": 67001265522, "status": 5, "subject": "Test", "association_type": null, "support_email": "support@newaccount1603334233301.freshdesk.com", "to_emails": ["support@newaccount1603334233301.freshdesk.com"], "product_id": null, "id": 21, "type": null, "due_by": "2022-11-19T15:59:28Z", "fr_due_by": "2022-11-17T15:59:28Z", "is_escalated": false, "custom_fields": {}, "created_at": "2022-11-16T15:59:28Z", "updated_at": "2022-11-16T15:59:28Z", "associated_tickets_count": null, "tags": [], "description": "
      Test
      ", "description_text": "Test", "requester": {"id": 67042965499, "name": "Iryna Grankova", "email": "iryna.grankova@airbyte.io", "mobile": "+380636306253", "phone": null}, "stats": {"agent_responded_at": null, "requester_responded_at": null, "first_responded_at": null, "status_updated_at": "2022-11-16T15:59:28Z", "reopened_at": null, "resolved_at": "2022-11-16T15:59:28Z", "closed_at": "2022-11-16T15:59:28Z", "pending_since": null}, "nr_due_by": null, "nr_escalated": false}, "emitted_at": 1682938733374} -{"stream": "conversations", "data": {"body": "
      Hi Jessica Rangel,

      Test reply

      \n
      ", "body_text": "Hi Jessica Rangel, Test reply", "id": 67105086049, "incoming": false, "private": false, "user_id": 67013469919, "support_email": "support@newaccount1603334233301.freshdesk.com", "source": 0, "category": 3, "to_emails": ["reyesamanda@example.com"], "from_email": "Airbyte ", "cc_emails": [], "bcc_emails": [], "email_failure_count": 1, "outgoing_failures": null, "thread_id": null, "thread_message_id": null, "created_at": "2022-11-16T16:16:58Z", "updated_at": "2022-11-16T16:16:58Z", "last_edited_at": null, "last_edited_user_id": null, "attachments": [], "automation_id": null, "automation_type_id": null, "auto_response": false, "ticket_id": 22, "source_additional_info": null}, "emitted_at": 1682938735052} -{"stream": "conversations", "data": {"body": "
      Test note
      ", "body_text": "Test note", "id": 67105086068, "incoming": false, "private": true, "user_id": 67013469919, "support_email": null, "source": 2, "category": 2, "to_emails": [], "from_email": null, "cc_emails": [], "bcc_emails": null, "email_failure_count": null, "outgoing_failures": null, "thread_id": null, "thread_message_id": null, "created_at": "2022-11-16T16:17:05Z", "updated_at": "2022-11-16T16:17:05Z", "last_edited_at": null, "last_edited_user_id": null, "attachments": [], "automation_id": null, "automation_type_id": null, "auto_response": false, "ticket_id": 22, "source_additional_info": null}, "emitted_at": 1682938735053} -{"stream": "conversations", "data": {"body": "
      Hi Iryna Grankova,


      \n
      ", "body_text": "Hi Iryna Grankova,", "id": 67105779551, "incoming": false, "private": false, "user_id": 67013469919, "support_email": "support@newaccount1603334233301.freshdesk.com", "source": 0, "category": 3, "to_emails": ["iryna.grankova@airbyte.io"], "from_email": "Airbyte ", "cc_emails": [], "bcc_emails": [], "email_failure_count": null, "outgoing_failures": null, "thread_id": null, "thread_message_id": null, "created_at": "2022-11-22T15:02:12Z", "updated_at": "2022-11-22T15:02:12Z", "last_edited_at": null, "last_edited_user_id": null, "attachments": [], "automation_id": null, "automation_type_id": null, "auto_response": false, "ticket_id": 23, "source_additional_info": null}, "emitted_at": 1682938735485} -{"stream": "conversations", "data": {"body": "
      test
      ", "body_text": "test", "id": 67105779604, "incoming": true, "private": false, "user_id": 67042965499, "support_email": null, "source": 6, "category": 7, "to_emails": null, "from_email": null, "cc_emails": [], "bcc_emails": null, "email_failure_count": null, "outgoing_failures": null, "thread_id": null, "thread_message_id": null, "created_at": "2022-11-22T15:02:29Z", "updated_at": "2022-11-22T15:02:29Z", "last_edited_at": null, "last_edited_user_id": null, "attachments": [], "automation_id": null, "automation_type_id": null, "auto_response": false, "ticket_id": 23, "source_additional_info": null}, "emitted_at": 1682938735486} +{"stream": "conversations", "data": {"body": "
      Hi Jessica Rangel,

      Test reply

      \n
      ", "body_text": "Hi Jessica Rangel, Test reply", "id": 67105086049, "incoming": false, "private": false, "user_id": 67013469919, "support_email": "support@newaccount1603334233301.freshdesk.com", "source": 0, "category": 3, "to_emails": ["reyesamanda@example.com"], "from_email": "Airbyte ", "cc_emails": [], "bcc_emails": [], "email_failure_count": 1, "outgoing_failures": null, "thread_id": null, "thread_message_id": null, "created_at": "2022-11-16T16:16:58Z", "updated_at": "2022-11-16T16:16:58Z", "last_edited_at": null, "last_edited_user_id": null, "attachments": [], "automation_id": null, "automation_type_id": null, "auto_response": false, "ticket_id": 22, "source_additional_info": null}, "emitted_at": 1701379065881} +{"stream": "conversations", "data": {"body": "
      Test note
      ", "body_text": "Test note", "id": 67105086068, "incoming": false, "private": true, "user_id": 67013469919, "support_email": null, "source": 2, "category": 2, "to_emails": [], "from_email": null, "cc_emails": [], "bcc_emails": null, "email_failure_count": null, "outgoing_failures": null, "thread_id": null, "thread_message_id": null, "created_at": "2022-11-16T16:17:05Z", "updated_at": "2022-11-16T16:17:05Z", "last_edited_at": null, "last_edited_user_id": null, "attachments": [], "automation_id": null, "automation_type_id": null, "auto_response": false, "ticket_id": 22, "source_additional_info": null}, "emitted_at": 1701379065883} +{"stream": "conversations", "data": {"body": "
      Hi Iryna Grankova,


      \n
      ", "body_text": "Hi Iryna Grankova,", "id": 67105779551, "incoming": false, "private": false, "user_id": 67013469919, "support_email": "support@newaccount1603334233301.freshdesk.com", "source": 0, "category": 3, "to_emails": ["iryna.grankova@airbyte.io"], "from_email": "Airbyte ", "cc_emails": [], "bcc_emails": [], "email_failure_count": null, "outgoing_failures": null, "thread_id": null, "thread_message_id": null, "created_at": "2022-11-22T15:02:12Z", "updated_at": "2022-11-22T15:02:12Z", "last_edited_at": null, "last_edited_user_id": null, "attachments": [], "automation_id": null, "automation_type_id": null, "auto_response": false, "ticket_id": 23, "source_additional_info": null}, "emitted_at": 1701379066147} +{"stream": "conversations", "data": {"body": "
      test
      ", "body_text": "test", "id": 67105779604, "incoming": true, "private": false, "user_id": 67042965499, "support_email": null, "source": 6, "category": 7, "to_emails": [], "from_email": null, "cc_emails": [], "bcc_emails": null, "email_failure_count": null, "outgoing_failures": null, "thread_id": null, "thread_message_id": null, "created_at": "2022-11-22T15:02:29Z", "updated_at": "2022-11-22T15:02:29Z", "last_edited_at": null, "last_edited_user_id": null, "attachments": [], "automation_id": null, "automation_type_id": null, "auto_response": false, "ticket_id": 23, "source_additional_info": null}, "emitted_at": 1701379066148} {"stream": "satisfaction_ratings", "data": {"id": 67000499229, "survey_id": 67000084490, "user_id": 67042965499, "agent_id": 67013469919, "feedback": "test", "group_id": null, "ticket_id": 23, "created_at": "2022-11-22T15:02:29Z", "updated_at": "2022-11-22T15:02:29Z", "ratings": {"default_question": 103, "question_67000084816": 103, "question_67000084817": 100, "question_67000084818": 100}}, "emitted_at": 1682938736407} {"stream": "surveys", "data": {"id": 67000039187, "title": "Default Survey", "active": false, "created_at": "2020-10-22T02:37:14Z", "updated_at": "2022-11-16T16:24:54Z", "questions": [{"id": "default_question", "label": "How would you rate your overall satisfaction for the resolution provided by the agent?", "accepted_ratings": [103, 100, -103], "default": true}]}, "emitted_at": 1682938737363} {"stream": "surveys", "data": {"id": 67000084490, "title": "Test survey", "active": true, "created_at": "2022-11-16T16:24:00Z", "updated_at": "2022-11-22T15:00:30Z", "questions": [{"id": "default_question", "label": " How would you rate your overall satisfaction for the resolution provided by the agent? ", "accepted_ratings": [103, 100, -103], "default": true}, {"id": "question_67000084816", "label": "Question 1?", "accepted_ratings": [103, 100, -103]}, {"id": "question_67000084817", "label": "Question 2?", "accepted_ratings": [103, 100, -103]}, {"id": "question_67000084818", "label": "Are you satisfied with our customer support experience?", "accepted_ratings": [103, 100, -103]}]}, "emitted_at": 1682938737365} diff --git a/airbyte-integrations/connectors/source-freshdesk/main.py b/airbyte-integrations/connectors/source-freshdesk/main.py index 319505ff4bb5..d32eaa6ca9e5 100644 --- a/airbyte-integrations/connectors/source-freshdesk/main.py +++ b/airbyte-integrations/connectors/source-freshdesk/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_freshdesk import SourceFreshdesk +from source_freshdesk.run import run if __name__ == "__main__": - source = SourceFreshdesk() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-freshdesk/metadata.yaml b/airbyte-integrations/connectors/source-freshdesk/metadata.yaml index 8a16f346e628..e532adfce2a5 100644 --- a/airbyte-integrations/connectors/source-freshdesk/metadata.yaml +++ b/airbyte-integrations/connectors/source-freshdesk/metadata.yaml @@ -1,12 +1,18 @@ data: + ab_internal: + ql: 200 + sl: 200 allowedHosts: hosts: - "*.freshdesk.com" + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: ec4b9503-13cb-48ab-a4ab-6ade4be46567 - dockerImageTag: 3.0.4 + dockerImageTag: 3.0.6 dockerRepository: airbyte/source-freshdesk + documentationUrl: https://docs.airbyte.com/integrations/sources/freshdesk githubIssueLabel: source-freshdesk icon: freshdesk.svg license: MIT @@ -17,11 +23,7 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/freshdesk + supportLevel: certified tags: - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-freshdesk/setup.py b/airbyte-integrations/connectors/source-freshdesk/setup.py index c0d8b408f781..b9cabbadddfc 100644 --- a/airbyte-integrations/connectors/source-freshdesk/setup.py +++ b/airbyte-integrations/connectors/source-freshdesk/setup.py @@ -15,6 +15,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-freshdesk=source_freshdesk.run:run", + ], + }, name="source_freshdesk", description="Source implementation for Freshdesk.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/run.py b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/run.py new file mode 100644 index 000000000000..5486a3c15061 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshdesk/source_freshdesk/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_freshdesk import SourceFreshdesk + + +def run(): + source = SourceFreshdesk() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_source.py b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_source.py index ce3a20a113ca..15cece664b68 100644 --- a/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-freshdesk/unit_tests/test_source.py @@ -26,7 +26,9 @@ def test_check_connection_invalid_api_key(requests_mock, config): requests_mock.register_uri("GET", "/api/v2/settings/helpdesk", responses) ok, error_msg = SourceFreshdesk().check_connection(logger, config=config) assert not ok - assert "The endpoint to access stream \'settings\' returned 401: Unauthorized. This is most likely due to wrong credentials. " in error_msg + assert ( + "The endpoint to access stream 'settings' returned 401: Unauthorized. This is most likely due to wrong credentials. " in error_msg + ) assert "You have to be logged in to perform this action." in error_msg diff --git a/airbyte-integrations/connectors/source-freshsales/Dockerfile b/airbyte-integrations/connectors/source-freshsales/Dockerfile index 1c46733029af..07b190ca4b88 100644 --- a/airbyte-integrations/connectors/source-freshsales/Dockerfile +++ b/airbyte-integrations/connectors/source-freshsales/Dockerfile @@ -34,5 +34,5 @@ COPY source_freshsales ./source_freshsales ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=1.0.0 LABEL io.airbyte.name=airbyte/source-freshsales diff --git a/airbyte-integrations/connectors/source-freshsales/README.md b/airbyte-integrations/connectors/source-freshsales/README.md index 48215c2de2aa..c4f2450e08e3 100644 --- a/airbyte-integrations/connectors/source-freshsales/README.md +++ b/airbyte-integrations/connectors/source-freshsales/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-freshsales:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/freshsales) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_freshsales/spec.json` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-freshsales:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-freshsales build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-freshsales:airbyteDocker +An image will be built with the tag `airbyte/source-freshsales:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-freshsales:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,45 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-freshsales:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-freshsales:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-freshsales:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-freshsales test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-freshsales:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-freshsales:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-freshsales:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -126,8 +90,10 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-freshsales test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/freshsales.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-freshsales/__int__.py b/airbyte-integrations/connectors/source-freshsales/__int__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshsales/__int__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-freshsales/acceptance-test-config.yml b/airbyte-integrations/connectors/source-freshsales/acceptance-test-config.yml index 8f382499e126..cefc9ad1fffa 100644 --- a/airbyte-integrations/connectors/source-freshsales/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-freshsales/acceptance-test-config.yml @@ -5,7 +5,7 @@ test_strictness_level: high acceptance_tests: spec: tests: - - spec_path: "source_freshsales/spec.json" + - spec_path: "source_freshsales/spec.yaml" connection: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-freshsales/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-freshsales/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-freshsales/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-freshsales/build.gradle b/airbyte-integrations/connectors/source-freshsales/build.gradle deleted file mode 100644 index 63f3c0b66a70..000000000000 --- a/airbyte-integrations/connectors/source-freshsales/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_freshsales' -} diff --git a/airbyte-integrations/connectors/source-freshsales/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-freshsales/integration_tests/expected_records.jsonl index 552ba6288536..c91403d5fb19 100644 --- a/airbyte-integrations/connectors/source-freshsales/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-freshsales/integration_tests/expected_records.jsonl @@ -1,39 +1,39 @@ -{"stream": "contacts", "data": {"id": 17008318589, "first_name": "Mail Delivery", "last_name": "Subsystem", "display_name": "Mail Delivery Subsystem", "avatar": null, "job_title": null, "city": null, "state": null, "zipcode": null, "country": null, "email": "mailer-daemon@googlemail.com", "emails": "[{'id': 17007443529, 'value': 'mailer-daemon@googlemail.com', 'is_primary': True, 'label': None, '_destroy': False}]", "time_zone": null, "work_number": null, "mobile_number": null, "address": null, "last_seen": null, "lead_score": 14, "last_contacted": null, "open_deals_amount": 7700.0, "won_deals_amount": 0.0, "links": {"conversations": "/crm/sales/contacts/17008318589/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "timeline_feeds": "/crm/sales/contacts/17008318589/timeline_feeds", "document_associations": "/crm/sales/contacts/17008318589/document_associations", "notes": "/crm/sales/contacts/17008318589/notes?include=creater", "tasks": "/crm/sales/contacts/17008318589/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/contacts/17008318589/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note", "reminders": "/crm/sales/contacts/17008318589/reminders?include=creater,owner,updater,targetable", "duplicates": "/crm/sales/contacts/17008318589/duplicates", "connections": "/crm/sales/contacts/17008318589/connections"}, "last_contacted_sales_activity_mode": null, "custom_field": {}, "created_at": "2021-10-19T00:28:18-06:00", "updated_at": "2023-03-23T05:19:22-06:00", "keyword": null, "medium": null, "last_contacted_mode": null, "recent_note": null, "won_deals_count": 0, "last_contacted_via_sales_activity": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_ids": null, "open_deals_count": 2, "last_assigned_at": "2021-10-19T00:28:19-06:00", "facebook": null, "twitter": null, "linkedin": null, "is_deleted": false, "team_user_ids": null, "external_id": null, "work_email": null, "subscription_status": "1", "subscription_types": "1;2;3;4;5", "unsubscription_reason": null, "other_unsubscription_reason": null, "customer_fit": 1, "whatsapp_subscription_status": 2, "sms_subscription_status": "2", "last_seen_chat": null, "first_seen_chat": null, "locale": null, "total_sessions": null, "system_tags": "[]", "first_campaign": null, "first_medium": null, "first_source": null, "last_campaign": null, "last_medium": null, "last_source": null, "latest_campaign": null, "latest_medium": null, "latest_source": null, "mcr_id": "1450348061427834880", "phone_numbers": [], "tags": []}, "emitted_at": 1682419136691} -{"stream": "contacts", "data": {"id": 17008066468, "first_name": "Jane", "last_name": "Sampleton (sample)", "display_name": "Jane Sampleton (sample)", "avatar": "https://img.fullcontact.com/static/4df0efb1ea1a7650fef74f5e44d50d35_ca437b79617f8bbfc40c317b729d32693be1463f356b5be1015b39739859659f", "job_title": "Sales Manager", "city": "Glendale", "state": "Arizona", "zipcode": "100652", "country": "USA", "email": "janesampleton@gmail.com", "emails": "[{'id': 17007194356, 'value': 'janesampleton@gmail.com', 'is_primary': True, 'label': None, '_destroy': False}]", "time_zone": "Arizona", "work_number": "3684932360", "mobile_number": "19266529503", "address": "604-5854 Beckford St.", "last_seen": null, "lead_score": 36, "last_contacted": "2021-10-12T10:06:38-06:00", "open_deals_amount": 22780.0, "won_deals_amount": 11000.0, "links": {"conversations": "/crm/sales/contacts/17008066468/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "timeline_feeds": "/crm/sales/contacts/17008066468/timeline_feeds", "document_associations": "/crm/sales/contacts/17008066468/document_associations", "notes": "/crm/sales/contacts/17008066468/notes?include=creater", "tasks": "/crm/sales/contacts/17008066468/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/contacts/17008066468/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note", "reminders": "/crm/sales/contacts/17008066468/reminders?include=creater,owner,updater,targetable", "duplicates": "/crm/sales/contacts/17008066468/duplicates", "connections": "/crm/sales/contacts/17008066468/connections"}, "last_contacted_sales_activity_mode": "Task", "custom_field": {}, "created_at": "2021-10-07T10:06:37-06:00", "updated_at": "2023-03-23T05:26:56-06:00", "keyword": "B2B Success", "medium": "Blog", "last_contacted_mode": "Email Opened", "recent_note": "Sample note for contact create", "won_deals_count": 4, "last_contacted_via_sales_activity": "2021-10-19T00:27:33-06:00", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_ids": null, "open_deals_count": 5, "last_assigned_at": "2022-09-05T04:44:49-06:00", "facebook": "100010587455650", "twitter": "janesampleton", "linkedin": "jane-sampleton-0b0039109", "is_deleted": false, "team_user_ids": null, "external_id": null, "work_email": null, "subscription_status": "1", "subscription_types": "1;2;3;4;5", "unsubscription_reason": null, "other_unsubscription_reason": null, "customer_fit": 2, "whatsapp_subscription_status": 2, "sms_subscription_status": "2", "last_seen_chat": null, "first_seen_chat": null, "locale": null, "total_sessions": null, "system_tags": "[]", "first_campaign": null, "first_medium": null, "first_source": null, "last_campaign": null, "last_medium": null, "last_source": null, "latest_campaign": null, "latest_medium": null, "latest_source": null, "mcr_id": "1450049339590340608", "phone_numbers": [], "tags": []}, "emitted_at": 1682419136692} -{"stream": "accounts", "data": {"id": 17001321830, "name": "Widgetz.io (sample)", "address": "160-6802 Aliquet Rd.", "city": "New Haven", "state": "Connecticut", "zipcode": "68089", "country": "United States", "number_of_employees": null, "annual_revenue": 0.0, "website": "widgetz.io", "owner_id": null, "phone": "5036153947", "open_deals_amount": 0.0, "open_deals_count": 0, "won_deals_amount": 0.0, "won_deals_count": 0, "last_contacted": "2021-10-12T10:06:38-06:00", "last_contacted_mode": "Email Opened", "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17001321830/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17001321830/document_associations", "notes": "/crm/sales/sales_accounts/17001321830/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17001321830/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17001321830/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2021-10-18T04:41:17-06:00", "updated_at": "2022-06-23T04:25:57-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": null, "last_contacted_via_sales_activity": "2021-10-19T00:27:33-06:00", "last_contacted_sales_activity_mode": "Task", "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2021-10-18T09:56:31-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1682419139166} -{"stream": "accounts", "data": {"id": 17001391875, "name": "Airbyte", "address": "San Francisco, CA", "city": null, "state": null, "zipcode": "94121", "country": null, "number_of_employees": 1, "annual_revenue": null, "website": null, "owner_id": 17000038922, "phone": "+1234567890", "open_deals_amount": 25.0, "open_deals_count": 1, "won_deals_amount": 0.0, "won_deals_count": 0, "last_contacted": "2021-10-19T05:04:54-06:00", "last_contacted_mode": "Call Outgoing", "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17001391875/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17001391875/document_associations", "notes": "/crm/sales/sales_accounts/17001391875/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17001391875/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17001391875/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2021-10-19T04:57:42-06:00", "updated_at": "2022-06-23T04:25:57-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": "activate 10/10/2021", "last_contacted_via_sales_activity": "2021-10-19T05:04:54-06:00", "last_contacted_sales_activity_mode": "Phone", "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2021-10-19T04:57:43-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1682419139167} -{"stream": "accounts", "data": {"id": 17004983219, "name": "Test Account 2", "address": null, "city": null, "state": null, "zipcode": null, "country": null, "number_of_employees": null, "annual_revenue": null, "website": null, "owner_id": 17000038922, "phone": null, "open_deals_amount": 0.0, "open_deals_count": 0, "won_deals_amount": 5000.0, "won_deals_count": 2, "last_contacted": null, "last_contacted_mode": null, "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17004983219/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17004983219/document_associations", "notes": "/crm/sales/sales_accounts/17004983219/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17004983219/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17004983219/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2023-03-23T05:14:37-06:00", "updated_at": "2023-03-23T05:25:59-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": null, "last_contacted_via_sales_activity": null, "last_contacted_sales_activity_mode": null, "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2023-03-23T05:14:38-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1682419139168} -{"stream": "accounts", "data": {"id": 17004983220, "name": "Test Account 3", "address": null, "city": null, "state": null, "zipcode": null, "country": null, "number_of_employees": null, "annual_revenue": null, "website": null, "owner_id": 17000038922, "phone": null, "open_deals_amount": 3200.0, "open_deals_count": 1, "won_deals_amount": 0.0, "won_deals_count": 0, "last_contacted": null, "last_contacted_mode": null, "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17004983220/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17004983220/document_associations", "notes": "/crm/sales/sales_accounts/17004983220/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17004983220/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17004983220/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2023-03-23T05:19:11-06:00", "updated_at": "2023-03-23T05:26:29-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": null, "last_contacted_via_sales_activity": null, "last_contacted_sales_activity_mode": null, "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2023-03-23T05:19:12-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1682419139169} -{"stream": "accounts", "data": {"id": 17004983221, "name": "Test Account 4", "address": null, "city": null, "state": null, "zipcode": null, "country": null, "number_of_employees": null, "annual_revenue": null, "website": null, "owner_id": 17000038922, "phone": null, "open_deals_amount": 8080.0, "open_deals_count": 2, "won_deals_amount": 0.0, "won_deals_count": 0, "last_contacted": null, "last_contacted_mode": null, "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17004983221/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17004983221/document_associations", "notes": "/crm/sales/sales_accounts/17004983221/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17004983221/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17004983221/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2023-03-23T05:19:46-06:00", "updated_at": "2023-03-23T05:26:56-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": null, "last_contacted_via_sales_activity": null, "last_contacted_sales_activity_mode": null, "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2023-03-23T05:19:47-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1682419139169} -{"stream": "accounts", "data": {"id": 17004983218, "name": "Test Account 1", "address": null, "city": null, "state": null, "zipcode": null, "country": null, "number_of_employees": null, "annual_revenue": null, "website": null, "owner_id": 17000038922, "phone": null, "open_deals_amount": 4500.0, "open_deals_count": 1, "won_deals_amount": 6000.0, "won_deals_count": 2, "last_contacted": null, "last_contacted_mode": null, "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17004983218/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17004983218/document_associations", "notes": "/crm/sales/sales_accounts/17004983218/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17004983218/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17004983218/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2023-03-23T05:13:41-06:00", "updated_at": "2023-03-23T05:31:35-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": null, "last_contacted_via_sales_activity": "2023-03-07T11:00:00-06:00", "last_contacted_sales_activity_mode": "Test chat", "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2023-03-23T05:13:42-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1682419139170} -{"stream": "open_deals", "data": {"id": 17000512184, "name": "Gold plan (sample)", "amount": 7000.0, "base_currency_amount": 7000.0, "expected_close": "2021-10-21", "closed_date": null, "stage_updated_time": "2021-10-13T10:06:38-06:00", "custom_field": {}, "probability": 100, "updated_at": "2021-10-14T10:06:38-06:00", "created_at": "2021-10-09T10:06:37-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237830, "age": 563, "links": {"conversations": "/crm/sales/deals/17000512184/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17000512184/document_associations", "notes": "/crm/sales/deals/17000512184/notes?include=creater", "tasks": "/crm/sales/deals/17000512184/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17000512184/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2021-10-09T10:06:37-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 7000.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": 2, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 533, "tags": []}, "emitted_at": 1682419141504} -{"stream": "open_deals", "data": {"id": 17000521380, "name": "Discaunt", "amount": 25.0, "base_currency_amount": 25.0, "expected_close": null, "closed_date": null, "stage_updated_time": "2021-10-19T05:10:53-06:00", "custom_field": {}, "probability": 100, "updated_at": "2021-10-19T05:10:53-06:00", "created_at": "2021-10-19T05:04:09-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237831, "age": 553, "links": {"conversations": "/crm/sales/deals/17000521380/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17000521380/document_associations", "notes": "/crm/sales/deals/17000521380/notes?include=creater", "tasks": "/crm/sales/deals/17000521380/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17000521380/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2021-10-19T05:04:10-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 25.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 523, "tags": []}, "emitted_at": 1682419141505} -{"stream": "open_deals", "data": {"id": 17015628751, "name": "Test Open Deal 1", "amount": 4500.0, "base_currency_amount": 4500.0, "expected_close": null, "closed_date": null, "stage_updated_time": "2023-03-23T05:18:44-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:18:44-06:00", "created_at": "2023-03-23T05:18:44-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237830, "age": 33, "links": {"conversations": "/crm/sales/deals/17015628751/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628751/document_associations", "notes": "/crm/sales/deals/17015628751/notes?include=creater", "tasks": "/crm/sales/deals/17015628751/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628751/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:18:45-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 4500.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 3, "tags": []}, "emitted_at": 1682419141506} -{"stream": "open_deals", "data": {"id": 17015628806, "name": "Test Open Deal 2", "amount": 3200.0, "base_currency_amount": 3200.0, "expected_close": null, "closed_date": null, "stage_updated_time": "2023-03-23T05:19:22-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:19:22-06:00", "created_at": "2023-03-23T05:19:22-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237830, "age": 33, "links": {"conversations": "/crm/sales/deals/17015628806/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628806/document_associations", "notes": "/crm/sales/deals/17015628806/notes?include=creater", "tasks": "/crm/sales/deals/17015628806/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628806/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:19:23-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 3200.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 3, "tags": []}, "emitted_at": 1682419141507} -{"stream": "open_deals", "data": {"id": 17015628849, "name": "Test Open Deal 3", "amount": 1580.0, "base_currency_amount": 1580.0, "expected_close": null, "closed_date": null, "stage_updated_time": "2023-03-23T05:19:51-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:19:51-06:00", "created_at": "2023-03-23T05:19:51-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237830, "age": 33, "links": {"conversations": "/crm/sales/deals/17015628849/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628849/document_associations", "notes": "/crm/sales/deals/17015628849/notes?include=creater", "tasks": "/crm/sales/deals/17015628849/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628849/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:19:52-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 1580.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 3, "tags": []}, "emitted_at": 1682419141508} -{"stream": "open_deals", "data": {"id": 17015628886, "name": "Test Open Deal 4", "amount": 6500.0, "base_currency_amount": 6500.0, "expected_close": null, "closed_date": null, "stage_updated_time": "2023-03-23T05:20:17-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:20:17-06:00", "created_at": "2023-03-23T05:20:17-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237830, "age": 33, "links": {"conversations": "/crm/sales/deals/17015628886/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628886/document_associations", "notes": "/crm/sales/deals/17015628886/notes?include=creater", "tasks": "/crm/sales/deals/17015628886/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628886/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:20:18-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 6500.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 3, "tags": []}, "emitted_at": 1682419141509} -{"stream": "won_deals", "data": {"id": 17015628496, "name": "Test Won Deal 4", "amount": 5000.0, "base_currency_amount": 5000.0, "expected_close": null, "closed_date": "2023-03-23", "stage_updated_time": "2023-03-23T05:16:34-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:16:36-06:00", "created_at": "2023-03-23T05:15:44-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237835, "age": 0, "links": {"conversations": "/crm/sales/deals/17015628496/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628496/document_associations", "notes": "/crm/sales/deals/17015628496/notes?include=creater", "tasks": "/crm/sales/deals/17015628496/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628496/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:15:45-06:00", "last_contacted_sales_activity_mode": "Test chat", "last_contacted_via_sales_activity": "2023-03-07T11:00:00-06:00", "expected_deal_value": 5000.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 6, "deal_prediction_last_updated_at": "2023-03-23T05:16:34-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1682419143931} -{"stream": "won_deals", "data": {"id": 17015628427, "name": "Test Won Deal 2", "amount": 3000.0, "base_currency_amount": 3000.0, "expected_close": null, "closed_date": "2023-03-23", "stage_updated_time": "2023-03-23T05:17:00-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:17:00-06:00", "created_at": "2023-03-23T05:14:55-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237835, "age": 0, "links": {"conversations": "/crm/sales/deals/17015628427/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628427/document_associations", "notes": "/crm/sales/deals/17015628427/notes?include=creater", "tasks": "/crm/sales/deals/17015628427/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628427/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:14:56-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 3000.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 6, "deal_prediction_last_updated_at": "2023-03-23T05:17:00-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1682419143932} -{"stream": "won_deals", "data": {"id": 17015628468, "name": "Test Won Deal 3", "amount": 2000.0, "base_currency_amount": 2000.0, "expected_close": null, "closed_date": "2023-03-23", "stage_updated_time": "2023-03-23T05:17:21-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:17:26-06:00", "created_at": "2023-03-23T05:15:23-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237835, "age": 0, "links": {"conversations": "/crm/sales/deals/17015628468/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628468/document_associations", "notes": "/crm/sales/deals/17015628468/notes?include=creater", "tasks": "/crm/sales/deals/17015628468/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628468/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:15:24-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 2000.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 6, "deal_prediction_last_updated_at": "2023-03-23T05:17:21-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1682419143933} -{"stream": "won_deals", "data": {"id": 17015628368, "name": "Test Won Deal 1", "amount": 1000.0, "base_currency_amount": 1000.0, "expected_close": null, "closed_date": "2023-03-14", "stage_updated_time": "2023-03-23T05:17:48-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:17:50-06:00", "created_at": "2023-03-23T05:14:16-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237835, "age": -9, "links": {"conversations": "/crm/sales/deals/17015628368/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628368/document_associations", "notes": "/crm/sales/deals/17015628368/notes?include=creater", "tasks": "/crm/sales/deals/17015628368/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628368/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:14:17-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 1000.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 6, "deal_prediction_last_updated_at": "2023-03-23T05:17:48-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1682419143934} -{"stream": "lost_deals", "data": {"id": 17015629024, "name": "Test Lost Deal 1", "amount": 800.0, "base_currency_amount": 800.0, "expected_close": null, "closed_date": "2023-03-17", "stage_updated_time": "2023-03-23T05:24:18-06:00", "custom_field": {}, "probability": 0, "updated_at": "2023-03-23T05:24:21-06:00", "created_at": "2023-03-23T05:21:53-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237836, "age": -6, "links": {"conversations": "/crm/sales/deals/17015629024/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015629024/document_associations", "notes": "/crm/sales/deals/17015629024/notes?include=creater", "tasks": "/crm/sales/deals/17015629024/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015629024/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:21:54-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 0.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 7, "deal_prediction_last_updated_at": "2023-03-23T05:24:18-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1682419146221} -{"stream": "lost_deals", "data": {"id": 17015629056, "name": "Test Lost Deal 2", "amount": 2800.0, "base_currency_amount": 2800.0, "expected_close": null, "closed_date": "2023-03-22", "stage_updated_time": "2023-03-23T05:25:59-06:00", "custom_field": {}, "probability": 0, "updated_at": "2023-03-23T05:26:01-06:00", "created_at": "2023-03-23T05:22:14-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237836, "age": -1, "links": {"conversations": "/crm/sales/deals/17015629056/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015629056/document_associations", "notes": "/crm/sales/deals/17015629056/notes?include=creater", "tasks": "/crm/sales/deals/17015629056/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015629056/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:22:15-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 0.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 7, "deal_prediction_last_updated_at": "2023-03-23T05:25:59-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1682419146222} -{"stream": "lost_deals", "data": {"id": 17015629086, "name": "Test Lost Deal 3", "amount": 3200.0, "base_currency_amount": 3200.0, "expected_close": null, "closed_date": "2023-03-09", "stage_updated_time": "2023-03-23T05:26:29-06:00", "custom_field": {}, "probability": 0, "updated_at": "2023-03-23T05:26:31-06:00", "created_at": "2023-03-23T05:22:33-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237836, "age": -14, "links": {"conversations": "/crm/sales/deals/17015629086/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015629086/document_associations", "notes": "/crm/sales/deals/17015629086/notes?include=creater", "tasks": "/crm/sales/deals/17015629086/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015629086/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:22:34-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 0.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 7, "deal_prediction_last_updated_at": "2023-03-23T05:26:29-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1682419146223} -{"stream": "lost_deals", "data": {"id": 17015629190, "name": "Test Lost Deal 4", "amount": 895.0, "base_currency_amount": 895.0, "expected_close": null, "closed_date": "2023-03-05", "stage_updated_time": "2023-03-23T05:26:55-06:00", "custom_field": {}, "probability": 0, "updated_at": "2023-03-23T05:26:57-06:00", "created_at": "2023-03-23T05:23:44-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237836, "age": -18, "links": {"conversations": "/crm/sales/deals/17015629190/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015629190/document_associations", "notes": "/crm/sales/deals/17015629190/notes?include=creater", "tasks": "/crm/sales/deals/17015629190/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015629190/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:23:45-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": 0.0, "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 7, "deal_prediction_last_updated_at": "2023-03-23T05:26:55-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1682419146223} -{"stream": "open_tasks", "data": {"id": 17000410092, "status": "0", "title": "All reports for meeting", "description": "All reports for meeting", "created_at": "2021-10-19T00:31:18-06:00", "updated_at": "2021-10-19T00:31:18-06:00", "owner_id": 17000038922, "due_date": "2021-10-20T00:30:00-06:00", "completed_date": null, "creater_id": 17000038922, "updater_id": 17000038922, "outcome_id": 17001154298, "task_type_id": 17000098963, "targetables": "[]"}, "emitted_at": 1682419147728} -{"stream": "open_tasks", "data": {"id": 17000407414, "status": "0", "title": "(Sample) Send the pricing quote", "description": "Coordinate with Steve for the pricing quote and send it to James.", "created_at": "2021-10-14T04:41:18-06:00", "updated_at": "2021-10-18T04:41:18-06:00", "owner_id": 17000038922, "due_date": "2021-10-20T03:00:00-06:00", "completed_date": null, "creater_id": 17000038922, "updater_id": 17000038922, "outcome_id": null, "task_type_id": null, "targetables": "[{'id': 17008066468, 'type': 'Contact'}]"}, "emitted_at": 1682419147729} -{"stream": "open_tasks", "data": {"id": 17000411384, "status": "0", "title": "Meeting", "description": "Meeting with Zazmic", "created_at": "2021-10-19T05:05:54-06:00", "updated_at": "2021-10-19T05:05:54-06:00", "owner_id": 17000038922, "due_date": "2021-10-29T05:30:00-06:00", "completed_date": null, "creater_id": 17000038922, "updater_id": 17000038922, "outcome_id": 17001154298, "task_type_id": 17000098963, "targetables": "[{'id': 17001391875, 'type': 'SalesAccount'}]"}, "emitted_at": 1682419147729} -{"stream": "open_tasks", "data": {"id": 17000408204, "status": "0", "title": "Sample Task", "description": "This is just a sample task.", "created_at": "2021-10-18T09:53:02-06:00", "updated_at": "2021-10-18T09:53:02-06:00", "owner_id": 17000038922, "due_date": "2022-06-21T05:00:00-06:00", "completed_date": null, "creater_id": 17000038922, "updater_id": 17000038922, "outcome_id": null, "task_type_id": null, "targetables": "[{'id': 17008066468, 'type': 'Contact'}]"}, "emitted_at": 1682419147730} -{"stream": "completed_tasks", "data": {"id": 17000407413, "status": "1", "title": "(Sample) Send the proposal document", "description": "Send the proposal document and follow up with this contact after it.", "created_at": "2021-10-14T04:41:18-06:00", "updated_at": "2021-10-19T00:27:33-06:00", "owner_id": 17000038922, "due_date": "2021-10-19T02:00:00-06:00", "completed_date": "2021-10-19T00:27:33-06:00", "creater_id": 17000038922, "updater_id": 17000038922, "outcome_id": 17001154300, "task_type_id": null, "targetables": "[{'id': 17008066468, 'type': 'Contact'}]"}, "emitted_at": 1682419148806} -{"stream": "past_appointments", "data": {"id": 17000384293, "time_zone": "Chennai", "title": "Sample Appointment", "description": "This is just a sample Appointment.", "location": "Chennai, TN, India", "is_allday": "False", "outcome_id": null, "from_date": "2021-06-19T23:00:00-06:00", "end_date": "2022-06-20T00:00:00-06:00", "created_at": "2021-10-18T09:55:47-06:00", "updated_at": "2021-10-18T09:55:47-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[{'id': 17008066468, 'type': 'Contact'}]"}, "emitted_at": 1682419150369} -{"stream": "past_appointments", "data": {"id": 17000384297, "time_zone": "Chennai", "title": "Sample Appointment", "description": "This is just a sample Appointment.", "location": "Chennai, TN, India", "is_allday": "False", "outcome_id": null, "from_date": "2021-06-19T23:00:00-06:00", "end_date": "2022-06-20T00:00:00-06:00", "created_at": "2021-10-18T09:56:29-06:00", "updated_at": "2021-10-18T09:56:29-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[{'id': 17008066468, 'type': 'Contact'}]"}, "emitted_at": 1682419150370} -{"stream": "past_appointments", "data": {"id": 17000386736, "time_zone": "Central America", "title": "Daily meeting", "description": "Daily meeting", "location": "Zoom", "is_allday": "False", "outcome_id": null, "from_date": "2021-10-19T00:30:06-06:00", "end_date": "2022-01-19T08:30:00-06:00", "created_at": "2021-10-19T00:25:24-06:00", "updated_at": "2021-10-19T00:25:24-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[]"}, "emitted_at": 1682419150370} -{"stream": "past_appointments", "data": {"id": 17000386761, "time_zone": "Central America", "title": "Discount discussion", "description": null, "location": "Zoom", "is_allday": "False", "outcome_id": null, "from_date": "2021-10-21T00:45:00-06:00", "end_date": "2021-10-21T01:15:00-06:00", "created_at": "2021-10-19T00:45:49-06:00", "updated_at": "2021-10-19T00:45:49-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[]"}, "emitted_at": 1682419150371} -{"stream": "past_appointments", "data": {"id": 17000382511, "time_zone": "Arizona", "title": "(Sample) Meeting - final discussion about the deal", "description": "Meeting James to resolve any concerns and close the deal.", "location": "Hilton Hotel, Bucks Road", "is_allday": "False", "outcome_id": null, "from_date": "2021-10-20T10:00:00-06:00", "end_date": "2021-10-20T12:00:00-06:00", "created_at": "2021-10-18T04:41:18-06:00", "updated_at": "2021-10-16T04:41:18-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[{'id': 17008066468, 'type': 'Contact'}]"}, "emitted_at": 1682419150371} -{"stream": "past_appointments", "data": {"id": 17000384327, "time_zone": "Central America", "title": "New meeting", "description": "test meeting", "location": "zoom", "is_allday": "False", "outcome_id": null, "from_date": "2021-10-18T10:15:40-06:00", "end_date": "2021-10-18T10:45:40-06:00", "created_at": "2021-10-18T10:03:30-06:00", "updated_at": "2021-10-18T10:03:30-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[]"}, "emitted_at": 1682419150372} -{"stream": "upcoming_appointments", "data": {"id": 17000384327, "time_zone": "Central America", "title": "New meeting", "description": "test meeting", "location": "zoom", "is_allday": "False", "outcome_id": null, "from_date": "2021-10-18T10:15:40-06:00", "end_date": "2021-10-18T10:45:40-06:00", "created_at": "2021-10-18T10:03:30-06:00", "updated_at": "2021-10-18T10:03:30-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[]"}, "emitted_at": 1682419151716} -{"stream": "upcoming_appointments", "data": {"id": 17000382511, "time_zone": "Arizona", "title": "(Sample) Meeting - final discussion about the deal", "description": "Meeting James to resolve any concerns and close the deal.", "location": "Hilton Hotel, Bucks Road", "is_allday": "False", "outcome_id": null, "from_date": "2021-10-20T10:00:00-06:00", "end_date": "2021-10-20T12:00:00-06:00", "created_at": "2021-10-18T04:41:18-06:00", "updated_at": "2021-10-16T04:41:18-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[{'id': 17008066468, 'type': 'Contact'}]"}, "emitted_at": 1682419151717} -{"stream": "upcoming_appointments", "data": {"id": 17000386761, "time_zone": "Central America", "title": "Discount discussion", "description": null, "location": "Zoom", "is_allday": "False", "outcome_id": null, "from_date": "2021-10-21T00:45:00-06:00", "end_date": "2021-10-21T01:15:00-06:00", "created_at": "2021-10-19T00:45:49-06:00", "updated_at": "2021-10-19T00:45:49-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[]"}, "emitted_at": 1682419151718} -{"stream": "upcoming_appointments", "data": {"id": 17000386736, "time_zone": "Central America", "title": "Daily meeting", "description": "Daily meeting", "location": "Zoom", "is_allday": "False", "outcome_id": null, "from_date": "2021-10-19T00:30:06-06:00", "end_date": "2022-01-19T08:30:00-06:00", "created_at": "2021-10-19T00:25:24-06:00", "updated_at": "2021-10-19T00:25:24-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[]"}, "emitted_at": 1682419151719} -{"stream": "upcoming_appointments", "data": {"id": 17000384293, "time_zone": "Chennai", "title": "Sample Appointment", "description": "This is just a sample Appointment.", "location": "Chennai, TN, India", "is_allday": "False", "outcome_id": null, "from_date": "2021-06-19T23:00:00-06:00", "end_date": "2022-06-20T00:00:00-06:00", "created_at": "2021-10-18T09:55:47-06:00", "updated_at": "2021-10-18T09:55:47-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[{'id': 17008066468, 'type': 'Contact'}]"}, "emitted_at": 1682419151719} -{"stream": "upcoming_appointments", "data": {"id": 17000384297, "time_zone": "Chennai", "title": "Sample Appointment", "description": "This is just a sample Appointment.", "location": "Chennai, TN, India", "is_allday": "False", "outcome_id": null, "from_date": "2021-06-19T23:00:00-06:00", "end_date": "2022-06-20T00:00:00-06:00", "created_at": "2021-10-18T09:56:29-06:00", "updated_at": "2021-10-18T09:56:29-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": "[{'id': 17008066468, 'type': 'Contact'}]"}, "emitted_at": 1682419151720} +{"stream": "contacts", "data": {"id": 17008318589, "first_name": "Mail Delivery", "last_name": "Subsystem", "display_name": "Mail Delivery Subsystem", "avatar": null, "job_title": null, "city": null, "state": null, "zipcode": null, "country": null, "email": "mailer-daemon@googlemail.com", "emails": [{"id": 17007443529, "value": "mailer-daemon@googlemail.com", "is_primary": true, "label": null, "_destroy": false}], "time_zone": null, "work_number": null, "mobile_number": null, "address": null, "last_seen": null, "lead_score": 14, "last_contacted": null, "open_deals_amount": "7700.0", "won_deals_amount": "0.0", "links": {"conversations": "/crm/sales/contacts/17008318589/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "timeline_feeds": "/crm/sales/contacts/17008318589/timeline_feeds", "document_associations": "/crm/sales/contacts/17008318589/document_associations", "notes": "/crm/sales/contacts/17008318589/notes?include=creater", "tasks": "/crm/sales/contacts/17008318589/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/contacts/17008318589/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note", "reminders": "/crm/sales/contacts/17008318589/reminders?include=creater,owner,updater,targetable", "duplicates": "/crm/sales/contacts/17008318589/duplicates", "connections": "/crm/sales/contacts/17008318589/connections"}, "last_contacted_sales_activity_mode": null, "custom_field": {}, "created_at": "2021-10-19T00:28:18-06:00", "updated_at": "2023-03-23T05:19:22-06:00", "keyword": null, "medium": null, "last_contacted_mode": null, "recent_note": null, "won_deals_count": 0, "last_contacted_via_sales_activity": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_ids": null, "open_deals_count": 2, "last_assigned_at": "2021-10-19T00:28:19-06:00", "facebook": null, "twitter": null, "linkedin": null, "is_deleted": false, "team_user_ids": null, "external_id": null, "work_email": null, "subscription_status": 1, "subscription_types": "1;2;3;4;5", "unsubscription_reason": null, "other_unsubscription_reason": null, "customer_fit": 1, "whatsapp_subscription_status": 2, "sms_subscription_status": 2, "last_seen_chat": null, "first_seen_chat": null, "locale": null, "total_sessions": null, "system_tags": [], "first_campaign": null, "first_medium": null, "first_source": null, "last_campaign": null, "last_medium": null, "last_source": null, "latest_campaign": null, "latest_medium": null, "latest_source": null, "mcr_id": 1450348061427834880, "phone_numbers": [], "tags": []}, "emitted_at": 1699903253739} +{"stream": "contacts", "data": {"id": 17008066468, "first_name": "Jane", "last_name": "Sampleton (sample)", "display_name": "Jane Sampleton (sample)", "avatar": "https://img.fullcontact.com/static/4df0efb1ea1a7650fef74f5e44d50d35_ca437b79617f8bbfc40c317b729d32693be1463f356b5be1015b39739859659f", "job_title": "Sales Manager", "city": "Glendale", "state": "Arizona", "zipcode": "100652", "country": "USA", "email": "janesampleton@gmail.com", "emails": [{"id": 17007194356, "value": "janesampleton@gmail.com", "is_primary": true, "label": null, "_destroy": false}], "time_zone": "Arizona", "work_number": "3684932360", "mobile_number": "19266529503", "address": "604-5854 Beckford St.", "last_seen": null, "lead_score": 33, "last_contacted": "2021-10-12T10:06:38-06:00", "open_deals_amount": "22780.0", "won_deals_amount": "11000.0", "links": {"conversations": "/crm/sales/contacts/17008066468/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "timeline_feeds": "/crm/sales/contacts/17008066468/timeline_feeds", "document_associations": "/crm/sales/contacts/17008066468/document_associations", "notes": "/crm/sales/contacts/17008066468/notes?include=creater", "tasks": "/crm/sales/contacts/17008066468/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/contacts/17008066468/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note", "reminders": "/crm/sales/contacts/17008066468/reminders?include=creater,owner,updater,targetable", "duplicates": "/crm/sales/contacts/17008066468/duplicates", "connections": "/crm/sales/contacts/17008066468/connections"}, "last_contacted_sales_activity_mode": "Task", "custom_field": {}, "created_at": "2021-10-07T10:06:37-06:00", "updated_at": "2023-09-08T23:42:26-06:00", "keyword": "B2B Success", "medium": "Blog", "last_contacted_mode": "Email opened by recipient", "recent_note": "Sample note for contact create", "won_deals_count": 4, "last_contacted_via_sales_activity": "2021-10-19T00:27:33-06:00", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_ids": null, "open_deals_count": 5, "last_assigned_at": "2022-09-05T04:44:49-06:00", "facebook": "100010587455650", "twitter": "janesampleton", "linkedin": "jane-sampleton-0b0039109", "is_deleted": false, "team_user_ids": null, "external_id": null, "work_email": null, "subscription_status": 1, "subscription_types": "1;2;3;4;5", "unsubscription_reason": null, "other_unsubscription_reason": null, "customer_fit": 2, "whatsapp_subscription_status": 2, "sms_subscription_status": 2, "last_seen_chat": null, "first_seen_chat": null, "locale": null, "total_sessions": null, "system_tags": [], "first_campaign": null, "first_medium": null, "first_source": null, "last_campaign": null, "last_medium": null, "last_source": null, "latest_campaign": null, "latest_medium": null, "latest_source": null, "mcr_id": 1450049339590340608, "phone_numbers": [], "tags": []}, "emitted_at": 1699903253746} +{"stream": "accounts", "data": {"id": 17001321830, "name": "Widgetz.io (sample)", "address": "160-6802 Aliquet Rd.", "city": "New Haven", "state": "Connecticut", "zipcode": "68089", "country": "United States", "number_of_employees": null, "annual_revenue": 0, "website": "widgetz.io", "owner_id": null, "phone": "5036153947", "open_deals_amount": "0.0", "open_deals_count": 0, "won_deals_amount": "0.0", "won_deals_count": 0, "last_contacted": "2021-10-12T10:06:38-06:00", "last_contacted_mode": "Email opened by recipient", "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17001321830/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17001321830/document_associations", "notes": "/crm/sales/sales_accounts/17001321830/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17001321830/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17001321830/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2021-10-18T04:41:17-06:00", "updated_at": "2022-06-23T04:25:57-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": null, "last_contacted_via_sales_activity": "2021-10-19T00:27:33-06:00", "last_contacted_sales_activity_mode": "Task", "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2021-10-18T09:56:31-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1699903255739} +{"stream": "accounts", "data": {"id": 17001391875, "name": "Airbyte", "address": "San Francisco, CA", "city": null, "state": null, "zipcode": "94121", "country": null, "number_of_employees": 1, "annual_revenue": null, "website": null, "owner_id": 17000038922, "phone": "+1234567890", "open_deals_amount": "25.0", "open_deals_count": 1, "won_deals_amount": "0.0", "won_deals_count": 0, "last_contacted": "2021-10-19T05:04:54-06:00", "last_contacted_mode": "Outgoing call", "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17001391875/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17001391875/document_associations", "notes": "/crm/sales/sales_accounts/17001391875/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17001391875/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17001391875/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2021-10-19T04:57:42-06:00", "updated_at": "2022-06-23T04:25:57-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": "activate 10/10/2021", "last_contacted_via_sales_activity": "2021-10-19T05:04:54-06:00", "last_contacted_sales_activity_mode": "Phone", "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2021-10-19T04:57:43-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1699903255745} +{"stream": "accounts", "data": {"id": 17004983219, "name": "Test Account 2", "address": null, "city": null, "state": null, "zipcode": null, "country": null, "number_of_employees": null, "annual_revenue": null, "website": null, "owner_id": 17000038922, "phone": null, "open_deals_amount": "0.0", "open_deals_count": 0, "won_deals_amount": "5000.0", "won_deals_count": 2, "last_contacted": null, "last_contacted_mode": null, "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17004983219/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17004983219/document_associations", "notes": "/crm/sales/sales_accounts/17004983219/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17004983219/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17004983219/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2023-03-23T05:14:37-06:00", "updated_at": "2023-03-23T05:25:59-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": null, "last_contacted_via_sales_activity": null, "last_contacted_sales_activity_mode": null, "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2023-03-23T05:14:38-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1699903255751} +{"stream": "accounts", "data": {"id": 17004983220, "name": "Test Account 3", "address": null, "city": null, "state": null, "zipcode": null, "country": null, "number_of_employees": null, "annual_revenue": null, "website": null, "owner_id": 17000038922, "phone": null, "open_deals_amount": "3200.0", "open_deals_count": 1, "won_deals_amount": "0.0", "won_deals_count": 0, "last_contacted": null, "last_contacted_mode": null, "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17004983220/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17004983220/document_associations", "notes": "/crm/sales/sales_accounts/17004983220/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17004983220/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17004983220/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2023-03-23T05:19:11-06:00", "updated_at": "2023-03-23T05:26:29-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": null, "last_contacted_via_sales_activity": null, "last_contacted_sales_activity_mode": null, "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2023-03-23T05:19:12-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1699903255756} +{"stream": "accounts", "data": {"id": 17004983221, "name": "Test Account 4", "address": null, "city": null, "state": null, "zipcode": null, "country": null, "number_of_employees": null, "annual_revenue": null, "website": null, "owner_id": 17000038922, "phone": null, "open_deals_amount": "8080.0", "open_deals_count": 2, "won_deals_amount": "0.0", "won_deals_count": 0, "last_contacted": null, "last_contacted_mode": null, "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17004983221/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17004983221/document_associations", "notes": "/crm/sales/sales_accounts/17004983221/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17004983221/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17004983221/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2023-03-23T05:19:46-06:00", "updated_at": "2023-03-23T05:26:56-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": null, "last_contacted_via_sales_activity": null, "last_contacted_sales_activity_mode": null, "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2023-03-23T05:19:47-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1699903255762} +{"stream": "accounts", "data": {"id": 17004983218, "name": "Test Account 1", "address": null, "city": null, "state": null, "zipcode": null, "country": null, "number_of_employees": null, "annual_revenue": null, "website": null, "owner_id": 17000038922, "phone": null, "open_deals_amount": "4500.0", "open_deals_count": 1, "won_deals_amount": "6000.0", "won_deals_count": 2, "last_contacted": null, "last_contacted_mode": null, "facebook": null, "twitter": null, "linkedin": null, "links": {"conversations": "/crm/sales/sales_accounts/17004983218/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/sales_accounts/17004983218/document_associations", "notes": "/crm/sales/sales_accounts/17004983218/notes?include=creater", "tasks": "/crm/sales/sales_accounts/17004983218/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/sales_accounts/17004983218/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "custom_field": {}, "created_at": "2023-03-23T05:13:41-06:00", "updated_at": "2023-03-23T05:31:35-06:00", "avatar": null, "parent_sales_account_id": null, "recent_note": null, "last_contacted_via_sales_activity": "2023-03-07T11:00:00-06:00", "last_contacted_sales_activity_mode": "Test chat", "completed_sales_sequences": null, "active_sales_sequences": null, "last_assigned_at": "2023-03-23T05:13:42-06:00", "is_deleted": false, "team_user_ids": null, "web_form_ids": null, "tags": []}, "emitted_at": 1699903255768} +{"stream": "open_deals", "data": {"id": 17000512184, "name": "Gold plan (sample)", "amount": "7000.0", "base_currency_amount": "7000.0", "expected_close": "2021-10-21", "closed_date": null, "stage_updated_time": "2021-10-13T10:06:38-06:00", "custom_field": {}, "probability": 100, "updated_at": "2021-10-14T10:06:38-06:00", "created_at": "2021-10-09T10:06:37-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237830, "age": 765, "links": {"conversations": "/crm/sales/deals/17000512184/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17000512184/document_associations", "notes": "/crm/sales/deals/17000512184/notes?include=creater", "tasks": "/crm/sales/deals/17000512184/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17000512184/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2021-10-09T10:06:37-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "7000.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": 2, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 735, "tags": []}, "emitted_at": 1699903257648} +{"stream": "open_deals", "data": {"id": 17000521380, "name": "Discaunt", "amount": "25.0", "base_currency_amount": "25.0", "expected_close": null, "closed_date": null, "stage_updated_time": "2021-10-19T05:10:53-06:00", "custom_field": {}, "probability": 100, "updated_at": "2021-10-19T05:10:53-06:00", "created_at": "2021-10-19T05:04:09-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237831, "age": 755, "links": {"conversations": "/crm/sales/deals/17000521380/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17000521380/document_associations", "notes": "/crm/sales/deals/17000521380/notes?include=creater", "tasks": "/crm/sales/deals/17000521380/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17000521380/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2021-10-19T05:04:10-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "25.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 725, "tags": []}, "emitted_at": 1699903257653} +{"stream": "open_deals", "data": {"id": 17015628751, "name": "Test Open Deal 1", "amount": "4500.0", "base_currency_amount": "4500.0", "expected_close": null, "closed_date": null, "stage_updated_time": "2023-03-23T05:18:44-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:18:44-06:00", "created_at": "2023-03-23T05:18:44-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237830, "age": 235, "links": {"conversations": "/crm/sales/deals/17015628751/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628751/document_associations", "notes": "/crm/sales/deals/17015628751/notes?include=creater", "tasks": "/crm/sales/deals/17015628751/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628751/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:18:45-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "4500.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 205, "tags": []}, "emitted_at": 1699903257657} +{"stream": "open_deals", "data": {"id": 17015628806, "name": "Test Open Deal 2", "amount": "3200.0", "base_currency_amount": "3200.0", "expected_close": null, "closed_date": null, "stage_updated_time": "2023-03-23T05:19:22-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:19:22-06:00", "created_at": "2023-03-23T05:19:22-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237830, "age": 235, "links": {"conversations": "/crm/sales/deals/17015628806/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628806/document_associations", "notes": "/crm/sales/deals/17015628806/notes?include=creater", "tasks": "/crm/sales/deals/17015628806/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628806/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:19:23-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "3200.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 205, "tags": []}, "emitted_at": 1699903257661} +{"stream": "open_deals", "data": {"id": 17015628849, "name": "Test Open Deal 3", "amount": "1580.0", "base_currency_amount": "1580.0", "expected_close": null, "closed_date": null, "stage_updated_time": "2023-03-23T05:19:51-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:19:51-06:00", "created_at": "2023-03-23T05:19:51-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237830, "age": 235, "links": {"conversations": "/crm/sales/deals/17015628849/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628849/document_associations", "notes": "/crm/sales/deals/17015628849/notes?include=creater", "tasks": "/crm/sales/deals/17015628849/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628849/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:19:52-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "1580.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 205, "tags": []}, "emitted_at": 1699903257666} +{"stream": "open_deals", "data": {"id": 17015628886, "name": "Test Open Deal 4", "amount": "6500.0", "base_currency_amount": "6500.0", "expected_close": null, "closed_date": null, "stage_updated_time": "2023-03-23T05:20:17-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:20:17-06:00", "created_at": "2023-03-23T05:20:17-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237830, "age": 235, "links": {"conversations": "/crm/sales/deals/17015628886/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628886/document_associations", "notes": "/crm/sales/deals/17015628886/notes?include=creater", "tasks": "/crm/sales/deals/17015628886/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628886/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": null, "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:20:18-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "6500.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 0, "deal_prediction_last_updated_at": null, "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": 205, "tags": []}, "emitted_at": 1699903257670} +{"stream": "won_deals", "data": {"id": 17015628496, "name": "Test Won Deal 4", "amount": "5000.0", "base_currency_amount": "5000.0", "expected_close": null, "closed_date": "2023-03-23", "stage_updated_time": "2023-03-23T05:16:34-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:16:36-06:00", "created_at": "2023-03-23T05:15:44-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237835, "age": 0, "links": {"conversations": "/crm/sales/deals/17015628496/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628496/document_associations", "notes": "/crm/sales/deals/17015628496/notes?include=creater", "tasks": "/crm/sales/deals/17015628496/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628496/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:15:45-06:00", "last_contacted_sales_activity_mode": "Test chat", "last_contacted_via_sales_activity": "2023-03-07T11:00:00-06:00", "expected_deal_value": "5000.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 6, "deal_prediction_last_updated_at": "2023-03-23T05:16:34-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1699903259466} +{"stream": "won_deals", "data": {"id": 17015628427, "name": "Test Won Deal 2", "amount": "3000.0", "base_currency_amount": "3000.0", "expected_close": null, "closed_date": "2023-03-23", "stage_updated_time": "2023-03-23T05:17:00-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:17:00-06:00", "created_at": "2023-03-23T05:14:55-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237835, "age": 0, "links": {"conversations": "/crm/sales/deals/17015628427/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628427/document_associations", "notes": "/crm/sales/deals/17015628427/notes?include=creater", "tasks": "/crm/sales/deals/17015628427/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628427/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:14:56-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "3000.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 6, "deal_prediction_last_updated_at": "2023-03-23T05:17:00-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1699903259472} +{"stream": "won_deals", "data": {"id": 17015628468, "name": "Test Won Deal 3", "amount": "2000.0", "base_currency_amount": "2000.0", "expected_close": null, "closed_date": "2023-03-23", "stage_updated_time": "2023-03-23T05:17:21-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:17:26-06:00", "created_at": "2023-03-23T05:15:23-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237835, "age": 0, "links": {"conversations": "/crm/sales/deals/17015628468/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628468/document_associations", "notes": "/crm/sales/deals/17015628468/notes?include=creater", "tasks": "/crm/sales/deals/17015628468/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628468/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:15:24-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "2000.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 6, "deal_prediction_last_updated_at": "2023-03-23T05:17:21-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1699903259480} +{"stream": "won_deals", "data": {"id": 17015628368, "name": "Test Won Deal 1", "amount": "1000.0", "base_currency_amount": "1000.0", "expected_close": null, "closed_date": "2023-03-14", "stage_updated_time": "2023-03-23T05:17:48-06:00", "custom_field": {}, "probability": 100, "updated_at": "2023-03-23T05:17:50-06:00", "created_at": "2023-03-23T05:14:16-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237835, "age": -9, "links": {"conversations": "/crm/sales/deals/17015628368/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015628368/document_associations", "notes": "/crm/sales/deals/17015628368/notes?include=creater", "tasks": "/crm/sales/deals/17015628368/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015628368/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:14:17-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "1000.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 6, "deal_prediction_last_updated_at": "2023-03-23T05:17:48-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1699903259485} +{"stream": "lost_deals", "data": {"id": 17015629024, "name": "Test Lost Deal 1", "amount": "800.0", "base_currency_amount": "800.0", "expected_close": null, "closed_date": "2023-03-17", "stage_updated_time": "2023-03-23T05:24:18-06:00", "custom_field": {}, "probability": 0, "updated_at": "2023-03-23T05:24:21-06:00", "created_at": "2023-03-23T05:21:53-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237836, "age": -6, "links": {"conversations": "/crm/sales/deals/17015629024/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015629024/document_associations", "notes": "/crm/sales/deals/17015629024/notes?include=creater", "tasks": "/crm/sales/deals/17015629024/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015629024/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:21:54-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "0.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 7, "deal_prediction_last_updated_at": "2023-03-23T05:24:18-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1699903261406} +{"stream": "lost_deals", "data": {"id": 17015629056, "name": "Test Lost Deal 2", "amount": "2800.0", "base_currency_amount": "2800.0", "expected_close": null, "closed_date": "2023-03-22", "stage_updated_time": "2023-03-23T05:25:59-06:00", "custom_field": {}, "probability": 0, "updated_at": "2023-03-23T05:26:01-06:00", "created_at": "2023-03-23T05:22:14-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237836, "age": -1, "links": {"conversations": "/crm/sales/deals/17015629056/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015629056/document_associations", "notes": "/crm/sales/deals/17015629056/notes?include=creater", "tasks": "/crm/sales/deals/17015629056/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015629056/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:22:15-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "0.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 7, "deal_prediction_last_updated_at": "2023-03-23T05:25:59-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1699903261411} +{"stream": "lost_deals", "data": {"id": 17015629086, "name": "Test Lost Deal 3", "amount": "3200.0", "base_currency_amount": "3200.0", "expected_close": null, "closed_date": "2023-03-09", "stage_updated_time": "2023-03-23T05:26:29-06:00", "custom_field": {}, "probability": 0, "updated_at": "2023-03-23T05:26:31-06:00", "created_at": "2023-03-23T05:22:33-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237836, "age": -14, "links": {"conversations": "/crm/sales/deals/17015629086/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015629086/document_associations", "notes": "/crm/sales/deals/17015629086/notes?include=creater", "tasks": "/crm/sales/deals/17015629086/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015629086/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:22:34-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "0.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 7, "deal_prediction_last_updated_at": "2023-03-23T05:26:29-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1699903261416} +{"stream": "lost_deals", "data": {"id": 17015629190, "name": "Test Lost Deal 4", "amount": "895.0", "base_currency_amount": "895.0", "expected_close": null, "closed_date": "2023-03-05", "stage_updated_time": "2023-03-23T05:26:55-06:00", "custom_field": {}, "probability": 0, "updated_at": "2023-03-23T05:26:57-06:00", "created_at": "2023-03-23T05:23:44-06:00", "deal_pipeline_id": 17000033935, "deal_stage_id": 17000237836, "age": -18, "links": {"conversations": "/crm/sales/deals/17015629190/conversations/all?include=email_conversation_recipients%2Ctargetable%2Cphone_number%2Cphone_caller%2Cnote%2Cuser&per_page=3", "document_associations": "/crm/sales/deals/17015629190/document_associations", "notes": "/crm/sales/deals/17015629190/notes?include=creater", "tasks": "/crm/sales/deals/17015629190/tasks?include=creater,owner,updater,targetable,users,task_type", "appointments": "/crm/sales/deals/17015629190/appointments?include=creater,owner,updater,targetable,appointment_attendees,conference,note"}, "recent_note": "Test notes", "completed_sales_sequences": null, "active_sales_sequences": null, "web_form_id": null, "upcoming_activities_time": null, "collaboration": {}, "last_assigned_at": "2023-03-23T05:23:45-06:00", "last_contacted_sales_activity_mode": null, "last_contacted_via_sales_activity": null, "expected_deal_value": "0.0", "is_deleted": false, "team_user_ids": null, "avatar": null, "forecast_category": null, "deal_prediction": 7, "deal_prediction_last_updated_at": "2023-03-23T05:26:55-06:00", "freddy_forecast_metrics": null, "last_deal_prediction": null, "rotten_days": null, "tags": []}, "emitted_at": 1699903261420} +{"stream": "open_tasks", "data": {"id": 17000410092, "status": 0, "title": "All reports for meeting", "description": "All reports for meeting", "created_at": "2021-10-19T00:31:18-06:00", "updated_at": "2021-10-19T00:31:18-06:00", "owner_id": 17000038922, "due_date": "2021-10-20T00:30:00-06:00", "completed_date": null, "creater_id": 17000038922, "updater_id": 17000038922, "outcome_id": 17001154298, "task_type_id": 17000098963, "targetables": []}, "emitted_at": 1699903262254} +{"stream": "open_tasks", "data": {"id": 17000407414, "status": 0, "title": "(Sample) Send the pricing quote", "description": "Coordinate with Steve for the pricing quote and send it to James.", "created_at": "2021-10-14T04:41:18-06:00", "updated_at": "2021-10-18T04:41:18-06:00", "owner_id": 17000038922, "due_date": "2021-10-20T03:00:00-06:00", "completed_date": null, "creater_id": 17000038922, "updater_id": 17000038922, "outcome_id": null, "task_type_id": null, "targetables": [{"id": 17008066468, "type": "Contact"}]}, "emitted_at": 1699903262258} +{"stream": "open_tasks", "data": {"id": 17000411384, "status": 0, "title": "Meeting", "description": "Meeting with Zazmic", "created_at": "2021-10-19T05:05:54-06:00", "updated_at": "2021-10-19T05:05:54-06:00", "owner_id": 17000038922, "due_date": "2021-10-29T05:30:00-06:00", "completed_date": null, "creater_id": 17000038922, "updater_id": 17000038922, "outcome_id": 17001154298, "task_type_id": 17000098963, "targetables": [{"id": 17001391875, "type": "SalesAccount"}]}, "emitted_at": 1699903262261} +{"stream": "open_tasks", "data": {"id": 17000408204, "status": 0, "title": "Sample Task", "description": "This is just a sample task.", "created_at": "2021-10-18T09:53:02-06:00", "updated_at": "2021-10-18T09:53:02-06:00", "owner_id": 17000038922, "due_date": "2022-06-21T05:00:00-06:00", "completed_date": null, "creater_id": 17000038922, "updater_id": 17000038922, "outcome_id": null, "task_type_id": null, "targetables": [{"id": 17008066468, "type": "Contact"}]}, "emitted_at": 1699903262265} +{"stream": "completed_tasks", "data": {"id": 17000407413, "status": 1, "title": "(Sample) Send the proposal document", "description": "Send the proposal document and follow up with this contact after it.", "created_at": "2021-10-14T04:41:18-06:00", "updated_at": "2021-10-19T00:27:33-06:00", "owner_id": 17000038922, "due_date": "2021-10-19T02:00:00-06:00", "completed_date": "2021-10-19T00:27:33-06:00", "creater_id": 17000038922, "updater_id": 17000038922, "outcome_id": 17001154300, "task_type_id": null, "targetables": [{"id": 17008066468, "type": "Contact"}]}, "emitted_at": 1699903263068} +{"stream": "past_appointments", "data": {"id": 17000384293, "time_zone": "Chennai", "title": "Sample Appointment", "description": "This is just a sample Appointment.", "location": "Chennai, TN, India", "is_allday": false, "outcome_id": null, "from_date": "2021-06-19T23:00:00-06:00", "end_date": "2022-06-20T00:00:00-06:00", "created_at": "2021-10-18T09:55:47-06:00", "updated_at": "2021-10-18T09:55:47-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": [{"id": 17008066468, "type": "Contact"}]}, "emitted_at": 1699903264022} +{"stream": "past_appointments", "data": {"id": 17000384297, "time_zone": "Chennai", "title": "Sample Appointment", "description": "This is just a sample Appointment.", "location": "Chennai, TN, India", "is_allday": false, "outcome_id": null, "from_date": "2021-06-19T23:00:00-06:00", "end_date": "2022-06-20T00:00:00-06:00", "created_at": "2021-10-18T09:56:29-06:00", "updated_at": "2021-10-18T09:56:29-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": [{"id": 17008066468, "type": "Contact"}]}, "emitted_at": 1699903264028} +{"stream": "past_appointments", "data": {"id": 17000386736, "time_zone": "Central America", "title": "Daily meeting", "description": "Daily meeting", "location": "Zoom", "is_allday": false, "outcome_id": null, "from_date": "2021-10-19T00:30:06-06:00", "end_date": "2022-01-19T08:30:00-06:00", "created_at": "2021-10-19T00:25:24-06:00", "updated_at": "2021-10-19T00:25:24-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": []}, "emitted_at": 1699903264032} +{"stream": "past_appointments", "data": {"id": 17000386761, "time_zone": "Central America", "title": "Discount discussion", "description": null, "location": "Zoom", "is_allday": false, "outcome_id": null, "from_date": "2021-10-21T00:45:00-06:00", "end_date": "2021-10-21T01:15:00-06:00", "created_at": "2021-10-19T00:45:49-06:00", "updated_at": "2021-10-19T00:45:49-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": []}, "emitted_at": 1699903264037} +{"stream": "past_appointments", "data": {"id": 17000382511, "time_zone": "Arizona", "title": "(Sample) Meeting - final discussion about the deal", "description": "Meeting James to resolve any concerns and close the deal.", "location": "Hilton Hotel, Bucks Road", "is_allday": false, "outcome_id": null, "from_date": "2021-10-20T10:00:00-06:00", "end_date": "2021-10-20T12:00:00-06:00", "created_at": "2021-10-18T04:41:18-06:00", "updated_at": "2021-10-16T04:41:18-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": [{"id": 17008066468, "type": "Contact"}]}, "emitted_at": 1699903264043} +{"stream": "past_appointments", "data": {"id": 17000384327, "time_zone": "Central America", "title": "New meeting", "description": "test meeting", "location": "zoom", "is_allday": false, "outcome_id": null, "from_date": "2021-10-18T10:15:40-06:00", "end_date": "2021-10-18T10:45:40-06:00", "created_at": "2021-10-18T10:03:30-06:00", "updated_at": "2021-10-18T10:03:30-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": []}, "emitted_at": 1699903264047} +{"stream": "upcoming_appointments", "data": {"id": 17000384327, "time_zone": "Central America", "title": "New meeting", "description": "test meeting", "location": "zoom", "is_allday": false, "outcome_id": null, "from_date": "2021-10-18T10:15:40-06:00", "end_date": "2021-10-18T10:45:40-06:00", "created_at": "2021-10-18T10:03:30-06:00", "updated_at": "2021-10-18T10:03:30-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": []}, "emitted_at": 1699903265232} +{"stream": "upcoming_appointments", "data": {"id": 17000382511, "time_zone": "Arizona", "title": "(Sample) Meeting - final discussion about the deal", "description": "Meeting James to resolve any concerns and close the deal.", "location": "Hilton Hotel, Bucks Road", "is_allday": false, "outcome_id": null, "from_date": "2021-10-20T10:00:00-06:00", "end_date": "2021-10-20T12:00:00-06:00", "created_at": "2021-10-18T04:41:18-06:00", "updated_at": "2021-10-16T04:41:18-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": [{"id": 17008066468, "type": "Contact"}]}, "emitted_at": 1699903265239} +{"stream": "upcoming_appointments", "data": {"id": 17000386761, "time_zone": "Central America", "title": "Discount discussion", "description": null, "location": "Zoom", "is_allday": false, "outcome_id": null, "from_date": "2021-10-21T00:45:00-06:00", "end_date": "2021-10-21T01:15:00-06:00", "created_at": "2021-10-19T00:45:49-06:00", "updated_at": "2021-10-19T00:45:49-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": []}, "emitted_at": 1699903265243} +{"stream": "upcoming_appointments", "data": {"id": 17000386736, "time_zone": "Central America", "title": "Daily meeting", "description": "Daily meeting", "location": "Zoom", "is_allday": false, "outcome_id": null, "from_date": "2021-10-19T00:30:06-06:00", "end_date": "2022-01-19T08:30:00-06:00", "created_at": "2021-10-19T00:25:24-06:00", "updated_at": "2021-10-19T00:25:24-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": []}, "emitted_at": 1699903265249} +{"stream": "upcoming_appointments", "data": {"id": 17000384293, "time_zone": "Chennai", "title": "Sample Appointment", "description": "This is just a sample Appointment.", "location": "Chennai, TN, India", "is_allday": false, "outcome_id": null, "from_date": "2021-06-19T23:00:00-06:00", "end_date": "2022-06-20T00:00:00-06:00", "created_at": "2021-10-18T09:55:47-06:00", "updated_at": "2021-10-18T09:55:47-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": [{"id": 17008066468, "type": "Contact"}]}, "emitted_at": 1699903265253} +{"stream": "upcoming_appointments", "data": {"id": 17000384297, "time_zone": "Chennai", "title": "Sample Appointment", "description": "This is just a sample Appointment.", "location": "Chennai, TN, India", "is_allday": false, "outcome_id": null, "from_date": "2021-06-19T23:00:00-06:00", "end_date": "2022-06-20T00:00:00-06:00", "created_at": "2021-10-18T09:56:29-06:00", "updated_at": "2021-10-18T09:56:29-06:00", "provider": "freshsales", "creater_id": 17000038922, "latitude": null, "longitude": null, "checkedin_at": null, "can_checkin_checkout": true, "checkedout_latitude": null, "checkedout_longitude": null, "checkedout_location": null, "checkedout_at": null, "checkedin_duration": null, "can_checkin": true, "conference_id": null, "targetables": [{"id": 17008066468, "type": "Contact"}]}, "emitted_at": 1699903265258} diff --git a/airbyte-integrations/connectors/source-freshsales/metadata.yaml b/airbyte-integrations/connectors/source-freshsales/metadata.yaml index 4b9db49454b2..afb486eb0a95 100644 --- a/airbyte-integrations/connectors/source-freshsales/metadata.yaml +++ b/airbyte-integrations/connectors/source-freshsales/metadata.yaml @@ -1,12 +1,21 @@ data: + ab_internal: + ql: 300 + sl: 100 allowedHosts: hosts: - "*.myfreshworks.com" connectorSubtype: api connectorType: source definitionId: eca08d79-7b92-4065-b7f3-79c14836ebe7 - dockerImageTag: 0.1.4 + dockerImageTag: 1.0.0 + releases: + breakingChanges: + 1.0.0: + message: "This version migrates the Freshsales connector to our low-code framework for greater maintainability. It also introduces changes to data types across most streams. You will need to run a reset after upgrading to continue syncing data with the connector." + upgradeDeadline: "2023-11-29" dockerRepository: airbyte/source-freshsales + documentationUrl: https://docs.airbyte.com/integrations/sources/freshsales githubIssueLabel: source-freshsales icon: freshsales.svg license: MIT @@ -17,11 +26,7 @@ data: oss: enabled: true releaseStage: beta - documentationUrl: https://docs.airbyte.com/integrations/sources/freshsales + supportLevel: community tags: - - language:python - ab_internal: - sl: 200 - ql: 300 - supportLevel: certified + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-freshsales/setup.py b/airbyte-integrations/connectors/source-freshsales/setup.py index 91ef829534e2..2cc1107f0c8b 100644 --- a/airbyte-integrations/connectors/source-freshsales/setup.py +++ b/airbyte-integrations/connectors/source-freshsales/setup.py @@ -6,23 +6,23 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", + "airbyte-cdk~=0.1", ] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", - "pytest", - "pytest-mock", + "pytest~=6.2", + "pytest-mock~=3.6.1", ] setup( name="source_freshsales", description="Source implementation for Freshsales.", - author="Tuan Nguyen", - author_email="anhtuan.nguyen@me.com", + author="Airbyte", + author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/manifest.yaml b/airbyte-integrations/connectors/source-freshsales/source_freshsales/manifest.yaml new file mode 100644 index 000000000000..bf292c6c56f0 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/manifest.yaml @@ -0,0 +1,286 @@ +version: 0.51.16 + +type: DeclarativeSource +check: + type: CheckStream + stream_names: + - contacts +definitions: + schema_loader: + type: JsonFileSchemaLoader + file_path: "./source_freshsales/schemas/{{ parameters['name'] }}.json" + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + basic_requester: + type: HttpRequester + url_base: "https://{{ config['domain_name'] }}/crm/sales/api/" + http_method: "GET" + authenticator: + type: ApiKeyAuthenticator + header: "Authorization" + api_token: "Token token={{ config['api_key'] }}" + requester: + $ref: "#/definitions/basic_requester" + request_parameters: + page: "{{ parameters.get('page', None) }}" + filter: "{{ parameters.get('filter', None) }}" + sort_type: "asc" + sort: "updated_at" + default_paginator: + type: "DefaultPaginator" + page_size_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "per_page" + pagination_strategy: + type: "PageIncrement" + page_size: 50 + start_from_page: 1 + inject_on_first_request: true + page_token_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "page" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + requester: + $ref: "#/definitions/requester" + paginator: + $ref: "#/definitions/default_paginator" + base_stream: + primary_key: "id" + retriever: + $ref: "#/definitions/retriever" + contacts_filters_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "contact_filters" + path: "contacts/filters" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/basic_requester" + record_selector: + type: RecordSelector + extractor: + field_path: + - filters + record_filter: + condition: "{{ record['name'] == parameters['filter'] }}" + contacts_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "contacts" + filter: "All Contacts" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + path: "contacts/view/{{ stream_slice.view_id }}" + record_selector: + type: RecordSelector + extractor: + field_path: + - contacts + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/contacts_filters_stream" + parent_key: "id" + partition_field: "view_id" + accounts_filters_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "sales_accounts_filter" + path: "sales_accounts/filters" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/basic_requester" + record_selector: + type: RecordSelector + extractor: + field_path: + - filters + record_filter: + condition: "{{ record['name'] == parameters['filter'] }}" + accounts_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "accounts" + filter: "All Accounts" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + path: "sales_accounts/view/{{ stream_slice.view_id }}" + record_selector: + type: RecordSelector + extractor: + field_path: + - sales_accounts + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/accounts_filters_stream" + parent_key: "id" + partition_field: "view_id" + deals_filters_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "deals_filter" + path: "deals/filters" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/basic_requester" + record_selector: + type: RecordSelector + extractor: + field_path: + - filters + record_filter: + condition: "{{ record['name'] == parameters['filter'] }}" + open_deals_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "open_deals" + filter: "Open Deals" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + path: "deals/view/{{ stream_slice.view_id }}" + record_selector: + type: RecordSelector + extractor: + field_path: + - deals + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/deals_filters_stream" + parent_key: "id" + partition_field: "view_id" + transformations: + - type: RemoveFields + field_pointers: + - ["fc_widget_collaboration"] + won_deals_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "won_deals" + filter: "Won Deals" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + path: "deals/view/{{ stream_slice.view_id }}" + record_selector: + type: RecordSelector + extractor: + field_path: + - deals + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/deals_filters_stream" + parent_key: "id" + partition_field: "view_id" + transformations: + - type: RemoveFields + field_pointers: + - ["fc_widget_collaboration"] + lost_deals_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "lost_deals" + filter: "Lost Deals" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + path: "deals/view/{{ stream_slice.view_id }}" + record_selector: + type: RecordSelector + extractor: + field_path: + - deals + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/deals_filters_stream" + parent_key: "id" + partition_field: "view_id" + transformations: + - type: RemoveFields + field_pointers: + - ["fc_widget_collaboration"] + open_tasks_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "open_tasks" + path: "tasks" + filter: "open" + retriever: + $ref: "#/definitions/retriever" + record_selector: + type: RecordSelector + extractor: + field_path: + - tasks + completed_tasks_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "completed_tasks" + path: "tasks" + filter: "completed" + retriever: + $ref: "#/definitions/retriever" + record_selector: + type: RecordSelector + extractor: + field_path: + - tasks + past_appointments_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "past_appointments" + path: "appointments" + filter: "past" + retriever: + $ref: "#/definitions/retriever" + record_selector: + type: RecordSelector + extractor: + field_path: + - appointments + upcoming_appointments_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "upcoming_appointments" + path: "appointments" + filter: "upcoming" + retriever: + $ref: "#/definitions/retriever" + record_selector: + type: RecordSelector + extractor: + field_path: + - appointments + +streams: + - "#/definitions/contacts_stream" + - "#/definitions/accounts_stream" + - "#/definitions/open_deals_stream" + - "#/definitions/won_deals_stream" + - "#/definitions/lost_deals_stream" + - "#/definitions/open_tasks_stream" + - "#/definitions/completed_tasks_stream" + - "#/definitions/past_appointments_stream" + - "#/definitions/upcoming_appointments_stream" diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/accounts.json b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/accounts.json index 231a18049eed..601134fea6b1 100644 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/accounts.json +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/accounts.json @@ -38,8 +38,8 @@ "last_seen": { "type": ["null", "string"] }, "lead_score": { "type": ["null", "integer"] }, "last_contacted": { "type": ["null", "string"] }, - "open_deals_amount": { "type": ["null", "number"] }, - "won_deals_amount": { "type": ["null", "number"] }, + "open_deals_amount": { "type": ["null", "string"] }, + "won_deals_amount": { "type": ["null", "string"] }, "links": { "type": ["null", "object"] }, "last_contacted_sales_activity_mode": { "type": ["null", "string"] }, "custom_field": { "type": ["null", "object"] }, diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/completed_tasks.json b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/completed_tasks.json index 7c5325c9d8e6..5d44344dfe8a 100644 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/completed_tasks.json +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/completed_tasks.json @@ -12,14 +12,14 @@ "targetable_type": { "type": ["null", "string"] }, "Possible": { "type": ["null", "string"] }, "owner_id": { "type": ["null", "integer"] }, - "status": { "type": ["null", "string"] }, + "status": { "type": ["null", "integer"] }, "creater_id": { "type": ["null", "integer"] }, "created_at": { "type": ["null", "string"] }, "updated_at": { "type": ["null", "string"] }, "outcome_id": { "type": ["null", "integer"] }, "task_type_id": { "type": ["null", "integer"] }, "updater_id": { "type": ["null", "integer"] }, - "targetables": { "type": ["null", "string"] }, + "targetables": { "type": ["null", "array"] }, "completed_date": { "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/contacts.json b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/contacts.json index cae7ab5ec745..db09dc756746 100644 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/contacts.json @@ -7,10 +7,10 @@ "id": { "type": ["null", "integer"] }, "first_name": { "type": ["null", "string"] }, "last_name": { "type": ["null", "string"] }, - "subscription_status": { "type": ["null", "string"] }, + "subscription_status": { "type": ["null", "integer"] }, "job_title": { "type": ["null", "string"] }, "email": { "type": ["null", "string"] }, - "emails": { "type": ["null", "string"] }, + "emails": { "type": ["null", "array"] }, "work_number": { "type": ["null", "string"] }, "external_id": { "type": ["null", "string"] }, "mobile_number": { "type": ["null", "string"] }, @@ -41,8 +41,8 @@ "last_seen": { "type": ["null", "string"] }, "lead_score": { "type": ["null", "integer"] }, "last_contacted": { "type": ["null", "string"] }, - "open_deals_amount": { "type": ["null", "number"] }, - "won_deals_amount": { "type": ["null", "number"] }, + "open_deals_amount": { "type": ["null", "string"] }, + "won_deals_amount": { "type": ["null", "string"] }, "links": { "type": ["null", "object"] }, "last_contacted_sales_activity_mode": { "type": ["null", "string"] }, "custom_field": { "type": ["null", "object"] }, @@ -68,16 +68,16 @@ "unsubscription_reason": { "type": ["null", "string"] }, "first_campaign": { "type": ["null", "string"] }, "total_sessions": { "type": ["null", "string"] }, - "mcr_id": { "type": ["null", "string"] }, + "mcr_id": { "type": ["null", "integer"] }, "last_campaign": { "type": ["null", "string"] }, "last_medium": { "type": ["null", "string"] }, "last_seen_chat": { "type": ["null", "string"] }, "first_medium": { "type": ["null", "string"] }, "other_unsubscription_reason": { "type": ["null", "string"] }, - "system_tags": { "type": ["null", "string"] }, + "system_tags": { "type": ["null", "array"] }, "latest_medium": { "type": ["null", "string"] }, "first_source": { "type": ["null", "string"] }, - "sms_subscription_status": { "type": ["null", "string"] }, + "sms_subscription_status": { "type": ["null", "integer"] }, "locale": { "type": ["null", "string"] }, "last_source": { "type": ["null", "string"] } } diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/lost_deals.json b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/lost_deals.json index 27067bba75d7..6f8dae4aaead 100644 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/lost_deals.json +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/lost_deals.json @@ -6,9 +6,9 @@ "properties": { "id": { "type": ["null", "integer"] }, "name": { "type": ["null", "string"] }, - "amount": { "type": ["null", "number"] }, + "amount": { "type": ["null", "string"] }, "currency_id": { "type": ["null", "integer"] }, - "base_currency_amount": { "type": ["null", "number"] }, + "base_currency_amount": { "type": ["null", "string"] }, "sales_account_id": { "type": ["null", "integer"] }, "deal_stage_id": { "type": ["null", "integer"] }, "deal_reason_id": { "type": ["null", "integer"] }, @@ -39,7 +39,7 @@ "tags": { "type": ["null", "array"] }, "last_contacted_sales_activity_mode": { "type": ["null", "string"] }, "last_contacted_via_sales_activity": { "type": ["null", "string"] }, - "expected_deal_value": { "type": ["null", "number"] }, + "expected_deal_value": { "type": ["null", "string"] }, "is_deleted": { "type": ["null", "boolean"] }, "team_user_ids": { "type": ["null", "string"] }, "avatar": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/open_deals.json b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/open_deals.json index 27067bba75d7..6f8dae4aaead 100644 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/open_deals.json +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/open_deals.json @@ -6,9 +6,9 @@ "properties": { "id": { "type": ["null", "integer"] }, "name": { "type": ["null", "string"] }, - "amount": { "type": ["null", "number"] }, + "amount": { "type": ["null", "string"] }, "currency_id": { "type": ["null", "integer"] }, - "base_currency_amount": { "type": ["null", "number"] }, + "base_currency_amount": { "type": ["null", "string"] }, "sales_account_id": { "type": ["null", "integer"] }, "deal_stage_id": { "type": ["null", "integer"] }, "deal_reason_id": { "type": ["null", "integer"] }, @@ -39,7 +39,7 @@ "tags": { "type": ["null", "array"] }, "last_contacted_sales_activity_mode": { "type": ["null", "string"] }, "last_contacted_via_sales_activity": { "type": ["null", "string"] }, - "expected_deal_value": { "type": ["null", "number"] }, + "expected_deal_value": { "type": ["null", "string"] }, "is_deleted": { "type": ["null", "boolean"] }, "team_user_ids": { "type": ["null", "string"] }, "avatar": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/open_tasks.json b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/open_tasks.json index 7c5325c9d8e6..5d44344dfe8a 100644 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/open_tasks.json +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/open_tasks.json @@ -12,14 +12,14 @@ "targetable_type": { "type": ["null", "string"] }, "Possible": { "type": ["null", "string"] }, "owner_id": { "type": ["null", "integer"] }, - "status": { "type": ["null", "string"] }, + "status": { "type": ["null", "integer"] }, "creater_id": { "type": ["null", "integer"] }, "created_at": { "type": ["null", "string"] }, "updated_at": { "type": ["null", "string"] }, "outcome_id": { "type": ["null", "integer"] }, "task_type_id": { "type": ["null", "integer"] }, "updater_id": { "type": ["null", "integer"] }, - "targetables": { "type": ["null", "string"] }, + "targetables": { "type": ["null", "array"] }, "completed_date": { "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/past_appointments.json b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/past_appointments.json index 018ba47b60f2..264f43f48153 100644 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/past_appointments.json +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/past_appointments.json @@ -20,7 +20,7 @@ "location": { "type": ["null", "string"] }, "created_at": { "type": ["null", "string"] }, "updated_at": { "type": ["null", "string"] }, - "is_allday": { "type": ["null", "string"] }, + "is_allday": { "type": ["null", "boolean"] }, "appointment_attendees_attributes": { "type": ["null", "array"] }, "outcome_id": { "type": ["null", "integer"] }, "latitude": { "type": ["null", "string"] }, @@ -31,7 +31,7 @@ "checkedout_latitude": { "type": ["null", "string"] }, "checkedin_duration": { "type": ["null", "string"] }, "checkedout_longitude": { "type": ["null", "string"] }, - "targetables": { "type": ["null", "string"] }, + "targetables": { "type": ["null", "array"] }, "can_checkin": { "type": ["null", "boolean"] }, "provider": { "type": ["null", "string"] }, "checkedout_at": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/upcoming_appointments.json b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/upcoming_appointments.json index 018ba47b60f2..264f43f48153 100644 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/upcoming_appointments.json +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/upcoming_appointments.json @@ -20,7 +20,7 @@ "location": { "type": ["null", "string"] }, "created_at": { "type": ["null", "string"] }, "updated_at": { "type": ["null", "string"] }, - "is_allday": { "type": ["null", "string"] }, + "is_allday": { "type": ["null", "boolean"] }, "appointment_attendees_attributes": { "type": ["null", "array"] }, "outcome_id": { "type": ["null", "integer"] }, "latitude": { "type": ["null", "string"] }, @@ -31,7 +31,7 @@ "checkedout_latitude": { "type": ["null", "string"] }, "checkedin_duration": { "type": ["null", "string"] }, "checkedout_longitude": { "type": ["null", "string"] }, - "targetables": { "type": ["null", "string"] }, + "targetables": { "type": ["null", "array"] }, "can_checkin": { "type": ["null", "boolean"] }, "provider": { "type": ["null", "string"] }, "checkedout_at": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/won_deals.json b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/won_deals.json index 27067bba75d7..6f8dae4aaead 100644 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/won_deals.json +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/schemas/won_deals.json @@ -6,9 +6,9 @@ "properties": { "id": { "type": ["null", "integer"] }, "name": { "type": ["null", "string"] }, - "amount": { "type": ["null", "number"] }, + "amount": { "type": ["null", "string"] }, "currency_id": { "type": ["null", "integer"] }, - "base_currency_amount": { "type": ["null", "number"] }, + "base_currency_amount": { "type": ["null", "string"] }, "sales_account_id": { "type": ["null", "integer"] }, "deal_stage_id": { "type": ["null", "integer"] }, "deal_reason_id": { "type": ["null", "integer"] }, @@ -39,7 +39,7 @@ "tags": { "type": ["null", "array"] }, "last_contacted_sales_activity_mode": { "type": ["null", "string"] }, "last_contacted_via_sales_activity": { "type": ["null", "string"] }, - "expected_deal_value": { "type": ["null", "number"] }, + "expected_deal_value": { "type": ["null", "string"] }, "is_deleted": { "type": ["null", "boolean"] }, "team_user_ids": { "type": ["null", "string"] }, "avatar": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/source.py b/airbyte-integrations/connectors/source-freshsales/source_freshsales/source.py index 45fba4481963..a2a6cc96dbd0 100644 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/source.py +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/source.py @@ -2,209 +2,16 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. +WARNING: Do not modify this file. +""" -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer - -# Basic full refresh stream -class FreshsalesStream(HttpStream, ABC): - - primary_key: str = "id" - order_field: str = "updated_at" - object_name: str = None - require_view_id: bool = False - filter_value: str = None - - transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - - def __init__(self, domain_name: str, **kwargs): - super().__init__(**kwargs) - self.domain_name = domain_name - self.page = 1 - - @property - def url_base(self) -> str: - return f"https://{self.domain_name}/crm/sales/api/" - - @property - def auth_headers(self) -> Mapping[str, Any]: - return self.authenticator.get_auth_header() - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - There is no next page token in the respond so incrementing the page param until there is no new result - """ - list_result = response.json().get(self.object_name, []) - if list_result: - self.page += 1 - return self.page - else: - return None - - def request_params(self, **kwargs) -> MutableMapping[str, Any]: - params = {"page": self.page, "sort": self.order_field, "sort_type": "asc"} - if self.filter_value: - params["filter"] = self.filter_value - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - json_response = response.json() or {} - records = json_response.get(self.object_name, []) if self.object_name else json_response - yield from records - - def _get_filters(self) -> List: - """ - Some streams require a filter_id to be passed in. This function gets all available filters. - """ - url = f"{self.url_base}{self.object_name}/filters" - try: - response = self._session.get(url=url, headers=self.auth_headers) - response.raise_for_status() - return response.json().get("filters") - except requests.exceptions.RequestException as e: - self.logger.error(f"Error occured while getting `Filters` for stream `{self.name}`, full message: {e}") - raise - - def get_view_id(self) -> int: - """ - This function finds a relevant filter_id among all available filters by its name. - """ - filters = self._get_filters() - return next(_filter["id"] for _filter in filters if _filter["name"] == self.filter_name) - - def path(self, **kwargs) -> str: - if self.require_view_id: - return f"{self.object_name}/view/{self.get_view_id()}" - else: - return self.object_name - - -class Contacts(FreshsalesStream): - """ - API docs: https://developers.freshworks.com/crm/api/#contacts - """ - - object_name = "contacts" - filter_name = "All Contacts" - require_view_id = True - - -class Accounts(FreshsalesStream): - """ - API docs: https://developers.freshworks.com/crm/api/#accounts - """ - - object_name = "sales_accounts" - filter_name = "All Accounts" - require_view_id = True - - -class Deals(FreshsalesStream): - object_name = "deals" - require_view_id = True - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - # This is to remove data form widget development. Keeping this in failed integration tests. - for record in super().parse_response(response): - record.pop("fc_widget_collaboration", None) - yield record - - -class OpenDeals(Deals): - """ - API docs: https://developers.freshworks.com/crm/api/#deals - """ - - filter_name = "Open Deals" - - -class WonDeals(Deals): - """ - API docs: https://developers.freshworks.com/crm/api/#deals - """ - - filter_name = "Won Deals" - - -class LostDeals(Deals): - """ - API docs: https://developers.freshworks.com/crm/api/#deals - """ - - filter_name = "Lost Deals" - - -class OpenTasks(FreshsalesStream): - """ - API docs: https://developers.freshworks.com/crm/api/#tasks - """ - - object_name = "tasks" - filter_value = "open" - - -class CompletedTasks(FreshsalesStream): - """ - API docs: https://developers.freshworks.com/crm/api/#tasks - """ - - object_name = "tasks" - filter_value = "completed" - - -class PastAppointments(FreshsalesStream): - """ - API docs: https://developers.freshworks.com/crm/api/#appointments - """ - - object_name = "appointments" - filter_value = "past" - - -class UpcomingAppointments(FreshsalesStream): - """ - API docs: https://developers.freshworks.com/crm/api/#appointments - """ - - object_name = "appointments" - filter_value = "upcoming" - - -# Source -class SourceFreshsales(AbstractSource): - @staticmethod - def get_input_stream_args(api_key: str, domain_name: str) -> Mapping[str, Any]: - return { - "authenticator": TokenAuthenticator(token=api_key, auth_method="Token"), - "domain_name": domain_name, - } - - def check_connection(self, logger, config) -> Tuple[bool, any]: - stream = Contacts(**self.get_input_stream_args(config["api_key"], config["domain_name"])) - try: - next(stream.read_records(sync_mode=None)) - return True, None - except requests.exceptions.RequestException as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - args = self.get_input_stream_args(config["api_key"], config["domain_name"]) - return [ - Contacts(**args), - Accounts(**args), - OpenDeals(**args), - WonDeals(**args), - LostDeals(**args), - OpenTasks(**args), - CompletedTasks(**args), - PastAppointments(**args), - UpcomingAppointments(**args), - ] +# Declarative Source +class SourceFreshsales(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/spec.json b/airbyte-integrations/connectors/source-freshsales/source_freshsales/spec.json deleted file mode 100644 index b3c15c2a28a8..000000000000 --- a/airbyte-integrations/connectors/source-freshsales/source_freshsales/spec.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/freshsales", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Freshsales Spec", - "type": "object", - "required": ["domain_name", "api_key"], - "additionalProperties": true, - "properties": { - "domain_name": { - "type": "string", - "title": "Domain Name", - "description": "The Name of your Freshsales domain", - "examples": ["mydomain.myfreshworks.com"] - }, - "api_key": { - "type": "string", - "title": "API Key", - "description": "Freshsales API Key. See here. The key is case sensitive.", - "airbyte_secret": true - } - } - } -} diff --git a/airbyte-integrations/connectors/source-freshsales/source_freshsales/spec.yaml b/airbyte-integrations/connectors/source-freshsales/source_freshsales/spec.yaml new file mode 100644 index 000000000000..4ee33199d1f3 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshsales/source_freshsales/spec.yaml @@ -0,0 +1,23 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/freshsales +connectionSpecification: + type: object + title: Freshsales Spec + $schema: http://json-schema.org/draft-07/schema# + required: + - domain_name + - api_key + properties: + domain_name: + type: string + order: 0 + title: Domain Name + description: "The Name of your Freshsales domain" + examples: + - "mydomain.myfreshworks.com" + api_key: + type: string + order: 1 + title: API Key + description: 'Freshsales API Key. See here. The key is case sensitive.' + airbyte_secret: true + additionalProperties: true diff --git a/airbyte-integrations/connectors/source-freshsales/unit_tests/__init__.py b/airbyte-integrations/connectors/source-freshsales/unit_tests/__init__.py deleted file mode 100644 index 46b7376756ec..000000000000 --- a/airbyte-integrations/connectors/source-freshsales/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-freshsales/unit_tests/conftest.py b/airbyte-integrations/connectors/source-freshsales/unit_tests/conftest.py deleted file mode 100644 index 73e06f0b1b16..000000000000 --- a/airbyte-integrations/connectors/source-freshsales/unit_tests/conftest.py +++ /dev/null @@ -1,19 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import json - -import pytest -from source_freshsales.source import SourceFreshsales - - -@pytest.fixture(scope="session", name="config") -def config_fixture(): - with open("secrets/config.json", "r") as config_file: - return json.load(config_file) - - -@pytest.fixture(name="stream_args") -def stream_args(config): - return SourceFreshsales().get_input_stream_args(config['api_key'], config["domain_name"]) diff --git a/airbyte-integrations/connectors/source-freshsales/unit_tests/test_source.py b/airbyte-integrations/connectors/source-freshsales/unit_tests/test_source.py deleted file mode 100644 index e2b32dbc924a..000000000000 --- a/airbyte-integrations/connectors/source-freshsales/unit_tests/test_source.py +++ /dev/null @@ -1,83 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock, patch - -import pytest -import requests -from source_freshsales.source import Contacts, FreshsalesStream, OpenDeals, OpenTasks, SourceFreshsales - - -def test_get_input_stream_args(config): - source = SourceFreshsales() - expected_keys = ["authenticator", "domain_name"] - actual = source.get_input_stream_args(config['api_key'], config["domain_name"]) - for key in expected_keys: - assert key in actual.keys() - - -def test_check_connection(mocker, config): - source = SourceFreshsales() - logger_mock = MagicMock() - assert source.check_connection(logger_mock, config) == (True, None) - - -def test_count_streams(mocker): - source = SourceFreshsales() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 9 - assert len(streams) == expected_streams_number - - -def test_url_base(stream_args): - stream = FreshsalesStream(**stream_args) - expected = f"https://{stream_args.get('domain_name')}/crm/sales/api/" - actual = stream.url_base - assert actual == expected - - -def test_next_page_token(stream_args, requests_mock): - stream = Contacts(**stream_args) - stream_filters = [{"id": 1, "name": stream.filter_name}] - with patch.object(stream, "_get_filters", return_value=stream_filters) as mock_method: - url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json={stream.name: [{'id': 123}]}) - response = requests.get(url) - assert stream.next_page_token(response) == 2 - mock_method.assert_called() - - -def test_request_params(stream_args): - stream = OpenTasks(**stream_args) - actual = stream.request_params() - expected = {'filter': 'open', 'page': 1, 'sort': 'updated_at', 'sort_type': 'asc'} - assert actual == expected - - -@pytest.mark.parametrize( - "stream, response, expected", - [ - (Contacts, [{'id': 123}], [{'id': 123}]), - (OpenDeals, [{'id': 234, "fc_widget_collaboration": {"test": "test"}}], [{'id': 234}]), - ], - ids=["Contacts", "OpenDeals"] -) -def test_parse_response(stream, response, expected, stream_args, requests_mock): - stream = stream(**stream_args) - stream_filters = [{"id": 1, "name": stream.filter_name}] - with patch.object(stream, "_get_filters", return_value=stream_filters) as mock_method: - url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json={stream.object_name: response}) - _resp = requests.get(url) - assert list(stream.parse_response(_resp)) == expected - mock_method.assert_called() - - -def test_path(stream_args): - stream = Contacts(**stream_args) - stream_filters = [{"id": 1, "name": stream.filter_name}] - with patch.object(stream, "_get_filters", return_value=stream_filters) as mock_method: - assert stream.path() == 'contacts/view/1' - mock_method.assert_called() diff --git a/airbyte-integrations/connectors/source-freshservice/Dockerfile b/airbyte-integrations/connectors/source-freshservice/Dockerfile index 7732b3d3d243..b8b34c49cf44 100644 --- a/airbyte-integrations/connectors/source-freshservice/Dockerfile +++ b/airbyte-integrations/connectors/source-freshservice/Dockerfile @@ -34,5 +34,5 @@ COPY source_freshservice ./source_freshservice ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.2.0 +LABEL io.airbyte.version=1.3.0 LABEL io.airbyte.name=airbyte/source-freshservice diff --git a/airbyte-integrations/connectors/source-freshservice/README.md b/airbyte-integrations/connectors/source-freshservice/README.md index bc19d433edc6..4be9ae0819b1 100644 --- a/airbyte-integrations/connectors/source-freshservice/README.md +++ b/airbyte-integrations/connectors/source-freshservice/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-freshservice:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/freshservice) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_freshservice/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-freshservice:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-freshservice build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-freshservice:airbyteDocker +An image will be built with the tag `airbyte/source-freshservice:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-freshservice:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-freshservice:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-freshservice:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-freshservice:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-freshservice test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-freshservice:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-freshservice:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-freshservice test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/freshservice.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-freshservice/acceptance-test-config.yml b/airbyte-integrations/connectors/source-freshservice/acceptance-test-config.yml index 5f2bc08d045b..e7d25b19ecf6 100644 --- a/airbyte-integrations/connectors/source-freshservice/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-freshservice/acceptance-test-config.yml @@ -39,7 +39,7 @@ acceptance_tests: bypass_reason: Test account does not have permissions - name: purchase_orders bypass_reason: Test account does not have permissions - incremental: + incremental: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-freshservice/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-freshservice/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-freshservice/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-freshservice/build.gradle b/airbyte-integrations/connectors/source-freshservice/build.gradle deleted file mode 100644 index 623148492311..000000000000 --- a/airbyte-integrations/connectors/source-freshservice/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_freshservice' -} diff --git a/airbyte-integrations/connectors/source-freshservice/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-freshservice/integration_tests/configured_catalog.json index 88cbb244c9f1..14bfe7e98d95 100644 --- a/airbyte-integrations/connectors/source-freshservice/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-freshservice/integration_tests/configured_catalog.json @@ -12,28 +12,37 @@ "sync_mode": "incremental", "destination_sync_mode": "overwrite" }, + { + "stream": { + "name": "requested_items", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "problems", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "supported_sync_modes": ["full_refresh"], "source_defined_cursor": true, - "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", + "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, { "stream": { "name": "changes", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "supported_sync_modes": ["full_refresh"], "source_defined_cursor": true, - "default_cursor_field": ["updated_at"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", + "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, { diff --git a/airbyte-integrations/connectors/source-freshservice/metadata.yaml b/airbyte-integrations/connectors/source-freshservice/metadata.yaml index 61c73b4c6b10..43c5dd48d13f 100644 --- a/airbyte-integrations/connectors/source-freshservice/metadata.yaml +++ b/airbyte-integrations/connectors/source-freshservice/metadata.yaml @@ -1,16 +1,16 @@ data: allowedHosts: hosts: - - TODO # Please change to the hostname of the source. + - ${domain_name}/api/v2 registries: oss: - enabled: false + enabled: true cloud: enabled: false connectorSubtype: api connectorType: source definitionId: 9bb85338-ea95-4c93-b267-6be89125b267 - dockerImageTag: 1.2.0 + dockerImageTag: 1.3.0 dockerRepository: airbyte/source-freshservice githubIssueLabel: source-freshservice icon: freshservice.svg @@ -21,5 +21,5 @@ data: supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/freshservice tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-freshservice/setup.py b/airbyte-integrations/connectors/source-freshservice/setup.py index 8ccedf94055a..422531f33640 100644 --- a/airbyte-integrations/connectors/source-freshservice/setup.py +++ b/airbyte-integrations/connectors/source-freshservice/setup.py @@ -6,14 +6,10 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk~=0.55.2", ] -TEST_REQUIREMENTS = [ - "pytest~=6.2", - "pytest-mock~=3.6.1", - "connector-acceptance-test", -] +TEST_REQUIREMENTS = ["pytest~=6.2", "pytest-mock~=3.6.1"] setup( name="source_freshservice", diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/manifest.yaml b/airbyte-integrations/connectors/source-freshservice/source_freshservice/manifest.yaml index fb2cbfe386c3..7d8414022f55 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/manifest.yaml +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/manifest.yaml @@ -78,12 +78,33 @@ definitions: parent_key: "id" partition_field: "parent_id" + requested_items_stream: + name: "requested_items" + primary_key: "id" + $parameters: + path_extractor: "requested_items" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + path: "tickets/{{ stream_slice.parent_id }}/requested_items" + error_handler: + type: DefaultErrorHandler + response_filters: + - http_codes: [404] + action: IGNORE + error_message: No data collected + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/tickets_stream" + parent_key: "id" + partition_field: "parent_id" + problems_stream: $ref: "#/definitions/base_stream" name: "problems" primary_key: "id" - incremental_sync: - $ref: "#/definitions/incremental_base" $parameters: path_extractor: "problems" path: "/problems" @@ -92,8 +113,6 @@ definitions: $ref: "#/definitions/base_stream" name: "changes" primary_key: "id" - incremental_sync: - $ref: "#/definitions/incremental_base" $parameters: path_extractor: "changes" path: "/changes" @@ -186,6 +205,7 @@ streams: - "#/definitions/assets_stream" - "#/definitions/purchase_orders_stream" - "#/definitions/software_stream" + - "#/definitions/requested_items_stream" check: type: CheckStream diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/agents.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/agents.json index 005af789f286..1b2d07f0fe09 100644 --- a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/agents.json +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/agents.json @@ -27,6 +27,12 @@ "mobile_phone_number": { "type": ["null", "string"] }, + "member_of_pending_approval": { + "type": ["null", "array"] + }, + "observer_of_pending_approval": { + "type": ["null", "array"] + }, "department_ids": { "type": ["null", "array"] }, diff --git a/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/requested_items.json b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/requested_items.json new file mode 100644 index 000000000000..909a9ed2621c --- /dev/null +++ b/airbyte-integrations/connectors/source-freshservice/source_freshservice/schemas/requested_items.json @@ -0,0 +1,47 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "stage": { + "type": ["null", "integer"] + }, + "loaned": { + "type": ["null", "boolean"] + }, + "cost_per_request": { + "type": ["null", "number"] + }, + "remarks": { + "type": ["null", "string"] + }, + "delivery_time": { + "type": ["null", "number"] + }, + "is_parent": { + "type": ["null", "boolean"] + }, + "service_item_id": { + "type": ["null", "integer"] + }, + "service_item_name": { + "type": ["null", "string"] + }, + "custom_fields": { + "type": ["null", "object"], + "additionalProperties": true + } + } +} diff --git a/airbyte-integrations/connectors/source-fullstory/README.md b/airbyte-integrations/connectors/source-fullstory/README.md index 731e849dc434..8780501576a2 100644 --- a/airbyte-integrations/connectors/source-fullstory/README.md +++ b/airbyte-integrations/connectors/source-fullstory/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-fullstory:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/fullstory) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_fullstory/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-fullstory:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-fullstory build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-fullstory:airbyteDocker +An image will be built with the tag `airbyte/source-fullstory:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-fullstory:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-fullstory:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-fullstory:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-fullstory:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-fullstory test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-fullstory:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-fullstory:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-fullstory test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/fullstory.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-fullstory/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-fullstory/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-fullstory/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-fullstory/build.gradle b/airbyte-integrations/connectors/source-fullstory/build.gradle deleted file mode 100644 index c4ae1e3fc85d..000000000000 --- a/airbyte-integrations/connectors/source-fullstory/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_fullstory' -} diff --git a/airbyte-integrations/connectors/source-gainsight-px/README.md b/airbyte-integrations/connectors/source-gainsight-px/README.md index 7373411b868a..5504ac343822 100644 --- a/airbyte-integrations/connectors/source-gainsight-px/README.md +++ b/airbyte-integrations/connectors/source-gainsight-px/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-gainsight-px:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/gainsight-px) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_gainsight_px/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-gainsight-px:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-gainsight-px build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-gainsight-px:airbyteDocker +An image will be built with the tag `airbyte/source-gainsight-px:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-gainsight-px:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-gainsight-px:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-gainsight-px:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-gainsight-px:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-gainsight-px test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-gainsight-px:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-gainsight-px:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-gainsight-px test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/gainsight-px.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-gainsight-px/acceptance-test-config.yml b/airbyte-integrations/connectors/source-gainsight-px/acceptance-test-config.yml index bd7e46aae2c5..cb5dc426f507 100644 --- a/airbyte-integrations/connectors/source-gainsight-px/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-gainsight-px/acceptance-test-config.yml @@ -31,20 +31,20 @@ acceptance_tests: bypass_reason: "Sandbox account cannot seed the stream" - name: users bypass_reason: "Sandbox account cannot seed the stream" -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.jsonl" -# extra_fields: no -# exact_order: no -# extra_records: yes - incremental: + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.jsonl" + # extra_fields: no + # exact_order: no + # extra_records: yes + incremental: bypass_reason: "This connector does not implement incremental sync" -# TODO uncomment this block this block if your connector implements incremental sync: -# tests: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# future_state: -# future_state_path: "integration_tests/abnormal_state.json" + # TODO uncomment this block this block if your connector implements incremental sync: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-gainsight-px/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-gainsight-px/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-gainsight-px/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-gainsight-px/build.gradle b/airbyte-integrations/connectors/source-gainsight-px/build.gradle deleted file mode 100644 index 735ea0dbde01..000000000000 --- a/airbyte-integrations/connectors/source-gainsight-px/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_gainsight_px' -} diff --git a/airbyte-integrations/connectors/source-gcs/Dockerfile b/airbyte-integrations/connectors/source-gcs/Dockerfile deleted file mode 100644 index 0d9dda5d897c..000000000000 --- a/airbyte-integrations/connectors/source-gcs/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" - -WORKDIR /airbyte/integration_code -COPY source_gcs ./source_gcs -COPY setup.py ./ -COPY main.py ./ -RUN pip install . - -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.2.0 -LABEL io.airbyte.name=airbyte/source-gcs diff --git a/airbyte-integrations/connectors/source-gcs/README.md b/airbyte-integrations/connectors/source-gcs/README.md index 3695276558a2..6938e3d96c0c 100644 --- a/airbyte-integrations/connectors/source-gcs/README.md +++ b/airbyte-integrations/connectors/source-gcs/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-gcs:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/gcs) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_gcs/spec.yaml` file. @@ -54,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-gcs:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-gcs build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-gcs:airbyteDocker +An image will be built with the tag `airbyte/source-gcs:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-gcs:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,44 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-gcs:dev check --config docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-gcs:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-gcs:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-gcs test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-gcs:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-gcs:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -122,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-gcs test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/gcs.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-gcs/acceptance-test-config.yml b/airbyte-integrations/connectors/source-gcs/acceptance-test-config.yml index c98deb180b7d..2114c72238d1 100644 --- a/airbyte-integrations/connectors/source-gcs/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-gcs/acceptance-test-config.yml @@ -4,24 +4,43 @@ connector_image: airbyte/source-gcs:dev acceptance_tests: spec: tests: - - spec_path: "source_gcs/spec.yaml" + - spec_path: integration_tests/spec.json + backward_compatibility_tests_config: + disable_for_version: 0.2.0 connection: tests: - config_path: "secrets/config.json" - status: "succeed" + status: succeed + - config_path: "secrets/old_config.json" + status: succeed - config_path: "integration_tests/invalid_config.json" - status: "failed" + status: exception discovery: tests: - config_path: "secrets/config.json" + timeout_seconds: 2400 basic_read: + tests: + - config_path: "secrets/old_config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + expect_trace_message_on_failure: false + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + expect_trace_message_on_failure: false + incremental: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] - incremental: - bypass_reason: "This connector does not implement incremental sync" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + ignored_fields: + example_1: + - name: _ab_source_file_url + bypass_reason: "Uri has autogenerated token in query params" + example_2: + - name: _ab_source_file_url + bypass_reason: "Uri has autogenerated token in query params" diff --git a/airbyte-integrations/connectors/source-gcs/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-gcs/acceptance-test-docker.sh deleted file mode 100755 index a8d6ac4bb608..000000000000 --- a/airbyte-integrations/connectors/source-gcs/acceptance-test-docker.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env sh - -# Build latest connector image -docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) - -# Pull latest acctest image -docker pull airbyte/connector-acceptance-test:latest - -# Run -docker run --rm -it \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /tmp:/tmp \ - -v $(pwd):/test_input \ - airbyte/connector-acceptance-test \ - --acceptance-test-config /test_input - diff --git a/airbyte-integrations/connectors/source-gcs/build.gradle b/airbyte-integrations/connectors/source-gcs/build.gradle deleted file mode 100644 index ad9dac856011..000000000000 --- a/airbyte-integrations/connectors/source-gcs/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_gcs_singer' -} diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-gcs/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..18702dfa6966 --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/abnormal_state.json @@ -0,0 +1,30 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "_ab_source_file_last_modified": "2023-02-27T10:34:32.664000Z_https://storage.googleapis.com/airbyte-integration-test-source-gcs/test_folder/example_1.csv", + "history": { + "https://storage.googleapis.com/airbyte-integration-test-source-gcs/test_folder/example_1.csv": "2023-02-27T10:34:32.664000Z" + } + }, + "stream_descriptor": { + "name": "example_1" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "_ab_source_file_last_modified": "2023-02-27T10:34:32.680000Z_https://storage.googleapis.com/airbyte-integration-test-source-gcs/test_folder/example_2.csv", + "history": { + "https://storage.googleapis.com/airbyte-integration-test-source-gcs/test_folder/example_2.csv": "2023-02-27T10:34:32.680000Z" + } + }, + "stream_descriptor": { + "name": "example_2" + } + } + } +] diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-gcs/integration_tests/configured_catalog.json index 9836beb15b42..f6dd1106608b 100644 --- a/airbyte-integrations/connectors/source-gcs/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/configured_catalog.json @@ -4,18 +4,18 @@ "stream": { "name": "example_1", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "full_refresh", + "sync_mode": "incremental", "destination_sync_mode": "overwrite" }, { "stream": { "name": "example_2", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "full_refresh", + "sync_mode": "incremental", "destination_sync_mode": "overwrite" } ] diff --git a/airbyte-integrations/connectors/source-gcs/integration_tests/spec.json b/airbyte-integrations/connectors/source-gcs/integration_tests/spec.json new file mode 100644 index 000000000000..5f69da41c02a --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/integration_tests/spec.json @@ -0,0 +1,259 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/gcs", + "connectionSpecification": { + "title": "Config", + "description": "NOTE: When this Spec is changed, legacy_config_transformer.py must also be\nmodified to uptake the changes because it is responsible for converting\nlegacy GCS configs into file based configs using the File-Based CDK.", + "type": "object", + "properties": { + "start_date": { + "title": "Start Date", + "description": "UTC date and time in the format 2017-01-25T00:00:00.000000Z. Any file modified before this date will not be replicated.", + "examples": ["2021-01-01T00:00:00.000000Z"], + "format": "date-time", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{6}Z$", + "pattern_descriptor": "YYYY-MM-DDTHH:mm:ss.SSSSSSZ", + "order": 1, + "type": "string" + }, + "streams": { + "title": "The list of streams to sync", + "description": "Each instance of this configuration defines a stream. Use this to define which files belong in the stream, their format, and how they should be parsed and validated. When sending data to warehouse destination such as Snowflake or BigQuery, each stream is a separate table.", + "order": 3, + "type": "array", + "items": { + "title": "SourceGCSStreamConfig", + "type": "object", + "properties": { + "name": { + "title": "Name", + "description": "The name of the stream.", + "order": 0, + "type": "string" + }, + "globs": { + "title": "Globs", + "description": "The pattern used to specify which files should be selected from the file system. For more information on glob pattern matching look here.", + "order": 1, + "type": "array", + "items": { + "type": "string" + } + }, + "legacy_prefix": { + "title": "Legacy Prefix", + "description": "The path prefix configured in previous versions of the GCS connector. This option is deprecated in favor of a single glob.", + "airbyte_hidden": true, + "type": "string" + }, + "validation_policy": { + "title": "Validation Policy", + "description": "The name of the validation policy that dictates sync behavior when a record does not adhere to the stream schema.", + "default": "Emit Record", + "enum": ["Emit Record", "Skip Record", "Wait for Discover"] + }, + "input_schema": { + "title": "Input Schema", + "description": "The schema that will be used to validate records extracted from the file. This will override the stream schema that is auto-detected from incoming files.", + "type": "string" + }, + "primary_key": { + "title": "Primary Key", + "description": "The column or columns (for a composite key) that serves as the unique identifier of a record. If empty, the primary key will default to the parser's default primary key.", + "airbyte_hidden": true, + "type": "string" + }, + "days_to_sync_if_history_is_full": { + "title": "Days To Sync If History Is Full", + "description": "When the state history of the file store is full, syncs will only read files that were last modified in the provided day range.", + "default": 3, + "type": "integer" + }, + "format": { + "title": "Format", + "description": "The configuration options that are used to alter how to read incoming files that deviate from the standard formatting.", + "order": 2, + "type": "object", + "oneOf": [ + { + "title": "CSV Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "csv", + "const": "csv", + "type": "string" + }, + "delimiter": { + "title": "Delimiter", + "description": "The character delimiting individual cells in the CSV data. This may only be a 1-character string. For tab-delimited data enter '\\t'.", + "default": ",", + "type": "string" + }, + "quote_char": { + "title": "Quote Character", + "description": "The character used for quoting CSV values. To disallow quoting, make this field blank.", + "default": "\"", + "type": "string" + }, + "escape_char": { + "title": "Escape Character", + "description": "The character used for escaping special characters. To disallow escaping, leave this field blank.", + "type": "string" + }, + "encoding": { + "title": "Encoding", + "description": "The character encoding of the CSV data. Leave blank to default to UTF8. See list of python encodings for allowable options.", + "default": "utf8", + "type": "string" + }, + "double_quote": { + "title": "Double Quote", + "description": "Whether two quotes in a quoted CSV value denote a single quote in the data.", + "default": true, + "type": "boolean" + }, + "null_values": { + "title": "Null Values", + "description": "A set of case-sensitive strings that should be interpreted as null values. For example, if the value 'NA' should be interpreted as null, enter 'NA' in this field.", + "default": [], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "strings_can_be_null": { + "title": "Strings Can Be Null", + "description": "Whether strings can be interpreted as null values. If true, strings that match the null_values set will be interpreted as null. If false, strings that match the null_values set will be interpreted as the string itself.", + "default": true, + "type": "boolean" + }, + "skip_rows_before_header": { + "title": "Skip Rows Before Header", + "description": "The number of rows to skip before the header row. For example, if the header row is on the 3rd row, enter 2 in this field.", + "default": 0, + "type": "integer" + }, + "skip_rows_after_header": { + "title": "Skip Rows After Header", + "description": "The number of rows to skip after the header row.", + "default": 0, + "type": "integer" + }, + "header_definition": { + "title": "CSV Header Definition", + "description": "How headers will be defined. `User Provided` assumes the CSV does not have a header row and uses the headers provided and `Autogenerated` assumes the CSV does not have a header row and the CDK will generate headers using for `f{i}` where `i` is the index starting from 0. Else, the default behavior is to use the header from the CSV file. If a user wants to autogenerate or provide column names for a CSV having headers, they can skip rows.", + "default": { + "header_definition_type": "From CSV" + }, + "oneOf": [ + { + "title": "From CSV", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "From CSV", + "const": "From CSV", + "type": "string" + } + }, + "required": ["header_definition_type"] + }, + { + "title": "Autogenerated", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "Autogenerated", + "const": "Autogenerated", + "type": "string" + } + }, + "required": ["header_definition_type"] + }, + { + "title": "User Provided", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "User Provided", + "const": "User Provided", + "type": "string" + }, + "column_names": { + "title": "Column Names", + "description": "The column names that will be used while emitting the CSV records", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": ["column_names", "header_definition_type"] + } + ], + "type": "object" + }, + "true_values": { + "title": "True Values", + "description": "A set of case-sensitive strings that should be interpreted as true values.", + "default": ["y", "yes", "t", "true", "on", "1"], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "false_values": { + "title": "False Values", + "description": "A set of case-sensitive strings that should be interpreted as false values.", + "default": ["n", "no", "f", "false", "off", "0"], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "inference_type": { + "title": "Inference Type", + "description": "How to infer the types of the columns. If none, inference default to strings.", + "default": "None", + "airbyte_hidden": true, + "enum": ["None", "Primitive Types Only"] + } + }, + "required": ["filetype"] + } + ] + }, + "schemaless": { + "title": "Schemaless", + "description": "When enabled, syncs will not validate or structure records against the stream's schema.", + "default": false, + "type": "boolean" + } + }, + "required": ["name", "format"] + } + }, + "service_account": { + "title": "Service Account Information", + "description": "Enter your Google Cloud service account key in JSON format", + "airbyte_secret": true, + "order": 0, + "type": "string" + }, + "bucket": { + "title": "Bucket", + "description": "Name of the GCS bucket where the file(s) exist.", + "order": 2, + "type": "string" + } + }, + "required": ["streams", "service_account", "bucket"] + } +} diff --git a/airbyte-integrations/connectors/source-gcs/main.py b/airbyte-integrations/connectors/source-gcs/main.py index 74e5bf63ab7e..c98b5b943cc7 100644 --- a/airbyte-integrations/connectors/source-gcs/main.py +++ b/airbyte-integrations/connectors/source-gcs/main.py @@ -5,9 +5,11 @@ import sys -from airbyte_cdk.entrypoint import launch -from source_gcs import SourceGCS +from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch +from source_gcs import Config, Cursor, SourceGCS, SourceGCSStreamReader if __name__ == "__main__": - source = SourceGCS() + _args = sys.argv[1:] + catalog_path = AirbyteEntrypoint.extract_catalog(_args) + source = SourceGCS(SourceGCSStreamReader(), Config, catalog_path, cursor_cls=Cursor) launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-gcs/metadata.yaml b/airbyte-integrations/connectors/source-gcs/metadata.yaml index 927b6b006180..1ed2d6757783 100644 --- a/airbyte-integrations/connectors/source-gcs/metadata.yaml +++ b/airbyte-integrations/connectors/source-gcs/metadata.yaml @@ -2,10 +2,12 @@ data: ab_internal: ql: 200 sl: 100 + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: file connectorType: source definitionId: 2a8c41ae-8c23-4be0-a73f-2ab10ca1a820 - dockerImageTag: 0.2.0 + dockerImageTag: 0.3.4 dockerRepository: airbyte/source-gcs documentationUrl: https://docs.airbyte.com/integrations/sources/gcs githubIssueLabel: source-gcs diff --git a/airbyte-integrations/connectors/source-gcs/setup.py b/airbyte-integrations/connectors/source-gcs/setup.py index 73669bd2ee21..b9574a838971 100644 --- a/airbyte-integrations/connectors/source-gcs/setup.py +++ b/airbyte-integrations/connectors/source-gcs/setup.py @@ -5,7 +5,12 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.2", "google-cloud-storage==2.5.0", "pandas==1.5.3"] +MAIN_REQUIREMENTS = [ + "airbyte-cdk[file-based]>=0.55.5", + "google-cloud-storage==2.12.0", + "smart-open[s3]==5.1.0", + "pandas==1.5.3", +] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/__init__.py b/airbyte-integrations/connectors/source-gcs/source_gcs/__init__.py index 917e71deaf26..852a5b50e206 100644 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/__init__.py +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/__init__.py @@ -2,7 +2,16 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - +from .config import Config +from .cursor import Cursor +from .legacy_config_transformer import LegacyConfigTransformer from .source import SourceGCS +from .stream_reader import SourceGCSStreamReader -__all__ = ["SourceGCS"] +__all__ = [ + "Config", + "Cursor", + "LegacyConfigTransformer", + "SourceGCS", + "SourceGCSStreamReader", +] diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/config.py b/airbyte-integrations/connectors/source-gcs/source_gcs/config.py new file mode 100644 index 000000000000..04dc1e16b8d3 --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/config.py @@ -0,0 +1,86 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import List, Optional + +from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec +from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig +from pydantic import AnyUrl, Field + + +class SourceGCSStreamConfig(FileBasedStreamConfig): + name: str = Field(title="Name", description="The name of the stream.", order=0) + globs: Optional[List[str]] = Field( + title="Globs", + description="The pattern used to specify which files should be selected from the file system. For more information on glob " + 'pattern matching look here.', + order=1, + ) + format: CsvFormat = Field( + title="Format", + description="The configuration options that are used to alter how to read incoming files that deviate from " + "the standard formatting.", + order=2, + ) + legacy_prefix: Optional[str] = Field( + title="Legacy Prefix", + description="The path prefix configured in previous versions of the GCS connector. " + "This option is deprecated in favor of a single glob.", + airbyte_hidden=True, + ) + + +class Config(AbstractFileBasedSpec): + """ + NOTE: When this Spec is changed, legacy_config_transformer.py must also be + modified to uptake the changes because it is responsible for converting + legacy GCS configs into file based configs using the File-Based CDK. + """ + + service_account: str = Field( + title="Service Account Information", + airbyte_secret=True, + description=( + "Enter your Google Cloud " + '' + "service account key in JSON format" + ), + order=0, + ) + + bucket: str = Field(title="Bucket", description="Name of the GCS bucket where the file(s) exist.", order=2) + + streams: List[SourceGCSStreamConfig] = Field( + title="The list of streams to sync", + description=( + "Each instance of this configuration defines a stream. " + "Use this to define which files belong in the stream, their format, and how they should be " + "parsed and validated. When sending data to warehouse destination such as Snowflake or " + "BigQuery, each stream is a separate table." + ), + order=3, + ) + + @classmethod + def documentation_url(cls) -> AnyUrl: + """ + Returns the documentation URL. + """ + return AnyUrl("https://docs.airbyte.com/integrations/sources/gcs", scheme="https") + + @staticmethod + def replace_enum_allOf_and_anyOf(schema): + """ + Replace allOf with anyOf when appropriate in the schema with one value. + """ + objects_to_check = schema["properties"]["streams"]["items"]["properties"]["format"] + if len(objects_to_check.get("allOf", [])) == 1: + objects_to_check["anyOf"] = objects_to_check.pop("allOf") + + return super(Config, Config).replace_enum_allOf_and_anyOf(schema) + + @staticmethod + def remove_discriminator(schema) -> None: + pass diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/cursor.py b/airbyte-integrations/connectors/source-gcs/source_gcs/cursor.py new file mode 100644 index 000000000000..e55bbd20e29c --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/cursor.py @@ -0,0 +1,58 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from datetime import datetime + +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.sources.file_based.stream.cursor import DefaultFileBasedCursor + + +class Cursor(DefaultFileBasedCursor): + @staticmethod + def get_file_uri(file: RemoteFile) -> str: + return file.uri.split("?")[0] + + def add_file(self, file: RemoteFile) -> None: + uri = self.get_file_uri(file) + self._file_to_datetime_history[uri] = file.last_modified.strftime(self.DATE_TIME_FORMAT) + if len(self._file_to_datetime_history) > self.DEFAULT_MAX_HISTORY_SIZE: + # Get the earliest file based on its last modified date and its uri + oldest_file = self._compute_earliest_file_in_history() + if oldest_file: + del self._file_to_datetime_history[oldest_file.uri] + else: + raise Exception( + "The history is full but there is no files in the history. This should never happen and might be indicative of a bug in the CDK." + ) + + def _should_sync_file(self, file: RemoteFile, logger: logging.Logger) -> bool: + uri = self.get_file_uri(file) + if uri in self._file_to_datetime_history: + # If the file's uri is in the history, we should sync the file if it has been modified since it was synced + updated_at_from_history = datetime.strptime(self._file_to_datetime_history[uri], self.DATE_TIME_FORMAT) + if file.last_modified < updated_at_from_history: + logger.warning( + f"The file {uri}'s last modified date is older than the last time it was synced. This is unexpected. Skipping the file." + ) + else: + return file.last_modified > updated_at_from_history + return file.last_modified > updated_at_from_history + if self._is_history_full(): + if self._initial_earliest_file_in_history is None: + return True + if file.last_modified > self._initial_earliest_file_in_history.last_modified: + # If the history is partial and the file's datetime is strictly greater than the earliest file in the history, + # we should sync it + return True + elif file.last_modified == self._initial_earliest_file_in_history.last_modified: + # If the history is partial and the file's datetime is equal to the earliest file in the history, + # we should sync it if its uri is strictly greater than the earliest file in the history + return uri > self._initial_earliest_file_in_history.uri + else: + # Otherwise, only sync the file if it has been modified since the start of the time window + return file.last_modified >= self.get_start_time() + else: + # The file is not in the history and the history is complete. We know we need to sync the file + return True diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/helpers.py b/airbyte-integrations/connectors/source-gcs/source_gcs/helpers.py index 164e6852c65f..3962cbef5458 100644 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/helpers.py +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/helpers.py @@ -12,16 +12,16 @@ def get_gcs_client(config): - credentials = service_account.Credentials.from_service_account_info(json.loads(config.get("service_account"))) + credentials = service_account.Credentials.from_service_account_info(json.loads(config.service_account)) client = storage.Client(credentials=credentials) return client def get_gcs_blobs(config): client = get_gcs_client(config) - bucket = client.get_bucket(config.get("gcs_bucket")) - blobs = bucket.list_blobs(prefix=config.get("gcs_path")) - # TODO: only support CSV intially. Change this check if implementing other file formats. + bucket = client.get_bucket(config.gcs_bucket) + blobs = bucket.list_blobs(prefix=config.gcs_path) + # TODO: only support CSV initially. Change this check if implementing other file formats. blobs = [blob for blob in blobs if "csv" in blob.name.lower()] return blobs diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/legacy_config_transformer.py b/airbyte-integrations/connectors/source-gcs/source_gcs/legacy_config_transformer.py new file mode 100644 index 000000000000..2a06e386370d --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/legacy_config_transformer.py @@ -0,0 +1,45 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Dict, Mapping + +from source_gcs.spec import SourceGCSSpec + +from .helpers import get_gcs_blobs, get_stream_name + + +class LegacyConfigTransformer: + """ + Transforms GCS source configs from legacy format to be compatible + with the new GCS source built with the file-based CDK. + """ + + @staticmethod + def _create_stream(blob: Any, legacy_prefix: str) -> Dict[str, Any]: + """ + Create a stream dict from a blob. + + :param blob: The blob from which to create the stream. + :param legacy_prefix: The legacy prefix path on GCS. + :return: A dictionary representing the stream. + """ + return { + "name": get_stream_name(blob), + "legacy_prefix": f"{legacy_prefix}/{blob.name.split('/')[-1]}", + "validation_policy": "Emit Record", + "format": {"filetype": "csv"}, + } + + @classmethod + def convert(cls, legacy_config: SourceGCSSpec) -> Mapping[str, Any]: + """ + Convert a legacy configuration to a transformed configuration. + + :param legacy_config: Legacy configuration of type SourceGCSSpec. + :return: Transformed configuration as a dictionary. + """ + blobs = get_gcs_blobs(legacy_config) + streams = [cls._create_stream(blob, legacy_config.gcs_path) for blob in blobs] + + return {"bucket": legacy_config.gcs_bucket, "service_account": legacy_config.service_account, "streams": streams} diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/source.py b/airbyte-integrations/connectors/source-gcs/source_gcs/source.py index 88975ef6b68c..b33ff2b87eb4 100644 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/source.py +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/source.py @@ -3,70 +3,28 @@ # -import json -from datetime import datetime -from typing import Dict, Generator +from typing import Any, Mapping -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import ( - AirbyteCatalog, - AirbyteConnectionStatus, - AirbyteMessage, - AirbyteRecordMessage, - AirbyteStream, - ConfiguredAirbyteCatalog, - Status, - Type, -) -from airbyte_cdk.sources import Source +from airbyte_cdk.config_observation import emit_configuration_as_airbyte_control_message +from airbyte_cdk.sources.file_based.file_based_source import FileBasedSource +from source_gcs.legacy_config_transformer import LegacyConfigTransformer +from source_gcs.spec import SourceGCSSpec -from .helpers import construct_file_schema, get_gcs_blobs, get_stream_name, read_csv_file - -class SourceGCS(Source): - def check(self, logger: AirbyteLogger, config: json) -> AirbyteConnectionStatus: +class SourceGCS(FileBasedSource): + def read_config(self, config_path: str) -> Mapping[str, Any]: """ - Check to see if a client can be created and list the files in the bucket. + Override the default read_config to transform the legacy config format + into the new one before validating it against the new spec. """ - try: - blobs = get_gcs_blobs(config) - if not blobs: - return AirbyteConnectionStatus(status=Status.FAILED, message="No compatible file found in bucket") - return AirbyteConnectionStatus(status=Status.SUCCEEDED) - except Exception as e: - return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {str(e)}") - - def discover(self, logger: AirbyteLogger, config: json) -> AirbyteCatalog: - streams = [] - - blobs = get_gcs_blobs(config) - for blob in blobs: - # Read the first 0.1MB of the file to determine schema - df = read_csv_file(blob, read_header_only=True) - stream_name = get_stream_name(blob) - json_schema = construct_file_schema(df) - streams.append(AirbyteStream(name=stream_name, json_schema=json_schema, supported_sync_modes=["full_refresh"])) - - return AirbyteCatalog(streams=streams) - - def read( - self, logger: AirbyteLogger, config: json, catalog: ConfiguredAirbyteCatalog, state: Dict[str, any] - ) -> Generator[AirbyteMessage, None, None]: - logger.info("Start reading") - blobs = get_gcs_blobs(config) - - # Read only selected stream(s) - selected_streams = [configged_stream.stream.name for configged_stream in catalog.streams] - selected_blobs = [blob for blob in blobs if get_stream_name(blob) in selected_streams] - - for blob in selected_blobs: - logger.info(blob.name) - df = read_csv_file(blob) - stream_name = get_stream_name(blob) - for _, row in df.iterrows(): - row_dict = row.to_dict() - row_dict = {k: str(v) for k, v in row_dict.items()} - yield AirbyteMessage( - type=Type.RECORD, - record=AirbyteRecordMessage(stream=stream_name, data=row_dict, emitted_at=int(datetime.now().timestamp()) * 1000), - ) + config = super().read_config(config_path) + if not self._is_file_based_config(config): + parsed_legacy_config = SourceGCSSpec(**config) + converted_config = LegacyConfigTransformer.convert(parsed_legacy_config) + emit_configuration_as_airbyte_control_message(converted_config) + return converted_config + return config + + @staticmethod + def _is_file_based_config(config: Mapping[str, Any]) -> bool: + return "streams" in config diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/spec.py b/airbyte-integrations/connectors/source-gcs/source_gcs/spec.py new file mode 100644 index 000000000000..790554aeef61 --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/spec.py @@ -0,0 +1,39 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from pydantic import BaseModel, Field + + +class SourceGCSSpec(BaseModel): + """ + The SourceGCSSpec class defines the expected input configuration + for the Google Cloud Storage (GCS) source. It uses Pydantic for data + validation through the defined data models. + + Note: When this Spec is changed, ensure that the legacy_config_transformer.py + is also modified to accommodate the changes, as it is responsible for + converting legacy GCS configs into file based configs using the File-Based CDK. + """ + + gcs_bucket: str = Field( + title="GCS bucket", + description="GCS bucket name", + order=0, + ) + + gcs_path: str = Field( + title="GCS Path", + description="GCS path to data", + order=1, + ) + + service_account: str = Field( + title="Service Account Information.", + airbyte_secret=True, + description=( + 'Enter your Google Cloud ' + "service account key in JSON format" + ), + ) diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/spec.yaml b/airbyte-integrations/connectors/source-gcs/source_gcs/spec.yaml deleted file mode 100644 index 6b042e975071..000000000000 --- a/airbyte-integrations/connectors/source-gcs/source_gcs/spec.yaml +++ /dev/null @@ -1,25 +0,0 @@ -documentationUrl: https://docsurl.com -connectionSpecification: - $schema: http://json-schema.org/draft-07/schema# - title: Gcs Spec - type: object - required: - - gcs_bucket - - gcs_path - - service_account - properties: - gcs_bucket: - type: string - title: GCS bucket - description: GCS bucket name - gcs_path: - type: string - title: GCS Path - description: GCS path to data - service_account: - type: string - title: Service Account Information. - description: 'Enter your Google Cloud service account key in JSON format' - airbyte_secret: true - examples: - - '{ "type": "service_account", "project_id": YOUR_PROJECT_ID, "private_key_id": YOUR_PRIVATE_KEY, ... }' diff --git a/airbyte-integrations/connectors/source-gcs/source_gcs/stream_reader.py b/airbyte-integrations/connectors/source-gcs/source_gcs/stream_reader.py new file mode 100644 index 000000000000..ec44dd27048e --- /dev/null +++ b/airbyte-integrations/connectors/source-gcs/source_gcs/stream_reader.py @@ -0,0 +1,106 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import itertools +import json +import logging +from datetime import datetime, timedelta +from io import IOBase +from typing import Iterable, List, Optional + +import pytz +import smart_open +from airbyte_cdk.sources.file_based.exceptions import ErrorListingFiles, FileBasedSourceError +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from google.cloud import storage +from google.oauth2 import service_account +from source_gcs.config import Config + +ERROR_MESSAGE_ACCESS = ( + "We don't have access to {uri}. The file appears to have become unreachable during sync." + "Check whether key {uri} exists in `{bucket}` bucket and/or has proper ACL permissions" +) +FILE_FORMAT = "csv" # TODO: Change if other file formats are implemented + + +class SourceGCSStreamReader(AbstractFileBasedStreamReader): + """ + Stream reader for Google Cloud Storage (GCS). + """ + + def __init__(self): + super().__init__() + self._gcs_client = None + self._config = None + + @property + def config(self) -> Config: + return self._config + + @config.setter + def config(self, value: Config): + assert isinstance(value, Config), "Config must be an instance of the expected Config class." + self._config = value + + def _initialize_gcs_client(self): + if self.config is None: + raise ValueError("Source config is missing; cannot create the GCS client.") + if self._gcs_client is None: + credentials = self._get_credentials() + self._gcs_client = storage.Client(credentials=credentials) + return self._gcs_client + + def _get_credentials(self): + return service_account.Credentials.from_service_account_info(json.loads(self.config.service_account)) + + @property + def gcs_client(self) -> storage.Client: + return self._initialize_gcs_client() + + def get_matching_files(self, globs: List[str], prefix: Optional[str], logger: logging.Logger) -> Iterable[RemoteFile]: + """ + Retrieve all files matching the specified glob patterns in GCS. + """ + try: + start_date = ( + datetime.strptime(self.config.start_date, self.DATE_TIME_FORMAT) if self.config and self.config.start_date else None + ) + prefixes = [prefix] if prefix else self.get_prefixes_from_globs(globs or []) + globs = globs or [None] + + for prefix, glob in itertools.product(prefixes, globs): + bucket = self.gcs_client.get_bucket(self.config.bucket) + blobs = bucket.list_blobs(prefix=prefix, match_glob=glob) + for blob in blobs: + last_modified = blob.updated.astimezone(pytz.utc).replace(tzinfo=None) + + if FILE_FORMAT in blob.name.lower() and (not start_date or last_modified >= start_date): + uri = blob.generate_signed_url(expiration=timedelta(hours=1), version="v4") + + yield RemoteFile(uri=uri, last_modified=last_modified) + + except Exception as exc: + self._handle_file_listing_error(exc, prefix, logger) + + def _handle_file_listing_error(self, exc: Exception, prefix: str, logger: logging.Logger): + logger.error(f"Error while listing files: {str(exc)}") + raise ErrorListingFiles( + FileBasedSourceError.ERROR_LISTING_FILES, + source="gcs", + bucket=self.config.bucket, + prefix=prefix, + ) from exc + + def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger) -> IOBase: + """ + Open and yield a remote file from GCS for reading. + """ + logger.debug(f"Trying to open {file.uri}") + try: + result = smart_open.open(file.uri, mode=mode.value, encoding=encoding) + except OSError as oe: + logger.warning(ERROR_MESSAGE_ACCESS.format(uri=file.uri, bucket=self.config.bucket)) + logger.exception(oe) + return result diff --git a/airbyte-integrations/connectors/source-gcs/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-gcs/unit_tests/unit_test.py index 1d15516be5ce..f9c7751344fa 100644 --- a/airbyte-integrations/connectors/source-gcs/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-gcs/unit_tests/unit_test.py @@ -10,14 +10,9 @@ class TestGCSFunctions(unittest.TestCase): - def setUp(self): # Initialize the mock config - self.config = { - 'service_account': '{"test_key": "test_value"}', - 'gcs_bucket': 'test_bucket', - 'gcs_path': 'test_path' - } + self.config = {"service_account": '{"test_key": "test_value"}', "gcs_bucket": "test_bucket", "gcs_path": "test_path"} def test_construct_file_schema(self): # Test that the function correctly constructs a JSON schema for a DataFrame @@ -26,9 +21,6 @@ def test_construct_file_schema(self): expected_schema = { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "properties": { - "id": {"type": "string"}, - "name": {"type": "string"} - } + "properties": {"id": {"type": "string"}, "name": {"type": "string"}}, } self.assertEqual(schema, expected_schema) diff --git a/airbyte-integrations/connectors/source-genesys/README.md b/airbyte-integrations/connectors/source-genesys/README.md index 58bcbe410178..6bed9a552aed 100644 --- a/airbyte-integrations/connectors/source-genesys/README.md +++ b/airbyte-integrations/connectors/source-genesys/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-genesys:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/genesys) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_genesys/spec.yaml` file. @@ -56,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-genesys:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-genesys build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-genesys:airbyteDocker +An image will be built with the tag `airbyte/source-genesys:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-genesys:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -77,44 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-genesys:dev check --co docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-genesys:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-genesys:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-genesys test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-genesys:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-genesys:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -124,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-genesys test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/genesys.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-genesys/acceptance-test-config.yml b/airbyte-integrations/connectors/source-genesys/acceptance-test-config.yml index c5484597064a..a825e3d38046 100644 --- a/airbyte-integrations/connectors/source-genesys/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-genesys/acceptance-test-config.yml @@ -14,23 +14,24 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [ - "routing_outbound_events", - "routing_routing_assessments", - "routing_routing_queues", - "telephony_locations", - "telephony_providers_edges_didpools", - "telephony_providers_edges_dids", - "telephony_providers_edges_extensions", - "telephony_providers_edges_lines", - "telephony_providers_edges_outboundroutes", - "telephony_providers_edges_phones", - "telephony_providers_edges_sites", - "telephony_providers_edges_trunks", - "telephony_providers_edges", - "telephony_stations", - "user_groups", - ] + empty_streams: + [ + "routing_outbound_events", + "routing_routing_assessments", + "routing_routing_queues", + "telephony_locations", + "telephony_providers_edges_didpools", + "telephony_providers_edges_dids", + "telephony_providers_edges_extensions", + "telephony_providers_edges_lines", + "telephony_providers_edges_outboundroutes", + "telephony_providers_edges_phones", + "telephony_providers_edges_sites", + "telephony_providers_edges_trunks", + "telephony_providers_edges", + "telephony_stations", + "user_groups", + ] full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-genesys/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-genesys/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-genesys/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-genesys/build.gradle b/airbyte-integrations/connectors/source-genesys/build.gradle deleted file mode 100644 index 7c7591465241..000000000000 --- a/airbyte-integrations/connectors/source-genesys/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_genesys' -} diff --git a/airbyte-integrations/connectors/source-getlago/Dockerfile b/airbyte-integrations/connectors/source-getlago/Dockerfile index 8c1e400964bb..faad542ae3bd 100644 --- a/airbyte-integrations/connectors/source-getlago/Dockerfile +++ b/airbyte-integrations/connectors/source-getlago/Dockerfile @@ -34,5 +34,5 @@ COPY source_getlago ./source_getlago ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.3.0 LABEL io.airbyte.name=airbyte/source-getlago diff --git a/airbyte-integrations/connectors/source-getlago/README.md b/airbyte-integrations/connectors/source-getlago/README.md index fd1225f162b0..79ef2999b3ce 100644 --- a/airbyte-integrations/connectors/source-getlago/README.md +++ b/airbyte-integrations/connectors/source-getlago/README.md @@ -1,18 +1,10 @@ -# Getlago Source +# Lago Source -This is the repository for the Getlago configuration based source connector. +This is the repository for the Lago configuration based source connector. For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/getlago). ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-getlago:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/getlago) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_getlago/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-getlago:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-getlago build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-getlago:airbyteDocker +An image will be built with the tag `airbyte/source-getlago:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-getlago:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-getlago:dev check --co docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-getlago:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-getlago:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-getlago test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-getlago:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-getlago:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-getlago test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/getlago.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-getlago/acceptance-test-config.yml b/airbyte-integrations/connectors/source-getlago/acceptance-test-config.yml index 0c2a1c28a382..cfe7c503642d 100644 --- a/airbyte-integrations/connectors/source-getlago/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-getlago/acceptance-test-config.yml @@ -18,7 +18,11 @@ acceptance_tests: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] + empty_streams: + - name: customer_usage + bypass_reason: "Sandbox account can't seed this stream" + - name: wallets + bypass_reason: "Sandbox account can't seed this stream" # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file # expect_records: # path: "integration_tests/expected_records.jsonl" diff --git a/airbyte-integrations/connectors/source-getlago/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-getlago/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-getlago/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-getlago/build.gradle b/airbyte-integrations/connectors/source-getlago/build.gradle deleted file mode 100644 index b3240ef1b99c..000000000000 --- a/airbyte-integrations/connectors/source-getlago/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_getlago' -} diff --git a/airbyte-integrations/connectors/source-getlago/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-getlago/integration_tests/configured_catalog.json index 1e8fc6d57a06..2e0e18efb2e7 100644 --- a/airbyte-integrations/connectors/source-getlago/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-getlago/integration_tests/configured_catalog.json @@ -62,6 +62,24 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "wallets", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "customer_usage", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-getlago/metadata.yaml b/airbyte-integrations/connectors/source-getlago/metadata.yaml index 539d110e928a..70779de3b016 100644 --- a/airbyte-integrations/connectors/source-getlago/metadata.yaml +++ b/airbyte-integrations/connectors/source-getlago/metadata.yaml @@ -2,12 +2,12 @@ data: connectorSubtype: api connectorType: source definitionId: e1a3866b-d3b2-43b6-b6d7-8c1ee4d7f53f - dockerImageTag: 0.1.0 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-getlago githubIssueLabel: source-getlago icon: getlago.svg license: MIT - name: GetLago + name: Lago registries: cloud: enabled: true diff --git a/airbyte-integrations/connectors/source-getlago/source_getlago/manifest.yaml b/airbyte-integrations/connectors/source-getlago/source_getlago/manifest.yaml index 13accde4d905..9cc722a4a0ea 100644 --- a/airbyte-integrations/connectors/source-getlago/source_getlago/manifest.yaml +++ b/airbyte-integrations/connectors/source-getlago/source_getlago/manifest.yaml @@ -5,7 +5,7 @@ definitions: extractor: field_path: ["{{ parameters['name'] }}"] requester: - url_base: "https://api.getlago.com/api/v1" + url_base: "{{ config.get('api_url', 'https://api.getlago.com/api/v1') }}" http_method: "GET" authenticator: type: BearerAuthenticator @@ -90,6 +90,24 @@ definitions: $ref: "#/definitions/customer_partition_router" record_selector: $ref: "#/definitions/selector" + wallets_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "wallets" + primary_key: "lago_id" + path: "/wallets" + customer_usage_stream: + name: "customer_usage" + primary_key: "lago_invoice_id" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + path: "/customers/{{stream_slice.customer_external_id}}/current_usage" + paginator: + type: NoPagination + partition_router: + $ref: "#/definitions/customer_partition_router" streams: - "#/definitions/billable_metrics_stream" @@ -99,6 +117,8 @@ streams: - "#/definitions/invoices_stream" - "#/definitions/customers_stream" - "#/definitions/subscriptions_stream" + - "#/definitions/wallets_stream" + - "#/definitions/customer_usage_stream" check: stream_names: diff --git a/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/add_ons.json b/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/add_ons.json index 81185dc445c6..30ef1672ea07 100644 --- a/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/add_ons.json +++ b/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/add_ons.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "lago_id": { "type": ["null", "string"] @@ -11,6 +12,9 @@ "created_at": { "type": ["null", "string"] }, + "invoice_display_name": { + "type": ["null", "string"] + }, "code": { "type": ["null", "string"] }, @@ -22,6 +26,9 @@ }, "amount_currency": { "type": ["null", "string"] + }, + "taxes": { + "type": ["array", "null"] } } } diff --git a/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/billable_metrics.json b/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/billable_metrics.json index 3c65d5b9f05a..4ec863bc071c 100644 --- a/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/billable_metrics.json +++ b/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/billable_metrics.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "lago_id": { "type": ["null", "string"] @@ -25,6 +26,21 @@ }, "group": { "type": ["null", "object"] + }, + "active_subscriptions_count": { + "type": ["null", "integer"] + }, + "draft_invoices_count": { + "type": ["null", "integer"] + }, + "plans_count": { + "type": ["null", "integer"] + }, + "recurring": { + "type": ["null", "boolean"] + }, + "weighted_interval": { + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/coupons.json b/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/coupons.json index 2fcae9c85b38..2c2387278733 100644 --- a/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/coupons.json +++ b/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/coupons.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "lago_id": { "type": ["null", "string"] @@ -37,6 +38,30 @@ }, "expiration_date": { "type": ["null", "string"] + }, + "billable_metric_codes": { + "type": ["null", "array"] + }, + "description": { + "type": ["null", "string"] + }, + "expiration_at": { + "type": ["null", "string"] + }, + "limited_billable_metrics": { + "type": ["null", "boolean"] + }, + "limited_plans": { + "type": ["null", "boolean"] + }, + "plan_codes": { + "type": ["null", "array"] + }, + "reusable": { + "type": ["null", "boolean"] + }, + "terminated_at": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/customer_usage.json b/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/customer_usage.json new file mode 100644 index 000000000000..88d9cb413038 --- /dev/null +++ b/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/customer_usage.json @@ -0,0 +1,105 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "from_datetime": { + "type": ["null", "string"] + }, + "to_datetime": { + "type": ["null", "string"] + }, + "issuing_date": { + "type": ["null", "string"] + }, + "lago_invoice_id": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "amount_cents": { + "type": ["null", "integer"] + }, + "taxes_amount_cents": { + "type": ["null", "integer"] + }, + "total_amount_cents": { + "type": ["null", "integer"] + }, + "charges_usage": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "units": { + "type": ["null", "string"] + }, + "events_count": { + "type": ["null", "integer"] + }, + "amount_cents": { + "type": ["null", "integer"] + }, + "amount_currency": { + "type": ["null", "string"] + }, + "charge": { + "type": ["null", "object"], + "properties": { + "lago_id": { + "type": ["null", "string"] + }, + "charge_model": { + "type": ["null", "string"] + } + } + }, + "billable_metric": { + "type": ["null", "object"], + "properties": { + "lago_id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "code": { + "type": ["null", "string"] + }, + "aggregation_type": { + "type": ["null", "string"] + } + } + }, + "groups": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "lago_id": { + "type": ["null", "string"] + }, + "key": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "units": { + "type": ["null", "string"] + }, + "events_count": { + "type": ["null", "integer"] + }, + "amount_cents": { + "type": ["null", "integer"] + } + } + } + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/customers.json b/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/customers.json index 415d75f01bdd..17bfd14a80d9 100644 --- a/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/customers.json +++ b/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/customers.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "lago_id": { "type": ["null", "string"] @@ -56,8 +57,62 @@ "currency": { "type": ["null", "string"] }, + "applicable_timezone": { + "type": ["null", "string"] + }, "billing_configuration": { - "type": ["null", "object"] + "additionalProperties": true, + "properties": { + "document_locale": { + "type": ["null", "string"] + }, + "invoice_grace_period": { + "type": ["null", "integer"] + }, + "payment_provider": { + "type": ["null", "string"] + }, + "provider_customer_id": { + "type": ["null", "string"] + }, + "provider_payment_methods": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "vat_rate": { + "type": ["null", "integer"] + } + }, + "type": "object" + }, + "country": { + "type": ["null", "string"] + }, + "external_salesforce_id": { + "type": ["null", "integer"] + }, + "metadata": { + "type": ["null", "array"] + }, + "net_payment_term": { + "type": ["null", "integer"] + }, + "tax_identification_number": { + "type": ["null", "integer"] + }, + "taxes": { + "type": ["null", "array"] + }, + "timezone": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "zipcode": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/invoices.json b/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/invoices.json index 198fb2b0d8a6..7b2d74b70aec 100644 --- a/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/invoices.json +++ b/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/invoices.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "lago_id": { "type": ["null", "string"] @@ -40,6 +41,174 @@ }, "file_url": { "type": ["null", "string"] + }, + "applied_taxes": { + "type": ["null", "array"] + }, + "coupons_amount_cents": { + "type": ["null", "integer"] + }, + "credit_amount_cents": { + "type": ["null", "integer"] + }, + "credit_amount_currency": { + "type": ["null", "string"] + }, + "credit_notes_amount_cents": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "customer": { + "additionalProperties": true, + "properties": { + "address_line1": { + "type": ["null", "string"] + }, + "address_line2": { + "type": ["null", "string"] + }, + "applicable_timezone": { + "type": ["null", "string"] + }, + "billing_configuration": { + "additionalProperties": true, + "properties": { + "document_locale": { + "type": ["null", "string"] + }, + "invoice_grace_period": { + "type": ["null", "integer"] + }, + "payment_provider": { + "type": ["null", "string"] + }, + "provider_customer_id": { + "type": ["null", "integer"] + }, + "provider_payment_methods": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "vat_rate": { + "type": ["null", "integer"] + } + }, + "type": "object" + }, + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "external_id": { + "type": ["null", "string"] + }, + "external_salesforce_id": { + "type": ["null", "integer"] + }, + "lago_id": { + "type": ["null", "string"] + }, + "legal_name": { + "type": ["null", "string"] + }, + "legal_number": { + "type": ["null", "string"] + }, + "logo_url": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "array"] + }, + "name": { + "type": ["null", "string"] + }, + "net_payment_term": { + "type": ["null", "integer"] + }, + "phone": { + "type": ["null", "string"] + }, + "sequential_id": { + "type": ["null", "integer"] + }, + "slug": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "tax_identification_number": { + "type": ["null", "integer"] + }, + "timezone": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "url": { + "type": "null" + }, + "zipcode": { + "type": ["null", "string"] + } + }, + "type": "object" + }, + "fees_amount_cents": { + "type": ["null", "integer"] + }, + "legacy": { + "type": ["null", "boolean"] + }, + "metadata": { + "type": ["null", "array"] + }, + "net_payment_term": { + "type": ["null", "integer"] + }, + "payment_due_date": { + "type": ["null", "string"] + }, + "payment_status": { + "type": ["null", "string"] + }, + "prepaid_credit_amount_cents": { + "type": ["null", "integer"] + }, + "sub_total_excluding_taxes_amount_cents": { + "type": ["null", "integer"] + }, + "sub_total_including_taxes_amount_cents": { + "type": ["null", "integer"] + }, + "sub_total_vat_excluded_amount_cents": { + "type": ["null", "integer"] + }, + "sub_total_vat_included_amount_cents": { + "type": ["null", "integer"] + }, + "taxes_amount_cents": { + "type": ["null", "integer"] + }, + "version_number": { + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/plans.json b/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/plans.json index 57385fe75d95..1552e2523f2f 100644 --- a/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/plans.json +++ b/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/plans.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "lago_id": { "type": ["null", "string"] @@ -35,7 +36,73 @@ "bill_charges_monthly": { "type": ["null", "boolean"] }, + "active_subscriptions_count": { + "type": ["null", "integer"] + }, "charges": { + "items": { + "additionalProperties": true, + "properties": { + "billable_metric_code": { + "type": ["null", "string"] + }, + "charge_model": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "group_properties": { + "type": ["null", "array"] + }, + "invoiceable": { + "type": ["null", "boolean"] + }, + "lago_billable_metric_id": { + "type": ["null", "string"] + }, + "lago_id": { + "type": ["null", "string"] + }, + "min_amount_cents": { + "type": ["null", "integer"] + }, + "pay_in_advance": { + "type": ["null", "boolean"] + }, + "properties": { + "additionalProperties": true, + "properties": { + "amount": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "prorated": { + "type": ["null", "boolean"] + }, + "taxes": { + "type": ["null", "array"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, + "customers_count": { + "type": ["null", "integer"] + }, + "draft_invoices_count": { + "type": ["null", "integer"] + }, + "invoice_display_name": { + "type": ["null", "string"] + }, + "parent_id": { + "type": ["null", "integer"] + }, + "taxes": { "type": ["null", "array"] } } diff --git a/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/subscriptions.json b/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/subscriptions.json index 82c35e69bac7..57a5a671fc3e 100644 --- a/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/subscriptions.json +++ b/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/subscriptions.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "lago_id": { "type": ["null", "string"] @@ -49,6 +50,12 @@ }, "downgrade_plan_date": { "type": ["null", "string"] + }, + "ending_at": { + "type": ["null", "string"] + }, + "subscription_at": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/wallets.json b/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/wallets.json new file mode 100644 index 000000000000..3a46c6e4b5c1 --- /dev/null +++ b/airbyte-integrations/connectors/source-getlago/source_getlago/schemas/wallets.json @@ -0,0 +1,52 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "lago_id": { + "type": ["null", "string"] + }, + "lago_customer_id": { + "type": ["null", "string"] + }, + "external_customer_id": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "rate_amount": { + "type": ["null", "string"] + }, + "credits_balance": { + "type": ["null", "string"] + }, + "balance_cents": { + "type": ["null", "integer"] + }, + "consumed_credits": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "expiration_at": { + "type": ["null", "string"] + }, + "last_balance_sync_at": { + "type": ["null", "string"] + }, + "last_consumed_credit_at": { + "type": ["null", "string"] + }, + "terminated_at": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-getlago/source_getlago/spec.yaml b/airbyte-integrations/connectors/source-getlago/source_getlago/spec.yaml index 2f9d5311e536..04921b0ecbf2 100644 --- a/airbyte-integrations/connectors/source-getlago/source_getlago/spec.yaml +++ b/airbyte-integrations/connectors/source-getlago/source_getlago/spec.yaml @@ -1,12 +1,17 @@ documentationUrl: https://docs.airbyte.com/integrations/sources/getlago connectionSpecification: $schema: http://json-schema.org/draft-07/schema# - title: Getlago Spec + title: Lago Spec type: object required: - api_key additionalProperties: true properties: + api_url: + title: API Url + type: string + description: Your Lago API URL + default: https://api.getlago.com/api/v1 api_key: title: API Key type: string diff --git a/airbyte-integrations/connectors/source-github/Dockerfile b/airbyte-integrations/connectors/source-github/Dockerfile deleted file mode 100644 index 14a7ee12b273..000000000000 --- a/airbyte-integrations/connectors/source-github/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY source_github ./source_github -COPY main.py ./ -COPY setup.py ./ -RUN pip install . - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=1.0.4 -LABEL io.airbyte.name=airbyte/source-github diff --git a/airbyte-integrations/connectors/source-github/README.md b/airbyte-integrations/connectors/source-github/README.md index a42dd899f3c5..0a0aad08fcc4 100644 --- a/airbyte-integrations/connectors/source-github/README.md +++ b/airbyte-integrations/connectors/source-github/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-github:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/github) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_github/spec.json` file. @@ -57,19 +49,70 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-github:dev + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-github build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-github:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. -You can also build the connector image via Gradle: +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-github:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-github:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-github:dev . +# Running the spec command against your patched connector +docker run airbyte/source-github:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -78,44 +121,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-github:dev check --con docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-github:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-github:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .'[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-github test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-github:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-github:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +140,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-github test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/github.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-github/acceptance-test-config.yml b/airbyte-integrations/connectors/source-github/acceptance-test-config.yml index ef37bf639fff..2ff5a27c1f45 100644 --- a/airbyte-integrations/connectors/source-github/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-github/acceptance-test-config.yml @@ -3,149 +3,129 @@ test_strictness_level: "high" acceptance_tests: spec: tests: - - spec_path: "source_github/spec.json" - backward_compatibility_tests_config: - disable_for_version: "0.5.0" + - spec_path: "source_github/spec.json" + backward_compatibility_tests_config: + disable_for_version: "1.4.3" connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "secrets/config_oauth.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "secrets/config_oauth.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: tests: - - config_path: "secrets/config.json" - backward_compatibility_tests_config: - disable_for_version: "0.4.8" - - config_path: "secrets/config_oauth.json" - backward_compatibility_tests_config: - disable_for_version: "0.4.8" + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "1.4.3" + - config_path: "secrets/config_oauth.json" + backward_compatibility_tests_config: + disable_for_version: "0.4.8" basic_read: tests: - - config_path: "secrets/config.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: no - extra_records: yes - empty_streams: - - name: "events" - bypass_reason: "Only events created within the past 90 days can be showed" - ignored_fields: - workflows: - - name: created_at - bypass_reason: value may be returned in different time zones - - name: updated_at - bypass_reason: value may be returned in different time zones - workflow_jobs: - - name: steps/*/started_at - bypass_reason: "depend on changing data" - - name: steps/*/completed_at - bypass_reason: "depend on changing data" - organizations: - - name: followers - bypass_reason: "fast changing data" - - name: updated_at - bypass_reason: "fast changing data" - - name: plan - bypass_reason: "fast changing data" - - name: public_repos - bypass_reason: "fast changing data" - - name: total_private_repos - bypass_reason: "fast changing data" - - name: owned_private_repos - bypass_reason: "fast changing data" - repositories: - - name: updated_at - bypass_reason: "fast changing data" - - name: pushed_at - bypass_reason: "fast changing data" - - name: size - bypass_reason: "fast changing data" - - name: stargazers_count - bypass_reason: "fast changing data" - - name: watchers_count - bypass_reason: "fast changing data" - - name: forks_count - bypass_reason: "fast changing data" - - name: forks - bypass_reason: "fast changing data" - - name: open_issues - bypass_reason: "fast changing data" - - name: open_issues_count - bypass_reason: "fast changing data" - - name: watchers - bypass_reason: "fast changing data" + - config_path: "secrets/config.json" + expect_records: + path: "integration_tests/expected_records.jsonl" + extra_fields: no + exact_order: no + extra_records: yes + empty_streams: + - name: "events" + bypass_reason: "Only events created within the past 90 days can be showed" + ignored_fields: + contributor_activity: + - name: weeks + bypass_reason: "depend on changing data" + - name: total + bypass_reason: "depend on changing data" + workflows: + - name: created_at + bypass_reason: value may be returned in different time zones + - name: updated_at + bypass_reason: value may be returned in different time zones + workflow_jobs: + - name: steps/*/started_at + bypass_reason: "depend on changing data" + - name: steps/*/completed_at + bypass_reason: "depend on changing data" + organizations: + - name: followers + bypass_reason: "fast changing data" + - name: updated_at + bypass_reason: "fast changing data" + - name: plan + bypass_reason: "fast changing data" + - name: public_repos + bypass_reason: "fast changing data" + - name: total_private_repos + bypass_reason: "fast changing data" + - name: owned_private_repos + bypass_reason: "fast changing data" + repositories: + - name: updated_at + bypass_reason: "fast changing data" + - name: pushed_at + bypass_reason: "fast changing data" + - name: size + bypass_reason: "fast changing data" + - name: stargazers_count + bypass_reason: "fast changing data" + - name: watchers_count + bypass_reason: "fast changing data" + - name: forks_count + bypass_reason: "fast changing data" + - name: forks + bypass_reason: "fast changing data" + - name: open_issues + bypass_reason: "fast changing data" + - name: open_issues_count + bypass_reason: "fast changing data" + - name: watchers + bypass_reason: "fast changing data" incremental: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" - cursor_paths: - comments: ["airbytehq/integration-test", "updated_at"] - commit_comment_reactions: ["airbytehq/integration-test", "55538825", "created_at"] - commit_comments: ["airbytehq/integration-test", "updated_at"] - commits: ["airbytehq/integration-test", "master", "created_at"] - deployments: ["airbytehq/integration-test", "updated_at"] - events: ["airbytehq/integration-test", "created_at"] - issue_comment_reactions: ["airbytehq/integration-test", "907296275", "created_at"] - issue_events: ["airbytehq/integration-test", "created_at"] - issue_milestones: ["airbytehq/integration-test", "updated_at"] - issue_reactions: ["airbytehq/integration-test", "created_at"] - issues: ["airbytehq/integration-test", "updated_at"] - project_cards: ["airbytehq/integration-test", "13167124", "17807006", "updated_at"] - project_columns: ["airbytehq/integration-test", "13167124", "updated_at"] - projects: ["airbytehq/integration-test", "updated_at"] - pull_request_comment_reactions: ["airbytehq/integration-test", "699253726", "created_at"] - pull_request_stats: ["airbytehq/integration-test", "updated_at"] - pull_requests: ["airbytehq/integration-test", "updated_at"] - releases: ["airbytehq/integration-test", "created_at"] - repositories: ["airbytehq", "updated_at"] - review_comments: ["airbytehq/integration-test", "updated_at"] - reviews: ["airbytehq/integration-test", "updated_at"] - stargazers: ["airbytehq/integration-test", "starred_at"] - workflow_runs: ["airbytehq/integration-test", "updated_at"] - workflows: ["airbytehq/integration-test", "updated_at"] - workflow_jobs: ["airbytehq/integration-test", "completed_at"] + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + skip_comprehensive_incremental_tests: true full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - ignored_fields: - organizations: - - name: followers - bypass_reason: "fast changing data" - - name: updated_at - bypass_reason: "fast changing data" - - name: plan - bypass_reason: "fast changing data" - - name: public_repos - bypass_reason: "fast changing data" - - name: total_private_repos - bypass_reason: "fast changing data" - - name: owned_private_repos - bypass_reason: "fast changing data" - repositories: - - name: updated_at - bypass_reason: "fast changing data" - - name: pushed_at - bypass_reason: "fast changing data" - - name: size - bypass_reason: "fast changing data" - - name: stargazers_count - bypass_reason: "fast changing data" - - name: watchers_count - bypass_reason: "fast changing data" - - name: forks_count - bypass_reason: "fast changing data" - - name: forks - bypass_reason: "fast changing data" - - name: open_issues - bypass_reason: "fast changing data" - - name: open_issues_count - bypass_reason: "fast changing data" - - name: watchers - bypass_reason: "fast changing data" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_full_refresh_test.json" + ignored_fields: + organizations: + - name: followers + bypass_reason: "fast changing data" + - name: updated_at + bypass_reason: "fast changing data" + - name: plan + bypass_reason: "fast changing data" + - name: public_repos + bypass_reason: "fast changing data" + - name: total_private_repos + bypass_reason: "fast changing data" + - name: owned_private_repos + bypass_reason: "fast changing data" + repositories: + - name: updated_at + bypass_reason: "fast changing data" + - name: pushed_at + bypass_reason: "fast changing data" + - name: size + bypass_reason: "fast changing data" + - name: stargazers_count + bypass_reason: "fast changing data" + - name: watchers_count + bypass_reason: "fast changing data" + - name: forks_count + bypass_reason: "fast changing data" + - name: forks + bypass_reason: "fast changing data" + - name: open_issues + bypass_reason: "fast changing data" + - name: open_issues_count + bypass_reason: "fast changing data" + - name: watchers + bypass_reason: "fast changing data" diff --git a/airbyte-integrations/connectors/source-github/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-github/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-github/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-github/build.gradle b/airbyte-integrations/connectors/source-github/build.gradle deleted file mode 100644 index dc3c0b0ab5d1..000000000000 --- a/airbyte-integrations/connectors/source-github/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_github' -} diff --git a/airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json index 331880b000fa..abd43868c68e 100644 --- a/airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json @@ -79,6 +79,16 @@ "destination_sync_mode": "append", "cursor_field": ["created_at"] }, + { + "stream": { + "name": "contributor_activity", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "deployments", @@ -410,6 +420,16 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "issue_timeline_events", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["repository"], ["issue_number"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-github/integration_tests/configured_catalog_full_refresh_test.json b/airbyte-integrations/connectors/source-github/integration_tests/configured_catalog_full_refresh_test.json new file mode 100644 index 000000000000..6c0b17fa6b33 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/integration_tests/configured_catalog_full_refresh_test.json @@ -0,0 +1,415 @@ +{ + "streams": [ + { + "stream": { + "name": "assignees", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "branches", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["repository"], ["name"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "collaborators", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "comments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "commit_comment_reactions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "commit_comments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "commits", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["sha"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["created_at"] + }, + { + "stream": { + "name": "contributor_activity", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "deployments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "events", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["created_at"] + }, + { + "stream": { + "name": "issue_comment_reactions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "issue_events", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["created_at"] + }, + { + "stream": { + "name": "issue_labels", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "issue_milestones", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "issue_reactions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "issues", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "organizations", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "project_cards", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "project_columns", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "projects", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "pull_request_comment_reactions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "pull_request_commits", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["sha"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "pull_request_stats", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "pull_requests", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "releases", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["created_at"] + }, + { + "stream": { + "name": "repositories", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "review_comments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "reviews", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["updated_at"] + }, + { + "stream": { + "name": "stargazers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["starred_at"], + "source_defined_primary_key": [["user_id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["starred_at"] + }, + { + "stream": { + "name": "tags", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["repository"], ["name"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "teams", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "workflows", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "workflow_runs", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "workflow_jobs", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "team_members", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"], ["team_slug"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-github/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-github/integration_tests/expected_records.jsonl index b445361bfe9f..9b65df5c424e 100644 --- a/airbyte-integrations/connectors/source-github/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-github/integration_tests/expected_records.jsonl @@ -5,9 +5,11 @@ {"stream":"commit_comment_reactions","data":{"id":154935429,"node_id":"REA_lADOF9hP9c4DT3SJzgk8IIU","user":{"login":"grubberr","id":195743,"node_id":"MDQ6VXNlcjE5NTc0Mw==","avatar_url":"https://avatars.githubusercontent.com/u/195743?v=4","gravatar_id":"","url":"https://api.github.com/users/grubberr","html_url":"https://github.com/grubberr","followers_url":"https://api.github.com/users/grubberr/followers","following_url":"https://api.github.com/users/grubberr/following{/other_user}","gists_url":"https://api.github.com/users/grubberr/gists{/gist_id}","starred_url":"https://api.github.com/users/grubberr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/grubberr/subscriptions","organizations_url":"https://api.github.com/users/grubberr/orgs","repos_url":"https://api.github.com/users/grubberr/repos","events_url":"https://api.github.com/users/grubberr/events{/privacy}","received_events_url":"https://api.github.com/users/grubberr/received_events","type":"User","site_admin":false},"content":"laugh","created_at":"2022-03-20T11:29:29Z","repository":"airbytehq/integration-test","comment_id":55538825},"emitted_at":1677668746490} {"stream":"commit_comments","data":{"url":"https://api.github.com/repos/airbytehq/integration-test/comments/55538825","html_url":"https://github.com/airbytehq/integration-test/commit/cbbeaf3ef6eb7217052eae2fe665f655e3813973#commitcomment-55538825","id":55538825,"node_id":"MDEzOkNvbW1pdENvbW1lbnQ1NTUzODgyNQ==","user":{"login":"gaart","id":743901,"node_id":"MDQ6VXNlcjc0MzkwMQ==","avatar_url":"https://avatars.githubusercontent.com/u/743901?v=4","gravatar_id":"","url":"https://api.github.com/users/gaart","html_url":"https://github.com/gaart","followers_url":"https://api.github.com/users/gaart/followers","following_url":"https://api.github.com/users/gaart/following{/other_user}","gists_url":"https://api.github.com/users/gaart/gists{/gist_id}","starred_url":"https://api.github.com/users/gaart/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gaart/subscriptions","organizations_url":"https://api.github.com/users/gaart/orgs","repos_url":"https://api.github.com/users/gaart/repos","events_url":"https://api.github.com/users/gaart/events{/privacy}","received_events_url":"https://api.github.com/users/gaart/received_events","type":"User","site_admin":false},"position":null,"line":null,"path":null,"commit_id":"cbbeaf3ef6eb7217052eae2fe665f655e3813973","created_at":"2021-08-27T15:43:32Z","updated_at":"2021-08-27T15:43:32Z","author_association":"CONTRIBUTOR","body":"comment for cbbeaf3ef6eb7217052eae2fe665f655e3813973 branch","reactions":{"url":"https://api.github.com/repos/airbytehq/integration-test/comments/55538825/reactions","total_count":2,"+1":0,"-1":0,"laugh":1,"hooray":0,"confused":0,"heart":1,"rocket":0,"eyes":0},"repository":"airbytehq/integration-test"},"emitted_at":1677668747441} {"stream":"commits","data":{"sha":"a12c9379604f7b32e54e5459122aa48473f806ee","node_id":"C_kwDOF9hP9doAKGExMmM5Mzc5NjA0ZjdiMzJlNTRlNTQ1OTEyMmFhNDg0NzNmODA2ZWU","commit":{"author":{"name":"Marcos Marx","email":"marcosmarxm@users.noreply.github.com","date":"2022-03-30T19:35:47Z"},"committer":{"name":"GitHub","email":"noreply@github.com","date":"2022-03-30T19:35:47Z"},"message":"Update secret","tree":{"sha":"20041b32912f78ebe33ba4475e6ee439b28eea91","url":"https://api.github.com/repos/airbytehq/integration-test/git/trees/20041b32912f78ebe33ba4475e6ee439b28eea91"},"url":"https://api.github.com/repos/airbytehq/integration-test/git/commits/a12c9379604f7b32e54e5459122aa48473f806ee","comment_count":0,"verification":{"verified":true,"reason":"valid","signature":"-----BEGIN PGP SIGNATURE-----\n\nwsBcBAABCAAQBQJiRLETCRBK7hj4Ov3rIwAA5HYIAG4TCKKHConYOP4KJw0FAbFC\nkoIySnA2Zw5npa261DWJhMuSHHRu1HV0O7jzANOYuu3WlJAVPNEs7lkA0e2+eqVJ\nXIwKvEcoJ7DnRU2IL8JQQ5Ivtkv6XRCYoCW0koL0wcD1ORb4ZwFtd60MCezOoPms\nIszZaj4zXjpIe8Dw5N7je9UlP8nlqMJP4ll3cv9mt5VX+NwPVHj2vLnBX6zdGEPE\nNW5HlFRX1G5+hAzZ6bbIcFfB8TMlFa74MbPVMR9TccVgucbCHQFPSkYM8zILUVKM\nLdNh1U3+oYKH6Di/b2tef2dgW24NobXoKD0aSeVx/7V0MlockF8AKH2JZiXnMDg=\n=f0MM\n-----END PGP SIGNATURE-----\n","payload":"tree 20041b32912f78ebe33ba4475e6ee439b28eea91\nparent 1a75ba447e31cc2d8908694db3371ee39bc783eb\nauthor Marcos Marx 1648668947 -0300\ncommitter GitHub 1648668947 -0300\n\nUpdate secret"}},"url":"https://api.github.com/repos/airbytehq/integration-test/commits/a12c9379604f7b32e54e5459122aa48473f806ee","html_url":"https://github.com/airbytehq/integration-test/commit/a12c9379604f7b32e54e5459122aa48473f806ee","comments_url":"https://api.github.com/repos/airbytehq/integration-test/commits/a12c9379604f7b32e54e5459122aa48473f806ee/comments","author":{"login":"marcosmarxm","id":5154322,"node_id":"MDQ6VXNlcjUxNTQzMjI=","avatar_url":"https://avatars.githubusercontent.com/u/5154322?v=4","gravatar_id":"","url":"https://api.github.com/users/marcosmarxm","html_url":"https://github.com/marcosmarxm","followers_url":"https://api.github.com/users/marcosmarxm/followers","following_url":"https://api.github.com/users/marcosmarxm/following{/other_user}","gists_url":"https://api.github.com/users/marcosmarxm/gists{/gist_id}","starred_url":"https://api.github.com/users/marcosmarxm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/marcosmarxm/subscriptions","organizations_url":"https://api.github.com/users/marcosmarxm/orgs","repos_url":"https://api.github.com/users/marcosmarxm/repos","events_url":"https://api.github.com/users/marcosmarxm/events{/privacy}","received_events_url":"https://api.github.com/users/marcosmarxm/received_events","type":"User","site_admin":false},"committer":{"login":"web-flow","id":19864447,"node_id":"MDQ6VXNlcjE5ODY0NDQ3","avatar_url":"https://avatars.githubusercontent.com/u/19864447?v=4","gravatar_id":"","url":"https://api.github.com/users/web-flow","html_url":"https://github.com/web-flow","followers_url":"https://api.github.com/users/web-flow/followers","following_url":"https://api.github.com/users/web-flow/following{/other_user}","gists_url":"https://api.github.com/users/web-flow/gists{/gist_id}","starred_url":"https://api.github.com/users/web-flow/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/web-flow/subscriptions","organizations_url":"https://api.github.com/users/web-flow/orgs","repos_url":"https://api.github.com/users/web-flow/repos","events_url":"https://api.github.com/users/web-flow/events{/privacy}","received_events_url":"https://api.github.com/users/web-flow/received_events","type":"User","site_admin":false},"parents":[{"sha":"1a75ba447e31cc2d8908694db3371ee39bc783eb","url":"https://api.github.com/repos/airbytehq/integration-test/commits/1a75ba447e31cc2d8908694db3371ee39bc783eb","html_url":"https://github.com/airbytehq/integration-test/commit/1a75ba447e31cc2d8908694db3371ee39bc783eb"}],"repository":"airbytehq/integration-test","created_at":"2022-03-30T19:35:47Z","branch":"master"},"emitted_at":1677668747823} +{"stream":"contributor_activity","data":{"total":4,"weeks":[{"w":1629590400,"a":0,"d":0,"c":0},{"w":1630195200,"a":0,"d":0,"c":0},{"w":1630800000,"a":0,"d":0,"c":0},{"w":1631404800,"a":0,"d":0,"c":0}],"repository":"airbytehq/integration-test","login":"marcosmarxm","id":5154322,"node_id":"MDQ6VXNlcjUxNTQzMjI=","avatar_url":"https://avatars.githubusercontent.com/u/5154322?v=4","gravatar_id":"","url":"https://api.github.com/users/marcosmarxm","html_url":"https://github.com/marcosmarxm","followers_url":"https://api.github.com/users/marcosmarxm/followers","following_url":"https://api.github.com/users/marcosmarxm/following{/other_user}","gists_url":"https://api.github.com/users/marcosmarxm/gists{/gist_id}","starred_url":"https://api.github.com/users/marcosmarxm/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/marcosmarxm/subscriptions","organizations_url":"https://api.github.com/users/marcosmarxm/orgs","repos_url":"https://api.github.com/users/marcosmarxm/repos","events_url":"https://api.github.com/users/marcosmarxm/events{/privacy}","received_events_url":"https://api.github.com/users/marcosmarxm/received_events","type":"User","site_admin":false},"emitted_at":1695214089426} +{"stream":"contributor_activity","data":{"total":6,"weeks":[{"w":1629590400,"a":5,"d":0,"c":6},{"w":1630195200,"a":0,"d":0,"c":0},{"w":1630800000,"a":0,"d":0,"c":0},{"w":1631404800,"a":0,"d":0,"c":0}],"repository":"airbytehq/integration-test","login":"gaart","id":743901,"node_id":"MDQ6VXNlcjc0MzkwMQ==","avatar_url":"https://avatars.githubusercontent.com/u/743901?v=4","gravatar_id":"","url":"https://api.github.com/users/gaart","html_url":"https://github.com/gaart","followers_url":"https://api.github.com/users/gaart/followers","following_url":"https://api.github.com/users/gaart/following{/other_user}","gists_url":"https://api.github.com/users/gaart/gists{/gist_id}","starred_url":"https://api.github.com/users/gaart/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gaart/subscriptions","organizations_url":"https://api.github.com/users/gaart/orgs","repos_url":"https://api.github.com/users/gaart/repos","events_url":"https://api.github.com/users/gaart/events{/privacy}","received_events_url":"https://api.github.com/users/gaart/received_events","type":"User","site_admin":false},"emitted_at":1695214089428} {"stream":"deployments","data":{"url":"https://api.github.com/repos/airbytehq/integration-test/deployments/508084910","id":508084910,"node_id":"DE_kwDOF9hP9c4eSMKu","task":"deploy","original_environment":"production","environment":"production","description":null,"created_at":"2022-02-16T10:17:53Z","updated_at":"2022-02-16T10:17:53Z","statuses_url":"https://api.github.com/repos/airbytehq/integration-test/deployments/508084910/statuses","repository_url":"https://api.github.com/repos/airbytehq/integration-test","creator":{"login":"sherifnada","id":6246757,"node_id":"MDQ6VXNlcjYyNDY3NTc=","avatar_url":"https://avatars.githubusercontent.com/u/6246757?v=4","gravatar_id":"","url":"https://api.github.com/users/sherifnada","html_url":"https://github.com/sherifnada","followers_url":"https://api.github.com/users/sherifnada/followers","following_url":"https://api.github.com/users/sherifnada/following{/other_user}","gists_url":"https://api.github.com/users/sherifnada/gists{/gist_id}","starred_url":"https://api.github.com/users/sherifnada/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sherifnada/subscriptions","organizations_url":"https://api.github.com/users/sherifnada/orgs","repos_url":"https://api.github.com/users/sherifnada/repos","events_url":"https://api.github.com/users/sherifnada/events{/privacy}","received_events_url":"https://api.github.com/users/sherifnada/received_events","type":"User","site_admin":false},"sha":"0c033903d2b400c262d75199db5f0bd3c6d81fe2","ref":"master","payload":{},"transient_environment":false,"production_environment":false,"performed_via_github_app":null,"repository":"airbytehq/integration-test"},"emitted_at":1677668748127} {"stream":"issue_comment_reactions","data":{"id":127042034,"node_id":"MDIwOklzc3VlQ29tbWVudFJlYWN0aW9uMTI3MDQyMDM0","user":{"login":"yevhenii-ldv","id":34103125,"node_id":"MDQ6VXNlcjM0MTAzMTI1","avatar_url":"https://avatars.githubusercontent.com/u/34103125?v=4","gravatar_id":"","url":"https://api.github.com/users/yevhenii-ldv","html_url":"https://github.com/yevhenii-ldv","followers_url":"https://api.github.com/users/yevhenii-ldv/followers","following_url":"https://api.github.com/users/yevhenii-ldv/following{/other_user}","gists_url":"https://api.github.com/users/yevhenii-ldv/gists{/gist_id}","starred_url":"https://api.github.com/users/yevhenii-ldv/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yevhenii-ldv/subscriptions","organizations_url":"https://api.github.com/users/yevhenii-ldv/orgs","repos_url":"https://api.github.com/users/yevhenii-ldv/repos","events_url":"https://api.github.com/users/yevhenii-ldv/events{/privacy}","received_events_url":"https://api.github.com/users/yevhenii-ldv/received_events","type":"User","site_admin":false},"content":"hooray","created_at":"2021-09-06T10:18:19Z","repository":"airbytehq/integration-test","comment_id":907296275},"emitted_at":1677668749798} -{"stream":"issue_events","data":{"id":6917774630,"node_id":"RRE_lADOF9hP9c5M9xnAzwAAAAGcVN0m","url":"https://api.github.com/repos/airbytehq/integration-test/issues/events/6917774630","actor":{"login":"annalvova05","id":37615075,"node_id":"MDQ6VXNlcjM3NjE1MDc1","avatar_url":"https://avatars.githubusercontent.com/u/37615075?v=4","gravatar_id":"","url":"https://api.github.com/users/annalvova05","html_url":"https://github.com/annalvova05","followers_url":"https://api.github.com/users/annalvova05/followers","following_url":"https://api.github.com/users/annalvova05/following{/other_user}","gists_url":"https://api.github.com/users/annalvova05/gists{/gist_id}","starred_url":"https://api.github.com/users/annalvova05/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/annalvova05/subscriptions","organizations_url":"https://api.github.com/users/annalvova05/orgs","repos_url":"https://api.github.com/users/annalvova05/repos","events_url":"https://api.github.com/users/annalvova05/events{/privacy}","received_events_url":"https://api.github.com/users/annalvova05/received_events","type":"User","site_admin":false},"event":"review_requested","commit_id":null,"commit_url":null,"created_at":"2022-07-01T11:22:20Z","review_requester":{"login":"annalvova05","id":37615075,"node_id":"MDQ6VXNlcjM3NjE1MDc1","avatar_url":"https://avatars.githubusercontent.com/u/37615075?v=4","gravatar_id":"","url":"https://api.github.com/users/annalvova05","html_url":"https://github.com/annalvova05","followers_url":"https://api.github.com/users/annalvova05/followers","following_url":"https://api.github.com/users/annalvova05/following{/other_user}","gists_url":"https://api.github.com/users/annalvova05/gists{/gist_id}","starred_url":"https://api.github.com/users/annalvova05/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/annalvova05/subscriptions","organizations_url":"https://api.github.com/users/annalvova05/orgs","repos_url":"https://api.github.com/users/annalvova05/repos","events_url":"https://api.github.com/users/annalvova05/events{/privacy}","received_events_url":"https://api.github.com/users/annalvova05/received_events","type":"User","site_admin":false},"requested_reviewer":{"login":"annalvova05","id":37615075,"node_id":"MDQ6VXNlcjM3NjE1MDc1","avatar_url":"https://avatars.githubusercontent.com/u/37615075?v=4","gravatar_id":"","url":"https://api.github.com/users/annalvova05","html_url":"https://github.com/annalvova05","followers_url":"https://api.github.com/users/annalvova05/followers","following_url":"https://api.github.com/users/annalvova05/following{/other_user}","gists_url":"https://api.github.com/users/annalvova05/gists{/gist_id}","starred_url":"https://api.github.com/users/annalvova05/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/annalvova05/subscriptions","organizations_url":"https://api.github.com/users/annalvova05/orgs","repos_url":"https://api.github.com/users/annalvova05/repos","events_url":"https://api.github.com/users/annalvova05/events{/privacy}","received_events_url":"https://api.github.com/users/annalvova05/received_events","type":"User","site_admin":false},"issue":{"url":"https://api.github.com/repos/airbytehq/integration-test/issues/14","repository_url":"https://api.github.com/repos/airbytehq/integration-test","labels_url":"https://api.github.com/repos/airbytehq/integration-test/issues/14/labels{/name}","comments_url":"https://api.github.com/repos/airbytehq/integration-test/issues/14/comments","events_url":"https://api.github.com/repos/airbytehq/integration-test/issues/14/events","html_url":"https://github.com/airbytehq/integration-test/pull/14","id":1291262400,"node_id":"PR_kwDOF9hP9c46s2Qa","number":14,"title":"New PR from feature/branch_5","user":{"login":"grubberr","id":195743,"node_id":"MDQ6VXNlcjE5NTc0Mw==","avatar_url":"https://avatars.githubusercontent.com/u/195743?v=4","gravatar_id":"","url":"https://api.github.com/users/grubberr","html_url":"https://github.com/grubberr","followers_url":"https://api.github.com/users/grubberr/followers","following_url":"https://api.github.com/users/grubberr/following{/other_user}","gists_url":"https://api.github.com/users/grubberr/gists{/gist_id}","starred_url":"https://api.github.com/users/grubberr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/grubberr/subscriptions","organizations_url":"https://api.github.com/users/grubberr/orgs","repos_url":"https://api.github.com/users/grubberr/repos","events_url":"https://api.github.com/users/grubberr/events{/privacy}","received_events_url":"https://api.github.com/users/grubberr/received_events","type":"User","site_admin":false},"labels":[{"id":3984065862,"node_id":"LA_kwDOF9hP9c7teAVG","url":"https://api.github.com/repos/airbytehq/integration-test/labels/labeler","name":"labeler","color":"ededed","default":false,"description":null}],"state":"open","locked":false,"assignee":null,"assignees":[],"milestone":null,"comments":0,"created_at":"2022-07-01T11:05:28Z","updated_at":"2022-10-04T17:41:29Z","closed_at":null,"author_association":"NONE","active_lock_reason":null,"draft":false,"pull_request":{"url":"https://api.github.com/repos/airbytehq/integration-test/pulls/14","html_url":"https://github.com/airbytehq/integration-test/pull/14","diff_url":"https://github.com/airbytehq/integration-test/pull/14.diff","patch_url":"https://github.com/airbytehq/integration-test/pull/14.patch","merged_at":null},"body":"Signed-off-by: Sergey Chvalyuk ","reactions":{"url":"https://api.github.com/repos/airbytehq/integration-test/issues/14/reactions","total_count":0,"+1":0,"-1":0,"laugh":0,"hooray":0,"confused":0,"heart":0,"rocket":0,"eyes":0},"timeline_url":"https://api.github.com/repos/airbytehq/integration-test/issues/14/timeline","performed_via_github_app":null,"state_reason":null},"performed_via_github_app":null,"repository":"airbytehq/integration-test"},"emitted_at":1677668750398} +{"stream":"issue_events","data":{"id": 6917774630, "node_id": "RRE_lADOF9hP9c5M9xnAzwAAAAGcVN0m", "url": "https://api.github.com/repos/airbytehq/integration-test/issues/events/6917774630", "actor": {"login": "annalvova05", "id": 37615075, "node_id": "MDQ6VXNlcjM3NjE1MDc1", "avatar_url": "https://avatars.githubusercontent.com/u/37615075?v=4", "gravatar_id": "", "url": "https://api.github.com/users/annalvova05", "html_url": "https://github.com/annalvova05", "followers_url": "https://api.github.com/users/annalvova05/followers", "following_url": "https://api.github.com/users/annalvova05/following{/other_user}", "gists_url": "https://api.github.com/users/annalvova05/gists{/gist_id}", "starred_url": "https://api.github.com/users/annalvova05/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/annalvova05/subscriptions", "organizations_url": "https://api.github.com/users/annalvova05/orgs", "repos_url": "https://api.github.com/users/annalvova05/repos", "events_url": "https://api.github.com/users/annalvova05/events{/privacy}", "received_events_url": "https://api.github.com/users/annalvova05/received_events", "type": "User", "site_admin": false}, "event": "review_requested", "commit_id": null, "commit_url": null, "created_at": "2022-07-01T11:22:20Z", "review_requester": {"login": "annalvova05", "id": 37615075, "node_id": "MDQ6VXNlcjM3NjE1MDc1", "avatar_url": "https://avatars.githubusercontent.com/u/37615075?v=4", "gravatar_id": "", "url": "https://api.github.com/users/annalvova05", "html_url": "https://github.com/annalvova05", "followers_url": "https://api.github.com/users/annalvova05/followers", "following_url": "https://api.github.com/users/annalvova05/following{/other_user}", "gists_url": "https://api.github.com/users/annalvova05/gists{/gist_id}", "starred_url": "https://api.github.com/users/annalvova05/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/annalvova05/subscriptions", "organizations_url": "https://api.github.com/users/annalvova05/orgs", "repos_url": "https://api.github.com/users/annalvova05/repos", "events_url": "https://api.github.com/users/annalvova05/events{/privacy}", "received_events_url": "https://api.github.com/users/annalvova05/received_events", "type": "User", "site_admin": false}, "requested_reviewer": {"login": "annalvova05", "id": 37615075, "node_id": "MDQ6VXNlcjM3NjE1MDc1", "avatar_url": "https://avatars.githubusercontent.com/u/37615075?v=4", "gravatar_id": "", "url": "https://api.github.com/users/annalvova05", "html_url": "https://github.com/annalvova05", "followers_url": "https://api.github.com/users/annalvova05/followers", "following_url": "https://api.github.com/users/annalvova05/following{/other_user}", "gists_url": "https://api.github.com/users/annalvova05/gists{/gist_id}", "starred_url": "https://api.github.com/users/annalvova05/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/annalvova05/subscriptions", "organizations_url": "https://api.github.com/users/annalvova05/orgs", "repos_url": "https://api.github.com/users/annalvova05/repos", "events_url": "https://api.github.com/users/annalvova05/events{/privacy}", "received_events_url": "https://api.github.com/users/annalvova05/received_events", "type": "User", "site_admin": false}, "issue": {"url": "https://api.github.com/repos/airbytehq/integration-test/issues/14", "repository_url": "https://api.github.com/repos/airbytehq/integration-test", "labels_url": "https://api.github.com/repos/airbytehq/integration-test/issues/14/labels{/name}", "comments_url": "https://api.github.com/repos/airbytehq/integration-test/issues/14/comments", "events_url": "https://api.github.com/repos/airbytehq/integration-test/issues/14/events", "html_url": "https://github.com/airbytehq/integration-test/pull/14", "id": 1291262400, "node_id": "PR_kwDOF9hP9c46s2Qa", "number": 14, "title": "New PR from feature/branch_5", "user": {"login": "grubberr", "id": 195743, "node_id": "MDQ6VXNlcjE5NTc0Mw==", "avatar_url": "https://avatars.githubusercontent.com/u/195743?v=4", "gravatar_id": "", "url": "https://api.github.com/users/grubberr", "html_url": "https://github.com/grubberr", "followers_url": "https://api.github.com/users/grubberr/followers", "following_url": "https://api.github.com/users/grubberr/following{/other_user}", "gists_url": "https://api.github.com/users/grubberr/gists{/gist_id}", "starred_url": "https://api.github.com/users/grubberr/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/grubberr/subscriptions", "organizations_url": "https://api.github.com/users/grubberr/orgs", "repos_url": "https://api.github.com/users/grubberr/repos", "events_url": "https://api.github.com/users/grubberr/events{/privacy}", "received_events_url": "https://api.github.com/users/grubberr/received_events", "type": "User", "site_admin": false}, "labels": [{"id": 3984065862, "node_id": "LA_kwDOF9hP9c7teAVG", "url": "https://api.github.com/repos/airbytehq/integration-test/labels/labeler", "name": "labeler", "color": "ededed", "default": false, "description": null}], "state": "open", "locked": false, "assignee": null, "assignees": [], "milestone": null, "comments": 0, "created_at": "2022-07-01T11:05:28Z", "updated_at": "2022-10-04T17:41:29Z", "closed_at": null, "author_association": "FIRST_TIME_CONTRIBUTOR", "active_lock_reason": null, "draft": false, "pull_request": {"url": "https://api.github.com/repos/airbytehq/integration-test/pulls/14", "html_url": "https://github.com/airbytehq/integration-test/pull/14", "diff_url": "https://github.com/airbytehq/integration-test/pull/14.diff", "patch_url": "https://github.com/airbytehq/integration-test/pull/14.patch", "merged_at": null}, "body": "Signed-off-by: Sergey Chvalyuk ", "reactions": {"url": "https://api.github.com/repos/airbytehq/integration-test/issues/14/reactions", "total_count": 0, "+1": 0, "-1": 0, "laugh": 0, "hooray": 0, "confused": 0, "heart": 0, "rocket": 0, "eyes": 0}, "timeline_url": "https://api.github.com/repos/airbytehq/integration-test/issues/14/timeline", "performed_via_github_app": null, "state_reason": null}, "performed_via_github_app": null, "repository": "airbytehq/integration-test"},"emitted_at":1677668750398} {"stream":"issue_labels","data":{"id":3295756566,"node_id":"MDU6TGFiZWwzMjk1NzU2NTY2","url":"https://api.github.com/repos/airbytehq/integration-test/labels/bug","name":"bug","color":"d73a4a","default":true,"description":"Something isn't working","repository":"airbytehq/integration-test"},"emitted_at":1677668750697} {"stream":"issue_milestones","data":{"url":"https://api.github.com/repos/airbytehq/integration-test/milestones/1","html_url":"https://github.com/airbytehq/integration-test/milestone/1","labels_url":"https://api.github.com/repos/airbytehq/integration-test/milestones/1/labels","id":7097357,"node_id":"MI_kwDOF9hP9c4AbEwN","number":1,"title":"main","description":null,"creator":{"login":"gaart","id":743901,"node_id":"MDQ6VXNlcjc0MzkwMQ==","avatar_url":"https://avatars.githubusercontent.com/u/743901?v=4","gravatar_id":"","url":"https://api.github.com/users/gaart","html_url":"https://github.com/gaart","followers_url":"https://api.github.com/users/gaart/followers","following_url":"https://api.github.com/users/gaart/following{/other_user}","gists_url":"https://api.github.com/users/gaart/gists{/gist_id}","starred_url":"https://api.github.com/users/gaart/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gaart/subscriptions","organizations_url":"https://api.github.com/users/gaart/orgs","repos_url":"https://api.github.com/users/gaart/repos","events_url":"https://api.github.com/users/gaart/events{/privacy}","received_events_url":"https://api.github.com/users/gaart/received_events","type":"User","site_admin":false},"open_issues":3,"closed_issues":1,"state":"open","created_at":"2021-08-27T15:43:44Z","updated_at":"2021-08-27T16:02:49Z","due_on":null,"closed_at":null,"repository":"airbytehq/integration-test"},"emitted_at":1677668751023} {"stream":"issue_reactions","data":{"node_id":"MDEzOklzc3VlUmVhY3Rpb24xMjcwNDg0NTY=","id":127048456,"content":"ROCKET","created_at":"2021-09-06T11:13:32Z","user":{"node_id":"MDQ6VXNlcjM0MTAzMTI1","id":34103125,"login":"yevhenii-ldv","avatar_url":"https://avatars.githubusercontent.com/u/34103125?u=3e49bb73177a9f70896e3d49b34656ab659c70a5&v=4","html_url":"https://github.com/yevhenii-ldv","site_admin":false,"type":"User"},"repository":"airbytehq/integration-test","issue_number":11},"emitted_at":1677668751465} @@ -16,20 +18,24 @@ {"stream":"project_cards","data":{"url":"https://api.github.com/projects/columns/cards/77859890","project_url":"https://api.github.com/projects/13167124","id":77859890,"node_id":"PRC_lALOF9hP9c4AyOoUzgSkDDI","note":"note_1","archived":false,"creator":{"login":"grubberr","id":195743,"node_id":"MDQ6VXNlcjE5NTc0Mw==","avatar_url":"https://avatars.githubusercontent.com/u/195743?v=4","gravatar_id":"","url":"https://api.github.com/users/grubberr","html_url":"https://github.com/grubberr","followers_url":"https://api.github.com/users/grubberr/followers","following_url":"https://api.github.com/users/grubberr/following{/other_user}","gists_url":"https://api.github.com/users/grubberr/gists{/gist_id}","starred_url":"https://api.github.com/users/grubberr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/grubberr/subscriptions","organizations_url":"https://api.github.com/users/grubberr/orgs","repos_url":"https://api.github.com/users/grubberr/repos","events_url":"https://api.github.com/users/grubberr/events{/privacy}","received_events_url":"https://api.github.com/users/grubberr/received_events","type":"User","site_admin":false},"created_at":"2022-02-17T09:56:51Z","updated_at":"2022-02-17T09:56:51Z","column_url":"https://api.github.com/projects/columns/17807006","repository":"airbytehq/integration-test","project_id":13167124,"column_id":17807006},"emitted_at":1677668754200} {"stream":"project_columns","data":{"url":"https://api.github.com/projects/columns/17807092","project_url":"https://api.github.com/projects/13167124","cards_url":"https://api.github.com/projects/columns/17807092/cards","id":17807092,"node_id":"PC_lATOF9hP9c4AyOoUzgEPtvQ","name":"column_2","created_at":"2022-02-17T09:57:27Z","updated_at":"2022-02-17T09:57:27Z","repository":"airbytehq/integration-test","project_id":13167124},"emitted_at":1677668754456} {"stream":"projects","data":{"owner_url":"https://api.github.com/repos/airbytehq/integration-test","url":"https://api.github.com/projects/13167124","html_url":"https://github.com/airbytehq/integration-test/projects/3","columns_url":"https://api.github.com/projects/13167124/columns","id":13167124,"node_id":"PRO_kwLOF9hP9c4AyOoU","name":"project_3","body":null,"number":3,"state":"open","creator":{"login":"gaart","id":743901,"node_id":"MDQ6VXNlcjc0MzkwMQ==","avatar_url":"https://avatars.githubusercontent.com/u/743901?v=4","gravatar_id":"","url":"https://api.github.com/users/gaart","html_url":"https://github.com/gaart","followers_url":"https://api.github.com/users/gaart/followers","following_url":"https://api.github.com/users/gaart/following{/other_user}","gists_url":"https://api.github.com/users/gaart/gists{/gist_id}","starred_url":"https://api.github.com/users/gaart/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gaart/subscriptions","organizations_url":"https://api.github.com/users/gaart/orgs","repos_url":"https://api.github.com/users/gaart/repos","events_url":"https://api.github.com/users/gaart/events{/privacy}","received_events_url":"https://api.github.com/users/gaart/received_events","type":"User","site_admin":false},"created_at":"2021-08-27T15:43:57Z","updated_at":"2022-02-17T12:16:56Z","repository":"airbytehq/integration-test"},"emitted_at":1677668754468} +{"stream":"projects_v2","data":{"closed":false,"created_at":"2023-09-25T18:34:52Z","closed_at":null,"updated_at":"2023-09-25T18:35:45Z","creator":{"avatarUrl":"https://avatars.githubusercontent.com/u/92915184?u=e53c87d81ec6fb0596bc0f75e12e84e8f0df8d83&v=4","login":"airbyteio","resourcePath":"/airbyteio","url":"https://github.com/airbyteio"},"node_id":"PVT_kwDOA4_XW84AV7NS","id":5747538,"number":58,"public":false,"readme":"# Title\nintegration test project","short_description":"integration test project description","template":false,"title":"integration test project","url":"https://github.com/orgs/airbytehq/projects/58","viewerCanClose":true,"viewerCanReopen":true,"viewerCanUpdate":true,"owner_id":"MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3","repository":"airbytehq/integration-test"},"emitted_at":1695666959656} {"stream":"pull_request_comment_reactions","data":{"node_id":"MDMyOlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudFJlYWN0aW9uMTI3MDUxNDM4","id":127051438,"content":"HEART","created_at":"2021-09-06T11:37:25Z","user":{"node_id":"MDQ6VXNlcjM0MTAzMTI1","id":34103125,"login":"yevhenii-ldv","avatar_url":"https://avatars.githubusercontent.com/u/34103125?u=3e49bb73177a9f70896e3d49b34656ab659c70a5&v=4","html_url":"https://github.com/yevhenii-ldv","site_admin":false,"type":"User"},"repository":"airbytehq/integration-test","comment_id":699253726},"emitted_at":1677668755106} {"stream":"pull_request_commits","data":{"sha":"00a74695eb754865a552196ee158a87f0b9dcff7","node_id":"MDY6Q29tbWl0NDAwMDUyMjEzOjAwYTc0Njk1ZWI3NTQ4NjVhNTUyMTk2ZWUxNThhODdmMGI5ZGNmZjc=","commit":{"author":{"name":"Arthur Galuza","email":"a.galuza@exaft.com","date":"2021-08-27T15:41:11Z"},"committer":{"name":"Arthur Galuza","email":"a.galuza@exaft.com","date":"2021-08-27T15:41:11Z"},"message":"commit number 0","tree":{"sha":"3f2a52f90f9acc30359b00065e5b989267fef1f5","url":"https://api.github.com/repos/airbytehq/integration-test/git/trees/3f2a52f90f9acc30359b00065e5b989267fef1f5"},"url":"https://api.github.com/repos/airbytehq/integration-test/git/commits/00a74695eb754865a552196ee158a87f0b9dcff7","comment_count":0,"verification":{"verified":false,"reason":"unsigned","signature":null,"payload":null}},"url":"https://api.github.com/repos/airbytehq/integration-test/commits/00a74695eb754865a552196ee158a87f0b9dcff7","html_url":"https://github.com/airbytehq/integration-test/commit/00a74695eb754865a552196ee158a87f0b9dcff7","comments_url":"https://api.github.com/repos/airbytehq/integration-test/commits/00a74695eb754865a552196ee158a87f0b9dcff7/comments","author":{"login":"gaart","id":743901,"node_id":"MDQ6VXNlcjc0MzkwMQ==","avatar_url":"https://avatars.githubusercontent.com/u/743901?v=4","gravatar_id":"","url":"https://api.github.com/users/gaart","html_url":"https://github.com/gaart","followers_url":"https://api.github.com/users/gaart/followers","following_url":"https://api.github.com/users/gaart/following{/other_user}","gists_url":"https://api.github.com/users/gaart/gists{/gist_id}","starred_url":"https://api.github.com/users/gaart/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gaart/subscriptions","organizations_url":"https://api.github.com/users/gaart/orgs","repos_url":"https://api.github.com/users/gaart/repos","events_url":"https://api.github.com/users/gaart/events{/privacy}","received_events_url":"https://api.github.com/users/gaart/received_events","type":"User","site_admin":false},"committer":{"login":"gaart","id":743901,"node_id":"MDQ6VXNlcjc0MzkwMQ==","avatar_url":"https://avatars.githubusercontent.com/u/743901?v=4","gravatar_id":"","url":"https://api.github.com/users/gaart","html_url":"https://github.com/gaart","followers_url":"https://api.github.com/users/gaart/followers","following_url":"https://api.github.com/users/gaart/following{/other_user}","gists_url":"https://api.github.com/users/gaart/gists{/gist_id}","starred_url":"https://api.github.com/users/gaart/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gaart/subscriptions","organizations_url":"https://api.github.com/users/gaart/orgs","repos_url":"https://api.github.com/users/gaart/repos","events_url":"https://api.github.com/users/gaart/events{/privacy}","received_events_url":"https://api.github.com/users/gaart/received_events","type":"User","site_admin":false},"parents":[{"sha":"978753aeb56f7b49872279d1b491411a6235aa90","url":"https://api.github.com/repos/airbytehq/integration-test/commits/978753aeb56f7b49872279d1b491411a6235aa90","html_url":"https://github.com/airbytehq/integration-test/commit/978753aeb56f7b49872279d1b491411a6235aa90"}],"repository":"airbytehq/integration-test","pull_number":5},"emitted_at":1677668756160} -{"stream":"pull_request_stats","data":{"node_id":"MDExOlB1bGxSZXF1ZXN0NzIxNDM1NTA2","id":721435506,"number":5,"updated_at":"2021-08-27T15:53:14Z","changed_files":5,"deletions":0,"additions":5,"merged":false,"mergeable":"MERGEABLE","can_be_rebased":true,"maintainer_can_modify":false,"merge_state_status":"BLOCKED","comments":0,"commits":5,"review_comments":0,"merged_by":null,"repository":"airbytehq/integration-test"},"emitted_at":1677668759962} -{"stream": "pull_requests", "data": {"url": "https://api.github.com/repos/airbytehq/integration-test/pulls/5", "id": 721435506, "node_id": "MDExOlB1bGxSZXF1ZXN0NzIxNDM1NTA2", "html_url": "https://github.com/airbytehq/integration-test/pull/5", "diff_url": "https://github.com/airbytehq/integration-test/pull/5.diff", "patch_url": "https://github.com/airbytehq/integration-test/pull/5.patch", "issue_url": "https://api.github.com/repos/airbytehq/integration-test/issues/5", "number": 5, "state": "open", "locked": false, "title": "New PR from feature/branch_4", "user": {"login": "gaart", "id": 743901, "node_id": "MDQ6VXNlcjc0MzkwMQ==", "avatar_url": "https://avatars.githubusercontent.com/u/743901?v=4", "gravatar_id": "", "url": "https://api.github.com/users/gaart", "html_url": "https://github.com/gaart", "followers_url": "https://api.github.com/users/gaart/followers", "following_url": "https://api.github.com/users/gaart/following{/other_user}", "gists_url": "https://api.github.com/users/gaart/gists{/gist_id}", "starred_url": "https://api.github.com/users/gaart/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/gaart/subscriptions", "organizations_url": "https://api.github.com/users/gaart/orgs", "repos_url": "https://api.github.com/users/gaart/repos", "events_url": "https://api.github.com/users/gaart/events{/privacy}", "received_events_url": "https://api.github.com/users/gaart/received_events", "type": "User", "site_admin": false}, "body": null, "created_at": "2021-08-27T15:43:40Z", "updated_at": "2021-08-27T15:53:14Z", "closed_at": null, "merged_at": null, "merge_commit_sha": "191309e3da8b36705156348ae73f4dca836533f9", "assignee": null, "assignees": [], "requested_reviewers": [], "requested_teams": [], "labels": [{"id": 3295756566, "node_id": "MDU6TGFiZWwzMjk1NzU2NTY2", "url": "https://api.github.com/repos/airbytehq/integration-test/labels/bug", "name": "bug", "color": "d73a4a", "default": true, "description": "Something isn't working"}, {"id": 3300346197, "node_id": "MDU6TGFiZWwzMzAwMzQ2MTk3", "url": "https://api.github.com/repos/airbytehq/integration-test/labels/critical", "name": "critical", "color": "ededed", "default": false, "description": null}], "milestone": null, "draft": false, "commits_url": "https://api.github.com/repos/airbytehq/integration-test/pulls/5/commits", "review_comments_url": "https://api.github.com/repos/airbytehq/integration-test/pulls/5/comments", "review_comment_url": "https://api.github.com/repos/airbytehq/integration-test/pulls/comments{/number}", "comments_url": "https://api.github.com/repos/airbytehq/integration-test/issues/5/comments", "statuses_url": "https://api.github.com/repos/airbytehq/integration-test/statuses/31a3e3f19fefce60fba6bfc69dd2b3fb5195a083", "head": {"label": "airbytehq:feature/branch_4", "ref": "feature/branch_4", "sha": "31a3e3f19fefce60fba6bfc69dd2b3fb5195a083", "user": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "repo_id": 400052213}, "base": {"label": "airbytehq:master", "ref": "master", "sha": "978753aeb56f7b49872279d1b491411a6235aa90", "user": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "repo": {"id": 400052213, "node_id": "MDEwOlJlcG9zaXRvcnk0MDAwNTIyMTM=", "name": "integration-test", "full_name": "airbytehq/integration-test", "private": false, "owner": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "html_url": "https://github.com/airbytehq/integration-test", "description": "Used for integration testing the Github source connector", "fork": false, "url": "https://api.github.com/repos/airbytehq/integration-test", "forks_url": "https://api.github.com/repos/airbytehq/integration-test/forks", "keys_url": "https://api.github.com/repos/airbytehq/integration-test/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/airbytehq/integration-test/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/airbytehq/integration-test/teams", "hooks_url": "https://api.github.com/repos/airbytehq/integration-test/hooks", "issue_events_url": "https://api.github.com/repos/airbytehq/integration-test/issues/events{/number}", "events_url": "https://api.github.com/repos/airbytehq/integration-test/events", "assignees_url": "https://api.github.com/repos/airbytehq/integration-test/assignees{/user}", "branches_url": "https://api.github.com/repos/airbytehq/integration-test/branches{/branch}", "tags_url": "https://api.github.com/repos/airbytehq/integration-test/tags", "blobs_url": "https://api.github.com/repos/airbytehq/integration-test/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/airbytehq/integration-test/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/airbytehq/integration-test/git/refs{/sha}", "trees_url": "https://api.github.com/repos/airbytehq/integration-test/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/airbytehq/integration-test/statuses/{sha}", "languages_url": "https://api.github.com/repos/airbytehq/integration-test/languages", "stargazers_url": "https://api.github.com/repos/airbytehq/integration-test/stargazers", "contributors_url": "https://api.github.com/repos/airbytehq/integration-test/contributors", "subscribers_url": "https://api.github.com/repos/airbytehq/integration-test/subscribers", "subscription_url": "https://api.github.com/repos/airbytehq/integration-test/subscription", "commits_url": "https://api.github.com/repos/airbytehq/integration-test/commits{/sha}", "git_commits_url": "https://api.github.com/repos/airbytehq/integration-test/git/commits{/sha}", "comments_url": "https://api.github.com/repos/airbytehq/integration-test/comments{/number}", "issue_comment_url": "https://api.github.com/repos/airbytehq/integration-test/issues/comments{/number}", "contents_url": "https://api.github.com/repos/airbytehq/integration-test/contents/{+path}", "compare_url": "https://api.github.com/repos/airbytehq/integration-test/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/airbytehq/integration-test/merges", "archive_url": "https://api.github.com/repos/airbytehq/integration-test/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/airbytehq/integration-test/downloads", "issues_url": "https://api.github.com/repos/airbytehq/integration-test/issues{/number}", "pulls_url": "https://api.github.com/repos/airbytehq/integration-test/pulls{/number}", "milestones_url": "https://api.github.com/repos/airbytehq/integration-test/milestones{/number}", "notifications_url": "https://api.github.com/repos/airbytehq/integration-test/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/airbytehq/integration-test/labels{/name}", "releases_url": "https://api.github.com/repos/airbytehq/integration-test/releases{/id}", "deployments_url": "https://api.github.com/repos/airbytehq/integration-test/deployments", "created_at": "2021-08-26T05:32:43Z", "updated_at": "2022-07-08T01:27:13Z", "pushed_at": "2023-05-03T16:40:56Z", "git_url": "git://github.com/airbytehq/integration-test.git", "ssh_url": "git@github.com:airbytehq/integration-test.git", "clone_url": "https://github.com/airbytehq/integration-test.git", "svn_url": "https://github.com/airbytehq/integration-test", "homepage": null, "size": 11, "stargazers_count": 4, "watchers_count": 4, "language": null, "has_issues": true, "has_projects": true, "has_downloads": true, "has_wiki": true, "has_pages": false, "has_discussions": false, "forks_count": 2, "mirror_url": null, "archived": false, "disabled": false, "open_issues_count": 10, "license": null, "allow_forking": true, "is_template": false, "web_commit_signoff_required": false, "topics": [], "visibility": "public", "forks": 2, "open_issues": 10, "watchers": 4, "default_branch": "master"}, "repo_id": null}, "_links": {"self": {"href": "https://api.github.com/repos/airbytehq/integration-test/pulls/5"}, "html": {"href": "https://github.com/airbytehq/integration-test/pull/5"}, "issue": {"href": "https://api.github.com/repos/airbytehq/integration-test/issues/5"}, "comments": {"href": "https://api.github.com/repos/airbytehq/integration-test/issues/5/comments"}, "review_comments": {"href": "https://api.github.com/repos/airbytehq/integration-test/pulls/5/comments"}, "review_comment": {"href": "https://api.github.com/repos/airbytehq/integration-test/pulls/comments{/number}"}, "commits": {"href": "https://api.github.com/repos/airbytehq/integration-test/pulls/5/commits"}, "statuses": {"href": "https://api.github.com/repos/airbytehq/integration-test/statuses/31a3e3f19fefce60fba6bfc69dd2b3fb5195a083"}}, "author_association": "CONTRIBUTOR", "auto_merge": null, "active_lock_reason": null, "repository": "airbytehq/integration-test"}, "emitted_at": 1685698519242} +{"stream": "pull_request_stats", "data": {"node_id": "MDExOlB1bGxSZXF1ZXN0NzIxNDM1NTA2", "id": 721435506, "number": 5, "updated_at": "2023-11-16T14:38:58Z", "changed_files": 5, "deletions": 0, "additions": 5, "merged": false, "mergeable": "MERGEABLE", "can_be_rebased": false, "maintainer_can_modify": false, "merge_state_status": "BLOCKED", "comments": 0, "commits": 5, "review_comments": 0, "merged_by": null, "repository": "airbytehq/integration-test"}, "emitted_at": 1700557306144} +{"stream": "pull_requests", "data": {"url": "https://api.github.com/repos/airbytehq/integration-test/pulls/5", "id": 721435506, "node_id": "MDExOlB1bGxSZXF1ZXN0NzIxNDM1NTA2", "html_url": "https://github.com/airbytehq/integration-test/pull/5", "diff_url": "https://github.com/airbytehq/integration-test/pull/5.diff", "patch_url": "https://github.com/airbytehq/integration-test/pull/5.patch", "issue_url": "https://api.github.com/repos/airbytehq/integration-test/issues/5", "number": 5, "state": "closed", "locked": false, "title": "New PR from feature/branch_4", "user": {"login": "gaart", "id": 743901, "node_id": "MDQ6VXNlcjc0MzkwMQ==", "avatar_url": "https://avatars.githubusercontent.com/u/743901?v=4", "gravatar_id": "", "url": "https://api.github.com/users/gaart", "html_url": "https://github.com/gaart", "followers_url": "https://api.github.com/users/gaart/followers", "following_url": "https://api.github.com/users/gaart/following{/other_user}", "gists_url": "https://api.github.com/users/gaart/gists{/gist_id}", "starred_url": "https://api.github.com/users/gaart/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/gaart/subscriptions", "organizations_url": "https://api.github.com/users/gaart/orgs", "repos_url": "https://api.github.com/users/gaart/repos", "events_url": "https://api.github.com/users/gaart/events{/privacy}", "received_events_url": "https://api.github.com/users/gaart/received_events", "type": "User", "site_admin": false}, "body": null, "created_at": "2021-08-27T15:43:40Z", "updated_at": "2023-11-16T14:38:58Z", "closed_at": "2023-11-16T14:38:58Z", "merged_at": null, "merge_commit_sha": "191309e3da8b36705156348ae73f4dca836533f9", "assignee": null, "assignees": [], "requested_reviewers": [], "requested_teams": [], "labels": [{"id": 3295756566, "node_id": "MDU6TGFiZWwzMjk1NzU2NTY2", "url": "https://api.github.com/repos/airbytehq/integration-test/labels/bug", "name": "bug", "color": "d73a4a", "default": true, "description": "Something isn't working"}, {"id": 3300346197, "node_id": "MDU6TGFiZWwzMzAwMzQ2MTk3", "url": "https://api.github.com/repos/airbytehq/integration-test/labels/critical", "name": "critical", "color": "ededed", "default": false, "description": null}], "milestone": null, "draft": false, "commits_url": "https://api.github.com/repos/airbytehq/integration-test/pulls/5/commits", "review_comments_url": "https://api.github.com/repos/airbytehq/integration-test/pulls/5/comments", "review_comment_url": "https://api.github.com/repos/airbytehq/integration-test/pulls/comments{/number}", "comments_url": "https://api.github.com/repos/airbytehq/integration-test/issues/5/comments", "statuses_url": "https://api.github.com/repos/airbytehq/integration-test/statuses/31a3e3f19fefce60fba6bfc69dd2b3fb5195a083", "head": {"label": "airbytehq:feature/branch_4", "ref": "feature/branch_4", "sha": "31a3e3f19fefce60fba6bfc69dd2b3fb5195a083", "user": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "repo_id": 400052213}, "base": {"label": "airbytehq:master", "ref": "master", "sha": "978753aeb56f7b49872279d1b491411a6235aa90", "user": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "repo": {"id": 400052213, "node_id": "MDEwOlJlcG9zaXRvcnk0MDAwNTIyMTM=", "name": "integration-test", "full_name": "airbytehq/integration-test", "private": false, "owner": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "html_url": "https://github.com/airbytehq/integration-test", "description": "Used for integration testing the Github source connector", "fork": false, "url": "https://api.github.com/repos/airbytehq/integration-test", "forks_url": "https://api.github.com/repos/airbytehq/integration-test/forks", "keys_url": "https://api.github.com/repos/airbytehq/integration-test/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/airbytehq/integration-test/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/airbytehq/integration-test/teams", "hooks_url": "https://api.github.com/repos/airbytehq/integration-test/hooks", "issue_events_url": "https://api.github.com/repos/airbytehq/integration-test/issues/events{/number}", "events_url": "https://api.github.com/repos/airbytehq/integration-test/events", "assignees_url": "https://api.github.com/repos/airbytehq/integration-test/assignees{/user}", "branches_url": "https://api.github.com/repos/airbytehq/integration-test/branches{/branch}", "tags_url": "https://api.github.com/repos/airbytehq/integration-test/tags", "blobs_url": "https://api.github.com/repos/airbytehq/integration-test/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/airbytehq/integration-test/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/airbytehq/integration-test/git/refs{/sha}", "trees_url": "https://api.github.com/repos/airbytehq/integration-test/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/airbytehq/integration-test/statuses/{sha}", "languages_url": "https://api.github.com/repos/airbytehq/integration-test/languages", "stargazers_url": "https://api.github.com/repos/airbytehq/integration-test/stargazers", "contributors_url": "https://api.github.com/repos/airbytehq/integration-test/contributors", "subscribers_url": "https://api.github.com/repos/airbytehq/integration-test/subscribers", "subscription_url": "https://api.github.com/repos/airbytehq/integration-test/subscription", "commits_url": "https://api.github.com/repos/airbytehq/integration-test/commits{/sha}", "git_commits_url": "https://api.github.com/repos/airbytehq/integration-test/git/commits{/sha}", "comments_url": "https://api.github.com/repos/airbytehq/integration-test/comments{/number}", "issue_comment_url": "https://api.github.com/repos/airbytehq/integration-test/issues/comments{/number}", "contents_url": "https://api.github.com/repos/airbytehq/integration-test/contents/{+path}", "compare_url": "https://api.github.com/repos/airbytehq/integration-test/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/airbytehq/integration-test/merges", "archive_url": "https://api.github.com/repos/airbytehq/integration-test/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/airbytehq/integration-test/downloads", "issues_url": "https://api.github.com/repos/airbytehq/integration-test/issues{/number}", "pulls_url": "https://api.github.com/repos/airbytehq/integration-test/pulls{/number}", "milestones_url": "https://api.github.com/repos/airbytehq/integration-test/milestones{/number}", "notifications_url": "https://api.github.com/repos/airbytehq/integration-test/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/airbytehq/integration-test/labels{/name}", "releases_url": "https://api.github.com/repos/airbytehq/integration-test/releases{/id}", "deployments_url": "https://api.github.com/repos/airbytehq/integration-test/deployments", "created_at": "2021-08-26T05:32:43Z", "updated_at": "2023-11-16T14:48:53Z", "pushed_at": "2023-05-03T16:40:56Z", "git_url": "git://github.com/airbytehq/integration-test.git", "ssh_url": "git@github.com:airbytehq/integration-test.git", "clone_url": "https://github.com/airbytehq/integration-test.git", "svn_url": "https://github.com/airbytehq/integration-test", "homepage": null, "size": 11, "stargazers_count": 4, "watchers_count": 4, "language": null, "has_issues": true, "has_projects": true, "has_downloads": true, "has_wiki": true, "has_pages": false, "has_discussions": false, "forks_count": 2, "mirror_url": null, "archived": false, "disabled": false, "open_issues_count": 6, "license": null, "allow_forking": true, "is_template": false, "web_commit_signoff_required": false, "topics": [], "visibility": "public", "forks": 2, "open_issues": 6, "watchers": 4, "default_branch": "master"}, "repo_id": null}, "_links": {"self": {"href": "https://api.github.com/repos/airbytehq/integration-test/pulls/5"}, "html": {"href": "https://github.com/airbytehq/integration-test/pull/5"}, "issue": {"href": "https://api.github.com/repos/airbytehq/integration-test/issues/5"}, "comments": {"href": "https://api.github.com/repos/airbytehq/integration-test/issues/5/comments"}, "review_comments": {"href": "https://api.github.com/repos/airbytehq/integration-test/pulls/5/comments"}, "review_comment": {"href": "https://api.github.com/repos/airbytehq/integration-test/pulls/comments{/number}"}, "commits": {"href": "https://api.github.com/repos/airbytehq/integration-test/pulls/5/commits"}, "statuses": {"href": "https://api.github.com/repos/airbytehq/integration-test/statuses/31a3e3f19fefce60fba6bfc69dd2b3fb5195a083"}}, "author_association": "CONTRIBUTOR", "auto_merge": null, "active_lock_reason": null, "repository": "airbytehq/integration-test"}, "emitted_at": 1700585060024} {"stream":"releases","data":{"url":"https://api.github.com/repos/airbytehq/integration-test/releases/48581586","assets_url":"https://api.github.com/repos/airbytehq/integration-test/releases/48581586/assets","upload_url":"https://uploads.github.com/repos/airbytehq/integration-test/releases/48581586/assets{?name,label}","html_url":"https://github.com/airbytehq/integration-test/releases/tag/dev-0.9","id":48581586,"author":{"login":"gaart","id":743901,"node_id":"MDQ6VXNlcjc0MzkwMQ==","avatar_url":"https://avatars.githubusercontent.com/u/743901?v=4","gravatar_id":"","url":"https://api.github.com/users/gaart","html_url":"https://github.com/gaart","followers_url":"https://api.github.com/users/gaart/followers","following_url":"https://api.github.com/users/gaart/following{/other_user}","gists_url":"https://api.github.com/users/gaart/gists{/gist_id}","starred_url":"https://api.github.com/users/gaart/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gaart/subscriptions","organizations_url":"https://api.github.com/users/gaart/orgs","repos_url":"https://api.github.com/users/gaart/repos","events_url":"https://api.github.com/users/gaart/events{/privacy}","received_events_url":"https://api.github.com/users/gaart/received_events","type":"User","site_admin":false},"node_id":"MDc6UmVsZWFzZTQ4NTgxNTg2","tag_name":"dev-0.9","target_commitish":"master","name":"9 global release","draft":false,"prerelease":false,"created_at":"2021-08-27T07:03:09Z","published_at":"2021-08-27T15:43:53Z","assets":[],"tarball_url":"https://api.github.com/repos/airbytehq/integration-test/tarball/dev-0.9","zipball_url":"https://api.github.com/repos/airbytehq/integration-test/zipball/dev-0.9","body":"","repository":"airbytehq/integration-test"},"emitted_at":1677668760424} -{"stream": "repositories", "data": {"id": 283046497, "node_id": "MDEwOlJlcG9zaXRvcnkyODMwNDY0OTc=", "name": "airbyte", "full_name": "airbytehq/airbyte", "private": false, "owner": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "html_url": "https://github.com/airbytehq/airbyte", "description": "Data integration platform for ELT pipelines from APIs, databases & files to warehouses & lakes.", "fork": false, "url": "https://api.github.com/repos/airbytehq/airbyte", "forks_url": "https://api.github.com/repos/airbytehq/airbyte/forks", "keys_url": "https://api.github.com/repos/airbytehq/airbyte/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/airbytehq/airbyte/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/airbytehq/airbyte/teams", "hooks_url": "https://api.github.com/repos/airbytehq/airbyte/hooks", "issue_events_url": "https://api.github.com/repos/airbytehq/airbyte/issues/events{/number}", "events_url": "https://api.github.com/repos/airbytehq/airbyte/events", "assignees_url": "https://api.github.com/repos/airbytehq/airbyte/assignees{/user}", "branches_url": "https://api.github.com/repos/airbytehq/airbyte/branches{/branch}", "tags_url": "https://api.github.com/repos/airbytehq/airbyte/tags", "blobs_url": "https://api.github.com/repos/airbytehq/airbyte/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/airbytehq/airbyte/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/airbytehq/airbyte/git/refs{/sha}", "trees_url": "https://api.github.com/repos/airbytehq/airbyte/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/airbytehq/airbyte/statuses/{sha}", "languages_url": "https://api.github.com/repos/airbytehq/airbyte/languages", "stargazers_url": "https://api.github.com/repos/airbytehq/airbyte/stargazers", "contributors_url": "https://api.github.com/repos/airbytehq/airbyte/contributors", "subscribers_url": "https://api.github.com/repos/airbytehq/airbyte/subscribers", "subscription_url": "https://api.github.com/repos/airbytehq/airbyte/subscription", "commits_url": "https://api.github.com/repos/airbytehq/airbyte/commits{/sha}", "git_commits_url": "https://api.github.com/repos/airbytehq/airbyte/git/commits{/sha}", "comments_url": "https://api.github.com/repos/airbytehq/airbyte/comments{/number}", "issue_comment_url": "https://api.github.com/repos/airbytehq/airbyte/issues/comments{/number}", "contents_url": "https://api.github.com/repos/airbytehq/airbyte/contents/{+path}", "compare_url": "https://api.github.com/repos/airbytehq/airbyte/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/airbytehq/airbyte/merges", "archive_url": "https://api.github.com/repos/airbytehq/airbyte/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/airbytehq/airbyte/downloads", "issues_url": "https://api.github.com/repos/airbytehq/airbyte/issues{/number}", "pulls_url": "https://api.github.com/repos/airbytehq/airbyte/pulls{/number}", "milestones_url": "https://api.github.com/repos/airbytehq/airbyte/milestones{/number}", "notifications_url": "https://api.github.com/repos/airbytehq/airbyte/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/airbytehq/airbyte/labels{/name}", "releases_url": "https://api.github.com/repos/airbytehq/airbyte/releases{/id}", "deployments_url": "https://api.github.com/repos/airbytehq/airbyte/deployments", "created_at": "2020-07-27T23:55:54Z", "updated_at": "2023-07-11T04:02:27Z", "pushed_at": "2023-07-11T08:47:50Z", "git_url": "git://github.com/airbytehq/airbyte.git", "ssh_url": "git@github.com:airbytehq/airbyte.git", "clone_url": "https://github.com/airbytehq/airbyte.git", "svn_url": "https://github.com/airbytehq/airbyte", "homepage": "https://airbyte.com", "size": 3671239, "stargazers_count": 11137, "watchers_count": 11137, "language": "Python", "has_issues": true, "has_projects": true, "has_downloads": true, "has_wiki": false, "has_pages": true, "has_discussions": true, "forks_count": 2881, "mirror_url": null, "archived": false, "disabled": false, "open_issues_count": 4723, "license": {"key": "other", "name": "Other", "spdx_id": "NOASSERTION", "url": null, "node_id": "MDc6TGljZW5zZTA="}, "allow_forking": true, "is_template": false, "web_commit_signoff_required": false, "topics": ["airbyte", "bigquery", "change-data-capture", "data", "data-analysis", "data-collection", "data-engineering", "data-ingestion", "data-integration", "elt", "etl", "java", "pipeline", "python", "redshift", "snowflake"], "visibility": "public", "forks": 2881, "open_issues": 4723, "watchers": 11137, "default_branch": "master", "permissions": {"admin": true, "maintain": true, "push": true, "triage": true, "pull": true}, "security_and_analysis": {"secret_scanning": {"status": "disabled"}, "secret_scanning_push_protection": {"status": "disabled"}, "dependabot_security_updates": {"status": "enabled"}}, "organization": "airbytehq"}, "emitted_at": 1689065865611} -{"stream":"review_comments","data":{"url":"https://api.github.com/repos/airbytehq/integration-test/pulls/comments/699253726","pull_request_review_id":742633128,"id":699253726,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDY5OTI1MzcyNg==","diff_hunk":"@@ -0,0 +1 @@\n+text_for_file_","path":"github_sources/file_1.txt","position":1,"original_position":1,"commit_id":"da5fa314f9b3a272d0aa47a453aec0f68a80cbae","original_commit_id":"da5fa314f9b3a272d0aa47a453aec0f68a80cbae","user":{"login":"yevhenii-ldv","id":34103125,"node_id":"MDQ6VXNlcjM0MTAzMTI1","avatar_url":"https://avatars.githubusercontent.com/u/34103125?v=4","gravatar_id":"","url":"https://api.github.com/users/yevhenii-ldv","html_url":"https://github.com/yevhenii-ldv","followers_url":"https://api.github.com/users/yevhenii-ldv/followers","following_url":"https://api.github.com/users/yevhenii-ldv/following{/other_user}","gists_url":"https://api.github.com/users/yevhenii-ldv/gists{/gist_id}","starred_url":"https://api.github.com/users/yevhenii-ldv/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yevhenii-ldv/subscriptions","organizations_url":"https://api.github.com/users/yevhenii-ldv/orgs","repos_url":"https://api.github.com/users/yevhenii-ldv/repos","events_url":"https://api.github.com/users/yevhenii-ldv/events{/privacy}","received_events_url":"https://api.github.com/users/yevhenii-ldv/received_events","type":"User","site_admin":false},"body":"Good point","created_at":"2021-08-31T12:01:15Z","updated_at":"2021-08-31T12:01:15Z","html_url":"https://github.com/airbytehq/integration-test/pull/4#discussion_r699253726","pull_request_url":"https://api.github.com/repos/airbytehq/integration-test/pulls/4","author_association":"NONE","_links":{"self":{"href":"https://api.github.com/repos/airbytehq/integration-test/pulls/comments/699253726"},"html":{"href":"https://github.com/airbytehq/integration-test/pull/4#discussion_r699253726"},"pull_request":{"href":"https://api.github.com/repos/airbytehq/integration-test/pulls/4"}},"reactions":{"url":"https://api.github.com/repos/airbytehq/integration-test/pulls/comments/699253726/reactions","total_count":1,"+1":0,"-1":0,"laugh":0,"hooray":0,"confused":0,"heart":1,"rocket":0,"eyes":0},"start_line":null,"original_start_line":null,"start_side":null,"line":1,"original_line":1,"side":"RIGHT","subject_type": "line","repository":"airbytehq/integration-test"},"emitted_at":1677668764426} +{"stream": "repositories", "data": {"id": 283046497, "node_id": "MDEwOlJlcG9zaXRvcnkyODMwNDY0OTc=", "name": "airbyte", "full_name": "airbytehq/airbyte", "private": false, "owner": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "html_url": "https://github.com/airbytehq/airbyte", "description": "Data integration platform for ELT pipelines from APIs, databases & files to warehouses & lakes.", "fork": false, "url": "https://api.github.com/repos/airbytehq/airbyte", "forks_url": "https://api.github.com/repos/airbytehq/airbyte/forks", "keys_url": "https://api.github.com/repos/airbytehq/airbyte/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/airbytehq/airbyte/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/airbytehq/airbyte/teams", "hooks_url": "https://api.github.com/repos/airbytehq/airbyte/hooks", "issue_events_url": "https://api.github.com/repos/airbytehq/airbyte/issues/events{/number}", "events_url": "https://api.github.com/repos/airbytehq/airbyte/events", "assignees_url": "https://api.github.com/repos/airbytehq/airbyte/assignees{/user}", "branches_url": "https://api.github.com/repos/airbytehq/airbyte/branches{/branch}", "tags_url": "https://api.github.com/repos/airbytehq/airbyte/tags", "blobs_url": "https://api.github.com/repos/airbytehq/airbyte/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/airbytehq/airbyte/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/airbytehq/airbyte/git/refs{/sha}", "trees_url": "https://api.github.com/repos/airbytehq/airbyte/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/airbytehq/airbyte/statuses/{sha}", "languages_url": "https://api.github.com/repos/airbytehq/airbyte/languages", "stargazers_url": "https://api.github.com/repos/airbytehq/airbyte/stargazers", "contributors_url": "https://api.github.com/repos/airbytehq/airbyte/contributors", "subscribers_url": "https://api.github.com/repos/airbytehq/airbyte/subscribers", "subscription_url": "https://api.github.com/repos/airbytehq/airbyte/subscription", "commits_url": "https://api.github.com/repos/airbytehq/airbyte/commits{/sha}", "git_commits_url": "https://api.github.com/repos/airbytehq/airbyte/git/commits{/sha}", "comments_url": "https://api.github.com/repos/airbytehq/airbyte/comments{/number}", "issue_comment_url": "https://api.github.com/repos/airbytehq/airbyte/issues/comments{/number}", "contents_url": "https://api.github.com/repos/airbytehq/airbyte/contents/{+path}", "compare_url": "https://api.github.com/repos/airbytehq/airbyte/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/airbytehq/airbyte/merges", "archive_url": "https://api.github.com/repos/airbytehq/airbyte/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/airbytehq/airbyte/downloads", "issues_url": "https://api.github.com/repos/airbytehq/airbyte/issues{/number}", "pulls_url": "https://api.github.com/repos/airbytehq/airbyte/pulls{/number}", "milestones_url": "https://api.github.com/repos/airbytehq/airbyte/milestones{/number}", "notifications_url": "https://api.github.com/repos/airbytehq/airbyte/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/airbytehq/airbyte/labels{/name}", "releases_url": "https://api.github.com/repos/airbytehq/airbyte/releases{/id}", "deployments_url": "https://api.github.com/repos/airbytehq/airbyte/deployments", "created_at": "2020-07-27T23:55:54Z", "updated_at": "2023-11-21T14:55:05Z", "pushed_at": "2023-11-21T16:55:37Z", "git_url": "git://github.com/airbytehq/airbyte.git", "ssh_url": "git@github.com:airbytehq/airbyte.git", "clone_url": "https://github.com/airbytehq/airbyte.git", "svn_url": "https://github.com/airbytehq/airbyte", "homepage": "https://airbyte.com", "size": 455477, "stargazers_count": 12328, "watchers_count": 12328, "language": "Python", "has_issues": true, "has_projects": true, "has_downloads": true, "has_wiki": false, "has_pages": false, "has_discussions": true, "forks_count": 3226, "mirror_url": null, "archived": false, "disabled": false, "open_issues_count": 5053, "license": {"key": "other", "name": "Other", "spdx_id": "NOASSERTION", "url": null, "node_id": "MDc6TGljZW5zZTA="}, "allow_forking": true, "is_template": false, "web_commit_signoff_required": false, "topics": ["airbyte", "bigquery", "change-data-capture", "data", "data-analysis", "data-collection", "data-engineering", "data-ingestion", "data-integration", "elt", "etl", "java", "pipeline", "python", "redshift", "snowflake"], "visibility": "public", "forks": 3226, "open_issues": 5053, "watchers": 12328, "default_branch": "master", "permissions": {"admin": true, "maintain": true, "push": true, "triage": true, "pull": true}, "security_and_analysis": {"secret_scanning": {"status": "disabled"}, "secret_scanning_push_protection": {"status": "disabled"}, "dependabot_security_updates": {"status": "enabled"}, "secret_scanning_validity_checks": {"status": "disabled"}}, "organization": "airbytehq"}, "emitted_at": 1700585836592} +{"stream":"review_comments","data":{"url":"https://api.github.com/repos/airbytehq/integration-test/pulls/comments/699253726","pull_request_review_id":742633128,"id":699253726,"node_id":"MDI0OlB1bGxSZXF1ZXN0UmV2aWV3Q29tbWVudDY5OTI1MzcyNg==","diff_hunk":"@@ -0,0 +1 @@\n+text_for_file_","path":"github_sources/file_1.txt","commit_id":"da5fa314f9b3a272d0aa47a453aec0f68a80cbae","original_commit_id":"da5fa314f9b3a272d0aa47a453aec0f68a80cbae","user":{"login":"yevhenii-ldv","id":34103125,"node_id":"MDQ6VXNlcjM0MTAzMTI1","avatar_url":"https://avatars.githubusercontent.com/u/34103125?v=4","gravatar_id":"","url":"https://api.github.com/users/yevhenii-ldv","html_url":"https://github.com/yevhenii-ldv","followers_url":"https://api.github.com/users/yevhenii-ldv/followers","following_url":"https://api.github.com/users/yevhenii-ldv/following{/other_user}","gists_url":"https://api.github.com/users/yevhenii-ldv/gists{/gist_id}","starred_url":"https://api.github.com/users/yevhenii-ldv/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yevhenii-ldv/subscriptions","organizations_url":"https://api.github.com/users/yevhenii-ldv/orgs","repos_url":"https://api.github.com/users/yevhenii-ldv/repos","events_url":"https://api.github.com/users/yevhenii-ldv/events{/privacy}","received_events_url":"https://api.github.com/users/yevhenii-ldv/received_events","type":"User","site_admin":false},"body":"Good point","created_at":"2021-08-31T12:01:15Z","updated_at":"2021-08-31T12:01:15Z","html_url":"https://github.com/airbytehq/integration-test/pull/4#discussion_r699253726","pull_request_url":"https://api.github.com/repos/airbytehq/integration-test/pulls/4","author_association":"MEMBER","_links":{"self":{"href":"https://api.github.com/repos/airbytehq/integration-test/pulls/comments/699253726"},"html":{"href":"https://github.com/airbytehq/integration-test/pull/4#discussion_r699253726"},"pull_request":{"href":"https://api.github.com/repos/airbytehq/integration-test/pulls/4"}},"reactions":{"url":"https://api.github.com/repos/airbytehq/integration-test/pulls/comments/699253726/reactions","total_count":1,"+1":0,"-1":0,"laugh":0,"hooray":0,"confused":0,"heart":1,"rocket":0,"eyes":0},"start_line":null,"original_start_line":null,"start_side":null,"line":1,"original_line":1,"side":"RIGHT","original_position":1,"position":1,"subject_type":"line","repository":"airbytehq/integration-test"},"emitted_at":1695375624151} {"stream":"reviews","data":{"node_id":"MDE3OlB1bGxSZXF1ZXN0UmV2aWV3NzQwNjU5Nzk4","id":740659798,"body":"Review commit for branch feature/branch_4","state":"COMMENTED","html_url":"https://github.com/airbytehq/integration-test/pull/5#pullrequestreview-740659798","author_association":"CONTRIBUTOR","submitted_at":"2021-08-27T15:43:42Z","created_at":"2021-08-27T15:43:42Z","updated_at":"2021-08-27T15:43:42Z","user":{"node_id":"MDQ6VXNlcjc0MzkwMQ==","id":743901,"login":"gaart","avatar_url":"https://avatars.githubusercontent.com/u/743901?v=4","html_url":"https://github.com/gaart","site_admin":false,"type":"User"},"repository":"airbytehq/integration-test","pull_request_url":"https://github.com/airbytehq/integration-test/pull/5","commit_id":"31a3e3f19fefce60fba6bfc69dd2b3fb5195a083","_links":{"html":{"href":"https://github.com/airbytehq/integration-test/pull/5#pullrequestreview-740659798"},"pull_request":{"href":"https://github.com/airbytehq/integration-test/pull/5"}}},"emitted_at":1677668764954} {"stream":"stargazers","data":{"starred_at":"2021-08-27T16:23:34Z","user":{"login":"VasylLazebnyk","id":68591643,"node_id":"MDQ6VXNlcjY4NTkxNjQz","avatar_url":"https://avatars.githubusercontent.com/u/68591643?v=4","gravatar_id":"","url":"https://api.github.com/users/VasylLazebnyk","html_url":"https://github.com/VasylLazebnyk","followers_url":"https://api.github.com/users/VasylLazebnyk/followers","following_url":"https://api.github.com/users/VasylLazebnyk/following{/other_user}","gists_url":"https://api.github.com/users/VasylLazebnyk/gists{/gist_id}","starred_url":"https://api.github.com/users/VasylLazebnyk/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/VasylLazebnyk/subscriptions","organizations_url":"https://api.github.com/users/VasylLazebnyk/orgs","repos_url":"https://api.github.com/users/VasylLazebnyk/repos","events_url":"https://api.github.com/users/VasylLazebnyk/events{/privacy}","received_events_url":"https://api.github.com/users/VasylLazebnyk/received_events","type":"User","site_admin":false},"repository":"airbytehq/integration-test","user_id":68591643},"emitted_at":1677668765231} {"stream":"tags","data":{"name":"dev-0.9","zipball_url":"https://api.github.com/repos/airbytehq/integration-test/zipball/refs/tags/dev-0.9","tarball_url":"https://api.github.com/repos/airbytehq/integration-test/tarball/refs/tags/dev-0.9","commit":{"sha":"978753aeb56f7b49872279d1b491411a6235aa90","url":"https://api.github.com/repos/airbytehq/integration-test/commits/978753aeb56f7b49872279d1b491411a6235aa90"},"node_id":"MDM6UmVmNDAwMDUyMjEzOnJlZnMvdGFncy9kZXYtMC45","repository":"airbytehq/integration-test"},"emitted_at":1677668765467} {"stream":"teams", "data": {"name": "Zazmic", "id": 4432406, "node_id": "MDQ6VGVhbTQ0MzI0MDY=", "slug": "zazmic", "description": "", "privacy": "closed", "notification_setting": "notifications_enabled", "url": "https://api.github.com/organizations/59758427/team/4432406", "html_url": "https://github.com/orgs/airbytehq/teams/zazmic", "members_url": "https://api.github.com/organizations/59758427/team/4432406/members{/member}", "repositories_url": "https://api.github.com/organizations/59758427/team/4432406/repos", "permission": "pull", "parent": null, "organization": "airbytehq"}, "emitted_at": 1681307598422} {"stream":"users","data":{"login":"AirbyteEricksson","id":101604444,"node_id":"U_kgDOBg5cXA","avatar_url":"https://avatars.githubusercontent.com/u/101604444?v=4","gravatar_id":"","url":"https://api.github.com/users/AirbyteEricksson","html_url":"https://github.com/AirbyteEricksson","followers_url":"https://api.github.com/users/AirbyteEricksson/followers","following_url":"https://api.github.com/users/AirbyteEricksson/following{/other_user}","gists_url":"https://api.github.com/users/AirbyteEricksson/gists{/gist_id}","starred_url":"https://api.github.com/users/AirbyteEricksson/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/AirbyteEricksson/subscriptions","organizations_url":"https://api.github.com/users/AirbyteEricksson/orgs","repos_url":"https://api.github.com/users/AirbyteEricksson/repos","events_url":"https://api.github.com/users/AirbyteEricksson/events{/privacy}","received_events_url":"https://api.github.com/users/AirbyteEricksson/received_events","type":"User","site_admin":false,"organization":"airbytehq"},"emitted_at":1677668766142} {"stream":"workflows","data":{"id":22952989,"node_id":"W_kwDOF9hP9c4BXjwd","name":"Pull Request Labeler","path":".github/workflows/labeler.yml","state":"active","created_at":"2022-03-30T21:30:37.000+02:00","updated_at":"2022-03-30T21:30:37.000+02:00","url":"https://api.github.com/repos/airbytehq/integration-test/actions/workflows/22952989","html_url":"https://github.com/airbytehq/integration-test/blob/master/.github/workflows/labeler.yml","badge_url":"https://github.com/airbytehq/integration-test/workflows/Pull%20Request%20Labeler/badge.svg","repository":"airbytehq/integration-test"},"emitted_at":1677668766580} -{"stream":"workflow_runs","data":{"id":3184250176,"name":"Pull Request Labeler","node_id":"WFR_kwLOF9hP9c69y81A","head_branch":"feature/branch_5","head_sha":"f71e5f6894578148d52b487dff07e55804fd9cfd","path":".github/workflows/labeler.yml","display_title":"New PR from feature/branch_5","run_number":3,"event":"pull_request_target","status":"completed","conclusion":"success","workflow_id":22952989,"check_suite_id":8611635614,"check_suite_node_id":"CS_kwDOF9hP9c8AAAACAUshng","url":"https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176","html_url":"https://github.com/airbytehq/integration-test/actions/runs/3184250176","pull_requests":[{"url":"https://api.github.com/repos/airbytehq/integration-test/pulls/14","id":984835098,"number":14,"head":{"ref":"feature/branch_5","sha":"f71e5f6894578148d52b487dff07e55804fd9cfd","repo":{"id":400052213,"url":"https://api.github.com/repos/airbytehq/integration-test","name":"integration-test"}},"base":{"ref":"master","sha":"a12c9379604f7b32e54e5459122aa48473f806ee","repo":{"id":400052213,"url":"https://api.github.com/repos/airbytehq/integration-test","name":"integration-test"}}}],"created_at":"2022-10-04T17:41:18Z","updated_at":"2022-10-04T17:41:32Z","actor":{"login":"grubberr","id":195743,"node_id":"MDQ6VXNlcjE5NTc0Mw==","avatar_url":"https://avatars.githubusercontent.com/u/195743?v=4","gravatar_id":"","url":"https://api.github.com/users/grubberr","html_url":"https://github.com/grubberr","followers_url":"https://api.github.com/users/grubberr/followers","following_url":"https://api.github.com/users/grubberr/following{/other_user}","gists_url":"https://api.github.com/users/grubberr/gists{/gist_id}","starred_url":"https://api.github.com/users/grubberr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/grubberr/subscriptions","organizations_url":"https://api.github.com/users/grubberr/orgs","repos_url":"https://api.github.com/users/grubberr/repos","events_url":"https://api.github.com/users/grubberr/events{/privacy}","received_events_url":"https://api.github.com/users/grubberr/received_events","type":"User","site_admin":false},"run_attempt":1,"referenced_workflows":[],"run_started_at":"2022-10-04T17:41:18Z","triggering_actor":{"login":"grubberr","id":195743,"node_id":"MDQ6VXNlcjE5NTc0Mw==","avatar_url":"https://avatars.githubusercontent.com/u/195743?v=4","gravatar_id":"","url":"https://api.github.com/users/grubberr","html_url":"https://github.com/grubberr","followers_url":"https://api.github.com/users/grubberr/followers","following_url":"https://api.github.com/users/grubberr/following{/other_user}","gists_url":"https://api.github.com/users/grubberr/gists{/gist_id}","starred_url":"https://api.github.com/users/grubberr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/grubberr/subscriptions","organizations_url":"https://api.github.com/users/grubberr/orgs","repos_url":"https://api.github.com/users/grubberr/repos","events_url":"https://api.github.com/users/grubberr/events{/privacy}","received_events_url":"https://api.github.com/users/grubberr/received_events","type":"User","site_admin":false},"jobs_url":"https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176/jobs","logs_url":"https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176/logs","check_suite_url":"https://api.github.com/repos/airbytehq/integration-test/check-suites/8611635614","artifacts_url":"https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176/artifacts","cancel_url":"https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176/cancel","rerun_url":"https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176/rerun","previous_attempt_url":null,"workflow_url":"https://api.github.com/repos/airbytehq/integration-test/actions/workflows/22952989","head_commit":{"id":"f71e5f6894578148d52b487dff07e55804fd9cfd","tree_id":"bb78ec62be8c5c640010e7c897f40932ce59e725","message":"file_5.txt updated\n\nSigned-off-by: Sergey Chvalyuk ","timestamp":"2022-10-04T17:41:08Z","author":{"name":"Sergey Chvalyuk","email":"grubberr@gmail.com"},"committer":{"name":"Sergey Chvalyuk","email":"grubberr@gmail.com"}},"repository":{"id":400052213,"node_id":"MDEwOlJlcG9zaXRvcnk0MDAwNTIyMTM=","name":"integration-test","full_name":"airbytehq/integration-test","private":false,"owner":{"login":"airbytehq","id":59758427,"node_id":"MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3","avatar_url":"https://avatars.githubusercontent.com/u/59758427?v=4","gravatar_id":"","url":"https://api.github.com/users/airbytehq","html_url":"https://github.com/airbytehq","followers_url":"https://api.github.com/users/airbytehq/followers","following_url":"https://api.github.com/users/airbytehq/following{/other_user}","gists_url":"https://api.github.com/users/airbytehq/gists{/gist_id}","starred_url":"https://api.github.com/users/airbytehq/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/airbytehq/subscriptions","organizations_url":"https://api.github.com/users/airbytehq/orgs","repos_url":"https://api.github.com/users/airbytehq/repos","events_url":"https://api.github.com/users/airbytehq/events{/privacy}","received_events_url":"https://api.github.com/users/airbytehq/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/airbytehq/integration-test","description":"Used for integration testing the Github source connector","fork":false,"url":"https://api.github.com/repos/airbytehq/integration-test","forks_url":"https://api.github.com/repos/airbytehq/integration-test/forks","keys_url":"https://api.github.com/repos/airbytehq/integration-test/keys{/key_id}","collaborators_url":"https://api.github.com/repos/airbytehq/integration-test/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/airbytehq/integration-test/teams","hooks_url":"https://api.github.com/repos/airbytehq/integration-test/hooks","issue_events_url":"https://api.github.com/repos/airbytehq/integration-test/issues/events{/number}","events_url":"https://api.github.com/repos/airbytehq/integration-test/events","assignees_url":"https://api.github.com/repos/airbytehq/integration-test/assignees{/user}","branches_url":"https://api.github.com/repos/airbytehq/integration-test/branches{/branch}","tags_url":"https://api.github.com/repos/airbytehq/integration-test/tags","blobs_url":"https://api.github.com/repos/airbytehq/integration-test/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/airbytehq/integration-test/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/airbytehq/integration-test/git/refs{/sha}","trees_url":"https://api.github.com/repos/airbytehq/integration-test/git/trees{/sha}","statuses_url":"https://api.github.com/repos/airbytehq/integration-test/statuses/{sha}","languages_url":"https://api.github.com/repos/airbytehq/integration-test/languages","stargazers_url":"https://api.github.com/repos/airbytehq/integration-test/stargazers","contributors_url":"https://api.github.com/repos/airbytehq/integration-test/contributors","subscribers_url":"https://api.github.com/repos/airbytehq/integration-test/subscribers","subscription_url":"https://api.github.com/repos/airbytehq/integration-test/subscription","commits_url":"https://api.github.com/repos/airbytehq/integration-test/commits{/sha}","git_commits_url":"https://api.github.com/repos/airbytehq/integration-test/git/commits{/sha}","comments_url":"https://api.github.com/repos/airbytehq/integration-test/comments{/number}","issue_comment_url":"https://api.github.com/repos/airbytehq/integration-test/issues/comments{/number}","contents_url":"https://api.github.com/repos/airbytehq/integration-test/contents/{+path}","compare_url":"https://api.github.com/repos/airbytehq/integration-test/compare/{base}...{head}","merges_url":"https://api.github.com/repos/airbytehq/integration-test/merges","archive_url":"https://api.github.com/repos/airbytehq/integration-test/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/airbytehq/integration-test/downloads","issues_url":"https://api.github.com/repos/airbytehq/integration-test/issues{/number}","pulls_url":"https://api.github.com/repos/airbytehq/integration-test/pulls{/number}","milestones_url":"https://api.github.com/repos/airbytehq/integration-test/milestones{/number}","notifications_url":"https://api.github.com/repos/airbytehq/integration-test/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/airbytehq/integration-test/labels{/name}","releases_url":"https://api.github.com/repos/airbytehq/integration-test/releases{/id}","deployments_url":"https://api.github.com/repos/airbytehq/integration-test/deployments"},"head_repository":{"id":400052213,"node_id":"MDEwOlJlcG9zaXRvcnk0MDAwNTIyMTM=","name":"integration-test","full_name":"airbytehq/integration-test","private":false,"owner":{"login":"airbytehq","id":59758427,"node_id":"MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3","avatar_url":"https://avatars.githubusercontent.com/u/59758427?v=4","gravatar_id":"","url":"https://api.github.com/users/airbytehq","html_url":"https://github.com/airbytehq","followers_url":"https://api.github.com/users/airbytehq/followers","following_url":"https://api.github.com/users/airbytehq/following{/other_user}","gists_url":"https://api.github.com/users/airbytehq/gists{/gist_id}","starred_url":"https://api.github.com/users/airbytehq/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/airbytehq/subscriptions","organizations_url":"https://api.github.com/users/airbytehq/orgs","repos_url":"https://api.github.com/users/airbytehq/repos","events_url":"https://api.github.com/users/airbytehq/events{/privacy}","received_events_url":"https://api.github.com/users/airbytehq/received_events","type":"Organization","site_admin":false},"html_url":"https://github.com/airbytehq/integration-test","description":"Used for integration testing the Github source connector","fork":false,"url":"https://api.github.com/repos/airbytehq/integration-test","forks_url":"https://api.github.com/repos/airbytehq/integration-test/forks","keys_url":"https://api.github.com/repos/airbytehq/integration-test/keys{/key_id}","collaborators_url":"https://api.github.com/repos/airbytehq/integration-test/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/airbytehq/integration-test/teams","hooks_url":"https://api.github.com/repos/airbytehq/integration-test/hooks","issue_events_url":"https://api.github.com/repos/airbytehq/integration-test/issues/events{/number}","events_url":"https://api.github.com/repos/airbytehq/integration-test/events","assignees_url":"https://api.github.com/repos/airbytehq/integration-test/assignees{/user}","branches_url":"https://api.github.com/repos/airbytehq/integration-test/branches{/branch}","tags_url":"https://api.github.com/repos/airbytehq/integration-test/tags","blobs_url":"https://api.github.com/repos/airbytehq/integration-test/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/airbytehq/integration-test/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/airbytehq/integration-test/git/refs{/sha}","trees_url":"https://api.github.com/repos/airbytehq/integration-test/git/trees{/sha}","statuses_url":"https://api.github.com/repos/airbytehq/integration-test/statuses/{sha}","languages_url":"https://api.github.com/repos/airbytehq/integration-test/languages","stargazers_url":"https://api.github.com/repos/airbytehq/integration-test/stargazers","contributors_url":"https://api.github.com/repos/airbytehq/integration-test/contributors","subscribers_url":"https://api.github.com/repos/airbytehq/integration-test/subscribers","subscription_url":"https://api.github.com/repos/airbytehq/integration-test/subscription","commits_url":"https://api.github.com/repos/airbytehq/integration-test/commits{/sha}","git_commits_url":"https://api.github.com/repos/airbytehq/integration-test/git/commits{/sha}","comments_url":"https://api.github.com/repos/airbytehq/integration-test/comments{/number}","issue_comment_url":"https://api.github.com/repos/airbytehq/integration-test/issues/comments{/number}","contents_url":"https://api.github.com/repos/airbytehq/integration-test/contents/{+path}","compare_url":"https://api.github.com/repos/airbytehq/integration-test/compare/{base}...{head}","merges_url":"https://api.github.com/repos/airbytehq/integration-test/merges","archive_url":"https://api.github.com/repos/airbytehq/integration-test/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/airbytehq/integration-test/downloads","issues_url":"https://api.github.com/repos/airbytehq/integration-test/issues{/number}","pulls_url":"https://api.github.com/repos/airbytehq/integration-test/pulls{/number}","milestones_url":"https://api.github.com/repos/airbytehq/integration-test/milestones{/number}","notifications_url":"https://api.github.com/repos/airbytehq/integration-test/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/airbytehq/integration-test/labels{/name}","releases_url":"https://api.github.com/repos/airbytehq/integration-test/releases{/id}","deployments_url":"https://api.github.com/repos/airbytehq/integration-test/deployments"}},"emitted_at":1677668766993} -{"stream":"workflow_jobs","data":{"id": 8705992587, "run_id": 3184250176, "workflow_name": "Pull Request Labeler", "head_branch": "feature/branch_5", "run_url": "https://api.github.com/repos/airbytehq/integration-test/actions/runs/3184250176", "run_attempt": 1, "node_id": "CR_kwDOF9hP9c8AAAACBurniw", "head_sha": "f71e5f6894578148d52b487dff07e55804fd9cfd", "url": "https://api.github.com/repos/airbytehq/integration-test/actions/jobs/8705992587", "html_url": "https://github.com/airbytehq/integration-test/actions/runs/3184250176/job/8705992587", "status": "completed", "conclusion": "success", "created_at": "2022-10-04T17:41:20Z", "started_at": "2022-10-04T17:41:27Z", "completed_at": "2022-10-04T17:41:30Z", "name": "triage", "steps": [{"name": "Set up job", "status": "completed", "conclusion": "success", "number": 1, "started_at": "2022-10-04T20:41:26.000+03:00", "completed_at": "2022-10-04T20:41:27.000+03:00"}, {"name": "Run actions/labeler@v3", "status": "completed", "conclusion": "success", "number": 2, "started_at": "2022-10-04T20:41:27.000+03:00", "completed_at": "2022-10-04T20:41:29.000+03:00"}, {"name": "Complete job", "status": "completed", "conclusion": "success", "number": 3, "started_at": "2022-10-04T20:41:29.000+03:00", "completed_at": "2022-10-04T20:41:29.000+03:00"}], "check_run_url": "https://api.github.com/repos/airbytehq/integration-test/check-runs/8705992587", "labels": ["ubuntu-latest"], "runner_id": 1, "runner_name": "Hosted Agent", "runner_group_id": 2, "runner_group_name": "GitHub Actions", "repository": "airbytehq/integration-test"},"emitted_at":1677668767830} -{"stream":"team_members","data":{"login":"sherifnada","id":6246757,"node_id":"MDQ6VXNlcjYyNDY3NTc=","avatar_url":"https://avatars.githubusercontent.com/u/6246757?v=4","gravatar_id":"","url":"https://api.github.com/users/sherifnada","html_url":"https://github.com/sherifnada","followers_url":"https://api.github.com/users/sherifnada/followers","following_url":"https://api.github.com/users/sherifnada/following{/other_user}","gists_url":"https://api.github.com/users/sherifnada/gists{/gist_id}","starred_url":"https://api.github.com/users/sherifnada/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/sherifnada/subscriptions","organizations_url":"https://api.github.com/users/sherifnada/orgs","repos_url":"https://api.github.com/users/sherifnada/repos","events_url":"https://api.github.com/users/sherifnada/events{/privacy}","received_events_url":"https://api.github.com/users/sherifnada/received_events","type":"User","site_admin":false,"organization":"airbytehq","team_slug":"zazmic"},"emitted_at":1677668768649} -{"stream":"team_memberships","data":{"state":"active","role":"maintainer","url":"https://api.github.com/organizations/59758427/team/4432406/memberships/sherifnada","organization":"airbytehq","team_slug":"zazmic","username":"sherifnada"},"emitted_at":1677668779034} \ No newline at end of file +{"stream": "workflow_runs", "data": {"id": 4871166142, "name": "Pull Request Labeler", "node_id": "WFR_kwLOF9hP9c8AAAABIlgYvg", "head_branch": "arsenlosenko/test-pending-comments-in-pr", "head_sha": "47c7a128f28791f657265eb89cdf7ab28a0ff51b", "path": ".github/workflows/labeler.yml", "display_title": "Update .gitignore", "run_number": 4, "event": "pull_request_target", "status": "completed", "conclusion": "success", "workflow_id": 22952989, "check_suite_id": 12643387080, "check_suite_node_id": "CS_kwDOF9hP9c8AAAAC8ZrGyA", "url": "https://api.github.com/repos/airbytehq/integration-test/actions/runs/4871166142", "html_url": "https://github.com/airbytehq/integration-test/actions/runs/4871166142", "pull_requests": [], "created_at": "2023-05-03T11:05:23Z", "updated_at": "2023-05-03T11:05:36Z", "actor": {"login": "arsenlosenko", "id": 20901439, "node_id": "MDQ6VXNlcjIwOTAxNDM5", "avatar_url": "https://avatars.githubusercontent.com/u/20901439?v=4", "gravatar_id": "", "url": "https://api.github.com/users/arsenlosenko", "html_url": "https://github.com/arsenlosenko", "followers_url": "https://api.github.com/users/arsenlosenko/followers", "following_url": "https://api.github.com/users/arsenlosenko/following{/other_user}", "gists_url": "https://api.github.com/users/arsenlosenko/gists{/gist_id}", "starred_url": "https://api.github.com/users/arsenlosenko/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/arsenlosenko/subscriptions", "organizations_url": "https://api.github.com/users/arsenlosenko/orgs", "repos_url": "https://api.github.com/users/arsenlosenko/repos", "events_url": "https://api.github.com/users/arsenlosenko/events{/privacy}", "received_events_url": "https://api.github.com/users/arsenlosenko/received_events", "type": "User", "site_admin": false}, "run_attempt": 1, "referenced_workflows": [], "run_started_at": "2023-05-03T11:05:23Z", "triggering_actor": {"login": "arsenlosenko", "id": 20901439, "node_id": "MDQ6VXNlcjIwOTAxNDM5", "avatar_url": "https://avatars.githubusercontent.com/u/20901439?v=4", "gravatar_id": "", "url": "https://api.github.com/users/arsenlosenko", "html_url": "https://github.com/arsenlosenko", "followers_url": "https://api.github.com/users/arsenlosenko/followers", "following_url": "https://api.github.com/users/arsenlosenko/following{/other_user}", "gists_url": "https://api.github.com/users/arsenlosenko/gists{/gist_id}", "starred_url": "https://api.github.com/users/arsenlosenko/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/arsenlosenko/subscriptions", "organizations_url": "https://api.github.com/users/arsenlosenko/orgs", "repos_url": "https://api.github.com/users/arsenlosenko/repos", "events_url": "https://api.github.com/users/arsenlosenko/events{/privacy}", "received_events_url": "https://api.github.com/users/arsenlosenko/received_events", "type": "User", "site_admin": false}, "jobs_url": "https://api.github.com/repos/airbytehq/integration-test/actions/runs/4871166142/jobs", "logs_url": "https://api.github.com/repos/airbytehq/integration-test/actions/runs/4871166142/logs", "check_suite_url": "https://api.github.com/repos/airbytehq/integration-test/check-suites/12643387080", "artifacts_url": "https://api.github.com/repos/airbytehq/integration-test/actions/runs/4871166142/artifacts", "cancel_url": "https://api.github.com/repos/airbytehq/integration-test/actions/runs/4871166142/cancel", "rerun_url": "https://api.github.com/repos/airbytehq/integration-test/actions/runs/4871166142/rerun", "previous_attempt_url": null, "workflow_url": "https://api.github.com/repos/airbytehq/integration-test/actions/workflows/22952989", "head_commit": {"id": "47c7a128f28791f657265eb89cdf7ab28a0ff51b", "tree_id": "3cc1c41924b3cb67150684024877f6e02d283afb", "message": "Update .gitignore", "timestamp": "2023-05-03T11:04:11Z", "author": {"name": "Arsen Losenko", "email": "20901439+arsenlosenko@users.noreply.github.com"}, "committer": {"name": "Arsen Losenko", "email": "20901439+arsenlosenko@users.noreply.github.com"}}, "repository": {"id": 400052213, "node_id": "MDEwOlJlcG9zaXRvcnk0MDAwNTIyMTM=", "name": "integration-test", "full_name": "airbytehq/integration-test", "private": false, "owner": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "html_url": "https://github.com/airbytehq/integration-test", "description": "Used for integration testing the Github source connector", "fork": false, "url": "https://api.github.com/repos/airbytehq/integration-test", "forks_url": "https://api.github.com/repos/airbytehq/integration-test/forks", "keys_url": "https://api.github.com/repos/airbytehq/integration-test/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/airbytehq/integration-test/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/airbytehq/integration-test/teams", "hooks_url": "https://api.github.com/repos/airbytehq/integration-test/hooks", "issue_events_url": "https://api.github.com/repos/airbytehq/integration-test/issues/events{/number}", "events_url": "https://api.github.com/repos/airbytehq/integration-test/events", "assignees_url": "https://api.github.com/repos/airbytehq/integration-test/assignees{/user}", "branches_url": "https://api.github.com/repos/airbytehq/integration-test/branches{/branch}", "tags_url": "https://api.github.com/repos/airbytehq/integration-test/tags", "blobs_url": "https://api.github.com/repos/airbytehq/integration-test/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/airbytehq/integration-test/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/airbytehq/integration-test/git/refs{/sha}", "trees_url": "https://api.github.com/repos/airbytehq/integration-test/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/airbytehq/integration-test/statuses/{sha}", "languages_url": "https://api.github.com/repos/airbytehq/integration-test/languages", "stargazers_url": "https://api.github.com/repos/airbytehq/integration-test/stargazers", "contributors_url": "https://api.github.com/repos/airbytehq/integration-test/contributors", "subscribers_url": "https://api.github.com/repos/airbytehq/integration-test/subscribers", "subscription_url": "https://api.github.com/repos/airbytehq/integration-test/subscription", "commits_url": "https://api.github.com/repos/airbytehq/integration-test/commits{/sha}", "git_commits_url": "https://api.github.com/repos/airbytehq/integration-test/git/commits{/sha}", "comments_url": "https://api.github.com/repos/airbytehq/integration-test/comments{/number}", "issue_comment_url": "https://api.github.com/repos/airbytehq/integration-test/issues/comments{/number}", "contents_url": "https://api.github.com/repos/airbytehq/integration-test/contents/{+path}", "compare_url": "https://api.github.com/repos/airbytehq/integration-test/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/airbytehq/integration-test/merges", "archive_url": "https://api.github.com/repos/airbytehq/integration-test/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/airbytehq/integration-test/downloads", "issues_url": "https://api.github.com/repos/airbytehq/integration-test/issues{/number}", "pulls_url": "https://api.github.com/repos/airbytehq/integration-test/pulls{/number}", "milestones_url": "https://api.github.com/repos/airbytehq/integration-test/milestones{/number}", "notifications_url": "https://api.github.com/repos/airbytehq/integration-test/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/airbytehq/integration-test/labels{/name}", "releases_url": "https://api.github.com/repos/airbytehq/integration-test/releases{/id}", "deployments_url": "https://api.github.com/repos/airbytehq/integration-test/deployments"}, "head_repository": {"id": 400052213, "node_id": "MDEwOlJlcG9zaXRvcnk0MDAwNTIyMTM=", "name": "integration-test", "full_name": "airbytehq/integration-test", "private": false, "owner": {"login": "airbytehq", "id": 59758427, "node_id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3", "avatar_url": "https://avatars.githubusercontent.com/u/59758427?v=4", "gravatar_id": "", "url": "https://api.github.com/users/airbytehq", "html_url": "https://github.com/airbytehq", "followers_url": "https://api.github.com/users/airbytehq/followers", "following_url": "https://api.github.com/users/airbytehq/following{/other_user}", "gists_url": "https://api.github.com/users/airbytehq/gists{/gist_id}", "starred_url": "https://api.github.com/users/airbytehq/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/airbytehq/subscriptions", "organizations_url": "https://api.github.com/users/airbytehq/orgs", "repos_url": "https://api.github.com/users/airbytehq/repos", "events_url": "https://api.github.com/users/airbytehq/events{/privacy}", "received_events_url": "https://api.github.com/users/airbytehq/received_events", "type": "Organization", "site_admin": false}, "html_url": "https://github.com/airbytehq/integration-test", "description": "Used for integration testing the Github source connector", "fork": false, "url": "https://api.github.com/repos/airbytehq/integration-test", "forks_url": "https://api.github.com/repos/airbytehq/integration-test/forks", "keys_url": "https://api.github.com/repos/airbytehq/integration-test/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/airbytehq/integration-test/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/airbytehq/integration-test/teams", "hooks_url": "https://api.github.com/repos/airbytehq/integration-test/hooks", "issue_events_url": "https://api.github.com/repos/airbytehq/integration-test/issues/events{/number}", "events_url": "https://api.github.com/repos/airbytehq/integration-test/events", "assignees_url": "https://api.github.com/repos/airbytehq/integration-test/assignees{/user}", "branches_url": "https://api.github.com/repos/airbytehq/integration-test/branches{/branch}", "tags_url": "https://api.github.com/repos/airbytehq/integration-test/tags", "blobs_url": "https://api.github.com/repos/airbytehq/integration-test/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/airbytehq/integration-test/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/airbytehq/integration-test/git/refs{/sha}", "trees_url": "https://api.github.com/repos/airbytehq/integration-test/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/airbytehq/integration-test/statuses/{sha}", "languages_url": "https://api.github.com/repos/airbytehq/integration-test/languages", "stargazers_url": "https://api.github.com/repos/airbytehq/integration-test/stargazers", "contributors_url": "https://api.github.com/repos/airbytehq/integration-test/contributors", "subscribers_url": "https://api.github.com/repos/airbytehq/integration-test/subscribers", "subscription_url": "https://api.github.com/repos/airbytehq/integration-test/subscription", "commits_url": "https://api.github.com/repos/airbytehq/integration-test/commits{/sha}", "git_commits_url": "https://api.github.com/repos/airbytehq/integration-test/git/commits{/sha}", "comments_url": "https://api.github.com/repos/airbytehq/integration-test/comments{/number}", "issue_comment_url": "https://api.github.com/repos/airbytehq/integration-test/issues/comments{/number}", "contents_url": "https://api.github.com/repos/airbytehq/integration-test/contents/{+path}", "compare_url": "https://api.github.com/repos/airbytehq/integration-test/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/airbytehq/integration-test/merges", "archive_url": "https://api.github.com/repos/airbytehq/integration-test/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/airbytehq/integration-test/downloads", "issues_url": "https://api.github.com/repos/airbytehq/integration-test/issues{/number}", "pulls_url": "https://api.github.com/repos/airbytehq/integration-test/pulls{/number}", "milestones_url": "https://api.github.com/repos/airbytehq/integration-test/milestones{/number}", "notifications_url": "https://api.github.com/repos/airbytehq/integration-test/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/airbytehq/integration-test/labels{/name}", "releases_url": "https://api.github.com/repos/airbytehq/integration-test/releases{/id}", "deployments_url": "https://api.github.com/repos/airbytehq/integration-test/deployments"}}, "emitted_at": 1700586521273} +{"stream": "workflow_jobs", "data": {"id": 13199605689, "run_id": 4871166142, "workflow_name": "Pull Request Labeler", "head_branch": "arsenlosenko/test-pending-comments-in-pr", "run_url": "https://api.github.com/repos/airbytehq/integration-test/actions/runs/4871166142", "run_attempt": 1, "node_id": "CR_kwDOF9hP9c8AAAADEsH_uQ", "head_sha": "47c7a128f28791f657265eb89cdf7ab28a0ff51b", "url": "https://api.github.com/repos/airbytehq/integration-test/actions/jobs/13199605689", "html_url": "https://github.com/airbytehq/integration-test/actions/runs/4871166142/job/13199605689", "status": "completed", "conclusion": "success", "created_at": "2023-05-03T11:05:25Z", "started_at": "2023-05-03T11:05:30Z", "completed_at": "2023-05-03T11:05:34Z", "name": "triage", "steps": [{"name": "Set up job", "status": "completed", "conclusion": "success", "number": 1, "started_at": "2023-05-03T14:05:30.000+03:00", "completed_at": "2023-05-03T14:05:31.000+03:00"}, {"name": "Run actions/labeler@v3", "status": "completed", "conclusion": "success", "number": 2, "started_at": "2023-05-03T14:05:32.000+03:00", "completed_at": "2023-05-03T14:05:32.000+03:00"}, {"name": "Complete job", "status": "completed", "conclusion": "success", "number": 3, "started_at": "2023-05-03T14:05:32.000+03:00", "completed_at": "2023-05-03T14:05:32.000+03:00"}], "check_run_url": "https://api.github.com/repos/airbytehq/integration-test/check-runs/13199605689", "labels": ["ubuntu-latest"], "runner_id": 4, "runner_name": "GitHub Actions 4", "runner_group_id": 2, "runner_group_name": "GitHub Actions", "repository": "airbytehq/integration-test"}, "emitted_at": 1700587195423} +{"stream": "team_members", "data": {"login": "johnlafleur", "id": 68561602, "node_id": "MDQ6VXNlcjY4NTYxNjAy", "avatar_url": "https://avatars.githubusercontent.com/u/68561602?v=4", "gravatar_id": "", "url": "https://api.github.com/users/johnlafleur", "html_url": "https://github.com/johnlafleur", "followers_url": "https://api.github.com/users/johnlafleur/followers", "following_url": "https://api.github.com/users/johnlafleur/following{/other_user}", "gists_url": "https://api.github.com/users/johnlafleur/gists{/gist_id}", "starred_url": "https://api.github.com/users/johnlafleur/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/johnlafleur/subscriptions", "organizations_url": "https://api.github.com/users/johnlafleur/orgs", "repos_url": "https://api.github.com/users/johnlafleur/repos", "events_url": "https://api.github.com/users/johnlafleur/events{/privacy}", "received_events_url": "https://api.github.com/users/johnlafleur/received_events", "type": "User", "site_admin": false, "organization": "airbytehq", "team_slug": "airbyte-eng"}, "emitted_at": 1698750584444} +{"stream": "team_memberships", "data": {"state": "active", "role": "member", "url": "https://api.github.com/organizations/59758427/team/4559297/memberships/johnlafleur", "organization": "airbytehq", "team_slug": "airbyte-core", "username": "johnlafleur"}, "emitted_at": 1698757985640} +{"stream": "issue_timeline_events", "data": {"repository": "airbytehq/integration-test", "issue_number": 6, "labeled": {"id": 5219398390, "node_id": "MDEyOkxhYmVsZWRFdmVudDUyMTkzOTgzOTA=", "url": "https://api.github.com/repos/airbytehq/integration-test/issues/events/5219398390", "actor": {"login": "gaart", "id": 743901, "node_id": "MDQ6VXNlcjc0MzkwMQ==", "avatar_url": "https://avatars.githubusercontent.com/u/743901?v=4", "gravatar_id": "", "url": "https://api.github.com/users/gaart", "html_url": "https://github.com/gaart", "followers_url": "https://api.github.com/users/gaart/followers", "following_url": "https://api.github.com/users/gaart/following{/other_user}", "gists_url": "https://api.github.com/users/gaart/gists{/gist_id}", "starred_url": "https://api.github.com/users/gaart/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/gaart/subscriptions", "organizations_url": "https://api.github.com/users/gaart/orgs", "repos_url": "https://api.github.com/users/gaart/repos", "events_url": "https://api.github.com/users/gaart/events{/privacy}", "received_events_url": "https://api.github.com/users/gaart/received_events", "type": "User", "site_admin": false}, "event": "labeled", "commit_id": null, "commit_url": null, "created_at": "2021-08-27T15:43:58Z", "label": {"name": "critical", "color": "ededed"}, "performed_via_github_app": null}, "milestoned": {"id": 5219398392, "node_id": "MDE1Ok1pbGVzdG9uZWRFdmVudDUyMTkzOTgzOTI=", "url": "https://api.github.com/repos/airbytehq/integration-test/issues/events/5219398392", "actor": {"login": "gaart", "id": 743901, "node_id": "MDQ6VXNlcjc0MzkwMQ==", "avatar_url": "https://avatars.githubusercontent.com/u/743901?v=4", "gravatar_id": "", "url": "https://api.github.com/users/gaart", "html_url": "https://github.com/gaart", "followers_url": "https://api.github.com/users/gaart/followers", "following_url": "https://api.github.com/users/gaart/following{/other_user}", "gists_url": "https://api.github.com/users/gaart/gists{/gist_id}", "starred_url": "https://api.github.com/users/gaart/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/gaart/subscriptions", "organizations_url": "https://api.github.com/users/gaart/orgs", "repos_url": "https://api.github.com/users/gaart/repos", "events_url": "https://api.github.com/users/gaart/events{/privacy}", "received_events_url": "https://api.github.com/users/gaart/received_events", "type": "User", "site_admin": false}, "event": "milestoned", "commit_id": null, "commit_url": null, "created_at": "2021-08-27T15:43:58Z", "milestone": {"title": "main"}, "performed_via_github_app": null}, "commented": {"url": "https://api.github.com/repos/airbytehq/integration-test/issues/comments/907296167", "html_url": "https://github.com/airbytehq/integration-test/issues/6#issuecomment-907296167", "issue_url": "https://api.github.com/repos/airbytehq/integration-test/issues/6", "id": 907296167, "node_id": "IC_kwDOF9hP9c42FD2n", "user": {"login": "gaart", "id": 743901, "node_id": "MDQ6VXNlcjc0MzkwMQ==", "avatar_url": "https://avatars.githubusercontent.com/u/743901?v=4", "gravatar_id": "", "url": "https://api.github.com/users/gaart", "html_url": "https://github.com/gaart", "followers_url": "https://api.github.com/users/gaart/followers", "following_url": "https://api.github.com/users/gaart/following{/other_user}", "gists_url": "https://api.github.com/users/gaart/gists{/gist_id}", "starred_url": "https://api.github.com/users/gaart/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/gaart/subscriptions", "organizations_url": "https://api.github.com/users/gaart/orgs", "repos_url": "https://api.github.com/users/gaart/repos", "events_url": "https://api.github.com/users/gaart/events{/privacy}", "received_events_url": "https://api.github.com/users/gaart/received_events", "type": "User", "site_admin": false}, "created_at": "2021-08-27T15:43:59Z", "updated_at": "2021-08-27T15:43:59Z", "author_association": "CONTRIBUTOR", "body": "comment for issues https://api.github.com/repos/airbytehq/integration-test/issues/6/comments", "reactions": {"url": "https://api.github.com/repos/airbytehq/integration-test/issues/comments/907296167/reactions", "total_count": 0, "+1": 0, "-1": 0, "laugh": 0, "hooray": 0, "confused": 0, "heart": 0, "rocket": 0, "eyes": 0}, "performed_via_github_app": null, "event": "commented", "actor": {"login": "gaart", "id": 743901, "node_id": "MDQ6VXNlcjc0MzkwMQ==", "avatar_url": "https://avatars.githubusercontent.com/u/743901?v=4", "gravatar_id": "", "url": "https://api.github.com/users/gaart", "html_url": "https://github.com/gaart", "followers_url": "https://api.github.com/users/gaart/followers", "following_url": "https://api.github.com/users/gaart/following{/other_user}", "gists_url": "https://api.github.com/users/gaart/gists{/gist_id}", "starred_url": "https://api.github.com/users/gaart/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/gaart/subscriptions", "organizations_url": "https://api.github.com/users/gaart/orgs", "repos_url": "https://api.github.com/users/gaart/repos", "events_url": "https://api.github.com/users/gaart/events{/privacy}", "received_events_url": "https://api.github.com/users/gaart/received_events", "type": "User", "site_admin": false}}}, "emitted_at": 1695815681406} +{"stream": "issue_timeline_events", "data": {"repository": "airbytehq/integration-test", "issue_number": 9, "labeled": {"id": 5219398888, "node_id": "MDEyOkxhYmVsZWRFdmVudDUyMTkzOTg4ODg=", "url": "https://api.github.com/repos/airbytehq/integration-test/issues/events/5219398888", "actor": {"login": "gaart", "id": 743901, "node_id": "MDQ6VXNlcjc0MzkwMQ==", "avatar_url": "https://avatars.githubusercontent.com/u/743901?v=4", "gravatar_id": "", "url": "https://api.github.com/users/gaart", "html_url": "https://github.com/gaart", "followers_url": "https://api.github.com/users/gaart/followers", "following_url": "https://api.github.com/users/gaart/following{/other_user}", "gists_url": "https://api.github.com/users/gaart/gists{/gist_id}", "starred_url": "https://api.github.com/users/gaart/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/gaart/subscriptions", "organizations_url": "https://api.github.com/users/gaart/orgs", "repos_url": "https://api.github.com/users/gaart/repos", "events_url": "https://api.github.com/users/gaart/events{/privacy}", "received_events_url": "https://api.github.com/users/gaart/received_events", "type": "User", "site_admin": false}, "event": "labeled", "commit_id": null, "commit_url": null, "created_at": "2021-08-27T15:44:04Z", "label": {"name": "important", "color": "ededed"}, "performed_via_github_app": null}, "milestoned": {"id": 5219398894, "node_id": "MDE1Ok1pbGVzdG9uZWRFdmVudDUyMTkzOTg4OTQ=", "url": "https://api.github.com/repos/airbytehq/integration-test/issues/events/5219398894", "actor": {"login": "gaart", "id": 743901, "node_id": "MDQ6VXNlcjc0MzkwMQ==", "avatar_url": "https://avatars.githubusercontent.com/u/743901?v=4", "gravatar_id": "", "url": "https://api.github.com/users/gaart", "html_url": "https://github.com/gaart", "followers_url": "https://api.github.com/users/gaart/followers", "following_url": "https://api.github.com/users/gaart/following{/other_user}", "gists_url": "https://api.github.com/users/gaart/gists{/gist_id}", "starred_url": "https://api.github.com/users/gaart/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/gaart/subscriptions", "organizations_url": "https://api.github.com/users/gaart/orgs", "repos_url": "https://api.github.com/users/gaart/repos", "events_url": "https://api.github.com/users/gaart/events{/privacy}", "received_events_url": "https://api.github.com/users/gaart/received_events", "type": "User", "site_admin": false}, "event": "milestoned", "commit_id": null, "commit_url": null, "created_at": "2021-08-27T15:44:04Z", "milestone": {"title": "main"}, "performed_via_github_app": null}, "commented": {"url": "https://api.github.com/repos/airbytehq/integration-test/issues/comments/907296239", "html_url": "https://github.com/airbytehq/integration-test/issues/9#issuecomment-907296239", "issue_url": "https://api.github.com/repos/airbytehq/integration-test/issues/9", "id": 907296239, "node_id": "IC_kwDOF9hP9c42FD3v", "user": {"login": "gaart", "id": 743901, "node_id": "MDQ6VXNlcjc0MzkwMQ==", "avatar_url": "https://avatars.githubusercontent.com/u/743901?v=4", "gravatar_id": "", "url": "https://api.github.com/users/gaart", "html_url": "https://github.com/gaart", "followers_url": "https://api.github.com/users/gaart/followers", "following_url": "https://api.github.com/users/gaart/following{/other_user}", "gists_url": "https://api.github.com/users/gaart/gists{/gist_id}", "starred_url": "https://api.github.com/users/gaart/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/gaart/subscriptions", "organizations_url": "https://api.github.com/users/gaart/orgs", "repos_url": "https://api.github.com/users/gaart/repos", "events_url": "https://api.github.com/users/gaart/events{/privacy}", "received_events_url": "https://api.github.com/users/gaart/received_events", "type": "User", "site_admin": false}, "created_at": "2021-08-27T15:44:05Z", "updated_at": "2021-08-27T15:44:05Z", "author_association": "CONTRIBUTOR", "body": "comment for issues https://api.github.com/repos/airbytehq/integration-test/issues/9/comments", "reactions": {"url": "https://api.github.com/repos/airbytehq/integration-test/issues/comments/907296239/reactions", "total_count": 0, "+1": 0, "-1": 0, "laugh": 0, "hooray": 0, "confused": 0, "heart": 0, "rocket": 0, "eyes": 0}, "performed_via_github_app": null, "event": "commented", "actor": {"login": "gaart", "id": 743901, "node_id": "MDQ6VXNlcjc0MzkwMQ==", "avatar_url": "https://avatars.githubusercontent.com/u/743901?v=4", "gravatar_id": "", "url": "https://api.github.com/users/gaart", "html_url": "https://github.com/gaart", "followers_url": "https://api.github.com/users/gaart/followers", "following_url": "https://api.github.com/users/gaart/following{/other_user}", "gists_url": "https://api.github.com/users/gaart/gists{/gist_id}", "starred_url": "https://api.github.com/users/gaart/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/gaart/subscriptions", "organizations_url": "https://api.github.com/users/gaart/orgs", "repos_url": "https://api.github.com/users/gaart/repos", "events_url": "https://api.github.com/users/gaart/events{/privacy}", "received_events_url": "https://api.github.com/users/gaart/received_events", "type": "User", "site_admin": false}}}, "emitted_at": 1695815681650} +{"stream": "issue_timeline_events", "data": {"repository": "airbytehq/integration-test", "issue_number": 11, "labeled": {"id": 5219399223, "node_id": "MDEyOkxhYmVsZWRFdmVudDUyMTkzOTkyMjM=", "url": "https://api.github.com/repos/airbytehq/integration-test/issues/events/5219399223", "actor": {"login": "gaart", "id": 743901, "node_id": "MDQ6VXNlcjc0MzkwMQ==", "avatar_url": "https://avatars.githubusercontent.com/u/743901?v=4", "gravatar_id": "", "url": "https://api.github.com/users/gaart", "html_url": "https://github.com/gaart", "followers_url": "https://api.github.com/users/gaart/followers", "following_url": "https://api.github.com/users/gaart/following{/other_user}", "gists_url": "https://api.github.com/users/gaart/gists{/gist_id}", "starred_url": "https://api.github.com/users/gaart/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/gaart/subscriptions", "organizations_url": "https://api.github.com/users/gaart/orgs", "repos_url": "https://api.github.com/users/gaart/repos", "events_url": "https://api.github.com/users/gaart/events{/privacy}", "received_events_url": "https://api.github.com/users/gaart/received_events", "type": "User", "site_admin": false}, "event": "labeled", "commit_id": null, "commit_url": null, "created_at": "2021-08-27T15:44:08Z", "label": {"name": "important", "color": "ededed"}, "performed_via_github_app": null}, "milestoned": {"id": 5219399233, "node_id": "MDE1Ok1pbGVzdG9uZWRFdmVudDUyMTkzOTkyMzM=", "url": "https://api.github.com/repos/airbytehq/integration-test/issues/events/5219399233", "actor": {"login": "gaart", "id": 743901, "node_id": "MDQ6VXNlcjc0MzkwMQ==", "avatar_url": "https://avatars.githubusercontent.com/u/743901?v=4", "gravatar_id": "", "url": "https://api.github.com/users/gaart", "html_url": "https://github.com/gaart", "followers_url": "https://api.github.com/users/gaart/followers", "following_url": "https://api.github.com/users/gaart/following{/other_user}", "gists_url": "https://api.github.com/users/gaart/gists{/gist_id}", "starred_url": "https://api.github.com/users/gaart/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/gaart/subscriptions", "organizations_url": "https://api.github.com/users/gaart/orgs", "repos_url": "https://api.github.com/users/gaart/repos", "events_url": "https://api.github.com/users/gaart/events{/privacy}", "received_events_url": "https://api.github.com/users/gaart/received_events", "type": "User", "site_admin": false}, "event": "milestoned", "commit_id": null, "commit_url": null, "created_at": "2021-08-27T15:44:08Z", "milestone": {"title": "main"}, "performed_via_github_app": null}, "commented": {"url": "https://api.github.com/repos/airbytehq/integration-test/issues/comments/907296275", "html_url": "https://github.com/airbytehq/integration-test/issues/11#issuecomment-907296275", "issue_url": "https://api.github.com/repos/airbytehq/integration-test/issues/11", "id": 907296275, "node_id": "IC_kwDOF9hP9c42FD4T", "user": {"login": "gaart", "id": 743901, "node_id": "MDQ6VXNlcjc0MzkwMQ==", "avatar_url": "https://avatars.githubusercontent.com/u/743901?v=4", "gravatar_id": "", "url": "https://api.github.com/users/gaart", "html_url": "https://github.com/gaart", "followers_url": "https://api.github.com/users/gaart/followers", "following_url": "https://api.github.com/users/gaart/following{/other_user}", "gists_url": "https://api.github.com/users/gaart/gists{/gist_id}", "starred_url": "https://api.github.com/users/gaart/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/gaart/subscriptions", "organizations_url": "https://api.github.com/users/gaart/orgs", "repos_url": "https://api.github.com/users/gaart/repos", "events_url": "https://api.github.com/users/gaart/events{/privacy}", "received_events_url": "https://api.github.com/users/gaart/received_events", "type": "User", "site_admin": false}, "created_at": "2021-08-27T15:44:09Z", "updated_at": "2021-08-27T15:44:09Z", "author_association": "CONTRIBUTOR", "body": "comment for issues https://api.github.com/repos/airbytehq/integration-test/issues/11/comments", "reactions": {"url": "https://api.github.com/repos/airbytehq/integration-test/issues/comments/907296275/reactions", "total_count": 3, "+1": 1, "-1": 0, "laugh": 1, "hooray": 1, "confused": 0, "heart": 0, "rocket": 0, "eyes": 0}, "performed_via_github_app": null, "event": "commented", "actor": {"login": "gaart", "id": 743901, "node_id": "MDQ6VXNlcjc0MzkwMQ==", "avatar_url": "https://avatars.githubusercontent.com/u/743901?v=4", "gravatar_id": "", "url": "https://api.github.com/users/gaart", "html_url": "https://github.com/gaart", "followers_url": "https://api.github.com/users/gaart/followers", "following_url": "https://api.github.com/users/gaart/following{/other_user}", "gists_url": "https://api.github.com/users/gaart/gists{/gist_id}", "starred_url": "https://api.github.com/users/gaart/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/gaart/subscriptions", "organizations_url": "https://api.github.com/users/gaart/orgs", "repos_url": "https://api.github.com/users/gaart/repos", "events_url": "https://api.github.com/users/gaart/events{/privacy}", "received_events_url": "https://api.github.com/users/gaart/received_events", "type": "User", "site_admin": false}}}, "emitted_at": 1695815681995} diff --git a/airbyte-integrations/connectors/source-github/main.py b/airbyte-integrations/connectors/source-github/main.py index eedaef37f125..aa6b652e953c 100644 --- a/airbyte-integrations/connectors/source-github/main.py +++ b/airbyte-integrations/connectors/source-github/main.py @@ -7,7 +7,10 @@ from airbyte_cdk.entrypoint import launch from source_github import SourceGithub +from source_github.config_migrations import MigrateBranch, MigrateRepository if __name__ == "__main__": source = SourceGithub() + MigrateRepository.migrate(sys.argv[1:], source) + MigrateBranch.migrate(sys.argv[1:], source) launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-github/metadata.yaml b/airbyte-integrations/connectors/source-github/metadata.yaml index 7684650afb9e..cece9f362bd4 100644 --- a/airbyte-integrations/connectors/source-github/metadata.yaml +++ b/airbyte-integrations/connectors/source-github/metadata.yaml @@ -1,16 +1,22 @@ data: + ab_internal: + ql: 400 + sl: 200 allowedHosts: hosts: - - api.github.com + - ${api_url} + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: api connectorType: source definitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e - dockerImageTag: 1.0.4 - maxSecondsBetweenMessages: 5400 + dockerImageTag: 1.5.5 dockerRepository: airbyte/source-github + documentationUrl: https://docs.airbyte.com/integrations/sources/github githubIssueLabel: source-github icon: github.svg license: MIT + maxSecondsBetweenMessages: 5400 name: GitHub registries: cloud: @@ -30,11 +36,7 @@ data: - tags - teams - users - documentationUrl: https://docs.airbyte.com/integrations/sources/github + supportLevel: certified tags: - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-github/setup.py b/airbyte-integrations/connectors/source-github/setup.py index 0c618c4eef04..8b5f90f29e12 100644 --- a/airbyte-integrations/connectors/source-github/setup.py +++ b/airbyte-integrations/connectors/source-github/setup.py @@ -5,9 +5,9 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.2", "pendulum~=2.1.2", "sgqlc"] +MAIN_REQUIREMENTS = ["airbyte-cdk", "sgqlc"] -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1", "responses~=0.23.1", "freezegun~=1.2.0"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.2", "responses~=0.23.1", "freezegun~=1.2"] setup( name="source_github", diff --git a/airbyte-integrations/connectors/source-github/source_github/config_migrations.py b/airbyte-integrations/connectors/source-github/source_github/config_migrations.py new file mode 100644 index 000000000000..79ec73a9cd2f --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/config_migrations.py @@ -0,0 +1,106 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import abc +import logging +from abc import ABC +from typing import Any, List, Mapping + +from airbyte_cdk.config_observation import create_connector_config_control_message +from airbyte_cdk.entrypoint import AirbyteEntrypoint +from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + +from .source import SourceGithub + +logger = logging.getLogger("airbyte_logger") + + +class MigrateStringToArray(ABC): + """ + This class stands for migrating the config at runtime, + while providing the backward compatibility when falling back to the previous source version. + + Specifically, starting from `1.4.6`, the `repository` and `branch` properties should be like : + > List(["", "", ..., ""]) + instead of, in `1.4.5`: + > JSON STR: "repository_1 repository_2" + """ + + message_repository: MessageRepository = InMemoryMessageRepository() + + @property + @abc.abstractmethod + def migrate_from_key(self) -> str: + ... + + @property + @abc.abstractmethod + def migrate_to_key(self) -> str: + ... + + @classmethod + def _should_migrate(cls, config: Mapping[str, Any]) -> bool: + """ + This method determines whether config require migration. + Returns: + > True, if the transformation is necessary + > False, otherwise. + """ + if cls.migrate_from_key in config and cls.migrate_to_key not in config: + return True + return False + + @classmethod + def _transform_to_array(cls, config: Mapping[str, Any], source: SourceGithub = None) -> Mapping[str, Any]: + # assign old values to new property that will be used within the new version + config[cls.migrate_to_key] = config[cls.migrate_to_key] if cls.migrate_to_key in config else [] + data = set(filter(None, config.get(cls.migrate_from_key).split(" "))) + config[cls.migrate_to_key] = list(data | set(config[cls.migrate_to_key])) + return config + + @classmethod + def _modify_and_save(cls, config_path: str, source: SourceGithub, config: Mapping[str, Any]) -> Mapping[str, Any]: + # modify the config + migrated_config = cls._transform_to_array(config, source) + # save the config + source.write_config(migrated_config, config_path) + # return modified config + return migrated_config + + @classmethod + def _emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: + # add the Airbyte Control Message to message repo + cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) + # emit the Airbyte Control Message from message queue to stdout + for message in cls.message_repository._message_queue: + print(message.json(exclude_unset=True)) + + @classmethod + def migrate(cls, args: List[str], source: SourceGithub) -> None: + """ + This method checks the input args, should the config be migrated, + transform if necessary and emit the CONTROL message. + """ + # get config path + config_path = AirbyteEntrypoint(source).extract_config(args) + # proceed only if `--config` arg is provided + if config_path: + # read the existing config + config = source.read_config(config_path) + # migration check + if cls._should_migrate(config): + cls._emit_control_message( + cls._modify_and_save(config_path, source, config), + ) + + +class MigrateRepository(MigrateStringToArray): + + migrate_from_key: str = "repository" + migrate_to_key: str = "repositories" + + +class MigrateBranch(MigrateStringToArray): + + migrate_from_key: str = "branch" + migrate_to_key: str = "branches" diff --git a/airbyte-integrations/connectors/source-github/source_github/github_schema.py b/airbyte-integrations/connectors/source-github/source_github/github_schema.py index 318f9fc3edfd..ace0d9c69e2d 100644 --- a/airbyte-integrations/connectors/source-github/source_github/github_schema.py +++ b/airbyte-integrations/connectors/source-github/source_github/github_schema.py @@ -51,6 +51,15 @@ class Base64String(sgqlc.types.Scalar): __schema__ = github_schema +class BigInt(sgqlc.types.Scalar): + """Represents non-fractional signed whole numeric values. Since the + value may exceed the size of a 32-bit integer, it's encoded as a + string. + """ + + __schema__ = github_schema + + Boolean = sgqlc.types.Boolean @@ -89,6 +98,47 @@ class CheckConclusionState(sgqlc.types.Enum): __choices__ = ("ACTION_REQUIRED", "CANCELLED", "FAILURE", "NEUTRAL", "SKIPPED", "STALE", "STARTUP_FAILURE", "SUCCESS", "TIMED_OUT") +class CheckRunState(sgqlc.types.Enum): + """The possible states of a check run in a status rollup. + + Enumeration Choices: + + * `ACTION_REQUIRED`: The check run requires action. + * `CANCELLED`: The check run has been cancelled. + * `COMPLETED`: The check run has been completed. + * `FAILURE`: The check run has failed. + * `IN_PROGRESS`: The check run is in progress. + * `NEUTRAL`: The check run was neutral. + * `PENDING`: The check run is in pending state. + * `QUEUED`: The check run has been queued. + * `SKIPPED`: The check run was skipped. + * `STALE`: The check run was marked stale by GitHub. Only GitHub + can use this conclusion. + * `STARTUP_FAILURE`: The check run has failed at startup. + * `SUCCESS`: The check run has succeeded. + * `TIMED_OUT`: The check run has timed out. + * `WAITING`: The check run is in waiting state. + """ + + __schema__ = github_schema + __choices__ = ( + "ACTION_REQUIRED", + "CANCELLED", + "COMPLETED", + "FAILURE", + "IN_PROGRESS", + "NEUTRAL", + "PENDING", + "QUEUED", + "SKIPPED", + "STALE", + "STARTUP_FAILURE", + "SUCCESS", + "TIMED_OUT", + "WAITING", + ) + + class CheckRunType(sgqlc.types.Enum): """The possible types of check runs. @@ -196,6 +246,22 @@ class CommitContributionOrderField(sgqlc.types.Enum): __choices__ = ("COMMIT_COUNT", "OCCURRED_AT") +class ComparisonStatus(sgqlc.types.Enum): + """The status of a git comparison between two refs. + + Enumeration Choices: + + * `AHEAD`: The head ref is ahead of the base ref. + * `BEHIND`: The head ref is behind the base ref. + * `DIVERGED`: The head ref is both ahead and behind of the base + ref, indicating git history has diverged. + * `IDENTICAL`: The head ref and base ref are identical. + """ + + __schema__ = github_schema + __choices__ = ("AHEAD", "BEHIND", "DIVERGED", "IDENTICAL") + + class ContributionLevel(sgqlc.types.Enum): """Varying levels of contributions from none to many. @@ -248,12 +314,13 @@ class DependencyGraphEcosystem(sgqlc.types.Enum): * `NPM`: JavaScript packages hosted at npmjs.com * `NUGET`: .NET packages hosted at the NuGet Gallery * `PIP`: Python packages hosted at PyPI.org + * `PUB`: Dart packages hosted at pub.dev * `RUBYGEMS`: Ruby gems hosted at RubyGems.org * `RUST`: Rust crates """ __schema__ = github_schema - __choices__ = ("ACTIONS", "COMPOSER", "GO", "MAVEN", "NPM", "NUGET", "PIP", "RUBYGEMS", "RUST") + __choices__ = ("ACTIONS", "COMPOSER", "GO", "MAVEN", "NPM", "NUGET", "PIP", "PUB", "RUBYGEMS", "RUST") class DeploymentOrderField(sgqlc.types.Enum): @@ -309,11 +376,24 @@ class DeploymentState(sgqlc.types.Enum): * `IN_PROGRESS`: The deployment is in progress. * `PENDING`: The deployment is pending. * `QUEUED`: The deployment has queued + * `SUCCESS`: The deployment was successful. * `WAITING`: The deployment is waiting. """ __schema__ = github_schema - __choices__ = ("ABANDONED", "ACTIVE", "DESTROYED", "ERROR", "FAILURE", "INACTIVE", "IN_PROGRESS", "PENDING", "QUEUED", "WAITING") + __choices__ = ( + "ABANDONED", + "ACTIVE", + "DESTROYED", + "ERROR", + "FAILURE", + "INACTIVE", + "IN_PROGRESS", + "PENDING", + "QUEUED", + "SUCCESS", + "WAITING", + ) class DeploymentStatusState(sgqlc.types.Enum): @@ -348,6 +428,20 @@ class DiffSide(sgqlc.types.Enum): __choices__ = ("LEFT", "RIGHT") +class DiscussionCloseReason(sgqlc.types.Enum): + """The possible reasons for closing a discussion. + + Enumeration Choices: + + * `DUPLICATE`: The discussion is a duplicate of another + * `OUTDATED`: The discussion is no longer relevant + * `RESOLVED`: The discussion has been resolved + """ + + __schema__ = github_schema + __choices__ = ("DUPLICATE", "OUTDATED", "RESOLVED") + + class DiscussionOrderField(sgqlc.types.Enum): """Properties by which discussion connections can be ordered. @@ -377,6 +471,34 @@ class DiscussionPollOptionOrderField(sgqlc.types.Enum): __choices__ = ("AUTHORED_ORDER", "VOTE_COUNT") +class DiscussionState(sgqlc.types.Enum): + """The possible states of a discussion. + + Enumeration Choices: + + * `CLOSED`: A discussion that has been closed + * `OPEN`: A discussion that is open + """ + + __schema__ = github_schema + __choices__ = ("CLOSED", "OPEN") + + +class DiscussionStateReason(sgqlc.types.Enum): + """The possible state reasons of a discussion. + + Enumeration Choices: + + * `DUPLICATE`: The discussion is a duplicate of another + * `OUTDATED`: The discussion is no longer relevant + * `REOPENED`: The discussion was reopened + * `RESOLVED`: The discussion has been resolved + """ + + __schema__ = github_schema + __choices__ = ("DUPLICATE", "OUTDATED", "REOPENED", "RESOLVED") + + class DismissReason(sgqlc.types.Enum): """The possible reasons that a Dependabot alert was dismissed. @@ -421,6 +543,39 @@ class EnterpriseAdministratorRole(sgqlc.types.Enum): __choices__ = ("BILLING_MANAGER", "OWNER") +class EnterpriseAllowPrivateRepositoryForkingPolicyValue(sgqlc.types.Enum): + """The possible values for the enterprise allow private repository + forking policy value. + + Enumeration Choices: + + * `ENTERPRISE_ORGANIZATIONS`: Members can fork a repository to an + organization within this enterprise. + * `ENTERPRISE_ORGANIZATIONS_USER_ACCOUNTS`: Members can fork a + repository to their enterprise-managed user account or an + organization inside this enterprise. + * `EVERYWHERE`: Members can fork a repository to their user + account or an organization, either inside or outside of this + enterprise. + * `SAME_ORGANIZATION`: Members can fork a repository only within + the same organization (intra-org). + * `SAME_ORGANIZATION_USER_ACCOUNTS`: Members can fork a repository + to their user account or within the same organization. + * `USER_ACCOUNTS`: Members can fork a repository to their user + account. + """ + + __schema__ = github_schema + __choices__ = ( + "ENTERPRISE_ORGANIZATIONS", + "ENTERPRISE_ORGANIZATIONS_USER_ACCOUNTS", + "EVERYWHERE", + "SAME_ORGANIZATION", + "SAME_ORGANIZATION_USER_ACCOUNTS", + "USER_ACCOUNTS", + ) + + class EnterpriseDefaultRepositoryPermissionSettingValue(sgqlc.types.Enum): """The possible values for the enterprise base repository permission setting. @@ -607,10 +762,13 @@ class EnterpriseUserAccountMembershipRole(sgqlc.types.Enum): enterprise. * `OWNER`: The user is an owner of an organization in the enterprise. + * `UNAFFILIATED`: The user is not an owner of the enterprise, and + not a member or owner of any organizations in the enterprise; + only for EMU-enabled enterprises. """ __schema__ = github_schema - __choices__ = ("MEMBER", "OWNER") + __choices__ = ("MEMBER", "OWNER", "UNAFFILIATED") class EnterpriseUserDeployment(sgqlc.types.Enum): @@ -1078,8 +1236,7 @@ class MergeStateStatus(sgqlc.types.Enum): * `BLOCKED`: The merge is blocked. * `CLEAN`: Mergeable and passing commit status. * `DIRTY`: The merge commit cannot be cleanly created. - * `HAS_HOOKS`: Mergeable with passing commit status and pre- - receive hooks. + * `HAS_HOOKS`: Mergeable with passing commit status and prereceive hooks. * `UNKNOWN`: The state cannot currently be determined. * `UNSTABLE`: Mergeable with non-passing commit status. """ @@ -1088,6 +1245,78 @@ class MergeStateStatus(sgqlc.types.Enum): __choices__ = ("BEHIND", "BLOCKED", "CLEAN", "DIRTY", "HAS_HOOKS", "UNKNOWN", "UNSTABLE") +class MannequinOrderField(sgqlc.types.Enum): + """Properties by which mannequins can be ordered. + + Enumeration Choices: + + * `CREATED_AT`: Order mannequins why when they were created. + * `LOGIN`: Order mannequins alphabetically by their source login. + """ + + __schema__ = github_schema + __choices__ = ("CREATED_AT", "LOGIN") + + +class MergeCommitMessage(sgqlc.types.Enum): + """The possible default commit messages for merges. + + Enumeration Choices: + + * `BLANK`: Default to a blank commit message. + * `PR_BODY`: Default to the pull request's body. + * `PR_TITLE`: Default to the pull request's title. + """ + + __schema__ = github_schema + __choices__ = ("BLANK", "PR_BODY", "PR_TITLE") + + +class MergeCommitTitle(sgqlc.types.Enum): + """The possible default commit titles for merges. + + Enumeration Choices: + + * `MERGE_MESSAGE`: Default to the classic title for a merge + message (e.g., Merge pull request #123 from branch-name). + * `PR_TITLE`: Default to the pull request's title. + """ + + __schema__ = github_schema + __choices__ = ("MERGE_MESSAGE", "PR_TITLE") + + +class MergeQueueEntryState(sgqlc.types.Enum): + """The possible states for a merge queue entry. + + Enumeration Choices: + + * `AWAITING_CHECKS`: The entry is currently waiting for checks to + pass. + * `LOCKED`: The entry is currently locked. + * `MERGEABLE`: The entry is currently mergeable. + * `QUEUED`: The entry is currently queued. + * `UNMERGEABLE`: The entry is currently unmergeable. + """ + + __schema__ = github_schema + __choices__ = ("AWAITING_CHECKS", "LOCKED", "MERGEABLE", "QUEUED", "UNMERGEABLE") + + +class MergeQueueMergingStrategy(sgqlc.types.Enum): + """The possible merging strategies for a merge queue. + + Enumeration Choices: + + * `ALLGREEN`: Entries only allowed to merge if they are passing. + * `HEADGREEN`: Failing Entires are allowed to merge if they are + with a passing entry. + """ + + __schema__ = github_schema + __choices__ = ("ALLGREEN", "HEADGREEN") + + class MergeableState(sgqlc.types.Enum): """Whether or not a PullRequest can be merged. @@ -1105,35 +1334,33 @@ class MergeableState(sgqlc.types.Enum): class MigrationSourceType(sgqlc.types.Enum): - """Represents the different Octoshift migration sources. + """Represents the different GitHub Enterprise Importer (GEI) + migration sources. Enumeration Choices: * `AZURE_DEVOPS`: An Azure DevOps migration source. * `BITBUCKET_SERVER`: A Bitbucket Server migration source. - * `GITHUB`: A GitHub migration source. * `GITHUB_ARCHIVE`: A GitHub Migration API source. - * `GITLAB`: A GitLab migration source. """ __schema__ = github_schema - __choices__ = ("AZURE_DEVOPS", "BITBUCKET_SERVER", "GITHUB", "GITHUB_ARCHIVE", "GITLAB") + __choices__ = ("AZURE_DEVOPS", "BITBUCKET_SERVER", "GITHUB_ARCHIVE") class MigrationState(sgqlc.types.Enum): - """The Octoshift migration state. + """The GitHub Enterprise Importer (GEI) migration state. Enumeration Choices: - * `FAILED`: The Octoshift migration has failed. - * `FAILED_VALIDATION`: The Octoshift migration has invalid - credentials. - * `IN_PROGRESS`: The Octoshift migration is in progress. - * `NOT_STARTED`: The Octoshift migration has not started. - * `PENDING_VALIDATION`: The Octoshift migration needs to have its + * `FAILED`: The migration has failed. + * `FAILED_VALIDATION`: The migration has invalid credentials. + * `IN_PROGRESS`: The migration is in progress. + * `NOT_STARTED`: The migration has not started. + * `PENDING_VALIDATION`: The migration needs to have its credentials validated. - * `QUEUED`: The Octoshift migration has been queued. - * `SUCCEEDED`: The Octoshift migration has succeeded. + * `QUEUED`: The migration has been queued. + * `SUCCEEDED`: The migration has succeeded. """ __schema__ = github_schema @@ -1194,15 +1421,15 @@ class OIDCProviderType(sgqlc.types.Enum): class OauthApplicationCreateAuditEntryState(sgqlc.types.Enum): - """The state of an OAuth Application when it was created. + """The state of an OAuth application when it was created. Enumeration Choices: - * `ACTIVE`: The OAuth Application was active and allowed to have + * `ACTIVE`: The OAuth application was active and allowed to have OAuth Accesses. - * `PENDING_DELETION`: The OAuth Application was in the process of + * `PENDING_DELETION`: The OAuth application was in the process of being deleted. - * `SUSPENDED`: The OAuth Application was suspended from generating + * `SUSPENDED`: The OAuth application was suspended from generating OAuth Accesses due to abuse or security concerns. """ @@ -1475,6 +1702,21 @@ class OrganizationInvitationRole(sgqlc.types.Enum): __choices__ = ("ADMIN", "BILLING_MANAGER", "DIRECT_MEMBER", "REINSTATE") +class OrganizationInvitationSource(sgqlc.types.Enum): + """The possible organization invitation sources. + + Enumeration Choices: + + * `MEMBER`: The invitation was created from the web interface or + from API + * `SCIM`: The invitation was created from SCIM + * `UNKNOWN`: The invitation was sent before this feature was added + """ + + __schema__ = github_schema + __choices__ = ("MEMBER", "SCIM", "UNKNOWN") + + class OrganizationInvitationType(sgqlc.types.Enum): """The possible organization invitation types. @@ -1521,6 +1763,43 @@ class OrganizationMembersCanCreateRepositoriesSettingValue(sgqlc.types.Enum): __choices__ = ("ALL", "DISABLED", "INTERNAL", "PRIVATE") +class OrganizationMigrationState(sgqlc.types.Enum): + """The Octoshift Organization migration state. + + Enumeration Choices: + + * `FAILED`: The Octoshift migration has failed. + * `FAILED_VALIDATION`: The Octoshift migration has invalid + credentials. + * `IN_PROGRESS`: The Octoshift migration is in progress. + * `NOT_STARTED`: The Octoshift migration has not started. + * `PENDING_VALIDATION`: The Octoshift migration needs to have its + credentials validated. + * `POST_REPO_MIGRATION`: The Octoshift migration is performing + post repository migrations. + * `PRE_REPO_MIGRATION`: The Octoshift migration is performing pre + repository migrations. + * `QUEUED`: The Octoshift migration has been queued. + * `REPO_MIGRATION`: The Octoshift org migration is performing + repository migrations. + * `SUCCEEDED`: The Octoshift migration has succeeded. + """ + + __schema__ = github_schema + __choices__ = ( + "FAILED", + "FAILED_VALIDATION", + "IN_PROGRESS", + "NOT_STARTED", + "PENDING_VALIDATION", + "POST_REPO_MIGRATION", + "PRE_REPO_MIGRATION", + "QUEUED", + "REPO_MIGRATION", + "SUCCEEDED", + ) + + class OrganizationOrderField(sgqlc.types.Enum): """Properties by which organization connections can be ordered. @@ -1564,15 +1843,11 @@ class PackageType(sgqlc.types.Enum): Enumeration Choices: * `DEBIAN`: A debian package. - * `MAVEN`: A maven package. - * `NPM`: An npm package. - * `NUGET`: A nuget package. * `PYPI`: A python package. - * `RUBYGEMS`: A rubygems package. """ __schema__ = github_schema - __choices__ = ("DEBIAN", "MAVEN", "NPM", "NUGET", "PYPI", "RUBYGEMS") + __choices__ = ("DEBIAN", "PYPI") class PackageVersionOrderField(sgqlc.types.Enum): @@ -1707,23 +1982,84 @@ class ProjectColumnPurpose(sgqlc.types.Enum): __choices__ = ("DONE", "IN_PROGRESS", "TODO") -class ProjectItemType(sgqlc.types.Enum): - """The type of a project item. +class ProjectOrderField(sgqlc.types.Enum): + """Properties by which project connections can be ordered. Enumeration Choices: - * `DRAFT_ISSUE`: Draft Issue - * `ISSUE`: Issue - * `PULL_REQUEST`: Pull Request - * `REDACTED`: Redacted Item + * `CREATED_AT`: Order projects by creation time + * `NAME`: Order projects by name + * `UPDATED_AT`: Order projects by update time """ __schema__ = github_schema - __choices__ = ("DRAFT_ISSUE", "ISSUE", "PULL_REQUEST", "REDACTED") + __choices__ = ("CREATED_AT", "NAME", "UPDATED_AT") -class ProjectNextFieldType(sgqlc.types.Enum): - """The type of a project next field. +class ProjectState(sgqlc.types.Enum): + """State of the project; either 'open' or 'closed' + + Enumeration Choices: + + * `CLOSED`: The project is closed. + * `OPEN`: The project is open. + """ + + __schema__ = github_schema + __choices__ = ("CLOSED", "OPEN") + + +class ProjectTemplate(sgqlc.types.Enum): + """GitHub-provided templates for Projects + + Enumeration Choices: + + * `AUTOMATED_KANBAN_V2`: Create a board with v2 triggers to + automatically move cards across To do, In progress and Done + columns. + * `AUTOMATED_REVIEWS_KANBAN`: Create a board with triggers to + automatically move cards across columns with review automation. + * `BASIC_KANBAN`: Create a board with columns for To do, In + progress and Done. + * `BUG_TRIAGE`: Create a board to triage and prioritize bugs with + To do, priority, and Done columns. + """ + + __schema__ = github_schema + __choices__ = ("AUTOMATED_KANBAN_V2", "AUTOMATED_REVIEWS_KANBAN", "BASIC_KANBAN", "BUG_TRIAGE") + + +class ProjectV2CustomFieldType(sgqlc.types.Enum): + """The type of a project field. + + Enumeration Choices: + + * `DATE`: Date + * `NUMBER`: Number + * `SINGLE_SELECT`: Single Select + * `TEXT`: Text + """ + + __schema__ = github_schema + __choices__ = ("DATE", "NUMBER", "SINGLE_SELECT", "TEXT") + + +class ProjectV2FieldOrderField(sgqlc.types.Enum): + """Properties by which project v2 field connections can be ordered. + + Enumeration Choices: + + * `CREATED_AT`: Order project v2 fields by creation time + * `NAME`: Order project v2 fields by name + * `POSITION`: Order project v2 fields by position + """ + + __schema__ = github_schema + __choices__ = ("CREATED_AT", "NAME", "POSITION") + + +class ProjectV2FieldType(sgqlc.types.Enum): + """The type of a project field. Enumeration Choices: @@ -1739,6 +2075,7 @@ class ProjectNextFieldType(sgqlc.types.Enum): * `SINGLE_SELECT`: Single Select * `TEXT`: Text * `TITLE`: Title + * `TRACKED_BY`: Tracked by * `TRACKS`: Tracks """ @@ -1756,12 +2093,55 @@ class ProjectNextFieldType(sgqlc.types.Enum): "SINGLE_SELECT", "TEXT", "TITLE", + "TRACKED_BY", "TRACKS", ) -class ProjectNextOrderField(sgqlc.types.Enum): - """Properties by which the return project can be ordered. +class ProjectV2ItemFieldValueOrderField(sgqlc.types.Enum): + """Properties by which project v2 item field value connections can be + ordered. + + Enumeration Choices: + + * `POSITION`: Order project v2 item field values by the their + position in the project + """ + + __schema__ = github_schema + __choices__ = ("POSITION",) + + +class ProjectV2ItemOrderField(sgqlc.types.Enum): + """Properties by which project v2 item connections can be ordered. + + Enumeration Choices: + + * `POSITION`: Order project v2 items by the their position in the + project + """ + + __schema__ = github_schema + __choices__ = ("POSITION",) + + +class ProjectV2ItemType(sgqlc.types.Enum): + """The type of a project item. + + Enumeration Choices: + + * `DRAFT_ISSUE`: Draft Issue + * `ISSUE`: Issue + * `PULL_REQUEST`: Pull Request + * `REDACTED`: Redacted Item + """ + + __schema__ = github_schema + __choices__ = ("DRAFT_ISSUE", "ISSUE", "PULL_REQUEST", "REDACTED") + + +class ProjectV2OrderField(sgqlc.types.Enum): + """Properties by which projects can be ordered. Enumeration Choices: @@ -1775,64 +2155,95 @@ class ProjectNextOrderField(sgqlc.types.Enum): __choices__ = ("CREATED_AT", "NUMBER", "TITLE", "UPDATED_AT") -class ProjectOrderField(sgqlc.types.Enum): - """Properties by which project connections can be ordered. +class ProjectV2Roles(sgqlc.types.Enum): + """The possible roles of a collaborator on a project. Enumeration Choices: - * `CREATED_AT`: Order projects by creation time - * `NAME`: Order projects by name - * `UPDATED_AT`: Order projects by update time + * `ADMIN`: The collaborator can view, edit, and maange the + settings of the project + * `NONE`: The collaborator has no direct access to the project + * `READER`: The collaborator can view the project + * `WRITER`: The collaborator can view and edit the project """ __schema__ = github_schema - __choices__ = ("CREATED_AT", "NAME", "UPDATED_AT") + __choices__ = ("ADMIN", "NONE", "READER", "WRITER") -class ProjectState(sgqlc.types.Enum): - """State of the project; either 'open' or 'closed' +class ProjectV2SingleSelectFieldOptionColor(sgqlc.types.Enum): + """The display color of a single-select field option. Enumeration Choices: - * `CLOSED`: The project is closed. - * `OPEN`: The project is open. + * `BLUE`: BLUE + * `GRAY`: GRAY + * `GREEN`: GREEN + * `ORANGE`: ORANGE + * `PINK`: PINK + * `PURPLE`: PURPLE + * `RED`: RED + * `YELLOW`: YELLOW """ __schema__ = github_schema - __choices__ = ("CLOSED", "OPEN") + __choices__ = ("BLUE", "GRAY", "GREEN", "ORANGE", "PINK", "PURPLE", "RED", "YELLOW") -class ProjectTemplate(sgqlc.types.Enum): - """GitHub-provided templates for Projects +class ProjectV2State(sgqlc.types.Enum): + """The possible states of a project v2. Enumeration Choices: - * `AUTOMATED_KANBAN_V2`: Create a board with v2 triggers to - automatically move cards across To do, In progress and Done - columns. - * `AUTOMATED_REVIEWS_KANBAN`: Create a board with triggers to - automatically move cards across columns with review automation. - * `BASIC_KANBAN`: Create a board with columns for To do, In - progress and Done. - * `BUG_TRIAGE`: Create a board to triage and prioritize bugs with - To do, priority, and Done columns. + * `CLOSED`: A project v2 that has been closed + * `OPEN`: A project v2 that is still open """ __schema__ = github_schema - __choices__ = ("AUTOMATED_KANBAN_V2", "AUTOMATED_REVIEWS_KANBAN", "BASIC_KANBAN", "BUG_TRIAGE") + __choices__ = ("CLOSED", "OPEN") -class ProjectViewLayout(sgqlc.types.Enum): - """The layout of a project view. +class ProjectV2ViewLayout(sgqlc.types.Enum): + """The layout of a project v2 view. Enumeration Choices: * `BOARD_LAYOUT`: Board layout + * `ROADMAP_LAYOUT`: Roadmap layout * `TABLE_LAYOUT`: Table layout """ __schema__ = github_schema - __choices__ = ("BOARD_LAYOUT", "TABLE_LAYOUT") + __choices__ = ("BOARD_LAYOUT", "ROADMAP_LAYOUT", "TABLE_LAYOUT") + + +class ProjectV2ViewOrderField(sgqlc.types.Enum): + """Properties by which project v2 view connections can be ordered. + + Enumeration Choices: + + * `CREATED_AT`: Order project v2 views by creation time + * `NAME`: Order project v2 views by name + * `POSITION`: Order project v2 views by position + """ + + __schema__ = github_schema + __choices__ = ("CREATED_AT", "NAME", "POSITION") + + +class ProjectV2WorkflowsOrderField(sgqlc.types.Enum): + """Properties by which project workflows can be ordered. + + Enumeration Choices: + + * `CREATED_AT`: The workflows' date and time of creation + * `NAME`: The workflows' name + * `NUMBER`: The workflows' number + * `UPDATED_AT`: The workflows' date and time of update + """ + + __schema__ = github_schema + __choices__ = ("CREATED_AT", "NAME", "NUMBER", "UPDATED_AT") class PullRequestMergeMethod(sgqlc.types.Enum): @@ -1928,6 +2339,21 @@ class PullRequestReviewState(sgqlc.types.Enum): __choices__ = ("APPROVED", "CHANGES_REQUESTED", "COMMENTED", "DISMISSED", "PENDING") +class PullRequestReviewThreadSubjectType(sgqlc.types.Enum): + """The possible subject types of a pull request review comment. + + Enumeration Choices: + + * `FILE`: A comment that has been made against the file of a pull + request + * `LINE`: A comment that has been made against the line of a pull + request + """ + + __schema__ = github_schema + __choices__ = ("FILE", "LINE") + + class PullRequestState(sgqlc.types.Enum): """The possible states of a pull request. @@ -2440,10 +2866,12 @@ class RepositoryLockReason(sgqlc.types.Enum): * `MIGRATING`: The repository is locked due to a migration. * `MOVING`: The repository is locked due to a move. * `RENAME`: The repository is locked due to a rename. + * `TRADE_RESTRICTION`: The repository is locked due to a trade + controls related reason. """ __schema__ = github_schema - __choices__ = ("BILLING", "MIGRATING", "MOVING", "RENAME") + __choices__ = ("BILLING", "MIGRATING", "MOVING", "RENAME", "TRADE_RESTRICTION") class RepositoryMigrationOrderDirection(sgqlc.types.Enum): @@ -2526,6 +2954,73 @@ class RepositoryPrivacy(sgqlc.types.Enum): __choices__ = ("PRIVATE", "PUBLIC") +class RepositoryRuleType(sgqlc.types.Enum): + """The rule types supported in rulesets + + Enumeration Choices: + + * `BRANCH_NAME_PATTERN`: Branch name pattern + * `COMMITTER_EMAIL_PATTERN`: Committer email pattern + * `COMMIT_AUTHOR_EMAIL_PATTERN`: Commit author email pattern + * `COMMIT_MESSAGE_PATTERN`: Commit message pattern + * `CREATION`: Only allow users with bypass permission to create + matching refs. + * `DELETION`: Only allow users with bypass permissions to delete + matching refs. + * `NON_FAST_FORWARD`: Prevent users with push access from force + pushing to branches. + * `PULL_REQUEST`: Require all commits be made to a non-target + branch and submitted via a pull request before they can be + merged. + * `REQUIRED_DEPLOYMENTS`: Choose which environments must be + successfully deployed to before branches can be merged into a + branch that matches this rule. + * `REQUIRED_LINEAR_HISTORY`: Prevent merge commits from being + pushed to matching branches. + * `REQUIRED_SIGNATURES`: Commits pushed to matching branches must + have verified signatures. + * `REQUIRED_STATUS_CHECKS`: Choose which status checks must pass + before branches can be merged into a branch that matches this + rule. When enabled, commits must first be pushed to another + branch, then merged or pushed directly to a branch that matches + this rule after status checks have passed. + * `TAG_NAME_PATTERN`: Tag name pattern + * `UPDATE`: Only allow users with bypass permission to update + matching refs. + """ + + __schema__ = github_schema + __choices__ = ( + "BRANCH_NAME_PATTERN", + "COMMITTER_EMAIL_PATTERN", + "COMMIT_AUTHOR_EMAIL_PATTERN", + "COMMIT_MESSAGE_PATTERN", + "CREATION", + "DELETION", + "NON_FAST_FORWARD", + "PULL_REQUEST", + "REQUIRED_DEPLOYMENTS", + "REQUIRED_LINEAR_HISTORY", + "REQUIRED_SIGNATURES", + "REQUIRED_STATUS_CHECKS", + "TAG_NAME_PATTERN", + "UPDATE", + ) + + +class RepositoryRulesetTarget(sgqlc.types.Enum): + """The targets supported for rulesets + + Enumeration Choices: + + * `BRANCH`: Branch + * `TAG`: Tag + """ + + __schema__ = github_schema + __choices__ = ("BRANCH", "TAG") + + class RepositoryVisibility(sgqlc.types.Enum): """The repository's visibility level. @@ -2542,18 +3037,34 @@ class RepositoryVisibility(sgqlc.types.Enum): __choices__ = ("INTERNAL", "PRIVATE", "PUBLIC") +class RepositoryVulnerabilityAlertDependencyScope(sgqlc.types.Enum): + """The possible scopes of an alert's dependency. + + Enumeration Choices: + + * `DEVELOPMENT`: A dependency that is only used in development + * `RUNTIME`: A dependency that is leveraged during application + runtime + """ + + __schema__ = github_schema + __choices__ = ("DEVELOPMENT", "RUNTIME") + + class RepositoryVulnerabilityAlertState(sgqlc.types.Enum): """The possible states of an alert Enumeration Choices: + * `AUTO_DISMISSED`: An alert that has been automatically closed by + Dependabot. * `DISMISSED`: An alert that has been manually closed by a user. * `FIXED`: An alert that has been resolved by a code change. * `OPEN`: An alert that is still open. """ __schema__ = github_schema - __choices__ = ("DISMISSED", "FIXED", "OPEN") + __choices__ = ("AUTO_DISMISSED", "DISMISSED", "FIXED", "OPEN") class RequestableCheckStatusState(sgqlc.types.Enum): @@ -2590,6 +3101,43 @@ class RoleInOrganization(sgqlc.types.Enum): __choices__ = ("DIRECT_MEMBER", "OWNER", "UNAFFILIATED") +class RuleBypassMode(sgqlc.types.Enum): + """The bypass mode for a rule or ruleset. + + Enumeration Choices: + + * `NONE`: Bypassing is disabled + * `ORGANIZATION`: Those with bypass permission at the organization + level can bypass + * `ORGANIZATION_ALWAYS`: Those with bypass permission at the + organization level can always bypass + * `ORGANIZATION_NONE`: Bypassing is disabled + * `ORGANIZATION_PRS_ONLY`: Those with bypass permission at the + organization level can bypass for pull requests only + * `REPOSITORY`: Those with bypass permission at the repository + level can bypass + """ + + __schema__ = github_schema + __choices__ = ("NONE", "ORGANIZATION", "ORGANIZATION_ALWAYS", "ORGANIZATION_NONE", "ORGANIZATION_PRS_ONLY", "REPOSITORY") + + +class RuleEnforcement(sgqlc.types.Enum): + """The level of enforcement for a rule or ruleset. + + Enumeration Choices: + + * `ACTIVE`: Rules will be enforced + * `DISABLED`: Do not evaluate or enforce rules + * `EVALUATE`: Allow admins to test rules before enforcing them. + Admins can view insights on the Rule Insights page (`evaluate` + is only available with GitHub Enterprise). + """ + + __schema__ = github_schema + __choices__ = ("ACTIVE", "DISABLED", "EVALUATE") + + class SamlDigestAlgorithm(sgqlc.types.Enum): """The possible digest algorithms used to sign SAML requests for an identity provider. @@ -2668,18 +3216,22 @@ class SecurityAdvisoryEcosystem(sgqlc.types.Enum): Enumeration Choices: + * `ACTIONS`: GitHub Actions * `COMPOSER`: PHP packages hosted at packagist.org + * `ERLANG`: Erlang/Elixir packages hosted at hex.pm * `GO`: Go modules * `MAVEN`: Java artifacts hosted at the Maven central repository * `NPM`: JavaScript packages hosted at npmjs.com * `NUGET`: .NET packages hosted at the NuGet Gallery * `PIP`: Python packages hosted at PyPI.org + * `PUB`: Dart packages hosted at pub.dev * `RUBYGEMS`: Ruby gems hosted at RubyGems.org * `RUST`: Rust crates + * `SWIFT`: Swift packages """ __schema__ = github_schema - __choices__ = ("COMPOSER", "GO", "MAVEN", "NPM", "NUGET", "PIP", "RUBYGEMS", "RUST") + __choices__ = ("ACTIONS", "COMPOSER", "ERLANG", "GO", "MAVEN", "NPM", "NUGET", "PIP", "PUB", "RUBYGEMS", "RUST", "SWIFT") class SecurityAdvisoryIdentifierType(sgqlc.types.Enum): @@ -2736,6 +3288,30 @@ class SecurityVulnerabilityOrderField(sgqlc.types.Enum): __choices__ = ("UPDATED_AT",) +class SocialAccountProvider(sgqlc.types.Enum): + """Software or company that hosts social media accounts. + + Enumeration Choices: + + * `FACEBOOK`: Social media and networking website. + * `GENERIC`: Catch-all for social media providers that do not yet + have specific handling. + * `HOMETOWN`: Fork of Mastodon with a greater focus on local + posting. + * `INSTAGRAM`: Social media website with a focus on photo and + video sharing. + * `LINKEDIN`: Professional networking website. + * `MASTODON`: Open-source federated microblogging service. + * `REDDIT`: Social news aggregation and discussion website. + * `TWITCH`: Live-streaming service. + * `TWITTER`: Microblogging website. + * `YOUTUBE`: Online video platform. + """ + + __schema__ = github_schema + __choices__ = ("FACEBOOK", "GENERIC", "HOMETOWN", "INSTAGRAM", "LINKEDIN", "MASTODON", "REDDIT", "TWITCH", "TWITTER", "YOUTUBE") + + class SponsorOrderField(sgqlc.types.Enum): """Properties by which sponsor connections can be ordered. @@ -2814,6 +3390,511 @@ class SponsorsActivityPeriod(sgqlc.types.Enum): __choices__ = ("ALL", "DAY", "MONTH", "WEEK") +class SponsorsCountryOrRegionCode(sgqlc.types.Enum): + """Represents countries or regions for billing and residence for a + GitHub Sponsors profile. + + Enumeration Choices: + + * `AD`: Andorra + * `AE`: United Arab Emirates + * `AF`: Afghanistan + * `AG`: Antigua and Barbuda + * `AI`: Anguilla + * `AL`: Albania + * `AM`: Armenia + * `AO`: Angola + * `AQ`: Antarctica + * `AR`: Argentina + * `AS`: American Samoa + * `AT`: Austria + * `AU`: Australia + * `AW`: Aruba + * `AX`: Åland + * `AZ`: Azerbaijan + * `BA`: Bosnia and Herzegovina + * `BB`: Barbados + * `BD`: Bangladesh + * `BE`: Belgium + * `BF`: Burkina Faso + * `BG`: Bulgaria + * `BH`: Bahrain + * `BI`: Burundi + * `BJ`: Benin + * `BL`: Saint Barthélemy + * `BM`: Bermuda + * `BN`: Brunei Darussalam + * `BO`: Bolivia + * `BQ`: Bonaire, Sint Eustatius and Saba + * `BR`: Brazil + * `BS`: Bahamas + * `BT`: Bhutan + * `BV`: Bouvet Island + * `BW`: Botswana + * `BY`: Belarus + * `BZ`: Belize + * `CA`: Canada + * `CC`: Cocos (Keeling) Islands + * `CD`: Congo (Kinshasa) + * `CF`: Central African Republic + * `CG`: Congo (Brazzaville) + * `CH`: Switzerland + * `CI`: Côte d'Ivoire + * `CK`: Cook Islands + * `CL`: Chile + * `CM`: Cameroon + * `CN`: China + * `CO`: Colombia + * `CR`: Costa Rica + * `CV`: Cape Verde + * `CW`: Curaçao + * `CX`: Christmas Island + * `CY`: Cyprus + * `CZ`: Czech Republic + * `DE`: Germany + * `DJ`: Djibouti + * `DK`: Denmark + * `DM`: Dominica + * `DO`: Dominican Republic + * `DZ`: Algeria + * `EC`: Ecuador + * `EE`: Estonia + * `EG`: Egypt + * `EH`: Western Sahara + * `ER`: Eritrea + * `ES`: Spain + * `ET`: Ethiopia + * `FI`: Finland + * `FJ`: Fiji + * `FK`: Falkland Islands + * `FM`: Micronesia + * `FO`: Faroe Islands + * `FR`: France + * `GA`: Gabon + * `GB`: United Kingdom + * `GD`: Grenada + * `GE`: Georgia + * `GF`: French Guiana + * `GG`: Guernsey + * `GH`: Ghana + * `GI`: Gibraltar + * `GL`: Greenland + * `GM`: Gambia + * `GN`: Guinea + * `GP`: Guadeloupe + * `GQ`: Equatorial Guinea + * `GR`: Greece + * `GS`: South Georgia and South Sandwich Islands + * `GT`: Guatemala + * `GU`: Guam + * `GW`: Guinea-Bissau + * `GY`: Guyana + * `HK`: Hong Kong + * `HM`: Heard and McDonald Islands + * `HN`: Honduras + * `HR`: Croatia + * `HT`: Haiti + * `HU`: Hungary + * `ID`: Indonesia + * `IE`: Ireland + * `IL`: Israel + * `IM`: Isle of Man + * `IN`: India + * `IO`: British Indian Ocean Territory + * `IQ`: Iraq + * `IR`: Iran + * `IS`: Iceland + * `IT`: Italy + * `JE`: Jersey + * `JM`: Jamaica + * `JO`: Jordan + * `JP`: Japan + * `KE`: Kenya + * `KG`: Kyrgyzstan + * `KH`: Cambodia + * `KI`: Kiribati + * `KM`: Comoros + * `KN`: Saint Kitts and Nevis + * `KR`: Korea, South + * `KW`: Kuwait + * `KY`: Cayman Islands + * `KZ`: Kazakhstan + * `LA`: Laos + * `LB`: Lebanon + * `LC`: Saint Lucia + * `LI`: Liechtenstein + * `LK`: Sri Lanka + * `LR`: Liberia + * `LS`: Lesotho + * `LT`: Lithuania + * `LU`: Luxembourg + * `LV`: Latvia + * `LY`: Libya + * `MA`: Morocco + * `MC`: Monaco + * `MD`: Moldova + * `ME`: Montenegro + * `MF`: Saint Martin (French part) + * `MG`: Madagascar + * `MH`: Marshall Islands + * `MK`: Macedonia + * `ML`: Mali + * `MM`: Myanmar + * `MN`: Mongolia + * `MO`: Macau + * `MP`: Northern Mariana Islands + * `MQ`: Martinique + * `MR`: Mauritania + * `MS`: Montserrat + * `MT`: Malta + * `MU`: Mauritius + * `MV`: Maldives + * `MW`: Malawi + * `MX`: Mexico + * `MY`: Malaysia + * `MZ`: Mozambique + * `NA`: Namibia + * `NC`: New Caledonia + * `NE`: Niger + * `NF`: Norfolk Island + * `NG`: Nigeria + * `NI`: Nicaragua + * `NL`: Netherlands + * `NO`: Norway + * `NP`: Nepal + * `NR`: Nauru + * `NU`: Niue + * `NZ`: New Zealand + * `OM`: Oman + * `PA`: Panama + * `PE`: Peru + * `PF`: French Polynesia + * `PG`: Papua New Guinea + * `PH`: Philippines + * `PK`: Pakistan + * `PL`: Poland + * `PM`: Saint Pierre and Miquelon + * `PN`: Pitcairn + * `PR`: Puerto Rico + * `PS`: Palestine + * `PT`: Portugal + * `PW`: Palau + * `PY`: Paraguay + * `QA`: Qatar + * `RE`: Reunion + * `RO`: Romania + * `RS`: Serbia + * `RU`: Russian Federation + * `RW`: Rwanda + * `SA`: Saudi Arabia + * `SB`: Solomon Islands + * `SC`: Seychelles + * `SD`: Sudan + * `SE`: Sweden + * `SG`: Singapore + * `SH`: Saint Helena + * `SI`: Slovenia + * `SJ`: Svalbard and Jan Mayen Islands + * `SK`: Slovakia + * `SL`: Sierra Leone + * `SM`: San Marino + * `SN`: Senegal + * `SO`: Somalia + * `SR`: Suriname + * `SS`: South Sudan + * `ST`: Sao Tome and Principe + * `SV`: El Salvador + * `SX`: Sint Maarten (Dutch part) + * `SZ`: Swaziland + * `TC`: Turks and Caicos Islands + * `TD`: Chad + * `TF`: French Southern Lands + * `TG`: Togo + * `TH`: Thailand + * `TJ`: Tajikistan + * `TK`: Tokelau + * `TL`: Timor-Leste + * `TM`: Turkmenistan + * `TN`: Tunisia + * `TO`: Tonga + * `TR`: Turkey + * `TT`: Trinidad and Tobago + * `TV`: Tuvalu + * `TW`: Taiwan + * `TZ`: Tanzania + * `UA`: Ukraine + * `UG`: Uganda + * `UM`: United States Minor Outlying Islands + * `US`: United States of America + * `UY`: Uruguay + * `UZ`: Uzbekistan + * `VA`: Vatican City + * `VC`: Saint Vincent and the Grenadines + * `VE`: Venezuela + * `VG`: Virgin Islands, British + * `VI`: Virgin Islands, U.S. + * `VN`: Vietnam + * `VU`: Vanuatu + * `WF`: Wallis and Futuna Islands + * `WS`: Samoa + * `YE`: Yemen + * `YT`: Mayotte + * `ZA`: South Africa + * `ZM`: Zambia + * `ZW`: Zimbabwe + """ + + __schema__ = github_schema + __choices__ = ( + "AD", + "AE", + "AF", + "AG", + "AI", + "AL", + "AM", + "AO", + "AQ", + "AR", + "AS", + "AT", + "AU", + "AW", + "AX", + "AZ", + "BA", + "BB", + "BD", + "BE", + "BF", + "BG", + "BH", + "BI", + "BJ", + "BL", + "BM", + "BN", + "BO", + "BQ", + "BR", + "BS", + "BT", + "BV", + "BW", + "BY", + "BZ", + "CA", + "CC", + "CD", + "CF", + "CG", + "CH", + "CI", + "CK", + "CL", + "CM", + "CN", + "CO", + "CR", + "CV", + "CW", + "CX", + "CY", + "CZ", + "DE", + "DJ", + "DK", + "DM", + "DO", + "DZ", + "EC", + "EE", + "EG", + "EH", + "ER", + "ES", + "ET", + "FI", + "FJ", + "FK", + "FM", + "FO", + "FR", + "GA", + "GB", + "GD", + "GE", + "GF", + "GG", + "GH", + "GI", + "GL", + "GM", + "GN", + "GP", + "GQ", + "GR", + "GS", + "GT", + "GU", + "GW", + "GY", + "HK", + "HM", + "HN", + "HR", + "HT", + "HU", + "ID", + "IE", + "IL", + "IM", + "IN", + "IO", + "IQ", + "IR", + "IS", + "IT", + "JE", + "JM", + "JO", + "JP", + "KE", + "KG", + "KH", + "KI", + "KM", + "KN", + "KR", + "KW", + "KY", + "KZ", + "LA", + "LB", + "LC", + "LI", + "LK", + "LR", + "LS", + "LT", + "LU", + "LV", + "LY", + "MA", + "MC", + "MD", + "ME", + "MF", + "MG", + "MH", + "MK", + "ML", + "MM", + "MN", + "MO", + "MP", + "MQ", + "MR", + "MS", + "MT", + "MU", + "MV", + "MW", + "MX", + "MY", + "MZ", + "NA", + "NC", + "NE", + "NF", + "NG", + "NI", + "NL", + "NO", + "NP", + "NR", + "NU", + "NZ", + "OM", + "PA", + "PE", + "PF", + "PG", + "PH", + "PK", + "PL", + "PM", + "PN", + "PR", + "PS", + "PT", + "PW", + "PY", + "QA", + "RE", + "RO", + "RS", + "RU", + "RW", + "SA", + "SB", + "SC", + "SD", + "SE", + "SG", + "SH", + "SI", + "SJ", + "SK", + "SL", + "SM", + "SN", + "SO", + "SR", + "SS", + "ST", + "SV", + "SX", + "SZ", + "TC", + "TD", + "TF", + "TG", + "TH", + "TJ", + "TK", + "TL", + "TM", + "TN", + "TO", + "TR", + "TT", + "TV", + "TW", + "TZ", + "UA", + "UG", + "UM", + "US", + "UY", + "UZ", + "VA", + "VC", + "VE", + "VG", + "VI", + "VN", + "VU", + "WF", + "WS", + "YE", + "YT", + "ZA", + "ZM", + "ZW", + ) + + class SponsorsGoalKind(sgqlc.types.Enum): """The different kinds of goals a GitHub Sponsors member can have. @@ -2829,6 +3910,22 @@ class SponsorsGoalKind(sgqlc.types.Enum): __choices__ = ("MONTHLY_SPONSORSHIP_AMOUNT", "TOTAL_SPONSORS_COUNT") +class SponsorsListingFeaturedItemFeatureableType(sgqlc.types.Enum): + """The different kinds of records that can be featured on a GitHub + Sponsors profile page. + + Enumeration Choices: + + * `REPOSITORY`: A repository owned by the user or organization + with the GitHub Sponsors profile. + * `USER`: A user who belongs to the organization with the GitHub + Sponsors profile. + """ + + __schema__ = github_schema + __choices__ = ("REPOSITORY", "USER") + + class SponsorsTierOrderField(sgqlc.types.Enum): """Properties by which Sponsors tiers connections can be ordered. @@ -2881,6 +3978,34 @@ class SponsorshipPrivacy(sgqlc.types.Enum): __choices__ = ("PRIVATE", "PUBLIC") +class SquashMergeCommitMessage(sgqlc.types.Enum): + """The possible default commit messages for squash merges. + + Enumeration Choices: + + * `BLANK`: Default to a blank commit message. + * `COMMIT_MESSAGES`: Default to the branch's commit messages. + * `PR_BODY`: Default to the pull request's body. + """ + + __schema__ = github_schema + __choices__ = ("BLANK", "COMMIT_MESSAGES", "PR_BODY") + + +class SquashMergeCommitTitle(sgqlc.types.Enum): + """The possible default commit titles for squash merges. + + Enumeration Choices: + + * `COMMIT_OR_PR_TITLE`: Default to the commit's title (if only one + commit) or the pull request's title (when more than one commit). + * `PR_TITLE`: Default to the pull request's title. + """ + + __schema__ = github_schema + __choices__ = ("COMMIT_OR_PR_TITLE", "PR_TITLE") + + class StarOrderField(sgqlc.types.Enum): """Properties by which star connections can be ordered. @@ -2997,6 +4122,20 @@ class TeamMembershipType(sgqlc.types.Enum): __choices__ = ("ALL", "CHILD_TEAM", "IMMEDIATE") +class TeamNotificationSetting(sgqlc.types.Enum): + """The possible team notification values. + + Enumeration Choices: + + * `NOTIFICATIONS_DISABLED`: No one will receive notifications. + * `NOTIFICATIONS_ENABLED`: Everyone will receive notifications + when the team is @mentioned. + """ + + __schema__ = github_schema + __choices__ = ("NOTIFICATIONS_DISABLED", "NOTIFICATIONS_ENABLED") + + class TeamOrderField(sgqlc.types.Enum): """Properties by which team connections can be ordered. @@ -3134,6 +4273,35 @@ class VerifiableDomainOrderField(sgqlc.types.Enum): __choices__ = ("CREATED_AT", "DOMAIN") +class WorkflowRunOrderField(sgqlc.types.Enum): + """Properties by which workflow run connections can be ordered. + + Enumeration Choices: + + * `CREATED_AT`: Order workflow runs by most recently created + """ + + __schema__ = github_schema + __choices__ = ("CREATED_AT",) + + +class WorkflowState(sgqlc.types.Enum): + """The possible states for a workflow. + + Enumeration Choices: + + * `ACTIVE`: The workflow is active. + * `DELETED`: The workflow was deleted from the git repository. + * `DISABLED_FORK`: The workflow was disabled by default on a fork. + * `DISABLED_INACTIVITY`: The workflow was disabled for inactivity + in the repository. + * `DISABLED_MANUALLY`: The workflow was disabled manually. + """ + + __schema__ = github_schema + __choices__ = ("ACTIVE", "DELETED", "DISABLED_FORK", "DISABLED_INACTIVITY", "DISABLED_MANUALLY") + + class X509Certificate(sgqlc.types.Scalar): """A valid x509 certificate string""" @@ -3246,6 +4414,27 @@ class AddDiscussionPollVoteInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" +class AddEnterpriseOrganizationMemberInput(sgqlc.types.Input): + """Autogenerated input type of AddEnterpriseOrganizationMember""" + + __schema__ = github_schema + __field_names__ = ("enterprise_id", "organization_id", "user_ids", "role", "client_mutation_id") + enterprise_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="enterpriseId") + """The ID of the enterprise which owns the organization.""" + + organization_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="organizationId") + """The ID of the organization the users will be added to.""" + + user_ids = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="userIds") + """The IDs of the enterprise members to add.""" + + role = sgqlc.types.Field(OrganizationMemberRole, graphql_name="role") + """The role to assign the users in the organization""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class AddEnterpriseSupportEntitlementInput(sgqlc.types.Input): """Autogenerated input type of AddEnterpriseSupportEntitlement""" @@ -3311,8 +4500,8 @@ class AddProjectColumnInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" -class AddProjectDraftIssueInput(sgqlc.types.Input): - """Autogenerated input type of AddProjectDraftIssue""" +class AddProjectV2DraftIssueInput(sgqlc.types.Input): + """Autogenerated input type of AddProjectV2DraftIssue""" __schema__ = github_schema __field_names__ = ("project_id", "title", "body", "assignee_ids", "client_mutation_id") @@ -3320,7 +4509,10 @@ class AddProjectDraftIssueInput(sgqlc.types.Input): """The ID of the Project to add the draft issue to.""" title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - """The title of the draft issue.""" + """The title of the draft issue. A project item can also be created + by providing the URL of an Issue or Pull Request if you have + access. + """ body = sgqlc.types.Field(String, graphql_name="body") """The body of the draft issue.""" @@ -3332,8 +4524,8 @@ class AddProjectDraftIssueInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" -class AddProjectNextItemInput(sgqlc.types.Input): - """Autogenerated input type of AddProjectNextItem""" +class AddProjectV2ItemByIdInput(sgqlc.types.Input): + """Autogenerated input type of AddProjectV2ItemById""" __schema__ = github_schema __field_names__ = ("project_id", "content_id", "client_mutation_id") @@ -3341,7 +4533,7 @@ class AddProjectNextItemInput(sgqlc.types.Input): """The ID of the Project to add the item to.""" content_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="contentId") - """The content id of the item (Issue or PullRequest).""" + """The id of the Issue or Pull Request to add.""" client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" @@ -3362,25 +4554,60 @@ class AddPullRequestReviewCommentInput(sgqlc.types.Input): "client_mutation_id", ) pull_request_id = sgqlc.types.Field(ID, graphql_name="pullRequestId") - """The node ID of the pull request reviewing""" + """The node ID of the pull request reviewing **Upcoming Change on + 2023-10-01 UTC** **Description:** `pullRequestId` will be removed. + use addPullRequestReviewThread or addPullRequestReviewThreadReply + instead **Reason:** We are deprecating the + addPullRequestReviewComment mutation + """ pull_request_review_id = sgqlc.types.Field(ID, graphql_name="pullRequestReviewId") - """The Node ID of the review to modify.""" + """The Node ID of the review to modify. **Upcoming Change on + 2023-10-01 UTC** **Description:** `pullRequestReviewId` will be + removed. use addPullRequestReviewThread or + addPullRequestReviewThreadReply instead **Reason:** We are + deprecating the addPullRequestReviewComment mutation + """ commit_oid = sgqlc.types.Field(GitObjectID, graphql_name="commitOID") - """The SHA of the commit to comment on.""" + """The SHA of the commit to comment on. **Upcoming Change on + 2023-10-01 UTC** **Description:** `commitOID` will be removed. use + addPullRequestReviewThread or addPullRequestReviewThreadReply + instead **Reason:** We are deprecating the + addPullRequestReviewComment mutation + """ - body = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="body") - """The text of the comment.""" + body = sgqlc.types.Field(String, graphql_name="body") + """The text of the comment. This field is required **Upcoming Change + on 2023-10-01 UTC** **Description:** `body` will be removed. use + addPullRequestReviewThread or addPullRequestReviewThreadReply + instead **Reason:** We are deprecating the + addPullRequestReviewComment mutation + """ path = sgqlc.types.Field(String, graphql_name="path") - """The relative path of the file to comment on.""" + """The relative path of the file to comment on. **Upcoming Change on + 2023-10-01 UTC** **Description:** `path` will be removed. use + addPullRequestReviewThread or addPullRequestReviewThreadReply + instead **Reason:** We are deprecating the + addPullRequestReviewComment mutation + """ position = sgqlc.types.Field(Int, graphql_name="position") - """The line index in the diff to comment on.""" + """The line index in the diff to comment on. **Upcoming Change on + 2023-10-01 UTC** **Description:** `position` will be removed. use + addPullRequestReviewThread or addPullRequestReviewThreadReply + instead **Reason:** We are deprecating the + addPullRequestReviewComment mutation + """ in_reply_to = sgqlc.types.Field(ID, graphql_name="inReplyTo") - """The comment id to reply to.""" + """The comment id to reply to. **Upcoming Change on 2023-10-01 UTC** + **Description:** `inReplyTo` will be removed. use + addPullRequestReviewThread or addPullRequestReviewThreadReply + instead **Reason:** We are deprecating the + addPullRequestReviewComment mutation + """ client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" @@ -3404,7 +4631,11 @@ class AddPullRequestReviewInput(sgqlc.types.Input): """The event to perform on the pull request review.""" comments = sgqlc.types.Field(sgqlc.types.list_of("DraftPullRequestReviewComment"), graphql_name="comments") - """The review line comments.""" + """The review line comments. **Upcoming Change on 2023-10-01 UTC** + **Description:** `comments` will be removed. use the `threads` + argument instead **Reason:** We are deprecating comment fields + that use diff-relative positioning + """ threads = sgqlc.types.Field(sgqlc.types.list_of("DraftPullRequestReviewThread"), graphql_name="threads") """The review line comment threads.""" @@ -3426,6 +4657,7 @@ class AddPullRequestReviewThreadInput(sgqlc.types.Input): "side", "start_line", "start_side", + "subject_type", "client_mutation_id", ) path = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="path") @@ -3440,9 +4672,10 @@ class AddPullRequestReviewThreadInput(sgqlc.types.Input): pull_request_review_id = sgqlc.types.Field(ID, graphql_name="pullRequestReviewId") """The Node ID of the review to modify.""" - line = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="line") - """The line of the blob to which the thread refers. The end of the - line range for multi-line comments. + line = sgqlc.types.Field(Int, graphql_name="line") + """The line of the blob to which the thread refers, required for + line-level threads. The end of the line range for multi-line + comments. """ side = sgqlc.types.Field(DiffSide, graphql_name="side") @@ -3456,6 +4689,11 @@ class AddPullRequestReviewThreadInput(sgqlc.types.Input): start_side = sgqlc.types.Field(DiffSide, graphql_name="startSide") """The side of the diff on which the start line resides.""" + subject_type = sgqlc.types.Field(PullRequestReviewThreadSubjectType, graphql_name="subjectType") + """The level at which the comments in the corresponding thread are + targeted, can be a diff line or a file + """ + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" @@ -3546,6 +4784,21 @@ class ApproveVerifiableDomainInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" +class ArchiveProjectV2ItemInput(sgqlc.types.Input): + """Autogenerated input type of ArchiveProjectV2Item""" + + __schema__ = github_schema + __field_names__ = ("project_id", "item_id", "client_mutation_id") + project_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="projectId") + """The ID of the Project to archive the item from.""" + + item_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="itemId") + """The ID of the ProjectV2Item to archive.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class ArchiveRepositoryInput(sgqlc.types.Input): """Autogenerated input type of ArchiveRepository""" @@ -3570,6 +4823,48 @@ class AuditLogOrder(sgqlc.types.Input): """The ordering direction.""" +class BranchNamePatternParametersInput(sgqlc.types.Input): + """Parameters to be used for the branch_name_pattern rule""" + + __schema__ = github_schema + __field_names__ = ("name", "negate", "operator", "pattern") + name = sgqlc.types.Field(String, graphql_name="name") + """How this rule will appear to users.""" + + negate = sgqlc.types.Field(Boolean, graphql_name="negate") + """If true, the rule will fail if the pattern matches.""" + + operator = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="operator") + """The operator to use for matching.""" + + pattern = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="pattern") + """The pattern to match with.""" + + +class BulkSponsorship(sgqlc.types.Input): + """Information about a sponsorship to make for a user or organization + with a GitHub Sponsors profile, as part of sponsoring many users + or organizations at once. + """ + + __schema__ = github_schema + __field_names__ = ("sponsorable_id", "sponsorable_login", "amount") + sponsorable_id = sgqlc.types.Field(ID, graphql_name="sponsorableId") + """The ID of the user or organization who is receiving the + sponsorship. Required if sponsorableLogin is not given. + """ + + sponsorable_login = sgqlc.types.Field(String, graphql_name="sponsorableLogin") + """The username of the user or organization who is receiving the + sponsorship. Required if sponsorableId is not given. + """ + + amount = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="amount") + """The amount to pay to the sponsorable in US dollars. Valid values: + 1-12000. + """ + + class CancelEnterpriseAdminInvitationInput(sgqlc.types.Input): """Autogenerated input type of CancelEnterpriseAdminInvitation""" @@ -3703,7 +4998,7 @@ class CheckRunFilter(sgqlc.types.Input): """The filters that are available when fetching check runs.""" __schema__ = github_schema - __field_names__ = ("check_type", "app_id", "check_name", "status") + __field_names__ = ("check_type", "app_id", "check_name", "status", "statuses", "conclusions") check_type = sgqlc.types.Field(CheckRunType, graphql_name="checkType") """Filters the check runs by this type.""" @@ -3714,7 +5009,13 @@ class CheckRunFilter(sgqlc.types.Input): """Filters the check runs by this name.""" status = sgqlc.types.Field(CheckStatusState, graphql_name="status") - """Filters the check runs by this status.""" + """Filters the check runs by this status. Superceded by statuses.""" + + statuses = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(CheckStatusState)), graphql_name="statuses") + """Filters the check runs by this status. Overrides status.""" + + conclusions = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(CheckConclusionState)), graphql_name="conclusions") + """Filters the check runs by these conclusions.""" class CheckRunOutput(sgqlc.types.Input): @@ -3795,6 +5096,24 @@ class ClearLabelsFromLabelableInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" +class ClearProjectV2ItemFieldValueInput(sgqlc.types.Input): + """Autogenerated input type of ClearProjectV2ItemFieldValue""" + + __schema__ = github_schema + __field_names__ = ("project_id", "item_id", "field_id", "client_mutation_id") + project_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="projectId") + """The ID of the Project.""" + + item_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="itemId") + """The ID of the item to be cleared.""" + + field_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="fieldId") + """The ID of the field to be cleared.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class CloneProjectInput(sgqlc.types.Input): """Autogenerated input type of CloneProject""" @@ -3852,6 +5171,21 @@ class CloneTemplateRepositoryInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" +class CloseDiscussionInput(sgqlc.types.Input): + """Autogenerated input type of CloseDiscussion""" + + __schema__ = github_schema + __field_names__ = ("discussion_id", "reason", "client_mutation_id") + discussion_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="discussionId") + """ID of the discussion to be closed.""" + + reason = sgqlc.types.Field(DiscussionCloseReason, graphql_name="reason") + """The reason why the discussion is being closed.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class CloseIssueInput(sgqlc.types.Input): """Autogenerated input type of CloseIssue""" @@ -3896,6 +5230,24 @@ class CommitAuthor(sgqlc.types.Input): """ +class CommitAuthorEmailPatternParametersInput(sgqlc.types.Input): + """Parameters to be used for the commit_author_email_pattern rule""" + + __schema__ = github_schema + __field_names__ = ("name", "negate", "operator", "pattern") + name = sgqlc.types.Field(String, graphql_name="name") + """How this rule will appear to users.""" + + negate = sgqlc.types.Field(Boolean, graphql_name="negate") + """If true, the rule will fail if the pattern matches.""" + + operator = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="operator") + """The operator to use for matching.""" + + pattern = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="pattern") + """The pattern to match with.""" + + class CommitContributionOrder(sgqlc.types.Input): """Ordering options for commit contribution connections.""" @@ -3920,16 +5272,35 @@ class CommitMessage(sgqlc.types.Input): """The body of the message.""" +class CommitMessagePatternParametersInput(sgqlc.types.Input): + """Parameters to be used for the commit_message_pattern rule""" + + __schema__ = github_schema + __field_names__ = ("name", "negate", "operator", "pattern") + name = sgqlc.types.Field(String, graphql_name="name") + """How this rule will appear to users.""" + + negate = sgqlc.types.Field(Boolean, graphql_name="negate") + """If true, the rule will fail if the pattern matches.""" + + operator = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="operator") + """The operator to use for matching.""" + + pattern = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="pattern") + """The pattern to match with.""" + + class CommittableBranch(sgqlc.types.Input): """A git ref for a commit to be appended to. The ref must be a branch, i.e. its fully qualified name must start with `refs/heads/` (although the input is not required to be fully qualified). The Ref may be specified by its global node ID or by - the repository nameWithOwner and branch name. ### Examples + the `repositoryNameWithOwner` and `branchName`. ### Examples Specify a branch using a global node ID: { "id": "MDM6UmVmMTpyZWZzL2hlYWRzL21haW4=" } Specify a branch using - nameWithOwner and branch name: { "nameWithOwner": - "github/graphql-client", "branchName": "main" } + `repositoryNameWithOwner` and `branchName`: { + "repositoryNameWithOwner": "github/graphql-client", + "branchName": "main" } """ __schema__ = github_schema @@ -3944,6 +5315,24 @@ class CommittableBranch(sgqlc.types.Input): """The unqualified name of the branch to append the commit to.""" +class CommitterEmailPatternParametersInput(sgqlc.types.Input): + """Parameters to be used for the committer_email_pattern rule""" + + __schema__ = github_schema + __field_names__ = ("name", "negate", "operator", "pattern") + name = sgqlc.types.Field(String, graphql_name="name") + """How this rule will appear to users.""" + + negate = sgqlc.types.Field(Boolean, graphql_name="negate") + """If true, the rule will fail if the pattern matches.""" + + operator = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="operator") + """The operator to use for matching.""" + + pattern = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="pattern") + """The pattern to match with.""" + + class ContributionOrder(sgqlc.types.Input): """Ordering options for contribution connections.""" @@ -3988,6 +5377,45 @@ class ConvertPullRequestToDraftInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" +class CopyProjectV2Input(sgqlc.types.Input): + """Autogenerated input type of CopyProjectV2""" + + __schema__ = github_schema + __field_names__ = ("project_id", "owner_id", "title", "include_draft_issues", "client_mutation_id") + project_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="projectId") + """The ID of the source Project to copy.""" + + owner_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="ownerId") + """The owner ID of the new project.""" + + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + """The title of the project.""" + + include_draft_issues = sgqlc.types.Field(Boolean, graphql_name="includeDraftIssues") + """Include draft issues in the new project""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + +class CreateAttributionInvitationInput(sgqlc.types.Input): + """Autogenerated input type of CreateAttributionInvitation""" + + __schema__ = github_schema + __field_names__ = ("owner_id", "source_id", "target_id", "client_mutation_id") + owner_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="ownerId") + """The Node ID of the owner scoping the reattributable data.""" + + source_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="sourceId") + """The Node ID of the account owning the data to reattribute.""" + + target_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="targetId") + """The Node ID of the account which may claim the data.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class CreateBranchProtectionRuleInput(sgqlc.types.Input): """Autogenerated input type of CreateBranchProtectionRule""" @@ -4015,7 +5443,12 @@ class CreateBranchProtectionRuleInput(sgqlc.types.Input): "push_actor_ids", "required_status_check_contexts", "required_status_checks", + "requires_deployments", + "required_deployment_environments", "requires_conversation_resolution", + "require_last_push_approval", + "lock_branch", + "lock_allows_fetch_and_merge", "client_mutation_id", ) repository_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="repositoryId") @@ -4104,9 +5537,33 @@ class CreateBranchProtectionRuleInput(sgqlc.types.Input): ) """The list of required status checks""" + requires_deployments = sgqlc.types.Field(Boolean, graphql_name="requiresDeployments") + """Are successful deployments required before merging.""" + + required_deployment_environments = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="requiredDeploymentEnvironments" + ) + """The list of required deployment environments""" + requires_conversation_resolution = sgqlc.types.Field(Boolean, graphql_name="requiresConversationResolution") """Are conversations required to be resolved before merging.""" + require_last_push_approval = sgqlc.types.Field(Boolean, graphql_name="requireLastPushApproval") + """Whether the most recent push must be approved by someone other + than the person who pushed it + """ + + lock_branch = sgqlc.types.Field(Boolean, graphql_name="lockBranch") + """Whether to set the branch as read-only. If this is true, users + will not be able to push to the branch. + """ + + lock_allows_fetch_and_merge = sgqlc.types.Field(Boolean, graphql_name="lockAllowsFetchAndMerge") + """Whether users can pull changes from upstream when the branch is + locked. Set to `true` to allow fork syncing. Set to `false` to + prevent fork syncing. + """ + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" @@ -4340,27 +5797,50 @@ class CreateIssueInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" +class CreateLinkedBranchInput(sgqlc.types.Input): + """Autogenerated input type of CreateLinkedBranch""" + + __schema__ = github_schema + __field_names__ = ("issue_id", "oid", "name", "repository_id", "client_mutation_id") + issue_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="issueId") + """ID of the issue to link to.""" + + oid = sgqlc.types.Field(sgqlc.types.non_null(GitObjectID), graphql_name="oid") + """The commit SHA to base the new branch on.""" + + name = sgqlc.types.Field(String, graphql_name="name") + """The name of the new branch. Defaults to issue number and title.""" + + repository_id = sgqlc.types.Field(ID, graphql_name="repositoryId") + """ID of the repository to create the branch in. Defaults to the + issue repository. + """ + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class CreateMigrationSourceInput(sgqlc.types.Input): """Autogenerated input type of CreateMigrationSource""" __schema__ = github_schema __field_names__ = ("name", "url", "access_token", "type", "owner_id", "github_pat", "client_mutation_id") name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - """The Octoshift migration source name.""" + """The migration source name.""" - url = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="url") - """The Octoshift migration source URL.""" + url = sgqlc.types.Field(String, graphql_name="url") + """The migration source URL, for example `https://github.com` or + `https://monalisa.ghe.com`. + """ access_token = sgqlc.types.Field(String, graphql_name="accessToken") - """The Octoshift migration source access token.""" + """The migration source access token.""" type = sgqlc.types.Field(sgqlc.types.non_null(MigrationSourceType), graphql_name="type") - """The Octoshift migration source type.""" + """The migration source type.""" owner_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="ownerId") - """The ID of the organization that will own the Octoshift migration - source. - """ + """The ID of the organization that will own the migration source.""" github_pat = sgqlc.types.Field(String, graphql_name="githubPat") """The GitHub personal access token of the user importing to the @@ -4397,6 +5877,54 @@ class CreateProjectInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" +class CreateProjectV2FieldInput(sgqlc.types.Input): + """Autogenerated input type of CreateProjectV2Field""" + + __schema__ = github_schema + __field_names__ = ("project_id", "data_type", "name", "single_select_options", "client_mutation_id") + project_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="projectId") + """The ID of the Project to create the field in.""" + + data_type = sgqlc.types.Field(sgqlc.types.non_null(ProjectV2CustomFieldType), graphql_name="dataType") + """The data type of the field.""" + + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + """The name of the field.""" + + single_select_options = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("ProjectV2SingleSelectFieldOptionInput")), graphql_name="singleSelectOptions" + ) + """Options for a single select field. At least one value is required + if data_type is SINGLE_SELECT + """ + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + +class CreateProjectV2Input(sgqlc.types.Input): + """Autogenerated input type of CreateProjectV2""" + + __schema__ = github_schema + __field_names__ = ("owner_id", "title", "repository_id", "team_id", "client_mutation_id") + owner_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="ownerId") + """The owner ID to create the project under.""" + + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + """The title of the project.""" + + repository_id = sgqlc.types.Field(ID, graphql_name="repositoryId") + """The repository to link the project to.""" + + team_id = sgqlc.types.Field(ID, graphql_name="teamId") + """The team to link the project to. The team will be granted read + permissions. + """ + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class CreatePullRequestInput(sgqlc.types.Input): """Autogenerated input type of CreatePullRequest""" @@ -4405,6 +5933,7 @@ class CreatePullRequestInput(sgqlc.types.Input): "repository_id", "base_ref_name", "head_ref_name", + "head_repository_id", "title", "body", "maintainer_can_modify", @@ -4427,6 +5956,9 @@ class CreatePullRequestInput(sgqlc.types.Input): `head_ref_name` with a user like this: `username:branch`. """ + head_repository_id = sgqlc.types.Field(ID, graphql_name="headRepositoryId") + """The Node ID of the head repository.""" + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") """The title of the pull request.""" @@ -4519,6 +6051,120 @@ class CreateRepositoryInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" +class CreateRepositoryRulesetInput(sgqlc.types.Input): + """Autogenerated input type of CreateRepositoryRuleset""" + + __schema__ = github_schema + __field_names__ = ( + "source_id", + "name", + "target", + "rules", + "conditions", + "enforcement", + "bypass_mode", + "bypass_actor_ids", + "client_mutation_id", + ) + source_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="sourceId") + """The global relay id of the source in which a new ruleset should be + created in. + """ + + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + """The name of the ruleset.""" + + target = sgqlc.types.Field(RepositoryRulesetTarget, graphql_name="target") + """The target of the ruleset.""" + + rules = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("RepositoryRuleInput")), graphql_name="rules") + """The list of rules for this ruleset""" + + conditions = sgqlc.types.Field(sgqlc.types.non_null("RepositoryRuleConditionsInput"), graphql_name="conditions") + """The set of conditions for this ruleset""" + + enforcement = sgqlc.types.Field(sgqlc.types.non_null(RuleEnforcement), graphql_name="enforcement") + """The enforcement level for this ruleset""" + + bypass_mode = sgqlc.types.Field(RuleBypassMode, graphql_name="bypassMode") + """The bypass mode for this ruleset""" + + bypass_actor_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="bypassActorIds") + """A list of Team or App IDs allowed to bypass rules in this ruleset.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + +class CreateSponsorsListingInput(sgqlc.types.Input): + """Autogenerated input type of CreateSponsorsListing""" + + __schema__ = github_schema + __field_names__ = ( + "sponsorable_login", + "fiscal_host_login", + "fiscally_hosted_project_profile_url", + "billing_country_or_region_code", + "residence_country_or_region_code", + "contact_email", + "full_description", + "client_mutation_id", + ) + sponsorable_login = sgqlc.types.Field(String, graphql_name="sponsorableLogin") + """The username of the organization to create a GitHub Sponsors + profile for, if desired. Defaults to creating a GitHub Sponsors + profile for the authenticated user if omitted. + """ + + fiscal_host_login = sgqlc.types.Field(String, graphql_name="fiscalHostLogin") + """The username of the supported fiscal host's GitHub organization, + if you want to receive sponsorship payouts through a fiscal host + rather than directly to a bank account. For example, 'Open-Source- + Collective' for Open Source Collective or 'numfocus' for numFOCUS. + Case insensitive. See https://docs.github.com/sponsors/receiving- + sponsorships-through-github-sponsors/using-a-fiscal-host-to- + receive-github-sponsors-payouts for more information. + """ + + fiscally_hosted_project_profile_url = sgqlc.types.Field(String, graphql_name="fiscallyHostedProjectProfileUrl") + """The URL for your profile page on the fiscal host's website, e.g., + https://opencollective.com/babel or + https://numfocus.org/project/bokeh. Required if fiscalHostLogin is + specified. + """ + + billing_country_or_region_code = sgqlc.types.Field(SponsorsCountryOrRegionCode, graphql_name="billingCountryOrRegionCode") + """The country or region where the sponsorable's bank account is + located. Required if fiscalHostLogin is not specified, ignored + when fiscalHostLogin is specified. + """ + + residence_country_or_region_code = sgqlc.types.Field(SponsorsCountryOrRegionCode, graphql_name="residenceCountryOrRegionCode") + """The country or region where the sponsorable resides. This is for + tax purposes. Required if the sponsorable is yourself, ignored + when sponsorableLogin specifies an organization. + """ + + contact_email = sgqlc.types.Field(String, graphql_name="contactEmail") + """The email address we should use to contact you about the GitHub + Sponsors profile being created. This will not be shared publicly. + Must be a verified email address already on your GitHub account. + Only relevant when the sponsorable is yourself. Defaults to your + primary email address on file if omitted. + """ + + full_description = sgqlc.types.Field(String, graphql_name="fullDescription") + """Provide an introduction to serve as the main focus that appears on + your GitHub Sponsors profile. It's a great opportunity to help + potential sponsors learn more about you, your work, and why their + sponsorship is important to you. GitHub-flavored Markdown is + supported. + """ + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class CreateSponsorsTierInput(sgqlc.types.Input): """Autogenerated input type of CreateSponsorsTier""" @@ -4660,6 +6306,36 @@ class CreateSponsorshipInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" +class CreateSponsorshipsInput(sgqlc.types.Input): + """Autogenerated input type of CreateSponsorships""" + + __schema__ = github_schema + __field_names__ = ("sponsor_login", "sponsorships", "receive_emails", "privacy_level", "client_mutation_id") + sponsor_login = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="sponsorLogin") + """The username of the user or organization who is acting as the + sponsor, paying for the sponsorships. + """ + + sponsorships = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(BulkSponsorship))), graphql_name="sponsorships" + ) + """The list of maintainers to sponsor and for how much apiece.""" + + receive_emails = sgqlc.types.Field(Boolean, graphql_name="receiveEmails") + """Whether the sponsor should receive email updates from the + sponsorables. + """ + + privacy_level = sgqlc.types.Field(SponsorshipPrivacy, graphql_name="privacyLevel") + """Specify whether others should be able to see that the sponsor is + sponsoring the sponsorables. Public visibility still does not + reveal the dollar value of the sponsorship. + """ + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class CreateTeamDiscussionCommentInput(sgqlc.types.Input): """Autogenerated input type of CreateTeamDiscussionComment""" @@ -4813,6 +6489,18 @@ class DeleteIssueInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" +class DeleteLinkedBranchInput(sgqlc.types.Input): + """Autogenerated input type of DeleteLinkedBranch""" + + __schema__ = github_schema + __field_names__ = ("linked_branch_id", "client_mutation_id") + linked_branch_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="linkedBranchId") + """The ID of the linked branch""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class DeleteProjectCardInput(sgqlc.types.Input): """Autogenerated input type of DeleteProjectCard""" @@ -4849,8 +6537,32 @@ class DeleteProjectInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" -class DeleteProjectNextItemInput(sgqlc.types.Input): - """Autogenerated input type of DeleteProjectNextItem""" +class DeleteProjectV2FieldInput(sgqlc.types.Input): + """Autogenerated input type of DeleteProjectV2Field""" + + __schema__ = github_schema + __field_names__ = ("field_id", "client_mutation_id") + field_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="fieldId") + """The ID of the field to delete.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + +class DeleteProjectV2Input(sgqlc.types.Input): + """Autogenerated input type of DeleteProjectV2""" + + __schema__ = github_schema + __field_names__ = ("project_id", "client_mutation_id") + project_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="projectId") + """The ID of the Project to delete.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + +class DeleteProjectV2ItemInput(sgqlc.types.Input): + """Autogenerated input type of DeleteProjectV2Item""" __schema__ = github_schema __field_names__ = ("project_id", "item_id", "client_mutation_id") @@ -4864,6 +6576,18 @@ class DeleteProjectNextItemInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" +class DeleteProjectV2WorkflowInput(sgqlc.types.Input): + """Autogenerated input type of DeleteProjectV2Workflow""" + + __schema__ = github_schema + __field_names__ = ("workflow_id", "client_mutation_id") + workflow_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="workflowId") + """The ID of the workflow to be removed.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class DeletePullRequestReviewCommentInput(sgqlc.types.Input): """Autogenerated input type of DeletePullRequestReviewComment""" @@ -4900,6 +6624,18 @@ class DeleteRefInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" +class DeleteRepositoryRulesetInput(sgqlc.types.Input): + """Autogenerated input type of DeleteRepositoryRuleset""" + + __schema__ = github_schema + __field_names__ = ("repository_ruleset_id", "client_mutation_id") + repository_ruleset_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="repositoryRulesetId") + """The global relay id of the repository ruleset to be deleted.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class DeleteTeamDiscussionCommentInput(sgqlc.types.Input): """Autogenerated input type of DeleteTeamDiscussionComment""" @@ -4948,6 +6684,18 @@ class DeploymentOrder(sgqlc.types.Input): """The ordering direction.""" +class DequeuePullRequestInput(sgqlc.types.Input): + """Autogenerated input type of DequeuePullRequest""" + + __schema__ = github_schema + __field_names__ = ("id", "client_mutation_id") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + """The ID of the pull request to be dequeued.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class DisablePullRequestAutoMergeInput(sgqlc.types.Input): """Autogenerated input type of DisablePullRequestAutoMerge""" @@ -5065,26 +6813,60 @@ class EnablePullRequestAutoMergeInput(sgqlc.types.Input): """Autogenerated input type of EnablePullRequestAutoMerge""" __schema__ = github_schema - __field_names__ = ("pull_request_id", "commit_headline", "commit_body", "merge_method", "author_email", "client_mutation_id") + __field_names__ = ( + "pull_request_id", + "commit_headline", + "commit_body", + "merge_method", + "author_email", + "expected_head_oid", + "client_mutation_id", + ) pull_request_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="pullRequestId") """ID of the pull request to enable auto-merge on.""" commit_headline = sgqlc.types.Field(String, graphql_name="commitHeadline") """Commit headline to use for the commit when the PR is mergable; if - omitted, a default message will be used. + omitted, a default message will be used. NOTE: when merging with a + merge queue any input value for commit headline is ignored. """ commit_body = sgqlc.types.Field(String, graphql_name="commitBody") """Commit body to use for the commit when the PR is mergable; if - omitted, a default message will be used. + omitted, a default message will be used. NOTE: when merging with a + merge queue any input value for commit message is ignored. """ merge_method = sgqlc.types.Field(PullRequestMergeMethod, graphql_name="mergeMethod") - """The merge method to use. If omitted, defaults to 'MERGE' """ + """The merge method to use. If omitted, defaults to `MERGE`. NOTE: + when merging with a merge queue any input value for merge method + is ignored. + """ author_email = sgqlc.types.Field(String, graphql_name="authorEmail") """The email address to associate with this merge.""" + expected_head_oid = sgqlc.types.Field(GitObjectID, graphql_name="expectedHeadOid") + """The expected head OID of the pull request.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + +class EnqueuePullRequestInput(sgqlc.types.Input): + """Autogenerated input type of EnqueuePullRequest""" + + __schema__ = github_schema + __field_names__ = ("pull_request_id", "jump", "expected_head_oid", "client_mutation_id") + pull_request_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="pullRequestId") + """The ID of the pull request to enqueue.""" + + jump = sgqlc.types.Field(Boolean, graphql_name="jump") + """Add the pull request to the front of the queue.""" + + expected_head_oid = sgqlc.types.Field(GitObjectID, graphql_name="expectedHeadOid") + """The expected head OID of the pull request.""" + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" @@ -5468,6 +7250,36 @@ class LanguageOrder(sgqlc.types.Input): """The ordering direction.""" +class LinkProjectV2ToRepositoryInput(sgqlc.types.Input): + """Autogenerated input type of LinkProjectV2ToRepository""" + + __schema__ = github_schema + __field_names__ = ("project_id", "repository_id", "client_mutation_id") + project_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="projectId") + """The ID of the project to link to the repository.""" + + repository_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="repositoryId") + """The ID of the repository to link to the project.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + +class LinkProjectV2ToTeamInput(sgqlc.types.Input): + """Autogenerated input type of LinkProjectV2ToTeam""" + + __schema__ = github_schema + __field_names__ = ("project_id", "team_id", "client_mutation_id") + project_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="projectId") + """The ID of the project to link to the team.""" + + team_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="teamId") + """The ID of the team to link to the project.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class LinkRepositoryToProjectInput(sgqlc.types.Input): """Autogenerated input type of LinkRepositoryToProject""" @@ -5498,6 +7310,18 @@ class LockLockableInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" +class MannequinOrder(sgqlc.types.Input): + """Ordering options for mannequins.""" + + __schema__ = github_schema + __field_names__ = ("field", "direction") + field = sgqlc.types.Field(sgqlc.types.non_null(MannequinOrderField), graphql_name="field") + """The field to order mannequins by.""" + + direction = sgqlc.types.Field(sgqlc.types.non_null(OrderDirection), graphql_name="direction") + """The ordering direction.""" + + class MarkDiscussionCommentAsAnswerInput(sgqlc.types.Input): """Autogenerated input type of MarkDiscussionCommentAsAnswer""" @@ -5525,6 +7349,18 @@ class MarkFileAsViewedInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" +class MarkProjectV2AsTemplateInput(sgqlc.types.Input): + """Autogenerated input type of MarkProjectV2AsTemplate""" + + __schema__ = github_schema + __field_names__ = ("project_id", "client_mutation_id") + project_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="projectId") + """The ID of the Project to mark as a template.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class MarkPullRequestReadyForReviewInput(sgqlc.types.Input): """Autogenerated input type of MarkPullRequestReadyForReview""" @@ -5766,6 +7602,154 @@ class ProjectOrder(sgqlc.types.Input): """The direction in which to order projects by the specified field.""" +class ProjectV2Collaborator(sgqlc.types.Input): + """A collaborator to update on a project. Only one of the userId or + teamId should be provided. + """ + + __schema__ = github_schema + __field_names__ = ("user_id", "team_id", "role") + user_id = sgqlc.types.Field(ID, graphql_name="userId") + """The ID of the user as a collaborator.""" + + team_id = sgqlc.types.Field(ID, graphql_name="teamId") + """The ID of the team as a collaborator.""" + + role = sgqlc.types.Field(sgqlc.types.non_null(ProjectV2Roles), graphql_name="role") + """The role to grant the collaborator""" + + +class ProjectV2FieldOrder(sgqlc.types.Input): + """Ordering options for project v2 field connections""" + + __schema__ = github_schema + __field_names__ = ("field", "direction") + field = sgqlc.types.Field(sgqlc.types.non_null(ProjectV2FieldOrderField), graphql_name="field") + """The field to order the project v2 fields by.""" + + direction = sgqlc.types.Field(sgqlc.types.non_null(OrderDirection), graphql_name="direction") + """The ordering direction.""" + + +class ProjectV2FieldValue(sgqlc.types.Input): + """The values that can be used to update a field of an item inside a + Project. Only 1 value can be updated at a time. + """ + + __schema__ = github_schema + __field_names__ = ("text", "number", "date", "single_select_option_id", "iteration_id") + text = sgqlc.types.Field(String, graphql_name="text") + """The text to set on the field.""" + + number = sgqlc.types.Field(Float, graphql_name="number") + """The number to set on the field.""" + + date = sgqlc.types.Field(Date, graphql_name="date") + """The ISO 8601 date to set on the field.""" + + single_select_option_id = sgqlc.types.Field(String, graphql_name="singleSelectOptionId") + """The id of the single select option to set on the field.""" + + iteration_id = sgqlc.types.Field(String, graphql_name="iterationId") + """The id of the iteration to set on the field.""" + + +class ProjectV2Filters(sgqlc.types.Input): + """Ways in which to filter lists of projects.""" + + __schema__ = github_schema + __field_names__ = ("state",) + state = sgqlc.types.Field(ProjectV2State, graphql_name="state") + """List project v2 filtered by the state given.""" + + +class ProjectV2ItemFieldValueOrder(sgqlc.types.Input): + """Ordering options for project v2 item field value connections""" + + __schema__ = github_schema + __field_names__ = ("field", "direction") + field = sgqlc.types.Field(sgqlc.types.non_null(ProjectV2ItemFieldValueOrderField), graphql_name="field") + """The field to order the project v2 item field values by.""" + + direction = sgqlc.types.Field(sgqlc.types.non_null(OrderDirection), graphql_name="direction") + """The ordering direction.""" + + +class ProjectV2ItemOrder(sgqlc.types.Input): + """Ordering options for project v2 item connections""" + + __schema__ = github_schema + __field_names__ = ("field", "direction") + field = sgqlc.types.Field(sgqlc.types.non_null(ProjectV2ItemOrderField), graphql_name="field") + """The field to order the project v2 items by.""" + + direction = sgqlc.types.Field(sgqlc.types.non_null(OrderDirection), graphql_name="direction") + """The ordering direction.""" + + +class ProjectV2Order(sgqlc.types.Input): + """Ways in which lists of projects can be ordered upon return.""" + + __schema__ = github_schema + __field_names__ = ("field", "direction") + field = sgqlc.types.Field(sgqlc.types.non_null(ProjectV2OrderField), graphql_name="field") + """The field in which to order projects by.""" + + direction = sgqlc.types.Field(sgqlc.types.non_null(OrderDirection), graphql_name="direction") + """The direction in which to order projects by the specified field.""" + + +class ProjectV2SingleSelectFieldOptionInput(sgqlc.types.Input): + """Represents a single select field option""" + + __schema__ = github_schema + __field_names__ = ("name", "color", "description") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + """The name of the option""" + + color = sgqlc.types.Field(sgqlc.types.non_null(ProjectV2SingleSelectFieldOptionColor), graphql_name="color") + """The display color of the option""" + + description = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="description") + """The description text of the option""" + + +class ProjectV2ViewOrder(sgqlc.types.Input): + """Ordering options for project v2 view connections""" + + __schema__ = github_schema + __field_names__ = ("field", "direction") + field = sgqlc.types.Field(sgqlc.types.non_null(ProjectV2ViewOrderField), graphql_name="field") + """The field to order the project v2 views by.""" + + direction = sgqlc.types.Field(sgqlc.types.non_null(OrderDirection), graphql_name="direction") + """The ordering direction.""" + + +class ProjectV2WorkflowOrder(sgqlc.types.Input): + """Ordering options for project v2 workflows connections""" + + __schema__ = github_schema + __field_names__ = ("field", "direction") + field = sgqlc.types.Field(sgqlc.types.non_null(ProjectV2WorkflowsOrderField), graphql_name="field") + """The field to order the project v2 workflows by.""" + + direction = sgqlc.types.Field(sgqlc.types.non_null(OrderDirection), graphql_name="direction") + """The ordering direction.""" + + +class PublishSponsorsTierInput(sgqlc.types.Input): + """Autogenerated input type of PublishSponsorsTier""" + + __schema__ = github_schema + __field_names__ = ("tier_id", "client_mutation_id") + tier_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="tierId") + """The ID of the draft tier to publish.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class PullRequestOrder(sgqlc.types.Input): """Ways in which lists of issues can be ordered upon return.""" @@ -5780,6 +7764,45 @@ class PullRequestOrder(sgqlc.types.Input): """ +class PullRequestParametersInput(sgqlc.types.Input): + """Require all commits be made to a non-target branch and submitted + via a pull request before they can be merged. + """ + + __schema__ = github_schema + __field_names__ = ( + "dismiss_stale_reviews_on_push", + "require_code_owner_review", + "require_last_push_approval", + "required_approving_review_count", + "required_review_thread_resolution", + ) + dismiss_stale_reviews_on_push = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="dismissStaleReviewsOnPush") + """New, reviewable commits pushed will dismiss previous pull request + review approvals. + """ + + require_code_owner_review = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requireCodeOwnerReview") + """Require an approving review in pull requests that modify files + that have a designated code owner. + """ + + require_last_push_approval = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requireLastPushApproval") + """Whether the most recent reviewable push must be approved by + someone other than the person who pushed it. + """ + + required_approving_review_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="requiredApprovingReviewCount") + """The number of approving reviews that are required before a pull + request can be merged. + """ + + required_review_thread_resolution = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiredReviewThreadResolution") + """All conversations on code must be resolved before a pull request + can be merged. + """ + + class ReactionOrder(sgqlc.types.Input): """Ways in which lists of reactions can be ordered upon return.""" @@ -5792,6 +7815,24 @@ class ReactionOrder(sgqlc.types.Input): """The direction in which to order reactions by the specified field.""" +class RefNameConditionTargetInput(sgqlc.types.Input): + """Parameters to be used for the ref_name condition""" + + __schema__ = github_schema + __field_names__ = ("exclude", "include") + exclude = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="exclude") + """Array of ref names or patterns to exclude. The condition will not + pass if any of these patterns match. + """ + + include = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="include") + """Array of ref names or patterns to include. One of these patterns + must match for the condition to pass. Also accepts + `~DEFAULT_BRANCH` to include the default branch or `~ALL` to + include all branches. + """ + + class RefOrder(sgqlc.types.Input): """Ways in which lists of git refs can be ordered upon return.""" @@ -5908,6 +7949,21 @@ class RemoveEnterpriseIdentityProviderInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" +class RemoveEnterpriseMemberInput(sgqlc.types.Input): + """Autogenerated input type of RemoveEnterpriseMember""" + + __schema__ = github_schema + __field_names__ = ("enterprise_id", "user_id", "client_mutation_id") + enterprise_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="enterpriseId") + """The ID of the enterprise from which the user should be removed.""" + + user_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="userId") + """The ID of the user to remove from the enterprise.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class RemoveEnterpriseOrganizationInput(sgqlc.types.Input): """Autogenerated input type of RemoveEnterpriseOrganization""" @@ -6011,6 +8067,18 @@ class RemoveUpvoteInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" +class ReopenDiscussionInput(sgqlc.types.Input): + """Autogenerated input type of ReopenDiscussion""" + + __schema__ = github_schema + __field_names__ = ("discussion_id", "client_mutation_id") + discussion_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="discussionId") + """ID of the discussion to be reopened.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class ReopenIssueInput(sgqlc.types.Input): """Autogenerated input type of ReopenIssue""" @@ -6059,6 +8127,28 @@ class RepositoryMigrationOrder(sgqlc.types.Input): """The ordering direction.""" +class RepositoryNameConditionTargetInput(sgqlc.types.Input): + """Parameters to be used for the repository_name condition""" + + __schema__ = github_schema + __field_names__ = ("exclude", "include", "protected") + exclude = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="exclude") + """Array of repository names or patterns to exclude. The condition + will not pass if any of these patterns match. + """ + + include = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="include") + """Array of repository names or patterns to include. One of these + patterns must match for the condition to pass. Also accepts `~ALL` + to include all repositories. + """ + + protected = sgqlc.types.Field(Boolean, graphql_name="protected") + """Target changes that match these patterns will be prevented except + by those with bypass permissions. + """ + + class RepositoryOrder(sgqlc.types.Input): """Ordering options for repository connections""" @@ -6071,6 +8161,33 @@ class RepositoryOrder(sgqlc.types.Input): """The ordering direction.""" +class RepositoryRuleConditionsInput(sgqlc.types.Input): + """Specifies the conditions required for a ruleset to evaluate""" + + __schema__ = github_schema + __field_names__ = ("ref_name", "repository_name") + ref_name = sgqlc.types.Field(RefNameConditionTargetInput, graphql_name="refName") + """Configuration for the ref_name condition""" + + repository_name = sgqlc.types.Field(RepositoryNameConditionTargetInput, graphql_name="repositoryName") + """Configuration for the repository_name condition""" + + +class RepositoryRuleInput(sgqlc.types.Input): + """Specifies the attributes for a new or updated rule.""" + + __schema__ = github_schema + __field_names__ = ("id", "type", "parameters") + id = sgqlc.types.Field(ID, graphql_name="id") + """Optional ID of this rule when updating""" + + type = sgqlc.types.Field(sgqlc.types.non_null(RepositoryRuleType), graphql_name="type") + """The type of rule to create.""" + + parameters = sgqlc.types.Field("RuleParametersInput", graphql_name="parameters") + """The parameters for the rule.""" + + class RequestReviewsInput(sgqlc.types.Input): """Autogenerated input type of RequestReviews""" @@ -6092,6 +8209,21 @@ class RequestReviewsInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" +class RequiredDeploymentsParametersInput(sgqlc.types.Input): + """Choose which environments must be successfully deployed to before + branches can be merged into a branch that matches this rule. + """ + + __schema__ = github_schema + __field_names__ = ("required_deployment_environments",) + required_deployment_environments = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="requiredDeploymentEnvironments" + ) + """The environments that must be successfully deployed to before + branches can be merged. + """ + + class RequiredStatusCheckInput(sgqlc.types.Input): """Specifies the attributes for a new or updated required status check. @@ -6112,6 +8244,29 @@ class RequiredStatusCheckInput(sgqlc.types.Input): """ +class RequiredStatusChecksParametersInput(sgqlc.types.Input): + """Choose which status checks must pass before branches can be merged + into a branch that matches this rule. When enabled, commits must + first be pushed to another branch, then merged or pushed directly + to a branch that matches this rule after status checks have + passed. + """ + + __schema__ = github_schema + __field_names__ = ("required_status_checks", "strict_required_status_checks_policy") + required_status_checks = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("StatusCheckConfigurationInput"))), + graphql_name="requiredStatusChecks", + ) + """Status checks that are required.""" + + strict_required_status_checks_policy = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="strictRequiredStatusChecksPolicy") + """Whether pull requests targeting a matching branch must be tested + with the latest code. This setting will not take effect unless at + least one status check is enabled. + """ + + class RerequestCheckSuiteInput(sgqlc.types.Input): """Autogenerated input type of RerequestCheckSuite""" @@ -6139,6 +8294,39 @@ class ResolveReviewThreadInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" +class RetireSponsorsTierInput(sgqlc.types.Input): + """Autogenerated input type of RetireSponsorsTier""" + + __schema__ = github_schema + __field_names__ = ("tier_id", "client_mutation_id") + tier_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="tierId") + """The ID of the published tier to retire.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + +class RevertPullRequestInput(sgqlc.types.Input): + """Autogenerated input type of RevertPullRequest""" + + __schema__ = github_schema + __field_names__ = ("pull_request_id", "title", "body", "draft", "client_mutation_id") + pull_request_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="pullRequestId") + """The ID of the pull request to revert.""" + + title = sgqlc.types.Field(String, graphql_name="title") + """The title of the revert pull request.""" + + body = sgqlc.types.Field(String, graphql_name="body") + """The description of the revert pull request.""" + + draft = sgqlc.types.Field(Boolean, graphql_name="draft") + """Indicates whether the revert pull request should be a draft.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class RevokeEnterpriseOrganizationsMigratorRoleInput(sgqlc.types.Input): """Autogenerated input type of RevokeEnterpriseOrganizationsMigratorRole @@ -6176,6 +8364,51 @@ class RevokeMigratorRoleInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" +class RuleParametersInput(sgqlc.types.Input): + """Specifies the parameters for a `RepositoryRule` object. Only one + of the fields should be specified. + """ + + __schema__ = github_schema + __field_names__ = ( + "update", + "required_deployments", + "pull_request", + "required_status_checks", + "commit_message_pattern", + "commit_author_email_pattern", + "committer_email_pattern", + "branch_name_pattern", + "tag_name_pattern", + ) + update = sgqlc.types.Field("UpdateParametersInput", graphql_name="update") + """Parameters used for the `update` rule type""" + + required_deployments = sgqlc.types.Field(RequiredDeploymentsParametersInput, graphql_name="requiredDeployments") + """Parameters used for the `required_deployments` rule type""" + + pull_request = sgqlc.types.Field(PullRequestParametersInput, graphql_name="pullRequest") + """Parameters used for the `pull_request` rule type""" + + required_status_checks = sgqlc.types.Field(RequiredStatusChecksParametersInput, graphql_name="requiredStatusChecks") + """Parameters used for the `required_status_checks` rule type""" + + commit_message_pattern = sgqlc.types.Field(CommitMessagePatternParametersInput, graphql_name="commitMessagePattern") + """Parameters used for the `commit_message_pattern` rule type""" + + commit_author_email_pattern = sgqlc.types.Field(CommitAuthorEmailPatternParametersInput, graphql_name="commitAuthorEmailPattern") + """Parameters used for the `commit_author_email_pattern` rule type""" + + committer_email_pattern = sgqlc.types.Field(CommitterEmailPatternParametersInput, graphql_name="committerEmailPattern") + """Parameters used for the `committer_email_pattern` rule type""" + + branch_name_pattern = sgqlc.types.Field(BranchNamePatternParametersInput, graphql_name="branchNamePattern") + """Parameters used for the `branch_name_pattern` rule type""" + + tag_name_pattern = sgqlc.types.Field("TagNamePatternParametersInput", graphql_name="tagNamePattern") + """Parameters used for the `tag_name_pattern` rule type""" + + class SavedReplyOrder(sgqlc.types.Input): """Ordering options for saved reply connections.""" @@ -6399,6 +8632,27 @@ class StarOrder(sgqlc.types.Input): """The direction in which to order nodes.""" +class StartOrganizationMigrationInput(sgqlc.types.Input): + """Autogenerated input type of StartOrganizationMigration""" + + __schema__ = github_schema + __field_names__ = ("source_org_url", "target_org_name", "target_enterprise_id", "source_access_token", "client_mutation_id") + source_org_url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="sourceOrgUrl") + """The URL of the organization to migrate.""" + + target_org_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="targetOrgName") + """The name of the target organization.""" + + target_enterprise_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="targetEnterpriseId") + """The ID of the enterprise the target organization belongs to.""" + + source_access_token = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="sourceAccessToken") + """The migration source access token.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class StartRepositoryMigrationInput(sgqlc.types.Input): """Autogenerated input type of StartRepositoryMigration""" @@ -6414,31 +8668,33 @@ class StartRepositoryMigrationInput(sgqlc.types.Input): "access_token", "github_pat", "skip_releases", + "target_repo_visibility", + "lock_source", "client_mutation_id", ) source_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="sourceId") - """The ID of the Octoshift migration source.""" + """The ID of the migration source.""" owner_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="ownerId") """The ID of the organization that will own the imported repository.""" - source_repository_url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="sourceRepositoryUrl") - """The Octoshift migration source repository URL.""" + source_repository_url = sgqlc.types.Field(URI, graphql_name="sourceRepositoryUrl") + """The URL of the source repository.""" repository_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="repositoryName") """The name of the imported repository.""" continue_on_error = sgqlc.types.Field(Boolean, graphql_name="continueOnError") - """Whether to continue the migration on error""" + """Whether to continue the migration on error. Defaults to `false`.""" git_archive_url = sgqlc.types.Field(String, graphql_name="gitArchiveUrl") - """The signed URL to access the user-uploaded git archive""" + """The signed URL to access the user-uploaded git archive.""" metadata_archive_url = sgqlc.types.Field(String, graphql_name="metadataArchiveUrl") - """The signed URL to access the user-uploaded metadata archive""" + """The signed URL to access the user-uploaded metadata archive.""" - access_token = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="accessToken") - """The Octoshift migration source access token.""" + access_token = sgqlc.types.Field(String, graphql_name="accessToken") + """The migration source access token.""" github_pat = sgqlc.types.Field(String, graphql_name="githubPat") """The GitHub personal access token of the user importing to the @@ -6448,10 +8704,30 @@ class StartRepositoryMigrationInput(sgqlc.types.Input): skip_releases = sgqlc.types.Field(Boolean, graphql_name="skipReleases") """Whether to skip migrating releases for the repository.""" + target_repo_visibility = sgqlc.types.Field(String, graphql_name="targetRepoVisibility") + """The visibility of the imported repository.""" + + lock_source = sgqlc.types.Field(Boolean, graphql_name="lockSource") + """Whether to lock the source repository.""" + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" +class StatusCheckConfigurationInput(sgqlc.types.Input): + """Required status check""" + + __schema__ = github_schema + __field_names__ = ("context", "integration_id") + context = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="context") + """The status check context name that must be present on the commit.""" + + integration_id = sgqlc.types.Field(Int, graphql_name="integrationId") + """The optional integration ID that this status check must originate + from. + """ + + class SubmitPullRequestReviewInput(sgqlc.types.Input): """Autogenerated input type of SubmitPullRequestReview""" @@ -6473,6 +8749,24 @@ class SubmitPullRequestReviewInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" +class TagNamePatternParametersInput(sgqlc.types.Input): + """Parameters to be used for the tag_name_pattern rule""" + + __schema__ = github_schema + __field_names__ = ("name", "negate", "operator", "pattern") + name = sgqlc.types.Field(String, graphql_name="name") + """How this rule will appear to users.""" + + negate = sgqlc.types.Field(Boolean, graphql_name="negate") + """If true, the rule will fail if the pattern matches.""" + + operator = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="operator") + """The operator to use for matching.""" + + pattern = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="pattern") + """The pattern to match with.""" + + class TeamDiscussionCommentOrder(sgqlc.types.Input): """Ways in which team discussion comment connections can be ordered.""" @@ -6533,17 +8827,54 @@ class TeamRepositoryOrder(sgqlc.types.Input): """The ordering direction.""" +class TransferEnterpriseOrganizationInput(sgqlc.types.Input): + """Autogenerated input type of TransferEnterpriseOrganization""" + + __schema__ = github_schema + __field_names__ = ("organization_id", "destination_enterprise_id", "client_mutation_id") + organization_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="organizationId") + """The ID of the organization to transfer.""" + + destination_enterprise_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="destinationEnterpriseId") + """The ID of the enterprise where the organization should be + transferred. + """ + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class TransferIssueInput(sgqlc.types.Input): """Autogenerated input type of TransferIssue""" __schema__ = github_schema - __field_names__ = ("issue_id", "repository_id", "client_mutation_id") + __field_names__ = ("issue_id", "repository_id", "create_labels_if_missing", "client_mutation_id") issue_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="issueId") """The Node ID of the issue to be transferred""" repository_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="repositoryId") """The Node ID of the repository the issue should be transferred to""" + create_labels_if_missing = sgqlc.types.Field(Boolean, graphql_name="createLabelsIfMissing") + """Whether to create labels if they don't exist in the target + repository (matched by name) + """ + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + +class UnarchiveProjectV2ItemInput(sgqlc.types.Input): + """Autogenerated input type of UnarchiveProjectV2Item""" + + __schema__ = github_schema + __field_names__ = ("project_id", "item_id", "client_mutation_id") + project_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="projectId") + """The ID of the Project to archive the item from.""" + + item_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="itemId") + """The ID of the ProjectV2Item to unarchive.""" + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" @@ -6584,6 +8915,36 @@ class UnfollowUserInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" +class UnlinkProjectV2FromRepositoryInput(sgqlc.types.Input): + """Autogenerated input type of UnlinkProjectV2FromRepository""" + + __schema__ = github_schema + __field_names__ = ("project_id", "repository_id", "client_mutation_id") + project_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="projectId") + """The ID of the project to unlink from the repository.""" + + repository_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="repositoryId") + """The ID of the repository to unlink from the project.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + +class UnlinkProjectV2FromTeamInput(sgqlc.types.Input): + """Autogenerated input type of UnlinkProjectV2FromTeam""" + + __schema__ = github_schema + __field_names__ = ("project_id", "team_id", "client_mutation_id") + project_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="projectId") + """The ID of the project to unlink from the team.""" + + team_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="teamId") + """The ID of the team to unlink from the project.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class UnlinkRepositoryFromProjectInput(sgqlc.types.Input): """Autogenerated input type of UnlinkRepositoryFromProject""" @@ -6655,6 +9016,18 @@ class UnmarkIssueAsDuplicateInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" +class UnmarkProjectV2AsTemplateInput(sgqlc.types.Input): + """Autogenerated input type of UnmarkProjectV2AsTemplate""" + + __schema__ = github_schema + __field_names__ = ("project_id", "client_mutation_id") + project_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="projectId") + """The ID of the Project to unmark as a template.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class UnminimizeCommentInput(sgqlc.types.Input): """Autogenerated input type of UnminimizeComment""" @@ -6718,7 +9091,12 @@ class UpdateBranchProtectionRuleInput(sgqlc.types.Input): "push_actor_ids", "required_status_check_contexts", "required_status_checks", + "requires_deployments", + "required_deployment_environments", "requires_conversation_resolution", + "require_last_push_approval", + "lock_branch", + "lock_allows_fetch_and_merge", "client_mutation_id", ) branch_protection_rule_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="branchProtectionRuleId") @@ -6805,9 +9183,33 @@ class UpdateBranchProtectionRuleInput(sgqlc.types.Input): ) """The list of required status checks""" + requires_deployments = sgqlc.types.Field(Boolean, graphql_name="requiresDeployments") + """Are successful deployments required before merging.""" + + required_deployment_environments = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="requiredDeploymentEnvironments" + ) + """The list of required deployment environments""" + requires_conversation_resolution = sgqlc.types.Field(Boolean, graphql_name="requiresConversationResolution") """Are conversations required to be resolved before merging.""" + require_last_push_approval = sgqlc.types.Field(Boolean, graphql_name="requireLastPushApproval") + """Whether the most recent push must be approved by someone other + than the person who pushed it + """ + + lock_branch = sgqlc.types.Field(Boolean, graphql_name="lockBranch") + """Whether to set the branch as read-only. If this is true, users + will not be able to push to the branch. + """ + + lock_allows_fetch_and_merge = sgqlc.types.Field(Boolean, graphql_name="lockAllowsFetchAndMerge") + """Whether users can pull changes from upstream when the branch is + locked. Set to `true` to allow fork syncing. Set to `false` to + prevent fork syncing. + """ + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" @@ -6951,7 +9353,7 @@ class UpdateEnterpriseAllowPrivateRepositoryForkingSettingInput(sgqlc.types.Inpu """ __schema__ = github_schema - __field_names__ = ("enterprise_id", "setting_value", "client_mutation_id") + __field_names__ = ("enterprise_id", "setting_value", "policy_value", "client_mutation_id") enterprise_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="enterpriseId") """The ID of the enterprise on which to set the allow private repository forking setting. @@ -6962,6 +9364,11 @@ class UpdateEnterpriseAllowPrivateRepositoryForkingSettingInput(sgqlc.types.Inpu enterprise. """ + policy_value = sgqlc.types.Field(EnterpriseAllowPrivateRepositoryForkingPolicyValue, graphql_name="policyValue") + """The value for the allow private repository forking policy on the + enterprise. + """ + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" @@ -7465,6 +9872,36 @@ class UpdateOrganizationAllowPrivateRepositoryForkingSettingInput(sgqlc.types.In """A unique identifier for the client performing the mutation.""" +class UpdateOrganizationWebCommitSignoffSettingInput(sgqlc.types.Input): + """Autogenerated input type of + UpdateOrganizationWebCommitSignoffSetting + """ + + __schema__ = github_schema + __field_names__ = ("organization_id", "web_commit_signoff_required", "client_mutation_id") + organization_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="organizationId") + """The ID of the organization on which to set the web commit signoff + setting. + """ + + web_commit_signoff_required = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="webCommitSignoffRequired") + """Enable signoff on web-based commits for repositories in the + organization? + """ + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + +class UpdateParametersInput(sgqlc.types.Input): + """Only allow users with bypass permission to update matching refs.""" + + __schema__ = github_schema + __field_names__ = ("update_allows_fetch_and_merge",) + update_allows_fetch_and_merge = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="updateAllowsFetchAndMerge") + """Branch can pull changes from its upstream repository""" + + class UpdateProjectCardInput(sgqlc.types.Input): """Autogenerated input type of UpdateProjectCard""" @@ -7498,27 +9935,6 @@ class UpdateProjectColumnInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" -class UpdateProjectDraftIssueInput(sgqlc.types.Input): - """Autogenerated input type of UpdateProjectDraftIssue""" - - __schema__ = github_schema - __field_names__ = ("draft_issue_id", "title", "body", "assignee_ids", "client_mutation_id") - draft_issue_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="draftIssueId") - """The ID of the draft issue to update.""" - - title = sgqlc.types.Field(String, graphql_name="title") - """The title of the draft issue.""" - - body = sgqlc.types.Field(String, graphql_name="body") - """The body of the draft issue.""" - - assignee_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="assigneeIds") - """The IDs of the assignees of the draft issue.""" - - client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") - """A unique identifier for the client performing the mutation.""" - - class UpdateProjectInput(sgqlc.types.Input): """Autogenerated input type of UpdateProject""" @@ -7543,23 +9959,61 @@ class UpdateProjectInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" -class UpdateProjectNextInput(sgqlc.types.Input): - """Autogenerated input type of UpdateProjectNext""" +class UpdateProjectV2CollaboratorsInput(sgqlc.types.Input): + """Autogenerated input type of UpdateProjectV2Collaborators""" __schema__ = github_schema - __field_names__ = ("project_id", "title", "description", "short_description", "closed", "public", "client_mutation_id") + __field_names__ = ("project_id", "collaborators", "client_mutation_id") + project_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="projectId") + """The ID of the project to update the collaborators for.""" + + collaborators = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProjectV2Collaborator))), graphql_name="collaborators" + ) + """The collaborators to update.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + +class UpdateProjectV2DraftIssueInput(sgqlc.types.Input): + """Autogenerated input type of UpdateProjectV2DraftIssue""" + + __schema__ = github_schema + __field_names__ = ("draft_issue_id", "title", "body", "assignee_ids", "client_mutation_id") + draft_issue_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="draftIssueId") + """The ID of the draft issue to update.""" + + title = sgqlc.types.Field(String, graphql_name="title") + """The title of the draft issue.""" + + body = sgqlc.types.Field(String, graphql_name="body") + """The body of the draft issue.""" + + assignee_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="assigneeIds") + """The IDs of the assignees of the draft issue.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + +class UpdateProjectV2Input(sgqlc.types.Input): + """Autogenerated input type of UpdateProjectV2""" + + __schema__ = github_schema + __field_names__ = ("project_id", "title", "short_description", "readme", "closed", "public", "client_mutation_id") project_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="projectId") """The ID of the Project to update.""" title = sgqlc.types.Field(String, graphql_name="title") """Set the title of the project.""" - description = sgqlc.types.Field(String, graphql_name="description") - """Set the readme description of the project.""" - short_description = sgqlc.types.Field(String, graphql_name="shortDescription") """Set the short description of the project.""" + readme = sgqlc.types.Field(String, graphql_name="readme") + """Set the readme description of the project.""" + closed = sgqlc.types.Field(Boolean, graphql_name="closed") """Set the project to closed or open.""" @@ -7570,8 +10024,8 @@ class UpdateProjectNextInput(sgqlc.types.Input): """A unique identifier for the client performing the mutation.""" -class UpdateProjectNextItemFieldInput(sgqlc.types.Input): - """Autogenerated input type of UpdateProjectNextItemField""" +class UpdateProjectV2ItemFieldValueInput(sgqlc.types.Input): + """Autogenerated input type of UpdateProjectV2ItemFieldValue""" __schema__ = github_schema __field_names__ = ("project_id", "item_id", "field_id", "value", "client_mutation_id") @@ -7579,18 +10033,38 @@ class UpdateProjectNextItemFieldInput(sgqlc.types.Input): """The ID of the Project.""" item_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="itemId") - """The id of the item to be updated.""" + """The ID of the item to be updated.""" - field_id = sgqlc.types.Field(ID, graphql_name="fieldId") - """The id of the field to be updated.""" + field_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="fieldId") + """The ID of the field to be updated.""" - value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") + value = sgqlc.types.Field(sgqlc.types.non_null(ProjectV2FieldValue), graphql_name="value") """The value which will be set on the field.""" client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" +class UpdateProjectV2ItemPositionInput(sgqlc.types.Input): + """Autogenerated input type of UpdateProjectV2ItemPosition""" + + __schema__ = github_schema + __field_names__ = ("project_id", "item_id", "after_id", "client_mutation_id") + project_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="projectId") + """The ID of the Project.""" + + item_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="itemId") + """The ID of the item to be moved.""" + + after_id = sgqlc.types.Field(ID, graphql_name="afterId") + """The ID of the item to position this item after. If omitted or set + to null the item will be moved to top. + """ + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class UpdatePullRequestBranchInput(sgqlc.types.Input): """Autogenerated input type of UpdatePullRequestBranch""" @@ -7722,6 +10196,7 @@ class UpdateRepositoryInput(sgqlc.types.Input): "has_wiki_enabled", "has_issues_enabled", "has_projects_enabled", + "has_discussions_enabled", "client_mutation_id", ) repository_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="repositoryId") @@ -7759,191 +10234,2128 @@ class UpdateRepositoryInput(sgqlc.types.Input): enabled. """ + has_discussions_enabled = sgqlc.types.Field(Boolean, graphql_name="hasDiscussionsEnabled") + """Indicates if the repository should have the discussions feature + enabled. + """ + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + +class UpdateRepositoryRulesetInput(sgqlc.types.Input): + """Autogenerated input type of UpdateRepositoryRuleset""" + + __schema__ = github_schema + __field_names__ = ( + "repository_ruleset_id", + "name", + "target", + "rules", + "conditions", + "enforcement", + "bypass_mode", + "bypass_actor_ids", + "client_mutation_id", + ) + repository_ruleset_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="repositoryRulesetId") + """The global relay id of the repository ruleset to be updated.""" + + name = sgqlc.types.Field(String, graphql_name="name") + """The name of the ruleset.""" + + target = sgqlc.types.Field(RepositoryRulesetTarget, graphql_name="target") + """The target of the ruleset.""" + + rules = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(RepositoryRuleInput)), graphql_name="rules") + """The list of rules for this ruleset""" + + conditions = sgqlc.types.Field(RepositoryRuleConditionsInput, graphql_name="conditions") + """The list of conditions for this ruleset""" + + enforcement = sgqlc.types.Field(RuleEnforcement, graphql_name="enforcement") + """The enforcement level for this ruleset""" + + bypass_mode = sgqlc.types.Field(RuleBypassMode, graphql_name="bypassMode") + """The bypass mode for this ruleset""" + + bypass_actor_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="bypassActorIds") + """A list of Team or App IDs allowed to bypass rules in this ruleset.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + +class UpdateRepositoryWebCommitSignoffSettingInput(sgqlc.types.Input): + """Autogenerated input type of + UpdateRepositoryWebCommitSignoffSetting + """ + + __schema__ = github_schema + __field_names__ = ("repository_id", "web_commit_signoff_required", "client_mutation_id") + repository_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="repositoryId") + """The ID of the repository to update.""" + + web_commit_signoff_required = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="webCommitSignoffRequired") + """Indicates if the repository should require signoff on web-based + commits. + """ + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + +class UpdateSponsorshipPreferencesInput(sgqlc.types.Input): + """Autogenerated input type of UpdateSponsorshipPreferences""" + + __schema__ = github_schema + __field_names__ = ( + "sponsor_id", + "sponsor_login", + "sponsorable_id", + "sponsorable_login", + "receive_emails", + "privacy_level", + "client_mutation_id", + ) + sponsor_id = sgqlc.types.Field(ID, graphql_name="sponsorId") + """The ID of the user or organization who is acting as the sponsor, + paying for the sponsorship. Required if sponsorLogin is not given. + """ + + sponsor_login = sgqlc.types.Field(String, graphql_name="sponsorLogin") + """The username of the user or organization who is acting as the + sponsor, paying for the sponsorship. Required if sponsorId is not + given. + """ + + sponsorable_id = sgqlc.types.Field(ID, graphql_name="sponsorableId") + """The ID of the user or organization who is receiving the + sponsorship. Required if sponsorableLogin is not given. + """ + + sponsorable_login = sgqlc.types.Field(String, graphql_name="sponsorableLogin") + """The username of the user or organization who is receiving the + sponsorship. Required if sponsorableId is not given. + """ + + receive_emails = sgqlc.types.Field(Boolean, graphql_name="receiveEmails") + """Whether the sponsor should receive email updates from the + sponsorable. + """ + + privacy_level = sgqlc.types.Field(SponsorshipPrivacy, graphql_name="privacyLevel") + """Specify whether others should be able to see that the sponsor is + sponsoring the sponsorable. Public visibility still does not + reveal which tier is used. + """ + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + +class UpdateSubscriptionInput(sgqlc.types.Input): + """Autogenerated input type of UpdateSubscription""" + + __schema__ = github_schema + __field_names__ = ("subscribable_id", "state", "client_mutation_id") + subscribable_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="subscribableId") + """The Node ID of the subscribable object to modify.""" + + state = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionState), graphql_name="state") + """The new state of the subscription.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + +class UpdateTeamDiscussionCommentInput(sgqlc.types.Input): + """Autogenerated input type of UpdateTeamDiscussionComment""" + + __schema__ = github_schema + __field_names__ = ("id", "body", "body_version", "client_mutation_id") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + """The ID of the comment to modify.""" + + body = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="body") + """The updated text of the comment.""" + + body_version = sgqlc.types.Field(String, graphql_name="bodyVersion") + """The current version of the body content.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + +class UpdateTeamDiscussionInput(sgqlc.types.Input): + """Autogenerated input type of UpdateTeamDiscussion""" + + __schema__ = github_schema + __field_names__ = ("id", "title", "body", "body_version", "pinned", "client_mutation_id") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + """The Node ID of the discussion to modify.""" + + title = sgqlc.types.Field(String, graphql_name="title") + """The updated title of the discussion.""" + + body = sgqlc.types.Field(String, graphql_name="body") + """The updated text of the discussion.""" + + body_version = sgqlc.types.Field(String, graphql_name="bodyVersion") + """The current version of the body content. If provided, this update + operation will be rejected if the given version does not match the + latest version on the server. + """ + + pinned = sgqlc.types.Field(Boolean, graphql_name="pinned") + """If provided, sets the pinned state of the updated discussion.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + +class UpdateTeamsRepositoryInput(sgqlc.types.Input): + """Autogenerated input type of UpdateTeamsRepository""" + + __schema__ = github_schema + __field_names__ = ("repository_id", "team_ids", "permission", "client_mutation_id") + repository_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="repositoryId") + """Repository ID being granted access to.""" + + team_ids = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="teamIds") + """A list of teams being granted access. Limit: 10""" + + permission = sgqlc.types.Field(sgqlc.types.non_null(RepositoryPermission), graphql_name="permission") + """Permission that should be granted to the teams.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + +class UpdateTopicsInput(sgqlc.types.Input): + """Autogenerated input type of UpdateTopics""" + + __schema__ = github_schema + __field_names__ = ("repository_id", "topic_names", "client_mutation_id") + repository_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="repositoryId") + """The Node ID of the repository.""" + + topic_names = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="topicNames") + """An array of topic names.""" + + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + +class UserStatusOrder(sgqlc.types.Input): + """Ordering options for user status connections.""" + + __schema__ = github_schema + __field_names__ = ("field", "direction") + field = sgqlc.types.Field(sgqlc.types.non_null(UserStatusOrderField), graphql_name="field") + """The field to order user statuses by.""" + + direction = sgqlc.types.Field(sgqlc.types.non_null(OrderDirection), graphql_name="direction") + """The ordering direction.""" + + +class VerifiableDomainOrder(sgqlc.types.Input): + """Ordering options for verifiable domain connections.""" + + __schema__ = github_schema + __field_names__ = ("field", "direction") + field = sgqlc.types.Field(sgqlc.types.non_null(VerifiableDomainOrderField), graphql_name="field") + """The field to order verifiable domains by.""" + + direction = sgqlc.types.Field(sgqlc.types.non_null(OrderDirection), graphql_name="direction") + """The ordering direction.""" + + +class VerifyVerifiableDomainInput(sgqlc.types.Input): + """Autogenerated input type of VerifyVerifiableDomain""" + + __schema__ = github_schema + __field_names__ = ("id", "client_mutation_id") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + """The ID of the verifiable domain to verify.""" + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" -class UpdateSponsorshipPreferencesInput(sgqlc.types.Input): - """Autogenerated input type of UpdateSponsorshipPreferences""" +class WorkflowRunOrder(sgqlc.types.Input): + """Ways in which lists of workflow runs can be ordered upon return.""" + + __schema__ = github_schema + __field_names__ = ("field", "direction") + field = sgqlc.types.Field(sgqlc.types.non_null(WorkflowRunOrderField), graphql_name="field") + """The field by which to order workflows.""" + + direction = sgqlc.types.Field(sgqlc.types.non_null(OrderDirection), graphql_name="direction") + """The direction in which to order workflow runs by the specified + field. + """ + + +######################################################################## +# Output Objects and Interfaces +######################################################################## +class Actor(sgqlc.types.Interface): + """Represents an object which can take actions on GitHub. Typically a + User or Bot. + """ + + __schema__ = github_schema + __field_names__ = ("avatar_url", "login", "resource_path", "url") + avatar_url = sgqlc.types.Field( + sgqlc.types.non_null(URI), + graphql_name="avatarUrl", + args=sgqlc.types.ArgDict((("size", sgqlc.types.Arg(Int, graphql_name="size", default=None)),)), + ) + """A URL pointing to the actor's public avatar. + + Arguments: + + * `size` (`Int`): The size of the resulting square image. + """ + + login = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="login") + """The username of the actor.""" + + resource_path = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="resourcePath") + """The HTTP path for this actor.""" + + url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="url") + """The HTTP URL for this actor.""" + + +class AnnouncementBanner(sgqlc.types.Interface): + """Represents an announcement banner.""" + + __schema__ = github_schema + __field_names__ = ("announcement", "announcement_expires_at", "announcement_user_dismissible") + announcement = sgqlc.types.Field(String, graphql_name="announcement") + """The text of the announcement""" + + announcement_expires_at = sgqlc.types.Field(DateTime, graphql_name="announcementExpiresAt") + """The expiration date of the announcement, if any""" + + announcement_user_dismissible = sgqlc.types.Field(Boolean, graphql_name="announcementUserDismissible") + """Whether the announcement can be dismissed by the user""" + + +class Assignable(sgqlc.types.Interface): + """An object that can have users assigned to it.""" + + __schema__ = github_schema + __field_names__ = ("assignees",) + assignees = sgqlc.types.Field( + sgqlc.types.non_null("UserConnection"), + graphql_name="assignees", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), + ) + """A list of Users assigned to this object. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + """ + + +class AuditEntry(sgqlc.types.Interface): + """An entry in the audit log.""" + + __schema__ = github_schema + __field_names__ = ( + "action", + "actor", + "actor_ip", + "actor_location", + "actor_login", + "actor_resource_path", + "actor_url", + "created_at", + "operation_type", + "user", + "user_login", + "user_resource_path", + "user_url", + ) + action = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="action") + """The action name""" + + actor = sgqlc.types.Field("AuditEntryActor", graphql_name="actor") + """The user who initiated the action""" + + actor_ip = sgqlc.types.Field(String, graphql_name="actorIp") + """The IP address of the actor""" + + actor_location = sgqlc.types.Field("ActorLocation", graphql_name="actorLocation") + """A readable representation of the actor's location""" + + actor_login = sgqlc.types.Field(String, graphql_name="actorLogin") + """The username of the user who initiated the action""" + + actor_resource_path = sgqlc.types.Field(URI, graphql_name="actorResourcePath") + """The HTTP path for the actor.""" + + actor_url = sgqlc.types.Field(URI, graphql_name="actorUrl") + """The HTTP URL for the actor.""" + + created_at = sgqlc.types.Field(sgqlc.types.non_null(PreciseDateTime), graphql_name="createdAt") + """The time the action was initiated""" + + operation_type = sgqlc.types.Field(OperationType, graphql_name="operationType") + """The corresponding operation type for the action""" + + user = sgqlc.types.Field("User", graphql_name="user") + """The user affected by the action""" + + user_login = sgqlc.types.Field(String, graphql_name="userLogin") + """For actions involving two users, the actor is the initiator and + the user is the affected user. + """ + + user_resource_path = sgqlc.types.Field(URI, graphql_name="userResourcePath") + """The HTTP path for the user.""" + + user_url = sgqlc.types.Field(URI, graphql_name="userUrl") + """The HTTP URL for the user.""" + + +class Closable(sgqlc.types.Interface): + """An object that can be closed""" + + __schema__ = github_schema + __field_names__ = ("closed", "closed_at", "viewer_can_close", "viewer_can_reopen") + closed = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="closed") + """Indicates if the object is closed (definition of closed may depend + on type) + """ + + closed_at = sgqlc.types.Field(DateTime, graphql_name="closedAt") + """Identifies the date and time when the object was closed.""" + + viewer_can_close = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanClose") + """Indicates if the object can be closed by the viewer.""" + + viewer_can_reopen = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanReopen") + """Indicates if the object can be reopened by the viewer.""" + + +class Comment(sgqlc.types.Interface): + """Represents a comment.""" + + __schema__ = github_schema + __field_names__ = ( + "author", + "author_association", + "body", + "body_html", + "body_text", + "created_at", + "created_via_email", + "editor", + "id", + "includes_created_edit", + "last_edited_at", + "published_at", + "updated_at", + "user_content_edits", + "viewer_did_author", + ) + author = sgqlc.types.Field(Actor, graphql_name="author") + """The actor who authored the comment.""" + + author_association = sgqlc.types.Field(sgqlc.types.non_null(CommentAuthorAssociation), graphql_name="authorAssociation") + """Author's association with the subject of the comment.""" + + body = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="body") + """The body as Markdown.""" + + body_html = sgqlc.types.Field(sgqlc.types.non_null(HTML), graphql_name="bodyHTML") + """The body rendered to HTML.""" + + body_text = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="bodyText") + """The body rendered to text.""" + + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + """Identifies the date and time when the object was created.""" + + created_via_email = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="createdViaEmail") + """Check if this comment was created via an email reply.""" + + editor = sgqlc.types.Field(Actor, graphql_name="editor") + """The actor who edited the comment.""" + + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + includes_created_edit = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="includesCreatedEdit") + """Check if this comment was edited and includes an edit with the + creation data + """ + + last_edited_at = sgqlc.types.Field(DateTime, graphql_name="lastEditedAt") + """The moment the editor made the last edit""" + + published_at = sgqlc.types.Field(DateTime, graphql_name="publishedAt") + """Identifies when the comment was published at.""" + + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + """Identifies the date and time when the object was last updated.""" + + user_content_edits = sgqlc.types.Field( + "UserContentEditConnection", + graphql_name="userContentEdits", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), + ) + """A list of edits to this content. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + """ + + viewer_did_author = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerDidAuthor") + """Did the viewer author this comment.""" + + +class Contribution(sgqlc.types.Interface): + """Represents a contribution a user made on GitHub, such as opening + an issue. + """ + + __schema__ = github_schema + __field_names__ = ("is_restricted", "occurred_at", "resource_path", "url", "user") + is_restricted = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isRestricted") + """Whether this contribution is associated with a record you do not + have access to. For example, your own 'first issue' contribution + may have been made on a repository you can no longer access. + """ + + occurred_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="occurredAt") + """When this contribution was made.""" + + resource_path = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="resourcePath") + """The HTTP path for this contribution.""" + + url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="url") + """The HTTP URL for this contribution.""" + + user = sgqlc.types.Field(sgqlc.types.non_null("User"), graphql_name="user") + """The user who made this contribution.""" + + +class Deletable(sgqlc.types.Interface): + """Entities that can be deleted.""" + + __schema__ = github_schema + __field_names__ = ("viewer_can_delete",) + viewer_can_delete = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanDelete") + """Check if the current viewer can delete this object.""" + + +class EnterpriseAuditEntryData(sgqlc.types.Interface): + """Metadata for an audit entry containing enterprise account + information. + """ + + __schema__ = github_schema + __field_names__ = ("enterprise_resource_path", "enterprise_slug", "enterprise_url") + enterprise_resource_path = sgqlc.types.Field(URI, graphql_name="enterpriseResourcePath") + """The HTTP path for this enterprise.""" + + enterprise_slug = sgqlc.types.Field(String, graphql_name="enterpriseSlug") + """The slug of the enterprise.""" + + enterprise_url = sgqlc.types.Field(URI, graphql_name="enterpriseUrl") + """The HTTP URL for this enterprise.""" + + +class GitObject(sgqlc.types.Interface): + """Represents a Git object.""" + + __schema__ = github_schema + __field_names__ = ("abbreviated_oid", "commit_resource_path", "commit_url", "id", "oid", "repository") + abbreviated_oid = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="abbreviatedOid") + """An abbreviated version of the Git object ID""" + + commit_resource_path = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="commitResourcePath") + """The HTTP path for this Git object""" + + commit_url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="commitUrl") + """The HTTP URL for this Git object""" + + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + oid = sgqlc.types.Field(sgqlc.types.non_null(GitObjectID), graphql_name="oid") + """The Git object ID""" + + repository = sgqlc.types.Field(sgqlc.types.non_null("Repository"), graphql_name="repository") + """The Repository the Git object belongs to""" + + +class GitSignature(sgqlc.types.Interface): + """Information about a signature (GPG or S/MIME) on a Commit or Tag.""" + + __schema__ = github_schema + __field_names__ = ("email", "is_valid", "payload", "signature", "signer", "state", "was_signed_by_git_hub") + email = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="email") + """Email used to sign this object.""" + + is_valid = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isValid") + """True if the signature is valid and verified by GitHub.""" + + payload = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="payload") + """Payload for GPG signing object. Raw ODB object without the + signature header. + """ + + signature = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="signature") + """ASCII-armored signature header from object.""" + + signer = sgqlc.types.Field("User", graphql_name="signer") + """GitHub user corresponding to the email signing this commit.""" + + state = sgqlc.types.Field(sgqlc.types.non_null(GitSignatureState), graphql_name="state") + """The state of this signature. `VALID` if signature is valid and + verified by GitHub, otherwise represents reason why signature is + considered invalid. + """ + + was_signed_by_git_hub = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="wasSignedByGitHub") + """True if the signature was made with GitHub's signing key.""" + + +class HovercardContext(sgqlc.types.Interface): + """An individual line of a hovercard""" + + __schema__ = github_schema + __field_names__ = ("message", "octicon") + message = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="message") + """A string describing this context""" + + octicon = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="octicon") + """An octicon to accompany this context""" + + +class Labelable(sgqlc.types.Interface): + """An object that can have labels assigned to it.""" + + __schema__ = github_schema + __field_names__ = ("labels",) + labels = sgqlc.types.Field( + "LabelConnection", + graphql_name="labels", + args=sgqlc.types.ArgDict( + ( + ("order_by", sgqlc.types.Arg(LabelOrder, graphql_name="orderBy", default={"field": "CREATED_AT", "direction": "ASC"})), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), + ) + """A list of labels associated with the object. + + Arguments: + + * `order_by` (`LabelOrder`): Ordering options for labels returned + from the connection. (default: `{field: CREATED_AT, direction: + ASC}`) + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + """ + + +class Lockable(sgqlc.types.Interface): + """An object that can be locked.""" + + __schema__ = github_schema + __field_names__ = ("active_lock_reason", "locked") + active_lock_reason = sgqlc.types.Field(LockReason, graphql_name="activeLockReason") + """Reason that the conversation was locked.""" + + locked = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="locked") + """`true` if the object is locked""" + + +class MemberStatusable(sgqlc.types.Interface): + """Entities that have members who can set status messages.""" + + __schema__ = github_schema + __field_names__ = ("member_statuses",) + member_statuses = sgqlc.types.Field( + sgqlc.types.non_null("UserStatusConnection"), + graphql_name="memberStatuses", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ( + "order_by", + sgqlc.types.Arg(UserStatusOrder, graphql_name="orderBy", default={"field": "UPDATED_AT", "direction": "DESC"}), + ), + ) + ), + ) + """Get the status messages members of this entity have set that are + either public or visible only to the organization. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `order_by` (`UserStatusOrder`): Ordering options for user + statuses returned from the connection. (default: `{field: + UPDATED_AT, direction: DESC}`) + """ + + +class Migration(sgqlc.types.Interface): + """Represents a GitHub Enterprise Importer (GEI) migration.""" + + __schema__ = github_schema + __field_names__ = ( + "continue_on_error", + "created_at", + "database_id", + "failure_reason", + "id", + "migration_log_url", + "migration_source", + "repository_name", + "source_url", + "state", + "warnings_count", + ) + continue_on_error = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="continueOnError") + """The migration flag to continue on error.""" + + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + """Identifies the date and time when the object was created.""" + + database_id = sgqlc.types.Field(String, graphql_name="databaseId") + """Identifies the primary key from the database.""" + + failure_reason = sgqlc.types.Field(String, graphql_name="failureReason") + """The reason the migration failed.""" + + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + migration_log_url = sgqlc.types.Field(URI, graphql_name="migrationLogUrl") + """The URL for the migration log (expires 1 day after migration + completes). + """ + + migration_source = sgqlc.types.Field(sgqlc.types.non_null("MigrationSource"), graphql_name="migrationSource") + """The migration source.""" + + repository_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="repositoryName") + """The target repository name.""" + + source_url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="sourceUrl") + """The migration source URL, for example `https://github.com` or + `https://monalisa.ghe.com`. + """ + + state = sgqlc.types.Field(sgqlc.types.non_null(MigrationState), graphql_name="state") + """The migration state.""" + + warnings_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="warningsCount") + """The number of warnings encountered for this migration. To review + the warnings, check the [Migration + Log](https://docs.github.com/en/migrations/using-github- + enterprise-importer/completing-your-migration-with-github- + enterprise-importer/accessing-your-migration-logs-for-github- + enterprise-importer). + """ + + +class Minimizable(sgqlc.types.Interface): + """Entities that can be minimized.""" + + __schema__ = github_schema + __field_names__ = ("is_minimized", "minimized_reason", "viewer_can_minimize") + is_minimized = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isMinimized") + """Returns whether or not a comment has been minimized.""" + + minimized_reason = sgqlc.types.Field(String, graphql_name="minimizedReason") + """Returns why the comment was minimized. One of `abuse`, `off- + topic`, `outdated`, `resolved`, `duplicate` and `spam`. Note that + the case and formatting of these values differs from the inputs to + the `MinimizeComment` mutation. + """ + + viewer_can_minimize = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanMinimize") + """Check if the current viewer can minimize this object.""" + + +class Node(sgqlc.types.Interface): + """An object with an ID.""" + + __schema__ = github_schema + __field_names__ = ("id",) + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + """ID of the object.""" + + +class OauthApplicationAuditEntryData(sgqlc.types.Interface): + """Metadata for an audit entry with action oauth_application.*""" + + __schema__ = github_schema + __field_names__ = ("oauth_application_name", "oauth_application_resource_path", "oauth_application_url") + oauth_application_name = sgqlc.types.Field(String, graphql_name="oauthApplicationName") + """The name of the OAuth application.""" + + oauth_application_resource_path = sgqlc.types.Field(URI, graphql_name="oauthApplicationResourcePath") + """The HTTP path for the OAuth application""" + + oauth_application_url = sgqlc.types.Field(URI, graphql_name="oauthApplicationUrl") + """The HTTP URL for the OAuth application""" + + +class OrganizationAuditEntryData(sgqlc.types.Interface): + """Metadata for an audit entry with action org.*""" + + __schema__ = github_schema + __field_names__ = ("organization", "organization_name", "organization_resource_path", "organization_url") + organization = sgqlc.types.Field("Organization", graphql_name="organization") + """The Organization associated with the Audit Entry.""" + + organization_name = sgqlc.types.Field(String, graphql_name="organizationName") + """The name of the Organization.""" + + organization_resource_path = sgqlc.types.Field(URI, graphql_name="organizationResourcePath") + """The HTTP path for the organization""" + + organization_url = sgqlc.types.Field(URI, graphql_name="organizationUrl") + """The HTTP URL for the organization""" + + +class PackageOwner(sgqlc.types.Interface): + """Represents an owner of a package.""" + + __schema__ = github_schema + __field_names__ = ("id", "packages") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + packages = sgqlc.types.Field( + sgqlc.types.non_null("PackageConnection"), + graphql_name="packages", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("names", sgqlc.types.Arg(sgqlc.types.list_of(String), graphql_name="names", default=None)), + ("repository_id", sgqlc.types.Arg(ID, graphql_name="repositoryId", default=None)), + ("package_type", sgqlc.types.Arg(PackageType, graphql_name="packageType", default=None)), + ("order_by", sgqlc.types.Arg(PackageOrder, graphql_name="orderBy", default={"field": "CREATED_AT", "direction": "DESC"})), + ) + ), + ) + """A list of packages under the owner. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `names` (`[String]`): Find packages by their names. + * `repository_id` (`ID`): Find packages in a repository by ID. + * `package_type` (`PackageType`): Filter registry package by type. + * `order_by` (`PackageOrder`): Ordering of the returned packages. + (default: `{field: CREATED_AT, direction: DESC}`) + """ + + +class ProfileOwner(sgqlc.types.Interface): + """Represents any entity on GitHub that has a profile page.""" + + __schema__ = github_schema + __field_names__ = ( + "any_pinnable_items", + "email", + "id", + "item_showcase", + "location", + "login", + "name", + "pinnable_items", + "pinned_items", + "pinned_items_remaining", + "viewer_can_change_pinned_items", + "website_url", + ) + any_pinnable_items = sgqlc.types.Field( + sgqlc.types.non_null(Boolean), + graphql_name="anyPinnableItems", + args=sgqlc.types.ArgDict((("type", sgqlc.types.Arg(PinnableItemType, graphql_name="type", default=None)),)), + ) + """Determine if this repository owner has any items that can be + pinned to their profile. + + Arguments: + + * `type` (`PinnableItemType`): Filter to only a particular kind of + pinnable item. + """ + + email = sgqlc.types.Field(String, graphql_name="email") + """The public profile email.""" + + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + item_showcase = sgqlc.types.Field(sgqlc.types.non_null("ProfileItemShowcase"), graphql_name="itemShowcase") + """Showcases a selection of repositories and gists that the profile + owner has either curated or that have been selected automatically + based on popularity. + """ + + location = sgqlc.types.Field(String, graphql_name="location") + """The public profile location.""" + + login = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="login") + """The username used to login.""" + + name = sgqlc.types.Field(String, graphql_name="name") + """The public profile name.""" + + pinnable_items = sgqlc.types.Field( + sgqlc.types.non_null("PinnableItemConnection"), + graphql_name="pinnableItems", + args=sgqlc.types.ArgDict( + ( + ("types", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(PinnableItemType)), graphql_name="types", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), + ) + """A list of repositories and gists this profile owner can pin to + their profile. + + Arguments: + + * `types` (`[PinnableItemType!]`): Filter the types of pinnable + items that are returned. + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + """ + + pinned_items = sgqlc.types.Field( + sgqlc.types.non_null("PinnableItemConnection"), + graphql_name="pinnedItems", + args=sgqlc.types.ArgDict( + ( + ("types", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(PinnableItemType)), graphql_name="types", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), + ) + """A list of repositories and gists this profile owner has pinned to + their profile + + Arguments: + + * `types` (`[PinnableItemType!]`): Filter the types of pinned + items that are returned. + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + """ + + pinned_items_remaining = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="pinnedItemsRemaining") + """Returns how many more items this profile owner can pin to their + profile. + """ + + viewer_can_change_pinned_items = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanChangePinnedItems") + """Can the viewer pin repositories and gists to the profile?""" + + website_url = sgqlc.types.Field(URI, graphql_name="websiteUrl") + """The public profile website URL.""" + + +class ProjectOwner(sgqlc.types.Interface): + """Represents an owner of a Project.""" + + __schema__ = github_schema + __field_names__ = ("id", "project", "projects", "projects_resource_path", "projects_url", "viewer_can_create_projects") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + project = sgqlc.types.Field( + "Project", + graphql_name="project", + args=sgqlc.types.ArgDict((("number", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="number", default=None)),)), + ) + """Find project by number. + + Arguments: + + * `number` (`Int!`): The project number to find. + """ + + projects = sgqlc.types.Field( + sgqlc.types.non_null("ProjectConnection"), + graphql_name="projects", + args=sgqlc.types.ArgDict( + ( + ("order_by", sgqlc.types.Arg(ProjectOrder, graphql_name="orderBy", default=None)), + ("search", sgqlc.types.Arg(String, graphql_name="search", default=None)), + ("states", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ProjectState)), graphql_name="states", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), + ) + """A list of projects under the owner. + + Arguments: + + * `order_by` (`ProjectOrder`): Ordering options for projects + returned from the connection + * `search` (`String`): Query to search projects by, currently only + searching by name. + * `states` (`[ProjectState!]`): A list of states to filter the + projects by. + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + """ + + projects_resource_path = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="projectsResourcePath") + """The HTTP path listing owners projects""" + + projects_url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="projectsUrl") + """The HTTP URL listing owners projects""" + + viewer_can_create_projects = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanCreateProjects") + """Can the current viewer create new projects on this owner.""" + + +class ProjectV2FieldCommon(sgqlc.types.Interface): + """Common fields across different project field types""" + + __schema__ = github_schema + __field_names__ = ("created_at", "data_type", "database_id", "id", "name", "project", "updated_at") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + """Identifies the date and time when the object was created.""" + + data_type = sgqlc.types.Field(sgqlc.types.non_null(ProjectV2FieldType), graphql_name="dataType") + """The field's type.""" + + database_id = sgqlc.types.Field(Int, graphql_name="databaseId") + """Identifies the primary key from the database.""" + + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + """The project field's name.""" + + project = sgqlc.types.Field(sgqlc.types.non_null("ProjectV2"), graphql_name="project") + """The project that contains this field.""" + + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + """Identifies the date and time when the object was last updated.""" + + +class ProjectV2ItemFieldValueCommon(sgqlc.types.Interface): + """Common fields across different project field value types""" + + __schema__ = github_schema + __field_names__ = ("created_at", "creator", "database_id", "field", "id", "item", "updated_at") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + """Identifies the date and time when the object was created.""" + + creator = sgqlc.types.Field(Actor, graphql_name="creator") + """The actor who created the item.""" + + database_id = sgqlc.types.Field(Int, graphql_name="databaseId") + """Identifies the primary key from the database.""" + + field = sgqlc.types.Field(sgqlc.types.non_null("ProjectV2FieldConfiguration"), graphql_name="field") + """The project field that contains this value.""" + + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + item = sgqlc.types.Field(sgqlc.types.non_null("ProjectV2Item"), graphql_name="item") + """The project item that contains this value.""" + + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + """Identifies the date and time when the object was last updated.""" + + +class ProjectV2Owner(sgqlc.types.Interface): + """Represents an owner of a project (beta).""" + + __schema__ = github_schema + __field_names__ = ("id", "project_v2", "projects_v2") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + project_v2 = sgqlc.types.Field( + "ProjectV2", + graphql_name="projectV2", + args=sgqlc.types.ArgDict((("number", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="number", default=None)),)), + ) + """Find a project by number. + + Arguments: + + * `number` (`Int!`): The project number. + """ + + projects_v2 = sgqlc.types.Field( + sgqlc.types.non_null("ProjectV2Connection"), + graphql_name="projectsV2", + args=sgqlc.types.ArgDict( + ( + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("order_by", sgqlc.types.Arg(ProjectV2Order, graphql_name="orderBy", default={"field": "NUMBER", "direction": "DESC"})), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), + ) + """A list of projects under the owner. + + Arguments: + + * `query` (`String`): A project to search for under the the owner. + * `order_by` (`ProjectV2Order`): How to order the returned + projects. (default: `{field: NUMBER, direction: DESC}`) + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + """ + + +class ProjectV2Recent(sgqlc.types.Interface): + """Recent projects for the owner.""" + + __schema__ = github_schema + __field_names__ = ("recent_projects",) + recent_projects = sgqlc.types.Field( + sgqlc.types.non_null("ProjectV2Connection"), + graphql_name="recentProjects", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), + ) + """Recent projects that this user has modified in the context of the + owner. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + """ + + +class Reactable(sgqlc.types.Interface): + """Represents a subject that can be reacted on.""" + + __schema__ = github_schema + __field_names__ = ("database_id", "id", "reaction_groups", "reactions", "viewer_can_react") + database_id = sgqlc.types.Field(Int, graphql_name="databaseId") + """Identifies the primary key from the database.""" + + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + reaction_groups = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("ReactionGroup")), graphql_name="reactionGroups") + """A list of reactions grouped by content left on the subject.""" + + reactions = sgqlc.types.Field( + sgqlc.types.non_null("ReactionConnection"), + graphql_name="reactions", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("content", sgqlc.types.Arg(ReactionContent, graphql_name="content", default=None)), + ("order_by", sgqlc.types.Arg(ReactionOrder, graphql_name="orderBy", default=None)), + ) + ), + ) + """A list of Reactions left on the Issue. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `content` (`ReactionContent`): Allows filtering Reactions by + emoji. + * `order_by` (`ReactionOrder`): Allows specifying the order in + which reactions are returned. + """ + + viewer_can_react = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanReact") + """Can user react to this subject""" + + +class RepositoryAuditEntryData(sgqlc.types.Interface): + """Metadata for an audit entry with action repo.*""" + + __schema__ = github_schema + __field_names__ = ("repository", "repository_name", "repository_resource_path", "repository_url") + repository = sgqlc.types.Field("Repository", graphql_name="repository") + """The repository associated with the action""" + + repository_name = sgqlc.types.Field(String, graphql_name="repositoryName") + """The name of the repository""" + + repository_resource_path = sgqlc.types.Field(URI, graphql_name="repositoryResourcePath") + """The HTTP path for the repository""" + + repository_url = sgqlc.types.Field(URI, graphql_name="repositoryUrl") + """The HTTP URL for the repository""" + + +class RepositoryDiscussionAuthor(sgqlc.types.Interface): + """Represents an author of discussions in repositories.""" + + __schema__ = github_schema + __field_names__ = ("repository_discussions",) + repository_discussions = sgqlc.types.Field( + sgqlc.types.non_null("DiscussionConnection"), + graphql_name="repositoryDiscussions", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ( + "order_by", + sgqlc.types.Arg(DiscussionOrder, graphql_name="orderBy", default={"field": "CREATED_AT", "direction": "DESC"}), + ), + ("repository_id", sgqlc.types.Arg(ID, graphql_name="repositoryId", default=None)), + ("answered", sgqlc.types.Arg(Boolean, graphql_name="answered", default=None)), + ("states", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(DiscussionState)), graphql_name="states", default=())), + ) + ), + ) + """Discussions this user has started. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `order_by` (`DiscussionOrder`): Ordering options for discussions + returned from the connection. (default: `{field: CREATED_AT, + direction: DESC}`) + * `repository_id` (`ID`): Filter discussions to only those in a + specific repository. + * `answered` (`Boolean`): Filter discussions to only those that + have been answered or not. Defaults to including both answered + and unanswered discussions. (default: `null`) + * `states` (`[DiscussionState!]`): A list of states to filter the + discussions by. (default: `[]`) + """ + + +class RepositoryDiscussionCommentAuthor(sgqlc.types.Interface): + """Represents an author of discussion comments in repositories.""" + + __schema__ = github_schema + __field_names__ = ("repository_discussion_comments",) + repository_discussion_comments = sgqlc.types.Field( + sgqlc.types.non_null("DiscussionCommentConnection"), + graphql_name="repositoryDiscussionComments", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("repository_id", sgqlc.types.Arg(ID, graphql_name="repositoryId", default=None)), + ("only_answers", sgqlc.types.Arg(Boolean, graphql_name="onlyAnswers", default=False)), + ) + ), + ) + """Discussion comments this user has authored. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `repository_id` (`ID`): Filter discussion comments to only those + in a specific repository. + * `only_answers` (`Boolean`): Filter discussion comments to only + those that were marked as the answer (default: `false`) + """ + + +class RepositoryInfo(sgqlc.types.Interface): + """A subset of repository info.""" + + __schema__ = github_schema + __field_names__ = ( + "archived_at", + "created_at", + "description", + "description_html", + "fork_count", + "has_discussions_enabled", + "has_issues_enabled", + "has_projects_enabled", + "has_wiki_enabled", + "homepage_url", + "is_archived", + "is_fork", + "is_in_organization", + "is_locked", + "is_mirror", + "is_private", + "is_template", + "license_info", + "lock_reason", + "mirror_url", + "name", + "name_with_owner", + "open_graph_image_url", + "owner", + "pushed_at", + "resource_path", + "short_description_html", + "updated_at", + "url", + "uses_custom_open_graph_image", + "visibility", + ) + archived_at = sgqlc.types.Field(DateTime, graphql_name="archivedAt") + """Identifies the date and time when the repository was archived.""" + + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + """Identifies the date and time when the object was created.""" + + description = sgqlc.types.Field(String, graphql_name="description") + """The description of the repository.""" + + description_html = sgqlc.types.Field(sgqlc.types.non_null(HTML), graphql_name="descriptionHTML") + """The description of the repository rendered to HTML.""" + + fork_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="forkCount") + """Returns how many forks there are of this repository in the whole + network. + """ + + has_discussions_enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasDiscussionsEnabled") + """Indicates if the repository has the Discussions feature enabled.""" + + has_issues_enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasIssuesEnabled") + """Indicates if the repository has issues feature enabled.""" + + has_projects_enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasProjectsEnabled") + """Indicates if the repository has the Projects feature enabled.""" + + has_wiki_enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasWikiEnabled") + """Indicates if the repository has wiki feature enabled.""" + + homepage_url = sgqlc.types.Field(URI, graphql_name="homepageUrl") + """The repository's URL.""" + + is_archived = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isArchived") + """Indicates if the repository is unmaintained.""" + + is_fork = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isFork") + """Identifies if the repository is a fork.""" + + is_in_organization = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isInOrganization") + """Indicates if a repository is either owned by an organization, or + is a private fork of an organization repository. + """ + + is_locked = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isLocked") + """Indicates if the repository has been locked or not.""" + + is_mirror = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isMirror") + """Identifies if the repository is a mirror.""" + + is_private = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isPrivate") + """Identifies if the repository is private or internal.""" + + is_template = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isTemplate") + """Identifies if the repository is a template that can be used to + generate new repositories. + """ + + license_info = sgqlc.types.Field("License", graphql_name="licenseInfo") + """The license associated with the repository""" + + lock_reason = sgqlc.types.Field(RepositoryLockReason, graphql_name="lockReason") + """The reason the repository has been locked.""" + + mirror_url = sgqlc.types.Field(URI, graphql_name="mirrorUrl") + """The repository's original mirror URL.""" + + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + """The name of the repository.""" + + name_with_owner = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="nameWithOwner") + """The repository's name with owner.""" + + open_graph_image_url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="openGraphImageUrl") + """The image used to represent this repository in Open Graph data.""" + + owner = sgqlc.types.Field(sgqlc.types.non_null("RepositoryOwner"), graphql_name="owner") + """The User owner of the repository.""" + + pushed_at = sgqlc.types.Field(DateTime, graphql_name="pushedAt") + """Identifies the date and time when the repository was last pushed + to. + """ + + resource_path = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="resourcePath") + """The HTTP path for this repository""" + + short_description_html = sgqlc.types.Field( + sgqlc.types.non_null(HTML), + graphql_name="shortDescriptionHTML", + args=sgqlc.types.ArgDict((("limit", sgqlc.types.Arg(Int, graphql_name="limit", default=200)),)), + ) + """A description of the repository, rendered to HTML without any + links in it. + + Arguments: + + * `limit` (`Int`): How many characters to return. (default: `200`) + """ + + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + """Identifies the date and time when the object was last updated.""" + + url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="url") + """The HTTP URL for this repository""" + + uses_custom_open_graph_image = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="usesCustomOpenGraphImage") + """Whether this repository has a custom image to use with Open Graph + as opposed to being represented by the owner's avatar. + """ + + visibility = sgqlc.types.Field(sgqlc.types.non_null(RepositoryVisibility), graphql_name="visibility") + """Indicates the repository's visibility level.""" + + +class RepositoryNode(sgqlc.types.Interface): + """Represents a object that belongs to a repository.""" + + __schema__ = github_schema + __field_names__ = ("repository",) + repository = sgqlc.types.Field(sgqlc.types.non_null("Repository"), graphql_name="repository") + """The repository associated with this node.""" + + +class RepositoryOwner(sgqlc.types.Interface): + """Represents an owner of a Repository.""" + + __schema__ = github_schema + __field_names__ = ("avatar_url", "id", "login", "repositories", "repository", "resource_path", "url") + avatar_url = sgqlc.types.Field( + sgqlc.types.non_null(URI), + graphql_name="avatarUrl", + args=sgqlc.types.ArgDict((("size", sgqlc.types.Arg(Int, graphql_name="size", default=None)),)), + ) + """A URL pointing to the owner's public avatar. + + Arguments: + + * `size` (`Int`): The size of the resulting square image. + """ + + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + login = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="login") + """The username used to login.""" + + repositories = sgqlc.types.Field( + sgqlc.types.non_null("RepositoryConnection"), + graphql_name="repositories", + args=sgqlc.types.ArgDict( + ( + ("privacy", sgqlc.types.Arg(RepositoryPrivacy, graphql_name="privacy", default=None)), + ("order_by", sgqlc.types.Arg(RepositoryOrder, graphql_name="orderBy", default=None)), + ("affiliations", sgqlc.types.Arg(sgqlc.types.list_of(RepositoryAffiliation), graphql_name="affiliations", default=None)), + ( + "owner_affiliations", + sgqlc.types.Arg( + sgqlc.types.list_of(RepositoryAffiliation), graphql_name="ownerAffiliations", default=("OWNER", "COLLABORATOR") + ), + ), + ("is_locked", sgqlc.types.Arg(Boolean, graphql_name="isLocked", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("is_fork", sgqlc.types.Arg(Boolean, graphql_name="isFork", default=None)), + ) + ), + ) + """A list of repositories that the user owns. + + Arguments: + + * `privacy` (`RepositoryPrivacy`): If non-null, filters + repositories according to privacy + * `order_by` (`RepositoryOrder`): Ordering options for + repositories returned from the connection + * `affiliations` (`[RepositoryAffiliation]`): Array of viewer's + affiliation options for repositories returned from the + connection. For example, OWNER will include only repositories + that the current viewer owns. + * `owner_affiliations` (`[RepositoryAffiliation]`): Array of + owner's affiliation options for repositories returned from the + connection. For example, OWNER will include only repositories + that the organization or user being viewed owns. (default: + `[OWNER, COLLABORATOR]`) + * `is_locked` (`Boolean`): If non-null, filters repositories + according to whether they have been locked + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `is_fork` (`Boolean`): If non-null, filters repositories + according to whether they are forks of another repository + """ + + repository = sgqlc.types.Field( + "Repository", + graphql_name="repository", + args=sgqlc.types.ArgDict( + ( + ("name", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="name", default=None)), + ("follow_renames", sgqlc.types.Arg(Boolean, graphql_name="followRenames", default=True)), + ) + ), + ) + """Find Repository. + + Arguments: + + * `name` (`String!`): Name of Repository to find. + * `follow_renames` (`Boolean`): Follow repository renames. If + disabled, a repository referenced by its old name will return an + error. (default: `true`) + """ + + resource_path = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="resourcePath") + """The HTTP URL for the owner.""" + + url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="url") + """The HTTP URL for the owner.""" + + +class RequirableByPullRequest(sgqlc.types.Interface): + """Represents a type that can be required by a pull request for + merging. + """ + + __schema__ = github_schema + __field_names__ = ("is_required",) + is_required = sgqlc.types.Field( + sgqlc.types.non_null(Boolean), + graphql_name="isRequired", + args=sgqlc.types.ArgDict( + ( + ("pull_request_id", sgqlc.types.Arg(ID, graphql_name="pullRequestId", default=None)), + ("pull_request_number", sgqlc.types.Arg(Int, graphql_name="pullRequestNumber", default=None)), + ) + ), + ) + """Whether this is required to pass before merging for a specific + pull request. + + Arguments: + + * `pull_request_id` (`ID`): The id of the pull request this is + required for + * `pull_request_number` (`Int`): The number of the pull request + this is required for + """ + + +class Sponsorable(sgqlc.types.Interface): + """Entities that can sponsor or be sponsored through GitHub Sponsors.""" + + __schema__ = github_schema + __field_names__ = ( + "estimated_next_sponsors_payout_in_cents", + "has_sponsors_listing", + "is_sponsored_by", + "is_sponsoring_viewer", + "monthly_estimated_sponsors_income_in_cents", + "sponsoring", + "sponsors", + "sponsors_activities", + "sponsors_listing", + "sponsorship_for_viewer_as_sponsor", + "sponsorship_for_viewer_as_sponsorable", + "sponsorship_newsletters", + "sponsorships_as_maintainer", + "sponsorships_as_sponsor", + "total_sponsorship_amount_as_sponsor_in_cents", + "viewer_can_sponsor", + "viewer_is_sponsoring", + ) + estimated_next_sponsors_payout_in_cents = sgqlc.types.Field( + sgqlc.types.non_null(Int), graphql_name="estimatedNextSponsorsPayoutInCents" + ) + """The estimated next GitHub Sponsors payout for this + user/organization in cents (USD). + """ + + has_sponsors_listing = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasSponsorsListing") + """True if this user/organization has a GitHub Sponsors listing.""" + + is_sponsored_by = sgqlc.types.Field( + sgqlc.types.non_null(Boolean), + graphql_name="isSponsoredBy", + args=sgqlc.types.ArgDict( + (("account_login", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="accountLogin", default=None)),) + ), + ) + """Whether the given account is sponsoring this user/organization. + + Arguments: + + * `account_login` (`String!`): The target account's login. + """ + + is_sponsoring_viewer = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isSponsoringViewer") + """True if the viewer is sponsored by this user/organization.""" + + monthly_estimated_sponsors_income_in_cents = sgqlc.types.Field( + sgqlc.types.non_null(Int), graphql_name="monthlyEstimatedSponsorsIncomeInCents" + ) + """The estimated monthly GitHub Sponsors income for this + user/organization in cents (USD). + """ + + sponsoring = sgqlc.types.Field( + sgqlc.types.non_null("SponsorConnection"), + graphql_name="sponsoring", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("order_by", sgqlc.types.Arg(SponsorOrder, graphql_name="orderBy", default={"field": "RELEVANCE", "direction": "DESC"})), + ) + ), + ) + """List of users and organizations this entity is sponsoring. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `order_by` (`SponsorOrder`): Ordering options for the users and + organizations returned from the connection. (default: `{field: + RELEVANCE, direction: DESC}`) + """ + + sponsors = sgqlc.types.Field( + sgqlc.types.non_null("SponsorConnection"), + graphql_name="sponsors", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("tier_id", sgqlc.types.Arg(ID, graphql_name="tierId", default=None)), + ("order_by", sgqlc.types.Arg(SponsorOrder, graphql_name="orderBy", default={"field": "RELEVANCE", "direction": "DESC"})), + ) + ), + ) + """List of sponsors for this user or organization. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `tier_id` (`ID`): If given, will filter for sponsors at the + given tier. Will only return sponsors whose tier the viewer is + permitted to see. + * `order_by` (`SponsorOrder`): Ordering options for sponsors + returned from the connection. (default: `{field: RELEVANCE, + direction: DESC}`) + """ + + sponsors_activities = sgqlc.types.Field( + sgqlc.types.non_null("SponsorsActivityConnection"), + graphql_name="sponsorsActivities", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("period", sgqlc.types.Arg(SponsorsActivityPeriod, graphql_name="period", default="MONTH")), + ("since", sgqlc.types.Arg(DateTime, graphql_name="since", default=None)), + ("until", sgqlc.types.Arg(DateTime, graphql_name="until", default=None)), + ( + "order_by", + sgqlc.types.Arg(SponsorsActivityOrder, graphql_name="orderBy", default={"field": "TIMESTAMP", "direction": "DESC"}), + ), + ( + "actions", + sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(SponsorsActivityAction)), graphql_name="actions", default=()), + ), + ("include_as_sponsor", sgqlc.types.Arg(Boolean, graphql_name="includeAsSponsor", default=False)), + ) + ), + ) + """Events involving this sponsorable, such as new sponsorships. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `period` (`SponsorsActivityPeriod`): Filter activities returned + to only those that occurred in the most recent specified time + period. Set to ALL to avoid filtering by when the activity + occurred. Will be ignored if `since` or `until` is given. + (default: `MONTH`) + * `since` (`DateTime`): Filter activities to those that occurred + on or after this time. + * `until` (`DateTime`): Filter activities to those that occurred + before this time. + * `order_by` (`SponsorsActivityOrder`): Ordering options for + activity returned from the connection. (default: `{field: + TIMESTAMP, direction: DESC}`) + * `actions` (`[SponsorsActivityAction!]`): Filter activities to + only the specified actions. (default: `[]`) + * `include_as_sponsor` (`Boolean`): Whether to include those + events where this sponsorable acted as the sponsor. Defaults to + only including events where this sponsorable was the recipient + of a sponsorship. (default: `false`) + """ + + sponsors_listing = sgqlc.types.Field("SponsorsListing", graphql_name="sponsorsListing") + """The GitHub Sponsors listing for this user or organization.""" - __schema__ = github_schema - __field_names__ = ( - "sponsor_id", - "sponsor_login", - "sponsorable_id", - "sponsorable_login", - "receive_emails", - "privacy_level", - "client_mutation_id", + sponsorship_for_viewer_as_sponsor = sgqlc.types.Field( + "Sponsorship", + graphql_name="sponsorshipForViewerAsSponsor", + args=sgqlc.types.ArgDict((("active_only", sgqlc.types.Arg(Boolean, graphql_name="activeOnly", default=True)),)), ) - sponsor_id = sgqlc.types.Field(ID, graphql_name="sponsorId") - """The ID of the user or organization who is acting as the sponsor, - paying for the sponsorship. Required if sponsorLogin is not given. - """ + """The sponsorship from the viewer to this user/organization; that + is, the sponsorship where you're the sponsor. - sponsor_login = sgqlc.types.Field(String, graphql_name="sponsorLogin") - """The username of the user or organization who is acting as the - sponsor, paying for the sponsorship. Required if sponsorId is not - given. - """ + Arguments: - sponsorable_id = sgqlc.types.Field(ID, graphql_name="sponsorableId") - """The ID of the user or organization who is receiving the - sponsorship. Required if sponsorableLogin is not given. + * `active_only` (`Boolean`): Whether to return the sponsorship + only if it's still active. Pass false to get the viewer's + sponsorship back even if it has been cancelled. (default: + `true`) """ - sponsorable_login = sgqlc.types.Field(String, graphql_name="sponsorableLogin") - """The username of the user or organization who is receiving the - sponsorship. Required if sponsorableId is not given. + sponsorship_for_viewer_as_sponsorable = sgqlc.types.Field( + "Sponsorship", + graphql_name="sponsorshipForViewerAsSponsorable", + args=sgqlc.types.ArgDict((("active_only", sgqlc.types.Arg(Boolean, graphql_name="activeOnly", default=True)),)), + ) + """The sponsorship from this user/organization to the viewer; that + is, the sponsorship you're receiving. + + Arguments: + + * `active_only` (`Boolean`): Whether to return the sponsorship + only if it's still active. Pass false to get the sponsorship + back even if it has been cancelled. (default: `true`) """ - receive_emails = sgqlc.types.Field(Boolean, graphql_name="receiveEmails") - """Whether the sponsor should receive email updates from the - sponsorable. + sponsorship_newsletters = sgqlc.types.Field( + sgqlc.types.non_null("SponsorshipNewsletterConnection"), + graphql_name="sponsorshipNewsletters", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ( + "order_by", + sgqlc.types.Arg( + SponsorshipNewsletterOrder, graphql_name="orderBy", default={"field": "CREATED_AT", "direction": "DESC"} + ), + ), + ) + ), + ) + """List of sponsorship updates sent from this sponsorable to + sponsors. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `order_by` (`SponsorshipNewsletterOrder`): Ordering options for + sponsorship updates returned from the connection. (default: + `{field: CREATED_AT, direction: DESC}`) """ - privacy_level = sgqlc.types.Field(SponsorshipPrivacy, graphql_name="privacyLevel") - """Specify whether others should be able to see that the sponsor is - sponsoring the sponsorable. Public visibility still does not - reveal which tier is used. + sponsorships_as_maintainer = sgqlc.types.Field( + sgqlc.types.non_null("SponsorshipConnection"), + graphql_name="sponsorshipsAsMaintainer", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("include_private", sgqlc.types.Arg(Boolean, graphql_name="includePrivate", default=False)), + ("order_by", sgqlc.types.Arg(SponsorshipOrder, graphql_name="orderBy", default=None)), + ("active_only", sgqlc.types.Arg(Boolean, graphql_name="activeOnly", default=True)), + ) + ), + ) + """The sponsorships where this user or organization is the maintainer + receiving the funds. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `include_private` (`Boolean`): Whether or not to include private + sponsorships in the result set (default: `false`) + * `order_by` (`SponsorshipOrder`): Ordering options for + sponsorships returned from this connection. If left blank, the + sponsorships will be ordered based on relevancy to the viewer. + * `active_only` (`Boolean`): Whether to include only sponsorships + that are active right now, versus all sponsorships this + maintainer has ever received. (default: `true`) """ - client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") - """A unique identifier for the client performing the mutation.""" + sponsorships_as_sponsor = sgqlc.types.Field( + sgqlc.types.non_null("SponsorshipConnection"), + graphql_name="sponsorshipsAsSponsor", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("order_by", sgqlc.types.Arg(SponsorshipOrder, graphql_name="orderBy", default=None)), + ( + "maintainer_logins", + sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="maintainerLogins", default=None), + ), + ("active_only", sgqlc.types.Arg(Boolean, graphql_name="activeOnly", default=True)), + ) + ), + ) + """The sponsorships where this user or organization is the funder. + Arguments: -class UpdateSubscriptionInput(sgqlc.types.Input): - """Autogenerated input type of UpdateSubscription""" + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `order_by` (`SponsorshipOrder`): Ordering options for + sponsorships returned from this connection. If left blank, the + sponsorships will be ordered based on relevancy to the viewer. + * `maintainer_logins` (`[String!]`): Filter sponsorships returned + to those for the specified maintainers. That is, the recipient + of the sponsorship is a user or organization with one of the + given logins. + * `active_only` (`Boolean`): Whether to include only sponsorships + that are active right now, versus all sponsorships this sponsor + has ever made. (default: `true`) + """ - __schema__ = github_schema - __field_names__ = ("subscribable_id", "state", "client_mutation_id") - subscribable_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="subscribableId") - """The Node ID of the subscribable object to modify.""" + total_sponsorship_amount_as_sponsor_in_cents = sgqlc.types.Field( + Int, + graphql_name="totalSponsorshipAmountAsSponsorInCents", + args=sgqlc.types.ArgDict( + ( + ("since", sgqlc.types.Arg(DateTime, graphql_name="since", default=None)), + ("until", sgqlc.types.Arg(DateTime, graphql_name="until", default=None)), + ( + "sponsorable_logins", + sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="sponsorableLogins", default=()), + ), + ) + ), + ) + """The amount in United States cents (e.g., 500 = $5.00 USD) that + this entity has spent on GitHub to fund sponsorships. Only returns + a value when viewed by the user themselves or by a user who can + manage sponsorships for the requested organization. - state = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionState), graphql_name="state") - """The new state of the subscription.""" + Arguments: - client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") - """A unique identifier for the client performing the mutation.""" + * `since` (`DateTime`): Filter payments to those that occurred on + or after this time. + * `until` (`DateTime`): Filter payments to those that occurred + before this time. + * `sponsorable_logins` (`[String!]`): Filter payments to those + made to the users or organizations with the specified usernames. + (default: `[]`) + """ + viewer_can_sponsor = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanSponsor") + """Whether or not the viewer is able to sponsor this + user/organization. + """ -class UpdateTeamDiscussionCommentInput(sgqlc.types.Input): - """Autogenerated input type of UpdateTeamDiscussionComment""" + viewer_is_sponsoring = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerIsSponsoring") + """True if the viewer is sponsoring this user/organization.""" + + +class Starrable(sgqlc.types.Interface): + """Things that can be starred.""" __schema__ = github_schema - __field_names__ = ("id", "body", "body_version", "client_mutation_id") + __field_names__ = ("id", "stargazer_count", "stargazers", "viewer_has_starred") id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - """The ID of the comment to modify.""" - body = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="body") - """The updated text of the comment.""" + stargazer_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="stargazerCount") + """Returns a count of how many stargazers there are on this object""" - body_version = sgqlc.types.Field(String, graphql_name="bodyVersion") - """The current version of the body content.""" + stargazers = sgqlc.types.Field( + sgqlc.types.non_null("StargazerConnection"), + graphql_name="stargazers", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("order_by", sgqlc.types.Arg(StarOrder, graphql_name="orderBy", default=None)), + ) + ), + ) + """A list of users who have starred this starrable. - client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") - """A unique identifier for the client performing the mutation.""" + Arguments: + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `order_by` (`StarOrder`): Order for connection + """ -class UpdateTeamDiscussionInput(sgqlc.types.Input): - """Autogenerated input type of UpdateTeamDiscussion""" + viewer_has_starred = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerHasStarred") + """Returns a boolean indicating whether the viewing user has starred + this starrable. + """ - __schema__ = github_schema - __field_names__ = ("id", "title", "body", "body_version", "pinned", "client_mutation_id") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - """The Node ID of the discussion to modify.""" - title = sgqlc.types.Field(String, graphql_name="title") - """The updated title of the discussion.""" +class Subscribable(sgqlc.types.Interface): + """Entities that can be subscribed to for web and email + notifications. + """ - body = sgqlc.types.Field(String, graphql_name="body") - """The updated text of the discussion.""" + __schema__ = github_schema + __field_names__ = ("id", "viewer_can_subscribe", "viewer_subscription") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - body_version = sgqlc.types.Field(String, graphql_name="bodyVersion") - """The current version of the body content. If provided, this update - operation will be rejected if the given version does not match the - latest version on the server. + viewer_can_subscribe = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanSubscribe") + """Check if the viewer is able to change their subscription status + for the repository. """ - pinned = sgqlc.types.Field(Boolean, graphql_name="pinned") - """If provided, sets the pinned state of the updated discussion.""" - - client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") - """A unique identifier for the client performing the mutation.""" + viewer_subscription = sgqlc.types.Field(SubscriptionState, graphql_name="viewerSubscription") + """Identifies if the viewer is watching, not watching, or ignoring + the subscribable entity. + """ -class UpdateTeamsRepositoryInput(sgqlc.types.Input): - """Autogenerated input type of UpdateTeamsRepository""" +class TeamAuditEntryData(sgqlc.types.Interface): + """Metadata for an audit entry with action team.*""" __schema__ = github_schema - __field_names__ = ("repository_id", "team_ids", "permission", "client_mutation_id") - repository_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="repositoryId") - """Repository ID being granted access to.""" + __field_names__ = ("team", "team_name", "team_resource_path", "team_url") + team = sgqlc.types.Field("Team", graphql_name="team") + """The team associated with the action""" - team_ids = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="teamIds") - """A list of teams being granted access. Limit: 10""" + team_name = sgqlc.types.Field(String, graphql_name="teamName") + """The name of the team""" - permission = sgqlc.types.Field(sgqlc.types.non_null(RepositoryPermission), graphql_name="permission") - """Permission that should be granted to the teams.""" + team_resource_path = sgqlc.types.Field(URI, graphql_name="teamResourcePath") + """The HTTP path for this team""" - client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") - """A unique identifier for the client performing the mutation.""" + team_url = sgqlc.types.Field(URI, graphql_name="teamUrl") + """The HTTP URL for this team""" -class UpdateTopicsInput(sgqlc.types.Input): - """Autogenerated input type of UpdateTopics""" +class TopicAuditEntryData(sgqlc.types.Interface): + """Metadata for an audit entry with a topic.""" __schema__ = github_schema - __field_names__ = ("repository_id", "topic_names", "client_mutation_id") - repository_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="repositoryId") - """The Node ID of the repository.""" - - topic_names = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="topicNames") - """An array of topic names.""" + __field_names__ = ("topic", "topic_name") + topic = sgqlc.types.Field("Topic", graphql_name="topic") + """The name of the topic added to the repository""" - client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") - """A unique identifier for the client performing the mutation.""" + topic_name = sgqlc.types.Field(String, graphql_name="topicName") + """The name of the topic added to the repository""" -class UserStatusOrder(sgqlc.types.Input): - """Ordering options for user status connections.""" +class UniformResourceLocatable(sgqlc.types.Interface): + """Represents a type that can be retrieved by a URL.""" __schema__ = github_schema - __field_names__ = ("field", "direction") - field = sgqlc.types.Field(sgqlc.types.non_null(UserStatusOrderField), graphql_name="field") - """The field to order user statuses by.""" + __field_names__ = ("resource_path", "url") + resource_path = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="resourcePath") + """The HTML path to this resource.""" - direction = sgqlc.types.Field(sgqlc.types.non_null(OrderDirection), graphql_name="direction") - """The ordering direction.""" + url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="url") + """The URL to this resource.""" -class VerifiableDomainOrder(sgqlc.types.Input): - """Ordering options for verifiable domain connections.""" +class Updatable(sgqlc.types.Interface): + """Entities that can be updated.""" __schema__ = github_schema - __field_names__ = ("field", "direction") - field = sgqlc.types.Field(sgqlc.types.non_null(VerifiableDomainOrderField), graphql_name="field") - """The field to order verifiable domains by.""" + __field_names__ = ("viewer_can_update",) + viewer_can_update = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanUpdate") + """Check if the current viewer can update this object.""" - direction = sgqlc.types.Field(sgqlc.types.non_null(OrderDirection), graphql_name="direction") - """The ordering direction.""" +class UpdatableComment(sgqlc.types.Interface): + """Comments that can be updated.""" + + __schema__ = github_schema + __field_names__ = ("viewer_cannot_update_reasons",) + viewer_cannot_update_reasons = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CommentCannotUpdateReason))), graphql_name="viewerCannotUpdateReasons" + ) + """Reasons why the current viewer can not update this comment.""" -class VerifyVerifiableDomainInput(sgqlc.types.Input): - """Autogenerated input type of VerifyVerifiableDomain""" + +class Votable(sgqlc.types.Interface): + """A subject that may be upvoted.""" __schema__ = github_schema - __field_names__ = ("id", "client_mutation_id") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - """The ID of the verifiable domain to verify.""" + __field_names__ = ("upvote_count", "viewer_can_upvote", "viewer_has_upvoted") + upvote_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="upvoteCount") + """Number of upvotes that this subject has received.""" - client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") - """A unique identifier for the client performing the mutation.""" + viewer_can_upvote = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanUpvote") + """Whether or not the current user can add or remove an upvote on + this subject. + """ + + viewer_has_upvoted = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerHasUpvoted") + """Whether or not the current user has already upvoted this subject.""" -######################################################################## -# Output Objects and Interfaces -######################################################################## class AbortQueuedMigrationsPayload(sgqlc.types.Type): """Autogenerated return type of AbortQueuedMigrations""" @@ -7987,35 +12399,6 @@ class AcceptTopicSuggestionPayload(sgqlc.types.Type): """The accepted topic.""" -class Actor(sgqlc.types.Interface): - """Represents an object which can take actions on GitHub. Typically a - User or Bot. - """ - - __schema__ = github_schema - __field_names__ = ("avatar_url", "login", "resource_path", "url") - avatar_url = sgqlc.types.Field( - sgqlc.types.non_null(URI), - graphql_name="avatarUrl", - args=sgqlc.types.ArgDict((("size", sgqlc.types.Arg(Int, graphql_name="size", default=None)),)), - ) - """A URL pointing to the actor's public avatar. - - Arguments: - - * `size` (`Int`): The size of the resulting square image. - """ - - login = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="login") - """The username of the actor.""" - - resource_path = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="resourcePath") - """The HTTP path for this actor.""" - - url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="url") - """The HTTP URL for this actor.""" - - class ActorLocation(sgqlc.types.Type): """Location information for an actor""" @@ -8042,7 +12425,7 @@ class AddAssigneesToAssignablePayload(sgqlc.types.Type): __schema__ = github_schema __field_names__ = ("assignable", "client_mutation_id") - assignable = sgqlc.types.Field("Assignable", graphql_name="assignable") + assignable = sgqlc.types.Field(Assignable, graphql_name="assignable") """The item that was assigned.""" client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") @@ -8060,7 +12443,7 @@ class AddCommentPayload(sgqlc.types.Type): comment_edge = sgqlc.types.Field("IssueCommentEdge", graphql_name="commentEdge") """The edge from the subject's comment connection.""" - subject = sgqlc.types.Field("Node", graphql_name="subject") + subject = sgqlc.types.Field(Node, graphql_name="subject") """The subject""" timeline_edge = sgqlc.types.Field("IssueTimelineItemEdge", graphql_name="timelineEdge") @@ -8091,6 +12474,18 @@ class AddDiscussionPollVotePayload(sgqlc.types.Type): """The poll option that a vote was added to.""" +class AddEnterpriseOrganizationMemberPayload(sgqlc.types.Type): + """Autogenerated return type of AddEnterpriseOrganizationMember""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "users") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + users = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("User")), graphql_name="users") + """The users who were added to the organization.""" + + class AddEnterpriseSupportEntitlementPayload(sgqlc.types.Type): """Autogenerated return type of AddEnterpriseSupportEntitlement""" @@ -8111,7 +12506,7 @@ class AddLabelsToLabelablePayload(sgqlc.types.Type): client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" - labelable = sgqlc.types.Field("Labelable", graphql_name="labelable") + labelable = sgqlc.types.Field(Labelable, graphql_name="labelable") """The item that was labeled.""" @@ -8145,27 +12540,27 @@ class AddProjectColumnPayload(sgqlc.types.Type): """The project""" -class AddProjectDraftIssuePayload(sgqlc.types.Type): - """Autogenerated return type of AddProjectDraftIssue""" +class AddProjectV2DraftIssuePayload(sgqlc.types.Type): + """Autogenerated return type of AddProjectV2DraftIssue""" __schema__ = github_schema - __field_names__ = ("client_mutation_id", "project_next_item") + __field_names__ = ("client_mutation_id", "project_item") client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" - project_next_item = sgqlc.types.Field("ProjectNextItem", graphql_name="projectNextItem") + project_item = sgqlc.types.Field("ProjectV2Item", graphql_name="projectItem") """The draft issue added to the project.""" -class AddProjectNextItemPayload(sgqlc.types.Type): - """Autogenerated return type of AddProjectNextItem""" +class AddProjectV2ItemByIdPayload(sgqlc.types.Type): + """Autogenerated return type of AddProjectV2ItemById""" __schema__ = github_schema - __field_names__ = ("client_mutation_id", "project_next_item") + __field_names__ = ("client_mutation_id", "item") client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" - project_next_item = sgqlc.types.Field("ProjectNextItem", graphql_name="projectNextItem") + item = sgqlc.types.Field("ProjectV2Item", graphql_name="item") """The item added to the project.""" @@ -8215,14 +12610,17 @@ class AddReactionPayload(sgqlc.types.Type): """Autogenerated return type of AddReaction""" __schema__ = github_schema - __field_names__ = ("client_mutation_id", "reaction", "subject") + __field_names__ = ("client_mutation_id", "reaction", "reaction_groups", "subject") client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" reaction = sgqlc.types.Field("Reaction", graphql_name="reaction") """The reaction object.""" - subject = sgqlc.types.Field("Reactable", graphql_name="subject") + reaction_groups = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("ReactionGroup")), graphql_name="reactionGroups") + """The reaction groups for the subject.""" + + subject = sgqlc.types.Field(Reactable, graphql_name="subject") """The reactable subject.""" @@ -8234,7 +12632,7 @@ class AddStarPayload(sgqlc.types.Type): client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" - starrable = sgqlc.types.Field("Starrable", graphql_name="starrable") + starrable = sgqlc.types.Field(Starrable, graphql_name="starrable") """The starrable.""" @@ -8246,7 +12644,7 @@ class AddUpvotePayload(sgqlc.types.Type): client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" - subject = sgqlc.types.Field("Votable", graphql_name="subject") + subject = sgqlc.types.Field(Votable, graphql_name="subject") """The votable subject.""" @@ -8286,107 +12684,28 @@ class ApproveVerifiableDomainPayload(sgqlc.types.Type): """The verifiable domain that was approved.""" -class ArchiveRepositoryPayload(sgqlc.types.Type): - """Autogenerated return type of ArchiveRepository""" +class ArchiveProjectV2ItemPayload(sgqlc.types.Type): + """Autogenerated return type of ArchiveProjectV2Item""" __schema__ = github_schema - __field_names__ = ("client_mutation_id", "repository") + __field_names__ = ("client_mutation_id", "item") client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" - repository = sgqlc.types.Field("Repository", graphql_name="repository") - """The repository that was marked as archived.""" - - -class Assignable(sgqlc.types.Interface): - """An object that can have users assigned to it.""" - - __schema__ = github_schema - __field_names__ = ("assignees",) - assignees = sgqlc.types.Field( - sgqlc.types.non_null("UserConnection"), - graphql_name="assignees", - args=sgqlc.types.ArgDict( - ( - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ) - ), - ) - """A list of Users assigned to this object. - - Arguments: - - * `after` (`String`): Returns the elements in the list that come - after the specified cursor. - * `before` (`String`): Returns the elements in the list that come - before the specified cursor. - * `first` (`Int`): Returns the first _n_ elements from the list. - * `last` (`Int`): Returns the last _n_ elements from the list. - """ + item = sgqlc.types.Field("ProjectV2Item", graphql_name="item") + """The item archived from the project.""" -class AuditEntry(sgqlc.types.Interface): - """An entry in the audit log.""" +class ArchiveRepositoryPayload(sgqlc.types.Type): + """Autogenerated return type of ArchiveRepository""" __schema__ = github_schema - __field_names__ = ( - "action", - "actor", - "actor_ip", - "actor_location", - "actor_login", - "actor_resource_path", - "actor_url", - "created_at", - "operation_type", - "user", - "user_login", - "user_resource_path", - "user_url", - ) - action = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="action") - """The action name""" - - actor = sgqlc.types.Field("AuditEntryActor", graphql_name="actor") - """The user who initiated the action""" - - actor_ip = sgqlc.types.Field(String, graphql_name="actorIp") - """The IP address of the actor""" - - actor_location = sgqlc.types.Field(ActorLocation, graphql_name="actorLocation") - """A readable representation of the actor's location""" - - actor_login = sgqlc.types.Field(String, graphql_name="actorLogin") - """The username of the user who initiated the action""" - - actor_resource_path = sgqlc.types.Field(URI, graphql_name="actorResourcePath") - """The HTTP path for the actor.""" - - actor_url = sgqlc.types.Field(URI, graphql_name="actorUrl") - """The HTTP URL for the actor.""" - - created_at = sgqlc.types.Field(sgqlc.types.non_null(PreciseDateTime), graphql_name="createdAt") - """The time the action was initiated""" - - operation_type = sgqlc.types.Field(OperationType, graphql_name="operationType") - """The corresponding operation type for the action""" - - user = sgqlc.types.Field("User", graphql_name="user") - """The user affected by the action""" - - user_login = sgqlc.types.Field(String, graphql_name="userLogin") - """For actions involving two users, the actor is the initiator and - the user is the affected user. - """ - - user_resource_path = sgqlc.types.Field(URI, graphql_name="userResourcePath") - """The HTTP path for the user.""" + __field_names__ = ("client_mutation_id", "repository") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" - user_url = sgqlc.types.Field(URI, graphql_name="userUrl") - """The HTTP URL for the user.""" + repository = sgqlc.types.Field("Repository", graphql_name="repository") + """The repository that was marked as archived.""" class AutoMergeRequest(sgqlc.types.Type): @@ -8456,6 +12775,24 @@ class BlameRange(sgqlc.types.Type): """The starting line for the range""" +class BranchNamePatternParameters(sgqlc.types.Type): + """Parameters to be used for the branch_name_pattern rule""" + + __schema__ = github_schema + __field_names__ = ("name", "negate", "operator", "pattern") + name = sgqlc.types.Field(String, graphql_name="name") + """How this rule will appear to users.""" + + negate = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="negate") + """If true, the rule will fail if the pattern matches.""" + + operator = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="operator") + """The operator to use for matching.""" + + pattern = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="pattern") + """The pattern to match with.""" + + class BranchProtectionRuleConflict(sgqlc.types.Type): """A conflict between two branch protection rules.""" @@ -8788,6 +13125,18 @@ class CheckRunEdge(sgqlc.types.Type): """The item at the end of the edge.""" +class CheckRunStateCount(sgqlc.types.Type): + """Represents a count of the state of a check run.""" + + __schema__ = github_schema + __field_names__ = ("count", "state") + count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="count") + """The number of check runs with this state.""" + + state = sgqlc.types.Field(sgqlc.types.non_null(CheckRunState), graphql_name="state") + """The state of a check run.""" + + class CheckStep(sgqlc.types.Type): """A single check step.""" @@ -8888,10 +13237,22 @@ class ClearLabelsFromLabelablePayload(sgqlc.types.Type): client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" - labelable = sgqlc.types.Field("Labelable", graphql_name="labelable") + labelable = sgqlc.types.Field(Labelable, graphql_name="labelable") """The item that was unlabeled.""" +class ClearProjectV2ItemFieldValuePayload(sgqlc.types.Type): + """Autogenerated return type of ClearProjectV2ItemFieldValue""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "project_v2_item") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + project_v2_item = sgqlc.types.Field("ProjectV2Item", graphql_name="projectV2Item") + """The updated item.""" + + class CloneProjectPayload(sgqlc.types.Type): """Autogenerated return type of CloneProject""" @@ -8919,18 +13280,16 @@ class CloneTemplateRepositoryPayload(sgqlc.types.Type): """The new repository.""" -class Closable(sgqlc.types.Interface): - """An object that can be closed""" +class CloseDiscussionPayload(sgqlc.types.Type): + """Autogenerated return type of CloseDiscussion""" __schema__ = github_schema - __field_names__ = ("closed", "closed_at") - closed = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="closed") - """`true` if the object is closed (definition of closed may depend on - type) - """ + __field_names__ = ("client_mutation_id", "discussion") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" - closed_at = sgqlc.types.Field(DateTime, graphql_name="closedAt") - """Identifies the date and time when the object was closed.""" + discussion = sgqlc.types.Field("Discussion", graphql_name="discussion") + """The discussion that was closed.""" class CloseIssuePayload(sgqlc.types.Type): @@ -8957,93 +13316,22 @@ class ClosePullRequestPayload(sgqlc.types.Type): """The pull request that was closed.""" -class Comment(sgqlc.types.Interface): - """Represents a comment.""" +class CommitAuthorEmailPatternParameters(sgqlc.types.Type): + """Parameters to be used for the commit_author_email_pattern rule""" __schema__ = github_schema - __field_names__ = ( - "author", - "author_association", - "body", - "body_html", - "body_text", - "created_at", - "created_via_email", - "editor", - "id", - "includes_created_edit", - "last_edited_at", - "published_at", - "updated_at", - "user_content_edits", - "viewer_did_author", - ) - author = sgqlc.types.Field(Actor, graphql_name="author") - """The actor who authored the comment.""" - - author_association = sgqlc.types.Field(sgqlc.types.non_null(CommentAuthorAssociation), graphql_name="authorAssociation") - """Author's association with the subject of the comment.""" - - body = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="body") - """The body as Markdown.""" - - body_html = sgqlc.types.Field(sgqlc.types.non_null(HTML), graphql_name="bodyHTML") - """The body rendered to HTML.""" - - body_text = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="bodyText") - """The body rendered to text.""" - - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - """Identifies the date and time when the object was created.""" - - created_via_email = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="createdViaEmail") - """Check if this comment was created via an email reply.""" - - editor = sgqlc.types.Field(Actor, graphql_name="editor") - """The actor who edited the comment.""" - - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - - includes_created_edit = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="includesCreatedEdit") - """Check if this comment was edited and includes an edit with the - creation data - """ - - last_edited_at = sgqlc.types.Field(DateTime, graphql_name="lastEditedAt") - """The moment the editor made the last edit""" - - published_at = sgqlc.types.Field(DateTime, graphql_name="publishedAt") - """Identifies when the comment was published at.""" - - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - """Identifies the date and time when the object was last updated.""" - - user_content_edits = sgqlc.types.Field( - "UserContentEditConnection", - graphql_name="userContentEdits", - args=sgqlc.types.ArgDict( - ( - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ) - ), - ) - """A list of edits to this content. + __field_names__ = ("name", "negate", "operator", "pattern") + name = sgqlc.types.Field(String, graphql_name="name") + """How this rule will appear to users.""" - Arguments: + negate = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="negate") + """If true, the rule will fail if the pattern matches.""" - * `after` (`String`): Returns the elements in the list that come - after the specified cursor. - * `before` (`String`): Returns the elements in the list that come - before the specified cursor. - * `first` (`Int`): Returns the first _n_ elements from the list. - * `last` (`Int`): Returns the last _n_ elements from the list. - """ + operator = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="operator") + """The operator to use for matching.""" - viewer_did_author = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerDidAuthor") - """Did the viewer author this comment.""" + pattern = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="pattern") + """The pattern to match with.""" class CommitCommentConnection(sgqlc.types.relay.Connection): @@ -9174,30 +13462,61 @@ class CommitHistoryConnection(sgqlc.types.relay.Connection): """Identifies the total count of items in the connection.""" -class Contribution(sgqlc.types.Interface): - """Represents a contribution a user made on GitHub, such as opening - an issue. - """ +class CommitMessagePatternParameters(sgqlc.types.Type): + """Parameters to be used for the commit_message_pattern rule""" __schema__ = github_schema - __field_names__ = ("is_restricted", "occurred_at", "resource_path", "url", "user") - is_restricted = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isRestricted") - """Whether this contribution is associated with a record you do not - have access to. For example, your own 'first issue' contribution - may have been made on a repository you can no longer access. - """ + __field_names__ = ("name", "negate", "operator", "pattern") + name = sgqlc.types.Field(String, graphql_name="name") + """How this rule will appear to users.""" - occurred_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="occurredAt") - """When this contribution was made.""" + negate = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="negate") + """If true, the rule will fail if the pattern matches.""" - resource_path = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="resourcePath") - """The HTTP path for this contribution.""" + operator = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="operator") + """The operator to use for matching.""" - url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="url") - """The HTTP URL for this contribution.""" + pattern = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="pattern") + """The pattern to match with.""" - user = sgqlc.types.Field(sgqlc.types.non_null("User"), graphql_name="user") - """The user who made this contribution.""" + +class CommitterEmailPatternParameters(sgqlc.types.Type): + """Parameters to be used for the committer_email_pattern rule""" + + __schema__ = github_schema + __field_names__ = ("name", "negate", "operator", "pattern") + name = sgqlc.types.Field(String, graphql_name="name") + """How this rule will appear to users.""" + + negate = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="negate") + """If true, the rule will fail if the pattern matches.""" + + operator = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="operator") + """The operator to use for matching.""" + + pattern = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="pattern") + """The pattern to match with.""" + + +class ComparisonCommitConnection(sgqlc.types.relay.Connection): + """The connection type for Commit.""" + + __schema__ = github_schema + __field_names__ = ("author_count", "edges", "nodes", "page_info", "total_count") + author_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="authorCount") + """The total count of authors and co-authors across all commits.""" + + edges = sgqlc.types.Field(sgqlc.types.list_of(CommitEdge), graphql_name="edges") + """A list of edges.""" + + nodes = sgqlc.types.Field(sgqlc.types.list_of("Commit"), graphql_name="nodes") + """A list of nodes.""" + + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + """Information to aid in pagination.""" + + total_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalCount") + """Identifies the total count of items in the connection.""" class ContributionCalendar(sgqlc.types.Type): @@ -9581,7 +13900,8 @@ class ContributionsCollection(sgqlc.types.Type): ) ), ) - """Pull request review contributions made by the user. + """Pull request review contributions made by the user. Returns the + most recently submitted review for each PR reviewed by the user. Arguments: @@ -9792,6 +14112,36 @@ class ConvertPullRequestToDraftPayload(sgqlc.types.Type): """The pull request that is now a draft.""" +class CopyProjectV2Payload(sgqlc.types.Type): + """Autogenerated return type of CopyProjectV2""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "project_v2") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + project_v2 = sgqlc.types.Field("ProjectV2", graphql_name="projectV2") + """The copied project.""" + + +class CreateAttributionInvitationPayload(sgqlc.types.Type): + """Autogenerated return type of CreateAttributionInvitation""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "owner", "source", "target") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + owner = sgqlc.types.Field("Organization", graphql_name="owner") + """The owner scoping the reattributable data.""" + + source = sgqlc.types.Field("Claimable", graphql_name="source") + """The account owning the data to reattribute.""" + + target = sgqlc.types.Field("Claimable", graphql_name="target") + """The account which may claim the data.""" + + class CreateBranchProtectionRulePayload(sgqlc.types.Type): """Autogenerated return type of CreateBranchProtectionRule""" @@ -9906,6 +14256,18 @@ class CreateIssuePayload(sgqlc.types.Type): """The new issue.""" +class CreateLinkedBranchPayload(sgqlc.types.Type): + """Autogenerated return type of CreateLinkedBranch""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "linked_branch") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + linked_branch = sgqlc.types.Field("LinkedBranch", graphql_name="linkedBranch") + """The new branch issue reference.""" + + class CreateMigrationSourcePayload(sgqlc.types.Type): """Autogenerated return type of CreateMigrationSource""" @@ -9915,7 +14277,7 @@ class CreateMigrationSourcePayload(sgqlc.types.Type): """A unique identifier for the client performing the mutation.""" migration_source = sgqlc.types.Field("MigrationSource", graphql_name="migrationSource") - """The created Octoshift migration source.""" + """The created migration source.""" class CreateProjectPayload(sgqlc.types.Type): @@ -9930,6 +14292,30 @@ class CreateProjectPayload(sgqlc.types.Type): """The new project.""" +class CreateProjectV2FieldPayload(sgqlc.types.Type): + """Autogenerated return type of CreateProjectV2Field""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "project_v2_field") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + project_v2_field = sgqlc.types.Field("ProjectV2FieldConfiguration", graphql_name="projectV2Field") + """The new field.""" + + +class CreateProjectV2Payload(sgqlc.types.Type): + """Autogenerated return type of CreateProjectV2""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "project_v2") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + project_v2 = sgqlc.types.Field("ProjectV2", graphql_name="projectV2") + """The new project.""" + + class CreatePullRequestPayload(sgqlc.types.Type): """Autogenerated return type of CreatePullRequest""" @@ -9966,6 +14352,30 @@ class CreateRepositoryPayload(sgqlc.types.Type): """The new repository.""" +class CreateRepositoryRulesetPayload(sgqlc.types.Type): + """Autogenerated return type of CreateRepositoryRuleset""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "ruleset") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + ruleset = sgqlc.types.Field("RepositoryRuleset", graphql_name="ruleset") + """The newly created Ruleset.""" + + +class CreateSponsorsListingPayload(sgqlc.types.Type): + """Autogenerated return type of CreateSponsorsListing""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "sponsors_listing") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + sponsors_listing = sgqlc.types.Field("SponsorsListing", graphql_name="sponsorsListing") + """The new GitHub Sponsors profile.""" + + class CreateSponsorsTierPayload(sgqlc.types.Type): """Autogenerated return type of CreateSponsorsTier""" @@ -9990,6 +14400,18 @@ class CreateSponsorshipPayload(sgqlc.types.Type): """The sponsorship that was started.""" +class CreateSponsorshipsPayload(sgqlc.types.Type): + """Autogenerated return type of CreateSponsorships""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "sponsorables") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + sponsorables = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(Sponsorable)), graphql_name="sponsorables") + """The users and organizations who received a sponsorship.""" + + class CreateTeamDiscussionCommentPayload(sgqlc.types.Type): """Autogenerated return type of CreateTeamDiscussionComment""" @@ -10178,15 +14600,6 @@ class DeclineTopicSuggestionPayload(sgqlc.types.Type): """The declined topic.""" -class Deletable(sgqlc.types.Interface): - """Entities that can be deleted.""" - - __schema__ = github_schema - __field_names__ = ("viewer_can_delete",) - viewer_can_delete = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanDelete") - """Check if the current viewer can delete this object.""" - - class DeleteBranchProtectionRulePayload(sgqlc.types.Type): """Autogenerated return type of DeleteBranchProtectionRule""" @@ -10271,6 +14684,18 @@ class DeleteIssuePayload(sgqlc.types.Type): """The repository the issue belonged to""" +class DeleteLinkedBranchPayload(sgqlc.types.Type): + """Autogenerated return type of DeleteLinkedBranch""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "issue") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + issue = sgqlc.types.Field("Issue", graphql_name="issue") + """The issue the linked branch was unlinked from.""" + + class DeleteProjectCardPayload(sgqlc.types.Type): """Autogenerated return type of DeleteProjectCard""" @@ -10301,8 +14726,32 @@ class DeleteProjectColumnPayload(sgqlc.types.Type): """The project the deleted column was in.""" -class DeleteProjectNextItemPayload(sgqlc.types.Type): - """Autogenerated return type of DeleteProjectNextItem""" +class DeleteProjectPayload(sgqlc.types.Type): + """Autogenerated return type of DeleteProject""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "owner") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + owner = sgqlc.types.Field(ProjectOwner, graphql_name="owner") + """The repository or organization the project was removed from.""" + + +class DeleteProjectV2FieldPayload(sgqlc.types.Type): + """Autogenerated return type of DeleteProjectV2Field""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "project_v2_field") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + project_v2_field = sgqlc.types.Field("ProjectV2FieldConfiguration", graphql_name="projectV2Field") + """The deleted field.""" + + +class DeleteProjectV2ItemPayload(sgqlc.types.Type): + """Autogenerated return type of DeleteProjectV2Item""" __schema__ = github_schema __field_names__ = ("client_mutation_id", "deleted_item_id") @@ -10313,29 +14762,47 @@ class DeleteProjectNextItemPayload(sgqlc.types.Type): """The ID of the deleted item.""" -class DeleteProjectPayload(sgqlc.types.Type): - """Autogenerated return type of DeleteProject""" +class DeleteProjectV2Payload(sgqlc.types.Type): + """Autogenerated return type of DeleteProjectV2""" __schema__ = github_schema - __field_names__ = ("client_mutation_id", "owner") + __field_names__ = ("client_mutation_id", "project_v2") client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" - owner = sgqlc.types.Field("ProjectOwner", graphql_name="owner") - """The repository or organization the project was removed from.""" + project_v2 = sgqlc.types.Field("ProjectV2", graphql_name="projectV2") + """The deleted Project.""" + + +class DeleteProjectV2WorkflowPayload(sgqlc.types.Type): + """Autogenerated return type of DeleteProjectV2Workflow""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "deleted_workflow_id", "project_v2") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + deleted_workflow_id = sgqlc.types.Field(ID, graphql_name="deletedWorkflowId") + """The ID of the deleted workflow.""" + + project_v2 = sgqlc.types.Field("ProjectV2", graphql_name="projectV2") + """The project the deleted workflow was in.""" class DeletePullRequestReviewCommentPayload(sgqlc.types.Type): """Autogenerated return type of DeletePullRequestReviewComment""" __schema__ = github_schema - __field_names__ = ("client_mutation_id", "pull_request_review") + __field_names__ = ("client_mutation_id", "pull_request_review", "pull_request_review_comment") client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" pull_request_review = sgqlc.types.Field("PullRequestReview", graphql_name="pullRequestReview") """The pull request review the deleted comment belonged to.""" + pull_request_review_comment = sgqlc.types.Field("PullRequestReviewComment", graphql_name="pullRequestReviewComment") + """The deleted pull request review comment.""" + class DeletePullRequestReviewPayload(sgqlc.types.Type): """Autogenerated return type of DeletePullRequestReview""" @@ -10358,6 +14825,15 @@ class DeleteRefPayload(sgqlc.types.Type): """A unique identifier for the client performing the mutation.""" +class DeleteRepositoryRulesetPayload(sgqlc.types.Type): + """Autogenerated return type of DeleteRepositoryRuleset""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id",) + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + class DeleteTeamDiscussionCommentPayload(sgqlc.types.Type): """Autogenerated return type of DeleteTeamDiscussionComment""" @@ -10694,6 +15170,18 @@ class DeploymentStatusEdge(sgqlc.types.Type): """The item at the end of the edge.""" +class DequeuePullRequestPayload(sgqlc.types.Type): + """Autogenerated return type of DequeuePullRequest""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "merge_queue_entry") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + merge_queue_entry = sgqlc.types.Field("MergeQueueEntry", graphql_name="mergeQueueEntry") + """The merge queue entry of the dequeued pull request.""" + + class DisablePullRequestAutoMergePayload(sgqlc.types.Type): """Autogenerated return type of DisablePullRequestAutoMerge""" @@ -10868,6 +15356,18 @@ class EnablePullRequestAutoMergePayload(sgqlc.types.Type): """The pull request auto-merge was enabled on.""" +class EnqueuePullRequestPayload(sgqlc.types.Type): + """Autogenerated return type of EnqueuePullRequest""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "merge_queue_entry") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + merge_queue_entry = sgqlc.types.Field("MergeQueueEntry", graphql_name="mergeQueueEntry") + """The merge queue entry for the enqueued pull request.""" + + class EnterpriseAdministratorConnection(sgqlc.types.relay.Connection): """The connection type for User.""" @@ -10931,23 +15431,6 @@ class EnterpriseAdministratorInvitationEdge(sgqlc.types.Type): """The item at the end of the edge.""" -class EnterpriseAuditEntryData(sgqlc.types.Interface): - """Metadata for an audit entry containing enterprise account - information. - """ - - __schema__ = github_schema - __field_names__ = ("enterprise_resource_path", "enterprise_slug", "enterprise_url") - enterprise_resource_path = sgqlc.types.Field(URI, graphql_name="enterpriseResourcePath") - """The HTTP path for this enterprise.""" - - enterprise_slug = sgqlc.types.Field(String, graphql_name="enterpriseSlug") - """The slug of the enterprise.""" - - enterprise_url = sgqlc.types.Field(URI, graphql_name="enterpriseUrl") - """The HTTP URL for this enterprise.""" - - class EnterpriseBillingInfo(sgqlc.types.Type): """Enterprise billing information visible to enterprise billing managers and owners. @@ -11009,6 +15492,39 @@ class EnterpriseBillingInfo(sgqlc.types.Type): """The total number of licenses allocated.""" +class EnterpriseFailedInvitationConnection(sgqlc.types.relay.Connection): + """The connection type for OrganizationInvitation.""" + + __schema__ = github_schema + __field_names__ = ("edges", "nodes", "page_info", "total_count", "total_unique_user_count") + edges = sgqlc.types.Field(sgqlc.types.list_of("EnterpriseFailedInvitationEdge"), graphql_name="edges") + """A list of edges.""" + + nodes = sgqlc.types.Field(sgqlc.types.list_of("OrganizationInvitation"), graphql_name="nodes") + """A list of nodes.""" + + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + """Information to aid in pagination.""" + + total_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalCount") + """Identifies the total count of items in the connection.""" + + total_unique_user_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalUniqueUserCount") + """Identifies the total count of unique users in the connection.""" + + +class EnterpriseFailedInvitationEdge(sgqlc.types.Type): + """A failed invitation to be a member in an enterprise organization.""" + + __schema__ = github_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + """A cursor for use in pagination.""" + + node = sgqlc.types.Field("OrganizationInvitation", graphql_name="node") + """The item at the end of the edge.""" + + class EnterpriseMemberConnection(sgqlc.types.relay.Connection): """The connection type for EnterpriseMember.""" @@ -11134,7 +15650,10 @@ class EnterpriseOutsideCollaboratorEdge(sgqlc.types.Type): class EnterpriseOwnerInfo(sgqlc.types.Type): - """Enterprise information only visible to enterprise owners.""" + """Enterprise information visible to enterprise owners or enterprise + owners' personal access tokens (classic) with read:enterprise or + admin:enterprise scope. + """ __schema__ = github_schema __field_names__ = ( @@ -11143,10 +15662,12 @@ class EnterpriseOwnerInfo(sgqlc.types.Type): "affiliated_users_with_two_factor_disabled_exist", "allow_private_repository_forking_setting", "allow_private_repository_forking_setting_organizations", + "allow_private_repository_forking_setting_policy_value", "default_repository_permission_setting", "default_repository_permission_setting_organizations", "domains", "enterprise_server_installations", + "failed_invitations", "ip_allow_list_enabled_setting", "ip_allow_list_entries", "ip_allow_list_for_installed_apps_enabled_setting", @@ -11203,6 +15724,7 @@ class EnterpriseOwnerInfo(sgqlc.types.Type): "order_by", sgqlc.types.Arg(EnterpriseMemberOrder, graphql_name="orderBy", default={"field": "LOGIN", "direction": "ASC"}), ), + ("has_two_factor_enabled", sgqlc.types.Arg(Boolean, graphql_name="hasTwoFactorEnabled", default=None)), ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), @@ -11221,6 +15743,8 @@ class EnterpriseOwnerInfo(sgqlc.types.Type): * `order_by` (`EnterpriseMemberOrder`): Ordering options for administrators returned from the connection. (default: `{field: LOGIN, direction: ASC}`) + * `has_two_factor_enabled` (`Boolean`): Only return administrators + with this two-factor authentication status. (default: `null`) * `after` (`String`): Returns the elements in the list that come after the specified cursor. * `before` (`String`): Returns the elements in the list that come @@ -11300,6 +15824,13 @@ class EnterpriseOwnerInfo(sgqlc.types.Type): direction: ASC}`) """ + allow_private_repository_forking_setting_policy_value = sgqlc.types.Field( + EnterpriseAllowPrivateRepositoryForkingPolicyValue, graphql_name="allowPrivateRepositoryForkingSettingPolicyValue" + ) + """The value for the allow private repository forking policy on the + enterprise. + """ + default_repository_permission_setting = sgqlc.types.Field( sgqlc.types.non_null(EnterpriseDefaultRepositoryPermissionSettingValue), graphql_name="defaultRepositoryPermissionSetting" ) @@ -11357,7 +15888,9 @@ class EnterpriseOwnerInfo(sgqlc.types.Type): ) ), ) - """A list of domains owned by the enterprise. + """A list of domains owned by the enterprise. Visible to enterprise + owners or enterprise owners' personal access tokens (classic) with + admin:enterprise scope. Arguments: @@ -11412,6 +15945,32 @@ class EnterpriseOwnerInfo(sgqlc.types.Type): `{field: HOST_NAME, direction: ASC}`) """ + failed_invitations = sgqlc.types.Field( + sgqlc.types.non_null(EnterpriseFailedInvitationConnection), + graphql_name="failedInvitations", + args=sgqlc.types.ArgDict( + ( + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), + ) + """A list of failed invitations in the enterprise. + + Arguments: + + * `query` (`String`): The search string to look for. + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + """ + ip_allow_list_enabled_setting = sgqlc.types.Field( sgqlc.types.non_null(IpAllowListEnabledSettingValue), graphql_name="ipAllowListEnabledSetting" ) @@ -11438,7 +15997,8 @@ class EnterpriseOwnerInfo(sgqlc.types.Type): ), ) """The IP addresses that are allowed to access resources owned by the - enterprise. + enterprise. Visible to enterprise owners or enterprise owners' + personal access tokens (classic) with admin:enterprise scope. Arguments: @@ -11958,6 +16518,7 @@ class EnterpriseOwnerInfo(sgqlc.types.Type): "organization_logins", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="organizationLogins", default=None), ), + ("invitation_source", sgqlc.types.Arg(OrganizationInvitationSource, graphql_name="invitationSource", default=None)), ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), @@ -11973,6 +16534,8 @@ class EnterpriseOwnerInfo(sgqlc.types.Type): * `query` (`String`): The search string to look for. * `organization_logins` (`[String!]`): Only return invitations within the organizations with these logins + * `invitation_source` (`OrganizationInvitationSource`): Only + return invitations matching this invitation source * `after` (`String`): Returns the elements in the list that come after the specified cursor. * `before` (`String`): Returns the elements in the list that come @@ -12021,10 +16584,7 @@ class EnterpriseOwnerInfo(sgqlc.types.Type): """ saml_identity_provider = sgqlc.types.Field("EnterpriseIdentityProvider", graphql_name="samlIdentityProvider") - """The SAML Identity Provider for the enterprise. When used by a - GitHub App, requires an installation token with read and write - access to members. - """ + """The SAML Identity Provider for the enterprise.""" saml_identity_provider_setting_organizations = sgqlc.types.Field( sgqlc.types.non_null("OrganizationConnection"), @@ -12261,15 +16821,15 @@ class EnterpriseServerInstallationEdge(sgqlc.types.Type): """The item at the end of the edge.""" -class EnterpriseServerUserAccountConnection(sgqlc.types.relay.Connection): - """The connection type for EnterpriseServerUserAccount.""" +class EnterpriseServerInstallationMembershipConnection(sgqlc.types.relay.Connection): + """The connection type for EnterpriseServerInstallation.""" __schema__ = github_schema __field_names__ = ("edges", "nodes", "page_info", "total_count") - edges = sgqlc.types.Field(sgqlc.types.list_of("EnterpriseServerUserAccountEdge"), graphql_name="edges") + edges = sgqlc.types.Field(sgqlc.types.list_of("EnterpriseServerInstallationMembershipEdge"), graphql_name="edges") """A list of edges.""" - nodes = sgqlc.types.Field(sgqlc.types.list_of("EnterpriseServerUserAccount"), graphql_name="nodes") + nodes = sgqlc.types.Field(sgqlc.types.list_of("EnterpriseServerInstallation"), graphql_name="nodes") """A list of nodes.""" page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") @@ -12279,27 +16839,30 @@ class EnterpriseServerUserAccountConnection(sgqlc.types.relay.Connection): """Identifies the total count of items in the connection.""" -class EnterpriseServerUserAccountEdge(sgqlc.types.Type): - """An edge in a connection.""" +class EnterpriseServerInstallationMembershipEdge(sgqlc.types.Type): + """An Enterprise Server installation that a user is a member of.""" __schema__ = github_schema - __field_names__ = ("cursor", "node") + __field_names__ = ("cursor", "node", "role") cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") """A cursor for use in pagination.""" - node = sgqlc.types.Field("EnterpriseServerUserAccount", graphql_name="node") + node = sgqlc.types.Field("EnterpriseServerInstallation", graphql_name="node") """The item at the end of the edge.""" + role = sgqlc.types.Field(sgqlc.types.non_null(EnterpriseUserAccountMembershipRole), graphql_name="role") + """The role of the user in the enterprise membership.""" + -class EnterpriseServerUserAccountEmailConnection(sgqlc.types.relay.Connection): - """The connection type for EnterpriseServerUserAccountEmail.""" +class EnterpriseServerUserAccountConnection(sgqlc.types.relay.Connection): + """The connection type for EnterpriseServerUserAccount.""" __schema__ = github_schema __field_names__ = ("edges", "nodes", "page_info", "total_count") - edges = sgqlc.types.Field(sgqlc.types.list_of("EnterpriseServerUserAccountEmailEdge"), graphql_name="edges") + edges = sgqlc.types.Field(sgqlc.types.list_of("EnterpriseServerUserAccountEdge"), graphql_name="edges") """A list of edges.""" - nodes = sgqlc.types.Field(sgqlc.types.list_of("EnterpriseServerUserAccountEmail"), graphql_name="nodes") + nodes = sgqlc.types.Field(sgqlc.types.list_of("EnterpriseServerUserAccount"), graphql_name="nodes") """A list of nodes.""" page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") @@ -12309,7 +16872,7 @@ class EnterpriseServerUserAccountEmailConnection(sgqlc.types.relay.Connection): """Identifies the total count of items in the connection.""" -class EnterpriseServerUserAccountEmailEdge(sgqlc.types.Type): +class EnterpriseServerUserAccountEdge(sgqlc.types.Type): """An edge in a connection.""" __schema__ = github_schema @@ -12317,19 +16880,19 @@ class EnterpriseServerUserAccountEmailEdge(sgqlc.types.Type): cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") """A cursor for use in pagination.""" - node = sgqlc.types.Field("EnterpriseServerUserAccountEmail", graphql_name="node") + node = sgqlc.types.Field("EnterpriseServerUserAccount", graphql_name="node") """The item at the end of the edge.""" -class EnterpriseServerUserAccountsUploadConnection(sgqlc.types.relay.Connection): - """The connection type for EnterpriseServerUserAccountsUpload.""" +class EnterpriseServerUserAccountEmailConnection(sgqlc.types.relay.Connection): + """The connection type for EnterpriseServerUserAccountEmail.""" __schema__ = github_schema __field_names__ = ("edges", "nodes", "page_info", "total_count") - edges = sgqlc.types.Field(sgqlc.types.list_of("EnterpriseServerUserAccountsUploadEdge"), graphql_name="edges") + edges = sgqlc.types.Field(sgqlc.types.list_of("EnterpriseServerUserAccountEmailEdge"), graphql_name="edges") """A list of edges.""" - nodes = sgqlc.types.Field(sgqlc.types.list_of("EnterpriseServerUserAccountsUpload"), graphql_name="nodes") + nodes = sgqlc.types.Field(sgqlc.types.list_of("EnterpriseServerUserAccountEmail"), graphql_name="nodes") """A list of nodes.""" page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") @@ -12339,7 +16902,7 @@ class EnterpriseServerUserAccountsUploadConnection(sgqlc.types.relay.Connection) """Identifies the total count of items in the connection.""" -class EnterpriseServerUserAccountsUploadEdge(sgqlc.types.Type): +class EnterpriseServerUserAccountEmailEdge(sgqlc.types.Type): """An edge in a connection.""" __schema__ = github_schema @@ -12347,19 +16910,19 @@ class EnterpriseServerUserAccountsUploadEdge(sgqlc.types.Type): cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") """A cursor for use in pagination.""" - node = sgqlc.types.Field("EnterpriseServerUserAccountsUpload", graphql_name="node") + node = sgqlc.types.Field("EnterpriseServerUserAccountEmail", graphql_name="node") """The item at the end of the edge.""" -class EnterpriseUserAccountConnection(sgqlc.types.relay.Connection): - """The connection type for EnterpriseUserAccount.""" +class EnterpriseServerUserAccountsUploadConnection(sgqlc.types.relay.Connection): + """The connection type for EnterpriseServerUserAccountsUpload.""" __schema__ = github_schema __field_names__ = ("edges", "nodes", "page_info", "total_count") - edges = sgqlc.types.Field(sgqlc.types.list_of("EnterpriseUserAccountEdge"), graphql_name="edges") + edges = sgqlc.types.Field(sgqlc.types.list_of("EnterpriseServerUserAccountsUploadEdge"), graphql_name="edges") """A list of edges.""" - nodes = sgqlc.types.Field(sgqlc.types.list_of("EnterpriseUserAccount"), graphql_name="nodes") + nodes = sgqlc.types.Field(sgqlc.types.list_of("EnterpriseServerUserAccountsUpload"), graphql_name="nodes") """A list of nodes.""" page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") @@ -12369,7 +16932,7 @@ class EnterpriseUserAccountConnection(sgqlc.types.relay.Connection): """Identifies the total count of items in the connection.""" -class EnterpriseUserAccountEdge(sgqlc.types.Type): +class EnterpriseServerUserAccountsUploadEdge(sgqlc.types.Type): """An edge in a connection.""" __schema__ = github_schema @@ -12377,7 +16940,7 @@ class EnterpriseUserAccountEdge(sgqlc.types.Type): cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") """A cursor for use in pagination.""" - node = sgqlc.types.Field("EnterpriseUserAccount", graphql_name="node") + node = sgqlc.types.Field("EnterpriseServerUserAccountsUpload", graphql_name="node") """The item at the end of the edge.""" @@ -12776,61 +17339,6 @@ class GitHubMetadata(sgqlc.types.Type): """IP addresses for GitHub Pages' A records""" -class GitObject(sgqlc.types.Interface): - """Represents a Git object.""" - - __schema__ = github_schema - __field_names__ = ("abbreviated_oid", "commit_resource_path", "commit_url", "id", "oid", "repository") - abbreviated_oid = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="abbreviatedOid") - """An abbreviated version of the Git object ID""" - - commit_resource_path = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="commitResourcePath") - """The HTTP path for this Git object""" - - commit_url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="commitUrl") - """The HTTP URL for this Git object""" - - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - - oid = sgqlc.types.Field(sgqlc.types.non_null(GitObjectID), graphql_name="oid") - """The Git object ID""" - - repository = sgqlc.types.Field(sgqlc.types.non_null("Repository"), graphql_name="repository") - """The Repository the Git object belongs to""" - - -class GitSignature(sgqlc.types.Interface): - """Information about a signature (GPG or S/MIME) on a Commit or Tag.""" - - __schema__ = github_schema - __field_names__ = ("email", "is_valid", "payload", "signature", "signer", "state", "was_signed_by_git_hub") - email = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="email") - """Email used to sign this object.""" - - is_valid = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isValid") - """True if the signature is valid and verified by GitHub.""" - - payload = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="payload") - """Payload for GPG signing object. Raw ODB object without the - signature header. - """ - - signature = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="signature") - """ASCII-armored signature header from object.""" - - signer = sgqlc.types.Field("User", graphql_name="signer") - """GitHub user corresponding to the email signing this commit.""" - - state = sgqlc.types.Field(sgqlc.types.non_null(GitSignatureState), graphql_name="state") - """The state of this signature. `VALID` if signature is valid and - verified by GitHub, otherwise represents reason why signature is - considered invalid. - """ - - was_signed_by_git_hub = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="wasSignedByGitHub") - """True if the signature was made with GitHub's signing key.""" - - class GrantEnterpriseOrganizationsMigratorRolePayload(sgqlc.types.Type): """Autogenerated return type of GrantEnterpriseOrganizationsMigratorRole @@ -12884,24 +17392,10 @@ class Hovercard(sgqlc.types.Type): __schema__ = github_schema __field_names__ = ("contexts",) - contexts = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("HovercardContext"))), graphql_name="contexts" - ) + contexts = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(HovercardContext))), graphql_name="contexts") """Each of the contexts for this hovercard""" -class HovercardContext(sgqlc.types.Interface): - """An individual line of a hovercard""" - - __schema__ = github_schema - __field_names__ = ("message", "octicon") - message = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="message") - """A string describing this context""" - - octicon = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="octicon") - """An octicon to accompany this context""" - - class InviteEnterpriseAdminPayload(sgqlc.types.Type): """Autogenerated return type of InviteEnterpriseAdmin""" @@ -13045,13 +17539,68 @@ class IssueTemplate(sgqlc.types.Type): """A repository issue template.""" __schema__ = github_schema - __field_names__ = ("about", "body", "name", "title") + __field_names__ = ("about", "assignees", "body", "filename", "labels", "name", "title") about = sgqlc.types.Field(String, graphql_name="about") """The template purpose.""" + assignees = sgqlc.types.Field( + sgqlc.types.non_null("UserConnection"), + graphql_name="assignees", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), + ) + """The suggested assignees. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + """ + body = sgqlc.types.Field(String, graphql_name="body") """The suggested issue body.""" + filename = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="filename") + """The template filename.""" + + labels = sgqlc.types.Field( + "LabelConnection", + graphql_name="labels", + args=sgqlc.types.ArgDict( + ( + ("order_by", sgqlc.types.Arg(LabelOrder, graphql_name="orderBy", default={"field": "CREATED_AT", "direction": "ASC"})), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), + ) + """The suggested issue labels + + Arguments: + + * `order_by` (`LabelOrder`): Ordering options for labels returned + from the connection. (default: `{field: CREATED_AT, direction: + ASC}`) + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + """ + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") """The template name.""" @@ -13162,40 +17711,6 @@ class LabelEdge(sgqlc.types.Type): """The item at the end of the edge.""" -class Labelable(sgqlc.types.Interface): - """An object that can have labels assigned to it.""" - - __schema__ = github_schema - __field_names__ = ("labels",) - labels = sgqlc.types.Field( - LabelConnection, - graphql_name="labels", - args=sgqlc.types.ArgDict( - ( - ("order_by", sgqlc.types.Arg(LabelOrder, graphql_name="orderBy", default={"field": "CREATED_AT", "direction": "ASC"})), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ) - ), - ) - """A list of labels associated with the object. - - Arguments: - - * `order_by` (`LabelOrder`): Ordering options for labels returned - from the connection. (default: `{field: CREATED_AT, direction: - ASC}`) - * `after` (`String`): Returns the elements in the list that come - after the specified cursor. - * `before` (`String`): Returns the elements in the list that come - before the specified cursor. - * `first` (`Int`): Returns the first _n_ elements from the list. - * `last` (`Int`): Returns the last _n_ elements from the list. - """ - - class LanguageConnection(sgqlc.types.relay.Connection): """A list of languages associated with the parent.""" @@ -13245,6 +17760,30 @@ class LicenseRule(sgqlc.types.Type): """The human-readable rule label""" +class LinkProjectV2ToRepositoryPayload(sgqlc.types.Type): + """Autogenerated return type of LinkProjectV2ToRepository""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "repository") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + repository = sgqlc.types.Field("Repository", graphql_name="repository") + """The repository the project is linked to.""" + + +class LinkProjectV2ToTeamPayload(sgqlc.types.Type): + """Autogenerated return type of LinkProjectV2ToTeam""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "team") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + team = sgqlc.types.Field("Team", graphql_name="team") + """The team the project is linked to""" + + class LinkRepositoryToProjectPayload(sgqlc.types.Type): """Autogenerated return type of LinkRepositoryToProject""" @@ -13260,6 +17799,36 @@ class LinkRepositoryToProjectPayload(sgqlc.types.Type): """The linked Repository.""" +class LinkedBranchConnection(sgqlc.types.relay.Connection): + """The connection type for LinkedBranch.""" + + __schema__ = github_schema + __field_names__ = ("edges", "nodes", "page_info", "total_count") + edges = sgqlc.types.Field(sgqlc.types.list_of("LinkedBranchEdge"), graphql_name="edges") + """A list of edges.""" + + nodes = sgqlc.types.Field(sgqlc.types.list_of("LinkedBranch"), graphql_name="nodes") + """A list of nodes.""" + + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + """Information to aid in pagination.""" + + total_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalCount") + """Identifies the total count of items in the connection.""" + + +class LinkedBranchEdge(sgqlc.types.Type): + """An edge in a connection.""" + + __schema__ = github_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + """A cursor for use in pagination.""" + + node = sgqlc.types.Field("LinkedBranch", graphql_name="node") + """The item at the end of the edge.""" + + class LockLockablePayload(sgqlc.types.Type): """Autogenerated return type of LockLockable""" @@ -13271,20 +17840,38 @@ class LockLockablePayload(sgqlc.types.Type): client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" - locked_record = sgqlc.types.Field("Lockable", graphql_name="lockedRecord") + locked_record = sgqlc.types.Field(Lockable, graphql_name="lockedRecord") """The item that was locked.""" -class Lockable(sgqlc.types.Interface): - """An object that can be locked.""" +class MannequinConnection(sgqlc.types.relay.Connection): + """The connection type for Mannequin.""" __schema__ = github_schema - __field_names__ = ("active_lock_reason", "locked") - active_lock_reason = sgqlc.types.Field(LockReason, graphql_name="activeLockReason") - """Reason that the conversation was locked.""" + __field_names__ = ("edges", "nodes", "page_info", "total_count") + edges = sgqlc.types.Field(sgqlc.types.list_of("MannequinEdge"), graphql_name="edges") + """A list of edges.""" - locked = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="locked") - """`true` if the object is locked""" + nodes = sgqlc.types.Field(sgqlc.types.list_of("Mannequin"), graphql_name="nodes") + """A list of nodes.""" + + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + """Information to aid in pagination.""" + + total_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalCount") + """Identifies the total count of items in the connection.""" + + +class MannequinEdge(sgqlc.types.Type): + """Represents a mannequin.""" + + __schema__ = github_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + """A cursor for use in pagination.""" + + node = sgqlc.types.Field("Mannequin", graphql_name="node") + """The item at the end of the edge.""" class MarkDiscussionCommentAsAnswerPayload(sgqlc.types.Type): @@ -13311,6 +17898,18 @@ class MarkFileAsViewedPayload(sgqlc.types.Type): """The updated pull request.""" +class MarkProjectV2AsTemplatePayload(sgqlc.types.Type): + """Autogenerated return type of MarkProjectV2AsTemplate""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "project_v2") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + project_v2 = sgqlc.types.Field("ProjectV2", graphql_name="projectV2") + """The project.""" + + class MarkPullRequestReadyForReviewPayload(sgqlc.types.Type): """Autogenerated return type of MarkPullRequestReadyForReview""" @@ -13353,44 +17952,6 @@ class MarketplaceListingEdge(sgqlc.types.Type): """The item at the end of the edge.""" -class MemberStatusable(sgqlc.types.Interface): - """Entities that have members who can set status messages.""" - - __schema__ = github_schema - __field_names__ = ("member_statuses",) - member_statuses = sgqlc.types.Field( - sgqlc.types.non_null("UserStatusConnection"), - graphql_name="memberStatuses", - args=sgqlc.types.ArgDict( - ( - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ( - "order_by", - sgqlc.types.Arg(UserStatusOrder, graphql_name="orderBy", default={"field": "UPDATED_AT", "direction": "DESC"}), - ), - ) - ), - ) - """Get the status messages members of this entity have set that are - either public or visible only to the organization. - - Arguments: - - * `after` (`String`): Returns the elements in the list that come - after the specified cursor. - * `before` (`String`): Returns the elements in the list that come - before the specified cursor. - * `first` (`Int`): Returns the first _n_ elements from the list. - * `last` (`Int`): Returns the last _n_ elements from the list. - * `order_by` (`UserStatusOrder`): Ordering options for user - statuses returned from the connection. (default: `{field: - UPDATED_AT, direction: DESC}`) - """ - - class MergeBranchPayload(sgqlc.types.Type): """Autogenerated return type of MergeBranch""" @@ -13418,48 +17979,74 @@ class MergePullRequestPayload(sgqlc.types.Type): """The pull request that was merged.""" -class Migration(sgqlc.types.Interface): - """Represents an Octoshift migration.""" +class MergeQueueConfiguration(sgqlc.types.Type): + """Configuration for a MergeQueue""" __schema__ = github_schema __field_names__ = ( - "continue_on_error", - "created_at", - "failure_reason", - "id", - "migration_log_url", - "migration_source", - "repository_name", - "source_url", - "state", + "check_response_timeout", + "maximum_entries_to_build", + "maximum_entries_to_merge", + "merge_method", + "merging_strategy", + "minimum_entries_to_merge", + "minimum_entries_to_merge_wait_time", ) - continue_on_error = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="continueOnError") - """The Octoshift migration flag to continue on error.""" + check_response_timeout = sgqlc.types.Field(Int, graphql_name="checkResponseTimeout") + """The amount of time in minutes to wait for a check response before + considering it a failure. + """ - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - """Identifies the date and time when the object was created.""" + maximum_entries_to_build = sgqlc.types.Field(Int, graphql_name="maximumEntriesToBuild") + """The maximum number of entries to build at once.""" - failure_reason = sgqlc.types.Field(String, graphql_name="failureReason") - """The reason the migration failed.""" + maximum_entries_to_merge = sgqlc.types.Field(Int, graphql_name="maximumEntriesToMerge") + """The maximum number of entries to merge at once.""" - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + merge_method = sgqlc.types.Field(PullRequestMergeMethod, graphql_name="mergeMethod") + """The merge method to use for this queue.""" - migration_log_url = sgqlc.types.Field(URI, graphql_name="migrationLogUrl") - """The URL for the migration log (expires 1 day after migration - completes). + merging_strategy = sgqlc.types.Field(MergeQueueMergingStrategy, graphql_name="mergingStrategy") + """The strategy to use when merging entries.""" + + minimum_entries_to_merge = sgqlc.types.Field(Int, graphql_name="minimumEntriesToMerge") + """The minimum number of entries required to merge at once.""" + + minimum_entries_to_merge_wait_time = sgqlc.types.Field(Int, graphql_name="minimumEntriesToMergeWaitTime") + """The amount of time in minutes to wait before ignoring the minumum + number of entries in the queue requirement and merging a + collection of entries """ - migration_source = sgqlc.types.Field(sgqlc.types.non_null("MigrationSource"), graphql_name="migrationSource") - """The Octoshift migration source.""" - repository_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="repositoryName") - """The target repository name.""" +class MergeQueueEntryConnection(sgqlc.types.relay.Connection): + """The connection type for MergeQueueEntry.""" - source_url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="sourceUrl") - """The Octoshift migration source URL.""" + __schema__ = github_schema + __field_names__ = ("edges", "nodes", "page_info", "total_count") + edges = sgqlc.types.Field(sgqlc.types.list_of("MergeQueueEntryEdge"), graphql_name="edges") + """A list of edges.""" - state = sgqlc.types.Field(sgqlc.types.non_null(MigrationState), graphql_name="state") - """The Octoshift migration state.""" + nodes = sgqlc.types.Field(sgqlc.types.list_of("MergeQueueEntry"), graphql_name="nodes") + """A list of nodes.""" + + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + """Information to aid in pagination.""" + + total_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalCount") + """Identifies the total count of items in the connection.""" + + +class MergeQueueEntryEdge(sgqlc.types.Type): + """An edge in a connection.""" + + __schema__ = github_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + """A cursor for use in pagination.""" + + node = sgqlc.types.Field("MergeQueueEntry", graphql_name="node") + """The item at the end of the edge.""" class MilestoneConnection(sgqlc.types.relay.Connection): @@ -13492,21 +18079,6 @@ class MilestoneEdge(sgqlc.types.Type): """The item at the end of the edge.""" -class Minimizable(sgqlc.types.Interface): - """Entities that can be minimized.""" - - __schema__ = github_schema - __field_names__ = ("is_minimized", "minimized_reason", "viewer_can_minimize") - is_minimized = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isMinimized") - """Returns whether or not a comment has been minimized.""" - - minimized_reason = sgqlc.types.Field(String, graphql_name="minimizedReason") - """Returns why the comment was minimized.""" - - viewer_can_minimize = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanMinimize") - """Check if the current viewer can minimize this object.""" - - class MinimizeCommentPayload(sgqlc.types.Type): """Autogenerated return type of MinimizeComment""" @@ -13555,12 +18127,13 @@ class Mutation(sgqlc.types.Type): "add_comment", "add_discussion_comment", "add_discussion_poll_vote", + "add_enterprise_organization_member", "add_enterprise_support_entitlement", "add_labels_to_labelable", "add_project_card", "add_project_column", - "add_project_draft_issue", - "add_project_next_item", + "add_project_v2_draft_issue", + "add_project_v2_item_by_id", "add_pull_request_review", "add_pull_request_review_comment", "add_pull_request_review_thread", @@ -13570,17 +18143,22 @@ class Mutation(sgqlc.types.Type): "add_verifiable_domain", "approve_deployments", "approve_verifiable_domain", + "archive_project_v2_item", "archive_repository", "cancel_enterprise_admin_invitation", "cancel_sponsorship", "change_user_status", "clear_labels_from_labelable", + "clear_project_v2_item_field_value", "clone_project", "clone_template_repository", + "close_discussion", "close_issue", "close_pull_request", "convert_project_card_note_to_issue", "convert_pull_request_to_draft", + "copy_project_v2", + "create_attribution_invitation", "create_branch_protection_rule", "create_check_run", "create_check_suite", @@ -13590,13 +18168,19 @@ class Mutation(sgqlc.types.Type): "create_environment", "create_ip_allow_list_entry", "create_issue", + "create_linked_branch", "create_migration_source", "create_project", + "create_project_v2", + "create_project_v2_field", "create_pull_request", "create_ref", "create_repository", + "create_repository_ruleset", + "create_sponsors_listing", "create_sponsors_tier", "create_sponsorship", + "create_sponsorships", "create_team_discussion", "create_team_discussion_comment", "decline_topic_suggestion", @@ -13608,29 +18192,39 @@ class Mutation(sgqlc.types.Type): "delete_ip_allow_list_entry", "delete_issue", "delete_issue_comment", + "delete_linked_branch", "delete_project", "delete_project_card", "delete_project_column", - "delete_project_next_item", + "delete_project_v2", + "delete_project_v2_field", + "delete_project_v2_item", + "delete_project_v2_workflow", "delete_pull_request_review", "delete_pull_request_review_comment", "delete_ref", + "delete_repository_ruleset", "delete_team_discussion", "delete_team_discussion_comment", "delete_verifiable_domain", + "dequeue_pull_request", "disable_pull_request_auto_merge", "dismiss_pull_request_review", "dismiss_repository_vulnerability_alert", "enable_pull_request_auto_merge", + "enqueue_pull_request", "follow_organization", "follow_user", "grant_enterprise_organizations_migrator_role", "grant_migrator_role", "invite_enterprise_admin", + "link_project_v2_to_repository", + "link_project_v2_to_team", "link_repository_to_project", "lock_lockable", "mark_discussion_comment_as_answer", "mark_file_as_viewed", + "mark_project_v2_as_template", "mark_pull_request_ready_for_review", "merge_branch", "merge_pull_request", @@ -13638,12 +18232,14 @@ class Mutation(sgqlc.types.Type): "move_project_card", "move_project_column", "pin_issue", + "publish_sponsors_tier", "regenerate_enterprise_identity_provider_recovery_codes", "regenerate_verifiable_domain_token", "reject_deployments", "remove_assignees_from_assignable", "remove_enterprise_admin", "remove_enterprise_identity_provider", + "remove_enterprise_member", "remove_enterprise_organization", "remove_enterprise_support_entitlement", "remove_labels_from_labelable", @@ -13651,28 +18247,37 @@ class Mutation(sgqlc.types.Type): "remove_reaction", "remove_star", "remove_upvote", + "reopen_discussion", "reopen_issue", "reopen_pull_request", "request_reviews", "rerequest_check_suite", "resolve_review_thread", + "retire_sponsors_tier", + "revert_pull_request", "revoke_enterprise_organizations_migrator_role", "revoke_migrator_role", "set_enterprise_identity_provider", "set_organization_interaction_limit", "set_repository_interaction_limit", "set_user_interaction_limit", + "start_organization_migration", "start_repository_migration", "submit_pull_request_review", + "transfer_enterprise_organization", "transfer_issue", + "unarchive_project_v2_item", "unarchive_repository", "unfollow_organization", "unfollow_user", + "unlink_project_v2_from_repository", + "unlink_project_v2_from_team", "unlink_repository_from_project", "unlock_lockable", "unmark_discussion_comment_as_answer", "unmark_file_as_viewed", "unmark_issue_as_duplicate", + "unmark_project_v2_as_template", "unminimize_comment", "unpin_issue", "unresolve_review_thread", @@ -13706,18 +18311,23 @@ class Mutation(sgqlc.types.Type): "update_issue_comment", "update_notification_restriction_setting", "update_organization_allow_private_repository_forking_setting", + "update_organization_web_commit_signoff_setting", "update_project", "update_project_card", "update_project_column", - "update_project_draft_issue", - "update_project_next", - "update_project_next_item_field", + "update_project_v2", + "update_project_v2_collaborators", + "update_project_v2_draft_issue", + "update_project_v2_item_field_value", + "update_project_v2_item_position", "update_pull_request", "update_pull_request_branch", "update_pull_request_review", "update_pull_request_review_comment", "update_ref", "update_repository", + "update_repository_ruleset", + "update_repository_web_commit_signoff_setting", "update_sponsorship_preferences", "update_subscription", "update_team_discussion", @@ -13835,6 +18445,21 @@ class Mutation(sgqlc.types.Type): AddDiscussionPollVote """ + add_enterprise_organization_member = sgqlc.types.Field( + AddEnterpriseOrganizationMemberPayload, + graphql_name="addEnterpriseOrganizationMember", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(AddEnterpriseOrganizationMemberInput), graphql_name="input", default=None)),) + ), + ) + """Adds enterprise members to an organization within the enterprise. + + Arguments: + + * `input` (`AddEnterpriseOrganizationMemberInput!`): Parameters + for AddEnterpriseOrganizationMember + """ + add_enterprise_support_entitlement = sgqlc.types.Field( AddEnterpriseSupportEntitlementPayload, graphql_name="addEnterpriseSupportEntitlement", @@ -13895,34 +18520,34 @@ class Mutation(sgqlc.types.Type): AddProjectColumn """ - add_project_draft_issue = sgqlc.types.Field( - AddProjectDraftIssuePayload, - graphql_name="addProjectDraftIssue", + add_project_v2_draft_issue = sgqlc.types.Field( + AddProjectV2DraftIssuePayload, + graphql_name="addProjectV2DraftIssue", args=sgqlc.types.ArgDict( - (("input", sgqlc.types.Arg(sgqlc.types.non_null(AddProjectDraftIssueInput), graphql_name="input", default=None)),) + (("input", sgqlc.types.Arg(sgqlc.types.non_null(AddProjectV2DraftIssueInput), graphql_name="input", default=None)),) ), ) """Creates a new draft issue and add it to a Project. Arguments: - * `input` (`AddProjectDraftIssueInput!`): Parameters for - AddProjectDraftIssue + * `input` (`AddProjectV2DraftIssueInput!`): Parameters for + AddProjectV2DraftIssue """ - add_project_next_item = sgqlc.types.Field( - AddProjectNextItemPayload, - graphql_name="addProjectNextItem", + add_project_v2_item_by_id = sgqlc.types.Field( + AddProjectV2ItemByIdPayload, + graphql_name="addProjectV2ItemById", args=sgqlc.types.ArgDict( - (("input", sgqlc.types.Arg(sgqlc.types.non_null(AddProjectNextItemInput), graphql_name="input", default=None)),) + (("input", sgqlc.types.Arg(sgqlc.types.non_null(AddProjectV2ItemByIdInput), graphql_name="input", default=None)),) ), ) - """Adds an existing item (Issue or PullRequest) to a Project. + """Links an existing content instance to a Project. Arguments: - * `input` (`AddProjectNextItemInput!`): Parameters for - AddProjectNextItem + * `input` (`AddProjectV2ItemByIdInput!`): Parameters for + AddProjectV2ItemById """ add_pull_request_review = sgqlc.types.Field( @@ -14051,6 +18676,21 @@ class Mutation(sgqlc.types.Type): ApproveVerifiableDomain """ + archive_project_v2_item = sgqlc.types.Field( + ArchiveProjectV2ItemPayload, + graphql_name="archiveProjectV2Item", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(ArchiveProjectV2ItemInput), graphql_name="input", default=None)),) + ), + ) + """Archives a ProjectV2Item + + Arguments: + + * `input` (`ArchiveProjectV2ItemInput!`): Parameters for + ArchiveProjectV2Item + """ + archive_repository = sgqlc.types.Field( ArchiveRepositoryPayload, graphql_name="archiveRepository", @@ -14127,6 +18767,23 @@ class Mutation(sgqlc.types.Type): ClearLabelsFromLabelable """ + clear_project_v2_item_field_value = sgqlc.types.Field( + ClearProjectV2ItemFieldValuePayload, + graphql_name="clearProjectV2ItemFieldValue", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(ClearProjectV2ItemFieldValueInput), graphql_name="input", default=None)),) + ), + ) + """This mutation clears the value of a field for an item in a + Project. Currently only text, number, date, assignees, labels, + single-select, iteration and milestone fields are supported. + + Arguments: + + * `input` (`ClearProjectV2ItemFieldValueInput!`): Parameters for + ClearProjectV2ItemFieldValue + """ + clone_project = sgqlc.types.Field( CloneProjectPayload, graphql_name="cloneProject", @@ -14158,6 +18815,21 @@ class Mutation(sgqlc.types.Type): CloneTemplateRepository """ + close_discussion = sgqlc.types.Field( + CloseDiscussionPayload, + graphql_name="closeDiscussion", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(CloseDiscussionInput), graphql_name="input", default=None)),) + ), + ) + """Close a discussion. + + Arguments: + + * `input` (`CloseDiscussionInput!`): Parameters for + CloseDiscussion + """ + close_issue = sgqlc.types.Field( CloseIssuePayload, graphql_name="closeIssue", @@ -14216,6 +18888,35 @@ class Mutation(sgqlc.types.Type): ConvertPullRequestToDraft """ + copy_project_v2 = sgqlc.types.Field( + CopyProjectV2Payload, + graphql_name="copyProjectV2", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(CopyProjectV2Input), graphql_name="input", default=None)),) + ), + ) + """Copy a project. + + Arguments: + + * `input` (`CopyProjectV2Input!`): Parameters for CopyProjectV2 + """ + + create_attribution_invitation = sgqlc.types.Field( + CreateAttributionInvitationPayload, + graphql_name="createAttributionInvitation", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(CreateAttributionInvitationInput), graphql_name="input", default=None)),) + ), + ) + """Invites a user to claim reattributable data + + Arguments: + + * `input` (`CreateAttributionInvitationInput!`): Parameters for + CreateAttributionInvitation + """ + create_branch_protection_rule = sgqlc.types.Field( CreateBranchProtectionRulePayload, graphql_name="createBranchProtectionRule", @@ -14372,6 +19073,21 @@ class Mutation(sgqlc.types.Type): * `input` (`CreateIssueInput!`): Parameters for CreateIssue """ + create_linked_branch = sgqlc.types.Field( + CreateLinkedBranchPayload, + graphql_name="createLinkedBranch", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(CreateLinkedBranchInput), graphql_name="input", default=None)),) + ), + ) + """Create a branch linked to an issue. + + Arguments: + + * `input` (`CreateLinkedBranchInput!`): Parameters for + CreateLinkedBranch + """ + create_migration_source = sgqlc.types.Field( CreateMigrationSourcePayload, graphql_name="createMigrationSource", @@ -14379,7 +19095,7 @@ class Mutation(sgqlc.types.Type): (("input", sgqlc.types.Arg(sgqlc.types.non_null(CreateMigrationSourceInput), graphql_name="input", default=None)),) ), ) - """Creates an Octoshift migration source. + """Creates a GitHub Enterprise Importer (GEI) migration source. Arguments: @@ -14401,6 +19117,36 @@ class Mutation(sgqlc.types.Type): * `input` (`CreateProjectInput!`): Parameters for CreateProject """ + create_project_v2 = sgqlc.types.Field( + CreateProjectV2Payload, + graphql_name="createProjectV2", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(CreateProjectV2Input), graphql_name="input", default=None)),) + ), + ) + """Creates a new project. + + Arguments: + + * `input` (`CreateProjectV2Input!`): Parameters for + CreateProjectV2 + """ + + create_project_v2_field = sgqlc.types.Field( + CreateProjectV2FieldPayload, + graphql_name="createProjectV2Field", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(CreateProjectV2FieldInput), graphql_name="input", default=None)),) + ), + ) + """Create a new project field. + + Arguments: + + * `input` (`CreateProjectV2FieldInput!`): Parameters for + CreateProjectV2Field + """ + create_pull_request = sgqlc.types.Field( CreatePullRequestPayload, graphql_name="createPullRequest", @@ -14443,6 +19189,37 @@ class Mutation(sgqlc.types.Type): CreateRepository """ + create_repository_ruleset = sgqlc.types.Field( + CreateRepositoryRulesetPayload, + graphql_name="createRepositoryRuleset", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(CreateRepositoryRulesetInput), graphql_name="input", default=None)),) + ), + ) + """Create a repository ruleset + + Arguments: + + * `input` (`CreateRepositoryRulesetInput!`): Parameters for + CreateRepositoryRuleset + """ + + create_sponsors_listing = sgqlc.types.Field( + CreateSponsorsListingPayload, + graphql_name="createSponsorsListing", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(CreateSponsorsListingInput), graphql_name="input", default=None)),) + ), + ) + """Create a GitHub Sponsors profile to allow others to sponsor you or + your organization. + + Arguments: + + * `input` (`CreateSponsorsListingInput!`): Parameters for + CreateSponsorsListing + """ + create_sponsors_tier = sgqlc.types.Field( CreateSponsorsTierPayload, graphql_name="createSponsorsTier", @@ -14474,6 +19251,23 @@ class Mutation(sgqlc.types.Type): CreateSponsorship """ + create_sponsorships = sgqlc.types.Field( + CreateSponsorshipsPayload, + graphql_name="createSponsorships", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(CreateSponsorshipsInput), graphql_name="input", default=None)),) + ), + ) + """Make many one-time sponsorships for different sponsorable users or + organizations at once. Can only sponsor those who have a public + GitHub Sponsors profile. + + Arguments: + + * `input` (`CreateSponsorshipsInput!`): Parameters for + CreateSponsorships + """ + create_team_discussion = sgqlc.types.Field( CreateTeamDiscussionPayload, graphql_name="createTeamDiscussion", @@ -14636,6 +19430,21 @@ class Mutation(sgqlc.types.Type): DeleteIssueComment """ + delete_linked_branch = sgqlc.types.Field( + DeleteLinkedBranchPayload, + graphql_name="deleteLinkedBranch", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(DeleteLinkedBranchInput), graphql_name="input", default=None)),) + ), + ) + """Unlink a branch from an issue. + + Arguments: + + * `input` (`DeleteLinkedBranchInput!`): Parameters for + DeleteLinkedBranch + """ + delete_project = sgqlc.types.Field( DeleteProjectPayload, graphql_name="deleteProject", @@ -14680,19 +19489,64 @@ class Mutation(sgqlc.types.Type): DeleteProjectColumn """ - delete_project_next_item = sgqlc.types.Field( - DeleteProjectNextItemPayload, - graphql_name="deleteProjectNextItem", + delete_project_v2 = sgqlc.types.Field( + DeleteProjectV2Payload, + graphql_name="deleteProjectV2", args=sgqlc.types.ArgDict( - (("input", sgqlc.types.Arg(sgqlc.types.non_null(DeleteProjectNextItemInput), graphql_name="input", default=None)),) + (("input", sgqlc.types.Arg(sgqlc.types.non_null(DeleteProjectV2Input), graphql_name="input", default=None)),) + ), + ) + """Delete a project. + + Arguments: + + * `input` (`DeleteProjectV2Input!`): Parameters for + DeleteProjectV2 + """ + + delete_project_v2_field = sgqlc.types.Field( + DeleteProjectV2FieldPayload, + graphql_name="deleteProjectV2Field", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(DeleteProjectV2FieldInput), graphql_name="input", default=None)),) + ), + ) + """Delete a project field. + + Arguments: + + * `input` (`DeleteProjectV2FieldInput!`): Parameters for + DeleteProjectV2Field + """ + + delete_project_v2_item = sgqlc.types.Field( + DeleteProjectV2ItemPayload, + graphql_name="deleteProjectV2Item", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(DeleteProjectV2ItemInput), graphql_name="input", default=None)),) ), ) """Deletes an item from a Project. Arguments: - * `input` (`DeleteProjectNextItemInput!`): Parameters for - DeleteProjectNextItem + * `input` (`DeleteProjectV2ItemInput!`): Parameters for + DeleteProjectV2Item + """ + + delete_project_v2_workflow = sgqlc.types.Field( + DeleteProjectV2WorkflowPayload, + graphql_name="deleteProjectV2Workflow", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(DeleteProjectV2WorkflowInput), graphql_name="input", default=None)),) + ), + ) + """Deletes a project workflow. + + Arguments: + + * `input` (`DeleteProjectV2WorkflowInput!`): Parameters for + DeleteProjectV2Workflow """ delete_pull_request_review = sgqlc.types.Field( @@ -14737,6 +19591,21 @@ class Mutation(sgqlc.types.Type): * `input` (`DeleteRefInput!`): Parameters for DeleteRef """ + delete_repository_ruleset = sgqlc.types.Field( + DeleteRepositoryRulesetPayload, + graphql_name="deleteRepositoryRuleset", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(DeleteRepositoryRulesetInput), graphql_name="input", default=None)),) + ), + ) + """Delete a repository ruleset + + Arguments: + + * `input` (`DeleteRepositoryRulesetInput!`): Parameters for + DeleteRepositoryRuleset + """ + delete_team_discussion = sgqlc.types.Field( DeleteTeamDiscussionPayload, graphql_name="deleteTeamDiscussion", @@ -14782,6 +19651,21 @@ class Mutation(sgqlc.types.Type): DeleteVerifiableDomain """ + dequeue_pull_request = sgqlc.types.Field( + DequeuePullRequestPayload, + graphql_name="dequeuePullRequest", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(DequeuePullRequestInput), graphql_name="input", default=None)),) + ), + ) + """Remove a pull request from the merge queue. + + Arguments: + + * `input` (`DequeuePullRequestInput!`): Parameters for + DequeuePullRequest + """ + disable_pull_request_auto_merge = sgqlc.types.Field( DisablePullRequestAutoMergePayload, graphql_name="disablePullRequestAutoMerge", @@ -14847,6 +19731,21 @@ class Mutation(sgqlc.types.Type): EnablePullRequestAutoMerge """ + enqueue_pull_request = sgqlc.types.Field( + EnqueuePullRequestPayload, + graphql_name="enqueuePullRequest", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(EnqueuePullRequestInput), graphql_name="input", default=None)),) + ), + ) + """Add a pull request to the merge queue. + + Arguments: + + * `input` (`EnqueuePullRequestInput!`): Parameters for + EnqueuePullRequest + """ + follow_organization = sgqlc.types.Field( FollowOrganizationPayload, graphql_name="followOrganization", @@ -14927,6 +19826,36 @@ class Mutation(sgqlc.types.Type): InviteEnterpriseAdmin """ + link_project_v2_to_repository = sgqlc.types.Field( + LinkProjectV2ToRepositoryPayload, + graphql_name="linkProjectV2ToRepository", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(LinkProjectV2ToRepositoryInput), graphql_name="input", default=None)),) + ), + ) + """Links a project to a repository. + + Arguments: + + * `input` (`LinkProjectV2ToRepositoryInput!`): Parameters for + LinkProjectV2ToRepository + """ + + link_project_v2_to_team = sgqlc.types.Field( + LinkProjectV2ToTeamPayload, + graphql_name="linkProjectV2ToTeam", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(LinkProjectV2ToTeamInput), graphql_name="input", default=None)),) + ), + ) + """Links a project to a team. + + Arguments: + + * `input` (`LinkProjectV2ToTeamInput!`): Parameters for + LinkProjectV2ToTeam + """ + link_repository_to_project = sgqlc.types.Field( LinkRepositoryToProjectPayload, graphql_name="linkRepositoryToProject", @@ -14987,6 +19916,22 @@ class Mutation(sgqlc.types.Type): MarkFileAsViewed """ + mark_project_v2_as_template = sgqlc.types.Field( + MarkProjectV2AsTemplatePayload, + graphql_name="markProjectV2AsTemplate", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(MarkProjectV2AsTemplateInput), graphql_name="input", default=None)),) + ), + ) + """Mark a project as a template. Note that only projects which are + owned by an Organization can be marked as a template. + + Arguments: + + * `input` (`MarkProjectV2AsTemplateInput!`): Parameters for + MarkProjectV2AsTemplate + """ + mark_pull_request_ready_for_review = sgqlc.types.Field( MarkPullRequestReadyForReviewPayload, graphql_name="markPullRequestReadyForReview", @@ -15086,6 +20031,22 @@ class Mutation(sgqlc.types.Type): * `input` (`PinIssueInput!`): Parameters for PinIssue """ + publish_sponsors_tier = sgqlc.types.Field( + "PublishSponsorsTierPayload", + graphql_name="publishSponsorsTier", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(PublishSponsorsTierInput), graphql_name="input", default=None)),) + ), + ) + """Publish an existing sponsorship tier that is currently still a + draft to a GitHub Sponsors profile. + + Arguments: + + * `input` (`PublishSponsorsTierInput!`): Parameters for + PublishSponsorsTier + """ + regenerate_enterprise_identity_provider_recovery_codes = sgqlc.types.Field( "RegenerateEnterpriseIdentityProviderRecoveryCodesPayload", graphql_name="regenerateEnterpriseIdentityProviderRecoveryCodes", @@ -15184,6 +20145,21 @@ class Mutation(sgqlc.types.Type): for RemoveEnterpriseIdentityProvider """ + remove_enterprise_member = sgqlc.types.Field( + "RemoveEnterpriseMemberPayload", + graphql_name="removeEnterpriseMember", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(RemoveEnterpriseMemberInput), graphql_name="input", default=None)),) + ), + ) + """Removes a user from all organizations within the enterprise + + Arguments: + + * `input` (`RemoveEnterpriseMemberInput!`): Parameters for + RemoveEnterpriseMember + """ + remove_enterprise_organization = sgqlc.types.Field( "RemoveEnterpriseOrganizationPayload", graphql_name="removeEnterpriseOrganization", @@ -15285,6 +20261,21 @@ class Mutation(sgqlc.types.Type): * `input` (`RemoveUpvoteInput!`): Parameters for RemoveUpvote """ + reopen_discussion = sgqlc.types.Field( + "ReopenDiscussionPayload", + graphql_name="reopenDiscussion", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(ReopenDiscussionInput), graphql_name="input", default=None)),) + ), + ) + """Reopen a discussion. + + Arguments: + + * `input` (`ReopenDiscussionInput!`): Parameters for + ReopenDiscussion + """ + reopen_issue = sgqlc.types.Field( "ReopenIssuePayload", graphql_name="reopenIssue", @@ -15356,6 +20347,38 @@ class Mutation(sgqlc.types.Type): ResolveReviewThread """ + retire_sponsors_tier = sgqlc.types.Field( + "RetireSponsorsTierPayload", + graphql_name="retireSponsorsTier", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(RetireSponsorsTierInput), graphql_name="input", default=None)),) + ), + ) + """Retire a published payment tier from your GitHub Sponsors profile + so it cannot be used to start new sponsorships. + + Arguments: + + * `input` (`RetireSponsorsTierInput!`): Parameters for + RetireSponsorsTier + """ + + revert_pull_request = sgqlc.types.Field( + "RevertPullRequestPayload", + graphql_name="revertPullRequest", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(RevertPullRequestInput), graphql_name="input", default=None)),) + ), + ) + """Create a pull request that reverts the changes from a merged pull + request. + + Arguments: + + * `input` (`RevertPullRequestInput!`): Parameters for + RevertPullRequest + """ + revoke_enterprise_organizations_migrator_role = sgqlc.types.Field( "RevokeEnterpriseOrganizationsMigratorRolePayload", graphql_name="revokeEnterpriseOrganizationsMigratorRole", @@ -15456,6 +20479,21 @@ class Mutation(sgqlc.types.Type): SetUserInteractionLimit """ + start_organization_migration = sgqlc.types.Field( + "StartOrganizationMigrationPayload", + graphql_name="startOrganizationMigration", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(StartOrganizationMigrationInput), graphql_name="input", default=None)),) + ), + ) + """Starts a GitHub Enterprise Importer organization migration. + + Arguments: + + * `input` (`StartOrganizationMigrationInput!`): Parameters for + StartOrganizationMigration + """ + start_repository_migration = sgqlc.types.Field( "StartRepositoryMigrationPayload", graphql_name="startRepositoryMigration", @@ -15463,7 +20501,7 @@ class Mutation(sgqlc.types.Type): (("input", sgqlc.types.Arg(sgqlc.types.non_null(StartRepositoryMigrationInput), graphql_name="input", default=None)),) ), ) - """Start a repository migration. + """Starts a GitHub Enterprise Importer (GEI) repository migration. Arguments: @@ -15486,6 +20524,22 @@ class Mutation(sgqlc.types.Type): SubmitPullRequestReview """ + transfer_enterprise_organization = sgqlc.types.Field( + "TransferEnterpriseOrganizationPayload", + graphql_name="transferEnterpriseOrganization", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(TransferEnterpriseOrganizationInput), graphql_name="input", default=None)),) + ), + ) + """Transfer an organization from one enterprise to another + enterprise. + + Arguments: + + * `input` (`TransferEnterpriseOrganizationInput!`): Parameters for + TransferEnterpriseOrganization + """ + transfer_issue = sgqlc.types.Field( "TransferIssuePayload", graphql_name="transferIssue", @@ -15500,6 +20554,21 @@ class Mutation(sgqlc.types.Type): * `input` (`TransferIssueInput!`): Parameters for TransferIssue """ + unarchive_project_v2_item = sgqlc.types.Field( + "UnarchiveProjectV2ItemPayload", + graphql_name="unarchiveProjectV2Item", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(UnarchiveProjectV2ItemInput), graphql_name="input", default=None)),) + ), + ) + """Unarchives a ProjectV2Item + + Arguments: + + * `input` (`UnarchiveProjectV2ItemInput!`): Parameters for + UnarchiveProjectV2Item + """ + unarchive_repository = sgqlc.types.Field( "UnarchiveRepositoryPayload", graphql_name="unarchiveRepository", @@ -15544,6 +20613,36 @@ class Mutation(sgqlc.types.Type): * `input` (`UnfollowUserInput!`): Parameters for UnfollowUser """ + unlink_project_v2_from_repository = sgqlc.types.Field( + "UnlinkProjectV2FromRepositoryPayload", + graphql_name="unlinkProjectV2FromRepository", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(UnlinkProjectV2FromRepositoryInput), graphql_name="input", default=None)),) + ), + ) + """Unlinks a project from a repository. + + Arguments: + + * `input` (`UnlinkProjectV2FromRepositoryInput!`): Parameters for + UnlinkProjectV2FromRepository + """ + + unlink_project_v2_from_team = sgqlc.types.Field( + "UnlinkProjectV2FromTeamPayload", + graphql_name="unlinkProjectV2FromTeam", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(UnlinkProjectV2FromTeamInput), graphql_name="input", default=None)),) + ), + ) + """Unlinks a project to a team. + + Arguments: + + * `input` (`UnlinkProjectV2FromTeamInput!`): Parameters for + UnlinkProjectV2FromTeam + """ + unlink_repository_from_project = sgqlc.types.Field( "UnlinkRepositoryFromProjectPayload", graphql_name="unlinkRepositoryFromProject", @@ -15619,6 +20718,21 @@ class Mutation(sgqlc.types.Type): UnmarkIssueAsDuplicate """ + unmark_project_v2_as_template = sgqlc.types.Field( + "UnmarkProjectV2AsTemplatePayload", + graphql_name="unmarkProjectV2AsTemplate", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(UnmarkProjectV2AsTemplateInput), graphql_name="input", default=None)),) + ), + ) + """Unmark a project as a template. + + Arguments: + + * `input` (`UnmarkProjectV2AsTemplateInput!`): Parameters for + UnmarkProjectV2AsTemplate + """ + unminimize_comment = sgqlc.types.Field( "UnminimizeCommentPayload", graphql_name="unminimizeComment", @@ -15668,7 +20782,7 @@ class Mutation(sgqlc.types.Type): (("input", sgqlc.types.Arg(sgqlc.types.non_null(UpdateBranchProtectionRuleInput), graphql_name="input", default=None)),) ), ) - """Create a new branch protection rule + """Update a branch protection rule Arguments: @@ -16265,6 +21379,29 @@ class Mutation(sgqlc.types.Type): UpdateOrganizationAllowPrivateRepositoryForkingSetting """ + update_organization_web_commit_signoff_setting = sgqlc.types.Field( + "UpdateOrganizationWebCommitSignoffSettingPayload", + graphql_name="updateOrganizationWebCommitSignoffSetting", + args=sgqlc.types.ArgDict( + ( + ( + "input", + sgqlc.types.Arg( + sgqlc.types.non_null(UpdateOrganizationWebCommitSignoffSettingInput), graphql_name="input", default=None + ), + ), + ) + ), + ) + """Sets whether contributors are required to sign off on web-based + commits for repositories in an organization. + + Arguments: + + * `input` (`UpdateOrganizationWebCommitSignoffSettingInput!`): + Parameters for UpdateOrganizationWebCommitSignoffSetting + """ + update_project = sgqlc.types.Field( "UpdateProjectPayload", graphql_name="updateProject", @@ -16309,49 +21446,82 @@ class Mutation(sgqlc.types.Type): UpdateProjectColumn """ - update_project_draft_issue = sgqlc.types.Field( - "UpdateProjectDraftIssuePayload", - graphql_name="updateProjectDraftIssue", + update_project_v2 = sgqlc.types.Field( + "UpdateProjectV2Payload", + graphql_name="updateProjectV2", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(UpdateProjectV2Input), graphql_name="input", default=None)),) + ), + ) + """Updates an existing project (beta). + + Arguments: + + * `input` (`UpdateProjectV2Input!`): Parameters for + UpdateProjectV2 + """ + + update_project_v2_collaborators = sgqlc.types.Field( + "UpdateProjectV2CollaboratorsPayload", + graphql_name="updateProjectV2Collaborators", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(UpdateProjectV2CollaboratorsInput), graphql_name="input", default=None)),) + ), + ) + """Update the collaborators on a team or a project + + Arguments: + + * `input` (`UpdateProjectV2CollaboratorsInput!`): Parameters for + UpdateProjectV2Collaborators + """ + + update_project_v2_draft_issue = sgqlc.types.Field( + "UpdateProjectV2DraftIssuePayload", + graphql_name="updateProjectV2DraftIssue", args=sgqlc.types.ArgDict( - (("input", sgqlc.types.Arg(sgqlc.types.non_null(UpdateProjectDraftIssueInput), graphql_name="input", default=None)),) + (("input", sgqlc.types.Arg(sgqlc.types.non_null(UpdateProjectV2DraftIssueInput), graphql_name="input", default=None)),) ), ) """Updates a draft issue within a Project. Arguments: - * `input` (`UpdateProjectDraftIssueInput!`): Parameters for - UpdateProjectDraftIssue + * `input` (`UpdateProjectV2DraftIssueInput!`): Parameters for + UpdateProjectV2DraftIssue """ - update_project_next = sgqlc.types.Field( - "UpdateProjectNextPayload", - graphql_name="updateProjectNext", + update_project_v2_item_field_value = sgqlc.types.Field( + "UpdateProjectV2ItemFieldValuePayload", + graphql_name="updateProjectV2ItemFieldValue", args=sgqlc.types.ArgDict( - (("input", sgqlc.types.Arg(sgqlc.types.non_null(UpdateProjectNextInput), graphql_name="input", default=None)),) + (("input", sgqlc.types.Arg(sgqlc.types.non_null(UpdateProjectV2ItemFieldValueInput), graphql_name="input", default=None)),) ), ) - """Updates an existing project (beta). + """This mutation updates the value of a field for an item in a + Project. Currently only single-select, text, number, date, and + iteration fields are supported. Arguments: - * `input` (`UpdateProjectNextInput!`): Parameters for - UpdateProjectNext + * `input` (`UpdateProjectV2ItemFieldValueInput!`): Parameters for + UpdateProjectV2ItemFieldValue """ - update_project_next_item_field = sgqlc.types.Field( - "UpdateProjectNextItemFieldPayload", - graphql_name="updateProjectNextItemField", + update_project_v2_item_position = sgqlc.types.Field( + "UpdateProjectV2ItemPositionPayload", + graphql_name="updateProjectV2ItemPosition", args=sgqlc.types.ArgDict( - (("input", sgqlc.types.Arg(sgqlc.types.non_null(UpdateProjectNextItemFieldInput), graphql_name="input", default=None)),) + (("input", sgqlc.types.Arg(sgqlc.types.non_null(UpdateProjectV2ItemPositionInput), graphql_name="input", default=None)),) ), ) - """Updates a field of an item from a Project. + """This mutation updates the position of the item in the project, + where the position represents the priority of an item. Arguments: - * `input` (`UpdateProjectNextItemFieldInput!`): Parameters for - UpdateProjectNextItemField + * `input` (`UpdateProjectV2ItemPositionInput!`): Parameters for + UpdateProjectV2ItemPosition """ update_pull_request = sgqlc.types.Field( @@ -16441,6 +21611,42 @@ class Mutation(sgqlc.types.Type): UpdateRepository """ + update_repository_ruleset = sgqlc.types.Field( + "UpdateRepositoryRulesetPayload", + graphql_name="updateRepositoryRuleset", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(UpdateRepositoryRulesetInput), graphql_name="input", default=None)),) + ), + ) + """Update a repository ruleset + + Arguments: + + * `input` (`UpdateRepositoryRulesetInput!`): Parameters for + UpdateRepositoryRuleset + """ + + update_repository_web_commit_signoff_setting = sgqlc.types.Field( + "UpdateRepositoryWebCommitSignoffSettingPayload", + graphql_name="updateRepositoryWebCommitSignoffSetting", + args=sgqlc.types.ArgDict( + ( + ( + "input", + sgqlc.types.Arg(sgqlc.types.non_null(UpdateRepositoryWebCommitSignoffSettingInput), graphql_name="input", default=None), + ), + ) + ), + ) + """Sets whether contributors are required to sign off on web-based + commits for a repository. + + Arguments: + + * `input` (`UpdateRepositoryWebCommitSignoffSettingInput!`): + Parameters for UpdateRepositoryWebCommitSignoffSetting + """ + update_sponsorship_preferences = sgqlc.types.Field( "UpdateSponsorshipPreferencesPayload", graphql_name="updateSponsorshipPreferences", @@ -16547,30 +21753,6 @@ class Mutation(sgqlc.types.Type): """ -class Node(sgqlc.types.Interface): - """An object with an ID.""" - - __schema__ = github_schema - __field_names__ = ("id",) - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - """ID of the object.""" - - -class OauthApplicationAuditEntryData(sgqlc.types.Interface): - """Metadata for an audit entry with action oauth_application.*""" - - __schema__ = github_schema - __field_names__ = ("oauth_application_name", "oauth_application_resource_path", "oauth_application_url") - oauth_application_name = sgqlc.types.Field(String, graphql_name="oauthApplicationName") - """The name of the OAuth Application.""" - - oauth_application_resource_path = sgqlc.types.Field(URI, graphql_name="oauthApplicationResourcePath") - """The HTTP path for the OAuth Application""" - - oauth_application_url = sgqlc.types.Field(URI, graphql_name="oauthApplicationUrl") - """The HTTP URL for the OAuth Application""" - - class OrganizationAuditEntryConnection(sgqlc.types.relay.Connection): """The connection type for OrganizationAuditEntry.""" @@ -16589,24 +21771,6 @@ class OrganizationAuditEntryConnection(sgqlc.types.relay.Connection): """Identifies the total count of items in the connection.""" -class OrganizationAuditEntryData(sgqlc.types.Interface): - """Metadata for an audit entry with action org.*""" - - __schema__ = github_schema - __field_names__ = ("organization", "organization_name", "organization_resource_path", "organization_url") - organization = sgqlc.types.Field("Organization", graphql_name="organization") - """The Organization associated with the Audit Entry.""" - - organization_name = sgqlc.types.Field(String, graphql_name="organizationName") - """The name of the Organization.""" - - organization_resource_path = sgqlc.types.Field(URI, graphql_name="organizationResourcePath") - """The HTTP path for the organization""" - - organization_url = sgqlc.types.Field(URI, graphql_name="organizationUrl") - """The HTTP URL for the organization""" - - class OrganizationAuditEntryEdge(sgqlc.types.Type): """An edge in a connection.""" @@ -16812,47 +21976,6 @@ class PackageFileEdge(sgqlc.types.Type): """The item at the end of the edge.""" -class PackageOwner(sgqlc.types.Interface): - """Represents an owner of a package.""" - - __schema__ = github_schema - __field_names__ = ("id", "packages") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - - packages = sgqlc.types.Field( - sgqlc.types.non_null(PackageConnection), - graphql_name="packages", - args=sgqlc.types.ArgDict( - ( - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("names", sgqlc.types.Arg(sgqlc.types.list_of(String), graphql_name="names", default=None)), - ("repository_id", sgqlc.types.Arg(ID, graphql_name="repositoryId", default=None)), - ("package_type", sgqlc.types.Arg(PackageType, graphql_name="packageType", default=None)), - ("order_by", sgqlc.types.Arg(PackageOrder, graphql_name="orderBy", default={"field": "CREATED_AT", "direction": "DESC"})), - ) - ), - ) - """A list of packages under the owner. - - Arguments: - - * `after` (`String`): Returns the elements in the list that come - after the specified cursor. - * `before` (`String`): Returns the elements in the list that come - before the specified cursor. - * `first` (`Int`): Returns the first _n_ elements from the list. - * `last` (`Int`): Returns the last _n_ elements from the list. - * `names` (`[String]`): Find packages by their names. - * `repository_id` (`ID`): Find packages in a repository by ID. - * `package_type` (`PackageType`): Filter registry package by type. - * `order_by` (`PackageOrder`): Ordering of the returned packages. - (default: `{field: CREATED_AT, direction: DESC}`) - """ - - class PackageStatistics(sgqlc.types.Type): """Represents a object that contains package activity statistics such as downloads. @@ -17079,126 +22202,6 @@ class ProfileItemShowcase(sgqlc.types.Type): """ -class ProfileOwner(sgqlc.types.Interface): - """Represents any entity on GitHub that has a profile page.""" - - __schema__ = github_schema - __field_names__ = ( - "any_pinnable_items", - "email", - "id", - "item_showcase", - "location", - "login", - "name", - "pinnable_items", - "pinned_items", - "pinned_items_remaining", - "viewer_can_change_pinned_items", - "website_url", - ) - any_pinnable_items = sgqlc.types.Field( - sgqlc.types.non_null(Boolean), - graphql_name="anyPinnableItems", - args=sgqlc.types.ArgDict((("type", sgqlc.types.Arg(PinnableItemType, graphql_name="type", default=None)),)), - ) - """Determine if this repository owner has any items that can be - pinned to their profile. - - Arguments: - - * `type` (`PinnableItemType`): Filter to only a particular kind of - pinnable item. - """ - - email = sgqlc.types.Field(String, graphql_name="email") - """The public profile email.""" - - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - - item_showcase = sgqlc.types.Field(sgqlc.types.non_null(ProfileItemShowcase), graphql_name="itemShowcase") - """Showcases a selection of repositories and gists that the profile - owner has either curated or that have been selected automatically - based on popularity. - """ - - location = sgqlc.types.Field(String, graphql_name="location") - """The public profile location.""" - - login = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="login") - """The username used to login.""" - - name = sgqlc.types.Field(String, graphql_name="name") - """The public profile name.""" - - pinnable_items = sgqlc.types.Field( - sgqlc.types.non_null(PinnableItemConnection), - graphql_name="pinnableItems", - args=sgqlc.types.ArgDict( - ( - ("types", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(PinnableItemType)), graphql_name="types", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ) - ), - ) - """A list of repositories and gists this profile owner can pin to - their profile. - - Arguments: - - * `types` (`[PinnableItemType!]`): Filter the types of pinnable - items that are returned. - * `after` (`String`): Returns the elements in the list that come - after the specified cursor. - * `before` (`String`): Returns the elements in the list that come - before the specified cursor. - * `first` (`Int`): Returns the first _n_ elements from the list. - * `last` (`Int`): Returns the last _n_ elements from the list. - """ - - pinned_items = sgqlc.types.Field( - sgqlc.types.non_null(PinnableItemConnection), - graphql_name="pinnedItems", - args=sgqlc.types.ArgDict( - ( - ("types", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(PinnableItemType)), graphql_name="types", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ) - ), - ) - """A list of repositories and gists this profile owner has pinned to - their profile - - Arguments: - - * `types` (`[PinnableItemType!]`): Filter the types of pinned - items that are returned. - * `after` (`String`): Returns the elements in the list that come - after the specified cursor. - * `before` (`String`): Returns the elements in the list that come - before the specified cursor. - * `first` (`Int`): Returns the first _n_ elements from the list. - * `last` (`Int`): Returns the last _n_ elements from the list. - """ - - pinned_items_remaining = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="pinnedItemsRemaining") - """Returns how many more items this profile owner can pin to their - profile. - """ - - viewer_can_change_pinned_items = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanChangePinnedItems") - """Can the viewer pin repositories and gists to the profile?""" - - website_url = sgqlc.types.Field(URI, graphql_name="websiteUrl") - """The public profile website URL.""" - - class ProjectCardConnection(sgqlc.types.relay.Connection): """The connection type for ProjectCard.""" @@ -17289,15 +22292,52 @@ class ProjectEdge(sgqlc.types.Type): """The item at the end of the edge.""" -class ProjectNextConnection(sgqlc.types.relay.Connection): - """The connection type for ProjectNext.""" +class ProjectProgress(sgqlc.types.Type): + """Project progress stats.""" + + __schema__ = github_schema + __field_names__ = ( + "done_count", + "done_percentage", + "enabled", + "in_progress_count", + "in_progress_percentage", + "todo_count", + "todo_percentage", + ) + done_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="doneCount") + """The number of done cards.""" + + done_percentage = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="donePercentage") + """The percentage of done cards.""" + + enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="enabled") + """Whether progress tracking is enabled and cards with purpose exist + for this project + """ + + in_progress_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="inProgressCount") + """The number of in-progress cards.""" + + in_progress_percentage = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="inProgressPercentage") + """The percentage of in-progress cards.""" + + todo_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="todoCount") + """The number of to do cards.""" + + todo_percentage = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="todoPercentage") + """The percentage of to do cards.""" + + +class ProjectV2ActorConnection(sgqlc.types.relay.Connection): + """The connection type for ProjectV2Actor.""" __schema__ = github_schema __field_names__ = ("edges", "nodes", "page_info", "total_count") - edges = sgqlc.types.Field(sgqlc.types.list_of("ProjectNextEdge"), graphql_name="edges") + edges = sgqlc.types.Field(sgqlc.types.list_of("ProjectV2ActorEdge"), graphql_name="edges") """A list of edges.""" - nodes = sgqlc.types.Field(sgqlc.types.list_of("ProjectNext"), graphql_name="nodes") + nodes = sgqlc.types.Field(sgqlc.types.list_of("ProjectV2Actor"), graphql_name="nodes") """A list of nodes.""" page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") @@ -17307,7 +22347,7 @@ class ProjectNextConnection(sgqlc.types.relay.Connection): """Identifies the total count of items in the connection.""" -class ProjectNextEdge(sgqlc.types.Type): +class ProjectV2ActorEdge(sgqlc.types.Type): """An edge in a connection.""" __schema__ = github_schema @@ -17315,48 +22355,49 @@ class ProjectNextEdge(sgqlc.types.Type): cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") """A cursor for use in pagination.""" - node = sgqlc.types.Field("ProjectNext", graphql_name="node") + node = sgqlc.types.Field("ProjectV2Actor", graphql_name="node") """The item at the end of the edge.""" -class ProjectNextFieldCommon(sgqlc.types.Interface): - """Common fields across different field types""" +class ProjectV2Connection(sgqlc.types.relay.Connection): + """The connection type for ProjectV2.""" __schema__ = github_schema - __field_names__ = ("created_at", "data_type", "database_id", "id", "name", "project", "settings", "updated_at") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - """Identifies the date and time when the object was created.""" + __field_names__ = ("edges", "nodes", "page_info", "total_count") + edges = sgqlc.types.Field(sgqlc.types.list_of("ProjectV2Edge"), graphql_name="edges") + """A list of edges.""" - data_type = sgqlc.types.Field(sgqlc.types.non_null(ProjectNextFieldType), graphql_name="dataType") - """The field's type.""" + nodes = sgqlc.types.Field(sgqlc.types.list_of("ProjectV2"), graphql_name="nodes") + """A list of nodes.""" - database_id = sgqlc.types.Field(Int, graphql_name="databaseId") - """Identifies the primary key from the database.""" + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + """Information to aid in pagination.""" - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + total_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalCount") + """Identifies the total count of items in the connection.""" - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - """The project field's name.""" - project = sgqlc.types.Field(sgqlc.types.non_null("ProjectNext"), graphql_name="project") - """The project that contains this field.""" +class ProjectV2Edge(sgqlc.types.Type): + """An edge in a connection.""" - settings = sgqlc.types.Field(String, graphql_name="settings") - """The field's settings.""" + __schema__ = github_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + """A cursor for use in pagination.""" - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - """Identifies the date and time when the object was last updated.""" + node = sgqlc.types.Field("ProjectV2", graphql_name="node") + """The item at the end of the edge.""" -class ProjectNextFieldConnection(sgqlc.types.relay.Connection): - """The connection type for ProjectNextField.""" +class ProjectV2FieldConfigurationConnection(sgqlc.types.relay.Connection): + """The connection type for ProjectV2FieldConfiguration.""" __schema__ = github_schema __field_names__ = ("edges", "nodes", "page_info", "total_count") - edges = sgqlc.types.Field(sgqlc.types.list_of("ProjectNextFieldEdge"), graphql_name="edges") + edges = sgqlc.types.Field(sgqlc.types.list_of("ProjectV2FieldConfigurationEdge"), graphql_name="edges") """A list of edges.""" - nodes = sgqlc.types.Field(sgqlc.types.list_of("ProjectNextField"), graphql_name="nodes") + nodes = sgqlc.types.Field(sgqlc.types.list_of("ProjectV2FieldConfiguration"), graphql_name="nodes") """A list of nodes.""" page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") @@ -17366,7 +22407,7 @@ class ProjectNextFieldConnection(sgqlc.types.relay.Connection): """Identifies the total count of items in the connection.""" -class ProjectNextFieldEdge(sgqlc.types.Type): +class ProjectV2FieldConfigurationEdge(sgqlc.types.Type): """An edge in a connection.""" __schema__ = github_schema @@ -17374,19 +22415,19 @@ class ProjectNextFieldEdge(sgqlc.types.Type): cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") """A cursor for use in pagination.""" - node = sgqlc.types.Field("ProjectNextField", graphql_name="node") + node = sgqlc.types.Field("ProjectV2FieldConfiguration", graphql_name="node") """The item at the end of the edge.""" -class ProjectNextItemConnection(sgqlc.types.relay.Connection): - """The connection type for ProjectNextItem.""" +class ProjectV2FieldConnection(sgqlc.types.relay.Connection): + """The connection type for ProjectV2Field.""" __schema__ = github_schema __field_names__ = ("edges", "nodes", "page_info", "total_count") - edges = sgqlc.types.Field(sgqlc.types.list_of("ProjectNextItemEdge"), graphql_name="edges") + edges = sgqlc.types.Field(sgqlc.types.list_of("ProjectV2FieldEdge"), graphql_name="edges") """A list of edges.""" - nodes = sgqlc.types.Field(sgqlc.types.list_of("ProjectNextItem"), graphql_name="nodes") + nodes = sgqlc.types.Field(sgqlc.types.list_of("ProjectV2Field"), graphql_name="nodes") """A list of nodes.""" page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") @@ -17396,7 +22437,7 @@ class ProjectNextItemConnection(sgqlc.types.relay.Connection): """Identifies the total count of items in the connection.""" -class ProjectNextItemEdge(sgqlc.types.Type): +class ProjectV2FieldEdge(sgqlc.types.Type): """An edge in a connection.""" __schema__ = github_schema @@ -17404,19 +22445,19 @@ class ProjectNextItemEdge(sgqlc.types.Type): cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") """A cursor for use in pagination.""" - node = sgqlc.types.Field("ProjectNextItem", graphql_name="node") + node = sgqlc.types.Field("ProjectV2Field", graphql_name="node") """The item at the end of the edge.""" -class ProjectNextItemFieldValueConnection(sgqlc.types.relay.Connection): - """The connection type for ProjectNextItemFieldValue.""" +class ProjectV2ItemConnection(sgqlc.types.relay.Connection): + """The connection type for ProjectV2Item.""" __schema__ = github_schema __field_names__ = ("edges", "nodes", "page_info", "total_count") - edges = sgqlc.types.Field(sgqlc.types.list_of("ProjectNextItemFieldValueEdge"), graphql_name="edges") + edges = sgqlc.types.Field(sgqlc.types.list_of("ProjectV2ItemEdge"), graphql_name="edges") """A list of edges.""" - nodes = sgqlc.types.Field(sgqlc.types.list_of("ProjectNextItemFieldValue"), graphql_name="nodes") + nodes = sgqlc.types.Field(sgqlc.types.list_of("ProjectV2Item"), graphql_name="nodes") """A list of nodes.""" page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") @@ -17426,7 +22467,7 @@ class ProjectNextItemFieldValueConnection(sgqlc.types.relay.Connection): """Identifies the total count of items in the connection.""" -class ProjectNextItemFieldValueEdge(sgqlc.types.Type): +class ProjectV2ItemEdge(sgqlc.types.Type): """An edge in a connection.""" __schema__ = github_schema @@ -17434,44 +22475,80 @@ class ProjectNextItemFieldValueEdge(sgqlc.types.Type): cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") """A cursor for use in pagination.""" - node = sgqlc.types.Field("ProjectNextItemFieldValue", graphql_name="node") + node = sgqlc.types.Field("ProjectV2Item", graphql_name="node") """The item at the end of the edge.""" -class ProjectNextOwner(sgqlc.types.Interface): - """Represents an owner of a project (beta).""" +class ProjectV2ItemFieldLabelValue(sgqlc.types.Type): + """The value of the labels field in a Project item.""" __schema__ = github_schema - __field_names__ = ("id", "project_next", "projects_next") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + __field_names__ = ("field", "labels") + field = sgqlc.types.Field(sgqlc.types.non_null("ProjectV2FieldConfiguration"), graphql_name="field") + """The field that contains this value.""" - project_next = sgqlc.types.Field( - "ProjectNext", - graphql_name="projectNext", - args=sgqlc.types.ArgDict((("number", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="number", default=None)),)), + labels = sgqlc.types.Field( + LabelConnection, + graphql_name="labels", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), ) - """Find a project by project (beta) number. + """Labels value of a field Arguments: - * `number` (`Int!`): The project (beta) number. + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. """ - projects_next = sgqlc.types.Field( - sgqlc.types.non_null(ProjectNextConnection), - graphql_name="projectsNext", + +class ProjectV2ItemFieldMilestoneValue(sgqlc.types.Type): + """The value of a milestone field in a Project item.""" + + __schema__ = github_schema + __field_names__ = ("field", "milestone") + field = sgqlc.types.Field(sgqlc.types.non_null("ProjectV2FieldConfiguration"), graphql_name="field") + """The field that contains this value.""" + + milestone = sgqlc.types.Field("Milestone", graphql_name="milestone") + """Milestone value of a field""" + + +class ProjectV2ItemFieldPullRequestValue(sgqlc.types.Type): + """The value of a pull request field in a Project item.""" + + __schema__ = github_schema + __field_names__ = ("field", "pull_requests") + field = sgqlc.types.Field(sgqlc.types.non_null("ProjectV2FieldConfiguration"), graphql_name="field") + """The field that contains this value.""" + + pull_requests = sgqlc.types.Field( + "PullRequestConnection", + graphql_name="pullRequests", args=sgqlc.types.ArgDict( ( ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("sort_by", sgqlc.types.Arg(ProjectNextOrderField, graphql_name="sortBy", default="TITLE")), + ( + "order_by", + sgqlc.types.Arg(PullRequestOrder, graphql_name="orderBy", default={"field": "CREATED_AT", "direction": "ASC"}), + ), ) ), ) - """A list of projects (beta) under the owner. + """The pull requests for this field Arguments: @@ -17481,40 +22558,69 @@ class ProjectNextOwner(sgqlc.types.Interface): before the specified cursor. * `first` (`Int`): Returns the first _n_ elements from the list. * `last` (`Int`): Returns the last _n_ elements from the list. - * `query` (`String`): A project (beta) to search for under the the - owner. - * `sort_by` (`ProjectNextOrderField`): How to order the returned - projects (beta). (default: `TITLE`) + * `order_by` (`PullRequestOrder`): Ordering options for pull + requests. (default: `{field: CREATED_AT, direction: ASC}`) """ -class ProjectOwner(sgqlc.types.Interface): - """Represents an owner of a Project.""" +class ProjectV2ItemFieldRepositoryValue(sgqlc.types.Type): + """The value of a repository field in a Project item.""" __schema__ = github_schema - __field_names__ = ("id", "project", "projects", "projects_resource_path", "projects_url", "viewer_can_create_projects") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + __field_names__ = ("field", "repository") + field = sgqlc.types.Field(sgqlc.types.non_null("ProjectV2FieldConfiguration"), graphql_name="field") + """The field that contains this value.""" - project = sgqlc.types.Field( - "Project", - graphql_name="project", - args=sgqlc.types.ArgDict((("number", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="number", default=None)),)), + repository = sgqlc.types.Field("Repository", graphql_name="repository") + """The repository for this field.""" + + +class ProjectV2ItemFieldReviewerValue(sgqlc.types.Type): + """The value of a reviewers field in a Project item.""" + + __schema__ = github_schema + __field_names__ = ("field", "reviewers") + field = sgqlc.types.Field(sgqlc.types.non_null("ProjectV2FieldConfiguration"), graphql_name="field") + """The field that contains this value.""" + + reviewers = sgqlc.types.Field( + "RequestedReviewerConnection", + graphql_name="reviewers", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), ) - """Find project by number. + """The reviewers for this field. Arguments: - * `number` (`Int!`): The project number to find. + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. """ - projects = sgqlc.types.Field( - sgqlc.types.non_null(ProjectConnection), - graphql_name="projects", + +class ProjectV2ItemFieldUserValue(sgqlc.types.Type): + """The value of a user field in a Project item.""" + + __schema__ = github_schema + __field_names__ = ("field", "users") + field = sgqlc.types.Field(sgqlc.types.non_null("ProjectV2FieldConfiguration"), graphql_name="field") + """The field that contains this value.""" + + users = sgqlc.types.Field( + "UserConnection", + graphql_name="users", args=sgqlc.types.ArgDict( ( - ("order_by", sgqlc.types.Arg(ProjectOrder, graphql_name="orderBy", default=None)), - ("search", sgqlc.types.Arg(String, graphql_name="search", default=None)), - ("states", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ProjectState)), graphql_name="states", default=None)), ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), @@ -17522,16 +22628,10 @@ class ProjectOwner(sgqlc.types.Interface): ) ), ) - """A list of projects under the owner. + """The users for this field Arguments: - * `order_by` (`ProjectOrder`): Ordering options for projects - returned from the connection - * `search` (`String`): Query to search projects by, currently only - searching by name. - * `states` (`[ProjectState!]`): A list of states to filter the - projects by. * `after` (`String`): Returns the elements in the list that come after the specified cursor. * `before` (`String`): Returns the elements in the list that come @@ -17540,62 +22640,159 @@ class ProjectOwner(sgqlc.types.Interface): * `last` (`Int`): Returns the last _n_ elements from the list. """ - projects_resource_path = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="projectsResourcePath") - """The HTTP path listing owners projects""" - projects_url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="projectsUrl") - """The HTTP URL listing owners projects""" +class ProjectV2ItemFieldValueConnection(sgqlc.types.relay.Connection): + """The connection type for ProjectV2ItemFieldValue.""" - viewer_can_create_projects = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanCreateProjects") - """Can the current viewer create new projects on this owner.""" + __schema__ = github_schema + __field_names__ = ("edges", "nodes", "page_info", "total_count") + edges = sgqlc.types.Field(sgqlc.types.list_of("ProjectV2ItemFieldValueEdge"), graphql_name="edges") + """A list of edges.""" + nodes = sgqlc.types.Field(sgqlc.types.list_of("ProjectV2ItemFieldValue"), graphql_name="nodes") + """A list of nodes.""" -class ProjectProgress(sgqlc.types.Type): - """Project progress stats.""" + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + """Information to aid in pagination.""" + + total_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalCount") + """Identifies the total count of items in the connection.""" + + +class ProjectV2ItemFieldValueEdge(sgqlc.types.Type): + """An edge in a connection.""" __schema__ = github_schema - __field_names__ = ( - "done_count", - "done_percentage", - "enabled", - "in_progress_count", - "in_progress_percentage", - "todo_count", - "todo_percentage", + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + """A cursor for use in pagination.""" + + node = sgqlc.types.Field("ProjectV2ItemFieldValue", graphql_name="node") + """The item at the end of the edge.""" + + +class ProjectV2IterationFieldConfiguration(sgqlc.types.Type): + """Iteration field configuration for a project.""" + + __schema__ = github_schema + __field_names__ = ("completed_iterations", "duration", "iterations", "start_day") + completed_iterations = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProjectV2IterationFieldIteration"))), + graphql_name="completedIterations", ) - done_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="doneCount") - """The number of done cards.""" + """The iteration's completed iterations""" - done_percentage = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="donePercentage") - """The percentage of done cards.""" + duration = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="duration") + """The iteration's duration in days""" - enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="enabled") - """Whether progress tracking is enabled and cards with purpose exist - for this project - """ + iterations = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProjectV2IterationFieldIteration"))), graphql_name="iterations" + ) + """The iteration's iterations""" - in_progress_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="inProgressCount") - """The number of in-progress cards.""" + start_day = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="startDay") + """The iteration's start day of the week""" - in_progress_percentage = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="inProgressPercentage") - """The percentage of in-progress cards.""" - todo_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="todoCount") - """The number of to do cards.""" +class ProjectV2IterationFieldIteration(sgqlc.types.Type): + """Iteration field iteration settings for a project.""" + + __schema__ = github_schema + __field_names__ = ("duration", "id", "start_date", "title", "title_html") + duration = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="duration") + """The iteration's duration in days""" + + id = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="id") + """The iteration's ID.""" + + start_date = sgqlc.types.Field(sgqlc.types.non_null(Date), graphql_name="startDate") + """The iteration's start date""" + + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + """The iteration's title.""" + + title_html = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="titleHTML") + """The iteration's html title.""" - todo_percentage = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="todoPercentage") - """The percentage of to do cards.""" +class ProjectV2SingleSelectFieldOption(sgqlc.types.Type): + """Single select field option for a configuration for a project.""" -class ProjectViewConnection(sgqlc.types.relay.Connection): - """The connection type for ProjectView.""" + __schema__ = github_schema + __field_names__ = ("id", "name", "name_html") + id = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="id") + """The option's ID.""" + + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + """The option's name.""" + + name_html = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="nameHTML") + """The option's html name.""" + + +class ProjectV2SortBy(sgqlc.types.Type): + """Represents a sort by field and direction.""" + + __schema__ = github_schema + __field_names__ = ("direction", "field") + direction = sgqlc.types.Field(sgqlc.types.non_null(OrderDirection), graphql_name="direction") + """The direction of the sorting. Possible values are ASC and DESC.""" + + field = sgqlc.types.Field(sgqlc.types.non_null("ProjectV2Field"), graphql_name="field") + """The field by which items are sorted.""" + + +class ProjectV2SortByConnection(sgqlc.types.relay.Connection): + """The connection type for ProjectV2SortBy.""" + + __schema__ = github_schema + __field_names__ = ("edges", "nodes", "page_info", "total_count") + edges = sgqlc.types.Field(sgqlc.types.list_of("ProjectV2SortByEdge"), graphql_name="edges") + """A list of edges.""" + + nodes = sgqlc.types.Field(sgqlc.types.list_of(ProjectV2SortBy), graphql_name="nodes") + """A list of nodes.""" + + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + """Information to aid in pagination.""" + + total_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalCount") + """Identifies the total count of items in the connection.""" + + +class ProjectV2SortByEdge(sgqlc.types.Type): + """An edge in a connection.""" + + __schema__ = github_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + """A cursor for use in pagination.""" + + node = sgqlc.types.Field(ProjectV2SortBy, graphql_name="node") + """The item at the end of the edge.""" + + +class ProjectV2SortByField(sgqlc.types.Type): + """Represents a sort by field and direction.""" + + __schema__ = github_schema + __field_names__ = ("direction", "field") + direction = sgqlc.types.Field(sgqlc.types.non_null(OrderDirection), graphql_name="direction") + """The direction of the sorting. Possible values are ASC and DESC.""" + + field = sgqlc.types.Field(sgqlc.types.non_null("ProjectV2FieldConfiguration"), graphql_name="field") + """The field by which items are sorted.""" + + +class ProjectV2SortByFieldConnection(sgqlc.types.relay.Connection): + """The connection type for ProjectV2SortByField.""" __schema__ = github_schema __field_names__ = ("edges", "nodes", "page_info", "total_count") - edges = sgqlc.types.Field(sgqlc.types.list_of("ProjectViewEdge"), graphql_name="edges") + edges = sgqlc.types.Field(sgqlc.types.list_of("ProjectV2SortByFieldEdge"), graphql_name="edges") """A list of edges.""" - nodes = sgqlc.types.Field(sgqlc.types.list_of("ProjectView"), graphql_name="nodes") + nodes = sgqlc.types.Field(sgqlc.types.list_of(ProjectV2SortByField), graphql_name="nodes") """A list of nodes.""" page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") @@ -17605,7 +22802,7 @@ class ProjectViewConnection(sgqlc.types.relay.Connection): """Identifies the total count of items in the connection.""" -class ProjectViewEdge(sgqlc.types.Type): +class ProjectV2SortByFieldEdge(sgqlc.types.Type): """An edge in a connection.""" __schema__ = github_schema @@ -17613,7 +22810,67 @@ class ProjectViewEdge(sgqlc.types.Type): cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") """A cursor for use in pagination.""" - node = sgqlc.types.Field("ProjectView", graphql_name="node") + node = sgqlc.types.Field(ProjectV2SortByField, graphql_name="node") + """The item at the end of the edge.""" + + +class ProjectV2ViewConnection(sgqlc.types.relay.Connection): + """The connection type for ProjectV2View.""" + + __schema__ = github_schema + __field_names__ = ("edges", "nodes", "page_info", "total_count") + edges = sgqlc.types.Field(sgqlc.types.list_of("ProjectV2ViewEdge"), graphql_name="edges") + """A list of edges.""" + + nodes = sgqlc.types.Field(sgqlc.types.list_of("ProjectV2View"), graphql_name="nodes") + """A list of nodes.""" + + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + """Information to aid in pagination.""" + + total_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalCount") + """Identifies the total count of items in the connection.""" + + +class ProjectV2ViewEdge(sgqlc.types.Type): + """An edge in a connection.""" + + __schema__ = github_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + """A cursor for use in pagination.""" + + node = sgqlc.types.Field("ProjectV2View", graphql_name="node") + """The item at the end of the edge.""" + + +class ProjectV2WorkflowConnection(sgqlc.types.relay.Connection): + """The connection type for ProjectV2Workflow.""" + + __schema__ = github_schema + __field_names__ = ("edges", "nodes", "page_info", "total_count") + edges = sgqlc.types.Field(sgqlc.types.list_of("ProjectV2WorkflowEdge"), graphql_name="edges") + """A list of edges.""" + + nodes = sgqlc.types.Field(sgqlc.types.list_of("ProjectV2Workflow"), graphql_name="nodes") + """A list of nodes.""" + + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + """Information to aid in pagination.""" + + total_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalCount") + """Identifies the total count of items in the connection.""" + + +class ProjectV2WorkflowEdge(sgqlc.types.Type): + """An edge in a connection.""" + + __schema__ = github_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + """A cursor for use in pagination.""" + + node = sgqlc.types.Field("ProjectV2Workflow", graphql_name="node") """The item at the end of the edge.""" @@ -17647,6 +22904,18 @@ class PublicKeyEdge(sgqlc.types.Type): """The item at the end of the edge.""" +class PublishSponsorsTierPayload(sgqlc.types.Type): + """Autogenerated return type of PublishSponsorsTier""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "sponsors_tier") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + sponsors_tier = sgqlc.types.Field("SponsorsTier", graphql_name="sponsorsTier") + """The tier that was published.""" + + class PullRequestChangedFile(sgqlc.types.Type): """A file changed in a pull request.""" @@ -17797,6 +23066,45 @@ class PullRequestEdge(sgqlc.types.Type): """The item at the end of the edge.""" +class PullRequestParameters(sgqlc.types.Type): + """Require all commits be made to a non-target branch and submitted + via a pull request before they can be merged. + """ + + __schema__ = github_schema + __field_names__ = ( + "dismiss_stale_reviews_on_push", + "require_code_owner_review", + "require_last_push_approval", + "required_approving_review_count", + "required_review_thread_resolution", + ) + dismiss_stale_reviews_on_push = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="dismissStaleReviewsOnPush") + """New, reviewable commits pushed will dismiss previous pull request + review approvals. + """ + + require_code_owner_review = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requireCodeOwnerReview") + """Require an approving review in pull requests that modify files + that have a designated code owner. + """ + + require_last_push_approval = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requireLastPushApproval") + """Whether the most recent reviewable push must be approved by + someone other than the person who pushed it. + """ + + required_approving_review_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="requiredApprovingReviewCount") + """The number of approving reviews that are required before a pull + request can be merged. + """ + + required_review_thread_resolution = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiredReviewThreadResolution") + """All conversations on code must be resolved before a pull request + can be merged. + """ + + class PullRequestReviewCommentConnection(sgqlc.types.relay.Connection): """The connection type for PullRequestReviewComment.""" @@ -18355,8 +23663,9 @@ class Query(sgqlc.types.Type): """ relay = sgqlc.types.Field(sgqlc.types.non_null("Query"), graphql_name="relay") - """Hack to workaround https://github.com/facebook/relay/issues/112 - re-exposing the root query object + """Workaround for re-exposing the root query object. (Refer to + https://github.com/facebook/relay/issues/112 for more + information.) """ repository = sgqlc.types.Field( @@ -18382,7 +23691,7 @@ class Query(sgqlc.types.Type): """ repository_owner = sgqlc.types.Field( - "RepositoryOwner", + RepositoryOwner, graphql_name="repositoryOwner", args=sgqlc.types.ArgDict((("login", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="login", default=None)),)), ) @@ -18395,7 +23704,7 @@ class Query(sgqlc.types.Type): """ resource = sgqlc.types.Field( - "UniformResourceLocatable", + UniformResourceLocatable, graphql_name="resource", args=sgqlc.types.ArgDict((("url", sgqlc.types.Arg(sgqlc.types.non_null(URI), graphql_name="url", default=None)),)), ) @@ -18420,7 +23729,8 @@ class Query(sgqlc.types.Type): ) ), ) - """Perform a search across resources. + """Perform a search across resources, returning a maximum of 1,000 + results. Arguments: @@ -18666,53 +23976,6 @@ class RateLimit(sgqlc.types.Type): """The number of points used in the current rate limit window.""" -class Reactable(sgqlc.types.Interface): - """Represents a subject that can be reacted on.""" - - __schema__ = github_schema - __field_names__ = ("database_id", "id", "reaction_groups", "reactions", "viewer_can_react") - database_id = sgqlc.types.Field(Int, graphql_name="databaseId") - """Identifies the primary key from the database.""" - - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - - reaction_groups = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("ReactionGroup")), graphql_name="reactionGroups") - """A list of reactions grouped by content left on the subject.""" - - reactions = sgqlc.types.Field( - sgqlc.types.non_null("ReactionConnection"), - graphql_name="reactions", - args=sgqlc.types.ArgDict( - ( - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("content", sgqlc.types.Arg(ReactionContent, graphql_name="content", default=None)), - ("order_by", sgqlc.types.Arg(ReactionOrder, graphql_name="orderBy", default=None)), - ) - ), - ) - """A list of Reactions left on the Issue. - - Arguments: - - * `after` (`String`): Returns the elements in the list that come - after the specified cursor. - * `before` (`String`): Returns the elements in the list that come - before the specified cursor. - * `first` (`Int`): Returns the first _n_ elements from the list. - * `last` (`Int`): Returns the last _n_ elements from the list. - * `content` (`ReactionContent`): Allows filtering Reactions by - emoji. - * `order_by` (`ReactionOrder`): Allows specifying the order in - which reactions are returned. - """ - - viewer_can_react = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanReact") - """Can user react to this subject""" - - class ReactingUserConnection(sgqlc.types.relay.Connection): """The connection type for User.""" @@ -18888,6 +24151,24 @@ class RefEdge(sgqlc.types.Type): """The item at the end of the edge.""" +class RefNameConditionTarget(sgqlc.types.Type): + """Parameters to be used for the ref_name condition""" + + __schema__ = github_schema + __field_names__ = ("exclude", "include") + exclude = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="exclude") + """Array of ref names or patterns to exclude. The condition will not + pass if any of these patterns match. + """ + + include = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="include") + """Array of ref names or patterns to include. One of these patterns + must match for the condition to pass. Also accepts + `~DEFAULT_BRANCH` to include the default branch or `~ALL` to + include all branches. + """ + + class RefUpdateRule(sgqlc.types.Type): """A ref update rules for a viewer.""" @@ -19088,6 +24369,24 @@ class RemoveEnterpriseIdentityProviderPayload(sgqlc.types.Type): """The identity provider that was removed from the enterprise.""" +class RemoveEnterpriseMemberPayload(sgqlc.types.Type): + """Autogenerated return type of RemoveEnterpriseMember""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "enterprise", "user", "viewer") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + enterprise = sgqlc.types.Field("Enterprise", graphql_name="enterprise") + """The updated enterprise.""" + + user = sgqlc.types.Field("User", graphql_name="user") + """The user that was removed from the enterprise.""" + + viewer = sgqlc.types.Field("User", graphql_name="viewer") + """The viewer performing the mutation.""" + + class RemoveEnterpriseOrganizationPayload(sgqlc.types.Type): """Autogenerated return type of RemoveEnterpriseOrganization""" @@ -19148,13 +24447,16 @@ class RemoveReactionPayload(sgqlc.types.Type): """Autogenerated return type of RemoveReaction""" __schema__ = github_schema - __field_names__ = ("client_mutation_id", "reaction", "subject") + __field_names__ = ("client_mutation_id", "reaction", "reaction_groups", "subject") client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" reaction = sgqlc.types.Field("Reaction", graphql_name="reaction") """The reaction object.""" + reaction_groups = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ReactionGroup)), graphql_name="reactionGroups") + """The reaction groups for the subject.""" + subject = sgqlc.types.Field(Reactable, graphql_name="subject") """The reactable subject.""" @@ -19167,7 +24469,7 @@ class RemoveStarPayload(sgqlc.types.Type): client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" - starrable = sgqlc.types.Field("Starrable", graphql_name="starrable") + starrable = sgqlc.types.Field(Starrable, graphql_name="starrable") """The starrable.""" @@ -19179,10 +24481,22 @@ class RemoveUpvotePayload(sgqlc.types.Type): client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" - subject = sgqlc.types.Field("Votable", graphql_name="subject") + subject = sgqlc.types.Field(Votable, graphql_name="subject") """The votable subject.""" +class ReopenDiscussionPayload(sgqlc.types.Type): + """Autogenerated return type of ReopenDiscussion""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "discussion") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + discussion = sgqlc.types.Field("Discussion", graphql_name="discussion") + """The discussion that was reopened.""" + + class ReopenIssuePayload(sgqlc.types.Type): """Autogenerated return type of ReopenIssue""" @@ -19207,24 +24521,6 @@ class ReopenPullRequestPayload(sgqlc.types.Type): """The pull request that was reopened.""" -class RepositoryAuditEntryData(sgqlc.types.Interface): - """Metadata for an audit entry with action repo.*""" - - __schema__ = github_schema - __field_names__ = ("repository", "repository_name", "repository_resource_path", "repository_url") - repository = sgqlc.types.Field("Repository", graphql_name="repository") - """The repository associated with the action""" - - repository_name = sgqlc.types.Field(String, graphql_name="repositoryName") - """The name of the repository""" - - repository_resource_path = sgqlc.types.Field(URI, graphql_name="repositoryResourcePath") - """The HTTP path for the repository""" - - repository_url = sgqlc.types.Field(URI, graphql_name="repositoryUrl") - """The HTTP URL for the repository""" - - class RepositoryCodeowners(sgqlc.types.Type): """Information extracted from a repository's `CODEOWNERS` file.""" @@ -19338,86 +24634,6 @@ class RepositoryContactLink(sgqlc.types.Type): """The contact link URL.""" -class RepositoryDiscussionAuthor(sgqlc.types.Interface): - """Represents an author of discussions in repositories.""" - - __schema__ = github_schema - __field_names__ = ("repository_discussions",) - repository_discussions = sgqlc.types.Field( - sgqlc.types.non_null(DiscussionConnection), - graphql_name="repositoryDiscussions", - args=sgqlc.types.ArgDict( - ( - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ( - "order_by", - sgqlc.types.Arg(DiscussionOrder, graphql_name="orderBy", default={"field": "CREATED_AT", "direction": "DESC"}), - ), - ("repository_id", sgqlc.types.Arg(ID, graphql_name="repositoryId", default=None)), - ("answered", sgqlc.types.Arg(Boolean, graphql_name="answered", default=None)), - ) - ), - ) - """Discussions this user has started. - - Arguments: - - * `after` (`String`): Returns the elements in the list that come - after the specified cursor. - * `before` (`String`): Returns the elements in the list that come - before the specified cursor. - * `first` (`Int`): Returns the first _n_ elements from the list. - * `last` (`Int`): Returns the last _n_ elements from the list. - * `order_by` (`DiscussionOrder`): Ordering options for discussions - returned from the connection. (default: `{field: CREATED_AT, - direction: DESC}`) - * `repository_id` (`ID`): Filter discussions to only those in a - specific repository. - * `answered` (`Boolean`): Filter discussions to only those that - have been answered or not. Defaults to including both answered - and unanswered discussions. (default: `null`) - """ - - -class RepositoryDiscussionCommentAuthor(sgqlc.types.Interface): - """Represents an author of discussion comments in repositories.""" - - __schema__ = github_schema - __field_names__ = ("repository_discussion_comments",) - repository_discussion_comments = sgqlc.types.Field( - sgqlc.types.non_null(DiscussionCommentConnection), - graphql_name="repositoryDiscussionComments", - args=sgqlc.types.ArgDict( - ( - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("repository_id", sgqlc.types.Arg(ID, graphql_name="repositoryId", default=None)), - ("only_answers", sgqlc.types.Arg(Boolean, graphql_name="onlyAnswers", default=False)), - ) - ), - ) - """Discussion comments this user has authored. - - Arguments: - - * `after` (`String`): Returns the elements in the list that come - after the specified cursor. - * `before` (`String`): Returns the elements in the list that come - before the specified cursor. - * `first` (`Int`): Returns the first _n_ elements from the list. - * `last` (`Int`): Returns the last _n_ elements from the list. - * `repository_id` (`ID`): Filter discussion comments to only those - in a specific repository. - * `only_answers` (`Boolean`): Filter discussion comments to only - those that were marked as the answer (default: `false`) - """ - - class RepositoryEdge(sgqlc.types.Type): """An edge in a connection.""" @@ -19430,171 +24646,124 @@ class RepositoryEdge(sgqlc.types.Type): """The item at the end of the edge.""" -class RepositoryInfo(sgqlc.types.Interface): - """A subset of repository info.""" +class RepositoryInteractionAbility(sgqlc.types.Type): + """Repository interaction limit that applies to this object.""" __schema__ = github_schema - __field_names__ = ( - "created_at", - "description", - "description_html", - "fork_count", - "has_issues_enabled", - "has_projects_enabled", - "has_wiki_enabled", - "homepage_url", - "is_archived", - "is_fork", - "is_in_organization", - "is_locked", - "is_mirror", - "is_private", - "is_template", - "license_info", - "lock_reason", - "mirror_url", - "name", - "name_with_owner", - "open_graph_image_url", - "owner", - "pushed_at", - "resource_path", - "short_description_html", - "updated_at", - "url", - "uses_custom_open_graph_image", - "visibility", - ) - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - """Identifies the date and time when the object was created.""" - - description = sgqlc.types.Field(String, graphql_name="description") - """The description of the repository.""" + __field_names__ = ("expires_at", "limit", "origin") + expires_at = sgqlc.types.Field(DateTime, graphql_name="expiresAt") + """The time the currently active limit expires.""" - description_html = sgqlc.types.Field(sgqlc.types.non_null(HTML), graphql_name="descriptionHTML") - """The description of the repository rendered to HTML.""" + limit = sgqlc.types.Field(sgqlc.types.non_null(RepositoryInteractionLimit), graphql_name="limit") + """The current limit that is enabled on this object.""" - fork_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="forkCount") - """Returns how many forks there are of this repository in the whole - network. - """ + origin = sgqlc.types.Field(sgqlc.types.non_null(RepositoryInteractionLimitOrigin), graphql_name="origin") + """The origin of the currently active interaction limit.""" - has_issues_enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasIssuesEnabled") - """Indicates if the repository has issues feature enabled.""" - has_projects_enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasProjectsEnabled") - """Indicates if the repository has the Projects feature enabled.""" +class RepositoryInvitationConnection(sgqlc.types.relay.Connection): + """A list of repository invitations.""" - has_wiki_enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasWikiEnabled") - """Indicates if the repository has wiki feature enabled.""" + __schema__ = github_schema + __field_names__ = ("edges", "nodes", "page_info", "total_count") + edges = sgqlc.types.Field(sgqlc.types.list_of("RepositoryInvitationEdge"), graphql_name="edges") + """A list of edges.""" - homepage_url = sgqlc.types.Field(URI, graphql_name="homepageUrl") - """The repository's URL.""" + nodes = sgqlc.types.Field(sgqlc.types.list_of("RepositoryInvitation"), graphql_name="nodes") + """A list of nodes.""" - is_archived = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isArchived") - """Indicates if the repository is unmaintained.""" + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + """Information to aid in pagination.""" - is_fork = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isFork") - """Identifies if the repository is a fork.""" + total_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalCount") + """Identifies the total count of items in the connection.""" - is_in_organization = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isInOrganization") - """Indicates if a repository is either owned by an organization, or - is a private fork of an organization repository. - """ - is_locked = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isLocked") - """Indicates if the repository has been locked or not.""" +class RepositoryInvitationEdge(sgqlc.types.Type): + """An edge in a connection.""" - is_mirror = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isMirror") - """Identifies if the repository is a mirror.""" + __schema__ = github_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + """A cursor for use in pagination.""" - is_private = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isPrivate") - """Identifies if the repository is private or internal.""" + node = sgqlc.types.Field("RepositoryInvitation", graphql_name="node") + """The item at the end of the edge.""" - is_template = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isTemplate") - """Identifies if the repository is a template that can be used to - generate new repositories. - """ - license_info = sgqlc.types.Field("License", graphql_name="licenseInfo") - """The license associated with the repository""" +class RepositoryMigrationConnection(sgqlc.types.relay.Connection): + """The connection type for RepositoryMigration.""" - lock_reason = sgqlc.types.Field(RepositoryLockReason, graphql_name="lockReason") - """The reason the repository has been locked.""" + __schema__ = github_schema + __field_names__ = ("edges", "nodes", "page_info", "total_count") + edges = sgqlc.types.Field(sgqlc.types.list_of("RepositoryMigrationEdge"), graphql_name="edges") + """A list of edges.""" - mirror_url = sgqlc.types.Field(URI, graphql_name="mirrorUrl") - """The repository's original mirror URL.""" + nodes = sgqlc.types.Field(sgqlc.types.list_of("RepositoryMigration"), graphql_name="nodes") + """A list of nodes.""" - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - """The name of the repository.""" + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + """Information to aid in pagination.""" - name_with_owner = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="nameWithOwner") - """The repository's name with owner.""" + total_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalCount") + """Identifies the total count of items in the connection.""" - open_graph_image_url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="openGraphImageUrl") - """The image used to represent this repository in Open Graph data.""" - owner = sgqlc.types.Field(sgqlc.types.non_null("RepositoryOwner"), graphql_name="owner") - """The User owner of the repository.""" +class RepositoryMigrationEdge(sgqlc.types.Type): + """Represents a repository migration.""" - pushed_at = sgqlc.types.Field(DateTime, graphql_name="pushedAt") - """Identifies when the repository was last pushed to.""" + __schema__ = github_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + """A cursor for use in pagination.""" - resource_path = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="resourcePath") - """The HTTP path for this repository""" + node = sgqlc.types.Field("RepositoryMigration", graphql_name="node") + """The item at the end of the edge.""" - short_description_html = sgqlc.types.Field( - sgqlc.types.non_null(HTML), - graphql_name="shortDescriptionHTML", - args=sgqlc.types.ArgDict((("limit", sgqlc.types.Arg(Int, graphql_name="limit", default=200)),)), - ) - """A description of the repository, rendered to HTML without any - links in it. - Arguments: +class RepositoryNameConditionTarget(sgqlc.types.Type): + """Parameters to be used for the repository_name condition""" - * `limit` (`Int`): How many characters to return. (default: `200`) + __schema__ = github_schema + __field_names__ = ("exclude", "include", "protected") + exclude = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="exclude") + """Array of repository names or patterns to exclude. The condition + will not pass if any of these patterns match. """ - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - """Identifies the date and time when the object was last updated.""" - - url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="url") - """The HTTP URL for this repository""" - - uses_custom_open_graph_image = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="usesCustomOpenGraphImage") - """Whether this repository has a custom image to use with Open Graph - as opposed to being represented by the owner's avatar. + include = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="include") + """Array of repository names or patterns to include. One of these + patterns must match for the condition to pass. Also accepts `~ALL` + to include all repositories. """ - visibility = sgqlc.types.Field(sgqlc.types.non_null(RepositoryVisibility), graphql_name="visibility") - """Indicates the repository's visibility level.""" + protected = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="protected") + """Target changes that match these patterns will be prevented except + by those with bypass permissions. + """ -class RepositoryInteractionAbility(sgqlc.types.Type): - """Repository interaction limit that applies to this object.""" +class RepositoryRuleConditions(sgqlc.types.Type): + """Set of conditions that determine if a ruleset will evaluate""" __schema__ = github_schema - __field_names__ = ("expires_at", "limit", "origin") - expires_at = sgqlc.types.Field(DateTime, graphql_name="expiresAt") - """The time the currently active limit expires.""" + __field_names__ = ("ref_name", "repository_name") + ref_name = sgqlc.types.Field(RefNameConditionTarget, graphql_name="refName") + """Configuration for the ref_name condition""" - limit = sgqlc.types.Field(sgqlc.types.non_null(RepositoryInteractionLimit), graphql_name="limit") - """The current limit that is enabled on this object.""" + repository_name = sgqlc.types.Field(RepositoryNameConditionTarget, graphql_name="repositoryName") + """Configuration for the repository_name condition""" - origin = sgqlc.types.Field(sgqlc.types.non_null(RepositoryInteractionLimitOrigin), graphql_name="origin") - """The origin of the currently active interaction limit.""" - -class RepositoryInvitationConnection(sgqlc.types.relay.Connection): - """A list of repository invitations.""" +class RepositoryRuleConnection(sgqlc.types.relay.Connection): + """The connection type for RepositoryRule.""" __schema__ = github_schema __field_names__ = ("edges", "nodes", "page_info", "total_count") - edges = sgqlc.types.Field(sgqlc.types.list_of("RepositoryInvitationEdge"), graphql_name="edges") + edges = sgqlc.types.Field(sgqlc.types.list_of("RepositoryRuleEdge"), graphql_name="edges") """A list of edges.""" - nodes = sgqlc.types.Field(sgqlc.types.list_of("RepositoryInvitation"), graphql_name="nodes") + nodes = sgqlc.types.Field(sgqlc.types.list_of("RepositoryRule"), graphql_name="nodes") """A list of nodes.""" page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") @@ -19604,7 +24773,7 @@ class RepositoryInvitationConnection(sgqlc.types.relay.Connection): """Identifies the total count of items in the connection.""" -class RepositoryInvitationEdge(sgqlc.types.Type): +class RepositoryRuleEdge(sgqlc.types.Type): """An edge in a connection.""" __schema__ = github_schema @@ -19612,19 +24781,19 @@ class RepositoryInvitationEdge(sgqlc.types.Type): cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") """A cursor for use in pagination.""" - node = sgqlc.types.Field("RepositoryInvitation", graphql_name="node") + node = sgqlc.types.Field("RepositoryRule", graphql_name="node") """The item at the end of the edge.""" -class RepositoryMigrationConnection(sgqlc.types.relay.Connection): - """The connection type for RepositoryMigration.""" +class RepositoryRulesetBypassActorConnection(sgqlc.types.relay.Connection): + """The connection type for RepositoryRulesetBypassActor.""" __schema__ = github_schema __field_names__ = ("edges", "nodes", "page_info", "total_count") - edges = sgqlc.types.Field(sgqlc.types.list_of("RepositoryMigrationEdge"), graphql_name="edges") + edges = sgqlc.types.Field(sgqlc.types.list_of("RepositoryRulesetBypassActorEdge"), graphql_name="edges") """A list of edges.""" - nodes = sgqlc.types.Field(sgqlc.types.list_of("RepositoryMigration"), graphql_name="nodes") + nodes = sgqlc.types.Field(sgqlc.types.list_of("RepositoryRulesetBypassActor"), graphql_name="nodes") """A list of nodes.""" page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") @@ -19634,126 +24803,46 @@ class RepositoryMigrationConnection(sgqlc.types.relay.Connection): """Identifies the total count of items in the connection.""" -class RepositoryMigrationEdge(sgqlc.types.Type): - """Represents a repository migration.""" +class RepositoryRulesetBypassActorEdge(sgqlc.types.Type): + """An edge in a connection.""" __schema__ = github_schema __field_names__ = ("cursor", "node") cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") """A cursor for use in pagination.""" - node = sgqlc.types.Field("RepositoryMigration", graphql_name="node") + node = sgqlc.types.Field("RepositoryRulesetBypassActor", graphql_name="node") """The item at the end of the edge.""" -class RepositoryNode(sgqlc.types.Interface): - """Represents a object that belongs to a repository.""" - - __schema__ = github_schema - __field_names__ = ("repository",) - repository = sgqlc.types.Field(sgqlc.types.non_null("Repository"), graphql_name="repository") - """The repository associated with this node.""" - - -class RepositoryOwner(sgqlc.types.Interface): - """Represents an owner of a Repository.""" +class RepositoryRulesetConnection(sgqlc.types.relay.Connection): + """The connection type for RepositoryRuleset.""" __schema__ = github_schema - __field_names__ = ("avatar_url", "id", "login", "repositories", "repository", "resource_path", "url") - avatar_url = sgqlc.types.Field( - sgqlc.types.non_null(URI), - graphql_name="avatarUrl", - args=sgqlc.types.ArgDict((("size", sgqlc.types.Arg(Int, graphql_name="size", default=None)),)), - ) - """A URL pointing to the owner's public avatar. - - Arguments: - - * `size` (`Int`): The size of the resulting square image. - """ - - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - - login = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="login") - """The username used to login.""" - - repositories = sgqlc.types.Field( - sgqlc.types.non_null(RepositoryConnection), - graphql_name="repositories", - args=sgqlc.types.ArgDict( - ( - ("privacy", sgqlc.types.Arg(RepositoryPrivacy, graphql_name="privacy", default=None)), - ("order_by", sgqlc.types.Arg(RepositoryOrder, graphql_name="orderBy", default=None)), - ("affiliations", sgqlc.types.Arg(sgqlc.types.list_of(RepositoryAffiliation), graphql_name="affiliations", default=None)), - ( - "owner_affiliations", - sgqlc.types.Arg( - sgqlc.types.list_of(RepositoryAffiliation), graphql_name="ownerAffiliations", default=("OWNER", "COLLABORATOR") - ), - ), - ("is_locked", sgqlc.types.Arg(Boolean, graphql_name="isLocked", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("is_fork", sgqlc.types.Arg(Boolean, graphql_name="isFork", default=None)), - ) - ), - ) - """A list of repositories that the user owns. + __field_names__ = ("edges", "nodes", "page_info", "total_count") + edges = sgqlc.types.Field(sgqlc.types.list_of("RepositoryRulesetEdge"), graphql_name="edges") + """A list of edges.""" - Arguments: + nodes = sgqlc.types.Field(sgqlc.types.list_of("RepositoryRuleset"), graphql_name="nodes") + """A list of nodes.""" - * `privacy` (`RepositoryPrivacy`): If non-null, filters - repositories according to privacy - * `order_by` (`RepositoryOrder`): Ordering options for - repositories returned from the connection - * `affiliations` (`[RepositoryAffiliation]`): Array of viewer's - affiliation options for repositories returned from the - connection. For example, OWNER will include only repositories - that the current viewer owns. - * `owner_affiliations` (`[RepositoryAffiliation]`): Array of - owner's affiliation options for repositories returned from the - connection. For example, OWNER will include only repositories - that the organization or user being viewed owns. (default: - `[OWNER, COLLABORATOR]`) - * `is_locked` (`Boolean`): If non-null, filters repositories - according to whether they have been locked - * `after` (`String`): Returns the elements in the list that come - after the specified cursor. - * `before` (`String`): Returns the elements in the list that come - before the specified cursor. - * `first` (`Int`): Returns the first _n_ elements from the list. - * `last` (`Int`): Returns the last _n_ elements from the list. - * `is_fork` (`Boolean`): If non-null, filters repositories - according to whether they are forks of another repository - """ + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + """Information to aid in pagination.""" - repository = sgqlc.types.Field( - "Repository", - graphql_name="repository", - args=sgqlc.types.ArgDict( - ( - ("name", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="name", default=None)), - ("follow_renames", sgqlc.types.Arg(Boolean, graphql_name="followRenames", default=True)), - ) - ), - ) - """Find Repository. + total_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalCount") + """Identifies the total count of items in the connection.""" - Arguments: - * `name` (`String!`): Name of Repository to find. - * `follow_renames` (`Boolean`): Follow repository renames. If - disabled, a repository referenced by its old name will return an - error. (default: `true`) - """ +class RepositoryRulesetEdge(sgqlc.types.Type): + """An edge in a connection.""" - resource_path = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="resourcePath") - """The HTTP URL for the owner.""" + __schema__ = github_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + """A cursor for use in pagination.""" - url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="url") - """The HTTP URL for the owner.""" + node = sgqlc.types.Field("RepositoryRuleset", graphql_name="node") + """The item at the end of the edge.""" class RepositoryTopicConnection(sgqlc.types.relay.Connection): @@ -19834,32 +24923,48 @@ class RequestReviewsPayload(sgqlc.types.Type): """The edge from the pull request to the requested reviewers.""" -class RequirableByPullRequest(sgqlc.types.Interface): - """Represents a type that can be required by a pull request for - merging. - """ +class RequestedReviewerConnection(sgqlc.types.relay.Connection): + """The connection type for RequestedReviewer.""" __schema__ = github_schema - __field_names__ = ("is_required",) - is_required = sgqlc.types.Field( - sgqlc.types.non_null(Boolean), - graphql_name="isRequired", - args=sgqlc.types.ArgDict( - ( - ("pull_request_id", sgqlc.types.Arg(ID, graphql_name="pullRequestId", default=None)), - ("pull_request_number", sgqlc.types.Arg(Int, graphql_name="pullRequestNumber", default=None)), - ) - ), - ) - """Whether this is required to pass before merging for a specific - pull request. + __field_names__ = ("edges", "nodes", "page_info", "total_count") + edges = sgqlc.types.Field(sgqlc.types.list_of("RequestedReviewerEdge"), graphql_name="edges") + """A list of edges.""" - Arguments: + nodes = sgqlc.types.Field(sgqlc.types.list_of("RequestedReviewer"), graphql_name="nodes") + """A list of nodes.""" - * `pull_request_id` (`ID`): The id of the pull request this is - required for - * `pull_request_number` (`Int`): The number of the pull request - this is required for + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + """Information to aid in pagination.""" + + total_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalCount") + """Identifies the total count of items in the connection.""" + + +class RequestedReviewerEdge(sgqlc.types.Type): + """An edge in a connection.""" + + __schema__ = github_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + """A cursor for use in pagination.""" + + node = sgqlc.types.Field("RequestedReviewer", graphql_name="node") + """The item at the end of the edge.""" + + +class RequiredDeploymentsParameters(sgqlc.types.Type): + """Choose which environments must be successfully deployed to before + branches can be merged into a branch that matches this rule. + """ + + __schema__ = github_schema + __field_names__ = ("required_deployment_environments",) + required_deployment_environments = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="requiredDeploymentEnvironments" + ) + """The environments that must be successfully deployed to before + branches can be merged. """ @@ -19879,6 +24984,28 @@ class RequiredStatusCheckDescription(sgqlc.types.Type): """The name of this status.""" +class RequiredStatusChecksParameters(sgqlc.types.Type): + """Choose which status checks must pass before branches can be merged + into a branch that matches this rule. When enabled, commits must + first be pushed to another branch, then merged or pushed directly + to a branch that matches this rule after status checks have + passed. + """ + + __schema__ = github_schema + __field_names__ = ("required_status_checks", "strict_required_status_checks_policy") + required_status_checks = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("StatusCheckConfiguration"))), graphql_name="requiredStatusChecks" + ) + """Status checks that are required.""" + + strict_required_status_checks_policy = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="strictRequiredStatusChecksPolicy") + """Whether pull requests targeting a matching branch must be tested + with the latest code. This setting will not take effect unless at + least one status check is enabled. + """ + + class RerequestCheckSuitePayload(sgqlc.types.Type): """Autogenerated return type of RerequestCheckSuite""" @@ -19903,6 +25030,33 @@ class ResolveReviewThreadPayload(sgqlc.types.Type): """The thread to resolve.""" +class RetireSponsorsTierPayload(sgqlc.types.Type): + """Autogenerated return type of RetireSponsorsTier""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "sponsors_tier") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + sponsors_tier = sgqlc.types.Field("SponsorsTier", graphql_name="sponsorsTier") + """The tier that was retired.""" + + +class RevertPullRequestPayload(sgqlc.types.Type): + """Autogenerated return type of RevertPullRequest""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "pull_request", "revert_pull_request") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + pull_request = sgqlc.types.Field("PullRequest", graphql_name="pullRequest") + """The pull request that was reverted.""" + + revert_pull_request = sgqlc.types.Field("PullRequest", graphql_name="revertPullRequest") + """The new pull request that reverts the input pull request.""" + + class ReviewDismissalAllowanceConnection(sgqlc.types.relay.Connection): """The connection type for ReviewDismissalAllowance.""" @@ -20042,7 +25196,10 @@ class SavedReplyEdge(sgqlc.types.Type): class SearchResultItemConnection(sgqlc.types.relay.Connection): - """A list of results that matched against a search query.""" + """A list of results that matched against a search query. Regardless + of the number of matches, a maximum of 1,000 results will be + available across all types, potentially split across many pages. + """ __schema__ = github_schema __field_names__ = ( @@ -20057,16 +25214,25 @@ class SearchResultItemConnection(sgqlc.types.relay.Connection): "wiki_count", ) code_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="codeCount") - """The number of pieces of code that matched the search query.""" + """The total number of pieces of code that matched the search query. + Regardless of the total number of matches, a maximum of 1,000 + results will be available across all types. + """ discussion_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="discussionCount") - """The number of discussions that matched the search query.""" + """The total number of discussions that matched the search query. + Regardless of the total number of matches, a maximum of 1,000 + results will be available across all types. + """ edges = sgqlc.types.Field(sgqlc.types.list_of("SearchResultItemEdge"), graphql_name="edges") """A list of edges.""" issue_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="issueCount") - """The number of issues that matched the search query.""" + """The total number of issues that matched the search query. + Regardless of the total number of matches, a maximum of 1,000 + results will be available across all types. + """ nodes = sgqlc.types.Field(sgqlc.types.list_of("SearchResultItem"), graphql_name="nodes") """A list of nodes.""" @@ -20075,13 +25241,22 @@ class SearchResultItemConnection(sgqlc.types.relay.Connection): """Information to aid in pagination.""" repository_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="repositoryCount") - """The number of repositories that matched the search query.""" + """The total number of repositories that matched the search query. + Regardless of the total number of matches, a maximum of 1,000 + results will be available across all types. + """ user_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="userCount") - """The number of users that matched the search query.""" + """The total number of users that matched the search query. + Regardless of the total number of matches, a maximum of 1,000 + results will be available across all types. + """ wiki_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="wikiCount") - """The number of wiki pages that matched the search query.""" + """The total number of wiki pages that matched the search query. + Regardless of the total number of matches, a maximum of 1,000 + results will be available across all types. + """ class SearchResultItemEdge(sgqlc.types.Type): @@ -20281,27 +25456,30 @@ class SetUserInteractionLimitPayload(sgqlc.types.Type): """The user that the interaction limit was set for.""" -class SortBy(sgqlc.types.Type): - """Represents a sort by field and direction.""" +class SocialAccount(sgqlc.types.Type): + """Social media profile associated with a user.""" __schema__ = github_schema - __field_names__ = ("direction", "field") - direction = sgqlc.types.Field(sgqlc.types.non_null(OrderDirection), graphql_name="direction") - """The direction of the sorting. Possible values are ASC and DESC.""" + __field_names__ = ("display_name", "provider", "url") + display_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="displayName") + """Name of the social media account as it appears on the profile.""" + + provider = sgqlc.types.Field(sgqlc.types.non_null(SocialAccountProvider), graphql_name="provider") + """Software or company that hosts the social media account.""" - field = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="field") - """The id of the field by which the column is sorted.""" + url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="url") + """URL of the social media account.""" -class SponsorConnection(sgqlc.types.relay.Connection): - """The connection type for Sponsor.""" +class SocialAccountConnection(sgqlc.types.relay.Connection): + """The connection type for SocialAccount.""" __schema__ = github_schema __field_names__ = ("edges", "nodes", "page_info", "total_count") - edges = sgqlc.types.Field(sgqlc.types.list_of("SponsorEdge"), graphql_name="edges") + edges = sgqlc.types.Field(sgqlc.types.list_of("SocialAccountEdge"), graphql_name="edges") """A list of edges.""" - nodes = sgqlc.types.Field(sgqlc.types.list_of("Sponsor"), graphql_name="nodes") + nodes = sgqlc.types.Field(sgqlc.types.list_of(SocialAccount), graphql_name="nodes") """A list of nodes.""" page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") @@ -20311,286 +25489,48 @@ class SponsorConnection(sgqlc.types.relay.Connection): """Identifies the total count of items in the connection.""" -class SponsorEdge(sgqlc.types.Type): - """Represents a user or organization who is sponsoring someone in - GitHub Sponsors. - """ +class SocialAccountEdge(sgqlc.types.Type): + """An edge in a connection.""" __schema__ = github_schema __field_names__ = ("cursor", "node") cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") """A cursor for use in pagination.""" - node = sgqlc.types.Field("Sponsor", graphql_name="node") + node = sgqlc.types.Field(SocialAccount, graphql_name="node") """The item at the end of the edge.""" -class Sponsorable(sgqlc.types.Interface): - """Entities that can be sponsored through GitHub Sponsors""" +class SponsorConnection(sgqlc.types.relay.Connection): + """The connection type for Sponsor.""" __schema__ = github_schema - __field_names__ = ( - "estimated_next_sponsors_payout_in_cents", - "has_sponsors_listing", - "is_sponsored_by", - "is_sponsoring_viewer", - "monthly_estimated_sponsors_income_in_cents", - "sponsoring", - "sponsors", - "sponsors_activities", - "sponsors_listing", - "sponsorship_for_viewer_as_sponsor", - "sponsorship_for_viewer_as_sponsorable", - "sponsorship_newsletters", - "sponsorships_as_maintainer", - "sponsorships_as_sponsor", - "viewer_can_sponsor", - "viewer_is_sponsoring", - ) - estimated_next_sponsors_payout_in_cents = sgqlc.types.Field( - sgqlc.types.non_null(Int), graphql_name="estimatedNextSponsorsPayoutInCents" - ) - """The estimated next GitHub Sponsors payout for this - user/organization in cents (USD). - """ - - has_sponsors_listing = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasSponsorsListing") - """True if this user/organization has a GitHub Sponsors listing.""" - - is_sponsored_by = sgqlc.types.Field( - sgqlc.types.non_null(Boolean), - graphql_name="isSponsoredBy", - args=sgqlc.types.ArgDict( - (("account_login", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="accountLogin", default=None)),) - ), - ) - """Check if the given account is sponsoring this user/organization. - - Arguments: - - * `account_login` (`String!`): The target account's login. - """ - - is_sponsoring_viewer = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isSponsoringViewer") - """True if the viewer is sponsored by this user/organization.""" - - monthly_estimated_sponsors_income_in_cents = sgqlc.types.Field( - sgqlc.types.non_null(Int), graphql_name="monthlyEstimatedSponsorsIncomeInCents" - ) - """The estimated monthly GitHub Sponsors income for this - user/organization in cents (USD). - """ - - sponsoring = sgqlc.types.Field( - sgqlc.types.non_null(SponsorConnection), - graphql_name="sponsoring", - args=sgqlc.types.ArgDict( - ( - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("order_by", sgqlc.types.Arg(SponsorOrder, graphql_name="orderBy", default={"field": "RELEVANCE", "direction": "DESC"})), - ) - ), - ) - """List of users and organizations this entity is sponsoring. - - Arguments: - - * `after` (`String`): Returns the elements in the list that come - after the specified cursor. - * `before` (`String`): Returns the elements in the list that come - before the specified cursor. - * `first` (`Int`): Returns the first _n_ elements from the list. - * `last` (`Int`): Returns the last _n_ elements from the list. - * `order_by` (`SponsorOrder`): Ordering options for the users and - organizations returned from the connection. (default: `{field: - RELEVANCE, direction: DESC}`) - """ - - sponsors = sgqlc.types.Field( - sgqlc.types.non_null(SponsorConnection), - graphql_name="sponsors", - args=sgqlc.types.ArgDict( - ( - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("tier_id", sgqlc.types.Arg(ID, graphql_name="tierId", default=None)), - ("order_by", sgqlc.types.Arg(SponsorOrder, graphql_name="orderBy", default={"field": "RELEVANCE", "direction": "DESC"})), - ) - ), - ) - """List of sponsors for this user or organization. - - Arguments: - - * `after` (`String`): Returns the elements in the list that come - after the specified cursor. - * `before` (`String`): Returns the elements in the list that come - before the specified cursor. - * `first` (`Int`): Returns the first _n_ elements from the list. - * `last` (`Int`): Returns the last _n_ elements from the list. - * `tier_id` (`ID`): If given, will filter for sponsors at the - given tier. Will only return sponsors whose tier the viewer is - permitted to see. - * `order_by` (`SponsorOrder`): Ordering options for sponsors - returned from the connection. (default: `{field: RELEVANCE, - direction: DESC}`) - """ - - sponsors_activities = sgqlc.types.Field( - sgqlc.types.non_null("SponsorsActivityConnection"), - graphql_name="sponsorsActivities", - args=sgqlc.types.ArgDict( - ( - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("period", sgqlc.types.Arg(SponsorsActivityPeriod, graphql_name="period", default="MONTH")), - ( - "order_by", - sgqlc.types.Arg(SponsorsActivityOrder, graphql_name="orderBy", default={"field": "TIMESTAMP", "direction": "DESC"}), - ), - ) - ), - ) - """Events involving this sponsorable, such as new sponsorships. - - Arguments: - - * `after` (`String`): Returns the elements in the list that come - after the specified cursor. - * `before` (`String`): Returns the elements in the list that come - before the specified cursor. - * `first` (`Int`): Returns the first _n_ elements from the list. - * `last` (`Int`): Returns the last _n_ elements from the list. - * `period` (`SponsorsActivityPeriod`): Filter activities returned - to only those that occurred in a given time range. (default: - `MONTH`) - * `order_by` (`SponsorsActivityOrder`): Ordering options for - activity returned from the connection. (default: `{field: - TIMESTAMP, direction: DESC}`) - """ - - sponsors_listing = sgqlc.types.Field("SponsorsListing", graphql_name="sponsorsListing") - """The GitHub Sponsors listing for this user or organization.""" - - sponsorship_for_viewer_as_sponsor = sgqlc.types.Field("Sponsorship", graphql_name="sponsorshipForViewerAsSponsor") - """The sponsorship from the viewer to this user/organization; that - is, the sponsorship where you're the sponsor. Only returns a - sponsorship if it is active. - """ - - sponsorship_for_viewer_as_sponsorable = sgqlc.types.Field("Sponsorship", graphql_name="sponsorshipForViewerAsSponsorable") - """The sponsorship from this user/organization to the viewer; that - is, the sponsorship you're receiving. Only returns a sponsorship - if it is active. - """ - - sponsorship_newsletters = sgqlc.types.Field( - sgqlc.types.non_null("SponsorshipNewsletterConnection"), - graphql_name="sponsorshipNewsletters", - args=sgqlc.types.ArgDict( - ( - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ( - "order_by", - sgqlc.types.Arg( - SponsorshipNewsletterOrder, graphql_name="orderBy", default={"field": "CREATED_AT", "direction": "DESC"} - ), - ), - ) - ), - ) - """List of sponsorship updates sent from this sponsorable to - sponsors. - - Arguments: - - * `after` (`String`): Returns the elements in the list that come - after the specified cursor. - * `before` (`String`): Returns the elements in the list that come - before the specified cursor. - * `first` (`Int`): Returns the first _n_ elements from the list. - * `last` (`Int`): Returns the last _n_ elements from the list. - * `order_by` (`SponsorshipNewsletterOrder`): Ordering options for - sponsorship updates returned from the connection. (default: - `{field: CREATED_AT, direction: DESC}`) - """ - - sponsorships_as_maintainer = sgqlc.types.Field( - sgqlc.types.non_null("SponsorshipConnection"), - graphql_name="sponsorshipsAsMaintainer", - args=sgqlc.types.ArgDict( - ( - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("include_private", sgqlc.types.Arg(Boolean, graphql_name="includePrivate", default=False)), - ("order_by", sgqlc.types.Arg(SponsorshipOrder, graphql_name="orderBy", default=None)), - ) - ), - ) - """This object's sponsorships as the maintainer. + __field_names__ = ("edges", "nodes", "page_info", "total_count") + edges = sgqlc.types.Field(sgqlc.types.list_of("SponsorEdge"), graphql_name="edges") + """A list of edges.""" - Arguments: + nodes = sgqlc.types.Field(sgqlc.types.list_of("Sponsor"), graphql_name="nodes") + """A list of nodes.""" - * `after` (`String`): Returns the elements in the list that come - after the specified cursor. - * `before` (`String`): Returns the elements in the list that come - before the specified cursor. - * `first` (`Int`): Returns the first _n_ elements from the list. - * `last` (`Int`): Returns the last _n_ elements from the list. - * `include_private` (`Boolean`): Whether or not to include private - sponsorships in the result set (default: `false`) - * `order_by` (`SponsorshipOrder`): Ordering options for - sponsorships returned from this connection. If left blank, the - sponsorships will be ordered based on relevancy to the viewer. - """ + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + """Information to aid in pagination.""" - sponsorships_as_sponsor = sgqlc.types.Field( - sgqlc.types.non_null("SponsorshipConnection"), - graphql_name="sponsorshipsAsSponsor", - args=sgqlc.types.ArgDict( - ( - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("order_by", sgqlc.types.Arg(SponsorshipOrder, graphql_name="orderBy", default=None)), - ) - ), - ) - """This object's sponsorships as the sponsor. + total_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalCount") + """Identifies the total count of items in the connection.""" - Arguments: - * `after` (`String`): Returns the elements in the list that come - after the specified cursor. - * `before` (`String`): Returns the elements in the list that come - before the specified cursor. - * `first` (`Int`): Returns the first _n_ elements from the list. - * `last` (`Int`): Returns the last _n_ elements from the list. - * `order_by` (`SponsorshipOrder`): Ordering options for - sponsorships returned from this connection. If left blank, the - sponsorships will be ordered based on relevancy to the viewer. +class SponsorEdge(sgqlc.types.Type): + """Represents a user or organization who is sponsoring someone in + GitHub Sponsors. """ - viewer_can_sponsor = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanSponsor") - """Whether or not the viewer is able to sponsor this - user/organization. - """ + __schema__ = github_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + """A cursor for use in pagination.""" - viewer_is_sponsoring = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerIsSponsoring") - """True if the viewer is sponsoring this user/organization.""" + node = sgqlc.types.Field("Sponsor", graphql_name="node") + """The item at the end of the edge.""" class SponsorableItemConnection(sgqlc.types.relay.Connection): @@ -20687,7 +25627,31 @@ class SponsorsTierAdminInfo(sgqlc.types.Type): """ __schema__ = github_schema - __field_names__ = ("sponsorships",) + __field_names__ = ("is_draft", "is_published", "is_retired", "sponsorships") + is_draft = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isDraft") + """Indicates whether this tier is still a work in progress by the + sponsorable and not yet published to the associated GitHub + Sponsors profile. Draft tiers cannot be used for new sponsorships + and will not be in use on existing sponsorships. Draft tiers + cannot be seen by anyone but the admins of the GitHub Sponsors + profile. + """ + + is_published = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isPublished") + """Indicates whether this tier is published to the associated GitHub + Sponsors profile. Published tiers are visible to anyone who can + see the GitHub Sponsors profile, and are available for use in + sponsorships if the GitHub Sponsors profile is publicly visible. + """ + + is_retired = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isRetired") + """Indicates whether this tier has been retired from the associated + GitHub Sponsors profile. Retired tiers are no longer shown on the + GitHub Sponsors profile and cannot be chosen for new sponsorships. + Existing sponsorships may still use retired tiers if the sponsor + selected the tier before it was retired. + """ + sponsorships = sgqlc.types.Field( sgqlc.types.non_null("SponsorshipConnection"), graphql_name="sponsorships", @@ -20702,7 +25666,7 @@ class SponsorsTierAdminInfo(sgqlc.types.Type): ) ), ) - """The sponsorships associated with this tier. + """The sponsorships using this tier. Arguments: @@ -20712,8 +25676,9 @@ class SponsorsTierAdminInfo(sgqlc.types.Type): before the specified cursor. * `first` (`Int`): Returns the first _n_ elements from the list. * `last` (`Int`): Returns the last _n_ elements from the list. - * `include_private` (`Boolean`): Whether or not to include private - sponsorships in the result set (default: `false`) + * `include_private` (`Boolean`): Whether or not to return private + sponsorships using this tier. Defaults to only returning public + sponsorships on this tier. (default: `false`) * `order_by` (`SponsorshipOrder`): Ordering options for sponsorships returned from this connection. If left blank, the sponsorships will be ordered based on relevancy to the viewer. @@ -20863,48 +25828,6 @@ class StargazerEdge(sgqlc.types.Type): """Identifies when the item was starred.""" -class Starrable(sgqlc.types.Interface): - """Things that can be starred.""" - - __schema__ = github_schema - __field_names__ = ("id", "stargazer_count", "stargazers", "viewer_has_starred") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - - stargazer_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="stargazerCount") - """Returns a count of how many stargazers there are on this object""" - - stargazers = sgqlc.types.Field( - sgqlc.types.non_null(StargazerConnection), - graphql_name="stargazers", - args=sgqlc.types.ArgDict( - ( - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("order_by", sgqlc.types.Arg(StarOrder, graphql_name="orderBy", default=None)), - ) - ), - ) - """A list of users who have starred this starrable. - - Arguments: - - * `after` (`String`): Returns the elements in the list that come - after the specified cursor. - * `before` (`String`): Returns the elements in the list that come - before the specified cursor. - * `first` (`Int`): Returns the first _n_ elements from the list. - * `last` (`Int`): Returns the last _n_ elements from the list. - * `order_by` (`StarOrder`): Order for connection - """ - - viewer_has_starred = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerHasStarred") - """Returns a boolean indicating whether the viewing user has starred - this starrable. - """ - - class StarredRepositoryConnection(sgqlc.types.relay.Connection): """The connection type for Repository.""" @@ -20942,6 +25865,18 @@ class StarredRepositoryEdge(sgqlc.types.Type): """Identifies when the item was starred.""" +class StartOrganizationMigrationPayload(sgqlc.types.Type): + """Autogenerated return type of StartOrganizationMigration""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "org_migration") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + org_migration = sgqlc.types.Field("OrganizationMigration", graphql_name="orgMigration") + """The new organization migration.""" + + class StartRepositoryMigrationPayload(sgqlc.types.Type): """Autogenerated return type of StartRepositoryMigration""" @@ -20951,14 +25886,45 @@ class StartRepositoryMigrationPayload(sgqlc.types.Type): """A unique identifier for the client performing the mutation.""" repository_migration = sgqlc.types.Field("RepositoryMigration", graphql_name="repositoryMigration") - """The new Octoshift repository migration.""" + """The new repository migration.""" + + +class StatusCheckConfiguration(sgqlc.types.Type): + """Required status check""" + + __schema__ = github_schema + __field_names__ = ("context", "integration_id") + context = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="context") + """The status check context name that must be present on the commit.""" + + integration_id = sgqlc.types.Field(Int, graphql_name="integrationId") + """The optional integration ID that this status check must originate + from. + """ class StatusCheckRollupContextConnection(sgqlc.types.relay.Connection): """The connection type for StatusCheckRollupContext.""" __schema__ = github_schema - __field_names__ = ("edges", "nodes", "page_info", "total_count") + __field_names__ = ( + "check_run_count", + "check_run_counts_by_state", + "edges", + "nodes", + "page_info", + "status_context_count", + "status_context_counts_by_state", + "total_count", + ) + check_run_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="checkRunCount") + """The number of check runs in this rollup.""" + + check_run_counts_by_state = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(CheckRunStateCount)), graphql_name="checkRunCountsByState" + ) + """Counts of check runs by state.""" + edges = sgqlc.types.Field(sgqlc.types.list_of("StatusCheckRollupContextEdge"), graphql_name="edges") """A list of edges.""" @@ -20968,6 +25934,14 @@ class StatusCheckRollupContextConnection(sgqlc.types.relay.Connection): page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") """Information to aid in pagination.""" + status_context_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="statusContextCount") + """The number of status contexts in this rollup.""" + + status_context_counts_by_state = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("StatusContextStateCount")), graphql_name="statusContextCountsByState" + ) + """Counts of status contexts by state.""" + total_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalCount") """Identifies the total count of items in the connection.""" @@ -20984,6 +25958,64 @@ class StatusCheckRollupContextEdge(sgqlc.types.Type): """The item at the end of the edge.""" +class StatusContextStateCount(sgqlc.types.Type): + """Represents a count of the state of a status context.""" + + __schema__ = github_schema + __field_names__ = ("count", "state") + count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="count") + """The number of statuses with this state.""" + + state = sgqlc.types.Field(sgqlc.types.non_null(StatusState), graphql_name="state") + """The state of a status context.""" + + +class StripeConnectAccount(sgqlc.types.Type): + """A Stripe Connect account for receiving sponsorship funds from + GitHub Sponsors. + """ + + __schema__ = github_schema + __field_names__ = ( + "account_id", + "billing_country_or_region", + "country_or_region", + "is_active", + "sponsors_listing", + "stripe_dashboard_url", + ) + account_id = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="accountId") + """The account number used to identify this Stripe Connect account.""" + + billing_country_or_region = sgqlc.types.Field(String, graphql_name="billingCountryOrRegion") + """The name of the country or region of an external account, such as + a bank account, tied to the Stripe Connect account. Will only + return a value when queried by the maintainer of the associated + GitHub Sponsors profile themselves, or by an admin of the + sponsorable organization. + """ + + country_or_region = sgqlc.types.Field(String, graphql_name="countryOrRegion") + """The name of the country or region of the Stripe Connect account. + Will only return a value when queried by the maintainer of the + associated GitHub Sponsors profile themselves, or by an admin of + the sponsorable organization. + """ + + is_active = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isActive") + """Whether this Stripe Connect account is currently in use for the + associated GitHub Sponsors profile. + """ + + sponsors_listing = sgqlc.types.Field(sgqlc.types.non_null("SponsorsListing"), graphql_name="sponsorsListing") + """The GitHub Sponsors profile associated with this Stripe Connect + account. + """ + + stripe_dashboard_url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="stripeDashboardUrl") + """The URL to access this Stripe Connect account on Stripe's website.""" + + class SubmitPullRequestReviewPayload(sgqlc.types.Type): """Autogenerated return type of SubmitPullRequestReview""" @@ -21002,7 +26034,7 @@ class Submodule(sgqlc.types.Type): """ __schema__ = github_schema - __field_names__ = ("branch", "git_url", "name", "path", "subproject_commit_oid") + __field_names__ = ("branch", "git_url", "name", "name_raw", "path", "path_raw", "subproject_commit_oid") branch = sgqlc.types.Field(String, graphql_name="branch") """The branch of the upstream submodule for tracking updates""" @@ -21012,9 +26044,17 @@ class Submodule(sgqlc.types.Type): name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") """The name of the submodule in .gitmodules""" + name_raw = sgqlc.types.Field(sgqlc.types.non_null(Base64String), graphql_name="nameRaw") + """The name of the submodule in .gitmodules (Base64-encoded)""" + path = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="path") """The path in the superproject that this submodule is located in""" + path_raw = sgqlc.types.Field(sgqlc.types.non_null(Base64String), graphql_name="pathRaw") + """The path in the superproject that this submodule is located in + (Base64-encoded) + """ + subproject_commit_oid = sgqlc.types.Field(GitObjectID, graphql_name="subprojectCommitOid") """The commit revision of the subproject repository being tracked by the submodule @@ -21051,26 +26091,6 @@ class SubmoduleEdge(sgqlc.types.Type): """The item at the end of the edge.""" -class Subscribable(sgqlc.types.Interface): - """Entities that can be subscribed to for web and email - notifications. - """ - - __schema__ = github_schema - __field_names__ = ("id", "viewer_can_subscribe", "viewer_subscription") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - - viewer_can_subscribe = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanSubscribe") - """Check if the viewer is able to change their subscription status - for the repository. - """ - - viewer_subscription = sgqlc.types.Field(SubscriptionState, graphql_name="viewerSubscription") - """Identifies if the viewer is watching, not watching, or ignoring - the subscribable entity. - """ - - class SuggestedReviewer(sgqlc.types.Type): """A suggestion to review a pull request based on a user's commit history and review comments. @@ -21088,22 +26108,22 @@ class SuggestedReviewer(sgqlc.types.Type): """Identifies the user suggested to review the pull request.""" -class TeamAuditEntryData(sgqlc.types.Interface): - """Metadata for an audit entry with action team.*""" +class TagNamePatternParameters(sgqlc.types.Type): + """Parameters to be used for the tag_name_pattern rule""" __schema__ = github_schema - __field_names__ = ("team", "team_name", "team_resource_path", "team_url") - team = sgqlc.types.Field("Team", graphql_name="team") - """The team associated with the action""" + __field_names__ = ("name", "negate", "operator", "pattern") + name = sgqlc.types.Field(String, graphql_name="name") + """How this rule will appear to users.""" - team_name = sgqlc.types.Field(String, graphql_name="teamName") - """The name of the team""" + negate = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="negate") + """If true, the rule will fail if the pattern matches.""" - team_resource_path = sgqlc.types.Field(URI, graphql_name="teamResourcePath") - """The HTTP path for this team""" + operator = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="operator") + """The operator to use for matching.""" - team_url = sgqlc.types.Field(URI, graphql_name="teamUrl") - """The HTTP URL for this team""" + pattern = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="pattern") + """The pattern to match with.""" class TeamConnection(sgqlc.types.relay.Connection): @@ -21298,16 +26318,16 @@ class TextMatchHighlight(sgqlc.types.Type): """The text matched.""" -class TopicAuditEntryData(sgqlc.types.Interface): - """Metadata for an audit entry with a topic.""" +class TransferEnterpriseOrganizationPayload(sgqlc.types.Type): + """Autogenerated return type of TransferEnterpriseOrganization""" __schema__ = github_schema - __field_names__ = ("topic", "topic_name") - topic = sgqlc.types.Field("Topic", graphql_name="topic") - """The name of the topic added to the repository""" + __field_names__ = ("client_mutation_id", "organization") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" - topic_name = sgqlc.types.Field(String, graphql_name="topicName") - """The name of the topic added to the repository""" + organization = sgqlc.types.Field("Organization", graphql_name="organization") + """The organization for which a transfer was initiated.""" class TransferIssuePayload(sgqlc.types.Type): @@ -21329,13 +26349,17 @@ class TreeEntry(sgqlc.types.Type): __field_names__ = ( "extension", "is_generated", + "language", "line_count", "mode", "name", + "name_raw", "object", "oid", "path", + "path_raw", "repository", + "size", "submodule", "type", ) @@ -21345,6 +26369,9 @@ class TreeEntry(sgqlc.types.Type): is_generated = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isGenerated") """Whether or not this tree entry is generated""" + language = sgqlc.types.Field("Language", graphql_name="language") + """The programming language this file is written in.""" + line_count = sgqlc.types.Field(Int, graphql_name="lineCount") """Number of lines in the file.""" @@ -21354,6 +26381,9 @@ class TreeEntry(sgqlc.types.Type): name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") """Entry file name.""" + name_raw = sgqlc.types.Field(sgqlc.types.non_null(Base64String), graphql_name="nameRaw") + """Entry file name. (Base64-encoded)""" + object = sgqlc.types.Field(GitObject, graphql_name="object") """Entry file object.""" @@ -21363,9 +26393,15 @@ class TreeEntry(sgqlc.types.Type): path = sgqlc.types.Field(String, graphql_name="path") """The full path of the file.""" + path_raw = sgqlc.types.Field(Base64String, graphql_name="pathRaw") + """The full path of the file. (Base64-encoded)""" + repository = sgqlc.types.Field(sgqlc.types.non_null("Repository"), graphql_name="repository") """The Repository the tree entry belongs to""" + size = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="size") + """Entry byte size""" + submodule = sgqlc.types.Field(Submodule, graphql_name="submodule") """If the TreeEntry is for a directory occupied by a submodule project, this returns the corresponding submodule @@ -21375,6 +26411,18 @@ class TreeEntry(sgqlc.types.Type): """Entry file type.""" +class UnarchiveProjectV2ItemPayload(sgqlc.types.Type): + """Autogenerated return type of UnarchiveProjectV2Item""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "item") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + item = sgqlc.types.Field("ProjectV2Item", graphql_name="item") + """The item unarchived from the project.""" + + class UnarchiveRepositoryPayload(sgqlc.types.Type): """Autogenerated return type of UnarchiveRepository""" @@ -21411,16 +26459,28 @@ class UnfollowUserPayload(sgqlc.types.Type): """The user that was unfollowed.""" -class UniformResourceLocatable(sgqlc.types.Interface): - """Represents a type that can be retrieved by a URL.""" +class UnlinkProjectV2FromRepositoryPayload(sgqlc.types.Type): + """Autogenerated return type of UnlinkProjectV2FromRepository""" __schema__ = github_schema - __field_names__ = ("resource_path", "url") - resource_path = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="resourcePath") - """The HTML path to this resource.""" + __field_names__ = ("client_mutation_id", "repository") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" - url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="url") - """The URL to this resource.""" + repository = sgqlc.types.Field("Repository", graphql_name="repository") + """The repository the project is no longer linked to.""" + + +class UnlinkProjectV2FromTeamPayload(sgqlc.types.Type): + """Autogenerated return type of UnlinkProjectV2FromTeam""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "team") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + team = sgqlc.types.Field("Team", graphql_name="team") + """The team the project is unlinked from""" class UnlinkRepositoryFromProjectPayload(sgqlc.types.Type): @@ -21489,6 +26549,18 @@ class UnmarkIssueAsDuplicatePayload(sgqlc.types.Type): """The issue or pull request that was marked as a duplicate.""" +class UnmarkProjectV2AsTemplatePayload(sgqlc.types.Type): + """Autogenerated return type of UnmarkProjectV2AsTemplate""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "project_v2") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + project_v2 = sgqlc.types.Field("ProjectV2", graphql_name="projectV2") + """The project.""" + + class UnminimizeCommentPayload(sgqlc.types.Type): """Autogenerated return type of UnminimizeComment""" @@ -21525,26 +26597,6 @@ class UnresolveReviewThreadPayload(sgqlc.types.Type): """The thread to resolve.""" -class Updatable(sgqlc.types.Interface): - """Entities that can be updated.""" - - __schema__ = github_schema - __field_names__ = ("viewer_can_update",) - viewer_can_update = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanUpdate") - """Check if the current viewer can update this object.""" - - -class UpdatableComment(sgqlc.types.Interface): - """Comments that can be updated.""" - - __schema__ = github_schema - __field_names__ = ("viewer_cannot_update_reasons",) - viewer_cannot_update_reasons = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CommentCannotUpdateReason))), graphql_name="viewerCannotUpdateReasons" - ) - """Reasons why the current viewer can not update this comment.""" - - class UpdateBranchProtectionRulePayload(sgqlc.types.Type): """Autogenerated return type of UpdateBranchProtectionRule""" @@ -22041,6 +27093,34 @@ class UpdateOrganizationAllowPrivateRepositoryForkingSettingPayload(sgqlc.types. """ +class UpdateOrganizationWebCommitSignoffSettingPayload(sgqlc.types.Type): + """Autogenerated return type of + UpdateOrganizationWebCommitSignoffSetting + """ + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "message", "organization") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + message = sgqlc.types.Field(String, graphql_name="message") + """A message confirming the result of updating the web commit signoff + setting. + """ + + organization = sgqlc.types.Field("Organization", graphql_name="organization") + """The organization with the updated web commit signoff setting.""" + + +class UpdateParameters(sgqlc.types.Type): + """Only allow users with bypass permission to update matching refs.""" + + __schema__ = github_schema + __field_names__ = ("update_allows_fetch_and_merge",) + update_allows_fetch_and_merge = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="updateAllowsFetchAndMerge") + """Branch can pull changes from its upstream repository""" + + class UpdateProjectCardPayload(sgqlc.types.Type): """Autogenerated return type of UpdateProjectCard""" @@ -22065,8 +27145,53 @@ class UpdateProjectColumnPayload(sgqlc.types.Type): """The updated project column.""" -class UpdateProjectDraftIssuePayload(sgqlc.types.Type): - """Autogenerated return type of UpdateProjectDraftIssue""" +class UpdateProjectPayload(sgqlc.types.Type): + """Autogenerated return type of UpdateProject""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "project") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + project = sgqlc.types.Field("Project", graphql_name="project") + """The updated project.""" + + +class UpdateProjectV2CollaboratorsPayload(sgqlc.types.Type): + """Autogenerated return type of UpdateProjectV2Collaborators""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "collaborators") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + collaborators = sgqlc.types.Field( + ProjectV2ActorConnection, + graphql_name="collaborators", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), + ) + """The collaborators granted a role + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + """ + + +class UpdateProjectV2DraftIssuePayload(sgqlc.types.Type): + """Autogenerated return type of UpdateProjectV2DraftIssue""" __schema__ = github_schema __field_names__ = ("client_mutation_id", "draft_issue") @@ -22077,40 +27202,61 @@ class UpdateProjectDraftIssuePayload(sgqlc.types.Type): """The draft issue updated in the project.""" -class UpdateProjectNextItemFieldPayload(sgqlc.types.Type): - """Autogenerated return type of UpdateProjectNextItemField""" +class UpdateProjectV2ItemFieldValuePayload(sgqlc.types.Type): + """Autogenerated return type of UpdateProjectV2ItemFieldValue""" __schema__ = github_schema - __field_names__ = ("client_mutation_id", "project_next_item") + __field_names__ = ("client_mutation_id", "project_v2_item") client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" - project_next_item = sgqlc.types.Field("ProjectNextItem", graphql_name="projectNextItem") + project_v2_item = sgqlc.types.Field("ProjectV2Item", graphql_name="projectV2Item") """The updated item.""" -class UpdateProjectNextPayload(sgqlc.types.Type): - """Autogenerated return type of UpdateProjectNext""" +class UpdateProjectV2ItemPositionPayload(sgqlc.types.Type): + """Autogenerated return type of UpdateProjectV2ItemPosition""" __schema__ = github_schema - __field_names__ = ("client_mutation_id", "project_next") + __field_names__ = ("client_mutation_id", "items") client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" - project_next = sgqlc.types.Field("ProjectNext", graphql_name="projectNext") - """The updated Project.""" + items = sgqlc.types.Field( + ProjectV2ItemConnection, + graphql_name="items", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), + ) + """The items in the new order + Arguments: -class UpdateProjectPayload(sgqlc.types.Type): - """Autogenerated return type of UpdateProject""" + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + """ + + +class UpdateProjectV2Payload(sgqlc.types.Type): + """Autogenerated return type of UpdateProjectV2""" __schema__ = github_schema - __field_names__ = ("client_mutation_id", "project") + __field_names__ = ("client_mutation_id", "project_v2") client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") """A unique identifier for the client performing the mutation.""" - project = sgqlc.types.Field("Project", graphql_name="project") - """The updated project.""" + project_v2 = sgqlc.types.Field("ProjectV2", graphql_name="projectV2") + """The updated Project.""" class UpdatePullRequestBranchPayload(sgqlc.types.Type): @@ -22188,6 +27334,37 @@ class UpdateRepositoryPayload(sgqlc.types.Type): """The updated repository.""" +class UpdateRepositoryRulesetPayload(sgqlc.types.Type): + """Autogenerated return type of UpdateRepositoryRuleset""" + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "ruleset") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + ruleset = sgqlc.types.Field("RepositoryRuleset", graphql_name="ruleset") + """The newly created Ruleset.""" + + +class UpdateRepositoryWebCommitSignoffSettingPayload(sgqlc.types.Type): + """Autogenerated return type of + UpdateRepositoryWebCommitSignoffSetting + """ + + __schema__ = github_schema + __field_names__ = ("client_mutation_id", "message", "repository") + client_mutation_id = sgqlc.types.Field(String, graphql_name="clientMutationId") + """A unique identifier for the client performing the mutation.""" + + message = sgqlc.types.Field(String, graphql_name="message") + """A message confirming the result of updating the web commit signoff + setting. + """ + + repository = sgqlc.types.Field("Repository", graphql_name="repository") + """The updated repository.""" + + class UpdateSponsorshipPreferencesPayload(sgqlc.types.Type): """Autogenerated return type of UpdateSponsorshipPreferences""" @@ -22413,21 +27590,57 @@ class VerifyVerifiableDomainPayload(sgqlc.types.Type): """The verifiable domain that was verified.""" -class Votable(sgqlc.types.Interface): - """A subject that may be upvoted.""" +class WorkflowRunConnection(sgqlc.types.relay.Connection): + """The connection type for WorkflowRun.""" __schema__ = github_schema - __field_names__ = ("upvote_count", "viewer_can_upvote", "viewer_has_upvoted") - upvote_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="upvoteCount") - """Number of upvotes that this subject has received.""" + __field_names__ = ("edges", "nodes", "page_info", "total_count") + edges = sgqlc.types.Field(sgqlc.types.list_of("WorkflowRunEdge"), graphql_name="edges") + """A list of edges.""" - viewer_can_upvote = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanUpvote") - """Whether or not the current user can add or remove an upvote on - this subject. + nodes = sgqlc.types.Field(sgqlc.types.list_of("WorkflowRun"), graphql_name="nodes") + """A list of nodes.""" + + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + """Information to aid in pagination.""" + + total_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalCount") + """Identifies the total count of items in the connection.""" + + +class WorkflowRunEdge(sgqlc.types.Type): + """An edge in a connection.""" + + __schema__ = github_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + """A cursor for use in pagination.""" + + node = sgqlc.types.Field("WorkflowRun", graphql_name="node") + """The item at the end of the edge.""" + + +class AddedToMergeQueueEvent(sgqlc.types.Type, Node): + """Represents an 'added_to_merge_queue' event on a given pull + request. """ - viewer_has_upvoted = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerHasUpvoted") - """Whether or not the current user has already upvoted this subject.""" + __schema__ = github_schema + __field_names__ = ("actor", "created_at", "enqueuer", "merge_queue", "pull_request") + actor = sgqlc.types.Field(Actor, graphql_name="actor") + """Identifies the actor who performed the event.""" + + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + """Identifies the date and time when the object was created.""" + + enqueuer = sgqlc.types.Field("User", graphql_name="enqueuer") + """The user who added this Pull Request to the merge queue""" + + merge_queue = sgqlc.types.Field("MergeQueue", graphql_name="mergeQueue") + """The merge queue where this pull request was added to.""" + + pull_request = sgqlc.types.Field("PullRequest", graphql_name="pullRequest") + """PullRequest referenced by event.""" class AddedToProjectEvent(sgqlc.types.Type, Node): @@ -22811,17 +28024,22 @@ class BranchProtectionRule(sgqlc.types.Type, Node): "database_id", "dismisses_stale_reviews", "is_admin_enforced", + "lock_allows_fetch_and_merge", + "lock_branch", "matching_refs", "pattern", "push_allowances", "repository", + "require_last_push_approval", "required_approving_review_count", + "required_deployment_environments", "required_status_check_contexts", "required_status_checks", "requires_approving_reviews", "requires_code_owner_reviews", "requires_commit_signatures", "requires_conversation_resolution", + "requires_deployments", "requires_linear_history", "requires_status_checks", "requires_strict_status_checks", @@ -22927,6 +28145,17 @@ class BranchProtectionRule(sgqlc.types.Type, Node): is_admin_enforced = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isAdminEnforced") """Can admins overwrite branch protection.""" + lock_allows_fetch_and_merge = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="lockAllowsFetchAndMerge") + """Whether users can pull changes from upstream when the branch is + locked. Set to `true` to allow fork syncing. Set to `false` to + prevent fork syncing. + """ + + lock_branch = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="lockBranch") + """Whether to set the branch as read-only. If this is true, users + will not be able to push to the branch. + """ + matching_refs = sgqlc.types.Field( sgqlc.types.non_null(RefConnection), graphql_name="matchingRefs", @@ -22983,9 +28212,19 @@ class BranchProtectionRule(sgqlc.types.Type, Node): repository = sgqlc.types.Field("Repository", graphql_name="repository") """The repository associated with this branch protection rule.""" + require_last_push_approval = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requireLastPushApproval") + """Whether the most recent push must be approved by someone other + than the person who pushed it + """ + required_approving_review_count = sgqlc.types.Field(Int, graphql_name="requiredApprovingReviewCount") """Number of approving reviews required to update matching branches.""" + required_deployment_environments = sgqlc.types.Field(sgqlc.types.list_of(String), graphql_name="requiredDeploymentEnvironments") + """List of required deployment environments that must be deployed + successfully to update matching branches + """ + required_status_check_contexts = sgqlc.types.Field(sgqlc.types.list_of(String), graphql_name="requiredStatusCheckContexts") """List of required status check contexts that must pass for commits to be accepted to matching branches. @@ -23010,6 +28249,11 @@ class BranchProtectionRule(sgqlc.types.Type, Node): requires_conversation_resolution = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresConversationResolution") """Are conversations required to be resolved before merging.""" + requires_deployments = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresDeployments") + """Does this branch require deployment to specific environments + before merging + """ + requires_linear_history = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresLinearHistory") """Are merge commits prohibited from being pushed to this branch.""" @@ -23431,7 +28675,7 @@ class Commit(sgqlc.types.Type, Node, GitObject, Subscribable, UniformResourceLoc "authored_date", "authors", "blame", - "changed_files", + "changed_files_if_available", "check_suites", "comments", "committed_date", @@ -23448,7 +28692,6 @@ class Commit(sgqlc.types.Type, Node, GitObject, Subscribable, UniformResourceLoc "message_headline_html", "on_behalf_of", "parents", - "pushed_date", "signature", "status", "status_check_rollup", @@ -23542,8 +28785,12 @@ class Commit(sgqlc.types.Type, Node, GitObject, Subscribable, UniformResourceLoc want. """ - changed_files = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="changedFiles") - """The number of changed files in this commit.""" + changed_files_if_available = sgqlc.types.Field(Int, graphql_name="changedFilesIfAvailable") + """The number of changed files in this commit. If GitHub is unable to + calculate the number of changed files (for example due to a + timeout), this will return `null`. We recommend using this field + instead of `changedFiles`. + """ check_suites = sgqlc.types.Field( CheckSuiteConnection, @@ -23733,9 +28980,6 @@ class Commit(sgqlc.types.Type, Node, GitObject, Subscribable, UniformResourceLoc * `last` (`Int`): Returns the last _n_ elements from the list. """ - pushed_date = sgqlc.types.Field(DateTime, graphql_name="pushedDate") - """The datetime when this commit was pushed.""" - signature = sgqlc.types.Field(GitSignature, graphql_name="signature") """Commit signing information, if present.""" @@ -23856,6 +29100,51 @@ class CommitCommentThread(sgqlc.types.Type, Node, RepositoryNode): """ +class Comparison(sgqlc.types.Type, Node): + """Represents a comparison between two commit revisions.""" + + __schema__ = github_schema + __field_names__ = ("ahead_by", "base_target", "behind_by", "commits", "head_target", "status") + ahead_by = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="aheadBy") + """The number of commits ahead of the base branch.""" + + base_target = sgqlc.types.Field(sgqlc.types.non_null(GitObject), graphql_name="baseTarget") + """The base revision of this comparison.""" + + behind_by = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="behindBy") + """The number of commits behind the base branch.""" + + commits = sgqlc.types.Field( + sgqlc.types.non_null(ComparisonCommitConnection), + graphql_name="commits", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), + ) + """The commits which compose this comparison. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + """ + + head_target = sgqlc.types.Field(sgqlc.types.non_null(GitObject), graphql_name="headTarget") + """The head revision of this comparison.""" + + status = sgqlc.types.Field(sgqlc.types.non_null(ComparisonStatus), graphql_name="status") + """The status of this comparison.""" + + class ConnectedEvent(sgqlc.types.Type, Node): """Represents a 'connected' event on a given issue or pull request.""" @@ -24312,7 +29601,7 @@ class DisconnectedEvent(sgqlc.types.Type, Node): class Discussion( - sgqlc.types.Type, Comment, Updatable, Deletable, Labelable, Lockable, RepositoryNode, Subscribable, Reactable, Votable, Node + sgqlc.types.Type, Closable, Comment, Updatable, Deletable, Labelable, Lockable, RepositoryNode, Subscribable, Reactable, Votable, Node ): """A discussion in a repository.""" @@ -24326,6 +29615,7 @@ class Discussion( "number", "poll", "resource_path", + "state_reason", "title", "url", ) @@ -24374,6 +29664,9 @@ class Discussion( resource_path = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="resourcePath") """The path for this discussion.""" + state_reason = sgqlc.types.Field(DiscussionStateReason, graphql_name="stateReason") + """Identifies the reason for the discussion's state.""" + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") """The title of this discussion.""" @@ -24385,7 +29678,7 @@ class DiscussionCategory(sgqlc.types.Type, Node, RepositoryNode): """A category for discussions in a repository.""" __schema__ = github_schema - __field_names__ = ("created_at", "description", "emoji", "emoji_html", "is_answerable", "name", "updated_at") + __field_names__ = ("created_at", "description", "emoji", "emoji_html", "is_answerable", "name", "slug", "updated_at") created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") """Identifies the date and time when the object was created.""" @@ -24406,6 +29699,9 @@ class DiscussionCategory(sgqlc.types.Type, Node, RepositoryNode): name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") """The name of this category.""" + slug = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="slug") + """The slug of this category.""" + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") """Identifies the date and time when the object was last updated.""" @@ -24557,8 +29853,8 @@ class DraftIssue(sgqlc.types.Type, Node): "body_text", "created_at", "creator", - "project", - "project_item", + "project_v2_items", + "projects_v2", "title", "updated_at", ) @@ -24601,11 +29897,55 @@ class DraftIssue(sgqlc.types.Type, Node): creator = sgqlc.types.Field(Actor, graphql_name="creator") """The actor who created this draft issue.""" - project = sgqlc.types.Field(sgqlc.types.non_null("ProjectNext"), graphql_name="project") - """The project (beta) that contains this draft issue.""" + project_v2_items = sgqlc.types.Field( + sgqlc.types.non_null(ProjectV2ItemConnection), + graphql_name="projectV2Items", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), + ) + """List of items linked with the draft issue (currently draft issue + can be linked to only one item). + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + """ + + projects_v2 = sgqlc.types.Field( + sgqlc.types.non_null(ProjectV2Connection), + graphql_name="projectsV2", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), + ) + """Projects that link to this draft issue (currently draft issue can + be linked to only one project). + + Arguments: - project_item = sgqlc.types.Field(sgqlc.types.non_null("ProjectNextItem"), graphql_name="projectItem") - """The project (beta) item that wraps this draft issue.""" + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + """ title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") """The title of the draft issue""" @@ -24614,7 +29954,7 @@ class DraftIssue(sgqlc.types.Type, Node): """Identifies the date and time when the object was last updated.""" -class Enterprise(sgqlc.types.Type, Node): +class Enterprise(sgqlc.types.Type, Node, AnnouncementBanner): """An account to manage multiple organizations with consolidated policy and billing. """ @@ -24651,7 +29991,7 @@ class Enterprise(sgqlc.types.Type, Node): """ billing_info = sgqlc.types.Field(EnterpriseBillingInfo, graphql_name="billingInfo") - """Enterprise billing information visible to enterprise billing + """Enterprise billing information visible to enterprise billing managers. """ @@ -24686,6 +30026,7 @@ class Enterprise(sgqlc.types.Type, Node): ), ("role", sgqlc.types.Arg(EnterpriseUserAccountMembershipRole, graphql_name="role", default=None)), ("deployment", sgqlc.types.Arg(EnterpriseUserDeployment, graphql_name="deployment", default=None)), + ("has_two_factor_enabled", sgqlc.types.Arg(Boolean, graphql_name="hasTwoFactorEnabled", default=None)), ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), @@ -24707,6 +30048,10 @@ class Enterprise(sgqlc.types.Type, Node): user in the enterprise organization or server. * `deployment` (`EnterpriseUserDeployment`): Only return members within the selected GitHub Enterprise deployment + * `has_two_factor_enabled` (`Boolean`): Only return members with + this two-factor authentication status. Does not include members + who only have an account on a GitHub Enterprise Server instance. + (default: `null`) * `after` (`String`): Returns the elements in the list that come after the specified cursor. * `before` (`String`): Returns the elements in the list that come @@ -24752,7 +30097,10 @@ class Enterprise(sgqlc.types.Type, Node): """ owner_info = sgqlc.types.Field(EnterpriseOwnerInfo, graphql_name="ownerInfo") - """Enterprise information only visible to enterprise owners.""" + """Enterprise information visible to enterprise owners or enterprise + owners' personal access tokens (classic) with read:enterprise or + admin:enterprise scope. + """ resource_path = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="resourcePath") """The HTTP path for this enterprise.""" @@ -24800,7 +30148,9 @@ class EnterpriseAdministratorInvitation(sgqlc.types.Type, Node): class EnterpriseIdentityProvider(sgqlc.types.Type, Node): """An identity provider configured to provision identities for an - enterprise. + enterprise. Visible to enterprise owners or enterprise owners' + personal access tokens (classic) with read:enterprise or + admin:enterprise scope. """ __schema__ = github_schema @@ -25130,13 +30480,52 @@ class EnterpriseUserAccount(sgqlc.types.Type, Actor, Node): """ __schema__ = github_schema - __field_names__ = ("created_at", "enterprise", "name", "organizations", "updated_at", "user") + __field_names__ = ("created_at", "enterprise", "enterprise_installations", "name", "organizations", "updated_at", "user") created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") """Identifies the date and time when the object was created.""" enterprise = sgqlc.types.Field(sgqlc.types.non_null(Enterprise), graphql_name="enterprise") """The enterprise in which this user account exists.""" + enterprise_installations = sgqlc.types.Field( + sgqlc.types.non_null(EnterpriseServerInstallationMembershipConnection), + graphql_name="enterpriseInstallations", + args=sgqlc.types.ArgDict( + ( + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ( + "order_by", + sgqlc.types.Arg( + EnterpriseServerInstallationOrder, graphql_name="orderBy", default={"field": "HOST_NAME", "direction": "ASC"} + ), + ), + ("role", sgqlc.types.Arg(EnterpriseUserAccountMembershipRole, graphql_name="role", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), + ) + """A list of Enterprise Server installations this user is a member + of. + + Arguments: + + * `query` (`String`): The search string to look for. + * `order_by` (`EnterpriseServerInstallationOrder`): Ordering + options for installations returned from the connection. + (default: `{field: HOST_NAME, direction: ASC}`) + * `role` (`EnterpriseUserAccountMembershipRole`): The role of the + user in the installation. + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + """ + name = sgqlc.types.Field(String, graphql_name="name") """The name of the enterprise user account""" @@ -25217,7 +30606,16 @@ class Environment(sgqlc.types.Type, Node): class ExternalIdentity(sgqlc.types.Type, Node): - """An external identity provisioned by SAML SSO or SCIM.""" + """An external identity provisioned by SAML SSO or SCIM. If SAML is + configured on the organization, the external identity is visible + to (1) organization owners, (2) organization owners' personal + access tokens (classic) with read:org or admin:org scope, (3) + GitHub App with an installation token with read or write access to + members. If SAML is configured on the enterprise, the external + identity is visible to (1) enterprise owners, (2) enterprise + owners' personal access tokens (classic) with read:enterprise or + admin:enterprise scope. + """ __schema__ = github_schema __field_names__ = ("guid", "organization_invitation", "saml_identity", "scim_identity", "user") @@ -25483,6 +30881,7 @@ class Issue( Assignable, Closable, Comment, + Deletable, Updatable, UpdatableComment, Labelable, @@ -25491,7 +30890,7 @@ class Issue( RepositoryNode, Subscribable, UniformResourceLocatable, - ProjectNextOwner, + ProjectV2Owner, ): """An Issue is a place to discuss ideas, enhancements, tasks, and bugs for a project. @@ -25502,14 +30901,16 @@ class Issue( "body_resource_path", "body_url", "comments", + "full_database_id", "hovercard", "is_pinned", "is_read_by_viewer", + "linked_branches", "milestone", "number", "participants", "project_cards", - "project_next_items", + "project_items", "state", "state_reason", "timeline_items", @@ -25552,6 +30953,9 @@ class Issue( * `last` (`Int`): Returns the last _n_ elements from the list. """ + full_database_id = sgqlc.types.Field(BigInt, graphql_name="fullDatabaseId") + """Identifies the primary key from the database as a BigInt.""" + hovercard = sgqlc.types.Field( sgqlc.types.non_null(Hovercard), graphql_name="hovercard", @@ -25575,6 +30979,30 @@ class Issue( is_read_by_viewer = sgqlc.types.Field(Boolean, graphql_name="isReadByViewer") """Is this issue read by the viewer""" + linked_branches = sgqlc.types.Field( + sgqlc.types.non_null(LinkedBranchConnection), + graphql_name="linkedBranches", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), + ) + """Branches linked to this issue. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + """ + milestone = sgqlc.types.Field("Milestone", graphql_name="milestone") """Identifies the milestone associated with the issue.""" @@ -25638,9 +31066,9 @@ class Issue( NOT_ARCHIVED]`) """ - project_next_items = sgqlc.types.Field( - sgqlc.types.non_null(ProjectNextItemConnection), - graphql_name="projectNextItems", + project_items = sgqlc.types.Field( + sgqlc.types.non_null(ProjectV2ItemConnection), + graphql_name="projectItems", args=sgqlc.types.ArgDict( ( ("include_archived", sgqlc.types.Arg(Boolean, graphql_name="includeArchived", default=True)), @@ -25651,7 +31079,7 @@ class Issue( ) ), ) - """List of project (beta) items associated with this issue. + """List of project items associated with this issue. Arguments: @@ -25783,7 +31211,10 @@ class IssueComment(sgqlc.types.Type, Node, Comment, Deletable, Minimizable, Upda """Represents a comment on an Issue.""" __schema__ = github_schema - __field_names__ = ("issue", "pull_request", "resource_path", "url") + __field_names__ = ("full_database_id", "issue", "pull_request", "resource_path", "url") + full_database_id = sgqlc.types.Field(BigInt, graphql_name="fullDatabaseId") + """Identifies the primary key from the database as a BigInt.""" + issue = sgqlc.types.Field(sgqlc.types.non_null(Issue), graphql_name="issue") """Identifies the issue associated with the comment.""" @@ -26026,6 +31457,15 @@ class License(sgqlc.types.Type, Node): """URL to the license on """ +class LinkedBranch(sgqlc.types.Type, Node): + """A branch linked to an issue.""" + + __schema__ = github_schema + __field_names__ = ("ref",) + ref = sgqlc.types.Field("Ref", graphql_name="ref") + """The branch's ref.""" + + class LockedEvent(sgqlc.types.Type, Node): """Represents a 'locked' event on a given issue or pull request.""" @@ -26462,6 +31902,106 @@ class MentionedEvent(sgqlc.types.Type, Node): """Identifies the primary key from the database.""" +class MergeQueue(sgqlc.types.Type, Node): + """The queue of pull request entries to be merged into a protected + branch in a repository. + """ + + __schema__ = github_schema + __field_names__ = ("configuration", "entries", "next_entry_estimated_time_to_merge", "repository", "resource_path", "url") + configuration = sgqlc.types.Field(MergeQueueConfiguration, graphql_name="configuration") + """The configuration for this merge queue""" + + entries = sgqlc.types.Field( + MergeQueueEntryConnection, + graphql_name="entries", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), + ) + """The entries in the queue + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + """ + + next_entry_estimated_time_to_merge = sgqlc.types.Field(Int, graphql_name="nextEntryEstimatedTimeToMerge") + """The estimated time in seconds until a newly added entry would be + merged + """ + + repository = sgqlc.types.Field("Repository", graphql_name="repository") + """The repository this merge queue belongs to""" + + resource_path = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="resourcePath") + """The HTTP path for this merge queue""" + + url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="url") + """The HTTP URL for this merge queue""" + + +class MergeQueueEntry(sgqlc.types.Type, Node): + """Entries in a MergeQueue""" + + __schema__ = github_schema + __field_names__ = ( + "base_commit", + "enqueued_at", + "enqueuer", + "estimated_time_to_merge", + "head_commit", + "jump", + "merge_queue", + "position", + "pull_request", + "solo", + "state", + ) + base_commit = sgqlc.types.Field(Commit, graphql_name="baseCommit") + """The base commit for this entry""" + + enqueued_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="enqueuedAt") + """The date and time this entry was added to the merge queue""" + + enqueuer = sgqlc.types.Field(sgqlc.types.non_null(Actor), graphql_name="enqueuer") + """The actor that enqueued this entry""" + + estimated_time_to_merge = sgqlc.types.Field(Int, graphql_name="estimatedTimeToMerge") + """The estimated time in seconds until this entry will be merged""" + + head_commit = sgqlc.types.Field(Commit, graphql_name="headCommit") + """The head commit for this entry""" + + jump = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="jump") + """Whether this pull request should jump the queue""" + + merge_queue = sgqlc.types.Field(MergeQueue, graphql_name="mergeQueue") + """The merge queue that this entry belongs to""" + + position = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="position") + """The position of this entry in the queue""" + + pull_request = sgqlc.types.Field("PullRequest", graphql_name="pullRequest") + """The pull request that will be added to a merge group""" + + solo = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="solo") + """Does this pull request need to be deployed on its own""" + + state = sgqlc.types.Field(sgqlc.types.non_null(MergeQueueEntryState), graphql_name="state") + """The state of this entry in the queue""" + + class MergedEvent(sgqlc.types.Type, Node, UniformResourceLocatable): """Represents a 'merged' event on a given pull request.""" @@ -26487,18 +32027,20 @@ class MergedEvent(sgqlc.types.Type, Node, UniformResourceLocatable): class MigrationSource(sgqlc.types.Type, Node): - """An Octoshift migration source.""" + """A GitHub Enterprise Importer (GEI) migration source.""" __schema__ = github_schema __field_names__ = ("name", "type", "url") name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - """The Octoshift migration source name.""" + """The migration source name.""" type = sgqlc.types.Field(sgqlc.types.non_null(MigrationSourceType), graphql_name="type") - """The Octoshift migration source type.""" + """The migration source type.""" url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="url") - """The Octoshift migration source URL.""" + """The migration source URL, for example `https://github.com` or + `https://monalisa.ghe.com`. + """ class Milestone(sgqlc.types.Type, Node, Closable, UniformResourceLocatable): @@ -26667,7 +32209,9 @@ class MovedColumnsInProjectEvent(sgqlc.types.Type, Node): class OIDCProvider(sgqlc.types.Type, Node): """An OIDC identity provider configured to provision identities for - an enterprise. + an enterprise. Visible to enterprise owners or enterprise owners' + personal access tokens (classic) with read:enterprise or + admin:enterprise scope. """ __schema__ = github_schema @@ -26721,16 +32265,16 @@ class OauthApplicationCreateAuditEntry(sgqlc.types.Type, Node, AuditEntry, Oauth __schema__ = github_schema __field_names__ = ("application_url", "callback_url", "rate_limit", "state") application_url = sgqlc.types.Field(URI, graphql_name="applicationUrl") - """The application URL of the OAuth Application.""" + """The application URL of the OAuth application.""" callback_url = sgqlc.types.Field(URI, graphql_name="callbackUrl") - """The callback URL of the OAuth Application.""" + """The callback URL of the OAuth application.""" rate_limit = sgqlc.types.Field(Int, graphql_name="rateLimit") - """The rate limit of the OAuth Application.""" + """The rate limit of the OAuth application.""" state = sgqlc.types.Field(OauthApplicationCreateAuditEntryState, graphql_name="state") - """The state of the OAuth Application.""" + """The state of the OAuth application.""" class OrgAddBillingManagerAuditEntry(sgqlc.types.Type, Node, AuditEntry, OrganizationAuditEntryData): @@ -27080,7 +32624,8 @@ class Organization( Actor, PackageOwner, ProjectOwner, - ProjectNextOwner, + ProjectV2Owner, + ProjectV2Recent, RepositoryDiscussionAuthor, RepositoryDiscussionCommentAuthor, RepositoryOwner, @@ -27088,6 +32633,7 @@ class Organization( MemberStatusable, ProfileOwner, Sponsorable, + AnnouncementBanner, ): """An account on GitHub, with one or more owners, that has repositories, members and teams. @@ -27107,6 +32653,7 @@ class Organization( "ip_allow_list_entries", "ip_allow_list_for_installed_apps_enabled_setting", "is_verified", + "mannequins", "members_can_fork_private_repositories", "members_with_role", "new_team_resource_path", @@ -27116,6 +32663,8 @@ class Organization( "pending_members", "repository_migrations", "requires_two_factor_authentication", + "ruleset", + "rulesets", "saml_identity_provider", "team", "teams", @@ -27128,6 +32677,7 @@ class Organization( "viewer_can_create_teams", "viewer_is_amember", "viewer_is_following", + "web_commit_signoff_required", ) audit_log = sgqlc.types.Field( sgqlc.types.non_null(OrganizationAuditEntryConnection), @@ -27300,6 +32850,34 @@ class Organization( website. """ + mannequins = sgqlc.types.Field( + sgqlc.types.non_null(MannequinConnection), + graphql_name="mannequins", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("order_by", sgqlc.types.Arg(MannequinOrder, graphql_name="orderBy", default={"field": "CREATED_AT", "direction": "ASC"})), + ) + ), + ) + """A list of all mannequins for this organization. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `order_by` (`MannequinOrder`): Ordering options for mannequins + returned from the connection. (default: `{field: CREATED_AT, + direction: ASC}`) + """ + members_can_fork_private_repositories = sgqlc.types.Field( sgqlc.types.non_null(Boolean), graphql_name="membersCanForkPrivateRepositories" ) @@ -27411,8 +32989,51 @@ class Organization( and outside collaborators to enable two-factor authentication. """ + ruleset = sgqlc.types.Field( + "RepositoryRuleset", + graphql_name="ruleset", + args=sgqlc.types.ArgDict((("database_id", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="databaseId", default=None)),)), + ) + """Returns a single ruleset from the current organization by ID. + + Arguments: + + * `database_id` (`Int!`): The ID of the ruleset to be returned. + """ + + rulesets = sgqlc.types.Field( + RepositoryRulesetConnection, + graphql_name="rulesets", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("include_parents", sgqlc.types.Arg(Boolean, graphql_name="includeParents", default=False)), + ) + ), + ) + """A list of rulesets for this organization. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `include_parents` (`Boolean`): Return rulesets configured at + higher levels that apply to this organization (default: `false`) + """ + saml_identity_provider = sgqlc.types.Field("OrganizationIdentityProvider", graphql_name="samlIdentityProvider") - """The Organization's SAML identity providers""" + """The Organization's SAML identity provider. Visible to (1) + organization owners, (2) organization owners' personal access + tokens (classic) with read:org or admin:org scope, (3) GitHub App + with an installation token with read or write access to members. + """ team = sgqlc.types.Field( "Team", @@ -27432,6 +33053,7 @@ class Organization( args=sgqlc.types.ArgDict( ( ("privacy", sgqlc.types.Arg(TeamPrivacy, graphql_name="privacy", default=None)), + ("notification_setting", sgqlc.types.Arg(TeamNotificationSetting, graphql_name="notificationSetting", default=None)), ("role", sgqlc.types.Arg(TeamRole, graphql_name="role", default=None)), ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), ( @@ -27454,6 +33076,8 @@ class Organization( * `privacy` (`TeamPrivacy`): If non-null, filters teams according to privacy + * `notification_setting` (`TeamNotificationSetting`): If non-null, + filters teams according to notification setting * `role` (`TeamRole`): If non-null, filters teams according to whether the viewer is an admin or member on team * `query` (`String`): If non-null, filters teams with query on @@ -27500,10 +33124,18 @@ class Organization( viewer_is_following = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerIsFollowing") """Whether or not this Organization is followed by the viewer.""" + web_commit_signoff_required = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="webCommitSignoffRequired") + """Whether contributors are required to sign off on web-based commits + for repositories in this organization. + """ + class OrganizationIdentityProvider(sgqlc.types.Type, Node): """An Identity Provider configured to provision SAML and SCIM - identities for Organizations + identities for Organizations. Visible to (1) organization owners, + (2) organization owners' personal access tokens (classic) with + read:org or admin:org scope, (3) GitHub App with an installation + token with read or write access to members. """ __schema__ = github_schema @@ -27570,13 +33202,16 @@ class OrganizationInvitation(sgqlc.types.Type, Node): """An Invitation for a user to an organization.""" __schema__ = github_schema - __field_names__ = ("created_at", "email", "invitation_type", "invitee", "inviter", "organization", "role") + __field_names__ = ("created_at", "email", "invitation_source", "invitation_type", "invitee", "inviter", "organization", "role") created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") """Identifies the date and time when the object was created.""" email = sgqlc.types.Field(String, graphql_name="email") """The email address of the user invited to the organization.""" + invitation_source = sgqlc.types.Field(sgqlc.types.non_null(OrganizationInvitationSource), graphql_name="invitationSource") + """The source of the invitation.""" + invitation_type = sgqlc.types.Field(sgqlc.types.non_null(OrganizationInvitationType), graphql_name="invitationType") """The type of invitation that was sent (e.g. email, user).""" @@ -27593,6 +33228,49 @@ class OrganizationInvitation(sgqlc.types.Type, Node): """The user's pending role in the organization (e.g. member, owner).""" +class OrganizationMigration(sgqlc.types.Type, Node): + """A GitHub Enterprise Importer (GEI) organization migration.""" + + __schema__ = github_schema + __field_names__ = ( + "created_at", + "database_id", + "failure_reason", + "remaining_repositories_count", + "source_org_name", + "source_org_url", + "state", + "target_org_name", + "total_repositories_count", + ) + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + """Identifies the date and time when the object was created.""" + + database_id = sgqlc.types.Field(String, graphql_name="databaseId") + """Identifies the primary key from the database.""" + + failure_reason = sgqlc.types.Field(String, graphql_name="failureReason") + """The reason the organization migration failed.""" + + remaining_repositories_count = sgqlc.types.Field(Int, graphql_name="remainingRepositoriesCount") + """The remaining amount of repos to be migrated.""" + + source_org_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="sourceOrgName") + """The name of the source organization to be migrated.""" + + source_org_url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="sourceOrgUrl") + """The URL of the source organization to migrate.""" + + state = sgqlc.types.Field(sgqlc.types.non_null(OrganizationMigrationState), graphql_name="state") + """The migration state.""" + + target_org_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="targetOrgName") + """The name of the target organization.""" + + total_repositories_count = sgqlc.types.Field(Int, graphql_name="totalRepositoriesCount") + """The total amount of repositories to be migrated.""" + + class OrganizationTeamsHovercardContext(sgqlc.types.Type, HovercardContext): """An organization teams hovercard context""" @@ -27643,6 +33321,7 @@ class OrganizationsHovercardContext(sgqlc.types.Type, HovercardContext): graphql_name="relevantOrganizations", args=sgqlc.types.ArgDict( ( + ("order_by", sgqlc.types.Arg(OrganizationOrder, graphql_name="orderBy", default=None)), ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), @@ -27654,6 +33333,8 @@ class OrganizationsHovercardContext(sgqlc.types.Type, HovercardContext): Arguments: + * `order_by` (`OrganizationOrder`): Ordering options for the + User's organizations. (default: `null`) * `after` (`String`): Returns the elements in the list that come after the specified cursor. * `before` (`String`): Returns the elements in the list that come @@ -27895,10 +33576,13 @@ class PinnedIssue(sgqlc.types.Type, Node): """A Pinned Issue is a issue pinned to a repository's index page.""" __schema__ = github_schema - __field_names__ = ("database_id", "issue", "pinned_by", "repository") + __field_names__ = ("database_id", "full_database_id", "issue", "pinned_by", "repository") database_id = sgqlc.types.Field(Int, graphql_name="databaseId") """Identifies the primary key from the database.""" + full_database_id = sgqlc.types.Field(BigInt, graphql_name="fullDatabaseId") + """Identifies the primary key from the database as a BigInt.""" + issue = sgqlc.types.Field(sgqlc.types.non_null(Issue), graphql_name="issue") """The issue that was pinned.""" @@ -28172,7 +33856,7 @@ class ProjectColumn(sgqlc.types.Type, Node): """The HTTP URL for this project column""" -class ProjectNext(sgqlc.types.Type, Node, Closable, Updatable): +class ProjectV2(sgqlc.types.Type, Closable, Updatable, Node): """New projects that manage issues, pull requests and drafts using tables and boards. """ @@ -28182,19 +33866,25 @@ class ProjectNext(sgqlc.types.Type, Node, Closable, Updatable): "created_at", "creator", "database_id", - "description", + "field", "fields", "items", "number", "owner", "public", + "readme", "repositories", "resource_path", "short_description", + "teams", + "template", "title", "updated_at", "url", + "view", "views", + "workflow", + "workflows", ) created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") """Identifies the date and time when the object was created.""" @@ -28205,11 +33895,20 @@ class ProjectNext(sgqlc.types.Type, Node, Closable, Updatable): database_id = sgqlc.types.Field(Int, graphql_name="databaseId") """Identifies the primary key from the database.""" - description = sgqlc.types.Field(String, graphql_name="description") - """The project's description.""" + field = sgqlc.types.Field( + "ProjectV2FieldConfiguration", + graphql_name="field", + args=sgqlc.types.ArgDict((("name", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="name", default=None)),)), + ) + """A field of the project + + Arguments: + + * `name` (`String!`): The name of the field + """ fields = sgqlc.types.Field( - sgqlc.types.non_null(ProjectNextFieldConnection), + sgqlc.types.non_null(ProjectV2FieldConfigurationConnection), graphql_name="fields", args=sgqlc.types.ArgDict( ( @@ -28217,10 +33916,14 @@ class ProjectNext(sgqlc.types.Type, Node, Closable, Updatable): ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ( + "order_by", + sgqlc.types.Arg(ProjectV2FieldOrder, graphql_name="orderBy", default={"field": "POSITION", "direction": "ASC"}), + ), ) ), ) - """List of fields in the project + """List of fields and their constraints in the project Arguments: @@ -28230,10 +33933,13 @@ class ProjectNext(sgqlc.types.Type, Node, Closable, Updatable): before the specified cursor. * `first` (`Int`): Returns the first _n_ elements from the list. * `last` (`Int`): Returns the last _n_ elements from the list. + * `order_by` (`ProjectV2FieldOrder`): Ordering options for project + v2 fields returned from the connection (default: `{field: + POSITION, direction: ASC}`) """ items = sgqlc.types.Field( - sgqlc.types.non_null(ProjectNextItemConnection), + sgqlc.types.non_null(ProjectV2ItemConnection), graphql_name="items", args=sgqlc.types.ArgDict( ( @@ -28241,6 +33947,10 @@ class ProjectNext(sgqlc.types.Type, Node, Closable, Updatable): ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ( + "order_by", + sgqlc.types.Arg(ProjectV2ItemOrder, graphql_name="orderBy", default={"field": "POSITION", "direction": "ASC"}), + ), ) ), ) @@ -28254,17 +33964,23 @@ class ProjectNext(sgqlc.types.Type, Node, Closable, Updatable): before the specified cursor. * `first` (`Int`): Returns the first _n_ elements from the list. * `last` (`Int`): Returns the last _n_ elements from the list. + * `order_by` (`ProjectV2ItemOrder`): Ordering options for project + v2 items returned from the connection (default: `{field: + POSITION, direction: ASC}`) """ number = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="number") """The project's number.""" - owner = sgqlc.types.Field(sgqlc.types.non_null(ProjectNextOwner), graphql_name="owner") + owner = sgqlc.types.Field(sgqlc.types.non_null(ProjectV2Owner), graphql_name="owner") """The project's owner. Currently limited to organizations and users.""" public = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="public") """Returns true if the project is public.""" + readme = sgqlc.types.Field(String, graphql_name="readme") + """The project's readme.""" + repositories = sgqlc.types.Field( sgqlc.types.non_null(RepositoryConnection), graphql_name="repositories", @@ -28274,6 +33990,10 @@ class ProjectNext(sgqlc.types.Type, Node, Closable, Updatable): ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ( + "order_by", + sgqlc.types.Arg(RepositoryOrder, graphql_name="orderBy", default={"field": "CREATED_AT", "direction": "DESC"}), + ), ) ), ) @@ -28287,6 +34007,9 @@ class ProjectNext(sgqlc.types.Type, Node, Closable, Updatable): before the specified cursor. * `first` (`Int`): Returns the first _n_ elements from the list. * `last` (`Int`): Returns the last _n_ elements from the list. + * `order_by` (`RepositoryOrder`): Ordering options for + repositories returned from the connection (default: `{field: + CREATED_AT, direction: DESC}`) """ resource_path = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="resourcePath") @@ -28295,7 +34018,37 @@ class ProjectNext(sgqlc.types.Type, Node, Closable, Updatable): short_description = sgqlc.types.Field(String, graphql_name="shortDescription") """The project's short description.""" - title = sgqlc.types.Field(String, graphql_name="title") + teams = sgqlc.types.Field( + sgqlc.types.non_null(TeamConnection), + graphql_name="teams", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("order_by", sgqlc.types.Arg(TeamOrder, graphql_name="orderBy", default={"field": "NAME", "direction": "ASC"})), + ) + ), + ) + """The teams the project is linked to. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `order_by` (`TeamOrder`): Ordering options for teams returned + from this connection. (default: `{field: NAME, direction: ASC}`) + """ + + template = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="template") + """Returns true if this project is a template.""" + + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") """The project's name.""" updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") @@ -28304,8 +34057,20 @@ class ProjectNext(sgqlc.types.Type, Node, Closable, Updatable): url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="url") """The HTTP URL for this project""" + view = sgqlc.types.Field( + "ProjectV2View", + graphql_name="view", + args=sgqlc.types.ArgDict((("number", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="number", default=None)),)), + ) + """A view of the project + + Arguments: + + * `number` (`Int!`): The number of a view belonging to the project + """ + views = sgqlc.types.Field( - sgqlc.types.non_null(ProjectViewConnection), + sgqlc.types.non_null(ProjectV2ViewConnection), graphql_name="views", args=sgqlc.types.ArgDict( ( @@ -28313,6 +34078,10 @@ class ProjectNext(sgqlc.types.Type, Node, Closable, Updatable): ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ( + "order_by", + sgqlc.types.Arg(ProjectV2ViewOrder, graphql_name="orderBy", default={"field": "POSITION", "direction": "ASC"}), + ), ) ), ) @@ -28326,18 +34095,65 @@ class ProjectNext(sgqlc.types.Type, Node, Closable, Updatable): before the specified cursor. * `first` (`Int`): Returns the first _n_ elements from the list. * `last` (`Int`): Returns the last _n_ elements from the list. + * `order_by` (`ProjectV2ViewOrder`): Ordering options for project + v2 views returned from the connection (default: `{field: + POSITION, direction: ASC}`) """ + workflow = sgqlc.types.Field( + "ProjectV2Workflow", + graphql_name="workflow", + args=sgqlc.types.ArgDict((("number", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="number", default=None)),)), + ) + """A workflow of the project + + Arguments: -class ProjectNextField(sgqlc.types.Type, ProjectNextFieldCommon, Node): + * `number` (`Int!`): The number of a workflow belonging to the + project + """ + + workflows = sgqlc.types.Field( + sgqlc.types.non_null(ProjectV2WorkflowConnection), + graphql_name="workflows", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ( + "order_by", + sgqlc.types.Arg(ProjectV2WorkflowOrder, graphql_name="orderBy", default={"field": "NAME", "direction": "ASC"}), + ), + ) + ), + ) + """List of the workflows in the project + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `order_by` (`ProjectV2WorkflowOrder`): Ordering options for + project v2 workflows returned from the connection (default: + `{field: NAME, direction: ASC}`) + """ + + +class ProjectV2Field(sgqlc.types.Type, ProjectV2FieldCommon, Node): """A field inside a project.""" __schema__ = github_schema __field_names__ = () -class ProjectNextItem(sgqlc.types.Type, Node): - """An item within a new Project.""" +class ProjectV2Item(sgqlc.types.Type, Node): + """An item within a Project.""" __schema__ = github_schema __field_names__ = ( @@ -28345,14 +34161,14 @@ class ProjectNextItem(sgqlc.types.Type, Node): "created_at", "creator", "database_id", + "field_value_by_name", "field_values", "is_archived", "project", - "title", "type", "updated_at", ) - content = sgqlc.types.Field("ProjectNextItemContent", graphql_name="content") + content = sgqlc.types.Field("ProjectV2ItemContent", graphql_name="content") """The content of the referenced draft issue, issue, or pull request""" created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") @@ -28364,8 +34180,22 @@ class ProjectNextItem(sgqlc.types.Type, Node): database_id = sgqlc.types.Field(Int, graphql_name="databaseId") """Identifies the primary key from the database.""" + field_value_by_name = sgqlc.types.Field( + "ProjectV2ItemFieldValue", + graphql_name="fieldValueByName", + args=sgqlc.types.ArgDict((("name", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="name", default=None)),)), + ) + """The field value of the first project field which matches the + 'name' argument that is set on the item. + + Arguments: + + * `name` (`String!`): The name of the field to return the field + value of + """ + field_values = sgqlc.types.Field( - sgqlc.types.non_null(ProjectNextItemFieldValueConnection), + sgqlc.types.non_null(ProjectV2ItemFieldValueConnection), graphql_name="fieldValues", args=sgqlc.types.ArgDict( ( @@ -28373,10 +34203,16 @@ class ProjectNextItem(sgqlc.types.Type, Node): ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ( + "order_by", + sgqlc.types.Arg( + ProjectV2ItemFieldValueOrder, graphql_name="orderBy", default={"field": "POSITION", "direction": "ASC"} + ), + ), ) ), ) - """List of field values + """The field values that are set on the item. Arguments: @@ -28386,69 +34222,134 @@ class ProjectNextItem(sgqlc.types.Type, Node): before the specified cursor. * `first` (`Int`): Returns the first _n_ elements from the list. * `last` (`Int`): Returns the last _n_ elements from the list. + * `order_by` (`ProjectV2ItemFieldValueOrder`): Ordering options + for project v2 item field values returned from the connection + (default: `{field: POSITION, direction: ASC}`) """ is_archived = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isArchived") """Whether the item is archived.""" - project = sgqlc.types.Field(sgqlc.types.non_null(ProjectNext), graphql_name="project") + project = sgqlc.types.Field(sgqlc.types.non_null(ProjectV2), graphql_name="project") """The project that contains this item.""" - title = sgqlc.types.Field(String, graphql_name="title") - """The title of the item""" - - type = sgqlc.types.Field(sgqlc.types.non_null(ProjectItemType), graphql_name="type") + type = sgqlc.types.Field(sgqlc.types.non_null(ProjectV2ItemType), graphql_name="type") """The type of the item.""" updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") """Identifies the date and time when the object was last updated.""" -class ProjectNextItemFieldValue(sgqlc.types.Type, Node): - """An value of a field in an item of a new Project.""" +class ProjectV2ItemFieldDateValue(sgqlc.types.Type, ProjectV2ItemFieldValueCommon, Node): + """The value of a date field in a Project item.""" __schema__ = github_schema - __field_names__ = ("created_at", "creator", "database_id", "project_field", "project_item", "updated_at", "value") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - """Identifies the date and time when the object was created.""" + __field_names__ = ("date",) + date = sgqlc.types.Field(Date, graphql_name="date") + """Date value for the field""" - creator = sgqlc.types.Field(Actor, graphql_name="creator") - """The actor who created the item.""" - database_id = sgqlc.types.Field(Int, graphql_name="databaseId") - """Identifies the primary key from the database.""" +class ProjectV2ItemFieldIterationValue(sgqlc.types.Type, ProjectV2ItemFieldValueCommon, Node): + """The value of an iteration field in a Project item.""" - project_field = sgqlc.types.Field(sgqlc.types.non_null(ProjectNextField), graphql_name="projectField") - """The project field that contains this value.""" + __schema__ = github_schema + __field_names__ = ("duration", "iteration_id", "start_date", "title", "title_html") + duration = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="duration") + """The duration of the iteration in days.""" - project_item = sgqlc.types.Field(sgqlc.types.non_null(ProjectNextItem), graphql_name="projectItem") - """The project item that contains this value.""" + iteration_id = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="iterationId") + """The ID of the iteration.""" - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - """Identifies the date and time when the object was last updated.""" + start_date = sgqlc.types.Field(sgqlc.types.non_null(Date), graphql_name="startDate") + """The start date of the iteration.""" + + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + """The title of the iteration.""" + + title_html = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="titleHTML") + """The title of the iteration, with HTML.""" - value = sgqlc.types.Field(String, graphql_name="value") - """The value of a field""" +class ProjectV2ItemFieldNumberValue(sgqlc.types.Type, ProjectV2ItemFieldValueCommon, Node): + """The value of a number field in a Project item.""" -class ProjectView(sgqlc.types.Type, Node): - """A view within a Project.""" + __schema__ = github_schema + __field_names__ = ("number",) + number = sgqlc.types.Field(Float, graphql_name="number") + """Number as a float(8)""" + + +class ProjectV2ItemFieldSingleSelectValue(sgqlc.types.Type, ProjectV2ItemFieldValueCommon, Node): + """The value of a single select field in a Project item.""" + + __schema__ = github_schema + __field_names__ = ("name", "name_html", "option_id") + name = sgqlc.types.Field(String, graphql_name="name") + """The name of the selected single select option.""" + + name_html = sgqlc.types.Field(String, graphql_name="nameHTML") + """The html name of the selected single select option.""" + + option_id = sgqlc.types.Field(String, graphql_name="optionId") + """The id of the selected single select option.""" + + +class ProjectV2ItemFieldTextValue(sgqlc.types.Type, ProjectV2ItemFieldValueCommon, Node): + """The value of a text field in a Project item.""" + + __schema__ = github_schema + __field_names__ = ("text",) + text = sgqlc.types.Field(String, graphql_name="text") + """Text value of a field""" + + +class ProjectV2IterationField(sgqlc.types.Type, ProjectV2FieldCommon, Node): + """An iteration field inside a project.""" + + __schema__ = github_schema + __field_names__ = ("configuration",) + configuration = sgqlc.types.Field(sgqlc.types.non_null(ProjectV2IterationFieldConfiguration), graphql_name="configuration") + """Iteration configuration settings""" + + +class ProjectV2SingleSelectField(sgqlc.types.Type, ProjectV2FieldCommon, Node): + """A single select field inside a project.""" + + __schema__ = github_schema + __field_names__ = ("options",) + options = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProjectV2SingleSelectFieldOption))), + graphql_name="options", + args=sgqlc.types.ArgDict( + (("names", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="names", default=None)),) + ), + ) + """Options for the single select field + + Arguments: + + * `names` (`[String!]`): Filter returned options to only those + matching these names, case insensitive. + """ + + +class ProjectV2View(sgqlc.types.Type, Node): + """A view within a ProjectV2.""" __schema__ = github_schema __field_names__ = ( "created_at", "database_id", + "fields", "filter", - "group_by", - "items", + "group_by_fields", "layout", "name", "number", "project", - "sort_by", + "sort_by_fields", "updated_at", - "vertical_group_by", - "visible_fields", + "vertical_group_by_fields", ) created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") """Identifies the date and time when the object was created.""" @@ -28456,25 +34357,57 @@ class ProjectView(sgqlc.types.Type, Node): database_id = sgqlc.types.Field(Int, graphql_name="databaseId") """Identifies the primary key from the database.""" + fields = sgqlc.types.Field( + ProjectV2FieldConfigurationConnection, + graphql_name="fields", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ( + "order_by", + sgqlc.types.Arg(ProjectV2FieldOrder, graphql_name="orderBy", default={"field": "POSITION", "direction": "ASC"}), + ), + ) + ), + ) + """The view's visible fields. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `order_by` (`ProjectV2FieldOrder`): Ordering options for the + project v2 fields returned from the connection. (default: + `{field: POSITION, direction: ASC}`) + """ + filter = sgqlc.types.Field(String, graphql_name="filter") """The project view's filter.""" - group_by = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(Int)), graphql_name="groupBy") - """The view's group-by field.""" - - items = sgqlc.types.Field( - sgqlc.types.non_null(ProjectNextItemConnection), - graphql_name="items", + group_by_fields = sgqlc.types.Field( + ProjectV2FieldConfigurationConnection, + graphql_name="groupByFields", args=sgqlc.types.ArgDict( ( ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ( + "order_by", + sgqlc.types.Arg(ProjectV2FieldOrder, graphql_name="orderBy", default={"field": "POSITION", "direction": "ASC"}), + ), ) ), ) - """The view's filtered items. + """The view's group-by field. Arguments: @@ -28484,9 +34417,12 @@ class ProjectView(sgqlc.types.Type, Node): before the specified cursor. * `first` (`Int`): Returns the first _n_ elements from the list. * `last` (`Int`): Returns the last _n_ elements from the list. + * `order_by` (`ProjectV2FieldOrder`): Ordering options for the + project v2 fields returned from the connection. (default: + `{field: POSITION, direction: ASC}`) """ - layout = sgqlc.types.Field(sgqlc.types.non_null(ProjectViewLayout), graphql_name="layout") + layout = sgqlc.types.Field(sgqlc.types.non_null(ProjectV2ViewLayout), graphql_name="layout") """The project view's layout.""" name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") @@ -28495,20 +34431,93 @@ class ProjectView(sgqlc.types.Type, Node): number = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="number") """The project view's number.""" - project = sgqlc.types.Field(sgqlc.types.non_null(ProjectNext), graphql_name="project") + project = sgqlc.types.Field(sgqlc.types.non_null(ProjectV2), graphql_name="project") """The project that contains this view.""" - sort_by = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(SortBy)), graphql_name="sortBy") - """The view's sort-by config.""" + sort_by_fields = sgqlc.types.Field( + ProjectV2SortByFieldConnection, + graphql_name="sortByFields", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), + ) + """The view's sort-by config. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + """ updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") """Identifies the date and time when the object was last updated.""" - vertical_group_by = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(Int)), graphql_name="verticalGroupBy") - """The view's vertical-group-by field.""" + vertical_group_by_fields = sgqlc.types.Field( + ProjectV2FieldConfigurationConnection, + graphql_name="verticalGroupByFields", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ( + "order_by", + sgqlc.types.Arg(ProjectV2FieldOrder, graphql_name="orderBy", default={"field": "POSITION", "direction": "ASC"}), + ), + ) + ), + ) + """The view's vertical-group-by field. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `order_by` (`ProjectV2FieldOrder`): Ordering options for the + project v2 fields returned from the connection. (default: + `{field: POSITION, direction: ASC}`) + """ + + +class ProjectV2Workflow(sgqlc.types.Type, Node): + """A workflow inside a project.""" + + __schema__ = github_schema + __field_names__ = ("created_at", "database_id", "enabled", "name", "number", "project", "updated_at") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + """Identifies the date and time when the object was created.""" + + database_id = sgqlc.types.Field(Int, graphql_name="databaseId") + """Identifies the primary key from the database.""" + + enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="enabled") + """The workflows' enabled state.""" - visible_fields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(Int)), graphql_name="visibleFields") - """The view's visible fields.""" + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + """The workflows' name.""" + + number = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="number") + """The workflows' number.""" + + project = sgqlc.types.Field(sgqlc.types.non_null(ProjectV2), graphql_name="project") + """The project that contains this workflow.""" + + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + """Identifies the date and time when the object was last updated.""" class PublicKey(sgqlc.types.Type, Node): @@ -28559,7 +34568,7 @@ class PullRequest( RepositoryNode, Subscribable, UniformResourceLocatable, - ProjectNextOwner, + ProjectV2Owner, ): """A repository pull request.""" @@ -28594,6 +34603,7 @@ class PullRequest( "maintainer_can_modify", "merge_commit", "merge_state_status", + "merge_queue_entry", "mergeable", "merged", "merged_at", @@ -28604,7 +34614,7 @@ class PullRequest( "permalink", "potential_merge_commit", "project_cards", - "project_next_items", + "project_items", "revert_resource_path", "revert_url", "review_decision", @@ -28616,11 +34626,14 @@ class PullRequest( "timeline_items", "title", "title_html", + "total_comments_count", "viewer_can_apply_suggestion", "viewer_can_delete_head_ref", "viewer_can_disable_auto_merge", + "viewer_can_edit_files", "viewer_can_enable_auto_merge", "viewer_can_merge_as_admin", + "viewer_can_update_branch", "viewer_latest_review", "viewer_latest_review_request", "viewer_merge_body_text", @@ -28880,6 +34893,11 @@ class PullRequest( status. """ + merge_queue_entry = sgqlc.types.Field(MergeQueueEntry, graphql_name="mergeQueueEntry") + """The merge queue entry of the pull request in the base branch's + merge queue + """ + mergeable = sgqlc.types.Field(sgqlc.types.non_null(MergeableState), graphql_name="mergeable") """Whether or not the pull request can be merged based on the existence of merge conflicts. @@ -28969,9 +34987,9 @@ class PullRequest( NOT_ARCHIVED]`) """ - project_next_items = sgqlc.types.Field( - sgqlc.types.non_null(ProjectNextItemConnection), - graphql_name="projectNextItems", + project_items = sgqlc.types.Field( + sgqlc.types.non_null(ProjectV2ItemConnection), + graphql_name="projectItems", args=sgqlc.types.ArgDict( ( ("include_archived", sgqlc.types.Arg(Boolean, graphql_name="includeArchived", default=True)), @@ -28982,7 +35000,7 @@ class PullRequest( ) ), ) - """List of project (beta) items associated with this pull request. + """List of project items associated with this pull request. Arguments: @@ -29139,6 +35157,11 @@ class PullRequest( title_html = sgqlc.types.Field(sgqlc.types.non_null(HTML), graphql_name="titleHTML") """Identifies the pull request title rendered to HTML.""" + total_comments_count = sgqlc.types.Field(Int, graphql_name="totalCommentsCount") + """Returns a count of how many comments this pull request has + received. + """ + viewer_can_apply_suggestion = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanApplySuggestion") """Whether or not the viewer can apply suggestion.""" @@ -29148,6 +35171,9 @@ class PullRequest( viewer_can_disable_auto_merge = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanDisableAutoMerge") """Whether or not the viewer can disable auto-merge""" + viewer_can_edit_files = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanEditFiles") + """Can the viewer edit files within this pull request.""" + viewer_can_enable_auto_merge = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanEnableAutoMerge") """Whether or not the viewer can enable auto-merge""" @@ -29156,6 +35182,12 @@ class PullRequest( merge the pull request immediately """ + viewer_can_update_branch = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanUpdateBranch") + """Whether or not the viewer can update the head ref of this PR, by + merging or rebasing the base ref. If the head ref is up to date or + unable to be updated by this user, this will return false. + """ + viewer_latest_review = sgqlc.types.Field("PullRequestReview", graphql_name="viewerLatestReview") """The latest review given from the viewer.""" @@ -29346,16 +35378,19 @@ class PullRequestReviewComment( "commit", "diff_hunk", "drafted_at", + "line", "original_commit", - "original_position", + "original_line", + "original_start_line", "outdated", "path", - "position", "pull_request", "pull_request_review", "reply_to", "resource_path", + "start_line", "state", + "subject_type", "url", ) commit = sgqlc.types.Field(Commit, graphql_name="commit") @@ -29367,11 +35402,21 @@ class PullRequestReviewComment( drafted_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="draftedAt") """Identifies when the comment was created in a draft state.""" + line = sgqlc.types.Field(Int, graphql_name="line") + """The end line number on the file to which the comment applies""" + original_commit = sgqlc.types.Field(Commit, graphql_name="originalCommit") """Identifies the original commit associated with the comment.""" - original_position = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="originalPosition") - """The original line index in the diff to which the comment applies.""" + original_line = sgqlc.types.Field(Int, graphql_name="originalLine") + """The end line number on the file to which the comment applied when + it was first created + """ + + original_start_line = sgqlc.types.Field(Int, graphql_name="originalStartLine") + """The start line number on the file to which the comment applied + when it was first created + """ outdated = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="outdated") """Identifies when the comment body is outdated""" @@ -29379,9 +35424,6 @@ class PullRequestReviewComment( path = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="path") """The path to which the comment applies.""" - position = sgqlc.types.Field(Int, graphql_name="position") - """The line index in the diff to which the comment applies.""" - pull_request = sgqlc.types.Field(sgqlc.types.non_null(PullRequest), graphql_name="pullRequest") """The pull request associated with this review comment.""" @@ -29394,9 +35436,17 @@ class PullRequestReviewComment( resource_path = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="resourcePath") """The HTTP path permalink for this review comment.""" + start_line = sgqlc.types.Field(Int, graphql_name="startLine") + """The start line number on the file to which the comment applies""" + state = sgqlc.types.Field(sgqlc.types.non_null(PullRequestReviewCommentState), graphql_name="state") """Identifies the state of the comment.""" + subject_type = sgqlc.types.Field(sgqlc.types.non_null(PullRequestReviewThreadSubjectType), graphql_name="subjectType") + """The level at which the comments in the corresponding thread are + targeted, can be a diff line or a file + """ + url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="url") """The HTTP URL permalink for this review comment.""" @@ -29420,6 +35470,7 @@ class PullRequestReviewThread(sgqlc.types.Type, Node): "resolved_by", "start_diff_side", "start_line", + "subject_type", "viewer_can_reply", "viewer_can_resolve", "viewer_can_unresolve", @@ -29495,6 +35546,99 @@ class PullRequestReviewThread(sgqlc.types.Type, Node): only) """ + subject_type = sgqlc.types.Field(sgqlc.types.non_null(PullRequestReviewThreadSubjectType), graphql_name="subjectType") + """The level at which the comments in the corresponding thread are + targeted, can be a diff line or a file + """ + + viewer_can_reply = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanReply") + """Indicates whether the current viewer can reply to this thread.""" + + viewer_can_resolve = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanResolve") + """Whether or not the viewer can resolve this thread""" + + viewer_can_unresolve = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanUnresolve") + """Whether or not the viewer can unresolve this thread""" + + +class PullRequestThread(sgqlc.types.Type, Node): + """A threaded list of comments for a given pull request.""" + + __schema__ = github_schema + __field_names__ = ( + "comments", + "diff_side", + "is_collapsed", + "is_outdated", + "is_resolved", + "line", + "pull_request", + "repository", + "resolved_by", + "start_diff_side", + "start_line", + "viewer_can_reply", + "viewer_can_resolve", + "viewer_can_unresolve", + ) + comments = sgqlc.types.Field( + sgqlc.types.non_null(PullRequestReviewCommentConnection), + graphql_name="comments", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("skip", sgqlc.types.Arg(Int, graphql_name="skip", default=None)), + ) + ), + ) + """A list of pull request comments associated with the thread. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `skip` (`Int`): Skips the first _n_ elements in the list. + """ + + diff_side = sgqlc.types.Field(sgqlc.types.non_null(DiffSide), graphql_name="diffSide") + """The side of the diff on which this thread was placed.""" + + is_collapsed = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isCollapsed") + """Whether or not the thread has been collapsed (resolved)""" + + is_outdated = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isOutdated") + """Indicates whether this thread was outdated by newer changes.""" + + is_resolved = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isResolved") + """Whether this thread has been resolved""" + + line = sgqlc.types.Field(Int, graphql_name="line") + """The line in the file to which this thread refers""" + + pull_request = sgqlc.types.Field(sgqlc.types.non_null(PullRequest), graphql_name="pullRequest") + """Identifies the pull request associated with this thread.""" + + repository = sgqlc.types.Field(sgqlc.types.non_null("Repository"), graphql_name="repository") + """Identifies the repository associated with this thread.""" + + resolved_by = sgqlc.types.Field("User", graphql_name="resolvedBy") + """The user who resolved this thread""" + + start_diff_side = sgqlc.types.Field(DiffSide, graphql_name="startDiffSide") + """The side of the diff that the first line of the thread starts on + (multi-line only) + """ + + start_line = sgqlc.types.Field(Int, graphql_name="startLine") + """The line of the first file diff in the thread.""" + viewer_can_reply = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanReply") """Indicates whether the current viewer can reply to this thread.""" @@ -29582,7 +35726,16 @@ class Ref(sgqlc.types.Type, Node): """Represents a Git reference.""" __schema__ = github_schema - __field_names__ = ("associated_pull_requests", "branch_protection_rule", "name", "prefix", "ref_update_rule", "repository", "target") + __field_names__ = ( + "associated_pull_requests", + "branch_protection_rule", + "compare", + "name", + "prefix", + "ref_update_rule", + "repository", + "target", + ) associated_pull_requests = sgqlc.types.Field( sgqlc.types.non_null(PullRequestConnection), graphql_name="associatedPullRequests", @@ -29628,6 +35781,19 @@ class Ref(sgqlc.types.Type, Node): branch_protection_rule = sgqlc.types.Field(BranchProtectionRule, graphql_name="branchProtectionRule") """Branch protection rules for this ref""" + compare = sgqlc.types.Field( + Comparison, + graphql_name="compare", + args=sgqlc.types.ArgDict((("head_ref", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="headRef", default=None)),)), + ) + """Compares the current ref as a base ref to another head ref, if the + comparison can be made. + + Arguments: + + * `head_ref` (`String!`): The head ref to compare against. + """ + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") """The ref name.""" @@ -29853,6 +36019,37 @@ class ReleaseAsset(sgqlc.types.Type, Node): """Identifies the URL of the release asset.""" +class RemovedFromMergeQueueEvent(sgqlc.types.Type, Node): + """Represents a 'removed_from_merge_queue' event on a given pull + request. + """ + + __schema__ = github_schema + __field_names__ = ("actor", "before_commit", "created_at", "enqueuer", "merge_queue", "pull_request", "reason") + actor = sgqlc.types.Field(Actor, graphql_name="actor") + """Identifies the actor who performed the event.""" + + before_commit = sgqlc.types.Field(Commit, graphql_name="beforeCommit") + """Identifies the before commit SHA for the + 'removed_from_merge_queue' event. + """ + + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + """Identifies the date and time when the object was created.""" + + enqueuer = sgqlc.types.Field("User", graphql_name="enqueuer") + """The user who removed this Pull Request from the merge queue""" + + merge_queue = sgqlc.types.Field(MergeQueue, graphql_name="mergeQueue") + """The merge queue where this pull request was removed from.""" + + pull_request = sgqlc.types.Field(PullRequest, graphql_name="pullRequest") + """PullRequest referenced by event.""" + + reason = sgqlc.types.Field(String, graphql_name="reason") + """The reason this pull request was removed from the queue.""" + + class RemovedFromProjectEvent(sgqlc.types.Type, Node): """Represents a 'removed_from_project' event on a given issue or pull request. @@ -30093,7 +36290,9 @@ class RepoRemoveTopicAuditEntry( __field_names__ = () -class Repository(sgqlc.types.Type, Node, ProjectOwner, PackageOwner, Subscribable, Starrable, UniformResourceLocatable, RepositoryInfo): +class Repository( + sgqlc.types.Type, Node, ProjectV2Recent, ProjectOwner, PackageOwner, Subscribable, Starrable, UniformResourceLocatable, RepositoryInfo +): """A repository contains the content for a project.""" __schema__ = github_schema @@ -30114,6 +36313,7 @@ class Repository(sgqlc.types.Type, Node, ProjectOwner, PackageOwner, Subscribabl "deployments", "discussion", "discussion_categories", + "discussion_category", "discussions", "disk_usage", "environment", @@ -30121,6 +36321,7 @@ class Repository(sgqlc.types.Type, Node, ProjectOwner, PackageOwner, Subscribabl "forking_allowed", "forks", "funding_links", + "has_vulnerability_alerts_enabled", "interaction_ability", "is_blank_issues_enabled", "is_disabled", @@ -30137,6 +36338,9 @@ class Repository(sgqlc.types.Type, Node, ProjectOwner, PackageOwner, Subscribabl "latest_release", "mentionable_users", "merge_commit_allowed", + "merge_commit_message", + "merge_commit_title", + "merge_queue", "milestone", "milestones", "object", @@ -30144,8 +36348,8 @@ class Repository(sgqlc.types.Type, Node, ProjectOwner, PackageOwner, Subscribabl "pinned_discussions", "pinned_issues", "primary_language", - "project_next", - "projects_next", + "project_v2", + "projects_v2", "pull_request", "pull_request_templates", "pull_requests", @@ -30155,9 +36359,12 @@ class Repository(sgqlc.types.Type, Node, ProjectOwner, PackageOwner, Subscribabl "release", "releases", "repository_topics", + "ruleset", + "rulesets", "security_policy_url", "squash_merge_allowed", - "squash_pr_title_used_as_default", + "squash_merge_commit_message", + "squash_merge_commit_title", "ssh_url", "submodules", "temp_clone_token", @@ -30168,8 +36375,10 @@ class Repository(sgqlc.types.Type, Node, ProjectOwner, PackageOwner, Subscribabl "viewer_default_merge_method", "viewer_permission", "viewer_possible_commit_emails", + "vulnerability_alert", "vulnerability_alerts", "watchers", + "web_commit_signoff_required", ) allow_update_branch = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="allowUpdateBranch") """Whether or not a pull request head branch that is behind its base @@ -30255,6 +36464,7 @@ class Repository(sgqlc.types.Type, Node, ProjectOwner, PackageOwner, Subscribabl args=sgqlc.types.ArgDict( ( ("affiliation", sgqlc.types.Arg(CollaboratorAffiliation, graphql_name="affiliation", default=None)), + ("login", sgqlc.types.Arg(String, graphql_name="login", default=None)), ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), @@ -30269,6 +36479,7 @@ class Repository(sgqlc.types.Type, Node, ProjectOwner, PackageOwner, Subscribabl * `affiliation` (`CollaboratorAffiliation`): Collaborators affiliation level with a repository. + * `login` (`String`): The login of one specific collaborator. * `query` (`String`): Filters users with query on user name and login * `after` (`String`): Returns the elements in the list that come @@ -30415,6 +36626,19 @@ class Repository(sgqlc.types.Type, Node, ProjectOwner, PackageOwner, Subscribabl are assignable by the viewer. (default: `false`) """ + discussion_category = sgqlc.types.Field( + DiscussionCategory, + graphql_name="discussionCategory", + args=sgqlc.types.ArgDict((("slug", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="slug", default=None)),)), + ) + """A discussion category by slug. + + Arguments: + + * `slug` (`String!`): The slug of the discussion category to be + returned. + """ + discussions = sgqlc.types.Field( sgqlc.types.non_null(DiscussionConnection), graphql_name="discussions", @@ -30425,6 +36649,7 @@ class Repository(sgqlc.types.Type, Node, ProjectOwner, PackageOwner, Subscribabl ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), ("category_id", sgqlc.types.Arg(ID, graphql_name="categoryId", default=None)), + ("states", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(DiscussionState)), graphql_name="states", default=())), ( "order_by", sgqlc.types.Arg(DiscussionOrder, graphql_name="orderBy", default={"field": "UPDATED_AT", "direction": "DESC"}), @@ -30444,6 +36669,8 @@ class Repository(sgqlc.types.Type, Node, ProjectOwner, PackageOwner, Subscribabl * `last` (`Int`): Returns the last _n_ elements from the list. * `category_id` (`ID`): Only include discussions that belong to the category with this ID. (default: `null`) + * `states` (`[DiscussionState!]`): A list of states to filter the + discussions by. (default: `[]`) * `order_by` (`DiscussionOrder`): Ordering options for discussions returned from the connection. (default: `{field: UPDATED_AT, direction: DESC}`) @@ -30546,6 +36773,9 @@ class Repository(sgqlc.types.Type, Node, ProjectOwner, PackageOwner, Subscribabl ) """The funding links for this repository""" + has_vulnerability_alerts_enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasVulnerabilityAlertsEnabled") + """Whether vulnerability alerts are enabled for the repository.""" + interaction_ability = sgqlc.types.Field(RepositoryInteractionAbility, graphql_name="interactionAbility") """The interaction ability settings for this repository.""" @@ -30734,6 +36964,30 @@ class Repository(sgqlc.types.Type, Node, ProjectOwner, PackageOwner, Subscribabl repository. """ + merge_commit_message = sgqlc.types.Field(sgqlc.types.non_null(MergeCommitMessage), graphql_name="mergeCommitMessage") + """How the default commit message will be generated when merging a + pull request. + """ + + merge_commit_title = sgqlc.types.Field(sgqlc.types.non_null(MergeCommitTitle), graphql_name="mergeCommitTitle") + """How the default commit title will be generated when merging a pull + request. + """ + + merge_queue = sgqlc.types.Field( + MergeQueue, + graphql_name="mergeQueue", + args=sgqlc.types.ArgDict((("branch", sgqlc.types.Arg(String, graphql_name="branch", default=None)),)), + ) + """The merge queue for a specified branch, otherwise the default + branch if not provided. + + Arguments: + + * `branch` (`String`): The name of the branch to get the merge + queue for. Case sensitive. + """ + milestone = sgqlc.types.Field( Milestone, graphql_name="milestone", @@ -30850,22 +37104,22 @@ class Repository(sgqlc.types.Type, Node, ProjectOwner, PackageOwner, Subscribabl primary_language = sgqlc.types.Field(Language, graphql_name="primaryLanguage") """The primary language of the repository's code.""" - project_next = sgqlc.types.Field( - ProjectNext, - graphql_name="projectNext", + project_v2 = sgqlc.types.Field( + ProjectV2, + graphql_name="projectV2", args=sgqlc.types.ArgDict((("number", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="number", default=None)),)), ) - """Finds and returns the Project (beta) according to the provided - Project (beta) number. + """Finds and returns the Project according to the provided Project + number. Arguments: - * `number` (`Int!`): The ProjectNext number. + * `number` (`Int!`): The Project number. """ - projects_next = sgqlc.types.Field( - sgqlc.types.non_null(ProjectNextConnection), - graphql_name="projectsNext", + projects_v2 = sgqlc.types.Field( + sgqlc.types.non_null(ProjectV2Connection), + graphql_name="projectsV2", args=sgqlc.types.ArgDict( ( ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), @@ -30873,11 +37127,11 @@ class Repository(sgqlc.types.Type, Node, ProjectOwner, PackageOwner, Subscribabl ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("sort_by", sgqlc.types.Arg(ProjectNextOrderField, graphql_name="sortBy", default="TITLE")), + ("order_by", sgqlc.types.Arg(ProjectV2Order, graphql_name="orderBy", default={"field": "NUMBER", "direction": "DESC"})), ) ), ) - """List of projects (beta) linked to this repository. + """List of projects linked to this repository. Arguments: @@ -30887,10 +37141,9 @@ class Repository(sgqlc.types.Type, Node, ProjectOwner, PackageOwner, Subscribabl before the specified cursor. * `first` (`Int`): Returns the first _n_ elements from the list. * `last` (`Int`): Returns the last _n_ elements from the list. - * `query` (`String`): A project (beta) to search for linked to the - repo. - * `sort_by` (`ProjectNextOrderField`): How to order the returned - project (beta) objects. (default: `TITLE`) + * `query` (`String`): A project to search for linked to the repo. + * `order_by` (`ProjectV2Order`): How to order the returned + projects. (default: `{field: NUMBER, direction: DESC}`) """ pull_request = sgqlc.types.Field( @@ -31075,15 +37328,59 @@ class Repository(sgqlc.types.Type, Node, ProjectOwner, PackageOwner, Subscribabl * `last` (`Int`): Returns the last _n_ elements from the list. """ + ruleset = sgqlc.types.Field( + "RepositoryRuleset", + graphql_name="ruleset", + args=sgqlc.types.ArgDict((("database_id", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="databaseId", default=None)),)), + ) + """Returns a single ruleset from the current repository by ID. + + Arguments: + + * `database_id` (`Int!`): The ID of the ruleset to be returned. + """ + + rulesets = sgqlc.types.Field( + RepositoryRulesetConnection, + graphql_name="rulesets", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("include_parents", sgqlc.types.Arg(Boolean, graphql_name="includeParents", default=False)), + ) + ), + ) + """A list of rulesets for this repository. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `include_parents` (`Boolean`): Return rulesets configured at + higher levels that apply to this repository (default: `false`) + """ + security_policy_url = sgqlc.types.Field(URI, graphql_name="securityPolicyUrl") """The security policy URL.""" squash_merge_allowed = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="squashMergeAllowed") """Whether or not squash-merging is enabled on this repository.""" - squash_pr_title_used_as_default = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="squashPrTitleUsedAsDefault") - """Whether a squash merge commit can use the pull request title as - default. + squash_merge_commit_message = sgqlc.types.Field(sgqlc.types.non_null(SquashMergeCommitMessage), graphql_name="squashMergeCommitMessage") + """How the default commit message will be generated when squash + merging a pull request. + """ + + squash_merge_commit_title = sgqlc.types.Field(sgqlc.types.non_null(SquashMergeCommitTitle), graphql_name="squashMergeCommitTitle") + """How the default commit title will be generated when squash merging + a pull request. """ ssh_url = sgqlc.types.Field(sgqlc.types.non_null(GitSSHRemote), graphql_name="sshUrl") @@ -31148,6 +37445,20 @@ class Repository(sgqlc.types.Type, Node, ProjectOwner, PackageOwner, Subscribabl ) """A list of emails this viewer can commit with.""" + vulnerability_alert = sgqlc.types.Field( + "RepositoryVulnerabilityAlert", + graphql_name="vulnerabilityAlert", + args=sgqlc.types.ArgDict((("number", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="number", default=None)),)), + ) + """Returns a single vulnerability alert from the current repository + by number. + + Arguments: + + * `number` (`Int!`): The number for the vulnerability alert to be + returned. + """ + vulnerability_alerts = sgqlc.types.Field( RepositoryVulnerabilityAlertConnection, graphql_name="vulnerabilityAlerts", @@ -31163,6 +37474,14 @@ class Repository(sgqlc.types.Type, Node, ProjectOwner, PackageOwner, Subscribabl sgqlc.types.list_of(sgqlc.types.non_null(RepositoryVulnerabilityAlertState)), graphql_name="states", default=None ), ), + ( + "dependency_scopes", + sgqlc.types.Arg( + sgqlc.types.list_of(sgqlc.types.non_null(RepositoryVulnerabilityAlertDependencyScope)), + graphql_name="dependencyScopes", + default=None, + ), + ), ) ), ) @@ -31178,6 +37497,9 @@ class Repository(sgqlc.types.Type, Node, ProjectOwner, PackageOwner, Subscribabl * `last` (`Int`): Returns the last _n_ elements from the list. * `states` (`[RepositoryVulnerabilityAlertState!]`): Filter by the state of the alert + * `dependency_scopes` + (`[RepositoryVulnerabilityAlertDependencyScope!]`): Filter by + the scope of the alert's dependency """ watchers = sgqlc.types.Field( @@ -31204,6 +37526,11 @@ class Repository(sgqlc.types.Type, Node, ProjectOwner, PackageOwner, Subscribabl * `last` (`Int`): Returns the last _n_ elements from the list. """ + web_commit_signoff_required = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="webCommitSignoffRequired") + """Whether contributors are required to sign off on web-based commits + in this repository. + """ + class RepositoryInvitation(sgqlc.types.Type, Node): """An invitation for a user to be added to a repository.""" @@ -31230,12 +37557,135 @@ class RepositoryInvitation(sgqlc.types.Type, Node): class RepositoryMigration(sgqlc.types.Type, Node, Migration): - """An Octoshift repository migration.""" + """A GitHub Enterprise Importer (GEI) repository migration.""" __schema__ = github_schema __field_names__ = () +class RepositoryRule(sgqlc.types.Type, Node): + """A repository rule.""" + + __schema__ = github_schema + __field_names__ = ("parameters", "type") + parameters = sgqlc.types.Field("RuleParameters", graphql_name="parameters") + """The parameters for this rule.""" + + type = sgqlc.types.Field(sgqlc.types.non_null(RepositoryRuleType), graphql_name="type") + """The type of rule.""" + + +class RepositoryRuleset(sgqlc.types.Type, Node): + """A repository ruleset.""" + + __schema__ = github_schema + __field_names__ = ( + "bypass_actors", + "bypass_mode", + "conditions", + "created_at", + "database_id", + "enforcement", + "name", + "rules", + "source", + "target", + "updated_at", + ) + bypass_actors = sgqlc.types.Field( + RepositoryRulesetBypassActorConnection, + graphql_name="bypassActors", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), + ) + """The actors that can bypass this ruleset + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + """ + + bypass_mode = sgqlc.types.Field(sgqlc.types.non_null(RuleBypassMode), graphql_name="bypassMode") + """The bypass mode of this ruleset""" + + conditions = sgqlc.types.Field(sgqlc.types.non_null(RepositoryRuleConditions), graphql_name="conditions") + """The set of conditions that must evaluate to true for this ruleset + to apply + """ + + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + """Identifies the date and time when the object was created.""" + + database_id = sgqlc.types.Field(Int, graphql_name="databaseId") + """Identifies the primary key from the database.""" + + enforcement = sgqlc.types.Field(sgqlc.types.non_null(RuleEnforcement), graphql_name="enforcement") + """The enforcement level of this ruleset""" + + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + """Name of the ruleset.""" + + rules = sgqlc.types.Field( + RepositoryRuleConnection, + graphql_name="rules", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("type", sgqlc.types.Arg(RepositoryRuleType, graphql_name="type", default=None)), + ) + ), + ) + """List of rules. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `type` (`RepositoryRuleType`): The type of rule. + """ + + source = sgqlc.types.Field(sgqlc.types.non_null("RuleSource"), graphql_name="source") + """Source of ruleset.""" + + target = sgqlc.types.Field(RepositoryRulesetTarget, graphql_name="target") + """Target of the ruleset.""" + + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + """Identifies the date and time when the object was last updated.""" + + +class RepositoryRulesetBypassActor(sgqlc.types.Type, Node): + """A team or app that has the ability to bypass a rules defined on a + ruleset + """ + + __schema__ = github_schema + __field_names__ = ("actor", "repository_ruleset") + actor = sgqlc.types.Field("BypassActor", graphql_name="actor") + """The actor that can bypass rules.""" + + repository_ruleset = sgqlc.types.Field(RepositoryRuleset, graphql_name="repositoryRuleset") + """Identifies the ruleset associated with the allowed actor""" + + class RepositoryTopic(sgqlc.types.Type, Node, UniformResourceLocatable): """A repository-topic connects a repository to a topic.""" @@ -31266,12 +37716,14 @@ class RepositoryVulnerabilityAlert(sgqlc.types.Type, Node, RepositoryNode): __schema__ = github_schema __field_names__ = ( + "auto_dismissed_at", "created_at", "dependabot_update", + "dependency_scope", + "dismiss_comment", "dismiss_reason", "dismissed_at", "dismisser", - "fix_reason", "fixed_at", "number", "security_advisory", @@ -31281,12 +37733,21 @@ class RepositoryVulnerabilityAlert(sgqlc.types.Type, Node, RepositoryNode): "vulnerable_manifest_path", "vulnerable_requirements", ) + auto_dismissed_at = sgqlc.types.Field(DateTime, graphql_name="autoDismissedAt") + """When was the alert auto-dismissed?""" + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") """When was the alert created?""" dependabot_update = sgqlc.types.Field(DependabotUpdate, graphql_name="dependabotUpdate") """The associated Dependabot update""" + dependency_scope = sgqlc.types.Field(RepositoryVulnerabilityAlertDependencyScope, graphql_name="dependencyScope") + """The scope of an alert's dependency""" + + dismiss_comment = sgqlc.types.Field(String, graphql_name="dismissComment") + """Comment explaining the reason the alert was dismissed""" + dismiss_reason = sgqlc.types.Field(String, graphql_name="dismissReason") """The reason the alert was dismissed""" @@ -31296,9 +37757,6 @@ class RepositoryVulnerabilityAlert(sgqlc.types.Type, Node, RepositoryNode): dismisser = sgqlc.types.Field("User", graphql_name="dismisser") """The user who dismissed the alert""" - fix_reason = sgqlc.types.Field(String, graphql_name="fixReason") - """The reason the alert was marked as fixed.""" - fixed_at = sgqlc.types.Field(DateTime, graphql_name="fixedAt") """When was the alert fixed?""" @@ -31655,7 +38113,7 @@ class SponsorsActivity(sgqlc.types.Type, Node): """An event related to sponsorship activity.""" __schema__ = github_schema - __field_names__ = ("action", "previous_sponsors_tier", "sponsor", "sponsorable", "sponsors_tier", "timestamp") + __field_names__ = ("action", "previous_sponsors_tier", "sponsor", "sponsorable", "sponsors_tier", "timestamp", "via_bulk_sponsorship") action = sgqlc.types.Field(sgqlc.types.non_null(SponsorsActivityAction), graphql_name="action") """What action this activity indicates took place.""" @@ -31676,6 +38134,11 @@ class SponsorsActivity(sgqlc.types.Type, Node): timestamp = sgqlc.types.Field(DateTime, graphql_name="timestamp") """The timestamp of this event.""" + via_bulk_sponsorship = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viaBulkSponsorship") + """Was this sponsorship made alongside other sponsorships at the same + time from the same sponsor? + """ + class SponsorsListing(sgqlc.types.Type, Node): """A GitHub Sponsors listing.""" @@ -31683,25 +38146,95 @@ class SponsorsListing(sgqlc.types.Type, Node): __schema__ = github_schema __field_names__ = ( "active_goal", + "active_stripe_connect_account", + "billing_country_or_region", + "contact_email_address", "created_at", + "dashboard_resource_path", + "dashboard_url", + "featured_items", + "fiscal_host", "full_description", "full_description_html", "is_public", "name", "next_payout_date", + "residence_country_or_region", + "resource_path", "short_description", "slug", "sponsorable", "tiers", + "url", ) active_goal = sgqlc.types.Field(SponsorsGoal, graphql_name="activeGoal") """The current goal the maintainer is trying to reach with GitHub Sponsors, if any. """ + active_stripe_connect_account = sgqlc.types.Field(StripeConnectAccount, graphql_name="activeStripeConnectAccount") + """The Stripe Connect account currently in use for payouts for this + Sponsors listing, if any. Will only return a value when queried by + the maintainer themselves, or by an admin of the sponsorable + organization. + """ + + billing_country_or_region = sgqlc.types.Field(String, graphql_name="billingCountryOrRegion") + """The name of the country or region with the maintainer's bank + account or fiscal host. Will only return a value when queried by + the maintainer themselves, or by an admin of the sponsorable + organization. + """ + + contact_email_address = sgqlc.types.Field(String, graphql_name="contactEmailAddress") + """The email address used by GitHub to contact the sponsorable about + their GitHub Sponsors profile. Will only return a value when + queried by the maintainer themselves, or by an admin of the + sponsorable organization. + """ + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") """Identifies the date and time when the object was created.""" + dashboard_resource_path = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="dashboardResourcePath") + """The HTTP path for the Sponsors dashboard for this Sponsors + listing. + """ + + dashboard_url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="dashboardUrl") + """The HTTP URL for the Sponsors dashboard for this Sponsors listing.""" + + featured_items = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SponsorsListingFeaturedItem"))), + graphql_name="featuredItems", + args=sgqlc.types.ArgDict( + ( + ( + "featureable_types", + sgqlc.types.Arg( + sgqlc.types.list_of(sgqlc.types.non_null(SponsorsListingFeaturedItemFeatureableType)), + graphql_name="featureableTypes", + default=("REPOSITORY", "USER"), + ), + ), + ) + ), + ) + """The records featured on the GitHub Sponsors profile. + + Arguments: + + * `featureable_types` + (`[SponsorsListingFeaturedItemFeatureableType!]`): The types of + featured items to return. (default: `[REPOSITORY, USER]`) + """ + + fiscal_host = sgqlc.types.Field(Organization, graphql_name="fiscalHost") + """The fiscal host used for payments, if any. Will only return a + value when queried by the maintainer themselves, or by an admin of + the sponsorable organization. + """ + full_description = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="fullDescription") """The full description of the listing.""" @@ -31719,6 +38252,15 @@ class SponsorsListing(sgqlc.types.Type, Node): payout. """ + residence_country_or_region = sgqlc.types.Field(String, graphql_name="residenceCountryOrRegion") + """The name of the country or region where the maintainer resides. + Will only return a value when queried by the maintainer + themselves, or by an admin of the sponsorable organization. + """ + + resource_path = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="resourcePath") + """The HTTP path for this Sponsors listing.""" + short_description = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="shortDescription") """The short description of the listing.""" @@ -31745,10 +38287,11 @@ class SponsorsListing(sgqlc.types.Type, Node): SponsorsTierOrder, graphql_name="orderBy", default={"field": "MONTHLY_PRICE_IN_CENTS", "direction": "ASC"} ), ), + ("include_unpublished", sgqlc.types.Arg(Boolean, graphql_name="includeUnpublished", default=False)), ) ), ) - """The published tiers for this GitHub Sponsors listing. + """The tiers for this GitHub Sponsors profile. Arguments: @@ -31761,8 +38304,47 @@ class SponsorsListing(sgqlc.types.Type, Node): * `order_by` (`SponsorsTierOrder`): Ordering options for Sponsors tiers returned from the connection. (default: `{field: MONTHLY_PRICE_IN_CENTS, direction: ASC}`) + * `include_unpublished` (`Boolean`): Whether to include tiers that + aren't published. Only admins of the Sponsors listing can see + draft tiers. Only admins of the Sponsors listing and viewers who + are currently sponsoring on a retired tier can see those retired + tiers. Defaults to including only published tiers, which are + visible to anyone who can see the GitHub Sponsors profile. + (default: `false`) + """ + + url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="url") + """The HTTP URL for this Sponsors listing.""" + + +class SponsorsListingFeaturedItem(sgqlc.types.Type, Node): + """A record that is promoted on a GitHub Sponsors profile.""" + + __schema__ = github_schema + __field_names__ = ("created_at", "description", "featureable", "position", "sponsors_listing", "updated_at") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + """Identifies the date and time when the object was created.""" + + description = sgqlc.types.Field(String, graphql_name="description") + """Will either be a description from the sponsorable maintainer about + why they featured this item, or the item's description itself, + such as a user's bio from their GitHub profile page. """ + featureable = sgqlc.types.Field(sgqlc.types.non_null("SponsorsListingFeatureableItem"), graphql_name="featureable") + """The record that is featured on the GitHub Sponsors profile.""" + + position = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="position") + """The position of this featured item on the GitHub Sponsors profile + with a lower position indicating higher precedence. Starts at 1. + """ + + sponsors_listing = sgqlc.types.Field(sgqlc.types.non_null(SponsorsListing), graphql_name="sponsorsListing") + """The GitHub Sponsors profile that features this record.""" + + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + """Identifies the date and time when the object was last updated.""" + class SponsorsTier(sgqlc.types.Type, Node): """A GitHub Sponsors tier associated with a GitHub Sponsors listing.""" @@ -31834,6 +38416,7 @@ class Sponsorship(sgqlc.types.Type, Node): __schema__ = github_schema __field_names__ = ( "created_at", + "is_active", "is_one_time_payment", "is_sponsor_opted_into_email", "privacy_level", @@ -31845,13 +38428,19 @@ class Sponsorship(sgqlc.types.Type, Node): created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") """Identifies the date and time when the object was created.""" + is_active = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isActive") + """Whether the sponsorship is active. False implies the sponsor is a + past sponsor of the maintainer, while true implies they are a + current sponsor. + """ + is_one_time_payment = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isOneTimePayment") """Whether this sponsorship represents a one-time payment versus a recurring sponsorship. """ is_sponsor_opted_into_email = sgqlc.types.Field(Boolean, graphql_name="isSponsorOptedIntoEmail") - """Check if the sponsor has chosen to receive sponsorship update + """Whether the sponsor has chosen to receive sponsorship update emails sent from the sponsorable. Only returns a non-null value when the viewer has permission to know this. """ @@ -31882,7 +38471,10 @@ class SponsorshipNewsletter(sgqlc.types.Type, Node): """ __schema__ = github_schema - __field_names__ = ("body", "created_at", "is_published", "sponsorable", "subject", "updated_at") + __field_names__ = ("author", "body", "created_at", "is_published", "sponsorable", "subject", "updated_at") + author = sgqlc.types.Field("User", graphql_name="author") + """The author of the newsletter.""" + body = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="body") """The contents of the newsletter, the message the sponsorable wanted to give. @@ -31904,6 +38496,15 @@ class SponsorshipNewsletter(sgqlc.types.Type, Node): """Identifies the date and time when the object was last updated.""" +class SshSignature(sgqlc.types.Type, GitSignature): + """Represents an SSH signature on a Commit or Tag.""" + + __schema__ = github_schema + __field_names__ = ("key_fingerprint",) + key_fingerprint = sgqlc.types.Field(String, graphql_name="keyFingerprint") + """Hex-encoded fingerprint of the key that signed this object.""" + + class Status(sgqlc.types.Type, Node): """Represents a commit status.""" @@ -32090,9 +38691,12 @@ class Team(sgqlc.types.Type, Node, Subscribable, MemberStatusable): "name", "new_team_resource_path", "new_team_url", + "notification_setting", "organization", "parent_team", "privacy", + "project_v2", + "projects_v2", "repositories", "repositories_resource_path", "repositories_url", @@ -32311,6 +38915,9 @@ class Team(sgqlc.types.Type, Node, Subscribable, MemberStatusable): new_team_url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="newTeamUrl") """The HTTP URL creating a new team""" + notification_setting = sgqlc.types.Field(sgqlc.types.non_null(TeamNotificationSetting), graphql_name="notificationSetting") + """The notification setting that the team has set.""" + organization = sgqlc.types.Field(sgqlc.types.non_null(Organization), graphql_name="organization") """The organization that owns this team.""" @@ -32320,6 +38927,52 @@ class Team(sgqlc.types.Type, Node, Subscribable, MemberStatusable): privacy = sgqlc.types.Field(sgqlc.types.non_null(TeamPrivacy), graphql_name="privacy") """The level of privacy the team has.""" + project_v2 = sgqlc.types.Field( + ProjectV2, + graphql_name="projectV2", + args=sgqlc.types.ArgDict((("number", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="number", default=None)),)), + ) + """Finds and returns the project according to the provided project + number. + + Arguments: + + * `number` (`Int!`): The Project number. + """ + + projects_v2 = sgqlc.types.Field( + sgqlc.types.non_null(ProjectV2Connection), + graphql_name="projectsV2", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("order_by", sgqlc.types.Arg(ProjectV2Order, graphql_name="orderBy", default={"field": "NUMBER", "direction": "DESC"})), + ("filter_by", sgqlc.types.Arg(ProjectV2Filters, graphql_name="filterBy", default={})), + ("query", sgqlc.types.Arg(String, graphql_name="query", default="")), + ) + ), + ) + """List of projects this team has collaborator access to. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `order_by` (`ProjectV2Order`): How to order the returned + projects. (default: `{field: NUMBER, direction: DESC}`) + * `filter_by` (`ProjectV2Filters`): Filtering options for projects + returned from this connection (default: `{}`) + * `query` (`String`): The query to search projects by. (default: + `""`) + """ + repositories = sgqlc.types.Field( sgqlc.types.non_null(TeamRepositoryConnection), graphql_name="repositories", @@ -32344,7 +38997,8 @@ class Team(sgqlc.types.Type, Node, Subscribable, MemberStatusable): before the specified cursor. * `first` (`Int`): Returns the first _n_ elements from the list. * `last` (`Int`): Returns the last _n_ elements from the list. - * `query` (`String`): The search string to look for. + * `query` (`String`): The search string to look for. Repositories + will be returned where the name contains your search string. * `order_by` (`TeamRepositoryOrder`): Order for the connection. """ @@ -32774,7 +39428,8 @@ class User( Actor, PackageOwner, ProjectOwner, - ProjectNextOwner, + ProjectV2Owner, + ProjectV2Recent, RepositoryDiscussionAuthor, RepositoryDiscussionCommentAuthor, RepositoryOwner, @@ -32818,10 +39473,12 @@ class User( "organization", "organization_verified_domain_emails", "organizations", + "pronouns", "public_keys", "pull_requests", "repositories_contributed_to", "saved_replies", + "social_accounts", "starred_repositories", "status", "top_repositories", @@ -33058,7 +39715,7 @@ class User( is_following_viewer = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isFollowingViewer") """Whether or not this user is following the viewer. Inverse of - viewer_is_following + viewerIsFollowing """ is_git_hub_star = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isGitHubStar") @@ -33167,6 +39824,7 @@ class User( graphql_name="organizations", args=sgqlc.types.ArgDict( ( + ("order_by", sgqlc.types.Arg(OrganizationOrder, graphql_name="orderBy", default=None)), ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), @@ -33178,6 +39836,8 @@ class User( Arguments: + * `order_by` (`OrganizationOrder`): Ordering options for the + User's organizations. (default: `null`) * `after` (`String`): Returns the elements in the list that come after the specified cursor. * `before` (`String`): Returns the elements in the list that come @@ -33186,6 +39846,9 @@ class User( * `last` (`Int`): Returns the last _n_ elements from the list. """ + pronouns = sgqlc.types.Field(String, graphql_name="pronouns") + """The user's profile pronouns""" + public_keys = sgqlc.types.Field( sgqlc.types.non_null(PublicKeyConnection), graphql_name="publicKeys", @@ -33325,6 +39988,31 @@ class User( by. (default: `{field: UPDATED_AT, direction: DESC}`) """ + social_accounts = sgqlc.types.Field( + sgqlc.types.non_null(SocialAccountConnection), + graphql_name="socialAccounts", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ) + ), + ) + """The user's social media accounts, ordered as they appear on the + user's profile. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + """ + starred_repositories = sgqlc.types.Field( sgqlc.types.non_null(StarredRepositoryConnection), graphql_name="starredRepositories", @@ -33399,7 +40087,7 @@ class User( viewer_is_following = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerIsFollowing") """Whether or not this user is followed by the viewer. Inverse of - is_following_viewer. + isFollowingViewer. """ watching = sgqlc.types.Field( @@ -33626,13 +40314,13 @@ class ViewerHovercardContext(sgqlc.types.Type, HovercardContext): """Identifies the user who is related to this context.""" -class Workflow(sgqlc.types.Type, Node): +class Workflow(sgqlc.types.Type, Node, UniformResourceLocatable): """A workflow contains meta information about an Actions workflow file. """ __schema__ = github_schema - __field_names__ = ("created_at", "database_id", "name", "updated_at") + __field_names__ = ("created_at", "database_id", "name", "runs", "state", "updated_at") created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") """Identifies the date and time when the object was created.""" @@ -33642,11 +40330,44 @@ class Workflow(sgqlc.types.Type, Node): name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") """The name of the workflow.""" + runs = sgqlc.types.Field( + sgqlc.types.non_null(WorkflowRunConnection), + graphql_name="runs", + args=sgqlc.types.ArgDict( + ( + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ( + "order_by", + sgqlc.types.Arg(WorkflowRunOrder, graphql_name="orderBy", default={"field": "CREATED_AT", "direction": "DESC"}), + ), + ) + ), + ) + """The runs of the workflow. + + Arguments: + + * `after` (`String`): Returns the elements in the list that come + after the specified cursor. + * `before` (`String`): Returns the elements in the list that come + before the specified cursor. + * `first` (`Int`): Returns the first _n_ elements from the list. + * `last` (`Int`): Returns the last _n_ elements from the list. + * `order_by` (`WorkflowRunOrder`): Ordering options for the + connection (default: `{field: CREATED_AT, direction: DESC}`) + """ + + state = sgqlc.types.Field(sgqlc.types.non_null(WorkflowState), graphql_name="state") + """The state of the workflow.""" + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") """Identifies the date and time when the object was last updated.""" -class WorkflowRun(sgqlc.types.Type, Node): +class WorkflowRun(sgqlc.types.Type, Node, UniformResourceLocatable): """A workflow run.""" __schema__ = github_schema @@ -33655,11 +40376,11 @@ class WorkflowRun(sgqlc.types.Type, Node): "created_at", "database_id", "deployment_reviews", + "event", + "file", "pending_deployment_requests", - "resource_path", "run_number", "updated_at", - "url", "workflow", ) check_suite = sgqlc.types.Field(sgqlc.types.non_null(CheckSuite), graphql_name="checkSuite") @@ -33695,6 +40416,12 @@ class WorkflowRun(sgqlc.types.Type, Node): * `last` (`Int`): Returns the last _n_ elements from the list. """ + event = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="event") + """The event that triggered the workflow run""" + + file = sgqlc.types.Field("WorkflowRunFile", graphql_name="file") + """The workflow file""" + pending_deployment_requests = sgqlc.types.Field( sgqlc.types.non_null(DeploymentRequestConnection), graphql_name="pendingDeploymentRequests", @@ -33720,9 +40447,6 @@ class WorkflowRun(sgqlc.types.Type, Node): * `last` (`Int`): Returns the last _n_ elements from the list. """ - resource_path = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="resourcePath") - """The HTTP path for this workflow run""" - run_number = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="runNumber") """A number that uniquely identifies this workflow run in its parent workflow. @@ -33731,13 +40455,40 @@ class WorkflowRun(sgqlc.types.Type, Node): updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") """Identifies the date and time when the object was last updated.""" - url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="url") - """The HTTP URL for this workflow run""" - workflow = sgqlc.types.Field(sgqlc.types.non_null(Workflow), graphql_name="workflow") """The workflow executed in this workflow run.""" +class WorkflowRunFile(sgqlc.types.Type, UniformResourceLocatable, Node): + """An executed workflow file for a workflow run.""" + + __schema__ = github_schema + __field_names__ = ("path", "repository_file_url", "repository_name", "run", "viewer_can_push_repository", "viewer_can_read_repository") + path = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="path") + """The path of the workflow file relative to its repository.""" + + repository_file_url = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="repositoryFileUrl") + """The direct link to the file in the repository which stores the + workflow file. + """ + + repository_name = sgqlc.types.Field(sgqlc.types.non_null(URI), graphql_name="repositoryName") + """The repository name and owner which stores the workflow file.""" + + run = sgqlc.types.Field(sgqlc.types.non_null(WorkflowRun), graphql_name="run") + """The parent workflow run execution for this file.""" + + viewer_can_push_repository = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanPushRepository") + """If the viewer has permissions to push to the repository which + stores the workflow. + """ + + viewer_can_read_repository = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="viewerCanReadRepository") + """If the viewer has permissions to read the repository which stores + the workflow. + """ + + ######################################################################## # Unions ######################################################################## @@ -33762,6 +40513,22 @@ class BranchActorAllowanceActor(sgqlc.types.Union): __types__ = (App, Team, User) +class BypassActor(sgqlc.types.Union): + """Types that can represent a repository ruleset bypass actor.""" + + __schema__ = github_schema + __types__ = (App, Team) + + +class Claimable(sgqlc.types.Union): + """An object which can have its data claimed or claim data from + another. + """ + + __schema__ = github_schema + __types__ = (Mannequin, User) + + class Closer(sgqlc.types.Union): """The object which triggered a `ClosedEvent`.""" @@ -33976,6 +40743,13 @@ class OrganizationAuditEntry(sgqlc.types.Union): ) +class OrganizationOrUser(sgqlc.types.Union): + """Used for argument of CreateProjectV2 mutation.""" + + __schema__ = github_schema + __types__ = (Organization, User) + + class PermissionGranter(sgqlc.types.Union): """Types that can grant permissions on a repository to a user""" @@ -33997,13 +40771,46 @@ class ProjectCardItem(sgqlc.types.Union): __types__ = (Issue, PullRequest) -class ProjectNextItemContent(sgqlc.types.Union): +class ProjectV2Actor(sgqlc.types.Union): + """Possible collaborators for a project.""" + + __schema__ = github_schema + __types__ = (Team, User) + + +class ProjectV2FieldConfiguration(sgqlc.types.Union): + """Configurations for project fields.""" + + __schema__ = github_schema + __types__ = (ProjectV2Field, ProjectV2IterationField, ProjectV2SingleSelectField) + + +class ProjectV2ItemContent(sgqlc.types.Union): """Types that can be inside Project Items.""" __schema__ = github_schema __types__ = (DraftIssue, Issue, PullRequest) +class ProjectV2ItemFieldValue(sgqlc.types.Union): + """Project field values""" + + __schema__ = github_schema + __types__ = ( + ProjectV2ItemFieldDateValue, + ProjectV2ItemFieldIterationValue, + ProjectV2ItemFieldLabelValue, + ProjectV2ItemFieldMilestoneValue, + ProjectV2ItemFieldNumberValue, + ProjectV2ItemFieldPullRequestValue, + ProjectV2ItemFieldRepositoryValue, + ProjectV2ItemFieldReviewerValue, + ProjectV2ItemFieldSingleSelectValue, + ProjectV2ItemFieldTextValue, + ProjectV2ItemFieldUserValue, + ) + + class PullRequestTimelineItem(sgqlc.types.Union): """An item in a pull request timeline""" @@ -34050,6 +40857,7 @@ class PullRequestTimelineItems(sgqlc.types.Union): __schema__ = github_schema __types__ = ( + AddedToMergeQueueEvent, AddedToProjectEvent, AssignedEvent, AutoMergeDisabledEvent, @@ -34091,6 +40899,7 @@ class PullRequestTimelineItems(sgqlc.types.Union): PullRequestRevisionMarker, ReadyForReviewEvent, ReferencedEvent, + RemovedFromMergeQueueEvent, RemovedFromProjectEvent, RenamedTitleEvent, ReopenedEvent, @@ -34151,6 +40960,30 @@ class ReviewDismissalAllowanceActor(sgqlc.types.Union): __types__ = (App, Team, User) +class RuleParameters(sgqlc.types.Union): + """Types which can be parameters for `RepositoryRule` objects.""" + + __schema__ = github_schema + __types__ = ( + BranchNamePatternParameters, + CommitAuthorEmailPatternParameters, + CommitMessagePatternParameters, + CommitterEmailPatternParameters, + PullRequestParameters, + RequiredDeploymentsParameters, + RequiredStatusChecksParameters, + TagNamePatternParameters, + UpdateParameters, + ) + + +class RuleSource(sgqlc.types.Union): + """Types which can have `RepositoryRule` objects.""" + + __schema__ = github_schema + __types__ = (Organization, Repository) + + class SearchResultItem(sgqlc.types.Union): """The results of a search.""" @@ -34172,6 +41005,13 @@ class SponsorableItem(sgqlc.types.Union): __types__ = (Organization, User) +class SponsorsListingFeatureableItem(sgqlc.types.Union): + """A record that can be featured on a GitHub Sponsors profile.""" + + __schema__ = github_schema + __types__ = (Repository, User) + + class StatusCheckRollupContext(sgqlc.types.Union): """Types that can be inside a StatusCheckRollup context.""" diff --git a/airbyte-integrations/connectors/source-github/source_github/graphql.py b/airbyte-integrations/connectors/source-github/source_github/graphql.py index 312972ca9f43..603e58f0182c 100644 --- a/airbyte-integrations/connectors/source-github/source_github/graphql.py +++ b/airbyte-integrations/connectors/source-github/source_github/graphql.py @@ -61,6 +61,40 @@ def get_query_pull_requests(owner, name, first, after, direction): return str(op) +def get_query_projectsV2(owner, name, first, after, direction): + kwargs = {"first": first, "order_by": {"field": "UPDATED_AT", "direction": direction}} + if after: + kwargs["after"] = after + + op = sgqlc.operation.Operation(_schema_root.query_type) + repository = op.repository(owner=owner, name=name) + repository.name() + repository.owner.login() + projects_v2 = repository.projects_v2(**kwargs) + projects_v2.nodes.__fields__( + closed=True, + created_at="created_at", + closed_at="closed_at", + updated_at="updated_at", + creator="creator", + id="node_id", + database_id="id", + number=True, + public=True, + readme="readme", + short_description="short_description", + template=True, + title="title", + url="url", + viewer_can_close=True, + viewer_can_reopen=True, + viewer_can_update=True, + ) + projects_v2.nodes.owner.__fields__(id="id") + projects_v2.page_info.__fields__(has_next_page=True, end_cursor=True) + return str(op) + + def get_query_reviews(owner, name, first, after, number=None): op = sgqlc.operation.Operation(_schema_root.query_type) repository = op.repository(owner=owner, name=name) diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/contributor_activity.json b/airbyte-integrations/connectors/source-github/source_github/schemas/contributor_activity.json new file mode 100644 index 000000000000..43bbe08efe1e --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/contributor_activity.json @@ -0,0 +1,109 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "title": "Contributor Activity", + "properties": { + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["string", "null"] + }, + "login": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "avatar_url": { + "type": ["null", "string"], + "format": "uri" + }, + "gravatar_id": { + "type": ["string", "null"] + }, + "url": { + "type": ["null", "string"], + "format": "uri" + }, + "html_url": { + "type": ["null", "string"], + "format": "uri" + }, + "followers_url": { + "type": ["null", "string"], + "format": "uri" + }, + "following_url": { + "type": ["null", "string"] + }, + "gists_url": { + "type": ["null", "string"] + }, + "starred_url": { + "type": ["null", "string"] + }, + "subscriptions_url": { + "type": ["null", "string"], + "format": "uri" + }, + "organizations_url": { + "type": ["null", "string"], + "format": "uri" + }, + "repos_url": { + "type": ["null", "string"], + "format": "uri" + }, + "events_url": { + "type": ["null", "string"] + }, + "repository": { + "type": ["null", "string"] + }, + "received_events_url": { + "type": ["null", "string"], + "format": "uri" + }, + "type": { + "type": ["null", "string"] + }, + "site_admin": { + "type": ["null", "boolean"] + }, + "starred_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "total": { + "type": ["null", "integer"] + }, + "weeks": { + "type": "array", + "items": { + "type": "object", + "properties": { + "w": { + "type": ["null", "integer"], + "description": "Start of the week, given as a Unix timestamp." + }, + "a": { + "type": ["null", "integer"], + "description": "Number of additions" + }, + "d": { + "type": ["null", "integer"], + "description": "Number of deletions" + }, + "c": { + "type": ["null", "integer"], + "description": "Number of commits" + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/issue_timeline_events.json b/airbyte-integrations/connectors/source-github/source_github/schemas/issue_timeline_events.json new file mode 100644 index 000000000000..3abd58ae3f08 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/issue_timeline_events.json @@ -0,0 +1,1056 @@ +{ + "definitions": { + "base_event": { + "id": { + "type": ["null", "integer"] + }, + "node_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "actor": { + "title": "Simple User", + "description": "A GitHub user.", + "type": ["null", "object"], + "properties": { + "name": { + "type": ["string", "null"] + }, + "email": { + "type": ["string", "null"] + }, + "login": { + "type": "string" + }, + "id": { + "type": "integer" + }, + "node_id": { + "type": "string" + }, + "avatar_url": { + "type": "string" + }, + "gravatar_id": { + "type": ["string", "null"] + }, + "url": { + "type": "string" + }, + "html_url": { + "type": "string" + }, + "followers_url": { + "type": "string" + }, + "following_url": { + "type": "string" + }, + "gists_url": { + "type": "string" + }, + "starred_url": { + "type": "string" + }, + "subscriptions_url": { + "type": "string" + }, + "organizations_url": { + "type": "string" + }, + "repos_url": { + "type": "string" + }, + "events_url": { + "type": "string" + }, + "received_events_url": { + "type": "string" + }, + "type": { + "type": "string" + }, + "site_admin": { + "type": "boolean" + }, + "starred_at": { + "type": "string", + "format": "date-time" + } + } + }, + "event": { + "type": ["null", "string"] + }, + "commit_id": { + "type": ["string", "null"] + }, + "commit_url": { + "type": ["string", "null"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "performed_via_github_app": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "slug": { + "type": "string" + }, + "node_id": { + "type": "string" + }, + "owner": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "properties": { + "name": { + "type": ["string", "null"] + }, + "email": { + "type": ["string", "null"] + }, + "login": { + "type": "string" + }, + "id": { + "type": "integer" + }, + "node_id": { + "type": "string" + }, + "avatar_url": { + "type": "string" + }, + "gravatar_id": { + "type": ["string", "null"] + }, + "url": { + "type": "string" + }, + "html_url": { + "type": "string" + }, + "followers_url": { + "type": "string" + }, + "following_url": { + "type": "string" + }, + "gists_url": { + "type": "string" + }, + "starred_url": { + "type": "string" + }, + "subscriptions_url": { + "type": "string" + }, + "organizations_url": { + "type": "string" + }, + "repos_url": { + "type": "string" + }, + "events_url": { + "type": "string" + }, + "received_events_url": { + "type": "string" + }, + "type": { + "type": "string" + }, + "site_admin": { + "type": "boolean" + }, + "starred_at": { + "type": "string", + "format": "date-time" + } + } + } + ] + }, + "name": { + "type": "string" + }, + "description": { + "type": ["string", "null"] + }, + "external_url": { + "type": "string" + }, + "html_url": { + "type": "string" + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "updated_at": { + "type": "string", + "format": "date-time" + }, + "permissions": { + "type": "object", + "properties": { + "issues": { + "type": "string" + }, + "checks": { + "type": "string" + }, + "metadata": { + "type": "string" + }, + "contents": { + "type": "string" + }, + "deployments": { + "type": "string" + } + }, + "additionalProperties": true + }, + "events": { + "type": "array", + "items": { + "type": "string" + } + }, + "installations_count": { + "type": "integer" + }, + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + }, + "webhook_secret": { + "type": ["string", "null"] + }, + "pem": { + "type": "string" + } + } + } + ] + } + } + }, + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "repository": { + "type": "string" + }, + "issue_number": { + "type": "integer" + }, + "labeled": { + "$ref": "#/definitions/base_event", + "label": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "color": { + "type": ["null", "string"] + } + } + } + }, + "unlabeled": { + "$ref": "#/definitions/base_event", + "label": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "color": { + "type": "string" + } + } + } + }, + "milestoned": { + "$ref": "#/definitions/base_event", + "milestone": { + "type": "object", + "properties": { + "title": { + "type": "string" + } + } + } + }, + "demilestoned": { + "$ref": "#/definitions/base_event", + "milestone": { + "type": "object", + "properties": { + "title": { + "type": "string" + } + } + } + }, + "renamed": { + "$ref": "#/definitions/base_event", + "rename": { + "type": "object", + "properties": { + "from": { + "type": "string" + }, + "to": { + "type": "string" + } + } + } + }, + "review_requested": { + "$ref": "#/definitions/base_event", + "review_requester": { + "type": "object", + "properties": { + "name": { + "type": ["string", "null"] + }, + "email": { + "type": ["string", "null"] + }, + "login": { + "type": "string" + }, + "id": { + "type": "integer" + }, + "node_id": { + "type": "string" + }, + "avatar_url": { + "type": "string" + }, + "gravatar_id": { + "type": ["string", "null"] + }, + "url": { + "type": "string" + }, + "html_url": { + "type": "string" + }, + "followers_url": { + "type": "string" + }, + "following_url": { + "type": "string" + }, + "gists_url": { + "type": "string" + }, + "starred_url": { + "type": "string" + }, + "subscriptions_url": { + "type": "string" + }, + "organizations_url": { + "type": "string" + }, + "repos_url": { + "type": "string" + }, + "events_url": { + "type": "string" + }, + "received_events_url": { + "type": "string" + }, + "type": { + "type": "string" + }, + "site_admin": { + "type": "boolean" + }, + "starred_at": { + "type": "string", + "format": "date-time" + } + } + }, + "requested_team": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "node_id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "slug": { + "type": "string" + }, + "description": { + "type": ["string", "null"] + }, + "privacy": { + "type": "string" + }, + "notification_setting": { + "type": "string" + }, + "permission": { + "type": "string" + }, + "permissions": { + "type": "object", + "properties": { + "pull": { + "type": "boolean" + }, + "triage": { + "type": "boolean" + }, + "push": { + "type": "boolean" + }, + "maintain": { + "type": "boolean" + }, + "admin": { + "type": "boolean" + } + } + }, + "url": { + "type": "string" + }, + "html_url": { + "type": "string" + }, + "members_url": { + "type": "string" + }, + "repositories_url": { + "type": "string" + }, + "parent": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "node_id": { + "type": "string" + }, + "url": { + "type": "string" + }, + "members_url": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": ["string", "null"] + }, + "permission": { + "type": "string" + }, + "privacy": { + "type": "string" + }, + "notification_setting": { + "type": "string" + }, + "html_url": { + "type": "string" + }, + "repositories_url": { + "type": "string" + }, + "slug": { + "type": "string" + }, + "ldap_dn": { + "type": "string" + } + } + } + ] + } + } + }, + "requested_reviewer": { + "type": "object", + "properties": { + "name": { + "type": ["string", "null"] + }, + "email": { + "type": ["string", "null"] + }, + "login": { + "type": "string" + }, + "id": { + "type": "integer" + }, + "node_id": { + "type": "string" + }, + "avatar_url": { + "type": "string" + }, + "gravatar_id": { + "type": ["string", "null"] + }, + "url": { + "type": "string" + }, + "html_url": { + "type": "string" + }, + "followers_url": { + "type": "string" + }, + "following_url": { + "type": "string" + }, + "gists_url": { + "type": "string" + }, + "starred_url": { + "type": "string" + }, + "subscriptions_url": { + "type": "string" + }, + "organizations_url": { + "type": "string" + }, + "repos_url": { + "type": "string" + }, + "events_url": { + "type": "string" + }, + "received_events_url": { + "type": "string" + }, + "type": { + "type": "string" + }, + "site_admin": { + "type": "boolean" + }, + "starred_at": { + "type": "string", + "format": "date-time" + } + } + } + }, + "review_request_removed": { + "$ref": "#/definitions/base_event", + "review_requester": { + "type": "object", + "properties": { + "name": { + "type": ["string", "null"] + }, + "email": { + "type": ["string", "null"] + }, + "login": { + "type": "string" + }, + "id": { + "type": "integer" + }, + "node_id": { + "type": "string" + }, + "avatar_url": { + "type": "string" + }, + "gravatar_id": { + "type": ["string", "null"] + }, + "url": { + "type": "string" + }, + "html_url": { + "type": "string" + }, + "followers_url": { + "type": "string" + }, + "following_url": { + "type": "string" + }, + "gists_url": { + "type": "string" + }, + "starred_url": { + "type": "string" + }, + "subscriptions_url": { + "type": "string" + }, + "organizations_url": { + "type": "string" + }, + "repos_url": { + "type": "string" + }, + "events_url": { + "type": "string" + }, + "received_events_url": { + "type": "string" + }, + "type": { + "type": "string" + }, + "site_admin": { + "type": "boolean" + }, + "starred_at": { + "type": "string", + "format": "date-time" + } + } + }, + "requested_team": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "node_id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "slug": { + "type": "string" + }, + "description": { + "type": ["string", "null"] + }, + "privacy": { + "type": "string" + }, + "notification_setting": { + "type": "string" + }, + "permission": { + "type": "string" + }, + "permissions": { + "type": "object", + "properties": { + "pull": { + "type": "boolean" + }, + "triage": { + "type": "boolean" + }, + "push": { + "type": "boolean" + }, + "maintain": { + "type": "boolean" + }, + "admin": { + "type": "boolean" + } + } + }, + "url": { + "type": "string" + }, + "html_url": { + "type": "string" + }, + "members_url": { + "type": "string" + }, + "repositories_url": { + "type": "string" + }, + "parent": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "node_id": { + "type": "string" + }, + "url": { + "type": "string" + }, + "members_url": { + "type": "string" + }, + "name": { + "type": "string" + }, + "description": { + "type": ["string", "null"] + }, + "permission": { + "type": "string" + }, + "privacy": { + "type": "string" + }, + "notification_setting": { + "type": "string" + }, + "html_url": { + "type": "string" + }, + "repositories_url": { + "type": "string" + }, + "slug": { + "type": "string" + }, + "ldap_dn": { + "type": "string" + } + } + } + ] + } + } + }, + "requested_reviewer": { + "type": "object", + "properties": { + "name": { + "type": ["string", "null"] + }, + "email": { + "type": ["string", "null"] + }, + "login": { + "type": "string" + }, + "id": { + "type": "integer" + }, + "node_id": { + "type": "string" + }, + "avatar_url": { + "type": "string" + }, + "gravatar_id": { + "type": ["string", "null"] + }, + "url": { + "type": "string" + }, + "html_url": { + "type": "string" + }, + "followers_url": { + "type": "string" + }, + "following_url": { + "type": "string" + }, + "gists_url": { + "type": "string" + }, + "starred_url": { + "type": "string" + }, + "subscriptions_url": { + "type": "string" + }, + "organizations_url": { + "type": "string" + }, + "repos_url": { + "type": "string" + }, + "events_url": { + "type": "string" + }, + "received_events_url": { + "type": "string" + }, + "type": { + "type": "string" + }, + "site_admin": { + "type": "boolean" + }, + "starred_at": { + "type": "string", + "format": "date-time" + } + } + } + }, + "review_dismissed": { + "$ref": "#/definitions/base_event", + "dismissed_review": { + "type": "object", + "properties": { + "state": { + "type": "string" + }, + "review_id": { + "type": "integer" + }, + "dismissal_message": { + "type": ["string", "null"] + }, + "dismissal_commit_id": { + "type": "string" + } + } + } + }, + "locked": { + "$ref": "#/definitions/base_event", + "lock_reason": { + "type": ["string", "null"] + } + }, + "added_to_project": { + "$ref": "#/definitions/base_event", + "project_card": { + "type": "object", + "properties": { + "id": { "type": "integer" }, + "url": { "type": "string" }, + "project_id": { "type": "integer" }, + "project_url": { "type": "string" }, + "column_name": { "type": "string" }, + "previous_column_name": { "type": "string" } + } + } + }, + "moved_columns_in_project": { + "$ref": "#/definitions/base_event", + "project_card": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "url": { + "type": "string", + "format": "uri" + }, + "project_id": { + "type": "integer" + }, + "project_url": { + "type": "string", + "format": "uri" + }, + "column_name": { + "type": "string" + }, + "previous_column_name": { + "type": "string" + } + } + } + }, + "removed_from_project": { + "$ref": "#/definitions/base_event", + "project_card": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "url": { + "type": "string", + "format": "uri" + }, + "project_id": { + "type": "integer" + }, + "project_url": { + "type": "string", + "format": "uri" + }, + "column_name": { + "type": "string" + }, + "previous_column_name": { + "type": "string" + } + } + } + }, + "converted_note_to_issue": { + "$ref": "#/definitions/base_event", + "project_card": { + "type": "object", + "properties": { + "id": { "type": "integer" }, + "url": { "type": "string" }, + "project_id": { "type": "integer" }, + "project_url": { "type": "string" }, + "column_name": { "type": "string" }, + "previous_column_name": { "type": "string" } + } + } + }, + "comment": { + "$ref": "events/comment.json" + }, + "cross-referenced": { + "$ref": "events/cross_referenced.json" + }, + "committed": { + "$ref": "events/committed.json" + }, + "closed": { + "$ref": "#/definitions/base_event" + }, + "head_ref_deleted": { + "$ref": "#/definitions/base_event" + }, + "head_ref_restored": { + "$ref": "#/definitions/base_event" + }, + "reopened": { + "$ref": "#/definitions/base_event" + }, + "reviewed": { + "$ref": "events/reviewed.json" + }, + "commented": { + "$ref": "events/commented.json" + }, + "commit_commented": { + "$ref": "events/commented.json" + }, + "assigned": { + "$ref": "#/definitions/base_event", + "assignee": { + "title": "Simple User", + "description": "A GitHub user.", + "type": "object", + "properties": { + "name": { "type": ["string", "null"] }, + "email": { "type": ["string", "null"] }, + "login": { "type": "string" }, + "id": { "type": "integer" }, + "node_id": { "type": "string" }, + "avatar_url": { "type": "string" }, + "gravatar_id": { "type": ["string", "null"] }, + "url": { "type": "string" }, + "html_url": { "type": "string" }, + "followers_url": { "type": "string" }, + "following_url": { "type": "string" }, + "gists_url": { "type": "string" }, + "starred_url": { "type": "string" }, + "subscriptions_url": { "type": "string" }, + "organizations_url": { "type": "string" }, + "repos_url": { "type": "string" }, + "events_url": { "type": "string" }, + "received_events_url": { "type": "string" }, + "type": { "type": "string" }, + "site_admin": { "type": "boolean" }, + "starred_at": { "type": "string", "format": "date-time" } + } + } + }, + "unassigned": { + "$ref": "#/definitions/base_event", + "assignee": { + "title": "Simple User", + "description": "A GitHub user.", + "type": "object", + "properties": { + "name": { "type": ["string", "null"] }, + "email": { "type": ["string", "null"] }, + "login": { "type": "string" }, + "id": { "type": "integer" }, + "node_id": { "type": "string" }, + "avatar_url": { "type": "string" }, + "gravatar_id": { "type": ["string", "null"] }, + "url": { "type": "string" }, + "html_url": { "type": "string" }, + "followers_url": { "type": "string" }, + "following_url": { "type": "string" }, + "gists_url": { "type": "string" }, + "starred_url": { "type": "string" }, + "subscriptions_url": { "type": "string" }, + "organizations_url": { "type": "string" }, + "repos_url": { "type": "string" }, + "events_url": { "type": "string" }, + "received_events_url": { "type": "string" }, + "type": { "type": "string" }, + "site_admin": { "type": "boolean" }, + "starred_at": { "type": "string", "format": "date-time" } + } + } + }, + "state_change": { + "$ref": "#/definitions/base_event", + "state_reason": { + "type": ["string", "null"] + } + }, + "connected": { + "$ref": "#/definitions/base_event" + }, + "auto_squash_enabled": { + "$ref": "#/definitions/base_event" + }, + "merged": { + "$ref": "#/definitions/base_event" + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/projects_v2.json b/airbyte-integrations/connectors/source-github/source_github/schemas/projects_v2.json new file mode 100644 index 000000000000..744fec6d7571 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/projects_v2.json @@ -0,0 +1,80 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "closed": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "creator": { + "type": ["null", "object"], + "properties": { + "avatarUrl": { + "type": ["null", "string"] + }, + "login": { + "type": ["null", "string"] + }, + "resourcePath": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + }, + "closed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "node_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "number": { + "type": ["null", "integer"] + }, + "public": { + "type": ["null", "boolean"] + }, + "readme": { + "type": ["null", "string"] + }, + "short_description": { + "type": ["null", "string"] + }, + "template": { + "type": ["null", "boolean"] + }, + "title": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "viewerCanClose": { + "type": ["null", "boolean"] + }, + "viewerCanReopen": { + "type": ["null", "boolean"] + }, + "viewerCanUpdate": { + "type": ["null", "boolean"] + }, + "owner_id": { + "type": ["null", "string"] + }, + "repository": { + "type": "string" + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/shared/events/comment.json b/airbyte-integrations/connectors/source-github/source_github/schemas/shared/events/comment.json new file mode 100644 index 000000000000..f3555b3b2adf --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/shared/events/comment.json @@ -0,0 +1,188 @@ +{ + "title": "Timeline Comment Event", + "description": "Timeline Comment Event", + "type": "object", + "properties": { + "event": { "type": "string" }, + "actor": { + "title": "Simple User", + "description": "A GitHub user.", + "type": "object", + "properties": { + "name": { "type": ["string", "null"] }, + "email": { "type": ["string", "null"] }, + "login": { "type": "string" }, + "id": { "type": "integer" }, + "node_id": { "type": "string" }, + "avatar_url": { "type": "string" }, + "gravatar_id": { "type": ["string", "null"] }, + "url": { "type": "string" }, + "html_url": { "type": "string" }, + "followers_url": { "type": "string" }, + "following_url": { "type": "string" }, + "gists_url": { "type": "string" }, + "starred_url": { "type": "string" }, + "subscriptions_url": { "type": "string" }, + "organizations_url": { "type": "string" }, + "repos_url": { "type": "string" }, + "events_url": { "type": "string" }, + "received_events_url": { "type": "string" }, + "type": { "type": "string" }, + "site_admin": { "type": "boolean" }, + "starred_at": { "type": "string", "format": "date-time" } + } + }, + "id": { + "description": "Unique identifier of the issue comment", + "type": "integer" + }, + "node_id": { "type": "string" }, + "url": { "description": "URL for the issue comment", "type": "string" }, + "body": { + "description": "Contents of the issue comment", + "type": "string" + }, + "body_text": { "type": "string" }, + "body_html": { "type": "string" }, + "html_url": { "type": "string" }, + "user": { + "title": "Simple User", + "description": "A GitHub user.", + "type": "object", + "properties": { + "name": { "type": ["string", "null"] }, + "email": { "type": ["string", "null"] }, + "login": { "type": "string" }, + "id": { "type": "integer" }, + "node_id": { "type": "string" }, + "avatar_url": { "type": "string" }, + "gravatar_id": { "type": ["string", "null"] }, + "url": { "type": "string" }, + "html_url": { "type": "string" }, + "followers_url": { "type": "string" }, + "following_url": { "type": "string" }, + "gists_url": { "type": "string" }, + "starred_url": { "type": "string" }, + "subscriptions_url": { "type": "string" }, + "organizations_url": { "type": "string" }, + "repos_url": { "type": "string" }, + "events_url": { "type": "string" }, + "received_events_url": { "type": "string" }, + "type": { "type": "string" }, + "site_admin": { "type": "boolean" }, + "starred_at": { "type": "string", "format": "date-time" } + } + }, + "created_at": { "type": "string", "format": "date-time" }, + "updated_at": { "type": "string", "format": "date-time" }, + "issue_url": { "type": "string" }, + "author_association": { "type": "string" }, + "performed_via_github_app": { + "anyOf": [ + { + "type": "null" + }, + { + "title": "GitHub app", + "description": "GitHub apps are a new way to extend GitHub. They can be installed directly on organizations and user accounts and granted access to specific repositories. They come with granular permissions and built-in webhooks. GitHub apps are first class actors within GitHub.", + "type": "object", + "properties": { + "id": { + "description": "Unique identifier of the GitHub app", + "type": "integer" + }, + "slug": { + "description": "The slug name of the GitHub app", + "type": "string" + }, + "node_id": { "type": "string" }, + "owner": { + "anyOf": [ + { + "type": "null" + }, + { + "title": "Simple User", + "description": "A GitHub user.", + "type": "object", + "properties": { + "name": { "type": ["string", "null"] }, + "email": { "type": ["string", "null"] }, + "login": { "type": "string" }, + "id": { "type": "integer" }, + "node_id": { "type": "string" }, + "avatar_url": { "type": "string" }, + "gravatar_id": { "type": ["string", "null"] }, + "url": { "type": "string" }, + "html_url": { "type": "string" }, + "followers_url": { "type": "string" }, + "following_url": { "type": "string" }, + "gists_url": { "type": "string" }, + "starred_url": { "type": "string" }, + "subscriptions_url": { "type": "string" }, + "organizations_url": { "type": "string" }, + "repos_url": { "type": "string" }, + "events_url": { "type": "string" }, + "received_events_url": { "type": "string" }, + "type": { "type": "string" }, + "site_admin": { "type": "boolean" }, + "starred_at": { "type": "string", "format": "date-time" } + } + } + ] + }, + "name": { + "description": "The name of the GitHub app", + "type": "string" + }, + "description": { "type": ["string", "null"] }, + "external_url": { "type": "string" }, + "html_url": { "type": "string" }, + "created_at": { "type": "string", "format": "date-time" }, + "updated_at": { "type": "string", "format": "date-time" }, + "permissions": { + "description": "The set of permissions for the GitHub app", + "type": "object", + "properties": { + "issues": { "type": "string" }, + "checks": { "type": "string" }, + "metadata": { "type": "string" }, + "contents": { "type": "string" }, + "deployments": { "type": "string" } + } + }, + "events": { + "description": "The list of events for the GitHub app", + "type": "array", + "items": { "type": "string" } + }, + "installations_count": { + "description": "The number of installations associated with the GitHub app", + "type": "integer" + }, + "client_id": { "type": "string" }, + "client_secret": { "type": "string" }, + "webhook_secret": { "type": ["string", "null"] }, + "pem": { "type": "string" } + } + } + ] + }, + "reactions": { + "title": "Reaction Rollup", + "type": "object", + "properties": { + "url": { "type": "string" }, + "total_count": { "type": "integer" }, + "+1": { "type": "integer" }, + "-1": { "type": "integer" }, + "laugh": { "type": "integer" }, + "confused": { "type": "integer" }, + "heart": { "type": "integer" }, + "hooray": { "type": "integer" }, + "eyes": { "type": "integer" }, + "rocket": { "type": "integer" } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/shared/events/commented.json b/airbyte-integrations/connectors/source-github/source_github/schemas/shared/events/commented.json new file mode 100644 index 000000000000..671f979a92f7 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/shared/events/commented.json @@ -0,0 +1,118 @@ +{ + "title": "Timeline Line Commented Event", + "description": "Timeline Line Commented Event", + "type": "object", + "properties": { + "event": { "type": "string" }, + "node_id": { "type": "string" }, + "comments": { + "type": "array", + "items": { + "title": "Pull Request Review Comment", + "type": "object", + "properties": { + "url": { + "type": "string" + }, + "pull_request_review_id": { + "type": ["integer", "null"] + }, + "id": { "type": "integer" }, + "node_id": { "type": "string" }, + "diff_hunk": { "type": "string" }, + "path": { "type": "string" }, + "position": { "type": "integer" }, + "original_position": { "type": "integer" }, + "commit_id": { "type": "string" }, + "original_commit_id": { "type": "string" }, + "in_reply_to_id": { "type": "integer" }, + "user": { + "type": "object", + "properties": { + "name": { "type": ["string", "null"] }, + "email": { "type": ["string", "null"] }, + "login": { "type": "string" }, + "id": { "type": "integer" }, + "node_id": { "type": "string" }, + "avatar_url": { "type": "string" }, + "gravatar_id": { "type": ["string", "null"] }, + "url": { "type": "string" }, + "html_url": { "type": "string" }, + "followers_url": { "type": "string" }, + "following_url": { "type": "string" }, + "gists_url": { "type": "string" }, + "starred_url": { "type": "string" }, + "subscriptions_url": { "type": "string" }, + "organizations_url": { "type": "string" }, + "repos_url": { "type": "string" }, + "events_url": { "type": "string" }, + "received_events_url": { "type": "string" }, + "type": { "type": "string" }, + "site_admin": { "type": "boolean" }, + "starred_at": { "type": "string", "format": "date-time" } + } + }, + "body": { "type": "string" }, + "created_at": { "type": "string", "format": "date-time" }, + "updated_at": { "type": "string", "format": "date-time" }, + "html_url": { "type": "string" }, + "pull_request_url": { "type": "string" }, + "author_association": { "type": "string" }, + "_links": { + "type": "object", + "properties": { + "self": { + "type": "object", + "properties": { + "href": { "type": "string" } + } + }, + "html": { + "type": "object", + "properties": { + "href": { "type": "string" } + } + }, + "pull_request": { + "type": "object", + "properties": { + "href": { "type": "string" } + } + } + } + }, + "start_line": { + "type": ["integer", "null"] + }, + "original_start_line": { + "type": ["integer", "null"] + }, + "start_side": { + "type": ["string", "null"] + }, + "line": { "type": "integer" }, + "original_line": { "type": "integer" }, + "side": { "type": "string" }, + "subject_type": { "type": "string" }, + "reactions": { + "type": "object", + "properties": { + "url": { "type": "string" }, + "total_count": { "type": "integer" }, + "+1": { "type": "integer" }, + "-1": { "type": "integer" }, + "laugh": { "type": "integer" }, + "confused": { "type": "integer" }, + "heart": { "type": "integer" }, + "hooray": { "type": "integer" }, + "eyes": { "type": "integer" }, + "rocket": { "type": "integer" } + } + }, + "body_html": { "type": "string" }, + "body_text": { "type": "string" } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/shared/events/committed.json b/airbyte-integrations/connectors/source-github/source_github/schemas/shared/events/committed.json new file mode 100644 index 000000000000..2cbcd16fb13a --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/shared/events/committed.json @@ -0,0 +1,56 @@ +{ + "title": "Timeline Committed Event", + "description": "Timeline Committed Event", + "type": "object", + "properties": { + "event": { "type": "string" }, + "sha": { "type": "string" }, + "node_id": { "type": "string" }, + "url": { "type": "string" }, + "author": { + "type": "object", + "properties": { + "date": { "format": "date-time", "type": "string" }, + "email": { "type": "string" }, + "name": { "type": "string" } + } + }, + "committer": { + "type": "object", + "properties": { + "date": { "format": "date-time", "type": "string" }, + "email": { "type": "string" }, + "name": { "type": "string" } + } + }, + "message": { "type": "string" }, + "tree": { + "type": "object", + "properties": { + "sha": { "type": "string" }, + "url": { "type": "string" } + } + }, + "parents": { + "type": "array", + "items": { + "type": "object", + "properties": { + "sha": { "type": "string" }, + "url": { "type": "string" }, + "html_url": { "type": "string" } + } + } + }, + "verification": { + "type": "object", + "properties": { + "verified": { "type": "boolean" }, + "reason": { "type": "string" }, + "signature": { "type": ["string", "null"] }, + "payload": { "type": ["string", "null"] } + } + }, + "html_url": { "type": "string" } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/shared/events/cross_referenced.json b/airbyte-integrations/connectors/source-github/source_github/schemas/shared/events/cross_referenced.json new file mode 100644 index 000000000000..19a0f40395ac --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/shared/events/cross_referenced.json @@ -0,0 +1,784 @@ +{ + "title": "Timeline Cross Referenced Event", + "description": "Timeline Cross Referenced Event", + "type": "object", + "properties": { + "event": { "type": "string" }, + "actor": { + "type": "object", + "properties": { + "name": { "type": ["string", "null"] }, + "email": { "type": ["string", "null"] }, + "login": { "type": "string" }, + "id": { "type": "integer" }, + "node_id": { "type": "string" }, + "avatar_url": { "type": "string" }, + "gravatar_id": { "type": ["string", "null"] }, + "url": { "type": "string" }, + "html_url": { "type": "string" }, + "followers_url": { "type": "string" }, + "following_url": { "type": "string" }, + "gists_url": { "type": "string" }, + "starred_url": { "type": "string" }, + "subscriptions_url": { "type": "string" }, + "organizations_url": { "type": "string" }, + "repos_url": { "type": "string" }, + "events_url": { "type": "string" }, + "received_events_url": { "type": "string" }, + "type": { "type": "string" }, + "site_admin": { "type": "boolean" }, + "starred_at": { "type": "string", "format": "date-time" } + } + }, + "created_at": { "type": "string", "format": "date-time" }, + "updated_at": { "type": "string", "format": "date-time" }, + "source": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "issue": { + "title": "Issue", + "description": "Issues are a great way to keep track of tasks, enhancements, and bugs for your projects.", + "type": "object", + "properties": { + "id": { "type": "integer" }, + "node_id": { "type": "string" }, + "url": { "type": "string" }, + "repository_url": { "type": "string" }, + "labels_url": { "type": "string" }, + "comments_url": { "type": "string" }, + "events_url": { "type": "string" }, + "html_url": { "type": "string" }, + "number": { "type": "integer" }, + "state": { "type": "string" }, + "state_reason": { "type": ["string", "null"] }, + "title": { "type": "string" }, + "body": { "type": ["string", "null"] }, + "user": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "properties": { + "name": { "type": ["string", "null"] }, + "email": { "type": ["string", "null"] }, + "login": { "type": "string" }, + "id": { "type": "integer" }, + "node_id": { "type": "string" }, + "avatar_url": { "type": "string" }, + "gravatar_id": { "type": ["string", "null"] }, + "url": { "type": "string" }, + "html_url": { "type": "string" }, + "followers_url": { "type": "string" }, + "following_url": { "type": "string" }, + "gists_url": { "type": "string" }, + "starred_url": { "type": "string" }, + "subscriptions_url": { "type": "string" }, + "organizations_url": { "type": "string" }, + "repos_url": { "type": "string" }, + "events_url": { "type": "string" }, + "received_events_url": { "type": "string" }, + "type": { "type": "string" }, + "site_admin": { "type": "boolean" }, + "starred_at": { "type": "string", "format": "date-time" } + } + } + ] + }, + "labels": { + "type": "array", + "items": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + "properties": { + "id": { "type": "integer" }, + "node_id": { "type": "string" }, + "url": { "type": "string" }, + "name": { "type": "string" }, + "description": { "type": ["string", "null"] }, + "color": { "type": ["string", "null"] }, + "default": { "type": "boolean" } + } + } + ] + } + }, + "assignee": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "properties": { + "name": { "type": ["string", "null"] }, + "email": { "type": ["string", "null"] }, + "login": { "type": "string" }, + "id": { "type": "integer" }, + "node_id": { "type": "string" }, + "avatar_url": { "type": "string" }, + "gravatar_id": { "type": ["string", "null"] }, + "url": { "type": "string" }, + "html_url": { "type": "string" }, + "followers_url": { "type": "string" }, + "following_url": { "type": "string" }, + "gists_url": { "type": "string" }, + "starred_url": { "type": "string" }, + "subscriptions_url": { "type": "string" }, + "organizations_url": { "type": "string" }, + "repos_url": { "type": "string" }, + "events_url": { "type": "string" }, + "received_events_url": { "type": "string" }, + "type": { "type": "string" }, + "site_admin": { "type": "boolean" }, + "starred_at": { "type": "string", "format": "date-time" } + } + } + ] + }, + "assignees": { + "type": ["array", "null"], + "items": { + "type": "object", + "properties": { + "name": { "type": ["string", "null"] }, + "email": { "type": ["string", "null"] }, + "login": { "type": "string" }, + "id": { "type": "integer" }, + "node_id": { "type": "string" }, + "avatar_url": { "type": "string" }, + "gravatar_id": { "type": ["string", "null"] }, + "url": { "type": "string" }, + "html_url": { "type": "string" }, + "followers_url": { "type": "string" }, + "following_url": { "type": "string" }, + "gists_url": { "type": "string" }, + "starred_url": { "type": "string" }, + "subscriptions_url": { "type": "string" }, + "organizations_url": { "type": "string" }, + "repos_url": { "type": "string" }, + "events_url": { "type": "string" }, + "received_events_url": { "type": "string" }, + "type": { "type": "string" }, + "site_admin": { + "type": "boolean" + }, + "starred_at": { "type": "string", "format": "date-time" } + } + } + }, + "milestone": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "properties": { + "url": { "type": "string" }, + "html_url": { "type": "string" }, + "labels_url": { "type": "string" }, + "id": { "type": "integer" }, + "node_id": { "type": "string" }, + "number": { "type": "integer" }, + "state": { "type": "string" }, + "title": { "type": "string" }, + "description": { "type": ["string", "null"] }, + "creator": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "properties": { + "name": { "type": ["string", "null"] }, + "email": { "type": ["string", "null"] }, + "login": { "type": "string" }, + "id": { "type": "integer" }, + "node_id": { "type": "string" }, + "avatar_url": { "type": "string" }, + "gravatar_id": { "type": ["string", "null"] }, + "url": { "type": "string" }, + "html_url": { "type": "string" }, + "followers_url": { "type": "string" }, + "following_url": { "type": "string" }, + "gists_url": { "type": "string" }, + "starred_url": { "type": "string" }, + "subscriptions_url": { "type": "string" }, + "organizations_url": { "type": "string" }, + "repos_url": { "type": "string" }, + "events_url": { "type": "string" }, + "received_events_url": { "type": "string" }, + "type": { "type": "string" }, + "site_admin": { "type": "boolean" }, + "starred_at": { + "type": "string", + "format": "date-time" + } + } + } + ] + }, + "open_issues": { "type": "integer" }, + "closed_issues": { "type": "integer" }, + "created_at": { "type": "string", "format": "date-time" }, + "updated_at": { "type": "string", "format": "date-time" }, + "closed_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "due_on": { + "type": ["string", "null"], + "format": "date-time" + } + } + } + ] + }, + "locked": { "type": "boolean" }, + "active_lock_reason": { "type": ["string", "null"] }, + "comments": { "type": "integer" }, + "pull_request": { + "type": "object", + "properties": { + "merged_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "diff_url": { "type": ["string", "null"] }, + "html_url": { "type": ["string", "null"] }, + "patch_url": { "type": ["string", "null"] }, + "url": { "type": ["string", "null"] } + } + }, + "closed_at": { "type": ["string", "null"], "format": "date-time" }, + "created_at": { "type": "string", "format": "date-time" }, + "updated_at": { "type": "string", "format": "date-time" }, + "draft": { "type": "boolean" }, + "closed_by": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "properties": { + "name": { "type": ["string", "null"] }, + "email": { "type": ["string", "null"] }, + "login": { "type": "string", "examples": ["octocat"] }, + "id": { "type": "integer" }, + "node_id": { "type": "string" }, + "avatar_url": { "type": "string" }, + "gravatar_id": { "type": ["string", "null"] }, + "url": { "type": "string" }, + "html_url": { "type": "string" }, + "followers_url": { "type": "string" }, + "following_url": { "type": "string" }, + "gists_url": { "type": "string" }, + "starred_url": { "type": "string" }, + "subscriptions_url": { "type": "string" }, + "organizations_url": { "type": "string" }, + "repos_url": { "type": "string" }, + "events_url": { "type": "string" }, + "received_events_url": { "type": "string" }, + "type": { "type": "string" }, + "site_admin": { "type": "boolean" }, + "starred_at": { "type": "string", "format": "date-time" } + } + } + ] + }, + "body_html": { "type": "string" }, + "body_text": { "type": "string" }, + "timeline_url": { "type": "string" }, + "repository": { + "type": "object", + "properties": { + "id": { "type": "integer" }, + "node_id": { "type": "string" }, + "name": { "type": "string" }, + "full_name": { "type": "string" }, + "license": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "properties": { + "key": { "type": "string" }, + "name": { "type": "string" }, + "url": { "type": ["string", "null"] }, + "spdx_id": { "type": ["string", "null"] }, + "node_id": { "type": "string" }, + "html_url": { "type": "string" } + } + } + ] + }, + "organization": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "properties": { + "name": { "type": ["string", "null"] }, + "email": { "type": ["string", "null"] }, + "login": { "type": "string" }, + "id": { "type": "integer" }, + "node_id": { "type": "string" }, + "avatar_url": { "type": "string" }, + "gravatar_id": { "type": ["string", "null"] }, + "url": { "type": "string" }, + "html_url": { "type": "string" }, + "followers_url": { "type": "string" }, + "following_url": { "type": "string" }, + "gists_url": { "type": "string" }, + "starred_url": { "type": "string" }, + "subscriptions_url": { "type": "string" }, + "organizations_url": { "type": "string" }, + "repos_url": { "type": "string" }, + "events_url": { "type": "string" }, + "received_events_url": { "type": "string" }, + "type": { "type": "string" }, + "site_admin": { "type": "boolean" }, + "starred_at": { + "type": "string", + "format": "date-time" + } + } + } + ] + }, + "forks": { "type": "integer" }, + "permissions": { + "type": "object", + "properties": { + "admin": { "type": "boolean" }, + "pull": { "type": "boolean" }, + "triage": { "type": "boolean" }, + "push": { "type": "boolean" }, + "maintain": { "type": "boolean" } + } + }, + "owner": { + "type": "object", + "properties": { + "name": { "type": ["string", "null"] }, + "email": { "type": ["string", "null"] }, + "login": { "type": "string" }, + "id": { "type": "integer" }, + "node_id": { "type": "string" }, + "avatar_url": { "type": "string" }, + "gravatar_id": { "type": ["string", "null"] }, + "url": { "type": "string" }, + "html_url": { "type": "string" }, + "followers_url": { "type": "string" }, + "following_url": { "type": "string" }, + "gists_url": { "type": "string" }, + "starred_url": { "type": "string" }, + "subscriptions_url": { "type": "string" }, + "organizations_url": { "type": "string" }, + "repos_url": { "type": "string" }, + "events_url": { "type": "string" }, + "received_events_url": { "type": "string" }, + "type": { "type": "string" }, + "site_admin": { "type": "boolean" }, + "starred_at": { "type": "string", "format": "date-time" } + } + }, + "private": { "type": "boolean" }, + "html_url": { "type": "string" }, + "description": { "type": ["string", "null"] }, + "fork": { "type": "boolean" }, + "url": { "type": "string" }, + "archive_url": { "type": "string" }, + "assignees_url": { "type": "string" }, + "blobs_url": { "type": "string" }, + "branches_url": { "type": "string" }, + "collaborators_url": { "type": "string" }, + "comments_url": { "type": "string" }, + "commits_url": { "type": "string" }, + "compare_url": { "type": "string" }, + "contents_url": { "type": "string" }, + "contributors_url": { "type": "string" }, + "deployments_url": { "type": "string" }, + "downloads_url": { "type": "string" }, + "events_url": { "type": "string" }, + "forks_url": { "type": "string" }, + "git_commits_url": { "type": "string" }, + "git_refs_url": { "type": "string" }, + "git_tags_url": { "type": "string" }, + "git_url": { "type": "string" }, + "issue_comment_url": { "type": "string" }, + "issue_events_url": { "type": "string" }, + "issues_url": { "type": "string" }, + "keys_url": { "type": "string" }, + "labels_url": { "type": "string" }, + "languages_url": { "type": "string" }, + "merges_url": { "type": "string" }, + "milestones_url": { "type": "string" }, + "notifications_url": { "type": "string" }, + "pulls_url": { "type": "string" }, + "releases_url": { "type": "string" }, + "ssh_url": { "type": "string" }, + "stargazers_url": { "type": "string" }, + "statuses_url": { "type": "string" }, + "subscribers_url": { "type": "string" }, + "subscription_url": { "type": "string" }, + "tags_url": { "type": "string" }, + "teams_url": { "type": "string" }, + "trees_url": { "type": "string" }, + "clone_url": { "type": "string" }, + "mirror_url": { "type": ["string", "null"] }, + "hooks_url": { "type": "string" }, + "svn_url": { "type": "string" }, + "homepage": { "type": ["string", "null"] }, + "language": { "type": ["string", "null"] }, + "forks_count": { "type": "integer" }, + "stargazers_count": { "type": "integer" }, + "watchers_count": { "type": "integer" }, + "size": { "type": "integer" }, + "default_branch": { "type": "string" }, + "open_issues_count": { "type": "integer" }, + "is_template": { "type": "boolean" }, + "topics": { + "type": "array", + "items": { "type": "string" } + }, + "has_issues": { "type": "boolean" }, + "has_projects": { "type": "boolean" }, + "has_wiki": { "type": "boolean" }, + "has_pages": { "type": "boolean" }, + "has_downloads": { "type": "boolean" }, + "has_discussions": { "type": "boolean" }, + "archived": { "type": "boolean" }, + "disabled": { "type": "boolean" }, + "visibility": { "type": "string" }, + "pushed_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "created_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "updated_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "allow_rebase_merge": { "type": "boolean" }, + "template_repository": { + "type": ["object", "null"], + "properties": { + "id": { "type": "integer" }, + "node_id": { "type": "string" }, + "name": { "type": "string" }, + "full_name": { "type": "string" }, + "owner": { + "type": "object", + "properties": { + "login": { "type": "string" }, + "id": { "type": "integer" }, + "node_id": { "type": "string" }, + "avatar_url": { "type": "string" }, + "gravatar_id": { "type": "string" }, + "url": { "type": "string" }, + "html_url": { "type": "string" }, + "followers_url": { "type": "string" }, + "following_url": { "type": "string" }, + "gists_url": { "type": "string" }, + "starred_url": { "type": "string" }, + "subscriptions_url": { "type": "string" }, + "organizations_url": { "type": "string" }, + "repos_url": { "type": "string" }, + "events_url": { "type": "string" }, + "received_events_url": { "type": "string" }, + "type": { "type": "string" }, + "site_admin": { "type": "boolean" } + } + }, + "private": { "type": "boolean" }, + "html_url": { "type": "string" }, + "description": { "type": "string" }, + "fork": { "type": "boolean" }, + "url": { "type": "string" }, + "archive_url": { "type": "string" }, + "assignees_url": { "type": "string" }, + "blobs_url": { "type": "string" }, + "branches_url": { "type": "string" }, + "collaborators_url": { "type": "string" }, + "comments_url": { "type": "string" }, + "commits_url": { "type": "string" }, + "compare_url": { "type": "string" }, + "contents_url": { "type": "string" }, + "contributors_url": { "type": "string" }, + "deployments_url": { "type": "string" }, + "downloads_url": { "type": "string" }, + "events_url": { "type": "string" }, + "forks_url": { "type": "string" }, + "git_commits_url": { "type": "string" }, + "git_refs_url": { "type": "string" }, + "git_tags_url": { "type": "string" }, + "git_url": { "type": "string" }, + "issue_comment_url": { "type": "string" }, + "issue_events_url": { "type": "string" }, + "issues_url": { "type": "string" }, + "keys_url": { "type": "string" }, + "labels_url": { "type": "string" }, + "languages_url": { "type": "string" }, + "merges_url": { "type": "string" }, + "milestones_url": { "type": "string" }, + "notifications_url": { "type": "string" }, + "pulls_url": { "type": "string" }, + "releases_url": { "type": "string" }, + "ssh_url": { "type": "string" }, + "stargazers_url": { "type": "string" }, + "statuses_url": { "type": "string" }, + "subscribers_url": { "type": "string" }, + "subscription_url": { "type": "string" }, + "tags_url": { "type": "string" }, + "teams_url": { "type": "string" }, + "trees_url": { "type": "string" }, + "clone_url": { "type": "string" }, + "mirror_url": { "type": "string" }, + "hooks_url": { "type": "string" }, + "svn_url": { "type": "string" }, + "homepage": { "type": "string" }, + "language": { "type": "string" }, + "forks_count": { "type": "integer" }, + "stargazers_count": { "type": "integer" }, + "watchers_count": { "type": "integer" }, + "size": { "type": "integer" }, + "default_branch": { "type": "string" }, + "open_issues_count": { "type": "integer" }, + "is_template": { "type": "boolean" }, + "topics": { + "type": "array", + "items": { "type": "string" } + }, + "has_issues": { "type": "boolean" }, + "has_projects": { "type": "boolean" }, + "has_wiki": { "type": "boolean" }, + "has_pages": { "type": "boolean" }, + "has_downloads": { "type": "boolean" }, + "archived": { "type": "boolean" }, + "disabled": { "type": "boolean" }, + "visibility": { "type": "string" }, + "pushed_at": { "type": "string", "format": "date-time" }, + "created_at": { "type": "string", "format": "date-time" }, + "updated_at": { "type": "string", "format": "date-time" }, + "permissions": { + "type": "object", + "properties": { + "admin": { "type": "boolean" }, + "maintain": { "type": "boolean" }, + "push": { "type": "boolean" }, + "triage": { "type": "boolean" }, + "pull": { "type": "boolean" } + } + }, + "allow_rebase_merge": { "type": "boolean" }, + "temp_clone_token": { "type": "string" }, + "allow_squash_merge": { "type": "boolean" }, + "allow_auto_merge": { "type": "boolean" }, + "delete_branch_on_merge": { "type": "boolean" }, + "allow_update_branch": { "type": "boolean" }, + "use_squash_pr_title_as_default": { "type": "boolean" }, + "squash_merge_commit_title": { "type": "string" }, + "squash_merge_commit_message": { "type": "string" }, + "merge_commit_title": { "type": "string" }, + "merge_commit_message": { "type": "string" }, + "allow_merge_commit": { "type": "boolean" }, + "subscribers_count": { "type": "integer" }, + "network_count": { "type": "integer" } + } + }, + "temp_clone_token": { "type": "string" }, + "allow_squash_merge": { "type": "boolean" }, + "allow_auto_merge": { "type": "boolean" }, + "delete_branch_on_merge": { "type": "boolean" }, + "allow_update_branch": { "type": "boolean" }, + "use_squash_pr_title_as_default": { "type": "boolean" }, + "squash_merge_commit_title": { "type": "string" }, + "squash_merge_commit_message": { "type": "string" }, + "merge_commit_title": { "type": "string" }, + "merge_commit_message": { "type": "string" }, + "allow_merge_commit": { "type": "boolean" }, + "allow_forking": { "type": "boolean" }, + "web_commit_signoff_required": { "type": "boolean" }, + "subscribers_count": { "type": "integer" }, + "network_count": { "type": "integer" }, + "open_issues": { "type": "integer" }, + "watchers": { "type": "integer" }, + "master_branch": { "type": "string" }, + "starred_at": { "type": "string", "format": "date-time" }, + "anonymous_access_enabled": { "type": "boolean" } + }, + "performed_via_github_app": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "properties": { + "id": { "type": "integer" }, + "slug": { "type": "string" }, + "node_id": { "type": "string" }, + "owner": { + "anyOf": [ + { + "type": "null" + }, + { + "type": "object", + "properties": { + "name": { "type": ["string", "null"] }, + "email": { "type": ["string", "null"] }, + "login": { "type": "string" }, + "id": { "type": "integer" }, + "node_id": { "type": "string" }, + "avatar_url": { "type": "string" }, + "gravatar_id": { "type": ["string", "null"] }, + "url": { "type": "string" }, + "html_url": { "type": "string" }, + "followers_url": { "type": "string" }, + "following_url": { "type": "string" }, + "gists_url": { "type": "string" }, + "starred_url": { "type": "string" }, + "subscriptions_url": { "type": "string" }, + "organizations_url": { "type": "string" }, + "repos_url": { "type": "string" }, + "events_url": { "type": "string" }, + "received_events_url": { "type": "string" }, + "type": { "type": "string" }, + "site_admin": { "type": "boolean" }, + "starred_at": { + "type": "string", + "format": "date-time" + } + } + } + ] + }, + "name": { "type": "string" }, + "description": { "type": ["string", "null"] }, + "external_url": { + "type": "string" + }, + "html_url": { + "type": "string" + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "updated_at": { + "type": "string", + "format": "date-time" + }, + "permissions": { + "type": "object", + "properties": { + "issues": { + "type": "string" + }, + "checks": { + "type": "string" + }, + "metadata": { + "type": "string" + }, + "contents": { + "type": "string" + }, + "deployments": { + "type": "string" + } + }, + "additionalProperties": true + }, + "events": { + "type": "array", + "items": { + "type": "string" + }, + "examples": ["label", "deployment"] + }, + "installations_count": { + "type": "integer", + "examples": [5] + }, + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + }, + "webhook_secret": { + "type": ["string", "null"] + }, + "pem": { + "type": "string" + } + } + } + ] + }, + "author_association": { + "type": "string" + }, + "reactions": { + "type": "object", + "properties": { + "url": { + "type": "string" + }, + "total_count": { + "type": "integer" + }, + "+1": { + "type": "integer" + }, + "-1": { + "type": "integer" + }, + "laugh": { + "type": "integer" + }, + "confused": { + "type": "integer" + }, + "heart": { + "type": "integer" + }, + "hooray": { + "type": "integer" + }, + "eyes": { + "type": "integer" + }, + "rocket": { + "type": "integer" + } + } + } + } + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/schemas/shared/events/reviewed.json b/airbyte-integrations/connectors/source-github/source_github/schemas/shared/events/reviewed.json new file mode 100644 index 000000000000..50fc00c05395 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/source_github/schemas/shared/events/reviewed.json @@ -0,0 +1,139 @@ +{ + "title": "Timeline Reviewed Event", + "description": "Timeline Reviewed Event", + "type": "object", + "properties": { + "event": { + "type": "string" + }, + "id": { + "type": "integer" + }, + "node_id": { + "type": "string" + }, + "user": { + "title": "Simple User", + "description": "A GitHub user.", + "type": "object", + "properties": { + "name": { + "type": ["string", "null"] + }, + "email": { + "type": ["string", "null"] + }, + "login": { + "type": "string" + }, + "id": { + "type": "integer" + }, + "node_id": { + "type": "string" + }, + "avatar_url": { + "type": "string" + }, + "gravatar_id": { + "type": ["string", "null"] + }, + "url": { + "type": "string" + }, + "html_url": { + "type": "string" + }, + "followers_url": { + "type": "string" + }, + "following_url": { + "type": "string" + }, + "gists_url": { + "type": "string" + }, + "starred_url": { + "type": "string" + }, + "subscriptions_url": { + "type": "string" + }, + "organizations_url": { + "type": "string" + }, + "repos_url": { + "type": "string" + }, + "events_url": { + "type": "string" + }, + "received_events_url": { + "type": "string" + }, + "type": { + "type": "string" + }, + "site_admin": { + "type": "boolean" + }, + "starred_at": { + "type": "string", + "format": "date-time" + } + } + }, + "body": { + "type": ["string", "null"] + }, + "state": { + "type": "string" + }, + "html_url": { + "type": "string" + }, + "pull_request_url": { + "type": "string" + }, + "_links": { + "type": "object", + "properties": { + "html": { + "type": "object", + "properties": { + "href": { + "type": "string" + } + }, + "required": ["href"] + }, + "pull_request": { + "type": "object", + "properties": { + "href": { + "type": "string" + } + }, + "required": ["href"] + } + }, + "required": ["html", "pull_request"] + }, + "submitted_at": { + "type": "string", + "format": "date-time" + }, + "commit_id": { + "type": "string" + }, + "body_html": { + "type": "string" + }, + "body_text": { + "type": "string" + }, + "author_association": { + "type": "string" + } + } +} diff --git a/airbyte-integrations/connectors/source-github/source_github/source.py b/airbyte-integrations/connectors/source-github/source_github/source.py index 5c42fab09aec..04e02fbadf21 100644 --- a/airbyte-integrations/connectors/source-github/source_github/source.py +++ b/airbyte-integrations/connectors/source-github/source_github/source.py @@ -2,7 +2,9 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import Any, Dict, List, Mapping, Set, Tuple +from os import getenv +from typing import Any, Dict, List, Mapping, MutableMapping, Tuple +from urllib.parse import urlparse from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import FailureType, SyncMode @@ -21,6 +23,7 @@ CommitCommentReactions, CommitComments, Commits, + ContributorActivity, Deployments, Events, IssueCommentReactions, @@ -29,10 +32,12 @@ IssueMilestones, IssueReactions, Issues, + IssueTimelineEvents, Organizations, ProjectCards, ProjectColumns, Projects, + ProjectsV2, PullRequestCommentReactions, PullRequestCommits, PullRequests, @@ -56,27 +61,15 @@ class SourceGithub(AbstractSource): - @staticmethod - def _get_and_prepare_repositories_config(config: Mapping[str, Any]) -> Set[str]: - """ - _get_and_prepare_repositories_config gets set of repositories names from config and removes simple errors that user could provide - Args: - config: Dict representing connector's config - Returns: - set of provided repositories - """ - config_repositories = set(filter(None, config["repository"].split(" "))) - return config_repositories - @staticmethod def _get_org_repositories(config: Mapping[str, Any], authenticator: MultipleTokenAuthenticator) -> Tuple[List[str], List[str]]: """ - Parse config.repository and produce two lists: organizations, repositories. + Parse config/repositories and produce two lists: organizations, repositories. Args: config (dict): Dict representing connector's config authenticator(MultipleTokenAuthenticator): authenticator object """ - config_repositories = SourceGithub._get_and_prepare_repositories_config(config) + config_repositories = set(config.get("repositories")) repositories = set() organizations = set() @@ -91,7 +84,7 @@ def _get_org_repositories(config: Mapping[str, Any], authenticator: MultipleToke unchecked_repos.add(org_repos) if unchecked_orgs: - stream = Repositories(authenticator=authenticator, organizations=unchecked_orgs) + stream = Repositories(authenticator=authenticator, organizations=unchecked_orgs, api_url=config.get("api_url")) for record in read_full_refresh(stream): repositories.add(record["full_name"]) organizations.add(record["organization"]) @@ -101,6 +94,7 @@ def _get_org_repositories(config: Mapping[str, Any], authenticator: MultipleToke stream = RepositoryStats( authenticator=authenticator, repositories=unchecked_repos, + api_url=config.get("api_url"), # This parameter is deprecated and in future will be used sane default, page_size: 10 page_size_for_large_streams=config.get("page_size_for_large_streams", constants.DEFAULT_PAGE_SIZE_FOR_LARGE_STREAM), ) @@ -138,9 +132,52 @@ def _get_authenticator(self, config: Mapping[str, Any]): ) return MultipleTokenAuthenticator(tokens=tokens, auth_method="token") + def _validate_and_transform_config(self, config: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + config = self._ensure_default_values(config) + config = self._validate_repositories(config) + config = self._validate_branches(config) + return config + + def _ensure_default_values(self, config: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + config.setdefault("api_url", "https://api.github.com") + api_url_parsed = urlparse(config["api_url"]) + + if not api_url_parsed.scheme.startswith("http"): + message = "Please enter a full url for `API URL` field starting with `http`" + elif api_url_parsed.scheme == "http" and not self._is_http_allowed(): + message = "HTTP connection is insecure and is not allowed in this environment. Please use `https` instead." + elif not api_url_parsed.netloc: + message = "Please provide a correct API URL." + else: + return config + + raise AirbyteTracedException(message=message, failure_type=FailureType.config_error) + + def _validate_repositories(self, config: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + if config.get("repositories"): + pass + elif config.get("repository"): + config["repositories"] = set(filter(None, config["repository"].split(" "))) + + return config + + def _validate_branches(self, config: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + if config.get("branches"): + pass + elif config.get("branch"): + config["branches"] = set(filter(None, config["branch"].split(" "))) + + return config + @staticmethod - def _get_branches_data(selected_branches: str, full_refresh_args: Dict[str, Any] = None) -> Tuple[Dict[str, str], Dict[str, List[str]]]: - selected_branches = set(filter(None, selected_branches.split(" "))) + def _is_http_allowed() -> bool: + return getenv("DEPLOYMENT_MODE", "").upper() != "CLOUD" + + @staticmethod + def _get_branches_data( + selected_branches: List, full_refresh_args: Dict[str, Any] = None + ) -> Tuple[Dict[str, str], Dict[str, List[str]]]: + selected_branches = set(selected_branches) # Get the default branch for each repository default_branches = {} @@ -184,18 +221,24 @@ def user_friendly_error_message(self, message: str) -> str: elif "404 Client Error: Not Found for url: https://api.github.com/orgs/" in message: # 404 Client Error: Not Found for url: https://api.github.com/orgs/airbytehqBLA/repos?per_page=100 org_name = message.split("https://api.github.com/orgs/")[1].split("/")[0] - user_message = f'Organization name: "{org_name}" is unknown, "repository" config option should be updated' + user_message = f'Organization name: "{org_name}" is unknown, "repository" config option should be updated. Please validate your repository config.' elif "401 Client Error: Unauthorized for url" in message: # 401 Client Error: Unauthorized for url: https://api.github.com/orgs/datarootsio/repos?per_page=100&sort=updated&direction=desc - user_message = "Bad credentials, re-authentication or access token renewal is required" + user_message = ( + "Github credentials have expired or changed, please review your credentials and re-authenticate or renew your access token." + ) return user_message def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: + config = self._validate_and_transform_config(config) try: authenticator = self._get_authenticator(config) _, repositories = self._get_org_repositories(config=config, authenticator=authenticator) if not repositories: - return False, "Invalid repositories. Valid examples: airbytehq/airbyte airbytehq/another-repo airbytehq/* airbytehq/airbyte" + return ( + False, + "Some of the provided repositories couldn't be found. Please verify if every entered repository has a valid name and it matches the following format: airbytehq/airbyte airbytehq/another-repo airbytehq/* airbytehq/airbyte.", + ) return True, None except Exception as e: @@ -205,6 +248,7 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> def streams(self, config: Mapping[str, Any]) -> List[Stream]: authenticator = self._get_authenticator(config) + config = self._validate_and_transform_config(config) try: organizations, repositories = self._get_org_repositories(config=config, authenticator=authenticator) except Exception as e: @@ -221,6 +265,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: user_message = ( "No streams available. Looks like your config for repositories or organizations is not valid." " Please, check your permissions, names of repositories and organizations." + " Needed scopes: repo, read:org, read:repo_hook, read:user, read:discussion, workflow." ) raise AirbyteTracedException( internal_message="No streams available. Please check permissions", @@ -232,18 +277,25 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: page_size = config.get("page_size_for_large_streams", constants.DEFAULT_PAGE_SIZE_FOR_LARGE_STREAM) access_token_type, _ = self.get_access_token(config) - organization_args = {"authenticator": authenticator, "organizations": organizations, "access_token_type": access_token_type} - organization_args_with_start_date = {**organization_args, "start_date": config["start_date"]} + organization_args = { + "authenticator": authenticator, + "organizations": organizations, + "api_url": config.get("api_url"), + "access_token_type": access_token_type, + } + start_date = config.get("start_date") + organization_args_with_start_date = {**organization_args, "start_date": start_date} repository_args = { "authenticator": authenticator, + "api_url": config.get("api_url"), "repositories": repositories, "page_size_for_large_streams": page_size, "access_token_type": access_token_type, } - repository_args_with_start_date = {**repository_args, "start_date": config["start_date"]} + repository_args_with_start_date = {**repository_args, "start_date": start_date} - default_branches, branches_to_pull = self._get_branches_data(config.get("branch", ""), repository_args) + default_branches, branches_to_pull = self._get_branches_data(config.get("branch", []), repository_args) pull_requests_stream = PullRequests(**repository_args_with_start_date) projects_stream = Projects(**repository_args_with_start_date) project_columns_stream = ProjectColumns(projects_stream, **repository_args_with_start_date) @@ -252,6 +304,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: workflow_runs_stream = WorkflowRuns(**repository_args_with_start_date) return [ + IssueTimelineEvents(**repository_args), Assignees(**repository_args), Branches(**repository_args), Collaborators(**repository_args), @@ -259,6 +312,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: CommitCommentReactions(**repository_args_with_start_date), CommitComments(**repository_args_with_start_date), Commits(**repository_args_with_start_date, branches_to_pull=branches_to_pull, default_branches=default_branches), + ContributorActivity(**repository_args), Deployments(**repository_args_with_start_date), Events(**repository_args_with_start_date), IssueCommentReactions(**repository_args_with_start_date), @@ -274,6 +328,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: PullRequestCommentReactions(**repository_args_with_start_date), PullRequestCommits(parent=pull_requests_stream, **repository_args), PullRequestStats(**repository_args_with_start_date), + ProjectsV2(**repository_args_with_start_date), pull_requests_stream, Releases(**repository_args_with_start_date), Repositories(**organization_args_with_start_date), diff --git a/airbyte-integrations/connectors/source-github/source_github/spec.json b/airbyte-integrations/connectors/source-github/source_github/spec.json index 5c2b905915eb..8c24d76278e7 100644 --- a/airbyte-integrations/connectors/source-github/source_github/spec.json +++ b/airbyte-integrations/connectors/source-github/source_github/spec.json @@ -4,7 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "GitHub Source Spec", "type": "object", - "required": ["start_date", "repository"], + "required": ["credentials", "repositories"], "additionalProperties": true, "properties": { "credentials": { @@ -64,15 +64,6 @@ } ] }, - "start_date": { - "type": "string", - "title": "Start date", - "description": "The date from which you'd like to replicate data from GitHub in the format YYYY-MM-DDT00:00:00Z. For the streams which support this configuration, only data generated on or after the start date will be replicated. This field doesn't apply to all streams, see the docs for more info", - "examples": ["2021-03-01T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "order": 1, - "format": "date-time" - }, "repository": { "type": "string", "examples": [ @@ -81,17 +72,63 @@ "airbytehq/airbyte" ], "title": "GitHub Repositories", - "description": "Space-delimited list of GitHub organizations/repositories, e.g. `airbytehq/airbyte` for single repository, `airbytehq/*` for get all repositories from organization and `airbytehq/airbyte airbytehq/another-repo` for multiple repositories.", - "order": 2, + "description": "(DEPRCATED) Space-delimited list of GitHub organizations/repositories, e.g. `airbytehq/airbyte` for single repository, `airbytehq/*` for get all repositories from organization and `airbytehq/airbyte airbytehq/another-repo` for multiple repositories.", + "airbyte_hidden": true, "pattern": "^([\\w.-]+/(\\*|[\\w.-]+(?docs for more info", + "examples": ["2021-03-01T00:00:00Z"], + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "pattern_descriptor": "YYYY-MM-DDTHH:mm:ssZ", + "order": 2, + "format": "date-time" + }, + "api_url": { + "type": "string", + "examples": ["https://github.com", "https://github.company.org"], + "title": "API URL", + "default": "https://api.github.com/", + "description": "Please enter your basic URL from self-hosted GitHub instance or leave it empty to use GitHub.", + "order": 3 + }, "branch": { "type": "string", "title": "Branch", "examples": ["airbytehq/airbyte/master airbytehq/airbyte/my-branch"], - "description": "Space-delimited list of GitHub repository branches to pull commits for, e.g. `airbytehq/airbyte/master`. If no branches are specified for a repository, the default branch will be pulled.", - "order": 3, + "description": "(DEPRCATED) Space-delimited list of GitHub repository branches to pull commits for, e.g. `airbytehq/airbyte/master`. If no branches are specified for a repository, the default branch will be pulled.", + "airbyte_hidden": true, + "pattern_descriptor": "org/repo/branch1 org/repo/branch2" + }, + "branches": { + "type": "array", + "items": { + "type": "string" + }, + "title": "Branches", + "examples": ["airbytehq/airbyte/master airbytehq/airbyte/my-branch"], + "description": "List of GitHub repository branches to pull commits for, e.g. `airbytehq/airbyte/master`. If no branches are specified for a repository, the default branch will be pulled.", + "order": 4, "pattern_descriptor": "org/repo/branch1 org/repo/branch2" }, "requests_per_hour": { @@ -99,7 +136,7 @@ "title": "Max requests per hour", "description": "The GitHub API allows for a maximum of 5000 requests per hour (15000 for Github Enterprise). You can specify a lower value to limit your use of the API quota.", "minimum": 1, - "order": 4 + "order": 5 } } }, diff --git a/airbyte-integrations/connectors/source-github/source_github/streams.py b/airbyte-integrations/connectors/source-github/source_github/streams.py index de37b2446fa3..3f7d710d04e8 100644 --- a/airbyte-integrations/connectors/source-github/source_github/streams.py +++ b/airbyte-integrations/connectors/source-github/source_github/streams.py @@ -9,19 +9,26 @@ import pendulum import requests -from airbyte_cdk.models import SyncMode +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level, SyncMode +from airbyte_cdk.models import Type as MessageType from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException from requests.exceptions import HTTPError from . import constants -from .graphql import CursorStorage, QueryReactions, get_query_issue_reactions, get_query_pull_requests, get_query_reviews +from .graphql import ( + CursorStorage, + QueryReactions, + get_query_issue_reactions, + get_query_projectsV2, + get_query_pull_requests, + get_query_reviews, +) from .utils import getter -class GithubStream(HttpStream, ABC): - url_base = "https://api.github.com/" +class GithubStreamABC(HttpStream, ABC): primary_key = "id" @@ -30,30 +37,20 @@ class GithubStream(HttpStream, ABC): stream_base_params = {} - def __init__(self, repositories: List[str], page_size_for_large_streams: int, access_token_type: str = "", **kwargs): + def __init__(self, api_url: str = "https://api.github.com", access_token_type: str = "", **kwargs): super().__init__(**kwargs) - self.repositories = repositories - self.access_token_type = access_token_type - # GitHub pagination could be from 1 to 100. - # This parameter is deprecated and in future will be used sane default, page_size: 10 - self.page_size = page_size_for_large_streams if self.large_stream else constants.DEFAULT_PAGE_SIZE + self.access_token_type = access_token_type + self.api_url = api_url - MAX_RETRIES = 3 - adapter = requests.adapters.HTTPAdapter(max_retries=MAX_RETRIES) - self._session.mount("https://", adapter) + @property + def url_base(self) -> str: + return self.api_url @property def availability_strategy(self) -> Optional["AvailabilityStrategy"]: return None - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return f"repos/{stream_slice['repository']}/{self.name}" - - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: - for repository in self.repositories: - yield {"repository": repository} - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: links = response.links if "next" in links: @@ -62,13 +59,32 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, page = dict(parse.parse_qsl(parsed_link.query)).get("page") return {"page": page} - def check_graphql_rate_limited(self, response_json) -> bool: - errors = response_json.get("errors") - if errors: - for error in errors: - if error.get("type") == "RATE_LIMITED": - return True - return False + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + + params = {"per_page": self.page_size} + + if next_page_token: + params.update(next_page_token) + + params.update(self.stream_base_params) + + return params + + def request_headers(self, **kwargs) -> Mapping[str, Any]: + # Without sending `User-Agent` header we will be getting `403 Client Error: Forbidden for url` error. + return {"User-Agent": "PostmanRuntime/7.28.0"} + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + for record in response.json(): # GitHub puts records in an array. + yield self.transform(record=record, stream_slice=stream_slice) def should_retry(self, response: requests.Response) -> bool: if super().should_retry(response): @@ -119,15 +135,13 @@ def backoff_time(self, response: requests.Response) -> Optional[float]: if reset_time: return max(float(reset_time) - time.time(), min_backoff_time) - def get_error_display_message(self, exception: BaseException) -> Optional[str]: - if ( - isinstance(exception, DefaultBackoffException) - and exception.response.status_code == requests.codes.BAD_GATEWAY - and self.large_stream - and self.page_size > 1 - ): - return f'Please try to decrease the "Page size for large streams" below {self.page_size}. The stream "{self.name}" is a large stream, such streams can fail with 502 for high "page_size" values.' - return super().get_error_display_message(exception) + def check_graphql_rate_limited(self, response_json) -> bool: + errors = response_json.get("errors") + if errors: + for error in errors: + if error.get("type") == "RATE_LIMITED": + return True + return False def read_records(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: # get out the stream_slice parts for later use. @@ -143,11 +157,11 @@ def read_records(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iter if e.response.status_code == requests.codes.NOT_FOUND: # A lot of streams are not available for repositories owned by a user instead of an organization. if isinstance(self, Organizations): - error_msg = ( - f"Syncing `{self.__class__.__name__}` stream isn't available for organization `{stream_slice['organization']}`." - ) + error_msg = f"Syncing `{self.__class__.__name__}` stream isn't available for organization `{organisation}`." + elif isinstance(self, TeamMemberships): + error_msg = f"Syncing `{self.__class__.__name__}` stream for organization `{organisation}`, team `{stream_slice.get('team_slug')}` and user `{stream_slice.get('username')}` isn't available: User has no team membership. Skipping..." else: - error_msg = f"Syncing `{self.__class__.__name__}` stream isn't available for repository `{stream_slice['repository']}`." + error_msg = f"Syncing `{self.__class__.__name__}` stream isn't available for repository `{repository}`." elif e.response.status_code == requests.codes.FORBIDDEN: error_msg = str(e.response.json().get("message")) # When using the `check_connection` method, we should raise an error if we do not have access to the repository. @@ -188,36 +202,33 @@ def read_records(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iter self.logger.error(f"Undefined error while reading records: {e.response.text}") raise e - self.logger.warn(error_msg) + self.logger.warning(error_msg) - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params = {"per_page": self.page_size} - - if next_page_token: - params.update(next_page_token) - - params.update(self.stream_base_params) +class GithubStream(GithubStreamABC): + def __init__(self, repositories: List[str], page_size_for_large_streams: int, **kwargs): + super().__init__(**kwargs) + self.repositories = repositories + # GitHub pagination could be from 1 to 100. + # This parameter is deprecated and in future will be used sane default, page_size: 10 + self.page_size = page_size_for_large_streams if self.large_stream else constants.DEFAULT_PAGE_SIZE - return params + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"repos/{stream_slice['repository']}/{self.name}" - def request_headers(self, **kwargs) -> Mapping[str, Any]: - # Without sending `User-Agent` header we will be getting `403 Client Error: Forbidden for url` error. - return { - "User-Agent": "PostmanRuntime/7.28.0", - } + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + for repository in self.repositories: + yield {"repository": repository} - def parse_response( - self, - response: requests.Response, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[Mapping]: - for record in response.json(): # GitHub puts records in an array. - yield self.transform(record=record, stream_slice=stream_slice) + def get_error_display_message(self, exception: BaseException) -> Optional[str]: + if ( + isinstance(exception, DefaultBackoffException) + and exception.response.status_code == requests.codes.BAD_GATEWAY + and self.large_stream + and self.page_size > 1 + ): + return f'Please try to decrease the "Page size for large streams" below {self.page_size}. The stream "{self.name}" is a large stream, such streams can fail with 502 for high "page_size" values.' + return super().get_error_display_message(exception) def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any]) -> MutableMapping[str, Any]: record["repository"] = stream_slice["repository"] @@ -282,7 +293,9 @@ def _get_starting_point(self, stream_state: Mapping[str, Any], stream_slice: Map state_path = [stream_slice[k] for k in self.slice_keys] + [self.cursor_field] stream_state_value = getter(stream_state, state_path, strict=False) if stream_state_value: - return max(self._start_date, stream_state_value) + if self._start_date: + return max(self._start_date, stream_state_value) + return stream_state_value return self._start_date def get_starting_point(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any]) -> str: @@ -303,7 +316,7 @@ def read_records( sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state ): cursor_value = self.convert_cursor_value(record[self.cursor_field]) - if cursor_value > start_point: + if not start_point or cursor_value > start_point: yield record elif self.is_sorted == "desc" and cursor_value < start_point: break @@ -340,13 +353,13 @@ def parse_response(self, response: requests.Response, stream_slice: Mapping[str, class Assignees(GithubStream): """ - API docs: https://docs.github.com/en/rest/reference/issues#list-assignees + API docs: https://docs.github.com/en/rest/issues/assignees?apiVersion=2022-11-28#list-assignees """ class Branches(GithubStream): """ - API docs: https://docs.github.com/en/rest/reference/repos#list-branches + API docs: https://docs.github.com/en/rest/branches/branches?apiVersion=2022-11-28#list-branches """ primary_key = ["repository", "name"] @@ -357,29 +370,29 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class Collaborators(GithubStream): """ - API docs: https://docs.github.com/en/rest/reference/repos#list-repository-collaborators + API docs: https://docs.github.com/en/rest/collaborators/collaborators?apiVersion=2022-11-28#list-repository-collaborators """ class IssueLabels(GithubStream): """ - API docs: https://docs.github.com/en/rest/issues/labels#list-labels-for-a-repository + API docs: https://docs.github.com/en/rest/issues/labels?apiVersion=2022-11-28#list-labels-for-a-repository """ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: return f"repos/{stream_slice['repository']}/labels" -class Organizations(GithubStream): +class Organizations(GithubStreamABC): """ - API docs: https://docs.github.com/en/rest/reference/orgs#get-an-organization + API docs: https://docs.github.com/en/rest/orgs/orgs?apiVersion=2022-11-28#list-organizations """ # GitHub pagination could be from 1 to 100. page_size = 100 def __init__(self, organizations: List[str], access_token_type: str = "", **kwargs): - super(GithubStream, self).__init__(**kwargs) + super().__init__(**kwargs) self.organizations = organizations self.access_token_type = access_token_type @@ -400,7 +413,7 @@ def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, class Repositories(SemiIncrementalMixin, Organizations): """ - API docs: https://docs.github.com/en/rest/reference/repos#list-organization-repositories + API docs: https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-organization-repositories """ is_sorted = "desc" @@ -419,7 +432,7 @@ def parse_response(self, response: requests.Response, stream_slice: Mapping[str, class Tags(GithubStream): """ - API docs: https://docs.github.com/en/rest/reference/repos#list-repository-tags + API docs: https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-repository-tags """ primary_key = ["repository", "name"] @@ -430,7 +443,7 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class Teams(Organizations): """ - API docs: https://docs.github.com/en/rest/reference/teams#list-teams + API docs: https://docs.github.com/en/rest/teams/teams?apiVersion=2022-11-28#list-teams """ use_cache = True @@ -445,7 +458,7 @@ def parse_response(self, response: requests.Response, stream_slice: Mapping[str, class Users(Organizations): """ - API docs: https://docs.github.com/en/rest/reference/orgs#list-organization-members + API docs: https://docs.github.com/en/rest/orgs/members?apiVersion=2022-11-28#list-organization-members """ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: @@ -461,7 +474,7 @@ def parse_response(self, response: requests.Response, stream_slice: Mapping[str, class Releases(SemiIncrementalMixin, GithubStream): """ - API docs: https://docs.github.com/en/rest/reference/repos#list-releases + API docs: https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#list-releases """ cursor_field = "created_at" @@ -479,7 +492,7 @@ def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, class Events(SemiIncrementalMixin, GithubStream): """ - API docs: https://docs.github.com/en/rest/reference/activity#list-repository-events + API docs: https://docs.github.com/en/rest/activity/events?apiVersion=2022-11-28#list-repository-events """ cursor_field = "created_at" @@ -487,7 +500,7 @@ class Events(SemiIncrementalMixin, GithubStream): class PullRequests(SemiIncrementalMixin, GithubStream): """ - API docs: https://docs.github.com/en/rest/pulls/pulls#list-pull-requests + API docs: https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28#list-pull-requests """ use_cache = True @@ -536,7 +549,7 @@ def is_sorted(self) -> str: class CommitComments(SemiIncrementalMixin, GithubStream): """ - API docs: https://docs.github.com/en/rest/reference/repos#list-commit-comments-for-a-repository + API docs: https://docs.github.com/en/rest/commits/comments?apiVersion=2022-11-28#list-commit-comments-for-a-repository """ use_cache = True @@ -547,7 +560,7 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class IssueMilestones(SemiIncrementalMixin, GithubStream): """ - API docs: https://docs.github.com/en/rest/reference/issues#list-milestones + API docs: https://docs.github.com/en/rest/issues/milestones?apiVersion=2022-11-28#list-milestones """ is_sorted = "desc" @@ -563,7 +576,7 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class Stargazers(SemiIncrementalMixin, GithubStream): """ - API docs: https://docs.github.com/en/rest/reference/activity#list-stargazers + API docs: https://docs.github.com/en/rest/activity/starring?apiVersion=2022-11-28#list-stargazers """ primary_key = "user_id" @@ -589,7 +602,7 @@ def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, class Projects(SemiIncrementalMixin, GithubStream): """ - API docs: https://docs.github.com/en/rest/reference/projects#list-repository-projects + API docs: https://docs.github.com/en/rest/projects/projects?apiVersion=2022-11-28#list-repository-projects """ use_cache = True @@ -608,7 +621,7 @@ def request_headers(self, **kwargs) -> Mapping[str, Any]: class IssueEvents(SemiIncrementalMixin, GithubStream): """ - API docs: https://docs.github.com/en/rest/reference/issues#list-issue-events-for-a-repository + API docs: https://docs.github.com/en/rest/issues/events?apiVersion=2022-11-28#list-issue-events-for-a-repository """ cursor_field = "created_at" @@ -622,7 +635,7 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class Comments(IncrementalMixin, GithubStream): """ - API docs: https://docs.github.com/en/rest/reference/issues#list-issue-comments-for-a-repository + API docs: https://docs.github.com/en/rest/issues/comments?apiVersion=2022-11-28#list-issue-comments-for-a-repository """ use_cache = True @@ -635,7 +648,7 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class Commits(IncrementalMixin, GithubStream): """ - API docs: https://docs.github.com/en/rest/reference/repos#list-commits + API docs: https://docs.github.com/en/rest/commits/commits?apiVersion=2022-11-28#list-commits Pull commits from each branch of each repository, tracking state for each branch """ @@ -651,7 +664,9 @@ def __init__(self, branches_to_pull: Mapping[str, List[str]], default_branches: def request_params(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: params = super(IncrementalMixin, self).request_params(stream_state=stream_state, stream_slice=stream_slice, **kwargs) - params["since"] = self.get_starting_point(stream_state=stream_state, stream_slice=stream_slice) + since = self.get_starting_point(stream_state=stream_state, stream_slice=stream_slice) + if since: + params["since"] = since params["sha"] = stream_slice["branch"] return params @@ -687,7 +702,7 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late class Issues(IncrementalMixin, GithubStream): """ - API docs: https://docs.github.com/en/rest/issues/issues#list-repository-issues + API docs: https://docs.github.com/en/rest/issues/issues?apiVersion=2022-11-28#list-repository-issues """ use_cache = True @@ -703,7 +718,7 @@ class Issues(IncrementalMixin, GithubStream): class ReviewComments(IncrementalMixin, GithubStream): """ - API docs: https://docs.github.com/en/rest/reference/pulls#list-review-comments-in-a-repository + API docs: https://docs.github.com/en/rest/pulls/comments?apiVersion=2022-11-28#list-review-comments-in-a-repository """ use_cache = True @@ -713,12 +728,8 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: return f"repos/{stream_slice['repository']}/pulls/comments" -class PullRequestStats(SemiIncrementalMixin, GithubStream): - """ - API docs: https://docs.github.com/en/graphql/reference/objects#pullrequest - """ +class GitHubGraphQLStream(GithubStream, ABC): - is_sorted = "asc" http_method = "POST" def path( @@ -726,15 +737,31 @@ def path( ) -> str: return "graphql" - def raise_error_from_response(self, response_json): - if "errors" in response_json: - raise Exception(str(response_json["errors"])) + def should_retry(self, response: requests.Response) -> bool: + if response.status_code in (requests.codes.BAD_GATEWAY, requests.codes.GATEWAY_TIMEOUT): + self.page_size = int(self.page_size / 2) + return True + self.page_size = constants.DEFAULT_PAGE_SIZE_FOR_LARGE_STREAM if self.large_stream else constants.DEFAULT_PAGE_SIZE + return super().should_retry(response) or response.json().get("errors") - def _get_name(self, repository): + def _get_repository_name(self, repository: Mapping[str, Any]) -> str: return repository["owner"]["login"] + "/" + repository["name"] + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {} + + +class PullRequestStats(SemiIncrementalMixin, GitHubGraphQLStream): + """ + API docs: https://docs.github.com/en/graphql/reference/objects#pullrequest + """ + + large_stream = True + is_sorted = "asc" + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - self.raise_error_from_response(response_json=response.json()) repository = response.json()["data"]["repository"] if repository: nodes = repository["pullRequests"]["nodes"] @@ -742,7 +769,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp record["review_comments"] = sum([node["comments"]["totalCount"] for node in record["review_comments"]["nodes"]]) record["comments"] = record["comments"]["totalCount"] record["commits"] = record["commits"]["totalCount"] - record["repository"] = self._get_name(repository) + record["repository"] = self._get_repository_name(repository) if record["merged_by"]: record["merged_by"]["type"] = record["merged_by"].pop("__typename") yield record @@ -754,11 +781,6 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, if pageInfo["hasNextPage"]: return {"after": pageInfo["endCursor"]} - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return {} - def request_body_json( self, stream_state: Mapping[str, Any], @@ -780,13 +802,12 @@ def request_headers(self, **kwargs) -> Mapping[str, Any]: return {**base_headers, **headers} -class Reviews(SemiIncrementalMixin, GithubStream): +class Reviews(SemiIncrementalMixin, GitHubGraphQLStream): """ - API docs: https://docs.github.com/en/graphql/reference/objects#pullrequestreview + API docs: https://docs.github.com/en/rest/pulls/reviews?apiVersion=2022-11-28#list-reviews-for-a-pull-request """ is_sorted = False - http_method = "POST" cursor_field = "updated_at" def __init__(self, **kwargs): @@ -794,20 +815,6 @@ def __init__(self, **kwargs): self.pull_requests_cursor = {} self.reviews_cursors = {} - def path( - self, *, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "graphql" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return {} - - def raise_error_from_response(self, response_json): - if "errors" in response_json: - raise Exception(str(response_json["errors"])) - def _get_records(self, pull_request, repository_name): "yield review records from pull_request" for record in pull_request["reviews"]["nodes"]: @@ -824,14 +831,10 @@ def _get_records(self, pull_request, repository_name): } yield record - def _get_name(self, repository): - return repository["owner"]["login"] + "/" + repository["name"] - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - self.raise_error_from_response(response_json=response.json()) repository = response.json()["data"]["repository"] if repository: - repository_name = self._get_name(repository) + repository_name = self._get_repository_name(repository) if "pullRequests" in repository: for pull_request in repository["pullRequests"]["nodes"]: yield from self._get_records(pull_request, repository_name) @@ -841,7 +844,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: repository = response.json()["data"]["repository"] if repository: - repository_name = self._get_name(repository) + repository_name = self._get_repository_name(repository) reviews_cursors = self.reviews_cursors.setdefault(repository_name, {}) if "pullRequests" in repository: if repository["pullRequests"]["pageInfo"]["hasNextPage"]: @@ -875,7 +878,7 @@ def request_body_json( class PullRequestCommits(GithubStream): """ - API docs: https://docs.github.com/en/rest/reference/pulls#list-commits-on-a-pull-request + API docs: https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28#list-commits-on-a-pull-request """ primary_key = "sha" @@ -906,6 +909,44 @@ def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, return record +class ProjectsV2(SemiIncrementalMixin, GitHubGraphQLStream): + """ + API docs: https://docs.github.com/en/graphql/reference/objects#projectv2 + """ + + is_sorted = "asc" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + repository = response.json()["data"]["repository"] + if repository: + nodes = repository["projectsV2"]["nodes"] + for record in nodes: + record["owner_id"] = record.pop("owner").get("id") + record["repository"] = self._get_repository_name(repository) + yield record + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + repository = response.json()["data"]["repository"] + if repository: + page_info = repository["projectsV2"]["pageInfo"] + if page_info["hasNextPage"]: + return {"after": page_info["endCursor"]} + + def request_body_json( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Optional[Mapping]: + organization, name = stream_slice["repository"].split("/") + if next_page_token: + next_page_token = next_page_token["after"] + query = get_query_projectsV2( + owner=organization, name=name, first=self.page_size, after=next_page_token, direction=self.is_sorted.upper() + ) + return {"query": query} + + # Reactions streams @@ -953,7 +994,9 @@ def get_starting_point(self, stream_state: Mapping[str, Any], stream_slice: Mapp parent_id = str(stream_slice[self.copy_parent_key]) stream_state_value = stream_state.get(repository, {}).get(parent_id, {}).get(self.cursor_field) if stream_state_value: - return max(self._start_date, stream_state_value) + if self._start_date: + return max(self._start_date, stream_state_value) + return stream_state_value return self._start_date def read_records( @@ -967,7 +1010,7 @@ def read_records( for record in super().read_records( sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state ): - if record[self.cursor_field] > starting_point: + if not starting_point or record[self.cursor_field] > starting_point: yield record def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any]) -> MutableMapping[str, Any]: @@ -978,7 +1021,7 @@ def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, class CommitCommentReactions(ReactionStream): """ - API docs: https://docs.github.com/en/rest/reference/reactions#list-reactions-for-a-commit-comment + API docs: https://docs.github.com/en/rest/reference/reactions?apiVersion=2022-11-28#list-reactions-for-a-commit-comment """ parent_entity = CommitComments @@ -986,19 +1029,18 @@ class CommitCommentReactions(ReactionStream): class IssueCommentReactions(ReactionStream): """ - API docs: https://docs.github.com/en/rest/reference/reactions#list-reactions-for-an-issue-comment + API docs: https://docs.github.com/en/rest/reactions/reactions?apiVersion=2022-11-28#list-reactions-for-an-issue-comment """ parent_entity = Comments -class IssueReactions(SemiIncrementalMixin, GithubStream): +class IssueReactions(SemiIncrementalMixin, GitHubGraphQLStream): """ https://docs.github.com/en/graphql/reference/objects#issue https://docs.github.com/en/graphql/reference/objects#reaction """ - http_method = "POST" cursor_field = "created_at" def __init__(self, **kwargs): @@ -1006,18 +1048,6 @@ def __init__(self, **kwargs): self.issues_cursor = {} self.reactions_cursors = {} - def path( - self, *, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "graphql" - - def raise_error_from_response(self, response_json): - if "errors" in response_json: - raise Exception(str(response_json["errors"])) - - def _get_name(self, repository): - return repository["owner"]["login"] + "/" + repository["name"] - def _get_reactions_from_issue(self, issue, repository_name): for reaction in issue["reactions"]["nodes"]: reaction["repository"] = repository_name @@ -1026,10 +1056,9 @@ def _get_reactions_from_issue(self, issue, repository_name): yield reaction def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - self.raise_error_from_response(response_json=response.json()) repository = response.json()["data"]["repository"] if repository: - repository_name = self._get_name(repository) + repository_name = self._get_repository_name(repository) if "issues" in repository: for issue in repository["issues"]["nodes"]: yield from self._get_reactions_from_issue(issue, repository_name) @@ -1039,7 +1068,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: repository = response.json()["data"]["repository"] if repository: - repository_name = self._get_name(repository) + repository_name = self._get_repository_name(repository) reactions_cursors = self.reactions_cursors.setdefault(repository_name, {}) if "issues" in repository: if repository["issues"]["pageInfo"]["hasNextPage"]: @@ -1071,14 +1100,13 @@ def request_body_json( return {"query": query} -class PullRequestCommentReactions(SemiIncrementalMixin, GithubStream): +class PullRequestCommentReactions(SemiIncrementalMixin, GitHubGraphQLStream): """ API docs: https://docs.github.com/en/graphql/reference/objects#pullrequestreviewcomment https://docs.github.com/en/graphql/reference/objects#reaction """ - http_method = "POST" cursor_field = "created_at" def __init__(self, **kwargs): @@ -1086,21 +1114,9 @@ def __init__(self, **kwargs): self.cursor_storage = CursorStorage(["PullRequest", "PullRequestReview", "PullRequestReviewComment", "Reaction"]) self.query_reactions = QueryReactions() - def path( - self, *, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "graphql" - - def raise_error_from_response(self, response_json): - if "errors" in response_json: - raise Exception(str(response_json["errors"])) - - def _get_name(self, repository): - return repository["owner"]["login"] + "/" + repository["name"] - def _get_reactions_from_comment(self, comment, repository): for reaction in comment["reactions"]["nodes"]: - reaction["repository"] = self._get_name(repository) + reaction["repository"] = self._get_repository_name(repository) reaction["comment_id"] = comment["id"] if reaction["user"]: reaction["user"]["type"] = "User" @@ -1119,7 +1135,6 @@ def _get_reactions_from_repository(self, repository): yield from self._get_reactions_from_pull_request(pull_request, repository) def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - self.raise_error_from_response(response_json=response.json()) data = response.json()["data"] repository = data.get("repository") if repository: @@ -1177,11 +1192,6 @@ def _add_cursor(self, node, link): link_to_object[link], pageInfo["endCursor"], node[link]["totalCount"], parent_id=node.get("node_id") ) - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return {} - def request_body_json( self, stream_state: Mapping[str, Any], @@ -1208,7 +1218,7 @@ def request_body_json( class Deployments(SemiIncrementalMixin, GithubStream): """ - API docs: https://docs.github.com/en/rest/deployments/deployments#list-deployments + API docs: https://docs.github.com/en/rest/deployments/deployments?apiVersion=2022-11-28#list-deployments """ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: @@ -1217,7 +1227,7 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class ProjectColumns(GithubStream): """ - API docs: https://docs.github.com/en/rest/reference/projects#list-project-columns + API docs: https://docs.github.com/en/rest/projects/columns?apiVersion=2022-11-28#list-project-columns """ use_cache = True @@ -1255,7 +1265,7 @@ def read_records( for record in super().read_records( sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state ): - if record[self.cursor_field] > starting_point: + if not starting_point or record[self.cursor_field] > starting_point: yield record def get_starting_point(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any]) -> str: @@ -1264,7 +1274,9 @@ def get_starting_point(self, stream_state: Mapping[str, Any], stream_slice: Mapp project_id = str(stream_slice["project_id"]) stream_state_value = stream_state.get(repository, {}).get(project_id, {}).get(self.cursor_field) if stream_state_value: - return max(self._start_date, stream_state_value) + if self._start_date: + return max(self._start_date, stream_state_value) + return stream_state_value return self._start_date def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): @@ -1285,10 +1297,11 @@ def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, class ProjectCards(GithubStream): """ - API docs: https://docs.github.com/en/rest/reference/projects#list-project-cards + API docs: https://docs.github.com/en/rest/projects/cards?apiVersion=2022-11-28#list-project-cards """ cursor_field = "updated_at" + stream_base_params = {"archived_state": "all"} def __init__(self, parent: HttpStream, start_date: str, **kwargs): super().__init__(**kwargs) @@ -1322,7 +1335,7 @@ def read_records( for record in super().read_records( sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state ): - if record[self.cursor_field] > starting_point: + if not starting_point or record[self.cursor_field] > starting_point: yield record def get_starting_point(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any]) -> str: @@ -1332,7 +1345,9 @@ def get_starting_point(self, stream_state: Mapping[str, Any], stream_slice: Mapp column_id = str(stream_slice["column_id"]) stream_state_value = stream_state.get(repository, {}).get(project_id, {}).get(column_id, {}).get(self.cursor_field) if stream_state_value: - return max(self._start_date, stream_state_value) + if self._start_date: + return max(self._start_date, stream_state_value) + return stream_state_value return self._start_date def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): @@ -1358,7 +1373,7 @@ def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, class Workflows(SemiIncrementalMixin, GithubStream): """ Get all workflows of a GitHub repository - API documentation: https://docs.github.com/en/rest/actions/workflows#list-repository-workflows + API documentation: https://docs.github.com/en/rest/actions/workflows?apiVersion=2022-11-28#list-repository-workflows """ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: @@ -1376,7 +1391,7 @@ def convert_cursor_value(self, value): class WorkflowRuns(SemiIncrementalMixin, GithubStream): """ Get all workflow runs for a GitHub repository - API documentation: https://docs.github.com/en/rest/actions/workflow-runs#list-workflow-runs-for-a-repository + API documentation: https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-repository """ # key for accessing slice value from record @@ -1407,22 +1422,24 @@ def read_records( # workflows_runs records cannot be updated. It means if we initially fully synced stream on subsequent incremental sync we need # only to look behind on 30 days to find all records which were updated. start_point = self.get_starting_point(stream_state=stream_state, stream_slice=stream_slice) - break_point = (pendulum.parse(start_point) - pendulum.duration(days=self.re_run_period)).to_iso8601_string() + break_point = None + if start_point: + break_point = (pendulum.parse(start_point) - pendulum.duration(days=self.re_run_period)).to_iso8601_string() for record in super(SemiIncrementalMixin, self).read_records( sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state ): cursor_value = record[self.cursor_field] created_at = record["created_at"] - if cursor_value > start_point: + if not start_point or cursor_value > start_point: yield record - if created_at < break_point: + if break_point and created_at < break_point: break class WorkflowJobs(SemiIncrementalMixin, GithubStream): """ Get all workflow jobs for a workflow run - API documentation: https://docs.github.com/pt/rest/actions/workflow-jobs#list-jobs-for-a-workflow-run + API documentation: https://docs.github.com/pt/rest/actions/workflow-jobs?apiVersion=2022-11-28#list-jobs-for-a-workflow-run """ cursor_field = "completed_at" @@ -1476,7 +1493,7 @@ def request_params( class TeamMembers(GithubStream): """ - API docs: https://docs.github.com/en/rest/reference/teams#list-team-members + API docs: https://docs.github.com/en/rest/teams/members?apiVersion=2022-11-28#list-team-members """ use_cache = True @@ -1510,7 +1527,7 @@ def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, class TeamMemberships(GithubStream): """ - API docs: https://docs.github.com/en/rest/reference/teams#get-team-membership-for-a-user + API docs: https://docs.github.com/en/rest/teams/members?apiVersion=2022-11-28#get-team-membership-for-a-user """ primary_key = ["url"] @@ -1543,3 +1560,104 @@ def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, record["team_slug"] = stream_slice["team_slug"] record["username"] = stream_slice["username"] return record + + +class ContributorActivity(GithubStream): + """ + API docs: https://docs.github.com/en/rest/metrics/statistics?apiVersion=2022-11-28#get-all-contributor-commit-activity + """ + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"repos/{stream_slice['repository']}/stats/contributors" + + def request_headers(self, **kwargs) -> Mapping[str, Any]: + params = super().request_headers(**kwargs) + params.update({"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"}) + return params + + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any]) -> MutableMapping[str, Any]: + record["repository"] = stream_slice["repository"] + record.update(record.pop("author")) + return record + + def should_retry(self, response: requests.Response) -> bool: + """ + If the data hasn't been cached when you query a repository's statistics, you'll receive a 202 response, need to retry to get results + see for more info https://docs.github.com/en/rest/metrics/statistics?apiVersion=2022-11-28#a-word-about-caching + """ + if super().should_retry(response) or response.status_code == requests.codes.ACCEPTED: + return True + + def backoff_time(self, response: requests.Response) -> Optional[float]: + return 90 if response.status_code == requests.codes.ACCEPTED else super().backoff_time(response) + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + if response.status_code == requests.codes.NO_CONTENT: + self.logger.warning(f"Empty response received for {self.name} stats in repository {stream_slice.get('repository')}") + else: + yield from super().parse_response( + response, stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token + ) + + def read_records(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + repository = stream_slice.get("repository", "") + try: + yield from super().read_records(stream_slice=stream_slice, **kwargs) + except HTTPError as e: + if e.response.status_code == requests.codes.ACCEPTED: + yield AirbyteMessage( + type=MessageType.LOG, + log=AirbyteLogMessage( + level=Level.INFO, + message=f"Syncing `{self.__class__.__name__}` " f"stream isn't available for repository `{repository}`.", + ), + ) + else: + raise e + + +class IssueTimelineEvents(GithubStream): + """ + API docs https://docs.github.com/en/rest/issues/timeline?apiVersion=2022-11-28#list-timeline-events-for-an-issue + """ + + primary_key = ["repository", "issue_number"] + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.parent = Issues(**kwargs) + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"repos/{stream_slice['repository']}/issues/{stream_slice['number']}/timeline" + + def stream_slices( + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + parent_stream_slices = self.parent.stream_slices( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_state=stream_state + ) + for stream_slice in parent_stream_slices: + parent_records = self.parent.read_records( + sync_mode=SyncMode.full_refresh, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + for record in parent_records: + yield {"repository": record["repository"], "number": record["number"]} + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + events_list = response.json() + record = {"repository": stream_slice["repository"], "issue_number": stream_slice["number"]} + for event in events_list: + record[event["event"]] = event + yield record diff --git a/airbyte-integrations/connectors/source-github/unit_tests/conftest.py b/airbyte-integrations/connectors/source-github/unit_tests/conftest.py new file mode 100644 index 000000000000..c3d9c1c98188 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/unit_tests/conftest.py @@ -0,0 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import os + +os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" diff --git a/airbyte-integrations/connectors/source-github/unit_tests/projects_v2_pull_requests_query.json b/airbyte-integrations/connectors/source-github/unit_tests/projects_v2_pull_requests_query.json new file mode 100644 index 000000000000..fc9dc4995fa0 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/unit_tests/projects_v2_pull_requests_query.json @@ -0,0 +1,3 @@ +{ + "query": "query {\n repository(owner: \"airbytehq\", name: \"airbyte\") {\n name\n owner {\n login\n }\n projectsV2(first: 100, orderBy: {field: UPDATED_AT, direction: ASC}) {\n nodes {\n closed\n created_at: createdAt\n closed_at: closedAt\n updated_at: updatedAt\n creator: creator {\n avatarUrl\n login\n resourcePath\n url\n }\n node_id: id\n id: databaseId\n number\n public\n readme: readme\n short_description: shortDescription\n template\n title: title\n url: url\n viewerCanClose\n viewerCanReopen\n viewerCanUpdate\n owner {\n id: id\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n}" +} diff --git a/airbyte-integrations/connectors/source-github/unit_tests/pull_request_stats_query.json b/airbyte-integrations/connectors/source-github/unit_tests/pull_request_stats_query.json new file mode 100644 index 000000000000..b825c3696376 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/unit_tests/pull_request_stats_query.json @@ -0,0 +1,3 @@ +{ + "query": "query {\n repository(owner: \"airbytehq\", name: \"airbyte\") {\n name\n owner {\n login\n }\n pullRequests(first: 10, orderBy: {field: UPDATED_AT, direction: ASC}) {\n nodes {\n node_id: id\n id: databaseId\n number\n updated_at: updatedAt\n changed_files: changedFiles\n deletions\n additions\n merged\n mergeable\n can_be_rebased: canBeRebased\n maintainer_can_modify: maintainerCanModify\n merge_state_status: mergeStateStatus\n comments {\n totalCount\n }\n commits {\n totalCount\n }\n review_comments: reviews(first: 100) {\n totalCount\n nodes {\n comments {\n totalCount\n }\n }\n }\n merged_by: mergedBy {\n __typename\n ... on User {\n node_id: id\n id: databaseId\n login\n avatar_url: avatarUrl\n html_url: url\n site_admin: isSiteAdmin\n }\n }\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n }\n}" +} diff --git a/airbyte-integrations/connectors/source-github/unit_tests/responses/contributor_activity_response.json b/airbyte-integrations/connectors/source-github/unit_tests/responses/contributor_activity_response.json new file mode 100644 index 000000000000..1f818bd9aa3f --- /dev/null +++ b/airbyte-integrations/connectors/source-github/unit_tests/responses/contributor_activity_response.json @@ -0,0 +1,33 @@ +[ + { + "author": { + "login": "octocat", + "id": 1, + "node_id": "MDQ6VXNlcjE=", + "avatar_url": "https://github.com/images/error/octocat_happy.gif", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": false + }, + "total": 135, + "weeks": [ + { + "w": 1367712000, + "a": 6898, + "d": 77, + "c": 10 + } + ] + } +] diff --git a/airbyte-integrations/connectors/source-github/unit_tests/graphql_reviews_responses.json b/airbyte-integrations/connectors/source-github/unit_tests/responses/graphql_reviews_responses.json similarity index 100% rename from airbyte-integrations/connectors/source-github/unit_tests/graphql_reviews_responses.json rename to airbyte-integrations/connectors/source-github/unit_tests/responses/graphql_reviews_responses.json diff --git a/airbyte-integrations/connectors/source-github/unit_tests/responses/issue_timeline_events.json b/airbyte-integrations/connectors/source-github/unit_tests/responses/issue_timeline_events.json new file mode 100644 index 000000000000..026243a2c6b7 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/unit_tests/responses/issue_timeline_events.json @@ -0,0 +1,166 @@ +[ + { + "id": 6430295168, + "node_id": "LOE_lADODwFebM5HwC0kzwAAAAF_RoSA", + "url": "https://api.github.com/repos/github/roadmap/issues/events/6430295168", + "actor": { + "login": "github", + "id": 9919, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjk5MTk=", + "avatar_url": "https://avatars.githubusercontent.com/u/9919?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github", + "html_url": "https://github.com/github", + "followers_url": "https://api.github.com/users/github/followers", + "following_url": "https://api.github.com/users/github/following{/other_user}", + "gists_url": "https://api.github.com/users/github/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github/subscriptions", + "organizations_url": "https://api.github.com/users/github/orgs", + "repos_url": "https://api.github.com/users/github/repos", + "events_url": "https://api.github.com/users/github/events{/privacy}", + "received_events_url": "https://api.github.com/users/github/received_events", + "type": "Organization", + "site_admin": false + }, + "event": "locked", + "commit_id": null, + "commit_url": null, + "created_at": "2022-04-13T20:49:13Z", + "lock_reason": null, + "performed_via_github_app": null + }, + { + "id": 6430296748, + "node_id": "LE_lADODwFebM5HwC0kzwAAAAF_Roqs", + "url": "https://api.github.com/repos/github/roadmap/issues/events/6430296748", + "actor": { + "login": "github-product-roadmap", + "id": 67656570, + "node_id": "MDQ6VXNlcjY3NjU2NTcw", + "avatar_url": "https://avatars.githubusercontent.com/u/67656570?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-product-roadmap", + "html_url": "https://github.com/github-product-roadmap", + "followers_url": "https://api.github.com/users/github-product-roadmap/followers", + "following_url": "https://api.github.com/users/github-product-roadmap/following{/other_user}", + "gists_url": "https://api.github.com/users/github-product-roadmap/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-product-roadmap/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-product-roadmap/subscriptions", + "organizations_url": "https://api.github.com/users/github-product-roadmap/orgs", + "repos_url": "https://api.github.com/users/github-product-roadmap/repos", + "events_url": "https://api.github.com/users/github-product-roadmap/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-product-roadmap/received_events", + "type": "User", + "site_admin": false + }, + "event": "labeled", + "commit_id": null, + "commit_url": null, + "created_at": "2022-04-13T20:49:34Z", + "label": { + "name": "beta", + "color": "99dd88" + }, + "performed_via_github_app": null + }, + { + "id": 6635165802, + "node_id": "RTE_lADODwFebM5HwC0kzwAAAAGLfJhq", + "url": "https://api.github.com/repos/github/roadmap/issues/events/6635165802", + "actor": { + "login": "github-product-roadmap", + "id": 67656570, + "node_id": "MDQ6VXNlcjY3NjU2NTcw", + "avatar_url": "https://avatars.githubusercontent.com/u/67656570?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-product-roadmap", + "html_url": "https://github.com/github-product-roadmap", + "followers_url": "https://api.github.com/users/github-product-roadmap/followers", + "following_url": "https://api.github.com/users/github-product-roadmap/following{/other_user}", + "gists_url": "https://api.github.com/users/github-product-roadmap/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-product-roadmap/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-product-roadmap/subscriptions", + "organizations_url": "https://api.github.com/users/github-product-roadmap/orgs", + "repos_url": "https://api.github.com/users/github-product-roadmap/repos", + "events_url": "https://api.github.com/users/github-product-roadmap/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-product-roadmap/received_events", + "type": "User", + "site_admin": false + }, + "event": "renamed", + "commit_id": null, + "commit_url": null, + "created_at": "2022-05-18T19:29:01Z", + "rename": { + "from": "Secret scanning: dry-runs for enterprise-level custom patterns (cloud)", + "to": "Secret scanning: dry-runs for enterprise-level custom patterns" + }, + "performed_via_github_app": null + }, + { + "url": "https://api.github.com/repos/github/roadmap/issues/comments/1130876857", + "html_url": "https://github.com/github/roadmap/issues/493#issuecomment-1130876857", + "issue_url": "https://api.github.com/repos/github/roadmap/issues/493", + "id": 1130876857, + "node_id": "IC_kwDODwFebM5DZ8-5", + "user": { + "login": "octocat", + "id": 94867353, + "node_id": "U_kgDOBaePmQ", + "avatar_url": "https://avatars.githubusercontent.com/u/94867353?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": true + }, + "created_at": "2022-05-19T00:52:15Z", + "updated_at": "2022-05-19T00:52:15Z", + "author_association": "COLLABORATOR", + "body": "🚢 Shipped to the cloud: https://github.blog/changelog/2022-05-12-secret-scanning-dry-runs-for-enterprise-level-custom-patterns/", + "reactions": { + "url": "https://api.github.com/repos/github/roadmap/issues/comments/1130876857/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "performed_via_github_app": null, + "event": "commented", + "actor": { + "login": "octocat", + "id": 94867353, + "node_id": "U_kgDOBaePmQ", + "avatar_url": "https://avatars.githubusercontent.com/u/94867353?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": true + } + } +] diff --git a/airbyte-integrations/connectors/source-github/unit_tests/responses/issue_timeline_events_response.json b/airbyte-integrations/connectors/source-github/unit_tests/responses/issue_timeline_events_response.json new file mode 100644 index 000000000000..ee6a8dc668b8 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/unit_tests/responses/issue_timeline_events_response.json @@ -0,0 +1,170 @@ +[ + { + "repository": "airbytehq/airbyte", + "issue_number": 1, + "locked": { + "id": 6430295168, + "node_id": "LOE_lADODwFebM5HwC0kzwAAAAF_RoSA", + "url": "https://api.github.com/repos/github/roadmap/issues/events/6430295168", + "actor": { + "login": "github", + "id": 9919, + "node_id": "MDEyOk9yZ2FuaXphdGlvbjk5MTk=", + "avatar_url": "https://avatars.githubusercontent.com/u/9919?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github", + "html_url": "https://github.com/github", + "followers_url": "https://api.github.com/users/github/followers", + "following_url": "https://api.github.com/users/github/following{/other_user}", + "gists_url": "https://api.github.com/users/github/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github/subscriptions", + "organizations_url": "https://api.github.com/users/github/orgs", + "repos_url": "https://api.github.com/users/github/repos", + "events_url": "https://api.github.com/users/github/events{/privacy}", + "received_events_url": "https://api.github.com/users/github/received_events", + "type": "Organization", + "site_admin": false + }, + "event": "locked", + "commit_id": null, + "commit_url": null, + "created_at": "2022-04-13T20:49:13Z", + "lock_reason": null, + "performed_via_github_app": null + }, + "labeled": { + "id": 6430296748, + "node_id": "LE_lADODwFebM5HwC0kzwAAAAF_Roqs", + "url": "https://api.github.com/repos/github/roadmap/issues/events/6430296748", + "actor": { + "login": "github-product-roadmap", + "id": 67656570, + "node_id": "MDQ6VXNlcjY3NjU2NTcw", + "avatar_url": "https://avatars.githubusercontent.com/u/67656570?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-product-roadmap", + "html_url": "https://github.com/github-product-roadmap", + "followers_url": "https://api.github.com/users/github-product-roadmap/followers", + "following_url": "https://api.github.com/users/github-product-roadmap/following{/other_user}", + "gists_url": "https://api.github.com/users/github-product-roadmap/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-product-roadmap/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-product-roadmap/subscriptions", + "organizations_url": "https://api.github.com/users/github-product-roadmap/orgs", + "repos_url": "https://api.github.com/users/github-product-roadmap/repos", + "events_url": "https://api.github.com/users/github-product-roadmap/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-product-roadmap/received_events", + "type": "User", + "site_admin": false + }, + "event": "labeled", + "commit_id": null, + "commit_url": null, + "created_at": "2022-04-13T20:49:34Z", + "label": { + "name": "beta", + "color": "99dd88" + }, + "performed_via_github_app": null + }, + "renamed": { + "id": 6635165802, + "node_id": "RTE_lADODwFebM5HwC0kzwAAAAGLfJhq", + "url": "https://api.github.com/repos/github/roadmap/issues/events/6635165802", + "actor": { + "login": "github-product-roadmap", + "id": 67656570, + "node_id": "MDQ6VXNlcjY3NjU2NTcw", + "avatar_url": "https://avatars.githubusercontent.com/u/67656570?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/github-product-roadmap", + "html_url": "https://github.com/github-product-roadmap", + "followers_url": "https://api.github.com/users/github-product-roadmap/followers", + "following_url": "https://api.github.com/users/github-product-roadmap/following{/other_user}", + "gists_url": "https://api.github.com/users/github-product-roadmap/gists{/gist_id}", + "starred_url": "https://api.github.com/users/github-product-roadmap/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/github-product-roadmap/subscriptions", + "organizations_url": "https://api.github.com/users/github-product-roadmap/orgs", + "repos_url": "https://api.github.com/users/github-product-roadmap/repos", + "events_url": "https://api.github.com/users/github-product-roadmap/events{/privacy}", + "received_events_url": "https://api.github.com/users/github-product-roadmap/received_events", + "type": "User", + "site_admin": false + }, + "event": "renamed", + "commit_id": null, + "commit_url": null, + "created_at": "2022-05-18T19:29:01Z", + "rename": { + "from": "Secret scanning: dry-runs for enterprise-level custom patterns (cloud)", + "to": "Secret scanning: dry-runs for enterprise-level custom patterns" + }, + "performed_via_github_app": null + }, + "commented": { + "url": "https://api.github.com/repos/github/roadmap/issues/comments/1130876857", + "html_url": "https://github.com/github/roadmap/issues/493#issuecomment-1130876857", + "issue_url": "https://api.github.com/repos/github/roadmap/issues/493", + "id": 1130876857, + "node_id": "IC_kwDODwFebM5DZ8-5", + "user": { + "login": "octocat", + "id": 94867353, + "node_id": "U_kgDOBaePmQ", + "avatar_url": "https://avatars.githubusercontent.com/u/94867353?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": true + }, + "created_at": "2022-05-19T00:52:15Z", + "updated_at": "2022-05-19T00:52:15Z", + "author_association": "COLLABORATOR", + "body": "🚢 Shipped to the cloud: https://github.blog/changelog/2022-05-12-secret-scanning-dry-runs-for-enterprise-level-custom-patterns/", + "reactions": { + "url": "https://api.github.com/repos/github/roadmap/issues/comments/1130876857/reactions", + "total_count": 0, + "+1": 0, + "-1": 0, + "laugh": 0, + "hooray": 0, + "confused": 0, + "heart": 0, + "rocket": 0, + "eyes": 0 + }, + "performed_via_github_app": null, + "event": "commented", + "actor": { + "login": "octocat", + "id": 94867353, + "node_id": "U_kgDOBaePmQ", + "avatar_url": "https://avatars.githubusercontent.com/u/94867353?v=4", + "gravatar_id": "", + "url": "https://api.github.com/users/octocat", + "html_url": "https://github.com/octocat", + "followers_url": "https://api.github.com/users/octocat/followers", + "following_url": "https://api.github.com/users/octocat/following{/other_user}", + "gists_url": "https://api.github.com/users/octocat/gists{/gist_id}", + "starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}", + "subscriptions_url": "https://api.github.com/users/octocat/subscriptions", + "organizations_url": "https://api.github.com/users/octocat/orgs", + "repos_url": "https://api.github.com/users/octocat/repos", + "events_url": "https://api.github.com/users/octocat/events{/privacy}", + "received_events_url": "https://api.github.com/users/octocat/received_events", + "type": "User", + "site_admin": true + } + } + } +] diff --git a/airbyte-integrations/connectors/source-github/unit_tests/responses/projects_v2_response.json b/airbyte-integrations/connectors/source-github/unit_tests/responses/projects_v2_response.json new file mode 100644 index 000000000000..2ae61a1d045f --- /dev/null +++ b/airbyte-integrations/connectors/source-github/unit_tests/responses/projects_v2_response.json @@ -0,0 +1,45 @@ +{ + "data": { + "repository": { + "name": "integration-test", + "owner": { + "login": "airbytehq" + }, + "projectsV2": { + "nodes": [ + { + "closed": false, + "created_at": "2023-09-25T18:34:52Z", + "closed_at": null, + "updated_at": "2023-09-25T18:35:45Z", + "creator": { + "avatarUrl": "https://avatars.githubusercontent.com/u/92915184?u=e53c87d81ec6fb0596bc0f75e12e84e8f0df8d83&v=4", + "login": "airbyteio", + "resourcePath": "/airbyteio", + "url": "https://github.com/airbyteio" + }, + "node_id": "PVT_kwDOA4_XW84AV7NS", + "id": 5747538, + "number": 58, + "public": false, + "readme": "# Title\nintegration test project", + "short_description": "integration test project description", + "template": false, + "title": "integration test project", + "url": "https://github.com/orgs/airbytehq/projects/58", + "viewerCanClose": true, + "viewerCanReopen": true, + "viewerCanUpdate": true, + "owner": { + "id": "MDEyOk9yZ2FuaXphdGlvbjU5NzU4NDI3" + } + } + ], + "pageInfo": { + "hasNextPage": false, + "endCursor": "MQ" + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-github/unit_tests/pull_request_comment_reactions.json b/airbyte-integrations/connectors/source-github/unit_tests/responses/pull_request_comment_reactions.json similarity index 100% rename from airbyte-integrations/connectors/source-github/unit_tests/pull_request_comment_reactions.json rename to airbyte-integrations/connectors/source-github/unit_tests/responses/pull_request_comment_reactions.json diff --git a/airbyte-integrations/connectors/source-github/unit_tests/responses/pull_request_stats_response.json b/airbyte-integrations/connectors/source-github/unit_tests/responses/pull_request_stats_response.json new file mode 100644 index 000000000000..f4a53929e237 --- /dev/null +++ b/airbyte-integrations/connectors/source-github/unit_tests/responses/pull_request_stats_response.json @@ -0,0 +1,317 @@ +{ + "data": { + "repository": { + "name": "integration-test", + "owner": { + "login": "airbytehq" + }, + "pullRequests": { + "nodes": [ + { + "node_id": "MDExOlB1bGxSZXF1ZXN0NzIxNDM1NTA2", + "id": 721435506, + "number": 5, + "updated_at": "2021-08-27T15:53:14Z", + "changed_files": 5, + "deletions": 0, + "additions": 5, + "merged": false, + "mergeable": "MERGEABLE", + "can_be_rebased": true, + "maintainer_can_modify": false, + "merge_state_status": "BLOCKED", + "comments": { + "totalCount": 0 + }, + "commits": { + "totalCount": 5 + }, + "review_comments": { + "totalCount": 1, + "nodes": [ + { + "comments": { + "totalCount": 0 + } + } + ] + }, + "merged_by": null + }, + { + "node_id": "MDExOlB1bGxSZXF1ZXN0NzIxNDM1NDA3", + "id": 721435407, + "number": 2, + "updated_at": "2021-08-27T15:53:27Z", + "changed_files": 5, + "deletions": 0, + "additions": 5, + "merged": false, + "mergeable": "MERGEABLE", + "can_be_rebased": true, + "maintainer_can_modify": false, + "merge_state_status": "BLOCKED", + "comments": { + "totalCount": 0 + }, + "commits": { + "totalCount": 5 + }, + "review_comments": { + "totalCount": 1, + "nodes": [ + { + "comments": { + "totalCount": 0 + } + } + ] + }, + "merged_by": null + }, + { + "node_id": "MDExOlB1bGxSZXF1ZXN0NzIxNDM1NDQw", + "id": 721435440, + "number": 3, + "updated_at": "2021-08-27T16:02:49Z", + "changed_files": 5, + "deletions": 0, + "additions": 5, + "merged": true, + "mergeable": "UNKNOWN", + "can_be_rebased": false, + "maintainer_can_modify": false, + "merge_state_status": "UNKNOWN", + "comments": { + "totalCount": 0 + }, + "commits": { + "totalCount": 5 + }, + "review_comments": { + "totalCount": 1, + "nodes": [ + { + "comments": { + "totalCount": 0 + } + } + ] + }, + "merged_by": { + "__typename": "User", + "node_id": "MDQ6VXNlcjc0MzkwMQ==", + "id": 743901, + "login": "gaart", + "avatar_url": "https://avatars.githubusercontent.com/u/743901?v=4", + "html_url": "https://github.com/gaart", + "site_admin": false + } + }, + { + "node_id": "MDExOlB1bGxSZXF1ZXN0NzIxNDM1NDY2", + "id": 721435466, + "number": 4, + "updated_at": "2021-08-31T12:01:15Z", + "changed_files": 5, + "deletions": 0, + "additions": 5, + "merged": false, + "mergeable": "MERGEABLE", + "can_be_rebased": true, + "maintainer_can_modify": false, + "merge_state_status": "BLOCKED", + "comments": { + "totalCount": 0 + }, + "commits": { + "totalCount": 5 + }, + "review_comments": { + "totalCount": 3, + "nodes": [ + { + "comments": { + "totalCount": 0 + } + }, + { + "comments": { + "totalCount": 0 + } + }, + { + "comments": { + "totalCount": 1 + } + } + ] + }, + "merged_by": null + }, + { + "node_id": "PR_kwDOF9hP9c4xmEi6", + "id": 832063674, + "number": 12, + "updated_at": "2022-01-26T03:46:56Z", + "changed_files": 1, + "deletions": 0, + "additions": 2, + "merged": true, + "mergeable": "UNKNOWN", + "can_be_rebased": false, + "maintainer_can_modify": false, + "merge_state_status": "UNKNOWN", + "comments": { + "totalCount": 0 + }, + "commits": { + "totalCount": 1 + }, + "review_comments": { + "totalCount": 1, + "nodes": [ + { + "comments": { + "totalCount": 0 + } + } + ] + }, + "merged_by": { + "__typename": "User", + "node_id": "MDQ6VXNlcjUxNTQzMjI=", + "id": 5154322, + "login": "marcosmarxm", + "avatar_url": "https://avatars.githubusercontent.com/u/5154322?u=92c89b82271d48f41fad03923b0a24083e049038&v=4", + "html_url": "https://github.com/marcosmarxm", + "site_admin": false + } + }, + { + "node_id": "MDExOlB1bGxSZXF1ZXN0NzIxNDM1Mzcz", + "id": 721435373, + "number": 1, + "updated_at": "2022-03-31T11:06:06Z", + "changed_files": 5, + "deletions": 0, + "additions": 5, + "merged": false, + "mergeable": "MERGEABLE", + "can_be_rebased": true, + "maintainer_can_modify": false, + "merge_state_status": "BLOCKED", + "comments": { + "totalCount": 0 + }, + "commits": { + "totalCount": 5 + }, + "review_comments": { + "totalCount": 1, + "nodes": [ + { + "comments": { + "totalCount": 0 + } + } + ] + }, + "merged_by": null + }, + { + "node_id": "PR_kwDOF9hP9c46s2Qa", + "id": 984835098, + "number": 14, + "updated_at": "2022-10-04T17:41:29Z", + "changed_files": 1, + "deletions": 0, + "additions": 1, + "merged": false, + "mergeable": "MERGEABLE", + "can_be_rebased": true, + "maintainer_can_modify": false, + "merge_state_status": "BLOCKED", + "comments": { + "totalCount": 0 + }, + "commits": { + "totalCount": 2 + }, + "review_comments": { + "totalCount": 1, + "nodes": [ + { + "comments": { + "totalCount": 0 + } + } + ] + }, + "merged_by": null + }, + { + "node_id": "PR_kwDOF9hP9c41Vftv", + "id": 894827375, + "number": 13, + "updated_at": "2023-05-03T06:50:23Z", + "changed_files": 1, + "deletions": 1, + "additions": 1, + "merged": false, + "mergeable": "MERGEABLE", + "can_be_rebased": true, + "maintainer_can_modify": false, + "merge_state_status": "BLOCKED", + "comments": { + "totalCount": 0 + }, + "commits": { + "totalCount": 1 + }, + "review_comments": { + "totalCount": 0, + "nodes": [] + }, + "merged_by": null + }, + { + "node_id": "PR_kwDOF9hP9c5PqZhG", + "id": 1336514630, + "number": 15, + "updated_at": "2023-05-04T10:07:46Z", + "changed_files": 1, + "deletions": 0, + "additions": 1, + "merged": false, + "mergeable": "MERGEABLE", + "can_be_rebased": false, + "maintainer_can_modify": false, + "merge_state_status": "BLOCKED", + "comments": { + "totalCount": 1 + }, + "commits": { + "totalCount": 1 + }, + "review_comments": { + "totalCount": 1, + "nodes": [ + { + "comments": { + "totalCount": 0 + } + } + ] + }, + "merged_by": null + } + ], + "pageInfo": { + "hasNextPage": false, + "endCursor": "Y3Vyc29yOnYyOpK5MjAyMy0wNS0wNFQxMzowNzo0NiswMzowMM5PqZhG" + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_migrations/test_config.json b/airbyte-integrations/connectors/source-github/unit_tests/test_migrations/test_config.json new file mode 100644 index 000000000000..5272233b5e0b --- /dev/null +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_migrations/test_config.json @@ -0,0 +1,8 @@ +{ + "credentials": { + "personal_access_token": "personal_access_token" + }, + "repository": "airbytehq/airbyte airbytehq/airbyte-platform", + "start_date": "2000-01-01T00:00:00Z", + "branch": "airbytehq/airbyte/master airbytehq/airbyte-platform/main" +} diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_migrations/test_config_migrations.py b/airbyte-integrations/connectors/source-github/unit_tests/test_migrations/test_config_migrations.py new file mode 100644 index 000000000000..fdebc3af470b --- /dev/null +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_migrations/test_config_migrations.py @@ -0,0 +1,76 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import json +import os +from typing import Any, Mapping + +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source +from source_github.config_migrations import MigrateBranch, MigrateRepository +from source_github.source import SourceGithub + +# BASE ARGS +CMD = "check" +TEST_CONFIG_PATH = f"{os.path.dirname(__file__)}/test_config.json" +NEW_TEST_CONFIG_PATH = f"{os.path.dirname(__file__)}/test_new_config.json" +SOURCE_INPUT_ARGS = [CMD, "--config", TEST_CONFIG_PATH] +SOURCE: Source = SourceGithub() + + +# HELPERS +def load_config(config_path: str = TEST_CONFIG_PATH) -> Mapping[str, Any]: + with open(config_path, "r") as config: + return json.load(config) + + +def revert_migration(config_path: str = TEST_CONFIG_PATH) -> None: + with open(config_path, "r") as test_config: + config = json.load(test_config) + config.pop("repositories") + with open(config_path, "w") as updated_config: + config = json.dumps(config) + updated_config.write(config) + + +def test_migrate_config(): + migration_instance = MigrateRepository + # migrate the test_config + migration_instance.migrate(SOURCE_INPUT_ARGS, SOURCE) + # load the updated config + test_migrated_config = load_config() + # check migrated property + assert "repositories" in test_migrated_config + assert isinstance(test_migrated_config["repositories"], list) + # check the old property is in place + assert "repository" in test_migrated_config + assert isinstance(test_migrated_config["repository"], str) + # test CONTROL MESSAGE was emitted + control_msg = migration_instance.message_repository._message_queue[0] + assert control_msg.type == Type.CONTROL + assert control_msg.control.type == OrchestratorType.CONNECTOR_CONFIG + # new repositories is of type(list) + assert isinstance(control_msg.control.connectorConfig.config["repositories"], list) + # check the migrated values + revert_migration() + + +def test_config_is_reverted(): + # check the test_config state, it has to be the same as before tests + test_config = load_config() + # check the config no longer has the migrated property + assert "repositories" not in test_config + assert "branches" not in test_config + # check the old property is still there + assert "repository" in test_config + assert "branch" in test_config + assert isinstance(test_config["repository"], str) + assert isinstance(test_config["branch"], str) + + +def test_should_not_migrate_new_config(): + new_config = load_config(NEW_TEST_CONFIG_PATH) + for instance in MigrateBranch, MigrateRepository: + assert not instance._should_migrate(new_config) diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_migrations/test_new_config.json b/airbyte-integrations/connectors/source-github/unit_tests/test_migrations/test_new_config.json new file mode 100644 index 000000000000..ba674dc7ef4d --- /dev/null +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_migrations/test_new_config.json @@ -0,0 +1,8 @@ +{ + "credentials": { + "personal_access_token": "personal_access_token" + }, + "repositories": ["airbytehq/airbyte", "airbytehq/airbyte-platform"], + "start_date": "2000-01-01T00:00:00Z", + "branches": ["airbytehq/airbyte/master", "airbytehq/airbyte-platform/main"] +} diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_source.py b/airbyte-integrations/connectors/source-github/unit_tests/test_source.py index 123e0fb8bb0d..8ec9d79c574d 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_source.py @@ -3,6 +3,8 @@ # import datetime +import logging +import os import time from unittest.mock import MagicMock @@ -25,6 +27,51 @@ def check_source(repo_line: str) -> AirbyteConnectionStatus: return source.check(logger_mock, config) +@responses.activate +@pytest.mark.parametrize( + "config, expected", + ( + ( + { + "start_date": "2021-08-27T00:00:46Z", + "access_token": "test_token", + "repository": "airbyte/test", + }, + True, + ), + ({"access_token": "test_token", "repository": "airbyte/test"}, True), + ), +) +def test_check_start_date(config, expected): + responses.add(responses.GET, "https://api.github.com/repos/airbyte/test?per_page=100", json={"full_name": "test_full_name"}) + source = SourceGithub() + status, _ = source.check_connection(logger=logging.getLogger("airbyte"), config=config) + assert status == expected + + +@pytest.mark.parametrize( + "api_url, deployment_env, expected_message", + ( + ("github.my.company.org", "CLOUD", "Please enter a full url for `API URL` field starting with `http`"), + ( + "http://github.my.company.org", + "CLOUD", + "HTTP connection is insecure and is not allowed in this environment. Please use `https` instead.", + ), + ("http:/github.my.company.org", "NOT_CLOUD", "Please provide a correct API URL."), + ("https:/github.my.company.org", "CLOUD", "Please provide a correct API URL."), + ), +) +def test_connection_fail_due_to_config_error(api_url, deployment_env, expected_message): + os.environ["DEPLOYMENT_MODE"] = deployment_env + source = SourceGithub() + config = {"access_token": "test_token", "repository": "airbyte/test", "api_url": api_url} + + with pytest.raises(AirbyteTracedException) as e: + source.check_connection(logging.getLogger(), config) + assert e.value.message == expected_message + + @responses.activate def test_check_connection_repos_only(): responses.add("GET", "https://api.github.com/repos/airbytehq/airbyte", json={"full_name": "airbytehq/airbyte"}) @@ -91,12 +138,16 @@ def test_get_branches_data(): ], ) - default_branches, branches_to_pull = source._get_branches_data("", repository_args) + default_branches, branches_to_pull = source._get_branches_data([], repository_args) assert default_branches == {"airbytehq/integration-test": "master"} assert branches_to_pull == {"airbytehq/integration-test": ["master"]} default_branches, branches_to_pull = source._get_branches_data( - "airbytehq/integration-test/feature/branch_0 airbytehq/integration-test/feature/branch_1 airbytehq/integration-test/feature/branch_3", + [ + "airbytehq/integration-test/feature/branch_0", + "airbytehq/integration-test/feature/branch_1", + "airbytehq/integration-test/feature/branch_3", + ], repository_args, ) @@ -108,9 +159,6 @@ def test_get_branches_data(): @responses.activate def test_get_org_repositories(): - - source = SourceGithub() - responses.add( "GET", "https://api.github.com/repos/airbytehq/integration-test", @@ -126,7 +174,9 @@ def test_get_org_repositories(): ], ) - config = {"repository": "airbytehq/integration-test docker/*"} + config = {"repositories": ["airbytehq/integration-test", "docker/*"]} + source = SourceGithub() + config = source._ensure_default_values(config) organisations, repositories = source._get_org_repositories(config, authenticator=None) assert set(repositories) == {"airbytehq/integration-test", "docker/docker-py", "docker/compose"} @@ -142,11 +192,6 @@ def test_organization_or_repo_available(monkeypatch): assert exc_info.value.args[0] == "No streams available. Please check permissions" -def tests_get_and_prepare_repositories_config(): - config = {"repository": "airbytehq/airbyte airbytehq/airbyte.test airbytehq/integration-test"} - assert SourceGithub._get_and_prepare_repositories_config(config) == {"airbytehq/airbyte", "airbytehq/airbyte.test", "airbytehq/integration-test"} - - def test_check_config_repository(): source = SourceGithub() source.check = MagicMock(return_value=True) @@ -176,32 +221,19 @@ def test_check_config_repository(): "https://github.com/airbytehq/airbyte", ] - config["repository"] = "" + config["repositories"] = [] with pytest.raises(AirbyteTracedException): assert command_check(source, config) - config["repository"] = " " + config["repositories"] = [] with pytest.raises(AirbyteTracedException): assert command_check(source, config) for repos in repos_ok: - config["repository"] = repos - assert command_check(source, config) - - for repos in repos_fail: - config["repository"] = repos - with pytest.raises(AirbyteTracedException): - assert command_check(source, config) - - config["repository"] = " ".join(repos_ok) - assert command_check(source, config) - config["repository"] = " ".join(repos_ok) - assert command_check(source, config) - config["repository"] = ",".join(repos_ok) - with pytest.raises(AirbyteTracedException): + config["repositories"] = [repos] assert command_check(source, config) for repos in repos_fail: - config["repository"] = " ".join(repos_ok[:len(repos_ok)//2] + [repos] + repos_ok[len(repos_ok)//2:]) + config["repositories"] = [repos] with pytest.raises(AirbyteTracedException): assert command_check(source, config) @@ -221,7 +253,7 @@ def sleep_mock(seconds): frozen_time.tick(delta=datetime.timedelta(seconds=seconds)) called_args.append(seconds) - monkeypatch.setattr(time, 'sleep', sleep_mock) + monkeypatch.setattr(time, "sleep", sleep_mock) with freeze_time("2021-01-01 12:00:00") as frozen_time: @@ -273,3 +305,64 @@ def test_streams_page_size(): assert stream.page_size == constants.DEFAULT_PAGE_SIZE_FOR_LARGE_STREAM else: assert stream.page_size == constants.DEFAULT_PAGE_SIZE + + +@responses.activate +@pytest.mark.parametrize( + "config, expected", + ( + ( + { + "start_date": "2021-08-27T00:00:46Z", + "access_token": "test_token", + "repository": "airbyte/test", + }, + 39, + ), + ({"access_token": "test_token", "repository": "airbyte/test"}, 39), + ), +) +def test_streams_config_start_date(config, expected): + responses.add(responses.GET, "https://api.github.com/repos/airbyte/test?per_page=100", json={"full_name": "airbyte/test"}) + responses.add( + responses.GET, + "https://api.github.com/repos/airbyte/test?per_page=100", + json={"full_name": "airbyte/test", "default_branch": "default_branch"}, + ) + responses.add( + responses.GET, + "https://api.github.com/repos/airbyte/test/branches?per_page=100", + json=[{"repository": "airbyte/test", "name": "name"}], + ) + source = SourceGithub() + streams = source.streams(config=config) + # projects stream that uses start date + project_stream = streams[4] + assert len(streams) == expected + if config.get("start_date"): + assert project_stream._start_date == "2021-08-27T00:00:46Z" + else: + assert not project_stream._start_date + + +@pytest.mark.parametrize( + "error_message, expected_user_friendly_message", + [ + ( + "404 Client Error: Not Found for url: https://api.github.com/repos/repo_name", + 'Repo name: "repo_name" is unknown, "repository" config option should use existing full repo name /', + ), + ( + "404 Client Error: Not Found for url: https://api.github.com/orgs/org_name", + 'Organization name: "org_name" is unknown, "repository" config option should be updated. Please validate your repository config.', + ), + ( + "401 Client Error: Unauthorized for url", + "Github credentials have expired or changed, please review your credentials and re-authenticate or renew your access token.", + ), + ], +) +def test_user_friendly_message(error_message, expected_user_friendly_message): + source = SourceGithub() + user_friendly_error_message = source.user_friendly_error_message(error_message) + assert user_friendly_error_message == expected_user_friendly_message diff --git a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py index c55958da5510..87d9c3478cd3 100644 --- a/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py +++ b/airbyte-integrations/connectors/source-github/unit_tests/test_stream.py @@ -10,10 +10,11 @@ import pytest import requests import responses -from airbyte_cdk.sources.streams.http.exceptions import BaseBackoffException +from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode +from airbyte_cdk.sources.streams.http.exceptions import BaseBackoffException, UserDefinedBackoffException from requests import HTTPError from responses import matchers -from source_github import constants +from source_github import SourceGithub, constants from source_github.streams import ( Branches, Collaborators, @@ -21,14 +22,17 @@ CommitCommentReactions, CommitComments, Commits, + ContributorActivity, Deployments, IssueEvents, IssueLabels, IssueMilestones, + IssueTimelineEvents, Organizations, ProjectCards, ProjectColumns, Projects, + ProjectsV2, PullRequestCommentReactions, PullRequestCommits, PullRequests, @@ -210,6 +214,12 @@ def test_stream_teams_502(sleep_mock): assert set(call.request.url for call in responses.calls).symmetric_difference({f"{url}?per_page=100"}) == set() +def test_stream_organizations_availability_report(): + organization_args = {"organizations": ["org1", "org2"]} + stream = Organizations(**organization_args) + assert stream.availability_strategy is None + + @responses.activate def test_stream_organizations_read(): organization_args = {"organizations": ["org1", "org2"]} @@ -431,7 +441,11 @@ def test_stream_commits_incremental_read(): "GET", api_url, json=data[5:7], - match=[matchers.query_param_matcher({"since": "2022-02-02T10:10:06Z", "sha": "branch", "per_page": "2", "page": "2"}, strict_match=False)], + match=[ + matchers.query_param_matcher( + {"since": "2022-02-02T10:10:06Z", "sha": "branch", "per_page": "2", "page": "2"}, strict_match=False + ) + ], ) stream_state = {} @@ -877,7 +891,7 @@ def test_stream_reviews_incremental_read(): stream = Reviews(**repository_args_with_start_date) stream.page_size = 2 - f = Path(__file__).parent / "graphql_reviews_responses.json" + f = Path(__file__).parent / "responses/graphql_reviews_responses.json" response_objects = json.load(open(f)) def request_callback(request): @@ -904,7 +918,7 @@ def request_callback(request): @responses.activate -def test_stream_team_members_full_refresh(): +def test_stream_team_members_full_refresh(caplog): organization_args = {"organizations": ["org1"]} repository_args = {"repositories": [], "page_size_for_large_streams": 100} @@ -912,8 +926,9 @@ def test_stream_team_members_full_refresh(): responses.add("GET", "https://api.github.com/orgs/org1/teams/team1/members", json=[{"login": "login1"}, {"login": "login2"}]) responses.add("GET", "https://api.github.com/orgs/org1/teams/team1/memberships/login1", json={"username": "login1"}) responses.add("GET", "https://api.github.com/orgs/org1/teams/team1/memberships/login2", json={"username": "login2"}) - responses.add("GET", "https://api.github.com/orgs/org1/teams/team2/members", json=[{"login": "login2"}]) + responses.add("GET", "https://api.github.com/orgs/org1/teams/team2/members", json=[{"login": "login2"}, {"login": "login3"}]) responses.add("GET", "https://api.github.com/orgs/org1/teams/team2/memberships/login2", json={"username": "login2"}) + responses.add("GET", "https://api.github.com/orgs/org1/teams/team2/memberships/login3", status=requests.codes.NOT_FOUND) teams_stream = Teams(**organization_args) stream = TeamMembers(parent=teams_stream, **repository_args) @@ -924,6 +939,7 @@ def test_stream_team_members_full_refresh(): {"login": "login1", "organization": "org1", "team_slug": "team1"}, {"login": "login2", "organization": "org1", "team_slug": "team1"}, {"login": "login2", "organization": "org1", "team_slug": "team2"}, + {"login": "login3", "organization": "org1", "team_slug": "team2"}, ] stream = TeamMemberships(parent=stream, **repository_args) @@ -934,6 +950,8 @@ def test_stream_team_members_full_refresh(): {"username": "login2", "organization": "org1", "team_slug": "team1"}, {"username": "login2", "organization": "org1", "team_slug": "team2"}, ] + expected_message = "Syncing `TeamMemberships` stream for organization `org1`, team `team2` and user `login3` isn't available: User has no team membership. Skipping..." + assert expected_message in caplog.messages @responses.activate @@ -1247,7 +1265,7 @@ def test_stream_pull_request_comment_reactions_read(): stream = PullRequestCommentReactions(**repository_args_with_start_date) stream.page_size = 2 - f = Path(__file__).parent / "pull_request_comment_reactions.json" + f = Path(__file__).parent / "responses/pull_request_comment_reactions.json" response_objects = json.load(open(f)) def request_callback(request): @@ -1284,3 +1302,167 @@ def request_callback(request): ] assert stream_state == {"airbytehq/airbyte": {"created_at": "2022-01-02T00:00:01Z"}} + + +@responses.activate +def test_stream_projects_v2_graphql_retry(): + repository_args_with_start_date = { + "start_date": "2022-01-01T00:00:00Z", + "page_size_for_large_streams": 20, + "repositories": ["airbytehq/airbyte"], + } + stream = ProjectsV2(**repository_args_with_start_date) + resp = responses.add( + responses.POST, + "https://api.github.com/graphql", + json={"errors": "not found"}, + status=200, + ) + + with patch.object(stream, "backoff_time", return_value=0.01), pytest.raises(UserDefinedBackoffException): + read_incremental(stream, stream_state={}) + assert resp.call_count == stream.max_retries + 1 + + +@responses.activate +def test_stream_projects_v2_graphql_query(): + repository_args_with_start_date = { + "start_date": "2022-01-01T00:00:00Z", + "page_size_for_large_streams": 20, + "repositories": ["airbytehq/airbyte"], + } + stream = ProjectsV2(**repository_args_with_start_date) + query = stream.request_body_json(stream_state={}, stream_slice={"repository": "airbytehq/airbyte"}) + responses.add( + responses.POST, + "https://api.github.com/graphql", + json=json.load(open(Path(__file__).parent / "responses/projects_v2_response.json")), + ) + f = Path(__file__).parent / "projects_v2_pull_requests_query.json" + expected_query = json.load(open(f)) + + records = list(read_full_refresh(stream)) + assert query == expected_query + assert records[0].get("owner_id") + assert records[0].get("repository") + + +@responses.activate +def test_stream_contributor_activity_parse_empty_response(caplog): + repository_args = { + "page_size_for_large_streams": 20, + "repositories": ["airbytehq/airbyte"], + } + stream = ContributorActivity(**repository_args) + resp = responses.add( + responses.GET, + "https://api.github.com/repos/airbytehq/airbyte/stats/contributors", + body="", + status=204, + ) + records = list(read_full_refresh(stream)) + expected_message = "Empty response received for contributor_activity stats in repository airbytehq/airbyte" + assert resp.call_count == 1 + assert records == [] + assert expected_message in caplog.messages + + +@responses.activate +def test_stream_contributor_activity_accepted_response(caplog): + responses.add( + responses.GET, + "https://api.github.com/repos/airbytehq/test_airbyte?per_page=100", + json={"full_name": "airbytehq/test_airbyte"}, + status=200, + ) + responses.add( + responses.GET, + "https://api.github.com/repos/airbytehq/test_airbyte?per_page=100", + json={"full_name": "airbytehq/test_airbyte", "default_branch": "default_branch"}, + status=200, + ) + responses.add( + responses.GET, + "https://api.github.com/repos/airbytehq/test_airbyte/branches?per_page=100", + json={}, + status=200, + ) + resp = responses.add( + responses.GET, + "https://api.github.com/repos/airbytehq/test_airbyte/stats/contributors?per_page=100", + body="", + status=202, + ) + + source = SourceGithub() + configured_catalog = { + "streams": [ + { + "stream": {"name": "contributor_activity", "json_schema": {}, "supported_sync_modes": ["full_refresh"],"source_defined_primary_key": [["id"]]}, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] + } + catalog = ConfiguredAirbyteCatalog.parse_obj(configured_catalog) + config = {"access_token": "test_token", "repository": "airbytehq/test_airbyte"} + logger_mock = MagicMock() + + with patch("time.sleep", return_value=0): + records = list(source.read(config=config, logger=logger_mock, catalog=catalog, state={})) + + assert records[2].log.message == "Syncing `ContributorActivity` stream isn't available for repository `airbytehq/test_airbyte`." + assert resp.call_count == 6 + + +@responses.activate +def test_stream_contributor_activity_parse_response(): + repository_args = { + "page_size_for_large_streams": 20, + "repositories": ["airbytehq/airbyte"], + } + stream = ContributorActivity(**repository_args) + responses.add( + responses.GET, + "https://api.github.com/repos/airbytehq/airbyte/stats/contributors", + json=json.load(open(Path(__file__).parent / "responses/contributor_activity_response.json")), + ) + records = list(read_full_refresh(stream)) + assert len(records) == 1 + + +@responses.activate +def test_issues_timeline_events(): + repository_args = { + "repositories": ["airbytehq/airbyte"], + "page_size_for_large_streams": 20, + } + response_file = Path(__file__).parent / "responses/issue_timeline_events.json" + response_json = json.load(open(response_file)) + responses.add(responses.GET, "https://api.github.com/repos/airbytehq/airbyte/issues/1/timeline?per_page=100", json=response_json) + expected_file = Path(__file__).parent / "responses/issue_timeline_events_response.json" + expected_records = json.load(open(expected_file)) + + stream = IssueTimelineEvents(**repository_args) + records = list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"repository": "airbytehq/airbyte", "number": 1})) + assert expected_records == records + + +@responses.activate +def test_pull_request_stats(): + repository_args = { + "page_size_for_large_streams": 10, + "repositories": ["airbytehq/airbyte"], + } + stream = PullRequestStats(**repository_args) + query = stream.request_body_json(stream_state={}, stream_slice={"repository": "airbytehq/airbyte"}) + responses.add( + responses.POST, + "https://api.github.com/graphql", + json=json.load(open(Path(__file__).parent / "responses/pull_request_stats_response.json")), + ) + f = Path(__file__).parent / "pull_request_stats_query.json" + expected_query = json.load(open(f)) + + list(read_full_refresh(stream)) + assert query == expected_query diff --git a/airbyte-integrations/connectors/source-gitlab/Dockerfile b/airbyte-integrations/connectors/source-gitlab/Dockerfile deleted file mode 100644 index 67e6a7b6081b..000000000000 --- a/airbyte-integrations/connectors/source-gitlab/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" - -WORKDIR /airbyte/integration_code -COPY setup.py ./ -RUN pip install . -COPY source_gitlab ./source_gitlab -COPY main.py ./ - -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=1.6.0 -LABEL io.airbyte.name=airbyte/source-gitlab diff --git a/airbyte-integrations/connectors/source-gitlab/README.md b/airbyte-integrations/connectors/source-gitlab/README.md index d117ecc233d5..6b4a4f0ad561 100644 --- a/airbyte-integrations/connectors/source-gitlab/README.md +++ b/airbyte-integrations/connectors/source-gitlab/README.md @@ -27,14 +27,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-gitlab:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/gitlab) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_gitlab/spec.json` file. @@ -54,19 +46,70 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-gitlab:dev + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-gitlab build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-gitlab:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. -You can also build the connector image via Gradle: +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-gitlab:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-gitlab:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-gitlab:dev . +# Running the spec command against your patched connector +docker run airbyte/source-gitlab:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -75,44 +118,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-gitlab:dev check --con docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-gitlab:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-gitlab:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-gitlab test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-gitlab:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-gitlab:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -122,8 +137,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-gitlab test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/gitlab.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml b/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml index df414a0a2876..7ad85ba02cee 100644 --- a/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml @@ -3,64 +3,60 @@ test_strictness_level: "high" acceptance_tests: spec: tests: - - spec_path: "source_gitlab/spec.json" + - spec_path: "source_gitlab/spec.json" + backward_compatibility_tests_config: + disable_for_version: 1.8.0 connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "secrets/config_oauth.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "secrets/config_oauth.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: tests: - - config_path: "secrets/config.json" + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: 1.8.4 basic_read: tests: - - config_path: "secrets/config.json" - timeout_seconds: 3600 - expect_records: - path: "integration_tests/expected_records.jsonl" - ignored_fields: - jobs: - - name: "user" - bypass_reason: "User object contains local_time which will be different each time test is run" - fail_on_extra_columns: false - - config_path: "secrets/config_with_ids.json" - timeout_seconds: 3600 - empty_streams: - - name: "epics" - bypass_reason: "Group in this config does not have epics. This stream is tested in the above TC." - - name: "epic_issues" - bypass_reason: "Group in this config does not have epics issues. This stream is tested in the above TC." - expect_records: - path: "integration_tests/expected_records_with_ids.jsonl" - ignored_fields: - jobs: - - name: "user" - bypass_reason: "User object contains local_time which will be different each time test is run" - fail_on_extra_columns: false - - config_path: "secrets/config_oauth.json" - timeout_seconds: 3600 - expect_records: - path: "integration_tests/expected_records.jsonl" - ignored_fields: - jobs: - - name: "user" - bypass_reason: "User object contains local_time which will be different each time test is run" - fail_on_extra_columns: false + - config_path: "secrets/config.json" + timeout_seconds: 3600 + expect_records: + path: "integration_tests/expected_records.jsonl" + ignored_fields: + jobs: + - name: "user" + bypass_reason: "User object contains local_time which will be different each time test is run" + - config_path: "secrets/config_with_ids.json" + timeout_seconds: 3600 + empty_streams: + - name: "epics" + bypass_reason: "Group in this config does not have epics. This stream is tested in the above TC." + - name: "epic_issues" + bypass_reason: "Group in this config does not have epics issues. This stream is tested in the above TC." + expect_records: + path: "integration_tests/expected_records_with_ids.jsonl" + ignored_fields: + jobs: + - name: "user" + bypass_reason: "User object contains local_time which will be different each time test is run" + - config_path: "secrets/config_oauth.json" + timeout_seconds: 3600 + expect_records: + path: "integration_tests/expected_records.jsonl" + ignored_fields: + jobs: + - name: "user" + bypass_reason: "User object contains local_time which will be different each time test is run" incremental: tests: - - config_path: "secrets/config_with_ids.json" - configured_catalog_path: "integration_tests/incremental_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" - cursor_paths: - commits: ["25157276", "created_at"] - issues: ["25157276", "updated_at"] - merge_requests: ["25157276", "updated_at"] - pipelines: ["25157276", "updated_at"] + - config_path: "secrets/config_with_ids.json" + configured_catalog_path: "integration_tests/incremental_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-gitlab/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-gitlab/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-gitlab/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-gitlab/build.gradle b/airbyte-integrations/connectors/source-gitlab/build.gradle deleted file mode 100644 index 91ccc64ce7cd..000000000000 --- a/airbyte-integrations/connectors/source-gitlab/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_gitlab' -} diff --git a/airbyte-integrations/connectors/source-gitlab/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-gitlab/integration_tests/configured_catalog.json index 1aa8a2060837..67009e00a6da 100644 --- a/airbyte-integrations/connectors/source-gitlab/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-gitlab/integration_tests/configured_catalog.json @@ -185,6 +185,19 @@ "destination_sync_mode": "overwrite", "primary_key": [["name"]] }, + { + "stream": { + "name": "deployments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "primary_key": [["id"]] + }, { "stream": { "name": "tags", diff --git a/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.jsonl index b6607d921ac9..8db7341432c9 100644 --- a/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.jsonl @@ -1,42 +1,40 @@ -{"stream": "project_milestones", "data": {"id": 1943563, "iid": 1, "project_id": 25156633, "title": "Sample GitLab Project Milestone 1", "description": "Sample GitLab Project Milestone 1", "state": "active", "created_at": "2021-02-15T16:00:33.349Z", "updated_at": "2021-02-15T16:00:33.349Z", "due_date": "2021-03-12", "start_date": null, "expired": true, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/milestones/1"}, "emitted_at": 1686568101690} -{"stream": "project_milestones", "data": {"id": 1943564, "iid": 2, "project_id": 25156633, "title": "Sample GitLab Project Milestone 2", "description": "Sample GitLab Project Milestone 2", "state": "active", "created_at": "2021-02-15T16:00:46.934Z", "updated_at": "2021-02-15T16:00:46.934Z", "due_date": "2021-03-19", "start_date": null, "expired": true, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/milestones/2"}, "emitted_at": 1686568101692} -{"stream": "project_milestones", "data": {"id": 1943565, "iid": 3, "project_id": 25156633, "title": "Sample GitLab Project Milestone 3", "description": "Sample GitLab Project Milestone 3", "state": "active", "created_at": "2021-02-15T16:01:07.153Z", "updated_at": "2021-02-15T16:01:07.153Z", "due_date": "2021-03-26", "start_date": null, "expired": true, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/milestones/3"}, "emitted_at": 1686568101692} -{"stream": "pipelines_extended", "data": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "before_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:51:07.816Z", "finished_at": "2021-03-18T12:51:52.000Z", "committed_at": null, "duration": 43, "queued_duration": 1, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1686568215223} -{"stream": "pipelines_extended", "data": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "before_sha": "0000000000000000000000000000000000000000", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:48:50.166Z", "finished_at": "2021-03-18T12:49:38.084Z", "committed_at": null, "duration": 47, "queued_duration": null, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1686568215999} +{"stream": "project_milestones", "data": {"id": 1943705, "iid": 51, "project_id": 25157276, "title": "Project Milestone 51", "description": null, "state": "active", "created_at": "2021-03-15T15:33:16.915Z", "updated_at": "2021-03-15T15:33:16.915Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/milestones/51"}, "emitted_at": 1696947569422} +{"stream": "project_milestones", "data": {"id": 1943704, "iid": 50, "project_id": 25157276, "title": "Project Milestone 50", "description": null, "state": "active", "created_at": "2021-03-15T15:33:16.329Z", "updated_at": "2021-03-15T15:33:16.329Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/milestones/50"}, "emitted_at": 1696947569423} +{"stream": "project_milestones", "data": {"id": 1943703, "iid": 49, "project_id": 25157276, "title": "Project Milestone 49", "description": null, "state": "active", "created_at": "2021-03-15T15:33:15.960Z", "updated_at": "2021-03-15T15:33:15.960Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/milestones/49"}, "emitted_at": 1696947569423} +{"stream": "pipelines_extended", "data": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "before_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:51:07.816Z", "finished_at": "2021-03-18T12:51:52.000Z", "committed_at": null, "duration": 43, "queued_duration": 1, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "Failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1696948219689} +{"stream": "pipelines_extended", "data": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "before_sha": "0000000000000000000000000000000000000000", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:48:50.166Z", "finished_at": "2021-03-18T12:49:38.084Z", "committed_at": null, "duration": 47, "queued_duration": null, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "Failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1696948220075} {"stream": "group_issue_boards", "data": {"id": 5099065, "name": "Development", "hide_backlog_list": false, "hide_closed_list": false, "project": null, "lists": [], "group": {"id": 11329647, "web_url": "https://gitlab.com/groups/new-group-airbute", "name": "New Group Airbute"}, "group_id": 11329647}, "emitted_at": 1686568061140} -{"stream": "merge_requests", "data": {"id": 92098762, "iid": 30, "project_id": 25156633, "title": "Id blanditiis consequatur ut.", "description": "##### Voluptatum\nPorro et quo. Laborum molestias ducimus. Labore dolorum adipisci. Quisquam est quis. Sint accusamus maxime.\n* Veritatis. \n* Eos. \n* Adipisci. \n* Quibusdam. \n* Sint. \n* Consequuntur. \n* Hic. \n* Voluptate. \n* Velit.", "state": "opened", "created_at": "2021-02-15T15:55:38.117Z", "updated_at": "2021-02-15T15:55:38.117Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "laudantium-unde-et-iste-et", "user_notes_count": 15, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25156633, "target_project_id": 25156633, "labels": ["asperiores-ex", "quidem-labore", "sed-consequuntur"], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "d49703b85913ee7a8c85e1893057ef4cdb06ff85", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": null, "prepared_at": "2021-02-15T15:55:38.117Z", "reference": "!30", "references": {"short": "!30", "relative": "!30", "full": "airbyte.io/ci-test-project!30"}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/merge_requests/30", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1686568137434} -{"stream": "merge_requests", "data": {"id": 92098761, "iid": 29, "project_id": 25156633, "title": "Fugiat aut voluptatem voluptas.", "description": "###### Unde\nEligendi nemo quam. Veritatis delectus iure. Placeat ut odit. Officiis accusantium sit. Eos sequi cupiditate.\n# Eum", "state": "opened", "created_at": "2021-02-15T15:55:34.534Z", "updated_at": "2021-02-15T15:55:34.534Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ea-dolor-quia-et-sint", "user_notes_count": 4, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25156633, "target_project_id": 25156633, "labels": ["earum-eaque", "nisi-et", "sed-voluptatem"], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "45afadfbf4eb1a9d6468950b23e8557bf72445fa", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": null, "prepared_at": "2021-02-15T15:55:34.534Z", "reference": "!29", "references": {"short": "!29", "relative": "!29", "full": "airbyte.io/ci-test-project!29"}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/merge_requests/29", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1686568137437} -{"stream": "merge_requests", "data": {"id": 92098758, "iid": 28, "project_id": 25156633, "title": "Delectus sit quod repellendus.", "description": "###### Corporis\nMolestias eius corrupti. Est maiores ut. Deleniti itaque deserunt. Perspiciatis quis et. Non et quia.\n### Dolorum\nNihil at et. Eligendi recusandae omnis. Eaque ratione dolorem.\n### Quam\nRem ad vel. Officiis sint voluptatem. Asperiores odit non.\n0. Hic. \n1. Labore. \n2. Voluptates. \n3. Dolores. \n4. Laborum. \n5. Non. \n6. Odit.", "state": "opened", "created_at": "2021-02-15T15:55:31.164Z", "updated_at": "2021-02-15T15:55:31.164Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ipsum-consequatur-et-in-et", "user_notes_count": 19, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25156633, "target_project_id": 25156633, "labels": ["nisi-et", "omnis-assumenda", "ut-incidunt"], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "5b8695fe02d2856fa9e3249d757aea89832b8d2e", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": null, "prepared_at": "2021-02-15T15:55:31.164Z", "reference": "!28", "references": {"short": "!28", "relative": "!28", "full": "airbyte.io/ci-test-project!28"}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/merge_requests/28", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1686568137439} -{"stream":"groups","data":{"id":68657749,"web_url":"https://gitlab.com/groups/empty-group4","name":"Empty Group","path":"empty-group4","description":"","visibility":"private","share_with_group_lock":false,"require_two_factor_authentication":false,"two_factor_grace_period":48,"project_creation_level":"developer","auto_devops_enabled":null,"subgroup_creation_level":"maintainer","emails_disabled":null,"mentions_disabled":null,"lfs_enabled":true,"default_branch_protection":2,"avatar_url":null,"request_access_enabled":true,"full_name":"Empty Group","full_path":"empty-group4","created_at":"2023-06-09T13:47:19.446Z","parent_id":null,"shared_runners_setting":"enabled","ldap_cn":null,"ldap_access":null,"wiki_access_level":"enabled","shared_with_groups":[],"runners_token":"GR1348941-x4xtBM6_zdFj-ED8QF8","prevent_sharing_groups_outside_hierarchy":false,"shared_projects":[],"shared_runners_minutes_limit":null,"extra_shared_runners_minutes_limit":null,"prevent_forking_outside_group":null,"membership_lock":false,"projects":[]},"emitted_at":1688127739135} -{"stream":"groups","data":{"id":11329647,"web_url":"https://gitlab.com/groups/new-group-airbute","name":"New Group Airbute","path":"new-group-airbute","description":"","visibility":"public","share_with_group_lock":false,"require_two_factor_authentication":false,"two_factor_grace_period":48,"project_creation_level":"developer","auto_devops_enabled":null,"subgroup_creation_level":"maintainer","emails_disabled":null,"mentions_disabled":null,"lfs_enabled":true,"default_branch_protection":2,"avatar_url":null,"request_access_enabled":true,"full_name":"New Group Airbute","full_path":"new-group-airbute","created_at":"2021-03-15T15:55:53.613Z","parent_id":null,"shared_runners_setting":"enabled","ldap_cn":null,"ldap_access":null,"wiki_access_level":"enabled","shared_with_groups":[],"runners_token":"GR1348941-PhosPap-Sf1UxL1g6m4","prevent_sharing_groups_outside_hierarchy":false,"shared_projects":[],"shared_runners_minutes_limit":null,"extra_shared_runners_minutes_limit":null,"prevent_forking_outside_group":null,"membership_lock":false,"projects":[{"id":25157276,"path_with_namespace":"new-group-airbute/new-ci-test-project"}]},"emitted_at":1688127739529} -{"stream":"groups","data":{"id":61014882,"web_url":"https://gitlab.com/groups/new-group-airbute/test-subgroup-airbyte/test-private-sg","name":"Test Private SG","path":"test-private-sg","description":"","visibility":"private","share_with_group_lock":false,"require_two_factor_authentication":false,"two_factor_grace_period":48,"project_creation_level":"developer","auto_devops_enabled":null,"subgroup_creation_level":"maintainer","emails_disabled":null,"mentions_disabled":null,"lfs_enabled":true,"default_branch_protection":2,"avatar_url":null,"request_access_enabled":true,"full_name":"New Group Airbute / Test Subgroup Airbyte / Test Private SG","full_path":"new-group-airbute/test-subgroup-airbyte/test-private-sg","created_at":"2022-12-02T08:46:22.648Z","parent_id":61014863,"shared_runners_setting":"enabled","ldap_cn":null,"ldap_access":null,"wiki_access_level":"enabled","shared_with_groups":[],"runners_token":"GR1348941bjUaJQy2zzar-JmNBjfq","shared_projects":[],"shared_runners_minutes_limit":null,"extra_shared_runners_minutes_limit":null,"prevent_forking_outside_group":null,"membership_lock":false,"projects":[]},"emitted_at":1688127739793} -{"stream": "epic_issues", "data": {"id": 120214448, "iid": 31, "project_id": 25156633, "title": "Unit tests", "description": null, "state": "opened", "created_at": "2022-12-11T10:50:25.940Z", "updated_at": "2022-12-11T10:50:25.940Z", "closed_at": null, "closed_by": null, "labels": [], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/issues/31", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "weight": null, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25156633/issues/31", "notes": "https://gitlab.com/api/v4/projects/25156633/issues/31/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25156633/issues/31/award_emoji", "project": "https://gitlab.com/api/v4/projects/25156633", "closed_as_duplicate_of": null}, "references": {"short": "#31", "relative": "#31", "full": "airbyte.io/ci-test-project#31"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "epic_iid": 1, "epic": {"id": 678569, "iid": 1, "title": "Source Gitlab: certify to Beta", "url": "/groups/airbyte.io/-/epics/1", "group_id": 11266951, "human_readable_end_date": "Dec 30, 2022", "human_readable_timestamp": "Past due"}, "iteration": null, "epic_issue_id": 1899479, "relative_position": 0, "milestone_id": null, "assignee_id": null, "author_id": 8375961}, "emitted_at": 1686568231753} -{"stream": "issues", "data": {"id": 80940833, "iid": 30, "project_id": 25156633, "title": "Nulla tempore voluptatibus error.", "description": "### Voluptate\nQui quaerat praesentium. Voluptates temporibus quae. Libero aliquid quod. Nihil rerum earum. Inventore et illum.\n`Quod.`", "state": "opened", "created_at": "2021-02-15T15:56:04.924Z", "updated_at": "2021-02-15T15:56:04.924Z", "closed_at": null, "closed_by": null, "labels": ["et-facere", "nisi-et", "suscipit-consectetur"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 20, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/issues/30", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "weight": null, "blocking_issues_count": 0, "has_tasks": true, "task_status": "0 of 0 checklist items completed", "_links": {"self": "https://gitlab.com/api/v4/projects/25156633/issues/30", "notes": "https://gitlab.com/api/v4/projects/25156633/issues/30/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25156633/issues/30/award_emoji", "project": "https://gitlab.com/api/v4/projects/25156633", "closed_as_duplicate_of": null}, "references": {"short": "#30", "relative": "#30", "full": "airbyte.io/ci-test-project#30"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "epic_iid": null, "epic": null, "iteration": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1686568070306} -{"stream": "issues", "data": {"id": 80940829, "iid": 29, "project_id": 25156633, "title": "Dolores tempora deserunt perspiciatis.", "description": "#### Magnam\nTempora optio eos. Quos quam ut. Accusamus aperiam consequatur. Saepe sit nam. Eaque tenetur qui.\n0. Aut. \n1. Ratione.", "state": "opened", "created_at": "2021-02-15T15:56:04.790Z", "updated_at": "2021-02-15T15:56:04.790Z", "closed_at": null, "closed_by": null, "labels": ["in-qui", "iste-est", "odio-ut"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 18, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/issues/29", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "weight": null, "blocking_issues_count": 0, "has_tasks": true, "task_status": "0 of 0 checklist items completed", "_links": {"self": "https://gitlab.com/api/v4/projects/25156633/issues/29", "notes": "https://gitlab.com/api/v4/projects/25156633/issues/29/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25156633/issues/29/award_emoji", "project": "https://gitlab.com/api/v4/projects/25156633", "closed_as_duplicate_of": null}, "references": {"short": "#29", "relative": "#29", "full": "airbyte.io/ci-test-project#29"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "epic_iid": null, "epic": null, "iteration": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1686568070307} -{"stream": "issues", "data": {"id": 80940824, "iid": 28, "project_id": 25156633, "title": "Ut quae nesciunt facere.", "description": "##### Error\nIpsum doloremque beatae. Non incidunt ut. Quia esse atque. Quo dolores repudiandae. Sint aliquid et.\n## A", "state": "opened", "created_at": "2021-02-15T15:56:04.658Z", "updated_at": "2021-02-15T15:56:04.658Z", "closed_at": null, "closed_by": null, "labels": ["officia-laborum", "sit-neque", "voluptas-officiis"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 13, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/issues/28", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "weight": null, "blocking_issues_count": 0, "has_tasks": true, "task_status": "0 of 0 checklist items completed", "_links": {"self": "https://gitlab.com/api/v4/projects/25156633/issues/28", "notes": "https://gitlab.com/api/v4/projects/25156633/issues/28/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25156633/issues/28/award_emoji", "project": "https://gitlab.com/api/v4/projects/25156633", "closed_as_duplicate_of": null}, "references": {"short": "#28", "relative": "#28", "full": "airbyte.io/ci-test-project#28"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "epic_iid": null, "epic": null, "iteration": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1686568070307} -{"stream": "project_members", "data": {"access_level": 40, "created_at": "2022-12-02T08:50:10.348Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "project_id": 41541858}, "emitted_at": 1686568108823} -{"stream": "project_members", "data": {"access_level": 40, "created_at": "2021-03-15T14:46:31.550Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "project_id": 25156633}, "emitted_at": 1686568109879} -{"stream": "project_members", "data": {"access_level": 40, "created_at": "2021-03-10T17:16:54.405Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "project_id": 25032440}, "emitted_at": 1686568110194} -{"stream": "epics", "data": {"id": 678569, "iid": 1, "color": "#1068bf", "text_color": "#FFFFFF", "group_id": 11266951, "parent_id": null, "parent_iid": null, "title": "Source Gitlab: certify to Beta", "description": "Lorem ipsum", "confidential": false, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "start_date": "2022-12-11", "start_date_is_fixed": true, "start_date_fixed": "2022-12-11", "start_date_from_inherited_source": null, "start_date_from_milestones": null, "end_date": "2022-12-30", "due_date": "2022-12-30", "due_date_is_fixed": true, "due_date_fixed": "2022-12-30", "due_date_from_inherited_source": null, "due_date_from_milestones": null, "state": "opened", "web_edit_url": "/groups/airbyte.io/-/epics/1", "web_url": "https://gitlab.com/groups/airbyte.io/-/epics/1", "references": {"short": "&1", "relative": "&1", "full": "airbyte.io&1"}, "created_at": "2022-12-11T10:50:04.280Z", "updated_at": "2022-12-11T10:50:26.276Z", "closed_at": null, "labels": [], "upvotes": 1, "downvotes": 0, "_links": {"self": "https://gitlab.com/api/v4/groups/11266951/epics/1", "epic_issues": "https://gitlab.com/api/v4/groups/11266951/epics/1/issues", "group": "https://gitlab.com/api/v4/groups/11266951", "parent": null}, "author_id": 8375961}, "emitted_at": 1686568224925} -{"stream": "commits", "data": {"id": "fb24e6736b3a959a59e49b56d2d83a28ea3ae15b", "short_id": "fb24e673", "created_at": "2022-12-02T14:26:55.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2022-12-02T14:26:55.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2022-12-02T14:26:55.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup/-/commit/fb24e6736b3a959a59e49b56d2d83a28ea3ae15b", "stats": {"additions": 92, "deletions": 0, "total": 92}, "project_id": 41551658}, "emitted_at": 1686568045674} -{"stream": "commits", "data": {"id": "27329d3afac51fbf2762428e12f2635d1137c549", "short_id": "27329d3a", "created_at": "2021-02-15T15:52:52.000+00:00", "parent_ids": ["b362ea7aa65515dc35ff3a93423478b2143e771d"], "title": "Update README.md", "message": "Update README.md", "author_name": "Administrator", "author_email": "admin@example.com", "authored_date": "2021-02-15T15:52:52.000+00:00", "committer_name": "Administrator", "committer_email": "admin@example.com", "committed_date": "2021-02-15T15:52:52.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/commit/27329d3afac51fbf2762428e12f2635d1137c549", "stats": {"additions": 6, "deletions": 0, "total": 6}, "project_id": 25156633}, "emitted_at": 1686568048416} -{"stream": "commits", "data": {"id": "b362ea7aa65515dc35ff3a93423478b2143e771d", "short_id": "b362ea7a", "created_at": "2021-02-15T15:52:03.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Administrator", "author_email": "admin@example.com", "authored_date": "2021-02-15T15:52:03.000+00:00", "committer_name": "Administrator", "committer_email": "admin@example.com", "committed_date": "2021-02-15T15:52:03.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/commit/b362ea7aa65515dc35ff3a93423478b2143e771d", "stats": {"additions": 2, "deletions": 0, "total": 2}, "project_id": 25156633}, "emitted_at": 1686568048417} -{"stream": "jobs", "data": {"id": 1108959782, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.294Z", "started_at": "2021-03-18T12:51:07.646Z", "finished_at": "2021-03-18T12:51:51.309Z", "erased_at": null, "duration": 43.662407, "queued_duration": 1.180926, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "11:08 AM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959782", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2200, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1686568098000} -{"stream": "jobs", "data": {"id": 1108959779, "status": "failed", "stage": "test", "name": "test-code-job1", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.279Z", "started_at": "2021-03-18T12:51:07.943Z", "finished_at": "2021-03-18T12:51:50.943Z", "erased_at": null, "duration": 42.999853, "queued_duration": 1.349274, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "11:08 AM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959779", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2182, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1686568098001} -{"stream": "jobs", "data": {"id": 1108952832, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "ykurochkin/add-fake-CI-config", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:48:49.222Z", "started_at": "2021-03-18T12:48:50.732Z", "finished_at": "2021-03-18T12:49:37.961Z", "erased_at": null, "duration": 47.229034, "queued_duration": 1.422541, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "11:08 AM"}, "commit": {"id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "short_id": "028c02d9", "created_at": "2021-03-18T14:48:41.000+02:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef"], "title": "add fake CI config", "message": "add fake CI config\n", "author_name": "ykurochkin", "author_email": "zhenia.kurochkin@gmail.com", "authored_date": "2021-03-18T14:48:41.000+02:00", "committer_name": "ykurochkin", "committer_email": "zhenia.kurochkin@gmail.com", "committed_date": "2021-03-18T14:48:41.000+02:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/028c02d96f40afe9b4d1173c1d0f712dd6d07302"}, "pipeline": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108952832", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2223, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272631271, "runner_id": null, "commit_id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "project_id": 25157276}, "emitted_at": 1686568098411} -{"stream": "project_labels", "data": {"id": 19117004, "name": "Label 1", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#008000", "subscribed": false, "priority": null, "is_project_label": false, "project_id": 41541858}, "emitted_at": 1686568116129} -{"stream": "project_labels", "data": {"id": 19117017, "name": "Label 10", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#000080", "subscribed": false, "priority": null, "is_project_label": false, "project_id": 41541858}, "emitted_at": 1686568116131} -{"stream": "project_labels", "data": {"id": 19117018, "name": "Label 11", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#808080", "subscribed": false, "priority": null, "is_project_label": false, "project_id": 41541858}, "emitted_at": 1686568116131} -{"stream": "releases", "data": {"name": "First release", "tag_name": "fake-tag-6", "description": "Test Release", "created_at": "2021-03-18T12:44:12.497Z", "released_at": "2021-03-18T12:44:12.497Z", "upcoming_release": false, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "milestones": [1943704], "commit_path": "/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef", "tag_path": "/new-group-airbute/new-ci-test-project/-/tags/fake-tag-6", "assets": {"count": 4, "sources": [{"format": "zip", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.zip"}, {"format": "tar.gz", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.gz"}, {"format": "tar.bz2", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.bz2"}, {"format": "tar", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar"}], "links": []}, "evidences": [{"sha": "a616fdca9312ca5aa451bc1060ce91a672fd24cc0f4d", "filepath": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/evidences/855895.json", "collected_at": "2021-03-18T12:44:12.650Z"}], "_links": {"closed_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=closed", "closed_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=closed", "edit_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/edit", "merged_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=merged", "opened_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=opened", "opened_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=opened", "self": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6"}, "author_id": 8375961, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686568182190} -{"stream": "projects", "data": {"id": 41541858, "description": "Project description", "name": "Test Project 1", "name_with_namespace": "New Group Airbute / Test Public SG / Test SG Public 2 / Test Project 1", "path": "test-project-1", "path_with_namespace": "new-group-airbute/test-public-sg/test-sg-public-2/test-project-1", "created_at": "2022-12-02T08:50:08.842Z", "default_branch": "master", "tag_list": [], "topics": [], "ssh_url_to_repo": "git@gitlab.com:new-group-airbute/test-public-sg/test-sg-public-2/test-project-1.git", "http_url_to_repo": "https://gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-project-1.git", "web_url": "https://gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-project-1", "readme_url": "https://gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-project-1/-/blob/master/README.md", "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2022-12-02T08:50:08.842Z", "namespace": {"id": 61014943, "name": "Test SG Public 2", "path": "test-sg-public-2", "kind": "group", "full_path": "new-group-airbute/test-public-sg/test-sg-public-2", "parent_id": 61014902, "avatar_url": null, "web_url": "https://gitlab.com/groups/new-group-airbute/test-public-sg/test-sg-public-2"}, "container_registry_image_prefix": "registry.gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-project-1", "_links": {"self": "https://gitlab.com/api/v4/projects/41541858", "issues": "https://gitlab.com/api/v4/projects/41541858/issues", "merge_requests": "https://gitlab.com/api/v4/projects/41541858/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/41541858/repository/branches", "labels": "https://gitlab.com/api/v4/projects/41541858/labels", "events": "https://gitlab.com/api/v4/projects/41541858/events", "members": "https://gitlab.com/api/v4/projects/41541858/members", "cluster_agents": "https://gitlab.com/api/v4/projects/41541858/cluster_agents"}, "packages_enabled": true, "empty_repo": false, "archived": false, "visibility": "private", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2022-12-03T08:50:08.883Z"}, "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "jobs_enabled": true, "snippets_enabled": true, "container_registry_enabled": true, "service_desk_enabled": true, "service_desk_address": "contact-project+new-group-airbute-test-public-sg-test-sg-public-2-test-project-41541858-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "enabled", "repository_access_level": "enabled", "merge_requests_access_level": "enabled", "forking_access_level": "enabled", "wiki_access_level": "enabled", "builds_access_level": "enabled", "snippets_access_level": "enabled", "pages_access_level": "private", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "enabled", "feature_flags_access_level": "enabled", "infrastructure_access_level": "enabled", "monitor_access_level": "enabled", "emails_disabled": false, "emails_enabled": true, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": "gitlab_project", "import_status": "finished", "import_error": null, "open_issues_count": 0, "description_html": "

      Project description

      ", "updated_at": "2022-12-02T08:50:11.170Z", "ci_default_git_depth": 20, "ci_forward_deployment_enabled": true, "ci_forward_deployment_rollback_allowed": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941JLqwDRN64-__uzBXcgc5", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": "", "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": false, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 1, "storage_size": 125829, "repository_size": 125829, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 0, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "autoclose_referenced_issues": true, "external_authorization_classification_label": "", "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "permissions": {"project_access": {"access_level": 40, "notification_level": 3}, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1690448723961} -{"stream": "projects", "data": {"id": 41551658, "description": null, "name": "Test_project_in_nested_subgroup", "name_with_namespace": "New Group Airbute / Test Public SG / Test SG Public 2 / Test Private SubSubG 1 / Test_project_in_nested_subgroup", "path": "test_project_in_nested_subgroup", "path_with_namespace": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup", "created_at": "2022-12-02T14:26:55.282Z", "default_branch": "main", "tag_list": [], "topics": [], "ssh_url_to_repo": "git@gitlab.com:new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup.git", "http_url_to_repo": "https://gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup.git", "web_url": "https://gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup", "readme_url": "https://gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup/-/blob/main/README.md", "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2022-12-02T14:26:55.282Z", "namespace": {"id": 61015181, "name": "Test Private SubSubG 1", "path": "test-private-subsubg-1", "kind": "group", "full_path": "new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1", "parent_id": 61014943, "avatar_url": null, "web_url": "https://gitlab.com/groups/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1"}, "container_registry_image_prefix": "registry.gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup", "_links": {"self": "https://gitlab.com/api/v4/projects/41551658", "issues": "https://gitlab.com/api/v4/projects/41551658/issues", "merge_requests": "https://gitlab.com/api/v4/projects/41551658/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/41551658/repository/branches", "labels": "https://gitlab.com/api/v4/projects/41551658/labels", "events": "https://gitlab.com/api/v4/projects/41551658/events", "members": "https://gitlab.com/api/v4/projects/41551658/members", "cluster_agents": "https://gitlab.com/api/v4/projects/41551658/cluster_agents"}, "packages_enabled": true, "empty_repo": false, "archived": false, "visibility": "private", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2022-12-03T14:26:55.314Z"}, "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "jobs_enabled": true, "snippets_enabled": true, "container_registry_enabled": true, "service_desk_enabled": true, "service_desk_address": "contact-project+new-group-airbute-test-public-sg-test-sg-public-2-test-private-41551658-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "enabled", "repository_access_level": "enabled", "merge_requests_access_level": "enabled", "forking_access_level": "enabled", "wiki_access_level": "enabled", "builds_access_level": "enabled", "snippets_access_level": "enabled", "pages_access_level": "private", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "enabled", "feature_flags_access_level": "enabled", "infrastructure_access_level": "enabled", "monitor_access_level": "enabled", "emails_disabled": false, "emails_enabled": true, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": null, "import_status": "none", "import_error": null, "open_issues_count": 0, "description_html": "", "updated_at": "2022-12-02T14:26:56.266Z", "ci_default_git_depth": 20, "ci_forward_deployment_enabled": true, "ci_forward_deployment_rollback_allowed": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941hyrJGkPgfF9b5KARxqHr", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": "", "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": true, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 1, "storage_size": 73400, "repository_size": 73400, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 0, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "autoclose_referenced_issues": true, "external_authorization_classification_label": "", "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "permissions": {"project_access": null, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1690448724778} -{"stream": "projects", "data": {"id": 25032439, "description": null, "name": "documentation", "name_with_namespace": "airbyte.io / documentation", "path": "documentation", "path_with_namespace": "airbyte.io/documentation", "created_at": "2021-03-10T17:16:53.200Z", "default_branch": "main", "tag_list": [], "topics": [], "ssh_url_to_repo": "git@gitlab.com:airbyte.io/documentation.git", "http_url_to_repo": "https://gitlab.com/airbyte.io/documentation.git", "web_url": "https://gitlab.com/airbyte.io/documentation", "readme_url": null, "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2021-03-10T17:16:53.200Z", "namespace": {"id": 11266951, "name": "airbyte.io", "path": "airbyte.io", "kind": "group", "full_path": "airbyte.io", "parent_id": null, "avatar_url": null, "web_url": "https://gitlab.com/groups/airbyte.io"}, "container_registry_image_prefix": "registry.gitlab.com/airbyte.io/documentation", "_links": {"self": "https://gitlab.com/api/v4/projects/25032439", "issues": "https://gitlab.com/api/v4/projects/25032439/issues", "merge_requests": "https://gitlab.com/api/v4/projects/25032439/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/25032439/repository/branches", "labels": "https://gitlab.com/api/v4/projects/25032439/labels", "events": "https://gitlab.com/api/v4/projects/25032439/events", "members": "https://gitlab.com/api/v4/projects/25032439/members", "cluster_agents": "https://gitlab.com/api/v4/projects/25032439/cluster_agents"}, "packages_enabled": true, "empty_repo": true, "archived": false, "visibility": "private", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2021-03-11T17:16:53.215Z"}, "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "jobs_enabled": true, "snippets_enabled": true, "container_registry_enabled": true, "service_desk_enabled": true, "service_desk_address": "contact-project+airbyte-io-documentation-25032439-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "enabled", "repository_access_level": "enabled", "merge_requests_access_level": "enabled", "forking_access_level": "enabled", "wiki_access_level": "enabled", "builds_access_level": "enabled", "snippets_access_level": "enabled", "pages_access_level": "private", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "enabled", "feature_flags_access_level": "enabled", "infrastructure_access_level": "enabled", "monitor_access_level": "enabled", "emails_disabled": false, "emails_enabled": true, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": null, "import_status": "none", "import_error": null, "open_issues_count": 0, "description_html": "", "updated_at": "2022-03-23T13:23:04.923Z", "ci_default_git_depth": 50, "ci_forward_deployment_enabled": true, "ci_forward_deployment_rollback_allowed": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941iwELAs9x3hqVbY3Bo_q4", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": "", "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": true, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 0, "storage_size": 0, "repository_size": 0, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 0, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "autoclose_referenced_issues": true, "approvals_before_merge": 0, "mirror": false, "external_authorization_classification_label": "", "marked_for_deletion_at": null, "marked_for_deletion_on": null, "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "issues_template": null, "merge_requests_template": null, "merge_pipelines_enabled": false, "merge_trains_enabled": false, "allow_pipeline_trigger_approve_deployment": false, "permissions": {"project_access": null, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1690448725290} -{"stream": "branches", "data": {"name": "master", "commit": {"id": "bcdfbfd57c8f3cd6cd65998464bb71a562d49948", "short_id": "bcdfbfd5", "created_at": "2019-03-06T09:52:24.000+01:00", "parent_ids": [], "title": "Initial template creation", "message": "Initial template creation\n", "author_name": "GitLab", "author_email": "root@localhost", "authored_date": "2019-03-06T09:52:24.000+01:00", "committer_name": "Jason Lenny", "committer_email": "jlenny@gitlab.com", "committed_date": "2019-03-06T09:52:24.000+01:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-project-1/-/commit/bcdfbfd57c8f3cd6cd65998464bb71a562d49948"}, "merged": false, "protected": true, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": true, "web_url": "https://gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-project-1/-/tree/master", "commit_id": "bcdfbfd57c8f3cd6cd65998464bb71a562d49948", "project_id": 41541858}, "emitted_at": 1686568039222} -{"stream": "branches", "data": {"name": "main", "commit": {"id": "fb24e6736b3a959a59e49b56d2d83a28ea3ae15b", "short_id": "fb24e673", "created_at": "2022-12-02T14:26:55.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2022-12-02T14:26:55.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2022-12-02T14:26:55.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup/-/commit/fb24e6736b3a959a59e49b56d2d83a28ea3ae15b"}, "merged": false, "protected": true, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": true, "web_url": "https://gitlab.com/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup/-/tree/main", "commit_id": "fb24e6736b3a959a59e49b56d2d83a28ea3ae15b", "project_id": 41551658}, "emitted_at": 1686568039632} -{"stream": "branches", "data": {"name": "at-adipisci-ducimus-qui-nihil", "commit": {"id": "e10493c095260599a73a32def40249a4c389e354", "short_id": "e10493c0", "created_at": "2021-02-15T15:55:06.000+00:00", "parent_ids": ["763258bc3b5803074eb2c23eb069275f9716a2c1"], "title": "Nisi ipsam rem repudiandae.", "message": "Nisi ipsam rem repudiandae.", "author_name": "Administrator", "author_email": "admin@example.com", "authored_date": "2021-02-15T15:55:06.000+00:00", "committer_name": "Administrator", "committer_email": "admin@example.com", "committed_date": "2021-02-15T15:55:06.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/commit/e10493c095260599a73a32def40249a4c389e354"}, "merged": false, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/tree/at-adipisci-ducimus-qui-nihil", "commit_id": "e10493c095260599a73a32def40249a4c389e354", "project_id": 25156633}, "emitted_at": 1686568040451} -{"stream": "merge_request_commits", "data": {"id": 92098762, "iid": 30, "project_id": 25156633, "title": "Id blanditiis consequatur ut.", "description": "##### Voluptatum\nPorro et quo. Laborum molestias ducimus. Labore dolorum adipisci. Quisquam est quis. Sint accusamus maxime.\n* Veritatis. \n* Eos. \n* Adipisci. \n* Quibusdam. \n* Sint. \n* Consequuntur. \n* Hic. \n* Voluptate. \n* Velit.", "state": "opened", "created_at": "2021-02-15T15:55:38.117Z", "updated_at": "2021-02-15T15:55:38.117Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "laudantium-unde-et-iste-et", "user_notes_count": 15, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25156633, "target_project_id": 25156633, "labels": ["asperiores-ex", "quidem-labore", "sed-consequuntur"], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "d49703b85913ee7a8c85e1893057ef4cdb06ff85", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": null, "prepared_at": "2021-02-15T15:55:38.117Z", "reference": "!30", "references": {"short": "!30", "relative": "!30", "full": "airbyte.io/ci-test-project!30"}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/merge_requests/30", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": "18", "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": null, "diff_refs": {"base_sha": "27329d3afac51fbf2762428e12f2635d1137c549", "head_sha": "d49703b85913ee7a8c85e1893057ef4cdb06ff85", "start_sha": "27329d3afac51fbf2762428e12f2635d1137c549"}, "merge_error": null, "first_contribution": true, "user": {"can_merge": true}, "merge_request_iid": 30}, "emitted_at": 1686568154022} -{"stream": "merge_request_commits", "data": {"id": 92098761, "iid": 29, "project_id": 25156633, "title": "Fugiat aut voluptatem voluptas.", "description": "###### Unde\nEligendi nemo quam. Veritatis delectus iure. Placeat ut odit. Officiis accusantium sit. Eos sequi cupiditate.\n# Eum", "state": "opened", "created_at": "2021-02-15T15:55:34.534Z", "updated_at": "2021-02-15T15:55:34.534Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ea-dolor-quia-et-sint", "user_notes_count": 4, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25156633, "target_project_id": 25156633, "labels": ["earum-eaque", "nisi-et", "sed-voluptatem"], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "45afadfbf4eb1a9d6468950b23e8557bf72445fa", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": null, "prepared_at": "2021-02-15T15:55:34.534Z", "reference": "!29", "references": {"short": "!29", "relative": "!29", "full": "airbyte.io/ci-test-project!29"}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/merge_requests/29", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": "17", "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": null, "diff_refs": {"base_sha": "27329d3afac51fbf2762428e12f2635d1137c549", "head_sha": "45afadfbf4eb1a9d6468950b23e8557bf72445fa", "start_sha": "27329d3afac51fbf2762428e12f2635d1137c549"}, "merge_error": null, "first_contribution": true, "user": {"can_merge": true}, "merge_request_iid": 29}, "emitted_at": 1686568154423} -{"stream": "merge_request_commits", "data": {"id": 92098758, "iid": 28, "project_id": 25156633, "title": "Delectus sit quod repellendus.", "description": "###### Corporis\nMolestias eius corrupti. Est maiores ut. Deleniti itaque deserunt. Perspiciatis quis et. Non et quia.\n### Dolorum\nNihil at et. Eligendi recusandae omnis. Eaque ratione dolorem.\n### Quam\nRem ad vel. Officiis sint voluptatem. Asperiores odit non.\n0. Hic. \n1. Labore. \n2. Voluptates. \n3. Dolores. \n4. Laborum. \n5. Non. \n6. Odit.", "state": "opened", "created_at": "2021-02-15T15:55:31.164Z", "updated_at": "2021-02-15T15:55:31.164Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ipsum-consequatur-et-in-et", "user_notes_count": 19, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25156633, "target_project_id": 25156633, "labels": ["nisi-et", "omnis-assumenda", "ut-incidunt"], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "5b8695fe02d2856fa9e3249d757aea89832b8d2e", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": null, "prepared_at": "2021-02-15T15:55:31.164Z", "reference": "!28", "references": {"short": "!28", "relative": "!28", "full": "airbyte.io/ci-test-project!28"}, "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/merge_requests/28", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": "15", "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": null, "diff_refs": {"base_sha": "27329d3afac51fbf2762428e12f2635d1137c549", "head_sha": "5b8695fe02d2856fa9e3249d757aea89832b8d2e", "start_sha": "27329d3afac51fbf2762428e12f2635d1137c549"}, "merge_error": null, "first_contribution": true, "user": {"can_merge": true}, "merge_request_iid": 28}, "emitted_at": 1686568154935} +{"stream": "merge_requests", "data": {"id": 92594931, "iid": 3, "project_id": 25157276, "title": "add fake CI config", "description": "", "state": "merged", "created_at": "2021-03-18T12:49:13.091Z", "updated_at": "2021-03-18T12:51:06.319Z", "merged_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merge_user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merged_at": "2021-03-18T12:51:06.470Z", "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/add-fake-CI-config", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "not_open", "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "merge_commit_sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:49:13.091Z", "reference": "!3", "references": {"short": "!3", "relative": "!3", "full": "new-group-airbute/new-ci-test-project!3"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/3", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": 8375961}, "emitted_at": 1696948541619} +{"stream": "merge_requests", "data": {"id": 92593913, "iid": 2, "project_id": 25157276, "title": "update readme.md", "description": "", "state": "opened", "created_at": "2021-03-18T12:42:30.200Z", "updated_at": "2021-03-18T12:42:30.200Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/test-branch", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:42:30.200Z", "reference": "!2", "references": {"short": "!2", "relative": "!2", "full": "new-group-airbute/new-ci-test-project!2"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/2", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1696948541622} +{"stream": "merge_requests", "data": {"id": 92111504, "iid": 1, "project_id": 25157276, "title": "Draft: Resolve \"Fake Issue 30\"", "description": "Closes #31", "state": "opened", "created_at": "2021-03-15T16:08:05.071Z", "updated_at": "2021-03-15T16:08:05.071Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "31-fake-issue-30", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [8375961], "assignee": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": ["bug"], "draft": true, "work_in_progress": true, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "cannot_be_merged", "detailed_merge_status": "draft_status", "sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-15T16:08:05.071Z", "reference": "!1", "references": {"short": "!1", "relative": "!1", "full": "new-group-airbute/new-ci-test-project!1"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/1", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": true, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": 8375961, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1696948541624} +{"stream": "groups", "data": {"id": 68657749, "web_url": "https://gitlab.com/groups/empty-group4", "name": "Empty Group", "path": "empty-group4", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": false, "emails_enabled": true, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "default_branch_protection_defaults": {"allowed_to_push": [{"access_level": 30}], "allow_force_push": true, "allowed_to_merge": [{"access_level": 30}]}, "avatar_url": null, "request_access_enabled": true, "full_name": "Empty Group", "full_path": "empty-group4", "created_at": "2023-06-09T13:47:19.446Z", "parent_id": null, "organization_id": 1, "shared_runners_setting": "enabled", "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941-x4xtBM6_zdFj-ED8QF8", "prevent_sharing_groups_outside_hierarchy": false, "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": []}, "emitted_at": 1704732558754} +{"stream": "groups", "data": {"id": 11329647, "web_url": "https://gitlab.com/groups/new-group-airbute", "name": "New Group Airbute", "path": "new-group-airbute", "description": "", "visibility": "public", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": false, "emails_enabled": true, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "default_branch_protection_defaults": {"allowed_to_push": [{"access_level": 30}], "allow_force_push": true, "allowed_to_merge": [{"access_level": 30}]}, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute", "full_path": "new-group-airbute", "created_at": "2021-03-15T15:55:53.613Z", "parent_id": null, "organization_id": 1, "shared_runners_setting": "enabled", "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941-PhosPap-Sf1UxL1g6m4", "prevent_sharing_groups_outside_hierarchy": false, "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 25157276, "path_with_namespace": "new-group-airbute/new-ci-test-project"}]}, "emitted_at": 1704732559167} +{"stream": "groups", "data": {"id": 61014882, "web_url": "https://gitlab.com/groups/new-group-airbute/test-subgroup-airbyte/test-private-sg", "name": "Test Private SG", "path": "test-private-sg", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": false, "emails_enabled": true, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "default_branch_protection_defaults": {"allowed_to_push": [{"access_level": 30}], "allow_force_push": true, "allowed_to_merge": [{"access_level": 30}]}, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Subgroup Airbyte / Test Private SG", "full_path": "new-group-airbute/test-subgroup-airbyte/test-private-sg", "created_at": "2022-12-02T08:46:22.648Z", "parent_id": 61014863, "organization_id": 1, "shared_runners_setting": "enabled", "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941bjUaJQy2zzar-JmNBjfq", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": []}, "emitted_at": 1704732559573} +{"stream": "epic_issues", "data": {"id": 120214448, "iid": 31, "project_id": 25156633, "title": "Unit tests", "description": null, "state": "opened", "created_at": "2022-12-11T10:50:25.940Z", "updated_at": "2022-12-11T10:50:25.940Z", "closed_at": null, "closed_by": null, "labels": [], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/airbyte.io/ci-test-project/-/issues/31", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "weight": null, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25156633/issues/31", "notes": "https://gitlab.com/api/v4/projects/25156633/issues/31/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25156633/issues/31/award_emoji", "project": "https://gitlab.com/api/v4/projects/25156633", "closed_as_duplicate_of": null}, "references": {"short": "#31", "relative": "#31", "full": "airbyte.io/ci-test-project#31"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "epic_iid": 1, "epic": {"id": 678569, "iid": 1, "title": "Source Gitlab: certify to Beta", "url": "/groups/airbyte.io/-/epics/1", "group_id": 11266951, "human_readable_end_date": "Dec 30, 2022", "human_readable_timestamp": "Past due"}, "iteration": null, "epic_issue_id": 1899479, "relative_position": 0, "milestone_id": null, "assignee_id": null, "author_id": 8375961}, "emitted_at": 1696949059273} +{"stream": "epic_issues", "data": {"id": 80659730, "iid": 13, "project_id": 25032440, "title": "Start a free trial of GitLab Gold - no credit card required :rocket:", "description": "At any point while using the free version of GitLab you can start a trial of GitLab Gold for free for 30 days. With a GitLab Gold trial, you'll get access to all of the most popular features across all of the paid tiers within GitLab. \n \n:white_check_mark: Reduce risk by requiring team leaders to approve merge requests.\n \n:white_check_mark: Ensure code quality with Multiple code reviews.\n \n:white_check_mark: Run your CI pipelines for up to 50,000 minutes (~9,500 CI builds).\n \n:white_check_mark: Plan and organize parallel development with multiple issue boards.\n \n:white_check_mark: Report on the productivity of each team in your organization by using issue analytics. \n \n:white_check_mark: Dynamically scan Docker images for vulnerabilities before production pushes. \n \n:white_check_mark: Scan security vulnerabilities, license compliance and dependencies in your CI pipelines. \n \n:white_check_mark: Get alerted when your application performance degrades. \n \n:white_check_mark: And so much more, [you can view all the features here](https://about.gitlab.com/pricing/gitlab-com/feature-comparison/). \n \n## Next steps\n* [ ] [Click here to start a trial of GitLab Gold.](https://gitlab.com/-/trial_registrations/new?glm_content=user_onboarding_whats_in_paid_tiers&glm_source=gitlab.com)", "state": "opened", "created_at": "2021-03-10T17:16:56.091Z", "updated_at": "2023-10-10T11:44:39.796Z", "closed_at": null, "closed_by": null, "labels": ["Novice"], "milestone": null, "assignees": [8375961], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/airbyte.io/learn-gitlab/-/issues/13", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 1, "completed_count": 0}, "weight": null, "blocking_issues_count": 0, "has_tasks": false, "_links": {"self": "https://gitlab.com/api/v4/projects/25032440/issues/13", "notes": "https://gitlab.com/api/v4/projects/25032440/issues/13/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25032440/issues/13/award_emoji", "project": "https://gitlab.com/api/v4/projects/25032440", "closed_as_duplicate_of": null}, "references": {"short": "#13", "relative": "#13", "full": "airbyte.io/learn-gitlab#13"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "epic_iid": 1, "epic": {"id": 678569, "iid": 1, "title": "Source Gitlab: certify to Beta", "url": "/groups/airbyte.io/-/epics/1", "group_id": 11266951, "human_readable_end_date": "Dec 30, 2022", "human_readable_timestamp": "Past due"}, "iteration": null, "epic_issue_id": 3762298, "relative_position": -513, "milestone_id": null, "assignee_id": 8375961, "author_id": 8375961}, "emitted_at": 1696949059274} +{"stream": "issues", "data": {"id": 80943819, "iid": 32, "project_id": 25157276, "title": "Fake Issue 31", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:42.206Z", "updated_at": "2021-03-15T15:22:42.206Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/32", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/32", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/32/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/32/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#32", "relative": "#32", "full": "new-group-airbute/new-ci-test-project#32"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1696949354572} +{"stream": "issues", "data": {"id": 80943818, "iid": 31, "project_id": 25157276, "title": "Fake Issue 30", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:41.337Z", "updated_at": "2021-03-15T16:08:06.041Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 1, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/31", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/31", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/31/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/31/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#31", "relative": "#31", "full": "new-group-airbute/new-ci-test-project#31"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1696949354574} +{"stream": "issues", "data": {"id": 80943817, "iid": 30, "project_id": 25157276, "title": "Fake Issue 29", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:40.529Z", "updated_at": "2021-03-15T15:22:40.529Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/30", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/30", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/30/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/30/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#30", "relative": "#30", "full": "new-group-airbute/new-ci-test-project#30"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1696949354576} +{"stream": "project_members", "data": {"access_level": 40, "created_at": "2021-03-15T15:08:36.746Z", "created_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "project_id": 25157276}, "emitted_at": 1696949674671} +{"stream": "epics", "data": {"id": 1977226, "iid": 2, "color": "#1068bf", "text_color": "#FFFFFF", "group_id": 11266951, "parent_id": null, "parent_iid": null, "title": "Test epic", "description": null, "confidential": false, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "start_date": null, "start_date_is_fixed": false, "start_date_fixed": null, "start_date_from_inherited_source": null, "start_date_from_milestones": null, "end_date": null, "due_date": null, "due_date_is_fixed": false, "due_date_fixed": null, "due_date_from_inherited_source": null, "due_date_from_milestones": null, "state": "opened", "web_edit_url": "/groups/airbyte.io/-/epics/2", "web_url": "https://gitlab.com/groups/airbyte.io/-/epics/2", "references": {"short": "&2", "relative": "&2", "full": "airbyte.io&2"}, "created_at": "2023-10-10T10:37:36.529Z", "updated_at": "2023-10-10T11:44:50.107Z", "closed_at": null, "labels": [], "upvotes": 0, "downvotes": 0, "_links": {"self": "https://gitlab.com/api/v4/groups/11266951/epics/2", "epic_issues": "https://gitlab.com/api/v4/groups/11266951/epics/2/issues", "group": "https://gitlab.com/api/v4/groups/11266951", "parent": null}, "author_id": 8375961}, "emitted_at": 1696949906098} +{"stream": "epics", "data": {"id": 678569, "iid": 1, "color": "#1068bf", "text_color": "#FFFFFF", "group_id": 11266951, "parent_id": null, "parent_iid": null, "title": "Source Gitlab: certify to Beta", "description": "Lorem ipsum", "confidential": false, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "start_date": "2022-12-11", "start_date_is_fixed": true, "start_date_fixed": "2022-12-11", "start_date_from_inherited_source": null, "start_date_from_milestones": null, "end_date": "2022-12-30", "due_date": "2022-12-30", "due_date_is_fixed": true, "due_date_fixed": "2022-12-30", "due_date_from_inherited_source": null, "due_date_from_milestones": null, "state": "opened", "web_edit_url": "/groups/airbyte.io/-/epics/1", "web_url": "https://gitlab.com/groups/airbyte.io/-/epics/1", "references": {"short": "&1", "relative": "&1", "full": "airbyte.io&1"}, "created_at": "2022-12-11T10:50:04.280Z", "updated_at": "2023-10-10T11:44:49.999Z", "closed_at": null, "labels": [], "upvotes": 1, "downvotes": 0, "_links": {"self": "https://gitlab.com/api/v4/groups/11266951/epics/1", "epic_issues": "https://gitlab.com/api/v4/groups/11266951/epics/1/issues", "group": "https://gitlab.com/api/v4/groups/11266951", "parent": null}, "author_id": 8375961}, "emitted_at": 1696949906100} +{"stream": "commits", "data": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98", "stats": {"additions": 14, "deletions": 0, "total": 14}, "project_id": 25157276}, "emitted_at": 1703256223650} +{"stream": "commits", "data": {"id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "short_id": "028c02d9", "created_at": "2021-03-18T14:48:41.000+02:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef"], "title": "add fake CI config", "message": "add fake CI config\n", "author_name": "ykurochkin", "author_email": "zhenia.kurochkin@gmail.com", "authored_date": "2021-03-18T14:48:41.000+02:00", "committer_name": "ykurochkin", "committer_email": "zhenia.kurochkin@gmail.com", "committed_date": "2021-03-18T14:48:41.000+02:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/028c02d96f40afe9b4d1173c1d0f712dd6d07302", "stats": {"additions": 14, "deletions": 0, "total": 14}, "project_id": 25157276}, "emitted_at": 1703256223651} +{"stream": "commits", "data": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef", "stats": {"additions": 2, "deletions": 0, "total": 2}, "project_id": 25157276}, "emitted_at": 1703256223652} +{"stream": "jobs", "data": {"id": 1108959782, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.294Z", "started_at": "2021-03-18T12:51:07.646Z", "finished_at": "2021-03-18T12:51:51.309Z", "erased_at": null, "duration": 43.662407, "queued_duration": 1.180926, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "5:02 PM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959782", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2200, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "archived": false, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1704733346934} +{"stream": "jobs", "data": {"id": 1108959779, "status": "failed", "stage": "test", "name": "test-code-job1", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.279Z", "started_at": "2021-03-18T12:51:07.943Z", "finished_at": "2021-03-18T12:51:50.943Z", "erased_at": null, "duration": 42.999853, "queued_duration": 1.349274, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "5:02 PM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959779", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2182, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "archived": false, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1704733346935} +{"stream": "jobs", "data": {"id": 1108952832, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "ykurochkin/add-fake-CI-config", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:48:49.222Z", "started_at": "2021-03-18T12:48:50.732Z", "finished_at": "2021-03-18T12:49:37.961Z", "erased_at": null, "duration": 47.229034, "queued_duration": 1.422541, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "5:02 PM"}, "commit": {"id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "short_id": "028c02d9", "created_at": "2021-03-18T14:48:41.000+02:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef"], "title": "add fake CI config", "message": "add fake CI config\n", "author_name": "ykurochkin", "author_email": "zhenia.kurochkin@gmail.com", "authored_date": "2021-03-18T14:48:41.000+02:00", "committer_name": "ykurochkin", "committer_email": "zhenia.kurochkin@gmail.com", "committed_date": "2021-03-18T14:48:41.000+02:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/028c02d96f40afe9b4d1173c1d0f712dd6d07302"}, "pipeline": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108952832", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2223, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "archived": false, "tag_list": [], "user_id": 8375961, "pipeline_id": 272631271, "runner_id": null, "commit_id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "project_id": 25157276}, "emitted_at": 1704733347322} +{"stream": "project_labels", "data": {"id": 19116944, "name": "Label 1", "description": null, "description_html": "", "text_color": "#1F1E24", "color": "#ffff00", "subscribed": false, "priority": null, "is_project_label": true, "project_id": 25157276}, "emitted_at": 1696950582334} +{"stream": "project_labels", "data": {"id": 19117004, "name": "Label 1", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#008000", "subscribed": false, "priority": null, "is_project_label": false, "project_id": 25157276}, "emitted_at": 1696950582334} +{"stream": "project_labels", "data": {"id": 19116954, "name": "Label 10", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#ff00ff", "subscribed": false, "priority": null, "is_project_label": true, "project_id": 25157276}, "emitted_at": 1696950582334} +{"stream": "releases", "data": {"name": "First release", "tag_name": "fake-tag-6", "description": "Test Release", "created_at": "2021-03-18T12:44:12.497Z", "released_at": "2021-03-18T12:44:12.497Z", "upcoming_release": false, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "milestones": [1943704], "commit_path": "/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef", "tag_path": "/new-group-airbute/new-ci-test-project/-/tags/fake-tag-6", "assets": {"count": 4, "sources": [{"format": "zip", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.zip"}, {"format": "tar.gz", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.gz"}, {"format": "tar.bz2", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.bz2"}, {"format": "tar", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar"}], "links": []}, "evidences": [{"sha": "a616fdca9312ca5aa451bc1060ce91a672fd24cc0f4d", "filepath": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/evidences/855895.json", "collected_at": "2021-03-18T12:44:12.650Z"}], "_links": {"closed_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=closed", "closed_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=closed", "edit_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/edit", "merged_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=merged", "opened_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=opened", "opened_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=opened", "self": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6"}, "author_id": 8375961, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1703256897835} +{"stream": "projects", "data": {"id": 25157276, "description": "", "name": "New CI Test Project ", "name_with_namespace": "New Group Airbute / New CI Test Project ", "path": "new-ci-test-project", "path_with_namespace": "new-group-airbute/new-ci-test-project", "created_at": "2021-03-15T15:08:36.498Z", "default_branch": "master", "tag_list": [], "topics": [], "ssh_url_to_repo": "git@gitlab.com:new-group-airbute/new-ci-test-project.git", "http_url_to_repo": "https://gitlab.com/new-group-airbute/new-ci-test-project.git", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project", "readme_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/blob/master/README.md", "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2022-12-13T09:39:47.235Z", "namespace": {"id": 11329647, "name": "New Group Airbute", "path": "new-group-airbute", "kind": "group", "full_path": "new-group-airbute", "parent_id": null, "avatar_url": null, "web_url": "https://gitlab.com/groups/new-group-airbute"}, "container_registry_image_prefix": "registry.gitlab.com/new-group-airbute/new-ci-test-project", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276", "issues": "https://gitlab.com/api/v4/projects/25157276/issues", "merge_requests": "https://gitlab.com/api/v4/projects/25157276/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/25157276/repository/branches", "labels": "https://gitlab.com/api/v4/projects/25157276/labels", "events": "https://gitlab.com/api/v4/projects/25157276/events", "members": "https://gitlab.com/api/v4/projects/25157276/members", "cluster_agents": "https://gitlab.com/api/v4/projects/25157276/cluster_agents"}, "code_suggestions": true, "packages_enabled": true, "empty_repo": false, "archived": false, "visibility": "private", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2021-03-16T15:08:36.518Z"}, "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "jobs_enabled": true, "snippets_enabled": true, "container_registry_enabled": true, "service_desk_enabled": true, "service_desk_address": "contact-project+new-group-airbute-new-ci-test-project-25157276-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "private", "repository_access_level": "private", "merge_requests_access_level": "private", "forking_access_level": "enabled", "wiki_access_level": "enabled", "builds_access_level": "private", "snippets_access_level": "enabled", "pages_access_level": "private", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "enabled", "feature_flags_access_level": "enabled", "infrastructure_access_level": "enabled", "monitor_access_level": "enabled", "model_experiments_access_level": "enabled", "model_registry_access_level": "enabled", "emails_disabled": false, "emails_enabled": true, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": null, "import_status": "none", "import_error": null, "open_issues_count": 31, "description_html": "", "updated_at": "2023-05-23T12:12:18.623Z", "ci_default_git_depth": 50, "ci_forward_deployment_enabled": true, "ci_forward_deployment_rollback_allowed": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941eMJgWDU69xyyshaNsaTZ", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": "", "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": true, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 3, "storage_size": 9061, "repository_size": 251, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 8810, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "autoclose_referenced_issues": true, "external_authorization_classification_label": "", "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "permissions": {"project_access": {"access_level": 40, "notification_level": 3}, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1703257466384} +{"stream": "branches", "data": {"name": "31-fake-issue-30", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/31-fake-issue-30", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1703257845052} +{"stream": "branches", "data": {"name": "master", "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "merged": false, "protected": true, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": true, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/master", "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1703257845053} +{"stream": "branches", "data": {"name": "new-test-branch", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/new-test-branch", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1703257845054} +{"stream": "merge_request_commits", "data": {"id": 92594931, "iid": 3, "project_id": 25157276, "title": "add fake CI config", "description": "", "state": "merged", "created_at": "2021-03-18T12:49:13.091Z", "updated_at": "2021-03-18T12:51:06.319Z", "merged_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merge_user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merged_at": "2021-03-18T12:51:06.470Z", "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/add-fake-CI-config", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "not_open", "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "merge_commit_sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:49:13.091Z", "reference": "!3", "references": {"short": "!3", "relative": "!3", "full": "new-group-airbute/new-ci-test-project!3"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/3", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": "1", "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "before_sha": "0000000000000000000000000000000000000000", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:48:50.166Z", "finished_at": "2021-03-18T12:49:38.084Z", "committed_at": null, "duration": 47, "queued_duration": null, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "Failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": true, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}}, "diff_refs": {"base_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "head_sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "start_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merge_error": null, "first_contribution": false, "user": {"can_merge": true}, "merge_request_iid": 3}, "emitted_at": 1696952086155} +{"stream": "merge_request_commits", "data": {"id": 92593913, "iid": 2, "project_id": 25157276, "title": "update readme.md", "description": "", "state": "opened", "created_at": "2021-03-18T12:42:30.200Z", "updated_at": "2021-03-18T12:42:30.200Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/test-branch", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:42:30.200Z", "reference": "!2", "references": {"short": "!2", "relative": "!2", "full": "new-group-airbute/new-ci-test-project!2"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/2", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": "1", "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": null, "diff_refs": {"base_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "head_sha": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "start_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merge_error": null, "first_contribution": false, "user": {"can_merge": true}, "merge_request_iid": 2}, "emitted_at": 1696952086460} +{"stream": "merge_request_commits", "data": {"id": 92111504, "iid": 1, "project_id": 25157276, "title": "Draft: Resolve \"Fake Issue 30\"", "description": "Closes #31", "state": "opened", "created_at": "2021-03-15T16:08:05.071Z", "updated_at": "2021-03-15T16:08:05.071Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "31-fake-issue-30", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [{"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}], "assignee": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": ["bug"], "draft": true, "work_in_progress": true, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "cannot_be_merged", "detailed_merge_status": "draft_status", "sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-15T16:08:05.071Z", "reference": "!1", "references": {"short": "!1", "relative": "!1", "full": "new-group-airbute/new-ci-test-project!1"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/1", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": true, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": null, "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": null, "diff_refs": {"base_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "head_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "start_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merge_error": null, "first_contribution": false, "user": {"can_merge": true}, "merge_request_iid": 1}, "emitted_at": 1696952086890} {"stream": "group_milestones", "data": {"id": 1943775, "iid": 21, "group_id": 11329647, "title": "Group Milestone 21", "description": null, "state": "active", "created_at": "2021-03-15T16:01:02.125Z", "updated_at": "2021-03-15T16:01:02.125Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/21"}, "emitted_at": 1686568104768} {"stream": "group_milestones", "data": {"id": 1943774, "iid": 20, "group_id": 11329647, "title": "Group Milestone 20", "description": null, "state": "active", "created_at": "2021-03-15T16:01:01.682Z", "updated_at": "2021-03-15T16:01:01.682Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/20"}, "emitted_at": 1686568104771} {"stream": "group_milestones", "data": {"id": 1943773, "iid": 19, "group_id": 11329647, "title": "Group Milestone 19", "description": null, "state": "active", "created_at": "2021-03-15T16:01:01.067Z", "updated_at": "2021-03-15T16:01:01.067Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/19"}, "emitted_at": 1686568104771} @@ -45,12 +43,12 @@ {"stream": "group_labels", "data": {"id": 19117004, "name": "Label 1", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#008000", "subscribed": false, "group_id": 11329647}, "emitted_at": 1686568123880} {"stream": "group_labels", "data": {"id": 19117017, "name": "Label 10", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#000080", "subscribed": false, "group_id": 11329647}, "emitted_at": 1686568123881} {"stream": "group_labels", "data": {"id": 19117018, "name": "Label 11", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#808080", "subscribed": false, "group_id": 11329647}, "emitted_at": 1686568123881} -{"stream": "users", "data": {"id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/760fcac88680c724a6b19c6bfd5b6718?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin"}, "emitted_at": 1686568218271} -{"stream": "users", "data": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "emitted_at": 1686568218274} -{"stream": "users", "data": {"id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/760fcac88680c724a6b19c6bfd5b6718?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin"}, "emitted_at": 1686568218612} -{"stream": "group_members", "data": {"access_level": 50, "created_at": "2023-06-09T13:47:19.592Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 68657749}, "emitted_at": 1686568111619} -{"stream": "group_members", "data": {"access_level": 50, "created_at": "2021-03-15T15:55:53.658Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1686568111909} -{"stream": "group_members", "data": {"access_level": 30, "created_at": "2021-03-15T15:55:53.998Z", "created_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "expires_at": null, "id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/760fcac88680c724a6b19c6bfd5b6718?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1686568111910} -{"stream": "tags", "data": {"name": "fake-tag-1", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686568185586} -{"stream": "tags", "data": {"name": "fake-tag-10", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686568185588} -{"stream": "tags", "data": {"name": "fake-tag-11", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686568185590} +{"stream": "users", "data": {"id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/760fcac88680c724a6b19c6bfd5b6718?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin"}, "emitted_at": 1696952237932} +{"stream": "users", "data": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "emitted_at": 1696952237934} +{"stream": "group_members", "data": {"access_level": 50, "created_at": "2021-03-15T15:55:53.658Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1696952386414} +{"stream": "group_members", "data": {"access_level": 30, "created_at": "2021-03-15T15:55:53.998Z", "created_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "expires_at": null, "id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/760fcac88680c724a6b19c6bfd5b6718?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1696952386416} +{"stream": "group_members", "data": {"access_level": 50, "created_at": "2022-12-02T08:46:22.834Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 61014882}, "emitted_at": 1696952387022} +{"stream": "tags", "data": {"name": "fake-tag-1", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1703258228301} +{"stream": "tags", "data": {"name": "fake-tag-10", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1703258228301} +{"stream": "tags", "data": {"name": "fake-tag-11", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1703258228301} +{"stream": "deployments", "data": {"id": 568087366, "iid": 1, "ref": "master", "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "created_at": "2023-10-10T09:56:02.273Z", "updated_at": "2023-10-10T09:56:02.273Z", "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "environment": {"id": 17305239, "name": "dev", "slug": "dev", "external_url": null, "created_at": "2023-10-10T09:56:02.188Z", "updated_at": "2023-10-10T09:56:02.188Z"}, "deployable": null, "status": "failed", "user_id": 8375961, "environment_id": 17305239, "user_username": "airbyte", "user_full_name": "Airbyte Team", "environment_name": "dev", "project_id": 25157276}, "emitted_at": 1696931771902} diff --git a/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records_with_ids.jsonl b/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records_with_ids.jsonl index 280b0ec036fa..69c1ed5a15d0 100644 --- a/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records_with_ids.jsonl +++ b/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records_with_ids.jsonl @@ -1,49 +1,50 @@ {"stream": "pipelines", "data": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "name": null}, "emitted_at": 1686567225920} {"stream": "pipelines", "data": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "name": null}, "emitted_at": 1686567225922} -{"stream": "releases", "data": {"name": "First release", "tag_name": "fake-tag-6", "description": "Test Release", "created_at": "2021-03-18T12:44:12.497Z", "released_at": "2021-03-18T12:44:12.497Z", "upcoming_release": false, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "milestones": [1943704], "commit_path": "/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef", "tag_path": "/new-group-airbute/new-ci-test-project/-/tags/fake-tag-6", "assets": {"count": 4, "sources": [{"format": "zip", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.zip"}, {"format": "tar.gz", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.gz"}, {"format": "tar.bz2", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.bz2"}, {"format": "tar", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar"}], "links": []}, "evidences": [{"sha": "a616fdca9312ca5aa451bc1060ce91a672fd24cc0f4d", "filepath": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/evidences/855895.json", "collected_at": "2021-03-18T12:44:12.650Z"}], "_links": {"closed_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=closed", "closed_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=closed", "edit_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/edit", "merged_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=merged", "opened_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=opened", "opened_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=opened", "self": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6"}, "author_id": 8375961, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686567224796} -{"stream": "jobs", "data": {"id": 1108959782, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.294Z", "started_at": "2021-03-18T12:51:07.646Z", "finished_at": "2021-03-18T12:51:51.309Z", "erased_at": null, "duration": 43.662407, "queued_duration": 1.180926, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "10:53 AM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959782", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2200, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1686567192490} -{"stream": "jobs", "data": {"id": 1108959779, "status": "failed", "stage": "test", "name": "test-code-job1", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.279Z", "started_at": "2021-03-18T12:51:07.943Z", "finished_at": "2021-03-18T12:51:50.943Z", "erased_at": null, "duration": 42.999853, "queued_duration": 1.349274, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "10:53 AM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959779", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2182, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1686567192491} -{"stream": "jobs", "data": {"id": 1108952832, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "ykurochkin/add-fake-CI-config", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:48:49.222Z", "started_at": "2021-03-18T12:48:50.732Z", "finished_at": "2021-03-18T12:49:37.961Z", "erased_at": null, "duration": 47.229034, "queued_duration": 1.422541, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "10:53 AM"}, "commit": {"id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "short_id": "028c02d9", "created_at": "2021-03-18T14:48:41.000+02:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef"], "title": "add fake CI config", "message": "add fake CI config\n", "author_name": "ykurochkin", "author_email": "zhenia.kurochkin@gmail.com", "authored_date": "2021-03-18T14:48:41.000+02:00", "committer_name": "ykurochkin", "committer_email": "zhenia.kurochkin@gmail.com", "committed_date": "2021-03-18T14:48:41.000+02:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/028c02d96f40afe9b4d1173c1d0f712dd6d07302"}, "pipeline": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108952832", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2223, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "tag_list": [], "user_id": 8375961, "pipeline_id": 272631271, "runner_id": null, "commit_id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "project_id": 25157276}, "emitted_at": 1686567192861} -{"stream": "merge_request_commits", "data": {"id": 92594931, "iid": 3, "project_id": 25157276, "title": "add fake CI config", "description": "", "state": "merged", "created_at": "2021-03-18T12:49:13.091Z", "updated_at": "2021-03-18T12:51:06.319Z", "merged_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merge_user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merged_at": "2021-03-18T12:51:06.470Z", "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/add-fake-CI-config", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "not_open", "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "merge_commit_sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:49:13.091Z", "reference": "!3", "references": {"short": "!3", "relative": "!3", "full": "new-group-airbute/new-ci-test-project!3"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/3", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": "1", "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "before_sha": "0000000000000000000000000000000000000000", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:48:50.166Z", "finished_at": "2021-03-18T12:49:38.084Z", "committed_at": null, "duration": 47, "queued_duration": null, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": true, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}}, "diff_refs": {"base_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "head_sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "start_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merge_error": null, "first_contribution": false, "user": {"can_merge": true}, "merge_request_iid": 3}, "emitted_at": 1686567221987} -{"stream": "merge_request_commits", "data": {"id": 92593913, "iid": 2, "project_id": 25157276, "title": "update readme.md", "description": "", "state": "opened", "created_at": "2021-03-18T12:42:30.200Z", "updated_at": "2021-03-18T12:42:30.200Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/test-branch", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:42:30.200Z", "reference": "!2", "references": {"short": "!2", "relative": "!2", "full": "new-group-airbute/new-ci-test-project!2"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/2", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": "1", "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": null, "diff_refs": {"base_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "head_sha": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "start_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merge_error": null, "first_contribution": false, "user": {"can_merge": true}, "merge_request_iid": 2}, "emitted_at": 1686567222383} -{"stream": "merge_request_commits", "data": {"id": 92111504, "iid": 1, "project_id": 25157276, "title": "Draft: Resolve \"Fake Issue 30\"", "description": "Closes #31", "state": "opened", "created_at": "2021-03-15T16:08:05.071Z", "updated_at": "2021-03-15T16:08:05.071Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "31-fake-issue-30", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [{"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}], "assignee": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": ["bug"], "draft": true, "work_in_progress": true, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "cannot_be_merged", "detailed_merge_status": "draft_status", "sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-15T16:08:05.071Z", "reference": "!1", "references": {"short": "!1", "relative": "!1", "full": "new-group-airbute/new-ci-test-project!1"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/1", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": true, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": null, "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": null, "diff_refs": {"base_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "head_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "start_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merge_error": null, "first_contribution": false, "user": {"can_merge": true}, "merge_request_iid": 1}, "emitted_at": 1686567223002} +{"stream": "releases", "data": {"name": "First release", "tag_name": "fake-tag-6", "description": "Test Release", "created_at": "2021-03-18T12:44:12.497Z", "released_at": "2021-03-18T12:44:12.497Z", "upcoming_release": false, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "milestones": [1943704], "commit_path": "/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef", "tag_path": "/new-group-airbute/new-ci-test-project/-/tags/fake-tag-6", "assets": {"count": 4, "sources": [{"format": "zip", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.zip"}, {"format": "tar.gz", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.gz"}, {"format": "tar.bz2", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar.bz2"}, {"format": "tar", "url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/archive/fake-tag-6/new-ci-test-project-fake-tag-6.tar"}], "links": []}, "evidences": [{"sha": "a616fdca9312ca5aa451bc1060ce91a672fd24cc0f4d", "filepath": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/evidences/855895.json", "collected_at": "2021-03-18T12:44:12.650Z"}], "_links": {"closed_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=closed", "closed_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=closed", "edit_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6/edit", "merged_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=merged", "opened_issues_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues?release_tag=fake-tag-6&scope=all&state=opened", "opened_merge_requests_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests?release_tag=fake-tag-6&scope=all&state=opened", "self": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/releases/fake-tag-6"}, "author_id": 8375961, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1696947713101} +{"stream": "jobs", "data": {"id": 1108959782, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.294Z", "started_at": "2021-03-18T12:51:07.646Z", "finished_at": "2021-03-18T12:51:51.309Z", "erased_at": null, "duration": 43.662407, "queued_duration": 1.180926, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "4:51 PM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959782", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2200, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "archived": false, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1704732685403} +{"stream": "jobs", "data": {"id": 1108959779, "status": "failed", "stage": "test", "name": "test-code-job1", "ref": "master", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:51:06.279Z", "started_at": "2021-03-18T12:51:07.943Z", "finished_at": "2021-03-18T12:51:50.943Z", "erased_at": null, "duration": 42.999853, "queued_duration": 1.349274, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "4:51 PM"}, "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "pipeline": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108959779", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2182, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "archived": false, "tag_list": [], "user_id": 8375961, "pipeline_id": 272632767, "runner_id": null, "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1704732685404} +{"stream": "jobs", "data": {"id": 1108952832, "status": "failed", "stage": "test", "name": "test-code-job2", "ref": "ykurochkin/add-fake-CI-config", "tag": false, "coverage": null, "allow_failure": false, "created_at": "2021-03-18T12:48:49.222Z", "started_at": "2021-03-18T12:48:50.732Z", "finished_at": "2021-03-18T12:49:37.961Z", "erased_at": null, "duration": 47.229034, "queued_duration": 1.422541, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "created_at": "2021-03-10T17:13:46.589Z", "bio": "", "location": "", "public_email": "", "skype": "", "linkedin": "", "twitter": "", "discord": "", "website_url": "", "organization": "", "job_title": "", "pronouns": "", "bot": false, "work_information": null, "followers": 0, "following": 0, "local_time": "4:51 PM"}, "commit": {"id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "short_id": "028c02d9", "created_at": "2021-03-18T14:48:41.000+02:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef"], "title": "add fake CI config", "message": "add fake CI config\n", "author_name": "ykurochkin", "author_email": "zhenia.kurochkin@gmail.com", "authored_date": "2021-03-18T14:48:41.000+02:00", "committer_name": "ykurochkin", "committer_email": "zhenia.kurochkin@gmail.com", "committed_date": "2021-03-18T14:48:41.000+02:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/028c02d96f40afe9b4d1173c1d0f712dd6d07302"}, "pipeline": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271"}, "failure_reason": "script_failure", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/jobs/1108952832", "project": {"ci_job_token_scope_enabled": false}, "artifacts": [{"file_type": "trace", "size": 2223, "filename": "job.log", "file_format": null}], "runner": null, "artifacts_expire_at": null, "archived": false, "tag_list": [], "user_id": 8375961, "pipeline_id": 272631271, "runner_id": null, "commit_id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "project_id": 25157276}, "emitted_at": 1704732685727} +{"stream": "merge_request_commits", "data": {"id": 92594931, "iid": 3, "project_id": 25157276, "title": "add fake CI config", "description": "", "state": "merged", "created_at": "2021-03-18T12:49:13.091Z", "updated_at": "2021-03-18T12:51:06.319Z", "merged_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merge_user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merged_at": "2021-03-18T12:51:06.470Z", "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/add-fake-CI-config", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "not_open", "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "merge_commit_sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:49:13.091Z", "reference": "!3", "references": {"short": "!3", "relative": "!3", "full": "new-group-airbute/new-ci-test-project!3"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/3", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": "1", "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "before_sha": "0000000000000000000000000000000000000000", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:48:50.166Z", "finished_at": "2021-03-18T12:49:38.084Z", "committed_at": null, "duration": 47, "queued_duration": null, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "Failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": true, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}}, "diff_refs": {"base_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "head_sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "start_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merge_error": null, "first_contribution": false, "user": {"can_merge": true}, "merge_request_iid": 3}, "emitted_at": 1696948297499} +{"stream": "merge_request_commits", "data": {"id": 92593913, "iid": 2, "project_id": 25157276, "title": "update readme.md", "description": "", "state": "opened", "created_at": "2021-03-18T12:42:30.200Z", "updated_at": "2021-03-18T12:42:30.200Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/test-branch", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:42:30.200Z", "reference": "!2", "references": {"short": "!2", "relative": "!2", "full": "new-group-airbute/new-ci-test-project!2"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/2", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": "1", "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": null, "diff_refs": {"base_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "head_sha": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "start_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merge_error": null, "first_contribution": false, "user": {"can_merge": true}, "merge_request_iid": 2}, "emitted_at": 1696948297896} +{"stream": "merge_request_commits", "data": {"id": 92111504, "iid": 1, "project_id": 25157276, "title": "Draft: Resolve \"Fake Issue 30\"", "description": "Closes #31", "state": "opened", "created_at": "2021-03-15T16:08:05.071Z", "updated_at": "2021-03-15T16:08:05.071Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "31-fake-issue-30", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [{"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}], "assignee": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": ["bug"], "draft": true, "work_in_progress": true, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "cannot_be_merged", "detailed_merge_status": "draft_status", "sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-15T16:08:05.071Z", "reference": "!1", "references": {"short": "!1", "relative": "!1", "full": "new-group-airbute/new-ci-test-project!1"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/1", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": true, "blocking_discussions_resolved": true, "approvals_before_merge": null, "subscribed": true, "changes_count": null, "latest_build_started_at": null, "latest_build_finished_at": null, "first_deployed_to_production_at": null, "pipeline": null, "head_pipeline": null, "diff_refs": {"base_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "head_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "start_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merge_error": null, "first_contribution": false, "user": {"can_merge": true}, "merge_request_iid": 1}, "emitted_at": 1696948298305} {"stream": "group_milestones", "data": {"id": 1943775, "iid": 21, "group_id": 11329647, "title": "Group Milestone 21", "description": null, "state": "active", "created_at": "2021-03-15T16:01:02.125Z", "updated_at": "2021-03-15T16:01:02.125Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/21"}, "emitted_at": 1686567198876} {"stream": "group_milestones", "data": {"id": 1943774, "iid": 20, "group_id": 11329647, "title": "Group Milestone 20", "description": null, "state": "active", "created_at": "2021-03-15T16:01:01.682Z", "updated_at": "2021-03-15T16:01:01.682Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/20"}, "emitted_at": 1686567198878} {"stream": "group_milestones", "data": {"id": 1943773, "iid": 19, "group_id": 11329647, "title": "Group Milestone 19", "description": null, "state": "active", "created_at": "2021-03-15T16:01:01.067Z", "updated_at": "2021-03-15T16:01:01.067Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/groups/new-group-airbute/-/milestones/19"}, "emitted_at": 1686567198878} -{"stream": "pipelines_extended", "data": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "before_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:51:07.816Z", "finished_at": "2021-03-18T12:51:52.000Z", "committed_at": null, "duration": 43, "queued_duration": 1, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1686567228326} -{"stream": "pipelines_extended", "data": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "before_sha": "0000000000000000000000000000000000000000", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:48:50.166Z", "finished_at": "2021-03-18T12:49:38.084Z", "committed_at": null, "duration": 47, "queued_duration": null, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1686567228728} -{"stream": "users", "data": {"id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/760fcac88680c724a6b19c6bfd5b6718?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin"}, "emitted_at": 1686567230821} -{"stream": "users", "data": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "emitted_at": 1686567230823} -{"stream": "group_labels", "data": {"id": 19117004, "name": "Label 1", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#008000", "subscribed": false, "group_id": 11329647}, "emitted_at": 1686567210102} -{"stream": "group_labels", "data": {"id": 19117017, "name": "Label 10", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#000080", "subscribed": false, "group_id": 11329647}, "emitted_at": 1686567210104} -{"stream": "group_labels", "data": {"id": 19117018, "name": "Label 11", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#808080", "subscribed": false, "group_id": 11329647}, "emitted_at": 1686567210104} -{"stream":"groups","data":{"id":11329647,"web_url":"https://gitlab.com/groups/new-group-airbute","name":"New Group Airbute","path":"new-group-airbute","description":"","visibility":"public","share_with_group_lock":false,"require_two_factor_authentication":false,"two_factor_grace_period":48,"project_creation_level":"developer","auto_devops_enabled":null,"subgroup_creation_level":"maintainer","emails_disabled":null,"mentions_disabled":null,"lfs_enabled":true,"default_branch_protection":2,"avatar_url":null,"request_access_enabled":true,"full_name":"New Group Airbute","full_path":"new-group-airbute","created_at":"2021-03-15T15:55:53.613Z","parent_id":null,"shared_runners_setting":"enabled","ldap_cn":null,"ldap_access":null,"wiki_access_level":"enabled","shared_with_groups":[],"runners_token":"GR1348941-PhosPap-Sf1UxL1g6m4","prevent_sharing_groups_outside_hierarchy":false,"shared_projects":[],"shared_runners_minutes_limit":null,"extra_shared_runners_minutes_limit":null,"prevent_forking_outside_group":null,"membership_lock":false,"projects":[{"id":25157276,"path_with_namespace":"new-group-airbute/new-ci-test-project"}]},"emitted_at":1688238188152} -{"stream":"groups","data":{"id":61014882,"web_url":"https://gitlab.com/groups/new-group-airbute/test-subgroup-airbyte/test-private-sg","name":"Test Private SG","path":"test-private-sg","description":"","visibility":"private","share_with_group_lock":false,"require_two_factor_authentication":false,"two_factor_grace_period":48,"project_creation_level":"developer","auto_devops_enabled":null,"subgroup_creation_level":"maintainer","emails_disabled":null,"mentions_disabled":null,"lfs_enabled":true,"default_branch_protection":2,"avatar_url":null,"request_access_enabled":true,"full_name":"New Group Airbute / Test Subgroup Airbyte / Test Private SG","full_path":"new-group-airbute/test-subgroup-airbyte/test-private-sg","created_at":"2022-12-02T08:46:22.648Z","parent_id":61014863,"shared_runners_setting":"enabled","ldap_cn":null,"ldap_access":null,"wiki_access_level":"enabled","shared_with_groups":[],"runners_token":"GR1348941bjUaJQy2zzar-JmNBjfq","shared_projects":[],"shared_runners_minutes_limit":null,"extra_shared_runners_minutes_limit":null,"prevent_forking_outside_group":null,"membership_lock":false,"projects":[]},"emitted_at":1688238188456} -{"stream":"groups","data":{"id":61015181,"web_url":"https://gitlab.com/groups/new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1","name":"Test Private SubSubG 1","path":"test-private-subsubg-1","description":"","visibility":"private","share_with_group_lock":false,"require_two_factor_authentication":false,"two_factor_grace_period":48,"project_creation_level":"developer","auto_devops_enabled":null,"subgroup_creation_level":"maintainer","emails_disabled":null,"mentions_disabled":null,"lfs_enabled":true,"default_branch_protection":2,"avatar_url":null,"request_access_enabled":true,"full_name":"New Group Airbute / Test Public SG / Test SG Public 2 / Test Private SubSubG 1","full_path":"new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1","created_at":"2022-12-02T08:54:42.252Z","parent_id":61014943,"shared_runners_setting":"enabled","ldap_cn":null,"ldap_access":null,"wiki_access_level":"enabled","shared_with_groups":[],"runners_token":"GR1348941x8xQf6K-UvnnyJ-bcut4","shared_projects":[],"shared_runners_minutes_limit":null,"extra_shared_runners_minutes_limit":null,"prevent_forking_outside_group":null,"membership_lock":false,"projects":[{"id":41551658,"path_with_namespace":"new-group-airbute/test-public-sg/test-sg-public-2/test-private-subsubg-1/test_project_in_nested_subgroup"}]},"emitted_at":1688238188865} -{"stream": "group_members", "data": {"access_level": 50, "created_at": "2021-03-15T15:55:53.658Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1686567203646} -{"stream": "group_members", "data": {"access_level": 30, "created_at": "2021-03-15T15:55:53.998Z", "created_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "expires_at": null, "id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/760fcac88680c724a6b19c6bfd5b6718?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1686567203646} -{"stream": "group_members", "data": {"access_level": 50, "created_at": "2022-12-02T08:46:22.834Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 61014882}, "emitted_at": 1686567204049} -{"stream": "branches", "data": {"name": "31-fake-issue-30", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/31-fake-issue-30", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686567183576} -{"stream": "branches", "data": {"name": "master", "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "merged": false, "protected": true, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": true, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/master", "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1686567183576} -{"stream": "branches", "data": {"name": "new-test-branch", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/new-test-branch", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686567183577} -{"stream": "commits", "data": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98", "stats": {"additions": 14, "deletions": 0, "total": 14}, "project_id": 25157276}, "emitted_at": 1686567184540} -{"stream": "commits", "data": {"id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "short_id": "028c02d9", "created_at": "2021-03-18T14:48:41.000+02:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef"], "title": "add fake CI config", "message": "add fake CI config\n", "author_name": "ykurochkin", "author_email": "zhenia.kurochkin@gmail.com", "authored_date": "2021-03-18T14:48:41.000+02:00", "committer_name": "ykurochkin", "committer_email": "zhenia.kurochkin@gmail.com", "committed_date": "2021-03-18T14:48:41.000+02:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/028c02d96f40afe9b4d1173c1d0f712dd6d07302", "stats": {"additions": 14, "deletions": 0, "total": 14}, "project_id": 25157276}, "emitted_at": 1686567184541} -{"stream": "commits", "data": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef", "stats": {"additions": 2, "deletions": 0, "total": 2}, "project_id": 25157276}, "emitted_at": 1686567184541} +{"stream": "pipelines_extended", "data": {"id": 272632767, "iid": 2, "project_id": 25157276, "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "ref": "master", "status": "failed", "source": "push", "created_at": "2021-03-18T12:51:06.262Z", "updated_at": "2021-03-18T12:51:52.007Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "before_sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:51:07.816Z", "finished_at": "2021-03-18T12:51:52.000Z", "committed_at": null, "duration": 43, "queued_duration": 1, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "Failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272632767", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1696948628546} +{"stream": "pipelines_extended", "data": {"id": 272631271, "iid": 1, "project_id": 25157276, "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "ref": "ykurochkin/add-fake-CI-config", "status": "failed", "source": "push", "created_at": "2021-03-18T12:48:49.174Z", "updated_at": "2021-03-18T12:49:38.092Z", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "before_sha": "0000000000000000000000000000000000000000", "tag": false, "yaml_errors": null, "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "started_at": "2021-03-18T12:48:50.166Z", "finished_at": "2021-03-18T12:49:38.084Z", "committed_at": null, "duration": 47, "queued_duration": null, "coverage": null, "detailed_status": {"icon": "status_failed", "text": "Failed", "label": "failed", "group": "failed", "tooltip": "failed", "has_details": false, "details_path": "/new-group-airbute/new-ci-test-project/-/pipelines/272631271", "illustration": null, "favicon": "/assets/ci_favicons/favicon_status_failed-41304d7f7e3828808b0c26771f0309e55296819a9beea3ea9fbf6689d9857c12.png"}, "name": null}, "emitted_at": 1696948628851} +{"stream": "users", "data": {"id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/760fcac88680c724a6b19c6bfd5b6718?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin"}, "emitted_at": 1696948873593} +{"stream": "users", "data": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "emitted_at": 1696948873594} +{"stream": "groups", "data": {"id": 11266951, "web_url": "https://gitlab.com/groups/airbyte.io", "name": "airbyte.io", "path": "airbyte.io", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": false, "emails_enabled": true, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "default_branch_protection_defaults": {"allowed_to_push": [{"access_level": 30}], "allow_force_push": true, "allowed_to_merge": [{"access_level": 30}]}, "avatar_url": null, "request_access_enabled": true, "full_name": "airbyte.io", "full_path": "airbyte.io", "created_at": "2021-03-10T17:16:37.549Z", "parent_id": null, "organization_id": 1, "shared_runners_setting": "enabled", "ldap_cn": null, "ldap_access": null, "marked_for_deletion_on": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941bzmDjXx-Cz48snUcJfK8", "prevent_sharing_groups_outside_hierarchy": false, "shared_projects": [], "shared_runners_minutes_limit": 10000, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": false, "service_access_tokens_expiration_enforced": true, "membership_lock": false, "ip_restriction_ranges": null, "projects": [{"id": 25156633, "path_with_namespace": "airbyte.io/ci-test-project"}, {"id": 25032440, "path_with_namespace": "airbyte.io/learn-gitlab"}, {"id": 25032439, "path_with_namespace": "airbyte.io/documentation"}]}, "emitted_at": 1704733464696} +{"stream": "groups", "data": {"id": 11329647, "web_url": "https://gitlab.com/groups/new-group-airbute", "name": "New Group Airbute", "path": "new-group-airbute", "description": "", "visibility": "public", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": false, "emails_enabled": true, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "default_branch_protection_defaults": {"allowed_to_push": [{"access_level": 30}], "allow_force_push": true, "allowed_to_merge": [{"access_level": 30}]}, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute", "full_path": "new-group-airbute", "created_at": "2021-03-15T15:55:53.613Z", "parent_id": null, "organization_id": 1, "shared_runners_setting": "enabled", "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941-PhosPap-Sf1UxL1g6m4", "prevent_sharing_groups_outside_hierarchy": false, "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": [{"id": 25157276, "path_with_namespace": "new-group-airbute/new-ci-test-project"}]}, "emitted_at": 1704733465104} +{"stream": "groups", "data": {"id": 61014882, "web_url": "https://gitlab.com/groups/new-group-airbute/test-subgroup-airbyte/test-private-sg", "name": "Test Private SG", "path": "test-private-sg", "description": "", "visibility": "private", "share_with_group_lock": false, "require_two_factor_authentication": false, "two_factor_grace_period": 48, "project_creation_level": "developer", "auto_devops_enabled": null, "subgroup_creation_level": "maintainer", "emails_disabled": false, "emails_enabled": true, "mentions_disabled": null, "lfs_enabled": true, "default_branch_protection": 2, "default_branch_protection_defaults": {"allowed_to_push": [{"access_level": 30}], "allow_force_push": true, "allowed_to_merge": [{"access_level": 30}]}, "avatar_url": null, "request_access_enabled": true, "full_name": "New Group Airbute / Test Subgroup Airbyte / Test Private SG", "full_path": "new-group-airbute/test-subgroup-airbyte/test-private-sg", "created_at": "2022-12-02T08:46:22.648Z", "parent_id": 61014863, "organization_id": 1, "shared_runners_setting": "enabled", "ldap_cn": null, "ldap_access": null, "wiki_access_level": "enabled", "shared_with_groups": [], "runners_token": "GR1348941bjUaJQy2zzar-JmNBjfq", "shared_projects": [], "shared_runners_minutes_limit": null, "extra_shared_runners_minutes_limit": null, "prevent_forking_outside_group": null, "membership_lock": false, "projects": []}, "emitted_at": 1704733465409} +{"stream": "group_labels", "data": {"id": 19117004, "name": "Label 1", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#008000", "subscribed": false, "group_id": 11329647}, "emitted_at": 1696949435261} +{"stream": "group_labels", "data": {"id": 19117017, "name": "Label 10", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#000080", "subscribed": false, "group_id": 11329647}, "emitted_at": 1696949435263} +{"stream": "group_labels", "data": {"id": 19117018, "name": "Label 11", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#808080", "subscribed": false, "group_id": 11329647}, "emitted_at": 1696949435264} +{"stream": "group_members", "data": {"access_level": 50, "created_at": "2021-03-15T15:55:53.658Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1696949993328} +{"stream": "group_members", "data": {"access_level": 30, "created_at": "2021-03-15T15:55:53.998Z", "created_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "expires_at": null, "id": 7904355, "username": "y.kurochkin", "name": "Yevhenii Kurochkin", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/760fcac88680c724a6b19c6bfd5b6718?s=80&d=identicon", "web_url": "https://gitlab.com/y.kurochkin", "membership_state": "active", "group_id": 11329647}, "emitted_at": 1696949993329} +{"stream": "group_members", "data": {"access_level": 50, "created_at": "2022-12-02T08:46:22.834Z", "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "group_id": 61014882}, "emitted_at": 1696949993941} +{"stream": "branches", "data": {"name": "31-fake-issue-30", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/31-fake-issue-30", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1703257027266} +{"stream": "branches", "data": {"name": "master", "commit": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98"}, "merged": false, "protected": true, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": true, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/master", "commit_id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "project_id": 25157276}, "emitted_at": 1703257027267} +{"stream": "branches", "data": {"name": "new-test-branch", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "merged": true, "protected": false, "developers_can_push": false, "developers_can_merge": false, "can_push": true, "default": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/tree/new-test-branch", "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1703257027267} +{"stream": "commits", "data": {"id": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "short_id": "6ad3dd49", "created_at": "2021-03-18T12:51:05.000+00:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef", "028c02d96f40afe9b4d1173c1d0f712dd6d07302"], "title": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'", "message": "Merge branch 'ykurochkin/add-fake-CI-config' into 'master'\n\nadd fake CI config\n\nSee merge request new-group-airbute/new-ci-test-project!3", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-18T12:51:05.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-18T12:51:05.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/6ad3dd49539391774db738c9e7b7d69f2d872c98", "stats": {"additions": 14, "deletions": 0, "total": 14}, "project_id": 25157276}, "emitted_at": 1703257635545} +{"stream": "commits", "data": {"id": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "short_id": "028c02d9", "created_at": "2021-03-18T14:48:41.000+02:00", "parent_ids": ["2831d897ba0214f8d3168647e8ad4232b83987ef"], "title": "add fake CI config", "message": "add fake CI config\n", "author_name": "ykurochkin", "author_email": "zhenia.kurochkin@gmail.com", "authored_date": "2021-03-18T14:48:41.000+02:00", "committer_name": "ykurochkin", "committer_email": "zhenia.kurochkin@gmail.com", "committed_date": "2021-03-18T14:48:41.000+02:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/028c02d96f40afe9b4d1173c1d0f712dd6d07302", "stats": {"additions": 14, "deletions": 0, "total": 14}, "project_id": 25157276}, "emitted_at": 1703257635547} +{"stream": "commits", "data": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef", "stats": {"additions": 2, "deletions": 0, "total": 2}, "project_id": 25157276}, "emitted_at": 1703257635548} {"stream": "group_issue_boards", "data": {"id": 5099065, "name": "Development", "hide_backlog_list": false, "hide_closed_list": false, "project": null, "lists": [], "group": {"id": 11329647, "web_url": "https://gitlab.com/groups/new-group-airbute", "name": "New Group Airbute"}, "group_id": 11329647}, "emitted_at": 1686567186609} -{"stream": "projects", "data": {"id": 25157276, "description": "", "name": "New CI Test Project ", "name_with_namespace": "New Group Airbute / New CI Test Project ", "path": "new-ci-test-project", "path_with_namespace": "new-group-airbute/new-ci-test-project", "created_at": "2021-03-15T15:08:36.498Z", "default_branch": "master", "tag_list": [], "topics": [], "ssh_url_to_repo": "git@gitlab.com:new-group-airbute/new-ci-test-project.git", "http_url_to_repo": "https://gitlab.com/new-group-airbute/new-ci-test-project.git", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project", "readme_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/blob/master/README.md", "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2022-12-13T09:39:47.235Z", "namespace": {"id": 11329647, "name": "New Group Airbute", "path": "new-group-airbute", "kind": "group", "full_path": "new-group-airbute", "parent_id": null, "avatar_url": null, "web_url": "https://gitlab.com/groups/new-group-airbute"}, "container_registry_image_prefix": "registry.gitlab.com/new-group-airbute/new-ci-test-project", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276", "issues": "https://gitlab.com/api/v4/projects/25157276/issues", "merge_requests": "https://gitlab.com/api/v4/projects/25157276/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/25157276/repository/branches", "labels": "https://gitlab.com/api/v4/projects/25157276/labels", "events": "https://gitlab.com/api/v4/projects/25157276/events", "members": "https://gitlab.com/api/v4/projects/25157276/members", "cluster_agents": "https://gitlab.com/api/v4/projects/25157276/cluster_agents"}, "packages_enabled": true, "empty_repo": false, "archived": false, "visibility": "private", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2021-03-16T15:08:36.518Z"}, "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "jobs_enabled": true, "snippets_enabled": true, "container_registry_enabled": true, "service_desk_enabled": true, "service_desk_address": "contact-project+new-group-airbute-new-ci-test-project-25157276-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "private", "repository_access_level": "private", "merge_requests_access_level": "private", "forking_access_level": "enabled", "wiki_access_level": "enabled", "builds_access_level": "private", "snippets_access_level": "enabled", "pages_access_level": "private", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "enabled", "feature_flags_access_level": "enabled", "infrastructure_access_level": "enabled", "monitor_access_level": "enabled", "emails_disabled": false, "emails_enabled": true, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": null, "import_status": "none", "import_error": null, "open_issues_count": 31, "description_html": "", "updated_at": "2023-05-23T12:12:18.623Z", "ci_default_git_depth": 50, "ci_forward_deployment_enabled": true, "ci_forward_deployment_rollback_allowed": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941eMJgWDU69xyyshaNsaTZ", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": "", "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": true, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 3, "storage_size": 291925, "repository_size": 283115, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 8810, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "autoclose_referenced_issues": true, "external_authorization_classification_label": "", "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "permissions": {"project_access": {"access_level": 40, "notification_level": 3}, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1690448724369} -{"stream": "tags", "data": {"name": "fake-tag-1", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686567225240} -{"stream": "tags", "data": {"name": "fake-tag-10", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686567225242} -{"stream": "tags", "data": {"name": "fake-tag-11", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1686567225243} -{"stream": "merge_requests", "data": {"id": 92594931, "iid": 3, "project_id": 25157276, "title": "add fake CI config", "description": "", "state": "merged", "created_at": "2021-03-18T12:49:13.091Z", "updated_at": "2021-03-18T12:51:06.319Z", "merged_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merge_user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merged_at": "2021-03-18T12:51:06.470Z", "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/add-fake-CI-config", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "not_open", "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "merge_commit_sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:49:13.091Z", "reference": "!3", "references": {"short": "!3", "relative": "!3", "full": "new-group-airbute/new-ci-test-project!3"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/3", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": 8375961}, "emitted_at": 1686567219461} -{"stream": "merge_requests", "data": {"id": 92593913, "iid": 2, "project_id": 25157276, "title": "update readme.md", "description": "", "state": "opened", "created_at": "2021-03-18T12:42:30.200Z", "updated_at": "2021-03-18T12:42:30.200Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/test-branch", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:42:30.200Z", "reference": "!2", "references": {"short": "!2", "relative": "!2", "full": "new-group-airbute/new-ci-test-project!2"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/2", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1686567219464} -{"stream": "merge_requests", "data": {"id": 92111504, "iid": 1, "project_id": 25157276, "title": "Draft: Resolve \"Fake Issue 30\"", "description": "Closes #31", "state": "opened", "created_at": "2021-03-15T16:08:05.071Z", "updated_at": "2021-03-15T16:08:05.071Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "31-fake-issue-30", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [8375961], "assignee": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": ["bug"], "draft": true, "work_in_progress": true, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "cannot_be_merged", "detailed_merge_status": "draft_status", "sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-15T16:08:05.071Z", "reference": "!1", "references": {"short": "!1", "relative": "!1", "full": "new-group-airbute/new-ci-test-project!1"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/1", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": true, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": 8375961, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1686567219466} -{"stream": "issues", "data": {"id": 80943819, "iid": 32, "project_id": 25157276, "title": "Fake Issue 31", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:42.206Z", "updated_at": "2021-03-15T15:22:42.206Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/32", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/32", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/32/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/32/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#32", "relative": "#32", "full": "new-group-airbute/new-ci-test-project#32"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1686567189959} -{"stream": "issues", "data": {"id": 80943818, "iid": 31, "project_id": 25157276, "title": "Fake Issue 30", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:41.337Z", "updated_at": "2021-03-15T16:08:06.041Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 1, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/31", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/31", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/31/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/31/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#31", "relative": "#31", "full": "new-group-airbute/new-ci-test-project#31"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1686567189960} -{"stream": "issues", "data": {"id": 80943817, "iid": 30, "project_id": 25157276, "title": "Fake Issue 29", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:40.529Z", "updated_at": "2021-03-15T15:22:40.529Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/30", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/30", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/30/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/30/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#30", "relative": "#30", "full": "new-group-airbute/new-ci-test-project#30"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1686567189960} -{"stream": "project_members", "data": {"access_level": 40, "created_at": "2021-03-15T15:08:36.746Z", "created_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "project_id": 25157276}, "emitted_at": 1686567202944} +{"stream": "projects", "data": {"id": 25157276, "description": "", "name": "New CI Test Project ", "name_with_namespace": "New Group Airbute / New CI Test Project ", "path": "new-ci-test-project", "path_with_namespace": "new-group-airbute/new-ci-test-project", "created_at": "2021-03-15T15:08:36.498Z", "default_branch": "master", "tag_list": [], "topics": [], "ssh_url_to_repo": "git@gitlab.com:new-group-airbute/new-ci-test-project.git", "http_url_to_repo": "https://gitlab.com/new-group-airbute/new-ci-test-project.git", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project", "readme_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/blob/master/README.md", "forks_count": 0, "avatar_url": null, "star_count": 0, "last_activity_at": "2022-12-13T09:39:47.235Z", "namespace": {"id": 11329647, "name": "New Group Airbute", "path": "new-group-airbute", "kind": "group", "full_path": "new-group-airbute", "parent_id": null, "avatar_url": null, "web_url": "https://gitlab.com/groups/new-group-airbute"}, "container_registry_image_prefix": "registry.gitlab.com/new-group-airbute/new-ci-test-project", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276", "issues": "https://gitlab.com/api/v4/projects/25157276/issues", "merge_requests": "https://gitlab.com/api/v4/projects/25157276/merge_requests", "repo_branches": "https://gitlab.com/api/v4/projects/25157276/repository/branches", "labels": "https://gitlab.com/api/v4/projects/25157276/labels", "events": "https://gitlab.com/api/v4/projects/25157276/events", "members": "https://gitlab.com/api/v4/projects/25157276/members", "cluster_agents": "https://gitlab.com/api/v4/projects/25157276/cluster_agents"}, "code_suggestions": true, "packages_enabled": true, "empty_repo": false, "archived": false, "visibility": "private", "resolve_outdated_diff_discussions": false, "container_expiration_policy": {"cadence": "1d", "enabled": false, "keep_n": 10, "older_than": "90d", "name_regex": ".*", "name_regex_keep": null, "next_run_at": "2021-03-16T15:08:36.518Z"}, "issues_enabled": true, "merge_requests_enabled": true, "wiki_enabled": true, "jobs_enabled": true, "snippets_enabled": true, "container_registry_enabled": true, "service_desk_enabled": true, "service_desk_address": "contact-project+new-group-airbute-new-ci-test-project-25157276-issue-@incoming.gitlab.com", "can_create_merge_request_in": true, "issues_access_level": "private", "repository_access_level": "private", "merge_requests_access_level": "private", "forking_access_level": "enabled", "wiki_access_level": "enabled", "builds_access_level": "private", "snippets_access_level": "enabled", "pages_access_level": "private", "analytics_access_level": "enabled", "container_registry_access_level": "enabled", "security_and_compliance_access_level": "private", "releases_access_level": "enabled", "environments_access_level": "enabled", "feature_flags_access_level": "enabled", "infrastructure_access_level": "enabled", "monitor_access_level": "enabled", "model_experiments_access_level": "enabled", "model_registry_access_level": "enabled", "emails_disabled": false, "emails_enabled": true, "shared_runners_enabled": true, "lfs_enabled": true, "creator_id": 8375961, "import_url": null, "import_type": null, "import_status": "none", "import_error": null, "open_issues_count": 31, "description_html": "", "updated_at": "2023-05-23T12:12:18.623Z", "ci_default_git_depth": 50, "ci_forward_deployment_enabled": true, "ci_forward_deployment_rollback_allowed": true, "ci_job_token_scope_enabled": false, "ci_separated_caches": true, "ci_allow_fork_pipelines_to_run_in_parent_project": true, "build_git_strategy": "fetch", "keep_latest_artifact": true, "restrict_user_defined_variables": false, "runners_token": "GR1348941eMJgWDU69xyyshaNsaTZ", "runner_token_expiration_interval": null, "group_runners_enabled": true, "auto_cancel_pending_pipelines": "enabled", "build_timeout": 3600, "auto_devops_enabled": false, "auto_devops_deploy_strategy": "continuous", "ci_config_path": "", "public_jobs": true, "shared_with_groups": [], "only_allow_merge_if_pipeline_succeeds": false, "allow_merge_on_skipped_pipeline": null, "request_access_enabled": true, "only_allow_merge_if_all_discussions_are_resolved": false, "remove_source_branch_after_merge": true, "printing_merge_request_link_enabled": true, "merge_method": "merge", "squash_option": "default_off", "enforce_auth_checks_on_uploads": true, "suggestion_commit_message": null, "merge_commit_template": null, "squash_commit_template": null, "issue_branch_template": null, "statistics": {"commit_count": 3, "storage_size": 9061, "repository_size": 251, "wiki_size": 0, "lfs_objects_size": 0, "job_artifacts_size": 8810, "pipeline_artifacts_size": 0, "packages_size": 0, "snippets_size": 0, "uploads_size": 0}, "autoclose_referenced_issues": true, "external_authorization_classification_label": "", "requirements_enabled": false, "requirements_access_level": "enabled", "security_and_compliance_enabled": true, "compliance_frameworks": [], "permissions": {"project_access": {"access_level": 40, "notification_level": 3}, "group_access": {"access_level": 50, "notification_level": 3}}}, "emitted_at": 1703258006161} +{"stream": "tags", "data": {"name": "fake-tag-1", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1703258326525} +{"stream": "tags", "data": {"name": "fake-tag-10", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1703258326527} +{"stream": "tags", "data": {"name": "fake-tag-11", "message": "", "target": "2831d897ba0214f8d3168647e8ad4232b83987ef", "commit": {"id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "short_id": "2831d897", "created_at": "2021-03-15T15:08:36.000+00:00", "parent_ids": [], "title": "Initial commit", "message": "Initial commit", "author_name": "Alexander Arhipenko", "author_email": "integration-test@airbyte.io", "authored_date": "2021-03-15T15:08:36.000+00:00", "committer_name": "Alexander Arhipenko", "committer_email": "integration-test@airbyte.io", "committed_date": "2021-03-15T15:08:36.000+00:00", "trailers": {}, "extended_trailers": {}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/commit/2831d897ba0214f8d3168647e8ad4232b83987ef"}, "release": null, "protected": false, "commit_id": "2831d897ba0214f8d3168647e8ad4232b83987ef", "project_id": 25157276}, "emitted_at": 1703258326528} +{"stream": "merge_requests", "data": {"id": 92594931, "iid": 3, "project_id": 25157276, "title": "add fake CI config", "description": "", "state": "merged", "created_at": "2021-03-18T12:49:13.091Z", "updated_at": "2021-03-18T12:51:06.319Z", "merged_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merge_user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "merged_at": "2021-03-18T12:51:06.470Z", "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/add-fake-CI-config", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "not_open", "sha": "028c02d96f40afe9b4d1173c1d0f712dd6d07302", "merge_commit_sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:49:13.091Z", "reference": "!3", "references": {"short": "!3", "relative": "!3", "full": "new-group-airbute/new-ci-test-project!3"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/3", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": 8375961}, "emitted_at": 1696950689861} +{"stream": "merge_requests", "data": {"id": 92593913, "iid": 2, "project_id": 25157276, "title": "update readme.md", "description": "", "state": "opened", "created_at": "2021-03-18T12:42:30.200Z", "updated_at": "2021-03-18T12:42:30.200Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "ykurochkin/test-branch", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [], "assignee": null, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": [], "draft": false, "work_in_progress": false, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "can_be_merged", "detailed_merge_status": "mergeable", "sha": "9b0c5cf345f0ca1a3fb3ae253e74e0616abf8129", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-18T12:42:30.200Z", "reference": "!2", "references": {"short": "!2", "relative": "!2", "full": "new-group-airbute/new-ci-test-project!2"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/2", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": false, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1696950689864} +{"stream": "merge_requests", "data": {"id": 92111504, "iid": 1, "project_id": 25157276, "title": "Draft: Resolve \"Fake Issue 30\"", "description": "Closes #31", "state": "opened", "created_at": "2021-03-15T16:08:05.071Z", "updated_at": "2021-03-15T16:08:05.071Z", "merged_by": null, "merge_user": null, "merged_at": null, "closed_by": null, "closed_at": null, "target_branch": "master", "source_branch": "31-fake-issue-30", "user_notes_count": 0, "upvotes": 0, "downvotes": 0, "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "assignees": [8375961], "assignee": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "reviewers": [], "source_project_id": 25157276, "target_project_id": 25157276, "labels": ["bug"], "draft": true, "work_in_progress": true, "milestone": null, "merge_when_pipeline_succeeds": false, "merge_status": "cannot_be_merged", "detailed_merge_status": "draft_status", "sha": "2831d897ba0214f8d3168647e8ad4232b83987ef", "merge_commit_sha": null, "squash_commit_sha": null, "discussion_locked": null, "should_remove_source_branch": null, "force_remove_source_branch": true, "prepared_at": "2021-03-15T16:08:05.071Z", "reference": "!1", "references": {"short": "!1", "relative": "!1", "full": "new-group-airbute/new-ci-test-project!1"}, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/merge_requests/1", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "squash": false, "squash_on_merge": false, "task_completion_status": {"count": 0, "completed_count": 0}, "has_conflicts": true, "blocking_discussions_resolved": true, "approvals_before_merge": null, "author_id": 8375961, "assignee_id": 8375961, "closed_by_id": null, "milestone_id": null, "merged_by_id": null}, "emitted_at": 1696950689866} +{"stream": "issues", "data": {"id": 80943819, "iid": 32, "project_id": 25157276, "title": "Fake Issue 31", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:42.206Z", "updated_at": "2021-03-15T15:22:42.206Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/32", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/32", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/32/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/32/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#32", "relative": "#32", "full": "new-group-airbute/new-ci-test-project#32"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1696950969206} +{"stream": "issues", "data": {"id": 80943818, "iid": 31, "project_id": 25157276, "title": "Fake Issue 30", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:41.337Z", "updated_at": "2021-03-15T16:08:06.041Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 1, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/31", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/31", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/31/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/31/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#31", "relative": "#31", "full": "new-group-airbute/new-ci-test-project#31"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1696950969209} +{"stream": "issues", "data": {"id": 80943817, "iid": 30, "project_id": 25157276, "title": "Fake Issue 29", "description": null, "state": "opened", "created_at": "2021-03-15T15:22:40.529Z", "updated_at": "2021-03-15T15:22:40.529Z", "closed_at": null, "closed_by": null, "labels": ["bug"], "milestone": null, "assignees": [], "author": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "type": "ISSUE", "assignee": null, "user_notes_count": 0, "merge_requests_count": 0, "upvotes": 0, "downvotes": 0, "due_date": null, "confidential": false, "discussion_locked": null, "issue_type": "issue", "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/issues/30", "time_stats": {"time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, "human_total_time_spent": null}, "task_completion_status": {"count": 0, "completed_count": 0}, "blocking_issues_count": 0, "has_tasks": true, "task_status": "", "_links": {"self": "https://gitlab.com/api/v4/projects/25157276/issues/30", "notes": "https://gitlab.com/api/v4/projects/25157276/issues/30/notes", "award_emoji": "https://gitlab.com/api/v4/projects/25157276/issues/30/award_emoji", "project": "https://gitlab.com/api/v4/projects/25157276", "closed_as_duplicate_of": null}, "references": {"short": "#30", "relative": "#30", "full": "new-group-airbute/new-ci-test-project#30"}, "severity": "UNKNOWN", "moved_to_id": null, "service_desk_reply_to": null, "author_id": 8375961, "assignee_id": null, "closed_by_id": null, "milestone_id": null}, "emitted_at": 1696950969210} +{"stream": "project_members", "data": {"access_level": 40, "created_at": "2021-03-15T15:08:36.746Z", "created_by": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "expires_at": null, "id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte", "membership_state": "active", "project_id": 25157276}, "emitted_at": 1696951717050} {"stream": "project_labels", "data": {"id": 19116944, "name": "Label 1", "description": null, "description_html": "", "text_color": "#1F1E24", "color": "#ffff00", "subscribed": false, "priority": null, "is_project_label": true, "project_id": 25157276}, "emitted_at": 1686567207747} {"stream": "project_labels", "data": {"id": 19117004, "name": "Label 1", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#008000", "subscribed": false, "priority": null, "is_project_label": false, "project_id": 25157276}, "emitted_at": 1686567207748} {"stream": "project_labels", "data": {"id": 19116954, "name": "Label 10", "description": null, "description_html": "", "text_color": "#FFFFFF", "color": "#ff00ff", "subscribed": false, "priority": null, "is_project_label": true, "project_id": 25157276}, "emitted_at": 1686567207748} {"stream": "project_milestones", "data": {"id": 1943705, "iid": 51, "project_id": 25157276, "title": "Project Milestone 51", "description": null, "state": "active", "created_at": "2021-03-15T15:33:16.915Z", "updated_at": "2021-03-15T15:33:16.915Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/milestones/51"}, "emitted_at": 1686567197935} {"stream": "project_milestones", "data": {"id": 1943704, "iid": 50, "project_id": 25157276, "title": "Project Milestone 50", "description": null, "state": "active", "created_at": "2021-03-15T15:33:16.329Z", "updated_at": "2021-03-15T15:33:16.329Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/milestones/50"}, "emitted_at": 1686567197937} {"stream": "project_milestones", "data": {"id": 1943703, "iid": 49, "project_id": 25157276, "title": "Project Milestone 49", "description": null, "state": "active", "created_at": "2021-03-15T15:33:15.960Z", "updated_at": "2021-03-15T15:33:15.960Z", "due_date": null, "start_date": null, "expired": false, "web_url": "https://gitlab.com/new-group-airbute/new-ci-test-project/-/milestones/49"}, "emitted_at": 1686567197937} +{"stream": "deployments", "data": {"id": 568087366, "iid": 1, "ref": "master", "sha": "6ad3dd49539391774db738c9e7b7d69f2d872c98", "created_at": "2023-10-10T09:56:02.273Z", "updated_at": "2023-10-10T09:56:02.273Z", "user": {"id": 8375961, "username": "airbyte", "name": "Airbyte Team", "state": "active", "locked": false, "avatar_url": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?s=80&d=identicon", "web_url": "https://gitlab.com/airbyte"}, "environment": {"id": 17305239, "name": "dev", "slug": "dev", "external_url": null, "created_at": "2023-10-10T09:56:02.188Z", "updated_at": "2023-10-10T09:56:02.188Z"}, "deployable": null, "status": "failed", "user_id": 8375961, "environment_id": 17305239, "user_username": "airbyte", "user_full_name": "Airbyte Team", "environment_name": "dev", "project_id": 25157276}, "emitted_at": 1696931771902} diff --git a/airbyte-integrations/connectors/source-gitlab/main.py b/airbyte-integrations/connectors/source-gitlab/main.py index 6a0fe4988cc0..1c322c2f2c48 100644 --- a/airbyte-integrations/connectors/source-gitlab/main.py +++ b/airbyte-integrations/connectors/source-gitlab/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_gitlab import SourceGitlab +from source_gitlab.run import run if __name__ == "__main__": - source = SourceGitlab() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-gitlab/metadata.yaml b/airbyte-integrations/connectors/source-gitlab/metadata.yaml index 10304aeaebbc..381278747dee 100644 --- a/airbyte-integrations/connectors/source-gitlab/metadata.yaml +++ b/airbyte-integrations/connectors/source-gitlab/metadata.yaml @@ -1,12 +1,18 @@ data: + ab_internal: + ql: 400 + sl: 200 allowedHosts: hosts: - - "*" + - ${api_url} + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: api connectorType: source definitionId: 5e6175e5-68e1-4c17-bff9-56103bbb0d80 - dockerImageTag: 1.6.0 + dockerImageTag: 2.1.1 dockerRepository: airbyte/source-gitlab + documentationUrl: https://docs.airbyte.com/integrations/sources/gitlab githubIssueLabel: source-gitlab icon: gitlab.svg license: MIT @@ -17,11 +23,22 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/gitlab + releases: + breakingChanges: + 2.0.0: + message: + In this release, several streams were updated to date-time field format, as declared in the Gitlab API. + These changes impact pipeline.created_at and pipeline.updated_at fields for stream Deployments and expires_at field for stream Group Members and stream Project Members. + Users will need to refresh the source schema and reset affected streams after upgrading. + upgradeDeadline: "2023-11-09" + suggestedStreams: + streams: + - merge_requests + - users + - issues + - projects + - commits + supportLevel: certified tags: - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-gitlab/setup.py b/airbyte-integrations/connectors/source-gitlab/setup.py index 682fadb8af03..2d16bcd7d058 100644 --- a/airbyte-integrations/connectors/source-gitlab/setup.py +++ b/airbyte-integrations/connectors/source-gitlab/setup.py @@ -10,6 +10,11 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "requests_mock", "pytest-mock"] setup( + entry_points={ + "console_scripts": [ + "source-gitlab=source_gitlab.run:run", + ], + }, name="source_gitlab", description="Source implementation for Gitlab.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/config_migrations.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/config_migrations.py new file mode 100644 index 000000000000..0f963256cbcb --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/config_migrations.py @@ -0,0 +1,106 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import abc +import logging +from abc import ABC +from typing import Any, List, Mapping + +from airbyte_cdk.config_observation import create_connector_config_control_message +from airbyte_cdk.entrypoint import AirbyteEntrypoint +from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + +from .source import SourceGitlab + +logger = logging.getLogger("airbyte_logger") + + +class MigrateStringToArray(ABC): + """ + This class stands for migrating the config at runtime, + while providing the backward compatibility when falling back to the previous source version. + + Specifically, starting from `1.7.1`, the `groups` and `projects` properties should be like : + > List(["", "", ..., ""]) + instead of, in ` 1.7.0`: + > JSON STR: "group1 group2" + """ + + message_repository: MessageRepository = InMemoryMessageRepository() + + @property + @abc.abstractmethod + def migrate_from_key(self) -> str: + ... + + @property + @abc.abstractmethod + def migrate_to_key(self) -> str: + ... + + @classmethod + def _should_migrate(cls, config: Mapping[str, Any]) -> bool: + """ + This method determines whether config require migration. + Returns: + > True, if the transformation is necessary + > False, otherwise. + """ + if cls.migrate_from_key in config and cls.migrate_to_key not in config: + return True + return False + + @classmethod + def _transform_to_array(cls, config: Mapping[str, Any], source: SourceGitlab = None) -> Mapping[str, Any]: + # assign old values to new property that will be used within the new version + config[cls.migrate_to_key] = config[cls.migrate_to_key] if cls.migrate_to_key in config else [] + data = set(filter(None, config.get(cls.migrate_from_key).split(" "))) + config[cls.migrate_to_key] = list(data | set(config[cls.migrate_to_key])) + return config + + @classmethod + def _modify_and_save(cls, config_path: str, source: SourceGitlab, config: Mapping[str, Any]) -> Mapping[str, Any]: + # modify the config + migrated_config = cls._transform_to_array(config, source) + # save the config + source.write_config(migrated_config, config_path) + # return modified config + return migrated_config + + @classmethod + def _emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: + # add the Airbyte Control Message to message repo + cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) + # emit the Airbyte Control Message from message queue to stdout + for message in cls.message_repository._message_queue: + print(message.json(exclude_unset=True)) + + @classmethod + def migrate(cls, args: List[str], source: SourceGitlab) -> None: + """ + This method checks the input args, should the config be migrated, + transform if necessary and emit the CONTROL message. + """ + # get config path + config_path = AirbyteEntrypoint(source).extract_config(args) + # proceed only if `--config` arg is provided + if config_path: + # read the existing config + config = source.read_config(config_path) + # migration check + if cls._should_migrate(config): + cls._emit_control_message( + cls._modify_and_save(config_path, source, config), + ) + + +class MigrateGroups(MigrateStringToArray): + + migrate_from_key: str = "groups" + migrate_to_key: str = "groups_list" + + +class MigrateProjects(MigrateStringToArray): + + migrate_from_key: str = "projects" + migrate_to_key: str = "projects_list" diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/run.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/run.py new file mode 100644 index 000000000000..ddaf36b55b1c --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/run.py @@ -0,0 +1,17 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_gitlab import SourceGitlab +from source_gitlab.config_migrations import MigrateGroups, MigrateProjects + + +def run(): + source = SourceGitlab() + MigrateGroups.migrate(sys.argv[1:], source) + MigrateProjects.migrate(sys.argv[1:], source) + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/commits.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/commits.json index 89a7d2f5ae31..55b6809a6683 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/commits.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/commits.json @@ -37,6 +37,17 @@ "type": ["null", "string"], "format": "date-time" }, + "extended_trailers": { + "type": ["null", "object"], + "properties": { + "Cc": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + } + } + }, "committer_name": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/deployments.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/deployments.json new file mode 100644 index 000000000000..f8e9f69e1561 --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/deployments.json @@ -0,0 +1,210 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "iid": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "string"] + }, + "ref": { + "type": ["null", "string"] + }, + "sha": { + "type": ["null", "string"] + }, + "environment_name": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "user": { + "type": ["null", "object"], + "additionalProperties": true + }, + "user_full_name": { + "type": ["null", "string"] + }, + "user_username": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "environment": { + "type": ["null", "object"], + "additionalProperties": true + }, + "environment_id": { + "type": ["null", "integer"] + }, + "project_id": { + "type": ["null", "integer"] + }, + "deployable": { + "type": ["null", "object"], + "properties": { + "commit": { + "type": ["null", "object"], + "properties": { + "author_email": { + "type": ["null", "string"] + }, + "author_name": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "id": { + "type": ["null", "string"] + }, + "message": { + "type": ["null", "string"] + }, + "short_id": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + } + } + }, + "coverage": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "finished_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "ref": { + "type": ["null", "string"] + }, + "runner": { + "type": ["null", "string"] + }, + "stage": { + "type": ["null", "string"] + }, + "started_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "status": { + "type": ["null", "string"] + }, + "tag": { + "type": ["null", "boolean"] + }, + "project": { + "type": ["null", "object"], + "properties": { + "ci_job_token_scope_enabled": { + "type": ["null", "boolean"] + } + } + }, + "user": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "username": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "avatar_url": { + "type": ["null", "string"] + }, + "web_url": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "bio": { + "type": ["null", "string"] + }, + "location": { + "type": ["null", "string"] + }, + "public_email": { + "type": ["null", "string"] + }, + "skype": { + "type": ["null", "string"] + }, + "linkedin": { + "type": ["null", "string"] + }, + "twitter": { + "type": ["null", "string"] + }, + "website_url": { + "type": ["null", "string"] + }, + "organization": { + "type": ["null", "string"] + } + } + }, + "pipeline": { + "type": ["null", "object"], + "properties": { + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "id": { + "type": ["null", "integer"] + }, + "ref": { + "type": ["null", "string"] + }, + "sha": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "web_url": { + "type": ["null", "string"] + } + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epic_issues.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epic_issues.json index 0b27bc602861..7750ef4fdbc9 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epic_issues.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epic_issues.json @@ -75,7 +75,8 @@ "type": ["null", "integer"] }, "due_date": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" }, "confidential": { "type": ["null", "boolean"] @@ -144,11 +145,11 @@ }, "start_date": { "type": ["null", "string"], - "format": "date-time" + "format": "date" }, "due_date": { "type": ["null", "string"], - "format": "date-time" + "format": "date" }, "web_url": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epics.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epics.json index 5be8c292c03e..eee279b6510a 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epics.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/epics.json @@ -49,7 +49,8 @@ "type": ["null", "boolean"] }, "start_date_fixed": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" }, "start_date_from_inherited_source": { "type": ["null", "string", "boolean"] @@ -64,7 +65,8 @@ "type": ["null", "boolean"] }, "due_date_fixed": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" }, "due_date_from_inherited_source": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_members.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_members.json index 52705d38be9a..f5c2a0a60209 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_members.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/group_members.json @@ -35,7 +35,7 @@ }, "expires_at": { "type": ["null", "string"], - "format": "date" + "format": "date-time" }, "created_by": { "avatar_url": { @@ -56,6 +56,9 @@ "web_url": { "type": ["null", "string"] } + }, + "locked": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/groups.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/groups.json index 3c8870097537..b04cb5cbb15d 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/groups.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/groups.json @@ -19,6 +19,28 @@ "id": { "type": ["null", "integer"] }, + "organization_id": { + "type": ["null", "integer"] + }, + "default_branch_protection_defaults": { + "type": ["null", "object"], + "properties": { + "allow_force_push": { + "type": ["null", "boolean"] + }, + "allowed_to_merge": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "access_level": { + "type": ["null", "integer"] + } + } + } + } + } + }, "web_url": { "type": ["null", "string"] }, @@ -55,6 +77,9 @@ "emails_disabled": { "type": ["null", "boolean"] }, + "emails_enabled": { + "type": ["null", "boolean"] + }, "mentions_disabled": { "type": ["null", "boolean"] }, @@ -125,6 +150,9 @@ }, "shared_runners_setting": { "type": ["null", "string"] + }, + "service_access_tokens_expiration_enforced": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/issues.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/issues.json index e02aa19fcf1a..11583cdad76e 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/issues.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/issues.json @@ -100,21 +100,72 @@ }, "author": { "type": ["null", "object"], - "additionalProperties": true + "properties": { + "state": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "web_url": { + "type": ["null", "string"] + }, + "avatar_url": { + "type": ["null", "string"] + }, + "username": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + } + } }, "author_id": { "type": ["null", "integer"] }, "assignee": { "type": ["null", "object"], - "additionalProperties": true + "properties": { + "state": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "web_url": { + "type": ["null", "string"] + }, + "avatar_url": { + "type": ["null", "string"] + }, + "username": { + "type": ["null", "string"] + } + } }, "assignee_id": { "type": ["null", "integer"] }, "closed_by": { "type": ["null", "object"], - "additionalProperties": true + "properties": { + "state": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "web_url": { + "type": ["null", "string"] + }, + "avatar_url": { + "type": ["null", "string"] + }, + "username": { + "type": ["null", "string"] + } + } }, "closed_by_id": { "type": ["null", "integer"] @@ -184,20 +235,23 @@ } }, "epic": { - "id": { - "type": ["null", "integer"] - }, - "iid": { - "type": ["null", "integer"] - }, - "title": { - "type": ["null", "string"] - }, - "url": { - "type": ["null", "string"] - }, - "group_id": { - "type": ["null", "integer"] + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "iid": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "group_id": { + "type": ["null", "integer"] + } } }, "epic_iid": { diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/jobs.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/jobs.json index 4c41e56b46a5..00fb3b5d6aad 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/jobs.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/jobs.json @@ -11,6 +11,9 @@ "stage": { "type": ["null", "string"] }, + "archived": { + "type": ["null", "boolean"] + }, "name": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/project_members.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/project_members.json index 691e9ca74768..2cb53f1d3ca0 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/project_members.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/project_members.json @@ -56,6 +56,9 @@ "web_url": { "type": ["null", "string"] } + }, + "locked": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/projects.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/projects.json index 3e4a9ce4b9ab..e86a72556e05 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/projects.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/projects.json @@ -471,6 +471,27 @@ "marked_for_deletion_on": { "type": ["null", "string"], "format": "date" + }, + "ci_forward_deployment_rollback_allowed": { + "type": ["null", "boolean"] + }, + "emails_enabled": { + "type": ["null", "boolean"] + }, + "model_experiments_access_level": { + "type": ["null", "string"] + }, + "merge_trains_skip_train_allowed": { + "type": ["null", "boolean"] + }, + "code_suggestions": { + "type": ["null", "boolean"] + }, + "model_registry_access_level": { + "type": ["null", "string"] + }, + "ci_restrict_pipeline_cancellation_role": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/users.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/users.json index 9c6369ce1721..7b574191bf29 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/users.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/users.json @@ -19,6 +19,9 @@ }, "web_url": { "type": ["null", "string"] + }, + "locked": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py index de3545dc89c8..8a69c840b616 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py @@ -13,11 +13,14 @@ from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.requests_native_auth.oauth import SingleUseRefreshTokenOauth2Authenticator from airbyte_cdk.sources.streams.http.requests_native_auth.token import TokenAuthenticator +from airbyte_cdk.utils import AirbyteTracedException from requests.auth import AuthBase +from requests.exceptions import HTTPError from .streams import ( Branches, Commits, + Deployments, EpicIssues, Epics, GitlabStream, @@ -81,7 +84,13 @@ def refresh_access_token(self) -> Tuple[str, int, int, str]: def get_authenticator(config: MutableMapping) -> AuthBase: if config["credentials"]["auth_type"] == "access_token": return TokenAuthenticator(token=config["credentials"]["access_token"]) - return SingleUseRefreshTokenGitlabOAuth2Authenticator(config, token_refresh_endpoint=f"https://{config['api_url']}/oauth/token") + return SingleUseRefreshTokenGitlabOAuth2Authenticator( + config, + token_refresh_endpoint=f"https://{config['api_url']}/oauth/token", + refresh_token_error_status_codes=(400,), + refresh_token_error_key="error", + refresh_token_error_values="invalid_grant", + ) class SourceGitlab(AbstractSource): @@ -106,7 +115,7 @@ def _groups_stream(self, config: MutableMapping[str, Any]) -> Groups: def _projects_stream(self, config: MutableMapping[str, Any]) -> Union[Projects, GroupProjects]: if not self.__projects_stream: auth_params = self._auth_params(config) - project_ids = list(filter(None, config.get("projects", "").split(" "))) + project_ids = config.get("projects_list", []) groups_stream = self._groups_stream(config) if groups_stream.group_ids: self.__projects_stream = GroupProjects(project_ids=project_ids, parent_stream=groups_stream, **auth_params) @@ -121,7 +130,7 @@ def _auth_params(self, config: MutableMapping[str, Any]) -> Mapping[str, Any]: return self.__auth_params def _get_group_list(self, config: MutableMapping[str, Any]) -> List[str]: - group_ids = list(filter(None, config.get("groups", "").split(" "))) + group_ids = config.get("groups_list") # Gitlab exposes different APIs to get a list of groups. # We use https://docs.gitlab.com/ee/api/groups.html#list-groups in case there's no group IDs in the input config. # This API provides full information about all available groups, including subgroups. @@ -140,7 +149,36 @@ def _get_group_list(self, config: MutableMapping[str, Any]) -> List[str]: def _is_http_allowed() -> bool: return os.environ.get("DEPLOYMENT_MODE", "").upper() != "CLOUD" - def check_connection(self, logger, config) -> Tuple[bool, any]: + def _try_refresh_access_token(self, logger, config: Mapping[str, Any]) -> Mapping[str, Any]: + """ + This method attempts to refresh the expired `access_token`, while `refresh_token` is still valid. + In order to obtain the new `refresh_token`, the Customer should `re-auth` in the source settings. + """ + # get current authenticator + authenticator: Union[SingleUseRefreshTokenOauth2Authenticator, TokenAuthenticator] = self.__auth_params.get("authenticator") + if isinstance(authenticator, SingleUseRefreshTokenOauth2Authenticator): + try: + creds = authenticator.refresh_access_token() + # update the actual config values + config["credentials"]["access_token"] = creds[0] + config["credentials"]["refresh_token"] = creds[3] + config["credentials"]["token_expiry_date"] = authenticator.get_new_token_expiry_date(creds[1], creds[2]).to_rfc3339_string() + # update the config + emit_configuration_as_airbyte_control_message(config) + logger.info("The `access_token` was successfully refreshed.") + return config + except (AirbyteTracedException, HTTPError) as http_error: + raise http_error + except Exception as e: + raise Exception(f"Unknown error occurred while refreshing the `access_token`, details: {e}") + + def _handle_expired_access_token_error(self, logger, config: Mapping[str, Any]) -> Tuple[bool, Any]: + try: + return self.check_connection(logger, self._try_refresh_access_token(logger, config)) + except HTTPError as http_error: + return False, f"Unable to refresh the `access_token`, please re-authenticate in Sources > Settings. Details: {http_error}" + + def check_connection(self, logger, config) -> Tuple[bool, Any]: config = self._ensure_default_values(config) is_valid, scheme, _ = parse_url(config["api_url"]) if not is_valid: @@ -150,30 +188,44 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: try: projects = self._projects_stream(config) for stream_slice in projects.stream_slices(sync_mode=SyncMode.full_refresh): - next(projects.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice)) - return True, None + try: + next(projects.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice)) + return True, None + except StopIteration: + # in case groups/projects provided and 404 occurs + return False, "Groups and/or projects that you provide are invalid or you don't have permission to view it." return True, None # in case there's no projects + except HTTPError as http_error: + if config["credentials"]["auth_type"] == "oauth2.0": + if http_error.response.status_code == 401: + return self._handle_expired_access_token_error(logger, config) + elif http_error.response.status_code == 500: + return False, f"Unable to connect to Gitlab API with the provided credentials - {repr(http_error)}" + else: + return False, f"Unable to connect to Gitlab API with the provided Private Access Token - {repr(http_error)}" except Exception as error: - return False, f"Unable to connect to Gitlab API with the provided credentials - {repr(error)}" + return False, f"Unknown error occurred while checking the connection - {repr(error)}" def streams(self, config: MutableMapping[str, Any]) -> List[Stream]: config = self._ensure_default_values(config) auth_params = self._auth_params(config) + start_date = config.get("start_date") groups, projects = self._groups_stream(config), self._projects_stream(config) - pipelines = Pipelines(parent_stream=projects, start_date=config["start_date"], **auth_params) - merge_requests = MergeRequests(parent_stream=projects, start_date=config["start_date"], **auth_params) + pipelines = Pipelines(parent_stream=projects, start_date=start_date, **auth_params) + merge_requests = MergeRequests(parent_stream=projects, start_date=start_date, **auth_params) epics = Epics(parent_stream=groups, **auth_params) streams = [ groups, projects, Branches(parent_stream=projects, repository_part=True, **auth_params), - Commits(parent_stream=projects, repository_part=True, start_date=config["start_date"], **auth_params), + Commits(parent_stream=projects, repository_part=True, start_date=start_date, **auth_params), epics, + Deployments(parent_stream=projects, **auth_params), EpicIssues(parent_stream=epics, **auth_params), GroupIssueBoards(parent_stream=groups, **auth_params), - Issues(parent_stream=projects, start_date=config["start_date"], **auth_params), + Issues(parent_stream=projects, start_date=start_date, **auth_params), Jobs(parent_stream=pipelines, **auth_params), ProjectMilestones(parent_stream=projects, **auth_params), GroupMilestones(parent_stream=groups, **auth_params), diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/spec.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/spec.json index 9061dadafb07..75b2fc3c487c 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/spec.json +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/spec.json @@ -4,7 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Source Gitlab Spec", "type": "object", - "required": ["start_date", "credentials"], + "required": ["credentials"], "additionalProperties": true, "properties": { "credentials": { @@ -76,7 +76,7 @@ "start_date": { "type": "string", "title": "Start Date", - "description": "The date from which you'd like to replicate data for GitLab API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", + "description": "The date from which you'd like to replicate data for GitLab API, in the format YYYY-MM-DDT00:00:00Z. Optional. If not set, all data will be replicated. All data generated after this date will be replicated.", "examples": ["2021-03-01T00:00:00Z"], "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", "order": 1, @@ -98,13 +98,33 @@ "type": "string", "examples": ["airbyte.io"], "title": "Groups", - "description": "Space-delimited list of groups. e.g. airbyte.io.", + "description": "[DEPRECATED] Space-delimited list of groups. e.g. airbyte.io.", + "airbyte_hidden": true + }, + "groups_list": { + "type": "array", + "items": { + "type": "string" + }, + "examples": ["airbyte.io"], + "title": "Groups", + "description": "List of groups. e.g. airbyte.io.", "order": 3 }, "projects": { "type": "string", "title": "Projects", "examples": ["airbyte.io/documentation"], + "description": "[DEPRECATED] Space-delimited list of projects. e.g. airbyte.io/documentation meltano/tap-gitlab.", + "airbyte_hidden": true + }, + "projects_list": { + "type": "array", + "items": { + "type": "string" + }, + "title": "Projects", + "examples": ["airbyte.io/documentation"], "description": "Space-delimited list of projects. e.g. airbyte.io/documentation meltano/tap-gitlab.", "order": 4 } diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py index de655d03e974..d1f14ca59eef 100644 --- a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py +++ b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py @@ -92,7 +92,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp elif isinstance(response_data, dict): yield self.transform(response_data, **kwargs) else: - Exception(f"Unsupported type of response data for stream {self.name}") + self.logger.info(f"Unsupported type of response data for stream {self.name}") def transform(self, record: Dict[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs): for key in self.flatten_id_keys: @@ -166,7 +166,7 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late current_state = current_state.get(self.cursor_field) current_state_value = current_state or latest_cursor_value max_value = max(pendulum.parse(current_state_value), pendulum.parse(latest_cursor_value)) - current_stream_state[str(project_id)] = {self.cursor_field: str(max_value)} + current_stream_state[str(project_id)] = {self.cursor_field: max_value.to_iso8601_string()} return current_stream_state @staticmethod @@ -187,22 +187,31 @@ def stream_slices( stream_state = stream_state or {} super_slices = super().stream_slices(sync_mode, cursor_field, stream_state) for super_slice in super_slices: - start_point = self._start_date state_project_value = stream_state.get(str(super_slice["id"])) - if state_project_value: - state_value = state_project_value.get(self.cursor_field) - if state_value: - start_point = max(start_point, state_value) - for start_dt, end_dt in self._chunk_date_range(pendulum.parse(start_point)): + if self._start_date or state_project_value: + start_point = self._start_date + if state_project_value: + state_value = state_project_value.get(self.cursor_field) + if state_value and start_point: + start_point = max(start_point, state_value) + else: + start_point = state_value or start_point + for start_dt, end_dt in self._chunk_date_range(pendulum.parse(start_point)): + stream_slice = {key: value for key, value in super_slice.items()} + stream_slice[self.lower_bound_filter] = start_dt + stream_slice[self.upper_bound_filter] = end_dt + yield stream_slice + else: stream_slice = {key: value for key, value in super_slice.items()} - stream_slice[self.lower_bound_filter] = start_dt - stream_slice[self.upper_bound_filter] = end_dt yield stream_slice def request_params(self, stream_state=None, stream_slice: Mapping[str, Any] = None, **kwargs): params = super().request_params(stream_state, stream_slice, **kwargs) - params[self.lower_bound_filter] = stream_slice[self.lower_bound_filter] - params[self.upper_bound_filter] = stream_slice[self.upper_bound_filter] + lower_bound_filter = stream_slice.get(self.lower_bound_filter) + upper_bound_filter = stream_slice.get(self.upper_bound_filter) + if lower_bound_filter and upper_bound_filter: + params[self.lower_bound_filter] = lower_bound_filter + params[self.upper_bound_filter] = upper_bound_filter return params @@ -404,3 +413,17 @@ class EpicIssues(GitlabChildStream): flatten_id_keys = ["milestone", "assignee", "author"] flatten_list_keys = ["assignees"] path_template = "groups/{group_id}/epics/{iid}/issues" + + +class Deployments(GitlabChildStream): + primary_key = "id" + flatten_id_keys = ["user", "environment"] + path_template = "projects/{id}/deployments" + + def transform(self, record, stream_slice: Mapping[str, Any] = None, **kwargs): + super().transform(record, stream_slice, **kwargs) + record["user_username"] = record["user"]["username"] + record["user_full_name"] = record["user"]["name"] + record["environment_name"] = record["environment"]["name"] + record["project_id"] = stream_slice["id"] + return record diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py index d250f5489258..8471fb8e8b06 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/conftest.py @@ -10,22 +10,38 @@ def config(request): return { "start_date": "2021-01-01T00:00:00Z", "api_url": request.param, - "credentials": { - "auth_type": "access_token", - "access_token": "token" - } + "credentials": {"auth_type": "access_token", "access_token": "token"}, } @pytest.fixture(autouse=True) def disable_cache(mocker): - mocker.patch( - "source_gitlab.streams.Projects.use_cache", - new_callable=mocker.PropertyMock, - return_value=False - ) - mocker.patch( - "source_gitlab.streams.Groups.use_cache", - new_callable=mocker.PropertyMock, - return_value=False - ) + mocker.patch("source_gitlab.streams.Projects.use_cache", new_callable=mocker.PropertyMock, return_value=False) + mocker.patch("source_gitlab.streams.Groups.use_cache", new_callable=mocker.PropertyMock, return_value=False) + + +@pytest.fixture +def oauth_config(): + return { + "api_url": "gitlab.com", + "credentials": { + "auth_type": "oauth2.0", + "client_id": "client_id", + "client_secret": "client_secret", + "access_token": "access_token", + "token_expiry_date": "2023-01-01T00:00:00Z", + "refresh_token": "refresh_token", + }, + "start_date": "2021-01-01T00:00:00Z", + } + + +@pytest.fixture +def config_with_project_groups(): + return { + "start_date": "2021-01-01T00:00:00Z", + "api_url": "https://gitlab.com", + "credentials": {"auth_type": "access_token", "access_token": "token"}, + "groups_list": ["g1"], + "projects_list": ["p1"], + } diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_config.json b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_config.json new file mode 100644 index 000000000000..71f30753dc6e --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_config.json @@ -0,0 +1 @@ +{ "groups": "a b c", "groups_list": ["a", "c", "b"] } diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_config_migrations.py new file mode 100644 index 000000000000..b61fb232906d --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_config_migrations.py @@ -0,0 +1,21 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import os + +from source_gitlab.config_migrations import MigrateGroups +from source_gitlab.source import SourceGitlab + +TEST_CONFIG_PATH = f"{os.path.dirname(__file__)}/test_config.json" + + +def test_should_migrate(): + assert MigrateGroups._should_migrate({"groups": "group group2 group3"}) is True + assert MigrateGroups._should_migrate({"groups_list": ["test", "group2", "group3"]}) is False + + +def test__modify_and_save(): + source = SourceGitlab() + expected = {"groups": "a b c", "groups_list": ["b", "c", "a"]} + modified_config = MigrateGroups._modify_and_save(config_path=TEST_CONFIG_PATH, source=source, config={"groups": "a b c"}) + assert modified_config["groups_list"].sort() == expected["groups_list"].sort() + assert modified_config.get("groups") diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py index 1c2b2a54757b..5454ee1d7d76 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py @@ -13,7 +13,7 @@ def test_streams(config, requests_mock): requests_mock.get("/api/v4/groups", json=[{"id": "g1"}, {"id": "g256"}]) source = SourceGitlab() streams = source.streams(config) - assert len(streams) == 22 + assert len(streams) == 23 assert all([isinstance(stream, GitlabStream) for stream in streams]) groups, projects, *_ = streams assert groups.group_ids == ["g1", "g256"] @@ -26,12 +26,10 @@ def test_streams(config, requests_mock): ( {"url": "/api/v4/groups", "json": [{"id": "g1"}]}, {"url": "/api/v4/groups/g1", "json": [{"id": "g1", "projects": [{"id": "p1", "path_with_namespace": "p1"}]}]}, - {"url": "/api/v4/projects/p1", "json": {"id": "p1"}} + {"url": "/api/v4/projects/p1", "json": {"id": "p1"}}, ), - ( - {"url": "/api/v4/groups", "json": []}, - ), - ) + ({"url": "/api/v4/groups", "json": []},), + ), ) def test_connection_success(config, requests_mock, url_mocks): for url_mock in url_mocks: @@ -41,20 +39,91 @@ def test_connection_success(config, requests_mock, url_mocks): assert (status, msg) == (True, None) -def test_connection_fail_due_to_api_error(config, mocker, requests_mock): +def test_connection_invalid_projects_and_projects(config_with_project_groups, requests_mock): + requests_mock.register_uri("GET", "https://gitlab.com/api/v4/groups/g1?per_page=50", status_code=404) + requests_mock.register_uri("GET", "https://gitlab.com/api/v4/groups/g1/descendant_groups?per_page=50", status_code=404) + requests_mock.register_uri("GET", "https://gitlab.com/api/v4/projects/p1?per_page=50&statistics=1", status_code=404) + source = SourceGitlab() + status, msg = source.check_connection(logging.getLogger(), config_with_project_groups) + assert (status, msg) == (False, "Groups and/or projects that you provide are invalid or you don't have permission to view it.") + + +@pytest.mark.parametrize( + "errror_code, expected_status", + ( + (500, False), + (401, False), + ), +) +def test_connection_fail_due_to_api_error(errror_code, expected_status, config, mocker, requests_mock): + mocker.patch("time.sleep") + requests_mock.get("/api/v4/groups", status_code=errror_code) + source = SourceGitlab() + status, msg = source.check_connection(logging.getLogger(), config) + assert status is False + assert msg.startswith("Unable to connect to Gitlab API with the provided Private Access Token") + + +def test_connection_fail_due_to_api_error_oauth(oauth_config, mocker, requests_mock): mocker.patch("time.sleep") + test_response = { + "access_token": "new_access_token", + "expires_in": 7200, + "created_at": 1735689600, + # (7200 + 1735689600).timestamp().to_rfc3339_string() = "2025-01-01T02:00:00+00:00" + "refresh_token": "new_refresh_token", + } + requests_mock.post("https://gitlab.com/oauth/token", status_code=200, json=test_response) requests_mock.get("/api/v4/groups", status_code=500) source = SourceGitlab() - status, msg = source.check_connection(logging.getLogger(), config) - assert status is False, msg.startswith('Unable to connect to Gitlab API with the provided credentials - "DefaultBackoffException"') + status, msg = source.check_connection(logging.getLogger(), oauth_config) + assert status is False + assert msg.startswith("Unable to connect to Gitlab API with the provided credentials") + + +def test_connection_fail_due_to_expired_access_token_error(oauth_config, requests_mock): + expected = "Unable to refresh the `access_token`, please re-authenticate in Sources > Settings." + requests_mock.post("https://gitlab.com/oauth/token", status_code=401) + source = SourceGitlab() + status, msg = source.check_connection(logging.getLogger("airbyte"), oauth_config) + assert status is False + assert expected in msg + + +def test_connection_refresh_access_token(oauth_config, requests_mock): + expected = "Unknown error occurred while checking the connection" + requests_mock.post("https://gitlab.com/oauth/token", status_code=200, json={"access_token": "new access token"}) + source = SourceGitlab() + status, msg = source.check_connection(logging.getLogger("airbyte"), oauth_config) + assert status is False + assert expected in msg + + +def test_refresh_expired_access_token_on_error(oauth_config, requests_mock): + test_response = { + "access_token": "new_access_token", + "expires_in": 7200, + "created_at": 1735689600, + # (7200 + 1735689600).timestamp().to_rfc3339_string() = "2025-01-01T02:00:00+00:00" + "refresh_token": "new_refresh_token", + } + expected_token_expiry_date = "2025-01-01T02:00:00+00:00" + requests_mock.post("https://gitlab.com/oauth/token", status_code=200, json=test_response) + requests_mock.get("https://gitlab.com/api/v4/groups?per_page=50", status_code=200, json=[]) + source = SourceGitlab() + source.check_connection(logging.getLogger("airbyte"), oauth_config) + # check the updated config values + assert test_response.get("access_token") == oauth_config.get("credentials").get("access_token") + assert test_response.get("refresh_token") == oauth_config.get("credentials").get("refresh_token") + assert expected_token_expiry_date == oauth_config.get("credentials").get("token_expiry_date") @pytest.mark.parametrize( "api_url, deployment_env, expected_message", ( ("http://gitlab.my.company.org", "CLOUD", "Http scheme is not allowed in this environment. Please use `https` instead."), - ("https://gitlab.com/api/v4", "CLOUD", "Invalid API resource locator.") - ) + ("https://gitlab.com/api/v4", "CLOUD", "Invalid API resource locator."), + ), ) def test_connection_fail_due_to_config_error(mocker, api_url, deployment_env, expected_message): mocker.patch("os.environ", {"DEPLOYMENT_MODE": deployment_env}) @@ -62,10 +131,31 @@ def test_connection_fail_due_to_config_error(mocker, api_url, deployment_env, ex config = { "start_date": "2021-01-01T00:00:00Z", "api_url": api_url, - "credentials": { - "auth_type": "access_token", - "access_token": "token" - } + "credentials": {"auth_type": "access_token", "access_token": "token"}, } status, msg = source.check_connection(logging.getLogger(), config) assert (status, msg) == (False, expected_message) + + +def test_try_refresh_access_token(oauth_config, requests_mock): + test_response = { + "access_token": "new_access_token", + "expires_in": 7200, + "created_at": 1735689600, + # (7200 + 1735689600).timestamp().to_rfc3339_string() = "2025-01-01T02:00:00+00:00" + "refresh_token": "new_refresh_token", + } + requests_mock.post("https://gitlab.com/oauth/token", status_code=200, json=test_response) + + expected = {"api_url": "gitlab.com", + "credentials": {"access_token": "new_access_token", + "auth_type": "oauth2.0", + "client_id": "client_id", + "client_secret": "client_secret", + "refresh_token": "new_refresh_token", + "token_expiry_date": "2025-01-01T02:00:00+00:00"}, + "start_date": "2021-01-01T00:00:00Z"} + + source = SourceGitlab() + source._auth_params(oauth_config) + assert source._try_refresh_access_token(logger=logging.getLogger(), config=oauth_config) == expected diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py index 2ed3aa88ac14..283288168998 100644 --- a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py @@ -3,10 +3,23 @@ # import datetime +from unittest.mock import MagicMock import pytest +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http.auth import NoAuth -from source_gitlab.streams import Branches, Commits, Jobs, MergeRequestCommits, MergeRequests, Pipelines, Projects, Releases, Tags +from source_gitlab.streams import ( + Branches, + Commits, + Deployments, + Jobs, + MergeRequestCommits, + MergeRequests, + Pipelines, + Projects, + Releases, + Tags, +) auth_params = {"authenticator": NoAuth(), "api_url": "gitlab.com"} start_date = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=14) @@ -42,6 +55,11 @@ def jobs(pipelines): return Jobs(parent_stream=pipelines, **auth_params) +@pytest.fixture() +def deployments(projects): + return Deployments(parent_stream=projects, **auth_params) + + @pytest.fixture() def merge_request_commits(merge_requests): return MergeRequestCommits(parent_stream=merge_requests, **auth_params) @@ -60,31 +78,17 @@ def commits(projects): @pytest.mark.parametrize( "stream, extra_mocks, expected_call_count", ( - ( - "projects", - ({"url": "/api/v4/projects/p_1", "status_code": 403},), - 1 - ), - ( - "projects", - ({"url": "/api/v4/projects/p_1", "status_code": 404},), - 1 - ), + ("projects", ({"url": "/api/v4/projects/p_1", "status_code": 403},), 1), + ("projects", ({"url": "/api/v4/projects/p_1", "status_code": 404},), 1), ( "branches", - ( - {"url": "/api/v4/projects/p_1", "json": [{"id": "p_1"}]}, - {"url": "/api/v4/projects/p_1/branches", "status_code": 403} - ), - 2 + ({"url": "/api/v4/projects/p_1", "json": [{"id": "p_1"}]}, {"url": "/api/v4/projects/p_1/branches", "status_code": 403}), + 2, ), ( "branches", - ( - {"url": "/api/v4/projects/p_1", "json": [{"id": "p_1"}]}, - {"url": "/api/v4/projects/p_1/branches", "status_code": 404} - ), - 2 + ({"url": "/api/v4/projects/p_1", "json": [{"id": "p_1"}]}, {"url": "/api/v4/projects/p_1/branches", "status_code": 404}), + 2, ), ), ) @@ -104,22 +108,39 @@ def test_should_retry(mocker, requests_mock, stream, extra_mocks, expected_call_ ( "jobs", ( - ("/api/v4/projects/p_1/pipelines", [{"project_id": "p_1", "id": "build_project_p1"}],), + ( + "/api/v4/projects/p_1/pipelines", + [{"project_id": "p_1", "id": "build_project_p1"}], + ), ( "/api/v4/projects/p_1/pipelines/build_project_p1/jobs", - [ - {"id": "j_1", "user": {"id": "u_1"}, "pipeline": {"id": "p_17"}, "runner": None, "commit": {"id": "c_23"}} - ] + [{"id": "j_1", "user": {"id": "u_1"}, "pipeline": {"id": "p_17"}, "runner": None, "commit": {"id": "c_23"}}], ), ), - [{"commit": {"id": "c_23"}, "commit_id": "c_23", "id": "j_1", "pipeline": {"id": "p_17"}, "pipeline_id": "p_17", "project_id": "p_1", "runner": None, "runner_id": None, "user": {"id": "u_1"}, "user_id": "u_1"}] + [ + { + "commit": {"id": "c_23"}, + "commit_id": "c_23", + "id": "j_1", + "pipeline": {"id": "p_17"}, + "pipeline_id": "p_17", + "project_id": "p_1", + "runner": None, + "runner_id": None, + "user": {"id": "u_1"}, + "user_id": "u_1", + } + ], ), ( "tags", ( - ("/api/v4/projects/p_1/repository/tags", [{"commit": {"id": "c_1"}, "name": "t_1", "target": "ddc89"}],), + ( + "/api/v4/projects/p_1/repository/tags", + [{"commit": {"id": "c_1"}, "name": "t_1", "target": "ddc89"}], + ), ), - [{"commit": {"id": "c_1"}, "commit_id": "c_1", "project_id": "p_1", "name": "t_1", "target": "ddc89"}] + [{"commit": {"id": "c_1"}, "commit_id": "c_1", "project_id": "p_1", "name": "t_1", "target": "ddc89"}], ), ( "releases", @@ -131,21 +152,71 @@ def test_should_retry(mocker, requests_mock, stream, extra_mocks, expected_call_ "id": "r_1", "author": {"name": "John", "id": "666"}, "commit": {"id": "abcd689"}, - "milestones": [{"id": "m1", "title": "Q1"}, {"id": "m2", "title": "Q2"}] + "milestones": [{"id": "m1", "title": "Q1"}, {"id": "m2", "title": "Q2"}], + } + ], + ), + ), + [ + { + "author": {"id": "666", "name": "John"}, + "author_id": "666", + "commit": {"id": "abcd689"}, + "commit_id": "abcd689", + "id": "r_1", + "milestones": ["m1", "m2"], + "project_id": "p_1", + } + ], + ), + ( + "deployments", + ( + ( + "/api/v4/projects/p_1/deployments", + [ + { + "id": "r_1", + "user": {"name": "John", "id": "666", "username": "john"}, + "environment": {"name": "dev"}, + "commit": {"id": "abcd689"}, } ], ), ), - [{"author": {"id": "666", "name": "John"}, "author_id": "666", "commit": {"id": "abcd689"}, "commit_id": "abcd689", "id": "r_1", "milestones": ["m1", "m2"], "project_id": "p_1"}] + [ + { + "id": "r_1", + "user": {"name": "John", "id": "666", "username": "john"}, + "environment": {"name": "dev"}, + "commit": {"id": "abcd689"}, + "user_id": "666", + "environment_id": None, + "user_username": "john", + "user_full_name": "John", + "environment_name": "dev", + "project_id": "p_1", + } + ], ), ( "merge_request_commits", ( - ("/api/v4/projects/p_1/merge_requests", [{"id": "mr_1", "iid": "mr_1", "project_id": "p_1"}],), - ("/api/v4/projects/p_1/merge_requests/mr_1", [{"id": "mrc_1",}],), + ( + "/api/v4/projects/p_1/merge_requests", + [{"id": "mr_1", "iid": "mr_1", "project_id": "p_1"}], + ), + ( + "/api/v4/projects/p_1/merge_requests/mr_1", + [ + { + "id": "mrc_1", + } + ], + ), ), - [{"id": "mrc_1", "project_id": "p_1", "merge_request_iid": "mr_1"}] - ) + [{"id": "mrc_1", "project_id": "p_1", "merge_request_iid": "mr_1"}], + ), ) @@ -166,44 +237,94 @@ def test_transform(requests_mock, stream, response_mocks, expected_records, requ @pytest.mark.parametrize( "stream, current_state, latest_record, new_state", ( - ( - "pipelines", - {"219445": {"updated_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"updated_at": "2021-03-11T08:56:40.001+02:00"}}, - {"project_id": "219445", "updated_at": "2022-12-16T00:12:41.005675+02:00"}, - {"219445": {"updated_at": "2022-12-16T00:12:41.005675+02:00"}, "211378": {"updated_at": "2021-03-11T08:56:40.001+02:00"}} - ), - ( - "pipelines", - {"219445": {"updated_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"updated_at": "2021-03-11T08:56:40.012001+02:00"}}, - {"project_id": "211378", "updated_at": "2021-03-10T23:58:58.011+02:00"}, - {"219445": {"updated_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"updated_at": "2021-03-11T08:56:40.012001+02:00"}} - ), - ( - "pipelines", - {}, - {"project_id": "211378", "updated_at": "2021-03-10T23:58:58.010001+02:00"}, - {"211378": {"updated_at": "2021-03-10T23:58:58.010001+02:00"}} - ), - ( - "commits", - {"219445": {"created_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"created_at": "2021-03-11T08:56:40.001+02:00"}}, - {"project_id": "219445", "created_at": "2022-12-16T00:12:41.005675+02:00"}, - {"219445": {"created_at": "2022-12-16T00:12:41.005675+02:00"}, "211378": {"created_at": "2021-03-11T08:56:40.001+02:00"}} - ), - ( - "commits", - {"219445": {"created_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"created_at": "2021-03-11T08:56:40.012001+02:00"}}, - {"project_id": "211378", "created_at": "2021-03-10T23:58:58.011+02:00"}, - {"219445": {"created_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"created_at": "2021-03-11T08:56:40.012001+02:00"}} - ), - ( - "commits", - {}, - {"project_id": "211378", "created_at": "2021-03-10T23:58:58.010001+02:00"}, - {"211378": {"created_at": "2021-03-10T23:58:58.010001+02:00"}} - ) - ) + ( + "pipelines", + {"219445": {"updated_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"updated_at": "2021-03-11T08:56:40.001+02:00"}}, + {"project_id": "219445", "updated_at": "2022-12-16T00:12:41.005675+02:00"}, + {"219445": {"updated_at": "2022-12-16T00:12:41.005675+02:00"}, "211378": {"updated_at": "2021-03-11T08:56:40.001+02:00"}}, + ), + ( + "pipelines", + {"219445": {"updated_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"updated_at": "2021-03-11T08:56:40.012001+02:00"}}, + {"project_id": "211378", "updated_at": "2021-03-10T23:58:58.011+02:00"}, + {"219445": {"updated_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"updated_at": "2021-03-11T08:56:40.012001+02:00"}}, + ), + ( + "pipelines", + {}, + {"project_id": "211378", "updated_at": "2021-03-10T23:58:58.010001+02:00"}, + {"211378": {"updated_at": "2021-03-10T23:58:58.010001+02:00"}}, + ), + ( + "commits", + {"219445": {"created_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"created_at": "2021-03-11T08:56:40.001+02:00"}}, + {"project_id": "219445", "created_at": "2022-12-16T00:12:41.005675+02:00"}, + {"219445": {"created_at": "2022-12-16T00:12:41.005675+02:00"}, "211378": {"created_at": "2021-03-11T08:56:40.001+02:00"}}, + ), + ( + "commits", + {"219445": {"created_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"created_at": "2021-03-11T08:56:40.012001+02:00"}}, + {"project_id": "211378", "created_at": "2021-03-10T23:58:58.011+02:00"}, + {"219445": {"created_at": "2022-12-14T17:07:34.005675+02:00"}, "211378": {"created_at": "2021-03-11T08:56:40.012001+02:00"}}, + ), + ( + "commits", + {}, + {"project_id": "211378", "created_at": "2021-03-10T23:58:58.010001+02:00"}, + {"211378": {"created_at": "2021-03-10T23:58:58.010001+02:00"}}, + ), + ), ) def test_updated_state(stream, current_state, latest_record, new_state, request): stream = request.getfixturevalue(stream) assert stream.get_updated_state(current_state, latest_record) == new_state + + +def test_parse_response_unsuported_response_type(request, caplog): + stream = request.getfixturevalue("pipelines") + from unittest.mock import MagicMock + response = MagicMock() + response.status_code = 200 + response.json = MagicMock(return_value="") + list(stream.parse_response(response=response)) + assert "Unsupported type of response data for stream pipelines" in caplog.text + + +def test_stream_slices_child_stream(request, requests_mock): + commits = request.getfixturevalue("commits") + requests_mock.get("https://gitlab.com/api/v4/projects/p_1?per_page=50&statistics=1", + json=[{"id": 13082000, "description": "", "name": "New CI Test Project"}]) + + slices = list(commits.stream_slices(sync_mode=SyncMode.full_refresh, stream_state={"13082000": {""'created_at': "2021-03-10T23:58:1213"}})) + assert slices + + +def test_next_page_token(request): + response = MagicMock() + response.status_code = 200 + response.json = MagicMock(return_value=["some data"]) + commits = request.getfixturevalue("commits") + assert not commits.next_page_token(response) + data = ["some data" for x in range(0, 50)] + response.json = MagicMock(return_value=data) + assert commits.next_page_token(response) == {'page': 2} + response.json = MagicMock(return_value={"data": "some data"}) + assert not commits.next_page_token(response) + + +def test_availability_strategy(request): + commits = request.getfixturevalue("commits") + assert not commits.availability_strategy + + +def test_request_params(request): + commits = request.getfixturevalue("commits") + expected = {'per_page': 50, 'page': 2, 'with_stats': True} + assert commits.request_params(stream_slice={"updated_after": "2021-03-10T23:58:1213"}, next_page_token={'page': 2}) == expected + + +def test_chunk_date_range(request): + commits = request.getfixturevalue("commits") + # start point in future + start_point = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=1) + assert not list(commits._chunk_date_range(start_point)) diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_utils.py new file mode 100644 index 000000000000..bd107e1a16dc --- /dev/null +++ b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_utils.py @@ -0,0 +1,17 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import pytest +from source_gitlab.utils import parse_url + + +@pytest.mark.parametrize( + "url, expected", + ( + ("http://example.com", (True, "http", "example.com")), + ("http://example", (True, "http", "example")), + ("test://example.com", (False, "", "")), + ("https://example.com/test/test2", (False, "", "")), + ) +) +def test_parse_url(url, expected): + assert parse_url(url) == expected diff --git a/airbyte-integrations/connectors/source-glassfrog/Dockerfile b/airbyte-integrations/connectors/source-glassfrog/Dockerfile index 6ac45d8e9f95..bc652bb843ce 100644 --- a/airbyte-integrations/connectors/source-glassfrog/Dockerfile +++ b/airbyte-integrations/connectors/source-glassfrog/Dockerfile @@ -34,5 +34,6 @@ COPY source_glassfrog ./source_glassfrog ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.2.0 + LABEL io.airbyte.name=airbyte/source-glassfrog diff --git a/airbyte-integrations/connectors/source-glassfrog/README.md b/airbyte-integrations/connectors/source-glassfrog/README.md index 7e5cde1f5cc1..ed98e29b2bb9 100644 --- a/airbyte-integrations/connectors/source-glassfrog/README.md +++ b/airbyte-integrations/connectors/source-glassfrog/README.md @@ -1,45 +1,12 @@ # Glassfrog Source -This is the repository for the Glassfrog source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/glassfrog). +This is the repository for the Glassfrog configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/glassfrog). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-glassfrog:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/glassfrog) +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/glassfrog) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_glassfrog/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. @@ -47,28 +14,21 @@ See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source glassfrog test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-glassfrog:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-glassfrog build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-glassfrog:airbyteDocker +An image will be built with the tag `airbyte/source-glassfrog:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-glassfrog:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-glassfrog:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-glassfrog:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-glassfrog:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-glassfrog test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-glassfrog:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-glassfrog:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-glassfrog test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/glassfrog.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-glassfrog/__init__.py b/airbyte-integrations/connectors/source-glassfrog/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-glassfrog/acceptance-test-config.yml b/airbyte-integrations/connectors/source-glassfrog/acceptance-test-config.yml index 91b1c13d09f2..0aa3c634fc42 100644 --- a/airbyte-integrations/connectors/source-glassfrog/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-glassfrog/acceptance-test-config.yml @@ -1,20 +1,31 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-glassfrog:dev -tests: +acceptance_tests: spec: - - spec_path: "source_glassfrog/spec.yaml" + tests: + - spec_path: "source_glassfrog/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: ["custom_fields", "checklist_items", "metrics", "projects"] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: custom_fields + - name: checklist_items + - name: metrics + - name: projects + incremental: + bypass_reason: "This connector does not implement incremental sync" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-glassfrog/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-glassfrog/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-glassfrog/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-glassfrog/bootstrap.md b/airbyte-integrations/connectors/source-glassfrog/bootstrap.md deleted file mode 100644 index 03a1255ab014..000000000000 --- a/airbyte-integrations/connectors/source-glassfrog/bootstrap.md +++ /dev/null @@ -1,28 +0,0 @@ -# Glassfrog - -## Overview - -Glassfrog is a team management platform that supports the implementation of [Holacracy](https://www.holacracy.org). It helps organizations gain transparency and hold meetings as required by the methodology. - -## Endpoints - -This Source is capable of syncing the following Streams: - -* [Assignments](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#db2934bd-8c07-1951-b273-51fbc2dc6422) -* [Checklist items](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#a81716d4-b492-79ff-1348-9048fd9dc527) -* [Circles](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#ed696857-c3d8-fba1-a174-fbe63de07798) -* [Custom fields](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#901f8ec2-a986-0291-2fa2-281c16622107) -* [Metrics](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#00d4f5fb-d6e5-5521-a77d-bdce50a9fb84) -* [People](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#78b74b9f-72b7-63fc-a18c-18518932944b) -* [Projects](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#110bde88-a319-ae9c-077a-9752fd2f0843) -* [Roles](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#d1f31f7a-1d42-8c86-be1d-a36e640bf993) - -## Additional notes - -* Authentication is handled via an API key, which is [obtained from the platform](https://app.glassfrog.com/org/27355/api_keys). -* API endpoints return all results for the resource, there is no documented pagination mechanism. -* There are no documented rate limits for the API. - -## API reference - -See the [API reference documents](https://support.glassfrog.com/support/solutions/articles/9000066846-how-do-i-get-api-keys-). diff --git a/airbyte-integrations/connectors/source-glassfrog/build.gradle b/airbyte-integrations/connectors/source-glassfrog/build.gradle deleted file mode 100644 index b09bea255f2d..000000000000 --- a/airbyte-integrations/connectors/source-glassfrog/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_glassfrog' -} diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-glassfrog/integration_tests/invalid_config.json index 40061e658e4d..d5ae35771339 100644 --- a/airbyte-integrations/connectors/source-glassfrog/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-glassfrog/integration_tests/invalid_config.json @@ -1,3 +1,3 @@ { - "api_key": "xxxxxxx" + "api_key": "invalid_api_key" } diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_config.json new file mode 100644 index 000000000000..0cc92f7a010d --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "api_key": "api_key" +} diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-glassfrog/metadata.yaml b/airbyte-integrations/connectors/source-glassfrog/metadata.yaml index 4e8922bd6d22..b75ae80dbeb8 100644 --- a/airbyte-integrations/connectors/source-glassfrog/metadata.yaml +++ b/airbyte-integrations/connectors/source-glassfrog/metadata.yaml @@ -1,22 +1,26 @@ data: + allowedHosts: + hosts: + - api.glassfrog.com + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: cf8ff320-6272-4faa-89e6-4402dc17e5d5 - dockerImageTag: 0.1.1 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-glassfrog githubIssueLabel: source-glassfrog icon: glassfrog.svg license: MIT name: Glassfrog - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: "2022-06-16" releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/glassfrog tags: - - language:python + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-glassfrog/setup.py b/airbyte-integrations/connectors/source-glassfrog/setup.py index 66e81d3d6700..e62098fca78b 100644 --- a/airbyte-integrations/connectors/source-glassfrog/setup.py +++ b/airbyte-integrations/connectors/source-glassfrog/setup.py @@ -11,7 +11,7 @@ TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", - "pytest~=6.1", + "pytest~=6.2", "pytest-mock~=3.6.1", ] diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/auth.py b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/auth.py deleted file mode 100644 index e4a7d8815704..000000000000 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/auth.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from typing import Any, Mapping - -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator - - -class GlassfrogAuthenticator(TokenAuthenticator): - def __init__(self, config: Mapping[str, Any]): - self.config = config - - def get_auth_header(self) -> Mapping[str, Any]: - return {"X-Auth-Token": self.config.get("api_key", "")} diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/manifest.yaml b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/manifest.yaml new file mode 100644 index 000000000000..37e08334441a --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/manifest.yaml @@ -0,0 +1,115 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["{{ parameters.path}}"] + requester: + type: HttpRequester + url_base: "https://api.glassfrog.com/api/v3/" + http_method: "GET" + authenticator: + type: ApiKeyAuthenticator + header: "X-Auth-Token" + api_token: "{{ config['api_key'] }}" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester" + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + assignments_stream: + $ref: "#/definitions/base_stream" + name: "assignments" + primary_key: "id" + $parameters: + path: "assignments" + + checklist_items_stream: + $ref: "#/definitions/base_stream" + name: "checklist_items" + primary_key: "id" + $parameters: + path: "checklist_items" + + circles_stream: + $ref: "#/definitions/base_stream" + name: "circles" + primary_key: "id" + $parameters: + path: "circles" + + custom_fields_stream: + $ref: "#/definitions/base_stream" + name: "custom_fields" + primary_key: "id" + $parameters: + path: "custom_fields" + + metrics_stream: + $ref: "#/definitions/base_stream" + name: "metrics" + primary_key: "id" + $parameters: + path: "metrics" + + people_stream: + $ref: "#/definitions/base_stream" + name: "people" + primary_key: "id" + $parameters: + path: "people" + + projects_stream: + $ref: "#/definitions/base_stream" + name: "projects" + primary_key: "id" + $parameters: + path: "projects" + + roles_stream: + $ref: "#/definitions/base_stream" + name: "roles" + primary_key: "id" + $parameters: + path: "roles" + +streams: + - "#/definitions/assignments_stream" + - "#/definitions/checklist_items_stream" + - "#/definitions/circles_stream" + - "#/definitions/custom_fields_stream" + - "#/definitions/metrics_stream" + - "#/definitions/people_stream" + - "#/definitions/projects_stream" + - "#/definitions/roles_stream" + +check: + type: CheckStream + stream_names: + - "assignments" + +spec: + type: Spec + documentationUrl: https://docs.airbyte.com/integrations/sources/glassfrog + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + title: Glassfrog Spec + type: object + required: + - api_key + additionalProperties: true + properties: + api_key: + type: string + description: API key provided by Glassfrog + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json index c1ee6ca93121..d9bb2d86887b 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json @@ -24,6 +24,9 @@ "external_id": { "type": ["null", "integer"] }, + "settings": { + "type": ["null", "object"] + }, "links": { "type": ["null", "object"], "properties": { diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/source.py b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/source.py index d6a63fea4691..28a1cc729429 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/source.py +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/source.py @@ -2,143 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from abc import ABC -from typing import Any, Iterable, List, Mapping, Optional, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream +WARNING: Do not modify this file. +""" -from .auth import GlassfrogAuthenticator - -# Basic full refresh stream -class GlassfrogStream(HttpStream, ABC): - url_base = "https://api.glassfrog.com/api/v3/" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - json_response = response.json() - records = json_response.get(self.data_field, []) if self.data_field is not None else json_response - - for record in records: - yield record - - -class Assignments(GlassfrogStream): - # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#db2934bd-8c07-1951-b273-51fbc2dc6422 - data_field = "assignments" - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return self.data_field - - -class ChecklistItems(GlassfrogStream): - # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#a81716d4-b492-79ff-1348-9048fd9dc527 - data_field = "checklist_items" - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return self.data_field - - -class Circles(GlassfrogStream): - # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#ed696857-c3d8-fba1-a174-fbe63de07798 - data_field = "circles" - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return self.data_field - - -class CustomFields(GlassfrogStream): - # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#901f8ec2-a986-0291-2fa2-281c16622107 - data_field = "custom_fields" - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return self.data_field - - -class Metrics(GlassfrogStream): - # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#00d4f5fb-d6e5-5521-a77d-bdce50a9fb84 - data_field = "metrics" - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return self.data_field - - -class People(GlassfrogStream): - # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#78b74b9f-72b7-63fc-a18c-18518932944b - data_field = "people" - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return self.data_field - - -class Projects(GlassfrogStream): - # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#110bde88-a319-ae9c-077a-9752fd2f0843 - data_field = "projects" - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return self.data_field - - -class Roles(GlassfrogStream): - # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#d1f31f7a-1d42-8c86-be1d-a36e640bf993 - data_field = "roles" - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return self.data_field - - -# Source -class SourceGlassfrog(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - try: - url = "https://api.glassfrog.com/api/v3/people" - headers = {"X-Auth-Token": config["api_key"]} - - r = requests.get(url, headers=headers) - r.raise_for_status() - return True, None - except Exception as error: - return False, f"Unable to connect to Glassfrog API with the provided credentials - {repr(error)}" - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - auth = GlassfrogAuthenticator(config=config) - return [ - Assignments(authenticator=auth), - ChecklistItems(authenticator=auth), - Circles(authenticator=auth), - CustomFields(authenticator=auth), - Metrics(authenticator=auth), - People(authenticator=auth), - Projects(authenticator=auth), - Roles(authenticator=auth), - ] +# Declarative Source +class SourceGlassfrog(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/spec.yaml b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/spec.yaml deleted file mode 100644 index e0e1c6faa12d..000000000000 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/spec.yaml +++ /dev/null @@ -1,13 +0,0 @@ -documentationUrl: https://docs.airbyte.com/integrations/sources/glassfrog -connectionSpecification: - $schema: http://json-schema.org/draft-07/schema# - title: Glassfrog Spec - type: object - required: - - api_key - additionalProperties: true - properties: - api_key: - type: string - description: API key provided by Glassfrog - airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_source.py b/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_source.py deleted file mode 100644 index 9b32206d1095..000000000000 --- a/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_source.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -from pytest import fixture -from source_glassfrog.source import SourceGlassfrog - - -@fixture() -def config(request): - args = {"api_key": "xxxxxxx"} - return args - - -def test_check_connection(mocker, config): - source = SourceGlassfrog() - logger_mock = MagicMock() - (connection_status, error) = source.check_connection(logger_mock, config) - expected_status = False - assert connection_status == expected_status - - -def test_streams(mocker, config): - source = SourceGlassfrog() - streams = source.streams(config) - expected_streams_number = 8 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_streams.py deleted file mode 100644 index d485fd6bf4c8..000000000000 --- a/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_streams.py +++ /dev/null @@ -1,66 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_glassfrog.source import GlassfrogStream - - -@pytest.fixture -def patch_base_class(mocker): - mocker.patch.object(GlassfrogStream, "path", "v0/example_endpoint") - mocker.patch.object(GlassfrogStream, "primary_key", "test_primary_key") - mocker.patch.object(GlassfrogStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = GlassfrogStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = GlassfrogStream() - inputs = {"response": MagicMock()} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_request_headers(patch_base_class): - stream = GlassfrogStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = GlassfrogStream() - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = GlassfrogStream() - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = GlassfrogStream() - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-gnews/README.md b/airbyte-integrations/connectors/source-gnews/README.md index ad314c997843..69852a8d6d5c 100644 --- a/airbyte-integrations/connectors/source-gnews/README.md +++ b/airbyte-integrations/connectors/source-gnews/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-gnews:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/gnews) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_gnews/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-gnews:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-gnews build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-gnews:airbyteDocker +An image will be built with the tag `airbyte/source-gnews:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-gnews:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-gnews:dev check --conf docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-gnews:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-gnews:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-gnews test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-gnews:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-gnews:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-gnews test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/gnews.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-gnews/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-gnews/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-gnews/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-gnews/build.gradle b/airbyte-integrations/connectors/source-gnews/build.gradle deleted file mode 100644 index b486b124362b..000000000000 --- a/airbyte-integrations/connectors/source-gnews/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_gnews' -} diff --git a/airbyte-integrations/connectors/source-gocardless/README.md b/airbyte-integrations/connectors/source-gocardless/README.md index 10498e005c8e..3698a91fb186 100644 --- a/airbyte-integrations/connectors/source-gocardless/README.md +++ b/airbyte-integrations/connectors/source-gocardless/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-gocardless:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/gocardless) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_gocardless/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-gocardless:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-gocardless build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-gocardless:airbyteDocker +An image will be built with the tag `airbyte/source-gocardless:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-gocardless:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-gocardless:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-gocardless:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-gocardless:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-gocardless test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-gocardless:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-gocardless:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-gocardless test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/gocardless.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-gocardless/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-gocardless/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-gocardless/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-gocardless/build.gradle b/airbyte-integrations/connectors/source-gocardless/build.gradle deleted file mode 100644 index 48bc120dc432..000000000000 --- a/airbyte-integrations/connectors/source-gocardless/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_gocardless' -} diff --git a/airbyte-integrations/connectors/source-gong/README.md b/airbyte-integrations/connectors/source-gong/README.md index 432a1a599e51..e6b687c0f615 100644 --- a/airbyte-integrations/connectors/source-gong/README.md +++ b/airbyte-integrations/connectors/source-gong/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-gong:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/gong) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_gong/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-gong:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-gong build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-gong:airbyteDocker +An image will be built with the tag `airbyte/source-gong:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-gong:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-gong:dev check --confi docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-gong:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-gong:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-gong test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-gong:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-gong:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-gong test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/gong.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-gong/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-gong/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-gong/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-gong/build.gradle b/airbyte-integrations/connectors/source-gong/build.gradle deleted file mode 100644 index f243d3955091..000000000000 --- a/airbyte-integrations/connectors/source-gong/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_gong' -} diff --git a/airbyte-integrations/connectors/source-google-ads/BOOTSTRAP.md b/airbyte-integrations/connectors/source-google-ads/BOOTSTRAP.md index 89c07e4750cb..4092d3c7075c 100644 --- a/airbyte-integrations/connectors/source-google-ads/BOOTSTRAP.md +++ b/airbyte-integrations/connectors/source-google-ads/BOOTSTRAP.md @@ -9,14 +9,14 @@ The resources are listed [here](https://developers.google.com/google-ads/api/ref When querying data, there are three categories of information that can be fetched: - **Attributes**: These are properties of the various entities in the API e.g: the title or ID of an ad campaign. -- **Metrics**: metrics are statistics related to entities in the API. For example, the number of impressions for an ad or an ad campaign. All available metrics can be found [here](https://developers.google.com/google-ads/api/fields/v11/metrics). +- **Metrics**: metrics are statistics related to entities in the API. For example, the number of impressions for an ad or an ad campaign. All available metrics can be found [here](https://developers.google.com/google-ads/api/fields/v15/metrics). - **Segments**: These are ways to partition metrics returned in the query by particular attributes. For example, one could query for the number of impressions (views of an ad) by running SELECT metrics.impressions FROM campaigns which would return the number of impressions for each campaign e.g: 10k impressions. Or you could query for impressions segmented by device type e.g; SELECT metrics.impressions, segments.device FROM campaigns which would return the number of impressions broken down by device type e.g: 3k iOS and 7k Android. When summing the result across all segments, the sum should be the same (approximately) as when requesting the whole query without segments. This is a useful feature for granular data analysis as an advertiser may for example want to know if their ad is successful with a particular kind of person over the other. See more about segmentation [here](https://developers.google.com/google-ads/api/docs/concepts/retrieving-objects). -If you want to get a representation of the raw resources in the API e.g: just know what are all the ads or campaigns in your google account, you would query only for attributes e.g. SELECT campaign.title FROM campaigns. +If you want to get a representation of the raw resources in the API e.g: just know what are all the ads or campaigns in your Google account, you would query only for attributes e.g. SELECT campaign.title FROM campaigns. But if you wanted to get reports about the data (a common use case is impression data for an ad campaign) then you would query for metrics, potentially with segmentation. diff --git a/airbyte-integrations/connectors/source-google-ads/Dockerfile b/airbyte-integrations/connectors/source-google-ads/Dockerfile deleted file mode 100644 index 6727c54ae432..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" - -WORKDIR /airbyte/integration_code -COPY setup.py ./ -RUN pip install . -COPY source_google_ads ./source_google_ads -COPY main.py ./ - -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.7.4 -LABEL io.airbyte.name=airbyte/source-google-ads diff --git a/airbyte-integrations/connectors/source-google-ads/README.md b/airbyte-integrations/connectors/source-google-ads/README.md index 2ecbb03251ca..d539875f4dc0 100644 --- a/airbyte-integrations/connectors/source-google-ads/README.md +++ b/airbyte-integrations/connectors/source-google-ads/README.md @@ -27,14 +27,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-google-ads:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/google-ads) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_ads/spec.json` file. @@ -54,19 +46,70 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-google-ads:dev + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-google-ads build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-google-ads:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. -You can also build the connector image via Gradle: +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-google-ads:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-google-ads:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-google-ads:dev . +# Running the spec command against your patched connector +docker run airbyte/source-google-ads:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -75,44 +118,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-ads:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-ads:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-google-ads:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install -e '.[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-google-ads test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-google-ads:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-google-ads:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -122,8 +137,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-google-ads test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/google-ads.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml index f0c3f9051b53..8c7f037815c4 100644 --- a/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml @@ -15,56 +15,119 @@ acceptance_tests: discovery: tests: - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "1.0.0" # schemas of default streams were updated basic_read: tests: - config_path: "secrets/config.json" expect_records: path: "integration_tests/expected_records.jsonl" - timeout_seconds: 600 + extra_fields: no + exact_order: yes + extra_records: yes # the file with all the records is 15 MB, so comparing only 3 records + timeout_seconds: 3600 empty_streams: - - name: "accounts" - bypass_reason: "Floating data" - - name: "display_topics_performance_report" - bypass_reason: "Stream not filled yet." - - name: "account_labels" - bypass_reason: "Unable to seed the stream" - - name: "ad_group_criterion_labels" - bypass_reason: "Unable to seed the stream" + - name: "customer_label" + bypass_reason: "Data is present in UI, but not in API: supposedly insufficient permissions" + - name: "shopping_performance_view" + bypass_reason: "No shopping campaign, need item for sale" + - name: "topic_view" + bypass_reason: "No data for this date range, tested in next config" - name: "click_view" - bypass_reason: "Stream not filled yet." - - name: "unhappytable" - bypass_reason: "Stream not filled yet." - - name: "shopping_performance_report" - bypass_reason: "Stream not filled yet." + bypass_reason: "Stream has data only for last 90 days, next config is used for testing it" + ignored_fields: + customer: + - name: customer.optimization_score_weight + bypass_reason: "Value can be updated by Google Ads" + - name: customer.optimization_score + bypass_reason: "Value can be updated by Google Ads" + - name: customer.pay_per_conversion_eligibility_failure_reasons + bypass_reason: "Value can be updated by Google Ads" + - config_path: "secrets/config_click_view.json" + expect_records: + path: "integration_tests/expected_records_click.jsonl" + timeout_seconds: 3600 + empty_streams: + - name: "customer_label" + bypass_reason: "Data is present in UI, but not in API: supposedly insufficient permissions" + - name: "shopping_performance_view" + bypass_reason: "No shopping campaign, need item for sale" + - name: "display_keyword_view" + bypass_reason: "No data for this date range, tested in previous config" + - name: "keyword_view" + bypass_reason: "No data for this date range, tested in previous config" + ignored_fields: + customer: + - name: customer.optimization_score_weight + bypass_reason: "Value can be updated by Google Ads" + - name: customer.optimization_score + bypass_reason: "Value can be updated by Google Ads" + - name: customer.pay_per_conversion_eligibility_failure_reasons + bypass_reason: "Value can be updated by Google Ads" + campaign_budget: + - name: campaign_budget.recommended_budget_estimated_change_weekly_interactions + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.all_conversions + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.all_conversions_from_interactions_rate + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.all_conversions_value + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.conversions + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.conversions_from_interactions_rate + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.conversions_value + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.cost_per_all_conversions + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.cost_per_conversion + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.value_per_all_conversions + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.value_per_conversion + bypass_reason: "Value can be updated by Google Ads" + campaign: + - name: campaign.optimization_score + bypass_reason: "Value can be updated by Google Ads" + ad_group_ad_legacy: + - name: metrics.all_conversions_from_interactions_rate + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.all_conversions_value + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.all_conversions + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.conversions_from_interactions_rate + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.conversions_value + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.conversions + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.cost_per_all_conversions + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.cost_per_conversion + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.cost_per_current_model_attributed_conversion + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.current_model_attributed_conversions_value + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.current_model_attributed_conversions + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.value_per_all_conversions + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.value_per_conversion + bypass_reason: "Value can be updated by Google Ads" + - name: metrics.value_per_current_model_attributed_conversion + bypass_reason: "Value can be updated by Google Ads" full_refresh: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - - config_path: "secrets/config_with_gaql.json" - configured_catalog_path: "integration_tests/configured_catalog_with_gaql_only.json" - # This config allows to read from the click_view stream which is empty in other configs. - # It should be tested anyway because it has different date range compared to other streams. - - config_path: "secrets/config_click_view.json" - configured_catalog_path: "integration_tests/configured_catalog_with_click_view.json" + - config_path: "secrets/config_manager_account.json" incremental: tests: - config_path: "secrets/incremental_config.json" + timeout_seconds: 3600 configured_catalog_path: "integration_tests/incremental_catalog.json" - threshold_days: 14 future_state: future_state_path: "integration_tests/abnormal_state.json" - cursor_paths: - account_performance_report: ["4651612872", "segments.date"] - click_view: ["4651612872", "segments.date"] - geographic_report: ["4651612872", "segments.date"] - keyword_report: ["4651612872", "segments.date"] - display_topics_performance_report: ["4651612872", "segments.date"] - shopping_performance_report: ["4651612872", "segments.date"] - ad_group_ads: ["4651612872", "segments.date"] - ad_groups: ["4651612872", "segments.date"] - accounts: ["4651612872", "segments.date"] - campaigns: ["4651612872", "segments.date"] - campaign_budget: ["4651612872", "segments.date"] - user_location_report: ["4651612872", "segments.date"] - ad_group_ad_report: ["4651612872", "segments.date"] - display_keyword_performance_report: ["4651612872", "segments.date"] diff --git a/airbyte-integrations/connectors/source-google-ads/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-google-ads/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-google-ads/build.gradle b/airbyte-integrations/connectors/source-google-ads/build.gradle deleted file mode 100644 index d0deb6f1ae4a..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_google_ads' -} diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/abnormal_state.json index 939fc0556265..3adb6391998c 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/abnormal_state.json @@ -17,77 +17,77 @@ "type": "STREAM", "stream": { "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, - "stream_descriptor": { "name": "geographic_report" } + "stream_descriptor": { "name": "geographic_view" } } }, { "type": "STREAM", "stream": { "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, - "stream_descriptor": { "name": "keyword_report" } + "stream_descriptor": { "name": "keyword_view" } } }, { "type": "STREAM", "stream": { "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, - "stream_descriptor": { "name": "display_topics_performance_report" } + "stream_descriptor": { "name": "topic_view" } } }, { "type": "STREAM", "stream": { "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, - "stream_descriptor": { "name": "shopping_performance_report" } + "stream_descriptor": { "name": "shopping_performance_view" } } }, { "type": "STREAM", "stream": { "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, - "stream_descriptor": { "name": "ad_group_ads" } + "stream_descriptor": { "name": "ad_group_ad" } } }, { "type": "STREAM", "stream": { "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, - "stream_descriptor": { "name": "ad_groups" } + "stream_descriptor": { "name": "ad_group" } } }, { "type": "STREAM", "stream": { "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, - "stream_descriptor": { "name": "accounts" } + "stream_descriptor": { "name": "customer" } } }, { "type": "STREAM", "stream": { "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, - "stream_descriptor": { "name": "campaigns" } + "stream_descriptor": { "name": "campaign" } } }, { "type": "STREAM", "stream": { "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, - "stream_descriptor": { "name": "user_location_report" } + "stream_descriptor": { "name": "user_location_view" } } }, { "type": "STREAM", "stream": { "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, - "stream_descriptor": { "name": "ad_group_ad_report" } + "stream_descriptor": { "name": "ad_group_ad_legacy" } } }, { "type": "STREAM", "stream": { "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, - "stream_descriptor": { "name": "display_keyword_performance_report" } + "stream_descriptor": { "name": "display_keyword_view" } } }, { @@ -101,14 +101,40 @@ "type": "STREAM", "stream": { "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, - "stream_descriptor": { "name": "campaign_bidding_strategies" } + "stream_descriptor": { "name": "campaign_bidding_strategy" } } }, { "type": "STREAM", "stream": { "stream_state": { "4651612872": { "segments.date": "2222-01-01" } }, - "stream_descriptor": { "name": "ad_group_bidding_strategies" } + "stream_descriptor": { "name": "ad_group_bidding_strategy" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "change_status": { + "4651612872": { + "change_status.last_change_date_time": "2024-08-16 13:20:01.003295" + } + } + }, + "stream_descriptor": { "name": "ad_group_criterion" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "change_status": { + "4651612872": { + "change_status.last_change_date_time": "2024-08-16 13:20:01.003295" + } + } + }, + "stream_descriptor": { "name": "campaign_criterion" } } } ] diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json index 7e19621f3bb4..47d5458aa18c 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json @@ -2,16 +2,54 @@ "streams": [ { "stream": { - "name": "ad_group_ad_report", + "name": "ad_group_ad_legacy", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, + "source_defined_primary_key": [ + ["ad_group.id"], + ["ad_group_ad.ad.id"], + ["segments.date"], + ["segments.ad_network_type"] + ], "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", + "primary_key": [ + ["ad_group.id"], + ["ad_group_ad.ad.id"], + ["segments.date"], + ["segments.ad_network_type"] + ], "cursor_field": ["segments.date"] }, + { + "stream": { + "name": "campaign_budget", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["segments.date"], + "source_defined_primary_key": [ + ["customer.id"], + ["campaign_budget.id"], + ["segments.date"], + ["segments.budget_campaign_association_status.campaign"], + ["segments.budget_campaign_association_status.status"] + ] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["segments.date"], + "primary_key": [ + ["customer.id"], + ["campaign_budget.id"], + ["segments.date"], + ["segments.budget_campaign_association_status.campaign"], + ["segments.budget_campaign_association_status.status"] + ] + }, { "stream": { "name": "ad_group_custom", @@ -29,11 +67,23 @@ "name": "account_performance_report", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [ + ["customer.id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", + "primary_key": [ + ["customer.id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "cursor_field": ["segments.date"] }, { @@ -42,65 +92,123 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "source_defined_primary_key": [["click_view.gclid"], ["segments.date"]], + "source_defined_primary_key": [ + ["click_view.gclid"], + ["segments.date"], + ["segments.ad_network_type"] + ], "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", "cursor_field": ["segments.date"], - "primary_key": [["click_view.gclid"], ["segments.date"]] + "primary_key": [ + ["click_view.gclid"], + ["segments.date"], + ["segments.ad_network_type"] + ] }, { "stream": { - "name": "geographic_report", + "name": "geographic_view", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [ + ["customer.id"], + ["geographic_view.country_criterion_id"], + ["geographic_view.location_type"], + ["segments.date"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", + "primary_key": [ + ["customer.id"], + ["geographic_view.country_criterion_id"], + ["geographic_view.location_type"], + ["segments.date"] + ], "cursor_field": ["segments.date"] }, { "stream": { - "name": "keyword_report", + "name": "keyword_view", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [ + ["ad_group_criterion.criterion_id"], + ["ad_group.id"], + ["segments.date"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", + "primary_key": [ + ["ad_group_criterion.criterion_id"], + ["ad_group.id"], + ["segments.date"] + ], "cursor_field": ["segments.date"] }, { "stream": { - "name": "display_keyword_performance_report", + "name": "display_keyword_view", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [ + ["ad_group.id"], + ["ad_group_criterion.criterion_id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", + "primary_key": [ + ["ad_group.id"], + ["ad_group_criterion.criterion_id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "cursor_field": ["segments.date"] }, { "stream": { - "name": "display_topics_performance_report", + "name": "topic_view", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [ + ["ad_group.id"], + ["ad_group_criterion.criterion_id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", + "primary_key": [ + ["ad_group.id"], + ["ad_group_criterion.criterion_id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "destination_sync_mode": "overwrite", "cursor_field": ["segments.date"] }, { "stream": { - "name": "shopping_performance_report", + "name": "shopping_performance_view", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -112,11 +220,12 @@ }, { "stream": { - "name": "ad_group_ads", + "name": "ad_group_ad", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "source_defined_primary_key": [ + ["ad_group.id"], ["ad_group_ad.ad.id"], ["segments.date"] ], @@ -125,11 +234,11 @@ "sync_mode": "incremental", "destination_sync_mode": "overwrite", "cursor_field": ["segments.date"], - "primary_key": [["ad_group_ad.ad.id"], ["segments.date"]] + "primary_key": [["ad_group.id"], ["ad_group_ad.ad.id"], ["segments.date"]] }, { "stream": { - "name": "ad_groups", + "name": "ad_group", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -143,7 +252,7 @@ }, { "stream": { - "name": "accounts", + "name": "customer", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -157,61 +266,89 @@ }, { "stream": { - "name": "campaigns", + "name": "campaign", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["segments.date"], - "source_defined_primary_key": [["campaign.id"], ["segments.date"]] + "source_defined_primary_key": [ + ["campaign.id"], + ["segments.date"], + ["segments.hour"], + ["segments.ad_network_type"] + ] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", "cursor_field": ["segments.date"], - "primary_key": [["campaign.id"], ["segments.date"]] + "primary_key": [ + ["campaign.id"], + ["segments.date"], + ["segments.hour"], + ["segments.ad_network_type"] + ] }, { "stream": { - "name": "campaign_labels", + "name": "campaign_label", "json_schema": {}, "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["campaign_label.resource_name"]] + "source_defined_primary_key": [["campaign.id"], ["label.id"]] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite", - "primary_key": [["campaign_label.resource_name"]] + "primary_key": [["campaign.id"], ["label.id"]] }, { "stream": { - "name": "ad_group_labels", + "name": "ad_group_label", "json_schema": {}, "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["ad_group_label.resource_name"]] + "source_defined_primary_key": [["ad_group.id"], ["label.id"]] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite", - "primary_key": [["ad_group_label.resource_name"]] + "primary_key": [["ad_group.id"], ["label.id"]] }, { "stream": { - "name": "ad_group_ad_labels", + "name": "ad_group_ad_label", "json_schema": {}, "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["ad_group_ad_label.resource_name"]] + "source_defined_primary_key": [ + ["ad_group.id"], + ["ad_group_ad.ad.id"], + ["label.id"] + ] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite", - "primary_key": [["ad_group_ad_label.resource_name"]] + "primary_key": [["ad_group.id"], ["ad_group_ad.ad.id"], ["label.id"]] }, { "stream": { - "name": "user_location_report", + "name": "user_location_view", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, + "source_defined_primary_key": [ + ["customer.id"], + ["user_location_view.country_criterion_id"], + ["user_location_view.targeting_location"], + ["segments.date"], + ["segments.ad_network_type"] + ], "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", + "primary_key": [ + ["customer.id"], + ["user_location_view.country_criterion_id"], + ["user_location_view.targeting_location"], + ["segments.date"], + ["segments.ad_network_type"] + ], "cursor_field": ["segments.date"] }, { @@ -226,18 +363,6 @@ "destination_sync_mode": "overwrite", "cursor_field": ["segments.date"] }, - { - "stream": { - "name": "unhappytable", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["segments.date"] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["segments.date"] - }, { "stream": { "name": "custom_audience", @@ -251,23 +376,27 @@ "stream": { "name": "audience", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["customer.id"], ["audience.id"]] }, "sync_mode": "full_refresh", + "primary_key": [["customer.id"], ["audience.id"]], "destination_sync_mode": "overwrite" }, { "stream": { "name": "user_interest", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["user_interest.user_interest_id"]] }, "sync_mode": "full_refresh", + "primary_key": [["user_interest.user_interest_id"]], "destination_sync_mode": "overwrite" }, { "stream": { - "name": "labels", + "name": "label", "json_schema": {}, "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["label.id"]] @@ -278,7 +407,7 @@ }, { "stream": { - "name": "account_labels", + "name": "customer_label", "json_schema": {}, "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["customer_label.resource_name"]] @@ -289,7 +418,7 @@ }, { "stream": { - "name": "campaign_bidding_strategies", + "name": "campaign_bidding_strategy", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_primary_key": [ @@ -311,7 +440,7 @@ }, { "stream": { - "name": "ad_group_bidding_strategies", + "name": "ad_group_bidding_strategy", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_primary_key": [ @@ -333,35 +462,29 @@ }, { "stream": { - "name": "ad_group_criterions", + "name": "ad_group_criterion", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [ - ["ad_group.id"], - ["ad_group_criterion.criterion_id"] - ] + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["ad_group_criterion.resource_name"]] }, - "sync_mode": "full_refresh", + "sync_mode": "incremental", "destination_sync_mode": "overwrite", - "primary_key": [["ad_group.id"], ["ad_group_criterion.criterion_id"]] + "primary_key": [["ad_group_criterion.resource_name"]] }, { "stream": { - "name": "ad_listing_group_criterions", + "name": "ad_listing_group_criterion", "json_schema": {}, "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [ - ["ad_group.id"], - ["ad_group_criterion.criterion_id"] - ] + "source_defined_primary_key": [["ad_group_criterion.resource_name"]] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite", - "primary_key": [["ad_group.id"], ["ad_group_criterion.criterion_id"]] + "primary_key": [["ad_group_criterion.resource_name"]] }, { "stream": { - "name": "ad_group_criterion_labels", + "name": "ad_group_criterion_label", "json_schema": {}, "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [ @@ -371,6 +494,17 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite", "primary_key": [["ad_group_criterion_label.resource_name"]] + }, + { + "stream": { + "name": "campaign_criterion", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["campaign_criterion.resource_name"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["campaign_criterion.resource_name"]] } ] } diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/conftest.py b/airbyte-integrations/connectors/source-google-ads/integration_tests/conftest.py deleted file mode 100644 index 044e962b5bc4..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/conftest.py +++ /dev/null @@ -1,13 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import json - -import pytest - - -@pytest.fixture(scope="session", name="config") -def config_fixture(): - with open("secrets/config.json", "r") as config_file: - return json.load(config_file) diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records.jsonl index 830126123dfc..875167ce8290 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records.jsonl @@ -1,100 +1,71 @@ -{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "segments.ad_network_type": "SEARCH", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 0, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0, "metrics.content_rank_lost_impression_share": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.date": "2022-04-08", "segments.day_of_week": "FRIDAY", "segments.device": "DESKTOP", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 0, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2022-04-01", "segments.quarter": "2022-04-01", "metrics.search_budget_lost_impression_share": 0.9001, "metrics.search_exact_match_impression_share": 0.0, "metrics.search_impression_share": 0.0999, "metrics.search_rank_lost_impression_share": 0.0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-04-04", "segments.year": 2022}, "emitted_at": 1671617272961} -{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "segments.ad_network_type": "SEARCH", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 0, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0, "metrics.content_rank_lost_impression_share": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.date": "2022-04-08", "segments.day_of_week": "FRIDAY", "segments.device": "MOBILE", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 2, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2022-04-01", "segments.quarter": "2022-04-01", "metrics.search_budget_lost_impression_share": 0.9001, "metrics.search_exact_match_impression_share": 0.0, "metrics.search_impression_share": 0.0999, "metrics.search_rank_lost_impression_share": 0.0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-04-04", "segments.year": 2022}, "emitted_at": 1671617272966} -{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "segments.ad_network_type": "SEARCH", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 0, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0, "metrics.content_rank_lost_impression_share": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.date": "2022-04-09", "segments.day_of_week": "SATURDAY", "segments.device": "DESKTOP", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 28, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2022-04-01", "segments.quarter": "2022-04-01", "metrics.search_budget_lost_impression_share": 0.6618578465869106, "metrics.search_exact_match_impression_share": 0.0999, "metrics.search_impression_share": 0.0999, "metrics.search_rank_lost_impression_share": 0.3282899366643209, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-04-04", "segments.year": 2022}, "emitted_at": 1671617272971} -{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "segments.ad_network_type": "SEARCH", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 70000.0, "metrics.average_cpc": 70000.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 1014492.7536231884, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 1, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0, "metrics.content_rank_lost_impression_share": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 70000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.014492753623188406, "segments.date": "2022-04-09", "segments.day_of_week": "SATURDAY", "segments.device": "MOBILE", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 69, "metrics.interaction_rate": 0.014492753623188406, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 1, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2022-04-01", "segments.quarter": "2022-04-01", "metrics.search_budget_lost_impression_share": 0.6715465465465466, "metrics.search_exact_match_impression_share": 0.0999, "metrics.search_impression_share": 0.0999, "metrics.search_rank_lost_impression_share": 0.30255255255255253, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-04-04", "segments.year": 2022}, "emitted_at": 1671617272975} -{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "segments.ad_network_type": "SEARCH", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 0, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0, "metrics.content_rank_lost_impression_share": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.date": "2022-04-09", "segments.day_of_week": "SATURDAY", "segments.device": "TABLET", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 1, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2022-04-01", "segments.quarter": "2022-04-01", "metrics.search_budget_lost_impression_share": 0.8473282442748091, "metrics.search_exact_match_impression_share": 0.0999, "metrics.search_impression_share": 0.0999, "metrics.search_rank_lost_impression_share": 0.1450381679389313, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-04-04", "segments.year": 2022}, "emitted_at": 1671617272980} -{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "segments.ad_network_type": "SEARCH_PARTNERS", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 0, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0, "metrics.content_rank_lost_impression_share": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.date": "2022-04-09", "segments.day_of_week": "SATURDAY", "segments.device": "DESKTOP", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 15, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2022-04-01", "segments.quarter": "2022-04-01", "metrics.search_budget_lost_impression_share": 0.0, "metrics.search_exact_match_impression_share": 0.0, "metrics.search_impression_share": 0.0, "metrics.search_rank_lost_impression_share": 0.0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-04-04", "segments.year": 2022}, "emitted_at": 1671617272984} -{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "segments.ad_network_type": "SEARCH_PARTNERS", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 0, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0, "metrics.content_rank_lost_impression_share": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.date": "2022-04-09", "segments.day_of_week": "SATURDAY", "segments.device": "MOBILE", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 26, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2022-04-01", "segments.quarter": "2022-04-01", "metrics.search_budget_lost_impression_share": 0.0, "metrics.search_exact_match_impression_share": 0.0, "metrics.search_impression_share": 0.0, "metrics.search_rank_lost_impression_share": 0.0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-04-04", "segments.year": 2022}, "emitted_at": 1671617272987} -{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "segments.ad_network_type": "SEARCH_PARTNERS", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 0, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0, "metrics.content_rank_lost_impression_share": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.date": "2022-04-09", "segments.day_of_week": "SATURDAY", "segments.device": "TABLET", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 3, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2022-04-01", "segments.quarter": "2022-04-01", "metrics.search_budget_lost_impression_share": 0.0, "metrics.search_exact_match_impression_share": 0.0, "metrics.search_impression_share": 0.0, "metrics.search_rank_lost_impression_share": 0.0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-04-04", "segments.year": 2022}, "emitted_at": 1671617272990} -{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "segments.ad_network_type": "SEARCH", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 262000.0, "metrics.average_cpc": 262000.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 93571428.57142857, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 5, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0, "metrics.content_rank_lost_impression_share": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 1310000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.35714285714285715, "segments.date": "2022-04-10", "segments.day_of_week": "SUNDAY", "segments.device": "DESKTOP", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 14, "metrics.interaction_rate": 0.35714285714285715, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 5, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2022-04-01", "segments.quarter": "2022-04-01", "metrics.search_budget_lost_impression_share": 0.6144693003331747, "metrics.search_exact_match_impression_share": 0.12244897959183673, "metrics.search_impression_share": 0.0999, "metrics.search_rank_lost_impression_share": 0.378867206092337, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-04-04", "segments.year": 2022}, "emitted_at": 1671617272992} -{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "segments.ad_network_type": "SEARCH", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 30000.0, "metrics.average_cpc": 30000.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 2195121.951219512, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 3, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0, "metrics.content_rank_lost_impression_share": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 90000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.07317073170731707, "segments.date": "2022-04-10", "segments.day_of_week": "SUNDAY", "segments.device": "MOBILE", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 41, "metrics.interaction_rate": 0.07317073170731707, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 3, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2022-04-01", "segments.quarter": "2022-04-01", "metrics.search_budget_lost_impression_share": 0.628, "metrics.search_exact_match_impression_share": 0.0999, "metrics.search_impression_share": 0.0999, "metrics.search_rank_lost_impression_share": 0.3556, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-04-04", "segments.year": 2022}, "emitted_at": 1671617272994} -{"stream": "geographic_report", "data": {"customer.descriptive_name": "", "geographic_view.country_criterion_id": 2124, "geographic_view.location_type": "LOCATION_OF_PRESENCE", "ad_group.id": 137051662444, "segments.date": "2022-04-08"}, "emitted_at": 1671617280435} -{"stream": "geographic_report", "data": {"customer.descriptive_name": "", "geographic_view.country_criterion_id": 2840, "geographic_view.location_type": "AREA_OF_INTEREST", "ad_group.id": 137020701042, "segments.date": "2022-04-09"}, "emitted_at": 1671617280437} -{"stream": "geographic_report", "data": {"customer.descriptive_name": "", "geographic_view.country_criterion_id": 2124, "geographic_view.location_type": "LOCATION_OF_PRESENCE", "ad_group.id": 137020701042, "segments.date": "2022-04-09"}, "emitted_at": 1671617280438} -{"stream": "geographic_report", "data": {"customer.descriptive_name": "", "geographic_view.country_criterion_id": 2840, "geographic_view.location_type": "LOCATION_OF_PRESENCE", "ad_group.id": 137020701042, "segments.date": "2022-04-09"}, "emitted_at": 1671617280440} -{"stream": "geographic_report", "data": {"customer.descriptive_name": "", "geographic_view.country_criterion_id": 2124, "geographic_view.location_type": "AREA_OF_INTEREST", "ad_group.id": 137051662444, "segments.date": "2022-04-09"}, "emitted_at": 1671617280442} -{"stream": "geographic_report", "data": {"customer.descriptive_name": "", "geographic_view.country_criterion_id": 2840, "geographic_view.location_type": "AREA_OF_INTEREST", "ad_group.id": 137051662444, "segments.date": "2022-04-09"}, "emitted_at": 1671617280443} -{"stream": "geographic_report", "data": {"customer.descriptive_name": "", "geographic_view.country_criterion_id": 2124, "geographic_view.location_type": "LOCATION_OF_PRESENCE", "ad_group.id": 137051662444, "segments.date": "2022-04-09"}, "emitted_at": 1671617280445} -{"stream": "geographic_report", "data": {"customer.descriptive_name": "", "geographic_view.country_criterion_id": 2840, "geographic_view.location_type": "LOCATION_OF_PRESENCE", "ad_group.id": 137051662444, "segments.date": "2022-04-09"}, "emitted_at": 1671617280447} -{"stream": "geographic_report", "data": {"customer.descriptive_name": "", "geographic_view.country_criterion_id": 2840, "geographic_view.location_type": "AREA_OF_INTEREST", "ad_group.id": 137020701042, "segments.date": "2022-04-10"}, "emitted_at": 1671617280448} -{"stream": "geographic_report", "data": {"customer.descriptive_name": "", "geographic_view.country_criterion_id": 2124, "geographic_view.location_type": "LOCATION_OF_PRESENCE", "ad_group.id": 137020701042, "segments.date": "2022-04-10"}, "emitted_at": 1671617280450} -{"stream":"keyword_report","data":{"customer.descriptive_name":"","ad_group.id":123273719655,"ad_group_criterion.type":"KEYWORD","ad_group_criterion.keyword.text":"open source analytics","ad_group_criterion.negative":false,"ad_group_criterion.keyword.match_type":"BROAD","metrics.historical_quality_score":1,"metrics.ctr":0,"segments.date":"2022-02-15","campaign.bidding_strategy_type":"TARGET_SPEND","metrics.clicks":0,"metrics.cost_micros":0,"metrics.impressions":0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0,"metrics.conversions":0,"metrics.conversions_value":0,"metrics.interactions":0,"metrics.interaction_event_types":[],"metrics.view_through_conversions":0,"ad_group_criterion.criterion_id":1732729676},"emitted_at":1688988446821} -{"stream":"keyword_report","data":{"customer.descriptive_name":"","ad_group.id":123273719655,"ad_group_criterion.type":"KEYWORD","ad_group_criterion.keyword.text":"etl pipeline","ad_group_criterion.negative":false,"ad_group_criterion.keyword.match_type":"BROAD","metrics.historical_quality_score":3,"metrics.ctr":0,"segments.date":"2022-02-15","campaign.bidding_strategy_type":"TARGET_SPEND","metrics.clicks":0,"metrics.cost_micros":0,"metrics.impressions":0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0,"metrics.conversions":0,"metrics.conversions_value":0,"metrics.interactions":0,"metrics.interaction_event_types":[],"metrics.view_through_conversions":0,"ad_group_criterion.criterion_id":297585172071},"emitted_at":1688988446824} -{"stream":"keyword_report","data":{"customer.descriptive_name":"","ad_group.id":123273719655,"ad_group_criterion.type":"KEYWORD","ad_group_criterion.keyword.text":"open source etl","ad_group_criterion.negative":false,"ad_group_criterion.keyword.match_type":"BROAD","metrics.historical_quality_score":7,"metrics.ctr":0,"segments.date":"2022-02-15","campaign.bidding_strategy_type":"TARGET_SPEND","metrics.clicks":0,"metrics.cost_micros":0,"metrics.impressions":0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0,"metrics.conversions":0,"metrics.conversions_value":0,"metrics.interactions":0,"metrics.interaction_event_types":[],"metrics.view_through_conversions":0,"ad_group_criterion.criterion_id":298270671583},"emitted_at":1688988446825} -{"stream":"keyword_report","data":{"customer.descriptive_name":"","ad_group.id":123273719655,"ad_group_criterion.type":"KEYWORD","ad_group_criterion.keyword.text":"data connectors","ad_group_criterion.negative":false,"ad_group_criterion.keyword.match_type":"BROAD","metrics.historical_quality_score":5,"metrics.ctr":0,"segments.date":"2022-02-15","campaign.bidding_strategy_type":"TARGET_SPEND","metrics.clicks":0,"metrics.cost_micros":0,"metrics.impressions":0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0,"metrics.conversions":0,"metrics.conversions_value":0,"metrics.interactions":0,"metrics.interaction_event_types":[],"metrics.view_through_conversions":0,"ad_group_criterion.criterion_id":303376179543},"emitted_at":1688988446826} -{"stream":"keyword_report","data":{"customer.descriptive_name":"","ad_group.id":123273719655,"ad_group_criterion.type":"KEYWORD","ad_group_criterion.keyword.text":"open source analytics","ad_group_criterion.negative":false,"ad_group_criterion.keyword.match_type":"BROAD","metrics.historical_quality_score":1,"metrics.ctr":0,"segments.date":"2022-02-16","campaign.bidding_strategy_type":"TARGET_SPEND","metrics.clicks":0,"metrics.cost_micros":0,"metrics.impressions":0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0,"metrics.conversions":0,"metrics.conversions_value":0,"metrics.interactions":0,"metrics.interaction_event_types":[],"metrics.view_through_conversions":0,"ad_group_criterion.criterion_id":1732729676},"emitted_at":1688988446827} -{"stream":"keyword_report","data":{"customer.descriptive_name":"","ad_group.id":123273719655,"ad_group_criterion.type":"KEYWORD","ad_group_criterion.keyword.text":"etl pipeline","ad_group_criterion.negative":false,"ad_group_criterion.keyword.match_type":"BROAD","metrics.historical_quality_score":3,"metrics.ctr":0,"segments.date":"2022-02-16","campaign.bidding_strategy_type":"TARGET_SPEND","metrics.clicks":0,"metrics.cost_micros":0,"metrics.impressions":0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0,"metrics.conversions":0,"metrics.conversions_value":0,"metrics.interactions":0,"metrics.interaction_event_types":[],"metrics.view_through_conversions":0,"ad_group_criterion.criterion_id":297585172071},"emitted_at":1688988446827} -{"stream":"keyword_report","data":{"customer.descriptive_name":"","ad_group.id":123273719655,"ad_group_criterion.type":"KEYWORD","ad_group_criterion.keyword.text":"open source etl","ad_group_criterion.negative":false,"ad_group_criterion.keyword.match_type":"BROAD","metrics.historical_quality_score":7,"metrics.ctr":0,"segments.date":"2022-02-16","campaign.bidding_strategy_type":"TARGET_SPEND","metrics.clicks":0,"metrics.cost_micros":0,"metrics.impressions":0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0,"metrics.conversions":0,"metrics.conversions_value":0,"metrics.interactions":0,"metrics.interaction_event_types":[],"metrics.view_through_conversions":0,"ad_group_criterion.criterion_id":298270671583},"emitted_at":1688988446828} -{"stream":"keyword_report","data":{"customer.descriptive_name":"","ad_group.id":123273719655,"ad_group_criterion.type":"KEYWORD","ad_group_criterion.keyword.text":"data connectors","ad_group_criterion.negative":false,"ad_group_criterion.keyword.match_type":"BROAD","metrics.historical_quality_score":5,"metrics.ctr":0,"segments.date":"2022-02-16","campaign.bidding_strategy_type":"TARGET_SPEND","metrics.clicks":0,"metrics.cost_micros":0,"metrics.impressions":0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0,"metrics.conversions":0,"metrics.conversions_value":0,"metrics.interactions":0,"metrics.interaction_event_types":[],"metrics.view_through_conversions":0,"ad_group_criterion.criterion_id":303376179543},"emitted_at":1688988446829} -{"stream":"keyword_report","data":{"customer.descriptive_name":"","ad_group.id":123273719655,"ad_group_criterion.type":"KEYWORD","ad_group_criterion.keyword.text":"open source analytics","ad_group_criterion.negative":false,"ad_group_criterion.keyword.match_type":"BROAD","metrics.historical_quality_score":1,"metrics.ctr":0,"segments.date":"2022-02-17","campaign.bidding_strategy_type":"TARGET_SPEND","metrics.clicks":0,"metrics.cost_micros":0,"metrics.impressions":0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0,"metrics.conversions":0,"metrics.conversions_value":0,"metrics.interactions":0,"metrics.interaction_event_types":[],"metrics.view_through_conversions":0,"ad_group_criterion.criterion_id":1732729676},"emitted_at":1688988446829} -{"stream":"keyword_report","data":{"customer.descriptive_name":"","ad_group.id":137020701042,"ad_group_criterion.type":"KEYWORD","ad_group_criterion.keyword.text":"airbytes","ad_group_criterion.negative":false,"ad_group_criterion.keyword.match_type":"BROAD","metrics.historical_quality_score":10,"metrics.ctr":0.06666666666666667,"segments.date":"2022-05-14","campaign.bidding_strategy_type":"MAXIMIZE_CONVERSIONS","metrics.clicks":2,"metrics.cost_micros":50000,"metrics.impressions":30,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0,"metrics.conversions":0,"metrics.conversions_value":0,"metrics.interactions":2,"metrics.interaction_event_types":["InteractionEventType.CLICK"],"metrics.view_through_conversions":0,"ad_group_criterion.criterion_id":423065099654},"emitted_at":1688988450597} -{"stream": "display_keyword_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 10012000.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 1, "metrics.active_view_measurability": 1.0, "metrics.active_view_measurable_cost_micros": 10012, "metrics.active_view_measurable_impressions": 1, "metrics.active_view_viewability": 1.0, "ad_group.id": 143992182864, "ad_group.name": "Video Non-skippable - 2022-05-30", "ad_group.status": "ENABLED", "segments.ad_network_type": "YOUTUBE_WATCH", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 10012000.0, "metrics.average_cpv": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/143992182864", "campaign.base_campaign": "customers/4651612872/campaigns/17354032686", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "TARGET_CPM", "campaign.id": 17354032686, "campaign.name": "Video Non-skippable - 2022-05-30", "campaign.status": "ENABLED", "metrics.clicks": 0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 10012, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "ad_group_criterion.effective_cpc_bid_micros": 10000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 10000, "ad_group_criterion.effective_cpv_bid_source": "AD_GROUP", "ad_group_criterion.keyword.text": "big data software", "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.day_of_week": "TUESDAY", "segments.device": "MOBILE", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_urls": [], "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_criterion.criterion_id": 26160872903, "metrics.impressions": 1, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "ad_group_criterion.negative": false, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n", "targeting_dimension: TOPIC\nbid_only: false\n"], "segments.month": "2022-05-01", "segments.quarter": "2022-04-01", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.url_custom_parameters": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-05-30", "segments.year": 2022, "segments.date": "2022-05-31"}, "emitted_at": 1671617298755} -{"stream": "campaign_labels", "data": {"campaign.resource_name": "customers/4651612872/campaigns/12124071339", "campaign_label.resource_name": "customers/4651612872/campaignLabels/12124071339~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1671617384908} -{"stream": "campaign_labels", "data": {"campaign.resource_name": "customers/4651612872/campaigns/13284356762", "campaign_label.resource_name": "customers/4651612872/campaignLabels/13284356762~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1671617384909} -{"stream": "ad_group_labels", "data": {"ad_group.resource_name": "customers/4651612872/adGroups/123273719655", "ad_group_label.resource_name": "customers/4651612872/adGroupLabels/123273719655~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1671617385395} -{"stream": "ad_group_ad_labels", "data": {"ad_group_ad.ad.resource_name": "customers/4651612872/ads/524518584182", "ad_group_ad_label.resource_name": "customers/4651612872/adGroupAdLabels/123273719655~524518584182~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1671617386108} -{"stream": "user_location_report", "data": {"segments.date": "2022-04-08", "segments.day_of_week": "FRIDAY", "segments.month": "2022-04-01", "segments.week": "2022-04-04", "segments.quarter": "2022-04-01", "segments.year": 2022, "segments.ad_network_type": "SEARCH", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2124, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2124~true", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a01", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137051662444", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.clicks": 0, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.impressions": 2, "metrics.interaction_event_types": [], "metrics.interaction_rate": 0.0, "metrics.interactions": 0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1671617390491} -{"stream": "user_location_report", "data": {"segments.date": "2022-04-09", "segments.day_of_week": "SATURDAY", "segments.month": "2022-04-01", "segments.week": "2022-04-04", "segments.quarter": "2022-04-01", "segments.year": 2022, "segments.ad_network_type": "SEARCH", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2124, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2124~true", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 70000.0, "metrics.average_cpc": 70000.0, "metrics.average_cpm": 3684210.5263157897, "metrics.average_cpv": 0.0, "metrics.clicks": 1, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 70000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.05263157894736842, "metrics.impressions": 19, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interaction_rate": 0.05263157894736842, "metrics.interactions": 1, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1671617390495} -{"stream": "user_location_report", "data": {"segments.date": "2022-04-09", "segments.day_of_week": "SATURDAY", "segments.month": "2022-04-01", "segments.week": "2022-04-04", "segments.quarter": "2022-04-01", "segments.year": 2022, "segments.ad_network_type": "SEARCH", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2840, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2840~true", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.clicks": 0, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.impressions": 3, "metrics.interaction_event_types": [], "metrics.interaction_rate": 0.0, "metrics.interactions": 0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1671617390499} -{"stream": "user_location_report", "data": {"segments.date": "2022-04-09", "segments.day_of_week": "SATURDAY", "segments.month": "2022-04-01", "segments.week": "2022-04-04", "segments.quarter": "2022-04-01", "segments.year": 2022, "segments.ad_network_type": "SEARCH", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2051, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2051~false", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a01", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137051662444", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.clicks": 0, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.impressions": 1, "metrics.interaction_event_types": [], "metrics.interaction_rate": 0.0, "metrics.interactions": 0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1671617390503} -{"stream": "user_location_report", "data": {"segments.date": "2022-04-09", "segments.day_of_week": "SATURDAY", "segments.month": "2022-04-01", "segments.week": "2022-04-04", "segments.quarter": "2022-04-01", "segments.year": 2022, "segments.ad_network_type": "SEARCH", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2170, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2170~false", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a01", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137051662444", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.clicks": 0, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.impressions": 2, "metrics.interaction_event_types": [], "metrics.interaction_rate": 0.0, "metrics.interactions": 0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1671617390507} -{"stream": "user_location_report", "data": {"segments.date": "2022-04-09", "segments.day_of_week": "SATURDAY", "segments.month": "2022-04-01", "segments.week": "2022-04-04", "segments.quarter": "2022-04-01", "segments.year": 2022, "segments.ad_network_type": "SEARCH", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2218, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2218~false", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a01", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137051662444", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.clicks": 0, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.impressions": 1, "metrics.interaction_event_types": [], "metrics.interaction_rate": 0.0, "metrics.interactions": 0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1671617390511} -{"stream": "user_location_report", "data": {"segments.date": "2022-04-09", "segments.day_of_week": "SATURDAY", "segments.month": "2022-04-01", "segments.week": "2022-04-04", "segments.quarter": "2022-04-01", "segments.year": 2022, "segments.ad_network_type": "SEARCH", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2356, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2356~false", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a01", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137051662444", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.clicks": 0, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.impressions": 1, "metrics.interaction_event_types": [], "metrics.interaction_rate": 0.0, "metrics.interactions": 0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1671617390514} -{"stream": "user_location_report", "data": {"segments.date": "2022-04-09", "segments.day_of_week": "SATURDAY", "segments.month": "2022-04-01", "segments.week": "2022-04-04", "segments.quarter": "2022-04-01", "segments.year": 2022, "segments.ad_network_type": "SEARCH", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2484, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2484~false", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a01", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137051662444", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.clicks": 0, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.impressions": 4, "metrics.interaction_event_types": [], "metrics.interaction_rate": 0.0, "metrics.interactions": 0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1671617390516} -{"stream": "user_location_report", "data": {"segments.date": "2022-04-09", "segments.day_of_week": "SATURDAY", "segments.month": "2022-04-01", "segments.week": "2022-04-04", "segments.quarter": "2022-04-01", "segments.year": 2022, "segments.ad_network_type": "SEARCH", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2504, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2504~false", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a01", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137051662444", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.clicks": 0, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.impressions": 1, "metrics.interaction_event_types": [], "metrics.interaction_rate": 0.0, "metrics.interactions": 0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1671617390518} -{"stream": "user_location_report", "data": {"segments.date": "2022-04-09", "segments.day_of_week": "SATURDAY", "segments.month": "2022-04-01", "segments.week": "2022-04-04", "segments.quarter": "2022-04-01", "segments.year": 2022, "segments.ad_network_type": "SEARCH", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2524, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2524~false", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a01", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137051662444", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.clicks": 0, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.impressions": 2, "metrics.interaction_event_types": [], "metrics.interaction_rate": 0.0, "metrics.interactions": 0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1671617390520} -{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2022-04-08", "campaign.end_date": "2037-12-30", "segments.date": "2022-04-08"}, "emitted_at": 1671617397632} -{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2022-04-08", "campaign.end_date": "2037-12-30", "segments.date": "2022-04-09"}, "emitted_at": 1671617397634} -{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2022-04-08", "campaign.end_date": "2037-12-30", "segments.date": "2022-04-10"}, "emitted_at": 1671617397634} -{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2022-04-08", "campaign.end_date": "2037-12-30", "segments.date": "2022-04-11"}, "emitted_at": 1671617397635} -{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2022-04-08", "campaign.end_date": "2037-12-30", "segments.date": "2022-04-12"}, "emitted_at": 1671617397635} -{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2022-04-08", "campaign.end_date": "2037-12-30", "segments.date": "2022-04-13"}, "emitted_at": 1671617397635} -{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2022-04-08", "campaign.end_date": "2037-12-30", "segments.date": "2022-04-14"}, "emitted_at": 1671617397636} -{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2022-04-08", "campaign.end_date": "2037-12-30", "segments.date": "2022-04-15"}, "emitted_at": 1671617397636} -{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2022-04-08", "campaign.end_date": "2037-12-30", "segments.date": "2022-04-16"}, "emitted_at": 1671617397636} -{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "WEBSITE", "campaign.start_date": "2022-04-08", "campaign.end_date": "2037-12-30", "segments.date": "2022-04-09"}, "emitted_at": 1671617397637} -{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2022-04-08"}, "emitted_at": 1671617407844} -{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2022-04-09"}, "emitted_at": 1671617407844} -{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2022-04-09"}, "emitted_at": 1671617407844} -{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2022-04-10"}, "emitted_at": 1671617407845} -{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2022-04-10"}, "emitted_at": 1671617407845} -{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2022-04-11"}, "emitted_at": 1671617407846} -{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2022-04-11"}, "emitted_at": 1671617407846} -{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2022-04-12"}, "emitted_at": 1671617407846} -{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2022-04-12"}, "emitted_at": 1671617407846} -{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2022-04-13"}, "emitted_at": 1671617407847} -{"stream":"custom_audience","data":{"custom_audience.description":"","custom_audience.name":"Airbyet","custom_audience.id":523469909,"custom_audience.members":["member_type: KEYWORD\nkeyword: \"etl elt\"\n","member_type: KEYWORD\nkeyword: \"cloud data management and analytics\"\n","member_type: KEYWORD\nkeyword: \"data integration\"\n","member_type: KEYWORD\nkeyword: \"big data analytics database\"\n","member_type: KEYWORD\nkeyword: \"data\"\n","member_type: KEYWORD\nkeyword: \"data sherid nada\"\n","member_type: KEYWORD\nkeyword: \"airbyteforeveryone\"\n","member_type: KEYWORD\nkeyword: \"Airbyte\"\n"],"custom_audience.resource_name":"customers/4651612872/customAudiences/523469909","custom_audience.status":"ENABLED","custom_audience.type":"AUTO"},"emitted_at":1676550195853} -{"stream":"ad_group_ad_report","data":{"ad_group_ad.ad.legacy_responsive_display_ad.accent_color":"","ad_group.id":137051662444,"customer.currency_code":"USD","customer.descriptive_name":"","customer.time_zone":"America/Los_Angeles","metrics.active_view_cpm":0.0,"metrics.active_view_ctr":0.0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0.0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0.0,"ad_group_ad.ad_group":"customers/4651612872/adGroups/137051662444","ad_group.name":"Группа объявлений 1","ad_group.status":"ENABLED","segments.ad_network_type":"SEARCH","ad_group_ad.ad_strength":"POOR","ad_group_ad.ad.type":"RESPONSIVE_SEARCH_AD","metrics.all_conversions_from_interactions_rate":0.0,"metrics.all_conversions_value":0.0,"metrics.all_conversions":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.added_by_google_ads":false,"metrics.average_cost":0.0,"metrics.average_cpc":0.0,"metrics.average_cpe":0.0,"metrics.average_cpm":0.0,"metrics.average_cpv":0.0,"metrics.average_page_views":0.0,"metrics.average_time_on_site":0.0,"ad_group.base_ad_group":"customers/4651612872/adGroups/137051662444","campaign.base_campaign":"customers/4651612872/campaigns/16820250687","metrics.bounce_rate":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.business_name":"","ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text":"","campaign.id":16820250687,"campaign.name":"Website traffic-Search-15","campaign.status":"PAUSED","metrics.clicks":0,"ad_group_ad.policy_summary.approval_status":"APPROVED","metrics.conversions_from_interactions_rate":0.0,"metrics.conversions_value":0.0,"metrics.conversions":0.0,"metrics.cost_micros":0,"metrics.cost_per_all_conversions":0.0,"metrics.cost_per_conversion":0.0,"metrics.cost_per_current_model_attributed_conversion":0.0,"ad_group_ad.ad.final_mobile_urls":[],"ad_group_ad.ad.final_urls":["https://airbyte.com"],"ad_group_ad.ad.tracking_url_template":"","ad_group_ad.ad.url_custom_parameters":[],"metrics.cross_device_conversions":0.0,"metrics.ctr":0.0,"metrics.current_model_attributed_conversions_value":0.0,"metrics.current_model_attributed_conversions":0.0,"segments.date":"2022-04-08","segments.day_of_week":"FRIDAY","ad_group_ad.ad.expanded_text_ad.description":"","ad_group_ad.ad.text_ad.description1":"","ad_group_ad.ad.text_ad.description2":"","ad_group_ad.ad.device_preference":"UNSPECIFIED","ad_group_ad.ad.display_url":"","metrics.engagement_rate":0.0,"metrics.engagements":0,"ad_group_ad.ad.legacy_responsive_display_ad.logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.marketing_image":"","ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image":"","ad_group_ad.ad.expanded_dynamic_search_ad.description":"","ad_group_ad.ad.expanded_text_ad.description2":"","ad_group_ad.ad.expanded_text_ad.headline_part3":"","customer.id":4651612872,"ad_group_ad.ad.legacy_responsive_display_ad.format_setting":"UNSPECIFIED","metrics.gmail_forwards":0,"metrics.gmail_saves":0,"metrics.gmail_secondary_clicks":0,"ad_group_ad.ad.text_ad.headline":"","ad_group_ad.ad.expanded_text_ad.headline_part1":"","ad_group_ad.ad.expanded_text_ad.headline_part2":"","ad_group_ad.ad.id":592078676857,"ad_group_ad.ad.image_ad.image_url":"","ad_group_ad.ad.image_ad.pixel_height":0,"ad_group_ad.ad.image_ad.pixel_width":0,"ad_group_ad.ad.image_ad.mime_type":"UNSPECIFIED","ad_group_ad.ad.image_ad.name":"","metrics.impressions":2,"metrics.interaction_rate":0.0,"metrics.interaction_event_types":[],"metrics.interactions":0,"ad_group_ad.ad.legacy_responsive_display_ad.long_headline":"","ad_group_ad.ad.legacy_responsive_display_ad.main_color":"","segments.month":"2022-04-01","ad_group_ad.ad.responsive_display_ad.accent_color":"","ad_group_ad.ad.responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.responsive_display_ad.business_name":"","ad_group_ad.ad.responsive_display_ad.call_to_action_text":"","ad_group_ad.ad.responsive_display_ad.descriptions":[],"ad_group_ad.ad.responsive_display_ad.price_prefix":"","ad_group_ad.ad.responsive_display_ad.promo_text":"","ad_group_ad.ad.responsive_display_ad.format_setting":"UNSPECIFIED","ad_group_ad.ad.responsive_display_ad.headlines":[],"ad_group_ad.ad.responsive_display_ad.logo_images":[],"ad_group_ad.ad.responsive_display_ad.square_logo_images":[],"ad_group_ad.ad.responsive_display_ad.long_headline":"","ad_group_ad.ad.responsive_display_ad.main_color":"","ad_group_ad.ad.responsive_display_ad.marketing_images":[],"ad_group_ad.ad.responsive_display_ad.square_marketing_images":[],"ad_group_ad.ad.responsive_display_ad.youtube_videos":[],"ad_group_ad.ad.expanded_text_ad.path1":"","ad_group_ad.ad.expanded_text_ad.path2":"","metrics.percent_new_visitors":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.price_prefix":"","ad_group_ad.ad.legacy_responsive_display_ad.promo_text":"","segments.quarter":"2022-04-01","ad_group_ad.ad.responsive_search_ad.descriptions":["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.headlines":["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.path1":"","ad_group_ad.ad.responsive_search_ad.path2":"","ad_group_ad.ad.legacy_responsive_display_ad.short_headline":"","ad_group_ad.status":"REMOVED","ad_group_ad.ad.system_managed_resource_source":"UNSPECIFIED","metrics.top_impression_percentage":0.0,"ad_group_ad.ad.app_ad.descriptions":[],"ad_group_ad.ad.app_ad.headlines":[],"ad_group_ad.ad.app_ad.html5_media_bundles":[],"ad_group_ad.ad.app_ad.images":[],"ad_group_ad.ad.app_ad.mandatory_ad_text":"","ad_group_ad.ad.app_ad.youtube_videos":[],"metrics.value_per_all_conversions":0.0,"metrics.value_per_conversion":0.0,"metrics.value_per_current_model_attributed_conversion":0.0,"metrics.video_quartile_p100_rate":0.0,"metrics.video_quartile_p25_rate":0.0,"metrics.video_quartile_p50_rate":0.0,"metrics.video_quartile_p75_rate":0.0,"metrics.video_view_rate":0.0,"metrics.video_views":0,"metrics.view_through_conversions":0,"segments.week":"2022-04-04","segments.year":2022},"emitted_at":1679508005462} -{"stream":"ad_group_ad_report","data":{"ad_group_ad.ad.legacy_responsive_display_ad.accent_color":"","ad_group.id":137020701042,"customer.currency_code":"USD","customer.descriptive_name":"","customer.time_zone":"America/Los_Angeles","metrics.active_view_cpm":0.0,"metrics.active_view_ctr":0.0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0.0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0.0,"ad_group_ad.ad_group":"customers/4651612872/adGroups/137020701042","ad_group.name":"Группа объявлений 2","ad_group.status":"ENABLED","segments.ad_network_type":"SEARCH","ad_group_ad.ad_strength":"POOR","ad_group_ad.ad.type":"RESPONSIVE_SEARCH_AD","metrics.all_conversions_from_interactions_rate":0.0,"metrics.all_conversions_value":0.0,"metrics.all_conversions":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.added_by_google_ads":false,"metrics.average_cost":70000.0,"metrics.average_cpc":70000.0,"metrics.average_cpe":0.0,"metrics.average_cpm":3181818.181818182,"metrics.average_cpv":0.0,"metrics.average_page_views":0.0,"metrics.average_time_on_site":0.0,"ad_group.base_ad_group":"customers/4651612872/adGroups/137020701042","campaign.base_campaign":"customers/4651612872/campaigns/16820250687","metrics.bounce_rate":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.business_name":"","ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text":"","campaign.id":16820250687,"campaign.name":"Website traffic-Search-15","campaign.status":"PAUSED","metrics.clicks":1,"ad_group_ad.policy_summary.approval_status":"APPROVED_LIMITED","metrics.conversions_from_interactions_rate":0.0,"metrics.conversions_value":0.0,"metrics.conversions":0.0,"metrics.cost_micros":70000,"metrics.cost_per_all_conversions":0.0,"metrics.cost_per_conversion":0.0,"metrics.cost_per_current_model_attributed_conversion":0.0,"ad_group_ad.ad.final_mobile_urls":[],"ad_group_ad.ad.final_urls":["https://airbyte.com"],"ad_group_ad.ad.tracking_url_template":"","ad_group_ad.ad.url_custom_parameters":[],"metrics.cross_device_conversions":0.0,"metrics.ctr":0.045454545454545456,"metrics.current_model_attributed_conversions_value":0.0,"metrics.current_model_attributed_conversions":0.0,"segments.date":"2022-04-09","segments.day_of_week":"SATURDAY","ad_group_ad.ad.expanded_text_ad.description":"","ad_group_ad.ad.text_ad.description1":"","ad_group_ad.ad.text_ad.description2":"","ad_group_ad.ad.device_preference":"UNSPECIFIED","ad_group_ad.ad.display_url":"","metrics.engagement_rate":0.0,"metrics.engagements":0,"ad_group_ad.ad.legacy_responsive_display_ad.logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.marketing_image":"","ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image":"","ad_group_ad.ad.expanded_dynamic_search_ad.description":"","ad_group_ad.ad.expanded_text_ad.description2":"","ad_group_ad.ad.expanded_text_ad.headline_part3":"","customer.id":4651612872,"ad_group_ad.ad.legacy_responsive_display_ad.format_setting":"UNSPECIFIED","metrics.gmail_forwards":0,"metrics.gmail_saves":0,"metrics.gmail_secondary_clicks":0,"ad_group_ad.ad.text_ad.headline":"","ad_group_ad.ad.expanded_text_ad.headline_part1":"","ad_group_ad.ad.expanded_text_ad.headline_part2":"","ad_group_ad.ad.id":592078631218,"ad_group_ad.ad.image_ad.image_url":"","ad_group_ad.ad.image_ad.pixel_height":0,"ad_group_ad.ad.image_ad.pixel_width":0,"ad_group_ad.ad.image_ad.mime_type":"UNSPECIFIED","ad_group_ad.ad.image_ad.name":"","metrics.impressions":22,"metrics.interaction_rate":0.045454545454545456,"metrics.interaction_event_types":["InteractionEventType.CLICK"],"metrics.interactions":1,"ad_group_ad.ad.legacy_responsive_display_ad.long_headline":"","ad_group_ad.ad.legacy_responsive_display_ad.main_color":"","segments.month":"2022-04-01","ad_group_ad.ad.responsive_display_ad.accent_color":"","ad_group_ad.ad.responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.responsive_display_ad.business_name":"","ad_group_ad.ad.responsive_display_ad.call_to_action_text":"","ad_group_ad.ad.responsive_display_ad.descriptions":[],"ad_group_ad.ad.responsive_display_ad.price_prefix":"","ad_group_ad.ad.responsive_display_ad.promo_text":"","ad_group_ad.ad.responsive_display_ad.format_setting":"UNSPECIFIED","ad_group_ad.ad.responsive_display_ad.headlines":[],"ad_group_ad.ad.responsive_display_ad.logo_images":[],"ad_group_ad.ad.responsive_display_ad.square_logo_images":[],"ad_group_ad.ad.responsive_display_ad.long_headline":"","ad_group_ad.ad.responsive_display_ad.main_color":"","ad_group_ad.ad.responsive_display_ad.marketing_images":[],"ad_group_ad.ad.responsive_display_ad.square_marketing_images":[],"ad_group_ad.ad.responsive_display_ad.youtube_videos":[],"ad_group_ad.ad.expanded_text_ad.path1":"","ad_group_ad.ad.expanded_text_ad.path2":"","metrics.percent_new_visitors":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.price_prefix":"","ad_group_ad.ad.legacy_responsive_display_ad.promo_text":"","segments.quarter":"2022-04-01","ad_group_ad.ad.responsive_search_ad.descriptions":["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n policy_topic_entries {\n topic: \"TRADEMARKS_IN_AD_TEXT\"\n type_: LIMITED\n evidences {\n text_list {\n texts: \"airbyte\"\n }\n }\n constraints {\n reseller_constraint {\n }\n }\n }\n review_status: REVIEWED\n approval_status: APPROVED_LIMITED\n}\n","text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.headlines":["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.path1":"","ad_group_ad.ad.responsive_search_ad.path2":"","ad_group_ad.ad.legacy_responsive_display_ad.short_headline":"","ad_group_ad.status":"ENABLED","ad_group_ad.ad.system_managed_resource_source":"UNSPECIFIED","metrics.top_impression_percentage":0.7727272727272727,"ad_group_ad.ad.app_ad.descriptions":[],"ad_group_ad.ad.app_ad.headlines":[],"ad_group_ad.ad.app_ad.html5_media_bundles":[],"ad_group_ad.ad.app_ad.images":[],"ad_group_ad.ad.app_ad.mandatory_ad_text":"","ad_group_ad.ad.app_ad.youtube_videos":[],"metrics.value_per_all_conversions":0.0,"metrics.value_per_conversion":0.0,"metrics.value_per_current_model_attributed_conversion":0.0,"metrics.video_quartile_p100_rate":0.0,"metrics.video_quartile_p25_rate":0.0,"metrics.video_quartile_p50_rate":0.0,"metrics.video_quartile_p75_rate":0.0,"metrics.video_view_rate":0.0,"metrics.video_views":0,"metrics.view_through_conversions":0,"segments.week":"2022-04-04","segments.year":2022},"emitted_at":1679508005467} -{"stream":"ad_group_ad_report","data":{"ad_group_ad.ad.legacy_responsive_display_ad.accent_color":"","ad_group.id":137051662444,"customer.currency_code":"USD","customer.descriptive_name":"","customer.time_zone":"America/Los_Angeles","metrics.active_view_cpm":0.0,"metrics.active_view_ctr":0.0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0.0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0.0,"ad_group_ad.ad_group":"customers/4651612872/adGroups/137051662444","ad_group.name":"Группа объявлений 1","ad_group.status":"ENABLED","segments.ad_network_type":"SEARCH","ad_group_ad.ad_strength":"POOR","ad_group_ad.ad.type":"RESPONSIVE_SEARCH_AD","metrics.all_conversions_from_interactions_rate":0.0,"metrics.all_conversions_value":0.0,"metrics.all_conversions":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.added_by_google_ads":false,"metrics.average_cost":0.0,"metrics.average_cpc":0.0,"metrics.average_cpe":0.0,"metrics.average_cpm":0.0,"metrics.average_cpv":0.0,"metrics.average_page_views":0.0,"metrics.average_time_on_site":0.0,"ad_group.base_ad_group":"customers/4651612872/adGroups/137051662444","campaign.base_campaign":"customers/4651612872/campaigns/16820250687","metrics.bounce_rate":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.business_name":"","ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text":"","campaign.id":16820250687,"campaign.name":"Website traffic-Search-15","campaign.status":"PAUSED","metrics.clicks":0,"ad_group_ad.policy_summary.approval_status":"APPROVED","metrics.conversions_from_interactions_rate":0.0,"metrics.conversions_value":0.0,"metrics.conversions":0.0,"metrics.cost_micros":0,"metrics.cost_per_all_conversions":0.0,"metrics.cost_per_conversion":0.0,"metrics.cost_per_current_model_attributed_conversion":0.0,"ad_group_ad.ad.final_mobile_urls":[],"ad_group_ad.ad.final_urls":["https://airbyte.com"],"ad_group_ad.ad.tracking_url_template":"","ad_group_ad.ad.url_custom_parameters":[],"metrics.cross_device_conversions":0.0,"metrics.ctr":0.0,"metrics.current_model_attributed_conversions_value":0.0,"metrics.current_model_attributed_conversions":0.0,"segments.date":"2022-04-09","segments.day_of_week":"SATURDAY","ad_group_ad.ad.expanded_text_ad.description":"","ad_group_ad.ad.text_ad.description1":"","ad_group_ad.ad.text_ad.description2":"","ad_group_ad.ad.device_preference":"UNSPECIFIED","ad_group_ad.ad.display_url":"","metrics.engagement_rate":0.0,"metrics.engagements":0,"ad_group_ad.ad.legacy_responsive_display_ad.logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.marketing_image":"","ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image":"","ad_group_ad.ad.expanded_dynamic_search_ad.description":"","ad_group_ad.ad.expanded_text_ad.description2":"","ad_group_ad.ad.expanded_text_ad.headline_part3":"","customer.id":4651612872,"ad_group_ad.ad.legacy_responsive_display_ad.format_setting":"UNSPECIFIED","metrics.gmail_forwards":0,"metrics.gmail_saves":0,"metrics.gmail_secondary_clicks":0,"ad_group_ad.ad.text_ad.headline":"","ad_group_ad.ad.expanded_text_ad.headline_part1":"","ad_group_ad.ad.expanded_text_ad.headline_part2":"","ad_group_ad.ad.id":592078676857,"ad_group_ad.ad.image_ad.image_url":"","ad_group_ad.ad.image_ad.pixel_height":0,"ad_group_ad.ad.image_ad.pixel_width":0,"ad_group_ad.ad.image_ad.mime_type":"UNSPECIFIED","ad_group_ad.ad.image_ad.name":"","metrics.impressions":76,"metrics.interaction_rate":0.0,"metrics.interaction_event_types":[],"metrics.interactions":0,"ad_group_ad.ad.legacy_responsive_display_ad.long_headline":"","ad_group_ad.ad.legacy_responsive_display_ad.main_color":"","segments.month":"2022-04-01","ad_group_ad.ad.responsive_display_ad.accent_color":"","ad_group_ad.ad.responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.responsive_display_ad.business_name":"","ad_group_ad.ad.responsive_display_ad.call_to_action_text":"","ad_group_ad.ad.responsive_display_ad.descriptions":[],"ad_group_ad.ad.responsive_display_ad.price_prefix":"","ad_group_ad.ad.responsive_display_ad.promo_text":"","ad_group_ad.ad.responsive_display_ad.format_setting":"UNSPECIFIED","ad_group_ad.ad.responsive_display_ad.headlines":[],"ad_group_ad.ad.responsive_display_ad.logo_images":[],"ad_group_ad.ad.responsive_display_ad.square_logo_images":[],"ad_group_ad.ad.responsive_display_ad.long_headline":"","ad_group_ad.ad.responsive_display_ad.main_color":"","ad_group_ad.ad.responsive_display_ad.marketing_images":[],"ad_group_ad.ad.responsive_display_ad.square_marketing_images":[],"ad_group_ad.ad.responsive_display_ad.youtube_videos":[],"ad_group_ad.ad.expanded_text_ad.path1":"","ad_group_ad.ad.expanded_text_ad.path2":"","metrics.percent_new_visitors":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.price_prefix":"","ad_group_ad.ad.legacy_responsive_display_ad.promo_text":"","segments.quarter":"2022-04-01","ad_group_ad.ad.responsive_search_ad.descriptions":["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.headlines":["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.path1":"","ad_group_ad.ad.responsive_search_ad.path2":"","ad_group_ad.ad.legacy_responsive_display_ad.short_headline":"","ad_group_ad.status":"REMOVED","ad_group_ad.ad.system_managed_resource_source":"UNSPECIFIED","metrics.top_impression_percentage":0.2894736842105263,"ad_group_ad.ad.app_ad.descriptions":[],"ad_group_ad.ad.app_ad.headlines":[],"ad_group_ad.ad.app_ad.html5_media_bundles":[],"ad_group_ad.ad.app_ad.images":[],"ad_group_ad.ad.app_ad.mandatory_ad_text":"","ad_group_ad.ad.app_ad.youtube_videos":[],"metrics.value_per_all_conversions":0.0,"metrics.value_per_conversion":0.0,"metrics.value_per_current_model_attributed_conversion":0.0,"metrics.video_quartile_p100_rate":0.0,"metrics.video_quartile_p25_rate":0.0,"metrics.video_quartile_p50_rate":0.0,"metrics.video_quartile_p75_rate":0.0,"metrics.video_view_rate":0.0,"metrics.video_views":0,"metrics.view_through_conversions":0,"segments.week":"2022-04-04","segments.year":2022},"emitted_at":1679508005471} -{"stream":"ad_group_ad_report","data":{"ad_group_ad.ad.legacy_responsive_display_ad.accent_color":"","ad_group.id":137020701042,"customer.currency_code":"USD","customer.descriptive_name":"","customer.time_zone":"America/Los_Angeles","metrics.active_view_cpm":0.0,"metrics.active_view_ctr":0.0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0.0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0.0,"ad_group_ad.ad_group":"customers/4651612872/adGroups/137020701042","ad_group.name":"Группа объявлений 2","ad_group.status":"ENABLED","segments.ad_network_type":"SEARCH_PARTNERS","ad_group_ad.ad_strength":"POOR","ad_group_ad.ad.type":"RESPONSIVE_SEARCH_AD","metrics.all_conversions_from_interactions_rate":0.0,"metrics.all_conversions_value":0.0,"metrics.all_conversions":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.added_by_google_ads":false,"metrics.average_cost":0.0,"metrics.average_cpc":0.0,"metrics.average_cpe":0.0,"metrics.average_cpm":0.0,"metrics.average_cpv":0.0,"metrics.average_page_views":0.0,"metrics.average_time_on_site":0.0,"ad_group.base_ad_group":"customers/4651612872/adGroups/137020701042","campaign.base_campaign":"customers/4651612872/campaigns/16820250687","metrics.bounce_rate":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.business_name":"","ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text":"","campaign.id":16820250687,"campaign.name":"Website traffic-Search-15","campaign.status":"PAUSED","metrics.clicks":0,"ad_group_ad.policy_summary.approval_status":"APPROVED_LIMITED","metrics.conversions_from_interactions_rate":0.0,"metrics.conversions_value":0.0,"metrics.conversions":0.0,"metrics.cost_micros":0,"metrics.cost_per_all_conversions":0.0,"metrics.cost_per_conversion":0.0,"metrics.cost_per_current_model_attributed_conversion":0.0,"ad_group_ad.ad.final_mobile_urls":[],"ad_group_ad.ad.final_urls":["https://airbyte.com"],"ad_group_ad.ad.tracking_url_template":"","ad_group_ad.ad.url_custom_parameters":[],"metrics.cross_device_conversions":0.0,"metrics.ctr":0.0,"metrics.current_model_attributed_conversions_value":0.0,"metrics.current_model_attributed_conversions":0.0,"segments.date":"2022-04-09","segments.day_of_week":"SATURDAY","ad_group_ad.ad.expanded_text_ad.description":"","ad_group_ad.ad.text_ad.description1":"","ad_group_ad.ad.text_ad.description2":"","ad_group_ad.ad.device_preference":"UNSPECIFIED","ad_group_ad.ad.display_url":"","metrics.engagement_rate":0.0,"metrics.engagements":0,"ad_group_ad.ad.legacy_responsive_display_ad.logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.marketing_image":"","ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image":"","ad_group_ad.ad.expanded_dynamic_search_ad.description":"","ad_group_ad.ad.expanded_text_ad.description2":"","ad_group_ad.ad.expanded_text_ad.headline_part3":"","customer.id":4651612872,"ad_group_ad.ad.legacy_responsive_display_ad.format_setting":"UNSPECIFIED","metrics.gmail_forwards":0,"metrics.gmail_saves":0,"metrics.gmail_secondary_clicks":0,"ad_group_ad.ad.text_ad.headline":"","ad_group_ad.ad.expanded_text_ad.headline_part1":"","ad_group_ad.ad.expanded_text_ad.headline_part2":"","ad_group_ad.ad.id":592078631218,"ad_group_ad.ad.image_ad.image_url":"","ad_group_ad.ad.image_ad.pixel_height":0,"ad_group_ad.ad.image_ad.pixel_width":0,"ad_group_ad.ad.image_ad.mime_type":"UNSPECIFIED","ad_group_ad.ad.image_ad.name":"","metrics.impressions":16,"metrics.interaction_rate":0.0,"metrics.interaction_event_types":[],"metrics.interactions":0,"ad_group_ad.ad.legacy_responsive_display_ad.long_headline":"","ad_group_ad.ad.legacy_responsive_display_ad.main_color":"","segments.month":"2022-04-01","ad_group_ad.ad.responsive_display_ad.accent_color":"","ad_group_ad.ad.responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.responsive_display_ad.business_name":"","ad_group_ad.ad.responsive_display_ad.call_to_action_text":"","ad_group_ad.ad.responsive_display_ad.descriptions":[],"ad_group_ad.ad.responsive_display_ad.price_prefix":"","ad_group_ad.ad.responsive_display_ad.promo_text":"","ad_group_ad.ad.responsive_display_ad.format_setting":"UNSPECIFIED","ad_group_ad.ad.responsive_display_ad.headlines":[],"ad_group_ad.ad.responsive_display_ad.logo_images":[],"ad_group_ad.ad.responsive_display_ad.square_logo_images":[],"ad_group_ad.ad.responsive_display_ad.long_headline":"","ad_group_ad.ad.responsive_display_ad.main_color":"","ad_group_ad.ad.responsive_display_ad.marketing_images":[],"ad_group_ad.ad.responsive_display_ad.square_marketing_images":[],"ad_group_ad.ad.responsive_display_ad.youtube_videos":[],"ad_group_ad.ad.expanded_text_ad.path1":"","ad_group_ad.ad.expanded_text_ad.path2":"","metrics.percent_new_visitors":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.price_prefix":"","ad_group_ad.ad.legacy_responsive_display_ad.promo_text":"","segments.quarter":"2022-04-01","ad_group_ad.ad.responsive_search_ad.descriptions":["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n policy_topic_entries {\n topic: \"TRADEMARKS_IN_AD_TEXT\"\n type_: LIMITED\n evidences {\n text_list {\n texts: \"airbyte\"\n }\n }\n constraints {\n reseller_constraint {\n }\n }\n }\n review_status: REVIEWED\n approval_status: APPROVED_LIMITED\n}\n","text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.headlines":["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.path1":"","ad_group_ad.ad.responsive_search_ad.path2":"","ad_group_ad.ad.legacy_responsive_display_ad.short_headline":"","ad_group_ad.status":"ENABLED","ad_group_ad.ad.system_managed_resource_source":"UNSPECIFIED","metrics.top_impression_percentage":0.0,"ad_group_ad.ad.app_ad.descriptions":[],"ad_group_ad.ad.app_ad.headlines":[],"ad_group_ad.ad.app_ad.html5_media_bundles":[],"ad_group_ad.ad.app_ad.images":[],"ad_group_ad.ad.app_ad.mandatory_ad_text":"","ad_group_ad.ad.app_ad.youtube_videos":[],"metrics.value_per_all_conversions":0.0,"metrics.value_per_conversion":0.0,"metrics.value_per_current_model_attributed_conversion":0.0,"metrics.video_quartile_p100_rate":0.0,"metrics.video_quartile_p25_rate":0.0,"metrics.video_quartile_p50_rate":0.0,"metrics.video_quartile_p75_rate":0.0,"metrics.video_view_rate":0.0,"metrics.video_views":0,"metrics.view_through_conversions":0,"segments.week":"2022-04-04","segments.year":2022},"emitted_at":1679508005476} -{"stream":"ad_group_ad_report","data":{"ad_group_ad.ad.legacy_responsive_display_ad.accent_color":"","ad_group.id":137051662444,"customer.currency_code":"USD","customer.descriptive_name":"","customer.time_zone":"America/Los_Angeles","metrics.active_view_cpm":0.0,"metrics.active_view_ctr":0.0,"metrics.active_view_impressions":0,"metrics.active_view_measurability":0.0,"metrics.active_view_measurable_cost_micros":0,"metrics.active_view_measurable_impressions":0,"metrics.active_view_viewability":0.0,"ad_group_ad.ad_group":"customers/4651612872/adGroups/137051662444","ad_group.name":"Группа объявлений 1","ad_group.status":"ENABLED","segments.ad_network_type":"SEARCH_PARTNERS","ad_group_ad.ad_strength":"POOR","ad_group_ad.ad.type":"RESPONSIVE_SEARCH_AD","metrics.all_conversions_from_interactions_rate":0.0,"metrics.all_conversions_value":0.0,"metrics.all_conversions":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.added_by_google_ads":false,"metrics.average_cost":0.0,"metrics.average_cpc":0.0,"metrics.average_cpe":0.0,"metrics.average_cpm":0.0,"metrics.average_cpv":0.0,"metrics.average_page_views":0.0,"metrics.average_time_on_site":0.0,"ad_group.base_ad_group":"customers/4651612872/adGroups/137051662444","campaign.base_campaign":"customers/4651612872/campaigns/16820250687","metrics.bounce_rate":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.business_name":"","ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text":"","campaign.id":16820250687,"campaign.name":"Website traffic-Search-15","campaign.status":"PAUSED","metrics.clicks":0,"ad_group_ad.policy_summary.approval_status":"APPROVED","metrics.conversions_from_interactions_rate":0.0,"metrics.conversions_value":0.0,"metrics.conversions":0.0,"metrics.cost_micros":0,"metrics.cost_per_all_conversions":0.0,"metrics.cost_per_conversion":0.0,"metrics.cost_per_current_model_attributed_conversion":0.0,"ad_group_ad.ad.final_mobile_urls":[],"ad_group_ad.ad.final_urls":["https://airbyte.com"],"ad_group_ad.ad.tracking_url_template":"","ad_group_ad.ad.url_custom_parameters":[],"metrics.cross_device_conversions":0.0,"metrics.ctr":0.0,"metrics.current_model_attributed_conversions_value":0.0,"metrics.current_model_attributed_conversions":0.0,"segments.date":"2022-04-09","segments.day_of_week":"SATURDAY","ad_group_ad.ad.expanded_text_ad.description":"","ad_group_ad.ad.text_ad.description1":"","ad_group_ad.ad.text_ad.description2":"","ad_group_ad.ad.device_preference":"UNSPECIFIED","ad_group_ad.ad.display_url":"","metrics.engagement_rate":0.0,"metrics.engagements":0,"ad_group_ad.ad.legacy_responsive_display_ad.logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image":"","ad_group_ad.ad.legacy_responsive_display_ad.marketing_image":"","ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image":"","ad_group_ad.ad.expanded_dynamic_search_ad.description":"","ad_group_ad.ad.expanded_text_ad.description2":"","ad_group_ad.ad.expanded_text_ad.headline_part3":"","customer.id":4651612872,"ad_group_ad.ad.legacy_responsive_display_ad.format_setting":"UNSPECIFIED","metrics.gmail_forwards":0,"metrics.gmail_saves":0,"metrics.gmail_secondary_clicks":0,"ad_group_ad.ad.text_ad.headline":"","ad_group_ad.ad.expanded_text_ad.headline_part1":"","ad_group_ad.ad.expanded_text_ad.headline_part2":"","ad_group_ad.ad.id":592078676857,"ad_group_ad.ad.image_ad.image_url":"","ad_group_ad.ad.image_ad.pixel_height":0,"ad_group_ad.ad.image_ad.pixel_width":0,"ad_group_ad.ad.image_ad.mime_type":"UNSPECIFIED","ad_group_ad.ad.image_ad.name":"","metrics.impressions":28,"metrics.interaction_rate":0.0,"metrics.interaction_event_types":[],"metrics.interactions":0,"ad_group_ad.ad.legacy_responsive_display_ad.long_headline":"","ad_group_ad.ad.legacy_responsive_display_ad.main_color":"","segments.month":"2022-04-01","ad_group_ad.ad.responsive_display_ad.accent_color":"","ad_group_ad.ad.responsive_display_ad.allow_flexible_color":false,"ad_group_ad.ad.responsive_display_ad.business_name":"","ad_group_ad.ad.responsive_display_ad.call_to_action_text":"","ad_group_ad.ad.responsive_display_ad.descriptions":[],"ad_group_ad.ad.responsive_display_ad.price_prefix":"","ad_group_ad.ad.responsive_display_ad.promo_text":"","ad_group_ad.ad.responsive_display_ad.format_setting":"UNSPECIFIED","ad_group_ad.ad.responsive_display_ad.headlines":[],"ad_group_ad.ad.responsive_display_ad.logo_images":[],"ad_group_ad.ad.responsive_display_ad.square_logo_images":[],"ad_group_ad.ad.responsive_display_ad.long_headline":"","ad_group_ad.ad.responsive_display_ad.main_color":"","ad_group_ad.ad.responsive_display_ad.marketing_images":[],"ad_group_ad.ad.responsive_display_ad.square_marketing_images":[],"ad_group_ad.ad.responsive_display_ad.youtube_videos":[],"ad_group_ad.ad.expanded_text_ad.path1":"","ad_group_ad.ad.expanded_text_ad.path2":"","metrics.percent_new_visitors":0.0,"ad_group_ad.ad.legacy_responsive_display_ad.price_prefix":"","ad_group_ad.ad.legacy_responsive_display_ad.promo_text":"","segments.quarter":"2022-04-01","ad_group_ad.ad.responsive_search_ad.descriptions":["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.headlines":["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n","text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"],"ad_group_ad.ad.responsive_search_ad.path1":"","ad_group_ad.ad.responsive_search_ad.path2":"","ad_group_ad.ad.legacy_responsive_display_ad.short_headline":"","ad_group_ad.status":"REMOVED","ad_group_ad.ad.system_managed_resource_source":"UNSPECIFIED","metrics.top_impression_percentage":0.0,"ad_group_ad.ad.app_ad.descriptions":[],"ad_group_ad.ad.app_ad.headlines":[],"ad_group_ad.ad.app_ad.html5_media_bundles":[],"ad_group_ad.ad.app_ad.images":[],"ad_group_ad.ad.app_ad.mandatory_ad_text":"","ad_group_ad.ad.app_ad.youtube_videos":[],"metrics.value_per_all_conversions":0.0,"metrics.value_per_conversion":0.0,"metrics.value_per_current_model_attributed_conversion":0.0,"metrics.video_quartile_p100_rate":0.0,"metrics.video_quartile_p25_rate":0.0,"metrics.video_quartile_p50_rate":0.0,"metrics.video_quartile_p75_rate":0.0,"metrics.video_view_rate":0.0,"metrics.video_views":0,"metrics.view_through_conversions":0,"segments.week":"2022-04-04","segments.year":2022},"emitted_at":1679508005481} -{"stream": "ad_group_ads", "data": {"ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 592078676857, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078676857", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/137051662444", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.labels": [], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/137051662444~592078676857", "ad_group_ad.status": "REMOVED", "segments.date": "2022-04-08"}, "emitted_at": 1692608495121} -{"stream": "ad_group_ads", "data": {"ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078631218", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n policy_topic_entries {\n topic: \"TRADEMARKS_IN_AD_TEXT\"\n type_: LIMITED\n evidences {\n text_list {\n texts: \"airbyte\"\n }\n }\n constraints {\n reseller_constraint {\n }\n }\n }\n review_status: REVIEWED\n approval_status: APPROVED_LIMITED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/137020701042", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.labels": ["customers/4651612872/labels/21906377810"], "ad_group_ad.policy_summary.approval_status": "APPROVED_LIMITED", "ad_group_ad.policy_summary.policy_topic_entries": ["topic: \"TRADEMARKS_IN_AD_TEXT\"\ntype_: LIMITED\nevidences {\n text_list {\n texts: \"airbyte\"\n }\n}\nconstraints {\n reseller_constraint {\n }\n}\n"], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/137020701042~592078631218", "ad_group_ad.status": "ENABLED", "segments.date": "2022-04-09"}, "emitted_at": 1692608495139} -{"stream": "ad_group_ads", "data": {"ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 592078676857, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078676857", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/137051662444", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.labels": [], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/137051662444~592078676857", "ad_group_ad.status": "REMOVED", "segments.date": "2022-04-09"}, "emitted_at": 1692608495148} -{"stream": "ad_group_ads", "data": {"ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078631218", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n policy_topic_entries {\n topic: \"TRADEMARKS_IN_AD_TEXT\"\n type_: LIMITED\n evidences {\n text_list {\n texts: \"airbyte\"\n }\n }\n constraints {\n reseller_constraint {\n }\n }\n }\n review_status: REVIEWED\n approval_status: APPROVED_LIMITED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/137020701042", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.labels": ["customers/4651612872/labels/21906377810"], "ad_group_ad.policy_summary.approval_status": "APPROVED_LIMITED", "ad_group_ad.policy_summary.policy_topic_entries": ["topic: \"TRADEMARKS_IN_AD_TEXT\"\ntype_: LIMITED\nevidences {\n text_list {\n texts: \"airbyte\"\n }\n}\nconstraints {\n reseller_constraint {\n }\n}\n"], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/137020701042~592078631218", "ad_group_ad.status": "ENABLED", "segments.date": "2022-04-10"}, "emitted_at": 1692608495155} -{"stream": "ad_group_ads", "data": {"ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 592078676857, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078676857", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/137051662444", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.labels": [], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/137051662444~592078676857", "ad_group_ad.status": "REMOVED", "segments.date": "2022-04-10"}, "emitted_at": 1692608495162} -{"stream": "campaigns", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "SEARCH", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/12862729190", "campaign_budget.amount_micros": 1000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 16820250687, "campaign.labels": ["customers/4651612872/labels/21906377810"], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "Website traffic-Search-15", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": true, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "CLICKS", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.shopping_setting.sales_country": "", "campaign.start_date": "2022-04-08", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n"], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 0, "metrics.ctr": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.impressions": 1.0, "metrics.video_views": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": "[]", "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2022-05-25", "segments.hour": 16.0, "segments.ad_network_type": "SEARCH"}, "emitted_at": 1692606908590} -{"stream": "campaigns", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "SEARCH", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/12862729190", "campaign_budget.amount_micros": 1000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 16820250687, "campaign.labels": ["customers/4651612872/labels/21906377810"], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "Website traffic-Search-15", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": true, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "CLICKS", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.shopping_setting.sales_country": "", "campaign.start_date": "2022-04-08", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n"], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 1, "metrics.ctr": 1.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 10000, "metrics.impressions": 1.0, "metrics.video_views": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.average_cost": 10000.0, "metrics.average_cpc": 10000.0, "metrics.average_cpm": 10000000.0, "metrics.interactions": 1, "metrics.interaction_event_types": "['InteractionEventType.CLICK']", "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2022-05-25", "segments.hour": 18.0, "segments.ad_network_type": "SEARCH"}, "emitted_at": 1692606908597} -{"stream": "campaigns", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "SEARCH", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/12862729190", "campaign_budget.amount_micros": 1000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 16820250687, "campaign.labels": ["customers/4651612872/labels/21906377810"], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "Website traffic-Search-15", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": true, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "CLICKS", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.shopping_setting.sales_country": "", "campaign.start_date": "2022-04-08", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n"], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 0, "metrics.ctr": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.impressions": 2.0, "metrics.video_views": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": "[]", "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2022-05-25", "segments.hour": 19.0, "segments.ad_network_type": "SEARCH"}, "emitted_at": 1692606908603} -{"stream": "campaigns", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "SEARCH", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/12862729190", "campaign_budget.amount_micros": 1000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 16820250687, "campaign.labels": ["customers/4651612872/labels/21906377810"], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "Website traffic-Search-15", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": true, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "CLICKS", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.shopping_setting.sales_country": "", "campaign.start_date": "2022-04-08", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n"], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 0, "metrics.ctr": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.impressions": 2.0, "metrics.video_views": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": "[]", "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2022-05-25", "segments.hour": 20.0, "segments.ad_network_type": "SEARCH"}, "emitted_at": 1692606908610} -{"stream": "campaigns", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "SEARCH", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/12862729190", "campaign_budget.amount_micros": 1000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 16820250687, "campaign.labels": ["customers/4651612872/labels/21906377810"], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "Website traffic-Search-15", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": true, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "CLICKS", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.shopping_setting.sales_country": "", "campaign.start_date": "2022-04-08", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n"], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 0, "metrics.ctr": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.impressions": 1.0, "metrics.video_views": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": "[]", "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2022-05-25", "segments.hour": 22.0, "segments.ad_network_type": "SEARCH"}, "emitted_at": 1692606908617} -{"stream": "ad_groups", "data": {"ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137051662444", "ad_group.campaign": "customers/4651612872/campaigns/16820250687", "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 137051662444, "ad_group.labels": [], "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a01", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/137051662444", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": [], "segments.date": "2022-04-08"}, "emitted_at": 1692610032155} -{"stream": "ad_groups", "data": {"ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "ad_group.campaign": "customers/4651612872/campaigns/16820250687", "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 137020701042, "ad_group.labels": ["customers/4651612872/labels/21906377810"], "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/137020701042", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": [], "segments.date": "2022-04-09"}, "emitted_at": 1692610032158} -{"stream": "ad_groups", "data": {"ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137051662444", "ad_group.campaign": "customers/4651612872/campaigns/16820250687", "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 137051662444, "ad_group.labels": [], "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a01", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/137051662444", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": [], "segments.date": "2022-04-09"}, "emitted_at": 1692610032159} -{"stream": "ad_groups", "data": {"ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "ad_group.campaign": "customers/4651612872/campaigns/16820250687", "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 137020701042, "ad_group.labels": ["customers/4651612872/labels/21906377810"], "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/137020701042", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": [], "segments.date": "2022-04-10"}, "emitted_at": 1692610032161} -{"stream": "ad_groups", "data": {"ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137051662444", "ad_group.campaign": "customers/4651612872/campaigns/16820250687", "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 137051662444, "ad_group.labels": [], "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a01", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/137051662444", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": [], "segments.date": "2022-04-10"}, "emitted_at": 1692610032162} -{"stream": "audience", "data": {"audience.description": "", "audience.dimensions": ["audience_segments {\n segments {\n custom_audience {\n custom_audience: \"customers/4651612872/customAudiences/523469909\"\n }\n }\n}\n"], "audience.exclusion_dimension": "", "audience.id": 47792633, "audience.name": "Audience name 1", "audience.resource_name": "customers/4651612872/audiences/47792633", "audience.status": "ENABLED"}, "emitted_at": 1688510825339} -{"stream": "user_interest", "data": {"user_interest.availabilities": ["channel {\n availability_mode: CHANNEL_TYPE_AND_ALL_SUBTYPES\n advertising_channel_type: SEARCH\n include_default_channel_sub_type: true\n}\nlocale {\n availability_mode: ALL_LOCALES\n}\n", "channel {\n availability_mode: CHANNEL_TYPE_AND_ALL_SUBTYPES\n advertising_channel_type: DISPLAY\n include_default_channel_sub_type: true\n}\nlocale {\n availability_mode: ALL_LOCALES\n}\n", "channel {\n availability_mode: CHANNEL_TYPE_AND_ALL_SUBTYPES\n advertising_channel_type: SHOPPING\n include_default_channel_sub_type: true\n}\nlocale {\n availability_mode: ALL_LOCALES\n}\n", "channel {\n availability_mode: CHANNEL_TYPE_AND_ALL_SUBTYPES\n advertising_channel_type: DISCOVERY\n include_default_channel_sub_type: true\n}\nlocale {\n availability_mode: ALL_LOCALES\n}\n", "channel {\n availability_mode: CHANNEL_TYPE_AND_SUBSET_SUBTYPES\n advertising_channel_type: VIDEO\n advertising_channel_sub_type: VIDEO_SEQUENCE\n advertising_channel_sub_type: VIDEO_OUTSTREAM\n advertising_channel_sub_type: VIDEO_ACTION\n advertising_channel_sub_type: VIDEO_NON_SKIPPABLE\n advertising_channel_sub_type: VIDEO_REACH_TARGET_FREQUENCY\n include_default_channel_sub_type: true\n}\nlocale {\n availability_mode: ALL_LOCALES\n}\n"], "user_interest.launched_to_all": false, "user_interest.name": "Cloud Services Power Users", "user_interest.resource_name": "customers/4651612872/userInterests/92931", "user_interest.taxonomy_type": "AFFINITY", "user_interest.user_interest_id": 92931, "user_interest.user_interest_parent": "customers/4651612872/userInterests/92507"}, "emitted_at": 1688510842265} -{"stream": "campaign_budget", "data": {"campaign_budget.aligned_bidding_strategy_id": 0, "campaign_budget.amount_micros": 750000, "campaign_budget.delivery_method": ["STANDARD"], "campaign_budget.explicitly_shared": false, "campaign_budget.has_recommended_budget": false, "campaign_budget.id": 10695604507, "campaign_budget.name": "Website traffic-Search-15", "campaign_budget.period": ["DAILY"], "campaign_budget.recommended_budget_amount_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_clicks": 0, "campaign_budget.recommended_budget_estimated_change_weekly_cost_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_interactions": 0, "campaign_budget.recommended_budget_estimated_change_weekly_views": 0, "campaign_budget.reference_count": 0, "campaign_budget.resource_name": "customers/4651612872/campaignBudgets/10695604507", "campaign_budget.status": ["REMOVED"], "campaign_budget.total_amount_micros": 0, "campaign_budget.type": ["STANDARD"], "segments.date": "2022-04-08", "segments.budget_campaign_association_status.campaign": "customers/4651612872/campaigns/16820250687", "segments.budget_campaign_association_status.status": ["REMOVED"], "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.clicks": 0, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "metrics.impressions": 2, "metrics.interaction_event_types": [], "metrics.interaction_rate": 0.0, "metrics.interactions": 0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1692610545110} -{"stream": "campaign_budget", "data": {"campaign_budget.aligned_bidding_strategy_id": 0, "campaign_budget.amount_micros": 750000, "campaign_budget.delivery_method": ["STANDARD"], "campaign_budget.explicitly_shared": false, "campaign_budget.has_recommended_budget": false, "campaign_budget.id": 10695604507, "campaign_budget.name": "Website traffic-Search-15", "campaign_budget.period": ["DAILY"], "campaign_budget.recommended_budget_amount_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_clicks": 0, "campaign_budget.recommended_budget_estimated_change_weekly_cost_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_interactions": 0, "campaign_budget.recommended_budget_estimated_change_weekly_views": 0, "campaign_budget.reference_count": 0, "campaign_budget.resource_name": "customers/4651612872/campaignBudgets/10695604507", "campaign_budget.status": ["REMOVED"], "campaign_budget.total_amount_micros": 0, "campaign_budget.type": ["STANDARD"], "segments.date": "2022-04-09", "segments.budget_campaign_association_status.campaign": "customers/4651612872/campaigns/16820250687", "segments.budget_campaign_association_status.status": ["REMOVED"], "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 70000.0, "metrics.average_cpc": 70000.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 492957.74647887325, "metrics.average_cpv": 0.0, "metrics.clicks": 1, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 70000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.007042253521126761, "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "metrics.impressions": 142, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interaction_rate": 0.007042253521126761, "metrics.interactions": 1, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1692610545113} -{"stream": "campaign_budget", "data": {"campaign_budget.aligned_bidding_strategy_id": 0, "campaign_budget.amount_micros": 750000, "campaign_budget.delivery_method": ["STANDARD"], "campaign_budget.explicitly_shared": false, "campaign_budget.has_recommended_budget": false, "campaign_budget.id": 10695604507, "campaign_budget.name": "Website traffic-Search-15", "campaign_budget.period": ["DAILY"], "campaign_budget.recommended_budget_amount_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_clicks": 0, "campaign_budget.recommended_budget_estimated_change_weekly_cost_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_interactions": 0, "campaign_budget.recommended_budget_estimated_change_weekly_views": 0, "campaign_budget.reference_count": 0, "campaign_budget.resource_name": "customers/4651612872/campaignBudgets/10695604507", "campaign_budget.status": ["REMOVED"], "campaign_budget.total_amount_micros": 0, "campaign_budget.type": ["STANDARD"], "segments.date": "2022-04-10", "segments.budget_campaign_association_status.campaign": "customers/4651612872/campaigns/16820250687", "segments.budget_campaign_association_status.status": ["REMOVED"], "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 175000.0, "metrics.average_cpc": 175000.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 17721518.987341773, "metrics.average_cpv": 0.0, "metrics.clicks": 8, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 1400000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.10126582278481013, "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "metrics.impressions": 79, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interaction_rate": 0.10126582278481013, "metrics.interactions": 8, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1692610545116} -{"stream": "labels", "data": {"label.id": 21585034471, "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471", "label.status": "ENABLED", "label.text_label.background_color": "#E993EB", "label.text_label.description": "example label for edgao"}, "emitted_at": 1689230737253} -{"stream": "campaign_bidding_strategies", "data": {"campaign.id": 17354032686, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-05-31"}, "emitted_at": 1689230742268} -{"stream": "campaign_bidding_strategies", "data": {"campaign.id": 17324459992, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-06-01"}, "emitted_at": 1689230742269} -{"stream": "ad_group_bidding_strategies", "data": {"ad_group.id": 143992182864, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-05-31"}, "emitted_at": 1689230746492} -{"stream": "ad_group_bidding_strategies", "data": {"ad_group.id": 138459160713, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-06-01"}, "emitted_at": 1689230746493} -{"stream": "ad_group_criterions", "data": {"ad_group.id": 143992182864, "ad_group_criterion.ad_group": "customers/4651612872/adGroups/143992182864", "ad_group_criterion.age_range.type": "UNSPECIFIED", "ad_group_criterion.app_payment_model.type": "UNSPECIFIED", "ad_group_criterion.approval_status": "APPROVED", "ad_group_criterion.audience.audience": "", "ad_group_criterion.bid_modifier": 0.0, "ad_group_criterion.combined_audience.combined_audience": "", "ad_group_criterion.cpc_bid_micros": 0, "ad_group_criterion.cpm_bid_micros": 0, "ad_group_criterion.cpv_bid_micros": 0, "ad_group_criterion.criterion_id": 82426333464, "ad_group_criterion.custom_affinity.custom_affinity": "", "ad_group_criterion.custom_audience.custom_audience": "", "ad_group_criterion.custom_intent.custom_intent": "", "ad_group_criterion.disapproval_reasons": [], "ad_group_criterion.display_name": "uservertical::80530", "ad_group_criterion.effective_cpc_bid_micros": 0, "ad_group_criterion.effective_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 10000, "ad_group_criterion.effective_cpv_bid_source": "AD_GROUP", "ad_group_criterion.effective_percent_cpc_bid_micros": 0, "ad_group_criterion.effective_percent_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_url_suffix": "", "ad_group_criterion.final_urls": [], "ad_group_criterion.gender.type": "UNSPECIFIED", "ad_group_criterion.income_range.type": "UNSPECIFIED", "ad_group_criterion.keyword.match_type": "UNSPECIFIED", "ad_group_criterion.keyword.text": "", "ad_group_criterion.labels": [], "ad_group_criterion.mobile_app_category.mobile_app_category_constant": "", "ad_group_criterion.mobile_application.app_id": "", "ad_group_criterion.mobile_application.name": "", "ad_group_criterion.negative": false, "ad_group_criterion.parental_status.type": "UNSPECIFIED", "ad_group_criterion.percent_cpc_bid_micros": 0, "ad_group_criterion.placement.url": "", "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.first_page_cpc_micros": 0, "ad_group_criterion.position_estimates.first_position_cpc_micros": 0, "ad_group_criterion.position_estimates.top_of_page_cpc_micros": 0, "ad_group_criterion.quality_info.creative_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.post_click_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.quality_score": 0, "ad_group_criterion.quality_info.search_predicted_ctr": "UNSPECIFIED", "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/143992182864~82426333464", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.system_serving_status": "ELIGIBLE", "ad_group_criterion.topic.path": [], "ad_group_criterion.topic.topic_constant": "", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.type": "USER_INTEREST", "ad_group_criterion.url_custom_parameters": [], "ad_group_criterion.user_interest.user_interest_category": "customers/4651612872/userInterests/80530", "ad_group_criterion.user_list.user_list": "", "ad_group_criterion.webpage.conditions": [], "ad_group_criterion.webpage.coverage_percentage": 0.0, "ad_group_criterion.webpage.criterion_name": "", "ad_group_criterion.webpage.sample.sample_urls": [], "ad_group_criterion.youtube_channel.channel_id": "", "ad_group_criterion.youtube_video.video_id": ""}, "emitted_at": 1689230748366} -{"stream": "ad_group_criterions", "data": {"ad_group.id": 143992182864, "ad_group_criterion.ad_group": "customers/4651612872/adGroups/143992182864", "ad_group_criterion.age_range.type": "UNSPECIFIED", "ad_group_criterion.app_payment_model.type": "UNSPECIFIED", "ad_group_criterion.approval_status": "APPROVED", "ad_group_criterion.audience.audience": "", "ad_group_criterion.bid_modifier": 0.0, "ad_group_criterion.combined_audience.combined_audience": "", "ad_group_criterion.cpc_bid_micros": 0, "ad_group_criterion.cpm_bid_micros": 0, "ad_group_criterion.cpv_bid_micros": 0, "ad_group_criterion.criterion_id": 297422806498, "ad_group_criterion.custom_affinity.custom_affinity": "", "ad_group_criterion.custom_audience.custom_audience": "", "ad_group_criterion.custom_intent.custom_intent": "", "ad_group_criterion.disapproval_reasons": [], "ad_group_criterion.display_name": "api integration", "ad_group_criterion.effective_cpc_bid_micros": 0, "ad_group_criterion.effective_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 10000, "ad_group_criterion.effective_cpv_bid_source": "AD_GROUP", "ad_group_criterion.effective_percent_cpc_bid_micros": 0, "ad_group_criterion.effective_percent_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_url_suffix": "", "ad_group_criterion.final_urls": [], "ad_group_criterion.gender.type": "UNSPECIFIED", "ad_group_criterion.income_range.type": "UNSPECIFIED", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.keyword.text": "api integration", "ad_group_criterion.labels": [], "ad_group_criterion.mobile_app_category.mobile_app_category_constant": "", "ad_group_criterion.mobile_application.app_id": "", "ad_group_criterion.mobile_application.name": "", "ad_group_criterion.negative": false, "ad_group_criterion.parental_status.type": "UNSPECIFIED", "ad_group_criterion.percent_cpc_bid_micros": 0, "ad_group_criterion.placement.url": "", "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.first_page_cpc_micros": 0, "ad_group_criterion.position_estimates.first_position_cpc_micros": 0, "ad_group_criterion.position_estimates.top_of_page_cpc_micros": 0, "ad_group_criterion.quality_info.creative_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.post_click_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.quality_score": 0, "ad_group_criterion.quality_info.search_predicted_ctr": "UNSPECIFIED", "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/143992182864~297422806498", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.system_serving_status": "ELIGIBLE", "ad_group_criterion.topic.path": [], "ad_group_criterion.topic.topic_constant": "", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.url_custom_parameters": [], "ad_group_criterion.user_interest.user_interest_category": "", "ad_group_criterion.user_list.user_list": "", "ad_group_criterion.webpage.conditions": [], "ad_group_criterion.webpage.coverage_percentage": 0.0, "ad_group_criterion.webpage.criterion_name": "", "ad_group_criterion.webpage.sample.sample_urls": [], "ad_group_criterion.youtube_channel.channel_id": "", "ad_group_criterion.youtube_video.video_id": ""}, "emitted_at": 1689230748368} -{"stream": "ad_listing_group_criterions", "data": {"ad_group.id": 143992182864, "ad_group_criterion.criterion_id": 82426333464, "ad_group_criterion.listing_group.case_value.activity_country.value": "", "ad_group_criterion.listing_group.case_value.activity_id.value": "", "ad_group_criterion.listing_group.case_value.activity_rating.value": 0, "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_class.value": 0, "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_id.value": "", "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": "", "ad_group_criterion.listing_group.case_value.product_bidding_category.id": 0, "ad_group_criterion.listing_group.case_value.product_bidding_category.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_brand.value": "", "ad_group_criterion.listing_group.case_value.product_channel.channel": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_condition.condition": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": "", "ad_group_criterion.listing_group.case_value.product_item_id.value": "", "ad_group_criterion.listing_group.case_value.product_type.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_type.value": "", "ad_group_criterion.listing_group.parent_ad_group_criterion": "", "ad_group_criterion.listing_group.type": "UNSPECIFIED"}, "emitted_at": 1689230749081} -{"stream": "ad_listing_group_criterions", "data": {"ad_group.id": 143992182864, "ad_group_criterion.criterion_id": 297422806498, "ad_group_criterion.listing_group.case_value.activity_country.value": "", "ad_group_criterion.listing_group.case_value.activity_id.value": "", "ad_group_criterion.listing_group.case_value.activity_rating.value": 0, "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_class.value": 0, "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_id.value": "", "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": "", "ad_group_criterion.listing_group.case_value.product_bidding_category.id": 0, "ad_group_criterion.listing_group.case_value.product_bidding_category.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_brand.value": "", "ad_group_criterion.listing_group.case_value.product_channel.channel": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_condition.condition": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": "", "ad_group_criterion.listing_group.case_value.product_item_id.value": "", "ad_group_criterion.listing_group.case_value.product_type.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_type.value": "", "ad_group_criterion.listing_group.parent_ad_group_criterion": "", "ad_group_criterion.listing_group.type": "UNSPECIFIED"}, "emitted_at": 1689230749082} +{"stream": "ad_group_ad_legacy", "data": {"ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group.id": 137020701042, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "ad_group_ad.ad_group": "customers/4651612872/adGroups/137020701042", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.status": "ENABLED", "segments.ad_network_type": "SEARCH", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.added_by_google_ads": false, "metrics.average_cost": 197500.0, "metrics.average_cpc": 197500.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 19750000.0, "metrics.average_cpv": 0.0, "metrics.average_page_views": 0.0, "metrics.average_time_on_site": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "metrics.bounce_rate": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "metrics.clicks": 4, "ad_group_ad.policy_summary.approval_status": "APPROVED", "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 790000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cost_per_current_model_attributed_conversion": 0.0, "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.url_custom_parameters": [], "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.1, "metrics.current_model_attributed_conversions_value": 0.0, "metrics.current_model_attributed_conversions": 0.0, "segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_url": "", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "customer.id": 4651612872, "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "metrics.impressions": 40, "metrics.interaction_rate": 0.1, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 4, "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "segments.month": "2022-05-01", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "metrics.percent_new_visitors": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "segments.quarter": "2022-04-01", "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.status": "ENABLED", "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "metrics.top_impression_percentage": 0.75, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.value_per_current_model_attributed_conversion": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-05-16", "segments.year": 2022}, "emitted_at": 1704407744443} +{"stream": "ad_group_ad_legacy", "data": {"ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group.id": 137020701042, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "ad_group_ad.ad_group": "customers/4651612872/adGroups/137020701042", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.status": "ENABLED", "segments.ad_network_type": "SEARCH_PARTNERS", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.added_by_google_ads": false, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.average_page_views": 0.0, "metrics.average_time_on_site": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "metrics.bounce_rate": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "metrics.clicks": 0, "ad_group_ad.policy_summary.approval_status": "APPROVED", "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cost_per_current_model_attributed_conversion": 0.0, "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.url_custom_parameters": [], "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.current_model_attributed_conversions_value": 0.0, "metrics.current_model_attributed_conversions": 0.0, "segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_url": "", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "customer.id": 4651612872, "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "metrics.impressions": 11, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "segments.month": "2022-05-01", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "metrics.percent_new_visitors": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "segments.quarter": "2022-04-01", "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.status": "ENABLED", "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "metrics.top_impression_percentage": 0.0, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.value_per_current_model_attributed_conversion": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-05-16", "segments.year": 2022}, "emitted_at": 1704407744444} +{"stream": "ad_group_ad_legacy", "data": {"ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group.id": 137020701042, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "ad_group_ad.ad_group": "customers/4651612872/adGroups/137020701042", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.status": "ENABLED", "segments.ad_network_type": "SEARCH", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.added_by_google_ads": false, "metrics.average_cost": 143333.33333333334, "metrics.average_cpc": 143333.33333333334, "metrics.average_cpe": 0.0, "metrics.average_cpm": 37391304.347826086, "metrics.average_cpv": 0.0, "metrics.average_page_views": 0.0, "metrics.average_time_on_site": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "metrics.bounce_rate": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "metrics.clicks": 6, "ad_group_ad.policy_summary.approval_status": "APPROVED", "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 860000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cost_per_current_model_attributed_conversion": 0.0, "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.url_custom_parameters": [], "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.2608695652173913, "metrics.current_model_attributed_conversions_value": 0.0, "metrics.current_model_attributed_conversions": 0.0, "segments.date": "2022-05-19", "segments.day_of_week": "THURSDAY", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_url": "", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "customer.id": 4651612872, "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "metrics.impressions": 23, "metrics.interaction_rate": 0.2608695652173913, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 6, "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "segments.month": "2022-05-01", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "metrics.percent_new_visitors": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "segments.quarter": "2022-04-01", "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.status": "ENABLED", "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "metrics.top_impression_percentage": 0.7391304347826086, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.value_per_current_model_attributed_conversion": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-05-16", "segments.year": 2022}, "emitted_at": 1704407744444} +{"stream": "campaign_budget", "data": {"customer.id": 4651612872, "campaign.id": 16820250687, "campaign_budget.aligned_bidding_strategy_id": 0, "campaign_budget.amount_micros": 750000, "campaign_budget.delivery_method": "STANDARD", "campaign_budget.explicitly_shared": false, "campaign_budget.has_recommended_budget": false, "campaign_budget.id": 10695604507, "campaign_budget.name": "Website traffic-Search-15", "campaign_budget.period": "DAILY", "campaign_budget.recommended_budget_amount_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_clicks": 0, "campaign_budget.recommended_budget_estimated_change_weekly_cost_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_interactions": 0, "campaign_budget.recommended_budget_estimated_change_weekly_views": 0, "campaign_budget.reference_count": 0, "campaign_budget.resource_name": "customers/4651612872/campaignBudgets/10695604507", "campaign_budget.status": "REMOVED", "campaign_budget.total_amount_micros": 0, "campaign_budget.type": "STANDARD", "segments.date": "2022-05-18", "segments.budget_campaign_association_status.campaign": "customers/4651612872/campaigns/16820250687", "segments.budget_campaign_association_status.status": "REMOVED", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 197500.0, "metrics.average_cpc": 197500.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 15490196.078431372, "metrics.average_cpv": 0.0, "metrics.clicks": 4, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 790000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0784313725490196, "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "metrics.impressions": 51, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interaction_rate": 0.0784313725490196, "metrics.interactions": 4, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1704407746549} +{"stream": "campaign_budget", "data": {"customer.id": 4651612872, "campaign.id": 16820250687, "campaign_budget.aligned_bidding_strategy_id": 0, "campaign_budget.amount_micros": 750000, "campaign_budget.delivery_method": "STANDARD", "campaign_budget.explicitly_shared": false, "campaign_budget.has_recommended_budget": false, "campaign_budget.id": 10695604507, "campaign_budget.name": "Website traffic-Search-15", "campaign_budget.period": "DAILY", "campaign_budget.recommended_budget_amount_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_clicks": 0, "campaign_budget.recommended_budget_estimated_change_weekly_cost_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_interactions": 0, "campaign_budget.recommended_budget_estimated_change_weekly_views": 0, "campaign_budget.reference_count": 0, "campaign_budget.resource_name": "customers/4651612872/campaignBudgets/10695604507", "campaign_budget.status": "REMOVED", "campaign_budget.total_amount_micros": 0, "campaign_budget.type": "STANDARD", "segments.date": "2022-05-19", "segments.budget_campaign_association_status.campaign": "customers/4651612872/campaigns/16820250687", "segments.budget_campaign_association_status.status": "REMOVED", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 143333.33333333334, "metrics.average_cpc": 143333.33333333334, "metrics.average_cpe": 0.0, "metrics.average_cpm": 31851851.85185185, "metrics.average_cpv": 0.0, "metrics.clicks": 6, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 860000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.2222222222222222, "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "metrics.impressions": 27, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interaction_rate": 0.2222222222222222, "metrics.interactions": 6, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1704407746559} +{"stream": "campaign_budget", "data": {"customer.id": 4651612872, "campaign.id": 16820250687, "campaign_budget.aligned_bidding_strategy_id": 0, "campaign_budget.amount_micros": 750000, "campaign_budget.delivery_method": "STANDARD", "campaign_budget.explicitly_shared": false, "campaign_budget.has_recommended_budget": false, "campaign_budget.id": 10695604507, "campaign_budget.name": "Website traffic-Search-15", "campaign_budget.period": "DAILY", "campaign_budget.recommended_budget_amount_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_clicks": 0, "campaign_budget.recommended_budget_estimated_change_weekly_cost_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_interactions": 0, "campaign_budget.recommended_budget_estimated_change_weekly_views": 0, "campaign_budget.reference_count": 0, "campaign_budget.resource_name": "customers/4651612872/campaignBudgets/10695604507", "campaign_budget.status": "REMOVED", "campaign_budget.total_amount_micros": 0, "campaign_budget.type": "STANDARD", "segments.date": "2022-05-20", "segments.budget_campaign_association_status.campaign": "customers/4651612872/campaigns/16820250687", "segments.budget_campaign_association_status.status": "REMOVED", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 215000.0, "metrics.average_cpc": 215000.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 16538461.53846154, "metrics.average_cpv": 0.0, "metrics.clicks": 2, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 430000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.07692307692307693, "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "metrics.impressions": 26, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interaction_rate": 0.07692307692307693, "metrics.interactions": 2, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1704407746561} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2022-05-18"}, "emitted_at": 1704407754204} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2022-05-19"}, "emitted_at": 1704407754210} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2022-05-20"}, "emitted_at": 1704407754210} +{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "segments.ad_network_type": "SEARCH", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 253333.33333333334, "metrics.average_cpc": 253333.33333333334, "metrics.average_cpe": 0.0, "metrics.average_cpm": 27142857.14285714, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 3, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0, "metrics.content_rank_lost_impression_share": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 760000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.10714285714285714, "segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "segments.device": "DESKTOP", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 28, "metrics.interaction_rate": 0.10714285714285714, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 3, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2022-05-01", "segments.quarter": "2022-04-01", "metrics.search_budget_lost_impression_share": 0.6935849056603773, "metrics.search_exact_match_impression_share": 0.0999, "metrics.search_impression_share": 0.0999, "metrics.search_rank_lost_impression_share": 0.2852830188679245, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-05-16", "segments.year": 2022}, "emitted_at": 1704407755279} +{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "segments.ad_network_type": "SEARCH", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 30000.0, "metrics.average_cpc": 30000.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 2500000.0, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 1, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0, "metrics.content_rank_lost_impression_share": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 30000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.08333333333333333, "segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "segments.device": "MOBILE", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 12, "metrics.interaction_rate": 0.08333333333333333, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 1, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2022-05-01", "segments.quarter": "2022-04-01", "metrics.search_budget_lost_impression_share": 0.7254437869822485, "metrics.search_exact_match_impression_share": 0.0999, "metrics.search_impression_share": 0.0999, "metrics.search_rank_lost_impression_share": 0.2603550295857988, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-05-16", "segments.year": 2022}, "emitted_at": 1704407755286} +{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "segments.ad_network_type": "SEARCH", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 0, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0, "metrics.content_rank_lost_impression_share": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "segments.device": "TABLET", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 0, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2022-05-01", "segments.quarter": "2022-04-01", "metrics.search_budget_lost_impression_share": 0.9001, "metrics.search_exact_match_impression_share": 0.0999, "metrics.search_impression_share": 0.0999, "metrics.search_rank_lost_impression_share": 0.0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-05-16", "segments.year": 2022}, "emitted_at": 1704407755286} +{"stream": "geographic_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "geographic_view.country_criterion_id": 2840, "geographic_view.location_type": "AREA_OF_INTEREST", "ad_group.id": 137020701042, "segments.date": "2022-05-18"}, "emitted_at": 1704407756403} +{"stream": "geographic_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "geographic_view.country_criterion_id": 2124, "geographic_view.location_type": "LOCATION_OF_PRESENCE", "ad_group.id": 137020701042, "segments.date": "2022-05-18"}, "emitted_at": 1704407756404} +{"stream": "geographic_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "geographic_view.country_criterion_id": 2840, "geographic_view.location_type": "LOCATION_OF_PRESENCE", "ad_group.id": 137020701042, "segments.date": "2022-05-18"}, "emitted_at": 1704407756405} +{"stream": "keyword_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "campaign.id": 16820250687, "ad_group.id": 137020701042, "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.keyword.text": "data integration software", "ad_group_criterion.negative": false, "ad_group_criterion.keyword.match_type": "BROAD", "metrics.historical_quality_score": 0, "metrics.ctr": 0.0, "segments.date": "2022-05-18", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "metrics.clicks": 0, "metrics.cost_micros": 0, "metrics.impressions": 2, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": [], "metrics.view_through_conversions": 0, "ad_group_criterion.criterion_id": 18697003}, "emitted_at": 1704407757564} +{"stream": "keyword_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "campaign.id": 16820250687, "ad_group.id": 137020701042, "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.keyword.text": "informatica software", "ad_group_criterion.negative": false, "ad_group_criterion.keyword.match_type": "BROAD", "metrics.historical_quality_score": 0, "metrics.ctr": 0.0, "segments.date": "2022-05-18", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "metrics.clicks": 0, "metrics.cost_micros": 0, "metrics.impressions": 3, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": [], "metrics.view_through_conversions": 0, "ad_group_criterion.criterion_id": 27723800}, "emitted_at": 1704407757565} +{"stream": "keyword_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "campaign.id": 16820250687, "ad_group.id": 137020701042, "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.keyword.text": "etl extract transform load", "ad_group_criterion.negative": false, "ad_group_criterion.keyword.match_type": "BROAD", "metrics.historical_quality_score": 0, "metrics.ctr": 0.0, "segments.date": "2022-05-18", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "metrics.clicks": 0, "metrics.cost_micros": 0, "metrics.impressions": 4, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": [], "metrics.view_through_conversions": 0, "ad_group_criterion.criterion_id": 439152736}, "emitted_at": 1704407757565} +{"stream": "display_keyword_view", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 10012000.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 1, "metrics.active_view_measurability": 1.0, "metrics.active_view_measurable_cost_micros": 10012, "metrics.active_view_measurable_impressions": 1, "metrics.active_view_viewability": 1.0, "ad_group.id": 143992182864, "ad_group.name": "Video Non-skippable - 2022-05-30", "ad_group.status": "ENABLED", "segments.ad_network_type": "YOUTUBE", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 10012000.0, "metrics.average_cpv": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/143992182864", "campaign.base_campaign": "customers/4651612872/campaigns/17354032686", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "TARGET_CPM", "campaign.id": 17354032686, "campaign.name": "Video Non-skippable - 2022-05-30", "campaign.status": "ENABLED", "metrics.clicks": 0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 10012, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "ad_group_criterion.effective_cpc_bid_micros": 10000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 10000, "ad_group_criterion.effective_cpv_bid_source": "AD_GROUP", "ad_group_criterion.keyword.text": "big data software", "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.day_of_week": "TUESDAY", "segments.device": "MOBILE", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_urls": [], "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_criterion.criterion_id": 26160872903, "metrics.impressions": 1, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "ad_group_criterion.negative": false, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n", "targeting_dimension: TOPIC\nbid_only: false\n"], "segments.month": "2022-05-01", "segments.quarter": "2022-04-01", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.url_custom_parameters": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2022-05-30", "segments.year": 2022, "segments.date": "2022-05-31"}, "emitted_at": 1704407759165} +{"stream": "ad_group_ad", "data": {"ad_group.id": 137020701042, "ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078631218", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/137020701042", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.labels": ["customers/4651612872/labels/21906377810"], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/137020701042~592078631218", "ad_group_ad.status": "ENABLED", "segments.date": "2022-05-18"}, "emitted_at": 1704407765438} +{"stream": "ad_group_ad", "data": {"ad_group.id": 137020701042, "ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078631218", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/137020701042", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.labels": ["customers/4651612872/labels/21906377810"], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/137020701042~592078631218", "ad_group_ad.status": "ENABLED", "segments.date": "2022-05-19"}, "emitted_at": 1704407765455} +{"stream": "ad_group_ad", "data": {"ad_group.id": 137020701042, "ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078631218", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"Behind The Scenes: Testing The Airbyte Maintainer Program\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Airbyte | Open-Source Data Integration Platform | ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Upgrading Our Discourse And Slack To Support Our Community Growth\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Consolidate your data in your data warehouses, lakes and databases\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"ELT tool\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open-source Data Integration\"\nasset_performance_label: PENDING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/137020701042", "ad_group_ad.ad_strength": "POOR", "ad_group_ad.labels": ["customers/4651612872/labels/21906377810"], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/137020701042~592078631218", "ad_group_ad.status": "ENABLED", "segments.date": "2022-05-20"}, "emitted_at": 1704407765456} +{"stream": "ad_group", "data": {"campaign.id": 16820250687, "ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "ad_group.campaign": "customers/4651612872/campaigns/16820250687", "metrics.cost_micros": 790000, "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 137020701042, "ad_group.labels": ["customers/4651612872/labels/21906377810"], "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/137020701042", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": [], "segments.date": "2022-05-18"}, "emitted_at": 1704715893659} +{"stream": "ad_group", "data": {"campaign.id": 16820250687, "ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "ad_group.campaign": "customers/4651612872/campaigns/16820250687", "metrics.cost_micros": 860000, "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 137020701042, "ad_group.labels": ["customers/4651612872/labels/21906377810"], "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/137020701042", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": [], "segments.date": "2022-05-19"}, "emitted_at": 1704715893662} +{"stream": "ad_group", "data": {"campaign.id": 16820250687, "ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "ad_group.campaign": "customers/4651612872/campaigns/16820250687", "metrics.cost_micros": 430000, "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 137020701042, "ad_group.labels": ["customers/4651612872/labels/21906377810"], "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/137020701042", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": [], "segments.date": "2022-05-20"}, "emitted_at": 1704715893662} +{"stream": "customer", "data": {"customer.auto_tagging_enabled": true, "customer.call_reporting_setting.call_conversion_action": "customers/4651612872/conversionActions/179", "customer.call_reporting_setting.call_conversion_reporting_enabled": true, "customer.call_reporting_setting.call_reporting_enabled": true, "customer.conversion_tracking_setting.conversion_tracking_id": 657981234, "customer.conversion_tracking_setting.cross_account_conversion_tracking_id": 0, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.final_url_suffix": "", "customer.has_partners_badge": false, "customer.id": 4651612872, "customer.manager": false, "customer.optimization_score": 0.7609283000000001, "customer.optimization_score_weight": 3182.4700059999996, "customer.pay_per_conversion_eligibility_failure_reasons": [], "customer.remarketing_setting.google_global_site_tag": "\n\n\n", "customer.resource_name": "customers/4651612872", "customer.test_account": false, "customer.time_zone": "America/Los_Angeles", "customer.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign={_utmcampaign}&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam={campaignid}&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "segments.date": "2022-05-18"}, "emitted_at": 1704407768194} +{"stream": "customer", "data": {"customer.auto_tagging_enabled": true, "customer.call_reporting_setting.call_conversion_action": "customers/4651612872/conversionActions/179", "customer.call_reporting_setting.call_conversion_reporting_enabled": true, "customer.call_reporting_setting.call_reporting_enabled": true, "customer.conversion_tracking_setting.conversion_tracking_id": 657981234, "customer.conversion_tracking_setting.cross_account_conversion_tracking_id": 0, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.final_url_suffix": "", "customer.has_partners_badge": false, "customer.id": 4651612872, "customer.manager": false, "customer.optimization_score": 0.7609283000000001, "customer.optimization_score_weight": 3182.4700059999996, "customer.pay_per_conversion_eligibility_failure_reasons": [], "customer.remarketing_setting.google_global_site_tag": "\n\n\n", "customer.resource_name": "customers/4651612872", "customer.test_account": false, "customer.time_zone": "America/Los_Angeles", "customer.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign={_utmcampaign}&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam={campaignid}&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "segments.date": "2022-05-19"}, "emitted_at": 1704407768194} +{"stream": "customer", "data": {"customer.auto_tagging_enabled": true, "customer.call_reporting_setting.call_conversion_action": "customers/4651612872/conversionActions/179", "customer.call_reporting_setting.call_conversion_reporting_enabled": true, "customer.call_reporting_setting.call_reporting_enabled": true, "customer.conversion_tracking_setting.conversion_tracking_id": 657981234, "customer.conversion_tracking_setting.cross_account_conversion_tracking_id": 0, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.final_url_suffix": "", "customer.has_partners_badge": false, "customer.id": 4651612872, "customer.manager": false, "customer.optimization_score": 0.7609283000000001, "customer.optimization_score_weight": 3182.4700059999996, "customer.pay_per_conversion_eligibility_failure_reasons": [], "customer.remarketing_setting.google_global_site_tag": "\n\n\n", "customer.resource_name": "customers/4651612872", "customer.test_account": false, "customer.time_zone": "America/Los_Angeles", "customer.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign={_utmcampaign}&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam={campaignid}&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "segments.date": "2022-05-20"}, "emitted_at": 1704407768195} +{"stream": "campaign", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "SEARCH", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/12862729190", "campaign_budget.amount_micros": 1000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 16820250687, "campaign.labels": ["customers/4651612872/labels/21906377810"], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "Website traffic-Search-15", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": true, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "CLICKS", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.start_date": "2022-04-08", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n"], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign=Website+traffic-Search-15&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam=16820250687&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 0, "metrics.ctr": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.impressions": 1, "metrics.video_views": 0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": [], "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2022-05-18", "segments.hour": 1, "segments.ad_network_type": "SEARCH"}, "emitted_at": 1704407769633} +{"stream": "campaign", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "SEARCH", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/12862729190", "campaign_budget.amount_micros": 1000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 16820250687, "campaign.labels": ["customers/4651612872/labels/21906377810"], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "Website traffic-Search-15", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": true, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "CLICKS", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.start_date": "2022-04-08", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n"], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign=Website+traffic-Search-15&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam=16820250687&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 0, "metrics.ctr": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.impressions": 2, "metrics.video_views": 0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": [], "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2022-05-18", "segments.hour": 2, "segments.ad_network_type": "SEARCH"}, "emitted_at": 1704407769640} +{"stream": "campaign", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "SEARCH", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/12862729190", "campaign_budget.amount_micros": 1000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 16820250687, "campaign.labels": ["customers/4651612872/labels/21906377810"], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "Website traffic-Search-15", "campaign.network_settings.target_content_network": true, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": true, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "CLICKS", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.start_date": "2022-04-08", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n"], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign=Website+traffic-Search-15&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam=16820250687&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 0, "metrics.ctr": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.impressions": 2, "metrics.video_views": 0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": [], "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2022-05-18", "segments.hour": 3, "segments.ad_network_type": "SEARCH"}, "emitted_at": 1704407769646} +{"stream": "campaign_label", "data": {"campaign.id": 12124071339, "label.id": 21585034471, "campaign.resource_name": "customers/4651612872/campaigns/12124071339", "campaign_label.resource_name": "customers/4651612872/campaignLabels/12124071339~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1704407771173} +{"stream": "campaign_label", "data": {"campaign.id": 13284356762, "label.id": 21585034471, "campaign.resource_name": "customers/4651612872/campaigns/13284356762", "campaign_label.resource_name": "customers/4651612872/campaignLabels/13284356762~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1704407771175} +{"stream": "campaign_label", "data": {"campaign.id": 16820250687, "label.id": 21906377810, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign_label.resource_name": "customers/4651612872/campaignLabels/16820250687~21906377810", "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810"}, "emitted_at": 1704407771175} +{"stream": "ad_group_label", "data": {"ad_group.id": 123273719655, "label.id": 21585034471, "ad_group.resource_name": "customers/4651612872/adGroups/123273719655", "ad_group_label.resource_name": "customers/4651612872/adGroupLabels/123273719655~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1704407771465} +{"stream": "ad_group_label", "data": {"ad_group.id": 138643385242, "label.id": 21585034471, "ad_group.resource_name": "customers/4651612872/adGroups/138643385242", "ad_group_label.resource_name": "customers/4651612872/adGroupLabels/138643385242~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1704407771468} +{"stream": "ad_group_label", "data": {"ad_group.id": 137020701042, "label.id": 21906377810, "ad_group.resource_name": "customers/4651612872/adGroups/137020701042", "ad_group_label.resource_name": "customers/4651612872/adGroupLabels/137020701042~21906377810", "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810"}, "emitted_at": 1704407771468} +{"stream": "ad_group_ad_label", "data": {"ad_group.id": 123273719655, "ad_group_ad.ad.id": 524518584182, "ad_group_ad.ad.resource_name": "customers/4651612872/ads/524518584182", "ad_group_ad_label.resource_name": "customers/4651612872/adGroupAdLabels/123273719655~524518584182~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471", "label.id": 21585034471}, "emitted_at": 1704407771926} +{"stream": "ad_group_ad_label", "data": {"ad_group.id": 137020701042, "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078631218", "ad_group_ad_label.resource_name": "customers/4651612872/adGroupAdLabels/137020701042~592078631218~21906377810", "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810", "label.id": 21906377810}, "emitted_at": 1704407771929} +{"stream": "user_location_view", "data": {"segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "segments.month": "2022-05-01", "segments.week": "2022-05-16", "segments.quarter": "2022-04-01", "segments.year": 2022, "segments.ad_network_type": "SEARCH", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2356, "user_location_view.targeting_location": false, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2356~false", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.clicks": 0, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.impressions": 3, "metrics.interaction_event_types": [], "metrics.interaction_rate": 0.0, "metrics.interactions": 0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1704407772615} +{"stream": "user_location_view", "data": {"segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "segments.month": "2022-05-01", "segments.week": "2022-05-16", "segments.quarter": "2022-04-01", "segments.year": 2022, "segments.ad_network_type": "SEARCH", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2484, "user_location_view.targeting_location": false, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2484~false", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.clicks": 0, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.impressions": 1, "metrics.interaction_event_types": [], "metrics.interaction_rate": 0.0, "metrics.interactions": 0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1704407772615} +{"stream": "user_location_view", "data": {"segments.date": "2022-05-18", "segments.day_of_week": "WEDNESDAY", "segments.month": "2022-05-01", "segments.week": "2022-05-16", "segments.quarter": "2022-04-01", "segments.year": 2022, "segments.ad_network_type": "SEARCH", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2124, "user_location_view.targeting_location": true, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2124~true", "campaign.base_campaign": "customers/4651612872/campaigns/16820250687", "campaign.id": 16820250687, "campaign.name": "Website traffic-Search-15", "campaign.status": "PAUSED", "ad_group.name": "\u0413\u0440\u0443\u043f\u043f\u0430 \u043e\u0431\u044a\u044f\u0432\u043b\u0435\u043d\u0438\u0439\u00a02", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/137020701042", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 253333.33333333334, "metrics.average_cpc": 253333.33333333334, "metrics.average_cpm": 36190476.190476194, "metrics.average_cpv": 0.0, "metrics.clicks": 3, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 760000, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.14285714285714285, "metrics.impressions": 21, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interaction_rate": 0.14285714285714285, "metrics.interactions": 3, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1704407772616} +{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2022-04-08", "campaign.end_date": "2037-12-30", "segments.date": "2022-05-18"}, "emitted_at": 1704407774246} +{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2022-04-08", "campaign.end_date": "2037-12-30", "segments.date": "2022-05-19"}, "emitted_at": 1704407774254} +{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2022-04-08", "campaign.end_date": "2037-12-30", "segments.date": "2022-05-20"}, "emitted_at": 1704407774254} +{"stream": "custom_audience", "data": {"custom_audience.description": "", "custom_audience.name": "Airbyte", "custom_audience.id": 523469909, "custom_audience.members": ["member_type: KEYWORD\nkeyword: \"etl elt\"\n", "member_type: KEYWORD\nkeyword: \"cloud data management and analytics\"\n", "member_type: KEYWORD\nkeyword: \"data integration\"\n", "member_type: KEYWORD\nkeyword: \"big data analytics database\"\n", "member_type: KEYWORD\nkeyword: \"data\"\n", "member_type: KEYWORD\nkeyword: \"data sherid nada\"\n", "member_type: KEYWORD\nkeyword: \"airbyteforeveryone\"\n", "member_type: KEYWORD\nkeyword: \"Airbyte\"\n"], "custom_audience.resource_name": "customers/4651612872/customAudiences/523469909", "custom_audience.status": "ENABLED", "custom_audience.type": "AUTO"}, "emitted_at": 1704407775427} +{"stream": "audience", "data": {"customer.id": 4651612872, "audience.description": "", "audience.dimensions": ["audience_segments {\n segments {\n custom_audience {\n custom_audience: \"customers/4651612872/customAudiences/523469909\"\n }\n }\n}\n"], "audience.exclusion_dimension": "", "audience.id": 47792633, "audience.name": "Audience name 1", "audience.resource_name": "customers/4651612872/audiences/47792633", "audience.status": "ENABLED"}, "emitted_at": 1704407775721} +{"stream": "audience", "data": {"customer.id": 4651612872, "audience.description": "", "audience.dimensions": ["audience_segments {\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80276\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80279\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80520\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80530\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/92931\"\n }\n }\n}\n"], "audience.exclusion_dimension": "", "audience.id": 97300129, "audience.name": "Upgraded Audience 1", "audience.resource_name": "customers/4651612872/audiences/97300129", "audience.status": "ENABLED"}, "emitted_at": 1704407775723} +{"stream": "user_interest", "data": {"user_interest.availabilities": [], "user_interest.launched_to_all": true, "user_interest.name": "Arts & Entertainment", "user_interest.resource_name": "customers/4651612872/userInterests/3", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 3, "user_interest.user_interest_parent": ""}, "emitted_at": 1704407777549} +{"stream": "user_interest", "data": {"user_interest.availabilities": [], "user_interest.launched_to_all": true, "user_interest.name": "Computers & Electronics", "user_interest.resource_name": "customers/4651612872/userInterests/5", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 5, "user_interest.user_interest_parent": ""}, "emitted_at": 1704407777550} +{"stream": "user_interest", "data": {"user_interest.availabilities": [], "user_interest.launched_to_all": true, "user_interest.name": "Finance", "user_interest.resource_name": "customers/4651612872/userInterests/7", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 7, "user_interest.user_interest_parent": ""}, "emitted_at": 1704407777551} +{"stream": "label", "data": {"customer.id": 4651612872, "label.id": 21585034471, "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471", "label.status": "ENABLED", "label.text_label.background_color": "#E993EB", "label.text_label.description": "example label for edgao"}, "emitted_at": 1704407779851} +{"stream": "label", "data": {"customer.id": 4651612872, "label.id": 21902092838, "label.name": "Test Label", "label.resource_name": "customers/4651612872/labels/21902092838", "label.status": "ENABLED", "label.text_label.background_color": "#8BCBD2", "label.text_label.description": "Description to test label"}, "emitted_at": 1704407779852} +{"stream": "label", "data": {"customer.id": 4651612872, "label.id": 21906377810, "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810", "label.status": "ENABLED", "label.text_label.background_color": "#8266C9", "label.text_label.description": ""}, "emitted_at": 1704407779852} +{"stream": "campaign_bidding_strategy", "data": {"customer.id": 4651612872, "campaign.id": 16820250687, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-05-18"}, "emitted_at": 1704407780704} +{"stream": "campaign_bidding_strategy", "data": {"customer.id": 4651612872, "campaign.id": 16820250687, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-05-19"}, "emitted_at": 1704407780707} +{"stream": "campaign_bidding_strategy", "data": {"customer.id": 4651612872, "campaign.id": 16820250687, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-05-20"}, "emitted_at": 1704407780713} +{"stream": "ad_group_bidding_strategy", "data": {"ad_group.id": 137020701042, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-05-18"}, "emitted_at": 1704407781887} +{"stream": "ad_group_bidding_strategy", "data": {"ad_group.id": 137020701042, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-05-19"}, "emitted_at": 1704407781888} +{"stream": "ad_group_bidding_strategy", "data": {"ad_group.id": 137020701042, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2022-05-20"}, "emitted_at": 1704407781889} +{"stream": "ad_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group.id": 117036054899, "ad_group_criterion.ad_group": "customers/4651612872/adGroups/117036054899", "ad_group_criterion.age_range.type": "UNSPECIFIED", "ad_group_criterion.app_payment_model.type": "UNSPECIFIED", "ad_group_criterion.approval_status": "APPROVED", "ad_group_criterion.audience.audience": "", "ad_group_criterion.bid_modifier": 0.0, "ad_group_criterion.combined_audience.combined_audience": "", "ad_group_criterion.cpc_bid_micros": 0, "ad_group_criterion.cpm_bid_micros": 0, "ad_group_criterion.cpv_bid_micros": 0, "ad_group_criterion.criterion_id": 18696703, "ad_group_criterion.custom_affinity.custom_affinity": "", "ad_group_criterion.custom_audience.custom_audience": "", "ad_group_criterion.custom_intent.custom_intent": "", "ad_group_criterion.disapproval_reasons": [], "ad_group_criterion.display_name": "data integrations", "ad_group_criterion.effective_cpc_bid_micros": 1000000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 0, "ad_group_criterion.effective_cpv_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_percent_cpc_bid_micros": 0, "ad_group_criterion.effective_percent_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_url_suffix": "", "ad_group_criterion.final_urls": [], "ad_group_criterion.gender.type": "UNSPECIFIED", "ad_group_criterion.income_range.type": "UNSPECIFIED", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.keyword.text": "data integrations", "ad_group_criterion.labels": [], "ad_group_criterion.mobile_app_category.mobile_app_category_constant": "", "ad_group_criterion.mobile_application.app_id": "", "ad_group_criterion.mobile_application.name": "", "ad_group_criterion.negative": false, "ad_group_criterion.parental_status.type": "UNSPECIFIED", "ad_group_criterion.percent_cpc_bid_micros": 0, "ad_group_criterion.placement.url": "", "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.first_page_cpc_micros": 0, "ad_group_criterion.position_estimates.first_position_cpc_micros": 0, "ad_group_criterion.position_estimates.top_of_page_cpc_micros": 0, "ad_group_criterion.quality_info.creative_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.post_click_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.quality_score": 0, "ad_group_criterion.quality_info.search_predicted_ctr": "UNSPECIFIED", "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~18696703", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.system_serving_status": "ELIGIBLE", "ad_group_criterion.topic.path": [], "ad_group_criterion.topic.topic_constant": "", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.url_custom_parameters": [], "ad_group_criterion.user_interest.user_interest_category": "", "ad_group_criterion.user_list.user_list": "", "ad_group_criterion.webpage.conditions": [], "ad_group_criterion.webpage.coverage_percentage": 0.0, "ad_group_criterion.webpage.criterion_name": "", "ad_group_criterion.webpage.sample.sample_urls": [], "ad_group_criterion.youtube_channel.channel_id": "", "ad_group_criterion.youtube_video.video_id": ""}, "emitted_at": 1704407786207} +{"stream": "ad_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group.id": 117036054899, "ad_group_criterion.ad_group": "customers/4651612872/adGroups/117036054899", "ad_group_criterion.age_range.type": "UNSPECIFIED", "ad_group_criterion.app_payment_model.type": "UNSPECIFIED", "ad_group_criterion.approval_status": "APPROVED", "ad_group_criterion.audience.audience": "", "ad_group_criterion.bid_modifier": 0.0, "ad_group_criterion.combined_audience.combined_audience": "", "ad_group_criterion.cpc_bid_micros": 0, "ad_group_criterion.cpm_bid_micros": 0, "ad_group_criterion.cpv_bid_micros": 0, "ad_group_criterion.criterion_id": 376833662, "ad_group_criterion.custom_affinity.custom_affinity": "", "ad_group_criterion.custom_audience.custom_audience": "", "ad_group_criterion.custom_intent.custom_intent": "", "ad_group_criterion.disapproval_reasons": [], "ad_group_criterion.display_name": "data integration services", "ad_group_criterion.effective_cpc_bid_micros": 1000000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 0, "ad_group_criterion.effective_cpv_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_percent_cpc_bid_micros": 0, "ad_group_criterion.effective_percent_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_url_suffix": "", "ad_group_criterion.final_urls": [], "ad_group_criterion.gender.type": "UNSPECIFIED", "ad_group_criterion.income_range.type": "UNSPECIFIED", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.keyword.text": "data integration services", "ad_group_criterion.labels": [], "ad_group_criterion.mobile_app_category.mobile_app_category_constant": "", "ad_group_criterion.mobile_application.app_id": "", "ad_group_criterion.mobile_application.name": "", "ad_group_criterion.negative": false, "ad_group_criterion.parental_status.type": "UNSPECIFIED", "ad_group_criterion.percent_cpc_bid_micros": 0, "ad_group_criterion.placement.url": "", "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.first_page_cpc_micros": 0, "ad_group_criterion.position_estimates.first_position_cpc_micros": 0, "ad_group_criterion.position_estimates.top_of_page_cpc_micros": 0, "ad_group_criterion.quality_info.creative_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.post_click_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.quality_score": 0, "ad_group_criterion.quality_info.search_predicted_ctr": "UNSPECIFIED", "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~376833662", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.system_serving_status": "ELIGIBLE", "ad_group_criterion.topic.path": [], "ad_group_criterion.topic.topic_constant": "", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.url_custom_parameters": [], "ad_group_criterion.user_interest.user_interest_category": "", "ad_group_criterion.user_list.user_list": "", "ad_group_criterion.webpage.conditions": [], "ad_group_criterion.webpage.coverage_percentage": 0.0, "ad_group_criterion.webpage.criterion_name": "", "ad_group_criterion.webpage.sample.sample_urls": [], "ad_group_criterion.youtube_channel.channel_id": "", "ad_group_criterion.youtube_video.video_id": ""}, "emitted_at": 1704407786208} +{"stream": "ad_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group.id": 117036054899, "ad_group_criterion.ad_group": "customers/4651612872/adGroups/117036054899", "ad_group_criterion.age_range.type": "UNSPECIFIED", "ad_group_criterion.app_payment_model.type": "UNSPECIFIED", "ad_group_criterion.approval_status": "APPROVED", "ad_group_criterion.audience.audience": "", "ad_group_criterion.bid_modifier": 0.0, "ad_group_criterion.combined_audience.combined_audience": "", "ad_group_criterion.cpc_bid_micros": 0, "ad_group_criterion.cpm_bid_micros": 0, "ad_group_criterion.cpv_bid_micros": 0, "ad_group_criterion.criterion_id": 13099056325, "ad_group_criterion.custom_affinity.custom_affinity": "", "ad_group_criterion.custom_audience.custom_audience": "", "ad_group_criterion.custom_intent.custom_intent": "", "ad_group_criterion.disapproval_reasons": [], "ad_group_criterion.display_name": "cloud data integration", "ad_group_criterion.effective_cpc_bid_micros": 1000000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 0, "ad_group_criterion.effective_cpv_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_percent_cpc_bid_micros": 0, "ad_group_criterion.effective_percent_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_url_suffix": "", "ad_group_criterion.final_urls": [], "ad_group_criterion.gender.type": "UNSPECIFIED", "ad_group_criterion.income_range.type": "UNSPECIFIED", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.keyword.text": "cloud data integration", "ad_group_criterion.labels": [], "ad_group_criterion.mobile_app_category.mobile_app_category_constant": "", "ad_group_criterion.mobile_application.app_id": "", "ad_group_criterion.mobile_application.name": "", "ad_group_criterion.negative": false, "ad_group_criterion.parental_status.type": "UNSPECIFIED", "ad_group_criterion.percent_cpc_bid_micros": 0, "ad_group_criterion.placement.url": "", "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.first_page_cpc_micros": 0, "ad_group_criterion.position_estimates.first_position_cpc_micros": 0, "ad_group_criterion.position_estimates.top_of_page_cpc_micros": 0, "ad_group_criterion.quality_info.creative_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.post_click_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.quality_score": 0, "ad_group_criterion.quality_info.search_predicted_ctr": "UNSPECIFIED", "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~13099056325", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.system_serving_status": "ELIGIBLE", "ad_group_criterion.topic.path": [], "ad_group_criterion.topic.topic_constant": "", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.url_custom_parameters": [], "ad_group_criterion.user_interest.user_interest_category": "", "ad_group_criterion.user_list.user_list": "", "ad_group_criterion.webpage.conditions": [], "ad_group_criterion.webpage.coverage_percentage": 0.0, "ad_group_criterion.webpage.criterion_name": "", "ad_group_criterion.webpage.sample.sample_urls": [], "ad_group_criterion.youtube_channel.channel_id": "", "ad_group_criterion.youtube_video.video_id": ""}, "emitted_at": 1704407786209} +{"stream": "ad_listing_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~18696703", "ad_group.id": 117036054899, "ad_group_criterion.criterion_id": 18696703, "ad_group_criterion.listing_group.case_value.activity_country.value": "", "ad_group_criterion.listing_group.case_value.activity_id.value": "", "ad_group_criterion.listing_group.case_value.activity_rating.value": 0, "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_class.value": 0, "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_id.value": "", "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": "", "ad_group_criterion.listing_group.case_value.product_category.category_id": 0, "ad_group_criterion.listing_group.case_value.product_category.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_brand.value": "", "ad_group_criterion.listing_group.case_value.product_channel.channel": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_condition.condition": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": "", "ad_group_criterion.listing_group.case_value.product_item_id.value": "", "ad_group_criterion.listing_group.case_value.product_type.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_type.value": "", "ad_group_criterion.listing_group.parent_ad_group_criterion": "", "ad_group_criterion.listing_group.type": "UNSPECIFIED"}, "emitted_at": 1704407823748} +{"stream": "ad_listing_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~376833662", "ad_group.id": 117036054899, "ad_group_criterion.criterion_id": 376833662, "ad_group_criterion.listing_group.case_value.activity_country.value": "", "ad_group_criterion.listing_group.case_value.activity_id.value": "", "ad_group_criterion.listing_group.case_value.activity_rating.value": 0, "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_class.value": 0, "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_id.value": "", "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": "", "ad_group_criterion.listing_group.case_value.product_category.category_id": 0, "ad_group_criterion.listing_group.case_value.product_category.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_brand.value": "", "ad_group_criterion.listing_group.case_value.product_channel.channel": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_condition.condition": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": "", "ad_group_criterion.listing_group.case_value.product_item_id.value": "", "ad_group_criterion.listing_group.case_value.product_type.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_type.value": "", "ad_group_criterion.listing_group.parent_ad_group_criterion": "", "ad_group_criterion.listing_group.type": "UNSPECIFIED"}, "emitted_at": 1704407823749} +{"stream": "ad_listing_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~13099056325", "ad_group.id": 117036054899, "ad_group_criterion.criterion_id": 13099056325, "ad_group_criterion.listing_group.case_value.activity_country.value": "", "ad_group_criterion.listing_group.case_value.activity_id.value": "", "ad_group_criterion.listing_group.case_value.activity_rating.value": 0, "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_class.value": 0, "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_id.value": "", "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": "", "ad_group_criterion.listing_group.case_value.product_category.category_id": 0, "ad_group_criterion.listing_group.case_value.product_category.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_brand.value": "", "ad_group_criterion.listing_group.case_value.product_channel.channel": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_condition.condition": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": "", "ad_group_criterion.listing_group.case_value.product_item_id.value": "", "ad_group_criterion.listing_group.case_value.product_type.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_type.value": "", "ad_group_criterion.listing_group.parent_ad_group_criterion": "", "ad_group_criterion.listing_group.type": "UNSPECIFIED"}, "emitted_at": 1704407823750} +{"stream": "ad_group_criterion_label", "data": {"ad_group.id": 137051662444, "label.id": 21902092838, "ad_group_criterion_label.ad_group_criterion": "customers/4651612872/adGroupCriteria/137051662444~10766861", "ad_group_criterion_label.label": "customers/4651612872/labels/21902092838", "ad_group_criterion_label.resource_name": "customers/4651612872/adGroupCriterionLabels/137051662444~10766861~21902092838", "ad_group_criterion.criterion_id": 10766861}, "emitted_at": 1704407848182} +{"stream": "ad_group_criterion_label", "data": {"ad_group.id": 137051662444, "label.id": 21906377810, "ad_group_criterion_label.ad_group_criterion": "customers/4651612872/adGroupCriteria/137051662444~528912986", "ad_group_criterion_label.label": "customers/4651612872/labels/21906377810", "ad_group_criterion_label.resource_name": "customers/4651612872/adGroupCriterionLabels/137051662444~528912986~21906377810", "ad_group_criterion.criterion_id": 528912986}, "emitted_at": 1704407848186} +{"stream": "campaign_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "campaign.id": 9660123292, "campaign_criterion.resource_name": "customers/4651612872/campaignCriteria/9660123292~2124", "campaign_criterion.campaign": "customers/4651612872/campaigns/9660123292", "campaign_criterion.age_range.type": "UNSPECIFIED", "campaign_criterion.mobile_application.name": "", "campaign_criterion.negative": false, "campaign_criterion.youtube_channel.channel_id": "", "campaign_criterion.youtube_video.video_id": ""}, "emitted_at": 1704407849655} +{"stream": "campaign_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "campaign.id": 9660123292, "campaign_criterion.resource_name": "customers/4651612872/campaignCriteria/9660123292~2250", "campaign_criterion.campaign": "customers/4651612872/campaigns/9660123292", "campaign_criterion.age_range.type": "UNSPECIFIED", "campaign_criterion.mobile_application.name": "", "campaign_criterion.negative": false, "campaign_criterion.youtube_channel.channel_id": "", "campaign_criterion.youtube_video.video_id": ""}, "emitted_at": 1704407849656} +{"stream": "campaign_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "campaign.id": 9660123292, "campaign_criterion.resource_name": "customers/4651612872/campaignCriteria/9660123292~2276", "campaign_criterion.campaign": "customers/4651612872/campaigns/9660123292", "campaign_criterion.age_range.type": "UNSPECIFIED", "campaign_criterion.mobile_application.name": "", "campaign_criterion.negative": false, "campaign_criterion.youtube_channel.channel_id": "", "campaign_criterion.youtube_video.video_id": ""}, "emitted_at": 1704407849656} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records_click.jsonl b/airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records_click.jsonl new file mode 100644 index 000000000000..46f5b1b428ad --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records_click.jsonl @@ -0,0 +1,69 @@ +{"stream": "ad_group_ad_legacy", "data": {"ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group.id": 155311392438, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "ad_group_ad.ad_group": "customers/4651612872/adGroups/155311392438", "ad_group.name": "Airbyte", "ad_group.status": "ENABLED", "segments.ad_network_type": "SEARCH", "ad_group_ad.ad_strength": "EXCELLENT", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "metrics.all_conversions_from_interactions_rate": 4.9339207, "metrics.all_conversions_value": 803.783622, "metrics.all_conversions": 148.017621, "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.added_by_google_ads": false, "metrics.average_cost": 5602666.666666667, "metrics.average_cpc": 5602666.666666667, "metrics.average_cpe": 0.0, "metrics.average_cpm": 1031165644.1717792, "metrics.average_cpv": 0.0, "metrics.average_page_views": 0.0, "metrics.average_time_on_site": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/155311392438", "campaign.base_campaign": "customers/4651612872/campaigns/20643300404", "metrics.bounce_rate": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "campaign.id": 20643300404, "campaign.name": "mm_search_brand", "campaign.status": "PAUSED", "metrics.clicks": 30, "ad_group_ad.policy_summary.approval_status": "APPROVED", "metrics.conversions_from_interactions_rate": 0.22079663333333333, "metrics.conversions_value": 662.3899, "metrics.conversions": 6.623899, "metrics.cost_micros": 168080000, "metrics.cost_per_all_conversions": 1135540.4773057392, "metrics.cost_per_conversion": 25374783.039415307, "metrics.cost_per_current_model_attributed_conversion": 25374783.039415307, "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_urls": ["https://airbyte.com/"], "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.url_custom_parameters": [], "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.18404907975460122, "metrics.current_model_attributed_conversions_value": 662.3899, "metrics.current_model_attributed_conversions": 6.623899, "segments.date": "2023-12-31", "segments.day_of_week": "SUNDAY", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_url": "", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "customer.id": 4651612872, "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.id": 676665180945, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "metrics.impressions": 163, "metrics.interaction_rate": 0.18404907975460122, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 30, "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "segments.month": "2023-12-01", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "metrics.percent_new_visitors": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "segments.quarter": "2023-10-01", "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"The most comprehensive catalog of connectors, trusted by 40,000K engineers\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"A high-performing and scalable data integration platform with advanced features.\"\nasset_performance_label: BEST\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Try Airbyte for free! Connect Any Data, Any User, & Any Application Effortlessly.\"\nasset_performance_label: BEST\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Build custom connectors in 10 min with our no-code connector builder.\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"The only ETL tool you need\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Build ELT Pipelines In Minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"No code, ELT Tool\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Replicate Data in Minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open Source Integration\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Try Airbyte Cloud Free\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Browse Our Catalog\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Trusted by over 40K Engineers\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"14 Day Free Trial\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"300+ off-the-shelf connectors\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"No-Code Connector Builder\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Get started in minutes\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Extract, Load & Transform\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Streamlined Data Pipeline\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.status": "ENABLED", "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "metrics.top_impression_percentage": 0.8834355828220859, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "metrics.value_per_all_conversions": 5.430323880154783, "metrics.value_per_conversion": 100.0, "metrics.value_per_current_model_attributed_conversion": 100.0, "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2023-12-25", "segments.year": 2023}, "emitted_at": 1705321896341} +{"stream": "ad_group_ad_legacy", "data": {"ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group.id": 155311392438, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "ad_group_ad.ad_group": "customers/4651612872/adGroups/155311392438", "ad_group.name": "Airbyte", "ad_group.status": "ENABLED", "segments.ad_network_type": "SEARCH", "ad_group_ad.ad_strength": "EXCELLENT", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "metrics.all_conversions_from_interactions_rate": 5.129510513513513, "metrics.all_conversions_value": 1315.049854, "metrics.all_conversions": 189.791889, "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.added_by_google_ads": false, "metrics.average_cost": 8890000.0, "metrics.average_cpc": 8890000.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 1279883268.4824903, "metrics.average_cpv": 0.0, "metrics.average_page_views": 0.0, "metrics.average_time_on_site": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/155311392438", "campaign.base_campaign": "customers/4651612872/campaigns/20643300404", "metrics.bounce_rate": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "campaign.id": 20643300404, "campaign.name": "mm_search_brand", "campaign.status": "PAUSED", "metrics.clicks": 37, "ad_group_ad.policy_summary.approval_status": "APPROVED", "metrics.conversions_from_interactions_rate": 0.35933878378378376, "metrics.conversions_value": 1129.5535, "metrics.conversions": 13.295535, "metrics.cost_micros": 328930000, "metrics.cost_per_all_conversions": 1733108.8369113603, "metrics.cost_per_conversion": 24739884.480015285, "metrics.cost_per_current_model_attributed_conversion": 24739884.480015285, "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_urls": ["https://airbyte.com/"], "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.url_custom_parameters": [], "metrics.cross_device_conversions": 3.0, "metrics.ctr": 0.14396887159533073, "metrics.current_model_attributed_conversions_value": 1129.5535, "metrics.current_model_attributed_conversions": 13.295535, "segments.date": "2024-01-01", "segments.day_of_week": "MONDAY", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_url": "", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "customer.id": 4651612872, "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.id": 676665180945, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "metrics.impressions": 257, "metrics.interaction_rate": 0.14396887159533073, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 37, "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "segments.month": "2024-01-01", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "metrics.percent_new_visitors": 0.0, "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "segments.quarter": "2024-01-01", "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"The most comprehensive catalog of connectors, trusted by 40,000K engineers\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"A high-performing and scalable data integration platform with advanced features.\"\nasset_performance_label: BEST\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Try Airbyte for free! Connect Any Data, Any User, & Any Application Effortlessly.\"\nasset_performance_label: BEST\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Build custom connectors in 10 min with our no-code connector builder.\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"The only ETL tool you need\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Build ELT Pipelines In Minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"No code, ELT Tool\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Replicate Data in Minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open Source Integration\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Try Airbyte Cloud Free\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Browse Our Catalog\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Trusted by over 40K Engineers\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"14 Day Free Trial\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"300+ off-the-shelf connectors\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"No-Code Connector Builder\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Get started in minutes\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Extract, Load & Transform\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Streamlined Data Pipeline\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.status": "ENABLED", "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "metrics.top_impression_percentage": 0.9688715953307393, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "metrics.value_per_all_conversions": 6.928904395909143, "metrics.value_per_conversion": 84.9573559845467, "metrics.value_per_current_model_attributed_conversion": 84.9573559845467, "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2024-01-01", "segments.year": 2024}, "emitted_at": 1705321896352} +{"stream": "campaign_budget", "data": {"customer.id": 4651612872, "campaign.id": 20643300404, "campaign_budget.aligned_bidding_strategy_id": 0, "campaign_budget.amount_micros": 330000000, "campaign_budget.delivery_method": "STANDARD", "campaign_budget.explicitly_shared": false, "campaign_budget.has_recommended_budget": false, "campaign_budget.id": 13022493317, "campaign_budget.name": "mm_search_brand", "campaign_budget.period": "DAILY", "campaign_budget.recommended_budget_amount_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_clicks": 0, "campaign_budget.recommended_budget_estimated_change_weekly_cost_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_interactions": 0, "campaign_budget.recommended_budget_estimated_change_weekly_views": 0, "campaign_budget.reference_count": 1, "campaign_budget.resource_name": "customers/4651612872/campaignBudgets/13022493317", "campaign_budget.status": "ENABLED", "campaign_budget.total_amount_micros": 0, "campaign_budget.type": "STANDARD", "segments.date": "2023-12-31", "segments.budget_campaign_association_status.campaign": "customers/4651612872/campaigns/20643300404", "segments.budget_campaign_association_status.status": "ENABLED", "metrics.all_conversions": 148.017621, "metrics.all_conversions_from_interactions_rate": 4.9339207, "metrics.all_conversions_value": 803.783622, "metrics.average_cost": 5602666.666666667, "metrics.average_cpc": 5602666.666666667, "metrics.average_cpe": 0.0, "metrics.average_cpm": 1031165644.1717792, "metrics.average_cpv": 0.0, "metrics.clicks": 30, "metrics.conversions": 6.623899, "metrics.conversions_from_interactions_rate": 0.22079663333333333, "metrics.conversions_value": 662.3899, "metrics.cost_micros": 168080000, "metrics.cost_per_all_conversions": 1135540.4773057392, "metrics.cost_per_conversion": 25374783.039415307, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.18404907975460122, "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "metrics.impressions": 163, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interaction_rate": 0.18404907975460122, "metrics.interactions": 30, "metrics.value_per_all_conversions": 5.430323880154783, "metrics.value_per_conversion": 100.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1705322166925} +{"stream": "campaign_budget", "data": {"customer.id": 4651612872, "campaign.id": 20643300404, "campaign_budget.aligned_bidding_strategy_id": 0, "campaign_budget.amount_micros": 330000000, "campaign_budget.delivery_method": "STANDARD", "campaign_budget.explicitly_shared": false, "campaign_budget.has_recommended_budget": false, "campaign_budget.id": 13022493317, "campaign_budget.name": "mm_search_brand", "campaign_budget.period": "DAILY", "campaign_budget.recommended_budget_amount_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_clicks": 0, "campaign_budget.recommended_budget_estimated_change_weekly_cost_micros": 0, "campaign_budget.recommended_budget_estimated_change_weekly_interactions": 0, "campaign_budget.recommended_budget_estimated_change_weekly_views": 0, "campaign_budget.reference_count": 1, "campaign_budget.resource_name": "customers/4651612872/campaignBudgets/13022493317", "campaign_budget.status": "ENABLED", "campaign_budget.total_amount_micros": 0, "campaign_budget.type": "STANDARD", "segments.date": "2024-01-01", "segments.budget_campaign_association_status.campaign": "customers/4651612872/campaigns/20643300404", "segments.budget_campaign_association_status.status": "ENABLED", "metrics.all_conversions": 189.791889, "metrics.all_conversions_from_interactions_rate": 5.129510513513513, "metrics.all_conversions_value": 1315.049854, "metrics.average_cost": 8890000.0, "metrics.average_cpc": 8890000.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 1279883268.4824903, "metrics.average_cpv": 0.0, "metrics.clicks": 37, "metrics.conversions": 13.295535, "metrics.conversions_from_interactions_rate": 0.35933878378378376, "metrics.conversions_value": 1129.5535, "metrics.cost_micros": 328930000, "metrics.cost_per_all_conversions": 1733108.8369113603, "metrics.cost_per_conversion": 24739884.480015285, "metrics.cross_device_conversions": 3.0, "metrics.ctr": 0.14396887159533073, "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "metrics.impressions": 257, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interaction_rate": 0.14396887159533073, "metrics.interactions": 37, "metrics.value_per_all_conversions": 6.928904395909143, "metrics.value_per_conversion": 84.9573559845467, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1705322166935} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2024-01-03"}, "emitted_at": 1704408105935} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n", "targeting_dimension: TOPIC\nbid_only: true\n"], "segments.date": "2024-01-03"}, "emitted_at": 1704408105942} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2024-01-02"}, "emitted_at": 1704408105943} +{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "segments.ad_network_type": "SEARCH", "metrics.all_conversions_from_interactions_rate": 2.9861930909090906, "metrics.all_conversions_value": 32.848124, "metrics.all_conversions": 32.848124, "metrics.average_cost": 1398181.8181818181, "metrics.average_cpc": 1398181.8181818181, "metrics.average_cpe": 0.0, "metrics.average_cpm": 640833333.3333334, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 11, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0, "metrics.content_rank_lost_impression_share": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 15380000, "metrics.cost_per_all_conversions": 468215.4755626227, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.4583333333333333, "segments.date": "2023-12-31", "segments.day_of_week": "SUNDAY", "segments.device": "MOBILE", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 24, "metrics.interaction_rate": 0.4583333333333333, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.interactions": 11, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2023-12-01", "segments.quarter": "2023-10-01", "metrics.search_budget_lost_impression_share": 0.0, "metrics.search_exact_match_impression_share": 0.6666666666666666, "metrics.search_impression_share": 0.6153846153846154, "metrics.search_rank_lost_impression_share": 0.38461538461538464, "metrics.value_per_all_conversions": 1.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2023-12-25", "segments.year": 2023}, "emitted_at": 1704408106623} +{"stream": "account_performance_report", "data": {"customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "segments.ad_network_type": "SEARCH", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "customer.manager": false, "metrics.clicks": 0, "metrics.content_budget_lost_impression_share": 0.0, "metrics.content_impression_share": 0.0, "metrics.content_rank_lost_impression_share": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.date": "2023-12-31", "segments.day_of_week": "SUNDAY", "segments.device": "TABLET", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "metrics.impressions": 2, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "customer.auto_tagging_enabled": true, "customer.test_account": false, "segments.month": "2023-12-01", "segments.quarter": "2023-10-01", "metrics.search_budget_lost_impression_share": 0.0, "metrics.search_exact_match_impression_share": 1.0, "metrics.search_impression_share": 1.0, "metrics.search_rank_lost_impression_share": 0.0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2023-12-25", "segments.year": 2023}, "emitted_at": 1704408106623} +{"stream": "click_view", "data": {"ad_group.name": "Airbyte", "click_view.gclid": "Cj0KCQiAv8SsBhC7ARIsALIkVT0aoRchs-JIhSNfsaUU1GQLPOaNU15XNhGEkNLQ0kpOpYoV_VDNNogaAl-2EALw_wcB", "click_view.ad_group_ad": "customers/4651612872/adGroupAds/155311392438~676665180945", "click_view.keyword": "", "click_view.keyword_info.match_type": "UNSPECIFIED", "click_view.keyword_info.text": "", "campaign.id": 20643300404, "ad_group.id": 155311392438, "segments.date": "2023-12-31", "customer.id": 4651612872, "campaign.name": "mm_search_brand", "segments.ad_network_type": "SEARCH", "campaign.network_settings.target_content_network": false, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": false}, "emitted_at": 1704408107339} +{"stream": "click_view", "data": {"ad_group.name": "Airbyte", "click_view.gclid": "Cj0KCQiAv8SsBhC7ARIsALIkVT17gRC4RsmoYczHLguLKTaojzCB4bPA0GjBSa3x44kKTbWVCvXEe58aAkeHEALw_wcB", "click_view.ad_group_ad": "customers/4651612872/adGroupAds/155311392438~676665180945", "click_view.keyword": "", "click_view.keyword_info.match_type": "UNSPECIFIED", "click_view.keyword_info.text": "", "campaign.id": 20643300404, "ad_group.id": 155311392438, "segments.date": "2023-12-31", "customer.id": 4651612872, "campaign.name": "mm_search_brand", "segments.ad_network_type": "SEARCH", "campaign.network_settings.target_content_network": false, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": false}, "emitted_at": 1704408107340} +{"stream": "click_view", "data": {"ad_group.name": "Airbyte", "click_view.gclid": "Cj0KCQiAv8SsBhC7ARIsALIkVT1H36_GC-jRtw1xNj-9Y5IdIZWa-1j-BqhYt5JSB82QzNE5-7OxgB4aAlU4EALw_wcB", "click_view.ad_group_ad": "customers/4651612872/adGroupAds/155311392438~676665180945", "click_view.keyword": "", "click_view.keyword_info.match_type": "UNSPECIFIED", "click_view.keyword_info.text": "", "campaign.id": 20643300404, "ad_group.id": 155311392438, "segments.date": "2023-12-31", "customer.id": 4651612872, "campaign.name": "mm_search_brand", "segments.ad_network_type": "SEARCH", "campaign.network_settings.target_content_network": false, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": false}, "emitted_at": 1704408107340} +{"stream": "geographic_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "geographic_view.country_criterion_id": 2840, "geographic_view.location_type": "AREA_OF_INTEREST", "ad_group.id": 155311392438, "segments.date": "2023-12-31"}, "emitted_at": 1704408109676} +{"stream": "geographic_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "geographic_view.country_criterion_id": 2840, "geographic_view.location_type": "LOCATION_OF_PRESENCE", "ad_group.id": 155311392438, "segments.date": "2023-12-31"}, "emitted_at": 1704408109677} +{"stream": "geographic_view", "data": {"customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "geographic_view.country_criterion_id": 2840, "geographic_view.location_type": "AREA_OF_INTEREST", "ad_group.id": 155311392438, "segments.date": "2024-01-01"}, "emitted_at": 1704408109677} +{"stream": "topic_view", "data": {"topic_view.resource_name": "customers/4651612872/topicViews/144799120517~945751797", "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 264196.96969696967, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 66, "metrics.active_view_measurability": 1.0, "metrics.active_view_measurable_cost_micros": 17437, "metrics.active_view_measurable_impressions": 90, "metrics.active_view_viewability": 0.7333333333333333, "ad_group.id": 144799120517, "ad_group.name": "Ad group 1", "ad_group.status": "ENABLED", "segments.ad_network_type": "CONTENT", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 193744.44444444444, "metrics.average_cpv": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/144799120517", "campaign.base_campaign": "customers/4651612872/campaigns/19410069806", "ad_group_criterion.bid_modifier": 0.0, "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MANUAL_CPM", "campaign.id": 19410069806, "campaign.name": "Brand awareness and reach-Display-1", "campaign.status": "PAUSED", "metrics.clicks": 0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 17437, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "ad_group_criterion.effective_cpc_bid_micros": 10000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 2000000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.topic.path": ["", "Online Communities"], "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.date": "2024-01-03", "segments.day_of_week": "WEDNESDAY", "segments.device": "DESKTOP", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_urls": [], "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_criterion.criterion_id": 945751797, "metrics.impressions": 90, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "ad_group_criterion.negative": false, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n", "targeting_dimension: TOPIC\nbid_only: true\n"], "segments.month": "2024-01-01", "segments.quarter": "2024-01-01", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.url_custom_parameters": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "ad_group_criterion.topic.topic_constant": "topicConstants/299", "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2024-01-01", "segments.year": 2024}, "emitted_at": 1704408113977} +{"stream": "topic_view", "data": {"topic_view.resource_name": "customers/4651612872/topicViews/144799120517~1543464477", "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 862000.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 2, "metrics.active_view_measurability": 1.0, "metrics.active_view_measurable_cost_micros": 1724, "metrics.active_view_measurable_impressions": 4, "metrics.active_view_viewability": 0.5, "ad_group.id": 144799120517, "ad_group.name": "Ad group 1", "ad_group.status": "ENABLED", "segments.ad_network_type": "CONTENT", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 431000.0, "metrics.average_cpv": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/144799120517", "campaign.base_campaign": "customers/4651612872/campaigns/19410069806", "ad_group_criterion.bid_modifier": 0.0, "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MANUAL_CPM", "campaign.id": 19410069806, "campaign.name": "Brand awareness and reach-Display-1", "campaign.status": "PAUSED", "metrics.clicks": 0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 1724, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "ad_group_criterion.effective_cpc_bid_micros": 10000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 2000000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.topic.path": ["", "Shopping"], "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.date": "2024-01-03", "segments.day_of_week": "WEDNESDAY", "segments.device": "DESKTOP", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_urls": [], "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_criterion.criterion_id": 1543464477, "metrics.impressions": 4, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "ad_group_criterion.negative": false, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n", "targeting_dimension: TOPIC\nbid_only: true\n"], "segments.month": "2024-01-01", "segments.quarter": "2024-01-01", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.url_custom_parameters": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "ad_group_criterion.topic.topic_constant": "topicConstants/18", "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2024-01-01", "segments.year": 2024}, "emitted_at": 1704408113979} +{"stream": "topic_view", "data": {"topic_view.resource_name": "customers/4651612872/topicViews/144799120517~1543465137", "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "metrics.active_view_cpm": 338986.6666666667, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 75, "metrics.active_view_measurability": 1.0, "metrics.active_view_measurable_cost_micros": 25424, "metrics.active_view_measurable_impressions": 104, "metrics.active_view_viewability": 0.7211538461538461, "ad_group.id": 144799120517, "ad_group.name": "Ad group 1", "ad_group.status": "ENABLED", "segments.ad_network_type": "CONTENT", "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.all_conversions": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpe": 0.0, "metrics.average_cpm": 244461.53846153844, "metrics.average_cpv": 0.0, "ad_group.base_ad_group": "customers/4651612872/adGroups/144799120517", "campaign.base_campaign": "customers/4651612872/campaigns/19410069806", "ad_group_criterion.bid_modifier": 0.0, "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MANUAL_CPM", "campaign.id": 19410069806, "campaign.name": "Brand awareness and reach-Display-1", "campaign.status": "PAUSED", "metrics.clicks": 0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.conversions": 0.0, "metrics.cost_micros": 25424, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "ad_group_criterion.effective_cpc_bid_micros": 10000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 2000000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.topic.path": ["", "Arts & Entertainment"], "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "segments.date": "2024-01-03", "segments.day_of_week": "WEDNESDAY", "segments.device": "DESKTOP", "metrics.engagement_rate": 0.0, "metrics.engagements": 0, "customer.id": 4651612872, "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_urls": [], "metrics.gmail_forwards": 0, "metrics.gmail_saves": 0, "metrics.gmail_secondary_clicks": 0, "ad_group_criterion.criterion_id": 1543465137, "metrics.impressions": 104, "metrics.interaction_rate": 0.0, "metrics.interaction_event_types": [], "metrics.interactions": 0, "ad_group_criterion.negative": false, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n", "targeting_dimension: TOPIC\nbid_only: true\n"], "segments.month": "2024-01-01", "segments.quarter": "2024-01-01", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.url_custom_parameters": [], "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "ad_group_criterion.topic.topic_constant": "topicConstants/3", "metrics.video_quartile_p100_rate": 0.0, "metrics.video_quartile_p25_rate": 0.0, "metrics.video_quartile_p50_rate": 0.0, "metrics.video_quartile_p75_rate": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0, "segments.week": "2024-01-01", "segments.year": 2024}, "emitted_at": 1704408113986} +{"stream": "ad_group_ad", "data": {"ad_group.id": 155311392438, "ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com/"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 676665180945, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/676665180945", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"The most comprehensive catalog of connectors, trusted by 40,000K engineers\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"A high-performing and scalable data integration platform with advanced features.\"\nasset_performance_label: BEST\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Try Airbyte for free! Connect Any Data, Any User, & Any Application Effortlessly.\"\nasset_performance_label: BEST\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Build custom connectors in 10 min with our no-code connector builder.\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"The only ETL tool you need\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Build ELT Pipelines In Minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"No code, ELT Tool\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Replicate Data in Minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open Source Integration\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Try Airbyte Cloud Free\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Browse Our Catalog\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Trusted by over 40K Engineers\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"14 Day Free Trial\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"300+ off-the-shelf connectors\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"No-Code Connector Builder\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Get started in minutes\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Extract, Load & Transform\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Streamlined Data Pipeline\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/155311392438", "ad_group_ad.ad_strength": "EXCELLENT", "ad_group_ad.labels": [], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/155311392438~676665180945", "ad_group_ad.status": "ENABLED", "segments.date": "2023-12-31"}, "emitted_at": 1704408116313} +{"stream": "ad_group_ad", "data": {"ad_group.id": 155311392438, "ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com/"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 676665180945, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/676665180945", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"The most comprehensive catalog of connectors, trusted by 40,000K engineers\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"A high-performing and scalable data integration platform with advanced features.\"\nasset_performance_label: BEST\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Try Airbyte for free! Connect Any Data, Any User, & Any Application Effortlessly.\"\nasset_performance_label: BEST\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Build custom connectors in 10 min with our no-code connector builder.\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"The only ETL tool you need\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Build ELT Pipelines In Minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"No code, ELT Tool\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Replicate Data in Minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open Source Integration\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Try Airbyte Cloud Free\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Browse Our Catalog\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Trusted by over 40K Engineers\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"14 Day Free Trial\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"300+ off-the-shelf connectors\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"No-Code Connector Builder\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Get started in minutes\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Extract, Load & Transform\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Streamlined Data Pipeline\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/155311392438", "ad_group_ad.ad_strength": "EXCELLENT", "ad_group_ad.labels": [], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/155311392438~676665180945", "ad_group_ad.status": "ENABLED", "segments.date": "2024-01-01"}, "emitted_at": 1704408116319} +{"stream": "ad_group_ad", "data": {"ad_group.id": 155311392438, "ad_group_ad.ad.added_by_google_ads": false, "ad_group_ad.ad.app_ad.descriptions": [], "ad_group_ad.ad.app_ad.headlines": [], "ad_group_ad.ad.app_ad.html5_media_bundles": [], "ad_group_ad.ad.app_ad.images": [], "ad_group_ad.ad.app_ad.mandatory_ad_text": "", "ad_group_ad.ad.app_ad.youtube_videos": [], "ad_group_ad.ad.app_engagement_ad.descriptions": [], "ad_group_ad.ad.app_engagement_ad.headlines": [], "ad_group_ad.ad.app_engagement_ad.images": [], "ad_group_ad.ad.app_engagement_ad.videos": [], "ad_group_ad.ad.call_ad.business_name": "", "ad_group_ad.ad.call_ad.call_tracked": false, "ad_group_ad.ad.call_ad.conversion_action": "", "ad_group_ad.ad.call_ad.conversion_reporting_state": "UNSPECIFIED", "ad_group_ad.ad.call_ad.country_code": "", "ad_group_ad.ad.call_ad.description1": "", "ad_group_ad.ad.call_ad.description2": "", "ad_group_ad.ad.call_ad.disable_call_conversion": false, "ad_group_ad.ad.call_ad.headline1": "", "ad_group_ad.ad.call_ad.headline2": "", "ad_group_ad.ad.call_ad.path1": "", "ad_group_ad.ad.call_ad.path2": "", "ad_group_ad.ad.call_ad.phone_number": "", "ad_group_ad.ad.call_ad.phone_number_verification_url": "", "ad_group_ad.ad.device_preference": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.display_upload_product_type": "UNSPECIFIED", "ad_group_ad.ad.display_upload_ad.media_bundle": "", "ad_group_ad.ad.display_url": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description": "", "ad_group_ad.ad.expanded_dynamic_search_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.description": "", "ad_group_ad.ad.expanded_text_ad.description2": "", "ad_group_ad.ad.expanded_text_ad.headline_part1": "", "ad_group_ad.ad.expanded_text_ad.headline_part2": "", "ad_group_ad.ad.expanded_text_ad.headline_part3": "", "ad_group_ad.ad.expanded_text_ad.path1": "", "ad_group_ad.ad.expanded_text_ad.path2": "", "ad_group_ad.ad.final_app_urls": [], "ad_group_ad.ad.final_mobile_urls": [], "ad_group_ad.ad.final_url_suffix": "", "ad_group_ad.ad.final_urls": ["https://airbyte.com/"], "ad_group_ad.ad.hotel_ad": "", "ad_group_ad.ad.id": 676665180945, "ad_group_ad.ad.image_ad.image_url": "", "ad_group_ad.ad.image_ad.mime_type": "UNSPECIFIED", "ad_group_ad.ad.image_ad.name": "", "ad_group_ad.ad.image_ad.pixel_height": 0, "ad_group_ad.ad.image_ad.pixel_width": 0, "ad_group_ad.ad.image_ad.preview_image_url": "", "ad_group_ad.ad.image_ad.preview_pixel_height": 0, "ad_group_ad.ad.image_ad.preview_pixel_width": 0, "ad_group_ad.ad.legacy_app_install_ad": "", "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.legacy_responsive_display_ad.business_name": "", "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.description": "", "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.main_color": "", "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": "", "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": "", "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": "", "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": "", "ad_group_ad.ad.local_ad.call_to_actions": [], "ad_group_ad.ad.local_ad.descriptions": [], "ad_group_ad.ad.local_ad.headlines": [], "ad_group_ad.ad.local_ad.logo_images": [], "ad_group_ad.ad.local_ad.marketing_images": [], "ad_group_ad.ad.local_ad.path1": "", "ad_group_ad.ad.local_ad.path2": "", "ad_group_ad.ad.local_ad.videos": [], "ad_group_ad.ad.name": "", "ad_group_ad.ad.resource_name": "customers/4651612872/ads/676665180945", "ad_group_ad.ad.responsive_display_ad.accent_color": "", "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": false, "ad_group_ad.ad.responsive_display_ad.business_name": "", "ad_group_ad.ad.responsive_display_ad.call_to_action_text": "", "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": false, "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": false, "ad_group_ad.ad.responsive_display_ad.descriptions": [], "ad_group_ad.ad.responsive_display_ad.format_setting": "UNSPECIFIED", "ad_group_ad.ad.responsive_display_ad.headlines": [], "ad_group_ad.ad.responsive_display_ad.logo_images": [], "ad_group_ad.ad.responsive_display_ad.long_headline": "", "ad_group_ad.ad.responsive_display_ad.main_color": "", "ad_group_ad.ad.responsive_display_ad.marketing_images": [], "ad_group_ad.ad.responsive_display_ad.price_prefix": "", "ad_group_ad.ad.responsive_display_ad.promo_text": "", "ad_group_ad.ad.responsive_display_ad.square_logo_images": [], "ad_group_ad.ad.responsive_display_ad.square_marketing_images": [], "ad_group_ad.ad.responsive_display_ad.youtube_videos": [], "ad_group_ad.ad.responsive_search_ad.descriptions": ["text: \"The most comprehensive catalog of connectors, trusted by 40,000K engineers\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"A high-performing and scalable data integration platform with advanced features.\"\nasset_performance_label: BEST\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Try Airbyte for free! Connect Any Data, Any User, & Any Application Effortlessly.\"\nasset_performance_label: BEST\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Build custom connectors in 10 min with our no-code connector builder.\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.headlines": ["text: \"Airbyte\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"The only ETL tool you need\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Build ELT Pipelines In Minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"No code, ELT Tool\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Replicate Data in Minutes\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Open Source Integration\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Try Airbyte Cloud Free\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Browse Our Catalog\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Trusted by over 40K Engineers\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"14 Day Free Trial\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"300+ off-the-shelf connectors\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"No-Code Connector Builder\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Get started in minutes\"\nasset_performance_label: GOOD\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Extract, Load & Transform\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n", "text: \"Streamlined Data Pipeline\"\nasset_performance_label: LEARNING\npolicy_summary_info {\n review_status: REVIEWED\n approval_status: APPROVED\n}\n"], "ad_group_ad.ad.responsive_search_ad.path1": "", "ad_group_ad.ad.responsive_search_ad.path2": "", "ad_group_ad.ad.shopping_comparison_listing_ad.headline": "", "ad_group_ad.ad.shopping_product_ad": "", "ad_group_ad.ad.shopping_smart_ad": "", "ad_group_ad.ad.smart_campaign_ad.descriptions": [], "ad_group_ad.ad.smart_campaign_ad.headlines": [], "ad_group_ad.ad.system_managed_resource_source": "UNSPECIFIED", "ad_group_ad.ad.text_ad.description1": "", "ad_group_ad.ad.text_ad.description2": "", "ad_group_ad.ad.text_ad.headline": "", "ad_group_ad.ad.tracking_url_template": "", "ad_group_ad.ad.type": "RESPONSIVE_SEARCH_AD", "ad_group_ad.ad.url_collections": [], "ad_group_ad.ad.url_custom_parameters": [], "ad_group_ad.ad.video_ad.in_feed.description1": "", "ad_group_ad.ad.video_ad.in_feed.description2": "", "ad_group_ad.ad.video_ad.in_feed.headline": "", "ad_group_ad.ad.video_ad.in_stream.action_button_label": "", "ad_group_ad.ad.video_ad.in_stream.action_headline": "", "ad_group_ad.ad.video_ad.out_stream.description": "", "ad_group_ad.ad.video_ad.out_stream.headline": "", "ad_group_ad.ad.video_responsive_ad.call_to_actions": [], "ad_group_ad.ad.video_responsive_ad.companion_banners": [], "ad_group_ad.ad.video_responsive_ad.descriptions": [], "ad_group_ad.ad.video_responsive_ad.headlines": [], "ad_group_ad.ad.video_responsive_ad.long_headlines": [], "ad_group_ad.ad.video_responsive_ad.videos": [], "ad_group_ad.ad_group": "customers/4651612872/adGroups/155311392438", "ad_group_ad.ad_strength": "EXCELLENT", "ad_group_ad.labels": [], "ad_group_ad.policy_summary.approval_status": "APPROVED", "ad_group_ad.policy_summary.policy_topic_entries": [], "ad_group_ad.policy_summary.review_status": "REVIEWED", "ad_group_ad.resource_name": "customers/4651612872/adGroupAds/155311392438~676665180945", "ad_group_ad.status": "ENABLED", "segments.date": "2024-01-02"}, "emitted_at": 1704408116320} +{"stream": "ad_group", "data": {"campaign.id": 20643300404, "ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/155311392438", "ad_group.campaign": "customers/4651612872/campaigns/20643300404", "metrics.cost_micros": 168080000, "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 155311392438, "ad_group.labels": [], "ad_group.name": "Airbyte", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/155311392438", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": ["key: \"adgroup\"\nvalue: \"Airbyte\"\n", "key: \"campaign\"\nvalue: \"mm_search_brand\"\n"], "segments.date": "2023-12-31"}, "emitted_at": 1704717743436} +{"stream": "ad_group", "data": {"campaign.id": 20643300404, "ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/155311392438", "ad_group.campaign": "customers/4651612872/campaigns/20643300404", "metrics.cost_micros": 328930000, "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 155311392438, "ad_group.labels": [], "ad_group.name": "Airbyte", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/155311392438", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": ["key: \"adgroup\"\nvalue: \"Airbyte\"\n", "key: \"campaign\"\nvalue: \"mm_search_brand\"\n"], "segments.date": "2024-01-01"}, "emitted_at": 1704717743438} +{"stream": "ad_group", "data": {"campaign.id": 20655886237, "ad_group.ad_rotation_mode": "UNSPECIFIED", "ad_group.base_ad_group": "customers/4651612872/adGroups/153930342465", "ad_group.campaign": "customers/4651612872/campaigns/20655886237", "metrics.cost_micros": 27110000, "ad_group.cpc_bid_micros": 10000, "ad_group.cpm_bid_micros": 10000, "ad_group.cpv_bid_micros": 0, "ad_group.display_custom_bid_dimension": "UNSPECIFIED", "ad_group.effective_target_cpa_micros": 0, "ad_group.effective_target_cpa_source": "UNSPECIFIED", "ad_group.effective_target_roas": 0.0, "ad_group.effective_target_roas_source": "UNSPECIFIED", "ad_group.excluded_parent_asset_field_types": [], "ad_group.optimized_targeting_enabled": false, "ad_group.final_url_suffix": "", "ad_group.id": 153930342465, "ad_group.labels": [], "ad_group.name": "Airflow", "ad_group.percent_cpc_bid_micros": 0, "ad_group.resource_name": "customers/4651612872/adGroups/153930342465", "ad_group.status": "ENABLED", "ad_group.target_cpa_micros": 0, "ad_group.target_cpm_micros": 10000, "ad_group.target_roas": 0.0, "ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "ad_group.tracking_url_template": "", "ad_group.type": "SEARCH_STANDARD", "ad_group.url_custom_parameters": ["key: \"adgroup\"\nvalue: \"Airflow\"\n", "key: \"campaign\"\nvalue: \"mm_search_competitors\"\n"], "segments.date": "2024-01-02"}, "emitted_at": 1704717743440} +{"stream": "customer", "data": {"customer.auto_tagging_enabled": true, "customer.call_reporting_setting.call_conversion_action": "customers/4651612872/conversionActions/179", "customer.call_reporting_setting.call_conversion_reporting_enabled": true, "customer.call_reporting_setting.call_reporting_enabled": true, "customer.conversion_tracking_setting.conversion_tracking_id": 657981234, "customer.conversion_tracking_setting.cross_account_conversion_tracking_id": 0, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.final_url_suffix": "", "customer.has_partners_badge": false, "customer.id": 4651612872, "customer.manager": false, "customer.optimization_score": 0.7609283000000001, "customer.optimization_score_weight": 3182.4700060000005, "customer.pay_per_conversion_eligibility_failure_reasons": [], "customer.remarketing_setting.google_global_site_tag": "\n\n\n", "customer.resource_name": "customers/4651612872", "customer.test_account": false, "customer.time_zone": "America/Los_Angeles", "customer.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign={_utmcampaign}&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam={campaignid}&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "segments.date": "2023-12-31"}, "emitted_at": 1704408117407} +{"stream": "customer", "data": {"customer.auto_tagging_enabled": true, "customer.call_reporting_setting.call_conversion_action": "customers/4651612872/conversionActions/179", "customer.call_reporting_setting.call_conversion_reporting_enabled": true, "customer.call_reporting_setting.call_reporting_enabled": true, "customer.conversion_tracking_setting.conversion_tracking_id": 657981234, "customer.conversion_tracking_setting.cross_account_conversion_tracking_id": 0, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.final_url_suffix": "", "customer.has_partners_badge": false, "customer.id": 4651612872, "customer.manager": false, "customer.optimization_score": 0.7609283000000001, "customer.optimization_score_weight": 3182.4700060000005, "customer.pay_per_conversion_eligibility_failure_reasons": [], "customer.remarketing_setting.google_global_site_tag": "\n\n\n", "customer.resource_name": "customers/4651612872", "customer.test_account": false, "customer.time_zone": "America/Los_Angeles", "customer.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign={_utmcampaign}&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam={campaignid}&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "segments.date": "2024-01-01"}, "emitted_at": 1704408117408} +{"stream": "customer", "data": {"customer.auto_tagging_enabled": true, "customer.call_reporting_setting.call_conversion_action": "customers/4651612872/conversionActions/179", "customer.call_reporting_setting.call_conversion_reporting_enabled": true, "customer.call_reporting_setting.call_reporting_enabled": true, "customer.conversion_tracking_setting.conversion_tracking_id": 657981234, "customer.conversion_tracking_setting.cross_account_conversion_tracking_id": 0, "customer.currency_code": "USD", "customer.descriptive_name": "Airbyte", "customer.final_url_suffix": "", "customer.has_partners_badge": false, "customer.id": 4651612872, "customer.manager": false, "customer.optimization_score": 0.7609283000000001, "customer.optimization_score_weight": 3182.4700060000005, "customer.pay_per_conversion_eligibility_failure_reasons": [], "customer.remarketing_setting.google_global_site_tag": "\n\n\n", "customer.resource_name": "customers/4651612872", "customer.test_account": false, "customer.time_zone": "America/Los_Angeles", "customer.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign={_utmcampaign}&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam={campaignid}&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "segments.date": "2024-01-02"}, "emitted_at": 1704408117408} +{"stream": "campaign", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "SEARCH", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/20643300404", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/13022493317", "campaign_budget.amount_micros": 330000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 20643300404, "campaign.labels": [], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "mm_search_brand", "campaign.network_settings.target_content_network": false, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": false, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "CLICKS", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/20643300404", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.start_date": "2023-10-10", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n"], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign=mm_search_brand&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam=20643300404&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 1, "metrics.ctr": 0.3333333333333333, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 2980000, "metrics.impressions": 3, "metrics.video_views": 0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.average_cost": 2980000.0, "metrics.average_cpc": 2980000.0, "metrics.average_cpm": 993333333.3333334, "metrics.interactions": 1, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2023-12-31", "segments.hour": 0, "segments.ad_network_type": "SEARCH"}, "emitted_at": 1704408118506} +{"stream": "campaign", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "SEARCH", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/20643300404", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/13022493317", "campaign_budget.amount_micros": 330000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 20643300404, "campaign.labels": [], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "mm_search_brand", "campaign.network_settings.target_content_network": false, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": false, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "CLICKS", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/20643300404", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.start_date": "2023-10-10", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n"], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign=mm_search_brand&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam=20643300404&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 0, "metrics.ctr": 0.0, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.impressions": 2, "metrics.video_views": 0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.interactions": 0, "metrics.interaction_event_types": [], "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2023-12-31", "segments.hour": 3, "segments.ad_network_type": "SEARCH"}, "emitted_at": 1704408118507} +{"stream": "campaign", "data": {"campaign.accessible_bidding_strategy": "", "campaign.ad_serving_optimization_status": "OPTIMIZE", "campaign.advertising_channel_sub_type": "UNSPECIFIED", "campaign.advertising_channel_type": "SEARCH", "campaign.app_campaign_setting.app_id": "", "campaign.app_campaign_setting.app_store": "UNSPECIFIED", "campaign.app_campaign_setting.bidding_strategy_goal_type": "UNSPECIFIED", "campaign.base_campaign": "customers/4651612872/campaigns/20643300404", "campaign.bidding_strategy": "", "campaign.bidding_strategy_type": "MAXIMIZE_CONVERSIONS", "campaign.campaign_budget": "customers/4651612872/campaignBudgets/13022493317", "campaign_budget.amount_micros": 330000000, "campaign.commission.commission_rate_micros": 0, "campaign.dynamic_search_ads_setting.domain_name": "", "campaign.dynamic_search_ads_setting.feeds": [], "campaign.dynamic_search_ads_setting.language_code": "", "campaign.dynamic_search_ads_setting.use_supplied_urls_only": false, "campaign.end_date": "2037-12-30", "campaign.excluded_parent_asset_field_types": [], "campaign.experiment_type": "BASE", "campaign.final_url_suffix": "", "campaign.frequency_caps": [], "campaign.geo_target_type_setting.negative_geo_target_type": "PRESENCE", "campaign.geo_target_type_setting.positive_geo_target_type": "PRESENCE_OR_INTEREST", "campaign.hotel_setting.hotel_center_id": 0, "campaign.id": 20643300404, "campaign.labels": [], "campaign.local_campaign_setting.location_source_type": "UNSPECIFIED", "campaign.manual_cpc.enhanced_cpc_enabled": false, "campaign.manual_cpm": "", "campaign.manual_cpv": "", "campaign.maximize_conversion_value.target_roas": 0.0, "campaign.maximize_conversions.target_cpa_micros": 0, "campaign.name": "mm_search_brand", "campaign.network_settings.target_content_network": false, "campaign.network_settings.target_google_search": true, "campaign.network_settings.target_partner_search_network": false, "campaign.network_settings.target_search_network": false, "campaign.optimization_goal_setting.optimization_goal_types": [], "campaign.optimization_score": 0.0, "campaign.payment_mode": "CLICKS", "campaign.percent_cpc.cpc_bid_ceiling_micros": 0, "campaign.percent_cpc.enhanced_cpc_enabled": false, "campaign.real_time_bidding_setting.opt_in": false, "campaign.resource_name": "customers/4651612872/campaigns/20643300404", "campaign.selective_optimization.conversion_actions": [], "campaign.serving_status": "SERVING", "campaign.shopping_setting.campaign_priority": 0, "campaign.shopping_setting.enable_local": false, "campaign.shopping_setting.merchant_id": 0, "campaign.start_date": "2023-10-10", "campaign.status": "PAUSED", "campaign.target_cpa.cpc_bid_ceiling_micros": 0, "campaign.target_cpa.cpc_bid_floor_micros": 0, "campaign.target_cpa.target_cpa_micros": 0, "campaign.target_cpm.target_frequency_goal.target_count": 0, "campaign.target_cpm.target_frequency_goal.time_unit": "UNSPECIFIED", "campaign.target_impression_share.cpc_bid_ceiling_micros": 0, "campaign.target_impression_share.location": "UNSPECIFIED", "campaign.target_impression_share.location_fraction_micros": 0, "campaign.target_roas.cpc_bid_ceiling_micros": 0, "campaign.target_roas.cpc_bid_floor_micros": 0, "campaign.target_roas.target_roas": 0.0, "campaign.target_spend.cpc_bid_ceiling_micros": 0, "campaign.target_spend.target_spend_micros": 0, "campaign.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n"], "campaign.tracking_setting.tracking_url": "", "campaign.tracking_url_template": "{lpurl}?utm_term={keyword}&utm_campaign=mm_search_brand&utm_source=adwords&utm_medium=ppc&hsa_acc=4651612872&hsa_cam=20643300404&hsa_grp={adgroupid}&hsa_ad={creative}&hsa_src={network}&hsa_tgt={targetid}&hsa_kw={keyword}&hsa_mt={matchtype}&hsa_net=adwords&hsa_ver=3", "campaign.url_custom_parameters": [], "campaign.vanity_pharma.vanity_pharma_display_url_mode": "UNSPECIFIED", "campaign.vanity_pharma.vanity_pharma_text": "UNSPECIFIED", "campaign.video_brand_safety_suitability": "UNSPECIFIED", "metrics.clicks": 2, "metrics.ctr": 0.5, "metrics.conversions": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 1180000, "metrics.impressions": 4, "metrics.video_views": 0, "metrics.video_quartile_p100_rate": 0.0, "metrics.active_view_cpm": 0.0, "metrics.active_view_ctr": 0.0, "metrics.active_view_impressions": 0, "metrics.active_view_measurability": 0.0, "metrics.active_view_measurable_cost_micros": 0, "metrics.active_view_measurable_impressions": 0, "metrics.active_view_viewability": 0.0, "metrics.average_cost": 590000.0, "metrics.average_cpc": 590000.0, "metrics.average_cpm": 295000000.0, "metrics.interactions": 2, "metrics.interaction_event_types": ["InteractionEventType.CLICK"], "metrics.value_per_conversion": 0.0, "metrics.cost_per_conversion": 0.0, "segments.date": "2023-12-31", "segments.hour": 5, "segments.ad_network_type": "SEARCH"}, "emitted_at": 1704408118508} +{"stream": "campaign_label", "data": {"campaign.id": 12124071339, "label.id": 21585034471, "campaign.resource_name": "customers/4651612872/campaigns/12124071339", "campaign_label.resource_name": "customers/4651612872/campaignLabels/12124071339~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1704408119170} +{"stream": "campaign_label", "data": {"campaign.id": 13284356762, "label.id": 21585034471, "campaign.resource_name": "customers/4651612872/campaigns/13284356762", "campaign_label.resource_name": "customers/4651612872/campaignLabels/13284356762~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1704408119172} +{"stream": "campaign_label", "data": {"campaign.id": 16820250687, "label.id": 21906377810, "campaign.resource_name": "customers/4651612872/campaigns/16820250687", "campaign_label.resource_name": "customers/4651612872/campaignLabels/16820250687~21906377810", "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810"}, "emitted_at": 1704408119173} +{"stream": "ad_group_label", "data": {"ad_group.id": 123273719655, "label.id": 21585034471, "ad_group.resource_name": "customers/4651612872/adGroups/123273719655", "ad_group_label.resource_name": "customers/4651612872/adGroupLabels/123273719655~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1704408119522} +{"stream": "ad_group_label", "data": {"ad_group.id": 138643385242, "label.id": 21585034471, "ad_group.resource_name": "customers/4651612872/adGroups/138643385242", "ad_group_label.resource_name": "customers/4651612872/adGroupLabels/138643385242~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471"}, "emitted_at": 1704408119524} +{"stream": "ad_group_label", "data": {"ad_group.id": 137020701042, "label.id": 21906377810, "ad_group.resource_name": "customers/4651612872/adGroups/137020701042", "ad_group_label.resource_name": "customers/4651612872/adGroupLabels/137020701042~21906377810", "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810"}, "emitted_at": 1704408119525} +{"stream": "ad_group_ad_label", "data": {"ad_group.id": 123273719655, "ad_group_ad.ad.id": 524518584182, "ad_group_ad.ad.resource_name": "customers/4651612872/ads/524518584182", "ad_group_ad_label.resource_name": "customers/4651612872/adGroupAdLabels/123273719655~524518584182~21585034471", "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471", "label.id": 21585034471}, "emitted_at": 1704408119841} +{"stream": "ad_group_ad_label", "data": {"ad_group.id": 137020701042, "ad_group_ad.ad.id": 592078631218, "ad_group_ad.ad.resource_name": "customers/4651612872/ads/592078631218", "ad_group_ad_label.resource_name": "customers/4651612872/adGroupAdLabels/137020701042~592078631218~21906377810", "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810", "label.id": 21906377810}, "emitted_at": 1704408119844} +{"stream": "user_location_view", "data": {"segments.date": "2023-12-31", "segments.day_of_week": "SUNDAY", "segments.month": "2023-12-01", "segments.week": "2023-12-25", "segments.quarter": "2023-10-01", "segments.year": 2023, "segments.ad_network_type": "SEARCH", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2124, "user_location_view.targeting_location": false, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2124~false", "campaign.base_campaign": "customers/4651612872/campaigns/20643300404", "campaign.id": 20643300404, "campaign.name": "mm_search_brand", "campaign.status": "PAUSED", "ad_group.name": "Airbyte", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/155311392438", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.clicks": 0, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.impressions": 3, "metrics.interaction_event_types": [], "metrics.interaction_rate": 0.0, "metrics.interactions": 0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1704408120544} +{"stream": "user_location_view", "data": {"segments.date": "2023-12-31", "segments.day_of_week": "SUNDAY", "segments.month": "2023-12-01", "segments.week": "2023-12-25", "segments.quarter": "2023-10-01", "segments.year": 2023, "segments.ad_network_type": "SEARCH", "customer.currency_code": "USD", "customer.id": 4651612872, "customer.descriptive_name": "Airbyte", "customer.time_zone": "America/Los_Angeles", "user_location_view.country_criterion_id": 2356, "user_location_view.targeting_location": false, "user_location_view.resource_name": "customers/4651612872/userLocationViews/2356~false", "campaign.base_campaign": "customers/4651612872/campaigns/20643300404", "campaign.id": 20643300404, "campaign.name": "mm_search_brand", "campaign.status": "PAUSED", "ad_group.name": "Airbyte", "ad_group.status": "ENABLED", "ad_group.base_ad_group": "customers/4651612872/adGroups/155311392438", "metrics.all_conversions": 0.0, "metrics.all_conversions_from_interactions_rate": 0.0, "metrics.all_conversions_value": 0.0, "metrics.average_cost": 0.0, "metrics.average_cpc": 0.0, "metrics.average_cpm": 0.0, "metrics.average_cpv": 0.0, "metrics.clicks": 0, "metrics.conversions": 0.0, "metrics.conversions_from_interactions_rate": 0.0, "metrics.conversions_value": 0.0, "metrics.cost_micros": 0, "metrics.cost_per_all_conversions": 0.0, "metrics.cost_per_conversion": 0.0, "metrics.cross_device_conversions": 0.0, "metrics.ctr": 0.0, "metrics.impressions": 1, "metrics.interaction_event_types": [], "metrics.interaction_rate": 0.0, "metrics.interactions": 0, "metrics.value_per_all_conversions": 0.0, "metrics.value_per_conversion": 0.0, "metrics.video_view_rate": 0.0, "metrics.video_views": 0, "metrics.view_through_conversions": 0}, "emitted_at": 1704408120545} +{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2023-10-10", "campaign.end_date": "2037-12-30", "segments.date": "2023-12-31"}, "emitted_at": 1704408121315} +{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2023-10-10", "campaign.end_date": "2037-12-30", "segments.date": "2024-01-01"}, "emitted_at": 1704408121315} +{"stream": "happytable", "data": {"campaign.accessible_bidding_strategy": "", "segments.ad_destination_type": "NOT_APPLICABLE", "campaign.start_date": "2023-10-10", "campaign.end_date": "2037-12-30", "segments.date": "2024-01-02"}, "emitted_at": 1704408121315} +{"stream": "custom_audience", "data": {"custom_audience.description": "", "custom_audience.name": "Airbyte", "custom_audience.id": 523469909, "custom_audience.members": ["member_type: KEYWORD\nkeyword: \"etl elt\"\n", "member_type: KEYWORD\nkeyword: \"cloud data management and analytics\"\n", "member_type: KEYWORD\nkeyword: \"data integration\"\n", "member_type: KEYWORD\nkeyword: \"big data analytics database\"\n", "member_type: KEYWORD\nkeyword: \"data\"\n", "member_type: KEYWORD\nkeyword: \"data sherid nada\"\n", "member_type: KEYWORD\nkeyword: \"airbyteforeveryone\"\n", "member_type: KEYWORD\nkeyword: \"Airbyte\"\n"], "custom_audience.resource_name": "customers/4651612872/customAudiences/523469909", "custom_audience.status": "ENABLED", "custom_audience.type": "AUTO"}, "emitted_at": 1704408121936} +{"stream": "audience", "data": {"customer.id": 4651612872, "audience.description": "", "audience.dimensions": ["audience_segments {\n segments {\n custom_audience {\n custom_audience: \"customers/4651612872/customAudiences/523469909\"\n }\n }\n}\n"], "audience.exclusion_dimension": "", "audience.id": 47792633, "audience.name": "Audience name 1", "audience.resource_name": "customers/4651612872/audiences/47792633", "audience.status": "ENABLED"}, "emitted_at": 1704408122314} +{"stream": "audience", "data": {"customer.id": 4651612872, "audience.description": "", "audience.dimensions": ["audience_segments {\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80276\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80279\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80520\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/80530\"\n }\n }\n segments {\n user_interest {\n user_interest_category: \"customers/4651612872/userInterests/92931\"\n }\n }\n}\n"], "audience.exclusion_dimension": "", "audience.id": 97300129, "audience.name": "Upgraded Audience 1", "audience.resource_name": "customers/4651612872/audiences/97300129", "audience.status": "ENABLED"}, "emitted_at": 1704408122315} +{"stream": "user_interest", "data": {"user_interest.availabilities": [], "user_interest.launched_to_all": true, "user_interest.name": "Arts & Entertainment", "user_interest.resource_name": "customers/4651612872/userInterests/3", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 3, "user_interest.user_interest_parent": ""}, "emitted_at": 1704408124247} +{"stream": "user_interest", "data": {"user_interest.availabilities": [], "user_interest.launched_to_all": true, "user_interest.name": "Computers & Electronics", "user_interest.resource_name": "customers/4651612872/userInterests/5", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 5, "user_interest.user_interest_parent": ""}, "emitted_at": 1704408124249} +{"stream": "user_interest", "data": {"user_interest.availabilities": [], "user_interest.launched_to_all": true, "user_interest.name": "Finance", "user_interest.resource_name": "customers/4651612872/userInterests/7", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 7, "user_interest.user_interest_parent": ""}, "emitted_at": 1704408124250} +{"stream": "label", "data": {"customer.id": 4651612872, "label.id": 21585034471, "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471", "label.status": "ENABLED", "label.text_label.background_color": "#E993EB", "label.text_label.description": "example label for edgao"}, "emitted_at": 1704408126496} +{"stream": "label", "data": {"customer.id": 4651612872, "label.id": 21902092838, "label.name": "Test Label", "label.resource_name": "customers/4651612872/labels/21902092838", "label.status": "ENABLED", "label.text_label.background_color": "#8BCBD2", "label.text_label.description": "Description to test label"}, "emitted_at": 1704408126498} +{"stream": "label", "data": {"customer.id": 4651612872, "label.id": 21906377810, "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810", "label.status": "ENABLED", "label.text_label.background_color": "#8266C9", "label.text_label.description": ""}, "emitted_at": 1704408126499} +{"stream": "campaign_bidding_strategy", "data": {"customer.id": 4651612872, "campaign.id": 20643300404, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2023-12-31"}, "emitted_at": 1704408127194} +{"stream": "campaign_bidding_strategy", "data": {"customer.id": 4651612872, "campaign.id": 20643300404, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2024-01-01"}, "emitted_at": 1704408127198} +{"stream": "campaign_bidding_strategy", "data": {"customer.id": 4651612872, "campaign.id": 20637264648, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2024-01-02"}, "emitted_at": 1704408127200} +{"stream": "ad_group_bidding_strategy", "data": {"ad_group.id": 155311392438, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2023-12-31"}, "emitted_at": 1704408127574} +{"stream": "ad_group_bidding_strategy", "data": {"ad_group.id": 155311392438, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2024-01-01"}, "emitted_at": 1704408127581} +{"stream": "ad_group_bidding_strategy", "data": {"ad_group.id": 154050719199, "bidding_strategy.aligned_campaign_budget_id": 0, "bidding_strategy.campaign_count": 0, "bidding_strategy.currency_code": "", "bidding_strategy.effective_currency_code": "", "bidding_strategy.enhanced_cpc": "", "bidding_strategy.id": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversion_value.target_roas": 0.0, "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": 0, "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": 0, "bidding_strategy.maximize_conversions.target_cpa_micros": 0, "bidding_strategy.name": "", "bidding_strategy.non_removed_campaign_count": 0, "bidding_strategy.resource_name": "", "bidding_strategy.status": "UNSPECIFIED", "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_cpa.cpc_bid_floor_micros": 0, "bidding_strategy.target_cpa.target_cpa_micros": 0, "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_impression_share.location": "UNSPECIFIED", "bidding_strategy.target_impression_share.location_fraction_micros": 0, "bidding_strategy.target_roas.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_roas.cpc_bid_floor_micros": 0, "bidding_strategy.target_roas.target_roas": 0.0, "bidding_strategy.target_spend.cpc_bid_ceiling_micros": 0, "bidding_strategy.target_spend.target_spend_micros": 0, "bidding_strategy.type": "UNSPECIFIED", "segments.date": "2024-01-02"}, "emitted_at": 1704408127583} +{"stream": "ad_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group.id": 117036054899, "ad_group_criterion.ad_group": "customers/4651612872/adGroups/117036054899", "ad_group_criterion.age_range.type": "UNSPECIFIED", "ad_group_criterion.app_payment_model.type": "UNSPECIFIED", "ad_group_criterion.approval_status": "APPROVED", "ad_group_criterion.audience.audience": "", "ad_group_criterion.bid_modifier": 0.0, "ad_group_criterion.combined_audience.combined_audience": "", "ad_group_criterion.cpc_bid_micros": 0, "ad_group_criterion.cpm_bid_micros": 0, "ad_group_criterion.cpv_bid_micros": 0, "ad_group_criterion.criterion_id": 18696703, "ad_group_criterion.custom_affinity.custom_affinity": "", "ad_group_criterion.custom_audience.custom_audience": "", "ad_group_criterion.custom_intent.custom_intent": "", "ad_group_criterion.disapproval_reasons": [], "ad_group_criterion.display_name": "data integrations", "ad_group_criterion.effective_cpc_bid_micros": 1000000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 0, "ad_group_criterion.effective_cpv_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_percent_cpc_bid_micros": 0, "ad_group_criterion.effective_percent_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_url_suffix": "", "ad_group_criterion.final_urls": [], "ad_group_criterion.gender.type": "UNSPECIFIED", "ad_group_criterion.income_range.type": "UNSPECIFIED", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.keyword.text": "data integrations", "ad_group_criterion.labels": [], "ad_group_criterion.mobile_app_category.mobile_app_category_constant": "", "ad_group_criterion.mobile_application.app_id": "", "ad_group_criterion.mobile_application.name": "", "ad_group_criterion.negative": false, "ad_group_criterion.parental_status.type": "UNSPECIFIED", "ad_group_criterion.percent_cpc_bid_micros": 0, "ad_group_criterion.placement.url": "", "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.first_page_cpc_micros": 0, "ad_group_criterion.position_estimates.first_position_cpc_micros": 0, "ad_group_criterion.position_estimates.top_of_page_cpc_micros": 0, "ad_group_criterion.quality_info.creative_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.post_click_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.quality_score": 0, "ad_group_criterion.quality_info.search_predicted_ctr": "UNSPECIFIED", "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~18696703", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.system_serving_status": "ELIGIBLE", "ad_group_criterion.topic.path": [], "ad_group_criterion.topic.topic_constant": "", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.url_custom_parameters": [], "ad_group_criterion.user_interest.user_interest_category": "", "ad_group_criterion.user_list.user_list": "", "ad_group_criterion.webpage.conditions": [], "ad_group_criterion.webpage.coverage_percentage": 0.0, "ad_group_criterion.webpage.criterion_name": "", "ad_group_criterion.webpage.sample.sample_urls": [], "ad_group_criterion.youtube_channel.channel_id": "", "ad_group_criterion.youtube_video.video_id": ""}, "emitted_at": 1704408130758} +{"stream": "ad_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group.id": 117036054899, "ad_group_criterion.ad_group": "customers/4651612872/adGroups/117036054899", "ad_group_criterion.age_range.type": "UNSPECIFIED", "ad_group_criterion.app_payment_model.type": "UNSPECIFIED", "ad_group_criterion.approval_status": "APPROVED", "ad_group_criterion.audience.audience": "", "ad_group_criterion.bid_modifier": 0.0, "ad_group_criterion.combined_audience.combined_audience": "", "ad_group_criterion.cpc_bid_micros": 0, "ad_group_criterion.cpm_bid_micros": 0, "ad_group_criterion.cpv_bid_micros": 0, "ad_group_criterion.criterion_id": 376833662, "ad_group_criterion.custom_affinity.custom_affinity": "", "ad_group_criterion.custom_audience.custom_audience": "", "ad_group_criterion.custom_intent.custom_intent": "", "ad_group_criterion.disapproval_reasons": [], "ad_group_criterion.display_name": "data integration services", "ad_group_criterion.effective_cpc_bid_micros": 1000000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 0, "ad_group_criterion.effective_cpv_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_percent_cpc_bid_micros": 0, "ad_group_criterion.effective_percent_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_url_suffix": "", "ad_group_criterion.final_urls": [], "ad_group_criterion.gender.type": "UNSPECIFIED", "ad_group_criterion.income_range.type": "UNSPECIFIED", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.keyword.text": "data integration services", "ad_group_criterion.labels": [], "ad_group_criterion.mobile_app_category.mobile_app_category_constant": "", "ad_group_criterion.mobile_application.app_id": "", "ad_group_criterion.mobile_application.name": "", "ad_group_criterion.negative": false, "ad_group_criterion.parental_status.type": "UNSPECIFIED", "ad_group_criterion.percent_cpc_bid_micros": 0, "ad_group_criterion.placement.url": "", "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.first_page_cpc_micros": 0, "ad_group_criterion.position_estimates.first_position_cpc_micros": 0, "ad_group_criterion.position_estimates.top_of_page_cpc_micros": 0, "ad_group_criterion.quality_info.creative_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.post_click_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.quality_score": 0, "ad_group_criterion.quality_info.search_predicted_ctr": "UNSPECIFIED", "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~376833662", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.system_serving_status": "ELIGIBLE", "ad_group_criterion.topic.path": [], "ad_group_criterion.topic.topic_constant": "", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.url_custom_parameters": [], "ad_group_criterion.user_interest.user_interest_category": "", "ad_group_criterion.user_list.user_list": "", "ad_group_criterion.webpage.conditions": [], "ad_group_criterion.webpage.coverage_percentage": 0.0, "ad_group_criterion.webpage.criterion_name": "", "ad_group_criterion.webpage.sample.sample_urls": [], "ad_group_criterion.youtube_channel.channel_id": "", "ad_group_criterion.youtube_video.video_id": ""}, "emitted_at": 1704408130764} +{"stream": "ad_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group.id": 117036054899, "ad_group_criterion.ad_group": "customers/4651612872/adGroups/117036054899", "ad_group_criterion.age_range.type": "UNSPECIFIED", "ad_group_criterion.app_payment_model.type": "UNSPECIFIED", "ad_group_criterion.approval_status": "APPROVED", "ad_group_criterion.audience.audience": "", "ad_group_criterion.bid_modifier": 0.0, "ad_group_criterion.combined_audience.combined_audience": "", "ad_group_criterion.cpc_bid_micros": 0, "ad_group_criterion.cpm_bid_micros": 0, "ad_group_criterion.cpv_bid_micros": 0, "ad_group_criterion.criterion_id": 13099056325, "ad_group_criterion.custom_affinity.custom_affinity": "", "ad_group_criterion.custom_audience.custom_audience": "", "ad_group_criterion.custom_intent.custom_intent": "", "ad_group_criterion.disapproval_reasons": [], "ad_group_criterion.display_name": "cloud data integration", "ad_group_criterion.effective_cpc_bid_micros": 1000000, "ad_group_criterion.effective_cpc_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpm_bid_micros": 10000, "ad_group_criterion.effective_cpm_bid_source": "AD_GROUP", "ad_group_criterion.effective_cpv_bid_micros": 0, "ad_group_criterion.effective_cpv_bid_source": "UNSPECIFIED", "ad_group_criterion.effective_percent_cpc_bid_micros": 0, "ad_group_criterion.effective_percent_cpc_bid_source": "UNSPECIFIED", "ad_group_criterion.final_mobile_urls": [], "ad_group_criterion.final_url_suffix": "", "ad_group_criterion.final_urls": [], "ad_group_criterion.gender.type": "UNSPECIFIED", "ad_group_criterion.income_range.type": "UNSPECIFIED", "ad_group_criterion.keyword.match_type": "BROAD", "ad_group_criterion.keyword.text": "cloud data integration", "ad_group_criterion.labels": [], "ad_group_criterion.mobile_app_category.mobile_app_category_constant": "", "ad_group_criterion.mobile_application.app_id": "", "ad_group_criterion.mobile_application.name": "", "ad_group_criterion.negative": false, "ad_group_criterion.parental_status.type": "UNSPECIFIED", "ad_group_criterion.percent_cpc_bid_micros": 0, "ad_group_criterion.placement.url": "", "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": 0, "ad_group_criterion.position_estimates.first_page_cpc_micros": 0, "ad_group_criterion.position_estimates.first_position_cpc_micros": 0, "ad_group_criterion.position_estimates.top_of_page_cpc_micros": 0, "ad_group_criterion.quality_info.creative_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.post_click_quality_score": "UNSPECIFIED", "ad_group_criterion.quality_info.quality_score": 0, "ad_group_criterion.quality_info.search_predicted_ctr": "UNSPECIFIED", "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~13099056325", "ad_group_criterion.status": "ENABLED", "ad_group_criterion.system_serving_status": "ELIGIBLE", "ad_group_criterion.topic.path": [], "ad_group_criterion.topic.topic_constant": "", "ad_group_criterion.tracking_url_template": "", "ad_group_criterion.type": "KEYWORD", "ad_group_criterion.url_custom_parameters": [], "ad_group_criterion.user_interest.user_interest_category": "", "ad_group_criterion.user_list.user_list": "", "ad_group_criterion.webpage.conditions": [], "ad_group_criterion.webpage.coverage_percentage": 0.0, "ad_group_criterion.webpage.criterion_name": "", "ad_group_criterion.webpage.sample.sample_urls": [], "ad_group_criterion.youtube_channel.channel_id": "", "ad_group_criterion.youtube_video.video_id": ""}, "emitted_at": 1704408130766} +{"stream": "ad_listing_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~18696703", "ad_group.id": 117036054899, "ad_group_criterion.criterion_id": 18696703, "ad_group_criterion.listing_group.case_value.activity_country.value": "", "ad_group_criterion.listing_group.case_value.activity_id.value": "", "ad_group_criterion.listing_group.case_value.activity_rating.value": 0, "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_class.value": 0, "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_id.value": "", "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": "", "ad_group_criterion.listing_group.case_value.product_category.category_id": 0, "ad_group_criterion.listing_group.case_value.product_category.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_brand.value": "", "ad_group_criterion.listing_group.case_value.product_channel.channel": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_condition.condition": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": "", "ad_group_criterion.listing_group.case_value.product_item_id.value": "", "ad_group_criterion.listing_group.case_value.product_type.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_type.value": "", "ad_group_criterion.listing_group.parent_ad_group_criterion": "", "ad_group_criterion.listing_group.type": "UNSPECIFIED"}, "emitted_at": 1704408168252} +{"stream": "ad_listing_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~376833662", "ad_group.id": 117036054899, "ad_group_criterion.criterion_id": 376833662, "ad_group_criterion.listing_group.case_value.activity_country.value": "", "ad_group_criterion.listing_group.case_value.activity_id.value": "", "ad_group_criterion.listing_group.case_value.activity_rating.value": 0, "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_class.value": 0, "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_id.value": "", "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": "", "ad_group_criterion.listing_group.case_value.product_category.category_id": 0, "ad_group_criterion.listing_group.case_value.product_category.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_brand.value": "", "ad_group_criterion.listing_group.case_value.product_channel.channel": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_condition.condition": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": "", "ad_group_criterion.listing_group.case_value.product_item_id.value": "", "ad_group_criterion.listing_group.case_value.product_type.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_type.value": "", "ad_group_criterion.listing_group.parent_ad_group_criterion": "", "ad_group_criterion.listing_group.type": "UNSPECIFIED"}, "emitted_at": 1704408168257} +{"stream": "ad_listing_group_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "ad_group_criterion.resource_name": "customers/4651612872/adGroupCriteria/117036054899~13099056325", "ad_group.id": 117036054899, "ad_group_criterion.criterion_id": 13099056325, "ad_group_criterion.listing_group.case_value.activity_country.value": "", "ad_group_criterion.listing_group.case_value.activity_id.value": "", "ad_group_criterion.listing_group.case_value.activity_rating.value": 0, "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_class.value": 0, "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": "", "ad_group_criterion.listing_group.case_value.hotel_id.value": "", "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": "", "ad_group_criterion.listing_group.case_value.product_category.category_id": 0, "ad_group_criterion.listing_group.case_value.product_category.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_brand.value": "", "ad_group_criterion.listing_group.case_value.product_channel.channel": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_condition.condition": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": "", "ad_group_criterion.listing_group.case_value.product_item_id.value": "", "ad_group_criterion.listing_group.case_value.product_type.level": "UNSPECIFIED", "ad_group_criterion.listing_group.case_value.product_type.value": "", "ad_group_criterion.listing_group.parent_ad_group_criterion": "", "ad_group_criterion.listing_group.type": "UNSPECIFIED"}, "emitted_at": 1704408168258} +{"stream": "ad_group_criterion_label", "data": {"ad_group.id": 137051662444, "label.id": 21902092838, "ad_group_criterion_label.ad_group_criterion": "customers/4651612872/adGroupCriteria/137051662444~10766861", "ad_group_criterion_label.label": "customers/4651612872/labels/21902092838", "ad_group_criterion_label.resource_name": "customers/4651612872/adGroupCriterionLabels/137051662444~10766861~21902092838", "ad_group_criterion.criterion_id": 10766861}, "emitted_at": 1704408192425} +{"stream": "ad_group_criterion_label", "data": {"ad_group.id": 137051662444, "label.id": 21906377810, "ad_group_criterion_label.ad_group_criterion": "customers/4651612872/adGroupCriteria/137051662444~528912986", "ad_group_criterion_label.label": "customers/4651612872/labels/21906377810", "ad_group_criterion_label.resource_name": "customers/4651612872/adGroupCriterionLabels/137051662444~528912986~21906377810", "ad_group_criterion.criterion_id": 528912986}, "emitted_at": 1704408192426} +{"stream": "campaign_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "campaign.id": 9660123292, "campaign_criterion.resource_name": "customers/4651612872/campaignCriteria/9660123292~2124", "campaign_criterion.campaign": "customers/4651612872/campaigns/9660123292", "campaign_criterion.age_range.type": "UNSPECIFIED", "campaign_criterion.mobile_application.name": "", "campaign_criterion.negative": false, "campaign_criterion.youtube_channel.channel_id": "", "campaign_criterion.youtube_video.video_id": ""}, "emitted_at": 1704408194062} +{"stream": "campaign_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "campaign.id": 9660123292, "campaign_criterion.resource_name": "customers/4651612872/campaignCriteria/9660123292~2250", "campaign_criterion.campaign": "customers/4651612872/campaigns/9660123292", "campaign_criterion.age_range.type": "UNSPECIFIED", "campaign_criterion.mobile_application.name": "", "campaign_criterion.negative": false, "campaign_criterion.youtube_channel.channel_id": "", "campaign_criterion.youtube_video.video_id": ""}, "emitted_at": 1704408194068} +{"stream": "campaign_criterion", "data": {"deleted_at": null, "change_status.last_change_date_time": null, "campaign.id": 9660123292, "campaign_criterion.resource_name": "customers/4651612872/campaignCriteria/9660123292~2276", "campaign_criterion.campaign": "customers/4651612872/campaigns/9660123292", "campaign_criterion.age_range.type": "UNSPECIFIED", "campaign_criterion.mobile_application.name": "", "campaign_criterion.negative": false, "campaign_criterion.youtube_channel.channel_id": "", "campaign_criterion.youtube_video.video_id": ""}, "emitted_at": 1704408194068} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/incremental_catalog.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/incremental_catalog.json index 12bb06914012..34604ac525ff 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/incremental_catalog.json +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/incremental_catalog.json @@ -5,11 +5,23 @@ "name": "account_performance_report", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [ + ["customer.id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", + "primary_key": [ + ["customer.id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "cursor_field": ["segments.date"] }, { @@ -18,69 +30,132 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "source_defined_primary_key": [["click_view.gclid"], ["segments.date"]], + "source_defined_primary_key": [ + ["customer.id"], + ["click_view.gclid"], + ["segments.date"], + ["segments.ad_network_type"] + ], "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", "cursor_field": ["segments.date"], - "primary_key": [["click_view.gclid"], ["segments.date"]] + "primary_key": [ + ["customer.id"], + ["click_view.gclid"], + ["segments.date"], + ["segments.ad_network_type"] + ] }, { "stream": { - "name": "geographic_report", + "name": "geographic_view", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [ + ["customer.id"], + ["geographic_view.country_criterion_id"], + ["geographic_view.location_type"], + ["segments.date"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", + "primary_key": [ + ["customer.id"], + ["geographic_view.country_criterion_id"], + ["geographic_view.location_type"], + ["segments.date"] + ], "cursor_field": ["segments.date"] }, { "stream": { - "name": "keyword_report", + "name": "keyword_view", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [ + ["customer.id"], + ["ad_group_criterion.criterion_id"], + ["ad_group.id"], + ["segments.date"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", + "primary_key": [ + ["customer.id"], + ["ad_group_criterion.criterion_id"], + ["ad_group.id"], + ["segments.date"] + ], "cursor_field": ["segments.date"] }, { "stream": { - "name": "display_topics_performance_report", + "name": "topic_view", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [ + ["customer.id"], + ["ad_group.id"], + ["ad_group_criterion.criterion_id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", - "cursor_field": ["segments.date"] + "cursor_field": ["segments.date"], + "primary_key": [ + ["customer.id"], + ["ad_group.id"], + ["ad_group_criterion.criterion_id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ] }, { "stream": { - "name": "shopping_performance_report", + "name": "shopping_performance_view", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [ + ["customer.id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", + "primary_key": [ + ["customer.id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "destination_sync_mode": "overwrite", "cursor_field": ["segments.date"] }, { "stream": { - "name": "ad_group_ads", + "name": "ad_group_ad", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "source_defined_primary_key": [ + ["customer.id"], ["ad_group_ad.ad.id"], ["segments.date"] ], @@ -89,25 +164,29 @@ "sync_mode": "incremental", "destination_sync_mode": "overwrite", "cursor_field": ["segments.date"], - "primary_key": [["ad_group_ad.ad.id"], ["segments.date"]] + "primary_key": [["customer.id"], ["ad_group_ad.ad.id"], ["segments.date"]] }, { "stream": { - "name": "ad_groups", + "name": "ad_group", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["segments.date"], - "source_defined_primary_key": [["ad_group.id"], ["segments.date"]] + "source_defined_primary_key": [ + ["customer.id"], + ["ad_group.id"], + ["segments.date"] + ] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", "cursor_field": ["segments.date"], - "primary_key": [["ad_group.id"], ["segments.date"]] + "primary_key": [["customer.id"], ["ad_group.id"], ["segments.date"]] }, { "stream": { - "name": "accounts", + "name": "customer", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -121,7 +200,7 @@ }, { "stream": { - "name": "campaigns", + "name": "campaign", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -152,46 +231,89 @@ }, { "stream": { - "name": "user_location_report", + "name": "user_location_view", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, + "source_defined_primary_key": [ + ["customer.id"], + ["user_location_view.country_criterion_id"], + ["user_location_view.targeting_location"], + ["segments.date"], + ["segments.ad_network_type"] + ], "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", + "primary_key": [ + ["customer.id"], + ["user_location_view.country_criterion_id"], + ["user_location_view.targeting_location"], + ["segments.date"], + ["segments.ad_network_type"] + ], "cursor_field": ["segments.date"] }, { "stream": { - "name": "ad_group_ad_report", + "name": "ad_group_ad_legacy", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, + "source_defined_primary_key": [ + ["customer.id"], + ["ad_group_ad.ad.id"], + ["segments.date"], + ["segments.ad_network_type"] + ], "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", + "primary_key": [ + ["customer.id"], + ["ad_group_ad.ad.id"], + ["segments.date"], + ["segments.ad_network_type"] + ], "cursor_field": ["segments.date"] }, { "stream": { - "name": "display_keyword_performance_report", + "name": "display_keyword_view", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [ + ["customer.id"], + ["ad_group.id"], + ["ad_group_criterion.criterion_id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, "sync_mode": "incremental", "destination_sync_mode": "overwrite", + "primary_key": [ + ["customer.id"], + ["ad_group.id"], + ["ad_group_criterion.criterion_id"], + ["segments.date"], + ["segments.ad_network_type"], + ["segments.device"] + ], "cursor_field": ["segments.date"] }, { "stream": { - "name": "campaign_bidding_strategies", + "name": "campaign_bidding_strategy", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_primary_key": [ + ["customer.id"], ["campaign.id"], ["bidding_strategy.id"], ["segments.date"] @@ -199,9 +321,10 @@ "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, - "sync_mode": "incremental", - "destination_sync_mode": "append", + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", "primary_key": [ + ["customer.id"], ["campaign.id"], ["bidding_strategy.id"], ["segments.date"] @@ -210,10 +333,11 @@ }, { "stream": { - "name": "ad_group_bidding_strategies", + "name": "ad_group_bidding_strategy", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_primary_key": [ + ["customer.id"], ["ad_group.id"], ["bidding_strategy.id"], ["segments.date"] @@ -221,14 +345,37 @@ "source_defined_cursor": true, "default_cursor_field": ["segments.date"] }, - "sync_mode": "incremental", - "destination_sync_mode": "append", + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", "primary_key": [ + ["customer.id"], ["ad_group.id"], ["bidding_strategy.id"], ["segments.date"] ], "cursor_field": ["segments.date"] + }, + { + "stream": { + "name": "ad_group_criterion", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["ad_group_criterion.resource_name"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["ad_group_criterion.resource_name"]] + }, + { + "stream": { + "name": "campaign_criterion", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["campaign_criterion.resource_name"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["campaign_criterion.resource_name"]] } ] } diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/integration_tests.py b/airbyte-integrations/connectors/source-google-ads/integration_tests/integration_tests.py new file mode 100644 index 000000000000..65f544425f63 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/integration_tests.py @@ -0,0 +1,225 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +from pathlib import Path + +import pytest +from airbyte_cdk.models import SyncMode +from google.ads.googleads.v15.services.types.google_ads_service import GoogleAdsRow +from source_google_ads.source import SourceGoogleAds + + +@pytest.fixture(scope="module") +def config(): + with open(Path(__file__).parent.parent / "secrets/config.json", "r") as file: + return json.loads(file.read()) + + +@pytest.fixture(scope="module") +def streams(config): + return SourceGoogleAds().streams(config=config) + + +@pytest.fixture(scope="module") +def account_labels(streams): + return next(filter(lambda s: s.name == "account_labels", streams)) + + +@pytest.fixture(scope="module") +def shopping_performance_report(streams): + return next(filter(lambda s: s.name == "shopping_performance_report", streams)) + + +def create_google_ads_row_from_dict(data: dict) -> GoogleAdsRow: + row = GoogleAdsRow() + + for key, value in data.items(): + # Split the key to check for nested fields + parts = key.split(".") + + # Handle nested fields + if len(parts) > 1: + parent_field = parts[0] + nested_field = parts[1] + + # Check if the parent field exists and create if not + if not hasattr(row, parent_field): + setattr( + row, parent_field, row.__class__() + ) # Assuming the nested message type is the same as the parent. Adjust if different. + + # Set the nested field value + nested_obj = getattr(row, parent_field) + setattr(nested_obj, nested_field, value) + else: + # Handle non-nested fields + if hasattr(row, key): + setattr(row, key, value) + else: + print(f"Warning: Unknown field '{key}' for GoogleAdsRow") + + return row + + +@pytest.mark.parametrize( + "stream_fixture_name, expected_records", + [ + ( + "account_labels", + [ + { + "customer_label.resource_name": "123", + "customer_label.customer": "customer", + "customer.id": 123, + "customer_label.label": "customer_label", + }, + { + "customer_label.resource_name": "1234", + "customer_label.customer": "customer", + "customer.id": 123, + "customer_label.label": "customer_label1", + }, + ], + ), + ( + "shopping_performance_report", + [ + { + "customer.descriptive_name": "Customer ABC", + "ad_group.id": 12345, + "ad_group.name": "Ad Group 1", + "ad_group.status": "REMOVED", + "segments.ad_network_type": "UNKNOWN", + "segments.product_aggregator_id": 67890, + "metrics.all_conversions_from_interactions_rate": 0.75, + "metrics.all_conversions_value": 150.25, + "metrics.all_conversions": 5.0, + "metrics.average_cpc": 0.5, + "segments.product_brand": "Brand XYZ", + "campaign.id": 11112, + "campaign.name": "Campaign 1", + "campaign.status": "UNKNOWN", + "segments.product_category_level1": "Electronics", + "segments.product_category_level2": "Mobile Phones", + "segments.product_category_level3": "Smartphones", + "segments.product_category_level4": "Android", + "segments.product_category_level5": "Samsung", + "segments.product_channel": "UNSPECIFIED", + "segments.product_channel_exclusivity": "SINGLE_CHANNEL", + "segments.click_type": "APP_DEEPLINK", + "metrics.clicks": 10, + "metrics.conversions_from_interactions_rate": 0.5, + "metrics.conversions_value": 100.5, + "metrics.conversions": 4.0, + "metrics.cost_micros": 5000000, + "metrics.cost_per_all_conversions": 25.05, + "metrics.cost_per_conversion": 6.25, + "segments.product_country": "US", + "metrics.cross_device_conversions": 2.0, + "metrics.ctr": 0.1, + "segments.product_custom_attribute0": "Attribute 0", + "segments.product_custom_attribute1": "Attribute 1", + "segments.product_custom_attribute2": "Attribute 2", + "segments.product_custom_attribute3": "Attribute 3", + "segments.product_custom_attribute4": "Attribute 4", + "segments.date": "2023-09-22", + "segments.day_of_week": "FRIDAY", + "segments.device": "TABLET", + "customer.id": 123, + "metrics.impressions": 100, + "segments.product_language": "English", + "segments.product_merchant_id": 54321, + "segments.month": "September", + "segments.product_item_id": "ITEM123", + "segments.product_condition": 2, + "segments.product_title": "Samsung Galaxy S23", + "segments.product_type_l1": "Electronics", + "segments.product_type_l2": "Phones", + "segments.product_type_l3": "Smartphones", + "segments.product_type_l4": "Android", + "segments.product_type_l5": "Samsung", + "segments.quarter": "Q3", + "segments.product_store_id": "STORE123", + "metrics.value_per_all_conversions": 30.05, + "metrics.value_per_conversion": 7.5, + "segments.week": "38", + "segments.year": 2023, + }, + { + "customer.descriptive_name": "Customer ABC", + "ad_group.id": 12345, + "ad_group.name": "Ad Group 1", + "ad_group.status": "REMOVED", + "segments.ad_network_type": "UNKNOWN", + "segments.product_aggregator_id": 67890, + "metrics.all_conversions_from_interactions_rate": 0.75, + "metrics.all_conversions_value": 150.25, + "metrics.all_conversions": 5.0, + "metrics.average_cpc": 0.5, + "segments.product_brand": "Brand XYZ", + "campaign.id": 11112, + "campaign.name": "Campaign 1", + "campaign.status": "UNKNOWN", + "segments.product_category_level1": "Electronics", + "segments.product_category_level2": "Mobile Phones", + "segments.product_category_level3": "Smartphones", + "segments.product_category_level4": "Android", + "segments.product_category_level5": "Samsung", + "segments.product_channel": "UNSPECIFIED", + "segments.product_channel_exclusivity": "SINGLE_CHANNEL", + "segments.click_type": "APP_DEEPLINK", + "metrics.clicks": 10, + "metrics.conversions_from_interactions_rate": 0.5, + "metrics.conversions_value": 100.5, + "metrics.conversions": 4.0, + "metrics.cost_micros": 5000000, + "metrics.cost_per_all_conversions": 25.05, + "metrics.cost_per_conversion": 6.25, + "segments.product_country": "US", + "metrics.cross_device_conversions": 2.0, + "metrics.ctr": 0.1, + "segments.product_custom_attribute0": "Attribute 0", + "segments.product_custom_attribute1": "Attribute 1", + "segments.product_custom_attribute2": "Attribute 2", + "segments.product_custom_attribute3": "Attribute 3", + "segments.product_custom_attribute4": "Attribute 4", + "segments.date": "2023-11-22", + "segments.day_of_week": "FRIDAY", + "segments.device": "TABLET", + "customer.id": 123, + "metrics.impressions": 100, + "segments.product_language": "English", + "segments.product_merchant_id": 54321, + "segments.month": "November", + "segments.product_item_id": "ITEM123", + "segments.product_condition": 2, + "segments.product_title": "Samsung Galaxy S23", + "segments.product_type_l1": "Electronics", + "segments.product_type_l2": "Phones", + "segments.product_type_l3": "Smartphones", + "segments.product_type_l4": "Android", + "segments.product_type_l5": "Samsung", + "segments.quarter": "Q4", + "segments.product_store_id": "STORE123", + "metrics.value_per_all_conversions": 30.05, + "metrics.value_per_conversion": 7.5, + "segments.week": "38", + "segments.year": 2023, + }, + ], + ), + ], +) +def test_empty_streams(mocker, stream_fixture_name, expected_records, request): + """ + A test with synthetic data since we are not able to test `annotations_stream` and `cohorts_stream` streams + due to free subscription plan for the sandbox + """ + stream = request.getfixturevalue(stream_fixture_name) + records_reader = stream.read_records(sync_mode=SyncMode.full_refresh, cursor_field=None, stream_slice={"customer_id": "123"}) + send_request_result = [create_google_ads_row_from_dict(expected_record) for expected_record in expected_records] + mocker.patch("source_google_ads.google_ads.GoogleAds.send_request", return_value=[send_request_result]) + + assert list(records_reader) == expected_records diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/state.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/state.json new file mode 100644 index 000000000000..60ad9790af5f --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/state.json @@ -0,0 +1,73 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "ad_groups" + }, + "stream_state": { + "4651612872": { + "segments.date": "2022-04-09" + } + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "campaigns" + }, + "stream_state": { + "4651612872": { + "segments.date": "2022-04-09" + } + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "ad_group_criterion" + }, + "stream_state": { + "change_status": { + "4651612872": { + "change_status.last_change_date_time": "2023-11-01 13:20:01.003295" + } + } + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "ad_listing_group_criterion" + }, + "stream_state": { + "change_status": { + "4651612872": { + "change_status.last_change_date_time": "2023-11-01 13:20:01.003295" + } + } + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "campaign_criterion" + }, + "stream_state": { + "change_status": { + "4651612872": { + "change_status.last_change_date_time": "2023-11-01 13:20:01.003295" + } + } + } + } + } +] diff --git a/airbyte-integrations/connectors/source-google-ads/main.py b/airbyte-integrations/connectors/source-google-ads/main.py index 74d321502526..2824c4955943 100644 --- a/airbyte-integrations/connectors/source-google-ads/main.py +++ b/airbyte-integrations/connectors/source-google-ads/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_google_ads import SourceGoogleAds +from source_google_ads.run import run if __name__ == "__main__": - source = SourceGoogleAds() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-google-ads/metadata.yaml b/airbyte-integrations/connectors/source-google-ads/metadata.yaml index 381177418962..4c09baf87a5c 100644 --- a/airbyte-integrations/connectors/source-google-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-ads/metadata.yaml @@ -1,13 +1,19 @@ data: + ab_internal: + ql: 400 + sl: 300 allowedHosts: hosts: - accounts.google.com - googleads.googleapis.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 253487c0-2246-43ba-a21f-5116b20a2c50 - dockerImageTag: 0.7.4 + dockerImageTag: 3.3.1 dockerRepository: airbyte/source-google-ads + documentationUrl: https://docs.airbyte.com/integrations/sources/google-ads githubIssueLabel: source-google-ads icon: google-adwords.svg license: Elv2 @@ -18,11 +24,45 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/google-ads + releases: + breakingChanges: + 1.0.0: + message: + This release introduces fixes to custom query schema creation. Users + should refresh the source schema and reset affected streams after upgrading + to ensure uninterrupted syncs. + upgradeDeadline: "2023-10-31" + 2.0.0: + message: + This release updates the Source Google Ads connector so that its + default streams and stream names match the related resources in Google Ads + API. Users should refresh the source schema and reset affected streams after + upgrading to ensure uninterrupted syncs. + upgradeDeadline: "2023-11-30" + 3.0.0: + message: Google is deprecating v13 of the Google Ads API in January. + This release upgrades the Google Ads API to the latest version (v15), which causes changes in several schemas. + Users should refresh the source schema and reset affected streams after upgrading to ensure uninterrupted syncs. + upgradeDeadline: "2024-01-12" + suggestedStreams: + streams: + - campaigns + - accounts + - ad_group_ads + - ad_group_ad_report + - ad_groups + - click_view + - account_performance_report + - keyword_report + - campaign_labels + - ad_group_labels + - ad_group_ad_labels + - user_location_report + - geographic_report + - display_keyword_performance_report + - shopping_performance_report + - display_topics_performance_report + supportLevel: certified tags: - language:python - ab_internal: - sl: 300 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-google-ads/setup.py b/airbyte-integrations/connectors/source-google-ads/setup.py index 35bfce507db0..7211a092ff54 100644 --- a/airbyte-integrations/connectors/source-google-ads/setup.py +++ b/airbyte-integrations/connectors/source-google-ads/setup.py @@ -7,11 +7,18 @@ # pin protobuf==3.20.0 as other versions may cause problems on different architectures # (see https://github.com/airbytehq/airbyte/issues/13580) -MAIN_REQUIREMENTS = ["airbyte-cdk>=0.2.2", "google-ads==20.0.0", "protobuf", "pendulum"] +# pendulum <3.0.0 is required to align with the CDK version, and should be updated once the next issue is resolved: +# https://github.com/airbytehq/airbyte/issues/33573 +MAIN_REQUIREMENTS = ["airbyte-cdk>=0.51.3", "google-ads==22.1.0", "protobuf", "pendulum<3.0.0"] TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock", "freezegun", "requests-mock"] setup( + entry_points={ + "console_scripts": [ + "source-google-ads=source_google_ads.run:run", + ], + }, name="source_google_ads", description="Source implementation for Google Ads.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/config_migrations.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/config_migrations.py new file mode 100644 index 000000000000..be206ee13e62 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/config_migrations.py @@ -0,0 +1,131 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import logging +from typing import Any, List, Mapping + +from airbyte_cdk.config_observation import create_connector_config_control_message +from airbyte_cdk.entrypoint import AirbyteEntrypoint +from airbyte_cdk.models import FailureType +from airbyte_cdk.sources import Source +from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository +from airbyte_cdk.utils import AirbyteTracedException + +from .utils import GAQL + +logger = logging.getLogger("airbyte_logger") + +FULL_REFRESH_CUSTOM_TABLE = [ + "asset", + "asset_group_listing_group_filter", + "custom_audience", + "geo_target_constant", + "change_event", + "change_status", +] + + +class MigrateCustomQuery: + """ + This class stands for migrating the config at runtime. + This migration is backwards compatible with the previous version, as new property will be created. + When falling back to the previous source version connector will use old property `custom_queries`. + + Add `segments.date` for all queries where it was previously added by IncrementalCustomQuery class. + """ + + message_repository: MessageRepository = InMemoryMessageRepository() + + @classmethod + def should_migrate(cls, config: Mapping[str, Any]) -> bool: + """ + Determines if a configuration requires migration. + + Args: + - config (Mapping[str, Any]): The configuration data to check. + + Returns: + - True: If the configuration requires migration. + - False: Otherwise. + """ + return "custom_queries_array" not in config + + @classmethod + def update_custom_queries(cls, config: Mapping[str, Any], source: Source = None) -> Mapping[str, Any]: + """ + Update custom queries with segments.date field. + + Args: + - config (Mapping[str, Any]): The configuration from which the key should be removed. + - source (Source, optional): The data source. Defaults to None. + + Returns: + - Mapping[str, Any]: The configuration after removing the key. + """ + custom_queries = [] + for query in config.get("custom_queries", []): + new_query = query.copy() + try: + query_object = GAQL.parse(query["query"]) + except ValueError: + message = f"The custom GAQL query {query['table_name']} failed. Validate your GAQL query with the Google Ads query validator. https://developers.google.com/google-ads/api/fields/v13/query_validator" + raise AirbyteTracedException(message=message, failure_type=FailureType.config_error) + + if query_object.resource_name not in FULL_REFRESH_CUSTOM_TABLE and "segments.date" not in query_object.fields: + query_object = query_object.append_field("segments.date") + + new_query["query"] = str(query_object) + custom_queries.append(new_query) + + config["custom_queries_array"] = custom_queries + return config + + @classmethod + def modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Modifies the configuration and then saves it back to the source. + + Args: + - config_path (str): The path where the configuration is stored. + - source (Source): The data source. + - config (Mapping[str, Any]): The current configuration. + + Returns: + - Mapping[str, Any]: The updated configuration. + """ + migrated_config = cls.update_custom_queries(config, source) + source.write_config(migrated_config, config_path) + return migrated_config + + @classmethod + def emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: + """ + Emits the control messages related to configuration migration. + + Args: + - migrated_config (Mapping[str, Any]): The migrated configuration. + """ + cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) + for message in cls.message_repository._message_queue: + print(message.json(exclude_unset=True)) + + @classmethod + def migrate(cls, args: List[str], source: Source) -> None: + """ + Orchestrates the configuration migration process. + + It first checks if the `--config` argument is provided, and if so, + determines whether migration is needed, and then performs the migration + if required. + + Args: + - args (List[str]): List of command-line arguments. + - source (Source): The data source. + """ + config_path = AirbyteEntrypoint(source).extract_config(args) + if config_path: + config = source.read_config(config_path) + if cls.should_migrate(config): + cls.emit_control_message(cls.modify_and_save(config_path, source, config)) diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py index 2cf6066c4882..4a3ac096cfdd 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py @@ -2,12 +2,15 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from functools import lru_cache from typing import Any, Dict, Mapping from .streams import GoogleAdsStream, IncrementalGoogleAdsStream from .utils import GAQL +DATE_TYPES = ("segments.date", "segments.month", "segments.quarter", "segments.week") + class CustomQueryMixin: def __init__(self, config, **kwargs): @@ -52,6 +55,8 @@ def get_json_schema(self) -> Dict[str, Any]: "STRING": "string", "BOOLEAN": "boolean", "DATE": "string", + "MESSAGE": "string", + "ENUM": "string", } fields = list(self.config["query"].fields) if self.cursor_field: @@ -62,25 +67,17 @@ def get_json_schema(self) -> Dict[str, Any]: node = google_schema.get(field) # Data type return in enum format: "GoogleAdsFieldDataType." google_data_type = node.data_type.name + field_value = {"type": [google_datatype_mapping.get(google_data_type, "string"), "null"]} + + # Google Ads doesn't differentiate between DATE and DATETIME, so we need to manually check for fields with known type + if google_data_type == "DATE" and field in DATE_TYPES: + field_value["format"] = "date" + if google_data_type == "ENUM": field_value = {"type": "string", "enum": list(node.enum_values)} - if node.is_repeated: - field_value = {"type": ["null", "array"], "items": field_value} - elif google_data_type == "MESSAGE": - # Represents protobuf message and could be anything, set custom - # attribute "protobuf_message" to convert it to a string (or - # array of strings) later. - # https://developers.google.com/google-ads/api/reference/rpc/v11/GoogleAdsFieldDataTypeEnum.GoogleAdsFieldDataType?hl=en#message - if node.is_repeated: - output_type = ["array", "null"] - else: - output_type = ["string", "null"] - field_value = {"type": output_type, "protobuf_message": True} - else: - output_type = [google_datatype_mapping.get(google_data_type, "string"), "null"] - field_value = {"type": output_type} - if google_data_type == "DATE": - field_value["format"] = "date" + + if node.is_repeated: + field_value = {"type": ["null", "array"], "items": field_value} local_json_schema["properties"][field] = field_value diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py index dc67173a5d85..c34833154ce6 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py @@ -5,48 +5,18 @@ import logging from enum import Enum -from typing import Any, Iterator, List, Mapping, MutableMapping +from typing import Any, Iterable, Iterator, List, Mapping, MutableMapping import backoff from airbyte_cdk.models import FailureType from airbyte_cdk.utils import AirbyteTracedException from google.ads.googleads.client import GoogleAdsClient -from google.ads.googleads.v13.services.types.google_ads_service import GoogleAdsRow, SearchGoogleAdsResponse -from google.api_core.exceptions import ServerError, TooManyRequests +from google.ads.googleads.v15.services.types.google_ads_service import GoogleAdsRow, SearchGoogleAdsResponse +from google.api_core.exceptions import InternalServerError, ServerError, TooManyRequests from google.auth import exceptions from proto.marshal.collections import Repeated, RepeatedComposite -REPORT_MAPPING = { - "accounts": "customer", - "account_labels": "customer_label", - "account_performance_report": "customer", - "ad_group_ads": "ad_group_ad", - "ad_group_ad_labels": "ad_group_ad_label", - "ad_group_ad_report": "ad_group_ad", - "ad_groups": "ad_group", - "ad_group_bidding_strategies": "ad_group", - "ad_group_criterions": "ad_group_criterion", - "ad_group_criterion_labels": "ad_group_criterion_label", - "ad_group_labels": "ad_group_label", - "ad_listing_group_criterions": "ad_group_criterion", - "audience": "audience", - "campaigns": "campaign", - "campaign_real_time_bidding_settings": "campaign", - "campaign_bidding_strategies": "campaign", - "campaign_budget": "campaign_budget", - "campaign_labels": "campaign_label", - "click_view": "click_view", - "display_keyword_performance_report": "display_keyword_view", - "display_topics_performance_report": "topic_view", - "geographic_report": "geographic_view", - "keyword_report": "keyword_view", - "labels": "label", - "service_accounts": "customer", - "shopping_performance_report": "shopping_performance_view", - "user_interest": "user_interest", - "user_location_report": "user_location_view", -} -API_VERSION = "v13" +API_VERSION = "v15" logger = logging.getLogger("airbyte") @@ -57,8 +27,28 @@ def __init__(self, credentials: MutableMapping[str, Any]): # `google-ads` library version `14.0.0` and higher requires an additional required parameter `use_proto_plus`. # More details can be found here: https://developers.google.com/google-ads/api/docs/client-libs/python/protobuf-messages credentials["use_proto_plus"] = True - self.client = self.get_google_ads_client(credentials) - self.ga_service = self.client.get_service("GoogleAdsService") + self.clients = {} + self.ga_services = {} + self.credentials = credentials + + self.clients["default"] = self.get_google_ads_client(credentials) + self.ga_services["default"] = self.clients["default"].get_service("GoogleAdsService") + + self.customer_service = self.clients["default"].get_service("CustomerService") + + def get_client(self, login_customer_id="default"): + if login_customer_id in self.clients: + return self.clients[login_customer_id] + new_creds = self.credentials.copy() + new_creds["login_customer_id"] = login_customer_id + self.clients[login_customer_id] = self.get_google_ads_client(new_creds) + return self.clients[login_customer_id] + + def ga_service(self, login_customer_id="default"): + if login_customer_id in self.ga_services: + return self.ga_services[login_customer_id] + self.ga_services[login_customer_id] = self.clients[login_customer_id].get_service("GoogleAdsService") + return self.ga_services[login_customer_id] @staticmethod def get_google_ads_client(credentials) -> GoogleAdsClient: @@ -68,21 +58,29 @@ def get_google_ads_client(credentials) -> GoogleAdsClient: message = "The authentication to Google Ads has expired. Re-authenticate to restore access to Google Ads." raise AirbyteTracedException(message=message, failure_type=FailureType.config_error) from e + def get_accessible_accounts(self): + customer_resource_names = self.customer_service.list_accessible_customers().resource_names + logger.info(f"Found {len(customer_resource_names)} accessible accounts: {customer_resource_names}") + + for customer_resource_name in customer_resource_names: + customer_id = self.ga_service().parse_customer_path(customer_resource_name)["customer_id"] + yield customer_id + @backoff.on_exception( backoff.expo, - (ServerError, TooManyRequests), + (InternalServerError, ServerError, TooManyRequests), on_backoff=lambda details: logger.info( f"Caught retryable error after {details['tries']} tries. Waiting {details['wait']} seconds then retrying..." ), max_tries=5, ) - def send_request(self, query: str, customer_id: str) -> Iterator[SearchGoogleAdsResponse]: - client = self.client + def send_request(self, query: str, customer_id: str, login_customer_id: str = "default") -> Iterator[SearchGoogleAdsResponse]: + client = self.get_client(login_customer_id) search_request = client.get_type("SearchGoogleAdsRequest") search_request.query = query search_request.page_size = self.DEFAULT_PAGE_SIZE search_request.customer_id = customer_id - return [self.ga_service.search(search_request)] + return [self.ga_service(login_customer_id).search(search_request)] def get_fields_metadata(self, fields: List[str]) -> Mapping[str, Any]: """ @@ -91,8 +89,8 @@ def get_fields_metadata(self, fields: List[str]) -> Mapping[str, Any]: :return dict of fields type info. """ - ga_field_service = self.client.get_service("GoogleAdsFieldService") - request = self.client.get_type("SearchGoogleAdsFieldsRequest") + ga_field_service = self.get_client().get_service("GoogleAdsFieldService") + request = self.get_client().get_type("SearchGoogleAdsFieldsRequest") request.page_size = len(fields) fields_sql = ",".join([f"'{field}'" for field in fields]) request.query = f""" @@ -113,16 +111,36 @@ def get_fields_from_schema(schema: Mapping[str, Any]) -> List[str]: @staticmethod def convert_schema_into_query( - schema: Mapping[str, Any], report_name: str, from_date: str = None, to_date: str = None, cursor_field: str = None + fields: Iterable[str], + table_name: str, + conditions: List[str] = None, + order_field: str = None, + limit: int = None, ) -> str: - from_category = REPORT_MAPPING[report_name] - fields = GoogleAds.get_fields_from_schema(schema) - fields = ", ".join(fields) + """ + Constructs a Google Ads query based on the provided parameters. + + Args: + - fields (Iterable[str]): List of fields to be selected in the query. + - table_name (str): Name of the table from which data will be selected. + - conditions (List[str], optional): List of conditions to be applied in the WHERE clause. Defaults to None. + - order_field (str, optional): Field by which the results should be ordered. Defaults to None. + - limit (int, optional): Maximum number of results to be returned. Defaults to None. + + Returns: + - str: Constructed Google Ads query. + """ - query_template = f"SELECT {fields} FROM {from_category}" + query_template = f"SELECT {', '.join(fields)} FROM {table_name}" - if cursor_field: - query_template += f" WHERE {cursor_field} >= '{from_date}' AND {cursor_field} <= '{to_date}' ORDER BY {cursor_field} ASC" + if conditions: + query_template += " WHERE " + " AND ".join(conditions) + + if order_field: + query_template += f" ORDER BY {order_field} ASC" + + if limit: + query_template += f" LIMIT {limit}" return query_template @@ -180,25 +198,18 @@ def get_field_value(field_value: GoogleAdsRow, field: str, schema_type: Mapping[ elif isinstance(field_value, (Repeated, RepeatedComposite)): field_value = [str(value) for value in field_value] - # Google Ads has a lot of entities inside itself and we cannot process them all separately, because: + # Google Ads has a lot of entities inside itself, and we cannot process them all separately, because: # 1. It will take a long time # 2. We have no way to get data on absolutely all entities to test. # # To prevent JSON from throwing an error during deserialization, we made such a hack. # For example: - # 1. ad_group_ad.ad.responsive_display_ad.long_headline - type AdTextAsset (https://developers.google.com/google-ads/api/reference/rpc/v6/AdTextAsset?hl=en). - # 2. ad_group_ad.ad.legacy_app_install_ad - type LegacyAppInstallAdInfo (https://developers.google.com/google-ads/api/reference/rpc/v7/LegacyAppInstallAdInfo?hl=en). - # - if not (isinstance(field_value, (list, int, float, str, bool, dict)) or field_value is None): + # 1. ad_group_ad.ad.responsive_display_ad.long_headline - type AdTextAsset + # (https://developers.google.com/google-ads/api/reference/rpc/v6/AdTextAsset?hl=en). + # 2. ad_group_ad.ad.legacy_app_install_ad - type LegacyAppInstallAdInfo + # (https://developers.google.com/google-ads/api/reference/rpc/v7/LegacyAppInstallAdInfo?hl=en). + if not isinstance(field_value, (list, int, float, str, bool, dict)) and field_value is not None: field_value = str(field_value) - # In case of custom query field has MESSAGE type it represents protobuf - # message and could be anything, convert it to a string or array of - # string if it has "repeated" flag on metadata - if schema_type.get("protobuf_message"): - if "array" in schema_type.get("type"): - field_value = [str(field) for field in field_value] - else: - field_value = str(field_value) return field_value diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/models.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/models.py index 768f0c61dba8..7da4ed7c2b9c 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/models.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/models.py @@ -2,28 +2,34 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from dataclasses import dataclass -from typing import Any, Iterable, Mapping, Union +from typing import Any, Iterable, Mapping -from pendulum import timezone +from pendulum import local_timezone, timezone from pendulum.tz.timezone import Timezone @dataclass -class Customer: +class CustomerModel: id: str - time_zone: Union[timezone, str] = "local" + time_zone: timezone = local_timezone() is_manager_account: bool = False + login_customer_id: str = None @classmethod - def from_accounts(cls, accounts: Iterable[Iterable[Mapping[str, Any]]]): + def from_accounts(cls, accounts: Iterable[Mapping[str, Any]]) -> Iterable["CustomerModel"]: data_objects = [] - for account_list in accounts: - for account in account_list: - time_zone_name = account.get("customer.time_zone") - tz = Timezone(time_zone_name) if time_zone_name else "local" + for account in accounts: + time_zone_name = account.get("customer_client.time_zone") + tz = Timezone(time_zone_name) if time_zone_name else local_timezone() - data_objects.append( - cls(id=str(account["customer.id"]), time_zone=tz, is_manager_account=bool(account.get("customer.manager"))) + data_objects.append( + cls( + id=str(account["customer_client.id"]), + time_zone=tz, + is_manager_account=bool(account.get("customer_client.manager")), + login_customer_id=account.get("login_customer_id"), ) + ) return data_objects diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/run.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/run.py new file mode 100644 index 000000000000..dd759a035015 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/run.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_google_ads import SourceGoogleAds +from source_google_ads.config_migrations import MigrateCustomQuery + + +def run(): + source = SourceGoogleAds() + MigrateCustomQuery.migrate(sys.argv[1:], source) + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group.json new file mode 100644 index 000000000000..96dbdc94edea --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group.json @@ -0,0 +1,109 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "campaign.id": { + "type": ["null", "integer"] + }, + "ad_group.ad_rotation_mode": { + "type": ["null", "string"] + }, + "ad_group.base_ad_group": { + "type": ["null", "string"] + }, + "ad_group.campaign": { + "type": ["null", "string"] + }, + "metrics.cost_micros": { + "type": ["null", "integer"] + }, + "ad_group.cpc_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group.cpm_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group.cpv_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group.display_custom_bid_dimension": { + "type": ["null", "string"] + }, + "ad_group.effective_target_cpa_micros": { + "type": ["null", "integer"] + }, + "ad_group.effective_target_cpa_source": { + "type": ["null", "string"] + }, + "ad_group.effective_target_roas": { + "type": ["null", "number"] + }, + "ad_group.effective_target_roas_source": { + "type": ["null", "string"] + }, + "ad_group.excluded_parent_asset_field_types": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group.optimized_targeting_enabled": { + "type": ["null", "boolean"] + }, + "ad_group.final_url_suffix": { + "type": ["null", "string"] + }, + "ad_group.id": { + "type": ["null", "integer"] + }, + "ad_group.labels": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group.name": { + "type": ["null", "string"] + }, + "ad_group.percent_cpc_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group.resource_name": { + "type": ["null", "string"] + }, + "ad_group.status": { + "type": ["null", "string"] + }, + "ad_group.target_cpa_micros": { + "type": ["null", "integer"] + }, + "ad_group.target_cpm_micros": { + "type": ["null", "integer"] + }, + "ad_group.target_roas": { + "type": ["null", "number"] + }, + "ad_group.targeting_setting.target_restrictions": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group.tracking_url_template": { + "type": ["null", "string"] + }, + "ad_group.type": { + "type": ["null", "string"] + }, + "ad_group.url_custom_parameters": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "segments.date": { + "type": ["null", "string"], + "format": "date" + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad.json new file mode 100644 index 000000000000..f11c82c5489b --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad.json @@ -0,0 +1,532 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ad_group.id": { + "type": ["null", "integer"] + }, + "ad_group_ad.ad.added_by_google_ads": { + "type": ["null", "boolean"] + }, + "ad_group_ad.ad.app_ad.descriptions": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.app_ad.headlines": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.app_ad.html5_media_bundles": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.app_ad.images": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.app_ad.mandatory_ad_text": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.app_ad.youtube_videos": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.app_engagement_ad.descriptions": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.app_engagement_ad.headlines": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.app_engagement_ad.images": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.app_engagement_ad.videos": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.call_ad.business_name": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.call_ad.call_tracked": { + "type": ["null", "boolean"] + }, + "ad_group_ad.ad.call_ad.conversion_action": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.call_ad.conversion_reporting_state": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.call_ad.country_code": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.call_ad.description1": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.call_ad.description2": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.call_ad.disable_call_conversion": { + "type": ["null", "boolean"] + }, + "ad_group_ad.ad.call_ad.headline1": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.call_ad.headline2": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.call_ad.path1": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.call_ad.path2": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.call_ad.phone_number": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.call_ad.phone_number_verification_url": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.device_preference": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.display_upload_ad.display_upload_product_type": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.display_upload_ad.media_bundle": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.display_url": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.expanded_dynamic_search_ad.description": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.expanded_dynamic_search_ad.description2": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.expanded_text_ad.description": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.expanded_text_ad.description2": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.expanded_text_ad.headline_part1": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.expanded_text_ad.headline_part2": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.expanded_text_ad.headline_part3": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.expanded_text_ad.path1": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.expanded_text_ad.path2": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.final_app_urls": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.final_mobile_urls": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.final_url_suffix": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.final_urls": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.hotel_ad": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.id": { + "type": ["null", "integer"] + }, + "ad_group_ad.ad.image_ad.image_url": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.image_ad.mime_type": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.image_ad.name": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.image_ad.pixel_height": { + "type": ["null", "integer"] + }, + "ad_group_ad.ad.image_ad.pixel_width": { + "type": ["null", "integer"] + }, + "ad_group_ad.ad.image_ad.preview_image_url": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.image_ad.preview_pixel_height": { + "type": ["null", "integer"] + }, + "ad_group_ad.ad.image_ad.preview_pixel_width": { + "type": ["null", "integer"] + }, + "ad_group_ad.ad.legacy_app_install_ad": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": { + "type": ["null", "boolean"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.business_name": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.description": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.main_color": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.local_ad.call_to_actions": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.local_ad.descriptions": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.local_ad.headlines": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.local_ad.logo_images": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.local_ad.marketing_images": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.local_ad.path1": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.local_ad.path2": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.local_ad.videos": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.name": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.resource_name": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.responsive_display_ad.accent_color": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": { + "type": ["null", "boolean"] + }, + "ad_group_ad.ad.responsive_display_ad.business_name": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.responsive_display_ad.call_to_action_text": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": { + "type": ["null", "boolean"] + }, + "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": { + "type": ["null", "boolean"] + }, + "ad_group_ad.ad.responsive_display_ad.descriptions": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.responsive_display_ad.format_setting": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.responsive_display_ad.headlines": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.responsive_display_ad.logo_images": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.responsive_display_ad.long_headline": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.responsive_display_ad.main_color": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.responsive_display_ad.marketing_images": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.responsive_display_ad.price_prefix": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.responsive_display_ad.promo_text": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.responsive_display_ad.square_logo_images": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.responsive_display_ad.square_marketing_images": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.responsive_display_ad.youtube_videos": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.responsive_search_ad.descriptions": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.responsive_search_ad.headlines": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.responsive_search_ad.path1": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.responsive_search_ad.path2": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.shopping_comparison_listing_ad.headline": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.shopping_product_ad": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.shopping_smart_ad": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.smart_campaign_ad.descriptions": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.smart_campaign_ad.headlines": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.system_managed_resource_source": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.text_ad.description1": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.text_ad.description2": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.text_ad.headline": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.tracking_url_template": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.type": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.url_collections": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.url_custom_parameters": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.video_ad.in_feed.description1": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.video_ad.in_feed.description2": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.video_ad.in_feed.headline": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.video_ad.in_stream.action_button_label": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.video_ad.in_stream.action_headline": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.video_ad.out_stream.description": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.video_ad.out_stream.headline": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.video_responsive_ad.call_to_actions": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.video_responsive_ad.companion_banners": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.video_responsive_ad.descriptions": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.video_responsive_ad.headlines": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.video_responsive_ad.long_headlines": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.video_responsive_ad.videos": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad_group": { + "type": ["null", "string"] + }, + "ad_group_ad.ad_strength": { + "type": ["null", "string"] + }, + "ad_group_ad.labels": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.policy_summary.approval_status": { + "type": ["null", "string"] + }, + "ad_group_ad.policy_summary.policy_topic_entries": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.policy_summary.review_status": { + "type": ["null", "string"] + }, + "ad_group_ad.resource_name": { + "type": ["null", "string"] + }, + "ad_group_ad.status": { + "type": ["null", "string"] + }, + "segments.date": { + "type": ["null", "string"], + "format": "date" + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_label.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_label.json new file mode 100644 index 000000000000..266649b8f125 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_label.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ad_group.id": { + "type": ["null", "integer"] + }, + "ad_group_ad.ad.id": { + "type": ["null", "integer"] + }, + "ad_group_ad.ad.resource_name": { + "type": ["null", "string"] + }, + "ad_group_ad_label.resource_name": { + "type": ["null", "string"] + }, + "label.name": { + "type": ["null", "string"] + }, + "label.resource_name": { + "type": ["null", "string"] + }, + "label.id": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_labels.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_labels.json deleted file mode 100644 index 50c0377ae578..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_labels.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "ad_group_ad.ad.resource_name": { - "type": ["null", "string"] - }, - "ad_group_ad_label.resource_name": { - "type": ["null", "string"] - }, - "label.name": { - "type": ["null", "string"] - }, - "label.resource_name": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_legacy.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_legacy.json new file mode 100644 index 000000000000..985a41e03bfe --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_legacy.json @@ -0,0 +1,481 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": { + "type": ["null", "string"] + }, + "ad_group.id": { + "type": ["null", "integer"] + }, + "customer.currency_code": { + "type": ["null", "string"] + }, + "customer.descriptive_name": { + "type": ["null", "string"] + }, + "customer.time_zone": { + "type": ["null", "string"] + }, + "metrics.active_view_cpm": { + "type": ["null", "number"] + }, + "metrics.active_view_ctr": { + "type": ["null", "number"] + }, + "metrics.active_view_impressions": { + "type": ["null", "integer"] + }, + "metrics.active_view_measurability": { + "type": ["null", "number"] + }, + "metrics.active_view_measurable_cost_micros": { + "type": ["null", "integer"] + }, + "metrics.active_view_measurable_impressions": { + "type": ["null", "integer"] + }, + "metrics.active_view_viewability": { + "type": ["null", "number"] + }, + "ad_group_ad.ad_group": { + "type": ["null", "string"] + }, + "ad_group.name": { + "type": ["null", "string"] + }, + "ad_group.status": { + "type": ["null", "string"] + }, + "segments.ad_network_type": { + "type": ["null", "string"] + }, + "ad_group_ad.ad_strength": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.type": { + "type": ["null", "string"] + }, + "metrics.all_conversions_from_interactions_rate": { + "type": ["null", "number"] + }, + "metrics.all_conversions_value": { + "type": ["null", "number"] + }, + "metrics.all_conversions": { + "type": ["null", "number"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": { + "type": ["null", "boolean"] + }, + "ad_group_ad.ad.added_by_google_ads": { + "type": ["null", "boolean"] + }, + "metrics.average_cost": { + "type": ["null", "number"] + }, + "metrics.average_cpc": { + "type": ["null", "number"] + }, + "metrics.average_cpe": { + "type": ["null", "number"] + }, + "metrics.average_cpm": { + "type": ["null", "number"] + }, + "metrics.average_cpv": { + "type": ["null", "number"] + }, + "metrics.average_page_views": { + "type": ["null", "number"] + }, + "metrics.average_time_on_site": { + "type": ["null", "number"] + }, + "ad_group.base_ad_group": { + "type": ["null", "string"] + }, + "campaign.base_campaign": { + "type": ["null", "string"] + }, + "metrics.bounce_rate": { + "type": ["null", "number"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.business_name": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": { + "type": ["null", "string"] + }, + "campaign.id": { + "type": ["null", "integer"] + }, + "campaign.name": { + "type": ["null", "string"] + }, + "campaign.status": { + "type": ["null", "string"] + }, + "metrics.clicks": { + "type": ["null", "integer"] + }, + "ad_group_ad.policy_summary.approval_status": { + "type": ["null", "string"] + }, + "metrics.conversions_from_interactions_rate": { + "type": ["null", "number"] + }, + "metrics.conversions_value": { + "type": ["null", "number"] + }, + "metrics.conversions": { + "type": ["null", "number"] + }, + "metrics.cost_micros": { + "type": ["null", "integer"] + }, + "metrics.cost_per_all_conversions": { + "type": ["null", "number"] + }, + "metrics.cost_per_conversion": { + "type": ["null", "number"] + }, + "metrics.cost_per_current_model_attributed_conversion": { + "type": ["null", "number"] + }, + "ad_group_ad.ad.final_mobile_urls": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.final_urls": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.tracking_url_template": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.url_custom_parameters": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "metrics.cross_device_conversions": { + "type": ["null", "number"] + }, + "metrics.ctr": { + "type": ["null", "number"] + }, + "metrics.current_model_attributed_conversions_value": { + "type": ["null", "number"] + }, + "metrics.current_model_attributed_conversions": { + "type": ["null", "number"] + }, + "segments.date": { + "type": ["null", "string"], + "format": "date" + }, + "segments.day_of_week": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.expanded_text_ad.description": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.text_ad.description1": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.text_ad.description2": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.device_preference": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.display_url": { + "type": ["null", "string"] + }, + "metrics.engagement_rate": { + "type": ["null", "number"] + }, + "metrics.engagements": { + "type": ["null", "integer"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.expanded_dynamic_search_ad.description": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.expanded_text_ad.description2": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.expanded_text_ad.headline_part3": { + "type": ["null", "string"] + }, + "customer.id": { + "type": ["null", "integer"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": { + "type": ["null", "string"] + }, + "metrics.gmail_forwards": { + "type": ["null", "integer"] + }, + "metrics.gmail_saves": { + "type": ["null", "integer"] + }, + "metrics.gmail_secondary_clicks": { + "type": ["null", "integer"] + }, + "ad_group_ad.ad.text_ad.headline": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.expanded_text_ad.headline_part1": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.expanded_text_ad.headline_part2": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.id": { + "type": ["null", "integer"] + }, + "ad_group_ad.ad.image_ad.image_url": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.image_ad.pixel_height": { + "type": ["null", "integer"] + }, + "ad_group_ad.ad.image_ad.pixel_width": { + "type": ["null", "integer"] + }, + "ad_group_ad.ad.image_ad.mime_type": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.image_ad.name": { + "type": ["null", "string"] + }, + "metrics.impressions": { + "type": ["null", "integer"] + }, + "metrics.interaction_rate": { + "type": ["null", "number"] + }, + "metrics.interaction_event_types": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "metrics.interactions": { + "type": ["null", "integer"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.main_color": { + "type": ["null", "string"] + }, + "segments.month": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.responsive_display_ad.accent_color": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": { + "type": ["null", "boolean"] + }, + "ad_group_ad.ad.responsive_display_ad.business_name": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.responsive_display_ad.call_to_action_text": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.responsive_display_ad.descriptions": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.responsive_display_ad.price_prefix": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.responsive_display_ad.promo_text": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.responsive_display_ad.format_setting": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.responsive_display_ad.headlines": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.responsive_display_ad.logo_images": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.responsive_display_ad.square_logo_images": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.responsive_display_ad.long_headline": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.responsive_display_ad.main_color": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.responsive_display_ad.marketing_images": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.responsive_display_ad.square_marketing_images": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.responsive_display_ad.youtube_videos": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.expanded_text_ad.path1": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.expanded_text_ad.path2": { + "type": ["null", "string"] + }, + "metrics.percent_new_visitors": { + "type": ["null", "number"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": { + "type": ["null", "string"] + }, + "segments.quarter": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.responsive_search_ad.descriptions": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.responsive_search_ad.headlines": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.responsive_search_ad.path1": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.responsive_search_ad.path2": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": { + "type": ["null", "string"] + }, + "ad_group_ad.status": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.system_managed_resource_source": { + "type": ["null", "string"] + }, + "metrics.top_impression_percentage": { + "type": ["null", "number"] + }, + "ad_group_ad.ad.app_ad.descriptions": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.app_ad.headlines": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.app_ad.html5_media_bundles": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.app_ad.images": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_ad.ad.app_ad.mandatory_ad_text": { + "type": ["null", "string"] + }, + "ad_group_ad.ad.app_ad.youtube_videos": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "metrics.value_per_all_conversions": { + "type": ["null", "number"] + }, + "metrics.value_per_conversion": { + "type": ["null", "number"] + }, + "metrics.value_per_current_model_attributed_conversion": { + "type": ["null", "number"] + }, + "metrics.video_quartile_p100_rate": { + "type": ["null", "number"] + }, + "metrics.video_quartile_p25_rate": { + "type": ["null", "number"] + }, + "metrics.video_quartile_p50_rate": { + "type": ["null", "number"] + }, + "metrics.video_quartile_p75_rate": { + "type": ["null", "number"] + }, + "metrics.video_view_rate": { + "type": ["null", "number"] + }, + "metrics.video_views": { + "type": ["null", "integer"] + }, + "metrics.view_through_conversions": { + "type": ["null", "integer"] + }, + "segments.week": { + "type": ["null", "string"] + }, + "segments.year": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_report.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_report.json deleted file mode 100644 index 52d02607d620..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ad_report.json +++ /dev/null @@ -1,427 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": { - "type": ["null", "string"] - }, - "ad_group.id": { - "type": ["null", "integer"] - }, - "customer.currency_code": { - "type": ["null", "string"] - }, - "customer.descriptive_name": { - "type": ["null", "string"] - }, - "customer.time_zone": { - "type": ["null", "string"] - }, - "metrics.active_view_cpm": { - "type": ["null", "number"] - }, - "metrics.active_view_ctr": { - "type": ["null", "number"] - }, - "metrics.active_view_impressions": { - "type": ["null", "integer"] - }, - "metrics.active_view_measurability": { - "type": ["null", "number"] - }, - "metrics.active_view_measurable_cost_micros": { - "type": ["null", "integer"] - }, - "metrics.active_view_measurable_impressions": { - "type": ["null", "integer"] - }, - "metrics.active_view_viewability": { - "type": ["null", "number"] - }, - "ad_group_ad.ad_group": { - "type": ["null", "string"] - }, - "ad_group.name": { - "type": ["null", "string"] - }, - "ad_group.status": { - "type": ["null", "string"] - }, - "segments.ad_network_type": { - "type": ["null", "string"] - }, - "ad_group_ad.ad_strength": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.type": { - "type": ["null", "string"] - }, - "metrics.all_conversions_from_interactions_rate": { - "type": ["null", "number"] - }, - "metrics.all_conversions_value": { - "type": ["null", "number"] - }, - "metrics.all_conversions": { - "type": ["null", "number"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": { - "type": ["null", "boolean"] - }, - "ad_group_ad.ad.added_by_google_ads": { - "type": ["null", "boolean"] - }, - "metrics.average_cost": { - "type": ["null", "number"] - }, - "metrics.average_cpc": { - "type": ["null", "number"] - }, - "metrics.average_cpe": { - "type": ["null", "number"] - }, - "metrics.average_cpm": { - "type": ["null", "number"] - }, - "metrics.average_cpv": { - "type": ["null", "number"] - }, - "metrics.average_page_views": { - "type": ["null", "number"] - }, - "metrics.average_time_on_site": { - "type": ["null", "number"] - }, - "ad_group.base_ad_group": { - "type": ["null", "string"] - }, - "campaign.base_campaign": { - "type": ["null", "string"] - }, - "metrics.bounce_rate": { - "type": ["null", "number"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.business_name": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": { - "type": ["null", "string"] - }, - "campaign.id": { - "type": ["null", "integer"] - }, - "campaign.name": { - "type": ["null", "string"] - }, - "campaign.status": { - "type": ["null", "string"] - }, - "metrics.clicks": { - "type": ["null", "integer"] - }, - "ad_group_ad.policy_summary.approval_status": { - "type": ["null", "string"] - }, - "metrics.conversions_from_interactions_rate": { - "type": ["null", "number"] - }, - "metrics.conversions_value": { - "type": ["null", "number"] - }, - "metrics.conversions": { - "type": ["null", "number"] - }, - "metrics.cost_micros": { - "type": ["null", "integer"] - }, - "metrics.cost_per_all_conversions": { - "type": ["null", "number"] - }, - "metrics.cost_per_conversion": { - "type": ["null", "number"] - }, - "metrics.cost_per_current_model_attributed_conversion": { - "type": ["null", "number"] - }, - "ad_group_ad.ad.final_mobile_urls": { - "type": ["null", "array"] - }, - "ad_group_ad.ad.final_urls": { - "type": ["null", "array"] - }, - "ad_group_ad.ad.tracking_url_template": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.url_custom_parameters": { - "type": ["null", "array"] - }, - "metrics.cross_device_conversions": { - "type": ["null", "number"] - }, - "metrics.ctr": { - "type": ["null", "number"] - }, - "metrics.current_model_attributed_conversions_value": { - "type": ["null", "number"] - }, - "metrics.current_model_attributed_conversions": { - "type": ["null", "number"] - }, - "segments.date": { - "type": ["null", "string"], - "format": "date" - }, - "segments.day_of_week": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.expanded_text_ad.description": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.text_ad.description1": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.text_ad.description2": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.device_preference": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.display_url": { - "type": ["null", "string"] - }, - "metrics.engagement_rate": { - "type": ["null", "number"] - }, - "metrics.engagements": { - "type": ["null", "integer"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.expanded_dynamic_search_ad.description": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.expanded_text_ad.description2": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.expanded_text_ad.headline_part3": { - "type": ["null", "string"] - }, - "customer.id": { - "type": ["null", "integer"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": { - "type": ["null", "string"] - }, - "metrics.gmail_forwards": { - "type": ["null", "integer"] - }, - "metrics.gmail_saves": { - "type": ["null", "integer"] - }, - "metrics.gmail_secondary_clicks": { - "type": ["null", "integer"] - }, - "ad_group_ad.ad.text_ad.headline": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.expanded_text_ad.headline_part1": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.expanded_text_ad.headline_part2": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.id": { - "type": ["null", "integer"] - }, - "ad_group_ad.ad.image_ad.image_url": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.image_ad.pixel_height": { - "type": ["null", "integer"] - }, - "ad_group_ad.ad.image_ad.pixel_width": { - "type": ["null", "integer"] - }, - "ad_group_ad.ad.image_ad.mime_type": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.image_ad.name": { - "type": ["null", "string"] - }, - "metrics.impressions": { - "type": ["null", "integer"] - }, - "metrics.interaction_rate": { - "type": ["null", "number"] - }, - "metrics.interaction_event_types": { - "type": ["null", "array"] - }, - "metrics.interactions": { - "type": ["null", "integer"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.main_color": { - "type": ["null", "string"] - }, - "segments.month": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.responsive_display_ad.accent_color": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": { - "type": ["null", "boolean"] - }, - "ad_group_ad.ad.responsive_display_ad.business_name": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.responsive_display_ad.call_to_action_text": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.responsive_display_ad.descriptions": { - "type": ["null", "array"] - }, - "ad_group_ad.ad.responsive_display_ad.price_prefix": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.responsive_display_ad.promo_text": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.responsive_display_ad.format_setting": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.responsive_display_ad.headlines": { - "type": ["null", "array"] - }, - "ad_group_ad.ad.responsive_display_ad.logo_images": { - "type": ["null", "array"] - }, - "ad_group_ad.ad.responsive_display_ad.square_logo_images": { - "type": ["null", "array"] - }, - "ad_group_ad.ad.responsive_display_ad.long_headline": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.responsive_display_ad.main_color": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.responsive_display_ad.marketing_images": { - "type": ["null", "array"] - }, - "ad_group_ad.ad.responsive_display_ad.square_marketing_images": { - "type": ["null", "array"] - }, - "ad_group_ad.ad.responsive_display_ad.youtube_videos": { - "type": ["null", "array"] - }, - "ad_group_ad.ad.expanded_text_ad.path1": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.expanded_text_ad.path2": { - "type": ["null", "string"] - }, - "metrics.percent_new_visitors": { - "type": ["null", "number"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": { - "type": ["null", "string"] - }, - "segments.quarter": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.responsive_search_ad.descriptions": { - "type": ["null", "array"] - }, - "ad_group_ad.ad.responsive_search_ad.headlines": { - "type": ["null", "array"] - }, - "ad_group_ad.ad.responsive_search_ad.path1": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.responsive_search_ad.path2": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": { - "type": ["null", "string"] - }, - "ad_group_ad.status": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.system_managed_resource_source": { - "type": ["null", "string"] - }, - "metrics.top_impression_percentage": { - "type": ["null", "number"] - }, - "ad_group_ad.ad.app_ad.descriptions": { - "type": ["null", "array"] - }, - "ad_group_ad.ad.app_ad.headlines": { - "type": ["null", "array"] - }, - "ad_group_ad.ad.app_ad.html5_media_bundles": { - "type": ["null", "array"] - }, - "ad_group_ad.ad.app_ad.images": { - "type": ["null", "array"] - }, - "ad_group_ad.ad.app_ad.mandatory_ad_text": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.app_ad.youtube_videos": { - "type": ["null", "array"] - }, - "metrics.value_per_all_conversions": { - "type": ["null", "number"] - }, - "metrics.value_per_conversion": { - "type": ["null", "number"] - }, - "metrics.value_per_current_model_attributed_conversion": { - "type": ["null", "number"] - }, - "metrics.video_quartile_p100_rate": { - "type": ["null", "number"] - }, - "metrics.video_quartile_p25_rate": { - "type": ["null", "number"] - }, - "metrics.video_quartile_p50_rate": { - "type": ["null", "number"] - }, - "metrics.video_quartile_p75_rate": { - "type": ["null", "number"] - }, - "metrics.video_view_rate": { - "type": ["null", "number"] - }, - "metrics.video_views": { - "type": ["null", "integer"] - }, - "metrics.view_through_conversions": { - "type": ["null", "integer"] - }, - "segments.week": { - "type": ["null", "string"] - }, - "segments.year": { - "type": ["null", "integer"] - } - } -} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ads.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ads.json deleted file mode 100644 index 58b73e318ec8..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_ads.json +++ /dev/null @@ -1,529 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "ad_group_ad.ad.added_by_google_ads": { - "type": ["null", "boolean"] - }, - "ad_group_ad.ad.app_ad.descriptions": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.app_ad.headlines": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.app_ad.html5_media_bundles": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.app_ad.images": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.app_ad.mandatory_ad_text": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.app_ad.youtube_videos": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.app_engagement_ad.descriptions": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.app_engagement_ad.headlines": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.app_engagement_ad.images": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.app_engagement_ad.videos": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.call_ad.business_name": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.call_ad.call_tracked": { - "type": ["null", "boolean"] - }, - "ad_group_ad.ad.call_ad.conversion_action": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.call_ad.conversion_reporting_state": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.call_ad.country_code": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.call_ad.description1": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.call_ad.description2": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.call_ad.disable_call_conversion": { - "type": ["null", "boolean"] - }, - "ad_group_ad.ad.call_ad.headline1": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.call_ad.headline2": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.call_ad.path1": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.call_ad.path2": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.call_ad.phone_number": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.call_ad.phone_number_verification_url": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.device_preference": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.display_upload_ad.display_upload_product_type": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.display_upload_ad.media_bundle": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.display_url": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.expanded_dynamic_search_ad.description": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.expanded_dynamic_search_ad.description2": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.expanded_text_ad.description": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.expanded_text_ad.description2": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.expanded_text_ad.headline_part1": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.expanded_text_ad.headline_part2": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.expanded_text_ad.headline_part3": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.expanded_text_ad.path1": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.expanded_text_ad.path2": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.final_app_urls": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.final_mobile_urls": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.final_url_suffix": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.final_urls": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.hotel_ad": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.id": { - "type": ["null", "integer"] - }, - "ad_group_ad.ad.image_ad.image_url": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.image_ad.mime_type": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.image_ad.name": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.image_ad.pixel_height": { - "type": ["null", "integer"] - }, - "ad_group_ad.ad.image_ad.pixel_width": { - "type": ["null", "integer"] - }, - "ad_group_ad.ad.image_ad.preview_image_url": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.image_ad.preview_pixel_height": { - "type": ["null", "integer"] - }, - "ad_group_ad.ad.image_ad.preview_pixel_width": { - "type": ["null", "integer"] - }, - "ad_group_ad.ad.legacy_app_install_ad": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.accent_color": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.allow_flexible_color": { - "type": ["null", "boolean"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.business_name": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.call_to_action_text": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.description": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.format_setting": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.logo_image": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.long_headline": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.main_color": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.marketing_image": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.price_prefix": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.promo_text": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.short_headline": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.square_logo_image": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.legacy_responsive_display_ad.square_marketing_image": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.local_ad.call_to_actions": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.local_ad.descriptions": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.local_ad.headlines": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.local_ad.logo_images": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.local_ad.marketing_images": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.local_ad.path1": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.local_ad.path2": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.local_ad.videos": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.name": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.resource_name": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.responsive_display_ad.accent_color": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.responsive_display_ad.allow_flexible_color": { - "type": ["null", "boolean"] - }, - "ad_group_ad.ad.responsive_display_ad.business_name": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.responsive_display_ad.call_to_action_text": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.responsive_display_ad.control_spec.enable_asset_enhancements": { - "type": ["null", "boolean"] - }, - "ad_group_ad.ad.responsive_display_ad.control_spec.enable_autogen_video": { - "type": ["null", "boolean"] - }, - "ad_group_ad.ad.responsive_display_ad.descriptions": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.responsive_display_ad.format_setting": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.responsive_display_ad.headlines": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.responsive_display_ad.logo_images": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.responsive_display_ad.long_headline": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.responsive_display_ad.main_color": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.responsive_display_ad.marketing_images": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.responsive_display_ad.price_prefix": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.responsive_display_ad.promo_text": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.responsive_display_ad.square_logo_images": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.responsive_display_ad.square_marketing_images": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.responsive_display_ad.youtube_videos": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.responsive_search_ad.descriptions": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.responsive_search_ad.headlines": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.responsive_search_ad.path1": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.responsive_search_ad.path2": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.shopping_comparison_listing_ad.headline": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.shopping_product_ad": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.shopping_smart_ad": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.smart_campaign_ad.descriptions": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.smart_campaign_ad.headlines": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.system_managed_resource_source": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.text_ad.description1": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.text_ad.description2": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.text_ad.headline": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.tracking_url_template": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.type": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.url_collections": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.url_custom_parameters": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.video_ad.in_feed.description1": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.video_ad.in_feed.description2": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.video_ad.in_feed.headline": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.video_ad.in_stream.action_button_label": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.video_ad.in_stream.action_headline": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.video_ad.out_stream.description": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.video_ad.out_stream.headline": { - "type": ["null", "string"] - }, - "ad_group_ad.ad.video_responsive_ad.call_to_actions": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.video_responsive_ad.companion_banners": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.video_responsive_ad.descriptions": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.video_responsive_ad.headlines": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.video_responsive_ad.long_headlines": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad.video_responsive_ad.videos": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.ad_group": { - "type": ["null", "string"] - }, - "ad_group_ad.ad_strength": { - "type": ["null", "string"] - }, - "ad_group_ad.labels": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.policy_summary.approval_status": { - "type": ["null", "string"] - }, - "ad_group_ad.policy_summary.policy_topic_entries": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group_ad.policy_summary.review_status": { - "type": ["null", "string"] - }, - "ad_group_ad.resource_name": { - "type": ["null", "string"] - }, - "ad_group_ad.status": { - "type": ["null", "string"] - }, - "segments.date": { - "type": ["null", "string"], - "format": "date" - } - } -} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_bidding_strategies.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_bidding_strategies.json deleted file mode 100644 index bc1b798dc61e..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_bidding_strategies.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "ad_group.id": { - "type": ["null", "integer"] - }, - "bidding_strategy.aligned_campaign_budget_id": { - "type": ["null", "integer"] - }, - "bidding_strategy.campaign_count": { - "type": ["null", "integer"] - }, - "bidding_strategy.currency_code": { - "type": ["null", "string"] - }, - "bidding_strategy.effective_currency_code": { - "type": ["null", "string"] - }, - "bidding_strategy.enhanced_cpc": { - "type": ["null", "string"] - }, - "bidding_strategy.id": { - "type": ["null", "integer"] - }, - "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.maximize_conversion_value.target_roas": { - "type": ["null", "number"] - }, - "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.maximize_conversions.target_cpa_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.name": { - "type": ["null", "string"] - }, - "bidding_strategy.non_removed_campaign_count": { - "type": ["null", "integer"] - }, - "bidding_strategy.resource_name": { - "type": ["null", "string"] - }, - "bidding_strategy.status": { - "type": ["null", "string"] - }, - "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.target_cpa.cpc_bid_floor_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.target_cpa.target_cpa_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.target_impression_share.location": { - "type": ["null", "string"] - }, - "bidding_strategy.target_impression_share.location_fraction_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.target_roas.cpc_bid_ceiling_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.target_roas.cpc_bid_floor_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.target_roas.target_roas": { - "type": ["null", "number"] - }, - "bidding_strategy.target_spend.cpc_bid_ceiling_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.target_spend.target_spend_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.type": { - "type": ["null", "string"] - }, - "segments.date": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_bidding_strategy.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_bidding_strategy.json new file mode 100644 index 000000000000..cd9390b1f622 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_bidding_strategy.json @@ -0,0 +1,97 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ad_group.id": { + "type": ["null", "integer"] + }, + "bidding_strategy.aligned_campaign_budget_id": { + "type": ["null", "integer"] + }, + "bidding_strategy.campaign_count": { + "type": ["null", "integer"] + }, + "bidding_strategy.currency_code": { + "type": ["null", "string"] + }, + "bidding_strategy.effective_currency_code": { + "type": ["null", "string"] + }, + "bidding_strategy.enhanced_cpc": { + "type": ["null", "string"] + }, + "bidding_strategy.id": { + "type": ["null", "integer"] + }, + "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.maximize_conversion_value.target_roas": { + "type": ["null", "number"] + }, + "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.maximize_conversions.target_cpa_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.name": { + "type": ["null", "string"] + }, + "bidding_strategy.non_removed_campaign_count": { + "type": ["null", "integer"] + }, + "bidding_strategy.resource_name": { + "type": ["null", "string"] + }, + "bidding_strategy.status": { + "type": ["null", "string"] + }, + "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_cpa.cpc_bid_floor_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_cpa.target_cpa_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_impression_share.location": { + "type": ["null", "string"] + }, + "bidding_strategy.target_impression_share.location_fraction_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_roas.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_roas.cpc_bid_floor_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_roas.target_roas": { + "type": ["null", "number"] + }, + "bidding_strategy.target_spend.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_spend.target_spend_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.type": { + "type": ["null", "string"] + }, + "segments.date": { + "type": ["null", "string"], + "format": "date" + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterion.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterion.json new file mode 100644 index 000000000000..48564345f31f --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterion.json @@ -0,0 +1,231 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "deleted_at": { + "type": ["null", "string"] + }, + "change_status.last_change_date_time": { + "type": ["null", "string"] + }, + "ad_group.id": { + "type": ["null", "integer"] + }, + "ad_group_criterion.ad_group": { + "type": ["null", "string"] + }, + "ad_group_criterion.age_range.type": { + "type": ["null", "string"] + }, + "ad_group_criterion.app_payment_model.type": { + "type": ["null", "string"] + }, + "ad_group_criterion.approval_status": { + "type": ["null", "string"] + }, + "ad_group_criterion.audience.audience": { + "type": ["null", "string"] + }, + "ad_group_criterion.bid_modifier": { + "type": ["null", "number"] + }, + "ad_group_criterion.combined_audience.combined_audience": { + "type": ["null", "string"] + }, + "ad_group_criterion.cpc_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.cpm_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.cpv_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.criterion_id": { + "type": ["null", "integer"] + }, + "ad_group_criterion.custom_affinity.custom_affinity": { + "type": ["null", "string"] + }, + "ad_group_criterion.custom_audience.custom_audience": { + "type": ["null", "string"] + }, + "ad_group_criterion.custom_intent.custom_intent": { + "type": ["null", "string"] + }, + "ad_group_criterion.disapproval_reasons": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "ad_group_criterion.display_name": { + "type": ["null", "string"] + }, + "ad_group_criterion.effective_cpc_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.effective_cpc_bid_source": { + "type": ["null", "string"] + }, + "ad_group_criterion.effective_cpm_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.effective_cpm_bid_source": { + "type": ["null", "string"] + }, + "ad_group_criterion.effective_cpv_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.effective_cpv_bid_source": { + "type": ["null", "string"] + }, + "ad_group_criterion.effective_percent_cpc_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.effective_percent_cpc_bid_source": { + "type": ["null", "string"] + }, + "ad_group_criterion.final_mobile_urls": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "ad_group_criterion.final_url_suffix": { + "type": ["null", "string"] + }, + "ad_group_criterion.final_urls": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "ad_group_criterion.gender.type": { + "type": ["null", "string"] + }, + "ad_group_criterion.income_range.type": { + "type": ["null", "string"] + }, + "ad_group_criterion.keyword.match_type": { + "type": ["null", "string"] + }, + "ad_group_criterion.keyword.text": { + "type": ["null", "string"] + }, + "ad_group_criterion.labels": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "ad_group_criterion.mobile_app_category.mobile_app_category_constant": { + "type": ["null", "string"] + }, + "ad_group_criterion.mobile_application.app_id": { + "type": ["null", "string"] + }, + "ad_group_criterion.mobile_application.name": { + "type": ["null", "string"] + }, + "ad_group_criterion.negative": { + "type": ["null", "boolean"] + }, + "ad_group_criterion.parental_status.type": { + "type": ["null", "string"] + }, + "ad_group_criterion.percent_cpc_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.placement.url": { + "type": ["null", "string"] + }, + "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": { + "type": ["null", "integer"] + }, + "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": { + "type": ["null", "integer"] + }, + "ad_group_criterion.position_estimates.first_page_cpc_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.position_estimates.first_position_cpc_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.position_estimates.top_of_page_cpc_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.quality_info.creative_quality_score": { + "type": ["null", "string"] + }, + "ad_group_criterion.quality_info.post_click_quality_score": { + "type": ["null", "string"] + }, + "ad_group_criterion.quality_info.quality_score": { + "type": ["null", "integer"] + }, + "ad_group_criterion.quality_info.search_predicted_ctr": { + "type": ["null", "string"] + }, + "ad_group_criterion.resource_name": { + "type": ["null", "string"] + }, + "ad_group_criterion.status": { + "type": ["null", "string"] + }, + "ad_group_criterion.system_serving_status": { + "type": ["null", "string"] + }, + "ad_group_criterion.topic.path": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "ad_group_criterion.topic.topic_constant": { + "type": ["null", "string"] + }, + "ad_group_criterion.tracking_url_template": { + "type": ["null", "string"] + }, + "ad_group_criterion.type": { + "type": ["null", "string"] + }, + "ad_group_criterion.url_custom_parameters": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "ad_group_criterion.user_interest.user_interest_category": { + "type": ["null", "string"] + }, + "ad_group_criterion.user_list.user_list": { + "type": ["null", "string"] + }, + "ad_group_criterion.webpage.conditions": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "ad_group_criterion.webpage.coverage_percentage": { + "type": ["null", "number"] + }, + "ad_group_criterion.webpage.criterion_name": { + "type": ["null", "string"] + }, + "ad_group_criterion.webpage.sample.sample_urls": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "ad_group_criterion.youtube_channel.channel_id": { + "type": ["null", "string"] + }, + "ad_group_criterion.youtube_video.video_id": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterion_label.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterion_label.json new file mode 100644 index 000000000000..c381686ebf58 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterion_label.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ad_group.id": { + "type": ["null", "integer"] + }, + "label.id": { + "type": ["null", "integer"] + }, + "ad_group_criterion_label.ad_group_criterion": { + "type": ["null", "string"] + }, + "ad_group_criterion_label.label": { + "type": ["null", "string"] + }, + "ad_group_criterion_label.resource_name": { + "type": ["null", "string"] + }, + "ad_group_criterion.criterion_id": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterion_labels.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterion_labels.json deleted file mode 100644 index 16d04dd4720c..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterion_labels.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "ad_group_criterion_label.ad_group_criterion": { - "type": ["null", "string"] - }, - "ad_group_criterion_label.label": { - "type": ["null", "string"] - }, - "ad_group_criterion_label.resource_name": { - "type": ["null", "string"] - }, - "ad_group_criterion.criterion_id": { - "type": ["null", "integer"] - } - } -} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterions.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterions.json deleted file mode 100644 index bf90272fbb3c..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_criterions.json +++ /dev/null @@ -1,225 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "ad_group.id": { - "type": ["null", "integer"] - }, - "ad_group_criterion.ad_group": { - "type": ["null", "string"] - }, - "ad_group_criterion.age_range.type": { - "type": ["null", "string"] - }, - "ad_group_criterion.app_payment_model.type": { - "type": ["null", "string"] - }, - "ad_group_criterion.approval_status": { - "type": ["null", "string"] - }, - "ad_group_criterion.audience.audience": { - "type": ["null", "string"] - }, - "ad_group_criterion.bid_modifier": { - "type": ["null", "number"] - }, - "ad_group_criterion.combined_audience.combined_audience": { - "type": ["null", "string"] - }, - "ad_group_criterion.cpc_bid_micros": { - "type": ["null", "integer"] - }, - "ad_group_criterion.cpm_bid_micros": { - "type": ["null", "integer"] - }, - "ad_group_criterion.cpv_bid_micros": { - "type": ["null", "integer"] - }, - "ad_group_criterion.criterion_id": { - "type": ["null", "integer"] - }, - "ad_group_criterion.custom_affinity.custom_affinity": { - "type": ["null", "string"] - }, - "ad_group_criterion.custom_audience.custom_audience": { - "type": ["null", "string"] - }, - "ad_group_criterion.custom_intent.custom_intent": { - "type": ["null", "string"] - }, - "ad_group_criterion.disapproval_reasons": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "ad_group_criterion.display_name": { - "type": ["null", "string"] - }, - "ad_group_criterion.effective_cpc_bid_micros": { - "type": ["null", "integer"] - }, - "ad_group_criterion.effective_cpc_bid_source": { - "type": ["null", "string"] - }, - "ad_group_criterion.effective_cpm_bid_micros": { - "type": ["null", "integer"] - }, - "ad_group_criterion.effective_cpm_bid_source": { - "type": ["null", "string"] - }, - "ad_group_criterion.effective_cpv_bid_micros": { - "type": ["null", "integer"] - }, - "ad_group_criterion.effective_cpv_bid_source": { - "type": ["null", "string"] - }, - "ad_group_criterion.effective_percent_cpc_bid_micros": { - "type": ["null", "integer"] - }, - "ad_group_criterion.effective_percent_cpc_bid_source": { - "type": ["null", "string"] - }, - "ad_group_criterion.final_mobile_urls": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "ad_group_criterion.final_url_suffix": { - "type": ["null", "string"] - }, - "ad_group_criterion.final_urls": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "ad_group_criterion.gender.type": { - "type": ["null", "string"] - }, - "ad_group_criterion.income_range.type": { - "type": ["null", "string"] - }, - "ad_group_criterion.keyword.match_type": { - "type": ["null", "string"] - }, - "ad_group_criterion.keyword.text": { - "type": ["null", "string"] - }, - "ad_group_criterion.labels": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "ad_group_criterion.mobile_app_category.mobile_app_category_constant": { - "type": ["null", "string"] - }, - "ad_group_criterion.mobile_application.app_id": { - "type": ["null", "string"] - }, - "ad_group_criterion.mobile_application.name": { - "type": ["null", "string"] - }, - "ad_group_criterion.negative": { - "type": ["null", "boolean"] - }, - "ad_group_criterion.parental_status.type": { - "type": ["null", "string"] - }, - "ad_group_criterion.percent_cpc_bid_micros": { - "type": ["null", "integer"] - }, - "ad_group_criterion.placement.url": { - "type": ["null", "string"] - }, - "ad_group_criterion.position_estimates.estimated_add_clicks_at_first_position_cpc": { - "type": ["null", "integer"] - }, - "ad_group_criterion.position_estimates.estimated_add_cost_at_first_position_cpc": { - "type": ["null", "integer"] - }, - "ad_group_criterion.position_estimates.first_page_cpc_micros": { - "type": ["null", "integer"] - }, - "ad_group_criterion.position_estimates.first_position_cpc_micros": { - "type": ["null", "integer"] - }, - "ad_group_criterion.position_estimates.top_of_page_cpc_micros": { - "type": ["null", "integer"] - }, - "ad_group_criterion.quality_info.creative_quality_score": { - "type": ["null", "string"] - }, - "ad_group_criterion.quality_info.post_click_quality_score": { - "type": ["null", "string"] - }, - "ad_group_criterion.quality_info.quality_score": { - "type": ["null", "integer"] - }, - "ad_group_criterion.quality_info.search_predicted_ctr": { - "type": ["null", "string"] - }, - "ad_group_criterion.resource_name": { - "type": ["null", "string"] - }, - "ad_group_criterion.status": { - "type": ["null", "string"] - }, - "ad_group_criterion.system_serving_status": { - "type": ["null", "string"] - }, - "ad_group_criterion.topic.path": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "ad_group_criterion.topic.topic_constant": { - "type": ["null", "string"] - }, - "ad_group_criterion.tracking_url_template": { - "type": ["null", "string"] - }, - "ad_group_criterion.type": { - "type": ["null", "string"] - }, - "ad_group_criterion.url_custom_parameters": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"] - } - }, - "ad_group_criterion.user_interest.user_interest_category": { - "type": ["null", "string"] - }, - "ad_group_criterion.user_list.user_list": { - "type": ["null", "string"] - }, - "ad_group_criterion.webpage.conditions": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "ad_group_criterion.webpage.coverage_percentage": { - "type": ["null", "number"] - }, - "ad_group_criterion.webpage.criterion_name": { - "type": ["null", "string"] - }, - "ad_group_criterion.webpage.sample.sample_urls": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "ad_group_criterion.youtube_channel.channel_id": { - "type": ["null", "string"] - }, - "ad_group_criterion.youtube_video.video_id": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_label.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_label.json new file mode 100644 index 000000000000..6338bda92b78 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_label.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ad_group.id": { + "type": ["null", "integer"] + }, + "label.id": { + "type": ["null", "integer"] + }, + "ad_group.resource_name": { + "type": ["null", "string"] + }, + "ad_group_label.resource_name": { + "type": ["null", "string"] + }, + "label.name": { + "type": ["null", "string"] + }, + "label.resource_name": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_labels.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_labels.json deleted file mode 100644 index ad0fb593eeeb..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_group_labels.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "ad_group.resource_name": { - "type": ["null", "string"] - }, - "ad_group_label.resource_name": { - "type": ["null", "string"] - }, - "label.name": { - "type": ["null", "string"] - }, - "label.resource_name": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_groups.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_groups.json deleted file mode 100644 index 08cafeb898b0..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_groups.json +++ /dev/null @@ -1,103 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "ad_group.ad_rotation_mode": { - "type": ["null", "string"] - }, - "ad_group.base_ad_group": { - "type": ["null", "string"] - }, - "ad_group.campaign": { - "type": ["null", "string"] - }, - "ad_group.cpc_bid_micros": { - "type": ["null", "integer"] - }, - "ad_group.cpm_bid_micros": { - "type": ["null", "integer"] - }, - "ad_group.cpv_bid_micros": { - "type": ["null", "integer"] - }, - "ad_group.display_custom_bid_dimension": { - "type": ["null", "string"] - }, - "ad_group.effective_target_cpa_micros": { - "type": ["null", "integer"] - }, - "ad_group.effective_target_cpa_source": { - "type": ["null", "string"] - }, - "ad_group.effective_target_roas": { - "type": ["null", "number"] - }, - "ad_group.effective_target_roas_source": { - "type": ["null", "string"] - }, - "ad_group.excluded_parent_asset_field_types": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group.optimized_targeting_enabled": { - "type": ["null", "boolean"] - }, - "ad_group.final_url_suffix": { - "type": ["null", "string"] - }, - "ad_group.id": { - "type": ["null", "integer"] - }, - "ad_group.labels": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group.name": { - "type": ["null", "string"] - }, - "ad_group.percent_cpc_bid_micros": { - "type": ["null", "integer"] - }, - "ad_group.resource_name": { - "type": ["null", "string"] - }, - "ad_group.status": { - "type": ["null", "string"] - }, - "ad_group.target_cpa_micros": { - "type": ["null", "integer"] - }, - "ad_group.target_cpm_micros": { - "type": ["null", "integer"] - }, - "ad_group.target_roas": { - "type": ["null", "number"] - }, - "ad_group.targeting_setting.target_restrictions": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "ad_group.tracking_url_template": { - "type": ["null", "string"] - }, - "ad_group.type": { - "type": ["null", "string"] - }, - "ad_group.url_custom_parameters": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "segments.date": { - "type": ["null", "string"], - "format": "date" - } - } -} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_listing_group_criterion.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_listing_group_criterion.json new file mode 100644 index 000000000000..fe5efc371589 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_listing_group_criterion.json @@ -0,0 +1,84 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "deleted_at": { + "type": ["null", "string"] + }, + "change_status.last_change_date_time": { + "type": ["null", "string"] + }, + "ad_group_criterion.resource_name": { + "type": ["null", "string"] + }, + "ad_group.id": { + "type": ["null", "integer"] + }, + "ad_group_criterion.criterion_id": { + "type": ["null", "integer"] + }, + "ad_group_criterion.listing_group.case_value.activity_country.value": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.activity_id.value": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.activity_rating.value": { + "type": ["null", "integer"] + }, + "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.hotel_class.value": { + "type": ["null", "integer"] + }, + "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.hotel_id.value": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.product_category.category_id": { + "type": ["null", "integer"] + }, + "ad_group_criterion.listing_group.case_value.product_category.level": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.product_brand.value": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.product_channel.channel": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.product_condition.condition": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.product_item_id.value": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.product_type.level": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.case_value.product_type.value": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.parent_ad_group_criterion": { + "type": ["null", "string"] + }, + "ad_group_criterion.listing_group.type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_listing_group_criterions.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_listing_group_criterions.json deleted file mode 100644 index bfd193ddef36..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/ad_listing_group_criterions.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "ad_group.id": { - "type": ["null", "integer"] - }, - "ad_group_criterion.criterion_id": { - "type": ["null", "integer"] - }, - "ad_group_criterion.listing_group.case_value.activity_country.value": { - "type": ["null", "string"] - }, - "ad_group_criterion.listing_group.case_value.activity_id.value": { - "type": ["null", "string"] - }, - "ad_group_criterion.listing_group.case_value.activity_rating.value": { - "type": ["null", "integer"] - }, - "ad_group_criterion.listing_group.case_value.hotel_city.city_criterion": { - "type": ["null", "string"] - }, - "ad_group_criterion.listing_group.case_value.hotel_class.value": { - "type": ["null", "integer"] - }, - "ad_group_criterion.listing_group.case_value.hotel_country_region.country_region_criterion": { - "type": ["null", "string"] - }, - "ad_group_criterion.listing_group.case_value.hotel_id.value": { - "type": ["null", "string"] - }, - "ad_group_criterion.listing_group.case_value.hotel_state.state_criterion": { - "type": ["null", "string"] - }, - "ad_group_criterion.listing_group.case_value.product_bidding_category.id": { - "type": ["null", "integer"] - }, - "ad_group_criterion.listing_group.case_value.product_bidding_category.level": { - "type": ["null", "string"] - }, - "ad_group_criterion.listing_group.case_value.product_brand.value": { - "type": ["null", "string"] - }, - "ad_group_criterion.listing_group.case_value.product_channel.channel": { - "type": ["null", "string"] - }, - "ad_group_criterion.listing_group.case_value.product_channel_exclusivity.channel_exclusivity": { - "type": ["null", "string"] - }, - "ad_group_criterion.listing_group.case_value.product_condition.condition": { - "type": ["null", "string"] - }, - "ad_group_criterion.listing_group.case_value.product_custom_attribute.index": { - "type": ["null", "string"] - }, - "ad_group_criterion.listing_group.case_value.product_custom_attribute.value": { - "type": ["null", "string"] - }, - "ad_group_criterion.listing_group.case_value.product_item_id.value": { - "type": ["null", "string"] - }, - "ad_group_criterion.listing_group.case_value.product_type.level": { - "type": ["null", "string"] - }, - "ad_group_criterion.listing_group.case_value.product_type.value": { - "type": ["null", "string"] - }, - "ad_group_criterion.listing_group.parent_ad_group_criterion": { - "type": ["null", "string"] - }, - "ad_group_criterion.listing_group.type": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/audience.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/audience.json index c1bd49e8eed2..314a2c19b771 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/audience.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/audience.json @@ -2,11 +2,17 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { + "customer.id": { + "type": ["null", "integer"] + }, "audience.description": { "type": ["null", "string"] }, "audience.dimensions": { - "type": ["null", "array"] + "type": ["null", "array"], + "items": { + "type": "string" + } }, "audience.exclusion_dimension": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign.json new file mode 100644 index 000000000000..6b7d4f334ca5 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign.json @@ -0,0 +1,325 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "campaign.accessible_bidding_strategy": { + "type": ["null", "string"] + }, + "campaign.ad_serving_optimization_status": { + "type": ["null", "string"] + }, + "campaign.advertising_channel_sub_type": { + "type": ["null", "string"] + }, + "campaign.advertising_channel_type": { + "type": ["null", "string"] + }, + "campaign.app_campaign_setting.app_id": { + "type": ["null", "string"] + }, + "campaign.app_campaign_setting.app_store": { + "type": ["null", "string"] + }, + "campaign.app_campaign_setting.bidding_strategy_goal_type": { + "type": ["null", "string"] + }, + "campaign.base_campaign": { + "type": ["null", "string"] + }, + "campaign.bidding_strategy": { + "type": ["null", "string"] + }, + "campaign.bidding_strategy_type": { + "type": ["null", "string"] + }, + "campaign.campaign_budget": { + "type": ["null", "string"] + }, + "campaign_budget.amount_micros": { + "type": ["null", "integer"] + }, + "campaign.commission.commission_rate_micros": { + "type": ["null", "integer"] + }, + "campaign.dynamic_search_ads_setting.domain_name": { + "type": ["null", "string"] + }, + "campaign.dynamic_search_ads_setting.feeds": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "campaign.dynamic_search_ads_setting.language_code": { + "type": ["null", "string"] + }, + "campaign.dynamic_search_ads_setting.use_supplied_urls_only": { + "type": ["null", "boolean"] + }, + "campaign.end_date": { + "type": ["null", "string"] + }, + "campaign.excluded_parent_asset_field_types": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "campaign.experiment_type": { + "type": ["null", "string"] + }, + "campaign.final_url_suffix": { + "type": ["null", "string"] + }, + "campaign.frequency_caps": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "campaign.geo_target_type_setting.negative_geo_target_type": { + "type": ["null", "string"] + }, + "campaign.geo_target_type_setting.positive_geo_target_type": { + "type": ["null", "string"] + }, + "campaign.hotel_setting.hotel_center_id": { + "type": ["null", "integer"] + }, + "campaign.id": { + "type": ["null", "integer"] + }, + "campaign.labels": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "campaign.local_campaign_setting.location_source_type": { + "type": ["null", "string"] + }, + "campaign.manual_cpc.enhanced_cpc_enabled": { + "type": ["null", "boolean"] + }, + "campaign.manual_cpm": { + "type": ["null", "string"] + }, + "campaign.manual_cpv": { + "type": ["null", "string"] + }, + "campaign.maximize_conversion_value.target_roas": { + "type": ["null", "number"] + }, + "campaign.maximize_conversions.target_cpa_micros": { + "type": ["null", "integer"] + }, + "campaign.name": { + "type": ["null", "string"] + }, + "campaign.network_settings.target_content_network": { + "type": ["null", "boolean"] + }, + "campaign.network_settings.target_google_search": { + "type": ["null", "boolean"] + }, + "campaign.network_settings.target_partner_search_network": { + "type": ["null", "boolean"] + }, + "campaign.network_settings.target_search_network": { + "type": ["null", "boolean"] + }, + "campaign.optimization_goal_setting.optimization_goal_types": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "campaign.optimization_score": { + "type": ["null", "number"] + }, + "campaign.payment_mode": { + "type": ["null", "string"] + }, + "campaign.percent_cpc.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "campaign.percent_cpc.enhanced_cpc_enabled": { + "type": ["null", "boolean"] + }, + "campaign.real_time_bidding_setting.opt_in": { + "type": ["null", "boolean"] + }, + "campaign.resource_name": { + "type": ["null", "string"] + }, + "campaign.selective_optimization.conversion_actions": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "campaign.serving_status": { + "type": ["null", "string"] + }, + "campaign.shopping_setting.campaign_priority": { + "type": ["null", "integer"] + }, + "campaign.shopping_setting.enable_local": { + "type": ["null", "boolean"] + }, + "campaign.shopping_setting.merchant_id": { + "type": ["null", "integer"] + }, + "campaign.start_date": { + "type": ["null", "string"] + }, + "campaign.status": { + "type": ["null", "string"] + }, + "campaign.target_cpa.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "campaign.target_cpa.cpc_bid_floor_micros": { + "type": ["null", "integer"] + }, + "campaign.target_cpa.target_cpa_micros": { + "type": ["null", "integer"] + }, + "campaign.target_cpm.target_frequency_goal.target_count": { + "type": ["null", "integer"] + }, + "campaign.target_cpm.target_frequency_goal.time_unit": { + "type": ["null", "string"] + }, + "campaign.target_impression_share.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "campaign.target_impression_share.location": { + "type": ["null", "string"] + }, + "campaign.target_impression_share.location_fraction_micros": { + "type": ["null", "integer"] + }, + "campaign.target_roas.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "campaign.target_roas.cpc_bid_floor_micros": { + "type": ["null", "integer"] + }, + "campaign.target_roas.target_roas": { + "type": ["null", "number"] + }, + "campaign.target_spend.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "campaign.target_spend.target_spend_micros": { + "type": ["null", "integer"] + }, + "campaign.targeting_setting.target_restrictions": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "campaign.tracking_setting.tracking_url": { + "type": ["null", "string"] + }, + "campaign.tracking_url_template": { + "type": ["null", "string"] + }, + "campaign.url_custom_parameters": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "campaign.vanity_pharma.vanity_pharma_display_url_mode": { + "type": ["null", "string"] + }, + "campaign.vanity_pharma.vanity_pharma_text": { + "type": ["null", "string"] + }, + "campaign.video_brand_safety_suitability": { + "type": ["null", "string"] + }, + "metrics.clicks": { + "type": ["null", "integer"] + }, + "metrics.ctr": { + "type": ["null", "number"] + }, + "metrics.conversions": { + "type": ["null", "number"] + }, + "metrics.conversions_value": { + "type": ["null", "number"] + }, + "metrics.cost_micros": { + "type": ["null", "integer"] + }, + "metrics.impressions": { + "type": ["null", "integer"] + }, + "metrics.video_views": { + "type": ["null", "integer"] + }, + "metrics.video_quartile_p100_rate": { + "type": ["null", "number"] + }, + "metrics.active_view_cpm": { + "type": ["null", "number"] + }, + "metrics.active_view_ctr": { + "type": ["null", "number"] + }, + "metrics.active_view_impressions": { + "type": ["null", "integer"] + }, + "metrics.active_view_measurability": { + "type": ["null", "number"] + }, + "metrics.active_view_measurable_cost_micros": { + "type": ["null", "integer"] + }, + "metrics.active_view_measurable_impressions": { + "type": ["null", "integer"] + }, + "metrics.active_view_viewability": { + "type": ["null", "number"] + }, + "metrics.average_cost": { + "type": ["null", "number"] + }, + "metrics.average_cpc": { + "type": ["null", "number"] + }, + "metrics.average_cpm": { + "type": ["null", "number"] + }, + "metrics.interactions": { + "type": ["null", "integer"] + }, + "metrics.interaction_event_types": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "metrics.value_per_conversion": { + "type": ["null", "number"] + }, + "metrics.cost_per_conversion": { + "type": ["null", "number"] + }, + "segments.date": { + "type": ["null", "string"], + "format": "date" + }, + "segments.hour": { + "type": ["null", "integer"] + }, + "segments.ad_network_type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_bidding_strategies.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_bidding_strategies.json deleted file mode 100644 index 32493bfd957e..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_bidding_strategies.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "campaign.id": { - "type": ["null", "integer"] - }, - "bidding_strategy.aligned_campaign_budget_id": { - "type": ["null", "integer"] - }, - "bidding_strategy.campaign_count": { - "type": ["null", "integer"] - }, - "bidding_strategy.currency_code": { - "type": ["null", "string"] - }, - "bidding_strategy.effective_currency_code": { - "type": ["null", "string"] - }, - "bidding_strategy.enhanced_cpc": { - "type": ["null", "string"] - }, - "bidding_strategy.id": { - "type": ["null", "integer"] - }, - "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.maximize_conversion_value.target_roas": { - "type": ["null", "number"] - }, - "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.maximize_conversions.target_cpa_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.name": { - "type": ["null", "string"] - }, - "bidding_strategy.non_removed_campaign_count": { - "type": ["null", "integer"] - }, - "bidding_strategy.resource_name": { - "type": ["null", "string"] - }, - "bidding_strategy.status": { - "type": ["null", "string"] - }, - "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.target_cpa.cpc_bid_floor_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.target_cpa.target_cpa_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.target_impression_share.location": { - "type": ["null", "string"] - }, - "bidding_strategy.target_impression_share.location_fraction_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.target_roas.cpc_bid_ceiling_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.target_roas.cpc_bid_floor_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.target_roas.target_roas": { - "type": ["null", "number"] - }, - "bidding_strategy.target_spend.cpc_bid_ceiling_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.target_spend.target_spend_micros": { - "type": ["null", "integer"] - }, - "bidding_strategy.type": { - "type": ["null", "string"] - }, - "segments.date": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_bidding_strategy.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_bidding_strategy.json new file mode 100644 index 000000000000..7bd9a868d9d2 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_bidding_strategy.json @@ -0,0 +1,100 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "customer.id": { + "type": ["null", "integer"] + }, + "campaign.id": { + "type": ["null", "integer"] + }, + "bidding_strategy.aligned_campaign_budget_id": { + "type": ["null", "integer"] + }, + "bidding_strategy.campaign_count": { + "type": ["null", "integer"] + }, + "bidding_strategy.currency_code": { + "type": ["null", "string"] + }, + "bidding_strategy.effective_currency_code": { + "type": ["null", "string"] + }, + "bidding_strategy.enhanced_cpc": { + "type": ["null", "string"] + }, + "bidding_strategy.id": { + "type": ["null", "integer"] + }, + "bidding_strategy.maximize_conversion_value.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.maximize_conversion_value.cpc_bid_floor_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.maximize_conversion_value.target_roas": { + "type": ["null", "number"] + }, + "bidding_strategy.maximize_conversions.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.maximize_conversions.cpc_bid_floor_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.maximize_conversions.target_cpa_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.name": { + "type": ["null", "string"] + }, + "bidding_strategy.non_removed_campaign_count": { + "type": ["null", "integer"] + }, + "bidding_strategy.resource_name": { + "type": ["null", "string"] + }, + "bidding_strategy.status": { + "type": ["null", "string"] + }, + "bidding_strategy.target_cpa.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_cpa.cpc_bid_floor_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_cpa.target_cpa_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_impression_share.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_impression_share.location": { + "type": ["null", "string"] + }, + "bidding_strategy.target_impression_share.location_fraction_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_roas.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_roas.cpc_bid_floor_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_roas.target_roas": { + "type": ["null", "number"] + }, + "bidding_strategy.target_spend.cpc_bid_ceiling_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.target_spend.target_spend_micros": { + "type": ["null", "integer"] + }, + "bidding_strategy.type": { + "type": ["null", "string"] + }, + "segments.date": { + "type": ["null", "string"], + "format": "date" + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_budget.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_budget.json index 2be876e64cbb..86b246f863bf 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_budget.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_budget.json @@ -2,6 +2,12 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { + "customer.id": { + "type": ["null", "integer"] + }, + "campaign.id": { + "type": ["null", "integer"] + }, "campaign_budget.aligned_bidding_strategy_id": { "type": ["null", "integer"] }, @@ -9,10 +15,7 @@ "type": ["null", "integer"] }, "campaign_budget.delivery_method": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } + "type": ["null", "string"] }, "campaign_budget.explicitly_shared": { "type": ["null", "boolean"] @@ -27,10 +30,7 @@ "type": ["null", "string"] }, "campaign_budget.period": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } + "type": ["null", "string"] }, "campaign_budget.recommended_budget_amount_micros": { "type": ["null", "integer"] @@ -54,19 +54,13 @@ "type": ["null", "string"] }, "campaign_budget.status": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } + "type": ["null", "string"] }, "campaign_budget.total_amount_micros": { "type": ["null", "integer"] }, "campaign_budget.type": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } + "type": ["null", "string"] }, "segments.date": { "type": ["null", "string"], @@ -76,10 +70,7 @@ "type": ["null", "string"] }, "segments.budget_campaign_association_status.status": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } + "type": ["null", "string"] }, "metrics.all_conversions": { "type": ["null", "number"] diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_criterion.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_criterion.json new file mode 100644 index 000000000000..dc5b2f8109a2 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_criterion.json @@ -0,0 +1,36 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "deleted_at": { + "type": ["null", "string"] + }, + "change_status.last_change_date_time": { + "type": ["null", "string"] + }, + "campaign.id": { + "type": ["null", "integer"] + }, + "campaign_criterion.resource_name": { + "type": ["null", "string"] + }, + "campaign_criterion.campaign": { + "type": ["null", "string"] + }, + "campaign_criterion.age_range.type": { + "type": ["null", "string"] + }, + "campaign_criterion.mobile_application.name": { + "type": ["null", "string"] + }, + "campaign_criterion.negative": { + "type": ["null", "boolean"] + }, + "campaign_criterion.youtube_channel.channel_id": { + "type": ["null", "string"] + }, + "campaign_criterion.youtube_video.video_id": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_label.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_label.json new file mode 100644 index 000000000000..6f64ee157f6e --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_label.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "campaign.id": { + "type": ["null", "integer"] + }, + "label.id": { + "type": ["null", "integer"] + }, + "campaign.resource_name": { + "type": ["null", "string"] + }, + "campaign_label.resource_name": { + "type": ["null", "string"] + }, + "label.name": { + "type": ["null", "string"] + }, + "label.resource_name": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_labels.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_labels.json deleted file mode 100644 index 022d767958f9..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaign_labels.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "campaign.resource_name": { - "type": ["null", "string"] - }, - "campaign_label.resource_name": { - "type": ["null", "string"] - }, - "label.name": { - "type": ["null", "string"] - }, - "label.resource_name": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaigns.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaigns.json deleted file mode 100644 index 7f7a54c07ce2..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/campaigns.json +++ /dev/null @@ -1,325 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "campaign.accessible_bidding_strategy": { - "type": ["null", "string"] - }, - "campaign.ad_serving_optimization_status": { - "type": ["null", "string"] - }, - "campaign.advertising_channel_sub_type": { - "type": ["null", "string"] - }, - "campaign.advertising_channel_type": { - "type": ["null", "string"] - }, - "campaign.app_campaign_setting.app_id": { - "type": ["null", "string"] - }, - "campaign.app_campaign_setting.app_store": { - "type": ["null", "string"] - }, - "campaign.app_campaign_setting.bidding_strategy_goal_type": { - "type": ["null", "string"] - }, - "campaign.base_campaign": { - "type": ["null", "string"] - }, - "campaign.bidding_strategy": { - "type": ["null", "string"] - }, - "campaign.bidding_strategy_type": { - "type": ["null", "string"] - }, - "campaign.campaign_budget": { - "type": ["null", "string"] - }, - "campaign_budget.amount_micros": { - "type": ["null", "integer"] - }, - "campaign.commission.commission_rate_micros": { - "type": ["null", "integer"] - }, - "campaign.dynamic_search_ads_setting.domain_name": { - "type": ["null", "string"] - }, - "campaign.dynamic_search_ads_setting.feeds": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "campaign.dynamic_search_ads_setting.language_code": { - "type": ["null", "string"] - }, - "campaign.dynamic_search_ads_setting.use_supplied_urls_only": { - "type": ["null", "boolean"] - }, - "campaign.end_date": { - "type": ["null", "string"] - }, - "campaign.excluded_parent_asset_field_types": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "campaign.experiment_type": { - "type": ["null", "string"] - }, - "campaign.final_url_suffix": { - "type": ["null", "string"] - }, - "campaign.frequency_caps": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "campaign.geo_target_type_setting.negative_geo_target_type": { - "type": ["null", "string"] - }, - "campaign.geo_target_type_setting.positive_geo_target_type": { - "type": ["null", "string"] - }, - "campaign.hotel_setting.hotel_center_id": { - "type": ["null", "integer"] - }, - "campaign.id": { - "type": ["null", "integer"] - }, - "campaign.labels": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "campaign.local_campaign_setting.location_source_type": { - "type": ["null", "string"] - }, - "campaign.manual_cpc.enhanced_cpc_enabled": { - "type": ["null", "boolean"] - }, - "campaign.manual_cpm": { - "type": ["null", "string"] - }, - "campaign.manual_cpv": { - "type": ["null", "string"] - }, - "campaign.maximize_conversion_value.target_roas": { - "type": ["null", "number"] - }, - "campaign.maximize_conversions.target_cpa_micros": { - "type": ["null", "integer"] - }, - "campaign.name": { - "type": ["null", "string"] - }, - "campaign.network_settings.target_content_network": { - "type": ["null", "boolean"] - }, - "campaign.network_settings.target_google_search": { - "type": ["null", "boolean"] - }, - "campaign.network_settings.target_partner_search_network": { - "type": ["null", "boolean"] - }, - "campaign.network_settings.target_search_network": { - "type": ["null", "boolean"] - }, - "campaign.optimization_goal_setting.optimization_goal_types": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "campaign.optimization_score": { - "type": ["null", "number"] - }, - "campaign.payment_mode": { - "type": ["null", "string"] - }, - "campaign.percent_cpc.cpc_bid_ceiling_micros": { - "type": ["null", "integer"] - }, - "campaign.percent_cpc.enhanced_cpc_enabled": { - "type": ["null", "boolean"] - }, - "campaign.real_time_bidding_setting.opt_in": { - "type": ["null", "boolean"] - }, - "campaign.resource_name": { - "type": ["null", "string"] - }, - "campaign.selective_optimization.conversion_actions": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "campaign.serving_status": { - "type": ["null", "string"] - }, - "campaign.shopping_setting.campaign_priority": { - "type": ["null", "integer"] - }, - "campaign.shopping_setting.enable_local": { - "type": ["null", "boolean"] - }, - "campaign.shopping_setting.merchant_id": { - "type": ["null", "integer"] - }, - "campaign.shopping_setting.sales_country": { - "type": ["null", "string"] - }, - "campaign.start_date": { - "type": ["null", "string"] - }, - "campaign.status": { - "type": ["null", "string"] - }, - "campaign.target_cpa.cpc_bid_ceiling_micros": { - "type": ["null", "integer"] - }, - "campaign.target_cpa.cpc_bid_floor_micros": { - "type": ["null", "integer"] - }, - "campaign.target_cpa.target_cpa_micros": { - "type": ["null", "integer"] - }, - "campaign.target_cpm.target_frequency_goal.target_count": { - "type": ["null", "integer"] - }, - "campaign.target_cpm.target_frequency_goal.time_unit": { - "type": ["null", "string"] - }, - "campaign.target_impression_share.cpc_bid_ceiling_micros": { - "type": ["null", "integer"] - }, - "campaign.target_impression_share.location": { - "type": ["null", "string"] - }, - "campaign.target_impression_share.location_fraction_micros": { - "type": ["null", "integer"] - }, - "campaign.target_roas.cpc_bid_ceiling_micros": { - "type": ["null", "integer"] - }, - "campaign.target_roas.cpc_bid_floor_micros": { - "type": ["null", "integer"] - }, - "campaign.target_roas.target_roas": { - "type": ["null", "number"] - }, - "campaign.target_spend.cpc_bid_ceiling_micros": { - "type": ["null", "integer"] - }, - "campaign.target_spend.target_spend_micros": { - "type": ["null", "integer"] - }, - "campaign.targeting_setting.target_restrictions": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "campaign.tracking_setting.tracking_url": { - "type": ["null", "string"] - }, - "campaign.tracking_url_template": { - "type": ["null", "string"] - }, - "campaign.url_custom_parameters": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "campaign.vanity_pharma.vanity_pharma_display_url_mode": { - "type": ["null", "string"] - }, - "campaign.vanity_pharma.vanity_pharma_text": { - "type": ["null", "string"] - }, - "campaign.video_brand_safety_suitability": { - "type": ["null", "string"] - }, - "metrics.clicks": { - "type": ["null", "integer"] - }, - "metrics.ctr": { - "type": ["null", "number"] - }, - "metrics.conversions": { - "type": ["null", "number"] - }, - "metrics.conversions_value": { - "type": ["null", "number"] - }, - "metrics.cost_micros": { - "type": ["null", "integer"] - }, - "metrics.impressions": { - "type": ["null", "number"] - }, - "metrics.video_views": { - "type": ["null", "number"] - }, - "metrics.video_quartile_p100_rate": { - "type": ["null", "number"] - }, - "metrics.active_view_cpm": { - "type": ["null", "number"] - }, - "metrics.active_view_ctr": { - "type": ["null", "number"] - }, - "metrics.active_view_impressions": { - "type": ["null", "integer"] - }, - "metrics.active_view_measurability": { - "type": ["null", "number"] - }, - "metrics.active_view_measurable_cost_micros": { - "type": ["null", "integer"] - }, - "metrics.active_view_measurable_impressions": { - "type": ["null", "integer"] - }, - "metrics.active_view_viewability": { - "type": ["null", "number"] - }, - "metrics.average_cost": { - "type": ["null", "number"] - }, - "metrics.average_cpc": { - "type": ["null", "number"] - }, - "metrics.average_cpm": { - "type": ["null", "number"] - }, - "metrics.interactions": { - "type": ["null", "integer"] - }, - "metrics.interaction_event_types": { - "type": ["null", "string"] - }, - "metrics.value_per_conversion": { - "type": ["null", "number"] - }, - "metrics.cost_per_conversion": { - "type": ["null", "number"] - }, - "segments.date": { - "type": ["null", "string"], - "format": "date" - }, - "segments.hour": { - "type": ["null", "number"] - }, - "segments.ad_network_type": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/change_status.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/change_status.json new file mode 100644 index 000000000000..55c9f2e90138 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/change_status.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "change_status.last_change_date_time": { + "type": ["null", "string"] + }, + "change_status.resource_type": { + "type": ["null", "string"] + }, + "change_status.resource_status": { + "type": ["null", "string"] + }, + "change_status.resource_name": { + "type": ["null", "string"] + }, + "change_status.ad_group_criterion": { + "type": ["null", "string"] + }, + "change_status.campaign_criterion": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/click_view.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/click_view.json index 4a8b6c3d682d..6f1f78316206 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/click_view.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/click_view.json @@ -27,7 +27,8 @@ "type": ["null", "integer"] }, "segments.date": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date" }, "customer.id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/accounts.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/customer.json similarity index 100% rename from airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/accounts.json rename to airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/customer.json diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/customer_client.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/customer_client.json new file mode 100644 index 000000000000..efb4bfd93f78 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/customer_client.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "customer_client.client_customer": { + "type": ["null", "boolean"] + }, + "customer_client.level": { + "type": ["null", "string"] + }, + "customer_client.id": { + "type": ["null", "integer"] + }, + "customer_client.manager": { + "type": ["null", "boolean"] + }, + "customer_client.time_zone": { + "type": ["null", "number"] + }, + "customer_client.status": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/account_labels.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/customer_label.json similarity index 100% rename from airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/account_labels.json rename to airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/customer_label.json diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/display_keyword_performance_report.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/display_keyword_performance_report.json deleted file mode 100644 index 0eb6676e558f..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/display_keyword_performance_report.json +++ /dev/null @@ -1,256 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "customer.currency_code": { - "type": ["null", "string"] - }, - "customer.descriptive_name": { - "type": ["null", "string"] - }, - "customer.time_zone": { - "type": ["null", "string"] - }, - "metrics.active_view_cpm": { - "type": ["null", "number"] - }, - "metrics.active_view_ctr": { - "type": ["null", "number"] - }, - "metrics.active_view_impressions": { - "type": ["null", "integer"] - }, - "metrics.active_view_measurability": { - "type": ["null", "number"] - }, - "metrics.active_view_measurable_cost_micros": { - "type": ["null", "integer"] - }, - "metrics.active_view_measurable_impressions": { - "type": ["null", "integer"] - }, - "metrics.active_view_viewability": { - "type": ["null", "number"] - }, - "ad_group.id": { - "type": ["null", "integer"] - }, - "ad_group.name": { - "type": ["null", "string"] - }, - "ad_group.status": { - "type": ["null", "string"] - }, - "segments.ad_network_type": { - "type": ["null", "string"] - }, - "metrics.all_conversions_from_interactions_rate": { - "type": ["null", "number"] - }, - "metrics.all_conversions_value": { - "type": ["null", "number"] - }, - "metrics.all_conversions": { - "type": ["null", "number"] - }, - "metrics.average_cost": { - "type": ["null", "number"] - }, - "metrics.average_cpc": { - "type": ["null", "number"] - }, - "metrics.average_cpe": { - "type": ["null", "number"] - }, - "metrics.average_cpm": { - "type": ["null", "number"] - }, - "metrics.average_cpv": { - "type": ["null", "number"] - }, - "ad_group.base_ad_group": { - "type": ["null", "string"] - }, - "campaign.base_campaign": { - "type": ["null", "string"] - }, - "campaign.bidding_strategy": { - "type": ["null", "string"] - }, - "campaign.bidding_strategy_type": { - "type": ["null", "string"] - }, - "campaign.id": { - "type": ["null", "integer"] - }, - "campaign.name": { - "type": ["null", "string"] - }, - "campaign.status": { - "type": ["null", "string"] - }, - "metrics.clicks": { - "type": ["null", "integer"] - }, - "metrics.conversions_from_interactions_rate": { - "type": ["null", "number"] - }, - "metrics.conversions_value": { - "type": ["null", "number"] - }, - "metrics.conversions": { - "type": ["null", "number"] - }, - "metrics.cost_micros": { - "type": ["null", "integer"] - }, - "metrics.cost_per_all_conversions": { - "type": ["null", "number"] - }, - "metrics.cost_per_conversion": { - "type": ["null", "number"] - }, - "ad_group_criterion.effective_cpc_bid_micros": { - "type": ["null", "integer"] - }, - "ad_group_criterion.effective_cpc_bid_source": { - "type": ["null", "string"] - }, - "ad_group_criterion.effective_cpm_bid_micros": { - "type": ["null", "integer"] - }, - "ad_group_criterion.effective_cpm_bid_source": { - "type": ["null", "string"] - }, - "ad_group_criterion.effective_cpv_bid_micros": { - "type": ["null", "integer"] - }, - "ad_group_criterion.effective_cpv_bid_source": { - "type": ["null", "string"] - }, - "ad_group_criterion.keyword.text": { - "type": ["null", "string"] - }, - "metrics.cross_device_conversions": { - "type": ["null", "number"] - }, - "metrics.ctr": { - "type": ["null", "number"] - }, - "segments.day_of_week": { - "type": ["null", "string"] - }, - "segments.device": { - "type": ["null", "string"] - }, - "metrics.engagement_rate": { - "type": ["null", "number"] - }, - "metrics.engagements": { - "type": ["null", "integer"] - }, - "customer.id": { - "type": ["null", "integer"] - }, - "ad_group_criterion.final_mobile_urls": { - "type": ["null", "string", "array"] - }, - "ad_group_criterion.final_urls": { - "type": ["null", "string", "array"] - }, - "metrics.gmail_forwards": { - "type": ["null", "integer"] - }, - "metrics.gmail_saves": { - "type": ["null", "integer"] - }, - "metrics.gmail_secondary_clicks": { - "type": ["null", "integer"] - }, - "ad_group_criterion.criterion_id": { - "type": ["null", "integer"] - }, - "metrics.impressions": { - "type": ["null", "integer"] - }, - "metrics.interaction_rate": { - "type": ["null", "number"] - }, - "metrics.interaction_event_types": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "metrics.interactions": { - "type": ["null", "integer"] - }, - "ad_group_criterion.negative": { - "type": ["null", "boolean"] - }, - "ad_group.targeting_setting.target_restrictions": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "segments.month": { - "type": ["null", "string"] - }, - "segments.quarter": { - "type": ["null", "string"] - }, - "ad_group_criterion.status": { - "type": ["null", "string"] - }, - "ad_group_criterion.tracking_url_template": { - "type": ["null", "string"] - }, - "ad_group_criterion.keyword.match_type": { - "type": ["null", "string"] - }, - "ad_group_criterion.url_custom_parameters": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "metrics.value_per_all_conversions": { - "type": ["null", "number"] - }, - "metrics.value_per_conversion": { - "type": ["null", "number"] - }, - "metrics.video_quartile_p100_rate": { - "type": ["null", "number"] - }, - "metrics.video_quartile_p25_rate": { - "type": ["null", "number"] - }, - "metrics.video_quartile_p50_rate": { - "type": ["null", "number"] - }, - "metrics.video_quartile_p75_rate": { - "type": ["null", "number"] - }, - "metrics.video_view_rate": { - "type": ["null", "number"] - }, - "metrics.video_views": { - "type": ["null", "integer"] - }, - "metrics.view_through_conversions": { - "type": ["null", "integer"] - }, - "segments.week": { - "type": ["null", "string"] - }, - "segments.year": { - "type": ["null", "integer"] - }, - "segments.date": { - "type": ["null", "string"], - "format": "date" - } - } -} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/display_keyword_view.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/display_keyword_view.json new file mode 100644 index 000000000000..60b84b43e051 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/display_keyword_view.json @@ -0,0 +1,262 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "customer.currency_code": { + "type": ["null", "string"] + }, + "customer.descriptive_name": { + "type": ["null", "string"] + }, + "customer.time_zone": { + "type": ["null", "string"] + }, + "metrics.active_view_cpm": { + "type": ["null", "number"] + }, + "metrics.active_view_ctr": { + "type": ["null", "number"] + }, + "metrics.active_view_impressions": { + "type": ["null", "integer"] + }, + "metrics.active_view_measurability": { + "type": ["null", "number"] + }, + "metrics.active_view_measurable_cost_micros": { + "type": ["null", "integer"] + }, + "metrics.active_view_measurable_impressions": { + "type": ["null", "integer"] + }, + "metrics.active_view_viewability": { + "type": ["null", "number"] + }, + "ad_group.id": { + "type": ["null", "integer"] + }, + "ad_group.name": { + "type": ["null", "string"] + }, + "ad_group.status": { + "type": ["null", "string"] + }, + "segments.ad_network_type": { + "type": ["null", "string"] + }, + "metrics.all_conversions_from_interactions_rate": { + "type": ["null", "number"] + }, + "metrics.all_conversions_value": { + "type": ["null", "number"] + }, + "metrics.all_conversions": { + "type": ["null", "number"] + }, + "metrics.average_cost": { + "type": ["null", "number"] + }, + "metrics.average_cpc": { + "type": ["null", "number"] + }, + "metrics.average_cpe": { + "type": ["null", "number"] + }, + "metrics.average_cpm": { + "type": ["null", "number"] + }, + "metrics.average_cpv": { + "type": ["null", "number"] + }, + "ad_group.base_ad_group": { + "type": ["null", "string"] + }, + "campaign.base_campaign": { + "type": ["null", "string"] + }, + "campaign.bidding_strategy": { + "type": ["null", "string"] + }, + "campaign.bidding_strategy_type": { + "type": ["null", "string"] + }, + "campaign.id": { + "type": ["null", "integer"] + }, + "campaign.name": { + "type": ["null", "string"] + }, + "campaign.status": { + "type": ["null", "string"] + }, + "metrics.clicks": { + "type": ["null", "integer"] + }, + "metrics.conversions_from_interactions_rate": { + "type": ["null", "number"] + }, + "metrics.conversions_value": { + "type": ["null", "number"] + }, + "metrics.conversions": { + "type": ["null", "number"] + }, + "metrics.cost_micros": { + "type": ["null", "integer"] + }, + "metrics.cost_per_all_conversions": { + "type": ["null", "number"] + }, + "metrics.cost_per_conversion": { + "type": ["null", "number"] + }, + "ad_group_criterion.effective_cpc_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.effective_cpc_bid_source": { + "type": ["null", "string"] + }, + "ad_group_criterion.effective_cpm_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.effective_cpm_bid_source": { + "type": ["null", "string"] + }, + "ad_group_criterion.effective_cpv_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.effective_cpv_bid_source": { + "type": ["null", "string"] + }, + "ad_group_criterion.keyword.text": { + "type": ["null", "string"] + }, + "metrics.cross_device_conversions": { + "type": ["null", "number"] + }, + "metrics.ctr": { + "type": ["null", "number"] + }, + "segments.day_of_week": { + "type": ["null", "string"] + }, + "segments.device": { + "type": ["null", "string"] + }, + "metrics.engagement_rate": { + "type": ["null", "number"] + }, + "metrics.engagements": { + "type": ["null", "integer"] + }, + "customer.id": { + "type": ["null", "integer"] + }, + "ad_group_criterion.final_mobile_urls": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "ad_group_criterion.final_urls": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "metrics.gmail_forwards": { + "type": ["null", "integer"] + }, + "metrics.gmail_saves": { + "type": ["null", "integer"] + }, + "metrics.gmail_secondary_clicks": { + "type": ["null", "integer"] + }, + "ad_group_criterion.criterion_id": { + "type": ["null", "integer"] + }, + "metrics.impressions": { + "type": ["null", "integer"] + }, + "metrics.interaction_rate": { + "type": ["null", "number"] + }, + "metrics.interaction_event_types": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "metrics.interactions": { + "type": ["null", "integer"] + }, + "ad_group_criterion.negative": { + "type": ["null", "boolean"] + }, + "ad_group.targeting_setting.target_restrictions": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "segments.month": { + "type": ["null", "string"] + }, + "segments.quarter": { + "type": ["null", "string"] + }, + "ad_group_criterion.status": { + "type": ["null", "string"] + }, + "ad_group_criterion.tracking_url_template": { + "type": ["null", "string"] + }, + "ad_group_criterion.keyword.match_type": { + "type": ["null", "string"] + }, + "ad_group_criterion.url_custom_parameters": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "metrics.value_per_all_conversions": { + "type": ["null", "number"] + }, + "metrics.value_per_conversion": { + "type": ["null", "number"] + }, + "metrics.video_quartile_p100_rate": { + "type": ["null", "number"] + }, + "metrics.video_quartile_p25_rate": { + "type": ["null", "number"] + }, + "metrics.video_quartile_p50_rate": { + "type": ["null", "number"] + }, + "metrics.video_quartile_p75_rate": { + "type": ["null", "number"] + }, + "metrics.video_view_rate": { + "type": ["null", "number"] + }, + "metrics.video_views": { + "type": ["null", "integer"] + }, + "metrics.view_through_conversions": { + "type": ["null", "integer"] + }, + "segments.week": { + "type": ["null", "string"] + }, + "segments.year": { + "type": ["null", "integer"] + }, + "segments.date": { + "type": ["null", "string"], + "format": "date" + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/display_topics_performance_report.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/display_topics_performance_report.json deleted file mode 100644 index 1d6cbeab5ede..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/display_topics_performance_report.json +++ /dev/null @@ -1,256 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "customer.currency_code": { - "type": ["null", "string"] - }, - "customer.descriptive_name": { - "type": ["null", "string"] - }, - "customer.time_zone": { - "type": ["null", "string"] - }, - "metrics.active_view_cpm": { - "type": ["null", "number"] - }, - "metrics.active_view_ctr": { - "type": ["null", "number"] - }, - "metrics.active_view_impressions": { - "type": ["null", "integer"] - }, - "metrics.active_view_measurability": { - "type": ["null", "number"] - }, - "metrics.active_view_measurable_cost_micros": { - "type": ["null", "integer"] - }, - "metrics.active_view_measurable_impressions": { - "type": ["null", "integer"] - }, - "metrics.active_view_viewability": { - "type": ["null", "number"] - }, - "ad_group.id": { - "type": ["null", "integer"] - }, - "ad_group.name": { - "type": ["null", "string"] - }, - "ad_group.status": { - "type": ["null", "string"] - }, - "segments.ad_network_type": { - "type": ["null", "string"] - }, - "metrics.all_conversions_from_interactions_rate": { - "type": ["null", "number"] - }, - "metrics.all_conversions_value": { - "type": ["null", "number"] - }, - "metrics.all_conversions": { - "type": ["null", "number"] - }, - "metrics.average_cost": { - "type": ["null", "number"] - }, - "metrics.average_cpc": { - "type": ["null", "number"] - }, - "metrics.average_cpe": { - "type": ["null", "number"] - }, - "metrics.average_cpm": { - "type": ["null", "number"] - }, - "metrics.average_cpv": { - "type": ["null", "number"] - }, - "ad_group.base_ad_group": { - "type": ["null", "string"] - }, - "campaign.base_campaign": { - "type": ["null", "string"] - }, - "ad_group_criterion.bid_modifier": { - "type": ["null", "number"] - }, - "campaign.bidding_strategy": { - "type": ["null", "string"] - }, - "campaign.bidding_strategy_type": { - "type": ["null", "string"] - }, - "campaign.id": { - "type": ["null", "integer"] - }, - "campaign.name": { - "type": ["null", "string"] - }, - "campaign.status": { - "type": ["null", "string"] - }, - "metrics.clicks": { - "type": ["null", "integer"] - }, - "metrics.conversions_from_interactions_rate": { - "type": ["null", "number"] - }, - "metrics.conversions_value": { - "type": ["null", "number"] - }, - "metrics.conversions": { - "type": ["null", "number"] - }, - "metrics.cost_micros": { - "type": ["null", "integer"] - }, - "metrics.cost_per_all_conversions": { - "type": ["null", "number"] - }, - "metrics.cost_per_conversion": { - "type": ["null", "number"] - }, - "ad_group_criterion.effective_cpc_bid_micros": { - "type": ["null", "integer"] - }, - "ad_group_criterion.effective_cpc_bid_source": { - "type": ["null", "string"] - }, - "ad_group_criterion.effective_cpm_bid_micros": { - "type": ["null", "integer"] - }, - "ad_group_criterion.effective_cpm_bid_source": { - "type": ["null", "string"] - }, - "ad_group_criterion.topic.path": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "metrics.cross_device_conversions": { - "type": ["null", "number"] - }, - "metrics.ctr": { - "type": ["null", "number"] - }, - "segments.date": { - "type": ["null", "string"], - "format": "date" - }, - "segments.day_of_week": { - "type": ["null", "string"] - }, - "segments.device": { - "type": ["null", "string"] - }, - "metrics.engagement_rate": { - "type": ["null", "number"] - }, - "metrics.engagements": { - "type": ["null", "integer"] - }, - "customer.id": { - "type": ["null", "integer"] - }, - "ad_group_criterion.final_mobile_urls": { - "type": ["null", "string", "array"] - }, - "ad_group_criterion.final_urls": { - "type": ["null", "string", "array"] - }, - "metrics.gmail_forwards": { - "type": ["null", "integer"] - }, - "metrics.gmail_saves": { - "type": ["null", "integer"] - }, - "metrics.gmail_secondary_clicks": { - "type": ["null", "integer"] - }, - "ad_group_criterion.criterion_id": { - "type": ["null", "integer"] - }, - "metrics.impressions": { - "type": ["null", "integer"] - }, - "metrics.interaction_rate": { - "type": ["null", "number"] - }, - "metrics.interaction_event_types": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "metrics.interactions": { - "type": ["null", "integer"] - }, - "ad_group_criterion.negative": { - "type": ["null", "boolean"] - }, - "ad_group.targeting_setting.target_restrictions": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "segments.month": { - "type": ["null", "string"] - }, - "segments.quarter": { - "type": ["null", "string"] - }, - "ad_group_criterion.status": { - "type": ["null", "string"] - }, - "ad_group_criterion.tracking_url_template": { - "type": ["null", "string"] - }, - "ad_group_criterion.url_custom_parameters": { - "type": ["null", "array"], - "items": { - "type": "string" - } - }, - "metrics.value_per_all_conversions": { - "type": ["null", "number"] - }, - "metrics.value_per_conversion": { - "type": ["null", "number"] - }, - "ad_group_criterion.topic.topic_constant": { - "type": ["null", "string"] - }, - "metrics.video_quartile_p100_rate": { - "type": ["null", "number"] - }, - "metrics.video_quartile_p25_rate": { - "type": ["null", "number"] - }, - "metrics.video_quartile_p50_rate": { - "type": ["null", "number"] - }, - "metrics.video_quartile_p75_rate": { - "type": ["null", "number"] - }, - "metrics.video_view_rate": { - "type": ["null", "number"] - }, - "metrics.video_views": { - "type": ["null", "integer"] - }, - "metrics.view_through_conversions": { - "type": ["null", "integer"] - }, - "segments.week": { - "type": ["null", "string"] - }, - "segments.year": { - "type": ["null", "integer"] - } - } -} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/geographic_report.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/geographic_report.json deleted file mode 100644 index 8c6904dff4e5..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/geographic_report.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "customer.descriptive_name": { - "type": ["null", "string"] - }, - "geographic_view.country_criterion_id": { - "type": ["null", "integer"] - }, - "geographic_view.location_type": { - "type": ["null", "string"], - "enum": [ - "AREA_OF_INTEREST", - "LOCATION_OF_PRESENCE", - "UNKNOWN", - "UNSPECIFIED" - ] - }, - "ad_group.id": { - "type": ["null", "integer"] - }, - "segments.date": { - "type": ["null", "string"], - "format": "date" - } - } -} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/geographic_view.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/geographic_view.json new file mode 100644 index 000000000000..2f67d121006f --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/geographic_view.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "customer.id": { + "type": ["null", "integer"] + }, + "customer.descriptive_name": { + "type": ["null", "string"] + }, + "geographic_view.country_criterion_id": { + "type": ["null", "integer"] + }, + "geographic_view.location_type": { + "type": ["null", "string"] + }, + "ad_group.id": { + "type": ["null", "integer"] + }, + "segments.date": { + "type": ["null", "string"], + "format": "date" + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/keyword_report.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/keyword_report.json deleted file mode 100644 index 6ce4e2a4de4c..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/keyword_report.json +++ /dev/null @@ -1,120 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "customer.descriptive_name": { - "type": ["null", "string"] - }, - "ad_group.id": { - "type": ["null", "integer"] - }, - "ad_group_criterion.type": { - "type": ["null", "string"], - "enum": [ - "AD_SCHEDULE", - "AGE_RANGE", - "APP_PAYMENT_MODEL", - "CARRIER", - "COMBINED_AUDIENCE", - "CONTENT_LABEL", - "CUSTOM_AFFINITY", - "CUSTOM_AUDIENCE", - "CUSTOM_INTENT", - "DEVICE", - "GENDER", - "INCOME_RANGE", - "IP_BLOCK", - "KEYWORD", - "KEYWORD_THEME", - "LANGUAGE", - "LISTING_GROUP", - "LISTING_SCOPE", - "LOCATION", - "LOCATION_GROUP", - "MOBILE_APPLICATION", - "MOBILE_APP_CATEGORY", - "MOBILE_DEVICE", - "OPERATING_SYSTEM_VERSION", - "PARENTAL_STATUS", - "PLACEMENT", - "PROXIMITY", - "TOPIC", - "UNKNOWN", - "UNSPECIFIED", - "USER_INTEREST", - "USER_LIST", - "WEBPAGE", - "YOUTUBE_CHANNEL", - "YOUTUBE_VIDEO" - ] - }, - "ad_group_criterion.keyword.text": { - "type": ["null", "string"] - }, - "ad_group_criterion.negative": { - "type": ["null", "boolean"] - }, - "ad_group_criterion.keyword.match_type": { - "type": ["null", "string"], - "enum": ["BROAD", "EXACT", "PHRASE", "UNKNOWN", "UNSPECIFIED"] - }, - "metrics.historical_quality_score": { - "type": ["null", "integer"] - }, - "metrics.ctr": { - "type": ["null", "number"] - }, - "segments.date": { - "type": ["null", "string"], - "format": "date" - }, - "campaign.bidding_strategy_type": { - "type": ["null", "string"] - }, - "metrics.clicks": { - "type": ["null", "integer"] - }, - "metrics.cost_micros": { - "type": ["null", "integer"] - }, - "metrics.impressions": { - "type": ["null", "integer"] - }, - "metrics.active_view_impressions": { - "type": ["null", "integer"] - }, - "metrics.active_view_measurability": { - "type": ["null", "number"] - }, - "metrics.active_view_measurable_cost_micros": { - "type": ["null", "integer"] - }, - "metrics.active_view_measurable_impressions": { - "type": ["null", "integer"] - }, - "metrics.active_view_viewability": { - "type": ["null", "number"] - }, - "metrics.conversions": { - "type": ["null", "number"] - }, - "metrics.conversions_value": { - "type": ["null", "number"] - }, - "metrics.interactions": { - "type": ["null", "integer"] - }, - "metrics.interaction_event_types": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "metrics.view_through_conversions": { - "type": ["null", "integer"] - }, - "ad_group_criterion.criterion_id": { - "type": ["null", "integer"] - } - } -} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/keyword_view.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/keyword_view.json new file mode 100644 index 000000000000..a4ade30ce77a --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/keyword_view.json @@ -0,0 +1,88 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "customer.id": { + "type": ["null", "integer"] + }, + "customer.descriptive_name": { + "type": ["null", "string"] + }, + "campaign.id": { + "type": ["null", "integer"] + }, + "ad_group.id": { + "type": ["null", "integer"] + }, + "ad_group_criterion.type": { + "type": ["null", "string"] + }, + "ad_group_criterion.keyword.text": { + "type": ["null", "string"] + }, + "ad_group_criterion.negative": { + "type": ["null", "boolean"] + }, + "ad_group_criterion.keyword.match_type": { + "type": ["null", "string"] + }, + "metrics.historical_quality_score": { + "type": ["null", "integer"] + }, + "metrics.ctr": { + "type": ["null", "number"] + }, + "segments.date": { + "type": ["null", "string"], + "format": "date" + }, + "campaign.bidding_strategy_type": { + "type": ["null", "string"] + }, + "metrics.clicks": { + "type": ["null", "integer"] + }, + "metrics.cost_micros": { + "type": ["null", "integer"] + }, + "metrics.impressions": { + "type": ["null", "integer"] + }, + "metrics.active_view_impressions": { + "type": ["null", "integer"] + }, + "metrics.active_view_measurability": { + "type": ["null", "number"] + }, + "metrics.active_view_measurable_cost_micros": { + "type": ["null", "integer"] + }, + "metrics.active_view_measurable_impressions": { + "type": ["null", "integer"] + }, + "metrics.active_view_viewability": { + "type": ["null", "number"] + }, + "metrics.conversions": { + "type": ["null", "number"] + }, + "metrics.conversions_value": { + "type": ["null", "number"] + }, + "metrics.interactions": { + "type": ["null", "integer"] + }, + "metrics.interaction_event_types": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "metrics.view_through_conversions": { + "type": ["null", "integer"] + }, + "ad_group_criterion.criterion_id": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/label.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/label.json new file mode 100644 index 000000000000..79043478d1f7 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/label.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "customer.id": { + "type": ["null", "integer"] + }, + "label.id": { + "type": ["null", "integer"] + }, + "label.name": { + "type": ["null", "string"] + }, + "label.resource_name": { + "type": ["null", "string"] + }, + "label.status": { + "type": ["null", "string"] + }, + "label.text_label.background_color": { + "type": ["null", "string"] + }, + "label.text_label.description": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/labels.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/labels.json deleted file mode 100644 index 1621d5831bc2..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/labels.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "label.id": { - "type": ["null", "integer"] - }, - "label.name": { - "type": ["null", "string"] - }, - "label.resource_name": { - "type": ["null", "string"] - }, - "label.status": { - "type": ["null", "string"] - }, - "label.text_label.background_color": { - "type": ["null", "string"] - }, - "label.text_label.description": { - "type": ["null", "string"] - } - } -} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/shopping_performance_report.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/shopping_performance_report.json deleted file mode 100644 index 965e5e3857d3..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/shopping_performance_report.json +++ /dev/null @@ -1,184 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "customer.descriptive_name": { - "type": ["null", "string"] - }, - "ad_group.id": { - "type": ["null", "integer"] - }, - "ad_group.name": { - "type": ["null", "string"] - }, - "ad_group.status": { - "type": ["null", "string"] - }, - "segments.ad_network_type": { - "type": ["null", "string"] - }, - "segments.product_aggregator_id": { - "type": ["null", "integer"] - }, - "metrics.all_conversions_from_interactions_rate": { - "type": ["null", "number"] - }, - "metrics.all_conversions_value": { - "type": ["null", "number"] - }, - "metrics.all_conversions": { - "type": ["null", "number"] - }, - "metrics.average_cpc": { - "type": ["null", "number"] - }, - "segments.product_brand": { - "type": ["null", "string"] - }, - "campaign.id": { - "type": ["null", "integer"] - }, - "campaign.name": { - "type": ["null", "string"] - }, - "campaign.status": { - "type": ["null", "string"] - }, - "segments.product_bidding_category_level1": { - "type": ["null", "string"] - }, - "segments.product_bidding_category_level2": { - "type": ["null", "string"] - }, - "segments.product_bidding_category_level3": { - "type": ["null", "string"] - }, - "segments.product_bidding_category_level4": { - "type": ["null", "string"] - }, - "segments.product_bidding_category_level5": { - "type": ["null", "string"] - }, - "segments.product_channel": { - "type": ["null", "string"] - }, - "segments.product_channel_exclusivity": { - "type": ["null", "string"] - }, - "segments.click_type": { - "type": ["null", "string"] - }, - "metrics.clicks": { - "type": ["null", "integer"] - }, - "metrics.conversions_from_interactions_rate": { - "type": ["null", "number"] - }, - "metrics.conversions_value": { - "type": ["null", "number"] - }, - "metrics.conversions": { - "type": ["null", "number"] - }, - "metrics.cost_micros": { - "type": ["null", "integer"] - }, - "metrics.cost_per_all_conversions": { - "type": ["null", "number"] - }, - "metrics.cost_per_conversion": { - "type": ["null", "number"] - }, - "segments.product_country": { - "type": ["null", "string"] - }, - "metrics.cross_device_conversions": { - "type": ["null", "number"] - }, - "metrics.ctr": { - "type": ["null", "number"] - }, - "segments.product_custom_attribute0": { - "type": ["null", "string"] - }, - "segments.product_custom_attribute1": { - "type": ["null", "string"] - }, - "segments.product_custom_attribute2": { - "type": ["null", "string"] - }, - "segments.product_custom_attribute3": { - "type": ["null", "string"] - }, - "segments.product_custom_attribute4": { - "type": ["null", "string"] - }, - "segments.date": { - "type": ["null", "string"], - "format": "date" - }, - "segments.day_of_week": { - "type": ["null", "string"] - }, - "segments.device": { - "type": ["null", "string"] - }, - "customer.id": { - "type": ["null", "integer"] - }, - "metrics.impressions": { - "type": ["null", "integer"] - }, - "segments.product_language": { - "type": ["null", "string"] - }, - "segments.product_merchant_id": { - "type": ["null", "integer"] - }, - "segments.month": { - "type": ["null", "string"] - }, - "segments.product_item_id": { - "type": ["null", "string"] - }, - "segments.product_condition": { - "type": ["null", "string"] - }, - "segments.product_title": { - "type": ["null", "string"] - }, - "segments.product_type_l1": { - "type": ["null", "string"] - }, - "segments.product_type_l2": { - "type": ["null", "string"] - }, - "segments.product_type_l3": { - "type": ["null", "string"] - }, - "segments.product_type_l4": { - "type": ["null", "string"] - }, - "segments.product_type_l5": { - "type": ["null", "string"] - }, - "segments.quarter": { - "type": ["null", "string"] - }, - "segments.product_store_id": { - "type": ["null", "string"] - }, - "metrics.value_per_all_conversions": { - "type": ["null", "number"] - }, - "metrics.value_per_conversion": { - "type": ["null", "number"] - }, - "segments.week": { - "type": ["null", "string"] - }, - "segments.year": { - "type": ["null", "integer"] - } - } -} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/shopping_performance_view.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/shopping_performance_view.json new file mode 100644 index 000000000000..f679be52592c --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/shopping_performance_view.json @@ -0,0 +1,184 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "customer.descriptive_name": { + "type": ["null", "string"] + }, + "ad_group.id": { + "type": ["null", "integer"] + }, + "ad_group.name": { + "type": ["null", "string"] + }, + "ad_group.status": { + "type": ["null", "string"] + }, + "segments.ad_network_type": { + "type": ["null", "string"] + }, + "segments.product_aggregator_id": { + "type": ["null", "integer"] + }, + "metrics.all_conversions_from_interactions_rate": { + "type": ["null", "number"] + }, + "metrics.all_conversions_value": { + "type": ["null", "number"] + }, + "metrics.all_conversions": { + "type": ["null", "number"] + }, + "metrics.average_cpc": { + "type": ["null", "number"] + }, + "segments.product_brand": { + "type": ["null", "string"] + }, + "campaign.id": { + "type": ["null", "integer"] + }, + "campaign.name": { + "type": ["null", "string"] + }, + "campaign.status": { + "type": ["null", "string"] + }, + "segments.product_category_level1": { + "type": ["null", "string"] + }, + "segments.product_category_level2": { + "type": ["null", "string"] + }, + "segments.product_category_level3": { + "type": ["null", "string"] + }, + "segments.product_category_level4": { + "type": ["null", "string"] + }, + "segments.product_category_level5": { + "type": ["null", "string"] + }, + "segments.product_channel": { + "type": ["null", "string"] + }, + "segments.product_channel_exclusivity": { + "type": ["null", "string"] + }, + "segments.click_type": { + "type": ["null", "string"] + }, + "metrics.clicks": { + "type": ["null", "integer"] + }, + "metrics.conversions_from_interactions_rate": { + "type": ["null", "number"] + }, + "metrics.conversions_value": { + "type": ["null", "number"] + }, + "metrics.conversions": { + "type": ["null", "number"] + }, + "metrics.cost_micros": { + "type": ["null", "integer"] + }, + "metrics.cost_per_all_conversions": { + "type": ["null", "number"] + }, + "metrics.cost_per_conversion": { + "type": ["null", "number"] + }, + "segments.product_country": { + "type": ["null", "string"] + }, + "metrics.cross_device_conversions": { + "type": ["null", "number"] + }, + "metrics.ctr": { + "type": ["null", "number"] + }, + "segments.product_custom_attribute0": { + "type": ["null", "string"] + }, + "segments.product_custom_attribute1": { + "type": ["null", "string"] + }, + "segments.product_custom_attribute2": { + "type": ["null", "string"] + }, + "segments.product_custom_attribute3": { + "type": ["null", "string"] + }, + "segments.product_custom_attribute4": { + "type": ["null", "string"] + }, + "segments.date": { + "type": ["null", "string"], + "format": "date" + }, + "segments.day_of_week": { + "type": ["null", "string"] + }, + "segments.device": { + "type": ["null", "string"] + }, + "customer.id": { + "type": ["null", "integer"] + }, + "metrics.impressions": { + "type": ["null", "integer"] + }, + "segments.product_language": { + "type": ["null", "string"] + }, + "segments.product_merchant_id": { + "type": ["null", "integer"] + }, + "segments.month": { + "type": ["null", "string"] + }, + "segments.product_item_id": { + "type": ["null", "string"] + }, + "segments.product_condition": { + "type": ["null", "string"] + }, + "segments.product_title": { + "type": ["null", "string"] + }, + "segments.product_type_l1": { + "type": ["null", "string"] + }, + "segments.product_type_l2": { + "type": ["null", "string"] + }, + "segments.product_type_l3": { + "type": ["null", "string"] + }, + "segments.product_type_l4": { + "type": ["null", "string"] + }, + "segments.product_type_l5": { + "type": ["null", "string"] + }, + "segments.quarter": { + "type": ["null", "string"] + }, + "segments.product_store_id": { + "type": ["null", "string"] + }, + "metrics.value_per_all_conversions": { + "type": ["null", "number"] + }, + "metrics.value_per_conversion": { + "type": ["null", "number"] + }, + "segments.week": { + "type": ["null", "string"] + }, + "segments.year": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/topic_view.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/topic_view.json new file mode 100644 index 000000000000..9defd19f5600 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/topic_view.json @@ -0,0 +1,265 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "topic_view.resource_name": { + "type": ["null", "string"] + }, + "customer.currency_code": { + "type": ["null", "string"] + }, + "customer.descriptive_name": { + "type": ["null", "string"] + }, + "customer.time_zone": { + "type": ["null", "string"] + }, + "metrics.active_view_cpm": { + "type": ["null", "number"] + }, + "metrics.active_view_ctr": { + "type": ["null", "number"] + }, + "metrics.active_view_impressions": { + "type": ["null", "integer"] + }, + "metrics.active_view_measurability": { + "type": ["null", "number"] + }, + "metrics.active_view_measurable_cost_micros": { + "type": ["null", "integer"] + }, + "metrics.active_view_measurable_impressions": { + "type": ["null", "integer"] + }, + "metrics.active_view_viewability": { + "type": ["null", "number"] + }, + "ad_group.id": { + "type": ["null", "integer"] + }, + "ad_group.name": { + "type": ["null", "string"] + }, + "ad_group.status": { + "type": ["null", "string"] + }, + "segments.ad_network_type": { + "type": ["null", "string"] + }, + "metrics.all_conversions_from_interactions_rate": { + "type": ["null", "number"] + }, + "metrics.all_conversions_value": { + "type": ["null", "number"] + }, + "metrics.all_conversions": { + "type": ["null", "number"] + }, + "metrics.average_cost": { + "type": ["null", "number"] + }, + "metrics.average_cpc": { + "type": ["null", "number"] + }, + "metrics.average_cpe": { + "type": ["null", "number"] + }, + "metrics.average_cpm": { + "type": ["null", "number"] + }, + "metrics.average_cpv": { + "type": ["null", "number"] + }, + "ad_group.base_ad_group": { + "type": ["null", "string"] + }, + "campaign.base_campaign": { + "type": ["null", "string"] + }, + "ad_group_criterion.bid_modifier": { + "type": ["null", "number"] + }, + "campaign.bidding_strategy": { + "type": ["null", "string"] + }, + "campaign.bidding_strategy_type": { + "type": ["null", "string"] + }, + "campaign.id": { + "type": ["null", "integer"] + }, + "campaign.name": { + "type": ["null", "string"] + }, + "campaign.status": { + "type": ["null", "string"] + }, + "metrics.clicks": { + "type": ["null", "integer"] + }, + "metrics.conversions_from_interactions_rate": { + "type": ["null", "number"] + }, + "metrics.conversions_value": { + "type": ["null", "number"] + }, + "metrics.conversions": { + "type": ["null", "number"] + }, + "metrics.cost_micros": { + "type": ["null", "integer"] + }, + "metrics.cost_per_all_conversions": { + "type": ["null", "number"] + }, + "metrics.cost_per_conversion": { + "type": ["null", "number"] + }, + "ad_group_criterion.effective_cpc_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.effective_cpc_bid_source": { + "type": ["null", "string"] + }, + "ad_group_criterion.effective_cpm_bid_micros": { + "type": ["null", "integer"] + }, + "ad_group_criterion.effective_cpm_bid_source": { + "type": ["null", "string"] + }, + "ad_group_criterion.topic.path": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "metrics.cross_device_conversions": { + "type": ["null", "number"] + }, + "metrics.ctr": { + "type": ["null", "number"] + }, + "segments.date": { + "type": ["null", "string"], + "format": "date" + }, + "segments.day_of_week": { + "type": ["null", "string"] + }, + "segments.device": { + "type": ["null", "string"] + }, + "metrics.engagement_rate": { + "type": ["null", "number"] + }, + "metrics.engagements": { + "type": ["null", "integer"] + }, + "customer.id": { + "type": ["null", "integer"] + }, + "ad_group_criterion.final_mobile_urls": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "ad_group_criterion.final_urls": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "metrics.gmail_forwards": { + "type": ["null", "integer"] + }, + "metrics.gmail_saves": { + "type": ["null", "integer"] + }, + "metrics.gmail_secondary_clicks": { + "type": ["null", "integer"] + }, + "ad_group_criterion.criterion_id": { + "type": ["null", "integer"] + }, + "metrics.impressions": { + "type": ["null", "integer"] + }, + "metrics.interaction_rate": { + "type": ["null", "number"] + }, + "metrics.interaction_event_types": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "metrics.interactions": { + "type": ["null", "integer"] + }, + "ad_group_criterion.negative": { + "type": ["null", "boolean"] + }, + "ad_group.targeting_setting.target_restrictions": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "segments.month": { + "type": ["null", "string"] + }, + "segments.quarter": { + "type": ["null", "string"] + }, + "ad_group_criterion.status": { + "type": ["null", "string"] + }, + "ad_group_criterion.tracking_url_template": { + "type": ["null", "string"] + }, + "ad_group_criterion.url_custom_parameters": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "metrics.value_per_all_conversions": { + "type": ["null", "number"] + }, + "metrics.value_per_conversion": { + "type": ["null", "number"] + }, + "ad_group_criterion.topic.topic_constant": { + "type": ["null", "string"] + }, + "metrics.video_quartile_p100_rate": { + "type": ["null", "number"] + }, + "metrics.video_quartile_p25_rate": { + "type": ["null", "number"] + }, + "metrics.video_quartile_p50_rate": { + "type": ["null", "number"] + }, + "metrics.video_quartile_p75_rate": { + "type": ["null", "number"] + }, + "metrics.video_view_rate": { + "type": ["null", "number"] + }, + "metrics.video_views": { + "type": ["null", "integer"] + }, + "metrics.view_through_conversions": { + "type": ["null", "integer"] + }, + "segments.week": { + "type": ["null", "string"] + }, + "segments.year": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/user_interest.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/user_interest.json index 350a40ef5906..1de930af34e4 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/user_interest.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/user_interest.json @@ -3,7 +3,10 @@ "type": "object", "properties": { "user_interest.availabilities": { - "type": ["null", "array"] + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, "user_interest.launched_to_all": { "type": ["null", "boolean"] diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/user_location_report.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/user_location_report.json deleted file mode 100644 index c9ef9cb5ae0c..000000000000 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/user_location_report.json +++ /dev/null @@ -1,142 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "segments.date": { - "type": ["null", "string"], - "format": "date" - }, - "segments.day_of_week": { - "type": ["null", "string"] - }, - "segments.month": { - "type": ["null", "string"] - }, - "segments.week": { - "type": ["null", "string"] - }, - "segments.quarter": { - "type": ["null", "string"] - }, - "segments.year": { - "type": ["null", "integer"] - }, - "segments.ad_network_type": { - "type": ["null", "string"] - }, - "customer.currency_code": { - "type": ["null", "string"] - }, - "customer.id": { - "type": ["null", "integer"] - }, - "customer.descriptive_name": { - "type": ["null", "string"] - }, - "customer.time_zone": { - "type": ["null", "string"] - }, - "user_location_view.country_criterion_id": { - "type": ["null", "integer"] - }, - "user_location_view.resource_name": { - "type": ["null", "string"] - }, - "campaign.base_campaign": { - "type": ["null", "string"] - }, - "campaign.id": { - "type": ["null", "integer"] - }, - "campaign.name": { - "type": ["null", "string"] - }, - "campaign.status": { - "type": ["null", "string"] - }, - "ad_group.name": { - "type": ["null", "string"] - }, - "ad_group.status": { - "type": ["null", "string"] - }, - "ad_group.base_ad_group": { - "type": ["null", "string"] - }, - "metrics.all_conversions": { - "type": ["null", "number"] - }, - "metrics.all_conversions_from_interactions_rate": { - "type": ["null", "number"] - }, - "metrics.all_conversions_value": { - "type": ["null", "number"] - }, - "metrics.average_cost": { - "type": ["null", "number"] - }, - "metrics.average_cpc": { - "type": ["null", "number"] - }, - "metrics.average_cpm": { - "type": ["null", "number"] - }, - "metrics.average_cpv": { - "type": ["null", "number"] - }, - "metrics.clicks": { - "type": ["null", "number"] - }, - "metrics.conversions": { - "type": ["null", "number"] - }, - "metrics.conversions_from_interactions_rate": { - "type": ["null", "number"] - }, - "metrics.conversions_value": { - "type": ["null", "number"] - }, - "metrics.cost_micros": { - "type": ["null", "number"] - }, - "metrics.cost_per_all_conversions": { - "type": ["null", "number"] - }, - "metrics.cost_per_conversion": { - "type": ["null", "number"] - }, - "metrics.cross_device_conversions": { - "type": ["null", "number"] - }, - "metrics.ctr": { - "type": ["null", "number"] - }, - "metrics.impressions": { - "type": ["null", "number"] - }, - "metrics.interaction_event_types": { - "type": ["null", "array"] - }, - "metrics.interaction_rate": { - "type": ["null", "number"] - }, - "metrics.interactions": { - "type": ["null", "number"] - }, - "metrics.value_per_all_conversions": { - "type": ["null", "number"] - }, - "metrics.value_per_conversion": { - "type": ["null", "number"] - }, - "metrics.video_view_rate": { - "type": ["null", "number"] - }, - "metrics.video_views": { - "type": ["null", "number"] - }, - "metrics.view_through_conversions": { - "type": ["null", "number"] - } - } -} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/user_location_view.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/user_location_view.json new file mode 100644 index 000000000000..d78f0d8f2e7b --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/schemas/user_location_view.json @@ -0,0 +1,148 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "segments.date": { + "type": ["null", "string"], + "format": "date" + }, + "segments.day_of_week": { + "type": ["null", "string"] + }, + "segments.month": { + "type": ["null", "string"] + }, + "segments.week": { + "type": ["null", "string"] + }, + "segments.quarter": { + "type": ["null", "string"] + }, + "segments.year": { + "type": ["null", "integer"] + }, + "segments.ad_network_type": { + "type": ["null", "string"] + }, + "customer.currency_code": { + "type": ["null", "string"] + }, + "customer.id": { + "type": ["null", "integer"] + }, + "customer.descriptive_name": { + "type": ["null", "string"] + }, + "customer.time_zone": { + "type": ["null", "string"] + }, + "user_location_view.country_criterion_id": { + "type": ["null", "integer"] + }, + "user_location_view.targeting_location": { + "type": ["null", "boolean"] + }, + "user_location_view.resource_name": { + "type": ["null", "string"] + }, + "campaign.base_campaign": { + "type": ["null", "string"] + }, + "campaign.id": { + "type": ["null", "integer"] + }, + "campaign.name": { + "type": ["null", "string"] + }, + "campaign.status": { + "type": ["null", "string"] + }, + "ad_group.name": { + "type": ["null", "string"] + }, + "ad_group.status": { + "type": ["null", "string"] + }, + "ad_group.base_ad_group": { + "type": ["null", "string"] + }, + "metrics.all_conversions": { + "type": ["null", "number"] + }, + "metrics.all_conversions_from_interactions_rate": { + "type": ["null", "number"] + }, + "metrics.all_conversions_value": { + "type": ["null", "number"] + }, + "metrics.average_cost": { + "type": ["null", "number"] + }, + "metrics.average_cpc": { + "type": ["null", "number"] + }, + "metrics.average_cpm": { + "type": ["null", "number"] + }, + "metrics.average_cpv": { + "type": ["null", "number"] + }, + "metrics.clicks": { + "type": ["null", "integer"] + }, + "metrics.conversions": { + "type": ["null", "number"] + }, + "metrics.conversions_from_interactions_rate": { + "type": ["null", "number"] + }, + "metrics.conversions_value": { + "type": ["null", "number"] + }, + "metrics.cost_micros": { + "type": ["null", "integer"] + }, + "metrics.cost_per_all_conversions": { + "type": ["null", "number"] + }, + "metrics.cost_per_conversion": { + "type": ["null", "number"] + }, + "metrics.cross_device_conversions": { + "type": ["null", "number"] + }, + "metrics.ctr": { + "type": ["null", "number"] + }, + "metrics.impressions": { + "type": ["null", "integer"] + }, + "metrics.interaction_event_types": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "metrics.interaction_rate": { + "type": ["null", "number"] + }, + "metrics.interactions": { + "type": ["null", "integer"] + }, + "metrics.value_per_all_conversions": { + "type": ["null", "number"] + }, + "metrics.value_per_conversion": { + "type": ["null", "number"] + }, + "metrics.video_view_rate": { + "type": ["null", "number"] + }, + "metrics.video_views": { + "type": ["null", "integer"] + }, + "metrics.view_through_conversions": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py index e7c485e08e63..2402cd18adbe 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py @@ -4,99 +4,143 @@ import logging -import traceback from typing import Any, Iterable, List, Mapping, MutableMapping, Tuple from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.utils import AirbyteTracedException -from google.ads.googleads.errors import GoogleAdsException -from google.ads.googleads.v13.errors.types.authentication_error import AuthenticationErrorEnum -from google.ads.googleads.v13.errors.types.authorization_error import AuthorizationErrorEnum from pendulum import parse, today from .custom_query_stream import CustomQuery, IncrementalCustomQuery from .google_ads import GoogleAds -from .models import Customer +from .models import CustomerModel from .streams import ( - AccountLabels, AccountPerformanceReport, - Accounts, - AdGroupAdLabels, - AdGroupAdReport, - AdGroupAds, - AdGroupBiddingStrategies, - AdGroupCriterionLabels, - AdGroupCriterions, - AdGroupLabels, - AdGroups, - AdListingGroupCriterions, + AdGroup, + AdGroupAd, + AdGroupAdLabel, + AdGroupAdLegacy, + AdGroupBiddingStrategy, + AdGroupCriterion, + AdGroupCriterionLabel, + AdGroupLabel, + AdListingGroupCriterion, Audience, - CampaignBiddingStrategies, + Campaign, + CampaignBiddingStrategy, CampaignBudget, - CampaignLabels, - Campaigns, + CampaignCriterion, + CampaignLabel, ClickView, - DisplayKeywordPerformanceReport, - DisplayTopicsPerformanceReport, - GeographicReport, - KeywordReport, - Labels, + Customer, + CustomerClient, + CustomerLabel, + DisplayKeywordView, + GeographicView, + KeywordView, + Label, ServiceAccounts, - ShoppingPerformanceReport, + ShoppingPerformanceView, + TopicView, UserInterest, - UserLocationReport, + UserLocationView, ) from .utils import GAQL -FULL_REFRESH_CUSTOM_TABLE = ["asset", "asset_group_listing_group_filter", "custom_audience", "geo_target_constant"] +logger = logging.getLogger("airbyte") class SourceGoogleAds(AbstractSource): + # Skip exceptions on missing streams + raise_exception_on_missing_stream = False + @staticmethod def _validate_and_transform(config: Mapping[str, Any]): if config.get("end_date") == "": config.pop("end_date") - for query in config.get("custom_queries", []): + for query in config.get("custom_queries_array", []): try: query["query"] = GAQL.parse(query["query"]) except ValueError: - message = f"The custom GAQL query {query['table_name']} failed. Validate your GAQL query with the Google Ads query validator. https://developers.google.com/google-ads/api/fields/v13/query_validator" + message = ( + f"The custom GAQL query {query['table_name']} failed. Validate your GAQL query with the Google Ads query validator. " + "https://developers.google.com/google-ads/api/fields/v15/query_validator" + ) raise AirbyteTracedException(message=message, failure_type=FailureType.config_error) + + if "customer_id" in config: + config["customer_ids"] = config["customer_id"].split(",") + config.pop("customer_id") + return config @staticmethod def get_credentials(config: Mapping[str, Any]) -> MutableMapping[str, Any]: credentials = config["credentials"] - # use_proto_plus is set to True, because setting to False returned wrong value types, which breakes the backward compatibility. + # use_proto_plus is set to True, because setting to False returned wrong value types, which breaks the backward compatibility. # For more info read the related PR's description: https://github.com/airbytehq/airbyte/pull/9996 credentials.update(use_proto_plus=True) - - # https://developers.google.com/google-ads/api/docs/concepts/call-structure#cid - if "login_customer_id" in config and config["login_customer_id"].strip(): - credentials["login_customer_id"] = config["login_customer_id"] return credentials @staticmethod - def get_incremental_stream_config(google_api: GoogleAds, config: Mapping[str, Any], customers: List[Customer]): + def get_incremental_stream_config(google_api: GoogleAds, config: Mapping[str, Any], customers: List[CustomerModel]): + # date range is mandatory parameter for incremental streams, so default start day is used + start_date = config.get("start_date", today().subtract(years=2).to_date_string()) + end_date = config.get("end_date") - if end_date: - end_date = min(today(), parse(end_date)).to_date_string() + # check if end_date is not in the future, set to today if it is + end_date = min(today(), parse(end_date)) if end_date else today() + end_date = end_date.to_date_string() + incremental_stream_config = dict( api=google_api, customers=customers, - conversion_window_days=config["conversion_window_days"], - start_date=config["start_date"], + conversion_window_days=config.get("conversion_window_days", 0), + start_date=start_date, end_date=end_date, ) return incremental_stream_config - def get_account_info(self, google_api: GoogleAds, config: Mapping[str, Any]) -> Iterable[Iterable[Mapping[str, Any]]]: - dummy_customers = [Customer(id=_id) for _id in config["customer_id"].split(",")] - accounts_stream = ServiceAccounts(google_api, customers=dummy_customers) - for slice_ in accounts_stream.stream_slices(): - yield accounts_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice_) + def get_all_accounts(self, google_api: GoogleAds, customers: List[CustomerModel], customer_status_filter: List[str]) -> List[str]: + customer_clients_stream = CustomerClient(api=google_api, customers=customers, customer_status_filter=customer_status_filter) + for slice in customer_clients_stream.stream_slices(): + for record in customer_clients_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice): + yield record + + def _get_all_connected_accounts( + self, google_api: GoogleAds, customer_status_filter: List[str] + ) -> Iterable[Iterable[Mapping[str, Any]]]: + customer_ids = [customer_id for customer_id in google_api.get_accessible_accounts()] + dummy_customers = [CustomerModel(id=_id, login_customer_id=_id) for _id in customer_ids] + + yield from self.get_all_accounts(google_api, dummy_customers, customer_status_filter) + + def get_customers(self, google_api: GoogleAds, config: Mapping[str, Any]) -> List[CustomerModel]: + customer_status_filter = config.get("customer_status_filter", []) + accounts = self._get_all_connected_accounts(google_api, customer_status_filter) + customers = CustomerModel.from_accounts(accounts) + + # filter duplicates as one customer can be accessible from mutiple connected accounts + unique_customers = [] + seen_ids = set() + for customer in customers: + if customer.id in seen_ids: + continue + seen_ids.add(customer.id) + unique_customers.append(customer) + customers = unique_customers + customers_dict = {customer.id: customer for customer in customers} + + # filter only selected accounts + if config.get("customer_ids"): + customers = [] + for customer_id in config["customer_ids"]: + if customer_id not in customers_dict: + logging.warning(f"Customer with id {customer_id} is not accessible. Skipping it.") + else: + customers.append(customers_dict[customer_id]) + return customers @staticmethod def is_metrics_in_custom_query(query: GAQL) -> bool: @@ -105,98 +149,120 @@ def is_metrics_in_custom_query(query: GAQL) -> bool: return True return False + @staticmethod + def is_custom_query_incremental(query: GAQL) -> bool: + time_segment_in_select, time_segment_in_where = ["segments.date" in clause for clause in [query.fields, query.where]] + return time_segment_in_select and not time_segment_in_where + + def create_custom_query_stream( + self, + google_api: GoogleAds, + single_query_config: Mapping[str, Any], + customers: List[CustomerModel], + non_manager_accounts: List[CustomerModel], + incremental_config: Mapping[str, Any], + non_manager_incremental_config: Mapping[str, Any], + ): + query = single_query_config["query"] + is_incremental = self.is_custom_query_incremental(query) + is_non_manager = self.is_metrics_in_custom_query(query) + + if is_non_manager: + # Skip query with metrics if there are no non-manager accounts + if not non_manager_accounts: + return + + customers = non_manager_accounts + incremental_config = non_manager_incremental_config + + if is_incremental: + return IncrementalCustomQuery(config=single_query_config, **incremental_config) + else: + return CustomQuery(config=single_query_config, api=google_api, customers=customers) + def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, any]: config = self._validate_and_transform(config) - try: - logger.info("Checking the config") - google_api = GoogleAds(credentials=self.get_credentials(config)) - - accounts = self.get_account_info(google_api, config) - customers = Customer.from_accounts(accounts) - # Check custom query request validity by sending metric request with non-existant time window - for customer in customers: - for query in config.get("custom_queries", []): - query = query["query"] - if customer.is_manager_account and self.is_metrics_in_custom_query(query): - logger.warning( - f"Metrics are not available for manager account {customer.id}. " - f"Please remove metrics fields in your custom query: {query}." - ) - if query.resource_name not in FULL_REFRESH_CUSTOM_TABLE: - if IncrementalCustomQuery.cursor_field in query.fields: - return False, f"Custom query should not contain {IncrementalCustomQuery.cursor_field}" - query = IncrementalCustomQuery.insert_segments_date_expr(query, "1980-01-01", "1980-01-01") - query = query.set_limit(1) - response = google_api.send_request(str(query), customer_id=customer.id) - # iterate over the response otherwise exceptions will not be raised! - for _ in response: - pass - return True, None - except GoogleAdsException as exception: - if AuthorizationErrorEnum.AuthorizationError.USER_PERMISSION_DENIED in ( - x.error_code.authorization_error for x in exception.failure.errors - ) or AuthenticationErrorEnum.AuthenticationError.CUSTOMER_NOT_FOUND in ( - x.error_code.authentication_error for x in exception.failure.errors - ): - message = f"Failed to access the customer '{exception.customer_id}'. Ensure the customer is linked to your manager account or check your permissions to access this customer account." - raise AirbyteTracedException(message=message, failure_type=FailureType.config_error) - error_messages = ", ".join([error.message for error in exception.failure.errors]) - logger.error(traceback.format_exc()) - return False, f"Unable to connect to Google Ads API with the provided configuration - {error_messages}" + + logger.info("Checking the config") + google_api = GoogleAds(credentials=self.get_credentials(config)) + + customers = self.get_customers(google_api, config) + logger.info(f"Found {len(customers)} customers: {[customer.id for customer in customers]}") + + # Check custom query request validity by sending metric request with non-existent time window + for customer in customers: + for query in config.get("custom_queries_array", []): + query = query["query"] + if customer.is_manager_account and self.is_metrics_in_custom_query(query): + logger.warning( + f"Metrics are not available for manager account {customer.id}. " + f'Skipping the custom query: "{query}" for manager account.' + ) + continue + + # Add segments.date to where clause of incremental custom queries if they are not present. + # The same will be done during read, but with start and end date from config + if self.is_custom_query_incremental(query): + query = IncrementalCustomQuery.insert_segments_date_expr(query, "1980-01-01", "1980-01-01") + + query = query.set_limit(1) + response = google_api.send_request(str(query), customer_id=customer.id, login_customer_id=customer.login_customer_id) + # iterate over the response otherwise exceptions will not be raised! + for _ in response: + pass + return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: config = self._validate_and_transform(config) google_api = GoogleAds(credentials=self.get_credentials(config)) - accounts = self.get_account_info(google_api, config) - customers = Customer.from_accounts(accounts) + + customers = self.get_customers(google_api, config) + logger.info(f"Found {len(customers)} customers: {[customer.id for customer in customers]}") + non_manager_accounts = [customer for customer in customers if not customer.is_manager_account] + default_config = dict(api=google_api, customers=customers) incremental_config = self.get_incremental_stream_config(google_api, config, customers) non_manager_incremental_config = self.get_incremental_stream_config(google_api, config, non_manager_accounts) streams = [ - AdGroupAds(**incremental_config), - AdGroupAdLabels(google_api, customers=customers), - AdGroups(**incremental_config), - AdGroupBiddingStrategies(**incremental_config), - AdGroupCriterions(google_api, customers=customers), - AdGroupCriterionLabels(google_api, customers=customers), - AdGroupLabels(google_api, customers=customers), - AdListingGroupCriterions(google_api, customers=customers), - Accounts(**incremental_config), - AccountLabels(google_api, customers=customers), - Audience(google_api, customers=customers), - CampaignBiddingStrategies(**incremental_config), - CampaignBudget(**incremental_config), - CampaignLabels(google_api, customers=customers), + AdGroup(**incremental_config), + AdGroupAd(**incremental_config), + AdGroupAdLabel(**default_config), + AdGroupBiddingStrategy(**incremental_config), + AdGroupCriterion(**default_config), + AdGroupCriterionLabel(**default_config), + AdGroupLabel(**default_config), + AdListingGroupCriterion(**default_config), + Audience(**default_config), + CampaignBiddingStrategy(**incremental_config), + CampaignCriterion(**default_config), + CampaignLabel(google_api, customers=customers), ClickView(**incremental_config), - Labels(google_api, customers=customers), - UserInterest(google_api, customers=customers), + Customer(**incremental_config), + CustomerLabel(**default_config), + Label(**default_config), + UserInterest(**default_config), ] # Metrics streams cannot be requested for a manager account. if non_manager_accounts: streams.extend( [ - Campaigns(**non_manager_incremental_config), - UserLocationReport(**non_manager_incremental_config), + Campaign(**non_manager_incremental_config), + CampaignBudget(**non_manager_incremental_config), + UserLocationView(**non_manager_incremental_config), AccountPerformanceReport(**non_manager_incremental_config), - DisplayTopicsPerformanceReport(**non_manager_incremental_config), - DisplayKeywordPerformanceReport(**non_manager_incremental_config), - ShoppingPerformanceReport(**non_manager_incremental_config), - AdGroupAdReport(**non_manager_incremental_config), - GeographicReport(**non_manager_incremental_config), - KeywordReport(**non_manager_incremental_config), + TopicView(**non_manager_incremental_config), + DisplayKeywordView(**non_manager_incremental_config), + ShoppingPerformanceView(**non_manager_incremental_config), + AdGroupAdLegacy(**non_manager_incremental_config), + GeographicView(**non_manager_incremental_config), + KeywordView(**non_manager_incremental_config), ] ) - for single_query_config in config.get("custom_queries", []): - query = single_query_config["query"] - if self.is_metrics_in_custom_query(query): - if non_manager_accounts: - if query.resource_name in FULL_REFRESH_CUSTOM_TABLE: - streams.append(CustomQuery(config=single_query_config, api=google_api, customers=non_manager_accounts)) - else: - streams.append(IncrementalCustomQuery(config=single_query_config, **non_manager_incremental_config)) - continue - if query.resource_name in FULL_REFRESH_CUSTOM_TABLE: - streams.append(CustomQuery(config=single_query_config, api=google_api, customers=customers)) - else: - streams.append(IncrementalCustomQuery(config=single_query_config, **incremental_config)) + + for single_query_config in config.get("custom_queries_array", []): + query_stream = self.create_custom_query_stream( + google_api, single_query_config, customers, non_manager_accounts, incremental_config, non_manager_incremental_config + ) + if query_stream: + streams.append(query_stream) return streams diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json index 73e4eca612b0..2b84f6bc1beb 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json @@ -4,7 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Google Ads Spec", "type": "object", - "required": ["credentials", "start_date", "customer_id"], + "required": ["credentials"], "additionalProperties": true, "properties": { "credentials": { @@ -64,31 +64,43 @@ "examples": ["6783948572,5839201945"], "order": 1 }, + "customer_status_filter": { + "title": "Customer Statuses Filter", + "description": "A list of customer statuses to filter on. For detailed info about what each status mean refer to Google Ads documentation.", + "default": [], + "order": 2, + "type": "array", + "items": { + "title": "CustomerStatus", + "description": "An enumeration.", + "enum": ["UNKNOWN", "ENABLED", "CANCELED", "SUSPENDED", "CLOSED"] + } + }, "start_date": { "type": "string", "title": "Start Date", - "description": "UTC date in the format YYYY-MM-DD. Any data before this date will not be replicated.", + "description": "UTC date in the format YYYY-MM-DD. Any data before this date will not be replicated. (Default value of two years ago is used if not set)", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", "pattern_descriptor": "YYYY-MM-DD", "examples": ["2017-01-25"], - "order": 2, + "order": 3, "format": "date" }, "end_date": { "type": "string", "title": "End Date", - "description": "UTC date in the format YYYY-MM-DD. Any data after this date will not be replicated.", + "description": "UTC date in the format YYYY-MM-DD. Any data after this date will not be replicated. (Default value of today is used if not set)", "pattern": "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}$", "pattern_descriptor": "YYYY-MM-DD", "examples": ["2017-01-30"], - "order": 6, + "order": 4, "format": "date" }, - "custom_queries": { + "custom_queries_array": { "type": "array", "title": "Custom GAQL Queries", "description": "", - "order": 3, + "order": 5, "items": { "type": "object", "required": ["query", "table_name"], @@ -110,15 +122,6 @@ } } }, - "login_customer_id": { - "type": "string", - "title": "Login Customer ID for Managed Accounts", - "description": "If your access to the customer account is through a manager account, this field is required, and must be set to the 10-digit customer ID of the manager account. For more information about this field, refer to Google's documentation.", - "pattern_descriptor": ": 10 digits, with no dashes.", - "pattern": "^([0-9]{10})?$", - "examples": ["7349206847"], - "order": 4 - }, "conversion_window_days": { "title": "Conversion Window", "type": "integer", @@ -127,7 +130,7 @@ "maximum": 1095, "default": 14, "examples": [14], - "order": 5 + "order": 6 } } }, diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py index 1cebedf2abd6..499bceca367e 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py @@ -2,136 +2,104 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import logging -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional +from abc import ABC, abstractmethod +from typing import Any, Iterable, Iterator, List, Mapping, MutableMapping, Optional + +import backoff import pendulum from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import IncrementalMixin, Stream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +from airbyte_cdk.utils import AirbyteTracedException +from airbyte_protocol.models import FailureType from google.ads.googleads.errors import GoogleAdsException -from google.ads.googleads.v11.errors.types.authorization_error import AuthorizationErrorEnum -from google.ads.googleads.v11.errors.types.request_error import RequestErrorEnum -from google.ads.googleads.v11.services.services.google_ads_service.pagers import SearchPager - -from .google_ads import GoogleAds -from .models import Customer - - -class cyclic_sieve: - def __init__(self, logger: logging.Logger, fraction: int = 10): - self._logger = logger - self._cycle_counter = 0 - self._fraction = fraction - - def __getattr__(self, item): - if self._cycle_counter % self._fraction == 0: - return getattr(self._logger, item) - return self.stub - - def stub(self, *args, **kwargs): - pass - - def bump(self): - self._cycle_counter += 1 - - -def parse_dates(stream_slice): - start_date = pendulum.parse(stream_slice["start_date"]) - end_date = pendulum.parse(stream_slice["end_date"]) - return start_date, end_date - - -def chunk_date_range( - start_date: str, - conversion_window: int, - end_date: str = None, - days_of_data_storage: int = None, - range_days: int = None, - time_zone=None, -) -> Iterable[Optional[MutableMapping[str, any]]]: - """ - Returns `start_date` and `end_date` for the given stream_slice. - If (end_date - start_date) is a big date range (>= 1 month), it can take more than 2 hours to process all the records from the given slice. - After 2 hours next page tokens will be expired, finally resulting in page token expired error - Currently this method returns `start_date` and `end_date` with `range_days` difference which is 15 days in most cases. - """ - today = pendulum.today(tz=time_zone) - end_date = min(pendulum.parse(end_date, tz=time_zone), today) if end_date else today - start_date = pendulum.parse(start_date, tz=time_zone) - - # For some metrics we can only get data not older than N days, it is Google Ads policy - if days_of_data_storage: - start_date = max(start_date, pendulum.now(tz=time_zone).subtract(days=days_of_data_storage - conversion_window)) +from google.ads.googleads.v15.services.services.google_ads_service.pagers import SearchPager +from google.ads.googleads.v15.services.types.google_ads_service import SearchGoogleAdsResponse +from google.api_core.exceptions import InternalServerError, ServerError, ServiceUnavailable, TooManyRequests, Unauthenticated - # As in to return some state when state in abnormal - if start_date > end_date: - return [None] - - # applying conversion window - start_date = start_date.subtract(days=conversion_window) - slice_start = start_date - - while slice_start.date() <= end_date.date(): - slice_end = min(end_date, slice_start.add(days=range_days - 1)) - yield { - "start_date": slice_start.to_date_string(), - "end_date": slice_end.to_date_string(), - } - slice_start = slice_end.add(days=1) +from .google_ads import GoogleAds, logger +from .models import CustomerModel +from .utils import ExpiredPageTokenError, chunk_date_range, detached, generator_backoff, get_resource_name, parse_dates, traced_exception class GoogleAdsStream(Stream, ABC): - CATCH_API_ERRORS = True + CATCH_CUSTOMER_NOT_ENABLED_ERROR = True - def __init__(self, api: GoogleAds, customers: List[Customer]): + def __init__(self, api: GoogleAds, customers: List[CustomerModel]): self.google_ads_client = api self.customers = customers - self.base_sieve_logger = cyclic_sieve(self.logger, 10) def get_query(self, stream_slice: Mapping[str, Any]) -> str: - query = GoogleAds.convert_schema_into_query(schema=self.get_json_schema(), report_name=self.name) + fields = GoogleAds.get_fields_from_schema(self.get_json_schema()) + table_name = get_resource_name(self.name) + query = GoogleAds.convert_schema_into_query(fields=fields, table_name=table_name) return query - def parse_response(self, response: SearchPager) -> Iterable[Mapping]: + def parse_response(self, response: SearchPager, stream_slice: Optional[Mapping[str, Any]] = None) -> Iterable[Mapping]: for result in response: yield self.google_ads_client.parse_single_result(self.get_json_schema(), result) def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: for customer in self.customers: - yield {"customer_id": customer.id} + yield {"customer_id": customer.id, "login_customer_id": customer.login_customer_id} + + @generator_backoff( + wait_gen=backoff.constant, + exception=(TimeoutError), + max_tries=5, + on_backoff=lambda details: logger.info( + f"Caught retryable error {details['exception']} after {details['tries']} tries. Waiting {details['wait']} seconds then retrying..." + ), + interval=1, + ) + @detached(timeout_minutes=5) + def request_records_job(self, customer_id, login_customer_id, query, stream_slice): + response_records = self.google_ads_client.send_request(query=query, customer_id=customer_id, login_customer_id=login_customer_id) + yield from self.parse_records_with_backoff(response_records, stream_slice) def read_records(self, sync_mode, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: - self.base_sieve_logger.bump() - self.base_sieve_logger.info(f"Read records using g-ads client. Stream slice is {stream_slice}") if stream_slice is None: return [] customer_id = stream_slice["customer_id"] + login_customer_id = stream_slice["login_customer_id"] + try: - response_records = self.google_ads_client.send_request(self.get_query(stream_slice), customer_id=customer_id) - for response in response_records: - yield from self.parse_response(response) - except GoogleAdsException as exc: - exc.customer_id = customer_id - if not self.CATCH_API_ERRORS: - raise - for error in exc.failure.errors: - if error.error_code.authorization_error == AuthorizationErrorEnum.AuthorizationError.CUSTOMER_NOT_ENABLED: - self.base_sieve_logger.error(error.message) - continue - # log and ignore only CUSTOMER_NOT_ENABLED error, otherwise - raise further - raise + yield from self.request_records_job(customer_id, login_customer_id, self.get_query(stream_slice), stream_slice) + except (GoogleAdsException, Unauthenticated) as exception: + traced_exception(exception, customer_id, self.CATCH_CUSTOMER_NOT_ENABLED_ERROR) + except TimeoutError as exception: + # Prevent sync failure + logger.warning(f"Timeout: Failed to access {self.name} stream data. {str(exception)}") + + @generator_backoff( + wait_gen=backoff.expo, + exception=(InternalServerError, ServerError, ServiceUnavailable, TooManyRequests), + max_tries=5, + max_time=600, + on_backoff=lambda details: logger.info( + f"Caught retryable error {details['exception']} after {details['tries']} tries. Waiting {details['wait']} seconds then retrying..." + ), + factor=5, + ) + def parse_records_with_backoff( + self, response_records: Iterator[SearchGoogleAdsResponse], stream_slice: Optional[Mapping[str, Any]] = None + ) -> Iterable[Mapping[str, Any]]: + for response in response_records: + yield from self.parse_response(response, stream_slice) class IncrementalGoogleAdsStream(GoogleAdsStream, IncrementalMixin, ABC): + primary_key = None days_of_data_storage = None cursor_field = "segments.date" - primary_key = None - # Date range is set to 15 days, because for conversion_window_days default value is 14. - # Range less than 15 days will break the integration tests. - range_days = 15 + cursor_time_format = "YYYY-MM-DD" + # Slice duration is set to 14 days, because for conversion_window_days default value is 14. + # Range less than 14 days will break the integration tests. + slice_duration = pendulum.duration(days=14) + # slice step is difference from one slice end_date and next slice start_date + slice_step = pendulum.duration(days=1) def __init__(self, start_date: str, conversion_window_days: int, end_date: str = None, **kwargs): self.conversion_window_days = conversion_window_days @@ -139,7 +107,11 @@ def __init__(self, start_date: str, conversion_window_days: int, end_date: str = self._end_date = end_date self._state = {} super().__init__(**kwargs) - self.incremental_sieve_logger = cyclic_sieve(self.logger, 10) + + @property + def state_checkpoint_interval(self) -> int: + # default page size is 10000, so set to 10% of it + return 1000 @property def state(self) -> MutableMapping[str, Any]: @@ -149,13 +121,12 @@ def state(self) -> MutableMapping[str, Any]: def state(self, value: MutableMapping[str, Any]): self._state.update(value) - def current_state(self, customer_id, default=None): + def get_current_state(self, customer_id, default=None): default = default or self.state.get(self.cursor_field) return self.state.get(customer_id, {}).get(self.cursor_field) or default def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[MutableMapping[str, any]]]: for customer in self.customers: - logger = cyclic_sieve(self.logger, 10) stream_state = stream_state or {} if stream_state.get(customer.id): start_date = stream_state[customer.id].get(self.cursor_field) or self._start_date @@ -167,22 +138,50 @@ def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Ite start_date = self._start_date end_date = self._end_date - logger.info(f"Generating slices for customer {customer.id}. Start date is {start_date}, end date is {end_date}") for chunk in chunk_date_range( start_date=start_date, end_date=end_date, conversion_window=self.conversion_window_days, days_of_data_storage=self.days_of_data_storage, - range_days=self.range_days, time_zone=customer.time_zone, + time_format=self.cursor_time_format, + slice_duration=self.slice_duration, + slice_step=self.slice_step, ): if chunk: chunk["customer_id"] = customer.id - logger.info(f"Next slice is {chunk}") - logger.bump() + chunk["login_customer_id"] = customer.login_customer_id yield chunk + def _update_state(self, customer_id: str, record: MutableMapping[str, Any]): + """Update the state based on the latest record's cursor value.""" + current_state = self.get_current_state(customer_id) + if current_state: + date_in_current_stream = pendulum.parse(current_state) + date_in_latest_record = pendulum.parse(record[self.cursor_field]) + cursor_value = (max(date_in_current_stream, date_in_latest_record)).format(self.cursor_time_format) + self.state = {customer_id: {self.cursor_field: cursor_value}} + else: + self.state = {customer_id: {self.cursor_field: record[self.cursor_field]}} + + def _handle_expired_page_exception(self, exception: ExpiredPageTokenError, stream_slice: MutableMapping[str, Any], customer_id: str): + """ + Handle Google Ads EXPIRED_PAGE_TOKEN error by updating the stream slice. + """ + start_date, end_date = parse_dates(stream_slice) + current_state = self.get_current_state(customer_id) + + if end_date - start_date <= self.slice_step: + # If range days less than slice_step, no need in retry, because it's the minimum date range + raise exception + elif current_state == stream_slice["start_date"]: + # It couldn't read all the records within one day, it will enter an infinite loop, + # so raise the error + raise exception + # Retry reading records from where it crushed + stream_slice["start_date"] = self.get_current_state(customer_id, default=stream_slice["start_date"]) + def read_records( self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_slice: MutableMapping[str, Any] = None, **kwargs ) -> Iterable[Mapping[str, Any]]: @@ -190,79 +189,107 @@ def read_records( This method is overridden to handle GoogleAdsException with EXPIRED_PAGE_TOKEN error code, and update `start_date` key in the `stream_slice` with the latest read record's cursor value, then retry the sync. """ - self.incremental_sieve_logger.bump() while True: - self.incremental_sieve_logger.info("Starting a while loop iteration") customer_id = stream_slice and stream_slice["customer_id"] + try: + # count records to update slice date range with latest record time when limit is hit records = super().read_records(sync_mode, stream_slice=stream_slice) for record in records: - current_state = self.current_state(customer_id) - if current_state: - date_in_current_stream = pendulum.parse(current_state) - date_in_latest_record = pendulum.parse(record[self.cursor_field]) - cursor_value = (max(date_in_current_stream, date_in_latest_record)).to_date_string() - self.state = {customer_id: {self.cursor_field: cursor_value}} - # When large amount of data this log produces so much records so the enire log is not usable - # See: https://github.com/airbytehq/oncall/issues/2460 - # self.incremental_sieve_logger.info(f"Updated state for customer {customer_id}. Full state is {self.state}.") - yield record - continue - self.state = {customer_id: {self.cursor_field: record[self.cursor_field]}} - self.incremental_sieve_logger.info(f"Initialized state for customer {customer_id}. Full state is {self.state}.") + self._update_state(customer_id, record) yield record - continue - except GoogleAdsException as exception: - self.incremental_sieve_logger.info(f"Caught a GoogleAdsException: {str(exception)}") - error = next(iter(exception.failure.errors)) - if error.error_code.request_error == RequestErrorEnum.RequestError.EXPIRED_PAGE_TOKEN: - start_date, end_date = parse_dates(stream_slice) - current_state = self.current_state(customer_id) - self.incremental_sieve_logger.info( - f"Start date is {start_date}. End date is {end_date}. Current state is {current_state}" - ) - if (end_date - start_date).days == 1: - # If range days is 1, no need in retry, because it's the minimum date range - self.incremental_sieve_logger.error("Page token has expired.") - raise exception - elif current_state == stream_slice["start_date"]: - # It couldn't read all the records within one day, it will enter an infinite loop, - # so raise the error - self.incremental_sieve_logger.error("Page token has expired.") - raise exception - # Retry reading records from where it crushed - stream_slice["start_date"] = self.current_state(customer_id, default=stream_slice["start_date"]) - self.incremental_sieve_logger.info(f"Retry reading records from where it crushed with a modified slice: {stream_slice}") - else: - # raise caught error for other error statuses - raise exception + except ExpiredPageTokenError as exception: + # handle expired page error that was caught in parent class by updating stream_slice + self._handle_expired_page_exception(exception, stream_slice, customer_id) else: - # return the control if no exception is raised - self.incremental_sieve_logger.info("Current slice has been read. Exiting read_records()") return def get_query(self, stream_slice: Mapping[str, Any] = None) -> str: + fields = GoogleAds.get_fields_from_schema(self.get_json_schema()) + table_name = get_resource_name(self.name) + + start_date, end_date = stream_slice.get("start_date"), stream_slice.get("end_date") + cursor_condition = [f"{self.cursor_field} >= '{start_date}' AND {self.cursor_field} <= '{end_date}'"] + query = GoogleAds.convert_schema_into_query( - schema=self.get_json_schema(), - report_name=self.name, - from_date=stream_slice.get("start_date"), - to_date=stream_slice.get("end_date"), - cursor_field=self.cursor_field, + fields=fields, table_name=table_name, conditions=cursor_condition, order_field=self.cursor_field ) return query -class Accounts(IncrementalGoogleAdsStream): +class Customer(IncrementalGoogleAdsStream): """ - Accounts stream: https://developers.google.com/google-ads/api/fields/v11/customer + Customer stream: https://developers.google.com/google-ads/api/fields/v15/customer """ primary_key = ["customer.id", "segments.date"] + def parse_response(self, response: SearchPager, stream_slice: Optional[Mapping[str, Any]] = None) -> Iterable[Mapping]: + for record in super().parse_response(response): + if isinstance(record.get("customer.optimization_score_weight"), int): + record["customer.optimization_score_weight"] = float(record["customer.optimization_score_weight"]) + yield record + -class AccountLabels(GoogleAdsStream): +class CustomerClient(GoogleAdsStream): """ - Account Labels stream: https://developers.google.com/google-ads/api/fields/v14/customer_label + Customer Client stream: https://developers.google.com/google-ads/api/fields/v15/customer_client + """ + + primary_key = ["customer_client.id"] + + def __init__(self, customer_status_filter: List[str], **kwargs): + self.customer_status_filter = customer_status_filter + super().__init__(**kwargs) + + def get_query(self, stream_slice: Mapping[str, Any] = None) -> str: + fields = GoogleAds.get_fields_from_schema(self.get_json_schema()) + table_name = get_resource_name(self.name) + + active_customers_condition = [] + if self.customer_status_filter: + customer_status_filter = ", ".join([f"'{status}'" for status in self.customer_status_filter]) + active_customers_condition = [f"customer_client.status in ({customer_status_filter})"] + + query = GoogleAds.convert_schema_into_query(fields=fields, table_name=table_name, conditions=active_customers_condition) + return query + + def read_records(self, sync_mode, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + """ + This method is overridden to avoid using login_customer_id from dummy_customers. + + login_customer_id is used in the stream_slices to pass it to child customers, + but we don't need it here as this class iterate over customers accessible from user creds. + """ + if stream_slice is None: + return [] + + customer_id = stream_slice["customer_id"] + + try: + response_records = self.google_ads_client.send_request(self.get_query(stream_slice), customer_id=customer_id) + + yield from self.parse_records_with_backoff(response_records, stream_slice) + except GoogleAdsException as exception: + traced_exception(exception, customer_id, self.CATCH_CUSTOMER_NOT_ENABLED_ERROR) + + def parse_response(self, response: SearchPager, stream_slice: Optional[Mapping[str, Any]] = None) -> Iterable[Mapping]: + """ + login_cusotmer_id is populated to child customers if they are under managers account + """ + records = [record for record in super().parse_response(response)] + + # read_records get all customers connected to customer_id from stream_slice + # if the result is more than one customer, it's a manager, otherwise it is client account for which we don't need login_customer_id + root_is_manager = len(records) > 1 + for record in records: + record["login_customer_id"] = stream_slice["login_customer_id"] if root_is_manager else "default" + yield record + + +class CustomerLabel(GoogleAdsStream): + """ + Customer Label stream: https://developers.google.com/google-ads/api/fields/v15/customer_label """ primary_key = ["customer_label.resource_name"] @@ -273,183 +300,219 @@ class ServiceAccounts(GoogleAdsStream): This stream is intended to be used as a service class, not exposed to a user """ - CATCH_API_ERRORS = False + CATCH_CUSTOMER_NOT_ENABLED_ERROR = False primary_key = ["customer.id"] -class Campaigns(IncrementalGoogleAdsStream): +class Campaign(IncrementalGoogleAdsStream): """ - Campaigns stream: https://developers.google.com/google-ads/api/fields/v11/campaign + Campaign stream: https://developers.google.com/google-ads/api/fields/v15/campaign """ transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - primary_key = ["campaign.id", "segments.date", "segments.hour"] + primary_key = ["campaign.id", "segments.date", "segments.hour", "segments.ad_network_type"] class CampaignBudget(IncrementalGoogleAdsStream): """ - Campaigns stream: https://developers.google.com/google-ads/api/fields/v13/campaign_budget + Campaigns stream: https://developers.google.com/google-ads/api/fields/v15/campaign_budget """ transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - primary_key = ["campaign_budget.id", "segments.date"] + primary_key = [ + "customer.id", + "campaign_budget.id", + "segments.date", + "segments.budget_campaign_association_status.campaign", + "segments.budget_campaign_association_status.status", + ] -class CampaignBiddingStrategies(IncrementalGoogleAdsStream): +class CampaignBiddingStrategy(IncrementalGoogleAdsStream): """ - Campaign Bidding Strategies stream: https://developers.google.com/google-ads/api/fields/v14/campaign + Campaign Bidding Strategy stream: https://developers.google.com/google-ads/api/fields/v15/campaign """ transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) primary_key = ["campaign.id", "bidding_strategy.id", "segments.date"] -class CampaignLabels(GoogleAdsStream): +class CampaignLabel(GoogleAdsStream): """ - Campaign labels stream: https://developers.google.com/google-ads/api/fields/v11/campaign_label + Campaign labels stream: https://developers.google.com/google-ads/api/fields/v15/campaign_label """ # Note that this is a string type. Google doesn't return a more convenient identifier. - primary_key = ["campaign_label.resource_name"] + primary_key = ["campaign.id", "label.id"] -class AdGroups(IncrementalGoogleAdsStream): +class AdGroup(IncrementalGoogleAdsStream): """ - AdGroups stream: https://developers.google.com/google-ads/api/fields/v11/ad_group + AdGroup stream: https://developers.google.com/google-ads/api/fields/v15/ad_group """ primary_key = ["ad_group.id", "segments.date"] + def get_query(self, stream_slice: Mapping[str, Any] = None) -> str: + fields = GoogleAds.get_fields_from_schema(self.get_json_schema()) + # validation that the customer is not a manager + # due to unsupported metrics.cost_micros field and removing it in case custom is a manager + if [customer for customer in self.customers if customer.id == stream_slice["customer_id"]][0].is_manager_account: + fields = [field for field in fields if field != "metrics.cost_micros"] + table_name = get_resource_name(self.name) + start_date, end_date = stream_slice.get("start_date"), stream_slice.get("end_date") + cursor_condition = [f"{self.cursor_field} >= '{start_date}' AND {self.cursor_field} <= '{end_date}'"] -class AdGroupLabels(GoogleAdsStream): - """ - Ad Group Labels stream: https://developers.google.com/google-ads/api/fields/v11/ad_group_label - """ - - # Note that this is a string type. Google doesn't return a more convenient identifier. - primary_key = ["ad_group_label.resource_name"] + query = GoogleAds.convert_schema_into_query( + fields=fields, table_name=table_name, conditions=cursor_condition, order_field=self.cursor_field + ) + return query -class AdGroupBiddingStrategies(IncrementalGoogleAdsStream): +class AdGroupLabel(GoogleAdsStream): """ - Ad Group Bidding Strategies stream: https://developers.google.com/google-ads/api/fields/v14/ad_group + Ad Group Labels stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_label """ - transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - primary_key = ["ad_group.id", "bidding_strategy.id", "segments.date"] + # Note that this is a string type. Google doesn't return a more convenient identifier. + primary_key = ["ad_group.id", "label.id"] -class AdGroupCriterions(GoogleAdsStream): +class AdGroupBiddingStrategy(IncrementalGoogleAdsStream): """ - Ad Group Criterions stream: https://developers.google.com/google-ads/api/fields/v14/ad_group_criterion + Ad Group Bidding Strategies stream: https://developers.google.com/google-ads/api/fields/v15/ad_group """ transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - primary_key = ["ad_group.id", "ad_group_criterion.criterion_id"] + primary_key = ["ad_group.id", "bidding_strategy.id", "segments.date"] -class AdGroupCriterionLabels(GoogleAdsStream): +class AdGroupCriterionLabel(GoogleAdsStream): """ - Ad Group Criterion Labels stream: https://developers.google.com/google-ads/api/fields/v14/ad_group_criterion_label + Ad Group Criterion Label stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_criterion_label """ transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) primary_key = ["ad_group_criterion_label.resource_name"] -class AdListingGroupCriterions(GoogleAdsStream): - """ - Ad Group Criterions stream: https://developers.google.com/google-ads/api/fields/v14/ad_group_criterion - """ - - transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - primary_key = ["ad_group.id", "ad_group_criterion.criterion_id"] - - -class AdGroupAds(IncrementalGoogleAdsStream): +class AdGroupAd(IncrementalGoogleAdsStream): """ - AdGroups stream: https://developers.google.com/google-ads/api/fields/v11/ad_group_ad + Ad Group Ad stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_ad """ - primary_key = ["ad_group_ad.ad.id", "segments.date"] + primary_key = ["ad_group.id", "ad_group_ad.ad.id", "segments.date"] -class AdGroupAdLabels(GoogleAdsStream): +class AdGroupAdLabel(GoogleAdsStream): """ - Ad Group Ad Labels stream: https://developers.google.com/google-ads/api/fields/v11/ad_group_ad_label + Ad Group Ad Labels stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_ad_label """ - # Note that this is a string type. Google doesn't return a more convenient identifier. - primary_key = ["ad_group_ad_label.resource_name"] + primary_key = ["ad_group.id", "ad_group_ad.ad.id", "label.id"] class AccountPerformanceReport(IncrementalGoogleAdsStream): """ - AccountPerformanceReport stream: https://developers.google.com/google-ads/api/fields/v11/customer + AccountPerformanceReport stream: https://developers.google.com/google-ads/api/fields/v15/customer Google Ads API field mapping: https://developers.google.com/google-ads/api/docs/migration/mapping#account_performance """ + primary_key = ["customer.id", "segments.date", "segments.ad_network_type", "segments.device"] + -class AdGroupAdReport(IncrementalGoogleAdsStream): +class AdGroupAdLegacy(IncrementalGoogleAdsStream): """ - AdGroupAdReport stream: https://developers.google.com/google-ads/api/fields/v11/ad_group_ad + AdGroupAdReport stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_ad Google Ads API field mapping: https://developers.google.com/google-ads/api/docs/migration/mapping#ad_performance """ + primary_key = ["ad_group.id", "ad_group_ad.ad.id", "segments.date", "segments.ad_network_type"] + -class DisplayKeywordPerformanceReport(IncrementalGoogleAdsStream): +class DisplayKeywordView(IncrementalGoogleAdsStream): """ - DisplayKeywordPerformanceReport stream: https://developers.google.com/google-ads/api/fields/v11/display_keyword_view + DisplayKeywordView stream: https://developers.google.com/google-ads/api/fields/v15/display_keyword_view Google Ads API field mapping: https://developers.google.com/google-ads/api/docs/migration/mapping#display_keyword_performance """ + primary_key = [ + "ad_group.id", + "ad_group_criterion.criterion_id", + "segments.date", + "segments.ad_network_type", + "segments.device", + ] -class DisplayTopicsPerformanceReport(IncrementalGoogleAdsStream): + +class TopicView(IncrementalGoogleAdsStream): """ - DisplayTopicsPerformanceReport stream: https://developers.google.com/google-ads/api/fields/v11/topic_view + DisplayTopicsPerformanceReport stream: https://developers.google.com/google-ads/api/fields/v15/topic_view Google Ads API field mapping: https://developers.google.com/google-ads/api/docs/migration/mapping#display_topics_performance """ + primary_key = [ + "ad_group.id", + "ad_group_criterion.criterion_id", + "segments.date", + "segments.ad_network_type", + "segments.device", + ] + -class ShoppingPerformanceReport(IncrementalGoogleAdsStream): +class ShoppingPerformanceView(IncrementalGoogleAdsStream): """ - ShoppingPerformanceReport stream: https://developers.google.com/google-ads/api/fields/v11/shopping_performance_view + ShoppingPerformanceView stream: https://developers.google.com/google-ads/api/fields/v15/shopping_performance_view Google Ads API field mapping: https://developers.google.com/google-ads/api/docs/migration/mapping#shopping_performance """ -class UserLocationReport(IncrementalGoogleAdsStream): +class UserLocationView(IncrementalGoogleAdsStream): """ - UserLocationReport stream: https://developers.google.com/google-ads/api/fields/v11/user_location_view + UserLocationView stream: https://developers.google.com/google-ads/api/fields/v15/user_location_view Google Ads API field mapping: https://developers.google.com/google-ads/api/docs/migration/mapping#geo_performance """ + primary_key = [ + "customer.id", + "user_location_view.country_criterion_id", + "user_location_view.targeting_location", + "segments.date", + "segments.ad_network_type", + ] + -class GeographicReport(IncrementalGoogleAdsStream): +class GeographicView(IncrementalGoogleAdsStream): """ - UserLocationReport stream: https://developers.google.com/google-ads/api/fields/v11/geographic_view + UserLocationReport stream: https://developers.google.com/google-ads/api/fields/v15/geographic_view """ + primary_key = ["customer.id", "geographic_view.country_criterion_id", "geographic_view.location_type", "segments.date"] -class KeywordReport(IncrementalGoogleAdsStream): + +class KeywordView(IncrementalGoogleAdsStream): """ - UserLocationReport stream: https://developers.google.com/google-ads/api/fields/v11/keyword_view + UserLocationReport stream: https://developers.google.com/google-ads/api/fields/v15/keyword_view """ + primary_key = ["ad_group.id", "ad_group_criterion.criterion_id", "segments.date"] + class ClickView(IncrementalGoogleAdsStream): """ - ClickView stream: https://developers.google.com/google-ads/api/reference/rpc/v11/ClickView + ClickView stream: https://developers.google.com/google-ads/api/reference/rpc/v15/ClickView """ primary_key = ["click_view.gclid", "segments.date", "segments.ad_network_type"] days_of_data_storage = 90 - range_days = 1 + # where clause for cursor is inclusive from both sides, duration 0 will result in - '"2022-01-01" <= cursor AND "2022-01-01" >= cursor' + # Queries including ClickView must have a filter limiting the results to one day + slice_duration = pendulum.duration(days=0) class UserInterest(GoogleAdsStream): """ - Ad Group Ad Labels stream: https://developers.google.com/google-ads/api/fields/v11/ad_group_ad_label + Ad Group Ad Labels stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_ad_label """ primary_key = ["user_interest.user_interest_id"] @@ -457,15 +520,323 @@ class UserInterest(GoogleAdsStream): class Audience(GoogleAdsStream): """ - Ad Group Ad Labels stream: https://developers.google.com/google-ads/api/fields/v11/ad_group_ad_label + Ad Group Ad Labels stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_ad_label """ - primary_key = ["audience.id"] + primary_key = ["customer.id", "audience.id"] -class Labels(GoogleAdsStream): +class Label(GoogleAdsStream): """ - Labels stream: https://developers.google.com/google-ads/api/fields/v14/label + Label stream: https://developers.google.com/google-ads/api/fields/v15/label """ primary_key = ["label.id"] + + +class ChangeStatus(IncrementalGoogleAdsStream): + """ + Change status stream: https://developers.google.com/google-ads/api/fields/v15/change_status + Stream is only used internally to implement incremental updates for child streams of IncrementalEventsStream + """ + + cursor_field = "change_status.last_change_date_time" + slice_step = pendulum.duration(microseconds=1) + days_of_data_storage = 90 + cursor_time_format = "YYYY-MM-DD HH:mm:ss.SSSSSS" + + def __init__(self, **kwargs): + # date range is not used for these streams, only state is used to sync recent records, otherwise full refresh + for key in ["start_date", "conversion_window_days", "end_date"]: + kwargs.pop(key, None) + super().__init__(start_date=None, conversion_window_days=0, end_date=None, **kwargs) + + @property + def query_limit(self) -> Optional[int]: + """Queries for ChangeStatus resource have to include limit in it""" + return 10000 + + def read_records( + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_slice: MutableMapping[str, Any] = None, **kwargs + ) -> Iterable[Mapping[str, Any]]: + """ + This method is overridden to handle GoogleAdsException with EXPIRED_PAGE_TOKEN error code, + and update `start_date` key in the `stream_slice` with the latest read record's cursor value, then retry the sync. + """ + while True: + records_count = 0 + customer_id = stream_slice and stream_slice["customer_id"] + + try: + # count records to update slice date range with latest record time when limit is hit + records = super().read_records(sync_mode, stream_slice=stream_slice) + for records_count, record in enumerate(records, start=1): + self._update_state(customer_id, record) + yield record + except ExpiredPageTokenError as exception: + # handle expired page error that was caught in parent class by updating stream_slice + self._handle_expired_page_exception(exception, stream_slice, customer_id) + else: + # if records limit is hit - update slice with new start_date to continue reading + if self.query_limit and records_count == self.query_limit: + # if state was not updated before hitting limit - raise error to avoid infinite loop + if stream_slice["start_date"] == self.get_current_state(customer_id): + raise AirbyteTracedException( + message=f"More than limit {self.query_limit} records with same cursor field. Incremental sync is not possible for this stream.", + failure_type=FailureType.system_error, + ) + + current_state = self.get_current_state(customer_id, default=stream_slice["start_date"]) + stream_slice["start_date"] = current_state + else: + return + + def get_query(self, stream_slice: Mapping[str, Any] = None) -> str: + fields = GoogleAds.get_fields_from_schema(self.get_json_schema()) + table_name = get_resource_name(self.name) + + start_date, end_date = stream_slice.get("start_date"), stream_slice.get("end_date") + conditions = [f"{self.cursor_field} >= '{start_date}' AND {self.cursor_field} <= '{end_date}'"] + + resource_type = stream_slice.get("resource_type") + conditions.append(f"change_status.resource_type = '{resource_type}'") + + query = GoogleAds.convert_schema_into_query( + fields=fields, table_name=table_name, conditions=conditions, order_field=self.cursor_field, limit=self.query_limit + ) + return query + + +class IncrementalEventsStream(GoogleAdsStream, IncrementalMixin, ABC): + """ + Abstract class used for getting incremental updates based on events returned from ChangeStatus stream. + Only Ad Group Criterion and Campaign Criterion streams are fetched using this class, for other resources + like Campaigns, Ad Groups, Ad Group Ads, and Campaign Budget we already fetch incremental updates based on date. + Also, these resources, unlike criterions, can't be deleted, only marked as "Removed". + """ + + def __init__(self, **kwargs): + self.parent_stream = ChangeStatus(api=kwargs.get("api"), customers=kwargs.get("customers")) + self.parent_stream_name: str = self.parent_stream.name + self.parent_cursor_field: str = self.parent_stream.cursor_field + + super().__init__(**kwargs) + + self._state = {self.parent_stream_name: {customer.id: None for customer in self.customers}} + + @property + @abstractmethod + def id_field(self) -> str: + """Name of field used for getting records by id""" + pass + + @property + @abstractmethod + def parent_id_field(self) -> str: + """Field name of id from parent record""" + pass + + @property + @abstractmethod + def resource_type(self) -> str: + """Resource type used for filtering parent records""" + pass + + @property + def state(self) -> MutableMapping[str, Any]: + return self._state + + @state.setter + def state(self, value: MutableMapping[str, Any]): + self._state.update(value) + self.parent_stream.state = self._state.get(self.parent_stream_name, {}) + + def get_current_state(self, customer_id, default=None): + return self.parent_stream.get_current_state(customer_id, default) + + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[MutableMapping[str, any]]]: + """ + If state exists read updates from parent stream otherwise return slices with only customer id to sync all records for stream + """ + if stream_state: + slices_generator = self.read_parent_stream(SyncMode.incremental, self.parent_cursor_field, stream_state) + yield from slices_generator + else: + for customer in self.customers: + yield { + "customer_id": customer.id, + "login_customer_id": customer.login_customer_id, + "updated_ids": set(), + "deleted_ids": set(), + "record_changed_time_map": dict(), + } + + def _process_parent_record(self, parent_record: MutableMapping[str, Any], child_slice: MutableMapping[str, Any]) -> bool: + """Process a single parent_record and update the child_slice.""" + substream_id = parent_record.get(self.parent_id_field) + if not substream_id: + return False + + # Save time of change + child_slice["record_changed_time_map"][substream_id] = parent_record[self.parent_cursor_field] + + # Add record id to list of changed or deleted items depending on status + slice_id_list = "deleted_ids" if parent_record.get("change_status.resource_status") == "REMOVED" else "updated_ids" + child_slice[slice_id_list].add(substream_id) + + return True + + def read_parent_stream( + self, sync_mode: SyncMode, cursor_field: Optional[str], stream_state: Mapping[str, Any] + ) -> Iterable[Mapping[str, Any]]: + for parent_slice in self.parent_stream.stream_slices( + sync_mode=sync_mode, cursor_field=cursor_field, stream_state=stream_state.get(self.parent_stream_name) + ): + customer_id = parent_slice.get("customer_id") + child_slice = { + "customer_id": customer_id, + "updated_ids": set(), + "deleted_ids": set(), + "record_changed_time_map": dict(), + "login_customer_id": parent_slice.get("login_customer_id"), + } + if not self.get_current_state(customer_id): + yield child_slice + continue + + parent_slice["resource_type"] = self.resource_type + for parent_record in self.parent_stream.read_records(sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=parent_slice): + self._process_parent_record(parent_record, child_slice) + + # yield child slice if any records where read + if child_slice["record_changed_time_map"]: + yield child_slice + + def parse_response(self, response: SearchPager, stream_slice: MutableMapping[str, Any] = None) -> Iterable[Mapping]: + # update records with time obtained from parent stream + for record in super().parse_response(response): + primary_key_value = record[self.primary_key[0]] + + # cursor value obtained from parent stream + cursor_value = stream_slice.get("record_changed_time_map", dict()).get(primary_key_value) + + record[self.cursor_field] = cursor_value + yield record + + def _update_state(self, stream_slice: MutableMapping[str, Any]): + customer_id = stream_slice.get("customer_id") + + # if parent stream was used - copy state from it, otherwise set default state + if isinstance(self.parent_stream.state, dict) and self.parent_stream.state.get(customer_id): + self._state[self.parent_stream_name][customer_id] = self.parent_stream.state[customer_id] + else: + parent_state = {self.parent_cursor_field: pendulum.today().start_of("day").format(self.parent_stream.cursor_time_format)} + # full refresh sync without parent stream + self._state[self.parent_stream_name].update({customer_id: parent_state}) + + def _read_deleted_records(self, stream_slice: MutableMapping[str, Any] = None): + # yield deleted records with id and time when record was deleted + for deleted_record_id in stream_slice.get("deleted_ids", []): + yield {self.id_field: deleted_record_id, "deleted_at": stream_slice["record_changed_time_map"].get(deleted_record_id)} + + @staticmethod + def _split_slice(child_slice: MutableMapping[str, Any], chunk_size: int = 10000) -> Iterable[Mapping[str, Any]]: + """ + Splits a child slice into smaller chunks based on the chunk_size. + + Parameters: + - child_slice (MutableMapping[str, Any]): The input dictionary to split. + - chunk_size (int, optional): The maximum number of ids per chunk. Defaults to 10000, + because it is the maximum number of ids that can be present in a query filter. + + Yields: + - Mapping[str, Any]: A dictionary with a similar structure to child_slice. + """ + updated_ids = list(child_slice["updated_ids"]) + if not updated_ids: + yield child_slice + return + + record_changed_time_map = child_slice["record_changed_time_map"] + customer_id = child_slice["customer_id"] + login_customer_id = child_slice["login_customer_id"] + + # Split the updated_ids into chunks and yield them + for i in range(0, len(updated_ids), chunk_size): + chunk_ids = set(updated_ids[i : i + chunk_size]) + chunk_time_map = {k: record_changed_time_map[k] for k in chunk_ids} + + yield { + "updated_ids": chunk_ids, + "record_changed_time_map": chunk_time_map, + "customer_id": customer_id, + "deleted_ids": set(), + "login_customer_id": login_customer_id, + } + + def read_records( + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_slice: MutableMapping[str, Any] = None, **kwargs + ) -> Iterable[Mapping[str, Any]]: + """ + This method is overridden to read records using parent stream + """ + # if state is present read records by ids from slice otherwise full refresh sync + for stream_slice_part in self._split_slice(stream_slice): + yield from super().read_records(sync_mode, stream_slice=stream_slice_part) + + # yield deleted items + yield from self._read_deleted_records(stream_slice) + + self._update_state(stream_slice) + + def get_query(self, stream_slice: Mapping[str, Any] = None) -> str: + table_name = get_resource_name(self.name) + + fields = GoogleAds.get_fields_from_schema(self.get_json_schema()) + # delete fields that are obtained from parent stream and should not be requested from API + delete_fields = ["change_status.last_change_date_time", "deleted_at"] + fields = [field_name for field_name in fields if field_name not in delete_fields] + + conditions = [] + # filter by ids obtained from parent stream + updated_ids = stream_slice.get("updated_ids") + if updated_ids: + id_list_str = ", ".join(f"'{str(id_)}'" for id_ in updated_ids) + conditions.append(f"{self.id_field} IN ({id_list_str})") + + query = GoogleAds.convert_schema_into_query(fields=fields, table_name=table_name, conditions=conditions) + return query + + +class AdGroupCriterion(IncrementalEventsStream): + """ + Ad Group Criterion stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_criterion + """ + + transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + primary_key = ["ad_group_criterion.resource_name"] + parent_id_field = "change_status.ad_group_criterion" + id_field = "ad_group_criterion.resource_name" + resource_type = "AD_GROUP_CRITERION" + cursor_field = "change_status.last_change_date_time" + + +class AdListingGroupCriterion(AdGroupCriterion): + """ + Ad Listing Group Criterion stream: https://developers.google.com/google-ads/api/fields/v15/ad_group_criterion + While this stream utilizes the same resource as the AdGroupCriterions, + it specifically targets the listing group and has distinct schemas. + """ + + +class CampaignCriterion(IncrementalEventsStream): + """ + Campaign Criterion stream: https://developers.google.com/google-ads/api/fields/v15/campaign_criterion + """ + + transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + primary_key = ["campaign_criterion.resource_name"] + parent_id_field = "change_status.campaign_criterion" + id_field = "campaign_criterion.resource_name" + resource_type = "CAMPAIGN_CRITERION" + cursor_field = "change_status.last_change_date_time" diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py index ff3c66feafb7..3085343c9278 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/utils.py @@ -2,9 +2,373 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import functools +import queue import re +import threading +import time from dataclasses import dataclass -from typing import Optional, Tuple +from datetime import datetime +from typing import Any, Callable, Generator, Iterable, MutableMapping, Optional, Tuple, Type, Union + +import pendulum +from airbyte_cdk.models import FailureType +from airbyte_cdk.utils import AirbyteTracedException +from google.ads.googleads.errors import GoogleAdsException +from google.ads.googleads.v15.errors.types.authentication_error import AuthenticationErrorEnum +from google.ads.googleads.v15.errors.types.authorization_error import AuthorizationErrorEnum +from google.ads.googleads.v15.errors.types.quota_error import QuotaErrorEnum +from google.ads.googleads.v15.errors.types.request_error import RequestErrorEnum +from google.api_core.exceptions import Unauthenticated +from source_google_ads.google_ads import logger + + +def get_resource_name(stream_name: str) -> str: + """Returns resource name for stream name""" + return REPORT_MAPPING[stream_name] if stream_name in REPORT_MAPPING else stream_name + + +# maps stream name to name of resource in Google Ads +REPORT_MAPPING = { + "account_performance_report": "customer", + "ad_group_ad_legacy": "ad_group_ad", + "ad_group_bidding_strategy": "ad_group", + "ad_listing_group_criterion": "ad_group_criterion", + "campaign_real_time_bidding_settings": "campaign", + "campaign_bidding_strategy": "campaign", + "service_accounts": "customer", +} + + +class ExpiredPageTokenError(AirbyteTracedException): + """ + Custom AirbyteTracedException exception to handle the scenario when the page token has expired + while processing a response from Google Ads. + """ + + pass + + +def is_error_type(error_value, target_enum_value): + """Compares error value with target enum value after converting both to integers.""" + return int(error_value) == int(target_enum_value) + + +def traced_exception(ga_exception: Union[GoogleAdsException, Unauthenticated], customer_id: str, catch_disabled_customer_error: bool): + """Add user-friendly message for GoogleAdsException""" + messages = [] + raise_exception = AirbyteTracedException + failure_type = FailureType.config_error + + if isinstance(ga_exception, Unauthenticated): + message = ( + f"Authentication failed for the customer '{customer_id}'. " + f"Please try to Re-authenticate your credentials on set up Google Ads page." + ) + raise raise_exception.from_exception(failure_type=failure_type, exc=ga_exception, message=message) from ga_exception + + for error in ga_exception.failure.errors: + # Get error codes + authorization_error = error.error_code.authorization_error + authentication_error = error.error_code.authentication_error + query_error = error.error_code.query_error + quota_error = error.error_code.quota_error + request_error = error.error_code.request_error + + if is_error_type(authorization_error, AuthorizationErrorEnum.AuthorizationError.USER_PERMISSION_DENIED) or is_error_type( + authentication_error, AuthenticationErrorEnum.AuthenticationError.CUSTOMER_NOT_FOUND + ): + message = ( + f"Failed to access the customer '{customer_id}'. " + f"Ensure the customer is linked to your manager account or check your permissions to access this customer account." + ) + + # If the error is encountered in the internally used class `ServiceAccounts`, an exception is raised. + # For other classes, the error is logged and skipped to prevent sync failure. See: https://github.com/airbytehq/airbyte/issues/12486 + elif is_error_type(authorization_error, AuthorizationErrorEnum.AuthorizationError.CUSTOMER_NOT_ENABLED): + if catch_disabled_customer_error: + logger.error(error.message) + continue + else: + message = ( + f"The customer account '{customer_id}' hasn't finished signup or has been deactivated. " + "Sign in to the Google Ads UI to verify its status. " + "For reactivating deactivated accounts, refer to: " + "https://support.google.com/google-ads/answer/2375392." + ) + + elif query_error: + message = f"Incorrect custom query. {error.message}" + + elif is_error_type(quota_error, QuotaErrorEnum.QuotaError.RESOURCE_EXHAUSTED): + message = ( + f"The operation limits for your Google Ads account '{customer_id}' have been exceeded for the last 24 hours. " + f"To avoid these limitations, consider applying for Standard access which offers unlimited operations per day. " + f"Learn more about access levels and how to apply for Standard access here: " + f"https://developers.google.com/google-ads/api/docs/access-levels#access_levels_2" + ) + + # This error occurs when the page token expires while processing results, it is partially handled in IncrementalGoogleAdsStream + elif is_error_type(request_error, RequestErrorEnum.RequestError.EXPIRED_PAGE_TOKEN): + message = ( + "Page token has expired during processing response. " + "Please contact the Airbyte team with the link of your connection for assistance." + ) + + # Raise new error for easier catch in child class - this error will be handled in IncrementalGoogleAdsStream + raise_exception = ExpiredPageTokenError + failure_type = FailureType.system_error + + else: + message = str(error.message) + failure_type = FailureType.system_error + + if message: + messages.append(message) + + if messages: + message = "\n".join(messages) + raise raise_exception.from_exception(failure_type=failure_type, exc=ga_exception, message=message) from ga_exception + + +def generator_backoff( + wait_gen: Callable, + exception: Union[Type[Exception], tuple], + max_tries: Optional[int] = None, + max_time: Optional[float] = None, + on_backoff: Optional[Callable] = None, + **wait_gen_kwargs: Any, +): + def decorator(func: Callable) -> Callable: + def wrapper(*args, **kwargs) -> Generator: + tries = 0 + start_time = datetime.now() + wait_times = wait_gen(**wait_gen_kwargs) + next(wait_times) # Skip the first yield which is None + + while True: + try: + yield from func(*args, **kwargs) + return # If the generator completes without error, return + except exception as e: + tries += 1 + elapsed_time = (datetime.now() - start_time).total_seconds() + + if max_time is not None and elapsed_time >= max_time: + print(f"Maximum time of {max_time} seconds exceeded.") + raise + + if max_tries is not None and tries >= max_tries: + print(f"Maximum tries of {max_tries} exceeded.") + raise + + # Get the next wait time from the exponential decay generator + sleep_time = next(wait_times) + + # Adjust sleep time if it exceeds the remaining max_time + if max_time is not None: + time_remaining = max_time - elapsed_time + sleep_time = min(sleep_time, time_remaining) + + if on_backoff: + on_backoff( + { + "target": func, + "args": args, + "kwargs": kwargs, + "tries": tries, + "elapsed": elapsed_time, + "wait": sleep_time, + "exception": e, + } + ) + + time.sleep(sleep_time) + + return wrapper + + return decorator + + +class RunAsThread: + """ + The `RunAsThread` decorator is designed to run a generator function in a separate thread with a specified timeout. + This is particularly useful when dealing with functions that involve potentially time-consuming operations, + and you want to enforce a time limit for their execution. + """ + + def __init__(self, timeout_minutes): + """ + :param timeout_minutes: The maximum allowed time (in minutes) for the generator function to idle. + If the timeout is reached, a TimeoutError is raised. + """ + self._timeout_seconds = timeout_minutes * 60 + + def __call__(self, generator_func): + @functools.wraps(generator_func) + def wrapper(*args, **kwargs): + """ + The wrapper function sets up threading components, starts a separate thread to run the generator function. + It uses events and a queue for communication and synchronization between the main thread and the thread running the generator function. + """ + # Event and Queue initialization + write_event = threading.Event() + exit_event = threading.Event() + the_queue = queue.Queue() + + # Thread initialization and start + thread = threading.Thread( + target=self.target, args=(the_queue, write_event, exit_event, generator_func, args, kwargs), daemon=True + ) + thread.start() + + # Records the starting time for the timeout calculation. + start_time = time.time() + while thread.is_alive() or not the_queue.empty(): + # The main thread waits for the `write_event` to be set or until the specified timeout. + if the_queue.empty(): + write_event.wait(self._timeout_seconds) + try: + # The main thread yields the result obtained from reading the queue. + yield self.read(the_queue) + # The timer is reset since a new result has been received, preventing the timeout from occurring. + start_time = time.time() + except queue.Empty: + # If exit_event is set it means that the generator function in the thread has completed its execution. + if exit_event.is_set(): + break + # Check if the timeout has been reached without new results. + if time.time() - start_time > self._timeout_seconds: + # The thread may continue to run for some time after reaching a timeout and even come to life and continue working. + # That is why the exit event is set to signal the generator function to stop producing data. + exit_event.set() + raise TimeoutError(f"Method '{generator_func.__name__}' timed out after {self._timeout_seconds / 60.0} minutes") + # The write event is cleared to reset it for the next iteration. + write_event.clear() + + return wrapper + + def target(self, the_queue, write_event, exit_event, func, args, kwargs): + """ + This is a target function for the thread. + It runs the actual generator function, writing its results to a queue. + Exceptions raised during execution are also written to the queue. + :param the_queue: A queue used for communication between the main thread and the thread running the generator function. + :param write_event: An event signaling the availability of new data in the queue. + :param exit_event: An event indicating whether the generator function should stop producing data due to a timeout. + :param func: The generator function to be executed. + :param args: Positional arguments for the generator function. + :param kwargs: Keyword arguments for the generator function. + :return: None + """ + try: + for value in func(*args, **kwargs): + # If the timeout has been reached we must stop producing any data + if exit_event.is_set(): + break + self.write(the_queue, value, write_event) + else: + # Notify the main thread that the generator function has completed its execution. + exit_event.set() + # Notify the main thread (even if the generator didn't produce any data) to prevent waiting for no reason. + if not write_event.is_set(): + write_event.set() + except Exception as e: + self.write(the_queue, e, write_event) + + @staticmethod + def write(the_queue, value, write_event): + """ + Puts a value into the queue and sets a write event to notify the main thread that new data is available. + :param the_queue: A queue used for communication between the main thread and the thread running the generator function. + :param value: The value to be put into the communication queue. + This can be any type of data produced by the generator function, including results or exceptions. + :param write_event: An event signaling the availability of new data in the queue. + :return: None + """ + the_queue.put(value) + write_event.set() + + @staticmethod + def read(the_queue, timeout=0.001): + """ + Retrieves a value from the queue, handling the case where the value is an exception, and raising it. + :param the_queue: A queue used for communication between the main thread and the thread running the generator function. + :param timeout: A time in seconds to wait for a value to be available in the queue. + If the timeout is reached and no new data is available, a `queue.Empty` exception is raised. + :return: a value retrieved from the queue + """ + value = the_queue.get(block=True, timeout=timeout) + if isinstance(value, Exception): + raise value + return value + + +detached = RunAsThread + + +def parse_dates(stream_slice): + start_date = pendulum.parse(stream_slice["start_date"]) + end_date = pendulum.parse(stream_slice["end_date"]) + return start_date, end_date + + +def chunk_date_range( + start_date: str, + end_date: str = None, + conversion_window: int = 0, + days_of_data_storage: int = None, + time_zone=None, + time_format="YYYY-MM-DD", + slice_duration: pendulum.Duration = pendulum.duration(days=14), + slice_step: pendulum.Duration = pendulum.duration(days=1), +) -> Iterable[Optional[MutableMapping[str, any]]]: + """ + Splits a date range into smaller chunks based on the provided parameters. + + Args: + start_date (str): The beginning date of the range. + end_date (str, optional): The ending date of the range. Defaults to today's date. + conversion_window (int): Number of days to subtract from the start date. Defaults to 0. + days_of_data_storage (int, optional): Maximum age of data that can be retrieved. Used to adjust the start date. + time_zone: Time zone to be used for date parsing and today's date calculation. If not provided, the default time zone is used. + time_format (str): Format to be used when returning dates. Defaults to 'YYYY-MM-DD'. + slice_duration (pendulum.Duration): Duration of each chunk. Defaults to 14 days. + slice_step (pendulum.Duration): Step size to move to the next chunk. Defaults to 1 day. + + Returns: + Iterable[Optional[MutableMapping[str, any]]]: An iterable of dictionaries containing start and end dates for each chunk. + If the adjusted start date is greater than the end date, returns a list with a None value. + + Notes: + - If the difference between `end_date` and `start_date` is large (e.g., >= 1 month), processing all records might take a long time. + - Tokens for fetching subsequent pages of data might expire after 2 hours, leading to potential errors. + - The function adjusts the start date based on `days_of_data_storage` and `conversion_window` to adhere to certain data retrieval policies, such as Google Ads' policy of only retrieving data not older than a certain number of days. + - The method returns `start_date` and `end_date` with a difference typically spanning 15 days to avoid token expiration issues. + """ + start_date = pendulum.parse(start_date, tz=time_zone) + today = pendulum.today(tz=time_zone) + end_date = pendulum.parse(end_date, tz=time_zone) if end_date else today + + # For some metrics we can only get data not older than N days, it is Google Ads policy + if days_of_data_storage: + start_date = max(start_date, pendulum.now(tz=time_zone).subtract(days=days_of_data_storage - conversion_window)) + + # As in to return some state when state in abnormal + if start_date > end_date: + return [None] + + # applying conversion window + start_date = start_date.subtract(days=conversion_window) + slice_start = start_date + + while slice_start <= end_date: + slice_end = min(end_date, slice_start + slice_duration) + yield { + "start_date": slice_start.format(time_format), + "end_date": slice_end.format(time_format), + } + slice_start = slice_end + slice_step @dataclass(repr=False, eq=False, frozen=True) diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/common.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/common.py index 426f3995dc85..b2bff404d6e2 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/common.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/common.py @@ -2,10 +2,15 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import json from google.ads.googleads.errors import GoogleAdsException -from google.ads.googleads.v11 import GoogleAdsFailure +from google.ads.googleads.v15 import GoogleAdsFailure +from google.ads.googleads.v15.errors.types.authentication_error import AuthenticationErrorEnum +from google.ads.googleads.v15.errors.types.authorization_error import AuthorizationErrorEnum +from google.ads.googleads.v15.errors.types.query_error import QueryErrorEnum +from google.ads.googleads.v15.errors.types.quota_error import QuotaErrorEnum class MockSearchRequest: @@ -13,6 +18,7 @@ class MockSearchRequest: query = None page_size = 100 page_token = None + next_page_token = None # Mocking Classes @@ -30,44 +36,85 @@ def get_type(self, type): return MockSearchRequest() def get_service(self, service): + if service == "GoogleAdsFieldService": + return MockGoogleAdsFieldService() return MockGoogleAdsService() @staticmethod - def load_from_dict(config): + def load_from_dict(config, version=None): return MockGoogleAdsClient(config) - def send_request(self, query, customer_id): + def send_request(self, query, customer_id, login_customer_id="none"): yield from () - -class MockErroringGoogleAdsService: - def search(self, search_request): - raise make_google_ads_exception(1) - - -def make_google_ads_exception(failure_code: int = 1, failure_msg: str = "it failed", error_type: str = "request_error"): - # There is no easy way I could find to mock a GoogleAdsException without doing something heinous like this - # Following the definition of the object here - # https://developers.google.com/google-ads/api/reference/rpc/v10/GoogleAdsFailure - protobuf_as_json = json.dumps({"errors": [{"error_code": {error_type: failure_code}, "message": failure_msg}], "request_id": "1"}) - failure = type(GoogleAdsFailure).from_json(GoogleAdsFailure, protobuf_as_json) - return GoogleAdsException(None, None, failure, 1) - - -class MockErroringGoogleAdsClient: - def __init__(self, credentials, **kwargs): - self.config = credentials - self.customer_ids = ["1"] - - def send_request(self, query, customer_id): - raise make_google_ads_exception(1) - - def get_type(self, type): - return MockSearchRequest() - - def get_service(self, service): - return MockErroringGoogleAdsService() - - @staticmethod - def load_from_dict(config): - return MockErroringGoogleAdsClient(config) + def get_accessible_accounts(self): + yield from ["fake_customer_id", "fake_customer_id_2"] + + +class MockGoogleAdsFieldService: + _instance = None + + def __new__(cls): + if cls._instance is None: + cls._instance = super(MockGoogleAdsFieldService, cls).__new__(cls) + cls._instance.request_query = None + return cls._instance + + def search_google_ads_fields(self, request): + self.request_query = request.query + + class MockResponse: + def __init__(self, name): + self.name = name + + fields = [name.strip("'") for name in request.query.split("WHERE name in (")[1].split(")")[0].split(",")] + return [MockResponse(name) for name in fields] + + +ERROR_MAP = { + "CUSTOMER_NOT_FOUND": { + "failure_code": AuthenticationErrorEnum.AuthenticationError.CUSTOMER_NOT_FOUND, + "failure_msg": "msg2", + "error_type": "authenticationError", + }, + "USER_PERMISSION_DENIED": { + "failure_code": AuthorizationErrorEnum.AuthorizationError.USER_PERMISSION_DENIED, + "failure_msg": "msg1", + "error_type": "authorizationError", + }, + "CUSTOMER_NOT_ENABLED": { + "failure_code": AuthorizationErrorEnum.AuthorizationError.CUSTOMER_NOT_ENABLED, + "failure_msg": "msg2", + "error_type": "authorizationError", + }, + "QUERY_ERROR": { + "failure_code": QueryErrorEnum.QueryError.UNEXPECTED_END_OF_QUERY, + "failure_msg": "Error in query: unexpected end of query.", + "error_type": "queryError", + }, + "RESOURCE_EXHAUSTED": {"failure_code": QuotaErrorEnum.QuotaError.RESOURCE_EXHAUSTED, "failure_msg": "msg4", "error_type": "quotaError"}, + "UNEXPECTED_ERROR": { + "failure_code": AuthorizationErrorEnum.AuthorizationError.UNKNOWN, + "failure_msg": "Unexpected error message", + "error_type": "authorizationError", + }, +} + + +def mock_google_ads_request_failure(mocker, error_names): + errors = [] + for error_name in error_names: + param = ERROR_MAP[error_name] + # Extract the parameter values from the request object + failure_code = param.get("failure_code", 1) + failure_msg = param.get("failure_msg", "it failed") + error_type = param.get("error_type", "requestError") + + errors.append({"error_code": {error_type: failure_code}, "message": failure_msg}) + + protobuf_as_json = json.dumps({"errors": errors, "request_id": "1"}) + failure = GoogleAdsFailure.from_json(protobuf_as_json) + + exception = GoogleAdsException(None, None, failure, 1) + + mocker.patch("source_google_ads.google_ads.GoogleAds.send_request", side_effect=exception) diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py index bb4bece88aba..0ed82b8024ec 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/conftest.py @@ -2,8 +2,11 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + +from unittest.mock import Mock + import pytest -from source_google_ads.models import Customer +from source_google_ads.models import CustomerModel @pytest.fixture(name="config") @@ -18,7 +21,7 @@ def test_config(): "customer_id": "123", "start_date": "2021-01-01", "conversion_window_days": 14, - "custom_queries": [ + "custom_queries_array": [ { "query": "SELECT campaign.accessible_bidding_strategy, segments.ad_destination_type, campaign.start_date, campaign.end_date FROM campaign", "primary_key": None, @@ -52,4 +55,9 @@ def mock_oauth_call(requests_mock): @pytest.fixture def customers(config): - return [Customer(id=_id, time_zone="local", is_manager_account=False) for _id in config["customer_id"].split(",")] + return [CustomerModel(id=_id, time_zone="local", is_manager_account=False) for _id in config["customer_id"].split(",")] + + +@pytest.fixture +def customers_manager(config): + return [CustomerModel(id=_id, time_zone="local", is_manager_account=True) for _id in config["customer_id"].split(",")] diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_config_migrations.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_config_migrations.py new file mode 100644 index 000000000000..4ca91bc77892 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_config_migrations.py @@ -0,0 +1,79 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import json +from typing import Any, Mapping + +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source +from source_google_ads.config_migrations import MigrateCustomQuery +from source_google_ads.source import SourceGoogleAds + +# BASE ARGS +CMD = "check" +TEST_CONFIG_PATH = "unit_tests/test_migrations/custom_query/test_config.json" +NEW_TEST_CONFIG_PATH = "unit_tests/test_migrations/custom_query/test_new_config.json" +SOURCE_INPUT_ARGS = [CMD, "--config", TEST_CONFIG_PATH] +SOURCE: Source = SourceGoogleAds() + + +# HELPERS +def load_config(config_path: str = TEST_CONFIG_PATH) -> Mapping[str, Any]: + with open(config_path, "r") as config: + return json.load(config) + + +def revert_migration(config_path: str = TEST_CONFIG_PATH) -> None: + with open(config_path, "r") as test_config: + config = json.load(test_config) + config.pop("custom_queries_array") + with open(config_path, "w") as updated_config: + config = json.dumps(config) + updated_config.write(config) + + +def test_migrate_config(): + migration_instance = MigrateCustomQuery() + original_config = load_config() + original_config_queries = original_config["custom_queries"].copy() + # migrate the test_config + migration_instance.migrate(SOURCE_INPUT_ARGS, SOURCE) + # load the updated config + test_migrated_config = load_config() + # check migrated property + assert "custom_queries_array" in test_migrated_config + assert "segments.date" in test_migrated_config["custom_queries_array"][0]["query"] + # check the old property is in place + assert "custom_queries" in test_migrated_config + assert test_migrated_config["custom_queries"] == original_config_queries + assert "segments.date" not in test_migrated_config["custom_queries"][0]["query"] + # check the migration should be skipped, once already done + assert not migration_instance.should_migrate(test_migrated_config) + # load the old custom reports VS migrated + new_config_queries = test_migrated_config["custom_queries_array"].copy() + new_config_queries[0]["query"] = new_config_queries[0]["query"].replace(", segments.date", "") + print(f"{original_config=} \n {test_migrated_config=}") + assert original_config["custom_queries"] == new_config_queries + # test CONTROL MESSAGE was emitted + control_msg = migration_instance.message_repository._message_queue[0] + assert control_msg.type == Type.CONTROL + assert control_msg.control.type == OrchestratorType.CONNECTOR_CONFIG + # revert the test_config to the starting point + revert_migration() + + +def test_config_is_reverted(): + # check the test_config state, it has to be the same as before tests + test_config = load_config() + # check the config no longer has the migarted property + assert "custom_queries_array" not in test_config + # check the old property is still there + assert "custom_queries" in test_config + + +def test_should_not_migrate_new_config(): + new_config = load_config(NEW_TEST_CONFIG_PATH) + migration_instance = MigrateCustomQuery() + assert not migration_instance.should_migrate(new_config) diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_custom_query.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_custom_query.py index dc5c1f086a06..862324d3c237 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_custom_query.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_custom_query.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from unittest.mock import MagicMock from source_google_ads.custom_query_stream import CustomQueryMixin, IncrementalCustomQuery @@ -27,29 +28,33 @@ def __init__(self, **entries): def test_get_json_schema(): - query_object = MagicMock(return_value={ - 'a': Obj(data_type=Obj(name='ENUM'), is_repeated=False, enum_values=['a', 'aa']), - 'b': Obj(data_type=Obj(name='ENUM'), is_repeated=True, enum_values=['b', 'bb']), - 'c': Obj(data_type=Obj(name='MESSAGE'), is_repeated=False), - 'd': Obj(data_type=Obj(name='MESSAGE'), is_repeated=True), - 'e': Obj(data_type=Obj(name='STRING')), - 'f': Obj(data_type=Obj(name='DATE')), - }) - instance = CustomQueryMixin(config={'query': Obj(fields=['a', 'b', 'c', 'd', 'e', 'f'])}) + query_object = MagicMock( + return_value={ + "a": Obj(data_type=Obj(name="ENUM"), is_repeated=False, enum_values=["a", "aa"]), + "b": Obj(data_type=Obj(name="ENUM"), is_repeated=True, enum_values=["b", "bb"]), + "c": Obj(data_type=Obj(name="MESSAGE"), is_repeated=False), + "d": Obj(data_type=Obj(name="MESSAGE"), is_repeated=True), + "e": Obj(data_type=Obj(name="STRING"), is_repeated=False), + "f": Obj(data_type=Obj(name="DATE"), is_repeated=False), + "segments.month": Obj(data_type=Obj(name="DATE"), is_repeated=False), + } + ) + instance = CustomQueryMixin(config={"query": Obj(fields=["a", "b", "c", "d", "e", "f", "segments.month"])}) instance.cursor_field = None instance.google_ads_client = Obj(get_fields_metadata=query_object) schema = instance.get_json_schema() assert schema == { - '$schema': 'http://json-schema.org/draft-07/schema#', - 'additionalProperties': True, - 'type': 'object', - 'properties': { - 'a': {'type': 'string', 'enum': ['a', 'aa']}, - 'b': {'type': ['null', 'array'], 'items': {'type': 'string', 'enum': ['b', 'bb']}}, - 'c': {'type': ['string', 'null'], 'protobuf_message': True}, - 'd': {'type': ['array', 'null'], 'protobuf_message': True}, - 'e': {'type': ['string', 'null']}, - 'f': {'type': ['string', 'null'], 'format': 'date'}, - } + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": True, + "type": "object", + "properties": { + "a": {"type": "string", "enum": ["a", "aa"]}, + "b": {"type": ["null", "array"], "items": {"type": "string", "enum": ["b", "bb"]}}, + "c": {"type": ["string", "null"]}, + "d": {"type": ["null", "array"], "items": {"type": ["string", "null"]}}, + "e": {"type": ["string", "null"]}, + "f": {"type": ["string", "null"]}, + "segments.month": {"type": ["string", "null"], "format": "date"}, + }, } diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_errors.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_errors.py new file mode 100644 index 000000000000..e71263296007 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_errors.py @@ -0,0 +1,156 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from contextlib import nullcontext as does_not_raise +from unittest.mock import Mock + +import pytest +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.utils import AirbyteTracedException +from source_google_ads.google_ads import GoogleAds +from source_google_ads.models import CustomerModel +from source_google_ads.source import SourceGoogleAds +from source_google_ads.streams import AdGroupLabel, Label, ServiceAccounts + +from .common import MockGoogleAdsClient, mock_google_ads_request_failure + + +@pytest.fixture +def mock_get_customers(mocker): + mocker.patch( + "source_google_ads.source.SourceGoogleAds.get_customers", + Mock(return_value=[CustomerModel(is_manager_account=False, time_zone="Europe/Berlin", id="123")]), + ) + + +params = [ + ( + ["USER_PERMISSION_DENIED"], + "Failed to access the customer '123'. Ensure the customer is linked to your manager account or check your permissions to access this customer account.", + ), + ( + ["CUSTOMER_NOT_FOUND"], + "Failed to access the customer '123'. Ensure the customer is linked to your manager account or check your permissions to access this customer account.", + ), + (["QUERY_ERROR"], "Incorrect custom query. Error in query: unexpected end of query."), + ( + ["RESOURCE_EXHAUSTED"], + ( + "The operation limits for your Google Ads account '123' have been exceeded for the last 24 hours. " + "To avoid these limitations, consider applying for Standard access which offers unlimited operations per day. " + "Learn more about access levels and how to apply for Standard access here: " + "https://developers.google.com/google-ads/api/docs/access-levels#access_levels_2" + ), + ), + (["UNEXPECTED_ERROR"], "Unexpected error message"), + (["QUERY_ERROR", "UNEXPECTED_ERROR"], "Incorrect custom query. Error in query: unexpected end of query.\nUnexpected error message"), +] + + +@pytest.mark.parametrize(("exception", "error_message"), params) +def test_expected_errors(mocker, config, exception, error_message): + mock_google_ads_request_failure(mocker, exception) + mocker.patch( + "source_google_ads.google_ads.GoogleAds.get_accessible_accounts", + Mock(return_value=["123", "12345"]), + ) + source = SourceGoogleAds() + with pytest.raises(AirbyteTracedException) as exception: + status_ok, error = source.check_connection(AirbyteLogger(), config) + assert exception.value.message == error_message + + +@pytest.mark.parametrize( + ("cls", "raise_expected"), + ( + (AdGroupLabel, False), + (Label, False), + (ServiceAccounts, True), + ), +) +def test_read_record_error_handling(mocker, config, customers, cls, raise_expected): + mock_google_ads_request_failure(mocker, ["CUSTOMER_NOT_ENABLED"]) + google_api = GoogleAds(credentials=config["credentials"]) + stream = cls(api=google_api, customers=customers) + + # Use nullcontext or pytest.raises based on raise_expected + context = pytest.raises(AirbyteTracedException) if raise_expected else does_not_raise() + + with context as exception: + for _ in stream.read_records(sync_mode=Mock(), stream_slice={"customer_id": "1234567890", "login_customer_id": "default"}): + pass + + if raise_expected: + assert exception.value.message == ( + "The customer account '1234567890' hasn't finished signup or has been deactivated. " + "Sign in to the Google Ads UI to verify its status. " + "For reactivating deactivated accounts, refer to: " + "https://support.google.com/google-ads/answer/2375392." + ) + + +@pytest.mark.parametrize( + "custom_query, is_manager_account, error_message, warning", + [ + ( + { + "query": "SELECT campaign.accessible_bidding_strategy, metrics.clicks from campaigns", + "primary_key": None, + "cursor_field": "None", + "table_name": "happytable", + }, + True, + None, + ( + "Metrics are not available for manager account 8765. " + 'Skipping the custom query: "SELECT campaign.accessible_bidding_strategy, ' + 'metrics.clicks FROM campaigns" for manager account.' + ), + ), + ( + { + "query": "SELECT campaign.accessible_bidding_strategy, metrics.clicks from campaigns", + "primary_key": None, + "cursor_field": None, + "table_name": "happytable", + }, + False, + None, + None, + ), + ( + { + "query": "SELECT segments.ad_destination_type, segments.date from campaigns", + "primary_key": "customer.id", + "cursor_field": None, + "table_name": "unhappytable", + }, + False, + None, + None, + ), + ], +) +def test_check_custom_queries(mocker, config, custom_query, is_manager_account, error_message, warning): + config["custom_queries_array"] = [custom_query] + mocker.patch( + "source_google_ads.source.SourceGoogleAds.get_customers", + Mock(return_value=[CustomerModel(is_manager_account=is_manager_account, time_zone="Europe/Berlin", id="8765")]), + ) + mocker.patch("source_google_ads.google_ads.GoogleAdsClient", return_value=MockGoogleAdsClient) + source = SourceGoogleAds() + logger_mock = Mock() + + # Use nullcontext or pytest.raises based on error_message + context = pytest.raises(AirbyteTracedException) if error_message else does_not_raise() + + with context as exception: + status_ok, error = source.check_connection(logger_mock, config) + + if error_message: + assert exception.value.message == error_message + + if warning: + logger_mock.warning.assert_called_with(warning) diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py index cdd5285a051d..3f66564846f4 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py @@ -2,8 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from datetime import date +import pendulum import pytest from airbyte_cdk.utils import AirbyteTracedException from google.auth import exceptions @@ -92,7 +94,11 @@ def test_interval_chunking(): {"start_date": "2021-07-27", "end_date": "2021-08-05"}, {"start_date": "2021-08-06", "end_date": "2021-08-10"}, ] - intervals = list(chunk_date_range("2021-07-01", 14, "2021-08-10", range_days=10, time_zone="UTC")) + intervals = list( + chunk_date_range( + start_date="2021-07-01", end_date="2021-08-10", conversion_window=14, slice_duration=pendulum.Duration(days=9), time_zone="UTC" + ) + ) assert mock_intervals == intervals @@ -100,60 +106,39 @@ def test_interval_chunking(): @pytest.mark.parametrize( - "stream_schema, report_name, slice_start, slice_end, cursor, expected_sql", + "fields, table_name, conditions, order_field, limit, expected_sql", ( + # Basic test case ( - generic_schema, - "ad_group_ads", - "2020-01-01", - "2020-01-10", + ["ad_group_id", "segments.date", "campaign_id", "account_id"], + "ad_group_ad", + ["segments.date >= '2020-01-01'", "segments.date <= '2020-01-10'"], "segments.date", - "SELECT ad_group_id, segments.date, campaign_id, account_id FROM ad_group_ad WHERE segments.date >= '2020-01-01' AND segments.date <= '2020-01-10' ORDER BY segments.date ASC" - ), - ( - generic_schema, - "ad_group_ads", - "2020-01-01", - "2020-01-02", - "segments.date", - "SELECT ad_group_id, segments.date, campaign_id, account_id FROM ad_group_ad WHERE segments.date >= '2020-01-01' AND segments.date <= '2020-01-02' ORDER BY segments.date ASC" + None, + "SELECT ad_group_id, segments.date, campaign_id, account_id FROM ad_group_ad WHERE segments.date >= '2020-01-01' AND segments.date <= '2020-01-10' ORDER BY segments.date ASC", ), + # Test with no conditions ( - generic_schema, - "ad_group_ads", + ["ad_group_id", "segments.date", "campaign_id", "account_id"], + "ad_group_ad", None, None, None, - "SELECT ad_group_id, segments.date, campaign_id, account_id FROM ad_group_ad" - ), - ( - generic_schema, - "click_view", - "2020-01-01", - "2020-01-10", - "segments.date", - "SELECT ad_group_id, segments.date, campaign_id, account_id FROM click_view WHERE segments.date >= '2020-01-01' AND segments.date <= '2020-01-10' ORDER BY segments.date ASC" - ), - ( - generic_schema, - "click_view", - "2020-01-01", - "2020-01-02", - "segments.date", - "SELECT ad_group_id, segments.date, campaign_id, account_id FROM click_view WHERE segments.date >= '2020-01-01' AND segments.date <= '2020-01-02' ORDER BY segments.date ASC" + "SELECT ad_group_id, segments.date, campaign_id, account_id FROM ad_group_ad", ), + # Test order with limit ( - generic_schema, + ["ad_group_id", "segments.date", "campaign_id", "account_id"], "click_view", None, - None, - None, - "SELECT ad_group_id, segments.date, campaign_id, account_id FROM click_view" + "ad_group_id", + 5, + "SELECT ad_group_id, segments.date, campaign_id, account_id FROM click_view ORDER BY ad_group_id ASC LIMIT 5", ), ), ) -def test_convert_schema_into_query(stream_schema, report_name, slice_start, slice_end, cursor, expected_sql): - query = GoogleAds.convert_schema_into_query(stream_schema, report_name, slice_start, slice_end, cursor) +def test_convert_schema_into_query(fields, table_name, conditions, order_field, limit, expected_sql): + query = GoogleAds.convert_schema_into_query(fields, table_name, conditions, order_field, limit) assert query == expected_sql @@ -168,3 +153,36 @@ def test_parse_single_result(): date = "2001-01-01" response = GoogleAds.parse_single_result(SAMPLE_SCHEMA, MockedDateSegment(date)) assert response == response + + +def test_get_fields_metadata(mocker): + # Mock the GoogleAdsClient to return our mock client + mocker.patch("source_google_ads.google_ads.GoogleAdsClient", MockGoogleAdsClient) + + # Instantiate the GoogleAds client + google_ads_client = GoogleAds(**SAMPLE_CONFIG) + + # Define the fields we want metadata for + fields = ["field1", "field2", "field3"] + + # Call the method to get fields metadata + response = google_ads_client.get_fields_metadata(fields) + + # Get the mock service to check the request query + mock_service = google_ads_client.get_client().get_service("GoogleAdsFieldService") + + # Assert the constructed request query + expected_query = """ + SELECT + name, + data_type, + enum_values, + is_repeated + WHERE name in ('field1','field2','field3') + """ + assert mock_service.request_query.strip() == expected_query.strip() + + # Assert the response + assert set(response.keys()) == set(fields) + for field in fields: + assert response[field].name == field diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_incremental_events_streams.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_incremental_events_streams.py new file mode 100644 index 000000000000..8ddf8bd80fba --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_incremental_events_streams.py @@ -0,0 +1,608 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from copy import deepcopy +from unittest.mock import DEFAULT, MagicMock, Mock, call + +import pendulum +import pytest +from airbyte_cdk.models import SyncMode +from airbyte_cdk.utils import AirbyteTracedException +from source_google_ads.google_ads import GoogleAds +from source_google_ads.streams import CampaignCriterion, ChangeStatus + + +def mock_response_parent(): + yield [ + { + "change_status.last_change_date_time": "2023-06-13 12:36:01.772447", + "change_status.resource_type": "CAMPAIGN_CRITERION", + "change_status.resource_status": "ADDED", + "change_status.campaign_criterion": "1", + }, + { + "change_status.last_change_date_time": "2023-06-13 12:36:02.772447", + "change_status.resource_type": "CAMPAIGN_CRITERION", + "change_status.resource_status": "ADDED", + "change_status.campaign_criterion": "2", + }, + { + "change_status.last_change_date_time": "2023-06-13 12:36:03.772447", + "change_status.resource_type": "CAMPAIGN_CRITERION", + "change_status.resource_status": "REMOVED", + "change_status.campaign_criterion": "3", + }, + { + "change_status.last_change_date_time": "2023-06-13 12:36:04.772447", + "change_status.resource_type": "CAMPAIGN_CRITERION", + "change_status.resource_status": "REMOVED", + "change_status.campaign_criterion": "4", + }, + ] + + +def mock_response_child(): + yield [ + {"customer.id": 123, "campaign.id": 1, "campaign_criterion.resource_name": "1"}, + {"customer.id": 123, "campaign.id": 1, "campaign_criterion.resource_name": "2"}, + ] + + +class MockGoogleAds(GoogleAds): + def parse_single_result(self, schema, result): + return result + + def send_request(self, query: str, customer_id: str, login_customer_id: str = "default"): + if query == "query_parent": + return mock_response_parent() + else: + return mock_response_child() + + +def test_change_status_stream(config, customers): + """ """ + customer_id = next(iter(customers)).id + stream_slice = {"customer_id": customer_id, "login_customer_id": "default"} + + google_api = MockGoogleAds(credentials=config["credentials"]) + + stream = ChangeStatus(api=google_api, customers=customers) + + stream.get_query = Mock() + stream.get_query.return_value = "query_parent" + + result = list( + stream.read_records(sync_mode=SyncMode.incremental, cursor_field=["change_status.last_change_date_time"], stream_slice=stream_slice) + ) + assert len(result) == 4 + assert stream.get_query.call_count == 1 + stream.get_query.assert_called_with({"customer_id": customer_id, "login_customer_id": "default"}) + + +def test_child_incremental_events_read(config, customers): + """ + Page token expired while reading records on date 2021-01-03 + The latest read record is {"segments.date": "2021-01-03", "click_view.gclid": "4"} + It should retry reading starting from 2021-01-03, already read records will be reread again from that date. + It shouldn't read records on 2021-01-01, 2021-01-02 + """ + customer_id = next(iter(customers)).id + parent_stream_slice = {"customer_id": customer_id, "resource_type": "CAMPAIGN_CRITERION", "login_customer_id": "default"} + stream_state = {"change_status": {customer_id: {"change_status.last_change_date_time": "2023-08-16 13:20:01.003295"}}} + + google_api = MockGoogleAds(credentials=config["credentials"]) + + stream = CampaignCriterion(api=google_api, customers=customers) + parent_stream = stream.parent_stream + + parent_stream.get_query = Mock() + parent_stream.get_query.return_value = "query_parent" + + parent_stream.stream_slices = Mock() + parent_stream.stream_slices.return_value = [parent_stream_slice] + + parent_stream.state = {customer_id: {"change_status.last_change_date_time": "2023-05-16 13:20:01.003295"}} + + stream.get_query = Mock() + stream.get_query.return_value = "query_child" + + stream_slices = list(stream.stream_slices(stream_state=stream_state)) + + assert stream_slices == [ + { + "customer_id": "123", + "updated_ids": {"2", "1"}, + "deleted_ids": {"3", "4"}, + "record_changed_time_map": { + "1": "2023-06-13 12:36:01.772447", + "2": "2023-06-13 12:36:02.772447", + "3": "2023-06-13 12:36:03.772447", + "4": "2023-06-13 12:36:04.772447", + }, + "login_customer_id": "default", + } + ] + + result = list( + stream.read_records( + sync_mode=SyncMode.incremental, cursor_field=["change_status.last_change_date_time"], stream_slice=stream_slices[0] + ) + ) + expected_result = [ + { + "campaign.id": 1, + "campaign_criterion.resource_name": "1", + "change_status.last_change_date_time": "2023-06-13 12:36:01.772447", + "customer.id": 123, + }, + { + "campaign.id": 1, + "campaign_criterion.resource_name": "2", + "change_status.last_change_date_time": "2023-06-13 12:36:02.772447", + "customer.id": 123, + }, + {"campaign_criterion.resource_name": "3", "deleted_at": "2023-06-13 12:36:03.772447"}, + {"campaign_criterion.resource_name": "4", "deleted_at": "2023-06-13 12:36:04.772447"}, + ] + + assert all([expected_row in result for expected_row in expected_result]) + + assert stream.state == {"change_status": {"123": {"change_status.last_change_date_time": "2023-06-13 12:36:04.772447"}}} + + assert stream.get_query.call_count == 1 + + +def mock_response_1(): + yield [ + { + "change_status.last_change_date_time": "2023-06-13 12:36:01.772447", + "change_status.resource_type": "CAMPAIGN_CRITERION", + "change_status.resource_status": "ADDED", + "change_status.campaign_criterion": "1", + }, + { + "change_status.last_change_date_time": "2023-06-13 12:36:02.772447", + "change_status.resource_type": "CAMPAIGN_CRITERION", + "change_status.resource_status": "ADDED", + "change_status.campaign_criterion": "2", + }, + ] + + +def mock_response_2(): + yield [ + { + "change_status.last_change_date_time": "2023-06-13 12:36:03.772447", + "change_status.resource_type": "CAMPAIGN_CRITERION", + "change_status.resource_status": "REMOVED", + "change_status.campaign_criterion": "3", + }, + { + "change_status.last_change_date_time": "2023-06-13 12:36:04.772447", + "change_status.resource_type": "CAMPAIGN_CRITERION", + "change_status.resource_status": "REMOVED", + "change_status.campaign_criterion": "4", + }, + ] + + +def mock_response_3(): + yield [ + { + "change_status.last_change_date_time": "2023-06-13 12:36:04.772447", + "change_status.resource_type": "CAMPAIGN_CRITERION", + "change_status.resource_status": "REMOVED", + "change_status.campaign_criterion": "6", + }, + ] + + +def mock_response_4(): + yield [ + { + "change_status.last_change_date_time": "2023-06-13 12:36:04.772447", + "change_status.resource_type": "CAMPAIGN_CRITERION", + "change_status.resource_status": "REMOVED", + "change_status.campaign_criterion": "6", + }, + { + "change_status.last_change_date_time": "2023-06-13 12:36:04.772447", + "change_status.resource_type": "CAMPAIGN_CRITERION", + "change_status.resource_status": "REMOVED", + "change_status.campaign_criterion": "7", + }, + ] + + +class MockGoogleAdsLimit(GoogleAds): + count = 0 + + def parse_single_result(self, schema, result): + return result + + def send_request(self, query: str, customer_id: str, login_customer_id: str = "default"): + self.count += 1 + if self.count == 1: + return mock_response_1() + elif self.count == 2: + return mock_response_2() + else: + return mock_response_3() + + +def mock_query_limit(self) -> int: + return 2 # or whatever value you want for testing + + +def copy_call_args(mock): + new_mock = Mock() + + def side_effect(*args, **kwargs): + args = deepcopy(args) + kwargs = deepcopy(kwargs) + new_mock(*args, **kwargs) + return DEFAULT + + mock.side_effect = side_effect + return new_mock + + +def test_query_limit_hit(config, customers): + """ + Test the behavior of the `read_records` method in the `ChangeStatus` stream when the query limit is hit. + + This test simulates a scenario where the limit is hit and slice start_date is updated with latest record cursor + """ + customer_id = next(iter(customers)).id + stream_slice = { + "customer_id": customer_id, + "start_date": "2023-06-13 11:35:04.772447", + "end_date": "2023-06-13 13:36:04.772447", + "login_customer_id": "default", + } + + google_api = MockGoogleAdsLimit(credentials=config["credentials"]) + stream_config = dict( + api=google_api, + customers=customers, + ) + stream = ChangeStatus(**stream_config) + ChangeStatus.query_limit = property(mock_query_limit) + stream.get_query = Mock(return_value="query") + get_query_mock = copy_call_args(stream.get_query) + + result = list( + stream.read_records(sync_mode=SyncMode.incremental, cursor_field=["change_status.last_change_date_time"], stream_slice=stream_slice) + ) + + assert len(result) == 5 + assert stream.get_query.call_count == 3 + + get_query_calls = [ + call( + { + "customer_id": "123", + "start_date": "2023-06-13 11:35:04.772447", + "end_date": "2023-06-13 13:36:04.772447", + "login_customer_id": "default", + } + ), + call( + { + "customer_id": "123", + "start_date": "2023-06-13 12:36:02.772447", + "end_date": "2023-06-13 13:36:04.772447", + "login_customer_id": "default", + } + ), + call( + { + "customer_id": "123", + "start_date": "2023-06-13 12:36:04.772447", + "end_date": "2023-06-13 13:36:04.772447", + "login_customer_id": "default", + } + ), + ] + + get_query_mock.assert_has_calls(get_query_calls) + + +class MockGoogleAdsLimitException(MockGoogleAdsLimit): + def send_request(self, query: str, customer_id: str, login_customer_id: str = "default"): + self.count += 1 + if self.count == 1: + return mock_response_1() + elif self.count == 2: + return mock_response_2() + elif self.count == 3: + return mock_response_4() + + +def test_query_limit_hit_exception(config, customers): + """ + Test the behavior of the `read_records` method in the `ChangeStatus` stream when the query limit is hit. + + This test simulates a scenario where the limit is hit and there are more than query_limit number of records with same cursor, + then error will be raised + """ + customer_id = next(iter(customers)).id + stream_slice = { + "customer_id": customer_id, + "start_date": "2023-06-13 11:35:04.772447", + "end_date": "2023-06-13 13:36:04.772447", + "login_customer_id": "default", + } + + google_api = MockGoogleAdsLimitException(credentials=config["credentials"]) + stream_config = dict( + api=google_api, + customers=customers, + ) + stream = ChangeStatus(**stream_config) + ChangeStatus.query_limit = property(mock_query_limit) + stream.get_query = Mock(return_value="query") + + with pytest.raises(AirbyteTracedException) as e: + list( + stream.read_records( + sync_mode=SyncMode.incremental, cursor_field=["change_status.last_change_date_time"], stream_slice=stream_slice + ) + ) + + expected_message = "More than limit 2 records with same cursor field. Incremental sync is not possible for this stream." + assert e.value.message == expected_message + + +def test_change_status_get_query(mocker, config, customers): + """ + Test the get_query method of ChangeStatus stream. + + Given a sample stream_slice, it verifies that the returned query is as expected. + """ + # Setup an instance of the ChangeStatus stream + google_api = MockGoogleAds(credentials=config["credentials"]) + stream = ChangeStatus(api=google_api, customers=customers) + + # Mock get_json_schema method of the stream to return a predefined schema + mocker.patch.object(stream, "get_json_schema", return_value={"properties": {"change_status.resource_type": {"type": "str"}}}) + + # Define a sample stream_slice for the test + stream_slice = { + "start_date": "2023-01-01 00:00:00.000000", + "end_date": "2023-09-19 00:00:00.000000", + "resource_type": "SOME_RESOURCE_TYPE", + "login_customer_id": "default", + } + + # Call the get_query method with the stream_slice + query = stream.get_query(stream_slice=stream_slice) + + # Expected result based on the provided sample + expected_query = """SELECT change_status.resource_type FROM change_status WHERE change_status.last_change_date_time >= '2023-01-01 00:00:00.000000' AND change_status.last_change_date_time <= '2023-09-19 00:00:00.000000' AND change_status.resource_type = 'SOME_RESOURCE_TYPE' ORDER BY change_status.last_change_date_time ASC LIMIT 2""" + + # Check that the result from the get_query method matches the expected query + assert query == expected_query + + +def are_queries_equivalent(query1, query2): + # Split the queries to extract the list of criteria + criteria1 = query1.split("IN (")[1].rstrip(")").split(", ") + criteria2 = query2.split("IN (")[1].rstrip(")").split(", ") + + # Sort the criteria for comparison + criteria1_sorted = sorted(criteria1) + criteria2_sorted = sorted(criteria2) + + # Replace the original criteria with the sorted version in the queries + query1_sorted = query1.replace(", ".join(criteria1), ", ".join(criteria1_sorted)) + query2_sorted = query2.replace(", ".join(criteria2), ", ".join(criteria2_sorted)) + + return query1_sorted == query2_sorted + + +def test_incremental_events_stream_get_query(mocker, config, customers): + """ + Test the get_query method of the IncrementalEventsStream class. + + Given a sample stream_slice, this test will verify that the returned query string is as expected. + """ + # Setup an instance of the CampaignCriterion stream + google_api = MockGoogleAds(credentials=config["credentials"]) + stream = CampaignCriterion(api=google_api, customers=customers) + + # Mock get_json_schema method of the stream to return a predefined schema + mocker.patch.object(stream, "get_json_schema", return_value={"properties": {"campaign_criterion.resource_name": {"type": "str"}}}) + + # Define a sample stream_slice for the test + stream_slice = { + "customer_id": "1234567890", + "updated_ids": { + "customers/1234567890/adGroupCriteria/111111111111~1", + "customers/1234567890/adGroupCriteria/111111111111~2", + "customers/1234567890/adGroupCriteria/111111111111~3", + }, + "deleted_ids": { + "customers/1234567890/adGroupCriteria/111111111111~4", + "customers/1234567890/adGroupCriteria/111111111111~5", + }, + "record_changed_time_map": { + "customers/1234567890/adGroupCriteria/111111111111~1": "2023-09-18 08:56:53.413023", + "customers/1234567890/adGroupCriteria/111111111111~2": "2023-09-18 08:56:59.165599", + "customers/1234567890/adGroupCriteria/111111111111~3": "2023-09-18 08:56:59.165599", + "customers/1234567890/adGroupCriteria/111111111111~4": "2023-09-18 08:56:59.165599", + "customers/1234567890/adGroupCriteria/111111111111~5": "2023-09-18 08:56:59.165599", + }, + "login_customer_id": "default", + } + + # Call the get_query method with the stream_slice + query = stream.get_query(stream_slice=stream_slice) + + # Assuming the generated query should look like: + expected_query = ( + "SELECT campaign_criterion.resource_name " + "FROM campaign_criterion " + "WHERE campaign_criterion.resource_name IN (" + "'customers/1234567890/adGroupCriteria/111111111111~1', " + "'customers/1234567890/adGroupCriteria/111111111111~2', " + "'customers/1234567890/adGroupCriteria/111111111111~3')" + ) + + # Check if the query generated by the get_query method matches the expected query + assert are_queries_equivalent(query, expected_query) + + +def test_read_records_with_slice_splitting(mocker, config): + """ + Test the read_records method to ensure it correctly splits the stream_slice and calls the parent's read_records. + """ + # Define a stream_slice with 15,000 ids to ensure it gets split during processing + stream_slice = { + "updated_ids": set(range(15000)), + "record_changed_time_map": {i: f"time_{i}" for i in range(15000)}, + "customer_id": "sample_customer_id", + "deleted_ids": set(), + "login_customer_id": "default", + } + + # Create a mock instance of the CampaignCriterion stream + google_api = MockGoogleAds(credentials=config["credentials"]) + stream = CampaignCriterion(api=google_api, customers=[]) + + # Mock methods that are expected to be called during the test + super_read_records_mock = MagicMock() + mocker.patch("source_google_ads.streams.GoogleAdsStream.read_records", super_read_records_mock) + read_deleted_records_mock = mocker.patch.object(stream, "_read_deleted_records", return_value=[]) + update_state_mock = mocker.patch.object(stream, "_update_state") + + # Execute the method under test + list(stream.read_records(SyncMode.incremental, stream_slice=stream_slice)) + + # Verify that the parent's read_records method was called twice due to splitting + assert super_read_records_mock.call_count == 2 + + # Define the expected slices after the stream_slice is split + expected_first_slice = { + "updated_ids": set(range(10000)), + "record_changed_time_map": {i: f"time_{i}" for i in range(10000)}, + "customer_id": "sample_customer_id", + "deleted_ids": set(), + "login_customer_id": "default", + } + expected_second_slice = { + "updated_ids": set(range(10000, 15000)), + "record_changed_time_map": {i: f"time_{i}" for i in range(10000, 15000)}, + "customer_id": "sample_customer_id", + "deleted_ids": set(), + "login_customer_id": "default", + } + + # Verify the arguments passed to the parent's read_records method for both calls + first_call_args, first_call_kwargs = super_read_records_mock.call_args_list[0] + assert first_call_args[0] == SyncMode.incremental + assert first_call_kwargs["stream_slice"] == expected_first_slice + second_call_args, second_call_kwargs = super_read_records_mock.call_args_list[1] + assert second_call_args[0] == SyncMode.incremental + assert second_call_kwargs["stream_slice"] == expected_second_slice + + # Ensure that the mocked methods were called as expected + read_deleted_records_mock.assert_called_once_with(stream_slice) + update_state_mock.assert_called_once_with(stream_slice) + + +def test_update_state_with_parent_state(mocker): + """ + Test the _update_state method when the parent_stream has a state. + """ + # Mock instance setup + stream = CampaignCriterion(api=MagicMock(), customers=[]) + + # Mock parent_stream with initial state + stream.parent_stream.state = {"customer_id_1": {"change_status.last_change_date_time": "2023-10-20 00:00:00.000000"}} + + # Call the _update_state method with the first stream_slice + stream_slice_first = {"customer_id": "customer_id_1"} + stream._update_state(stream_slice_first) + + # Assert the state after the first call + expected_state_first_call = {"change_status": {"customer_id_1": {"change_status.last_change_date_time": "2023-10-20 00:00:00.000000"}}} + assert stream._state == expected_state_first_call + + # Update the parent_stream state for the second call + stream.parent_stream.state = {"customer_id_2": {"change_status.last_change_date_time": "2023-10-21 00:00:00.000000"}} + + # Call the _update_state method with the second stream_slice + stream_slice_second = {"customer_id": "customer_id_2"} + stream._update_state(stream_slice_second) + + # Assert the state after the second call + expected_state_second_call = { + "change_status": { + "customer_id_1": {"change_status.last_change_date_time": "2023-10-20 00:00:00.000000"}, + "customer_id_2": {"change_status.last_change_date_time": "2023-10-21 00:00:00.000000"}, + } + } + assert stream._state == expected_state_second_call + + # Set pendulum to return a consistent value + now = pendulum.datetime(2023, 11, 2, 12, 53, 7) + pendulum.set_test_now(now) + + # Call the _update_state method with the third stream_slice + stream_slice_third = {"customer_id": "customer_id_3"} + stream._update_state(stream_slice_third) + + # Assert the state after the third call + expected_state_third_call = { + "change_status": { + "customer_id_1": {"change_status.last_change_date_time": "2023-10-20 00:00:00.000000"}, + "customer_id_2": {"change_status.last_change_date_time": "2023-10-21 00:00:00.000000"}, + "customer_id_3": {"change_status.last_change_date_time": "2023-11-02 00:00:00.000000"}, + } + } + assert stream._state == expected_state_third_call + + # Reset the pendulum mock to its original state + pendulum.set_test_now() + + +def test_update_state_without_parent_state(mocker): + """ + Test the _update_state method when the parent_stream does not have a state. + """ + # Reset any previous mock state for pendulum + pendulum.set_test_now() + + # Mock instance setup + stream = CampaignCriterion(api=MagicMock(), customers=[]) + + # Mock pendulum call to return a consistent value + now = pendulum.datetime(2023, 11, 2, 12, 53, 7) + pendulum.set_test_now(now) + + # Call the _update_state method with the first stream_slice + stream_slice_first = {"customer_id": "customer_id_1"} + stream._update_state(stream_slice_first) + + # Assert the state after the first call + expected_state_first_call = {"change_status": {"customer_id_1": {"change_status.last_change_date_time": "2023-11-02 00:00:00.000000"}}} + assert stream._state == expected_state_first_call + + # Call the _update_state method with the second stream_slice + stream_slice_second = {"customer_id": "customer_id_2"} + stream._update_state(stream_slice_second) + + # Assert the state after the second call + expected_state_second_call = { + "change_status": { + "customer_id_1": {"change_status.last_change_date_time": "2023-11-02 00:00:00.000000"}, + "customer_id_2": {"change_status.last_change_date_time": "2023-11-02 00:00:00.000000"}, + } + } + assert stream._state == expected_state_second_call + + # Reset the pendulum mock to its original state + pendulum.set_test_now() diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_migrations/custom_query/test_config.json b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_migrations/custom_query/test_config.json new file mode 100644 index 000000000000..2ce005d03ec0 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_migrations/custom_query/test_config.json @@ -0,0 +1,18 @@ +{ + "credentials": { + "developer_token": "developer_token", + "client_id": "client_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token" + }, + "customer_id": "1234567890", + "start_date": "2023-09-04", + "conversion_window_days": 14, + "custom_queries": [ + { + "query": "SELECT campaign.name, metrics.clicks FROM campaign", + "primary_key": null, + "table_name": "test_query" + } + ] +} diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_migrations/custom_query/test_new_config.json b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_migrations/custom_query/test_new_config.json new file mode 100644 index 000000000000..7d8097055f09 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_migrations/custom_query/test_new_config.json @@ -0,0 +1,12 @@ +{ + "credentials": { + "developer_token": "developer_token", + "client_id": "client_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token" + }, + "customer_id": "1234567890", + "start_date": "2023-09-04", + "conversion_window_days": 14, + "custom_queries_array": [] +} diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_models.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_models.py index 649fe36abad4..7606a76bc7bf 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_models.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_models.py @@ -2,20 +2,26 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + +from unittest.mock import Mock + import pytest -from source_google_ads.models import Customer +from pendulum.tz.timezone import Timezone +from source_google_ads.models import CustomerModel + +def test_time_zone(mocker): + mocker.patch("source_google_ads.models.local_timezone", Mock(return_value=Timezone("Europe/Riga"))) -def test_time_zone(): - mock_account_info = [[{"customer.id": "8765"}]] - customers = Customer.from_accounts(mock_account_info) + mock_account_info = [{"customer_client.id": "8765"}] + customers = CustomerModel.from_accounts(mock_account_info) for customer in customers: - assert customer.time_zone == "local" + assert customer.time_zone.name == Timezone("Europe/Riga").name @pytest.mark.parametrize("is_manager_account", (True, False)) def test_manager_account(is_manager_account): - mock_account_info = [[{"customer.manager": is_manager_account, "customer.id": "8765"}]] - customers = Customer.from_accounts(mock_account_info) + mock_account_info = [{"customer_client.manager": is_manager_account, "customer_client.id": "8765"}] + customers = CustomerModel.from_accounts(mock_account_info) for customer in customers: assert customer.is_manager_account is is_manager_account diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py index c520140822b7..6394817edd99 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py @@ -2,30 +2,31 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + import re from collections import namedtuple -from unittest.mock import Mock +from unittest.mock import Mock, call +import pendulum import pytest from airbyte_cdk import AirbyteLogger -from freezegun import freeze_time -from google.ads.googleads.errors import GoogleAdsException -from google.ads.googleads.v11.errors.types.authorization_error import AuthorizationErrorEnum +from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode from pendulum import today from source_google_ads.custom_query_stream import IncrementalCustomQuery from source_google_ads.google_ads import GoogleAds +from source_google_ads.models import CustomerModel from source_google_ads.source import SourceGoogleAds -from source_google_ads.streams import AdGroupAdReport, AdGroupLabels, ServiceAccounts, chunk_date_range +from source_google_ads.streams import AdGroupAdLegacy, chunk_date_range from source_google_ads.utils import GAQL -from .common import MockErroringGoogleAdsClient, MockGoogleAdsClient, make_google_ads_exception +from .common import MockGoogleAdsClient @pytest.fixture -def mock_account_info(mocker): +def mock_get_customers(mocker): mocker.patch( - "source_google_ads.source.SourceGoogleAds.get_account_info", - Mock(return_value=[[{"customer.manager": False, "customer.time_zone": "Europe/Berlin", "customer.id": "8765"}]]), + "source_google_ads.source.SourceGoogleAds.get_customers", + Mock(return_value=[CustomerModel(is_manager_account=False, time_zone="Europe/Berlin", id="8765")]), ) @@ -34,7 +35,7 @@ def stream_mock(mocker, config, customers): def mock(latest_record): mocker.patch("source_google_ads.streams.GoogleAdsStream.read_records", Mock(return_value=[latest_record])) google_api = GoogleAds(credentials=config["credentials"]) - client = AdGroupAdReport( + client = AdGroupAdLegacy( start_date=config["start_date"], api=google_api, conversion_window_days=config["conversion_window_days"], customers=customers ) return client @@ -42,23 +43,6 @@ def mock(latest_record): return mock -@pytest.fixture -def mocked_gads_api(mocker): - def mock(response=None, failure_code=1, failure_msg="", error_type=""): - def side_effect_func(): - raise make_google_ads_exception(failure_code=failure_code, failure_msg=failure_msg, error_type=error_type) - yield - - side_effect = [] - if response: - side_effect.append(response) - if failure_msg or failure_code or error_type: - side_effect.append(side_effect_func()) - mocker.patch("source_google_ads.google_ads.GoogleAds.send_request", side_effect=side_effect) - - return mock - - @pytest.fixture() def mock_fields_meta_data(): DataType = namedtuple("DataType", ["name"]) @@ -105,31 +89,19 @@ def mock_fields_meta_data(): return Mock(get_fields_metadata=Mock(return_value={node.name: node for node in nodes})) -# Test chunk date range without end date -@freeze_time("2022-01-30") -def test_chunk_date_range_without_end_date(): - start_date_str = "2022-01-24" - conversion_window = 0 - slices = list(chunk_date_range( - start_date=start_date_str, conversion_window=conversion_window, end_date=None, days_of_data_storage=None, range_days=1, time_zone="UTC" - )) - expected_response = [ - {"start_date": "2022-01-24", "end_date": "2022-01-24"}, - {"start_date": "2022-01-25", "end_date": "2022-01-25"}, - {"start_date": "2022-01-26", "end_date": "2022-01-26"}, - {"start_date": "2022-01-27", "end_date": "2022-01-27"}, - {"start_date": "2022-01-28", "end_date": "2022-01-28"}, - {"start_date": "2022-01-29", "end_date": "2022-01-29"}, - {"start_date": "2022-01-30", "end_date": "2022-01-30"}, - ] - assert expected_response == slices - - def test_chunk_date_range(): start_date = "2021-03-04" end_date = "2021-05-04" conversion_window = 14 - slices = list(chunk_date_range(start_date, conversion_window, end_date, range_days=10, time_zone="UTC")) + slices = list( + chunk_date_range( + start_date=start_date, + end_date=end_date, + conversion_window=conversion_window, + slice_duration=pendulum.Duration(days=9), + time_zone="UTC", + ) + ) assert [ {"start_date": "2021-02-18", "end_date": "2021-02-27"}, {"start_date": "2021-02-28", "end_date": "2021-03-09"}, @@ -142,13 +114,37 @@ def test_chunk_date_range(): ] == slices -def test_streams_count(config, mock_account_info): +def test_streams_count(config, mock_get_customers): source = SourceGoogleAds() streams = source.streams(config) - expected_streams_number = 29 + expected_streams_number = 30 + print(f"{config=} \n{streams=}") assert len(streams) == expected_streams_number +def test_read_missing_stream(config, mock_get_customers): + source = SourceGoogleAds() + + catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream( + name="fake_stream", + json_schema={}, + supported_sync_modes=[SyncMode.full_refresh], + ), + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + ] + ) + + try: + list(source.read(AirbyteLogger(), config=config, catalog=catalog)) + except KeyError as error: + pytest.fail(str(error)) + + @pytest.mark.parametrize( ( "query", @@ -198,97 +194,95 @@ def stream_instance(query, api_mock, **kwargs): [ ( """ -SELECT - campaign.id, - campaign.name, - campaign.status, - metrics.impressions -FROM campaign -WHERE campaign.status = 'PAUSED' -AND metrics.impressions > 100 -ORDER BY campaign.status -""", + SELECT + campaign.id, + campaign.name, + campaign.status, + metrics.impressions + FROM campaign + WHERE campaign.status = 'PAUSED' + AND metrics.impressions > 100 + ORDER BY campaign.status + """, """ -SELECT - campaign.id, - campaign.name, - campaign.status, - metrics.impressions, - segments.date -FROM campaign -WHERE campaign.status = 'PAUSED' -AND metrics.impressions > 100 - AND segments.date BETWEEN '1980-01-01' AND '2000-01-01' -ORDER BY campaign.status -""", + SELECT + campaign.id, + campaign.name, + campaign.status, + metrics.impressions, + segments.date + FROM campaign + WHERE campaign.status = 'PAUSED' + AND metrics.impressions > 100 + AND segments.date BETWEEN '1980-01-01' AND '2000-01-01' + ORDER BY campaign.status + """, ), ( """ -SELECT - campaign.id, - campaign.name, - campaign.status, - metrics.impressions -FROM campaign -ORDER BY campaign.status -""", + SELECT + campaign.id, + campaign.name, + campaign.status, + metrics.impressions + FROM campaign + ORDER BY campaign.status + """, """ -SELECT - campaign.id, - campaign.name, - campaign.status, - metrics.impressions, - segments.date -FROM campaign - -WHERE segments.date BETWEEN '1980-01-01' AND '2000-01-01' -ORDER BY campaign.status -""", + SELECT + campaign.id, + campaign.name, + campaign.status, + metrics.impressions, + segments.date + FROM campaign + WHERE segments.date BETWEEN '1980-01-01' AND '2000-01-01' + ORDER BY campaign.status + """, ), ( """ -SELECT - campaign.id, - campaign.name, - campaign.status, - metrics.impressions -FROM campaign -WHERE campaign.status = 'PAUSED' -AND metrics.impressions > 100 -""", + SELECT + campaign.id, + campaign.name, + campaign.status, + metrics.impressions + FROM campaign + WHERE campaign.status = 'PAUSED' + AND metrics.impressions > 100 + """, """ -SELECT - campaign.id, - campaign.name, - campaign.status, - metrics.impressions, - segments.date -FROM campaign -WHERE campaign.status = 'PAUSED' -AND metrics.impressions > 100 - AND segments.date BETWEEN '1980-01-01' AND '2000-01-01' -""", + SELECT + campaign.id, + campaign.name, + campaign.status, + metrics.impressions, + segments.date + FROM campaign + WHERE campaign.status = 'PAUSED' + AND metrics.impressions > 100 + AND segments.date BETWEEN '1980-01-01' AND '2000-01-01' + """, ), ( """ -SELECT - campaign.accessible_bidding_strategy, - segments.ad_destination_type, - campaign.start_date, - campaign.end_date -FROM campaign -""", + SELECT + campaign.accessible_bidding_strategy, + segments.ad_destination_type, + campaign.start_date, + campaign.end_date + FROM campaign + """, """ -SELECT - campaign.accessible_bidding_strategy, - segments.ad_destination_type, - campaign.start_date, - campaign.end_date, - segments.date -FROM campaign - -WHERE segments.date BETWEEN '1980-01-01' AND '2000-01-01' -""", + SELECT + campaign.accessible_bidding_strategy, + segments.ad_destination_type, + campaign.start_date, + campaign.end_date, + segments.date + FROM campaign + WHERE segments.date BETWEEN '1980-01-01' AND '2000-01-01' + """, ), ], ) @@ -399,7 +393,7 @@ def test_check_connection_should_pass_when_config_valid(mocker): "customer_id": "fake_customer_id", "start_date": "2022-01-01", "conversion_window_days": 14, - "custom_queries": [ + "custom_queries_array": [ { "query": "SELECT campaign.accessible_bidding_strategy, segments.ad_destination_type, campaign.start_date, campaign.end_date FROM campaign", "primary_key": None, @@ -425,49 +419,6 @@ def test_check_connection_should_pass_when_config_valid(mocker): assert message is None -def test_check_connection_should_fail_when_api_call_fails(mocker): - # We patch the object inside source.py because that's the calling context - # https://docs.python.org/3/library/unittest.mock.html#where-to-patch - mocker.patch("source_google_ads.source.GoogleAds", MockErroringGoogleAdsClient) - source = SourceGoogleAds() - check_successful, message = source.check_connection( - AirbyteLogger(), - { - "credentials": { - "developer_token": "fake_developer_token", - "client_id": "fake_client_id", - "client_secret": "fake_client_secret", - "refresh_token": "fake_refresh_token", - }, - "customer_id": "fake_customer_id", - "start_date": "2022-01-01", - "conversion_window_days": 14, - "custom_queries": [ - { - "query": "SELECT campaign.accessible_bidding_strategy, segments.ad_destination_type, campaign.start_date, campaign.end_date FROM campaign", - "primary_key": None, - "cursor_field": "campaign.start_date", - "table_name": "happytable", - }, - { - "query": "SELECT segments.ad_destination_type, segments.ad_network_type, segments.day_of_week, customer.auto_tagging_enabled, customer.id, metrics.conversions, campaign.start_date FROM campaign", - "primary_key": "customer.id", - "cursor_field": None, - "table_name": "unhappytable", - }, - { - "query": "SELECT ad_group.targeting_setting.target_restrictions FROM ad_group", - "primary_key": "customer.id", - "cursor_field": None, - "table_name": "ad_group_custom", - }, - ], - }, - ) - assert not check_successful - assert message.startswith("Unable to connect to Google Ads API with the provided configuration") - - def test_end_date_is_not_in_the_future(customers): source = SourceGoogleAds() config = source.get_incremental_stream_config( @@ -476,49 +427,9 @@ def test_end_date_is_not_in_the_future(customers): assert config.get("end_date") == today().to_date_string() -def test_invalid_custom_query_handled(mocked_gads_api, config): - # limit to one custom query, otherwise need to mock more side effects - config["custom_queries"] = [next(iter(config["custom_queries"]))] - mocked_gads_api( - response=[{"customer.id": "8765"}], - failure_msg="Unrecognized field in the query: 'ad_group_ad.ad.video_ad.media_file'", - error_type="request_error", - ) - source = SourceGoogleAds() - status_ok, error = source.check_connection(AirbyteLogger(), config) - assert not status_ok - assert error == ( - "Unable to connect to Google Ads API with the provided configuration - Unrecognized field in the query: " - "'ad_group_ad.ad.video_ad.media_file'" - ) - - -@pytest.mark.parametrize( - ("cls", "error", "failure_code", "raise_expected"), - ( - (AdGroupLabels, "authorization_error", AuthorizationErrorEnum.AuthorizationError.CUSTOMER_NOT_ENABLED, False), - (AdGroupLabels, "internal_error", 1, True), - (ServiceAccounts, "authentication_error", 1, True), - (ServiceAccounts, "internal_error", 1, True), - ), -) -def test_read_record_error_handling(config, customers, mocked_gads_api, cls, error, failure_code, raise_expected): - error_msg = "Some unexpected error" - mocked_gads_api(failure_code=failure_code, failure_msg=error_msg, error_type=error) - google_api = GoogleAds(credentials=config["credentials"]) - stream = cls(api=google_api, customers=customers) - if raise_expected: - with pytest.raises(GoogleAdsException): - for _ in stream.read_records(sync_mode=Mock(), stream_slice={"customer_id": "1234567890"}): - pass - else: - for _ in stream.read_records(sync_mode=Mock(), stream_slice={"customer_id": "1234567890"}): - pass - - def test_stream_slices(config, customers): google_api = GoogleAds(credentials=config["credentials"]) - stream = AdGroupAdReport( + stream = AdGroupAdLegacy( start_date=config["start_date"], api=google_api, conversion_window_days=config["conversion_window_days"], @@ -527,8 +438,84 @@ def test_stream_slices(config, customers): ) slices = list(stream.stream_slices()) assert slices == [ - {"start_date": "2020-12-18", "end_date": "2021-01-01", "customer_id": "123"}, - {"start_date": "2021-01-02", "end_date": "2021-01-16", "customer_id": "123"}, - {"start_date": "2021-01-17", "end_date": "2021-01-31", "customer_id": "123"}, - {"start_date": "2021-02-01", "end_date": "2021-02-10", "customer_id": "123"}, + {"start_date": "2020-12-18", "end_date": "2021-01-01", "customer_id": "123", "login_customer_id": None}, + {"start_date": "2021-01-02", "end_date": "2021-01-16", "customer_id": "123", "login_customer_id": None}, + {"start_date": "2021-01-17", "end_date": "2021-01-31", "customer_id": "123", "login_customer_id": None}, + {"start_date": "2021-02-01", "end_date": "2021-02-10", "customer_id": "123", "login_customer_id": None}, ] + + +def mock_send_request(query: str, customer_id: str, login_customer_id: str = "default"): + print(query, customer_id, login_customer_id) + if customer_id == "123": + if "WHERE customer_client.status in ('active')" in query: + return [ + [ + {"customer_client.id": "123", "customer_client.status": "active"}, + ] + ] + else: + return [ + [ + {"customer_client.id": "123", "customer_client.status": "active"}, + {"customer_client.id": "456", "customer_client.status": "disabled"}, + ] + ] + else: + return [ + [ + {"customer_client.id": "789", "customer_client.status": "active"}, + ] + ] + + +@pytest.mark.parametrize( + "customer_status_filter, expected_ids, send_request_calls", + [ + ( + [], + ["123", "456", "789"], + [ + call( + "SELECT customer_client.client_customer, customer_client.level, customer_client.id, customer_client.manager, customer_client.time_zone, customer_client.status FROM customer_client", + customer_id="123", + ), + call( + "SELECT customer_client.client_customer, customer_client.level, customer_client.id, customer_client.manager, customer_client.time_zone, customer_client.status FROM customer_client", + customer_id="789", + ), + ], + ), # Empty filter, expect all customers + ( + ["active"], + ["123", "789"], + [ + call( + "SELECT customer_client.client_customer, customer_client.level, customer_client.id, customer_client.manager, customer_client.time_zone, customer_client.status FROM customer_client WHERE customer_client.status in ('active')", + customer_id="123", + ), + call( + "SELECT customer_client.client_customer, customer_client.level, customer_client.id, customer_client.manager, customer_client.time_zone, customer_client.status FROM customer_client WHERE customer_client.status in ('active')", + customer_id="789", + ), + ], + ), # Non-empty filter, expect filtered customers + ], +) +def test_get_customers(mocker, customer_status_filter, expected_ids, send_request_calls): + mock_google_api = Mock() + + mock_google_api.get_accessible_accounts.return_value = ["123", "789"] + mock_google_api.send_request.side_effect = mock_send_request + mock_google_api.parse_single_result.side_effect = lambda schema, result: result + + mock_config = {"customer_status_filter": customer_status_filter, "customer_ids": ["123", "456", "789"]} + + source = SourceGoogleAds() + + customers = source.get_customers(mock_google_api, mock_config) + + mock_google_api.send_request.assert_has_calls(send_request_calls) + + assert len(customers) == len(expected_ids) + assert {customer.id for customer in customers} == set(expected_ids) diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py index 7c0f38fdd34a..a171b869d3d2 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py @@ -2,27 +2,19 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import logging + from unittest.mock import Mock import pytest from airbyte_cdk.models import SyncMode +from airbyte_cdk.utils import AirbyteTracedException from google.ads.googleads.errors import GoogleAdsException -from google.ads.googleads.v11.errors.types.errors import ErrorCode, GoogleAdsError, GoogleAdsFailure -from google.ads.googleads.v11.errors.types.request_error import RequestErrorEnum -from google.api_core.exceptions import DataLoss, InternalServerError, ResourceExhausted, TooManyRequests +from google.ads.googleads.v15.errors.types.errors import ErrorCode, GoogleAdsError, GoogleAdsFailure +from google.ads.googleads.v15.errors.types.request_error import RequestErrorEnum +from google.api_core.exceptions import DataLoss, InternalServerError, ResourceExhausted, TooManyRequests, Unauthenticated from grpc import RpcError from source_google_ads.google_ads import GoogleAds -from source_google_ads.streams import ClickView, cyclic_sieve - -from .common import MockGoogleAdsClient as MockGoogleAdsClient - - -@pytest.fixture -def mock_ads_client(mocker, config): - """Mock google ads library method, so it returns mocked Client""" - mocker.patch("source_google_ads.google_ads.GoogleAdsClient.load_from_dict", return_value=MockGoogleAdsClient(config)) - +from source_google_ads.streams import AdGroup, ClickView, Customer, CustomerLabel # EXPIRED_PAGE_TOKEN exception will be raised when page token has expired. exception = GoogleAdsException( @@ -59,7 +51,7 @@ class MockGoogleAds(GoogleAds): def parse_single_result(self, schema, result): return result - def send_request(self, query: str, customer_id: str): + def send_request(self, query: str, customer_id: str, login_customer_id: str = "none"): self.count += 1 if self.count == 1: return mock_response_1() @@ -67,7 +59,7 @@ def send_request(self, query: str, customer_id: str): return mock_response_2() -def test_page_token_expired_retry_succeeds(mock_ads_client, config, customers): +def test_page_token_expired_retry_succeeds(config, customers): """ Page token expired while reading records on date 2021-01-03 The latest read record is {"segments.date": "2021-01-03", "click_view.gclid": "4"} @@ -75,7 +67,7 @@ def test_page_token_expired_retry_succeeds(mock_ads_client, config, customers): It shouldn't read records on 2021-01-01, 2021-01-02 """ customer_id = next(iter(customers)).id - stream_slice = {"customer_id": customer_id, "start_date": "2021-01-01", "end_date": "2021-01-15"} + stream_slice = {"customer_id": customer_id, "start_date": "2021-01-01", "end_date": "2021-01-15", "login_customer_id": customer_id} google_api = MockGoogleAds(credentials=config["credentials"]) incremental_stream_config = dict( @@ -92,7 +84,9 @@ def test_page_token_expired_retry_succeeds(mock_ads_client, config, customers): result = list(stream.read_records(sync_mode=SyncMode.incremental, cursor_field=["segments.date"], stream_slice=stream_slice)) assert len(result) == 9 assert stream.get_query.call_count == 2 - stream.get_query.assert_called_with({"customer_id": customer_id, "start_date": "2021-01-03", "end_date": "2021-01-15"}) + stream.get_query.assert_called_with( + {"customer_id": customer_id, "start_date": "2021-01-03", "end_date": "2021-01-15", "login_customer_id": customer_id} + ) def mock_response_fails_1(): @@ -118,7 +112,7 @@ def mock_response_fails_2(): class MockGoogleAdsFails(MockGoogleAds): - def send_request(self, query: str, customer_id: str): + def send_request(self, query: str, customer_id: str, login_customer_id: str = "none"): self.count += 1 if self.count == 1: return mock_response_fails_1() @@ -126,13 +120,13 @@ def send_request(self, query: str, customer_id: str): return mock_response_fails_2() -def test_page_token_expired_retry_fails(mock_ads_client, config, customers): +def test_page_token_expired_retry_fails(config, customers): """ Page token has expired while reading records within date "2021-01-03", it should raise error, because Google Ads API doesn't allow filter by datetime. """ customer_id = next(iter(customers)).id - stream_slice = {"customer_id": customer_id, "start_date": "2021-01-01", "end_date": "2021-01-15"} + stream_slice = {"customer_id": customer_id, "start_date": "2021-01-01", "end_date": "2021-01-15", "login_customer_id": customer_id} google_api = MockGoogleAdsFails(credentials=config["credentials"]) incremental_stream_config = dict( @@ -146,10 +140,16 @@ def test_page_token_expired_retry_fails(mock_ads_client, config, customers): stream.get_query = Mock() stream.get_query.return_value = "query" - with pytest.raises(GoogleAdsException): + with pytest.raises(AirbyteTracedException) as exception: list(stream.read_records(sync_mode=SyncMode.incremental, cursor_field=["segments.date"], stream_slice=stream_slice)) + assert exception.value.message == ( + "Page token has expired during processing response. " + "Please contact the Airbyte team with the link of your connection for assistance." + ) - stream.get_query.assert_called_with({"customer_id": customer_id, "start_date": "2021-01-03", "end_date": "2021-01-15"}) + stream.get_query.assert_called_with( + {"customer_id": customer_id, "start_date": "2021-01-03", "end_date": "2021-01-15", "login_customer_id": customer_id} + ) assert stream.get_query.call_count == 2 @@ -165,18 +165,18 @@ def mock_response_fails_one_date(): class MockGoogleAdsFailsOneDate(MockGoogleAds): - def send_request(self, query: str, customer_id: str): + def send_request(self, query: str, customer_id: str, login_customer_id: str = "none"): return mock_response_fails_one_date() -def test_page_token_expired_it_should_fail_date_range_1_day(mock_ads_client, config, customers): +def test_page_token_expired_it_should_fail_date_range_1_day(config, customers): """ Page token has expired while reading records within date "2021-01-03", it should raise error, because Google Ads API doesn't allow filter by datetime. Minimum date range is 1 day. """ customer_id = next(iter(customers)).id - stream_slice = {"customer_id": customer_id, "start_date": "2021-01-03", "end_date": "2021-01-04"} + stream_slice = {"customer_id": customer_id, "start_date": "2021-01-03", "end_date": "2021-01-04", "login_customer_id": customer_id} google_api = MockGoogleAdsFailsOneDate(credentials=config["credentials"]) incremental_stream_config = dict( @@ -190,20 +190,27 @@ def test_page_token_expired_it_should_fail_date_range_1_day(mock_ads_client, con stream.get_query = Mock() stream.get_query.return_value = "query" - with pytest.raises(GoogleAdsException): + with pytest.raises(AirbyteTracedException) as exception: list(stream.read_records(sync_mode=SyncMode.incremental, cursor_field=["segments.date"], stream_slice=stream_slice)) - - stream.get_query.assert_called_with({"customer_id": customer_id, "start_date": "2021-01-03", "end_date": "2021-01-04"}) + assert exception.value.message == ( + "Page token has expired during processing response. " + "Please contact the Airbyte team with the link of your connection for assistance." + ) + stream.get_query.assert_called_with( + {"customer_id": customer_id, "start_date": "2021-01-03", "end_date": "2021-01-04", "login_customer_id": customer_id} + ) assert stream.get_query.call_count == 1 @pytest.mark.parametrize("error_cls", (ResourceExhausted, TooManyRequests, InternalServerError, DataLoss)) def test_retry_transient_errors(mocker, config, customers, error_cls): + customer_id = next(iter(customers)).id + mocker.patch("time.sleep") credentials = config["credentials"] credentials.update(use_proto_plus=True) api = GoogleAds(credentials=credentials) - mocked_search = mocker.patch.object(api.ga_service, "search", side_effect=error_cls("Error message")) + mocked_search = mocker.patch.object(api.ga_services["default"], "search", side_effect=error_cls("Error message")) incremental_stream_config = dict( api=api, conversion_window_days=config["conversion_window_days"], @@ -212,22 +219,82 @@ def test_retry_transient_errors(mocker, config, customers, error_cls): customers=customers, ) stream = ClickView(**incremental_stream_config) - customer_id = next(iter(customers)).id - stream_slice = {"customer_id": customer_id, "start_date": "2021-01-03", "end_date": "2021-01-04"} + stream_slice = {"customer_id": customer_id, "start_date": "2021-01-03", "end_date": "2021-01-04", "login_customer_id": "default"} records = [] - with pytest.raises(error_cls): + with pytest.raises(error_cls) as exception: records = list(stream.read_records(sync_mode=SyncMode.incremental, cursor_field=["segments.date"], stream_slice=stream_slice)) + assert exception.value.message == "Error message" + assert mocked_search.call_count == 5 assert records == [] -def test_cyclic_sieve(caplog): - original_logger = logging.getLogger("test") - original_logger.setLevel(logging.DEBUG) - sieve = cyclic_sieve(original_logger, fraction=10) - for _ in range(20): - sieve.info("Ground Control to Major Tom") - sieve.info("Your circuit's dead, there's something wrong") - sieve.info("Can you hear me, Major Tom?") - sieve.bump() - assert len(caplog.records) == 6 # 20 * 3 / 10 +def test_parse_response(mocker, customers, config): + """ + Tests the `parse_response` method of the `Customer` class. + The test checks if the optimization_score_weight of type int is converted to float. + """ + + # Prepare sample input data + response = [ + {"customer.id": "1", "segments.date": "2023-09-19", "customer.optimization_score_weight": 80}, + {"customer.id": "2", "segments.date": "2023-09-20", "customer.optimization_score_weight": 80.0}, + {"customer.id": "3", "segments.date": "2023-09-21"}, + ] + mocker.patch("source_google_ads.streams.GoogleAdsStream.parse_response", Mock(return_value=response)) + + credentials = config["credentials"] + api = GoogleAds(credentials=credentials) + + incremental_stream_config = dict( + api=api, + conversion_window_days=config["conversion_window_days"], + start_date=config["start_date"], + end_date="2021-04-04", + customers=customers, + ) + + # Create an instance of the Customer class + accounts = Customer(**incremental_stream_config) + + # Use the parse_response method and get the output + output = list(accounts.parse_response(response)) + + # Expected output after the method's logic + expected_output = [ + {"customer.id": "1", "segments.date": "2023-09-19", "customer.optimization_score_weight": 80.0}, + {"customer.id": "2", "segments.date": "2023-09-20", "customer.optimization_score_weight": 80.0}, + {"customer.id": "3", "segments.date": "2023-09-21"}, + ] + + assert output == expected_output + + +def test_read_records_unauthenticated(mocker, customers, config): + credentials = config["credentials"] + api = GoogleAds(credentials=credentials) + + mocker.patch.object(api, "parse_single_result", side_effect=Unauthenticated(message="Unauthenticated")) + + stream_config = dict( + api=api, + customers=customers, + ) + stream = CustomerLabel(**stream_config) + with pytest.raises(AirbyteTracedException) as exc_info: + list(stream.read_records(SyncMode.full_refresh, {"customer_id": "customer_id", "login_customer_id": "default"})) + + assert exc_info.value.message == ( + "Authentication failed for the customer 'customer_id'. " "Please try to Re-authenticate your credentials on set up Google Ads page." + ) + + +def test_ad_group_stream_query_removes_metrics_field_for_manager(customers_manager, customers, config): + credentials = config["credentials"] + api = GoogleAds(credentials=credentials) + stream_config = dict(api=api, customers=customers_manager, start_date="2020-01-01", conversion_window_days=10) + stream = AdGroup(**stream_config) + assert "metrics" not in stream.get_query(stream_slice={"customer_id": "123"}) + stream_config = dict(api=api, customers=customers, start_date="2020-01-01", conversion_window_days=10) + stream = AdGroup(**stream_config) + assert "metrics" in stream.get_query(stream_slice={"customer_id": "123"}) diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py index 350fc51f8378..7c42cd50360e 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_utils.py @@ -2,10 +2,15 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + +from datetime import datetime +from unittest.mock import Mock + +import backoff import pytest from airbyte_cdk.utils import AirbyteTracedException from source_google_ads import SourceGoogleAds -from source_google_ads.utils import GAQL +from source_google_ads.utils import GAQL, generator_backoff def test_parse_GAQL_ok(): @@ -54,13 +59,15 @@ def test_parse_GAQL_ok(): assert sql.parameters == "" assert str(sql) == "SELECT t.field1, t.field2 FROM x_Table ORDER BY field2, field1 LIMIT 10" - sql = GAQL.parse(""" + sql = GAQL.parse( + """ SELECT field1, field2 FROM x_Table WHERE date = '2020-01-01' ORDER BY field2 ASC, field1 DESC LIMIT 10 - PARAMETERS include_drafts=true """) + PARAMETERS include_drafts=true """ + ) assert sql.fields == ("field1", "field2") assert sql.resource_name == "x_Table" @@ -68,46 +75,25 @@ def test_parse_GAQL_ok(): assert sql.order_by == "field2 ASC, field1 DESC" assert sql.limit == 10 assert sql.parameters == "include_drafts=true" - assert str(sql) == "SELECT field1, field2 FROM x_Table WHERE date = '2020-01-01' ORDER BY field2 ASC, field1 DESC LIMIT 10 PARAMETERS include_drafts=true" + assert ( + str(sql) + == "SELECT field1, field2 FROM x_Table WHERE date = '2020-01-01' ORDER BY field2 ASC, field1 DESC LIMIT 10 PARAMETERS include_drafts=true" + ) @pytest.mark.parametrize( "config", [ - { - "custom_queries": [ - { - "query": "SELECT field1, field2 FROM x_Table2", - "table_name": "test_table" - }] - }, - { - "custom_queries": [ - { - "query": "SELECT field1, field2 FROM x_Table WHERE ", - "table_name": "test_table" - }] - }, - { - "custom_queries": [ - { - "query": "SELECT field1, , field2 FROM table", - "table_name": "test_table" - }] - }, - { - "custom_queries": [ - { - "query": "SELECT fie ld1, field2 FROM table", - "table_name": "test_table" - }] - }, - ] + {"custom_queries_array": [{"query": "SELECT field1, field2 FROM x_Table2", "table_name": "test_table"}]}, + {"custom_queries_array": [{"query": "SELECT field1, field2 FROM x_Table WHERE ", "table_name": "test_table"}]}, + {"custom_queries_array": [{"query": "SELECT field1, , field2 FROM table", "table_name": "test_table"}]}, + {"custom_queries_array": [{"query": "SELECT fie ld1, field2 FROM table", "table_name": "test_table"}]}, + ], ) def test_parse_GAQL_fail(config): with pytest.raises(AirbyteTracedException) as e: SourceGoogleAds._validate_and_transform(config) - expected_message = "The custom GAQL query test_table failed. Validate your GAQL query with the Google Ads query validator. https://developers.google.com/google-ads/api/fields/v13/query_validator" + expected_message = "The custom GAQL query test_table failed. Validate your GAQL query with the Google Ads query validator. https://developers.google.com/google-ads/api/fields/v15/query_validator" assert e.value.message == expected_message @@ -143,3 +129,75 @@ def test_parse_GAQL_fail(config): ) def test_get_query_fields(query, fields): assert list(GAQL.parse(query).fields) == fields + + +def test_generator_backoff_retries_until_success(): + tries = 0 + + def flaky_function(): + nonlocal tries # Declare tries as nonlocal to modify it within the function + if tries < 2: + tries += 1 + raise ValueError("Simulated failure") + else: + yield "Success" + + # Mock on_backoff callable + mock_on_backoff = Mock() + + # Apply the decorator to the flaky_function + decorated_flaky_function = generator_backoff( + wait_gen=backoff.expo, + exception=ValueError, + max_tries=4, + max_time=5, + on_backoff=mock_on_backoff, + factor=2, + )(flaky_function) + + # Start the clock + start_time = datetime.now() + + # Run the decorated function and collect results + results = list(decorated_flaky_function()) + + # Check that the function succeeded after retries + assert results == ["Success"] + + # Check that the function was retried the correct number of times + assert mock_on_backoff.call_count == 2 + + # Check that the elapsed time is reasonable + elapsed_time = (datetime.now() - start_time).total_seconds() + # The wait times are 3 and then 2 seconds, so the elapsed time should be at least 5 seconds + assert elapsed_time >= 5 + + # Check that on_backoff was called with the correct parameters + expected_calls = [ + { + "target": flaky_function, + "args": (), + "kwargs": {}, + "tries": 1, + "elapsed": pytest.approx(0.1, abs=0.1), + "wait": pytest.approx(2, abs=0.1), + "exception": "Simulated failure", + }, + { + "target": flaky_function, + "args": (), + "kwargs": {}, + "tries": 2, + "elapsed": pytest.approx(2, abs=0.1), + "wait": pytest.approx(3, abs=0.1), + "exception": "Simulated failure", + }, + ] + + # Convert actual calls to a list of dictionaries + actual_calls = [{**c.args[0], "exception": str(c.args[0]["exception"])} for c in mock_on_backoff.call_args_list] + print(actual_calls) + + # Compare each expected call with the actual call + for expected, actual in zip(expected_calls, actual_calls): + assert expected == actual diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/Dockerfile b/airbyte-integrations/connectors/source-google-analytics-data-api/Dockerfile deleted file mode 100644 index 6c3b35f5861b..000000000000 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM python:3.9.11-slim as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apt update -y && apt upgrade -y - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# copy payload code only -COPY main.py ./ -COPY source_google_analytics_data_api ./source_google_analytics_data_api - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=1.1.3 -LABEL io.airbyte.name=airbyte/source-google-analytics-data-api diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/README.md b/airbyte-integrations/connectors/source-google-analytics-data-api/README.md index 87677dfc98e0..2ca089c74cd0 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/README.md +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-google-analytics-data-api:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/google-analytics-data-api) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_analytics_data_api/spec.{yaml,json}` file. @@ -54,19 +48,70 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-google-analytics-data-api:dev + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-google-analytics-data-api build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-google-analytics-data-api:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations -You can also build the connector image via Gradle: +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-google-analytics-data-api:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-google-analytics-data-api:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-google-analytics-data-api:dev . +# Running the spec command against your patched connector +docker run airbyte/source-google-analytics-data-api:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -75,45 +120,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-analytics-data- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-analytics-data-api:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-google-analytics-data-api:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install '.[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-google-analytics-data-api test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-google-analytics-data-api:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-google-analytics-data-api:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-google-analytics-data-api:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -123,8 +139,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-google-analytics-data-api test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/google-analytics-data-api.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-analytics-data-api/acceptance-test-config.yml index c3206c26c842..4e07a4a4ce28 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/acceptance-test-config.yml @@ -7,7 +7,8 @@ acceptance_tests: tests: - spec_path: "source_google_analytics_data_api/spec.json" backward_compatibility_tests_config: - disable_for_version: 0.2.1 + # changed the structure of `custom_reports` -> `cohortSpec` + disable_for_version: 2.1.0 connection: tests: - config_path: "secrets/config.json" @@ -20,54 +21,33 @@ acceptance_tests: basic_read: tests: - config_path: "secrets/config.json" - empty_streams: - - name: "traffic_sources" - bypass_reason: "The data contains business information" expect_records: path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: no - extra_records: yes - ignored_fields: - devices: - - name: averageSessionDuration - bypass_reason: "dynamic field" - locations: - - name: averageSessionDuration - bypass_reason: "dynamic field" - pages: - - name: screenPageViews - bypass_reason: "dynamically created field" - - name: bounceRate - bypass_reason: "dynamically created field" - website_overview: - - name: averageSessionDuration - bypass_reason: "dynamically created field" - pivot_report: - - name: sessions - bypass_reason: "volatile data" + empty_streams: + - name: "cohort_report" + bypass_reason: "The test resource does not support cohort report" + - name: "demographic_interest_report" + bypass_reason: "The test resource does not collect interest" + - name: "demographic_age_report" + bypass_reason: "The test resource does not collect age" + - name: "demographic_gender_report" + bypass_reason: "The test resource does not collect gender" + - name: "publisher_ads_ad_unit_report" + bypass_reason: "The test resource does not work with publisher ads" + - name: "publisher_ads_ad_source_report" + bypass_reason: "The test resource does not work with publisher ads" + - name: "publisher_ads_page_path_report" + bypass_reason: "The test resource does not work with publisher ads" + - name: "publisher_ads_ad_format_report" + bypass_reason: "The test resource does not work with publisher ads" full_refresh: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - ignored_fields: - devices: - - name: averageSessionDuration - bypass_reason: "dynamic field" - locations: - - name: averageSessionDuration - bypass_reason: "dynamic field" - traffic_sources: - - name: averageSessionDuration - bypass_reason: "dynamically created field" - website_overview: - - name: averageSessionDuration - bypass_reason: "dynamically created field" incremental: tests: - config_path: "secrets/config.json" - timeout_seconds: 3600 + timeout_seconds: 10800 configured_catalog_path: "integration_tests/configured_catalog.json" future_state: future_state_path: "integration_tests/abnormal_state.json" - threshold_days: 2 diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-google-analytics-data-api/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/build.gradle b/airbyte-integrations/connectors/source-google-analytics-data-api/build.gradle deleted file mode 100644 index cfd55d7805c9..000000000000 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_google_analytics_data_api' -} diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/abnormal_state.json index cd3adac6a364..5e20a30c4498 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/abnormal_state.json @@ -86,5 +86,544 @@ "date": "20990101" } } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "user_acquisition_first_user_medium_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "user_acquisition_first_user_source_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "user_acquisition_first_user_source_medium_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "user_acquisition_first_user_source_platform_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "user_acquisition_first_user_campaign_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "user_acquisition_first_user_google_ads_ad_network_type_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "user_acquisition_first_user_google_ads_ad_group_name_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "traffic_acquisition_session_source_medium_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "traffic_acquisition_session_medium_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "traffic_acquisition_session_source_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "traffic_acquisition_session_campaign_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "traffic_acquisition_session_default_channel_grouping_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "traffic_acquisition_session_source_platform_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "events_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "conversions_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "pages_title_and_screen_class_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "pages_path_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "pages_title_and_screen_name_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "content_group_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "ecommerce_purchases_item_id_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "demographic_country_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "demographic_region_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "demographic_city_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "demographic_language_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "tech_browser_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "tech_device_category_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "tech_device_model_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "tech_screen_resolution_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "tech_app_version_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "tech_platform_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "tech_platform_device_category_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "tech_operating_system_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "tech_os_with_version_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "demographic_interest_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "ecommerce_purchases_item_category_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "ecommerce_purchases_item_category_report_combined" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "ecommerce_purchases_item_category_3_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "publisher_ads_ad_unit_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "demographic_age_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "publisher_ads_ad_format_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "ecommerce_purchases_item_category_4_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "ecommerce_purchases_item_category_5_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "ecommerce_purchases_item_category_2_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "ecommerce_purchases_item_name_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "publisher_ads_page_path_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "demographic_gender_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "ecommerce_purchases_item_brand_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "publisher_ads_ad_source_report" + }, + "stream_state": { + "date": "20990101" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "weekly_events_report" + }, + "stream_state": { + "yearWeek": "209915" + } + } } ] diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/configured_catalog.json index c1da32d744dc..8f212a2ae02b 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/configured_catalog.json @@ -187,6 +187,647 @@ ["startDate"], ["endDate"] ] + }, + { + "stream": { + "name": "user_acquisition_first_user_medium_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "user_acquisition_first_user_source_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "user_acquisition_first_user_source_medium_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "user_acquisition_first_user_source_platform_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "user_acquisition_first_user_campaign_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "user_acquisition_first_user_google_ads_ad_network_type_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "user_acquisition_first_user_google_ads_ad_group_name_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "traffic_acquisition_session_source_medium_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "traffic_acquisition_session_medium_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "traffic_acquisition_session_source_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "traffic_acquisition_session_campaign_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "traffic_acquisition_session_default_channel_grouping_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "traffic_acquisition_session_source_platform_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "events_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "conversions_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "pages_title_and_screen_class_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "pages_path_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "pages_title_and_screen_name_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "content_group_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "ecommerce_purchases_item_name_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "ecommerce_purchases_item_id_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "ecommerce_purchases_item_category_report_combined", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "ecommerce_purchases_item_category_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "ecommerce_purchases_item_category_2_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "ecommerce_purchases_item_category_3_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "ecommerce_purchases_item_category_4_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "ecommerce_purchases_item_category_5_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "ecommerce_purchases_item_brand_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "publisher_ads_ad_unit_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "publisher_ads_page_path_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "publisher_ads_ad_format_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "publisher_ads_ad_source_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "demographic_country_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "demographic_region_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "demographic_city_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "demographic_language_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "demographic_age_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "demographic_gender_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "demographic_interest_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "tech_browser_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "tech_device_category_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "tech_device_model_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "tech_screen_resolution_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "tech_app_version_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "tech_platform_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "tech_platform_device_category_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "tech_operating_system_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "tech_os_with_version_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["property_id"], ["date"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["date"]] + }, + { + "stream": { + "name": "weekly_events_report", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["yearWeek"], + "source_defined_primary_key": [ + ["property_id"], + ["yearWeek"], + ["eventName"] + ] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "primary_key": [["property_id"], ["yearWeek"], ["eventName"]] } ] } diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/expected_records.jsonl index 861748826431..78be1140ebe1 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/expected_records.jsonl @@ -1,90 +1,51 @@ -{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230406","active1DayUsers":2562},"emitted_at":1681405954033} -{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230403","active1DayUsers":2521},"emitted_at":1681405954034} -{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230404","active1DayUsers":2386},"emitted_at":1681405954034} -{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230405","active1DayUsers":2318},"emitted_at":1681405954035} -{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230411","active1DayUsers":2248},"emitted_at":1681405954035} -{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230412","active1DayUsers":2164},"emitted_at":1681405954036} -{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230410","active1DayUsers":2021},"emitted_at":1681405954036} -{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230407","active1DayUsers":1628},"emitted_at":1681405954037} -{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230409","active1DayUsers":1009},"emitted_at":1681405954037} -{"stream":"daily_active_users","data":{"property_id":"314186564","date":"20230402","active1DayUsers":978},"emitted_at":1681405954038} -{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230403","active7DayUsers":11840},"emitted_at":1681405954684} -{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230406","active7DayUsers":11828},"emitted_at":1681405954685} -{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230404","active7DayUsers":11812},"emitted_at":1681405954685} -{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230405","active7DayUsers":11751},"emitted_at":1681405954685} -{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230408","active7DayUsers":11745},"emitted_at":1681405954685} -{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230409","active7DayUsers":11739},"emitted_at":1681405954685} -{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230407","active7DayUsers":11637},"emitted_at":1681405954685} -{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230401","active7DayUsers":11547},"emitted_at":1681405954685} -{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230402","active7DayUsers":11521},"emitted_at":1681405954685} -{"stream":"weekly_active_users","data":{"property_id":"314186564","date":"20230410","active7DayUsers":11369},"emitted_at":1681405954686} -{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230401","active28DayUsers":48082},"emitted_at":1681405955854} -{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230402","active28DayUsers":47927},"emitted_at":1681405955854} -{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230403","active28DayUsers":44678},"emitted_at":1681405955854} -{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230404","active28DayUsers":42997},"emitted_at":1681405955854} -{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230405","active28DayUsers":42219},"emitted_at":1681405955855} -{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230406","active28DayUsers":42028},"emitted_at":1681405955855} -{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230407","active28DayUsers":41851},"emitted_at":1681405955855} -{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230408","active28DayUsers":41775},"emitted_at":1681405955855} -{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230409","active28DayUsers":41717},"emitted_at":1681405955855} -{"stream":"four_weekly_active_users","data":{"property_id":"314186564","date":"20230410","active28DayUsers":41212},"emitted_at":1681405955855} -{"stream":"devices","data":{"property_id":"314186564","date":"20230411","deviceCategory":"desktop","operatingSystem":"Macintosh","browser":"Chrome","totalUsers":973,"newUsers":368,"sessions":1667,"sessionsPerUser":2.110126582278481,"averageSessionDuration":308.8923676994601,"screenPageViews":5367,"screenPageViewsPerSession":3.2195560887822436,"bounceRate":0.498500299940012},"emitted_at":1681405958296} -{"stream":"devices","data":{"property_id":"314186564","date":"20230412","deviceCategory":"desktop","operatingSystem":"Macintosh","browser":"Chrome","totalUsers":969,"newUsers":350,"sessions":1588,"sessionsPerUser":2.0025220680958387,"averageSessionDuration":336.108126070529,"screenPageViews":4726,"screenPageViewsPerSession":2.9760705289672544,"bounceRate":0.5012594458438288},"emitted_at":1681405958296} -{"stream":"devices","data":{"property_id":"314186564","date":"20230404","deviceCategory":"desktop","operatingSystem":"Macintosh","browser":"Chrome","totalUsers":942,"newUsers":352,"sessions":1554,"sessionsPerUser":2.007751937984496,"averageSessionDuration":328.66656451029604,"screenPageViews":5217,"screenPageViewsPerSession":3.357142857142857,"bounceRate":0.4954954954954955},"emitted_at":1681405958296} -{"stream":"devices","data":{"property_id":"314186564","date":"20230406","deviceCategory":"desktop","operatingSystem":"Macintosh","browser":"Chrome","totalUsers":942,"newUsers":389,"sessions":1551,"sessionsPerUser":1.9783163265306123,"averageSessionDuration":357.5382107272727,"screenPageViews":5102,"screenPageViewsPerSession":3.289490651192779,"bounceRate":0.49258542875564154},"emitted_at":1681405958297} -{"stream":"devices","data":{"property_id":"314186564","date":"20230403","deviceCategory":"desktop","operatingSystem":"Macintosh","browser":"Chrome","totalUsers":929,"newUsers":341,"sessions":1546,"sessionsPerUser":2.0558510638297873,"averageSessionDuration":315.4776974385511,"screenPageViews":5116,"screenPageViewsPerSession":3.309184993531695,"bounceRate":0.5071151358344114},"emitted_at":1681405958297} -{"stream":"devices","data":{"property_id":"314186564","date":"20230405","deviceCategory":"desktop","operatingSystem":"Macintosh","browser":"Chrome","totalUsers":926,"newUsers":363,"sessions":1573,"sessionsPerUser":2.0428571428571427,"averageSessionDuration":346.09502719898285,"screenPageViews":5032,"screenPageViewsPerSession":3.1989828353464715,"bounceRate":0.4869675778766688},"emitted_at":1681405958297} -{"stream":"devices","data":{"property_id":"314186564","date":"20230410","deviceCategory":"desktop","operatingSystem":"Macintosh","browser":"Chrome","totalUsers":920,"newUsers":374,"sessions":1524,"sessionsPerUser":2.0456375838926175,"averageSessionDuration":255.77025801837266,"screenPageViews":4025,"screenPageViewsPerSession":2.641076115485564,"bounceRate":0.5255905511811023},"emitted_at":1681405958297} -{"stream":"devices","data":{"property_id":"314186564","date":"20230403","deviceCategory":"desktop","operatingSystem":"Windows","browser":"Chrome","totalUsers":781,"newUsers":366,"sessions":1184,"sessionsPerUser":1.8528951486697967,"averageSessionDuration":278.84846059881755,"screenPageViews":2993,"screenPageViewsPerSession":2.5278716216216215,"bounceRate":0.5616554054054054},"emitted_at":1681405958297} -{"stream":"devices","data":{"property_id":"314186564","date":"20230411","deviceCategory":"desktop","operatingSystem":"Windows","browser":"Chrome","totalUsers":760,"newUsers":365,"sessions":1155,"sessionsPerUser":1.896551724137931,"averageSessionDuration":264.1307251896104,"screenPageViews":2452,"screenPageViewsPerSession":2.122943722943723,"bounceRate":0.5316017316017316},"emitted_at":1681405958298} -{"stream":"devices","data":{"property_id":"314186564","date":"20230404","deviceCategory":"desktop","operatingSystem":"Windows","browser":"Chrome","totalUsers":727,"newUsers":345,"sessions":1137,"sessionsPerUser":1.8517915309446253,"averageSessionDuration":252.06245670272648,"screenPageViews":2601,"screenPageViewsPerSession":2.287598944591029,"bounceRate":0.5488126649076517},"emitted_at":1681405958298} -{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230406","totalUsers":108,"newUsers":62,"sessions":157,"sessionsPerUser":1.6354166666666667,"averageSessionDuration":435.44268001273895,"screenPageViews":534,"screenPageViewsPerSession":3.4012738853503186,"bounceRate":0.5031847133757962},"emitted_at":1681405962136} -{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230405","totalUsers":95,"newUsers":54,"sessions":123,"sessionsPerUser":1.5769230769230769,"averageSessionDuration":499.2074986666667,"screenPageViews":481,"screenPageViewsPerSession":3.910569105691057,"bounceRate":0.44715447154471544},"emitted_at":1681405962136} -{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230403","totalUsers":94,"newUsers":46,"sessions":126,"sessionsPerUser":1.68,"averageSessionDuration":424.00281903174607,"screenPageViews":499,"screenPageViewsPerSession":3.9603174603174605,"bounceRate":0.5238095238095238},"emitted_at":1681405962136} -{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230404","totalUsers":85,"newUsers":47,"sessions":121,"sessionsPerUser":1.6575342465753424,"averageSessionDuration":378.81275640495863,"screenPageViews":434,"screenPageViewsPerSession":3.5867768595041323,"bounceRate":0.48760330578512395},"emitted_at":1681405962136} -{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230412","totalUsers":85,"newUsers":49,"sessions":131,"sessionsPerUser":1.8194444444444444,"averageSessionDuration":379.1322029236641,"screenPageViews":391,"screenPageViewsPerSession":2.984732824427481,"bounceRate":0.5267175572519084},"emitted_at":1681405962137} -{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230410","totalUsers":81,"newUsers":42,"sessions":135,"sessionsPerUser":1.9565217391304348,"averageSessionDuration":303.13140742962963,"screenPageViews":376,"screenPageViewsPerSession":2.785185185185185,"bounceRate":0.5407407407407407},"emitted_at":1681405962137} -{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230411","totalUsers":81,"newUsers":38,"sessions":123,"sessionsPerUser":1.9523809523809523,"averageSessionDuration":362.51537134146344,"screenPageViews":312,"screenPageViewsPerSession":2.5365853658536586,"bounceRate":0.5934959349593496},"emitted_at":1681405962137} -{"stream":"locations","data":{"property_id":"314186564","region":"Karnataka","country":"India","city":"Bengaluru","date":"20230411","totalUsers":76,"newUsers":52,"sessions":123,"sessionsPerUser":1.8636363636363635,"averageSessionDuration":203.00314456910567,"screenPageViews":261,"screenPageViewsPerSession":2.1219512195121952,"bounceRate":0.4959349593495935},"emitted_at":1681405962137} -{"stream":"locations","data":{"property_id":"314186564","region":"Karnataka","country":"India","city":"Bengaluru","date":"20230403","totalUsers":69,"newUsers":34,"sessions":102,"sessionsPerUser":1.728813559322034,"averageSessionDuration":256.4942830490196,"screenPageViews":216,"screenPageViewsPerSession":2.1176470588235294,"bounceRate":0.5490196078431373},"emitted_at":1681405962137} -{"stream":"locations","data":{"property_id":"314186564","region":"New York","country":"United States","city":"New York","date":"20230407","totalUsers":69,"newUsers":30,"sessions":98,"sessionsPerUser":1.849056603773585,"averageSessionDuration":489.54009168367344,"screenPageViews":376,"screenPageViewsPerSession":3.836734693877551,"bounceRate":0.4489795918367347},"emitted_at":1681405962137} -{"stream":"pages","data":{"property_id":"314186564","date":"20230405","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":1190,"bounceRate":0.5516393442622951},"emitted_at":1681405967183} -{"stream":"pages","data":{"property_id":"314186564","date":"20230411","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":1151,"bounceRate":0.5400641025641025},"emitted_at":1681405967184} -{"stream":"pages","data":{"property_id":"314186564","date":"20230404","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":1137,"bounceRate":0.5617232808616405},"emitted_at":1681405967184} -{"stream":"pages","data":{"property_id":"314186564","date":"20230410","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":1099,"bounceRate":0.5416666666666666},"emitted_at":1681405967184} -{"stream":"pages","data":{"property_id":"314186564","date":"20230403","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":1092,"bounceRate":0.5569070373588184},"emitted_at":1681405967184} -{"stream":"pages","data":{"property_id":"314186564","date":"20230412","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":1089,"bounceRate":0.5690515806988353},"emitted_at":1681405967184} -{"stream":"pages","data":{"property_id":"314186564","date":"20230406","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":1005,"bounceRate":0.5516279069767441},"emitted_at":1681405967185} -{"stream":"pages","data":{"property_id":"314186564","date":"20230407","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":734,"bounceRate":0.571619812583668},"emitted_at":1681405967185} -{"stream":"pages","data":{"property_id":"314186564","date":"20230403","hostName":"airbyte.com","pagePathPlusQueryString":"/blog/data-modeling-unsung-hero-data-engineering-introduction","screenPageViews":541,"bounceRate":0.7192691029900332},"emitted_at":1681405967185} -{"stream":"pages","data":{"property_id":"314186564","date":"20230402","hostName":"airbyte.com","pagePathPlusQueryString":"/","screenPageViews":529,"bounceRate":0.5614678899082569},"emitted_at":1681405967185} -{"stream":"website_overview","data":{"property_id":"314186564","date":"20230406","totalUsers":3014,"newUsers":1539,"sessions":4257,"sessionsPerUser":1.661592505854801,"averageSessionDuration":270.9253856281419,"screenPageViews":10839,"screenPageViewsPerSession":2.5461592670894997,"bounceRate":0.5391120507399577},"emitted_at":1681405971634} -{"stream":"website_overview","data":{"property_id":"314186564","date":"20230403","totalUsers":2988,"newUsers":1461,"sessions":4350,"sessionsPerUser":1.725505751685839,"averageSessionDuration":246.36103450390806,"screenPageViews":10749,"screenPageViewsPerSession":2.4710344827586206,"bounceRate":0.5618390804597702},"emitted_at":1681405971634} -{"stream":"website_overview","data":{"property_id":"314186564","date":"20230404","totalUsers":2817,"newUsers":1367,"sessions":4153,"sessionsPerUser":1.7405699916177704,"averageSessionDuration":259.69049313965803,"screenPageViews":10653,"screenPageViewsPerSession":2.5651336383337346,"bounceRate":0.5379243920057789},"emitted_at":1681405971635} -{"stream":"website_overview","data":{"property_id":"314186564","date":"20230405","totalUsers":2754,"newUsers":1333,"sessions":4004,"sessionsPerUser":1.727351164797239,"averageSessionDuration":290.08648263536463,"screenPageViews":10737,"screenPageViewsPerSession":2.6815684315684316,"bounceRate":0.5072427572427572},"emitted_at":1681405971635} -{"stream":"website_overview","data":{"property_id":"314186564","date":"20230411","totalUsers":2730,"newUsers":1273,"sessions":4006,"sessionsPerUser":1.7820284697508897,"averageSessionDuration":256.8832527284074,"screenPageViews":10073,"screenPageViewsPerSession":2.514478282576136,"bounceRate":0.5162256615077384},"emitted_at":1681405971635} -{"stream":"website_overview","data":{"property_id":"314186564","date":"20230412","totalUsers":2642,"newUsers":1215,"sessions":3940,"sessionsPerUser":1.820702402957486,"averageSessionDuration":281.3629124893401,"screenPageViews":10621,"screenPageViewsPerSession":2.6956852791878174,"bounceRate":0.5309644670050762},"emitted_at":1681405971635} -{"stream":"website_overview","data":{"property_id":"314186564","date":"20230410","totalUsers":2409,"newUsers":1173,"sessions":3602,"sessionsPerUser":1.7822859970311726,"averageSessionDuration":252.51497996779568,"screenPageViews":8973,"screenPageViewsPerSession":2.491116046640755,"bounceRate":0.524153248195447},"emitted_at":1681405971635} -{"stream":"website_overview","data":{"property_id":"314186564","date":"20230407","totalUsers":1950,"newUsers":974,"sessions":2710,"sessionsPerUser":1.6646191646191646,"averageSessionDuration":261.6388968815498,"screenPageViews":6972,"screenPageViewsPerSession":2.572693726937269,"bounceRate":0.5431734317343173},"emitted_at":1681405971635} -{"stream":"website_overview","data":{"property_id":"314186564","date":"20230409","totalUsers":1277,"newUsers":664,"sessions":1661,"sessionsPerUser":1.6461843409316155,"averageSessionDuration":199.5610062384106,"screenPageViews":3300,"screenPageViewsPerSession":1.9867549668874172,"bounceRate":0.5605057194461168},"emitted_at":1681405971635} -{"stream":"website_overview","data":{"property_id":"314186564","date":"20230402","totalUsers":1185,"newUsers":605,"sessions":1505,"sessionsPerUser":1.5388548057259714,"averageSessionDuration":221.2044838358804,"screenPageViews":3260,"screenPageViewsPerSession":2.166112956810631,"bounceRate":0.5348837209302325},"emitted_at":1681405971636} -{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0000","cohortActiveUsers":731},"emitted_at":1681405973101} -{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0001","cohortActiveUsers":25},"emitted_at":1681405973101} -{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0002","cohortActiveUsers":9},"emitted_at":1681405973101} -{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0003","cohortActiveUsers":6},"emitted_at":1681405973101} -{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0004","cohortActiveUsers":4},"emitted_at":1681405973101} -{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0009","cohortActiveUsers":4},"emitted_at":1681405973102} -{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0010","cohortActiveUsers":4},"emitted_at":1681405973102} -{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0011","cohortActiveUsers":4},"emitted_at":1681405973102} -{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0013","cohortActiveUsers":4},"emitted_at":1681405973102} -{"stream":"cohort_report","data":{"property_id":"314186564","cohort":"cohort_0","cohortNthDay":"0025","cohortActiveUsers":4},"emitted_at":1681405973102} -{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"United States","language":"English","sessions":24293,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261616} -{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"India","language":"English","sessions":9419,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261618} -{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Safari","country":"United States","language":"English","sessions":3863,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261618} -{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"Canada","language":"English","sessions":2560,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261618} -{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"United Kingdom","language":"English","sessions":1964,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261618} -{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Edge","country":"United States","language":"English","sessions":1351,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261619} -{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"Australia","language":"English","sessions":1307,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261619} -{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"Brazil","language":"Portuguese","sessions":1302,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261619} -{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"Vietnam","language":"English","sessions":1196,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261619} -{"stream":"pivot_report","data":{"property_id":"314186564","browser":"Chrome","country":"Germany","language":"English","sessions":963,"startDate":"2023-04-01","endDate":"2023-04-30"},"emitted_at":1685012261619} +{"stream": "daily_active_users", "data": {"property_id": "320370985", "date": "20230424", "active1DayUsers": 4}, "emitted_at": 1695314546574} +{"stream": "weekly_active_users", "data": {"property_id": "320370985", "date": "20230425", "active7DayUsers": 5}, "emitted_at": 1695314550248} +{"stream": "four_weekly_active_users", "data": {"property_id": "320370985", "date": "20230425", "active28DayUsers": 5}, "emitted_at": 1695314554589} +{"stream": "devices", "data": {"property_id": "320370985", "date": "20230424", "deviceCategory": "desktop", "operatingSystem": "Windows", "browser": "Edge", "totalUsers": 3, "newUsers": 1, "sessions": 3, "sessionsPerUser": 1.0, "averageSessionDuration": 145.55264966666667, "screenPageViews": 21, "screenPageViewsPerSession": 7.0, "bounceRate": 0.6666666666666666}, "emitted_at": 1695314559264} +{"stream": "locations", "data": {"property_id": "320370985", "region": "Kyiv city", "country": "Ukraine", "city": "Kyiv", "date": "20230424", "totalUsers": 3, "newUsers": 1, "sessions": 3, "sessionsPerUser": 1.0, "averageSessionDuration": 145.55264966666667, "screenPageViews": 21, "screenPageViewsPerSession": 7.0, "bounceRate": 0.6666666666666666}, "emitted_at": 1695314565241} +{"stream": "pages", "data": {"property_id": "320370985", "date": "20230420", "hostName": "integration-test.club", "pagePathPlusQueryString": "/klmuo3qdsubv4yfe/index.php?controller=AdminModules&configure=ps_googleanalytics&token=0cbe63f56492969a25fd47b98dbb2bf6", "screenPageViews": 4, "bounceRate": 0.0}, "emitted_at": 1695314570329} +{"stream": "traffic_sources", "data": {"property_id": "320370985", "date": "20230424", "sessionSource": "(direct)", "sessionMedium": "(none)", "totalUsers": 4, "newUsers": 2, "sessions": 4, "sessionsPerUser": 1.0, "averageSessionDuration": 109.16448725000001, "screenPageViews": 22, "screenPageViewsPerSession": 5.5, "bounceRate": 0.75}, "emitted_at": 1695314575210} +{"stream": "website_overview", "data": {"property_id": "320370985", "date": "20230424", "totalUsers": 4, "newUsers": 2, "sessions": 4, "sessionsPerUser": 1.0, "averageSessionDuration": 109.16448725000001, "screenPageViews": 22, "screenPageViewsPerSession": 5.5, "bounceRate": 0.75}, "emitted_at": 1695314578676} +{"stream": "pivot_report", "data": {"property_id": "320370985", "browser": "Edge", "country": "Ukraine", "language": "English", "sessions": 7, "startDate": "2023-04-01", "endDate": "2023-04-30"}, "emitted_at": 1695314584144} +{"stream": "user_acquisition_first_user_medium_report", "data": {"property_id": "320370985", "date": "20230424", "firstUserMedium": "(none)", "newUsers": 2, "engagedSessions": 1, "engagementRate": 0.25, "eventCount": 80, "conversions": 1.0, "totalRevenue": 35.72, "totalUsers": 4, "userEngagementDuration": 296.0}, "emitted_at": 1695314588898} +{"stream": "user_acquisition_first_user_source_report", "data": {"property_id": "320370985", "date": "20230424", "firstUserSource": "(direct)", "newUsers": 2, "engagedSessions": 1, "engagementRate": 0.25, "eventCount": 80, "conversions": 1.0, "totalRevenue": 35.72, "totalUsers": 4, "userEngagementDuration": 296.0}, "emitted_at": 1695314595303} +{"stream": "user_acquisition_first_user_source_medium_report", "data": {"property_id": "320370985", "date": "20230424", "firstUserSource": "(direct)", "firstUserMedium": "(none)", "newUsers": 2, "engagedSessions": 1, "engagementRate": 0.25, "eventCount": 80, "conversions": 1.0, "totalRevenue": 35.72, "totalUsers": 4, "userEngagementDuration": 296.0}, "emitted_at": 1695314599507} +{"stream": "user_acquisition_first_user_source_platform_report", "data": {"property_id": "320370985", "date": "20230424", "firstUserSourcePlatform": "(not set)", "newUsers": 2, "engagedSessions": 1, "engagementRate": 0.25, "eventCount": 80, "conversions": 1.0, "totalRevenue": 35.72, "totalUsers": 4, "userEngagementDuration": 296.0}, "emitted_at": 1695314604420} +{"stream": "user_acquisition_first_user_campaign_report", "data": {"property_id": "320370985", "date": "20230424", "firstUserCampaignName": "(direct)", "newUsers": 2, "engagedSessions": 1, "engagementRate": 0.25, "eventCount": 80, "conversions": 1.0, "totalRevenue": 35.72, "totalUsers": 4, "userEngagementDuration": 296.0}, "emitted_at": 1695314611280} +{"stream": "user_acquisition_first_user_google_ads_ad_network_type_report", "data": {"property_id": "320370985", "date": "20230424", "firstUserGoogleAdsAdNetworkType": "(not set)", "newUsers": 2, "engagedSessions": 1, "engagementRate": 0.25, "eventCount": 80, "conversions": 1.0, "totalRevenue": 35.72, "totalUsers": 4, "userEngagementDuration": 296.0}, "emitted_at": 1695314618449} +{"stream": "user_acquisition_first_user_google_ads_ad_group_name_report", "data": {"property_id": "320370985", "date": "20230424", "firstUserGoogleAdsAdGroupName": "(not set)", "newUsers": 2, "engagedSessions": 1, "engagementRate": 0.25, "eventCount": 80, "conversions": 1.0, "totalRevenue": 35.72, "totalUsers": 4, "userEngagementDuration": 296.0}, "emitted_at": 1695314623189} +{"stream": "traffic_acquisition_session_source_medium_report", "data": {"property_id": "320370985", "date": "20230424", "sessionSource": "(direct)", "sessionMedium": "(none)", "totalUsers": 4, "sessions": 4, "engagedSessions": 1, "eventsPerSession": 20.0, "engagementRate": 0.25, "eventCount": 80, "conversions": 1.0, "totalRevenue": 35.72, "userEngagementDuration": 296.0}, "emitted_at": 1695314626349} +{"stream": "traffic_acquisition_session_medium_report", "data": {"property_id": "320370985", "date": "20230424", "sessionMedium": "(none)", "totalUsers": 4, "sessions": 4, "engagedSessions": 1, "eventsPerSession": 20.0, "engagementRate": 0.25, "eventCount": 80, "conversions": 1.0, "totalRevenue": 35.72, "userEngagementDuration": 296.0}, "emitted_at": 1695314629603} +{"stream": "traffic_acquisition_session_source_report", "data": {"property_id": "320370985", "date": "20230424", "sessionSource": "(direct)", "totalUsers": 4, "sessions": 4, "engagedSessions": 1, "eventsPerSession": 20.0, "engagementRate": 0.25, "eventCount": 80, "conversions": 1.0, "totalRevenue": 35.72, "userEngagementDuration": 296.0}, "emitted_at": 1695314632467} +{"stream": "traffic_acquisition_session_campaign_report", "data": {"property_id": "320370985", "date": "20230424", "sessionCampaignName": "(direct)", "totalUsers": 4, "sessions": 4, "engagedSessions": 1, "eventsPerSession": 20.0, "engagementRate": 0.25, "eventCount": 80, "conversions": 1.0, "totalRevenue": 35.72, "userEngagementDuration": 296.0}, "emitted_at": 1695314636002} +{"stream": "traffic_acquisition_session_default_channel_grouping_report", "data": {"property_id": "320370985", "date": "20230424", "sessionDefaultChannelGrouping": "Direct", "totalUsers": 4, "sessions": 4, "engagedSessions": 1, "eventsPerSession": 20.0, "engagementRate": 0.25, "eventCount": 80, "conversions": 1.0, "totalRevenue": 35.72, "userEngagementDuration": 296.0}, "emitted_at": 1695314640780} +{"stream": "traffic_acquisition_session_source_platform_report", "data": {"property_id": "320370985", "date": "20230424", "sessionSourcePlatform": "(not set)", "totalUsers": 4, "sessions": 4, "engagedSessions": 1, "eventsPerSession": 20.0, "engagementRate": 0.25, "eventCount": 80, "conversions": 1.0, "totalRevenue": 35.72, "userEngagementDuration": 296.0}, "emitted_at": 1695314644198} +{"stream": "events_report", "data": {"property_id": "320370985", "date": "20230424", "eventName": "page_view", "eventCount": 22, "totalUsers": 4, "eventCountPerUser": 5.5, "totalRevenue": 0.0}, "emitted_at": 1695314648936} +{"stream": "conversions_report", "data": {"property_id": "320370985", "date": "20230424", "eventName": "purchase", "conversions": 1.0, "totalUsers": 1, "totalRevenue": 35.72}, "emitted_at": 1695314653577} +{"stream": "pages_title_and_screen_class_report", "data": {"property_id": "320370985", "date": "20230424", "unifiedScreenClass": "Integration Test", "screenPageViews": 5, "totalUsers": 2, "newUsers": 1, "eventCount": 24, "conversions": 0.0, "totalRevenue": 0.0, "userEngagementDuration": 55.0}, "emitted_at": 1695314657480} +{"stream": "pages_path_report", "data": {"property_id": "320370985", "date": "20230420", "pagePath": "/klmuo3qdsubv4yfe/index.php", "screenPageViews": 6, "totalUsers": 1, "newUsers": 1, "eventCount": 20, "conversions": 0.0, "totalRevenue": 0.0, "userEngagementDuration": 98.0}, "emitted_at": 1695314661338} +{"stream": "pages_title_and_screen_name_report", "data": {"property_id": "320370985", "date": "20230424", "unifiedScreenName": "Integration Test", "screenPageViews": 5, "totalUsers": 2, "newUsers": 1, "eventCount": 24, "conversions": 0.0, "totalRevenue": 0.0, "userEngagementDuration": 55.0}, "emitted_at": 1695314664466} +{"stream": "content_group_report", "data": {"property_id": "320370985", "date": "20230424", "contentGroup": "(not set)", "screenPageViews": 22, "totalUsers": 4, "newUsers": 2, "eventCount": 80, "conversions": 1.0, "totalRevenue": 35.72, "userEngagementDuration": 296.0}, "emitted_at": 1695314667933} +{"stream": "ecommerce_purchases_item_name_report", "data": {"property_id": "320370985", "date": "20230424", "itemName": "Hummingbird printed sweater", "cartToViewRate": 1.0, "purchaseToViewRate": 1.0, "itemsPurchased": 1, "itemRevenue": 28.72, "itemsAddedToCart": 1, "itemsViewed": 1}, "emitted_at": 1695314670777} +{"stream": "ecommerce_purchases_item_id_report", "data": {"property_id": "320370985", "date": "20230424", "itemId": "2", "cartToViewRate": 1.0, "purchaseToViewRate": 1.0, "itemsPurchased": 1, "itemRevenue": 28.72, "itemsAddedToCart": 1, "itemsViewed": 1}, "emitted_at": 1695314673575} +{"stream": "ecommerce_purchases_item_category_report_combined", "data": {"property_id": "320370985", "date": "20230420", "itemCategory": "Home Accessories", "itemCategory2": "(not set)", "itemCategory3": "(not set)", "itemCategory4": "(not set)", "itemCategory5": "(not set)", "cartToViewRate": 0.0, "purchaseToViewRate": 0.0, "itemsPurchased": 0, "itemRevenue": 0.0, "itemsAddedToCart": 0, "itemsViewed": 1}, "emitted_at": 1695314677000} +{"stream": "ecommerce_purchases_item_category_report", "data": {"property_id": "320370985", "date": "20230420", "itemCategory": "Home Accessories", "cartToViewRate": 0.0, "purchaseToViewRate": 0.0, "itemsPurchased": 0, "itemRevenue": 0.0, "itemsAddedToCart": 0, "itemsViewed": 1}, "emitted_at": 1695314680621} +{"stream": "ecommerce_purchases_item_category_2_report", "data": {"property_id": "320370985", "date": "20230424", "itemCategory2": "(not set)", "cartToViewRate": 1.0, "purchaseToViewRate": 1.0, "itemsPurchased": 1, "itemRevenue": 28.72, "itemsAddedToCart": 1, "itemsViewed": 1}, "emitted_at": 1695314683859} +{"stream": "ecommerce_purchases_item_category_3_report", "data": {"property_id": "320370985", "date": "20230424", "itemCategory3": "(not set)", "cartToViewRate": 1.0, "purchaseToViewRate": 1.0, "itemsPurchased": 1, "itemRevenue": 28.72, "itemsAddedToCart": 1, "itemsViewed": 1}, "emitted_at": 1695314686389} +{"stream": "ecommerce_purchases_item_category_4_report", "data": {"property_id": "320370985", "date": "20230424", "itemCategory4": "(not set)", "cartToViewRate": 1.0, "purchaseToViewRate": 1.0, "itemsPurchased": 1, "itemRevenue": 28.72, "itemsAddedToCart": 1, "itemsViewed": 1}, "emitted_at": 1695314688847} +{"stream": "ecommerce_purchases_item_category_5_report", "data": {"property_id": "320370985", "date": "20230424", "itemCategory5": "(not set)", "cartToViewRate": 1.0, "purchaseToViewRate": 1.0, "itemsPurchased": 1, "itemRevenue": 28.72, "itemsAddedToCart": 1, "itemsViewed": 1}, "emitted_at": 1695314691680} +{"stream": "ecommerce_purchases_item_brand_report", "data": {"property_id": "320370985", "date": "20230420", "itemBrand": "", "cartToViewRate": 0.0, "purchaseToViewRate": 0.0, "itemsPurchased": 0, "itemRevenue": 0.0, "itemsAddedToCart": 0, "itemsViewed": 1}, "emitted_at": 1695314694040} +{"stream": "demographic_country_report", "data": {"property_id": "320370985", "date": "20230424", "country": "Ukraine", "totalUsers": 3, "newUsers": 1, "engagedSessions": 1, "engagementRate": 0.3333333333333333, "conversions": 1.0, "totalRevenue": 35.72}, "emitted_at": 1695314708261} +{"stream": "demographic_region_report", "data": {"property_id": "320370985", "date": "20230424", "region": "Kyiv city", "totalUsers": 3, "newUsers": 1, "engagedSessions": 1, "engagementRate": 0.3333333333333333, "conversions": 1.0, "totalRevenue": 35.72}, "emitted_at": 1695314711057} +{"stream": "demographic_city_report", "data": {"property_id": "320370985", "date": "20230424", "city": "Kyiv", "totalUsers": 3, "newUsers": 1, "engagedSessions": 1, "engagementRate": 0.3333333333333333, "conversions": 1.0, "totalRevenue": 35.72}, "emitted_at": 1695314714227} +{"stream": "demographic_language_report", "data": {"property_id": "320370985", "date": "20230424", "language": "English", "totalUsers": 4, "newUsers": 2, "engagedSessions": 1, "engagementRate": 0.25, "conversions": 1.0, "totalRevenue": 35.72}, "emitted_at": 1695314717303} +{"stream": "tech_browser_report", "data": {"property_id": "320370985", "date": "20230424", "browser": "Edge", "totalUsers": 3, "newUsers": 1, "engagedSessions": 1, "engagementRate": 0.3333333333333333, "eventCount": 77, "conversions": 1.0, "totalRevenue": 35.72}, "emitted_at": 1695314732162} +{"stream": "tech_device_category_report", "data": {"property_id": "320370985", "date": "20230424", "deviceCategory": "desktop", "totalUsers": 4, "newUsers": 2, "engagedSessions": 1, "engagementRate": 0.25, "eventCount": 80, "conversions": 1.0, "totalRevenue": 35.72}, "emitted_at": 1695314735763} +{"stream": "tech_device_model_report", "data": {"property_id": "320370985", "date": "20230424", "deviceModel": "(not set)", "totalUsers": 4, "newUsers": 2, "engagedSessions": 1, "engagementRate": 0.25, "eventCount": 80, "conversions": 1.0, "totalRevenue": 35.72}, "emitted_at": 1695314739817} +{"stream": "tech_screen_resolution_report", "data": {"property_id": "320370985", "date": "20230424", "screenResolution": "1536x864", "totalUsers": 3, "newUsers": 1, "engagedSessions": 1, "engagementRate": 0.3333333333333333, "eventCount": 77, "conversions": 1.0, "totalRevenue": 35.72}, "emitted_at": 1695314743588} +{"stream": "tech_app_version_report", "data": {"property_id": "320370985", "date": "20230424", "appVersion": "(not set)", "totalUsers": 4, "newUsers": 2, "engagedSessions": 1, "engagementRate": 0.25, "eventCount": 80, "conversions": 1.0, "totalRevenue": 35.72}, "emitted_at": 1695314747185} +{"stream": "tech_platform_report", "data": {"property_id": "320370985", "date": "20230424", "platform": "web", "totalUsers": 4, "newUsers": 2, "engagedSessions": 1, "engagementRate": 0.25, "eventCount": 80, "conversions": 1.0, "totalRevenue": 35.72}, "emitted_at": 1695314750338} +{"stream": "tech_platform_device_category_report", "data": {"property_id": "320370985", "date": "20230424", "platform": "web", "deviceCategory": "desktop", "totalUsers": 4, "newUsers": 2, "engagedSessions": 1, "engagementRate": 0.25, "eventCount": 80, "conversions": 1.0, "totalRevenue": 35.72}, "emitted_at": 1695314753532} +{"stream": "tech_operating_system_report", "data": {"property_id": "320370985", "date": "20230424", "operatingSystem": "Windows", "totalUsers": 3, "newUsers": 1, "engagedSessions": 1, "engagementRate": 0.3333333333333333, "eventCount": 77, "conversions": 1.0, "totalRevenue": 35.72}, "emitted_at": 1695314758191} +{"stream": "tech_os_with_version_report", "data": {"property_id": "320370985", "date": "20230424", "operatingSystemWithVersion": "Windows 10", "totalUsers": 3, "newUsers": 1, "engagedSessions": 1, "engagementRate": 0.3333333333333333, "eventCount": 77, "conversions": 1.0, "totalRevenue": 35.72}, "emitted_at": 1695314762337} +{"stream": "weekly_events_report", "data": {"property_id": "320370985", "yearWeek": "202317", "eventName": "page_view", "eventCount": 23, "totalUsers": 5, "eventCountPerUser": 4.6, "totalRevenue": 0.0, "startDate": "2023-04-01", "endDate": "2023-04-30"}, "emitted_at": 1695314765979} diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/invalid_config.json index 10e4173e92e8..ade383bf022c 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/integration_tests/invalid_config.json @@ -1,5 +1,5 @@ { - "property_id": "1", + "property_id": ["1"], "json_credentials": "wrong", "report_name": "crash_report", "dimensions": "date, operatingSystem, streamId", diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/main.py b/airbyte-integrations/connectors/source-google-analytics-data-api/main.py index 3e6c130dfeb8..93839ed0e51a 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/main.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_google_analytics_data_api import SourceGoogleAnalyticsDataApi +from source_google_analytics_data_api.run import run if __name__ == "__main__": - source = SourceGoogleAnalyticsDataApi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/metadata.yaml b/airbyte-integrations/connectors/source-google-analytics-data-api/metadata.yaml index 26825ee071fc..d91736309fd9 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/metadata.yaml @@ -1,14 +1,20 @@ data: + ab_internal: + ql: 400 + sl: 300 allowedHosts: hosts: - oauth2.googleapis.com - www.googleapis.com - analyticsdata.googleapis.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 3cc2eafd-84aa-4dca-93af-322d9dfeec1a - dockerImageTag: 1.1.3 + dockerImageTag: 2.2.1 dockerRepository: airbyte/source-google-analytics-data-api + documentationUrl: https://docs.airbyte.com/integrations/sources/google-analytics-data-api githubIssueLabel: source-google-analytics-data-api icon: google-analytics.svg license: Elv2 @@ -19,11 +25,27 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/google-analytics-data-api + releases: + breakingChanges: + 2.0.0: + message: + Version 2.0.0 introduces changes to stream names for those syncing + more than one Google Analytics 4 property. It allows streams from all properties + to sync successfully. Please upgrade the connector to enable this additional + functionality. + upgradeDeadline: "2023-10-16" + suggestedStreams: + streams: + - website_overview + - daily_active_users + - traffic_sources + - pages + - weekly_active_users + - devices + - locations + - four_weekly_active_users + - sessions + supportLevel: certified tags: - language:python - ab_internal: - sl: 300 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/setup.py b/airbyte-integrations/connectors/source-google-analytics-data-api/setup.py index d93d394a4354..f2a10ce3101c 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/setup.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk", "PyJWT==2.4.0", "cryptography==37.0.4", "requests"] +MAIN_REQUIREMENTS = ["airbyte-cdk", "PyJWT==2.4.0", "cryptography==37.0.4", "requests", "pandas"] TEST_REQUIREMENTS = [ "freezegun", @@ -15,6 +15,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-google-analytics-data-api=source_google_analytics_data_api.run:run", + ], + }, name="source_google_analytics_data_api", description="Source implementation for Google Analytics Data Api.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/api_quota.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/api_quota.py index 7c476baae98f..e90560e8bfee 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/api_quota.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/api_quota.py @@ -8,6 +8,7 @@ from typing import Any, Iterable, Mapping, Optional import requests +from requests.exceptions import JSONDecodeError from .utils import API_LIMIT_PER_HOUR @@ -144,7 +145,7 @@ def _check_for_errors(self, response: requests.Response) -> None: self._set_retry_attrs_for_quota(quota_name) self.logger.warn(f"The `{quota_name}` quota is exceeded!") return None - except AttributeError as attr_e: + except (AttributeError, JSONDecodeError) as attr_e: self.logger.warning( f"`GoogleAnalyticsApiQuota._check_for_errors`: Received non JSON response from the API. Full error: {attr_e}. Bypassing." ) @@ -159,7 +160,7 @@ def _check_quota(self, response: requests.Response): # try get json from response try: parsed_response = response.json() - except AttributeError as e: + except (AttributeError, JSONDecodeError) as e: self.logger.warn( f"`GoogleAnalyticsApiQuota._check_quota`: Received non JSON response from the API. Full error: {e}. Bypassing." ) diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/config_migrations.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/config_migrations.py new file mode 100644 index 000000000000..621b5bbafcaf --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/config_migrations.py @@ -0,0 +1,236 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import logging +from typing import Any, List, Mapping + +import dpath.util +from airbyte_cdk.config_observation import create_connector_config_control_message +from airbyte_cdk.entrypoint import AirbyteEntrypoint +from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + +from .source import SourceGoogleAnalyticsDataApi + +logger = logging.getLogger("airbyte_logger") + + +class MigratePropertyID: + """ + This class stands for migrating the config at runtime, + while providing the backward compatibility when falling back to the previous source version. + + Specifically, starting from `1.3.0`, the `property_id` property should be like : + > List(["", "", ..., ""]) + instead of, in `1.2.0`: + > JSON STR: "" + """ + + message_repository: MessageRepository = InMemoryMessageRepository() + migrate_from_key: str = "property_id" + migrate_to_key: str = "property_ids" + + @classmethod + def _should_migrate(cls, config: Mapping[str, Any]) -> bool: + """ + This method determines whether config requires migration. + Returns: + > True, if the transformation is necessary + > False, otherwise. + """ + if cls.migrate_from_key in config: + return True + return False + + @classmethod + def _transform_to_array(cls, config: Mapping[str, Any], source: SourceGoogleAnalyticsDataApi = None) -> Mapping[str, Any]: + # assign old values to new property that will be used within the new version + config[cls.migrate_to_key] = config[cls.migrate_to_key] if cls.migrate_to_key in config else [] + data = config.pop(cls.migrate_from_key) + if data not in config[cls.migrate_to_key]: + config[cls.migrate_to_key].append(data) + return config + + @classmethod + def _modify_and_save(cls, config_path: str, source: SourceGoogleAnalyticsDataApi, config: Mapping[str, Any]) -> Mapping[str, Any]: + # modify the config + migrated_config = cls._transform_to_array(config, source) + # save the config + source.write_config(migrated_config, config_path) + # return modified config + return migrated_config + + @classmethod + def _emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: + # add the Airbyte Control Message to message repo + cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) + # emit the Airbyte Control Message from message queue to stdout + for message in cls.message_repository.consume_queue(): + print(message.json(exclude_unset=True)) + + @classmethod + def migrate(cls, args: List[str], source: SourceGoogleAnalyticsDataApi) -> None: + """ + This method checks the input args, should the config be migrated, + transform if necessary and emit the CONTROL message. + """ + # get config path + config_path = AirbyteEntrypoint(source).extract_config(args) + # proceed only if `--config` arg is provided + if config_path: + # read the existing config + config = source.read_config(config_path) + # migration check + if cls._should_migrate(config): + cls._emit_control_message( + cls._modify_and_save(config_path, source, config), + ) + + +class MigrateCustomReports: + """ + This class stands for migrating the config at runtime, + while providing the backward compatibility when falling back to the previous source version. + Specifically, starting from `1.3.3`, the `custom_reports` property should be like : + > List([{name: my_report}, {dimensions: [a,b,c]}], [], ...) + instead of, in `1.3.2`: + > JSON STR: "{name: my_report}, {dimensions: [a,b,c]}" + """ + + message_repository: MessageRepository = InMemoryMessageRepository() + migrate_from_key: str = "custom_reports" + migrate_to_key: str = "custom_reports_array" + + @classmethod + def _should_migrate(cls, config: Mapping[str, Any]) -> bool: + """ + This method determines whether the config should be migrated to have the new structure for the `custom_reports`, + based on the source spec. + Returns: + > True, if the transformation is necessary + > False, otherwise. + > Raises the Exception if the structure could not be migrated. + """ + + # If the config has been migrated and has entries, no need to migrate again. + if config.get(cls.migrate_to_key, []): + return False + + # If the old config key is present and its value is a string, migration is needed. + if config.get(cls.migrate_from_key, None) is not None and isinstance(config[cls.migrate_from_key], str): + return True + + return False + + @classmethod + def _transform_to_array(cls, config: Mapping[str, Any], source: SourceGoogleAnalyticsDataApi = None) -> Mapping[str, Any]: + # assign old values to new property that will be used within the new version + config[cls.migrate_to_key] = config[cls.migrate_from_key] + # transform `json_str` to `list` of objects + return source._validate_custom_reports(config) + + @classmethod + def _modify_and_save(cls, config_path: str, source: SourceGoogleAnalyticsDataApi, config: Mapping[str, Any]) -> Mapping[str, Any]: + # modify the config + migrated_config = cls._transform_to_array(config, source) + # save the config + source.write_config(migrated_config, config_path) + # return modified config + return migrated_config + + @classmethod + def _emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: + # add the Airbyte Control Message to message repo + cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) + # emit the Airbyte Control Message from message queue to stdout + for message in cls.message_repository.consume_queue(): + print(message.json(exclude_unset=True)) + + @classmethod + def migrate(cls, args: List[str], source: SourceGoogleAnalyticsDataApi) -> None: + """ + This method checks the input args, should the config be migrated, + transform if necessary and emit the CONTROL message. + """ + # get config path + config_path = AirbyteEntrypoint(source).extract_config(args) + # proceed only if `--config` arg is provided + if config_path: + # read the existing config + config = source.read_config(config_path) + # migration check + if cls._should_migrate(config): + cls._emit_control_message( + cls._modify_and_save(config_path, source, config), + ) + + +class MigrateCustomReportsCohortSpec: + """ + This class stands for migrating the config at runtime, + Specifically, starting from `2.1.0`; the `cohortSpec` property will be added tp `custom_reports_array` with flag `enabled`: + > List([{name: my_report, "cohortSpec": { "enabled": "true" } }, ...]) + """ + + message_repository: MessageRepository = InMemoryMessageRepository() + + @classmethod + def _should_migrate(cls, config: Mapping[str, Any]) -> bool: + """ + This method determines whether the config should be migrated to have the new structure for the `cohortSpec` inside `custom_reports`, + based on the source spec. + Returns: + > True, if the transformation is necessary + > False, otherwise. + """ + + return not dpath.util.search(config, "custom_reports_array/**/cohortSpec/enabled") + + @classmethod + def _transform_custom_reports_cohort_spec( + cls, + config: Mapping[str, Any], + ) -> Mapping[str, Any]: + """Assign `enabled` property that will be used within the new version""" + for report in config.get("custom_reports_array", []): + if report.get("cohortSpec"): + report["cohortSpec"]["enabled"] = "true" + else: + report.setdefault("cohortSpec", {})["enabled"] = "false" + return config + + @classmethod + def _modify_and_save(cls, config_path: str, source: SourceGoogleAnalyticsDataApi, config: Mapping[str, Any]) -> Mapping[str, Any]: + # modify the config + migrated_config = cls._transform_custom_reports_cohort_spec(config) + # save the config + source.write_config(migrated_config, config_path) + # return modified config + return migrated_config + + @classmethod + def _emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: + # add the Airbyte Control Message to message repo + cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) + # emit the Airbyte Control Message from message queue to stdout + for message in cls.message_repository.consume_queue(): + print(message.json(exclude_unset=True)) + + @classmethod + def migrate(cls, args: List[str], source: SourceGoogleAnalyticsDataApi) -> None: + """ + This method checks the input args, should the config be migrated, + transform if necessary and emit the CONTROL message. + """ + # get config path + config_path = AirbyteEntrypoint(source).extract_config(args) + # proceed only if `--config` arg is provided + if config_path: + # read the existing config + config = source.read_config(config_path) + # migration check + if cls._should_migrate(config): + cls._emit_control_message( + cls._modify_and_save(config_path, source, config), + ) diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/defaults/default_reports.json b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/defaults/default_reports.json index 53ad0ee14727..559c52f37852 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/defaults/default_reports.json +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/defaults/default_reports.json @@ -74,5 +74,616 @@ "screenPageViewsPerSession", "bounceRate" ] + }, + { + "name": "user_acquisition_first_user_medium_report", + "dimensions": ["date", "firstUserMedium"], + "metrics": [ + "newUsers", + "engagedSessions", + "engagementRate", + "eventCount", + "conversions", + "totalRevenue", + "totalUsers", + "userEngagementDuration" + ] + }, + { + "name": "user_acquisition_first_user_source_report", + "dimensions": ["date", "firstUserSource"], + "metrics": [ + "newUsers", + "engagedSessions", + "engagementRate", + "eventCount", + "conversions", + "totalRevenue", + "totalUsers", + "userEngagementDuration" + ] + }, + { + "name": "user_acquisition_first_user_source_medium_report", + "dimensions": ["date", "firstUserSource", "firstUserMedium"], + "metrics": [ + "newUsers", + "engagedSessions", + "engagementRate", + "eventCount", + "conversions", + "totalRevenue", + "totalUsers", + "userEngagementDuration" + ] + }, + { + "name": "user_acquisition_first_user_source_platform_report", + "dimensions": ["date", "firstUserSourcePlatform"], + "metrics": [ + "newUsers", + "engagedSessions", + "engagementRate", + "eventCount", + "conversions", + "totalRevenue", + "totalUsers", + "userEngagementDuration" + ] + }, + { + "name": "user_acquisition_first_user_campaign_report", + "dimensions": ["date", "firstUserCampaignName"], + "metrics": [ + "newUsers", + "engagedSessions", + "engagementRate", + "eventCount", + "conversions", + "totalRevenue", + "totalUsers", + "userEngagementDuration" + ] + }, + { + "name": "user_acquisition_first_user_google_ads_ad_network_type_report", + "dimensions": ["date", "firstUserGoogleAdsAdNetworkType"], + "metrics": [ + "newUsers", + "engagedSessions", + "engagementRate", + "eventCount", + "conversions", + "totalRevenue", + "totalUsers", + "userEngagementDuration" + ] + }, + { + "name": "user_acquisition_first_user_google_ads_ad_group_name_report", + "dimensions": ["date", "firstUserGoogleAdsAdGroupName"], + "metrics": [ + "newUsers", + "engagedSessions", + "engagementRate", + "eventCount", + "conversions", + "totalRevenue", + "totalUsers", + "userEngagementDuration" + ] + }, + { + "name": "traffic_acquisition_session_source_medium_report", + "dimensions": ["date", "sessionSource", "sessionMedium"], + "metrics": [ + "totalUsers", + "sessions", + "engagedSessions", + "eventsPerSession", + "engagementRate", + "eventCount", + "conversions", + "totalRevenue", + "userEngagementDuration" + ] + }, + { + "name": "traffic_acquisition_session_medium_report", + "dimensions": ["date", "sessionMedium"], + "metrics": [ + "totalUsers", + "sessions", + "engagedSessions", + "eventsPerSession", + "engagementRate", + "eventCount", + "conversions", + "totalRevenue", + "userEngagementDuration" + ] + }, + { + "name": "traffic_acquisition_session_source_report", + "dimensions": ["date", "sessionSource"], + "metrics": [ + "totalUsers", + "sessions", + "engagedSessions", + "eventsPerSession", + "engagementRate", + "eventCount", + "conversions", + "totalRevenue", + "userEngagementDuration" + ] + }, + { + "name": "traffic_acquisition_session_campaign_report", + "dimensions": ["date", "sessionCampaignName"], + "metrics": [ + "totalUsers", + "sessions", + "engagedSessions", + "eventsPerSession", + "engagementRate", + "eventCount", + "conversions", + "totalRevenue", + "userEngagementDuration" + ] + }, + { + "name": "traffic_acquisition_session_default_channel_grouping_report", + "dimensions": ["date", "sessionDefaultChannelGrouping"], + "metrics": [ + "totalUsers", + "sessions", + "engagedSessions", + "eventsPerSession", + "engagementRate", + "eventCount", + "conversions", + "totalRevenue", + "userEngagementDuration" + ] + }, + { + "name": "traffic_acquisition_session_source_platform_report", + "dimensions": ["date", "sessionSourcePlatform"], + "metrics": [ + "totalUsers", + "sessions", + "engagedSessions", + "eventsPerSession", + "engagementRate", + "eventCount", + "conversions", + "totalRevenue", + "userEngagementDuration" + ] + }, + { + "name": "events_report", + "dimensions": ["date", "eventName"], + "metrics": ["eventCount", "totalUsers", "eventCountPerUser", "totalRevenue"] + }, + { + "name": "weekly_events_report", + "dimensions": ["yearWeek", "eventName"], + "metrics": ["eventCount", "totalUsers", "eventCountPerUser", "totalRevenue"] + }, + { + "name": "conversions_report", + "dimensions": ["date", "eventName"], + "metrics": ["conversions", "totalUsers", "totalRevenue"] + }, + { + "name": "pages_title_and_screen_class_report", + "dimensions": ["date", "unifiedScreenClass"], + "metrics": [ + "screenPageViews", + "totalUsers", + "newUsers", + "eventCount", + "conversions", + "totalRevenue", + "userEngagementDuration" + ] + }, + { + "name": "pages_path_report", + "dimensions": ["date", "pagePath"], + "metrics": [ + "screenPageViews", + "totalUsers", + "newUsers", + "eventCount", + "conversions", + "totalRevenue", + "userEngagementDuration" + ] + }, + { + "name": "pages_title_and_screen_name_report", + "dimensions": ["date", "unifiedScreenName"], + "metrics": [ + "screenPageViews", + "totalUsers", + "newUsers", + "eventCount", + "conversions", + "totalRevenue", + "userEngagementDuration" + ] + }, + { + "name": "content_group_report", + "dimensions": ["date", "contentGroup"], + "metrics": [ + "screenPageViews", + "totalUsers", + "newUsers", + "eventCount", + "conversions", + "totalRevenue", + "userEngagementDuration" + ] + }, + { + "name": "ecommerce_purchases_item_name_report", + "dimensions": ["date", "itemName"], + "metrics": [ + "cartToViewRate", + "purchaseToViewRate", + "itemsPurchased", + "itemRevenue", + "itemsAddedToCart", + "itemsViewed" + ] + }, + { + "name": "ecommerce_purchases_item_id_report", + "dimensions": ["date", "itemId"], + "metrics": [ + "cartToViewRate", + "purchaseToViewRate", + "itemsPurchased", + "itemRevenue", + "itemsAddedToCart", + "itemsViewed" + ] + }, + { + "name": "ecommerce_purchases_item_category_report_combined", + "dimensions": [ + "date", + "itemCategory", + "itemCategory2", + "itemCategory3", + "itemCategory4", + "itemCategory5" + ], + "metrics": [ + "cartToViewRate", + "purchaseToViewRate", + "itemsPurchased", + "itemRevenue", + "itemsAddedToCart", + "itemsViewed" + ] + }, + { + "name": "ecommerce_purchases_item_category_report", + "dimensions": ["date", "itemCategory"], + "metrics": [ + "cartToViewRate", + "purchaseToViewRate", + "itemsPurchased", + "itemRevenue", + "itemsAddedToCart", + "itemsViewed" + ] + }, + { + "name": "ecommerce_purchases_item_category_2_report", + "dimensions": ["date", "itemCategory2"], + "metrics": [ + "cartToViewRate", + "purchaseToViewRate", + "itemsPurchased", + "itemRevenue", + "itemsAddedToCart", + "itemsViewed" + ] + }, + { + "name": "ecommerce_purchases_item_category_3_report", + "dimensions": ["date", "itemCategory3"], + "metrics": [ + "cartToViewRate", + "purchaseToViewRate", + "itemsPurchased", + "itemRevenue", + "itemsAddedToCart", + "itemsViewed" + ] + }, + { + "name": "ecommerce_purchases_item_category_4_report", + "dimensions": ["date", "itemCategory4"], + "metrics": [ + "cartToViewRate", + "purchaseToViewRate", + "itemsPurchased", + "itemRevenue", + "itemsAddedToCart", + "itemsViewed" + ] + }, + { + "name": "ecommerce_purchases_item_category_5_report", + "dimensions": ["date", "itemCategory5"], + "metrics": [ + "cartToViewRate", + "purchaseToViewRate", + "itemsPurchased", + "itemRevenue", + "itemsAddedToCart", + "itemsViewed" + ] + }, + { + "name": "ecommerce_purchases_item_brand_report", + "dimensions": ["date", "itemBrand"], + "metrics": [ + "cartToViewRate", + "purchaseToViewRate", + "itemsPurchased", + "itemRevenue", + "itemsAddedToCart", + "itemsViewed" + ] + }, + { + "name": "publisher_ads_ad_unit_report", + "dimensions": ["date", "adUnitName"], + "metrics": [ + "publisherAdImpressions", + "adUnitExposure", + "publisherAdClicks", + "totalAdRevenue" + ] + }, + { + "name": "publisher_ads_page_path_report", + "dimensions": ["date", "pagePath"], + "metrics": [ + "publisherAdImpressions", + "adUnitExposure", + "publisherAdClicks", + "totalAdRevenue" + ] + }, + { + "name": "publisher_ads_ad_format_report", + "dimensions": ["date", "adFormat"], + "metrics": [ + "publisherAdImpressions", + "adUnitExposure", + "publisherAdClicks", + "totalAdRevenue" + ] + }, + { + "name": "publisher_ads_ad_source_report", + "dimensions": ["date", "adSourceName"], + "metrics": [ + "publisherAdImpressions", + "adUnitExposure", + "publisherAdClicks", + "totalAdRevenue" + ] + }, + { + "name": "demographic_country_report", + "dimensions": ["date", "country"], + "metrics": [ + "totalUsers", + "newUsers", + "engagedSessions", + "engagementRate", + "conversions", + "totalRevenue" + ] + }, + { + "name": "demographic_region_report", + "dimensions": ["date", "region"], + "metrics": [ + "totalUsers", + "newUsers", + "engagedSessions", + "engagementRate", + "conversions", + "totalRevenue" + ] + }, + { + "name": "demographic_city_report", + "dimensions": ["date", "city"], + "metrics": [ + "totalUsers", + "newUsers", + "engagedSessions", + "engagementRate", + "conversions", + "totalRevenue" + ] + }, + { + "name": "demographic_language_report", + "dimensions": ["date", "language"], + "metrics": [ + "totalUsers", + "newUsers", + "engagedSessions", + "engagementRate", + "conversions", + "totalRevenue" + ] + }, + { + "name": "demographic_age_report", + "dimensions": ["date", "userAgeBracket"], + "metrics": [ + "totalUsers", + "newUsers", + "engagedSessions", + "engagementRate", + "conversions", + "totalRevenue" + ] + }, + { + "name": "demographic_gender_report", + "dimensions": ["date", "userGender"], + "metrics": [ + "totalUsers", + "newUsers", + "engagedSessions", + "engagementRate", + "conversions", + "totalRevenue" + ] + }, + { + "name": "demographic_interest_report", + "dimensions": ["date", "brandingInterest"], + "metrics": [ + "totalUsers", + "newUsers", + "engagedSessions", + "engagementRate", + "conversions", + "totalRevenue" + ] + }, + { + "name": "tech_browser_report", + "dimensions": ["date", "browser"], + "metrics": [ + "totalUsers", + "newUsers", + "engagedSessions", + "engagementRate", + "eventCount", + "conversions", + "totalRevenue" + ] + }, + { + "name": "tech_device_category_report", + "dimensions": ["date", "deviceCategory"], + "metrics": [ + "totalUsers", + "newUsers", + "engagedSessions", + "engagementRate", + "eventCount", + "conversions", + "totalRevenue" + ] + }, + { + "name": "tech_device_model_report", + "dimensions": ["date", "deviceModel"], + "metrics": [ + "totalUsers", + "newUsers", + "engagedSessions", + "engagementRate", + "eventCount", + "conversions", + "totalRevenue" + ] + }, + { + "name": "tech_screen_resolution_report", + "dimensions": ["date", "screenResolution"], + "metrics": [ + "totalUsers", + "newUsers", + "engagedSessions", + "engagementRate", + "eventCount", + "conversions", + "totalRevenue" + ] + }, + { + "name": "tech_app_version_report", + "dimensions": ["date", "appVersion"], + "metrics": [ + "totalUsers", + "newUsers", + "engagedSessions", + "engagementRate", + "eventCount", + "conversions", + "totalRevenue" + ] + }, + { + "name": "tech_platform_report", + "dimensions": ["date", "platform"], + "metrics": [ + "totalUsers", + "newUsers", + "engagedSessions", + "engagementRate", + "eventCount", + "conversions", + "totalRevenue" + ] + }, + { + "name": "tech_platform_device_category_report", + "dimensions": ["date", "platform", "deviceCategory"], + "metrics": [ + "totalUsers", + "newUsers", + "engagedSessions", + "engagementRate", + "eventCount", + "conversions", + "totalRevenue" + ] + }, + { + "name": "tech_operating_system_report", + "dimensions": ["date", "operatingSystem"], + "metrics": [ + "totalUsers", + "newUsers", + "engagedSessions", + "engagementRate", + "eventCount", + "conversions", + "totalRevenue" + ] + }, + { + "name": "tech_os_with_version_report", + "dimensions": ["date", "operatingSystemWithVersion"], + "metrics": [ + "totalUsers", + "newUsers", + "engagedSessions", + "engagementRate", + "eventCount", + "conversions", + "totalRevenue" + ] } ] diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/run.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/run.py new file mode 100644 index 000000000000..ed4ec25e9250 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/run.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_google_analytics_data_api import SourceGoogleAnalyticsDataApi +from source_google_analytics_data_api.config_migrations import MigrateCustomReports, MigrateCustomReportsCohortSpec, MigratePropertyID + + +def run(): + source = SourceGoogleAnalyticsDataApi() + MigratePropertyID.migrate(sys.argv[1:], source) + MigrateCustomReports.migrate(sys.argv[1:], source) + MigrateCustomReportsCohortSpec.migrate(sys.argv[1:], source) + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py index 5d7091cf431c..d58bb1673ac7 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py @@ -13,6 +13,7 @@ import dpath import jsonschema +import pendulum import requests from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.sources import AbstractSource @@ -21,7 +22,13 @@ from airbyte_cdk.utils import AirbyteTracedException from requests import HTTPError from source_google_analytics_data_api import utils -from source_google_analytics_data_api.utils import DATE_FORMAT, WRONG_DIMENSIONS, WRONG_JSON_SYNTAX, WRONG_METRICS +from source_google_analytics_data_api.utils import ( + DATE_FORMAT, + WRONG_CUSTOM_REPORT_CONFIG, + WRONG_DIMENSIONS, + WRONG_JSON_SYNTAX, + WRONG_METRICS, +) from .api_quota import GoogleAnalyticsApiQuota from .utils import ( @@ -32,10 +39,12 @@ get_metrics_type, get_source_defined_primary_key, metrics_type_to_python, + serialize_to_date_string, + transform_json, ) -# set the quota handler globaly since limitations are the same for all streams -# the initial values should be saved once and tracked for each stream, inclusivelly. +# set the quota handler globally since limitations are the same for all streams +# the initial values should be saved once and tracked for each stream, inclusively. GoogleAnalyticsQuotaHandler: GoogleAnalyticsApiQuota = GoogleAnalyticsApiQuota() LOOKBACK_WINDOW = datetime.timedelta(days=2) @@ -52,7 +61,20 @@ def __init__(self): def __get__(self, instance, owner): if not self._metadata: stream = GoogleAnalyticsDataApiMetadataStream(config=instance.config, authenticator=instance.config["authenticator"]) - metadata = next(stream.read_records(sync_mode=SyncMode.full_refresh), None) + + metadata = None + try: + metadata = next(stream.read_records(sync_mode=SyncMode.full_refresh), None) + except HTTPError as e: + if e.response.status_code in [HTTPStatus.UNAUTHORIZED, HTTPStatus.FORBIDDEN]: + internal_message = "Unauthorized error reached." + message = "Can not get metadata with unauthorized credentials. Try to re-authenticate in source settings." + + unauthorized_error = AirbyteTracedException( + message=message, internal_message=internal_message, failure_type=FailureType.config_error + ) + raise unauthorized_error + if not metadata: raise Exception("failed to get metadata, over quota, try later") self._metadata = { @@ -118,7 +140,11 @@ class GoogleAnalyticsDataApiBaseStream(GoogleAnalyticsDataApiAbstractStream): @property def cursor_field(self) -> Optional[str]: - return "date" if "date" in self.config.get("dimensions", []) else [] + date_fields = ["date", "yearWeek", "yearMonth", "year"] + for field in date_fields: + if field in self.config.get("dimensions", []): + return field + return [] @property def primary_key(self): @@ -156,7 +182,10 @@ def get_json_schema(self) -> Mapping[str, Any]: schema["properties"].update( { - d: {"type": get_dimensions_type(d), "description": self.metadata["dimensions"].get(d, {}).get("description", d)} + d: { + "type": get_dimensions_type(d), + "description": self.metadata["dimensions"].get(d, {}).get("description", d), + } for d in self.config["dimensions"] } ) @@ -215,7 +244,7 @@ def parse_response( dimensions = [h.get("name") for h in r.get("dimensionHeaders", [{}])] metrics = [h.get("name") for h in r.get("metricHeaders", [{}])] - metrics_type_map = {h.get("name"): h.get("type") for h in r.get("metricHeaders", [{}])} + metrics_type_map = {h.get("name"): h.get("type") for h in r.get("metricHeaders", [{}]) if "name" in h} for row in r.get("rows", []): record = { @@ -238,12 +267,22 @@ def parse_response( yield record def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): - updated_state = utils.string_to_date(latest_record[self.cursor_field], self._record_date_format) + updated_state = ( + utils.string_to_date(latest_record[self.cursor_field], self._record_date_format) + if self.cursor_field == "date" + else latest_record[self.cursor_field] + ) stream_state_value = current_stream_state.get(self.cursor_field) if stream_state_value: - stream_state_value = utils.string_to_date(stream_state_value, self._record_date_format, old_format=DATE_FORMAT) + stream_state_value = ( + utils.string_to_date(stream_state_value, self._record_date_format, old_format=DATE_FORMAT) + if self.cursor_field == "date" + else stream_state_value + ) updated_state = max(updated_state, stream_state_value) - current_stream_state[self.cursor_field] = updated_state.strftime(self._record_date_format) + current_stream_state[self.cursor_field] = ( + updated_state.strftime(self._record_date_format) if self.cursor_field == "date" else updated_state + ) return current_stream_state def request_body_json( @@ -252,7 +291,6 @@ def request_body_json( stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, ) -> Optional[Mapping]: - payload = { "metrics": [{"name": m} for m in self.config["metrics"]], "dimensions": [{"name": d} for d in self.config["dimensions"]], @@ -260,7 +298,17 @@ def request_body_json( "returnPropertyQuota": True, "offset": str(0), "limit": str(self.page_size), + "keepEmptyRows": self.config.get("keep_empty_rows", False), } + + dimension_filter = self.config.get("dimensionFilter") + if dimension_filter: + payload.update({"dimensionFilter": dimension_filter}) + + metrics_filter = self.config.get("metricsFilter") + if metrics_filter: + payload.update({"metricsFilter": metrics_filter}) + if next_page_token and next_page_token.get("offset") is not None: payload.update({"offset": str(next_page_token["offset"])}) return payload @@ -268,11 +316,13 @@ def request_body_json( def stream_slices( self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - today: datetime.date = datetime.date.today() start_date = stream_state and stream_state.get(self.cursor_field) if start_date: + start_date = ( + serialize_to_date_string(start_date, DATE_FORMAT, self.cursor_field) if not self.cursor_field == "date" else start_date + ) start_date = utils.string_to_date(start_date, self._record_date_format, old_format=DATE_FORMAT) start_date -= LOOKBACK_WINDOW start_date = max(start_date, self.config["date_ranges_start_date"]) @@ -358,37 +408,52 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp class SourceGoogleAnalyticsDataApi(AbstractSource): - def _validate_and_transform(self, config: Mapping[str, Any], report_names: Set[str]): - if "custom_reports" in config: - if isinstance(config["custom_reports"], str): + @property + def default_date_ranges_start_date(self) -> str: + # set default date ranges start date to 2 years ago + return pendulum.now(tz="UTC").subtract(years=2).format("YYYY-MM-DD") + + def _validate_and_transform_start_date(self, start_date: str) -> datetime.date: + start_date = self.default_date_ranges_start_date if not start_date else start_date + + try: + start_date = utils.string_to_date(start_date) + except ValueError as e: + raise ConfigurationError(str(e)) + + return start_date + + def _validate_custom_reports(self, config: Mapping[str, Any]) -> Mapping[str, Any]: + if "custom_reports_array" in config: + if isinstance(config["custom_reports_array"], str): try: - config["custom_reports"] = json.loads(config["custom_reports"]) - if not isinstance(config["custom_reports"], list): + config["custom_reports_array"] = json.loads(config["custom_reports_array"]) + if not isinstance(config["custom_reports_array"], list): raise ValueError except ValueError: raise ConfigurationError(WRONG_JSON_SYNTAX) else: - config["custom_reports"] = [] + config["custom_reports_array"] = [] + + return config + + def _validate_and_transform(self, config: Mapping[str, Any], report_names: Set[str]): + config = self._validate_custom_reports(config) schema = json.loads(pkgutil.get_data("source_google_analytics_data_api", "defaults/custom_reports_schema.json")) try: - jsonschema.validate(instance=config["custom_reports"], schema=schema) + jsonschema.validate(instance=config["custom_reports_array"], schema=schema) except jsonschema.ValidationError as e: if message := check_no_property_error(e): raise ConfigurationError(message) if message := check_invalid_property_error(e): - report_name = dpath.util.get(config["custom_reports"], str(e.absolute_path[0])).get("name") + report_name = dpath.util.get(config["custom_reports_array"], str(e.absolute_path[0])).get("name") raise ConfigurationError(message.format(fields=e.message, report_name=report_name)) - key_path = "custom_reports" - if e.path: - key_path += "." + ".".join(map(str, e.path)) - raise ConfigurationError(f"{key_path}: {e.message}") - - existing_names = {r["name"] for r in config["custom_reports"]} & report_names + existing_names = {r["name"] for r in config["custom_reports_array"]} & report_names if existing_names: existing_names = ", ".join(existing_names) - raise ConfigurationError(f"custom_reports: {existing_names} already exist as a default report(s).") + raise ConfigurationError(f"Custom reports: {existing_names} already exist as a default report(s).") if "credentials_json" in config["credentials"]: try: @@ -396,10 +461,7 @@ def _validate_and_transform(self, config: Mapping[str, Any], report_names: Set[s except ValueError: raise ConfigurationError("credentials.credentials_json is not valid JSON") - try: - config["date_ranges_start_date"] = utils.string_to_date(config["date_ranges_start_date"]) - except ValueError as e: - raise ConfigurationError(str(e)) + config["date_ranges_start_date"] = self._validate_and_transform_start_date(config.get("date_ranges_start_date")) if not config.get("window_in_days"): source_spec = self.spec(logging.getLogger("airbyte")) @@ -413,73 +475,106 @@ def get_authenticator(self, config: Mapping[str, Any]): return authenticator_class(**get_credentials(credentials)) def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: - reports = json.loads(pkgutil.get_data("source_google_analytics_data_api", "defaults/default_reports.json")) - try: - config = self._validate_and_transform(config, report_names={r["name"] for r in reports}) - except ConfigurationError as e: - return False, str(e) - config["authenticator"] = self.get_authenticator(config) + for property_id in config["property_ids"]: + reports = json.loads(pkgutil.get_data("source_google_analytics_data_api", "defaults/default_reports.json")) + try: + config = self._validate_and_transform(config, report_names={r["name"] for r in reports}) + except ConfigurationError as e: + return False, str(e) + config["authenticator"] = self.get_authenticator(config) - metadata = None - try: - # explicitly setting small page size for the check operation not to cause OOM issues - stream = GoogleAnalyticsDataApiMetadataStream(config=config, authenticator=config["authenticator"]) - metadata = next(stream.read_records(sync_mode=SyncMode.full_refresh), None) - except HTTPError as e: - error_list = [HTTPStatus.BAD_REQUEST, HTTPStatus.FORBIDDEN] - if e.response.status_code in error_list: - internal_message = f"Incorrect Property ID: {config['property_id']}" - property_id_docs_url = ( - "https://developers.google.com/analytics/devguides/reporting/data/v1/property-id#what_is_my_property_id" - ) - message = f"Access was denied to the property ID entered. Check your access to the Property ID or use Google Analytics {property_id_docs_url} to find your Property ID." - - wrong_property_id_error = AirbyteTracedException( - message=message, internal_message=internal_message, failure_type=FailureType.config_error - ) - raise wrong_property_id_error - - if not metadata: - return False, "failed to get metadata, over quota, try later" - - dimensions = {d["apiName"] for d in metadata["dimensions"]} - metrics = {d["apiName"] for d in metadata["metrics"]} - - for report in config["custom_reports"]: - invalid_dimensions = set(report["dimensions"]) - dimensions - if invalid_dimensions: - invalid_dimensions = ", ".join(invalid_dimensions) - return False, WRONG_DIMENSIONS.format(fields=invalid_dimensions, report_name=report["name"]) - invalid_metrics = set(report["metrics"]) - metrics - if invalid_metrics: - invalid_metrics = ", ".join(invalid_metrics) - return False, WRONG_METRICS.format(fields=invalid_metrics, report_name=report["name"]) - report_stream = self.instantiate_report_class(report, config, page_size=100) - # check if custom_report dimensions + metrics can be combined and report generated - stream_slice = next(report_stream.stream_slices(sync_mode=SyncMode.full_refresh)) - next(report_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice), None) - return True, None + _config = config.copy() + _config["property_id"] = property_id + + metadata = None + try: + # explicitly setting small page size for the check operation not to cause OOM issues + stream = GoogleAnalyticsDataApiMetadataStream(config=_config, authenticator=_config["authenticator"]) + metadata = next(stream.read_records(sync_mode=SyncMode.full_refresh), None) + except HTTPError as e: + error_list = [HTTPStatus.BAD_REQUEST, HTTPStatus.FORBIDDEN] + if e.response.status_code in error_list: + internal_message = f"Incorrect Property ID: {property_id}" + property_id_docs_url = ( + "https://developers.google.com/analytics/devguides/reporting/data/v1/property-id#what_is_my_property_id" + ) + message = f"Access was denied to the property ID entered. Check your access to the Property ID or use Google Analytics {property_id_docs_url} to find your Property ID." + + wrong_property_id_error = AirbyteTracedException( + message=message, internal_message=internal_message, failure_type=FailureType.config_error + ) + raise wrong_property_id_error + + if not metadata: + return False, "Failed to get metadata, over quota, try later" + + dimensions = {d["apiName"] for d in metadata["dimensions"]} + metrics = {d["apiName"] for d in metadata["metrics"]} + + for report in _config["custom_reports_array"]: + # Check if custom report dimensions supported. Compare them with dimensions provided by GA API + invalid_dimensions = set(report["dimensions"]) - dimensions + if invalid_dimensions: + invalid_dimensions = ", ".join(invalid_dimensions) + return False, WRONG_DIMENSIONS.format(fields=invalid_dimensions, report_name=report["name"]) + + # Check if custom report metrics supported. Compare them with metrics provided by GA API + invalid_metrics = set(report["metrics"]) - metrics + if invalid_metrics: + invalid_metrics = ", ".join(invalid_metrics) + return False, WRONG_METRICS.format(fields=invalid_metrics, report_name=report["name"]) + + report_stream = self.instantiate_report_class(report, False, _config, page_size=100) + # check if custom_report dimensions + metrics can be combined and report generated + try: + stream_slice = next(report_stream.stream_slices(sync_mode=SyncMode.full_refresh)) + next(report_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice), None) + except HTTPError as e: + error_response = "" + if e.response.status_code == HTTPStatus.BAD_REQUEST: + error_response = e.response.json().get("error", {}).get("message", "") + return False, WRONG_CUSTOM_REPORT_CONFIG.format(report=report["name"], error_response=error_response) + + return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: reports = json.loads(pkgutil.get_data("source_google_analytics_data_api", "defaults/default_reports.json")) config = self._validate_and_transform(config, report_names={r["name"] for r in reports}) config["authenticator"] = self.get_authenticator(config) - return [self.instantiate_report_class(report, config) for report in reports + config["custom_reports"]] + return [stream for report in reports + config["custom_reports_array"] for stream in self.instantiate_report_streams(report, config)] + + def instantiate_report_streams(self, report: dict, config: Mapping[str, Any], **extra_kwargs) -> GoogleAnalyticsDataApiBaseStream: + add_name_suffix = False + for property_id in config["property_ids"]: + yield self.instantiate_report_class( + report=report, add_name_suffix=add_name_suffix, config={**config, "property_id": property_id} + ) + # Append property ID to stream name only for the second and subsequent properties. + # This will make a release non-breaking for users with a single property. + # This is a temporary solution until https://github.com/airbytehq/airbyte/issues/30926 is implemented. + add_name_suffix = True @staticmethod - def instantiate_report_class(report: dict, config: Mapping[str, Any], **extra_kwargs) -> GoogleAnalyticsDataApiBaseStream: - cohort_spec = report.get("cohortSpec") + def instantiate_report_class( + report: dict, add_name_suffix: bool, config: Mapping[str, Any], **extra_kwargs + ) -> GoogleAnalyticsDataApiBaseStream: + cohort_spec = report.get("cohortSpec", {}) pivots = report.get("pivots") stream_config = { + **config, "metrics": report["metrics"], "dimensions": report["dimensions"], - **config, + "dimensionFilter": transform_json(report.get("dimensionFilter", {})), + "metricsFilter": transform_json(report.get("metricsFilter", {})), } report_class_tuple = (GoogleAnalyticsDataApiBaseStream,) if pivots: stream_config["pivots"] = pivots report_class_tuple = (PivotReport,) - if cohort_spec: + if cohort_spec.pop("enabled", "") == "true": stream_config["cohort_spec"] = cohort_spec report_class_tuple = (CohortReportMixin, *report_class_tuple) - return type(report["name"], report_class_tuple, {})(config=stream_config, authenticator=config["authenticator"], **extra_kwargs) + name = report["name"] + if add_name_suffix: + name = f"{name}Property{config['property_id']}" + return type(name, report_class_tuple, {})(config=stream_config, authenticator=config["authenticator"], **extra_kwargs) diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/spec.json b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/spec.json index 051338214db1..c487c6ff3572 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/spec.json +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/spec.json @@ -4,7 +4,7 @@ "$schema": "https://json-schema.org/draft-07/schema#", "title": "Google Analytics (Data API) Spec", "type": "object", - "required": ["property_id", "date_ranges_start_date"], + "required": ["property_ids"], "additionalProperties": true, "properties": { "credentials": { @@ -76,14 +76,16 @@ } ] }, - "property_id": { - "type": "string", - "title": "Property ID", - "description": "The Property ID is a unique number assigned to each property in Google Analytics, found in your GA4 property URL. This ID allows the connector to track the specific events associated with your property. Refer to the Google Analytics documentation to locate your property ID.", - "pattern": "^[0-9]*$", - "pattern_descriptor": "123...", - "examples": ["1738294", "5729978930"], - "order": 1 + "property_ids": { + "title": "Property IDs", + "description": "A list of your Property IDs. The Property ID is a unique number assigned to each property in Google Analytics, found in your GA4 property URL. This ID allows the connector to track the specific events associated with your property. Refer to the Google Analytics documentation to locate your property ID.", + "order": 1, + "type": "array", + "items": { + "type": "string", + "pattern": "^[0-9]*$" + }, + "examples": [["1738294", "5729978930"]] }, "date_ranges_start_date": { "type": "string", @@ -95,11 +97,2134 @@ "examples": ["2021-01-01"], "order": 2 }, - "custom_reports": { - "order": 3, - "type": "string", + "custom_reports_array": { "title": "Custom Reports", - "description": "A JSON array describing the custom reports you want to sync from Google Analytics. See the documentation for more information about the exact format you can use to fill out this field." + "description": "You can add your Custom Analytics report by creating one.", + "order": 4, + "type": "array", + "items": { + "title": "Custom Report Config", + "type": "object", + "properties": { + "name": { + "title": "Name", + "description": "The name of the custom report, this name would be used as stream name.", + "type": "string", + "order": 0 + }, + "dimensions": { + "title": "Dimensions", + "description": "A list of dimensions.", + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "order": 1 + }, + "metrics": { + "title": "Metrics", + "description": "A list of metrics.", + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "order": 2 + }, + "dimensionFilter": { + "title": "Dimensions filter", + "description": "Dimensions filter", + "type": "object", + "order": 3, + "oneOf": [ + { + "title": "andGroup", + "description": "The FilterExpressions in andGroup have an AND relationship.", + "type": "object", + "properties": { + "filter_type": { + "type": "string", + "const": "andGroup", + "order": 0 + }, + "expressions": { + "title": "Expressions", + "type": "array", + "order": 1, + "items": { + "title": "Expression", + "type": "object", + "properties": { + "field_name": { + "title": "fieldName", + "type": "string", + "order": 1 + }, + "filter": { + "title": "filter", + "type": "object", + "order": 2, + "oneOf": [ + { + "title": "stringFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "stringFilter" + }, + "matchType": { + "title": "matchType", + "type": "array", + "order": 1, + "items": { + "title": "ValidEnums", + "enum": [ + "MATCH_TYPE_UNSPECIFIED", + "EXACT", + "BEGINS_WITH", + "ENDS_WITH", + "CONTAINS", + "FULL_REGEXP", + "PARTIAL_REGEXP" + ] + } + }, + "value": { + "tittle": "value", + "type": "string", + "order": 0 + }, + "caseSensitive": { + "tittle": "caseSensitive", + "type": "boolean", + "order": 2 + } + }, + "required": ["filter_name", "value"] + }, + { + "title": "inListFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "inListFilter" + }, + "values": { + "tittle": "values", + "type": "array", + "minItems": 1, + "order": 0, + "items": { + "type": "string" + } + }, + "caseSensitive": { + "tittle": "caseSensitive", + "type": "boolean", + "order": 1 + } + }, + "required": ["filter_name", "values"] + }, + { + "title": "numericFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "numericFilter" + }, + "operation": { + "title": "operation", + "type": "array", + "order": 1, + "items": { + "title": "ValidEnums", + "enum": [ + "OPERATION_UNSPECIFIED", + "EQUAL", + "LESS_THAN", + "LESS_THAN_OR_EQUAL", + "GREATER_THAN", + "GREATER_THAN_OR_EQUAL" + ] + } + }, + "value": { + "tittle": "value", + "type": "object", + "order": 0, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + } + }, + "required": [ + "filter_name", + "operation", + "value" + ] + }, + { + "title": "betweenFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "betweenFilter" + }, + "fromValue": { + "tittle": "fromValue", + "type": "object", + "order": 0, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + }, + "toValue": { + "tittle": "toValue", + "type": "object", + "order": 1, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + } + }, + "required": [ + "filter_name", + "fromValue", + "toValue" + ] + } + ] + } + }, + "required": ["field_name", "filter"] + } + } + }, + "required": ["filter_type", "expressions"] + }, + { + "title": "orGroup", + "type": "object", + "description": "The FilterExpressions in orGroup have an OR relationship.", + "properties": { + "filter_type": { + "type": "string", + "const": "orGroup", + "order": 0 + }, + "expressions": { + "title": "Expressions", + "type": "array", + "order": 1, + "items": { + "title": "Expression", + "type": "object", + "properties": { + "field_name": { + "title": "fieldName", + "type": "string", + "order": 1 + }, + "filter": { + "title": "filter", + "type": "object", + "order": 2, + "oneOf": [ + { + "title": "stringFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "stringFilter" + }, + "matchType": { + "title": "matchType", + "type": "array", + "order": 1, + "items": { + "title": "ValidEnums", + "enum": [ + "MATCH_TYPE_UNSPECIFIED", + "EXACT", + "BEGINS_WITH", + "ENDS_WITH", + "CONTAINS", + "FULL_REGEXP", + "PARTIAL_REGEXP" + ] + } + }, + "value": { + "tittle": "value", + "type": "string", + "order": 0 + }, + "caseSensitive": { + "tittle": "caseSensitive", + "type": "boolean", + "order": 2 + } + }, + "required": ["filter_name", "value"] + }, + { + "title": "inListFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "inListFilter" + }, + "values": { + "tittle": "values", + "type": "array", + "minItems": 1, + "order": 0, + "items": { + "type": "string" + } + }, + "caseSensitive": { + "tittle": "caseSensitive", + "type": "boolean", + "order": 1 + } + }, + "required": ["filter_name", "values"] + }, + { + "title": "numericFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "numericFilter" + }, + "operation": { + "title": "operation", + "type": "array", + "order": 1, + "items": { + "title": "ValidEnums", + "enum": [ + "OPERATION_UNSPECIFIED", + "EQUAL", + "LESS_THAN", + "LESS_THAN_OR_EQUAL", + "GREATER_THAN", + "GREATER_THAN_OR_EQUAL" + ] + } + }, + "value": { + "tittle": "value", + "type": "object", + "order": 0, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + } + }, + "required": [ + "filter_name", + "operation", + "value" + ] + }, + { + "title": "betweenFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "betweenFilter" + }, + "fromValue": { + "tittle": "fromValue", + "type": "object", + "order": 0, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + }, + "toValue": { + "tittle": "toValue", + "type": "object", + "order": 1, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + } + }, + "required": [ + "filter_name", + "fromValue", + "toValue" + ] + } + ] + } + }, + "required": ["field_name", "filter"] + } + } + }, + "required": ["filter_type", "expressions"] + }, + { + "title": "notExpression", + "type": "object", + "description": "The FilterExpression is NOT of notExpression.", + "properties": { + "filter_type": { + "type": "string", + "const": "notExpression", + "order": 0 + }, + "expression": { + "title": "Expression", + "type": "object", + "properties": { + "field_name": { + "title": "fieldName", + "type": "string", + "order": 1 + }, + "filter": { + "title": "filter", + "type": "object", + "order": 2, + "oneOf": [ + { + "title": "stringFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "stringFilter" + }, + "matchType": { + "title": "matchType", + "type": "array", + "order": 1, + "items": { + "title": "ValidEnums", + "enum": [ + "MATCH_TYPE_UNSPECIFIED", + "EXACT", + "BEGINS_WITH", + "ENDS_WITH", + "CONTAINS", + "FULL_REGEXP", + "PARTIAL_REGEXP" + ] + } + }, + "value": { + "tittle": "value", + "type": "string", + "order": 0 + }, + "caseSensitive": { + "tittle": "caseSensitive", + "type": "boolean", + "order": 2 + } + }, + "required": ["filter_name", "value"] + }, + { + "title": "inListFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "inListFilter" + }, + "values": { + "tittle": "values", + "type": "array", + "minItems": 1, + "order": 0, + "items": { + "type": "string" + } + }, + "caseSensitive": { + "tittle": "caseSensitive", + "type": "boolean", + "order": 1 + } + }, + "required": ["filter_name", "values"] + }, + { + "title": "numericFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "numericFilter" + }, + "operation": { + "title": "operation", + "type": "array", + "order": 1, + "items": { + "title": "ValidEnums", + "enum": [ + "OPERATION_UNSPECIFIED", + "EQUAL", + "LESS_THAN", + "LESS_THAN_OR_EQUAL", + "GREATER_THAN", + "GREATER_THAN_OR_EQUAL" + ] + } + }, + "value": { + "tittle": "value", + "type": "object", + "order": 0, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + } + }, + "required": ["filter_name", "operation", "value"] + }, + { + "title": "betweenFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "betweenFilter" + }, + "fromValue": { + "tittle": "fromValue", + "type": "object", + "order": 0, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + }, + "toValue": { + "tittle": "toValue", + "type": "object", + "order": 1, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + } + }, + "required": [ + "filter_name", + "fromValue", + "toValue" + ] + } + ] + } + }, + "required": ["field_name", "filter"] + } + } + }, + { + "title": "filter", + "type": "object", + "description": "A primitive filter. In the same FilterExpression, all of the filter's field names need to be either all dimensions.", + "properties": { + "filter_type": { + "type": "string", + "const": "filter", + "order": 0 + }, + "field_name": { + "title": "fieldName", + "type": "string", + "order": 1 + }, + "filter": { + "title": "filter", + "type": "object", + "order": 2, + "oneOf": [ + { + "title": "stringFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "stringFilter" + }, + "matchType": { + "title": "matchType", + "type": "array", + "order": 1, + "items": { + "title": "ValidEnums", + "enum": [ + "MATCH_TYPE_UNSPECIFIED", + "EXACT", + "BEGINS_WITH", + "ENDS_WITH", + "CONTAINS", + "FULL_REGEXP", + "PARTIAL_REGEXP" + ] + } + }, + "value": { + "tittle": "value", + "type": "string", + "order": 0 + }, + "caseSensitive": { + "tittle": "caseSensitive", + "type": "boolean", + "order": 2 + } + }, + "required": ["filter_name", "value"] + }, + { + "title": "inListFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "inListFilter" + }, + "values": { + "tittle": "values", + "type": "array", + "minItems": 1, + "order": 0, + "items": { + "type": "string" + } + }, + "caseSensitive": { + "tittle": "caseSensitive", + "type": "boolean", + "order": 1 + } + }, + "required": ["filter_name", "values"] + }, + { + "title": "numericFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "numericFilter" + }, + "operation": { + "title": "operation", + "type": "array", + "order": 1, + "items": { + "title": "ValidEnums", + "enum": [ + "OPERATION_UNSPECIFIED", + "EQUAL", + "LESS_THAN", + "LESS_THAN_OR_EQUAL", + "GREATER_THAN", + "GREATER_THAN_OR_EQUAL" + ] + } + }, + "value": { + "tittle": "value", + "type": "object", + "order": 0, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + } + }, + "required": ["filter_name", "operation", "value"] + }, + { + "title": "betweenFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "betweenFilter" + }, + "fromValue": { + "tittle": "fromValue", + "type": "object", + "order": 0, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + }, + "toValue": { + "tittle": "toValue", + "type": "object", + "order": 1, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + } + }, + "required": ["filter_name", "fromValue", "toValue"] + } + ] + } + }, + "required": ["field_name", "filter"] + } + ] + }, + "metricFilter": { + "title": "Metrics filter", + "description": "Metrics filter", + "type": "object", + "order": 4, + "oneOf": [ + { + "title": "andGroup", + "description": "The FilterExpressions in andGroup have an AND relationship.", + "type": "object", + "properties": { + "filter_type": { + "type": "string", + "const": "andGroup", + "order": 0 + }, + "expressions": { + "title": "Expressions", + "type": "array", + "order": 1, + "items": { + "title": "Expression", + "type": "object", + "properties": { + "field_name": { + "title": "fieldName", + "type": "string", + "order": 1 + }, + "filter": { + "title": "filter", + "type": "object", + "order": 2, + "oneOf": [ + { + "title": "stringFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "stringFilter" + }, + "matchType": { + "title": "matchType", + "type": "array", + "order": 1, + "items": { + "title": "ValidEnums", + "enum": [ + "MATCH_TYPE_UNSPECIFIED", + "EXACT", + "BEGINS_WITH", + "ENDS_WITH", + "CONTAINS", + "FULL_REGEXP", + "PARTIAL_REGEXP" + ] + } + }, + "value": { + "tittle": "value", + "type": "string", + "order": 0 + }, + "caseSensitive": { + "tittle": "caseSensitive", + "type": "boolean", + "order": 2 + } + }, + "required": ["filter_name", "value"] + }, + { + "title": "inListFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "inListFilter" + }, + "values": { + "tittle": "values", + "type": "array", + "minItems": 1, + "order": 0, + "items": { + "type": "string" + } + }, + "caseSensitive": { + "tittle": "caseSensitive", + "type": "boolean", + "order": 1 + } + }, + "required": ["filter_name", "values"] + }, + { + "title": "numericFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "numericFilter" + }, + "operation": { + "title": "operation", + "type": "array", + "order": 1, + "items": { + "title": "ValidEnums", + "enum": [ + "OPERATION_UNSPECIFIED", + "EQUAL", + "LESS_THAN", + "LESS_THAN_OR_EQUAL", + "GREATER_THAN", + "GREATER_THAN_OR_EQUAL" + ] + } + }, + "value": { + "tittle": "value", + "type": "object", + "order": 0, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + } + }, + "required": [ + "filter_name", + "operation", + "value" + ] + }, + { + "title": "betweenFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "betweenFilter" + }, + "fromValue": { + "tittle": "fromValue", + "type": "object", + "order": 0, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + }, + "toValue": { + "tittle": "toValue", + "type": "object", + "order": 1, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + } + }, + "required": [ + "filter_name", + "fromValue", + "toValue" + ] + } + ] + } + }, + "required": ["field_name", "filter"] + } + } + }, + "required": ["filter_type", "expressions"] + }, + { + "title": "orGroup", + "type": "object", + "description": "The FilterExpressions in orGroup have an OR relationship.", + "properties": { + "filter_type": { + "type": "string", + "const": "orGroup", + "order": 0 + }, + "expressions": { + "title": "Expressions", + "type": "array", + "order": 1, + "items": { + "title": "Expression", + "type": "object", + "properties": { + "field_name": { + "title": "fieldName", + "type": "string", + "order": 1 + }, + "filter": { + "title": "filter", + "type": "object", + "order": 2, + "oneOf": [ + { + "title": "stringFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "stringFilter" + }, + "matchType": { + "title": "matchType", + "type": "array", + "order": 1, + "items": { + "title": "ValidEnums", + "enum": [ + "MATCH_TYPE_UNSPECIFIED", + "EXACT", + "BEGINS_WITH", + "ENDS_WITH", + "CONTAINS", + "FULL_REGEXP", + "PARTIAL_REGEXP" + ] + } + }, + "value": { + "tittle": "value", + "type": "string", + "order": 0 + }, + "caseSensitive": { + "tittle": "caseSensitive", + "type": "boolean", + "order": 2 + } + }, + "required": ["filter_name", "value"] + }, + { + "title": "inListFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "inListFilter" + }, + "values": { + "tittle": "values", + "type": "array", + "minItems": 1, + "order": 0, + "items": { + "type": "string" + } + }, + "caseSensitive": { + "tittle": "caseSensitive", + "type": "boolean", + "order": 1 + } + }, + "required": ["filter_name", "values"] + }, + { + "title": "numericFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "numericFilter" + }, + "operation": { + "title": "operation", + "type": "array", + "order": 1, + "items": { + "title": "ValidEnums", + "enum": [ + "OPERATION_UNSPECIFIED", + "EQUAL", + "LESS_THAN", + "LESS_THAN_OR_EQUAL", + "GREATER_THAN", + "GREATER_THAN_OR_EQUAL" + ] + } + }, + "value": { + "tittle": "value", + "type": "object", + "order": 0, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + } + }, + "required": [ + "filter_name", + "operation", + "value" + ] + }, + { + "title": "betweenFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "betweenFilter" + }, + "fromValue": { + "tittle": "fromValue", + "type": "object", + "order": 0, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + }, + "toValue": { + "tittle": "toValue", + "type": "object", + "order": 1, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + } + }, + "required": [ + "filter_name", + "fromValue", + "toValue" + ] + } + ] + } + }, + "required": ["field_name", "filter"] + } + } + }, + "required": ["filter_type", "expressions"] + }, + { + "title": "notExpression", + "type": "object", + "description": "The FilterExpression is NOT of notExpression.", + "properties": { + "filter_type": { + "type": "string", + "const": "notExpression", + "order": 0 + }, + "expression": { + "title": "Expression", + "type": "object", + "properties": { + "field_name": { + "title": "fieldName", + "type": "string", + "order": 1 + }, + "filter": { + "title": "filter", + "type": "object", + "order": 2, + "oneOf": [ + { + "title": "stringFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "stringFilter" + }, + "matchType": { + "title": "matchType", + "type": "array", + "order": 1, + "items": { + "title": "ValidEnums", + "enum": [ + "MATCH_TYPE_UNSPECIFIED", + "EXACT", + "BEGINS_WITH", + "ENDS_WITH", + "CONTAINS", + "FULL_REGEXP", + "PARTIAL_REGEXP" + ] + } + }, + "value": { + "tittle": "value", + "type": "string", + "order": 0 + }, + "caseSensitive": { + "tittle": "caseSensitive", + "type": "boolean", + "order": 2 + } + }, + "required": ["filter_name", "value"] + }, + { + "title": "inListFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "inListFilter" + }, + "values": { + "tittle": "values", + "type": "array", + "minItems": 1, + "order": 0, + "items": { + "type": "string" + } + }, + "caseSensitive": { + "tittle": "caseSensitive", + "type": "boolean", + "order": 1 + } + }, + "required": ["filter_name", "values"] + }, + { + "title": "numericFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "numericFilter" + }, + "operation": { + "title": "operation", + "type": "array", + "order": 1, + "items": { + "title": "ValidEnums", + "enum": [ + "OPERATION_UNSPECIFIED", + "EQUAL", + "LESS_THAN", + "LESS_THAN_OR_EQUAL", + "GREATER_THAN", + "GREATER_THAN_OR_EQUAL" + ] + } + }, + "value": { + "tittle": "value", + "type": "object", + "order": 0, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + } + }, + "required": ["filter_name", "operation", "value"] + }, + { + "title": "betweenFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "betweenFilter" + }, + "fromValue": { + "tittle": "fromValue", + "type": "object", + "order": 0, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + }, + "toValue": { + "tittle": "toValue", + "type": "object", + "order": 1, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + } + }, + "required": [ + "filter_name", + "fromValue", + "toValue" + ] + } + ] + } + }, + "required": ["field_name", "filter"] + } + } + }, + { + "title": "filter", + "type": "object", + "description": "A primitive filter. In the same FilterExpression, all of the filter's field names need to be either all metrics.", + "properties": { + "filter_type": { + "type": "string", + "const": "filter", + "order": 0 + }, + "field_name": { + "title": "fieldName", + "type": "string", + "order": 1 + }, + "filter": { + "title": "filter", + "type": "object", + "order": 2, + "oneOf": [ + { + "title": "stringFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "stringFilter" + }, + "matchType": { + "title": "matchType", + "type": "array", + "order": 1, + "items": { + "title": "ValidEnums", + "enum": [ + "MATCH_TYPE_UNSPECIFIED", + "EXACT", + "BEGINS_WITH", + "ENDS_WITH", + "CONTAINS", + "FULL_REGEXP", + "PARTIAL_REGEXP" + ] + } + }, + "value": { + "tittle": "value", + "type": "string", + "order": 0 + }, + "caseSensitive": { + "tittle": "caseSensitive", + "type": "boolean", + "order": 2 + } + }, + "required": ["filter_name", "value"] + }, + { + "title": "inListFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "inListFilter" + }, + "values": { + "tittle": "values", + "type": "array", + "minItems": 1, + "order": 0, + "items": { + "type": "string" + } + }, + "caseSensitive": { + "tittle": "caseSensitive", + "type": "boolean", + "order": 1 + } + }, + "required": ["filter_name", "values"] + }, + { + "title": "numericFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "numericFilter" + }, + "operation": { + "title": "operation", + "type": "array", + "order": 1, + "items": { + "title": "ValidEnums", + "enum": [ + "OPERATION_UNSPECIFIED", + "EQUAL", + "LESS_THAN", + "LESS_THAN_OR_EQUAL", + "GREATER_THAN", + "GREATER_THAN_OR_EQUAL" + ] + } + }, + "value": { + "tittle": "value", + "type": "object", + "order": 0, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + } + }, + "required": ["filter_name", "operation", "value"] + }, + { + "title": "betweenFilter", + "type": "object", + "properties": { + "filter_name": { + "type": "string", + "const": "betweenFilter" + }, + "fromValue": { + "tittle": "fromValue", + "type": "object", + "order": 0, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + }, + "toValue": { + "tittle": "toValue", + "type": "object", + "order": 1, + "oneOf": [ + { + "title": "int64Value", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "int64Value" + }, + "value": { + "type": "string" + } + }, + "required": ["value_type", "value"] + }, + { + "title": "doubleValue", + "type": "object", + "properties": { + "value_type": { + "type": "string", + "const": "doubleValue" + }, + "value": { + "type": "number" + } + }, + "required": ["value_type", "value"] + } + ] + } + }, + "required": ["filter_name", "fromValue", "toValue"] + } + ] + } + }, + "required": ["field_name", "filter"] + } + ] + }, + "cohortSpec": { + "title": "Cohort Reports", + "description": "Cohort reports creates a time series of user retention for the cohort.", + "type": "object", + "order": 5, + "oneOf": [ + { + "title": "Disabled", + "type": "object", + "properties": { + "enabled": { + "type": "string", + "const": "false" + } + } + }, + { + "title": "Enabled", + "type": "object", + "properties": { + "enabled": { + "type": "string", + "const": "true" + }, + "cohorts": { + "name": "Cohorts", + "order": 0, + "type": "array", + "always_show": true, + "items": { + "title": "Cohorts", + "type": "object", + "required": ["dimension", "dateRange"], + "properties": { + "name": { + "title": "Name", + "type": "string", + "always_show": true, + "pattern": "^(?!(cohort_|RESERVED_)).*$", + "description": "Assigns a name to this cohort. If not set, cohorts are named by their zero based index cohort_0, cohort_1, etc.", + "order": 0 + }, + "dimension": { + "title": "Dimension", + "description": "Dimension used by the cohort. Required and only supports `firstSessionDate`", + "type": "string", + "enum": ["firstSessionDate"], + "order": 1 + }, + "dateRange": { + "type": "object", + "required": ["startDate", "endDate"], + "properties": { + "startDate": { + "title": "Start Date", + "type": "string", + "format": "date", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "pattern_descriptor": "YYYY-MM-DD", + "examples": ["2021-01-01"], + "order": 2 + }, + "endDate": { + "title": "End Date", + "type": "string", + "format": "date", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "pattern_descriptor": "YYYY-MM-DD", + "examples": ["2021-01-01"], + "order": 3 + } + } + } + } + } + }, + "cohortsRange": { + "type": "object", + "order": 1, + "required": ["granularity", "endOffset"], + "properties": { + "granularity": { + "title": "Granularity", + "description": "The granularity used to interpret the startOffset and endOffset for the extended reporting date range for a cohort report.", + "type": "string", + "enum": [ + "GRANULARITY_UNSPECIFIED", + "DAILY", + "WEEKLY", + "MONTHLY" + ], + "order": 0 + }, + "startOffset": { + "title": "Start Offset", + "description": "Specifies the start date of the extended reporting date range for a cohort report.", + "type": "integer", + "minimum": 0, + "order": 1 + }, + "endOffset": { + "title": "End Offset", + "description": "Specifies the end date of the extended reporting date range for a cohort report.", + "type": "integer", + "minimum": 0, + "order": 2 + } + } + }, + "cohortReportSettings": { + "type": "object", + "title": "Cohort Report Settings", + "description": "Optional settings for a cohort report.", + "properties": { + "accumulate": { + "always_show": true, + "title": "Accumulate", + "description": "If true, accumulates the result from first touch day to the end day", + "type": "boolean" + } + } + } + } + } + ] + } + }, + "required": ["name", "dimensions", "metrics"] + } }, "window_in_days": { "type": "integer", @@ -109,7 +2234,14 @@ "minimum": 1, "maximum": 364, "default": 1, - "order": 4 + "order": 5 + }, + "keep_empty_rows": { + "type": "boolean", + "title": "Keep Empty Rows", + "description": "If false, each row with all metrics equal to 0 will not be returned. If true, these rows will be returned if they are not separately removed by a filter. More information is available in the documentation.", + "default": false, + "order": 6 } } }, diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/utils.py b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/utils.py index c4336453ca06..5a77a16f9a32 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/utils.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/utils.py @@ -11,6 +11,7 @@ from typing import Dict import jsonschema +import pandas as pd from airbyte_cdk.sources.streams.http import auth from source_google_analytics_data_api.authenticator import GoogleServiceKeyAuthenticator @@ -70,6 +71,7 @@ WRONG_METRICS = "The custom report {report_name} entered contains invalid metrics: {fields}. Validate your custom query with the GA 4 Query Explorer (https://ga-dev-tools.google/ga4/query-explorer/)." WRONG_PIVOTS = "The custom report {report_name} entered contains invalid pivots: {fields}. Ensure the pivot follow the syntax described in the docs (https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/Pivot)." API_LIMIT_PER_HOUR = "Your API key has reached its limit for the hour. Wait until the quota refreshes in an hour to retry." +WRONG_CUSTOM_REPORT_CONFIG = "Please check configuration for custom report {report}. {error_response}" def datetime_to_secs(dt: datetime.datetime) -> int: @@ -135,3 +137,111 @@ def get_source_defined_primary_key(stream): catalog = json.loads(open(args.catalog).read()) res = {s["stream"]["name"]: s["stream"].get("source_defined_primary_key") for s in catalog["streams"]} return res.get(stream) + + +def transform_string_filter(filter): + string_filter = {"value": filter.get("value")} + if "matchType" in filter: + string_filter["matchType"] = filter.get("matchType")[0] + if "caseSensitive" in filter: + string_filter["caseSensitive"] = filter.get("caseSensitive") + return {"stringFilter": string_filter} + + +def transform_in_list_filter(filter): + in_list_filter = {"values": filter.get("values")} + if "caseSensitive" in filter: + in_list_filter["caseSensitive"] = filter.get("caseSensitive") + return {"inListFilter": in_list_filter} + + +def transform_numeric_filter(filter): + numeric_filter = { + "value": {filter.get("value").get("value_type"): filter.get("value").get("value")}, + } + if "operation" in filter: + numeric_filter["operation"] = filter.get("operation")[0] + return {"numericFilter": numeric_filter} + + +def transform_between_filter(filter): + from_value = filter.get("fromValue") + to_value = filter.get("toValue") + + from_value_type = from_value.get("value_type") + to_value_type = to_value.get("value_type") + + if from_value_type == "doubleValue" and isinstance(from_value.get("value"), str): + from_value["value"] = float(from_value.get("value")) + if to_value_type == "doubleValue" and isinstance(to_value.get("value"), str): + to_value["value"] = float(to_value.get("value")) + + return { + "betweenFilter": { + "fromValue": {from_value_type: from_value.get("value")}, + "toValue": {to_value_type: to_value.get("value")}, + } + } + + +def transform_expression(expression): + transformed_expression = {"fieldName": expression.get("field_name")} + filter = expression.get("filter") + filter_name = filter.get("filter_name") + + if filter_name == "stringFilter": + transformed_expression.update(transform_string_filter(filter)) + elif filter_name == "inListFilter": + transformed_expression.update(transform_in_list_filter(filter)) + elif filter_name == "numericFilter": + transformed_expression.update(transform_numeric_filter(filter)) + elif filter_name == "betweenFilter": + transformed_expression.update(transform_between_filter(filter)) + + return {"filter": transformed_expression} + + +def transform_json(original_json): + transformed_json = {} + filter_type = original_json.get("filter_type") + + if filter_type in ["andGroup", "orGroup"]: + expressions = original_json.get("expressions", []) + transformed_expressions = [transform_expression(exp) for exp in expressions] + transformed_json = {filter_type: {"expressions": transformed_expressions}} if transformed_expressions else {} + + elif filter_type == "notExpression": + expression = original_json.get("expression") + transformed_expression = transform_expression(expression) + transformed_json = {filter_type: transformed_expression} + + elif filter_type == "filter": + transformed_json = transform_expression(original_json) + + return transformed_json + + +def serialize_to_date_string(date: str, date_format: str, date_type: str) -> str: + """ + Serialize a date string to a different date format based on the date_type. + + Parameters: + - date (str): The input date string. + - date_format (str): The desired output format for the date string. + - date_type (str): The type of the date string ('yearWeek', 'yearMonth', or 'year'). + + Returns: + str: The date string formatted according to date_format. + + Examples: + '202245' -> '2022-11-07' + '202210' -> '2022-10-01' + '2022' -> '2022-01-01' + """ + if date_type == "yearWeek": + return pd.to_datetime(f"{date}1", format="%Y%W%w").strftime(date_format) + elif date_type == "yearMonth": + year = int(date[:-2]) + month = int(date[-2:]) + return datetime.datetime(year, month, 1).strftime(date_format) + return datetime.datetime(int(date), 1, 1).strftime(date_format) diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/conftest.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/conftest.py new file mode 100644 index 000000000000..fcd8e1b879be --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/conftest.py @@ -0,0 +1,99 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import datetime +import json +from copy import deepcopy + +import pytest + +# json credentials with fake private key +json_credentials = """ +{ + "type": "service_account", + "project_id": "unittest-project-id", + "private_key_id": "9qf98e52oda52g5ne23al6evnf13649c2u077162c", + "private_key": "-----BEGIN PRIVATE KEY-----\\nMIIBVQIBADANBgkqhkiG9w0BAQEFAASCAT8wggE7AgEAAkEA3slcXL+dA36ESmOi\\n1xBhZmp5Hn0WkaHDtW4naba3plva0ibloBNWhFhjQOh7Ff01PVjhT4D5jgqXBIgc\\nz9Gv3QIDAQABAkEArlhYPoD5SB2/O1PjwHgiMPrL1C9B9S/pr1cH4vPJnpY3VKE3\\n5hvdil14YwRrcbmIxMkK2iRLi9lM4mJmdWPy4QIhAPsRFXZSGx0TZsDxD9V0ZJmZ\\n0AuDCj/NF1xB5KPLmp7pAiEA4yoFox6w7ql/a1pUVaLt0NJkDfE+22pxYGNQaiXU\\nuNUCIQCsFLaIJZiN4jlgbxlyLVeya9lLuqIwvqqPQl6q4ad12QIgS9gG48xmdHig\\n8z3IdIMedZ8ZCtKmEun6Cp1+BsK0wDUCIF0nHfSuU+eTQ2qAON2SHIrJf8UeFO7N\\nzdTN1IwwQqjI\\n-----END PRIVATE KEY-----\\n", + "client_email": "google-analytics-access@unittest-project-id.iam.gserviceaccount.com", + "client_id": "213243192021686092537", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/google-analytics-access%40unittest-project-id.iam.gserviceaccount.com" +} +""" + + +@pytest.fixture +def one_year_ago(): + return datetime.datetime.strftime((datetime.datetime.now() - datetime.timedelta(days=1)), "%Y-%m-%d") + + +@pytest.fixture +def config(one_year_ago): + return { + "property_id": "108176369", + "property_ids": ["108176369"], + "credentials": {"auth_type": "Service", "credentials_json": json_credentials}, + "date_ranges_start_date": one_year_ago, + "dimensions": ["date", "deviceCategory", "operatingSystem", "browser"], + "metrics": [ + "totalUsers", + "newUsers", + "sessions", + "sessionsPerUser", + "averageSessionDuration", + "screenPageViews", + "screenPageViewsPerSession", + "bounceRate", + ], + "keep_empty_rows": True, + "custom_reports": json.dumps( + [ + { + "name": "report1", + "dimensions": ["date", "browser"], + "metrics": ["totalUsers", "sessions", "screenPageViews"], + } + ] + ), + } + + +@pytest.fixture +def config_without_date_range(): + return { + "property_id": "108176369", + "property_ids": ["108176369"], + "credentials": {"auth_type": "Service", "credentials_json": json_credentials}, + "dimensions": ["deviceCategory", "operatingSystem", "browser"], + "metrics": [ + "totalUsers", + "newUsers", + "sessions", + "sessionsPerUser", + "averageSessionDuration", + "screenPageViews", + "screenPageViewsPerSession", + "bounceRate", + ], + "custom_reports": [], + } + + +@pytest.fixture +def patch_base_class(one_year_ago, config_without_date_range): + return {"config": config_without_date_range} + + +@pytest.fixture +def config_gen(config): + def inner(**kwargs): + new_config = deepcopy(config) + # WARNING, no support deep dictionaries + new_config.update(kwargs) + return {k: v for k, v in new_config.items() if v is not ...} + + return inner diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_api_quota.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_api_quota.py index 0a7fd7653808..2bbbc6b6b3f5 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_api_quota.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_api_quota.py @@ -9,10 +9,10 @@ TEST_QUOTA_INSTANCE: GoogleAnalyticsApiQuota = GoogleAnalyticsApiQuota() -@pytest.fixture(name='expected_quota_list') +@pytest.fixture(name="expected_quota_list") def expected_quota_list(): - """ The Quota were currently handle """ - return ['concurrentRequests', 'tokensPerProjectPerHour', 'potentiallyThresholdedRequestsPerHour'] + """The Quota were currently handle""" + return ["concurrentRequests", "tokensPerProjectPerHour", "potentiallyThresholdedRequestsPerHour"] def test_check_initial_quota_is_empty(): @@ -28,106 +28,90 @@ def test_check_initial_quota_is_empty(): # Full Quota ( { - 'propertyQuota': { - 'concurrentRequests': { - 'consumed': 0, - 'remaining': 10 - }, - 'tokensPerProjectPerHour': { - 'consumed': 1, - 'remaining': 1735 - }, - 'potentiallyThresholdedRequestsPerHour': { - 'consumed': 1, - 'remaining': 26 - } + "propertyQuota": { + "concurrentRequests": {"consumed": 0, "remaining": 10}, + "tokensPerProjectPerHour": {"consumed": 1, "remaining": 1735}, + "potentiallyThresholdedRequestsPerHour": {"consumed": 1, "remaining": 26}, } }, - False, True, None, True, False, + False, + True, + None, + True, + False, ), # Partial Quota ( { - 'propertyQuota': { - 'concurrentRequests': { - 'consumed': 0, - 'remaining': 10 - }, - 'tokensPerProjectPerHour': { - 'consumed': 5, - 'remaining': 955 - }, - 'potentiallyThresholdedRequestsPerHour': { - 'consumed': 3, - 'remaining': 26 - } + "propertyQuota": { + "concurrentRequests": {"consumed": 0, "remaining": 10}, + "tokensPerProjectPerHour": {"consumed": 5, "remaining": 955}, + "potentiallyThresholdedRequestsPerHour": {"consumed": 3, "remaining": 26}, } }, - True, True, None, True, False, + True, + True, + None, + True, + False, ), # Running out `tokensPerProjectPerHour` ( { - 'propertyQuota': { - 'concurrentRequests': { - 'consumed': 2, - 'remaining': 8 - }, - 'tokensPerProjectPerHour': { - 'consumed': 5, + "propertyQuota": { + "concurrentRequests": {"consumed": 2, "remaining": 8}, + "tokensPerProjectPerHour": { + "consumed": 5, # ~9% from original quota is left - 'remaining': 172 + "remaining": 172, }, - 'potentiallyThresholdedRequestsPerHour': { - 'consumed': 3, - 'remaining': 26 - } + "potentiallyThresholdedRequestsPerHour": {"consumed": 3, "remaining": 26}, } }, - True, True, 1800, False, False, + True, + True, + 1800, + False, + False, ), # Running out `concurrentRequests` ( { - 'propertyQuota': { - 'concurrentRequests': { - 'consumed': 9, + "propertyQuota": { + "concurrentRequests": { + "consumed": 9, # 10% from original quota is left - 'remaining': 1 - }, - 'tokensPerProjectPerHour': { - 'consumed': 5, - 'remaining': 935 + "remaining": 1, }, - 'potentiallyThresholdedRequestsPerHour': { - 'consumed': 1, - 'remaining': 26 - } + "tokensPerProjectPerHour": {"consumed": 5, "remaining": 935}, + "potentiallyThresholdedRequestsPerHour": {"consumed": 1, "remaining": 26}, } }, - True, True, 30, False, False, + True, + True, + 30, + False, + False, ), # Running out `potentiallyThresholdedRequestsPerHour` ( { - 'propertyQuota': { - 'concurrentRequests': { - 'consumed':1, - 'remaining': 9 - }, - 'tokensPerProjectPerHour': { - 'consumed': 5, - 'remaining': 935 - }, - 'potentiallyThresholdedRequestsPerHour': { + "propertyQuota": { + "concurrentRequests": {"consumed": 1, "remaining": 9}, + "tokensPerProjectPerHour": {"consumed": 5, "remaining": 935}, + "potentiallyThresholdedRequestsPerHour": { # 7% from original quota is left - 'consumed': 26, - 'remaining': 2 - } + "consumed": 26, + "remaining": 2, + }, } }, - True, True, 1800, False, False, - ) + True, + True, + 1800, + False, + False, + ), ], ids=[ "Full", @@ -135,7 +119,7 @@ def test_check_initial_quota_is_empty(): "Running out tokensPerProjectPerHour", "Running out concurrentRequests", "Running out potentiallyThresholdedRequestsPerHour", - ] + ], ) def test_check_full_quota( requests_mock, @@ -166,7 +150,7 @@ def test_check_full_quota( # Check the CURRENT QUOTA is different from Initial if partial_quota: - current_quota = TEST_QUOTA_INSTANCE._get_known_quota_from_response(response.json().get('propertyQuota')) + current_quota = TEST_QUOTA_INSTANCE._get_known_quota_from_response(response.json().get("propertyQuota")) assert not current_quota == TEST_QUOTA_INSTANCE.initial_quota # Check the scenario is applied based on Quota Values diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_authenticator.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_authenticator.py index ef1aec41889c..4118c86b116a 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_authenticator.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_authenticator.py @@ -12,19 +12,19 @@ def test_token_rotation(requests_mock): credentials = { "client_email": "client_email", "private_key": "-----BEGIN PRIVATE KEY-----\nMIIBVQIBADANBgkqhkiG9w0BAQEFAASCAT8wggE7AgEAAkEA3slcXL+dA36ESmOi\n1xBhZmp5Hn0WkaHDtW4naba3plva0ibloBNWhFhjQOh7Ff01PVjhT4D5jgqXBIgc\nz9Gv3QIDAQABAkEArlhYPoD5SB2/O1PjwHgiMPrL1C9B9S/pr1cH4vPJnpY3VKE3\n5hvdil14YwRrcbmIxMkK2iRLi9lM4mJmdWPy4QIhAPsRFXZSGx0TZsDxD9V0ZJmZ\n0AuDCj/NF1xB5KPLmp7pAiEA4yoFox6w7ql/a1pUVaLt0NJkDfE+22pxYGNQaiXU\nuNUCIQCsFLaIJZiN4jlgbxlyLVeya9lLuqIwvqqPQl6q4ad12QIgS9gG48xmdHig\n8z3IdIMedZ8ZCtKmEun6Cp1+BsK0wDUCIF0nHfSuU+eTQ2qAON2SHIrJf8UeFO7N\nzdTN1IwwQqjI\n-----END PRIVATE KEY-----\n", - "client_id": "client_id" + "client_id": "client_id", } authenticator = GoogleServiceKeyAuthenticator(credentials) auth_request = requests_mock.register_uri( - "POST", - authenticator._google_oauth2_token_endpoint, - json={"access_token": "bearer_token", "expires_in": 3600} + "POST", authenticator._google_oauth2_token_endpoint, json={"access_token": "bearer_token", "expires_in": 3600} ) authenticated_request = authenticator(requests.Request()) assert auth_request.call_count == 1 - assert auth_request.last_request.qs.get("assertion") == ['eyj0exaioijkv1qilcjhbgcioijsuzi1niisimtpzci6imnsawvudf9pzcj9.eyjpc3mioijjbgllbnrfzw1hawwilcjzy29wzsi6imh0dhbzoi8vd3d3lmdvb2dszwfwaxmuy29tl2f1dggvyw5hbhl0awnzlnjlywrvbmx5iiwiyxvkijoiahr0chm6ly9vyxv0adiuz29vz2xlyxbpcy5jb20vdg9rzw4ilcjlehaioje2nzi1mzq4mdasimlhdci6mty3mjuzmtiwmh0.u1gpfmncrtlsy_ujxpc2iazpvdzb6eq4mobq3xez5v6gqtj0xgou__c6neu9d7qvb8h0jkynggsfibkoci_g7a'] + assert auth_request.last_request.qs.get("assertion") == [ + "eyj0exaioijkv1qilcjhbgcioijsuzi1niisimtpzci6imnsawvudf9pzcj9.eyjpc3mioijjbgllbnrfzw1hawwilcjzy29wzsi6imh0dhbzoi8vd3d3lmdvb2dszwfwaxmuy29tl2f1dggvyw5hbhl0awnzlnjlywrvbmx5iiwiyxvkijoiahr0chm6ly9vyxv0adiuz29vz2xlyxbpcy5jb20vdg9rzw4ilcjlehaioje2nzi1mzq4mdasimlhdci6mty3mjuzmtiwmh0.u1gpfmncrtlsy_ujxpc2iazpvdzb6eq4mobq3xez5v6gqtj0xgou__c6neu9d7qvb8h0jkynggsfibkoci_g7a" + ] assert auth_request.last_request.qs.get("grant_type") == ["urn:ietf:params:oauth:grant-type:jwt-bearer"] assert authenticator._token.get("expires_at") == 1672534800 assert authenticated_request.headers.get("Authorization") == "Bearer bearer_token" diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration.py new file mode 100644 index 000000000000..a0df1b3f59d2 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration.py @@ -0,0 +1,36 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import patch + +from airbyte_cdk.entrypoint import AirbyteEntrypoint +from source_google_analytics_data_api import SourceGoogleAnalyticsDataApi +from source_google_analytics_data_api.config_migrations import MigratePropertyID + + +@patch.object(SourceGoogleAnalyticsDataApi, "read_config") +@patch.object(SourceGoogleAnalyticsDataApi, "write_config") +@patch.object(AirbyteEntrypoint, "extract_config") +def test_migration(ab_entrypoint_extract_config_mock, source_write_config_mock, source_read_config_mock): + source = SourceGoogleAnalyticsDataApi() + + source_read_config_mock.return_value = { + "credentials": {"auth_type": "Service", "credentials_json": ""}, + "custom_reports": "", + "date_ranges_start_date": "2023-09-01", + "window_in_days": 30, + "property_id": "111111111", + } + ab_entrypoint_extract_config_mock.return_value = "/path/to/config.json" + + def check_migrated_value(new_config, path): + assert path == "/path/to/config.json" + assert "property_id" not in new_config + assert "property_ids" in new_config + assert "111111111" in new_config["property_ids"] + assert len(new_config["property_ids"]) == 1 + + source_write_config_mock.side_effect = check_migrated_value + + MigratePropertyID.migrate(["--config", "/path/to/config.json"], source) diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config.json b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config.json new file mode 100644 index 000000000000..245a01f07016 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config.json @@ -0,0 +1,59 @@ +{ + "credentials": { + "auth_type": "Service", + "credentials_json": "" + }, + "date_ranges_start_date": "2023-09-01", + "window_in_days": 30, + "property_ids": "314186564", + "custom_reports_array": [ + { + "name": "cohort_report", + "dimensions": ["cohort", "cohortNthDay"], + "metrics": ["cohortActiveUsers"], + "cohortSpec": { + "cohorts": [ + { + "dimension": "firstSessionDate", + "dateRange": { + "startDate": "2023-04-24", + "endDate": "2023-04-24" + } + } + ], + "cohortsRange": { + "endOffset": 100, + "granularity": "DAILY" + }, + "cohortReportSettings": { + "accumulate": false + } + } + }, + { + "name": "pivot_report", + "dateRanges": [ + { + "startDate": "2020-09-01", + "endDate": "2020-09-15" + } + ], + "dimensions": ["browser", "country", "language"], + "metrics": ["sessions"], + "pivots": [ + { + "fieldNames": ["browser"], + "limit": 5 + }, + { + "fieldNames": ["country"], + "limit": 250 + }, + { + "fieldNames": ["language"], + "limit": 15 + } + ] + } + ] +} diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config_migration_cohortspec.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config_migration_cohortspec.py new file mode 100644 index 000000000000..de76bda4e8a4 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_config_migration_cohortspec.py @@ -0,0 +1,45 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import json +import os +from typing import Any, Mapping + +import dpath.util +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source +from source_google_analytics_data_api.config_migrations import MigrateCustomReportsCohortSpec +from source_google_analytics_data_api.source import SourceGoogleAnalyticsDataApi + +# BASE ARGS +CMD = "check" +TEST_CONFIG_PATH = f"{os.path.dirname(__file__)}/test_config.json" +NEW_TEST_CONFIG_PATH = f"{os.path.dirname(__file__)}/test_new_config.json" +SOURCE_INPUT_ARGS = [CMD, "--config", TEST_CONFIG_PATH] +SOURCE: Source = SourceGoogleAnalyticsDataApi() + + +# HELPERS +def load_config(config_path: str = TEST_CONFIG_PATH) -> Mapping[str, Any]: + with open(config_path, "r") as config: + return json.load(config) + + +def test_migrate_config(capsys): + migration_instance = MigrateCustomReportsCohortSpec() + # migrate the test_config + migration_instance.migrate(SOURCE_INPUT_ARGS, SOURCE) + + control_msg = json.loads(capsys.readouterr().out) + assert control_msg["type"] == Type.CONTROL.value + assert control_msg["control"]["type"] == OrchestratorType.CONNECTOR_CONFIG.value + + assert control_msg["control"]["connectorConfig"]["config"]["custom_reports_array"][0]["cohortSpec"]["enabled"] == "true" + assert control_msg["control"]["connectorConfig"]["config"]["custom_reports_array"][1]["cohortSpec"]["enabled"] == "false" + + +def test_should_not_migrate_new_config(): + new_config = load_config(NEW_TEST_CONFIG_PATH) + assert not MigrateCustomReportsCohortSpec._should_migrate(new_config) diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_new_config.json b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_new_config.json new file mode 100644 index 000000000000..fd7ddcd7ce9f --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migration_cohortspec/test_new_config.json @@ -0,0 +1,63 @@ +{ + "credentials": { + "auth_type": "Service", + "credentials_json": "" + }, + "date_ranges_start_date": "2023-09-01", + "window_in_days": 30, + "property_ids": "314186564", + "custom_reports_array": [ + { + "name": "cohort_report", + "dimensions": ["cohort", "cohortNthDay"], + "metrics": ["cohortActiveUsers"], + "cohortSpec": { + "cohorts": [ + { + "dimension": "firstSessionDate", + "dateRange": { + "startDate": "2023-04-24", + "endDate": "2023-04-24" + } + } + ], + "cohortsRange": { + "endOffset": 100, + "granularity": "DAILY" + }, + "cohortReportSettings": { + "accumulate": false + }, + "enable": "true" + } + }, + { + "name": "pivot_report", + "dateRanges": [ + { + "startDate": "2020-09-01", + "endDate": "2020-09-15" + } + ], + "dimensions": ["browser", "country", "language"], + "metrics": ["sessions"], + "pivots": [ + { + "fieldNames": ["browser"], + "limit": 5 + }, + { + "fieldNames": ["country"], + "limit": 250 + }, + { + "fieldNames": ["language"], + "limit": 15 + } + ], + "cohortSpec": { + "enabled": "false" + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migrations/test_config.json b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migrations/test_config.json new file mode 100644 index 000000000000..9b00b00f949f --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migrations/test_config.json @@ -0,0 +1,10 @@ +{ + "credentials": { + "auth_type": "Service", + "credentials_json": "" + }, + "custom_reports": "[{\"name\": \"custom_dimensions\", \"dimensions\": [\"date\", \"country\", \"device\"]}]", + "date_ranges_start_date": "2023-09-01", + "window_in_days": 30, + "property_ids": "314186564" +} diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migrations/test_config_migrations.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migrations/test_config_migrations.py new file mode 100644 index 000000000000..5bee2f8ab6b9 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_migrations/test_config_migrations.py @@ -0,0 +1,75 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import json +from typing import Any, Mapping + +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source +from source_google_analytics_data_api.config_migrations import MigrateCustomReports +from source_google_analytics_data_api.source import SourceGoogleAnalyticsDataApi + +# BASE ARGS +CMD = "check" +TEST_CONFIG_PATH = "unit_tests/test_migrations/test_config.json" +SOURCE_INPUT_ARGS = [CMD, "--config", TEST_CONFIG_PATH] +SOURCE: Source = SourceGoogleAnalyticsDataApi() + + +# HELPERS +def load_config(config_path: str = TEST_CONFIG_PATH) -> Mapping[str, Any]: + with open(config_path, "r") as config: + return json.load(config) + + +def revert_migration(config_path: str = TEST_CONFIG_PATH) -> None: + with open(config_path, "r") as test_config: + config = json.load(test_config) + config.pop("custom_reports_array") + with open(config_path, "w") as updated_config: + config = json.dumps(config) + updated_config.write(config) + + +def test_migrate_config(capsys): + migration_instance = MigrateCustomReports() + original_config = load_config() + # migrate the test_config + migration_instance.migrate(SOURCE_INPUT_ARGS, SOURCE) + # load the updated config + test_migrated_config = load_config() + # check migrated property + assert "custom_reports_array" in test_migrated_config + assert isinstance(test_migrated_config["custom_reports_array"], list) + # check the old property is in place + assert "custom_reports" in test_migrated_config + assert isinstance(test_migrated_config["custom_reports"], str) + # check the migration should be skipped, once already done + assert not migration_instance._should_migrate(test_migrated_config) + # load the old custom reports VS migrated + assert json.loads(original_config["custom_reports"]) == test_migrated_config["custom_reports_array"] + # test CONTROL MESSAGE was emitted + control_msg = json.loads(capsys.readouterr().out) + assert control_msg["type"] == Type.CONTROL.value + assert control_msg["control"]["type"] == OrchestratorType.CONNECTOR_CONFIG.value + # old custom_reports are stil type(str) + assert isinstance(control_msg["control"]["connectorConfig"]["config"]["custom_reports"], str) + # new custom_reports are type(list) + assert isinstance(control_msg["control"]["connectorConfig"]["config"]["custom_reports_array"], list) + # check the migrated values + assert control_msg["control"]["connectorConfig"]["config"]["custom_reports_array"][0]["name"] == "custom_dimensions" + assert control_msg["control"]["connectorConfig"]["config"]["custom_reports_array"][0]["dimensions"] == ["date", "country", "device"] + # revert the test_config to the starting point + revert_migration() + + +def test_config_is_reverted(): + # check the test_config state, it has to be the same as before tests + test_config = load_config() + # check the config no longer has the migarted property + assert "custom_reports_array" not in test_config + # check the old property is still there + assert "custom_reports" in test_config + assert isinstance(test_config["custom_reports"], str) diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py index e2ca394d1147..631b2a1d8683 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py @@ -2,131 +2,167 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import datetime -import json -from copy import deepcopy from unittest.mock import MagicMock import pytest from airbyte_cdk.models import AirbyteConnectionStatus, FailureType, Status from airbyte_cdk.utils import AirbyteTracedException from source_google_analytics_data_api import SourceGoogleAnalyticsDataApi +from source_google_analytics_data_api.source import MetadataDescriptor from source_google_analytics_data_api.utils import NO_DIMENSIONS, NO_METRICS, NO_NAME, WRONG_JSON_SYNTAX -json_credentials = """ -{ - "type": "service_account", - "project_id": "unittest-project-id", - "private_key_id": "9qf98e52oda52g5ne23al6evnf13649c2u077162c", - "private_key": "-----BEGIN PRIVATE KEY-----\\nMIIBVQIBADANBgkqhkiG9w0BAQEFAASCAT8wggE7AgEAAkEA3slcXL+dA36ESmOi\\n1xBhZmp5Hn0WkaHDtW4naba3plva0ibloBNWhFhjQOh7Ff01PVjhT4D5jgqXBIgc\\nz9Gv3QIDAQABAkEArlhYPoD5SB2/O1PjwHgiMPrL1C9B9S/pr1cH4vPJnpY3VKE3\\n5hvdil14YwRrcbmIxMkK2iRLi9lM4mJmdWPy4QIhAPsRFXZSGx0TZsDxD9V0ZJmZ\\n0AuDCj/NF1xB5KPLmp7pAiEA4yoFox6w7ql/a1pUVaLt0NJkDfE+22pxYGNQaiXU\\nuNUCIQCsFLaIJZiN4jlgbxlyLVeya9lLuqIwvqqPQl6q4ad12QIgS9gG48xmdHig\\n8z3IdIMedZ8ZCtKmEun6Cp1+BsK0wDUCIF0nHfSuU+eTQ2qAON2SHIrJf8UeFO7N\\nzdTN1IwwQqjI\\n-----END PRIVATE KEY-----\\n", - "client_email": "google-analytics-access@unittest-project-id.iam.gserviceaccount.com", - "client_id": "213243192021686092537", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/google-analytics-access%40unittest-project-id.iam.gserviceaccount.com" -} -""" - - -@pytest.fixture -def patch_base_class(): - return { - "config": { - "property_id": "108176369", - "credentials": {"auth_type": "Service", "credentials_json": json_credentials}, - "date_ranges_start_date": datetime.datetime.strftime((datetime.datetime.now() - datetime.timedelta(days=1)), "%Y-%m-%d"), - } - } - - -@pytest.fixture -def config(): - return { - "property_id": "108176369", - "credentials": {"auth_type": "Service", "credentials_json": json_credentials}, - "date_ranges_start_date": datetime.datetime.strftime((datetime.datetime.now() - datetime.timedelta(days=1)), "%Y-%m-%d"), - "custom_reports": json.dumps([{ - "name": "report1", - "dimensions": ["date", "country"], - "metrics": ["totalUsers", "screenPageViews"] - }]), - } - - -@pytest.fixture -def config_gen(config): - def inner(**kwargs): - new_config = deepcopy(config) - # WARNING, no support deep dictionaries - new_config.update(kwargs) - return {k: v for k, v in new_config.items() if v is not ...} - - return inner - @pytest.mark.parametrize( "config_values, is_successful, message", [ ({}, Status.SUCCEEDED, None), - ({"custom_reports": ...}, Status.SUCCEEDED, None), - ({"custom_reports": "[]"}, Status.SUCCEEDED, None), - ({"custom_reports": "invalid"}, Status.FAILED, f"'{WRONG_JSON_SYNTAX}'"), - ({"custom_reports": "{}"}, Status.FAILED, f"'{WRONG_JSON_SYNTAX}'"), - ({"custom_reports": "[{}]"}, Status.FAILED, f"'{NO_NAME}'"), - ({"custom_reports": "[{\"name\": \"name\"}]"}, Status.FAILED, f"'{NO_DIMENSIONS}'"), - ({"custom_reports": "[{\"name\": \"daily_active_users\", \"dimensions\": [\"date\"]}]"}, Status.FAILED, f"'{NO_METRICS}'"), - ({"custom_reports": "[{\"name\": \"daily_active_users\", \"metrics\": [\"totalUsers\"], \"dimensions\": [{\"name\": \"city\"}]}]"}, Status.FAILED, '"The custom report daily_active_users entered contains invalid dimensions: {\'name\': \'city\'} is not of type \'string\'. Validate your custom query with the GA 4 Query Explorer (https://ga-dev-tools.google/ga4/query-explorer/)."'), - ({"date_ranges_start_date": "2022-20-20"}, Status.FAILED, '"time data \'2022-20-20\' does not match format \'%Y-%m-%d\'"'), - ({"credentials": {"auth_type": "Service", "credentials_json": "invalid"}}, - Status.FAILED, "'credentials.credentials_json is not valid JSON'"), - ({"custom_reports": "[{\"name\": \"name\", \"dimensions\": [], \"metrics\": []}]"}, Status.FAILED, "'The custom report name entered contains invalid dimensions: [] is too short. Validate your custom query with the GA 4 Query Explorer (https://ga-dev-tools.google/ga4/query-explorer/).'"), - ({"custom_reports": "[{\"name\": \"daily_active_users\", \"dimensions\": [\"date\"], \"metrics\": [\"totalUsers\"]}]"}, Status.FAILED, "'custom_reports: daily_active_users already exist as a default report(s).'"), - ({"custom_reports": "[{\"name\": \"name\", \"dimensions\": [\"unknown\"], \"metrics\": [\"totalUsers\"]}]"}, - Status.FAILED, "'The custom report name entered contains invalid dimensions: unknown. Validate your custom query with the GA 4 Query Explorer (https://ga-dev-tools.google/ga4/query-explorer/).'"), - ({"custom_reports": "[{\"name\": \"name\", \"dimensions\": [\"date\"], \"metrics\": [\"unknown\"]}]"}, Status.FAILED, "'The custom report name entered contains invalid metrics: unknown. Validate your custom query with the GA 4 Query Explorer (https://ga-dev-tools.google/ga4/query-explorer/).'"), - ({"custom_reports": "[{\"name\": \"cohort_report\", \"dimensions\": [\"cohort\", \"cohortNthDay\"], \"metrics\": " - "[\"cohortActiveUsers\"], \"cohortSpec\": {\"cohorts\": [{\"dimension\": \"firstSessionDate\", \"dateRange\": " - "{\"startDate\": \"2023-01-01\", \"endDate\": \"2023-01-01\"}}], \"cohortsRange\": {\"endOffset\": 100}}}]"}, - Status.FAILED, '"custom_reports.0.cohortSpec.cohortsRange: \'granularity\' is a required property"'), - ({"custom_reports": "[{\"name\": \"pivot_report\", \"dateRanges\": [{ \"startDate\": \"2020-09-01\", \"endDate\": " - "\"2020-09-15\" }], \"dimensions\": [\"browser\", \"country\", \"language\"], \"metrics\": [\"sessions\"], " - "\"pivots\": {}}]"}, - Status.FAILED, '"The custom report pivot_report entered contains invalid pivots: {} is not of type \'null\', \'array\'. Ensure the pivot follow the syntax described in the docs (https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/Pivot)."'), + ({"custom_reports_array": ...}, Status.SUCCEEDED, None), + ({"custom_reports_array": "[]"}, Status.SUCCEEDED, None), + ({"custom_reports_array": "invalid"}, Status.FAILED, f"'{WRONG_JSON_SYNTAX}'"), + ({"custom_reports_array": "{}"}, Status.FAILED, f"'{WRONG_JSON_SYNTAX}'"), + ({"custom_reports_array": "[{}]"}, Status.FAILED, f"'{NO_NAME}'"), + ({"custom_reports_array": '[{"name": "name"}]'}, Status.FAILED, f"'{NO_DIMENSIONS}'"), + ({"custom_reports_array": '[{"name": "daily_active_users", "dimensions": ["date"]}]'}, Status.FAILED, f"'{NO_METRICS}'"), + ( + {"custom_reports_array": '[{"name": "daily_active_users", "metrics": ["totalUsers"], "dimensions": [{"name": "city"}]}]'}, + Status.FAILED, + "\"The custom report daily_active_users entered contains invalid dimensions: {'name': 'city'} is not of type 'string'. Validate your custom query with the GA 4 Query Explorer (https://ga-dev-tools.google/ga4/query-explorer/).\"", + ), + ({"date_ranges_start_date": "2022-20-20"}, Status.FAILED, "\"time data '2022-20-20' does not match format '%Y-%m-%d'\""), + ( + {"credentials": {"auth_type": "Service", "credentials_json": "invalid"}}, + Status.FAILED, + "'credentials.credentials_json is not valid JSON'", + ), + ( + {"custom_reports_array": '[{"name": "name", "dimensions": [], "metrics": []}]'}, + Status.FAILED, + "'The custom report name entered contains invalid dimensions: [] is too short. Validate your custom query with the GA 4 Query Explorer (https://ga-dev-tools.google/ga4/query-explorer/).'", + ), + ( + {"custom_reports_array": '[{"name": "daily_active_users", "dimensions": ["date"], "metrics": ["totalUsers"]}]'}, + Status.FAILED, + "'Custom reports: daily_active_users already exist as a default report(s).'", + ), + ( + {"custom_reports_array": '[{"name": "name", "dimensions": ["unknown"], "metrics": ["totalUsers"]}]'}, + Status.FAILED, + "'The custom report name entered contains invalid dimensions: unknown. Validate your custom query with the GA 4 Query Explorer (https://ga-dev-tools.google/ga4/query-explorer/).'", + ), + ( + {"custom_reports_array": '[{"name": "name", "dimensions": ["date"], "metrics": ["unknown"]}]'}, + Status.FAILED, + "'The custom report name entered contains invalid metrics: unknown. Validate your custom query with the GA 4 Query Explorer (https://ga-dev-tools.google/ga4/query-explorer/).'", + ), + ( + { + "custom_reports_array": '[{"name": "pivot_report", "dateRanges": [{ "startDate": "2020-09-01", "endDate": "2020-09-15" }], "dimensions": ["browser", "country", "language"], "metrics": ["sessions"], "pivots": {}}]' + }, + Status.FAILED, + "\"The custom report pivot_report entered contains invalid pivots: {} is not of type 'null', 'array'. Ensure the pivot follow the syntax described in the docs (https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/Pivot).\"", + ), ], ) def test_check(requests_mock, config_gen, config_values, is_successful, message): - requests_mock.register_uri("POST", "https://oauth2.googleapis.com/token", - json={"access_token": "access_token", "expires_in": 3600, "token_type": "Bearer"}) - - requests_mock.register_uri("GET", "https://analyticsdata.googleapis.com/v1beta/properties/108176369/metadata", - json={"dimensions": [{"apiName": "date"}, {"apiName": "country"}, - {"apiName": "language"}, {"apiName": "browser"}], - "metrics": [{"apiName": "totalUsers"}, {"apiName": "screenPageViews"}, {"apiName": "sessions"}]}) - requests_mock.register_uri("POST", "https://analyticsdata.googleapis.com/v1beta/properties/108176369:runReport", - json={"dimensionHeaders": [{"name": "date"}, {"name": "country"}], - "metricHeaders": [{"name": "totalUsers", "type": "s"}, - {"name": "screenPageViews", "type": "m"}], - "rows": [] - }) - requests_mock.register_uri("GET", "https://analyticsdata.googleapis.com/v1beta/properties/UA-11111111/metadata", - json={}, status_code=403) + requests_mock.register_uri( + "POST", "https://oauth2.googleapis.com/token", json={"access_token": "access_token", "expires_in": 3600, "token_type": "Bearer"} + ) + + requests_mock.register_uri( + "GET", + "https://analyticsdata.googleapis.com/v1beta/properties/108176369/metadata", + json={ + "dimensions": [{"apiName": "date"}, {"apiName": "country"}, {"apiName": "language"}, {"apiName": "browser"}], + "metrics": [{"apiName": "totalUsers"}, {"apiName": "screenPageViews"}, {"apiName": "sessions"}], + }, + ) + requests_mock.register_uri( + "POST", + "https://analyticsdata.googleapis.com/v1beta/properties/108176369:runReport", + json={ + "dimensionHeaders": [{"name": "date"}, {"name": "country"}], + "metricHeaders": [{"name": "totalUsers", "type": "s"}, {"name": "screenPageViews", "type": "m"}], + "rows": [], + }, + ) source = SourceGoogleAnalyticsDataApi() logger = MagicMock() assert source.check(logger, config_gen(**config_values)) == AirbyteConnectionStatus(status=is_successful, message=message) - if not is_successful: - with pytest.raises(AirbyteTracedException) as e: - source.check(logger, config_gen(property_id="UA-11111111")) - assert e.value.failure_type == FailureType.config_error -def test_streams(mocker, patch_base_class): +def test_check_failure(requests_mock, config_gen): + requests_mock.register_uri( + "POST", "https://oauth2.googleapis.com/token", json={"access_token": "access_token", "expires_in": 3600, "token_type": "Bearer"} + ) + requests_mock.register_uri( + "GET", "https://analyticsdata.googleapis.com/v1beta/properties/UA-11111111/metadata", json={}, status_code=403 + ) + source = SourceGoogleAnalyticsDataApi() + logger = MagicMock() + with pytest.raises(AirbyteTracedException) as e: + source.check(logger, config_gen(property_ids=["UA-11111111"])) + assert e.value.failure_type == FailureType.config_error + assert "Access was denied to the property ID entered." in e.value.message + + +@pytest.mark.parametrize( + ("status_code", "expected_message"), + ( + (403, "Please check configuration for custom report cohort_report. "), + (400, "Please check configuration for custom report cohort_report. Granularity in the cohortsRange is required."), + ), +) +def test_check_incorrect_custom_reports_config(requests_mock, config_gen, status_code, expected_message): + requests_mock.register_uri( + "POST", "https://oauth2.googleapis.com/token", json={"access_token": "access_token", "expires_in": 3600, "token_type": "Bearer"} + ) + requests_mock.register_uri( + "GET", + "https://analyticsdata.googleapis.com/v1beta/properties/108176369/metadata", + json={ + "dimensions": [{"apiName": "date"}, {"apiName": "country"}, {"apiName": "language"}, {"apiName": "browser"}], + "metrics": [{"apiName": "totalUsers"}, {"apiName": "screenPageViews"}, {"apiName": "sessions"}], + }, + ) + requests_mock.register_uri( + "POST", + "https://analyticsdata.googleapis.com/v1beta/properties/108176369:runReport", + status_code=status_code, + json={"error": {"message": "Granularity in the cohortsRange is required."}}, + ) + config = {"custom_reports_array": '[{"name": "cohort_report", "dimensions": ["date"], "metrics": ["totalUsers"]}]'} source = SourceGoogleAnalyticsDataApi() + logger = MagicMock() + status, message = source.check_connection(logger, config_gen(**config)) + assert status is False + assert message == expected_message + + +@pytest.mark.parametrize("status_code", (403, 401)) +def test_missing_metadata(requests_mock, status_code): + # required for MetadataDescriptor $instance input + class TestConfig: + config = { + "authenticator": None, + "property_id": 123, + } + + # mocking the url for metadata + requests_mock.register_uri( + "GET", "https://analyticsdata.googleapis.com/v1beta/properties/123/metadata", json={}, status_code=status_code + ) - config_mock = MagicMock() - config_mock.__getitem__.side_effect = patch_base_class["config"].__getitem__ + metadata_descriptor = MetadataDescriptor() + with pytest.raises(AirbyteTracedException) as e: + metadata_descriptor.__get__(TestConfig(), None) + assert e.value.failure_type == FailureType.config_error - streams = source.streams(patch_base_class["config"]) - expected_streams_number = 8 - assert len(streams) == expected_streams_number + +def test_streams(patch_base_class, config_gen): + config = config_gen(property_ids=["Prop1", "PropN"]) + source = SourceGoogleAnalyticsDataApi() + streams = source.streams(config) + expected_streams_number = 57 * 2 + assert len([stream for stream in streams if "_property_" in stream.name]) == 57 + assert len(set(streams)) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py index 4654981e70bb..94597c7dc184 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_streams.py @@ -14,47 +14,41 @@ from .utils import read_incremental -json_credentials = """ -{ - "type": "service_account", - "project_id": "unittest-project-id", - "private_key_id": "9qf98e52oda52g5ne23al6evnf13649c2u077162c", - "private_key": "", - "client_email": "google-analytics-access@unittest-project-id.iam.gserviceaccount.com", - "client_id": "213243192021686092537", - "auth_uri": "https://accounts.google.com/o/oauth2/auth", - "token_uri": "https://oauth2.googleapis.com/token", - "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/google-analytics-access%40unittest-project-id.iam.gserviceaccount.com" -} -""" - @pytest.fixture -def patch_base_class(mocker): +def patch_base_class(mocker, config, config_without_date_range): # Mock abstract methods to enable instantiating abstract class mocker.patch.object(GoogleAnalyticsDataApiBaseStream, "path", f"{random.randint(100000000, 999999999)}:runReport") mocker.patch.object(GoogleAnalyticsDataApiBaseStream, "primary_key", "test_primary_key") mocker.patch.object(GoogleAnalyticsDataApiBaseStream, "__abstractmethods__", set()) - return { - "config": { - "property_id": "496180525", - "credentials": {"auth_type": "Service", "credentials_json": json_credentials}, - "dimensions": ["date", "deviceCategory", "operatingSystem", "browser"], - "metrics": [ - "totalUsers", - "newUsers", - "sessions", - "sessionsPerUser", - "averageSessionDuration", - "screenPageViews", - "screenPageViewsPerSession", - "bounceRate", - ], - "date_ranges_start_date": datetime.datetime.strftime((datetime.datetime.now() - datetime.timedelta(days=1)), "%Y-%m-%d"), - } - } + return {"config": config, "config_without_date_range": config_without_date_range} + + +def test_json_schema(requests_mock, patch_base_class): + requests_mock.register_uri( + "POST", "https://oauth2.googleapis.com/token", json={"access_token": "access_token", "expires_in": 3600, "token_type": "Bearer"} + ) + requests_mock.register_uri( + "GET", + "https://analyticsdata.googleapis.com/v1beta/properties/108176369/metadata", + json={ + "dimensions": [{"apiName": "date"}, {"apiName": "country"}, {"apiName": "language"}, {"apiName": "browser"}], + "metrics": [{"apiName": "totalUsers"}, {"apiName": "screenPageViews"}, {"apiName": "sessions"}], + }, + ) + schema = GoogleAnalyticsDataApiBaseStream( + authenticator=MagicMock(), config={"authenticator": MagicMock(), **patch_base_class["config_without_date_range"]} + ).get_json_schema() + + for d in patch_base_class["config_without_date_range"]["dimensions"]: + assert d in schema["properties"] + + for p in patch_base_class["config_without_date_range"]["metrics"]: + assert p in schema["properties"] + + assert "startDate" in schema["properties"] + assert "endDate" in schema["properties"] def test_request_params(patch_base_class): @@ -86,6 +80,7 @@ def test_request_body_json(patch_base_class): {"name": "operatingSystem"}, {"name": "browser"}, ], + "keepEmptyRows": True, "dateRanges": [request_body_params["stream_slice"]], "returnPropertyQuota": True, "offset": str(0), @@ -159,7 +154,7 @@ def test_parse_response(patch_base_class): {"name": "totalUsers", "type": "TYPE_INTEGER"}, {"name": "newUsers", "type": "TYPE_INTEGER"}, {"name": "sessions", "type": "TYPE_INTEGER"}, - {"name": "sessionsPerUser", "type": "TYPE_FLOAT"}, + {"name": "sessionsPerUser:parameter", "type": "TYPE_FLOAT"}, {"name": "averageSessionDuration", "type": "TYPE_SECONDS"}, {"name": "screenPageViews", "type": "TYPE_INTEGER"}, {"name": "screenPageViewsPerSession", "type": "TYPE_FLOAT"}, @@ -200,7 +195,7 @@ def test_parse_response(patch_base_class): expected_data = [ { - "property_id": "496180525", + "property_id": "108176369", "date": "20220731", "deviceCategory": "desktop", "operatingSystem": "Macintosh", @@ -208,14 +203,14 @@ def test_parse_response(patch_base_class): "totalUsers": 344, "newUsers": 169, "sessions": 420, - "sessionsPerUser": 1.2209302325581395, + "sessionsPerUser:parameter": 1.2209302325581395, "averageSessionDuration": 194.76313766428572, "screenPageViews": 614, "screenPageViewsPerSession": 1.4619047619047618, "bounceRate": 0.47857142857142859, }, { - "property_id": "496180525", + "property_id": "108176369", "date": "20220731", "deviceCategory": "desktop", "operatingSystem": "Windows", @@ -223,7 +218,7 @@ def test_parse_response(patch_base_class): "totalUsers": 322, "newUsers": 211, "sessions": 387, - "sessionsPerUser": 1.2018633540372672, + "sessionsPerUser:parameter": 1.2018633540372672, "averageSessionDuration": 249.21595714211884, "screenPageViews": 669, "screenPageViewsPerSession": 1.7286821705426356, @@ -307,10 +302,11 @@ def test_stream_slices(): def test_read_incremental(requests_mock): config = { + "property_ids": [123], "property_id": 123, - "date_ranges_start_date": datetime.date(2022, 12, 29), + "date_ranges_start_date": datetime.date(2022, 1, 6), "window_in_days": 1, - "dimensions": ["date"], + "dimensions": ["yearWeek"], "metrics": ["totalUsers"], } @@ -319,52 +315,52 @@ def test_read_incremental(requests_mock): responses = [ { - "dimensionHeaders": [{"name": "date"}], + "dimensionHeaders": [{"name": "yearWeek"}], "metricHeaders": [{"name": "totalUsers", "type": "TYPE_INTEGER"}], - "rows": [{"dimensionValues": [{"value": "20221229"}], "metricValues": [{"value": "100"}]}], + "rows": [{"dimensionValues": [{"value": "202201"}], "metricValues": [{"value": "100"}]}], "rowCount": 1, }, { - "dimensionHeaders": [{"name": "date"}], + "dimensionHeaders": [{"name": "yearWeek"}], "metricHeaders": [{"name": "totalUsers", "type": "TYPE_INTEGER"}], - "rows": [{"dimensionValues": [{"value": "20221230"}], "metricValues": [{"value": "110"}]}], + "rows": [{"dimensionValues": [{"value": "202201"}], "metricValues": [{"value": "110"}]}], "rowCount": 1, }, { - "dimensionHeaders": [{"name": "date"}], + "dimensionHeaders": [{"name": "yearWeek"}], "metricHeaders": [{"name": "totalUsers", "type": "TYPE_INTEGER"}], - "rows": [{"dimensionValues": [{"value": "20221231"}], "metricValues": [{"value": "120"}]}], + "rows": [{"dimensionValues": [{"value": "202201"}], "metricValues": [{"value": "120"}]}], "rowCount": 1, }, { - "dimensionHeaders": [{"name": "date"}], + "dimensionHeaders": [{"name": "yearWeek"}], "metricHeaders": [{"name": "totalUsers", "type": "TYPE_INTEGER"}], - "rows": [{"dimensionValues": [{"value": "20230101"}], "metricValues": [{"value": "130"}]}], + "rows": [{"dimensionValues": [{"value": "202202"}], "metricValues": [{"value": "130"}]}], "rowCount": 1, }, # 2-nd incremental read { - "dimensionHeaders": [{"name": "date"}], + "dimensionHeaders": [{"name": "yearWeek"}], "metricHeaders": [{"name": "totalUsers", "type": "TYPE_INTEGER"}], - "rows": [{"dimensionValues": [{"value": "20221230"}], "metricValues": [{"value": "112"}]}], - "rowCount": 1 + "rows": [{"dimensionValues": [{"value": "202202"}], "metricValues": [{"value": "112"}]}], + "rowCount": 1, }, { - "dimensionHeaders": [{"name": "date"}], + "dimensionHeaders": [{"name": "yearWeek"}], "metricHeaders": [{"name": "totalUsers", "type": "TYPE_INTEGER"}], - "rows": [{"dimensionValues": [{"value": "20221231"}], "metricValues": [{"value": "125"}]}], - "rowCount": 1 + "rows": [{"dimensionValues": [{"value": "202202"}], "metricValues": [{"value": "125"}]}], + "rowCount": 1, }, { - "dimensionHeaders": [{"name": "date"}], + "dimensionHeaders": [{"name": "yearWeek"}], "metricHeaders": [{"name": "totalUsers", "type": "TYPE_INTEGER"}], - "rows": [{"dimensionValues": [{"value": "20230101"}], "metricValues": [{"value": "140"}]}], + "rows": [{"dimensionValues": [{"value": "202202"}], "metricValues": [{"value": "140"}]}], "rowCount": 1, }, { - "dimensionHeaders": [{"name": "date"}], + "dimensionHeaders": [{"name": "yearWeek"}], "metricHeaders": [{"name": "totalUsers", "type": "TYPE_INTEGER"}], - "rows": [{"dimensionValues": [{"value": "20230102"}], "metricValues": [{"value": "150"}]}], + "rows": [{"dimensionValues": [{"value": "202202"}], "metricValues": [{"value": "150"}]}], "rowCount": 1, }, ] @@ -375,24 +371,23 @@ def test_read_incremental(requests_mock): json=lambda request, context: responses.pop(0), ) - with freeze_time("2023-01-01 12:00:00"): + with freeze_time("2022-01-09 12:00:00"): records = list(read_incremental(stream, stream_state)) - + print(records) assert records == [ - {"date": "20221229", "totalUsers": 100, "property_id": 123}, - {"date": "20221230", "totalUsers": 110, "property_id": 123}, - {"date": "20221231", "totalUsers": 120, "property_id": 123}, - {"date": "20230101", "totalUsers": 130, "property_id": 123}, + {"property_id": 123, "yearWeek": "202201", "totalUsers": 100, "startDate": "2022-01-06", "endDate": "2022-01-06"}, + {"property_id": 123, "yearWeek": "202201", "totalUsers": 110, "startDate": "2022-01-07", "endDate": "2022-01-07"}, + {"property_id": 123, "yearWeek": "202201", "totalUsers": 120, "startDate": "2022-01-08", "endDate": "2022-01-08"}, + {"property_id": 123, "yearWeek": "202202", "totalUsers": 130, "startDate": "2022-01-09", "endDate": "2022-01-09"}, ] - assert stream_state == {"date": "20230101"} + assert stream_state == {"yearWeek": "202202"} - with freeze_time("2023-01-02 12:00:00"): + with freeze_time("2022-01-10 12:00:00"): records = list(read_incremental(stream, stream_state)) assert records == [ - {"date": "20221230", "totalUsers": 112, "property_id": 123}, - {"date": "20221231", "totalUsers": 125, "property_id": 123}, - {"date": "20230101", "totalUsers": 140, "property_id": 123}, - {"date": "20230102", "totalUsers": 150, "property_id": 123}, + {"property_id": 123, "yearWeek": "202202", "totalUsers": 112, "startDate": "2022-01-08", "endDate": "2022-01-08"}, + {"property_id": 123, "yearWeek": "202202", "totalUsers": 125, "startDate": "2022-01-09", "endDate": "2022-01-09"}, + {"property_id": 123, "yearWeek": "202202", "totalUsers": 140, "startDate": "2022-01-10", "endDate": "2022-01-10"}, ] diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_utils.py new file mode 100644 index 000000000000..6a203cc4dc74 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_utils.py @@ -0,0 +1,164 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import sys +from unittest.mock import Mock, mock_open, patch + +import pytest +from source_google_analytics_data_api.utils import ( + get_source_defined_primary_key, + serialize_to_date_string, + transform_between_filter, + transform_expression, + transform_in_list_filter, + transform_json, + transform_numeric_filter, + transform_string_filter, +) + + +class TestSerializeToDateString: + @pytest.mark.parametrize( + "input_date, date_format, date_type, expected", + [ + ("202105", "%Y-%m-%d", "yearWeek", "2021-02-01"), + ("202105", "%Y-%m-%d", "yearMonth", "2021-05-01"), + ("202245", "%Y-%m-%d", "yearWeek", "2022-11-07"), + ("202210", "%Y-%m-%d", "yearMonth", "2022-10-01"), + ("2022", "%Y-%m-%d", "year", "2022-01-01"), + ], + ) + def test_valid_cases(self, input_date, date_format, date_type, expected): + result = serialize_to_date_string(input_date, date_format, date_type) + assert result == expected + + def test_invalid_type(self): + with pytest.raises(ValueError): + serialize_to_date_string("202105", "%Y-%m-%d", "invalidType") + + +class TestTransformFilters: + def test_transform_string_filter(self): + filter_data = {"value": "test", "matchType": ["partial"], "caseSensitive": True} + expected = {"stringFilter": {"value": "test", "matchType": "partial", "caseSensitive": True}} + result = transform_string_filter(filter_data) + assert result == expected + + def test_transform_in_list_filter(self): + filter_data = {"values": ["test1", "test2"], "caseSensitive": False} + expected = {"inListFilter": {"values": ["test1", "test2"], "caseSensitive": False}} + result = transform_in_list_filter(filter_data) + assert result == expected + + def test_transform_numeric_filter(self): + filter_data = {"value": {"value_type": "doubleValue", "value": 5.5}, "operation": ["equals"]} + expected = {"numericFilter": {"value": {"doubleValue": 5.5}, "operation": "equals"}} + result = transform_numeric_filter(filter_data) + assert result == expected + + @pytest.mark.parametrize( + "filter_data, expected", + [ + ( + {"fromValue": {"value_type": "doubleValue", "value": "10.5"}, "toValue": {"value_type": "doubleValue", "value": "20.5"}}, + {"betweenFilter": {"fromValue": {"doubleValue": 10.5}, "toValue": {"doubleValue": 20.5}}}, + ), + ( + {"fromValue": {"value_type": "stringValue", "value": "hello"}, "toValue": {"value_type": "stringValue", "value": "world"}}, + {"betweenFilter": {"fromValue": {"stringValue": "hello"}, "toValue": {"stringValue": "world"}}}, + ), + ( + {"fromValue": {"value_type": "doubleValue", "value": 10.5}, "toValue": {"value_type": "doubleValue", "value": 20.5}}, + {"betweenFilter": {"fromValue": {"doubleValue": 10.5}, "toValue": {"doubleValue": 20.5}}}, + ), + ], + ) + def test_transform_between_filter(self, filter_data, expected): + result = transform_between_filter(filter_data) + assert result == expected + + +class TestTransformExpression: + @patch("source_google_analytics_data_api.utils.transform_string_filter", Mock(return_value={"stringFilter": "mocked_string_filter"})) + @patch("source_google_analytics_data_api.utils.transform_in_list_filter", Mock(return_value={"inListFilter": "mocked_in_list_filter"})) + @patch("source_google_analytics_data_api.utils.transform_numeric_filter", Mock(return_value={"numericFilter": "mocked_numeric_filter"})) + def test_between_filter(self): + expression = { + "field_name": "some_field", + "filter": { + "filter_name": "betweenFilter", + "fromValue": {"value_type": "doubleValue", "value": "10.5"}, + "toValue": {"value_type": "doubleValue", "value": "20.5"}, + }, + } + expected = { + "filter": {"fieldName": "some_field", "betweenFilter": {"fromValue": {"doubleValue": 10.5}, "toValue": {"doubleValue": 20.5}}} + } + result = transform_expression(expression) + assert result == expected + + +class TestGetSourceDefinedPrimaryKey: + @pytest.mark.parametrize( + "stream_name, mocked_content, expected", + [ + ("sample_stream", {"streams": [{"stream": {"name": "sample_stream", "source_defined_primary_key": ["id"]}}]}, ["id"]), + ("sample_stream", {"streams": [{"stream": {"name": "different_stream", "source_defined_primary_key": ["id"]}}]}, None), + ], + ) + def test_primary_key(self, stream_name, mocked_content, expected): + sys.argv = ["script_name", "read", "--catalog", "mocked_catalog_path"] + m = mock_open(read_data=json.dumps(mocked_content)) + with patch("builtins.open", m): + with patch("json.loads", return_value=mocked_content): + result = get_source_defined_primary_key(stream_name) + assert result == expected + + +class TestTransformJson: + @staticmethod + def mock_transform_expression(expression): + return {"transformed": expression} + + # Applying pytest monkeypatch for the mock_transform_expression + @pytest.fixture(autouse=True) + def mock_transform_functions(self, monkeypatch): + monkeypatch.setattr("source_google_analytics_data_api.utils.transform_expression", self.mock_transform_expression) + + @pytest.mark.parametrize( + "original, expected", + [ + ( + { + "filter_type": "andGroup", + "expressions": [{"field": "field1", "condition": "cond1"}, {"field": "field2", "condition": "cond2"}], + }, + { + "andGroup": { + "expressions": [ + {"transformed": {"field": "field1", "condition": "cond1"}}, + {"transformed": {"field": "field2", "condition": "cond2"}}, + ] + } + }, + ), + ( + {"filter_type": "orGroup", "expressions": [{"field": "field1", "condition": "cond1"}]}, + {"orGroup": {"expressions": [{"transformed": {"field": "field1", "condition": "cond1"}}]}}, + ), + ( + {"filter_type": "notExpression", "expression": {"field": "field1", "condition": "cond1"}}, + {"notExpression": {"transformed": {"field": "field1", "condition": "cond1"}}}, + ), + ( + {"filter_type": "filter", "field": "field1", "condition": "cond1"}, + {"transformed": {"condition": "cond1", "field": "field1", "filter_type": "filter"}}, + ), + ({"filter_type": "andGroup"}, {}), + ], + ) + def test_cases(self, original, expected): + result = transform_json(original) + assert result == expected diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile b/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile deleted file mode 100644 index 076145b76938..000000000000 --- a/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY setup.py ./ -COPY source_google_analytics_v4 ./source_google_analytics_v4 -COPY main.py ./ -RUN pip install . - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.2.1 -LABEL io.airbyte.name=airbyte/source-google-analytics-v4 diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/README.md b/airbyte-integrations/connectors/source-google-analytics-v4/README.md index 93aaa5533ceb..4530bcf8a465 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/README.md +++ b/airbyte-integrations/connectors/source-google-analytics-v4/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-google-analytics-v4:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/google-analytics-v4) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_analytics_v4/spec.json` file. @@ -56,19 +48,70 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-google-analytics-v4:dev + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-google-analytics-v4 build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-google-analytics-v4:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. -You can also build the connector image via Gradle: +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-google-analytics-v4:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-google-analytics-v4:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-google-analytics-v4:dev . +# Running the spec command against your patched connector +docker run airbyte/source-google-analytics-v4:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -77,44 +120,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-analytics-v4:de docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-analytics-v4:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-google-analytics-v4:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install '.[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-google-analytics-v4 test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-google-analytics-v4:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-google-analytics-v4:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -124,8 +139,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-google-analytics-v4 test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/google-analytics-v4.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-analytics-v4/acceptance-test-config.yml index f14f9a0d4cf7..e7e9b896674e 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-analytics-v4/acceptance-test-config.yml @@ -28,7 +28,6 @@ acceptance_tests: timeout_seconds: 2400 future_state: future_state_path: integration_tests/abnormal_state.json - threshold_days: 2 spec: tests: - spec_path: source_google_analytics_v4/spec.json diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-google-analytics-v4/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-google-analytics-v4/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/build.gradle b/airbyte-integrations/connectors/source-google-analytics-v4/build.gradle deleted file mode 100644 index 24f594ce9be6..000000000000 --- a/airbyte-integrations/connectors/source-google-analytics-v4/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_google_analytics_v4' -} diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/metadata.yaml b/airbyte-integrations/connectors/source-google-analytics-v4/metadata.yaml index 4b17e0c7619f..921875383f55 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-analytics-v4/metadata.yaml @@ -1,30 +1,32 @@ data: + ab_internal: + ql: 400 + sl: 200 allowedHosts: hosts: - oauth2.googleapis.com - www.googleapis.com - analyticsdata.googleapis.com - analyticsreporting.googleapis.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: api connectorType: source definitionId: eff3616a-f9c3-11eb-9a03-0242ac130003 - dockerImageTag: 0.2.1 + dockerImageTag: 0.2.3 dockerRepository: airbyte/source-google-analytics-v4 + documentationUrl: https://docs.airbyte.com/integrations/sources/google-analytics-v4 githubIssueLabel: source-google-analytics-v4 icon: google-analytics.svg license: Elv2 name: Google Analytics (Universal Analytics) registries: cloud: - enabled: true + enabled: false oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/google-analytics-v4 + supportLevel: certified tags: - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py index d607904310ce..3d81036fa5d1 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py @@ -102,6 +102,7 @@ class GoogleAnalyticsV4Stream(HttpStream, ABC): def __init__(self, config: MutableMapping): super().__init__(authenticator=config["authenticator"]) self.start_date = config["start_date"] + self.end_date = config.get("end_date") self.window_in_days: int = config.get("window_in_days", 1) self.view_id = config["view_id"] self.metrics = config["metrics"] @@ -255,7 +256,7 @@ def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs: Any) - ...] """ - end_date = pendulum.now().date() + end_date = (pendulum.parse(self.end_date) if self.end_date else pendulum.now()).date() start_date = pendulum.parse(self.start_date).date() if stream_state: prev_end_date = pendulum.parse(stream_state.get(self.cursor_field)).date() diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json index 33fefa97a7ca..c4a8078bfbd0 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json @@ -91,9 +91,18 @@ "title": "View ID", "description": "The ID for the Google Analytics View you want to fetch data from. This can be found from the Google Analytics Account Explorer." }, - "custom_reports": { + "end_date": { "order": 3, "type": "string", + "title": "Replication End Date", + "description": "The date in the format YYYY-MM-DD. Any data after this date will not be replicated.", + "examples": ["2020-06-01"], + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$|^$|[\\s\\S]+$", + "format": "date" + }, + "custom_reports": { + "order": 4, + "type": "string", "title": "Custom Reports", "description": "A JSON array describing the custom reports you want to sync from Google Analytics. See the docs for more information about the exact format you can use to fill out this field." }, @@ -103,7 +112,7 @@ "description": "The time increment used by the connector when requesting data from the Google Analytics API. More information is available in the the docs. The bigger this value is, the faster the sync will be, but the more likely that sampling will be applied to your data, potentially causing inaccuracies in the returned results. We recommend setting this to 1 unless you have a hard requirement to make the sync faster at the expense of accuracy. The minimum allowed value for this field is 1, and the maximum is 364. ", "examples": [30, 60, 90, 120, 200, 364], "default": 1, - "order": 4 + "order": 5 } } }, diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/test_custom_reports_validator.py b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/test_custom_reports_validator.py index f724857256fe..4becd6c2d323 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/test_custom_reports_validator.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/test_custom_reports_validator.py @@ -10,19 +10,16 @@ @pytest.mark.parametrize( "custom_reports, expected", ( - ([{"name": "test", "dimensions": ["ga+test"], "metrics": ["ga!test"]}], "errors: incorrect field reference"), - ([{"name": [], "dimensions": ["ga:test"], "metrics": ["ga:test"]}], "errors: type errors"), - ([{"name": "test", "dimensions": ["ga:test"], "metrics": ["ga:test"], "added_field": "test"}], "errors: fields not permitted"), - ([{"name": "missing_segment_dimension", "dimensions": ["ga:test"], "segments": ["another_segment"],"metrics": ["ga:test"]}], "errors: `ga:segment` is required"), - ([{"missing_name": "test", "dimensions": ["ga:test"], "metrics": ["ga:test"]}], "errors: fields required"), + ([{"name": "test", "dimensions": ["ga+test"], "metrics": ["ga!test"]}], "errors: incorrect field reference"), + ([{"name": [], "dimensions": ["ga:test"], "metrics": ["ga:test"]}], "errors: type errors"), + ([{"name": "test", "dimensions": ["ga:test"], "metrics": ["ga:test"], "added_field": "test"}], "errors: fields not permitted"), + ( + [{"name": "missing_segment_dimension", "dimensions": ["ga:test"], "segments": ["another_segment"], "metrics": ["ga:test"]}], + "errors: `ga:segment` is required", + ), + ([{"missing_name": "test", "dimensions": ["ga:test"], "metrics": ["ga:test"]}], "errors: fields required"), ), - ids=[ - "incorrrect field reference", - "type_error", - "not_permitted", - "missing", - "missing_segment_dimension" - ] + ids=["incorrrect field reference", "type_error", "not_permitted", "missing", "missing_segment_dimension"], ) def test_custom_reports_validator(custom_reports, expected): try: diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py index 791cd28d3391..a4a9f276ba14 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py @@ -92,28 +92,24 @@ def test_no_regressions_for_result_is_sampled_and_data_is_golden_warnings( @patch("source_google_analytics_v4.source.jwt") def test_check_connection_fails_jwt( - jwt_encode_mock, - test_config_auth_service, - requests_mock, - mock_metrics_dimensions_type_list_link, - mock_auth_call + jwt_encode_mock, test_config_auth_service, requests_mock, mock_metrics_dimensions_type_list_link, mock_auth_call ): """ check_connection fails because of the API returns no records, then we assume than user doesn't have permission to read requested `view` """ source = SourceGoogleAnalyticsV4() - requests_mock.register_uri("POST", "https://analyticsreporting.googleapis.com/v4/reports:batchGet", - [{"status_code": 403, - "json": {"results": [], - "error": "User does not have sufficient permissions for this profile."}}]) + requests_mock.register_uri( + "POST", + "https://analyticsreporting.googleapis.com/v4/reports:batchGet", + [{"status_code": 403, "json": {"results": [], "error": "User does not have sufficient permissions for this profile."}}], + ) is_success, msg = source.check_connection(MagicMock(), test_config_auth_service) assert is_success is False assert ( - msg - == f"Please check the permissions for the requested view_id: {test_config_auth_service['view_id']}. " - f"User does not have sufficient permissions for this profile." + msg == f"Please check the permissions for the requested view_id: {test_config_auth_service['view_id']}. " + f"User does not have sufficient permissions for this profile." ) jwt_encode_mock.encode.assert_called() assert mock_auth_call.called @@ -142,27 +138,22 @@ def test_check_connection_success_jwt( @patch("source_google_analytics_v4.source.jwt") -def test_check_connection_fails_oauth( - jwt_encode_mock, - test_config, - mock_metrics_dimensions_type_list_link, - mock_auth_call, - requests_mock -): +def test_check_connection_fails_oauth(jwt_encode_mock, test_config, mock_metrics_dimensions_type_list_link, mock_auth_call, requests_mock): """ check_connection fails because of the API returns no records, then we assume than user doesn't have permission to read requested `view` """ source = SourceGoogleAnalyticsV4() - requests_mock.register_uri("POST", "https://analyticsreporting.googleapis.com/v4/reports:batchGet", - [{"status_code": 403, - "json": {"results": [], - "error": "User does not have sufficient permissions for this profile."}}]) + requests_mock.register_uri( + "POST", + "https://analyticsreporting.googleapis.com/v4/reports:batchGet", + [{"status_code": 403, "json": {"results": [], "error": "User does not have sufficient permissions for this profile."}}], + ) is_success, msg = source.check_connection(MagicMock(), test_config) assert is_success is False assert ( msg == f"Please check the permissions for the requested view_id: {test_config['view_id']}." - f" User does not have sufficient permissions for this profile." + f" User does not have sufficient permissions for this profile." ) jwt_encode_mock.encode.assert_not_called() assert "https://www.googleapis.com/auth/analytics.readonly" in unquote(mock_auth_call.last_request.body) diff --git a/airbyte-integrations/connectors/source-google-directory/README.md b/airbyte-integrations/connectors/source-google-directory/README.md index da4643af524c..103cf550af2b 100644 --- a/airbyte-integrations/connectors/source-google-directory/README.md +++ b/airbyte-integrations/connectors/source-google-directory/README.md @@ -1,7 +1,7 @@ -# Google Directory Source +# Freshsales Source -This is the repository for the Google Directory source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/google-directory). +This is the repository for the Freshsales source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/freshsales). ## Local development @@ -21,6 +21,7 @@ development environment of choice. To activate it from the terminal, run: ``` source .venv/bin/activate pip install -r requirements.txt +pip install '.[tests]' ``` If you are in an IDE, follow your IDE's instructions to activate the virtualenv. @@ -29,72 +30,71 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-google-directory:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/google-directory) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_directory/spec.json` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/freshsales) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_freshsales/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source google-directory test creds` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source freshsales test creds` and place them into `secrets/config.json`. - ### Locally running the connector ``` -python main_dev.py spec -python main_dev.py check --config secrets/config.json -python main_dev.py discover --config secrets/config.json -python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json ``` ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-google-directory:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name source-freshsales build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-google-directory:airbyteDocker +An image will be built with the tag `airbyte/source-freshsales:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-freshsales:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-google-directory:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-directory:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-directory:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-google-directory:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm airbyte/source-freshsales:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-freshsales:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-freshsales:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-freshsales:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-google-directory:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-google-directory test +``` + +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-google-directory test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/google-directory.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-google-directory/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-google-directory/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-google-directory/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-google-directory/build.gradle b/airbyte-integrations/connectors/source-google-directory/build.gradle deleted file mode 100644 index b05ed036c08e..000000000000 --- a/airbyte-integrations/connectors/source-google-directory/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_google_directory' -} diff --git a/airbyte-integrations/connectors/source-google-drive/.dockerignore b/airbyte-integrations/connectors/source-google-drive/.dockerignore new file mode 100644 index 000000000000..648834f6fa95 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/.dockerignore @@ -0,0 +1,7 @@ +* +!Dockerfile +!Dockerfile.test +!main.py +!source_google_drive +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-google-drive/README.md b/airbyte-integrations/connectors/source-google-drive/README.md new file mode 100644 index 000000000000..586d5cdecae9 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/README.md @@ -0,0 +1,100 @@ +# Google Drive Source + +This is the repository for the Google Drive source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/google-drive). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.10.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/google-drive) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_drive/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source google-drive test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + + +#### Build +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-google-drive build +``` + +An image will be built with the tag `airbyte/source-google-drive:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-google-drive:dev . +``` + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-google-drive:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-drive:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-drive:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-google-drive:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-google-drive test +``` + +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-google-drive test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/google-drive.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-google-drive/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-drive/acceptance-test-config.yml new file mode 100644 index 000000000000..5714d50eb620 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/acceptance-test-config.yml @@ -0,0 +1,44 @@ +acceptance_tests: + basic_read: + tests: + - config_path: secrets/config.json + expect_records: + path: integration_tests/expected_records.jsonl + exact_order: false + timeout_seconds: 1800 + expect_trace_message_on_failure: false + - config_path: secrets/oauth_config.json + expect_records: + path: integration_tests/expected_records.jsonl + exact_order: false + timeout_seconds: 1800 + expect_trace_message_on_failure: false + + connection: + tests: + - config_path: secrets/config.json + status: succeed + - config_path: secrets/oauth_config.json + status: succeed + - config_path: integration_tests/invalid_config.json + status: failed + discovery: + tests: + - config_path: secrets/config.json + full_refresh: + tests: + - config_path: secrets/config.json + configured_catalog_path: integration_tests/configured_catalog.json + timeout_seconds: 1800 + + incremental: + tests: + - config_path: secrets/config.json + configured_catalog_path: integration_tests/configured_catalog.json + future_state: + future_state_path: integration_tests/abnormal_state.json + timeout_seconds: 1800 + spec: + tests: + - spec_path: integration_tests/spec.json +connector_image: airbyte/source-google-drive:dev diff --git a/airbyte-integrations/connectors/source-google-drive/examples/configured_catalog.json b/airbyte-integrations/connectors/source-google-drive/examples/configured_catalog.json new file mode 100644 index 000000000000..31f1bb6e02ee --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/examples/configured_catalog.json @@ -0,0 +1,13 @@ +{ + "streams": [ + { + "stream": { + "name": "my_file_stream", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-google-drive/examples/state.json b/airbyte-integrations/connectors/source-google-drive/examples/state.json new file mode 100644 index 000000000000..341d344b3c0c --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/examples/state.json @@ -0,0 +1,16 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "my_file_stream" + }, + "stream_state": { + "history": { + "https://drive.google.com/file/d/1kbXDc8UFkQcWCRfKKtSjCtlgD7EUXzuM": "2023-10-16T14:51:33.000000Z" + }, + "_ab_source_file_last_modified": "2023-10-16T14:51:33.000000Z_https://drive.google.com/file/d/1kbXDc8UFkQcWCRfKKtSjCtlgD7EUXzuM" + } + } + } +] diff --git a/airbyte-integrations/connectors/source-google-drive/icon.svg b/airbyte-integrations/connectors/source-google-drive/icon.svg new file mode 100644 index 000000000000..1133b624babf --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/icon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-google-drive/integration_tests/__init__.py b/airbyte-integrations/connectors/source-google-drive/integration_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-google-drive/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-google-drive/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..49a1209da298 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/integration_tests/abnormal_state.json @@ -0,0 +1,35 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "test" + }, + "stream_state": { + "history": { + "test.jsonl": "2023-10-16T06:16:06.000000Z", + "subfolder/test2.jsonl": "2023-10-19T01:43:56.000000Z" + }, + "_ab_source_file_last_modified": "2023-10-19T01:43:56.000000Z_subfolder/test2.jsonl" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "test_unstructured" + }, + "stream_state": { + "history": { + "testdoc_docx.docx": "2023-10-27T00:45:54.000000Z", + "testdoc_pdf.pdf": "2023-10-27T00:45:58.000000Z", + "testdoc_ocr_pdf.pdf": "2023-10-27T00:46:04.000000Z", + "testdoc_google": "2023-11-10T13:46:18.551000Z", + "testdoc_presentation": "2023-11-10T13:49:06.640000Z" + }, + "_ab_source_file_last_modified": "2023-11-10T13:49:06.640000Z_testdoc_presentation" + } + } + } +] diff --git a/airbyte-integrations/connectors/source-google-drive/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-drive/integration_tests/acceptance.py new file mode 100644 index 000000000000..6b0c294530cd --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from typing import Iterable + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup() -> Iterable[None]: + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-google-drive/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-google-drive/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..de9b2ddee6a8 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/integration_tests/configured_catalog.json @@ -0,0 +1,60 @@ +{ + "streams": [ + { + "stream": { + "name": "test", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": ["null", "integer"] + }, + "x": { + "type": ["null", "integer"] + }, + "_ab_source_file_last_modified": { + "type": "string", + "format": "date-time" + }, + "_ab_source_file_url": { + "type": "string" + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["_ab_source_file_last_modified"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "test_unstructured", + "json_schema": { + "type": "object", + "properties": { + "document_key": { + "type": ["null", "integer"] + }, + "content": { + "type": ["null", "integer"] + }, + "_ab_source_file_last_modified": { + "type": "string", + "format": "date-time" + }, + "_ab_source_file_url": { + "type": "string" + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["_ab_source_file_last_modified"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-google-drive/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-google-drive/integration_tests/expected_records.jsonl new file mode 100644 index 000000000000..005c1cb48aa7 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/integration_tests/expected_records.jsonl @@ -0,0 +1,9 @@ +{"stream": "test", "data": {"x": 999, "_ab_source_file_last_modified": "2023-10-16T06:16:06.000000Z", "_ab_source_file_url": "test.jsonl"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"x": 9999, "_ab_source_file_last_modified": "2023-10-16T06:16:06.000000Z", "_ab_source_file_url": "test.jsonl"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"y": 9999, "_ab_source_file_last_modified": "2023-10-19T01:43:56.000000Z", "_ab_source_file_url": "subfolder/test2.jsonl"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"y": 123, "_ab_source_file_last_modified": "2023-10-19T01:43:56.000000Z", "_ab_source_file_url": "subfolder/test2.jsonl"}, "emitted_at": 162727468000} +{"stream": "test_unstructured", "data": {"content": "# Heading\n\nThis is the content which is not just a single word", "document_key": "testdoc_docx.docx", "_ab_source_file_last_modified": "2023-10-27T00:45:54.000000Z", "_ab_source_file_url": "testdoc_docx.docx", "_ab_source_file_parse_error": null}, "emitted_at": 1698400261867} +{"stream": "test_unstructured", "data": {"content": "# Heading\n\nThis is the content which is not just a single word", "document_key": "testdoc_pdf.pdf", "_ab_source_file_last_modified": "2023-10-27T00:45:58.000000Z", "_ab_source_file_url": "testdoc_pdf.pdf", "_ab_source_file_parse_error": null}, "emitted_at": 1698400264556} +{"stream": "test_unstructured", "data": {"content": "This is a test", "document_key": "testdoc_ocr_pdf.pdf", "_ab_source_file_last_modified": "2023-10-27T00:46:04.000000Z", "_ab_source_file_url": "testdoc_ocr_pdf.pdf", "_ab_source_file_parse_error": null}, "emitted_at": 1698400267184} +{"stream": "test_unstructured", "data": {"content": "# Heading\n\nThis is the content which is not just a single word", "document_key": "testdoc_google", "_ab_source_file_last_modified": "2023-11-10T13:46:18.551000Z", "_ab_source_file_url": "testdoc_google", "_ab_source_file_parse_error": null}, "emitted_at": 1698400261074} +{"stream": "test_unstructured", "data": {"content": "This is a test", "document_key": "testdoc_presentation", "_ab_source_file_last_modified": "2023-11-10T13:49:06.640000Z", "_ab_source_file_url": "testdoc_presentation", "_ab_source_file_parse_error": null}, "emitted_at": 1698402779268} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-google-drive/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-google-drive/integration_tests/invalid_config.json new file mode 100644 index 000000000000..eca04cd9ad7d --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/integration_tests/invalid_config.json @@ -0,0 +1,19 @@ +{ + "folder_url": "https://drive.google.com/drive/folders/yyy", + "credentials": { + "auth_type": "Service", + "service_account_info": "abc" + }, + "streams": [ + { + "name": "test", + "globs": ["**/*.jsonl"], + "format": { + "filetype": "jsonl" + }, + "schemaless": false, + "validation_policy": "Emit Record", + "days_to_sync_if_history_is_full": 3 + } + ] +} diff --git a/airbyte-integrations/connectors/source-google-drive/integration_tests/spec.json b/airbyte-integrations/connectors/source-google-drive/integration_tests/spec.json new file mode 100644 index 000000000000..e1341d1bbe27 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/integration_tests/spec.json @@ -0,0 +1,456 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/google-drive", + "connectionSpecification": { + "title": "Google Drive Source Spec", + "description": "Used during spec; allows the developer to configure the cloud provider specific options\nthat are needed when users configure a file-based source.", + "type": "object", + "properties": { + "start_date": { + "title": "Start Date", + "description": "UTC date and time in the format 2017-01-25T00:00:00.000000Z. Any file modified before this date will not be replicated.", + "examples": ["2021-01-01T00:00:00.000000Z"], + "format": "date-time", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{6}Z$", + "pattern_descriptor": "YYYY-MM-DDTHH:mm:ss.SSSSSSZ", + "order": 1, + "type": "string" + }, + "streams": { + "title": "The list of streams to sync", + "description": "Each instance of this configuration defines a stream. Use this to define which files belong in the stream, their format, and how they should be parsed and validated. When sending data to warehouse destination such as Snowflake or BigQuery, each stream is a separate table.", + "order": 10, + "type": "array", + "items": { + "title": "FileBasedStreamConfig", + "type": "object", + "properties": { + "name": { + "title": "Name", + "description": "The name of the stream.", + "type": "string" + }, + "globs": { + "title": "Globs", + "default": ["**"], + "order": 1, + "description": "The pattern used to specify which files should be selected from the file system. For more information on glob pattern matching look here.", + "type": "array", + "items": { + "type": "string" + } + }, + "validation_policy": { + "title": "Validation Policy", + "description": "The name of the validation policy that dictates sync behavior when a record does not adhere to the stream schema.", + "default": "Emit Record", + "enum": ["Emit Record", "Skip Record", "Wait for Discover"] + }, + "input_schema": { + "title": "Input Schema", + "description": "The schema that will be used to validate records extracted from the file. This will override the stream schema that is auto-detected from incoming files.", + "type": "string" + }, + "primary_key": { + "title": "Primary Key", + "description": "The column or columns (for a composite key) that serves as the unique identifier of a record. If empty, the primary key will default to the parser's default primary key.", + "type": "string", + "airbyte_hidden": true + }, + "days_to_sync_if_history_is_full": { + "title": "Days To Sync If History Is Full", + "description": "When the state history of the file store is full, syncs will only read files that were last modified in the provided day range.", + "default": 3, + "type": "integer" + }, + "format": { + "title": "Format", + "description": "The configuration options that are used to alter how to read incoming files that deviate from the standard formatting.", + "type": "object", + "oneOf": [ + { + "title": "Avro Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "avro", + "const": "avro", + "type": "string" + }, + "double_as_string": { + "title": "Convert Double Fields to Strings", + "description": "Whether to convert double fields to strings. This is recommended if you have decimal numbers with a high degree of precision because there can be a loss precision when handling floating point numbers.", + "default": false, + "type": "boolean" + } + }, + "required": ["filetype"] + }, + { + "title": "CSV Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "csv", + "const": "csv", + "type": "string" + }, + "delimiter": { + "title": "Delimiter", + "description": "The character delimiting individual cells in the CSV data. This may only be a 1-character string. For tab-delimited data enter '\\t'.", + "default": ",", + "type": "string" + }, + "quote_char": { + "title": "Quote Character", + "description": "The character used for quoting CSV values. To disallow quoting, make this field blank.", + "default": "\"", + "type": "string" + }, + "escape_char": { + "title": "Escape Character", + "description": "The character used for escaping special characters. To disallow escaping, leave this field blank.", + "type": "string" + }, + "encoding": { + "title": "Encoding", + "description": "The character encoding of the CSV data. Leave blank to default to UTF8. See list of python encodings for allowable options.", + "default": "utf8", + "type": "string" + }, + "double_quote": { + "title": "Double Quote", + "description": "Whether two quotes in a quoted CSV value denote a single quote in the data.", + "default": true, + "type": "boolean" + }, + "null_values": { + "title": "Null Values", + "description": "A set of case-sensitive strings that should be interpreted as null values. For example, if the value 'NA' should be interpreted as null, enter 'NA' in this field.", + "default": [], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "strings_can_be_null": { + "title": "Strings Can Be Null", + "description": "Whether strings can be interpreted as null values. If true, strings that match the null_values set will be interpreted as null. If false, strings that match the null_values set will be interpreted as the string itself.", + "default": true, + "type": "boolean" + }, + "skip_rows_before_header": { + "title": "Skip Rows Before Header", + "description": "The number of rows to skip before the header row. For example, if the header row is on the 3rd row, enter 2 in this field.", + "default": 0, + "type": "integer" + }, + "skip_rows_after_header": { + "title": "Skip Rows After Header", + "description": "The number of rows to skip after the header row.", + "default": 0, + "type": "integer" + }, + "header_definition": { + "title": "CSV Header Definition", + "description": "How headers will be defined. `User Provided` assumes the CSV does not have a header row and uses the headers provided and `Autogenerated` assumes the CSV does not have a header row and the CDK will generate headers using for `f{i}` where `i` is the index starting from 0. Else, the default behavior is to use the header from the CSV file. If a user wants to autogenerate or provide column names for a CSV having headers, they can skip rows.", + "default": { + "header_definition_type": "From CSV" + }, + "oneOf": [ + { + "title": "From CSV", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "From CSV", + "const": "From CSV", + "type": "string" + } + }, + "required": ["header_definition_type"] + }, + { + "title": "Autogenerated", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "Autogenerated", + "const": "Autogenerated", + "type": "string" + } + }, + "required": ["header_definition_type"] + }, + { + "title": "User Provided", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "User Provided", + "const": "User Provided", + "type": "string" + }, + "column_names": { + "title": "Column Names", + "description": "The column names that will be used while emitting the CSV records", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": ["column_names", "header_definition_type"] + } + ], + "type": "object" + }, + "true_values": { + "title": "True Values", + "description": "A set of case-sensitive strings that should be interpreted as true values.", + "default": ["y", "yes", "t", "true", "on", "1"], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "false_values": { + "title": "False Values", + "description": "A set of case-sensitive strings that should be interpreted as false values.", + "default": ["n", "no", "f", "false", "off", "0"], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "required": ["filetype"] + }, + { + "title": "Jsonl Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "jsonl", + "const": "jsonl", + "type": "string" + } + }, + "required": ["filetype"] + }, + { + "title": "Parquet Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "parquet", + "const": "parquet", + "type": "string" + }, + "decimal_as_float": { + "title": "Convert Decimal Fields to Floats", + "description": "Whether to convert decimal fields to floats. There is a loss of precision when converting decimals to floats, so this is not recommended.", + "default": false, + "type": "boolean" + } + }, + "required": ["filetype"] + }, + { + "title": "Document File Type Format (Experimental)", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "unstructured", + "const": "unstructured", + "type": "string" + }, + "skip_unprocessable_files": { + "type": "boolean", + "default": true, + "title": "Skip Unprocessable Files", + "description": "If true, skip files that cannot be parsed and pass the error message along as the _ab_source_file_parse_error field. If false, fail the sync.", + "always_show": true + }, + "strategy": { + "type": "string", + "always_show": true, + "order": 0, + "default": "auto", + "title": "Parsing Strategy", + "enum": ["auto", "fast", "ocr_only", "hi_res"], + "description": "The strategy used to parse documents. `fast` extracts text directly from the document which doesn't work for all files. `ocr_only` is more reliable, but slower. `hi_res` is the most reliable, but requires an API key and a hosted instance of unstructured and can't be used with local mode. See the unstructured.io documentation for more details: https://unstructured-io.github.io/unstructured/core/partition.html#partition-pdf" + }, + "processing": { + "title": "Processing", + "description": "Processing configuration", + "default": { + "mode": "local" + }, + "type": "object", + "oneOf": [ + { + "title": "Local", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "local", + "const": "local", + "enum": ["local"], + "type": "string" + } + }, + "description": "Process files locally, supporting `fast` and `ocr` modes. This is the default option.", + "required": ["mode"] + } + ] + } + }, + "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file.", + "required": ["filetype"] + } + ] + }, + "schemaless": { + "title": "Schemaless", + "description": "When enabled, syncs will not validate or structure records against the stream's schema.", + "default": false, + "type": "boolean" + } + }, + "required": ["name", "format"] + } + }, + "folder_url": { + "title": "Folder Url", + "description": "URL for the folder you want to sync. Using individual streams and glob patterns, it's possible to only sync a subset of all files located in the folder.", + "examples": [ + "https://drive.google.com/drive/folders/1Xaz0vXXXX2enKnNYU5qSt9NS70gvMyYn" + ], + "order": 0, + "pattern": "^https://drive.google.com/.+", + "pattern_descriptor": "https://drive.google.com/drive/folders/MY-FOLDER-ID", + "type": "string" + }, + "credentials": { + "title": "Authentication", + "description": "Credentials for connecting to the Google Drive API", + "type": "object", + "oneOf": [ + { + "title": "Authenticate via Google (OAuth)", + "type": "object", + "properties": { + "auth_type": { + "title": "Auth Type", + "default": "Client", + "const": "Client", + "enum": ["Client"], + "type": "string" + }, + "client_id": { + "title": "Client ID", + "description": "Client ID for the Google Drive API", + "airbyte_secret": true, + "type": "string" + }, + "client_secret": { + "title": "Client Secret", + "description": "Client Secret for the Google Drive API", + "airbyte_secret": true, + "type": "string" + }, + "refresh_token": { + "title": "Refresh Token", + "description": "Refresh Token for the Google Drive API", + "airbyte_secret": true, + "type": "string" + } + }, + "required": [ + "client_id", + "client_secret", + "refresh_token", + "auth_type" + ] + }, + { + "title": "Service Account Key Authentication", + "type": "object", + "properties": { + "auth_type": { + "title": "Auth Type", + "default": "Service", + "const": "Service", + "enum": ["Service"], + "type": "string" + }, + "service_account_info": { + "title": "Service Account Information", + "description": "The JSON key of the service account to use for authorization. Read more here.", + "airbyte_secret": true, + "type": "string" + } + }, + "required": ["service_account_info", "auth_type"] + } + ] + } + }, + "required": ["streams", "folder_url", "credentials"] + }, + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "Client", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "refresh_token": { + "type": "string", + "path_in_connector_config": ["credentials", "refresh_token"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["credentials", "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["credentials", "client_secret"] + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-google-drive/main.py b/airbyte-integrations/connectors/source-google-drive/main.py new file mode 100644 index 000000000000..606e4f7641e8 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/main.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from source_google_drive.run import run + +if __name__ == "__main__": + run() diff --git a/airbyte-integrations/connectors/source-google-drive/metadata.yaml b/airbyte-integrations/connectors/source-google-drive/metadata.yaml new file mode 100644 index 000000000000..bf40629baae0 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/metadata.yaml @@ -0,0 +1,26 @@ +data: + allowedHosts: + hosts: + - "www.googleapis.com" + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 + connectorSubtype: file + connectorType: source + definitionId: 9f8dda77-1048-4368-815b-269bf54ee9b8 + dockerImageTag: 0.0.6 + dockerRepository: airbyte/source-google-drive + githubIssueLabel: source-google-drive + icon: google-drive.svg + license: ELv2 + name: Google Drive + registries: + cloud: + enabled: true + oss: + enabled: true + releaseStage: alpha + documentationUrl: https://docs.airbyte.com/integrations/sources/google-drive + tags: + - language:python + supportLevel: community +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-google-drive/requirements.txt b/airbyte-integrations/connectors/source-google-drive/requirements.txt new file mode 100644 index 000000000000..d6e1198b1ab1 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/requirements.txt @@ -0,0 +1 @@ +-e . diff --git a/airbyte-integrations/connectors/source-google-drive/setup.py b/airbyte-integrations/connectors/source-google-drive/setup.py new file mode 100644 index 000000000000..1015092ff1c8 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/setup.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk[file-based]>=0.57.7", + "google-api-python-client==2.104.0", + "google-auth-httplib2==0.1.1", + "google-auth-oauthlib==1.1.0", + "google-api-python-client-stubs==1.18.0", +] + +TEST_REQUIREMENTS = [ + "pytest-mock~=3.6.1", + "pytest~=6.1", +] + +setup( + name="source_google_drive", + description="Source implementation for Google Drive.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, + entry_points={ + "console_scripts": [ + "source-google-drive=source_google_drive.run:run", + ], + }, +) diff --git a/airbyte-integrations/connectors/source-google-drive/source_google_drive/__init__.py b/airbyte-integrations/connectors/source-google-drive/source_google_drive/__init__.py new file mode 100644 index 000000000000..db5171dbaa57 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/source_google_drive/__init__.py @@ -0,0 +1,6 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from .source import SourceGoogleDrive + +__all__ = ["SourceGoogleDrive"] diff --git a/airbyte-integrations/connectors/source-google-drive/source_google_drive/run.py b/airbyte-integrations/connectors/source-google-drive/source_google_drive/run.py new file mode 100644 index 000000000000..a5605f39c2b1 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/source_google_drive/run.py @@ -0,0 +1,17 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk import AirbyteEntrypoint +from airbyte_cdk.entrypoint import launch +from source_google_drive import SourceGoogleDrive + + +def run(): + args = sys.argv[1:] + catalog_path = AirbyteEntrypoint.extract_catalog(args) + source = SourceGoogleDrive(catalog_path) + launch(source, args) diff --git a/airbyte-integrations/connectors/source-google-drive/source_google_drive/source.py b/airbyte-integrations/connectors/source-google-drive/source_google_drive/source.py new file mode 100644 index 000000000000..fe49fba7fe8c --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/source_google_drive/source.py @@ -0,0 +1,55 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from typing import Any + +from airbyte_cdk.models import AdvancedAuth, ConnectorSpecification, OAuthConfigSpecification +from airbyte_cdk.sources.file_based.file_based_source import FileBasedSource +from airbyte_cdk.sources.file_based.stream.cursor.default_file_based_cursor import DefaultFileBasedCursor +from source_google_drive.spec import SourceGoogleDriveSpec +from source_google_drive.stream_reader import SourceGoogleDriveStreamReader + + +class SourceGoogleDrive(FileBasedSource): + def __init__(self, catalog_path: str): + super().__init__( + stream_reader=SourceGoogleDriveStreamReader(), + spec_class=SourceGoogleDriveSpec, + catalog_path=catalog_path, + cursor_cls=DefaultFileBasedCursor, + ) + + def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: + """ + Returns the specification describing what fields can be configured by a user when setting up a file-based source. + """ + + return ConnectorSpecification( + documentationUrl=self.spec_class.documentation_url(), + connectionSpecification=self.spec_class.schema(), + advanced_auth=AdvancedAuth( + auth_flow_type="oauth2.0", + predicate_key=["credentials", "auth_type"], + predicate_value="Client", + oauth_config_specification=OAuthConfigSpecification( + complete_oauth_output_specification={ + "type": "object", + "additionalProperties": False, + "properties": {"refresh_token": {"type": "string", "path_in_connector_config": ["credentials", "refresh_token"]}}, + }, + complete_oauth_server_input_specification={ + "type": "object", + "additionalProperties": False, + "properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}}, + }, + complete_oauth_server_output_specification={ + "type": "object", + "additionalProperties": False, + "properties": { + "client_id": {"type": "string", "path_in_connector_config": ["credentials", "client_id"]}, + "client_secret": {"type": "string", "path_in_connector_config": ["credentials", "client_secret"]}, + }, + }, + ), + ), + ) diff --git a/airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py b/airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py new file mode 100644 index 000000000000..4bc354dbf4d5 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/source_google_drive/spec.py @@ -0,0 +1,85 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from typing import Any, Dict, Literal, Union + +import dpath.util +from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec +from airbyte_cdk.utils.oneof_option_config import OneOfOptionConfig +from pydantic import BaseModel, Field + + +class OAuthCredentials(BaseModel): + class Config(OneOfOptionConfig): + title = "Authenticate via Google (OAuth)" + discriminator = "auth_type" + + auth_type: Literal["Client"] = Field("Client", const=True) + client_id: str = Field( + title="Client ID", + description="Client ID for the Google Drive API", + airbyte_secret=True, + ) + client_secret: str = Field( + title="Client Secret", + description="Client Secret for the Google Drive API", + airbyte_secret=True, + ) + refresh_token: str = Field( + title="Refresh Token", + description="Refresh Token for the Google Drive API", + airbyte_secret=True, + ) + + +class ServiceAccountCredentials(BaseModel): + class Config(OneOfOptionConfig): + title = "Service Account Key Authentication" + discriminator = "auth_type" + + auth_type: Literal["Service"] = Field("Service", const=True) + service_account_info: str = Field( + title="Service Account Information", + description='The JSON key of the service account to use for authorization. Read more here.', + airbyte_secret=True, + ) + + +class SourceGoogleDriveSpec(AbstractFileBasedSpec, BaseModel): + class Config: + title = "Google Drive Source Spec" + + folder_url: str = Field( + description="URL for the folder you want to sync. Using individual streams and glob patterns, it's possible to only sync a subset of all files located in the folder.", + examples=["https://drive.google.com/drive/folders/1Xaz0vXXXX2enKnNYU5qSt9NS70gvMyYn"], + order=0, + pattern="^https://drive.google.com/.+", + pattern_descriptor="https://drive.google.com/drive/folders/MY-FOLDER-ID", + ) + + credentials: Union[OAuthCredentials, ServiceAccountCredentials] = Field( + title="Authentication", description="Credentials for connecting to the Google Drive API", discriminator="auth_type", type="object" + ) + + @classmethod + def documentation_url(cls) -> str: + return "https://docs.airbyte.com/integrations/sources/google-drive" + + @classmethod + def schema(cls, *args: Any, **kwargs: Any) -> Dict[str, Any]: + """ + Generates the mapping comprised of the config fields + """ + schema = super().schema(*args, **kwargs) + + # Remove legacy settings + dpath.util.delete(schema, "properties/streams/items/properties/legacy_prefix") + dpath.util.delete(schema, "properties/streams/items/properties/format/oneOf/*/properties/inference_type") + + # Hide API processing option until https://github.com/airbytehq/airbyte-platform-internal/issues/10354 is fixed + processing_options = dpath.util.get(schema, "properties/streams/items/properties/format/oneOf/4/properties/processing/oneOf") + dpath.util.set(schema, "properties/streams/items/properties/format/oneOf/4/properties/processing/oneOf", processing_options[:1]) + + return schema diff --git a/airbyte-integrations/connectors/source-google-drive/source_google_drive/stream_reader.py b/airbyte-integrations/connectors/source-google-drive/source_google_drive/stream_reader.py new file mode 100644 index 000000000000..dd786360f7a3 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/source_google_drive/stream_reader.py @@ -0,0 +1,185 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import io +import json +import logging +import re +from datetime import datetime +from io import IOBase +from typing import Iterable, List, Optional, Set + +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.utils.traced_exception import AirbyteTracedException, FailureType +from google.oauth2 import credentials, service_account +from googleapiclient.discovery import build +from googleapiclient.http import MediaIoBaseDownload +from source_google_drive.utils import get_folder_id + +from .spec import SourceGoogleDriveSpec + +FOLDER_MIME_TYPE = "application/vnd.google-apps.folder" +GOOGLE_DOC_MIME_TYPE = "application/vnd.google-apps.document" +EXPORTABLE_DOCUMENTS_MIME_TYPES = [ + GOOGLE_DOC_MIME_TYPE, + "application/vnd.google-apps.presentation", + "application/vnd.google-apps.drawing", +] + + +class GoogleDriveRemoteFile(RemoteFile): + id: str + # The mime type of the file as returned by the Google Drive API + # This is not the same as the mime type when opened by the parser (e.g. google docs is exported as docx) + original_mime_type: str + + +class SourceGoogleDriveStreamReader(AbstractFileBasedStreamReader): + def __init__(self): + super().__init__() + self._drive_service = None + + @property + def config(self) -> SourceGoogleDriveSpec: + return self._config + + @config.setter + def config(self, value: SourceGoogleDriveSpec): + """ + FileBasedSource reads the config from disk and parses it, and once parsed, the source sets the config on its StreamReader. + + Note: FileBasedSource only requires the keys defined in the abstract config, whereas concrete implementations of StreamReader + will require keys that (for example) allow it to authenticate with the 3rd party. + + Therefore, concrete implementations of AbstractFileBasedStreamReader's config setter should assert that `value` is of the correct + config type for that type of StreamReader. + """ + assert isinstance(value, SourceGoogleDriveSpec) + self._config = value + + @property + def google_drive_service(self): + if self.config is None: + # We shouldn't hit this; config should always get set before attempting to + # list or read files. + raise ValueError("Source config is missing; cannot create the Google Drive client.") + try: + if self._drive_service is None: + if self.config.credentials.auth_type == "Client": + creds = credentials.Credentials.from_authorized_user_info(self.config.credentials.dict()) + else: + creds = service_account.Credentials.from_service_account_info(json.loads(self.config.credentials.service_account_info)) + self._drive_service = build("drive", "v3", credentials=creds) + except Exception as e: + raise AirbyteTracedException( + internal_message=str(e), + message="Could not authenticate with Google Drive. Please check your credentials.", + failure_type=FailureType.config_error, + exception=e, + ) + + return self._drive_service + + def get_matching_files(self, globs: List[str], prefix: Optional[str], logger: logging.Logger) -> Iterable[RemoteFile]: + """ + Get all files matching the specified glob patterns. + """ + service = self.google_drive_service + root_folder_id = get_folder_id(self.config.folder_url) + # ignore prefix argument as it's legacy only and this is a new connector + prefixes = self.get_prefixes_from_globs(globs) + + folder_id_queue = [("", root_folder_id)] + seen: Set[str] = set() + while len(folder_id_queue) > 0: + (path, folder_id) = folder_id_queue.pop() + # fetch all files in this folder (1000 is the max page size) + # supportsAllDrives and includeItemsFromAllDrives are required to access files in shared drives + request = service.files().list( + q=f"'{folder_id}' in parents", + pageSize=1000, + fields="nextPageToken, files(id, name, modifiedTime, mimeType)", + supportsAllDrives=True, + includeItemsFromAllDrives=True, + ) + while True: + results = request.execute() + new_files = results.get("files", []) + for new_file in new_files: + # It's possible files and folders are linked up multiple times, this prevents us from getting stuck in a loop + if new_file["id"] in seen: + continue + seen.add(new_file["id"]) + file_name = path + new_file["name"] + if new_file["mimeType"] == FOLDER_MIME_TYPE: + folder_name = f"{file_name}/" + # check prefix matching in both directions to handle + prefix_matches_folder_name = any(prefix.startswith(folder_name) for prefix in prefixes) + folder_name_matches_prefix = any(folder_name.startswith(prefix) for prefix in prefixes) + if prefix_matches_folder_name or folder_name_matches_prefix or len(prefixes) == 0: + folder_id_queue.append((folder_name, new_file["id"])) + continue + else: + last_modified = datetime.strptime(new_file["modifiedTime"], "%Y-%m-%dT%H:%M:%S.%fZ") + original_mime_type = new_file["mimeType"] + mime_type = ( + self._get_export_mime_type(original_mime_type) + if self._is_exportable_document(original_mime_type) + else original_mime_type + ) + remote_file = GoogleDriveRemoteFile( + uri=file_name, + last_modified=last_modified, + id=new_file["id"], + original_mime_type=original_mime_type, + mime_type=mime_type, + ) + if self.file_matches_globs(remote_file, globs): + yield remote_file + request = service.files().list_next(request, results) + if request is None: + break + + def _is_exportable_document(self, mime_type: str): + """ + Returns true if the given file is a Google App document that can be exported. + """ + return mime_type in EXPORTABLE_DOCUMENTS_MIME_TYPES + + def open_file(self, file: GoogleDriveRemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger) -> IOBase: + if self._is_exportable_document(file.original_mime_type): + if mode == FileReadMode.READ: + raise ValueError( + "Google Docs/Drawings/Presentations can only be processed using the document file type format. Please set the format accordingly or adjust the glob pattern." + ) + request = self.google_drive_service.files().export_media(fileId=file.id, mimeType=file.mime_type) + else: + request = self.google_drive_service.files().get_media(fileId=file.id) + handle = io.BytesIO() + downloader = MediaIoBaseDownload(handle, request) + done = False + while done is False: + _, done = downloader.next_chunk() + + handle.seek(0) + + if mode == FileReadMode.READ_BINARY: + return handle + else: + # repack the bytes into a string with the right encoding + text_handle = io.StringIO(handle.read().decode(encoding or "utf-8")) + handle.close() + return text_handle + + def _get_export_mime_type(self, original_mime_type: str): + """ + Returns the mime type to export Google App documents as. + + Google Docs are exported as Docx to preserve as much formatting as possible, everything else goes through PDF. + """ + if original_mime_type.startswith(GOOGLE_DOC_MIME_TYPE): + return "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + else: + return "application/pdf" diff --git a/airbyte-integrations/connectors/source-google-drive/source_google_drive/utils.py b/airbyte-integrations/connectors/source-google-drive/source_google_drive/utils.py new file mode 100644 index 000000000000..c0994802358b --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/source_google_drive/utils.py @@ -0,0 +1,21 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from urllib.parse import urlparse + + +def get_folder_id(url_string: str) -> str: + """ + Extract the folder ID from a Google Drive folder URL. + + Takes the last path segment of the URL, which is the folder ID (ignoring trailing slashes and query parameters). + """ + try: + parsed_url = urlparse(url_string) + if parsed_url.scheme != "https" or parsed_url.netloc != "drive.google.com": + raise ValueError("Folder URL has to be of the form https://drive.google.com/drive/folders/") + path_segments = list(filter(None, parsed_url.path.split("/"))) + if path_segments[-2] != "folders" or len(path_segments) < 3: + raise ValueError("Folder URL has to be of the form https://drive.google.com/drive/folders/") + return path_segments[-1] + except Exception: + raise ValueError("Folder URL is invalid") diff --git a/airbyte-integrations/connectors/source-google-drive/unit_tests/test_reader.py b/airbyte-integrations/connectors/source-google-drive/unit_tests/test_reader.py new file mode 100644 index 000000000000..c3b095d3a5d1 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/unit_tests/test_reader.py @@ -0,0 +1,643 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import datetime +from unittest.mock import MagicMock, call, patch + +import pytest +from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig +from airbyte_cdk.sources.file_based.config.jsonl_format import JsonlFormat +from airbyte_cdk.sources.file_based.file_based_stream_reader import FileReadMode +from source_google_drive.spec import ServiceAccountCredentials, SourceGoogleDriveSpec +from source_google_drive.stream_reader import GoogleDriveRemoteFile, SourceGoogleDriveStreamReader + + +def create_reader( + config=SourceGoogleDriveSpec( + folder_url="https://drive.google.com/drive/folders/1Z2Q3", + streams=[FileBasedStreamConfig(name="test", format=JsonlFormat())], + credentials=ServiceAccountCredentials(auth_type="Service", service_account_info='{"test": "abc"}'), + ) +): + reader = SourceGoogleDriveStreamReader() + reader.config = config + + return reader + + +def flatten_list(list_of_lists): + return [item for sublist in list_of_lists for item in sublist] + + +@pytest.mark.parametrize( + "glob, listing_results, matched_files", + [ + pytest.param( + "*", + [[{"files": [{"id": "abc", "mimeType": "text/csv", "name": "test.csv", "modifiedTime": "2021-01-01T00:00:00.000Z"}]}]], + [ + GoogleDriveRemoteFile( + uri="test.csv", + id="abc", + mime_type="text/csv", + original_mime_type="text/csv", + last_modified=datetime.datetime(2021, 1, 1), + ) + ], + id="Single file", + ), + pytest.param( + "*", + [ + [ + { + "files": [ + {"id": "abc", "mimeType": "text/csv", "name": "test.csv", "modifiedTime": "2021-01-01T00:00:00.000Z"}, + {"id": "def", "mimeType": "text/csv", "name": "another_file.csv", "modifiedTime": "2021-01-01T00:00:00.000Z"}, + ] + }, + ] + ], + [ + GoogleDriveRemoteFile( + uri="test.csv", + id="abc", + mime_type="text/csv", + original_mime_type="text/csv", + last_modified=datetime.datetime(2021, 1, 1), + ), + GoogleDriveRemoteFile( + uri="another_file.csv", + id="def", + mime_type="text/csv", + original_mime_type="text/csv", + last_modified=datetime.datetime(2021, 1, 1), + ), + ], + id="Multiple files", + ), + pytest.param( + "*", + [ + [ + {"files": [{"id": "abc", "mimeType": "text/csv", "name": "test.csv", "modifiedTime": "2021-01-01T00:00:00.000Z"}]}, + { + "files": [ + {"id": "def", "mimeType": "text/csv", "name": "another_file.csv", "modifiedTime": "2021-01-01T00:00:00.000Z"} + ] + }, + ] + ], + [ + GoogleDriveRemoteFile( + uri="test.csv", + id="abc", + mime_type="text/csv", + original_mime_type="text/csv", + last_modified=datetime.datetime(2021, 1, 1), + ), + GoogleDriveRemoteFile( + uri="another_file.csv", + id="def", + mime_type="text/csv", + original_mime_type="text/csv", + last_modified=datetime.datetime(2021, 1, 1), + ), + ], + id="Multiple pages", + ), + pytest.param( + "*", + [ + [ + {"files": []}, + ] + ], + [], + id="No files", + ), + pytest.param( + "**/*", + [ + [ + { + "files": [ + {"id": "abc", "mimeType": "text/csv", "name": "test.csv", "modifiedTime": "2021-01-01T00:00:00.000Z"}, + { + "id": "sub", + "mimeType": "application/vnd.google-apps.folder", + "name": "subfolder", + "modifiedTime": "2021-01-01T00:00:00.000Z", + }, + ] + }, + ], + [ + # second request is for requesting the subfolder + { + "files": [ + {"id": "def", "mimeType": "text/csv", "name": "another_file.csv", "modifiedTime": "2021-01-01T00:00:00.000Z"}, + { + "id": "subsub", + "mimeType": "application/vnd.google-apps.folder", + "name": "subsubfolder", + "modifiedTime": "2021-01-01T00:00:00.000Z", + }, + ] + }, + ], + [ + # third request is for requesting the subsubfolder + { + "files": [ + { + "id": "ghi", + "mimeType": "text/csv", + "name": "yet_another_file.csv", + "modifiedTime": "2021-01-01T00:00:00.000Z", + }, + ] + }, + ], + ], + [ + GoogleDriveRemoteFile( + uri="test.csv", + id="abc", + mime_type="text/csv", + original_mime_type="text/csv", + last_modified=datetime.datetime(2021, 1, 1), + ), + GoogleDriveRemoteFile( + uri="subfolder/another_file.csv", + id="def", + mime_type="text/csv", + original_mime_type="text/csv", + last_modified=datetime.datetime(2021, 1, 1), + ), + GoogleDriveRemoteFile( + uri="subfolder/subsubfolder/yet_another_file.csv", + id="ghi", + mime_type="text/csv", + original_mime_type="text/csv", + last_modified=datetime.datetime(2021, 1, 1), + ), + ], + id="Nested directories", + ), + pytest.param( + "**/*", + [ + [ + { + "files": [ + {"id": "abc", "mimeType": "text/csv", "name": "test.csv", "modifiedTime": "2021-01-01T00:00:00.000Z"}, + { + "id": "sub", + "mimeType": "application/vnd.google-apps.folder", + "name": "subfolder", + "modifiedTime": "2021-01-01T00:00:00.000Z", + }, + ] + }, + ], + [ + # second request is for requesting the subfolder + { + "files": [ + {"id": "abc", "mimeType": "text/csv", "name": "test.csv", "modifiedTime": "2021-01-01T00:00:00.000Z"}, + { + "id": "subsub", + "mimeType": "application/vnd.google-apps.folder", + "name": "subsubfolder", + "modifiedTime": "2021-01-01T00:00:00.000Z", + }, + ] + }, + ], + [ + # third request is for requesting the subsubfolder + { + "files": [ + {"id": "abc", "mimeType": "text/csv", "name": "test.csv", "modifiedTime": "2021-01-01T00:00:00.000Z"}, + { + "id": "sub", + "mimeType": "application/vnd.google-apps.folder", + "name": "link_to_subfolder", + "modifiedTime": "2021-01-01T00:00:00.000Z", + }, + ] + }, + ], + ], + [ + GoogleDriveRemoteFile( + uri="test.csv", + id="abc", + mime_type="text/csv", + original_mime_type="text/csv", + last_modified=datetime.datetime(2021, 1, 1), + ), + ], + id="Duplicates", + ), + pytest.param( + "subfolder/**/*.csv", + [ + [ + { + "files": [ + {"id": "abc", "mimeType": "text/csv", "name": "test.csv", "modifiedTime": "2021-01-01T00:00:00.000Z"}, + { + "id": "sub", + "mimeType": "application/vnd.google-apps.folder", + "name": "subfolder", + "modifiedTime": "2021-01-01T00:00:00.000Z", + }, + ] + }, + ], + [ + # second request is for requesting the subfolder + { + "files": [ + {"id": "def", "mimeType": "text/csv", "name": "another_file.csv", "modifiedTime": "2021-01-01T00:00:00.000Z"}, + { + "id": "ghi", + "mimeType": "text/jsonl", + "name": "non_matching.jsonl", + "modifiedTime": "2021-01-01T00:00:00.000Z", + }, + ] + }, + ], + ], + [ + GoogleDriveRemoteFile( + uri="subfolder/another_file.csv", + id="def", + mime_type="text/csv", + original_mime_type="text/csv", + last_modified=datetime.datetime(2021, 1, 1), + ), + ], + id="Glob matching and subdirectories", + ), + pytest.param( + "subfolder/*.csv", + [ + [ + { + "files": [ + {"id": "abc", "mimeType": "text/csv", "name": "test.csv", "modifiedTime": "2021-01-01T00:00:00.000Z"}, + { + "id": "sub", + "mimeType": "application/vnd.google-apps.folder", + "name": "subfolder", + "modifiedTime": "2021-01-01T00:00:00.000Z", + }, + # This won't get queued because it has no chance of matching the glob + { + "id": "sub", + "mimeType": "application/vnd.google-apps.folder", + "name": "ignored_subfolder", + "modifiedTime": "2021-01-01T00:00:00.000Z", + }, + ] + }, + ], + [ + # second request is for requesting the subfolder + { + "files": [ + {"id": "def", "mimeType": "text/csv", "name": "another_file.csv", "modifiedTime": "2021-01-01T00:00:00.000Z"}, + # This will get queued because it matches the prefix (event though it can't match the glob) + { + "id": "subsub", + "mimeType": "application/vnd.google-apps.folder", + "name": "subsubfolder", + "modifiedTime": "2021-01-01T00:00:00.000Z", + }, + ] + }, + ], + [ + # third request is for requesting the subsubfolder + { + "files": [ + { + "id": "ghi", + "mimeType": "text/csv", + "name": "yet_another_file.csv", + "modifiedTime": "2021-01-01T00:00:00.000Z", + }, + ] + }, + ], + ], + [ + GoogleDriveRemoteFile( + uri="subfolder/another_file.csv", + id="def", + mime_type="text/csv", + original_mime_type="text/csv", + last_modified=datetime.datetime(2021, 1, 1), + ), + ], + id="Glob matching and ignoring most subdirectories that can't be matched", + ), + pytest.param( + "subfolder/subsubfolder/*.csv", + [ + [ + { + "files": [ + {"id": "abc", "mimeType": "text/csv", "name": "test.csv", "modifiedTime": "2021-01-01T00:00:00.000Z"}, + { + "id": "sub", + "mimeType": "application/vnd.google-apps.folder", + "name": "subfolder", + "modifiedTime": "2021-01-01T00:00:00.000Z", + }, + ] + }, + ], + [ + # second request is for requesting the subfolder + { + "files": [ + {"id": "def", "mimeType": "text/csv", "name": "another_file.csv", "modifiedTime": "2021-01-01T00:00:00.000Z"}, + # This will get queued because it matches the prefix (event though it can't match the glob) + { + "id": "subsub", + "mimeType": "application/vnd.google-apps.folder", + "name": "subsubfolder", + "modifiedTime": "2021-01-01T00:00:00.000Z", + }, + ] + }, + ], + [ + # third request is for requesting the subsubfolder + { + "files": [ + { + "id": "ghi", + "mimeType": "text/csv", + "name": "yet_another_file.csv", + "modifiedTime": "2021-01-01T00:00:00.000Z", + }, + # This will get queued because it matches the prefix (event though it can't match the glob) + { + "id": "subsubsub", + "mimeType": "application/vnd.google-apps.folder", + "name": "ignored_subsubsubfolder", + "modifiedTime": "2021-01-01T00:00:00.000Z", + }, + ] + }, + ], + [{"files": []}], + ], + [ + GoogleDriveRemoteFile( + uri="subfolder/subsubfolder/yet_another_file.csv", + id="ghi", + mime_type="text/csv", + original_mime_type="text/csv", + last_modified=datetime.datetime(2021, 1, 1), + ), + ], + id="Glob matching and ignoring subdirectories that can't be matched, multiple levels", + ), + pytest.param( + "*", + [ + [ + { + "files": [ + { + "id": "abc", + "mimeType": "application/vnd.google-apps.document", + "name": "MyDoc", + "modifiedTime": "2021-01-01T00:00:00.000Z", + } + ] + } + ] + ], + [ + GoogleDriveRemoteFile( + uri="MyDoc", + id="abc", + original_mime_type="application/vnd.google-apps.document", + mime_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document", + last_modified=datetime.datetime(2021, 1, 1), + ) + ], + id="Google Doc as docx", + ), + pytest.param( + "*", + [ + [ + { + "files": [ + { + "id": "abc", + "mimeType": "application/vnd.google-apps.presentation", + "name": "MySlides", + "modifiedTime": "2021-01-01T00:00:00.000Z", + } + ] + } + ] + ], + [ + GoogleDriveRemoteFile( + uri="MySlides", + id="abc", + original_mime_type="application/vnd.google-apps.presentation", + mime_type="application/pdf", + last_modified=datetime.datetime(2021, 1, 1), + ) + ], + id="Presentation as pdf", + ), + pytest.param( + "*", + [ + [ + { + "files": [ + { + "id": "abc", + "mimeType": "application/vnd.google-apps.drawing", + "name": "MyDrawing", + "modifiedTime": "2021-01-01T00:00:00.000Z", + } + ] + } + ] + ], + [ + GoogleDriveRemoteFile( + uri="MyDrawing", + id="abc", + original_mime_type="application/vnd.google-apps.drawing", + mime_type="application/pdf", + last_modified=datetime.datetime(2021, 1, 1), + ) + ], + id="Drawing as pdf", + ), + pytest.param( + "*", + [ + [ + { + "files": [ + { + "id": "abc", + "mimeType": "application/vnd.google-apps.video", + "name": "MyVideo", + "modifiedTime": "2021-01-01T00:00:00.000Z", + } + ] + } + ] + ], + [ + GoogleDriveRemoteFile( + uri="MyVideo", + id="abc", + original_mime_type="application/vnd.google-apps.video", + mime_type="application/vnd.google-apps.video", + last_modified=datetime.datetime(2021, 1, 1), + ) + ], + id="Other google file types as is", + ), + ], +) +@patch("source_google_drive.stream_reader.service_account") +@patch("source_google_drive.stream_reader.build") +def test_matching_files(mock_build_service, mock_service_account, glob, listing_results, matched_files): + mock_request = MagicMock() + # execute returns all results from all pages for all listings + flattened_results = flatten_list(listing_results) + + mock_request.execute.side_effect = flattened_results + files_service = MagicMock() + files_service.list.return_value = mock_request + # list next returns a new fake "request" for each page and None at the end of each page (simulating the end of the listing like the Google Drive API behaves in practice) + files_service.list_next.side_effect = flatten_list( + [[*[mock_request for _ in range(len(listing) - 1)], None] for listing in listing_results] + ) + drive_service = MagicMock() + drive_service.files.return_value = files_service + mock_build_service.return_value = drive_service + + reader = create_reader() + + found_files = list(reader.get_matching_files([glob], None, MagicMock())) + assert files_service.list.call_count == len(listing_results) + assert matched_files == found_files + assert files_service.list_next.call_count == len(flattened_results) + + +@pytest.mark.parametrize( + "file, file_content, mode, expect_export, expected_mime_type, expected_read, expect_raise", + [ + pytest.param( + GoogleDriveRemoteFile( + uri="avro_file", id="abc", mime_type="text/csv", original_mime_type="text/csv", last_modified=datetime.datetime(2021, 1, 1) + ), + b"test", + FileReadMode.READ_BINARY, + False, + None, + b"test", + False, + id="Read binary file", + ), + pytest.param( + GoogleDriveRemoteFile( + uri="test.csv", id="abc", mime_type="text/csv", original_mime_type="text/csv", last_modified=datetime.datetime(2021, 1, 1) + ), + b"test", + FileReadMode.READ, + False, + None, + "test", + False, + id="Read text file", + ), + pytest.param( + GoogleDriveRemoteFile( + uri="abc", + id="abc", + mime_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document", + original_mime_type="application/vnd.google-apps.document", + last_modified=datetime.datetime(2021, 1, 1), + ), + b"test", + FileReadMode.READ_BINARY, + True, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + b"test", + False, + id="Read google doc as binary file with export", + ), + ], +) +@patch("source_google_drive.stream_reader.MediaIoBaseDownload") +@patch("source_google_drive.stream_reader.service_account") +@patch("source_google_drive.stream_reader.build") +def test_open_file( + mock_build_service, + mock_service_account, + mock_basedownload, + file, + file_content, + mode, + expect_export, + expected_mime_type, + expected_read, + expect_raise, +): + mock_request = MagicMock() + mock_downloader = MagicMock() + + def mock_next_chunk(): + handle = mock_basedownload.call_args[0][0] + if handle.tell() > 0: + return (None, True) + else: + handle.write(file_content) + return (None, False) + + mock_downloader.next_chunk.side_effect = mock_next_chunk + + mock_basedownload.return_value = mock_downloader + + files_service = MagicMock() + if expect_export: + files_service.export_media.return_value = mock_request + else: + files_service.get_media.return_value = mock_request + drive_service = MagicMock() + drive_service.files.return_value = files_service + mock_build_service.return_value = drive_service + + if expect_raise: + with pytest.raises(ValueError): + create_reader().open_file(file, mode, None, MagicMock()).read() + else: + assert expected_read == create_reader().open_file(file, mode, None, MagicMock()).read() + assert mock_downloader.next_chunk.call_count == 2 + if expect_export: + files_service.export_media.assert_has_calls([call(fileId=file.id, mimeType=expected_mime_type)]) + else: + files_service.get_media.assert_has_calls([call(fileId=file.id)]) diff --git a/airbyte-integrations/connectors/source-google-drive/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-google-drive/unit_tests/test_utils.py new file mode 100644 index 000000000000..8dcb7e52e223 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-drive/unit_tests/test_utils.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + + +import pytest +from source_google_drive.utils import get_folder_id + + +@pytest.mark.parametrize( + "input, output, raises", + [ + ("https://drive.google.com/drive/folders/1q2w3e4r5t6y7u8i9o0p", "1q2w3e4r5t6y7u8i9o0p", False), + ("https://drive.google.com/drive/folders/1q2w3e4r5t6y7u8i9o0p/", "1q2w3e4r5t6y7u8i9o0p", False), + ("https://drive.google.com/drive/folders/1q2w3e4r5t6y7u8i9o0p?usp=link_sharing", "1q2w3e4r5t6y7u8i9o0p", False), + ("https://drive.google.com/drive/u/0/folders/1q2w3e4r5t6y7u8i9o0p/", "1q2w3e4r5t6y7u8i9o0p", False), + ("https://drive.google.com/drive/u/0/folders/1q2w3e4r5t6y7u8i9o0p?usp=link_sharing", "1q2w3e4r5t6y7u8i9o0p", False), + ("https://drive.google.com/drive/u/0/folders/1q2w3e4r5t6y7u8i9o0p#abc", "1q2w3e4r5t6y7u8i9o0p", False), + ("https://docs.google.com/document/d/fsgfjdsh", None, True), + ("https://drive.google.com/drive/my-drive", None, True), + ("http://drive.google.com/drive/u/0/folders/1q2w3e4r5t6y7u8i9o0p/", None, True), + ("https://drive.google.com/", None, True), + ] +) +def test_get_folder_id(input, output, raises): + if raises: + with pytest.raises(ValueError): + get_folder_id(input) + else: + assert get_folder_id(input) == output \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-google-pagespeed-insights/README.md b/airbyte-integrations/connectors/source-google-pagespeed-insights/README.md index e02be14d79f2..ffb79152d10e 100644 --- a/airbyte-integrations/connectors/source-google-pagespeed-insights/README.md +++ b/airbyte-integrations/connectors/source-google-pagespeed-insights/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-google-pagespeed-insights:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/google-pagespeed-insights) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_pagespeed_insights/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-google-pagespeed-insights:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-google-pagespeed-insights build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-google-pagespeed-insights:airbyteDocker +An image will be built with the tag `airbyte/source-google-pagespeed-insights:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-google-pagespeed-insights:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-pagespeed-insig docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-pagespeed-insights:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-google-pagespeed-insights:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-google-pagespeed-insights test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-google-pagespeed-insights:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-google-pagespeed-insights:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-google-pagespeed-insights test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/google-pagespeed-insights.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-google-pagespeed-insights/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-google-pagespeed-insights/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-google-pagespeed-insights/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-google-pagespeed-insights/build.gradle b/airbyte-integrations/connectors/source-google-pagespeed-insights/build.gradle deleted file mode 100644 index 51d3f622d9a3..000000000000 --- a/airbyte-integrations/connectors/source-google-pagespeed-insights/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_google_pagespeed_insights' -} diff --git a/airbyte-integrations/connectors/source-google-search-console/Dockerfile b/airbyte-integrations/connectors/source-google-search-console/Dockerfile deleted file mode 100755 index b77accca79fe..000000000000 --- a/airbyte-integrations/connectors/source-google-search-console/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY source_google_search_console ./source_google_search_console -COPY main.py ./ -COPY setup.py ./ -RUN pip install . - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=1.3.2 -LABEL io.airbyte.name=airbyte/source-google-search-console diff --git a/airbyte-integrations/connectors/source-google-search-console/README.md b/airbyte-integrations/connectors/source-google-search-console/README.md index 88a1e80b9119..3fc8aacd2bd3 100755 --- a/airbyte-integrations/connectors/source-google-search-console/README.md +++ b/airbyte-integrations/connectors/source-google-search-console/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-google-search-console:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/google-search-console) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_search_console/spec.json` file. @@ -54,19 +48,70 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-google-search-console:dev + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-google-search-console build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-google-search-console:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations -You can also build the connector image via Gradle: +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-google-search-console:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-google-search-console:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-google-search-console:dev . +# Running the spec command against your patched connector +docker run airbyte/source-google-search-console:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -75,44 +120,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-search-console: docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-search-console:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-google-search-console:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .'[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-google-search-console test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-google-search-console:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-google-search-console:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -122,8 +139,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -2. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -3. Create a Pull Request. -4. Pat yourself on the back for being an awesome contributor. -5. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-google-search-console test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/google-search-console.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-google-search-console/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-search-console/acceptance-test-config.yml index de661af6035d..09fb7528c86b 100755 --- a/airbyte-integrations/connectors/source-google-search-console/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-search-console/acceptance-test-config.yml @@ -6,6 +6,10 @@ acceptance_tests: spec: tests: - spec_path: "source_google_search_console/spec.json" + backward_compatibility_tests_config: + # changed the structure of `custom_reports` + # from `json string` to `list[reports]` + disable_for_version: "1.3.2" connection: tests: - config_path: "secrets/config.json" @@ -154,3 +158,6 @@ acceptance_tests: timeout_seconds: 3600 future_state: future_state_path: "integration_tests/abnormal_state.json" + # Incremental read with current config produces multiple empty state messages before emitting first record. + # This leads to identical consecutive sync results which fail the test + skip_comprehensive_incremental_tests: true diff --git a/airbyte-integrations/connectors/source-google-search-console/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-google-search-console/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-google-search-console/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-google-search-console/build.gradle b/airbyte-integrations/connectors/source-google-search-console/build.gradle deleted file mode 100755 index 569a0ebd89b6..000000000000 --- a/airbyte-integrations/connectors/source-google-search-console/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_google_search_console' -} diff --git a/airbyte-integrations/connectors/source-google-search-console/main.py b/airbyte-integrations/connectors/source-google-search-console/main.py index 656bc5fa8a61..117df652ca76 100755 --- a/airbyte-integrations/connectors/source-google-search-console/main.py +++ b/airbyte-integrations/connectors/source-google-search-console/main.py @@ -7,7 +7,11 @@ from airbyte_cdk.entrypoint import launch from source_google_search_console import SourceGoogleSearchConsole +from source_google_search_console.config_migrations import MigrateCustomReports if __name__ == "__main__": source = SourceGoogleSearchConsole() + # migrate config at runtime + MigrateCustomReports.migrate(sys.argv[1:], source) + # run the connector launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-google-search-console/metadata.yaml b/airbyte-integrations/connectors/source-google-search-console/metadata.yaml index be4242be9cbf..e3d321483fd5 100644 --- a/airbyte-integrations/connectors/source-google-search-console/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-search-console/metadata.yaml @@ -1,12 +1,18 @@ data: + ab_internal: + ql: 400 + sl: 300 allowedHosts: hosts: - "*.googleapis.com" + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: api connectorType: source definitionId: eb4c9e00-db83-4d63-a386-39cfa91012a8 - dockerImageTag: 1.3.2 + dockerImageTag: 1.3.6 dockerRepository: airbyte/source-google-search-console + documentationUrl: https://docs.airbyte.com/integrations/sources/google-search-console githubIssueLabel: source-google-search-console icon: googlesearchconsole.svg license: Elv2 @@ -26,11 +32,7 @@ data: - sitemaps - sites - search_analytics_all_fields - documentationUrl: https://docs.airbyte.com/integrations/sources/google-search-console + supportLevel: certified tags: - language:python - ab_internal: - sl: 300 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/config_migrations.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/config_migrations.py new file mode 100644 index 000000000000..2774f64703c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/config_migrations.py @@ -0,0 +1,97 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import logging +from typing import Any, List, Mapping + +from airbyte_cdk.config_observation import create_connector_config_control_message +from airbyte_cdk.entrypoint import AirbyteEntrypoint +from airbyte_cdk.sources import Source +from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + +logger = logging.getLogger("airbyte_logger") + + +class MigrateCustomReports: + """ + This class stands for migrating the config at runtime, + while providing the backward compatibility when falling back to the previous source version. + + Specifically, starting from `1.3.3`, the `custom_reports` property should be like : + > List([{name: my_report}, {dimensions: [a,b,c]}], [], ...) + instead of, in `1.3.2`: + > JSON STR: "{name: my_report}, {dimensions: [a,b,c]}" + """ + + message_repository: MessageRepository = InMemoryMessageRepository() + migrate_from_key: str = "custom_reports" + migrate_to_key: str = "custom_reports_array" + + @classmethod + def should_migrate(cls, config: Mapping[str, Any]) -> bool: + """ + This method determines whether the config should be migrated to have the new structure for the `custom_reports`, + based on the source spec. + Returns: + > True, if the transformation is necessary + > False, otherwise. + > Raises the Exception if the structure could not be migrated. + """ + # If the config was already migrated, there is no need to do this again. + # but if the customer has already switched to the new version, + # corrected the old config and switches back to the new version, + # we should try to migrate the modified old custom reports. + if cls.migrate_to_key in config: + # also we need to make sure this is not a newly created connection + return len(config[cls.migrate_to_key]) == 0 and cls.migrate_from_key in config + + if cls.migrate_from_key in config: + custom_reports = config[cls.migrate_from_key] + # check the old structure vs new spec + if isinstance(custom_reports, str): + return True + return False + + @classmethod + def transform_to_array(cls, config: Mapping[str, Any], source: Source = None) -> Mapping[str, Any]: + # assign old values to new property that will be used within the new version + config[cls.migrate_to_key] = config[cls.migrate_from_key] + # transfom `json_str` to `list` of objects + return source._validate_custom_reports(config) + + @classmethod + def modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, Any]) -> Mapping[str, Any]: + # modify the config + migrated_config = cls.transform_to_array(config, source) + # save the config + source.write_config(migrated_config, config_path) + # return modified config + return migrated_config + + @classmethod + def emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: + # add the Airbyte Control Message to message repo + cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) + # emit the Airbyte Control Message from message queue to stdout + for message in cls.message_repository._message_queue: + print(message.json(exclude_unset=True)) + + @classmethod + def migrate(cls, args: List[str], source: Source) -> None: + """ + This method checks the input args, should the config be migrated, + transform if neccessary and emit the CONTROL message. + """ + # get config path + config_path = AirbyteEntrypoint(source).extract_config(args) + # proceed only if `--config` arg is provided + if config_path: + # read the existing config + config = source.read_config(config_path) + # migration check + if cls.should_migrate(config): + cls.emit_control_message( + cls.modify_and_save(config_path, source, config), + ) diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/exceptions.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/exceptions.py index 77dc53148762..04cc13a6adf8 100644 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/exceptions.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/exceptions.py @@ -14,13 +14,15 @@ def __init__(self, invalid_site_url: Union[Set, List]) -> None: class UnauthorizedOauthError(Exception): def __init__(self): - message = "Unable to connect with privided OAuth credentials. The `access token` or `refresh token` is expired. Please re-authrenticate using valid account credenials." + message = "Unable to connect with provided OAuth credentials. The `access token` or `refresh token` is expired. Please re-authrenticate using valid account credenials." super().__init__(message) class UnauthorizedServiceAccountError(Exception): def __init__(self): - message = "Unable to connect with privided Service Account credentials. Make sure the `sevice account crdentials` povided is valid." + message = ( + "Unable to connect with provided Service Account credentials. Make sure the `sevice account credentials` provided are valid." + ) super().__init__(message) diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/service_account_authenticator.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/service_account_authenticator.py index decbb448107f..15759c39bec7 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/service_account_authenticator.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/service_account_authenticator.py @@ -14,14 +14,18 @@ class ServiceAccountAuthenticator(AuthBase): def __init__(self, service_account_info: str, email: str, scopes=None): self.scopes = scopes or DEFAULT_SCOPES - self.credentials: Credentials = Credentials.from_service_account_info(service_account_info, scopes=self.scopes).with_subject(email) + self.service_account_info = service_account_info + self.email = email def __call__(self, request: requests.PreparedRequest) -> requests.PreparedRequest: try: - if not self.credentials.valid: + credentials: Credentials = Credentials.from_service_account_info(self.service_account_info, scopes=self.scopes).with_subject( + self.email + ) + if not credentials.valid: # We pass a dummy request because the refresh iface requires it - self.credentials.refresh(Request()) - self.credentials.apply(request.headers) + credentials.refresh(Request()) + credentials.apply(request.headers) return request except Exception: raise UnauthorizedServiceAccountError diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py index 5fbc3b7f9a77..48f59c11d9d5 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/source.py @@ -10,10 +10,11 @@ import pendulum import requests from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import SyncMode +from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator +from airbyte_cdk.utils import AirbyteTracedException from source_google_search_console.exceptions import ( InvalidSiteURLValidationError, UnauthorizedOauthError, @@ -67,19 +68,11 @@ def _validate_and_transform(self, config: Mapping[str, Any]): try: authorization["service_account_info"] = json.loads(authorization["service_account_info"]) except ValueError: - raise Exception("authorization.service_account_info is not valid JSON") + message = "authorization.service_account_info is not valid JSON" + raise AirbyteTracedException(message=message, internal_message=message, failure_type=FailureType.config_error) # custom report validation - if "custom_reports" in config: - try: - config["custom_reports"] = json.loads(config["custom_reports"]) - except ValueError: - raise Exception("custom_reports is not valid JSON") - jsonschema.validate(config["custom_reports"], custom_reports_schema) - for report in config["custom_reports"]: - for dimension in report["dimensions"]: - if dimension not in SearchAnalyticsByCustomDimensions.dimension_to_property_schema_map: - raise Exception(f"dimension: '{dimension}' not found") + config = self._validate_custom_reports(config) # start date checks pendulum.parse(config.get("start_date", "2021-01-01")) # `2021-01-01` is the default value @@ -95,6 +88,26 @@ def _validate_and_transform(self, config: Mapping[str, Any]): config["data_state"] = config.get("data_state", "final") return config + def _validate_custom_reports(self, config: Mapping[str, Any]) -> Mapping[str, Any]: + if "custom_reports_array" in config: + try: + custom_reports = config["custom_reports_array"] + if isinstance(custom_reports, str): + # load the json_str old report structure and transform it into valid JSON Object + config["custom_reports_array"] = json.loads(config["custom_reports_array"]) + elif isinstance(custom_reports, list): + pass # allow the list structure only + except ValueError: + message = "Custom Reports provided is not valid List of Object (reports)" + raise AirbyteTracedException(message=message, internal_message=message, failure_type=FailureType.config_error) + jsonschema.validate(config["custom_reports_array"], custom_reports_schema) + for report in config["custom_reports_array"]: + for dimension in report["dimensions"]: + if dimension not in SearchAnalyticsByCustomDimensions.DIMENSION_TO_PROPERTY_SCHEMA_MAP: + message = f"dimension: '{dimension}' not found" + raise AirbyteTracedException(message=message, internal_message=message, failure_type=FailureType.config_error) + return config + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: try: config = self._validate_and_transform(config) @@ -181,7 +194,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: def get_custom_reports(self, config: Mapping[str, Any], stream_config: Mapping[str, Any]) -> List[Optional[Stream]]: return [ type(report["name"], (SearchAnalyticsByCustomDimensions,), {})(dimensions=report["dimensions"], **stream_config) - for report in config.get("custom_reports", []) + for report in config.get("custom_reports_array", []) ] def get_stream_kwargs(self, config: Mapping[str, Any]) -> Mapping[str, Any]: diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/spec.json b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/spec.json index ff26efd0e224..8fa0e72ca5f5 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/spec.json +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/spec.json @@ -23,6 +23,7 @@ "default": "2021-01-01", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", "pattern_descriptor": "YYYY-MM-DD", + "always_show": true, "order": 1, "format": "date" }, @@ -115,7 +116,38 @@ "order": 4, "type": "string", "title": "Custom Reports", - "description": "A JSON array describing the custom reports you want to sync from Google Search Console. See our documentation for more information on formulating custom reports." + "airbyte_hidden": true, + "description": "(DEPRCATED) A JSON array describing the custom reports you want to sync from Google Search Console. See our documentation for more information on formulating custom reports." + }, + "custom_reports_array": { + "title": "Custom Reports", + "description": "You can add your Custom Analytics report by creating one.", + "order": 5, + "type": "array", + "items": { + "title": "Custom Report Config", + "type": "object", + "properties": { + "name": { + "title": "Name", + "description": "The name of the custom report, this name would be used as stream name", + "type": "string" + }, + "dimensions": { + "title": "Dimensions", + "description": "A list of available dimensions. Please note, that for technical reasons `date` is the default dimension which will be included in your query whether you specify it or not. Primary key will consist of your custom dimensions and the default dimension along with `site_url` and `search_type`.", + "type": "array", + "items": { + "title": "ValidEnums", + "description": "An enumeration of dimensions.", + "enum": ["country", "date", "device", "page", "query"] + }, + "default": ["date"], + "minItems": 0 + } + }, + "required": ["name", "dimensions"] + } }, "data_state": { "type": "string", @@ -124,7 +156,7 @@ "description": "If set to 'final', the returned data will include only finalized, stable data. If set to 'all', fresh data will be included. When using Incremental sync mode, we do not recommend setting this parameter to 'all' as it may cause data loss. More information can be found in our full documentation.", "examples": ["final", "all"], "default": "final", - "order": 5 + "order": 6 } } }, diff --git a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py index f707e527f8ff..461f55177c59 100755 --- a/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py +++ b/airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py @@ -25,7 +25,6 @@ class QueryAggregationType(Enum): class GoogleSearchConsole(HttpStream, ABC): url_base = BASE_URL - primary_key = None data_field = "" raise_on_http_errors = True @@ -74,6 +73,11 @@ def should_retry(self, response: requests.Response) -> bool: self.logger.error(f"Stream {self.name}. {error.get('message')}. Skipping.") setattr(self, "raise_on_http_errors", False) return False + # handle the `HTTP-400` - Bad query params with `aggregationType` + if error.get("code", 0) == 400: + self.logger.error(f"Stream `{self.name}`. {error.get('message')}. Trying with `aggregationType = auto` instead.") + self.aggregation_type = QueryAggregationType.auto + setattr(self, "raise_on_http_errors", False) return super().should_retry(response) @@ -82,6 +86,8 @@ class Sites(GoogleSearchConsole): API docs: https://developers.google.com/webmaster-tools/search-console-api-original/v3/sites """ + primary_key = None + def path( self, stream_state: Mapping[str, Any] = None, @@ -96,6 +102,7 @@ class Sitemaps(GoogleSearchConsole): API docs: https://developers.google.com/webmaster-tools/search-console-api-original/v3/sitemaps """ + primary_key = None data_field = "sitemap" def path( @@ -212,6 +219,7 @@ def request_body_json( "rowLimit": ROW_LIMIT, "dataState": stream_slice.get("data_state"), } + return data def _get_end_date(self) -> pendulum.date: @@ -300,30 +308,36 @@ def get_updated_state( class SearchAnalyticsByDate(SearchAnalytics): + primary_key = ["site_url", "date", "search_type"] search_types = ["web", "news", "image", "video", "discover", "googleNews"] dimensions = ["date"] class SearchAnalyticsByCountry(SearchAnalytics): + primary_key = ["site_url", "date", "country", "search_type"] search_types = ["web", "news", "image", "video", "discover", "googleNews"] dimensions = ["date", "country"] class SearchAnalyticsByDevice(SearchAnalytics): + primary_key = ["site_url", "date", "device", "search_type"] search_types = ["web", "news", "image", "video", "googleNews"] dimensions = ["date", "device"] class SearchAnalyticsByPage(SearchAnalytics): + primary_key = ["site_url", "date", "page", "search_type"] search_types = ["web", "news", "image", "video", "discover", "googleNews"] dimensions = ["date", "page"] class SearchAnalyticsByQuery(SearchAnalytics): + primary_key = ["site_url", "date", "query", "search_type"] dimensions = ["date", "query"] class SearchAnalyticsAllFields(SearchAnalytics): + primary_key = ["site_url", "date", "country", "device", "query", "page", "search_type"] dimensions = ["date", "country", "device", "page", "query"] @@ -334,6 +348,7 @@ class SearchAppearance(SearchAnalytics): https://developers.google.com/webmaster-tools/v1/how-tos/all-your-data#search-appearance-data """ + primary_key = None dimensions = ["searchAppearance"] @@ -354,7 +369,6 @@ def request_body_json( stream = SearchAppearance(self.authenticator, self._site_urls, self._start_date, self._end_date) keywords_records = stream.read_records(sync_mode=SyncMode.full_refresh, stream_state=stream_state, stream_slice=stream_slice) keywords = {record["searchAppearance"] for record in keywords_records} - filters = [] for keyword in keywords: filters.append({"dimension": "searchAppearance", "operator": "equals", "expression": keyword}) @@ -365,76 +379,89 @@ def request_body_json( class SearchAnalyticsKeywordPageReport(SearchByKeyword): + primary_key = ["site_url", "date", "country", "device", "query", "page", "search_type"] dimensions = ["date", "country", "device", "query", "page"] class SearchAnalyticsKeywordSiteReportByPage(SearchByKeyword): + primary_key = ["site_url", "date", "country", "device", "query", "search_type"] dimensions = ["date", "country", "device", "query"] aggregation_type = QueryAggregationType.by_page class SearchAnalyticsKeywordSiteReportBySite(SearchByKeyword): + primary_key = ["site_url", "date", "country", "device", "query", "search_type"] dimensions = ["date", "country", "device", "query"] aggregation_type = QueryAggregationType.by_property class SearchAnalyticsSiteReportBySite(SearchAnalytics): + primary_key = ["site_url", "date", "country", "device", "search_type"] dimensions = ["date", "country", "device"] aggregation_type = QueryAggregationType.by_property class SearchAnalyticsSiteReportByPage(SearchAnalytics): + primary_key = ["site_url", "date", "country", "device", "search_type"] search_types = ["web", "news", "image", "video", "googleNews"] dimensions = ["date", "country", "device"] aggregation_type = QueryAggregationType.by_page class SearchAnalyticsPageReport(SearchAnalytics): + primary_key = ["site_url", "date", "country", "device", "search_type", "page"] search_types = ["web", "news", "image", "video", "googleNews"] dimensions = ["date", "country", "device", "page"] class SearchAnalyticsByCustomDimensions(SearchAnalytics): - dimension_to_property_schema_map = { + # `date` is a cursor field therefore should be mandatory + DEFAULT_DIMENSIONS = ["date"] + DIMENSION_TO_PROPERTY_SCHEMA_MAP = { "country": [{"country": {"type": ["null", "string"]}}], - "date": [], + "date": [{"date": {"type": ["null", "string"], "format": "date"}}], "device": [{"device": {"type": ["null", "string"]}}], "page": [{"page": {"type": ["null", "string"]}}], "query": [{"query": {"type": ["null", "string"]}}], } + primary_key = None + def __init__(self, dimensions: List[str], *args, **kwargs): super(SearchAnalyticsByCustomDimensions, self).__init__(*args, **kwargs) - self.dimensions = dimensions + self.dimensions = dimensions + [dimension for dimension in self.DEFAULT_DIMENSIONS if dimension not in dimensions] + # Assign the dimensions as PK for the custom report stream. + # Site URL and Search Type are included in the API call thus affect the resulting data. + # `site_url` is a required URL param for making API calls; + # `search_type` remains a query param for historical reasons, we do not want to remove it to not break existing connections. + self.primary_key = self.dimensions + ["site_url", "search_type"] def get_json_schema(self) -> Mapping[str, Any]: - try: - return super(SearchAnalyticsByCustomDimensions, self).get_json_schema() - except FileNotFoundError: - schema: Mapping[str, Any] = { - "$schema": "https://json-schema.org/draft-07/schema#", - "type": ["null", "object"], - "additionalProperties": True, - "properties": { - "clicks": {"type": ["null", "integer"]}, - "ctr": {"type": ["null", "number"], "multipleOf": 1e-25}, - "date": {"type": ["null", "string"], "format": "date"}, - "impressions": {"type": ["null", "integer"]}, - "position": {"type": ["null", "number"], "multipleOf": 1e-25}, - "search_type": {"type": ["null", "string"]}, - "site_url": {"type": ["null", "string"]}, - }, - } - - dimension_properties = self.dimension_to_property_schema() - schema["properties"].update(dimension_properties) - - return schema + schema: Mapping[str, Any] = { + "$schema": "https://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "additionalProperties": True, + "properties": { + # metrics + "clicks": {"type": ["null", "integer"]}, + "ctr": {"type": ["null", "number"], "multipleOf": 1e-25}, + "impressions": {"type": ["null", "integer"]}, + "position": {"type": ["null", "number"], "multipleOf": 1e-25}, + # default fields + "search_type": {"type": ["null", "string"]}, + "site_url": {"type": ["null", "string"]}, + }, + } + + # dimensions + dimension_properties = self.dimension_to_property_schema() + schema["properties"].update(dimension_properties) + return schema def dimension_to_property_schema(self) -> dict: properties = {} for dimension in sorted(self.dimensions): - fields = self.dimension_to_property_schema_map[dimension] + fields = self.DIMENSION_TO_PROPERTY_SCHEMA_MAP[dimension] for field in fields: properties = {**properties, **field} return properties diff --git a/airbyte-integrations/connectors/source-google-search-console/unit_tests/conftest.py b/airbyte-integrations/connectors/source-google-search-console/unit_tests/conftest.py index 3c201316fcaf..7245db70436e 100644 --- a/airbyte-integrations/connectors/source-google-search-console/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-google-search-console/unit_tests/conftest.py @@ -20,6 +20,7 @@ def config_fixture(requests_mock): "refresh_token": "refresh_token", }, "custom_reports": '[{"name": "custom_dimensions", "dimensions": ["date", "country", "device"]}]', + "custom_reports_array": [{"name": "custom_dimensions", "dimensions": ["date", "country", "device"]}], } @@ -31,15 +32,16 @@ def config_service_account_fixture(requests_mock): "end_date": "2022-02-01", "authorization": { "auth_type": "Service", - "service_account_info": "{\n \"type\": \"service_account\",\n \"project_id\": \"test\",\n \"private_key_id\": \"123\",\n \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nMIIEvQIBADANBgkqhkiG9w0BBQEFAASCBKcwggSjAgEAAoIBAQDCJPUvep0vXeWb\\nqiwnDxWdd8D75FWJBaYB3rjZvBBhZiY3sA7DmEOj+NJHl4PiPzP8tDZl9MyLBWEc\\neTFSmHSBYSqxax9AOLzWXfLzUezjediIRsGC/Eq9Ue0rkDdMcdcfzQ5J9RDDI1DF\\n1UBxVHFOf7DOSOU7meNPFjAO68aITErvnTh/XL1wWC28PYL351hs57WwLSQTuW0e\\ncUw9XUOE977+qJ4Cs3ZM5c10eid5DDWS4heFG/9hEkobXy34BNdeDodfe9xGSJxD\\nFoAhADj6jMn1z7YgsUG7zpsyW8yh2LtnYdT+fMqIl0FeB4dt0kB3uU1f6vqgo97p\\ndibK6DQ3AgMBAAECggEADWZPz+eZreyNDbnokzrDMw7SMIof1AKIsZKMPDPAE1Pu\\ndlbkWS0LGhs92p6DOUKqFWdHMkrh/tEvuy94L1G1CAr+gqe4mY4KjPPuC7I1wRuM\\n50ovWtlliGL9SIDxkbw+IB4SJIBrS3SgCg+AA6WgezQ5lHtLUXPh6ivHXfhGLlKR\\nI+Gow93UklbxcT57ezeDZVn0U3iUG1H7NkE0livyTTGEMm6GxUqxje7axA4ZVfRL\\nRVrNAHQTihPTThmN/p47Wbh6C8m7o1/cutYDk52CuCjuifxNINlak1ZimSEJ7mcY\\nSIglnTmndQImwiyeDbITtJ3gyYiJerjHnMAYH+VInQKBgQD5HH1tKBxZouozdweu\\n6lpTyko+TBa/3Eo2pgFxbJrKe3pBhkNWVLrCukZxWDFkKSbC+5xaSNGnh/lP/6FX\\nWHWBuBL8R5os9bfNQ9xnArZX7OhzN+aIh8aK5gmEPJE1JaepPyC0X8vaTBqFiQlK\\n6aRB89RqOUlB86B9vzJca7p7LQKBgQDHg1h9A6X9EkWewW5cSOuScw4FElK8N62v\\n5oVByBZZb/Ys9zP04m0yG7VdRSjk8xyCH5+GDS5m9jTxJdctON2AOPL7de8KOtga\\nJSHivUdDLkt7wSmvblc/JYnNs5+B783gTOpdBrXhV6Wo+QpVw1Pcx15b10WLAs8l\\nMzk7LG27cwKBgDJPorVNCIzB7nL+czrMcfnCPURfsaiGISbwWBJEUO7cCVD6gNcK\\nvb1eSaPSoAcOmJmAn49MbatcNuoFQtyVLQZJ2uvAuk6iQcdfF8BmN9WCL2A1xgWF\\nBoA+/WULpngJZtczvLMxNcac4C5gAtRyY44+ZIQflcAQKDW9S7qGt17xAoGBAJ37\\npLtBg1PU/yoJ81DCMT/DOYvMiZUe5bsO5+BCB2iE3sOWcB7umRb/l+qmVA6Pb7ie\\nP9yPXXoMZbm6hBv8FnFtJwL1zPYlyG9TjfSUevR4mS8CsvaGhjGvkOJA5QKoGDcP\\n0Nke8jDhDX2yzntA84w0lsRUv22nKM5FNIFl2fJ/AoGAOAVtlKRPPi2YrjUqqy6F\\nYr9RXwDZIaHQv9RKzkhPN346zXrYOuAGoL7V7F/MyUH3nX3pzHJDns71+S4Ms5qq\\n6ZjMCu/ic/RsCIoCH5IQsubLpI5bnSsHVt8wLMNR9LwQ/lbRJPWF4LmMnDNJCuC0\\nqJd/bEiNrFhu8IgD6NCT7dQ=\\n-----END PRIVATE KEY-----\\n\",\n \"client_email\": \"search-console-integration-tes@airbyte.iam.gserviceaccount.com\",\n \"client_id\": \"123\",\n \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\n \"token_uri\": \"https://oauth2.googleapis.com/token\",\n \"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\n \"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/search-console-integration-test%40dairbyte.iam.gserviceaccount.com\"\n}", - "email": "test@example.com" + "service_account_info": '{\n "type": "service_account",\n "project_id": "test",\n "private_key_id": "123",\n "private_key": "-----BEGIN PRIVATE KEY-----\\nMIIEvQIBADANBgkqhkiG9w0BBQEFAASCBKcwggSjAgEAAoIBAQDCJPUvep0vXeWb\\nqiwnDxWdd8D75FWJBaYB3rjZvBBhZiY3sA7DmEOj+NJHl4PiPzP8tDZl9MyLBWEc\\neTFSmHSBYSqxax9AOLzWXfLzUezjediIRsGC/Eq9Ue0rkDdMcdcfzQ5J9RDDI1DF\\n1UBxVHFOf7DOSOU7meNPFjAO68aITErvnTh/XL1wWC28PYL351hs57WwLSQTuW0e\\ncUw9XUOE977+qJ4Cs3ZM5c10eid5DDWS4heFG/9hEkobXy34BNdeDodfe9xGSJxD\\nFoAhADj6jMn1z7YgsUG7zpsyW8yh2LtnYdT+fMqIl0FeB4dt0kB3uU1f6vqgo97p\\ndibK6DQ3AgMBAAECggEADWZPz+eZreyNDbnokzrDMw7SMIof1AKIsZKMPDPAE1Pu\\ndlbkWS0LGhs92p6DOUKqFWdHMkrh/tEvuy94L1G1CAr+gqe4mY4KjPPuC7I1wRuM\\n50ovWtlliGL9SIDxkbw+IB4SJIBrS3SgCg+AA6WgezQ5lHtLUXPh6ivHXfhGLlKR\\nI+Gow93UklbxcT57ezeDZVn0U3iUG1H7NkE0livyTTGEMm6GxUqxje7axA4ZVfRL\\nRVrNAHQTihPTThmN/p47Wbh6C8m7o1/cutYDk52CuCjuifxNINlak1ZimSEJ7mcY\\nSIglnTmndQImwiyeDbITtJ3gyYiJerjHnMAYH+VInQKBgQD5HH1tKBxZouozdweu\\n6lpTyko+TBa/3Eo2pgFxbJrKe3pBhkNWVLrCukZxWDFkKSbC+5xaSNGnh/lP/6FX\\nWHWBuBL8R5os9bfNQ9xnArZX7OhzN+aIh8aK5gmEPJE1JaepPyC0X8vaTBqFiQlK\\n6aRB89RqOUlB86B9vzJca7p7LQKBgQDHg1h9A6X9EkWewW5cSOuScw4FElK8N62v\\n5oVByBZZb/Ys9zP04m0yG7VdRSjk8xyCH5+GDS5m9jTxJdctON2AOPL7de8KOtga\\nJSHivUdDLkt7wSmvblc/JYnNs5+B783gTOpdBrXhV6Wo+QpVw1Pcx15b10WLAs8l\\nMzk7LG27cwKBgDJPorVNCIzB7nL+czrMcfnCPURfsaiGISbwWBJEUO7cCVD6gNcK\\nvb1eSaPSoAcOmJmAn49MbatcNuoFQtyVLQZJ2uvAuk6iQcdfF8BmN9WCL2A1xgWF\\nBoA+/WULpngJZtczvLMxNcac4C5gAtRyY44+ZIQflcAQKDW9S7qGt17xAoGBAJ37\\npLtBg1PU/yoJ81DCMT/DOYvMiZUe5bsO5+BCB2iE3sOWcB7umRb/l+qmVA6Pb7ie\\nP9yPXXoMZbm6hBv8FnFtJwL1zPYlyG9TjfSUevR4mS8CsvaGhjGvkOJA5QKoGDcP\\n0Nke8jDhDX2yzntA84w0lsRUv22nKM5FNIFl2fJ/AoGAOAVtlKRPPi2YrjUqqy6F\\nYr9RXwDZIaHQv9RKzkhPN346zXrYOuAGoL7V7F/MyUH3nX3pzHJDns71+S4Ms5qq\\n6ZjMCu/ic/RsCIoCH5IQsubLpI5bnSsHVt8wLMNR9LwQ/lbRJPWF4LmMnDNJCuC0\\nqJd/bEiNrFhu8IgD6NCT7dQ=\\n-----END PRIVATE KEY-----\\n",\n "client_email": "search-console-integration-tes@airbyte.iam.gserviceaccount.com",\n "client_id": "123",\n "auth_uri": "https://accounts.google.com/o/oauth2/auth",\n "token_uri": "https://oauth2.googleapis.com/token",\n "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",\n "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/search-console-integration-test%40dairbyte.iam.gserviceaccount.com"\n}', + "email": "test@example.com", }, - "custom_reports": "[{\"name\": \"custom_dimensions\", \"dimensions\": [\"date\", \"country\", \"device\"]}]" + "custom_reports": '[{"name": "custom_dimensions", "dimensions": ["date", "country", "device"]}]', + "custom_reports_array": [{"name": "custom_dimensions", "dimensions": ["date", "country", "device"]}], } @fixture(name="forbidden_error_message_json") -def forbidden_error_message_json(requests_mock): +def forbidden_error_message_json(): return { "error": { "code": 403, @@ -48,9 +50,28 @@ def forbidden_error_message_json(requests_mock): { "message": "User does not have sufficient permission for site 'https://test-site-test.com/'. See also: https://support.google.com/webmasters/answer/9999999.", "domain": "global", - "reason": "forbidden" + "reason": "forbidden", + } + ], + } + } + + +@fixture(name="bad_aggregation_type") +def bad_aggregation_type(): + return { + "error": { + "code": 400, + "message": "'BY_PROPERTY' is not a valid aggregation type in the context of the request.", + "errors": [ + { + "message": "'BY_PROPERTY' is not a valid aggregation type in the context of the request.", + "domain": "global", + "reason": "invalidParameter", + "location": "aggregation_type", + "locationType": "parameter", } - ] + ], } } diff --git a/airbyte-integrations/connectors/source-google-search-console/unit_tests/test_migrations/test_config.json b/airbyte-integrations/connectors/source-google-search-console/unit_tests/test_migrations/test_config.json new file mode 100644 index 000000000000..dbde2f0aa6d2 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-search-console/unit_tests/test_migrations/test_config.json @@ -0,0 +1,13 @@ +{ + "site_urls": ["https://airbyte.io/"], + "_limit": 6, + "start_date": "2021-09-14", + "end_date": "2025-08-31", + "authorization": { + "auth_type": "Client", + "client_id": "client_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token" + }, + "custom_reports": "[{\"name\": \"custom_dimensions\", \"dimensions\": [\"date\", \"country\", \"device\"]}]" +} diff --git a/airbyte-integrations/connectors/source-google-search-console/unit_tests/test_migrations/test_config_migrations.py b/airbyte-integrations/connectors/source-google-search-console/unit_tests/test_migrations/test_config_migrations.py new file mode 100644 index 000000000000..1a321480fc1f --- /dev/null +++ b/airbyte-integrations/connectors/source-google-search-console/unit_tests/test_migrations/test_config_migrations.py @@ -0,0 +1,82 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import json +from typing import Any, Mapping + +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source +from source_google_search_console.config_migrations import MigrateCustomReports +from source_google_search_console.source import SourceGoogleSearchConsole + +# BASE ARGS +CMD = "check" +TEST_CONFIG_PATH = "unit_tests/test_migrations/test_config.json" +NEW_TEST_CONFIG_PATH = "unit_tests/test_migrations/test_new_config.json" +SOURCE_INPUT_ARGS = [CMD, "--config", TEST_CONFIG_PATH] +SOURCE: Source = SourceGoogleSearchConsole() + + +# HELPERS +def load_config(config_path: str = TEST_CONFIG_PATH) -> Mapping[str, Any]: + with open(config_path, "r") as config: + return json.load(config) + + +def revert_migration(config_path: str = TEST_CONFIG_PATH) -> None: + with open(config_path, "r") as test_config: + config = json.load(test_config) + config.pop("custom_reports_array") + with open(config_path, "w") as updated_config: + config = json.dumps(config) + updated_config.write(config) + + +def test_migrate_config(): + migration_instance = MigrateCustomReports() + original_config = load_config() + # migrate the test_config + migration_instance.migrate(SOURCE_INPUT_ARGS, SOURCE) + # load the updated config + test_migrated_config = load_config() + # check migrated property + assert "custom_reports_array" in test_migrated_config + assert isinstance(test_migrated_config["custom_reports_array"], list) + # check the old property is in place + assert "custom_reports" in test_migrated_config + assert isinstance(test_migrated_config["custom_reports"], str) + # check the migration should be skipped, once already done + assert not migration_instance.should_migrate(test_migrated_config) + # load the old custom reports VS migrated + assert json.loads(original_config["custom_reports"]) == test_migrated_config["custom_reports_array"] + # test CONTROL MESSAGE was emitted + control_msg = migration_instance.message_repository._message_queue[0] + assert control_msg.type == Type.CONTROL + assert control_msg.control.type == OrchestratorType.CONNECTOR_CONFIG + # old custom_reports are stil type(str) + assert isinstance(control_msg.control.connectorConfig.config["custom_reports"], str) + # new custom_reports are type(list) + assert isinstance(control_msg.control.connectorConfig.config["custom_reports_array"], list) + # check the migrated values + assert control_msg.control.connectorConfig.config["custom_reports_array"][0]["name"] == "custom_dimensions" + assert control_msg.control.connectorConfig.config["custom_reports_array"][0]["dimensions"] == ["date", "country", "device"] + # revert the test_config to the starting point + revert_migration() + + +def test_config_is_reverted(): + # check the test_config state, it has to be the same as before tests + test_config = load_config() + # check the config no longer has the migarted property + assert "custom_reports_array" not in test_config + # check the old property is still there + assert "custom_reports" in test_config + assert isinstance(test_config["custom_reports"], str) + + +def test_should_not_migrate_new_config(): + new_config = load_config(NEW_TEST_CONFIG_PATH) + migration_instance = MigrateCustomReports() + assert not migration_instance.should_migrate(new_config) diff --git a/airbyte-integrations/connectors/source-google-search-console/unit_tests/test_migrations/test_new_config.json b/airbyte-integrations/connectors/source-google-search-console/unit_tests/test_migrations/test_new_config.json new file mode 100644 index 000000000000..b901e8ddefec --- /dev/null +++ b/airbyte-integrations/connectors/source-google-search-console/unit_tests/test_migrations/test_new_config.json @@ -0,0 +1,13 @@ +{ + "site_urls": ["https://airbyte.io/"], + "_limit": 6, + "start_date": "2021-09-14", + "end_date": "2025-08-31", + "authorization": { + "auth_type": "Client", + "client_id": "client_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token" + }, + "custom_reports_array": [] +} diff --git a/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py index 304b5e3cf540..84a6ed75f91e 100755 --- a/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py @@ -15,8 +15,10 @@ from source_google_search_console.streams import ( ROW_LIMIT, GoogleSearchConsole, + QueryAggregationType, SearchAnalyticsByCustomDimensions, SearchAnalyticsByDate, + SearchAnalyticsKeywordSiteReportBySite, Sites, ) from utils import command_check @@ -49,9 +51,7 @@ def test_pagination(count, expected): @pytest.mark.parametrize( "site_urls", - [ - ["https://example1.com", "https://example2.com"], ["https://example.com"] - ], + [["https://example1.com", "https://example2.com"], ["https://example.com"]], ) @pytest.mark.parametrize("sync_mode", [SyncMode.full_refresh, SyncMode.incremental]) @pytest.mark.parametrize("data_state", ["all", "final"]) @@ -131,6 +131,21 @@ def test_forbidden_should_retry(requests_mock, forbidden_error_message_json): assert stream.raise_on_http_errors is False +def test_bad_aggregation_type_should_retry(requests_mock, bad_aggregation_type): + stream = SearchAnalyticsKeywordSiteReportBySite(None, ["https://example.com"], "2021-01-01", "2021-01-02") + slice = list(stream.stream_slices(None))[0] + url = stream.url_base + stream.path(None, slice) + requests_mock.get(url, status_code=400, json=bad_aggregation_type) + test_response = requests.get(url) + # before should_retry, the aggregation_type should be set to `by_propety` + assert stream.aggregation_type == QueryAggregationType.by_property + # trigger should retry + assert stream.should_retry(test_response) is False + # after should_retry, the aggregation_type should be set to `auto` + assert stream.aggregation_type == QueryAggregationType.auto + assert stream.raise_on_http_errors is False + + @pytest.mark.parametrize( "stream_class, expected", [ @@ -198,13 +213,15 @@ def test_check_connection(config_gen, config, mocker, requests_mock): ) # test custom_reports - assert command_check(source, config_gen(custom_reports="")) == AirbyteConnectionStatus( - status=Status.FAILED, - message="\"Unable to check connectivity to Google Search Console API - Exception('custom_reports is not valid JSON')\"", - ) - assert command_check(source, config_gen(custom_reports="{}")) == AirbyteConnectionStatus( - status=Status.FAILED, message="''" - ) + with pytest.raises(AirbyteTracedException): + assert command_check(source, config_gen(custom_reports_array="")) == AirbyteConnectionStatus( + status=Status.FAILED, + message="''", + ) + with pytest.raises(AirbyteTracedException): + assert command_check(source, config_gen(custom_reports_array="{}")) == AirbyteConnectionStatus( + status=Status.FAILED, message="''" + ) @pytest.mark.parametrize( @@ -212,10 +229,18 @@ def test_check_connection(config_gen, config, mocker, requests_mock): [ ( lazy_fixture("config"), - (False, "UnauthorizedOauthError('Unable to connect with privided OAuth credentials. The `access token` or `refresh token` is expired. Please re-authrenticate using valid account credenials.')")), + ( + False, + "UnauthorizedOauthError('Unable to connect with privided OAuth credentials. The `access token` or `refresh token` is expired. Please re-authrenticate using valid account credenials.')", + ), + ), ( lazy_fixture("service_account_config"), - (False, "UnauthorizedServiceAccountError('Unable to connect with privided Service Account credentials. Make sure the `sevice account crdentials` povided is valid.')")) + ( + False, + "UnauthorizedServiceAccountError('Unable to connect with privided Service Account credentials. Make sure the `sevice account crdentials` povided is valid.')", + ), + ), ], ) def test_unauthorized_creds_exceptions(test_config, expected, requests_mock): @@ -229,7 +254,7 @@ def test_streams(config_gen): source = SourceGoogleSearchConsole() streams = source.streams(config_gen()) assert len(streams) == 15 - streams = source.streams(config_gen(custom_reports=...)) + streams = source.streams(config_gen(custom_reports_array=...)) assert len(streams) == 14 @@ -243,10 +268,46 @@ def test_get_start_date(): assert date == str(state_date) -def test_custom_streams(): - dimensions = ["date", "country"] +@pytest.mark.parametrize( + "dimensions, expected_status, schema_props, primary_key", + ( + (["impressions"], Status.FAILED, None, None), + ( + [], + Status.SUCCEEDED, + ["clicks", "ctr", "impressions", "position", "date", "site_url", "search_type"], + ["date", "site_url", "search_type"], + ), + ( + ["date"], + Status.SUCCEEDED, + ["clicks", "ctr", "impressions", "position", "date", "site_url", "search_type"], + ["date", "site_url", "search_type"], + ), + ( + ["country", "device", "page", "query"], + Status.SUCCEEDED, + ["clicks", "ctr", "impressions", "position", "date", "site_url", "search_type", "country", "device", "page", "query"], + ["date", "country", "device", "page", "query", "site_url", "search_type"], + ), + ( + ["country", "device", "page", "query", "date"], + Status.SUCCEEDED, + ["clicks", "ctr", "impressions", "position", "date", "site_url", "search_type", "country", "device", "page", "query"], + ["date", "country", "device", "page", "query", "site_url", "search_type"], + ), + ), +) +def test_custom_streams(config_gen, requests_mock, dimensions, expected_status, schema_props, primary_key): + requests_mock.get("https://www.googleapis.com/webmasters/v3/sites/https%3A%2F%2Fexample.com%2F", json={}) + requests_mock.get("https://www.googleapis.com/webmasters/v3/sites", json={"siteEntry": [{"siteUrl": "https://example.com/"}]}) + requests_mock.post("https://oauth2.googleapis.com/token", json={"access_token": "token", "expires_in": 10}) + custom_reports = [{"name": "custom", "dimensions": dimensions}] + status = SourceGoogleSearchConsole().check(config=config_gen(custom_reports_array=custom_reports), logger=None).status + assert status is expected_status + if status is Status.FAILED: + return stream = SearchAnalyticsByCustomDimensions(dimensions, None, ["https://domain1.com", "https://domain2.com"], "2021-09-01", "2021-09-07") schema = stream.get_json_schema() - - for d in ["clicks", "ctr", "date", "impressions", "position", "search_type", "site_url", "country"]: - assert d in schema["properties"] + assert set(schema["properties"]) == set(schema_props) + assert set(stream.primary_key) == set(primary_key) diff --git a/airbyte-integrations/connectors/source-google-sheets/Dockerfile b/airbyte-integrations/connectors/source-google-sheets/Dockerfile deleted file mode 100644 index 647779e246e1..000000000000 --- a/airbyte-integrations/connectors/source-google-sheets/Dockerfile +++ /dev/null @@ -1,40 +0,0 @@ -FROM python:3.9.16-alpine3.18 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash && \ - # upgrading openssl due to https://nvd.nist.gov/vuln/detail/CVE-2023-2650 - apk upgrade - -# copy payload code only -COPY main.py ./ -COPY source_google_sheets ./source_google_sheets - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.3.7 -LABEL io.airbyte.name=airbyte/source-google-sheets diff --git a/airbyte-integrations/connectors/source-google-sheets/README.md b/airbyte-integrations/connectors/source-google-sheets/README.md index de4f98dd178d..313109d6eff3 100644 --- a/airbyte-integrations/connectors/source-google-sheets/README.md +++ b/airbyte-integrations/connectors/source-google-sheets/README.md @@ -1,100 +1,67 @@ -# Google Sheets Source +# Pypi Source -This is the repository for the Google Sheets source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/google-sheets). +This is the repository for the Pypi configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/pypi). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-google-sheets:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/google-sheets) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_sheets/spec.yaml` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/pypi) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_pypi/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source google-sheets test creds` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source pypi test creds` and place them into `secrets/config.json`. - -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-google-sheets:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name source-pypi build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-google-sheets:airbyteDocker +An image will be built with the tag `airbyte/source-pypi:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-pypi:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-google-sheets:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-sheets:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-sheets:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-google-sheets:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm airbyte/source-pypi:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pypi:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pypi:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-pypi:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-google-sheets:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-google-sheets test +``` + +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-google-sheets test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/google-sheets.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml index 6bdbb7516264..63c87ce3137f 100644 --- a/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-sheets/acceptance-test-config.yml @@ -3,31 +3,30 @@ test_strictness_level: high acceptance_tests: basic_read: tests: - - config_path: secrets/service_config.json - expect_records: - path: integration_tests/expected_records.txt + - config_path: secrets/service_config.json + expect_records: + path: integration_tests/expected_records.txt connection: tests: - - config_path: secrets/config.json - status: succeed - - config_path: secrets/config_with_url.json - status: succeed - - config_path: secrets/service_config.json - status: succeed - - config_path: integration_tests/invalid_config.json - status: exception + - config_path: secrets/config.json + status: succeed + - config_path: secrets/config_with_url.json + status: succeed + - config_path: secrets/service_config.json + status: succeed + - config_path: integration_tests/invalid_config.json + status: exception discovery: tests: - - config_path: secrets/service_config.json + - config_path: secrets/service_config.json full_refresh: tests: - - config_path: secrets/service_config.json - configured_catalog_path: integration_tests/configured_catalog.json + - config_path: secrets/service_config.json + configured_catalog_path: integration_tests/configured_catalog.json incremental: bypass_reason: "Incremental sync are not supported on this connector" spec: tests: - - spec_path: source_google_sheets/spec.yaml - backward_compatibility_tests_config: - disable_for_version: "0.3.6" - + - spec_path: source_google_sheets/spec.yaml + backward_compatibility_tests_config: + disable_for_version: "0.3.6" diff --git a/airbyte-integrations/connectors/source-google-sheets/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-google-sheets/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-google-sheets/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-google-sheets/build.gradle b/airbyte-integrations/connectors/source-google-sheets/build.gradle deleted file mode 100644 index bbe959e6f949..000000000000 --- a/airbyte-integrations/connectors/source-google-sheets/build.gradle +++ /dev/null @@ -1,10 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_google_sheets' -} - diff --git a/airbyte-integrations/connectors/source-google-sheets/main.py b/airbyte-integrations/connectors/source-google-sheets/main.py index 4aaa9a106d9b..806ac60fbefe 100644 --- a/airbyte-integrations/connectors/source-google-sheets/main.py +++ b/airbyte-integrations/connectors/source-google-sheets/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_google_sheets import SourceGoogleSheets +from source_google_sheets.run import run if __name__ == "__main__": - source = SourceGoogleSheets() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-google-sheets/metadata.yaml b/airbyte-integrations/connectors/source-google-sheets/metadata.yaml index 8e7c2e79be2d..deea25511ecc 100644 --- a/airbyte-integrations/connectors/source-google-sheets/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-sheets/metadata.yaml @@ -1,12 +1,18 @@ data: + ab_internal: + ql: 400 + sl: 300 allowedHosts: hosts: - "*.googleapis.com" + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: file connectorType: source definitionId: 71607ba1-c0ac-4799-8049-7f4b90dd50f7 - dockerImageTag: 0.3.7 + dockerImageTag: 0.3.13 dockerRepository: airbyte/source-google-sheets + documentationUrl: https://docs.airbyte.com/integrations/sources/google-sheets githubIssueLabel: source-google-sheets icon: google-sheets.svg license: Elv2 @@ -17,11 +23,7 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/google-sheets + supportLevel: certified tags: - language:python - ab_internal: - sl: 300 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-google-sheets/setup.py b/airbyte-integrations/connectors/source-google-sheets/setup.py index 1dd377a9357e..921d19138dbe 100644 --- a/airbyte-integrations/connectors/source-google-sheets/setup.py +++ b/airbyte-integrations/connectors/source-google-sheets/setup.py @@ -33,4 +33,9 @@ extras_require={ "tests": TEST_REQUIREMENTS, }, + entry_points={ + "console_scripts": [ + "source-google-sheets=source_google_sheets.run:run", + ], + }, ) diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/client.py b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/client.py index cbc988ca70ba..1a521ae2467f 100644 --- a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/client.py +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/client.py @@ -33,7 +33,7 @@ def give_up(error): def __init__(self, credentials: Dict[str, str], scopes: List[str] = SCOPES): self.client = Helpers.get_authenticated_sheets_client(credentials, scopes) - def create_range(self, sheet, row_cursor): + def _create_range(self, sheet, row_cursor): range = f"{sheet}!{row_cursor}:{row_cursor + self.Backoff.row_batch_size}" return range @@ -47,7 +47,9 @@ def create(self, **kwargs): @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=Backoff.give_up, on_backoff=Backoff.increase_row_batch_size) def get_values(self, **kwargs): - return self.client.values().batchGet(**kwargs).execute() + range = self._create_range(kwargs.pop("sheet"), kwargs.pop("row_cursor")) + logger.info(f"Fetching range {range}") + return self.client.values().batchGet(ranges=range, **kwargs).execute() @backoff.on_exception(backoff.expo, errors.HttpError, max_time=120, giveup=Backoff.give_up, on_backoff=Backoff.increase_row_batch_size) def update_values(self, **kwargs): diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/helpers.py b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/helpers.py index d8f367baddb6..6043d6b9adf3 100644 --- a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/helpers.py +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/helpers.py @@ -52,7 +52,7 @@ def headers_to_airbyte_stream(logger: AirbyteLogger, sheet_name: str, header_row """ fields, duplicate_fields = Helpers.get_valid_headers_and_duplicates(header_row_values) if duplicate_fields: - logger.warn(f"Duplicate headers found in {sheet_name}. Ignoring them :{duplicate_fields}") + logger.warn(f"Duplicate headers found in {sheet_name}. Ignoring them: {duplicate_fields}") sheet_json_schema = { "$schema": "http://json-schema.org/draft-07/schema#", @@ -85,8 +85,8 @@ def get_valid_headers_and_duplicates(header_row_values: List[str]) -> (List[str] @staticmethod def get_formatted_row_values(row_data: RowData) -> List[str]: """ - Gets the formatted values of all cell data in this row. A formatted value is the final value a user sees in a spreadsheet. It can be a raw - string input by the user, or the result of a sheets function call. + Gets the formatted values of all cell data in this row. A formatted value is the final value a user sees in a spreadsheet. + It can be a raw string input by the user, or the result of a sheets function call. """ return [value.formattedValue for value in row_data.values] @@ -114,7 +114,8 @@ def get_first_row(client, spreadsheet_id: str, sheet_name: str) -> List[str]: first_row_data = all_row_data[0] - return Helpers.get_formatted_row_values(first_row_data) + # When a column is deleted by backspace, a cell with None value is returned, so should be filtered here + return [header_cell for header_cell in Helpers.get_formatted_row_values(first_row_data) if header_cell is not None] @staticmethod def parse_sheet_and_column_names_from_catalog(catalog: ConfiguredAirbyteCatalog) -> Dict[str, FrozenSet[str]]: @@ -151,6 +152,9 @@ def get_available_sheets_to_column_index_to_name( first_row = Helpers.get_first_row(client, spreadsheet_id, sheet) if names_conversion: first_row = [safe_name_conversion(h) for h in first_row] + # When performing names conversion, they won't match what is listed in catalog for the majority of cases, + # so they should be cast here in order to have them in records + columns = {safe_name_conversion(c) for c in columns if c is not None} # Find the column index of each header value idx = 0 for cell_value in first_row: diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/run.py b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/run.py new file mode 100644 index 000000000000..a34dfe611d01 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/run.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch + +from .source import SourceGoogleSheets + + +def run(): + source = SourceGoogleSheets() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/source.py b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/source.py index cf7416a48276..8b7e8faa303e 100644 --- a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/source.py +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/source.py @@ -171,10 +171,13 @@ def _read( # if the last row of the interval goes outside the sheet - this is normal, we will return # only the real data of the sheet and in the next iteration we will loop out. while row_cursor <= sheet_row_counts[sheet]: - range = client.create_range(sheet, row_cursor) - logger.info(f"Fetching range {range}") row_batch = SpreadsheetValues.parse_obj( - client.get_values(spreadsheetId=spreadsheet_id, ranges=range, majorDimension="ROWS") + client.get_values( + sheet=sheet, + row_cursor=row_cursor, + spreadsheetId=spreadsheet_id, + majorDimension="ROWS", + ) ) row_cursor += client.Backoff.row_batch_size + 1 diff --git a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/utils.py b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/utils.py index 689d9856a8d7..0e2168c4aff2 100644 --- a/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/utils.py +++ b/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/utils.py @@ -12,7 +12,7 @@ DEFAULT_SEPARATOR = "_" -def name_conversion(text): +def name_conversion(text: str) -> str: """ convert name using a set of rules, for example: '1MyName' -> '_1_my_name' """ @@ -36,7 +36,9 @@ def name_conversion(text): return text -def safe_name_conversion(text): +def safe_name_conversion(text: str) -> str: + if not text: + return text new = name_conversion(text) if not new: raise Exception(f"initial string '{text}' converted to empty") diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/conftest.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/conftest.py index 315fb8898cb1..c808785e01de 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/conftest.py @@ -9,11 +9,10 @@ def invalid_config(): return { "spreadsheet_id": "invalid_spreadsheet_id", - "credentials": - { - "auth_type": "Client", - "client_id": "fake_client_id", - "client_secret": "fake_client_secret", - "refresh_token": "fake_refresh_token" - } + "credentials": { + "auth_type": "Client", + "client_id": "fake_client_id", + "client_secret": "fake_client_secret", + "refresh_token": "fake_refresh_token", + }, } diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_client.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_client.py index 425559f8edb7..f4c3ed88dfaf 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_client.py @@ -1,30 +1,68 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from unittest.mock import MagicMock import pytest import requests from source_google_sheets.client import GoogleSheetsClient -@pytest.mark.parametrize( - "status, need_give_up", - [ - (429, False), (500, False), (404, True) - ] -) +@pytest.mark.parametrize("status, need_give_up", [(429, False), (500, False), (404, True)]) def test_backoff_give_up(status, need_give_up, mocker): - e = requests.HTTPError('error') + e = requests.HTTPError("error") e.resp = mocker.Mock(status=status) assert need_give_up is GoogleSheetsClient.Backoff.give_up(e) -def test_backoff_increase_row_batch_size(mocker): - assert GoogleSheetsClient.Backoff.row_batch_size == 200 - e = requests.HTTPError('error') +def test_backoff_increase_row_batch_size(): + client = GoogleSheetsClient( + {"auth_type": "Client", "client_id": "fake_client_id", "client_secret": "fake_client_secret", "refresh_token": "fake_refresh_token"} + ) + assert client.Backoff.row_batch_size == 200 + assert client._create_range("spreadsheet_id", 0) == "spreadsheet_id!0:200" + e = requests.HTTPError("error") e.status_code = 429 - GoogleSheetsClient.Backoff.increase_row_batch_size({"exception": e}) - assert GoogleSheetsClient.Backoff.row_batch_size == 210 - GoogleSheetsClient.Backoff.row_batch_size = 1000 - GoogleSheetsClient.Backoff.increase_row_batch_size({"exception": e}) - assert GoogleSheetsClient.Backoff.row_batch_size == 1000 + client.Backoff.increase_row_batch_size({"exception": e}) + assert client.Backoff.row_batch_size == 210 + assert client._create_range("spreadsheet_id", 0) == "spreadsheet_id!0:210" + client.Backoff.row_batch_size = 1000 + client.Backoff.increase_row_batch_size({"exception": e}) + assert client.Backoff.row_batch_size == 1000 + assert client._create_range("spreadsheet_id", 0) == "spreadsheet_id!0:1000" + + +def test_client_get_values_on_backoff(caplog): + client_google_sheets = GoogleSheetsClient( + { + "auth_type": "Client", + "client_id": "fake_client_id", + "client_secret": "fake_client_secret", + "refresh_token": "fake_refresh_token", + }, + ) + client_google_sheets.Backoff.row_batch_size = 210 + client_google_sheets.client.values = MagicMock(return_value=MagicMock(batchGet=MagicMock())) + + assert client_google_sheets.Backoff.row_batch_size == 210 + client_google_sheets.get_values( + sheet="sheet", + row_cursor=0, + spreadsheetId="spreadsheet_id", + majorDimension="ROWS", + ) + + assert "Fetching range sheet!0:210" in caplog.text + assert client_google_sheets.Backoff.row_batch_size == 210 + e = requests.HTTPError("error") + e.status_code = 429 + client_google_sheets.Backoff.increase_row_batch_size({"exception": e}) + assert client_google_sheets.Backoff.row_batch_size == 220 + client_google_sheets.get_values( + sheet="sheet", + row_cursor=0, + spreadsheetId="spreadsheet_id", + majorDimension="ROWS", + ) + + assert "Fetching range sheet!0:220" in caplog.text diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py index 3743f3e46513..768133371828 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_helpers.py @@ -65,7 +65,7 @@ def test_headers_to_airbyte_stream(self): actual_stream = Helpers.headers_to_airbyte_stream(logger, sheet_name, header_values) self.assertEqual(expected_stream, actual_stream) - def test_duplicate_headers_retrived(self): + def test_duplicate_headers_retrieved(self): header_values = ["h1", "h1", "h3"] expected_valid_header_values = ["h3"] @@ -266,10 +266,35 @@ def mock_client_call(spreadsheetId, includeGridData, ranges=None): with patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes: None): sheet_client = GoogleSheetsClient({"fake": "credentials"}, ["auth_scopes"]) sheet_client.client = client + + expected = {sheet1: {0: "1", 1: "2", 2: "3", 3: "4"}} + + # names_conversion = False actual = Helpers.get_available_sheets_to_column_index_to_name( - sheet_client, spreadsheet_id, {sheet1: frozenset(sheet1_first_row), "doesnotexist": frozenset(["1", "2"])} + client=sheet_client, + spreadsheet_id=spreadsheet_id, + requested_sheets_and_columns={sheet1: frozenset(sheet1_first_row), "doesnotexist": frozenset(["1", "2"])}, + ) + self.assertEqual(expected, actual) + + # names_conversion = False, with null header cell + sheet1_first_row = ["1", "2", "3", "4", None] + actual = Helpers.get_available_sheets_to_column_index_to_name( + client=sheet_client, + spreadsheet_id=spreadsheet_id, + requested_sheets_and_columns={sheet1: frozenset(sheet1_first_row), "doesnotexist": frozenset(["1", "2"])}, + ) + self.assertEqual(expected, actual) + + # names_conversion = True, with null header cell + sheet1_first_row = ["AB", "Some Header", "Header", "4", "1MyName", None] + expected = {sheet1: {0: "ab", 1: "some_header", 2: "header", 3: "_4", 4: "_1_my_name"}} + actual = Helpers.get_available_sheets_to_column_index_to_name( + client=sheet_client, + spreadsheet_id=spreadsheet_id, + requested_sheets_and_columns={sheet1: frozenset(sheet1_first_row), "doesnotexist": frozenset(["1", "2"])}, + names_conversion=True, ) - expected = {sheet1: {0: "1", 1: "2", 2: "3", 3: "4"}} self.assertEqual(expected, actual) diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py index 372b2e7d4dc6..20e8d5f862bc 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_stream.py @@ -23,7 +23,7 @@ def set_http_error_for_google_sheets_client(mocker, resp): mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) - mocker.patch.object(GoogleSheetsClient, "get", side_effect=errors.HttpError(resp=resp, content=b'')) + mocker.patch.object(GoogleSheetsClient, "get", side_effect=errors.HttpError(resp=resp, content=b"")) def set_resp_http_error(status_code, error_message=None): @@ -35,10 +35,8 @@ def set_resp_http_error(status_code, error_message=None): def set_sheets_type_grid(sheet_first_row): - data = [ - GridData(rowData=[RowData(values=[CellData(formattedValue=v) for v in sheet_first_row]) - ])] - sheet = Sheet(properties=SheetProperties(title='sheet1', gridProperties='true', sheetType="GRID"), data=data) + data = [GridData(rowData=[RowData(values=[CellData(formattedValue=v) for v in sheet_first_row])])] + sheet = Sheet(properties=SheetProperties(title="sheet1", gridProperties="true", sheetType="GRID"), data=data) return sheet @@ -46,7 +44,7 @@ def test_invalid_credentials_error_message(invalid_config): source = SourceGoogleSheets() with pytest.raises(AirbyteTracedException) as e: source.check(logger=None, config=invalid_config) - assert e.value.args[0] == 'Access to the spreadsheet expired or was revoked. Re-authenticate to restore access.' + assert e.value.args[0] == "Access to the spreadsheet expired or was revoked. Re-authenticate to restore access." def test_invalid_link_error_message(mocker, invalid_config): @@ -54,7 +52,7 @@ def test_invalid_link_error_message(mocker, invalid_config): set_http_error_for_google_sheets_client(mocker, set_resp_http_error(404)) with pytest.raises(AirbyteTracedException) as e: source.check(logger=None, config=invalid_config) - expected_message = 'Config error: The spreadsheet link is not valid. Enter the URL of the Google spreadsheet you want to sync.' + expected_message = "Config error: The spreadsheet link is not valid. Enter the URL of the Google spreadsheet you want to sync." assert e.value.args[0] == expected_message @@ -64,8 +62,10 @@ def test_discover_404_error(mocker, invalid_config): with pytest.raises(AirbyteTracedException) as e: source.discover(logger=mocker.MagicMock(), config=invalid_config) - expected_message = ("The requested Google Sheets spreadsheet with id invalid_spreadsheet_id does not exist." - " Please ensure the Spreadsheet Link you have set is valid and the spreadsheet exists. If the issue persists, contact support. Requested entity was not found.") + expected_message = ( + "The requested Google Sheets spreadsheet with id invalid_spreadsheet_id does not exist." + " Please ensure the Spreadsheet Link you have set is valid and the spreadsheet exists. If the issue persists, contact support. Requested entity was not found." + ) assert e.value.args[0] == expected_message @@ -75,23 +75,25 @@ def test_discover_403_error(mocker, invalid_config): with pytest.raises(AirbyteTracedException) as e: source.discover(logger=mocker.MagicMock(), config=invalid_config) - expected_message = ("The authenticated Google Sheets user does not have permissions to view the " - "spreadsheet with id invalid_spreadsheet_id. Please ensure the authenticated user has access" - " to the Spreadsheet and reauthenticate. If the issue persists, contact support. " - "The caller does not have right permissions.") + expected_message = ( + "The authenticated Google Sheets user does not have permissions to view the " + "spreadsheet with id invalid_spreadsheet_id. Please ensure the authenticated user has access" + " to the Spreadsheet and reauthenticate. If the issue persists, contact support. " + "The caller does not have right permissions." + ) assert e.value.args[0] == expected_message def test_check_invalid_creds_json_file(invalid_config): source = SourceGoogleSheets() res = source.check(logger=None, config={""}) - assert 'Please use valid credentials json file' in res.message + assert "Please use valid credentials json file" in res.message def test_check_access_expired(mocker, invalid_config): source = SourceGoogleSheets() set_http_error_for_google_sheets_client(mocker, set_resp_http_error(403)) - expected_message = 'Access to the spreadsheet expired or was revoked. Re-authenticate to restore access.' + expected_message = "Access to the spreadsheet expired or was revoked. Re-authenticate to restore access." with pytest.raises(AirbyteTracedException): res = source.check(logger=None, config=invalid_config) assert res.message == expected_message @@ -99,23 +101,25 @@ def test_check_access_expired(mocker, invalid_config): def test_check_expected_to_read_data_from_1_sheet(mocker, invalid_config, caplog): source = SourceGoogleSheets() - spreadsheet = Spreadsheet(spreadsheetId='spreadsheet_id', sheets=[set_sheets_type_grid(["1", "2"]), set_sheets_type_grid(["3", "4"])]) + spreadsheet = Spreadsheet(spreadsheetId="spreadsheet_id", sheets=[set_sheets_type_grid(["1", "2"]), set_sheets_type_grid(["3", "4"])]) source = SourceGoogleSheets() mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) mocker.patch.object(GoogleSheetsClient, "get", return_value=spreadsheet) - res = source.check(logger=logging.getLogger('airbyte'), config=invalid_config) + res = source.check(logger=logging.getLogger("airbyte"), config=invalid_config) assert str(res.status) == "Status.FAILED" assert "Unexpected return result: Sheet sheet1 was expected to contain data on exactly 1 sheet." in caplog.text def test_check_duplicated_headers(invalid_config, mocker, caplog): source = SourceGoogleSheets() - spreadsheet = Spreadsheet(spreadsheetId='spreadsheet_id', sheets=[set_sheets_type_grid(["1", "1", "3", "4"])]) + spreadsheet = Spreadsheet(spreadsheetId="spreadsheet_id", sheets=[set_sheets_type_grid(["1", "1", "3", "4"])]) source = SourceGoogleSheets() - expected_message = "The following duplicate headers were found in the following sheets. Please fix them to continue: [sheet:sheet1, headers:['1']]" + expected_message = ( + "The following duplicate headers were found in the following sheets. Please fix them to continue: [sheet:sheet1, headers:['1']]" + ) mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) mocker.patch.object(GoogleSheetsClient, "get", return_value=spreadsheet) - res = source.check(logger=logging.getLogger('airbyte'), config=invalid_config) + res = source.check(logger=logging.getLogger("airbyte"), config=invalid_config) assert str(res.status) == "Status.FAILED" assert expected_message in res.message @@ -123,9 +127,13 @@ def test_check_duplicated_headers(invalid_config, mocker, caplog): def test_check_status_succeeded(mocker, invalid_config): source = SourceGoogleSheets() mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) - mocker.patch.object(GoogleSheetsClient, "get", return_value=Spreadsheet( - spreadsheetId='spreadsheet_id', sheets=[Sheet(properties=SheetProperties(title=t)) for t in ["1", "2", "3", "4"]] - )) + mocker.patch.object( + GoogleSheetsClient, + "get", + return_value=Spreadsheet( + spreadsheetId="spreadsheet_id", sheets=[Sheet(properties=SheetProperties(title=t)) for t in ["1", "2", "3", "4"]] + ), + ) res = source.check(logger=None, config=invalid_config) assert str(res.status) == "Status.SUCCEEDED" @@ -134,16 +142,20 @@ def test_check_status_succeeded(mocker, invalid_config): def test_discover_with_non_grid_sheets(mocker, invalid_config): source = SourceGoogleSheets() mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) - mocker.patch.object(GoogleSheetsClient, "get", return_value=Spreadsheet( - spreadsheetId='spreadsheet_id', sheets=[Sheet(properties=SheetProperties(title=t)) for t in ["1", "2", "3", "4"]] - )) + mocker.patch.object( + GoogleSheetsClient, + "get", + return_value=Spreadsheet( + spreadsheetId="spreadsheet_id", sheets=[Sheet(properties=SheetProperties(title=t)) for t in ["1", "2", "3", "4"]] + ), + ) res = source.discover(logger=mocker.MagicMock(), config=invalid_config) assert res.streams == [] def test_discover(mocker, invalid_config): source = SourceGoogleSheets() - spreadsheet = Spreadsheet(spreadsheetId='spreadsheet_id', sheets=[set_sheets_type_grid(["1", "2", "3", "4"])]) + spreadsheet = Spreadsheet(spreadsheetId="spreadsheet_id", sheets=[set_sheets_type_grid(["1", "2", "3", "4"])]) mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) mocker.patch.object(GoogleSheetsClient, "get", return_value=spreadsheet) res = source.discover(logger=mocker.MagicMock(), config=invalid_config) @@ -152,7 +164,7 @@ def test_discover(mocker, invalid_config): def test_discover_with_names_conversion(mocker, invalid_config): invalid_config["names_conversion"] = True - spreadsheet = Spreadsheet(spreadsheetId='spreadsheet_id', sheets=[set_sheets_type_grid(["1 тест", "2", "3", "4"])]) + spreadsheet = Spreadsheet(spreadsheetId="spreadsheet_id", sheets=[set_sheets_type_grid(["1 тест", "2", "3", "4"])]) source = SourceGoogleSheets() mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) mocker.patch.object(GoogleSheetsClient, "get", return_value=spreadsheet) @@ -162,7 +174,7 @@ def test_discover_with_names_conversion(mocker, invalid_config): def test_discover_incorrect_spreadsheet_name(mocker, invalid_config): - spreadsheet = Spreadsheet(spreadsheetId='spreadsheet_id', sheets=[set_sheets_type_grid(["1", "2", "3", "4"])]) + spreadsheet = Spreadsheet(spreadsheetId="spreadsheet_id", sheets=[set_sheets_type_grid(["1", "2", "3", "4"])]) source = SourceGoogleSheets() mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) mocker.patch.object(GoogleSheetsClient, "get", return_value=spreadsheet) @@ -176,24 +188,26 @@ def test_discover_could_not_run_discover(mocker, invalid_config): with pytest.raises(Exception) as e: source.discover(logger=mocker.MagicMock(), config=invalid_config) - expected_message = ("Could not discover the schema of your spreadsheet. There was an issue with the Google Sheets API." - " This is usually a temporary issue from Google's side. Please try again. If this issue persists, contact support. Interval Server error.") + expected_message = ( + "Could not discover the schema of your spreadsheet. There was an issue with the Google Sheets API." + " This is usually a temporary issue from Google's side. Please try again. If this issue persists, contact support. Interval Server error." + ) assert e.value.args[0] == expected_message def test_get_credentials(invalid_config): expected_config = { - 'auth_type': 'Client', 'client_id': 'fake_client_id', - 'client_secret': 'fake_client_secret', 'refresh_token': 'fake_refresh_token' + "auth_type": "Client", + "client_id": "fake_client_id", + "client_secret": "fake_client_secret", + "refresh_token": "fake_refresh_token", } assert expected_config == SourceGoogleSheets.get_credentials(invalid_config) def test_get_credentials_old_style(): - old_style_config = { - "credentials_json": "some old style data" - } - expected_config = {'auth_type': 'Service', 'service_account_info': 'some old style data'} + old_style_config = {"credentials_json": "some old style data"} + expected_config = {"auth_type": "Service", "service_account_info": "some old style data"} assert expected_config == SourceGoogleSheets.get_credentials(old_style_config) @@ -201,7 +215,11 @@ def test_read_429_error(mocker, invalid_config, caplog): source = SourceGoogleSheets() mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) mocker.patch.object(GoogleSheetsClient, "get", return_value=mocker.Mock) - mocker.patch.object(Helpers, "get_sheets_in_spreadsheet", side_effect=errors.HttpError(resp=set_resp_http_error(429, "Request a higher quota limit"), content=b'')) + mocker.patch.object( + Helpers, + "get_sheets_in_spreadsheet", + side_effect=errors.HttpError(resp=set_resp_http_error(429, "Request a higher quota limit"), content=b""), + ) sheet1 = "soccer_team" sheet1_columns = frozenset(["arsenal", "chelsea", "manutd", "liverpool"]) @@ -217,14 +235,19 @@ def test_read_429_error(mocker, invalid_config, caplog): ) records = list(source.read(logger=logging.getLogger("airbyte"), config=invalid_config, catalog=catalog)) assert [] == records - assert "Stopped syncing process due to rate limits. Rate limit has been reached. Please try later or request a higher quota for your account" in caplog.text + assert ( + "Stopped syncing process due to rate limits. Rate limit has been reached. Please try later or request a higher quota for your account" + in caplog.text + ) def test_read_403_error(mocker, invalid_config, caplog): source = SourceGoogleSheets() mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) mocker.patch.object(GoogleSheetsClient, "get", return_value=mocker.Mock) - mocker.patch.object(Helpers, "get_sheets_in_spreadsheet", side_effect=errors.HttpError(resp=set_resp_http_error(403, "Permission denied"), content=b'')) + mocker.patch.object( + Helpers, "get_sheets_in_spreadsheet", side_effect=errors.HttpError(resp=set_resp_http_error(403, "Permission denied"), content=b"") + ) sheet1 = "soccer_team" sheet1_columns = frozenset(["arsenal", "chelsea", "manutd", "liverpool"]) @@ -240,7 +263,10 @@ def test_read_403_error(mocker, invalid_config, caplog): ) with pytest.raises(AirbyteTracedException) as e: next(source.read(logger=logging.getLogger("airbyte"), config=invalid_config, catalog=catalog)) - assert str(e.value) == "The authenticated Google Sheets user does not have permissions to view the spreadsheet with id invalid_spreadsheet_id. Please ensure the authenticated user has access to the Spreadsheet and reauthenticate. If the issue persists, contact support" + assert ( + str(e.value) + == "The authenticated Google Sheets user does not have permissions to view the spreadsheet with id invalid_spreadsheet_id. Please ensure the authenticated user has access to the Spreadsheet and reauthenticate. If the issue persists, contact support" + ) def test_read_expected_data_on_1_sheet(invalid_config, mocker, caplog): @@ -248,9 +274,13 @@ def test_read_expected_data_on_1_sheet(invalid_config, mocker, caplog): mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) sheet1 = "soccer_team" sheet2 = "soccer_team2" - mocker.patch.object(GoogleSheetsClient, "get", return_value=Spreadsheet( - spreadsheetId='spreadsheet_id', sheets=[Sheet(properties=SheetProperties(title=t)) for t in [sheet1, sheet2]] - )) + mocker.patch.object( + GoogleSheetsClient, + "get", + return_value=Spreadsheet( + spreadsheetId="spreadsheet_id", sheets=[Sheet(properties=SheetProperties(title=t)) for t in [sheet1, sheet2]] + ), + ) sheet1_columns = frozenset(["arsenal", "chelsea", "manutd", "liverpool"]) sheet1_schema = {"properties": {c: {"type": "string"} for c in sheet1_columns}} @@ -279,10 +309,24 @@ def test_read_emply_sheet(invalid_config, mocker, caplog): mocker.patch.object(GoogleSheetsClient, "__init__", lambda s, credentials, scopes=SCOPES: None) sheet1 = "soccer_team" sheet2 = "soccer_team2" - mocker.patch.object(GoogleSheetsClient, "get", return_value=Spreadsheet( - spreadsheetId=invalid_config["spreadsheet_id"], - sheets=[Sheet(properties=SheetProperties(title=t), data=[{"test1": "12", "test2": "123"},]) for t in [sheet1, ]] - )) + mocker.patch.object( + GoogleSheetsClient, + "get", + return_value=Spreadsheet( + spreadsheetId=invalid_config["spreadsheet_id"], + sheets=[ + Sheet( + properties=SheetProperties(title=t), + data=[ + {"test1": "12", "test2": "123"}, + ], + ) + for t in [ + sheet1, + ] + ], + ), + ) sheet1_columns = frozenset(["arsenal", "chelsea"]) sheet1_schema = {"properties": {c: {"type": "string"} for c in sheet1_columns}} @@ -300,6 +344,6 @@ def test_read_emply_sheet(invalid_config, mocker, caplog): ), ] ) - records = list(source.read(logger=logging.getLogger("airbyte"), catalog=catalog,config=invalid_config)) + records = list(source.read(logger=logging.getLogger("airbyte"), catalog=catalog, config=invalid_config)) assert records == [] assert "The sheet soccer_team (ID invalid_spreadsheet_id) is empty!" in caplog.text diff --git a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_utils.py index 8e44ce2b3e37..dde2391f7172 100644 --- a/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-google-sheets/unit_tests/test_utils.py @@ -29,11 +29,20 @@ def test_safe_name_conversion(): @pytest.mark.parametrize( "status_code, expected_message", [ - (404, "The requested Google Sheets spreadsheet with id spreadsheet_id does not exist. Please ensure the Spreadsheet Link you have set is valid and the spreadsheet exists. If the issue persists, contact support"), + ( + 404, + "The requested Google Sheets spreadsheet with id spreadsheet_id does not exist. Please ensure the Spreadsheet Link you have set is valid and the spreadsheet exists. If the issue persists, contact support", + ), (429, "Rate limit has been reached. Please try later or request a higher quota for your account."), - (500, "There was an issue with the Google Sheets API. This is usually a temporary issue from Google's side. Please try again. If this issue persists, contact support"), - (403, "The authenticated Google Sheets user does not have permissions to view the spreadsheet with id spreadsheet_id. Please ensure the authenticated user has access to the Spreadsheet and reauthenticate. If the issue persists, contact support"), - ] + ( + 500, + "There was an issue with the Google Sheets API. This is usually a temporary issue from Google's side. Please try again. If this issue persists, contact support", + ), + ( + 403, + "The authenticated Google Sheets user does not have permissions to view the spreadsheet with id spreadsheet_id. Please ensure the authenticated user has access to the Spreadsheet and reauthenticate. If the issue persists, contact support", + ), + ], ) def test_exception_description_by_status_code(status_code, expected_message): assert expected_message == exception_description_by_status_code(status_code, "spreadsheet_id") diff --git a/airbyte-integrations/connectors/source-google-webfonts/README.md b/airbyte-integrations/connectors/source-google-webfonts/README.md index fec59a21cbba..60a616d26191 100644 --- a/airbyte-integrations/connectors/source-google-webfonts/README.md +++ b/airbyte-integrations/connectors/source-google-webfonts/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-google-webfonts:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/google-webfonts) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_webfonts/spec.yaml` file. @@ -48,18 +40,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-google-webfonts:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-google-webfonts build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-google-webfonts:airbyteDocker +An image will be built with the tag `airbyte/source-google-webfonts:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-google-webfonts:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -69,25 +62,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-webfonts:dev ch docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-webfonts:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-google-webfonts:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-google-webfonts test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-google-webfonts:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-google-webfonts:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -96,8 +81,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-google-webfonts test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/google-webfonts.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-google-webfonts/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-google-webfonts/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-google-webfonts/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-google-webfonts/build.gradle b/airbyte-integrations/connectors/source-google-webfonts/build.gradle deleted file mode 100644 index 2edf95d72c3d..000000000000 --- a/airbyte-integrations/connectors/source-google-webfonts/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_google_webfonts' -} diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/README.md b/airbyte-integrations/connectors/source-google-workspace-admin-reports/README.md index ef56cd73de6d..baf8c2eeffab 100644 --- a/airbyte-integrations/connectors/source-google-workspace-admin-reports/README.md +++ b/airbyte-integrations/connectors/source-google-workspace-admin-reports/README.md @@ -1,100 +1,67 @@ -# Google Workspace Admin Reports Source +# Wikipedia Pageviews Source -This is the repository for the Google Workspace Admin Reports source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/google-workspace-admin-reports). +This is the repository for the Wikipedia Pageviews configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/wikipedia-pageviews). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-google-workspace-admin-reports:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/google-workspace-admin-reports) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_google_workspace_admin_reports/spec.json` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/wikipedia-pageviews) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_wikipedia_pageviews/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source google-workspace-admin-reports test creds` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source wikipedia-pageviews test creds` and place them into `secrets/config.json`. - -### Locally running the connector -``` -python main_dev.py spec -python main_dev.py check --config secrets/config.json -python main_dev.py discover --config secrets/config.json -python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-google-workspace-admin-reports:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name source-wikipedia-pageviews build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-google-workspace-admin-reports:airbyteDocker +An image will be built with the tag `airbyte/source-wikipedia-pageviews:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-wikipedia-pageviews:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-google-workspace-admin-reports:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-workspace-admin-reports:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-google-workspace-admin-reports:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-google-workspace-admin-reports:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm airbyte/source-wikipedia-pageviews:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-wikipedia-pageviews:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-wikipedia-pageviews:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-wikipedia-pageviews:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-google-workspace-admin-reports:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-google-workspace-admin-reports test +``` + +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-google-workspace-admin-reports test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/google-workspace-admin-reports.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-workspace-admin-reports/acceptance-test-config.yml index 636c0727117a..78e2d1fe2aa6 100755 --- a/airbyte-integrations/connectors/source-google-workspace-admin-reports/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-workspace-admin-reports/acceptance-test-config.yml @@ -15,11 +15,11 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: ["admin"] -# We have active test account. New records in reports appear frequently. -# Therefore, second activity differs from first, and it brakes test. -# full_refresh: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" + # We have active test account. New records in reports appear frequently. + # Therefore, second activity differs from first, and it brakes test. + # full_refresh: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-google-workspace-admin-reports/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-google-workspace-admin-reports/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-google-workspace-admin-reports/build.gradle b/airbyte-integrations/connectors/source-google-workspace-admin-reports/build.gradle deleted file mode 100644 index 965b837a734f..000000000000 --- a/airbyte-integrations/connectors/source-google-workspace-admin-reports/build.gradle +++ /dev/null @@ -1,10 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_google_workspace_admin_reports' -} - diff --git a/airbyte-integrations/connectors/source-greenhouse/Dockerfile b/airbyte-integrations/connectors/source-greenhouse/Dockerfile deleted file mode 100644 index a6c69afd9724..000000000000 --- a/airbyte-integrations/connectors/source-greenhouse/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY setup.py ./ -RUN pip install . -COPY source_greenhouse ./source_greenhouse -COPY main.py ./ - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.4.2 -LABEL io.airbyte.name=airbyte/source-greenhouse diff --git a/airbyte-integrations/connectors/source-greenhouse/README.md b/airbyte-integrations/connectors/source-greenhouse/README.md index aa8fa3af70dd..7c78083f0569 100644 --- a/airbyte-integrations/connectors/source-greenhouse/README.md +++ b/airbyte-integrations/connectors/source-greenhouse/README.md @@ -1,6 +1,6 @@ -# Greenhouse Source +# Firebolt Source -This is the repository for the Greenhouse source connector, written in Python. +This is the repository for the Firebolt source connector, written in Python. For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/greenhouse). ## Local development @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-greenhouse:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/greenhouse) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_greenhouse/spec.json` file. @@ -44,7 +38,6 @@ See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source greenhouse test creds` and place them into `secrets/config.json`. - ### Locally running the connector ``` python main.py spec @@ -53,36 +46,70 @@ python main.py discover --config secrets/config.json python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json ``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. -If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-greenhouse:dev \ -&& python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-greenhouse:dev +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name source-greenhouse build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-greenhouse:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations -You can also build the connector image via Gradle: +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-greenhouse:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-greenhouse:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-greenhouse:dev . +# Running the spec command against your patched connector +docker run airbyte/source-greenhouse:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -92,18 +119,28 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-greenhouse:dev discove docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-greenhouse:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-greenhouse:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-greenhouse test +``` + +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-greenhouse test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/greenhouse.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-greenhouse/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-greenhouse/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-greenhouse/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-greenhouse/build.gradle b/airbyte-integrations/connectors/source-greenhouse/build.gradle deleted file mode 100644 index 1e2273f6cdb7..000000000000 --- a/airbyte-integrations/connectors/source-greenhouse/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_greenhouse' -} diff --git a/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.jsonl index ed7fa8cd13b4..b0f80448e3d2 100644 --- a/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.jsonl @@ -7,15 +7,13 @@ {"stream": "applications_interviews", "data": {"id": 40387397003, "application_id": 44937562003, "external_event_id": "123456789", "start": {"date_time": "2021-12-12T13:15:00.000Z"}, "end": {"date_time": "2021-12-12T14:15:00.000Z"}, "location": null, "video_conferencing_url": null, "status": "awaiting_feedback", "created_at": "2021-10-10T16:21:44.107Z", "updated_at": "2021-12-12T15:15:02.894Z", "interview": {"id": 5628615003, "name": "Preliminary Screening Call"}, "organizer": {"id": 4218085003, "first_name": "Greenhouse", "last_name": "Admin", "name": "Greenhouse Admin", "employee_id": null}, "interviewers": [{"id": 4218085003, "employee_id": null, "name": "Greenhouse Admin", "email": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "response_status": "accepted", "scorecard_id": null}]}, "emitted_at": 1691572565721} {"stream": "applications_interviews", "data": {"id": 40387426003, "application_id": 44937562003, "external_event_id": "12345678", "start": {"date_time": "2021-12-13T13:15:00.000Z"}, "end": {"date_time": "2021-12-13T14:15:00.000Z"}, "location": null, "video_conferencing_url": null, "status": "awaiting_feedback", "created_at": "2021-10-10T16:22:04.561Z", "updated_at": "2021-12-13T15:15:13.252Z", "interview": {"id": 5628615003, "name": "Preliminary Screening Call"}, "organizer": {"id": 4218085003, "first_name": "Greenhouse", "last_name": "Admin", "name": "Greenhouse Admin", "employee_id": null}, "interviewers": [{"id": 4218085003, "employee_id": null, "name": "Greenhouse Admin", "email": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "response_status": "accepted", "scorecard_id": null}]}, "emitted_at": 1691572565725} {"stream": "applications_interviews", "data": {"id": 40387431003, "application_id": 44937562003, "external_event_id": "1234567", "start": {"date_time": "2021-12-14T13:15:00.000Z"}, "end": {"date_time": "2021-12-14T14:15:00.000Z"}, "location": null, "video_conferencing_url": null, "status": "awaiting_feedback", "created_at": "2021-10-10T16:22:13.681Z", "updated_at": "2021-12-14T15:15:12.118Z", "interview": {"id": 5628615003, "name": "Preliminary Screening Call"}, "organizer": {"id": 4218085003, "first_name": "Greenhouse", "last_name": "Admin", "name": "Greenhouse Admin", "employee_id": null}, "interviewers": [{"id": 4218085003, "employee_id": null, "name": "Greenhouse Admin", "email": "scrubbed_email_vq8-rm4513etm7xxd9d1qq@example.com", "response_status": "accepted", "scorecard_id": null}]}, "emitted_at": 1691572565728} -{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2020-11-24T23:24:37.050Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Test", "last_activity": "2020-11-24T23:24:37.049Z", "is_private": false, "id": 17130511003, "first_name": "Test", "employments": [], "email_addresses": [], "educations": [], "created_at": "2020-11-24T23:24:37.018Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "HRMARKET", "id": 4000067003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": {"name": "Airbyte Team", "id": 4218086003}}, "prospect": true, "location": null, "last_activity_at": "2020-11-24T23:24:37.049Z", "jobs": [], "job_post_id": null, "id": 19214950003, "current_stage": null, "credited_to": {"name": "Airbyte Team", "last_name": "Team", "id": 4218086003, "first_name": "Airbyte", "employee_id": null}, "candidate_id": 17130511003, "attachments": [], "applied_at": "2020-11-24T23:24:37.023Z", "answers": []}], "application_ids": [19214950003], "addresses": []}, "emitted_at": 1691572566540} -{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2020-11-24T23:25:13.806Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Test2", "last_activity": "2020-11-24T23:25:13.804Z", "is_private": false, "id": 17130554003, "first_name": "Test2", "employments": [], "email_addresses": [], "educations": [], "created_at": "2020-11-24T23:25:13.777Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "Jobs page on your website", "id": 4000177003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": {"name": "Airbyte Team", "id": 4218086003}}, "prospect": true, "location": null, "last_activity_at": "2020-11-24T23:25:13.804Z", "jobs": [], "job_post_id": null, "id": 19214993003, "current_stage": null, "credited_to": {"name": "Airbyte Team", "last_name": "Team", "id": 4218086003, "first_name": "Airbyte", "employee_id": null}, "candidate_id": 17130554003, "attachments": [], "applied_at": "2020-11-24T23:25:13.781Z", "answers": []}], "application_ids": [19214993003], "addresses": []}, "emitted_at": 1691572566544} -{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2020-11-24T23:28:19.781Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Lastname", "last_activity": "2020-11-24T23:28:19.779Z", "is_private": false, "id": 17130732003, "first_name": "Name", "employments": [], "email_addresses": [], "educations": [], "created_at": "2020-11-24T23:28:19.710Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "Internal Applicant", "id": 4000142003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2020-11-24T23:28:19.779Z", "jobs": [{"name": "Test job", "id": 4177046003}], "job_post_id": null, "id": 19215172003, "current_stage": {"name": "Preliminary Phone Screen", "id": 5245804003}, "credited_to": {"name": "Airbyte Team", "last_name": "Team", "id": 4218086003, "first_name": "Airbyte", "employee_id": null}, "candidate_id": 17130732003, "attachments": [], "applied_at": "2020-11-24T23:28:19.712Z", "answers": []}], "application_ids": [19215172003], "addresses": []}, "emitted_at": 1691572566548} -{"stream": "close_reasons", "data": {"id": 4010635003, "name": "Not Filling"}, "emitted_at": 1691572567002} +{"stream": "candidates", "data": {"id": 44081361003, "first_name": "Ivan", "last_name": "Petrov", "company": null, "title": null, "created_at": "2021-11-22T08:41:55.634Z", "updated_at": "2023-01-17T08:23:44.812Z", "last_activity": "2023-01-17T08:23:44.791Z", "is_private": false, "photo_url": null, "attachments": [], "application_ids": [48693310003], "phone_numbers": [], "addresses": [], "email_addresses": [], "website_addresses": [], "social_media_addresses": [], "recruiter": null, "coordinator": null, "can_email": true, "tags": [], "applications": [{"id": 48693310003, "candidate_id": 44081361003, "prospect": false, "applied_at": "2021-11-22T08:41:55.640Z", "rejected_at": null, "last_activity_at": "2023-01-17T08:23:44.791Z", "location": null, "attachments": [], "source": {"id": 4000032003, "public_name": "Bubblesort"}, "credited_to": {"id": 4218087003, "first_name": null, "last_name": null, "name": "emily.brooks+airbyte_integration@greenhouse.io", "employee_id": null}, "rejection_reason": null, "rejection_details": null, "jobs": [{"id": 4446240003, "name": "Copy of Test Job 2"}], "job_post_id": null, "status": "active", "current_stage": {"id": 7179760003, "name": "Application Review"}, "answers": [], "prospective_department": null, "prospective_office": null, "prospect_detail": {"prospect_pool": null, "prospect_stage": null, "prospect_owner": null}}], "educations": [], "employments": [], "linked_user_ids": []}, "emitted_at": 1701307364287} +{"stream": "candidates", "data": {"id": 40513954003, "first_name": "Test", "last_name": "User", "company": null, "title": null, "created_at": "2021-09-29T16:37:27.585Z", "updated_at": "2021-09-29T16:38:03.672Z", "last_activity": "2021-09-29T16:38:03.660Z", "is_private": false, "photo_url": null, "attachments": [], "application_ids": [44933447003], "phone_numbers": [], "addresses": [], "email_addresses": [], "website_addresses": [], "social_media_addresses": [], "recruiter": null, "coordinator": null, "can_email": true, "tags": [], "applications": [{"id": 44933447003, "candidate_id": 40513954003, "prospect": false, "applied_at": "2021-09-29T16:37:27.589Z", "rejected_at": "2021-09-29T16:38:03.637Z", "last_activity_at": "2021-09-29T16:38:03.660Z", "location": null, "attachments": [], "source": {"id": 4013544003, "public_name": "Test agency"}, "credited_to": {"id": 4218086003, "first_name": "Airbyte", "last_name": "Team", "name": "Airbyte Team", "employee_id": null}, "rejection_reason": {"id": 4000004003, "name": "Other (add notes below)", "type": {"id": 4000000003, "name": "We rejected them"}}, "rejection_details": {}, "jobs": [{"id": 4177046003, "name": "Test job"}], "job_post_id": null, "status": "rejected", "current_stage": {"id": 5245805003, "name": "Phone Interview"}, "answers": [], "prospective_department": null, "prospective_office": null, "prospect_detail": {"prospect_pool": null, "prospect_stage": null, "prospect_owner": null}}], "educations": [], "employments": [], "linked_user_ids": []}, "emitted_at": 1701304885157} +{"stream": "candidates", "data": {"id": 17130732003, "first_name": "Name", "last_name": "Lastname", "company": null, "title": null, "created_at": "2020-11-24T23:28:19.710Z", "updated_at": "2020-11-24T23:28:19.781Z", "last_activity": "2020-11-24T23:28:19.779Z", "is_private": false, "photo_url": null, "attachments": [], "application_ids": [19215172003], "phone_numbers": [], "addresses": [], "email_addresses": [], "website_addresses": [], "social_media_addresses": [], "recruiter": null, "coordinator": null, "can_email": true, "tags": [], "applications": [{"id": 19215172003, "candidate_id": 17130732003, "prospect": false, "applied_at": "2020-11-24T23:28:19.712Z", "rejected_at": null, "last_activity_at": "2020-11-24T23:28:19.779Z", "location": null, "attachments": [], "source": {"id": 4000142003, "public_name": "Internal Applicant"}, "credited_to": {"id": 4218086003, "first_name": "Airbyte", "last_name": "Team", "name": "Airbyte Team", "employee_id": null}, "rejection_reason": null, "rejection_details": null, "jobs": [{"id": 4177046003, "name": "Test job"}], "job_post_id": null, "status": "active", "current_stage": {"id": 5245804003, "name": "Preliminary Phone Screen"}, "answers": [], "prospective_department": null, "prospective_office": null, "prospect_detail": {"prospect_pool": null, "prospect_stage": null, "prospect_owner": null}}], "educations": [], "employments": [], "linked_user_ids": []}, "emitted_at": 1701304885151} {"stream": "close_reasons", "data": {"id": 4010634003, "name": "On Hold"}, "emitted_at": 1691572567004} {"stream": "close_reasons", "data": {"id": 4010633003, "name": "Hire - New Headcount"}, "emitted_at": 1691572567006} -{"stream": "custom_fields", "data": {"id": 4680898003, "name": "School Name", "active": true, "field_type": "candidate", "priority": 0, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "school_name", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10845822003, "name": "Abraham Baldwin Agricultural College", "priority": 0, "external_id": null}, {"id": 10845823003, "name": "Academy of Art University", "priority": 1, "external_id": null}, {"id": 10845824003, "name": "Acadia University", "priority": 2, "external_id": null}, {"id": 10845825003, "name": "Adams State University", "priority": 3, "external_id": null}, {"id": 10845826003, "name": "Adelphi University", "priority": 4, "external_id": null}, {"id": 10845827003, "name": "Adrian College", "priority": 5, "external_id": null}, {"id": 10845828003, "name": "Adventist University of Health Sciences", "priority": 6, "external_id": null}, {"id": 10845829003, "name": "Agnes Scott College", "priority": 7, "external_id": null}, {"id": 10845830003, "name": "AIB College of Business", "priority": 8, "external_id": null}, {"id": 10845831003, "name": "Alaska Pacific University", "priority": 9, "external_id": null}, {"id": 10845832003, "name": "Albany College of Pharmacy and Health Sciences", "priority": 10, "external_id": null}, {"id": 10845833003, "name": "Albany State University", "priority": 11, "external_id": null}, {"id": 10845834003, "name": "Albertus Magnus College", "priority": 12, "external_id": null}, {"id": 10845835003, "name": "Albion College", "priority": 13, "external_id": null}, {"id": 10845836003, "name": "Albright College", "priority": 14, "external_id": null}, {"id": 10845837003, "name": "Alderson Broaddus University", "priority": 15, "external_id": null}, {"id": 10845838003, "name": "Alfred University", "priority": 16, "external_id": null}, {"id": 10845839003, "name": "Alice Lloyd College", "priority": 17, "external_id": null}, {"id": 10845840003, "name": "Allegheny College", "priority": 18, "external_id": null}, {"id": 10845841003, "name": "Allen College", "priority": 19, "external_id": null}, {"id": 10845842003, "name": "Allen University", "priority": 20, "external_id": null}, {"id": 10845843003, "name": "Alliant International University", "priority": 21, "external_id": null}, {"id": 10845844003, "name": "Alma College", "priority": 22, "external_id": null}, {"id": 10845845003, "name": "Alvernia University", "priority": 23, "external_id": null}, {"id": 10845846003, "name": "Alverno College", "priority": 24, "external_id": null}, {"id": 10845847003, "name": "Amberton University", "priority": 25, "external_id": null}, {"id": 10845848003, "name": "American Academy of Art", "priority": 26, "external_id": null}, {"id": 10845849003, "name": "American Indian College of the Assemblies of God", "priority": 27, "external_id": null}, {"id": 10845850003, "name": "American InterContinental University", "priority": 28, "external_id": null}, {"id": 10845851003, "name": "American International College", "priority": 29, "external_id": null}, {"id": 10845852003, "name": "American Jewish University", "priority": 30, "external_id": null}, {"id": 10845853003, "name": "American Public University System", "priority": 31, "external_id": null}, {"id": 10845854003, "name": "American University", "priority": 32, "external_id": null}, {"id": 10845855003, "name": "American University in Bulgaria", "priority": 33, "external_id": null}, {"id": 10845856003, "name": "American University in Cairo", "priority": 34, "external_id": null}, {"id": 10845857003, "name": "American University of Beirut", "priority": 35, "external_id": null}, {"id": 10845858003, "name": "American University of Paris", "priority": 36, "external_id": null}, {"id": 10845859003, "name": "American University of Puerto Rico", "priority": 37, "external_id": null}, {"id": 10845860003, "name": "Amherst College", "priority": 38, "external_id": null}, {"id": 10845861003, "name": "Amridge University", "priority": 39, "external_id": null}, {"id": 10845862003, "name": "Anderson University", "priority": 40, "external_id": null}, {"id": 10845863003, "name": "Andrews University", "priority": 41, "external_id": null}, {"id": 10845864003, "name": "Angelo State University", "priority": 42, "external_id": null}, {"id": 10845865003, "name": "Anna Maria College", "priority": 43, "external_id": null}, {"id": 10845866003, "name": "Antioch University", "priority": 44, "external_id": null}, {"id": 10845867003, "name": "Appalachian Bible College", "priority": 45, "external_id": null}, {"id": 10845868003, "name": "Aquinas College", "priority": 46, "external_id": null}, {"id": 10845869003, "name": "Arcadia University", "priority": 47, "external_id": null}, {"id": 10845870003, "name": "Argosy University", "priority": 48, "external_id": null}, {"id": 10845871003, "name": "Arizona Christian University", "priority": 49, "external_id": null}, {"id": 10845872003, "name": "Arizona State University - West", "priority": 50, "external_id": null}, {"id": 10845873003, "name": "Arkansas Baptist College", "priority": 51, "external_id": null}, {"id": 10845874003, "name": "Arkansas Tech University", "priority": 52, "external_id": null}, {"id": 10845875003, "name": "Armstrong Atlantic State University", "priority": 53, "external_id": null}, {"id": 10845876003, "name": "Art Academy of Cincinnati", "priority": 54, "external_id": null}, {"id": 10845877003, "name": "Art Center College of Design", "priority": 55, "external_id": null}, {"id": 10845878003, "name": "Art Institute of Atlanta", "priority": 56, "external_id": null}, {"id": 10845879003, "name": "Art Institute of Colorado", "priority": 57, "external_id": null}, {"id": 10845880003, "name": "Art Institute of Houston", "priority": 58, "external_id": null}, {"id": 10845881003, "name": "Art Institute of Pittsburgh", "priority": 59, "external_id": null}, {"id": 10845882003, "name": "Art Institute of Portland", "priority": 60, "external_id": null}, {"id": 10845883003, "name": "Art Institute of Seattle", "priority": 61, "external_id": null}, {"id": 10845884003, "name": "Asbury University", "priority": 62, "external_id": null}, {"id": 10845885003, "name": "Ashford University", "priority": 63, "external_id": null}, {"id": 10845886003, "name": "Ashland University", "priority": 64, "external_id": null}, {"id": 10845887003, "name": "Assumption College", "priority": 65, "external_id": null}, {"id": 10845888003, "name": "Athens State University", "priority": 66, "external_id": null}, {"id": 10845889003, "name": "Auburn University - Montgomery", "priority": 67, "external_id": null}, {"id": 10845890003, "name": "Augsburg College", "priority": 68, "external_id": null}, {"id": 10845891003, "name": "Augustana College", "priority": 69, "external_id": null}, {"id": 10845892003, "name": "Aurora University", "priority": 70, "external_id": null}, {"id": 10845893003, "name": "Austin College", "priority": 71, "external_id": null}, {"id": 10845894003, "name": "Alcorn State University", "priority": 72, "external_id": null}, {"id": 10845895003, "name": "Ave Maria University", "priority": 73, "external_id": null}, {"id": 10845896003, "name": "Averett University", "priority": 74, "external_id": null}, {"id": 10845897003, "name": "Avila University", "priority": 75, "external_id": null}, {"id": 10845898003, "name": "Azusa Pacific University", "priority": 76, "external_id": null}, {"id": 10845899003, "name": "Babson College", "priority": 77, "external_id": null}, {"id": 10845900003, "name": "Bacone College", "priority": 78, "external_id": null}, {"id": 10845901003, "name": "Baker College of Flint", "priority": 79, "external_id": null}, {"id": 10845902003, "name": "Baker University", "priority": 80, "external_id": null}, {"id": 10845903003, "name": "Baldwin Wallace University", "priority": 81, "external_id": null}, {"id": 10845904003, "name": "Christian Brothers University", "priority": 82, "external_id": null}, {"id": 10845905003, "name": "Abilene Christian University", "priority": 83, "external_id": null}, {"id": 10845906003, "name": "Arizona State University", "priority": 84, "external_id": null}, {"id": 10845907003, "name": "Auburn University", "priority": 85, "external_id": null}, {"id": 10845908003, "name": "Alabama A&M University", "priority": 86, "external_id": null}, {"id": 10845909003, "name": "Alabama State University", "priority": 87, "external_id": null}, {"id": 10845910003, "name": "Arkansas State University", "priority": 88, "external_id": null}, {"id": 10845911003, "name": "Baptist Bible College", "priority": 89, "external_id": null}, {"id": 10845912003, "name": "Baptist Bible College and Seminary", "priority": 90, "external_id": null}, {"id": 10845913003, "name": "Baptist College of Florida", "priority": 91, "external_id": null}, {"id": 10845914003, "name": "Baptist Memorial College of Health Sciences", "priority": 92, "external_id": null}, {"id": 10845915003, "name": "Baptist Missionary Association Theological Seminary", "priority": 93, "external_id": null}, {"id": 10845916003, "name": "Bard College", "priority": 94, "external_id": null}, {"id": 10845917003, "name": "Bard College at Simon's Rock", "priority": 95, "external_id": null}, {"id": 10845918003, "name": "Barnard College", "priority": 96, "external_id": null}, {"id": 10845919003, "name": "Barry University", "priority": 97, "external_id": null}, {"id": 10845920003, "name": "Barton College", "priority": 98, "external_id": null}, {"id": 10845921003, "name": "Bastyr University", "priority": 99, "external_id": null}, {"id": 10845922003, "name": "Bates College", "priority": 100, "external_id": null}, {"id": 10845923003, "name": "Bauder College", "priority": 101, "external_id": null}, {"id": 10845924003, "name": "Bay Path College", "priority": 102, "external_id": null}, {"id": 10845925003, "name": "Bay State College", "priority": 103, "external_id": null}, {"id": 10845926003, "name": "Bayamon Central University", "priority": 104, "external_id": null}, {"id": 10845927003, "name": "Beacon College", "priority": 105, "external_id": null}, {"id": 10845928003, "name": "Becker College", "priority": 106, "external_id": null}, {"id": 10845929003, "name": "Belhaven University", "priority": 107, "external_id": null}, {"id": 10845930003, "name": "Bellarmine University", "priority": 108, "external_id": null}, {"id": 10845931003, "name": "Bellevue College", "priority": 109, "external_id": null}, {"id": 10845932003, "name": "Bellevue University", "priority": 110, "external_id": null}, {"id": 10845933003, "name": "Bellin College", "priority": 111, "external_id": null}, {"id": 10845934003, "name": "Belmont Abbey College", "priority": 112, "external_id": null}, {"id": 10845935003, "name": "Belmont University", "priority": 113, "external_id": null}, {"id": 10845936003, "name": "Beloit College", "priority": 114, "external_id": null}, {"id": 10845937003, "name": "Bemidji State University", "priority": 115, "external_id": null}, {"id": 10845938003, "name": "Benedict College", "priority": 116, "external_id": null}, {"id": 10845939003, "name": "Benedictine College", "priority": 117, "external_id": null}, {"id": 10845940003, "name": "Benedictine University", "priority": 118, "external_id": null}, {"id": 10845941003, "name": "Benjamin Franklin Institute of Technology", "priority": 119, "external_id": null}, {"id": 10845942003, "name": "Bennett College", "priority": 120, "external_id": null}, {"id": 10845943003, "name": "Bennington College", "priority": 121, "external_id": null}, {"id": 10845944003, "name": "Bentley University", "priority": 122, "external_id": null}, {"id": 10845945003, "name": "Berea College", "priority": 123, "external_id": null}, {"id": 10845946003, "name": "Berkeley College", "priority": 124, "external_id": null}, {"id": 10845947003, "name": "Berklee College of Music", "priority": 125, "external_id": null}, {"id": 10845948003, "name": "Berry College", "priority": 126, "external_id": null}, {"id": 10845949003, "name": "Bethany College", "priority": 127, "external_id": null}, {"id": 10845950003, "name": "Bethany Lutheran College", "priority": 128, "external_id": null}, {"id": 10845951003, "name": "Bethel College", "priority": 129, "external_id": null}, {"id": 10845952003, "name": "Bethel University", "priority": 130, "external_id": null}, {"id": 10845953003, "name": "BI Norwegian Business School", "priority": 131, "external_id": null}, {"id": 10845954003, "name": "Binghamton University - SUNY", "priority": 132, "external_id": null}, {"id": 10845955003, "name": "Biola University", "priority": 133, "external_id": null}, {"id": 10845956003, "name": "Birmingham-Southern College", "priority": 134, "external_id": null}, {"id": 10845957003, "name": "Bismarck State College", "priority": 135, "external_id": null}, {"id": 10845958003, "name": "Black Hills State University", "priority": 136, "external_id": null}, {"id": 10845959003, "name": "Blackburn College", "priority": 137, "external_id": null}, {"id": 10845960003, "name": "Blessing-Rieman College of Nursing", "priority": 138, "external_id": null}, {"id": 10845961003, "name": "Bloomfield College", "priority": 139, "external_id": null}, {"id": 10845962003, "name": "Bloomsburg University of Pennsylvania", "priority": 140, "external_id": null}, {"id": 10845963003, "name": "Blue Mountain College", "priority": 141, "external_id": null}, {"id": 10845964003, "name": "Bluefield College", "priority": 142, "external_id": null}, {"id": 10845965003, "name": "Bluefield State College", "priority": 143, "external_id": null}, {"id": 10845966003, "name": "Bluffton University", "priority": 144, "external_id": null}, {"id": 10845967003, "name": "Boricua College", "priority": 145, "external_id": null}, {"id": 10845968003, "name": "Boston Architectural College", "priority": 146, "external_id": null}, {"id": 10845969003, "name": "Boston Conservatory", "priority": 147, "external_id": null}, {"id": 10845970003, "name": "Boston University", "priority": 148, "external_id": null}, {"id": 10845971003, "name": "Bowdoin College", "priority": 149, "external_id": null}, {"id": 10845972003, "name": "Bowie State University", "priority": 150, "external_id": null}, {"id": 10845973003, "name": "Bradley University", "priority": 151, "external_id": null}, {"id": 10845974003, "name": "Brandeis University", "priority": 152, "external_id": null}, {"id": 10845975003, "name": "Brandman University", "priority": 153, "external_id": null}, {"id": 10845976003, "name": "Brazosport College", "priority": 154, "external_id": null}, {"id": 10845977003, "name": "Brenau University", "priority": 155, "external_id": null}, {"id": 10845978003, "name": "Brescia University", "priority": 156, "external_id": null}, {"id": 10845979003, "name": "Brevard College", "priority": 157, "external_id": null}, {"id": 10845980003, "name": "Brewton-Parker College", "priority": 158, "external_id": null}, {"id": 10845981003, "name": "Briar Cliff University", "priority": 159, "external_id": null}, {"id": 10845982003, "name": "Briarcliffe College", "priority": 160, "external_id": null}, {"id": 10845983003, "name": "Bridgewater College", "priority": 161, "external_id": null}, {"id": 10845984003, "name": "Bridgewater State University", "priority": 162, "external_id": null}, {"id": 10845985003, "name": "Brigham Young University - Hawaii", "priority": 163, "external_id": null}, {"id": 10845986003, "name": "Brigham Young University - Idaho", "priority": 164, "external_id": null}, {"id": 10845987003, "name": "Brock University", "priority": 165, "external_id": null}, {"id": 10845988003, "name": "Bryan College", "priority": 166, "external_id": null}, {"id": 10845989003, "name": "Bryn Athyn College of the New Church", "priority": 167, "external_id": null}, {"id": 10845990003, "name": "Bryn Mawr College", "priority": 168, "external_id": null}, {"id": 10845991003, "name": "Boston College", "priority": 169, "external_id": null}, {"id": 10845992003, "name": "Buena Vista University", "priority": 170, "external_id": null}, {"id": 10845993003, "name": "Burlington College", "priority": 171, "external_id": null}, {"id": 10845994003, "name": "Bowling Green State University", "priority": 172, "external_id": null}, {"id": 10845995003, "name": "Brown University", "priority": 173, "external_id": null}, {"id": 10845996003, "name": "Appalachian State University", "priority": 174, "external_id": null}, {"id": 10845997003, "name": "Brigham Young University - Provo", "priority": 175, "external_id": null}, {"id": 10845998003, "name": "Boise State University", "priority": 176, "external_id": null}, {"id": 10845999003, "name": "Bethune-Cookman University", "priority": 177, "external_id": null}, {"id": 10846000003, "name": "Bryant University", "priority": 178, "external_id": null}, {"id": 10846001003, "name": "Cabarrus College of Health Sciences", "priority": 179, "external_id": null}, {"id": 10846002003, "name": "Cabrini College", "priority": 180, "external_id": null}, {"id": 10846003003, "name": "Cairn University", "priority": 181, "external_id": null}, {"id": 10846004003, "name": "Caldwell College", "priority": 182, "external_id": null}, {"id": 10846005003, "name": "California Baptist University", "priority": 183, "external_id": null}, {"id": 10846006003, "name": "California College of the Arts", "priority": 184, "external_id": null}, {"id": 10846007003, "name": "California Institute of Integral Studies", "priority": 185, "external_id": null}, {"id": 10846008003, "name": "California Institute of Technology", "priority": 186, "external_id": null}, {"id": 10846009003, "name": "California Institute of the Arts", "priority": 187, "external_id": null}, {"id": 10846010003, "name": "California Lutheran University", "priority": 188, "external_id": null}, {"id": 10846011003, "name": "California Maritime Academy", "priority": 189, "external_id": null}, {"id": 10846012003, "name": "California State Polytechnic University - Pomona", "priority": 190, "external_id": null}, {"id": 10846013003, "name": "California State University - Bakersfield", "priority": 191, "external_id": null}, {"id": 10846014003, "name": "California State University - Channel Islands", "priority": 192, "external_id": null}, {"id": 10846015003, "name": "California State University - Chico", "priority": 193, "external_id": null}, {"id": 10846016003, "name": "California State University - Dominguez Hills", "priority": 194, "external_id": null}, {"id": 10846017003, "name": "California State University - East Bay", "priority": 195, "external_id": null}, {"id": 10846018003, "name": "California State University - Fullerton", "priority": 196, "external_id": null}, {"id": 10846019003, "name": "California State University - Los Angeles", "priority": 197, "external_id": null}, {"id": 10846020003, "name": "California State University - Monterey Bay", "priority": 198, "external_id": null}, {"id": 10846021003, "name": "California State University - Northridge", "priority": 199, "external_id": null}, {"id": 10846022003, "name": "California State University - San Bernardino", "priority": 200, "external_id": null}, {"id": 10846023003, "name": "California State University - San Marcos", "priority": 201, "external_id": null}, {"id": 10846024003, "name": "California State University - Stanislaus", "priority": 202, "external_id": null}, {"id": 10846025003, "name": "California University of Pennsylvania", "priority": 203, "external_id": null}, {"id": 10846026003, "name": "Calumet College of St. Joseph", "priority": 204, "external_id": null}, {"id": 10846027003, "name": "Calvary Bible College and Theological Seminary", "priority": 205, "external_id": null}, {"id": 10846028003, "name": "Calvin College", "priority": 206, "external_id": null}, {"id": 10846029003, "name": "Cambridge College", "priority": 207, "external_id": null}, {"id": 10846030003, "name": "Cameron University", "priority": 208, "external_id": null}, {"id": 10846031003, "name": "Campbellsville University", "priority": 209, "external_id": null}, {"id": 10846032003, "name": "Canisius College", "priority": 210, "external_id": null}, {"id": 10846033003, "name": "Capella University", "priority": 211, "external_id": null}, {"id": 10846034003, "name": "Capital University", "priority": 212, "external_id": null}, {"id": 10846035003, "name": "Capitol College", "priority": 213, "external_id": null}, {"id": 10846036003, "name": "Cardinal Stritch University", "priority": 214, "external_id": null}, {"id": 10846037003, "name": "Caribbean University", "priority": 215, "external_id": null}, {"id": 10846038003, "name": "Carleton College", "priority": 216, "external_id": null}, {"id": 10846039003, "name": "Carleton University", "priority": 217, "external_id": null}, {"id": 10846040003, "name": "Carlos Albizu University", "priority": 218, "external_id": null}, {"id": 10846041003, "name": "Carlow University", "priority": 219, "external_id": null}, {"id": 10846042003, "name": "Carnegie Mellon University", "priority": 220, "external_id": null}, {"id": 10846043003, "name": "Carroll College", "priority": 221, "external_id": null}, {"id": 10846044003, "name": "Carroll University", "priority": 222, "external_id": null}, {"id": 10846045003, "name": "Carson-Newman University", "priority": 223, "external_id": null}, {"id": 10846046003, "name": "Carthage College", "priority": 224, "external_id": null}, {"id": 10846047003, "name": "Case Western Reserve University", "priority": 225, "external_id": null}, {"id": 10846048003, "name": "Castleton State College", "priority": 226, "external_id": null}, {"id": 10846049003, "name": "Catawba College", "priority": 227, "external_id": null}, {"id": 10846050003, "name": "Cazenovia College", "priority": 228, "external_id": null}, {"id": 10846051003, "name": "Cedar Crest College", "priority": 229, "external_id": null}, {"id": 10846052003, "name": "Cedarville University", "priority": 230, "external_id": null}, {"id": 10846053003, "name": "Centenary College", "priority": 231, "external_id": null}, {"id": 10846054003, "name": "Centenary College of Louisiana", "priority": 232, "external_id": null}, {"id": 10846055003, "name": "Central Baptist College", "priority": 233, "external_id": null}, {"id": 10846056003, "name": "Central Bible College", "priority": 234, "external_id": null}, {"id": 10846057003, "name": "Central Christian College", "priority": 235, "external_id": null}, {"id": 10846058003, "name": "Central College", "priority": 236, "external_id": null}, {"id": 10846059003, "name": "Central Methodist University", "priority": 237, "external_id": null}, {"id": 10846060003, "name": "Central Penn College", "priority": 238, "external_id": null}, {"id": 10846061003, "name": "Central State University", "priority": 239, "external_id": null}, {"id": 10846062003, "name": "Central Washington University", "priority": 240, "external_id": null}, {"id": 10846063003, "name": "Centre College", "priority": 241, "external_id": null}, {"id": 10846064003, "name": "Chadron State College", "priority": 242, "external_id": null}, {"id": 10846065003, "name": "Chamberlain College of Nursing", "priority": 243, "external_id": null}, {"id": 10846066003, "name": "Chaminade University of Honolulu", "priority": 244, "external_id": null}, {"id": 10846067003, "name": "Champlain College", "priority": 245, "external_id": null}, {"id": 10846068003, "name": "Chancellor University", "priority": 246, "external_id": null}, {"id": 10846069003, "name": "Chapman University", "priority": 247, "external_id": null}, {"id": 10846070003, "name": "Charles R. Drew University of Medicine and Science", "priority": 248, "external_id": null}, {"id": 10846071003, "name": "Charter Oak State College", "priority": 249, "external_id": null}, {"id": 10846072003, "name": "Chatham University", "priority": 250, "external_id": null}, {"id": 10846073003, "name": "Chestnut Hill College", "priority": 251, "external_id": null}, {"id": 10846074003, "name": "Cheyney University of Pennsylvania", "priority": 252, "external_id": null}, {"id": 10846075003, "name": "Chicago State University", "priority": 253, "external_id": null}, {"id": 10846076003, "name": "Chipola College", "priority": 254, "external_id": null}, {"id": 10846077003, "name": "Chowan University", "priority": 255, "external_id": null}, {"id": 10846078003, "name": "Christendom College", "priority": 256, "external_id": null}, {"id": 10846079003, "name": "Baylor University", "priority": 257, "external_id": null}, {"id": 10846080003, "name": "Central Connecticut State University", "priority": 258, "external_id": null}, {"id": 10846081003, "name": "Central Michigan University", "priority": 259, "external_id": null}, {"id": 10846082003, "name": "Charleston Southern University", "priority": 260, "external_id": null}, {"id": 10846083003, "name": "California State University - Sacramento", "priority": 261, "external_id": null}, {"id": 10846084003, "name": "California State University - Fresno", "priority": 262, "external_id": null}, {"id": 10846085003, "name": "Campbell University", "priority": 263, "external_id": null}, {"id": 10846086003, "name": "Christopher Newport University", "priority": 264, "external_id": null}, {"id": 10846087003, "name": "Cincinnati Christian University", "priority": 265, "external_id": null}, {"id": 10846088003, "name": "Cincinnati College of Mortuary Science", "priority": 266, "external_id": null}, {"id": 10846089003, "name": "City University of Seattle", "priority": 267, "external_id": null}, {"id": 10846090003, "name": "Claflin University", "priority": 268, "external_id": null}, {"id": 10846091003, "name": "Claremont McKenna College", "priority": 269, "external_id": null}, {"id": 10846092003, "name": "Clarion University of Pennsylvania", "priority": 270, "external_id": null}, {"id": 10846093003, "name": "Clark Atlanta University", "priority": 271, "external_id": null}, {"id": 10846094003, "name": "Clark University", "priority": 272, "external_id": null}, {"id": 10846095003, "name": "Clarke University", "priority": 273, "external_id": null}, {"id": 10846096003, "name": "Clarkson College", "priority": 274, "external_id": null}, {"id": 10846097003, "name": "Clarkson University", "priority": 275, "external_id": null}, {"id": 10846098003, "name": "Clayton State University", "priority": 276, "external_id": null}, {"id": 10846099003, "name": "Clear Creek Baptist Bible College", "priority": 277, "external_id": null}, {"id": 10846100003, "name": "Clearwater Christian College", "priority": 278, "external_id": null}, {"id": 10846101003, "name": "Cleary University", "priority": 279, "external_id": null}, {"id": 10846102003, "name": "College of William and Mary", "priority": 280, "external_id": null}, {"id": 10846103003, "name": "Cleveland Chiropractic College", "priority": 281, "external_id": null}, {"id": 10846104003, "name": "Cleveland Institute of Art", "priority": 282, "external_id": null}, {"id": 10846105003, "name": "Cleveland Institute of Music", "priority": 283, "external_id": null}, {"id": 10846106003, "name": "Cleveland State University", "priority": 284, "external_id": null}, {"id": 10846107003, "name": "Coe College", "priority": 285, "external_id": null}, {"id": 10846108003, "name": "Cogswell Polytechnical College", "priority": 286, "external_id": null}, {"id": 10846109003, "name": "Coker College", "priority": 287, "external_id": null}, {"id": 10846110003, "name": "Colby College", "priority": 288, "external_id": null}, {"id": 10846111003, "name": "Colby-Sawyer College", "priority": 289, "external_id": null}, {"id": 10846112003, "name": "College at Brockport - SUNY", "priority": 290, "external_id": null}, {"id": 10846113003, "name": "College for Creative Studies", "priority": 291, "external_id": null}, {"id": 10846114003, "name": "College of Charleston", "priority": 292, "external_id": null}, {"id": 10846115003, "name": "College of Idaho", "priority": 293, "external_id": null}, {"id": 10846116003, "name": "College of Mount St. Joseph", "priority": 294, "external_id": null}, {"id": 10846117003, "name": "College of Mount St. Vincent", "priority": 295, "external_id": null}, {"id": 10846118003, "name": "College of New Jersey", "priority": 296, "external_id": null}, {"id": 10846119003, "name": "College of New Rochelle", "priority": 297, "external_id": null}, {"id": 10846120003, "name": "College of Our Lady of the Elms", "priority": 298, "external_id": null}, {"id": 10846121003, "name": "College of Saints John Fisher & Thomas More", "priority": 299, "external_id": null}, {"id": 10846122003, "name": "College of Southern Nevada", "priority": 300, "external_id": null}, {"id": 10846123003, "name": "College of St. Benedict", "priority": 301, "external_id": null}, {"id": 10846124003, "name": "College of St. Elizabeth", "priority": 302, "external_id": null}, {"id": 10846125003, "name": "College of St. Joseph", "priority": 303, "external_id": null}, {"id": 10846126003, "name": "College of St. Mary", "priority": 304, "external_id": null}, {"id": 10846127003, "name": "College of St. Rose", "priority": 305, "external_id": null}, {"id": 10846128003, "name": "College of St. Scholastica", "priority": 306, "external_id": null}, {"id": 10846129003, "name": "College of the Atlantic", "priority": 307, "external_id": null}, {"id": 10846130003, "name": "College of the Holy Cross", "priority": 308, "external_id": null}, {"id": 10846131003, "name": "College of the Ozarks", "priority": 309, "external_id": null}, {"id": 10846132003, "name": "College of Wooster", "priority": 310, "external_id": null}, {"id": 10846133003, "name": "Colorado Christian University", "priority": 311, "external_id": null}, {"id": 10846134003, "name": "Colorado College", "priority": 312, "external_id": null}, {"id": 10846135003, "name": "Colorado Mesa University", "priority": 313, "external_id": null}, {"id": 10846136003, "name": "Colorado School of Mines", "priority": 314, "external_id": null}, {"id": 10846137003, "name": "Colorado State University - Pueblo", "priority": 315, "external_id": null}, {"id": 10846138003, "name": "Colorado Technical University", "priority": 316, "external_id": null}, {"id": 10846139003, "name": "Columbia College", "priority": 317, "external_id": null}, {"id": 10846140003, "name": "Columbia College Chicago", "priority": 318, "external_id": null}, {"id": 10846141003, "name": "Columbia College of Nursing", "priority": 319, "external_id": null}, {"id": 10846142003, "name": "Columbia International University", "priority": 320, "external_id": null}, {"id": 10846143003, "name": "Columbus College of Art and Design", "priority": 321, "external_id": null}, {"id": 10846144003, "name": "Columbus State University", "priority": 322, "external_id": null}, {"id": 10846145003, "name": "Conception Seminary College", "priority": 323, "external_id": null}, {"id": 10846146003, "name": "Concord University", "priority": 324, "external_id": null}, {"id": 10846147003, "name": "Concordia College", "priority": 325, "external_id": null}, {"id": 10846148003, "name": "Concordia College - Moorhead", "priority": 326, "external_id": null}, {"id": 10846149003, "name": "Concordia University", "priority": 327, "external_id": null}, {"id": 10846150003, "name": "Concordia University Chicago", "priority": 328, "external_id": null}, {"id": 10846151003, "name": "Concordia University Texas", "priority": 329, "external_id": null}, {"id": 10846152003, "name": "Concordia University Wisconsin", "priority": 330, "external_id": null}, {"id": 10846153003, "name": "Concordia University - St. Paul", "priority": 331, "external_id": null}, {"id": 10846154003, "name": "Connecticut College", "priority": 332, "external_id": null}, {"id": 10846155003, "name": "Converse College", "priority": 333, "external_id": null}, {"id": 10846156003, "name": "Cooper Union", "priority": 334, "external_id": null}, {"id": 10846157003, "name": "Coppin State University", "priority": 335, "external_id": null}, {"id": 10846158003, "name": "Corban University", "priority": 336, "external_id": null}, {"id": 10846159003, "name": "Corcoran College of Art and Design", "priority": 337, "external_id": null}, {"id": 10846160003, "name": "Cornell College", "priority": 338, "external_id": null}, {"id": 10846161003, "name": "Cornerstone University", "priority": 339, "external_id": null}, {"id": 10846162003, "name": "Cornish College of the Arts", "priority": 340, "external_id": null}, {"id": 10846163003, "name": "Covenant College", "priority": 341, "external_id": null}, {"id": 10846164003, "name": "Cox College", "priority": 342, "external_id": null}, {"id": 10846165003, "name": "Creighton University", "priority": 343, "external_id": null}, {"id": 10846166003, "name": "Criswell College", "priority": 344, "external_id": null}, {"id": 10846167003, "name": "Crown College", "priority": 345, "external_id": null}, {"id": 10846168003, "name": "Culinary Institute of America", "priority": 346, "external_id": null}, {"id": 10846169003, "name": "Culver-Stockton College", "priority": 347, "external_id": null}, {"id": 10846170003, "name": "Cumberland University", "priority": 348, "external_id": null}, {"id": 10846171003, "name": "Columbia University", "priority": 349, "external_id": null}, {"id": 10846172003, "name": "Cornell University", "priority": 350, "external_id": null}, {"id": 10846173003, "name": "Colorado State University", "priority": 351, "external_id": null}, {"id": 10846174003, "name": "University of Virginia", "priority": 352, "external_id": null}, {"id": 10846175003, "name": "Colgate University", "priority": 353, "external_id": null}, {"id": 10846176003, "name": "CUNY - Baruch College", "priority": 354, "external_id": null}, {"id": 10846177003, "name": "CUNY - Brooklyn College", "priority": 355, "external_id": null}, {"id": 10846178003, "name": "CUNY - City College", "priority": 356, "external_id": null}, {"id": 10846179003, "name": "CUNY - College of Staten Island", "priority": 357, "external_id": null}, {"id": 10846180003, "name": "CUNY - Hunter College", "priority": 358, "external_id": null}, {"id": 10846181003, "name": "CUNY - John Jay College of Criminal Justice", "priority": 359, "external_id": null}, {"id": 10846182003, "name": "CUNY - Lehman College", "priority": 360, "external_id": null}, {"id": 10846183003, "name": "CUNY - Medgar Evers College", "priority": 361, "external_id": null}, {"id": 10846184003, "name": "CUNY - New York City College of Technology", "priority": 362, "external_id": null}, {"id": 10846185003, "name": "CUNY - Queens College", "priority": 363, "external_id": null}, {"id": 10846186003, "name": "CUNY - York College", "priority": 364, "external_id": null}, {"id": 10846187003, "name": "Curry College", "priority": 365, "external_id": null}, {"id": 10846188003, "name": "Curtis Institute of Music", "priority": 366, "external_id": null}, {"id": 10846189003, "name": "D'Youville College", "priority": 367, "external_id": null}, {"id": 10846190003, "name": "Daemen College", "priority": 368, "external_id": null}, {"id": 10846191003, "name": "Dakota State University", "priority": 369, "external_id": null}, {"id": 10846192003, "name": "Dakota Wesleyan University", "priority": 370, "external_id": null}, {"id": 10846193003, "name": "Dalhousie University", "priority": 371, "external_id": null}, {"id": 10846194003, "name": "Dallas Baptist University", "priority": 372, "external_id": null}, {"id": 10846195003, "name": "Dallas Christian College", "priority": 373, "external_id": null}, {"id": 10846196003, "name": "Dalton State College", "priority": 374, "external_id": null}, {"id": 10846197003, "name": "Daniel Webster College", "priority": 375, "external_id": null}, {"id": 10846198003, "name": "Davenport University", "priority": 376, "external_id": null}, {"id": 10846199003, "name": "Davis and Elkins College", "priority": 377, "external_id": null}, {"id": 10846200003, "name": "Davis College", "priority": 378, "external_id": null}, {"id": 10846201003, "name": "Daytona State College", "priority": 379, "external_id": null}, {"id": 10846202003, "name": "Dean College", "priority": 380, "external_id": null}, {"id": 10846203003, "name": "Defiance College", "priority": 381, "external_id": null}, {"id": 10846204003, "name": "Delaware Valley College", "priority": 382, "external_id": null}, {"id": 10846205003, "name": "Delta State University", "priority": 383, "external_id": null}, {"id": 10846206003, "name": "Denison University", "priority": 384, "external_id": null}, {"id": 10846207003, "name": "DePaul University", "priority": 385, "external_id": null}, {"id": 10846208003, "name": "DePauw University", "priority": 386, "external_id": null}, {"id": 10846209003, "name": "DEREE - The American College of Greece", "priority": 387, "external_id": null}, {"id": 10846210003, "name": "DeSales University", "priority": 388, "external_id": null}, {"id": 10846211003, "name": "DeVry University", "priority": 389, "external_id": null}, {"id": 10846212003, "name": "Dickinson College", "priority": 390, "external_id": null}, {"id": 10846213003, "name": "Dickinson State University", "priority": 391, "external_id": null}, {"id": 10846214003, "name": "Dillard University", "priority": 392, "external_id": null}, {"id": 10846215003, "name": "Divine Word College", "priority": 393, "external_id": null}, {"id": 10846216003, "name": "Dixie State College of Utah", "priority": 394, "external_id": null}, {"id": 10846217003, "name": "Doane College", "priority": 395, "external_id": null}, {"id": 10846218003, "name": "Dominican College", "priority": 396, "external_id": null}, {"id": 10846219003, "name": "Dominican University", "priority": 397, "external_id": null}, {"id": 10846220003, "name": "Dominican University of California", "priority": 398, "external_id": null}, {"id": 10846221003, "name": "Donnelly College", "priority": 399, "external_id": null}, {"id": 10846222003, "name": "Dordt College", "priority": 400, "external_id": null}, {"id": 10846223003, "name": "Dowling College", "priority": 401, "external_id": null}, {"id": 10846224003, "name": "Drew University", "priority": 402, "external_id": null}, {"id": 10846225003, "name": "Drexel University", "priority": 403, "external_id": null}, {"id": 10846226003, "name": "Drury University", "priority": 404, "external_id": null}, {"id": 10846227003, "name": "Dunwoody College of Technology", "priority": 405, "external_id": null}, {"id": 10846228003, "name": "Earlham College", "priority": 406, "external_id": null}, {"id": 10846229003, "name": "Drake University", "priority": 407, "external_id": null}, {"id": 10846230003, "name": "East Central University", "priority": 408, "external_id": null}, {"id": 10846231003, "name": "East Stroudsburg University of Pennsylvania", "priority": 409, "external_id": null}, {"id": 10846232003, "name": "East Tennessee State University", "priority": 410, "external_id": null}, {"id": 10846233003, "name": "East Texas Baptist University", "priority": 411, "external_id": null}, {"id": 10846234003, "name": "East-West University", "priority": 412, "external_id": null}, {"id": 10846235003, "name": "Eastern Connecticut State University", "priority": 413, "external_id": null}, {"id": 10846236003, "name": "Eastern Mennonite University", "priority": 414, "external_id": null}, {"id": 10846237003, "name": "Eastern Nazarene College", "priority": 415, "external_id": null}, {"id": 10846238003, "name": "Eastern New Mexico University", "priority": 416, "external_id": null}, {"id": 10846239003, "name": "Eastern Oregon University", "priority": 417, "external_id": null}, {"id": 10846240003, "name": "Eastern University", "priority": 418, "external_id": null}, {"id": 10846241003, "name": "Eckerd College", "priority": 419, "external_id": null}, {"id": 10846242003, "name": "ECPI University", "priority": 420, "external_id": null}, {"id": 10846243003, "name": "Edgewood College", "priority": 421, "external_id": null}, {"id": 10846244003, "name": "Edinboro University of Pennsylvania", "priority": 422, "external_id": null}, {"id": 10846245003, "name": "Edison State College", "priority": 423, "external_id": null}, {"id": 10846246003, "name": "Edward Waters College", "priority": 424, "external_id": null}, {"id": 10846247003, "name": "Elizabeth City State University", "priority": 425, "external_id": null}, {"id": 10846248003, "name": "Elizabethtown College", "priority": 426, "external_id": null}, {"id": 10846249003, "name": "Elmhurst College", "priority": 427, "external_id": null}, {"id": 10846250003, "name": "Elmira College", "priority": 428, "external_id": null}, {"id": 10846251003, "name": "Embry-Riddle Aeronautical University", "priority": 429, "external_id": null}, {"id": 10846252003, "name": "Embry-Riddle Aeronautical University - Prescott", "priority": 430, "external_id": null}, {"id": 10846253003, "name": "Emerson College", "priority": 431, "external_id": null}, {"id": 10846254003, "name": "Duquesne University", "priority": 432, "external_id": null}, {"id": 10846255003, "name": "Eastern Washington University", "priority": 433, "external_id": null}, {"id": 10846256003, "name": "Eastern Illinois University", "priority": 434, "external_id": null}, {"id": 10846257003, "name": "Eastern Kentucky University", "priority": 435, "external_id": null}, {"id": 10846258003, "name": "Eastern Michigan University", "priority": 436, "external_id": null}, {"id": 10846259003, "name": "Elon University", "priority": 437, "external_id": null}, {"id": 10846260003, "name": "Delaware State University", "priority": 438, "external_id": null}, {"id": 10846261003, "name": "Duke University", "priority": 439, "external_id": null}, {"id": 10846262003, "name": "California Polytechnic State University - San Luis Obispo", "priority": 440, "external_id": null}, {"id": 10846263003, "name": "Emmanuel College", "priority": 441, "external_id": null}, {"id": 10846264003, "name": "Emmaus Bible College", "priority": 442, "external_id": null}, {"id": 10846265003, "name": "Emory and Henry College", "priority": 443, "external_id": null}, {"id": 10846266003, "name": "Emory University", "priority": 444, "external_id": null}, {"id": 10846267003, "name": "Emporia State University", "priority": 445, "external_id": null}, {"id": 10846268003, "name": "Endicott College", "priority": 446, "external_id": null}, {"id": 10846269003, "name": "Erskine College", "priority": 447, "external_id": null}, {"id": 10846270003, "name": "Escuela de Artes Plasticas de Puerto Rico", "priority": 448, "external_id": null}, {"id": 10846271003, "name": "Eureka College", "priority": 449, "external_id": null}, {"id": 10846272003, "name": "Evangel University", "priority": 450, "external_id": null}, {"id": 10846273003, "name": "Everest College - Phoenix", "priority": 451, "external_id": null}, {"id": 10846274003, "name": "Everglades University", "priority": 452, "external_id": null}, {"id": 10846275003, "name": "Evergreen State College", "priority": 453, "external_id": null}, {"id": 10846276003, "name": "Excelsior College", "priority": 454, "external_id": null}, {"id": 10846277003, "name": "Fairfield University", "priority": 455, "external_id": null}, {"id": 10846278003, "name": "Fairleigh Dickinson University", "priority": 456, "external_id": null}, {"id": 10846279003, "name": "Fairmont State University", "priority": 457, "external_id": null}, {"id": 10846280003, "name": "Faith Baptist Bible College and Theological Seminary", "priority": 458, "external_id": null}, {"id": 10846281003, "name": "Farmingdale State College - SUNY", "priority": 459, "external_id": null}, {"id": 10846282003, "name": "Fashion Institute of Technology", "priority": 460, "external_id": null}, {"id": 10846283003, "name": "Faulkner University", "priority": 461, "external_id": null}, {"id": 10846284003, "name": "Fayetteville State University", "priority": 462, "external_id": null}, {"id": 10846285003, "name": "Felician College", "priority": 463, "external_id": null}, {"id": 10846286003, "name": "Ferris State University", "priority": 464, "external_id": null}, {"id": 10846287003, "name": "Ferrum College", "priority": 465, "external_id": null}, {"id": 10846288003, "name": "Finlandia University", "priority": 466, "external_id": null}, {"id": 10846289003, "name": "Fisher College", "priority": 467, "external_id": null}, {"id": 10846290003, "name": "Fisk University", "priority": 468, "external_id": null}, {"id": 10846291003, "name": "Fitchburg State University", "priority": 469, "external_id": null}, {"id": 10846292003, "name": "Five Towns College", "priority": 470, "external_id": null}, {"id": 10846293003, "name": "Flagler College", "priority": 471, "external_id": null}, {"id": 10846294003, "name": "Florida Christian College", "priority": 472, "external_id": null}, {"id": 10846295003, "name": "Florida College", "priority": 473, "external_id": null}, {"id": 10846296003, "name": "Florida Gulf Coast University", "priority": 474, "external_id": null}, {"id": 10846297003, "name": "Florida Institute of Technology", "priority": 475, "external_id": null}, {"id": 10846298003, "name": "Florida Memorial University", "priority": 476, "external_id": null}, {"id": 10846299003, "name": "Florida Southern College", "priority": 477, "external_id": null}, {"id": 10846300003, "name": "Florida State College - Jacksonville", "priority": 478, "external_id": null}, {"id": 10846301003, "name": "Fontbonne University", "priority": 479, "external_id": null}, {"id": 10846302003, "name": "Fort Hays State University", "priority": 480, "external_id": null}, {"id": 10846303003, "name": "Fort Lewis College", "priority": 481, "external_id": null}, {"id": 10846304003, "name": "Fort Valley State University", "priority": 482, "external_id": null}, {"id": 10846305003, "name": "Framingham State University", "priority": 483, "external_id": null}, {"id": 10846306003, "name": "Francis Marion University", "priority": 484, "external_id": null}, {"id": 10846307003, "name": "Franciscan University of Steubenville", "priority": 485, "external_id": null}, {"id": 10846308003, "name": "Frank Lloyd Wright School of Architecture", "priority": 486, "external_id": null}, {"id": 10846309003, "name": "Franklin and Marshall College", "priority": 487, "external_id": null}, {"id": 10846310003, "name": "Franklin College", "priority": 488, "external_id": null}, {"id": 10846311003, "name": "Franklin College Switzerland", "priority": 489, "external_id": null}, {"id": 10846312003, "name": "Franklin Pierce University", "priority": 490, "external_id": null}, {"id": 10846313003, "name": "Franklin University", "priority": 491, "external_id": null}, {"id": 10846314003, "name": "Franklin W. Olin College of Engineering", "priority": 492, "external_id": null}, {"id": 10846315003, "name": "Freed-Hardeman University", "priority": 493, "external_id": null}, {"id": 10846316003, "name": "Fresno Pacific University", "priority": 494, "external_id": null}, {"id": 10846317003, "name": "Friends University", "priority": 495, "external_id": null}, {"id": 10846318003, "name": "Frostburg State University", "priority": 496, "external_id": null}, {"id": 10846319003, "name": "Gallaudet University", "priority": 497, "external_id": null}, {"id": 10846320003, "name": "Gannon University", "priority": 498, "external_id": null}, {"id": 10846321003, "name": "Geneva College", "priority": 499, "external_id": null}, {"id": 10846322003, "name": "George Fox University", "priority": 500, "external_id": null}, {"id": 10846323003, "name": "George Mason University", "priority": 501, "external_id": null}, {"id": 10846324003, "name": "George Washington University", "priority": 502, "external_id": null}, {"id": 10846325003, "name": "Georgetown College", "priority": 503, "external_id": null}, {"id": 10846326003, "name": "Georgia College & State University", "priority": 504, "external_id": null}, {"id": 10846327003, "name": "Georgia Gwinnett College", "priority": 505, "external_id": null}, {"id": 10846328003, "name": "Georgia Regents University", "priority": 506, "external_id": null}, {"id": 10846329003, "name": "Georgia Southwestern State University", "priority": 507, "external_id": null}, {"id": 10846330003, "name": "Georgian Court University", "priority": 508, "external_id": null}, {"id": 10846331003, "name": "Gettysburg College", "priority": 509, "external_id": null}, {"id": 10846332003, "name": "Glenville State College", "priority": 510, "external_id": null}, {"id": 10846333003, "name": "God's Bible School and College", "priority": 511, "external_id": null}, {"id": 10846334003, "name": "Goddard College", "priority": 512, "external_id": null}, {"id": 10846335003, "name": "Golden Gate University", "priority": 513, "external_id": null}, {"id": 10846336003, "name": "Goldey-Beacom College", "priority": 514, "external_id": null}, {"id": 10846337003, "name": "Goldfarb School of Nursing at Barnes-Jewish College", "priority": 515, "external_id": null}, {"id": 10846338003, "name": "Gonzaga University", "priority": 516, "external_id": null}, {"id": 10846339003, "name": "Gordon College", "priority": 517, "external_id": null}, {"id": 10846340003, "name": "Fordham University", "priority": 518, "external_id": null}, {"id": 10846341003, "name": "Georgia Institute of Technology", "priority": 519, "external_id": null}, {"id": 10846342003, "name": "Gardner-Webb University", "priority": 520, "external_id": null}, {"id": 10846343003, "name": "Georgia Southern University", "priority": 521, "external_id": null}, {"id": 10846344003, "name": "Georgia State University", "priority": 522, "external_id": null}, {"id": 10846345003, "name": "Florida State University", "priority": 523, "external_id": null}, {"id": 10846346003, "name": "Dartmouth College", "priority": 524, "external_id": null}, {"id": 10846347003, "name": "Florida International University", "priority": 525, "external_id": null}, {"id": 10846348003, "name": "Georgetown University", "priority": 526, "external_id": null}, {"id": 10846349003, "name": "Furman University", "priority": 527, "external_id": null}, {"id": 10846350003, "name": "Gordon State College", "priority": 528, "external_id": null}, {"id": 10846351003, "name": "Goshen College", "priority": 529, "external_id": null}, {"id": 10846352003, "name": "Goucher College", "priority": 530, "external_id": null}, {"id": 10846353003, "name": "Governors State University", "priority": 531, "external_id": null}, {"id": 10846354003, "name": "Grace Bible College", "priority": 532, "external_id": null}, {"id": 10846355003, "name": "Grace College and Seminary", "priority": 533, "external_id": null}, {"id": 10846356003, "name": "Grace University", "priority": 534, "external_id": null}, {"id": 10846357003, "name": "Graceland University", "priority": 535, "external_id": null}, {"id": 10846358003, "name": "Grand Canyon University", "priority": 536, "external_id": null}, {"id": 10846359003, "name": "Grand Valley State University", "priority": 537, "external_id": null}, {"id": 10846360003, "name": "Grand View University", "priority": 538, "external_id": null}, {"id": 10846361003, "name": "Granite State College", "priority": 539, "external_id": null}, {"id": 10846362003, "name": "Gratz College", "priority": 540, "external_id": null}, {"id": 10846363003, "name": "Great Basin College", "priority": 541, "external_id": null}, {"id": 10846364003, "name": "Great Lakes Christian College", "priority": 542, "external_id": null}, {"id": 10846365003, "name": "Green Mountain College", "priority": 543, "external_id": null}, {"id": 10846366003, "name": "Greensboro College", "priority": 544, "external_id": null}, {"id": 10846367003, "name": "Greenville College", "priority": 545, "external_id": null}, {"id": 10846368003, "name": "Grinnell College", "priority": 546, "external_id": null}, {"id": 10846369003, "name": "Grove City College", "priority": 547, "external_id": null}, {"id": 10846370003, "name": "Guilford College", "priority": 548, "external_id": null}, {"id": 10846371003, "name": "Gustavus Adolphus College", "priority": 549, "external_id": null}, {"id": 10846372003, "name": "Gwynedd-Mercy College", "priority": 550, "external_id": null}, {"id": 10846373003, "name": "Hamilton College", "priority": 551, "external_id": null}, {"id": 10846374003, "name": "Hamline University", "priority": 552, "external_id": null}, {"id": 10846375003, "name": "Hampden-Sydney College", "priority": 553, "external_id": null}, {"id": 10846376003, "name": "Hampshire College", "priority": 554, "external_id": null}, {"id": 10846377003, "name": "Hannibal-LaGrange University", "priority": 555, "external_id": null}, {"id": 10846378003, "name": "Hanover College", "priority": 556, "external_id": null}, {"id": 10846379003, "name": "Hardin-Simmons University", "priority": 557, "external_id": null}, {"id": 10846380003, "name": "Harding University", "priority": 558, "external_id": null}, {"id": 10846381003, "name": "Harrington College of Design", "priority": 559, "external_id": null}, {"id": 10846382003, "name": "Harris-Stowe State University", "priority": 560, "external_id": null}, {"id": 10846383003, "name": "Harrisburg University of Science and Technology", "priority": 561, "external_id": null}, {"id": 10846384003, "name": "Hartwick College", "priority": 562, "external_id": null}, {"id": 10846385003, "name": "Harvey Mudd College", "priority": 563, "external_id": null}, {"id": 10846386003, "name": "Haskell Indian Nations University", "priority": 564, "external_id": null}, {"id": 10846387003, "name": "Hastings College", "priority": 565, "external_id": null}, {"id": 10846388003, "name": "Haverford College", "priority": 566, "external_id": null}, {"id": 10846389003, "name": "Hawaii Pacific University", "priority": 567, "external_id": null}, {"id": 10846390003, "name": "Hebrew Theological College", "priority": 568, "external_id": null}, {"id": 10846391003, "name": "Heidelberg University", "priority": 569, "external_id": null}, {"id": 10846392003, "name": "Hellenic College", "priority": 570, "external_id": null}, {"id": 10846393003, "name": "Henderson State University", "priority": 571, "external_id": null}, {"id": 10846394003, "name": "Hendrix College", "priority": 572, "external_id": null}, {"id": 10846395003, "name": "Heritage University", "priority": 573, "external_id": null}, {"id": 10846396003, "name": "Herzing University", "priority": 574, "external_id": null}, {"id": 10846397003, "name": "Hesser College", "priority": 575, "external_id": null}, {"id": 10846398003, "name": "High Point University", "priority": 576, "external_id": null}, {"id": 10846399003, "name": "Hilbert College", "priority": 577, "external_id": null}, {"id": 10846400003, "name": "Hillsdale College", "priority": 578, "external_id": null}, {"id": 10846401003, "name": "Hiram College", "priority": 579, "external_id": null}, {"id": 10846402003, "name": "Hobart and William Smith Colleges", "priority": 580, "external_id": null}, {"id": 10846403003, "name": "Hodges University", "priority": 581, "external_id": null}, {"id": 10846404003, "name": "Hofstra University", "priority": 582, "external_id": null}, {"id": 10846405003, "name": "Hollins University", "priority": 583, "external_id": null}, {"id": 10846406003, "name": "Holy Apostles College and Seminary", "priority": 584, "external_id": null}, {"id": 10846407003, "name": "Indiana State University", "priority": 585, "external_id": null}, {"id": 10846408003, "name": "Holy Family University", "priority": 586, "external_id": null}, {"id": 10846409003, "name": "Holy Names University", "priority": 587, "external_id": null}, {"id": 10846410003, "name": "Hood College", "priority": 588, "external_id": null}, {"id": 10846411003, "name": "Hope College", "priority": 589, "external_id": null}, {"id": 10846412003, "name": "Hope International University", "priority": 590, "external_id": null}, {"id": 10846413003, "name": "Houghton College", "priority": 591, "external_id": null}, {"id": 10846414003, "name": "Howard Payne University", "priority": 592, "external_id": null}, {"id": 10846415003, "name": "Hult International Business School", "priority": 593, "external_id": null}, {"id": 10846416003, "name": "Humboldt State University", "priority": 594, "external_id": null}, {"id": 10846417003, "name": "Humphreys College", "priority": 595, "external_id": null}, {"id": 10846418003, "name": "Huntingdon College", "priority": 596, "external_id": null}, {"id": 10846419003, "name": "Huntington University", "priority": 597, "external_id": null}, {"id": 10846420003, "name": "Husson University", "priority": 598, "external_id": null}, {"id": 10846421003, "name": "Huston-Tillotson University", "priority": 599, "external_id": null}, {"id": 10846422003, "name": "Illinois College", "priority": 600, "external_id": null}, {"id": 10846423003, "name": "Illinois Institute of Art at Chicago", "priority": 601, "external_id": null}, {"id": 10846424003, "name": "Illinois Institute of Technology", "priority": 602, "external_id": null}, {"id": 10846425003, "name": "Illinois Wesleyan University", "priority": 603, "external_id": null}, {"id": 10846426003, "name": "Immaculata University", "priority": 604, "external_id": null}, {"id": 10846427003, "name": "Indian River State College", "priority": 605, "external_id": null}, {"id": 10846428003, "name": "Indiana Institute of Technology", "priority": 606, "external_id": null}, {"id": 10846429003, "name": "Indiana University East", "priority": 607, "external_id": null}, {"id": 10846430003, "name": "Indiana University Northwest", "priority": 608, "external_id": null}, {"id": 10846431003, "name": "Indiana University of Pennsylvania", "priority": 609, "external_id": null}, {"id": 10846432003, "name": "Indiana University Southeast", "priority": 610, "external_id": null}, {"id": 10846433003, "name": "Illinois State University", "priority": 611, "external_id": null}, {"id": 10846434003, "name": "Indiana University - Bloomington", "priority": 612, "external_id": null}, {"id": 10846435003, "name": "Davidson College", "priority": 613, "external_id": null}, {"id": 10846436003, "name": "Idaho State University", "priority": 614, "external_id": null}, {"id": 10846437003, "name": "Harvard University", "priority": 615, "external_id": null}, {"id": 10846438003, "name": "Howard University", "priority": 616, "external_id": null}, {"id": 10846439003, "name": "Houston Baptist University", "priority": 617, "external_id": null}, {"id": 10846440003, "name": "Indiana University - Kokomo", "priority": 618, "external_id": null}, {"id": 10846441003, "name": "Indiana University - South Bend", "priority": 619, "external_id": null}, {"id": 10846442003, "name": "Indiana University-Purdue University - Fort Wayne", "priority": 620, "external_id": null}, {"id": 10846443003, "name": "Indiana University-Purdue University - Indianapolis", "priority": 621, "external_id": null}, {"id": 10846444003, "name": "Indiana Wesleyan University", "priority": 622, "external_id": null}, {"id": 10846445003, "name": "Institute of American Indian and Alaska Native Culture and Arts Development", "priority": 623, "external_id": null}, {"id": 10846446003, "name": "Inter American University of Puerto Rico - Aguadilla", "priority": 624, "external_id": null}, {"id": 10846447003, "name": "Inter American University of Puerto Rico - Arecibo", "priority": 625, "external_id": null}, {"id": 10846448003, "name": "Inter American University of Puerto Rico - Barranquitas", "priority": 626, "external_id": null}, {"id": 10846449003, "name": "Inter American University of Puerto Rico - Bayamon", "priority": 627, "external_id": null}, {"id": 10846450003, "name": "Inter American University of Puerto Rico - Fajardo", "priority": 628, "external_id": null}, {"id": 10846451003, "name": "Inter American University of Puerto Rico - Guayama", "priority": 629, "external_id": null}, {"id": 10846452003, "name": "Inter American University of Puerto Rico - Metropolitan Campus", "priority": 630, "external_id": null}, {"id": 10846453003, "name": "Inter American University of Puerto Rico - Ponce", "priority": 631, "external_id": null}, {"id": 10846454003, "name": "Inter American University of Puerto Rico - San German", "priority": 632, "external_id": null}, {"id": 10846455003, "name": "International College of the Cayman Islands", "priority": 633, "external_id": null}, {"id": 10846456003, "name": "Iona College", "priority": 634, "external_id": null}, {"id": 10846457003, "name": "Iowa Wesleyan College", "priority": 635, "external_id": null}, {"id": 10846458003, "name": "Ithaca College", "priority": 636, "external_id": null}, {"id": 10846459003, "name": "Jarvis Christian College", "priority": 637, "external_id": null}, {"id": 10846460003, "name": "Jewish Theological Seminary of America", "priority": 638, "external_id": null}, {"id": 10846461003, "name": "John Brown University", "priority": 639, "external_id": null}, {"id": 10846462003, "name": "John Carroll University", "priority": 640, "external_id": null}, {"id": 10846463003, "name": "John F. Kennedy University", "priority": 641, "external_id": null}, {"id": 10846464003, "name": "Johns Hopkins University", "priority": 642, "external_id": null}, {"id": 10846465003, "name": "Johnson & Wales University", "priority": 643, "external_id": null}, {"id": 10846466003, "name": "Johnson C. Smith University", "priority": 644, "external_id": null}, {"id": 10846467003, "name": "Johnson State College", "priority": 645, "external_id": null}, {"id": 10846468003, "name": "Johnson University", "priority": 646, "external_id": null}, {"id": 10846469003, "name": "Jones International University", "priority": 647, "external_id": null}, {"id": 10846470003, "name": "Judson College", "priority": 648, "external_id": null}, {"id": 10846471003, "name": "Judson University", "priority": 649, "external_id": null}, {"id": 10846472003, "name": "Juilliard School", "priority": 650, "external_id": null}, {"id": 10846473003, "name": "Juniata College", "priority": 651, "external_id": null}, {"id": 10846474003, "name": "Kalamazoo College", "priority": 652, "external_id": null}, {"id": 10846475003, "name": "Kansas City Art Institute", "priority": 653, "external_id": null}, {"id": 10846476003, "name": "Kansas Wesleyan University", "priority": 654, "external_id": null}, {"id": 10846477003, "name": "Kaplan University", "priority": 655, "external_id": null}, {"id": 10846478003, "name": "Kean University", "priority": 656, "external_id": null}, {"id": 10846479003, "name": "Keene State College", "priority": 657, "external_id": null}, {"id": 10846480003, "name": "Keiser University", "priority": 658, "external_id": null}, {"id": 10846481003, "name": "Kendall College", "priority": 659, "external_id": null}, {"id": 10846482003, "name": "Kennesaw State University", "priority": 660, "external_id": null}, {"id": 10846483003, "name": "Kentucky Christian University", "priority": 661, "external_id": null}, {"id": 10846484003, "name": "Kentucky State University", "priority": 662, "external_id": null}, {"id": 10846485003, "name": "Kentucky Wesleyan College", "priority": 663, "external_id": null}, {"id": 10846486003, "name": "Kenyon College", "priority": 664, "external_id": null}, {"id": 10846487003, "name": "Kettering College", "priority": 665, "external_id": null}, {"id": 10846488003, "name": "Kettering University", "priority": 666, "external_id": null}, {"id": 10846489003, "name": "Keuka College", "priority": 667, "external_id": null}, {"id": 10846490003, "name": "Keystone College", "priority": 668, "external_id": null}, {"id": 10846491003, "name": "King University", "priority": 669, "external_id": null}, {"id": 10846492003, "name": "King's College", "priority": 670, "external_id": null}, {"id": 10846493003, "name": "Knox College", "priority": 671, "external_id": null}, {"id": 10846494003, "name": "Kutztown University of Pennsylvania", "priority": 672, "external_id": null}, {"id": 10846495003, "name": "Kuyper College", "priority": 673, "external_id": null}, {"id": 10846496003, "name": "La Roche College", "priority": 674, "external_id": null}, {"id": 10846497003, "name": "La Salle University", "priority": 675, "external_id": null}, {"id": 10846498003, "name": "La Sierra University", "priority": 676, "external_id": null}, {"id": 10846499003, "name": "LaGrange College", "priority": 677, "external_id": null}, {"id": 10846500003, "name": "Laguna College of Art and Design", "priority": 678, "external_id": null}, {"id": 10846501003, "name": "Lake Erie College", "priority": 679, "external_id": null}, {"id": 10846502003, "name": "Lake Forest College", "priority": 680, "external_id": null}, {"id": 10846503003, "name": "Lake Superior State University", "priority": 681, "external_id": null}, {"id": 10846504003, "name": "Lakeland College", "priority": 682, "external_id": null}, {"id": 10846505003, "name": "Lakeview College of Nursing", "priority": 683, "external_id": null}, {"id": 10846506003, "name": "Lancaster Bible College", "priority": 684, "external_id": null}, {"id": 10846507003, "name": "Lander University", "priority": 685, "external_id": null}, {"id": 10846508003, "name": "Lane College", "priority": 686, "external_id": null}, {"id": 10846509003, "name": "Langston University", "priority": 687, "external_id": null}, {"id": 10846510003, "name": "Lasell College", "priority": 688, "external_id": null}, {"id": 10846511003, "name": "Lawrence Technological University", "priority": 689, "external_id": null}, {"id": 10846512003, "name": "Lawrence University", "priority": 690, "external_id": null}, {"id": 10846513003, "name": "Le Moyne College", "priority": 691, "external_id": null}, {"id": 10846514003, "name": "Lebanon Valley College", "priority": 692, "external_id": null}, {"id": 10846515003, "name": "Lee University", "priority": 693, "external_id": null}, {"id": 10846516003, "name": "Lees-McRae College", "priority": 694, "external_id": null}, {"id": 10846517003, "name": "Kansas State University", "priority": 695, "external_id": null}, {"id": 10846518003, "name": "James Madison University", "priority": 696, "external_id": null}, {"id": 10846519003, "name": "Lafayette College", "priority": 697, "external_id": null}, {"id": 10846520003, "name": "Jacksonville University", "priority": 698, "external_id": null}, {"id": 10846521003, "name": "Kent State University", "priority": 699, "external_id": null}, {"id": 10846522003, "name": "Lamar University", "priority": 700, "external_id": null}, {"id": 10846523003, "name": "Jackson State University", "priority": 701, "external_id": null}, {"id": 10846524003, "name": "Lehigh University", "priority": 702, "external_id": null}, {"id": 10846525003, "name": "Jacksonville State University", "priority": 703, "external_id": null}, {"id": 10846526003, "name": "LeMoyne-Owen College", "priority": 704, "external_id": null}, {"id": 10846527003, "name": "Lenoir-Rhyne University", "priority": 705, "external_id": null}, {"id": 10846528003, "name": "Lesley University", "priority": 706, "external_id": null}, {"id": 10846529003, "name": "LeTourneau University", "priority": 707, "external_id": null}, {"id": 10846530003, "name": "Lewis & Clark College", "priority": 708, "external_id": null}, {"id": 10846531003, "name": "Lewis University", "priority": 709, "external_id": null}, {"id": 10846532003, "name": "Lewis-Clark State College", "priority": 710, "external_id": null}, {"id": 10846533003, "name": "Lexington College", "priority": 711, "external_id": null}, {"id": 10846534003, "name": "Life Pacific College", "priority": 712, "external_id": null}, {"id": 10846535003, "name": "Life University", "priority": 713, "external_id": null}, {"id": 10846536003, "name": "LIM College", "priority": 714, "external_id": null}, {"id": 10846537003, "name": "Limestone College", "priority": 715, "external_id": null}, {"id": 10846538003, "name": "Lincoln Christian University", "priority": 716, "external_id": null}, {"id": 10846539003, "name": "Lincoln College", "priority": 717, "external_id": null}, {"id": 10846540003, "name": "Lincoln Memorial University", "priority": 718, "external_id": null}, {"id": 10846541003, "name": "Lincoln University", "priority": 719, "external_id": null}, {"id": 10846542003, "name": "Lindenwood University", "priority": 720, "external_id": null}, {"id": 10846543003, "name": "Lindsey Wilson College", "priority": 721, "external_id": null}, {"id": 10846544003, "name": "Linfield College", "priority": 722, "external_id": null}, {"id": 10846545003, "name": "Lipscomb University", "priority": 723, "external_id": null}, {"id": 10846546003, "name": "LIU Post", "priority": 724, "external_id": null}, {"id": 10846547003, "name": "Livingstone College", "priority": 725, "external_id": null}, {"id": 10846548003, "name": "Lock Haven University of Pennsylvania", "priority": 726, "external_id": null}, {"id": 10846549003, "name": "Loma Linda University", "priority": 727, "external_id": null}, {"id": 10846550003, "name": "Longwood University", "priority": 728, "external_id": null}, {"id": 10846551003, "name": "Loras College", "priority": 729, "external_id": null}, {"id": 10846552003, "name": "Louisiana College", "priority": 730, "external_id": null}, {"id": 10846553003, "name": "Louisiana State University Health Sciences Center", "priority": 731, "external_id": null}, {"id": 10846554003, "name": "Louisiana State University - Alexandria", "priority": 732, "external_id": null}, {"id": 10846555003, "name": "Louisiana State University - Shreveport", "priority": 733, "external_id": null}, {"id": 10846556003, "name": "Lourdes University", "priority": 734, "external_id": null}, {"id": 10846557003, "name": "Loyola Marymount University", "priority": 735, "external_id": null}, {"id": 10846558003, "name": "Loyola University Chicago", "priority": 736, "external_id": null}, {"id": 10846559003, "name": "Loyola University Maryland", "priority": 737, "external_id": null}, {"id": 10846560003, "name": "Loyola University New Orleans", "priority": 738, "external_id": null}, {"id": 10846561003, "name": "Lubbock Christian University", "priority": 739, "external_id": null}, {"id": 10846562003, "name": "Luther College", "priority": 740, "external_id": null}, {"id": 10846563003, "name": "Lycoming College", "priority": 741, "external_id": null}, {"id": 10846564003, "name": "Lyme Academy College of Fine Arts", "priority": 742, "external_id": null}, {"id": 10846565003, "name": "Lynchburg College", "priority": 743, "external_id": null}, {"id": 10846566003, "name": "Lyndon State College", "priority": 744, "external_id": null}, {"id": 10846567003, "name": "Lynn University", "priority": 745, "external_id": null}, {"id": 10846568003, "name": "Lyon College", "priority": 746, "external_id": null}, {"id": 10846569003, "name": "Macalester College", "priority": 747, "external_id": null}, {"id": 10846570003, "name": "MacMurray College", "priority": 748, "external_id": null}, {"id": 10846571003, "name": "Madonna University", "priority": 749, "external_id": null}, {"id": 10846572003, "name": "Maharishi University of Management", "priority": 750, "external_id": null}, {"id": 10846573003, "name": "Maine College of Art", "priority": 751, "external_id": null}, {"id": 10846574003, "name": "Maine Maritime Academy", "priority": 752, "external_id": null}, {"id": 10846575003, "name": "Malone University", "priority": 753, "external_id": null}, {"id": 10846576003, "name": "Manchester University", "priority": 754, "external_id": null}, {"id": 10846577003, "name": "Manhattan Christian College", "priority": 755, "external_id": null}, {"id": 10846578003, "name": "Manhattan College", "priority": 756, "external_id": null}, {"id": 10846579003, "name": "Manhattan School of Music", "priority": 757, "external_id": null}, {"id": 10846580003, "name": "Manhattanville College", "priority": 758, "external_id": null}, {"id": 10846581003, "name": "Mansfield University of Pennsylvania", "priority": 759, "external_id": null}, {"id": 10846582003, "name": "Maranatha Baptist Bible College", "priority": 760, "external_id": null}, {"id": 10846583003, "name": "Marian University", "priority": 761, "external_id": null}, {"id": 10846584003, "name": "Marietta College", "priority": 762, "external_id": null}, {"id": 10846585003, "name": "Marlboro College", "priority": 763, "external_id": null}, {"id": 10846586003, "name": "Marquette University", "priority": 764, "external_id": null}, {"id": 10846587003, "name": "Mars Hill University", "priority": 765, "external_id": null}, {"id": 10846588003, "name": "Martin Luther College", "priority": 766, "external_id": null}, {"id": 10846589003, "name": "Martin Methodist College", "priority": 767, "external_id": null}, {"id": 10846590003, "name": "Martin University", "priority": 768, "external_id": null}, {"id": 10846591003, "name": "Mary Baldwin College", "priority": 769, "external_id": null}, {"id": 10846592003, "name": "Marygrove College", "priority": 770, "external_id": null}, {"id": 10846593003, "name": "Maryland Institute College of Art", "priority": 771, "external_id": null}, {"id": 10846594003, "name": "Marylhurst University", "priority": 772, "external_id": null}, {"id": 10846595003, "name": "Marymount Manhattan College", "priority": 773, "external_id": null}, {"id": 10846596003, "name": "Marymount University", "priority": 774, "external_id": null}, {"id": 10846597003, "name": "Maryville College", "priority": 775, "external_id": null}, {"id": 10846598003, "name": "Maryville University of St. Louis", "priority": 776, "external_id": null}, {"id": 10846599003, "name": "Marywood University", "priority": 777, "external_id": null}, {"id": 10846600003, "name": "Massachusetts College of Art and Design", "priority": 778, "external_id": null}, {"id": 10846601003, "name": "Massachusetts College of Liberal Arts", "priority": 779, "external_id": null}, {"id": 10846602003, "name": "Massachusetts College of Pharmacy and Health Sciences", "priority": 780, "external_id": null}, {"id": 10846603003, "name": "Massachusetts Institute of Technology", "priority": 781, "external_id": null}, {"id": 10846604003, "name": "Massachusetts Maritime Academy", "priority": 782, "external_id": null}, {"id": 10846605003, "name": "Master's College and Seminary", "priority": 783, "external_id": null}, {"id": 10846606003, "name": "Mayville State University", "priority": 784, "external_id": null}, {"id": 10846607003, "name": "McDaniel College", "priority": 785, "external_id": null}, {"id": 10846608003, "name": "McGill University", "priority": 786, "external_id": null}, {"id": 10846609003, "name": "McKendree University", "priority": 787, "external_id": null}, {"id": 10846610003, "name": "McMurry University", "priority": 788, "external_id": null}, {"id": 10846611003, "name": "McPherson College", "priority": 789, "external_id": null}, {"id": 10846612003, "name": "Medaille College", "priority": 790, "external_id": null}, {"id": 10846613003, "name": "Marist College", "priority": 791, "external_id": null}, {"id": 10846614003, "name": "McNeese State University", "priority": 792, "external_id": null}, {"id": 10846615003, "name": "Louisiana Tech University", "priority": 793, "external_id": null}, {"id": 10846616003, "name": "Marshall University", "priority": 794, "external_id": null}, {"id": 10846617003, "name": "Medical University of South Carolina", "priority": 795, "external_id": null}, {"id": 10846618003, "name": "Memorial University of Newfoundland", "priority": 796, "external_id": null}, {"id": 10846619003, "name": "Memphis College of Art", "priority": 797, "external_id": null}, {"id": 10846620003, "name": "Menlo College", "priority": 798, "external_id": null}, {"id": 10846621003, "name": "Mercy College", "priority": 799, "external_id": null}, {"id": 10846622003, "name": "Mercy College of Health Sciences", "priority": 800, "external_id": null}, {"id": 10846623003, "name": "Mercy College of Ohio", "priority": 801, "external_id": null}, {"id": 10846624003, "name": "Mercyhurst University", "priority": 802, "external_id": null}, {"id": 10846625003, "name": "Meredith College", "priority": 803, "external_id": null}, {"id": 10846626003, "name": "Merrimack College", "priority": 804, "external_id": null}, {"id": 10846627003, "name": "Messiah College", "priority": 805, "external_id": null}, {"id": 10846628003, "name": "Methodist University", "priority": 806, "external_id": null}, {"id": 10846629003, "name": "Metropolitan College of New York", "priority": 807, "external_id": null}, {"id": 10846630003, "name": "Metropolitan State University", "priority": 808, "external_id": null}, {"id": 10846631003, "name": "Metropolitan State University of Denver", "priority": 809, "external_id": null}, {"id": 10846632003, "name": "Miami Dade College", "priority": 810, "external_id": null}, {"id": 10846633003, "name": "Miami International University of Art & Design", "priority": 811, "external_id": null}, {"id": 10846634003, "name": "Michigan Technological University", "priority": 812, "external_id": null}, {"id": 10846635003, "name": "Mid-America Christian University", "priority": 813, "external_id": null}, {"id": 10846636003, "name": "Mid-Atlantic Christian University", "priority": 814, "external_id": null}, {"id": 10846637003, "name": "Mid-Continent University", "priority": 815, "external_id": null}, {"id": 10846638003, "name": "MidAmerica Nazarene University", "priority": 816, "external_id": null}, {"id": 10846639003, "name": "Middle Georgia State College", "priority": 817, "external_id": null}, {"id": 10846640003, "name": "Middlebury College", "priority": 818, "external_id": null}, {"id": 10846641003, "name": "Midland College", "priority": 819, "external_id": null}, {"id": 10846642003, "name": "Midland University", "priority": 820, "external_id": null}, {"id": 10846643003, "name": "Midstate College", "priority": 821, "external_id": null}, {"id": 10846644003, "name": "Midway College", "priority": 822, "external_id": null}, {"id": 10846645003, "name": "Midwestern State University", "priority": 823, "external_id": null}, {"id": 10846646003, "name": "Miles College", "priority": 824, "external_id": null}, {"id": 10846647003, "name": "Millersville University of Pennsylvania", "priority": 825, "external_id": null}, {"id": 10846648003, "name": "Milligan College", "priority": 826, "external_id": null}, {"id": 10846649003, "name": "Millikin University", "priority": 827, "external_id": null}, {"id": 10846650003, "name": "Mills College", "priority": 828, "external_id": null}, {"id": 10846651003, "name": "Millsaps College", "priority": 829, "external_id": null}, {"id": 10846652003, "name": "Milwaukee Institute of Art and Design", "priority": 830, "external_id": null}, {"id": 10846653003, "name": "Milwaukee School of Engineering", "priority": 831, "external_id": null}, {"id": 10846654003, "name": "Minneapolis College of Art and Design", "priority": 832, "external_id": null}, {"id": 10846655003, "name": "Minnesota State University - Mankato", "priority": 833, "external_id": null}, {"id": 10846656003, "name": "Minnesota State University - Moorhead", "priority": 834, "external_id": null}, {"id": 10846657003, "name": "Minot State University", "priority": 835, "external_id": null}, {"id": 10846658003, "name": "Misericordia University", "priority": 836, "external_id": null}, {"id": 10846659003, "name": "Mississippi College", "priority": 837, "external_id": null}, {"id": 10846660003, "name": "Mississippi University for Women", "priority": 838, "external_id": null}, {"id": 10846661003, "name": "Missouri Baptist University", "priority": 839, "external_id": null}, {"id": 10846662003, "name": "Missouri Southern State University", "priority": 840, "external_id": null}, {"id": 10846663003, "name": "Missouri University of Science & Technology", "priority": 841, "external_id": null}, {"id": 10846664003, "name": "Missouri Valley College", "priority": 842, "external_id": null}, {"id": 10846665003, "name": "Missouri Western State University", "priority": 843, "external_id": null}, {"id": 10846666003, "name": "Mitchell College", "priority": 844, "external_id": null}, {"id": 10846667003, "name": "Molloy College", "priority": 845, "external_id": null}, {"id": 10846668003, "name": "Monmouth College", "priority": 846, "external_id": null}, {"id": 10846669003, "name": "Monroe College", "priority": 847, "external_id": null}, {"id": 10846670003, "name": "Montana State University - Billings", "priority": 848, "external_id": null}, {"id": 10846671003, "name": "Montana State University - Northern", "priority": 849, "external_id": null}, {"id": 10846672003, "name": "Montana Tech of the University of Montana", "priority": 850, "external_id": null}, {"id": 10846673003, "name": "Montclair State University", "priority": 851, "external_id": null}, {"id": 10846674003, "name": "Monterrey Institute of Technology and Higher Education - Monterrey", "priority": 852, "external_id": null}, {"id": 10846675003, "name": "Montreat College", "priority": 853, "external_id": null}, {"id": 10846676003, "name": "Montserrat College of Art", "priority": 854, "external_id": null}, {"id": 10846677003, "name": "Moody Bible Institute", "priority": 855, "external_id": null}, {"id": 10846678003, "name": "Moore College of Art & Design", "priority": 856, "external_id": null}, {"id": 10846679003, "name": "Moravian College", "priority": 857, "external_id": null}, {"id": 10846680003, "name": "Morehouse College", "priority": 858, "external_id": null}, {"id": 10846681003, "name": "Morningside College", "priority": 859, "external_id": null}, {"id": 10846682003, "name": "Morris College", "priority": 860, "external_id": null}, {"id": 10846683003, "name": "Morrisville State College", "priority": 861, "external_id": null}, {"id": 10846684003, "name": "Mount Aloysius College", "priority": 862, "external_id": null}, {"id": 10846685003, "name": "Mount Angel Seminary", "priority": 863, "external_id": null}, {"id": 10846686003, "name": "Mount Carmel College of Nursing", "priority": 864, "external_id": null}, {"id": 10846687003, "name": "Mount Holyoke College", "priority": 865, "external_id": null}, {"id": 10846688003, "name": "Mount Ida College", "priority": 866, "external_id": null}, {"id": 10846689003, "name": "Mount Marty College", "priority": 867, "external_id": null}, {"id": 10846690003, "name": "Mount Mary University", "priority": 868, "external_id": null}, {"id": 10846691003, "name": "Mount Mercy University", "priority": 869, "external_id": null}, {"id": 10846692003, "name": "Mount Olive College", "priority": 870, "external_id": null}, {"id": 10846693003, "name": "Mississippi State University", "priority": 871, "external_id": null}, {"id": 10846694003, "name": "Montana State University", "priority": 872, "external_id": null}, {"id": 10846695003, "name": "Mississippi Valley State University", "priority": 873, "external_id": null}, {"id": 10846696003, "name": "Monmouth University", "priority": 874, "external_id": null}, {"id": 10846697003, "name": "Morehead State University", "priority": 875, "external_id": null}, {"id": 10846698003, "name": "Miami University - Oxford", "priority": 876, "external_id": null}, {"id": 10846699003, "name": "Morgan State University", "priority": 877, "external_id": null}, {"id": 10846700003, "name": "Missouri State University", "priority": 878, "external_id": null}, {"id": 10846701003, "name": "Michigan State University", "priority": 879, "external_id": null}, {"id": 10846702003, "name": "Mount St. Mary College", "priority": 880, "external_id": null}, {"id": 10846703003, "name": "Mount St. Mary's College", "priority": 881, "external_id": null}, {"id": 10846704003, "name": "Mount St. Mary's University", "priority": 882, "external_id": null}, {"id": 10846705003, "name": "Mount Vernon Nazarene University", "priority": 883, "external_id": null}, {"id": 10846706003, "name": "Muhlenberg College", "priority": 884, "external_id": null}, {"id": 10846707003, "name": "Multnomah University", "priority": 885, "external_id": null}, {"id": 10846708003, "name": "Muskingum University", "priority": 886, "external_id": null}, {"id": 10846709003, "name": "Naropa University", "priority": 887, "external_id": null}, {"id": 10846710003, "name": "National American University", "priority": 888, "external_id": null}, {"id": 10846711003, "name": "National Graduate School of Quality Management", "priority": 889, "external_id": null}, {"id": 10846712003, "name": "National Hispanic University", "priority": 890, "external_id": null}, {"id": 10846713003, "name": "National Labor College", "priority": 891, "external_id": null}, {"id": 10846714003, "name": "National University", "priority": 892, "external_id": null}, {"id": 10846715003, "name": "National-Louis University", "priority": 893, "external_id": null}, {"id": 10846716003, "name": "Nazarene Bible College", "priority": 894, "external_id": null}, {"id": 10846717003, "name": "Nazareth College", "priority": 895, "external_id": null}, {"id": 10846718003, "name": "Nebraska Methodist College", "priority": 896, "external_id": null}, {"id": 10846719003, "name": "Nebraska Wesleyan University", "priority": 897, "external_id": null}, {"id": 10846720003, "name": "Neumann University", "priority": 898, "external_id": null}, {"id": 10846721003, "name": "Nevada State College", "priority": 899, "external_id": null}, {"id": 10846722003, "name": "New College of Florida", "priority": 900, "external_id": null}, {"id": 10846723003, "name": "New England College", "priority": 901, "external_id": null}, {"id": 10846724003, "name": "New England Conservatory of Music", "priority": 902, "external_id": null}, {"id": 10846725003, "name": "New England Institute of Art", "priority": 903, "external_id": null}, {"id": 10846726003, "name": "New England Institute of Technology", "priority": 904, "external_id": null}, {"id": 10846727003, "name": "New Jersey City University", "priority": 905, "external_id": null}, {"id": 10846728003, "name": "New Jersey Institute of Technology", "priority": 906, "external_id": null}, {"id": 10846729003, "name": "New Mexico Highlands University", "priority": 907, "external_id": null}, {"id": 10846730003, "name": "New Mexico Institute of Mining and Technology", "priority": 908, "external_id": null}, {"id": 10846731003, "name": "New Orleans Baptist Theological Seminary", "priority": 909, "external_id": null}, {"id": 10846732003, "name": "New School", "priority": 910, "external_id": null}, {"id": 10846733003, "name": "New York Institute of Technology", "priority": 911, "external_id": null}, {"id": 10846734003, "name": "New York University", "priority": 912, "external_id": null}, {"id": 10846735003, "name": "Newberry College", "priority": 913, "external_id": null}, {"id": 10846736003, "name": "Newbury College", "priority": 914, "external_id": null}, {"id": 10846737003, "name": "Newman University", "priority": 915, "external_id": null}, {"id": 10846738003, "name": "Niagara University", "priority": 916, "external_id": null}, {"id": 10846739003, "name": "Nichols College", "priority": 917, "external_id": null}, {"id": 10846740003, "name": "North Carolina Wesleyan College", "priority": 918, "external_id": null}, {"id": 10846741003, "name": "North Central College", "priority": 919, "external_id": null}, {"id": 10846742003, "name": "North Central University", "priority": 920, "external_id": null}, {"id": 10846743003, "name": "North Greenville University", "priority": 921, "external_id": null}, {"id": 10846744003, "name": "North Park University", "priority": 922, "external_id": null}, {"id": 10846745003, "name": "Northcentral University", "priority": 923, "external_id": null}, {"id": 10846746003, "name": "Northeastern Illinois University", "priority": 924, "external_id": null}, {"id": 10846747003, "name": "Northeastern State University", "priority": 925, "external_id": null}, {"id": 10846748003, "name": "Northeastern University", "priority": 926, "external_id": null}, {"id": 10846749003, "name": "Northern Kentucky University", "priority": 927, "external_id": null}, {"id": 10846750003, "name": "Northern Michigan University", "priority": 928, "external_id": null}, {"id": 10846751003, "name": "Northern New Mexico College", "priority": 929, "external_id": null}, {"id": 10846752003, "name": "Northern State University", "priority": 930, "external_id": null}, {"id": 10846753003, "name": "Northland College", "priority": 931, "external_id": null}, {"id": 10846754003, "name": "Northwest Christian University", "priority": 932, "external_id": null}, {"id": 10846755003, "name": "Northwest Florida State College", "priority": 933, "external_id": null}, {"id": 10846756003, "name": "Northwest Missouri State University", "priority": 934, "external_id": null}, {"id": 10846757003, "name": "Northwest Nazarene University", "priority": 935, "external_id": null}, {"id": 10846758003, "name": "Northwest University", "priority": 936, "external_id": null}, {"id": 10846759003, "name": "Northwestern College", "priority": 937, "external_id": null}, {"id": 10846760003, "name": "Northwestern Health Sciences University", "priority": 938, "external_id": null}, {"id": 10846761003, "name": "Northwestern Oklahoma State University", "priority": 939, "external_id": null}, {"id": 10846762003, "name": "Northwood University", "priority": 940, "external_id": null}, {"id": 10846763003, "name": "Norwich University", "priority": 941, "external_id": null}, {"id": 10846764003, "name": "Notre Dame College of Ohio", "priority": 942, "external_id": null}, {"id": 10846765003, "name": "Notre Dame de Namur University", "priority": 943, "external_id": null}, {"id": 10846766003, "name": "Notre Dame of Maryland University", "priority": 944, "external_id": null}, {"id": 10846767003, "name": "Nova Scotia College of Art and Design", "priority": 945, "external_id": null}, {"id": 10846768003, "name": "Nova Southeastern University", "priority": 946, "external_id": null}, {"id": 10846769003, "name": "Nyack College", "priority": 947, "external_id": null}, {"id": 10846770003, "name": "Oakland City University", "priority": 948, "external_id": null}, {"id": 10846771003, "name": "Oakland University", "priority": 949, "external_id": null}, {"id": 10846772003, "name": "Oakwood University", "priority": 950, "external_id": null}, {"id": 10846773003, "name": "Oberlin College", "priority": 951, "external_id": null}, {"id": 10846774003, "name": "Occidental College", "priority": 952, "external_id": null}, {"id": 10846775003, "name": "Oglala Lakota College", "priority": 953, "external_id": null}, {"id": 10846776003, "name": "North Carolina A&T State University", "priority": 954, "external_id": null}, {"id": 10846777003, "name": "Northern Illinois University", "priority": 955, "external_id": null}, {"id": 10846778003, "name": "North Dakota State University", "priority": 956, "external_id": null}, {"id": 10846779003, "name": "Nicholls State University", "priority": 957, "external_id": null}, {"id": 10846780003, "name": "North Carolina Central University", "priority": 958, "external_id": null}, {"id": 10846781003, "name": "Norfolk State University", "priority": 959, "external_id": null}, {"id": 10846782003, "name": "Northwestern State University of Louisiana", "priority": 960, "external_id": null}, {"id": 10846783003, "name": "Northern Arizona University", "priority": 961, "external_id": null}, {"id": 10846784003, "name": "North Carolina State University - Raleigh", "priority": 962, "external_id": null}, {"id": 10846785003, "name": "Northwestern University", "priority": 963, "external_id": null}, {"id": 10846786003, "name": "Oglethorpe University", "priority": 964, "external_id": null}, {"id": 10846787003, "name": "Ohio Christian University", "priority": 965, "external_id": null}, {"id": 10846788003, "name": "Ohio Dominican University", "priority": 966, "external_id": null}, {"id": 10846789003, "name": "Ohio Northern University", "priority": 967, "external_id": null}, {"id": 10846790003, "name": "Ohio Valley University", "priority": 968, "external_id": null}, {"id": 10846791003, "name": "Ohio Wesleyan University", "priority": 969, "external_id": null}, {"id": 10846792003, "name": "Oklahoma Baptist University", "priority": 970, "external_id": null}, {"id": 10846793003, "name": "Oklahoma Christian University", "priority": 971, "external_id": null}, {"id": 10846794003, "name": "Oklahoma City University", "priority": 972, "external_id": null}, {"id": 10846795003, "name": "Oklahoma Panhandle State University", "priority": 973, "external_id": null}, {"id": 10846796003, "name": "Oklahoma State University Institute of Technology - Okmulgee", "priority": 974, "external_id": null}, {"id": 10846797003, "name": "Oklahoma State University - Oklahoma City", "priority": 975, "external_id": null}, {"id": 10846798003, "name": "Oklahoma Wesleyan University", "priority": 976, "external_id": null}, {"id": 10846799003, "name": "Olivet College", "priority": 977, "external_id": null}, {"id": 10846800003, "name": "Olivet Nazarene University", "priority": 978, "external_id": null}, {"id": 10846801003, "name": "Olympic College", "priority": 979, "external_id": null}, {"id": 10846802003, "name": "Oral Roberts University", "priority": 980, "external_id": null}, {"id": 10846803003, "name": "Oregon College of Art and Craft", "priority": 981, "external_id": null}, {"id": 10846804003, "name": "Oregon Health and Science University", "priority": 982, "external_id": null}, {"id": 10846805003, "name": "Oregon Institute of Technology", "priority": 983, "external_id": null}, {"id": 10846806003, "name": "Otis College of Art and Design", "priority": 984, "external_id": null}, {"id": 10846807003, "name": "Ottawa University", "priority": 985, "external_id": null}, {"id": 10846808003, "name": "Otterbein University", "priority": 986, "external_id": null}, {"id": 10846809003, "name": "Ouachita Baptist University", "priority": 987, "external_id": null}, {"id": 10846810003, "name": "Our Lady of Holy Cross College", "priority": 988, "external_id": null}, {"id": 10846811003, "name": "Our Lady of the Lake College", "priority": 989, "external_id": null}, {"id": 10846812003, "name": "Our Lady of the Lake University", "priority": 990, "external_id": null}, {"id": 10846813003, "name": "Pace University", "priority": 991, "external_id": null}, {"id": 10846814003, "name": "Pacific Lutheran University", "priority": 992, "external_id": null}, {"id": 10846815003, "name": "Pacific Northwest College of Art", "priority": 993, "external_id": null}, {"id": 10846816003, "name": "Pacific Oaks College", "priority": 994, "external_id": null}, {"id": 10846817003, "name": "Pacific Union College", "priority": 995, "external_id": null}, {"id": 10846818003, "name": "Pacific University", "priority": 996, "external_id": null}, {"id": 10846819003, "name": "Paine College", "priority": 997, "external_id": null}, {"id": 10846820003, "name": "Palm Beach Atlantic University", "priority": 998, "external_id": null}, {"id": 10846821003, "name": "Palmer College of Chiropractic", "priority": 999, "external_id": null}, {"id": 10846822003, "name": "Park University", "priority": 1000, "external_id": null}, {"id": 10846823003, "name": "Parker University", "priority": 1001, "external_id": null}, {"id": 10846824003, "name": "Patten University", "priority": 1002, "external_id": null}, {"id": 10846825003, "name": "Paul Smith's College", "priority": 1003, "external_id": null}, {"id": 10846826003, "name": "Peirce College", "priority": 1004, "external_id": null}, {"id": 10846827003, "name": "Peninsula College", "priority": 1005, "external_id": null}, {"id": 10846828003, "name": "Pennsylvania College of Art and Design", "priority": 1006, "external_id": null}, {"id": 10846829003, "name": "Pennsylvania College of Technology", "priority": 1007, "external_id": null}, {"id": 10846830003, "name": "Pennsylvania State University - Erie, The Behrend College", "priority": 1008, "external_id": null}, {"id": 10846831003, "name": "Pennsylvania State University - Harrisburg", "priority": 1009, "external_id": null}, {"id": 10846832003, "name": "Pepperdine University", "priority": 1010, "external_id": null}, {"id": 10846833003, "name": "Peru State College", "priority": 1011, "external_id": null}, {"id": 10846834003, "name": "Pfeiffer University", "priority": 1012, "external_id": null}, {"id": 10846835003, "name": "Philadelphia University", "priority": 1013, "external_id": null}, {"id": 10846836003, "name": "Philander Smith College", "priority": 1014, "external_id": null}, {"id": 10846837003, "name": "Piedmont College", "priority": 1015, "external_id": null}, {"id": 10846838003, "name": "Pine Manor College", "priority": 1016, "external_id": null}, {"id": 10846839003, "name": "Pittsburg State University", "priority": 1017, "external_id": null}, {"id": 10846840003, "name": "Pitzer College", "priority": 1018, "external_id": null}, {"id": 10846841003, "name": "Plaza College", "priority": 1019, "external_id": null}, {"id": 10846842003, "name": "Plymouth State University", "priority": 1020, "external_id": null}, {"id": 10846843003, "name": "Point Loma Nazarene University", "priority": 1021, "external_id": null}, {"id": 10846844003, "name": "Point Park University", "priority": 1022, "external_id": null}, {"id": 10846845003, "name": "Point University", "priority": 1023, "external_id": null}, {"id": 10846846003, "name": "Polytechnic Institute of New York University", "priority": 1024, "external_id": null}, {"id": 10846847003, "name": "Pomona College", "priority": 1025, "external_id": null}, {"id": 10846848003, "name": "Pontifical Catholic University of Puerto Rico", "priority": 1026, "external_id": null}, {"id": 10846849003, "name": "Pontifical College Josephinum", "priority": 1027, "external_id": null}, {"id": 10846850003, "name": "Post University", "priority": 1028, "external_id": null}, {"id": 10846851003, "name": "Potomac College", "priority": 1029, "external_id": null}, {"id": 10846852003, "name": "Pratt Institute", "priority": 1030, "external_id": null}, {"id": 10846853003, "name": "Prescott College", "priority": 1031, "external_id": null}, {"id": 10846854003, "name": "Presentation College", "priority": 1032, "external_id": null}, {"id": 10846855003, "name": "Principia College", "priority": 1033, "external_id": null}, {"id": 10846856003, "name": "Providence College", "priority": 1034, "external_id": null}, {"id": 10846857003, "name": "Puerto Rico Conservatory of Music", "priority": 1035, "external_id": null}, {"id": 10846858003, "name": "Purchase College - SUNY", "priority": 1036, "external_id": null}, {"id": 10846859003, "name": "Purdue University - Calumet", "priority": 1037, "external_id": null}, {"id": 10846860003, "name": "Purdue University - North Central", "priority": 1038, "external_id": null}, {"id": 10846861003, "name": "Queens University of Charlotte", "priority": 1039, "external_id": null}, {"id": 10846862003, "name": "Oklahoma State University", "priority": 1040, "external_id": null}, {"id": 10846863003, "name": "Oregon State University", "priority": 1041, "external_id": null}, {"id": 10846864003, "name": "Portland State University", "priority": 1042, "external_id": null}, {"id": 10846865003, "name": "Old Dominion University", "priority": 1043, "external_id": null}, {"id": 10846866003, "name": "Prairie View A&M University", "priority": 1044, "external_id": null}, {"id": 10846867003, "name": "Presbyterian College", "priority": 1045, "external_id": null}, {"id": 10846868003, "name": "Purdue University - West Lafayette", "priority": 1046, "external_id": null}, {"id": 10846869003, "name": "Ohio University", "priority": 1047, "external_id": null}, {"id": 10846870003, "name": "Princeton University", "priority": 1048, "external_id": null}, {"id": 10846871003, "name": "Quincy University", "priority": 1049, "external_id": null}, {"id": 10846872003, "name": "Quinnipiac University", "priority": 1050, "external_id": null}, {"id": 10846873003, "name": "Radford University", "priority": 1051, "external_id": null}, {"id": 10846874003, "name": "Ramapo College of New Jersey", "priority": 1052, "external_id": null}, {"id": 10846875003, "name": "Randolph College", "priority": 1053, "external_id": null}, {"id": 10846876003, "name": "Randolph-Macon College", "priority": 1054, "external_id": null}, {"id": 10846877003, "name": "Ranken Technical College", "priority": 1055, "external_id": null}, {"id": 10846878003, "name": "Reed College", "priority": 1056, "external_id": null}, {"id": 10846879003, "name": "Regent University", "priority": 1057, "external_id": null}, {"id": 10846880003, "name": "Regent's American College London", "priority": 1058, "external_id": null}, {"id": 10846881003, "name": "Regis College", "priority": 1059, "external_id": null}, {"id": 10846882003, "name": "Regis University", "priority": 1060, "external_id": null}, {"id": 10846883003, "name": "Reinhardt University", "priority": 1061, "external_id": null}, {"id": 10846884003, "name": "Rensselaer Polytechnic Institute", "priority": 1062, "external_id": null}, {"id": 10846885003, "name": "Research College of Nursing", "priority": 1063, "external_id": null}, {"id": 10846886003, "name": "Resurrection University", "priority": 1064, "external_id": null}, {"id": 10846887003, "name": "Rhode Island College", "priority": 1065, "external_id": null}, {"id": 10846888003, "name": "Rhode Island School of Design", "priority": 1066, "external_id": null}, {"id": 10846889003, "name": "Rhodes College", "priority": 1067, "external_id": null}, {"id": 10846890003, "name": "Richard Stockton College of New Jersey", "priority": 1068, "external_id": null}, {"id": 10846891003, "name": "Richmond - The American International University in London", "priority": 1069, "external_id": null}, {"id": 10846892003, "name": "Rider University", "priority": 1070, "external_id": null}, {"id": 10846893003, "name": "Ringling College of Art and Design", "priority": 1071, "external_id": null}, {"id": 10846894003, "name": "Ripon College", "priority": 1072, "external_id": null}, {"id": 10846895003, "name": "Rivier University", "priority": 1073, "external_id": null}, {"id": 10846896003, "name": "Roanoke College", "priority": 1074, "external_id": null}, {"id": 10846897003, "name": "Robert B. Miller College", "priority": 1075, "external_id": null}, {"id": 10846898003, "name": "Roberts Wesleyan College", "priority": 1076, "external_id": null}, {"id": 10846899003, "name": "Rochester College", "priority": 1077, "external_id": null}, {"id": 10846900003, "name": "Rochester Institute of Technology", "priority": 1078, "external_id": null}, {"id": 10846901003, "name": "Rockford University", "priority": 1079, "external_id": null}, {"id": 10846902003, "name": "Rockhurst University", "priority": 1080, "external_id": null}, {"id": 10846903003, "name": "Rocky Mountain College", "priority": 1081, "external_id": null}, {"id": 10846904003, "name": "Rocky Mountain College of Art and Design", "priority": 1082, "external_id": null}, {"id": 10846905003, "name": "Roger Williams University", "priority": 1083, "external_id": null}, {"id": 10846906003, "name": "Rogers State University", "priority": 1084, "external_id": null}, {"id": 10846907003, "name": "Rollins College", "priority": 1085, "external_id": null}, {"id": 10846908003, "name": "Roosevelt University", "priority": 1086, "external_id": null}, {"id": 10846909003, "name": "Rosalind Franklin University of Medicine and Science", "priority": 1087, "external_id": null}, {"id": 10846910003, "name": "Rose-Hulman Institute of Technology", "priority": 1088, "external_id": null}, {"id": 10846911003, "name": "Rosemont College", "priority": 1089, "external_id": null}, {"id": 10846912003, "name": "Rowan University", "priority": 1090, "external_id": null}, {"id": 10846913003, "name": "Rush University", "priority": 1091, "external_id": null}, {"id": 10846914003, "name": "Rust College", "priority": 1092, "external_id": null}, {"id": 10846915003, "name": "Rutgers, the State University of New Jersey - Camden", "priority": 1093, "external_id": null}, {"id": 10846916003, "name": "Rutgers, the State University of New Jersey - Newark", "priority": 1094, "external_id": null}, {"id": 10846917003, "name": "Ryerson University", "priority": 1095, "external_id": null}, {"id": 10846918003, "name": "Sacred Heart Major Seminary", "priority": 1096, "external_id": null}, {"id": 10846919003, "name": "Saginaw Valley State University", "priority": 1097, "external_id": null}, {"id": 10846920003, "name": "Salem College", "priority": 1098, "external_id": null}, {"id": 10846921003, "name": "Salem International University", "priority": 1099, "external_id": null}, {"id": 10846922003, "name": "Salem State University", "priority": 1100, "external_id": null}, {"id": 10846923003, "name": "Salisbury University", "priority": 1101, "external_id": null}, {"id": 10846924003, "name": "Salish Kootenai College", "priority": 1102, "external_id": null}, {"id": 10846925003, "name": "Salve Regina University", "priority": 1103, "external_id": null}, {"id": 10846926003, "name": "Samuel Merritt University", "priority": 1104, "external_id": null}, {"id": 10846927003, "name": "San Diego Christian College", "priority": 1105, "external_id": null}, {"id": 10846928003, "name": "San Francisco Art Institute", "priority": 1106, "external_id": null}, {"id": 10846929003, "name": "San Francisco Conservatory of Music", "priority": 1107, "external_id": null}, {"id": 10846930003, "name": "San Francisco State University", "priority": 1108, "external_id": null}, {"id": 10846931003, "name": "Sanford College of Nursing", "priority": 1109, "external_id": null}, {"id": 10846932003, "name": "Santa Clara University", "priority": 1110, "external_id": null}, {"id": 10846933003, "name": "Santa Fe University of Art and Design", "priority": 1111, "external_id": null}, {"id": 10846934003, "name": "Sarah Lawrence College", "priority": 1112, "external_id": null}, {"id": 10846935003, "name": "Savannah College of Art and Design", "priority": 1113, "external_id": null}, {"id": 10846936003, "name": "School of the Art Institute of Chicago", "priority": 1114, "external_id": null}, {"id": 10846937003, "name": "School of Visual Arts", "priority": 1115, "external_id": null}, {"id": 10846938003, "name": "Schreiner University", "priority": 1116, "external_id": null}, {"id": 10846939003, "name": "Scripps College", "priority": 1117, "external_id": null}, {"id": 10846940003, "name": "Seattle Pacific University", "priority": 1118, "external_id": null}, {"id": 10846941003, "name": "Seattle University", "priority": 1119, "external_id": null}, {"id": 10846942003, "name": "Seton Hall University", "priority": 1120, "external_id": null}, {"id": 10846943003, "name": "Seton Hill University", "priority": 1121, "external_id": null}, {"id": 10846944003, "name": "Sewanee - University of the South", "priority": 1122, "external_id": null}, {"id": 10846945003, "name": "Shaw University", "priority": 1123, "external_id": null}, {"id": 10846946003, "name": "Shawnee State University", "priority": 1124, "external_id": null}, {"id": 10846947003, "name": "Shenandoah University", "priority": 1125, "external_id": null}, {"id": 10846948003, "name": "Shepherd University", "priority": 1126, "external_id": null}, {"id": 10846949003, "name": "Shimer College", "priority": 1127, "external_id": null}, {"id": 10846950003, "name": "Sacred Heart University", "priority": 1128, "external_id": null}, {"id": 10846951003, "name": "Robert Morris University", "priority": 1129, "external_id": null}, {"id": 10846952003, "name": "Sam Houston State University", "priority": 1130, "external_id": null}, {"id": 10846953003, "name": "Samford University", "priority": 1131, "external_id": null}, {"id": 10846954003, "name": "Savannah State University", "priority": 1132, "external_id": null}, {"id": 10846955003, "name": "San Jose State University", "priority": 1133, "external_id": null}, {"id": 10846956003, "name": "Rutgers, the State University of New Jersey - New Brunswick", "priority": 1134, "external_id": null}, {"id": 10846957003, "name": "San Diego State University", "priority": 1135, "external_id": null}, {"id": 10846958003, "name": "Shippensburg University of Pennsylvania", "priority": 1136, "external_id": null}, {"id": 10846959003, "name": "Shorter University", "priority": 1137, "external_id": null}, {"id": 10846960003, "name": "Siena College", "priority": 1138, "external_id": null}, {"id": 10846961003, "name": "Siena Heights University", "priority": 1139, "external_id": null}, {"id": 10846962003, "name": "Sierra Nevada College", "priority": 1140, "external_id": null}, {"id": 10846963003, "name": "Silver Lake College", "priority": 1141, "external_id": null}, {"id": 10846964003, "name": "Simmons College", "priority": 1142, "external_id": null}, {"id": 10846965003, "name": "Simon Fraser University", "priority": 1143, "external_id": null}, {"id": 10846966003, "name": "Simpson College", "priority": 1144, "external_id": null}, {"id": 10846967003, "name": "Simpson University", "priority": 1145, "external_id": null}, {"id": 10846968003, "name": "Sinte Gleska University", "priority": 1146, "external_id": null}, {"id": 10846969003, "name": "Sitting Bull College", "priority": 1147, "external_id": null}, {"id": 10846970003, "name": "Skidmore College", "priority": 1148, "external_id": null}, {"id": 10846971003, "name": "Slippery Rock University of Pennsylvania", "priority": 1149, "external_id": null}, {"id": 10846972003, "name": "Smith College", "priority": 1150, "external_id": null}, {"id": 10846973003, "name": "Sojourner-Douglass College", "priority": 1151, "external_id": null}, {"id": 10846974003, "name": "Soka University of America", "priority": 1152, "external_id": null}, {"id": 10846975003, "name": "Sonoma State University", "priority": 1153, "external_id": null}, {"id": 10846976003, "name": "South College", "priority": 1154, "external_id": null}, {"id": 10846977003, "name": "South Dakota School of Mines and Technology", "priority": 1155, "external_id": null}, {"id": 10846978003, "name": "South Seattle Community College", "priority": 1156, "external_id": null}, {"id": 10846979003, "name": "South Texas College", "priority": 1157, "external_id": null}, {"id": 10846980003, "name": "South University", "priority": 1158, "external_id": null}, {"id": 10846981003, "name": "Southeastern Oklahoma State University", "priority": 1159, "external_id": null}, {"id": 10846982003, "name": "Southeastern University", "priority": 1160, "external_id": null}, {"id": 10846983003, "name": "Southern Adventist University", "priority": 1161, "external_id": null}, {"id": 10846984003, "name": "Southern Arkansas University", "priority": 1162, "external_id": null}, {"id": 10846985003, "name": "Southern Baptist Theological Seminary", "priority": 1163, "external_id": null}, {"id": 10846986003, "name": "Southern California Institute of Architecture", "priority": 1164, "external_id": null}, {"id": 10846987003, "name": "Southern Connecticut State University", "priority": 1165, "external_id": null}, {"id": 10846988003, "name": "Southern Illinois University - Edwardsville", "priority": 1166, "external_id": null}, {"id": 10846989003, "name": "Southern Nazarene University", "priority": 1167, "external_id": null}, {"id": 10846990003, "name": "Southern New Hampshire University", "priority": 1168, "external_id": null}, {"id": 10846991003, "name": "Southern Oregon University", "priority": 1169, "external_id": null}, {"id": 10846992003, "name": "Southern Polytechnic State University", "priority": 1170, "external_id": null}, {"id": 10846993003, "name": "Southern University - New Orleans", "priority": 1171, "external_id": null}, {"id": 10846994003, "name": "Southern Vermont College", "priority": 1172, "external_id": null}, {"id": 10846995003, "name": "Southern Wesleyan University", "priority": 1173, "external_id": null}, {"id": 10846996003, "name": "Southwest Baptist University", "priority": 1174, "external_id": null}, {"id": 10846997003, "name": "Southwest Minnesota State University", "priority": 1175, "external_id": null}, {"id": 10846998003, "name": "Southwest University of Visual Arts", "priority": 1176, "external_id": null}, {"id": 10846999003, "name": "Southwestern Adventist University", "priority": 1177, "external_id": null}, {"id": 10847000003, "name": "Southwestern Assemblies of God University", "priority": 1178, "external_id": null}, {"id": 10847001003, "name": "Southwestern Christian College", "priority": 1179, "external_id": null}, {"id": 10847002003, "name": "Southwestern Christian University", "priority": 1180, "external_id": null}, {"id": 10847003003, "name": "Southwestern College", "priority": 1181, "external_id": null}, {"id": 10847004003, "name": "Southwestern Oklahoma State University", "priority": 1182, "external_id": null}, {"id": 10847005003, "name": "Southwestern University", "priority": 1183, "external_id": null}, {"id": 10847006003, "name": "Spalding University", "priority": 1184, "external_id": null}, {"id": 10847007003, "name": "Spelman College", "priority": 1185, "external_id": null}, {"id": 10847008003, "name": "Spring Arbor University", "priority": 1186, "external_id": null}, {"id": 10847009003, "name": "Spring Hill College", "priority": 1187, "external_id": null}, {"id": 10847010003, "name": "Springfield College", "priority": 1188, "external_id": null}, {"id": 10847011003, "name": "St. Ambrose University", "priority": 1189, "external_id": null}, {"id": 10847012003, "name": "St. Anselm College", "priority": 1190, "external_id": null}, {"id": 10847013003, "name": "St. Anthony College of Nursing", "priority": 1191, "external_id": null}, {"id": 10847014003, "name": "St. Augustine College", "priority": 1192, "external_id": null}, {"id": 10847015003, "name": "St. Augustine's University", "priority": 1193, "external_id": null}, {"id": 10847016003, "name": "St. Bonaventure University", "priority": 1194, "external_id": null}, {"id": 10847017003, "name": "St. Catharine College", "priority": 1195, "external_id": null}, {"id": 10847018003, "name": "St. Catherine University", "priority": 1196, "external_id": null}, {"id": 10847019003, "name": "St. Charles Borromeo Seminary", "priority": 1197, "external_id": null}, {"id": 10847020003, "name": "St. Cloud State University", "priority": 1198, "external_id": null}, {"id": 10847021003, "name": "St. Edward's University", "priority": 1199, "external_id": null}, {"id": 10847022003, "name": "St. Francis College", "priority": 1200, "external_id": null}, {"id": 10847023003, "name": "St. Francis Medical Center College of Nursing", "priority": 1201, "external_id": null}, {"id": 10847024003, "name": "St. Gregory's University", "priority": 1202, "external_id": null}, {"id": 10847025003, "name": "St. John Fisher College", "priority": 1203, "external_id": null}, {"id": 10847026003, "name": "St. John Vianney College Seminary", "priority": 1204, "external_id": null}, {"id": 10847027003, "name": "St. John's College", "priority": 1205, "external_id": null}, {"id": 10847028003, "name": "St. John's University", "priority": 1206, "external_id": null}, {"id": 10847029003, "name": "St. Joseph Seminary College", "priority": 1207, "external_id": null}, {"id": 10847030003, "name": "St. Joseph's College", "priority": 1208, "external_id": null}, {"id": 10847031003, "name": "St. Joseph's College New York", "priority": 1209, "external_id": null}, {"id": 10847032003, "name": "St. Joseph's University", "priority": 1210, "external_id": null}, {"id": 10847033003, "name": "St. Lawrence University", "priority": 1211, "external_id": null}, {"id": 10847034003, "name": "St. Leo University", "priority": 1212, "external_id": null}, {"id": 10847035003, "name": "Southern University and A&M College", "priority": 1213, "external_id": null}, {"id": 10847036003, "name": "Southern Methodist University", "priority": 1214, "external_id": null}, {"id": 10847037003, "name": "Southeast Missouri State University", "priority": 1215, "external_id": null}, {"id": 10847038003, "name": "Southern Utah University", "priority": 1216, "external_id": null}, {"id": 10847039003, "name": "South Dakota State University", "priority": 1217, "external_id": null}, {"id": 10847040003, "name": "St. Francis University", "priority": 1218, "external_id": null}, {"id": 10847041003, "name": "Southeastern Louisiana University", "priority": 1219, "external_id": null}, {"id": 10847042003, "name": "Southern Illinois University - Carbondale", "priority": 1220, "external_id": null}, {"id": 10847043003, "name": "St. Louis College of Pharmacy", "priority": 1221, "external_id": null}, {"id": 10847044003, "name": "St. Louis University", "priority": 1222, "external_id": null}, {"id": 10847045003, "name": "St. Luke's College of Health Sciences", "priority": 1223, "external_id": null}, {"id": 10847046003, "name": "St. Martin's University", "priority": 1224, "external_id": null}, {"id": 10847047003, "name": "St. Mary's College", "priority": 1225, "external_id": null}, {"id": 10847048003, "name": "St. Mary's College of California", "priority": 1226, "external_id": null}, {"id": 10847049003, "name": "St. Mary's College of Maryland", "priority": 1227, "external_id": null}, {"id": 10847050003, "name": "St. Mary's Seminary and University", "priority": 1228, "external_id": null}, {"id": 10847051003, "name": "St. Mary's University of Minnesota", "priority": 1229, "external_id": null}, {"id": 10847052003, "name": "St. Mary's University of San Antonio", "priority": 1230, "external_id": null}, {"id": 10847053003, "name": "St. Mary-of-the-Woods College", "priority": 1231, "external_id": null}, {"id": 10847054003, "name": "St. Michael's College", "priority": 1232, "external_id": null}, {"id": 10847055003, "name": "St. Norbert College", "priority": 1233, "external_id": null}, {"id": 10847056003, "name": "St. Olaf College", "priority": 1234, "external_id": null}, {"id": 10847057003, "name": "St. Paul's College", "priority": 1235, "external_id": null}, {"id": 10847058003, "name": "St. Peter's University", "priority": 1236, "external_id": null}, {"id": 10847059003, "name": "St. Petersburg College", "priority": 1237, "external_id": null}, {"id": 10847060003, "name": "St. Thomas Aquinas College", "priority": 1238, "external_id": null}, {"id": 10847061003, "name": "St. Thomas University", "priority": 1239, "external_id": null}, {"id": 10847062003, "name": "St. Vincent College", "priority": 1240, "external_id": null}, {"id": 10847063003, "name": "St. Xavier University", "priority": 1241, "external_id": null}, {"id": 10847064003, "name": "Stephens College", "priority": 1242, "external_id": null}, {"id": 10847065003, "name": "Sterling College", "priority": 1243, "external_id": null}, {"id": 10847066003, "name": "Stevens Institute of Technology", "priority": 1244, "external_id": null}, {"id": 10847067003, "name": "Stevenson University", "priority": 1245, "external_id": null}, {"id": 10847068003, "name": "Stillman College", "priority": 1246, "external_id": null}, {"id": 10847069003, "name": "Stonehill College", "priority": 1247, "external_id": null}, {"id": 10847070003, "name": "Strayer University", "priority": 1248, "external_id": null}, {"id": 10847071003, "name": "Suffolk University", "priority": 1249, "external_id": null}, {"id": 10847072003, "name": "Sul Ross State University", "priority": 1250, "external_id": null}, {"id": 10847073003, "name": "Sullivan University", "priority": 1251, "external_id": null}, {"id": 10847074003, "name": "SUNY Buffalo State", "priority": 1252, "external_id": null}, {"id": 10847075003, "name": "SUNY College of Agriculture and Technology - Cobleskill", "priority": 1253, "external_id": null}, {"id": 10847076003, "name": "SUNY College of Environmental Science and Forestry", "priority": 1254, "external_id": null}, {"id": 10847077003, "name": "SUNY College of Technology - Alfred", "priority": 1255, "external_id": null}, {"id": 10847078003, "name": "SUNY College of Technology - Canton", "priority": 1256, "external_id": null}, {"id": 10847079003, "name": "SUNY College of Technology - Delhi", "priority": 1257, "external_id": null}, {"id": 10847080003, "name": "SUNY College - Cortland", "priority": 1258, "external_id": null}, {"id": 10847081003, "name": "SUNY College - Old Westbury", "priority": 1259, "external_id": null}, {"id": 10847082003, "name": "SUNY College - Oneonta", "priority": 1260, "external_id": null}, {"id": 10847083003, "name": "SUNY College - Potsdam", "priority": 1261, "external_id": null}, {"id": 10847084003, "name": "SUNY Downstate Medical Center", "priority": 1262, "external_id": null}, {"id": 10847085003, "name": "SUNY Empire State College", "priority": 1263, "external_id": null}, {"id": 10847086003, "name": "SUNY Institute of Technology - Utica/Rome", "priority": 1264, "external_id": null}, {"id": 10847087003, "name": "SUNY Maritime College", "priority": 1265, "external_id": null}, {"id": 10847088003, "name": "SUNY Upstate Medical University", "priority": 1266, "external_id": null}, {"id": 10847089003, "name": "SUNY - Fredonia", "priority": 1267, "external_id": null}, {"id": 10847090003, "name": "SUNY - Geneseo", "priority": 1268, "external_id": null}, {"id": 10847091003, "name": "SUNY - New Paltz", "priority": 1269, "external_id": null}, {"id": 10847092003, "name": "SUNY - Oswego", "priority": 1270, "external_id": null}, {"id": 10847093003, "name": "SUNY - Plattsburgh", "priority": 1271, "external_id": null}, {"id": 10847094003, "name": "Swarthmore College", "priority": 1272, "external_id": null}, {"id": 10847095003, "name": "Sweet Briar College", "priority": 1273, "external_id": null}, {"id": 10847096003, "name": "Tabor College", "priority": 1274, "external_id": null}, {"id": 10847097003, "name": "Talladega College", "priority": 1275, "external_id": null}, {"id": 10847098003, "name": "Tarleton State University", "priority": 1276, "external_id": null}, {"id": 10847099003, "name": "Taylor University", "priority": 1277, "external_id": null}, {"id": 10847100003, "name": "Tennessee Wesleyan College", "priority": 1278, "external_id": null}, {"id": 10847101003, "name": "Texas A&M International University", "priority": 1279, "external_id": null}, {"id": 10847102003, "name": "Texas A&M University - Commerce", "priority": 1280, "external_id": null}, {"id": 10847103003, "name": "Texas A&M University - Corpus Christi", "priority": 1281, "external_id": null}, {"id": 10847104003, "name": "Texas A&M University - Galveston", "priority": 1282, "external_id": null}, {"id": 10847105003, "name": "Texas A&M University - Kingsville", "priority": 1283, "external_id": null}, {"id": 10847106003, "name": "Texas A&M University - Texarkana", "priority": 1284, "external_id": null}, {"id": 10847107003, "name": "Texas College", "priority": 1285, "external_id": null}, {"id": 10847108003, "name": "Texas Lutheran University", "priority": 1286, "external_id": null}, {"id": 10847109003, "name": "Bucknell University", "priority": 1287, "external_id": null}, {"id": 10847110003, "name": "Butler University", "priority": 1288, "external_id": null}, {"id": 10847111003, "name": "Stephen F. Austin State University", "priority": 1289, "external_id": null}, {"id": 10847112003, "name": "Texas A&M University - College Station", "priority": 1290, "external_id": null}, {"id": 10847113003, "name": "Stanford University", "priority": 1291, "external_id": null}, {"id": 10847114003, "name": "Stetson University", "priority": 1292, "external_id": null}, {"id": 10847115003, "name": "Stony Brook University - SUNY", "priority": 1293, "external_id": null}, {"id": 10847116003, "name": "Syracuse University", "priority": 1294, "external_id": null}, {"id": 10847117003, "name": "Texas Christian University", "priority": 1295, "external_id": null}, {"id": 10847118003, "name": "Temple University", "priority": 1296, "external_id": null}, {"id": 10847119003, "name": "Clemson University", "priority": 1297, "external_id": null}, {"id": 10847120003, "name": "Texas Southern University", "priority": 1298, "external_id": null}, {"id": 10847121003, "name": "Austin Peay State University", "priority": 1299, "external_id": null}, {"id": 10847122003, "name": "Tennessee State University", "priority": 1300, "external_id": null}, {"id": 10847123003, "name": "Ball State University", "priority": 1301, "external_id": null}, {"id": 10847124003, "name": "Texas Tech University Health Sciences Center", "priority": 1302, "external_id": null}, {"id": 10847125003, "name": "Texas Wesleyan University", "priority": 1303, "external_id": null}, {"id": 10847126003, "name": "Texas Woman's University", "priority": 1304, "external_id": null}, {"id": 10847127003, "name": "The Catholic University of America", "priority": 1305, "external_id": null}, {"id": 10847128003, "name": "The Sage Colleges", "priority": 1306, "external_id": null}, {"id": 10847129003, "name": "Thiel College", "priority": 1307, "external_id": null}, {"id": 10847130003, "name": "Thomas Aquinas College", "priority": 1308, "external_id": null}, {"id": 10847131003, "name": "Thomas College", "priority": 1309, "external_id": null}, {"id": 10847132003, "name": "Thomas Edison State College", "priority": 1310, "external_id": null}, {"id": 10847133003, "name": "Thomas Jefferson University", "priority": 1311, "external_id": null}, {"id": 10847134003, "name": "Thomas More College", "priority": 1312, "external_id": null}, {"id": 10847135003, "name": "Thomas More College of Liberal Arts", "priority": 1313, "external_id": null}, {"id": 10847136003, "name": "Thomas University", "priority": 1314, "external_id": null}, {"id": 10847137003, "name": "Tiffin University", "priority": 1315, "external_id": null}, {"id": 10847138003, "name": "Tilburg University", "priority": 1316, "external_id": null}, {"id": 10847139003, "name": "Toccoa Falls College", "priority": 1317, "external_id": null}, {"id": 10847140003, "name": "Tougaloo College", "priority": 1318, "external_id": null}, {"id": 10847141003, "name": "Touro College", "priority": 1319, "external_id": null}, {"id": 10847142003, "name": "Transylvania University", "priority": 1320, "external_id": null}, {"id": 10847143003, "name": "Trent University", "priority": 1321, "external_id": null}, {"id": 10847144003, "name": "Trevecca Nazarene University", "priority": 1322, "external_id": null}, {"id": 10847145003, "name": "Trident University International", "priority": 1323, "external_id": null}, {"id": 10847146003, "name": "Trine University", "priority": 1324, "external_id": null}, {"id": 10847147003, "name": "Trinity Christian College", "priority": 1325, "external_id": null}, {"id": 10847148003, "name": "Trinity College", "priority": 1326, "external_id": null}, {"id": 10847149003, "name": "Trinity College of Nursing & Health Sciences", "priority": 1327, "external_id": null}, {"id": 10847150003, "name": "Trinity International University", "priority": 1328, "external_id": null}, {"id": 10847151003, "name": "Trinity Lutheran College", "priority": 1329, "external_id": null}, {"id": 10847152003, "name": "Trinity University", "priority": 1330, "external_id": null}, {"id": 10847153003, "name": "Trinity Western University", "priority": 1331, "external_id": null}, {"id": 10847154003, "name": "Truett McConnell College", "priority": 1332, "external_id": null}, {"id": 10847155003, "name": "Truman State University", "priority": 1333, "external_id": null}, {"id": 10847156003, "name": "Tufts University", "priority": 1334, "external_id": null}, {"id": 10847157003, "name": "Tusculum College", "priority": 1335, "external_id": null}, {"id": 10847158003, "name": "Tuskegee University", "priority": 1336, "external_id": null}, {"id": 10847159003, "name": "Union College", "priority": 1337, "external_id": null}, {"id": 10847160003, "name": "Union Institute and University", "priority": 1338, "external_id": null}, {"id": 10847161003, "name": "Union University", "priority": 1339, "external_id": null}, {"id": 10847162003, "name": "United States Coast Guard Academy", "priority": 1340, "external_id": null}, {"id": 10847163003, "name": "United States International University - Kenya", "priority": 1341, "external_id": null}, {"id": 10847164003, "name": "United States Merchant Marine Academy", "priority": 1342, "external_id": null}, {"id": 10847165003, "name": "United States Sports Academy", "priority": 1343, "external_id": null}, {"id": 10847166003, "name": "Unity College", "priority": 1344, "external_id": null}, {"id": 10847167003, "name": "Universidad Adventista de las Antillas", "priority": 1345, "external_id": null}, {"id": 10847168003, "name": "Universidad del Este", "priority": 1346, "external_id": null}, {"id": 10847169003, "name": "Universidad del Turabo", "priority": 1347, "external_id": null}, {"id": 10847170003, "name": "Universidad Metropolitana", "priority": 1348, "external_id": null}, {"id": 10847171003, "name": "Universidad Politecnica De Puerto Rico", "priority": 1349, "external_id": null}, {"id": 10847172003, "name": "University of Advancing Technology", "priority": 1350, "external_id": null}, {"id": 10847173003, "name": "University of Alabama - Huntsville", "priority": 1351, "external_id": null}, {"id": 10847174003, "name": "University of Alaska - Anchorage", "priority": 1352, "external_id": null}, {"id": 10847175003, "name": "University of Alaska - Fairbanks", "priority": 1353, "external_id": null}, {"id": 10847176003, "name": "University of Alaska - Southeast", "priority": 1354, "external_id": null}, {"id": 10847177003, "name": "University of Alberta", "priority": 1355, "external_id": null}, {"id": 10847178003, "name": "University of Arkansas for Medical Sciences", "priority": 1356, "external_id": null}, {"id": 10847179003, "name": "University of Arkansas - Fort Smith", "priority": 1357, "external_id": null}, {"id": 10847180003, "name": "University of Arkansas - Little Rock", "priority": 1358, "external_id": null}, {"id": 10847181003, "name": "University of Arkansas - Monticello", "priority": 1359, "external_id": null}, {"id": 10847182003, "name": "University of Baltimore", "priority": 1360, "external_id": null}, {"id": 10847183003, "name": "University of Bridgeport", "priority": 1361, "external_id": null}, {"id": 10847184003, "name": "University of British Columbia", "priority": 1362, "external_id": null}, {"id": 10847185003, "name": "University of Calgary", "priority": 1363, "external_id": null}, {"id": 10847186003, "name": "University of California - Riverside", "priority": 1364, "external_id": null}, {"id": 10847187003, "name": "Holy Cross College", "priority": 1365, "external_id": null}, {"id": 10847188003, "name": "Towson University", "priority": 1366, "external_id": null}, {"id": 10847189003, "name": "United States Military Academy", "priority": 1367, "external_id": null}, {"id": 10847190003, "name": "The Citadel", "priority": 1368, "external_id": null}, {"id": 10847191003, "name": "Troy University", "priority": 1369, "external_id": null}, {"id": 10847192003, "name": "University of California - Davis", "priority": 1370, "external_id": null}, {"id": 10847193003, "name": "Grambling State University", "priority": 1371, "external_id": null}, {"id": 10847194003, "name": "University at Albany - SUNY", "priority": 1372, "external_id": null}, {"id": 10847195003, "name": "University at Buffalo - SUNY", "priority": 1373, "external_id": null}, {"id": 10847196003, "name": "United States Naval Academy", "priority": 1374, "external_id": null}, {"id": 10847197003, "name": "University of Arizona", "priority": 1375, "external_id": null}, {"id": 10847198003, "name": "University of California - Los Angeles", "priority": 1376, "external_id": null}, {"id": 10847199003, "name": "Florida A&M University", "priority": 1377, "external_id": null}, {"id": 10847200003, "name": "Texas State University", "priority": 1378, "external_id": null}, {"id": 10847201003, "name": "University of Alabama - Birmingham", "priority": 1379, "external_id": null}, {"id": 10847202003, "name": "University of California - Santa Cruz", "priority": 1380, "external_id": null}, {"id": 10847203003, "name": "University of Central Missouri", "priority": 1381, "external_id": null}, {"id": 10847204003, "name": "University of Central Oklahoma", "priority": 1382, "external_id": null}, {"id": 10847205003, "name": "University of Charleston", "priority": 1383, "external_id": null}, {"id": 10847206003, "name": "University of Chicago", "priority": 1384, "external_id": null}, {"id": 10847207003, "name": "University of Cincinnati - UC Blue Ash College", "priority": 1385, "external_id": null}, {"id": 10847208003, "name": "University of Colorado - Colorado Springs", "priority": 1386, "external_id": null}, {"id": 10847209003, "name": "University of Colorado - Denver", "priority": 1387, "external_id": null}, {"id": 10847210003, "name": "University of Dallas", "priority": 1388, "external_id": null}, {"id": 10847211003, "name": "University of Denver", "priority": 1389, "external_id": null}, {"id": 10847212003, "name": "University of Detroit Mercy", "priority": 1390, "external_id": null}, {"id": 10847213003, "name": "University of Dubuque", "priority": 1391, "external_id": null}, {"id": 10847214003, "name": "University of Evansville", "priority": 1392, "external_id": null}, {"id": 10847215003, "name": "University of Findlay", "priority": 1393, "external_id": null}, {"id": 10847216003, "name": "University of Great Falls", "priority": 1394, "external_id": null}, {"id": 10847217003, "name": "University of Guam", "priority": 1395, "external_id": null}, {"id": 10847218003, "name": "University of Guelph", "priority": 1396, "external_id": null}, {"id": 10847219003, "name": "University of Hartford", "priority": 1397, "external_id": null}, {"id": 10847220003, "name": "University of Hawaii - Hilo", "priority": 1398, "external_id": null}, {"id": 10847221003, "name": "University of Hawaii - Maui College", "priority": 1399, "external_id": null}, {"id": 10847222003, "name": "University of Hawaii - West Oahu", "priority": 1400, "external_id": null}, {"id": 10847223003, "name": "University of Houston - Clear Lake", "priority": 1401, "external_id": null}, {"id": 10847224003, "name": "University of Houston - Downtown", "priority": 1402, "external_id": null}, {"id": 10847225003, "name": "University of Houston - Victoria", "priority": 1403, "external_id": null}, {"id": 10847226003, "name": "University of Illinois - Chicago", "priority": 1404, "external_id": null}, {"id": 10847227003, "name": "University of Illinois - Springfield", "priority": 1405, "external_id": null}, {"id": 10847228003, "name": "University of Indianapolis", "priority": 1406, "external_id": null}, {"id": 10847229003, "name": "University of Jamestown", "priority": 1407, "external_id": null}, {"id": 10847230003, "name": "University of La Verne", "priority": 1408, "external_id": null}, {"id": 10847231003, "name": "University of Maine - Augusta", "priority": 1409, "external_id": null}, {"id": 10847232003, "name": "University of Maine - Farmington", "priority": 1410, "external_id": null}, {"id": 10847233003, "name": "University of Maine - Fort Kent", "priority": 1411, "external_id": null}, {"id": 10847234003, "name": "University of Maine - Machias", "priority": 1412, "external_id": null}, {"id": 10847235003, "name": "University of Maine - Presque Isle", "priority": 1413, "external_id": null}, {"id": 10847236003, "name": "University of Mary", "priority": 1414, "external_id": null}, {"id": 10847237003, "name": "University of Mary Hardin-Baylor", "priority": 1415, "external_id": null}, {"id": 10847238003, "name": "University of Mary Washington", "priority": 1416, "external_id": null}, {"id": 10847239003, "name": "University of Maryland - Baltimore", "priority": 1417, "external_id": null}, {"id": 10847240003, "name": "University of Maryland - Baltimore County", "priority": 1418, "external_id": null}, {"id": 10847241003, "name": "University of Maryland - Eastern Shore", "priority": 1419, "external_id": null}, {"id": 10847242003, "name": "University of Maryland - University College", "priority": 1420, "external_id": null}, {"id": 10847243003, "name": "University of Massachusetts - Boston", "priority": 1421, "external_id": null}, {"id": 10847244003, "name": "University of Massachusetts - Dartmouth", "priority": 1422, "external_id": null}, {"id": 10847245003, "name": "University of Massachusetts - Lowell", "priority": 1423, "external_id": null}, {"id": 10847246003, "name": "University of Medicine and Dentistry of New Jersey", "priority": 1424, "external_id": null}, {"id": 10847247003, "name": "University of Michigan - Dearborn", "priority": 1425, "external_id": null}, {"id": 10847248003, "name": "University of Michigan - Flint", "priority": 1426, "external_id": null}, {"id": 10847249003, "name": "University of Minnesota - Crookston", "priority": 1427, "external_id": null}, {"id": 10847250003, "name": "University of Minnesota - Duluth", "priority": 1428, "external_id": null}, {"id": 10847251003, "name": "University of Minnesota - Morris", "priority": 1429, "external_id": null}, {"id": 10847252003, "name": "University of Mississippi Medical Center", "priority": 1430, "external_id": null}, {"id": 10847253003, "name": "University of Missouri - Kansas City", "priority": 1431, "external_id": null}, {"id": 10847254003, "name": "University of Missouri - St. Louis", "priority": 1432, "external_id": null}, {"id": 10847255003, "name": "University of Mobile", "priority": 1433, "external_id": null}, {"id": 10847256003, "name": "University of Montana - Western", "priority": 1434, "external_id": null}, {"id": 10847257003, "name": "University of Montevallo", "priority": 1435, "external_id": null}, {"id": 10847258003, "name": "University of Mount Union", "priority": 1436, "external_id": null}, {"id": 10847259003, "name": "University of Nebraska Medical Center", "priority": 1437, "external_id": null}, {"id": 10847260003, "name": "University of Nebraska - Kearney", "priority": 1438, "external_id": null}, {"id": 10847261003, "name": "University of Dayton", "priority": 1439, "external_id": null}, {"id": 10847262003, "name": "University of Delaware", "priority": 1440, "external_id": null}, {"id": 10847263003, "name": "University of Florida", "priority": 1441, "external_id": null}, {"id": 10847264003, "name": "University of Iowa", "priority": 1442, "external_id": null}, {"id": 10847265003, "name": "University of Idaho", "priority": 1443, "external_id": null}, {"id": 10847266003, "name": "University of Kentucky", "priority": 1444, "external_id": null}, {"id": 10847267003, "name": "University of Massachusetts - Amherst", "priority": 1445, "external_id": null}, {"id": 10847268003, "name": "University of Maine", "priority": 1446, "external_id": null}, {"id": 10847269003, "name": "University of Michigan - Ann Arbor", "priority": 1447, "external_id": null}, {"id": 10847270003, "name": "University of Cincinnati", "priority": 1448, "external_id": null}, {"id": 10847271003, "name": "University of Miami", "priority": 1449, "external_id": null}, {"id": 10847272003, "name": "University of Louisiana - Monroe", "priority": 1450, "external_id": null}, {"id": 10847273003, "name": "University of Missouri", "priority": 1451, "external_id": null}, {"id": 10847274003, "name": "University of Mississippi", "priority": 1452, "external_id": null}, {"id": 10847275003, "name": "University of Memphis", "priority": 1453, "external_id": null}, {"id": 10847276003, "name": "University of Houston", "priority": 1454, "external_id": null}, {"id": 10847277003, "name": "University of Colorado - Boulder", "priority": 1455, "external_id": null}, {"id": 10847278003, "name": "University of Nebraska - Omaha", "priority": 1456, "external_id": null}, {"id": 10847279003, "name": "University of New Brunswick", "priority": 1457, "external_id": null}, {"id": 10847280003, "name": "University of New England", "priority": 1458, "external_id": null}, {"id": 10847281003, "name": "University of New Haven", "priority": 1459, "external_id": null}, {"id": 10847282003, "name": "University of New Orleans", "priority": 1460, "external_id": null}, {"id": 10847283003, "name": "University of North Alabama", "priority": 1461, "external_id": null}, {"id": 10847284003, "name": "University of North Carolina School of the Arts", "priority": 1462, "external_id": null}, {"id": 10847285003, "name": "University of North Carolina - Asheville", "priority": 1463, "external_id": null}, {"id": 10847286003, "name": "University of North Carolina - Greensboro", "priority": 1464, "external_id": null}, {"id": 10847287003, "name": "University of North Carolina - Pembroke", "priority": 1465, "external_id": null}, {"id": 10847288003, "name": "University of North Carolina - Wilmington", "priority": 1466, "external_id": null}, {"id": 10847289003, "name": "University of North Florida", "priority": 1467, "external_id": null}, {"id": 10847290003, "name": "University of North Georgia", "priority": 1468, "external_id": null}, {"id": 10847291003, "name": "University of Northwestern Ohio", "priority": 1469, "external_id": null}, {"id": 10847292003, "name": "University of Northwestern - St. Paul", "priority": 1470, "external_id": null}, {"id": 10847293003, "name": "University of Ottawa", "priority": 1471, "external_id": null}, {"id": 10847294003, "name": "University of Phoenix", "priority": 1472, "external_id": null}, {"id": 10847295003, "name": "University of Pikeville", "priority": 1473, "external_id": null}, {"id": 10847296003, "name": "University of Portland", "priority": 1474, "external_id": null}, {"id": 10847297003, "name": "University of Prince Edward Island", "priority": 1475, "external_id": null}, {"id": 10847298003, "name": "University of Puerto Rico - Aguadilla", "priority": 1476, "external_id": null}, {"id": 10847299003, "name": "University of Puerto Rico - Arecibo", "priority": 1477, "external_id": null}, {"id": 10847300003, "name": "University of Puerto Rico - Bayamon", "priority": 1478, "external_id": null}, {"id": 10847301003, "name": "University of Puerto Rico - Cayey", "priority": 1479, "external_id": null}, {"id": 10847302003, "name": "University of Puerto Rico - Humacao", "priority": 1480, "external_id": null}, {"id": 10847303003, "name": "University of Puerto Rico - Mayaguez", "priority": 1481, "external_id": null}, {"id": 10847304003, "name": "University of Puerto Rico - Medical Sciences Campus", "priority": 1482, "external_id": null}, {"id": 10847305003, "name": "University of Puerto Rico - Ponce", "priority": 1483, "external_id": null}, {"id": 10847306003, "name": "University of Puerto Rico - Rio Piedras", "priority": 1484, "external_id": null}, {"id": 10847307003, "name": "University of Puget Sound", "priority": 1485, "external_id": null}, {"id": 10847308003, "name": "University of Redlands", "priority": 1486, "external_id": null}, {"id": 10847309003, "name": "University of Regina", "priority": 1487, "external_id": null}, {"id": 10847310003, "name": "University of Rio Grande", "priority": 1488, "external_id": null}, {"id": 10847311003, "name": "University of Rochester", "priority": 1489, "external_id": null}, {"id": 10847312003, "name": "University of San Francisco", "priority": 1490, "external_id": null}, {"id": 10847313003, "name": "University of Saskatchewan", "priority": 1491, "external_id": null}, {"id": 10847314003, "name": "University of Science and Arts of Oklahoma", "priority": 1492, "external_id": null}, {"id": 10847315003, "name": "University of Scranton", "priority": 1493, "external_id": null}, {"id": 10847316003, "name": "University of Sioux Falls", "priority": 1494, "external_id": null}, {"id": 10847317003, "name": "University of South Carolina - Aiken", "priority": 1495, "external_id": null}, {"id": 10847318003, "name": "University of South Carolina - Beaufort", "priority": 1496, "external_id": null}, {"id": 10847319003, "name": "University of South Carolina - Upstate", "priority": 1497, "external_id": null}, {"id": 10847320003, "name": "University of South Florida - St. Petersburg", "priority": 1498, "external_id": null}, {"id": 10847321003, "name": "University of Southern Indiana", "priority": 1499, "external_id": null}, {"id": 10847322003, "name": "University of Southern Maine", "priority": 1500, "external_id": null}, {"id": 10847323003, "name": "University of St. Francis", "priority": 1501, "external_id": null}, {"id": 10847324003, "name": "University of St. Joseph", "priority": 1502, "external_id": null}, {"id": 10847325003, "name": "University of St. Mary", "priority": 1503, "external_id": null}, {"id": 10847326003, "name": "University of St. Thomas", "priority": 1504, "external_id": null}, {"id": 10847327003, "name": "University of Tampa", "priority": 1505, "external_id": null}, {"id": 10847328003, "name": "University of Texas Health Science Center - Houston", "priority": 1506, "external_id": null}, {"id": 10847329003, "name": "University of Texas Health Science Center - San Antonio", "priority": 1507, "external_id": null}, {"id": 10847330003, "name": "University of Texas Medical Branch - Galveston", "priority": 1508, "external_id": null}, {"id": 10847331003, "name": "University of Texas of the Permian Basin", "priority": 1509, "external_id": null}, {"id": 10847332003, "name": "University of Texas - Arlington", "priority": 1510, "external_id": null}, {"id": 10847333003, "name": "University of Texas - Brownsville", "priority": 1511, "external_id": null}, {"id": 10847334003, "name": "University of Texas - Pan American", "priority": 1512, "external_id": null}, {"id": 10847335003, "name": "University of Oregon", "priority": 1513, "external_id": null}, {"id": 10847336003, "name": "University of New Mexico", "priority": 1514, "external_id": null}, {"id": 10847337003, "name": "University of Pennsylvania", "priority": 1515, "external_id": null}, {"id": 10847338003, "name": "University of North Dakota", "priority": 1516, "external_id": null}, {"id": 10847339003, "name": "University of Nevada - Reno", "priority": 1517, "external_id": null}, {"id": 10847340003, "name": "University of New Hampshire", "priority": 1518, "external_id": null}, {"id": 10847341003, "name": "University of Texas - Austin", "priority": 1519, "external_id": null}, {"id": 10847342003, "name": "University of Southern Mississippi", "priority": 1520, "external_id": null}, {"id": 10847343003, "name": "University of Rhode Island", "priority": 1521, "external_id": null}, {"id": 10847344003, "name": "University of South Dakota", "priority": 1522, "external_id": null}, {"id": 10847345003, "name": "University of Tennessee", "priority": 1523, "external_id": null}, {"id": 10847346003, "name": "University of North Texas", "priority": 1524, "external_id": null}, {"id": 10847347003, "name": "University of North Carolina - Charlotte", "priority": 1525, "external_id": null}, {"id": 10847348003, "name": "University of Texas - San Antonio", "priority": 1526, "external_id": null}, {"id": 10847349003, "name": "University of Notre Dame", "priority": 1527, "external_id": null}, {"id": 10847350003, "name": "University of Southern California", "priority": 1528, "external_id": null}, {"id": 10847351003, "name": "University of Texas - Tyler", "priority": 1529, "external_id": null}, {"id": 10847352003, "name": "University of the Arts", "priority": 1530, "external_id": null}, {"id": 10847353003, "name": "University of the Cumberlands", "priority": 1531, "external_id": null}, {"id": 10847354003, "name": "University of the District of Columbia", "priority": 1532, "external_id": null}, {"id": 10847355003, "name": "University of the Ozarks", "priority": 1533, "external_id": null}, {"id": 10847356003, "name": "University of the Pacific", "priority": 1534, "external_id": null}, {"id": 10847357003, "name": "University of the Sacred Heart", "priority": 1535, "external_id": null}, {"id": 10847358003, "name": "University of the Sciences", "priority": 1536, "external_id": null}, {"id": 10847359003, "name": "University of the Southwest", "priority": 1537, "external_id": null}, {"id": 10847360003, "name": "University of the Virgin Islands", "priority": 1538, "external_id": null}, {"id": 10847361003, "name": "University of the West", "priority": 1539, "external_id": null}, {"id": 10847362003, "name": "University of Toronto", "priority": 1540, "external_id": null}, {"id": 10847363003, "name": "University of Vermont", "priority": 1541, "external_id": null}, {"id": 10847364003, "name": "University of Victoria", "priority": 1542, "external_id": null}, {"id": 10847365003, "name": "University of Virginia - Wise", "priority": 1543, "external_id": null}, {"id": 10847366003, "name": "University of Waterloo", "priority": 1544, "external_id": null}, {"id": 10847367003, "name": "University of West Alabama", "priority": 1545, "external_id": null}, {"id": 10847368003, "name": "University of West Florida", "priority": 1546, "external_id": null}, {"id": 10847369003, "name": "University of West Georgia", "priority": 1547, "external_id": null}, {"id": 10847370003, "name": "University of Windsor", "priority": 1548, "external_id": null}, {"id": 10847371003, "name": "University of Winnipeg", "priority": 1549, "external_id": null}, {"id": 10847372003, "name": "University of Wisconsin - Eau Claire", "priority": 1550, "external_id": null}, {"id": 10847373003, "name": "University of Wisconsin - Green Bay", "priority": 1551, "external_id": null}, {"id": 10847374003, "name": "University of Wisconsin - La Crosse", "priority": 1552, "external_id": null}, {"id": 10847375003, "name": "University of Wisconsin - Milwaukee", "priority": 1553, "external_id": null}, {"id": 10847376003, "name": "University of Wisconsin - Oshkosh", "priority": 1554, "external_id": null}, {"id": 10847377003, "name": "University of Wisconsin - Parkside", "priority": 1555, "external_id": null}, {"id": 10847378003, "name": "University of Wisconsin - Platteville", "priority": 1556, "external_id": null}, {"id": 10847379003, "name": "University of Wisconsin - River Falls", "priority": 1557, "external_id": null}, {"id": 10847380003, "name": "University of Wisconsin - Stevens Point", "priority": 1558, "external_id": null}, {"id": 10847381003, "name": "University of Wisconsin - Stout", "priority": 1559, "external_id": null}, {"id": 10847382003, "name": "University of Wisconsin - Superior", "priority": 1560, "external_id": null}, {"id": 10847383003, "name": "University of Wisconsin - Whitewater", "priority": 1561, "external_id": null}, {"id": 10847384003, "name": "Upper Iowa University", "priority": 1562, "external_id": null}, {"id": 10847385003, "name": "Urbana University", "priority": 1563, "external_id": null}, {"id": 10847386003, "name": "Ursinus College", "priority": 1564, "external_id": null}, {"id": 10847387003, "name": "Ursuline College", "priority": 1565, "external_id": null}, {"id": 10847388003, "name": "Utah Valley University", "priority": 1566, "external_id": null}, {"id": 10847389003, "name": "Utica College", "priority": 1567, "external_id": null}, {"id": 10847390003, "name": "Valdosta State University", "priority": 1568, "external_id": null}, {"id": 10847391003, "name": "Valley City State University", "priority": 1569, "external_id": null}, {"id": 10847392003, "name": "Valley Forge Christian College", "priority": 1570, "external_id": null}, {"id": 10847393003, "name": "VanderCook College of Music", "priority": 1571, "external_id": null}, {"id": 10847394003, "name": "Vanguard University of Southern California", "priority": 1572, "external_id": null}, {"id": 10847395003, "name": "Vassar College", "priority": 1573, "external_id": null}, {"id": 10847396003, "name": "Vaughn College of Aeronautics and Technology", "priority": 1574, "external_id": null}, {"id": 10847397003, "name": "Vermont Technical College", "priority": 1575, "external_id": null}, {"id": 10847398003, "name": "Victory University", "priority": 1576, "external_id": null}, {"id": 10847399003, "name": "Vincennes University", "priority": 1577, "external_id": null}, {"id": 10847400003, "name": "Virginia Commonwealth University", "priority": 1578, "external_id": null}, {"id": 10847401003, "name": "Virginia Intermont College", "priority": 1579, "external_id": null}, {"id": 10847402003, "name": "Virginia State University", "priority": 1580, "external_id": null}, {"id": 10847403003, "name": "Virginia Union University", "priority": 1581, "external_id": null}, {"id": 10847404003, "name": "Virginia Wesleyan College", "priority": 1582, "external_id": null}, {"id": 10847405003, "name": "Viterbo University", "priority": 1583, "external_id": null}, {"id": 10847406003, "name": "Voorhees College", "priority": 1584, "external_id": null}, {"id": 10847407003, "name": "Wabash College", "priority": 1585, "external_id": null}, {"id": 10847408003, "name": "Walden University", "priority": 1586, "external_id": null}, {"id": 10847409003, "name": "Waldorf College", "priority": 1587, "external_id": null}, {"id": 10847410003, "name": "Walla Walla University", "priority": 1588, "external_id": null}, {"id": 10847411003, "name": "Walsh College of Accountancy and Business Administration", "priority": 1589, "external_id": null}, {"id": 10847412003, "name": "Walsh University", "priority": 1590, "external_id": null}, {"id": 10847413003, "name": "Warner Pacific College", "priority": 1591, "external_id": null}, {"id": 10847414003, "name": "Warner University", "priority": 1592, "external_id": null}, {"id": 10847415003, "name": "Warren Wilson College", "priority": 1593, "external_id": null}, {"id": 10847416003, "name": "Wartburg College", "priority": 1594, "external_id": null}, {"id": 10847417003, "name": "Washburn University", "priority": 1595, "external_id": null}, {"id": 10847418003, "name": "Washington Adventist University", "priority": 1596, "external_id": null}, {"id": 10847419003, "name": "Washington and Jefferson College", "priority": 1597, "external_id": null}, {"id": 10847420003, "name": "Washington and Lee University", "priority": 1598, "external_id": null}, {"id": 10847421003, "name": "Washington College", "priority": 1599, "external_id": null}, {"id": 10847422003, "name": "Washington University in St. Louis", "priority": 1600, "external_id": null}, {"id": 10847423003, "name": "Watkins College of Art, Design & Film", "priority": 1601, "external_id": null}, {"id": 10847424003, "name": "Wayland Baptist University", "priority": 1602, "external_id": null}, {"id": 10847425003, "name": "Wayne State College", "priority": 1603, "external_id": null}, {"id": 10847426003, "name": "Wayne State University", "priority": 1604, "external_id": null}, {"id": 10847427003, "name": "Waynesburg University", "priority": 1605, "external_id": null}, {"id": 10847428003, "name": "Valparaiso University", "priority": 1606, "external_id": null}, {"id": 10847429003, "name": "Villanova University", "priority": 1607, "external_id": null}, {"id": 10847430003, "name": "Virginia Tech", "priority": 1608, "external_id": null}, {"id": 10847431003, "name": "Washington State University", "priority": 1609, "external_id": null}, {"id": 10847432003, "name": "University of Toledo", "priority": 1610, "external_id": null}, {"id": 10847433003, "name": "Wagner College", "priority": 1611, "external_id": null}, {"id": 10847434003, "name": "University of Wyoming", "priority": 1612, "external_id": null}, {"id": 10847435003, "name": "University of Wisconsin - Madison", "priority": 1613, "external_id": null}, {"id": 10847436003, "name": "University of Tulsa", "priority": 1614, "external_id": null}, {"id": 10847437003, "name": "Webb Institute", "priority": 1615, "external_id": null}, {"id": 10847438003, "name": "Webber International University", "priority": 1616, "external_id": null}, {"id": 10847439003, "name": "Webster University", "priority": 1617, "external_id": null}, {"id": 10847440003, "name": "Welch College", "priority": 1618, "external_id": null}, {"id": 10847441003, "name": "Wellesley College", "priority": 1619, "external_id": null}, {"id": 10847442003, "name": "Wells College", "priority": 1620, "external_id": null}, {"id": 10847443003, "name": "Wentworth Institute of Technology", "priority": 1621, "external_id": null}, {"id": 10847444003, "name": "Wesley College", "priority": 1622, "external_id": null}, {"id": 10847445003, "name": "Wesleyan College", "priority": 1623, "external_id": null}, {"id": 10847446003, "name": "Wesleyan University", "priority": 1624, "external_id": null}, {"id": 10847447003, "name": "West Chester University of Pennsylvania", "priority": 1625, "external_id": null}, {"id": 10847448003, "name": "West Liberty University", "priority": 1626, "external_id": null}, {"id": 10847449003, "name": "West Texas A&M University", "priority": 1627, "external_id": null}, {"id": 10847450003, "name": "West Virginia State University", "priority": 1628, "external_id": null}, {"id": 10847451003, "name": "West Virginia University Institute of Technology", "priority": 1629, "external_id": null}, {"id": 10847452003, "name": "West Virginia University - Parkersburg", "priority": 1630, "external_id": null}, {"id": 10847453003, "name": "West Virginia Wesleyan College", "priority": 1631, "external_id": null}, {"id": 10847454003, "name": "Western Connecticut State University", "priority": 1632, "external_id": null}, {"id": 10847455003, "name": "Western Governors University", "priority": 1633, "external_id": null}, {"id": 10847456003, "name": "Western International University", "priority": 1634, "external_id": null}, {"id": 10847457003, "name": "Western Nevada College", "priority": 1635, "external_id": null}, {"id": 10847458003, "name": "Western New England University", "priority": 1636, "external_id": null}, {"id": 10847459003, "name": "Western New Mexico University", "priority": 1637, "external_id": null}, {"id": 10847460003, "name": "Western Oregon University", "priority": 1638, "external_id": null}, {"id": 10847461003, "name": "Western State Colorado University", "priority": 1639, "external_id": null}, {"id": 10847462003, "name": "Western University", "priority": 1640, "external_id": null}, {"id": 10847463003, "name": "Western Washington University", "priority": 1641, "external_id": null}, {"id": 10847464003, "name": "Westfield State University", "priority": 1642, "external_id": null}, {"id": 10847465003, "name": "Westminster College", "priority": 1643, "external_id": null}, {"id": 10847466003, "name": "Westmont College", "priority": 1644, "external_id": null}, {"id": 10847467003, "name": "Wheaton College", "priority": 1645, "external_id": null}, {"id": 10847468003, "name": "Wheeling Jesuit University", "priority": 1646, "external_id": null}, {"id": 10847469003, "name": "Wheelock College", "priority": 1647, "external_id": null}, {"id": 10847470003, "name": "Whitman College", "priority": 1648, "external_id": null}, {"id": 10847471003, "name": "Whittier College", "priority": 1649, "external_id": null}, {"id": 10847472003, "name": "Whitworth University", "priority": 1650, "external_id": null}, {"id": 10847473003, "name": "Wichita State University", "priority": 1651, "external_id": null}, {"id": 10847474003, "name": "Widener University", "priority": 1652, "external_id": null}, {"id": 10847475003, "name": "Wilberforce University", "priority": 1653, "external_id": null}, {"id": 10847476003, "name": "Wiley College", "priority": 1654, "external_id": null}, {"id": 10847477003, "name": "Wilkes University", "priority": 1655, "external_id": null}, {"id": 10847478003, "name": "Willamette University", "priority": 1656, "external_id": null}, {"id": 10847479003, "name": "William Carey University", "priority": 1657, "external_id": null}, {"id": 10847480003, "name": "William Jessup University", "priority": 1658, "external_id": null}, {"id": 10847481003, "name": "William Jewell College", "priority": 1659, "external_id": null}, {"id": 10847482003, "name": "William Paterson University of New Jersey", "priority": 1660, "external_id": null}, {"id": 10847483003, "name": "William Peace University", "priority": 1661, "external_id": null}, {"id": 10847484003, "name": "William Penn University", "priority": 1662, "external_id": null}, {"id": 10847485003, "name": "William Woods University", "priority": 1663, "external_id": null}, {"id": 10847486003, "name": "Williams Baptist College", "priority": 1664, "external_id": null}, {"id": 10847487003, "name": "Williams College", "priority": 1665, "external_id": null}, {"id": 10847488003, "name": "Wilmington College", "priority": 1666, "external_id": null}, {"id": 10847489003, "name": "Wilmington University", "priority": 1667, "external_id": null}, {"id": 10847490003, "name": "Wilson College", "priority": 1668, "external_id": null}, {"id": 10847491003, "name": "Wingate University", "priority": 1669, "external_id": null}, {"id": 10847492003, "name": "Winona State University", "priority": 1670, "external_id": null}, {"id": 10847493003, "name": "Winston-Salem State University", "priority": 1671, "external_id": null}, {"id": 10847494003, "name": "Winthrop University", "priority": 1672, "external_id": null}, {"id": 10847495003, "name": "Wisconsin Lutheran College", "priority": 1673, "external_id": null}, {"id": 10847496003, "name": "Wittenberg University", "priority": 1674, "external_id": null}, {"id": 10847497003, "name": "Woodbury University", "priority": 1675, "external_id": null}, {"id": 10847498003, "name": "Worcester Polytechnic Institute", "priority": 1676, "external_id": null}, {"id": 10847499003, "name": "Worcester State University", "priority": 1677, "external_id": null}, {"id": 10847500003, "name": "Wright State University", "priority": 1678, "external_id": null}, {"id": 10847501003, "name": "Xavier University", "priority": 1679, "external_id": null}, {"id": 10847502003, "name": "Xavier University of Louisiana", "priority": 1680, "external_id": null}, {"id": 10847503003, "name": "Yeshiva University", "priority": 1681, "external_id": null}, {"id": 10847504003, "name": "York College", "priority": 1682, "external_id": null}, {"id": 10847505003, "name": "York College of Pennsylvania", "priority": 1683, "external_id": null}, {"id": 10847506003, "name": "York University", "priority": 1684, "external_id": null}, {"id": 10847507003, "name": "University of Cambridge", "priority": 1685, "external_id": null}, {"id": 10847508003, "name": "UCL (University College London)", "priority": 1686, "external_id": null}, {"id": 10847509003, "name": "Imperial College London", "priority": 1687, "external_id": null}, {"id": 10847510003, "name": "University of Oxford", "priority": 1688, "external_id": null}, {"id": 10847511003, "name": "ETH Zurich (Swiss Federal Institute of Technology)", "priority": 1689, "external_id": null}, {"id": 10847512003, "name": "University of Edinburgh", "priority": 1690, "external_id": null}, {"id": 10847513003, "name": "Ecole Polytechnique F\u00e9d\u00e9rale de Lausanne", "priority": 1691, "external_id": null}, {"id": 10847514003, "name": "King's College London (KCL)", "priority": 1692, "external_id": null}, {"id": 10847515003, "name": "National University of Singapore (NUS)", "priority": 1693, "external_id": null}, {"id": 10847516003, "name": "University of Hong Kong", "priority": 1694, "external_id": null}, {"id": 10847517003, "name": "Australian National University", "priority": 1695, "external_id": null}, {"id": 10847518003, "name": "Ecole normale sup\u00e9rieure, Paris", "priority": 1696, "external_id": null}, {"id": 10847519003, "name": "University of Bristol", "priority": 1697, "external_id": null}, {"id": 10847520003, "name": "The University of Melbourne", "priority": 1698, "external_id": null}, {"id": 10847521003, "name": "The University of Tokyo", "priority": 1699, "external_id": null}, {"id": 10847522003, "name": "The University of Manchester", "priority": 1700, "external_id": null}, {"id": 10847523003, "name": "Western Illinois University", "priority": 1701, "external_id": null}, {"id": 10847524003, "name": "Wofford College", "priority": 1702, "external_id": null}, {"id": 10847525003, "name": "Western Carolina University", "priority": 1703, "external_id": null}, {"id": 10847526003, "name": "West Virginia University", "priority": 1704, "external_id": null}, {"id": 10847527003, "name": "Yale University", "priority": 1705, "external_id": null}, {"id": 10847528003, "name": "The Hong Kong University of Science and Technology", "priority": 1706, "external_id": null}, {"id": 10847529003, "name": "Kyoto University", "priority": 1707, "external_id": null}, {"id": 10847530003, "name": "Seoul National University", "priority": 1708, "external_id": null}, {"id": 10847531003, "name": "The University of Sydney", "priority": 1709, "external_id": null}, {"id": 10847532003, "name": "The Chinese University of Hong Kong", "priority": 1710, "external_id": null}, {"id": 10847533003, "name": "Ecole Polytechnique", "priority": 1711, "external_id": null}, {"id": 10847534003, "name": "Nanyang Technological University (NTU)", "priority": 1712, "external_id": null}, {"id": 10847535003, "name": "The University of Queensland", "priority": 1713, "external_id": null}, {"id": 10847536003, "name": "University of Copenhagen", "priority": 1714, "external_id": null}, {"id": 10847537003, "name": "Peking University", "priority": 1715, "external_id": null}, {"id": 10847538003, "name": "Tsinghua University", "priority": 1716, "external_id": null}, {"id": 10847539003, "name": "Ruprecht-Karls-Universit\u00e4t Heidelberg", "priority": 1717, "external_id": null}, {"id": 10847540003, "name": "University of Glasgow", "priority": 1718, "external_id": null}, {"id": 10847541003, "name": "The University of New South Wales", "priority": 1719, "external_id": null}, {"id": 10847542003, "name": "Technische Universit\u00e4t M\u00fcnchen", "priority": 1720, "external_id": null}, {"id": 10847543003, "name": "Osaka University", "priority": 1721, "external_id": null}, {"id": 10847544003, "name": "University of Amsterdam", "priority": 1722, "external_id": null}, {"id": 10847545003, "name": "KAIST - Korea Advanced Institute of Science & Technology", "priority": 1723, "external_id": null}, {"id": 10847546003, "name": "Trinity College Dublin", "priority": 1724, "external_id": null}, {"id": 10847547003, "name": "University of Birmingham", "priority": 1725, "external_id": null}, {"id": 10847548003, "name": "The University of Warwick", "priority": 1726, "external_id": null}, {"id": 10847549003, "name": "Ludwig-Maximilians-Universit\u00e4t M\u00fcnchen", "priority": 1727, "external_id": null}, {"id": 10847550003, "name": "Tokyo Institute of Technology", "priority": 1728, "external_id": null}, {"id": 10847551003, "name": "Lund University", "priority": 1729, "external_id": null}, {"id": 10847552003, "name": "London School of Economics and Political Science (LSE)", "priority": 1730, "external_id": null}, {"id": 10847553003, "name": "Monash University", "priority": 1731, "external_id": null}, {"id": 10847554003, "name": "University of Helsinki", "priority": 1732, "external_id": null}, {"id": 10847555003, "name": "The University of Sheffield", "priority": 1733, "external_id": null}, {"id": 10847556003, "name": "University of Geneva", "priority": 1734, "external_id": null}, {"id": 10847557003, "name": "Leiden University", "priority": 1735, "external_id": null}, {"id": 10847558003, "name": "The University of Nottingham", "priority": 1736, "external_id": null}, {"id": 10847559003, "name": "Tohoku University", "priority": 1737, "external_id": null}, {"id": 10847560003, "name": "KU Leuven", "priority": 1738, "external_id": null}, {"id": 10847561003, "name": "University of Zurich", "priority": 1739, "external_id": null}, {"id": 10847562003, "name": "Uppsala University", "priority": 1740, "external_id": null}, {"id": 10847563003, "name": "Utrecht University", "priority": 1741, "external_id": null}, {"id": 10847564003, "name": "National Taiwan University (NTU)", "priority": 1742, "external_id": null}, {"id": 10847565003, "name": "University of St Andrews", "priority": 1743, "external_id": null}, {"id": 10847566003, "name": "The University of Western Australia", "priority": 1744, "external_id": null}, {"id": 10847567003, "name": "University of Southampton", "priority": 1745, "external_id": null}, {"id": 10847568003, "name": "Fudan University", "priority": 1746, "external_id": null}, {"id": 10847569003, "name": "University of Oslo", "priority": 1747, "external_id": null}, {"id": 10847570003, "name": "Durham University", "priority": 1748, "external_id": null}, {"id": 10847571003, "name": "Aarhus University", "priority": 1749, "external_id": null}, {"id": 10847572003, "name": "Erasmus University Rotterdam", "priority": 1750, "external_id": null}, {"id": 10847573003, "name": "Universit\u00e9 de Montr\u00e9al", "priority": 1751, "external_id": null}, {"id": 10847574003, "name": "The University of Auckland", "priority": 1752, "external_id": null}, {"id": 10847575003, "name": "Delft University of Technology", "priority": 1753, "external_id": null}, {"id": 10847576003, "name": "University of Groningen", "priority": 1754, "external_id": null}, {"id": 10847577003, "name": "University of Leeds", "priority": 1755, "external_id": null}, {"id": 10847578003, "name": "Nagoya University", "priority": 1756, "external_id": null}, {"id": 10847579003, "name": "Universit\u00e4t Freiburg", "priority": 1757, "external_id": null}, {"id": 10847580003, "name": "City University of Hong Kong", "priority": 1758, "external_id": null}, {"id": 10847581003, "name": "The University of Adelaide", "priority": 1759, "external_id": null}, {"id": 10847582003, "name": "Pohang University of Science And Technology (POSTECH)", "priority": 1760, "external_id": null}, {"id": 10847583003, "name": "Freie Universit\u00e4t Berlin", "priority": 1761, "external_id": null}, {"id": 10847584003, "name": "University of Basel", "priority": 1762, "external_id": null}, {"id": 10847585003, "name": "University of Lausanne", "priority": 1763, "external_id": null}, {"id": 10847586003, "name": "Universit\u00e9 Pierre et Marie Curie (UPMC)", "priority": 1764, "external_id": null}, {"id": 10847587003, "name": "Yonsei University", "priority": 1765, "external_id": null}, {"id": 10847588003, "name": "University of York", "priority": 1766, "external_id": null}, {"id": 10847589003, "name": "Queen Mary, University of London (QMUL)", "priority": 1767, "external_id": null}, {"id": 10847590003, "name": "Karlsruhe Institute of Technology (KIT)", "priority": 1768, "external_id": null}, {"id": 10847591003, "name": "KTH, Royal Institute of Technology", "priority": 1769, "external_id": null}, {"id": 10847592003, "name": "Lomonosov Moscow State University", "priority": 1770, "external_id": null}, {"id": 10847593003, "name": "Maastricht University", "priority": 1771, "external_id": null}, {"id": 10847594003, "name": "University of Ghent", "priority": 1772, "external_id": null}, {"id": 10847595003, "name": "Shanghai Jiao Tong University", "priority": 1773, "external_id": null}, {"id": 10847596003, "name": "Humboldt-Universit\u00e4t zu Berlin", "priority": 1774, "external_id": null}, {"id": 10847597003, "name": "Universidade de S\u00e3o Paulo (USP)", "priority": 1775, "external_id": null}, {"id": 10847598003, "name": "Georg-August-Universit\u00e4t G\u00f6ttingen", "priority": 1776, "external_id": null}, {"id": 10847599003, "name": "Newcastle University", "priority": 1777, "external_id": null}, {"id": 10847600003, "name": "University of Liverpool", "priority": 1778, "external_id": null}, {"id": 10847601003, "name": "Kyushu University", "priority": 1779, "external_id": null}, {"id": 10847602003, "name": "Eberhard Karls Universit\u00e4t T\u00fcbingen", "priority": 1780, "external_id": null}, {"id": 10847603003, "name": "Technical University of Denmark", "priority": 1781, "external_id": null}, {"id": 10847604003, "name": "Cardiff University", "priority": 1782, "external_id": null}, {"id": 10847605003, "name": "Universit\u00e9 Catholique de Louvain (UCL)", "priority": 1783, "external_id": null}, {"id": 10847606003, "name": "University College Dublin", "priority": 1784, "external_id": null}, {"id": 10847607003, "name": "McMaster University", "priority": 1785, "external_id": null}, {"id": 10847608003, "name": "Hebrew University of Jerusalem", "priority": 1786, "external_id": null}, {"id": 10847609003, "name": "Radboud University Nijmegen", "priority": 1787, "external_id": null}, {"id": 10847610003, "name": "Hokkaido University", "priority": 1788, "external_id": null}, {"id": 10847611003, "name": "Korea University", "priority": 1789, "external_id": null}, {"id": 10847612003, "name": "University of Cape Town", "priority": 1790, "external_id": null}, {"id": 10847613003, "name": "Rheinisch-Westf\u00e4lische Technische Hochschule Aachen", "priority": 1791, "external_id": null}, {"id": 10847614003, "name": "University of Aberdeen", "priority": 1792, "external_id": null}, {"id": 10847615003, "name": "Wageningen University", "priority": 1793, "external_id": null}, {"id": 10847616003, "name": "University of Bergen", "priority": 1794, "external_id": null}, {"id": 10847617003, "name": "University of Bern", "priority": 1795, "external_id": null}, {"id": 10847618003, "name": "University of Otago", "priority": 1796, "external_id": null}, {"id": 10847619003, "name": "Lancaster University", "priority": 1797, "external_id": null}, {"id": 10847620003, "name": "Eindhoven University of Technology", "priority": 1798, "external_id": null}, {"id": 10847621003, "name": "Ecole Normale Sup\u00e9rieure de Lyon", "priority": 1799, "external_id": null}, {"id": 10847622003, "name": "University of Vienna", "priority": 1800, "external_id": null}, {"id": 10847623003, "name": "The Hong Kong Polytechnic University", "priority": 1801, "external_id": null}, {"id": 10847624003, "name": "Sungkyunkwan University", "priority": 1802, "external_id": null}, {"id": 10847625003, "name": "Rheinische Friedrich-Wilhelms-Universit\u00e4t Bonn", "priority": 1803, "external_id": null}, {"id": 10847626003, "name": "Universidad Nacional Aut\u00f3noma de M\u00e9xico (UNAM)", "priority": 1804, "external_id": null}, {"id": 10847627003, "name": "Zhejiang University", "priority": 1805, "external_id": null}, {"id": 10847628003, "name": "Pontificia Universidad Cat\u00f3lica de Chile", "priority": 1806, "external_id": null}, {"id": 10847629003, "name": "Universiti Malaya (UM)", "priority": 1807, "external_id": null}, {"id": 10847630003, "name": "Universit\u00e9 Libre de Bruxelles (ULB)", "priority": 1808, "external_id": null}, {"id": 10847631003, "name": "University of Exeter", "priority": 1809, "external_id": null}, {"id": 10847632003, "name": "Stockholm University", "priority": 1810, "external_id": null}, {"id": 10847633003, "name": "Queen's University of Belfast", "priority": 1811, "external_id": null}, {"id": 10847634003, "name": "Vrije Universiteit Brussel (VUB)", "priority": 1812, "external_id": null}, {"id": 10847635003, "name": "University of Science and Technology of China", "priority": 1813, "external_id": null}, {"id": 10847636003, "name": "Nanjing University", "priority": 1814, "external_id": null}, {"id": 10847637003, "name": "Universitat Aut\u00f3noma de Barcelona", "priority": 1815, "external_id": null}, {"id": 10847638003, "name": "University of Barcelona", "priority": 1816, "external_id": null}, {"id": 10847639003, "name": "VU University Amsterdam", "priority": 1817, "external_id": null}, {"id": 10847640003, "name": "Technion - Israel Institute of Technology", "priority": 1818, "external_id": null}, {"id": 10847641003, "name": "Technische Universit\u00e4t Berlin", "priority": 1819, "external_id": null}, {"id": 10847642003, "name": "University of Antwerp", "priority": 1820, "external_id": null}, {"id": 10847643003, "name": "Universit\u00e4t Hamburg", "priority": 1821, "external_id": null}, {"id": 10847644003, "name": "University of Bath", "priority": 1822, "external_id": null}, {"id": 10847645003, "name": "University of Bologna", "priority": 1823, "external_id": null}, {"id": 10847646003, "name": "Queen's University, Ontario", "priority": 1824, "external_id": null}, {"id": 10847647003, "name": "Universit\u00e9 Paris-Sud 11", "priority": 1825, "external_id": null}, {"id": 10847648003, "name": "Keio University", "priority": 1826, "external_id": null}, {"id": 10847649003, "name": "University of Sussex", "priority": 1827, "external_id": null}, {"id": 10847650003, "name": "Universidad Aut\u00f3noma de Madrid", "priority": 1828, "external_id": null}, {"id": 10847651003, "name": "Aalto University", "priority": 1829, "external_id": null}, {"id": 10847652003, "name": "Sapienza University of Rome", "priority": 1830, "external_id": null}, {"id": 10847653003, "name": "Tel Aviv University", "priority": 1831, "external_id": null}, {"id": 10847654003, "name": "National Tsing Hua University", "priority": 1832, "external_id": null}, {"id": 10847655003, "name": "Chalmers University of Technology", "priority": 1833, "external_id": null}, {"id": 10847656003, "name": "University of Leicester", "priority": 1834, "external_id": null}, {"id": 10847657003, "name": "Universit\u00e9 Paris Diderot - Paris 7", "priority": 1835, "external_id": null}, {"id": 10847658003, "name": "University of Gothenburg", "priority": 1836, "external_id": null}, {"id": 10847659003, "name": "University of Turku", "priority": 1837, "external_id": null}, {"id": 10847660003, "name": "Universit\u00e4t Frankfurt am Main", "priority": 1838, "external_id": null}, {"id": 10847661003, "name": "Universidad de Buenos Aires", "priority": 1839, "external_id": null}, {"id": 10847662003, "name": "University College Cork", "priority": 1840, "external_id": null}, {"id": 10847663003, "name": "University of Tsukuba", "priority": 1841, "external_id": null}, {"id": 10847664003, "name": "University of Reading", "priority": 1842, "external_id": null}, {"id": 10847665003, "name": "Sciences Po Paris", "priority": 1843, "external_id": null}, {"id": 10847666003, "name": "Universidade Estadual de Campinas", "priority": 1844, "external_id": null}, {"id": 10847667003, "name": "King Fahd University of Petroleum & Minerals", "priority": 1845, "external_id": null}, {"id": 10847668003, "name": "University Complutense Madrid", "priority": 1846, "external_id": null}, {"id": 10847669003, "name": "Universit\u00e9 Paris-Sorbonne (Paris IV)", "priority": 1847, "external_id": null}, {"id": 10847670003, "name": "University of Dundee", "priority": 1848, "external_id": null}, {"id": 10847671003, "name": "Universit\u00e9 Joseph Fourier - Grenoble 1", "priority": 1849, "external_id": null}, {"id": 10847672003, "name": "Waseda University", "priority": 1850, "external_id": null}, {"id": 10847673003, "name": "Indian Institute of Technology Delhi (IITD)", "priority": 1851, "external_id": null}, {"id": 10847674003, "name": "Universidad de Chile", "priority": 1852, "external_id": null}, {"id": 10847675003, "name": "Universit\u00e9 Paris 1 Panth\u00e9on-Sorbonne", "priority": 1853, "external_id": null}, {"id": 10847676003, "name": "Universit\u00e9 de Strasbourg", "priority": 1854, "external_id": null}, {"id": 10847677003, "name": "University of Twente", "priority": 1855, "external_id": null}, {"id": 10847678003, "name": "University of East Anglia (UEA)", "priority": 1856, "external_id": null}, {"id": 10847679003, "name": "National Chiao Tung University", "priority": 1857, "external_id": null}, {"id": 10847680003, "name": "Politecnico di Milano", "priority": 1858, "external_id": null}, {"id": 10847681003, "name": "Charles University", "priority": 1859, "external_id": null}, {"id": 10847682003, "name": "Indian Institute of Technology Bombay (IITB)", "priority": 1860, "external_id": null}, {"id": 10847683003, "name": "University of Milano", "priority": 1861, "external_id": null}, {"id": 10847684003, "name": "Westf\u00e4lische Wilhelms-Universit\u00e4t M\u00fcnster", "priority": 1862, "external_id": null}, {"id": 10847685003, "name": "University of Canterbury", "priority": 1863, "external_id": null}, {"id": 10847686003, "name": "Chulalongkorn University", "priority": 1864, "external_id": null}, {"id": 10847687003, "name": "Saint-Petersburg State University", "priority": 1865, "external_id": null}, {"id": 10847688003, "name": "University of Liege", "priority": 1866, "external_id": null}, {"id": 10847689003, "name": "Universit\u00e4t zu K\u00f6ln", "priority": 1867, "external_id": null}, {"id": 10847690003, "name": "Loughborough University", "priority": 1868, "external_id": null}, {"id": 10847691003, "name": "National Cheng Kung University", "priority": 1869, "external_id": null}, {"id": 10847692003, "name": "Universit\u00e4t Stuttgart", "priority": 1870, "external_id": null}, {"id": 10847693003, "name": "Hanyang University", "priority": 1871, "external_id": null}, {"id": 10847694003, "name": "American University of Beirut (AUB)", "priority": 1872, "external_id": null}, {"id": 10847695003, "name": "Norwegian University of Science And Technology", "priority": 1873, "external_id": null}, {"id": 10847696003, "name": "Beijing Normal University", "priority": 1874, "external_id": null}, {"id": 10847697003, "name": "King Saud University", "priority": 1875, "external_id": null}, {"id": 10847698003, "name": "University of Oulu", "priority": 1876, "external_id": null}, {"id": 10847699003, "name": "Kyung Hee University", "priority": 1877, "external_id": null}, {"id": 10847700003, "name": "University of Strathclyde", "priority": 1878, "external_id": null}, {"id": 10847701003, "name": "Universit\u00e4t Ulm", "priority": 1879, "external_id": null}, {"id": 10847702003, "name": "University of Pisa", "priority": 1880, "external_id": null}, {"id": 10847703003, "name": "Technische Universit\u00e4t Darmstadt", "priority": 1881, "external_id": null}, {"id": 10847704003, "name": "Technische Universit\u00e4t Dresden", "priority": 1882, "external_id": null}, {"id": 10847705003, "name": "Macquarie University", "priority": 1883, "external_id": null}, {"id": 10847706003, "name": "Vienna University of Technology", "priority": 1884, "external_id": null}, {"id": 10847707003, "name": "Royal Holloway University of London", "priority": 1885, "external_id": null}, {"id": 10847708003, "name": "Victoria University of Wellington", "priority": 1886, "external_id": null}, {"id": 10847709003, "name": "University of Padua", "priority": 1887, "external_id": null}, {"id": 10847710003, "name": "Universiti Kebangsaan Malaysia (UKM)", "priority": 1888, "external_id": null}, {"id": 10847711003, "name": "University of Technology, Sydney", "priority": 1889, "external_id": null}, {"id": 10847712003, "name": "Universit\u00e4t Konstanz", "priority": 1890, "external_id": null}, {"id": 10847713003, "name": "Universidad de Los Andes Colombia", "priority": 1891, "external_id": null}, {"id": 10847714003, "name": "Universit\u00e9 Paris Descartes", "priority": 1892, "external_id": null}, {"id": 10847715003, "name": "Tokyo Medical and Dental University", "priority": 1893, "external_id": null}, {"id": 10847716003, "name": "University of Wollongong", "priority": 1894, "external_id": null}, {"id": 10847717003, "name": "Universit\u00e4t Erlangen-N\u00fcrnberg", "priority": 1895, "external_id": null}, {"id": 10847718003, "name": "Queensland University of Technology", "priority": 1896, "external_id": null}, {"id": 10847719003, "name": "Tecnol\u00f3gico de Monterrey (ITESM)", "priority": 1897, "external_id": null}, {"id": 10847720003, "name": "Universit\u00e4t Mannheim", "priority": 1898, "external_id": null}, {"id": 10847721003, "name": "Universitat Pompeu Fabra", "priority": 1899, "external_id": null}, {"id": 10847722003, "name": "Mahidol University", "priority": 1900, "external_id": null}, {"id": 10847723003, "name": "Curtin University", "priority": 1901, "external_id": null}, {"id": 10847724003, "name": "National University of Ireland, Galway", "priority": 1902, "external_id": null}, {"id": 10847725003, "name": "Universidade Federal do Rio de Janeiro", "priority": 1903, "external_id": null}, {"id": 10847726003, "name": "University of Surrey", "priority": 1904, "external_id": null}, {"id": 10847727003, "name": "Hong Kong Baptist University", "priority": 1905, "external_id": null}, {"id": 10847728003, "name": "Ume\u00e5 University", "priority": 1906, "external_id": null}, {"id": 10847729003, "name": "Universit\u00e4t Innsbruck", "priority": 1907, "external_id": null}, {"id": 10847730003, "name": "RMIT University", "priority": 1908, "external_id": null}, {"id": 10847731003, "name": "University of Eastern Finland", "priority": 1909, "external_id": null}, {"id": 10847732003, "name": "Christian-Albrechts-Universit\u00e4t zu Kiel", "priority": 1910, "external_id": null}, {"id": 10847733003, "name": "Indian Institute of Technology Kanpur (IITK)", "priority": 1911, "external_id": null}, {"id": 10847734003, "name": "National Yang Ming University", "priority": 1912, "external_id": null}, {"id": 10847735003, "name": "Johannes Gutenberg Universit\u00e4t Mainz", "priority": 1913, "external_id": null}, {"id": 10847736003, "name": "The University of Newcastle", "priority": 1914, "external_id": null}, {"id": 10847737003, "name": "Al-Farabi Kazakh National University", "priority": 1915, "external_id": null}, {"id": 10847738003, "name": "\u00c9cole des Ponts ParisTech", "priority": 1916, "external_id": null}, {"id": 10847739003, "name": "University of Jyv\u00e4skyl\u00e4", "priority": 1917, "external_id": null}, {"id": 10847740003, "name": "L.N. Gumilyov Eurasian National University", "priority": 1918, "external_id": null}, {"id": 10847741003, "name": "Kobe University", "priority": 1919, "external_id": null}, {"id": 10847742003, "name": "University of Tromso", "priority": 1920, "external_id": null}, {"id": 10847743003, "name": "Hiroshima University", "priority": 1921, "external_id": null}, {"id": 10847744003, "name": "Universit\u00e9 Bordeaux 1, Sciences Technologies", "priority": 1922, "external_id": null}, {"id": 10847745003, "name": "University of Indonesia", "priority": 1923, "external_id": null}, {"id": 10847746003, "name": "Universit\u00e4t Leipzig", "priority": 1924, "external_id": null}, {"id": 10847747003, "name": "University of Southern Denmark", "priority": 1925, "external_id": null}, {"id": 10847748003, "name": "Indian Institute of Technology Madras (IITM)", "priority": 1926, "external_id": null}, {"id": 10847749003, "name": "University of The Witwatersrand", "priority": 1927, "external_id": null}, {"id": 10847750003, "name": "University of Navarra", "priority": 1928, "external_id": null}, {"id": 10847751003, "name": "Universidad Austral - Argentina", "priority": 1929, "external_id": null}, {"id": 10847752003, "name": "Universidad Carlos III de Madrid", "priority": 1930, "external_id": null}, {"id": 10847753003, "name": "Universit\u00e0\u00a1 degli Studi di Roma - Tor Vergata", "priority": 1931, "external_id": null}, {"id": 10847754003, "name": "Pontificia Universidad Cat\u00f3lica Argentina Santa Mar\u00eda de los Buenos Aires", "priority": 1932, "external_id": null}, {"id": 10847755003, "name": "UCA", "priority": 1933, "external_id": null}, {"id": 10847756003, "name": "Julius-Maximilians-Universit\u00e4t W\u00fcrzburg", "priority": 1934, "external_id": null}, {"id": 10847757003, "name": "Universidad Nacional de Colombia", "priority": 1935, "external_id": null}, {"id": 10847758003, "name": "Laval University", "priority": 1936, "external_id": null}, {"id": 10847759003, "name": "Ben Gurion University of The Negev", "priority": 1937, "external_id": null}, {"id": 10847760003, "name": "Link\u00f6ping University", "priority": 1938, "external_id": null}, {"id": 10847761003, "name": "Aalborg University", "priority": 1939, "external_id": null}, {"id": 10847762003, "name": "Bauman Moscow State Technical University", "priority": 1940, "external_id": null}, {"id": 10847763003, "name": "Ecole Normale Sup\u00e9rieure de Cachan", "priority": 1941, "external_id": null}, {"id": 10847764003, "name": "SOAS - School of Oriental and African Studies, University of London", "priority": 1942, "external_id": null}, {"id": 10847765003, "name": "University of Essex", "priority": 1943, "external_id": null}, {"id": 10847766003, "name": "University of Warsaw", "priority": 1944, "external_id": null}, {"id": 10847767003, "name": "Griffith University", "priority": 1945, "external_id": null}, {"id": 10847768003, "name": "University of South Australia", "priority": 1946, "external_id": null}, {"id": 10847769003, "name": "Massey University", "priority": 1947, "external_id": null}, {"id": 10847770003, "name": "University of Porto", "priority": 1948, "external_id": null}, {"id": 10847771003, "name": "Universitat Polit\u00e8cnica de Catalunya", "priority": 1949, "external_id": null}, {"id": 10847772003, "name": "Indian Institute of Technology Kharagpur (IITKGP)", "priority": 1950, "external_id": null}, {"id": 10847773003, "name": "City University London", "priority": 1951, "external_id": null}, {"id": 10847774003, "name": "Dublin City University", "priority": 1952, "external_id": null}, {"id": 10847775003, "name": "Pontificia Universidad Javeriana", "priority": 1953, "external_id": null}, {"id": 10847776003, "name": "James Cook University", "priority": 1954, "external_id": null}, {"id": 10847777003, "name": "Novosibirsk State University", "priority": 1955, "external_id": null}, {"id": 10847778003, "name": "Universidade Nova de Lisboa", "priority": 1956, "external_id": null}, {"id": 10847779003, "name": "Universit\u00e9 Aix-Marseille", "priority": 1957, "external_id": null}, {"id": 10847780003, "name": "Universiti Sains Malaysia (USM)", "priority": 1958, "external_id": null}, {"id": 10847781003, "name": "Universiti Teknologi Malaysia (UTM)", "priority": 1959, "external_id": null}, {"id": 10847782003, "name": "Universit\u00e9 Paris Dauphine", "priority": 1960, "external_id": null}, {"id": 10847783003, "name": "University of Coimbra", "priority": 1961, "external_id": null}, {"id": 10847784003, "name": "Brunel University", "priority": 1962, "external_id": null}, {"id": 10847785003, "name": "King Abdul Aziz University (KAU)", "priority": 1963, "external_id": null}, {"id": 10847786003, "name": "Ewha Womans University", "priority": 1964, "external_id": null}, {"id": 10847787003, "name": "Nankai University", "priority": 1965, "external_id": null}, {"id": 10847788003, "name": "Taipei Medical University", "priority": 1966, "external_id": null}, {"id": 10847789003, "name": "Universit\u00e4t Jena", "priority": 1967, "external_id": null}, {"id": 10847790003, "name": "Ruhr-Universit\u00e4t Bochum", "priority": 1968, "external_id": null}, {"id": 10847791003, "name": "Heriot-Watt University", "priority": 1969, "external_id": null}, {"id": 10847792003, "name": "Politecnico di Torino", "priority": 1970, "external_id": null}, {"id": 10847793003, "name": "Universit\u00e4t Bremen", "priority": 1971, "external_id": null}, {"id": 10847794003, "name": "Xi'an Jiaotong University", "priority": 1972, "external_id": null}, {"id": 10847795003, "name": "Birkbeck College, University of London", "priority": 1973, "external_id": null}, {"id": 10847796003, "name": "Oxford Brookes University", "priority": 1974, "external_id": null}, {"id": 10847797003, "name": "Jagiellonian University", "priority": 1975, "external_id": null}, {"id": 10847798003, "name": "University of Tampere", "priority": 1976, "external_id": null}, {"id": 10847799003, "name": "University of Florence", "priority": 1977, "external_id": null}, {"id": 10847800003, "name": "Deakin University", "priority": 1978, "external_id": null}, {"id": 10847801003, "name": "University of the Philippines", "priority": 1979, "external_id": null}, {"id": 10847802003, "name": "Universitat Polit\u00e8cnica de Val\u00e8ncia", "priority": 1980, "external_id": null}, {"id": 10847803003, "name": "Sun Yat-sen University", "priority": 1981, "external_id": null}, {"id": 10847804003, "name": "Universit\u00e9 Montpellier 2, Sciences et Techniques du Languedoc", "priority": 1982, "external_id": null}, {"id": 10847805003, "name": "Moscow State Institute of International Relations (MGIMO-University)", "priority": 1983, "external_id": null}, {"id": 10847806003, "name": "Stellenbosch University", "priority": 1984, "external_id": null}, {"id": 10847807003, "name": "Polit\u00e9cnica de Madrid", "priority": 1985, "external_id": null}, {"id": 10847808003, "name": "Instituto Tecnol\u00f3gico de Buenos Aires (ITBA)", "priority": 1986, "external_id": null}, {"id": 10847809003, "name": "La Trobe University", "priority": 1987, "external_id": null}, {"id": 10847810003, "name": "Universit\u00e9 Paul Sabatier Toulouse III", "priority": 1988, "external_id": null}, {"id": 10847811003, "name": "Karl-Franzens-Universit\u00e4t Graz", "priority": 1989, "external_id": null}, {"id": 10847812003, "name": "Universit\u00e4t D\u00fcsseldorf", "priority": 1990, "external_id": null}, {"id": 10847813003, "name": "University of Naples - Federico Ii", "priority": 1991, "external_id": null}, {"id": 10847814003, "name": "Aston University", "priority": 1992, "external_id": null}, {"id": 10847815003, "name": "University of Turin", "priority": 1993, "external_id": null}, {"id": 10847816003, "name": "Beihang University (former BUAA)", "priority": 1994, "external_id": null}, {"id": 10847817003, "name": "Indian Institute of Technology Roorkee (IITR)", "priority": 1995, "external_id": null}, {"id": 10847818003, "name": "National Central University", "priority": 1996, "external_id": null}, {"id": 10847819003, "name": "Sogang University", "priority": 1997, "external_id": null}, {"id": 10847820003, "name": "Universit\u00e4t Regensburg", "priority": 1998, "external_id": null}, {"id": 10847821003, "name": "Universit\u00e9 Lille 1, Sciences et Technologie", "priority": 1999, "external_id": null}, {"id": 10847822003, "name": "University of Tasmania", "priority": 2000, "external_id": null}, {"id": 10847823003, "name": "University of Waikato", "priority": 2001, "external_id": null}, {"id": 10847824003, "name": "Wuhan University", "priority": 2002, "external_id": null}, {"id": 10847825003, "name": "National Taiwan University of Science And Technology", "priority": 2003, "external_id": null}, {"id": 10847826003, "name": "Universidade Federal de S\u00e3o Paulo (UNIFESP)", "priority": 2004, "external_id": null}, {"id": 10847827003, "name": "Universit\u00e0 degli Studi di Pavia", "priority": 2005, "external_id": null}, {"id": 10847828003, "name": "Universit\u00e4t Bayreuth", "priority": 2006, "external_id": null}, {"id": 10847829003, "name": "Universit\u00e9 Claude Bernard Lyon 1", "priority": 2007, "external_id": null}, {"id": 10847830003, "name": "Universit\u00e9 du Qu\u00e9bec", "priority": 2008, "external_id": null}, {"id": 10847831003, "name": "Universiti Putra Malaysia (UPM)", "priority": 2009, "external_id": null}, {"id": 10847832003, "name": "University of Kent", "priority": 2010, "external_id": null}, {"id": 10847833003, "name": "University of St Gallen (HSG)", "priority": 2011, "external_id": null}, {"id": 10847834003, "name": "Bond University", "priority": 2012, "external_id": null}, {"id": 10847835003, "name": "United Arab Emirates University", "priority": 2013, "external_id": null}, {"id": 10847836003, "name": "Universidad de San Andr\u00c3\u00a9s", "priority": 2014, "external_id": null}, {"id": 10847837003, "name": "Universidad Nacional de La Plata", "priority": 2015, "external_id": null}, {"id": 10847838003, "name": "Universit\u00e4t des Saarlandes", "priority": 2016, "external_id": null}, {"id": 10847839003, "name": "American University of Sharjah (AUS)", "priority": 2017, "external_id": null}, {"id": 10847840003, "name": "Bilkent University", "priority": 2018, "external_id": null}, {"id": 10847841003, "name": "Flinders University", "priority": 2019, "external_id": null}, {"id": 10847842003, "name": "Hankuk (Korea) University of Foreign Studies", "priority": 2020, "external_id": null}, {"id": 10847843003, "name": "Middle East Technical University", "priority": 2021, "external_id": null}, {"id": 10847844003, "name": "Philipps-Universit\u00e4t Marburg", "priority": 2022, "external_id": null}, {"id": 10847845003, "name": "Swansea University", "priority": 2023, "external_id": null}, {"id": 10847846003, "name": "Tampere University of Technology", "priority": 2024, "external_id": null}, {"id": 10847847003, "name": "Universit\u00e4t Bielefeld", "priority": 2025, "external_id": null}, {"id": 10847848003, "name": "University of Manitoba", "priority": 2026, "external_id": null}, {"id": 10847849003, "name": "Chiba University", "priority": 2027, "external_id": null}, {"id": 10847850003, "name": "Moscow Institute of Physics and Technology State University", "priority": 2028, "external_id": null}, {"id": 10847851003, "name": "Tallinn University of Technology", "priority": 2029, "external_id": null}, {"id": 10847852003, "name": "Taras Shevchenko National University of Kyiv", "priority": 2030, "external_id": null}, {"id": 10847853003, "name": "Tokyo University of Science", "priority": 2031, "external_id": null}, {"id": 10847854003, "name": "University of Salamanca", "priority": 2032, "external_id": null}, {"id": 10847855003, "name": "University of Trento", "priority": 2033, "external_id": null}, {"id": 10847856003, "name": "Universit\u00e9 de Sherbrooke", "priority": 2034, "external_id": null}, {"id": 10847857003, "name": "Universit\u00e9 Panth\u00e9on-Assas (Paris 2)", "priority": 2035, "external_id": null}, {"id": 10847858003, "name": "University of Delhi", "priority": 2036, "external_id": null}, {"id": 10847859003, "name": "Abo Akademi University", "priority": 2037, "external_id": null}, {"id": 10847860003, "name": "Czech Technical University In Prague", "priority": 2038, "external_id": null}, {"id": 10847861003, "name": "Leibniz Universit\u00e4t Hannover", "priority": 2039, "external_id": null}, {"id": 10847862003, "name": "Pusan National University", "priority": 2040, "external_id": null}, {"id": 10847863003, "name": "Shanghai University", "priority": 2041, "external_id": null}, {"id": 10847864003, "name": "St. Petersburg State Politechnical University", "priority": 2042, "external_id": null}, {"id": 10847865003, "name": "Universit\u00e0 Cattolica del Sacro Cuore", "priority": 2043, "external_id": null}, {"id": 10847866003, "name": "University of Genoa", "priority": 2044, "external_id": null}, {"id": 10847867003, "name": "Bandung Institute of Technology (ITB)", "priority": 2045, "external_id": null}, {"id": 10847868003, "name": "Bogazici University", "priority": 2046, "external_id": null}, {"id": 10847869003, "name": "Goldsmiths, University of London", "priority": 2047, "external_id": null}, {"id": 10847870003, "name": "National Sun Yat-sen University", "priority": 2048, "external_id": null}, {"id": 10847871003, "name": "Renmin (People\u2019s) University of China", "priority": 2049, "external_id": null}, {"id": 10847872003, "name": "Universidad de Costa Rica", "priority": 2050, "external_id": null}, {"id": 10847873003, "name": "Universidad de Santiago de Chile - USACH", "priority": 2051, "external_id": null}, {"id": 10847874003, "name": "University of Tartu", "priority": 2052, "external_id": null}, {"id": 10847875003, "name": "Aristotle University of Thessaloniki", "priority": 2053, "external_id": null}, {"id": 10847876003, "name": "Auckland University of Technology", "priority": 2054, "external_id": null}, {"id": 10847877003, "name": "Bangor University", "priority": 2055, "external_id": null}, {"id": 10847878003, "name": "Charles Darwin University", "priority": 2056, "external_id": null}, {"id": 10847879003, "name": "Kingston University, London", "priority": 2057, "external_id": null}, {"id": 10847880003, "name": "Universitat de Valencia", "priority": 2058, "external_id": null}, {"id": 10847881003, "name": "Universit\u00e9 Montpellier 1", "priority": 2059, "external_id": null}, {"id": 10847882003, "name": "University of Pretoria", "priority": 2060, "external_id": null}, {"id": 10847883003, "name": "Lincoln University", "priority": 2061, "external_id": null}, {"id": 10847884003, "name": "National Taiwan Normal University", "priority": 2062, "external_id": null}, {"id": 10847885003, "name": "National University of Sciences And Technology (NUST) Islamabad", "priority": 2063, "external_id": null}, {"id": 10847886003, "name": "Swinburne University of Technology", "priority": 2064, "external_id": null}, {"id": 10847887003, "name": "Tongji University", "priority": 2065, "external_id": null}, {"id": 10847888003, "name": "Universidad de Zaragoza", "priority": 2066, "external_id": null}, {"id": 10847889003, "name": "Universidade Federal de Minas Gerais", "priority": 2067, "external_id": null}, {"id": 10847890003, "name": "Universit\u00e4t Duisburg-Essen", "priority": 2068, "external_id": null}, {"id": 10847891003, "name": "Al-Imam Mohamed Ibn Saud Islamic University", "priority": 2069, "external_id": null}, {"id": 10847892003, "name": "Harbin Institute of Technology", "priority": 2070, "external_id": null}, {"id": 10847893003, "name": "People's Friendship University of Russia", "priority": 2071, "external_id": null}, {"id": 10847894003, "name": "Universidade Estadual PaulistaJ\u00falio de Mesquita Filho' (UNESP)", "priority": 2072, "external_id": null}, {"id": 10847895003, "name": "Universit\u00e9 Nice Sophia-Antipolis", "priority": 2073, "external_id": null}, {"id": 10847896003, "name": "University of Crete", "priority": 2074, "external_id": null}, {"id": 10847897003, "name": "University of Milano-Bicocca", "priority": 2075, "external_id": null}, {"id": 10847898003, "name": "Ateneo de Manila University", "priority": 2076, "external_id": null}, {"id": 10847899003, "name": "Beijing Institute of Technology", "priority": 2077, "external_id": null}, {"id": 10847900003, "name": "Chang Gung University", "priority": 2078, "external_id": null}, {"id": 10847901003, "name": "hung-Ang University", "priority": 2079, "external_id": null}, {"id": 10847902003, "name": "Dublin Institute of Technology", "priority": 2080, "external_id": null}, {"id": 10847903003, "name": "Huazhong University of Science and Technology", "priority": 2081, "external_id": null}, {"id": 10847904003, "name": "International Islamic University Malaysia (IIUM)", "priority": 2082, "external_id": null}, {"id": 10847905003, "name": "Johannes Kepler University Linz", "priority": 2083, "external_id": null}, {"id": 10847906003, "name": "Justus-Liebig-Universit\u00e4t Gie\u00dfen", "priority": 2084, "external_id": null}, {"id": 10847907003, "name": "Kanazawa University", "priority": 2085, "external_id": null}, {"id": 10847908003, "name": "Keele University", "priority": 2086, "external_id": null}, {"id": 10847909003, "name": "Koc University", "priority": 2087, "external_id": null}, {"id": 10847910003, "name": "National and Kapodistrian University of Athens", "priority": 2088, "external_id": null}, {"id": 10847911003, "name": "National Research University \u2013 Higher School of Economics (HSE)", "priority": 2089, "external_id": null}, {"id": 10847912003, "name": "National Technical University of Athens", "priority": 2090, "external_id": null}, {"id": 10847913003, "name": "Okayama University", "priority": 2091, "external_id": null}, {"id": 10847914003, "name": "Sabanci University", "priority": 2092, "external_id": null}, {"id": 10847915003, "name": "Southeast University", "priority": 2093, "external_id": null}, {"id": 10847916003, "name": "Sultan Qaboos University", "priority": 2094, "external_id": null}, {"id": 10847917003, "name": "Technische Universit\u00e4t Braunschweig", "priority": 2095, "external_id": null}, {"id": 10847918003, "name": "Technische Universit\u00e4t Dortmund", "priority": 2096, "external_id": null}, {"id": 10847919003, "name": "The Catholic University of Korea", "priority": 2097, "external_id": null}, {"id": 10847920003, "name": "Tianjin University", "priority": 2098, "external_id": null}, {"id": 10847921003, "name": "Tokyo Metropolitan University", "priority": 2099, "external_id": null}, {"id": 10847922003, "name": "Universidad de Antioquia", "priority": 2100, "external_id": null}, {"id": 10847923003, "name": "University of Granada", "priority": 2101, "external_id": null}, {"id": 10847924003, "name": "Universidad de Palermo", "priority": 2102, "external_id": null}, {"id": 10847925003, "name": "Universidad Nacional de C\u00f3rdoba", "priority": 2103, "external_id": null}, {"id": 10847926003, "name": "Universidade de Santiago de Compostela", "priority": 2104, "external_id": null}, {"id": 10847927003, "name": "Universidade Federal do Rio Grande Do Sul", "priority": 2105, "external_id": null}, {"id": 10847928003, "name": "University of Siena", "priority": 2106, "external_id": null}, {"id": 10847929003, "name": "University of Trieste", "priority": 2107, "external_id": null}, {"id": 10847930003, "name": "Universitas Gadjah Mada", "priority": 2108, "external_id": null}, {"id": 10847931003, "name": "Universit\u00e9 de Lorraine", "priority": 2109, "external_id": null}, {"id": 10847932003, "name": "Universit\u00e9 de Rennes 1", "priority": 2110, "external_id": null}, {"id": 10847933003, "name": "University of Bradford", "priority": 2111, "external_id": null}, {"id": 10847934003, "name": "University of Hull", "priority": 2112, "external_id": null}, {"id": 10847935003, "name": "University of Kwazulu-Natal", "priority": 2113, "external_id": null}, {"id": 10847936003, "name": "University of Limerick", "priority": 2114, "external_id": null}, {"id": 10847937003, "name": "University of Stirling", "priority": 2115, "external_id": null}, {"id": 10847938003, "name": "University of Szeged", "priority": 2116, "external_id": null}, {"id": 10847939003, "name": "Ural Federal University", "priority": 2117, "external_id": null}, {"id": 10847940003, "name": "Xiamen University", "priority": 2118, "external_id": null}, {"id": 10847941003, "name": "Yokohama City University", "priority": 2119, "external_id": null}, {"id": 10847942003, "name": "Aberystwyth University", "priority": 2120, "external_id": null}, {"id": 10847943003, "name": "Belarus State University", "priority": 2121, "external_id": null}, {"id": 10847944003, "name": "Cairo University", "priority": 2122, "external_id": null}, {"id": 10847945003, "name": "Chiang Mai University", "priority": 2123, "external_id": null}, {"id": 10847946003, "name": "Chonbuk National University", "priority": 2124, "external_id": null}, {"id": 10847947003, "name": "E\u00f6tv\u00f6s Lor\u00e1nd University", "priority": 2125, "external_id": null}, {"id": 10847948003, "name": "Inha University", "priority": 2126, "external_id": null}, {"id": 10847949003, "name": "Instituto Polit\u00e9cnico Nacional (IPN)", "priority": 2127, "external_id": null}, {"id": 10847950003, "name": "Istanbul Technical University", "priority": 2128, "external_id": null}, {"id": 10847951003, "name": "Kumamoto University", "priority": 2129, "external_id": null}, {"id": 10847952003, "name": "Kyungpook National University", "priority": 2130, "external_id": null}, {"id": 10847953003, "name": "Lingnan University (Hong Kong)", "priority": 2131, "external_id": null}, {"id": 10847954003, "name": "Masaryk University", "priority": 2132, "external_id": null}, {"id": 10847955003, "name": "Murdoch University", "priority": 2133, "external_id": null}, {"id": 10847956003, "name": "Nagasaki University", "priority": 2134, "external_id": null}, {"id": 10847957003, "name": "National Chung Hsing University", "priority": 2135, "external_id": null}, {"id": 10847958003, "name": "National Taipei University of Technology", "priority": 2136, "external_id": null}, {"id": 10847959003, "name": "National University of Ireland Maynooth", "priority": 2137, "external_id": null}, {"id": 10847960003, "name": "Osaka City University", "priority": 2138, "external_id": null}, {"id": 10847961003, "name": "Pontificia Universidad Cat\u00f3lica del Per\u00fa", "priority": 2139, "external_id": null}, {"id": 10847962003, "name": "Pontificia Universidade Cat\u00f3lica de S\u00e3o Paulo (PUC -SP)", "priority": 2140, "external_id": null}, {"id": 10847963003, "name": "Pontificia Universidade Cat\u00f3lica do Rio de Janeiro (PUC - Rio)", "priority": 2141, "external_id": null}, {"id": 10847964003, "name": "Qatar University", "priority": 2142, "external_id": null}, {"id": 10847965003, "name": "Rhodes University", "priority": 2143, "external_id": null}, {"id": 10847966003, "name": "Tokyo University of Agriculture and Technology", "priority": 2144, "external_id": null}, {"id": 10847967003, "name": "Tomsk Polytechnic University", "priority": 2145, "external_id": null}, {"id": 10847968003, "name": "Tomsk State University", "priority": 2146, "external_id": null}, {"id": 10847969003, "name": "Umm Al-Qura University", "priority": 2147, "external_id": null}, {"id": 10847970003, "name": "Universidad Cat\u00f3lica Andr\u00e9s Bello - UCAB", "priority": 2148, "external_id": null}, {"id": 10847971003, "name": "Universidad Central de Venezuela - UCV", "priority": 2149, "external_id": null}, {"id": 10847972003, "name": "Universidad de Belgrano", "priority": 2150, "external_id": null}, {"id": 10847973003, "name": "Universidad de Concepci\u00f3n", "priority": 2151, "external_id": null}, {"id": 10847974003, "name": "Universidad de Sevilla", "priority": 2152, "external_id": null}, {"id": 10847975003, "name": "Universidade Catolica Portuguesa, Lisboa", "priority": 2153, "external_id": null}, {"id": 10847976003, "name": "Universidade de Brasilia (UnB)", "priority": 2154, "external_id": null}, {"id": 10847977003, "name": "University of Lisbon", "priority": 2155, "external_id": null}, {"id": 10847978003, "name": "University of Ljubljana", "priority": 2156, "external_id": null}, {"id": 10847979003, "name": "University of Seoul", "priority": 2157, "external_id": null}, {"id": 10847980003, "name": "Abu Dhabi University", "priority": 2158, "external_id": null}, {"id": 10847981003, "name": "Ain Shams University", "priority": 2159, "external_id": null}, {"id": 10847982003, "name": "Ajou University", "priority": 2160, "external_id": null}, {"id": 10847983003, "name": "De La Salle University", "priority": 2161, "external_id": null}, {"id": 10847984003, "name": "Dongguk University", "priority": 2162, "external_id": null}, {"id": 10847985003, "name": "Gifu University", "priority": 2163, "external_id": null}, {"id": 10847986003, "name": "Hacettepe University", "priority": 2164, "external_id": null}, {"id": 10847987003, "name": "Indian Institute of Technology Guwahati (IITG)", "priority": 2165, "external_id": null}, {"id": 10847988003, "name": "Jilin University", "priority": 2166, "external_id": null}, {"id": 10847989003, "name": "Kazan Federal University", "priority": 2167, "external_id": null}, {"id": 10847990003, "name": "King Khalid University", "priority": 2168, "external_id": null}, {"id": 10847991003, "name": "Martin-Luther-Universit\u00e4t Halle-Wittenberg", "priority": 2169, "external_id": null}, {"id": 10847992003, "name": "National Chengchi University", "priority": 2170, "external_id": null}, {"id": 10847993003, "name": "National Technical University of UkraineKyiv Polytechnic Institute'", "priority": 2171, "external_id": null}, {"id": 10847994003, "name": "Niigata University", "priority": 2172, "external_id": null}, {"id": 10847995003, "name": "Osaka Prefecture University", "priority": 2173, "external_id": null}, {"id": 10847996003, "name": "Paris Lodron University of Salzburg", "priority": 2174, "external_id": null}, {"id": 10847997003, "name": "Sharif University of Technology", "priority": 2175, "external_id": null}, {"id": 10847998003, "name": "Southern Federal University", "priority": 2176, "external_id": null}, {"id": 10847999003, "name": "Thammasat University", "priority": 2177, "external_id": null}, {"id": 10848000003, "name": "Universidad de Guadalajara (UDG)", "priority": 2178, "external_id": null}, {"id": 10848001003, "name": "Universidad de la Rep\u00fablica (UdelaR)", "priority": 2179, "external_id": null}, {"id": 10848002003, "name": "Universidad Iberoamericana (UIA)", "priority": 2180, "external_id": null}, {"id": 10848003003, "name": "Universidad Torcuato Di Tella", "priority": 2181, "external_id": null}, {"id": 10848004003, "name": "Universidade Federal da Bahia", "priority": 2182, "external_id": null}, {"id": 10848005003, "name": "Universidade Federal de S\u00e3o Carlos", "priority": 2183, "external_id": null}, {"id": 10848006003, "name": "Universidade Federal de Vi\u00e7osa", "priority": 2184, "external_id": null}, {"id": 10848007003, "name": "Perugia University", "priority": 2185, "external_id": null}, {"id": 10848008003, "name": "Universit\u00e9 de Nantes", "priority": 2186, "external_id": null}, {"id": 10848009003, "name": "Universit\u00e9 Saint-Joseph de Beyrouth", "priority": 2187, "external_id": null}, {"id": 10848010003, "name": "University of Canberra", "priority": 2188, "external_id": null}, {"id": 10848011003, "name": "University of Debrecen", "priority": 2189, "external_id": null}, {"id": 10848012003, "name": "University of Johannesburg", "priority": 2190, "external_id": null}, {"id": 10848013003, "name": "University of Mumbai", "priority": 2191, "external_id": null}, {"id": 10848014003, "name": "University of Patras", "priority": 2192, "external_id": null}, {"id": 10848015003, "name": "University of Tehran", "priority": 2193, "external_id": null}, {"id": 10848016003, "name": "University of Ulsan", "priority": 2194, "external_id": null}, {"id": 10848017003, "name": "University of Ulster", "priority": 2195, "external_id": null}, {"id": 10848018003, "name": "University of Zagreb", "priority": 2196, "external_id": null}, {"id": 10848019003, "name": "Vilnius University", "priority": 2197, "external_id": null}, {"id": 10848020003, "name": "Warsaw University of Technology", "priority": 2198, "external_id": null}, {"id": 10848021003, "name": "Al Azhar University", "priority": 2199, "external_id": null}, {"id": 10848022003, "name": "Bar-Ilan University", "priority": 2200, "external_id": null}, {"id": 10848023003, "name": "Brno University of Technology", "priority": 2201, "external_id": null}, {"id": 10848024003, "name": "Chonnam National University", "priority": 2202, "external_id": null}, {"id": 10848025003, "name": "Chungnam National University", "priority": 2203, "external_id": null}, {"id": 10848026003, "name": "Corvinus University of Budapest", "priority": 2204, "external_id": null}, {"id": 10848027003, "name": "Gunma University", "priority": 2205, "external_id": null}, {"id": 10848028003, "name": "Hallym University", "priority": 2206, "external_id": null}, {"id": 10848029003, "name": "Instituto Tecnol\u00f3gico Autonomo de M\u00e9xico (ITAM)", "priority": 2207, "external_id": null}, {"id": 10848030003, "name": "Istanbul University", "priority": 2208, "external_id": null}, {"id": 10848031003, "name": "Jordan University of Science & Technology", "priority": 2209, "external_id": null}, {"id": 10848032003, "name": "Kasetsart University", "priority": 2210, "external_id": null}, {"id": 10848033003, "name": "Kazakh-British Technical University", "priority": 2211, "external_id": null}, {"id": 10848034003, "name": "Khazar University", "priority": 2212, "external_id": null}, {"id": 10848035003, "name": "London Metropolitan University", "priority": 2213, "external_id": null}, {"id": 10848036003, "name": "Middlesex University", "priority": 2214, "external_id": null}, {"id": 10848037003, "name": "Universidad Industrial de Santander", "priority": 2215, "external_id": null}, {"id": 10848038003, "name": "Pontificia Universidad Cat\u00f3lica de Valpara\u00edso", "priority": 2216, "external_id": null}, {"id": 10848039003, "name": "Pontificia Universidade Cat\u00f3lica do Rio Grande do Sul", "priority": 2217, "external_id": null}, {"id": 10848040003, "name": "Qafqaz University", "priority": 2218, "external_id": null}, {"id": 10848041003, "name": "Ritsumeikan University", "priority": 2219, "external_id": null}, {"id": 10848042003, "name": "Shandong University", "priority": 2220, "external_id": null}, {"id": 10848043003, "name": "University of St. Kliment Ohridski", "priority": 2221, "external_id": null}, {"id": 10848044003, "name": "South Kazakhstan State University (SKSU)", "priority": 2222, "external_id": null}, {"id": 10848045003, "name": "Universidad Adolfo Ib\u00e1\u00f1ez", "priority": 2223, "external_id": null}, {"id": 10848046003, "name": "Universidad Aut\u00f3noma del Estado de M\u00e9xico", "priority": 2224, "external_id": null}, {"id": 10848047003, "name": "Universidad Aut\u00f3noma Metropolitana (UAM)", "priority": 2225, "external_id": null}, {"id": 10848048003, "name": "Universidad de Alcal\u00e1", "priority": 2226, "external_id": null}, {"id": 10848049003, "name": "Universidad Nacional Costa Rica", "priority": 2227, "external_id": null}, {"id": 10848050003, "name": "Universidad Nacional de Mar del Plata", "priority": 2228, "external_id": null}, {"id": 10848051003, "name": "Universidad Peruana Cayetano Heredia", "priority": 2229, "external_id": null}, {"id": 10848052003, "name": "Universidad Sim\u00f3n Bol\u00edvar Venezuela", "priority": 2230, "external_id": null}, {"id": 10848053003, "name": "Universidade Federal de Santa Catarina", "priority": 2231, "external_id": null}, {"id": 10848054003, "name": "Universidade Federal do Paran\u00e1 (UFPR)", "priority": 2232, "external_id": null}, {"id": 10848055003, "name": "Universidade Federal Fluminense", "priority": 2233, "external_id": null}, {"id": 10848056003, "name": "University of Modena", "priority": 2234, "external_id": null}, {"id": 10848057003, "name": "Universit\u00e9 Lumi\u00e8re Lyon 2", "priority": 2235, "external_id": null}, {"id": 10848058003, "name": "Universit\u00e9 Toulouse 1, Capitole", "priority": 2236, "external_id": null}, {"id": 10848059003, "name": "University of Economics Prague", "priority": 2237, "external_id": null}, {"id": 10848060003, "name": "University of Hertfordshire", "priority": 2238, "external_id": null}, {"id": 10848061003, "name": "University of Plymouth", "priority": 2239, "external_id": null}, {"id": 10848062003, "name": "University of Salford", "priority": 2240, "external_id": null}, {"id": 10848063003, "name": "University of Science and Technology Beijing", "priority": 2241, "external_id": null}, {"id": 10848064003, "name": "University of Western Sydney", "priority": 2242, "external_id": null}, {"id": 10848065003, "name": "Yamaguchi University", "priority": 2243, "external_id": null}, {"id": 10848066003, "name": "Yokohama National University", "priority": 2244, "external_id": null}, {"id": 10848067003, "name": "Airlangga University", "priority": 2245, "external_id": null}, {"id": 10848068003, "name": "Alexandria University", "priority": 2246, "external_id": null}, {"id": 10848069003, "name": "Alexandru Ioan Cuza University", "priority": 2247, "external_id": null}, {"id": 10848070003, "name": "Alpen-Adria-Universit\u00e4t Klagenfurt", "priority": 2248, "external_id": null}, {"id": 10848071003, "name": "Aoyama Gakuin University", "priority": 2249, "external_id": null}, {"id": 10848072003, "name": "Athens University of Economy And Business", "priority": 2250, "external_id": null}, {"id": 10848073003, "name": "Babes-Bolyai University", "priority": 2251, "external_id": null}, {"id": 10848074003, "name": "Baku State University", "priority": 2252, "external_id": null}, {"id": 10848075003, "name": "Belarusian National Technical University", "priority": 2253, "external_id": null}, {"id": 10848076003, "name": "Benem\u00e9rita Universidad Aut\u00f3noma de Puebla", "priority": 2254, "external_id": null}, {"id": 10848077003, "name": "Bogor Agricultural University", "priority": 2255, "external_id": null}, {"id": 10848078003, "name": "Coventry University", "priority": 2256, "external_id": null}, {"id": 10848079003, "name": "Cukurova University", "priority": 2257, "external_id": null}, {"id": 10848080003, "name": "Diponegoro University", "priority": 2258, "external_id": null}, {"id": 10848081003, "name": "Donetsk National University", "priority": 2259, "external_id": null}, {"id": 10848082003, "name": "Doshisha University", "priority": 2260, "external_id": null}, {"id": 10848083003, "name": "E.A.Buketov Karaganda State University", "priority": 2261, "external_id": null}, {"id": 10848084003, "name": "Far Eastern Federal University", "priority": 2262, "external_id": null}, {"id": 10848085003, "name": "Fu Jen Catholic University", "priority": 2263, "external_id": null}, {"id": 10848086003, "name": "Kagoshima University", "priority": 2264, "external_id": null}, {"id": 10848087003, "name": "Kaunas University of Technology", "priority": 2265, "external_id": null}, {"id": 10848088003, "name": "Kazakh Ablai khan University of International Relations and World Languages", "priority": 2266, "external_id": null}, {"id": 10848089003, "name": "Kazakh National Pedagogical University Abai", "priority": 2267, "external_id": null}, {"id": 10848090003, "name": "Kazakh National Technical University", "priority": 2268, "external_id": null}, {"id": 10848091003, "name": "Khon Kaen University", "priority": 2269, "external_id": null}, {"id": 10848092003, "name": "King Faisal University", "priority": 2270, "external_id": null}, {"id": 10848093003, "name": "King Mongkut''s University of Technology Thonburi", "priority": 2271, "external_id": null}, {"id": 10848094003, "name": "Kuwait University", "priority": 2272, "external_id": null}, {"id": 10848095003, "name": "Lodz University", "priority": 2273, "external_id": null}, {"id": 10848096003, "name": "Manchester Metropolitan University", "priority": 2274, "external_id": null}, {"id": 10848097003, "name": "Lobachevsky State University of Nizhni Novgorod", "priority": 2275, "external_id": null}, {"id": 10848098003, "name": "National Technical UniversityKharkiv Polytechnic Institute'", "priority": 2276, "external_id": null}, {"id": 10848099003, "name": "Nicolaus Copernicus University", "priority": 2277, "external_id": null}, {"id": 10848100003, "name": "Northumbria University at Newcastle", "priority": 2278, "external_id": null}, {"id": 10848101003, "name": "Nottingham Trent University", "priority": 2279, "external_id": null}, {"id": 10848102003, "name": "Ochanomizu University", "priority": 2280, "external_id": null}, {"id": 10848103003, "name": "Plekhanov Russian University of Economics", "priority": 2281, "external_id": null}, {"id": 10848104003, "name": "Pontificia Universidad Catolica del Ecuador", "priority": 2282, "external_id": null}, {"id": 10848105003, "name": "Prince of Songkla University", "priority": 2283, "external_id": null}, {"id": 10848106003, "name": "S.Seifullin Kazakh Agro Technical University", "priority": 2284, "external_id": null}, {"id": 10848107003, "name": "Saitama University", "priority": 2285, "external_id": null}, {"id": 10848108003, "name": "Sepuluh Nopember Institute of Technology", "priority": 2286, "external_id": null}, {"id": 10848109003, "name": "Shinshu University", "priority": 2287, "external_id": null}, {"id": 10848110003, "name": "The Robert Gordon University", "priority": 2288, "external_id": null}, {"id": 10848111003, "name": "Tokai University", "priority": 2289, "external_id": null}, {"id": 10848112003, "name": "Universidad ANAHUAC", "priority": 2290, "external_id": null}, {"id": 10848113003, "name": "Universidad Austral de Chile", "priority": 2291, "external_id": null}, {"id": 10848114003, "name": "University Aut\u00f3noma de Nuevo Le\u00f3n (UANL)", "priority": 2292, "external_id": null}, {"id": 10848115003, "name": "Universidad de la Habana", "priority": 2293, "external_id": null}, {"id": 10848116003, "name": "Universidad de La Sabana", "priority": 2294, "external_id": null}, {"id": 10848117003, "name": "Universidad de las Am\u00e9ricas Puebla (UDLAP)", "priority": 2295, "external_id": null}, {"id": 10848118003, "name": "Universidad de los Andes M\u00e9rida", "priority": 2296, "external_id": null}, {"id": 10848119003, "name": "University of Murcia", "priority": 2297, "external_id": null}, {"id": 10848120003, "name": "Universidad de Puerto Rico", "priority": 2298, "external_id": null}, {"id": 10848121003, "name": "Universidad de San Francisco de Quito", "priority": 2299, "external_id": null}, {"id": 10848122003, "name": "Universidad de Talca", "priority": 2300, "external_id": null}, {"id": 10848123003, "name": "Universidad del Norte", "priority": 2301, "external_id": null}, {"id": 10848124003, "name": "Universidad del Rosario", "priority": 2302, "external_id": null}, {"id": 10848125003, "name": "Universidad del Valle", "priority": 2303, "external_id": null}, {"id": 10848126003, "name": "Universidad Nacional de Cuyo", "priority": 2304, "external_id": null}, {"id": 10848127003, "name": "Universidad Nacional de Rosario", "priority": 2305, "external_id": null}, {"id": 10848128003, "name": "Universidad Nacional de Tucum\u00e1n", "priority": 2306, "external_id": null}, {"id": 10848129003, "name": "Universidad Nacional del Sur", "priority": 2307, "external_id": null}, {"id": 10848130003, "name": "Universidad Nacional Mayor de San Marcos", "priority": 2308, "external_id": null}, {"id": 10848131003, "name": "Universidad T\u00e9cnica Federico Santa Mar\u00eda", "priority": 2309, "external_id": null}, {"id": 10848132003, "name": "Universidad Tecnol\u00f3gica Nacional (UTN)", "priority": 2310, "external_id": null}, {"id": 10848133003, "name": "Universidade do Estado do Rio de Janeiro (UERJ)", "priority": 2311, "external_id": null}, {"id": 10848134003, "name": "Universidade Estadual de Londrina (UEL)", "priority": 2312, "external_id": null}, {"id": 10848135003, "name": "Universidade Federal de Santa Maria", "priority": 2313, "external_id": null}, {"id": 10848136003, "name": "Universidade Federal do Cear\u00e1 (UFC)", "priority": 2314, "external_id": null}, {"id": 10848137003, "name": "Universidade Federal do Pernambuco", "priority": 2315, "external_id": null}, {"id": 10848138003, "name": "Universit\u00e0 Ca'' Foscari Venezia", "priority": 2316, "external_id": null}, {"id": 10848139003, "name": "Catania University", "priority": 2317, "external_id": null}, {"id": 10848140003, "name": "Universit\u00e0 degli Studi Roma Tre", "priority": 2318, "external_id": null}, {"id": 10848141003, "name": "Universit\u00e9 Charles-de-Gaulle Lille 3", "priority": 2319, "external_id": null}, {"id": 10848142003, "name": "Universit\u00e9 de Caen Basse-Normandie", "priority": 2320, "external_id": null}, {"id": 10848143003, "name": "Universit\u00e9 de Cergy-Pontoise", "priority": 2321, "external_id": null}, {"id": 10848144003, "name": "Universit\u00e9 de Poitiers", "priority": 2322, "external_id": null}, {"id": 10848145003, "name": "Universit\u00e9 Jean Moulin Lyon 3", "priority": 2323, "external_id": null}, {"id": 10848146003, "name": "Universit\u00e9 Lille 2 Droit et Sant\u00e9", "priority": 2324, "external_id": null}, {"id": 10848147003, "name": "Universit\u00e9 Paris Ouest Nanterre La D\u00e9fense", "priority": 2325, "external_id": null}, {"id": 10848148003, "name": "Universit\u00e9 Paul-Val\u00e9ry Montpellier 3", "priority": 2326, "external_id": null}, {"id": 10848149003, "name": "Universit\u00e9 Pierre Mend\u00e8s France - Grenoble 2", "priority": 2327, "external_id": null}, {"id": 10848150003, "name": "Universit\u00e9 Stendhal Grenoble 3", "priority": 2328, "external_id": null}, {"id": 10848151003, "name": "Universit\u00e9 Toulouse II, Le Mirail", "priority": 2329, "external_id": null}, {"id": 10848152003, "name": "Universiti Teknologi MARA - UiTM", "priority": 2330, "external_id": null}, {"id": 10848153003, "name": "University of Baghdad", "priority": 2331, "external_id": null}, {"id": 10848154003, "name": "University of Bahrain", "priority": 2332, "external_id": null}, {"id": 10848155003, "name": "University of Bari", "priority": 2333, "external_id": null}, {"id": 10848156003, "name": "University of Belgrade", "priority": 2334, "external_id": null}, {"id": 10848157003, "name": "University of Brawijaya", "priority": 2335, "external_id": null}, {"id": 10848158003, "name": "University of Brescia", "priority": 2336, "external_id": null}, {"id": 10848159003, "name": "University of Bucharest", "priority": 2337, "external_id": null}, {"id": 10848160003, "name": "University of Calcutta", "priority": 2338, "external_id": null}, {"id": 10848161003, "name": "University of Central Lancashire", "priority": 2339, "external_id": null}, {"id": 10848162003, "name": "University of Colombo", "priority": 2340, "external_id": null}, {"id": 10848163003, "name": "University of Dhaka", "priority": 2341, "external_id": null}, {"id": 10848164003, "name": "University of East London", "priority": 2342, "external_id": null}, {"id": 10848165003, "name": "University of Engineering & Technology (UET) Lahore", "priority": 2343, "external_id": null}, {"id": 10848166003, "name": "University of Greenwich", "priority": 2344, "external_id": null}, {"id": 10848167003, "name": "University of Jordan", "priority": 2345, "external_id": null}, {"id": 10848168003, "name": "University of Karachi", "priority": 2346, "external_id": null}, {"id": 10848169003, "name": "University of Lahore", "priority": 2347, "external_id": null}, {"id": 10848170003, "name": "University of Latvia", "priority": 2348, "external_id": null}, {"id": 10848171003, "name": "University of New England", "priority": 2349, "external_id": null}, {"id": 10848172003, "name": "University of Pune", "priority": 2350, "external_id": null}, {"id": 10848173003, "name": "University of Santo Tomas", "priority": 2351, "external_id": null}, {"id": 10848174003, "name": "University of Southern Queensland", "priority": 2352, "external_id": null}, {"id": 10848175003, "name": "University of Wroclaw", "priority": 2353, "external_id": null}, {"id": 10848176003, "name": "Verona University", "priority": 2354, "external_id": null}, {"id": 10848177003, "name": "Victoria University", "priority": 2355, "external_id": null}, {"id": 10848178003, "name": "Vilnius Gediminas Technical University", "priority": 2356, "external_id": null}, {"id": 10848179003, "name": "Voronezh State University", "priority": 2357, "external_id": null}, {"id": 10848180003, "name": "Vytautas Magnus University", "priority": 2358, "external_id": null}, {"id": 10848181003, "name": "West University of Timisoara", "priority": 2359, "external_id": null}, {"id": 10848182003, "name": "University of South Alabama", "priority": 2360, "external_id": null}, {"id": 10848183003, "name": "University of Arkansas", "priority": 2361, "external_id": null}, {"id": 10848184003, "name": "University of California - Berkeley", "priority": 2362, "external_id": null}, {"id": 10848185003, "name": "University of Connecticut", "priority": 2363, "external_id": null}, {"id": 10848186003, "name": "University of South Florida", "priority": 2364, "external_id": null}, {"id": 10848187003, "name": "University of Georgia", "priority": 2365, "external_id": null}, {"id": 10848188003, "name": "University of Hawaii - Manoa", "priority": 2366, "external_id": null}, {"id": 10848189003, "name": "Iowa State University", "priority": 2367, "external_id": null}, {"id": 10848190003, "name": "Murray State University", "priority": 2368, "external_id": null}, {"id": 10848191003, "name": "University of Louisville", "priority": 2369, "external_id": null}, {"id": 10848192003, "name": "Western Kentucky University", "priority": 2370, "external_id": null}, {"id": 10848193003, "name": "Louisiana State University - Baton Rouge", "priority": 2371, "external_id": null}, {"id": 10848194003, "name": "University of Maryland - College Park", "priority": 2372, "external_id": null}, {"id": 10848195003, "name": "University of Minnesota - Twin Cities", "priority": 2373, "external_id": null}, {"id": 10848196003, "name": "University of Montana", "priority": 2374, "external_id": null}, {"id": 10848197003, "name": "East Carolina University", "priority": 2375, "external_id": null}, {"id": 10848198003, "name": "University of North Carolina - Chapel Hill", "priority": 2376, "external_id": null}, {"id": 10848199003, "name": "Wake Forest University", "priority": 2377, "external_id": null}, {"id": 10848200003, "name": "University of Nebraska - Lincoln", "priority": 2378, "external_id": null}, {"id": 10848201003, "name": "New Mexico State University", "priority": 2379, "external_id": null}, {"id": 10848202003, "name": "Ohio State University - Columbus", "priority": 2380, "external_id": null}, {"id": 10848203003, "name": "University of Oklahoma", "priority": 2381, "external_id": null}, {"id": 10848204003, "name": "Pennsylvania State University - University Park", "priority": 2382, "external_id": null}, {"id": 10848205003, "name": "University of Pittsburgh", "priority": 2383, "external_id": null}, {"id": 10848206003, "name": "University of Tennessee - Chattanooga", "priority": 2384, "external_id": null}, {"id": 10848207003, "name": "Vanderbilt University", "priority": 2385, "external_id": null}, {"id": 10848208003, "name": "Rice University", "priority": 2386, "external_id": null}, {"id": 10848209003, "name": "University of Utah", "priority": 2387, "external_id": null}, {"id": 10848210003, "name": "University of Richmond", "priority": 2388, "external_id": null}, {"id": 10848211003, "name": "University of Arkansas - Pine Bluff", "priority": 2389, "external_id": null}, {"id": 10848212003, "name": "University of Central Florida", "priority": 2390, "external_id": null}, {"id": 10848213003, "name": "Florida Atlantic University", "priority": 2391, "external_id": null}, {"id": 10848214003, "name": "Hampton University", "priority": 2392, "external_id": null}, {"id": 10848215003, "name": "Liberty University", "priority": 2393, "external_id": null}, {"id": 10848216003, "name": "Mercer University", "priority": 2394, "external_id": null}, {"id": 10848217003, "name": "Middle Tennessee State University", "priority": 2395, "external_id": null}, {"id": 10848218003, "name": "University of Nevada - Las Vegas", "priority": 2396, "external_id": null}, {"id": 10848219003, "name": "South Carolina State University", "priority": 2397, "external_id": null}, {"id": 10848220003, "name": "University of Tennessee - Martin", "priority": 2398, "external_id": null}, {"id": 10848221003, "name": "Weber State University", "priority": 2399, "external_id": null}, {"id": 10848222003, "name": "Youngstown State University", "priority": 2400, "external_id": null}, {"id": 10848223003, "name": "University of the Incarnate Word", "priority": 2401, "external_id": null}, {"id": 10848224003, "name": "University of Washington", "priority": 2402, "external_id": null}, {"id": 10848225003, "name": "University of Louisiana - Lafayette", "priority": 2403, "external_id": null}, {"id": 10848226003, "name": "Coastal Carolina University", "priority": 2404, "external_id": null}, {"id": 10848227003, "name": "Utah State University", "priority": 2405, "external_id": null}, {"id": 10848228003, "name": "University of Alabama", "priority": 2406, "external_id": null}, {"id": 10848229003, "name": "University of Illinois - Urbana-Champaign", "priority": 2407, "external_id": null}, {"id": 10848230003, "name": "United States Air Force Academy", "priority": 2408, "external_id": null}, {"id": 10848231003, "name": "University of Akron", "priority": 2409, "external_id": null}, {"id": 10848232003, "name": "University of Central Arkansas", "priority": 2410, "external_id": null}, {"id": 10848233003, "name": "University of Kansas", "priority": 2411, "external_id": null}, {"id": 10848234003, "name": "University of Northern Colorado", "priority": 2412, "external_id": null}, {"id": 10848235003, "name": "University of Northern Iowa", "priority": 2413, "external_id": null}, {"id": 10848236003, "name": "University of South Carolina", "priority": 2414, "external_id": null}, {"id": 10848237003, "name": "Tennessee Technological University", "priority": 2415, "external_id": null}, {"id": 10848238003, "name": "University of Texas - El Paso", "priority": 2416, "external_id": null}, {"id": 10848239003, "name": "Texas Tech University", "priority": 2417, "external_id": null}, {"id": 10848240003, "name": "Tulane University", "priority": 2418, "external_id": null}, {"id": 10848241003, "name": "Virginia Military Institute", "priority": 2419, "external_id": null}, {"id": 10848242003, "name": "Western Michigan University", "priority": 2420, "external_id": null}, {"id": 10848243003, "name": "Wilfrid Laurier University", "priority": 2421, "external_id": null}, {"id": 10848244003, "name": "University of San Diego", "priority": 2422, "external_id": null}, {"id": 10848245003, "name": "University of California - San Diego", "priority": 2423, "external_id": null}, {"id": 10848246003, "name": "Brooks Institute of Photography", "priority": 2424, "external_id": null}, {"id": 10848247003, "name": "Acupuncture and Integrative Medicine College - Berkeley", "priority": 2425, "external_id": null}, {"id": 10848248003, "name": "Southern Alberta Institute of Technology", "priority": 2426, "external_id": null}, {"id": 10848249003, "name": "Susquehanna University", "priority": 2427, "external_id": null}, {"id": 10848250003, "name": "University of Texas - Dallas", "priority": 2428, "external_id": null}, {"id": 10848251003, "name": "Thunderbird School of Global Management", "priority": 2429, "external_id": null}, {"id": 10848252003, "name": "Presidio Graduate School", "priority": 2430, "external_id": null}, {"id": 10848253003, "name": "\u00c9cole sup\u00e9rieure de commerce de Dijon", "priority": 2431, "external_id": null}, {"id": 10848254003, "name": "University of California - San Francisco", "priority": 2432, "external_id": null}, {"id": 10848255003, "name": "Hack Reactor", "priority": 2433, "external_id": null}, {"id": 10848256003, "name": "St. Mary''s College of California", "priority": 2434, "external_id": null}, {"id": 10848257003, "name": "New England Law", "priority": 2435, "external_id": null}, {"id": 10848258003, "name": "University of California, Merced", "priority": 2436, "external_id": null}, {"id": 10848259003, "name": "University of California, Hastings College of the Law", "priority": 2437, "external_id": null}, {"id": 10848260003, "name": "V.N. Karazin Kharkiv National University", "priority": 2438, "external_id": null}, {"id": 10848261003, "name": "SIM University (UniSIM)", "priority": 2439, "external_id": null}, {"id": 10848262003, "name": "Singapore Management University (SMU)", "priority": 2440, "external_id": null}, {"id": 10848263003, "name": "Singapore University of Technology and Design (SUTD)", "priority": 2441, "external_id": null}, {"id": 10848264003, "name": "Singapore Institute of Technology (SIT)", "priority": 2442, "external_id": null}, {"id": 10848265003, "name": "Nanyang Polytechnic (NYP)", "priority": 2443, "external_id": null}, {"id": 10848266003, "name": "Ngee Ann Polytechnic (NP)", "priority": 2444, "external_id": null}, {"id": 10848267003, "name": "Republic Polytechnic (RP)", "priority": 2445, "external_id": null}, {"id": 10848268003, "name": "Singapore Polytechnic (SP)", "priority": 2446, "external_id": null}, {"id": 10848269003, "name": "Temasek Polytechnic (TP)", "priority": 2447, "external_id": null}, {"id": 10848270003, "name": "INSEAD", "priority": 2448, "external_id": null}, {"id": 10848271003, "name": "Funda\u00e7\u00e3o Get\u00falio Vargas", "priority": 2449, "external_id": null}, {"id": 10848272003, "name": "Acharya Nagarjuna University", "priority": 2450, "external_id": null}, {"id": 10848273003, "name": "University of California - Santa Barbara", "priority": 2451, "external_id": null}, {"id": 10848274003, "name": "University of California - Irvine", "priority": 2452, "external_id": null}, {"id": 10848275003, "name": "California State University - Long Beach", "priority": 2453, "external_id": null}, {"id": 10848276003, "name": "Robert Morris University Illinois", "priority": 2454, "external_id": null}, {"id": 10848277003, "name": "Harold Washington College - City Colleges of Chicago", "priority": 2455, "external_id": null}, {"id": 10848278003, "name": "Harry S Truman College - City Colleges of Chicago", "priority": 2456, "external_id": null}, {"id": 10848279003, "name": "Kennedy-King College - City Colleges of Chicago", "priority": 2457, "external_id": null}, {"id": 10848280003, "name": "Malcolm X College - City Colleges of Chicago", "priority": 2458, "external_id": null}, {"id": 10848281003, "name": "Olive-Harvey College - City Colleges of Chicago", "priority": 2459, "external_id": null}, {"id": 10848282003, "name": "Richard J Daley College - City Colleges of Chicago", "priority": 2460, "external_id": null}, {"id": 10848283003, "name": "Wilbur Wright College - City Colleges of Chicago", "priority": 2461, "external_id": null}, {"id": 10848284003, "name": "Abertay University", "priority": 2462, "external_id": null}, {"id": 10848285003, "name": "Pontif\u00edcia Universidade Cat\u00f3lica de Minas Gerais", "priority": 2463, "external_id": null}, {"id": 10848286003, "name": "Other", "priority": 2464, "external_id": null}, {"id": 19126655003, "name": "Atlanta College of Arts", "priority": 2465, "external_id": null}]}, "emitted_at": 1691572567819} -{"stream": "custom_fields", "data": {"id": 4680899003, "name": "Degree", "active": true, "field_type": "candidate", "priority": 1, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "degree", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10848287003, "name": "High School", "priority": 0, "external_id": null}, {"id": 10848288003, "name": "Associate's Degree", "priority": 1, "external_id": null}, {"id": 10848289003, "name": "Bachelor's Degree", "priority": 2, "external_id": null}, {"id": 10848290003, "name": "Master's Degree", "priority": 3, "external_id": null}, {"id": 10848291003, "name": "Master of Business Administration (M.B.A.)", "priority": 4, "external_id": null}, {"id": 10848292003, "name": "Juris Doctor (J.D.)", "priority": 5, "external_id": null}, {"id": 10848293003, "name": "Doctor of Medicine (M.D.)", "priority": 6, "external_id": null}, {"id": 10848294003, "name": "Doctor of Philosophy (Ph.D.)", "priority": 7, "external_id": null}, {"id": 10848295003, "name": "Engineer's Degree", "priority": 8, "external_id": null}, {"id": 10848296003, "name": "Other", "priority": 9, "external_id": null}]}, "emitted_at": 1691572567849} -{"stream": "custom_fields", "data": {"id": 4680900003, "name": "Discipline", "active": true, "field_type": "candidate", "priority": 2, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "discipline", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10848297003, "name": "Accounting", "priority": 0, "external_id": null}, {"id": 10848298003, "name": "African Studies", "priority": 1, "external_id": null}, {"id": 10848299003, "name": "Agriculture", "priority": 2, "external_id": null}, {"id": 10848300003, "name": "Anthropology", "priority": 3, "external_id": null}, {"id": 10848301003, "name": "Applied Health Services", "priority": 4, "external_id": null}, {"id": 10848302003, "name": "Architecture", "priority": 5, "external_id": null}, {"id": 10848303003, "name": "Art", "priority": 6, "external_id": null}, {"id": 10848304003, "name": "Asian Studies", "priority": 7, "external_id": null}, {"id": 10848305003, "name": "Biology", "priority": 8, "external_id": null}, {"id": 10848306003, "name": "Business", "priority": 9, "external_id": null}, {"id": 10848307003, "name": "Business Administration", "priority": 10, "external_id": null}, {"id": 10848308003, "name": "Chemistry", "priority": 11, "external_id": null}, {"id": 10848309003, "name": "Classical Languages", "priority": 12, "external_id": null}, {"id": 10848310003, "name": "Communications & Film", "priority": 13, "external_id": null}, {"id": 10848311003, "name": "Computer Science", "priority": 14, "external_id": null}, {"id": 10848312003, "name": "Dentistry", "priority": 15, "external_id": null}, {"id": 10848313003, "name": "Developing Nations", "priority": 16, "external_id": null}, {"id": 10848314003, "name": "Discipline Unknown", "priority": 17, "external_id": null}, {"id": 10848315003, "name": "Earth Sciences", "priority": 18, "external_id": null}, {"id": 10848316003, "name": "Economics", "priority": 19, "external_id": null}, {"id": 10848317003, "name": "Education", "priority": 20, "external_id": null}, {"id": 10848318003, "name": "Electronics", "priority": 21, "external_id": null}, {"id": 10848319003, "name": "Engineering", "priority": 22, "external_id": null}, {"id": 10848320003, "name": "English Studies", "priority": 23, "external_id": null}, {"id": 10848321003, "name": "Environmental Studies", "priority": 24, "external_id": null}, {"id": 10848322003, "name": "European Studies", "priority": 25, "external_id": null}, {"id": 10848323003, "name": "Fashion", "priority": 26, "external_id": null}, {"id": 10848324003, "name": "Finance", "priority": 27, "external_id": null}, {"id": 10848325003, "name": "Fine Arts", "priority": 28, "external_id": null}, {"id": 10848326003, "name": "General Studies", "priority": 29, "external_id": null}, {"id": 10848327003, "name": "Health Services", "priority": 30, "external_id": null}, {"id": 10848328003, "name": "History", "priority": 31, "external_id": null}, {"id": 10848329003, "name": "Human Resources Management", "priority": 32, "external_id": null}, {"id": 10848330003, "name": "Humanities", "priority": 33, "external_id": null}, {"id": 10848331003, "name": "Industrial Arts & Carpentry", "priority": 34, "external_id": null}, {"id": 10848332003, "name": "Information Systems", "priority": 35, "external_id": null}, {"id": 10848333003, "name": "International Relations", "priority": 36, "external_id": null}, {"id": 10848334003, "name": "Journalism", "priority": 37, "external_id": null}, {"id": 10848335003, "name": "Languages", "priority": 38, "external_id": null}, {"id": 10848336003, "name": "Latin American Studies", "priority": 39, "external_id": null}, {"id": 10848337003, "name": "Law", "priority": 40, "external_id": null}, {"id": 10848338003, "name": "Linguistics", "priority": 41, "external_id": null}, {"id": 10848339003, "name": "Manufacturing & Mechanics", "priority": 42, "external_id": null}, {"id": 10848340003, "name": "Mathematics", "priority": 43, "external_id": null}, {"id": 10848341003, "name": "Medicine", "priority": 44, "external_id": null}, {"id": 10848342003, "name": "Middle Eastern Studies", "priority": 45, "external_id": null}, {"id": 10848343003, "name": "Naval Science", "priority": 46, "external_id": null}, {"id": 10848344003, "name": "North American Studies", "priority": 47, "external_id": null}, {"id": 10848345003, "name": "Nuclear Technics", "priority": 48, "external_id": null}, {"id": 10848346003, "name": "Operations Research & Strategy", "priority": 49, "external_id": null}, {"id": 10848347003, "name": "Organizational Theory", "priority": 50, "external_id": null}, {"id": 10848348003, "name": "Philosophy", "priority": 51, "external_id": null}, {"id": 10848349003, "name": "Physical Education", "priority": 52, "external_id": null}, {"id": 10848350003, "name": "Physical Sciences", "priority": 53, "external_id": null}, {"id": 10848351003, "name": "Physics", "priority": 54, "external_id": null}, {"id": 10848352003, "name": "Political Science", "priority": 55, "external_id": null}, {"id": 10848353003, "name": "Psychology", "priority": 56, "external_id": null}, {"id": 10848354003, "name": "Public Policy", "priority": 57, "external_id": null}, {"id": 10848355003, "name": "Public Service", "priority": 58, "external_id": null}, {"id": 10848356003, "name": "Religious Studies", "priority": 59, "external_id": null}, {"id": 10848357003, "name": "Russian & Soviet Studies", "priority": 60, "external_id": null}, {"id": 10848358003, "name": "Scandinavian Studies", "priority": 61, "external_id": null}, {"id": 10848359003, "name": "Science", "priority": 62, "external_id": null}, {"id": 10848360003, "name": "Slavic Studies", "priority": 63, "external_id": null}, {"id": 10848361003, "name": "Social Science", "priority": 64, "external_id": null}, {"id": 10848362003, "name": "Social Sciences", "priority": 65, "external_id": null}, {"id": 10848363003, "name": "Sociology", "priority": 66, "external_id": null}, {"id": 10848364003, "name": "Speech", "priority": 67, "external_id": null}, {"id": 10848365003, "name": "Statistics & Decision Theory", "priority": 68, "external_id": null}, {"id": 10848366003, "name": "Urban Studies", "priority": 69, "external_id": null}, {"id": 10848367003, "name": "Veterinary Medicine", "priority": 70, "external_id": null}, {"id": 10848368003, "name": "Other", "priority": 71, "external_id": null}]}, "emitted_at": 1691572567852} +{"stream": "custom_fields", "data": {"id": 4680902003, "name": "Start Date", "active": true, "field_type": "offer", "priority": 0, "value_type": "date", "private": true, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "start_date", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "use_for_job_approvals": false, "use_for_offer_approvals": false, "custom_field_options": []}, "emitted_at": 1701307661390} +{"stream": "custom_fields", "data": {"id": 4680903003, "name": "Employment Type", "active": true, "field_type": "offer", "priority": 1, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": true, "name_key": "employment_type", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "use_for_job_approvals": false, "use_for_offer_approvals": false, "custom_field_options": [{"id": 10845801003, "name": "Full-time", "priority": 0, "external_id": null}, {"id": 10845802003, "name": "Part-time", "priority": 1, "external_id": null}, {"id": 10845803003, "name": "Intern", "priority": 2, "external_id": null}, {"id": 10845804003, "name": "Contract", "priority": 3, "external_id": null}, {"id": 10845805003, "name": "Temporary", "priority": 4, "external_id": null}]}, "emitted_at": 1701307661392} {"stream": "degrees", "data": {"id": 10848287003, "name": "High School", "priority": 0, "external_id": null}, "emitted_at": 1691572568468} {"stream": "degrees", "data": {"id": 10848288003, "name": "Associate's Degree", "priority": 1, "external_id": null}, "emitted_at": 1691572568471} {"stream": "degrees", "data": {"id": 10848289003, "name": "Bachelor's Degree", "priority": 2, "external_id": null}, "emitted_at": 1691572568473} diff --git a/airbyte-integrations/connectors/source-greenhouse/metadata.yaml b/airbyte-integrations/connectors/source-greenhouse/metadata.yaml index bf5927079078..e732c0f18fa6 100644 --- a/airbyte-integrations/connectors/source-greenhouse/metadata.yaml +++ b/airbyte-integrations/connectors/source-greenhouse/metadata.yaml @@ -1,12 +1,18 @@ data: + ab_internal: + ql: 200 + sl: 200 allowedHosts: hosts: - harvest.greenhouse.io + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 59f1e50a-331f-4f09-b3e8-2e8d4d355f44 - dockerImageTag: 0.4.2 + dockerImageTag: 0.4.4 dockerRepository: airbyte/source-greenhouse + documentationUrl: https://docs.airbyte.com/integrations/sources/greenhouse githubIssueLabel: source-greenhouse icon: greenhouse.svg license: MIT @@ -17,12 +23,8 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/greenhouse + supportLevel: certified tags: - language:low-code - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/components.py b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/components.py index 7eeba4b7ed8d..fe355f0984bb 100644 --- a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/components.py +++ b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/components.py @@ -61,8 +61,8 @@ def is_greater_than_or_equal(self, first: Record, second: Record) -> bool: """ Evaluating which record is greater in terms of cursor. This is used to avoid having to capture all the records to close a slice """ - first_cursor_value = first.get(self.cursor_field) - second_cursor_value = second.get(self.cursor_field) + first_cursor_value = first.get(self.cursor_field, "") + second_cursor_value = second.get(self.cursor_field, "") if first_cursor_value and second_cursor_value: return first_cursor_value >= second_cursor_value elif first_cursor_value: diff --git a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/candidates.json b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/candidates.json index 5e1161a10b96..8e3f9d7a301c 100644 --- a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/candidates.json +++ b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/schemas/candidates.json @@ -217,6 +217,16 @@ }, "addresses": { "type": "array" + }, + "custom_fields": { + "properties": {}, + "additionalProperties": true, + "type": ["null", "object"] + }, + "keyed_custom_fields": { + "properties": {}, + "additionalProperties": true, + "type": ["null", "object"] } } } diff --git a/airbyte-integrations/connectors/source-greenhouse/unit_tests/conftest.py b/airbyte-integrations/connectors/source-greenhouse/unit_tests/conftest.py new file mode 100644 index 000000000000..605c45c1ea2f --- /dev/null +++ b/airbyte-integrations/connectors/source-greenhouse/unit_tests/conftest.py @@ -0,0 +1,21 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock, Mock + +import pytest +from airbyte_cdk.sources.streams import Stream +from source_greenhouse.components import GreenHouseSlicer, GreenHouseSubstreamSlicer + + +@pytest.fixture +def greenhouse_slicer(): + date_time = "2022-09-05T10:10:10.000000Z" + return GreenHouseSlicer(cursor_field=date_time, parameters={}, request_cursor_field=None) + + +@pytest.fixture +def greenhouse_substream_slicer(): + parent_stream = MagicMock(spec=Stream) + return GreenHouseSubstreamSlicer(cursor_field='cursor_field', stream_slice_field='slice_field', parent_stream=parent_stream, parent_key='parent_key', parameters={}, request_cursor_field=None) diff --git a/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py b/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py index 0e9d1b5bb96a..48db265f477f 100644 --- a/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py @@ -10,10 +10,10 @@ from source_greenhouse.components import GreenHouseSlicer, GreenHouseSubstreamSlicer -def test_slicer(): +def test_slicer(greenhouse_slicer): date_time = "2022-09-05T10:10:10.000000Z" date_time_dict = {date_time: date_time} - slicer = GreenHouseSlicer(cursor_field=date_time, parameters={}, request_cursor_field=None) + slicer = greenhouse_slicer slicer.close_slice(date_time_dict, date_time_dict) assert slicer.get_stream_state() == {date_time: "2022-09-05T10:10:10.000Z"} assert slicer.get_request_headers() == {} @@ -48,3 +48,90 @@ def test_sub_slicer(last_record, expected, records): stream_slice = next(slicer.stream_slices()) if records else {} slicer.close_slice(stream_slice, last_record) assert slicer.get_stream_state() == expected + + +@pytest.mark.parametrize( + "stream_state, cursor_field, expected_state", + [ + ({'cursor_field_1': '2022-09-05T10:10:10.000Z'}, 'cursor_field_1', {'cursor_field_1': '2022-09-05T10:10:10.000Z'}), + ({'cursor_field_2': '2022-09-05T10:10:100000Z'}, 'cursor_field_3', {}), + ({'cursor_field_4': None}, 'cursor_field_4', {}), + ({'cursor_field_5': ''}, 'cursor_field_5', {}), + ], + ids=[ + "cursor_value_present", + "cursor_value_not_present", + "cursor_value_is_None", + "cursor_value_is_empty_string" + ] +) +def test_slicer_set_initial_state(stream_state, cursor_field, expected_state): + slicer = GreenHouseSlicer(cursor_field=cursor_field, parameters={}, request_cursor_field=None) + # Set initial state + slicer.set_initial_state(stream_state) + assert slicer.get_stream_state() == expected_state + +@pytest.mark.parametrize( + "stream_state, initial_state, expected_state", + [ + ( + {'id1': {'cursor_field': '2023-01-01T10:00:00.000Z'}}, + {'id2': {'cursor_field': '2023-01-02T11:00:00.000Z'}}, + { + 'id1': {'cursor_field': '2023-01-01T10:00:00.000Z'}, + 'id2': {'cursor_field': '2023-01-02T11:00:00.000Z'} + } + ), + ( + {'id1': {'cursor_field': '2023-01-01T10:00:00.000Z'}}, + {'id1': {'cursor_field': '2023-01-01T09:00:00.000Z'}}, + {'id1': {'cursor_field': '2023-01-01T10:00:00.000Z'}} + ), + ( + {}, + {}, + {} + ), + ], + ids=[ + "stream_state and initial_state have different keys", + "stream_state and initial_state have overlapping keys with different values", + "stream_state and initial_state are empty" + ] +) +def test_substream_set_initial_state(greenhouse_substream_slicer, stream_state, initial_state, expected_state): + slicer = greenhouse_substream_slicer + # Set initial state + slicer._state = initial_state + slicer.set_initial_state(stream_state) + assert slicer._state == expected_state + + +@pytest.mark.parametrize( + "first_record, second_record, expected_result", + [ + ( + {'cursor_field': '2023-01-01T00:00:00.000Z'}, + {'cursor_field': '2023-01-02T00:00:00.000Z'}, + False + ), + ( + {'cursor_field': '2023-02-01T00:00:00.000Z'}, + {'cursor_field': '2023-01-01T00:00:00.000Z'}, + True + ), + ( + {'cursor_field': '2023-01-02T00:00:00.000Z'}, + {'cursor_field': ''}, + True + ), + ( + {'cursor_field': ''}, + {'cursor_field': '2023-01-02T00:00:00.000Z'}, + False + ), + ] +) +def test_is_greater_than_or_equal(greenhouse_substream_slicer, first_record, second_record, expected_result): + slicer = greenhouse_substream_slicer + assert slicer.is_greater_than_or_equal(first_record, second_record) == expected_result diff --git a/airbyte-integrations/connectors/source-gridly/README.md b/airbyte-integrations/connectors/source-gridly/README.md index b3a0ea66f9db..f06931947767 100644 --- a/airbyte-integrations/connectors/source-gridly/README.md +++ b/airbyte-integrations/connectors/source-gridly/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-gridly:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/gridly) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_gridly/spec.yaml` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-gridly:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-gridly build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-gridly:airbyteDocker +An image will be built with the tag `airbyte/source-gridly:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-gridly:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-gridly:dev check --con docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-gridly:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-gridly:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-gridly test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-gridly:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-gridly:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-gridly test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/gridly.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-gridly/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-gridly/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-gridly/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-gridly/build.gradle b/airbyte-integrations/connectors/source-gridly/build.gradle deleted file mode 100644 index bbecc83978a7..000000000000 --- a/airbyte-integrations/connectors/source-gridly/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_gridly' -} diff --git a/airbyte-integrations/connectors/source-gutendex/README.md b/airbyte-integrations/connectors/source-gutendex/README.md index 42eea363acf7..3423fa3c754d 100644 --- a/airbyte-integrations/connectors/source-gutendex/README.md +++ b/airbyte-integrations/connectors/source-gutendex/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-gutendex:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/gutendex) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_gutendex/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-gutendex:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-gutendex build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-gutendex:airbyteDocker +An image will be built with the tag `airbyte/source-gutendex:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-gutendex:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-gutendex:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-gutendex:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-gutendex:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-gutendex test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-gutendex:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-gutendex:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-gutendex test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/gutendex.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-gutendex/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-gutendex/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-gutendex/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-gutendex/build.gradle b/airbyte-integrations/connectors/source-gutendex/build.gradle deleted file mode 100644 index 4a0eced2c958..000000000000 --- a/airbyte-integrations/connectors/source-gutendex/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_gutendex' -} diff --git a/airbyte-integrations/connectors/source-harness/.dockerignore b/airbyte-integrations/connectors/source-harness/.dockerignore new file mode 100644 index 000000000000..40467f139afd --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_harness +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-harness/Dockerfile b/airbyte-integrations/connectors/source-harness/Dockerfile new file mode 100644 index 000000000000..8542d6eca698 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_harness ./source_harness + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-harness diff --git a/airbyte-integrations/connectors/source-harness/README.md b/airbyte-integrations/connectors/source-harness/README.md new file mode 100644 index 000000000000..2956defc0ddd --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/README.md @@ -0,0 +1,67 @@ +# Harness Source + +This is the repository for the Harness configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/harness). + +## Local development + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/harness) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_harness/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source harness test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + + +#### Build +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-harness build +``` + +An image will be built with the tag `airbyte/source-harness:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-harness:dev . +``` + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-harness:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-harness:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-harness:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-harness:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-harness test +``` + +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-harness test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/harness.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-harness/__init__.py b/airbyte-integrations/connectors/source-harness/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-harness/acceptance-test-config.yml b/airbyte-integrations/connectors/source-harness/acceptance-test-config.yml new file mode 100644 index 000000000000..28456c8a61fb --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/acceptance-test-config.yml @@ -0,0 +1,39 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-harness:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_harness/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.jsonl" + # extra_fields: no + # exact_order: no + # extra_records: yes + incremental: + bypass_reason: "This connector does not implement incremental sync" + # TODO uncomment this block this block if your connector implements incremental sync: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-harness/icon.svg b/airbyte-integrations/connectors/source-harness/icon.svg new file mode 100644 index 000000000000..e1770dde603c --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/icon.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/airbyte-integrations/connectors/source-harness/integration_tests/__init__.py b/airbyte-integrations/connectors/source-harness/integration_tests/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-harness/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-harness/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-jdbc/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-harness/integration_tests/acceptance.py similarity index 100% rename from airbyte-integrations/connectors/source-jdbc/integration_tests/acceptance.py rename to airbyte-integrations/connectors/source-harness/integration_tests/acceptance.py diff --git a/airbyte-integrations/connectors/source-harness/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-harness/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..4cf0f64f27b0 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/integration_tests/configured_catalog.json @@ -0,0 +1,13 @@ +{ + "streams": [ + { + "stream": { + "name": "organizations", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-harness/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-harness/integration_tests/invalid_config.json new file mode 100644 index 000000000000..f0d7d0c01c14 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/integration_tests/invalid_config.json @@ -0,0 +1,5 @@ +{ + "api_key": "", + "account_id": "xxxxxxxxxxxxxxxxx", + "api_url": "https://app.harness.io" +} diff --git a/airbyte-integrations/connectors/source-harness/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-harness/integration_tests/sample_config.json new file mode 100644 index 000000000000..b792a18806ff --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/integration_tests/sample_config.json @@ -0,0 +1,5 @@ +{ + "api_key": "xxxxxxxxxxxxxxxxxxxxxxxxxxx", + "account_id": "xxxxxxxxxxxxxxxxx", + "api_url": "https://app.harness.io" +} diff --git a/airbyte-integrations/connectors/source-harness/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-harness/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-harness/main.py b/airbyte-integrations/connectors/source-harness/main.py new file mode 100644 index 000000000000..b323465b96c8 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_harness import SourceHarness + +if __name__ == "__main__": + source = SourceHarness() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-harness/metadata.yaml b/airbyte-integrations/connectors/source-harness/metadata.yaml new file mode 100644 index 000000000000..a33d83b93859 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/metadata.yaml @@ -0,0 +1,25 @@ +data: + allowedHosts: + hosts: + - api.harness.io + registries: + oss: + enabled: false + cloud: + enabled: false + connectorSubtype: api + connectorType: source + definitionId: b0e46f61-e143-47cc-a595-4bb73bfa8a15 + dockerImageTag: 0.1.0 + dockerRepository: airbyte/source-harness + githubIssueLabel: source-harness + icon: harness.svg + license: MIT + name: Harness + releaseDate: 2023-10-10 + releaseStage: alpha + supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/sources/harness + tags: + - language:low-code +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-harness/requirements.txt b/airbyte-integrations/connectors/source-harness/requirements.txt new file mode 100644 index 000000000000..cc57334ef619 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/connector-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-harness/setup.py b/airbyte-integrations/connectors/source-harness/setup.py new file mode 100644 index 000000000000..6bef3ce1447c --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/setup.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = ["airbyte-cdk"] + +TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest~=6.2", + "pytest-mock~=3.6.1", +] + +setup( + name="source_harness", + description="Source implementation for Harness.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-harness/source_harness/__init__.py b/airbyte-integrations/connectors/source-harness/source_harness/__init__.py new file mode 100644 index 000000000000..1af39ecab1b1 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/source_harness/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceHarness + +__all__ = ["SourceHarness"] diff --git a/airbyte-integrations/connectors/source-harness/source_harness/manifest.yaml b/airbyte-integrations/connectors/source-harness/source_harness/manifest.yaml new file mode 100644 index 000000000000..e01c0d3bd201 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/source_harness/manifest.yaml @@ -0,0 +1,46 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["data", "content"] + requester: + type: HttpRequester + url_base: "{{ config['api_url'] }}" + http_method: "GET" + authenticator: + type: "ApiKeyAuthenticator" + header: "x-api-key" + api_token: "{{ config['api_key'] }}" + request_parameters: + accountIdentifier: "{{ config['account_id'] }}" + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + organizations_stream: + $ref: "#/definitions/base_stream" + name: "organizations" + $parameters: + path: "/ng/api/organizations" + +streams: + - "#/definitions/organizations_stream" + +check: + type: CheckStream + stream_names: + - "organizations" diff --git a/airbyte-integrations/connectors/source-harness/source_harness/schemas/organizations.json b/airbyte-integrations/connectors/source-harness/source_harness/schemas/organizations.json new file mode 100644 index 000000000000..88372907ea2b --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/source_harness/schemas/organizations.json @@ -0,0 +1,39 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Organizations schema", + "additionalProperties": true, + "type": ["object", "null"], + "properties": { + "organization": { + "type": ["object", "null"], + "properties": { + "identifier": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "description": { + "type": ["string", "null"] + }, + "tags": { + "type": ["object", "null"], + "properties": { + "identifier": { + "type": ["string", "null"] + } + } + } + } + }, + "createdAt": { + "type": ["number", "null"] + }, + "lastModifiedAt": { + "type": ["number", "null"] + }, + "harnessManaged": { + "type": ["boolean", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-harness/source_harness/source.py b/airbyte-integrations/connectors/source-harness/source_harness/source.py new file mode 100644 index 000000000000..a52f4c06db86 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/source_harness/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceHarness(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-harness/source_harness/spec.yaml b/airbyte-integrations/connectors/source-harness/source_harness/spec.yaml new file mode 100644 index 000000000000..d0bcc389d5f7 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/source_harness/spec.yaml @@ -0,0 +1,25 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/harness +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Harness Spec + type: object + required: + - api_key + - account_id + additionalProperties: true + properties: + api_key: + type: string + title: API key + airbyte_secret: true + account_id: + type: string + title: Account ID + description: Harness Account ID + api_url: + type: string + title: API URL + description: The API URL for fetching data from Harness + default: https://app.harness.io + examples: + - https://my-harness-server.example.com diff --git a/airbyte-integrations/connectors/source-harvest/Dockerfile b/airbyte-integrations/connectors/source-harvest/Dockerfile deleted file mode 100644 index 44fed7755bb3..000000000000 --- a/airbyte-integrations/connectors/source-harvest/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY source_harvest ./source_harvest -COPY main.py ./ -COPY setup.py ./ -RUN pip install . - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.18 -LABEL io.airbyte.name=airbyte/source-harvest diff --git a/airbyte-integrations/connectors/source-harvest/README.md b/airbyte-integrations/connectors/source-harvest/README.md index 837b66aaba6a..dbf16de24033 100644 --- a/airbyte-integrations/connectors/source-harvest/README.md +++ b/airbyte-integrations/connectors/source-harvest/README.md @@ -27,14 +27,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-harvest:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/harvest) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_harvest/spec.json` file. @@ -54,19 +46,70 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-harvest:dev + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-harvest build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-harvest:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. -You can also build the connector image via Gradle: +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-harvest:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-harvest:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-harvest:dev . +# Running the spec command against your patched connector +docker run airbyte/source-harvest:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -75,44 +118,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-harvest:dev check --co docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-harvest:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-harvest:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .'[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-harvest test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-harvest:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-harvest:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -122,8 +137,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-harvest test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/harvest.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-harvest/acceptance-test-config.yml b/airbyte-integrations/connectors/source-harvest/acceptance-test-config.yml index c8c4baf80d8d..8333c153e825 100644 --- a/airbyte-integrations/connectors/source-harvest/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-harvest/acceptance-test-config.yml @@ -3,37 +3,34 @@ test_strictness_level: "high" acceptance_tests: spec: tests: - - spec_path: "source_harvest/spec.json" + - spec_path: "source_harvest/spec.json" connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "secrets/old_config.json" - status: "succeed" - - config_path: "secrets/config_oauth.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "secrets/old_config.json" + status: "succeed" + - config_path: "secrets/config_oauth.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: tests: - - config_path: "secrets/config.json" + - config_path: "secrets/config.json" basic_read: tests: - - config_path: "secrets/config.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - fail_on_extra_columns: false + - config_path: "secrets/config.json" + expect_records: + path: "integration_tests/expected_records.jsonl" + fail_on_extra_columns: false incremental: tests: - - config_path: "secrets/config_with_date_range.json" - configured_catalog_path: "integration_tests/incremental_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" - cursor_paths: - contacts: ["updated_at"] - expenses_clients: ["to"] - timeout_seconds: 2400 + - config_path: "secrets/config_with_date_range.json" + configured_catalog_path: "integration_tests/incremental_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + timeout_seconds: 2400 full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-harvest/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-harvest/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-harvest/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-harvest/build.gradle b/airbyte-integrations/connectors/source-harvest/build.gradle deleted file mode 100644 index 22cf57235d7d..000000000000 --- a/airbyte-integrations/connectors/source-harvest/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_harvest' -} diff --git a/airbyte-integrations/connectors/source-harvest/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-harvest/integration_tests/expected_records.jsonl index 249c503bd2c4..4b23bf6a6cea 100644 --- a/airbyte-integrations/connectors/source-harvest/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-harvest/integration_tests/expected_records.jsonl @@ -28,9 +28,9 @@ {"stream": "time_entries", "data": {"id": 1494415019, "spent_date": "2021-05-05", "hours": 0.01, "hours_without_timer": 0.01, "rounded_hours": 0.01, "notes": "", "is_locked": true, "locked_reason": "Item Invoiced and Locked for this Time Period", "is_closed": false, "is_billed": true, "timer_started_at": null, "started_time": null, "ended_time": null, "is_running": false, "billable": true, "budgeted": false, "billable_rate": 10.0, "cost_rate": null, "created_at": "2021-05-05T12:53:38Z", "updated_at": "2021-05-25T16:16:52Z", "user": {"id": 3758380, "name": "Airbyte Developer"}, "client": {"id": 10749825, "name": "First client", "currency": "USD"}, "project": {"id": 28674500, "name": "First project", "code": "FP"}, "task": {"id": 16575206, "name": "Design"}, "user_assignment": {"id": 286365663, "is_project_manager": true, "is_active": true, "use_default_rates": false, "budget": null, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": 10.0}, "task_assignment": {"id": 307640132, "billable": true, "is_active": true, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": null, "budget": null}, "invoice": {"id": 28174531, "number": "1"}, "external_reference": null}, "emitted_at": 1690884282692} {"stream": "time_entries", "data": {"id": 1494414737, "spent_date": "2021-05-05", "hours": 8.0, "hours_without_timer": 8.0, "rounded_hours": 8.0, "notes": "", "is_locked": true, "locked_reason": "Item Invoiced and Locked for this Time Period", "is_closed": false, "is_billed": true, "timer_started_at": null, "started_time": null, "ended_time": null, "is_running": false, "billable": true, "budgeted": false, "billable_rate": 10.0, "cost_rate": null, "created_at": "2021-05-05T12:53:23Z", "updated_at": "2021-05-25T16:16:52Z", "user": {"id": 3758380, "name": "Airbyte Developer"}, "client": {"id": 10749825, "name": "First client", "currency": "USD"}, "project": {"id": 28674500, "name": "First project", "code": "FP"}, "task": {"id": 16575207, "name": "Programming"}, "user_assignment": {"id": 286365663, "is_project_manager": true, "is_active": true, "use_default_rates": false, "budget": null, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": 10.0}, "task_assignment": {"id": 307640134, "billable": true, "is_active": true, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": null, "budget": null}, "invoice": {"id": 28174531, "number": "1"}, "external_reference": null}, "emitted_at": 1690884282693} {"stream": "time_entries", "data": {"id": 1494238685, "spent_date": "2021-05-05", "hours": 0.71, "hours_without_timer": 0.71, "rounded_hours": 0.71, "notes": "This is a sample time entry.", "is_locked": false, "locked_reason": null, "is_closed": false, "is_billed": false, "timer_started_at": null, "started_time": null, "ended_time": null, "is_running": false, "billable": false, "budgeted": true, "billable_rate": null, "cost_rate": 60.0, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "user": {"id": 3758384, "name": "[SAMPLE] Warrin Wristwatch"}, "client": {"id": 10748670, "name": "[SAMPLE] Client A", "currency": "USD"}, "project": {"id": 28671449, "name": "Non-Billable Project", "code": "SAMPLE"}, "task": {"id": 16575208, "name": "Marketing"}, "user_assignment": {"id": 286326482, "is_project_manager": false, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:33Z", "updated_at": "2021-05-05T08:19:33Z", "hourly_rate": 175.0}, "task_assignment": {"id": 307607000, "billable": false, "is_active": true, "created_at": "2021-05-05T08:19:32Z", "updated_at": "2021-05-05T08:19:32Z", "hourly_rate": null, "budget": null}, "invoice": null, "external_reference": null}, "emitted_at": 1690884282694} -{"stream": "user_assignments", "data": {"id": 286365663, "is_project_manager": true, "is_active": true, "use_default_rates": false, "budget": null, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": 10.0, "project": {"id": 28674500, "name": "First project", "code": "FP"}, "user": {"id": 3758380, "name": "Airbyte Developer"}}, "emitted_at": 1690884285723} -{"stream": "user_assignments", "data": {"id": 286326492, "is_project_manager": true, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "hourly_rate": null, "project": {"id": 28671451, "name": "Airbyte", "code": null}, "user": {"id": 3758380, "name": "Airbyte Developer"}}, "emitted_at": 1690884285748} -{"stream": "user_assignments", "data": {"id": 286326482, "is_project_manager": false, "is_active": true, "use_default_rates": true, "budget": null, "created_at": "2021-05-05T08:19:33Z", "updated_at": "2021-05-05T08:19:33Z", "hourly_rate": 175.0, "project": {"id": 28671449, "name": "Non-Billable Project", "code": "SAMPLE"}, "user": {"id": 3758384, "name": "[SAMPLE] Warrin Wristwatch"}}, "emitted_at": 1690884285748} +{"stream": "user_assignments", "data": {"id": 286365663, "is_project_manager": true, "is_active": true, "use_default_rates": false, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": 10.0, "budget": null, "project": {"id": 28674500, "name": "First project", "code": "FP"}, "user": {"id": 3758380, "name": "Airbyte Developer"}}, "emitted_at": 1701383318239} +{"stream": "user_assignments", "data": {"id": 286326492, "is_project_manager": true, "is_active": true, "use_default_rates": true, "created_at": "2021-05-05T08:19:35Z", "updated_at": "2021-05-05T08:19:35Z", "hourly_rate": null, "budget": null, "project": {"id": 28671451, "name": "Airbyte", "code": null}, "user": {"id": 3758380, "name": "Airbyte Developer"}}, "emitted_at": 1701383318241} +{"stream": "user_assignments", "data": {"id": 286326482, "is_project_manager": false, "is_active": true, "use_default_rates": true, "created_at": "2021-05-05T08:19:33Z", "updated_at": "2021-05-05T08:19:33Z", "hourly_rate": null, "budget": null, "project": {"id": 28671449, "name": "Non-Billable Project", "code": "SAMPLE"}, "user": {"id": 3758384, "name": "[SAMPLE] Warrin Wristwatch"}}, "emitted_at": 1701383318242} {"stream": "task_assignments", "data": {"id": 307640135, "billable": true, "is_active": true, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": null, "budget": null, "project": {"id": 28674500, "name": "First project", "code": "FP"}, "task": {"id": 16575209, "name": "Project Management"}}, "emitted_at": 1690884286243} {"stream": "task_assignments", "data": {"id": 307640134, "billable": true, "is_active": true, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": null, "budget": null, "project": {"id": 28674500, "name": "First project", "code": "FP"}, "task": {"id": 16575207, "name": "Programming"}}, "emitted_at": 1690884286244} {"stream": "task_assignments", "data": {"id": 307640133, "billable": true, "is_active": true, "created_at": "2021-05-05T12:52:20Z", "updated_at": "2021-05-05T12:52:20Z", "hourly_rate": null, "budget": null, "project": {"id": 28674500, "name": "First project", "code": "FP"}, "task": {"id": 16575208, "name": "Marketing"}}, "emitted_at": 1690884286244} diff --git a/airbyte-integrations/connectors/source-harvest/metadata.yaml b/airbyte-integrations/connectors/source-harvest/metadata.yaml index 6930f5cc5963..0bb7f453aea5 100644 --- a/airbyte-integrations/connectors/source-harvest/metadata.yaml +++ b/airbyte-integrations/connectors/source-harvest/metadata.yaml @@ -1,12 +1,18 @@ data: + ab_internal: + ql: 200 + sl: 200 allowedHosts: hosts: - api.harvestapp.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: api connectorType: source definitionId: fe2b4084-3386-4d3b-9ad6-308f61a6f1e6 - dockerImageTag: 0.1.18 + dockerImageTag: 0.1.21 dockerRepository: airbyte/source-harvest + documentationUrl: https://docs.airbyte.com/integrations/sources/harvest githubIssueLabel: source-harvest icon: harvest.svg license: MIT @@ -17,11 +23,7 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/harvest + supportLevel: certified tags: - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py b/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py index ffbcb24543bd..3e7d1b2e5617 100644 --- a/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py +++ b/airbyte-integrations/connectors/source-harvest/source_harvest/streams.py @@ -299,7 +299,7 @@ def __init__(self, from_date: Optional[pendulum.date] = None, to_date: Optional[ super().__init__(**kwargs) current_date = pendulum.now().date() - self._from_date = from_date or current_date.subtract(years=1) + self._from_date = from_date or current_date.subtract(days=365) self._to_date = to_date or current_date # `to` date greater than `from` date causes an exception on Harvest if self._from_date > current_date: @@ -353,9 +353,9 @@ def stream_slices(self, sync_mode, stream_state: Mapping[str, Any] = None, **kwa start_date = pendulum.parse(stream_state.get(self.cursor_field)).date() while start_date < end_date: - # Max size of date chunks is 1 year + # Max size of date chunks is 365 days # Docs: https://help.getharvest.com/api-v2/reports-api/reports/time-reports/ - end_date_slice = end_date if start_date >= end_date.subtract(years=1) else start_date.add(years=1) + end_date_slice = end_date if start_date >= end_date.subtract(days=365) else start_date.add(days=365) date_slice = {"from": start_date.strftime(self.date_param_template), "to": end_date_slice.strftime(self.date_param_template)} start_date = end_date_slice diff --git a/airbyte-integrations/connectors/source-harvest/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-harvest/unit_tests/test_streams.py index 7df9dbdf5a24..e3c75953c74d 100644 --- a/airbyte-integrations/connectors/source-harvest/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-harvest/unit_tests/test_streams.py @@ -11,21 +11,23 @@ def test_skip_stream_default_availability_strategy(config, requests_mock): requests_mock.get("https://api.harvestapp.com/v2/estimates", status_code=403, json={"error": "error"}) - catalog = ConfiguredAirbyteCatalog.parse_obj({ - "streams": [ - { - "stream": { - "name": "estimates", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": True, - "default_cursor_field": ["updated_at"] - }, - "sync_mode": "incremental", - "cursor_field": ["updated_at"], - "destination_sync_mode": "append" - } - ] - }) + catalog = ConfiguredAirbyteCatalog.parse_obj( + { + "streams": [ + { + "stream": { + "name": "estimates", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": True, + "default_cursor_field": ["updated_at"], + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append", + } + ] + } + ) list(SourceHarvest().read(logger, config, catalog, {})) diff --git a/airbyte-integrations/connectors/source-hellobaton/README.md b/airbyte-integrations/connectors/source-hellobaton/README.md index e626764601dd..c56dd42ea657 100644 --- a/airbyte-integrations/connectors/source-hellobaton/README.md +++ b/airbyte-integrations/connectors/source-hellobaton/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-hellobaton:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/hellobaton) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_hellobaton/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-hellobaton:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-hellobaton build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-hellobaton:airbyteDocker +An image will be built with the tag `airbyte/source-hellobaton:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-hellobaton:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-hellobaton:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-hellobaton:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-hellobaton:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-hellobaton test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-hellobaton:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-hellobaton:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-hellobaton test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/hellobaton.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-hellobaton/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-hellobaton/acceptance-test-docker.sh deleted file mode 100644 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-hellobaton/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-hellobaton/build.gradle b/airbyte-integrations/connectors/source-hellobaton/build.gradle deleted file mode 100644 index 08e2290bc576..000000000000 --- a/airbyte-integrations/connectors/source-hellobaton/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_hellobaton' -} diff --git a/airbyte-integrations/connectors/source-hubplanner/Dockerfile b/airbyte-integrations/connectors/source-hubplanner/Dockerfile index ba50ba0758fb..7472843e16cb 100644 --- a/airbyte-integrations/connectors/source-hubplanner/Dockerfile +++ b/airbyte-integrations/connectors/source-hubplanner/Dockerfile @@ -1,16 +1,38 @@ -FROM python:3.9-slim +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base WORKDIR /airbyte/integration_code -COPY source_hubplanner ./source_hubplanner + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only COPY main.py ./ -COPY setup.py ./ -RUN pip install . +COPY source_hubplanner ./source_hubplanner ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-hubplanner diff --git a/airbyte-integrations/connectors/source-hubplanner/README.md b/airbyte-integrations/connectors/source-hubplanner/README.md index e58e8e540395..e7c245255f86 100644 --- a/airbyte-integrations/connectors/source-hubplanner/README.md +++ b/airbyte-integrations/connectors/source-hubplanner/README.md @@ -1,74 +1,34 @@ # Hubplanner Source -This is the repository for the Hubplanner source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/hubplanner). +This is the repository for the Hubplanner configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/hubplanner). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-hubplanner:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/hubplanner) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_hubplanner/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/hubplanner) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_hubplanner/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source hubplanner test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-hubplanner:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-hubplanner build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-hubplanner:airbyteDocker +An image will be built with the tag `airbyte/source-hubplanner:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-hubplanner:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-hubplanner:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-hubplanner:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-hubplanner:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-hubplanner test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-hubplanner:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-hubplanner:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-hubplanner test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/hubplanner.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-hubplanner/__init__.py b/airbyte-integrations/connectors/source-hubplanner/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-hubplanner/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hubplanner/acceptance-test-config.yml index 8ca311be523e..bc397a012de0 100644 --- a/airbyte-integrations/connectors/source-hubplanner/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-hubplanner/acceptance-test-config.yml @@ -1,26 +1,31 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-hubplanner:dev -tests: +acceptance_tests: spec: - - spec_path: "source_hubplanner/spec.json" + tests: + - spec_path: "source_hubplanner/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.0" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: ["holidays"] - # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file - # expect_records: - # path: "integration_tests/expected_records.jsonl" - # extra_fields: no - # exact_order: no - # extra_records: yes + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: clients + - name: holidays + incremental: + bypass_reason: "This connector does not implement incremental sync" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-hubplanner/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-hubplanner/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-hubplanner/build.gradle b/airbyte-integrations/connectors/source-hubplanner/build.gradle deleted file mode 100644 index 54ec4fbb1116..000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_hubplanner' -} diff --git a/airbyte-integrations/connectors/source-hubplanner/integration_tests/__init__.py b/airbyte-integrations/connectors/source-hubplanner/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-hubplanner/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-hubplanner/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-hubplanner/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-hubplanner/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-hubplanner/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-hubplanner/integration_tests/invalid_config.json index 92a71d900e59..d5ae35771339 100644 --- a/airbyte-integrations/connectors/source-hubplanner/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-hubplanner/integration_tests/invalid_config.json @@ -1,3 +1,3 @@ { - "api_key": "invalid-api-key" + "api_key": "invalid_api_key" } diff --git a/airbyte-integrations/connectors/source-hubplanner/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-hubplanner/integration_tests/sample_config.json new file mode 100644 index 000000000000..0cc92f7a010d --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "api_key": "api_key" +} diff --git a/airbyte-integrations/connectors/source-hubplanner/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-hubplanner/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-hubplanner/metadata.yaml b/airbyte-integrations/connectors/source-hubplanner/metadata.yaml index 8b0907752bb1..59487dacb6f1 100644 --- a/airbyte-integrations/connectors/source-hubplanner/metadata.yaml +++ b/airbyte-integrations/connectors/source-hubplanner/metadata.yaml @@ -1,24 +1,28 @@ data: + allowedHosts: + hosts: + - "*" # Please change to the hostname of the source. + registries: + cloud: + enabled: true + oss: + enabled: true connectorSubtype: api connectorType: source definitionId: 8097ceb9-383f-42f6-9f92-d3fd4bcc7689 - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-hubplanner githubIssueLabel: source-hubplanner icon: hubplanner.svg license: MIT name: Hubplanner - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: "2021-08-10" releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/hubplanner tags: - - language:python + - language:low-code ab_internal: sl: 100 ql: 100 - supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-hubplanner/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-hubplanner/sample_files/configured_catalog.json deleted file mode 100644 index 4fac0c64d90b..000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/sample_files/configured_catalog.json +++ /dev/null @@ -1,137 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "billing_rates", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "api_key": { - "type": "string", - "description": "Hubplanner API key. See here for more details.", - "airbyte_secret": true - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "bookings", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "api_key": { - "type": "string", - "description": "Hubplanner API key. See here for more details.", - "airbyte_secret": true - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "clients", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "api_key": { - "type": "string", - "description": "Hubplanner API key. See here for more details.", - "airbyte_secret": true - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "events", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "api_key": { - "type": "string", - "description": "Hubplanner API key. See here for more details.", - "airbyte_secret": true - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "holidays", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "api_key": { - "type": "string", - "description": "Hubplanner API key. See here for more details.", - "airbyte_secret": true - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "projects", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "api_key": { - "type": "string", - "description": "Hubplanner API key. See here for more details.", - "airbyte_secret": true - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "resources", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "api_key": { - "type": "string", - "description": "Hubplanner API key. See here for more details.", - "airbyte_secret": true - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-hubplanner/setup.py b/airbyte-integrations/connectors/source-hubplanner/setup.py index 0aa78f28fe34..5a7474d7c180 100644 --- a/airbyte-integrations/connectors/source-hubplanner/setup.py +++ b/airbyte-integrations/connectors/source-hubplanner/setup.py @@ -10,6 +10,7 @@ ] TEST_REQUIREMENTS = [ + "pytest~=6.2", "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", @@ -22,7 +23,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/__init__.py b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/__init__.py index 3bcd0c2b2b7a..3e8b2578cf36 100644 --- a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/__init__.py +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/manifest.yaml b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/manifest.yaml new file mode 100644 index 000000000000..984b017ebd8b --- /dev/null +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/manifest.yaml @@ -0,0 +1,108 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + requester: + type: HttpRequester + url_base: "https://api.hubplanner.com/v1" + http_method: "GET" + authenticator: + type: ApiKeyAuthenticator + header: Authorization + api_token: "{{ config['api_key'] }}" + request_parameters: + limit: "100" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "PageIncrement" + page_size: 100 + page_token_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "page" + requester: + $ref: "#/definitions/requester" + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + billing_rates_stream: + $ref: "#/definitions/base_stream" + name: "billing_rates" + $parameters: + path: "billingRate" + + bookings_stream: + $ref: "#/definitions/base_stream" + name: "bookings" + $parameters: + path: "booking" + + clients_stream: + $ref: "#/definitions/base_stream" + name: "clients" + $parameters: + path: "client" + + events_stream: + $ref: "#/definitions/base_stream" + name: "events" + $parameters: + path: "event" + + holidays_stream: + $ref: "#/definitions/base_stream" + name: "holidays" + $parameters: + path: "holiday" + + projects_stream: + $ref: "#/definitions/base_stream" + name: "projects" + $parameters: + path: "project" + + resources_stream: + $ref: "#/definitions/base_stream" + name: "resources" + $parameters: + path: "resource" + +streams: + - "#/definitions/billing_rates_stream" + - "#/definitions/bookings_stream" + - "#/definitions/clients_stream" + - "#/definitions/events_stream" + - "#/definitions/holidays_stream" + - "#/definitions/projects_stream" + - "#/definitions/resources_stream" + +check: + type: CheckStream + stream_names: + - "billing_rates" + +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/hubplanner + connection_specification: + title: Hubplanner Spec + type: object + required: + - api_key + additionalProperties: true + properties: + api_key: + type: string + description: Hubplanner API key. See https://github.com/hubplanner/API#authentication for more details. + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/bookings.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/bookings.json index d359bcf702a9..ec9db2e860c4 100644 --- a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/bookings.json +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/bookings.json @@ -149,6 +149,12 @@ "lastUpdatedById": { "type": ["null", "string"] }, + "deletedById": { + "type": ["null", "string"] + }, + "billable": { + "type": ["null", "boolean"] + }, "backgroundColor": { "type": ["null", "string"] } diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/projects.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/projects.json index af3395edabc2..24e456bf861a 100644 --- a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/projects.json +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/projects.json @@ -160,6 +160,36 @@ "metadata": { "type": ["null", "string"] }, + "defaultCategory": { + "type": ["null", "string"] + }, + "categoryGroups": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "customers": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "budgetCategories": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"] + } + }, + "private": { + "type": ["null", "boolean"] + }, + "billable": { + "type": ["null", "boolean"] + }, + "fixedCosts": { + "type": ["null", "array"] + }, "customFields": { "items": { "properties": { diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/resources.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/resources.json index 1b123381b9cc..fc646a589cb0 100644 --- a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/resources.json +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/schemas/resources.json @@ -39,6 +39,18 @@ "properties": {}, "type": ["null", "object"] }, + "billable": { + "type": ["null", "boolean"] + }, + "calendarIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "isApprover": { + "type": ["null", "boolean"] + }, "billing": { "properties": { "useDefault": { diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/source.py b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/source.py index 0267c67e86c5..5266d29cbcc3 100644 --- a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/source.py +++ b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/source.py @@ -2,247 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator +WARNING: Do not modify this file. +""" -# Basic full refresh stream -class HubplannerStream(HttpStream, ABC): - - url_base = "https://api.hubplanner.com/v1" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - TODO: Override this method to define a pagination strategy. If you will not be using pagination, no action is required - just return None. - - This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed - to most other methods in this class to help you form headers, request bodies, query params, etc.. - - For example, if the API accepts a 'page' parameter to determine which page of the result to return, and a response from the API contains a - 'page' number, then this method should probably return a dict {'page': response.json()['page'] + 1} to increment the page count by 1. - The request_params method should then read the input next_page_token and set the 'page' param to next_page_token['page']. - - :param response: the most recent response from the API - :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. - If there are no more pages in the result, return None. - """ - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - """ - TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. - Usually contains common params e.g. pagination size etc. - """ - return {} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - TODO: Override this method to define how a response is parsed. - :return an iterable containing each record in the response - """ - yield {} - - -# Basic incremental stream -class IncrementalHubplannerStream(HubplannerStream, ABC): - """ - TODO fill in details of this class to implement functionality related to incremental syncs for your connector. - if you do not need to implement incremental sync for any streams, remove this class. - """ - - # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. - state_checkpoint_interval = None - - @property - def cursor_field(self) -> str: - """ - TODO - Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is - usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. - - :return str: The name of the cursor field. - """ - return [] - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and - the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. - """ - return {} - - -class HubplannerAuthenticator(HttpAuthenticator): - def __init__(self, token: str, auth_header: str = "Authorization"): - self.auth_header = auth_header - self._token = token - - def get_auth_header(self) -> Mapping[str, Any]: - return {self.auth_header: f"{self._token}"} - - -class BillingRates(HubplannerStream): - primary_key = "_id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "v1/billingRate" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return super().request_params(stream_state, stream_slice, next_page_token) - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - return response.json() - - -class Bookings(HubplannerStream): - primary_key = "_id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "v1/booking" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return super().request_params(stream_state, stream_slice, next_page_token) - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - return response.json() - - -class Clients(HubplannerStream): - primary_key = "_id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "v1/client" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return super().request_params(stream_state, stream_slice, next_page_token) - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - return response.json() - - -class Events(HubplannerStream): - primary_key = "_id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "v1/event" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return super().request_params(stream_state, stream_slice, next_page_token) - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - return response.json() - - -class Holidays(HubplannerStream): - primary_key = "_id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "v1/holiday" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return super().request_params(stream_state, stream_slice, next_page_token) - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - return response.json() - - -class Projects(HubplannerStream): - primary_key = "_id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "v1/project" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return super().request_params(stream_state, stream_slice, next_page_token) - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - return response.json() - - -class Resources(HubplannerStream): - primary_key = "_id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "v1/resource" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return super().request_params(stream_state, stream_slice, next_page_token) - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - return response.json() - - -# Source -class SourceHubplanner(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - """ - :param config: the user-input config object conforming to the connector's spec.json - :param logger: logger object - :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. - """ - - url_base = "https://api.hubplanner.com/v1" - - try: - url = f"{url_base}/project" - - authenticator = HubplannerAuthenticator(token=config["api_key"]) - - session = requests.get(url, headers=authenticator.get_auth_header()) - session.raise_for_status() - - return True, None - except requests.exceptions.RequestException as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - authenticator = HubplannerAuthenticator(token=config["api_key"]) - return [ - BillingRates(authenticator=authenticator), - Bookings(authenticator=authenticator), - Clients(authenticator=authenticator), - Events(authenticator=authenticator), - Holidays(authenticator=authenticator), - Projects(authenticator=authenticator), - Resources(authenticator=authenticator), - ] +# Declarative Source +class SourceHubplanner(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/spec.json b/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/spec.json deleted file mode 100644 index 97897dc9370a..000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/source_hubplanner/spec.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/hubplanner", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Hubplanner Spec", - "type": "object", - "required": ["api_key"], - "additionalProperties": true, - "properties": { - "api_key": { - "type": "string", - "description": "Hubplanner API key. See https://github.com/hubplanner/API#authentication for more details.", - "airbyte_secret": true - } - } - } -} diff --git a/airbyte-integrations/connectors/source-hubplanner/unit_tests/__init__.py b/airbyte-integrations/connectors/source-hubplanner/unit_tests/__init__.py deleted file mode 100644 index 46b7376756ec..000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_source.py b/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_source.py deleted file mode 100644 index 3412ccc3a1da..000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_source.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -from source_hubplanner.source import SourceHubplanner - - -def test_streams(mocker): - source = SourceHubplanner() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 7 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_streams.py deleted file mode 100644 index f716cf012dbb..000000000000 --- a/airbyte-integrations/connectors/source-hubplanner/unit_tests/test_streams.py +++ /dev/null @@ -1,79 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_hubplanner.source import HubplannerStream - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(HubplannerStream, "path", "v0/example_endpoint") - mocker.patch.object(HubplannerStream, "primary_key", "test_primary_key") - mocker.patch.object(HubplannerStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = HubplannerStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = HubplannerStream() - inputs = {"response": MagicMock()} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_parse_response(patch_base_class): - stream = HubplannerStream() - # TODO: replace this with your input parameters - inputs = {"response": MagicMock()} - # TODO: replace this with your expected parced object - expected_parsed_object = {} - assert next(stream.parse_response(**inputs)) == expected_parsed_object - - -def test_request_headers(patch_base_class): - stream = HubplannerStream() - # TODO: replace this with your input parameters - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request headers - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = HubplannerStream() - # TODO: replace this with your expected http request method - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = HubplannerStream() - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = HubplannerStream() - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile deleted file mode 100644 index ac381183e833..000000000000 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - -COPY setup.py ./ - -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_hubspot ./source_hubspot - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=1.4.1 -LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/airbyte-integrations/connectors/source-hubspot/README.md b/airbyte-integrations/connectors/source-hubspot/README.md index fba382b6bc7b..aae879872bf3 100644 --- a/airbyte-integrations/connectors/source-hubspot/README.md +++ b/airbyte-integrations/connectors/source-hubspot/README.md @@ -74,14 +74,6 @@ If you are in an IDE, follow your IDE's instructions to activate the virtualenv. Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle - -From the Airbyte repository root, run: - -``` -./gradlew :airbyte-integrations:connectors:source-hubspot:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/hubspot) @@ -99,104 +91,28 @@ python main.py discover --config secrets/config.json python main.py read --config secrets/config_oauth.json --catalog sample_files/basic_read_catalog.json ``` -## Testing - -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. First install test dependencies into your virtual environment: - -``` -pip install .'[tests]' -``` - -### Unit Tests - -To run unit tests locally, from the connector directory run: - -``` -python -m pytest unit_tests -``` - -### Integration Tests - -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). - -#### Custom Integration tests - -Place custom tests inside `integration_tests/` folder, then, from the connector root, run - -``` -python -m pytest integration_tests -``` - -#### Acceptance Tests - -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. - -To run your integration tests with acceptance tests, from the connector root, run - -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests - -All commands should be run from airbyte project root. To run unit tests: - -``` -./gradlew :airbyte-integrations:connectors:source-hubspot:unitTest -``` - -To run acceptance and custom integration tests: - -``` -./gradlew :airbyte-integrations:connectors:source-hubspot:integrationTest -``` - -### Locally running the connector docker image - -#### Build - -First, make sure you build the latest Docker image: - -``` -docker build . -t airbyte/source-hubspot:dev -``` - -You can also build the connector image via Gradle: - -``` -./gradlew :airbyte-integrations:connectors:source-hubspot:airbyteDocker -``` - -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in the Dockerfile. - -#### Run - -Then run any of the connector commands as follows: - -``` -docker run --rm airbyte/source-hubspot:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-hubspot:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-hubspot:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-hubspot:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-hubspot test ``` -### Integration Tests - -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-hubspot:integrationTest` to run the standard integration test suite. -2. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. ### Publishing a new version of the connector - You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-hubspot test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/hubspot.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. -1. Make sure your changes are passing unit and integration tests -2. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -3. Create a Pull Request -4. Pat yourself on the back for being an awesome contributor -5. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master diff --git a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml index 94fa9c282e4c..dacff1cfa126 100644 --- a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml @@ -19,6 +19,8 @@ acceptance_tests: discovery: tests: - config_path: secrets/config_oauth.json + backward_compatibility_tests_config: + disable_for_version: 1.9.0 basic_read: tests: - config_path: secrets/config_oauth.json @@ -43,68 +45,154 @@ acceptance_tests: bypass_reason: Unable to populate - name: owners_archived bypass_reason: unable to populate + - name: tickets_web_analytics + bypass_reason: Unable to populate + - name: deals_web_analytics + bypass_reason: Unable to populate + - name: companies_web_analytics + bypass_reason: Unable to populate + - name: engagements_calls_web_analytics + bypass_reason: Unable to populate + - name: engagements_emails_web_analytics + bypass_reason: Unable to populate + - name: engagements_meetings_web_analytics + bypass_reason: Unable to populate + - name: engagements_notes_web_analytics + bypass_reason: Unable to populate + - name: engagements_tasks_web_analytics + bypass_reason: Unable to populate + - name: goals_web_analytics + bypass_reason: Unable to populate + - name: line_items_web_analytics + bypass_reason: Unable to populate + - name: products_web_analytics + bypass_reason: Unable to populate + - name: pets_web_analytics + bypass_reason: Unable to populate + - name: cars_web_analytics + bypass_reason: Unable to populate ignored_fields: contact_lists: - name: ilsFilterBranch bypass_reason: Floating fields order companies: + - name: properties_hs_time_* + bypass_reason: Hubspot time depend on current time - name: properties/hs_time_* bypass_reason: Hubspot time depend on current time + - name: properties_hs_was_imported + bypass_reason: attribute is not stable - name: properties/hs_was_imported bypass_reason: attribute is not stable contacts: + - name: properties_hs_time_* + bypass_reason: Hubspot time depend on current time - name: properties/hs_time_* bypass_reason: Hubspot time depend on current time + - name: properties_hs_time_in_subscriber + bypass_reason: Hubspot time depend on current time - name: properties/hs_time_in_subscriber bypass_reason: Hubspot time depend on current time + - name: properties_hs_latest_source_timestamp + bypass_reason: Hubspot time depend on current time - name: properties/hs_latest_source_timestamp bypass_reason: Hubspot time depend on current time + - name: properties_hs_predictivescoringtier + bypass_reason: Hubspot prediction changes - name: properties/hs_predictivescoringtier bypass_reason: Hubspot prediction changes + - name: properties_lastmodifieddate + bypass_reason: Hubspot time depend on current time - name: properties/lastmodifieddate bypass_reason: Hubspot time depend on current time + - name: properties_hs_time_in_lead + bypass_reason: Hubspot time depend on current time - name: properties/hs_time_in_lead bypass_reason: Hubspot time depend on current time + - name: properties_hs_time_in_opportunity + bypass_reason: Hubspot time depend on current time - name: properties/hs_time_in_opportunity bypass_reason: Hubspot time depend on current time + - name: properties_hs_was_imported + bypass_reason: attribute is not stable - name: properties/hs_was_imported bypass_reason: attribute is not stable - name: updatedAt bypass_reason: Hubspot time depend on current time + - name: properties/hs_v2_cumulative_time_* + bypass_reason: Hubspot time depend on current time + - name: properties/hs_v2_latest_time_* + bypass_reason: Hubspot time depend on current time + - name: properties_hs_v2_cumulative_time_* + bypass_reason: Hubspot time depend on current time + - name: properties_hs_v2_latest_time_* + bypass_reason: Hubspot time depend on current time deals: + - name: properties_hs_time_* + bypass_reason: Hubspot time depend on current time - name: properties/hs_time_* bypass_reason: Hubspot time depend on current time + - name: properties_hs_acv + bypass_reason: value can be an integer or float - name: properties/hs_acv bypass_reason: value can be an integer or float + - name: properties_hs_arr + bypass_reason: value can be an integer or float - name: properties/hs_arr bypass_reason: value can be an integer or float + - name: properties_hs_mrr + bypass_reason: value can be an integer or float - name: properties/hs_mrr bypass_reason: value can be an integer or float + - name: properties_hs_tcv + bypass_reason: value can be an integer or float - name: properties/hs_tcv bypass_reason: value can be an integer or float + - name: properties_hs_num_of_associated_line_items + bypass_reason: value can be an integer or float - name: properties/hs_num_of_associated_line_items bypass_reason: value can be an integer or float deals_archived: + - name: properties_hs_time_* + bypass_reason: Hubspot time depend on current time - name: properties/hs_time_* bypass_reason: Hubspot time depend on current time + - name: properties_hs_acv + bypass_reason: value can be an integer or float - name: properties/hs_acv bypass_reason: value can be an integer or float + - name: properties_hs_arr + bypass_reason: value can be an integer or float - name: properties/hs_arr bypass_reason: value can be an integer or float + - name: properties_hs_mrr + bypass_reason: value can be an integer or float - name: properties/hs_mrr bypass_reason: value can be an integer or float + - name: properties_hs_tcv + bypass_reason: value can be an integer or float - name: properties/hs_tcv bypass_reason: value can be an integer or float + - name: properties_hs_num_of_associated_line_items + bypass_reason: value can be an integer or float - name: properties/hs_num_of_associated_line_items bypass_reason: value can be an integer or float tickets: + - name: properties_hs_time_* + bypass_reason: Hubspot time depend on current time - name: properties/hs_time_* bypass_reason: Hubspot time depend on current time goals: + - name: properties_hs_time_* + bypass_reason: Hubspot time depend on current time - name: properties/hs_time_* bypass_reason: Hubspot time depend on current time + - name: properties_hs_lastmodifieddate + bypass_reason: Hubspot time depend on current time - name: properties/hs_lastmodifieddate bypass_reason: Hubspot time depend on current time + - name: properties_hs_kpi_value_last_calculated_at + bypass_reason: Hubspot time depend on current time - name: properties/hs_kpi_value_last_calculated_at bypass_reason: Hubspot time depend on current time - name: updatedAt @@ -120,15 +208,31 @@ acceptance_tests: - name: ilsFilterBranch bypass_reason: Floating fields order companies: + - name: properties_hs_time_* + bypass_reason: Hubspot time depend on current time - name: properties/hs_time_* bypass_reason: Hubspot time depend on current time contacts: + - name: properties_hs_time_* + bypass_reason: Hubspot time depend on current time - name: properties/hs_time_* bypass_reason: Hubspot time depend on current time + - name: properties/hs_v2_cumulative_time_* + bypass_reason: Hubspot time depend on current time + - name: properties/hs_v2_latest_time_* + bypass_reason: Hubspot time depend on current time + - name: properties_hs_v2_cumulative_time_* + bypass_reason: Hubspot time depend on current time + - name: properties_hs_v2_latest_time_* + bypass_reason: Hubspot time depend on current time deals: + - name: properties_hs_time_* + bypass_reason: Hubspot time depend on current time - name: properties/hs_time_* bypass_reason: Hubspot time depend on current time tickets: + - name: properties_hs_time_* + bypass_reason: Hubspot time depend on current time - name: properties/hs_time_* bypass_reason: Hubspot time depend on current time incremental: @@ -137,3 +241,4 @@ acceptance_tests: configured_catalog_path: sample_files/incremental_catalog.json future_state: future_state_path: integration_tests/abnormal_state.json + timeout_seconds: 7200 diff --git a/airbyte-integrations/connectors/source-hubspot/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-hubspot/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-hubspot/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-hubspot/build.gradle b/airbyte-integrations/connectors/source-hubspot/build.gradle deleted file mode 100644 index a7e1c44dad63..000000000000 --- a/airbyte-integrations/connectors/source-hubspot/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_hubspot' -} diff --git a/airbyte-integrations/connectors/source-hubspot/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-hubspot/integration_tests/abnormal_state.json index 5ad10ada31e8..882b0b1a8268 100644 --- a/airbyte-integrations/connectors/source-hubspot/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-hubspot/integration_tests/abnormal_state.json @@ -199,5 +199,42 @@ "stream_descriptor": { "name": "workflows" }, "stream_state": { "updatedAt": 7945393076000 } } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { "name": "contacts_property_history" }, + "stream_state": { "updatedAt": 7945393076000 } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { "name": "companies_property_history" }, + "stream_state": { "updatedAt": 7945393076000 } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { "name": "deals_property_history" }, + "stream_state": { "updatedAt": 7945393076000 } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { "name": "contacts_web_analytics" }, + "stream_state": { + "151": { "occurredAt": "2050-01-01T00:00:00Z" }, + "251": { "occurredAt": "2050-01-01T00:00:00Z" }, + "401": { "occurredAt": "2050-01-01T00:00:00Z" }, + "601": { "occurredAt": "2050-01-01T00:00:00Z" }, + "651": { "occurredAt": "2050-01-01T00:00:00Z" }, + "2501": { "occurredAt": "2050-01-01T00:00:00Z" }, + "2551": { "occurredAt": "2050-01-01T00:00:00Z" }, + "2601": { "occurredAt": "2050-01-01T00:00:00Z" } + } + } } ] diff --git a/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl index 643ca48f835d..0109a703be27 100644 --- a/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl @@ -1,52 +1,72 @@ -{"stream": "campaigns", "data": {"id": 243851494, "lastUpdatedTime": 1675121674226, "appId": 113, "appName": "Batch", "contentId": 100523515217, "subject": "test", "name": "test", "counters": {"dropped": 1}, "lastProcessingFinishedAt": 1675121674000, "lastProcessingStartedAt": 1675121671000, "lastProcessingStateChangeAt": 1675121674000, "numIncluded": 1, "processingState": "DONE", "type": "BATCH_EMAIL"}, "emitted_at": 1688723569670} -{"stream": "companies", "data": {"id": "4992593519", "properties": {"about_us": null, "address": null, "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": null, "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-10T07:58:09.554000+00:00", "custom_company_property": null, "days_to_close": null, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "airbyte.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": null, "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": "2021-05-21T10:17:06.028000+00:00", "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": null, "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_num_page_views": null, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": null, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": null, "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": "2021-05-21T10:17:28.964000+00:00", "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-01-26T11:45:49.817000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": null, "hs_num_blockers": null, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": null, "hs_num_decision_makers": null, "hs_num_open_deals": 1, "hs_object_id": 4992593519, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": null, "hs_predictivecontactscore_v2": null, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.5476861596107483, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": 68102534447, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2020-12-10T07:58:09.554000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": false, "lifecyclestage": "opportunity", "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Airbyte test1", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_associated_deals": 1, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 200, "phone": "+1 415-307-4864", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": null, "recent_deal_close_date": null, "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": null, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;segment;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "website": "airbyte.io", "zip": "94114"}, "createdAt": "2020-12-10T07:58:09.554Z", "updatedAt": "2023-01-26T11:45:49.817Z", "archived": false}, "emitted_at": 1689694783559} -{"stream": "companies", "data": {"id": "5000526215", "properties": {"about_us": null, "address": null, "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": "2023-04-04T15:00:58.081000+00:00", "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-11T01:27:40.002000+00:00", "custom_company_property": null, "days_to_close": 844, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "dataline.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": "2020-12-11T01:29:50.116000+00:00", "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": "2021-01-13T10:30:42.221000+00:00", "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": "OFFLINE", "hs_analytics_latest_source_data_1": "CONTACTS", "hs_analytics_latest_source_data_2": "CRM_UI", "hs_analytics_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "hs_analytics_num_page_views": 0, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": 0, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": "CRM_UI", "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": "2023-04-04T15:00:58.081000+00:00", "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": "2021-02-23T20:21:06.027000+00:00", "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": "2023-04-04T15:00:58.081000+00:00", "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-04-04T15:12:52.778000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": "5183403213", "hs_num_blockers": 0, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": 0, "hs_num_decision_makers": 0, "hs_num_open_deals": 2, "hs_object_id": 5000526215, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "companies-lifecycle-pipeline", "hs_predictivecontactscore_v2": 0.29, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.46257445216178894, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": 9074325326, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": 66508792054, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": 60010, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2020-12-11T01:27:40.002000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": false, "lifecyclestage": "customer", "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Dataline", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 1, "num_associated_deals": 3, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 25, "phone": "", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": 60000, "recent_deal_close_date": "2023-04-04T14:59:45.103000+00:00", "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": 60000, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;segment;google_tag_manager;cloud_flare;google_analytics;intercom;lever;google_apps", "website": "dataline.io", "zip": ""}, "createdAt": "2020-12-11T01:27:40.002Z", "updatedAt": "2023-04-04T15:12:52.778Z", "archived": false, "contacts": ["151", "151"]}, "emitted_at": 1689694783560} -{"stream": "companies", "data": {"id": "5000787595", "properties": {"about_us": null, "address": "2261 Market Street", "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": null, "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-11T01:28:27.673000+00:00", "custom_company_property": null, "days_to_close": null, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "Daxtarity.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": null, "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": null, "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": null, "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": "", "hs_analytics_latest_source_data_1": "", "hs_analytics_latest_source_data_2": "", "hs_analytics_latest_source_timestamp": null, "hs_analytics_num_page_views": null, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": null, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": "", "hs_analytics_source_data_1": "", "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": "", "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-01-23T15:41:56.644000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": null, "hs_num_blockers": 0, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": 0, "hs_num_decision_makers": 0, "hs_num_open_deals": 0, "hs_object_id": 5000787595, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": null, "hs_predictivecontactscore_v2": null, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.4076234698295593, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2020-12-11T01:28:27.673000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": false, "lifecyclestage": null, "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Daxtarity", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 50, "phone": "+1 415-307-4864", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": null, "recent_deal_close_date": null, "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": null, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "website": "Daxtarity.com", "zip": "94114"}, "createdAt": "2020-12-11T01:28:27.673Z", "updatedAt": "2023-01-23T15:41:56.644Z", "archived": false}, "emitted_at": 1689694783560} -{"stream": "contact_lists", "data": {"portalId": 8727216, "listId": 166, "createdAt": 1675120756833, "updatedAt": 1675120852460, "name": "Test", "listType": "DYNAMIC", "authorId": 12282590, "filters": [], "metaData": {"size": 3, "lastSizeChangeAt": 1675257270514, "processing": "DONE", "lastProcessingStateChangeAt": 1675120853286, "error": "", "listReferencesCount": null, "parentFolderId": null}, "archived": false, "teamIds": [], "ilsFilterBranch": "{\"filterBranchOperator\":\"OR\",\"filters\":[],\"filterBranches\":[{\"filterBranchOperator\":\"AND\",\"filters\":[{\"filterType\":\"PROPERTY\",\"property\":\"createdate\",\"operation\":{\"propertyType\":\"datetime\",\"operator\":\"IS_AFTER\",\"timestamp\":1669957199999,\"defaultValue\":null,\"includeObjectsWithNoValueSet\":false,\"requiresTimeZoneConversion\":true,\"operationType\":\"datetime\",\"operatorName\":\"IS_AFTER\"},\"frameworkFilterId\":null}],\"filterBranches\":[],\"filterBranchType\":\"AND\"}],\"filterBranchType\":\"OR\"}", "readOnly": false, "internal": false, "limitExempt": false, "dynamic": true}, "emitted_at": 1685387174847} -{"stream": "contacts", "data": {"id": "151", "properties": {"address": null, "annualrevenue": null, "associatedcompanyid": 5000526215, "associatedcompanylastupdated": null, "city": null, "closedate": null, "company": null, "company_size": null, "country": null, "createdate": "2020-12-11T01:29:50.116000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "shef@dne.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "sh", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "151", "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_2": "CRM_UI", "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": 1, "hs_count_is_worked": 0, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": "2020-12-11T01:29:50.116000+00:00", "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "dne.io", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_meeting_activity": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "CONTACTS", "hs_latest_source_data_2": "CRM_UI", "hs_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": null, "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": "2020-12-11T01:29:50.116000+00:00", "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 151, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.29, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_4", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": null, "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": 82747504126, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": "2020-12-11T01:29:50.093000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-03-21T19:28:17.125000+00:00", "lastname": "na", "lifecyclestage": "subscriber", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": null, "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": null, "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": null, "work_email": null, "zip": null}, "createdAt": "2020-12-11T01:29:50.116Z", "updatedAt": "2023-03-21T19:28:17.125Z", "archived": false, "companies": ["5000526215", "5000526215"]}, "emitted_at": 1690397694270} -{"stream": "contacts", "data": {"id": "251", "properties": {"address": "25000000 First Street", "annualrevenue": null, "associatedcompanyid": 5170561229, "associatedcompanylastupdated": null, "city": "Cambridge", "closedate": null, "company": "HubSpot", "company_size": null, "country": "USA", "createdate": "2021-02-22T14:05:09.944000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "testingdsapis@hubspot.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "Test User 5001", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "251", "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2021-02-22T14:05:09.944000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "API", "hs_analytics_source_data_2": null, "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": null, "hs_count_is_worked": null, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": "2021-02-22T14:05:09.944000+00:00", "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "hubspot.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_meeting_activity": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "API", "hs_latest_source_data_2": null, "hs_latest_source_timestamp": "2021-02-22T14:05:10.036000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": null, "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": "2021-02-22T14:05:09.944000+00:00", "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 251, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.29, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_4", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": "5551222323", "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": 76394984297, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-03-21T19:29:13.036000+00:00", "lastname": "Test Lastname 5001", "lifecyclestage": "subscriber", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": "555-122-2323", "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": "MA", "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": "http://hubspot.com", "work_email": null, "zip": "02139"}, "createdAt": "2021-02-22T14:05:09.944Z", "updatedAt": "2023-03-21T19:29:13.036Z", "archived": false, "companies": ["5170561229", "5170561229"]}, "emitted_at": 1690397694271} -{"stream": "contacts", "data": {"id": "401", "properties": {"address": "25 First Street", "annualrevenue": null, "associatedcompanyid": null, "associatedcompanylastupdated": null, "city": "Cambridge", "closedate": null, "company": null, "company_size": null, "country": null, "createdate": "2021-02-23T20:10:36.191000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "macmitch@hubspot.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "Mac", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "401", "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2021-02-23T20:10:36.181000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "IMPORT", "hs_analytics_source_data_2": "13256565", "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": "+18884827768", "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": "US", "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": 1, "hs_count_is_worked": 0, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": "2021-02-23T20:10:36.181000+00:00", "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "hubspot.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": "OTHER", "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_meeting_activity": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "IMPORT", "hs_latest_source_data_2": "13256565", "hs_latest_source_timestamp": "2021-02-23T20:10:36.210000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": "2021-02-23T20:10:36.181000+00:00", "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": null, "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 401, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.29, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_4", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": "8884827768", "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": 76286658061, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": true, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": "2021-05-21T10:20:30.963000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-03-21T19:31:00.563000+00:00", "lastname": "Mitchell", "lifecyclestage": "lead", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": "1(888) 482-7768", "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": "MA", "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": null, "work_email": null, "zip": "21430"}, "createdAt": "2021-02-23T20:10:36.191Z", "updatedAt": "2023-03-21T19:31:00.563Z", "archived": false}, "emitted_at": 1690397694271} -{"stream": "contacts", "data": {"id": "601", "properties": {"address": "0 First Street", "annualrevenue": null, "associatedcompanyid": null, "associatedcompanylastupdated": null, "city": "Cambridge", "closedate": null, "company": "HubSpot Test", "company_size": null, "country": null, "createdate": "2021-10-12T13:22:50.930000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "testingapicontact_0@hubspot.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "test contact 0", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "601", "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2021-10-12T13:22:50.930000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "API", "hs_analytics_source_data_2": null, "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": null, "hs_count_is_worked": null, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": "2021-10-12T13:22:50.930000+00:00", "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "hubspot.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_meeting_activity": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "API", "hs_latest_source_data_2": null, "hs_latest_source_timestamp": "2021-10-12T13:22:51.107000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": null, "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": "2021-10-12T13:22:50.930000+00:00", "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 601, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.31, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_2", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": "5551222323", "hs_sequences_actively_enrolled_count": 0, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": 56352723312, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-04-03T20:29:57.224000+00:00", "lastname": "testerson number 0", "lifecyclestage": "subscriber", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": "555-122-2323", "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": "MA", "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": "http://hubspot.com", "work_email": null, "zip": "02139"}, "createdAt": "2021-10-12T13:22:50.930Z", "updatedAt": "2023-04-03T20:29:57.224Z", "archived": false}, "emitted_at": 1690397694272} -{"stream": "contacts", "data": {"id": "651", "properties": {"address": "1 First Street", "annualrevenue": null, "associatedcompanyid": 5170561229, "associatedcompanylastupdated": null, "city": "Cambridge", "closedate": null, "company": "HubSpot Test", "company_size": null, "country": null, "createdate": "2021-01-14T14:26:17.014000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "testingapicontact_1@hubspot.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "test contact 1-1", "gender": null, "graduation_date": null, "hs_additional_emails": "testingapis@hubspot.com", "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "201;651", "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2021-01-14T14:26:17.014000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "API", "hs_analytics_source_data_2": null, "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": "201:1688758327178", "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": null, "hs_count_is_worked": null, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": "2021-10-12T13:23:01.830000+00:00", "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "hubspot.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": "", "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_meeting_activity": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "API", "hs_latest_source_data_2": null, "hs_latest_source_timestamp": "2021-01-14T14:26:17.081000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": null, "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": "2021-10-12T13:23:01.830000+00:00", "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": "201", "hs_object_id": 651, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.39, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_2", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": "5551222323", "hs_sequences_actively_enrolled_count": 0, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": 56352712412, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-07-07T19:33:25.177000+00:00", "lastname": "testerson number 1", "lifecyclestage": "subscriber", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": "555-122-2323", "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": "MA", "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": "http://hubspot.com", "work_email": null, "zip": "02139"}, "createdAt": "2021-01-14T14:26:17.014Z", "updatedAt": "2023-07-07T19:33:25.177Z", "archived": false, "companies": ["5170561229", "5170561229"]}, "emitted_at": 1690397694272} -{"stream": "contacts", "data": {"id": "2501", "properties": {"address": null, "annualrevenue": null, "associatedcompanyid": null, "associatedcompanylastupdated": null, "city": null, "closedate": null, "company": null, "company_size": null, "country": null, "createdate": "2023-01-30T23:17:09.904000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "test@test.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "test", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "2501", "hs_all_owner_ids": "", "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2023-01-30T23:17:09.904000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CRM_UI", "hs_analytics_source_data_2": "userId:12282590", "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": "+555555555555", "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": "BR", "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": null, "hs_count_is_worked": null, "hs_created_by_conversations": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": "2023-01-30T23:17:09.904000+00:00", "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": "2023-01-31T00:31:07.832000+00:00", "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": "2023-01-31T00:31:07.832000+00:00", "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "test.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_meeting_activity": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "CRM_UI", "hs_latest_source_data_2": "userId:12282590", "hs_latest_source_timestamp": "2023-01-30T23:17:10.053000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": "Legitimate interest \u2013 prospect/lead", "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": "2023-01-30T23:17:09.904000+00:00", "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": "2023-01-31T00:31:07.832000+00:00", "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": null, "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 2501, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 2.93, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_1", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": "5555555555", "hs_sequences_actively_enrolled_count": 0, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": 4437928, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": 15272626409, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "", "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": "", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-04-04T15:11:47.143000+00:00", "lastname": "test", "lifecyclestage": "opportunity", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": "+555555555555", "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": null, "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": null, "work_email": null, "zip": null}, "createdAt": "2023-01-30T23:17:09.904Z", "updatedAt": "2023-04-04T15:11:47.143Z", "archived": false}, "emitted_at": 1690397694273} -{"stream": "contacts", "data": {"id": "2551", "properties": {"address": null, "annualrevenue": null, "associatedcompanyid": null, "associatedcompanylastupdated": null, "city": null, "closedate": null, "company": null, "company_size": null, "country": null, "createdate": "2023-01-31T00:11:58.499000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "test-integration-test-user-4@testmail.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": null, "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "2551", "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2023-01-31T00:11:58.499000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CRM_UI", "hs_analytics_source_data_2": "userId:12282590", "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": 1, "hs_count_is_worked": 0, "hs_created_by_conversations": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": "2023-01-31T00:11:58.499000+00:00", "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "testmail.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_meeting_activity": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "CRM_UI", "hs_latest_source_data_2": "userId:12282590", "hs_latest_source_timestamp": "2023-01-31T00:11:58.582000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": "Legitimate interest \u2013 prospect/lead", "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": "2023-01-31T00:11:58.499000+00:00", "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": null, "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 2551, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 2.93, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_1", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": null, "hs_sequences_actively_enrolled_count": 0, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": 15273775742, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": "2023-01-31T00:20:40.680000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-04-03T20:29:54.560000+00:00", "lastname": null, "lifecyclestage": "lead", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": null, "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": null, "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": null, "work_email": null, "zip": null}, "createdAt": "2023-01-31T00:11:58.499Z", "updatedAt": "2023-04-03T20:29:54.560Z", "archived": false}, "emitted_at": 1690397694273} -{"stream": "contacts", "data": {"id": "2601", "properties": {"address": null, "annualrevenue": null, "associatedcompanyid": null, "associatedcompanylastupdated": null, "city": null, "closedate": null, "company": null, "company_size": null, "country": null, "createdate": "2023-02-01T13:08:49.766000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "test.person@airbyte.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "TEST", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "2601", "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2023-02-01T13:08:49.735000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CRM_UI", "hs_analytics_source_data_2": "userId:12282590", "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": 1, "hs_count_is_worked": 0, "hs_created_by_conversations": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": "2023-02-01T13:08:49.735000+00:00", "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "airbyte.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_meeting_activity": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "CRM_UI", "hs_latest_source_data_2": "userId:12282590", "hs_latest_source_timestamp": "2023-02-01T13:08:49.863000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": "Performance of a contract", "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": "2023-02-01T13:08:49.735000+00:00", "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": null, "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": "12282590", "hs_marketable_reason_type": "USER_SET", "hs_marketable_status": "true", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 2601, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 3.15, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_1", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": null, "hs_sequences_actively_enrolled_count": 0, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": 15140764507, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": "2023-02-01T13:08:49.735000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-04-03T20:29:58.691000+00:00", "lastname": "TEST", "lifecyclestage": "lead", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": null, "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": null, "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": null, "work_email": null, "zip": null}, "createdAt": "2023-02-01T13:08:49.766Z", "updatedAt": "2023-04-03T20:29:58.691Z", "archived": false}, "emitted_at": 1690397694274} -{"stream": "contacts_list_memberships", "data": {"canonical-vid": 2501, "static-list-id": 60, "internal-list-id": 2147483643, "timestamp": 1675124235515, "vid": 2501, "is-member": true}, "emitted_at": 1685387177140} -{"stream": "contacts_list_memberships", "data": {"canonical-vid": 2501, "static-list-id": 61, "internal-list-id": 2147483643, "timestamp": 1675124259228, "vid": 2501, "is-member": true}, "emitted_at": 1685387177141} -{"stream": "contacts_list_memberships", "data": {"canonical-vid": 2501, "static-list-id": 166, "internal-list-id": 2147483643, "timestamp": 1675120848102, "vid": 2501, "is-member": true}, "emitted_at": 1685387177141} -{"stream": "deal_pipelines", "data": {"label": "New Business Pipeline", "displayOrder": 3, "active": true, "stages": [{"label": "Initial Qualification", "displayOrder": 0, "metadata": {"isClosed": "false", "probability": "0.1"}, "stageId": "9567448", "createdAt": 1610635973956, "updatedAt": 1680620354263, "active": true}, {"label": "Success! Closed Won", "displayOrder": 2, "metadata": {"isClosed": "true", "probability": "1.0"}, "stageId": "customclosedwonstage", "createdAt": 1610635973956, "updatedAt": 1680620354263, "active": true}, {"label": "Negotiation", "displayOrder": 1, "metadata": {"isClosed": "false", "probability": "0.5"}, "stageId": "9567449", "createdAt": 1610635973956, "updatedAt": 1680620354263, "active": true}, {"label": "Closed Lost", "displayOrder": 3, "metadata": {"isClosed": "false", "probability": "0.1"}, "stageId": "66894120", "createdAt": 1680620354263, "updatedAt": 1680620354263, "active": true}], "objectType": "DEAL", "objectTypeId": "0-3", "pipelineId": "b9152945-a594-4835-9676-a6f405fecd71", "createdAt": 1610635973956, "updatedAt": 1680620354263, "default": false}, "emitted_at": 1685387177933} -{"stream": "deals", "data": {"id": "4280411910", "properties": {"amount": 6, "amount_in_home_currency": 6, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2014-08-31T00:00:00+00:00", "createdate": "2021-02-22T14:01:11.762000+00:00", "days_to_close": 0, "dealname": "Test Deal 332", "dealstage": "appointmentscheduled", "dealtype": "newbusiness", "description": "Test deal", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": 95.0, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_2": null, "hs_arr": 0.0, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": null, "hs_createdate": "2021-02-22T14:01:11.762000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-02-22T14:01:11.762000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_days_to_close_raw": 0, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": null, "hs_exchange_rate": null, "hs_forecast_amount": 6, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_is_open_count": 1, "hs_lastmodifieddate": "2023-04-04T21:28:37.824000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_next_step": null, "hs_num_associated_deal_splits": null, "hs_num_of_associated_line_items": 1, "hs_num_target_accounts": 0, "hs_object_id": 4280411910, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 1.2000000000000002, "hs_projected_amount_in_home_currency": 1.2000000000000002, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": 95.0, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 77505247423, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-02-22T14:01:11.762000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": null, "num_notes": null, "pipeline": "default"}, "createdAt": "2021-02-22T14:01:11.762Z", "updatedAt": "2023-04-04T21:28:37.824Z", "archived": false, "companies": ["5183409178", "5183409178"], "line_items": ["5153237390"]}, "emitted_at": 1691507719264} -{"stream": "deals", "data": {"id": "4315375411", "properties": {"amount": 10, "amount_in_home_currency": 10, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2021-02-28T20:20:10.826000+00:00", "createdate": "2021-02-23T20:20:10.826000+00:00", "days_to_close": 5, "dealname": "Test deal 2", "dealstage": "appointmentscheduled", "dealtype": null, "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": 10.0, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": "", "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": "", "hs_analytics_latest_source_data_1": "", "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": "", "hs_analytics_latest_source_data_2": "", "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": "", "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": "", "hs_analytics_source_data_1": "", "hs_analytics_source_data_2": "", "hs_arr": 0.0, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2021-02-23T20:21:32.862000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-02-23T20:21:32.862000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_days_to_close_raw": 5, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": null, "hs_exchange_rate": null, "hs_forecast_amount": 10, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_is_open_count": 1, "hs_lastmodifieddate": "2023-01-30T23:10:56.577000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_next_step": null, "hs_num_associated_deal_splits": null, "hs_num_of_associated_line_items": 1, "hs_num_target_accounts": null, "hs_object_id": 4315375411, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 2.0, "hs_projected_amount_in_home_currency": 2.0, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": 10.0, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 77396026323, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-02-23T20:21:32.862000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": "2021-02-26T06:00:00+00:00", "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": 0, "num_notes": 2, "pipeline": "default"}, "createdAt": "2021-02-23T20:20:10.826Z", "updatedAt": "2023-01-30T23:10:56.577Z", "archived": false, "line_items": ["1188257165"]}, "emitted_at": 1691507719265} -{"stream": "deals", "data": {"id": "5313445525", "properties": {"amount": 60, "amount_in_home_currency": 60, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2021-05-31T10:21:28.593000+00:00", "createdate": "2021-05-21T10:21:28.593000+00:00", "days_to_close": 10, "dealname": "Test Deal AAAA", "dealstage": "appointmentscheduled", "dealtype": "newbusiness", "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": 60.0, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": "", "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": "", "hs_analytics_latest_source_data_1": "", "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": "", "hs_analytics_latest_source_data_2": "", "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": "", "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": "", "hs_analytics_source_data_1": "", "hs_analytics_source_data_2": "", "hs_arr": 60.0, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2021-05-21T10:22:40.228000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-05-21T10:22:40.228000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_days_to_close_raw": 10, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": 0.2, "hs_exchange_rate": null, "hs_forecast_amount": 60, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_is_open_count": 1, "hs_lastmodifieddate": "2023-01-23T15:35:59.701000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": 20.0, "hs_next_step": null, "hs_num_associated_deal_splits": 0, "hs_num_of_associated_line_items": 1, "hs_num_target_accounts": 0, "hs_object_id": 5313445525, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": "medium", "hs_projected_amount": 12.0, "hs_projected_amount_in_home_currency": 12.0, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": 60.0, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 69915158957, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-05-21T10:22:40.228000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": null, "num_notes": null, "pipeline": "default"}, "createdAt": "2021-05-21T10:21:28.593Z", "updatedAt": "2023-01-23T15:35:59.701Z", "archived": false, "companies": ["5438025334", "5438025334"], "line_items": ["1510167477"]}, "emitted_at": 1691507719265} -{"stream": "email_events", "data": {"id": "cd276838-3925-4649-9a38-2b61761362c4", "source": "SOURCE_HUBSPOT_CUSTOMER", "recipient": "testingapicontact_0@hubspot.com", "subscriptions": [{"id": 23704464, "status": "SUBSCRIBED", "legalBasisChange": {"legalBasisType": "LEGITIMATE_INTEREST_CLIENT", "legalBasisExplanation": "test", "optState": "OPT_IN"}}], "sourceId": "Self Service Resubscription", "created": 1675123491624, "type": "STATUSCHANGE", "portalId": 8727216, "appId": 0, "emailCampaignId": 0}, "emitted_at": 1688977609453} -{"stream": "email_subscriptions", "data": {"id": 23704464, "portalId": 8727216, "name": "Test sub", "description": "Test sub", "active": true, "internal": false, "category": "Marketing", "channel": "Email", "businessUnitId": 0}, "emitted_at": 1685387183303} -{"stream": "email_subscriptions", "data": {"id": 10798197, "portalId": 8727216, "name": "DONT USE ME", "description": "Receive feedback requests and customer service information.", "active": true, "internal": true, "category": "Service", "channel": "Email", "order": 0, "internalName": "SERVICE_HUB_FEEDBACK", "businessUnitId": 0}, "emitted_at": 1685387183304} -{"stream": "email_subscriptions", "data": {"id": 11890603, "portalId": 8727216, "name": "DONT USE ME ", "description": "TTTT", "active": true, "internal": false, "category": "", "channel": "", "order": 1, "businessUnitId": 0}, "emitted_at": 1685387183304} -{"stream": "engagements", "data": {"id": 11257289597, "portalId": 8727216, "active": true, "createdAt": 1614111907503, "lastUpdated": 1681915963485, "createdBy": 12282590, "modifiedBy": 12282590, "ownerId": 52550153, "type": "TASK", "timestamp": 1614319200000, "allAccessibleTeamIds": [], "bodyPreview": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "queueMembershipIds": [], "bodyPreviewIsTruncated": false, "bodyPreviewHtml": "\n \n \n Regarding note logged on Tuesday, February 23, 2021 10:25 PM\n \n", "gdprDeleted": false, "associations": {"contactIds": [], "companyIds": [], "dealIds": [4315375411], "ownerIds": [], "workflowIds": [], "ticketIds": [], "contentIds": [], "quoteIds": [], "marketingEventIds": []}, "attachments": [], "scheduledTasks": [{"engagementId": 11257289597, "portalId": 8727216, "engagementType": "TASK", "taskType": "REMINDER", "timestamp": 1614319200000, "uuid": "TASK:e41fd851-f7c7-4381-85fa-796d076163aa"}], "metadata": {"body": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "status": "NOT_STARTED", "forObjectType": "OWNER", "subject": "Follow up on Test deal 2", "taskType": "TODO", "reminders": [1614319200000], "priority": "NONE", "isAllDay": false}}, "emitted_at": 1685387185031} -{"stream": "engagements", "data": {"id": 30652596616, "portalId": 8727216, "active": true, "createdAt": 1675122083198, "lastUpdated": 1675122083198, "createdBy": 12282590, "modifiedBy": 12282590, "ownerId": 52550153, "type": "NOTE", "timestamp": 1675122083198, "source": "CRM_UI", "allAccessibleTeamIds": [], "bodyPreview": "test", "queueMembershipIds": [], "bodyPreviewIsTruncated": false, "bodyPreviewHtml": "\n \n \n
      \n

      test

      \n
      \n \n", "associations": {"contactIds": [], "companyIds": [], "dealIds": [], "ownerIds": [], "workflowIds": [], "ticketIds": [], "contentIds": [], "quoteIds": [], "marketingEventIds": []}, "attachments": [], "metadata": {"body": "

      test

      "}}, "emitted_at": 1685387185033} -{"stream": "engagements", "data": {"id": 30652597343, "portalId": 8727216, "active": true, "createdAt": 1675122108834, "lastUpdated": 1680621107231, "createdBy": 12282590, "modifiedBy": 12282590, "ownerId": 52550153, "type": "TASK", "timestamp": 1675407600000, "source": "CRM_UI", "allAccessibleTeamIds": [], "queueMembershipIds": [], "bodyPreviewIsTruncated": false, "associations": {"contactIds": [], "companyIds": [], "dealIds": [], "ownerIds": [], "workflowIds": [], "ticketIds": [], "contentIds": [], "quoteIds": [], "marketingEventIds": []}, "attachments": [], "scheduledTasks": [], "metadata": {"status": "NOT_STARTED", "forObjectType": "OWNER", "subject": "test", "taskType": "TODO", "reminders": [], "sendDefaultReminder": false, "priority": "NONE", "isAllDay": false}}, "emitted_at": 1685387185033} -{"stream": "engagements_notes", "data": {"id": "30652596616", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": "test", "hs_body_preview_html": "\n \n \n
      \n

      test

      \n
      \n \n", "hs_body_preview_is_truncated": false, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:41:23.198000+00:00", "hs_engagement_source": "CRM_UI", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_lastmodifieddate": "2023-01-30T23:41:23.198000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_note_body": "

      test

      ", "hs_object_id": 30652596616, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_timestamp": "2023-01-30T23:41:23.198000+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2023-01-30T23:41:23.198000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2023-01-30T23:41:23.198Z", "updatedAt": "2023-01-30T23:41:23.198Z", "archived": false}, "emitted_at": 1689697216801} -{"stream": "engagements_notes", "data": {"id": "30652613125", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": "test", "hs_body_preview_html": "\n \n \n
      \n

      test

      \n
      \n \n", "hs_body_preview_is_truncated": false, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:51:47.542000+00:00", "hs_engagement_source": "CRM_UI", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_lastmodifieddate": "2023-01-30T23:51:47.542000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_note_body": "

      test

      ", "hs_object_id": 30652613125, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_timestamp": "2023-01-30T23:51:47.542000+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2023-01-30T23:51:47.542000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2023-01-30T23:51:47.542Z", "updatedAt": "2023-01-30T23:51:47.542Z", "archived": false, "companies": ["11481383026"]}, "emitted_at": 1689697216801} -{"stream": "engagements_tasks", "data": {"id": "11257289597", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "hs_body_preview_html": "\n \n \n Regarding note logged on Tuesday, February 23, 2021 10:25 PM\n \n", "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2021-02-23T20:25:07.503000+00:00", "hs_engagement_source": null, "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": false, "hs_lastmodifieddate": "2023-04-19T14:52:43.485000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_num_associated_companies": 0, "hs_num_associated_contacts": 0, "hs_num_associated_deals": 1, "hs_num_associated_queue_objects": 1, "hs_num_associated_tickets": 0, "hs_object_id": 11257289597, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[{\"engagementId\":11257289597,\"portalId\":8727216,\"engagementType\":\"TASK\",\"taskType\":\"REMINDER\",\"timestamp\":1614319200000,\"uuid\":\"TASK:e41fd851-f7c7-4381-85fa-796d076163aa\"}]}", "hs_task_body": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": null, "hs_task_reminders": "1614319200000", "hs_task_repeat_interval": null, "hs_task_send_default_reminder": null, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "Follow up on Test deal 2", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_timestamp": "2021-02-26T06:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-02-23T20:25:07.503000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2021-02-23T20:25:07.503Z", "updatedAt": "2023-04-19T14:52:43.485Z", "archived": false, "deals": ["4315375411"]}, "emitted_at": 1689697218307} -{"stream": "engagements_tasks", "data": {"id": "30652597343", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": null, "hs_body_preview_html": null, "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:41:48.834000+00:00", "hs_engagement_source": "CRM_UI", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_lastmodifieddate": "2023-04-04T15:11:47.231000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_num_associated_companies": 0, "hs_num_associated_contacts": 0, "hs_num_associated_deals": 0, "hs_num_associated_queue_objects": 0, "hs_num_associated_tickets": 0, "hs_object_id": 30652597343, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "hs_task_body": null, "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": "[]", "hs_task_reminders": null, "hs_task_repeat_interval": null, "hs_task_send_default_reminder": false, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "test", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_timestamp": "2023-02-03T07:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2023-01-30T23:41:48.834000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2023-01-30T23:41:48.834Z", "updatedAt": "2023-04-04T15:11:47.231Z", "archived": false}, "emitted_at": 1689697218308} -{"stream": "engagements_tasks", "data": {"id": "30652613208", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": null, "hs_body_preview_html": null, "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:51:52.099000+00:00", "hs_engagement_source": "CRM_UI", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_lastmodifieddate": "2023-01-30T23:51:54.343000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_num_associated_companies": 1, "hs_num_associated_contacts": 0, "hs_num_associated_deals": 0, "hs_num_associated_queue_objects": 1, "hs_num_associated_tickets": 0, "hs_object_id": 30652613208, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "hs_task_body": null, "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": "[]", "hs_task_reminders": null, "hs_task_repeat_interval": null, "hs_task_send_default_reminder": false, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "test", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_timestamp": "2023-02-03T07:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2023-01-30T23:51:52.099000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2023-01-30T23:51:52.099Z", "updatedAt": "2023-01-30T23:51:54.343Z", "archived": false, "companies": ["11481383026"]}, "emitted_at": 1689697218309} -{"stream": "forms", "data": {"id": "03e69987-1dcb-4d55-9cb6-d3812ac00ee6", "name": "New form 93", "createdAt": "2023-02-13T16:56:33.108Z", "updatedAt": "2023-02-13T16:56:33.108Z", "archived": false, "fieldGroups": [{"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "email", "label": "Email", "required": true, "hidden": false, "fieldType": "email", "validation": {"blockedEmailDomains": [], "useDefaultBlockList": false}}]}], "configuration": {"language": "en", "cloneable": true, "postSubmitAction": {"type": "thank_you", "value": "Thanks for submitting the form."}, "editable": true, "archivable": true, "recaptchaEnabled": false, "notifyContactOwner": false, "notifyRecipients": ["12282590"], "createNewContactForNewEmail": false, "prePopulateKnownValues": true, "allowLinkToResetKnownValues": false}, "displayOptions": {"renderRawHtml": false, "theme": "default_style", "submitButtonText": "Submit", "style": {"fontFamily": "arial, helvetica, sans-serif", "backgroundWidth": "100%", "labelTextColor": "#33475b", "labelTextSize": "14px", "helpTextColor": "#7C98B6", "helpTextSize": "11px", "legalConsentTextColor": "#33475b", "legalConsentTextSize": "14px", "submitColor": "#ff7a59", "submitAlignment": "left", "submitFontColor": "#ffffff", "submitSize": "12px"}, "cssClass": "hs-form stacked"}, "legalConsentOptions": {"type": "implicit_consent_to_process", "communicationConsentText": "integrationtest is committed to protecting and respecting your privacy, and we\u2019ll only use your personal information to administer your account and to provide the products and services you requested from us. From time to time, we would like to contact you about our products and services, as well as other content that may be of interest to you. If you consent to us contacting you for this purpose, please tick below to say how you would like us to contact you:", "communicationsCheckboxes": [{"required": false, "subscriptionTypeId": 23704464, "label": "I agree to receive other communications from [MAIN] integration test account."}], "privacyText": "You may unsubscribe from these communications at any time. For more information on how to unsubscribe, our privacy practices, and how we are committed to protecting and respecting your privacy, please review our Privacy Policy.", "consentToProcessText": "By clicking submit below, you consent to allow integrationtest to store and process the personal information submitted above to provide you the content requested."}, "formType": "hubspot"}, "emitted_at": 1685387195424} -{"stream": "forms", "data": {"id": "0a7fd84f-471e-444a-a4e0-ca36d39f8af7", "name": "New form 27", "createdAt": "2023-02-13T16:45:22.640Z", "updatedAt": "2023-02-13T16:45:22.640Z", "archived": false, "fieldGroups": [{"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "email", "label": "Email", "required": true, "hidden": false, "fieldType": "email", "validation": {"blockedEmailDomains": [], "useDefaultBlockList": false}}]}], "configuration": {"language": "en", "cloneable": true, "postSubmitAction": {"type": "thank_you", "value": "Thanks for submitting the form."}, "editable": true, "archivable": true, "recaptchaEnabled": false, "notifyContactOwner": false, "notifyRecipients": ["12282590"], "createNewContactForNewEmail": false, "prePopulateKnownValues": true, "allowLinkToResetKnownValues": false}, "displayOptions": {"renderRawHtml": false, "theme": "default_style", "submitButtonText": "Submit", "style": {"fontFamily": "arial, helvetica, sans-serif", "backgroundWidth": "100%", "labelTextColor": "#33475b", "labelTextSize": "14px", "helpTextColor": "#7C98B6", "helpTextSize": "11px", "legalConsentTextColor": "#33475b", "legalConsentTextSize": "14px", "submitColor": "#ff7a59", "submitAlignment": "left", "submitFontColor": "#ffffff", "submitSize": "12px"}, "cssClass": "hs-form stacked"}, "legalConsentOptions": {"type": "implicit_consent_to_process", "communicationConsentText": "integrationtest is committed to protecting and respecting your privacy, and we\u2019ll only use your personal information to administer your account and to provide the products and services you requested from us. From time to time, we would like to contact you about our products and services, as well as other content that may be of interest to you. If you consent to us contacting you for this purpose, please tick below to say how you would like us to contact you:", "communicationsCheckboxes": [{"required": false, "subscriptionTypeId": 23704464, "label": "I agree to receive other communications from [MAIN] integration test account."}], "privacyText": "You may unsubscribe from these communications at any time. For more information on how to unsubscribe, our privacy practices, and how we are committed to protecting and respecting your privacy, please review our Privacy Policy.", "consentToProcessText": "By clicking submit below, you consent to allow integrationtest to store and process the personal information submitted above to provide you the content requested."}, "formType": "hubspot"}, "emitted_at": 1685387195425} -{"stream": "forms", "data": {"id": "0bf0c00f-e68d-4de2-8cd9-d9b04e41072f", "name": "New form 55", "createdAt": "2023-02-13T16:50:27.345Z", "updatedAt": "2023-02-13T16:50:27.345Z", "archived": false, "fieldGroups": [{"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "email", "label": "Email", "required": true, "hidden": false, "fieldType": "email", "validation": {"blockedEmailDomains": [], "useDefaultBlockList": false}}]}], "configuration": {"language": "en", "cloneable": true, "postSubmitAction": {"type": "thank_you", "value": "Thanks for submitting the form."}, "editable": true, "archivable": true, "recaptchaEnabled": false, "notifyContactOwner": false, "notifyRecipients": ["12282590"], "createNewContactForNewEmail": false, "prePopulateKnownValues": true, "allowLinkToResetKnownValues": false}, "displayOptions": {"renderRawHtml": false, "theme": "default_style", "submitButtonText": "Submit", "style": {"fontFamily": "arial, helvetica, sans-serif", "backgroundWidth": "100%", "labelTextColor": "#33475b", "labelTextSize": "14px", "helpTextColor": "#7C98B6", "helpTextSize": "11px", "legalConsentTextColor": "#33475b", "legalConsentTextSize": "14px", "submitColor": "#ff7a59", "submitAlignment": "left", "submitFontColor": "#ffffff", "submitSize": "12px"}, "cssClass": "hs-form stacked"}, "legalConsentOptions": {"type": "implicit_consent_to_process", "communicationConsentText": "integrationtest is committed to protecting and respecting your privacy, and we\u2019ll only use your personal information to administer your account and to provide the products and services you requested from us. From time to time, we would like to contact you about our products and services, as well as other content that may be of interest to you. If you consent to us contacting you for this purpose, please tick below to say how you would like us to contact you:", "communicationsCheckboxes": [{"required": false, "subscriptionTypeId": 23704464, "label": "I agree to receive other communications from [MAIN] integration test account."}], "privacyText": "You may unsubscribe from these communications at any time. For more information on how to unsubscribe, our privacy practices, and how we are committed to protecting and respecting your privacy, please review our Privacy Policy.", "consentToProcessText": "By clicking submit below, you consent to allow integrationtest to store and process the personal information submitted above to provide you the content requested."}, "formType": "hubspot"}, "emitted_at": 1685387195425} -{"stream": "goals", "data": {"id": "221880757009", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-07-31T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]},{\"property\":\"hubspot_owner_id\",\"operator\":\"EQ\",\"value\":\"111730024\"}]}]", "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-08-01T00:45:14.830000+00:00", "hs_lastmodifieddate": "2023-08-18T14:59:25.726000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757009, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_outcome": "completed", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-07-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "achieved", "hs_status_display_order": 4, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-08-18T14:59:25.726Z", "archived": false}, "emitted_at": 1692530575531} -{"stream": "goals", "data": {"id": "221880757010", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-09-30T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]},{\"property\":\"hubspot_owner_id\",\"operator\":\"EQ\",\"value\":\"111730024\"}]}]", "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-04-10T22:31:22.345000+00:00", "hs_lastmodifieddate": "2023-08-18T14:59:25.726000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757010, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_outcome": "in_progress", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-09-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "pending", "hs_status_display_order": 5, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-08-18T14:59:25.726Z", "archived": false}, "emitted_at": 1692530575532} -{"stream": "goals", "data": {"id": "221880757011", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-08-31T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]},{\"property\":\"hubspot_owner_id\",\"operator\":\"EQ\",\"value\":\"111730024\"}]}]", "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-08-19T22:11:13.040000+00:00", "hs_lastmodifieddate": "2023-08-19T22:11:13.080000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757011, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_outcome": "in_progress", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-08-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "in_progress", "hs_status_display_order": 1, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-08-19T22:11:13.080Z", "archived": false}, "emitted_at": 1692530575532} -{"stream": "line_items", "data": {"id": "4617680695", "properties": {"amount": 34.0, "createdate": "2023-01-31T00:31:29.812000+00:00", "description": null, "discount": null, "hs_acv": 34.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_allow_buyer_selected_quantity": null, "hs_arr": 0.0, "hs_billing_start_delay_days": null, "hs_billing_start_delay_months": null, "hs_billing_start_delay_type": null, "hs_cost_of_goods_sold": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_external_id": null, "hs_images": null, "hs_lastmodifieddate": "2023-01-31T00:31:29.812000+00:00", "hs_line_item_currency_code": null, "hs_margin": 34.0, "hs_margin_acv": 34.0, "hs_margin_arr": 0.0, "hs_margin_mrr": 0.0, "hs_margin_tcv": 34.0, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_object_id": 4617680695, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_position_on_quote": 0, "hs_pre_discount_amount": 34.0, "hs_product_id": null, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_end_date": null, "hs_recurring_billing_number_of_payments": 1, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_recurring_billing_terms": null, "hs_sku": null, "hs_sync_amount": null, "hs_tcv": 34.0, "hs_term_in_months": null, "hs_total_discount": 0.0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_variant_id": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "test", "price": 34, "quantity": 1, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2023-01-31T00:31:29.812Z", "updatedAt": "2023-01-31T00:31:29.812Z", "archived": false}, "emitted_at": 1689697250135} -{"stream": "line_items", "data": {"id": "5153237390", "properties": {"amount": 95.0, "createdate": "2023-04-04T21:28:36.663000+00:00", "description": "Baseball hat, medium", "discount": 5, "hs_acv": 95.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_allow_buyer_selected_quantity": null, "hs_arr": 0.0, "hs_billing_start_delay_days": null, "hs_billing_start_delay_months": null, "hs_billing_start_delay_type": null, "hs_cost_of_goods_sold": 5, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_external_id": null, "hs_images": null, "hs_lastmodifieddate": "2023-04-04T21:28:36.663000+00:00", "hs_line_item_currency_code": null, "hs_margin": 90.0, "hs_margin_acv": 90.0, "hs_margin_arr": 0.0, "hs_margin_mrr": 0.0, "hs_margin_tcv": 90.0, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_object_id": 5153237390, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_position_on_quote": 0, "hs_pre_discount_amount": 100.0, "hs_product_id": 646778218, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_end_date": null, "hs_recurring_billing_number_of_payments": 1, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_recurring_billing_terms": null, "hs_sku": null, "hs_sync_amount": null, "hs_tcv": 95.0, "hs_term_in_months": null, "hs_total_discount": 5.0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_variant_id": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Blue Hat", "price": 100, "quantity": 1, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2023-04-04T21:28:36.663Z", "updatedAt": "2023-04-04T21:28:36.663Z", "archived": false}, "emitted_at": 1689697250136} -{"stream":"marketing_emails","data":{"ab":false,"abHoursToWait":4,"abSampleSizeDefault":null,"abSamplingDefault":null,"abSuccessMetric":null,"abTestPercentage":50,"abVariation":false,"absoluteUrl":"http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de","allEmailCampaignIds":[243851494],"analyticsPageId":"100523515217","analyticsPageType":"email","archivedAt":0,"archivedInDashboard":false,"audienceAccess":"PUBLIC","author":"integration-test@airbyte.io","authorName":"Team-1 Airbyte","blogRssSettings":null,"canSpamSettingsId":36765207029,"categoryId":2,"contentAccessRuleIds":[],"contentAccessRuleTypes":[],"contentTypeCategory":2,"createPage":false,"created":1675121582718,"createdById":12282590,"currentState":"PUBLISHED","currentlyPublished":true,"customReplyTo":"","customReplyToEnabled":false,"domain":"","emailBody":"{% content_attribute \"email_body\" %}{{ default_email_body }}{% end_content_attribute %}","emailNote":"","emailTemplateMode":"DRAG_AND_DROP","emailType":"BATCH_EMAIL","emailbodyPlaintext":"","feedbackSurveyId":null,"flexAreas":{"main":{"boxed":false,"isSingleColumnFullWidth":false,"sections":[{"columns":[{"id":"column-0-0","widgets":["module-0-0-0"],"width":12}],"id":"section-0","style":{"backgroundColor":"#eaf0f6","backgroundType":"CONTENT","paddingBottom":"10px","paddingTop":"10px"}},{"columns":[{"id":"column-1-0","widgets":["module-1-0-0"],"width":12}],"id":"section-1","style":{"backgroundType":"CONTENT","paddingBottom":"30px","paddingTop":"30px"}},{"columns":[{"id":"column-2-0","widgets":["module-2-0-0"],"width":12}],"id":"section-2","style":{"backgroundColor":"","backgroundType":"CONTENT","paddingBottom":"20px","paddingTop":"20px"}}]}},"freezeDate":1675121645993,"fromName":"Team Airbyte","hasContentAccessRules":false,"htmlTitle":"","id":100523515217,"isCreatedFomSandboxSync":false,"isGraymailSuppressionEnabled":true,"isInstanceLayoutPage":false,"isPublished":true,"isRecipientFatigueSuppressionEnabled":null,"language":"en","layoutSections":{},"liveDomain":"integrationtest-dev-8727216-8727216.hs-sites.com","mailingListsExcluded":[],"mailingListsIncluded":[],"maxRssEntries":5,"metaDescription":"","name":"test","pageExpiryEnabled":false,"pageRedirected":false,"pastMabExperimentIds":[],"portalId":8727216,"previewKey":"nlkwziGL","primaryEmailCampaignId":243851494,"processingStatus":"PUBLISHED","publishDate":1675121645997,"publishImmediately":true,"publishedAt":1675121646297,"publishedByEmail":"integration-test@airbyte.io","publishedById":12282590,"publishedByName":"Team-1 Airbyte","publishedUrl":"http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de","replyTo":"integration-test@airbyte.io","resolvedDomain":"integrationtest-dev-8727216-8727216.hs-sites.com","rssEmailByText":"By","rssEmailClickThroughText":"Read more »","rssEmailCommentText":"Comment »","rssEmailEntryTemplateEnabled":false,"rssEmailImageMaxWidth":0,"rssEmailUrl":"","sections":{},"securityState":"NONE","selected":0,"slug":"-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de","smartEmailFields":{},"state":"PUBLISHED","stats":{"counters":{"sent":0,"open":0,"delivered":0,"bounce":0,"unsubscribed":0,"click":0,"reply":0,"dropped":1,"selected":1,"spamreport":0,"suppressed":0,"hardbounced":0,"softbounced":0,"pending":0,"contactslost":0,"notsent":1},"deviceBreakdown":{"open_device_type":{"computer":0,"mobile":0,"unknown":0},"click_device_type":{"computer":0,"mobile":0,"unknown":0}},"failedToLoad":false,"qualifierStats":{},"ratios":{"clickratio":0,"clickthroughratio":0,"deliveredratio":0,"openratio":0,"replyratio":0,"unsubscribedratio":0,"spamreportratio":0,"bounceratio":0,"hardbounceratio":0,"softbounceratio":0,"contactslostratio":0,"pendingratio":0,"notsentratio":100}},"styleSettings":{"background_color":"#EAF0F6","background_image":null,"background_image_type":null,"body_border_color":"#EAF0F6","body_border_color_choice":"BORDER_MANUAL","body_border_width":"1","body_color":"#ffffff","color_picker_favorite1":null,"color_picker_favorite2":null,"color_picker_favorite3":null,"color_picker_favorite4":null,"color_picker_favorite5":null,"color_picker_favorite6":null,"email_body_padding":null,"email_body_width":null,"heading_one_font":{"bold":null,"color":null,"font":null,"font_style":{},"italic":null,"size":"28","underline":null},"heading_two_font":{"bold":null,"color":null,"font":null,"font_style":{},"italic":null,"size":"22","underline":null},"links_font":{"bold":false,"color":"#00a4bd","font":null,"font_style":{},"italic":false,"size":null,"underline":true},"primary_accent_color":null,"primary_font":"Arial, sans-serif","primary_font_color":"#23496d","primary_font_line_height":null,"primary_font_size":"15","secondary_accent_color":null,"secondary_font":"Arial, sans-serif","secondary_font_color":"#23496d","secondary_font_line_height":null,"secondary_font_size":"12","use_email_client_default_settings":false,"user_module_defaults":{"button_email":{"background_color":"#00a4bd","corner_radius":8,"font":"Arial, sans-serif","font_color":"#ffffff","font_size":16,"font_style":{"color":"#ffffff","font":"Arial, sans-serif","size":{"units":"px","value":16},"styles":{"bold":false,"italic":false,"underline":false}}},"email_divider":{"color":{"color":"#23496d","opacity":100},"height":1,"line_type":"solid"}}},"subcategory":"batch","subject":"test","subscription":23704464,"subscriptionName":"Test sub","teamPerms":[],"templatePath":"@hubspot/email/dnd/welcome.html","transactional":false,"translations":{},"unpublishedAt":0,"updated":1675121702583,"updatedById":12282590,"url":"http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de","useRssHeadlineAsSubject":false,"userPerms":[],"vidsExcluded":[],"vidsIncluded":[2501],"visibleToAll":true},"emitted_at":1688060624527} -{"stream":"owners", "data": {"id": "52550153", "email": "integration-test@airbyte.io", "firstName": "Team-1", "lastName": "Airbyte", "userId": 12282590, "createdAt": "2020-10-28T21:17:56.082Z", "updatedAt": "2023-01-31T00:25:34.448Z", "archived": false}, "emitted_at": 1685387219734} -{"stream": "products", "data": {"id": "1783898388", "properties": {"amount": null, "createdate": "2023-01-31T00:08:27.149000+00:00", "description": null, "discount": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_avatar_filemanager_key": null, "hs_cost_of_goods_sold": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_folder_id": null, "hs_images": null, "hs_lastmodifieddate": "2023-01-31T00:28:58.829000+00:00", "hs_merged_object_ids": null, "hs_object_id": 1783898388, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_product_type": "inventory", "hs_read_only": null, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_sku": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "test", "price": 1, "quantity": null, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2023-01-31T00:08:27.149Z", "updatedAt": "2023-01-31T00:28:58.829Z", "archived": false}, "emitted_at": 1689697253487} -{"stream": "property_history", "data": {"value": "sh", "source-type": "CRM_UI", "source-id": "userId:12282590", "source-label": null, "updated-by-user-id": 12282590, "timestamp": 1673944973763, "selected": false, "property": "firstname", "vid": 151}, "emitted_at": 1685387222765} -{"stream": "subscription_changes", "data": {"timestamp": 1675123491624, "recipient": "testingapicontact_0@hubspot.com", "portalId": 8727216, "normalizedEmailId": "6b59e963-cabc-4bf8-baec-feab401bdd98", "changes": [{"source": "SOURCE_HUBSPOT_CUSTOMER", "timestamp": 1675123491624, "change": "SUBSCRIBED", "portalId": 8727216, "subscriptionId": 23704464, "causedByEvent": {"id": "cd276838-3925-4649-9a38-2b61761362c4", "created": 1675123491624}, "changeType": "SUBSCRIPTION_STATUS"}]}, "emitted_at": 1685387223625} -{"stream": "tickets", "data": {"id": "1401690016", "properties": {"closed_date": null, "content": null, "created_by": null, "createdate": "2023-01-30T23:52:42.464000+00:00", "first_agent_reply_date": null, "hs_all_accessible_team_ids": null, "hs_all_conversation_mentions": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_assignment_method": null, "hs_auto_generated_from_thread_id": null, "hs_conversations_originating_message_id": null, "hs_conversations_originating_thread_id": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_custom_inbox": null, "hs_date_entered_1": "2023-01-30T23:52:42.464000+00:00", "hs_date_entered_2": null, "hs_date_entered_3": null, "hs_date_entered_4": null, "hs_date_exited_1": null, "hs_date_exited_2": null, "hs_date_exited_3": null, "hs_date_exited_4": null, "hs_external_object_ids": null, "hs_feedback_last_ces_follow_up": null, "hs_feedback_last_ces_rating": null, "hs_feedback_last_survey_date": null, "hs_file_upload": null, "hs_first_agent_message_sent_at": null, "hs_helpdesk_sort_timestamp": "2023-01-30T23:52:42.464000+00:00", "hs_in_helpdesk": null, "hs_inbox_id": null, "hs_last_email_activity": null, "hs_last_email_date": null, "hs_last_message_from_visitor": false, "hs_last_message_received_at": null, "hs_last_message_sent_at": null, "hs_lastactivitydate": null, "hs_lastcontacted": null, "hs_lastmodifieddate": "2023-01-30T23:52:43.939000+00:00", "hs_latest_message_seen_by_agent_ids": null, "hs_merged_object_ids": null, "hs_most_relevant_sla_status": null, "hs_most_relevant_sla_type": null, "hs_msteams_message_id": null, "hs_nextactivitydate": null, "hs_num_associated_companies": 0, "hs_num_times_contacted": 0, "hs_object_id": 1401690016, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_originating_channel_instance_id": null, "hs_originating_email_engagement_id": null, "hs_originating_generic_channel_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "0", "hs_pipeline_stage": "1", "hs_read_only": null, "hs_resolution": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_thread_ids_to_restore": null, "hs_ticket_category": null, "hs_ticket_id": 1401690016, "hs_ticket_priority": null, "hs_time_in_1": 17411822718, "hs_time_in_2": null, "hs_time_in_3": null, "hs_time_in_4": null, "hs_time_to_close_sla_at": null, "hs_time_to_close_sla_status": null, "hs_time_to_first_response_sla_at": null, "hs_time_to_first_response_sla_status": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2023-01-30T23:52:42.464000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "last_engagement_date": null, "last_reply_date": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "nps_follow_up_answer": null, "nps_follow_up_question_version": null, "nps_score": null, "num_contacted_notes": null, "num_notes": null, "source_ref": null, "source_thread_id": null, "source_type": null, "subject": "test", "tags": null, "time_to_close": null, "time_to_first_agent_reply": null}, "createdAt": "2023-01-30T23:52:42.464Z", "updatedAt": "2023-01-30T23:52:43.939Z", "archived": false}, "emitted_at": 1692534585461} -{"stream": "workflows", "data": {"migrationStatus": {"portalId": 8727216, "workflowId": 40032127, "migrationStatus": "EXECUTION_MIGRATED", "enrollmentMigrationStatus": "PLATFORM_OWNED", "platformOwnsActions": true, "lastSuccessfulMigrationTimestamp": null, "enrollmentMigrationTimestamp": null, "flowId": 321690519}, "name": "Unnamed workflow - Mon Mar 15 2021 12:58:03 GMT+0200 (cloned)", "id": 40032127, "type": "DRIP_DELAY", "enabled": true, "creationSource": {"sourceApplication": {"source": "DIRECT_API", "serviceName": "AutomationPlatformService-userweb"}, "createdByUser": {"userId": 12282590, "userEmail": "integration-test@airbyte.io"}, "clonedFromWorkflowId": 23314874, "createdAt": 1675124258186}, "updateSource": {"sourceApplication": {"source": "DIRECT_API", "serviceName": "AutomationPlatformService-userweb"}, "updatedByUser": {"userId": 12282590, "userEmail": "integration-test@airbyte.io"}, "updatedAt": 1675124308226}, "originalAuthorUserId": 12282590, "contactListIds": {"enrolled": 167, "active": 168, "completed": 169, "succeeded": 170}, "personaTagIds": [], "lastUpdatedByUserId": 12282590, "contactCounts": {"active": 0, "enrolled": 0}, "portalId": 8727216, "insertedAt": 1675124258190, "updatedAt": 1675124308226, "description": ""}, "emitted_at": 1685387227678} -{"stream": "pets", "data": {"id": "5936415312", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:08:50.632000+00:00", "hs_lastmodifieddate": "2023-04-12T17:08:50.632000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5936415312, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "pet_name": "Marcos Pet", "pet_type": "Dog"}, "createdAt": "2023-04-12T17:08:50.632Z", "updatedAt": "2023-04-12T17:08:50.632Z", "archived": false}, "emitted_at": 1689697266624} -{"stream": "pets", "data": {"id": "5938880054", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:53:12.692000+00:00", "hs_lastmodifieddate": "2023-04-12T17:53:12.692000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5938880054, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "pet_name": "Integration Test Pet", "pet_type": "Unknown"}, "createdAt": "2023-04-12T17:53:12.692Z", "updatedAt": "2023-04-12T17:53:12.692Z", "archived": false}, "emitted_at": 1689697266625} -{"stream": "cars", "data": {"id": "5938880072", "properties": {"car_id": 1, "car_name": 3232324, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:57:15.836000+00:00", "hs_lastmodifieddate": "2023-04-12T17:59:20.189000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5938880072, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-12T17:57:15.836Z", "updatedAt": "2023-04-12T17:59:20.189Z", "archived": false}, "emitted_at": 1689697267882} -{"stream": "cars", "data": {"id": "5938880073", "properties": {"car_id": 2, "car_name": 23232, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:57:20.583000+00:00", "hs_lastmodifieddate": "2023-04-12T17:59:20.189000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5938880073, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-12T17:57:20.583Z", "updatedAt": "2023-04-12T17:59:20.189Z", "archived": false}, "emitted_at": 1689697267883} -{"stream": "contacts_merged_audit", "data": {"canonical-vid": 651, "vid-to-merge": 201, "timestamp": 1688758327178, "entity-id": "auth:app-cookie | auth-level:app | login-id:integration-test@airbyte.io-1688758203663 | hub-id:8727216 | user-id:12282590 | origin-ip:2804:1b3:8402:b1f4:7d1b:f62e:b071:593d | correlation-id:3f139cd7-66fc-4300-8cbc-e6c1fe9ea7d1", "user-id": 12282590, "num-properties-moved": 45, "merged_from_email": {"value": "testingapis@hubspot.com", "source-type": "API", "source-id": null, "source-label": null, "updated-by-user-id": null, "timestamp": 1610634377014, "selected": false}, "merged_to_email": {"value": "testingapicontact_1@hubspot.com", "source-type": "API", "source-id": null, "source-label": null, "updated-by-user-id": null, "timestamp": 1634044981830, "selected": false}, "first-name": "test", "last-name": "testerson"}, "emitted_at": 1688758844966} +{"stream": "campaigns", "data": {"id": 243851494, "lastUpdatedTime": 1675121674226, "appId": 113, "appName": "Batch", "contentId": 100523515217, "subject": "test", "name": "test", "counters": {"dropped": 1}, "lastProcessingFinishedAt": 1675121674000, "lastProcessingStartedAt": 1675121671000, "lastProcessingStateChangeAt": 1675121674000, "numIncluded": 1, "processingState": "DONE", "type": "BATCH_EMAIL", "counters_dropped": 1}, "emitted_at": 1697714185530} +{"stream": "campaigns", "data": {"id": 115429485, "lastUpdatedTime": 1615506409286, "appId": 113, "appName": "Batch", "contentId": 42931043849, "subject": "Test subj", "name": "Test subj", "counters": {"processed": 1, "deferred": 1, "mta_dropped": 1, "dropped": 3, "sent": 0}, "lastProcessingFinishedAt": 1615504712000, "lastProcessingStartedAt": 1615504687000, "lastProcessingStateChangeAt": 1615504712000, "numIncluded": 3, "processingState": "DONE", "type": "BATCH_EMAIL", "counters_processed": 1, "counters_deferred": 1, "counters_mta_dropped": 1, "counters_dropped": 3, "counters_sent": 0}, "emitted_at": 1697714185763} +{"stream": "companies", "data": {"id": "4992593519", "properties": {"about_us": null, "address": null, "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": null, "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-10T07:58:09.554000+00:00", "custom_company_property": null, "days_to_close": null, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "airbyte.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": null, "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": "2021-05-21T10:17:06.028000+00:00", "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": null, "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_num_page_views": null, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": null, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": null, "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": "2021-05-21T10:17:28.964000+00:00", "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-01-26T11:45:49.817000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": null, "hs_num_blockers": null, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": null, "hs_num_decision_makers": null, "hs_num_open_deals": 1, "hs_object_id": 4992593519, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": null, "hs_predictivecontactscore_v2": null, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.5476861596107483, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": 76121938222, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2020-12-10T07:58:09.554000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": false, "lifecyclestage": "opportunity", "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Airbyte test1", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_associated_deals": 1, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 200, "phone": "+1 415-307-4864", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": null, "recent_deal_close_date": null, "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": null, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;segment;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "website": "airbyte.io", "zip": "94114"}, "createdAt": "2020-12-10T07:58:09.554Z", "updatedAt": "2023-01-26T11:45:49.817Z", "archived": false, "properties_about_us": null, "properties_address": null, "properties_address2": null, "properties_annualrevenue": null, "properties_city": "San Francisco", "properties_closedate": null, "properties_closedate_timestamp_earliest_value_a2a17e6e": null, "properties_country": "United States", "properties_createdate": "2020-12-10T07:58:09.554000+00:00", "properties_custom_company_property": null, "properties_days_to_close": null, "properties_description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_domain": "airbyte.io", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_facebook_company_page": null, "properties_facebookfans": null, "properties_first_contact_createdate": null, "properties_first_contact_createdate_timestamp_earliest_value_78b50eea": null, "properties_first_conversion_date": null, "properties_first_conversion_date_timestamp_earliest_value_61f58f2c": null, "properties_first_conversion_event_name": null, "properties_first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "properties_first_deal_created_date": "2021-05-21T10:17:06.028000+00:00", "properties_founded_year": "2020", "properties_googleplus_page": null, "properties_hs_additional_domains": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_first_timestamp": null, "properties_hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "properties_hs_analytics_latest_source": null, "properties_hs_analytics_latest_source_data_1": null, "properties_hs_analytics_latest_source_data_2": null, "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_num_page_views": null, "properties_hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "properties_hs_analytics_num_visits": null, "properties_hs_analytics_num_visits_cardinality_sum_53d952a6": null, "properties_hs_analytics_source": null, "properties_hs_analytics_source_data_1": null, "properties_hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "properties_hs_analytics_source_data_2": null, "properties_hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "properties_hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "properties_hs_annual_revenue_currency_code": "USD", "properties_hs_avatar_filemanager_key": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": "2021-05-21T10:17:28.964000+00:00", "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": null, "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_ideal_customer_profile": null, "properties_hs_is_target_account": null, "properties_hs_last_booked_meeting_date": null, "properties_hs_last_logged_call_date": null, "properties_hs_last_open_task_date": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": "2023-01-26T11:45:49.817000+00:00", "properties_hs_latest_createdate_of_active_subscriptions": null, "properties_hs_latest_meeting_activity": null, "properties_hs_lead_status": null, "properties_hs_merged_object_ids": null, "properties_hs_num_blockers": null, "properties_hs_num_child_companies": 0, "properties_hs_num_contacts_with_buying_roles": null, "properties_hs_num_decision_makers": null, "properties_hs_num_open_deals": 1, "properties_hs_object_id": 4992593519, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_parent_company_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": null, "properties_hs_predictivecontactscore_v2": null, "properties_hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_target_account": null, "properties_hs_target_account_probability": 0.5476861596107483, "properties_hs_target_account_recommendation_snooze_time": null, "properties_hs_target_account_recommendation_state": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": 76121938222, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": null, "properties_hs_total_deal_value": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2020-12-10T07:58:09.554000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_is_public": false, "properties_lifecyclestage": "opportunity", "properties_linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "properties_linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_name": "Airbyte test1", "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_associated_deals": 1, "properties_num_contacted_notes": null, "properties_num_conversion_events": null, "properties_num_conversion_events_cardinality_sum_d095f14b": null, "properties_num_notes": null, "properties_numberofemployees": 200, "properties_phone": "+1 415-307-4864", "properties_recent_conversion_date": null, "properties_recent_conversion_date_timestamp_latest_value_72856da1": null, "properties_recent_conversion_event_name": null, "properties_recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_state": "CA", "properties_timezone": "America/Los_Angeles", "properties_total_money_raised": null, "properties_total_revenue": null, "properties_twitterbio": null, "properties_twitterfollowers": null, "properties_twitterhandle": "AirbyteHQ", "properties_type": null, "properties_web_technologies": "slack;segment;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "properties_website": "airbyte.io", "properties_zip": "94114"}, "emitted_at": 1697714187356} +{"stream": "companies", "data": {"id": "5000526215", "properties": {"about_us": null, "address": null, "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": "2023-04-04T15:00:58.081000+00:00", "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-11T01:27:40.002000+00:00", "custom_company_property": null, "days_to_close": 844, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "dataline.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": "2020-12-11T01:29:50.116000+00:00", "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": "2021-01-13T10:30:42.221000+00:00", "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": "OFFLINE", "hs_analytics_latest_source_data_1": "CONTACTS", "hs_analytics_latest_source_data_2": "CRM_UI", "hs_analytics_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "hs_analytics_num_page_views": 0, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": 0, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": "CRM_UI", "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": "2023-04-04T15:00:58.081000+00:00", "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": "2021-02-23T20:21:06.027000+00:00", "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": "2023-04-04T15:00:58.081000+00:00", "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-09-07T03:58:14.126000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": "5183403213", "hs_num_blockers": 0, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": 0, "hs_num_decision_makers": 0, "hs_num_open_deals": 2, "hs_object_id": 5000526215, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "companies-lifecycle-pipeline", "hs_predictivecontactscore_v2": 0.3, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.46257445216178894, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": 17093729103, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": 66508792054, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": 60010, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2020-12-11T01:27:40.002000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": false, "lifecyclestage": "customer", "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Dataline", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 1, "num_associated_deals": 3, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 25, "phone": "", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": 60000, "recent_deal_close_date": "2023-04-04T14:59:45.103000+00:00", "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": 60000, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;segment;google_tag_manager;cloud_flare;google_analytics;intercom;lever;google_apps", "website": "dataline.io", "zip": ""}, "createdAt": "2020-12-11T01:27:40.002Z", "updatedAt": "2023-09-07T03:58:14.126Z", "archived": false, "contacts": ["151", "151"], "properties_about_us": null, "properties_address": null, "properties_address2": null, "properties_annualrevenue": null, "properties_city": "San Francisco", "properties_closedate": "2023-04-04T15:00:58.081000+00:00", "properties_closedate_timestamp_earliest_value_a2a17e6e": null, "properties_country": "United States", "properties_createdate": "2020-12-11T01:27:40.002000+00:00", "properties_custom_company_property": null, "properties_days_to_close": 844, "properties_description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_domain": "dataline.io", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_facebook_company_page": null, "properties_facebookfans": null, "properties_first_contact_createdate": "2020-12-11T01:29:50.116000+00:00", "properties_first_contact_createdate_timestamp_earliest_value_78b50eea": null, "properties_first_conversion_date": null, "properties_first_conversion_date_timestamp_earliest_value_61f58f2c": null, "properties_first_conversion_event_name": null, "properties_first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "properties_first_deal_created_date": "2021-01-13T10:30:42.221000+00:00", "properties_founded_year": "2020", "properties_googleplus_page": null, "properties_hs_additional_domains": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "properties_hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "properties_hs_analytics_latest_source": "OFFLINE", "properties_hs_analytics_latest_source_data_1": "CONTACTS", "properties_hs_analytics_latest_source_data_2": "CRM_UI", "properties_hs_analytics_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "properties_hs_analytics_num_page_views": 0, "properties_hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "properties_hs_analytics_num_visits": 0, "properties_hs_analytics_num_visits_cardinality_sum_53d952a6": null, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "CONTACTS", "properties_hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "properties_hs_analytics_source_data_2": "CRM_UI", "properties_hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "properties_hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "properties_hs_annual_revenue_currency_code": "USD", "properties_hs_avatar_filemanager_key": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_date_entered_customer": "2023-04-04T15:00:58.081000+00:00", "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": "2021-02-23T20:21:06.027000+00:00", "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": null, "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": "2023-04-04T15:00:58.081000+00:00", "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_ideal_customer_profile": null, "properties_hs_is_target_account": null, "properties_hs_last_booked_meeting_date": null, "properties_hs_last_logged_call_date": null, "properties_hs_last_open_task_date": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": "2023-09-07T03:58:14.126000+00:00", "properties_hs_latest_createdate_of_active_subscriptions": null, "properties_hs_latest_meeting_activity": null, "properties_hs_lead_status": null, "properties_hs_merged_object_ids": "5183403213", "properties_hs_num_blockers": 0, "properties_hs_num_child_companies": 0, "properties_hs_num_contacts_with_buying_roles": 0, "properties_hs_num_decision_makers": 0, "properties_hs_num_open_deals": 2, "properties_hs_object_id": 5000526215, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_parent_company_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "companies-lifecycle-pipeline", "properties_hs_predictivecontactscore_v2": 0.3, "properties_hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_target_account": null, "properties_hs_target_account_probability": 0.46257445216178894, "properties_hs_target_account_recommendation_snooze_time": null, "properties_hs_target_account_recommendation_state": null, "properties_hs_time_in_customer": 17093729103, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": 66508792054, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": null, "properties_hs_total_deal_value": 60010, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2020-12-11T01:27:40.002000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_is_public": false, "properties_lifecyclestage": "customer", "properties_linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "properties_linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_name": "Dataline", "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 1, "properties_num_associated_deals": 3, "properties_num_contacted_notes": null, "properties_num_conversion_events": null, "properties_num_conversion_events_cardinality_sum_d095f14b": null, "properties_num_notes": null, "properties_numberofemployees": 25, "properties_phone": "", "properties_recent_conversion_date": null, "properties_recent_conversion_date_timestamp_latest_value_72856da1": null, "properties_recent_conversion_event_name": null, "properties_recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "properties_recent_deal_amount": 60000, "properties_recent_deal_close_date": "2023-04-04T14:59:45.103000+00:00", "properties_state": "CA", "properties_timezone": "America/Los_Angeles", "properties_total_money_raised": null, "properties_total_revenue": 60000, "properties_twitterbio": null, "properties_twitterfollowers": null, "properties_twitterhandle": "AirbyteHQ", "properties_type": null, "properties_web_technologies": "slack;segment;google_tag_manager;cloud_flare;google_analytics;intercom;lever;google_apps", "properties_website": "dataline.io", "properties_zip": ""}, "emitted_at": 1697714187359} +{"stream": "companies", "data": {"id": "5000787595", "properties": {"about_us": null, "address": "2261 Market Street", "address2": null, "annualrevenue": null, "city": "San Francisco", "closedate": null, "closedate_timestamp_earliest_value_a2a17e6e": null, "country": "United States", "createdate": "2020-12-11T01:28:27.673000+00:00", "custom_company_property": null, "days_to_close": null, "description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "domain": "Daxtarity.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "facebook_company_page": null, "facebookfans": null, "first_contact_createdate": null, "first_contact_createdate_timestamp_earliest_value_78b50eea": null, "first_conversion_date": null, "first_conversion_date_timestamp_earliest_value_61f58f2c": null, "first_conversion_event_name": null, "first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "first_deal_created_date": null, "founded_year": "2020", "googleplus_page": null, "hs_additional_domains": null, "hs_all_accessible_team_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_first_timestamp": null, "hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "hs_analytics_latest_source": "", "hs_analytics_latest_source_data_1": "", "hs_analytics_latest_source_data_2": "", "hs_analytics_latest_source_timestamp": null, "hs_analytics_num_page_views": null, "hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "hs_analytics_num_visits": null, "hs_analytics_num_visits_cardinality_sum_53d952a6": null, "hs_analytics_source": "", "hs_analytics_source_data_1": "", "hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "hs_analytics_source_data_2": "", "hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "hs_annual_revenue_currency_code": "USD", "hs_avatar_filemanager_key": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_ideal_customer_profile": null, "hs_is_target_account": null, "hs_last_booked_meeting_date": null, "hs_last_logged_call_date": null, "hs_last_open_task_date": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": "2023-01-23T15:41:56.644000+00:00", "hs_latest_createdate_of_active_subscriptions": null, "hs_latest_meeting_activity": null, "hs_lead_status": null, "hs_merged_object_ids": null, "hs_num_blockers": 0, "hs_num_child_companies": 0, "hs_num_contacts_with_buying_roles": 0, "hs_num_decision_makers": 0, "hs_num_open_deals": 0, "hs_object_id": 5000787595, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_parent_company_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": null, "hs_predictivecontactscore_v2": null, "hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_target_account": null, "hs_target_account_probability": 0.4076234698295593, "hs_target_account_recommendation_snooze_time": null, "hs_target_account_recommendation_state": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_total_deal_value": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2020-12-11T01:28:27.673000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "is_public": false, "lifecyclestage": null, "linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "name": "Daxtarity", "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": null, "num_conversion_events_cardinality_sum_d095f14b": null, "num_notes": null, "numberofemployees": 50, "phone": "+1 415-307-4864", "recent_conversion_date": null, "recent_conversion_date_timestamp_latest_value_72856da1": null, "recent_conversion_event_name": null, "recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "recent_deal_amount": null, "recent_deal_close_date": null, "state": "CA", "timezone": "America/Los_Angeles", "total_money_raised": null, "total_revenue": null, "twitterbio": null, "twitterfollowers": null, "twitterhandle": "AirbyteHQ", "type": null, "web_technologies": "slack;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "website": "Daxtarity.com", "zip": "94114"}, "createdAt": "2020-12-11T01:28:27.673Z", "updatedAt": "2023-01-23T15:41:56.644Z", "archived": false, "properties_about_us": null, "properties_address": "2261 Market Street", "properties_address2": null, "properties_annualrevenue": null, "properties_city": "San Francisco", "properties_closedate": null, "properties_closedate_timestamp_earliest_value_a2a17e6e": null, "properties_country": "United States", "properties_createdate": "2020-12-11T01:28:27.673000+00:00", "properties_custom_company_property": null, "properties_days_to_close": null, "properties_description": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_domain": "Daxtarity.com", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_facebook_company_page": null, "properties_facebookfans": null, "properties_first_contact_createdate": null, "properties_first_contact_createdate_timestamp_earliest_value_78b50eea": null, "properties_first_conversion_date": null, "properties_first_conversion_date_timestamp_earliest_value_61f58f2c": null, "properties_first_conversion_event_name": null, "properties_first_conversion_event_name_timestamp_earliest_value_68ddae0a": null, "properties_first_deal_created_date": null, "properties_founded_year": "2020", "properties_googleplus_page": null, "properties_hs_additional_domains": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_first_timestamp": null, "properties_hs_analytics_first_timestamp_timestamp_earliest_value_11e3a63a": null, "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_touch_converting_campaign_timestamp_earliest_value_4757fe10": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_first_visit_timestamp_timestamp_earliest_value_accc17ae": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_timestamp_timestamp_latest_value_4e16365a": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_touch_converting_campaign_timestamp_latest_value_81a64e30": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_last_visit_timestamp_timestamp_latest_value_999a0fce": null, "properties_hs_analytics_latest_source": "", "properties_hs_analytics_latest_source_data_1": "", "properties_hs_analytics_latest_source_data_2": "", "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_num_page_views": null, "properties_hs_analytics_num_page_views_cardinality_sum_e46e85b0": null, "properties_hs_analytics_num_visits": null, "properties_hs_analytics_num_visits_cardinality_sum_53d952a6": null, "properties_hs_analytics_source": "", "properties_hs_analytics_source_data_1": "", "properties_hs_analytics_source_data_1_timestamp_earliest_value_9b2f1fa1": null, "properties_hs_analytics_source_data_2": "", "properties_hs_analytics_source_data_2_timestamp_earliest_value_9b2f9400": null, "properties_hs_analytics_source_timestamp_earliest_value_25a3a52c": null, "properties_hs_annual_revenue_currency_code": "USD", "properties_hs_avatar_filemanager_key": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": null, "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": null, "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_ideal_customer_profile": null, "properties_hs_is_target_account": null, "properties_hs_last_booked_meeting_date": null, "properties_hs_last_logged_call_date": null, "properties_hs_last_open_task_date": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": "2023-01-23T15:41:56.644000+00:00", "properties_hs_latest_createdate_of_active_subscriptions": null, "properties_hs_latest_meeting_activity": null, "properties_hs_lead_status": null, "properties_hs_merged_object_ids": null, "properties_hs_num_blockers": 0, "properties_hs_num_child_companies": 0, "properties_hs_num_contacts_with_buying_roles": 0, "properties_hs_num_decision_makers": 0, "properties_hs_num_open_deals": 0, "properties_hs_object_id": 5000787595, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_parent_company_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": null, "properties_hs_predictivecontactscore_v2": null, "properties_hs_predictivecontactscore_v2_next_max_max_d4e58c1e": null, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_target_account": null, "properties_hs_target_account_probability": 0.4076234698295593, "properties_hs_target_account_recommendation_snooze_time": null, "properties_hs_target_account_recommendation_state": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": null, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": null, "properties_hs_total_deal_value": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2020-12-11T01:28:27.673000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_is_public": false, "properties_lifecyclestage": null, "properties_linkedin_company_page": "https://www.linkedin.com/company/airbytehq", "properties_linkedinbio": "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", "properties_name": "Daxtarity", "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_associated_deals": null, "properties_num_contacted_notes": null, "properties_num_conversion_events": null, "properties_num_conversion_events_cardinality_sum_d095f14b": null, "properties_num_notes": null, "properties_numberofemployees": 50, "properties_phone": "+1 415-307-4864", "properties_recent_conversion_date": null, "properties_recent_conversion_date_timestamp_latest_value_72856da1": null, "properties_recent_conversion_event_name": null, "properties_recent_conversion_event_name_timestamp_latest_value_66c820bf": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_state": "CA", "properties_timezone": "America/Los_Angeles", "properties_total_money_raised": null, "properties_total_revenue": null, "properties_twitterbio": null, "properties_twitterfollowers": null, "properties_twitterhandle": "AirbyteHQ", "properties_type": null, "properties_web_technologies": "slack;google_tag_manager;greenhouse;google_analytics;intercom;piwik;google_apps;hubspot;facebook_advertiser", "properties_website": "Daxtarity.com", "properties_zip": "94114"}, "emitted_at": 1697714187363} +{"stream": "contact_lists", "data": {"portalId": 8727216, "listId": 1, "createdAt": 1610634707370, "updatedAt": 1610634721116, "name": "tweeters", "listType": "DYNAMIC", "authorId": 0, "filters": [], "metaData": {"size": 0, "lastSizeChangeAt": 1625270400000, "processing": "DONE", "lastProcessingStateChangeAt": 1610634721950, "error": "", "listReferencesCount": null, "parentFolderId": null}, "archived": false, "teamIds": [], "ilsFilterBranch": "{\"filterBranchOperator\":\"OR\",\"filters\":[],\"filterBranches\":[{\"filterBranchOperator\":\"AND\",\"filters\":[{\"filterType\":\"PROPERTY\",\"property\":\"twitterhandle\",\"operation\":{\"propertyType\":\"string\",\"operator\":\"IS_EQUAL_TO\",\"value\":\"@hubspot\",\"defaultValue\":null,\"includeObjectsWithNoValueSet\":false,\"operationType\":\"string\",\"operatorName\":\"IS_EQUAL_TO\"},\"frameworkFilterId\":null}],\"filterBranches\":[],\"filterBranchType\":\"AND\"}],\"filterBranchType\":\"OR\"}", "readOnly": false, "dynamic": true, "internal": false, "limitExempt": false, "metaData_size": 0, "metaData_lastSizeChangeAt": 1625270400000, "metaData_processing": "DONE", "metaData_lastProcessingStateChangeAt": 1610634721950, "metaData_error": "", "metaData_listReferencesCount": null, "metaData_parentFolderId": null}, "emitted_at": 1697714189110} +{"stream": "contact_lists", "data": {"portalId": 8727216, "listId": 2, "createdAt": 1610634770432, "updatedAt": 1610634780637, "name": "tweeters 1", "listType": "DYNAMIC", "authorId": 0, "filters": [], "metaData": {"size": 0, "lastSizeChangeAt": 1625270400000, "processing": "DONE", "lastProcessingStateChangeAt": 1610634781147, "error": "", "listReferencesCount": null, "parentFolderId": null}, "archived": false, "teamIds": [], "ilsFilterBranch": "{\"filterBranchOperator\":\"OR\",\"filters\":[],\"filterBranches\":[{\"filterBranchOperator\":\"AND\",\"filters\":[{\"filterType\":\"PROPERTY\",\"property\":\"twitterhandle\",\"operation\":{\"propertyType\":\"string\",\"operator\":\"IS_EQUAL_TO\",\"value\":\"@hubspot\",\"defaultValue\":null,\"includeObjectsWithNoValueSet\":false,\"operationType\":\"string\",\"operatorName\":\"IS_EQUAL_TO\"},\"frameworkFilterId\":null}],\"filterBranches\":[],\"filterBranchType\":\"AND\"}],\"filterBranchType\":\"OR\"}", "readOnly": false, "dynamic": true, "internal": false, "limitExempt": false, "metaData_size": 0, "metaData_lastSizeChangeAt": 1625270400000, "metaData_processing": "DONE", "metaData_lastProcessingStateChangeAt": 1610634781147, "metaData_error": "", "metaData_listReferencesCount": null, "metaData_parentFolderId": null}, "emitted_at": 1697714189112} +{"stream": "contact_lists", "data": {"portalId": 8727216, "listId": 3, "createdAt": 1610634774356, "updatedAt": 1610634787734, "name": "tweeters 2", "listType": "DYNAMIC", "authorId": 0, "filters": [], "metaData": {"size": 0, "lastSizeChangeAt": 1625270400000, "processing": "DONE", "lastProcessingStateChangeAt": 1610634788528, "error": "", "listReferencesCount": null, "parentFolderId": null}, "archived": false, "teamIds": [], "ilsFilterBranch": "{\"filterBranchOperator\":\"OR\",\"filters\":[],\"filterBranches\":[{\"filterBranchOperator\":\"AND\",\"filters\":[{\"filterType\":\"PROPERTY\",\"property\":\"twitterhandle\",\"operation\":{\"propertyType\":\"string\",\"operator\":\"IS_EQUAL_TO\",\"value\":\"@hubspot\",\"defaultValue\":null,\"includeObjectsWithNoValueSet\":false,\"operationType\":\"string\",\"operatorName\":\"IS_EQUAL_TO\"},\"frameworkFilterId\":null}],\"filterBranches\":[],\"filterBranchType\":\"AND\"}],\"filterBranchType\":\"OR\"}", "readOnly": false, "dynamic": true, "internal": false, "limitExempt": false, "metaData_size": 0, "metaData_lastSizeChangeAt": 1625270400000, "metaData_processing": "DONE", "metaData_lastProcessingStateChangeAt": 1610634788528, "metaData_error": "", "metaData_listReferencesCount": null, "metaData_parentFolderId": null}, "emitted_at": 1697714189113} +{"stream": "contacts", "data": {"id": "151", "properties": {"address": null, "annualrevenue": null, "associatedcompanyid": 5000526215, "associatedcompanylastupdated": null, "city": null, "closedate": null, "company": null, "company_size": null, "country": null, "createdate": "2020-12-11T01:29:50.116000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "shef@dne.io", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "she", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "151", "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_2": "CRM_UI", "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": 1, "hs_count_is_worked": 0, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": "2020-12-11T01:29:50.116000+00:00", "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "dne.io", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_disqualified_lead_date": null, "hs_latest_meeting_activity": null, "hs_latest_open_lead_date": null, "hs_latest_qualified_lead_date": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "CONTACTS", "hs_latest_source_data_2": "CRM_UI", "hs_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": null, "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": "2020-12-11T01:29:50.116000+00:00", "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 151, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.3, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_3", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": null, "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": 94172907549, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_v2_cumulative_time_in_customer": null, "hs_v2_cumulative_time_in_evangelist": null, "hs_v2_cumulative_time_in_lead": null, "hs_v2_cumulative_time_in_marketingqualifiedlead": null, "hs_v2_cumulative_time_in_opportunity": null, "hs_v2_cumulative_time_in_other": null, "hs_v2_cumulative_time_in_salesqualifiedlead": null, "hs_v2_cumulative_time_in_subscriber": null, "hs_v2_date_entered_customer": null, "hs_v2_date_entered_evangelist": null, "hs_v2_date_entered_lead": null, "hs_v2_date_entered_marketingqualifiedlead": null, "hs_v2_date_entered_opportunity": null, "hs_v2_date_entered_other": null, "hs_v2_date_entered_salesqualifiedlead": null, "hs_v2_date_entered_subscriber": "2020-12-11T01:29:50.116000+00:00", "hs_v2_date_exited_customer": null, "hs_v2_date_exited_evangelist": null, "hs_v2_date_exited_lead": null, "hs_v2_date_exited_marketingqualifiedlead": null, "hs_v2_date_exited_opportunity": null, "hs_v2_date_exited_other": null, "hs_v2_date_exited_salesqualifiedlead": null, "hs_v2_date_exited_subscriber": null, "hs_v2_latest_time_in_customer": null, "hs_v2_latest_time_in_evangelist": null, "hs_v2_latest_time_in_lead": null, "hs_v2_latest_time_in_marketingqualifiedlead": null, "hs_v2_latest_time_in_opportunity": null, "hs_v2_latest_time_in_other": null, "hs_v2_latest_time_in_salesqualifiedlead": null, "hs_v2_latest_time_in_subscriber": null, "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": "2020-12-11T01:29:50.093000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-11-22T21:10:04.346000+00:00", "lastname": "nad", "lifecyclestage": "subscriber", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": null, "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": null, "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": null, "work_email": null, "zip": null}, "createdAt": "2020-12-11T01:29:50.116Z", "updatedAt": "2023-11-22T21:10:04.346Z", "archived": false, "companies": ["5000526215", "5000526215"], "properties_address": null, "properties_annualrevenue": null, "properties_associatedcompanyid": 5000526215, "properties_associatedcompanylastupdated": null, "properties_city": null, "properties_closedate": null, "properties_company": null, "properties_company_size": null, "properties_country": null, "properties_createdate": "2020-12-11T01:29:50.116000+00:00", "properties_currentlyinworkflow": null, "properties_date_of_birth": null, "properties_days_to_close": null, "properties_degree": null, "properties_email": "shef@dne.io", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_fax": null, "properties_field_of_study": null, "properties_first_conversion_date": null, "properties_first_conversion_event_name": null, "properties_first_deal_created_date": null, "properties_firstname": "she", "properties_gender": null, "properties_graduation_date": null, "properties_hs_additional_emails": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_contact_vids": "151", "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_average_page_views": 0, "properties_hs_analytics_first_referrer": null, "properties_hs_analytics_first_timestamp": "2020-12-11T01:29:50.116000+00:00", "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_url": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_last_referrer": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_url": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_num_event_completions": 0, "properties_hs_analytics_num_page_views": 0, "properties_hs_analytics_num_visits": 0, "properties_hs_analytics_revenue": 0.0, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "CONTACTS", "properties_hs_analytics_source_data_2": "CRM_UI", "properties_hs_avatar_filemanager_key": null, "properties_hs_buying_role": null, "properties_hs_calculated_form_submissions": null, "properties_hs_calculated_merged_vids": null, "properties_hs_calculated_mobile_number": null, "properties_hs_calculated_phone_number": null, "properties_hs_calculated_phone_number_area_code": null, "properties_hs_calculated_phone_number_country_code": null, "properties_hs_calculated_phone_number_region_code": null, "properties_hs_clicked_linkedin_ad": null, "properties_hs_content_membership_email": null, "properties_hs_content_membership_email_confirmed": null, "properties_hs_content_membership_notes": null, "properties_hs_content_membership_registered_at": null, "properties_hs_content_membership_registration_domain_sent_to": null, "properties_hs_content_membership_registration_email_sent_at": null, "properties_hs_content_membership_status": null, "properties_hs_conversations_visitor_email": null, "properties_hs_count_is_unworked": 1, "properties_hs_count_is_worked": 0, "properties_hs_created_by_conversations": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": null, "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": "2020-12-11T01:29:50.116000+00:00", "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_document_last_revisited": null, "properties_hs_email_bad_address": null, "properties_hs_email_bounce": null, "properties_hs_email_click": null, "properties_hs_email_customer_quarantined_reason": null, "properties_hs_email_delivered": null, "properties_hs_email_domain": "dne.io", "properties_hs_email_first_click_date": null, "properties_hs_email_first_open_date": null, "properties_hs_email_first_reply_date": null, "properties_hs_email_first_send_date": null, "properties_hs_email_hard_bounce_reason": null, "properties_hs_email_hard_bounce_reason_enum": null, "properties_hs_email_is_ineligible": null, "properties_hs_email_last_click_date": null, "properties_hs_email_last_email_name": null, "properties_hs_email_last_open_date": null, "properties_hs_email_last_reply_date": null, "properties_hs_email_last_send_date": null, "properties_hs_email_open": null, "properties_hs_email_optout": null, "properties_hs_email_optout_10798197": null, "properties_hs_email_optout_11890603": null, "properties_hs_email_optout_11890831": null, "properties_hs_email_optout_23704464": null, "properties_hs_email_optout_94692364": null, "properties_hs_email_quarantined": null, "properties_hs_email_quarantined_reason": null, "properties_hs_email_recipient_fatigue_recovery_time": null, "properties_hs_email_replied": null, "properties_hs_email_sends_since_last_engagement": null, "properties_hs_emailconfirmationstatus": null, "properties_hs_facebook_ad_clicked": null, "properties_hs_facebook_click_id": null, "properties_hs_feedback_last_nps_follow_up": null, "properties_hs_feedback_last_nps_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_feedback_show_nps_web_survey": null, "properties_hs_first_engagement_object_id": null, "properties_hs_first_outreach_date": null, "properties_hs_first_subscription_create_date": null, "properties_hs_google_click_id": null, "properties_hs_has_active_subscription": null, "properties_hs_ip_timezone": null, "properties_hs_is_contact": true, "properties_hs_is_unworked": true, "properties_hs_language": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": null, "properties_hs_latest_disqualified_lead_date": null, "properties_hs_latest_meeting_activity": null, "properties_hs_latest_open_lead_date": null, "properties_hs_latest_qualified_lead_date": null, "properties_hs_latest_sequence_ended_date": null, "properties_hs_latest_sequence_enrolled": null, "properties_hs_latest_sequence_enrolled_date": null, "properties_hs_latest_sequence_finished_date": null, "properties_hs_latest_sequence_unenrolled_date": null, "properties_hs_latest_source": "OFFLINE", "properties_hs_latest_source_data_1": "CONTACTS", "properties_hs_latest_source_data_2": "CRM_UI", "properties_hs_latest_source_timestamp": "2020-12-11T01:29:50.153000+00:00", "properties_hs_latest_subscription_create_date": null, "properties_hs_lead_status": null, "properties_hs_legal_basis": null, "properties_hs_lifecyclestage_customer_date": null, "properties_hs_lifecyclestage_evangelist_date": null, "properties_hs_lifecyclestage_lead_date": null, "properties_hs_lifecyclestage_marketingqualifiedlead_date": null, "properties_hs_lifecyclestage_opportunity_date": null, "properties_hs_lifecyclestage_other_date": null, "properties_hs_lifecyclestage_salesqualifiedlead_date": null, "properties_hs_lifecyclestage_subscriber_date": "2020-12-11T01:29:50.116000+00:00", "properties_hs_linkedin_ad_clicked": null, "properties_hs_marketable_reason_id": null, "properties_hs_marketable_reason_type": null, "properties_hs_marketable_status": "false", "properties_hs_marketable_until_renewal": "false", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 151, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_persona": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "contacts-lifecycle-pipeline", "properties_hs_predictivecontactscore": null, "properties_hs_predictivecontactscore_v2": 0.3, "properties_hs_predictivecontactscorebucket": null, "properties_hs_predictivescoringtier": "tier_3", "properties_hs_read_only": null, "properties_hs_sa_first_engagement_date": null, "properties_hs_sa_first_engagement_descr": null, "properties_hs_sa_first_engagement_object_type": null, "properties_hs_sales_email_last_clicked": null, "properties_hs_sales_email_last_opened": null, "properties_hs_sales_email_last_replied": null, "properties_hs_searchable_calculated_international_mobile_number": null, "properties_hs_searchable_calculated_international_phone_number": null, "properties_hs_searchable_calculated_mobile_number": null, "properties_hs_searchable_calculated_phone_number": null, "properties_hs_sequences_actively_enrolled_count": null, "properties_hs_sequences_enrolled_count": null, "properties_hs_sequences_is_enrolled": null, "properties_hs_testpurge": null, "properties_hs_testrollback": null, "properties_hs_time_between_contact_creation_and_deal_close": null, "properties_hs_time_between_contact_creation_and_deal_creation": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": null, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": 94172907549, "properties_hs_time_to_first_engagement": null, "properties_hs_time_to_move_from_lead_to_customer": null, "properties_hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_opportunity_to_customer": null, "properties_hs_time_to_move_from_salesqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_subscriber_to_customer": null, "properties_hs_timezone": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_v2_cumulative_time_in_customer": null, "properties_hs_v2_cumulative_time_in_evangelist": null, "properties_hs_v2_cumulative_time_in_lead": null, "properties_hs_v2_cumulative_time_in_marketingqualifiedlead": null, "properties_hs_v2_cumulative_time_in_opportunity": null, "properties_hs_v2_cumulative_time_in_other": null, "properties_hs_v2_cumulative_time_in_salesqualifiedlead": null, "properties_hs_v2_cumulative_time_in_subscriber": null, "properties_hs_v2_date_entered_customer": null, "properties_hs_v2_date_entered_evangelist": null, "properties_hs_v2_date_entered_lead": null, "properties_hs_v2_date_entered_marketingqualifiedlead": null, "properties_hs_v2_date_entered_opportunity": null, "properties_hs_v2_date_entered_other": null, "properties_hs_v2_date_entered_salesqualifiedlead": null, "properties_hs_v2_date_entered_subscriber": "2020-12-11T01:29:50.116000+00:00", "properties_hs_v2_date_exited_customer": null, "properties_hs_v2_date_exited_evangelist": null, "properties_hs_v2_date_exited_lead": null, "properties_hs_v2_date_exited_marketingqualifiedlead": null, "properties_hs_v2_date_exited_opportunity": null, "properties_hs_v2_date_exited_other": null, "properties_hs_v2_date_exited_salesqualifiedlead": null, "properties_hs_v2_date_exited_subscriber": null, "properties_hs_v2_latest_time_in_customer": null, "properties_hs_v2_latest_time_in_evangelist": null, "properties_hs_v2_latest_time_in_lead": null, "properties_hs_v2_latest_time_in_marketingqualifiedlead": null, "properties_hs_v2_latest_time_in_opportunity": null, "properties_hs_v2_latest_time_in_other": null, "properties_hs_v2_latest_time_in_salesqualifiedlead": null, "properties_hs_v2_latest_time_in_subscriber": null, "properties_hs_was_imported": null, "properties_hs_whatsapp_phone_number": null, "properties_hubspot_owner_assigneddate": "2020-12-11T01:29:50.093000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_ip_city": null, "properties_ip_country": null, "properties_ip_country_code": null, "properties_ip_latlon": null, "properties_ip_state": null, "properties_ip_state_code": null, "properties_ip_zipcode": null, "properties_job_function": null, "properties_jobtitle": null, "properties_lastmodifieddate": "2023-11-22T21:10:04.346000+00:00", "properties_lastname": "nad", "properties_lifecyclestage": "subscriber", "properties_marital_status": null, "properties_message": null, "properties_military_status": null, "properties_mobilephone": null, "properties_my_custom_test_property": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_deals": null, "properties_num_contacted_notes": null, "properties_num_conversion_events": 0, "properties_num_notes": null, "properties_num_unique_conversion_events": 0, "properties_numemployees": null, "properties_phone": null, "properties_recent_conversion_date": null, "properties_recent_conversion_event_name": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_relationship_status": null, "properties_salutation": null, "properties_school": null, "properties_seniority": null, "properties_start_date": null, "properties_state": null, "properties_surveymonkeyeventlastupdated": null, "properties_test": null, "properties_total_revenue": null, "properties_twitterhandle": null, "properties_webinareventlastupdated": null, "properties_website": null, "properties_work_email": null, "properties_zip": null}, "emitted_at": 1701823098427} +{"stream": "contacts", "data": {"id": "251", "properties": {"address": "25000000 First Street", "annualrevenue": null, "associatedcompanyid": 5170561229, "associatedcompanylastupdated": null, "city": "Cambridge", "closedate": null, "company": "HubSpot", "company_size": null, "country": "USA", "createdate": "2021-02-22T14:05:09.944000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "testingdsapis@hubspot.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "Test User 5001", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "251", "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2021-02-22T14:05:09.944000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "API", "hs_analytics_source_data_2": null, "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": null, "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": null, "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": null, "hs_count_is_worked": null, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": null, "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": "2021-02-22T14:05:09.944000+00:00", "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "hubspot.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": null, "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_disqualified_lead_date": null, "hs_latest_meeting_activity": null, "hs_latest_open_lead_date": null, "hs_latest_qualified_lead_date": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "API", "hs_latest_source_data_2": null, "hs_latest_source_timestamp": "2021-02-22T14:05:10.036000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": null, "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": "2021-02-22T14:05:09.944000+00:00", "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 251, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.29, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_4", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": "5551222323", "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": null, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": 87820387720, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_v2_cumulative_time_in_customer": null, "hs_v2_cumulative_time_in_evangelist": null, "hs_v2_cumulative_time_in_lead": null, "hs_v2_cumulative_time_in_marketingqualifiedlead": null, "hs_v2_cumulative_time_in_opportunity": null, "hs_v2_cumulative_time_in_other": null, "hs_v2_cumulative_time_in_salesqualifiedlead": null, "hs_v2_cumulative_time_in_subscriber": null, "hs_v2_date_entered_customer": null, "hs_v2_date_entered_evangelist": null, "hs_v2_date_entered_lead": null, "hs_v2_date_entered_marketingqualifiedlead": null, "hs_v2_date_entered_opportunity": null, "hs_v2_date_entered_other": null, "hs_v2_date_entered_salesqualifiedlead": null, "hs_v2_date_entered_subscriber": "2021-02-22T14:05:09.944000+00:00", "hs_v2_date_exited_customer": null, "hs_v2_date_exited_evangelist": null, "hs_v2_date_exited_lead": null, "hs_v2_date_exited_marketingqualifiedlead": null, "hs_v2_date_exited_opportunity": null, "hs_v2_date_exited_other": null, "hs_v2_date_exited_salesqualifiedlead": null, "hs_v2_date_exited_subscriber": null, "hs_v2_latest_time_in_customer": null, "hs_v2_latest_time_in_evangelist": null, "hs_v2_latest_time_in_lead": null, "hs_v2_latest_time_in_marketingqualifiedlead": null, "hs_v2_latest_time_in_opportunity": null, "hs_v2_latest_time_in_other": null, "hs_v2_latest_time_in_salesqualifiedlead": null, "hs_v2_latest_time_in_subscriber": null, "hs_was_imported": null, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-03-21T19:29:13.036000+00:00", "lastname": "Test Lastname 5001", "lifecyclestage": "subscriber", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": "555-122-2323", "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": "MA", "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": "http://hubspot.com", "work_email": null, "zip": "02139"}, "createdAt": "2021-02-22T14:05:09.944Z", "updatedAt": "2023-03-21T19:29:13.036Z", "archived": false, "companies": ["5170561229", "5170561229"], "properties_address": "25000000 First Street", "properties_annualrevenue": null, "properties_associatedcompanyid": 5170561229, "properties_associatedcompanylastupdated": null, "properties_city": "Cambridge", "properties_closedate": null, "properties_company": "HubSpot", "properties_company_size": null, "properties_country": "USA", "properties_createdate": "2021-02-22T14:05:09.944000+00:00", "properties_currentlyinworkflow": null, "properties_date_of_birth": null, "properties_days_to_close": null, "properties_degree": null, "properties_email": "testingdsapis@hubspot.com", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_fax": null, "properties_field_of_study": null, "properties_first_conversion_date": null, "properties_first_conversion_event_name": null, "properties_first_deal_created_date": null, "properties_firstname": "Test User 5001", "properties_gender": null, "properties_graduation_date": null, "properties_hs_additional_emails": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_contact_vids": "251", "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_analytics_average_page_views": 0, "properties_hs_analytics_first_referrer": null, "properties_hs_analytics_first_timestamp": "2021-02-22T14:05:09.944000+00:00", "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_url": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_last_referrer": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_url": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_num_event_completions": 0, "properties_hs_analytics_num_page_views": 0, "properties_hs_analytics_num_visits": 0, "properties_hs_analytics_revenue": 0.0, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "API", "properties_hs_analytics_source_data_2": null, "properties_hs_avatar_filemanager_key": null, "properties_hs_buying_role": null, "properties_hs_calculated_form_submissions": null, "properties_hs_calculated_merged_vids": null, "properties_hs_calculated_mobile_number": null, "properties_hs_calculated_phone_number": null, "properties_hs_calculated_phone_number_area_code": null, "properties_hs_calculated_phone_number_country_code": null, "properties_hs_calculated_phone_number_region_code": null, "properties_hs_clicked_linkedin_ad": null, "properties_hs_content_membership_email": null, "properties_hs_content_membership_email_confirmed": null, "properties_hs_content_membership_notes": null, "properties_hs_content_membership_registered_at": null, "properties_hs_content_membership_registration_domain_sent_to": null, "properties_hs_content_membership_registration_email_sent_at": null, "properties_hs_content_membership_status": null, "properties_hs_conversations_visitor_email": null, "properties_hs_count_is_unworked": null, "properties_hs_count_is_worked": null, "properties_hs_created_by_conversations": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": null, "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": null, "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": "2021-02-22T14:05:09.944000+00:00", "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_document_last_revisited": null, "properties_hs_email_bad_address": null, "properties_hs_email_bounce": null, "properties_hs_email_click": null, "properties_hs_email_customer_quarantined_reason": null, "properties_hs_email_delivered": null, "properties_hs_email_domain": "hubspot.com", "properties_hs_email_first_click_date": null, "properties_hs_email_first_open_date": null, "properties_hs_email_first_reply_date": null, "properties_hs_email_first_send_date": null, "properties_hs_email_hard_bounce_reason": null, "properties_hs_email_hard_bounce_reason_enum": null, "properties_hs_email_is_ineligible": null, "properties_hs_email_last_click_date": null, "properties_hs_email_last_email_name": null, "properties_hs_email_last_open_date": null, "properties_hs_email_last_reply_date": null, "properties_hs_email_last_send_date": null, "properties_hs_email_open": null, "properties_hs_email_optout": null, "properties_hs_email_optout_10798197": null, "properties_hs_email_optout_11890603": null, "properties_hs_email_optout_11890831": null, "properties_hs_email_optout_23704464": null, "properties_hs_email_optout_94692364": null, "properties_hs_email_quarantined": null, "properties_hs_email_quarantined_reason": null, "properties_hs_email_recipient_fatigue_recovery_time": null, "properties_hs_email_replied": null, "properties_hs_email_sends_since_last_engagement": null, "properties_hs_emailconfirmationstatus": null, "properties_hs_facebook_ad_clicked": null, "properties_hs_facebook_click_id": null, "properties_hs_feedback_last_nps_follow_up": null, "properties_hs_feedback_last_nps_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_feedback_show_nps_web_survey": null, "properties_hs_first_engagement_object_id": null, "properties_hs_first_outreach_date": null, "properties_hs_first_subscription_create_date": null, "properties_hs_google_click_id": null, "properties_hs_has_active_subscription": null, "properties_hs_ip_timezone": null, "properties_hs_is_contact": true, "properties_hs_is_unworked": true, "properties_hs_language": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": null, "properties_hs_latest_disqualified_lead_date": null, "properties_hs_latest_meeting_activity": null, "properties_hs_latest_open_lead_date": null, "properties_hs_latest_qualified_lead_date": null, "properties_hs_latest_sequence_ended_date": null, "properties_hs_latest_sequence_enrolled": null, "properties_hs_latest_sequence_enrolled_date": null, "properties_hs_latest_sequence_finished_date": null, "properties_hs_latest_sequence_unenrolled_date": null, "properties_hs_latest_source": "OFFLINE", "properties_hs_latest_source_data_1": "API", "properties_hs_latest_source_data_2": null, "properties_hs_latest_source_timestamp": "2021-02-22T14:05:10.036000+00:00", "properties_hs_latest_subscription_create_date": null, "properties_hs_lead_status": null, "properties_hs_legal_basis": null, "properties_hs_lifecyclestage_customer_date": null, "properties_hs_lifecyclestage_evangelist_date": null, "properties_hs_lifecyclestage_lead_date": null, "properties_hs_lifecyclestage_marketingqualifiedlead_date": null, "properties_hs_lifecyclestage_opportunity_date": null, "properties_hs_lifecyclestage_other_date": null, "properties_hs_lifecyclestage_salesqualifiedlead_date": null, "properties_hs_lifecyclestage_subscriber_date": "2021-02-22T14:05:09.944000+00:00", "properties_hs_linkedin_ad_clicked": null, "properties_hs_marketable_reason_id": null, "properties_hs_marketable_reason_type": null, "properties_hs_marketable_status": "false", "properties_hs_marketable_until_renewal": "false", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 251, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_persona": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "contacts-lifecycle-pipeline", "properties_hs_predictivecontactscore": null, "properties_hs_predictivecontactscore_v2": 0.29, "properties_hs_predictivecontactscorebucket": null, "properties_hs_predictivescoringtier": "tier_4", "properties_hs_read_only": null, "properties_hs_sa_first_engagement_date": null, "properties_hs_sa_first_engagement_descr": null, "properties_hs_sa_first_engagement_object_type": null, "properties_hs_sales_email_last_clicked": null, "properties_hs_sales_email_last_opened": null, "properties_hs_sales_email_last_replied": null, "properties_hs_searchable_calculated_international_mobile_number": null, "properties_hs_searchable_calculated_international_phone_number": null, "properties_hs_searchable_calculated_mobile_number": null, "properties_hs_searchable_calculated_phone_number": "5551222323", "properties_hs_sequences_actively_enrolled_count": null, "properties_hs_sequences_enrolled_count": null, "properties_hs_sequences_is_enrolled": null, "properties_hs_testpurge": null, "properties_hs_testrollback": null, "properties_hs_time_between_contact_creation_and_deal_close": null, "properties_hs_time_between_contact_creation_and_deal_creation": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": null, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": null, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": 87820387720, "properties_hs_time_to_first_engagement": null, "properties_hs_time_to_move_from_lead_to_customer": null, "properties_hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_opportunity_to_customer": null, "properties_hs_time_to_move_from_salesqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_subscriber_to_customer": null, "properties_hs_timezone": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_v2_cumulative_time_in_customer": null, "properties_hs_v2_cumulative_time_in_evangelist": null, "properties_hs_v2_cumulative_time_in_lead": null, "properties_hs_v2_cumulative_time_in_marketingqualifiedlead": null, "properties_hs_v2_cumulative_time_in_opportunity": null, "properties_hs_v2_cumulative_time_in_other": null, "properties_hs_v2_cumulative_time_in_salesqualifiedlead": null, "properties_hs_v2_cumulative_time_in_subscriber": null, "properties_hs_v2_date_entered_customer": null, "properties_hs_v2_date_entered_evangelist": null, "properties_hs_v2_date_entered_lead": null, "properties_hs_v2_date_entered_marketingqualifiedlead": null, "properties_hs_v2_date_entered_opportunity": null, "properties_hs_v2_date_entered_other": null, "properties_hs_v2_date_entered_salesqualifiedlead": null, "properties_hs_v2_date_entered_subscriber": "2021-02-22T14:05:09.944000+00:00", "properties_hs_v2_date_exited_customer": null, "properties_hs_v2_date_exited_evangelist": null, "properties_hs_v2_date_exited_lead": null, "properties_hs_v2_date_exited_marketingqualifiedlead": null, "properties_hs_v2_date_exited_opportunity": null, "properties_hs_v2_date_exited_other": null, "properties_hs_v2_date_exited_salesqualifiedlead": null, "properties_hs_v2_date_exited_subscriber": null, "properties_hs_v2_latest_time_in_customer": null, "properties_hs_v2_latest_time_in_evangelist": null, "properties_hs_v2_latest_time_in_lead": null, "properties_hs_v2_latest_time_in_marketingqualifiedlead": null, "properties_hs_v2_latest_time_in_opportunity": null, "properties_hs_v2_latest_time_in_other": null, "properties_hs_v2_latest_time_in_salesqualifiedlead": null, "properties_hs_v2_latest_time_in_subscriber": null, "properties_hs_was_imported": null, "properties_hs_whatsapp_phone_number": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_ip_city": null, "properties_ip_country": null, "properties_ip_country_code": null, "properties_ip_latlon": null, "properties_ip_state": null, "properties_ip_state_code": null, "properties_ip_zipcode": null, "properties_job_function": null, "properties_jobtitle": null, "properties_lastmodifieddate": "2023-03-21T19:29:13.036000+00:00", "properties_lastname": "Test Lastname 5001", "properties_lifecyclestage": "subscriber", "properties_marital_status": null, "properties_message": null, "properties_military_status": null, "properties_mobilephone": null, "properties_my_custom_test_property": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_deals": null, "properties_num_contacted_notes": null, "properties_num_conversion_events": 0, "properties_num_notes": null, "properties_num_unique_conversion_events": 0, "properties_numemployees": null, "properties_phone": "555-122-2323", "properties_recent_conversion_date": null, "properties_recent_conversion_event_name": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_relationship_status": null, "properties_salutation": null, "properties_school": null, "properties_seniority": null, "properties_start_date": null, "properties_state": "MA", "properties_surveymonkeyeventlastupdated": null, "properties_test": null, "properties_total_revenue": null, "properties_twitterhandle": null, "properties_webinareventlastupdated": null, "properties_website": "http://hubspot.com", "properties_work_email": null, "properties_zip": "02139"}, "emitted_at": 1701823098430} +{"stream": "contacts", "data": {"id": "401", "properties": {"address": "25 First Street", "annualrevenue": null, "associatedcompanyid": null, "associatedcompanylastupdated": null, "city": "Cambridge", "closedate": null, "company": null, "company_size": null, "country": null, "createdate": "2021-02-23T20:10:36.191000+00:00", "currentlyinworkflow": null, "date_of_birth": null, "days_to_close": null, "degree": null, "email": "macmitch@hubspot.com", "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "fax": null, "field_of_study": null, "first_conversion_date": null, "first_conversion_event_name": null, "first_deal_created_date": null, "firstname": "Mac", "gender": null, "graduation_date": null, "hs_additional_emails": null, "hs_all_accessible_team_ids": null, "hs_all_contact_vids": "401", "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_average_page_views": 0, "hs_analytics_first_referrer": null, "hs_analytics_first_timestamp": "2021-02-23T20:10:36.181000+00:00", "hs_analytics_first_touch_converting_campaign": null, "hs_analytics_first_url": null, "hs_analytics_first_visit_timestamp": null, "hs_analytics_last_referrer": null, "hs_analytics_last_timestamp": null, "hs_analytics_last_touch_converting_campaign": null, "hs_analytics_last_url": null, "hs_analytics_last_visit_timestamp": null, "hs_analytics_num_event_completions": 0, "hs_analytics_num_page_views": 0, "hs_analytics_num_visits": 0, "hs_analytics_revenue": 0.0, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "IMPORT", "hs_analytics_source_data_2": "13256565", "hs_avatar_filemanager_key": null, "hs_buying_role": null, "hs_calculated_form_submissions": null, "hs_calculated_merged_vids": null, "hs_calculated_mobile_number": null, "hs_calculated_phone_number": "+18884827768", "hs_calculated_phone_number_area_code": null, "hs_calculated_phone_number_country_code": "US", "hs_calculated_phone_number_region_code": null, "hs_clicked_linkedin_ad": null, "hs_content_membership_email": null, "hs_content_membership_email_confirmed": null, "hs_content_membership_notes": null, "hs_content_membership_registered_at": null, "hs_content_membership_registration_domain_sent_to": null, "hs_content_membership_registration_email_sent_at": null, "hs_content_membership_status": null, "hs_conversations_visitor_email": null, "hs_count_is_unworked": 1, "hs_count_is_worked": 0, "hs_created_by_conversations": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_date_entered_customer": null, "hs_date_entered_evangelist": null, "hs_date_entered_lead": "2021-02-23T20:10:36.181000+00:00", "hs_date_entered_marketingqualifiedlead": null, "hs_date_entered_opportunity": null, "hs_date_entered_other": null, "hs_date_entered_salesqualifiedlead": null, "hs_date_entered_subscriber": null, "hs_date_exited_customer": null, "hs_date_exited_evangelist": null, "hs_date_exited_lead": null, "hs_date_exited_marketingqualifiedlead": null, "hs_date_exited_opportunity": null, "hs_date_exited_other": null, "hs_date_exited_salesqualifiedlead": null, "hs_date_exited_subscriber": null, "hs_document_last_revisited": null, "hs_email_bad_address": null, "hs_email_bounce": null, "hs_email_click": null, "hs_email_customer_quarantined_reason": null, "hs_email_delivered": null, "hs_email_domain": "hubspot.com", "hs_email_first_click_date": null, "hs_email_first_open_date": null, "hs_email_first_reply_date": null, "hs_email_first_send_date": null, "hs_email_hard_bounce_reason": null, "hs_email_hard_bounce_reason_enum": "OTHER", "hs_email_is_ineligible": null, "hs_email_last_click_date": null, "hs_email_last_email_name": null, "hs_email_last_open_date": null, "hs_email_last_reply_date": null, "hs_email_last_send_date": null, "hs_email_open": null, "hs_email_optout": null, "hs_email_optout_10798197": null, "hs_email_optout_11890603": null, "hs_email_optout_11890831": null, "hs_email_optout_23704464": null, "hs_email_optout_94692364": null, "hs_email_quarantined": null, "hs_email_quarantined_reason": null, "hs_email_recipient_fatigue_recovery_time": null, "hs_email_replied": null, "hs_email_sends_since_last_engagement": null, "hs_emailconfirmationstatus": null, "hs_facebook_ad_clicked": null, "hs_facebook_click_id": null, "hs_feedback_last_nps_follow_up": null, "hs_feedback_last_nps_rating": null, "hs_feedback_last_survey_date": null, "hs_feedback_show_nps_web_survey": null, "hs_first_engagement_object_id": null, "hs_first_outreach_date": null, "hs_first_subscription_create_date": null, "hs_google_click_id": null, "hs_has_active_subscription": null, "hs_ip_timezone": null, "hs_is_contact": true, "hs_is_unworked": true, "hs_language": null, "hs_last_sales_activity_date": null, "hs_last_sales_activity_timestamp": null, "hs_last_sales_activity_type": null, "hs_lastmodifieddate": null, "hs_latest_disqualified_lead_date": null, "hs_latest_meeting_activity": null, "hs_latest_open_lead_date": null, "hs_latest_qualified_lead_date": null, "hs_latest_sequence_ended_date": null, "hs_latest_sequence_enrolled": null, "hs_latest_sequence_enrolled_date": null, "hs_latest_sequence_finished_date": null, "hs_latest_sequence_unenrolled_date": null, "hs_latest_source": "OFFLINE", "hs_latest_source_data_1": "IMPORT", "hs_latest_source_data_2": "13256565", "hs_latest_source_timestamp": "2021-02-23T20:10:36.210000+00:00", "hs_latest_subscription_create_date": null, "hs_lead_status": null, "hs_legal_basis": null, "hs_lifecyclestage_customer_date": null, "hs_lifecyclestage_evangelist_date": null, "hs_lifecyclestage_lead_date": "2021-02-23T20:10:36.181000+00:00", "hs_lifecyclestage_marketingqualifiedlead_date": null, "hs_lifecyclestage_opportunity_date": null, "hs_lifecyclestage_other_date": null, "hs_lifecyclestage_salesqualifiedlead_date": null, "hs_lifecyclestage_subscriber_date": null, "hs_linkedin_ad_clicked": null, "hs_marketable_reason_id": null, "hs_marketable_reason_type": null, "hs_marketable_status": "false", "hs_marketable_until_renewal": "false", "hs_merged_object_ids": null, "hs_object_id": 401, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_persona": null, "hs_pinned_engagement_id": null, "hs_pipeline": "contacts-lifecycle-pipeline", "hs_predictivecontactscore": null, "hs_predictivecontactscore_v2": 0.29, "hs_predictivecontactscorebucket": null, "hs_predictivescoringtier": "tier_4", "hs_read_only": null, "hs_sa_first_engagement_date": null, "hs_sa_first_engagement_descr": null, "hs_sa_first_engagement_object_type": null, "hs_sales_email_last_clicked": null, "hs_sales_email_last_opened": null, "hs_sales_email_last_replied": null, "hs_searchable_calculated_international_mobile_number": null, "hs_searchable_calculated_international_phone_number": null, "hs_searchable_calculated_mobile_number": null, "hs_searchable_calculated_phone_number": "8884827768", "hs_sequences_actively_enrolled_count": null, "hs_sequences_enrolled_count": null, "hs_sequences_is_enrolled": null, "hs_testpurge": null, "hs_testrollback": null, "hs_time_between_contact_creation_and_deal_close": null, "hs_time_between_contact_creation_and_deal_creation": null, "hs_time_in_customer": null, "hs_time_in_evangelist": null, "hs_time_in_lead": 87712061483, "hs_time_in_marketingqualifiedlead": null, "hs_time_in_opportunity": null, "hs_time_in_other": null, "hs_time_in_salesqualifiedlead": null, "hs_time_in_subscriber": null, "hs_time_to_first_engagement": null, "hs_time_to_move_from_lead_to_customer": null, "hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "hs_time_to_move_from_opportunity_to_customer": null, "hs_time_to_move_from_salesqualifiedlead_to_customer": null, "hs_time_to_move_from_subscriber_to_customer": null, "hs_timezone": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_v2_cumulative_time_in_customer": null, "hs_v2_cumulative_time_in_evangelist": null, "hs_v2_cumulative_time_in_lead": null, "hs_v2_cumulative_time_in_marketingqualifiedlead": null, "hs_v2_cumulative_time_in_opportunity": null, "hs_v2_cumulative_time_in_other": null, "hs_v2_cumulative_time_in_salesqualifiedlead": null, "hs_v2_cumulative_time_in_subscriber": null, "hs_v2_date_entered_customer": null, "hs_v2_date_entered_evangelist": null, "hs_v2_date_entered_lead": "2021-02-23T20:10:36.181000+00:00", "hs_v2_date_entered_marketingqualifiedlead": null, "hs_v2_date_entered_opportunity": null, "hs_v2_date_entered_other": null, "hs_v2_date_entered_salesqualifiedlead": null, "hs_v2_date_entered_subscriber": null, "hs_v2_date_exited_customer": null, "hs_v2_date_exited_evangelist": null, "hs_v2_date_exited_lead": null, "hs_v2_date_exited_marketingqualifiedlead": null, "hs_v2_date_exited_opportunity": null, "hs_v2_date_exited_other": null, "hs_v2_date_exited_salesqualifiedlead": null, "hs_v2_date_exited_subscriber": null, "hs_v2_latest_time_in_customer": null, "hs_v2_latest_time_in_evangelist": null, "hs_v2_latest_time_in_lead": null, "hs_v2_latest_time_in_marketingqualifiedlead": null, "hs_v2_latest_time_in_opportunity": null, "hs_v2_latest_time_in_other": null, "hs_v2_latest_time_in_salesqualifiedlead": null, "hs_v2_latest_time_in_subscriber": null, "hs_was_imported": true, "hs_whatsapp_phone_number": null, "hubspot_owner_assigneddate": "2021-05-21T10:20:30.963000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "hubspotscore": null, "industry": null, "ip_city": null, "ip_country": null, "ip_country_code": null, "ip_latlon": null, "ip_state": null, "ip_state_code": null, "ip_zipcode": null, "job_function": null, "jobtitle": null, "lastmodifieddate": "2023-03-21T19:31:00.563000+00:00", "lastname": "Mitchell", "lifecyclestage": "lead", "marital_status": null, "message": null, "military_status": null, "mobilephone": null, "my_custom_test_property": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_deals": null, "num_contacted_notes": null, "num_conversion_events": 0, "num_notes": null, "num_unique_conversion_events": 0, "numemployees": null, "phone": "1(888) 482-7768", "recent_conversion_date": null, "recent_conversion_event_name": null, "recent_deal_amount": null, "recent_deal_close_date": null, "relationship_status": null, "salutation": null, "school": null, "seniority": null, "start_date": null, "state": "MA", "surveymonkeyeventlastupdated": null, "test": null, "total_revenue": null, "twitterhandle": null, "webinareventlastupdated": null, "website": null, "work_email": null, "zip": "21430"}, "createdAt": "2021-02-23T20:10:36.191Z", "updatedAt": "2023-03-21T19:31:00.563Z", "archived": false, "properties_address": "25 First Street", "properties_annualrevenue": null, "properties_associatedcompanyid": null, "properties_associatedcompanylastupdated": null, "properties_city": "Cambridge", "properties_closedate": null, "properties_company": null, "properties_company_size": null, "properties_country": null, "properties_createdate": "2021-02-23T20:10:36.191000+00:00", "properties_currentlyinworkflow": null, "properties_date_of_birth": null, "properties_days_to_close": null, "properties_degree": null, "properties_email": "macmitch@hubspot.com", "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_fax": null, "properties_field_of_study": null, "properties_first_conversion_date": null, "properties_first_conversion_event_name": null, "properties_first_deal_created_date": null, "properties_firstname": "Mac", "properties_gender": null, "properties_graduation_date": null, "properties_hs_additional_emails": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_contact_vids": "401", "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_average_page_views": 0, "properties_hs_analytics_first_referrer": null, "properties_hs_analytics_first_timestamp": "2021-02-23T20:10:36.181000+00:00", "properties_hs_analytics_first_touch_converting_campaign": null, "properties_hs_analytics_first_url": null, "properties_hs_analytics_first_visit_timestamp": null, "properties_hs_analytics_last_referrer": null, "properties_hs_analytics_last_timestamp": null, "properties_hs_analytics_last_touch_converting_campaign": null, "properties_hs_analytics_last_url": null, "properties_hs_analytics_last_visit_timestamp": null, "properties_hs_analytics_num_event_completions": 0, "properties_hs_analytics_num_page_views": 0, "properties_hs_analytics_num_visits": 0, "properties_hs_analytics_revenue": 0.0, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "IMPORT", "properties_hs_analytics_source_data_2": "13256565", "properties_hs_avatar_filemanager_key": null, "properties_hs_buying_role": null, "properties_hs_calculated_form_submissions": null, "properties_hs_calculated_merged_vids": null, "properties_hs_calculated_mobile_number": null, "properties_hs_calculated_phone_number": "+18884827768", "properties_hs_calculated_phone_number_area_code": null, "properties_hs_calculated_phone_number_country_code": "US", "properties_hs_calculated_phone_number_region_code": null, "properties_hs_clicked_linkedin_ad": null, "properties_hs_content_membership_email": null, "properties_hs_content_membership_email_confirmed": null, "properties_hs_content_membership_notes": null, "properties_hs_content_membership_registered_at": null, "properties_hs_content_membership_registration_domain_sent_to": null, "properties_hs_content_membership_registration_email_sent_at": null, "properties_hs_content_membership_status": null, "properties_hs_conversations_visitor_email": null, "properties_hs_count_is_unworked": 1, "properties_hs_count_is_worked": 0, "properties_hs_created_by_conversations": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_date_entered_customer": null, "properties_hs_date_entered_evangelist": null, "properties_hs_date_entered_lead": "2021-02-23T20:10:36.181000+00:00", "properties_hs_date_entered_marketingqualifiedlead": null, "properties_hs_date_entered_opportunity": null, "properties_hs_date_entered_other": null, "properties_hs_date_entered_salesqualifiedlead": null, "properties_hs_date_entered_subscriber": null, "properties_hs_date_exited_customer": null, "properties_hs_date_exited_evangelist": null, "properties_hs_date_exited_lead": null, "properties_hs_date_exited_marketingqualifiedlead": null, "properties_hs_date_exited_opportunity": null, "properties_hs_date_exited_other": null, "properties_hs_date_exited_salesqualifiedlead": null, "properties_hs_date_exited_subscriber": null, "properties_hs_document_last_revisited": null, "properties_hs_email_bad_address": null, "properties_hs_email_bounce": null, "properties_hs_email_click": null, "properties_hs_email_customer_quarantined_reason": null, "properties_hs_email_delivered": null, "properties_hs_email_domain": "hubspot.com", "properties_hs_email_first_click_date": null, "properties_hs_email_first_open_date": null, "properties_hs_email_first_reply_date": null, "properties_hs_email_first_send_date": null, "properties_hs_email_hard_bounce_reason": null, "properties_hs_email_hard_bounce_reason_enum": "OTHER", "properties_hs_email_is_ineligible": null, "properties_hs_email_last_click_date": null, "properties_hs_email_last_email_name": null, "properties_hs_email_last_open_date": null, "properties_hs_email_last_reply_date": null, "properties_hs_email_last_send_date": null, "properties_hs_email_open": null, "properties_hs_email_optout": null, "properties_hs_email_optout_10798197": null, "properties_hs_email_optout_11890603": null, "properties_hs_email_optout_11890831": null, "properties_hs_email_optout_23704464": null, "properties_hs_email_optout_94692364": null, "properties_hs_email_quarantined": null, "properties_hs_email_quarantined_reason": null, "properties_hs_email_recipient_fatigue_recovery_time": null, "properties_hs_email_replied": null, "properties_hs_email_sends_since_last_engagement": null, "properties_hs_emailconfirmationstatus": null, "properties_hs_facebook_ad_clicked": null, "properties_hs_facebook_click_id": null, "properties_hs_feedback_last_nps_follow_up": null, "properties_hs_feedback_last_nps_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_feedback_show_nps_web_survey": null, "properties_hs_first_engagement_object_id": null, "properties_hs_first_outreach_date": null, "properties_hs_first_subscription_create_date": null, "properties_hs_google_click_id": null, "properties_hs_has_active_subscription": null, "properties_hs_ip_timezone": null, "properties_hs_is_contact": true, "properties_hs_is_unworked": true, "properties_hs_language": null, "properties_hs_last_sales_activity_date": null, "properties_hs_last_sales_activity_timestamp": null, "properties_hs_last_sales_activity_type": null, "properties_hs_lastmodifieddate": null, "properties_hs_latest_disqualified_lead_date": null, "properties_hs_latest_meeting_activity": null, "properties_hs_latest_open_lead_date": null, "properties_hs_latest_qualified_lead_date": null, "properties_hs_latest_sequence_ended_date": null, "properties_hs_latest_sequence_enrolled": null, "properties_hs_latest_sequence_enrolled_date": null, "properties_hs_latest_sequence_finished_date": null, "properties_hs_latest_sequence_unenrolled_date": null, "properties_hs_latest_source": "OFFLINE", "properties_hs_latest_source_data_1": "IMPORT", "properties_hs_latest_source_data_2": "13256565", "properties_hs_latest_source_timestamp": "2021-02-23T20:10:36.210000+00:00", "properties_hs_latest_subscription_create_date": null, "properties_hs_lead_status": null, "properties_hs_legal_basis": null, "properties_hs_lifecyclestage_customer_date": null, "properties_hs_lifecyclestage_evangelist_date": null, "properties_hs_lifecyclestage_lead_date": "2021-02-23T20:10:36.181000+00:00", "properties_hs_lifecyclestage_marketingqualifiedlead_date": null, "properties_hs_lifecyclestage_opportunity_date": null, "properties_hs_lifecyclestage_other_date": null, "properties_hs_lifecyclestage_salesqualifiedlead_date": null, "properties_hs_lifecyclestage_subscriber_date": null, "properties_hs_linkedin_ad_clicked": null, "properties_hs_marketable_reason_id": null, "properties_hs_marketable_reason_type": null, "properties_hs_marketable_status": "false", "properties_hs_marketable_until_renewal": "false", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 401, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_persona": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "contacts-lifecycle-pipeline", "properties_hs_predictivecontactscore": null, "properties_hs_predictivecontactscore_v2": 0.29, "properties_hs_predictivecontactscorebucket": null, "properties_hs_predictivescoringtier": "tier_4", "properties_hs_read_only": null, "properties_hs_sa_first_engagement_date": null, "properties_hs_sa_first_engagement_descr": null, "properties_hs_sa_first_engagement_object_type": null, "properties_hs_sales_email_last_clicked": null, "properties_hs_sales_email_last_opened": null, "properties_hs_sales_email_last_replied": null, "properties_hs_searchable_calculated_international_mobile_number": null, "properties_hs_searchable_calculated_international_phone_number": null, "properties_hs_searchable_calculated_mobile_number": null, "properties_hs_searchable_calculated_phone_number": "8884827768", "properties_hs_sequences_actively_enrolled_count": null, "properties_hs_sequences_enrolled_count": null, "properties_hs_sequences_is_enrolled": null, "properties_hs_testpurge": null, "properties_hs_testrollback": null, "properties_hs_time_between_contact_creation_and_deal_close": null, "properties_hs_time_between_contact_creation_and_deal_creation": null, "properties_hs_time_in_customer": null, "properties_hs_time_in_evangelist": null, "properties_hs_time_in_lead": 87712061483, "properties_hs_time_in_marketingqualifiedlead": null, "properties_hs_time_in_opportunity": null, "properties_hs_time_in_other": null, "properties_hs_time_in_salesqualifiedlead": null, "properties_hs_time_in_subscriber": null, "properties_hs_time_to_first_engagement": null, "properties_hs_time_to_move_from_lead_to_customer": null, "properties_hs_time_to_move_from_marketingqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_opportunity_to_customer": null, "properties_hs_time_to_move_from_salesqualifiedlead_to_customer": null, "properties_hs_time_to_move_from_subscriber_to_customer": null, "properties_hs_timezone": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_v2_cumulative_time_in_customer": null, "properties_hs_v2_cumulative_time_in_evangelist": null, "properties_hs_v2_cumulative_time_in_lead": null, "properties_hs_v2_cumulative_time_in_marketingqualifiedlead": null, "properties_hs_v2_cumulative_time_in_opportunity": null, "properties_hs_v2_cumulative_time_in_other": null, "properties_hs_v2_cumulative_time_in_salesqualifiedlead": null, "properties_hs_v2_cumulative_time_in_subscriber": null, "properties_hs_v2_date_entered_customer": null, "properties_hs_v2_date_entered_evangelist": null, "properties_hs_v2_date_entered_lead": "2021-02-23T20:10:36.181000+00:00", "properties_hs_v2_date_entered_marketingqualifiedlead": null, "properties_hs_v2_date_entered_opportunity": null, "properties_hs_v2_date_entered_other": null, "properties_hs_v2_date_entered_salesqualifiedlead": null, "properties_hs_v2_date_entered_subscriber": null, "properties_hs_v2_date_exited_customer": null, "properties_hs_v2_date_exited_evangelist": null, "properties_hs_v2_date_exited_lead": null, "properties_hs_v2_date_exited_marketingqualifiedlead": null, "properties_hs_v2_date_exited_opportunity": null, "properties_hs_v2_date_exited_other": null, "properties_hs_v2_date_exited_salesqualifiedlead": null, "properties_hs_v2_date_exited_subscriber": null, "properties_hs_v2_latest_time_in_customer": null, "properties_hs_v2_latest_time_in_evangelist": null, "properties_hs_v2_latest_time_in_lead": null, "properties_hs_v2_latest_time_in_marketingqualifiedlead": null, "properties_hs_v2_latest_time_in_opportunity": null, "properties_hs_v2_latest_time_in_other": null, "properties_hs_v2_latest_time_in_salesqualifiedlead": null, "properties_hs_v2_latest_time_in_subscriber": null, "properties_hs_was_imported": true, "properties_hs_whatsapp_phone_number": null, "properties_hubspot_owner_assigneddate": "2021-05-21T10:20:30.963000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_hubspotscore": null, "properties_industry": null, "properties_ip_city": null, "properties_ip_country": null, "properties_ip_country_code": null, "properties_ip_latlon": null, "properties_ip_state": null, "properties_ip_state_code": null, "properties_ip_zipcode": null, "properties_job_function": null, "properties_jobtitle": null, "properties_lastmodifieddate": "2023-03-21T19:31:00.563000+00:00", "properties_lastname": "Mitchell", "properties_lifecyclestage": "lead", "properties_marital_status": null, "properties_message": null, "properties_military_status": null, "properties_mobilephone": null, "properties_my_custom_test_property": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_deals": null, "properties_num_contacted_notes": null, "properties_num_conversion_events": 0, "properties_num_notes": null, "properties_num_unique_conversion_events": 0, "properties_numemployees": null, "properties_phone": "1(888) 482-7768", "properties_recent_conversion_date": null, "properties_recent_conversion_event_name": null, "properties_recent_deal_amount": null, "properties_recent_deal_close_date": null, "properties_relationship_status": null, "properties_salutation": null, "properties_school": null, "properties_seniority": null, "properties_start_date": null, "properties_state": "MA", "properties_surveymonkeyeventlastupdated": null, "properties_test": null, "properties_total_revenue": null, "properties_twitterhandle": null, "properties_webinareventlastupdated": null, "properties_website": null, "properties_work_email": null, "properties_zip": "21430"}, "emitted_at": 1701823098432} +{"stream": "contacts_list_memberships", "data": {"canonical-vid": 401, "static-list-id": 60, "internal-list-id": 2147483643, "timestamp": 1614111042672, "vid": 401, "is-member": true}, "emitted_at": 1697714191502} +{"stream": "contacts_list_memberships", "data": {"canonical-vid": 401, "static-list-id": 61, "internal-list-id": 2147483643, "timestamp": 1615502112726, "vid": 401, "is-member": true}, "emitted_at": 1697714191513} +{"stream": "contacts_list_memberships", "data": {"canonical-vid": 2501, "static-list-id": 60, "internal-list-id": 2147483643, "timestamp": 1675124235515, "vid": 2501, "is-member": true}, "emitted_at": 1697714191513} +{"stream": "contacts_merged_audit", "data": {"canonical-vid": 651, "vid-to-merge": 201, "timestamp": 1688758327178, "entity-id": "auth:app-cookie | auth-level:app | login-id:integration-test@airbyte.io-1688758203663 | hub-id:8727216 | user-id:12282590 | origin-ip:2804:1b3:8402:b1f4:7d1b:f62e:b071:593d | correlation-id:3f139cd7-66fc-4300-8cbc-e6c1fe9ea7d1", "user-id": 12282590, "num-properties-moved": 45, "merged_from_email": {"value": "testingapis@hubspot.com", "source-type": "API", "source-id": null, "source-label": null, "updated-by-user-id": null, "timestamp": 1610634377014, "selected": false}, "merged_to_email": {"value": "testingapicontact_1@hubspot.com", "source-type": "API", "source-id": null, "source-label": null, "updated-by-user-id": null, "timestamp": 1634044981830, "selected": false}, "first-name": "test", "last-name": "testerson", "merged_from_email_value": "testingapis@hubspot.com", "merged_from_email_source-type": "API", "merged_from_email_source-id": null, "merged_from_email_source-label": null, "merged_from_email_updated-by-user-id": null, "merged_from_email_timestamp": 1610634377014, "merged_from_email_selected": false, "merged_to_email_value": "testingapicontact_1@hubspot.com", "merged_to_email_source-type": "API", "merged_to_email_source-id": null, "merged_to_email_source-label": null, "merged_to_email_updated-by-user-id": null, "merged_to_email_timestamp": 1634044981830, "merged_to_email_selected": false}, "emitted_at": 1697714194351} +{"stream": "deal_pipelines", "data": {"label": "New Business Pipeline", "displayOrder": 3, "active": true, "stages": [{"label": "Initial Qualification", "displayOrder": 0, "metadata": {"isClosed": "false", "probability": "0.1"}, "stageId": "9567448", "createdAt": 1610635973956, "updatedAt": 1680620354263, "active": true}, {"label": "Success! Closed Won", "displayOrder": 2, "metadata": {"isClosed": "true", "probability": "1.0"}, "stageId": "customclosedwonstage", "createdAt": 1610635973956, "updatedAt": 1680620354263, "active": true}, {"label": "Negotiation", "displayOrder": 1, "metadata": {"isClosed": "false", "probability": "0.5"}, "stageId": "9567449", "createdAt": 1610635973956, "updatedAt": 1680620354263, "active": true}, {"label": "Closed Lost", "displayOrder": 3, "metadata": {"isClosed": "false", "probability": "0.1"}, "stageId": "66894120", "createdAt": 1680620354263, "updatedAt": 1680620354263, "active": true}], "objectType": "DEAL", "objectTypeId": "0-3", "pipelineId": "b9152945-a594-4835-9676-a6f405fecd71", "createdAt": 1610635973956, "updatedAt": 1680620354263, "default": false}, "emitted_at": 1697714195524} +{"stream": "deals", "data": {"id": "3980651569", "properties": {"amount": 60000, "amount_in_home_currency": 60000, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2014-08-31T00:00:00+00:00", "createdate": "2021-01-13T10:30:42.221000+00:00", "days_to_close": 0, "dealname": "Tim's Newer Deal", "dealstage": "appointmentscheduled", "dealtype": "newbusiness", "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": null, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": "OFFLINE", "hs_analytics_latest_source_company": "OFFLINE", "hs_analytics_latest_source_contact": null, "hs_analytics_latest_source_data_1": "CONTACTS", "hs_analytics_latest_source_data_1_company": "CONTACTS", "hs_analytics_latest_source_data_1_contact": null, "hs_analytics_latest_source_data_2": "CRM_UI", "hs_analytics_latest_source_data_2_company": "CRM_UI", "hs_analytics_latest_source_data_2_contact": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": "OFFLINE", "hs_analytics_source_data_1": "CONTACTS", "hs_analytics_source_data_2": "CRM_UI", "hs_arr": null, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-13T10:30:42.221000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-01-13T10:30:42.221000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_days_to_close_raw": 0, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": null, "hs_exchange_rate": null, "hs_forecast_amount": 60000, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_is_open_count": 1, "hs_lastmodifieddate": "2021-09-07T02:36:16.363000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": null, "hs_next_step": null, "hs_num_associated_deal_splits": null, "hs_num_of_associated_line_items": 0, "hs_num_target_accounts": 0, "hs_object_id": 3980651569, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 12000.0, "hs_projected_amount_in_home_currency": 12000.0, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": null, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 92113120720, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_v2_cumulative_time_in_66894120": null, "hs_v2_cumulative_time_in_9567448": null, "hs_v2_cumulative_time_in_9567449": null, "hs_v2_cumulative_time_in_customclosedwonstage": null, "hs_v2_date_entered_66894120": null, "hs_v2_date_entered_9567448": null, "hs_v2_date_entered_9567449": null, "hs_v2_date_entered_customclosedwonstage": null, "hs_v2_date_exited_66894120": null, "hs_v2_date_exited_9567448": null, "hs_v2_date_exited_9567449": null, "hs_v2_date_exited_customclosedwonstage": null, "hs_v2_latest_time_in_66894120": null, "hs_v2_latest_time_in_9567448": null, "hs_v2_latest_time_in_9567449": null, "hs_v2_latest_time_in_customclosedwonstage": null, "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-01-13T10:30:42.221000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": null, "num_notes": null, "pipeline": "default"}, "createdAt": "2021-01-13T10:30:42.221Z", "updatedAt": "2021-09-07T02:36:16.363Z", "archived": false, "companies": ["5000526215", "5000526215"], "properties_amount": 60000, "properties_amount_in_home_currency": 60000, "properties_closed_lost_reason": null, "properties_closed_won_reason": null, "properties_closedate": "2014-08-31T00:00:00+00:00", "properties_createdate": "2021-01-13T10:30:42.221000+00:00", "properties_days_to_close": 0, "properties_dealname": "Tim's Newer Deal", "properties_dealstage": "appointmentscheduled", "properties_dealtype": "newbusiness", "properties_description": null, "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_hs_acv": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_collaborator_owner_ids": null, "properties_hs_all_deal_split_owner_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_latest_source": "OFFLINE", "properties_hs_analytics_latest_source_company": "OFFLINE", "properties_hs_analytics_latest_source_contact": null, "properties_hs_analytics_latest_source_data_1": "CONTACTS", "properties_hs_analytics_latest_source_data_1_company": "CONTACTS", "properties_hs_analytics_latest_source_data_1_contact": null, "properties_hs_analytics_latest_source_data_2": "CRM_UI", "properties_hs_analytics_latest_source_data_2_company": "CRM_UI", "properties_hs_analytics_latest_source_data_2_contact": null, "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_latest_source_timestamp_company": null, "properties_hs_analytics_latest_source_timestamp_contact": null, "properties_hs_analytics_source": "OFFLINE", "properties_hs_analytics_source_data_1": "CONTACTS", "properties_hs_analytics_source_data_2": "CRM_UI", "properties_hs_arr": null, "properties_hs_campaign": null, "properties_hs_closed_amount": 0, "properties_hs_closed_amount_in_home_currency": 0, "properties_hs_closed_won_count": null, "properties_hs_closed_won_date": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-13T10:30:42.221000+00:00", "properties_hs_date_entered_66894120": null, "properties_hs_date_entered_9567448": null, "properties_hs_date_entered_9567449": null, "properties_hs_date_entered_appointmentscheduled": "2021-01-13T10:30:42.221000+00:00", "properties_hs_date_entered_closedlost": null, "properties_hs_date_entered_closedwon": null, "properties_hs_date_entered_contractsent": null, "properties_hs_date_entered_customclosedwonstage": null, "properties_hs_date_entered_decisionmakerboughtin": null, "properties_hs_date_entered_presentationscheduled": null, "properties_hs_date_entered_qualifiedtobuy": null, "properties_hs_date_exited_66894120": null, "properties_hs_date_exited_9567448": null, "properties_hs_date_exited_9567449": null, "properties_hs_date_exited_appointmentscheduled": null, "properties_hs_date_exited_closedlost": null, "properties_hs_date_exited_closedwon": null, "properties_hs_date_exited_contractsent": null, "properties_hs_date_exited_customclosedwonstage": null, "properties_hs_date_exited_decisionmakerboughtin": null, "properties_hs_date_exited_presentationscheduled": null, "properties_hs_date_exited_qualifiedtobuy": null, "properties_hs_days_to_close_raw": 0, "properties_hs_deal_amount_calculation_preference": null, "properties_hs_deal_stage_probability": 0.2, "properties_hs_deal_stage_probability_shadow": null, "properties_hs_exchange_rate": null, "properties_hs_forecast_amount": 60000, "properties_hs_forecast_probability": null, "properties_hs_is_closed": false, "properties_hs_is_closed_won": false, "properties_hs_is_deal_split": false, "properties_hs_is_open_count": 1, "properties_hs_lastmodifieddate": "2021-09-07T02:36:16.363000+00:00", "properties_hs_latest_meeting_activity": null, "properties_hs_likelihood_to_close": null, "properties_hs_line_item_global_term_hs_discount_percentage": null, "properties_hs_line_item_global_term_hs_discount_percentage_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_period": null, "properties_hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "properties_hs_line_item_global_term_recurringbillingfrequency": null, "properties_hs_line_item_global_term_recurringbillingfrequency_enabled": null, "properties_hs_manual_forecast_category": null, "properties_hs_merged_object_ids": null, "properties_hs_mrr": null, "properties_hs_next_step": null, "properties_hs_num_associated_deal_splits": null, "properties_hs_num_of_associated_line_items": 0, "properties_hs_num_target_accounts": 0, "properties_hs_object_id": 3980651569, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_predicted_amount": null, "properties_hs_predicted_amount_in_home_currency": null, "properties_hs_priority": null, "properties_hs_projected_amount": 12000.0, "properties_hs_projected_amount_in_home_currency": 12000.0, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_tcv": null, "properties_hs_time_in_66894120": null, "properties_hs_time_in_9567448": null, "properties_hs_time_in_9567449": null, "properties_hs_time_in_appointmentscheduled": 92113120720, "properties_hs_time_in_closedlost": null, "properties_hs_time_in_closedwon": null, "properties_hs_time_in_contractsent": null, "properties_hs_time_in_customclosedwonstage": null, "properties_hs_time_in_decisionmakerboughtin": null, "properties_hs_time_in_presentationscheduled": null, "properties_hs_time_in_qualifiedtobuy": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_v2_cumulative_time_in_66894120": null, "properties_hs_v2_cumulative_time_in_9567448": null, "properties_hs_v2_cumulative_time_in_9567449": null, "properties_hs_v2_cumulative_time_in_customclosedwonstage": null, "properties_hs_v2_date_entered_66894120": null, "properties_hs_v2_date_entered_9567448": null, "properties_hs_v2_date_entered_9567449": null, "properties_hs_v2_date_entered_customclosedwonstage": null, "properties_hs_v2_date_exited_66894120": null, "properties_hs_v2_date_exited_9567448": null, "properties_hs_v2_date_exited_9567449": null, "properties_hs_v2_date_exited_customclosedwonstage": null, "properties_hs_v2_latest_time_in_66894120": null, "properties_hs_v2_latest_time_in_9567448": null, "properties_hs_v2_latest_time_in_9567449": null, "properties_hs_v2_latest_time_in_customclosedwonstage": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2021-01-13T10:30:42.221000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_pipeline": "default"}, "emitted_at": 1702646964420} +{"stream": "deals", "data": {"id": "3980673856", "properties": {"amount": 60000, "amount_in_home_currency": 60000, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2014-08-31T00:00:00+00:00", "createdate": "2021-01-13T10:31:51.154000+00:00", "days_to_close": 0, "dealname": "Tim's Newer Deal", "dealstage": "appointmentscheduled", "dealtype": "newbusiness", "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": null, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_2": null, "hs_arr": null, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-13T10:31:51.154000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-01-13T10:31:51.154000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_days_to_close_raw": 0, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": null, "hs_exchange_rate": null, "hs_forecast_amount": 60000, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_is_open_count": 1, "hs_lastmodifieddate": "2021-09-07T18:11:59.757000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": null, "hs_next_step": null, "hs_num_associated_deal_splits": null, "hs_num_of_associated_line_items": 0, "hs_num_target_accounts": null, "hs_object_id": 3980673856, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 12000.0, "hs_projected_amount_in_home_currency": 12000.0, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": null, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 92113051787, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_v2_cumulative_time_in_66894120": null, "hs_v2_cumulative_time_in_9567448": null, "hs_v2_cumulative_time_in_9567449": null, "hs_v2_cumulative_time_in_customclosedwonstage": null, "hs_v2_date_entered_66894120": null, "hs_v2_date_entered_9567448": null, "hs_v2_date_entered_9567449": null, "hs_v2_date_entered_customclosedwonstage": null, "hs_v2_date_exited_66894120": null, "hs_v2_date_exited_9567448": null, "hs_v2_date_exited_9567449": null, "hs_v2_date_exited_customclosedwonstage": null, "hs_v2_latest_time_in_66894120": null, "hs_v2_latest_time_in_9567448": null, "hs_v2_latest_time_in_9567449": null, "hs_v2_latest_time_in_customclosedwonstage": null, "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-01-13T10:31:51.154000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": null, "num_notes": null, "pipeline": "default"}, "createdAt": "2021-01-13T10:31:51.154Z", "updatedAt": "2021-09-07T18:11:59.757Z", "archived": false, "properties_amount": 60000, "properties_amount_in_home_currency": 60000, "properties_closed_lost_reason": null, "properties_closed_won_reason": null, "properties_closedate": "2014-08-31T00:00:00+00:00", "properties_createdate": "2021-01-13T10:31:51.154000+00:00", "properties_days_to_close": 0, "properties_dealname": "Tim's Newer Deal", "properties_dealstage": "appointmentscheduled", "properties_dealtype": "newbusiness", "properties_description": null, "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_hs_acv": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_collaborator_owner_ids": null, "properties_hs_all_deal_split_owner_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_latest_source": null, "properties_hs_analytics_latest_source_company": null, "properties_hs_analytics_latest_source_contact": null, "properties_hs_analytics_latest_source_data_1": null, "properties_hs_analytics_latest_source_data_1_company": null, "properties_hs_analytics_latest_source_data_1_contact": null, "properties_hs_analytics_latest_source_data_2": null, "properties_hs_analytics_latest_source_data_2_company": null, "properties_hs_analytics_latest_source_data_2_contact": null, "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_latest_source_timestamp_company": null, "properties_hs_analytics_latest_source_timestamp_contact": null, "properties_hs_analytics_source": null, "properties_hs_analytics_source_data_1": null, "properties_hs_analytics_source_data_2": null, "properties_hs_arr": null, "properties_hs_campaign": null, "properties_hs_closed_amount": 0, "properties_hs_closed_amount_in_home_currency": 0, "properties_hs_closed_won_count": null, "properties_hs_closed_won_date": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-13T10:31:51.154000+00:00", "properties_hs_date_entered_66894120": null, "properties_hs_date_entered_9567448": null, "properties_hs_date_entered_9567449": null, "properties_hs_date_entered_appointmentscheduled": "2021-01-13T10:31:51.154000+00:00", "properties_hs_date_entered_closedlost": null, "properties_hs_date_entered_closedwon": null, "properties_hs_date_entered_contractsent": null, "properties_hs_date_entered_customclosedwonstage": null, "properties_hs_date_entered_decisionmakerboughtin": null, "properties_hs_date_entered_presentationscheduled": null, "properties_hs_date_entered_qualifiedtobuy": null, "properties_hs_date_exited_66894120": null, "properties_hs_date_exited_9567448": null, "properties_hs_date_exited_9567449": null, "properties_hs_date_exited_appointmentscheduled": null, "properties_hs_date_exited_closedlost": null, "properties_hs_date_exited_closedwon": null, "properties_hs_date_exited_contractsent": null, "properties_hs_date_exited_customclosedwonstage": null, "properties_hs_date_exited_decisionmakerboughtin": null, "properties_hs_date_exited_presentationscheduled": null, "properties_hs_date_exited_qualifiedtobuy": null, "properties_hs_days_to_close_raw": 0, "properties_hs_deal_amount_calculation_preference": null, "properties_hs_deal_stage_probability": 0.2, "properties_hs_deal_stage_probability_shadow": null, "properties_hs_exchange_rate": null, "properties_hs_forecast_amount": 60000, "properties_hs_forecast_probability": null, "properties_hs_is_closed": false, "properties_hs_is_closed_won": false, "properties_hs_is_deal_split": false, "properties_hs_is_open_count": 1, "properties_hs_lastmodifieddate": "2021-09-07T18:11:59.757000+00:00", "properties_hs_latest_meeting_activity": null, "properties_hs_likelihood_to_close": null, "properties_hs_line_item_global_term_hs_discount_percentage": null, "properties_hs_line_item_global_term_hs_discount_percentage_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_period": null, "properties_hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "properties_hs_line_item_global_term_recurringbillingfrequency": null, "properties_hs_line_item_global_term_recurringbillingfrequency_enabled": null, "properties_hs_manual_forecast_category": null, "properties_hs_merged_object_ids": null, "properties_hs_mrr": null, "properties_hs_next_step": null, "properties_hs_num_associated_deal_splits": null, "properties_hs_num_of_associated_line_items": 0, "properties_hs_num_target_accounts": null, "properties_hs_object_id": 3980673856, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_predicted_amount": null, "properties_hs_predicted_amount_in_home_currency": null, "properties_hs_priority": null, "properties_hs_projected_amount": 12000.0, "properties_hs_projected_amount_in_home_currency": 12000.0, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_tcv": null, "properties_hs_time_in_66894120": null, "properties_hs_time_in_9567448": null, "properties_hs_time_in_9567449": null, "properties_hs_time_in_appointmentscheduled": 92113051787, "properties_hs_time_in_closedlost": null, "properties_hs_time_in_closedwon": null, "properties_hs_time_in_contractsent": null, "properties_hs_time_in_customclosedwonstage": null, "properties_hs_time_in_decisionmakerboughtin": null, "properties_hs_time_in_presentationscheduled": null, "properties_hs_time_in_qualifiedtobuy": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_v2_cumulative_time_in_66894120": null, "properties_hs_v2_cumulative_time_in_9567448": null, "properties_hs_v2_cumulative_time_in_9567449": null, "properties_hs_v2_cumulative_time_in_customclosedwonstage": null, "properties_hs_v2_date_entered_66894120": null, "properties_hs_v2_date_entered_9567448": null, "properties_hs_v2_date_entered_9567449": null, "properties_hs_v2_date_entered_customclosedwonstage": null, "properties_hs_v2_date_exited_66894120": null, "properties_hs_v2_date_exited_9567448": null, "properties_hs_v2_date_exited_9567449": null, "properties_hs_v2_date_exited_customclosedwonstage": null, "properties_hs_v2_latest_time_in_66894120": null, "properties_hs_v2_latest_time_in_9567448": null, "properties_hs_v2_latest_time_in_9567449": null, "properties_hs_v2_latest_time_in_customclosedwonstage": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2021-01-13T10:31:51.154000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_pipeline": "default"}, "emitted_at": 1702646964423} +{"stream": "deals", "data": {"id": "3986867076", "properties": {"amount": 6, "amount_in_home_currency": 6, "closed_lost_reason": null, "closed_won_reason": null, "closedate": "2014-08-31T00:00:00+00:00", "createdate": "2021-01-14T14:38:00.797000+00:00", "days_to_close": 0, "dealname": "Test Deal 2", "dealstage": "appointmentscheduled", "dealtype": "newbusiness", "description": null, "engagements_last_meeting_booked": null, "engagements_last_meeting_booked_campaign": null, "engagements_last_meeting_booked_medium": null, "engagements_last_meeting_booked_source": null, "hs_acv": null, "hs_all_accessible_team_ids": null, "hs_all_collaborator_owner_ids": null, "hs_all_deal_split_owner_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_analytics_latest_source": null, "hs_analytics_latest_source_company": null, "hs_analytics_latest_source_contact": null, "hs_analytics_latest_source_data_1": null, "hs_analytics_latest_source_data_1_company": null, "hs_analytics_latest_source_data_1_contact": null, "hs_analytics_latest_source_data_2": null, "hs_analytics_latest_source_data_2_company": null, "hs_analytics_latest_source_data_2_contact": null, "hs_analytics_latest_source_timestamp": null, "hs_analytics_latest_source_timestamp_company": null, "hs_analytics_latest_source_timestamp_contact": null, "hs_analytics_source": null, "hs_analytics_source_data_1": null, "hs_analytics_source_data_2": null, "hs_arr": null, "hs_campaign": null, "hs_closed_amount": 0, "hs_closed_amount_in_home_currency": 0, "hs_closed_won_count": null, "hs_closed_won_date": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-14T14:38:00.797000+00:00", "hs_date_entered_66894120": null, "hs_date_entered_9567448": null, "hs_date_entered_9567449": null, "hs_date_entered_appointmentscheduled": "2021-01-14T14:38:00.797000+00:00", "hs_date_entered_closedlost": null, "hs_date_entered_closedwon": null, "hs_date_entered_contractsent": null, "hs_date_entered_customclosedwonstage": null, "hs_date_entered_decisionmakerboughtin": null, "hs_date_entered_presentationscheduled": null, "hs_date_entered_qualifiedtobuy": null, "hs_date_exited_66894120": null, "hs_date_exited_9567448": null, "hs_date_exited_9567449": null, "hs_date_exited_appointmentscheduled": null, "hs_date_exited_closedlost": null, "hs_date_exited_closedwon": null, "hs_date_exited_contractsent": null, "hs_date_exited_customclosedwonstage": null, "hs_date_exited_decisionmakerboughtin": null, "hs_date_exited_presentationscheduled": null, "hs_date_exited_qualifiedtobuy": null, "hs_days_to_close_raw": 0, "hs_deal_amount_calculation_preference": null, "hs_deal_stage_probability": 0.2, "hs_deal_stage_probability_shadow": null, "hs_exchange_rate": null, "hs_forecast_amount": 6, "hs_forecast_probability": null, "hs_is_closed": false, "hs_is_closed_won": false, "hs_is_deal_split": false, "hs_is_open_count": 1, "hs_lastmodifieddate": "2021-09-07T00:24:18.932000+00:00", "hs_latest_meeting_activity": null, "hs_likelihood_to_close": null, "hs_line_item_global_term_hs_discount_percentage": null, "hs_line_item_global_term_hs_discount_percentage_enabled": null, "hs_line_item_global_term_hs_recurring_billing_period": null, "hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "hs_line_item_global_term_hs_recurring_billing_start_date": null, "hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "hs_line_item_global_term_recurringbillingfrequency": null, "hs_line_item_global_term_recurringbillingfrequency_enabled": null, "hs_manual_forecast_category": null, "hs_merged_object_ids": null, "hs_mrr": null, "hs_next_step": null, "hs_num_associated_deal_splits": null, "hs_num_of_associated_line_items": 0, "hs_num_target_accounts": 0, "hs_object_id": 3986867076, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_pinned_engagement_id": null, "hs_predicted_amount": null, "hs_predicted_amount_in_home_currency": null, "hs_priority": null, "hs_projected_amount": 1.2000000000000002, "hs_projected_amount_in_home_currency": 1.2000000000000002, "hs_read_only": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_tcv": null, "hs_time_in_66894120": null, "hs_time_in_9567448": null, "hs_time_in_9567449": null, "hs_time_in_appointmentscheduled": 92011882144, "hs_time_in_closedlost": null, "hs_time_in_closedwon": null, "hs_time_in_contractsent": null, "hs_time_in_customclosedwonstage": null, "hs_time_in_decisionmakerboughtin": null, "hs_time_in_presentationscheduled": null, "hs_time_in_qualifiedtobuy": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_v2_cumulative_time_in_66894120": null, "hs_v2_cumulative_time_in_9567448": null, "hs_v2_cumulative_time_in_9567449": null, "hs_v2_cumulative_time_in_customclosedwonstage": null, "hs_v2_date_entered_66894120": null, "hs_v2_date_entered_9567448": null, "hs_v2_date_entered_9567449": null, "hs_v2_date_entered_customclosedwonstage": null, "hs_v2_date_exited_66894120": null, "hs_v2_date_exited_9567448": null, "hs_v2_date_exited_9567449": null, "hs_v2_date_exited_customclosedwonstage": null, "hs_v2_latest_time_in_66894120": null, "hs_v2_latest_time_in_9567448": null, "hs_v2_latest_time_in_9567449": null, "hs_v2_latest_time_in_customclosedwonstage": null, "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-01-14T14:38:00.797000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "num_associated_contacts": 0, "num_contacted_notes": null, "num_notes": null, "pipeline": "default"}, "createdAt": "2021-01-14T14:38:00.797Z", "updatedAt": "2021-09-07T00:24:18.932Z", "archived": false, "companies": ["5183409178", "5183409178"], "properties_amount": 6, "properties_amount_in_home_currency": 6, "properties_closed_lost_reason": null, "properties_closed_won_reason": null, "properties_closedate": "2014-08-31T00:00:00+00:00", "properties_createdate": "2021-01-14T14:38:00.797000+00:00", "properties_days_to_close": 0, "properties_dealname": "Test Deal 2", "properties_dealstage": "appointmentscheduled", "properties_dealtype": "newbusiness", "properties_description": null, "properties_engagements_last_meeting_booked": null, "properties_engagements_last_meeting_booked_campaign": null, "properties_engagements_last_meeting_booked_medium": null, "properties_engagements_last_meeting_booked_source": null, "properties_hs_acv": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_collaborator_owner_ids": null, "properties_hs_all_deal_split_owner_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_analytics_latest_source": null, "properties_hs_analytics_latest_source_company": null, "properties_hs_analytics_latest_source_contact": null, "properties_hs_analytics_latest_source_data_1": null, "properties_hs_analytics_latest_source_data_1_company": null, "properties_hs_analytics_latest_source_data_1_contact": null, "properties_hs_analytics_latest_source_data_2": null, "properties_hs_analytics_latest_source_data_2_company": null, "properties_hs_analytics_latest_source_data_2_contact": null, "properties_hs_analytics_latest_source_timestamp": null, "properties_hs_analytics_latest_source_timestamp_company": null, "properties_hs_analytics_latest_source_timestamp_contact": null, "properties_hs_analytics_source": null, "properties_hs_analytics_source_data_1": null, "properties_hs_analytics_source_data_2": null, "properties_hs_arr": null, "properties_hs_campaign": null, "properties_hs_closed_amount": 0, "properties_hs_closed_amount_in_home_currency": 0, "properties_hs_closed_won_count": null, "properties_hs_closed_won_date": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-14T14:38:00.797000+00:00", "properties_hs_date_entered_66894120": null, "properties_hs_date_entered_9567448": null, "properties_hs_date_entered_9567449": null, "properties_hs_date_entered_appointmentscheduled": "2021-01-14T14:38:00.797000+00:00", "properties_hs_date_entered_closedlost": null, "properties_hs_date_entered_closedwon": null, "properties_hs_date_entered_contractsent": null, "properties_hs_date_entered_customclosedwonstage": null, "properties_hs_date_entered_decisionmakerboughtin": null, "properties_hs_date_entered_presentationscheduled": null, "properties_hs_date_entered_qualifiedtobuy": null, "properties_hs_date_exited_66894120": null, "properties_hs_date_exited_9567448": null, "properties_hs_date_exited_9567449": null, "properties_hs_date_exited_appointmentscheduled": null, "properties_hs_date_exited_closedlost": null, "properties_hs_date_exited_closedwon": null, "properties_hs_date_exited_contractsent": null, "properties_hs_date_exited_customclosedwonstage": null, "properties_hs_date_exited_decisionmakerboughtin": null, "properties_hs_date_exited_presentationscheduled": null, "properties_hs_date_exited_qualifiedtobuy": null, "properties_hs_days_to_close_raw": 0, "properties_hs_deal_amount_calculation_preference": null, "properties_hs_deal_stage_probability": 0.2, "properties_hs_deal_stage_probability_shadow": null, "properties_hs_exchange_rate": null, "properties_hs_forecast_amount": 6, "properties_hs_forecast_probability": null, "properties_hs_is_closed": false, "properties_hs_is_closed_won": false, "properties_hs_is_deal_split": false, "properties_hs_is_open_count": 1, "properties_hs_lastmodifieddate": "2021-09-07T00:24:18.932000+00:00", "properties_hs_latest_meeting_activity": null, "properties_hs_likelihood_to_close": null, "properties_hs_line_item_global_term_hs_discount_percentage": null, "properties_hs_line_item_global_term_hs_discount_percentage_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_period": null, "properties_hs_line_item_global_term_hs_recurring_billing_period_enabled": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date": null, "properties_hs_line_item_global_term_hs_recurring_billing_start_date_enabled": null, "properties_hs_line_item_global_term_recurringbillingfrequency": null, "properties_hs_line_item_global_term_recurringbillingfrequency_enabled": null, "properties_hs_manual_forecast_category": null, "properties_hs_merged_object_ids": null, "properties_hs_mrr": null, "properties_hs_next_step": null, "properties_hs_num_associated_deal_splits": null, "properties_hs_num_of_associated_line_items": 0, "properties_hs_num_target_accounts": 0, "properties_hs_object_id": 3986867076, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_predicted_amount": null, "properties_hs_predicted_amount_in_home_currency": null, "properties_hs_priority": null, "properties_hs_projected_amount": 1.2000000000000002, "properties_hs_projected_amount_in_home_currency": 1.2000000000000002, "properties_hs_read_only": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_tcv": null, "properties_hs_time_in_66894120": null, "properties_hs_time_in_9567448": null, "properties_hs_time_in_9567449": null, "properties_hs_time_in_appointmentscheduled": 92011882144, "properties_hs_time_in_closedlost": null, "properties_hs_time_in_closedwon": null, "properties_hs_time_in_contractsent": null, "properties_hs_time_in_customclosedwonstage": null, "properties_hs_time_in_decisionmakerboughtin": null, "properties_hs_time_in_presentationscheduled": null, "properties_hs_time_in_qualifiedtobuy": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_v2_cumulative_time_in_66894120": null, "properties_hs_v2_cumulative_time_in_9567448": null, "properties_hs_v2_cumulative_time_in_9567449": null, "properties_hs_v2_cumulative_time_in_customclosedwonstage": null, "properties_hs_v2_date_entered_66894120": null, "properties_hs_v2_date_entered_9567448": null, "properties_hs_v2_date_entered_9567449": null, "properties_hs_v2_date_entered_customclosedwonstage": null, "properties_hs_v2_date_exited_66894120": null, "properties_hs_v2_date_exited_9567448": null, "properties_hs_v2_date_exited_9567449": null, "properties_hs_v2_date_exited_customclosedwonstage": null, "properties_hs_v2_latest_time_in_66894120": null, "properties_hs_v2_latest_time_in_9567448": null, "properties_hs_v2_latest_time_in_9567449": null, "properties_hs_v2_latest_time_in_customclosedwonstage": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2021-01-14T14:38:00.797000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_num_associated_contacts": 0, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_pipeline": "default"}, "emitted_at": 1702646964424} +{"stream": "email_events", "data": {"appName": "BatchTest", "location": {"country": "Unknown", "state": "Unknown", "city": "Unknown", "zipcode": "Unknown"}, "id": "17d3fcc4-bc34-38b4-9103-69b5896bbdde", "duration": 0, "browser": {"name": "Google Image Cache", "family": "Google Image Cache", "producer": "", "producerUrl": "", "type": "Proxy", "url": "", "version": []}, "created": 1614191191202, "userAgent": "Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0 (via ggpht.com GoogleImageProxy)", "deviceType": "COMPUTER", "type": "OPEN", "recipient": "integration-test@airbyte.io", "portalId": 8727216, "sentBy": {"id": "dd239309-7866-4705-a3e9-c571dd349477", "created": 1614119023182}, "smtpId": null, "filteredEvent": false, "appId": 20053, "emailCampaignId": 2}, "emitted_at": 1697714199237} +{"stream": "email_events", "data": {"appName": "BatchTest", "location": {"country": "Unknown", "state": "Unknown", "city": "Unknown", "zipcode": "Unknown"}, "id": "e5cbe134-db76-32cb-9e82-9dafcbaf8b64", "duration": 0, "browser": {"name": "Google Image Cache", "family": "Google Image Cache", "producer": "", "producerUrl": "", "type": "Proxy", "url": "", "version": []}, "created": 1614122124339, "userAgent": "Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko Firefox/11.0 (via ggpht.com GoogleImageProxy)", "deviceType": "COMPUTER", "type": "OPEN", "recipient": "integration-test@airbyte.io", "portalId": 8727216, "sentBy": {"id": "dd239309-7866-4705-a3e9-c571dd349477", "created": 1614119023182}, "smtpId": null, "filteredEvent": false, "appId": 20053, "emailCampaignId": 2}, "emitted_at": 1697714199238} +{"stream": "email_events", "data": {"appName": "BatchTest", "location": {"country": "UNITED STATES", "state": "california", "city": "mountain view", "latitude": 37.40599, "longitude": -122.078514, "zipcode": "94043"}, "id": "35b79cd1-3527-3ae7-b316-be0bbf872839", "duration": 1229, "browser": {"name": "Microsoft Edge 12.246", "family": "Microsoft Edge", "producer": "Microsoft Corporation.", "producerUrl": "https://www.microsoft.com/about/", "type": "Browser", "url": "https://en.wikipedia.org/wiki/Microsoft_Edge", "version": ["12.246"]}, "created": 1614119026757, "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.246 Mozilla/5.0", "deviceType": "COMPUTER", "type": "OPEN", "recipient": "integration-test@airbyte.io", "portalId": 8727216, "sentBy": {"id": "dd239309-7866-4705-a3e9-c571dd349477", "created": 1614119023182}, "smtpId": null, "filteredEvent": true, "appId": 20053, "emailCampaignId": 2}, "emitted_at": 1697714199239} +{"stream": "email_subscriptions", "data": {"id": 23704464, "portalId": 8727216, "name": "Test sub", "description": "Test sub", "active": true, "internal": false, "category": "Marketing", "channel": "Email", "businessUnitId": 0}, "emitted_at": 1697714208242} +{"stream": "email_subscriptions", "data": {"id": 94692364, "portalId": 8727216, "name": "One to One", "description": "One to One emails", "active": true, "internal": true, "category": "Sales", "channel": "Email", "internalName": "ONE_TO_ONE", "businessUnitId": 0}, "emitted_at": 1697714208243} +{"stream": "email_subscriptions", "data": {"id": 10798197, "portalId": 8727216, "name": "DONT USE ME", "description": "Receive feedback requests and customer service information.", "active": true, "internal": true, "category": "Service", "channel": "Email", "order": 0, "internalName": "SERVICE_HUB_FEEDBACK", "businessUnitId": 0}, "emitted_at": 1697714208243} +{"stream": "engagements", "data": {"id": 10584327028, "portalId": 8727216, "active": true, "createdAt": 1610636372009, "lastUpdated": 1610636372009, "type": "NOTE", "timestamp": 1409172644778, "allAccessibleTeamIds": [], "bodyPreview": "note body 5", "queueMembershipIds": [], "bodyPreviewIsTruncated": false, "bodyPreviewHtml": "\n \n \n note body 5\n \n", "gdprDeleted": false, "associations": {"contactIds": [], "companyIds": [], "dealIds": [], "ownerIds": [], "workflowIds": [], "ticketIds": [], "contentIds": [], "quoteIds": [], "marketingEventIds": []}, "attachments": [{"id": 4241968539}], "metadata": {"body": "note body 5"}, "associations_contactIds": [], "associations_companyIds": [], "associations_dealIds": [], "associations_ownerIds": [], "associations_workflowIds": [], "associations_ticketIds": [], "associations_contentIds": [], "associations_quoteIds": [], "associations_marketingEventIds": [], "metadata_body": "note body 5"}, "emitted_at": 1697714210187} +{"stream": "engagements", "data": {"id": 10584327043, "portalId": 8727216, "active": true, "createdAt": 1610636372714, "lastUpdated": 1610636372714, "type": "NOTE", "timestamp": 1409172644778, "allAccessibleTeamIds": [], "bodyPreview": "note body 7", "queueMembershipIds": [], "bodyPreviewIsTruncated": false, "bodyPreviewHtml": "\n \n \n note body 7\n \n", "gdprDeleted": false, "associations": {"contactIds": [], "companyIds": [], "dealIds": [], "ownerIds": [], "workflowIds": [], "ticketIds": [], "contentIds": [], "quoteIds": [], "marketingEventIds": []}, "attachments": [{"id": 4241968539}], "metadata": {"body": "note body 7"}, "associations_contactIds": [], "associations_companyIds": [], "associations_dealIds": [], "associations_ownerIds": [], "associations_workflowIds": [], "associations_ticketIds": [], "associations_contentIds": [], "associations_quoteIds": [], "associations_marketingEventIds": [], "metadata_body": "note body 7"}, "emitted_at": 1697714210189} +{"stream": "engagements", "data": {"id": 10584344127, "portalId": 8727216, "active": true, "createdAt": 1610636320990, "lastUpdated": 1610636320990, "type": "NOTE", "timestamp": 1409172644778, "allAccessibleTeamIds": [], "bodyPreview": "note body", "queueMembershipIds": [], "bodyPreviewIsTruncated": false, "bodyPreviewHtml": "\n \n \n note body\n \n", "gdprDeleted": false, "associations": {"contactIds": [], "companyIds": [], "dealIds": [], "ownerIds": [], "workflowIds": [], "ticketIds": [], "contentIds": [], "quoteIds": [], "marketingEventIds": []}, "attachments": [{"id": 4241968539}], "metadata": {"body": "note body"}, "associations_contactIds": [], "associations_companyIds": [], "associations_dealIds": [], "associations_ownerIds": [], "associations_workflowIds": [], "associations_ticketIds": [], "associations_contentIds": [], "associations_quoteIds": [], "associations_marketingEventIds": [], "metadata_body": "note body"}, "emitted_at": 1697714210190} +{"stream": "engagements_notes", "data": {"id": "10584327028", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": "4241968539", "hs_body_preview": "note body 5", "hs_body_preview_html": "\n \n \n note body 5\n \n", "hs_body_preview_is_truncated": false, "hs_created_by": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-14T14:59:32.009000+00:00", "hs_engagement_source": null, "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": false, "hs_lastmodifieddate": "2021-01-14T14:59:32.009000+00:00", "hs_merged_object_ids": null, "hs_modified_by": null, "hs_note_body": "note body 5", "hs_object_id": 10584327028, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": "", "hubspot_team_id": null}, "createdAt": "2021-01-14T14:59:32.009Z", "updatedAt": "2021-01-14T14:59:32.009Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": "4241968539", "properties_hs_body_preview": "note body 5", "properties_hs_body_preview_html": "\n \n \n note body 5\n \n", "properties_hs_body_preview_is_truncated": false, "properties_hs_created_by": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-14T14:59:32.009000+00:00", "properties_hs_engagement_source": null, "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": false, "properties_hs_lastmodifieddate": "2021-01-14T14:59:32.009000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": null, "properties_hs_note_body": "note body 5", "properties_hs_object_id": 10584327028, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": "", "properties_hubspot_team_id": null}, "emitted_at": 1697714218669} +{"stream": "engagements_notes", "data": {"id": "10584327043", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": "4241968539", "hs_body_preview": "note body 7", "hs_body_preview_html": "\n \n \n note body 7\n \n", "hs_body_preview_is_truncated": false, "hs_created_by": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-14T14:59:32.714000+00:00", "hs_engagement_source": null, "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": false, "hs_lastmodifieddate": "2021-01-14T14:59:32.714000+00:00", "hs_merged_object_ids": null, "hs_modified_by": null, "hs_note_body": "note body 7", "hs_object_id": 10584327043, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": "", "hubspot_team_id": null}, "createdAt": "2021-01-14T14:59:32.714Z", "updatedAt": "2021-01-14T14:59:32.714Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": "4241968539", "properties_hs_body_preview": "note body 7", "properties_hs_body_preview_html": "\n \n \n note body 7\n \n", "properties_hs_body_preview_is_truncated": false, "properties_hs_created_by": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-14T14:59:32.714000+00:00", "properties_hs_engagement_source": null, "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": false, "properties_hs_lastmodifieddate": "2021-01-14T14:59:32.714000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": null, "properties_hs_note_body": "note body 7", "properties_hs_object_id": 10584327043, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": "", "properties_hubspot_team_id": null}, "emitted_at": 1697714218670} +{"stream": "engagements_notes", "data": {"id": "10584344127", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": "4241968539", "hs_body_preview": "note body", "hs_body_preview_html": "\n \n \n note body\n \n", "hs_body_preview_is_truncated": false, "hs_created_by": null, "hs_created_by_user_id": null, "hs_createdate": "2021-01-14T14:58:40.990000+00:00", "hs_engagement_source": null, "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": false, "hs_lastmodifieddate": "2021-01-14T14:58:40.990000+00:00", "hs_merged_object_ids": null, "hs_modified_by": null, "hs_note_body": "note body", "hs_object_id": 10584344127, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": "", "hubspot_team_id": null}, "createdAt": "2021-01-14T14:58:40.990Z", "updatedAt": "2021-01-14T14:58:40.990Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": "4241968539", "properties_hs_body_preview": "note body", "properties_hs_body_preview_html": "\n \n \n note body\n \n", "properties_hs_body_preview_is_truncated": false, "properties_hs_created_by": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": "2021-01-14T14:58:40.990000+00:00", "properties_hs_engagement_source": null, "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": false, "properties_hs_lastmodifieddate": "2021-01-14T14:58:40.990000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": null, "properties_hs_note_body": "note body", "properties_hs_object_id": 10584344127, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_timestamp": "2014-08-27T20:50:44.778000+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": "", "properties_hubspot_team_id": null}, "emitted_at": 1697714218671} +{"stream": "engagements_tasks", "data": {"id": "11257289597", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "hs_body_preview_html": "\n \n \n Regarding note logged on Tuesday, February 23, 2021 10:25 PM\n \n", "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2021-02-23T20:25:07.503000+00:00", "hs_engagement_source": null, "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": false, "hs_lastmodifieddate": "2023-04-19T14:52:43.485000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_num_associated_companies": 0, "hs_num_associated_contacts": 0, "hs_num_associated_deals": 1, "hs_num_associated_queue_objects": 1, "hs_num_associated_tickets": 0, "hs_object_id": 11257289597, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[{\"engagementId\":11257289597,\"portalId\":8727216,\"engagementType\":\"TASK\",\"taskType\":\"REMINDER\",\"timestamp\":1614319200000,\"uuid\":\"TASK:e41fd851-f7c7-4381-85fa-796d076163aa\"}]}", "hs_task_body": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_is_overdue": true, "hs_task_is_past_due_date": true, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_missed_due_date": true, "hs_task_missed_due_date_count": 1, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": null, "hs_task_reminders": "1614319200000", "hs_task_repeat_interval": null, "hs_task_send_default_reminder": null, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "Follow up on Test deal 2", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_timestamp": "2021-02-26T06:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2021-02-23T20:25:07.503000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2021-02-23T20:25:07.503Z", "updatedAt": "2023-04-19T14:52:43.485Z", "archived": false, "deals": ["4315375411"], "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": null, "properties_hs_body_preview": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "properties_hs_body_preview_html": "\n \n \n Regarding note logged on Tuesday, February 23, 2021 10:25 PM\n \n", "properties_hs_body_preview_is_truncated": false, "properties_hs_calendar_event_id": null, "properties_hs_created_by": 12282590, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2021-02-23T20:25:07.503000+00:00", "properties_hs_engagement_source": null, "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": false, "properties_hs_lastmodifieddate": "2023-04-19T14:52:43.485000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": 12282590, "properties_hs_msteams_message_id": null, "properties_hs_num_associated_companies": 0, "properties_hs_num_associated_contacts": 0, "properties_hs_num_associated_deals": 1, "properties_hs_num_associated_queue_objects": 1, "properties_hs_num_associated_tickets": 0, "properties_hs_object_id": 11257289597, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_repeat_status": null, "properties_hs_scheduled_tasks": "{\"scheduledTasks\":[{\"engagementId\":11257289597,\"portalId\":8727216,\"engagementType\":\"TASK\",\"taskType\":\"REMINDER\",\"timestamp\":1614319200000,\"uuid\":\"TASK:e41fd851-f7c7-4381-85fa-796d076163aa\"}]}", "properties_hs_task_body": "Regarding note logged on Tuesday, February 23, 2021 10:25 PM", "properties_hs_task_completion_count": null, "properties_hs_task_completion_date": null, "properties_hs_task_contact_timezone": null, "properties_hs_task_family": "SALES", "properties_hs_task_for_object_type": "OWNER", "properties_hs_task_is_all_day": false, "properties_hs_task_is_completed": 0, "properties_hs_task_is_completed_call": 0, "properties_hs_task_is_completed_email": 0, "properties_hs_task_is_completed_linked_in": 0, "properties_hs_task_is_completed_sequence": 0, "properties_hs_task_is_overdue": true, "properties_hs_task_is_past_due_date": true, "properties_hs_task_last_contact_outreach": null, "properties_hs_task_last_sales_activity_timestamp": null, "properties_hs_task_missed_due_date": true, "properties_hs_task_missed_due_date_count": 1, "properties_hs_task_priority": "NONE", "properties_hs_task_probability_to_complete": null, "properties_hs_task_relative_reminders": null, "properties_hs_task_reminders": "1614319200000", "properties_hs_task_repeat_interval": null, "properties_hs_task_send_default_reminder": null, "properties_hs_task_sequence_enrollment_active": null, "properties_hs_task_sequence_step_enrollment_id": null, "properties_hs_task_sequence_step_order": null, "properties_hs_task_status": "NOT_STARTED", "properties_hs_task_subject": "Follow up on Test deal 2", "properties_hs_task_template_id": null, "properties_hs_task_type": "TODO", "properties_hs_timestamp": "2021-02-26T06:00:00+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2021-02-23T20:25:07.503000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null}, "emitted_at": 1700237230220} +{"stream": "engagements_tasks", "data": {"id": "30652597343", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": null, "hs_body_preview_html": null, "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:41:48.834000+00:00", "hs_engagement_source": "CRM_UI", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_lastmodifieddate": "2023-04-04T15:11:47.231000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_num_associated_companies": 0, "hs_num_associated_contacts": 0, "hs_num_associated_deals": 0, "hs_num_associated_queue_objects": 0, "hs_num_associated_tickets": 0, "hs_object_id": 30652597343, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "hs_task_body": null, "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_is_overdue": true, "hs_task_is_past_due_date": true, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_missed_due_date": true, "hs_task_missed_due_date_count": 1, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": "[]", "hs_task_reminders": null, "hs_task_repeat_interval": null, "hs_task_send_default_reminder": false, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "test", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_timestamp": "2023-02-03T07:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2023-01-30T23:41:48.834000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2023-01-30T23:41:48.834Z", "updatedAt": "2023-04-04T15:11:47.231Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": null, "properties_hs_body_preview": null, "properties_hs_body_preview_html": null, "properties_hs_body_preview_is_truncated": false, "properties_hs_calendar_event_id": null, "properties_hs_created_by": 12282590, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-01-30T23:41:48.834000+00:00", "properties_hs_engagement_source": "CRM_UI", "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": null, "properties_hs_lastmodifieddate": "2023-04-04T15:11:47.231000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": 12282590, "properties_hs_msteams_message_id": null, "properties_hs_num_associated_companies": 0, "properties_hs_num_associated_contacts": 0, "properties_hs_num_associated_deals": 0, "properties_hs_num_associated_queue_objects": 0, "properties_hs_num_associated_tickets": 0, "properties_hs_object_id": 30652597343, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_repeat_status": null, "properties_hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "properties_hs_task_body": null, "properties_hs_task_completion_count": null, "properties_hs_task_completion_date": null, "properties_hs_task_contact_timezone": null, "properties_hs_task_family": "SALES", "properties_hs_task_for_object_type": "OWNER", "properties_hs_task_is_all_day": false, "properties_hs_task_is_completed": 0, "properties_hs_task_is_completed_call": 0, "properties_hs_task_is_completed_email": 0, "properties_hs_task_is_completed_linked_in": 0, "properties_hs_task_is_completed_sequence": 0, "properties_hs_task_is_overdue": true, "properties_hs_task_is_past_due_date": true, "properties_hs_task_last_contact_outreach": null, "properties_hs_task_last_sales_activity_timestamp": null, "properties_hs_task_missed_due_date": true, "properties_hs_task_missed_due_date_count": 1, "properties_hs_task_priority": "NONE", "properties_hs_task_probability_to_complete": null, "properties_hs_task_relative_reminders": "[]", "properties_hs_task_reminders": null, "properties_hs_task_repeat_interval": null, "properties_hs_task_send_default_reminder": false, "properties_hs_task_sequence_enrollment_active": null, "properties_hs_task_sequence_step_enrollment_id": null, "properties_hs_task_sequence_step_order": null, "properties_hs_task_status": "NOT_STARTED", "properties_hs_task_subject": "test", "properties_hs_task_template_id": null, "properties_hs_task_type": "TODO", "properties_hs_timestamp": "2023-02-03T07:00:00+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2023-01-30T23:41:48.834000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null}, "emitted_at": 1700237230222} +{"stream": "engagements_tasks", "data": {"id": "30652613208", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": "52550153", "hs_all_team_ids": null, "hs_at_mentioned_owner_ids": null, "hs_attachment_ids": null, "hs_body_preview": null, "hs_body_preview_html": null, "hs_body_preview_is_truncated": false, "hs_calendar_event_id": null, "hs_created_by": 12282590, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-01-30T23:51:52.099000+00:00", "hs_engagement_source": "CRM_UI", "hs_engagement_source_id": null, "hs_follow_up_action": null, "hs_gdpr_deleted": null, "hs_lastmodifieddate": "2023-01-30T23:51:54.343000+00:00", "hs_merged_object_ids": null, "hs_modified_by": 12282590, "hs_msteams_message_id": null, "hs_num_associated_companies": 1, "hs_num_associated_contacts": 0, "hs_num_associated_deals": 0, "hs_num_associated_queue_objects": 1, "hs_num_associated_tickets": 0, "hs_object_id": 30652613208, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_name": null, "hs_queue_membership_ids": null, "hs_read_only": null, "hs_repeat_status": null, "hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "hs_task_body": null, "hs_task_completion_count": null, "hs_task_completion_date": null, "hs_task_contact_timezone": null, "hs_task_family": "SALES", "hs_task_for_object_type": "OWNER", "hs_task_is_all_day": false, "hs_task_is_completed": 0, "hs_task_is_completed_call": 0, "hs_task_is_completed_email": 0, "hs_task_is_completed_linked_in": 0, "hs_task_is_completed_sequence": 0, "hs_task_is_overdue": true, "hs_task_is_past_due_date": true, "hs_task_last_contact_outreach": null, "hs_task_last_sales_activity_timestamp": null, "hs_task_missed_due_date": true, "hs_task_missed_due_date_count": 1, "hs_task_priority": "NONE", "hs_task_probability_to_complete": null, "hs_task_relative_reminders": "[]", "hs_task_reminders": null, "hs_task_repeat_interval": null, "hs_task_send_default_reminder": false, "hs_task_sequence_enrollment_active": null, "hs_task_sequence_step_enrollment_id": null, "hs_task_sequence_step_order": null, "hs_task_status": "NOT_STARTED", "hs_task_subject": "test", "hs_task_template_id": null, "hs_task_type": "TODO", "hs_timestamp": "2023-02-03T07:00:00+00:00", "hs_unique_creation_key": null, "hs_unique_id": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "12282590", "hs_was_imported": null, "hubspot_owner_assigneddate": "2023-01-30T23:51:52.099000+00:00", "hubspot_owner_id": "52550153", "hubspot_team_id": null}, "createdAt": "2023-01-30T23:51:52.099Z", "updatedAt": "2023-01-30T23:51:54.343Z", "archived": false, "companies": ["11481383026"], "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": "52550153", "properties_hs_all_team_ids": null, "properties_hs_at_mentioned_owner_ids": null, "properties_hs_attachment_ids": null, "properties_hs_body_preview": null, "properties_hs_body_preview_html": null, "properties_hs_body_preview_is_truncated": false, "properties_hs_calendar_event_id": null, "properties_hs_created_by": 12282590, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-01-30T23:51:52.099000+00:00", "properties_hs_engagement_source": "CRM_UI", "properties_hs_engagement_source_id": null, "properties_hs_follow_up_action": null, "properties_hs_gdpr_deleted": null, "properties_hs_lastmodifieddate": "2023-01-30T23:51:54.343000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_modified_by": 12282590, "properties_hs_msteams_message_id": null, "properties_hs_num_associated_companies": 1, "properties_hs_num_associated_contacts": 0, "properties_hs_num_associated_deals": 0, "properties_hs_num_associated_queue_objects": 1, "properties_hs_num_associated_tickets": 0, "properties_hs_object_id": 30652613208, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_name": null, "properties_hs_queue_membership_ids": null, "properties_hs_read_only": null, "properties_hs_repeat_status": null, "properties_hs_scheduled_tasks": "{\"scheduledTasks\":[]}", "properties_hs_task_body": null, "properties_hs_task_completion_count": null, "properties_hs_task_completion_date": null, "properties_hs_task_contact_timezone": null, "properties_hs_task_family": "SALES", "properties_hs_task_for_object_type": "OWNER", "properties_hs_task_is_all_day": false, "properties_hs_task_is_completed": 0, "properties_hs_task_is_completed_call": 0, "properties_hs_task_is_completed_email": 0, "properties_hs_task_is_completed_linked_in": 0, "properties_hs_task_is_completed_sequence": 0, "properties_hs_task_is_overdue": true, "properties_hs_task_is_past_due_date": true, "properties_hs_task_last_contact_outreach": null, "properties_hs_task_last_sales_activity_timestamp": null, "properties_hs_task_missed_due_date": true, "properties_hs_task_missed_due_date_count": 1, "properties_hs_task_priority": "NONE", "properties_hs_task_probability_to_complete": null, "properties_hs_task_relative_reminders": "[]", "properties_hs_task_reminders": null, "properties_hs_task_repeat_interval": null, "properties_hs_task_send_default_reminder": false, "properties_hs_task_sequence_enrollment_active": null, "properties_hs_task_sequence_step_enrollment_id": null, "properties_hs_task_sequence_step_order": null, "properties_hs_task_status": "NOT_STARTED", "properties_hs_task_subject": "test", "properties_hs_task_template_id": null, "properties_hs_task_type": "TODO", "properties_hs_timestamp": "2023-02-03T07:00:00+00:00", "properties_hs_unique_creation_key": null, "properties_hs_unique_id": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "12282590", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": "2023-01-30T23:51:52.099000+00:00", "properties_hubspot_owner_id": "52550153", "properties_hubspot_team_id": null}, "emitted_at": 1700237230223} +{"stream": "forms", "data": {"id": "01ba116c-f3a8-4957-8884-ff0c4420af76", "name": "DemoForm", "createdAt": "2021-01-14T14:44:48.278Z", "updatedAt": "2021-01-14T14:44:48.278Z", "archived": false, "fieldGroups": [{"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "firstname", "label": "First Name", "required": false, "hidden": false, "fieldType": "single_line_text"}]}, {"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "lastname", "label": "Last Name", "required": false, "hidden": false, "fieldType": "single_line_text"}]}, {"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "adress_1", "label": "Adress 1", "required": false, "hidden": false, "fieldType": "single_line_text"}]}], "configuration": {"language": "en", "cloneable": true, "postSubmitAction": {"type": "thank_you", "value": ""}, "editable": true, "archivable": true, "recaptchaEnabled": false, "notifyContactOwner": false, "notifyRecipients": [], "createNewContactForNewEmail": false, "prePopulateKnownValues": true, "allowLinkToResetKnownValues": false, "lifecycleStages": []}, "displayOptions": {"renderRawHtml": false, "theme": "default_style", "submitButtonText": "Submit", "style": {"fontFamily": "arial, helvetica, sans-serif", "backgroundWidth": "100%", "labelTextColor": "#33475b", "labelTextSize": "11px", "helpTextColor": "#7C98B6", "helpTextSize": "11px", "legalConsentTextColor": "#33475b", "legalConsentTextSize": "14px", "submitColor": "#ff7a59", "submitAlignment": "left", "submitFontColor": "#ffffff", "submitSize": "12px"}, "cssClass": null}, "legalConsentOptions": {"type": "none"}, "formType": "hubspot"}, "emitted_at": 1697714221520} +{"stream": "forms", "data": {"id": "03e69987-1dcb-4d55-9cb6-d3812ac00ee6", "name": "New form 93", "createdAt": "2023-02-13T16:56:33.108Z", "updatedAt": "2023-02-13T16:56:33.108Z", "archived": false, "fieldGroups": [{"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "email", "label": "Email", "required": true, "hidden": false, "fieldType": "email", "validation": {"blockedEmailDomains": [], "useDefaultBlockList": false}}]}], "configuration": {"language": "en", "cloneable": true, "postSubmitAction": {"type": "thank_you", "value": "Thanks for submitting the form."}, "editable": true, "archivable": true, "recaptchaEnabled": false, "notifyContactOwner": false, "notifyRecipients": ["12282590"], "createNewContactForNewEmail": false, "prePopulateKnownValues": true, "allowLinkToResetKnownValues": false, "lifecycleStages": []}, "displayOptions": {"renderRawHtml": false, "theme": "default_style", "submitButtonText": "Submit", "style": {"fontFamily": "arial, helvetica, sans-serif", "backgroundWidth": "100%", "labelTextColor": "#33475b", "labelTextSize": "14px", "helpTextColor": "#7C98B6", "helpTextSize": "11px", "legalConsentTextColor": "#33475b", "legalConsentTextSize": "14px", "submitColor": "#ff7a59", "submitAlignment": "left", "submitFontColor": "#ffffff", "submitSize": "12px"}, "cssClass": "hs-form stacked"}, "legalConsentOptions": {"type": "implicit_consent_to_process", "communicationConsentText": "integrationtest is committed to protecting and respecting your privacy, and we\u2019ll only use your personal information to administer your account and to provide the products and services you requested from us. From time to time, we would like to contact you about our products and services, as well as other content that may be of interest to you. If you consent to us contacting you for this purpose, please tick below to say how you would like us to contact you:", "communicationsCheckboxes": [{"required": false, "subscriptionTypeId": 23704464, "label": "I agree to receive other communications from [MAIN] integration test account."}], "privacyText": "You may unsubscribe from these communications at any time. For more information on how to unsubscribe, our privacy practices, and how we are committed to protecting and respecting your privacy, please review our Privacy Policy.", "consentToProcessText": "By clicking submit below, you consent to allow integrationtest to store and process the personal information submitted above to provide you the content requested."}, "formType": "hubspot"}, "emitted_at": 1697714221521} +{"stream": "forms", "data": {"id": "0a7fd84f-471e-444a-a4e0-ca36d39f8af7", "name": "New form 27", "createdAt": "2023-02-13T16:45:22.640Z", "updatedAt": "2023-02-13T16:45:22.640Z", "archived": false, "fieldGroups": [{"groupType": "default_group", "richTextType": "text", "fields": [{"objectTypeId": "0-1", "name": "email", "label": "Email", "required": true, "hidden": false, "fieldType": "email", "validation": {"blockedEmailDomains": [], "useDefaultBlockList": false}}]}], "configuration": {"language": "en", "cloneable": true, "postSubmitAction": {"type": "thank_you", "value": "Thanks for submitting the form."}, "editable": true, "archivable": true, "recaptchaEnabled": false, "notifyContactOwner": false, "notifyRecipients": ["12282590"], "createNewContactForNewEmail": false, "prePopulateKnownValues": true, "allowLinkToResetKnownValues": false, "lifecycleStages": []}, "displayOptions": {"renderRawHtml": false, "theme": "default_style", "submitButtonText": "Submit", "style": {"fontFamily": "arial, helvetica, sans-serif", "backgroundWidth": "100%", "labelTextColor": "#33475b", "labelTextSize": "14px", "helpTextColor": "#7C98B6", "helpTextSize": "11px", "legalConsentTextColor": "#33475b", "legalConsentTextSize": "14px", "submitColor": "#ff7a59", "submitAlignment": "left", "submitFontColor": "#ffffff", "submitSize": "12px"}, "cssClass": "hs-form stacked"}, "legalConsentOptions": {"type": "implicit_consent_to_process", "communicationConsentText": "integrationtest is committed to protecting and respecting your privacy, and we\u2019ll only use your personal information to administer your account and to provide the products and services you requested from us. From time to time, we would like to contact you about our products and services, as well as other content that may be of interest to you. If you consent to us contacting you for this purpose, please tick below to say how you would like us to contact you:", "communicationsCheckboxes": [{"required": false, "subscriptionTypeId": 23704464, "label": "I agree to receive other communications from [MAIN] integration test account."}], "privacyText": "You may unsubscribe from these communications at any time. For more information on how to unsubscribe, our privacy practices, and how we are committed to protecting and respecting your privacy, please review our Privacy Policy.", "consentToProcessText": "By clicking submit below, you consent to allow integrationtest to store and process the personal information submitted above to provide you the content requested."}, "formType": "hubspot"}, "emitted_at": 1697714221522} +{"stream": "goals", "data": {"id": "221880757009", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-07-31T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "hs_kpi_is_team_rollup": false, "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-08-01T00:45:14.830000+00:00", "hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757009, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_outcome": "completed", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-07-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "achieved", "hs_status_display_order": 4, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_template_id": 4, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-12-11T20:46:14.473Z", "archived": false, "properties_hs__migration_soft_delete": null, "properties_hs_ad_account_asset_ids": null, "properties_hs_ad_campaign_asset_ids": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignee_team_id": null, "properties_hs_assignee_user_id": 26748728, "properties_hs_contact_lifecycle_stage": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-10T13:57:36.691000+00:00", "properties_hs_currency": null, "properties_hs_deal_pipeline_ids": null, "properties_hs_edit_updates_notification_frequency": "weekly", "properties_hs_end_date": null, "properties_hs_end_datetime": "2023-07-31T23:59:59.999000+00:00", "properties_hs_fiscal_year_offset": 0, "properties_hs_goal_name": "Integration Test Goal Hubspot", "properties_hs_goal_target_group_id": 221880750627, "properties_hs_goal_type": "average_ticket_response_time", "properties_hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "properties_hs_is_forecastable": "true", "properties_hs_is_legacy": null, "properties_hs_kpi_display_unit": "hour", "properties_hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "properties_hs_kpi_is_team_rollup": false, "properties_hs_kpi_metric_type": "AVG", "properties_hs_kpi_object_type": "TICKET", "properties_hs_kpi_object_type_id": "0-5", "properties_hs_kpi_progress_percent": null, "properties_hs_kpi_property_name": "time_to_first_agent_reply", "properties_hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "properties_hs_kpi_time_period_property": "createdate", "properties_hs_kpi_tracking_method": "LOWER_IS_BETTER", "properties_hs_kpi_unit_type": "duration", "properties_hs_kpi_value": 0.0, "properties_hs_kpi_value_calculated_at": null, "properties_hs_kpi_value_last_calculated_at": "2023-08-01T00:45:14.830000+00:00", "properties_hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "properties_hs_legacy_active": null, "properties_hs_legacy_created_at": null, "properties_hs_legacy_created_by": null, "properties_hs_legacy_quarterly_target_composite_id": null, "properties_hs_legacy_sql_id": null, "properties_hs_legacy_unique_sql_id": null, "properties_hs_legacy_updated_at": null, "properties_hs_legacy_updated_by": null, "properties_hs_merged_object_ids": null, "properties_hs_migration_soft_delete": null, "properties_hs_milestone": "monthly", "properties_hs_object_id": 221880757009, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_outcome": "completed", "properties_hs_owner_ids_of_all_owners": "111730024", "properties_hs_participant_type": "users", "properties_hs_pipelines": "0", "properties_hs_progress_updates_notification_frequency": "weekly", "properties_hs_read_only": null, "properties_hs_should_notify_on_achieved": "false", "properties_hs_should_notify_on_edit_updates": "false", "properties_hs_should_notify_on_exceeded": "false", "properties_hs_should_notify_on_kickoff": "false", "properties_hs_should_notify_on_missed": "false", "properties_hs_should_notify_on_progress_updates": "false", "properties_hs_should_recalculate": "false", "properties_hs_start_date": null, "properties_hs_start_datetime": "2023-07-01T00:00:00+00:00", "properties_hs_static_kpi_filter_groups": "[]", "properties_hs_status": "achieved", "properties_hs_status_display_order": 4, "properties_hs_target_amount": 0.0, "properties_hs_target_amount_in_home_currency": 0.0, "properties_hs_team_id": null, "properties_hs_template_id": 4, "properties_hs_ticket_pipeline_ids": "0", "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "26748728", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null}, "emitted_at": 1702410363120} +{"stream": "goals", "data": {"id": "221880757010", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-09-30T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "hs_kpi_is_team_rollup": false, "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-10-01T22:31:08.621000+00:00", "hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757010, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_outcome": "completed", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-09-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "achieved", "hs_status_display_order": 4, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_template_id": 4, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-12-11T20:46:14.473Z", "archived": false, "properties_hs__migration_soft_delete": null, "properties_hs_ad_account_asset_ids": null, "properties_hs_ad_campaign_asset_ids": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignee_team_id": null, "properties_hs_assignee_user_id": 26748728, "properties_hs_contact_lifecycle_stage": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-10T13:57:36.691000+00:00", "properties_hs_currency": null, "properties_hs_deal_pipeline_ids": null, "properties_hs_edit_updates_notification_frequency": "weekly", "properties_hs_end_date": null, "properties_hs_end_datetime": "2023-09-30T23:59:59.999000+00:00", "properties_hs_fiscal_year_offset": 0, "properties_hs_goal_name": "Integration Test Goal Hubspot", "properties_hs_goal_target_group_id": 221880750627, "properties_hs_goal_type": "average_ticket_response_time", "properties_hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "properties_hs_is_forecastable": "true", "properties_hs_is_legacy": null, "properties_hs_kpi_display_unit": "hour", "properties_hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "properties_hs_kpi_is_team_rollup": false, "properties_hs_kpi_metric_type": "AVG", "properties_hs_kpi_object_type": "TICKET", "properties_hs_kpi_object_type_id": "0-5", "properties_hs_kpi_progress_percent": null, "properties_hs_kpi_property_name": "time_to_first_agent_reply", "properties_hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "properties_hs_kpi_time_period_property": "createdate", "properties_hs_kpi_tracking_method": "LOWER_IS_BETTER", "properties_hs_kpi_unit_type": "duration", "properties_hs_kpi_value": 0.0, "properties_hs_kpi_value_calculated_at": null, "properties_hs_kpi_value_last_calculated_at": "2023-10-01T22:31:08.621000+00:00", "properties_hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "properties_hs_legacy_active": null, "properties_hs_legacy_created_at": null, "properties_hs_legacy_created_by": null, "properties_hs_legacy_quarterly_target_composite_id": null, "properties_hs_legacy_sql_id": null, "properties_hs_legacy_unique_sql_id": null, "properties_hs_legacy_updated_at": null, "properties_hs_legacy_updated_by": null, "properties_hs_merged_object_ids": null, "properties_hs_migration_soft_delete": null, "properties_hs_milestone": "monthly", "properties_hs_object_id": 221880757010, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_outcome": "completed", "properties_hs_owner_ids_of_all_owners": "111730024", "properties_hs_participant_type": "users", "properties_hs_pipelines": "0", "properties_hs_progress_updates_notification_frequency": "weekly", "properties_hs_read_only": null, "properties_hs_should_notify_on_achieved": "false", "properties_hs_should_notify_on_edit_updates": "false", "properties_hs_should_notify_on_exceeded": "false", "properties_hs_should_notify_on_kickoff": "false", "properties_hs_should_notify_on_missed": "false", "properties_hs_should_notify_on_progress_updates": "false", "properties_hs_should_recalculate": "false", "properties_hs_start_date": null, "properties_hs_start_datetime": "2023-09-01T00:00:00+00:00", "properties_hs_static_kpi_filter_groups": "[]", "properties_hs_status": "achieved", "properties_hs_status_display_order": 4, "properties_hs_target_amount": 0.0, "properties_hs_target_amount_in_home_currency": 0.0, "properties_hs_team_id": null, "properties_hs_template_id": 4, "properties_hs_ticket_pipeline_ids": "0", "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "26748728", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null}, "emitted_at": 1702410363124} +{"stream": "goals", "data": {"id": "221880757011", "properties": {"hs__migration_soft_delete": null, "hs_ad_account_asset_ids": null, "hs_ad_campaign_asset_ids": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignee_team_id": null, "hs_assignee_user_id": 26748728, "hs_contact_lifecycle_stage": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-10T13:57:36.691000+00:00", "hs_currency": null, "hs_deal_pipeline_ids": null, "hs_edit_updates_notification_frequency": "weekly", "hs_end_date": null, "hs_end_datetime": "2023-08-31T23:59:59.999000+00:00", "hs_fiscal_year_offset": 0, "hs_goal_name": "Integration Test Goal Hubspot", "hs_goal_target_group_id": 221880750627, "hs_goal_type": "average_ticket_response_time", "hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "hs_is_forecastable": "true", "hs_is_legacy": null, "hs_kpi_display_unit": "hour", "hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "hs_kpi_is_team_rollup": false, "hs_kpi_metric_type": "AVG", "hs_kpi_object_type": "TICKET", "hs_kpi_object_type_id": "0-5", "hs_kpi_progress_percent": null, "hs_kpi_property_name": "time_to_first_agent_reply", "hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "hs_kpi_time_period_property": "createdate", "hs_kpi_tracking_method": "LOWER_IS_BETTER", "hs_kpi_unit_type": "duration", "hs_kpi_value": 0.0, "hs_kpi_value_calculated_at": null, "hs_kpi_value_last_calculated_at": "2023-09-01T15:26:00.500000+00:00", "hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "hs_legacy_active": null, "hs_legacy_created_at": null, "hs_legacy_created_by": null, "hs_legacy_quarterly_target_composite_id": null, "hs_legacy_sql_id": null, "hs_legacy_unique_sql_id": null, "hs_legacy_updated_at": null, "hs_legacy_updated_by": null, "hs_merged_object_ids": null, "hs_migration_soft_delete": null, "hs_milestone": "monthly", "hs_object_id": 221880757011, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_outcome": "completed", "hs_owner_ids_of_all_owners": "111730024", "hs_participant_type": "users", "hs_pipelines": "0", "hs_progress_updates_notification_frequency": "weekly", "hs_read_only": null, "hs_should_notify_on_achieved": "false", "hs_should_notify_on_edit_updates": "false", "hs_should_notify_on_exceeded": "false", "hs_should_notify_on_kickoff": "false", "hs_should_notify_on_missed": "false", "hs_should_notify_on_progress_updates": "false", "hs_should_recalculate": "false", "hs_start_date": null, "hs_start_datetime": "2023-08-01T00:00:00+00:00", "hs_static_kpi_filter_groups": "[]", "hs_status": "achieved", "hs_status_display_order": 4, "hs_target_amount": 0.0, "hs_target_amount_in_home_currency": 0.0, "hs_team_id": null, "hs_template_id": 4, "hs_ticket_pipeline_ids": "0", "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": "26748728", "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-10T13:57:36.691Z", "updatedAt": "2023-12-11T20:46:14.473Z", "archived": false, "properties_hs__migration_soft_delete": null, "properties_hs_ad_account_asset_ids": null, "properties_hs_ad_campaign_asset_ids": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignee_team_id": null, "properties_hs_assignee_user_id": 26748728, "properties_hs_contact_lifecycle_stage": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-10T13:57:36.691000+00:00", "properties_hs_currency": null, "properties_hs_deal_pipeline_ids": null, "properties_hs_edit_updates_notification_frequency": "weekly", "properties_hs_end_date": null, "properties_hs_end_datetime": "2023-08-31T23:59:59.999000+00:00", "properties_hs_fiscal_year_offset": 0, "properties_hs_goal_name": "Integration Test Goal Hubspot", "properties_hs_goal_target_group_id": 221880750627, "properties_hs_goal_type": "average_ticket_response_time", "properties_hs_group_correlation_uuid": "5c49f251-be20-43c6-87c7-dd273732b3a4", "properties_hs_is_forecastable": "true", "properties_hs_is_legacy": null, "properties_hs_kpi_display_unit": "hour", "properties_hs_kpi_filter_groups": "[{\"filters\":[{\"property\":\"hs_pipeline\",\"operator\":\"IN\",\"values\":[\"0\"]}]}]", "properties_hs_kpi_is_team_rollup": false, "properties_hs_kpi_metric_type": "AVG", "properties_hs_kpi_object_type": "TICKET", "properties_hs_kpi_object_type_id": "0-5", "properties_hs_kpi_progress_percent": null, "properties_hs_kpi_property_name": "time_to_first_agent_reply", "properties_hs_kpi_single_object_custom_goal_type_name": "avg_time_to_first_agent_reply_0-5", "properties_hs_kpi_time_period_property": "createdate", "properties_hs_kpi_tracking_method": "LOWER_IS_BETTER", "properties_hs_kpi_unit_type": "duration", "properties_hs_kpi_value": 0.0, "properties_hs_kpi_value_calculated_at": null, "properties_hs_kpi_value_last_calculated_at": "2023-09-01T15:26:00.500000+00:00", "properties_hs_lastmodifieddate": "2023-12-11T20:46:14.473000+00:00", "properties_hs_legacy_active": null, "properties_hs_legacy_created_at": null, "properties_hs_legacy_created_by": null, "properties_hs_legacy_quarterly_target_composite_id": null, "properties_hs_legacy_sql_id": null, "properties_hs_legacy_unique_sql_id": null, "properties_hs_legacy_updated_at": null, "properties_hs_legacy_updated_by": null, "properties_hs_merged_object_ids": null, "properties_hs_migration_soft_delete": null, "properties_hs_milestone": "monthly", "properties_hs_object_id": 221880757011, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_outcome": "completed", "properties_hs_owner_ids_of_all_owners": "111730024", "properties_hs_participant_type": "users", "properties_hs_pipelines": "0", "properties_hs_progress_updates_notification_frequency": "weekly", "properties_hs_read_only": null, "properties_hs_should_notify_on_achieved": "false", "properties_hs_should_notify_on_edit_updates": "false", "properties_hs_should_notify_on_exceeded": "false", "properties_hs_should_notify_on_kickoff": "false", "properties_hs_should_notify_on_missed": "false", "properties_hs_should_notify_on_progress_updates": "false", "properties_hs_should_recalculate": "false", "properties_hs_start_date": null, "properties_hs_start_datetime": "2023-08-01T00:00:00+00:00", "properties_hs_static_kpi_filter_groups": "[]", "properties_hs_status": "achieved", "properties_hs_status_display_order": 4, "properties_hs_target_amount": 0.0, "properties_hs_target_amount_in_home_currency": 0.0, "properties_hs_team_id": null, "properties_hs_template_id": 4, "properties_hs_ticket_pipeline_ids": "0", "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": "26748728", "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null}, "emitted_at": 1702410363125} +{"stream": "line_items", "data": {"id": "1188257165", "properties": {"amount": 10.0, "createdate": "2021-02-23T20:11:54.030000+00:00", "description": "Baseball hat, medium", "discount": null, "hs_acv": 10.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_allow_buyer_selected_quantity": null, "hs_arr": 0.0, "hs_billing_period_end_date": null, "hs_billing_period_start_date": null, "hs_billing_start_delay_days": null, "hs_billing_start_delay_months": null, "hs_billing_start_delay_type": null, "hs_cost_of_goods_sold": 5, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_external_id": null, "hs_images": null, "hs_lastmodifieddate": "2021-07-17T23:50:32.502000+00:00", "hs_line_item_currency_code": null, "hs_margin": 5.0, "hs_margin_acv": 5.0, "hs_margin_arr": 0.0, "hs_margin_mrr": 0.0, "hs_margin_tcv": 5.0, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_object_id": 1188257165, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_position_on_quote": 0, "hs_pre_discount_amount": 10, "hs_product_id": 646778218, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_end_date": null, "hs_recurring_billing_number_of_payments": 1, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_recurring_billing_terms": null, "hs_sku": null, "hs_sync_amount": null, "hs_tcv": 10.0, "hs_term_in_months": null, "hs_total_discount": 0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_variant_id": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Blue Hat", "price": 10, "quantity": 1, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:11:54.030Z", "updatedAt": "2021-07-17T23:50:32.502Z", "archived": false, "properties_amount": 10.0, "properties_createdate": "2021-02-23T20:11:54.030000+00:00", "properties_description": "Baseball hat, medium", "properties_discount": null, "properties_hs_acv": 10.0, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_allow_buyer_selected_quantity": null, "properties_hs_arr": 0.0, "properties_hs_billing_period_end_date": null, "properties_hs_billing_period_start_date": null, "properties_hs_billing_start_delay_days": null, "properties_hs_billing_start_delay_months": null, "properties_hs_billing_start_delay_type": null, "properties_hs_cost_of_goods_sold": 5, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_external_id": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-07-17T23:50:32.502000+00:00", "properties_hs_line_item_currency_code": null, "properties_hs_margin": 5.0, "properties_hs_margin_acv": 5.0, "properties_hs_margin_arr": 0.0, "properties_hs_margin_mrr": 0.0, "properties_hs_margin_tcv": 5.0, "properties_hs_merged_object_ids": null, "properties_hs_mrr": 0.0, "properties_hs_object_id": 1188257165, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_position_on_quote": 0, "properties_hs_pre_discount_amount": 10, "properties_hs_product_id": 646778218, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_end_date": null, "properties_hs_recurring_billing_number_of_payments": 1, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_recurring_billing_terms": null, "properties_hs_sku": null, "properties_hs_sync_amount": null, "properties_hs_tcv": 10.0, "properties_hs_term_in_months": null, "properties_hs_total_discount": 0, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_variant_id": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Blue Hat", "properties_price": 10, "properties_quantity": 1, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1697714248811} +{"stream": "line_items", "data": {"id": "1188257309", "properties": {"amount": 10.0, "createdate": "2021-02-23T20:11:54.030000+00:00", "description": "Baseball hat, medium", "discount": null, "hs_acv": 10.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_allow_buyer_selected_quantity": null, "hs_arr": 0.0, "hs_billing_period_end_date": null, "hs_billing_period_start_date": null, "hs_billing_start_delay_days": null, "hs_billing_start_delay_months": null, "hs_billing_start_delay_type": null, "hs_cost_of_goods_sold": 5, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_external_id": null, "hs_images": null, "hs_lastmodifieddate": "2021-07-19T03:57:09.834000+00:00", "hs_line_item_currency_code": null, "hs_margin": 5.0, "hs_margin_acv": 5.0, "hs_margin_arr": 0.0, "hs_margin_mrr": 0.0, "hs_margin_tcv": 5.0, "hs_merged_object_ids": null, "hs_mrr": 0.0, "hs_object_id": 1188257309, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_position_on_quote": 0, "hs_pre_discount_amount": 10, "hs_product_id": 646778218, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_end_date": null, "hs_recurring_billing_number_of_payments": 1, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_recurring_billing_terms": null, "hs_sku": null, "hs_sync_amount": null, "hs_tcv": 10.0, "hs_term_in_months": null, "hs_total_discount": 0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_variant_id": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Blue Hat", "price": 10, "quantity": 1, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:11:54.030Z", "updatedAt": "2021-07-19T03:57:09.834Z", "archived": false, "properties_amount": 10.0, "properties_createdate": "2021-02-23T20:11:54.030000+00:00", "properties_description": "Baseball hat, medium", "properties_discount": null, "properties_hs_acv": 10.0, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_allow_buyer_selected_quantity": null, "properties_hs_arr": 0.0, "properties_hs_billing_period_end_date": null, "properties_hs_billing_period_start_date": null, "properties_hs_billing_start_delay_days": null, "properties_hs_billing_start_delay_months": null, "properties_hs_billing_start_delay_type": null, "properties_hs_cost_of_goods_sold": 5, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_external_id": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-07-19T03:57:09.834000+00:00", "properties_hs_line_item_currency_code": null, "properties_hs_margin": 5.0, "properties_hs_margin_acv": 5.0, "properties_hs_margin_arr": 0.0, "properties_hs_margin_mrr": 0.0, "properties_hs_margin_tcv": 5.0, "properties_hs_merged_object_ids": null, "properties_hs_mrr": 0.0, "properties_hs_object_id": 1188257309, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_position_on_quote": 0, "properties_hs_pre_discount_amount": 10, "properties_hs_product_id": 646778218, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_end_date": null, "properties_hs_recurring_billing_number_of_payments": 1, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_recurring_billing_terms": null, "properties_hs_sku": null, "properties_hs_sync_amount": null, "properties_hs_tcv": 10.0, "properties_hs_term_in_months": null, "properties_hs_total_discount": 0, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_variant_id": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Blue Hat", "properties_price": 10, "properties_quantity": 1, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1697714248814} +{"stream": "line_items", "data": {"id": "1510167477", "properties": {"amount": 20.0, "createdate": "2021-05-21T10:22:40.683000+00:00", "description": "Top hat, large", "discount": null, "hs_acv": 60.0, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_allow_buyer_selected_quantity": null, "hs_arr": 60.0, "hs_billing_period_end_date": null, "hs_billing_period_start_date": null, "hs_billing_start_delay_days": null, "hs_billing_start_delay_months": null, "hs_billing_start_delay_type": null, "hs_cost_of_goods_sold": 10, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_external_id": null, "hs_images": null, "hs_lastmodifieddate": "2022-02-23T08:09:16.555000+00:00", "hs_line_item_currency_code": null, "hs_margin": 10.0, "hs_margin_acv": 30.0, "hs_margin_arr": 30.0, "hs_margin_mrr": 10.0, "hs_margin_tcv": 30.0, "hs_merged_object_ids": null, "hs_mrr": 20.0, "hs_object_id": 1510167477, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_position_on_quote": null, "hs_pre_discount_amount": 20, "hs_product_id": 646777910, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_end_date": "2022-05-28", "hs_recurring_billing_number_of_payments": 3, "hs_recurring_billing_period": "P3M", "hs_recurring_billing_start_date": "2022-02-28", "hs_recurring_billing_terms": null, "hs_sku": null, "hs_sync_amount": null, "hs_tcv": 60.0, "hs_term_in_months": 3, "hs_total_discount": 0, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_variant_id": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Red Hat", "price": 20, "quantity": 1, "recurringbillingfrequency": "monthly", "tax": null, "test": "2022-02-24", "test_product_price": "2022-02-23"}, "createdAt": "2021-05-21T10:22:40.683Z", "updatedAt": "2022-02-23T08:09:16.555Z", "archived": false, "properties_amount": 20.0, "properties_createdate": "2021-05-21T10:22:40.683000+00:00", "properties_description": "Top hat, large", "properties_discount": null, "properties_hs_acv": 60.0, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_allow_buyer_selected_quantity": null, "properties_hs_arr": 60.0, "properties_hs_billing_period_end_date": null, "properties_hs_billing_period_start_date": null, "properties_hs_billing_start_delay_days": null, "properties_hs_billing_start_delay_months": null, "properties_hs_billing_start_delay_type": null, "properties_hs_cost_of_goods_sold": 10, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_external_id": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2022-02-23T08:09:16.555000+00:00", "properties_hs_line_item_currency_code": null, "properties_hs_margin": 10.0, "properties_hs_margin_acv": 30.0, "properties_hs_margin_arr": 30.0, "properties_hs_margin_mrr": 10.0, "properties_hs_margin_tcv": 30.0, "properties_hs_merged_object_ids": null, "properties_hs_mrr": 20.0, "properties_hs_object_id": 1510167477, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_position_on_quote": null, "properties_hs_pre_discount_amount": 20, "properties_hs_product_id": 646777910, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_end_date": "2022-05-28", "properties_hs_recurring_billing_number_of_payments": 3, "properties_hs_recurring_billing_period": "P3M", "properties_hs_recurring_billing_start_date": "2022-02-28", "properties_hs_recurring_billing_terms": null, "properties_hs_sku": null, "properties_hs_sync_amount": null, "properties_hs_tcv": 60.0, "properties_hs_term_in_months": 3, "properties_hs_total_discount": 0, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_variant_id": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Red Hat", "properties_price": 20, "properties_quantity": 1, "properties_recurringbillingfrequency": "monthly", "properties_tax": null, "properties_test": "2022-02-24", "properties_test_product_price": "2022-02-23"}, "emitted_at": 1697714248816} +{"stream": "marketing_emails", "data": {"ab": false, "abHoursToWait": 4, "abSampleSizeDefault": null, "abSamplingDefault": null, "abSuccessMetric": null, "abTestPercentage": 50, "abVariation": false, "absoluteUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "aifeatures": null, "allEmailCampaignIds": [243851494], "analyticsPageId": "100523515217", "analyticsPageType": "email", "archivedAt": 0, "archivedInDashboard": false, "audienceAccess": "PUBLIC", "author": "integration-test@airbyte.io", "authorName": "Team-1 Airbyte", "blogRssSettings": null, "canSpamSettingsId": 36765207029, "categoryId": 2, "contentAccessRuleIds": [], "contentAccessRuleTypes": [], "contentTypeCategory": 2, "createPage": false, "created": 1675121582718, "createdById": 12282590, "currentState": "PUBLISHED", "currentlyPublished": true, "customReplyTo": "", "customReplyToEnabled": false, "domain": "", "emailBody": "{% content_attribute \"email_body\" %}{{ default_email_body }}{% end_content_attribute %}", "emailNote": "", "emailTemplateMode": "DRAG_AND_DROP", "emailType": "BATCH_EMAIL", "emailbodyPlaintext": "", "feedbackSurveyId": null, "flexAreas": {"main": {"boxed": false, "isSingleColumnFullWidth": false, "sections": [{"columns": [{"id": "column-0-0", "widgets": ["module-0-0-0"], "width": 12}], "id": "section-0", "style": {"backgroundColor": "#eaf0f6", "backgroundType": "CONTENT", "paddingBottom": "10px", "paddingTop": "10px"}}, {"columns": [{"id": "column-1-0", "widgets": ["module-1-0-0"], "width": 12}], "id": "section-1", "style": {"backgroundType": "CONTENT", "paddingBottom": "30px", "paddingTop": "30px"}}, {"columns": [{"id": "column-2-0", "widgets": ["module-2-0-0"], "width": 12}], "id": "section-2", "style": {"backgroundColor": "", "backgroundType": "CONTENT", "paddingBottom": "20px", "paddingTop": "20px"}}]}}, "freezeDate": 1675121645993, "fromName": "Team Airbyte", "hasContentAccessRules": false, "htmlTitle": "", "id": 100523515217, "isCreatedFomSandboxSync": false, "isGraymailSuppressionEnabled": true, "isInstanceLayoutPage": false, "isPublished": true, "isRecipientFatigueSuppressionEnabled": null, "language": "en", "layoutSections": {}, "liveDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "mailingListsExcluded": [], "mailingListsIncluded": [], "maxRssEntries": 5, "metaDescription": "", "name": "test", "pageExpiryEnabled": false, "pageRedirected": false, "pastMabExperimentIds": [], "portalId": 8727216, "previewKey": "nlkwziGL", "primaryEmailCampaignId": 243851494, "processingStatus": "PUBLISHED", "publishDate": 1675121645997, "publishImmediately": true, "publishedAt": 1675121646297, "publishedByEmail": "integration-test@airbyte.io", "publishedById": 12282590, "publishedByName": "Team-1 Airbyte", "publishedUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "replyTo": "integration-test@airbyte.io", "resolvedDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "rssEmailByText": "By", "rssEmailClickThroughText": "Read more »", "rssEmailCommentText": "Comment »", "rssEmailEntryTemplateEnabled": false, "rssEmailImageMaxWidth": 0, "rssEmailUrl": "", "sections": {}, "securityState": "NONE", "selected": 0, "slug": "-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "smartEmailFields": {}, "state": "PUBLISHED", "stats": {"counters": {"sent": 0, "open": 0, "delivered": 0, "bounce": 0, "unsubscribed": 0, "click": 0, "reply": 0, "dropped": 1, "selected": 1, "spamreport": 0, "suppressed": 0, "hardbounced": 0, "softbounced": 0, "pending": 0, "contactslost": 0, "notsent": 1}, "deviceBreakdown": {"open_device_type": {"computer": 0, "mobile": 0, "unknown": 0}, "click_device_type": {"computer": 0, "mobile": 0, "unknown": 0}}, "failedToLoad": false, "qualifierStats": {}, "ratios": {"clickratio": 0, "clickthroughratio": 0, "deliveredratio": 0, "openratio": 0, "replyratio": 0, "unsubscribedratio": 0, "spamreportratio": 0, "bounceratio": 0, "hardbounceratio": 0, "softbounceratio": 0, "contactslostratio": 0, "pendingratio": 0, "notsentratio": 100.0}}, "styleSettings": {"background_color": "#EAF0F6", "background_image": null, "background_image_type": null, "body_border_color": "#EAF0F6", "body_border_color_choice": "BORDER_MANUAL", "body_border_width": "1", "body_color": "#ffffff", "color_picker_favorite1": null, "color_picker_favorite2": null, "color_picker_favorite3": null, "color_picker_favorite4": null, "color_picker_favorite5": null, "color_picker_favorite6": null, "email_body_padding": null, "email_body_width": null, "heading_one_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "28", "underline": null}, "heading_two_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "22", "underline": null}, "links_font": {"bold": false, "color": "#00a4bd", "font": null, "font_style": {}, "italic": false, "size": null, "underline": true}, "primary_accent_color": null, "primary_font": "Arial, sans-serif", "primary_font_color": "#23496d", "primary_font_line_height": null, "primary_font_size": "15", "secondary_accent_color": null, "secondary_font": "Arial, sans-serif", "secondary_font_color": "#23496d", "secondary_font_line_height": null, "secondary_font_size": "12", "use_email_client_default_settings": false, "user_module_defaults": {"button_email": {"background_color": "#00a4bd", "corner_radius": 8, "font": "Arial, sans-serif", "font_color": "#ffffff", "font_size": 16, "font_style": {"color": "#ffffff", "font": "Arial, sans-serif", "size": {"units": "px", "value": 16}, "styles": {"bold": false, "italic": false, "underline": false}}}, "email_divider": {"color": {"color": "#23496d", "opacity": 100}, "height": 1, "line_type": "solid"}}}, "subcategory": "batch", "subject": "test", "subscription": 23704464, "subscriptionName": "Test sub", "teamPerms": [], "templatePath": "@hubspot/email/dnd/welcome.html", "transactional": false, "translations": {}, "unpublishedAt": 0, "updated": 1675121702583, "updatedById": 12282590, "url": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-86812db1-e3c8-43cd-ae80-69a0934cd1de", "useRssHeadlineAsSubject": false, "userPerms": [], "vidsExcluded": [], "vidsIncluded": [2501], "visibleToAll": true}, "emitted_at": 1697714249852} +{"stream": "marketing_emails", "data": {"ab": false, "abHoursToWait": 4, "abSampleSizeDefault": null, "abSamplingDefault": null, "abSuccessMetric": null, "abTestPercentage": 50, "abVariation": false, "absoluteUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-f142cfbc-0d58-4eb5-b442-0d221f27b420", "aifeatures": null, "allEmailCampaignIds": [169919555], "analyticsPageId": "57347028995", "analyticsPageType": "email", "archivedAt": 0, "archivedInDashboard": false, "audienceAccess": "PUBLIC", "author": "integration-test@airbyte.io", "authorName": "Team-1 Airbyte", "blogRssSettings": null, "canSpamSettingsId": 36765207029, "categoryId": 2, "contentAccessRuleIds": [], "contentAccessRuleTypes": [], "contentTypeCategory": 2, "createPage": false, "created": 1634050240841, "createdById": 12282590, "currentState": "PUBLISHED", "currentlyPublished": true, "customReplyTo": "", "customReplyToEnabled": false, "domain": "", "emailBody": "{% content_attribute \"email_body\" %}{{ default_email_body }}{% end_content_attribute %}", "emailNote": "", "emailTemplateMode": "DRAG_AND_DROP", "emailType": "BATCH_EMAIL", "emailbodyPlaintext": "", "feedbackSurveyId": null, "flexAreas": {"main": {"boxed": false, "isSingleColumnFullWidth": false, "sections": [{"columns": [{"id": "column-0-0", "widgets": ["module-0-0-0"], "width": 12}], "id": "section-0", "style": {"backgroundType": "CONTENT", "paddingBottom": "40px", "paddingTop": "40px"}}, {"columns": [{"id": "column-1-0", "widgets": ["module-1-0-0"], "width": 12}], "id": "section-1", "style": {"backgroundColor": "", "backgroundType": "CONTENT", "paddingBottom": "0px", "paddingTop": "0px"}}]}}, "freezeDate": 1634050421336, "fromName": "Team Airbyte", "hasContentAccessRules": false, "htmlTitle": "", "id": 57347028995, "isCreatedFomSandboxSync": false, "isGraymailSuppressionEnabled": true, "isInstanceLayoutPage": false, "isPublished": true, "isRecipientFatigueSuppressionEnabled": null, "language": "en", "layoutSections": {}, "liveDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "mailingListsExcluded": [], "mailingListsIncluded": [130, 129, 131, 128, 126, 127, 125, 124, 123, 122, 121, 120, 119, 118, 117, 116], "maxRssEntries": 5, "metaDescription": "", "name": "First test email - 1", "pageExpiryEnabled": false, "pageRedirected": false, "pastMabExperimentIds": [], "portalId": 8727216, "previewKey": "bgNuSvDn", "primaryEmailCampaignId": 169919555, "processingStatus": "PUBLISHED", "publishDate": 1634050421341, "publishImmediately": true, "publishedAt": 1634050421580, "publishedByEmail": "integration-test@airbyte.io", "publishedById": 12282590, "publishedByName": "Team-1 Airbyte", "publishedUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-f142cfbc-0d58-4eb5-b442-0d221f27b420", "replyTo": "integration-test@airbyte.io", "resolvedDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "rssEmailByText": "By", "rssEmailClickThroughText": "Read more »", "rssEmailCommentText": "Comment »", "rssEmailEntryTemplateEnabled": false, "rssEmailImageMaxWidth": 0, "rssEmailUrl": "", "sections": {}, "securityState": "NONE", "selected": 0, "slug": "-temporary-slug-f142cfbc-0d58-4eb5-b442-0d221f27b420", "smartEmailFields": {}, "state": "PUBLISHED", "stats": {"counters": {"sent": 0}, "deviceBreakdown": {}, "failedToLoad": false, "qualifierStats": {}, "ratios": {"clickratio": 0, "clickthroughratio": 0, "deliveredratio": 0, "openratio": 0, "replyratio": 0, "unsubscribedratio": 0, "spamreportratio": 0, "bounceratio": 0, "hardbounceratio": 0, "softbounceratio": 0, "contactslostratio": 0, "pendingratio": 0, "notsentratio": 0}}, "styleSettings": {"background_color": "#ffffff", "background_image": null, "background_image_type": null, "body_border_color": null, "body_border_color_choice": null, "body_border_width": "1", "body_color": "#ffffff", "color_picker_favorite1": null, "color_picker_favorite2": null, "color_picker_favorite3": null, "color_picker_favorite4": null, "color_picker_favorite5": null, "color_picker_favorite6": null, "email_body_padding": null, "email_body_width": null, "heading_one_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "28", "underline": null}, "heading_two_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "22", "underline": null}, "links_font": {"bold": false, "color": "#00a4bd", "font": null, "font_style": {}, "italic": false, "size": null, "underline": true}, "primary_accent_color": null, "primary_font": "Arial, sans-serif", "primary_font_color": "#23496d", "primary_font_line_height": null, "primary_font_size": "15", "secondary_accent_color": null, "secondary_font": "Arial, sans-serif", "secondary_font_color": "#23496d", "secondary_font_line_height": null, "secondary_font_size": "12", "use_email_client_default_settings": false, "user_module_defaults": {"button_email": {"background_color": null, "corner_radius": 8, "font": "Arial, sans-serif", "font_color": "#ffffff", "font_size": 16, "font_style": {"color": "#ffffff", "font": "Arial, sans-serif", "size": {"units": "px", "value": 16}, "styles": {"bold": false, "italic": false, "underline": false}}}, "email_divider": {"color": {"color": "#000000", "opacity": 100}, "height": 1, "line_type": null}}}, "subcategory": "batch", "subject": "Subject l", "subscription": 23704464, "subscriptionName": "Test sub", "teamPerms": [], "templatePath": "@hubspot/email/dnd/plain_text.html", "transactional": false, "translations": {}, "unpublishedAt": 0, "updated": 1634050455543, "updatedById": 12282590, "url": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-f142cfbc-0d58-4eb5-b442-0d221f27b420", "useRssHeadlineAsSubject": false, "userPerms": [], "vidsExcluded": [], "vidsIncluded": [], "visibleToAll": true}, "emitted_at": 1697714249853} +{"stream": "marketing_emails", "data": {"ab": false, "abHoursToWait": 4, "abSampleSizeDefault": null, "abSamplingDefault": null, "abSuccessMetric": null, "abTestPercentage": 50, "abVariation": false, "absoluteUrl": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-fb53d6bf-1eb6-4ee6-90fe-610fc2569ea7", "aifeatures": null, "allEmailCampaignIds": [], "analyticsPageId": "42930862366", "analyticsPageType": "email", "archivedAt": 0, "archivedInDashboard": false, "audienceAccess": "PUBLIC", "author": "integration-test@airbyte.io", "authorName": "Team-1 Airbyte", "blogRssSettings": null, "canSpamSettingsId": 36765207029, "categoryId": 2, "clonedFrom": 41886608509, "contentAccessRuleIds": [], "contentAccessRuleTypes": [], "contentTypeCategory": 2, "createPage": false, "created": 1615502115346, "createdById": 100, "currentState": "AUTOMATED_DRAFT", "currentlyPublished": false, "customReplyTo": "", "customReplyToEnabled": false, "domain": "", "emailBody": "{% content_attribute \"email_body\" %}{{ default_email_body }}{% end_content_attribute %}", "emailNote": "", "emailTemplateMode": "DRAG_AND_DROP", "emailType": "AUTOMATED_EMAIL", "emailbodyPlaintext": "", "feedbackSurveyId": null, "flexAreas": {"main": {"boxed": false, "isSingleColumnFullWidth": false, "sections": [{"columns": [{"id": "column-0-1", "widgets": ["module-0-1-1"], "width": 12}], "id": "section-0", "style": {"backgroundColor": "#eaf0f6", "backgroundType": "CONTENT", "paddingBottom": "10px", "paddingTop": "10px"}}, {"columns": [{"id": "column-1-1", "widgets": ["module-1-1-1"], "width": 12}], "id": "section-1", "style": {"backgroundType": "CONTENT", "paddingBottom": "30px", "paddingTop": "30px"}}, {"columns": [{"id": "column-2-1", "widgets": ["module-2-1-1"], "width": 12}], "id": "section-2", "style": {"backgroundColor": "", "backgroundType": "CONTENT", "paddingBottom": "20px", "paddingTop": "20px"}}]}}, "freezeDate": 1634042970319, "fromName": "Team Airbyte", "hasContentAccessRules": false, "htmlTitle": "", "id": 42930862366, "isCreatedFomSandboxSync": false, "isGraymailSuppressionEnabled": false, "isInstanceLayoutPage": false, "isPublished": false, "isRecipientFatigueSuppressionEnabled": null, "language": "en", "lastEditSessionId": 1634042969643, "lastEditUpdateId": 0, "layoutSections": {}, "liveDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "mailingListsExcluded": [], "mailingListsIncluded": [], "maxRssEntries": 5, "metaDescription": "", "name": "Test subject (Test campaing - Clone)", "pageExpiryEnabled": false, "pageRedirected": false, "pastMabExperimentIds": [], "portalId": 8727216, "previewKey": "UmZGYZsU", "processingStatus": "UNDEFINED", "publishDate": 1634042970320, "publishImmediately": true, "publishedUrl": "", "replyTo": "integration-test@airbyte.io", "resolvedDomain": "integrationtest-dev-8727216-8727216.hs-sites.com", "rssEmailByText": "By", "rssEmailClickThroughText": "Read more »", "rssEmailCommentText": "Comment »", "rssEmailEntryTemplateEnabled": false, "rssEmailImageMaxWidth": 0, "rssEmailUrl": "", "sections": {}, "securityState": "NONE", "slug": "-temporary-slug-fb53d6bf-1eb6-4ee6-90fe-610fc2569ea7", "smartEmailFields": {}, "state": "AUTOMATED_DRAFT", "styleSettings": {"background_color": "#EAF0F6", "background_image": null, "background_image_type": null, "body_border_color": "#EAF0F6", "body_border_color_choice": "BORDER_MANUAL", "body_border_width": "1", "body_color": "#ffffff", "color_picker_favorite1": null, "color_picker_favorite2": null, "color_picker_favorite3": null, "color_picker_favorite4": null, "color_picker_favorite5": null, "color_picker_favorite6": null, "email_body_padding": null, "email_body_width": null, "heading_one_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "28", "underline": null}, "heading_two_font": {"bold": null, "color": null, "font": null, "font_style": {}, "italic": null, "size": "22", "underline": null}, "links_font": {"bold": false, "color": "#00a4bd", "font": null, "font_style": {}, "italic": false, "size": null, "underline": true}, "primary_accent_color": null, "primary_font": "Arial, sans-serif", "primary_font_color": "#23496d", "primary_font_line_height": null, "primary_font_size": "15", "secondary_accent_color": null, "secondary_font": "Arial, sans-serif", "secondary_font_color": "#23496d", "secondary_font_line_height": null, "secondary_font_size": "12", "use_email_client_default_settings": false, "user_module_defaults": {"button_email": {"background_color": "#00a4bd", "corner_radius": 8, "font": "Arial, sans-serif", "font_color": "#ffffff", "font_size": 16, "font_style": {"color": "#ffffff", "font": "Arial, sans-serif", "size": {"units": "px", "value": 16}, "styles": {"bold": false, "italic": false, "underline": false}}}, "email_divider": {"color": {"color": "#23496d", "opacity": 100}, "height": 1, "line_type": "solid"}}}, "subcategory": "automated", "subject": "Test subject", "subscription": 11890831, "subscriptionName": "Test subscription", "teamPerms": [], "templatePath": "@hubspot/email/dnd/welcome.html", "transactional": false, "translations": {}, "unpublishedAt": 0, "updated": 1634042970321, "updatedById": 12282590, "url": "http://integrationtest-dev-8727216-8727216.hs-sites.com/-temporary-slug-fb53d6bf-1eb6-4ee6-90fe-610fc2569ea7", "useRssHeadlineAsSubject": false, "userPerms": [], "vidsExcluded": [], "vidsIncluded": [], "visibleToAll": true}, "emitted_at": 1697714249854} +{"stream": "owners", "data": {"id": "52550153", "email": "integration-test@airbyte.io", "firstName": "Team-1", "lastName": "Airbyte", "userId": 12282590, "createdAt": "2020-10-28T21:17:56.082Z", "updatedAt": "2023-01-31T00:25:34.448Z", "archived": false}, "emitted_at": 1697714250730} +{"stream": "owners", "data": {"id": "65568071", "email": "test-integration-test-user1@airbyte.io", "firstName": "", "lastName": "", "userId": 23660227, "createdAt": "2021-03-15T11:00:50.053Z", "updatedAt": "2021-03-15T11:00:50.053Z", "archived": false}, "emitted_at": 1697714250731} +{"stream": "owners", "data": {"id": "65568800", "email": "test-integration-test-user2@airbyte.io", "firstName": "", "lastName": "", "userId": 23660229, "createdAt": "2021-03-15T11:01:02.183Z", "updatedAt": "2021-03-15T11:01:02.183Z", "archived": false}, "emitted_at": 1697714250732} +{"stream": "products", "data": {"id": "646176421", "properties": {"amount": null, "createdate": "2021-02-23T20:03:18.336000+00:00", "description": null, "discount": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_avatar_filemanager_key": null, "hs_cost_of_goods_sold": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_folder_id": null, "hs_folder_name": null, "hs_images": null, "hs_lastmodifieddate": "2021-02-23T20:03:18.336000+00:00", "hs_merged_object_ids": null, "hs_object_id": 646176421, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_sku": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Test product", "price": 100, "quantity": null, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:03:18.336Z", "updatedAt": "2021-02-23T20:03:18.336Z", "archived": false, "properties_amount": null, "properties_createdate": "2021-02-23T20:03:18.336000+00:00", "properties_description": null, "properties_discount": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_avatar_filemanager_key": null, "properties_hs_cost_of_goods_sold": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_folder_id": null, "properties_hs_folder_name": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-02-23T20:03:18.336000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 646176421, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_sku": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Test product", "properties_price": 100, "properties_quantity": null, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1697714252635} +{"stream": "products", "data": {"id": "646176423", "properties": {"amount": null, "createdate": "2021-02-23T20:03:48.577000+00:00", "description": null, "discount": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_avatar_filemanager_key": null, "hs_cost_of_goods_sold": null, "hs_created_by_user_id": 12282590, "hs_createdate": null, "hs_discount_percentage": null, "hs_folder_id": 2430008, "hs_folder_name": "test folder", "hs_images": null, "hs_lastmodifieddate": "2021-02-23T20:03:48.577000+00:00", "hs_merged_object_ids": null, "hs_object_id": 646176423, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_sku": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Test product 1", "price": 123, "quantity": null, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:03:48.577Z", "updatedAt": "2021-02-23T20:03:48.577Z", "archived": false, "properties_amount": null, "properties_createdate": "2021-02-23T20:03:48.577000+00:00", "properties_description": null, "properties_discount": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_avatar_filemanager_key": null, "properties_hs_cost_of_goods_sold": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_folder_id": 2430008, "properties_hs_folder_name": "test folder", "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-02-23T20:03:48.577000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 646176423, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_sku": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Test product 1", "properties_price": 123, "properties_quantity": null, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1697714252637} +{"stream": "products", "data": {"id": "646316535", "properties": {"amount": null, "createdate": "2021-02-23T20:11:54.030000+00:00", "description": "baseball hat, large", "discount": null, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_avatar_filemanager_key": null, "hs_cost_of_goods_sold": 5, "hs_created_by_user_id": null, "hs_createdate": null, "hs_discount_percentage": null, "hs_folder_id": null, "hs_folder_name": null, "hs_images": null, "hs_lastmodifieddate": "2021-02-23T20:11:54.030000+00:00", "hs_merged_object_ids": null, "hs_object_id": 646316535, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_product_type": null, "hs_read_only": null, "hs_recurring_billing_period": null, "hs_recurring_billing_start_date": null, "hs_sku": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_url": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": true, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "name": "Green Hat", "price": 10, "quantity": null, "recurringbillingfrequency": null, "tax": null, "test": null, "test_product_price": null}, "createdAt": "2021-02-23T20:11:54.030Z", "updatedAt": "2021-02-23T20:11:54.030Z", "archived": false, "properties_amount": null, "properties_createdate": "2021-02-23T20:11:54.030000+00:00", "properties_description": "baseball hat, large", "properties_discount": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_avatar_filemanager_key": null, "properties_hs_cost_of_goods_sold": 5, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_discount_percentage": null, "properties_hs_folder_id": null, "properties_hs_folder_name": null, "properties_hs_images": null, "properties_hs_lastmodifieddate": "2021-02-23T20:11:54.030000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 646316535, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_product_type": null, "properties_hs_read_only": null, "properties_hs_recurring_billing_period": null, "properties_hs_recurring_billing_start_date": null, "properties_hs_sku": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_url": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": true, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_name": "Green Hat", "properties_price": 10, "properties_quantity": null, "properties_recurringbillingfrequency": null, "properties_tax": null, "properties_test": null, "properties_test_product_price": null}, "emitted_at": 1697714252638} +{"stream": "contacts_property_history", "data": {"value": "testo", "source-type": "CRM_UI", "source-id": "userId:12282590", "source-label": null, "updated-by-user-id": 12282590, "timestamp": 1700681340515, "selected": false, "property": "firstname", "vid": 2501, "portal-id": 8727216, "is-contact": true, "canonical-vid": 2501}, "emitted_at": 1701905506064} +{"stream": "contacts_property_history", "data": {"value": "test", "source-type": "CRM_UI", "source-id": "userId:12282590", "source-label": null, "updated-by-user-id": 12282590, "timestamp": 1675120629904, "selected": false, "property": "firstname", "vid": 2501, "portal-id": 8727216, "is-contact": true, "canonical-vid": 2501}, "emitted_at": 1701905506064} +{"stream": "companies_property_history", "data": {"name": "hs_analytics_latest_source_data_2", "value": "CRM_UI", "timestamp": 1657222285656, "sourceId": "RollupProperties", "source": "MIGRATION", "sourceVid": [], "property": "hs_analytics_latest_source_data_2", "companyId": 5000526215, "portalId": 8727216, "isDeleted": false}, "emitted_at": 1701905731242} +{"stream": "companies_property_history", "data": {"name": "hs_analytics_latest_source_data_1", "value": "CONTACTS", "timestamp": 1657222285656, "sourceId": "RollupProperties", "source": "MIGRATION", "sourceVid": [], "property": "hs_analytics_latest_source_data_1", "companyId": 5000526215, "portalId": 8727216, "isDeleted": false}, "emitted_at": 1701905731242} +{"stream": "deals_property_history", "data": {"name": "dealname", "value": "Test deal 2", "timestamp": 1614111692862, "sourceId": "userId:12282590", "source": "CRM_UI", "sourceVid": [], "requestId": "1ce13074-883d-4d9c-9d07-e01e8f23f363", "updatedByUserId": 12282590, "property": "dealname", "dealId": 4315375411, "portalId": 8727216, "isDeleted": false}, "emitted_at": 1701905810513} +{"stream": "subscription_changes", "data": {"timestamp": 1616173134301, "portalId": 8727216, "recipient": "0c90ecf5-629e-4fe4-8516-05f75636c3e3@gdpr-forgotten.hubspot.com", "normalizedEmailId": "0c90ecf5-629e-4fe4-8516-05f75636c3e3", "changes": [{"source": "SOURCE_HUBSPOT_CUSTOMER", "timestamp": 1616173134301, "portalId": 8727216, "causedByEvent": {"id": "d70b78b9-a411-4d3e-808b-fe931be35b43", "created": 1616173134301}, "changeType": "PORTAL_STATUS", "change": "SUBSCRIBED"}]}, "emitted_at": 1697714255435} +{"stream": "subscription_changes", "data": {"timestamp": 1616173134301, "portalId": 8727216, "recipient": "0c90ecf5-629e-4fe4-8516-05f75636c3e3@gdpr-forgotten.hubspot.com", "normalizedEmailId": "0c90ecf5-629e-4fe4-8516-05f75636c3e3", "changes": [{"source": "SOURCE_HUBSPOT_CUSTOMER", "timestamp": 1616173134301, "subscriptionId": 10798197, "portalId": 8727216, "causedByEvent": {"id": "ff118718-786d-4a35-94f9-6bbd413654de", "created": 1616173134301}, "changeType": "SUBSCRIPTION_STATUS", "change": "SUBSCRIBED"}]}, "emitted_at": 1697714255436} +{"stream": "subscription_changes", "data": {"timestamp": 1616173106737, "portalId": 8727216, "recipient": "0c90ecf5-629e-4fe4-8516-05f75636c3e3@gdpr-forgotten.hubspot.com", "normalizedEmailId": "0c90ecf5-629e-4fe4-8516-05f75636c3e3", "changes": [{"source": "SOURCE_HUBSPOT_CUSTOMER", "timestamp": 1616173106737, "portalId": 8727216, "causedByEvent": {"id": "24539f1f-0b20-4296-a5bf-6ba3bb9dc1b8", "created": 1616173106737}, "changeType": "PORTAL_STATUS", "change": "SUBSCRIBED"}]}, "emitted_at": 1697714255437} +{"stream": "tickets", "data": {"id": "312929579", "properties": {"closed_date": "2021-02-23T20:08:49.603000+00:00", "content": null, "created_by": null, "createdate": "2021-02-23T20:08:49.603000+00:00", "first_agent_reply_date": null, "hs_all_accessible_team_ids": null, "hs_all_associated_contact_companies": null, "hs_all_associated_contact_emails": null, "hs_all_associated_contact_firstnames": null, "hs_all_associated_contact_lastnames": null, "hs_all_associated_contact_mobilephones": null, "hs_all_associated_contact_phones": null, "hs_all_conversation_mentions": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignment_method": null, "hs_auto_generated_from_thread_id": null, "hs_conversations_originating_message_id": null, "hs_conversations_originating_thread_id": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_custom_inbox": null, "hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_2": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_3": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_4": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_2": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_3": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_4": null, "hs_external_object_ids": null, "hs_feedback_last_ces_follow_up": null, "hs_feedback_last_ces_rating": null, "hs_feedback_last_survey_date": null, "hs_file_upload": null, "hs_first_agent_message_sent_at": null, "hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "hs_in_helpdesk": null, "hs_inbox_id": null, "hs_is_visible_in_help_desk": null, "hs_last_email_activity": null, "hs_last_email_date": null, "hs_last_message_from_visitor": false, "hs_last_message_received_at": null, "hs_last_message_sent_at": null, "hs_lastactivitydate": null, "hs_lastcontacted": null, "hs_lastmodifieddate": "2021-02-23T20:08:53.371000+00:00", "hs_latest_message_seen_by_agent_ids": null, "hs_merged_object_ids": null, "hs_most_relevant_sla_status": null, "hs_most_relevant_sla_type": null, "hs_msteams_message_id": null, "hs_nextactivitydate": null, "hs_num_associated_companies": 0, "hs_num_associated_conversations": null, "hs_num_times_contacted": null, "hs_object_id": 312929579, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_originating_channel_instance_id": null, "hs_originating_email_engagement_id": null, "hs_originating_generic_channel_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "0", "hs_pipeline_stage": "4", "hs_primary_company": null, "hs_primary_company_id": null, "hs_primary_company_name": null, "hs_read_only": null, "hs_resolution": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_thread_ids_to_restore": null, "hs_ticket_category": null, "hs_ticket_id": 312929579, "hs_ticket_priority": "LOW", "hs_time_in_1": 0, "hs_time_in_2": 0, "hs_time_in_3": 0, "hs_time_in_4": 87748604133, "hs_time_to_close_sla_at": null, "hs_time_to_close_sla_status": null, "hs_time_to_first_response_sla_at": null, "hs_time_to_first_response_sla_status": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": true, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "last_engagement_date": null, "last_reply_date": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "nps_follow_up_answer": null, "nps_follow_up_question_version": null, "nps_score": null, "num_contacted_notes": null, "num_notes": null, "source_ref": null, "source_thread_id": null, "source_type": "CHAT", "subject": "Marketing Starter", "tags": null, "time_to_close": 0, "time_to_first_agent_reply": null}, "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:53.371Z", "archived": false, "properties_closed_date": "2021-02-23T20:08:49.603000+00:00", "properties_content": null, "properties_created_by": null, "properties_createdate": "2021-02-23T20:08:49.603000+00:00", "properties_first_agent_reply_date": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_associated_contact_companies": null, "properties_hs_all_associated_contact_emails": null, "properties_hs_all_associated_contact_firstnames": null, "properties_hs_all_associated_contact_lastnames": null, "properties_hs_all_associated_contact_mobilephones": null, "properties_hs_all_associated_contact_phones": null, "properties_hs_all_conversation_mentions": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignment_method": null, "properties_hs_auto_generated_from_thread_id": null, "properties_hs_conversations_originating_message_id": null, "properties_hs_conversations_originating_thread_id": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_custom_inbox": null, "properties_hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_2": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_3": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_4": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_2": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_3": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_4": null, "properties_hs_external_object_ids": null, "properties_hs_feedback_last_ces_follow_up": null, "properties_hs_feedback_last_ces_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_file_upload": null, "properties_hs_first_agent_message_sent_at": null, "properties_hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "properties_hs_in_helpdesk": null, "properties_hs_inbox_id": null, "properties_hs_is_visible_in_help_desk": null, "properties_hs_last_email_activity": null, "properties_hs_last_email_date": null, "properties_hs_last_message_from_visitor": false, "properties_hs_last_message_received_at": null, "properties_hs_last_message_sent_at": null, "properties_hs_lastactivitydate": null, "properties_hs_lastcontacted": null, "properties_hs_lastmodifieddate": "2021-02-23T20:08:53.371000+00:00", "properties_hs_latest_message_seen_by_agent_ids": null, "properties_hs_merged_object_ids": null, "properties_hs_most_relevant_sla_status": null, "properties_hs_most_relevant_sla_type": null, "properties_hs_msteams_message_id": null, "properties_hs_nextactivitydate": null, "properties_hs_num_associated_companies": 0, "properties_hs_num_associated_conversations": null, "properties_hs_num_times_contacted": null, "properties_hs_object_id": 312929579, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_originating_channel_instance_id": null, "properties_hs_originating_email_engagement_id": null, "properties_hs_originating_generic_channel_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "0", "properties_hs_pipeline_stage": "4", "properties_hs_primary_company": null, "properties_hs_primary_company_id": null, "properties_hs_primary_company_name": null, "properties_hs_read_only": null, "properties_hs_resolution": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_thread_ids_to_restore": null, "properties_hs_ticket_category": null, "properties_hs_ticket_id": 312929579, "properties_hs_ticket_priority": "LOW", "properties_hs_time_in_1": 0, "properties_hs_time_in_2": 0, "properties_hs_time_in_3": 0, "properties_hs_time_in_4": 87748604133, "properties_hs_time_to_close_sla_at": null, "properties_hs_time_to_close_sla_status": null, "properties_hs_time_to_first_response_sla_at": null, "properties_hs_time_to_first_response_sla_status": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": true, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_last_engagement_date": null, "properties_last_reply_date": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_nps_follow_up_answer": null, "properties_nps_follow_up_question_version": null, "properties_nps_score": null, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_source_ref": null, "properties_source_thread_id": null, "properties_source_type": "CHAT", "properties_subject": "Marketing Starter", "properties_tags": null, "properties_time_to_close": 0, "properties_time_to_first_agent_reply": null}, "emitted_at": 1701859534671} +{"stream": "tickets", "data": {"id": "312972611", "properties": {"closed_date": null, "content": null, "created_by": null, "createdate": "2021-02-23T20:08:49.603000+00:00", "first_agent_reply_date": null, "hs_all_accessible_team_ids": null, "hs_all_associated_contact_companies": null, "hs_all_associated_contact_emails": null, "hs_all_associated_contact_firstnames": null, "hs_all_associated_contact_lastnames": null, "hs_all_associated_contact_mobilephones": null, "hs_all_associated_contact_phones": null, "hs_all_conversation_mentions": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignment_method": null, "hs_auto_generated_from_thread_id": null, "hs_conversations_originating_message_id": null, "hs_conversations_originating_thread_id": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_custom_inbox": null, "hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_2": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_3": null, "hs_date_entered_4": null, "hs_date_exited_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_exited_2": null, "hs_date_exited_3": null, "hs_date_exited_4": null, "hs_external_object_ids": null, "hs_feedback_last_ces_follow_up": null, "hs_feedback_last_ces_rating": null, "hs_feedback_last_survey_date": null, "hs_file_upload": null, "hs_first_agent_message_sent_at": null, "hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "hs_in_helpdesk": null, "hs_inbox_id": null, "hs_is_visible_in_help_desk": null, "hs_last_email_activity": null, "hs_last_email_date": null, "hs_last_message_from_visitor": false, "hs_last_message_received_at": null, "hs_last_message_sent_at": null, "hs_lastactivitydate": null, "hs_lastcontacted": null, "hs_lastmodifieddate": "2021-02-23T20:08:52.663000+00:00", "hs_latest_message_seen_by_agent_ids": null, "hs_merged_object_ids": null, "hs_most_relevant_sla_status": null, "hs_most_relevant_sla_type": null, "hs_msteams_message_id": null, "hs_nextactivitydate": null, "hs_num_associated_companies": 0, "hs_num_associated_conversations": null, "hs_num_times_contacted": null, "hs_object_id": 312972611, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_originating_channel_instance_id": null, "hs_originating_email_engagement_id": null, "hs_originating_generic_channel_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "0", "hs_pipeline_stage": "2", "hs_primary_company": null, "hs_primary_company_id": null, "hs_primary_company_name": null, "hs_read_only": null, "hs_resolution": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_thread_ids_to_restore": null, "hs_ticket_category": null, "hs_ticket_id": 312972611, "hs_ticket_priority": "LOW", "hs_time_in_1": 0, "hs_time_in_2": 87748604132, "hs_time_in_3": null, "hs_time_in_4": null, "hs_time_to_close_sla_at": null, "hs_time_to_close_sla_status": null, "hs_time_to_first_response_sla_at": null, "hs_time_to_first_response_sla_status": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": true, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "last_engagement_date": null, "last_reply_date": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "nps_follow_up_answer": null, "nps_follow_up_question_version": null, "nps_score": null, "num_contacted_notes": null, "num_notes": null, "source_ref": null, "source_thread_id": null, "source_type": "FORM", "subject": "Sales Starter", "tags": null, "time_to_close": null, "time_to_first_agent_reply": null}, "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:52.663Z", "archived": false, "properties_closed_date": null, "properties_content": null, "properties_created_by": null, "properties_createdate": "2021-02-23T20:08:49.603000+00:00", "properties_first_agent_reply_date": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_associated_contact_companies": null, "properties_hs_all_associated_contact_emails": null, "properties_hs_all_associated_contact_firstnames": null, "properties_hs_all_associated_contact_lastnames": null, "properties_hs_all_associated_contact_mobilephones": null, "properties_hs_all_associated_contact_phones": null, "properties_hs_all_conversation_mentions": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignment_method": null, "properties_hs_auto_generated_from_thread_id": null, "properties_hs_conversations_originating_message_id": null, "properties_hs_conversations_originating_thread_id": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_custom_inbox": null, "properties_hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_2": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_3": null, "properties_hs_date_entered_4": null, "properties_hs_date_exited_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_exited_2": null, "properties_hs_date_exited_3": null, "properties_hs_date_exited_4": null, "properties_hs_external_object_ids": null, "properties_hs_feedback_last_ces_follow_up": null, "properties_hs_feedback_last_ces_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_file_upload": null, "properties_hs_first_agent_message_sent_at": null, "properties_hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "properties_hs_in_helpdesk": null, "properties_hs_inbox_id": null, "properties_hs_is_visible_in_help_desk": null, "properties_hs_last_email_activity": null, "properties_hs_last_email_date": null, "properties_hs_last_message_from_visitor": false, "properties_hs_last_message_received_at": null, "properties_hs_last_message_sent_at": null, "properties_hs_lastactivitydate": null, "properties_hs_lastcontacted": null, "properties_hs_lastmodifieddate": "2021-02-23T20:08:52.663000+00:00", "properties_hs_latest_message_seen_by_agent_ids": null, "properties_hs_merged_object_ids": null, "properties_hs_most_relevant_sla_status": null, "properties_hs_most_relevant_sla_type": null, "properties_hs_msteams_message_id": null, "properties_hs_nextactivitydate": null, "properties_hs_num_associated_companies": 0, "properties_hs_num_associated_conversations": null, "properties_hs_num_times_contacted": null, "properties_hs_object_id": 312972611, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_originating_channel_instance_id": null, "properties_hs_originating_email_engagement_id": null, "properties_hs_originating_generic_channel_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "0", "properties_hs_pipeline_stage": "2", "properties_hs_primary_company": null, "properties_hs_primary_company_id": null, "properties_hs_primary_company_name": null, "properties_hs_read_only": null, "properties_hs_resolution": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_thread_ids_to_restore": null, "properties_hs_ticket_category": null, "properties_hs_ticket_id": 312972611, "properties_hs_ticket_priority": "LOW", "properties_hs_time_in_1": 0, "properties_hs_time_in_2": 87748604132, "properties_hs_time_in_3": null, "properties_hs_time_in_4": null, "properties_hs_time_to_close_sla_at": null, "properties_hs_time_to_close_sla_status": null, "properties_hs_time_to_first_response_sla_at": null, "properties_hs_time_to_first_response_sla_status": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": true, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_last_engagement_date": null, "properties_last_reply_date": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_nps_follow_up_answer": null, "properties_nps_follow_up_question_version": null, "properties_nps_score": null, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_source_ref": null, "properties_source_thread_id": null, "properties_source_type": "FORM", "properties_subject": "Sales Starter", "properties_tags": null, "properties_time_to_close": null, "properties_time_to_first_agent_reply": null}, "emitted_at": 1701859534672} +{"stream": "tickets", "data": {"id": "312975112", "properties": {"closed_date": null, "content": null, "created_by": null, "createdate": "2021-02-23T20:08:49.603000+00:00", "first_agent_reply_date": null, "hs_all_accessible_team_ids": null, "hs_all_associated_contact_companies": null, "hs_all_associated_contact_emails": null, "hs_all_associated_contact_firstnames": null, "hs_all_associated_contact_lastnames": null, "hs_all_associated_contact_mobilephones": null, "hs_all_associated_contact_phones": null, "hs_all_conversation_mentions": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_assignment_method": null, "hs_auto_generated_from_thread_id": null, "hs_conversations_originating_message_id": null, "hs_conversations_originating_thread_id": null, "hs_created_by_user_id": null, "hs_createdate": null, "hs_custom_inbox": null, "hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "hs_date_entered_2": null, "hs_date_entered_3": null, "hs_date_entered_4": null, "hs_date_exited_1": null, "hs_date_exited_2": null, "hs_date_exited_3": null, "hs_date_exited_4": null, "hs_external_object_ids": null, "hs_feedback_last_ces_follow_up": null, "hs_feedback_last_ces_rating": null, "hs_feedback_last_survey_date": null, "hs_file_upload": null, "hs_first_agent_message_sent_at": null, "hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "hs_in_helpdesk": null, "hs_inbox_id": null, "hs_is_visible_in_help_desk": null, "hs_last_email_activity": null, "hs_last_email_date": null, "hs_last_message_from_visitor": false, "hs_last_message_received_at": null, "hs_last_message_sent_at": null, "hs_lastactivitydate": null, "hs_lastcontacted": null, "hs_lastmodifieddate": "2021-02-23T20:08:52.515000+00:00", "hs_latest_message_seen_by_agent_ids": null, "hs_merged_object_ids": null, "hs_most_relevant_sla_status": null, "hs_most_relevant_sla_type": null, "hs_msteams_message_id": null, "hs_nextactivitydate": null, "hs_num_associated_companies": 0, "hs_num_associated_conversations": null, "hs_num_times_contacted": null, "hs_object_id": 312975112, "hs_object_source": null, "hs_object_source_id": null, "hs_object_source_label": null, "hs_object_source_user_id": null, "hs_originating_channel_instance_id": null, "hs_originating_email_engagement_id": null, "hs_originating_generic_channel_id": null, "hs_pinned_engagement_id": null, "hs_pipeline": "0", "hs_pipeline_stage": "1", "hs_primary_company": null, "hs_primary_company_id": null, "hs_primary_company_name": null, "hs_read_only": null, "hs_resolution": null, "hs_sales_email_last_replied": null, "hs_tag_ids": null, "hs_thread_ids_to_restore": null, "hs_ticket_category": null, "hs_ticket_id": 312975112, "hs_ticket_priority": "MEDIUM", "hs_time_in_1": 87748604134, "hs_time_in_2": null, "hs_time_in_3": null, "hs_time_in_4": null, "hs_time_to_close_sla_at": null, "hs_time_to_close_sla_status": null, "hs_time_to_first_response_sla_at": null, "hs_time_to_first_response_sla_status": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": null, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": true, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "last_engagement_date": null, "last_reply_date": null, "notes_last_contacted": null, "notes_last_updated": null, "notes_next_activity_date": null, "nps_follow_up_answer": null, "nps_follow_up_question_version": null, "nps_score": null, "num_contacted_notes": null, "num_notes": null, "source_ref": null, "source_thread_id": null, "source_type": "PHONE", "subject": "Free CRM", "tags": null, "time_to_close": null, "time_to_first_agent_reply": null}, "createdAt": "2021-02-23T20:08:49.603Z", "updatedAt": "2021-02-23T20:08:52.515Z", "archived": false, "properties_closed_date": null, "properties_content": null, "properties_created_by": null, "properties_createdate": "2021-02-23T20:08:49.603000+00:00", "properties_first_agent_reply_date": null, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_associated_contact_companies": null, "properties_hs_all_associated_contact_emails": null, "properties_hs_all_associated_contact_firstnames": null, "properties_hs_all_associated_contact_lastnames": null, "properties_hs_all_associated_contact_mobilephones": null, "properties_hs_all_associated_contact_phones": null, "properties_hs_all_conversation_mentions": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_assignment_method": null, "properties_hs_auto_generated_from_thread_id": null, "properties_hs_conversations_originating_message_id": null, "properties_hs_conversations_originating_thread_id": null, "properties_hs_created_by_user_id": null, "properties_hs_createdate": null, "properties_hs_custom_inbox": null, "properties_hs_date_entered_1": "2021-02-23T20:08:49.603000+00:00", "properties_hs_date_entered_2": null, "properties_hs_date_entered_3": null, "properties_hs_date_entered_4": null, "properties_hs_date_exited_1": null, "properties_hs_date_exited_2": null, "properties_hs_date_exited_3": null, "properties_hs_date_exited_4": null, "properties_hs_external_object_ids": null, "properties_hs_feedback_last_ces_follow_up": null, "properties_hs_feedback_last_ces_rating": null, "properties_hs_feedback_last_survey_date": null, "properties_hs_file_upload": null, "properties_hs_first_agent_message_sent_at": null, "properties_hs_helpdesk_sort_timestamp": "2021-02-23T20:08:49.603000+00:00", "properties_hs_in_helpdesk": null, "properties_hs_inbox_id": null, "properties_hs_is_visible_in_help_desk": null, "properties_hs_last_email_activity": null, "properties_hs_last_email_date": null, "properties_hs_last_message_from_visitor": false, "properties_hs_last_message_received_at": null, "properties_hs_last_message_sent_at": null, "properties_hs_lastactivitydate": null, "properties_hs_lastcontacted": null, "properties_hs_lastmodifieddate": "2021-02-23T20:08:52.515000+00:00", "properties_hs_latest_message_seen_by_agent_ids": null, "properties_hs_merged_object_ids": null, "properties_hs_most_relevant_sla_status": null, "properties_hs_most_relevant_sla_type": null, "properties_hs_msteams_message_id": null, "properties_hs_nextactivitydate": null, "properties_hs_num_associated_companies": 0, "properties_hs_num_associated_conversations": null, "properties_hs_num_times_contacted": null, "properties_hs_object_id": 312975112, "properties_hs_object_source": null, "properties_hs_object_source_id": null, "properties_hs_object_source_label": null, "properties_hs_object_source_user_id": null, "properties_hs_originating_channel_instance_id": null, "properties_hs_originating_email_engagement_id": null, "properties_hs_originating_generic_channel_id": null, "properties_hs_pinned_engagement_id": null, "properties_hs_pipeline": "0", "properties_hs_pipeline_stage": "1", "properties_hs_primary_company": null, "properties_hs_primary_company_id": null, "properties_hs_primary_company_name": null, "properties_hs_read_only": null, "properties_hs_resolution": null, "properties_hs_sales_email_last_replied": null, "properties_hs_tag_ids": null, "properties_hs_thread_ids_to_restore": null, "properties_hs_ticket_category": null, "properties_hs_ticket_id": 312975112, "properties_hs_ticket_priority": "MEDIUM", "properties_hs_time_in_1": 87748604134, "properties_hs_time_in_2": null, "properties_hs_time_in_3": null, "properties_hs_time_in_4": null, "properties_hs_time_to_close_sla_at": null, "properties_hs_time_to_close_sla_status": null, "properties_hs_time_to_first_response_sla_at": null, "properties_hs_time_to_first_response_sla_status": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": null, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": true, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_last_engagement_date": null, "properties_last_reply_date": null, "properties_notes_last_contacted": null, "properties_notes_last_updated": null, "properties_notes_next_activity_date": null, "properties_nps_follow_up_answer": null, "properties_nps_follow_up_question_version": null, "properties_nps_score": null, "properties_num_contacted_notes": null, "properties_num_notes": null, "properties_source_ref": null, "properties_source_thread_id": null, "properties_source_type": "PHONE", "properties_subject": "Free CRM", "properties_tags": null, "properties_time_to_close": null, "properties_time_to_first_agent_reply": null}, "emitted_at": 1701859534673} +{"stream": "workflows", "data": {"migrationStatus": {"portalId": 8727216, "workflowId": 21058115, "migrationStatus": "EXECUTION_MIGRATED", "enrollmentMigrationStatus": "PLATFORM_OWNED", "platformOwnsActions": true, "lastSuccessfulMigrationTimestamp": null, "enrollmentMigrationTimestamp": null, "flowId": 50206671}, "name": "Test Workflow", "id": 21058115, "type": "DRIP_DELAY", "enabled": false, "creationSource": {"sourceApplication": {"source": "DIRECT_API"}, "createdAt": 1610635826795}, "updateSource": {"sourceApplication": {"source": "DIRECT_API", "serviceName": "AutomationPlatformService-web_BackfillILSListIds"}, "updatedAt": 1611847907577}, "contactListIds": {"enrolled": 12, "active": 13, "completed": 14, "succeeded": 15}, "personaTagIds": [], "contactCounts": {"active": 0, "enrolled": 0}, "portalId": 8727216, "insertedAt": 1610635826921, "updatedAt": 1611847907577, "contactListIds_enrolled": 12, "contactListIds_active": 13, "contactListIds_completed": 14, "contactListIds_succeeded": 15}, "emitted_at": 1697714264418} +{"stream": "workflows", "data": {"migrationStatus": {"portalId": 8727216, "workflowId": 21058121, "migrationStatus": "EXECUTION_MIGRATED", "enrollmentMigrationStatus": "PLATFORM_OWNED", "platformOwnsActions": true, "lastSuccessfulMigrationTimestamp": null, "enrollmentMigrationTimestamp": null, "flowId": 50205684}, "name": "Test Workflow 1", "id": 21058121, "type": "DRIP_DELAY", "enabled": false, "creationSource": {"sourceApplication": {"source": "DIRECT_API"}, "createdAt": 1610635850713}, "updateSource": {"sourceApplication": {"source": "DIRECT_API", "serviceName": "AutomationPlatformService-web_BackfillILSListIds"}, "updatedAt": 1611847907579}, "contactListIds": {"enrolled": 16, "active": 17, "completed": 18, "succeeded": 19}, "personaTagIds": [], "contactCounts": {"active": 0, "enrolled": 0}, "portalId": 8727216, "insertedAt": 1610635850758, "updatedAt": 1611847907579, "contactListIds_enrolled": 16, "contactListIds_active": 17, "contactListIds_completed": 18, "contactListIds_succeeded": 19}, "emitted_at": 1697714264419} +{"stream": "workflows", "data": {"migrationStatus": {"portalId": 8727216, "workflowId": 21058122, "migrationStatus": "EXECUTION_MIGRATED", "enrollmentMigrationStatus": "PLATFORM_OWNED", "platformOwnsActions": true, "lastSuccessfulMigrationTimestamp": null, "enrollmentMigrationTimestamp": null, "flowId": 50205036}, "name": "Test Workflow 2", "id": 21058122, "type": "DRIP_DELAY", "enabled": false, "creationSource": {"sourceApplication": {"source": "DIRECT_API"}, "createdAt": 1610635859664}, "updateSource": {"sourceApplication": {"source": "DIRECT_API", "serviceName": "AutomationPlatformService-web_BackfillILSListIds"}, "updatedAt": 1611847907578}, "contactListIds": {"enrolled": 20, "active": 21, "completed": 22, "succeeded": 23}, "personaTagIds": [], "contactCounts": {"active": 0, "enrolled": 0}, "portalId": 8727216, "insertedAt": 1610635859748, "updatedAt": 1611847907578, "contactListIds_enrolled": 20, "contactListIds_active": 21, "contactListIds_completed": 22, "contactListIds_succeeded": 23}, "emitted_at": 1697714264420} +{"stream": "cars", "data": {"id": "5938880072", "properties": {"car_id": 1, "car_name": 3232324, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:57:15.836000+00:00", "hs_lastmodifieddate": "2023-04-12T17:59:20.189000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5938880072, "hs_object_source": "CRM_UI", "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-12T17:57:15.836Z", "updatedAt": "2023-04-12T17:59:20.189Z", "archived": false, "properties_car_id": 1, "properties_car_name": 3232324, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-12T17:57:15.836000+00:00", "properties_hs_lastmodifieddate": "2023-04-12T17:59:20.189000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 5938880072, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_pinned_engagement_id": null, "properties_hs_read_only": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null}, "emitted_at": 1703882548289} +{"stream": "cars", "data": {"id": "5938880073", "properties": {"car_id": 2, "car_name": 23232, "hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:57:20.583000+00:00", "hs_lastmodifieddate": "2023-04-12T17:59:20.189000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5938880073, "hs_object_source": "CRM_UI", "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null}, "createdAt": "2023-04-12T17:57:20.583Z", "updatedAt": "2023-04-12T17:59:20.189Z", "archived": false, "properties_car_id": 2, "properties_car_name": 23232, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-12T17:57:20.583000+00:00", "properties_hs_lastmodifieddate": "2023-04-12T17:59:20.189000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 5938880073, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_pinned_engagement_id": null, "properties_hs_read_only": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null}, "emitted_at": 1703882548293} +{"stream": "pets", "data": {"id": "5936415312", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:08:50.632000+00:00", "hs_lastmodifieddate": "2023-04-12T17:08:50.632000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5936415312, "hs_object_source": "CRM_UI", "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "pet_name": "Marcos Pet", "pet_type": "Dog"}, "createdAt": "2023-04-12T17:08:50.632Z", "updatedAt": "2023-04-12T17:08:50.632Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-12T17:08:50.632000+00:00", "properties_hs_lastmodifieddate": "2023-04-12T17:08:50.632000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 5936415312, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_pinned_engagement_id": null, "properties_hs_read_only": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_pet_name": "Marcos Pet", "properties_pet_type": "Dog"}, "emitted_at": 1703886126793} +{"stream": "pets", "data": {"id": "5938880054", "properties": {"hs_all_accessible_team_ids": null, "hs_all_assigned_business_unit_ids": null, "hs_all_owner_ids": null, "hs_all_team_ids": null, "hs_created_by_user_id": 12282590, "hs_createdate": "2023-04-12T17:53:12.692000+00:00", "hs_lastmodifieddate": "2023-04-12T17:53:12.692000+00:00", "hs_merged_object_ids": null, "hs_object_id": 5938880054, "hs_object_source": "CRM_UI", "hs_object_source_id": "userId:12282590", "hs_object_source_label": "CRM_UI", "hs_object_source_user_id": 12282590, "hs_pinned_engagement_id": null, "hs_read_only": null, "hs_unique_creation_key": null, "hs_updated_by_user_id": 12282590, "hs_user_ids_of_all_notification_followers": null, "hs_user_ids_of_all_notification_unfollowers": null, "hs_user_ids_of_all_owners": null, "hs_was_imported": null, "hubspot_owner_assigneddate": null, "hubspot_owner_id": null, "hubspot_team_id": null, "pet_name": "Integration Test Pet", "pet_type": "Unknown"}, "createdAt": "2023-04-12T17:53:12.692Z", "updatedAt": "2023-04-12T17:53:12.692Z", "archived": false, "properties_hs_all_accessible_team_ids": null, "properties_hs_all_assigned_business_unit_ids": null, "properties_hs_all_owner_ids": null, "properties_hs_all_team_ids": null, "properties_hs_created_by_user_id": 12282590, "properties_hs_createdate": "2023-04-12T17:53:12.692000+00:00", "properties_hs_lastmodifieddate": "2023-04-12T17:53:12.692000+00:00", "properties_hs_merged_object_ids": null, "properties_hs_object_id": 5938880054, "properties_hs_object_source": "CRM_UI", "properties_hs_object_source_id": "userId:12282590", "properties_hs_object_source_label": "CRM_UI", "properties_hs_object_source_user_id": 12282590, "properties_hs_pinned_engagement_id": null, "properties_hs_read_only": null, "properties_hs_unique_creation_key": null, "properties_hs_updated_by_user_id": 12282590, "properties_hs_user_ids_of_all_notification_followers": null, "properties_hs_user_ids_of_all_notification_unfollowers": null, "properties_hs_user_ids_of_all_owners": null, "properties_hs_was_imported": null, "properties_hubspot_owner_assigneddate": null, "properties_hubspot_owner_id": null, "properties_hubspot_team_id": null, "properties_pet_name": "Integration Test Pet", "properties_pet_type": "Unknown"}, "emitted_at": 1703886126795} +{"stream": "contacts_web_analytics", "data": {"objectType": "CONTACT", "objectId": "401", "eventType": "pe8727216_airbyte_contact_custom_event", "occurredAt": "2023-12-01T22:08:25.435Z", "id": "d287cdb7-3e8a-4f4d-92db-486e32f99ad4", "properties_hs_region": "officiis exercitationem modi adipisicing odit Hic", "properties_hs_campaign_id": "libero", "properties_hs_page_url": "Lorem", "properties_hs_element_id": "dolor sit", "properties_hs_browser": "architecto molestias, officiis exercitationem sit", "properties_hs_screen_width": "1531.0", "properties_hs_device_type": "sit adipisicing nobis officiis modi dolor sit", "properties_hs_link_href": "dolor magnam,", "properties_hs_element_class": "exercitationem modi nobis amet odit molestias,", "properties_hs_operating_system": "culpa! ipsum adipisicing consectetur nobis culpa!", "properties_hs_touchpoint_source": "libero modi odit ipsum Lorem accusantium culpa!", "properties_hs_utm_medium": "elit. ipsum officiis molestias, ipsum dolor quas"}, "emitted_at": 1701822848687} +{"stream": "contacts_web_analytics", "data": {"objectType": "CONTACT", "objectId": "401", "eventType": "pe8727216_airbyte_contact_custom_event", "occurredAt": "2023-12-01T22:08:25.723Z", "id": "2f756b9a-a68d-4566-8e63-bc66b9149b41", "properties_hs_page_id": "modi sit", "properties_hs_city": "possimus modi culpa! veniam Lorem odit Lorem quas", "properties_hs_parent_module_id": "reprehenderit exercitationem dolor adipisicing", "properties_hs_user_agent": "possimus reprehenderit architecto odit ipsum, sit", "properties_hs_operating_version": "adipisicing", "properties_hs_element_id": "architecto exercitationem consectetur modi Lorem", "properties_hs_page_content_type": "amet", "properties_hs_screen_height": "4588.0", "properties_hs_operating_system": "reiciendis placeat possimus ipsum, adipisicing", "properties_hs_language": "adipisicing reprehenderit sit ipsum, amet nobis", "properties_hs_region": "placeat accusantium adipisicing culpa! modi quas", "properties_hs_utm_source": "molestias, reprehenderit reprehenderit", "properties_hs_referrer": "possimus consectetur odit sit Lorem nobis culpa!"}, "emitted_at": 1701822848688} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-hubspot/metadata.yaml b/airbyte-integrations/connectors/source-hubspot/metadata.yaml index 4507f326b015..e68f2ffd9280 100644 --- a/airbyte-integrations/connectors/source-hubspot/metadata.yaml +++ b/airbyte-integrations/connectors/source-hubspot/metadata.yaml @@ -1,12 +1,18 @@ data: + ab_internal: + ql: 400 + sl: 300 allowedHosts: hosts: - api.hubapi.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c - dockerImageTag: 1.4.1 + dockerImageTag: 2.0.2 dockerRepository: airbyte/source-hubspot + documentationUrl: https://docs.airbyte.com/integrations/sources/hubspot githubIssueLabel: source-hubspot icon: hubspot.svg license: ELv2 @@ -14,15 +20,22 @@ data: registries: cloud: enabled: true - dockerImageTag: 1.4.1 oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/hubspot + releases: + breakingChanges: + 2.0.0: + message: >- + This version eliminates the Property History stream in favor of creating 3 different streams, Contacts, Companies, and Deals, which can now all fetch their property history. + It will affect only users who use Property History stream, who will need to fix schema conflicts and sync Contacts Property History stream instead of Property History. + upgradeDeadline: 2024-01-15 + suggestedStreams: + streams: + - contacts + - companies + - deals + supportLevel: certified tags: - language:python - ab_internal: - sl: 300 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_catalog.json index 90d38d0e6d12..94d4a262b18f 100644 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_catalog.json +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/basic_read_catalog.json @@ -245,7 +245,29 @@ }, { "stream": { - "name": "property_history", + "name": "contacts_property_history", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "default_cursor_field": ["timestamp"] + }, + "sync_mode": "full_refresh", + "cursor_field": ["timestamp"], + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "companies_property_history", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "default_cursor_field": ["timestamp"] + }, + "sync_mode": "full_refresh", + "cursor_field": ["timestamp"], + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "deals_property_history", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "default_cursor_field": ["timestamp"] @@ -322,6 +344,15 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "contacts_web_analytics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/incremental_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/incremental_catalog.json index 01698f9824b9..58c8610e38b0 100644 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/incremental_catalog.json +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/incremental_catalog.json @@ -275,6 +275,54 @@ "sync_mode": "incremental", "cursor_field": ["updatedAt"], "destination_sync_mode": "append" + }, + { + "stream": { + "name": "contacts_property_history", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["timestamp"] + }, + "sync_mode": "incremental", + "cursor_field": ["timestamp"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "companies_property_history", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["timestamp"] + }, + "sync_mode": "incremental", + "cursor_field": ["timestamp"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "deals_property_history", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["timestamp"] + }, + "sync_mode": "incremental", + "cursor_field": ["timestamp"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "contacts_web_analytics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["occurredAt"] + }, + "sync_mode": "incremental", + "cursor_field": ["occurredAt"], + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/sample_state.json b/airbyte-integrations/connectors/source-hubspot/sample_files/sample_state.json index fa9036e9b3a0..d9a123b99c7a 100644 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/sample_state.json +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/sample_state.json @@ -1,38 +1,139 @@ -{ - "companies": { - "updatedAt": "2021-02-23T00:00:00Z" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2050-05-01" }, + "stream_descriptor": { "name": "companies" } + } }, - "contact_lists": { - "timestamp": "2021-02-23T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "timestamp": "2021-02-23T00:00:00Z" }, + "stream_descriptor": { "name": "contact_lists" } + } }, - "contacts": { - "updatedAt": "2021-02-23T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2021-02-23T00:00:00Z" }, + "stream_descriptor": { "name": "contacts" } + } }, - "deals": { - "updatedAt": "2021-02-23T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2021-02-23T00:00:00Z" }, + "stream_descriptor": { "name": "deals" } + } }, - "email_events": { - "timestamp": "2021-02-23T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "timestamp": "2021-02-23T00:00:00Z" }, + "stream_descriptor": { "name": "email_events" } + } }, - "engagements": { - "lastUpdated": 1614038400000 + { + "type": "STREAM", + "stream": { + "stream_state": { "lastUpdated": 1614038400000 }, + "stream_descriptor": { "name": "engagements" } + } }, - "goals": { - "updatedAt": "2021-02-23T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2021-02-23T00:00:00Z" }, + "stream_descriptor": { "name": "goals" } + } }, - "line_items": { - "updatedAt": "2021-02-23T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2021-02-23T00:00:00Z" }, + "stream_descriptor": { "name": "line_items" } + } }, - "products": { - "updatedAt": "2021-02-23T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2021-02-23T00:00:00Z" }, + "stream_descriptor": { "name": "products" } + } }, - "subscription_changes": { - "timestamp": "2021-02-23T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "timestamp": "2021-02-23T00:00:00Z" }, + "stream_descriptor": { "name": "subscription_changes" } + } }, - "tickets": { - "updatedAt": "2021-02-23T00:00:00Z" + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "2021-02-23T00:00:00Z" }, + "stream_descriptor": { "name": "tickets" } + } }, - "forms": { - "updatedAt": "" + { + "type": "STREAM", + "stream": { + "stream_state": { "updatedAt": "" }, + "stream_descriptor": { "name": "forms" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "timestamp": 1700681340514 }, + "stream_descriptor": { "name": "contacts_property_history" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "timestamp": 1700681340514 }, + "stream_descriptor": { "name": "companies_property_history" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "timestamp": 1700681340514 }, + "stream_descriptor": { "name": "deals_property_history" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { "name": "contacts_web_analytics" }, + "stream_state": { + "601": { + "occurredAt": "2023-12-01T22:09:29.626Z" + }, + "2601": { + "occurredAt": "2023-12-01T22:11:15.870Z" + }, + "251": { + "occurredAt": "2023-12-01T22:08:24.346Z" + }, + "2501": { + "occurredAt": "2023-12-01T22:10:02.613Z" + }, + "401": { + "occurredAt": "2023-12-01T22:08:57.083Z" + }, + "151": { + "occurredAt": "2023-12-01T22:07:17.836Z" + }, + "2551": { + "occurredAt": "2023-12-01T22:10:43.512Z" + }, + "651": { + "occurredAt": "2023-12-01T22:07:51.996Z" + } + } + } } -} +] diff --git a/airbyte-integrations/connectors/source-hubspot/setup.py b/airbyte-integrations/connectors/source-hubspot/setup.py index 8adc9bbbdf2e..0f0721b230ff 100644 --- a/airbyte-integrations/connectors/source-hubspot/setup.py +++ b/airbyte-integrations/connectors/source-hubspot/setup.py @@ -7,9 +7,6 @@ MAIN_REQUIREMENTS = [ "airbyte-cdk", - "backoff==1.11.1", - "pendulum==2.1.2", - "requests==2.26.0", ] TEST_REQUIREMENTS = [ diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/helpers.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/helpers.py index a54b0de21fb3..06abe8cf4bff 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/helpers.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/helpers.py @@ -106,8 +106,26 @@ def as_url_param(self): return {"property": self.properties} +class APIv2Property(IURLPropertyRepresentation): + _term_representation = "property={property}&" + + def as_url_param(self): + return {"property": self.properties} + + class APIv3Property(IURLPropertyRepresentation): _term_representation = "{property}," def as_url_param(self): return {"properties": ",".join(self.properties)} + + +class APIPropertiesWithHistory(IURLPropertyRepresentation): + """ + It works for both v1 and v2 versions of API + """ + + _term_representation = "propertiesWithHistory={property}&" + + def as_url_param(self): + return "&".join(map(lambda prop: f"propertiesWithHistory={prop}", self.properties)) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/campaigns.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/campaigns.json index 7b2b881ae805..86a39b9f86bb 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/campaigns.json @@ -64,6 +64,54 @@ } } }, + "counters_open": { + "type": ["null", "integer"] + }, + "counters_processed": { + "type": ["null", "integer"] + }, + "counters_sent": { + "type": ["null", "integer"] + }, + "counters_deferred": { + "type": ["null", "integer"] + }, + "counters_unsubscribed": { + "type": ["null", "integer"] + }, + "counters_statuschange": { + "type": ["null", "integer"] + }, + "counters_bounce": { + "type": ["null", "integer"] + }, + "counters_mta_dropped": { + "type": ["null", "integer"] + }, + "counters_dropped": { + "type": ["null", "integer"] + }, + "counters_suppressed": { + "type": ["null", "integer"] + }, + "counters_click": { + "type": ["null", "integer"] + }, + "counters_delivered": { + "type": ["null", "integer"] + }, + "counters_forward": { + "type": ["null", "integer"] + }, + "counters_print": { + "type": ["null", "integer"] + }, + "counters_reply": { + "type": ["null", "integer"] + }, + "counters_spamreport": { + "type": ["null", "integer"] + }, "id": { "type": ["null", "integer"] }, diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/companies_property_history.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/companies_property_history.json new file mode 100644 index 000000000000..17bab830c91a --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/companies_property_history.json @@ -0,0 +1,52 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "updatedByUserId": { + "type": ["null", "number"] + }, + "requestId": { + "type": ["null", "string"] + }, + "source": { + "type": ["null", "string"] + }, + "portalId": { + "type": ["null", "number"] + }, + "isDeleted": { + "type": ["null", "boolean"] + }, + "timestamp": { + "type": ["null", "number"] + }, + "property": { + "type": ["null", "string"] + }, + "persistenceTimestamp": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "sourceVid": { + "type": ["null", "array"] + }, + "useTimestampAsPersistenceTimestamp": { + "type": ["null", "boolean"] + }, + "sourceMetadata": { + "type": ["null", "string"] + }, + "companyId": { + "type": ["null", "number"] + }, + "sourceId": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json index 500d1460eda3..4da808123cc1 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json @@ -25,6 +25,27 @@ } } }, + "metaData_processing": { + "type": ["null", "string"] + }, + "metaData_size": { + "type": ["null", "integer"] + }, + "metaData_error": { + "type": ["null", "string"] + }, + "metaData_lastProcessingStateChangeAt": { + "type": ["null", "integer"] + }, + "metaData_lastSizeChangeAt": { + "type": ["null", "integer"] + }, + "metaData_listReferencesCount": { + "type": ["null", "integer"] + }, + "metaData_parentFolderId": { + "type": ["null", "integer"] + }, "dynamic": { "type": ["null", "boolean"] }, diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts_merged_audit.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts_merged_audit.json index f3c66139aef9..29d4496e4764 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts_merged_audit.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts_merged_audit.json @@ -54,6 +54,33 @@ } } }, + "merged_from_email_source-vids": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "merged_from_email_updated-by-user-id": { + "type": ["null", "integer"] + }, + "merged_from_email_source-label": { + "type": ["null", "string"] + }, + "merged_from_email_source-type": { + "type": ["null", "string"] + }, + "merged_from_email_value": { + "type": ["null", "string"] + }, + "merged_from_email_source-id": { + "type": ["null", "string"] + }, + "merged_from_email_selected": { + "type": ["null", "boolean"] + }, + "merged_from_email_timestamp": { + "type": ["null", "integer"] + }, "merged_to_email": { "type": ["null", "object"], "additionalProperties": true, @@ -81,6 +108,27 @@ } } }, + "merged_to_email_updated-by-user-id": { + "type": ["null", "integer"] + }, + "merged_to_email_source-label": { + "type": ["null", "string"] + }, + "merged_to_email_source-type": { + "type": ["null", "string"] + }, + "merged_to_email_value": { + "type": ["null", "string"] + }, + "merged_to_email_source-id": { + "type": ["null", "string"] + }, + "merged_to_email_selected": { + "type": ["null", "boolean"] + }, + "merged_to_email_timestamp": { + "type": ["null", "integer"] + }, "first-name": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts_property_history.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts_property_history.json new file mode 100644 index 000000000000..f11a3834b0bd --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contacts_property_history.json @@ -0,0 +1,49 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "value": { + "type": ["null", "string"] + }, + "source-type": { + "type": ["null", "string"] + }, + "source-id": { + "type": ["null", "string"] + }, + "source-label": { + "type": ["null", "string"] + }, + "updated-by-user-id": { + "type": ["null", "integer"] + }, + "timestamp": { + "type": ["null", "integer"] + }, + "selected": { + "type": ["null", "boolean"] + }, + "is-contact": { + "type": ["null", "boolean"] + }, + "property": { + "type": ["null", "string"] + }, + "vid": { + "type": ["null", "integer"] + }, + "canonical-vid": { + "type": ["null", "integer"] + }, + "portal-id": { + "type": ["null", "integer"] + }, + "source-vids": { + "type": ["array", "null"], + "items": { + "type": ["null", "integer"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals.json index 478c1c84cd51..7eb20b91f26f 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals.json @@ -313,6 +313,309 @@ } } }, + "properties_amount": { + "type": ["null", "string"] + }, + "properties_amount_in_home_currency": { + "type": ["null", "string"] + }, + "properties_closed_lost_reason": { + "type": ["null", "string"] + }, + "properties_closed_won_reason": { + "type": ["null", "string"] + }, + "properties_closedate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_createdate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_days_to_close": { + "type": ["null", "string"] + }, + "properties_dealname": { + "type": ["null", "string"] + }, + "properties_dealstage": { + "type": ["null", "string"] + }, + "properties_dealtype": { + "type": ["null", "string"] + }, + "properties_description": { + "type": ["null", "string"] + }, + "properties_engagements_last_meeting_booked": { + "type": ["null", "string"] + }, + "properties_engagements_last_meeting_booked_campaign": { + "type": ["null", "string"] + }, + "properties_engagements_last_meeting_booked_medium": { + "type": ["null", "string"] + }, + "properties_engagements_last_meeting_booked_source": { + "type": ["null", "string"] + }, + "properties_hs_acv": { + "type": ["null", "string"] + }, + "properties_hs_all_accessible_team_ids": { + "type": ["null", "string"] + }, + "properties_hs_all_assigned_business_unit_ids": { + "type": ["null", "string"] + }, + "properties_hs_all_owner_ids": { + "type": ["null", "string"] + }, + "properties_hs_all_team_ids": { + "type": ["null", "string"] + }, + "properties_hs_analytics_source": { + "type": ["null", "string"] + }, + "properties_hs_analytics_source_data_1": { + "type": ["null", "string"] + }, + "properties_hs_analytics_source_data_2": { + "type": ["null", "string"] + }, + "properties_hs_arr": { + "type": ["null", "string"] + }, + "properties_hs_closed_amount": { + "type": ["null", "string"] + }, + "properties_hs_closed_amount_in_home_currency": { + "type": ["null", "string"] + }, + "properties_hs_created_by_user_id": { + "type": ["null", "string"] + }, + "properties_hs_createdate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_entered_9567448": { + "type": ["null", "string"] + }, + "properties_hs_date_entered_9567449": { + "type": ["null", "string"] + }, + "properties_hs_date_entered_appointmentscheduled": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_entered_closedlost": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_entered_closedwon": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_entered_contractsent": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_entered_customclosedwonstage": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_entered_decisionmakerboughtin": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_entered_presentationscheduled": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_entered_qualifiedtobuy": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_exited_9567448": { + "type": ["null", "string"] + }, + "properties_hs_date_exited_9567449": { + "type": ["null", "string"] + }, + "properties_hs_date_exited_appointmentscheduled": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_exited_closedlost": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_exited_closedwon": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_exited_contractsent": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_exited_customclosedwonstage": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_exited_decisionmakerboughtin": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_exited_presentationscheduled": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_exited_qualifiedtobuy": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_deal_amount_calculation_preference": { + "type": ["null", "string"] + }, + "properties_hs_deal_stage_probability": { + "type": ["null", "string"] + }, + "properties_hs_deal_stage_probability_shadow": { + "type": ["null", "string"] + }, + "properties_hs_forecast_amount": { + "type": ["null", "string"] + }, + "properties_hs_forecast_probability": { + "type": ["null", "string"] + }, + "properties_hs_is_closed": { + "type": ["null", "boolean"] + }, + "properties_hs_is_closed_won": { + "type": ["null", "boolean"] + }, + "properties_hs_lastmodifieddate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_latest_meeting_activity": { + "type": ["null", "string"] + }, + "properties_hs_likelihood_to_close": { + "type": ["null", "string"] + }, + "properties_hs_line_item_global_term_hs_discount_percentage": { + "type": ["null", "string"] + }, + "properties_hs_line_item_global_term_hs_discount_percentage_enabled": { + "type": ["null", "boolean"] + }, + "properties_hs_line_item_global_term_hs_recurring_billing_period": { + "type": ["null", "string"] + }, + "properties_hs_line_item_global_term_hs_recurring_billing_period_enabled": { + "type": ["null", "boolean"] + }, + "properties_hs_line_item_global_term_hs_recurring_billing_start_date": { + "type": ["null", "string"] + }, + "properties_hs_line_item_global_term_hs_recurring_billing_start_date_enabled": { + "type": ["null", "boolean"] + }, + "properties_hs_line_item_global_term_recurringbillingfrequency": { + "type": ["null", "string"] + }, + "properties_hs_line_item_global_term_recurringbillingfrequency_enabled": { + "type": ["null", "boolean"] + }, + "properties_hs_manual_forecast_category": { + "type": ["null", "string"] + }, + "properties_hs_merged_object_ids": { + "type": ["null", "string"] + }, + "properties_hs_mrr": { + "type": ["null", "string"] + }, + "properties_hs_next_step": { + "type": ["null", "string"] + }, + "properties_hs_num_target_accounts": { + "type": ["null", "string"] + }, + "properties_hs_object_id": { + "type": ["null", "string"] + }, + "properties_hs_predicted_amount": { + "type": ["null", "string"] + }, + "properties_hs_predicted_amount_in_home_currency": { + "type": ["null", "string"] + }, + "properties_hs_priority": { + "type": ["null", "string"] + }, + "properties_hs_projected_amount": { + "type": ["null", "string"] + }, + "properties_hs_projected_amount_in_home_currency": { + "type": ["null", "string"] + }, + "properties_hs_sales_email_last_replied": { + "type": ["null", "string"] + }, + "properties_hs_tcv": { + "type": ["null", "string"] + }, + "properties_hs_unique_creation_key": { + "type": ["null", "string"] + }, + "properties_hs_updated_by_user_id": { + "type": ["null", "string"] + }, + "properties_hs_user_ids_of_all_notification_followers": { + "type": ["null", "string"] + }, + "properties_hs_user_ids_of_all_notification_unfollowers": { + "type": ["null", "string"] + }, + "properties_hs_user_ids_of_all_owners": { + "type": ["null", "string"] + }, + "properties_hubspot_owner_assigneddate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hubspot_owner_id": { + "type": ["null", "string"] + }, + "properties_hubspot_team_id": { + "type": ["null", "string"] + }, + "properties_notes_last_contacted": { + "type": ["null", "string"] + }, + "properties_notes_last_updated": { + "type": ["null", "string"] + }, + "properties_notes_next_activity_date": { + "type": ["null", "string"] + }, + "properties_num_associated_contacts": { + "type": ["null", "string"] + }, + "properties_num_contacted_notes": { + "type": ["null", "string"] + }, + "properties_num_notes": { + "type": ["null", "string"] + }, + "properties_pipeline": { + "type": ["null", "string"] + }, "createdAt": { "type": ["null", "string"], "format": "date-time" diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals_archived.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals_archived.json index 420b17768b87..8e9ca2b439f0 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals_archived.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals_archived.json @@ -313,6 +313,309 @@ } } }, + "properties_amount": { + "type": ["null", "string"] + }, + "properties_amount_in_home_currency": { + "type": ["null", "string"] + }, + "properties_closed_lost_reason": { + "type": ["null", "string"] + }, + "properties_closed_won_reason": { + "type": ["null", "string"] + }, + "properties_closedate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_createdate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_days_to_close": { + "type": ["null", "string"] + }, + "properties_dealname": { + "type": ["null", "string"] + }, + "properties_dealstage": { + "type": ["null", "string"] + }, + "properties_dealtype": { + "type": ["null", "string"] + }, + "properties_description": { + "type": ["null", "string"] + }, + "properties_engagements_last_meeting_booked": { + "type": ["null", "string"] + }, + "properties_engagements_last_meeting_booked_campaign": { + "type": ["null", "string"] + }, + "properties_engagements_last_meeting_booked_medium": { + "type": ["null", "string"] + }, + "properties_engagements_last_meeting_booked_source": { + "type": ["null", "string"] + }, + "properties_hs_acv": { + "type": ["null", "string"] + }, + "properties_hs_all_accessible_team_ids": { + "type": ["null", "string"] + }, + "properties_hs_all_assigned_business_unit_ids": { + "type": ["null", "string"] + }, + "properties_hs_all_owner_ids": { + "type": ["null", "string"] + }, + "properties_hs_all_team_ids": { + "type": ["null", "string"] + }, + "properties_hs_analytics_source": { + "type": ["null", "string"] + }, + "properties_hs_analytics_source_data_1": { + "type": ["null", "string"] + }, + "properties_hs_analytics_source_data_2": { + "type": ["null", "string"] + }, + "properties_hs_arr": { + "type": ["null", "string"] + }, + "properties_hs_closed_amount": { + "type": ["null", "string"] + }, + "properties_hs_closed_amount_in_home_currency": { + "type": ["null", "string"] + }, + "properties_hs_created_by_user_id": { + "type": ["null", "string"] + }, + "properties_hs_createdate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_entered_9567448": { + "type": ["null", "string"] + }, + "properties_hs_date_entered_9567449": { + "type": ["null", "string"] + }, + "properties_hs_date_entered_appointmentscheduled": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_entered_closedlost": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_entered_closedwon": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_entered_contractsent": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_entered_customclosedwonstage": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_entered_decisionmakerboughtin": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_entered_presentationscheduled": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_entered_qualifiedtobuy": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_exited_9567448": { + "type": ["null", "string"] + }, + "properties_hs_date_exited_9567449": { + "type": ["null", "string"] + }, + "properties_hs_date_exited_appointmentscheduled": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_exited_closedlost": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_exited_closedwon": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_exited_contractsent": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_exited_customclosedwonstage": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_exited_decisionmakerboughtin": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_exited_presentationscheduled": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_date_exited_qualifiedtobuy": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_deal_amount_calculation_preference": { + "type": ["null", "string"] + }, + "properties_hs_deal_stage_probability": { + "type": ["null", "string"] + }, + "properties_hs_deal_stage_probability_shadow": { + "type": ["null", "string"] + }, + "properties_hs_forecast_amount": { + "type": ["null", "string"] + }, + "properties_hs_forecast_probability": { + "type": ["null", "string"] + }, + "properties_hs_is_closed": { + "type": ["null", "string"] + }, + "properties_hs_is_closed_won": { + "type": ["null", "string"] + }, + "properties_hs_lastmodifieddate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_latest_meeting_activity": { + "type": ["null", "string"] + }, + "properties_hs_likelihood_to_close": { + "type": ["null", "string"] + }, + "properties_hs_line_item_global_term_hs_discount_percentage": { + "type": ["null", "string"] + }, + "properties_hs_line_item_global_term_hs_discount_percentage_enabled": { + "type": ["null", "string"] + }, + "properties_hs_line_item_global_term_hs_recurring_billing_period": { + "type": ["null", "string"] + }, + "properties_hs_line_item_global_term_hs_recurring_billing_period_enabled": { + "type": ["null", "string"] + }, + "properties_hs_line_item_global_term_hs_recurring_billing_start_date": { + "type": ["null", "string"] + }, + "properties_hs_line_item_global_term_hs_recurring_billing_start_date_enabled": { + "type": ["null", "string"] + }, + "properties_hs_line_item_global_term_recurringbillingfrequency": { + "type": ["null", "string"] + }, + "properties_hs_line_item_global_term_recurringbillingfrequency_enabled": { + "type": ["null", "string"] + }, + "properties_hs_manual_forecast_category": { + "type": ["null", "string"] + }, + "properties_hs_merged_object_ids": { + "type": ["null", "string"] + }, + "properties_hs_mrr": { + "type": ["null", "string"] + }, + "properties_hs_next_step": { + "type": ["null", "string"] + }, + "properties_hs_num_target_accounts": { + "type": ["null", "string"] + }, + "properties_hs_object_id": { + "type": ["null", "string"] + }, + "properties_hs_predicted_amount": { + "type": ["null", "string"] + }, + "properties_hs_predicted_amount_in_home_currency": { + "type": ["null", "string"] + }, + "properties_hs_priority": { + "type": ["null", "string"] + }, + "properties_hs_projected_amount": { + "type": ["null", "string"] + }, + "properties_hs_projected_amount_in_home_currency": { + "type": ["null", "string"] + }, + "properties_hs_sales_email_last_replied": { + "type": ["null", "string"] + }, + "properties_hs_tcv": { + "type": ["null", "string"] + }, + "properties_hs_unique_creation_key": { + "type": ["null", "string"] + }, + "properties_hs_updated_by_user_id": { + "type": ["null", "string"] + }, + "properties_hs_user_ids_of_all_notification_followers": { + "type": ["null", "string"] + }, + "properties_hs_user_ids_of_all_notification_unfollowers": { + "type": ["null", "string"] + }, + "properties_hs_user_ids_of_all_owners": { + "type": ["null", "string"] + }, + "properties_hubspot_owner_assigneddate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hubspot_owner_id": { + "type": ["null", "string"] + }, + "properties_hubspot_team_id": { + "type": ["null", "string"] + }, + "properties_notes_last_contacted": { + "type": ["null", "string"] + }, + "properties_notes_last_updated": { + "type": ["null", "string"] + }, + "properties_notes_next_activity_date": { + "type": ["null", "string"] + }, + "properties_num_associated_contacts": { + "type": ["null", "string"] + }, + "properties_num_contacted_notes": { + "type": ["null", "string"] + }, + "properties_num_notes": { + "type": ["null", "string"] + }, + "properties_pipeline": { + "type": ["null", "string"] + }, "createdAt": { "type": ["null", "string"], "format": "date-time" diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals_property_history.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals_property_history.json new file mode 100644 index 000000000000..f5f4dec16ae0 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/deals_property_history.json @@ -0,0 +1,52 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "updatedByUserId": { + "type": ["null", "number"] + }, + "requestId": { + "type": ["null", "string"] + }, + "source": { + "type": ["null", "string"] + }, + "portalId": { + "type": ["null", "number"] + }, + "isDeleted": { + "type": ["null", "boolean"] + }, + "timestamp": { + "type": ["null", "number"] + }, + "property": { + "type": ["null", "string"] + }, + "persistenceTimestamp": { + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "sourceVid": { + "type": ["null", "array"] + }, + "useTimestampAsPersistenceTimestamp": { + "type": ["null", "boolean"] + }, + "sourceMetadata": { + "type": ["null", "string"] + }, + "dealId": { + "type": ["null", "number"] + }, + "sourceId": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json index 07212bcbf3e4..43ac501b429f 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json @@ -112,6 +112,60 @@ } } }, + "associations_contactIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "associations_contentIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "associations_companyIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "associations_dealIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "associations_marketingEventIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "associations_ownerIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "associations_quoteIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "associations_workflowIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "associations_ticketIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, "attachments": { "type": ["null", "array"], "items": { @@ -329,6 +383,208 @@ "type": ["null", "boolean"] } } + }, + "metadata_body": { + "type": ["null", "string"] + }, + "metadata_from": { + "type": ["null", "object"], + "properties": { + "email": { + "type": ["null", "string"] + }, + "firstName": { + "type": ["null", "string"] + }, + "lastName": { + "type": ["null", "string"] + }, + "raw": { + "type": ["null", "string"] + } + } + }, + "metadata_sender": { + "type": ["null", "object"], + "properties": { + "email": { + "type": ["null", "string"] + } + } + }, + "metadata_to": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "email": { + "type": ["null", "string"] + }, + "firstName": { + "type": ["null", "string"] + }, + "lastName": { + "type": ["null", "string"] + }, + "raw": { + "type": ["null", "string"] + } + } + } + }, + "metadata_cc": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "email": { + "type": ["null", "string"] + }, + "firstName": { + "type": ["null", "string"] + }, + "lastName": { + "type": ["null", "string"] + }, + "raw": { + "type": ["null", "string"] + } + } + } + }, + "metadata_bcc": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "email": { + "type": ["null", "string"] + } + } + } + }, + "metadata_subject": { + "type": ["null", "string"] + }, + "metadata_html": { + "type": ["null", "string"] + }, + "metadata_text": { + "type": ["null", "string"] + }, + "metadata_status": { + "type": ["null", "string"] + }, + "metadata_forObjectType": { + "type": ["null", "string"] + }, + "metadata_startTime": { + "type": ["null", "integer"] + }, + "metadata_endTime": { + "type": ["null", "integer"] + }, + "metadata_title": { + "type": ["null", "string"] + }, + "metadata_toNumber": { + "type": ["null", "string"] + }, + "metadata_fromNumber": { + "type": ["null", "string"] + }, + "metadata_externalId": { + "type": ["null", "string"] + }, + "metadata_durationMilliseconds": { + "type": ["null", "integer"] + }, + "metadata_externalAccountId": { + "type": ["null", "string"] + }, + "metadata_recordingUrl": { + "type": ["null", "string"] + }, + "metadata_disposition": { + "type": ["null", "string"] + }, + "metadata_completionDate": { + "type": ["null", "integer"] + }, + "metadata_taskType": { + "type": ["null", "string"] + }, + "metadata_reminders": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "metadata_threadId": { + "type": ["null", "string"] + }, + "metadata_messageId": { + "type": ["null", "string"] + }, + "metadata_loggedFrom": { + "type": ["null", "string"] + }, + "metadata_attachedVideoOpened": { + "type": ["null", "boolean"] + }, + "metadata_attachedVideoWatched": { + "type": ["null", "boolean"] + }, + "metadata_trackerKey": { + "type": ["null", "string"] + }, + "metadata_sendDefaultReminder": { + "type": ["null", "boolean"] + }, + "metadata_source": { + "type": ["null", "string"] + }, + "metadata_unknownVisitorConversation": { + "type": ["null", "boolean"] + }, + "metadata_facsimileSendId": { + "type": ["null", "string"] + }, + "metadata_sentVia": { + "type": ["null", "string"] + }, + "metadata_sequenceStepOrder": { + "type": ["null", "integer"] + }, + "metadata_externalUrl": { + "type": ["null", "string"] + }, + "metadata_postSendStatus": { + "type": ["null", "string"] + }, + "metadata_errorMessage": { + "type": ["null", "string"] + }, + "metadata_recipientDropReasons": { + "type": ["null", "string"] + }, + "metadata_calleeObjectId": { + "type": ["null", "integer"] + }, + "metadata_calleeObjectType": { + "type": ["null", "string"] + }, + "metadata_mediaProcessingStatus": { + "type": ["null", "string"] + }, + "metadata_sourceId": { + "type": ["null", "string"] + }, + "metadata_priority": { + "type": ["null", "string"] + }, + "metadata_isAllDay": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements_calls.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements_calls.json index 648ba44b171c..468f4c477cfa 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements_calls.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements_calls.json @@ -176,6 +176,172 @@ } } }, + "properties_hs_activity_type": { + "type": ["null", "string"] + }, + "properties_hs_all_assigned_business_unit_ids": { + "type": ["null", "string"] + }, + "properties_hs_at_mentioned_owner_ids": { + "type": ["null", "string"] + }, + "properties_hs_attachment_ids": { + "type": ["null", "string"] + }, + "properties_hs_body_preview": { + "type": ["null", "string"] + }, + "properties_hs_body_preview_html": { + "type": ["null", "string"] + }, + "properties_hs_body_preview_is_truncated": { + "type": ["null", "boolean"] + }, + "properties_hs_call_app_id": { + "type": ["null", "number"] + }, + "properties_hs_call_authed_url_provider": { + "type": ["null", "string"] + }, + "properties_hs_call_body": { + "type": ["null", "string"] + }, + "properties_hs_call_callee_object_id": { + "type": ["null", "number"] + }, + "properties_hs_call_callee_object_type": { + "type": ["null", "string"] + }, + "properties_hs_call_disposition": { + "type": ["null", "string"] + }, + "properties_hs_call_duration": { + "type": ["null", "number"] + }, + "properties_hs_call_external_account_id": { + "type": ["null", "string"] + }, + "properties_hs_call_external_id": { + "type": ["null", "string"] + }, + "properties_hs_call_from_number": { + "type": ["null", "string"] + }, + "properties_hs_call_has_transcript": { + "type": ["null", "boolean"] + }, + "properties_hs_call_recording_url": { + "type": ["null", "string"] + }, + "properties_hs_call_source": { + "type": ["null", "string"] + }, + "properties_hs_call_status": { + "type": ["null", "string"] + }, + "properties_hs_call_title": { + "type": ["null", "string"] + }, + "properties_hs_call_to_number": { + "type": ["null", "string"] + }, + "properties_hs_call_transcription_id": { + "type": ["null", "number"] + }, + "properties_hs_call_video_recording_url": { + "type": ["null", "string"] + }, + "properties_hs_call_zoom_meeting_uuid": { + "type": ["null", "string"] + }, + "properties_hs_calls_service_call_id": { + "type": ["null", "number"] + }, + "properties_hs_created_by": { + "type": ["null", "number"] + }, + "properties_hs_created_by_user_id": { + "type": ["null", "number"] + }, + "properties_hs_createdate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_engagement_source": { + "type": ["null", "string"] + }, + "properties_hs_engagement_source_id": { + "type": ["null", "string"] + }, + "properties_hs_follow_up_action": { + "type": ["null", "string"] + }, + "properties_hs_gdpr_deleted": { + "type": ["null", "boolean"] + }, + "properties_hs_lastmodifieddate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_merged_object_ids": { + "type": ["null", "string"] + }, + "properties_hs_modified_by": { + "type": ["null", "number"] + }, + "properties_hs_object_id": { + "type": ["null", "number"] + }, + "properties_hs_product_name": { + "type": ["null", "string"] + }, + "properties_hs_queue_membership_ids": { + "type": ["null", "string"] + }, + "properties_hs_timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_unique_creation_key": { + "type": ["null", "string"] + }, + "properties_hs_unique_id": { + "type": ["null", "string"] + }, + "properties_hs_unknown_visitor_conversation": { + "type": ["null", "boolean"] + }, + "properties_hs_updated_by_user_id": { + "type": ["null", "number"] + }, + "properties_hs_user_ids_of_all_notification_followers": { + "type": ["null", "string"] + }, + "properties_hs_user_ids_of_all_notification_unfollowers": { + "type": ["null", "string"] + }, + "properties_hs_user_ids_of_all_owners": { + "type": ["null", "string"] + }, + "properties_hubspot_owner_assigneddate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hubspot_owner_id": { + "type": ["null", "string"] + }, + "properties_hubspot_team_id": { + "type": ["null", "string"] + }, + "properties_hs_all_owner_ids": { + "type": ["null", "string"] + }, + "properties_hs_all_team_ids": { + "type": ["null", "string"] + }, + "properties_hs_all_accessible_team_ids": { + "type": ["null", "string"] + }, "createdAt": { "type": ["null", "string"], "format": "date-time" diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements_emails.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements_emails.json index 3d683f44cd3a..60d4377f707d 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements_emails.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements_emails.json @@ -255,6 +255,251 @@ } } }, + "properties_hs_all_assigned_business_unit_ids": { + "type": ["null", "string"] + }, + "properties_hs_at_mentioned_owner_ids": { + "type": ["null", "string"] + }, + "properties_hs_attachment_ids": { + "type": ["null", "string"] + }, + "properties_hs_body_preview": { + "type": ["null", "string"] + }, + "properties_hs_body_preview_html": { + "type": ["null", "string"] + }, + "properties_hs_body_preview_is_truncated": { + "type": ["null", "boolean"] + }, + "properties_hs_created_by": { + "type": ["null", "string"] + }, + "properties_hs_created_by_user_id": { + "type": ["null", "number"] + }, + "properties_hs_createdate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_direction_and_unique_id": { + "type": ["null", "string"] + }, + "properties_hs_email_attached_video_id": { + "type": ["null", "string"] + }, + "properties_hs_email_attached_video_name": { + "type": ["null", "string"] + }, + "properties_hs_email_attached_video_opened": { + "type": ["null", "boolean"] + }, + "properties_hs_email_attached_video_watched": { + "type": ["null", "boolean"] + }, + "properties_hs_email_bcc_email": { + "type": ["null", "string"] + }, + "properties_hs_email_bcc_firstname": { + "type": ["null", "string"] + }, + "properties_hs_email_bcc_lastname": { + "type": ["null", "string"] + }, + "properties_hs_email_bcc_raw": { + "type": ["null", "string"] + }, + "properties_hs_email_cc_email": { + "type": ["null", "string"] + }, + "properties_hs_email_cc_firstname": { + "type": ["null", "string"] + }, + "properties_hs_email_cc_lastname": { + "type": ["null", "string"] + }, + "properties_hs_email_cc_raw": { + "type": ["null", "string"] + }, + "properties_hs_email_direction": { + "type": ["null", "string"] + }, + "properties_hs_email_encoded_email_associations_request": { + "type": ["null", "string"] + }, + "properties_hs_email_error_message": { + "type": ["null", "string"] + }, + "properties_hs_email_facsimile_send_id": { + "type": ["null", "string"] + }, + "properties_hs_email_from_email": { + "type": ["null", "string"] + }, + "properties_hs_email_from_firstname": { + "type": ["null", "string"] + }, + "properties_hs_email_from_lastname": { + "type": ["null", "string"] + }, + "properties_hs_email_from_raw": { + "type": ["null", "string"] + }, + "properties_hs_email_headers": { + "type": ["null", "string"] + }, + "properties_hs_email_html": { + "type": ["null", "string"] + }, + "properties_hs_email_logged_from": { + "type": ["null", "string"] + }, + "properties_hs_email_media_processing_status": { + "type": ["null", "string"] + }, + "properties_hs_email_member_of_forwarded_subthread": { + "type": ["null", "boolean"] + }, + "properties_hs_email_message_id": { + "type": ["null", "string"] + }, + "properties_hs_email_migrated_via_portal_data_migration": { + "type": ["null", "string"] + }, + "properties_hs_email_pending_inline_image_ids": { + "type": ["null", "string"] + }, + "properties_hs_email_post_send_status": { + "type": ["null", "string"] + }, + "properties_hs_email_recipient_drop_reasons": { + "type": ["null", "string"] + }, + "properties_hs_email_send_event_id": { + "type": ["null", "string"] + }, + "properties_hs_email_send_event_id_created": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_email_sender_email": { + "type": ["null", "string"] + }, + "properties_hs_email_sender_firstname": { + "type": ["null", "string"] + }, + "properties_hs_email_sender_lastname": { + "type": ["null", "string"] + }, + "properties_hs_email_sender_raw": { + "type": ["null", "string"] + }, + "properties_hs_email_sent_via": { + "type": ["null", "string"] + }, + "properties_hs_email_status": { + "type": ["null", "string"] + }, + "properties_hs_email_subject": { + "type": ["null", "string"] + }, + "properties_hs_email_text": { + "type": ["null", "string"] + }, + "properties_hs_email_thread_id": { + "type": ["null", "string"] + }, + "properties_hs_email_to_email": { + "type": ["null", "string"] + }, + "properties_hs_email_to_firstname": { + "type": ["null", "string"] + }, + "properties_hs_email_to_lastname": { + "type": ["null", "string"] + }, + "properties_hs_email_to_raw": { + "type": ["null", "string"] + }, + "properties_hs_email_tracker_key": { + "type": ["null", "string"] + }, + "properties_hs_email_validation_skipped": { + "type": ["null", "string"] + }, + "properties_hs_engagement_source": { + "type": ["null", "string"] + }, + "properties_hs_engagement_source_id": { + "type": ["null", "string"] + }, + "properties_hs_follow_up_action": { + "type": ["null", "string"] + }, + "properties_hs_gdpr_deleted": { + "type": ["null", "boolean"] + }, + "properties_hs_lastmodifieddate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_merged_object_ids": { + "type": ["null", "string"] + }, + "properties_hs_modified_by": { + "type": ["null", "string"] + }, + "properties_hs_object_id": { + "type": ["null", "number"] + }, + "properties_hs_product_name": { + "type": ["null", "string"] + }, + "properties_hs_queue_membership_ids": { + "type": ["null", "string"] + }, + "properties_hs_timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_unique_creation_key": { + "type": ["null", "string"] + }, + "properties_hs_unique_id": { + "type": ["null", "string"] + }, + "properties_hs_updated_by_user_id": { + "type": ["null", "number"] + }, + "properties_hs_user_ids_of_all_notification_followers": { + "type": ["null", "string"] + }, + "properties_hs_user_ids_of_all_notification_unfollowers": { + "type": ["null", "string"] + }, + "properties_hs_user_ids_of_all_owners": { + "type": ["null", "string"] + }, + "properties_hubspot_owner_assigneddate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hubspot_owner_id": { + "type": ["null", "string"] + }, + "properties_hubspot_team_id": { + "type": ["null", "string"] + }, + "properties_hs_all_owner_ids": { + "type": ["null", "string"] + }, + "properties_hs_all_team_ids": { + "type": ["null", "string"] + }, + "properties_hs_all_accessible_team_ids": { + "type": ["null", "string"] + }, "createdAt": { "type": ["null", "string"], "format": "date-time" diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements_meetings.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements_meetings.json index fe40c4da9a96..0c6ed26b3b76 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements_meetings.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements_meetings.json @@ -172,6 +172,168 @@ } } }, + "properties_hs_activity_type": { + "type": ["null", "string"] + }, + "properties_hs_all_assigned_business_unit_ids": { + "type": ["null", "string"] + }, + "properties_hs_at_mentioned_owner_ids": { + "type": ["null", "string"] + }, + "properties_hs_attachment_ids": { + "type": ["null", "string"] + }, + "properties_hs_attendee_owner_ids": { + "type": ["null", "string"] + }, + "properties_hs_body_preview": { + "type": ["null", "string"] + }, + "properties_hs_body_preview_html": { + "type": ["null", "string"] + }, + "properties_hs_body_preview_is_truncated": { + "type": ["null", "boolean"] + }, + "properties_hs_created_by": { + "type": ["null", "number"] + }, + "properties_hs_created_by_user_id": { + "type": ["null", "number"] + }, + "properties_hs_createdate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_engagement_source": { + "type": ["null", "string"] + }, + "properties_hs_engagement_source_id": { + "type": ["null", "string"] + }, + "properties_hs_follow_up_action": { + "type": ["null", "string"] + }, + "properties_hs_gdpr_deleted": { + "type": ["null", "boolean"] + }, + "properties_hs_i_cal_uid": { + "type": ["null", "string"] + }, + "properties_hs_internal_meeting_notes": { + "type": ["null", "string"] + }, + "properties_hs_lastmodifieddate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_meeting_body": { + "type": ["null", "string"] + }, + "properties_hs_meeting_calendar_event_hash": { + "type": ["null", "string"] + }, + "properties_hs_meeting_change_id": { + "type": ["null", "string"] + }, + "properties_hs_meeting_created_from_link_id": { + "type": ["null", "string"] + }, + "properties_hs_meeting_end_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_meeting_external_url": { + "type": ["null", "string"] + }, + "properties_hs_meeting_location": { + "type": ["null", "string"] + }, + "properties_hs_meeting_location_type": { + "type": ["null", "string"] + }, + "properties_hs_meeting_outcome": { + "type": ["null", "string"] + }, + "properties_hs_meeting_pre_meeting_prospect_reminders": { + "type": ["null", "string"] + }, + "properties_hs_meeting_source": { + "type": ["null", "string"] + }, + "properties_hs_meeting_source_id": { + "type": ["null", "string"] + }, + "properties_hs_meeting_start_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_meeting_title": { + "type": ["null", "string"] + }, + "properties_hs_meeting_web_conference_meeting_id": { + "type": ["null", "string"] + }, + "properties_hs_merged_object_ids": { + "type": ["null", "string"] + }, + "properties_hs_modified_by": { + "type": ["null", "number"] + }, + "properties_hs_object_id": { + "type": ["null", "number"] + }, + "properties_hs_product_name": { + "type": ["null", "string"] + }, + "properties_hs_queue_membership_ids": { + "type": ["null", "string"] + }, + "properties_hs_scheduled_tasks": { + "type": ["null", "string"] + }, + "properties_hs_timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_unique_creation_key": { + "type": ["null", "string"] + }, + "properties_hs_unique_id": { + "type": ["null", "string"] + }, + "properties_hs_updated_by_user_id": { + "type": ["null", "number"] + }, + "properties_hs_user_ids_of_all_notification_followers": { + "type": ["null", "string"] + }, + "properties_hs_user_ids_of_all_notification_unfollowers": { + "type": ["null", "string"] + }, + "properties_hs_user_ids_of_all_owners": { + "type": ["null", "string"] + }, + "properties_hubspot_owner_assigneddate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hubspot_owner_id": { + "type": ["null", "string"] + }, + "properties_hubspot_team_id": { + "type": ["null", "string"] + }, + "properties_hs_all_owner_ids": { + "type": ["null", "string"] + }, + "properties_hs_all_team_ids": { + "type": ["null", "string"] + }, + "properties_hs_all_accessible_team_ids": { + "type": ["null", "string"] + }, "createdAt": { "type": ["null", "string"], "format": "date-time" diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements_notes.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements_notes.json index 38aeaba107c5..803346044063 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements_notes.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements_notes.json @@ -113,6 +113,109 @@ } } }, + "properties_hs_all_assigned_business_unit_ids": { + "type": ["null", "string"] + }, + "properties_hs_at_mentioned_owner_ids": { + "type": ["null", "string"] + }, + "properties_hs_attachment_ids": { + "type": ["null", "string"] + }, + "properties_hs_body_preview": { + "type": ["null", "string"] + }, + "properties_hs_body_preview_html": { + "type": ["null", "string"] + }, + "properties_hs_body_preview_is_truncated": { + "type": ["null", "boolean"] + }, + "properties_hs_created_by": { + "type": ["null", "number"] + }, + "properties_hs_created_by_user_id": { + "type": ["null", "number"] + }, + "properties_hs_createdate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_engagement_source": { + "type": ["null", "string"] + }, + "properties_hs_engagement_source_id": { + "type": ["null", "string"] + }, + "properties_hs_follow_up_action": { + "type": ["null", "string"] + }, + "properties_hs_gdpr_deleted": { + "type": ["null", "boolean"] + }, + "properties_hs_lastmodifieddate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_merged_object_ids": { + "type": ["null", "string"] + }, + "properties_hs_modified_by": { + "type": ["null", "number"] + }, + "properties_hs_note_body": { + "type": ["null", "string"] + }, + "properties_hs_object_id": { + "type": ["null", "number"] + }, + "properties_hs_product_name": { + "type": ["null", "string"] + }, + "properties_hs_queue_membership_ids": { + "type": ["null", "string"] + }, + "properties_hs_timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_unique_creation_key": { + "type": ["null", "string"] + }, + "properties_hs_unique_id": { + "type": ["null", "string"] + }, + "properties_hs_updated_by_user_id": { + "type": ["null", "number"] + }, + "properties_hs_user_ids_of_all_notification_followers": { + "type": ["null", "string"] + }, + "properties_hs_user_ids_of_all_notification_unfollowers": { + "type": ["null", "string"] + }, + "properties_hs_user_ids_of_all_owners": { + "type": ["null", "string"] + }, + "properties_hubspot_owner_assigneddate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hubspot_owner_id": { + "type": ["null", "string"] + }, + "properties_hubspot_team_id": { + "type": ["null", "string"] + }, + "properties_hs_all_owner_ids": { + "type": ["null", "string"] + }, + "properties_hs_all_team_ids": { + "type": ["null", "string"] + }, + "properties_hs_all_accessible_team_ids": { + "type": ["null", "string"] + }, "createdAt": { "type": ["null", "string"], "format": "date-time" diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements_tasks.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements_tasks.json index 7232b8ba270b..ffde1acbf652 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements_tasks.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements_tasks.json @@ -197,6 +197,193 @@ } } }, + "properties_hs_all_assigned_business_unit_ids": { + "type": ["null", "string"] + }, + "properties_hs_at_mentioned_owner_ids": { + "type": ["null", "string"] + }, + "properties_hs_attachment_ids": { + "type": ["null", "string"] + }, + "properties_hs_body_preview": { + "type": ["null", "string"] + }, + "properties_hs_body_preview_html": { + "type": ["null", "string"] + }, + "properties_hs_body_preview_is_truncated": { + "type": ["null", "boolean"] + }, + "properties_hs_calendar_event_id": { + "type": ["null", "string"] + }, + "properties_hs_created_by": { + "type": ["null", "number"] + }, + "properties_hs_created_by_user_id": { + "type": ["null", "number"] + }, + "properties_hs_createdate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_engagement_source": { + "type": ["null", "string"] + }, + "properties_hs_engagement_source_id": { + "type": ["null", "string"] + }, + "properties_hs_follow_up_action": { + "type": ["null", "string"] + }, + "properties_hs_gdpr_deleted": { + "type": ["null", "boolean"] + }, + "properties_hs_lastmodifieddate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_merged_object_ids": { + "type": ["null", "string"] + }, + "properties_hs_modified_by": { + "type": ["null", "number"] + }, + "properties_hs_msteams_message_id": { + "type": ["null", "string"] + }, + "properties_hs_num_associated_companies": { + "type": ["null", "number"] + }, + "properties_hs_num_associated_contacts": { + "type": ["null", "number"] + }, + "properties_hs_num_associated_deals": { + "type": ["null", "number"] + }, + "properties_hs_num_associated_queue_objects": { + "type": ["null", "number"] + }, + "properties_hs_num_associated_tickets": { + "type": ["null", "number"] + }, + "properties_hs_object_id": { + "type": ["null", "number"] + }, + "properties_hs_product_name": { + "type": ["null", "string"] + }, + "properties_hs_queue_membership_ids": { + "type": ["null", "string"] + }, + "properties_hs_scheduled_tasks": { + "type": ["null", "string"] + }, + "properties_hs_task_body": { + "type": ["null", "string"] + }, + "properties_hs_task_completion_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_task_contact_timezone": { + "type": ["null", "string"] + }, + "properties_hs_task_for_object_type": { + "type": ["null", "string"] + }, + "properties_hs_task_is_all_day": { + "type": ["null", "boolean"] + }, + "properties_hs_task_last_contact_outreach": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_task_last_sales_activity_timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_task_priority": { + "type": ["null", "string"] + }, + "properties_hs_task_probability_to_complete": { + "type": ["null", "number"] + }, + "properties_hs_task_relative_reminders": { + "type": ["null", "string"] + }, + "properties_hs_task_reminders": { + "type": ["null", "string"] + }, + "properties_hs_task_repeat_interval": { + "type": ["null", "string"] + }, + "properties_hs_task_send_default_reminder": { + "type": ["null", "boolean"] + }, + "properties_hs_task_sequence_enrollment_active": { + "type": ["null", "boolean"] + }, + "properties_hs_task_sequence_step_enrollment_id": { + "type": ["null", "string"] + }, + "properties_hs_task_sequence_step_order": { + "type": ["null", "number"] + }, + "properties_hs_task_status": { + "type": ["null", "string"] + }, + "properties_hs_task_subject": { + "type": ["null", "string"] + }, + "properties_hs_task_template_id": { + "type": ["null", "number"] + }, + "properties_hs_task_type": { + "type": ["null", "string"] + }, + "properties_hs_timestamp": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_unique_creation_key": { + "type": ["null", "string"] + }, + "properties_hs_unique_id": { + "type": ["null", "string"] + }, + "properties_hs_updated_by_user_id": { + "type": ["null", "number"] + }, + "properties_hs_user_ids_of_all_notification_followers": { + "type": ["null", "string"] + }, + "properties_hs_user_ids_of_all_notification_unfollowers": { + "type": ["null", "string"] + }, + "properties_hs_user_ids_of_all_owners": { + "type": ["null", "string"] + }, + "properties_hubspot_owner_assigneddate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hubspot_owner_id": { + "type": ["null", "string"] + }, + "properties_hubspot_team_id": { + "type": ["null", "string"] + }, + "properties_hs_all_owner_ids": { + "type": ["null", "string"] + }, + "properties_hs_all_team_ids": { + "type": ["null", "string"] + }, + "properties_hs_all_accessible_team_ids": { + "type": ["null", "string"] + }, "createdAt": { "type": ["null", "string"], "format": "date-time" diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json index 89a39011f6fc..c1f20f05459f 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/forms.json @@ -108,6 +108,12 @@ }, "allowLinkToResetKnownValues": { "type": ["null", "boolean"] + }, + "lifecycleStages": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } } } }, diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/goals.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/goals.json index 0a6e4a507111..acb126224023 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/goals.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/goals.json @@ -42,6 +42,38 @@ } } }, + "properties_hs_created_by_user_id": { + "type": ["null", "string"] + }, + "properties_hs_createdate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_start_datetime": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_end_datetime": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_goal_name": { + "type": ["null", "string"] + }, + "properties_hs_lastmodifieddate": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_kpi_value_last_calculated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "properties_hs_object_id": { + "type": ["null", "string"] + }, + "properties_hs_target_amount": { + "type": ["null", "string"] + }, "createdAt": { "type": ["null", "string"], "format": "date-time" diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/marketing_emails.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/marketing_emails.json index 735b16fc8d82..aa15243a3daf 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/marketing_emails.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/marketing_emails.json @@ -32,6 +32,9 @@ "absoluteUrl": { "type": ["null", "string"] }, + "aifeatures": { + "type": ["null", "string"] + }, "allEmailCampaignIds": { "type": ["null", "array"], "items": { diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/property_history.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/property_history.json deleted file mode 100644 index de0f9bbe9c86..000000000000 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/property_history.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "string"] - }, - "source-type": { - "type": ["null", "string"] - }, - "source-id": { - "type": ["null", "string"] - }, - "source-label": { - "type": ["null", "string"] - }, - "updated-by-user-id": { - "type": ["null", "integer"] - }, - "timestamp": { - "type": ["null", "integer"] - }, - "selected": { - "type": ["null", "boolean"] - }, - "property": { - "type": ["null", "string"] - }, - "vid": { - "type": ["null", "integer"] - }, - "source-vids": { - "type": ["array", "null"], - "items": { - "type": ["null", "integer"] - } - } - } -} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/shared/default_event_properties.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/shared/default_event_properties.json new file mode 100644 index 000000000000..569707354cbc --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/shared/default_event_properties.json @@ -0,0 +1,118 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "properties_hs_asset_description": { + "type": ["null", "string"] + }, + "properties_hs_asset_type": { + "type": ["null", "string"] + }, + "properties_hs_browser": { + "type": ["null", "string"] + }, + "properties_hs_campaign_id": { + "type": ["null", "string"] + }, + "properties_hs_city": { + "type": ["null", "string"] + }, + "properties_hs_country": { + "type": ["null", "string"] + }, + "properties_hs_device_name": { + "type": ["null", "string"] + }, + "properties_hs_device_type": { + "type": ["null", "string"] + }, + "properties_hs_element_class": { + "type": ["null", "string"] + }, + "properties_hs_element_id": { + "type": ["null", "string"] + }, + "properties_hs_element_text": { + "type": ["null", "string"] + }, + "properties_hs_language": { + "type": ["null", "string"] + }, + "properties_hs_link_href": { + "type": ["null", "string"] + }, + "properties_hs_operating_system": { + "type": ["null", "string"] + }, + "properties_hs_operating_version": { + "type": ["null", "string"] + }, + "properties_hs_page_content_type": { + "type": ["null", "string"] + }, + "properties_hs_page_id": { + "type": ["null", "string"] + }, + "properties_hs_page_title": { + "type": ["null", "string"] + }, + "properties_hs_page_url": { + "type": ["null", "string"] + }, + "properties_hs_parent_module_id": { + "type": ["null", "string"] + }, + "properties_hs_referrer": { + "type": ["null", "string"] + }, + "properties_hs_region": { + "type": ["null", "string"] + }, + "properties_hs_screen_height": { + "type": ["null", "string"] + }, + "properties_hs_screen_width": { + "type": ["null", "string"] + }, + "properties_hs_touchpoint_source": { + "type": ["null", "string"] + }, + "properties_hs_tracking_name": { + "type": ["null", "string"] + }, + "properties_hs_user_agent": { + "type": ["null", "string"] + }, + "properties_hs_utm_campaign": { + "type": ["null", "string"] + }, + "properties_hs_utm_content": { + "type": ["null", "string"] + }, + "properties_hs_utm_medium": { + "type": ["null", "string"] + }, + "properties_hs_utm_source": { + "type": ["null", "string"] + }, + "properties_hs_utm_term": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "objectId": { + "type": ["null", "string"] + }, + "objectType": { + "type": ["null", "string"] + }, + "eventType": { + "type": ["null", "string"] + }, + "occurredAt": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/workflows.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/workflows.json index 4d89d65055cb..228121336cd7 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/workflows.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/workflows.json @@ -43,6 +43,24 @@ } } }, + "contactListIds_enrolled": { + "type": ["null", "integer"] + }, + "contactListIds_active": { + "type": ["null", "integer"] + }, + "contactListIds_completed": { + "type": ["null", "integer"] + }, + "contactListIds_succeeded": { + "type": ["null", "integer"] + }, + "contactListIds_steps": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, "lastUpdatedByUserId": { "type": ["null", "integer"] }, diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py index 2d1c6a642dca..97d42860a344 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/source.py @@ -17,34 +17,49 @@ API, Campaigns, Companies, + CompaniesPropertyHistory, + CompaniesWebAnalytics, ContactLists, Contacts, ContactsListMemberships, ContactsMergedAudit, + ContactsPropertyHistory, + ContactsWebAnalytics, CustomObject, DealPipelines, Deals, DealsArchived, + DealsPropertyHistory, + DealsWebAnalytics, EmailEvents, EmailSubscriptions, Engagements, EngagementsCalls, + EngagementsCallsWebAnalytics, EngagementsEmails, + EngagementsEmailsWebAnalytics, EngagementsMeetings, + EngagementsMeetingsWebAnalytics, EngagementsNotes, + EngagementsNotesWebAnalytics, EngagementsTasks, + EngagementsTasksWebAnalytics, Forms, FormSubmissions, Goals, + GoalsWebAnalytics, LineItems, + LineItemsWebAnalytics, MarketingEmails, Owners, OwnersArchived, Products, - PropertyHistory, + ProductsWebAnalytics, SubscriptionChanges, TicketPipelines, Tickets, + TicketsWebAnalytics, + WebAnalyticsStream, Workflows, ) @@ -123,13 +138,35 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Owners(**common_params), OwnersArchived(**common_params), Products(**common_params), - PropertyHistory(**common_params), + ContactsPropertyHistory(**common_params), + CompaniesPropertyHistory(**common_params), + DealsPropertyHistory(**common_params), SubscriptionChanges(**common_params), Tickets(**common_params), TicketPipelines(**common_params), Workflows(**common_params), ] + enable_experimental_streams = "enable_experimental_streams" in config and config["enable_experimental_streams"] + + if enable_experimental_streams: + streams.extend( + [ + ContactsWebAnalytics(**common_params), + CompaniesWebAnalytics(**common_params), + DealsWebAnalytics(**common_params), + TicketsWebAnalytics(**common_params), + EngagementsCallsWebAnalytics(**common_params), + EngagementsEmailsWebAnalytics(**common_params), + EngagementsMeetingsWebAnalytics(**common_params), + EngagementsNotesWebAnalytics(**common_params), + EngagementsTasksWebAnalytics(**common_params), + GoalsWebAnalytics(**common_params), + LineItemsWebAnalytics(**common_params), + ProductsWebAnalytics(**common_params), + ] + ) + api = API(credentials=credentials) if api.is_oauth2(): authenticator = api.get_authenticator() @@ -151,8 +188,42 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: available_streams.extend(self.get_custom_object_streams(api=api, common_params=common_params)) + if enable_experimental_streams: + custom_objects_web_analytics_streams = self.get_web_analytics_custom_objects_stream( + custom_object_stream_instances=self.get_custom_object_streams(api=api, common_params=common_params), + common_params=common_params, + ) + available_streams.extend(custom_objects_web_analytics_streams) + return available_streams def get_custom_object_streams(self, api: API, common_params: Mapping[str, Any]): - for (entity, fully_qualified_name, schema) in api.get_custom_objects_metadata(): - yield CustomObject(entity=entity, schema=schema, fully_qualified_name=fully_qualified_name, **common_params) + for entity, fully_qualified_name, schema, custom_properties in api.get_custom_objects_metadata(): + yield CustomObject( + entity=entity, + schema=schema, + fully_qualified_name=fully_qualified_name, + custom_properties=custom_properties, + **common_params, + ) + + def get_web_analytics_custom_objects_stream( + self, custom_object_stream_instances: List[CustomObject], common_params: Any + ) -> WebAnalyticsStream: + for custom_object_stream_instance in custom_object_stream_instances: + + def __init__(self, **kwargs: Any): + parent = custom_object_stream_instance.__class__( + entity=custom_object_stream_instance.entity, + schema=custom_object_stream_instance.schema, + fully_qualified_name=custom_object_stream_instance.fully_qualified_name, + custom_properties=custom_object_stream_instance.custom_properties, + **common_params, + ) + super(self.__class__, self).__init__(parent=parent, **kwargs) + + custom_web_analytics_stream_class = type( + f"{custom_object_stream_instance.name.capitalize()}WebAnalytics", (WebAnalyticsStream,), {"__init__": __init__} + ) + + yield custom_web_analytics_stream_class(**common_params) diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.yaml b/airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.yaml index 8dd2b446cc2c..3510c2b97d3b 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.yaml +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/spec.yaml @@ -86,6 +86,11 @@ connectionSpecification: if you need help finding this token. type: string airbyte_secret: true + enable_experimental_streams: + title: Enable experimental streams + description: If enabled then experimental streams become available for sync. + type: boolean + default: false advanced_auth: auth_flow_type: oauth2.0 predicate_key: diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py index d272bff24316..e388558f0682 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py @@ -8,7 +8,7 @@ import time from abc import ABC, abstractmethod from datetime import timedelta -from functools import cached_property, lru_cache +from functools import cached_property, lru_cache, reduce from http import HTTPStatus from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Set, Tuple, Union @@ -16,21 +16,31 @@ import pendulum as pendulum import requests from airbyte_cdk.entrypoint import logger -from airbyte_cdk.models import FailureType +from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.models.airbyte_protocol import SyncMode from airbyte_cdk.sources import Source from airbyte_cdk.sources.streams import IncrementalMixin, Stream from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.core import StreamData -from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator, TokenAuthenticator +from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from airbyte_cdk.utils import AirbyteTracedException from requests import HTTPError, codes from source_hubspot.constants import OAUTH_CREDENTIALS, PRIVATE_APP_CREDENTIALS from source_hubspot.errors import HubspotAccessDenied, HubspotInvalidAuth, HubspotRateLimited, HubspotTimeout, InvalidStartDateConfigError -from source_hubspot.helpers import APIv1Property, APIv3Property, GroupByKey, IRecordPostProcessor, IURLPropertyRepresentation, StoreAsIs +from source_hubspot.helpers import ( + APIPropertiesWithHistory, + APIv1Property, + APIv2Property, + APIv3Property, + GroupByKey, + IRecordPostProcessor, + IURLPropertyRepresentation, + StoreAsIs, +) # we got this when provided API Token has incorrect format CLOUDFLARE_ORIGIN_DNS_ERROR = 530 @@ -74,6 +84,35 @@ def retry_token_expired_handler(**kwargs): ) +class RecordUnnester: + def __init__(self, fields: Optional[List[str]] = None): + self.fields = fields or [] + + def unnest(self, records: Iterable[MutableMapping[str, Any]]) -> Iterable[MutableMapping[str, Any]]: + """ + In order to not make the users query their destinations for complicated json fields, duplicate some nested data as top level fields. + For instance: + {"id": 1, "updatedAt": "2020-01-01", "properties": {"hs_note_body": "World's best boss", "hs_created_by": "Michael Scott"}} + becomes + { + "id": 1, + "updatedAt": "2020-01-01", + "properties": {"hs_note_body": "World's best boss", "hs_created_by": "Michael Scott"}, + "properties_hs_note_body": "World's best boss", + "properties_hs_created_by": "Michael Scott" + } + """ + + for record in records: + fields_to_unnest = self.fields + ["properties"] + data_to_unnest = {field: record.get(field, {}) for field in fields_to_unnest} + unnested_data = { + f"{top_level_name}_{name}": value for (top_level_name, data) in data_to_unnest.items() for (name, value) in data.items() + } + final = {**record, **unnested_data} + yield final + + def retry_connection_handler(**kwargs): """Retry helper, log each attempt""" @@ -229,16 +268,16 @@ def get_custom_objects_metadata(self) -> Iterable[Tuple[str, str, Mapping[str, A if not response.ok or "results" not in data: self.logger.warn(self._parse_and_handle_errors(response)) return () + for metadata in data["results"]: + properties = self.get_properties(raw_schema=metadata) + schema = self.generate_schema(properties) + yield metadata["name"], metadata["fullyQualifiedName"], schema, properties - return ( - (metadata["name"], metadata["fullyQualifiedName"], self.generate_schema(raw_schema=metadata)) for metadata in data["results"] - ) - - def generate_schema(self, raw_schema: Mapping[str, Any]) -> Mapping[str, Any]: - properties = {} - for field in raw_schema["properties"]: - properties[field["name"]] = self._field_to_property_schema(field) + def get_properties(self, raw_schema: Mapping[str, Any]) -> Mapping[str, Any]: + return {field["name"]: self._field_to_property_schema(field) for field in raw_schema["properties"]} + def generate_schema(self, properties: Mapping[str, Any]) -> Mapping[str, Any]: + unnested_properties = {f"properties_{property_name}": property_value for (property_name, property_value) in properties.items()} schema = { "$schema": "http://json-schema.org/draft-07/schema#", "type": ["null", "object"], @@ -249,6 +288,7 @@ def generate_schema(self, raw_schema: Mapping[str, Any]) -> Mapping[str, Any]: "updatedAt": {"type": ["null", "string"], "format": "date-time"}, "archived": {"type": ["null", "boolean"]}, "properties": {"type": ["null", "object"], "properties": properties}, + **unnested_properties, }, } @@ -292,6 +332,11 @@ class Stream(HttpStream, ABC): denormalize_records: bool = False # one record from API response can result in multiple records emitted granted_scopes: Set = None properties_scopes: Set = None + unnest_fields: Optional[List[str]] = None + + @cached_property + def record_unnester(self): + return RecordUnnester(self.unnest_fields) @property @abstractmethod @@ -320,6 +365,7 @@ def path( stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, + properties: IURLPropertyRepresentation = None, ) -> str: return self.url @@ -328,6 +374,8 @@ def _property_wrapper(self) -> IURLPropertyRepresentation: properties = list(self.properties.keys()) if "v1" in self.url: return APIv1Property(properties) + if "v2" in self.url: + return APIv2Property(properties) return APIv3Property(properties) def __init__(self, api: API, start_date: Union[str, pendulum.datetime], credentials: Mapping[str, Any] = None, **kwargs): @@ -366,9 +414,18 @@ def request_headers( def get_json_schema(self) -> Mapping[str, Any]: json_schema = super().get_json_schema() if self.properties: - json_schema["properties"]["properties"] = {"type": "object", "properties": self.properties} + properties = {"properties": {"type": "object", "properties": self.properties}} + unnested_properties = { + f"properties_{property_name}": property_value for (property_name, property_value) in self.properties.items() + } + default_props = json_schema["properties"] + json_schema["properties"] = {**default_props, **properties, **unnested_properties} return json_schema + def update_request_properties(self, params: Mapping[str, Any], properties: IURLPropertyRepresentation) -> None: + if properties: + params.update(properties.as_url_param()) + @retry_token_expired_handler(max_tries=5) def handle_request( self, @@ -379,11 +436,10 @@ def handle_request( ) -> requests.Response: request_headers = self.request_headers(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) request_params = self.request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - if properties: - request_params.update(properties.as_url_param()) + self.update_request_properties(request_params, properties) request = self._create_prepared_request( - path=self.path(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + path=self.path(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token, properties=properties), headers=dict(request_headers, **self.authenticator.get_auth_header()), params=request_params, json=self.request_body_json(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), @@ -461,7 +517,7 @@ def read_records( if self.filter_old_records: records = self._filter_old_records(records) - yield from records + yield from self.record_unnester.unnest(records) next_page_token = self.next_page_token(response) if not next_page_token: @@ -863,6 +919,7 @@ def path( stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, + properties: IURLPropertyRepresentation = None, ) -> str: return f"/crm/v4/associations/{self.parent_stream.entity}/{stream_slice}/batch/read" @@ -1112,6 +1169,7 @@ def read_records( ) records = self._flat_associations(records) records = self._filter_old_records(records) + records = self.record_unnester.unnest(records) for record in records: cursor = self._field_to_datetime(record[self.updated_at_field]) @@ -1256,6 +1314,7 @@ class Campaigns(ClientSideIncrementalStream): cursor_field_datetime_format = "x" primary_key = "id" scopes = {"crm.lists.read"} + unnest_fields = ["counters"] def read_records( self, @@ -1267,7 +1326,7 @@ def read_records( for row in super().read_records(sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state): record, response = self._api.get(f"/email/public/v1/campaigns/{row['id']}") if self.filter_by_state(stream_state=stream_state, record=row): - yield {**row, **record} + yield from self.record_unnester.unnest([{**row, **record}]) class ContactLists(IncrementalStream): @@ -1286,6 +1345,7 @@ class ContactLists(IncrementalStream): primary_key = "listId" need_chunk = False scopes = {"crm.lists.read"} + unnest_fields = ["metaData"] class ContactsListMemberships(Stream): @@ -1452,6 +1512,8 @@ class EngagementsAll(EngagementsABC): Note: Returns all engagements records ordered by 'createdAt' (not 'lastUpdated') field """ + unnest_fields = ["associations", "metadata"] + @property def url(self): return "/engagements/v1/engagements/paged" @@ -1473,6 +1535,7 @@ class EngagementsRecent(EngagementsABC): total_records_limit = 10000 last_days_limit = 29 + unnest_fields = ["associations", "metadata"] @property def url(self): @@ -1528,6 +1591,8 @@ class Engagements(EngagementsABC, IncrementalStream): - EngagementsAll which extracts all records, but supports filter on connector side """ + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + @property def url(self): return "/engagements/v1/engagements/paged" @@ -1620,6 +1685,7 @@ def path( stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, + properties: IURLPropertyRepresentation = None, ) -> str: return f"{self.url}/{stream_slice['form_id']}" @@ -1713,27 +1779,90 @@ def request_params( return params -class PropertyHistory(Stream): +class PropertyHistory(ClientSideIncrementalStream): """Contacts Endpoint, API v1 Is used to get all Contacts and the history of their respective Properties. Whenever a property is changed it is added here. Docs: https://legacydocs.hubspot.com/docs/methods/contacts/get_contacts """ - more_key = "has-more" - url = "/contacts/v1/lists/all/contacts/all" updated_at_field = "timestamp" created_at_field = "timestamp" - entity = "contacts" - data_field = "contacts" - page_field = "vid-offset" - page_filter = "vidOffset" - primary_key = "vid" denormalize_records = True - limit_field = "count" limit = 100 - scopes = {"crm.objects.contacts.read"} - properties_scopes = {"crm.schemas.contacts.read"} + + @property + @abstractmethod + def page_field(self) -> str: + """Page offset field""" + + @property + @abstractmethod + def limit_field(self) -> str: + """Limit query field""" + + @property + @abstractmethod + def page_filter(self) -> str: + """Query param name that indicates page offset""" + + @property + @abstractmethod + def more_key(self) -> str: + """Field that indicates that are more records""" + + @property + @abstractmethod + def scopes(self) -> set: + """Scopes needed to get access to CRM object""" + + @property + @abstractmethod + def properties_scopes(self) -> set: + """Scopes needed to get access to CRM object properies""" + + @property + @abstractmethod + def entity(self) -> str: + """ + CRM object entity name. + This is usually a part of some URL or key that contains data in response + """ + + @property + @abstractmethod + def primary_key(self) -> str: + """Indicates a field name which is considered to be a primary key of the stream""" + + @property + @abstractmethod + def entity_primary_key(self) -> str: + """Indicates a field name which is considered to be a primary key of the parent entity""" + + @property + @abstractmethod + def additional_keys(self) -> list: + """The root keys to be placed into each record while iterating through versions""" + + @property + @abstractmethod + def last_modified_date_field_name(self) -> str: + """Last modified date field name""" + + @property + @abstractmethod + def data_field(self) -> str: + """A key that contains data in response""" + + @property + @abstractmethod + def url(self) -> str: + """An API url""" + + @property + def cursor_field_datetime_format(self) -> str: + """Cursor value expected to be a timestamp in milliseconds""" + return "x" def request_params( self, @@ -1749,11 +1878,12 @@ def request_params( def _transform(self, records: Iterable) -> Iterable: for record in records: properties = record.get("properties") - vid = record.get("vid") + primary_key = record.get(self.entity_primary_key) + additional_keys = {additional_key: record.get(additional_key) for additional_key in self.additional_keys} value_dict: Dict - for key, value_dict in properties.items(): + for property_name, value_dict in properties.items(): versions = value_dict.get("versions") - if key == "lastmodifieddate": + if property_name == self.last_modified_date_field_name: # Skipping the lastmodifieddate since it only returns the value # when one field of a contact was changed no matter which # field was changed. It therefore creates overhead, since for @@ -1762,9 +1892,207 @@ def _transform(self, records: Iterable) -> Iterable: continue if versions: for version in versions: - version["property"] = key - version["vid"] = vid - yield version + version["property"] = property_name + version[self.entity_primary_key] = primary_key + yield version | additional_keys + + +class ContactsPropertyHistory(PropertyHistory): + @property + def scopes(self): + return {"crm.objects.contacts.read"} + + @property + def properties_scopes(self): + return {"crm.schemas.contacts.read"} + + @property + def page_field(self) -> str: + return "vid-offset" + + @property + def limit_field(self) -> str: + return "count" + + @property + def page_filter(self) -> str: + return "vidOffset" + + @property + def more_key(self) -> str: + return "has-more" + + @property + def entity(self): + return "contacts" + + @property + def entity_primary_key(self) -> list: + return "vid" + + @property + def primary_key(self) -> list: + return ["vid", "property", "timestamp"] + + @property + def additional_keys(self) -> list: + return ["portal-id", "is-contact", "canonical-vid"] + + @property + def last_modified_date_field_name(self): + return "lastmodifieddate" + + @property + def data_field(self): + return "contacts" + + @property + def url(self): + return "/contacts/v1/lists/all/contacts/all" + + +class CompaniesPropertyHistory(PropertyHistory): + @cached_property + def _property_wrapper(self) -> IURLPropertyRepresentation: + properties = list(self.properties.keys()) + return APIPropertiesWithHistory(properties=properties) + + @property + def scopes(self) -> set: + return {"crm.objects.companies.read"} + + @property + def properties_scopes(self) -> set: + return {"crm.schemas.companies.read"} + + @property + def page_field(self) -> str: + return "offset" + + @property + def limit_field(self) -> str: + return "limit" + + @property + def page_filter(self) -> str: + return "offset" + + @property + def more_key(self) -> str: + return "hasMore" + + @property + def entity(self) -> str: + return "companies" + + @property + def entity_primary_key(self) -> list: + return "companyId" + + @property + def primary_key(self) -> list: + return ["companyId", "property", "timestamp"] + + @property + def additional_keys(self) -> list: + return ["portalId", "isDeleted"] + + @property + def last_modified_date_field_name(self) -> str: + return "hs_lastmodifieddate" + + @property + def data_field(self) -> str: + return "companies" + + @property + def url(self) -> str: + return "/companies/v2/companies/paged" + + def update_request_properties(self, params: Mapping[str, Any], properties: IURLPropertyRepresentation) -> None: + pass + + def path( + self, + *, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + properties: IURLPropertyRepresentation = None, + ) -> str: + return f"{self.url}?{properties.as_url_param()}" + + +class DealsPropertyHistory(PropertyHistory): + @cached_property + def _property_wrapper(self) -> IURLPropertyRepresentation: + properties = list(self.properties.keys()) + return APIPropertiesWithHistory(properties=properties) + + @property + def scopes(self) -> set: + return {"crm.objects.deals.read"} + + @property + def properties_scopes(self): + return {"crm.schemas.deals.read"} + + @property + def page_field(self) -> str: + return "offset" + + @property + def limit_field(self) -> str: + return "limit" + + @property + def page_filter(self) -> str: + return "offset" + + @property + def more_key(self) -> str: + return "hasMore" + + @property + def entity(self) -> set: + return "deals" + + @property + def entity_primary_key(self) -> list: + return "dealId" + + @property + def primary_key(self) -> list: + return ["dealId", "property", "timestamp"] + + @property + def additional_keys(self) -> list: + return ["portalId", "isDeleted"] + + @property + def last_modified_date_field_name(self) -> str: + return "hs_lastmodifieddate" + + @property + def data_field(self) -> str: + return "deals" + + @property + def url(self) -> str: + return "/deals/v1/deal/paged" + + def update_request_properties(self, params: Mapping[str, Any], properties: IURLPropertyRepresentation) -> None: + pass + + def path( + self, + *, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + properties: IURLPropertyRepresentation = None, + ) -> str: + return f"{self.url}?{properties.as_url_param()}" class SubscriptionChanges(IncrementalStream): @@ -1791,6 +2119,7 @@ class Workflows(ClientSideIncrementalStream): cursor_field_datetime_format = "x" primary_key = "id" scopes = {"automation"} + unnest_fields = ["contactListIds"] class Companies(CRMSearchStream): @@ -1813,19 +2142,12 @@ class ContactsMergedAudit(Stream): url = "/contacts/v1/contact/vids/batch/" updated_at_field = "timestamp" scopes = {"crm.objects.contacts.read"} + unnest_fields = ["merged_from_email", "merged_to_email"] def __init__(self, **kwargs): super().__init__(**kwargs) self.config = kwargs - def get_json_schema(self) -> Mapping[str, Any]: - """Override get_json_schema defined in Stream class - Final object does not have properties field - We return JSON schema as defined in : - source_hubspot/schemas/contacts_merged_audit.json - """ - return super(Stream, self).get_json_schema() - def stream_slices( self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None, **kwargs ) -> Iterable[Mapping[str, Any]]: @@ -1841,7 +2163,7 @@ def stream_slices( contacts.filter_old_records = False for contact in contacts.read_records(sync_mode=SyncMode.full_refresh): - if contact["properties"].get("hs_merged_object_ids"): + if contact.get("properties_hs_merged_object_ids"): contact_batch.append(contact["id"]) if len(contact_batch) == max_contacts: @@ -1956,11 +2278,12 @@ class CustomObject(CRMSearchStream, ABC): primary_key = "id" scopes = {"crm.schemas.custom.read", "crm.objects.custom.read"} - def __init__(self, entity: str, schema: Mapping[str, Any], fully_qualified_name: str, **kwargs): + def __init__(self, entity: str, schema: Mapping[str, Any], fully_qualified_name: str, custom_properties: Mapping[str, Any], **kwargs): super().__init__(**kwargs) self.entity = entity self.schema = schema self.fully_qualified_name = fully_qualified_name + self.custom_properties = custom_properties @property def name(self) -> str: @@ -1971,8 +2294,8 @@ def get_json_schema(self) -> Mapping[str, Any]: @property def properties(self) -> Mapping[str, Any]: - # do not make extra api queries - return self.get_json_schema()["properties"]["properties"]["properties"] + # do not make extra api calls + return self.custom_properties class EmailSubscriptions(Stream): @@ -1985,3 +2308,230 @@ class EmailSubscriptions(Stream): primary_key = "id" scopes = {"content"} filter_old_records = False + + +class WebAnalyticsStream(IncrementalMixin, HttpSubStream, Stream): + """ + A base class for Web Analytics API + Docs: https://developers.hubspot.com/docs/api/events/web-analytics + """ + + transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) + + cursor_field: str = "occurredAt" + slicing_period: int = 30 + state_checkpoint_interval: int = 100 + + # Set this flag to `False` as we don't need client side incremental logic + filter_old_records: bool = False + + def __init__(self, parent: HttpStream, **kwargs: Any): + super().__init__(parent, **kwargs) + self._state: MutableMapping[str, Any] = {} + + @property + def scopes(self) -> Set[str]: + return getattr(self.parent, "scopes") | {"oauth"} + + @property + def state(self) -> MutableMapping[str, Any]: + return self._state + + @state.setter + def state(self, value: MutableMapping[str, Any]): + self._state = value + + def get_json_schema(self): + raw_schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "$ref": "default_event_properties.json", + } + return ResourceSchemaLoader("source_hubspot")._resolve_schema_references(raw_schema=raw_schema) + + def get_updated_state( + self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any] + ) -> MutableMapping[str, Any]: + """ + Returns current state. At the moment when this method is called by sources we already have updated state stored in self._state, + because it is calculated each time we produce new record + """ + return self.state + + def get_latest_state( + self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any] + ) -> MutableMapping[str, Any]: + """ + State is a composite object that keeps latest datetime of an event for each parent object: + { + "100": {"occurredAt": "2023-11-24T23:23:23.000Z"}, + "200": {"occurredAt": "2023-11-24T23:23:23.000Z"}, + ..., + "": {"occurredAt": ""} + } + """ + if latest_record["objectId"] in current_stream_state: + # Not sure whether records are sorted and what kind of sorting is used, + # so trying to keep higher datetime value + latest_datetime = max( + current_stream_state[latest_record["objectId"]][self.cursor_field], + latest_record[self.cursor_field], + ) + else: + latest_datetime = latest_record[self.cursor_field] + return {**self.state, latest_record["objectId"]: {self.cursor_field: latest_datetime}} + + def records_transformer(self, records: Iterable[Mapping[str, Any]]) -> Iterable[Mapping[str, Any]]: + for record in records: + # We don't need `properties` as all the fields are unnested to the root + if "properties" in record: + record.pop("properties") + yield record + + def stream_slices( + self, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + now = pendulum.now(tz="UTC") + for parent_slice in super().stream_slices(sync_mode, cursor_field, stream_state): + + object_id = parent_slice["parent"][self.object_id_field] + + # Take the initial datetime either form config or from state depending whichever value is higher + # In case when state is detected add a 1 millisecond to avoid duplicates from previous sync + from_datetime = ( + max( + self._start_date, + self._field_to_datetime(self.state[object_id][self.cursor_field]) + timedelta(milliseconds=1), + ) + if object_id in self.state + else self._start_date + ) + + # Making slices of given slice period + while ( + (to_datetime := min(from_datetime.add(days=self.slicing_period), now)) <= now + and from_datetime != now + and from_datetime <= to_datetime + ): + yield { + "occurredAfter": from_datetime.to_iso8601_string(), + "occurredBefore": to_datetime.to_iso8601_string(), + "objectId": object_id, + "objectType": self.object_type, + } + # Shift time window to the next checkpoint interval + from_datetime = to_datetime + + @property + def object_type(self) -> str: + """ + The name of the CRM Object for which Web Analytics is requested + List of available CRM objects: https://developers.hubspot.com/docs/api/crm/understanding-the-crm + """ + return getattr(self.parent, "entity") + + @property + def object_id_field(self) -> str: + """ + The ID field name of the CRM Object for which Web Analytics is requested + List of available CRM objects: https://developers.hubspot.com/docs/api/crm/understanding-the-crm + """ + return getattr(self.parent, "primary_key") + + @property + def url(self) -> str: + return "/events/v3/events" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + """ + Preparing the request params dictionary for the following query string: + ?objectType= + &objectId= + &occurredAfter= + &occurredBefore= + """ + params = super().request_params(stream_state, stream_slice, next_page_token) + return params | stream_slice + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Mapping[str, Any]]: + record_generator = super().read_records(sync_mode, cursor_field, stream_slice, stream_state) + + record_generator = self.records_transformer(record_generator) + for record in record_generator: + yield record + # Update state with latest datetime each time we have a record + if sync_mode == SyncMode.incremental: + self.state = self.get_latest_state(self.state, record) + + +class ContactsWebAnalytics(WebAnalyticsStream): + def __init__(self, **kwargs: Any): + super().__init__(parent=Contacts(**kwargs), **kwargs) + + +class CompaniesWebAnalytics(WebAnalyticsStream): + def __init__(self, **kwargs: Any): + super().__init__(parent=Companies(**kwargs), **kwargs) + + +class DealsWebAnalytics(WebAnalyticsStream): + def __init__(self, **kwargs: Any): + super().__init__(parent=Deals(**kwargs), **kwargs) + + +class TicketsWebAnalytics(WebAnalyticsStream): + def __init__(self, **kwargs: Any): + super().__init__(parent=Tickets(**kwargs), **kwargs) + + +class EngagementsCallsWebAnalytics(WebAnalyticsStream): + def __init__(self, **kwargs: Any): + super().__init__(parent=EngagementsCalls(**kwargs), **kwargs) + + +class EngagementsEmailsWebAnalytics(WebAnalyticsStream): + def __init__(self, **kwargs: Any): + super().__init__(parent=EngagementsEmails(**kwargs), **kwargs) + + +class EngagementsMeetingsWebAnalytics(WebAnalyticsStream): + def __init__(self, **kwargs: Any): + super().__init__(parent=EngagementsMeetings(**kwargs), **kwargs) + + +class EngagementsNotesWebAnalytics(WebAnalyticsStream): + def __init__(self, **kwargs: Any): + super().__init__(parent=EngagementsNotes(**kwargs), **kwargs) + + +class EngagementsTasksWebAnalytics(WebAnalyticsStream): + def __init__(self, **kwargs: Any): + super().__init__(parent=EngagementsTasks(**kwargs), **kwargs) + + +class GoalsWebAnalytics(WebAnalyticsStream): + def __init__(self, **kwargs: Any): + super().__init__(parent=Goals(**kwargs), **kwargs) + + +class LineItemsWebAnalytics(WebAnalyticsStream): + def __init__(self, **kwargs: Any): + super().__init__(parent=LineItems(**kwargs), **kwargs) + + +class ProductsWebAnalytics(WebAnalyticsStream): + def __init__(self, **kwargs: Any): + super().__init__(parent=Products(**kwargs), **kwargs) + + +class FeedbackSubmissionsWebAnalytics(WebAnalyticsStream): + def __init__(self, **kwargs: Any): + super().__init__(parent=FeedbackSubmissions(**kwargs), **kwargs) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py index 0c4ff9c8f613..08fe337e61c8 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/conftest.py @@ -34,17 +34,42 @@ def common_params_fixture(config): @pytest.fixture(name="config_invalid_client_id") def config_invalid_client_id_fixture(): - return {"start_date": "2021-01-10T00:00:00Z", "credentials": {"credentials_title": "OAuth Credentials", "client_id": "invalid_client_id", "client_secret": "invalid_client_secret", "access_token": "test_access_token", "refresh_token": "test_refresh_token"}} + return { + "start_date": "2021-01-10T00:00:00Z", + "credentials": { + "credentials_title": "OAuth Credentials", + "client_id": "invalid_client_id", + "client_secret": "invalid_client_secret", + "access_token": "test_access_token", + "refresh_token": "test_refresh_token", + }, + } @pytest.fixture(name="config") def config_fixture(): - return {"start_date": "2021-01-10T00:00:00Z", "credentials": {"credentials_title": "Private App Credentials", "access_token": "test_access_token"}} + return { + "start_date": "2021-01-10T00:00:00Z", + "credentials": {"credentials_title": "Private App Credentials", "access_token": "test_access_token"}, + "enable_experimental_streams": False + } + + +@pytest.fixture(name="config_experimental") +def config_eperimantal_fixture(): + return { + "start_date": "2021-01-10T00:00:00Z", + "credentials": {"credentials_title": "Private App Credentials", "access_token": "test_access_token"}, + "enable_experimental_streams": True + } @pytest.fixture(name="config_invalid_date") def config_invalid_date_fixture(): - return {"start_date": "2000-00-00T00:00:00Z", "credentials": {"credentials_title": "Private App Credentials", "access_token": "test_access_token"}} + return { + "start_date": "2000-00-00T00:00:00Z", + "credentials": {"credentials_title": "Private App Credentials", "access_token": "test_access_token"}, + } @pytest.fixture(name="some_credentials") diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py index 14a5737ab6c4..6989843ff717 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_source.py @@ -85,7 +85,30 @@ def test_streams(requests_mock, config): streams = SourceHubspot().streams(config) - assert len(streams) == 30 + assert len(streams) == 32 + + +@mock.patch("source_hubspot.source.SourceHubspot.get_custom_object_streams") +def test_streams(requests_mock, config_experimental): + + streams = SourceHubspot().streams(config_experimental) + + assert len(streams) == 44 + + +def test_custom_streams(config_experimental): + custom_object_stream_instances = [ + MagicMock() + ] + streams = SourceHubspot().get_web_analytics_custom_objects_stream( + custom_object_stream_instances=custom_object_stream_instances, + common_params={ + "api": MagicMock(), + "start_date": "2021-01-01T00:00:00Z", + "credentials": config_experimental["credentials"] + } + ) + assert len(list(streams)) == 1 def test_check_credential_title_exception(config): @@ -120,7 +143,7 @@ def test_cast_datetime(common_params, caplog): "type": "LOG", "log": { "level": "WARN", - "message": f"Couldn't parse date/datetime string in {field_name}, trying to parse timestamp... Field value: {field_value}. Ex: argument of type 'DateTime' is not iterable", + "message": f"Couldn't parse date/datetime string in {field_name}, trying to parse timestamp... Field value: {field_value}. Ex: argument 'input': 'DateTime' object cannot be converted to 'PyString'", }, } assert expected_warining_message["log"]["message"] in caplog.text @@ -297,6 +320,8 @@ def test_stream_with_splitting_properties(self, requests_mock, api, fake_propert assert len(stream_records) == sum([len(ids) for ids in record_ids_paginated]) for record in stream_records: assert len(record["properties"]) == NUMBER_OF_PROPERTIES + properties = [field for field in record if field.startswith("properties_")] + assert len(properties) == NUMBER_OF_PROPERTIES def test_stream_with_splitting_properties_with_pagination(self, requests_mock, common_params, api, fake_properties_list): """ @@ -329,6 +354,8 @@ def test_stream_with_splitting_properties_with_pagination(self, requests_mock, c assert len(stream_records) == 5 for record in stream_records: assert len(record["properties"]) == NUMBER_OF_PROPERTIES + properties = [field for field in record if field.startswith("properties_")] + assert len(properties) == NUMBER_OF_PROPERTIES def test_stream_with_splitting_properties_with_new_record(self, requests_mock, common_params, api, fake_properties_list): """ @@ -451,7 +478,11 @@ def test_search_based_stream_should_not_attempt_to_get_more_than_10k_records(req requests_mock.register_uri("POST", test_stream.url, responses) test_stream._sync_mode = None requests_mock.register_uri("GET", "/properties/v2/company/properties", properties_response) - requests_mock.register_uri("POST", "/crm/v4/associations/company/contacts/batch/read", [{"status_code": 200, "json": {"results": []}}]) + requests_mock.register_uri( + "POST", + "/crm/v4/associations/company/contacts/batch/read", + [{"status_code": 200, "json": {"results": [{"from": {"id": "1"}, "to": [{"toObjectId": "2"}]}]}}] + ) records, _ = read_incremental(test_stream, {}) # The stream should not attempt to get more than 10K records. @@ -553,7 +584,7 @@ def test_engagements_stream_since_old_date(requests_mock, common_params, fake_pr "results": [{"engagement": {"id": f"{y}", "lastUpdated": old_date}} for y in range(100)], "hasMore": False, "offset": 0, - "total": 100 + "total": 100, }, "status_code": 200, } @@ -582,7 +613,7 @@ def test_engagements_stream_since_recent_date(requests_mock, common_params, fake "results": [{"engagement": {"id": f"{y}", "lastUpdated": recent_date}} for y in range(100)], "hasMore": False, "offset": 0, - "total": 100 + "total": 100, }, "status_code": 200, } @@ -613,7 +644,7 @@ def test_engagements_stream_since_recent_date_more_than_10k(requests_mock, commo "results": [{"engagement": {"id": f"{y}", "lastUpdated": recent_date}} for y in range(100)], "hasMore": False, "offset": 0, - "total": 10001 + "total": 10001, }, "status_code": 200, } @@ -673,3 +704,38 @@ def test_pagination_marketing_emails_stream(requests_mock, common_params): records = read_full_refresh(test_stream) # The stream should handle pagination correctly and output 600 records. assert len(records) == 600 + + +def test_get_granted_scopes(requests_mock, mocker): + authenticator = mocker.Mock() + authenticator.get_access_token.return_value = "the-token" + + expected_scopes = ["a", "b", "c"] + response = [ + {"json": {"scopes": expected_scopes}, "status_code": 200}, + ] + requests_mock.register_uri("GET", "https://api.hubapi.com/oauth/v1/access-tokens/the-token", response) + + actual_scopes = SourceHubspot().get_granted_scopes(authenticator) + + assert expected_scopes == actual_scopes + + +def test_streams_oauth_2_auth_no_suitable_scopes(requests_mock, mocker, config): + authenticator = mocker.Mock() + authenticator.get_access_token.return_value = "the-token" + + mocker.patch("source_hubspot.streams.API.is_oauth2", return_value=True) + mocker.patch("source_hubspot.streams.API.get_authenticator", return_value=authenticator) + + requests_mock.get("https://api.hubapi.com/crm/v3/schemas", json={}, status_code=200) + + expected_scopes = ["no.scopes.granted"] + response = [ + {"json": {"scopes": expected_scopes}, "status_code": 200}, + ] + requests_mock.register_uri("GET", "https://api.hubapi.com/oauth/v1/access-tokens/the-token", response) + + streams = SourceHubspot().streams(config) + + assert len(streams) == 0 diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py index bc7a150474cb..24f7991badfb 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py @@ -10,7 +10,10 @@ Companies, ContactLists, Contacts, + ContactsListMemberships, ContactsMergedAudit, + ContactsPropertyHistory, + ContactsWebAnalytics, CustomObject, DealPipelines, Deals, @@ -30,6 +33,7 @@ Owners, OwnersArchived, Products, + RecordUnnester, TicketPipelines, Tickets, Workflows, @@ -62,8 +66,7 @@ def test_updated_at_field_non_exist_handler(requests_mock, common_params, fake_p properties_response = [ { "json": [ - {"name": property_name, "type": "string", - "updatedAt": 1571085954360, "createdAt": 1565059306048} + {"name": property_name, "type": "string", "updatedAt": 1571085954360, "createdAt": 1565059306048} for property_name in fake_properties_list ], "status_code": 200, @@ -71,13 +74,11 @@ def test_updated_at_field_non_exist_handler(requests_mock, common_params, fake_p ] requests_mock.register_uri("GET", stream.url, responses) - requests_mock.register_uri( - "GET", "/properties/v2/contact/properties", properties_response) + requests_mock.register_uri("GET", "/properties/v2/contact/properties", properties_response) _, stream_state = read_incremental(stream, {}) - expected = int(pendulum.parse( - common_params["start_date"]).timestamp() * 1000) + expected = int(pendulum.parse(common_params["start_date"]).timestamp() * 1000) assert stream_state[stream.updated_at_field] == expected @@ -89,8 +90,7 @@ def test_updated_at_field_non_exist_handler(requests_mock, common_params, fake_p (Companies, "company", {"updatedAt": "2022-02-25T16:43:11Z"}), (ContactLists, "contact", {"updatedAt": "2022-02-25T16:43:11Z"}), (Contacts, "contact", {"updatedAt": "2022-02-25T16:43:11Z"}), - (ContactsMergedAudit, "contact", { - "updatedAt": "2022-02-25T16:43:11Z"}), + (ContactsMergedAudit, "contact", {"updatedAt": "2022-02-25T16:43:11Z"}), (Deals, "deal", {"updatedAt": "2022-02-25T16:43:11Z"}), (DealsArchived, "deal", {"archivedAt": "2022-02-25T16:43:11Z"}), (DealPipelines, "deal", {"updatedAt": 1675121674226}), @@ -98,8 +98,7 @@ def test_updated_at_field_non_exist_handler(requests_mock, common_params, fake_p (EmailSubscriptions, "", {"updatedAt": "2022-02-25T16:43:11Z"}), (EngagementsCalls, "calls", {"updatedAt": "2022-02-25T16:43:11Z"}), (EngagementsEmails, "emails", {"updatedAt": "2022-02-25T16:43:11Z"}), - (EngagementsMeetings, "meetings", { - "updatedAt": "2022-02-25T16:43:11Z"}), + (EngagementsMeetings, "meetings", {"updatedAt": "2022-02-25T16:43:11Z"}), (EngagementsNotes, "notes", {"updatedAt": "2022-02-25T16:43:11Z"}), (EngagementsTasks, "tasks", {"updatedAt": "2022-02-25T16:43:11Z"}), (Forms, "form", {"updatedAt": "2022-02-25T16:43:11Z"}), @@ -133,8 +132,7 @@ def test_streams_read(stream, endpoint, cursor_value, requests_mock, common_para properties_response = [ { "json": [ - {"name": property_name, "type": "string", - "updatedAt": 1571085954360, "createdAt": 1565059306048} + {"name": property_name, "type": "string", "updatedAt": 1571085954360, "createdAt": 1565059306048} for property_name in fake_properties_list ], "status_code": 200, @@ -144,14 +142,7 @@ def test_streams_read(stream, endpoint, cursor_value, requests_mock, common_para { "json": { stream.data_field: [ - { - "id": "test_id", - "created": "2022-06-25T16:43:11Z", - "properties": { - "hs_merged_object_ids": "test_id" - } - } - | cursor_value + {"id": "test_id", "created": "2022-06-25T16:43:11Z", "properties": {"hs_merged_object_ids": "test_id"}} | cursor_value ], } } @@ -159,16 +150,7 @@ def test_streams_read(stream, endpoint, cursor_value, requests_mock, common_para read_batch_contact_v1_response = [ { "json": { - "test_id": { - "vid": "test_id", - 'merge-audits': [ - { - 'canonical-vid': 2, - 'vid-to-merge': 5608, - 'timestamp': 1653322839932 - } - ] - } + "test_id": {"vid": "test_id", "merge-audits": [{"canonical-vid": 2, "vid-to-merge": 5608, "timestamp": 1653322839932}]} }, "status_code": 200, } @@ -179,15 +161,11 @@ def test_streams_read(stream, endpoint, cursor_value, requests_mock, common_para stream._sync_mode = None requests_mock.register_uri("GET", stream_url, responses) - requests_mock.register_uri( - "GET", "/crm/v3/objects/contact", contact_reponse) + requests_mock.register_uri("GET", "/crm/v3/objects/contact", contact_reponse) requests_mock.register_uri("GET", "/marketing/v3/forms", responses) - requests_mock.register_uri( - "GET", "/email/public/v1/campaigns/test_id", responses) - requests_mock.register_uri( - "GET", f"/properties/v2/{endpoint}/properties", properties_response) - requests_mock.register_uri( - "GET", "/contacts/v1/contact/vids/batch/", read_batch_contact_v1_response) + requests_mock.register_uri("GET", "/email/public/v1/campaigns/test_id", responses) + requests_mock.register_uri("GET", f"/properties/v2/{endpoint}/properties", properties_response) + requests_mock.register_uri("GET", "/contacts/v1/contact/vids/batch/", read_batch_contact_v1_response) records = read_full_refresh(stream) assert records @@ -204,8 +182,7 @@ def test_streams_read(stream, endpoint, cursor_value, requests_mock, common_para def test_common_error_retry(error_response, requests_mock, common_params, fake_properties_list): """Error once, check that we retry and not fail""" properties_response = [ - {"name": property_name, "type": "string", - "updatedAt": 1571085954360, "createdAt": 1565059306048} + {"name": property_name, "type": "string", "updatedAt": 1571085954360, "createdAt": 1565059306048} for property_name in fake_properties_list ] responses = [ @@ -228,8 +205,7 @@ def test_common_error_retry(error_response, requests_mock, common_params, fake_p } ], } - requests_mock.register_uri( - "GET", "/properties/v2/company/properties", responses) + requests_mock.register_uri("GET", "/properties/v2/company/properties", responses) stream._sync_mode = SyncMode.full_refresh stream_url = stream.url stream._sync_mode = None @@ -282,12 +258,9 @@ def test_client_side_incremental_stream(requests_mock, common_params, fake_prope { "json": { stream.data_field: [ - {"id": "test_id_1", "createdAt": "2022-03-25T16:43:11Z", - "updatedAt": "2023-01-30T23:46:36.287Z"}, - {"id": "test_id_2", "createdAt": "2022-03-25T16:43:11Z", - "updatedAt": latest_cursor_value}, - {"id": "test_id_3", "createdAt": "2022-03-25T16:43:11Z", - "updatedAt": "2023-02-20T23:46:36.287Z"}, + {"id": "test_id_1", "createdAt": "2022-03-25T16:43:11Z", "updatedAt": "2023-01-30T23:46:36.287Z"}, + {"id": "test_id_2", "createdAt": "2022-03-25T16:43:11Z", "updatedAt": latest_cursor_value}, + {"id": "test_id_3", "createdAt": "2022-03-25T16:43:11Z", "updatedAt": "2023-02-20T23:46:36.287Z"}, ], } } @@ -295,8 +268,7 @@ def test_client_side_incremental_stream(requests_mock, common_params, fake_prope properties_response = [ { "json": [ - {"name": property_name, "type": "string", "createdAt": "2023-01-30T23:46:24.355Z", - "updatedAt": "2023-01-30T23:46:36.287Z"} + {"name": property_name, "type": "string", "createdAt": "2023-01-30T23:46:24.355Z", "updatedAt": "2023-01-30T23:46:36.287Z"} for property_name in fake_properties_list ], "status_code": 200, @@ -304,26 +276,30 @@ def test_client_side_incremental_stream(requests_mock, common_params, fake_prope ] requests_mock.register_uri("GET", stream.url, responses) - requests_mock.register_uri( - "GET", "/properties/v2/form/properties", properties_response) + requests_mock.register_uri("GET", "/properties/v2/form/properties", properties_response) list(stream.read_records(SyncMode.incremental)) - assert stream.state == {stream.cursor_field: pendulum.parse( - latest_cursor_value).to_rfc3339_string()} + assert stream.state == {stream.cursor_field: pendulum.parse(latest_cursor_value).to_rfc3339_string()} @pytest.mark.parametrize( "state, record, expected", [ - ({"updatedAt": ""}, {"id": "test_id_1", "updatedAt": "2023-01-30T23:46:36.287Z"}, - (True, {"updatedAt": "2023-01-30T23:46:36.287000+00:00"})), - ({"updatedAt": "2023-01-30T23:46:36.287000+00:00"}, {"id": "test_id_1", - "updatedAt": "2023-01-29T01:02:03.123Z"}, (False, {"updatedAt": "2023-01-30T23:46:36.287000+00:00"})), + ( + {"updatedAt": ""}, + {"id": "test_id_1", "updatedAt": "2023-01-30T23:46:36.287Z"}, + (True, {"updatedAt": "2023-01-30T23:46:36.287000+00:00"}), + ), + ( + {"updatedAt": "2023-01-30T23:46:36.287000+00:00"}, + {"id": "test_id_1", "updatedAt": "2023-01-29T01:02:03.123Z"}, + (False, {"updatedAt": "2023-01-30T23:46:36.287000+00:00"}), + ), ], ids=[ "Empty Sting in state + new record", "State + old record", - ] + ], ) def test_empty_string_in_state(state, record, expected, requests_mock, common_params, fake_properties_list): stream = Forms(**common_params) @@ -333,16 +309,14 @@ def test_empty_string_in_state(state, record, expected, requests_mock, common_pa properties_response = [ { "json": [ - {"name": property_name, "type": "string", "CreatedAt": "2023-01-30T23:46:24.355Z", - "updatedAt": "2023-01-30T23:46:36.287Z"} + {"name": property_name, "type": "string", "CreatedAt": "2023-01-30T23:46:24.355Z", "updatedAt": "2023-01-30T23:46:36.287Z"} for property_name in fake_properties_list ], "status_code": 200, } ] requests_mock.register_uri("GET", stream.url, json=record) - requests_mock.register_uri( - "GET", "/properties/v2/form/properties", properties_response) + requests_mock.register_uri("GET", "/properties/v2/form/properties", properties_response) # end of mocking `availability strategy` result = stream.filter_by_state(stream.state, record) @@ -402,6 +376,7 @@ def expected_custom_object_json_schema(): "updatedAt": {"type": ["null", "string"], "format": "date-time"}, "archived": {"type": ["null", "boolean"]}, "properties": {"type": ["null", "object"], "properties": {"name": {"type": ["null", "string"]}}}, + "properties_name": {"type": ["null", "string"]}, }, } @@ -409,11 +384,15 @@ def expected_custom_object_json_schema(): def test_custom_object_stream_doesnt_call_hubspot_to_get_json_schema_if_available( requests_mock, custom_object_schema, expected_custom_object_json_schema, common_params ): - stream = CustomObject(entity="animals", schema=expected_custom_object_json_schema, - fully_qualified_name="p123_animals", **common_params) + stream = CustomObject( + entity="animals", + schema=expected_custom_object_json_schema, + fully_qualified_name="p123_animals", + custom_properties={"name": {"type": ["null", "string"]}}, + **common_params, + ) - adapter = requests_mock.register_uri( - "GET", "/crm/v3/schemas", [{"json": {"results": [custom_object_schema]}}]) + adapter = requests_mock.register_uri("GET", "/crm/v3/schemas", [{"json": {"results": [custom_object_schema]}}]) json_schema = stream.get_json_schema() assert json_schema == expected_custom_object_json_schema @@ -430,13 +409,13 @@ def test_contacts_merged_audit_stream_doesnt_call_hubspot_to_get_json_schema(req { "json": [ { - 'name': 'hs_object_id', - 'label': 'Record ID', - 'type': 'number', + "name": "hs_object_id", + "label": "Record ID", + "type": "number", } ] } - ] + ], ) _ = stream.get_json_schema() @@ -444,9 +423,193 @@ def test_contacts_merged_audit_stream_doesnt_call_hubspot_to_get_json_schema(req def test_get_custom_objects_metadata_success(requests_mock, custom_object_schema, expected_custom_object_json_schema, api): - requests_mock.register_uri( - "GET", "/crm/v3/schemas", json={"results": [custom_object_schema]}) - for (entity, fully_qualified_name, schema) in api.get_custom_objects_metadata(): + requests_mock.register_uri("GET", "/crm/v3/schemas", json={"results": [custom_object_schema]}) + for (entity, fully_qualified_name, schema, custom_properties) in api.get_custom_objects_metadata(): assert entity == "animals" assert fully_qualified_name == "p19936848_Animal" assert schema == expected_custom_object_json_schema + + +@pytest.mark.parametrize( + "input_data, unnest_fields, expected_output", + ( + ( + [{"id": 1, "createdAt": "2020-01-01", "email": {"from": "integration-test@airbyte.io", "to": "michael_scott@gmail.com"}}], + [], + [{"id": 1, "createdAt": "2020-01-01", "email": {"from": "integration-test@airbyte.io", "to": "michael_scott@gmail.com"}}], + ), + ( + [ + { + "id": 1, + "createdAt": "2020-01-01", + "email": {"from": "integration-test@airbyte.io", "to": "michael_scott@gmail.com"}, + "properties": {"phone": "+38044-111-111", "address": "31, Cleveland str, Washington DC"}, + } + ], + [], + [ + { + "id": 1, + "createdAt": "2020-01-01", + "email": {"from": "integration-test@airbyte.io", "to": "michael_scott@gmail.com"}, + "properties": {"phone": "+38044-111-111", "address": "31, Cleveland str, Washington DC"}, + "properties_phone": "+38044-111-111", + "properties_address": "31, Cleveland str, Washington DC", + } + ], + ), + ( + [ + { + "id": 1, + "createdAt": "2020-01-01", + "email": {"from": "integration-test@airbyte.io", "to": "michael_scott@gmail.com"}, + } + ], + ["email"], + [ + { + "id": 1, + "createdAt": "2020-01-01", + "email": {"from": "integration-test@airbyte.io", "to": "michael_scott@gmail.com"}, + "email_from": "integration-test@airbyte.io", + "email_to": "michael_scott@gmail.com", + } + ], + ), + ( + [ + { + "id": 1, + "createdAt": "2020-01-01", + "email": {"from": "integration-test@airbyte.io", "to": "michael_scott@gmail.com"}, + "properties": {"phone": "+38044-111-111", "address": "31, Cleveland str, Washington DC"}, + } + ], + ["email"], + [ + { + "id": 1, + "createdAt": "2020-01-01", + "email": {"from": "integration-test@airbyte.io", "to": "michael_scott@gmail.com"}, + "email_from": "integration-test@airbyte.io", + "email_to": "michael_scott@gmail.com", + "properties": {"phone": "+38044-111-111", "address": "31, Cleveland str, Washington DC"}, + "properties_phone": "+38044-111-111", + "properties_address": "31, Cleveland str, Washington DC", + } + ], + ), + ), +) +def test_records_unnester(input_data, unnest_fields, expected_output): + unnester = RecordUnnester(fields=unnest_fields) + assert list(unnester.unnest(input_data)) == expected_output + + +def test_web_analytics_stream_slices(common_params, mocker): + parent_slicer_mock = mocker.patch("airbyte_cdk.sources.streams.http.HttpSubStream.stream_slices") + parent_slicer_mock.return_value = (_ for _ in [{"parent": {"id": 1}}]) + + pendulum_now_mock = mocker.patch("pendulum.now") + pendulum_now_mock.return_value = pendulum.parse(common_params["start_date"]).add(days=50) + + stream = ContactsWebAnalytics(**common_params) + slices = list(stream.stream_slices(SyncMode.incremental, cursor_field="occurredAt")) + + assert len(slices) == 2 + assert all(map(lambda slice: slice["objectId"] == 1, slices)) + + assert [ + ("2021-01-10T00:00:00Z", "2021-02-09T00:00:00Z"), + ("2021-02-09T00:00:00Z", "2021-03-01T00:00:00Z") + ] == [ + (s["occurredAfter"], s["occurredBefore"]) for s in slices + ] + + +def test_web_analytics_latest_state(common_params, mocker): + parent_slicer_mock = mocker.patch("airbyte_cdk.sources.streams.http.HttpSubStream.stream_slices") + parent_slicer_mock.return_value = (_ for _ in [{"parent": {"id": "1"}}]) + + pendulum_now_mock = mocker.patch("pendulum.now") + pendulum_now_mock.return_value = pendulum.parse(common_params["start_date"]).add(days=10) + + parent_slicer_mock = mocker.patch("source_hubspot.streams.Stream.read_records") + parent_slicer_mock.return_value = (_ for _ in [{"objectId": "1", "occurredAt": "2021-01-02T00:00:00Z"}]) + + stream = ContactsWebAnalytics(**common_params) + stream.state = {"1": {"occurredAt": "2021-01-01T00:00:00Z"}} + slices = list(stream.stream_slices(SyncMode.incremental, cursor_field="occurredAt")) + records = [list(stream.read_records(SyncMode.incremental, cursor_field="occurredAt", stream_slice=stream_slice)) for stream_slice in slices] + + assert len(slices) == 1 + assert len(records) == 1 + assert len(records[0]) == 1 + assert records[0][0]["objectId"] == "1" + assert stream.state["1"]["occurredAt"] == "2021-01-02T00:00:00Z" + + +def test_property_history_transform(common_params): + stream = ContactsPropertyHistory(**common_params) + versions = [ + { + "value": "Georgia", + "timestamp": 1645135236625 + } + ] + records = [ + { + "vid": 1, + "canonical-vid": 1, + "portal-id": 1, + "is-contact": True, + "properties": { + "hs_country": {"versions": versions}, + "lastmodifieddate": {"value": 1645135236625} + } + } + ] + assert [ + { + "vid": 1, + "canonical-vid": 1, + "portal-id": 1, + "is-contact": True, + "property": "hs_country", + **version + } for version in versions + ] == list(stream._transform(records=records)) + + +def test_contacts_membership_transform(common_params): + stream = ContactsListMemberships(**common_params) + versions = [ + { + "value": "Georgia", + "timestamp": 1645135236625 + } + ] + memberships = [ + {"membership": 1} + ] + records = [ + { + "vid": 1, + "canonical-vid": 1, + "portal-id": 1, + "is-contact": True, + "properties": { + "hs_country": {"versions": versions}, + "lastmodifieddate": {"value": 1645135236625} + }, + "list-memberships": memberships + } + ] + assert [ + { + "membership": 1, + "canonical-vid": 1 + } for _ in versions + ] == list(stream._transform(records=records)) diff --git a/airbyte-integrations/connectors/source-insightly/Dockerfile b/airbyte-integrations/connectors/source-insightly/Dockerfile index c4ac2c8fb73a..6a744a678910 100644 --- a/airbyte-integrations/connectors/source-insightly/Dockerfile +++ b/airbyte-integrations/connectors/source-insightly/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.13-alpine3.15 as base +FROM python:3.9.11-alpine3.15 as base # build and load all requirements FROM base as builder @@ -34,5 +34,5 @@ COPY source_insightly ./source_insightly ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-insightly diff --git a/airbyte-integrations/connectors/source-insightly/README.md b/airbyte-integrations/connectors/source-insightly/README.md index 61fbc856b46f..92b977856b8b 100644 --- a/airbyte-integrations/connectors/source-insightly/README.md +++ b/airbyte-integrations/connectors/source-insightly/README.md @@ -1,45 +1,23 @@ # Insightly Source -This is the repository for the Insightly source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/insightly). +This is the repository for the Insightly configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/insightly). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle + You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. To build using Gradle, from the Airbyte repository root, run: + ``` ./gradlew :airbyte-integrations:connectors:source-insightly:build ``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/insightly) + +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/insightly) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_insightly/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. @@ -47,86 +25,95 @@ See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source insightly test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-insightly:dev -``` -You can also build the connector image via Gradle: +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** + +```bash +airbyte-ci connectors --name=source-insightly build ``` -./gradlew :airbyte-integrations:connectors:source-insightly:airbyteDocker + +An image will be built with the tag `airbyte/source-insightly:dev`. + +**Via `docker build`:** + +```bash +docker build -t airbyte/source-insightly:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run + Then run any of the connector commands as follows: + ``` docker run --rm airbyte/source-insightly:dev spec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-insightly:dev check --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-insightly:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-insightly:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` +<<<<<<< HEAD + #### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. + +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run + +To run your integration tests with Docker, run: + ``` -python -m pytest integration_tests -p integration_tests.acceptance +./acceptance-test-docker.sh ``` -To run your integration tests with docker ### Using gradle to run tests + All commands should be run from airbyte project root. To run unit tests: + ``` ./gradlew :airbyte-integrations:connectors:source-insightly:unitTest ``` + To run acceptance and custom integration tests: + ``` ./gradlew :airbyte-integrations:connectors:source-insightly:integrationTest ``` +======= +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): + +```bash +airbyte-ci connectors --name=source-insightly test +``` + +### Customizing acceptance Tests + +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +> > > > > > > master + ## Dependency Management + All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list + +- required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +- required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector + You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-insightly test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/insightly.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-insightly/__init__.py b/airbyte-integrations/connectors/source-insightly/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-insightly/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-insightly/acceptance-test-config.yml b/airbyte-integrations/connectors/source-insightly/acceptance-test-config.yml index a7de796e79a5..b0f10cdef373 100644 --- a/airbyte-integrations/connectors/source-insightly/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-insightly/acceptance-test-config.yml @@ -18,10 +18,46 @@ acceptance_tests: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: emails + bypass_reason: "no data for this stream in our sandbox account" + - name: events + bypass_reason: "no data for this stream in our sandbox account" + - name: milestones + bypass_reason: "no data for this stream in our sandbox account" + - name: notes + bypass_reason: "no data for this stream in our sandbox account" + - name: opportunity_categories + bypass_reason: "no data for this stream in our sandbox account" + - name: project_categories + bypass_reason: "no data for this stream in our sandbox account" + - name: knowledge_article_categories + bypass_reason: "current sandbox account does not have permissions to access this stream (403 error)" + - name: knowledge_article_folders + bypass_reason: "current sandbox account does not have permissions to access this stream (403 error)" + - name: knowledge_articles + bypass_reason: "current sandbox account does not have permissions to access this stream (403 error)" + - name: lead_sources + bypass_reason: "current sandbox account does not have permissions to access this stream (403 error)" + - name: lead_statuses + bypass_reason: "current sandbox account does not have permissions to access this stream (403 error)" + - name: prospects + bypass_reason: "current sandbox account does not have permissions to access this stream (403 error)" + full_refresh: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + ignored_fields: + contacts: + - name: "IMAGE_URL" + bypass_reason: "image url is a dynamic s3 url with a changing expiration date in the query parameter" + organisations: + - name: "IMAGE_URL" + bypass_reason: "image url is a dynamic s3 url with a changing expiration date in the query parameter" + users: + - name: "USER_CURRENCY" + bypass_reason: "this can change between sequential reads" incremental: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-insightly/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-insightly/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-insightly/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-insightly/bootstrap.md b/airbyte-integrations/connectors/source-insightly/bootstrap.md deleted file mode 100644 index d52b29577dea..000000000000 --- a/airbyte-integrations/connectors/source-insightly/bootstrap.md +++ /dev/null @@ -1,13 +0,0 @@ -# Insightly -OpenWeather is an online service offering an API to retrieve historical, current and forecasted weather data over the globe. - -### Auth -API calls are authenticated through an API key. An API key can be retrieved from Insightly User Settings page in the API section. - -### Rate limits -The API has different rate limits for different account types. Keep that in mind when syncing large amounts of data: -* Free/Gratis - 1,000 requests/day/instance -* Legacy plans - 20,000 requests/day/instance -* Plus - 40,000 requests/day/instance -* Professional - 60,000 requests/day/instance -* Enterprise - 100,000 requests/day/instance diff --git a/airbyte-integrations/connectors/source-insightly/build.gradle b/airbyte-integrations/connectors/source-insightly/build.gradle deleted file mode 100644 index 1b5baf0891c1..000000000000 --- a/airbyte-integrations/connectors/source-insightly/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_insightly' -} diff --git a/airbyte-integrations/connectors/source-insightly/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-insightly/integration_tests/abnormal_state.json index c06e9d0a75c0..f49326b9fd77 100644 --- a/airbyte-integrations/connectors/source-insightly/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-insightly/integration_tests/abnormal_state.json @@ -1,4 +1,136 @@ [ + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "contacts" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "events" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "knowledge_article_categories" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "knowledge_article_folders" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "knowledge_articles" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "milestones" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "notes" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "opportunities" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "organisations" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "projects" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "prospects" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "tasks" + }, + "stream_state": { + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" + } + } + }, { "type": "STREAM", "stream": { @@ -6,7 +138,7 @@ "name": "users" }, "stream_state": { - "DATE_UPDATED_UTC": "2122-10-17T19:10:14+00:00" + "DATE_UPDATED_UTC": "2122-10-17 19:10:14" } } } diff --git a/airbyte-integrations/connectors/source-insightly/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-insightly/integration_tests/configured_catalog.json index 0873c8cb593f..60107bacf103 100644 --- a/airbyte-integrations/connectors/source-insightly/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-insightly/integration_tests/configured_catalog.json @@ -1,26 +1,424 @@ { "streams": [ { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { - "name": "team_members", + "default_cursor_field": null, "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["MEMBER_USER_ID"]] + "name": "activity_sets", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["ACTIVITYSET_ID"]], + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "sync_mode": "full_refresh" }, { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { - "name": "users", + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "contacts", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["CONTACT_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "countries", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["COUNTRY_NAME"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "currencies", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["CURRENCY_CODE"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "emails", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["EMAIL_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "events", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["EVENT_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "knowledge_article_categories", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["CATEGORY_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "knowledge_article_folders", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["FOLDER_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "knowledge_articles", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["ARTICLE_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "lead_sources", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["LEAD_SOURCE_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "lead_statuses", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["LEAD_STATUS_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "milestones", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["MILESTONE_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "notes", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["NOTE_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "opportunities", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["OPPORTUNITY_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "opportunity_categories", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["CATEGORY_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "opportunity_state_reasons", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["STATE_REASON_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "name": "organisations", + "namespace": null, "source_defined_cursor": true, + "source_defined_primary_key": [["ORGANISATION_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "pipelines", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["PIPELINE_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "pipeline_stages", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["STAGE_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "project_categories", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["CATEGORY_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { "default_cursor_field": ["DATE_UPDATED_UTC"], - "source_defined_primary_key": [["USER_ID"]] + "json_schema": {}, + "name": "projects", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["PROJECT_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "prospects", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["PROSPECT_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "relationships", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["RELATIONSHIP_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "task_categories", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["CATEGORY_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "tasks", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["TASK_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "team_members", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["MEMBER_USER_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": null, + "json_schema": {}, + "name": "teams", + "namespace": null, + "source_defined_cursor": null, + "source_defined_primary_key": [["TEAM_ID"]], + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { + "default_cursor_field": ["DATE_UPDATED_UTC"], + "json_schema": {}, + "name": "users", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["USER_ID"]], + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "incremental", - "destination_sync_mode": "append" + "sync_mode": "full_refresh" } ] } diff --git a/airbyte-integrations/connectors/source-insightly/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-insightly/integration_tests/sample_state.json index 3127937e0f0e..672363e87f6e 100644 --- a/airbyte-integrations/connectors/source-insightly/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-insightly/integration_tests/sample_state.json @@ -6,7 +6,7 @@ "name": "users" }, "stream_state": { - "DATE_UPDATED_UTC": "2022-10-17T19:10:14+00:00" + "DATE_UPDATED_UTC": "2022-10-17 19:10:14" } } } diff --git a/airbyte-integrations/connectors/source-insightly/metadata.yaml b/airbyte-integrations/connectors/source-insightly/metadata.yaml index 06202981ce95..78f6fc859bf6 100644 --- a/airbyte-integrations/connectors/source-insightly/metadata.yaml +++ b/airbyte-integrations/connectors/source-insightly/metadata.yaml @@ -1,24 +1,27 @@ data: + allowedHosts: + hosts: + - TODO # Please change to the hostname of the source. + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: 38f84314-fe6a-4257-97be-a8dcd942d693 - dockerImageTag: 0.1.3 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-insightly githubIssueLabel: source-insightly icon: insightly.svg license: MIT name: Insightly - registries: - cloud: - enabled: true - oss: - enabled: true releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/insightly tags: - - language:python + - language:low-code ab_internal: sl: 100 ql: 100 - supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-insightly/setup.py b/airbyte-integrations/connectors/source-insightly/setup.py index a8c4637c1342..a3c070098791 100644 --- a/airbyte-integrations/connectors/source-insightly/setup.py +++ b/airbyte-integrations/connectors/source-insightly/setup.py @@ -5,14 +5,11 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", - "pendulum==2.1.2", -] +MAIN_REQUIREMENTS = ["airbyte-cdk"] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", - "pytest~=6.1", + "pytest~=6.2", "pytest-mock~=3.6.1", ] diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml b/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml new file mode 100644 index 000000000000..5f28969be084 --- /dev/null +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/manifest.yaml @@ -0,0 +1,424 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + + requester: + type: HttpRequester + url_base: "https://api.na1.insightly.com/v3.1/" + http_method: "GET" + authenticator: + type: BasicHttpAuthenticator + username: "{{ config['token'] }}" + request_parameters: + count_total: "True" + updated_after_utc: "{{ stream_state['DATE_UPDATED_UTC'] }}" + error_handler: + type: "CompositeErrorHandler" + error_handlers: + - response_filters: + - http_codes: [403] + action: IGNORE #ignore 403 errors for knowledge_article streams, lead streams and prospects + - response_filters: + - http_codes: [429] + action: RETRY + backoff_strategies: + - type: "ConstantBackoffStrategy" + backoff_time_in_seconds: 6.0 + + date_incremental_sync: + type: DatetimeBasedCursor + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + datetime: "{{ now_utc() }}" + datetime_format: "%Y-%m-%d %H:%M:%S.%f+00:00" + datetime_format: "%Y-%m-%d %H:%M:%S" + cursor_granularity: PT1S + step: P30D # Step should reflect the actual time granularity you need + cursor_field: "DATE_UPDATED_UTC" + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: DefaultPaginator + pagination_strategy: + type: "OffsetIncrement" + page_size: 500 + page_token_option: + type: RequestOption + inject_into: "request_parameter" + field_name: "skip" + page_size_option: + inject_into: "request_parameter" + field_name: "top" + requester: + $ref: "#/definitions/requester" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + base_incremental_stream: + incremental_sync: + $ref: "#/definitions/date_incremental_sync" + retriever: + $ref: "#/definitions/retriever" + + activity_sets_stream: + $ref: "#/definitions/base_stream" + name: "activity_sets" + primary_key: "ACTIVITYSET_ID" + $parameters: + path: "/ActivitySets" + + contacts_stream: + $ref: "#/definitions/base_incremental_stream" + name: "contacts" + primary_key: "CONTACT_ID" + $parameters: + path: "/Contacts/Search" + + countries_stream: + $ref: "#/definitions/base_stream" + name: "countries" + primary_key: "COUNTRY_NAME" + $parameters: + path: "/Countries" + + currencies_stream: + $ref: "#/definitions/base_stream" + name: "currencies" + primary_key: "CURRENCY_CODE" + $parameters: + path: "/Currencies" + + emails_stream: + $ref: "#/definitions/base_stream" + name: "emails" + primary_key: "EMAIL_ID" + $parameters: + path: "/Emails/Search" + + events_stream: + $ref: "#/definitions/base_incremental_stream" + name: "events" + primary_key: "EVENT_ID" + $parameters: + path: "/Events/Search" + + knowledge_article_categories_stream: + $ref: "#/definitions/base_incremental_stream" + name: "knowledge_article_categories" + primary_key: "CATEGORY_ID" + $parameters: + path: "/KnowledgeArticleCategory/Search" + + knowledge_article_folders_stream: + $ref: "#/definitions/base_incremental_stream" + name: "knowledge_article_folders" + primary_key: "FOLDER_ID" + $parameters: + path: "/KnowledgeArticleFolder/Search" + + knowledge_articles_stream: + $ref: "#/definitions/base_incremental_stream" + name: "knowledge_articles" + primary_key: "ARTICLE_ID" + $parameters: + path: "/KnowledgeArticle/Search" + + leads_stream: + $ref: "#/definitions/base_incremental_stream" + name: "leads" + primary_key: "LEAD_ID" + $parameters: + path: "/Leads/Search" + + lead_sources_stream: + $ref: "#/definitions/base_stream" + name: "lead_sources" + primary_key: "LEAD_SOURCE_ID" + $parameters: + path: "/LeadSources" + + lead_statuses_stream: + $ref: "#/definitions/base_stream" + name: "lead_statuses" + primary_key: "LEAD_STATUS_ID" + $parameters: + path: "/LeadStatuses" + + milestones_stream: + $ref: "#/definitions/base_incremental_stream" + name: "milestones" + primary_key: "MILESTONE_ID" + $parameters: + path: "/Milestones/Search" + + notes_stream: + $ref: "#/definitions/base_incremental_stream" + name: "notes" + primary_key: "NOTE_ID" + $parameters: + path: "/Notes/Search" + + opportunities_stream: + $ref: "#/definitions/base_incremental_stream" + name: "opportunities" + primary_key: "OPPORTUNITY_ID" + $parameters: + path: "/Opportunities/Search" + + opportunity_categories_stream: + $ref: "#/definitions/base_stream" + name: "opportunity_categories" + primary_key: "CATEGORY_ID" + $parameters: + path: "/OpportunityCategories" + + opportunity_products_stream: + $ref: "#/definitions/base_incremental_stream" + name: "opportunity_products" + primary_key: "OPPORTUNITY_ITEM_ID" + $parameters: + path: "/OpportunityLineItem/Search" + + opportunity_state_reasons_stream: + $ref: "#/definitions/base_stream" + name: "opportunity_state_reasons" + primary_key: "STATE_REASON_ID" + $parameters: + path: "/OpportunityStateReasons" + + organisations_stream: + $ref: "#/definitions/base_incremental_stream" + name: "organisations" + primary_key: "ORGANISATION_ID" + $parameters: + path: "/Organisations/Search" + + pipelines_stream: + $ref: "#/definitions/base_stream" + name: "pipelines" + primary_key: "PIPELINE_ID" + $parameters: + path: "/Pipelines" + + pipeline_stages_stream: + $ref: "#/definitions/base_stream" + name: "pipeline_stages" + primary_key: "STAGE_ID" + $parameters: + path: "/PipelineStages" + + pricebook_entries_stream: + $ref: "#/definitions/base_incremental_stream" + name: "price_book_entries" + primary_key: "PRICEBOOK_ENTRY_ID" + $parameters: + path: "/PricebookEntry/Search" + + pricebooks_stream: + $ref: "#/definitions/base_incremental_stream" + name: "price_books" + primary_key: "PRICEBOOK_ID" + $parameters: + path: "/Pricebook/Search" + + products_stream: + $ref: "#/definitions/base_incremental_stream" + name: "products" + primary_key: "PRODUCT_ID" + $parameters: + path: "/Product/Search" + + project_categories_stream: + $ref: "#/definitions/base_stream" + name: "project_categories" + primary_key: "CATEGORY_ID" + $parameters: + path: "/ProjectCategories" + + projects_stream: + $ref: "#/definitions/base_incremental_stream" + name: "projects" + primary_key: "PROJECT_ID" + $parameters: + path: "/Projects/Search" + + prospects_stream: + $ref: "#/definitions/base_incremental_stream" + name: "prospects" + primary_key: "PROSPECT_ID" + $parameters: + path: "/Prospect/Search" + + quote_products_stream: + $ref: "#/definitions/base_incremental_stream" + name: "quote_products" + primary_key: "QUOTATION_ITEM_ID" + $parameters: + path: "/QuotationLineItem/Search" + + quotes_stream: + $ref: "#/definitions/base_incremental_stream" + name: "quotes" + primary_key: "QUOTE_ID" + $parameters: + path: "/Quotation/Search" + + relationships_stream: + $ref: "#/definitions/base_stream" + name: "relationships" + primary_key: "RELATIONSHIP_ID" + $parameters: + path: "/Relationships" + + tags_stream: + $ref: "#/definitions/base_stream" + name: "tags" + primary_key: "TAG_NAME" + $parameters: + path: "/Tags" + + task_categories_stream: + $ref: "#/definitions/base_stream" + name: "task_categories" + primary_key: "CATEGORY_ID" + $parameters: + path: "/TaskCategories" + + tasks_stream: + $ref: "#/definitions/base_incremental_stream" + name: "tasks" + primary_key: "TASK_ID" + $parameters: + path: "/Tasks/Search" + + team_members_stream: + $ref: "#/definitions/base_stream" + name: "team_members" + primary_key: "MEMBER_USER_ID" + $parameters: + path: "/TeamMembers" + + teams_stream: + $ref: "#/definitions/base_stream" + name: "teams" + primary_key: "TEAM_ID" + $parameters: + path: "/Teams" + + tickets_stream: + $ref: "#/definitions/base_incremental_stream" + name: "tickets" + primary_key: "TICKET_ID" + $parameters: + path: "/Ticket/Search" + + users_stream: + $ref: "#/definitions/base_incremental_stream" + name: "users" + primary_key: "USER_ID" + $parameters: + path: "/Users/Search" + +streams: + - "#/definitions/activity_sets_stream" + - "#/definitions/contacts_stream" + - "#/definitions/countries_stream" + - "#/definitions/currencies_stream" + - "#/definitions/emails_stream" + - "#/definitions/events_stream" + - "#/definitions/knowledge_article_categories_stream" + - "#/definitions/knowledge_article_folders_stream" + - "#/definitions/knowledge_articles_stream" + # - "#/definitions/leads_stream" + - "#/definitions/lead_sources_stream" + - "#/definitions/lead_statuses_stream" + - "#/definitions/milestones_stream" + - "#/definitions/notes_stream" + - "#/definitions/opportunities_stream" + - "#/definitions/opportunity_categories_stream" + # - "#/definitions/opportunity_products_stream" + - "#/definitions/opportunity_state_reasons_stream" + - "#/definitions/organisations_stream" + - "#/definitions/pipelines_stream" + - "#/definitions/pipeline_stages_stream" + # - "#/definitions/pricebook_entries_stream" + # - "#/definitions/pricebooks_stream" + # - "#/definitions/products_stream" + - "#/definitions/project_categories_stream" + - "#/definitions/projects_stream" + - "#/definitions/prospects_stream" + # - "#/definitions/quote_products_stream" + # - "#/definitions/quotes_stream" + - "#/definitions/relationships_stream" + # - "#/definitions/tags_stream" + - "#/definitions/task_categories_stream" + - "#/definitions/tasks_stream" + - "#/definitions/team_members_stream" + - "#/definitions/teams_stream" + # - "#/definitions/tickets_stream" + - "#/definitions/users_stream" + +check: + type: CheckStream + stream_names: + - "activity_sets" + - "contacts" + # - "countries" + # - "currencies" + # - "opportunities" + # - "opportunity_state_reasons" + # - "organizations" + # - "pipelines" + # - "pipeline_stages" + # - "projects" + # - "relationships" + # - "task_categories" + # - "tasks" + # - "team_members" + # - "teams" + # - "users" + +spec: + type: Spec + documentationUrl: https://docs.airbyte.com/integrations/sources/insightly + connection_specification: + "$schema": http://json-schema.org/draft-07/schema# + title: Insightly Spec + type: object + required: + - token + - start_date + additionalProperties: true + properties: + token: + type: + - string + - "null" + title: API Token + description: Your Insightly API token. + airbyte_secret: true + start_date: + type: + - string + - "null" + title: Start Date + description: + The date from which you'd like to replicate data for Insightly + in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will + be replicated. Note that it will be used only for incremental streams. + examples: + - "2021-03-01T00:00:00Z" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/contacts.json b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/contacts.json index dd0b5039fe01..6b6fd0672c44 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/contacts.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, "type": "object", "properties": { "CONTACT_ID": { @@ -114,6 +115,12 @@ "TITLE": { "type": ["string", "null"] }, + "VISIBLE_TEAM_ID": { + "type": ["integer", "null"] + }, + "VISIBLE_TO": { + "type": ["string", "null"] + }, "EMAIL_OPTED_OUT": { "type": ["boolean", "null"] }, diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/opportunities.json b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/opportunities.json index e154e26090d2..ca20e33d36b8 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/opportunities.json +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/opportunities.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": true, "type": "object", "properties": { "OPPORTUNITY_ID": { @@ -27,7 +28,7 @@ "type": ["string", "null"] }, "BID_AMOUNT": { - "type": ["integer", "null"] + "type": ["number", "null"] }, "BID_TYPE": { "type": ["string", "null"] @@ -48,7 +49,7 @@ "format": "date-time" }, "OPPORTUNITY_VALUE": { - "type": ["integer", "null"] + "type": ["number", "null"] }, "PROBABILITY": { "type": ["integer", "null"] @@ -83,6 +84,12 @@ "PRICEBOOK_ID": { "type": ["integer", "null"] }, + "VISIBLE_TEAM_ID": { + "type": ["integer", "null"] + }, + "VISIBLE_TO": { + "type": ["string", "null"] + }, "CUSTOMFIELDS": { "type": "array", "items": { diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/organisations.json b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/organisations.json index b2dd05169110..9f7c844de6ad 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/organisations.json +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/organisations.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": true, "type": "object", "properties": { "ORGANISATION_ID": { @@ -84,6 +85,12 @@ "SOCIAL_TWITTER": { "type": ["string", "null"] }, + "VISIBLE_TEAM_ID": { + "type": ["integer", "null"] + }, + "VISIBLE_TO": { + "type": ["string", "null"] + }, "CUSTOMFIELDS": { "type": "array", "items": { diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/projects.json b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/projects.json index fa91ca651ef4..d11b04949b52 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/projects.json +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/projects.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": true, "type": "object", "properties": { "PROJECT_ID": { @@ -60,6 +61,12 @@ "RESPONSIBLE_USER_ID": { "type": ["integer", "null"] }, + "VISIBLE_TEAM_ID": { + "type": ["integer", "null"] + }, + "VISIBLE_TO": { + "type": ["string", "null"] + }, "CUSTOMFIELDS": { "type": "array", "items": { diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/tasks.json b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/tasks.json index 6b41220e9e10..752a64002f8f 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/tasks.json +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/tasks.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": true, "type": "object", "properties": { "TASK_ID": { @@ -34,6 +35,9 @@ "PERCENT_COMPLETE": { "type": ["integer", "null"] }, + "PUBLICLY_VISIBLE": { + "type": ["null", "boolean"] + }, "START_DATE": { "type": ["string", "null"], "format": "date-time" diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/teams.json b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/teams.json index b8454b8b393e..367acda41e53 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/teams.json +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/schemas/teams.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, "type": "object", "properties": { "TEAM_ID": { @@ -20,7 +21,7 @@ "format": "date-time" }, "TEAMMEMBERS": { - "type": "object" + "type": "array" } } } diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/source.py b/airbyte-integrations/connectors/source-insightly/source_insightly/source.py index 68133dfb1df3..29fa855efc5b 100644 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/source.py +++ b/airbyte-integrations/connectors/source-insightly/source_insightly/source.py @@ -2,409 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from abc import ABC -from datetime import datetime -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple -from urllib.parse import parse_qs, urlparse +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import pendulum -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import BasicHttpAuthenticator -from requests.auth import AuthBase +WARNING: Do not modify this file. +""" -PAGE_SIZE = 500 -BASE_URL = "https://api.insightly.com/v3.1/" - -# Basic full refresh stream -class InsightlyStream(HttpStream, ABC): - total_count: int = 0 - page_size: Optional[int] = PAGE_SIZE - - url_base = BASE_URL - - def __init__(self, authenticator: AuthBase, start_date: str = None, **kwargs): - self.start_date = start_date - super().__init__(authenticator=authenticator) - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - parsed = urlparse(response.request.url) - previous_skip = parse_qs(parsed.query)["skip"][0] - new_skip = int(previous_skip) + self.page_size - return new_skip if new_skip <= self.total_count else None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return { - "count_total": True, - "top": self.page_size, - "skip": next_page_token or 0, - } - - def request_headers(self, **kwargs) -> Mapping[str, Any]: - return {"Accept": "application/json"} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - self.total_count = int(response.headers.get("X-Total-Count", 0)) - results = response.json() - yield from results - - -class ActivitySets(InsightlyStream): - primary_key = "ACTIVITYSET_ID" - - def path(self, **kwargs) -> str: - return "ActivitySets" - - -class Countries(InsightlyStream): - primary_key = "COUNTRY_NAME" - - def path(self, **kwargs) -> str: - return "Countries" - - -class Currencies(InsightlyStream): - primary_key = "CURRENCY_CODE" - - def path(self, **kwargs) -> str: - return "Currencies" - - -class Emails(InsightlyStream): - primary_key = "EMAIL_ID" - - def path(self, **kwargs) -> str: - return "Emails" - - -class LeadSources(InsightlyStream): - primary_key = "LEAD_SOURCE_ID" - - def path(self, **kwargs) -> str: - return "LeadSources" - - -class LeadStatuses(InsightlyStream): - primary_key = "LEAD_STATUS_ID" - - def path(self, **kwargs) -> str: - return "LeadStatuses" - - -class OpportunityCategories(InsightlyStream): - primary_key = "CATEGORY_ID" - - def path(self, **kwargs) -> str: - return "OpportunityCategories" - - -class OpportunityStateReasons(InsightlyStream): - primary_key = "STATE_REASON_ID" - - def path(self, **kwargs) -> str: - return "OpportunityStateReasons" - - -class Pipelines(InsightlyStream): - primary_key = "PIPELINE_ID" - - def path(self, **kwargs) -> str: - return "Pipelines" - - -class PipelineStages(InsightlyStream): - primary_key = "STAGE_ID" - - def path(self, **kwargs) -> str: - return "PipelineStages" - - -class ProjectCategories(InsightlyStream): - primary_key = "CATEGORY_ID" - - def path(self, **kwargs) -> str: - return "ProjectCategories" - - -class Relationships(InsightlyStream): - primary_key = "RELATIONSHIP_ID" - - def path(self, **kwargs) -> str: - return "Relationships" - - -class Tags(InsightlyStream): - primary_key = "TAG_NAME" - - def path(self, **kwargs) -> str: - return "Tags" - - -class TaskCategories(InsightlyStream): - primary_key = "CATEGORY_ID" - - def path(self, **kwargs) -> str: - return "TaskCategories" - - -class TeamMembers(InsightlyStream): - primary_key = "MEMBER_USER_ID" - - def path(self, **kwargs) -> str: - return "TeamMembers" - - -class Teams(InsightlyStream): - primary_key = "TEAM_ID" - - def path(self, **kwargs) -> str: - return "Teams" - - -class IncrementalInsightlyStream(InsightlyStream, ABC): - """Insighlty incremental stream using `updated_after_utc` filter""" - - cursor_field = "DATE_UPDATED_UTC" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - - start_datetime = pendulum.parse(self.start_date) - cursor_datetime = stream_state.get(self.cursor_field) - if cursor_datetime: - if isinstance(cursor_datetime, datetime): - start_datetime = cursor_datetime - else: - start_datetime = pendulum.parse(cursor_datetime) - - # subtract 1 second to make the incremental request inclusive - start_datetime = start_datetime.subtract(seconds=1) - - params.update({"updated_after_utc": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")}) - return params - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - record_time = pendulum.parse(latest_record[self.cursor_field]) - current_state = current_stream_state.get(self.cursor_field) - if current_state: - current_state = current_state if isinstance(current_state, datetime) else pendulum.parse(current_state) - - current_stream_state[self.cursor_field] = max(record_time, current_state) if current_state else record_time - return current_stream_state - - -class Contacts(IncrementalInsightlyStream): - primary_key = "CONTACT_ID" - - def path(self, **kwargs) -> str: - return "Contacts/Search" - - -class Events(IncrementalInsightlyStream): - primary_key = "EVENT_ID" - - def path(self, **kwargs) -> str: - return "Events/Search" - - -class KnowledgeArticleCategories(IncrementalInsightlyStream): - primary_key = "CATEGORY_ID" - - def path(self, **kwargs) -> str: - return "KnowledgeArticleCategory/Search" - - -class KnowledgeArticleFolders(IncrementalInsightlyStream): - primary_key = "FOLDER_ID" - - def path(self, **kwargs) -> str: - return "KnowledgeArticleFolder/Search" - - -class KnowledgeArticles(IncrementalInsightlyStream): - primary_key = "ARTICLE_ID" - - def path(self, **kwargs) -> str: - return "KnowledgeArticle/Search" - - -class Leads(IncrementalInsightlyStream): - primary_key = "LEAD_ID" - - def path(self, **kwargs) -> str: - return "Leads/Search" - - -class Milestones(IncrementalInsightlyStream): - primary_key = "MILESTONE_ID" - - def path(self, **kwargs) -> str: - return "Milestones/Search" - - -class Notes(IncrementalInsightlyStream): - primary_key = "NOTE_ID" - - def path(self, **kwargs) -> str: - return "Notes/Search" - - -class Opportunities(IncrementalInsightlyStream): - primary_key = "OPPORTUNITY_ID" - - def path(self, **kwargs) -> str: - return "Opportunities/Search" - - -class OpportunityProducts(IncrementalInsightlyStream): - primary_key = "OPPORTUNITY_ITEM_ID" - - def path(self, **kwargs) -> str: - return "OpportunityLineItem/Search" - - -class Organisations(IncrementalInsightlyStream): - primary_key = "ORGANISATION_ID" - - def path(self, **kwargs) -> str: - return "Organisations/Search" - - -class PricebookEntries(IncrementalInsightlyStream): - primary_key = "PRICEBOOK_ENTRY_ID" - - def path(self, **kwargs) -> str: - return "PricebookEntry/Search" - - -class Pricebooks(IncrementalInsightlyStream): - primary_key = "PRICEBOOK_ID" - - def path(self, **kwargs) -> str: - return "Pricebook/Search" - - -class Products(IncrementalInsightlyStream): - primary_key = "PRODUCT_ID" - - def path(self, **kwargs) -> str: - return "Product/Search" - - -class Projects(IncrementalInsightlyStream): - primary_key = "PROJECT_ID" - - def path(self, **kwargs) -> str: - return "Projects/Search" - - -class Prospects(IncrementalInsightlyStream): - primary_key = "PROSPECT_ID" - - def path(self, **kwargs) -> str: - return "Prospect/Search" - - -class QuoteProducts(IncrementalInsightlyStream): - primary_key = "QUOTATION_ITEM_ID" - - def path(self, **kwargs) -> str: - return "QuotationLineItem/Search" - - -class Quotes(IncrementalInsightlyStream): - primary_key = "QUOTE_ID" - - def path(self, **kwargs) -> str: - return "Quotation/Search" - - -class Tasks(IncrementalInsightlyStream): - primary_key = "TASK_ID" - - def path(self, **kwargs) -> str: - return "Tasks/Search" - - -class Tickets(IncrementalInsightlyStream): - primary_key = "TICKET_ID" - - def path(self, **kwargs) -> str: - return "Ticket/Search" - - -class Users(IncrementalInsightlyStream): - primary_key = "USER_ID" - - def path(self, **kwargs) -> str: - return "Users/Search" - - -# Source -class SourceInsightly(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - try: - token = config.get("token") - response = requests.get(f"{BASE_URL}Instance", auth=(token, "")) - response.raise_for_status() - - result = response.json() - logger.info(result) - - return True, None - except Exception as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - - auth = BasicHttpAuthenticator(username=config.get("token"), password="") - return [ - ActivitySets(authenticator=auth, **config), - Contacts(authenticator=auth, **config), - Countries(authenticator=auth, **config), - Currencies(authenticator=auth, **config), - Emails(authenticator=auth, **config), - Events(authenticator=auth, **config), - KnowledgeArticleCategories(authenticator=auth, **config), - KnowledgeArticleFolders(authenticator=auth, **config), - KnowledgeArticles(authenticator=auth, **config), - LeadSources(authenticator=auth, **config), - LeadStatuses(authenticator=auth, **config), - Leads(authenticator=auth, **config), - Milestones(authenticator=auth, **config), - Notes(authenticator=auth, **config), - Opportunities(authenticator=auth, **config), - OpportunityCategories(authenticator=auth, **config), - OpportunityProducts(authenticator=auth, **config), - OpportunityStateReasons(authenticator=auth, **config), - Organisations(authenticator=auth, **config), - PipelineStages(authenticator=auth, **config), - Pipelines(authenticator=auth, **config), - PricebookEntries(authenticator=auth, **config), - Pricebooks(authenticator=auth, **config), - Products(authenticator=auth, **config), - ProjectCategories(authenticator=auth, **config), - Projects(authenticator=auth, **config), - Prospects(authenticator=auth, **config), - QuoteProducts(authenticator=auth, **config), - Quotes(authenticator=auth, **config), - Relationships(authenticator=auth, **config), - Tags(authenticator=auth, **config), - TaskCategories(authenticator=auth, **config), - Tasks(authenticator=auth, **config), - TeamMembers(authenticator=auth, **config), - Teams(authenticator=auth, **config), - Tickets(authenticator=auth, **config), - Users(authenticator=auth, **config), - ] +# Declarative Source +class SourceInsightly(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-insightly/source_insightly/spec.json b/airbyte-integrations/connectors/source-insightly/source_insightly/spec.json deleted file mode 100644 index a21504f3e553..000000000000 --- a/airbyte-integrations/connectors/source-insightly/source_insightly/spec.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/insightly", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Insightly Spec", - "type": "object", - "required": ["token", "start_date"], - "additionalProperties": true, - "properties": { - "token": { - "type": ["string", "null"], - "title": "API Token", - "description": "Your Insightly API token.", - "airbyte_secret": true - }, - "start_date": { - "type": ["string", "null"], - "title": "Start Date", - "description": "The date from which you'd like to replicate data for Insightly in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated. Note that it will be used only for incremental streams.", - "examples": ["2021-03-01T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" - } - } - } -} diff --git a/airbyte-integrations/connectors/source-insightly/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-insightly/unit_tests/test_incremental_streams.py deleted file mode 100644 index eb854c3bae84..000000000000 --- a/airbyte-integrations/connectors/source-insightly/unit_tests/test_incremental_streams.py +++ /dev/null @@ -1,81 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import pendulum -from airbyte_cdk.sources.streams.http.auth import BasicHttpAuthenticator -from pytest import fixture -from source_insightly.source import IncrementalInsightlyStream - -start_date = "2021-01-01T00:00:00Z" -authenticator = BasicHttpAuthenticator(username="test", password="") - - -@fixture -def patch_incremental_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(IncrementalInsightlyStream, "path", "v0/example_endpoint") - mocker.patch.object(IncrementalInsightlyStream, "primary_key", "test_primary_key") - mocker.patch.object(IncrementalInsightlyStream, "__abstractmethods__", set()) - - -def test_cursor_field(patch_incremental_base_class): - stream = IncrementalInsightlyStream(authenticator=authenticator, start_date=start_date) - expected_cursor_field = "DATE_UPDATED_UTC" - assert stream.cursor_field == expected_cursor_field - - -def test_incremental_params(patch_incremental_base_class): - """ - After talking to the insightly team we learned that the DATE_UPDATED_UTC - cursor is exclusive. Subtracting 1 second from the previous state makes it inclusive. - """ - stream = IncrementalInsightlyStream(authenticator=authenticator, start_date=start_date) - inputs = { - "stream_slice": None, - "stream_state": {"DATE_UPDATED_UTC": pendulum.datetime(2023, 5, 15, 18, 12, 44, tz="UTC")}, - "next_page_token": None, - } - expected_params = { - "count_total": True, - "skip": 0, - "top": 500, - "updated_after_utc": "2023-05-15T18:12:43Z", # 1 second subtracted from stream_state - } - assert stream.request_params(**inputs) == expected_params - - -def test_get_updated_state(patch_incremental_base_class): - stream = IncrementalInsightlyStream(authenticator=authenticator, start_date=start_date) - inputs = { - "current_stream_state": {"DATE_UPDATED_UTC": "2021-01-01T00:00:00Z"}, - "latest_record": {"DATE_UPDATED_UTC": "2021-02-01T00:00:00Z"}, - } - expected_state = {"DATE_UPDATED_UTC": pendulum.datetime(2021, 2, 1, 0, 0, 0, tz="UTC")} - assert stream.get_updated_state(**inputs) == expected_state - - -def test_get_updated_state_no_current_state(patch_incremental_base_class): - stream = IncrementalInsightlyStream(authenticator=authenticator, start_date=start_date) - inputs = {"current_stream_state": {}, "latest_record": {"DATE_UPDATED_UTC": "2021-01-01T00:00:00Z"}} - expected_state = {"DATE_UPDATED_UTC": pendulum.datetime(2021, 1, 1, 0, 0, 0, tz="UTC")} - assert stream.get_updated_state(**inputs) == expected_state - - -def test_supports_incremental(patch_incremental_base_class, mocker): - mocker.patch.object(IncrementalInsightlyStream, "cursor_field", "dummy_field") - stream = IncrementalInsightlyStream(authenticator=authenticator, start_date=start_date) - assert stream.supports_incremental - - -def test_source_defined_cursor(patch_incremental_base_class): - stream = IncrementalInsightlyStream(authenticator=authenticator, start_date=start_date) - assert stream.source_defined_cursor - - -def test_stream_checkpoint_interval(patch_incremental_base_class): - stream = IncrementalInsightlyStream(authenticator=authenticator, start_date=start_date) - # TODO: replace this with your expected checkpoint interval - expected_checkpoint_interval = None - assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-insightly/unit_tests/test_source.py b/airbyte-integrations/connectors/source-insightly/unit_tests/test_source.py deleted file mode 100644 index 4e9f2c408756..000000000000 --- a/airbyte-integrations/connectors/source-insightly/unit_tests/test_source.py +++ /dev/null @@ -1,56 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock, patch - -from source_insightly.source import SourceInsightly - - -class MockResponse: - def __init__(self, json_data, status_code): - self.json_data = json_data - self.status_code = status_code - - def json(self): - return self.json_data - - def raise_for_status(self): - if self.status_code != 200: - raise Exception("Bad things happened") - - -def mocked_requests_get(fail=False): - def wrapper(*args, **kwargs): - if fail: - return MockResponse(None, 404) - - return MockResponse( - {"INSTANCE_NAME": "bossco", "INSTANCE_SUBDOMAIN": None, "PLAN_NAME": "Gratis", "NEW_USER_EXPERIENCE_ENABLED": True}, 200 - ) - - return wrapper - - -@patch("requests.get", side_effect=mocked_requests_get()) -def test_check_connection(mocker): - source = SourceInsightly() - logger_mock, config_mock = MagicMock(), MagicMock() - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -@patch("requests.get", side_effect=mocked_requests_get(fail=True)) -def test_check_connection_fail(mocker): - source = SourceInsightly() - logger_mock, config_mock = MagicMock(), MagicMock() - assert source.check_connection(logger_mock, config_mock)[0] is False - assert source.check_connection(logger_mock, config_mock)[1] is not None - - -def test_streams(mocker): - source = SourceInsightly() - config_mock = MagicMock() - streams = source.streams(config_mock) - - expected_streams_number = 37 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-insightly/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-insightly/unit_tests/test_streams.py deleted file mode 100644 index e44895f5db26..000000000000 --- a/airbyte-integrations/connectors/source-insightly/unit_tests/test_streams.py +++ /dev/null @@ -1,126 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from airbyte_cdk.sources.streams.http.auth import BasicHttpAuthenticator -from source_insightly.source import InsightlyStream - -authenticator = BasicHttpAuthenticator(username="test", password="") - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(InsightlyStream, "path", "v0/example_endpoint") - mocker.patch.object(InsightlyStream, "primary_key", "test_primary_key") - mocker.patch.object(InsightlyStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {"count_total": True, "skip": 0, "top": 500} - assert stream.request_params(**inputs) == expected_params - - -def test_request_param_with_next_page_token(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": 1000} - expected_params = {"count_total": True, "skip": 1000, "top": 500} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - stream.total_count = 10000 - - request = MagicMock() - request.url = "https://api.insight.ly/v0/example_endpoint?count_total=True&skip=0&top=500" - response = MagicMock() - response.status_code = HTTPStatus.OK - response.request = request - - inputs = {"response": response} - expected_token = 500 - assert stream.next_page_token(**inputs) == expected_token - - -def test_next_page_token_last_records(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - stream.total_count = 2100 - - request = MagicMock() - request.url = "https://api.insight.ly/v0/example_endpoint?count_total=True&skip=1500&top=500" - response = MagicMock() - response.status_code = HTTPStatus.OK - response.request = request - - inputs = {"response": response} - expected_token = 2000 - assert stream.next_page_token(**inputs) == expected_token - - -def test_next_page_token_no_more_records(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - stream.total_count = 1000 - - request = MagicMock() - request.url = "https://api.insight.ly/v0/example_endpoint?count_total=True&skip=1000&top=500" - response = MagicMock() - response.status_code = HTTPStatus.OK - response.request = request - - inputs = {"response": response} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_parse_response(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - - response = MagicMock() - response.json = MagicMock(return_value=[{"data_field": [{"keys": ["keys"]}]}]) - - inputs = {"stream_state": "test_stream_state", "response": response} - expected_parsed_object = response.json()[0] - assert next(stream.parse_response(**inputs)) == expected_parsed_object - - -def test_request_headers(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {"Accept": "application/json"} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = InsightlyStream(authenticator=authenticator) - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = InsightlyStream(authenticator=authenticator) - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = InsightlyStream(authenticator=authenticator) - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-instagram/Dockerfile b/airbyte-integrations/connectors/source-instagram/Dockerfile deleted file mode 100644 index 529b56cc22fb..000000000000 --- a/airbyte-integrations/connectors/source-instagram/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY source_instagram ./source_instagram -COPY main.py ./ -COPY setup.py ./ -RUN pip install . - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=1.0.11 -LABEL io.airbyte.name=airbyte/source-instagram diff --git a/airbyte-integrations/connectors/source-instagram/README.md b/airbyte-integrations/connectors/source-instagram/README.md index 241324c6002c..8cfb12455f66 100644 --- a/airbyte-integrations/connectors/source-instagram/README.md +++ b/airbyte-integrations/connectors/source-instagram/README.md @@ -1,6 +1,6 @@ -# Instagram Source +# Instagram Source -This is the repository for the Instagram source connector, written in Python. +This is the repository for the Instagram source connector, written in Python. For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/instagram). ## Local development @@ -29,22 +29,15 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-instagram:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/instagram) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_instagram/spec.json` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source instagram test creds` and place them into `secrets/config.json`. - ### Locally running the connector ``` python main.py spec @@ -53,41 +46,21 @@ python main.py discover --config secrets/config.json python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json ``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` - -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. -If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-instagram:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` -or -``` -./acceptance-test-docker.sh -``` - -To run your integration tests with docker - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-instagram:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name source-instagram build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-instagram:airbyteDocker +An image will be built with the tag `airbyte/source-instagram:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-instagram:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -95,21 +68,32 @@ Then run any of the connector commands as follows: docker run --rm airbyte/source-instagram:dev spec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-instagram:dev check --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-instagram:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-instagram:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-instagram:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-instagram:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-instagram test +``` + +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-instagram test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/instagram.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-instagram/acceptance-test-config.yml b/airbyte-integrations/connectors/source-instagram/acceptance-test-config.yml index 295efd229c13..cbb6e96e83a4 100644 --- a/airbyte-integrations/connectors/source-instagram/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-instagram/acceptance-test-config.yml @@ -31,30 +31,34 @@ acceptance_tests: bypass_reason: Stories available only 24 hours, so do the insights ignored_fields: users: - - name: media_count - bypass_reason: Updated each time when new post was added to live account - - name: followers_count - bypass_reason: Updated each time when followers updated was added to live account - - name: follows_count - bypass_reason: Updated each time when follows updated was added to live account - - name: profile_picture_url - bypass_reason: Contains auto generated hash + - name: media_count + bypass_reason: Updated each time when new post was added to live account + - name: followers_count + bypass_reason: Updated each time when followers updated was added to live account + - name: follows_count + bypass_reason: Updated each time when follows updated was added to live account + - name: profile_picture_url + bypass_reason: Contains auto generated hash user_lifetime_insights: - - name: value - bypass_reason: Contains PII data - - name: date - bypass_reason: Depend on current date + - name: value + bypass_reason: Contains PII data + - name: date + bypass_reason: Depend on current date user_insights: - - name: date - bypass_reason: Anonymization for exactly for which date statistics anonymization - - name: online_followers - bypass_reason: Depend on each online user + - name: date + bypass_reason: Anonymization for exactly for which date statistics anonymization + - name: online_followers + bypass_reason: Depend on each online user media: - - name: media_url - bypass_reason: Contains auto generated hash + - name: like_count + bypass_reason: Auto updated field + - name: media_url + bypass_reason: Contains auto generated hash + - name: thumbnail_url + bypass_reason: Contains auto generated hash media_insights: - - name: id - bypass_reason: For statistic anonymization + - name: id + bypass_reason: For statistic anonymization full_refresh: tests: - config_path: "secrets/config.json" @@ -72,10 +76,8 @@ acceptance_tests: configured_catalog_path: "integration_tests/incremental_catalog.json" future_state: future_state_path: "integration_tests/abnormal_state.json" - cursor_paths: - user_insights: ["17841408147298757", "date"] # because state is complex and stores values for different accounts on one hand # and there's no way we can set multiple cursor paths for a single stream on the other, # this test should be skipped as it is false negative. # (we can not restrict accounts via config as well) - skip_comprehensive_incremental_tests: true \ No newline at end of file + skip_comprehensive_incremental_tests: true diff --git a/airbyte-integrations/connectors/source-instagram/build.gradle b/airbyte-integrations/connectors/source-instagram/build.gradle deleted file mode 100644 index ef064e903c7e..000000000000 --- a/airbyte-integrations/connectors/source-instagram/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_instagram' -} diff --git a/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl index 925793748eef..4d4baa753a0e 100644 --- a/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-instagram/integration_tests/expected_records.jsonl @@ -1,8 +1,7 @@ -{"stream": "users", "data": {"id": "17841408147298757", "website": "https://www.airbyte.io/", "media_count": 0, "username": "airbytehq", "followers_count": 0, "follows_count": 0, "profile_picture_url": "https://scontent-mrs2-1.xx.fbcdn.net/v/t51.2885-15/153169696_890787328349641_8382928081987798464_n.jpg?_nc_cat=111&_nc_sid=86c713&_nc_ohc=EZf9l6dXYAkAX9UKDmz&_nc_ht=scontent-mrs2-1.xx&edm=AL-3X8kEAAAA&oh=00_AfDy5GVLylNGrfrbuixG1fol12GiBAKo0YkkRKu08nZfpg&oe=6402B9FE", "biography": "Airbyte is the new open-source data integration platform that consolidates your data into your warehouses.", "ig_id": 8070063576, "name": "Airbyte", "page_id": "112704783733939"}, "emitted_at": 1677611621424} -{"stream": "user_lifetime_insights", "data": {"page_id": "112704783733939", "business_account_id": "17841408147298757", "metric": "audience_city"}, "emitted_at": 1675080684688} -{"stream": "user_lifetime_insights", "data": {"page_id": "112704783733939", "business_account_id": "17841408147298757", "metric": "audience_country"}, "emitted_at": 1675080684688} -{"stream": "user_lifetime_insights", "data": {"page_id": "112704783733939", "business_account_id": "17841408147298757", "metric": "audience_gender_age"}, "emitted_at": 1675080684688} -{"stream": "user_lifetime_insights", "data": {"page_id": "112704783733939", "business_account_id": "17841408147298757", "metric": "audience_locale"}, "emitted_at": 1675080684688} -{"stream": "media", "data": {"id": "17922055321172142", "media_url": "https://scontent-mrs2-1.cdninstagram.com/v/t51.2885-15/35575181_234396600491925_6359096645375754240_n.jpg?_nc_cat=109&ccb=1-7&_nc_sid=8ae9d6&_nc_ohc=Ozc82doXbDYAX-5EB14&_nc_ht=scontent-mrs2-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfDtSblYdZs5I_S1Buhk3AUc-qU1EUTtEtmoB41dJ-zlHw&oe=6403D2F5", "timestamp": "2018-06-27T09:22:01+0000", "shortcode": "Bkhd1E4gEyu", "owner": {"id": "17841408147298757"}, "like_count": 1, "is_comment_enabled": true, "permalink": "https://www.instagram.com/p/Bkhd1E4gEyu/", "ig_id": "1810859715903638702", "media_type": "IMAGE", "username": "airbytehq", "caption": "And now we will open the fortune cookies that were made before 05/05!!", "comments_count": 0, "page_id": "112704783733939", "business_account_id": "17841408147298757"}, "emitted_at": 1677610834989} -{"stream": "media", "data": {"id": "17930423551138131", "media_url": "https://scontent-mrs2-2.cdninstagram.com/v/t51.2885-15/35999650_494254794340080_7942251990459875328_n.jpg?_nc_cat=108&ccb=1-7&_nc_sid=8ae9d6&_nc_ohc=p92Ke0D-uSMAX_8M5eN&_nc_ht=scontent-mrs2-2.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfAZ0f80E_esComf-cW1OTocAx5y1IhWudZ6c3sWj9ZJjw&oe=640435B5", "timestamp": "2018-06-26T09:22:01+0000", "shortcode": "Bke5CHUj6RZ", "owner": {"id": "17841408147298757"}, "like_count": 0, "is_comment_enabled": true, "permalink": "https://www.instagram.com/p/Bke5CHUj6RZ/", "ig_id": "1810134934200755289", "media_type": "IMAGE", "username": "airbytehq", "caption": "Every time !!! Come on...we're teasing you...are we \ud83e\udd14?", "comments_count": 0, "page_id": "112704783733939", "business_account_id": "17841408147298757"}, "emitted_at": 1677610834990} -{"stream": "media", "data": {"id": "17915811973199640", "media_url": "https://scontent-mrs2-2.cdninstagram.com/v/t51.2885-15/35574691_612934559075657_3171610615386996736_n.jpg?_nc_cat=108&ccb=1-7&_nc_sid=8ae9d6&_nc_ohc=UPpYhQeKF08AX95XXF6&_nc_ht=scontent-mrs2-2.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfCo5yj_JzcOAxPi61wATijfwkTusLcoZZdnQJosZz8Dbg&oe=64024D5C", "timestamp": "2018-06-25T09:23:03+0000", "shortcode": "BkcUW0-DK2x", "owner": {"id": "17841408147298757"}, "like_count": 0, "is_comment_enabled": true, "permalink": "https://www.instagram.com/p/BkcUW0-DK2x/", "ig_id": "1809410679930400177", "media_type": "IMAGE", "username": "airbytehq", "caption": "\"Can we postpone this?\" \"Nope\" \"Ok...\"", "comments_count": 0, "page_id": "112704783733939", "business_account_id": "17841408147298757"}, "emitted_at": 1677610834990} +{"stream": "users", "data": {"id": "17841408147298757", "website": "https://www.airbyte.io/", "ig_id": 8070063576, "followers_count": 1252, "name": "Jean Lafleur", "media_count": 258, "username": "airbytehq", "follows_count": 14, "biography": "Airbyte is the new open-source data integration platform that consolidates your data into your warehouses.", "profile_picture_url": "https://scontent-iev1-1.xx.fbcdn.net/v/t51.2885-15/153169696_890787328349641_8382928081987798464_n.jpg?_nc_cat=111&_nc_sid=7d201b&_nc_ohc=DFFn_25gYVMAX8nPfUd&_nc_ht=scontent-iev1-1.xx&edm=AL-3X8kEAAAA&oh=00_AfBHQPJ5aiFU1qw88d3gTF5jmg-Rpd5TX_gxAQt3jrSA4g&oe=655CCBBE", "page_id": "144706962067225"}, "emitted_at": 1700230802579} +{"stream": "media", "data": {"id": "17884386203808767", "caption": "Terraform Explained Part 1\n.\n.\n.\n#airbyte #dataengineering #tech #terraform #cloud #cloudengineer #coding #reels", "ig_id": "3123724930722523505", "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/B34BFFBB0614049AD69F066D153FDD8C_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=107&vs=986202625710684_1200838240&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC9CMzRCRkZCQjA2MTQwNDlBRDY5RjA2NkQxNTNGREQ4Q192aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dDQm9HQlV3a2JxUWwtY0JBRnZGTnFBUkdQeHpicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJtDf4euHnbtAFQIoAkMzLBdAUBtDlYEGJRgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfBPpWnNa8TFbux-TpRO48bJGSkaIKPFOnmXhcv39jLd_A&oe=6559369A&_nc_sid=1d576d", "owner": {"id": "17841408147298757"}, "shortcode": "CtZs0Y3v2lx", "username": "airbytehq", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/353022694_609901831117241_2447211336606431614_n.jpg?_nc_cat=100&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=1ZTHPkRhzl8AX-hZcw_&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfBdTKQTru0U2JNSqNnuPN0cWYv1u6o6t6u3EHIFteUV7w&oe=655C7D4E", "is_comment_enabled": true, "permalink": "https://www.instagram.com/reel/CtZs0Y3v2lx/", "timestamp": "2023-06-12T19:20:02+00:00", "like_count": 9, "comments_count": 2, "media_product_type": "REELS", "media_type": "VIDEO", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1700230757119} +{"stream": "media", "data": {"id": "17864256500936159", "caption": "When and why you should be using Rust for Data Engineering! \n\n#rust #airbyte #coding #programming #tech #dataengineering #data", "ig_id": "3106359072491902976", "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/BE4F848CC97FBA35A1AE1B1150B989A7_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNzIwLmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=110&vs=6290041361087047_1877877688&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC9CRTRGODQ4Q0M5N0ZCQTM1QTFBRTFCMTE1MEI5ODlBN192aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dEaE94aFJJdk1BWGZaWURBQXQyS0FLWWxOSlhicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJrD%2B6LaRwf1AFQIoAkMzLBdARDmZmZmZmhgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfC6GeTJWR8KJZ3-eb1-faBZ8P8G8AFyswEDdD4gFzmPMg&oe=65594B26&_nc_sid=1d576d", "owner": {"id": "17841408147298757"}, "shortcode": "CscAR5EsRgA", "username": "airbytehq", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/347441626_604256678433845_716271787932876577_n.jpg?_nc_cat=108&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=jLyY4sWj0v0AX-iadbF&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfA-x6QyIXxT7o_lEwDH0k7tDb_bgCGeP61AseCpluCtPA&oe=655D3C59", "is_comment_enabled": true, "permalink": "https://www.instagram.com/reel/CscAR5EsRgA/", "timestamp": "2023-05-19T20:08:33+00:00", "like_count": 7, "comments_count": 0, "media_product_type": "REELS", "media_type": "VIDEO", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1700230757120} +{"stream": "media", "data": {"id": "17964324206288599", "caption": "We've all been there right? \ud83e\udd23\n\n#airbyte #data #dataengineering #datascience #dataanalytics #tech #softwareengineer", "ig_id": "3104241732634871967", "media_url": "https://scontent-iev1-1.cdninstagram.com/o1/v/t16/f1/m82/274503D36EA0F6E79A7CF3797A8D5985_video_dashinit.mp4?efg=eyJ2ZW5jb2RlX3RhZyI6InZ0c192b2RfdXJsZ2VuLmNsaXBzLnVua25vd24tQzMuNTc2LmRhc2hfYmFzZWxpbmVfMV92MSJ9&_nc_ht=scontent-iev1-1.cdninstagram.com&_nc_cat=106&vs=1336282350269744_3931649106&_nc_vs=HBksFQIYT2lnX3hwdl9yZWVsc19wZXJtYW5lbnRfcHJvZC8yNzQ1MDNEMzZFQTBGNkU3OUE3Q0YzNzk3QThENTk4NV92aWRlb19kYXNoaW5pdC5tcDQVAALIAQAVAhg6cGFzc3Rocm91Z2hfZXZlcnN0b3JlL0dQdzNzaFRId3VlSlBFWURBSDFmTjUzcUNhd0JicV9FQUFBRhUCAsgBACgAGAAbAYgHdXNlX29pbAExFQAAJrDwmtqO44lAFQIoAkMzLBdAIewIMSbpeRgSZGFzaF9iYXNlbGluZV8xX3YxEQB1AAA%3D&ccb=9-4&oh=00_AfACHaQfoSJ_vMXbm4Xw3gmWnG_vnJgUsIYUePDdtIUS-w&oe=6558DBB2&_nc_sid=1d576d", "owner": {"id": "17841408147298757"}, "shortcode": "CsUe2iqpQif", "username": "airbytehq", "thumbnail_url": "https://scontent-iev1-1.cdninstagram.com/v/t51.36329-15/347429218_1848940842145573_5975413208994727174_n.jpg?_nc_cat=101&ccb=1-7&_nc_sid=c4dd86&_nc_ohc=Y6VzeGH_9lkAX_wkzpd&_nc_ht=scontent-iev1-1.cdninstagram.com&edm=AM6HXa8EAAAA&oh=00_AfDil0e2W7Iqq0-d7rf9JkdOluS7U2C3nhK17EfQ3c07fw&oe=655D28FC", "is_comment_enabled": true, "permalink": "https://www.instagram.com/reel/CsUe2iqpQif/", "timestamp": "2023-05-16T22:01:45+00:00", "like_count": 13, "comments_count": 0, "media_product_type": "REELS", "media_type": "VIDEO", "page_id": "144706962067225", "business_account_id": "17841408147298757"}, "emitted_at": 1700230757120} +{"stream":"user_lifetime_insights","data":{"page_id":"144706962067225","breakdown":"city","business_account_id":"17841408147298757","metric":"follower_demographics"},"emitted_at":1704378481116} +{"stream":"user_lifetime_insights","data":{"page_id":"144706962067225","breakdown":"country","business_account_id":"17841408147298757","metric":"follower_demographics"},"emitted_at":1704378481343} +{"stream":"user_lifetime_insights","data":{"page_id":"144706962067225","breakdown":"age,gender","business_account_id":"17841408147298757","metric":"follower_demographics"},"emitted_at":1704378481574} diff --git a/airbyte-integrations/connectors/source-instagram/integration_tests/spec.json b/airbyte-integrations/connectors/source-instagram/integration_tests/spec.json index 81d9934b99b8..f3fbd6e9dc22 100644 --- a/airbyte-integrations/connectors/source-instagram/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-instagram/integration_tests/spec.json @@ -7,7 +7,7 @@ "properties": { "start_date": { "title": "Start Date", - "description": "The date from which you'd like to replicate data for User Insights, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", + "description": "The date from which you'd like to replicate data for User Insights, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated. If left blank, the start date will be set to 2 years before the present date.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", "examples": ["2017-01-25T00:00:00Z"], "type": "string", @@ -34,7 +34,7 @@ "type": "string" } }, - "required": ["start_date", "access_token"] + "required": ["access_token"] }, "supportsIncremental": true, "supported_destination_sync_modes": ["append"], diff --git a/airbyte-integrations/connectors/source-instagram/main.py b/airbyte-integrations/connectors/source-instagram/main.py index 7dfe30785519..0a871930a015 100644 --- a/airbyte-integrations/connectors/source-instagram/main.py +++ b/airbyte-integrations/connectors/source-instagram/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_instagram import SourceInstagram +from source_instagram.run import run if __name__ == "__main__": - source = SourceInstagram() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-instagram/metadata.yaml b/airbyte-integrations/connectors/source-instagram/metadata.yaml index d15bd17a275f..0c669ca2fe5d 100644 --- a/airbyte-integrations/connectors/source-instagram/metadata.yaml +++ b/airbyte-integrations/connectors/source-instagram/metadata.yaml @@ -2,10 +2,12 @@ data: allowedHosts: hosts: - graph.facebook.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 6acf6b55-4f1e-4fca-944e-1a3caef8aba8 - dockerImageTag: 1.0.11 + dockerImageTag: 3.0.2 dockerRepository: airbyte/source-instagram githubIssueLabel: source-instagram icon: instagram.svg @@ -17,6 +19,29 @@ data: oss: enabled: true releaseStage: generally_available + releases: + breakingChanges: + 3.0.0: + message: "The existing Instagram API (v11) has been deprecated. Customers who use streams `Media Insights`, `Story Insights` or `User Lifetime Insights` must take action with their connections. Please follow the to update to the latest Instagram API (v18). For more details, see our migration guide." + upgradeDeadline: "2024-01-05" + scopedImpact: + - scopeType: stream + impactedScopes: + ["media_insights", "story_insights", "user_lifetime_insights"] + 2.0.0: + message: + This release introduces a default primary key for the streams UserLifetimeInsights and UserInsights. + Additionally, the format of timestamp fields has been updated in the UserLifetimeInsights, UserInsights, Media and Stories streams to include timezone information. + upgradeDeadline: "2023-12-11" + suggestedStreams: + streams: + - media + - media_insights + - stories + - user_insights + - story_insights + - users + - user_lifetime_insights documentationUrl: https://docs.airbyte.com/integrations/sources/instagram tags: - language:python diff --git a/airbyte-integrations/connectors/source-instagram/setup.py b/airbyte-integrations/connectors/source-instagram/setup.py index a15d5f7ab25e..cfaf2e20122f 100644 --- a/airbyte-integrations/connectors/source-instagram/setup.py +++ b/airbyte-integrations/connectors/source-instagram/setup.py @@ -8,7 +8,7 @@ MAIN_REQUIREMENTS = [ "airbyte-cdk", "cached_property~=1.5", - "facebook_business~=11.0", + "facebook_business~=18.0.5", ] TEST_REQUIREMENTS = [ @@ -19,6 +19,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-instagram=source_instagram.run:run", + ], + }, name="source_instagram", description="Source implementation for Instagram.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/api.py b/airbyte-integrations/connectors/source-instagram/source_instagram/api.py index 1f73d0460141..16426efc1f9a 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/api.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/api.py @@ -17,7 +17,7 @@ from facebook_business.exceptions import FacebookRequestError from source_instagram.common import InstagramAPIException, retry_pattern -backoff_policy = retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5) +backoff_policy = retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5, max_time=600) class MyFacebookAdsApi(FacebookAdsApi): @@ -65,7 +65,6 @@ def call( class InstagramAPI: def __init__(self, access_token: str): - self._api = FacebookAdsApi.init(access_token=access_token) # design flaw in MyFacebookAdsApi requires such strange set of new default api instance self.api = MyFacebookAdsApi.init(access_token=access_token, crash_log=False) FacebookAdsApi.set_default_api(self.api) diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/common.py b/airbyte-integrations/connectors/source-instagram/source_instagram/common.py index 1899bbead023..25f8673d0ebf 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/common.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/common.py @@ -37,6 +37,20 @@ def should_retry_api_error(exc: FacebookRequestError): if exc.api_error_code() in (4, 17, 32, 613): return True + if ( + exc.http_status() == status_codes.INTERNAL_SERVER_ERROR + and exc.api_error_code() == 1 + and exc.api_error_message() == "Please reduce the amount of data you're asking for, then retry your request" + ): + return True + + if ( + exc.http_status() == status_codes.INTERNAL_SERVER_ERROR + and exc.api_error_code() == 1 + and exc.api_error_message() == "An unknown error occurred" + ): + return True + if exc.http_status() == status_codes.TOO_MANY_REQUESTS: return True diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/run.py b/airbyte-integrations/connectors/source-instagram/source_instagram/run.py new file mode 100644 index 000000000000..c012b2e2292a --- /dev/null +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_instagram import SourceInstagram + + +def run(): + source = SourceInstagram() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media.json b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media.json index 02eea4f43f92..03c77796f5a0 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media.json +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media.json @@ -28,6 +28,9 @@ "media_type": { "type": ["null", "string"] }, + "media_product_type": { + "type": ["null", "string"] + }, "media_url": { "type": ["null", "string"] }, @@ -50,7 +53,8 @@ }, "timestamp": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "username": { "type": ["null", "string"] @@ -91,7 +95,8 @@ }, "timestamp": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "username": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media_insights.json b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media_insights.json index 8a1759549e83..63aa03b6efcc 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media_insights.json +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media_insights.json @@ -10,8 +10,11 @@ "id": { "type": ["null", "string"] }, - "engagement": { - "type": ["null", "integer"] + "ig_reels_avg_watch_time": { + "type": ["null", "number"] + }, + "ig_reels_video_view_total_time": { + "type": ["null", "number"] }, "impressions": { "type": ["null", "integer"] @@ -25,18 +28,6 @@ "video_views": { "type": ["null", "integer"] }, - "carousel_album_engagement": { - "type": ["null", "integer"] - }, - "carousel_album_impressions": { - "type": ["null", "integer"] - }, - "carousel_album_reach": { - "type": ["null", "integer"] - }, - "carousel_album_saved": { - "type": ["null", "integer"] - }, "comments": { "type": ["null", "integer"] }, diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/stories.json b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/stories.json index 47e4af00aeb9..876edf95ea41 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/stories.json +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/stories.json @@ -22,6 +22,9 @@ "media_type": { "type": ["null", "string"] }, + "media_product_type": { + "type": ["null", "string"] + }, "media_url": { "type": ["null", "string"] }, @@ -44,7 +47,8 @@ }, "timestamp": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "username": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/story_insights.json b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/story_insights.json index 81513dcd8246..cf81cd498060 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/story_insights.json +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/story_insights.json @@ -10,9 +10,6 @@ "id": { "type": ["null", "string"] }, - "exits": { - "type": ["null", "integer"] - }, "impressions": { "type": ["null", "integer"] }, @@ -21,12 +18,6 @@ }, "replies": { "type": ["null", "integer"] - }, - "taps_forward": { - "type": ["null", "integer"] - }, - "taps_back": { - "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_insights.json b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_insights.json index ab81d27d09ea..91bc309d8eb6 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_insights.json +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_insights.json @@ -9,7 +9,8 @@ }, "date": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "follower_count": { "type": ["null", "integer"] @@ -49,6 +50,9 @@ }, "online_followers": { "type": ["null", "object"] + }, + "email_contacts": { + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_lifetime_insights.json b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_lifetime_insights.json index eb9bb57fc720..40265de413f6 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_lifetime_insights.json +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/user_lifetime_insights.json @@ -7,15 +7,14 @@ "business_account_id": { "type": ["null", "string"] }, - "date": { - "type": ["null", "string"], - "format": "date-time" + "breakdown": { + "type": ["null", "string"] }, "metric": { "type": ["null", "string"] }, "value": { - "type": ["integer", "object"] + "type": ["null", "object"] } } } diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/source.py b/airbyte-integrations/connectors/source-instagram/source_instagram/source.py index 6f78da2614c7..4a41d013c1a9 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/source.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/source.py @@ -5,6 +5,7 @@ from datetime import datetime from typing import Any, List, Mapping, Optional, Tuple +import pendulum from airbyte_cdk.models import AdvancedAuth, ConnectorSpecification, DestinationSyncMode, OAuthConfigSpecification from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -17,8 +18,8 @@ class ConnectorConfig(BaseModel): class Config: title = "Source Instagram" - start_date: datetime = Field( - description="The date from which you'd like to replicate data for User Insights, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", + start_date: Optional[datetime] = Field( + description="The date from which you'd like to replicate data for User Insights, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated. If left blank, the start date will be set to 2 years before the present date.", pattern="^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", examples=["2017-01-25T00:00:00Z"], ) @@ -59,8 +60,9 @@ def check_connection(self, logger, config: Mapping[str, Any]) -> Tuple[bool, Any error_msg = None try: - config = ConnectorConfig.parse_obj(config) # FIXME: this will be not need after we fix CDK - api = InstagramAPI(access_token=config.access_token) + self._validate_start_date(config) + + api = InstagramAPI(access_token=config["access_token"]) logger.info(f"Available accounts: {api.accounts}") ok = True except Exception as exc: @@ -68,13 +70,22 @@ def check_connection(self, logger, config: Mapping[str, Any]) -> Tuple[bool, Any return ok, error_msg + def _validate_start_date(self, config): + # If start_date is not found in config, set it to 2 years ago + if not config.get("start_date"): + config["start_date"] = pendulum.now().subtract(years=2).in_timezone("UTC").format("YYYY-MM-DDTHH:mm:ss.SSS[Z]") + else: + if pendulum.parse(config["start_date"]) > pendulum.now(): + raise ValueError("Please fix the start_date parameter in config, it cannot be in the future") + def streams(self, config: Mapping[str, Any]) -> List[Stream]: """Discovery method, returns available streams :param config: A Mapping of the user input configuration as defined in the connector spec. """ - config: ConnectorConfig = ConnectorConfig.parse_obj(config) # FIXME: this will be not need after we fix CDK - api = InstagramAPI(access_token=config.access_token) + api = InstagramAPI(access_token=config["access_token"]) + + self._validate_start_date(config) return [ Media(api=api), @@ -83,7 +94,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: StoryInsights(api=api), Users(api=api), UserLifetimeInsights(api=api), - UserInsights(api=api, start_date=config.start_date), + UserInsights(api=api, start_date=config["start_date"]), ] def spec(self, *args, **kwargs) -> ConnectorSpecification: diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py index 72239476831f..4e6d27c4fb2c 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py @@ -11,6 +11,7 @@ import pendulum from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import IncrementalMixin, Stream +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from cached_property import cached_property from facebook_business.adobjects.igmedia import IGMedia from facebook_business.exceptions import FacebookRequestError @@ -19,6 +20,24 @@ from .common import remove_params_from_url +class DatetimeTransformerMixin: + transformer: TypeTransformer = TypeTransformer(TransformConfig.CustomSchemaNormalization) + + @staticmethod + @transformer.registerCustomTransform + def custom_transform_datetime_rfc3339(original_value, field_schema): + """ + Transform datetime string to RFC 3339 format + """ + if original_value and field_schema.get("format") == "date-time" and field_schema.get("airbyte_type") == "timestamp_with_timezone": + # Parse the ISO format timestamp + dt = pendulum.parse(original_value) + + # Convert to RFC 3339 format + return dt.to_rfc3339_string() + return original_value + + class InstagramStream(Stream, ABC): """Base stream class""" @@ -81,7 +100,7 @@ class InstagramIncrementalStream(InstagramStream, IncrementalMixin): def __init__(self, start_date: datetime, **kwargs): super().__init__(**kwargs) - self._start_date = pendulum.instance(start_date) + self._start_date = pendulum.parse(start_date) self._state = {} @property @@ -121,13 +140,21 @@ def read_records( yield self.transform(record) -class UserLifetimeInsights(InstagramStream): +class UserLifetimeInsights(DatetimeTransformerMixin, InstagramStream): """Docs: https://developers.facebook.com/docs/instagram-api/reference/ig-user/insights""" - primary_key = None - LIFETIME_METRICS = ["audience_city", "audience_country", "audience_gender_age", "audience_locale"] + primary_key = ["business_account_id", "breakdown"] + BREAKDOWNS = ["city", "country", "age,gender"] + BASE_METRIC = ["follower_demographics"] period = "lifetime" + def stream_slices( + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + for slice in super().stream_slices(sync_mode=sync_mode, cursor_field=cursor_field, stream_state=stream_state): + for breakdown in self.BREAKDOWNS: + yield slice | {"breakdown": breakdown} + def read_records( self, sync_mode: SyncMode, @@ -137,13 +164,14 @@ def read_records( ) -> Iterable[Mapping[str, Any]]: account = stream_slice["account"] ig_account = account["instagram_business_account"] - for insight in ig_account.get_insights(params=self.request_params()): + for insight in ig_account.get_insights(params=self.request_params(stream_slice=stream_slice)): + insight_data = insight.export_all_data() yield { "page_id": account["page_id"], + "breakdown": stream_slice["breakdown"], "business_account_id": ig_account.get("id"), "metric": insight["name"], - "date": insight["values"][0].get("end_time"), - "value": insight["values"][0].get("value"), + "value": self._transform_breakdown_results(insight_data["total_value"]["breakdowns"][0]["results"]), } def request_params( @@ -152,11 +180,17 @@ def request_params( stream_state: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: params = super().request_params(stream_slice=stream_slice, stream_state=stream_state) - params.update({"metric": self.LIFETIME_METRICS, "period": self.period}) + params.update( + {"metric": self.BASE_METRIC, "metric_type": "total_value", "period": self.period, "breakdown": stream_slice["breakdown"]} + ) return params + @staticmethod + def _transform_breakdown_results(breakdown_results: Iterable[Mapping[str, Any]]) -> Mapping[str, Any]: + return {res.get("dimension_values")[0]: res.get("value") for res in breakdown_results} + -class UserInsights(InstagramIncrementalStream): +class UserInsights(DatetimeTransformerMixin, InstagramIncrementalStream): """Docs: https://developers.facebook.com/docs/instagram-api/reference/ig-user/insights""" METRICS_BY_PERIOD = { @@ -176,7 +210,7 @@ class UserInsights(InstagramIncrementalStream): "lifetime": ["online_followers"], } - primary_key = None + primary_key = ["business_account_id", "date"] cursor_field = "date" # For some metrics we can only get insights not older than 30 days, it is Facebook policy @@ -295,13 +329,13 @@ def _state_has_legacy_format(self, state: Mapping[str, Any]) -> bool: return False -class Media(InstagramStream): +class Media(DatetimeTransformerMixin, InstagramStream): """Children objects can only be of the media_type == "CAROUSEL_ALBUM". - And children object does not support INVALID_CHILDREN_FIELDS fields, + And children objects do not support INVALID_CHILDREN_FIELDS fields, so they are excluded when trying to get child objects to avoid the error """ - INVALID_CHILDREN_FIELDS = ["caption", "comments_count", "is_comment_enabled", "like_count", "children"] + INVALID_CHILDREN_FIELDS = ["caption", "comments_count", "is_comment_enabled", "like_count", "children", "media_product_type"] def read_records( self, @@ -339,9 +373,20 @@ def _get_children(self, ids: List): class MediaInsights(Media): """Docs: https://developers.facebook.com/docs/instagram-api/reference/ig-media/insights""" - MEDIA_METRICS = ["engagement", "impressions", "reach", "saved"] - CAROUSEL_ALBUM_METRICS = ["carousel_album_engagement", "carousel_album_impressions", "carousel_album_reach", "carousel_album_saved"] - REELS_METRICS = ["comments", "likes", "reach", "saved", "shares", "total_interactions", "plays"] + MEDIA_METRICS = ["total_interactions", "impressions", "reach", "saved", "video_views", "likes", "comments", "shares"] + CAROUSEL_ALBUM_METRICS = ["total_interactions", "impressions", "reach", "saved", "video_views"] + + REELS_METRICS = [ + "comments", + "ig_reels_avg_watch_time", + "ig_reels_video_view_total_time", + "likes", + "plays", + "reach", + "saved", + "shares", + "total_interactions", + ] def read_records( self, @@ -368,6 +413,8 @@ def _get_insights(self, item, account_id) -> Optional[MutableMapping[str, Any]]: """Get insights for specific media""" if item.get("media_product_type") == "REELS": metrics = self.REELS_METRICS + elif item.get("media_type") == "VIDEO" and item.get("media_product_type") == "FEED": + metrics = ["impressions", "reach", "saved", "video_views", "video_views"] elif item.get("media_type") == "VIDEO": metrics = self.MEDIA_METRICS + ["video_views"] elif item.get("media_type") == "CAROUSEL_ALBUM": @@ -380,21 +427,30 @@ def _get_insights(self, item, account_id) -> Optional[MutableMapping[str, Any]]: insights = item.get_insights(params={"metric": metrics}) return {record.get("name"): record.get("values")[0]["value"] for record in insights} except FacebookRequestError as error: + error_code = error.api_error_code() + error_subcode = error.api_error_subcode() + error_message = error.api_error_message() + # An error might occur if the media was posted before the most recent time that # the user's account was converted to a business account from a personal account - if error.api_error_subcode() == 2108006: - details = error.body().get("error", {}).get("error_user_title") or error.api_error_message() + if error_subcode == 2108006: + details = error.body().get("error", {}).get("error_user_title") or error_message self.logger.error(f"Insights error for business_account_id {account_id}: {details}") # We receive all Media starting from the last one, and if on the next Media we get an Insight error, # then no reason to make inquiries for each Media further, since they were published even earlier. return None - elif error.api_error_code() == 100 and error.api_error_subcode() == 33: - self.logger.error(f"Check provided permissions for {account_id}: {error.api_error_message()}") + elif ( + error_code == 100 + and error_subcode == 33 + or error_code == 10 + and error_message == "(#10) Application does not have permission for this action" + ): + self.logger.error(f"Check provided permissions for {account_id}: {error_message}") return None raise error -class Stories(InstagramStream): +class Stories(DatetimeTransformerMixin, InstagramStream): """Docs: https://developers.facebook.com/docs/instagram-api/reference/ig-user/stories""" def read_records( @@ -417,7 +473,7 @@ def read_records( class StoryInsights(Stories): """Docs: https://developers.facebook.com/docs/instagram-api/reference/ig-media/insights""" - metrics = ["exits", "impressions", "reach", "replies", "taps_forward", "taps_back"] + metrics = ["impressions", "reach", "replies"] def read_records( self, diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py b/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py index cbe45c836813..44be2de8ca2c 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/conftest.py @@ -36,6 +36,11 @@ def some_config_fixture(account_id): return {"start_date": "2021-01-23T00:00:00Z", "access_token": "unknown_token"} +@fixture(scope="session", name="some_config_future_date") +def some_config_future_date_fixture(account_id): + return {"start_date": "2030-01-23T00:00:00Z", "access_token": "unknown_token"} + + @fixture(name="fb_account_response") def fb_account_response_fixture(account_id, some_config, requests_mock): account = {"id": "test_id", "instagram_business_account": {"id": "test_id"}} @@ -49,13 +54,16 @@ def fb_account_response_fixture(account_id, some_config, requests_mock): "json": { "data": [ { - "account_id": account_id, + "access_token": "access_token", + "category": "Software company", "id": f"act_{account_id}", - } - ], - "paging": {"cursors": {"before": "MjM4NDYzMDYyMTcyNTAwNzEZD", "after": "MjM4NDYzMDYyMTcyNTAwNzEZD"}}, - }, - "status_code": 200, + "paging": {"cursors": { + "before": "cursor", + "after": "cursor"}}, + "summary": {"total_count": 1}, + "status_code": 200 + }] + } } @@ -94,6 +102,22 @@ def user_insight_data_fixture(): } +@fixture(name="user_lifetime_insight_data") +def user_lifetime_insight_data_fixture(): + return { + "name": "impressions", + "period": "day", + "total_value": {"breakdowns": [ + {"dimension_keys": ["city"], "results": [{"dimension_values": ["London, England"], "value": 22}, + {"dimension_values": ["Sydney, New South Wales"], "value": 33} + ]} + ]}, + "title": "Impressions", + "description": "Total number of times this profile has been seen", + "id": "17841400008460056/insights/impressions/day", + } + + @fixture(name="user_lifetime_insights") def user_lifetime_insights(): class UserLiftimeInsightEntityMock: @@ -129,7 +153,6 @@ class UserInsightEntityMock: # reference Issue: # https://github.com/airbytehq/airbyte/issues/24697 class UserInsight: - def __init__(self, values: dict): self.data = { "description": "test_insight", diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py b/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py index 41b6467f5512..add26ad1a33f 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/test_source.py @@ -32,14 +32,21 @@ def test_check_connection_empty_config(api): assert error_msg -def test_check_connection_invalid_config(api, some_config): - some_config.pop("start_date") - ok, error_msg = SourceInstagram().check_connection(logger, config=some_config) +def test_check_connection_invalid_config_future_date(api, some_config_future_date): + ok, error_msg = SourceInstagram().check_connection(logger, config=some_config_future_date) assert not ok assert error_msg +def test_check_connection_no_date_config(api, some_config): + some_config.pop("start_date") + ok, error_msg = SourceInstagram().check_connection(logger, config=some_config) + + assert ok + assert not error_msg + + def test_check_connection_exception(api, config): api.side_effect = RuntimeError("Something went wrong!") ok, error_msg = SourceInstagram().check_connection(logger, config=config) diff --git a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py index 7d016c67f483..0d6d1779272b 100644 --- a/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-instagram/unit_tests/test_streams.py @@ -9,6 +9,7 @@ from airbyte_cdk.models import SyncMode from facebook_business import FacebookAdsApi, FacebookSession from source_instagram.streams import ( + DatetimeTransformerMixin, InstagramStream, Media, MediaInsights, @@ -32,15 +33,11 @@ def test_clear_url(config): def test_state_outdated(api, config): - assert UserInsights(api=api, start_date=datetime.strptime(config["start_date"], "%Y-%m-%dT%H:%M:%S"))._state_has_legacy_format( - {"state": MagicMock()} - ) + assert UserInsights(api=api, start_date=config["start_date"])._state_has_legacy_format({"state": MagicMock()}) def test_state_is_not_outdated(api, config): - assert not UserInsights(api=api, start_date=datetime.strptime(config["start_date"], "%Y-%m-%dT%H:%M:%S"))._state_has_legacy_format( - {"state": {}} - ) + assert not UserInsights(api=api, start_date=config["start_date"])._state_has_legacy_format({"state": {}}) def test_media_get_children(api, requests_mock, some_config): @@ -76,7 +73,7 @@ def test_media_insights_read(api, user_stories_data, user_media_insights_data, r def test_media_insights_read_error(api, requests_mock): test_id = "test_id" stream = MediaInsights(api=api) - media_response = [{"id": "test_id"}, {"id": "test_id_2"}, {"id": "test_id_3"}, {"id": "test_id_4"}] + media_response = [{"id": "test_id"}, {"id": "test_id_2"}, {"id": "test_id_3"}, {"id": "test_id_4"}, {"id": "test_id_5"}] requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{test_id}/media", json={"data": media_response}) media_insights_response_test_id = { @@ -99,10 +96,12 @@ def test_media_insights_read_error(api, requests_mock): "is_transient": False, "error_user_title": "Media posted before business account conversion", "error_user_msg": "The media was posted before the most recent time that the user's account was converted to a business account from a personal account.", - "fbtrace_id": "fake_trace_id" + "fbtrace_id": "fake_trace_id", } } - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/test_id_2/insights", json=error_response_oauth, status_code=400) + requests_mock.register_uri( + "GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/test_id_2/insights", json=error_response_oauth, status_code=400 + ) error_response_wrong_permissions = { "error": { @@ -113,10 +112,12 @@ def test_media_insights_read_error(api, requests_mock): "error_subcode": 33, "is_transient": False, "error_user_msg": "Unsupported get request. Object with ID 'test_id_3' does not exist, cannot be loaded due to missing permissions, or does not support this operation.", - "fbtrace_id": "fake_trace_id" + "fbtrace_id": "fake_trace_id", } } - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/test_id_3/insights", json=error_response_wrong_permissions, status_code=400) + requests_mock.register_uri( + "GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/test_id_3/insights", json=error_response_wrong_permissions, status_code=400 + ) media_insights_response_test_id_4 = { "name": "impressions", @@ -126,17 +127,30 @@ def test_media_insights_read_error(api, requests_mock): "description": "Total number of times the media object has been seen", "id": "test_id_3/insights/impressions/lifetime", } - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/test_id_4/insights", json=media_insights_response_test_id_4) + requests_mock.register_uri( + "GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/test_id_4/insights", json=media_insights_response_test_id_4 + ) + + error_response_wrong_permissions_code_10 = { + "error": { + "message": "(#10) Application does not have permission for this action", + "type": "OAuthException", + "code": 10, + "fbtrace_id": "fake_trace_id", + } + } + requests_mock.register_uri( + "GET", + FacebookSession.GRAPH + f"/{FB_API_VERSION}/test_id_5/insights", + json=error_response_wrong_permissions_code_10, + status_code=400, + ) records = read_full_refresh(stream) - expected_records = [{"business_account_id": "test_id", - "id": "test_id", - "impressions": 264, - "page_id": "act_unknown_account"}, - {"business_account_id": "test_id", - "id": "test_id_4", - "impressions": 300, - "page_id": "act_unknown_account"}] + expected_records = [ + {"business_account_id": "test_id", "id": "test_id", "impressions": 264, "page_id": "act_unknown_account"}, + {"business_account_id": "test_id", "id": "test_id_4", "impressions": 300, "page_id": "act_unknown_account"}, + ] assert records == expected_records @@ -161,7 +175,7 @@ def test_user_read(api, user_data, requests_mock): def test_user_insights_read(api, config, user_insight_data, requests_mock): test_id = "test_id" - stream = UserInsights(api=api, start_date=datetime.strptime(config["start_date"], "%Y-%m-%dT%H:%M:%S")) + stream = UserInsights(api=api, start_date=config["start_date"]) requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{test_id}/insights", [{"json": user_insight_data}]) @@ -169,50 +183,22 @@ def test_user_insights_read(api, config, user_insight_data, requests_mock): assert records -def test_user_lifetime_insights_read(api, config, user_insight_data, requests_mock): +def test_user_lifetime_insights_read(api, config, user_lifetime_insight_data, requests_mock): test_id = "test_id" stream = UserLifetimeInsights(api=api) - requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{test_id}/insights", [{"json": user_insight_data}]) + requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{test_id}/insights", [{"json": user_lifetime_insight_data}]) records = read_full_refresh(stream) - assert records == [ - { - "page_id": "act_unknown_account", - "business_account_id": "test_id", - "metric": "impressions", - "date": "2020-05-04T07:00:00+0000", - "value": 4, - } - ] - - -@pytest.mark.parametrize( - "values,expected", - [ - ({"end_time": "test_end_time", "value": "test_value"}, {"date": "test_end_time", "value": "test_value"}), - ({"value": "test_value"}, {"date": None, "value": "test_value"}), - ({"end_time": "test_end_time"}, {"date": "test_end_time", "value": None}), - ({}, {"date": None, "value": None}), - ], - ids=[ - "`end_time` and `value` are present", - "no `end_time`, but `value` is present", - "`end_time` is present, but no `value`", - "no `end_time` and no `value`", - ] -) -def test_user_lifetime_insights_read_with_missing_keys(api, user_lifetime_insights, values, expected): - """ - This tests shows the behaviour of the `read_records` when either `end_time` or `value` key is not present in the data. - """ - stream = UserLifetimeInsights(api=api) - user_lifetime_insights(values) - test_slice = {"account": {"page_id": 1, "instagram_business_account": user_lifetime_insights}} - for insight in stream.read_records(sync_mode=None, stream_slice=test_slice): - assert insight["date"] == expected.get("date") - assert insight["value"] == expected.get("value") + expected_record = { + "breakdown": "city", + "business_account_id": "test_id", + "metric": "impressions", + "page_id": "act_unknown_account", + "value": {"London, England": 22, "Sydney, New South Wales": 33} + } + assert expected_record in records @pytest.mark.parametrize( @@ -249,7 +235,7 @@ def test_user_lifetime_insights_read_with_missing_keys(api, user_lifetime_insigh "No `end_time` value in record", "No `value` in record", "No `end_time` and no `value` in record", - ] + ], ) def test_user_insights_state(api, user_insights, values, slice_dates, expected): """ @@ -258,7 +244,7 @@ def test_user_insights_state(api, user_insights, values, slice_dates, expected): import pendulum # UserInsights stream - stream = UserInsights(api=api, start_date=pendulum.parse("2023-01-01T01:01:01Z")) + stream = UserInsights(api=api, start_date="2023-01-01T01:01:01Z") # Populate the fixute with `values` user_insights(values) # simulate `read_recods` generator job @@ -297,6 +283,11 @@ def test_stories_insights_read(api, requests_mock, user_stories_data, user_media {"json": {"error": {"type": "OAuthException", "code": 1}}}, {"json": {"error": {"code": 4}}}, {"json": {}, "status_code": 429}, + { + "json": {"error": {"code": 1, "message": "Please reduce the amount of data you're asking for, then retry your request"}}, + "status_code": 500, + }, + {"json": {"error": {"code": 1, "message": "An unknown error occurred"}}, "status_code": 500}, {"json": {"error": {"type": "OAuthException", "message": "(#10) Not enough viewers for the media to show insights", "code": 10}}}, {"json": {"error": {"code": 100, "error_subcode": 33}}, "status_code": 400}, {"json": {"error": {"is_transient": True}}}, @@ -306,6 +297,8 @@ def test_stories_insights_read(api, requests_mock, user_stories_data, user_media "oauth_error", "rate_limit_error", "too_many_request_error", + "reduce_amount_of_data_error", + "unknown_error", "viewers_insights_error", "4028_issue_error", "transient_error", @@ -333,9 +326,28 @@ def test_common_error_retry(error_response, requests_mock, api, account_id): def test_exit_gracefully(api, config, requests_mock, caplog): test_id = "test_id" - stream = UserInsights(api=api, start_date=datetime.strptime(config["start_date"], "%Y-%m-%dT%H:%M:%S")) + stream = UserInsights(api=api, start_date=config["start_date"]) requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FB_API_VERSION}/{test_id}/insights", json={"data": []}) records = read_incremental(stream, {}) assert not records assert requests_mock.call_count == 6 # 4 * 1 per `metric_to_period` map + 1 `summary` request + 1 `business_account_id` request assert "Stopping syncing stream 'user_insights'" in caplog.text + + +@pytest.mark.parametrize( + "original_value, field_schema, expected", + [ + ("2020-01-01T12:00:00Z", {"format": "date-time", "airbyte_type": "timestamp_with_timezone"}, "2020-01-01T12:00:00+00:00"), + ("2020-05-04T07:00:00+0000", {"format": "date-time", "airbyte_type": "timestamp_with_timezone"}, "2020-05-04T07:00:00+00:00"), + (None, {"format": "date-time", "airbyte_type": "timestamp_with_timezone"}, None), + ("2020-01-01T12:00:00", {"format": "date-time", "airbyte_type": "timestamp_without_timezone"}, "2020-01-01T12:00:00"), + ("2020-01-01T14:00:00", {"format": "date-time"}, "2020-01-01T14:00:00"), + ("2020-02-03T12:00:00", {"type": "string"}, "2020-02-03T12:00:00"), + ], +) +def test_custom_transform_datetime_rfc3339(original_value, field_schema, expected): + # Call the static method + result = DatetimeTransformerMixin.custom_transform_datetime_rfc3339(original_value, field_schema) + + # Assert the result matches the expected output + assert result == expected diff --git a/airbyte-integrations/connectors/source-instatus/README.md b/airbyte-integrations/connectors/source-instatus/README.md index 8ea3e3f1b455..3eaac6c4cbf1 100644 --- a/airbyte-integrations/connectors/source-instatus/README.md +++ b/airbyte-integrations/connectors/source-instatus/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-instatus:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/instatus) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_instatus/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-instatus:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-instatus build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-instatus:airbyteDocker +An image will be built with the tag `airbyte/source-instatus:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-instatus:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-instatus:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-instatus:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-instatus:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-instatus test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-instatus:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-instatus:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-instatus test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/instatus.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-instatus/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-instatus/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-instatus/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-instatus/build.gradle b/airbyte-integrations/connectors/source-instatus/build.gradle deleted file mode 100644 index f266aa6a3538..000000000000 --- a/airbyte-integrations/connectors/source-instatus/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_instatus' -} diff --git a/airbyte-integrations/connectors/source-intercom/Dockerfile b/airbyte-integrations/connectors/source-intercom/Dockerfile deleted file mode 100644 index d1ffa9e1739c..000000000000 --- a/airbyte-integrations/connectors/source-intercom/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_intercom ./source_intercom - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.3.0 -LABEL io.airbyte.name=airbyte/source-intercom diff --git a/airbyte-integrations/connectors/source-intercom/README.md b/airbyte-integrations/connectors/source-intercom/README.md index 66182dd1920d..d5b904c935df 100644 --- a/airbyte-integrations/connectors/source-intercom/README.md +++ b/airbyte-integrations/connectors/source-intercom/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-intercom:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/intercom) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_intercom/spec.yaml` file. @@ -56,19 +48,70 @@ python main.py discover --config secrets/config.json python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json ``` -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-intercom:dev + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-intercom build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-intercom:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. -You can also build the connector image via Gradle: +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-intercom:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-intercom:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-intercom:dev . +# Running the spec command against your patched connector +docker run airbyte/source-intercom:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -77,33 +120,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-intercom:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-intercom:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-intercom:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing - -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. -If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. - -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-intercom:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-intercom test ``` -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-intercom:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-intercom:integrationTest -``` +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -113,8 +139,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-intercom test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/intercom.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml b/airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml index 2bcc47ce0751..b2c5d59e44cc 100644 --- a/airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml @@ -5,42 +5,34 @@ test_strictness_level: high acceptance_tests: spec: tests: - - spec_path: "source_intercom/spec.json" - # Spec fix: advanced auth configuration contain `client_id` and `client_secret` fields but they were missing in spec. - backward_compatibility_tests_config: - disable_for_version: "0.2.1" + - spec_path: "source_intercom/spec.json" + # Spec fix: advanced auth configuration contain `client_id` and `client_secret` fields but they were missing in spec. + backward_compatibility_tests_config: + disable_for_version: "0.2.1" connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: tests: - - config_path: "secrets/config.json" - # Schema fix: update schemas with undeclared fields which is not breaking change - backward_compatibility_tests_config: - disable_for_version: "0.2.1" + - config_path: "secrets/config.json" + # Schema fix: update schemas with undeclared fields which is not breaking change + backward_compatibility_tests_config: + disable_for_version: "0.2.1" basic_read: tests: - - config_path: "secrets/config.json" - expect_records: - path: "integration_tests/expected_records.jsonl" + - config_path: "secrets/config.json" + expect_records: + path: "integration_tests/expected_records.jsonl" incremental: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/incremental_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" - threshold_days: 365 - cursor_paths: - companies: ["updated_at"] - company_segments: ["updated_at"] - conversations: ["updated_at"] - conversation_parts: ["updated_at"] - contacts: ["updated_at"] - segments: ["updated_at"] + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/incremental_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-intercom/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-intercom/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-intercom/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-intercom/build.gradle b/airbyte-integrations/connectors/source-intercom/build.gradle deleted file mode 100644 index 3446296ba962..000000000000 --- a/airbyte-integrations/connectors/source-intercom/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_intercom' -} diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json index 2bd1cb003b2c..e874bc451c67 100755 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json @@ -73,5 +73,16 @@ "updated_at": 7626086649 } } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "activity_logs" + }, + "stream_state": { + "created_at": 7626086649 + } + } } ] diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json index 2e0e4e62a618..66ccdc871d86 100644 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json @@ -11,6 +11,20 @@ "primary_key": [["id"]], "destination_sync_mode": "append" }, + { + "stream": { + "name": "activity_logs", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "cursor_field": ["created_at"], + "primary_key": [["id"]], + "destination_sync_mode": "append" + }, { "stream": { "name": "companies", diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-intercom/integration_tests/expected_records.jsonl index 6823ccd4dbb4..cb29963e07f5 100644 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/expected_records.jsonl @@ -1,23 +1,31 @@ -{"stream": "admins", "data": {"type": "admin", "email": "integration-test@airbyte.io", "id": "4423433", "name": "Airbyte Team", "job_title": "Admin", "away_mode_enabled": false, "away_mode_reassign": false, "has_inbox_seat": true, "team_ids": [5077733], "team_priority_level": {"primary_team_ids": [5077733]}}, "emitted_at": 1680518975287} -{"stream": "admins", "data": {"type": "admin", "email": "operator+wjw5eps7@intercom.io", "id": "4423434", "name": "Operator", "away_mode_enabled": false, "away_mode_reassign": false, "has_inbox_seat": false, "team_ids": [], "team_priority_level": {}}, "emitted_at": 1680518975289} -{"stream": "admins", "data": {"type": "admin", "email": "jared@daxtarity.com", "id": "4425337", "name": "Jared Rhizor", "away_mode_enabled": false, "away_mode_reassign": false, "has_inbox_seat": true, "team_ids": [], "team_priority_level": {}}, "emitted_at": 1680518975291} -{"stream": "admins", "data": {"type": "admin", "email": "user2.sample.airbyte@gmail.com", "id": "6405371", "name": "user2", "away_mode_enabled": false, "away_mode_reassign": false, "has_inbox_seat": true, "team_ids": [6407165], "team_priority_level": {"primary_team_ids": [6407165]}}, "emitted_at": 1680518975292} -{"stream": "admins", "data": {"type": "admin", "email": "user1.sample@zohomail.eu", "id": "6405388", "name": "User1", "away_mode_enabled": false, "away_mode_reassign": false, "has_inbox_seat": true, "team_ids": [6407164, 6407165, 6407170, 6407175], "team_priority_level": {"primary_team_ids": [6407164, 6407165, 6407170, 6407175]}}, "emitted_at": 1680518975294} -{"stream": "admins", "data": {"type": "admin", "email": "user3.sample.airbyte@outlook.com", "id": "6407134", "name": "User3 Sample", "away_mode_enabled": false, "away_mode_reassign": false, "has_inbox_seat": true, "team_ids": [6407164, 6407165, 6407173], "team_priority_level": {"primary_team_ids": [6407164, 6407165, 6407173]}}, "emitted_at": 1680518975296} -{"stream": "admins", "data": {"type": "admin", "email": "user4.sample.airbyte@outlook.com", "id": "6407142", "name": "User4 Sample", "away_mode_enabled": false, "away_mode_reassign": false, "has_inbox_seat": true, "team_ids": [6407166, 6407170, 6407174], "team_priority_level": {"primary_team_ids": [6407166, 6407170, 6407174]}}, "emitted_at": 1680518975298} -{"stream": "admins", "data": {"type": "admin", "email": "user5.sample.airbyte@outlook.com", "id": "6407146", "name": "User5 Sample", "away_mode_enabled": false, "away_mode_reassign": false, "has_inbox_seat": true, "team_ids": [6407166], "team_priority_level": {"primary_team_ids": [6407166]}}, "emitted_at": 1680518975299} -{"stream": "admins", "data": {"type": "admin", "email": "user6.sample.airbyte@outlook.com", "id": "6407148", "name": "User6 Sample", "away_mode_enabled": false, "away_mode_reassign": false, "has_inbox_seat": true, "team_ids": [6407166, 6407167, 6407168, 6407173, 6407174], "team_priority_level": {"primary_team_ids": [6407166, 6407167, 6407168, 6407173, 6407174]}}, "emitted_at": 1680518975301} -{"stream": "admins", "data": {"type": "admin", "email": "user7.sample.airbyte@outlook.com", "id": "6407153", "name": "User7 Sample", "away_mode_enabled": false, "away_mode_reassign": false, "has_inbox_seat": true, "team_ids": [6407166, 6407167, 6407170], "team_priority_level": {"primary_team_ids": [6407166, 6407167, 6407170]}}, "emitted_at": 1680518975303} -{"stream": "admins", "data": {"type": "admin", "email": "user8.sample.airbyte@outlook.com", "id": "6407155", "name": "User8 Sample", "away_mode_enabled": false, "away_mode_reassign": false, "has_inbox_seat": true, "team_ids": [6407167, 6407174], "team_priority_level": {"primary_team_ids": [6407167, 6407174]}}, "emitted_at": 1680518975305} -{"stream": "admins", "data": {"type": "admin", "email": "user9.sample.airbyte@outlook.com", "id": "6407156", "name": "User9 Sample", "away_mode_enabled": false, "away_mode_reassign": false, "has_inbox_seat": true, "team_ids": [6407168, 6407170, 6407173], "team_priority_level": {"primary_team_ids": [6407168, 6407170, 6407173]}}, "emitted_at": 1680518975306} -{"stream": "admins", "data": {"type": "admin", "email": "user10.sample.airbyte@outlook.com", "id": "6407160", "name": "User10 Sample", "away_mode_enabled": false, "away_mode_reassign": false, "has_inbox_seat": true, "team_ids": [6407168, 6407175], "team_priority_level": {"primary_team_ids": [6407168, 6407175]}}, "emitted_at": 1680518975308} +{"stream":"admins","data":{"type":"admin","email":"integration-test@airbyte.io","id":"4423433","name":"Airbyte Team","job_title":"Admin","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366492} +{"stream":"admins","data":{"type":"admin","email":"operator+wjw5eps7@intercom.io","id":"4423434","name":"Operator","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366495} +{"stream":"admins","data":{"type":"admin","email":"jared@daxtarity.com","id":"4425337","name":"Jared Rhizor","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366498} +{"stream":"admins","data":{"type":"admin","email":"user2.sample.airbyte@gmail.com","id":"6405371","name":"user2","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366500} +{"stream":"admins","data":{"type":"admin","email":"user1.sample@zohomail.eu","id":"6405388","name":"User1","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366502} +{"stream":"admins","data":{"type":"admin","email":"user3.sample.airbyte@outlook.com","id":"6407134","name":"User3 Sample","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366504} +{"stream":"admins","data":{"type":"admin","email":"user4.sample.airbyte@outlook.com","id":"6407142","name":"User4 Sample","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366506} +{"stream":"admins","data":{"type":"admin","email":"user5.sample.airbyte@outlook.com","id":"6407146","name":"User5 Sample","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366508} +{"stream":"admins","data":{"type":"admin","email":"user6.sample.airbyte@outlook.com","id":"6407148","name":"User6 Sample","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366509} +{"stream":"admins","data":{"type":"admin","email":"user7.sample.airbyte@outlook.com","id":"6407153","name":"User7 Sample","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366511} +{"stream":"admins","data":{"type":"admin","email":"user8.sample.airbyte@outlook.com","id":"6407155","name":"User8 Sample","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366513} +{"stream":"admins","data":{"type":"admin","email":"user9.sample.airbyte@outlook.com","id":"6407156","name":"User9 Sample","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366514} +{"stream":"admins","data":{"type":"admin","email":"user10.sample.airbyte@outlook.com","id":"6407160","name":"User10 Sample","away_mode_enabled":false,"away_mode_reassign":false,"has_inbox_seat":false,"team_ids":[],"team_priority_level":{}},"emitted_at":1695811366516} +{"stream": "activity_logs", "data": {"id": "f7cf4eba-3a37-44b0-aecf-f347fe116712", "performed_by": {"type": "admin", "id": "4423433", "email": "integration-test@airbyte.io", "ip": "93.74.108.30"}, "metadata": {"admin": {"id": 4423433, "first_name": "John", "last_name": "Lafleur"}, "before": {"permissions": {"access_billing_settings": true, "access_developer_hub": true, "access_product_settings": true, "access_reporting": true, "access_workspace_settings": true, "create_and_edit_bots": true, "export_data": true, "manage_apps_and_integrations": true, "manage_articles": true, "manage_inbox_rules": true, "manage_inbox_views": true, "manage_messages_settings": true, "manage_messenger_settings": true, "manage_saved_replies": true, "manage_tags": true, "manage_teammates": true, "reassign_conversations": true, "redact_conversation_parts": true, "send_messages": true}, "conversation_access": {}}, "after": {"permissions": {"access_billing_settings": true, "access_developer_hub": true, "access_product_settings": true, "access_reporting": true, "access_workspace_settings": true, "create_and_edit_bots": true, "export_data": true, "manage_apps_and_integrations": true, "manage_articles": true, "manage_inbox_rules": true, "manage_inbox_views": true, "manage_messages_settings": true, "manage_messenger_settings": true, "manage_saved_replies": true, "manage_tags": true, "manage_teammates": true, "reassign_conversations": true, "redact_conversation_parts": true, "send_messages": true}, "conversation_access": {"access_type": "all", "assignee_blocked_list": null, "include_unassigned": false}}}, "created_at": 1625657753, "activity_type": "admin_permission_change", "activity_description": "Airbyte Team changed John Lafleur's permissions."}, "emitted_at": 1704967352753} +{"stream": "activity_logs", "data": {"id": "1fb8c7f2-bb57-49c9-bffc-7c49e0e54b40", "performed_by": {"type": "admin", "id": "4423433", "email": "integration-test@airbyte.io", "ip": "93.74.108.30"}, "metadata": {"team": {"id": 5077733, "name": "test", "member_count": 1}}, "created_at": 1625657582, "activity_type": "app_team_creation", "activity_description": "Airbyte Team created a new team, test, with 1 member."}, "emitted_at": 1704967352755} +{"stream": "activity_logs", "data": {"id": "5f569e46-45c3-4f76-93b7-9096bca00431", "performed_by": {"type": "admin", "id": "4423433", "email": "integration-test@airbyte.io", "ip": "93.74.108.30"}, "metadata": {"admin": {"id": 4425337, "first_name": "Jared", "last_name": "Rhizor"}, "before": {"permissions": {"access_billing_settings": false, "access_developer_hub": true, "access_product_settings": true, "access_reporting": true, "access_workspace_settings": true, "create_and_edit_bots": false, "export_data": true, "manage_apps_and_integrations": true, "manage_articles": false, "manage_inbox_rules": true, "manage_inbox_views": false, "manage_messages_settings": false, "manage_messenger_settings": false, "manage_saved_replies": false, "manage_tags": true, "manage_teammates": true, "reassign_conversations": false, "redact_conversation_parts": false, "send_messages": false}, "conversation_access": {}}, "after": {"permissions": {"access_billing_settings": false, "access_developer_hub": true, "access_product_settings": true, "access_reporting": true, "access_workspace_settings": true, "create_and_edit_bots": false, "export_data": true, "manage_apps_and_integrations": true, "manage_articles": false, "manage_inbox_rules": true, "manage_inbox_views": false, "manage_messages_settings": false, "manage_messenger_settings": false, "manage_saved_replies": false, "manage_tags": true, "manage_teammates": true, "reassign_conversations": false, "redact_conversation_parts": false, "send_messages": false}, "conversation_access": {"access_type": "all", "assignee_blocked_list": null, "include_unassigned": false}}}, "created_at": 1625657461, "activity_type": "admin_permission_change", "activity_description": "Airbyte Team changed Jared Rhizor's permissions."}, "emitted_at": 1704967352757} +{"stream": "activity_logs", "data": {"id": "766a7c71-5c41-415e-8984-ca8873b00b78", "performed_by": {"type": "admin", "id": "4423433", "email": "integration-test@airbyte.io", "ip": "unknown"}, "metadata": {"app_package": {"name": "Airbyte", "description": null}}, "created_at": 1634283727, "activity_type": "app_package_installation", "activity_description": "Airbyte Team installed the Airbyte app package for Airbyte [DEV]."}, "emitted_at": 1704967353629} +{"stream": "activity_logs", "data": {"id": "47e43c9a-509e-43f9-9867-16b8fb8f8f88", "performed_by": {"type": "admin", "id": "4423433", "email": "integration-test@airbyte.io", "ip": "unknown"}, "metadata": {"app_package": {"name": "Airbyte", "description": null}}, "created_at": 1634281951, "activity_type": "app_package_installation", "activity_description": "Airbyte Team installed the Airbyte app package for Airbyte [DEV]."}, "emitted_at": 1704967353635} +{"stream": "activity_logs", "data": {"id": "1a4baeb9-4cf7-4c3c-ac88-c8e8a4d2b8d7", "performed_by": {"type": "admin", "id": "4423433", "email": "integration-test@airbyte.io", "ip": "unknown"}, "metadata": {"app_package": {"name": "Airbyte Application", "description": null}}, "created_at": 1634235978, "activity_type": "app_package_installation", "activity_description": "Airbyte Team installed the Airbyte Application app package for Airbyte [DEV]."}, "emitted_at": 1704967353639} +{"stream": "activity_logs", "data": {"id": "2fb77f78-b37e-4db1-8357-b990ea1f9c33", "performed_by": {"type": "admin", "id": "4423433", "email": "integration-test@airbyte.io", "ip": "unknown"}, "metadata": {"app_package": {"name": "Airbyte", "description": null}}, "created_at": 1633429956, "activity_type": "app_package_installation", "activity_description": "Airbyte Team installed the Airbyte app package for Airbyte [DEV]."}, "emitted_at": 1704967353642} +{"stream": "activity_logs", "data": {"id": "db71bd41-665d-4589-876a-900857f462ec", "performed_by": {"type": "admin", "id": "4423433", "email": "integration-test@airbyte.io", "ip": "93.73.161.112"}, "metadata": {"app_package": {"name": "Airbyte Application", "description": null}}, "created_at": 1632482635, "activity_type": "app_package_installation", "activity_description": "Airbyte Team installed the Airbyte Application app package for Airbyte [DEV]."}, "emitted_at": 1704967353644} +{"stream": "activity_logs", "data": {"id": "d8fc8709-3ed7-46fc-ae43-96f7a04e8682", "performed_by": {"type": "admin", "id": "4423433", "email": "integration-test@airbyte.io", "ip": "93.73.161.112"}, "metadata": {"app_package": {"name": "Airbyte App", "description": null}}, "created_at": 1632482041, "activity_type": "app_package_installation", "activity_description": "Airbyte Team installed the Airbyte App app package for Airbyte [DEV]."}, "emitted_at": 1704967353646} +{"stream": "activity_logs", "data": {"id": "940efd94-6625-4033-8cd8-42eec9063b5e", "performed_by": {"type": "admin", "id": "4423433", "email": "integration-test@airbyte.io", "ip": "93.73.161.112"}, "metadata": {"app_package": {"name": "Airbyte", "description": null}}, "created_at": 1632481922, "activity_type": "app_package_installation", "activity_description": "Airbyte Team installed the Airbyte app package for Airbyte [DEV]."}, "emitted_at": 1704967353647} {"stream": "companies", "data": {"type": "company", "company_id": "63ecc5731d460cdc137c906d-qualification-company", "id": "63ecc5731d460cdc137c906c", "app_id": "wjw5eps7", "name": "Test Company 8", "created_at": 1676461427, "updated_at": 1679484652, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 49, "website": "www.company8.com", "industry": "Manufacturing", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867526} {"stream": "companies", "data": {"type": "company", "company_id": "63ecc52f00fc87e58e8fb1f2-qualification-company", "id": "63ecc52f00fc87e58e8fb1f1", "app_id": "wjw5eps7", "name": "Test Company 7", "created_at": 1676461359, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 23, "website": "www.company7.com", "industry": "Production", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867529} {"stream": "companies", "data": {"type": "company", "company_id": "63ecc46a811f1737ded479ef-qualification-company", "id": "63ecc46a811f1737ded479ee", "app_id": "wjw5eps7", "name": "Test Company 4", "created_at": 1676461162, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 150, "website": "www.company4.com", "industry": "Software", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867531} {"stream": "companies", "data": {"type": "company", "company_id": "63ecc5d32059cdacf4ac6171-qualification-company", "id": "63ecc5d32059cdacf4ac6170", "app_id": "wjw5eps7", "name": "Test Company 9", "created_at": 1676461523, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 75, "website": "www.company9.com", "industry": "Sales", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867536} {"stream": "companies", "data": {"type": "company", "company_id": "63ecc61266325d8ebd24ed11-qualification-company", "id": "63ecc61266325d8ebd24ed10", "app_id": "wjw5eps7", "name": "Test Company 10", "created_at": 1676461586, "updated_at": 1679484652, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 38, "website": "www.company10.com", "industry": "IT", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867538} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecbfccb064f24a4941d219-qualification-company", "id": "63ecbfccb064f24a4941d218", "app_id": "wjw5eps7", "name": "Test Company", "created_at": 1676459980, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 123, "website": "http://test.com", "industry": "IT", "tags": {"type": "tag.list", "tags": [{"type": "tag", "id": "7799571", "name": "Tag1"}, {"type": "tag", "id": "7799570", "name": "Tag2"}, {"type": "tag", "id": "7799640", "name": "Tag10"}]}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867533} -{"stream": "companies", "data": {"type": "company", "company_id": "63ecbfef66325dc8a0ac006f-qualification-company", "id": "63ecbfef66325dc8a0ac006e", "app_id": "wjw5eps7", "name": "Test Company 2", "created_at": 1676460015, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 123, "website": "http://test.com", "industry": "IT 123", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867540} {"stream": "companies", "data": {"type": "company", "company_id": "63ecc41866325d2e90b0d3c6-qualification-company", "id": "63ecc41866325d2e90b0d3c5", "app_id": "wjw5eps7", "name": "Test Company 3", "created_at": 1676461080, "updated_at": 1679484653, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 50, "website": "www.company3.com", "industry": "IT", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867542} {"stream": "companies", "data": {"type": "company", "company_id": "63ecc3d60e3c81baaad9f9ef-qualification-company", "id": "63ecc3d60e3c81baaad9f9ee", "app_id": "wjw5eps7", "name": "Company 1", "created_at": 1676461015, "updated_at": 1689068298, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 25, "website": "www.company1.com", "industry": "Sales", "tags": {"type": "tag.list", "tags": []}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867548} {"stream": "companies", "data": {"type": "company", "company_id": "63ecc4e99a2c64721f435a23-qualification-company", "id": "63ecc4e99a2c64721f435a22", "app_id": "wjw5eps7", "name": "Test Company 6", "created_at": 1676461289, "updated_at": 1679484652, "monthly_spend": 0, "session_count": 0, "user_count": 1, "size": 55, "website": "www.company6.com", "industry": "Sales", "tags": {"type": "tag.list", "tags": [{"type": "tag", "id": "7799570", "name": "Tag2"}, {"type": "tag", "id": "7799640", "name": "Tag10"}]}, "segments": {"type": "segment.list", "segments": []}, "plan": {}, "custom_attributes": {"creation_source": "api"}}, "emitted_at": 1689152867544} @@ -50,17 +58,15 @@ {"stream": "company_segments", "data": {"type": "segment", "id": "63ea1a43d9c86cceefd8796e", "name": "Revenue", "created_at": 1676286531, "updated_at": 1676462321, "person_type": "user"}, "emitted_at": 1680518982259} {"stream": "company_segments", "data": {"type": "segment", "id": "63ecc7f36d40e8184b5d47a6", "name": "Sales", "created_at": 1676462067, "updated_at": 1676462069, "person_type": "user"}, "emitted_at": 1680518982262} {"stream": "company_segments", "data": {"type": "segment", "id": "6241a4b8c8b709894fa54df1", "name": "Test_1", "created_at": 1648469176, "updated_at": 1676462341, "person_type": "user"}, "emitted_at": 1680518982266} -{"stream": "conversations", "data": {"type": "conversation", "id": "1", "created_at": 1607553243, "updated_at": 1626346673, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "701718739", "delivered_as": "customer_initiated", "subject": "", "body": "

      hey there

      ", "author": {"type": "lead", "id": "5fd150d50697b6d0bbc4a2c2", "name": null, "email": ""}, "attachments": [], "url": "http://localhost:63342/airbyte-python/airbyte-integrations/bases/base-java/build/tmp/expandedArchives/org.jacoco.agent-0.8.5.jar_6a2df60c47de373ea127d14406367999/about.html?_ijt=uosck1k6vmp2dnl4oqib2g3u9d", "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "5fd150d50697b6d0bbc4a2c2"}]}, "first_contact_reply": {"created_at": 1607553243, "type": "conversation", "url": "http://localhost:63342/airbyte-python/airbyte-integrations/bases/base-java/build/tmp/expandedArchives/org.jacoco.agent-0.8.5.jar_6a2df60c47de373ea127d14406367999/about.html?_ijt=uosck1k6vmp2dnl4oqib2g3u9d"}, "admin_assignee_id": null, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": 4317957, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": 4317954, "first_contact_reply_at": 1607553243, "first_assignment_at": null, "first_admin_reply_at": 1625654131, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": 1607553246, "last_admin_reply_at": 1625656000, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 7}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": null, "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153694977} -{"stream": "conversations", "data": {"type": "conversation", "id": "59", "created_at": 1676460979, "updated_at": 1689068230, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51952658", "delivered_as": "automated", "subject": "", "body": "

      Test 1

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea418c0931f79d99a197ff"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": false, "state": "closed", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 3}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 1", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153695018} -{"stream": "conversations", "data": {"type": "conversation", "id": "60", "created_at": 1676461133, "updated_at": 1676461134, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51952871", "delivered_as": "automated", "subject": "", "body": "

      Test 3

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a0eddb9b625fb712c9"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test3", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153694982} -{"stream": "conversations", "data": {"type": "conversation", "id": "61", "created_at": 1676461196, "updated_at": 1676461197, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51952963", "delivered_as": "automated", "subject": "", "body": "

      Test 4

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a1b0e17c53248c7956"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 4", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153694985} -{"stream": "conversations", "data": {"type": "conversation", "id": "63", "created_at": 1676461327, "updated_at": 1676461328, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953153", "delivered_as": "automated", "subject": "", "body": "

      Test 6

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a2b2d44e63848146e7"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 6", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153694989} -{"stream": "conversations", "data": {"type": "conversation", "id": "64", "created_at": 1676461395, "updated_at": 1676461396, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953262", "delivered_as": "automated", "subject": "", "body": "

      Test 7

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a2c340f850172f2905"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 7", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153694994} -{"stream": "conversations", "data": {"type": "conversation", "id": "65", "created_at": 1676461499, "updated_at": 1676461499, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953436", "delivered_as": "automated", "subject": "", "body": "

      Test Lead 1

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "5fd150d50697b6d0bbc4a2c2"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test Lead 1", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153694998} -{"stream": "conversations", "data": {"type": "conversation", "id": "66", "created_at": 1676461563, "updated_at": 1676461564, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953541", "delivered_as": "automated", "subject": "", "body": "

      Test 9

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a3b0e17c505e52044d"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 9", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153695003} -{"stream": "conversations", "data": {"type": "conversation", "id": "67", "created_at": 1676461636, "updated_at": 1676461637, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953649", "delivered_as": "automated", "subject": "", "body": "

      Test 10

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a7b0e17c5039fbb824"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 10", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153695007} -{"stream": "conversations", "data": {"type": "conversation", "id": "68", "created_at": 1676461800, "updated_at": 1676461800, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953852", "delivered_as": "automated", "subject": "", "body": "

      Test Lead 5001

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ecc6c2811f17873ed2d007"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test Lead 5001", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153695011} -{"stream": "conversations", "data": {"type": "conversation", "id": "69", "created_at": 1676462031, "updated_at": 1676462031, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51954139", "delivered_as": "automated", "subject": "", "body": "

      Test 11

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a80931f79b6998e89f"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 11", "custom_attributes": {}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1689153695015} +{"stream": "conversations", "data": {"type": "conversation", "id": "60", "created_at": 1676461133, "updated_at": 1676461134, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51952871", "delivered_as": "automated", "subject": "", "body": "

      Test 3

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a0eddb9b625fb712c9"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test3", "custom_attributes": {"Created by": 4423433}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1705349379561} +{"stream": "conversations", "data": {"type": "conversation", "id": "61", "created_at": 1676461196, "updated_at": 1676461197, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51952963", "delivered_as": "automated", "subject": "", "body": "

      Test 4

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a1b0e17c53248c7956"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 4", "custom_attributes": {"Created by": 4423433}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1705349379565} +{"stream": "conversations", "data": {"type": "conversation", "id": "63", "created_at": 1676461327, "updated_at": 1676461328, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953153", "delivered_as": "automated", "subject": "", "body": "

      Test 6

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a2b2d44e63848146e7"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 6", "custom_attributes": {"Created by": 4423433}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1705349379569} +{"stream": "conversations", "data": {"type": "conversation", "id": "64", "created_at": 1676461395, "updated_at": 1676461396, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953262", "delivered_as": "automated", "subject": "", "body": "

      Test 7

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a2c340f850172f2905"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 7", "custom_attributes": {"Created by": 4423433}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1705349379573} +{"stream": "conversations", "data": {"type": "conversation", "id": "65", "created_at": 1676461499, "updated_at": 1676461499, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953436", "delivered_as": "automated", "subject": "", "body": "

      Test Lead 1

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "5fd150d50697b6d0bbc4a2c2"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test Lead 1", "custom_attributes": {"Created by": 4423433}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1705349379577} +{"stream": "conversations", "data": {"type": "conversation", "id": "66", "created_at": 1676461563, "updated_at": 1676461564, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953541", "delivered_as": "automated", "subject": "", "body": "

      Test 9

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a3b0e17c505e52044d"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 9", "custom_attributes": {"Created by": 4423433}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1705349379581} +{"stream": "conversations", "data": {"type": "conversation", "id": "67", "created_at": 1676461636, "updated_at": 1676461637, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953649", "delivered_as": "automated", "subject": "", "body": "

      Test 10

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a7b0e17c5039fbb824"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 10", "custom_attributes": {"Created by": 4423433}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1705349379585} +{"stream": "conversations", "data": {"type": "conversation", "id": "68", "created_at": 1676461800, "updated_at": 1676461800, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51953852", "delivered_as": "automated", "subject": "", "body": "

      Test Lead 5001

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ecc6c2811f17873ed2d007"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test Lead 5001", "custom_attributes": {"Created by": 4423433}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1705349379589} +{"stream": "conversations", "data": {"type": "conversation", "id": "69", "created_at": 1676462031, "updated_at": 1676462031, "waiting_since": null, "snoozed_until": null, "source": {"type": "conversation", "id": "51954139", "delivered_as": "automated", "subject": "", "body": "

      Test 11

      ", "author": {"type": "admin", "id": "4423433", "name": "Airbyte Team", "email": "integration-test@airbyte.io"}, "attachments": [], "url": null, "redacted": false}, "contacts": {"type": "contact.list", "contacts": [{"type": "contact", "id": "63ea41a80931f79b6998e89f"}]}, "first_contact_reply": null, "admin_assignee_id": 4423433, "team_assignee_id": null, "open": true, "state": "open", "read": false, "tags": {"type": "tag.list", "tags": []}, "priority": "not_priority", "sla_applied": null, "statistics": {"type": "conversation_statistics", "time_to_assignment": null, "time_to_admin_reply": null, "time_to_first_close": null, "time_to_last_close": null, "median_time_to_reply": null, "first_contact_reply_at": null, "first_assignment_at": null, "first_admin_reply_at": null, "first_close_at": null, "last_assignment_at": null, "last_assignment_admin_reply_at": null, "last_contact_reply_at": null, "last_admin_reply_at": null, "last_close_at": null, "last_closed_by_id": null, "count_reopens": 0, "count_assignments": 0, "count_conversation_parts": 2}, "conversation_rating": null, "teammates": {"type": "admin.list", "admins": [{"type": "admin", "id": "4423433"}]}, "title": "Test 11", "custom_attributes": {"Created by": 4423433}, "topics": {"type": "topic.list", "topics": [], "total_count": 0}}, "emitted_at": 1705349379593} {"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "7288120839", "part_type": "comment", "body": "

      is this showing up

      ", "created_at": 1607553246, "updated_at": 1607553246, "notified_at": 1607553246, "assigned_to": null, "author": {"id": "5fd150d50697b6d0bbc4a2c2", "type": "user", "name": null, "email": ""}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "1"}, "emitted_at": 1688632241806} {"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "7288121348", "part_type": "comment", "body": "

      Airbyte [DEV] will reply as soon as they can.

      ", "created_at": 1607553249, "updated_at": 1607553249, "notified_at": 1607553249, "assigned_to": null, "author": {"id": "4423434", "type": "bot", "name": "Operator", "email": "operator+wjw5eps7@intercom.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "1"}, "emitted_at": 1688632241811} {"stream": "conversation_parts", "data": {"type": "conversation_part", "id": "7288121392", "part_type": "comment", "body": "

      Give the team a way to reach you:

      ", "created_at": 1607553250, "updated_at": 1607553250, "notified_at": 1607553250, "assigned_to": null, "author": {"id": "4423434", "type": "bot", "name": "Operator", "email": "operator+wjw5eps7@intercom.io"}, "attachments": [], "external_id": null, "redacted": false, "conversation_id": "1"}, "emitted_at": 1688632241815} @@ -115,13 +121,13 @@ {"stream": "tags", "data": {"type": "tag", "id": "7799637", "name": "Tag7"}, "emitted_at": 1680518991642} {"stream": "tags", "data": {"type": "tag", "id": "7799636", "name": "Tag8"}, "emitted_at": 1680518991643} {"stream": "tags", "data": {"type": "tag", "id": "7799641", "name": "Tag9"}, "emitted_at": 1680518991644} -{"stream": "teams", "data": {"type": "team", "id": "5077733", "name": "test", "admin_ids": [4423433]}, "emitted_at": 1680518992219} -{"stream": "teams", "data": {"type": "team", "id": "6407164", "name": "Test team 2", "admin_ids": [6405388, 6407134]}, "emitted_at": 1680518992220} -{"stream": "teams", "data": {"type": "team", "id": "6407165", "name": "Test team 3", "admin_ids": [6405388, 6405371, 6407134]}, "emitted_at": 1680518992222} -{"stream": "teams", "data": {"type": "team", "id": "6407166", "name": "Test team 4", "admin_ids": [6407142, 6407146, 6407148, 6407153]}, "emitted_at": 1680518992223} -{"stream": "teams", "data": {"type": "team", "id": "6407167", "name": "Test team 5", "admin_ids": [6407148, 6407153, 6407155]}, "emitted_at": 1680518992224} -{"stream": "teams", "data": {"type": "team", "id": "6407168", "name": "Test team 6", "admin_ids": [6407148, 6407156, 6407160]}, "emitted_at": 1680518992226} -{"stream": "teams", "data": {"type": "team", "id": "6407170", "name": "Test team 7", "admin_ids": [6405388, 6407142, 6407153, 6407156]}, "emitted_at": 1680518992227} -{"stream": "teams", "data": {"type": "team", "id": "6407173", "name": "Test team 8", "admin_ids": [6407134, 6407148, 6407156]}, "emitted_at": 1680518992228} -{"stream": "teams", "data": {"type": "team", "id": "6407174", "name": "Test team 9", "admin_ids": [6407142, 6407148, 6407155]}, "emitted_at": 1680518992230} -{"stream": "teams", "data": {"type": "team", "id": "6407175", "name": "Test team 10", "admin_ids": [6405388, 6407160]}, "emitted_at": 1680518992231} +{"stream":"teams","data":{"type":"team","id":"5077733","name":"test","admin_ids":[]},"emitted_at":1695812351359} +{"stream":"teams","data":{"type":"team","id":"6407164","name":"Test team 2","admin_ids":[]},"emitted_at":1695812351360} +{"stream":"teams","data":{"type":"team","id":"6407165","name":"Test team 3","admin_ids":[]},"emitted_at":1695812351361} +{"stream":"teams","data":{"type":"team","id":"6407166","name":"Test team 4","admin_ids":[]},"emitted_at":1695812351362} +{"stream":"teams","data":{"type":"team","id":"6407167","name":"Test team 5","admin_ids":[]},"emitted_at":1695812351363} +{"stream":"teams","data":{"type":"team","id":"6407168","name":"Test team 6","admin_ids":[]},"emitted_at":1695812351364} +{"stream":"teams","data":{"type":"team","id":"6407170","name":"Test team 7","admin_ids":[]},"emitted_at":1695812351365} +{"stream":"teams","data":{"type":"team","id":"6407173","name":"Test team 8","admin_ids":[]},"emitted_at":1695812351366} +{"stream":"teams","data":{"type":"team","id":"6407174","name":"Test team 9","admin_ids":[]},"emitted_at":1695812351367} +{"stream":"teams","data":{"type":"team","id":"6407175","name":"Test team 10","admin_ids":[]},"emitted_at":1695812351368} diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/incremental_catalog.json b/airbyte-integrations/connectors/source-intercom/integration_tests/incremental_catalog.json index 2c4a3735e86d..04647c9bf1a7 100644 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/incremental_catalog.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/incremental_catalog.json @@ -1,5 +1,19 @@ { "streams": [ + { + "stream": { + "name": "activity_logs", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "cursor_field": ["created_at"], + "primary_key": [["id"]], + "destination_sync_mode": "append" + }, { "stream": { "name": "companies", diff --git a/airbyte-integrations/connectors/source-intercom/metadata.yaml b/airbyte-integrations/connectors/source-intercom/metadata.yaml index c9feadaca378..beb243445d31 100644 --- a/airbyte-integrations/connectors/source-intercom/metadata.yaml +++ b/airbyte-integrations/connectors/source-intercom/metadata.yaml @@ -1,12 +1,18 @@ data: + ab_internal: + ql: 400 + sl: 300 allowedHosts: hosts: - api.intercom.io + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: d8313939-3782-41b0-be29-b3ca20d8dd3a - dockerImageTag: 0.3.0 + dockerImageTag: 0.4.0 dockerRepository: airbyte/source-intercom + documentationUrl: https://docs.airbyte.com/integrations/sources/intercom githubIssueLabel: source-intercom icon: intercom.svg license: MIT @@ -17,12 +23,15 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/intercom + suggestedStreams: + streams: + - conversations + - contacts + - conversation_parts + - teams + - companies + supportLevel: certified tags: - language:low-code - language:python - ab_internal: - sl: 300 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-intercom/setup.py b/airbyte-integrations/connectors/source-intercom/setup.py index f5fce35eb718..0432b7d7f10a 100644 --- a/airbyte-integrations/connectors/source-intercom/setup.py +++ b/airbyte-integrations/connectors/source-intercom/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", + "airbyte-cdk>=0.58.8", # previous versions had a bug with http_method value from the manifest ] TEST_REQUIREMENTS = [ diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/components.py b/airbyte-integrations/connectors/source-intercom/source_intercom/components.py index 6e87c9b9e8a1..600ba64945b1 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/components.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/components.py @@ -167,7 +167,6 @@ def read_parent_stream( ) for parent_slice in parent_stream_slices_gen: - parent_records_gen = self.parent_stream.read_records( sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=parent_slice, stream_state=stream_state ) diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/manifest.yaml b/airbyte-integrations/connectors/source-intercom/source_intercom/manifest.yaml index a87842f5ef04..4dd78b43ea9d 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/manifest.yaml +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/manifest.yaml @@ -234,6 +234,14 @@ definitions: $ref: "#/definitions/substream_semi_incremental/retriever/record_selector" extractor: field_path: ["conversation_parts", "conversation_parts"] + requester: + $ref: "#/definitions/requester" + error_handler: + type: DefaultErrorHandler + description: "404 - conversation is not found while requesting, ignore" + response_filters: + - http_codes: [404] + action: IGNORE company_segments: $ref: "#/definitions/substream_semi_incremental" $parameters: @@ -289,7 +297,46 @@ definitions: data_field: "conversations" page_size: 150 + # activity logs stream is incremental based on created_at field + activity_logs: + $ref: "#/definitions/stream_full_refresh" + primary_key: id + $parameters: + name: "activity_logs" + path: "admins/activity_logs" + data_field: "activity_logs" + retriever: + $ref: "#/definitions/retriever" + description: "The Retriever without passing page size option" + paginator: + type: "DefaultPaginator" + url_base: "#/definitions/requester/url_base" + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response.get('pages', {}).get('next') }}" + stop_condition: "{{ 'next' not in response.get('pages', {}) }}" + page_token_option: + type: RequestPath + incremental_sync: + type: DatetimeBasedCursor + cursor_field: created_at + cursor_datetime_formats: + - "%s" + datetime_format: "%s" + cursor_granularity: "PT1S" + step: "P30D" + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_time_option: + field_name: "created_at_before" + inject_into: "request_parameter" + start_time_option: + field_name: "created_at_after" + inject_into: "request_parameter" + streams: + - "#/definitions/activity_logs" - "#/definitions/admins" - "#/definitions/tags" - "#/definitions/teams" diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/activity_logs.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/activity_logs.json new file mode 100644 index 000000000000..3136288524e5 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/activity_logs.json @@ -0,0 +1,37 @@ +{ + "type": "object", + "properties": { + "performed_by": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "ip": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + } + } + }, + "id": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"] + }, + "activity_type": { + "type": ["null", "string"] + }, + "activity_description": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-intercom/unit_tests/test_components.py b/airbyte-integrations/connectors/source-intercom/unit_tests/test_components.py index 455793c8e7de..2da6517110ec 100644 --- a/airbyte-integrations/connectors/source-intercom/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-intercom/unit_tests/test_components.py @@ -2,12 +2,52 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from unittest.mock import MagicMock, Mock +from unittest.mock import MagicMock, Mock, patch import pytest +import requests +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig from airbyte_cdk.sources.streams import Stream -from source_intercom.components import IncrementalSingleSliceCursor, IncrementalSubstreamSlicerCursor +from source_intercom.components import ( + HttpRequesterWithRateLimiter, + IncrementalSingleSliceCursor, + IncrementalSubstreamSlicerCursor, + IntercomRateLimiter, +) + + +def get_requester(): + request_options_provider = MagicMock() + request_params = {"param": "value"} + request_body_data = "body_key_1=value_1&body_key_2=value2" + request_body_json = {"body_field": "body_value"} + request_options_provider.get_request_params.return_value = request_params + request_options_provider.get_request_body_data.return_value = request_body_data + request_options_provider.get_request_body_json.return_value = request_body_json + + error_handler = MagicMock() + max_retries = 10 + backoff_time = 1000 + response_status = MagicMock() + response_status.retry_in.return_value = 10 + error_handler.max_retries = max_retries + error_handler.interpret_response.return_value = response_status + error_handler.backoff_time.return_value = backoff_time + + config = {"url": "https://airbyte.io"} + + return HttpRequesterWithRateLimiter( + name="stream_name", + url_base=InterpolatedString.create("{{ config['url'] }}", parameters={}), + path=InterpolatedString.create("v1/{{ stream_slice['id'] }}", parameters={}), + http_method="GET", + request_options_provider=request_options_provider, + authenticator=MagicMock(), + error_handler=error_handler, + config=config, + parameters={}, + ) def test_slicer(): @@ -25,7 +65,11 @@ def test_slicer(): [ ( {"first_stream_cursor": 1662459010}, - {'first_stream_cursor': 1662459010, 'prior_state': {'first_stream_cursor': 1662459010, 'parent_stream_name': {'parent_cursor_field': 1662459010}}, 'parent_stream_name': {'parent_cursor_field': 1662459010}}, + { + "first_stream_cursor": 1662459010, + "prior_state": {"first_stream_cursor": 1662459010, "parent_stream_name": {"parent_cursor_field": 1662459010}}, + "parent_stream_name": {"parent_cursor_field": 1662459010}, + }, [{"first_stream_cursor": 1662459010}], ) ], @@ -53,3 +97,52 @@ def test_sub_slicer(last_record, expected, records): stream_slice = next(slicer.stream_slices()) if records else {} slicer.close_slice(stream_slice, last_record) assert slicer.get_stream_state() == expected + + +@pytest.mark.parametrize( + "rate_limit_header, backoff_time", + [ + ({"X-RateLimit-Limit": 167, "X-RateLimit-Remaining": 167}, 0.01), + ({"X-RateLimit-Limit": 167, "X-RateLimit-Remaining": 100}, 0.01), + ({"X-RateLimit-Limit": 167, "X-RateLimit-Remaining": 83}, 1.5), + ({"X-RateLimit-Limit": 167, "X-RateLimit-Remaining": 16}, 8.0), + ({}, 1.0), + ], +) +def test_rate_limiter(rate_limit_header, backoff_time): + def check_backoff_time(t): + """A replacer for original `IntercomRateLimiter.backoff_time`""" + assert backoff_time == t, f"Expected {backoff_time}, got {t}" + + class Requester: + @IntercomRateLimiter.balance_rate_limit() + def interpret_response_status(self, response: requests.Response): + """A stub for the decorator function being tested""" + + with patch.object(IntercomRateLimiter, "backoff_time") as backoff_time_mock: + # Call `check_backoff_time` instead of original `IntercomRateLimiter.backoff_time` method + backoff_time_mock.side_effect = check_backoff_time + + requester = Requester() + + # Prepare requester object with headers + response = requests.models.Response() + response.headers = rate_limit_header + + # Call a decorated method + requester.interpret_response_status(response) + + +def test_requester_get_request_params(): + requester = get_requester() + assert {} == requester.get_request_params() + + +def test_requester_get_request_body_json(): + requester = get_requester() + assert {} == requester.get_request_body_json() + + +def test_requester_get_request_headers(): + requester = get_requester() + assert {} == requester.get_request_headers() diff --git a/airbyte-integrations/connectors/source-intercom/unit_tests/test_source.py b/airbyte-integrations/connectors/source-intercom/unit_tests/test_source.py new file mode 100644 index 000000000000..fe7a765a2ee0 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/unit_tests/test_source.py @@ -0,0 +1,9 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from source_intercom import SourceIntercom + + +def test_source(): + assert SourceIntercom() diff --git a/airbyte-integrations/connectors/source-intruder/README.md b/airbyte-integrations/connectors/source-intruder/README.md index 46a577610c6e..94b8b878d4ea 100644 --- a/airbyte-integrations/connectors/source-intruder/README.md +++ b/airbyte-integrations/connectors/source-intruder/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-intruder:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/intruder) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_intruder/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-intruder:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-intruder build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-intruder:airbyteDocker +An image will be built with the tag `airbyte/source-intruder:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-intruder:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-intruder:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-intruder:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-intruder:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-intruder test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-intruder:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-intruder:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-intruder test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/intruder.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-intruder/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-intruder/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-intruder/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-intruder/build.gradle b/airbyte-integrations/connectors/source-intruder/build.gradle deleted file mode 100644 index d346ef5c1805..000000000000 --- a/airbyte-integrations/connectors/source-intruder/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_intruder' -} diff --git a/airbyte-integrations/connectors/source-ip2whois/README.md b/airbyte-integrations/connectors/source-ip2whois/README.md index db558c7826f9..22bed4f38b3f 100644 --- a/airbyte-integrations/connectors/source-ip2whois/README.md +++ b/airbyte-integrations/connectors/source-ip2whois/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-ip2whois:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/ip2whois) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_ip2whois/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-ip2whois:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-ip2whois build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-ip2whois:airbyteDocker +An image will be built with the tag `airbyte/source-ip2whois:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-ip2whois:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-ip2whois:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-ip2whois:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-ip2whois:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-ip2whois test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-ip2whois:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-ip2whois:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-ip2whois test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/ip2whois.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-ip2whois/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-ip2whois/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-ip2whois/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-ip2whois/build.gradle b/airbyte-integrations/connectors/source-ip2whois/build.gradle deleted file mode 100644 index ba74c959a17a..000000000000 --- a/airbyte-integrations/connectors/source-ip2whois/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_ip2whois' -} diff --git a/airbyte-integrations/connectors/source-iterable/Dockerfile b/airbyte-integrations/connectors/source-iterable/Dockerfile deleted file mode 100644 index d05453734efe..000000000000 --- a/airbyte-integrations/connectors/source-iterable/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY source_iterable ./source_iterable -COPY main.py ./ -COPY setup.py ./ -RUN pip install . - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.30 -LABEL io.airbyte.name=airbyte/source-iterable diff --git a/airbyte-integrations/connectors/source-iterable/README.md b/airbyte-integrations/connectors/source-iterable/README.md index e06072c307f0..b7b19c7da8be 100644 --- a/airbyte-integrations/connectors/source-iterable/README.md +++ b/airbyte-integrations/connectors/source-iterable/README.md @@ -1,101 +1,117 @@ -# Iterable Source +# Coinmarketcap Source -This is the repository for the Iterable source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/iterable). +This is the repository for the Coinmarketcap configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/coinmarketcap). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/coinmarketcap) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_coinmarketcap/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. -#### Minimum Python version required `= 3.7.0` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source coinmarketcap test creds` +and place them into `secrets/config.json`. -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` +### Locally running the connector docker image -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-iterable:build + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name source-iterable build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-iterable:dev`. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/iterable) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_iterable/spec.json` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source iterable test creds` -and place them into `secrets/config.json`. +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations +from typing import TYPE_CHECKING -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Locally running the connector docker image +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-iterable:dev +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-iterable:airbyteDocker +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-iterable:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-iterable:dev . +# Running the spec command against your patched connector +docker run airbyte/source-iterable:dev spec +``` #### Run Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-iterable:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-iterable:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-iterable:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-iterable:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm airbyte/source-coinmarketcap:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-coinmarketcap:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-coinmarketcap:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-coinmarketcap:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-iterable test ``` -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-iterable:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-iterable test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/iterable.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-iterable/acceptance-test-config.yml b/airbyte-integrations/connectors/source-iterable/acceptance-test-config.yml index 034e9eb109ae..e6e5ed3decab 100644 --- a/airbyte-integrations/connectors/source-iterable/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-iterable/acceptance-test-config.yml @@ -5,109 +5,111 @@ test_strictness_level: "high" acceptance_tests: spec: tests: - - spec_path: "source_iterable/spec.json" + - spec_path: "source_iterable/spec.json" connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: tests: - - config_path: "secrets/config.json" + - config_path: "secrets/config.json" basic_read: tests: - - config_path: "secrets/config.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: no - extra_records: yes - empty_streams: - - name: "web_push_click" - bypass_reason: "Can not populate; need messaging service configured" - - name: "sms_bounce" - bypass_reason: "Can not populate; need messaging service configured" - - name: "hosted_unsubscribe_click" - bypass_reason: "Can not populate; need messaging service configured" - - name: "in_app_click" - bypass_reason: "Can not populate; need messaging service configured" - - name: "inbox_session" - bypass_reason: "Can not populate; need messaging service configured" - - name: "purchase" - bypass_reason: "Can not populate; need messaging service configured" - - name: "in_app_delivery" - bypass_reason: "Can not populate; need messaging service configured" - - name: "in_app_send_skip" - bypass_reason: "Can not populate; need messaging service configured" - - name: "push_open" - bypass_reason: "Can not populate; need messaging service configured" - - name: "in_app_open" - bypass_reason: "Can not populate; need messaging service configured" - - name: "sms_click" - bypass_reason: "Can not populate; need messaging service configured" - - name: "push_bounce" - bypass_reason: "Can not populate; need messaging service configured" - - name: "sms_received" - bypass_reason: "Can not populate; need messaging service configured" - - name: "in_app_delete" - bypass_reason: "Can not populate; need messaging service configured" - - name: "email_complaint" - bypass_reason: "Can not populate; need messaging service configured" - - name: "push_send_skip" - bypass_reason: "Can not populate; need messaging service configured" - - name: "email_send_skip" - bypass_reason: "Can not populate; need messaging service configured" - - name: "sms_send" - bypass_reason: "Can not populate; need messaging service configured" - - name: "inbox_message_impression" - bypass_reason: "Can not populate; need messaging service configured" - - name: "in_app_close" - bypass_reason: "Can not populate; need messaging service configured" - - name: "sms_send_skip" - bypass_reason: "Can not populate; need messaging service configured" - - name: "custom_event" - bypass_reason: "Can not populate; need messaging service configured" - - name: "web_push_send" - bypass_reason: "Can not populate; need messaging service configured" - - name: "sms_usage_info" - bypass_reason: "Can not populate; need messaging service configured" - - name: "push_uninstall" - bypass_reason: "Can not populate; need messaging service configured" - - name: "push_send" - bypass_reason: "Can not populate; need messaging service configured" - - name: "in_app_send" - bypass_reason: "Can not populate; need messaging service configured" - - name: "web_push_send_skip" - bypass_reason: "Can not populate; need messaging service configured" - - name: "email_open" - bypass_reason: "Can not populate; need messaging service configured" - - name: "email_bounce" - bypass_reason: "Can not populate; need messaging service configured" - - name: "email_click" - bypass_reason: "Can not populate; need messaging service configured" - - name: "email_send" - bypass_reason: "Can not populate; need messaging service configured" - - name: "email_subscribe" - bypass_reason: "Can not populate; need messaging service configured" - - name: "email_unsubscribe" - bypass_reason: "Can not populate; need messaging service configured" - timeout_seconds: 3600 - fail_on_extra_columns: false + - config_path: "secrets/config.json" + expect_records: + path: "integration_tests/expected_records.jsonl" + extra_fields: no + exact_order: no + extra_records: yes + empty_streams: + - name: "web_push_click" + bypass_reason: "Can not populate; need messaging service configured" + - name: "sms_bounce" + bypass_reason: "Can not populate; need messaging service configured" + - name: "hosted_unsubscribe_click" + bypass_reason: "Can not populate; need messaging service configured" + - name: "in_app_click" + bypass_reason: "Can not populate; need messaging service configured" + - name: "inbox_session" + bypass_reason: "Can not populate; need messaging service configured" + - name: "purchase" + bypass_reason: "Can not populate; need messaging service configured" + - name: "in_app_delivery" + bypass_reason: "Can not populate; need messaging service configured" + - name: "in_app_send_skip" + bypass_reason: "Can not populate; need messaging service configured" + - name: "push_open" + bypass_reason: "Can not populate; need messaging service configured" + - name: "in_app_open" + bypass_reason: "Can not populate; need messaging service configured" + - name: "sms_click" + bypass_reason: "Can not populate; need messaging service configured" + - name: "push_bounce" + bypass_reason: "Can not populate; need messaging service configured" + - name: "sms_received" + bypass_reason: "Can not populate; need messaging service configured" + - name: "in_app_delete" + bypass_reason: "Can not populate; need messaging service configured" + - name: "email_complaint" + bypass_reason: "Can not populate; need messaging service configured" + - name: "push_send_skip" + bypass_reason: "Can not populate; need messaging service configured" + - name: "email_send_skip" + bypass_reason: "Can not populate; need messaging service configured" + - name: "sms_send" + bypass_reason: "Can not populate; need messaging service configured" + - name: "inbox_message_impression" + bypass_reason: "Can not populate; need messaging service configured" + - name: "in_app_close" + bypass_reason: "Can not populate; need messaging service configured" + - name: "sms_send_skip" + bypass_reason: "Can not populate; need messaging service configured" + - name: "custom_event" + bypass_reason: "Can not populate; need messaging service configured" + - name: "web_push_send" + bypass_reason: "Can not populate; need messaging service configured" + - name: "sms_usage_info" + bypass_reason: "Can not populate; need messaging service configured" + - name: "push_uninstall" + bypass_reason: "Can not populate; need messaging service configured" + - name: "push_send" + bypass_reason: "Can not populate; need messaging service configured" + - name: "in_app_send" + bypass_reason: "Can not populate; need messaging service configured" + - name: "web_push_send_skip" + bypass_reason: "Can not populate; need messaging service configured" + - name: "email_open" + bypass_reason: "Can not populate; need messaging service configured" + - name: "email_bounce" + bypass_reason: "Can not populate; need messaging service configured" + - name: "email_click" + bypass_reason: "Can not populate; need messaging service configured" + - name: "email_send" + bypass_reason: "Can not populate; need messaging service configured" + - name: "email_subscribe" + bypass_reason: "Can not populate; need messaging service configured" + - name: "email_unsubscribe" + bypass_reason: "Can not populate; need messaging service configured" + timeout_seconds: 3600 + fail_on_extra_columns: false full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/catalog.json" - timeout_seconds: 3600 + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/catalog.json" + timeout_seconds: 3600 incremental: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" - missing_streams: - - name: "email_complaint" - bypass_reason: "Can not populate; need messaging service configured" - - name: "email_send_skip" - bypass_reason: "Can not populate; need messaging service configured" - timeout_seconds: 3600 + - config_path: "secrets/config.json" + # Temporarily skipping icnremental tests as email_complaint is failing despite being included in missing_streams + skip_comprehensive_incremental_tests: true + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + missing_streams: + - name: "email_complaint" + bypass_reason: "Can not populate; need messaging service configured" + - name: "email_send_skip" + bypass_reason: "Can not populate; need messaging service configured" + timeout_seconds: 3600 diff --git a/airbyte-integrations/connectors/source-iterable/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-iterable/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-iterable/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-iterable/build.gradle b/airbyte-integrations/connectors/source-iterable/build.gradle deleted file mode 100644 index 0bff51ae3e63..000000000000 --- a/airbyte-integrations/connectors/source-iterable/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_iterable' -} diff --git a/airbyte-integrations/connectors/source-iterable/main.py b/airbyte-integrations/connectors/source-iterable/main.py index 3a4a2f7982ff..eef7d894cbc4 100644 --- a/airbyte-integrations/connectors/source-iterable/main.py +++ b/airbyte-integrations/connectors/source-iterable/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_iterable import SourceIterable +from source_iterable.run import run if __name__ == "__main__": - source = SourceIterable() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-iterable/metadata.yaml b/airbyte-integrations/connectors/source-iterable/metadata.yaml index 3df40f4c8d7b..94c0cd2c6846 100644 --- a/airbyte-integrations/connectors/source-iterable/metadata.yaml +++ b/airbyte-integrations/connectors/source-iterable/metadata.yaml @@ -1,12 +1,18 @@ data: + ab_internal: + ql: 200 + sl: 200 allowedHosts: hosts: - api.iterable.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 2e875208-0c0b-4ee4-9e92-1cb3156ea799 - dockerImageTag: 0.1.30 + dockerImageTag: 0.2.1 dockerRepository: airbyte/source-iterable + documentationUrl: https://docs.airbyte.com/integrations/sources/iterable githubIssueLabel: source-iterable icon: iterable.svg license: MIT @@ -17,11 +23,7 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/iterable + supportLevel: certified tags: - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-iterable/setup.py b/airbyte-integrations/connectors/source-iterable/setup.py index 5d2e499d31c7..fd2061fb89fb 100644 --- a/airbyte-integrations/connectors/source-iterable/setup.py +++ b/airbyte-integrations/connectors/source-iterable/setup.py @@ -16,6 +16,11 @@ setup( + entry_points={ + "console_scripts": [ + "source-iterable=source_iterable.run:run", + ], + }, name="source_iterable", description="Source implementation for Iterable.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/run.py b/airbyte-integrations/connectors/source-iterable/source_iterable/run.py new file mode 100644 index 000000000000..c2e01ead95e5 --- /dev/null +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_iterable import SourceIterable + + +def run(): + source = SourceIterable() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_bounce.json b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_bounce.json index fd74f6a40f9f..14cc02a90c99 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_bounce.json +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_bounce.json @@ -29,6 +29,9 @@ "email": { "type": ["null", "string"] }, + "userId": { + "type": ["null", "string"] + }, "recipientState": { "type": ["null", "string"] } diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_click.json b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_click.json index 5a0fecaf3447..f8439312858c 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_click.json +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_click.json @@ -55,6 +55,9 @@ }, "email": { "type": ["null", "string"] + }, + "userId": { + "type": ["null", "string"] } }, "type": ["null", "object"] diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_complaint.json b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_complaint.json index fd74f6a40f9f..14cc02a90c99 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_complaint.json +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_complaint.json @@ -29,6 +29,9 @@ "email": { "type": ["null", "string"] }, + "userId": { + "type": ["null", "string"] + }, "recipientState": { "type": ["null", "string"] } diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_open.json b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_open.json index 2e085dceeff0..36064e7ab3c9 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_open.json +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_open.json @@ -46,6 +46,9 @@ }, "email": { "type": ["null", "string"] + }, + "userId": { + "type": ["null", "string"] } }, "type": ["null", "object"] diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_send.json b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_send.json index e2614d971b18..1f328b78436b 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_send.json +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_send.json @@ -122,6 +122,9 @@ "email": { "type": ["null", "string"] }, + "userId": { + "type": ["null", "string"] + }, "channelId": { "type": ["null", "integer"] } diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_send_skip.json b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_send_skip.json index 374a9671f998..a96ce2d53e7a 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_send_skip.json +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_send_skip.json @@ -122,6 +122,9 @@ "email": { "type": ["null", "string"] }, + "userId": { + "type": ["null", "string"] + }, "channelId": { "type": ["null", "integer"] } diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_subscribe.json b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_subscribe.json index 3ac82b5cecba..8839d6d76ea2 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_subscribe.json +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_subscribe.json @@ -30,6 +30,9 @@ "email": { "type": ["null", "string"] }, + "userId": { + "type": ["null", "string"] + }, "profileUpdatedAt": { "type": ["null", "string"], "format": "date-time" diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_unsubscribe.json b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_unsubscribe.json index 03b00577f7ba..c69cfa5bcb31 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_unsubscribe.json +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/email_unsubscribe.json @@ -46,6 +46,9 @@ "email": { "type": ["null", "string"] }, + "userId": { + "type": ["null", "string"] + }, "channelId": { "type": ["null", "integer"] } diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/events.json b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/events.json index 028d32c78854..3c88b02b1ab9 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/events.json +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/schemas/events.json @@ -23,6 +23,9 @@ "email": { "type": ["null", "string"] }, + "userId": { + "type": ["null", "string"] + }, "data": { "type": ["null", "object"] } diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/test_exports_stream.py b/airbyte-integrations/connectors/source-iterable/unit_tests/test_exports_stream.py index 3294e110df46..2592b27d5fe0 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/test_exports_stream.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/test_exports_stream.py @@ -23,17 +23,6 @@ def session_mock(): response_mock.status_code = 200 yield session_mock - -def test_send_email_stream(session_mock): - stream = Users(start_date="2020", authenticator=None) - stream_slice = StreamSlice(start_date=pendulum.parse("2020"), end_date=pendulum.parse("2021")) - _ = list(stream.read_records(sync_mode=SyncMode.full_refresh, cursor_field=None, stream_slice=stream_slice, stream_state={})) - - assert session_mock.send.called - send_args = session_mock.send.call_args[1] - assert send_args.get("stream") is True - - @responses.activate def test_stream_correct(): stream_slice = StreamSlice(start_date=pendulum.parse("2020"), end_date=pendulum.parse("2021")) diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py index cbf7c611059b..68634a9a7ac2 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py @@ -207,12 +207,12 @@ def test_listuser_stream_keep_working_on_500(): responses.get("https://api.iterable.com/api/lists/getUsers?listId=3000", body="one@d2.com\ntwo@d2.com\nthree@d2.com") expected_records = [ - {'email': 'one@d1.com', 'listId': 2000}, - {'email': 'two@d1.com', 'listId': 2000}, - {'email': 'three@d1.com', 'listId': 2000}, - {'email': 'one@d2.com', 'listId': 3000}, - {'email': 'two@d2.com', 'listId': 3000}, - {'email': 'three@d2.com', 'listId': 3000}, + {"email": "one@d1.com", "listId": 2000}, + {"email": "two@d1.com", "listId": 2000}, + {"email": "three@d1.com", "listId": 2000}, + {"email": "one@d2.com", "listId": 3000}, + {"email": "two@d2.com", "listId": 3000}, + {"email": "three@d2.com", "listId": 3000}, ] records = list(read_full_refresh(users_stream)) @@ -223,7 +223,7 @@ def test_listuser_stream_keep_working_on_500(): def test_events_read_full_refresh(): stream = Events(authenticator=None) responses.get("https://api.iterable.com/api/lists", json={"lists": [{"id": 1}]}) - responses.get("https://api.iterable.com/api/lists/getUsers?listId=1", body='user1\nuser2\nuser3\nuser4\nuser5\nuser6') + responses.get("https://api.iterable.com/api/lists/getUsers?listId=1", body="user1\nuser2\nuser3\nuser4\nuser5\nuser6") def get_body(emails): return "\n".join([json.dumps({"email": email}) for email in emails]) + "\n" @@ -245,10 +245,12 @@ def get_body(emails): responses.get("https://api.iterable.com/api/export/userEvents?email=user5&includeCustomEvents=true", json=generic_error2, status=500) responses.get("https://api.iterable.com/api/export/userEvents?email=user5&includeCustomEvents=true", body=get_body(["user5"])) - m = responses.get("https://api.iterable.com/api/export/userEvents?email=user6&includeCustomEvents=true", json=generic_error2, status=500) + m = responses.get( + "https://api.iterable.com/api/export/userEvents?email=user6&includeCustomEvents=true", json=generic_error2, status=500 + ) records = list(read_full_refresh(stream)) - assert [r["email"] for r in records] == ['user1', 'user2', 'user3', 'user5'] + assert [r["email"] for r in records] == ["user1", "user2", "user3", "user5"] assert m.call_count == 3 @@ -257,12 +259,12 @@ def test_retry_read_timeout(): stream._session.send = MagicMock(side_effect=requests.exceptions.ReadTimeout) with pytest.raises(requests.exceptions.ReadTimeout): list(read_full_refresh(stream)) - stream._session.send.call_args[1] == {'timeout': (60, 300)} + stream._session.send.call_args[1] == {"timeout": (60, 300)} assert stream._session.send.call_count == stream.max_retries + 1 stream = Campaigns(authenticator=None) stream._session.send = MagicMock(side_effect=requests.exceptions.ConnectionError) with pytest.raises(requests.exceptions.ConnectionError): list(read_full_refresh(stream)) - stream._session.send.call_args[1] == {'timeout': (60, 300)} + stream._session.send.call_args[1] == {"timeout": (60, 300)} assert stream._session.send.call_count == stream.max_retries + 1 diff --git a/airbyte-integrations/connectors/source-jdbc/.dockerignore b/airbyte-integrations/connectors/source-jdbc/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-jdbc/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-jdbc/Dockerfile b/airbyte-integrations/connectors/source-jdbc/Dockerfile deleted file mode 100644 index 69eaa15294e4..000000000000 --- a/airbyte-integrations/connectors/source-jdbc/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-jdbc - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-jdbc - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.3.5 -LABEL io.airbyte.name=airbyte/source-jdbc diff --git a/airbyte-integrations/connectors/source-jdbc/acceptance-test-config.yml b/airbyte-integrations/connectors/source-jdbc/acceptance-test-config.yml deleted file mode 100644 index b039c6cc8a2f..000000000000 --- a/airbyte-integrations/connectors/source-jdbc/acceptance-test-config.yml +++ /dev/null @@ -1,7 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-jdbc:dev -tests: - spec: - - spec_path: "src/test-integration/resources/expected_spec.json" - config_path: "src/test-integration/resources/dummy_config.json" diff --git a/airbyte-integrations/connectors/source-jdbc/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-jdbc/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-jdbc/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-jdbc/build.gradle b/airbyte-integrations/connectors/source-jdbc/build.gradle deleted file mode 100644 index 488357c8ec51..000000000000 --- a/airbyte-integrations/connectors/source-jdbc/build.gradle +++ /dev/null @@ -1,57 +0,0 @@ -plugins { - id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' - id 'airbyte-connector-acceptance-test' - id "java-library" - // https://docs.gradle.org/current/userguide/java_testing.html#sec:java_test_fixtures - id "java-test-fixtures" -} - -application { - mainClass = 'io.airbyte.integrations.source.jdbc.JdbcSource' -} - -project.configurations { - testFixturesImplementation.extendsFrom implementation - testFixturesRuntimeOnly.extendsFrom runtimeOnly -} - -dependencies { - implementation project(':airbyte-commons') - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:connectors:source-relational-db') - - implementation 'org.apache.commons:commons-lang3:3.11' - implementation libs.bundles.datadog - - testImplementation project(':airbyte-test-utils') - - testImplementation libs.postgresql - testImplementation libs.connectors.testcontainers.postgresql - - testImplementation libs.junit.jupiter.system.stubs - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - integrationTestJavaImplementation libs.connectors.testcontainers.postgresql - - testFixturesImplementation "org.hamcrest:hamcrest-all:1.3" - testFixturesImplementation libs.airbyte.protocol - testFixturesImplementation project(':airbyte-db:db-lib') - testFixturesImplementation project(':airbyte-integrations:bases:base-java') - - // todo (cgardens) - the java-test-fixtures plugin doesn't by default extend from test. - // we cannot make it depend on the dependencies of source-jdbc:test, because source-jdbc:test - // is going to depend on these fixtures. need to find a way to get fixtures to inherit the - // common test classes without duplicating them. this should be part of whatever solution we - // decide on for a "test-java-lib". the current implementation is leveraging the existing - // plugin, but we can something different if we don't like this tool. - testFixturesRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.4.2' - testFixturesImplementation 'org.junit.jupiter:junit-jupiter-api:5.4.2' - testFixturesImplementation 'org.junit.jupiter:junit-jupiter-params:5.4.2' - testFixturesImplementation group: 'org.mockito', name: 'mockito-junit-jupiter', version: '4.0.0' - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) -} diff --git a/airbyte-integrations/connectors/source-jdbc/readme.md b/airbyte-integrations/connectors/source-jdbc/readme.md deleted file mode 100644 index 30ba2fa6dc6c..000000000000 --- a/airbyte-integrations/connectors/source-jdbc/readme.md +++ /dev/null @@ -1,7 +0,0 @@ -# JDBC Source - -We are not planning to expose this source in the UI yet. It serves as a base upon which we can build all of our other JDBC-compliant sources. - -The reasons we are not exposing this source by itself are: -1. It is not terribly user-friendly (jdbc urls are hard for a human to parse) -1. Each JDBC-compliant db, we need to make sure the appropriate drivers are installed on the image. We don't want to frontload installing all possible drivers, and instead would like to be more methodical. Instead for each JDBC-compliant source, we will extend this one and then install only the necessary JDBC drivers on that source's image. diff --git a/airbyte-integrations/connectors/source-jdbc/src/test-integration/java/io/airbyte/integrations/source/jdbc/JdbcSourceSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-jdbc/src/test-integration/java/io/airbyte/integrations/source/jdbc/JdbcSourceSourceAcceptanceTest.java deleted file mode 100644 index 10b535c704e0..000000000000 --- a/airbyte-integrations/connectors/source-jdbc/src/test-integration/java/io/airbyte/integrations/source/jdbc/JdbcSourceSourceAcceptanceTest.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.jdbc; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import java.sql.SQLException; -import java.util.HashMap; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.testcontainers.containers.PostgreSQLContainer; - -/** - * The name here intentionally is a little weird to avoid conflicting with { @link - * JdbcSourceAcceptanceTest} This class is running { @link SourceAcceptanceTest } for the { @link - * JdbcSource }. { @link JdbcSourceAcceptanceTest} is the base class for JDBC sources. - */ -public class JdbcSourceSourceAcceptanceTest extends SourceAcceptanceTest { - - private static final String SCHEMA_NAME = "public"; - private static final String STREAM_NAME = "id_and_name"; - private PostgreSQLContainer container; - private JsonNode config; - - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws SQLException { - container = new PostgreSQLContainer<>("postgres:13-alpine"); - container.start(); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put(JdbcUtils.JDBC_URL_KEY, String.format("jdbc:postgresql://%s:%s/%s", - container.getHost(), - container.getFirstMappedPort(), - container.getDatabaseName())) - .build()); - - try (final DSLContext dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - config.get(JdbcUtils.JDBC_URL_KEY).asText(), - SQLDialect.POSTGRES)) { - final Database database = new Database(dslContext); - - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - return null; - }); - } - } - - @Override - protected void tearDown(final TestDestinationEnv testEnv) throws Exception { - container.close(); - } - - @Override - protected String getImageName() { - return "airbyte/source-jdbc:dev"; - } - - @Override - protected ConnectorSpecification getSpec() throws Exception { - return Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class); - } - - @Override - protected JsonNode getConfig() { - return config; - } - - @Override - protected ConfiguredAirbyteCatalog getConfiguredCatalog() { - return CatalogHelpers.createConfiguredAirbyteCatalog( - STREAM_NAME, - SCHEMA_NAME, - Field.of("id", JsonSchemaType.NUMBER), - Field.of("name", JsonSchemaType.STRING)); - } - - @Override - protected JsonNode getState() { - return Jsons.jsonNode(new HashMap<>()); - } - -} diff --git a/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/DefaultJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/DefaultJdbcSourceAcceptanceTest.java deleted file mode 100644 index 8971fe9adea7..000000000000 --- a/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/DefaultJdbcSourceAcceptanceTest.java +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.jdbc; - -import static io.airbyte.integrations.source.jdbc.JdbcDataSourceUtils.assertCustomParametersDontOverwriteDefaultParameters; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.io.IOs; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.string.Strings; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; -import io.airbyte.integrations.source.relationaldb.models.CdcState; -import io.airbyte.integrations.util.HostPortResolver; -import io.airbyte.protocol.models.v0.AirbyteGlobalState; -import io.airbyte.protocol.models.v0.AirbyteStateMessage; -import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; -import io.airbyte.protocol.models.v0.AirbyteStreamState; -import io.airbyte.test.utils.PostgreSQLContainerHelper; -import java.sql.JDBCType; -import java.util.List; -import java.util.Map; -import java.util.Set; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.utility.MountableFile; -import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; -import uk.org.webcompere.systemstubs.jupiter.SystemStub; -import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; - -/** - * Runs the acceptance tests in the source-jdbc test module. We want this module to run these tests - * itself as a sanity check. The trade off here is that this class is duplicated from the one used - * in source-postgres. - */ -@ExtendWith(SystemStubsExtension.class) -class DefaultJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { - - @SystemStub - private EnvironmentVariables environmentVariables; - - private static PostgreSQLContainer PSQL_DB; - - private JsonNode config; - private String dbName; - - @BeforeAll - static void init() { - PSQL_DB = new PostgreSQLContainer<>("postgres:13-alpine"); - PSQL_DB.start(); - CREATE_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s BIT(3) NOT NULL);"; - INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES(B'101');"; - } - - @BeforeEach - public void setup() throws Exception { - dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, PSQL_DB.getHost()) - .put(JdbcUtils.PORT_KEY, PSQL_DB.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, dbName) - .put(JdbcUtils.USERNAME_KEY, PSQL_DB.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, PSQL_DB.getPassword()) - .build()); - - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - - final String initScriptName = "init_" + dbName.concat(".sql"); - final String tmpFilePath = IOs.writeFileToRandomTmpDir(initScriptName, "CREATE DATABASE " + dbName + ";"); - PostgreSQLContainerHelper.runSqlScript(MountableFile.forHostPath(tmpFilePath), PSQL_DB); - - super.setup(); - } - - @Override - public boolean supportsSchemas() { - return true; - } - - @Override - public AbstractJdbcSource getJdbcSource() { - return new PostgresTestSource(); - } - - @Override - public JsonNode getConfig() { - return config; - } - - public JsonNode getConfigWithConnectionProperties(final PostgreSQLContainer psqlDb, final String dbName, final String additionalParameters) { - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(psqlDb)) - .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(psqlDb)) - .put(JdbcUtils.DATABASE_KEY, dbName) - .put(JdbcUtils.SCHEMAS_KEY, List.of(SCHEMA_NAME)) - .put(JdbcUtils.USERNAME_KEY, psqlDb.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, psqlDb.getPassword()) - .put(JdbcUtils.CONNECTION_PROPERTIES_KEY, additionalParameters) - .build()); - } - - @Override - public String getDriverClass() { - return PostgresTestSource.DRIVER_CLASS; - } - - @Override - protected boolean supportsPerStream() { - return true; - } - - @AfterAll - static void cleanUp() { - PSQL_DB.close(); - } - - private static class PostgresTestSource extends AbstractJdbcSource implements Source { - - private static final Logger LOGGER = LoggerFactory.getLogger(PostgresTestSource.class); - - static final String DRIVER_CLASS = DatabaseDriver.POSTGRESQL.getDriverClassName(); - - public PostgresTestSource() { - super(DRIVER_CLASS, AdaptiveStreamingQueryConfig::new, JdbcUtils.getDefaultSourceOperations()); - } - - @Override - public JsonNode toDatabaseConfig(final JsonNode config) { - final ImmutableMap.Builder configBuilder = ImmutableMap.builder() - .put(JdbcUtils.USERNAME_KEY, config.get(JdbcUtils.USERNAME_KEY).asText()) - .put(JdbcUtils.JDBC_URL_KEY, String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asInt(), - config.get(JdbcUtils.DATABASE_KEY).asText())); - - if (config.has(JdbcUtils.PASSWORD_KEY)) { - configBuilder.put(JdbcUtils.PASSWORD_KEY, config.get(JdbcUtils.PASSWORD_KEY).asText()); - } - - return Jsons.jsonNode(configBuilder.build()); - } - - @Override - public Set getExcludedInternalNameSpaces() { - return Set.of("information_schema", "pg_catalog", "pg_internal", "catalog_history"); - } - - // TODO This is a temporary override so that the Postgres source can take advantage of per-stream - // state - @Override - protected List generateEmptyInitialState(final JsonNode config) { - if (getSupportedStateType(config) == AirbyteStateType.GLOBAL) { - final AirbyteGlobalState globalState = new AirbyteGlobalState() - .withSharedState(Jsons.jsonNode(new CdcState())) - .withStreamStates(List.of()); - return List.of(new AirbyteStateMessage().withType(AirbyteStateType.GLOBAL).withGlobal(globalState)); - } else { - return List.of(new AirbyteStateMessage() - .withType(AirbyteStateType.STREAM) - .withStream(new AirbyteStreamState())); - } - } - - @Override - protected AirbyteStateType getSupportedStateType(final JsonNode config) { - return AirbyteStateType.STREAM; - } - - public static void main(final String[] args) throws Exception { - final Source source = new PostgresTestSource(); - LOGGER.info("starting source: {}", PostgresTestSource.class); - new IntegrationRunner(source).run(args); - LOGGER.info("completed source: {}", PostgresTestSource.class); - } - - } - - @Test - void testCustomParametersOverwriteDefaultParametersExpectException() { - final String connectionPropertiesUrl = "ssl=false"; - final JsonNode config = getConfigWithConnectionProperties(PSQL_DB, dbName, connectionPropertiesUrl); - final Map customParameters = JdbcUtils.parseJdbcParameters(config, JdbcUtils.CONNECTION_PROPERTIES_KEY, "&"); - final Map defaultParameters = Map.of( - "ssl", "true", - "sslmode", "require"); - assertThrows(IllegalArgumentException.class, () -> { - assertCustomParametersDontOverwriteDefaultParameters(customParameters, defaultParameters); - }); - } - -} diff --git a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java deleted file mode 100644 index 3f4c420cac77..000000000000 --- a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java +++ /dev/null @@ -1,1277 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.jdbc.test; - -import static io.airbyte.db.jdbc.JdbcUtils.getDefaultSourceOperations; -import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifier; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.doCallRealMethod; -import static org.mockito.Mockito.spy; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.commons.string.Strings; -import io.airbyte.commons.util.MoreIterators; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcSourceOperations; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.StreamingJdbcDatabase; -import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils; -import io.airbyte.integrations.source.relationaldb.models.DbState; -import io.airbyte.integrations.source.relationaldb.models.DbStreamState; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.AirbyteCatalog; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteMessage.Type; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import io.airbyte.protocol.models.v0.AirbyteStateMessage; -import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; -import io.airbyte.protocol.models.v0.AirbyteStream; -import io.airbyte.protocol.models.v0.AirbyteStreamState; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.StreamDescriptor; -import io.airbyte.protocol.models.v0.SyncMode; -import java.math.BigDecimal; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.stream.Collectors; -import javax.sql.DataSource; -import org.hamcrest.Matchers; -import org.junit.jupiter.api.Test; - -/** - * Tests that should be run on all Sources that extend the AbstractJdbcSource. - */ -// How leverage these tests: -// 1. Extend this class in the test module of the Source. -// 2. From the class that extends this one, you MUST call super.setup() in a @BeforeEach method. -// Otherwise you'll see many NPE issues. Your before each should also handle providing a fresh -// database between each test. -// 3. From the class that extends this one, implement a @AfterEach that cleans out the database -// between each test. -// 4. Then implement the abstract methods documented below. -@SuppressFBWarnings( - value = {"MS_SHOULD_BE_FINAL"}, - justification = "The static variables are updated in sub classes for convenience, and cannot be final.") -public abstract class JdbcSourceAcceptanceTest { - - // schema name must be randomized for each test run, - // otherwise parallel runs can interfere with each other - public static String SCHEMA_NAME = Strings.addRandomSuffix("jdbc_integration_test1", "_", 5).toLowerCase(); - public static String SCHEMA_NAME2 = Strings.addRandomSuffix("jdbc_integration_test2", "_", 5).toLowerCase(); - public static Set TEST_SCHEMAS = Set.of(SCHEMA_NAME, SCHEMA_NAME2); - - public static String TABLE_NAME = "id_and_name"; - public static String TABLE_NAME_WITH_SPACES = "id and name"; - public static String TABLE_NAME_WITHOUT_PK = "id_and_name_without_pk"; - public static String TABLE_NAME_COMPOSITE_PK = "full_name_composite_pk"; - public static String TABLE_NAME_WITHOUT_CURSOR_TYPE = "table_without_cursor_type"; - public static String TABLE_NAME_WITH_NULLABLE_CURSOR_TYPE = "table_with_null_cursor_type"; - // this table is used in testing incremental sync with concurrent insertions - public static String TABLE_NAME_AND_TIMESTAMP = "name_and_timestamp"; - - public static String COL_ID = "id"; - public static String COL_NAME = "name"; - public static String COL_UPDATED_AT = "updated_at"; - public static String COL_FIRST_NAME = "first_name"; - public static String COL_LAST_NAME = "last_name"; - public static String COL_LAST_NAME_WITH_SPACE = "last name"; - public static String COL_CURSOR = "cursor_field"; - public static String COL_TIMESTAMP = "timestamp"; - public static String COL_TIMESTAMP_TYPE = "TIMESTAMP"; - public static Number ID_VALUE_1 = 1; - public static Number ID_VALUE_2 = 2; - public static Number ID_VALUE_3 = 3; - public static Number ID_VALUE_4 = 4; - public static Number ID_VALUE_5 = 5; - - public static String DROP_SCHEMA_QUERY = "DROP SCHEMA IF EXISTS %s CASCADE"; - public static String COLUMN_CLAUSE_WITH_PK = "id INTEGER, name VARCHAR(200) NOT NULL, updated_at DATE NOT NULL"; - public static String COLUMN_CLAUSE_WITHOUT_PK = "id INTEGER, name VARCHAR(200) NOT NULL, updated_at DATE NOT NULL"; - public static String COLUMN_CLAUSE_WITH_COMPOSITE_PK = - "first_name VARCHAR(200) NOT NULL, last_name VARCHAR(200) NOT NULL, updated_at DATE NOT NULL"; - - public static String CREATE_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s bit NOT NULL);"; - public static String INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES(0);"; - public static String CREATE_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s VARCHAR(20));"; - public static String INSERT_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES('Hello world :)');"; - public static String INSERT_TABLE_NAME_AND_TIMESTAMP_QUERY = "INSERT INTO %s (name, timestamp) VALUES ('%s', '%s')"; - - public JsonNode config; - public DataSource dataSource; - public JdbcDatabase database; - public JdbcSourceOperations sourceOperations = getSourceOperations(); - public Source source; - public static String streamName; - - /** - * These tests write records without specifying a namespace (schema name). They will be written into - * whatever the default schema is for the database. When they are discovered they will be namespaced - * by the schema name (e.g. .). Thus the source needs to tell the - * tests what that default schema name is. If the database does not support schemas, then database - * name should used instead. - * - * @return name that will be used to namespace the record. - */ - public abstract boolean supportsSchemas(); - - /** - * A valid configuration to connect to a test database. - * - * @return config - */ - public abstract JsonNode getConfig(); - - /** - * Full qualified class name of the JDBC driver for the database. - * - * @return driver - */ - public abstract String getDriverClass(); - - /** - * An instance of the source that should be tests. - * - * @return abstract jdbc source - */ - public abstract AbstractJdbcSource getJdbcSource(); - - /** - * In some cases the Source that is being tested may be an AbstractJdbcSource, but because it is - * decorated, Java cannot recognize it as such. In these cases, as a workaround a user can choose to - * override getJdbcSource and have it return null. Then they can override this method with the - * decorated source AND override getToDatabaseConfigFunction with the appropriate - * toDatabaseConfigFunction that is hidden behind the decorator. - * - * @return source - */ - public Source getSource() { - return getJdbcSource(); - } - - /** - * See getSource() for when to override this method. - * - * @return a function that maps a source's config to a jdbc config. - */ - public Function getToDatabaseConfigFunction() { - return getJdbcSource()::toDatabaseConfig; - } - - protected JdbcSourceOperations getSourceOperations() { - return getDefaultSourceOperations(); - } - - protected String createTableQuery(final String tableName, final String columnClause, final String primaryKeyClause) { - return String.format("CREATE TABLE %s(%s %s %s)", - tableName, columnClause, primaryKeyClause.equals("") ? "" : ",", primaryKeyClause); - } - - protected String primaryKeyClause(final List columns) { - if (columns.isEmpty()) { - return ""; - } - - final StringBuilder clause = new StringBuilder(); - clause.append("PRIMARY KEY ("); - for (int i = 0; i < columns.size(); i++) { - clause.append(columns.get(i)); - if (i != (columns.size() - 1)) { - clause.append(","); - } - } - clause.append(")"); - return clause.toString(); - } - - protected String getJdbcParameterDelimiter() { - return "&"; - } - - public void setup() throws Exception { - source = getSource(); - config = getConfig(); - final JsonNode jdbcConfig = getToDatabaseConfigFunction().apply(config); - - streamName = TABLE_NAME; - - dataSource = getDataSource(jdbcConfig); - - database = new StreamingJdbcDatabase(dataSource, - getDefaultSourceOperations(), - AdaptiveStreamingQueryConfig::new); - - if (supportsSchemas()) { - createSchemas(); - } - - if (getDriverClass().toLowerCase().contains("oracle")) { - database.execute(connection -> connection.createStatement() - .execute("ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD'")); - } - - database.execute(connection -> { - - connection.createStatement().execute( - createTableQuery(getFullyQualifiedTableName(TABLE_NAME), COLUMN_CLAUSE_WITH_PK, - primaryKeyClause(Collections.singletonList("id")))); - connection.createStatement().execute( - String.format("INSERT INTO %s(id, name, updated_at) VALUES (1,'picard', '2004-10-19')", - getFullyQualifiedTableName(TABLE_NAME))); - connection.createStatement().execute( - String.format("INSERT INTO %s(id, name, updated_at) VALUES (2, 'crusher', '2005-10-19')", - getFullyQualifiedTableName(TABLE_NAME))); - connection.createStatement().execute( - String.format("INSERT INTO %s(id, name, updated_at) VALUES (3, 'vash', '2006-10-19')", - getFullyQualifiedTableName(TABLE_NAME))); - - connection.createStatement().execute( - createTableQuery(getFullyQualifiedTableName(TABLE_NAME_WITHOUT_PK), - COLUMN_CLAUSE_WITHOUT_PK, "")); - connection.createStatement().execute( - String.format("INSERT INTO %s(id, name, updated_at) VALUES (1,'picard', '2004-10-19')", - getFullyQualifiedTableName(TABLE_NAME_WITHOUT_PK))); - connection.createStatement().execute( - String.format("INSERT INTO %s(id, name, updated_at) VALUES (2, 'crusher', '2005-10-19')", - getFullyQualifiedTableName(TABLE_NAME_WITHOUT_PK))); - connection.createStatement().execute( - String.format("INSERT INTO %s(id, name, updated_at) VALUES (3, 'vash', '2006-10-19')", - getFullyQualifiedTableName(TABLE_NAME_WITHOUT_PK))); - - connection.createStatement().execute( - createTableQuery(getFullyQualifiedTableName(TABLE_NAME_COMPOSITE_PK), - COLUMN_CLAUSE_WITH_COMPOSITE_PK, - primaryKeyClause(List.of("first_name", "last_name")))); - connection.createStatement().execute( - String.format( - "INSERT INTO %s(first_name, last_name, updated_at) VALUES ('first' ,'picard', '2004-10-19')", - getFullyQualifiedTableName(TABLE_NAME_COMPOSITE_PK))); - connection.createStatement().execute( - String.format( - "INSERT INTO %s(first_name, last_name, updated_at) VALUES ('second', 'crusher', '2005-10-19')", - getFullyQualifiedTableName(TABLE_NAME_COMPOSITE_PK))); - connection.createStatement().execute( - String.format( - "INSERT INTO %s(first_name, last_name, updated_at) VALUES ('third', 'vash', '2006-10-19')", - getFullyQualifiedTableName(TABLE_NAME_COMPOSITE_PK))); - - }); - } - - protected DataSource getDataSource(final JsonNode jdbcConfig) { - return DataSourceFactory.create( - jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText(), - jdbcConfig.has(JdbcUtils.PASSWORD_KEY) ? jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, - getDriverClass(), - jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), - JdbcUtils.parseJdbcParameters(jdbcConfig, JdbcUtils.CONNECTION_PROPERTIES_KEY, getJdbcParameterDelimiter())); - } - - public void tearDown() throws SQLException { - dropSchemas(); - } - - @Test - void testSpec() throws Exception { - final ConnectorSpecification actual = source.spec(); - final String resourceString = MoreResources.readResource("spec.json"); - final ConnectorSpecification expected = Jsons.deserialize(resourceString, ConnectorSpecification.class); - - assertEquals(expected, actual); - } - - @Test - void testCheckSuccess() throws Exception { - final AirbyteConnectionStatus actual = source.check(config); - final AirbyteConnectionStatus expected = new AirbyteConnectionStatus().withStatus(Status.SUCCEEDED); - assertEquals(expected, actual); - } - - @Test - void testCheckFailure() throws Exception { - ((ObjectNode) config).put(JdbcUtils.PASSWORD_KEY, "fake"); - final AirbyteConnectionStatus actual = source.check(config); - assertEquals(Status.FAILED, actual.getStatus()); - } - - @Test - void testDiscover() throws Exception { - final AirbyteCatalog actual = filterOutOtherSchemas(source.discover(config)); - final AirbyteCatalog expected = getCatalog(getDefaultNamespace()); - assertEquals(expected.getStreams().size(), actual.getStreams().size()); - actual.getStreams().forEach(actualStream -> { - final Optional expectedStream = - expected.getStreams().stream() - .filter(stream -> stream.getNamespace().equals(actualStream.getNamespace()) && stream.getName().equals(actualStream.getName())) - .findAny(); - assertTrue(expectedStream.isPresent(), String.format("Unexpected stream %s", actualStream.getName())); - assertEquals(expectedStream.get(), actualStream); - }); - } - - @Test - protected void testDiscoverWithNonCursorFields() throws Exception { - database.execute(connection -> { - connection.createStatement() - .execute(String.format(CREATE_TABLE_WITHOUT_CURSOR_TYPE_QUERY, getFullyQualifiedTableName(TABLE_NAME_WITHOUT_CURSOR_TYPE), COL_CURSOR)); - connection.createStatement().execute(String.format(INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY, - getFullyQualifiedTableName(TABLE_NAME_WITHOUT_CURSOR_TYPE))); - }); - final AirbyteCatalog actual = filterOutOtherSchemas(source.discover(config)); - final AirbyteStream stream = - actual.getStreams().stream().filter(s -> s.getName().equalsIgnoreCase(TABLE_NAME_WITHOUT_CURSOR_TYPE)).findFirst().orElse(null); - assertNotNull(stream); - assertEquals(TABLE_NAME_WITHOUT_CURSOR_TYPE.toLowerCase(), stream.getName().toLowerCase()); - assertEquals(1, stream.getSupportedSyncModes().size()); - assertEquals(SyncMode.FULL_REFRESH, stream.getSupportedSyncModes().get(0)); - } - - @Test - protected void testDiscoverWithNullableCursorFields() throws Exception { - database.execute(connection -> { - connection.createStatement() - .execute(String.format(CREATE_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY, getFullyQualifiedTableName(TABLE_NAME_WITH_NULLABLE_CURSOR_TYPE), - COL_CURSOR)); - connection.createStatement().execute(String.format(INSERT_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY, - getFullyQualifiedTableName(TABLE_NAME_WITH_NULLABLE_CURSOR_TYPE))); - }); - final AirbyteCatalog actual = filterOutOtherSchemas(source.discover(config)); - final AirbyteStream stream = - actual.getStreams().stream().filter(s -> s.getName().equalsIgnoreCase(TABLE_NAME_WITH_NULLABLE_CURSOR_TYPE)).findFirst().orElse(null); - assertNotNull(stream); - assertEquals(TABLE_NAME_WITH_NULLABLE_CURSOR_TYPE.toLowerCase(), stream.getName().toLowerCase()); - assertEquals(2, stream.getSupportedSyncModes().size()); - assertTrue(stream.getSupportedSyncModes().contains(SyncMode.FULL_REFRESH)); - assertTrue(stream.getSupportedSyncModes().contains(SyncMode.INCREMENTAL)); - } - - protected AirbyteCatalog filterOutOtherSchemas(final AirbyteCatalog catalog) { - if (supportsSchemas()) { - final AirbyteCatalog filteredCatalog = Jsons.clone(catalog); - filteredCatalog.setStreams(filteredCatalog.getStreams() - .stream() - .filter(stream -> TEST_SCHEMAS.stream().anyMatch(schemaName -> stream.getNamespace().startsWith(schemaName))) - .collect(Collectors.toList())); - return filteredCatalog; - } else { - return catalog; - } - - } - - @Test - void testDiscoverWithMultipleSchemas() throws Exception { - // clickhouse and mysql do not have a concept of schemas, so this test does not make sense for them. - String driverClass = getDriverClass().toLowerCase(); - if (driverClass.contains("mysql") || driverClass.contains("clickhouse") || driverClass.contains("teradata")) { - return; - } - - // add table and data to a separate schema. - database.execute(connection -> { - connection.createStatement().execute( - String.format("CREATE TABLE %s(id VARCHAR(200) NOT NULL, name VARCHAR(200) NOT NULL)", - RelationalDbQueryUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))); - connection.createStatement() - .execute(String.format("INSERT INTO %s(id, name) VALUES ('1','picard')", - RelationalDbQueryUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))); - connection.createStatement() - .execute(String.format("INSERT INTO %s(id, name) VALUES ('2', 'crusher')", - RelationalDbQueryUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))); - connection.createStatement() - .execute(String.format("INSERT INTO %s(id, name) VALUES ('3', 'vash')", - RelationalDbQueryUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))); - }); - - final AirbyteCatalog actual = source.discover(config); - - final AirbyteCatalog expected = getCatalog(getDefaultNamespace()); - final List catalogStreams = new ArrayList<>(); - catalogStreams.addAll(expected.getStreams()); - catalogStreams.add(CatalogHelpers - .createAirbyteStream(TABLE_NAME, - SCHEMA_NAME2, - Field.of(COL_ID, JsonSchemaType.STRING), - Field.of(COL_NAME, JsonSchemaType.STRING)) - .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))); - expected.setStreams(catalogStreams); - // sort streams by name so that we are comparing lists with the same order. - final Comparator schemaTableCompare = Comparator.comparing(stream -> stream.getNamespace() + "." + stream.getName()); - expected.getStreams().sort(schemaTableCompare); - actual.getStreams().sort(schemaTableCompare); - assertEquals(expected, filterOutOtherSchemas(actual)); - } - - @Test - void testReadSuccess() throws Exception { - final List actualMessages = - MoreIterators.toList( - source.read(config, getConfiguredCatalogWithOneStream(getDefaultNamespace()), null)); - - setEmittedAtToNull(actualMessages); - final List expectedMessages = getTestMessages(); - assertThat(expectedMessages, Matchers.containsInAnyOrder(actualMessages.toArray())); - assertThat(actualMessages, Matchers.containsInAnyOrder(expectedMessages.toArray())); - } - - @Test - void testReadOneColumn() throws Exception { - final ConfiguredAirbyteCatalog catalog = CatalogHelpers - .createConfiguredAirbyteCatalog(streamName, getDefaultNamespace(), Field.of(COL_ID, JsonSchemaType.NUMBER)); - final List actualMessages = MoreIterators - .toList(source.read(config, catalog, null)); - - setEmittedAtToNull(actualMessages); - - final List expectedMessages = getAirbyteMessagesReadOneColumn(); - assertEquals(expectedMessages.size(), actualMessages.size()); - assertTrue(expectedMessages.containsAll(actualMessages)); - assertTrue(actualMessages.containsAll(expectedMessages)); - } - - protected List getAirbyteMessagesReadOneColumn() { - final List expectedMessages = getTestMessages().stream() - .map(Jsons::clone) - .peek(m -> { - ((ObjectNode) m.getRecord().getData()).remove(COL_NAME); - ((ObjectNode) m.getRecord().getData()).remove(COL_UPDATED_AT); - ((ObjectNode) m.getRecord().getData()).replace(COL_ID, - convertIdBasedOnDatabase(m.getRecord().getData().get(COL_ID).asInt())); - }) - .collect(Collectors.toList()); - return expectedMessages; - } - - @Test - void testReadMultipleTables() throws Exception { - final ConfiguredAirbyteCatalog catalog = getConfiguredCatalogWithOneStream( - getDefaultNamespace()); - final List expectedMessages = new ArrayList<>(getTestMessages()); - - for (int i = 2; i < 10; i++) { - final int iFinal = i; - final String streamName2 = streamName + i; - database.execute(connection -> { - connection.createStatement() - .execute( - createTableQuery(getFullyQualifiedTableName(TABLE_NAME + iFinal), - "id INTEGER, name VARCHAR(200)", "")); - connection.createStatement() - .execute(String.format("INSERT INTO %s(id, name) VALUES (1,'picard')", - getFullyQualifiedTableName(TABLE_NAME + iFinal))); - connection.createStatement() - .execute(String.format("INSERT INTO %s(id, name) VALUES (2, 'crusher')", - getFullyQualifiedTableName(TABLE_NAME + iFinal))); - connection.createStatement() - .execute(String.format("INSERT INTO %s(id, name) VALUES (3, 'vash')", - getFullyQualifiedTableName(TABLE_NAME + iFinal))); - }); - catalog.getStreams().add(CatalogHelpers.createConfiguredAirbyteStream( - streamName2, - getDefaultNamespace(), - Field.of(COL_ID, JsonSchemaType.NUMBER), - Field.of(COL_NAME, JsonSchemaType.STRING))); - - expectedMessages.addAll(getAirbyteMessagesSecondSync(streamName2)); - } - - final List actualMessages = MoreIterators - .toList(source.read(config, catalog, null)); - - setEmittedAtToNull(actualMessages); - - assertEquals(expectedMessages.size(), actualMessages.size()); - assertTrue(expectedMessages.containsAll(actualMessages)); - assertTrue(actualMessages.containsAll(expectedMessages)); - } - - protected List getAirbyteMessagesSecondSync(final String streamName2) { - return getTestMessages() - .stream() - .map(Jsons::clone) - .peek(m -> { - m.getRecord().setStream(streamName2); - m.getRecord().setNamespace(getDefaultNamespace()); - ((ObjectNode) m.getRecord().getData()).remove(COL_UPDATED_AT); - ((ObjectNode) m.getRecord().getData()).replace(COL_ID, - convertIdBasedOnDatabase(m.getRecord().getData().get(COL_ID).asInt())); - }) - .collect(Collectors.toList()); - - } - - @Test - void testTablesWithQuoting() throws Exception { - final ConfiguredAirbyteStream streamForTableWithSpaces = createTableWithSpaces(); - - final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() - .withStreams(List.of( - getConfiguredCatalogWithOneStream(getDefaultNamespace()).getStreams().get(0), - streamForTableWithSpaces)); - final List actualMessages = MoreIterators - .toList(source.read(config, catalog, null)); - - setEmittedAtToNull(actualMessages); - - final List expectedMessages = new ArrayList<>(getTestMessages()); - expectedMessages.addAll(getAirbyteMessagesForTablesWithQuoting(streamForTableWithSpaces)); - - assertEquals(expectedMessages.size(), actualMessages.size()); - assertTrue(expectedMessages.containsAll(actualMessages)); - assertTrue(actualMessages.containsAll(expectedMessages)); - } - - protected List getAirbyteMessagesForTablesWithQuoting(final ConfiguredAirbyteStream streamForTableWithSpaces) { - return getTestMessages() - .stream() - .map(Jsons::clone) - .peek(m -> { - m.getRecord().setStream(streamForTableWithSpaces.getStream().getName()); - ((ObjectNode) m.getRecord().getData()).set(COL_LAST_NAME_WITH_SPACE, - ((ObjectNode) m.getRecord().getData()).remove(COL_NAME)); - ((ObjectNode) m.getRecord().getData()).remove(COL_UPDATED_AT); - ((ObjectNode) m.getRecord().getData()).replace(COL_ID, - convertIdBasedOnDatabase(m.getRecord().getData().get(COL_ID).asInt())); - }) - .collect(Collectors.toList()); - } - - @SuppressWarnings("ResultOfMethodCallIgnored") - @Test - void testReadFailure() { - final ConfiguredAirbyteStream spiedAbStream = spy( - getConfiguredCatalogWithOneStream(getDefaultNamespace()).getStreams().get(0)); - final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() - .withStreams(List.of(spiedAbStream)); - doCallRealMethod().doThrow(new RuntimeException()).when(spiedAbStream).getStream(); - - assertThrows(RuntimeException.class, () -> source.read(config, catalog, null)); - } - - @Test - void testIncrementalNoPreviousState() throws Exception { - incrementalCursorCheck( - COL_ID, - null, - "3", - getTestMessages()); - } - - @Test - void testIncrementalIntCheckCursor() throws Exception { - incrementalCursorCheck( - COL_ID, - "2", - "3", - List.of(getTestMessages().get(2))); - } - - @Test - void testIncrementalStringCheckCursor() throws Exception { - incrementalCursorCheck( - COL_NAME, - "patent", - "vash", - List.of(getTestMessages().get(0), getTestMessages().get(2))); - } - - @Test - void testIncrementalStringCheckCursorSpaceInColumnName() throws Exception { - final ConfiguredAirbyteStream streamWithSpaces = createTableWithSpaces(); - - final List expectedRecordMessages = getAirbyteMessagesCheckCursorSpaceInColumnName(streamWithSpaces); - incrementalCursorCheck( - COL_LAST_NAME_WITH_SPACE, - COL_LAST_NAME_WITH_SPACE, - "patent", - "vash", - expectedRecordMessages, - streamWithSpaces); - } - - protected List getAirbyteMessagesCheckCursorSpaceInColumnName(final ConfiguredAirbyteStream streamWithSpaces) { - final AirbyteMessage firstMessage = getTestMessages().get(0); - firstMessage.getRecord().setStream(streamWithSpaces.getStream().getName()); - ((ObjectNode) firstMessage.getRecord().getData()).remove(COL_UPDATED_AT); - ((ObjectNode) firstMessage.getRecord().getData()).set(COL_LAST_NAME_WITH_SPACE, - ((ObjectNode) firstMessage.getRecord().getData()).remove(COL_NAME)); - - final AirbyteMessage secondMessage = getTestMessages().get(2); - secondMessage.getRecord().setStream(streamWithSpaces.getStream().getName()); - ((ObjectNode) secondMessage.getRecord().getData()).remove(COL_UPDATED_AT); - ((ObjectNode) secondMessage.getRecord().getData()).set(COL_LAST_NAME_WITH_SPACE, - ((ObjectNode) secondMessage.getRecord().getData()).remove(COL_NAME)); - - return List.of(firstMessage, secondMessage); - } - - @Test - void testIncrementalDateCheckCursor() throws Exception { - incrementalDateCheck(); - } - - protected void incrementalDateCheck() throws Exception { - incrementalCursorCheck( - COL_UPDATED_AT, - "2005-10-18", - "2006-10-19", - List.of(getTestMessages().get(1), getTestMessages().get(2))); - } - - @Test - void testIncrementalCursorChanges() throws Exception { - incrementalCursorCheck( - COL_ID, - COL_NAME, - // cheesing this value a little bit. in the correct implementation this initial cursor value should - // be ignored because the cursor field changed. setting it to a value that if used, will cause - // records to (incorrectly) be filtered out. - "data", - "vash", - getTestMessages()); - } - - @Test - void testReadOneTableIncrementallyTwice() throws Exception { - final String namespace = getDefaultNamespace(); - final ConfiguredAirbyteCatalog configuredCatalog = getConfiguredCatalogWithOneStream(namespace); - configuredCatalog.getStreams().forEach(airbyteStream -> { - airbyteStream.setSyncMode(SyncMode.INCREMENTAL); - airbyteStream.setCursorField(List.of(COL_ID)); - airbyteStream.setDestinationSyncMode(DestinationSyncMode.APPEND); - }); - - final List actualMessagesFirstSync = MoreIterators - .toList(source.read(config, configuredCatalog, createEmptyState(streamName, namespace))); - - final Optional stateAfterFirstSyncOptional = actualMessagesFirstSync.stream() - .filter(r -> r.getType() == Type.STATE).findFirst(); - assertTrue(stateAfterFirstSyncOptional.isPresent()); - - executeStatementReadIncrementallyTwice(); - - final List actualMessagesSecondSync = MoreIterators - .toList(source.read(config, configuredCatalog, extractState(stateAfterFirstSyncOptional.get()))); - - assertEquals(2, - (int) actualMessagesSecondSync.stream().filter(r -> r.getType() == Type.RECORD).count()); - final List expectedMessages = getExpectedAirbyteMessagesSecondSync(namespace); - - setEmittedAtToNull(actualMessagesSecondSync); - - assertEquals(expectedMessages.size(), actualMessagesSecondSync.size()); - assertTrue(expectedMessages.containsAll(actualMessagesSecondSync)); - assertTrue(actualMessagesSecondSync.containsAll(expectedMessages)); - } - - protected void executeStatementReadIncrementallyTwice() throws SQLException { - database.execute(connection -> { - connection.createStatement().execute( - String.format("INSERT INTO %s(id, name, updated_at) VALUES (4,'riker', '2006-10-19')", - getFullyQualifiedTableName(TABLE_NAME))); - connection.createStatement().execute( - String.format("INSERT INTO %s(id, name, updated_at) VALUES (5, 'data', '2006-10-19')", - getFullyQualifiedTableName(TABLE_NAME))); - }); - } - - protected List getExpectedAirbyteMessagesSecondSync(final String namespace) { - final List expectedMessages = new ArrayList<>(); - expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_4, - COL_NAME, "riker", - COL_UPDATED_AT, "2006-10-19"))))); - expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_5, - COL_NAME, "data", - COL_UPDATED_AT, "2006-10-19"))))); - final DbStreamState state = new DbStreamState() - .withStreamName(streamName) - .withStreamNamespace(namespace) - .withCursorField(List.of(COL_ID)) - .withCursor("5") - .withCursorRecordCount(1L); - expectedMessages.addAll(createExpectedTestMessages(List.of(state))); - return expectedMessages; - } - - @Test - void testReadMultipleTablesIncrementally() throws Exception { - final String tableName2 = TABLE_NAME + 2; - final String streamName2 = streamName + 2; - database.execute(ctx -> { - ctx.createStatement().execute( - createTableQuery(getFullyQualifiedTableName(tableName2), "id INTEGER, name VARCHAR(200)", "")); - ctx.createStatement().execute( - String.format("INSERT INTO %s(id, name) VALUES (1,'picard')", - getFullyQualifiedTableName(tableName2))); - ctx.createStatement().execute( - String.format("INSERT INTO %s(id, name) VALUES (2, 'crusher')", - getFullyQualifiedTableName(tableName2))); - ctx.createStatement().execute( - String.format("INSERT INTO %s(id, name) VALUES (3, 'vash')", - getFullyQualifiedTableName(tableName2))); - }); - - final String namespace = getDefaultNamespace(); - final ConfiguredAirbyteCatalog configuredCatalog = getConfiguredCatalogWithOneStream( - namespace); - configuredCatalog.getStreams().add(CatalogHelpers.createConfiguredAirbyteStream( - streamName2, - namespace, - Field.of(COL_ID, JsonSchemaType.NUMBER), - Field.of(COL_NAME, JsonSchemaType.STRING))); - configuredCatalog.getStreams().forEach(airbyteStream -> { - airbyteStream.setSyncMode(SyncMode.INCREMENTAL); - airbyteStream.setCursorField(List.of(COL_ID)); - airbyteStream.setDestinationSyncMode(DestinationSyncMode.APPEND); - }); - - final List actualMessagesFirstSync = MoreIterators - .toList(source.read(config, configuredCatalog, createEmptyState(streamName, namespace))); - - // get last state message. - final Optional stateAfterFirstSyncOptional = actualMessagesFirstSync.stream() - .filter(r -> r.getType() == Type.STATE) - .reduce((first, second) -> second); - assertTrue(stateAfterFirstSyncOptional.isPresent()); - - // we know the second streams messages are the same as the first minus the updated at column. so we - // cheat and generate the expected messages off of the first expected messages. - final List secondStreamExpectedMessages = getAirbyteMessagesSecondStreamWithNamespace(streamName2); - - // Represents the state after the first stream has been updated - final List expectedStateStreams1 = List.of( - new DbStreamState() - .withStreamName(streamName) - .withStreamNamespace(namespace) - .withCursorField(List.of(COL_ID)) - .withCursor("3") - .withCursorRecordCount(1L), - new DbStreamState() - .withStreamName(streamName2) - .withStreamNamespace(namespace) - .withCursorField(List.of(COL_ID))); - - // Represents the state after both streams have been updated - final List expectedStateStreams2 = List.of( - new DbStreamState() - .withStreamName(streamName) - .withStreamNamespace(namespace) - .withCursorField(List.of(COL_ID)) - .withCursor("3") - .withCursorRecordCount(1L), - new DbStreamState() - .withStreamName(streamName2) - .withStreamNamespace(namespace) - .withCursorField(List.of(COL_ID)) - .withCursor("3") - .withCursorRecordCount(1L)); - - final List expectedMessagesFirstSync = new ArrayList<>(getTestMessages()); - expectedMessagesFirstSync.add(createStateMessage(expectedStateStreams1.get(0), expectedStateStreams1)); - expectedMessagesFirstSync.addAll(secondStreamExpectedMessages); - expectedMessagesFirstSync.add(createStateMessage(expectedStateStreams2.get(1), expectedStateStreams2)); - - setEmittedAtToNull(actualMessagesFirstSync); - - assertEquals(expectedMessagesFirstSync.size(), actualMessagesFirstSync.size()); - assertTrue(expectedMessagesFirstSync.containsAll(actualMessagesFirstSync)); - assertTrue(actualMessagesFirstSync.containsAll(expectedMessagesFirstSync)); - } - - protected List getAirbyteMessagesSecondStreamWithNamespace(final String streamName2) { - return getTestMessages() - .stream() - .map(Jsons::clone) - .peek(m -> { - m.getRecord().setStream(streamName2); - ((ObjectNode) m.getRecord().getData()).remove(COL_UPDATED_AT); - ((ObjectNode) m.getRecord().getData()).replace(COL_ID, - convertIdBasedOnDatabase(m.getRecord().getData().get(COL_ID).asInt())); - }) - .collect(Collectors.toList()); - } - - // when initial and final cursor fields are the same. - protected void incrementalCursorCheck( - final String cursorField, - final String initialCursorValue, - final String endCursorValue, - final List expectedRecordMessages) - throws Exception { - incrementalCursorCheck(cursorField, cursorField, initialCursorValue, endCursorValue, - expectedRecordMessages); - } - - // See https://github.com/airbytehq/airbyte/issues/14732 for rationale and details. - @Test - public void testIncrementalWithConcurrentInsertion() throws Exception { - final String driverName = getDriverClass().toLowerCase(); - final String namespace = getDefaultNamespace(); - final String fullyQualifiedTableName = getFullyQualifiedTableName(TABLE_NAME_AND_TIMESTAMP); - final String columnDefinition = String.format("name VARCHAR(200) NOT NULL, %s %s NOT NULL", COL_TIMESTAMP, COL_TIMESTAMP_TYPE); - - // 1st sync - database.execute(ctx -> { - ctx.createStatement().execute(createTableQuery(fullyQualifiedTableName, columnDefinition, "")); - ctx.createStatement().execute(String.format(INSERT_TABLE_NAME_AND_TIMESTAMP_QUERY, fullyQualifiedTableName, "a", "2021-01-01 00:00:00")); - ctx.createStatement().execute(String.format(INSERT_TABLE_NAME_AND_TIMESTAMP_QUERY, fullyQualifiedTableName, "b", "2021-01-01 00:00:00")); - }); - - final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog( - new AirbyteCatalog().withStreams(List.of( - CatalogHelpers.createAirbyteStream( - TABLE_NAME_AND_TIMESTAMP, - namespace, - Field.of(COL_NAME, JsonSchemaType.STRING), - Field.of(COL_TIMESTAMP, JsonSchemaType.STRING_TIMESTAMP_WITHOUT_TIMEZONE))))); - configuredCatalog.getStreams().forEach(airbyteStream -> { - airbyteStream.setSyncMode(SyncMode.INCREMENTAL); - airbyteStream.setCursorField(List.of(COL_TIMESTAMP)); - airbyteStream.setDestinationSyncMode(DestinationSyncMode.APPEND); - }); - - final List firstSyncActualMessages = MoreIterators.toList( - source.read(config, configuredCatalog, createEmptyState(TABLE_NAME_AND_TIMESTAMP, namespace))); - - // cursor after 1st sync: 2021-01-01 00:00:00, count 2 - final Optional firstSyncStateOptional = firstSyncActualMessages.stream().filter(r -> r.getType() == Type.STATE).findFirst(); - assertTrue(firstSyncStateOptional.isPresent()); - final JsonNode firstSyncState = getStateData(firstSyncStateOptional.get(), TABLE_NAME_AND_TIMESTAMP); - assertEquals(firstSyncState.get("cursor_field").elements().next().asText(), COL_TIMESTAMP); - assertTrue(firstSyncState.get("cursor").asText().contains("2021-01-01")); - assertTrue(firstSyncState.get("cursor").asText().contains("00:00:00")); - assertEquals(2L, firstSyncState.get("cursor_record_count").asLong()); - - final List firstSyncNames = firstSyncActualMessages.stream() - .filter(r -> r.getType() == Type.RECORD) - .map(r -> r.getRecord().getData().get(COL_NAME).asText()) - .toList(); - // teradata doesn't make insertion order guarantee when equal ordering value - if (driverName.contains("teradata")) { - assertThat(List.of("a", "b"), Matchers.containsInAnyOrder(firstSyncNames.toArray())); - } else { - assertEquals(List.of("a", "b"), firstSyncNames); - } - - // 2nd sync - database.execute(ctx -> { - ctx.createStatement().execute(String.format(INSERT_TABLE_NAME_AND_TIMESTAMP_QUERY, fullyQualifiedTableName, "c", "2021-01-02 00:00:00")); - }); - - final List secondSyncActualMessages = MoreIterators.toList( - source.read(config, configuredCatalog, createState(TABLE_NAME_AND_TIMESTAMP, namespace, firstSyncState))); - - // cursor after 2nd sync: 2021-01-02 00:00:00, count 1 - final Optional secondSyncStateOptional = secondSyncActualMessages.stream().filter(r -> r.getType() == Type.STATE).findFirst(); - assertTrue(secondSyncStateOptional.isPresent()); - final JsonNode secondSyncState = getStateData(secondSyncStateOptional.get(), TABLE_NAME_AND_TIMESTAMP); - assertEquals(secondSyncState.get("cursor_field").elements().next().asText(), COL_TIMESTAMP); - assertTrue(secondSyncState.get("cursor").asText().contains("2021-01-02")); - assertTrue(secondSyncState.get("cursor").asText().contains("00:00:00")); - assertEquals(1L, secondSyncState.get("cursor_record_count").asLong()); - - final List secondSyncNames = secondSyncActualMessages.stream() - .filter(r -> r.getType() == Type.RECORD) - .map(r -> r.getRecord().getData().get(COL_NAME).asText()) - .toList(); - assertEquals(List.of("c"), secondSyncNames); - - // 3rd sync has records with duplicated cursors - database.execute(ctx -> { - ctx.createStatement().execute(String.format(INSERT_TABLE_NAME_AND_TIMESTAMP_QUERY, fullyQualifiedTableName, "d", "2021-01-02 00:00:00")); - ctx.createStatement().execute(String.format(INSERT_TABLE_NAME_AND_TIMESTAMP_QUERY, fullyQualifiedTableName, "e", "2021-01-02 00:00:00")); - ctx.createStatement().execute(String.format(INSERT_TABLE_NAME_AND_TIMESTAMP_QUERY, fullyQualifiedTableName, "f", "2021-01-03 00:00:00")); - }); - - final List thirdSyncActualMessages = MoreIterators.toList( - source.read(config, configuredCatalog, createState(TABLE_NAME_AND_TIMESTAMP, namespace, secondSyncState))); - - // Cursor after 3rd sync is: 2021-01-03 00:00:00, count 1. - final Optional thirdSyncStateOptional = thirdSyncActualMessages.stream().filter(r -> r.getType() == Type.STATE).findFirst(); - assertTrue(thirdSyncStateOptional.isPresent()); - final JsonNode thirdSyncState = getStateData(thirdSyncStateOptional.get(), TABLE_NAME_AND_TIMESTAMP); - assertEquals(thirdSyncState.get("cursor_field").elements().next().asText(), COL_TIMESTAMP); - assertTrue(thirdSyncState.get("cursor").asText().contains("2021-01-03")); - assertTrue(thirdSyncState.get("cursor").asText().contains("00:00:00")); - assertEquals(1L, thirdSyncState.get("cursor_record_count").asLong()); - - // The c, d, e, f are duplicated records from this sync, because the cursor - // record count in the database is different from that in the state. - final List thirdSyncExpectedNames = thirdSyncActualMessages.stream() - .filter(r -> r.getType() == Type.RECORD) - .map(r -> r.getRecord().getData().get(COL_NAME).asText()) - .toList(); - - // teradata doesn't make insertion order guarantee when equal ordering value - if (driverName.contains("teradata")) { - assertThat(List.of("c", "d", "e", "f"), Matchers.containsInAnyOrder(thirdSyncExpectedNames.toArray())); - } else { - assertEquals(List.of("c", "d", "e", "f"), thirdSyncExpectedNames); - } - - } - - protected JsonNode getStateData(final AirbyteMessage airbyteMessage, final String streamName) { - for (final JsonNode stream : airbyteMessage.getState().getData().get("streams")) { - if (stream.get("stream_name").asText().equals(streamName)) { - return stream; - } - } - throw new IllegalArgumentException("Stream not found in state message: " + streamName); - } - - private void incrementalCursorCheck( - final String initialCursorField, - final String cursorField, - final String initialCursorValue, - final String endCursorValue, - final List expectedRecordMessages) - throws Exception { - incrementalCursorCheck(initialCursorField, cursorField, initialCursorValue, endCursorValue, - expectedRecordMessages, - getConfiguredCatalogWithOneStream(getDefaultNamespace()).getStreams().get(0)); - } - - protected void incrementalCursorCheck( - final String initialCursorField, - final String cursorField, - final String initialCursorValue, - final String endCursorValue, - final List expectedRecordMessages, - final ConfiguredAirbyteStream airbyteStream) - throws Exception { - airbyteStream.setSyncMode(SyncMode.INCREMENTAL); - airbyteStream.setCursorField(List.of(cursorField)); - airbyteStream.setDestinationSyncMode(DestinationSyncMode.APPEND); - - final ConfiguredAirbyteCatalog configuredCatalog = new ConfiguredAirbyteCatalog() - .withStreams(List.of(airbyteStream)); - - final DbStreamState dbStreamState = buildStreamState(airbyteStream, initialCursorField, initialCursorValue); - - final List actualMessages = MoreIterators - .toList(source.read(config, configuredCatalog, Jsons.jsonNode(createState(List.of(dbStreamState))))); - - setEmittedAtToNull(actualMessages); - - final List expectedStreams = List.of(buildStreamState(airbyteStream, cursorField, endCursorValue)); - - final List expectedMessages = new ArrayList<>(expectedRecordMessages); - expectedMessages.addAll(createExpectedTestMessages(expectedStreams)); - - assertEquals(expectedMessages.size(), actualMessages.size()); - assertTrue(expectedMessages.containsAll(actualMessages)); - assertTrue(actualMessages.containsAll(expectedMessages)); - } - - protected DbStreamState buildStreamState(final ConfiguredAirbyteStream configuredAirbyteStream, - final String cursorField, - final String cursorValue) { - return new DbStreamState() - .withStreamName(configuredAirbyteStream.getStream().getName()) - .withStreamNamespace(configuredAirbyteStream.getStream().getNamespace()) - .withCursorField(List.of(cursorField)) - .withCursor(cursorValue) - .withCursorRecordCount(1L); - } - - // get catalog and perform a defensive copy. - protected ConfiguredAirbyteCatalog getConfiguredCatalogWithOneStream(final String defaultNamespace) { - final ConfiguredAirbyteCatalog catalog = CatalogHelpers.toDefaultConfiguredCatalog(getCatalog(defaultNamespace)); - // Filter to only keep the main stream name as configured stream - catalog.withStreams( - catalog.getStreams().stream().filter(s -> s.getStream().getName().equals(streamName)) - .collect(Collectors.toList())); - return catalog; - } - - protected AirbyteCatalog getCatalog(final String defaultNamespace) { - return new AirbyteCatalog().withStreams(List.of( - CatalogHelpers.createAirbyteStream( - TABLE_NAME, - defaultNamespace, - Field.of(COL_ID, JsonSchemaType.INTEGER), - Field.of(COL_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING)) - .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) - .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID))), - CatalogHelpers.createAirbyteStream( - TABLE_NAME_WITHOUT_PK, - defaultNamespace, - Field.of(COL_ID, JsonSchemaType.INTEGER), - Field.of(COL_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING)) - .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) - .withSourceDefinedPrimaryKey(Collections.emptyList()), - CatalogHelpers.createAirbyteStream( - TABLE_NAME_COMPOSITE_PK, - defaultNamespace, - Field.of(COL_FIRST_NAME, JsonSchemaType.STRING), - Field.of(COL_LAST_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING)) - .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) - .withSourceDefinedPrimaryKey( - List.of(List.of(COL_FIRST_NAME), List.of(COL_LAST_NAME))))); - } - - protected List getTestMessages() { - return List.of( - new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_1, - COL_NAME, "picard", - COL_UPDATED_AT, "2004-10-19")))), - new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_2, - COL_NAME, "crusher", - COL_UPDATED_AT, - "2005-10-19")))), - new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_3, - COL_NAME, "vash", - COL_UPDATED_AT, "2006-10-19"))))); - } - - protected List createExpectedTestMessages(final List states) { - return supportsPerStream() - ? states.stream() - .map(s -> new AirbyteMessage().withType(Type.STATE) - .withState( - new AirbyteStateMessage().withType(AirbyteStateType.STREAM) - .withStream(new AirbyteStreamState() - .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) - .withStreamState(Jsons.jsonNode(s))) - .withData(Jsons.jsonNode(new DbState().withCdc(false).withStreams(states))))) - .collect( - Collectors.toList()) - : List.of(new AirbyteMessage().withType(Type.STATE).withState(new AirbyteStateMessage().withType(AirbyteStateType.LEGACY) - .withData(Jsons.jsonNode(new DbState().withCdc(false).withStreams(states))))); - } - - protected List createState(final List states) { - return supportsPerStream() - ? states.stream() - .map(s -> new AirbyteStateMessage().withType(AirbyteStateType.STREAM) - .withStream(new AirbyteStreamState() - .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) - .withStreamState(Jsons.jsonNode(s)))) - .collect( - Collectors.toList()) - : List.of(new AirbyteStateMessage().withType(AirbyteStateType.LEGACY).withData(Jsons.jsonNode(new DbState().withStreams(states)))); - } - - protected ConfiguredAirbyteStream createTableWithSpaces() throws SQLException { - final String tableNameWithSpaces = TABLE_NAME_WITH_SPACES + "2"; - final String streamName2 = tableNameWithSpaces; - - database.execute(connection -> { - final String identifierQuoteString = connection.getMetaData().getIdentifierQuoteString(); - connection.createStatement() - .execute( - createTableQuery(getFullyQualifiedTableName( - enquoteIdentifier(tableNameWithSpaces, identifierQuoteString)), - "id INTEGER, " + enquoteIdentifier(COL_LAST_NAME_WITH_SPACE, identifierQuoteString) - + " VARCHAR(200)", - "")); - connection.createStatement() - .execute(String.format("INSERT INTO %s(id, %s) VALUES (1,'picard')", - getFullyQualifiedTableName( - enquoteIdentifier(tableNameWithSpaces, identifierQuoteString)), - enquoteIdentifier(COL_LAST_NAME_WITH_SPACE, identifierQuoteString))); - connection.createStatement() - .execute(String.format("INSERT INTO %s(id, %s) VALUES (2, 'crusher')", - getFullyQualifiedTableName( - enquoteIdentifier(tableNameWithSpaces, identifierQuoteString)), - enquoteIdentifier(COL_LAST_NAME_WITH_SPACE, identifierQuoteString))); - connection.createStatement() - .execute(String.format("INSERT INTO %s(id, %s) VALUES (3, 'vash')", - getFullyQualifiedTableName( - enquoteIdentifier(tableNameWithSpaces, identifierQuoteString)), - enquoteIdentifier(COL_LAST_NAME_WITH_SPACE, identifierQuoteString))); - }); - - return CatalogHelpers.createConfiguredAirbyteStream( - streamName2, - getDefaultNamespace(), - Field.of(COL_ID, JsonSchemaType.NUMBER), - Field.of(COL_LAST_NAME_WITH_SPACE, JsonSchemaType.STRING)); - } - - public String getFullyQualifiedTableName(final String tableName) { - return RelationalDbQueryUtils.getFullyQualifiedTableName(getDefaultSchemaName(), tableName); - } - - public void createSchemas() throws SQLException { - if (supportsSchemas()) { - for (final String schemaName : TEST_SCHEMAS) { - final String createSchemaQuery = String.format("CREATE SCHEMA %s;", schemaName); - database.execute(connection -> connection.createStatement().execute(createSchemaQuery)); - } - } - } - - public void dropSchemas() throws SQLException { - if (supportsSchemas()) { - for (final String schemaName : TEST_SCHEMAS) { - final String dropSchemaQuery = String - .format(DROP_SCHEMA_QUERY, schemaName); - database.execute(connection -> connection.createStatement().execute(dropSchemaQuery)); - } - } - } - - private JsonNode convertIdBasedOnDatabase(final int idValue) { - final var driverClass = getDriverClass().toLowerCase(); - if (driverClass.contains("oracle") || driverClass.contains("snowflake")) { - return Jsons.jsonNode(BigDecimal.valueOf(idValue)); - } else { - return Jsons.jsonNode(idValue); - } - } - - private String getDefaultSchemaName() { - return supportsSchemas() ? SCHEMA_NAME : null; - } - - protected String getDefaultNamespace() { - // mysql does not support schemas. it namespaces using database names instead. - if (getDriverClass().toLowerCase().contains("mysql") || getDriverClass().toLowerCase().contains("clickhouse") || - getDriverClass().toLowerCase().contains("teradata")) { - return config.get(JdbcUtils.DATABASE_KEY).asText(); - } else { - return SCHEMA_NAME; - } - } - - protected static void setEmittedAtToNull(final Iterable messages) { - for (final AirbyteMessage actualMessage : messages) { - if (actualMessage.getRecord() != null) { - actualMessage.getRecord().setEmittedAt(null); - } - } - } - - /** - * Tests whether the connector under test supports the per-stream state format or should use the - * legacy format for data generated by this test. - * - * @return {@code true} if the connector supports the per-stream state format or {@code false} if it - * does not support the per-stream state format (e.g. legacy format supported). Default - * value is {@code false}. - */ - protected boolean supportsPerStream() { - return false; - } - - /** - * Creates empty state with the provided stream name and namespace. - * - * @param streamName The stream name. - * @param streamNamespace The stream namespace. - * @return {@link JsonNode} representation of the generated empty state. - */ - protected JsonNode createEmptyState(final String streamName, final String streamNamespace) { - if (supportsPerStream()) { - final AirbyteStateMessage airbyteStateMessage = new AirbyteStateMessage() - .withType(AirbyteStateType.STREAM) - .withStream(new AirbyteStreamState().withStreamDescriptor(new StreamDescriptor().withName(streamName).withNamespace(streamNamespace))); - return Jsons.jsonNode(List.of(airbyteStateMessage)); - } else { - final DbState dbState = new DbState() - .withStreams(List.of(new DbStreamState().withStreamName(streamName).withStreamNamespace(streamNamespace))); - return Jsons.jsonNode(dbState); - } - } - - protected JsonNode createState(final String streamName, final String streamNamespace, final JsonNode stateData) { - if (supportsPerStream()) { - final AirbyteStateMessage airbyteStateMessage = new AirbyteStateMessage() - .withType(AirbyteStateType.STREAM) - .withStream( - new AirbyteStreamState() - .withStreamDescriptor(new StreamDescriptor().withName(streamName).withNamespace(streamNamespace)) - .withStreamState(stateData)); - return Jsons.jsonNode(List.of(airbyteStateMessage)); - } else { - final List cursorFields = MoreIterators.toList(stateData.get("cursor_field").elements()).stream().map(JsonNode::asText).toList(); - final DbState dbState = new DbState().withStreams(List.of( - new DbStreamState() - .withStreamName(streamName) - .withStreamNamespace(streamNamespace) - .withCursor(stateData.get("cursor").asText()) - .withCursorField(cursorFields) - .withCursorRecordCount(stateData.get("cursor_record_count").asLong()))); - return Jsons.jsonNode(dbState); - } - } - - /** - * Extracts the state component from the provided {@link AirbyteMessage} based on the value returned - * by {@link #supportsPerStream()}. - * - * @param airbyteMessage An {@link AirbyteMessage} that contains state. - * @return A {@link JsonNode} representation of the state contained in the {@link AirbyteMessage}. - */ - protected JsonNode extractState(final AirbyteMessage airbyteMessage) { - if (supportsPerStream()) { - return Jsons.jsonNode(List.of(airbyteMessage.getState())); - } else { - return airbyteMessage.getState().getData(); - } - } - - protected AirbyteMessage createStateMessage(final DbStreamState dbStreamState, final List legacyStates) { - if (supportsPerStream()) { - return new AirbyteMessage().withType(Type.STATE) - .withState( - new AirbyteStateMessage().withType(AirbyteStateType.STREAM) - .withStream(new AirbyteStreamState() - .withStreamDescriptor(new StreamDescriptor().withNamespace(dbStreamState.getStreamNamespace()) - .withName(dbStreamState.getStreamName())) - .withStreamState(Jsons.jsonNode(dbStreamState))) - .withData(Jsons.jsonNode(new DbState().withCdc(false).withStreams(legacyStates)))); - } else { - return new AirbyteMessage().withType(Type.STATE).withState(new AirbyteStateMessage().withType(AirbyteStateType.LEGACY) - .withData(Jsons.jsonNode(new DbState().withCdc(false).withStreams(legacyStates)))); - } - } - -} diff --git a/airbyte-integrations/connectors/source-jira/Dockerfile b/airbyte-integrations/connectors/source-jira/Dockerfile deleted file mode 100644 index 5a183bdb9b25..000000000000 --- a/airbyte-integrations/connectors/source-jira/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY source_jira ./source_jira -COPY main.py ./ -COPY setup.py ./ -RUN pip install . - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.3.12 -LABEL io.airbyte.name=airbyte/source-jira diff --git a/airbyte-integrations/connectors/source-jira/README.md b/airbyte-integrations/connectors/source-jira/README.md index 627f1f3c880d..be5d429e9d94 100644 --- a/airbyte-integrations/connectors/source-jira/README.md +++ b/airbyte-integrations/connectors/source-jira/README.md @@ -1,109 +1,67 @@ -# Jira Source +# News Api Source -This is the repository for the Jira source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/jira). +This is the repository for the News Api configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/news-api). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-jira:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/jira) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_jira/spec.json` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/news-api) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_news_api/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source jira test creds` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source news-api test creds` and place them into `secrets/config.json`. - -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-jira:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name source-news-api build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-jira:airbyteDocker +An image will be built with the tag `airbyte/source-news-api:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-news-api:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-jira:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-jira:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-jira:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-jira:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +docker run --rm airbyte/source-news-api:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-news-api:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-news-api:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-news-api:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. -If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-jira:dev \ -&& python -m pytest -p connector_acceptance_test.plugin +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-jira test ``` -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-jira:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-jira test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/jira.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-jira/acceptance-test-config.yml b/airbyte-integrations/connectors/source-jira/acceptance-test-config.yml index 669f5e2e4ccc..7790ca76c93a 100644 --- a/airbyte-integrations/connectors/source-jira/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-jira/acceptance-test-config.yml @@ -5,39 +5,40 @@ test_strictness_level: high acceptance_tests: spec: tests: - - spec_path: "source_jira/spec.json" + - spec_path: "source_jira/spec.json" connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + - config_path: "integration_tests/invalid_config_domain.json" + status: "failed" discovery: tests: - - config_path: "secrets/config.json" + - config_path: "secrets/config.json" basic_read: tests: - - config_path: "secrets/config.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - empty_streams: - - name: "project_permission_schemes" - bypass_reason: "unable to populate" - ignored_fields: - sprint_issues: - - name: updated - bypass_reason: "Unstable data" - - name: fields/updated - bypass_reason: "Unstable data" - timeout_seconds: 2400 - fail_on_extra_columns: false + - config_path: "secrets/config.json" + expect_records: + path: "integration_tests/expected_records.jsonl" + empty_streams: + - name: "project_permission_schemes" + bypass_reason: "Unable to populate. Jira doesn't support issue security for Free plan." + ignored_fields: + sprint_issues: + - name: updated + bypass_reason: "Unstable data" + - name: fields/updated + bypass_reason: "Unstable data" + timeout_seconds: 2400 incremental: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-jira/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-jira/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-jira/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-jira/build.gradle b/airbyte-integrations/connectors/source-jira/build.gradle deleted file mode 100644 index ae596c85ef07..000000000000 --- a/airbyte-integrations/connectors/source-jira/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_jira' -} diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-jira/integration_tests/abnormal_state.json index af6f2b9ee050..727d1fa0207e 100644 --- a/airbyte-integrations/connectors/source-jira/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-jira/integration_tests/abnormal_state.json @@ -6,7 +6,15 @@ "name": "board_issues" }, "stream_state": { - "updated": "2122-01-01T00:00:00Z" + "1": { + "updated": "2122-01-01T00:00:00Z" + }, + "17": { + "updated": "2122-01-01T00:00:00Z" + }, + "58": { + "updated": "2122-01-01T00:00:00Z" + } } } }, diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json index 41150ae12b29..637227111add 100644 --- a/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-jira/integration_tests/configured_catalog.json @@ -136,6 +136,16 @@ "sync_mode": "full_refresh", "destination_sync_mode": "append" }, + { + "stream": { + "name": "issue_custom_field_options", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, { "stream": { "name": "issue_link_types", @@ -215,6 +225,16 @@ "sync_mode": "full_refresh", "destination_sync_mode": "append" }, + { + "stream": { + "name": "issue_transitions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["issueId"], ["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, { "stream": { "name": "issue_type_schemes", @@ -225,6 +245,16 @@ "sync_mode": "full_refresh", "destination_sync_mode": "append" }, + { + "stream": { + "name": "issue_types", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, { "stream": { "name": "issue_type_screen_schemes", @@ -365,6 +395,16 @@ "sync_mode": "full_refresh", "destination_sync_mode": "append" }, + { + "stream": { + "name": "project_roles", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, { "stream": { "name": "project_types", diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-jira/integration_tests/expected_records.jsonl index 8b1db037ebac..d5338e2d7ffd 100644 --- a/airbyte-integrations/connectors/source-jira/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-jira/integration_tests/expected_records.jsonl @@ -1,145 +1,160 @@ -{"stream":"application_roles","data":{"key":"jira-servicedesk","groups":["jira-administrators","jira-software-users","jira-users","Test group 1","Test group 0","atlassian-addons-admin","integration-test-group","jira-servicemanagement-users-airbyteio","jira-admins-airbyteio","site-admins","Test group 10","administrators"],"groupDetails":[{"name":"jira-software-users","groupId":"4452b254-035d-469a-a422-1f4666dce50e","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=4452b254-035d-469a-a422-1f4666dce50e"},{"name":"Test group 0","groupId":"ee8d15d1-6462-406a-b0a6-8065b7e4cdd7","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=ee8d15d1-6462-406a-b0a6-8065b7e4cdd7"},{"name":"administrators","groupId":"0ca6e087-7a61-4986-a269-98fe268854a1","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=0ca6e087-7a61-4986-a269-98fe268854a1"},{"name":"integration-test-group","groupId":"5f1ec851-f8da-4f90-ab42-8dc50a9f99d8","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=5f1ec851-f8da-4f90-ab42-8dc50a9f99d8"},{"name":"jira-users","groupId":"2513da2e-08cf-4415-9bcd-cbbd32fa227d","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=2513da2e-08cf-4415-9bcd-cbbd32fa227d"},{"name":"atlassian-addons-admin","groupId":"90b9ffb1-ed26-4b5e-af59-8f684900ce83","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=90b9ffb1-ed26-4b5e-af59-8f684900ce83"},{"name":"jira-servicemanagement-users-airbyteio","groupId":"aab99a7c-3ce3-4123-b580-e4e00460754d","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=aab99a7c-3ce3-4123-b580-e4e00460754d"},{"name":"jira-admins-airbyteio","groupId":"2d55cbe0-4cab-46a4-853e-ec31162ab9a3","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=2d55cbe0-4cab-46a4-853e-ec31162ab9a3"},{"name":"jira-administrators","groupId":"58582f33-a5a6-43b9-92a6-ff0bbacb49ae","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=58582f33-a5a6-43b9-92a6-ff0bbacb49ae"},{"name":"Test group 1","groupId":"bda1faf1-1a1a-42d1-82e4-a428c8b8f67c","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=bda1faf1-1a1a-42d1-82e4-a428c8b8f67c"},{"name":"site-admins","groupId":"76dad095-fc1a-467a-88b4-fde534220985","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=76dad095-fc1a-467a-88b4-fde534220985"},{"name":"Test group 10","groupId":"e9f74708-e33c-4158-919d-6457f50c6e74","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=e9f74708-e33c-4158-919d-6457f50c6e74"}],"name":"Jira Service Desk","defaultGroups":["jira-servicemanagement-users-airbyteio"],"defaultGroupsDetails":[{"name":"jira-servicemanagement-users-airbyteio","groupId":"aab99a7c-3ce3-4123-b580-e4e00460754d","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=aab99a7c-3ce3-4123-b580-e4e00460754d"}],"selectedByDefault":false,"defined":true,"numberOfSeats":35000,"remainingSeats":34995,"userCount":5,"userCountDescription":"agents","hasUnlimitedSeats":false,"platform":false},"emitted_at":1685635876760} -{"stream":"application_roles","data":{"key":"jira-software","groups":["jira-users","Test group 1","Test group 0","system-administrators","atlassian-addons-admin","jira-servicemanagement-users-airbyteio","jira-admins-airbyteio","site-admins","jira-administrators","jira-software-users","integration-test-group","Test group 10","administrators"],"groupDetails":[{"name":"administrators","groupId":"0ca6e087-7a61-4986-a269-98fe268854a1","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=0ca6e087-7a61-4986-a269-98fe268854a1"},{"name":"jira-users","groupId":"2513da2e-08cf-4415-9bcd-cbbd32fa227d","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=2513da2e-08cf-4415-9bcd-cbbd32fa227d"},{"name":"jira-administrators","groupId":"58582f33-a5a6-43b9-92a6-ff0bbacb49ae","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=58582f33-a5a6-43b9-92a6-ff0bbacb49ae"},{"name":"Test group 1","groupId":"bda1faf1-1a1a-42d1-82e4-a428c8b8f67c","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=bda1faf1-1a1a-42d1-82e4-a428c8b8f67c"},{"name":"system-administrators","groupId":"ed0ab3a1-afa4-4ff5-a878-fc90c1574818","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=ed0ab3a1-afa4-4ff5-a878-fc90c1574818"},{"name":"site-admins","groupId":"76dad095-fc1a-467a-88b4-fde534220985","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=76dad095-fc1a-467a-88b4-fde534220985"},{"name":"jira-software-users","groupId":"4452b254-035d-469a-a422-1f4666dce50e","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=4452b254-035d-469a-a422-1f4666dce50e"},{"name":"Test group 0","groupId":"ee8d15d1-6462-406a-b0a6-8065b7e4cdd7","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=ee8d15d1-6462-406a-b0a6-8065b7e4cdd7"},{"name":"integration-test-group","groupId":"5f1ec851-f8da-4f90-ab42-8dc50a9f99d8","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=5f1ec851-f8da-4f90-ab42-8dc50a9f99d8"},{"name":"atlassian-addons-admin","groupId":"90b9ffb1-ed26-4b5e-af59-8f684900ce83","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=90b9ffb1-ed26-4b5e-af59-8f684900ce83"},{"name":"jira-admins-airbyteio","groupId":"2d55cbe0-4cab-46a4-853e-ec31162ab9a3","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=2d55cbe0-4cab-46a4-853e-ec31162ab9a3"},{"name":"jira-servicemanagement-users-airbyteio","groupId":"aab99a7c-3ce3-4123-b580-e4e00460754d","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=aab99a7c-3ce3-4123-b580-e4e00460754d"},{"name":"Test group 10","groupId":"e9f74708-e33c-4158-919d-6457f50c6e74","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=e9f74708-e33c-4158-919d-6457f50c6e74"}],"name":"Jira Software","defaultGroups":["jira-software-users"],"defaultGroupsDetails":[{"name":"jira-software-users","groupId":"4452b254-035d-469a-a422-1f4666dce50e","self":"https://airbyteio.atlassian.net/rest/api/3/group?groupId=4452b254-035d-469a-a422-1f4666dce50e"}],"selectedByDefault":false,"defined":true,"numberOfSeats":10,"remainingSeats":5,"userCount":5,"userCountDescription":"users","hasUnlimitedSeats":false,"platform":false},"emitted_at":1685635876761} -{"stream": "avatars", "data": {"id": "10300", "isSystemAvatar": true, "isSelected": false, "isDeletable": false, "urls": {"16x16": "/secure/useravatar?size=xsmall&avatarId=10300", "24x24": "/secure/useravatar?size=small&avatarId=10300", "32x32": "/secure/useravatar?size=medium&avatarId=10300", "48x48": "/secure/useravatar?avatarId=10300"}}, "emitted_at": 1685112891085} -{"stream": "avatars", "data": {"id": "10303", "isSystemAvatar": true, "isSelected": false, "isDeletable": false, "urls": {"16x16": "/secure/useravatar?size=xsmall&avatarId=10303", "24x24": "/secure/useravatar?size=small&avatarId=10303", "32x32": "/secure/useravatar?size=medium&avatarId=10303", "48x48": "/secure/useravatar?avatarId=10303"}}, "emitted_at": 1685112891086} -{"stream": "avatars", "data": {"id": "10304", "isSystemAvatar": true, "isSelected": false, "isDeletable": false, "urls": {"16x16": "/secure/useravatar?size=xsmall&avatarId=10304", "24x24": "/secure/useravatar?size=small&avatarId=10304", "32x32": "/secure/useravatar?size=medium&avatarId=10304", "48x48": "/secure/useravatar?avatarId=10304"}}, "emitted_at": 1685112891086} -{"stream": "boards", "data": {"id": 1, "self": "https://airbyteio.atlassian.net/rest/agile/1.0/board/1", "name": "IT board", "type": "scrum", "location": {"projectId": 10000, "displayName": "integration-tests (IT)", "projectName": "integration-tests", "projectKey": "IT", "projectTypeKey": "software", "avatarURI": "https://airbyteio.atlassian.net/rest/api/2/universal_avatar/view/type/project/avatar/10424?size=small", "name": "integration-tests (IT)"}, "projectId": "10000", "projectKey": "IT"}, "emitted_at": 1685112893105} -{"stream": "boards", "data": {"id": 17, "self": "https://airbyteio.atlassian.net/rest/agile/1.0/board/17", "name": "TESTKEY13 board", "type": "scrum", "location": {"projectId": 10016, "displayName": "Test project 13 (TESTKEY13)", "projectName": "Test project 13", "projectKey": "TESTKEY13", "projectTypeKey": "software", "avatarURI": "https://airbyteio.atlassian.net/rest/api/2/universal_avatar/view/type/project/avatar/10425?size=small", "name": "Test project 13 (TESTKEY13)"}, "projectId": "10016", "projectKey": "TESTKEY13"}, "emitted_at": 1685112893106} -{"stream": "board_issues", "data": {"expand": "operations,versionedRepresentations,editmeta,changelog,renderedFields", "id": "10012", "self": "https://airbyteio.atlassian.net/rest/agile/1.0/issue/10012", "key": "IT-6", "fields": {"updated": "2022-05-17T04:26:21.613-0700", "created": "2021-03-11T06:14:18.085-0800"}, "boardId": 1, "created": "2021-03-11T06:14:18.085-0800", "updated": "2022-05-17T04:26:21.613-0700"}, "emitted_at": 1685112894918} -{"stream": "board_issues", "data": {"expand": "operations,versionedRepresentations,editmeta,changelog,renderedFields", "id": "10019", "self": "https://airbyteio.atlassian.net/rest/agile/1.0/issue/10019", "key": "IT-9", "fields": {"updated": "2023-04-05T04:57:18.118-0700", "created": "2021-03-11T06:14:24.791-0800"}, "boardId": 1, "created": "2021-03-11T06:14:24.791-0800", "updated": "2023-04-05T04:57:18.118-0700"}, "emitted_at": 1685112894919} -{"stream": "board_issues", "data": {"expand": "operations,versionedRepresentations,editmeta,changelog,renderedFields", "id": "10000", "self": "https://airbyteio.atlassian.net/rest/agile/1.0/issue/10000", "key": "IT-1", "fields": {"updated": "2022-05-17T04:26:28.885-0700", "created": "2020-12-07T06:12:17.863-0800"}, "boardId": 1, "created": "2020-12-07T06:12:17.863-0800", "updated": "2022-05-17T04:26:28.885-0700"}, "emitted_at": 1685112894919} -{"stream": "dashboards", "data": {"id": "10000", "isFavourite": false, "name": "Default dashboard", "popularity": 0, "self": "https://airbyteio.atlassian.net/rest/api/3/dashboard/10000", "sharePermissions": [{"id": 10000, "type": "global"}], "editPermissions": [], "view": "/jira/dashboards/10000", "isWritable": true, "systemDashboard": true}, "emitted_at": 1685112896976} -{"stream": "dashboards", "data": {"description": "A dashboard to help auditors identify sample of issues to check.", "id": "10002", "isFavourite": true, "name": "Test dashboard 1", "owner": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "displayName": "integration test", "active": true, "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}}, "popularity": 1, "rank": 0, "self": "https://airbyteio.atlassian.net/rest/api/3/dashboard/10002", "sharePermissions": [], "editPermissions": [], "view": "/jira/dashboards/10002", "isWritable": true, "systemDashboard": false}, "emitted_at": 1685112896977} -{"stream": "dashboards", "data": {"description": "A dashboard to help auditors identify sample of issues to check.", "id": "10011", "isFavourite": true, "name": "Test dashboard 10", "owner": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "displayName": "integration test", "active": true, "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}}, "popularity": 1, "rank": 9, "self": "https://airbyteio.atlassian.net/rest/api/3/dashboard/10011", "sharePermissions": [], "editPermissions": [], "view": "/jira/dashboards/10011", "isWritable": true, "systemDashboard": false}, "emitted_at": 1685112896977} -{"stream": "filters", "data": {"expand": "description,owner,jql,viewUrl,searchUrl,favourite,favouritedCount,sharePermissions,editPermissions,isWritable,subscriptions", "self": "https://airbyteio.atlassian.net/rest/api/3/filter/10003", "id": "10003", "name": "Filter for EX board", "owner": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "jql": "project = EX ORDER BY Rank ASC", "viewUrl": "https://airbyteio.atlassian.net/issues/?filter=10003", "searchUrl": "https://airbyteio.atlassian.net/rest/api/3/search?jql=project+%3D+EX+ORDER+BY+Rank+ASC", "favourite": false, "favouritedCount": 0, "sharePermissions": [{"id": 10004, "type": "project", "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10003", "id": "10003", "key": "EX", "assigneeType": "PROJECT_LEAD", "name": "Example", "roles": {}, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10406", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10406?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10406?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10406?size=medium"}, "projectTypeKey": "software", "simplified": false, "style": "classic", "properties": {}}}], "isWritable": true, "subscriptions": []}, "emitted_at": 1685112898295} -{"stream": "filters", "data": {"expand": "description,owner,jql,viewUrl,searchUrl,favourite,favouritedCount,sharePermissions,editPermissions,isWritable,subscriptions", "self": "https://airbyteio.atlassian.net/rest/api/3/filter/10000", "id": "10000", "name": "Filter for IT board", "owner": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "jql": "project = IT ORDER BY Rank ASC", "viewUrl": "https://airbyteio.atlassian.net/issues/?filter=10000", "searchUrl": "https://airbyteio.atlassian.net/rest/api/3/search?jql=project+%3D+IT+ORDER+BY+Rank+ASC", "favourite": false, "favouritedCount": 0, "sharePermissions": [{"id": 10058, "type": "group", "group": {"name": "Test group 2", "groupId": "5ddb26f1-2d31-414a-ac34-b2d6de38805d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=5ddb26f1-2d31-414a-ac34-b2d6de38805d"}}, {"id": 10059, "type": "group", "group": {"name": "Test group 0", "groupId": "ee8d15d1-6462-406a-b0a6-8065b7e4cdd7", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=ee8d15d1-6462-406a-b0a6-8065b7e4cdd7"}}, {"id": 10057, "type": "project", "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10000", "id": "10000", "key": "IT", "assigneeType": "PROJECT_LEAD", "name": "integration-tests", "roles": {}, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10004", "id": "10004", "name": "Test category 2", "description": "Test Project Category 2"}, "projectTypeKey": "software", "simplified": false, "style": "classic", "properties": {}}}], "isWritable": true, "subscriptions": []}, "emitted_at": 1685112898296} -{"stream": "filters", "data": {"expand": "description,owner,jql,viewUrl,searchUrl,favourite,favouritedCount,sharePermissions,editPermissions,isWritable,subscriptions", "self": "https://airbyteio.atlassian.net/rest/api/3/filter/10001", "id": "10001", "name": "Filter for P2 board", "owner": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "jql": "project = P2 ORDER BY Rank ASC", "viewUrl": "https://airbyteio.atlassian.net/issues/?filter=10001", "searchUrl": "https://airbyteio.atlassian.net/rest/api/3/search?jql=project+%3D+P2+ORDER+BY+Rank+ASC", "favourite": false, "favouritedCount": 0, "sharePermissions": [{"id": 10063, "type": "group", "group": {"name": "Test group 0", "groupId": "ee8d15d1-6462-406a-b0a6-8065b7e4cdd7", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=ee8d15d1-6462-406a-b0a6-8065b7e4cdd7"}}, {"id": 10064, "type": "group", "group": {"name": "Test group 1", "groupId": "bda1faf1-1a1a-42d1-82e4-a428c8b8f67c", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=bda1faf1-1a1a-42d1-82e4-a428c8b8f67c"}}, {"id": 10062, "type": "project", "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10001", "id": "10001", "key": "P2", "assigneeType": "PROJECT_LEAD", "name": "project-2", "roles": {}, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10411", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10411?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10411?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10411?size=medium"}, "projectTypeKey": "software", "simplified": false, "style": "classic", "properties": {}}}], "isWritable": true, "subscriptions": []}, "emitted_at": 1685112898296} -{"stream": "filter_sharing", "data": {"id": 10004, "type": "project", "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10003", "id": "10003", "key": "EX", "assigneeType": "PROJECT_LEAD", "name": "Example", "roles": {}, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10406", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10406?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10406?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10406?size=medium"}, "projectTypeKey": "software", "simplified": false, "style": "classic", "properties": {}}}, "emitted_at": 1685112899652} -{"stream": "filter_sharing", "data": {"id": 10058, "type": "group", "group": {"name": "Test group 2", "groupId": "5ddb26f1-2d31-414a-ac34-b2d6de38805d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=5ddb26f1-2d31-414a-ac34-b2d6de38805d"}}, "emitted_at": 1685112899952} -{"stream": "filter_sharing", "data": {"id": 10059, "type": "group", "group": {"name": "Test group 0", "groupId": "ee8d15d1-6462-406a-b0a6-8065b7e4cdd7", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=ee8d15d1-6462-406a-b0a6-8065b7e4cdd7"}}, "emitted_at": 1685112899952} -{"stream": "groups", "data": {"name": "Test group 17", "groupId": "022bc924-ac57-442d-80c9-df042b73ad87"}, "emitted_at": 1685112927902} -{"stream": "groups", "data": {"name": "administrators", "groupId": "0ca6e087-7a61-4986-a269-98fe268854a1"}, "emitted_at": 1685112927903} -{"stream": "groups", "data": {"name": "jira-users", "groupId": "2513da2e-08cf-4415-9bcd-cbbd32fa227d"}, "emitted_at": 1685112927903} -{"stream": "issues", "data": {"expand": "operations,customfield_10030.properties,versionedRepresentations,editmeta,changelog,customfield_10029.properties,customfield_10010.requestTypePractice,renderedFields", "id": "10627", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10627", "key": "TESTKEY13-1", "fields": {"statuscategorychangedate": "2022-06-09T16:29:32.382-0700", "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}, "timespent": null, "customfield_10030": null, "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10016", "id": "10016", "key": "TESTKEY13", "name": "Test project 13", "projectTypeKey": "software", "simplified": false, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10000", "id": "10000", "description": "Category 1", "name": "Category 1"}}, "fixVersions": [{"self": "https://airbyteio.atlassian.net/rest/api/3/version/10066", "id": "10066", "description": "An excellent version", "name": "New Version 1", "archived": false, "released": true, "releaseDate": "2010-07-06"}], "aggregatetimespent": null, "resolution": null, "customfield_10225": null, "customfield_10226": null, "customfield_10227": null, "customfield_10029": null, "customfield_10228": null, "resolutiondate": null, "workratio": -1, "lastViewed": "2023-07-05T12:49:36.121-0700", "watches": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/TESTKEY13-1/watchers", "watchCount": 1, "isWatching": true}, "issuerestriction": {"issuerestrictions": {}, "shouldDisplay": false}, "customfield_10181": null, "created": "2022-06-09T16:29:31.871-0700", "customfield_10020": [{"id": 2, "name": "IT Sprint 1", "state": "active", "boardId": 1, "goal": "Deliver results", "startDate": "2022-05-17T11:25:59.072Z", "endDate": "2022-05-31T11:25:00.000Z"}], "customfield_10021": null, "customfield_10022": null, "customfield_10220": null, "customfield_10221": null, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "customfield_10023": null, "customfield_10222": null, "customfield_10024": null, "customfield_10025": null, "customfield_10223": null, "labels": ["test"], "customfield_10026": 3.0, "customfield_10224": null, "customfield_10016": null, "customfield_10214": null, "customfield_10017": "dark_orange", "customfield_10215": null, "customfield_10216": null, "customfield_10018": {"hasEpicLinkFieldDependency": false, "showField": false, "nonEditableReason": {"reason": "PLUGIN_LICENSE_ERROR", "message": "The Parent Link is only available to Jira Premium users."}}, "customfield_10217": [], "customfield_10019": "0|i0077b:", "customfield_10218": null, "timeestimate": null, "aggregatetimeoriginalestimate": null, "versions": [], "customfield_10219": null, "issuelinks": [], "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updated": "2023-04-04T04:36:21.195-0700", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "components": [{"self": "https://airbyteio.atlassian.net/rest/api/3/component/10065", "id": "10065", "name": "Component 0", "description": "This is a Jira component"}], "timeoriginalestimate": null, "description": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Test issue"}]}]}, "customfield_10010": null, "customfield_10011": "EPIC NAME TEXT", "customfield_10210": null, "customfield_10012": {"self": "https://airbyteio.atlassian.net/rest/api/3/customFieldOption/10016", "value": "To Do", "id": "10016"}, "customfield_10013": "ghx-label-14", "customfield_10211": null, "customfield_10014": null, "customfield_10212": null, "customfield_10015": null, "customfield_10213": null, "timetracking": {}, "customfield_10005": null, "customfield_10006": null, "customfield_10007": null, "security": null, "customfield_10008": null, "attachment": [], "aggregatetimeestimate": null, "customfield_10009": "2022-12-09T00:00:00.000-0800", "customfield_10209": null, "summary": "My Summary", "creator": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "subtasks": [], "reporter": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "aggregateprogress": {"progress": 0, "total": 0}, "customfield_10001": null, "customfield_10002": null, "customfield_10047": null, "customfield_10003": [{"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}], "customfield_10004": null, "environment": null, "duedate": null, "progress": {"progress": 0, "total": 0}, "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10627/comment", "maxResults": 0, "total": 0, "startAt": 0}, "votes": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/TESTKEY13-1/votes", "votes": 0, "hasVoted": false}, "worklog": {"startAt": 0, "maxResults": 20, "total": 0, "worklogs": []}}, "projectId": "10016", "projectKey": "TESTKEY13", "created": "2022-06-09T16:29:31.871-0700", "updated": "2023-04-04T04:36:21.195-0700"}, "emitted_at": 1690193760166} -{"stream": "issues", "data": {"expand": "operations,customfield_10030.properties,versionedRepresentations,editmeta,changelog,customfield_10029.properties,customfield_10010.requestTypePractice,renderedFields", "id": "10625", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625", "key": "IT-25", "fields": {"statuscategorychangedate": "2022-05-17T04:06:24.675-0700", "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}, "timespent": null, "customfield_10030": null, "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10000", "id": "10000", "key": "IT", "name": "integration-tests", "projectTypeKey": "software", "simplified": false, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10004", "id": "10004", "description": "Test Project Category 2", "name": "Test category 2"}}, "fixVersions": [], "aggregatetimespent": null, "resolution": null, "customfield_10225": null, "customfield_10226": null, "customfield_10029": null, "customfield_10227": null, "customfield_10228": null, "resolutiondate": null, "workratio": -1, "issuerestriction": {"issuerestrictions": {}, "shouldDisplay": false}, "lastViewed": null, "watches": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-25/watchers", "watchCount": 1, "isWatching": true}, "customfield_10181": null, "created": "2022-05-17T04:06:24.048-0700", "customfield_10020": null, "customfield_10021": null, "customfield_10220": null, "customfield_10022": null, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "customfield_10221": null, "customfield_10023": null, "customfield_10024": null, "customfield_10222": null, "customfield_10223": null, "customfield_10025": null, "customfield_10224": null, "labels": [], "customfield_10026": null, "customfield_10016": null, "customfield_10214": null, "customfield_10017": "dark_yellow", "customfield_10215": null, "customfield_10216": null, "customfield_10018": {"hasEpicLinkFieldDependency": false, "showField": false, "nonEditableReason": {"reason": "PLUGIN_LICENSE_ERROR", "message": "The Parent Link is only available to Jira Premium users."}}, "customfield_10019": "0|i0076v:", "customfield_10217": [], "customfield_10218": null, "aggregatetimeoriginalestimate": null, "timeestimate": null, "versions": [], "customfield_10219": null, "issuelinks": [{"id": "10263", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLink/10263", "type": {"id": "10001", "name": "Cloners", "inward": "is cloned by", "outward": "clones", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10001"}, "inwardIssue": {"id": "10626", "key": "IT-26", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626", "fields": {"summary": "CLONE - Aggregate issues", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}}}}], "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updated": "2022-05-17T04:28:19.876-0700", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "components": [{"self": "https://airbyteio.atlassian.net/rest/api/3/component/10049", "id": "10049", "name": "Component 3", "description": "This is a Jira component"}], "timeoriginalestimate": null, "description": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Implement OAUth"}]}]}, "customfield_10010": null, "customfield_10011": "Test 2", "customfield_10210": null, "customfield_10012": {"self": "https://airbyteio.atlassian.net/rest/api/3/customFieldOption/10016", "value": "To Do", "id": "10016"}, "customfield_10211": null, "customfield_10013": "ghx-label-2", "customfield_10212": null, "customfield_10014": null, "customfield_10015": null, "customfield_10213": null, "timetracking": {}, "customfield_10005": null, "customfield_10006": null, "customfield_10007": null, "security": null, "customfield_10008": null, "attachment": [], "customfield_10009": null, "aggregatetimeestimate": null, "customfield_10209": null, "summary": "Aggregate issues", "creator": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "subtasks": [], "reporter": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "aggregateprogress": {"progress": 0, "total": 0}, "customfield_10001": null, "customfield_10002": null, "customfield_10047": null, "customfield_10003": null, "customfield_10004": null, "environment": null, "duedate": null, "progress": {"progress": 0, "total": 0}, "votes": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-25/votes", "votes": 0, "hasVoted": false}, "comment": {"comments": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625/comment/10755", "id": "10755", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "body": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Closed"}]}]}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2022-05-17T04:06:55.076-0700", "updated": "2022-05-17T04:06:55.076-0700", "jsdPublic": true}], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625/comment", "maxResults": 1, "total": 1, "startAt": 0}, "worklog": {"startAt": 0, "maxResults": 20, "total": 0, "worklogs": []}}, "projectId": "10000", "projectKey": "IT", "created": "2022-05-17T04:06:24.048-0700", "updated": "2022-05-17T04:28:19.876-0700"}, "emitted_at": 1690193759636} -{"stream": "issue_comments", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625/comment/10755", "id": "10755", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "body": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Closed"}]}]}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2022-05-17T04:06:55.076-0700", "updated": "2022-05-17T04:06:55.076-0700", "jsdPublic": true}, "emitted_at": 1685112937324} -{"stream": "issue_comments", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10075/comment/10521", "id": "10521", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "body": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque eget venenatis elit. Duis eu justo eget augue iaculis fermentum. Sed semper quam laoreet nisi egestas at posuere augue semper.", "type": "text"}]}]}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-14T14:32:43.099-0700", "updated": "2021-04-14T14:32:43.099-0700", "jsdPublic": true}, "emitted_at": 1685112937947} -{"stream": "issue_comments", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10075/comment/10639", "id": "10639", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "body": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "Linked related issue!", "type": "text"}]}]}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T00:08:48.998-0700", "updated": "2021-04-15T00:08:48.998-0700", "jsdPublic": true}, "emitted_at": 1685112937947} -{"stream": "issue_fields", "data": {"id": "statuscategorychangedate", "key": "statuscategorychangedate", "name": "Status Category Changed", "custom": false, "orderable": false, "navigable": true, "searchable": true, "clauseNames": ["statusCategoryChangedDate"], "schema": {"type": "datetime", "system": "statuscategorychangedate"}}, "emitted_at": 1685112945817} -{"stream": "issue_fields", "data": {"id": "parent", "key": "parent", "name": "Parent", "custom": false, "orderable": false, "navigable": true, "searchable": false, "clauseNames": ["parent"]}, "emitted_at": 1685112945818} -{"stream": "issue_fields", "data": {"id": "issuetype", "key": "issuetype", "name": "Issue Type", "custom": false, "orderable": true, "navigable": true, "searchable": true, "clauseNames": ["issuetype", "type"], "schema": {"type": "issuetype", "system": "issuetype"}}, "emitted_at": 1685112945818} -{"stream": "issue_field_configurations", "data": {"id": 10000, "name": "Default Field Configuration", "description": "The default field configuration", "isDefault": true}, "emitted_at": 1685112946538} -{"stream": "issue_field_configurations", "data": {"id": 10001, "name": "Field Config 1", "description": "Field Config 1 test"}, "emitted_at": 1685112946539} -{"stream": "issue_field_configurations", "data": {"id": 10002, "name": "Field Config 2", "description": "Field Config 2 test"}, "emitted_at": 1685112946539} -{"stream": "issue_custom_field_contexts", "data": {"id": "10130", "name": "Default Configuration Scheme for Account", "description": "Default configuration scheme generated by Jira", "isGlobalContext": true, "isAnyIssueType": true}, "emitted_at": 1685112947151} -{"stream": "issue_custom_field_contexts", "data": {"id": "10382", "name": "Default Configuration Scheme for Time to first response", "description": "Default configuration scheme generated by Jira", "isGlobalContext": true, "isAnyIssueType": true}, "emitted_at": 1685112947419} -{"stream": "issue_custom_field_contexts", "data": {"id": "10384", "name": "Default Configuration Scheme for Time to done", "description": "Default configuration scheme generated by Jira", "isGlobalContext": true, "isAnyIssueType": true}, "emitted_at": 1685112947714} -{"stream": "issue_link_types", "data": {"id": "10000", "name": "Blocks", "inward": "is blocked by", "outward": "blocks", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10000"}, "emitted_at": 1685112962538} -{"stream": "issue_link_types", "data": {"id": "10001", "name": "Cloners", "inward": "is cloned by", "outward": "clones", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10001"}, "emitted_at": 1685112962539} -{"stream": "issue_link_types", "data": {"id": "10002", "name": "Duplicate", "inward": "is duplicated by", "outward": "duplicates", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10002"}, "emitted_at": 1685112962539} -{"stream": "issue_navigator_settings", "data": {"label": "Issue Type", "value": "issuetype"}, "emitted_at": 1685112963331} -{"stream": "issue_navigator_settings", "data": {"label": "Key", "value": "issuekey"}, "emitted_at": 1685112963332} -{"stream": "issue_navigator_settings", "data": {"label": "Summary", "value": "summary"}, "emitted_at": 1685112963332} -{"stream": "issue_notification_schemes", "data": {"expand": "notificationSchemeEvents,user,group,projectRole,field,all", "id": 10000, "self": "https://airbyteio.atlassian.net/rest/api/3/notificationscheme/10000", "name": "Default Notification Scheme"}, "emitted_at": 1685112963963} -{"stream": "issue_notification_schemes", "data": {"expand": "notificationSchemeEvents,user,group,projectRole,field,all", "id": 10001, "self": "https://airbyteio.atlassian.net/rest/api/3/notificationscheme/10001", "name": "Notification Scheme 1", "description": "Notification Scheme 1 test"}, "emitted_at": 1685112963964} -{"stream": "issue_notification_schemes", "data": {"expand": "notificationSchemeEvents,user,group,projectRole,field,all", "id": 10002, "self": "https://airbyteio.atlassian.net/rest/api/3/notificationscheme/10002", "name": "Notification Scheme 2", "description": "Notification Scheme 2 test"}, "emitted_at": 1685112963964} -{"stream": "issue_priorities", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/1", "statusColor": "#d04437", "description": "This problem will block progress.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/highest.svg", "name": "Highest", "id": "1", "isDefault": false}, "emitted_at": 1685112964545} -{"stream": "issue_priorities", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/2", "statusColor": "#f15C75", "description": "Serious problem that could block progress.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/high.svg", "name": "High", "id": "2", "isDefault": false}, "emitted_at": 1685112964546} -{"stream": "issue_priorities", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/3", "statusColor": "#f79232", "description": "Has the potential to affect progress.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/medium.svg", "name": "Medium", "id": "3", "isDefault": false}, "emitted_at": 1685112964547} -{"stream": "issue_properties", "data": {"key": "myProperty0", "value": {"owner": "admin", "weight": 100}}, "emitted_at": 1685112969564} -{"stream": "issue_properties", "data": {"key": "myProperty1", "value": {"owner": "admin", "weight": 100}}, "emitted_at": 1685112969876} -{"stream": "issue_properties", "data": {"key": "myProperty2", "value": {"owner": "admin", "weight": 100}}, "emitted_at": 1685112970164} -{"stream": "issue_remote_links", "data": {"id": 10046, "self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-24/remotelink/10046", "globalId": "system=https://www.mycompany.com/support&id=1", "application": {"type": "com.acme.tracker", "name": "My Acme Tracker"}, "relationship": "causes", "object": {"url": "https://www.mycompany.com/support?id=1", "title": "TSTSUP-111", "summary": "Customer support issue", "icon": {"url16x16": "https://www.mycompany.com/support/ticket.png", "title": "Support Ticket"}, "status": {"resolved": true, "icon": {"url16x16": "https://www.mycompany.com/support/resolved.png", "title": "Case Closed", "link": "https://www.mycompany.com/support?id=1&details=closed"}}}}, "emitted_at": 1685112990750} -{"stream": "issue_remote_links", "data": {"id": 10047, "self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-23/remotelink/10047", "globalId": "system=https://www.mycompany.com/support&id=1", "application": {"type": "com.acme.tracker", "name": "My Acme Tracker"}, "relationship": "causes", "object": {"url": "https://www.mycompany.com/support?id=1", "title": "TSTSUP-111", "summary": "Customer support issue", "icon": {"url16x16": "https://www.mycompany.com/support/ticket.png", "title": "Support Ticket"}, "status": {"resolved": true, "icon": {"url16x16": "https://www.mycompany.com/support/resolved.png", "title": "Case Closed", "link": "https://www.mycompany.com/support?id=1&details=closed"}}}}, "emitted_at": 1685112991027} -{"stream": "issue_remote_links", "data": {"id": 10048, "self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-22/remotelink/10048", "globalId": "system=https://www.mycompany.com/support&id=1", "application": {"type": "com.acme.tracker", "name": "My Acme Tracker"}, "relationship": "causes", "object": {"url": "https://www.mycompany.com/support?id=1", "title": "TSTSUP-111", "summary": "Customer support issue", "icon": {"url16x16": "https://www.mycompany.com/support/ticket.png", "title": "Support Ticket"}, "status": {"resolved": true, "icon": {"url16x16": "https://www.mycompany.com/support/resolved.png", "title": "Case Closed", "link": "https://www.mycompany.com/support?id=1&details=closed"}}}}, "emitted_at": 1685112991394} -{"stream": "issue_resolutions", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/resolution/10000", "id": "10000", "description": "Work has been completed on this issue.", "name": "Done", "isDefault": false}, "emitted_at": 1685112998451} -{"stream": "issue_resolutions", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/resolution/10001", "id": "10001", "description": "This issue won't be actioned.", "name": "Won't Do", "isDefault": false}, "emitted_at": 1685112998452} -{"stream": "issue_resolutions", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/resolution/10002", "id": "10002", "description": "The problem is a duplicate of an existing issue.", "name": "Duplicate", "isDefault": false}, "emitted_at": 1685112998452} -{"stream": "issue_security_schemes", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuesecurityschemes/10001", "id": 10001, "name": "Security scheme 2", "description": "Security scheme 2"}, "emitted_at": 1685112999194} -{"stream": "issue_security_schemes", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuesecurityschemes/10000", "id": 10000, "name": "Security scheme 1", "description": "Security scheme 1", "defaultSecurityLevelId": 10002}, "emitted_at": 1685112999195} -{"stream": "issue_security_schemes", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuesecurityschemes/10002", "id": 10002, "name": "Security scheme 3", "description": "Security scheme 3 test", "defaultSecurityLevelId": 10003}, "emitted_at": 1685112999195} -{"stream": "issue_type_schemes", "data": {"id": "10000", "name": "Default Issue Type Scheme", "description": "Default issue type scheme is the list of global issue types. All newly created issue types will automatically be added to this scheme.", "isDefault": true}, "emitted_at": 1685112999823} -{"stream": "issue_type_schemes", "data": {"id": "10126", "name": "IT: Scrum Issue Type Scheme", "defaultIssueTypeId": "10001"}, "emitted_at": 1685112999824} -{"stream": "issue_type_schemes", "data": {"id": "10128", "name": "P2: Scrum Issue Type Scheme", "defaultIssueTypeId": "10001"}, "emitted_at": 1685112999824} -{"stream": "issue_type_screen_schemes", "data": {"id": "1", "name": "Default Issue Type Screen Scheme", "description": "The default issue type screen scheme"}, "emitted_at": 1685113000857} -{"stream": "issue_type_screen_schemes", "data": {"id": "10000", "name": "IT: Scrum Issue Type Screen Scheme", "description": ""}, "emitted_at": 1685113000858} -{"stream": "issue_type_screen_schemes", "data": {"id": "10001", "name": "P2: Scrum Issue Type Screen Scheme", "description": ""}, "emitted_at": 1685113000859} -{"stream": "issue_votes", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-2/votes", "votes": 1, "hasVoted": true, "voters": [{"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}]}, "emitted_at": 1686151528794} -{"stream": "issue_votes", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-1/votes", "votes": 1, "hasVoted": true, "voters": [{"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}]}, "emitted_at": 1686151529331} -{"stream": "issue_votes", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/TESTKEY13-1/votes", "votes": 0, "hasVoted": false, "voters": []}, "emitted_at": 1686151530871} -{"stream": "issue_watchers", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-2/watchers", "isWatching": true, "watchCount": 1, "watchers": [{"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}]}, "emitted_at": 1686152488275} -{"stream": "issue_watchers", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-1/watchers", "isWatching": true, "watchCount": 1, "watchers": [{"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}]}, "emitted_at": 1686152488658} -{"stream": "issue_watchers", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/TESTKEY13-1/watchers", "isWatching": true, "watchCount": 1, "watchers": [{"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}]}, "emitted_at": 1686152489518} -{"stream": "issue_worklogs", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/worklog/11820", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "comment": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "time-tracking"}]}]}, "created": "2023-04-05T05:08:50.033-0700", "updated": "2023-04-05T05:08:50.033-0700", "started": "2023-04-05T01:00:00.000-0700", "timeSpent": "1d", "timeSpentSeconds": 28800, "id": "11820", "issueId": "10626"}, "emitted_at": 1686153001542} -{"stream": "issue_worklogs", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11709", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 1", "type": "text"}]}]}, "created": "2021-04-15T11:39:47.215-0700", "updated": "2021-04-15T11:39:47.215-0700", "started": "2021-04-14T18:48:52.747-0700", "timeSpent": "37m", "timeSpentSeconds": 2220, "id": "11709", "issueId": "10080"}, "emitted_at": 1686153002291} -{"stream": "issue_worklogs", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10075/worklog/11711", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 0", "type": "text"}]}]}, "created": "2021-04-15T11:39:48.447-0700", "updated": "2021-04-15T11:39:48.447-0700", "started": "2021-04-14T18:48:52.747-0700", "timeSpent": "1h 28m", "timeSpentSeconds": 5280, "id": "11711", "issueId": "10075"}, "emitted_at": 1686153002726} -{"stream": "issue_worklogs", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10075/worklog/11714", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 3", "type": "text"}]}]}, "created": "2021-04-15T11:39:50.205-0700", "updated": "2021-04-15T11:39:50.205-0700", "started": "2021-04-14T18:48:52.747-0700", "timeSpent": "1h 23m", "timeSpentSeconds": 4980, "id": "11714", "issueId": "10075"}, "emitted_at": 1686153002728} -{"stream": "jira_settings", "data": {"id": "jira.issuenav.criteria.autoupdate", "key": "jira.issuenav.criteria.autoupdate", "value": "true", "name": "Auto Update Criteria", "desc": "Turn on to update search results automatically", "type": "boolean"}, "emitted_at": 1685113041270} -{"stream": "jira_settings", "data": {"id": "jira.clone.prefix", "key": "jira.clone.prefix", "value": "CLONE -", "name": "The prefix added to the Summary field of cloned issues", "type": "string"}, "emitted_at": 1685113041271} -{"stream": "jira_settings", "data": {"id": "jira.date.picker.java.format", "key": "jira.date.picker.java.format", "value": "d/MMM/yy", "name": "Date Picker Format (Java)", "desc": "This part is only for the Java (server side) generated dates. Note that this should correspond to the javascript date picker format (jira.date.picker.javascript.format) setting.", "type": "string"}, "emitted_at": 1685113041271} -{"stream": "labels", "data": {"label": "Label10"}, "emitted_at": 1685113042126} -{"stream": "labels", "data": {"label": "Label3"}, "emitted_at": 1685113042127} -{"stream": "labels", "data": {"label": "Label4"}, "emitted_at": 1685113042127} -{"stream": "permissions", "data": {"key": "ADD_COMMENTS", "name": "Add Comments", "type": "PROJECT", "description": "Ability to comment on issues."}, "emitted_at": 1685113042849} -{"stream": "permissions", "data": {"key": "ADMINISTER", "name": "Administer Jira", "type": "GLOBAL", "description": "Create and administer projects, issue types, fields, workflows, and schemes for all projects. Users with this permission can perform most administration tasks, except: managing users, importing data, and editing system email settings."}, "emitted_at": 1685113042850} -{"stream": "permissions", "data": {"key": "ADMINISTER_PROJECTS", "name": "Administer Projects", "type": "PROJECT", "description": "Ability to administer a project in Jira."}, "emitted_at": 1685113042851} -{"stream": "permission_schemes", "data": {"expand": "permissions,user,group,projectRole,field,all", "id": 10056, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056", "name": "CAW software permission scheme", "description": "The permission scheme for Jira Software Free. In Free, any registered user can access and administer this project.", "permissions": [{"id": 14200, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14200", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ADD_COMMENTS"}, {"id": 14201, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14201", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ADMINISTER_PROJECTS"}, {"id": 14202, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14202", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ASSIGNABLE_USER"}, {"id": 14203, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14203", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ASSIGN_ISSUES"}, {"id": 14204, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14204", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "BROWSE_PROJECTS"}, {"id": 14205, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14205", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CLOSE_ISSUES"}, {"id": 14206, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14206", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CREATE_ATTACHMENTS"}, {"id": 14207, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14207", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CREATE_ISSUES"}, {"id": 14208, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14208", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_ATTACHMENTS"}, {"id": 14209, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14209", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_COMMENTS"}, {"id": 14210, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14210", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_WORKLOGS"}, {"id": 14211, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14211", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ISSUES"}, {"id": 14212, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14212", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_ATTACHMENTS"}, {"id": 14213, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14213", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_COMMENTS"}, {"id": 14214, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14214", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_WORKLOGS"}, {"id": 14215, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14215", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ALL_COMMENTS"}, {"id": 14216, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14216", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ALL_WORKLOGS"}, {"id": 14217, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14217", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ISSUES"}, {"id": 14218, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14218", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_OWN_COMMENTS"}, {"id": 14219, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14219", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_OWN_WORKLOGS"}, {"id": 14220, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14220", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "LINK_ISSUES"}, {"id": 14221, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14221", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MANAGE_SPRINTS_PERMISSION"}, {"id": 14222, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14222", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MANAGE_WATCHERS"}, {"id": 14223, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14223", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MODIFY_REPORTER"}, {"id": 14224, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14224", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MOVE_ISSUES"}, {"id": 14225, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14225", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "RESOLVE_ISSUES"}, {"id": 14226, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14226", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SCHEDULE_ISSUES"}, {"id": 14227, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14227", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "TRANSITION_ISSUES"}, {"id": 14228, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14228", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_DEV_TOOLS"}, {"id": 14229, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14229", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_READONLY_WORKFLOW"}, {"id": 14230, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14230", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_VOTERS_AND_WATCHERS"}, {"id": 14231, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14231", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "WORK_ON_ISSUES"}, {"id": 14232, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14232", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__log-work-for-others"}, {"id": 14233, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14233", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__set-billable-hours"}, {"id": 14234, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14234", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__view-all-worklogs"}, {"id": 14235, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14235", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__view-issue-hours"}, {"id": 14236, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14236", "holder": {"type": "applicationRole"}, "permission": "ADMINISTER_PROJECTS"}, {"id": 14237, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14237", "holder": {"type": "applicationRole"}, "permission": "BROWSE_PROJECTS"}, {"id": 14238, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14238", "holder": {"type": "applicationRole"}, "permission": "MANAGE_SPRINTS_PERMISSION"}, {"id": 14239, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14239", "holder": {"type": "applicationRole"}, "permission": "VIEW_DEV_TOOLS"}, {"id": 14240, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14240", "holder": {"type": "applicationRole"}, "permission": "VIEW_READONLY_WORKFLOW"}, {"id": 14241, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14241", "holder": {"type": "applicationRole"}, "permission": "ASSIGNABLE_USER"}, {"id": 14242, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14242", "holder": {"type": "applicationRole"}, "permission": "CLOSE_ISSUES"}, {"id": 14243, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14243", "holder": {"type": "applicationRole"}, "permission": "CREATE_ISSUES"}, {"id": 14244, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14244", "holder": {"type": "applicationRole"}, "permission": "DELETE_ISSUES"}, {"id": 14245, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14245", "holder": {"type": "applicationRole"}, "permission": "EDIT_ISSUES"}, {"id": 14246, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14246", "holder": {"type": "applicationRole"}, "permission": "LINK_ISSUES"}, {"id": 14247, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14247", "holder": {"type": "applicationRole"}, "permission": "MODIFY_REPORTER"}, {"id": 14248, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14248", "holder": {"type": "applicationRole"}, "permission": "MOVE_ISSUES"}, {"id": 14249, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14249", "holder": {"type": "applicationRole"}, "permission": "RESOLVE_ISSUES"}, {"id": 14250, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14250", "holder": {"type": "applicationRole"}, "permission": "SCHEDULE_ISSUES"}, {"id": 14251, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14251", "holder": {"type": "applicationRole"}, "permission": "TRANSITION_ISSUES"}, {"id": 14252, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14252", "holder": {"type": "applicationRole"}, "permission": "MANAGE_WATCHERS"}, {"id": 14253, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14253", "holder": {"type": "applicationRole"}, "permission": "VIEW_VOTERS_AND_WATCHERS"}, {"id": 14254, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14254", "holder": {"type": "applicationRole"}, "permission": "ADD_COMMENTS"}, {"id": 14255, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14255", "holder": {"type": "applicationRole"}, "permission": "DELETE_ALL_COMMENTS"}, {"id": 14256, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14256", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_COMMENTS"}, {"id": 14257, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14257", "holder": {"type": "applicationRole"}, "permission": "EDIT_ALL_COMMENTS"}, {"id": 14258, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14258", "holder": {"type": "applicationRole"}, "permission": "EDIT_OWN_COMMENTS"}, {"id": 14259, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14259", "holder": {"type": "applicationRole"}, "permission": "CREATE_ATTACHMENTS"}, {"id": 14260, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14260", "holder": {"type": "applicationRole"}, "permission": "DELETE_ALL_ATTACHMENTS"}, {"id": 14261, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14261", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_ATTACHMENTS"}, {"id": 14262, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14262", "holder": {"type": "applicationRole"}, "permission": "DELETE_ALL_WORKLOGS"}, {"id": 14263, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14263", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_WORKLOGS"}, {"id": 14264, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14264", "holder": {"type": "applicationRole"}, "permission": "EDIT_ALL_WORKLOGS"}, {"id": 14265, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14265", "holder": {"type": "applicationRole"}, "permission": "EDIT_OWN_WORKLOGS"}, {"id": 14266, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14266", "holder": {"type": "applicationRole"}, "permission": "WORK_ON_ISSUES"}, {"id": 14267, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14267", "holder": {"type": "applicationRole"}, "permission": "ASSIGN_ISSUES"}, {"id": 14404, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14404", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SET_ISSUE_SECURITY"}, {"id": 14481, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14481", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_AGGREGATED_DATA"}, {"id": 14542, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14542", "holder": {"type": "applicationRole"}, "permission": "VIEW_AGGREGATED_DATA"}, {"id": 14714, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14714", "holder": {"type": "applicationRole"}, "permission": "VIEW_PROJECTS"}, {"id": 14715, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14715", "holder": {"type": "applicationRole"}, "permission": "VIEW_ISSUES"}, {"id": 14836, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14836", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_PROJECTS"}, {"id": 14837, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14837", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_ISSUES"}, {"id": 15117, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/15117", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SERVICEDESK_AGENT"}]}, "emitted_at": 1685113044013} -{"stream": "permission_schemes", "data": {"expand": "permissions,user,group,projectRole,field,all", "id": 10055, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055", "name": "CLK software permission scheme", "description": "The permission scheme for Jira Software Free. In Free, any registered user can access and administer this project.", "permissions": [{"id": 14132, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14132", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ADD_COMMENTS"}, {"id": 14133, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14133", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ADMINISTER_PROJECTS"}, {"id": 14134, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14134", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ASSIGNABLE_USER"}, {"id": 14135, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14135", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ASSIGN_ISSUES"}, {"id": 14136, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14136", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "BROWSE_PROJECTS"}, {"id": 14137, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14137", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CLOSE_ISSUES"}, {"id": 14138, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14138", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CREATE_ATTACHMENTS"}, {"id": 14139, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14139", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CREATE_ISSUES"}, {"id": 14140, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14140", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_ATTACHMENTS"}, {"id": 14141, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14141", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_COMMENTS"}, {"id": 14142, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14142", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_WORKLOGS"}, {"id": 14143, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14143", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ISSUES"}, {"id": 14144, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14144", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_ATTACHMENTS"}, {"id": 14145, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14145", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_COMMENTS"}, {"id": 14146, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14146", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_WORKLOGS"}, {"id": 14147, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14147", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ALL_COMMENTS"}, {"id": 14148, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14148", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ALL_WORKLOGS"}, {"id": 14149, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14149", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ISSUES"}, {"id": 14150, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14150", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_OWN_COMMENTS"}, {"id": 14151, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14151", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_OWN_WORKLOGS"}, {"id": 14152, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14152", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "LINK_ISSUES"}, {"id": 14153, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14153", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MANAGE_SPRINTS_PERMISSION"}, {"id": 14154, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14154", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MANAGE_WATCHERS"}, {"id": 14155, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14155", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MODIFY_REPORTER"}, {"id": 14156, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14156", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MOVE_ISSUES"}, {"id": 14157, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14157", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "RESOLVE_ISSUES"}, {"id": 14158, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14158", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SCHEDULE_ISSUES"}, {"id": 14159, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14159", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "TRANSITION_ISSUES"}, {"id": 14160, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14160", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_DEV_TOOLS"}, {"id": 14161, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14161", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_READONLY_WORKFLOW"}, {"id": 14162, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14162", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_VOTERS_AND_WATCHERS"}, {"id": 14163, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14163", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "WORK_ON_ISSUES"}, {"id": 14164, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14164", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__log-work-for-others"}, {"id": 14165, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14165", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__set-billable-hours"}, {"id": 14166, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14166", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__view-all-worklogs"}, {"id": 14167, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14167", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__view-issue-hours"}, {"id": 14168, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14168", "holder": {"type": "applicationRole"}, "permission": "ADMINISTER_PROJECTS"}, {"id": 14169, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14169", "holder": {"type": "applicationRole"}, "permission": "BROWSE_PROJECTS"}, {"id": 14170, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14170", "holder": {"type": "applicationRole"}, "permission": "MANAGE_SPRINTS_PERMISSION"}, {"id": 14171, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14171", "holder": {"type": "applicationRole"}, "permission": "VIEW_DEV_TOOLS"}, {"id": 14172, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14172", "holder": {"type": "applicationRole"}, "permission": "VIEW_READONLY_WORKFLOW"}, {"id": 14173, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14173", "holder": {"type": "applicationRole"}, "permission": "ASSIGNABLE_USER"}, {"id": 14174, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14174", "holder": {"type": "applicationRole"}, "permission": "CLOSE_ISSUES"}, {"id": 14175, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14175", "holder": {"type": "applicationRole"}, "permission": "CREATE_ISSUES"}, {"id": 14176, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14176", "holder": {"type": "applicationRole"}, "permission": "DELETE_ISSUES"}, {"id": 14177, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14177", "holder": {"type": "applicationRole"}, "permission": "EDIT_ISSUES"}, {"id": 14178, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14178", "holder": {"type": "applicationRole"}, "permission": "LINK_ISSUES"}, {"id": 14179, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14179", "holder": {"type": "applicationRole"}, "permission": "MODIFY_REPORTER"}, {"id": 14180, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14180", "holder": {"type": "applicationRole"}, "permission": "MOVE_ISSUES"}, {"id": 14181, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14181", "holder": {"type": "applicationRole"}, "permission": "RESOLVE_ISSUES"}, {"id": 14182, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14182", "holder": {"type": "applicationRole"}, "permission": "SCHEDULE_ISSUES"}, {"id": 14183, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14183", "holder": {"type": "applicationRole"}, "permission": "TRANSITION_ISSUES"}, {"id": 14184, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14184", "holder": {"type": "applicationRole"}, "permission": "MANAGE_WATCHERS"}, {"id": 14185, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14185", "holder": {"type": "applicationRole"}, "permission": "VIEW_VOTERS_AND_WATCHERS"}, {"id": 14186, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14186", "holder": {"type": "applicationRole"}, "permission": "ADD_COMMENTS"}, {"id": 14187, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14187", "holder": {"type": "applicationRole"}, "permission": "DELETE_ALL_COMMENTS"}, {"id": 14188, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14188", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_COMMENTS"}, {"id": 14189, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14189", "holder": {"type": "applicationRole"}, "permission": "EDIT_ALL_COMMENTS"}, {"id": 14190, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14190", "holder": {"type": "applicationRole"}, "permission": "EDIT_OWN_COMMENTS"}, {"id": 14191, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14191", "holder": {"type": "applicationRole"}, "permission": "CREATE_ATTACHMENTS"}, {"id": 14192, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14192", "holder": {"type": "applicationRole"}, "permission": "DELETE_ALL_ATTACHMENTS"}, {"id": 14193, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14193", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_ATTACHMENTS"}, {"id": 14194, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14194", "holder": {"type": "applicationRole"}, "permission": "DELETE_ALL_WORKLOGS"}, {"id": 14195, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14195", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_WORKLOGS"}, {"id": 14196, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14196", "holder": {"type": "applicationRole"}, "permission": "EDIT_ALL_WORKLOGS"}, {"id": 14197, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14197", "holder": {"type": "applicationRole"}, "permission": "EDIT_OWN_WORKLOGS"}, {"id": 14198, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14198", "holder": {"type": "applicationRole"}, "permission": "WORK_ON_ISSUES"}, {"id": 14199, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14199", "holder": {"type": "applicationRole"}, "permission": "ASSIGN_ISSUES"}, {"id": 14405, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14405", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SET_ISSUE_SECURITY"}, {"id": 14482, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14482", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_AGGREGATED_DATA"}, {"id": 14543, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14543", "holder": {"type": "applicationRole"}, "permission": "VIEW_AGGREGATED_DATA"}, {"id": 14712, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14712", "holder": {"type": "applicationRole"}, "permission": "VIEW_PROJECTS"}, {"id": 14713, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14713", "holder": {"type": "applicationRole"}, "permission": "VIEW_ISSUES"}, {"id": 14834, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14834", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_PROJECTS"}, {"id": 14835, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14835", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_ISSUES"}, {"id": 15118, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/15118", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SERVICEDESK_AGENT"}]}, "emitted_at": 1685113044015} -{"stream": "permission_schemes", "data": {"expand": "permissions,user,group,projectRole,field,all", "id": 0, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0", "name": "Default Permission Scheme", "description": "This is the default Permission Scheme. Any new projects that are created will be assigned this scheme.", "permissions": [{"id": 10004, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10004", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "ADMINISTER_PROJECTS"}, {"id": 10005, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10005", "holder": {"type": "applicationRole"}, "permission": "BROWSE_PROJECTS"}, {"id": 10006, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10006", "holder": {"type": "applicationRole"}, "permission": "CREATE_ISSUES"}, {"id": 10007, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10007", "holder": {"type": "applicationRole"}, "permission": "ADD_COMMENTS"}, {"id": 10008, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10008", "holder": {"type": "applicationRole"}, "permission": "CREATE_ATTACHMENTS"}, {"id": 10009, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10009", "holder": {"type": "applicationRole"}, "permission": "ASSIGN_ISSUES"}, {"id": 10010, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10010", "holder": {"type": "applicationRole"}, "permission": "ASSIGNABLE_USER"}, {"id": 10011, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10011", "holder": {"type": "applicationRole"}, "permission": "RESOLVE_ISSUES"}, {"id": 10012, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10012", "holder": {"type": "applicationRole"}, "permission": "LINK_ISSUES"}, {"id": 10013, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10013", "holder": {"type": "applicationRole"}, "permission": "EDIT_ISSUES"}, {"id": 10014, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10014", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "DELETE_ISSUES"}, {"id": 10015, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10015", "holder": {"type": "applicationRole"}, "permission": "CLOSE_ISSUES"}, {"id": 10016, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10016", "holder": {"type": "applicationRole"}, "permission": "MOVE_ISSUES"}, {"id": 10017, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10017", "holder": {"type": "applicationRole"}, "permission": "SCHEDULE_ISSUES"}, {"id": 10018, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10018", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "MODIFY_REPORTER"}, {"id": 10019, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10019", "holder": {"type": "applicationRole"}, "permission": "WORK_ON_ISSUES"}, {"id": 10020, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10020", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "DELETE_ALL_WORKLOGS"}, {"id": 10021, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10021", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_WORKLOGS"}, {"id": 10022, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10022", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "EDIT_ALL_WORKLOGS"}, {"id": 10023, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10023", "holder": {"type": "applicationRole"}, "permission": "EDIT_OWN_WORKLOGS"}, {"id": 10024, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10024", "holder": {"type": "applicationRole"}, "permission": "VIEW_VOTERS_AND_WATCHERS"}, {"id": 10025, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10025", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "MANAGE_WATCHERS"}, {"id": 10026, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10026", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "EDIT_ALL_COMMENTS"}, {"id": 10027, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10027", "holder": {"type": "applicationRole"}, "permission": "EDIT_OWN_COMMENTS"}, {"id": 10028, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10028", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "DELETE_ALL_COMMENTS"}, {"id": 10029, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10029", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_COMMENTS"}, {"id": 10030, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10030", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "DELETE_ALL_ATTACHMENTS"}, {"id": 10031, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10031", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_ATTACHMENTS"}, {"id": 10033, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10033", "holder": {"type": "applicationRole"}, "permission": "VIEW_DEV_TOOLS"}, {"id": 10200, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10200", "holder": {"type": "applicationRole"}, "permission": "VIEW_READONLY_WORKFLOW"}, {"id": 10300, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10300", "holder": {"type": "applicationRole"}, "permission": "TRANSITION_ISSUES"}, {"id": 10301, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10301", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ADD_COMMENTS"}, {"id": 10302, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10302", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ADMINISTER_PROJECTS"}, {"id": 10303, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10303", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ASSIGNABLE_USER"}, {"id": 10304, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10304", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ASSIGN_ISSUES"}, {"id": 10305, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10305", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "BROWSE_PROJECTS"}, {"id": 10306, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10306", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CLOSE_ISSUES"}, {"id": 10307, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10307", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CREATE_ATTACHMENTS"}, {"id": 10308, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10308", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CREATE_ISSUES"}, {"id": 10309, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10309", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_ATTACHMENTS"}, {"id": 10310, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10310", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_COMMENTS"}, {"id": 10311, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10311", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_WORKLOGS"}, {"id": 10312, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10312", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ISSUES"}, {"id": 10313, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10313", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_ATTACHMENTS"}, {"id": 10314, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10314", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_COMMENTS"}, {"id": 10315, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10315", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_WORKLOGS"}, {"id": 10316, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10316", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ALL_COMMENTS"}, {"id": 10317, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10317", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ALL_WORKLOGS"}, {"id": 10318, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10318", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ISSUES"}, {"id": 10319, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10319", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_OWN_COMMENTS"}, {"id": 10320, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10320", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_OWN_WORKLOGS"}, {"id": 10321, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10321", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "LINK_ISSUES"}, {"id": 10322, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10322", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MANAGE_SPRINTS_PERMISSION"}, {"id": 10323, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10323", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MANAGE_WATCHERS"}, {"id": 10324, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10324", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MODIFY_REPORTER"}, {"id": 10325, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10325", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MOVE_ISSUES"}, {"id": 10326, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10326", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "RESOLVE_ISSUES"}, {"id": 10327, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10327", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SCHEDULE_ISSUES"}, {"id": 10328, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10328", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SET_ISSUE_SECURITY"}, {"id": 10329, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10329", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "TRANSITION_ISSUES"}, {"id": 10330, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10330", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_DEV_TOOLS"}, {"id": 10331, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10331", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_READONLY_WORKFLOW"}, {"id": 10332, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10332", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_VOTERS_AND_WATCHERS"}, {"id": 10333, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10333", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "WORK_ON_ISSUES"}, {"id": 10464, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10464", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__log-work-for-others"}, {"id": 10465, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10465", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__set-billable-hours"}, {"id": 10466, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10466", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__view-all-worklogs"}, {"id": 10467, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10467", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__view-issue-hours"}, {"id": 14538, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/14538", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_AGGREGATED_DATA"}, {"id": 14599, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/14599", "holder": {"type": "applicationRole"}, "permission": "VIEW_AGGREGATED_DATA"}, {"id": 14600, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/14600", "holder": {"type": "applicationRole"}, "permission": "VIEW_PROJECTS"}, {"id": 14601, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/14601", "holder": {"type": "applicationRole"}, "permission": "VIEW_ISSUES"}, {"id": 14722, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/14722", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_PROJECTS"}, {"id": 14723, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/14723", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_ISSUES"}, {"id": 15119, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/15119", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SERVICEDESK_AGENT"}]}, "emitted_at": 1685113044017} -{"stream": "projects", "data": {"expand": "description,lead,issueTypes,url,projectKeys,permissions,insight", "self": "https://airbyteio.atlassian.net/rest/api/3/project/10000", "id": "10000", "key": "IT", "description": "Test", "lead": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "name": "integration-tests", "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10004", "id": "10004", "name": "Test category 2", "description": "Test Project Category 2"}, "projectTypeKey": "software", "simplified": false, "style": "classic", "isPrivate": false, "properties": {}}, "emitted_at": 1686153767914} -{"stream": "projects", "data": {"expand": "description,lead,issueTypes,url,projectKeys,permissions,insight", "self": "https://airbyteio.atlassian.net/rest/api/3/project/10016", "id": "10016", "key": "TESTKEY13", "description": "Test project 13 description", "lead": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "name": "Test project 13", "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10000", "id": "10000", "name": "Category 1", "description": "Category 1"}, "projectTypeKey": "software", "simplified": false, "style": "classic", "isPrivate": false, "properties": {}}, "emitted_at": 1686153767915} -{"stream": "project_avatars", "data": {"id": "10400", "isSystemAvatar": true, "isSelected": false, "isDeletable": false, "urls": {"16x16": "/secure/viewavatar?size=xsmall&avatarId=10400&avatarType=project", "24x24": "/secure/viewavatar?size=small&avatarId=10400&avatarType=project", "32x32": "/secure/viewavatar?size=medium&avatarId=10400&avatarType=project", "48x48": "/secure/viewavatar?avatarId=10400&avatarType=project"}}, "emitted_at": 1685113044864} -{"stream": "project_avatars", "data": {"id": "10401", "isSystemAvatar": true, "isSelected": false, "isDeletable": false, "urls": {"16x16": "/secure/viewavatar?size=xsmall&avatarId=10401&avatarType=project", "24x24": "/secure/viewavatar?size=small&avatarId=10401&avatarType=project", "32x32": "/secure/viewavatar?size=medium&avatarId=10401&avatarType=project", "48x48": "/secure/viewavatar?avatarId=10401&avatarType=project"}}, "emitted_at": 1685113044864} -{"stream": "project_avatars", "data": {"id": "10402", "isSystemAvatar": true, "isSelected": false, "isDeletable": false, "urls": {"16x16": "/secure/viewavatar?size=xsmall&avatarId=10402&avatarType=project", "24x24": "/secure/viewavatar?size=small&avatarId=10402&avatarType=project", "32x32": "/secure/viewavatar?size=medium&avatarId=10402&avatarType=project", "48x48": "/secure/viewavatar?avatarId=10402&avatarType=project"}}, "emitted_at": 1685113044865} -{"stream": "project_categories", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10000", "id": "10000", "description": "Category 1", "name": "Category 1"}, "emitted_at": 1685113046251} -{"stream": "project_categories", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10001", "id": "10001", "description": "Category 2", "name": "Category 2"}, "emitted_at": 1685113046252} -{"stream": "project_categories", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10002", "id": "10002", "description": "Test Project Category 0", "name": "Test category 0"}, "emitted_at": 1685113046253} -{"stream": "project_components", "data": {"componentBean": {"self": "https://airbyteio.atlassian.net/rest/api/3/component/10047", "id": "10047", "name": "Component 0", "description": "This is a Jira component", "lead": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "assigneeType": "PROJECT_LEAD", "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "realAssigneeType": "PROJECT_LEAD", "realAssignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "isAssigneeTypeValid": true, "project": "IT", "projectId": 10000}, "issueCount": 0, "realAssignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "isAssigneeTypeValid": true, "realAssigneeType": "PROJECT_LEAD", "description": "This is a Jira component", "name": "Component 0", "id": "10047", "projectId": 10000, "project": "IT", "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "assigneeType": "PROJECT_LEAD", "lead": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "self": "https://airbyteio.atlassian.net/rest/api/3/component/10047"}, "emitted_at": 1685113046991} -{"stream": "project_components", "data": {"componentBean": {"self": "https://airbyteio.atlassian.net/rest/api/3/component/10000", "id": "10000", "name": "Component 1", "description": "Component 1", "lead": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "assigneeType": "PROJECT_DEFAULT", "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "realAssigneeType": "PROJECT_DEFAULT", "realAssignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "isAssigneeTypeValid": true, "project": "IT", "projectId": 10000}, "issueCount": 0, "realAssignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "isAssigneeTypeValid": true, "realAssigneeType": "PROJECT_DEFAULT", "description": "Component 1", "name": "Component 1", "id": "10000", "projectId": 10000, "project": "IT", "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "assigneeType": "PROJECT_DEFAULT", "lead": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "self": "https://airbyteio.atlassian.net/rest/api/3/component/10000"}, "emitted_at": 1685113046992} -{"stream": "project_components", "data": {"componentBean": {"self": "https://airbyteio.atlassian.net/rest/api/3/component/10048", "id": "10048", "name": "Component 2", "description": "This is a Jira component", "lead": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "assigneeType": "PROJECT_LEAD", "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "realAssigneeType": "PROJECT_LEAD", "realAssignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "isAssigneeTypeValid": true, "project": "IT", "projectId": 10000}, "issueCount": 0, "realAssignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "isAssigneeTypeValid": true, "realAssigneeType": "PROJECT_LEAD", "description": "This is a Jira component", "name": "Component 2", "id": "10048", "projectId": 10000, "project": "IT", "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "assigneeType": "PROJECT_LEAD", "lead": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "self": "https://airbyteio.atlassian.net/rest/api/3/component/10048"}, "emitted_at": 1685113046993} -{"stream": "project_email", "data": {"emailAddress": "jira@airbyteio.atlassian.net", "projectId": "10000"}, "emitted_at": 1685113048530} -{"stream": "project_email", "data": {"emailAddress": "jira@airbyteio.atlassian.net", "projectId": "10016"}, "emitted_at": 1685113049055} -{"stream": "project_types", "data": {"key": "product_discovery", "formattedKey": "Product Discovery", "descriptionI18nKey": "jira.project.type.polaris.description", "icon": "PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iaXNvLTg4NTktMSI/Pg0KPCEtLSBHZW5lcmF0b3I6IEFkb2JlIElsbHVzdHJhdG9yIDE4LjEuMSwgU1ZHIEV4cG9ydCBQbHVnLUluIC4gU1ZHIFZlcnNpb246IDYuMDAgQnVpbGQgMCkgIC0tPg0KPHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJMYXllcl8xIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCINCgkgdmlld0JveD0iMCAwIDMwMCAzMDAiIHN0eWxlPSJlbmFibGUtYmFja2dyb3VuZDpuZXcgMCAwIDMwMCAzMDA7IiB4bWw6c3BhY2U9InByZXNlcnZlIj4NCjxnIGlkPSJMYXllcl8yIj4NCgk8cGF0aCBzdHlsZT0iZmlsbDojRjc5MjMyOyIgZD0iTTE1MCwwQzY2LjY2NywwLDAsNjYuNjY3LDAsMTUwczY2LjY2NywxNTAsMTUwLDE1MHMxNTAtNjYuNjY3LDE1MC0xNTBTMjMzLjMzMywwLDE1MCwweg0KCQkgTTEzNi42NjcsMTc4LjMzM0wxMjUsMTkwbC00MS42NjctNDBMOTUsMTM4LjMzM2wzMC0zMEwxMzYuNjY3LDEyMGwtMzAsMzBMMTM2LjY2NywxNzguMzMzeiBNMjA1LDE2MS42NjdsLTMwLDMwTDE2My4zMzMsMTgwDQoJCWwzMC0zMGwtMzAtMzBMMTc1LDEwOC4zMzNMMjE2LjY2NywxNTBMMjA1LDE2MS42Njd6Ii8+DQo8L2c+DQo8Zz4NCgk8cG9seWdvbiBzdHlsZT0iZmlsbDojRkZGRkZGOyIgcG9pbnRzPSIxNzUsMTkxLjY2NyAyMDUsMTYxLjY2NyAyMTYuNjY3LDE1MCAxNzUsMTA4LjMzMyAxNjMuMzMzLDEyMCAxOTMuMzMzLDE1MCAxNjMuMzMzLDE4MCAJIi8+DQoJPHBvbHlnb24gc3R5bGU9ImZpbGw6I0ZGRkZGRjsiIHBvaW50cz0iMTI1LDEwOC4zMzMgOTUsMTM4LjMzMyA4My4zMzMsMTUwIDEyNSwxOTAgMTM2LjY2NywxNzguMzMzIDEwNi42NjcsMTUwIDEzNi42NjcsMTIwIAkiLz4NCjwvZz4NCjwvc3ZnPg0K", "color": "#F5A623"}, "emitted_at": 1685113053300} -{"stream": "project_types", "data": {"key": "software", "formattedKey": "Software", "descriptionI18nKey": "jira.project.type.software.description", "icon": "PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iaXNvLTg4NTktMSI/Pg0KPCEtLSBHZW5lcmF0b3I6IEFkb2JlIElsbHVzdHJhdG9yIDE4LjEuMSwgU1ZHIEV4cG9ydCBQbHVnLUluIC4gU1ZHIFZlcnNpb246IDYuMDAgQnVpbGQgMCkgIC0tPg0KPHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJMYXllcl8xIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCINCgkgdmlld0JveD0iMCAwIDMwMCAzMDAiIHN0eWxlPSJlbmFibGUtYmFja2dyb3VuZDpuZXcgMCAwIDMwMCAzMDA7IiB4bWw6c3BhY2U9InByZXNlcnZlIj4NCjxnIGlkPSJMYXllcl8yIj4NCgk8cGF0aCBzdHlsZT0iZmlsbDojRjc5MjMyOyIgZD0iTTE1MCwwQzY2LjY2NywwLDAsNjYuNjY3LDAsMTUwczY2LjY2NywxNTAsMTUwLDE1MHMxNTAtNjYuNjY3LDE1MC0xNTBTMjMzLjMzMywwLDE1MCwweg0KCQkgTTEzNi42NjcsMTc4LjMzM0wxMjUsMTkwbC00MS42NjctNDBMOTUsMTM4LjMzM2wzMC0zMEwxMzYuNjY3LDEyMGwtMzAsMzBMMTM2LjY2NywxNzguMzMzeiBNMjA1LDE2MS42NjdsLTMwLDMwTDE2My4zMzMsMTgwDQoJCWwzMC0zMGwtMzAtMzBMMTc1LDEwOC4zMzNMMjE2LjY2NywxNTBMMjA1LDE2MS42Njd6Ii8+DQo8L2c+DQo8Zz4NCgk8cG9seWdvbiBzdHlsZT0iZmlsbDojRkZGRkZGOyIgcG9pbnRzPSIxNzUsMTkxLjY2NyAyMDUsMTYxLjY2NyAyMTYuNjY3LDE1MCAxNzUsMTA4LjMzMyAxNjMuMzMzLDEyMCAxOTMuMzMzLDE1MCAxNjMuMzMzLDE4MCAJIi8+DQoJPHBvbHlnb24gc3R5bGU9ImZpbGw6I0ZGRkZGRjsiIHBvaW50cz0iMTI1LDEwOC4zMzMgOTUsMTM4LjMzMyA4My4zMzMsMTUwIDEyNSwxOTAgMTM2LjY2NywxNzguMzMzIDEwNi42NjcsMTUwIDEzNi42NjcsMTIwIAkiLz4NCjwvZz4NCjwvc3ZnPg0K", "color": "#F5A623"}, "emitted_at": 1685113053300} -{"stream": "project_types", "data": {"key": "service_desk", "formattedKey": "Service Desk", "descriptionI18nKey": "jira.project.type.servicedesk.description.jsm", "icon": "PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iaXNvLTg4NTktMSI/Pg0KPCEtLSBHZW5lcmF0b3I6IEFkb2JlIElsbHVzdHJhdG9yIDE4LjEuMSwgU1ZHIEV4cG9ydCBQbHVnLUluIC4gU1ZHIFZlcnNpb246IDYuMDAgQnVpbGQgMCkgIC0tPg0KPHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJMYXllcl8xIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCINCgkgdmlld0JveD0iMCAwIDMwMCAzMDAiIHN0eWxlPSJlbmFibGUtYmFja2dyb3VuZDpuZXcgMCAwIDMwMCAzMDA7IiB4bWw6c3BhY2U9InByZXNlcnZlIj4NCjxnIGlkPSJMYXllcl8yIj4NCgk8Zz4NCgkJPHJlY3QgeD0iMTAwIiB5PSIxMDAiIHN0eWxlPSJmaWxsOiM2N0FCNDk7IiB3aWR0aD0iMTAwIiBoZWlnaHQ9IjY2LjY2NyIvPg0KCQk8cGF0aCBzdHlsZT0iZmlsbDojNjdBQjQ5OyIgZD0iTTE1MCwwQzY2LjY2NywwLDAsNjYuNjY3LDAsMTUwczY2LjY2NywxNTAsMTUwLDE1MHMxNTAtNjYuNjY3LDE1MC0xNTBTMjMzLjMzMywwLDE1MCwweg0KCQkJIE0yMTYuNjY3LDEwMHY2Ni42Njd2MTYuNjY3aC01MFYyMDBIMjAwdjE2LjY2N0gxMDBWMjAwaDMzLjMzM3YtMTYuNjY3aC01MHYtMTYuNjY3VjEwMFY4My4zMzNoMTMzLjMzM1YxMDB6Ii8+DQoJPC9nPg0KPC9nPg0KPHBhdGggc3R5bGU9ImZpbGw6I0ZGRkZGRjsiIGQ9Ik0yMTYuNjY3LDE4My4zMzN2LTE2LjY2N1YxMDBWODMuMzMzSDgzLjMzM1YxMDB2NjYuNjY3djE2LjY2N2g1MFYyMDBIMTAwdjE2LjY2N2gxMDBWMjAwaC0zMy4zMzMNCgl2LTE2LjY2N0gyMTYuNjY3eiBNMTAwLDE2Ni42NjdWMTAwaDEwMHY2Ni42NjdIMTAweiIvPg0KPC9zdmc+DQo=", "color": "#67AB49"}, "emitted_at": 1685113053300} -{"stream": "project_versions", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/version/10000", "id": "10000", "description": "Version 1", "name": "Version 1", "archived": false, "released": false, "startDate": "2021-02-18", "releaseDate": "2021-02-25", "overdue": true, "userStartDate": "17/Feb/21", "userReleaseDate": "24/Feb/21", "projectId": 10000}, "emitted_at": 1685113053916} -{"stream": "project_versions", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/version/10040", "id": "10040", "description": "An excellent version", "name": "New Version 0", "archived": false, "released": true, "releaseDate": "2010-07-06", "userReleaseDate": "05/Jul/10", "projectId": 10000}, "emitted_at": 1685113053917} -{"stream": "project_versions", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/version/10041", "id": "10041", "description": "An excellent version", "name": "New Version 1", "archived": false, "released": true, "releaseDate": "2010-07-06", "userReleaseDate": "05/Jul/10", "projectId": 10000}, "emitted_at": 1685113053917} -{"stream": "screens", "data": {"id": 1, "name": "Default Screen", "description": "Allows to update all system fields."}, "emitted_at": 1685113054887} -{"stream": "screens", "data": {"id": 2, "name": "Workflow Screen", "description": "This screen is used in the workflow and enables you to assign issues"}, "emitted_at": 1685113054888} -{"stream": "screens", "data": {"id": 3, "name": "Resolve Issue Screen", "description": "Allows to set resolution, change fix versions and assign an issue."}, "emitted_at": 1685113054888} -{"stream": "screen_tabs", "data": {"id": 10000, "name": "Field Tab"}, "emitted_at": 1685113056194} -{"stream": "screen_tabs", "data": {"id": 10148, "name": "Tab1"}, "emitted_at": 1685113056194} -{"stream": "screen_tabs", "data": {"id": 10149, "name": "Tab2"}, "emitted_at": 1685113056194} -{"stream": "screen_tab_fields", "data": {"id": "summary", "name": "Summary"}, "emitted_at": 1685113102761} -{"stream": "screen_tab_fields", "data": {"id": "issuetype", "name": "Issue Type"}, "emitted_at": 1685113102762} -{"stream": "screen_tab_fields", "data": {"id": "security", "name": "Security Level"}, "emitted_at": 1685113102763} -{"stream": "screen_schemes", "data": {"id": 1, "name": "Default Screen Scheme", "description": "Default Screen Scheme", "screens": {"default": 1}}, "emitted_at": 1685113161369} -{"stream": "screen_schemes", "data": {"id": 10000, "name": "IT: Scrum Default Screen Scheme", "description": "", "screens": {"default": 10000}}, "emitted_at": 1685113161370} -{"stream": "screen_schemes", "data": {"id": 10001, "name": "IT: Scrum Bug Screen Scheme", "description": "", "screens": {"default": 10001}}, "emitted_at": 1685113161371} -{"stream": "sprints", "data": {"id": 2, "self": "https://airbyteio.atlassian.net/rest/agile/1.0/sprint/2", "state": "active", "name": "IT Sprint 1", "startDate": "2022-05-17T11:25:59.072Z", "endDate": "2022-05-31T11:25:00.000Z", "originBoardId": 1, "goal": "Deliver results"}, "emitted_at": 1685113162412} -{"stream": "sprints", "data": {"id": 3, "self": "https://airbyteio.atlassian.net/rest/agile/1.0/sprint/3", "state": "future", "name": "IT Sprint 2", "startDate": "2022-05-31T11:25:59.072Z", "endDate": "2022-06-14T11:25:00.000Z", "originBoardId": 1}, "emitted_at": 1685113162413} -{"stream": "sprints", "data": {"id": 4, "self": "https://airbyteio.atlassian.net/rest/agile/1.0/sprint/4", "state": "future", "name": "IT Sprint 3", "startDate": "2022-06-14T11:25:59.072Z", "endDate": "2022-06-28T11:25:00.000Z", "originBoardId": 1}, "emitted_at": 1685113162413} -{"stream": "sprint_issues", "data": {"expand": "operations,versionedRepresentations,editmeta,changelog,renderedFields", "id": "2-10012", "self": "https://airbyteio.atlassian.net/rest/agile/1.0/issue/10012", "key": "IT-6", "fields": {"customfield_10016": null, "updated": "2022-05-17T04:26:21.613-0700", "created": "2021-03-11T06:14:18.085-0800", "status": {"self": "https://airbyteio.atlassian.net/rest/api/2/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/2/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}}, "issueId": "10012", "sprintId": 2, "created": "2021-03-11T06:14:18.085-0800", "updated": "2022-05-17T04:26:21.613-0700"}, "emitted_at": 1685113164271} -{"stream": "sprint_issues", "data": {"expand": "operations,versionedRepresentations,editmeta,changelog,renderedFields", "id": "2-10019", "self": "https://airbyteio.atlassian.net/rest/agile/1.0/issue/10019", "key": "IT-9", "fields": {"customfield_10016": null, "updated": "2023-04-05T04:57:18.118-0700", "created": "2021-03-11T06:14:24.791-0800", "status": {"self": "https://airbyteio.atlassian.net/rest/api/2/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/2/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}}, "issueId": "10019", "sprintId": 2, "created": "2021-03-11T06:14:24.791-0800", "updated": "2023-04-05T04:57:18.118-0700"}, "emitted_at": 1685113164272} -{"stream": "sprint_issues", "data": {"expand": "operations,versionedRepresentations,editmeta,changelog,renderedFields", "id": "2-10000", "self": "https://airbyteio.atlassian.net/rest/agile/1.0/issue/10000", "key": "IT-1", "fields": {"customfield_10016": null, "updated": "2022-05-17T04:26:28.885-0700", "created": "2020-12-07T06:12:17.863-0800", "status": {"self": "https://airbyteio.atlassian.net/rest/api/2/status/10001", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "Done", "id": "10001", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/2/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "customfield_10026": null}, "issueId": "10000", "sprintId": 2, "created": "2020-12-07T06:12:17.863-0800", "updated": "2022-05-17T04:26:28.885-0700"}, "emitted_at": 1685113164272} -{"stream": "time_tracking", "data": {"key": "JIRA", "name": "JIRA provided time tracking"}, "emitted_at": 1685113169776} -{"stream": "time_tracking", "data": {"key": "is.origo.jira.tempo-plugin__timetracking-provider", "name": "Tempo Timesheets"}, "emitted_at": 1685113169777} -{"stream": "users", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "locale": "en_US"}, "emitted_at": 1685113170166} -{"stream": "users", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058:f58131cb-b67d-43c7-b30d-6b58d40bd077", "accountId": "557058:f58131cb-b67d-43c7-b30d-6b58d40bd077", "accountType": "app", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "24x24": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "16x16": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "32x32": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png"}, "displayName": "Automation for Jira", "active": true}, "emitted_at": 1685113170167} -{"stream": "users", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5d53f3cbc6b9320d9ea5bdc2", "accountId": "5d53f3cbc6b9320d9ea5bdc2", "accountType": "app", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "24x24": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "16x16": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "32x32": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png"}, "displayName": "Jira Outlook", "active": true}, "emitted_at": 1685113170167} -{"stream": "users_groups_detailed", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "locale": "en_US", "groups": {"size": 30, "items": [{"name": "administrators", "groupId": "0ca6e087-7a61-4986-a269-98fe268854a1", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=0ca6e087-7a61-4986-a269-98fe268854a1"}, {"name": "confluence-users", "groupId": "38d808e9-113f-45c4-817b-099e953b687a", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=38d808e9-113f-45c4-817b-099e953b687a"}, {"name": "integration-test-group", "groupId": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=5f1ec851-f8da-4f90-ab42-8dc50a9f99d8"}, {"name": "jira-administrators", "groupId": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=58582f33-a5a6-43b9-92a6-ff0bbacb49ae"}, {"name": "jira-admins-airbyteio", "groupId": "2d55cbe0-4cab-46a4-853e-ec31162ab9a3", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=2d55cbe0-4cab-46a4-853e-ec31162ab9a3"}, {"name": "jira-servicemanagement-customers-airbyteio", "groupId": "125680d3-7e85-41ad-a662-892b6590272e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=125680d3-7e85-41ad-a662-892b6590272e"}, {"name": "jira-servicemanagement-users-airbyteio", "groupId": "aab99a7c-3ce3-4123-b580-e4e00460754d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=aab99a7c-3ce3-4123-b580-e4e00460754d"}, {"name": "jira-software-users", "groupId": "4452b254-035d-469a-a422-1f4666dce50e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=4452b254-035d-469a-a422-1f4666dce50e"}, {"name": "jira-users", "groupId": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=2513da2e-08cf-4415-9bcd-cbbd32fa227d"}, {"name": "site-admins", "groupId": "76dad095-fc1a-467a-88b4-fde534220985", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=76dad095-fc1a-467a-88b4-fde534220985"}, {"name": "Test group 0", "groupId": "ee8d15d1-6462-406a-b0a6-8065b7e4cdd7", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=ee8d15d1-6462-406a-b0a6-8065b7e4cdd7"}, {"name": "Test group 1", "groupId": "bda1faf1-1a1a-42d1-82e4-a428c8b8f67c", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=bda1faf1-1a1a-42d1-82e4-a428c8b8f67c"}, {"name": "Test group 10", "groupId": "e9f74708-e33c-4158-919d-6457f50c6e74", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=e9f74708-e33c-4158-919d-6457f50c6e74"}, {"name": "Test group 11", "groupId": "b0e6d76f-701a-4208-a88d-4478f242edde", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=b0e6d76f-701a-4208-a88d-4478f242edde"}, {"name": "Test group 12", "groupId": "dddc24a0-ef00-407e-abef-5a660b6f55cf", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=dddc24a0-ef00-407e-abef-5a660b6f55cf"}, {"name": "Test group 13", "groupId": "dbe4af74-8387-4b08-843b-86af78dd738e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=dbe4af74-8387-4b08-843b-86af78dd738e"}, {"name": "Test group 14", "groupId": "d4570a20-38d8-44cc-a63b-0924d0d0d0ff", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=d4570a20-38d8-44cc-a63b-0924d0d0d0ff"}, {"name": "Test group 15", "groupId": "87bde5c0-7231-44a7-88b5-421da2ab8052", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=87bde5c0-7231-44a7-88b5-421da2ab8052"}, {"name": "Test group 16", "groupId": "538b6aa2-bf57-402f-93c0-c2e2d68b7155", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=538b6aa2-bf57-402f-93c0-c2e2d68b7155"}, {"name": "Test group 17", "groupId": "022bc924-ac57-442d-80c9-df042b73ad87", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=022bc924-ac57-442d-80c9-df042b73ad87"}, {"name": "Test group 18", "groupId": "bbfc6fc9-96db-4e66-88f4-c55b08298272", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=bbfc6fc9-96db-4e66-88f4-c55b08298272"}, {"name": "Test group 19", "groupId": "3c4fef5d-9721-4f20-9a68-346d222de3cf", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=3c4fef5d-9721-4f20-9a68-346d222de3cf"}, {"name": "Test group 2", "groupId": "5ddb26f1-2d31-414a-ac34-b2d6de38805d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=5ddb26f1-2d31-414a-ac34-b2d6de38805d"}, {"name": "Test group 3", "groupId": "638aa1ad-8707-4d56-9361-f5959b6c4785", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=638aa1ad-8707-4d56-9361-f5959b6c4785"}, {"name": "Test group 4", "groupId": "532554e0-43be-4eca-9186-b417dcf38547", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=532554e0-43be-4eca-9186-b417dcf38547"}, {"name": "Test group 5", "groupId": "6b663734-85b6-4185-8fb2-9ac27709b3aa", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=6b663734-85b6-4185-8fb2-9ac27709b3aa"}, {"name": "Test group 6", "groupId": "2d4af5cf-cd34-4e78-9445-abc000cdd5cc", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=2d4af5cf-cd34-4e78-9445-abc000cdd5cc"}, {"name": "Test group 7", "groupId": "e8a97909-d807-4f79-8548-1f2c156ae6f0", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=e8a97909-d807-4f79-8548-1f2c156ae6f0"}, {"name": "Test group 8", "groupId": "3ee851e7-6688-495a-a6f6-737e85a23878", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=3ee851e7-6688-495a-a6f6-737e85a23878"}, {"name": "Test group 9", "groupId": "af27d0b1-4378-443f-9a6d-f878848b144a", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=af27d0b1-4378-443f-9a6d-f878848b144a"}]}, "applicationRoles": {"size": 2, "items": [{"key": "jira-servicedesk", "name": "Jira Service Desk"}, {"key": "jira-software", "name": "Jira Software"}]}, "expand": "groups,applicationRoles"}, "emitted_at": 1688723244956} -{"stream": "users_groups_detailed", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058:f58131cb-b67d-43c7-b30d-6b58d40bd077", "accountId": "557058:f58131cb-b67d-43c7-b30d-6b58d40bd077", "accountType": "app", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "24x24": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "16x16": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "32x32": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png"}, "displayName": "Automation for Jira", "active": true, "timeZone": "America/Los_Angeles", "locale": "en_US", "groups": {"size": 3, "items": [{"name": "atlassian-addons-admin", "groupId": "90b9ffb1-ed26-4b5e-af59-8f684900ce83", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=90b9ffb1-ed26-4b5e-af59-8f684900ce83"}, {"name": "jira-servicemanagement-users-airbyteio", "groupId": "aab99a7c-3ce3-4123-b580-e4e00460754d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=aab99a7c-3ce3-4123-b580-e4e00460754d"}, {"name": "jira-software-users", "groupId": "4452b254-035d-469a-a422-1f4666dce50e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=4452b254-035d-469a-a422-1f4666dce50e"}]}, "applicationRoles": {"size": 2, "items": [{"key": "jira-servicedesk", "name": "Jira Service Desk"}, {"key": "jira-software", "name": "Jira Software"}]}, "expand": "groups,applicationRoles"}, "emitted_at": 1688723245241} -{"stream": "users_groups_detailed", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5d53f3cbc6b9320d9ea5bdc2", "accountId": "5d53f3cbc6b9320d9ea5bdc2", "accountType": "app", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "24x24": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "16x16": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "32x32": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png"}, "displayName": "Jira Outlook", "active": true, "timeZone": "America/Los_Angeles", "locale": "en_US", "groups": {"size": 1, "items": [{"name": "jira-software-users", "groupId": "4452b254-035d-469a-a422-1f4666dce50e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=4452b254-035d-469a-a422-1f4666dce50e"}]}, "applicationRoles": {"size": 2, "items": [{"key": "jira-servicedesk", "name": "Jira Service Desk"}, {"key": "jira-software", "name": "Jira Software"}]}, "expand": "groups,applicationRoles"}, "emitted_at": 1688723245508} -{"stream": "workflows", "data": {"id": {"name": "Builds Workflow", "entityId": "Builds Workflow"}, "description": "Builds Workflow", "created": "1969-12-31T16:00:00.000-0800", "updated": "1969-12-31T16:00:00.000-0800"}, "emitted_at": 1688485038533} -{"stream": "workflows", "data": {"id": {"name": "classic default workflow", "entityId": "385bb764-dfb6-89a7-2e43-a25bdd0cbaf4"}, "description": "The classic JIRA default workflow", "created": "2020-12-03T23:41:38.951-0800", "updated": "2023-06-30T02:33:48.808-0700"}, "emitted_at": 1688485038534} -{"stream": "workflows", "data": {"id": {"name": "jira", "entityId": "jira"}, "description": "The default Jira workflow.", "created": "1969-12-31T16:00:00.000-0800", "updated": "1969-12-31T16:00:00.000-0800"}, "emitted_at": 1688485038534} -{"stream": "workflow_schemes", "data": {"id": 10000, "name": "classic", "description": "classic", "defaultWorkflow": "classic default workflow", "issueTypeMappings": {}, "self": "https://airbyteio.atlassian.net/rest/api/3/workflowscheme/10000"}, "emitted_at": 1685113186843} -{"stream": "workflow_schemes", "data": {"id": 10001, "name": "IT: Software Simplified Workflow Scheme", "description": "Generated by JIRA Software version 1001.0.0-SNAPSHOT. This workflow scheme is managed internally by Jira Software. Do not manually modify this workflow scheme.", "defaultWorkflow": "Software Simplified Workflow for Project IT", "issueTypeMappings": {}, "self": "https://airbyteio.atlassian.net/rest/api/3/workflowscheme/10001"}, "emitted_at": 1685113186844} -{"stream": "workflow_schemes", "data": {"id": 10002, "name": "P2: Software Simplified Workflow Scheme", "description": "Generated by JIRA Software version 1001.0.0-SNAPSHOT. This workflow scheme is managed internally by Jira Software. Do not manually modify this workflow scheme.", "defaultWorkflow": "Software Simplified Workflow for Project P2", "issueTypeMappings": {}, "self": "https://airbyteio.atlassian.net/rest/api/3/workflowscheme/10002"}, "emitted_at": 1685113186844} -{"stream": "workflow_statuses", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/1", "description": "The issue is open and ready for the assignee to start work on it.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/open.png", "name": "Open", "untranslatedName": "Open", "id": "1", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "emitted_at": 1685113187861} -{"stream": "workflow_statuses", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/3", "description": "This issue is being actively worked on at the moment by the assignee.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/inprogress.png", "name": "In Progress", "untranslatedName": "In Progress", "id": "3", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "emitted_at": 1685113187862} -{"stream": "workflow_statuses", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/4", "description": "This issue was once resolved, but the resolution was deemed incorrect. From here issues are either marked assigned or resolved.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/reopened.png", "name": "Reopened", "untranslatedName": "Reopened", "id": "4", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "emitted_at": 1685113187862} -{"stream": "workflow_status_categories", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/1", "id": 1, "key": "undefined", "colorName": "medium-gray", "name": "No Category"}, "emitted_at": 1685113188477} -{"stream": "workflow_status_categories", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}, "emitted_at": 1685113188478} -{"stream": "workflow_status_categories", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}, "emitted_at": 1685113188479} +{"stream": "application_roles", "data": {"key": "jira-servicedesk", "groups": ["jira-administrators", "jira-software-users", "jira-users", "Test group 1", "Test group 0", "atlassian-addons-admin", "integration-test-group", "jira-servicemanagement-users-airbyteio", "jira-admins-airbyteio", "site-admins", "Test group 10", "administrators"], "groupDetails": [{"name": "jira-software-users", "groupId": "4452b254-035d-469a-a422-1f4666dce50e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=4452b254-035d-469a-a422-1f4666dce50e"}, {"name": "Test group 0", "groupId": "ee8d15d1-6462-406a-b0a6-8065b7e4cdd7", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=ee8d15d1-6462-406a-b0a6-8065b7e4cdd7"}, {"name": "administrators", "groupId": "0ca6e087-7a61-4986-a269-98fe268854a1", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=0ca6e087-7a61-4986-a269-98fe268854a1"}, {"name": "integration-test-group", "groupId": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=5f1ec851-f8da-4f90-ab42-8dc50a9f99d8"}, {"name": "jira-users", "groupId": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=2513da2e-08cf-4415-9bcd-cbbd32fa227d"}, {"name": "atlassian-addons-admin", "groupId": "90b9ffb1-ed26-4b5e-af59-8f684900ce83", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=90b9ffb1-ed26-4b5e-af59-8f684900ce83"}, {"name": "jira-servicemanagement-users-airbyteio", "groupId": "aab99a7c-3ce3-4123-b580-e4e00460754d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=aab99a7c-3ce3-4123-b580-e4e00460754d"}, {"name": "jira-admins-airbyteio", "groupId": "2d55cbe0-4cab-46a4-853e-ec31162ab9a3", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=2d55cbe0-4cab-46a4-853e-ec31162ab9a3"}, {"name": "jira-administrators", "groupId": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=58582f33-a5a6-43b9-92a6-ff0bbacb49ae"}, {"name": "Test group 1", "groupId": "bda1faf1-1a1a-42d1-82e4-a428c8b8f67c", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=bda1faf1-1a1a-42d1-82e4-a428c8b8f67c"}, {"name": "site-admins", "groupId": "76dad095-fc1a-467a-88b4-fde534220985", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=76dad095-fc1a-467a-88b4-fde534220985"}, {"name": "Test group 10", "groupId": "e9f74708-e33c-4158-919d-6457f50c6e74", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=e9f74708-e33c-4158-919d-6457f50c6e74"}], "name": "Jira Service Desk", "defaultGroups": ["jira-servicemanagement-users-airbyteio"], "defaultGroupsDetails": [{"name": "jira-servicemanagement-users-airbyteio", "groupId": "aab99a7c-3ce3-4123-b580-e4e00460754d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=aab99a7c-3ce3-4123-b580-e4e00460754d"}], "selectedByDefault": false, "defined": true, "numberOfSeats": 35000, "remainingSeats": 34995, "userCount": 5, "userCountDescription": "agents", "hasUnlimitedSeats": false, "platform": false}, "emitted_at": 1697453210985} +{"stream": "application_roles", "data": {"key": "jira-software", "groups": ["jira-users", "Test group 1", "Test group 0", "system-administrators", "atlassian-addons-admin", "jira-servicemanagement-users-airbyteio", "jira-admins-airbyteio", "site-admins", "jira-administrators", "jira-software-users", "integration-test-group", "Test group 10", "administrators"], "groupDetails": [{"name": "administrators", "groupId": "0ca6e087-7a61-4986-a269-98fe268854a1", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=0ca6e087-7a61-4986-a269-98fe268854a1"}, {"name": "jira-users", "groupId": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=2513da2e-08cf-4415-9bcd-cbbd32fa227d"}, {"name": "jira-administrators", "groupId": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=58582f33-a5a6-43b9-92a6-ff0bbacb49ae"}, {"name": "Test group 1", "groupId": "bda1faf1-1a1a-42d1-82e4-a428c8b8f67c", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=bda1faf1-1a1a-42d1-82e4-a428c8b8f67c"}, {"name": "system-administrators", "groupId": "ed0ab3a1-afa4-4ff5-a878-fc90c1574818", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=ed0ab3a1-afa4-4ff5-a878-fc90c1574818"}, {"name": "site-admins", "groupId": "76dad095-fc1a-467a-88b4-fde534220985", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=76dad095-fc1a-467a-88b4-fde534220985"}, {"name": "jira-software-users", "groupId": "4452b254-035d-469a-a422-1f4666dce50e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=4452b254-035d-469a-a422-1f4666dce50e"}, {"name": "Test group 0", "groupId": "ee8d15d1-6462-406a-b0a6-8065b7e4cdd7", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=ee8d15d1-6462-406a-b0a6-8065b7e4cdd7"}, {"name": "integration-test-group", "groupId": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=5f1ec851-f8da-4f90-ab42-8dc50a9f99d8"}, {"name": "atlassian-addons-admin", "groupId": "90b9ffb1-ed26-4b5e-af59-8f684900ce83", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=90b9ffb1-ed26-4b5e-af59-8f684900ce83"}, {"name": "jira-admins-airbyteio", "groupId": "2d55cbe0-4cab-46a4-853e-ec31162ab9a3", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=2d55cbe0-4cab-46a4-853e-ec31162ab9a3"}, {"name": "jira-servicemanagement-users-airbyteio", "groupId": "aab99a7c-3ce3-4123-b580-e4e00460754d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=aab99a7c-3ce3-4123-b580-e4e00460754d"}, {"name": "Test group 10", "groupId": "e9f74708-e33c-4158-919d-6457f50c6e74", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=e9f74708-e33c-4158-919d-6457f50c6e74"}], "name": "Jira Software", "defaultGroups": ["jira-software-users"], "defaultGroupsDetails": [{"name": "jira-software-users", "groupId": "4452b254-035d-469a-a422-1f4666dce50e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=4452b254-035d-469a-a422-1f4666dce50e"}], "selectedByDefault": false, "defined": true, "numberOfSeats": 10, "remainingSeats": 5, "userCount": 5, "userCountDescription": "users", "hasUnlimitedSeats": false, "platform": false}, "emitted_at": 1697453210987} +{"stream": "avatars", "data": {"id": "10300", "isSystemAvatar": true, "isSelected": false, "isDeletable": false, "urls": {"16x16": "/secure/useravatar?size=xsmall&avatarId=10300", "24x24": "/secure/useravatar?size=small&avatarId=10300", "32x32": "/secure/useravatar?size=medium&avatarId=10300", "48x48": "/secure/useravatar?avatarId=10300"}}, "emitted_at": 1697453211799} +{"stream": "avatars", "data": {"id": "10303", "isSystemAvatar": true, "isSelected": false, "isDeletable": false, "urls": {"16x16": "/secure/useravatar?size=xsmall&avatarId=10303", "24x24": "/secure/useravatar?size=small&avatarId=10303", "32x32": "/secure/useravatar?size=medium&avatarId=10303", "48x48": "/secure/useravatar?avatarId=10303"}}, "emitted_at": 1697453211800} +{"stream": "avatars", "data": {"id": "10304", "isSystemAvatar": true, "isSelected": false, "isDeletable": false, "urls": {"16x16": "/secure/useravatar?size=xsmall&avatarId=10304", "24x24": "/secure/useravatar?size=small&avatarId=10304", "32x32": "/secure/useravatar?size=medium&avatarId=10304", "48x48": "/secure/useravatar?avatarId=10304"}}, "emitted_at": 1697453211800} +{"stream": "boards", "data": {"id": 1, "self": "https://airbyteio.atlassian.net/rest/agile/1.0/board/1", "name": "IT board", "type": "scrum", "location": {"projectId": 10000, "displayName": "integration-tests (IT)", "projectName": "integration-tests", "projectKey": "IT", "projectTypeKey": "software", "avatarURI": "https://airbyteio.atlassian.net/rest/api/2/universal_avatar/view/type/project/avatar/10424?size=small", "name": "integration-tests (IT)"}, "projectId": "10000", "projectKey": "IT"}, "emitted_at": 1697453213161} +{"stream": "boards", "data": {"id": 17, "self": "https://airbyteio.atlassian.net/rest/agile/1.0/board/17", "name": "TESTKEY13 board", "type": "scrum", "location": {"projectId": 10016, "displayName": "Test project 13 (TESTKEY13)", "projectName": "Test project 13", "projectKey": "TESTKEY13", "projectTypeKey": "software", "avatarURI": "https://airbyteio.atlassian.net/rest/api/2/universal_avatar/view/type/project/avatar/10425?size=small", "name": "Test project 13 (TESTKEY13)"}, "projectId": "10016", "projectKey": "TESTKEY13"}, "emitted_at": 1697453213162} +{"stream": "boards", "data": {"id": 58, "self": "https://airbyteio.atlassian.net/rest/agile/1.0/board/58", "name": "TTMP2 board", "type": "simple", "location": {"projectId": 10064, "displayName": "Test Team Managed Project 2 (TTMP2)", "projectName": "Test Team Managed Project 2", "projectKey": "TTMP2", "projectTypeKey": "software", "avatarURI": "https://airbyteio.atlassian.net/rest/api/2/universal_avatar/view/type/project/avatar/10412?size=small", "name": "Test Team Managed Project 2 (TTMP2)"}, "projectId": "10064", "projectKey": "TTMP2"}, "emitted_at": 1697453213464} +{"stream": "board_issues", "data": {"expand": "operations,versionedRepresentations,editmeta,changelog,renderedFields", "id": "10012", "self": "https://airbyteio.atlassian.net/rest/agile/1.0/issue/10012", "key": "IT-6", "fields": {"updated": "2023-10-12T13:30:02.307000-07:00", "created": "2021-03-11T06:14:18.085-0800"}, "boardId": 1, "created": "2021-03-11T06:14:18.085000-08:00", "updated": "2023-10-12T13:30:02.307000-07:00"}, "emitted_at": 1697453214841} +{"stream": "board_issues", "data": {"expand": "operations,versionedRepresentations,editmeta,changelog,renderedFields", "id": "10019", "self": "https://airbyteio.atlassian.net/rest/agile/1.0/issue/10019", "key": "IT-9", "fields": {"updated": "2023-04-05T04:57:18.118000-07:00", "created": "2021-03-11T06:14:24.791-0800"}, "boardId": 1, "created": "2021-03-11T06:14:24.791000-08:00", "updated": "2023-04-05T04:57:18.118000-07:00"}, "emitted_at": 1697453214846} +{"stream": "board_issues", "data": {"expand": "operations,versionedRepresentations,editmeta,changelog,renderedFields", "id": "10000", "self": "https://airbyteio.atlassian.net/rest/agile/1.0/issue/10000", "key": "IT-1", "fields": {"updated": "2022-05-17T04:26:28.885000-07:00", "created": "2020-12-07T06:12:17.863-0800"}, "boardId": 1, "created": "2020-12-07T06:12:17.863000-08:00", "updated": "2022-05-17T04:26:28.885000-07:00"}, "emitted_at": 1697453214847} +{"stream": "dashboards", "data": {"id": "10000", "isFavourite": false, "name": "Default dashboard", "popularity": 0, "self": "https://airbyteio.atlassian.net/rest/api/3/dashboard/10000", "sharePermissions": [{"id": 10000, "type": "global"}], "editPermissions": [], "view": "/jira/dashboards/10000", "isWritable": true, "systemDashboard": true}, "emitted_at": 1697453217135} +{"stream": "dashboards", "data": {"description": "A dashboard to help auditors identify sample of issues to check.", "id": "10002", "isFavourite": true, "name": "Test dashboard 1", "owner": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "displayName": "integration test", "active": true, "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}}, "popularity": 1, "rank": 0, "self": "https://airbyteio.atlassian.net/rest/api/3/dashboard/10002", "sharePermissions": [], "editPermissions": [], "view": "/jira/dashboards/10002", "isWritable": true, "systemDashboard": false}, "emitted_at": 1697453217136} +{"stream": "dashboards", "data": {"description": "A dashboard to help auditors identify sample of issues to check.", "id": "10011", "isFavourite": true, "name": "Test dashboard 10", "owner": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "displayName": "integration test", "active": true, "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}}, "popularity": 1, "rank": 9, "self": "https://airbyteio.atlassian.net/rest/api/3/dashboard/10011", "sharePermissions": [], "editPermissions": [], "view": "/jira/dashboards/10011", "isWritable": true, "systemDashboard": false}, "emitted_at": 1697453217137} +{"stream": "filters", "data": {"expand": "description,owner,jql,viewUrl,searchUrl,favourite,favouritedCount,sharePermissions,editPermissions,isWritable,subscriptions", "self": "https://airbyteio.atlassian.net/rest/api/3/filter/10003", "id": "10003", "name": "Filter for EX board", "owner": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "jql": "project = EX ORDER BY Rank ASC", "viewUrl": "https://airbyteio.atlassian.net/issues/?filter=10003", "searchUrl": "https://airbyteio.atlassian.net/rest/api/3/search?jql=project+%3D+EX+ORDER+BY+Rank+ASC", "favourite": false, "favouritedCount": 0, "sharePermissions": [{"id": 10004, "type": "project", "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10003", "id": "10003", "key": "EX", "assigneeType": "PROJECT_LEAD", "name": "Example", "roles": {}, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10406", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10406?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10406?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10406?size=medium"}, "projectTypeKey": "software", "simplified": false, "style": "classic", "properties": {}}}], "isWritable": true, "subscriptions": []}, "emitted_at": 1697453218286} +{"stream": "filters", "data": {"expand": "description,owner,jql,viewUrl,searchUrl,favourite,favouritedCount,sharePermissions,editPermissions,isWritable,subscriptions", "self": "https://airbyteio.atlassian.net/rest/api/3/filter/10000", "id": "10000", "name": "Filter for IT board", "owner": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "jql": "project = IT ORDER BY Rank ASC", "viewUrl": "https://airbyteio.atlassian.net/issues/?filter=10000", "searchUrl": "https://airbyteio.atlassian.net/rest/api/3/search?jql=project+%3D+IT+ORDER+BY+Rank+ASC", "favourite": false, "favouritedCount": 0, "sharePermissions": [{"id": 10058, "type": "group", "group": {"name": "Test group 2", "groupId": "5ddb26f1-2d31-414a-ac34-b2d6de38805d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=5ddb26f1-2d31-414a-ac34-b2d6de38805d"}}, {"id": 10059, "type": "group", "group": {"name": "Test group 0", "groupId": "ee8d15d1-6462-406a-b0a6-8065b7e4cdd7", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=ee8d15d1-6462-406a-b0a6-8065b7e4cdd7"}}, {"id": 10057, "type": "project", "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10000", "id": "10000", "key": "IT", "assigneeType": "PROJECT_LEAD", "name": "integration-tests", "roles": {}, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10004", "id": "10004", "name": "Test category 2", "description": "Test Project Category 2"}, "projectTypeKey": "software", "simplified": false, "style": "classic", "properties": {}}}], "isWritable": true, "subscriptions": []}, "emitted_at": 1697453218287} +{"stream": "filters", "data": {"expand": "description,owner,jql,viewUrl,searchUrl,favourite,favouritedCount,sharePermissions,editPermissions,isWritable,subscriptions", "self": "https://airbyteio.atlassian.net/rest/api/3/filter/10001", "id": "10001", "name": "Filter for P2 board", "owner": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "jql": "project = P2 ORDER BY Rank ASC", "viewUrl": "https://airbyteio.atlassian.net/issues/?filter=10001", "searchUrl": "https://airbyteio.atlassian.net/rest/api/3/search?jql=project+%3D+P2+ORDER+BY+Rank+ASC", "favourite": false, "favouritedCount": 0, "sharePermissions": [{"id": 10063, "type": "group", "group": {"name": "Test group 0", "groupId": "ee8d15d1-6462-406a-b0a6-8065b7e4cdd7", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=ee8d15d1-6462-406a-b0a6-8065b7e4cdd7"}}, {"id": 10064, "type": "group", "group": {"name": "Test group 1", "groupId": "bda1faf1-1a1a-42d1-82e4-a428c8b8f67c", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=bda1faf1-1a1a-42d1-82e4-a428c8b8f67c"}}, {"id": 10062, "type": "project", "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10001", "id": "10001", "key": "P2", "assigneeType": "PROJECT_LEAD", "name": "project-2", "roles": {}, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10411", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10411?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10411?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10411?size=medium"}, "projectTypeKey": "software", "simplified": false, "style": "classic", "properties": {}}}], "isWritable": true, "subscriptions": []}, "emitted_at": 1697453218288} +{"stream": "filter_sharing", "data": {"id": 10004, "type": "project", "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10003", "id": "10003", "key": "EX", "assigneeType": "PROJECT_LEAD", "name": "Example", "roles": {}, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10406", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10406?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10406?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10406?size=medium"}, "projectTypeKey": "software", "simplified": false, "style": "classic", "properties": {}}, "filterId": "10003"}, "emitted_at": 1697453219638} +{"stream": "filter_sharing", "data": {"id": 10058, "type": "group", "group": {"name": "Test group 2", "groupId": "5ddb26f1-2d31-414a-ac34-b2d6de38805d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=5ddb26f1-2d31-414a-ac34-b2d6de38805d"}, "filterId": "10000"}, "emitted_at": 1697453219940} +{"stream": "filter_sharing", "data": {"id": 10059, "type": "group", "group": {"name": "Test group 0", "groupId": "ee8d15d1-6462-406a-b0a6-8065b7e4cdd7", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=ee8d15d1-6462-406a-b0a6-8065b7e4cdd7"}, "filterId": "10000"}, "emitted_at": 1697453219940} +{"stream": "groups", "data": {"name": "Test group 17", "groupId": "022bc924-ac57-442d-80c9-df042b73ad87"}, "emitted_at": 1697453247031} +{"stream": "groups", "data": {"name": "administrators", "groupId": "0ca6e087-7a61-4986-a269-98fe268854a1"}, "emitted_at": 1697453247032} +{"stream": "groups", "data": {"name": "jira-servicemanagement-customers-airbyteio", "groupId": "125680d3-7e85-41ad-a662-892b6590272e"}, "emitted_at": 1697453247033} +{"stream": "issues", "data": {"expand": "customfield_10030.properties,operations,versionedRepresentations,editmeta,changelog,customfield_10029.properties,customfield_10010.requestTypePractice,transitions,renderedFields,customfield_10229.properties", "id": "10625", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625", "key": "IT-25", "renderedFields": {"statuscategorychangedate": "17/May/22 4:06 AM", "created": "17/May/22 4:06 AM", "customfield_10017": "dark_yellow", "updated": "17/May/22 4:28 AM", "description": "

      Implement OAUth

      ", "customfield_10011": "Test 2", "customfield_10013": "ghx-label-2", "timetracking": {}, "attachment": [], "environment": "", "comment": {"comments": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625/comment/10755", "id": "10755", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "body": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Closed"}]}]}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "17/May/22 4:06 AM", "updated": "17/May/22 4:06 AM", "jsdPublic": true}], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625/comment", "maxResults": 1, "total": 1, "startAt": 0}, "worklog": {"startAt": 0, "maxResults": 20, "total": 0, "worklogs": []}}, "transitions": [{"id": "11", "name": "To Do", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "21", "name": "In Progress", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/3", "description": "This issue is being actively worked on at the moment by the assignee.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/inprogress.png", "name": "In Progress", "id": "3", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "31", "name": "Done", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10001", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "Done", "id": "10001", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "41", "name": "Approved", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10005", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Approved", "id": "10005", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "51", "name": "In review", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10004", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "In review", "id": "10004", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "61", "name": "Reopened", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/4", "description": "This issue was once resolved, but the resolution was deemed incorrect. From here issues are either marked assigned or resolved.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/reopened.png", "name": "Reopened", "id": "4", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "71", "name": "Declined", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10002", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/generic.png", "name": "Declined", "id": "10002", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "81", "name": "Open", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/1", "description": "The issue is open and ready for the assignee to start work on it.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/open.png", "name": "Open", "id": "1", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "91", "name": "Pending", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10003", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Pending", "id": "10003", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "101", "name": "Closed", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/6", "description": "The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/closed.png", "name": "Closed", "id": "6", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}], "changelog": {"startAt": 0, "maxResults": 1, "total": 1, "histories": [{"id": "15129", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2022-05-17T04:28:19.880-0700", "items": [{"field": "Link", "fieldtype": "jira", "from": null, "fromString": null, "to": "IT-26", "toString": "This issue is cloned by IT-26"}]}]}, "fields": {"statuscategorychangedate": "2022-05-17T04:06:24.675-0700", "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}, "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10000", "id": "10000", "key": "IT", "name": "integration-tests", "projectTypeKey": "software", "simplified": false, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10004", "id": "10004", "description": "Test Project Category 2", "name": "Test category 2"}}, "fixVersions": [], "workratio": -1, "watches": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-25/watchers", "watchCount": 1, "isWatching": true}, "issuerestriction": {"issuerestrictions": {}, "shouldDisplay": false}, "created": "2022-05-17T04:06:24.048000-07:00", "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "labels": [], "customfield_10017": "dark_yellow", "customfield_10018": {"hasEpicLinkFieldDependency": false, "showField": false, "nonEditableReason": {"reason": "PLUGIN_LICENSE_ERROR", "message": "The Parent Link is only available to Jira Premium users."}}, "customfield_10019": "0|i0076v:", "customfield_10217": [], "versions": [], "issuelinks": [{"id": "10263", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLink/10263", "type": {"id": "10001", "name": "Cloners", "inward": "is cloned by", "outward": "clones", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10001"}, "inwardIssue": {"id": "10626", "key": "IT-26", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626", "fields": {"summary": "CLONE - Aggregate issues", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}}}}], "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updated": "2022-05-17T04:28:19.876000-07:00", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "components": [{"self": "https://airbyteio.atlassian.net/rest/api/3/component/10049", "id": "10049", "name": "Component 3", "description": "This is a Jira component"}], "description": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Implement OAUth"}]}]}, "customfield_10011": "Test 2", "customfield_10012": {"self": "https://airbyteio.atlassian.net/rest/api/3/customFieldOption/10016", "value": "To Do", "id": "10016"}, "customfield_10013": "ghx-label-2", "timetracking": {}, "attachment": [], "summary": "Aggregate issues", "creator": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "subtasks": [], "reporter": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "aggregateprogress": {"progress": 0, "total": 0}, "progress": {"progress": 0, "total": 0}, "comment": {"comments": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625/comment/10755", "id": "10755", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "body": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Closed"}]}]}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2022-05-17T04:06:55.076-0700", "updated": "2022-05-17T04:06:55.076-0700", "jsdPublic": true}], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625/comment", "maxResults": 1, "total": 1, "startAt": 0}, "votes": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-25/votes", "votes": 0, "hasVoted": false}, "worklog": {"startAt": 0, "maxResults": 20, "total": 0, "worklogs": []}}, "projectId": "10000", "projectKey": "IT", "created": "2022-05-17T04:06:24.048000-07:00", "updated": "2022-05-17T04:28:19.876000-07:00"}, "emitted_at": 1701283916831} +{"stream": "issues", "data": {"expand": "customfield_10030.properties,operations,versionedRepresentations,editmeta,changelog,customfield_10029.properties,customfield_10010.requestTypePractice,transitions,renderedFields,customfield_10229.properties", "id": "10080", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080", "key": "IT-24", "renderedFields": {"statuscategorychangedate": "11/Mar/21 6:17 AM", "timespent": "5 hours, 48 minutes", "aggregatetimespent": "5 hours, 48 minutes", "created": "11/Mar/21 6:17 AM", "customfield_10017": "", "timeestimate": "0 minutes", "updated": "05/Apr/23 4:58 AM", "description": "

      Test description 74

      ", "timetracking": {"remainingEstimate": "0 minutes", "timeSpent": "5 hours, 48 minutes", "remainingEstimateSeconds": 0, "timeSpentSeconds": 20880}, "attachment": [{"self": "https://airbyteio.atlassian.net/rest/api/3/attachment/10123", "id": "10123", "filename": "demo.xlsx", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "14/Apr/21 2:11 PM", "size": "7 kB", "content": "https://airbyteio.atlassian.net/rest/api/3/attachment/content/10123"}], "aggregatetimeestimate": "0 minutes", "environment": "", "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/comment", "maxResults": 0, "total": 0, "startAt": 0}, "worklog": {"startAt": 0, "maxResults": 20, "total": 3, "worklogs": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11708", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "15/Apr/21 11:39 AM", "updated": "15/Apr/21 11:39 AM", "started": "14/Apr/21 6:48 PM", "timeSpent": "2 hours, 21 minutes", "id": "11708", "issueId": "10080"}, {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11709", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "15/Apr/21 11:39 AM", "updated": "15/Apr/21 11:39 AM", "started": "14/Apr/21 6:48 PM", "timeSpent": "37 minutes", "id": "11709", "issueId": "10080"}, {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11710", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "15/Apr/21 11:39 AM", "updated": "15/Apr/21 11:39 AM", "started": "14/Apr/21 6:48 PM", "timeSpent": "2 hours, 50 minutes", "id": "11710", "issueId": "10080"}]}}, "transitions": [{"id": "11", "name": "To Do", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "21", "name": "In Progress", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/3", "description": "This issue is being actively worked on at the moment by the assignee.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/inprogress.png", "name": "In Progress", "id": "3", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "31", "name": "Done", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10001", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "Done", "id": "10001", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "41", "name": "Approved", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10005", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Approved", "id": "10005", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "51", "name": "In review", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10004", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "In review", "id": "10004", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "61", "name": "Reopened", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/4", "description": "This issue was once resolved, but the resolution was deemed incorrect. From here issues are either marked assigned or resolved.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/reopened.png", "name": "Reopened", "id": "4", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "71", "name": "Declined", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10002", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/generic.png", "name": "Declined", "id": "10002", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "81", "name": "Open", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/1", "description": "The issue is open and ready for the assignee to start work on it.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/open.png", "name": "Open", "id": "1", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "91", "name": "Pending", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10003", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Pending", "id": "10003", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "101", "name": "Closed", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/6", "description": "The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/closed.png", "name": "Closed", "id": "6", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}], "changelog": {"startAt": 0, "maxResults": 8, "total": 8, "histories": [{"id": "15179", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2023-04-05T04:58:35.333-0700", "items": [{"field": "Sprint", "fieldtype": "custom", "fieldId": "customfield_10020", "from": "", "fromString": "", "to": "10", "toString": "IT Sprint 9"}]}, {"id": "14989", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T11:39:47.917-0700", "items": [{"field": "timeestimate", "fieldtype": "jira", "fieldId": "timeestimate", "from": "0", "fromString": "0", "to": "0", "toString": "0"}, {"field": "timespent", "fieldtype": "jira", "fieldId": "timespent", "from": "10680", "fromString": "10680", "to": "20880", "toString": "20880"}, {"field": "WorklogId", "fieldtype": "jira", "from": null, "fromString": null, "to": "11710", "toString": "11710"}]}, {"id": "14988", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T11:39:47.314-0700", "items": [{"field": "timeestimate", "fieldtype": "jira", "fieldId": "timeestimate", "from": "0", "fromString": "0", "to": "0", "toString": "0"}, {"field": "timespent", "fieldtype": "jira", "fieldId": "timespent", "from": "8460", "fromString": "8460", "to": "10680", "toString": "10680"}, {"field": "WorklogId", "fieldtype": "jira", "from": null, "fromString": null, "to": "11709", "toString": "11709"}]}, {"id": "14987", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T11:39:46.691-0700", "items": [{"field": "timeestimate", "fieldtype": "jira", "fieldId": "timeestimate", "from": null, "fromString": null, "to": "0", "toString": "0"}, {"field": "timespent", "fieldtype": "jira", "fieldId": "timespent", "from": null, "fromString": null, "to": "8460", "toString": "8460"}, {"field": "WorklogId", "fieldtype": "jira", "from": null, "fromString": null, "to": "11708", "toString": "11708"}]}, {"id": "14800", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T07:18:07.884-0700", "items": [{"field": "RemoteIssueLink", "fieldtype": "jira", "from": null, "fromString": null, "to": "10046", "toString": "This issue links to \"TSTSUP-111 (My Acme Tracker)\""}]}, {"id": "14718", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T00:08:54.455-0700", "items": [{"field": "Link", "fieldtype": "jira", "from": null, "fromString": null, "to": "IT-22", "toString": "This issue is duplicated by IT-22"}]}, {"id": "14716", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T00:08:48.880-0700", "items": [{"field": "Link", "fieldtype": "jira", "from": null, "fromString": null, "to": "IT-23", "toString": "This issue is duplicated by IT-23"}]}, {"id": "14596", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-14T14:11:01.899-0700", "items": [{"field": "Attachment", "fieldtype": "jira", "fieldId": "attachment", "from": null, "fromString": null, "to": "10123", "toString": "demo.xlsx"}]}]}, "fields": {"statuscategorychangedate": "2021-03-11T06:17:33.483-0800", "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10004", "id": "10004", "description": "A problem or error.", "iconUrl": "https://airbyteio.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10303?size=medium", "name": "Bug", "subtask": false, "avatarId": 10303, "hierarchyLevel": 0}, "timespent": 20880, "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10000", "id": "10000", "key": "IT", "name": "integration-tests", "projectTypeKey": "software", "simplified": false, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10004", "id": "10004", "description": "Test Project Category 2", "name": "Test category 2"}}, "fixVersions": [], "aggregatetimespent": 20880, "workratio": -1, "issuerestriction": {"issuerestrictions": {}, "shouldDisplay": false}, "watches": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-24/watchers", "watchCount": 1, "isWatching": true}, "created": "2021-03-11T06:17:33.169000-08:00", "customfield_10020": [{"id": 10, "name": "IT Sprint 9", "state": "future", "boardId": 1, "startDate": "2022-09-06T11:25:59.072Z", "endDate": "2022-09-20T11:25:00.000Z"}], "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/3", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/medium.svg", "name": "Medium", "id": "3"}, "labels": [], "customfield_10018": {"hasEpicLinkFieldDependency": false, "showField": false, "nonEditableReason": {"reason": "PLUGIN_LICENSE_ERROR", "message": "The Parent Link is only available to Jira Premium users."}}, "customfield_10217": [], "customfield_10019": "0|i000hr:", "timeestimate": 0, "versions": [], "issuelinks": [{"id": "10244", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLink/10244", "type": {"id": "10002", "name": "Duplicate", "inward": "is duplicated by", "outward": "duplicates", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10002"}, "inwardIssue": {"id": "10069", "key": "IT-22", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10069", "fields": {"summary": "Test 63", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/3", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/medium.svg", "name": "Medium", "id": "3"}, "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}}}}, {"id": "10243", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLink/10243", "type": {"id": "10002", "name": "Duplicate", "inward": "is duplicated by", "outward": "duplicates", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10002"}, "inwardIssue": {"id": "10075", "key": "IT-23", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10075", "fields": {"summary": "Test 69", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/3", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/medium.svg", "name": "Medium", "id": "3"}, "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10004", "id": "10004", "description": "A problem or error.", "iconUrl": "https://airbyteio.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10303?size=medium", "name": "Bug", "subtask": false, "avatarId": 10303, "hierarchyLevel": 0}}}}], "updated": "2023-04-05T04:58:35.329000-07:00", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "components": [], "description": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Test description 74"}]}]}, "timetracking": {"remainingEstimate": "0m", "timeSpent": "5h 48m", "remainingEstimateSeconds": 0, "timeSpentSeconds": 20880}, "attachment": [{"self": "https://airbyteio.atlassian.net/rest/api/3/attachment/10123", "id": "10123", "filename": "demo.xlsx", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-14T14:11:01.652-0700", "size": 7360, "content": "https://airbyteio.atlassian.net/rest/api/3/attachment/content/10123"}], "aggregatetimeestimate": 0, "summary": "Test 74", "creator": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "subtasks": [], "reporter": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "aggregateprogress": {"progress": 20880, "total": 20880, "percent": 100}, "progress": {"progress": 20880, "total": 20880, "percent": 100}, "votes": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-24/votes", "votes": 1, "hasVoted": true}, "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/comment", "maxResults": 0, "total": 0, "startAt": 0}, "worklog": {"startAt": 0, "maxResults": 20, "total": 3, "worklogs": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11708", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 0", "type": "text"}]}]}, "created": "2021-04-15T11:39:46.574-0700", "updated": "2021-04-15T11:39:46.574-0700", "started": "2021-04-14T18:48:52.747-0700", "timeSpent": "2h 21m", "timeSpentSeconds": 8460, "id": "11708", "issueId": "10080"}, {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11709", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 1", "type": "text"}]}]}, "created": "2021-04-15T11:39:47.215-0700", "updated": "2021-04-15T11:39:47.215-0700", "started": "2021-04-14T18:48:52.747-0700", "timeSpent": "37m", "timeSpentSeconds": 2220, "id": "11709", "issueId": "10080"}, {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11710", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 2", "type": "text"}]}]}, "created": "2021-04-15T11:39:47.834-0700", "updated": "2021-04-15T11:39:47.834-0700", "started": "2021-04-14T18:48:52.747-0700", "timeSpent": "2h 50m", "timeSpentSeconds": 10200, "id": "11710", "issueId": "10080"}]}}, "projectId": "10000", "projectKey": "IT", "created": "2021-03-11T06:17:33.169000-08:00", "updated": "2023-04-05T04:58:35.329000-07:00"}, "emitted_at": 1701283916937} +{"stream": "issues", "data": {"expand": "customfield_10030.properties,operations,versionedRepresentations,editmeta,changelog,customfield_10029.properties,customfield_10010.requestTypePractice,transitions,renderedFields,customfield_10229.properties", "id": "10626", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626", "key": "IT-26", "renderedFields": {"statuscategorychangedate": "17/May/22 4:28 AM", "timespent": "1 day", "aggregatetimespent": "1 day", "lastViewed": "12/Oct/23 1:43 PM", "created": "17/May/22 4:28 AM", "customfield_10017": "dark_yellow", "timeestimate": "1 week, 1 day", "aggregatetimeoriginalestimate": "2 weeks, 4 days, 5 hours", "updated": "12/Oct/23 1:43 PM", "timeoriginalestimate": "2 weeks, 4 days, 5 hours", "description": "

      Implement OAUth

      ", "customfield_10011": "Test 2", "customfield_10013": "ghx-label-2", "timetracking": {"originalEstimate": "2 weeks, 4 days, 5 hours", "remainingEstimate": "1 week, 1 day", "timeSpent": "1 day", "originalEstimateSeconds": 421200, "remainingEstimateSeconds": 172800, "timeSpentSeconds": 28800}, "attachment": [], "aggregatetimeestimate": "1 week, 1 day", "environment": "", "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/comment", "maxResults": 0, "total": 0, "startAt": 0}, "worklog": {"startAt": 0, "maxResults": 20, "total": 1, "worklogs": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/worklog/11820", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "created": "05/Apr/23 5:08 AM", "updated": "05/Apr/23 5:08 AM", "started": "05/Apr/23 1:00 AM", "timeSpent": "1 day", "id": "11820", "issueId": "10626"}]}}, "transitions": [{"id": "11", "name": "To Do", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "21", "name": "In Progress", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/3", "description": "This issue is being actively worked on at the moment by the assignee.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/inprogress.png", "name": "In Progress", "id": "3", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "31", "name": "Done", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10001", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "Done", "id": "10001", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "41", "name": "Approved", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10005", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Approved", "id": "10005", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "51", "name": "In review", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10004", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "In review", "id": "10004", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "61", "name": "Reopened", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/4", "description": "This issue was once resolved, but the resolution was deemed incorrect. From here issues are either marked assigned or resolved.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/reopened.png", "name": "Reopened", "id": "4", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "71", "name": "Declined", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10002", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/generic.png", "name": "Declined", "id": "10002", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "81", "name": "Open", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/1", "description": "The issue is open and ready for the assignee to start work on it.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/open.png", "name": "Open", "id": "1", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "91", "name": "Pending", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10003", "description": "This was auto-generated by Jira Service Management during workflow import", "iconUrl": "https://airbyteio.atlassian.net/images/icons/status_generic.gif", "name": "Pending", "id": "10003", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}, {"id": "101", "name": "Closed", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/6", "description": "The issue is considered finished, the resolution is correct. Issues which are closed can be reopened.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/closed.png", "name": "Closed", "id": "6", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false}], "changelog": {"startAt": 0, "maxResults": 4, "total": 4, "histories": [{"id": "15198", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2023-10-12T13:43:15.036-0700", "items": [{"field": "timeestimate", "fieldtype": "jira", "fieldId": "timeestimate", "from": null, "fromString": null, "to": "172800", "toString": "172800"}]}, {"id": "15197", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2023-10-12T13:43:05.182-0700", "items": [{"field": "timeoriginalestimate", "fieldtype": "jira", "fieldId": "timeoriginalestimate", "from": null, "fromString": null, "to": "421200", "toString": "421200"}]}, {"id": "15186", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "created": "2023-04-05T05:08:50.115-0700", "items": [{"field": "timespent", "fieldtype": "jira", "fieldId": "timespent", "from": null, "fromString": null, "to": "28800", "toString": "28800"}, {"field": "WorklogId", "fieldtype": "jira", "from": null, "fromString": null, "to": "11820", "toString": "11820"}]}, {"id": "15128", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2022-05-17T04:28:19.837-0700", "items": [{"field": "Link", "fieldtype": "jira", "from": null, "fromString": null, "to": "IT-25", "toString": "This issue clones IT-25"}]}]}, "fields": {"statuscategorychangedate": "2022-05-17T04:28:19.775-0700", "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}, "timespent": 28800, "project": {"self": "https://airbyteio.atlassian.net/rest/api/3/project/10000", "id": "10000", "key": "IT", "name": "integration-tests", "projectTypeKey": "software", "simplified": false, "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10004", "id": "10004", "description": "Test Project Category 2", "name": "Test category 2"}}, "fixVersions": [], "aggregatetimespent": 28800, "workratio": 6, "issuerestriction": {"issuerestrictions": {}, "shouldDisplay": false}, "lastViewed": "2023-10-12T13:43:22.992-0700", "watches": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-26/watchers", "watchCount": 1, "isWatching": true}, "created": "2022-05-17T04:28:19.523000-07:00", "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "labels": [], "customfield_10017": "dark_yellow", "customfield_10018": {"hasEpicLinkFieldDependency": false, "showField": false, "nonEditableReason": {"reason": "PLUGIN_LICENSE_ERROR", "message": "The Parent Link is only available to Jira Premium users."}}, "customfield_10217": [], "customfield_10019": "0|i00773:", "timeestimate": 172800, "aggregatetimeoriginalestimate": 421200, "versions": [], "issuelinks": [{"id": "10263", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLink/10263", "type": {"id": "10001", "name": "Cloners", "inward": "is cloned by", "outward": "clones", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10001"}, "outwardIssue": {"id": "10625", "key": "IT-25", "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625", "fields": {"summary": "Aggregate issues", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "priority": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/4", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/low.svg", "name": "Low", "id": "4"}, "issuetype": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", "id": "10000", "description": "A big user story that needs to be broken down. Created by Jira Software - do not edit or delete.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/issuetypes/epic.svg", "name": "Epic", "subtask": false, "hierarchyLevel": 1}}}}], "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updated": "2023-10-12T13:43:15.025000-07:00", "status": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "components": [{"self": "https://airbyteio.atlassian.net/rest/api/3/component/10049", "id": "10049", "name": "Component 3", "description": "This is a Jira component"}], "timeoriginalestimate": 421200, "description": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Implement OAUth"}]}]}, "customfield_10011": "Test 2", "customfield_10012": {"self": "https://airbyteio.atlassian.net/rest/api/3/customFieldOption/10016", "value": "To Do", "id": "10016"}, "customfield_10013": "ghx-label-2", "timetracking": {"originalEstimate": "2w 4d 5h", "remainingEstimate": "1w 1d", "timeSpent": "1d", "originalEstimateSeconds": 421200, "remainingEstimateSeconds": 172800, "timeSpentSeconds": 28800}, "attachment": [], "aggregatetimeestimate": 172800, "summary": "CLONE - Aggregate issues", "creator": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "subtasks": [], "reporter": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "aggregateprogress": {"progress": 28800, "total": 201600, "percent": 14}, "progress": {"progress": 28800, "total": 201600, "percent": 14}, "comment": {"comments": [], "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/comment", "maxResults": 0, "total": 0, "startAt": 0}, "votes": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-26/votes", "votes": 0, "hasVoted": false}, "worklog": {"startAt": 0, "maxResults": 20, "total": 1, "worklogs": [{"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/worklog/11820", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "comment": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "time-tracking"}]}]}, "created": "2023-04-05T05:08:50.033-0700", "updated": "2023-04-05T05:08:50.033-0700", "started": "2023-04-05T01:00:00.000-0700", "timeSpent": "1d", "timeSpentSeconds": 28800, "id": "11820", "issueId": "10626"}]}}, "projectId": "10000", "projectKey": "IT", "created": "2022-05-17T04:28:19.523000-07:00", "updated": "2023-10-12T13:43:15.025000-07:00"}, "emitted_at": 1701283916963} +{"stream": "issue_comments", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10625/comment/10755", "id": "10755", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "body": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "Closed"}]}]}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2022-05-17T04:06:55.076000-07:00", "updated": "2022-05-17T04:06:55.076000-07:00", "jsdPublic": true, "issueId": "IT-25"}, "emitted_at": 1697453253441} +{"stream": "issue_comments", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10075/comment/10521", "id": "10521", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "body": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque eget venenatis elit. Duis eu justo eget augue iaculis fermentum. Sed semper quam laoreet nisi egestas at posuere augue semper.", "type": "text"}]}]}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-14T14:32:43.099000-07:00", "updated": "2021-04-14T14:32:43.099000-07:00", "jsdPublic": true, "issueId": "IT-23"}, "emitted_at": 1697453254086} +{"stream": "issue_comments", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10075/comment/10639", "id": "10639", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "body": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "Linked related issue!", "type": "text"}]}]}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "created": "2021-04-15T00:08:48.998000-07:00", "updated": "2021-04-15T00:08:48.998000-07:00", "jsdPublic": true, "issueId": "IT-23"}, "emitted_at": 1697453254087} +{"stream": "issue_fields", "data": {"id": "statuscategorychangedate", "key": "statuscategorychangedate", "name": "Status Category Changed", "custom": false, "orderable": false, "navigable": true, "searchable": true, "clauseNames": ["statusCategoryChangedDate"], "schema": {"type": "datetime", "system": "statuscategorychangedate"}}, "emitted_at": 1697453262531} +{"stream": "issue_fields", "data": {"id": "parent", "key": "parent", "name": "Parent", "custom": false, "orderable": false, "navigable": true, "searchable": false, "clauseNames": ["parent"]}, "emitted_at": 1697453262532} +{"stream": "issue_fields", "data": {"id": "issuetype", "key": "issuetype", "name": "Issue Type", "custom": false, "orderable": true, "navigable": true, "searchable": true, "clauseNames": ["issuetype", "type"], "schema": {"type": "issuetype", "system": "issuetype"}}, "emitted_at": 1697453262533} +{"stream": "issue_field_configurations", "data": {"id": 10000, "name": "Default Field Configuration", "description": "The default field configuration", "isDefault": true}, "emitted_at": 1697453263242} +{"stream": "issue_field_configurations", "data": {"id": 10001, "name": "Field Config 1", "description": "Field Config 1 test"}, "emitted_at": 1697453263243} +{"stream": "issue_field_configurations", "data": {"id": 10002, "name": "Field Config 2", "description": "Field Config 2 test"}, "emitted_at": 1697453263243} +{"stream": "issue_custom_field_contexts", "data": {"id": "10130", "name": "Default Configuration Scheme for Account", "description": "Default configuration scheme generated by Jira", "isGlobalContext": true, "isAnyIssueType": true, "fieldId": "customfield_10030", "fieldType": "option2"}, "emitted_at": 1697453263627} +{"stream": "issue_custom_field_contexts", "data": {"id": "10382", "name": "Default Configuration Scheme for Time to first response", "description": "Default configuration scheme generated by Jira", "isGlobalContext": true, "isAnyIssueType": true, "fieldId": "customfield_10225", "fieldType": "sd-servicelevelagreement"}, "emitted_at": 1697453263898} +{"stream": "issue_custom_field_contexts", "data": {"id": "10384", "name": "Default Configuration Scheme for Time to done", "description": "Default configuration scheme generated by Jira", "isGlobalContext": true, "isAnyIssueType": true, "fieldId": "customfield_10226", "fieldType": "sd-servicelevelagreement"}, "emitted_at": 1697453264172} +{"stream": "issue_custom_field_options", "data": {"id": "10016", "value": "To Do", "disabled": false, "fieldId": "customfield_10012", "contextId": "10112"}, "emitted_at": 1697453280554} +{"stream": "issue_custom_field_options", "data": {"id": "10017", "value": "In Progress", "disabled": false, "fieldId": "customfield_10012", "contextId": "10112"}, "emitted_at": 1697453280555} +{"stream": "issue_custom_field_options", "data": {"id": "10018", "value": "Done", "disabled": false, "fieldId": "customfield_10012", "contextId": "10112"}, "emitted_at": 1697453280555} +{"stream": "issue_link_types", "data": {"id": "10000", "name": "Blocks", "inward": "is blocked by", "outward": "blocks", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10000"}, "emitted_at": 1697453282443} +{"stream": "issue_link_types", "data": {"id": "10001", "name": "Cloners", "inward": "is cloned by", "outward": "clones", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10001"}, "emitted_at": 1697453282444} +{"stream": "issue_link_types", "data": {"id": "10002", "name": "Duplicate", "inward": "is duplicated by", "outward": "duplicates", "self": "https://airbyteio.atlassian.net/rest/api/3/issueLinkType/10002"}, "emitted_at": 1697453282444} +{"stream": "issue_navigator_settings", "data": {"label": "Issue Type", "value": "issuetype"}, "emitted_at": 1697453283275} +{"stream": "issue_navigator_settings", "data": {"label": "Key", "value": "issuekey"}, "emitted_at": 1697453283276} +{"stream": "issue_navigator_settings", "data": {"label": "Summary", "value": "summary"}, "emitted_at": 1697453283277} +{"stream": "issue_notification_schemes", "data": {"expand": "notificationSchemeEvents,user,group,projectRole,field,all", "id": 10000, "self": "https://airbyteio.atlassian.net/rest/api/3/notificationscheme/10000", "name": "Default Notification Scheme"}, "emitted_at": 1697453284065} +{"stream": "issue_notification_schemes", "data": {"expand": "notificationSchemeEvents,user,group,projectRole,field,all", "id": 10001, "self": "https://airbyteio.atlassian.net/rest/api/3/notificationscheme/10001", "name": "Notification Scheme 1", "description": "Notification Scheme 1 test"}, "emitted_at": 1697453284066} +{"stream": "issue_notification_schemes", "data": {"expand": "notificationSchemeEvents,user,group,projectRole,field,all", "id": 10002, "self": "https://airbyteio.atlassian.net/rest/api/3/notificationscheme/10002", "name": "Notification Scheme 2", "description": "Notification Scheme 2 test"}, "emitted_at": 1697453284066} +{"stream": "issue_priorities", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/1", "statusColor": "#d04437", "description": "This problem will block progress.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/highest.svg", "name": "Highest", "id": "1", "isDefault": false}, "emitted_at": 1697453284694} +{"stream": "issue_priorities", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/2", "statusColor": "#f15C75", "description": "Serious problem that could block progress.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/high.svg", "name": "High", "id": "2", "isDefault": false}, "emitted_at": 1697453284696} +{"stream": "issue_priorities", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/priority/3", "statusColor": "#f79232", "description": "Has the potential to affect progress.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/priorities/medium.svg", "name": "Medium", "id": "3", "isDefault": false}, "emitted_at": 1697453284696} +{"stream": "issue_properties", "data": {"key": "myProperty", "value": {"owner": "admin", "weight": 100}, "issueId": "IT-16"}, "emitted_at": 1697453288836} +{"stream": "issue_properties", "data": {"key": "myProperty1", "value": {"owner": "admin", "weight": 100}, "issueId": "IT-16"}, "emitted_at": 1697453289165} +{"stream": "issue_properties", "data": {"key": "myProperty2", "value": {"owner": "admin", "weight": 100}, "issueId": "IT-16"}, "emitted_at": 1697453289439} +{"stream": "issue_remote_links", "data": {"id": 10046, "self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-24/remotelink/10046", "globalId": "system=https://www.mycompany.com/support&id=1", "application": {"type": "com.acme.tracker", "name": "My Acme Tracker"}, "relationship": "causes", "object": {"url": "https://www.mycompany.com/support?id=1", "title": "TSTSUP-111", "summary": "Customer support issue", "icon": {"url16x16": "https://www.mycompany.com/support/ticket.png", "title": "Support Ticket"}, "status": {"resolved": true, "icon": {"url16x16": "https://www.mycompany.com/support/resolved.png", "title": "Case Closed", "link": "https://www.mycompany.com/support?id=1&details=closed"}}}, "issueId": "IT-24"}, "emitted_at": 1697453301736} +{"stream": "issue_remote_links", "data": {"id": 10047, "self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-23/remotelink/10047", "globalId": "system=https://www.mycompany.com/support&id=1", "application": {"type": "com.acme.tracker", "name": "My Acme Tracker"}, "relationship": "causes", "object": {"url": "https://www.mycompany.com/support?id=1", "title": "TSTSUP-111", "summary": "Customer support issue", "icon": {"url16x16": "https://www.mycompany.com/support/ticket.png", "title": "Support Ticket"}, "status": {"resolved": true, "icon": {"url16x16": "https://www.mycompany.com/support/resolved.png", "title": "Case Closed", "link": "https://www.mycompany.com/support?id=1&details=closed"}}}, "issueId": "IT-23"}, "emitted_at": 1697453302033} +{"stream": "issue_remote_links", "data": {"id": 10048, "self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-22/remotelink/10048", "globalId": "system=https://www.mycompany.com/support&id=1", "application": {"type": "com.acme.tracker", "name": "My Acme Tracker"}, "relationship": "causes", "object": {"url": "https://www.mycompany.com/support?id=1", "title": "TSTSUP-111", "summary": "Customer support issue", "icon": {"url16x16": "https://www.mycompany.com/support/ticket.png", "title": "Support Ticket"}, "status": {"resolved": true, "icon": {"url16x16": "https://www.mycompany.com/support/resolved.png", "title": "Case Closed", "link": "https://www.mycompany.com/support?id=1&details=closed"}}}, "issueId": "IT-22"}, "emitted_at": 1697453302333} +{"stream": "issue_resolutions", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/resolution/10000", "id": "10000", "description": "Work has been completed on this issue.", "name": "Done", "isDefault": false}, "emitted_at": 1697453310592} +{"stream": "issue_resolutions", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/resolution/10001", "id": "10001", "description": "This issue won't be actioned.", "name": "Won't Do", "isDefault": false}, "emitted_at": 1697453310593} +{"stream": "issue_resolutions", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/resolution/10002", "id": "10002", "description": "The problem is a duplicate of an existing issue.", "name": "Duplicate", "isDefault": false}, "emitted_at": 1697453310594} +{"stream": "issue_security_schemes", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuesecurityschemes/10001", "id": 10001, "name": "Security scheme 2", "description": "Security scheme 2"}, "emitted_at": 1697453311388} +{"stream": "issue_security_schemes", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuesecurityschemes/10000", "id": 10000, "name": "Security scheme 1", "description": "Security scheme 1", "defaultSecurityLevelId": 10002}, "emitted_at": 1697453311389} +{"stream": "issue_security_schemes", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuesecurityschemes/10002", "id": 10002, "name": "Security scheme 3", "description": "Security scheme 3 test", "defaultSecurityLevelId": 10003}, "emitted_at": 1697453311389} +{"stream": "issue_transitions", "data": {"id": "11", "name": "To Do", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false, "issueId": "IT-26"}, "emitted_at": 1697453314405} +{"stream": "issue_transitions", "data": {"id": "21", "name": "In Progress", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/3", "description": "This issue is being actively worked on at the moment by the assignee.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/inprogress.png", "name": "In Progress", "id": "3", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false, "issueId": "IT-26"}, "emitted_at": 1697453314407} +{"stream": "issue_transitions", "data": {"id": "31", "name": "Done", "to": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/10001", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "Done", "id": "10001", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "hasScreen": false, "isGlobal": true, "isInitial": false, "isAvailable": true, "isConditional": false, "isLooped": false, "issueId": "IT-26"}, "emitted_at": 1697453314407} +{"stream": "issue_type_schemes", "data": {"id": "10000", "name": "Default Issue Type Scheme", "description": "Default issue type scheme is the list of global issue types. All newly created issue types will automatically be added to this scheme.", "isDefault": true}, "emitted_at": 1697453325645} +{"stream": "issue_type_schemes", "data": {"id": "10126", "name": "IT: Scrum Issue Type Scheme", "defaultIssueTypeId": "10001"}, "emitted_at": 1697453325646} +{"stream": "issue_type_schemes", "data": {"id": "10128", "name": "P2: Scrum Issue Type Scheme", "defaultIssueTypeId": "10001"}, "emitted_at": 1697453325647} +{"stream": "issue_types", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10023", "id": "10023", "description": "Subtasks track small pieces of work that are part of a larger task.", "iconUrl": "https://airbyteio.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10316?size=medium", "name": "Subtask", "untranslatedName": "Subtask", "subtask": true, "avatarId": 10316, "hierarchyLevel": -1, "scope": {"type": "PROJECT", "project": {"id": "10064"}}}, "emitted_at": 1697453326610} +{"stream": "issue_types", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10026", "id": "10026", "description": "Subtasks track small pieces of work that are part of a larger task.", "iconUrl": "https://airbyteio.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10316?size=medium", "name": "Subtask", "untranslatedName": "Subtask", "subtask": true, "avatarId": 10316, "hierarchyLevel": -1, "scope": {"type": "PROJECT", "project": {"id": "10065"}}}, "emitted_at": 1697453326611} +{"stream": "issue_types", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10003", "id": "10003", "description": "A small piece of work that's part of a larger task.", "iconUrl": "https://airbyteio.atlassian.net/rest/api/2/universal_avatar/view/type/issuetype/avatar/10316?size=medium", "name": "Sub-task", "untranslatedName": "Sub-task", "subtask": true, "avatarId": 10316, "hierarchyLevel": -1}, "emitted_at": 1697453326611} +{"stream": "issue_type_screen_schemes", "data": {"id": "1", "name": "Default Issue Type Screen Scheme", "description": "The default issue type screen scheme"}, "emitted_at": 1697453327278} +{"stream": "issue_type_screen_schemes", "data": {"id": "10000", "name": "IT: Scrum Issue Type Screen Scheme", "description": ""}, "emitted_at": 1697453327279} +{"stream": "issue_type_screen_schemes", "data": {"id": "10001", "name": "P2: Scrum Issue Type Screen Scheme", "description": ""}, "emitted_at": 1697453327279} +{"stream": "issue_votes", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-26/votes", "votes": 0, "hasVoted": false, "voters": [], "issueId": "IT-26"}, "emitted_at": 1697453328350} +{"stream": "issue_votes", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-25/votes", "votes": 0, "hasVoted": false, "voters": [], "issueId": "IT-25"}, "emitted_at": 1697453328636} +{"stream": "issue_votes", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-24/votes", "votes": 1, "hasVoted": true, "voters": [{"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}], "issueId": "IT-24"}, "emitted_at": 1697453328922} +{"stream": "issue_watchers", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-26/watchers", "isWatching": true, "watchCount": 1, "watchers": [{"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}], "issueId": "IT-26"}, "emitted_at": 1697453337918} +{"stream": "issue_watchers", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-25/watchers", "isWatching": true, "watchCount": 1, "watchers": [{"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}], "issueId": "IT-25"}, "emitted_at": 1697453338218} +{"stream": "issue_watchers", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/IT-24/watchers", "isWatching": true, "watchCount": 1, "watchers": [{"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}], "issueId": "IT-24"}, "emitted_at": 1697453338484} +{"stream": "issue_worklogs", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10626/worklog/11820", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058%3A295406f3-a1fc-4733-b906-dd15d021bd79", "accountId": "557058:295406f3-a1fc-4733-b906-dd15d021bd79", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "24x24": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "16x16": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png", "32x32": "https://secure.gravatar.com/avatar/182fc208a1a2e6cc41393ab6c9363d9c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FTT-6.png"}, "displayName": "Tempo Timesheets", "active": true, "timeZone": "America/Los_Angeles", "accountType": "app"}, "comment": {"version": 1, "type": "doc", "content": [{"type": "paragraph", "content": [{"type": "text", "text": "time-tracking"}]}]}, "created": "2023-04-05T05:08:50.033000-07:00", "updated": "2023-04-05T05:08:50.033000-07:00", "started": "2023-04-05T01:00:00-07:00", "timeSpent": "1d", "timeSpentSeconds": 28800, "id": "11820", "issueId": "10626"}, "emitted_at": 1697453347512} +{"stream": "issue_worklogs", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11708", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 0", "type": "text"}]}]}, "created": "2021-04-15T11:39:46.574000-07:00", "updated": "2021-04-15T11:39:46.574000-07:00", "started": "2021-04-14T18:48:52.747000-07:00", "timeSpent": "2h 21m", "timeSpentSeconds": 8460, "id": "11708", "issueId": "10080"}, "emitted_at": 1697453348137} +{"stream": "issue_worklogs", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/issue/10080/worklog/11709", "author": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "updateAuthor": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "accountType": "atlassian"}, "comment": {"type": "doc", "version": 1, "content": [{"type": "paragraph", "content": [{"text": "I did some work here. 1", "type": "text"}]}]}, "created": "2021-04-15T11:39:47.215000-07:00", "updated": "2021-04-15T11:39:47.215000-07:00", "started": "2021-04-14T18:48:52.747000-07:00", "timeSpent": "37m", "timeSpentSeconds": 2220, "id": "11709", "issueId": "10080"}, "emitted_at": 1697453348138} +{"stream": "jira_settings", "data": {"id": "jira.issuenav.criteria.autoupdate", "key": "jira.issuenav.criteria.autoupdate", "value": "true", "name": "Auto Update Criteria", "desc": "Turn on to update search results automatically", "type": "boolean"}, "emitted_at": 1697453357398} +{"stream": "jira_settings", "data": {"id": "jira.clone.prefix", "key": "jira.clone.prefix", "value": "CLONE -", "name": "The prefix added to the Summary field of cloned issues", "type": "string"}, "emitted_at": 1697453357400} +{"stream": "jira_settings", "data": {"id": "jira.date.picker.java.format", "key": "jira.date.picker.java.format", "value": "d/MMM/yy", "name": "Date Picker Format (Java)", "desc": "This part is only for the Java (server side) generated dates. Note that this should correspond to the javascript date picker format (jira.date.picker.javascript.format) setting.", "type": "string"}, "emitted_at": 1697453357400} +{"stream": "labels", "data": {"label": "Label10"}, "emitted_at": 1697453358179} +{"stream": "labels", "data": {"label": "Label3"}, "emitted_at": 1697453358180} +{"stream": "labels", "data": {"label": "Label4"}, "emitted_at": 1697453358180} +{"stream": "permissions", "data": {"key": "ADD_COMMENTS", "name": "Add Comments", "type": "PROJECT", "description": "Ability to comment on issues."}, "emitted_at": 1697453358813} +{"stream": "permissions", "data": {"key": "ADMINISTER", "name": "Administer Jira", "type": "GLOBAL", "description": "Create and administer projects, issue types, fields, workflows, and schemes for all projects. Users with this permission can perform most administration tasks, except: managing users, importing data, and editing system email settings."}, "emitted_at": 1697453358814} +{"stream": "permissions", "data": {"key": "ADMINISTER_PROJECTS", "name": "Administer Projects", "type": "PROJECT", "description": "Ability to administer a project in Jira."}, "emitted_at": 1697453358814} +{"stream": "permission_schemes", "data": {"expand": "permissions,user,group,projectRole,field,all", "id": 10056, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056", "name": "CAW software permission scheme", "description": "The permission scheme for Jira Software Free. In Free, any registered user can access and administer this project.", "permissions": [{"id": 14200, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14200", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ADD_COMMENTS"}, {"id": 14201, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14201", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ADMINISTER_PROJECTS"}, {"id": 14202, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14202", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ASSIGNABLE_USER"}, {"id": 14203, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14203", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ASSIGN_ISSUES"}, {"id": 14204, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14204", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "BROWSE_PROJECTS"}, {"id": 14205, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14205", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CLOSE_ISSUES"}, {"id": 14206, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14206", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CREATE_ATTACHMENTS"}, {"id": 14207, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14207", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CREATE_ISSUES"}, {"id": 14208, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14208", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_ATTACHMENTS"}, {"id": 14209, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14209", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_COMMENTS"}, {"id": 14210, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14210", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_WORKLOGS"}, {"id": 14211, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14211", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ISSUES"}, {"id": 14212, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14212", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_ATTACHMENTS"}, {"id": 14213, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14213", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_COMMENTS"}, {"id": 14214, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14214", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_WORKLOGS"}, {"id": 14215, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14215", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ALL_COMMENTS"}, {"id": 14216, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14216", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ALL_WORKLOGS"}, {"id": 14217, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14217", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ISSUES"}, {"id": 14218, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14218", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_OWN_COMMENTS"}, {"id": 14219, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14219", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_OWN_WORKLOGS"}, {"id": 14220, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14220", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "LINK_ISSUES"}, {"id": 14221, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14221", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MANAGE_SPRINTS_PERMISSION"}, {"id": 14222, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14222", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MANAGE_WATCHERS"}, {"id": 14223, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14223", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MODIFY_REPORTER"}, {"id": 14224, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14224", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MOVE_ISSUES"}, {"id": 14225, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14225", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "RESOLVE_ISSUES"}, {"id": 14226, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14226", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SCHEDULE_ISSUES"}, {"id": 14227, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14227", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "TRANSITION_ISSUES"}, {"id": 14228, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14228", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_DEV_TOOLS"}, {"id": 14229, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14229", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_READONLY_WORKFLOW"}, {"id": 14230, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14230", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_VOTERS_AND_WATCHERS"}, {"id": 14231, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14231", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "WORK_ON_ISSUES"}, {"id": 14232, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14232", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__log-work-for-others"}, {"id": 14233, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14233", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__set-billable-hours"}, {"id": 14234, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14234", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__view-all-worklogs"}, {"id": 14235, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14235", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__view-issue-hours"}, {"id": 14236, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14236", "holder": {"type": "applicationRole"}, "permission": "ADMINISTER_PROJECTS"}, {"id": 14237, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14237", "holder": {"type": "applicationRole"}, "permission": "BROWSE_PROJECTS"}, {"id": 14238, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14238", "holder": {"type": "applicationRole"}, "permission": "MANAGE_SPRINTS_PERMISSION"}, {"id": 14239, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14239", "holder": {"type": "applicationRole"}, "permission": "VIEW_DEV_TOOLS"}, {"id": 14240, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14240", "holder": {"type": "applicationRole"}, "permission": "VIEW_READONLY_WORKFLOW"}, {"id": 14241, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14241", "holder": {"type": "applicationRole"}, "permission": "ASSIGNABLE_USER"}, {"id": 14242, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14242", "holder": {"type": "applicationRole"}, "permission": "CLOSE_ISSUES"}, {"id": 14243, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14243", "holder": {"type": "applicationRole"}, "permission": "CREATE_ISSUES"}, {"id": 14244, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14244", "holder": {"type": "applicationRole"}, "permission": "DELETE_ISSUES"}, {"id": 14245, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14245", "holder": {"type": "applicationRole"}, "permission": "EDIT_ISSUES"}, {"id": 14246, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14246", "holder": {"type": "applicationRole"}, "permission": "LINK_ISSUES"}, {"id": 14247, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14247", "holder": {"type": "applicationRole"}, "permission": "MODIFY_REPORTER"}, {"id": 14248, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14248", "holder": {"type": "applicationRole"}, "permission": "MOVE_ISSUES"}, {"id": 14249, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14249", "holder": {"type": "applicationRole"}, "permission": "RESOLVE_ISSUES"}, {"id": 14250, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14250", "holder": {"type": "applicationRole"}, "permission": "SCHEDULE_ISSUES"}, {"id": 14251, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14251", "holder": {"type": "applicationRole"}, "permission": "TRANSITION_ISSUES"}, {"id": 14252, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14252", "holder": {"type": "applicationRole"}, "permission": "MANAGE_WATCHERS"}, {"id": 14253, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14253", "holder": {"type": "applicationRole"}, "permission": "VIEW_VOTERS_AND_WATCHERS"}, {"id": 14254, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14254", "holder": {"type": "applicationRole"}, "permission": "ADD_COMMENTS"}, {"id": 14255, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14255", "holder": {"type": "applicationRole"}, "permission": "DELETE_ALL_COMMENTS"}, {"id": 14256, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14256", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_COMMENTS"}, {"id": 14257, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14257", "holder": {"type": "applicationRole"}, "permission": "EDIT_ALL_COMMENTS"}, {"id": 14258, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14258", "holder": {"type": "applicationRole"}, "permission": "EDIT_OWN_COMMENTS"}, {"id": 14259, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14259", "holder": {"type": "applicationRole"}, "permission": "CREATE_ATTACHMENTS"}, {"id": 14260, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14260", "holder": {"type": "applicationRole"}, "permission": "DELETE_ALL_ATTACHMENTS"}, {"id": 14261, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14261", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_ATTACHMENTS"}, {"id": 14262, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14262", "holder": {"type": "applicationRole"}, "permission": "DELETE_ALL_WORKLOGS"}, {"id": 14263, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14263", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_WORKLOGS"}, {"id": 14264, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14264", "holder": {"type": "applicationRole"}, "permission": "EDIT_ALL_WORKLOGS"}, {"id": 14265, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14265", "holder": {"type": "applicationRole"}, "permission": "EDIT_OWN_WORKLOGS"}, {"id": 14266, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14266", "holder": {"type": "applicationRole"}, "permission": "WORK_ON_ISSUES"}, {"id": 14267, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14267", "holder": {"type": "applicationRole"}, "permission": "ASSIGN_ISSUES"}, {"id": 14404, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14404", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SET_ISSUE_SECURITY"}, {"id": 14481, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14481", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_AGGREGATED_DATA"}, {"id": 14542, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14542", "holder": {"type": "applicationRole"}, "permission": "VIEW_AGGREGATED_DATA"}, {"id": 14714, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14714", "holder": {"type": "applicationRole"}, "permission": "VIEW_PROJECTS"}, {"id": 14715, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14715", "holder": {"type": "applicationRole"}, "permission": "VIEW_ISSUES"}, {"id": 14836, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14836", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_PROJECTS"}, {"id": 14837, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/14837", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_ISSUES"}, {"id": 15117, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10056/permission/15117", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SERVICEDESK_AGENT"}]}, "emitted_at": 1697453360118} +{"stream": "permission_schemes", "data": {"expand": "permissions,user,group,projectRole,field,all", "id": 10055, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055", "name": "CLK software permission scheme", "description": "The permission scheme for Jira Software Free. In Free, any registered user can access and administer this project.", "permissions": [{"id": 14132, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14132", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ADD_COMMENTS"}, {"id": 14133, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14133", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ADMINISTER_PROJECTS"}, {"id": 14134, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14134", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ASSIGNABLE_USER"}, {"id": 14135, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14135", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ASSIGN_ISSUES"}, {"id": 14136, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14136", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "BROWSE_PROJECTS"}, {"id": 14137, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14137", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CLOSE_ISSUES"}, {"id": 14138, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14138", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CREATE_ATTACHMENTS"}, {"id": 14139, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14139", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CREATE_ISSUES"}, {"id": 14140, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14140", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_ATTACHMENTS"}, {"id": 14141, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14141", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_COMMENTS"}, {"id": 14142, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14142", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_WORKLOGS"}, {"id": 14143, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14143", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ISSUES"}, {"id": 14144, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14144", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_ATTACHMENTS"}, {"id": 14145, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14145", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_COMMENTS"}, {"id": 14146, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14146", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_WORKLOGS"}, {"id": 14147, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14147", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ALL_COMMENTS"}, {"id": 14148, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14148", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ALL_WORKLOGS"}, {"id": 14149, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14149", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ISSUES"}, {"id": 14150, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14150", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_OWN_COMMENTS"}, {"id": 14151, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14151", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_OWN_WORKLOGS"}, {"id": 14152, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14152", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "LINK_ISSUES"}, {"id": 14153, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14153", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MANAGE_SPRINTS_PERMISSION"}, {"id": 14154, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14154", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MANAGE_WATCHERS"}, {"id": 14155, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14155", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MODIFY_REPORTER"}, {"id": 14156, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14156", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MOVE_ISSUES"}, {"id": 14157, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14157", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "RESOLVE_ISSUES"}, {"id": 14158, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14158", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SCHEDULE_ISSUES"}, {"id": 14159, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14159", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "TRANSITION_ISSUES"}, {"id": 14160, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14160", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_DEV_TOOLS"}, {"id": 14161, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14161", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_READONLY_WORKFLOW"}, {"id": 14162, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14162", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_VOTERS_AND_WATCHERS"}, {"id": 14163, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14163", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "WORK_ON_ISSUES"}, {"id": 14164, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14164", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__log-work-for-others"}, {"id": 14165, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14165", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__set-billable-hours"}, {"id": 14166, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14166", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__view-all-worklogs"}, {"id": 14167, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14167", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__view-issue-hours"}, {"id": 14168, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14168", "holder": {"type": "applicationRole"}, "permission": "ADMINISTER_PROJECTS"}, {"id": 14169, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14169", "holder": {"type": "applicationRole"}, "permission": "BROWSE_PROJECTS"}, {"id": 14170, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14170", "holder": {"type": "applicationRole"}, "permission": "MANAGE_SPRINTS_PERMISSION"}, {"id": 14171, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14171", "holder": {"type": "applicationRole"}, "permission": "VIEW_DEV_TOOLS"}, {"id": 14172, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14172", "holder": {"type": "applicationRole"}, "permission": "VIEW_READONLY_WORKFLOW"}, {"id": 14173, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14173", "holder": {"type": "applicationRole"}, "permission": "ASSIGNABLE_USER"}, {"id": 14174, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14174", "holder": {"type": "applicationRole"}, "permission": "CLOSE_ISSUES"}, {"id": 14175, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14175", "holder": {"type": "applicationRole"}, "permission": "CREATE_ISSUES"}, {"id": 14176, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14176", "holder": {"type": "applicationRole"}, "permission": "DELETE_ISSUES"}, {"id": 14177, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14177", "holder": {"type": "applicationRole"}, "permission": "EDIT_ISSUES"}, {"id": 14178, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14178", "holder": {"type": "applicationRole"}, "permission": "LINK_ISSUES"}, {"id": 14179, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14179", "holder": {"type": "applicationRole"}, "permission": "MODIFY_REPORTER"}, {"id": 14180, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14180", "holder": {"type": "applicationRole"}, "permission": "MOVE_ISSUES"}, {"id": 14181, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14181", "holder": {"type": "applicationRole"}, "permission": "RESOLVE_ISSUES"}, {"id": 14182, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14182", "holder": {"type": "applicationRole"}, "permission": "SCHEDULE_ISSUES"}, {"id": 14183, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14183", "holder": {"type": "applicationRole"}, "permission": "TRANSITION_ISSUES"}, {"id": 14184, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14184", "holder": {"type": "applicationRole"}, "permission": "MANAGE_WATCHERS"}, {"id": 14185, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14185", "holder": {"type": "applicationRole"}, "permission": "VIEW_VOTERS_AND_WATCHERS"}, {"id": 14186, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14186", "holder": {"type": "applicationRole"}, "permission": "ADD_COMMENTS"}, {"id": 14187, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14187", "holder": {"type": "applicationRole"}, "permission": "DELETE_ALL_COMMENTS"}, {"id": 14188, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14188", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_COMMENTS"}, {"id": 14189, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14189", "holder": {"type": "applicationRole"}, "permission": "EDIT_ALL_COMMENTS"}, {"id": 14190, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14190", "holder": {"type": "applicationRole"}, "permission": "EDIT_OWN_COMMENTS"}, {"id": 14191, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14191", "holder": {"type": "applicationRole"}, "permission": "CREATE_ATTACHMENTS"}, {"id": 14192, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14192", "holder": {"type": "applicationRole"}, "permission": "DELETE_ALL_ATTACHMENTS"}, {"id": 14193, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14193", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_ATTACHMENTS"}, {"id": 14194, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14194", "holder": {"type": "applicationRole"}, "permission": "DELETE_ALL_WORKLOGS"}, {"id": 14195, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14195", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_WORKLOGS"}, {"id": 14196, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14196", "holder": {"type": "applicationRole"}, "permission": "EDIT_ALL_WORKLOGS"}, {"id": 14197, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14197", "holder": {"type": "applicationRole"}, "permission": "EDIT_OWN_WORKLOGS"}, {"id": 14198, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14198", "holder": {"type": "applicationRole"}, "permission": "WORK_ON_ISSUES"}, {"id": 14199, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14199", "holder": {"type": "applicationRole"}, "permission": "ASSIGN_ISSUES"}, {"id": 14405, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14405", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SET_ISSUE_SECURITY"}, {"id": 14482, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14482", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_AGGREGATED_DATA"}, {"id": 14543, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14543", "holder": {"type": "applicationRole"}, "permission": "VIEW_AGGREGATED_DATA"}, {"id": 14712, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14712", "holder": {"type": "applicationRole"}, "permission": "VIEW_PROJECTS"}, {"id": 14713, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14713", "holder": {"type": "applicationRole"}, "permission": "VIEW_ISSUES"}, {"id": 14834, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14834", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_PROJECTS"}, {"id": 14835, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/14835", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_ISSUES"}, {"id": 15118, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/10055/permission/15118", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SERVICEDESK_AGENT"}]}, "emitted_at": 1697453360125} +{"stream": "permission_schemes", "data": {"expand": "permissions,user,group,projectRole,field,all", "id": 0, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0", "name": "Default Permission Scheme", "description": "This is the default Permission Scheme. Any new projects that are created will be assigned this scheme.", "permissions": [{"id": 10004, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10004", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "ADMINISTER_PROJECTS"}, {"id": 10005, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10005", "holder": {"type": "applicationRole"}, "permission": "BROWSE_PROJECTS"}, {"id": 10006, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10006", "holder": {"type": "applicationRole"}, "permission": "CREATE_ISSUES"}, {"id": 10007, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10007", "holder": {"type": "applicationRole"}, "permission": "ADD_COMMENTS"}, {"id": 10008, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10008", "holder": {"type": "applicationRole"}, "permission": "CREATE_ATTACHMENTS"}, {"id": 10009, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10009", "holder": {"type": "applicationRole"}, "permission": "ASSIGN_ISSUES"}, {"id": 10010, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10010", "holder": {"type": "applicationRole"}, "permission": "ASSIGNABLE_USER"}, {"id": 10011, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10011", "holder": {"type": "applicationRole"}, "permission": "RESOLVE_ISSUES"}, {"id": 10012, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10012", "holder": {"type": "applicationRole"}, "permission": "LINK_ISSUES"}, {"id": 10013, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10013", "holder": {"type": "applicationRole"}, "permission": "EDIT_ISSUES"}, {"id": 10014, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10014", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "DELETE_ISSUES"}, {"id": 10015, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10015", "holder": {"type": "applicationRole"}, "permission": "CLOSE_ISSUES"}, {"id": 10016, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10016", "holder": {"type": "applicationRole"}, "permission": "MOVE_ISSUES"}, {"id": 10017, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10017", "holder": {"type": "applicationRole"}, "permission": "SCHEDULE_ISSUES"}, {"id": 10018, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10018", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "MODIFY_REPORTER"}, {"id": 10019, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10019", "holder": {"type": "applicationRole"}, "permission": "WORK_ON_ISSUES"}, {"id": 10020, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10020", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "DELETE_ALL_WORKLOGS"}, {"id": 10021, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10021", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_WORKLOGS"}, {"id": 10022, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10022", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "EDIT_ALL_WORKLOGS"}, {"id": 10023, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10023", "holder": {"type": "applicationRole"}, "permission": "EDIT_OWN_WORKLOGS"}, {"id": 10024, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10024", "holder": {"type": "applicationRole"}, "permission": "VIEW_VOTERS_AND_WATCHERS"}, {"id": 10025, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10025", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "MANAGE_WATCHERS"}, {"id": 10026, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10026", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "EDIT_ALL_COMMENTS"}, {"id": 10027, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10027", "holder": {"type": "applicationRole"}, "permission": "EDIT_OWN_COMMENTS"}, {"id": 10028, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10028", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "DELETE_ALL_COMMENTS"}, {"id": 10029, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10029", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_COMMENTS"}, {"id": 10030, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10030", "holder": {"type": "projectRole", "parameter": "10002", "value": "10002", "expand": "projectRole"}, "permission": "DELETE_ALL_ATTACHMENTS"}, {"id": 10031, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10031", "holder": {"type": "applicationRole"}, "permission": "DELETE_OWN_ATTACHMENTS"}, {"id": 10033, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10033", "holder": {"type": "applicationRole"}, "permission": "VIEW_DEV_TOOLS"}, {"id": 10200, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10200", "holder": {"type": "applicationRole"}, "permission": "VIEW_READONLY_WORKFLOW"}, {"id": 10300, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10300", "holder": {"type": "applicationRole"}, "permission": "TRANSITION_ISSUES"}, {"id": 10301, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10301", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ADD_COMMENTS"}, {"id": 10302, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10302", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ADMINISTER_PROJECTS"}, {"id": 10303, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10303", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ASSIGNABLE_USER"}, {"id": 10304, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10304", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "ASSIGN_ISSUES"}, {"id": 10305, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10305", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "BROWSE_PROJECTS"}, {"id": 10306, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10306", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CLOSE_ISSUES"}, {"id": 10307, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10307", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CREATE_ATTACHMENTS"}, {"id": 10308, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10308", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "CREATE_ISSUES"}, {"id": 10309, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10309", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_ATTACHMENTS"}, {"id": 10310, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10310", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_COMMENTS"}, {"id": 10311, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10311", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ALL_WORKLOGS"}, {"id": 10312, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10312", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_ISSUES"}, {"id": 10313, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10313", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_ATTACHMENTS"}, {"id": 10314, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10314", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_COMMENTS"}, {"id": 10315, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10315", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "DELETE_OWN_WORKLOGS"}, {"id": 10316, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10316", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ALL_COMMENTS"}, {"id": 10317, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10317", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ALL_WORKLOGS"}, {"id": 10318, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10318", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_ISSUES"}, {"id": 10319, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10319", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_OWN_COMMENTS"}, {"id": 10320, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10320", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "EDIT_OWN_WORKLOGS"}, {"id": 10321, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10321", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "LINK_ISSUES"}, {"id": 10322, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10322", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MANAGE_SPRINTS_PERMISSION"}, {"id": 10323, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10323", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MANAGE_WATCHERS"}, {"id": 10324, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10324", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MODIFY_REPORTER"}, {"id": 10325, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10325", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "MOVE_ISSUES"}, {"id": 10326, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10326", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "RESOLVE_ISSUES"}, {"id": 10327, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10327", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SCHEDULE_ISSUES"}, {"id": 10328, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10328", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SET_ISSUE_SECURITY"}, {"id": 10329, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10329", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "TRANSITION_ISSUES"}, {"id": 10330, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10330", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_DEV_TOOLS"}, {"id": 10331, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10331", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_READONLY_WORKFLOW"}, {"id": 10332, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10332", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_VOTERS_AND_WATCHERS"}, {"id": 10333, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10333", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "WORK_ON_ISSUES"}, {"id": 10464, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10464", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__log-work-for-others"}, {"id": 10465, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10465", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__set-billable-hours"}, {"id": 10466, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10466", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__view-all-worklogs"}, {"id": 10467, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/10467", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "io.tempo.jira__view-issue-hours"}, {"id": 14538, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/14538", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_AGGREGATED_DATA"}, {"id": 14599, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/14599", "holder": {"type": "applicationRole"}, "permission": "VIEW_AGGREGATED_DATA"}, {"id": 14600, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/14600", "holder": {"type": "applicationRole"}, "permission": "VIEW_PROJECTS"}, {"id": 14601, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/14601", "holder": {"type": "applicationRole"}, "permission": "VIEW_ISSUES"}, {"id": 14722, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/14722", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_PROJECTS"}, {"id": 14723, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/14723", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "VIEW_ISSUES"}, {"id": 15119, "self": "https://airbyteio.atlassian.net/rest/api/3/permissionscheme/0/permission/15119", "holder": {"type": "projectRole", "parameter": "10003", "value": "10003", "expand": "projectRole"}, "permission": "SERVICEDESK_AGENT"}]}, "emitted_at": 1697453360131} +{"stream": "projects", "data": {"expand": "description,lead,issueTypes,url,projectKeys,permissions,insight", "self": "https://airbyteio.atlassian.net/rest/api/3/project/10000", "id": "10000", "key": "IT", "description": "Test", "lead": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "name": "integration-tests", "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10424?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10004", "id": "10004", "name": "Test category 2", "description": "Test Project Category 2"}, "projectTypeKey": "software", "simplified": false, "style": "classic", "isPrivate": false, "properties": {}}, "emitted_at": 1697453360572} +{"stream": "projects", "data": {"expand": "description,lead,issueTypes,url,projectKeys,permissions,insight", "self": "https://airbyteio.atlassian.net/rest/api/3/project/10016", "id": "10016", "key": "TESTKEY13", "description": "Test project 13 description", "lead": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "name": "Test project 13", "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10425?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10000", "id": "10000", "name": "Category 1", "description": "Category 1"}, "projectTypeKey": "software", "simplified": false, "style": "classic", "isPrivate": false, "properties": {}}, "emitted_at": 1697453360573} +{"stream": "projects", "data": {"expand": "description,lead,issueTypes,url,projectKeys,permissions,insight", "self": "https://airbyteio.atlassian.net/rest/api/3/project/10064", "id": "10064", "key": "TTMP2", "description": "", "lead": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "name": "Test Team Managed Project 2", "avatarUrls": {"48x48": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10412", "24x24": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10412?size=small", "16x16": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10412?size=xsmall", "32x32": "https://airbyteio.atlassian.net/rest/api/3/universal_avatar/view/type/project/avatar/10412?size=medium"}, "projectCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10000", "id": "10000", "name": "Category 1", "description": "Category 1"}, "projectTypeKey": "software", "simplified": true, "style": "next-gen", "isPrivate": false, "properties": {}, "entityId": "6fc48839-dfa5-487d-ad8f-8b540f1748d7", "uuid": "6fc48839-dfa5-487d-ad8f-8b540f1748d7"}, "emitted_at": 1697453360578} +{"stream": "project_roles", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/role/10032", "name": "Administrator", "id": 10032, "description": "Admins can do most things, like update settings and add other admins.", "scope": {"type": "PROJECT", "project": {"id": "10064"}}}, "emitted_at": 1697453361611} +{"stream": "project_roles", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/role/10035", "name": "atlassian-addons-project-access", "id": 10035, "description": "A project role that represents Connect add-ons declaring a scope that requires more than read issue permissions", "scope": {"type": "PROJECT", "project": {"id": "10064"}}}, "emitted_at": 1697453361612} +{"stream": "project_roles", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/role/10033", "name": "Member", "id": 10033, "description": "Members are part of the team, and can add, edit, and collaborate on all work.", "scope": {"type": "PROJECT", "project": {"id": "10064"}}}, "emitted_at": 1697453361612} +{"stream": "project_avatars", "data": {"id": "10400", "isSystemAvatar": true, "isSelected": false, "isDeletable": false, "urls": {"16x16": "/secure/viewavatar?size=xsmall&avatarId=10400&avatarType=project", "24x24": "/secure/viewavatar?size=small&avatarId=10400&avatarType=project", "32x32": "/secure/viewavatar?size=medium&avatarId=10400&avatarType=project", "48x48": "/secure/viewavatar?avatarId=10400&avatarType=project"}, "projectId": "IT"}, "emitted_at": 1697453362355} +{"stream": "project_avatars", "data": {"id": "10401", "isSystemAvatar": true, "isSelected": false, "isDeletable": false, "urls": {"16x16": "/secure/viewavatar?size=xsmall&avatarId=10401&avatarType=project", "24x24": "/secure/viewavatar?size=small&avatarId=10401&avatarType=project", "32x32": "/secure/viewavatar?size=medium&avatarId=10401&avatarType=project", "48x48": "/secure/viewavatar?avatarId=10401&avatarType=project"}, "projectId": "IT"}, "emitted_at": 1697453362356} +{"stream": "project_avatars", "data": {"id": "10402", "isSystemAvatar": true, "isSelected": false, "isDeletable": false, "urls": {"16x16": "/secure/viewavatar?size=xsmall&avatarId=10402&avatarType=project", "24x24": "/secure/viewavatar?size=small&avatarId=10402&avatarType=project", "32x32": "/secure/viewavatar?size=medium&avatarId=10402&avatarType=project", "48x48": "/secure/viewavatar?avatarId=10402&avatarType=project"}, "projectId": "IT"}, "emitted_at": 1697453362357} +{"stream": "project_categories", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10000", "id": "10000", "description": "Category 1", "name": "Category 1"}, "emitted_at": 1697453364009} +{"stream": "project_categories", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10001", "id": "10001", "description": "Category 2", "name": "Category 2"}, "emitted_at": 1697453364010} +{"stream": "project_categories", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/projectCategory/10002", "id": "10002", "description": "Test Project Category 0", "name": "Test category 0"}, "emitted_at": 1697453364011} +{"stream": "project_components", "data": {"componentBean": {"self": "https://airbyteio.atlassian.net/rest/api/3/component/10047", "id": "10047", "name": "Component 0", "description": "This is a Jira component", "lead": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "assigneeType": "PROJECT_LEAD", "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "realAssigneeType": "PROJECT_LEAD", "realAssignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "isAssigneeTypeValid": true, "project": "IT", "projectId": 10000}, "issueCount": 0, "realAssignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "isAssigneeTypeValid": true, "realAssigneeType": "PROJECT_LEAD", "self": "https://airbyteio.atlassian.net/rest/api/3/component/10047", "description": "This is a Jira component", "projectId": 10000, "project": "IT", "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "assigneeType": "PROJECT_LEAD", "lead": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "name": "Component 0", "id": "10047"}, "emitted_at": 1697453364690} +{"stream": "project_components", "data": {"componentBean": {"self": "https://airbyteio.atlassian.net/rest/api/3/component/10000", "id": "10000", "name": "Component 1", "description": "Component 1", "lead": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "assigneeType": "PROJECT_DEFAULT", "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "realAssigneeType": "PROJECT_DEFAULT", "realAssignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "isAssigneeTypeValid": true, "project": "IT", "projectId": 10000}, "issueCount": 0, "realAssignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "isAssigneeTypeValid": true, "realAssigneeType": "PROJECT_DEFAULT", "self": "https://airbyteio.atlassian.net/rest/api/3/component/10000", "description": "Component 1", "projectId": 10000, "project": "IT", "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "assigneeType": "PROJECT_DEFAULT", "lead": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "name": "Component 1", "id": "10000"}, "emitted_at": 1697453364691} +{"stream": "project_components", "data": {"componentBean": {"self": "https://airbyteio.atlassian.net/rest/api/3/component/10048", "id": "10048", "name": "Component 2", "description": "This is a Jira component", "lead": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "assigneeType": "PROJECT_LEAD", "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "realAssigneeType": "PROJECT_LEAD", "realAssignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "isAssigneeTypeValid": true, "project": "IT", "projectId": 10000}, "issueCount": 0, "realAssignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "isAssigneeTypeValid": true, "realAssigneeType": "PROJECT_LEAD", "self": "https://airbyteio.atlassian.net/rest/api/3/component/10048", "description": "This is a Jira component", "projectId": 10000, "project": "IT", "assignee": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "assigneeType": "PROJECT_LEAD", "lead": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true}, "name": "Component 2", "id": "10048"}, "emitted_at": 1697453364692} +{"stream": "project_email", "data": {"emailAddress": "jira@airbyteio.atlassian.net", "projectId": "10000"}, "emitted_at": 1697453366010} +{"stream": "project_email", "data": {"emailAddress": "jira@airbyteio.atlassian.net", "projectId": "10016"}, "emitted_at": 1697453366292} +{"stream": "project_email", "data": {"emailAddress": "jira@airbyteio.atlassian.net", "projectId": "10064"}, "emitted_at": 1697453366602} +{"stream": "project_types", "data": {"key": "product_discovery", "formattedKey": "Product Discovery", "descriptionI18nKey": "jira.project.type.polaris.description", "icon": "PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iaXNvLTg4NTktMSI/Pg0KPCEtLSBHZW5lcmF0b3I6IEFkb2JlIElsbHVzdHJhdG9yIDE4LjEuMSwgU1ZHIEV4cG9ydCBQbHVnLUluIC4gU1ZHIFZlcnNpb246IDYuMDAgQnVpbGQgMCkgIC0tPg0KPHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJMYXllcl8xIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCINCgkgdmlld0JveD0iMCAwIDMwMCAzMDAiIHN0eWxlPSJlbmFibGUtYmFja2dyb3VuZDpuZXcgMCAwIDMwMCAzMDA7IiB4bWw6c3BhY2U9InByZXNlcnZlIj4NCjxnIGlkPSJMYXllcl8yIj4NCgk8cGF0aCBzdHlsZT0iZmlsbDojRjc5MjMyOyIgZD0iTTE1MCwwQzY2LjY2NywwLDAsNjYuNjY3LDAsMTUwczY2LjY2NywxNTAsMTUwLDE1MHMxNTAtNjYuNjY3LDE1MC0xNTBTMjMzLjMzMywwLDE1MCwweg0KCQkgTTEzNi42NjcsMTc4LjMzM0wxMjUsMTkwbC00MS42NjctNDBMOTUsMTM4LjMzM2wzMC0zMEwxMzYuNjY3LDEyMGwtMzAsMzBMMTM2LjY2NywxNzguMzMzeiBNMjA1LDE2MS42NjdsLTMwLDMwTDE2My4zMzMsMTgwDQoJCWwzMC0zMGwtMzAtMzBMMTc1LDEwOC4zMzNMMjE2LjY2NywxNTBMMjA1LDE2MS42Njd6Ii8+DQo8L2c+DQo8Zz4NCgk8cG9seWdvbiBzdHlsZT0iZmlsbDojRkZGRkZGOyIgcG9pbnRzPSIxNzUsMTkxLjY2NyAyMDUsMTYxLjY2NyAyMTYuNjY3LDE1MCAxNzUsMTA4LjMzMyAxNjMuMzMzLDEyMCAxOTMuMzMzLDE1MCAxNjMuMzMzLDE4MCAJIi8+DQoJPHBvbHlnb24gc3R5bGU9ImZpbGw6I0ZGRkZGRjsiIHBvaW50cz0iMTI1LDEwOC4zMzMgOTUsMTM4LjMzMyA4My4zMzMsMTUwIDEyNSwxOTAgMTM2LjY2NywxNzguMzMzIDEwNi42NjcsMTUwIDEzNi42NjcsMTIwIAkiLz4NCjwvZz4NCjwvc3ZnPg0K", "color": "#F5A623"}, "emitted_at": 1697453369323} +{"stream": "project_types", "data": {"key": "software", "formattedKey": "Software", "descriptionI18nKey": "jira.project.type.software.description", "icon": "PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iaXNvLTg4NTktMSI/Pg0KPCEtLSBHZW5lcmF0b3I6IEFkb2JlIElsbHVzdHJhdG9yIDE4LjEuMSwgU1ZHIEV4cG9ydCBQbHVnLUluIC4gU1ZHIFZlcnNpb246IDYuMDAgQnVpbGQgMCkgIC0tPg0KPHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJMYXllcl8xIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCINCgkgdmlld0JveD0iMCAwIDMwMCAzMDAiIHN0eWxlPSJlbmFibGUtYmFja2dyb3VuZDpuZXcgMCAwIDMwMCAzMDA7IiB4bWw6c3BhY2U9InByZXNlcnZlIj4NCjxnIGlkPSJMYXllcl8yIj4NCgk8cGF0aCBzdHlsZT0iZmlsbDojRjc5MjMyOyIgZD0iTTE1MCwwQzY2LjY2NywwLDAsNjYuNjY3LDAsMTUwczY2LjY2NywxNTAsMTUwLDE1MHMxNTAtNjYuNjY3LDE1MC0xNTBTMjMzLjMzMywwLDE1MCwweg0KCQkgTTEzNi42NjcsMTc4LjMzM0wxMjUsMTkwbC00MS42NjctNDBMOTUsMTM4LjMzM2wzMC0zMEwxMzYuNjY3LDEyMGwtMzAsMzBMMTM2LjY2NywxNzguMzMzeiBNMjA1LDE2MS42NjdsLTMwLDMwTDE2My4zMzMsMTgwDQoJCWwzMC0zMGwtMzAtMzBMMTc1LDEwOC4zMzNMMjE2LjY2NywxNTBMMjA1LDE2MS42Njd6Ii8+DQo8L2c+DQo8Zz4NCgk8cG9seWdvbiBzdHlsZT0iZmlsbDojRkZGRkZGOyIgcG9pbnRzPSIxNzUsMTkxLjY2NyAyMDUsMTYxLjY2NyAyMTYuNjY3LDE1MCAxNzUsMTA4LjMzMyAxNjMuMzMzLDEyMCAxOTMuMzMzLDE1MCAxNjMuMzMzLDE4MCAJIi8+DQoJPHBvbHlnb24gc3R5bGU9ImZpbGw6I0ZGRkZGRjsiIHBvaW50cz0iMTI1LDEwOC4zMzMgOTUsMTM4LjMzMyA4My4zMzMsMTUwIDEyNSwxOTAgMTM2LjY2NywxNzguMzMzIDEwNi42NjcsMTUwIDEzNi42NjcsMTIwIAkiLz4NCjwvZz4NCjwvc3ZnPg0K", "color": "#F5A623"}, "emitted_at": 1697453369324} +{"stream": "project_types", "data": {"key": "service_desk", "formattedKey": "Service Desk", "descriptionI18nKey": "jira.project.type.servicedesk.description.jsm", "icon": "PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iaXNvLTg4NTktMSI/Pg0KPCEtLSBHZW5lcmF0b3I6IEFkb2JlIElsbHVzdHJhdG9yIDE4LjEuMSwgU1ZHIEV4cG9ydCBQbHVnLUluIC4gU1ZHIFZlcnNpb246IDYuMDAgQnVpbGQgMCkgIC0tPg0KPHN2ZyB2ZXJzaW9uPSIxLjEiIGlkPSJMYXllcl8xIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB4PSIwcHgiIHk9IjBweCINCgkgdmlld0JveD0iMCAwIDMwMCAzMDAiIHN0eWxlPSJlbmFibGUtYmFja2dyb3VuZDpuZXcgMCAwIDMwMCAzMDA7IiB4bWw6c3BhY2U9InByZXNlcnZlIj4NCjxnIGlkPSJMYXllcl8yIj4NCgk8Zz4NCgkJPHJlY3QgeD0iMTAwIiB5PSIxMDAiIHN0eWxlPSJmaWxsOiM2N0FCNDk7IiB3aWR0aD0iMTAwIiBoZWlnaHQ9IjY2LjY2NyIvPg0KCQk8cGF0aCBzdHlsZT0iZmlsbDojNjdBQjQ5OyIgZD0iTTE1MCwwQzY2LjY2NywwLDAsNjYuNjY3LDAsMTUwczY2LjY2NywxNTAsMTUwLDE1MHMxNTAtNjYuNjY3LDE1MC0xNTBTMjMzLjMzMywwLDE1MCwweg0KCQkJIE0yMTYuNjY3LDEwMHY2Ni42Njd2MTYuNjY3aC01MFYyMDBIMjAwdjE2LjY2N0gxMDBWMjAwaDMzLjMzM3YtMTYuNjY3aC01MHYtMTYuNjY3VjEwMFY4My4zMzNoMTMzLjMzM1YxMDB6Ii8+DQoJPC9nPg0KPC9nPg0KPHBhdGggc3R5bGU9ImZpbGw6I0ZGRkZGRjsiIGQ9Ik0yMTYuNjY3LDE4My4zMzN2LTE2LjY2N1YxMDBWODMuMzMzSDgzLjMzM1YxMDB2NjYuNjY3djE2LjY2N2g1MFYyMDBIMTAwdjE2LjY2N2gxMDBWMjAwaC0zMy4zMzMNCgl2LTE2LjY2N0gyMTYuNjY3eiBNMTAwLDE2Ni42NjdWMTAwaDEwMHY2Ni42NjdIMTAweiIvPg0KPC9zdmc+DQo=", "color": "#67AB49"}, "emitted_at": 1697453369324} +{"stream": "project_versions", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/version/10000", "id": "10000", "description": "Version 1", "name": "Version 1", "archived": false, "released": false, "startDate": "2021-02-18", "releaseDate": "2021-02-25", "overdue": true, "userStartDate": "17/Feb/21", "userReleaseDate": "24/Feb/21", "projectId": 10000}, "emitted_at": 1697453369954} +{"stream": "project_versions", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/version/10040", "id": "10040", "description": "An excellent version", "name": "New Version 0", "archived": false, "released": true, "releaseDate": "2010-07-06", "userReleaseDate": "05/Jul/10", "projectId": 10000}, "emitted_at": 1697453369955} +{"stream": "project_versions", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/version/10041", "id": "10041", "description": "An excellent version", "name": "New Version 1", "archived": false, "released": true, "releaseDate": "2010-07-06", "userReleaseDate": "05/Jul/10", "projectId": 10000}, "emitted_at": 1697453369955} +{"stream": "screens", "data": {"id": 1, "name": "Default Screen", "description": "Allows to update all system fields."}, "emitted_at": 1697453371057} +{"stream": "screens", "data": {"id": 2, "name": "Workflow Screen", "description": "This screen is used in the workflow and enables you to assign issues"}, "emitted_at": 1697453371058} +{"stream": "screens", "data": {"id": 3, "name": "Resolve Issue Screen", "description": "Allows to set resolution, change fix versions and assign an issue."}, "emitted_at": 1697453371058} +{"stream": "screen_tabs", "data": {"id": 10000, "name": "Field Tab", "screenId": 1}, "emitted_at": 1697453372487} +{"stream": "screen_tabs", "data": {"id": 10148, "name": "Tab1", "screenId": 1}, "emitted_at": 1697453372488} +{"stream": "screen_tabs", "data": {"id": 10149, "name": "Tab2", "screenId": 1}, "emitted_at": 1697453372489} +{"stream": "screen_tab_fields", "data": {"id": "summary", "name": "Summary", "screenId": 1, "tabId": 10000}, "emitted_at": 1697453418442} +{"stream": "screen_tab_fields", "data": {"id": "issuetype", "name": "Issue Type", "screenId": 1, "tabId": 10000}, "emitted_at": 1697453418443} +{"stream": "screen_tab_fields", "data": {"id": "security", "name": "Security Level", "screenId": 1, "tabId": 10000}, "emitted_at": 1697453418443} +{"stream": "screen_schemes", "data": {"id": 1, "name": "Default Screen Scheme", "description": "Default Screen Scheme", "screens": {"default": 1}}, "emitted_at": 1697453468346} +{"stream": "screen_schemes", "data": {"id": 10000, "name": "IT: Scrum Default Screen Scheme", "description": "", "screens": {"default": 10000}}, "emitted_at": 1697453468347} +{"stream": "screen_schemes", "data": {"id": 10001, "name": "IT: Scrum Bug Screen Scheme", "description": "", "screens": {"default": 10001}}, "emitted_at": 1697453468348} +{"stream": "sprints", "data": {"id": 2, "self": "https://airbyteio.atlassian.net/rest/agile/1.0/sprint/2", "state": "active", "name": "IT Sprint 1", "startDate": "2022-05-17T11:25:59.072000+00:00", "endDate": "2022-05-31T11:25:00+00:00", "createdDate": "2022-05-17T11:24:12.933000+00:00", "originBoardId": 1, "goal": "Deliver results", "boardId": 1}, "emitted_at": 1697453469489} +{"stream": "sprints", "data": {"id": 3, "self": "https://airbyteio.atlassian.net/rest/agile/1.0/sprint/3", "state": "future", "name": "IT Sprint 2", "startDate": "2022-05-31T11:25:59.072000+00:00", "endDate": "2022-06-14T11:25:00+00:00", "createdDate": "2023-04-05T11:57:09.557000+00:00", "originBoardId": 1, "boardId": 1}, "emitted_at": 1697453469490} +{"stream": "sprints", "data": {"id": 4, "self": "https://airbyteio.atlassian.net/rest/agile/1.0/sprint/4", "state": "future", "name": "IT Sprint 3", "startDate": "2022-06-14T11:25:59.072000+00:00", "endDate": "2022-06-28T11:25:00+00:00", "createdDate": "2023-04-05T11:57:30.379000+00:00", "originBoardId": 1, "boardId": 1}, "emitted_at": 1697453469490} +{"stream": "sprint_issues", "data": {"expand": "operations,versionedRepresentations,editmeta,changelog,renderedFields", "id": "2-10012", "self": "https://airbyteio.atlassian.net/rest/agile/1.0/issue/10012", "key": "IT-6", "fields": {"customfield_10016": null, "updated": "2023-10-12T13:30:02.307000-07:00", "created": "2021-03-11T06:14:18.085-0800", "status": {"self": "https://airbyteio.atlassian.net/rest/api/2/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/2/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}}, "issueId": "10012", "sprintId": 2, "created": "2021-03-11T06:14:18.085000-08:00", "updated": "2023-10-12T13:30:02.307000-07:00"}, "emitted_at": 1697453471411} +{"stream": "sprint_issues", "data": {"expand": "operations,versionedRepresentations,editmeta,changelog,renderedFields", "id": "2-10019", "self": "https://airbyteio.atlassian.net/rest/agile/1.0/issue/10019", "key": "IT-9", "fields": {"customfield_10016": null, "updated": "2023-04-05T04:57:18.118000-07:00", "created": "2021-03-11T06:14:24.791-0800", "status": {"self": "https://airbyteio.atlassian.net/rest/api/2/status/10000", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "To Do", "id": "10000", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/2/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}}, "issueId": "10019", "sprintId": 2, "created": "2021-03-11T06:14:24.791000-08:00", "updated": "2023-04-05T04:57:18.118000-07:00"}, "emitted_at": 1697453471413} +{"stream": "sprint_issues", "data": {"expand": "operations,versionedRepresentations,editmeta,changelog,renderedFields", "id": "2-10000", "self": "https://airbyteio.atlassian.net/rest/agile/1.0/issue/10000", "key": "IT-1", "fields": {"customfield_10016": null, "updated": "2022-05-17T04:26:28.885000-07:00", "created": "2020-12-07T06:12:17.863-0800", "status": {"self": "https://airbyteio.atlassian.net/rest/api/2/status/10001", "description": "", "iconUrl": "https://airbyteio.atlassian.net/", "name": "Done", "id": "10001", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/2/statuscategory/3", "id": 3, "key": "done", "colorName": "green", "name": "Done"}}, "customfield_10026": null}, "issueId": "10000", "sprintId": 2, "created": "2020-12-07T06:12:17.863000-08:00", "updated": "2022-05-17T04:26:28.885000-07:00"}, "emitted_at": 1697453471414} +{"stream": "time_tracking", "data": {"key": "JIRA", "name": "JIRA provided time tracking"}, "emitted_at": 1697453477445} +{"stream": "time_tracking", "data": {"key": "is.origo.jira.tempo-plugin__timetracking-provider", "name": "Tempo Timesheets"}, "emitted_at": 1697453477446} +{"stream": "users", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "locale": "en_US"}, "emitted_at": 1697453477834} +{"stream": "users", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058:f58131cb-b67d-43c7-b30d-6b58d40bd077", "accountId": "557058:f58131cb-b67d-43c7-b30d-6b58d40bd077", "accountType": "app", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "24x24": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "16x16": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "32x32": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png"}, "displayName": "Automation for Jira", "active": true}, "emitted_at": 1697453477835} +{"stream": "users", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5d53f3cbc6b9320d9ea5bdc2", "accountId": "5d53f3cbc6b9320d9ea5bdc2", "accountType": "app", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "24x24": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "16x16": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "32x32": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png"}, "displayName": "Jira Outlook", "active": true}, "emitted_at": 1697453477836} +{"stream": "users_groups_detailed", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5fc9e78d2730d800760becc4", "accountId": "5fc9e78d2730d800760becc4", "accountType": "atlassian", "emailAddress": "integration-test@airbyte.io", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "24x24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "16x16": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png", "32x32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FIT-5.png"}, "displayName": "integration test", "active": true, "timeZone": "America/Los_Angeles", "locale": "en_US", "groups": {"size": 30, "items": [{"name": "administrators", "groupId": "0ca6e087-7a61-4986-a269-98fe268854a1", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=0ca6e087-7a61-4986-a269-98fe268854a1"}, {"name": "confluence-users", "groupId": "38d808e9-113f-45c4-817b-099e953b687a", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=38d808e9-113f-45c4-817b-099e953b687a"}, {"name": "integration-test-group", "groupId": "5f1ec851-f8da-4f90-ab42-8dc50a9f99d8", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=5f1ec851-f8da-4f90-ab42-8dc50a9f99d8"}, {"name": "jira-administrators", "groupId": "58582f33-a5a6-43b9-92a6-ff0bbacb49ae", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=58582f33-a5a6-43b9-92a6-ff0bbacb49ae"}, {"name": "jira-admins-airbyteio", "groupId": "2d55cbe0-4cab-46a4-853e-ec31162ab9a3", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=2d55cbe0-4cab-46a4-853e-ec31162ab9a3"}, {"name": "jira-servicemanagement-customers-airbyteio", "groupId": "125680d3-7e85-41ad-a662-892b6590272e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=125680d3-7e85-41ad-a662-892b6590272e"}, {"name": "jira-servicemanagement-users-airbyteio", "groupId": "aab99a7c-3ce3-4123-b580-e4e00460754d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=aab99a7c-3ce3-4123-b580-e4e00460754d"}, {"name": "jira-software-users", "groupId": "4452b254-035d-469a-a422-1f4666dce50e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=4452b254-035d-469a-a422-1f4666dce50e"}, {"name": "jira-users", "groupId": "2513da2e-08cf-4415-9bcd-cbbd32fa227d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=2513da2e-08cf-4415-9bcd-cbbd32fa227d"}, {"name": "site-admins", "groupId": "76dad095-fc1a-467a-88b4-fde534220985", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=76dad095-fc1a-467a-88b4-fde534220985"}, {"name": "Test group 0", "groupId": "ee8d15d1-6462-406a-b0a6-8065b7e4cdd7", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=ee8d15d1-6462-406a-b0a6-8065b7e4cdd7"}, {"name": "Test group 1", "groupId": "bda1faf1-1a1a-42d1-82e4-a428c8b8f67c", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=bda1faf1-1a1a-42d1-82e4-a428c8b8f67c"}, {"name": "Test group 10", "groupId": "e9f74708-e33c-4158-919d-6457f50c6e74", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=e9f74708-e33c-4158-919d-6457f50c6e74"}, {"name": "Test group 11", "groupId": "b0e6d76f-701a-4208-a88d-4478f242edde", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=b0e6d76f-701a-4208-a88d-4478f242edde"}, {"name": "Test group 12", "groupId": "dddc24a0-ef00-407e-abef-5a660b6f55cf", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=dddc24a0-ef00-407e-abef-5a660b6f55cf"}, {"name": "Test group 13", "groupId": "dbe4af74-8387-4b08-843b-86af78dd738e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=dbe4af74-8387-4b08-843b-86af78dd738e"}, {"name": "Test group 14", "groupId": "d4570a20-38d8-44cc-a63b-0924d0d0d0ff", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=d4570a20-38d8-44cc-a63b-0924d0d0d0ff"}, {"name": "Test group 15", "groupId": "87bde5c0-7231-44a7-88b5-421da2ab8052", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=87bde5c0-7231-44a7-88b5-421da2ab8052"}, {"name": "Test group 16", "groupId": "538b6aa2-bf57-402f-93c0-c2e2d68b7155", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=538b6aa2-bf57-402f-93c0-c2e2d68b7155"}, {"name": "Test group 17", "groupId": "022bc924-ac57-442d-80c9-df042b73ad87", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=022bc924-ac57-442d-80c9-df042b73ad87"}, {"name": "Test group 18", "groupId": "bbfc6fc9-96db-4e66-88f4-c55b08298272", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=bbfc6fc9-96db-4e66-88f4-c55b08298272"}, {"name": "Test group 19", "groupId": "3c4fef5d-9721-4f20-9a68-346d222de3cf", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=3c4fef5d-9721-4f20-9a68-346d222de3cf"}, {"name": "Test group 2", "groupId": "5ddb26f1-2d31-414a-ac34-b2d6de38805d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=5ddb26f1-2d31-414a-ac34-b2d6de38805d"}, {"name": "Test group 3", "groupId": "638aa1ad-8707-4d56-9361-f5959b6c4785", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=638aa1ad-8707-4d56-9361-f5959b6c4785"}, {"name": "Test group 4", "groupId": "532554e0-43be-4eca-9186-b417dcf38547", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=532554e0-43be-4eca-9186-b417dcf38547"}, {"name": "Test group 5", "groupId": "6b663734-85b6-4185-8fb2-9ac27709b3aa", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=6b663734-85b6-4185-8fb2-9ac27709b3aa"}, {"name": "Test group 6", "groupId": "2d4af5cf-cd34-4e78-9445-abc000cdd5cc", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=2d4af5cf-cd34-4e78-9445-abc000cdd5cc"}, {"name": "Test group 7", "groupId": "e8a97909-d807-4f79-8548-1f2c156ae6f0", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=e8a97909-d807-4f79-8548-1f2c156ae6f0"}, {"name": "Test group 8", "groupId": "3ee851e7-6688-495a-a6f6-737e85a23878", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=3ee851e7-6688-495a-a6f6-737e85a23878"}, {"name": "Test group 9", "groupId": "af27d0b1-4378-443f-9a6d-f878848b144a", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=af27d0b1-4378-443f-9a6d-f878848b144a"}]}, "applicationRoles": {"size": 2, "items": [{"key": "jira-servicedesk", "name": "Jira Service Desk"}, {"key": "jira-software", "name": "Jira Software"}]}, "expand": "groups,applicationRoles"}, "emitted_at": 1697453478823} +{"stream": "users_groups_detailed", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=557058:f58131cb-b67d-43c7-b30d-6b58d40bd077", "accountId": "557058:f58131cb-b67d-43c7-b30d-6b58d40bd077", "accountType": "app", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "24x24": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "16x16": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png", "32x32": "https://secure.gravatar.com/avatar/600529a9c8bfef89daa848e6db28ed2d?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FAJ-0.png"}, "displayName": "Automation for Jira", "active": true, "timeZone": "America/Los_Angeles", "locale": "en_US", "groups": {"size": 3, "items": [{"name": "atlassian-addons-admin", "groupId": "90b9ffb1-ed26-4b5e-af59-8f684900ce83", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=90b9ffb1-ed26-4b5e-af59-8f684900ce83"}, {"name": "jira-servicemanagement-users-airbyteio", "groupId": "aab99a7c-3ce3-4123-b580-e4e00460754d", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=aab99a7c-3ce3-4123-b580-e4e00460754d"}, {"name": "jira-software-users", "groupId": "4452b254-035d-469a-a422-1f4666dce50e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=4452b254-035d-469a-a422-1f4666dce50e"}]}, "applicationRoles": {"size": 2, "items": [{"key": "jira-servicedesk", "name": "Jira Service Desk"}, {"key": "jira-software", "name": "Jira Software"}]}, "expand": "groups,applicationRoles"}, "emitted_at": 1697453479094} +{"stream": "users_groups_detailed", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/user?accountId=5d53f3cbc6b9320d9ea5bdc2", "accountId": "5d53f3cbc6b9320d9ea5bdc2", "accountType": "app", "avatarUrls": {"48x48": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "24x24": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "16x16": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png", "32x32": "https://secure.gravatar.com/avatar/40cff14f727dbf6d865576d575c6bdd2?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FJO-4.png"}, "displayName": "Jira Outlook", "active": true, "timeZone": "America/Los_Angeles", "locale": "en_US", "groups": {"size": 1, "items": [{"name": "jira-software-users", "groupId": "4452b254-035d-469a-a422-1f4666dce50e", "self": "https://airbyteio.atlassian.net/rest/api/3/group?groupId=4452b254-035d-469a-a422-1f4666dce50e"}]}, "applicationRoles": {"size": 2, "items": [{"key": "jira-servicedesk", "name": "Jira Service Desk"}, {"key": "jira-software", "name": "Jira Software"}]}, "expand": "groups,applicationRoles"}, "emitted_at": 1697453479412} +{"stream": "workflows", "data": {"id": {"name": "Builds Workflow", "entityId": "Builds Workflow"}, "description": "Builds Workflow", "created": "1969-12-31T16:00:00-08:00", "updated": "1969-12-31T16:00:00-08:00"}, "emitted_at": 1697453494031} +{"stream": "workflows", "data": {"id": {"name": "classic default workflow", "entityId": "385bb764-dfb6-89a7-2e43-a25bdd0cbaf4"}, "description": "The classic JIRA default workflow", "created": "2020-12-03T23:41:38.951000-08:00", "updated": "2023-06-30T02:33:48.808000-07:00"}, "emitted_at": 1697453494032} +{"stream": "workflows", "data": {"id": {"name": "jira", "entityId": "jira"}, "description": "The default Jira workflow.", "created": "1969-12-31T16:00:00-08:00", "updated": "1969-12-31T16:00:00-08:00"}, "emitted_at": 1697453494033} +{"stream": "workflow_schemes", "data": {"id": 10000, "name": "classic", "description": "classic", "defaultWorkflow": "classic default workflow", "issueTypeMappings": {}, "self": "https://airbyteio.atlassian.net/rest/api/3/workflowscheme/10000"}, "emitted_at": 1697453495072} +{"stream": "workflow_schemes", "data": {"id": 10001, "name": "IT: Software Simplified Workflow Scheme", "description": "Generated by JIRA Software version 1001.0.0-SNAPSHOT. This workflow scheme is managed internally by Jira Software. Do not manually modify this workflow scheme.", "defaultWorkflow": "Software Simplified Workflow for Project IT", "issueTypeMappings": {}, "self": "https://airbyteio.atlassian.net/rest/api/3/workflowscheme/10001"}, "emitted_at": 1697453495073} +{"stream": "workflow_schemes", "data": {"id": 10002, "name": "P2: Software Simplified Workflow Scheme", "description": "Generated by JIRA Software version 1001.0.0-SNAPSHOT. This workflow scheme is managed internally by Jira Software. Do not manually modify this workflow scheme.", "defaultWorkflow": "Software Simplified Workflow for Project P2", "issueTypeMappings": {}, "self": "https://airbyteio.atlassian.net/rest/api/3/workflowscheme/10002"}, "emitted_at": 1697453495074} +{"stream": "workflow_statuses", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/1", "description": "The issue is open and ready for the assignee to start work on it.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/open.png", "name": "Open", "untranslatedName": "Open", "id": "1", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "emitted_at": 1697453496110} +{"stream": "workflow_statuses", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/3", "description": "This issue is being actively worked on at the moment by the assignee.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/inprogress.png", "name": "In Progress", "untranslatedName": "In Progress", "id": "3", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}}, "emitted_at": 1697453496111} +{"stream": "workflow_statuses", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/status/4", "description": "This issue was once resolved, but the resolution was deemed incorrect. From here issues are either marked assigned or resolved.", "iconUrl": "https://airbyteio.atlassian.net/images/icons/statuses/reopened.png", "name": "Reopened", "untranslatedName": "Reopened", "id": "4", "statusCategory": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}}, "emitted_at": 1697453496111} +{"stream": "workflow_status_categories", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/1", "id": 1, "key": "undefined", "colorName": "medium-gray", "name": "No Category"}, "emitted_at": 1697453496723} +{"stream": "workflow_status_categories", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/2", "id": 2, "key": "new", "colorName": "blue-gray", "name": "To Do"}, "emitted_at": 1697453496724} +{"stream": "workflow_status_categories", "data": {"self": "https://airbyteio.atlassian.net/rest/api/3/statuscategory/4", "id": 4, "key": "indeterminate", "colorName": "yellow", "name": "In Progress"}, "emitted_at": 1697453496724} diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-jira/integration_tests/invalid_config.json index 9232da9f03d8..022ba9936650 100644 --- a/airbyte-integrations/connectors/source-jira/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-jira/integration_tests/invalid_config.json @@ -1,6 +1,5 @@ { "api_token": "invalid_token", - "domain": "invaliddomain.atlassian.net", "email": "test@test.com", "projects": ["invalidproject"], "start_date": "2021-09-25T00:00:00Z" diff --git a/airbyte-integrations/connectors/source-jira/integration_tests/invalid_config_domain.json b/airbyte-integrations/connectors/source-jira/integration_tests/invalid_config_domain.json new file mode 100644 index 000000000000..834377ca8a8d --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/integration_tests/invalid_config_domain.json @@ -0,0 +1,5 @@ +{ + "api_token": "some_token", + "domain": "https://withprotocol.atlassian.net", + "start_date": "2021-09-25T00:00:00Z" +} diff --git a/airbyte-integrations/connectors/source-jira/metadata.yaml b/airbyte-integrations/connectors/source-jira/metadata.yaml index ad0753c1bea0..a46d1dad743d 100644 --- a/airbyte-integrations/connectors/source-jira/metadata.yaml +++ b/airbyte-integrations/connectors/source-jira/metadata.yaml @@ -1,16 +1,22 @@ data: + ab_internal: + ql: 400 + sl: 200 allowedHosts: hosts: - ${domain} + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: api connectorType: source definitionId: 68e63de2-bb83-4c7e-93fa-a8a9051e3993 - dockerImageTag: 0.3.12 - maxSecondsBetweenMessages: 21600 + dockerImageTag: 1.0.0 dockerRepository: airbyte/source-jira + documentationUrl: https://docs.airbyte.com/integrations/sources/jira githubIssueLabel: source-jira icon: jira.svg license: MIT + maxSecondsBetweenMessages: 21600 name: Jira registries: cloud: @@ -18,11 +24,21 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/jira + releases: + breakingChanges: + 1.0.0: + message: "Stream state will be saved for every board in stream `Boards Issues`. Customers who use stream `Board Issues` in Incremental Sync mode must take action with their connections." + upgradeDeadline: "2024-01-25" + scopedImpact: + - scopeType: stream + impactedScopes: ["board_issues"] + suggestedStreams: + streams: + - issues + - projects + - users + - issue_fields + supportLevel: certified tags: - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-jira/setup.py b/airbyte-integrations/connectors/source-jira/setup.py index e481ac92aa12..800525b0a829 100644 --- a/airbyte-integrations/connectors/source-jira/setup.py +++ b/airbyte-integrations/connectors/source-jira/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk", "requests==2.25.1", "pendulum~=2.1.2"] +MAIN_REQUIREMENTS = ["airbyte-cdk>=0.51.19"] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", diff --git a/airbyte-integrations/connectors/source-jira/source_jira/config_migrations.py b/airbyte-integrations/connectors/source-jira/source_jira/config_migrations.py new file mode 100644 index 000000000000..5667cdea454f --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/source_jira/config_migrations.py @@ -0,0 +1,102 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import logging +from typing import Any, List, Mapping + +from airbyte_cdk.config_observation import create_connector_config_control_message +from airbyte_cdk.entrypoint import AirbyteEntrypoint +from airbyte_cdk.sources import Source +from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + +logger = logging.getLogger("airbyte") + + +class MigrateIssueExpandProperties: + """ + This class stands for migrating the config at runtime, + while providing the backward compatibility when falling back to the previous source version. + + Specifically, starting from `0.6.1`, the `issues_stream_expand` property should be like : + > List("renderedFields", "transitions", "changelog" ...) + instead of, in `0.6.0`: + > expand_issue_changelog: bool: True + > render_fields: bool: True + > expand_issue_transition: bool: True + """ + + message_repository: MessageRepository = InMemoryMessageRepository() + migrate_from_keys_map: dict = { + "expand_issue_changelog": "changelog", + "render_fields": "renderedFields", + "expand_issue_transition": "transitions", + } + migrate_to_key: str = "issues_stream_expand_with" + + @classmethod + def should_migrate(cls, config: Mapping[str, Any]) -> bool: + """ + This method determines whether the config should be migrated to have the new structure for the `issues_stream_expand_with`, + based on the source spec. + Returns: + > True, if the transformation is necessary + > False, otherwise. + > Raises the Exception if the structure could not be migrated. + """ + # If the config was already migrated, there is no need to do this again. + # but if the customer has already switched to the new version, + # corrected the old config and switches back to the new version, + # we should try to migrate the modified old issue expand properties. + if cls.migrate_to_key in config: + return not len(config[cls.migrate_to_key]) > 0 + + if any(config.get(key) for key in cls.migrate_from_keys_map): + return True + return False + + @classmethod + def transform_to_array(cls, config: Mapping[str, Any]) -> Mapping[str, Any]: + # assign old values to new property that will be used within the new version + config[cls.migrate_to_key] = [] + for k, v in cls.migrate_from_keys_map.items(): + if config.get(k): + config[cls.migrate_to_key].append(v) + # transform boolean flags to `list` of objects + return config + + @classmethod + def modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, Any]) -> Mapping[str, Any]: + # modify the config + migrated_config = cls.transform_to_array(config) + # save the config + source.write_config(migrated_config, config_path) + # return modified config + return migrated_config + + @classmethod + def emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: + # add the Airbyte Control Message to message repo + cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) + # emit the Airbyte Control Message from message queue to stdout + for message in cls.message_repository._message_queue: + print(message.json(exclude_unset=True)) + + @classmethod + def migrate(cls, args: List[str], source: Source) -> None: + """ + This method checks the input args, should the config be migrated, + transform if necessary and emit the CONTROL message. + """ + # get config path + config_path = AirbyteEntrypoint(source).extract_config(args) + # proceed only if `--config` arg is provided + if config_path: + # read the existing config + config = source.read_config(config_path) + # migration check + if cls.should_migrate(config): + cls.emit_control_message( + cls.modify_and_save(config_path, source, config), + ) diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/application_roles.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/application_roles.json index 66bed202fa4d..c313a68d905e 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/application_roles.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/application_roles.json @@ -52,6 +52,23 @@ "type": ["null", "array"], "description": "Group Details", "items": { "type": ["null", "object"] } + }, + "defaultGroupsDetails": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "groupId": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + } + } + } } }, "additionalProperties": true, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/boards.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/boards.json index e8ef7e03931f..8062977e129d 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/boards.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/boards.json @@ -3,22 +3,54 @@ "type": "object", "properties": { "id": { - "type": "integer" + "type": ["null", "integer"] }, "self": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "type": { - "type": "string" + "type": ["null", "string"] }, "projectId": { - "type": "string" + "type": ["null", "string"] }, "projectKey": { - "type": "string" + "type": ["null", "string"] + }, + "location": { + "type": ["null", "object"], + "properties": { + "projectId": { + "type": ["null", "integer"] + }, + "userId": { + "type": ["null", "integer"] + }, + "userAccountId": { + "type": ["null", "string"] + }, + "displayName": { + "type": ["null", "string"] + }, + "projectName": { + "type": ["null", "string"] + }, + "projectKey": { + "type": ["null", "string"] + }, + "projectTypeKey": { + "type": ["null", "string"] + }, + "avatarURI": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } } } } diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/dashboards.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/dashboards.json index f9a2b9d0a2ef..ea39c4d60056 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/dashboards.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/dashboards.json @@ -2087,6 +2087,44 @@ "view": { "type": "string", "description": "The URL of the dashboard." + }, + "editpermission": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "group": { + "type": ["null", "object"] + }, + "id": { + "type": ["null", "integer"] + }, + "project": { + "type": ["null", "object"] + }, + "role": { + "type": ["null", "object"] + }, + "type": { + "type": ["null", "string"] + }, + "user": { + "type": ["null", "object"] + } + } + } + }, + "isWritable": { + "type": ["null", "boolean"] + }, + "systemDashboard": { + "type": ["null", "boolean"] + }, + "editPermissions": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"] + } } }, "additionalProperties": true, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/filter_sharing.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/filter_sharing.json index 06dac9d9c556..e3c19d2e8876 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/filter_sharing.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/filter_sharing.json @@ -7,6 +7,10 @@ "description": "The unique identifier of the share permission.", "readOnly": true }, + "filterId": { + "type": ["null", "string"], + "description": "Id of the related filter" + }, "type": { "type": "string", "description": "The type of share permission:\n\n * `group` Shared with a group. If set in a request, then specify `sharePermission.group` as well.\n * `project` Shared with a project. If set in a request, then specify `sharePermission.project` as well.\n * `projectRole` Share with a project role in a project. This value is not returned in responses. It is used in requests, where it needs to be specify with `projectId` and `projectRoleId`.\n * `global` Shared globally. If set in a request, no other `sharePermission` properties need to be specified.\n * `loggedin` Shared with all logged-in users. Note: This value is set in a request by specifying `authenticated` as the `type`.\n * `project-unknown` Shared with a project that the user does not have access to. Cannot be set in a request.", diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_comments.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_comments.json index fe184e1dc9ea..538bf16bfac6 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_comments.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_comments.json @@ -12,6 +12,11 @@ "description": "The ID of the comment.", "readOnly": true }, + "issueId": { + "type": ["null", "string"], + "description": "Id of the related issue.", + "readOnly": true + }, "author": { "description": "The ID of the user who created the comment.", "readOnly": true diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_custom_field_contexts.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_custom_field_contexts.json index 6d35043889c0..a56d15e4e169 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_custom_field_contexts.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_custom_field_contexts.json @@ -1,28 +1,34 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", + "description": "The details of a custom field context.", "properties": { "id": { - "type": "string", + "type": ["null", "string"], "description": "The ID of the context." }, + "fieldId": { + "type": ["null", "string"], + "description": "Id of the related field" + }, "name": { - "type": "string", + "type": ["null", "string"], "description": "The name of the context." }, "description": { - "type": "string", + "type": ["null", "string"], "description": "The description of the context." }, "isGlobalContext": { - "type": "boolean", + "type": ["null", "boolean"], "description": "Whether the context is global." }, "isAnyIssueType": { - "type": "boolean", + "type": ["null", "boolean"], "description": "Whether the context apply to all issue types." + }, + "fieldType": { + "type": ["null", "string"] } - }, - "additionalProperties": true, - "description": "The details of a custom field context." + } } diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_custom_field_options.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_custom_field_options.json index a7735025f5e1..57955820d4e2 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_custom_field_options.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_custom_field_options.json @@ -1,24 +1,29 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", + "description": "Details of the custom field options for a context.", "properties": { "id": { - "type": "string", + "type": ["null", "string"], "description": "The ID of the custom field option." }, "value": { - "type": "string", + "type": ["null", "string"], "description": "The value of the custom field option." }, "optionId": { - "type": "string", + "type": ["null", "string"], "description": "For cascading options, the ID of the custom field option containing the cascading option." }, "disabled": { - "type": "boolean", + "type": ["null", "boolean"], "description": "Whether the option is disabled." + }, + "fieldId": { + "type": ["null", "string"] + }, + "contextId": { + "type": ["null", "string"] } - }, - "additionalProperties": true, - "description": "Details of the custom field options for a context." + } } diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_fields.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_fields.json index 3e9f78138908..301272f3a40b 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_fields.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_fields.json @@ -173,6 +173,9 @@ "readOnly": true } } + }, + "untranslatedName": { + "type": ["null", "string"] } }, "additionalProperties": true, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_priorities.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_priorities.json index 5ea312354108..7e6af637a694 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_priorities.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_priorities.json @@ -25,6 +25,9 @@ "id": { "type": "string", "description": "The ID of the issue priority." + }, + "isDefault": { + "type": ["null", "boolean"] } }, "additionalProperties": true, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_properties.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_properties.json index c4ac2141d29f..faaa84ba2ec9 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_properties.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_properties.json @@ -6,10 +6,17 @@ "type": "string", "description": "The key of the property. Required on create and update." }, + "issueId": { + "type": ["null", "string"], + "description": "Id of the related issue.", + "readOnly": true + }, "value": { "description": "The value of the property. Required on create and update." + }, + "isdefault": { + "type": ["null", "boolean"] } }, - "additionalProperties": true, "description": "An entity property, for more information see [Entity properties](https://developer.atlassian.com/cloud/jira/platform/jira-entity-properties/)." } diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_remote_links.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_remote_links.json index 7a3a9d938f04..102a5fba5563 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_remote_links.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_remote_links.json @@ -6,6 +6,10 @@ "type": "integer", "description": "The ID of the link." }, + "issueId": { + "type": ["null", "string"], + "description": "Id of the related issue." + }, "self": { "type": "string", "description": "The URL of the link." diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_resolutions.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_resolutions.json index 226ad2dd2b1a..cac6937b08aa 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_resolutions.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_resolutions.json @@ -17,6 +17,9 @@ "name": { "type": "string", "description": "The name of the issue resolution." + }, + "isDefault": { + "type": ["null", "boolean"] } }, "additionalProperties": true, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_transitions.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_transitions.json new file mode 100644 index 000000000000..741a6dcec2a5 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_transitions.json @@ -0,0 +1,77 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "Issue Transitions", + "type": "object", + "properties": { + "fields": { + "type": ["null", "string"] + }, + "hasScreen": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "string"] + }, + "issueId": { + "type": ["null", "string"] + }, + "isAvailable": { + "type": ["null", "boolean"] + }, + "isConditional": { + "type": ["null", "boolean"] + }, + "isGlobal": { + "type": ["null", "boolean"] + }, + "isInitial": { + "type": ["null", "boolean"] + }, + "isLooped": { + "type": ["null", "boolean"] + }, + "name": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "object"], + "properties": { + "description": { + "type": ["null", "string"] + }, + "iconUrl": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "statusCategory": { + "type": ["null", "object"], + "properties": { + "colorName": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "key": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + } + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_types.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_types.json index 6ba2b5beb999..4068de21bc05 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_types.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_types.json @@ -1,89 +1,89 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", + "$schema": "https://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "description": "Details about an issue type.", "properties": { - "self": { - "type": "string", - "description": "The URL of these issue type details.", - "readOnly": true - }, - "id": { - "type": "string", - "description": "The ID of the issue type.", + "avatarId": { + "type": ["null", "integer"], + "description": "The ID of the issue type's avatar.", "readOnly": true }, "description": { - "type": "string", + "type": ["null", "string"], "description": "The description of the issue type.", "readOnly": true }, - "iconUrl": { - "type": "string", - "description": "The URL of the issue type's avatar.", + "entityId": { + "type": ["null", "string"], + "description": "Unique ID for next-gen projects.", "readOnly": true }, - "name": { - "type": "string", - "description": "The name of the issue type.", + "hierarchyLevel": { + "type": ["null", "integer"], + "description": "Hierarchy level of the issue type.", "readOnly": true }, - "subtask": { - "type": "boolean", - "description": "Whether this issue type is used to create subtasks.", + "iconUrl": { + "type": ["null", "string"], + "description": "The URL of the issue type's avatar.", "readOnly": true }, - "avatarId": { - "type": "integer", - "description": "The ID of the issue type's avatar.", + "id": { + "type": ["null", "string"], + "description": "The ID of the issue type.", "readOnly": true }, - "entityId": { - "type": "string", - "description": "Unique ID for next-gen projects.", + "name": { + "type": ["null", "string"], + "description": "The name of the issue type.", "readOnly": true }, - "hierarchyLevel": { - "type": "integer", - "description": "Hierarchy level of the issue type.", + "self": { + "type": ["null", "string"], + "description": "The URL of these issue type details.", + "readOnly": true + }, + "subtask": { + "type": ["null", "boolean"], + "description": "The URL of these issue type details.", "readOnly": true }, "scope": { "description": "Details of the next-gen projects the issue type is available in.", "readOnly": true, - "type": "object", + "type": ["null", "object"], "properties": { "type": { - "type": "string", + "type": ["null", "string"], "description": "The type of scope.", - "readOnly": true, - "enum": ["PROJECT", "TEMPLATE"] + "readOnly": true }, "project": { "description": "The project the item has scope in.", "readOnly": true, - "type": "object", + "type": ["null", "object"], "properties": { "self": { - "type": "string", + "type": ["null", "string"], "description": "The URL of the project details.", "readOnly": true }, "id": { - "type": "string", + "type": ["null", "string"], "description": "The ID of the project." }, "key": { - "type": "string", + "type": ["null", "string"], "description": "The key of the project.", "readOnly": true }, "name": { - "type": "string", + "type": ["null", "string"], "description": "The name of the project.", "readOnly": true }, "projectTypeKey": { - "type": "string", + "type": ["null", "string"], "description": "The [project type](https://confluence.atlassian.com/x/GwiiLQ#Jiraapplicationsoverview-Productfeaturesandprojecttypes) of the project.", "readOnly": true, "enum": ["software", "service_desk", "business"] @@ -96,22 +96,22 @@ "avatarUrls": { "description": "The URLs of the project's avatars.", "readOnly": true, - "type": "object", + "type": ["null", "object"], "properties": { "16x16": { - "type": "string", + "type": ["null", "string"], "description": "The URL of the item's 16x16 pixel avatar." }, "24x24": { - "type": "string", + "type": ["null", "string"], "description": "The URL of the item's 24x24 pixel avatar." }, "32x32": { - "type": "string", + "type": ["null", "string"], "description": "The URL of the item's 32x32 pixel avatar." }, "48x48": { - "type": "string", + "type": ["null", "string"], "description": "The URL of the item's 48x48 pixel avatar." } } @@ -119,25 +119,25 @@ "projectCategory": { "description": "The category the project belongs to.", "readOnly": true, - "type": "object", + "type": ["null", "object"], "properties": { "self": { - "type": "string", + "type": ["null", "string"], "description": "The URL of the project category.", "readOnly": true }, "id": { - "type": "string", + "type": ["null", "string"], "description": "The ID of the project category.", "readOnly": true }, "description": { - "type": "string", + "type": ["null", "string"], "description": "The name of the project category.", "readOnly": true }, "name": { - "type": "string", + "type": ["null", "string"], "description": "The description of the project category.", "readOnly": true } @@ -146,8 +146,9 @@ } } } + }, + "untranslatedName": { + "type": ["null", "string"] } - }, - "additionalProperties": true, - "description": "Details about an issue type." + } } diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_votes.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_votes.json index 10fbcb1334d7..913638b8b4f7 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_votes.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_votes.json @@ -7,6 +7,11 @@ "description": "The URL of these issue vote details.", "readOnly": true }, + "issueId": { + "type": ["null", "string"], + "description": "Id of the related issue.", + "readOnly": true + }, "votes": { "type": "integer", "description": "The number of votes on the issue.", diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_watchers.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_watchers.json index a0a1acb1eaf7..069c52a189d3 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_watchers.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issue_watchers.json @@ -7,6 +7,10 @@ "description": "The URL of these issue watcher details.", "readOnly": true }, + "issueId": { + "type": ["null", "string"], + "description": "Id of the related issue." + }, "isWatching": { "type": "boolean", "description": "Whether the calling user is watching this issue.", diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issues.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issues.json index 5c4e5e096a03..1af2ea566e43 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/issues.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/issues.json @@ -29,7 +29,311 @@ "type": "object", "additionalProperties": true, "description": "The rendered value of each field present on the issue.", - "readOnly": true + "readOnly": true, + "properties": { + "statuscategorychangedate": { + "type": ["null", "string"] + }, + "issuetype": { + "type": ["null", "string"] + }, + "timespent": { + "type": ["null", "string"] + }, + "project": { + "type": ["null", "string"] + }, + "fixVersions": { + "type": ["null", "string"] + }, + "aggregatetimespent": { + "type": ["null", "string"] + }, + "resolution": { + "type": ["null", "string"] + }, + "resolutiondate": { + "type": ["null", "string"] + }, + "workratio": { + "type": ["null", "string"] + }, + "watches": { + "type": ["null", "string"] + }, + "lastViewed": { + "type": ["null", "string"] + }, + "issuerestriction": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"] + }, + "priority": { + "type": ["null", "string"] + }, + "labels": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "timeestimate": { + "type": ["null", "string"] + }, + "aggregatetimeoriginalestimate": { + "type": ["null", "string"] + }, + "versions": { + "type": ["null", "string"] + }, + "issuelinks": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "assignee": { + "type": ["null", "string"] + }, + "updated": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "components": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"] + } + }, + "timeoriginalestimate": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "timetracking": { + "type": ["null", "object"] + }, + "security": { + "type": ["null", "string"] + }, + "aggregatetimeestimate": { + "type": ["null", "string"] + }, + "attachment": { + "type": ["null", "array"] + }, + "summary": { + "type": ["null", "string"] + }, + "creator": { + "type": ["null", "string"] + }, + "subtasks": { + "type": ["null", "array"] + }, + "reporter": { + "type": ["null", "string"] + }, + "aggregateprogress": { + "type": ["null", "string"] + }, + "environment": { + "type": ["null", "string"] + }, + "duedate": { + "type": ["null", "string"] + }, + "progress": { + "type": ["null", "string"] + }, + "comment": { + "type": ["null", "object"], + "properties": { + "comments": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "self": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "author": { + "type": ["null", "object"], + "properties": { + "self": { + "type": ["null", "string"] + }, + "accountId": { + "type": ["null", "string"] + }, + "emailAddress": { + "type": ["null", "string"] + }, + "avatarUrls": { + "type": ["null", "object"], + "properties": { + "48x48": { + "type": ["null", "string"] + }, + "24x24": { + "type": ["null", "string"] + }, + "16x16": { + "type": ["null", "string"] + }, + "32x32": { + "type": ["null", "string"] + } + } + }, + "displayName": { + "type": ["null", "string"] + }, + "active": { + "type": "boolean" + }, + "timeZone": { + "type": ["null", "string"] + }, + "accountType": { + "type": ["null", "string"] + } + } + }, + "body": { + "type": ["null", "object"], + "properties": { + "version": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + }, + "content": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "content": { + "type": "array", + "items": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "text": { + "type": ["null", "string"] + } + } + } + } + } + } + } + } + }, + "updateAuthor": { + "type": ["null", "object"], + "properties": { + "self": { + "type": ["null", "string"] + }, + "accountId": { + "type": ["null", "string"] + }, + "emailAddress": { + "type": ["null", "string"] + }, + "avatarUrls": { + "type": ["null", "object"], + "properties": { + "48x48": { + "type": ["null", "string"] + }, + "24x24": { + "type": ["null", "string"] + }, + "16x16": { + "type": ["null", "string"] + }, + "32x32": { + "type": ["null", "string"] + } + } + }, + "displayName": { + "type": ["null", "string"] + }, + "active": { + "type": "boolean" + }, + "timeZone": { + "type": ["null", "string"] + }, + "accountType": { + "type": ["null", "string"] + } + } + }, + "created": { + "type": ["null", "string"] + }, + "updated": { + "type": ["null", "string"] + }, + "jsdPublic": { + "type": "boolean" + } + } + } + }, + "self": { + "type": ["null", "string"] + }, + "maxResults": { + "type": ["null", "integer"] + }, + "total": { + "type": ["null", "integer"] + }, + "startAt": { + "type": ["null", "integer"] + } + } + }, + "votes": { + "type": ["null", "string"] + }, + "worklog": { + "type": ["null", "object"], + "properties": { + "startAt": { + "type": ["null", "integer"] + }, + "maxResults": { + "type": ["null", "integer"] + }, + "total": { + "type": "integer" + }, + "worklogs": { + "type": ["null", "array"] + } + } + } + } }, "properties": { "type": "object", @@ -51,7 +355,76 @@ "transitions": { "type": "array", "description": "The transitions that can be performed on the issue.", - "readOnly": true + "readOnly": true, + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "object"], + "properties": { + "self": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "iconUrl": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "statusCategory": { + "type": ["null", "object"], + "properties": { + "self": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "key": { + "type": ["null", "string"] + }, + "colorName": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } + } + } + }, + "hasScreen": { + "type": "boolean" + }, + "isGlobal": { + "type": "boolean" + }, + "isInitial": { + "type": "boolean" + }, + "isAvailable": { + "type": "boolean" + }, + "isConditional": { + "type": "boolean" + }, + "isLooped": { + "type": "boolean" + } + } + } }, "operations": { "type": ["object", "null"], @@ -66,7 +439,101 @@ "changelog": { "type": ["object", "null"], "description": "Details of changelogs associated with the issue.", - "readOnly": true + "readOnly": true, + "properties": { + "startAt": { + "type": ["null", "integer"] + }, + "maxResults": { + "type": ["null", "integer"] + }, + "total": { + "type": ["null", "integer"] + }, + "histories": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "author": { + "type": ["null", "object"], + "properties": { + "self": { + "type": ["null", "string"] + }, + "accountId": { + "type": ["null", "string"] + }, + "emailAddress": { + "type": ["null", "string"] + }, + "avatarUrls": { + "type": ["null", "object"], + "properties": { + "48x48": { + "type": ["null", "string"] + }, + "24x24": { + "type": ["null", "string"] + }, + "16x16": { + "type": ["null", "string"] + }, + "32x32": { + "type": ["null", "string"] + } + } + }, + "displayName": { + "type": "string" + }, + "active": { + "type": "boolean" + }, + "timeZone": { + "type": ["null", "string"] + }, + "accountType": { + "type": ["null", "string"] + } + } + }, + "created": { + "type": ["null", "string"] + }, + "items": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "field": { + "type": ["null", "string"] + }, + "fieldtype": { + "type": ["null", "string"] + }, + "from": { + "type": ["null", "string"] + }, + "fromString": { + "type": ["null", "string"] + }, + "to": { + "type": ["null", "string"] + }, + "toString": { + "type": ["null", "string"] + } + } + } + } + } + } + } + } }, "versionedRepresentations": { "type": "object", diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_avatars.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_avatars.json index 2f22374544a1..afb993abe8e3 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_avatars.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_avatars.json @@ -6,6 +6,10 @@ "type": "string", "description": "The ID of the avatar." }, + "projectId": { + "type": ["null", "string"], + "description": "Id of the related project." + }, "owner": { "type": "string", "description": "The owner of the avatar. For a system avatar the owner is null (and nothing is returned). For non-system avatars this is the appropriate identifier, such as the ID for a project or the account ID for a user.", diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_components.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_components.json index 23d450ee9c9a..ef1cbbcf0691 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_components.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_components.json @@ -505,6 +505,12 @@ "type": "integer", "description": "The ID of the project the component is assigned to.", "readOnly": true + }, + "componentBean": { + "type": ["null", "object"] + }, + "issueCount": { + "type": ["null", "integer"] } }, "additionalProperties": true, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_permission_schemes.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_permission_schemes.json index 0e7c33aa24f1..ff7c28191eec 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_permission_schemes.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_permission_schemes.json @@ -6,6 +6,10 @@ "type": ["null", "string"], "description": "The URL of the issue level security item." }, + "projectId": { + "type": ["null", "string"], + "description": "Id of the related project." + }, "id": { "type": ["null", "string"], "description": "The ID of the issue level security item." diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_roles.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_roles.json new file mode 100644 index 000000000000..dd4f970fd0a5 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/project_roles.json @@ -0,0 +1,177 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "description": "Project Roles", + "properties": { + "actors": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "actorGroup": { + "type": ["null", "object"], + "properties": { + "displayName": { + "type": ["null", "string"] + }, + "groupId": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "actorUser": { + "type": ["null", "object"], + "properties": { + "accountId": { + "type": ["null", "string"] + } + } + }, + "avatarUrl": { + "type": ["null", "string"] + }, + "displayName": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + } + }, + "admin": { + "type": ["null", "boolean"] + }, + "currentUserRole": { + "type": ["null", "boolean"] + }, + "default": { + "type": ["null", "boolean"] + }, + "description": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "roleConfigurable": { + "type": ["null", "boolean"] + }, + "scope": { + "description": "Details of the next-gen projects the issue type is available in.", + "readOnly": true, + "type": "object", + "properties": { + "type": { + "type": ["null", "string"], + "description": "The type of scope.", + "readOnly": true + }, + "project": { + "description": "The project the item has scope in.", + "readOnly": true, + "type": "object", + "properties": { + "self": { + "type": ["null", "string"], + "description": "The URL of the project details.", + "readOnly": true + }, + "id": { + "type": ["null", "string"], + "description": "The ID of the project." + }, + "key": { + "type": ["null", "string"], + "description": "The key of the project.", + "readOnly": true + }, + "name": { + "type": ["null", "string"], + "description": "The name of the project.", + "readOnly": true + }, + "projectTypeKey": { + "type": ["null", "string"], + "description": "The [project type](https://confluence.atlassian.com/x/GwiiLQ#Jiraapplicationsoverview-Productfeaturesandprojecttypes) of the project.", + "readOnly": true + }, + "simplified": { + "type": ["null", "boolean"], + "description": "Whether or not the project is simplified.", + "readOnly": true + }, + "avatarUrls": { + "description": "The URLs of the project's avatars.", + "readOnly": true, + "type": "object", + "properties": { + "16x16": { + "type": ["null", "string"], + "description": "The URL of the item's 16x16 pixel avatar." + }, + "24x24": { + "type": ["null", "string"], + "description": "The URL of the item's 24x24 pixel avatar." + }, + "32x32": { + "type": ["null", "string"], + "description": "The URL of the item's 32x32 pixel avatar." + }, + "48x48": { + "type": ["null", "string"], + "description": "The URL of the item's 48x48 pixel avatar." + } + } + }, + "projectCategory": { + "description": "The category the project belongs to.", + "readOnly": true, + "type": "object", + "properties": { + "self": { + "type": ["null", "string"], + "description": "The URL of the project category.", + "readOnly": true + }, + "id": { + "type": ["null", "string"], + "description": "The ID of the project category.", + "readOnly": true + }, + "description": { + "type": ["null", "string"], + "description": "The name of the project category.", + "readOnly": true + }, + "name": { + "type": ["null", "string"], + "description": "The description of the project category.", + "readOnly": true + } + } + } + } + } + } + }, + "self": { + "type": ["null", "string"] + }, + "translatedName": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/projects.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/projects.json index 3183161171e4..98d0b8457fd7 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/projects.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/projects.json @@ -166,6 +166,9 @@ "archivedBy": { "description": "The user who archived the project.", "readOnly": true + }, + "entityId": { + "type": ["null", "string"] } }, "additionalProperties": true, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tab_fields.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tab_fields.json index 097a398ff712..0180e3dd6d18 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tab_fields.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tab_fields.json @@ -10,6 +10,14 @@ "name": { "type": "string", "description": "The name of the screen tab field. Required on create and update. The maximum length is 255 characters." + }, + "screenId": { + "type": ["null", "integer"], + "description": "Id of the related screen." + }, + "tabId": { + "type": ["null", "integer"], + "description": "Id of the related tab." } }, "additionalProperties": true, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tabs.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tabs.json index 09e13c932da5..013814e7057d 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tabs.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/screen_tabs.json @@ -11,6 +11,10 @@ "name": { "type": "string", "description": "The name of the screen tab. The maximum length is 255 characters." + }, + "screenId": { + "type": ["null", "integer"], + "description": "Id of the related screen." } }, "additionalProperties": true, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/sprints.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/sprints.json index d91f1e1ff40f..12c13c227471 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/sprints.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/sprints.json @@ -29,8 +29,16 @@ "originBoardId": { "type": "integer" }, + "boardId": { + "type": "integer", + "description": "Used to determine which board the sprint is a part of. (Not always the same as originBoardId)" + }, "goal": { "type": "string" + }, + "createdDate": { + "type": ["null", "string"], + "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/workflow_statuses.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/workflow_statuses.json index 8fcd1436dbd3..b42f1a8b79d9 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/workflow_statuses.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/workflow_statuses.json @@ -58,6 +58,12 @@ "readOnly": true } } + }, + "scope": { + "type": ["null", "object"] + }, + "untranslatedName": { + "type": ["null", "string"] } }, "additionalProperties": true, diff --git a/airbyte-integrations/connectors/source-jira/source_jira/schemas/workflows.json b/airbyte-integrations/connectors/source-jira/source_jira/schemas/workflows.json index 7d65a897489a..3e6990ac18b6 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/schemas/workflows.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/schemas/workflows.json @@ -140,6 +140,14 @@ } } } + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated": { + "type": ["null", "string"], + "format": "date-time" } }, "readOnly": true diff --git a/airbyte-integrations/connectors/source-jira/source_jira/source.py b/airbyte-integrations/connectors/source-jira/source_jira/source.py index f8ed4ea43abd..a34a74949070 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/source.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/source.py @@ -2,14 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging from typing import Any, List, Mapping, Optional, Tuple import pendulum import requests from airbyte_cdk import AirbyteLogger +from airbyte_cdk.models import FailureType from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.auth import BasicHttpAuthenticator +from airbyte_cdk.utils.traced_exception import AirbyteTracedException from pydantic.error_wrappers import ValidationError from .streams import ( @@ -23,6 +26,7 @@ Groups, IssueComments, IssueCustomFieldContexts, + IssueCustomFieldOptions, IssueFieldConfigurations, IssueFields, IssueLinkTypes, @@ -34,6 +38,8 @@ IssueResolutions, Issues, IssueSecuritySchemes, + IssueTransitions, + IssueTypes, IssueTypeSchemes, IssueTypeScreenSchemes, IssueVotes, @@ -48,6 +54,7 @@ ProjectComponents, ProjectEmail, ProjectPermissionSchemes, + ProjectRoles, Projects, ProjectTypes, ProjectVersions, @@ -68,13 +75,15 @@ ) from .utils import read_full_refresh +logger = logging.getLogger("airbyte") + class SourceJira(AbstractSource): def _validate_and_transform(self, config: Mapping[str, Any]): start_date = config.get("start_date") if start_date: config["start_date"] = pendulum.parse(start_date) - + config["lookback_window_minutes"] = pendulum.duration(minutes=config.get("lookback_window_minutes", 0)) config["projects"] = config.get("projects", []) return config @@ -84,32 +93,64 @@ def get_authenticator(config: Mapping[str, Any]): def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: try: + original_config = config.copy() config = self._validate_and_transform(config) authenticator = self.get_authenticator(config) kwargs = {"authenticator": authenticator, "domain": config["domain"], "projects": config["projects"]} - labels_stream = Labels(**kwargs) - next(read_full_refresh(labels_stream), None) + # check projects projects_stream = Projects(**kwargs) projects = {project["key"] for project in read_full_refresh(projects_stream)} unknown_projects = set(config["projects"]) - projects if unknown_projects: return False, "unknown project(s): " + ", ".join(unknown_projects) - return True, None - except (requests.exceptions.RequestException, ValidationError) as e: - return False, e + + # Get streams to check access to any of them + streams = self.streams(original_config) + for stream in streams: + try: + next(read_full_refresh(stream), None) + except: + logger.warning("No access to stream: " + stream.name) + else: + logger.info(f"API Token have access to stream: {stream.name}, so check is successful.") + return True, None + return False, "This API Token does not have permission to read any of the resources." + except ValidationError as validation_error: + return False, validation_error + except requests.exceptions.RequestException as request_error: + has_response = request_error.response is not None + is_invalid_domain = ( + isinstance(request_error, requests.exceptions.InvalidURL) + or has_response + and request_error.response.status_code == requests.codes.not_found + ) + + if is_invalid_domain: + raise AirbyteTracedException( + message="Config validation error: please check that your domain is valid and does not include protocol (e.g: https://).", + internal_message=str(request_error), + failure_type=FailureType.config_error, + ) from None + + # sometimes jira returns non json response + if has_response and request_error.response.headers.get("content-type") == "application/json": + message = " ".join(map(str, request_error.response.json().get("errorMessages", ""))) + return False, f"{message} {request_error}" + + # we don't know what this is, rethrow it + raise request_error def streams(self, config: Mapping[str, Any]) -> List[Stream]: config = self._validate_and_transform(config) authenticator = self.get_authenticator(config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config["projects"]} - incremental_args = {**args, "start_date": config.get("start_date")} - render_fields = config.get("render_fields", False) - issues_stream = Issues( - **incremental_args, - expand_changelog=config.get("expand_issue_changelog", False), - render_fields=render_fields, - ) + incremental_args = { + **args, + "start_date": config.get("start_date"), + "lookback_window_minutes": config.get("lookback_window_minutes"), + } + issues_stream = Issues(**incremental_args) issue_fields_stream = IssueFields(**args) experimental_streams = [] if config.get("enable_experimental_streams", False): @@ -130,6 +171,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: issue_fields_stream, IssueFieldConfigurations(**args), IssueCustomFieldContexts(**args), + IssueCustomFieldOptions(**args), IssueLinkTypes(**args), IssueNavigatorSettings(**args), IssueNotificationSchemes(**args), @@ -138,7 +180,9 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: IssueRemoteLinks(**incremental_args), IssueResolutions(**args), IssueSecuritySchemes(**args), + IssueTransitions(**args), IssueTypeSchemes(**args), + IssueTypes(**args), IssueTypeScreenSchemes(**args), IssueVotes(**incremental_args), IssueWatchers(**incremental_args), @@ -148,6 +192,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Permissions(**args), PermissionSchemes(**args), Projects(**args), + ProjectRoles(**args), ProjectAvatars(**args), ProjectCategories(**args), ProjectComponents(**args), diff --git a/airbyte-integrations/connectors/source-jira/source_jira/spec.json b/airbyte-integrations/connectors/source-jira/source_jira/spec.json index 3fa5fe792cb1..fff67ec349e6 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/spec.json +++ b/airbyte-integrations/connectors/source-jira/source_jira/spec.json @@ -53,23 +53,51 @@ "expand_issue_changelog": { "type": "boolean", "title": "Expand Issue Changelog", - "description": "Expand the changelog when replicating issues.", - "default": false, - "order": 5 + "airbyte_hidden": true, + "description": "(DEPRECATED) Expand the changelog when replicating issues.", + "default": false }, "render_fields": { "type": "boolean", "title": "Render Issue Fields", - "description": "Render issue fields in HTML format in addition to Jira JSON-like format.", - "default": false, - "order": 6 + "airbyte_hidden": true, + "description": "(DEPRECATED) Render issue fields in HTML format in addition to Jira JSON-like format.", + "default": false + }, + "expand_issue_transition": { + "type": "boolean", + "title": "Expand Issue Transitions", + "airbyte_hidden": true, + "description": "(DEPRECATED) Expand the transitions when replicating issues.", + "default": false + }, + "issues_stream_expand_with": { + "type": "array", + "items": { + "type": "string", + "enum": ["renderedFields", "transitions", "changelog"] + }, + "title": "Expand Issues stream", + "airbyte_hidden": true, + "description": "Select fields to Expand the `Issues` stream when replicating with: ", + "default": [] + }, + "lookback_window_minutes": { + "title": "Lookback window", + "description": "When set to N, the connector will always refresh resources created within the past N minutes. By default, updated objects that are not newly created are not incrementally synced.", + "examples": [60], + "default": 0, + "minimum": 0, + "maximum": 576000, + "type": "integer", + "order": 5 }, "enable_experimental_streams": { "type": "boolean", "title": "Enable Experimental Streams", "description": "Allow the use of experimental streams which rely on undocumented Jira API endpoints. See https://docs.airbyte.com/integrations/sources/jira#experimental-tables for more info.", "default": false, - "order": 7 + "order": 6 } } } diff --git a/airbyte-integrations/connectors/source-jira/source_jira/streams.py b/airbyte-integrations/connectors/source-jira/source_jira/streams.py index cddb6fb1e54e..542ea67ea59d 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/streams.py +++ b/airbyte-integrations/connectors/source-jira/source_jira/streams.py @@ -5,19 +5,39 @@ import re import urllib.parse as urlparse from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional +from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Union from urllib.parse import parse_qsl import pendulum import requests +from airbyte_cdk.logger import AirbyteLogger as Logger +from airbyte_cdk.sources import Source +from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy +from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from requests.exceptions import HTTPError +from source_jira.type_transfromer import DateTimeTransformer from .utils import read_full_refresh, read_incremental, safe_max API_VERSION = 3 +class JiraAvailabilityStrategy(HttpAvailabilityStrategy): + """ + Inherit from HttpAvailabilityStrategy with slight modification to 403 and 401 error messages. + """ + + def reasons_for_unavailable_status_codes(self, stream: Stream, logger: Logger, source: Source, error: HTTPError) -> Dict[int, str]: + reasons_for_codes: Dict[int, str] = { + requests.codes.FORBIDDEN: "Please check the 'READ' permission(Scopes for Connect apps) and/or the user has Jira Software rights and access.", + requests.codes.UNAUTHORIZED: "Invalid creds were provided, please check your api token, domain and/or email.", + requests.codes.NOT_FOUND: "Please check the 'READ' permission(Scopes for Connect apps) and/or the user has Jira Software rights and access.", + } + return reasons_for_codes + + class JiraStream(HttpStream, ABC): """ Jira API Reference: https://developer.atlassian.com/cloud/jira/platform/rest/v3/intro/ @@ -27,8 +47,13 @@ class JiraStream(HttpStream, ABC): primary_key: Optional[str] = "id" extract_field: Optional[str] = None api_v1 = False - skip_http_status_codes = [] + # Defines the HTTP status codes for which the slice should be skipped. + # Refernce issue: https://github.com/airbytehq/oncall/issues/2133 + # we should skip the slice with `board id` which doesn't support `sprints` + # it's generally applied to all streams that might have the same error hit in the future. + skip_http_status_codes = [requests.codes.BAD_REQUEST] raise_on_http_errors = True + transformer: TypeTransformer = DateTimeTransformer(TransformConfig.DefaultSchemaNormalization) def __init__(self, domain: str, projects: List[str], **kwargs): super().__init__(**kwargs) @@ -41,6 +66,19 @@ def url_base(self) -> str: return f"https://{self._domain}/rest/agile/1.0/" return f"https://{self._domain}/rest/api/{API_VERSION}/" + @property + def availability_strategy(self) -> HttpAvailabilityStrategy: + return JiraAvailabilityStrategy() + + def _get_custom_error(self, response: requests.Response) -> str: + """Method for specifying custom error messages for errors that will be skipped.""" + return "" + + @property + def max_retries(self) -> Union[int, None]: + """Number of retries increased from default 5 to 10, based on issues with Jira. Max waiting time is still default 10 minutes.""" + return 10 + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: response_json = response.json() if isinstance(response_json, dict): @@ -92,24 +130,20 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: except HTTPError as e: if not (self.skip_http_status_codes and e.response.status_code in self.skip_http_status_codes): raise e - - def should_retry(self, response: requests.Response) -> bool: - if response.status_code == requests.codes.bad_request: - # Refernce issue: https://github.com/airbytehq/oncall/issues/2133 - # we should skip the slice with `board id` which doesn't support `sprints` - # it's generally applied to all streams that might have the same error hit in the future. - errors = response.json().get("errorMessages") - self.logger.error(f"Stream `{self.name}`. An error occured, details: {errors}. Skipping.") - setattr(self, "raise_on_http_errors", False) - return False - else: - # for all other HTTP errors the defaul handling is applied - return super().should_retry(response) + errors = e.response.json().get("errorMessages") + custom_error = self._get_custom_error(e.response) + self.logger.warning(f"Stream `{self.name}`. An error occurred, details: {errors}. Skipping for now. {custom_error}") class StartDateJiraStream(JiraStream, ABC): - def __init__(self, start_date: Optional[pendulum.DateTime] = None, **kwargs): + def __init__( + self, + start_date: Optional[pendulum.DateTime] = None, + lookback_window_minutes: pendulum.Duration = pendulum.duration(minutes=0), + **kwargs, + ): super().__init__(**kwargs) + self._lookback_window_minutes = lookback_window_minutes self._start_date = start_date @@ -141,7 +175,7 @@ def _get_starting_point(self, stream_state: Mapping[str, Any]) -> Optional[pendu if stream_state: stream_state_value = stream_state.get(self.cursor_field) if stream_state_value: - stream_state_value = pendulum.parse(stream_state_value) + stream_state_value = pendulum.parse(stream_state_value) - self._lookback_window_minutes return safe_max(stream_state_value, self._start_date) return self._start_date @@ -165,10 +199,6 @@ class ApplicationRoles(JiraStream): """ primary_key = "key" - skip_http_status_codes = [ - # Application access permissions can only be edited or viewed by administrators. - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "applicationrole" @@ -216,7 +246,7 @@ def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, return record -class BoardIssues(IncrementalJiraStream): +class BoardIssues(StartDateJiraStream): """ https://developer.atlassian.com/cloud/jira/software/rest/api-group-board/#api-rest-agile-1-0-board-boardid-issue-get """ @@ -224,9 +254,11 @@ class BoardIssues(IncrementalJiraStream): cursor_field = "updated" extract_field = "issues" api_v1 = True + state_checkpoint_interval = 50 # default page size is 50 def __init__(self, **kwargs): super().__init__(**kwargs) + self._starting_point_cache = {} self.boards_stream = Boards(authenticator=self.authenticator, domain=self._domain, projects=self._projects) def path(self, stream_slice: Mapping[str, Any], **kwargs) -> str: @@ -240,14 +272,68 @@ def request_params( ) -> MutableMapping[str, Any]: params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) params["fields"] = ["key", "created", "updated"] - jql = self.jql_compare_date(stream_state) + jql = self.jql_compare_date(stream_state, stream_slice) if jql: params["jql"] = jql return params + def jql_compare_date(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any]) -> Optional[str]: + compare_date = self.get_starting_point(stream_state, stream_slice) + if compare_date: + compare_date = compare_date.strftime("%Y/%m/%d %H:%M") + return f"{self.cursor_field} >= '{compare_date}'" + + def _is_board_error(self, response): + """Check if board has error and should be skipped""" + if response.status_code == 500: + if "This board has no columns with a mapped status." in response.text: + return True + + def should_retry(self, response: requests.Response) -> bool: + if self._is_board_error(response): + return False + + # for all other HTTP errors the default handling is applied + return super().should_retry(response) + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + yield from read_full_refresh(self.boards_stream) + def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: - for board in read_full_refresh(self.boards_stream): - yield from super().read_records(stream_slice={"board_id": board["id"]}, **kwargs) + try: + yield from super().read_records(stream_slice={"board_id": stream_slice["id"]}, **kwargs) + except HTTPError as e: + if self._is_board_error(e.response): + # Wrong board is skipped + self.logger.warning(f"Board {stream_slice['id']} has no columns with a mapped status. Skipping.") + else: + raise + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): + updated_state = latest_record[self.cursor_field] + board_id = str(latest_record["boardId"]) + stream_state_value = current_stream_state.get(board_id, {}).get(self.cursor_field) + if stream_state_value: + updated_state = max(updated_state, stream_state_value) + current_stream_state.setdefault(board_id, {})[self.cursor_field] = updated_state + return current_stream_state + + def get_starting_point(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any]) -> Optional[pendulum.DateTime]: + board_id = str(stream_slice["board_id"]) + if self.cursor_field not in self._starting_point_cache: + self._starting_point_cache.setdefault(board_id, {})[self.cursor_field] = self._get_starting_point( + stream_state=stream_state, stream_slice=stream_slice + ) + return self._starting_point_cache[board_id][self.cursor_field] + + def _get_starting_point(self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any]) -> Optional[pendulum.DateTime]: + if stream_state: + board_id = str(stream_slice["board_id"]) + stream_state_value = stream_state.get(board_id, {}).get(self.cursor_field) + if stream_state_value: + stream_state_value = pendulum.parse(stream_state_value) - self._lookback_window_minutes + return safe_max(stream_state_value, self._start_date) + return self._start_date def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: record["boardId"] = stream_slice["board_id"] @@ -300,6 +386,10 @@ def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwarg for filters in read_full_refresh(self.filters_stream): yield from super().read_records(stream_slice={"filter_id": filters["id"]}, **kwargs) + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + record["filterId"] = stream_slice["filter_id"] + return record + class Groups(JiraStream): """ @@ -320,12 +410,16 @@ class Issues(IncrementalJiraStream): cursor_field = "updated" extract_field = "issues" - use_cache = False # disable caching due to OOM errors in kubernetes + use_cache = True + _expand_fields_list = ["renderedFields", "transitions", "changelog"] - def __init__(self, expand_changelog: bool = False, render_fields: bool = False, **kwargs): + # Issue: https://github.com/airbytehq/airbyte/issues/26712 + # we should skip the slice with wrong permissions on project level + skip_http_status_codes = [requests.codes.FORBIDDEN, requests.codes.BAD_REQUEST] + state_checkpoint_interval = 50 # default page size is 50 + + def __init__(self, **kwargs): super().__init__(**kwargs) - self._expand_changelog = expand_changelog - self._render_fields = render_fields self._project_ids = [] self.issue_fields_stream = IssueFields(authenticator=self.authenticator, domain=self._domain, projects=self._projects) self.projects_stream = Projects(authenticator=self.authenticator, domain=self._domain, projects=self._projects) @@ -341,17 +435,14 @@ def request_params( ) -> MutableMapping[str, Any]: params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) params["fields"] = "*all" + jql_parts = [self.jql_compare_date(stream_state)] if self._project_ids: jql_parts.append(f"project in ({stream_slice.get('project_id')})") params["jql"] = " and ".join([p for p in jql_parts if p]) - expand = [] - if self._expand_changelog: - expand.append("changelog") - if self._render_fields: - expand.append("renderedFields") - if expand: - params["expand"] = ",".join(expand) + params["jql"] += f" ORDER BY {self.cursor_field} asc" + + params["expand"] = ",".join(self._expand_fields_list) return params def transform(self, record: MutableMapping[str, Any], **kwargs) -> MutableMapping[str, Any]: @@ -359,6 +450,12 @@ def transform(self, record: MutableMapping[str, Any], **kwargs) -> MutableMappin record["projectKey"] = record["fields"]["project"]["key"] record["created"] = record["fields"]["created"] record["updated"] = record["fields"]["updated"] + + # remove fields that are None + if "renderedFields" in record: + record["renderedFields"] = {k: v for k, v in record["renderedFields"].items() if v is not None} + if "fields" in record: + record["fields"] = {k: v for k, v in record["fields"].items() if v is not None} return record def get_project_ids(self): @@ -376,18 +473,10 @@ def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: else: yield from super().stream_slices(**kwargs) - def should_retry(self, response: requests.Response) -> bool: - if response.status_code == requests.codes.bad_request: - # Issue: https://github.com/airbytehq/airbyte/issues/26712 - # we should skip the slice with wrong permissions on project level - errors = response.json().get("errorMessages") - self.logger.error( - f"Stream `{self.name}`. An error occurred, details: {errors}." f"Check permissions for this project. Skipping for now." - ) - setattr(self, "raise_on_http_errors", False) - return False - else: - return super().should_retry(response) + def _get_custom_error(self, response: requests.Response) -> str: + if response.status_code == requests.codes.BAD_REQUEST: + return "The user doesn't have permission to the project. Please grant the user to the project." + return "" class IssueComments(IncrementalJiraStream): @@ -417,6 +506,10 @@ def read_records( stream_slice = {"key": issue["key"]} yield from super().read_records(stream_slice=stream_slice, stream_state=stream_state, **kwargs) + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + record["issueId"] = stream_slice["key"] + return record + class IssueFields(JiraStream): """ @@ -441,10 +534,6 @@ class IssueFieldConfigurations(JiraStream): """ extract_field = "values" - skip_http_status_codes = [ - # Only Jira administrators can access field configurations - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "fieldconfiguration" @@ -455,6 +544,7 @@ class IssueCustomFieldContexts(JiraStream): https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-custom-field-contexts/#api-rest-api-3-field-fieldid-context-get """ + use_cache = True extract_field = "values" skip_http_status_codes = [ # https://community.developer.atlassian.com/t/get-custom-field-contexts-not-found-returned/48408/2 @@ -462,6 +552,7 @@ class IssueCustomFieldContexts(JiraStream): requests.codes.NOT_FOUND, # Only Jira administrators can access custom field contexts. requests.codes.FORBIDDEN, + requests.codes.BAD_REQUEST, ] def __init__(self, **kwargs): @@ -474,7 +565,48 @@ def path(self, stream_slice: Mapping[str, Any], **kwargs) -> str: def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: for field in read_full_refresh(self.issue_fields_stream): if field.get("custom", False): - yield from super().read_records(stream_slice={"field_id": field["id"]}, **kwargs) + yield from super().read_records( + stream_slice={"field_id": field["id"], "field_type": field.get("schema", {}).get("type")}, **kwargs + ) + + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + record["fieldId"] = stream_slice["field_id"] + record["fieldType"] = stream_slice["field_type"] + return record + + +class IssueCustomFieldOptions(JiraStream): + """ + https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-custom-field-options/#api-rest-api-3-field-fieldid-context-contextid-option-get + """ + + skip_http_status_codes = [ + requests.codes.NOT_FOUND, + # Only Jira administrators can access custom field options. + requests.codes.FORBIDDEN, + requests.codes.BAD_REQUEST, + ] + + extract_field = "values" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.issue_custom_field_contexts_stream = IssueCustomFieldContexts( + authenticator=self.authenticator, domain=self._domain, projects=self._projects + ) + + def path(self, stream_slice: Mapping[str, Any], **kwargs) -> str: + return f"field/{stream_slice['field_id']}/context/{stream_slice['context_id']}/option" + + def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + for record in read_full_refresh(self.issue_custom_field_contexts_stream): + if record.get("fieldType") == "option": + yield from super().read_records(stream_slice={"field_id": record["fieldId"], "context_id": record["id"]}, **kwargs) + + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + record["fieldId"] = stream_slice["field_id"] + record["contextId"] = stream_slice["context_id"] + return record class IssueLinkTypes(JiraStream): @@ -494,10 +626,6 @@ class IssueNavigatorSettings(JiraStream): """ primary_key = None - skip_http_status_codes = [ - # You need Administrator permission to perform this operation. - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "settings/columns" @@ -532,6 +660,11 @@ class IssuePropertyKeys(JiraStream): extract_field = "keys" use_cache = True + skip_http_status_codes = [ + # Issue does not exist or you do not have permission to see it. + requests.codes.NOT_FOUND, + requests.codes.BAD_REQUEST, + ] def path(self, stream_slice: Mapping[str, Any], **kwargs) -> str: key = stream_slice["key"] @@ -567,6 +700,10 @@ def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwarg for property_key in self.issue_property_keys_stream.read_records(stream_slice={"key": issue["key"]}, **kwargs): yield from super().read_records(stream_slice={"key": property_key["key"], "issue_key": issue["key"]}, **kwargs) + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + record["issueId"] = stream_slice["issue_key"] + return record + class IssueRemoteLinks(StartDateJiraStream): """ @@ -592,6 +729,10 @@ def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwarg def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: return None + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + record["issueId"] = stream_slice["key"] + return record + class IssueResolutions(JiraStream): """ @@ -610,25 +751,26 @@ class IssueSecuritySchemes(JiraStream): """ extract_field = "issueSecuritySchemes" - skip_http_status_codes = [ - # You need to be a Jira administrator to perform this operation - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "issuesecurityschemes" +class IssueTypes(JiraStream): + """ + https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-types/#api-group-issue-types + """ + + def path(self, **kwargs) -> str: + return "issuetype" + + class IssueTypeSchemes(JiraStream): """ https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-type-schemes/#api-rest-api-3-issuetypescheme-get """ extract_field = "values" - skip_http_status_codes = [ - # Only Jira administrators can access issue type schemes. - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "issuetypescheme" @@ -640,15 +782,43 @@ class IssueTypeScreenSchemes(JiraStream): """ extract_field = "values" - skip_http_status_codes = [ - # Only Jira administrators can access issue type screen schemes. - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "issuetypescreenscheme" +class IssueTransitions(StartDateJiraStream): + """ + https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issues/#api-rest-api-3-issue-issueidorkey-transitions-get + """ + + primary_key = ["issueId", "id"] + extract_field = "transitions" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.issues_stream = Issues( + authenticator=self.authenticator, + domain=self._domain, + projects=self._projects, + start_date=self._start_date, + ) + + def path(self, stream_slice: Mapping[str, Any], **kwargs) -> str: + return f"issue/{stream_slice['key']}/transitions" + + def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + for issue in read_full_refresh(self.issues_stream): + yield from super().read_records(stream_slice={"key": issue["key"]}, **kwargs) + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + record["issueId"] = stream_slice["key"] + return record + + class IssueVotes(StartDateJiraStream): """ https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-votes/#api-rest-api-3-issue-issueidorkey-votes-get @@ -678,6 +848,10 @@ def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwarg for issue in read_full_refresh(self.issues_stream): yield from super().read_records(stream_slice={"key": issue["key"]}, **kwargs) + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + record["issueId"] = stream_slice["key"] + return record + class IssueWatchers(StartDateJiraStream): """ @@ -688,6 +862,11 @@ class IssueWatchers(StartDateJiraStream): # extract_field = "watchers" primary_key = None + skip_http_status_codes = [ + # Issue is not found or the user does not have permission to view it. + requests.codes.NOT_FOUND, + requests.codes.BAD_REQUEST, + ] def __init__(self, **kwargs): super().__init__(**kwargs) @@ -705,6 +884,10 @@ def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwarg for issue in read_full_refresh(self.issues_stream): yield from super().read_records(stream_slice={"key": issue["key"]}, **kwargs) + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + record["issueId"] = stream_slice["key"] + return record + class IssueWorklogs(IncrementalJiraStream): """ @@ -739,11 +922,6 @@ class JiraSettings(JiraStream): https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-jira-settings/#api-rest-api-3-application-properties-get """ - skip_http_status_codes = [ - # No permission - requests.codes.FORBIDDEN - ] - def path(self, **kwargs) -> str: return "application-properties" @@ -770,10 +948,6 @@ class Permissions(JiraStream): extract_field = "permissions" primary_key = "key" - skip_http_status_codes = [ - # You need to have Administer permissions to view this resource - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "permissions" @@ -809,6 +983,7 @@ def path(self, **kwargs) -> str: def request_params(self, **kwargs): params = super().request_params(**kwargs) params["expand"] = "description,lead" + params["status"] = ["live", "archived", "deleted"] return params def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: @@ -822,6 +997,12 @@ class ProjectAvatars(JiraStream): https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-avatars/#api-rest-api-3-project-projectidorkey-avatars-get """ + skip_http_status_codes = [ + # Project is not found or the user does not have permission to view the project. + requests.codes.UNAUTHORIZED, + requests.codes.NOT_FOUND, + ] + def __init__(self, **kwargs): super().__init__(**kwargs) self.projects_stream = Projects(authenticator=self.authenticator, domain=self._domain, projects=self._projects) @@ -831,8 +1012,11 @@ def path(self, stream_slice: Mapping[str, Any], **kwargs) -> str: def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: response_json = response.json() + stream_slice = kwargs["stream_slice"] for records in response_json.values(): - yield from records + for record in records: + record["projectId"] = stream_slice["key"] + yield record def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: for project in read_full_refresh(self.projects_stream): @@ -844,6 +1028,12 @@ class ProjectCategories(JiraStream): https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-categories/#api-rest-api-3-projectcategory-get """ + skip_http_status_codes = [ + # Project is not found or the user does not have permission to view the project. + requests.codes.UNAUTHORIZED, + requests.codes.NOT_FOUND, + ] + def path(self, **kwargs) -> str: return "projectCategory" @@ -875,7 +1065,8 @@ class ProjectEmail(JiraStream): primary_key = "projectId" skip_http_status_codes = [ # You cannot edit the configuration of this project. - requests.codes.FORBIDDEN + requests.codes.FORBIDDEN, + requests.codes.BAD_REQUEST, ] def __init__(self, **kwargs): @@ -912,6 +1103,21 @@ def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwarg for project in read_full_refresh(self.projects_stream): yield from super().read_records(stream_slice={"key": project["key"]}, **kwargs) + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + record["projectId"] = stream_slice["key"] + return record + + +class ProjectRoles(JiraStream): + """ + https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-roles#api-rest-api-3-role-get + """ + + primary_key = "id" + + def path(self, **kwargs) -> str: + return "role" + class ProjectTypes(JiraStream): """ @@ -1016,10 +1222,6 @@ class Screens(JiraStream): extract_field = "values" use_cache = True - skip_http_status_codes = [ - # Only Jira administrators can manage screens. - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "screens" @@ -1057,6 +1259,10 @@ def read_tab_records(self, stream_slice: Mapping[str, Any], **kwargs) -> Iterabl return yield screen_tab + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + record["screenId"] = stream_slice["screen_id"] + return record + class ScreenTabFields(JiraStream): """ @@ -1077,6 +1283,11 @@ def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwarg if "id" in tab: # Check for proper tab record since the ScreenTabs stream doesn't throw http errors yield from super().read_records(stream_slice={"screen_id": screen["id"], "tab_id": tab["id"]}, **kwargs) + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + record["screenId"] = stream_slice["screen_id"] + record["tabId"] = stream_slice["tab_id"] + return record + class ScreenSchemes(JiraStream): """ @@ -1084,10 +1295,6 @@ class ScreenSchemes(JiraStream): """ extract_field = "values" - skip_http_status_codes = [ - # Only Jira administrators can access screen schemes. - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "screenscheme" @@ -1106,6 +1313,18 @@ def __init__(self, **kwargs): super().__init__(**kwargs) self.boards_stream = Boards(authenticator=self.authenticator, domain=self._domain, projects=self._projects) + def _get_custom_error(self, response: requests.Response) -> str: + if response.status_code == requests.codes.BAD_REQUEST: + errors = response.json().get("errorMessages") + for error_message in errors: + if "The board does not support sprints" in error_message: + return ( + "The board does not support sprints. The board does not have a sprint board. if it's a team-managed one, " + "does it have sprints enabled under project settings? If it's a company-managed one," + " check that it has at least one Scrum board associated with it." + ) + return "" + def path(self, stream_slice: Mapping[str, Any], **kwargs) -> str: return f"board/{stream_slice['board_id']}/sprint" @@ -1117,6 +1336,10 @@ def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwarg self.logger.info(f"Fetching sprints for board: {board_details}") yield from super().read_records(stream_slice={"board_id": board["id"]}, **kwargs) + def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any], **kwargs) -> MutableMapping[str, Any]: + record["boardId"] = stream_slice["board_id"] + return record + class SprintIssues(IncrementalJiraStream): """ @@ -1177,10 +1400,6 @@ class TimeTracking(JiraStream): """ primary_key = "key" - skip_http_status_codes = [ - # This resource is only available to administrators - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "configuration/timetracking/list" @@ -1234,10 +1453,6 @@ class Workflows(JiraStream): """ extract_field = "values" - skip_http_status_codes = [ - # Only Jira administrators can access workflows. - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "workflow/search" @@ -1249,10 +1464,6 @@ class WorkflowSchemes(JiraStream): """ extract_field = "values" - skip_http_status_codes = [ - # Only Jira administrators can access workflow scheme associations. - requests.codes.FORBIDDEN - ] def path(self, **kwargs) -> str: return "workflowscheme" diff --git a/airbyte-integrations/connectors/source-jira/source_jira/type_transfromer.py b/airbyte-integrations/connectors/source-jira/source_jira/type_transfromer.py new file mode 100644 index 000000000000..16fdfb000a4c --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/source_jira/type_transfromer.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from datetime import datetime +from typing import Any, Dict + +from airbyte_cdk.sources.utils.transform import TypeTransformer + +logger = logging.getLogger("airbyte") + + +class DateTimeTransformer(TypeTransformer): + api_date_time_format = "%Y-%m-%dT%H:%M:%S.%f%z" + + @staticmethod + def default_convert(original_item: Any, subschema: Dict[str, Any]) -> Any: + target_format = subschema.get("format", "") + if target_format == "date-time": + if isinstance(original_item, str): + try: + date = datetime.strptime(original_item, DateTimeTransformer.api_date_time_format) + return date.isoformat() + except ValueError: + logger.warning(f"{original_item}: doesn't match expected format.") + # returning original item in case we received another date format + return original_item + # we don't need to convert other types + return original_item diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/conftest.py b/airbyte-integrations/connectors/source-jira/unit_tests/conftest.py index 7c5ff589790b..4421dbc4641f 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/conftest.py @@ -8,7 +8,114 @@ import responses from pytest import fixture from responses import matchers -from source_jira.streams import Projects +from source_jira.streams import ( + ApplicationRoles, + Avatars, + BoardIssues, + Boards, + Dashboards, + Filters, + FilterSharing, + Groups, + IssueComments, + IssueCustomFieldContexts, + IssueFieldConfigurations, + IssueFields, + IssueLinkTypes, + IssueNavigatorSettings, + IssueNotificationSchemes, + IssuePriorities, + IssuePropertyKeys, + IssueRemoteLinks, + IssueResolutions, + Issues, + IssueSecuritySchemes, + IssueTypeSchemes, + IssueVotes, + IssueWatchers, + IssueWorklogs, + JiraSettings, + Labels, + Permissions, + ProjectAvatars, + ProjectCategories, + ProjectComponents, + ProjectEmail, + ProjectPermissionSchemes, + Projects, + ProjectVersions, + Screens, + ScreenTabs, + SprintIssues, + Sprints, + TimeTracking, + Users, + UsersGroupsDetailed, + Workflows, + WorkflowSchemes, + WorkflowStatusCategories, + WorkflowStatuses, +) + + +@fixture(scope="session", autouse=True) +def disable_cache(): + classes = [ + ApplicationRoles, + Avatars, + BoardIssues, + Boards, + Dashboards, + Filters, + FilterSharing, + Groups, + IssueComments, + IssueCustomFieldContexts, + IssueFieldConfigurations, + IssueFields, + IssueLinkTypes, + IssueNavigatorSettings, + IssueNotificationSchemes, + IssuePriorities, + IssuePropertyKeys, + IssueRemoteLinks, + IssueResolutions, + Issues, + IssueSecuritySchemes, + IssueTypeSchemes, + IssueVotes, + IssueWatchers, + IssueWorklogs, + JiraSettings, + Labels, + Permissions, + ProjectAvatars, + ProjectCategories, + ProjectComponents, + ProjectEmail, + ProjectPermissionSchemes, + Projects, + ProjectVersions, + Screens, + ScreenTabs, + SprintIssues, + Sprints, + TimeTracking, + Users, + UsersGroupsDetailed, + Workflows, + WorkflowSchemes, + WorkflowStatusCategories, + WorkflowStatuses, + ] + for cls in classes: + # Disabling cache for all streams to assess the number of calls made for each stream. + # Additionally, this is necessary as the responses library has been returning unexpected call counts + # following the recent update to HttpStream + cls.use_cache = False + + +os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" @fixture @@ -18,7 +125,7 @@ def config(): "domain": "domain", "email": "email@email.com", "start_date": "2021-01-01T00:00:00Z", - "projects": ["Project1"] + "projects": ["Project1"], } @@ -206,6 +313,11 @@ def issue_custom_field_contexts_response(): return json.loads(load_file("issue_custom_field_contexts.json")) +@fixture +def issue_custom_field_options_response(): + return json.loads(load_file("issue_custom_field_options.json")) + + @fixture def issue_property_keys_response(): return json.loads(load_file("issue_property_keys.json")) @@ -266,7 +378,18 @@ def mock_projects_responses(config, projects_response): Projects.use_cache = False responses.add( responses.GET, - f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description%2Clead", + f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description%2Clead&status=live&status=archived&status=deleted", + json=projects_response, + ) + + +@fixture +def mock_projects_responses_additional_project(config, projects_response): + Projects.use_cache = False + projects_response["values"] += [{"id": "3", "key": "Project3"}, {"id": "4", "key": "Project4"}] + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description%2Clead&status=live&status=archived&status=deleted", json=projects_response, ) @@ -276,12 +399,229 @@ def mock_issues_responses(config, issues_response): responses.add( responses.GET, f"https://{config['domain']}/rest/api/3/search", - match=[matchers.query_param_matcher({"maxResults": 50, "fields": '*all', "jql": "project in (1)"})], + match=[ + matchers.query_param_matcher( + { + "maxResults": 50, + "fields": "*all", + "jql": "project in (1) ORDER BY updated asc", + "expand": "renderedFields,transitions,changelog", + } + ) + ], json=issues_response, ) responses.add( responses.GET, f"https://{config['domain']}/rest/api/3/search", - match=[matchers.query_param_matcher({"maxResults": 50, "fields": '*all', "jql": "project in (2)"})], + match=[ + matchers.query_param_matcher( + { + "maxResults": 50, + "fields": "*all", + "jql": "project in (2) ORDER BY updated asc", + "expand": "renderedFields,transitions,changelog", + } + ) + ], json={}, ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/search", + match=[ + matchers.query_param_matcher( + { + "maxResults": 50, + "fields": "*all", + "jql": "project in (3) ORDER BY updated asc", + "expand": "renderedFields,transitions,changelog", + } + ) + ], + json={"errorMessages": ["The value '3' does not exist for the field 'project'."]}, + status=400, + ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/search", + match=[ + matchers.query_param_matcher( + { + "maxResults": 50, + "fields": "*all", + "jql": "project in (4) ORDER BY updated asc", + "expand": "renderedFields,transitions,changelog", + } + ) + ], + json={ + "issues": [ + { + "key": "TESTKEY13-2", + "fields": { + "project": { + "id": "10016", + "key": "TESTKEY13", + }, + "created": "2022-06-09T16:29:31.871-0700", + "updated": "2022-12-08T02:22:18.889-0800", + }, + } + ] + }, + ) + + +@fixture +def mock_project_emails(config, project_email_response): + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/project/1/email?maxResults=50", + json=project_email_response, + ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/project/2/email?maxResults=50", + json=project_email_response, + ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/project/3/email?maxResults=50", + json={"errorMessages": ["No access to emails for project 3"]}, + status=403, + ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/project/4/email?maxResults=50", + json=project_email_response, + ) + + +@fixture +def mock_issue_watchers_responses(config, issue_watchers_response): + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/issue/TESTKEY13-1/watchers?maxResults=50", + json=issue_watchers_response, + ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/issue/TESTKEY13-2/watchers?maxResults=50", + json={"errorMessages": ["Not found watchers for issue TESTKEY13-2"]}, + status=404, + ) + + +@fixture +def mock_issue_custom_field_contexts_response(config, issue_custom_field_contexts_response): + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/field/issuetype/context?maxResults=50", + json=issue_custom_field_contexts_response, + ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/field/issuetype2/context?maxResults=50", + json={}, + ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/field/issuetype3/context?maxResults=50", + json={}, + ) + + +@fixture +def mock_issue_custom_field_contexts_response_error(config, issue_custom_field_contexts_response): + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/field/issuetype/context?maxResults=50", + json=issue_custom_field_contexts_response, + ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/field/issuetype2/context?maxResults=50", + json={"errorMessages": ["Not found issue custom field context for issue fields issuetype2"]}, + status=404, + ) + responses.add(responses.GET, f"https://{config['domain']}/rest/api/3/field/issuetype3/context?maxResults=50", json={}) + + +@fixture +def mock_issue_custom_field_options_response(config, issue_custom_field_options_response): + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/field/issuetype/context/10130/option?maxResults=50", + json=issue_custom_field_options_response, + ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/field/issuetype/context/10129/option?maxResults=50", + json={"errorMessages": ["Not found issue custom field options for issue fields issuetype3"]}, + status=404, + ) + + +@fixture +def mock_fields_response(config, issue_fields_response): + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/field?maxResults=50", + json=issue_fields_response, + ) + + +@fixture +def mock_users_response(config, users_response): + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/users/search?maxResults=50", + json=users_response, + ) + + +@fixture +def mock_board_response(config, boards_response): + responses.add( + responses.GET, + f"https://{config['domain']}/rest/agile/1.0/board?maxResults=50", + json=boards_response, + ) + + +@fixture +def mock_screen_response(config, screens_response): + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/screens?maxResults=50", + json=screens_response, + ) + + +@fixture +def mock_filter_response(config, filters_response): + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/filter/search?maxResults=50&expand=description%2Cowner%2Cjql%2CviewUrl%2CsearchUrl%2Cfavourite%2CfavouritedCount%2CsharePermissions%2CisWritable%2Csubscriptions", + json=filters_response, + ) + + +@fixture +def mock_sprints_response(config, sprints_response): + responses.add( + responses.GET, + f"https://{config['domain']}/rest/agile/1.0/board/1/sprint?maxResults=50", + json=sprints_response, + ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/agile/1.0/board/2/sprint?maxResults=50", + json=sprints_response, + ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/agile/1.0/board/3/sprint?maxResults=50", + json=sprints_response, + ) diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_custom_field_contexts.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_custom_field_contexts.json index ead4a03913ba..39e40191fcd1 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_custom_field_contexts.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_custom_field_contexts.json @@ -5,14 +5,16 @@ "name": "Default Configuration Scheme for Account", "description": "Default configuration scheme generated by Jira", "isGlobalContext": true, - "isAnyIssueType": true + "isAnyIssueType": true, + "fieldType": "option" }, { "id": "10129", "name": "Default Configuration Scheme for Team", "description": "Default configuration scheme generated by Jira", "isGlobalContext": true, - "isAnyIssueType": true + "isAnyIssueType": true, + "fieldType": "option" } ] } diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_custom_field_options.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_custom_field_options.json new file mode 100644 index 000000000000..2be8d7e1c7e5 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_custom_field_options.json @@ -0,0 +1,11 @@ +{ + "values": [ + { + "id": "10016", + "value": "To Do", + "disabled": false, + "fieldId": "customfield_10012", + "contextId": "10112" + } + ] +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_fields.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_fields.json index 05a312fc78a8..37ac23f16ab3 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_fields.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issue_fields.json @@ -23,7 +23,7 @@ "searchable": true, "clauseNames": ["issuetype", "type"], "schema": { - "type": "issuetype", + "type": "option", "system": "issuetype" } }, @@ -36,5 +36,33 @@ "navigable": true, "searchable": false, "clauseNames": ["parent"] + }, + { + "id": "issuetype2", + "key": "issuetype2", + "name": "Issue Type2", + "custom": true, + "orderable": true, + "navigable": true, + "searchable": true, + "clauseNames": ["issuetype", "type"], + "schema": { + "type": "option", + "system": "issuetype" + } + }, + { + "id": "issuetype3", + "key": "issuetype3", + "name": "Issue Type3", + "custom": true, + "orderable": true, + "navigable": true, + "searchable": true, + "clauseNames": ["issuetype", "type"], + "schema": { + "type": "option", + "system": "issuetype" + } } ] diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issues.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issues.json index 612f3b35a262..3ece380f3da0 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/issues.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/issues.json @@ -6,6 +6,8 @@ "self": "https://airbyteio.atlassian.net/rest/api/3/issue/10627", "key": "TESTKEY13-1", "fields": { + "empty_field": null, + "non_empty_field": "", "statuscategorychangedate": "2022-06-09T16:29:32.382-0700", "issuetype": { "self": "https://airbyteio.atlassian.net/rest/api/3/issuetype/10000", diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/responses/projects.json b/airbyte-integrations/connectors/source-jira/unit_tests/responses/projects.json index a463e6c679dd..b82385672f43 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/responses/projects.json +++ b/airbyte-integrations/connectors/source-jira/unit_tests/responses/projects.json @@ -23,7 +23,7 @@ "expand": "description,lead,issueTypes,url,projectKeys,permissions,insight", "self": "https://airbyteio.atlassian.net/rest/api/3/project/10016", "id": "2", - "key": "Project1", + "key": "Project2", "description": "Test project 13 description", "name": "Test project 13", "avatarUrls": { diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_date_time_transformer.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_date_time_transformer.py new file mode 100644 index 000000000000..263443b0c2f7 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_date_time_transformer.py @@ -0,0 +1,25 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import pytest +from source_jira.source import SourceJira +from source_jira.streams import ApplicationRoles + + +@pytest.mark.parametrize( + "origin_item,subschema,expected", + [ + ("2023-05-08T03:04:45.139-0700", {"type": "string", "format": "date-time"}, "2023-05-08T03:04:45.139000-07:00"), + ("2022-10-31T09:00:00.594Z", {"type": "string", "format": "date-time"}, "2022-10-31T09:00:00.594000+00:00"), + ("2023-09-11t17:51:41.666-0700", {"type": "string", "format": "date-time"}, "2023-09-11T17:51:41.666000-07:00"), + ("some string", {"type": "string"}, "some string"), + (1234, {"type": "integer"}, 1234), + ], +) +def test_converting_date_to_date_time(origin_item, subschema, expected, config): + authenticator = SourceJira().get_authenticator(config=config) + args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} + stream = ApplicationRoles(**args) + actual = stream.transformer.default_convert(origin_item, subschema) + assert actual == expected diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_migrations/test_config.json b/airbyte-integrations/connectors/source-jira/unit_tests/test_migrations/test_config.json new file mode 100644 index 000000000000..8745a889bf96 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_migrations/test_config.json @@ -0,0 +1,10 @@ +{ + "api_token": "invalid_token", + "domain": "invaliddomain.atlassian.net", + "email": "no-reply@domain.com", + "start_date": "2023-01-01T00:00:00Z", + "projects": ["IT1", "IT1", "IT1"], + "expand_issue_changelog": true, + "render_fields": true, + "expand_issue_transition": false +} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_migrations/test_config_migrations.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_migrations/test_config_migrations.py new file mode 100644 index 000000000000..2b124e0e6323 --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_migrations/test_config_migrations.py @@ -0,0 +1,68 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import json +import os +from typing import Any, Mapping + +from airbyte_cdk.models import OrchestratorType, Type +from airbyte_cdk.sources import Source +from source_jira.config_migrations import MigrateIssueExpandProperties +from source_jira.source import SourceJira + +# BASE ARGS +CMD = "check" +TEST_CONFIG_PATH = f"{os.path.dirname(__file__)}/test_config.json" +SOURCE_INPUT_ARGS = [CMD, "--config", TEST_CONFIG_PATH] +SOURCE: Source = SourceJira() + + +# HELPERS +def load_config(config_path: str = TEST_CONFIG_PATH) -> Mapping[str, Any]: + with open(config_path, "r") as config: + return json.load(config) + + +def revert_migration(config_path: str = TEST_CONFIG_PATH) -> None: + with open(config_path, "r") as test_config: + config = json.load(test_config) + config.pop("issues_stream_expand_with") + with open(config_path, "w") as updated_config: + config = json.dumps(config) + updated_config.write(config) + + +def test_migrate_config(): + migration_instance = MigrateIssueExpandProperties() + # migrate the test_config + migration_instance.migrate(SOURCE_INPUT_ARGS, SOURCE) + # load the updated config + test_migrated_config = load_config() + # check migrated property + assert "issues_stream_expand_with" in test_migrated_config + assert isinstance(test_migrated_config["issues_stream_expand_with"], list) + # check the old property is in place + assert all(key in test_migrated_config for key in migration_instance.migrate_from_keys_map) + assert all(isinstance(test_migrated_config[key], bool) for key in migration_instance.migrate_from_keys_map) + # check the migration should be skipped, once already done + assert not migration_instance.should_migrate(test_migrated_config) + # test CONTROL MESSAGE was emitted + control_msg = migration_instance.message_repository._message_queue[0] + assert control_msg.type == Type.CONTROL + assert control_msg.control.type == OrchestratorType.CONNECTOR_CONFIG + # check the migrated values + assert control_msg.control.connectorConfig.config["issues_stream_expand_with"] == ["changelog", "renderedFields"] + # revert the test_config to the starting point + revert_migration() + + +def test_config_is_reverted(): + # check the test_config state, it has to be the same as before tests + test_config = load_config() + # check the config no longer has the migrated property + assert "issues_stream_expand_with" not in test_config + # check the old property is still there + assert all(key in test_config for key in MigrateIssueExpandProperties.migrate_from_keys_map) + assert all(isinstance(test_config[key], bool) for key in MigrateIssueExpandProperties.migrate_from_keys_map) diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_pagination.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_pagination.py index f50c96a57cf6..bd9d2d65dd07 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/test_pagination.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_pagination.py @@ -36,9 +36,42 @@ def test_pagination_projects(): def test_pagination_issues(): domain = "domain.com" responses_json = [ - (HTTPStatus.OK, {}, json.dumps({"startAt": 0, "maxResults": 2, "total": 6, "issues": [{"id": "1", "updated": "2022-01-01"}, {"id": "2", "updated": "2022-01-01"}]})), - (HTTPStatus.OK, {}, json.dumps({"startAt": 2, "maxResults": 2, "total": 6, "issues": [{"id": "3", "updated": "2022-01-01"}, {"id": "4", "updated": "2022-01-01"}]})), - (HTTPStatus.OK, {}, json.dumps({"startAt": 4, "maxResults": 2, "total": 6, "issues": [{"id": "5", "updated": "2022-01-01"}, {"id": "6", "updated": "2022-01-01"}]})), + ( + HTTPStatus.OK, + {}, + json.dumps( + { + "startAt": 0, + "maxResults": 2, + "total": 6, + "issues": [{"id": "1", "updated": "2022-01-01"}, {"id": "2", "updated": "2022-01-01"}], + } + ), + ), + ( + HTTPStatus.OK, + {}, + json.dumps( + { + "startAt": 2, + "maxResults": 2, + "total": 6, + "issues": [{"id": "3", "updated": "2022-01-01"}, {"id": "4", "updated": "2022-01-01"}], + } + ), + ), + ( + HTTPStatus.OK, + {}, + json.dumps( + { + "startAt": 4, + "maxResults": 2, + "total": 6, + "issues": [{"id": "5", "updated": "2022-01-01"}, {"id": "6", "updated": "2022-01-01"}], + } + ), + ), ] responses.add_callback( @@ -57,7 +90,7 @@ def test_pagination_issues(): {"id": "3", "updated": "2022-01-01"}, {"id": "4", "updated": "2022-01-01"}, {"id": "5", "updated": "2022-01-01"}, - {"id": "6", "updated": "2022-01-01"} + {"id": "6", "updated": "2022-01-01"}, ] diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_source.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_source.py index cb9ba9330782..4cec82b00478 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_source.py @@ -4,7 +4,9 @@ from unittest.mock import MagicMock +import pytest import responses +from airbyte_cdk.utils.traced_exception import AirbyteTracedException from source_jira.source import SourceJira @@ -12,30 +14,53 @@ def test_streams(config): source = SourceJira() streams = source.streams(config) - expected_streams_number = 51 + expected_streams_number = 55 assert len(streams) == expected_streams_number @responses.activate -def test_check_connection(config, projects_response, labels_response): +def test_check_connection_config_no_access_to_one_stream(config, caplog, projects_response, avatars_response): responses.add( responses.GET, - f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description%2Clead", + f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description%2Clead&status=live&status=archived&status=deleted", json=projects_response, ) responses.add( responses.GET, - f"https://{config['domain']}/rest/api/3/label?maxResults=50", - json=labels_response, + f"https://{config['domain']}/rest/api/3/applicationrole?maxResults=50", + status=401, ) + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/avatar/issuetype/system?maxResults=50", + json=avatars_response, + ) + responses.add(responses.GET, f"https://{config['domain']}/rest/api/3/label?maxResults=50", status=401) source = SourceJira() logger_mock = MagicMock() - assert source.check_connection(logger=logger_mock, config=config) == (True, None) +@responses.activate +def test_check_connection_404_error(config): + responses.add( + responses.GET, + f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description%2Clead&status=live&status=archived&status=deleted", + status=404, + ) + responses.add(responses.GET, f"https://{config['domain']}/rest/api/3/label?maxResults=50", status=404) + source = SourceJira() + logger_mock = MagicMock() + with pytest.raises(AirbyteTracedException) as e: + source.check_connection(logger=logger_mock, config=config) + + assert ( + e.value.message == "Config validation error: please check that your domain is valid and does not include protocol (e.g: https://)." + ) + + def test_get_authenticator(config): source = SourceJira() authenticator = source.get_authenticator(config=config) - assert authenticator.get_auth_header() == {'Authorization': 'Basic ZW1haWxAZW1haWwuY29tOnRva2Vu'} + assert authenticator.get_auth_header() == {"Authorization": "Basic ZW1haWxAZW1haWwuY29tOnRva2Vu"} diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py index 654b49b72715..00675fa25ab1 100644 --- a/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-jira/unit_tests/test_streams.py @@ -2,6 +2,9 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging + +import pendulum import pytest import requests import responses @@ -20,6 +23,7 @@ Groups, IssueComments, IssueCustomFieldContexts, + IssueCustomFieldOptions, IssueFieldConfigurations, IssueFields, IssueLinkTypes, @@ -60,6 +64,24 @@ from source_jira.utils import read_full_refresh +@responses.activate +def test_application_roles_stream_401_error(config, caplog): + config["domain"] = "test_application_domain" + responses.add(responses.GET, f"https://{config['domain']}/rest/api/3/applicationrole?maxResults=50", status=401) + + authenticator = SourceJira().get_authenticator(config=config) + args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} + stream = ApplicationRoles(**args) + + is_available, reason = stream.check_availability(logger=logging.Logger, source=SourceJira()) + + assert is_available is False + + assert reason == ( + "Unable to read application_roles stream. The endpoint https://test_application_domain/rest/api/3/applicationrole?maxResults=50 returned 401: Unauthorized. Invalid creds were provided, please check your api token, domain and/or email.. Please visit https://docs.airbyte.com/integrations/sources/jira to learn more. " + ) + + @responses.activate def test_application_roles_stream(config, application_roles_response): responses.add( @@ -80,9 +102,7 @@ def test_application_roles_stream(config, application_roles_response): @responses.activate def test_application_roles_stream_http_error(config, application_roles_response): responses.add( - responses.GET, - f"https://{config['domain']}/rest/api/3/applicationrole?maxResults=50", - json={'error': 'not found'}, status=404 + responses.GET, f"https://{config['domain']}/rest/api/3/applicationrole?maxResults=50", json={"error": "not found"}, status=404 ) authenticator = SourceJira().get_authenticator(config=config) @@ -93,20 +113,41 @@ def test_application_roles_stream_http_error(config, application_roles_response) @responses.activate -def test_boards_stream(config, boards_response): +def test_boards_stream(config, mock_board_response): + authenticator = SourceJira().get_authenticator(config=config) + args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} + stream = Boards(**args) + + records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh)] + assert len(records) == 3 + assert len(responses.calls) == 1 + + +@responses.activate +def test_board_stream_forbidden(config, boards_response, caplog): + config["domain"] = "test_boards_domain" responses.add( responses.GET, f"https://{config['domain']}/rest/agile/1.0/board?maxResults=50", - json=boards_response, + json={"error": f"403 Client Error: Forbidden for url: https://{config['domain']}/rest/agile/1.0/board?maxResults=50"}, + status=403, ) - authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = Boards(**args) + is_available, reason = stream.check_availability(logger=logging.Logger, source=SourceJira()) - records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh)] - assert len(records) == 3 - assert len(responses.calls) == 1 + assert is_available is False + + assert reason == ( + "Unable to read boards stream. The endpoint " + "https://test_boards_domain/rest/agile/1.0/board?maxResults=50 returned 403: " + "Forbidden. Please check the 'READ' permission(Scopes for Connect apps) " + "and/or the user has Jira Software rights and access.. Please visit " + "https://docs.airbyte.com/integrations/sources/jira to learn more. " + "403 Client Error: Forbidden for url: " + "https://test_boards_domain/rest/agile/1.0/board?maxResults=50" + ) @responses.activate @@ -127,13 +168,7 @@ def test_dashboards_stream(config, dashboards_response): @responses.activate -def test_filters_stream(config, filters_response): - responses.add( - responses.GET, - f"https://{config['domain']}/rest/api/3/filter/search?maxResults=50&expand=description%2Cowner%2Cjql%2CviewUrl%2CsearchUrl%2Cfavourite%2CfavouritedCount%2CsharePermissions%2CisWritable%2Csubscriptions", - json=filters_response, - ) - +def test_filters_stream(config, mock_filter_response): authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = Filters(**args) @@ -161,19 +196,13 @@ def test_groups_stream(config, groups_response): @responses.activate -def test_issues_fields_stream(config, issue_fields_response): - responses.add( - responses.GET, - f"https://{config['domain']}/rest/api/3/field?maxResults=50", - json=issue_fields_response, - ) - +def test_issues_fields_stream(config, mock_fields_response): authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = IssueFields(**args) records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh)] - assert len(records) == 3 + assert len(records) == 5 assert len(responses.calls) == 1 @@ -331,7 +360,7 @@ def test_jira_settings_stream(config, jira_settings_response): @responses.activate -def test_board_issues_stream(config, board_issues_response): +def test_board_issues_stream(config, mock_board_response, board_issues_response): responses.add( responses.GET, f"https://{config['domain']}/rest/agile/1.0/board/1/issue?maxResults=50&fields=key&fields=created&fields=updated", @@ -340,7 +369,8 @@ def test_board_issues_stream(config, board_issues_response): responses.add( responses.GET, f"https://{config['domain']}/rest/agile/1.0/board/2/issue?maxResults=50&fields=key&fields=created&fields=updated", - json={}, + json={"errorMessages": ["This board has no columns with a mapped status."], "errors": {}}, + status=500, ) responses.add( responses.GET, @@ -351,13 +381,24 @@ def test_board_issues_stream(config, board_issues_response): authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = BoardIssues(**args) - records = [r for r in stream.read_records(sync_mode=SyncMode.incremental)] + records = list(read_full_refresh(stream)) assert len(records) == 1 - assert len(responses.calls) == 3 + assert len(responses.calls) == 4 + + +def test_stream_updated_state(config): + authenticator = SourceJira().get_authenticator(config=config) + args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} + stream = BoardIssues(**args) + + current_stream_state = {"22": {"updated": "2023-10-01T00:00:00Z"}} + latest_record = {"boardId": 22, "updated": "2023-09-01T00:00:00Z"} + + assert {"22": {"updated": "2023-10-01T00:00:00Z"}} == stream.get_updated_state(current_stream_state=current_stream_state, latest_record=latest_record) @responses.activate -def test_filter_sharing_stream(config, filter_sharing_response): +def test_filter_sharing_stream(config, mock_filter_response, filter_sharing_response): responses.add( responses.GET, f"https://{config['domain']}/rest/api/3/filter/1/permission?maxResults=50", @@ -369,26 +410,20 @@ def test_filter_sharing_stream(config, filter_sharing_response): stream = FilterSharing(**args) records = [r for r in stream.read_records(sync_mode=SyncMode.incremental)] assert len(records) == 1 - assert len(responses.calls) == 1 + assert len(responses.calls) == 2 @responses.activate -def test_projects_stream(config, projects_response): - responses.add( - responses.GET, - f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description%2Clead", - json=projects_response, - ) - +def test_projects_stream(config, mock_projects_responses): authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = Projects(**args) records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh)] - assert len(records) == 2 + assert len(records) == 1 @responses.activate -def test_projects_avatars_stream(config, projects_avatars_response): +def test_projects_avatars_stream(config, mock_projects_responses, projects_avatars_response): responses.add( responses.GET, f"https://{config['domain']}/rest/api/3/project/Project1/avatars?maxResults=50", @@ -399,7 +434,7 @@ def test_projects_avatars_stream(config, projects_avatars_response): args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = ProjectAvatars(**args) records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh)] - assert len(records) == 4 + assert len(records) == 2 assert len(responses.calls) == 2 @@ -420,13 +455,7 @@ def test_projects_categories_stream(config, projects_categories_response): @responses.activate -def test_screens_stream(config, screens_response): - responses.add( - responses.GET, - f"https://{config['domain']}/rest/api/3/screens?maxResults=50", - json=screens_response, - ) - +def test_screens_stream(config, mock_screen_response): authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = Screens(**args) @@ -436,7 +465,7 @@ def test_screens_stream(config, screens_response): @responses.activate -def test_screen_tabs_stream(config, screen_tabs_response): +def test_screen_tabs_stream(config, mock_screen_response, screen_tabs_response): responses.add( responses.GET, f"https://{config['domain']}/rest/api/3/screens/1/tabs?maxResults=50", @@ -453,50 +482,49 @@ def test_screen_tabs_stream(config, screen_tabs_response): stream = ScreenTabs(**args) records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh)] assert len(records) == 3 - assert len(responses.calls) == 2 + assert len(responses.calls) == 3 @responses.activate -def test_sprints_stream(config, sprints_response): +def test_sprints_stream(config, mock_board_response, mock_sprints_response): + authenticator = SourceJira().get_authenticator(config=config) + args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} + stream = Sprints(**args) + records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh)] + assert len(records) == 3 + assert len(responses.calls) == 4 + + +@responses.activate +def test_board_does_not_support_sprints(config, mock_board_response, sprints_response, caplog): responses.add( responses.GET, f"https://{config['domain']}/rest/agile/1.0/board/1/sprint?maxResults=50", json=sprints_response, ) - responses.add( - responses.GET, - f"https://{config['domain']}/rest/agile/1.0/board/2/sprint?maxResults=50", - json=sprints_response, - ) responses.add( responses.GET, f"https://{config['domain']}/rest/agile/1.0/board/3/sprint?maxResults=50", json=sprints_response, ) - + url = f"https://{config['domain']}/rest/agile/1.0/board/2/sprint?maxResults=50" + error = {"errorMessages": ["The board does not support sprints"], "errors": {}} + responses.add(responses.GET, url, json=error, status=400) authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = Sprints(**args) records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh)] - assert len(records) == 3 - assert len(responses.calls) == 3 - + assert len(records) == 2 -@responses.activate -def test_board_does_not_support_sprints(config): - url = f"https://{config['domain']}/rest/agile/1.0/board/4/sprint?maxResults=50" - error = {'errorMessages': ['The board does not support sprints'], 'errors': {}} - responses.add(responses.GET, url, json=error, status=400) - authenticator = SourceJira().get_authenticator(config=config) - args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} - stream = Sprints(**args) - response = requests.get(url) - actual = stream.should_retry(response) - assert actual is False + assert ( + "The board does not support sprints. The board does not have a sprint board. if it's a team-managed one, " + "does it have sprints enabled under project settings? If it's a company-managed one," + " check that it has at least one Scrum board associated with it." + ) in caplog.text @responses.activate -def test_sprint_issues_stream(config, sprints_issues_response): +def test_sprint_issues_stream(config, mock_board_response, mock_fields_response, mock_sprints_response, sprints_issues_response): responses.add( responses.GET, f"https://{config['domain']}/rest/agile/1.0/sprint/2/issue?maxResults=50&fields=key&fields=status&fields=created&fields=updated", @@ -508,7 +536,7 @@ def test_sprint_issues_stream(config, sprints_issues_response): stream = SprintIssues(**args) records = [r for r in stream.read_records(sync_mode=SyncMode.incremental)] assert len(records) == 3 - assert len(responses.calls) == 3 + assert len(responses.calls) == 8 @responses.activate @@ -528,13 +556,7 @@ def test_time_tracking_stream(config, time_tracking_response): @responses.activate -def test_users_stream(config, users_response): - responses.add( - responses.GET, - f"https://{config['domain']}/rest/api/3/users/search?maxResults=50", - json=users_response, - ) - +def test_users_stream(config, mock_users_response): authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = Users(**args) @@ -544,7 +566,7 @@ def test_users_stream(config, users_response): @responses.activate -def test_users_groups_detailed_stream(config, users_groups_detailed_response): +def test_users_groups_detailed_stream(config, mock_users_response, users_groups_detailed_response): responses.add( responses.GET, f"https://{config['domain']}/rest/api/3/user?maxResults=50&accountId=1&expand=groups%2CapplicationRoles", @@ -561,7 +583,7 @@ def test_users_groups_detailed_stream(config, users_groups_detailed_response): stream = UsersGroupsDetailed(**args) records = [r for r in stream.read_records(sync_mode=SyncMode.incremental)] assert len(records) == 4 - assert len(responses.calls) == 2 + assert len(responses.calls) == 3 @responses.activate @@ -639,37 +661,57 @@ def test_avatars_stream(config, avatars_response): authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = Avatars(**args) - records = [r for r in - stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"avatar_type": "issuetype"})] + records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"avatar_type": "issuetype"})] assert len(records) == 2 assert len(responses.calls) == 1 @responses.activate -def test_issues_stream(config, projects_response, mock_issues_responses, issues_response, caplog): - Projects.use_cache = False - projects_response['values'].append({"id": "3", "key": "Project1"}) - responses.add( - responses.GET, - f"https://{config['domain']}/rest/api/3/project/search?maxResults=50&expand=description%2Clead", - json=projects_response, - ) - responses.add( - responses.GET, - f"https://{config['domain']}/rest/api/3/search", - match=[matchers.query_param_matcher({"maxResults": 50, "fields": '*all', "jql": "project in (3)"})], - json={"errorMessages": ["The value '3' does not exist for the field 'project'."]}, - status=400 - ) +def test_avatars_stream_should_retry(config, caplog): + url = f"https://{config['domain']}/rest/api/3/avatar/issuetype/system?maxResults=50" + responses.add(method=responses.GET, url=url, json={"errorMessages": ["The error message"], "errors": {}}, status=400) + authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} + stream = Avatars(**args) + records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"avatar_type": "issuetype"})] + assert len(records) == 0 + + assert "The error message" in caplog.text + + +@responses.activate +def test_issues_stream(config, mock_projects_responses_additional_project, mock_issues_responses, caplog): + authenticator = SourceJira().get_authenticator(config=config) + args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", []) + ["Project3"]} stream = Issues(**args) records = list(read_full_refresh(stream)) assert len(records) == 1 - assert len(responses.calls) == 4 - error_message = "Stream `issues`. An error occurred, details: [\"The value '3' does not exist for the field 'project'.\"].Check permissions for this project. Skipping for now." + + # check if only None values was filtered out from 'fields' field + assert "empty_field" not in records[0]["fields"] + assert "non_empty_field" in records[0]["fields"] + + assert len(responses.calls) == 3 + error_message = "Stream `issues`. An error occurred, details: [\"The value '3' does not exist for the field 'project'.\"]. Skipping for now. The user doesn't have permission to the project. Please grant the user to the project." assert error_message in caplog.messages +@pytest.mark.parametrize( + "start_date, lookback_window, stream_state, expected_query", + [ + (pendulum.parse("2023-09-09T00:00:00Z"), 0, None, None), + (None, 10, {"updated": "2023-12-14T09:47:00"}, "updated >= '2023/12/14 09:37'"), + (None, 0, {"updated": "2023-12-14T09:47:00"}, "updated >= '2023/12/14 09:47'") + ] +) +def test_issues_stream_jql_compare_date(config, start_date, lookback_window, stream_state, expected_query, caplog): + authenticator = SourceJira().get_authenticator(config=config) + args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", []) + ["Project3"], + "lookback_window_minutes": pendulum.duration(minutes=lookback_window)} + stream = Issues(**args) + assert stream.jql_compare_date(stream_state) == expected_query + + @responses.activate def test_issue_comments_stream(config, mock_projects_responses, mock_issues_responses, issue_comments_response): @@ -682,42 +724,56 @@ def test_issue_comments_stream(config, mock_projects_responses, mock_issues_resp authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = IssueComments(**args) - records = [r for r in - stream.read_records(sync_mode=SyncMode.full_refresh)] + records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh)] + assert len(records) == 2 + assert len(responses.calls) == 3 + + +@responses.activate +def test_issue_custom_field_contexts_stream(config, mock_fields_response, mock_issue_custom_field_contexts_response): + authenticator = SourceJira().get_authenticator(config=config) + args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} + stream = IssueCustomFieldContexts(**args) + records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh)] assert len(records) == 2 assert len(responses.calls) == 4 @responses.activate -def test_issue_custom_field_contexts_stream(config, issue_custom_field_contexts_response): +def test_issue_property_keys_stream(config, issue_property_keys_response): responses.add( responses.GET, - f"https://{config['domain']}/rest/api/3/field/issuetype/context?maxResults=50", - json=issue_custom_field_contexts_response, + f"https://{config['domain']}/rest/api/3/issue/TESTKEY13-1/properties?maxResults=50", + json=issue_property_keys_response, ) authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} - stream = IssueCustomFieldContexts(**args) - records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"field_id": "10130"})] + stream = IssuePropertyKeys(**args) + records = [ + r for r in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"issue_key": "TESTKEY13-1", "key": "TESTKEY13-1"}) + ] assert len(records) == 2 assert len(responses.calls) == 1 @responses.activate -def test_issue_property_keys_stream(config, issue_property_keys_response): +def test_issue_property_keys_stream_not_found_skip(config, issue_property_keys_response): + config["domain"] = "test_skip_properties" responses.add( responses.GET, f"https://{config['domain']}/rest/api/3/issue/TESTKEY13-1/properties?maxResults=50", - json=issue_property_keys_response, + json={"errorMessages": ["Issue does not exist or you do not have permission to see it."], "errors": {}}, + status=404, ) authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = IssuePropertyKeys(**args) - records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh, - stream_slice={"issue_key": "TESTKEY13-1", "key": "TESTKEY13-1"})] - assert len(records) == 2 + records = [ + r for r in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"issue_key": "TESTKEY13-1", "key": "TESTKEY13-1"}) + ] + assert len(records) == 0 assert len(responses.calls) == 1 @@ -732,31 +788,35 @@ def test_project_permissions_stream(config, mock_projects_responses, project_per authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = ProjectPermissionSchemes(**args) - records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh, - stream_slice={"key": "TESTKEY13-1"})] - assert len(records) == 4 + records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"key": "Project1"})] + expected_records = [ + { + "description": "Only the reporter and internal staff can see this issue.", + "id": "100000", + "name": "Reporter Only", + "projectId": "Project1", + "self": "https://your-domain.atlassian.net/rest/api/3/securitylevel/100000", + }, + { + "description": "Only internal staff can see this issue.", + "id": "100001", + "name": "Staff Only", + "projectId": "Project1", + "self": "https://your-domain.atlassian.net/rest/api/3/securitylevel/100001", + }, + ] + assert len(records) == 2 + assert records == expected_records @responses.activate -def test_project_email_stream(config, mock_projects_responses, project_email_response): - responses.add( - responses.GET, - f"https://{config['domain']}/rest/api/3/project/1/email?maxResults=50", - json=project_email_response, - ) - responses.add( - responses.GET, - f"https://{config['domain']}/rest/api/3/project/2/email?maxResults=50", - json=project_email_response, - ) - +def test_project_email_stream(config, mock_projects_responses, mock_project_emails): authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = ProjectEmail(**args) - records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh, - stream_slice={"key": "TESTKEY13-1"})] - assert len(records) == 4 - assert len(responses.calls) == 3 + records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh)] + assert len(records) == 2 + assert len(responses.calls) == 2 @responses.activate @@ -770,10 +830,9 @@ def test_project_components_stream(config, mock_projects_responses, project_comp authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = ProjectComponents(**args) - records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh, - stream_slice={"key": "Project1"})] - assert len(records) == 4 - assert len(responses.calls) == 3 + records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"key": "Project1"})] + assert len(records) == 2 + assert len(responses.calls) == 2 @responses.activate @@ -826,23 +885,17 @@ def test_issue_worklogs_stream(config, mock_projects_responses, mock_issues_resp stream = IssueWorklogs(**args) records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh)] assert len(records) == 1 - assert len(responses.calls) == 4 + assert len(responses.calls) == 3 @responses.activate -def test_issue_watchers_stream(config, mock_projects_responses, mock_issues_responses, issue_watchers_response): - responses.add( - responses.GET, - f"https://{config['domain']}/rest/api/3/issue/TESTKEY13-1/watchers?maxResults=50", - json=issue_watchers_response, - ) - +def test_issue_watchers_stream(config, mock_projects_responses, mock_issues_responses, mock_issue_watchers_responses): authenticator = SourceJira().get_authenticator(config=config) args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", [])} stream = IssueWatchers(**args) records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh)] assert len(records) == 1 - assert len(responses.calls) == 4 + assert len(responses.calls) == 3 @responses.activate @@ -859,7 +912,7 @@ def test_issue_votes_stream(config, mock_projects_responses, mock_issues_respons records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"key": "Project1"})] assert len(records) == 1 - assert len(responses.calls) == 4 + assert len(responses.calls) == 3 @responses.activate @@ -876,7 +929,7 @@ def test_issue_remote_links_stream(config, mock_projects_responses, mock_issues_ records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"key": "Project1"})] assert len(records) == 2 - assert len(responses.calls) == 4 + assert len(responses.calls) == 3 @responses.activate @@ -892,5 +945,68 @@ def test_project_versions_stream(config, mock_projects_responses, projects_versi stream = ProjectVersions(**args) records = [r for r in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice={"key": "Project1"})] - assert len(records) == 4 - assert len(responses.calls) == 3 + assert len(records) == 2 + assert len(responses.calls) == 2 + + +@pytest.mark.parametrize( + "stream, expected_records_number, expected_calls_number, log_message", + [ + ( + Issues, + 2, + 4, + "Stream `issues`. An error occurred, details: [\"The value '3' does not " + "exist for the field 'project'.\"]. Skipping for now. The user doesn't have " + "permission to the project. Please grant the user to the project.", + ), + ( + IssueCustomFieldContexts, + 2, + 4, + "Stream `issue_custom_field_contexts`. An error occurred, details: ['Not found issue custom field context for issue fields issuetype2']. Skipping for now. ", + ), + ( + IssueCustomFieldOptions, + 1, + 6, + "Stream `issue_custom_field_options`. An error occurred, details: ['Not found issue custom field options for issue fields issuetype3']. Skipping for now. ", + ), + ( + IssueWatchers, + 1, + 6, + "Stream `issue_watchers`. An error occurred, details: ['Not found watchers for issue TESTKEY13-2']. Skipping for now. ", + ), + ( + ProjectEmail, + 4, + 4, + "Stream `project_email`. An error occurred, details: ['No access to emails for project 3']. Skipping for now. ", + ), + ], +) +@responses.activate +def test_skip_slice( + config, + mock_projects_responses_additional_project, + mock_issues_responses, + mock_project_emails, + mock_issue_watchers_responses, + mock_issue_custom_field_contexts_response_error, + mock_issue_custom_field_options_response, + mock_fields_response, + caplog, + stream, + expected_records_number, + expected_calls_number, + log_message, +): + authenticator = SourceJira().get_authenticator(config=config) + args = {"authenticator": authenticator, "domain": config["domain"], "projects": config.get("projects", []) + ["Project3", "Project4"]} + stream = stream(**args) + records = list(read_full_refresh(stream)) + assert len(records) == expected_records_number + + assert len(responses.calls) == expected_calls_number + assert log_message in caplog.messages diff --git a/airbyte-integrations/connectors/source-k6-cloud/README.md b/airbyte-integrations/connectors/source-k6-cloud/README.md index 33afbeff8310..b60caf6610b5 100644 --- a/airbyte-integrations/connectors/source-k6-cloud/README.md +++ b/airbyte-integrations/connectors/source-k6-cloud/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-k6-cloud:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/k6-cloud) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_k6_cloud/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-k6-cloud:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-k6-cloud build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-k6-cloud:airbyteDocker +An image will be built with the tag `airbyte/source-k6-cloud:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-k6-cloud:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-k6-cloud:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-k6-cloud:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-k6-cloud:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-k6-cloud test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-k6-cloud:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-k6-cloud:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-k6-cloud test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/k6-cloud.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-k6-cloud/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-k6-cloud/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-k6-cloud/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-k6-cloud/build.gradle b/airbyte-integrations/connectors/source-k6-cloud/build.gradle deleted file mode 100644 index a18607214137..000000000000 --- a/airbyte-integrations/connectors/source-k6-cloud/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_k6_cloud' -} diff --git a/airbyte-integrations/connectors/source-kafka/Dockerfile b/airbyte-integrations/connectors/source-kafka/Dockerfile deleted file mode 100644 index 7ca85493e476..000000000000 --- a/airbyte-integrations/connectors/source-kafka/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-kafka - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-kafka - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.2.3 -LABEL io.airbyte.name=airbyte/source-kafka diff --git a/airbyte-integrations/connectors/source-kafka/README.md b/airbyte-integrations/connectors/source-kafka/README.md index d658d739c97a..342b3758e161 100644 --- a/airbyte-integrations/connectors/source-kafka/README.md +++ b/airbyte-integrations/connectors/source-kafka/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:source-kafka:airbyteDocker +./gradlew :airbyte-integrations:connectors:source-kafka:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/source-kafka:dev`. the Dockerfile. #### Run @@ -55,8 +56,11 @@ All commands should be run from airbyte project root. To run acceptance and cust ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. \ No newline at end of file +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-kafka test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/kafka.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-kafka/acceptance-test-config.yml b/airbyte-integrations/connectors/source-kafka/acceptance-test-config.yml index 61502139adf1..d6958e091324 100644 --- a/airbyte-integrations/connectors/source-kafka/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-kafka/acceptance-test-config.yml @@ -4,4 +4,4 @@ connector_image: airbyte/source-kafka:dev tests: spec: - spec_path: "src/test-integration/resources/expected_spec.json" - config_path: "src/test-integration/resources/dummy_config.json" \ No newline at end of file + config_path: "src/test-integration/resources/dummy_config.json" diff --git a/airbyte-integrations/connectors/source-kafka/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-kafka/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-kafka/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-kafka/build.gradle b/airbyte-integrations/connectors/source-kafka/build.gradle index 7edb1fce90a0..4a3137bec286 100644 --- a/airbyte-integrations/connectors/source-kafka/build.gradle +++ b/airbyte-integrations/connectors/source-kafka/build.gradle @@ -1,37 +1,35 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' - id 'airbyte-connector-acceptance-test' + id 'airbyte-java-connector' } -application { - mainClass = 'io.airbyte.integrations.source.kafka.KafkaSource' - applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] +airbyteJavaConnector { + cdkVersionRequired = '0.2.0' + features = ['db-sources'] + useLocalCdk = false } -repositories { - mavenLocal() - mavenCentral() - maven { - url "https://packages.confluent.io/maven" +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") } +} +airbyteJavaConnector.addCdkDependencies() + +application { + mainClass = 'io.airbyte.integrations.source.kafka.KafkaSource' + applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.connectors.testcontainers.kafka implementation 'org.apache.kafka:kafka-clients:3.2.1' implementation 'org.apache.kafka:connect-json:3.2.1' implementation 'io.confluent:kafka-avro-serializer:7.2.1' - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-kafka') - integrationTestJavaImplementation libs.connectors.testcontainers.kafka + testImplementation libs.testcontainers.kafka - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + integrationTestJavaImplementation libs.testcontainers.kafka } diff --git a/airbyte-integrations/connectors/source-kafka/src/main/java/io/airbyte/integrations/source/kafka/KafkaSource.java b/airbyte-integrations/connectors/source-kafka/src/main/java/io/airbyte/integrations/source/kafka/KafkaSource.java index e7c452073e91..77a909fcfca8 100644 --- a/airbyte-integrations/connectors/source-kafka/src/main/java/io/airbyte/integrations/source/kafka/KafkaSource.java +++ b/airbyte-integrations/connectors/source-kafka/src/main/java/io/airbyte/integrations/source/kafka/KafkaSource.java @@ -5,10 +5,10 @@ package io.airbyte.integrations.source.kafka; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; import io.airbyte.commons.util.AutoCloseableIterator; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; import io.airbyte.integrations.source.kafka.format.KafkaFormat; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; diff --git a/airbyte-integrations/connectors/source-kafka/src/test-integration/java/io/airbyte/integrations/source/kafka/KafkaSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-kafka/src/test-integration/java/io/airbyte/integrations/source/kafka/KafkaSourceAcceptanceTest.java index da26c88475ad..2ae90e827d74 100644 --- a/airbyte-integrations/connectors/source-kafka/src/test-integration/java/io/airbyte/integrations/source/kafka/KafkaSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-kafka/src/test-integration/java/io/airbyte/integrations/source/kafka/KafkaSourceAcceptanceTest.java @@ -8,11 +8,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.CatalogHelpers; diff --git a/airbyte-integrations/connectors/source-klarna/Dockerfile b/airbyte-integrations/connectors/source-klarna/Dockerfile index ce2d2878dd50..68bdf79029bc 100644 --- a/airbyte-integrations/connectors/source-klarna/Dockerfile +++ b/airbyte-integrations/connectors/source-klarna/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.13-alpine3.15 as base +FROM python:3.9.11-alpine3.15 as base # build and load all requirements FROM base as builder @@ -34,5 +34,5 @@ COPY source_klarna ./source_klarna ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-klarna diff --git a/airbyte-integrations/connectors/source-klarna/README.md b/airbyte-integrations/connectors/source-klarna/README.md index e8c59a8d0a8f..42fcf61dfc02 100644 --- a/airbyte-integrations/connectors/source-klarna/README.md +++ b/airbyte-integrations/connectors/source-klarna/README.md @@ -1,45 +1,12 @@ # Klarna Source -This is the repository for the Klarna source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/klarna). +This is the repository for the Klarna configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/klarna). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-klarna:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/klarna) +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/klarna) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_klarna/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. @@ -47,28 +14,21 @@ See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source klarna test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-klarna:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-klarna build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-klarna:airbyteDocker +An image will be built with the tag `airbyte/source-klarna:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-klarna:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-klarna:dev check --con docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-klarna:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-klarna:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-klarna test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-klarna:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-klarna:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-klarna test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/klarna.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-klarna/__init__.py b/airbyte-integrations/connectors/source-klarna/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-klarna/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-klarna/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-klarna/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-klarna/build.gradle b/airbyte-integrations/connectors/source-klarna/build.gradle deleted file mode 100644 index e707216c6638..000000000000 --- a/airbyte-integrations/connectors/source-klarna/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_klarna' -} diff --git a/airbyte-integrations/connectors/source-klarna/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-klarna/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-klarna/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-klarna/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-klarna/metadata.yaml b/airbyte-integrations/connectors/source-klarna/metadata.yaml index 6dc2f5b3bdce..8eff9ee1fc7b 100644 --- a/airbyte-integrations/connectors/source-klarna/metadata.yaml +++ b/airbyte-integrations/connectors/source-klarna/metadata.yaml @@ -1,24 +1,31 @@ data: + allowedHosts: + hosts: + - api.klarna.com + - api.playground.klarna.com + - api-${config.region}.klarna.com + - api-${config.region}.playground.klarna.com + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: 60c24725-00ae-490c-991d-55b78c3197e0 - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-klarna githubIssueLabel: source-klarna icon: klarna.svg license: MIT name: Klarna - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: 2022-10-24 releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/klarna tags: - - language:python + - language:low-code ab_internal: sl: 100 ql: 100 - supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-klarna/setup.py b/airbyte-integrations/connectors/source-klarna/setup.py index 046558436d0f..a4742e88dd56 100644 --- a/airbyte-integrations/connectors/source-klarna/setup.py +++ b/airbyte-integrations/connectors/source-klarna/setup.py @@ -5,13 +5,14 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.2", ""] +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", - "pytest~=6.1", + "pytest~=6.2", "pytest-mock~=3.6.1", - "responses~=0.22.0", ] setup( diff --git a/airbyte-integrations/connectors/source-klarna/source_klarna/manifest.yaml b/airbyte-integrations/connectors/source-klarna/source_klarna/manifest.yaml new file mode 100644 index 000000000000..1f6d1db32a83 --- /dev/null +++ b/airbyte-integrations/connectors/source-klarna/source_klarna/manifest.yaml @@ -0,0 +1,117 @@ +version: "0.29.0" + +type: DeclarativeSource +check: + type: CheckStream + stream_names: + - payouts +streams: + - type: DeclarativeStream + name: payouts + primary_key: + - payout_date + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api{{ '-' + config.region if config.region != 'eu' }}.{{ 'playground.' if config.playground }}klarna.com/ + path: /settlements/v1/payouts + http_method: GET + request_parameters: {} + request_headers: {} + authenticator: + type: BasicHttpAuthenticator + username: "{{ config['username'] }}" + password: "{{ config['password'] }}" + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - payouts + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + inject_into: request_parameter + type: RequestOption + field_name: size + pagination_strategy: + type: CursorPagination + page_size: 500 + cursor_value: '{{ response.get("pagination", {}).get("next", {}) }}' + stop_condition: '{{ not response.get("pagination", {}).get("next", {}) }}' + - type: DeclarativeStream + name: transactions + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: >- + https://api{{ '-'+config.region if config.region != 'eu' }}.{{ + 'playground.' if config.playground }}klarna.com/ + path: /settlements/v1/transactions + http_method: GET + request_parameters: {} + request_headers: {} + authenticator: + type: BasicHttpAuthenticator + username: "{{ config['username'] }}" + password: "{{ config['password'] }}" + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - transactions + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + inject_into: request_parameter + type: RequestOption + field_name: size + pagination_strategy: + type: CursorPagination + page_size: 500 + cursor_value: '{{ response.get("pagination", {}).get("next", {}) }}' + stop_condition: '{{ not response.get("pagination", {}).get("next", {}) }}' +spec: + documentation_url: https://docs.airbyte.com/integrations/sources/klarna + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + title: Klarna Spec + type: object + required: + - region + - playground + - username + - password + additionalProperties: true + properties: + region: + title: Region + type: string + enum: + - eu + - us + - oc + description: Base url region (For playground eu https://docs.klarna.com/klarna-payments/api/payments-api/#tag/API-URLs). Supported 'eu', 'us', 'oc' + playground: + title: Playground + type: boolean + description: Propertie defining if connector is used against playground or production environment + default: false + username: + title: Username + type: string + description: Consists of your Merchant ID (eid) - a unique number that identifies your e-store, combined with a random string (https://developers.klarna.com/api/#authentication) + password: + title: Password + type: string + description: A string which is associated with your Merchant ID and is used to authorize use of Klarna's APIs (https://developers.klarna.com/api/#authentication) + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-klarna/source_klarna/schemas/payouts.json b/airbyte-integrations/connectors/source-klarna/source_klarna/schemas/payouts.json index e600cca16b69..ed12942e0f8b 100644 --- a/airbyte-integrations/connectors/source-klarna/source_klarna/schemas/payouts.json +++ b/airbyte-integrations/connectors/source-klarna/source_klarna/schemas/payouts.json @@ -1,17 +1,10 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "additionalProperties": true, - "required": [ - "totals", - "payment_reference", - "payout_date", - "currency_code", - "merchant_settlement_type", - "merchant_id" - ], "properties": { "totals": { - "type": "object", + "type": ["null", "object"], "additionalProperties": true, "properties": { "commission_amount": { @@ -130,6 +123,5 @@ "example": "https://{settlements_api}/transactions?payment_reference=XISA93DJ", "type": "string" } - }, - "$schema": "http://json-schema.org/schema#" + } } diff --git a/airbyte-integrations/connectors/source-klarna/source_klarna/schemas/transactions.json b/airbyte-integrations/connectors/source-klarna/source_klarna/schemas/transactions.json index 2ab0d30627c1..82b7a0faf690 100644 --- a/airbyte-integrations/connectors/source-klarna/source_klarna/schemas/transactions.json +++ b/airbyte-integrations/connectors/source-klarna/source_klarna/schemas/transactions.json @@ -1,7 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "additionalProperties": true, - "required": ["capture_id", "sale_date", "capture_date", "order_id"], "properties": { "amount": { "description": "Total amount of the specific transaction, in minor units", @@ -9,6 +9,20 @@ "type": "integer", "format": "int64" }, + "merchant_id": { + "type": ["null", "string"] + }, + "shipping_address_country": { + "type": ["null", "string"] + }, + "consumer_vat": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} + } + }, "capture_id": { "description": "The Klarna assigned id reference of a specific capture", "example": "33db6f16-9f43-43fa-a587-cc51411c98e4", @@ -186,6 +200,5 @@ "description": "ISO 4217 Currency Code of the country you are registered in.", "example": "EUR" } - }, - "$schema": "http://json-schema.org/schema#" + } } diff --git a/airbyte-integrations/connectors/source-klarna/source_klarna/source.py b/airbyte-integrations/connectors/source-klarna/source_klarna/source.py index dbc2a848d012..af5eb612a3c6 100644 --- a/airbyte-integrations/connectors/source-klarna/source_klarna/source.py +++ b/airbyte-integrations/connectors/source-klarna/source_klarna/source.py @@ -2,117 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple -from urllib.parse import parse_qs, urlparse +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.requests_native_auth import BasicHttpAuthenticator +WARNING: Do not modify this file. +""" -# Basic full refresh stream -class KlarnaStream(HttpStream, ABC): - def __init__(self, region: str, playground: bool, authenticator: BasicHttpAuthenticator, **kwargs): - self.region = region - self.playground = playground - self.kwargs = kwargs - super().__init__(authenticator=authenticator) - - page_size = 500 - data_api_field: str - - @property - def url_base(self) -> str: - playground_path = "playground." if self.playground else "" - if self.region == "eu": - endpoint = f"https://api.{playground_path}klarna.com/" - else: - endpoint = f"https://api-{self.region}.{playground_path}klarna.com/" - return endpoint - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - response_json = response.json() - if "next" in response_json.get("pagination", {}).keys(): - parsed_url = urlparse(response_json["pagination"]["next"]) - query_params = parse_qs(parsed_url.query) - # noinspection PyTypeChecker - return query_params - else: - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - if next_page_token: - return dict(next_page_token) - else: - return {"offset": 0, "size": self.page_size} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - :return an iterable containing each record in the response - """ - payouts = response.json().get(self.data_api_field, []) - yield from payouts - - -class Payouts(KlarnaStream): - """ - Payouts read from Klarna Settlements API https://developers.klarna.com/api/?json#settlements-api - """ - - primary_key = "payout_date" # TODO verify - data_api_field = "payouts" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "/settlements/v1/payouts" - - -class Transactions(KlarnaStream): - """ - Transactions read from Klarna Settlements API https://developers.klarna.com/api/?json#settlements-api - """ - - primary_key = "capture_id" # TODO verify - data_api_field = "transactions" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "/settlements/v1/transactions" - - -# Source -class SourceKlarna(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - """ - :param config: the user-input config object conforming to the connector's spec.yaml - :param logger: logger object - :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. - """ - try: - auth = BasicHttpAuthenticator(username=config["username"], password=config["password"]) - conn_test_stream = Transactions(authenticator=auth, **config) - conn_test_stream.page_size = 1 - conn_test_stream.next_page_token = lambda x: None - records = conn_test_stream.read_records(sync_mode=SyncMode.full_refresh) - # Try to read one value from records iterator - next(records, None) - return True, None - except Exception as e: - print(e) - return False, repr(e) - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - auth = BasicHttpAuthenticator(username=config["username"], password=config["password"]) - return [Payouts(authenticator=auth, **config), Transactions(authenticator=auth, **config)] +# Declarative Source +class SourceKlarna(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-klarna/source_klarna/spec.yaml b/airbyte-integrations/connectors/source-klarna/source_klarna/spec.yaml deleted file mode 100644 index e2d7dfc9c71f..000000000000 --- a/airbyte-integrations/connectors/source-klarna/source_klarna/spec.yaml +++ /dev/null @@ -1,34 +0,0 @@ -documentationUrl: https://docs.airbyte.com/integrations/sources/klarna -connectionSpecification: - $schema: http://json-schema.org/draft-07/schema# - title: Klarna Spec - type: object - required: - - region - - playground - - username - - password - additionalProperties: true - properties: - region: - title: Region - type: string - enum: - - eu - - us - - oc - description: Base url region (For playground eu https://docs.klarna.com/klarna-payments/api/payments-api/#tag/API-URLs). Supported 'eu', 'us', 'oc' - playground: - title: Playground - type: boolean - description: Propertie defining if connector is used against playground or production environment - default: false - username: - title: Username - type: string - description: Consists of your Merchant ID (eid) - a unique number that identifies your e-store, combined with a random string (https://developers.klarna.com/api/#authentication) - password: - title: Password - type: string - description: A string which is associated with your Merchant ID and is used to authorize use of Klarna's APIs (https://developers.klarna.com/api/#authentication) - airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-klarna/unit_tests/conftest.py b/airbyte-integrations/connectors/source-klarna/unit_tests/conftest.py deleted file mode 100644 index d06b5a1cdb35..000000000000 --- a/airbyte-integrations/connectors/source-klarna/unit_tests/conftest.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pytest as pytest -from airbyte_cdk.sources.streams.http.requests_native_auth import BasicHttpAuthenticator -from source_klarna import SourceKlarna -from source_klarna.source import KlarnaStream - - -@pytest.fixture(name="source_klarna") -def get_source_klarna(): - return SourceKlarna() - - -@pytest.fixture(name="klarna_config") -def get_klarna_config(): - return dict(playground=False, region="eu", username="user", password="password") - - -@pytest.fixture(name="klarna_stream") -def get_klarna_stream(klarna_config): - return KlarnaStream(authenticator=BasicHttpAuthenticator("", ""), **klarna_config) diff --git a/airbyte-integrations/connectors/source-klarna/unit_tests/test_source.py b/airbyte-integrations/connectors/source-klarna/unit_tests/test_source.py deleted file mode 100644 index 2b6b012b5ca2..000000000000 --- a/airbyte-integrations/connectors/source-klarna/unit_tests/test_source.py +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -import responses -from source_klarna.source import SourceKlarna - - -@responses.activate -def test_check_connection(mocker, source_klarna, klarna_config): - responses.add(responses.GET, "https://api.klarna.com/settlements/v1/transactions?offset=0&size=1", json={}) - - logger_mock, config_mock = MagicMock(), klarna_config - assert source_klarna.check_connection(logger_mock, config_mock) == (True, None) - - -def test_streams(mocker, klarna_config): - source = SourceKlarna() - config_mock = klarna_config - streams = source.streams(config_mock) - expected_streams_number = 2 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-klarna/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-klarna/unit_tests/test_streams.py deleted file mode 100644 index ca2f5dc37bcc..000000000000 --- a/airbyte-integrations/connectors/source-klarna/unit_tests/test_streams.py +++ /dev/null @@ -1,92 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from airbyte_cdk.sources.streams.http.requests_native_auth import BasicHttpAuthenticator -from source_klarna.source import KlarnaStream, Payouts, Transactions - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(KlarnaStream, "path", "v0/example_endpoint") - mocker.patch.object(KlarnaStream, "primary_key", "test_primary_key") - mocker.patch.object(KlarnaStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class, klarna_stream): - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {"offset": 0, "size": 500} - assert klarna_stream.request_params(**inputs) == expected_params - - -@pytest.mark.parametrize( - "total,count,offset,next_,expected_params", - [ - (9, 4, 0, "https://api.playground.klarna.com/settlements/v1/payouts?offset=4&size=4", {"offset": ["4"], "size": ["4"]}), - (9, 4, 4, "https://api.playground.klarna.com/settlements/v1/payouts?offset=48&size=4", {"offset": ["48"], "size": ["4"]}), - ], -) -def test_next_page_token(patch_base_class, klarna_stream, total, count, offset, next_, expected_params): - response_mock = MagicMock() - response_mock.json.return_value = { - "pagination": { - "total": total, - "count": count, - "offset": offset, - "next": next_, - } - } - inputs = {"response": response_mock} - assert klarna_stream.next_page_token(**inputs) == expected_params - - -@pytest.mark.parametrize( - ("specific_klarna_stream", "response"), - [ - (Payouts, {"payouts": [{}]}), - (Transactions, {"transactions": [{}]}), - ], -) -def test_parse_response(patch_base_class, klarna_config, specific_klarna_stream, response): - mock_response = MagicMock() - mock_response.json.return_value = response - inputs = {"response": mock_response, "stream_state": {}} - stream = specific_klarna_stream(authenticator=BasicHttpAuthenticator("", ""), **klarna_config) - assert next(stream.parse_response(**inputs)) == {} - - -def test_request_headers(patch_base_class, klarna_stream): - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {} - assert klarna_stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class, klarna_stream): - expected_method = "GET" - assert klarna_stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry, klarna_stream): - response_mock = MagicMock() - response_mock.status_code = http_status - assert klarna_stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class, klarna_stream): - response_mock = MagicMock() - expected_backoff_time = None - assert klarna_stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-klaus-api/.dockerignore b/airbyte-integrations/connectors/source-klaus-api/.dockerignore new file mode 100644 index 000000000000..23ff0658a006 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_klaus_api +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-klaus-api/Dockerfile b/airbyte-integrations/connectors/source-klaus-api/Dockerfile new file mode 100644 index 000000000000..0d8eb064db7c --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.13-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_klaus_api ./source_klaus_api + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-klaus-api diff --git a/airbyte-integrations/connectors/source-klaus-api/README.md b/airbyte-integrations/connectors/source-klaus-api/README.md new file mode 100644 index 000000000000..34a602108bc2 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/README.md @@ -0,0 +1,138 @@ +# Klaus Api Source + +This is the repository for the Klaus Api source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/klaus-api). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-klaus-api:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/klaus-api) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_klaus_api/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source klaus-api test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-klaus-api:dev +``` + +If you want to build the Docker image with the CDK on your local machine (rather than the most recent package published to pypi), from the airbyte base directory run: +```bash +CONNECTOR_TAG= CONNECTOR_NAME= sh airbyte-integrations/scripts/build-connector-image-with-local-cdk.sh +``` + + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-klaus-api:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-klaus-api:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-klaus-api:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-klaus-api:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-klaus-api:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-klaus-api:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-klaus-api:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-klaus-api/acceptance-test-config.yml b/airbyte-integrations/connectors/source-klaus-api/acceptance-test-config.yml new file mode 100644 index 000000000000..2d3795193802 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/acceptance-test-config.yml @@ -0,0 +1,33 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-klaus-api:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_klaus_api/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + + # TODO: Seed sandbox account with records + # incremental: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog_inc.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-klaus-api/icon.svg b/airbyte-integrations/connectors/source-klaus-api/icon.svg new file mode 100644 index 000000000000..303d0f728a6e Binary files /dev/null and b/airbyte-integrations/connectors/source-klaus-api/icon.svg differ diff --git a/airbyte-integrations/connectors/source-klaus-api/integration_tests/__init__.py b/airbyte-integrations/connectors/source-klaus-api/integration_tests/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-klaus-api/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-klaus-api/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..e1294d793511 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/integration_tests/abnormal_state.json @@ -0,0 +1,6 @@ +[ + { + "stream_descriptor": { "name": "reviews" }, + "stream_state": { "lastUpdatedISO": "2123-05-18T23:26:29Z" } + } +] diff --git a/airbyte-integrations/connectors/source-klaus-api/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-klaus-api/integration_tests/acceptance.py new file mode 100644 index 000000000000..82823254d266 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-klaus-api/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-klaus-api/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..3657c9da3fb5 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/integration_tests/configured_catalog.json @@ -0,0 +1,22 @@ +{ + "streams": [ + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "categories", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-klaus-api/integration_tests/configured_catalog_inc.json b/airbyte-integrations/connectors/source-klaus-api/integration_tests/configured_catalog_inc.json new file mode 100644 index 000000000000..8e3f315f3a12 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/integration_tests/configured_catalog_inc.json @@ -0,0 +1,15 @@ +{ + "streams": [ + { + "stream": { + "name": "reviews", + "json_schema": {}, + "source_defined_cursor": true, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["lastUpdatedISO"] + } + ] +} diff --git a/airbyte-integrations/connectors/source-klaus-api/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-klaus-api/integration_tests/expected_records.jsonl new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-klaus-api/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-klaus-api/integration_tests/invalid_config.json new file mode 100644 index 000000000000..0ab3c2e92767 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/integration_tests/invalid_config.json @@ -0,0 +1,4 @@ +{ + "account": 1111, + "workspace": 2222 +} diff --git a/airbyte-integrations/connectors/source-klaus-api/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-klaus-api/integration_tests/sample_config.json new file mode 100644 index 000000000000..62073d5283dd --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/integration_tests/sample_config.json @@ -0,0 +1,5 @@ +{ + "api_key": "i8pjas76t6gd9gfjn8inoh53m6fzed5434r83bi608kh1f4rei", + "account": 1111, + "workspace": 2222 +} diff --git a/airbyte-integrations/connectors/source-klaus-api/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-klaus-api/integration_tests/sample_state.json new file mode 100644 index 000000000000..d03ccbd39194 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/integration_tests/sample_state.json @@ -0,0 +1,6 @@ +[ + { + "stream_descriptor": { "name": "reviews" }, + "stream_state": { "lastUpdatedISO": "2023-05-18T23:26:29Z" } + } +] diff --git a/airbyte-integrations/connectors/source-klaus-api/main.py b/airbyte-integrations/connectors/source-klaus-api/main.py new file mode 100644 index 000000000000..9be6460ab485 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_klaus_api import SourceKlausApi + +if __name__ == "__main__": + source = SourceKlausApi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-klaus-api/metadata.yaml b/airbyte-integrations/connectors/source-klaus-api/metadata.yaml new file mode 100644 index 000000000000..24e538bf888a --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/metadata.yaml @@ -0,0 +1,27 @@ +data: + allowedHosts: + hosts: + - "*" # Please change to the hostname of the source. + registries: + oss: + enabled: true + cloud: + enabled: false + connectorSubtype: api + connectorType: source + definitionId: aad35903-2c0d-4e25-8010-d62ed909e0b7 + dockerImageTag: 0.1.0 + dockerRepository: airbyte/source-klaus-api + documentationUrl: https://docs.airbyte.com/integrations/sources/klaus-api + githubIssueLabel: source-klaus-api + icon: klaus-api.svg + license: MIT + name: Klaus Api + releaseStage: alpha + supportLevel: community + ab_internal: + ql: 300 + sl: 100 + tags: + - language:python +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-klaus-api/requirements.txt b/airbyte-integrations/connectors/source-klaus-api/requirements.txt new file mode 100644 index 000000000000..cc57334ef619 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/connector-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-klaus-api/setup.py b/airbyte-integrations/connectors/source-klaus-api/setup.py new file mode 100644 index 000000000000..b4444f99dd06 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/setup.py @@ -0,0 +1,28 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.2", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.2", + "pytest-mock~=3.6.1", +] + +setup( + name="source_klaus_api", + description="Source implementation for Klaus Api.", + author="Deke Li", + author_email="deke.li@sendinblue.com", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/__init__.py b/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/__init__.py new file mode 100644 index 000000000000..0083ac32f314 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceKlausApi + +__all__ = ["SourceKlausApi"] diff --git a/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/manifest.yaml b/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/manifest.yaml new file mode 100644 index 000000000000..e6c29d79d406 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/manifest.yaml @@ -0,0 +1,95 @@ +version: "0.29.0" + +definitions: + requester: + type: HttpRequester + url_base: "https://kibbles.klausapp.com/api/v2" + http_method: "GET" + authenticator: + type: "BearerAuthenticator" + api_token: "{{ config['api_key'] }}" + + datetime_cursor: + type: "DatetimeBasedCursor" + start_time_option: + field_name: "fromDate" + inject_into: "request_parameter" + end_time_option: + field_name: "toDate" + inject_into: "request_parameter" + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: "P1W" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + cursor_granularity: "PT0.001S" + cursor_field: "lastUpdatedISO" + + record_retriever: + type: SimpleRetriever + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["{{ parameters.name }}"] + paginator: + type: NoPagination + + reviews_retriever: + type: SimpleRetriever + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["conversations"] + paginator: + type: NoPagination + + reviews_stream: + type: DeclarativeStream + $parameters: + name: "reviews" + path: "/account/{{ config['account'] }}/workspace/{{ config['workspace'] }}/reviews" + retriever: + $ref: "#/definitions/reviews_retriever" + requester: + $ref: "#/definitions/requester" + incremental_sync: + $ref: "#/definitions/datetime_cursor" + + users_stream: + type: DeclarativeStream + $parameters: + name: "users" + path: "/account/{{ config['account'] }}/users" + retriever: + $ref: "#/definitions/record_retriever" + requester: + $ref: "#/definitions/requester" + primary_key: "id" + + categories_stream: + type: DeclarativeStream + $parameters: + name: "categories" + path: "/account/{{ config['account'] }}/workspace/{{ config['workspace'] }}/categories" + retriever: + $ref: "#/definitions/record_retriever" + requester: + $ref: "#/definitions/requester" + primary_key: "id" + +streams: + - "#/definitions/reviews_stream" + - "#/definitions/users_stream" + - "#/definitions/categories_stream" + +check: + type: CheckStream + stream_names: + - "reviews" + - "users" + - "categories" diff --git a/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/schemas/categories.json b/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/schemas/categories.json new file mode 100644 index 000000000000..3c59e46bdf23 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/schemas/categories.json @@ -0,0 +1,20 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { "type": ["string", "null"] }, + "name": { "type": ["string", "null"] }, + "description": { "type": ["string", "null"] }, + "critical": { "type": ["boolean", "null"] }, + "archived": { "type": ["boolean", "null"] }, + "weight": { "type": ["number", "null"] }, + "maxRating": { "type": ["integer", "null"] }, + "scorecards": { "type": "array", "items": { "type": "string" } }, + "rootCauses": { "type": "array", "items": { "type": "string" } }, + "position": { "type": ["integer", "null"] }, + "groupId": { "type": ["string", "null"] }, + "groupName": { "type": ["string", "null"] }, + "groupPosition": { "type": ["integer", "null"] } + } +} diff --git a/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/schemas/reviews.json b/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/schemas/reviews.json new file mode 100644 index 000000000000..deb183f56ada --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/schemas/reviews.json @@ -0,0 +1,396 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "comments": { + "items": { + "properties": { + "comment": { + "type": ["string", "null"] + }, + "created": { + "type": ["string", "null"] + }, + "createdISO": { + "type": ["string", "null"] + }, + "id": { + "type": ["string", "null"] + }, + "owner": { + "properties": { + "avatar": { + "type": ["string", "null"] + }, + "email": { + "type": ["string", "null"] + }, + "groups": { + "items": { + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "type": "array" + }, + "name": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "tags": { + "type": ["array", "null"] + }, + "thread": { + "items": { + "properties": { + "comment": { + "type": ["string", "null"] + }, + "created": { + "type": ["string", "null"] + }, + "createdISO": { + "type": ["string", "null"] + }, + "id": { + "type": ["string", "null"] + }, + "owner": { + "properties": { + "avatar": { + "type": ["string", "null"] + }, + "email": { + "type": ["string", "null"] + }, + "groups": { + "type": ["array", "null"] + }, + "name": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "tags": { + "items": { + "properties": { + "tag": { + "type": ["string", "null"] + }, + "user": { + "properties": { + "avatar": { + "type": ["string", "null"] + }, + "email": { + "type": ["string", "null"] + }, + "groups": { + "items": { + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "type": ["array", "null"] + }, + "name": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + } + }, + "type": ["object", "null"] + }, + "type": ["array", "null"] + }, + "thread": { + "type": ["array", "null"] + }, + "updated": { + "type": ["string", "null"] + }, + "updatedISO": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "type": ["array", "null"] + }, + "updated": { + "type": ["string", "null"] + }, + "updatedISO": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "type": ["array", "null"] + }, + "createdAt": { + "type": ["string", "null"] + }, + "createdAtISO": { + "type": ["string", "null"] + }, + "externalId": { + "type": ["string", "null"] + }, + "externalUrl": { + "type": ["string", "null"] + }, + "lastUpdatedISO": { + "type": ["string", "null"] + }, + "reviews": { + "items": { + "properties": { + "comment": { + "type": ["null", "string"] + }, + "created": { + "type": ["string", "null"] + }, + "createdISO": { + "type": ["string", "null"] + }, + "id": { + "type": ["string", "null"] + }, + "ratings": { + "items": { + "properties": { + "categoryId": { + "type": ["string", "null"] + }, + "categoryName": { + "type": ["string", "null"] + }, + "cause": { + "type": "null" + }, + "critical": { + "type": ["boolean", "null"] + }, + "score": { + "type": ["integer", "null"] + }, + "weight": { + "type": ["number", "null"] + } + }, + "type": ["object", "null"] + }, + "type": ["array", "null"] + }, + "received": { + "type": ["boolean", "null"] + }, + "reviewTime": { + "type": ["string", "null"] + }, + "reviewee": { + "properties": { + "avatar": { + "type": ["string", "null"] + }, + "email": { + "type": ["string", "null"] + }, + "groups": { + "items": { + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "type": ["array", "null"] + }, + "name": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "reviewer": { + "properties": { + "avatar": { + "type": ["string", "null"] + }, + "email": { + "type": ["string", "null"] + }, + "groups": { + "items": { + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "type": ["array", "null"] + }, + "name": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "score": { + "type": ["number", "null"] + }, + "scorecard": { + "type": "null" + }, + "tags": { + "type": ["array", "null"] + }, + "thread": { + "items": { + "properties": { + "comment": { + "type": ["string", "null"] + }, + "created": { + "type": ["string", "null"] + }, + "createdISO": { + "type": ["string", "null"] + }, + "id": { + "type": ["string", "null"] + }, + "owner": { + "properties": { + "avatar": { + "type": ["string", "null"] + }, + "email": { + "type": ["string", "null"] + }, + "groups": { + "items": { + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "type": ["array", "null"] + }, + "name": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "tags": { + "items": { + "properties": { + "tag": { + "type": ["string", "null"] + }, + "user": { + "properties": { + "avatar": { + "type": ["string", "null"] + }, + "email": { + "type": ["string", "null"] + }, + "groups": { + "items": { + "properties": { + "id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "type": ["array", "null"] + }, + "name": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + } + }, + "type": ["object", "null"] + }, + "type": ["array", "null"] + }, + "thread": { + "type": ["array", "null"] + }, + "updated": { + "type": ["string", "null"] + }, + "updatedISO": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "type": ["array", "null"] + }, + "updated": { + "type": ["string", "null"] + }, + "updatedISO": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "type": ["array", "null"] + }, + "sourceType": { + "type": ["string", "null"] + }, + "updated_at": { + "type": ["string", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "workspaceId": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] +} diff --git a/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/schemas/users.json b/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/schemas/users.json new file mode 100644 index 000000000000..214543d899ea --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/schemas/users.json @@ -0,0 +1,10 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { "type": ["string", "null"] }, + "name": { "type": ["string", "null"] }, + "email": { "type": ["string", "null"] } + } +} diff --git a/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/source.py b/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/source.py new file mode 100644 index 000000000000..6071fc1ffd29 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceKlausApi(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/spec.yaml b/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/spec.yaml new file mode 100644 index 000000000000..248413f0f2cf --- /dev/null +++ b/airbyte-integrations/connectors/source-klaus-api/source_klaus_api/spec.yaml @@ -0,0 +1,34 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/klaus-api +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Klaus Api Spec + type: object + required: + - api_key + - account + - workspace + additionalProperties: true + properties: + api_key: + type: string + description: API access key used to retrieve data from the KLAUS API. + airbyte_secret: true + order: 1 + account: + type: integer + title: account + description: getting data by account + order: 2 + workspace: + type: integer + title: workspace + description: getting data by workspace + order: 3 + start_date: + type: string + description: Start getting data from that date. + format: date-time + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ + examples: + - "2020-10-15T00:00:00Z" + order: 4 diff --git a/airbyte-integrations/connectors/source-klaviyo/Dockerfile b/airbyte-integrations/connectors/source-klaviyo/Dockerfile deleted file mode 100644 index e36a1a15073e..000000000000 --- a/airbyte-integrations/connectors/source-klaviyo/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_klaviyo ./source_klaviyo - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.3.2 -LABEL io.airbyte.name=airbyte/source-klaviyo diff --git a/airbyte-integrations/connectors/source-klaviyo/README.md b/airbyte-integrations/connectors/source-klaviyo/README.md index c456c2ac231e..61e657f87278 100644 --- a/airbyte-integrations/connectors/source-klaviyo/README.md +++ b/airbyte-integrations/connectors/source-klaviyo/README.md @@ -32,14 +32,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-klaviyo:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/klaviyo) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_klaviyo/spec.json` file. @@ -59,18 +51,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-klaviyo:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-klaviyo build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-klaviyo:airbyteDocker +An image will be built with the tag `airbyte/source-klaviyo:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-klaviyo:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -80,44 +73,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-klaviyo:dev check --co docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-klaviyo:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-klaviyo:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json ``` + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-klaviyo test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unittest run: -``` -./gradlew :airbyte-integrations:connectors:source-klaviyo:unitTest -``` -To run acceptance and custom integration tests run: -``` -./gradlew :airbyte-integrations:connectors:source-klaviyo:IntegrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -127,8 +92,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-klaviyo test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/klaviyo.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-klaviyo/acceptance-test-config.yml b/airbyte-integrations/connectors/source-klaviyo/acceptance-test-config.yml index 955fd79d0665..3558908788bd 100644 --- a/airbyte-integrations/connectors/source-klaviyo/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-klaviyo/acceptance-test-config.yml @@ -1,38 +1,37 @@ acceptance_tests: basic_read: tests: - - config_path: secrets/config.json - empty_streams: - - name: flows - bypass_reason: "the 'trigger' object may mutate often" - expect_records: - path: integration_tests/expected_records.jsonl - extra_records: true - ignored_fields: - email_templates: - - name: html - bypass_reason: unstable data + - config_path: secrets/config.json + expect_records: + path: integration_tests/expected_records.jsonl + extra_records: true + ignored_fields: + email_templates: + - name: "attributes/html" + bypass_reason: unstable data connection: tests: - - config_path: secrets/config.json - status: succeed - - config_path: integration_tests/invalid_config.json - status: failed + - config_path: secrets/config.json + status: succeed + - config_path: integration_tests/invalid_config.json + status: failed discovery: tests: - - config_path: secrets/config.json + - config_path: secrets/config.json + backward_compatibility_tests_config: + disable_for_version: "1.1.0" full_refresh: tests: - - config_path: secrets/config.json - - configured_catalog_path: integration_tests/configured_catalog.json + - config_path: secrets/config.json + - configured_catalog_path: integration_tests/configured_catalog.json incremental: tests: - - config_path: secrets/config.json - configured_catalog_path: integration_tests/configured_catalog.json - future_state: - future_state_path: integration_tests/abnormal_state.json + - config_path: secrets/config.json + configured_catalog_path: integration_tests/configured_catalog.json + future_state: + future_state_path: integration_tests/abnormal_state.json spec: tests: - - spec_path: source_klaviyo/spec.json + - spec_path: source_klaviyo/spec.json connector_image: airbyte/source-klaviyo:dev test_strictness_level: high diff --git a/airbyte-integrations/connectors/source-klaviyo/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-klaviyo/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-klaviyo/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-klaviyo/build.gradle b/airbyte-integrations/connectors/source-klaviyo/build.gradle deleted file mode 100644 index 90472c426112..000000000000 --- a/airbyte-integrations/connectors/source-klaviyo/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_klaviyo' -} diff --git a/airbyte-integrations/connectors/source-klaviyo/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-klaviyo/integration_tests/abnormal_state.json index c0374b6b23f1..5c4fe027f477 100644 --- a/airbyte-integrations/connectors/source-klaviyo/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-klaviyo/integration_tests/abnormal_state.json @@ -2,42 +2,52 @@ { "type": "STREAM", "stream": { - "stream_state": { "timestamp": 9621295127 }, + "stream_state": { "datetime": "2120-10-10T00:00:00Z" }, "stream_descriptor": { "name": "events" } } }, { "type": "STREAM", "stream": { - "stream_state": { "timestamp": "2120-10-10T00:00:00Z" }, + "stream_state": { "updated": "2120-10-10T00:00:00Z" }, "stream_descriptor": { "name": "global_exclusions" } } }, { "type": "STREAM", "stream": { - "stream_state": { "created": "2120-10-10 00:00:00" }, + "stream_state": { + "updated": "2120-10-10 00:00:00", + "archived": { + "updated": "2120-10-10 00:00:00" + } + }, "stream_descriptor": { "name": "flows" } } }, { "type": "STREAM", "stream": { - "stream_state": { "created": "2120-10-10 00:00:00" }, + "stream_state": { "updated": "2120-10-10 00:00:00" }, "stream_descriptor": { "name": "metrics" } } }, { "type": "STREAM", "stream": { - "stream_state": { "created": "2120-10-10 00:00:00" }, + "stream_state": { "updated": "2120-10-10 00:00:00" }, "stream_descriptor": { "name": "lists" } } }, { "type": "STREAM", "stream": { - "stream_state": { "created": "2120-10-10 00:00:00" }, + "stream_state": { + "updated_at": "2120-10-10 00:00:00", + "archived": { + "updated_at": "2120-10-10 00:00:00" + } + }, "stream_descriptor": { "name": "campaigns" } } }, @@ -47,5 +57,12 @@ "stream_state": { "updated": "2120-10-10 00:00:00" }, "stream_descriptor": { "name": "profiles" } } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated": "2120-10-10 00:00:00" }, + "stream_descriptor": { "name": "email_templates" } + } } ] diff --git a/airbyte-integrations/connectors/source-klaviyo/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-klaviyo/integration_tests/configured_catalog.json index bcb7e7a93dc1..7203e0bc0a6e 100644 --- a/airbyte-integrations/connectors/source-klaviyo/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-klaviyo/integration_tests/configured_catalog.json @@ -41,7 +41,7 @@ "namespace": null }, "sync_mode": "full_refresh", - "cursor_field": ["timestamp"], + "cursor_field": ["datetime"], "destination_sync_mode": "append", "primary_key": [["id"]] }, @@ -52,13 +52,13 @@ "supported_sync_modes": ["full_refresh"], "source_defined_cursor": null, "default_cursor_field": null, - "source_defined_primary_key": [["email"]], + "source_defined_primary_key": [["id"]], "namespace": null }, "sync_mode": "full_refresh", "cursor_field": null, "destination_sync_mode": "append", - "primary_key": [["email"]] + "primary_key": [["id"]] }, { "stream": { diff --git a/airbyte-integrations/connectors/source-klaviyo/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-klaviyo/integration_tests/expected_records.jsonl index 1cbe453f08c0..419da2f2429b 100644 --- a/airbyte-integrations/connectors/source-klaviyo/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-klaviyo/integration_tests/expected_records.jsonl @@ -1,81 +1,91 @@ -{"stream": "campaigns", "data": {"object": "campaign", "id": "VFaYVy", "name": "Email Campaign 2021-05-16 19:17:45", "subject": "My Test subject", "from_email": "integration-test@airbyte.io", "from_name": "Airbyte", "lists": [{"object": "list", "id": "RnsiHB", "name": "Newsletter", "list_type": "list", "folder": null, "created": "2021-03-31T10:50:36+00:00", "updated": "2021-03-31T10:50:36+00:00", "person_count": 1}, {"object": "list", "id": "TaSce6", "name": "Preview List", "list_type": "list", "folder": null, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00", "person_count": 1}], "excluded_lists": [{"object": "list", "id": "Ukh37W", "name": "Unengaged (3 Months)", "list_type": "segment", "folder": null, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:43+00:00", "person_count": 0}], "status": "sent", "status_id": 1, "status_label": "Sent", "sent_at": "2021-05-26T23:30:13+00:00", "send_time": "2021-05-26T23:30:00+00:00", "created": "2021-05-16T23:17:45+00:00", "updated": "2021-05-26T23:30:13+00:00", "num_recipients": 1, "campaign_type": "Batch", "is_segmented": true, "message_type": "email", "template_id": "VR2KEG"}, "emitted_at": 1663367156487} -{"stream": "campaigns", "data": {"object": "campaign", "id": "T4hgvQ", "name": "Email Campaign 2021-05-12 16:45:46", "subject": "", "from_email": "integration-test@airbyte.io", "from_name": "Airbyte", "lists": [], "excluded_lists": [], "status": "draft", "status_id": 2, "status_label": "Draft", "sent_at": null, "send_time": null, "created": "2021-05-12T20:45:47+00:00", "updated": "2021-05-12T20:45:47+00:00", "num_recipients": 0, "campaign_type": "Regular", "is_segmented": false, "message_type": "email", "template_id": null}, "emitted_at": 1663367156491} -{"stream": "events", "data": {"object": "event", "id": "3qvdbYg3", "statistic_id": "VFFb4u", "timestamp": 1621295008, "event_name": "Clicked Email", "event_properties": { "$event_id": "1621295008" }, "datetime": "2021-05-17 23:43:28+00:00", "uuid": "adc8d000-b769-11eb-8001-28a6687f81c3", "person": { "object": "person", "id": "01F5YBDQE9W7WDSH9KK398CAYX", "$address1": "", "$address2": "", "$city": "", "$country": "", "$latitude": "", "$longitude": "", "$region": "", "$zip": "", "$last_name": "", "$title": "", "$organization": "", "$phone_number": "", "$email": "some.email.that.dont.exist.{seed}@airbyte.io", "$first_name": "", "$timezone": "", "$id": "", "email": "some.email.that.dont.exist.{seed}@airbyte.io", "first_name": "", "last_name": "", "created": "2021-05-17 23:43:50", "updated": "2021-05-17 23:43:50" }, "flow_id": null, "flow_message_id": null, "campaign_id": null }, "emitted_at": 1663367160652} -{"stream": "events", "data": {"object": "event", "id": "3qvdgpzF", "statistic_id": "VFFb4u", "timestamp": 1621295124, "event_name": "Clicked Email", "event_properties": { "$event_id": "1621295124" }, "datetime": "2021-05-17 23:45:24+00:00", "uuid": "f2ed0200-b769-11eb-8001-76152f6b1c82", "person": { "object": "person", "id": "01F5YBGKW1SQN453RM293PHH37", "$address1": "", "$address2": "", "$city": "Springfield", "$country": "", "$latitude": "", "$longitude": "", "$region": "Illinois", "$zip": "", "$last_name": "Last Name 0", "$title": "", "$organization": "", "$phone_number": "", "$email": "some.email.that.dont.exist.0@airbyte.io", "$first_name": "First Name 0", "$timezone": "", "$id": "", "email": "some.email.that.dont.exist.0@airbyte.io", "first_name": "First Name 0", "last_name": "Last Name 0", "created": "2021-05-17 23:45:24", "updated": "2021-05-17 23:45:25" }, "flow_id": null, "flow_message_id": null, "campaign_id": null }, "emitted_at": 1663367160652} -{"stream": "events", "data": {"object": "event", "id": "3qvdgr5Z", "statistic_id": "VFFb4u", "timestamp": 1621295124, "event_name": "Clicked Email", "event_properties": { "$event_id": "1621295124" }, "datetime": "2021-05-17 23:45:24+00:00", "uuid": "f2ed0200-b769-11eb-8001-b642ddab48ad", "person": { "object": "person", "id": "01F5YBGM7J4YD4P6EYK5Q87BG4", "$address1": "", "$address2": "", "$city": "Springfield", "$country": "", "$latitude": "", "$longitude": "", "$region": "Illinois", "$zip": "", "$last_name": "Last Name 1", "$title": "", "$organization": "", "$phone_number": "", "$email": "some.email.that.dont.exist.1@airbyte.io", "$first_name": "First Name 1", "$timezone": "", "$id": "", "email": "some.email.that.dont.exist.1@airbyte.io", "first_name": "First Name 1", "last_name": "Last Name 1", "created": "2021-05-17 23:45:25", "updated": "2021-05-17 23:45:26" }, "flow_id": null, "flow_message_id": null, "campaign_id": null }, "emitted_at": 1663367160652} -{"stream": "events", "data": {"object": "event", "id": "3qvdgBgK", "statistic_id": "VFFb4u", "timestamp": 1621295124, "event_name": "Clicked Email", "event_properties": { "$event_id": "1621295124" }, "datetime": "2021-05-17 23:45:24+00:00", "uuid": "f2ed0200-b769-11eb-8001-2006a2b2b6e7", "person": { "object": "person", "id": "01F5YBGMK62AJR0955G7NW6EP7", "$address1": "", "$address2": "", "$city": "Springfield", "$country": "", "$latitude": "", "$longitude": "", "$region": "Illinois", "$zip": "", "$last_name": "Last Name 2", "$title": "", "$organization": "", "$phone_number": "", "$email": "some.email.that.dont.exist.2@airbyte.io", "$first_name": "First Name 2", "$timezone": "", "$id": "", "email": "some.email.that.dont.exist.2@airbyte.io", "first_name": "First Name 2", "last_name": "Last Name 2", "created": "2021-05-17 23:45:25", "updated": "2021-05-17 23:45:38" }, "flow_id": null, "flow_message_id": null, "campaign_id": null }, "emitted_at": 1663367160652} -{"stream": "events", "data": {"object": "event", "id": "3qvdgs9P", "statistic_id": "VFFb4u", "timestamp": 1621295125, "event_name": "Clicked Email", "event_properties": { "$event_id": "1621295125" }, "datetime": "2021-05-17 23:45:25+00:00", "uuid": "f3859880-b769-11eb-8001-f6a061424b91", "person": { "object": "person", "id": "01F5YBGMK62AJR0955G7NW6EP7", "$address1": "", "$address2": "", "$city": "Springfield", "$country": "", "$latitude": "", "$longitude": "", "$region": "Illinois", "$zip": "", "$last_name": "Last Name 2", "$title": "", "$organization": "", "$phone_number": "", "$email": "some.email.that.dont.exist.2@airbyte.io", "$first_name": "First Name 2", "$timezone": "", "$id": "", "email": "some.email.that.dont.exist.2@airbyte.io", "first_name": "First Name 2", "last_name": "Last Name 2", "created": "2021-05-17 23:45:25", "updated": "2021-05-17 23:45:38" }, "flow_id": null, "flow_message_id": null, "campaign_id": null }, "emitted_at": 1663367160652} -{"stream": "global_exclusions", "data": {"object": "exclusion", "email": "some.email.that.dont.exist.9@airbyte.io", "reason": "manually_excluded", "timestamp": "2021-05-18T01:20:01+00:00"}, "emitted_at": 1663367161413} -{"stream": "global_exclusions", "data": {"object": "exclusion", "email": "some.email.that.dont.exist.8@airbyte.io", "reason": "manually_excluded", "timestamp": "2021-05-18T01:29:51+00:00"}, "emitted_at": 1663367161413} -{"stream": "lists", "data": {"object": "list", "id": "RnsiHB", "name": "Newsletter", "list_type": "list", "folder": null, "created": "2021-03-31T10:50:36+00:00", "updated": "2021-03-31T10:50:36+00:00", "person_count": 1}, "emitted_at": 1663367161878} -{"stream": "lists", "data": {"object": "list", "id": "TaSce6", "name": "Preview List", "list_type": "list", "folder": null, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00", "person_count": 3}, "emitted_at": 1663367161881} -{"stream": "lists", "data": {"object": "list", "id": "UXi5Jz", "name": "New Subscribers", "list_type": "segment", "folder": null, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:43+00:00", "person_count": 0}, "emitted_at": 1663367161881} -{"stream": "lists", "data": {"object": "list", "id": "R2p3ry", "name": "Test2", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:04+00:00", "updated": "2021-11-16T14:24:04+00:00", "person_count": 0}, "emitted_at": 1663367161882} -{"stream": "lists", "data": {"object": "list", "id": "S7aBY2", "name": "Test1", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:07+00:00", "updated": "2021-11-16T14:24:07+00:00", "person_count": 0}, "emitted_at": 1663367161882} -{"stream": "lists", "data": {"object": "list", "id": "XpP2a5", "name": "Test4", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:10+00:00", "updated": "2021-11-16T14:24:10+00:00", "person_count": 0}, "emitted_at": 1663367161884} -{"stream": "lists", "data": {"object": "list", "id": "TDGJsj", "name": "Test3", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:14+00:00", "updated": "2021-11-16T14:24:14+00:00", "person_count": 0}, "emitted_at": 1663367161884} -{"stream": "lists", "data": {"object": "list", "id": "WBxsQE", "name": "Test5", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:17+00:00", "updated": "2021-11-16T14:24:17+00:00", "person_count": 0}, "emitted_at": 1663367161884} -{"stream": "lists", "data": {"object": "list", "id": "VmvmBq", "name": "Test5", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:18+00:00", "updated": "2021-11-16T14:24:18+00:00", "person_count": 0}, "emitted_at": 1663367161885} -{"stream": "lists", "data": {"object": "list", "id": "XGj3p8", "name": "Test5", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:20+00:00", "updated": "2021-11-16T14:24:20+00:00", "person_count": 0}, "emitted_at": 1663367161885} -{"stream": "lists", "data": {"object": "list", "id": "R4ZhCr", "name": "Test5", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:21+00:00", "updated": "2021-11-16T14:24:21+00:00", "person_count": 0}, "emitted_at": 1663367161886} -{"stream": "lists", "data": {"object": "list", "id": "Seq8wh", "name": "Test5", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:22+00:00", "updated": "2021-11-16T14:24:22+00:00", "person_count": 0}, "emitted_at": 1663367161886} -{"stream": "lists", "data": {"object": "list", "id": "TpNXq9", "name": "Test5", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:23+00:00", "updated": "2021-11-16T14:24:23+00:00", "person_count": 0}, "emitted_at": 1663367161886} -{"stream": "lists", "data": {"object": "list", "id": "UzdNhZ", "name": "Test5", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:24+00:00", "updated": "2021-11-16T14:24:24+00:00", "person_count": 0}, "emitted_at": 1663367161886} -{"stream": "lists", "data": {"object": "list", "id": "TWcKFn", "name": "Test5", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:25+00:00", "updated": "2021-11-16T14:24:25+00:00", "person_count": 0}, "emitted_at": 1663367161887} -{"stream": "lists", "data": {"object": "list", "id": "Ya5ziX", "name": "Test5", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:25+00:00", "updated": "2021-11-16T14:24:25+00:00", "person_count": 0}, "emitted_at": 1663367161887} -{"stream": "lists", "data": {"object": "list", "id": "RwKPyg", "name": "Test5", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:26+00:00", "updated": "2021-11-16T14:24:26+00:00", "person_count": 0}, "emitted_at": 1663367161887} -{"stream": "lists", "data": {"object": "list", "id": "VJCDbR", "name": "Test5", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:26+00:00", "updated": "2021-11-16T14:24:26+00:00", "person_count": 0}, "emitted_at": 1663367161887} -{"stream": "lists", "data": {"object": "list", "id": "TjbH4K", "name": "Test5", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:27+00:00", "updated": "2021-11-16T14:24:27+00:00", "person_count": 0}, "emitted_at": 1663367161888} -{"stream": "lists", "data": {"object": "list", "id": "VDZnQt", "name": "Test5", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:28+00:00", "updated": "2021-11-16T14:24:28+00:00", "person_count": 0}, "emitted_at": 1663367161888} -{"stream": "lists", "data": {"object": "list", "id": "WJLXnV", "name": "Test5", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:28+00:00", "updated": "2021-11-16T14:24:28+00:00", "person_count": 0}, "emitted_at": 1663367161888} -{"stream": "lists", "data": {"object": "list", "id": "XUbNgM", "name": "Test5", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:29+00:00", "updated": "2021-11-16T14:24:29+00:00", "person_count": 0}, "emitted_at": 1663367161888} -{"stream": "lists", "data": {"object": "list", "id": "RgS4w6", "name": "Test5", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:30+00:00", "updated": "2021-11-16T14:24:30+00:00", "person_count": 0}, "emitted_at": 1663367161888} -{"stream": "lists", "data": {"object": "list", "id": "UeGLUr", "name": "Test5", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:30+00:00", "updated": "2021-11-16T14:24:30+00:00", "person_count": 0}, "emitted_at": 1663367161889} -{"stream": "lists", "data": {"object": "list", "id": "RPfQMj", "name": "Test5", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:31+00:00", "updated": "2021-11-16T15:01:15+00:00", "person_count": 0}, "emitted_at": 1663367161889} -{"stream": "lists", "data": {"object": "list", "id": "SYEFFb", "name": "Test_5", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:31+00:00", "updated": "2021-11-16T14:24:31+00:00", "person_count": 0}, "emitted_at": 1663367161889} -{"stream": "lists", "data": {"object": "list", "id": "SmDD4y", "name": "Test5__x", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:32+00:00", "updated": "2021-11-16T14:24:32+00:00", "person_count": 0}, "emitted_at": 1663367161890} -{"stream": "lists", "data": {"object": "list", "id": "X7UeXn", "name": "Test5___", "list_type": "list", "folder": null, "created": "2021-11-16T14:24:34+00:00", "updated": "2021-11-16T14:24:34+00:00", "person_count": 1}, "emitted_at": 1663367161890} -{"stream": "lists", "data": {"object": "list", "id": "S8nmQ9", "name": "Test AAAB", "list_type": "list", "folder": null, "created": "2021-11-16T15:02:51+00:00", "updated": "2021-11-16T15:02:51+00:00", "person_count": 1}, "emitted_at": 1663367161890} -{"stream": "lists", "data": {"object": "list", "id": "SBYgiK", "name": "SMS Subscribers", "list_type": "list", "folder": null, "created": "2022-05-31T06:52:26+00:00", "updated": "2022-05-31T06:52:26+00:00", "person_count": 0}, "emitted_at": 1663367161890} -{"stream": "metrics", "data": {"object": "metric", "id": "RUQ6YQ", "name": "Active on Site", "integration": {"object": "integration", "id": "7FtS4J", "name": "API", "category": "API"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1688724105477} -{"stream": "metrics", "data": {"object": "metric", "id": "Xi7Kwh", "name": "Bounced Email", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1688724105478} -{"stream": "metrics", "data": {"object": "metric", "id": "VePdj9", "name": "Cancelled Order", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1688724105478} -{"stream": "metrics", "data": {"object": "metric", "id": "SPnhc3", "name": "Checkout Started", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1688724105478} -{"stream": "metrics", "data": {"object": "metric", "id": "SxR9Bt", "name": "Clicked Email", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1688724105478} -{"stream": "metrics", "data": {"object": "metric", "id": "VFFb4u", "name": "Clicked Email", "integration": {"object": "integration", "id": "7FtS4J", "name": "API", "category": "API"}, "created": "2021-05-17T23:43:50+00:00", "updated": "2021-05-17T23:43:50+00:00"}, "emitted_at": 1688724105479} -{"stream": "metrics", "data": {"object": "metric", "id": "Y5TbbA", "name": "Clicked SMS", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1688724105479} -{"stream": "metrics", "data": {"object": "metric", "id": "RhP4nd", "name": "Dropped Email", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1688724105479} -{"stream": "metrics", "data": {"object": "metric", "id": "TeZiVn", "name": "Failed to deliver Automated Response SMS", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1688724105479} -{"stream": "metrics", "data": {"object": "metric", "id": "S5Au3w", "name": "Failed to Deliver SMS", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1688724105479} -{"stream": "metrics", "data": {"object": "metric", "id": "X3f6PC", "name": "Fulfilled Order", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1688724105480} -{"stream": "metrics", "data": {"object": "metric", "id": "RcjEmN", "name": "Fulfilled Partial Order", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}, "created": "2022-05-31T06:45:47+00:00", "updated": "2022-05-31T06:45:47+00:00"}, "emitted_at": 1688724105480} -{"stream": "metrics", "data": {"object": "metric", "id": "THfYvj", "name": "Marked Email as Spam", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1688724105480} -{"stream": "metrics", "data": {"object": "metric", "id": "Yy9QKx", "name": "Opened Email", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1688724105480} -{"stream": "metrics", "data": {"object": "metric", "id": "RDXsib", "name": "Ordered Product", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1688724105480} -{"stream": "metrics", "data": {"object": "metric", "id": "TspjNE", "name": "Placed Order", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1688724105480} -{"stream": "metrics", "data": {"object": "metric", "id": "RszrqT", "name": "Received Automated Response SMS", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1688724105480} -{"stream": "metrics", "data": {"object": "metric", "id": "WKHXf4", "name": "Received Email", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1688724105480} -{"stream": "metrics", "data": {"object": "metric", "id": "WhthF7", "name": "Received SMS", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1688724105481} -{"stream": "metrics", "data": {"object": "metric", "id": "R2WpFy", "name": "Refunded Order", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1688724105481} -{"stream": "metrics", "data": {"object": "metric", "id": "XsS8yX", "name": "Sent SMS", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1688724105481} -{"stream": "metrics", "data": {"object": "metric", "id": "UBNaGw", "name": "Subscribed to Back in Stock", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1688724105481} -{"stream": "metrics", "data": {"object": "metric", "id": "Tp8t7d", "name": "Subscribed to List", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-11-16T15:05:22+00:00", "updated": "2021-11-16T15:05:22+00:00"}, "emitted_at": 1688724105481} -{"stream": "metrics", "data": {"object": "metric", "id": "VEsf4u", "name": "Subscribed to SMS Marketing", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1688724105481} -{"stream": "metrics", "data": {"object": "metric", "id": "VvFRZN", "name": "Unsubscribed", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1688724105481} -{"stream": "metrics", "data": {"object": "metric", "id": "TS2mxZ", "name": "Unsubscribed from SMS Marketing", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}, "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1688724105481} -{"stream": "metrics", "data": {"object": "metric", "id": "YcDVHu", "name": "Viewed Product", "integration": {"object": "integration", "id": "7FtS4J", "name": "API", "category": "API"}, "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1688724105482} -{"stream": "email_templates", "data": {"object": "email-template", "id": "RdbN2P", "name": "Newsletter #1 (Images & Text)", "html": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n
      \n
      \n\n\n\n\n\n\n
      \n\n
      \n\n\n\n\n\n\n
      \n\n
      \n\n\n\n\n\n\n
      \n\n
      \n\n
      \n\n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n
      \n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n

      \n

      \n\n
      \n
      \n
      \n
      \n\n
      \n\n\n
      \n\n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n

      This template starts with images.

      \n
      \n
      \n
      \n
      \n\n
      \n\n\n
      \n\n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n\n
      \n\n
      \n
      \n
      \n
      \n
      \n
      \n\n\n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n\n
      \n\n
      \n
      \n
      \n
      \n
      \n
      \n\n\n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n\n
      \n\n
      \n
      \n
      \n
      \n
      \n
      \n\n
      \n\n\n
      \n\n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n

      Everyone loves pictures. They're more engaging that text by itself and the images in this template will neatly stack on mobile devices for the best viewing experience.

      \n

      Use the text area below to add additional content or add more images to create a larger image gallery. You can drag blocks from the left sidebar to add content to your template. You can customize this colors, fonts and styling of this template to match your brand by clicking the \"Styles\" button to the left.

      \n

      Happy emailing!

      \n

      The Klaviyo Team

      \n
      \n
      \n
      \n
      \n\n
      \n\n\n
      \n\n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n
      \n\n
      \n\n
      \n\n\"Facebook\"\n\n
      \n\n
      \n
      \n\n
      \n\n\"Twitter\"\n\n
      \n\n
      \n
      \n\n
      \n\n\"LinkedIn\"\n\n
      \n\n
      \n\n
      \n
      \n
      \n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n
      No longer want to receive these emails? {% unsubscribe %}.
      {{ organization.name }} {{ organization.full_address }}
      \n
      \n
      \n
      \n
      \n\n
      \n\n
      \n\n
      \n
      \n\n
      \n
      \n\n
      \n\n
      \n\n\n\n\n\n\n
      \n\n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n\n\"Powered\n\n
      \n
      \n
      \n\n
      \n
      \n\n
      \n
      \n\n", "is_writeable": true, "created": "2021-03-31T10:50:37+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1686259883909} -{"stream": "profiles", "data": {"type": "profile", "id": "01F5VTP8THZD8CGS2AKNE63370", "attributes": {"email": "some.email.that.dont.exist@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name", "last_name": "Last Name", "organization": null, "title": null, "image": null, "created": "2021-05-17T00:12:55+00:00", "updated": "2021-05-17T00:12:55+00:00", "last_event_date": null, "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": {"marketing": null}}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTP8THZD8CGS2AKNE63370/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTP8THZD8CGS2AKNE63370/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5VTP8THZD8CGS2AKNE63370/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTP8THZD8CGS2AKNE63370/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5VTP8THZD8CGS2AKNE63370/segments/"}}}, "updated": "2021-05-17T00:12:55+00:00"}, "emitted_at": 1679533540462} -{"stream": "profiles", "data": {"type": "profile", "id": "01F5VTQ44548K2TBCG1EWPZEDN", "attributes": {"email": "some.email.that.dont.exist2@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "Strange Name1", "last_name": "Funny Name1", "organization": null, "title": null, "image": null, "created": "2021-05-17T00:13:23+00:00", "updated": "2021-05-17T00:16:44+00:00", "last_event_date": null, "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": {"marketing": null}}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTQ44548K2TBCG1EWPZEDN/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTQ44548K2TBCG1EWPZEDN/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5VTQ44548K2TBCG1EWPZEDN/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTQ44548K2TBCG1EWPZEDN/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5VTQ44548K2TBCG1EWPZEDN/segments/"}}}, "updated": "2021-05-17T00:16:44+00:00"}, "emitted_at": 1679533540462} -{"stream": "profiles", "data": {"type": "profile", "id": "01F5VTX8KP49GGQ4BG77HZ9FRH", "attributes": {"email": "some.email.that.dont.exist3@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "Strange Name2", "last_name": "Funny Name2", "organization": null, "title": null, "image": null, "created": "2021-05-17T00:16:44+00:00", "updated": "2021-05-17T00:16:44+00:00", "last_event_date": null, "location": {"address1": null, "address2": null, "city": null, "country": null, "latitude": null, "longitude": null, "region": null, "zip": null, "timezone": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": {"marketing": null}}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTX8KP49GGQ4BG77HZ9FRH/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTX8KP49GGQ4BG77HZ9FRH/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5VTX8KP49GGQ4BG77HZ9FRH/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTX8KP49GGQ4BG77HZ9FRH/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5VTX8KP49GGQ4BG77HZ9FRH/segments/"}}}, "updated": "2021-05-17T00:16:44+00:00"}, "emitted_at": 1679533540463} -{"stream": "profiles", "data": {"type": "profile", "id": "01F5YBDQE9W7WDSH9KK398CAYX", "attributes": {"email": "some.email.that.dont.exist.{seed}@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": null, "last_name": null, "organization": null, "title": null, "image": null, "created": "2021-05-17T23:43:50+00:00", "updated": "2021-05-17T23:43:50+00:00", "last_event_date": "2021-05-17T23:43:28+00:00", "location": {"address1": null, "address2": null, "city": null, "country": null, "latitude": null, "longitude": null, "region": null, "zip": null, "timezone": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": {"marketing": null}}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBDQE9W7WDSH9KK398CAYX/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBDQE9W7WDSH9KK398CAYX/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBDQE9W7WDSH9KK398CAYX/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBDQE9W7WDSH9KK398CAYX/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBDQE9W7WDSH9KK398CAYX/segments/"}}}, "updated": "2021-05-17T23:43:50+00:00"}, "emitted_at": 1679533540463} -{"stream": "profiles", "data": {"type": "profile", "id": "01F5YBGKW1SQN453RM293PHH37", "attributes": {"email": "some.email.that.dont.exist.0@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 0", "last_name": "Last Name 0", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:24+00:00", "updated": "2021-05-17T23:45:25+00:00", "last_event_date": "2021-05-17T23:45:24+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": {"marketing": null}}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGKW1SQN453RM293PHH37/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGKW1SQN453RM293PHH37/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGKW1SQN453RM293PHH37/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGKW1SQN453RM293PHH37/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGKW1SQN453RM293PHH37/segments/"}}}, "updated": "2021-05-17T23:45:25+00:00"}, "emitted_at": 1679533540463} -{"stream": "profiles", "data": {"type": "profile", "id": "01F5YBGMTSM3B56W37QB9Q9CAD", "attributes": {"email": "some.email.that.dont.exist.3@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 3", "last_name": "Last Name 3", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:25+00:00", "updated": "2021-05-17T23:45:25+00:00", "last_event_date": "2021-05-17T23:45:25+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": {"marketing": null}}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGMTSM3B56W37QB9Q9CAD/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGMTSM3B56W37QB9Q9CAD/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGMTSM3B56W37QB9Q9CAD/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGMTSM3B56W37QB9Q9CAD/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGMTSM3B56W37QB9Q9CAD/segments/"}}}, "updated": "2021-05-17T23:45:25+00:00"}, "emitted_at": 1679533540463} -{"stream": "profiles", "data": {"type": "profile", "id": "01F5YBGM7J4YD4P6EYK5Q87BG4", "attributes": {"email": "some.email.that.dont.exist.1@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 1", "last_name": "Last Name 1", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:25+00:00", "updated": "2021-05-17T23:45:26+00:00", "last_event_date": "2021-05-17T23:45:24+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": {"marketing": null}}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGM7J4YD4P6EYK5Q87BG4/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGM7J4YD4P6EYK5Q87BG4/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGM7J4YD4P6EYK5Q87BG4/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGM7J4YD4P6EYK5Q87BG4/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGM7J4YD4P6EYK5Q87BG4/segments/"}}}, "updated": "2021-05-17T23:45:26+00:00"}, "emitted_at": 1679533540464} -{"stream": "profiles", "data": {"type": "profile", "id": "01F5YBGN65NTCBGTAR1Y7P5285", "attributes": {"email": "some.email.that.dont.exist.4@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 4", "last_name": "Last Name 4", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:26+00:00", "updated": "2021-05-17T23:45:26+00:00", "last_event_date": "2021-05-17T23:45:26+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": {"marketing": null}}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGN65NTCBGTAR1Y7P5285/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGN65NTCBGTAR1Y7P5285/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGN65NTCBGTAR1Y7P5285/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGN65NTCBGTAR1Y7P5285/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGN65NTCBGTAR1Y7P5285/segments/"}}}, "updated": "2021-05-17T23:45:26+00:00"}, "emitted_at": 1679533540464} -{"stream": "profiles", "data": {"type": "profile", "id": "01F5YBGNK6H122QRC1K96GXY8C", "attributes": {"email": "some.email.that.dont.exist.5@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 5", "last_name": "Last Name 5", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:26+00:00", "updated": "2021-05-17T23:45:26+00:00", "last_event_date": "2021-05-17T23:45:26+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": {"marketing": null}}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGNK6H122QRC1K96GXY8C/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGNK6H122QRC1K96GXY8C/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGNK6H122QRC1K96GXY8C/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGNK6H122QRC1K96GXY8C/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGNK6H122QRC1K96GXY8C/segments/"}}}, "updated": "2021-05-17T23:45:26+00:00"}, "emitted_at": 1679533540464} -{"stream": "profiles", "data": {"type": "profile", "id": "01F5YBGP0P02E9Q64KF26VB2MH", "attributes": {"email": "some.email.that.dont.exist.6@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 6", "last_name": "Last Name 6", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:27+00:00", "updated": "2021-05-17T23:45:27+00:00", "last_event_date": "2021-05-17T23:45:26+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": {"marketing": null}}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGP0P02E9Q64KF26VB2MH/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGP0P02E9Q64KF26VB2MH/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGP0P02E9Q64KF26VB2MH/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGP0P02E9Q64KF26VB2MH/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGP0P02E9Q64KF26VB2MH/segments/"}}}, "updated": "2021-05-17T23:45:27+00:00"}, "emitted_at": 1679533540464} -{"stream": "profiles", "data": {"type": "profile", "id": "01F5YBGPSXF1N23RBJZ947R1N1", "attributes": {"email": "some.email.that.dont.exist.8@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 8", "last_name": "Last Name 8", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:27+00:00", "updated": "2021-05-17T23:45:27+00:00", "last_event_date": "2021-05-17T23:45:27+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [{"reason": "USER_SUPPRESSED", "timestamp": "2021-05-18T01:29:51+00:00"}], "list_suppressions": []}}, "sms": {"marketing": null}}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/segments/"}}}, "updated": "2021-05-17T23:45:27+00:00"}, "emitted_at": 1679533540465} -{"stream": "profiles", "data": {"type": "profile", "id": "01F5YBGPCQESZDRKGW3DB1WPZ0", "attributes": {"email": "some.email.that.dont.exist.7@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 7", "last_name": "Last Name 7", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:27+00:00", "updated": "2021-05-17T23:45:30+00:00", "last_event_date": "2021-05-17T23:45:27+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": {"marketing": null}}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGPCQESZDRKGW3DB1WPZ0/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGPCQESZDRKGW3DB1WPZ0/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGPCQESZDRKGW3DB1WPZ0/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGPCQESZDRKGW3DB1WPZ0/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGPCQESZDRKGW3DB1WPZ0/segments/"}}}, "updated": "2021-05-17T23:45:30+00:00"}, "emitted_at": 1679533540465} -{"stream": "profiles", "data": {"type": "profile", "id": "01F5YBGQ6X21SSWPGRDK9QK97C", "attributes": {"email": "some.email.that.dont.exist.9@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 9", "last_name": "Last Name 9", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:28+00:00", "updated": "2021-05-17T23:45:30+00:00", "last_event_date": "2021-05-17T23:45:28+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [{"reason": "USER_SUPPRESSED", "timestamp": "2021-05-18T01:20:01+00:00"}], "list_suppressions": []}}, "sms": {"marketing": null}}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/segments/"}}}, "updated": "2021-05-17T23:45:30+00:00"}, "emitted_at": 1679533540465} -{"stream": "profiles", "data": {"type": "profile", "id": "01F5YBGMK62AJR0955G7NW6EP7", "attributes": {"email": "some.email.that.dont.exist.2@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 2", "last_name": "Last Name 2", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:25+00:00", "updated": "2021-05-17T23:45:38+00:00", "last_event_date": "2021-05-17T23:45:25+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": {"marketing": null}}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGMK62AJR0955G7NW6EP7/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGMK62AJR0955G7NW6EP7/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGMK62AJR0955G7NW6EP7/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGMK62AJR0955G7NW6EP7/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGMK62AJR0955G7NW6EP7/segments/"}}}, "updated": "2021-05-17T23:45:38+00:00"}, "emitted_at": 1679533540465} +{"stream":"campaigns","data":{"type":"campaign","id":"T4hgvQ","attributes":{"name":"Email Campaign 2021-05-12 16:45:46","status":"Draft","archived":false,"channel":"email","audiences":{"included":[],"excluded":[]},"send_options":{"use_smart_sending":true,"ignore_unsubscribes":false},"message":"01GF99SBS5Q8NM5YXRQDFRS5R2","tracking_options":{"is_tracking_opens":true,"is_tracking_clicks":true,"is_add_utm":false,"utm_params":[]},"send_strategy":{"method":"immediate","options_static":null,"options_throttled":null,"options_sto":null},"created_at":"2021-05-12T20:45:47+00:00","scheduled_at":null,"updated_at":"2021-05-12T20:45:47+00:00","send_time":null},"relationships":{"tags":{"links":{"self":"https://a.klaviyo.com/api/campaigns/T4hgvQ/relationships/tags/","related":"https://a.klaviyo.com/api/campaigns/T4hgvQ/tags/"}}},"links":{"self":"https://a.klaviyo.com/api/campaigns/T4hgvQ/"},"updated_at":"2021-05-12T20:45:47+00:00"},"emitted_at":1701797442924} +{"stream":"campaigns","data":{"type":"campaign","id":"VFaYVy","attributes":{"name":"Email Campaign 2021-05-16 19:17:45","status":"Sent","archived":false,"channel":"email","audiences":{"included":["RnsiHB","TaSce6"],"excluded":["Ukh37W"]},"send_options":{"use_smart_sending":true,"ignore_unsubscribes":false},"message":"01GF9SD7YH28Q9CW5199E0TWYM","tracking_options":{"is_tracking_opens":true,"is_tracking_clicks":true,"is_add_utm":true,"utm_params":[{"name":"utm_source","value":"{segment}"},{"name":"utm_medium","value":"email"},{"name":"utm_campaign","value":"{name} ({id})"},{"name":"test_utm_param","value":"{customer_external_id}"}]},"send_strategy":{"method":"throttled","options_static":null,"options_throttled":{"datetime":"2021-05-26T23:30:00+00:00","throttle_percentage":20},"options_sto":null},"created_at":"2021-05-16T23:17:45+00:00","scheduled_at":"2021-05-16T23:21:19+00:00","updated_at":"2021-05-26T23:30:13+00:00","send_time":"2021-05-26T23:30:00+00:00"},"relationships":{"tags":{"links":{"self":"https://a.klaviyo.com/api/campaigns/VFaYVy/relationships/tags/","related":"https://a.klaviyo.com/api/campaigns/VFaYVy/tags/"}}},"links":{"self":"https://a.klaviyo.com/api/campaigns/VFaYVy/"},"updated_at":"2021-05-26T23:30:13+00:00"},"emitted_at":1701797442925} +{"stream":"campaigns","data":{"type":"campaign","id":"01HE82EVNPCB3YP0TZYNXAPJKQ","attributes":{"name":"Email Campaign - Nov 2 2023 3:09 PM","status":"Draft","archived":false,"channel":"email","audiences":{"included":[],"excluded":[]},"send_options":{"use_smart_sending":true,"ignore_unsubscribes":false},"message":"01HE82EVP0TMKNH6RGFE4ED0P6","tracking_options":{"is_tracking_opens":true,"is_tracking_clicks":true,"is_add_utm":false,"utm_params":[]},"send_strategy":{"method":"static","options_static":{"datetime":"2023-11-01T22:00:00+00:00","is_local":false,"send_past_recipients_immediately":null},"options_throttled":null,"options_sto":null},"created_at":"2023-11-02T13:09:45.276362+00:00","scheduled_at":null,"updated_at":"2023-11-02T13:09:45.276403+00:00","send_time":null},"relationships":{"tags":{"links":{"self":"https://a.klaviyo.com/api/campaigns/01HE82EVNPCB3YP0TZYNXAPJKQ/relationships/tags/","related":"https://a.klaviyo.com/api/campaigns/01HE82EVNPCB3YP0TZYNXAPJKQ/tags/"}}},"links":{"self":"https://a.klaviyo.com/api/campaigns/01HE82EVNPCB3YP0TZYNXAPJKQ/"},"updated_at":"2023-11-02T13:09:45.276403+00:00"},"emitted_at":1701797442926} +{"stream":"campaigns","data":{"type":"campaign","id":"01HE2PASG4GSV564GPXCAW8TFJ","attributes":{"name":"Email Campaign Archived - Nov 01 2023 12:55 PM","status":"Sent","archived":true,"channel":"email","audiences":{"included":["RnsiHB","UXi5Jz"],"excluded":[]},"send_options":{"use_smart_sending":true,"ignore_unsubscribes":false},"message":"01HE2PASMWR4TBTD7JD79N675K","tracking_options":{"is_tracking_opens":true,"is_tracking_clicks":true,"is_add_utm":false,"utm_params":[]},"send_strategy":{"method":"static","options_static":{"datetime":"2023-10-31T11:01:53+00:00","is_local":false,"send_past_recipients_immediately":null},"options_throttled":null,"options_sto":null},"created_at":"2023-10-31T11:01:36.900676+00:00","scheduled_at":"2023-10-31T11:01:53.122496+00:00","updated_at":"2023-10-31T11:02:12.888185+00:00","send_time":"2023-10-31T11:01:53+00:00"},"relationships":{"tags":{"links":{"self":"https://a.klaviyo.com/api/campaigns/01HE2PASG4GSV564GPXCAW8TFJ/relationships/tags/","related":"https://a.klaviyo.com/api/campaigns/01HE2PASG4GSV564GPXCAW8TFJ/tags/"}}},"links":{"self":"https://a.klaviyo.com/api/campaigns/01HE2PASG4GSV564GPXCAW8TFJ/"},"updated_at":"2023-10-31T11:02:12.888185+00:00"},"emitted_at":1701797443275} +{"stream":"campaigns","data":{"type":"campaign","id":"01HEHY2911JYEGQ4EMAWRWMGKE","attributes":{"name":"Email Campaign Archived 2 - Nov 6 2023 11:05 AM","status":"Sent","archived":true,"channel":"email","audiences":{"included":["RnsiHB"],"excluded":[]},"send_options":{"use_smart_sending":true,"ignore_unsubscribes":false},"message":"01HEHY291CNK9TAWBSKJ28P032","tracking_options":{"is_tracking_opens":true,"is_tracking_clicks":true,"is_add_utm":false,"utm_params":[]},"send_strategy":{"method":"static","options_static":{"datetime":"2023-11-06T09:06:34+00:00","is_local":false,"send_past_recipients_immediately":null},"options_throttled":null,"options_sto":null},"created_at":"2023-11-06T09:05:22.984339+00:00","scheduled_at":"2023-11-06T09:06:34.279041+00:00","updated_at":"2023-11-06T09:07:01.148389+00:00","send_time":"2023-11-06T09:06:34+00:00"},"relationships":{"tags":{"links":{"self":"https://a.klaviyo.com/api/campaigns/01HEHY2911JYEGQ4EMAWRWMGKE/relationships/tags/","related":"https://a.klaviyo.com/api/campaigns/01HEHY2911JYEGQ4EMAWRWMGKE/tags/"}}},"links":{"self":"https://a.klaviyo.com/api/campaigns/01HEHY2911JYEGQ4EMAWRWMGKE/"},"updated_at":"2023-11-06T09:07:01.148389+00:00"},"emitted_at":1701797443276} +{"stream": "events", "data": {"type": "event", "id": "3qvdbYg3", "attributes": {"timestamp": 1621295008, "event_properties": {"$event_id": "1621295008"}, "datetime": "2021-05-17 23:43:28+00:00", "uuid": "adc8d000-b769-11eb-8001-28a6687f81c3"}, "relationships": {"profile": {"data": {"type": "profile", "id": "01F5YBDQE9W7WDSH9KK398CAYX"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdbYg3/relationships/profile/", "related": "https://a.klaviyo.com/api/events/3qvdbYg3/profile/"}}, "metric": {"data": {"type": "metric", "id": "VFFb4u"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdbYg3/relationships/metric/", "related": "https://a.klaviyo.com/api/events/3qvdbYg3/metric/"}}}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdbYg3/"}, "datetime": "2021-05-17 23:43:28+00:00"}, "emitted_at": 1699980660456} +{"stream": "events", "data": {"type": "event", "id": "3qvdgpzF", "attributes": {"timestamp": 1621295124, "event_properties": {"$event_id": "1621295124"}, "datetime": "2021-05-17 23:45:24+00:00", "uuid": "f2ed0200-b769-11eb-8001-76152f6b1c82"}, "relationships": {"profile": {"data": {"type": "profile", "id": "01F5YBGKW1SQN453RM293PHH37"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgpzF/relationships/profile/", "related": "https://a.klaviyo.com/api/events/3qvdgpzF/profile/"}}, "metric": {"data": {"type": "metric", "id": "VFFb4u"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgpzF/relationships/metric/", "related": "https://a.klaviyo.com/api/events/3qvdgpzF/metric/"}}}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgpzF/"}, "datetime": "2021-05-17 23:45:24+00:00"}, "emitted_at": 1699980660457} +{"stream": "events", "data": {"type": "event", "id": "3qvdgr5Z", "attributes": {"timestamp": 1621295124, "event_properties": {"$event_id": "1621295124"}, "datetime": "2021-05-17 23:45:24+00:00", "uuid": "f2ed0200-b769-11eb-8001-b642ddab48ad"}, "relationships": {"profile": {"data": {"type": "profile", "id": "01F5YBGM7J4YD4P6EYK5Q87BG4"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgr5Z/relationships/profile/", "related": "https://a.klaviyo.com/api/events/3qvdgr5Z/profile/"}}, "metric": {"data": {"type": "metric", "id": "VFFb4u"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgr5Z/relationships/metric/", "related": "https://a.klaviyo.com/api/events/3qvdgr5Z/metric/"}}}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgr5Z/"}, "datetime": "2021-05-17 23:45:24+00:00"}, "emitted_at": 1699980660457} +{"stream": "events", "data": {"type": "event", "id": "3qvdgBgK", "attributes": {"timestamp": 1621295124, "event_properties": {"$event_id": "1621295124"}, "datetime": "2021-05-17 23:45:24+00:00", "uuid": "f2ed0200-b769-11eb-8001-2006a2b2b6e7"}, "relationships": {"profile": {"data": {"type": "profile", "id": "01F5YBGMK62AJR0955G7NW6EP7"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgBgK/relationships/profile/", "related": "https://a.klaviyo.com/api/events/3qvdgBgK/profile/"}}, "metric": {"data": {"type": "metric", "id": "VFFb4u"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgBgK/relationships/metric/", "related": "https://a.klaviyo.com/api/events/3qvdgBgK/metric/"}}}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgBgK/"}, "datetime": "2021-05-17 23:45:24+00:00"}, "emitted_at": 1699980660457} +{"stream": "events", "data": {"type": "event", "id": "3qvdgs9P", "attributes": {"timestamp": 1621295125, "event_properties": {"$event_id": "1621295125"}, "datetime": "2021-05-17 23:45:25+00:00", "uuid": "f3859880-b769-11eb-8001-f6a061424b91"}, "relationships": {"profile": {"data": {"type": "profile", "id": "01F5YBGMK62AJR0955G7NW6EP7"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgs9P/relationships/profile/", "related": "https://a.klaviyo.com/api/events/3qvdgs9P/profile/"}}, "metric": {"data": {"type": "metric", "id": "VFFb4u"}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgs9P/relationships/metric/", "related": "https://a.klaviyo.com/api/events/3qvdgs9P/metric/"}}}, "links": {"self": "https://a.klaviyo.com/api/events/3qvdgs9P/"}, "datetime": "2021-05-17 23:45:25+00:00"}, "emitted_at": 1699980660457} +{"stream": "global_exclusions", "data": {"type": "profile", "id": "01F5YBGPSXF1N23RBJZ947R1N1", "attributes": {"email": "some.email.that.dont.exist.8@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 8", "last_name": "Last Name 8", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:27+00:00", "updated": "2021-05-17T23:45:27+00:00", "last_event_date": "2021-05-17T23:45:27+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null, "ip": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [{"reason": "USER_SUPPRESSED", "timestamp": "2021-05-18T01:29:51+00:00"}], "list_suppressions": []}}, "sms": null}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/segments/"}}}, "updated": "2021-05-17T23:45:27+00:00"}, "emitted_at": 1663367161413} +{"stream": "global_exclusions", "data": {"type": "profile", "id": "01F5YBGQ6X21SSWPGRDK9QK97C", "attributes": {"email": "some.email.that.dont.exist.9@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 9", "last_name": "Last Name 9", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:28+00:00", "updated": "2021-05-17T23:45:30+00:00", "last_event_date": "2021-05-17T23:45:28+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null, "ip": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [{"reason": "USER_SUPPRESSED", "timestamp": "2021-05-18T01:20:01+00:00"}], "list_suppressions": []}}, "sms": null}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/segments/"}}}, "updated": "2021-05-17T23:45:30+00:00"}, "emitted_at": 1663367161413} +{"stream": "lists", "data": {"type": "list", "id": "RnsiHB", "attributes": {"name": "Newsletter", "created": "2021-03-31T10:50:36+00:00", "updated": "2021-03-31T10:50:36+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/RnsiHB/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/RnsiHB/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/RnsiHB/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/RnsiHB/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/RnsiHB/"}, "updated": "2021-03-31T10:50:36+00:00"}, "emitted_at": 1698942733516} +{"stream": "lists", "data": {"type": "list", "id": "TaSce6", "attributes": {"name": "Preview List", "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/TaSce6/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/TaSce6/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/TaSce6/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/TaSce6/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/TaSce6/"}, "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1698942733517} +{"stream": "lists", "data": {"type": "list", "id": "R2p3ry", "attributes": {"name": "Test2", "created": "2021-11-16T14:24:04+00:00", "updated": "2021-11-16T14:24:04+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/R2p3ry/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/R2p3ry/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/R2p3ry/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/R2p3ry/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/R2p3ry/"}, "updated": "2021-11-16T14:24:04+00:00"}, "emitted_at": 1698942733517} +{"stream": "lists", "data": {"type": "list", "id": "S7aBY2", "attributes": {"name": "Test1", "created": "2021-11-16T14:24:07+00:00", "updated": "2021-11-16T14:24:07+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/S7aBY2/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/S7aBY2/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/S7aBY2/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/S7aBY2/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/S7aBY2/"}, "updated": "2021-11-16T14:24:07+00:00"}, "emitted_at": 1698942733518} +{"stream": "lists", "data": {"type": "list", "id": "XpP2a5", "attributes": {"name": "Test4", "created": "2021-11-16T14:24:10+00:00", "updated": "2021-11-16T14:24:10+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/XpP2a5/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/XpP2a5/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/XpP2a5/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/XpP2a5/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/XpP2a5/"}, "updated": "2021-11-16T14:24:10+00:00"}, "emitted_at": 1698942733518} +{"stream": "lists", "data": {"type": "list", "id": "TDGJsj", "attributes": {"name": "Test3", "created": "2021-11-16T14:24:14+00:00", "updated": "2021-11-16T14:24:14+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/TDGJsj/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/TDGJsj/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/TDGJsj/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/TDGJsj/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/TDGJsj/"}, "updated": "2021-11-16T14:24:14+00:00"}, "emitted_at": 1698942733518} +{"stream": "lists", "data": {"type": "list", "id": "WBxsQE", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:17+00:00", "updated": "2021-11-16T14:24:17+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/WBxsQE/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/WBxsQE/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/WBxsQE/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/WBxsQE/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/WBxsQE/"}, "updated": "2021-11-16T14:24:17+00:00"}, "emitted_at": 1698942733518} +{"stream": "lists", "data": {"type": "list", "id": "VmvmBq", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:18+00:00", "updated": "2021-11-16T14:24:18+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/VmvmBq/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/VmvmBq/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/VmvmBq/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/VmvmBq/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/VmvmBq/"}, "updated": "2021-11-16T14:24:18+00:00"}, "emitted_at": 1698942733519} +{"stream": "lists", "data": {"type": "list", "id": "XGj3p8", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:20+00:00", "updated": "2021-11-16T14:24:20+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/XGj3p8/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/XGj3p8/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/XGj3p8/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/XGj3p8/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/XGj3p8/"}, "updated": "2021-11-16T14:24:20+00:00"}, "emitted_at": 1698942733520} +{"stream": "lists", "data": {"type": "list", "id": "R4ZhCr", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:21+00:00", "updated": "2021-11-16T14:24:21+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/R4ZhCr/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/R4ZhCr/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/R4ZhCr/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/R4ZhCr/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/R4ZhCr/"}, "updated": "2021-11-16T14:24:21+00:00"}, "emitted_at": 1698942733520} +{"stream": "lists", "data": {"type": "list", "id": "Seq8wh", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:22+00:00", "updated": "2021-11-16T14:24:22+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/Seq8wh/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/Seq8wh/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/Seq8wh/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/Seq8wh/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/Seq8wh/"}, "updated": "2021-11-16T14:24:22+00:00"}, "emitted_at": 1698942733520} +{"stream": "lists", "data": {"type": "list", "id": "TpNXq9", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:23+00:00", "updated": "2021-11-16T14:24:23+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/TpNXq9/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/TpNXq9/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/TpNXq9/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/TpNXq9/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/TpNXq9/"}, "updated": "2021-11-16T14:24:23+00:00"}, "emitted_at": 1698942733521} +{"stream": "lists", "data": {"type": "list", "id": "UzdNhZ", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:24+00:00", "updated": "2021-11-16T14:24:24+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/UzdNhZ/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/UzdNhZ/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/UzdNhZ/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/UzdNhZ/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/UzdNhZ/"}, "updated": "2021-11-16T14:24:24+00:00"}, "emitted_at": 1698942733521} +{"stream": "lists", "data": {"type": "list", "id": "TWcKFn", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:25+00:00", "updated": "2021-11-16T14:24:25+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/TWcKFn/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/TWcKFn/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/TWcKFn/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/TWcKFn/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/TWcKFn/"}, "updated": "2021-11-16T14:24:25+00:00"}, "emitted_at": 1698942733521} +{"stream": "lists", "data": {"type": "list", "id": "Ya5ziX", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:25+00:00", "updated": "2021-11-16T14:24:25+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/Ya5ziX/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/Ya5ziX/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/Ya5ziX/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/Ya5ziX/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/Ya5ziX/"}, "updated": "2021-11-16T14:24:25+00:00"}, "emitted_at": 1698942733521} +{"stream": "lists", "data": {"type": "list", "id": "RwKPyg", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:26+00:00", "updated": "2021-11-16T14:24:26+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/RwKPyg/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/RwKPyg/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/RwKPyg/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/RwKPyg/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/RwKPyg/"}, "updated": "2021-11-16T14:24:26+00:00"}, "emitted_at": 1698942733522} +{"stream": "lists", "data": {"type": "list", "id": "VJCDbR", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:26+00:00", "updated": "2021-11-16T14:24:26+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/VJCDbR/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/VJCDbR/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/VJCDbR/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/VJCDbR/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/VJCDbR/"}, "updated": "2021-11-16T14:24:26+00:00"}, "emitted_at": 1698942733522} +{"stream": "lists", "data": {"type": "list", "id": "TjbH4K", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:27+00:00", "updated": "2021-11-16T14:24:27+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/TjbH4K/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/TjbH4K/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/TjbH4K/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/TjbH4K/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/TjbH4K/"}, "updated": "2021-11-16T14:24:27+00:00"}, "emitted_at": 1698942733522} +{"stream": "lists", "data": {"type": "list", "id": "VDZnQt", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:28+00:00", "updated": "2021-11-16T14:24:28+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/VDZnQt/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/VDZnQt/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/VDZnQt/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/VDZnQt/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/VDZnQt/"}, "updated": "2021-11-16T14:24:28+00:00"}, "emitted_at": 1698942733523} +{"stream": "lists", "data": {"type": "list", "id": "WJLXnV", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:28+00:00", "updated": "2021-11-16T14:24:28+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/WJLXnV/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/WJLXnV/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/WJLXnV/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/WJLXnV/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/WJLXnV/"}, "updated": "2021-11-16T14:24:28+00:00"}, "emitted_at": 1698942733523} +{"stream": "lists", "data": {"type": "list", "id": "XUbNgM", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:29+00:00", "updated": "2021-11-16T14:24:29+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/XUbNgM/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/XUbNgM/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/XUbNgM/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/XUbNgM/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/XUbNgM/"}, "updated": "2021-11-16T14:24:29+00:00"}, "emitted_at": 1698942733523} +{"stream": "lists", "data": {"type": "list", "id": "RgS4w6", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:30+00:00", "updated": "2021-11-16T14:24:30+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/RgS4w6/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/RgS4w6/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/RgS4w6/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/RgS4w6/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/RgS4w6/"}, "updated": "2021-11-16T14:24:30+00:00"}, "emitted_at": 1698942733524} +{"stream": "lists", "data": {"type": "list", "id": "UeGLUr", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:30+00:00", "updated": "2021-11-16T14:24:30+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/UeGLUr/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/UeGLUr/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/UeGLUr/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/UeGLUr/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/UeGLUr/"}, "updated": "2021-11-16T14:24:30+00:00"}, "emitted_at": 1698942733524} +{"stream": "lists", "data": {"type": "list", "id": "SYEFFb", "attributes": {"name": "Test_5", "created": "2021-11-16T14:24:31+00:00", "updated": "2021-11-16T14:24:31+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/SYEFFb/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/SYEFFb/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/SYEFFb/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/SYEFFb/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/SYEFFb/"}, "updated": "2021-11-16T14:24:31+00:00"}, "emitted_at": 1698942733524} +{"stream": "lists", "data": {"type": "list", "id": "SmDD4y", "attributes": {"name": "Test5__x", "created": "2021-11-16T14:24:32+00:00", "updated": "2021-11-16T14:24:32+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/SmDD4y/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/SmDD4y/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/SmDD4y/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/SmDD4y/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/SmDD4y/"}, "updated": "2021-11-16T14:24:32+00:00"}, "emitted_at": 1698942733524} +{"stream": "lists", "data": {"type": "list", "id": "X7UeXn", "attributes": {"name": "Test5___", "created": "2021-11-16T14:24:34+00:00", "updated": "2021-11-16T14:24:34+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/X7UeXn/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/X7UeXn/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/X7UeXn/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/X7UeXn/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/X7UeXn/"}, "updated": "2021-11-16T14:24:34+00:00"}, "emitted_at": 1698942733525} +{"stream": "lists", "data": {"type": "list", "id": "RPfQMj", "attributes": {"name": "Test5", "created": "2021-11-16T14:24:31+00:00", "updated": "2021-11-16T15:01:15+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/RPfQMj/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/RPfQMj/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/RPfQMj/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/RPfQMj/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/RPfQMj/"}, "updated": "2021-11-16T15:01:15+00:00"}, "emitted_at": 1698942733525} +{"stream": "lists", "data": {"type": "list", "id": "S8nmQ9", "attributes": {"name": "Test AAAB", "created": "2021-11-16T15:02:51+00:00", "updated": "2021-11-16T15:02:51+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/S8nmQ9/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/S8nmQ9/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/S8nmQ9/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/S8nmQ9/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/S8nmQ9/"}, "updated": "2021-11-16T15:02:51+00:00"}, "emitted_at": 1698942733525} +{"stream": "lists", "data": {"type": "list", "id": "SBYgiK", "attributes": {"name": "SMS Subscribers", "created": "2022-05-31T06:52:26+00:00", "updated": "2022-05-31T06:52:26+00:00"}, "relationships": {"profiles": {"links": {"self": "https://a.klaviyo.com/api/lists/SBYgiK/relationships/profiles/", "related": "https://a.klaviyo.com/api/lists/SBYgiK/profiles/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/lists/SBYgiK/relationships/tags/", "related": "https://a.klaviyo.com/api/lists/SBYgiK/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/lists/SBYgiK/"}, "updated": "2022-05-31T06:52:26+00:00"}, "emitted_at": 1698942733525} +{"stream": "email_templates", "data": {"type": "template", "id": "RdbN2P", "attributes": {"name": "Newsletter #1 (Images & Text)", "editor_type": "SYSTEM_DRAGGABLE", "html": "\n\n\n\n\n\n\n\n\n\n\n\n\n\n
      \n
      \n\n\n\n\n\n\n
      \n\n
      \n\n\n\n\n\n\n
      \n\n
      \n\n\n\n\n\n\n
      \n\n
      \n\n
      \n\n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n
      \n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n

      \n

      \n\n
      \n
      \n
      \n
      \n\n
      \n\n\n
      \n\n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n

      This template starts with images.

      \n
      \n
      \n
      \n
      \n\n
      \n\n\n
      \n\n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n\n
      \n\n
      \n
      \n
      \n
      \n
      \n
      \n\n\n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n\n
      \n\n
      \n
      \n
      \n
      \n
      \n
      \n\n\n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n\n
      \n\n
      \n
      \n
      \n
      \n
      \n
      \n\n
      \n\n\n
      \n\n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n

      Everyone loves pictures. They're more engaging that text by itself and the images in this template will neatly stack on mobile devices for the best viewing experience.

      \n

      Use the text area below to add additional content or add more images to create a larger image gallery. You can drag blocks from the left sidebar to add content to your template. You can customize this colors, fonts and styling of this template to match your brand by clicking the \"Styles\" button to the left.

      \n

      Happy emailing!

      \n

      The Klaviyo Team

      \n
      \n
      \n
      \n
      \n\n
      \n\n\n
      \n\n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n
      \n\n
      \n\n
      \n\n\"Facebook\"\n\n
      \n\n
      \n
      \n\n
      \n\n\"Twitter\"\n\n
      \n\n
      \n
      \n\n
      \n\n\"LinkedIn\"\n\n
      \n\n
      \n\n
      \n
      \n
      \n
      \n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n
      No longer want to receive these emails? {% unsubscribe %}.
      {{ organization.name }} {{ organization.full_address }}
      \n
      \n
      \n
      \n
      \n\n
      \n\n
      \n\n
      \n
      \n\n
      \n
      \n\n
      \n\n
      \n\n\n\n\n\n\n
      \n\n
      \n\n\n\n\n\n\n
      \n\n\n\n\n\n\n
      \n\n\"Powered\n\n
      \n
      \n
      \n\n
      \n
      \n\n
      \n
      \n\n", "text": null, "created": "2021-03-31T10:50:37+00:00", "updated": "2022-05-31T06:36:45+00:00"}, "links": {"self": "https://a.klaviyo.com/api/templates/RdbN2P/"}, "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1698938827838} +{"stream": "metrics", "data": {"type": "metric", "id": "RUQ6YQ", "attributes": {"name": "Active on Site", "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00", "integration": {"object": "integration", "id": "7FtS4J", "name": "API", "category": "API"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/RUQ6YQ/"}, "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1698943412889} +{"stream": "metrics", "data": {"type": "metric", "id": "RhP4nd", "attributes": {"name": "Dropped Email", "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/RhP4nd/"}, "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1698943412891} +{"stream": "metrics", "data": {"type": "metric", "id": "SxR9Bt", "attributes": {"name": "Clicked Email", "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/SxR9Bt/"}, "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1698943412891} +{"stream": "metrics", "data": {"type": "metric", "id": "THfYvj", "attributes": {"name": "Marked Email as Spam", "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/THfYvj/"}, "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1698943412891} +{"stream": "metrics", "data": {"type": "metric", "id": "VvFRZN", "attributes": {"name": "Unsubscribed", "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/VvFRZN/"}, "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1698943412891} +{"stream": "metrics", "data": {"type": "metric", "id": "WKHXf4", "attributes": {"name": "Received Email", "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/WKHXf4/"}, "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1698943412892} +{"stream": "metrics", "data": {"type": "metric", "id": "Xi7Kwh", "attributes": {"name": "Bounced Email", "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/Xi7Kwh/"}, "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1698943412892} +{"stream": "metrics", "data": {"type": "metric", "id": "Yy9QKx", "attributes": {"name": "Opened Email", "created": "2021-03-31T10:50:37+00:00", "updated": "2021-03-31T10:50:37+00:00", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/Yy9QKx/"}, "updated": "2021-03-31T10:50:37+00:00"}, "emitted_at": 1698943412892} +{"stream": "metrics", "data": {"type": "metric", "id": "VFFb4u", "attributes": {"name": "Clicked Email", "created": "2021-05-17T23:43:50+00:00", "updated": "2021-05-17T23:43:50+00:00", "integration": {"object": "integration", "id": "7FtS4J", "name": "API", "category": "API"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/VFFb4u/"}, "updated": "2021-05-17T23:43:50+00:00"}, "emitted_at": 1698943412892} +{"stream": "metrics", "data": {"type": "metric", "id": "Tp8t7d", "attributes": {"name": "Subscribed to List", "created": "2021-11-16T15:05:22+00:00", "updated": "2021-11-16T15:05:22+00:00", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/Tp8t7d/"}, "updated": "2021-11-16T15:05:22+00:00"}, "emitted_at": 1698943412893} +{"stream": "metrics", "data": {"type": "metric", "id": "R2WpFy", "attributes": {"name": "Refunded Order", "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/R2WpFy/"}, "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1698943412893} +{"stream": "metrics", "data": {"type": "metric", "id": "RDXsib", "attributes": {"name": "Ordered Product", "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/RDXsib/"}, "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1698943412893} +{"stream": "metrics", "data": {"type": "metric", "id": "SPnhc3", "attributes": {"name": "Checkout Started", "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/SPnhc3/"}, "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1698943412893} +{"stream": "metrics", "data": {"type": "metric", "id": "TspjNE", "attributes": {"name": "Placed Order", "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/TspjNE/"}, "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1698943412894} +{"stream": "metrics", "data": {"type": "metric", "id": "UBNaGw", "attributes": {"name": "Subscribed to Back in Stock", "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/UBNaGw/"}, "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1698943412894} +{"stream": "metrics", "data": {"type": "metric", "id": "VePdj9", "attributes": {"name": "Cancelled Order", "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/VePdj9/"}, "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1698943412894} +{"stream": "metrics", "data": {"type": "metric", "id": "X3f6PC", "attributes": {"name": "Fulfilled Order", "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/X3f6PC/"}, "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1698943412894} +{"stream": "metrics", "data": {"type": "metric", "id": "YcDVHu", "attributes": {"name": "Viewed Product", "created": "2022-05-31T06:36:45+00:00", "updated": "2022-05-31T06:36:45+00:00", "integration": {"object": "integration", "id": "7FtS4J", "name": "API", "category": "API"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/YcDVHu/"}, "updated": "2022-05-31T06:36:45+00:00"}, "emitted_at": 1698943412895} +{"stream": "metrics", "data": {"type": "metric", "id": "RcjEmN", "attributes": {"name": "Fulfilled Partial Order", "created": "2022-05-31T06:45:47+00:00", "updated": "2022-05-31T06:45:47+00:00", "integration": {"object": "integration", "id": "0eMvjm", "name": "Shopify", "category": "eCommerce"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/RcjEmN/"}, "updated": "2022-05-31T06:45:47+00:00"}, "emitted_at": 1698943412895} +{"stream": "metrics", "data": {"type": "metric", "id": "RszrqT", "attributes": {"name": "Received Automated Response SMS", "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/RszrqT/"}, "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1698943412895} +{"stream": "metrics", "data": {"type": "metric", "id": "S5Au3w", "attributes": {"name": "Failed to Deliver SMS", "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/S5Au3w/"}, "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1698943412895} +{"stream": "metrics", "data": {"type": "metric", "id": "TS2mxZ", "attributes": {"name": "Unsubscribed from SMS Marketing", "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/TS2mxZ/"}, "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1698943412896} +{"stream": "metrics", "data": {"type": "metric", "id": "TeZiVn", "attributes": {"name": "Failed to deliver Automated Response SMS", "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/TeZiVn/"}, "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1698943412896} +{"stream": "metrics", "data": {"type": "metric", "id": "VEsf4u", "attributes": {"name": "Subscribed to SMS Marketing", "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/VEsf4u/"}, "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1698943412896} +{"stream": "metrics", "data": {"type": "metric", "id": "WhthF7", "attributes": {"name": "Received SMS", "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/WhthF7/"}, "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1698943412896} +{"stream": "metrics", "data": {"type": "metric", "id": "XsS8yX", "attributes": {"name": "Sent SMS", "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/XsS8yX/"}, "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1698943412897} +{"stream": "metrics", "data": {"type": "metric", "id": "Y5TbbA", "attributes": {"name": "Clicked SMS", "created": "2022-05-31T06:52:24+00:00", "updated": "2022-05-31T06:52:24+00:00", "integration": {"object": "integration", "id": "0rG4eQ", "name": "Klaviyo", "category": "Internal"}}, "links": {"self": "https://a.klaviyo.com/api/metrics/Y5TbbA/"}, "updated": "2022-05-31T06:52:24+00:00"}, "emitted_at": 1698943412897} +{"stream": "profiles", "data": {"type": "profile", "id": "01F5VTP8THZD8CGS2AKNE63370", "attributes": {"email": "some.email.that.dont.exist@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name", "last_name": "Last Name", "organization": null, "title": null, "image": null, "created": "2021-05-17T00:12:55+00:00", "updated": "2021-05-17T00:12:55+00:00", "last_event_date": null, "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null, "ip": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": null}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTP8THZD8CGS2AKNE63370/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTP8THZD8CGS2AKNE63370/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5VTP8THZD8CGS2AKNE63370/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTP8THZD8CGS2AKNE63370/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5VTP8THZD8CGS2AKNE63370/segments/"}}}, "updated": "2021-05-17T00:12:55+00:00"}, "emitted_at": 1679533540462} +{"stream": "profiles", "data": {"type": "profile", "id": "01F5VTQ44548K2TBCG1EWPZEDN", "attributes": {"email": "some.email.that.dont.exist2@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "Strange Name1", "last_name": "Funny Name1", "organization": null, "title": null, "image": null, "created": "2021-05-17T00:13:23+00:00", "updated": "2021-05-17T00:16:44+00:00", "last_event_date": null, "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null, "ip": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": null}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTQ44548K2TBCG1EWPZEDN/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTQ44548K2TBCG1EWPZEDN/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5VTQ44548K2TBCG1EWPZEDN/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTQ44548K2TBCG1EWPZEDN/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5VTQ44548K2TBCG1EWPZEDN/segments/"}}}, "updated": "2021-05-17T00:16:44+00:00"}, "emitted_at": 1679533540462} +{"stream": "profiles", "data": {"type": "profile", "id": "01F5VTX8KP49GGQ4BG77HZ9FRH", "attributes": {"email": "some.email.that.dont.exist3@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "Strange Name2", "last_name": "Funny Name2", "organization": null, "title": null, "image": null, "created": "2021-05-17T00:16:44+00:00", "updated": "2021-05-17T00:16:44+00:00", "last_event_date": null, "location": {"address1": null, "address2": null, "city": null, "country": null, "latitude": null, "longitude": null, "region": null, "zip": null, "timezone": null, "ip": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": null}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTX8KP49GGQ4BG77HZ9FRH/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTX8KP49GGQ4BG77HZ9FRH/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5VTX8KP49GGQ4BG77HZ9FRH/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5VTX8KP49GGQ4BG77HZ9FRH/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5VTX8KP49GGQ4BG77HZ9FRH/segments/"}}}, "updated": "2021-05-17T00:16:44+00:00"}, "emitted_at": 1679533540463} +{"stream": "profiles", "data": {"type": "profile", "id": "01F5YBDQE9W7WDSH9KK398CAYX", "attributes": {"email": "some.email.that.dont.exist.{seed}@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": null, "last_name": null, "organization": null, "title": null, "image": null, "created": "2021-05-17T23:43:50+00:00", "updated": "2021-05-17T23:43:50+00:00", "last_event_date": "2021-05-17T23:43:28+00:00", "location": {"address1": null, "address2": null, "city": null, "country": null, "latitude": null, "longitude": null, "region": null, "zip": null, "timezone": null, "ip": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": null}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBDQE9W7WDSH9KK398CAYX/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBDQE9W7WDSH9KK398CAYX/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBDQE9W7WDSH9KK398CAYX/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBDQE9W7WDSH9KK398CAYX/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBDQE9W7WDSH9KK398CAYX/segments/"}}}, "updated": "2021-05-17T23:43:50+00:00"}, "emitted_at": 1679533540463} +{"stream": "profiles", "data": {"type": "profile", "id": "01F5YBGKW1SQN453RM293PHH37", "attributes": {"email": "some.email.that.dont.exist.0@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 0", "last_name": "Last Name 0", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:24+00:00", "updated": "2021-05-17T23:45:25+00:00", "last_event_date": "2021-05-17T23:45:24+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null, "ip": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": null}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGKW1SQN453RM293PHH37/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGKW1SQN453RM293PHH37/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGKW1SQN453RM293PHH37/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGKW1SQN453RM293PHH37/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGKW1SQN453RM293PHH37/segments/"}}}, "updated": "2021-05-17T23:45:25+00:00"}, "emitted_at": 1679533540463} +{"stream": "profiles", "data": {"type": "profile", "id": "01F5YBGMTSM3B56W37QB9Q9CAD", "attributes": {"email": "some.email.that.dont.exist.3@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 3", "last_name": "Last Name 3", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:25+00:00", "updated": "2021-05-17T23:45:25+00:00", "last_event_date": "2021-05-17T23:45:25+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null, "ip": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": null}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGMTSM3B56W37QB9Q9CAD/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGMTSM3B56W37QB9Q9CAD/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGMTSM3B56W37QB9Q9CAD/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGMTSM3B56W37QB9Q9CAD/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGMTSM3B56W37QB9Q9CAD/segments/"}}}, "updated": "2021-05-17T23:45:25+00:00"}, "emitted_at": 1679533540463} +{"stream": "profiles", "data": {"type": "profile", "id": "01F5YBGM7J4YD4P6EYK5Q87BG4", "attributes": {"email": "some.email.that.dont.exist.1@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 1", "last_name": "Last Name 1", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:25+00:00", "updated": "2021-05-17T23:45:26+00:00", "last_event_date": "2021-05-17T23:45:24+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null, "ip": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": null}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGM7J4YD4P6EYK5Q87BG4/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGM7J4YD4P6EYK5Q87BG4/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGM7J4YD4P6EYK5Q87BG4/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGM7J4YD4P6EYK5Q87BG4/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGM7J4YD4P6EYK5Q87BG4/segments/"}}}, "updated": "2021-05-17T23:45:26+00:00"}, "emitted_at": 1679533540464} +{"stream": "profiles", "data": {"type": "profile", "id": "01F5YBGN65NTCBGTAR1Y7P5285", "attributes": {"email": "some.email.that.dont.exist.4@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 4", "last_name": "Last Name 4", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:26+00:00", "updated": "2021-05-17T23:45:26+00:00", "last_event_date": "2021-05-17T23:45:26+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null, "ip": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": null}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGN65NTCBGTAR1Y7P5285/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGN65NTCBGTAR1Y7P5285/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGN65NTCBGTAR1Y7P5285/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGN65NTCBGTAR1Y7P5285/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGN65NTCBGTAR1Y7P5285/segments/"}}}, "updated": "2021-05-17T23:45:26+00:00"}, "emitted_at": 1679533540464} +{"stream": "profiles", "data": {"type": "profile", "id": "01F5YBGNK6H122QRC1K96GXY8C", "attributes": {"email": "some.email.that.dont.exist.5@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 5", "last_name": "Last Name 5", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:26+00:00", "updated": "2021-05-17T23:45:26+00:00", "last_event_date": "2021-05-17T23:45:26+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null, "ip": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": null}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGNK6H122QRC1K96GXY8C/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGNK6H122QRC1K96GXY8C/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGNK6H122QRC1K96GXY8C/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGNK6H122QRC1K96GXY8C/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGNK6H122QRC1K96GXY8C/segments/"}}}, "updated": "2021-05-17T23:45:26+00:00"}, "emitted_at": 1679533540464} +{"stream": "profiles", "data": {"type": "profile", "id": "01F5YBGP0P02E9Q64KF26VB2MH", "attributes": {"email": "some.email.that.dont.exist.6@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 6", "last_name": "Last Name 6", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:27+00:00", "updated": "2021-05-17T23:45:27+00:00", "last_event_date": "2021-05-17T23:45:26+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null, "ip": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": null}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGP0P02E9Q64KF26VB2MH/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGP0P02E9Q64KF26VB2MH/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGP0P02E9Q64KF26VB2MH/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGP0P02E9Q64KF26VB2MH/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGP0P02E9Q64KF26VB2MH/segments/"}}}, "updated": "2021-05-17T23:45:27+00:00"}, "emitted_at": 1679533540464} +{"stream": "profiles", "data": {"type": "profile", "id": "01F5YBGPSXF1N23RBJZ947R1N1", "attributes": {"email": "some.email.that.dont.exist.8@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 8", "last_name": "Last Name 8", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:27+00:00", "updated": "2021-05-17T23:45:27+00:00", "last_event_date": "2021-05-17T23:45:27+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null, "ip": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [{"reason": "USER_SUPPRESSED", "timestamp": "2021-05-18T01:29:51+00:00"}], "list_suppressions": []}}, "sms": null}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGPSXF1N23RBJZ947R1N1/segments/"}}}, "updated": "2021-05-17T23:45:27+00:00"}, "emitted_at": 1679533540465} +{"stream": "profiles", "data": {"type": "profile", "id": "01F5YBGPCQESZDRKGW3DB1WPZ0", "attributes": {"email": "some.email.that.dont.exist.7@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 7", "last_name": "Last Name 7", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:27+00:00", "updated": "2021-05-17T23:45:30+00:00", "last_event_date": "2021-05-17T23:45:27+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null, "ip": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": null}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGPCQESZDRKGW3DB1WPZ0/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGPCQESZDRKGW3DB1WPZ0/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGPCQESZDRKGW3DB1WPZ0/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGPCQESZDRKGW3DB1WPZ0/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGPCQESZDRKGW3DB1WPZ0/segments/"}}}, "updated": "2021-05-17T23:45:30+00:00"}, "emitted_at": 1679533540465} +{"stream": "profiles", "data": {"type": "profile", "id": "01F5YBGQ6X21SSWPGRDK9QK97C", "attributes": {"email": "some.email.that.dont.exist.9@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 9", "last_name": "Last Name 9", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:28+00:00", "updated": "2021-05-17T23:45:30+00:00", "last_event_date": "2021-05-17T23:45:28+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null, "ip": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [{"reason": "USER_SUPPRESSED", "timestamp": "2021-05-18T01:20:01+00:00"}], "list_suppressions": []}}, "sms": null}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGQ6X21SSWPGRDK9QK97C/segments/"}}}, "updated": "2021-05-17T23:45:30+00:00"}, "emitted_at": 1679533540465} +{"stream": "profiles", "data": {"type": "profile", "id": "01F5YBGMK62AJR0955G7NW6EP7", "attributes": {"email": "some.email.that.dont.exist.2@airbyte.io", "phone_number": null, "external_id": null, "anonymous_id": null, "first_name": "First Name 2", "last_name": "Last Name 2", "organization": null, "title": null, "image": null, "created": "2021-05-17T23:45:25+00:00", "updated": "2021-05-17T23:45:38+00:00", "last_event_date": "2021-05-17T23:45:25+00:00", "location": {"address1": null, "address2": null, "city": "Springfield", "country": null, "latitude": null, "longitude": null, "region": "Illinois", "zip": null, "timezone": null, "ip": null}, "properties": {}, "subscriptions": {"email": {"marketing": {"consent": "NEVER_SUBSCRIBED", "timestamp": null, "method": null, "method_detail": null, "custom_method_detail": null, "double_optin": null, "suppressions": [], "list_suppressions": []}}, "sms": null}}, "links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGMK62AJR0955G7NW6EP7/"}, "relationships": {"lists": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGMK62AJR0955G7NW6EP7/relationships/lists/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGMK62AJR0955G7NW6EP7/lists/"}}, "segments": {"links": {"self": "https://a.klaviyo.com/api/profiles/01F5YBGMK62AJR0955G7NW6EP7/relationships/segments/", "related": "https://a.klaviyo.com/api/profiles/01F5YBGMK62AJR0955G7NW6EP7/segments/"}}}, "updated": "2021-05-17T23:45:38+00:00"}, "emitted_at": 1679533540465} +{"stream": "flows", "data": {"type": "flow", "id": "YfYbWb", "attributes": {"name": "Abandoned Cart", "status": "live", "archived": false, "created": "2022-05-31T06:48:46+00:00", "updated": "2022-05-31T06:50:35+00:00", "trigger_type": "Metric"}, "relationships": {"flow-actions": {"links": {"self": "https://a.klaviyo.com/api/flows/YfYbWb/relationships/flow-actions/", "related": "https://a.klaviyo.com/api/flows/YfYbWb/flow-actions/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/flows/YfYbWb/relationships/tags/", "related": "https://a.klaviyo.com/api/flows/YfYbWb/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/flows/YfYbWb/"}, "updated": "2022-05-31T06:50:35+00:00"}, "emitted_at": 1698938560373} +{"stream": "flows", "data": {"type": "flow", "id": "Usr9XK", "attributes": {"name": "Welcome Series", "status": "live", "archived": false, "created": "2022-05-31T06:51:39+00:00", "updated": "2022-05-31T06:52:14+00:00", "trigger_type": "Added to List"}, "relationships": {"flow-actions": {"links": {"self": "https://a.klaviyo.com/api/flows/Usr9XK/relationships/flow-actions/", "related": "https://a.klaviyo.com/api/flows/Usr9XK/flow-actions/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/flows/Usr9XK/relationships/tags/", "related": "https://a.klaviyo.com/api/flows/Usr9XK/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/flows/Usr9XK/"}, "updated": "2022-05-31T06:52:14+00:00"}, "emitted_at": 1698938560374} +{"stream": "flows", "data": {"type": "flow", "id": "Ub7CPq", "attributes": {"name": "Browse Abandonment", "status": "manual", "archived": false, "created": "2022-05-31T06:54:12+00:00", "updated": "2022-05-31T06:54:13+00:00", "trigger_type": "Metric"}, "relationships": {"flow-actions": {"links": {"self": "https://a.klaviyo.com/api/flows/Ub7CPq/relationships/flow-actions/", "related": "https://a.klaviyo.com/api/flows/Ub7CPq/flow-actions/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/flows/Ub7CPq/relationships/tags/", "related": "https://a.klaviyo.com/api/flows/Ub7CPq/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/flows/Ub7CPq/"}, "updated": "2022-05-31T06:54:13+00:00"}, "emitted_at": 1698938560374} +{"stream": "flows", "data": {"type": "flow", "id": "Uxqqk4", "attributes": {"name": "Customer Thank You", "status": "live", "archived": false, "created": "2022-05-31T06:55:43+00:00", "updated": "2022-05-31T06:55:44+00:00", "trigger_type": "Metric"}, "relationships": {"flow-actions": {"links": {"self": "https://a.klaviyo.com/api/flows/Uxqqk4/relationships/flow-actions/", "related": "https://a.klaviyo.com/api/flows/Uxqqk4/flow-actions/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/flows/Uxqqk4/relationships/tags/", "related": "https://a.klaviyo.com/api/flows/Uxqqk4/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/flows/Uxqqk4/"}, "updated": "2022-05-31T06:55:44+00:00"}, "emitted_at": 1698938560375} +{"stream": "flows", "data": {"type": "flow", "id": "VcNt87", "attributes": {"name": "Welcome Series - Standard (Email & SMS)", "status": "manual", "archived": false, "created": "2022-05-31T06:58:03+00:00", "updated": "2022-05-31T06:58:04+00:00", "trigger_type": "Added to List"}, "relationships": {"flow-actions": {"links": {"self": "https://a.klaviyo.com/api/flows/VcNt87/relationships/flow-actions/", "related": "https://a.klaviyo.com/api/flows/VcNt87/flow-actions/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/flows/VcNt87/relationships/tags/", "related": "https://a.klaviyo.com/api/flows/VcNt87/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/flows/VcNt87/"}, "updated": "2022-05-31T06:58:04+00:00"}, "emitted_at": 1698938560375} +{"stream": "flows", "data": {"type": "flow", "id": "U5LCpF", "attributes": {"name": "Welcome Series - Customer v. Non-Customer", "status": "live", "archived": false, "created": "2022-05-31T07:00:28+00:00", "updated": "2022-05-31T07:00:29+00:00", "trigger_type": "Added to List"}, "relationships": {"flow-actions": {"links": {"self": "https://a.klaviyo.com/api/flows/U5LCpF/relationships/flow-actions/", "related": "https://a.klaviyo.com/api/flows/U5LCpF/flow-actions/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/flows/U5LCpF/relationships/tags/", "related": "https://a.klaviyo.com/api/flows/U5LCpF/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/flows/U5LCpF/"}, "updated": "2022-05-31T07:00:29+00:00"}, "emitted_at": 1698938560375} +{"stream": "flows", "data": {"type": "flow", "id": "VueJfU", "attributes": {"name": "Happy Birthday Email - Standard", "status": "manual", "archived": false, "created": "2022-05-31T07:01:40+00:00", "updated": "2022-05-31T07:01:41+00:00", "trigger_type": "Date Based"}, "relationships": {"flow-actions": {"links": {"self": "https://a.klaviyo.com/api/flows/VueJfU/relationships/flow-actions/", "related": "https://a.klaviyo.com/api/flows/VueJfU/flow-actions/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/flows/VueJfU/relationships/tags/", "related": "https://a.klaviyo.com/api/flows/VueJfU/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/flows/VueJfU/"}, "updated": "2022-05-31T07:01:41+00:00"}, "emitted_at": 1698938560376} +{"stream": "flows", "data": {"type": "flow", "id": "ShbZ4B", "attributes": {"name": "Abandoned Cart Archived", "status": "draft", "archived": true, "created": "2023-10-31T11:37:03+00:00", "updated": "2023-10-31T11:40:31+00:00", "trigger_type": "Metric"}, "relationships": {"flow-actions": {"links": {"self": "https://a.klaviyo.com/api/flows/ShbZ4B/relationships/flow-actions/", "related": "https://a.klaviyo.com/api/flows/ShbZ4B/flow-actions/"}}, "tags": {"links": {"self": "https://a.klaviyo.com/api/flows/ShbZ4B/relationships/tags/", "related": "https://a.klaviyo.com/api/flows/ShbZ4B/tags/"}}}, "links": {"self": "https://a.klaviyo.com/api/flows/ShbZ4B/"}, "updated": "2023-10-31T11:40:31+00:00"}, "emitted_at": 1698938560690} diff --git a/airbyte-integrations/connectors/source-klaviyo/metadata.yaml b/airbyte-integrations/connectors/source-klaviyo/metadata.yaml index c062944c3905..71bcca6adeb0 100644 --- a/airbyte-integrations/connectors/source-klaviyo/metadata.yaml +++ b/airbyte-integrations/connectors/source-klaviyo/metadata.yaml @@ -6,7 +6,9 @@ data: connectorSubtype: api connectorType: source definitionId: 95e8cffd-b8c4-4039-968e-d32fb4a69bde - dockerImageTag: 0.3.2 + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c + dockerImageTag: 2.1.0 dockerRepository: airbyte/source-klaviyo githubIssueLabel: source-klaviyo icon: klaviyo.svg @@ -18,6 +20,21 @@ data: oss: enabled: true releaseStage: generally_available + suggestedStreams: + streams: + - events + - campaigns + - lists + - metrics + - flows + releases: + breakingChanges: + 1.0.0: + message: In this release, for 'events' stream changed type of 'event_properties/items/quantity' field from integer to number. Users will need to refresh the source schema and reset events streams after upgrading. + upgradeDeadline: "2023-11-30" + 2.0.0: + message: In this release, streams 'campaigns', 'email_templates', 'events', 'flows', 'global_exclusions', 'lists', and 'metrics' are now pulling data using latest API which has a different schema. Users will need to refresh the source schemas and reset these streams after upgrading. + upgradeDeadline: "2023-11-30" documentationUrl: https://docs.airbyte.com/integrations/sources/klaviyo tags: - language:python diff --git a/airbyte-integrations/connectors/source-klaviyo/setup.py b/airbyte-integrations/connectors/source-klaviyo/setup.py index 3603d9d4885b..32a31edb0848 100644 --- a/airbyte-integrations/connectors/source-klaviyo/setup.py +++ b/airbyte-integrations/connectors/source-klaviyo/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1"] +MAIN_REQUIREMENTS = ["airbyte-cdk"] TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock", "requests_mock~=1.8"] diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/availability_strategy.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/availability_strategy.py new file mode 100644 index 000000000000..4154d8f26401 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/availability_strategy.py @@ -0,0 +1,24 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from typing import Dict, Optional + +from airbyte_cdk.sources import Source +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy +from requests import HTTPError, codes + + +class KlaviyoAvailabilityStrategy(HttpAvailabilityStrategy): + def reasons_for_unavailable_status_codes( + self, stream: Stream, logger: logging.Logger, source: Optional[Source], error: HTTPError + ) -> Dict[int, str]: + reasons_for_codes: Dict[int, str] = super().reasons_for_unavailable_status_codes(stream, logger, source, error) + reasons_for_codes[codes.UNAUTHORIZED] = ( + "This is most likely due to insufficient permissions on the credentials in use. " + f"Try to create and use an API key with read permission for the '{stream.name}' stream granted" + ) + + return reasons_for_codes diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/exceptions.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/exceptions.py new file mode 100644 index 000000000000..63df81365345 --- /dev/null +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/exceptions.py @@ -0,0 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + + +class KlaviyoBackoffError(Exception): + """An exception which is raised when 'retry-after' time is longer than 'max_time' specified""" diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/campaigns.json b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/campaigns.json index 6e494a5d9b42..d568f492604a 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/campaigns.json @@ -1,55 +1,50 @@ { "type": "object", + "additionalProperties": true, "properties": { - "object": { "type": "string" }, + "type": { "type": "string" }, "id": { "type": "string" }, - "name": { "type": "string" }, - "created": { "type": ["null", "string"], "format": "date-time" }, - "updated": { "type": ["null", "string"], "format": "date-time" }, - "status": { "type": "string" }, - "status_id": { "type": "integer" }, - "status_label": { "type": "string" }, - "from_name": { "type": "string" }, - "from_email": { "type": "string" }, - "num_recipients": { "type": "integer" }, - "lists": { - "type": "array", - "items": { - "type": "object", - "properties": { - "object": { "type": "string" }, - "id": { "type": "string" }, - "name": { "type": "string" }, - "created": { "type": "string", "format": "date-time" }, - "updated": { "type": "string", "format": "date-time" }, - "person_count": { "type": "integer" }, - "list_type": { "type": "string" }, - "folder": { "type": ["null", "string"] } - } + "updated_at": { "type": ["null", "string"], "format": "date-time" }, + "attributes": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "name": { "type": "string" }, + "status": { "type": "string" }, + "archived": { "type": "boolean" }, + "channel": { "type": "string" }, + "audiences": { + "type": ["null", "object"], + "additionalProperties": true + }, + "send_options": { + "type": ["null", "object"], + "properties": { + "use_smart_sending": { "type": "boolean" } + } + }, + "message": { "type": "string" }, + "tracking_options": { + "type": ["null", "object"], + "additionalProperties": true + }, + "send_strategy": { + "type": ["null", "object"], + "additionalProperties": true + }, + "created_at": { "type": ["null", "string"], "format": "date-time" }, + "scheduled_at": { "type": ["null", "string"], "format": "date-time" }, + "updated_at": { "type": ["null", "string"], "format": "date-time" }, + "send_time": { "type": ["null", "string"], "format": "date-time" } } }, - "excluded_lists": { - "type": "array", - "items": { - "type": "object", - "properties": { - "object": { "type": "string" }, - "id": { "type": "string" }, - "name": { "type": "string" }, - "created": { "type": "string", "format": "date-time" }, - "updated": { "type": "string", "format": "date-time" }, - "person_count": { "type": "integer" }, - "list_type": { "type": "string" }, - "folder": { "type": ["null", "string"] } - } - } + "links": { + "type": ["null", "object"], + "additionalProperties": true }, - "is_segmented": { "type": "boolean" }, - "send_time": { "type": ["null", "string"], "format": "date-time" }, - "sent_at": { "type": ["null", "string"], "format": "date-time" }, - "campaign_type": { "type": "string" }, - "subject": { "type": ["null", "string"] }, - "message_type": { "type": "string" }, - "template_id": { "type": ["null", "string"] } + "relationships": { + "type": ["null", "object"], + "additionalProperties": true + } } } diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/email_templates.json b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/email_templates.json index da46b6550410..c3590f804f83 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/email_templates.json +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/email_templates.json @@ -2,28 +2,25 @@ "type": "object", "additionalProperties": true, "properties": { - "object": { - "type": "string" + "type": { "type": "string" }, + "id": { "type": "string" }, + "updated": { "type": ["null", "string"], "format": "date-time" }, + "attributes": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "name": { "type": "string" }, + "editor_type": { "type": ["null", "string"] }, + "html": { "type": "string" }, + "text": { "type": ["null", "string"] }, + "created": { "type": ["null", "string"], "format": "date-time" }, + "updated": { "type": ["null", "string"], "format": "date-time" }, + "company_id": { "type": ["null", "string"] } + } }, - "id": { - "type": "string" - }, - "name": { - "type": ["null", "string"] - }, - "html": { - "type": ["null", "string"] - }, - "is_writeable": { - "type": ["null", "boolean"] - }, - "created": { - "type": "string", - "format": "date-time" - }, - "updated": { - "type": "string", - "format": "date-time" + "links": { + "type": ["null", "object"], + "additionalProperties": true } } } diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/events.json b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/events.json index 7cf830f697a8..23006e1f3c36 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/events.json +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/events.json @@ -1,42 +1,64 @@ { "type": "object", + "additionalProperties": true, "properties": { - "object": { "type": "string" }, + "type": { "type": "string" }, "id": { "type": "string" }, - "uuid": { "type": "string" }, - "event_name": { "type": "string" }, - "timestamp": { "type": "integer" }, - "datetime": { "type": "string" }, - "statistic_id": { "type": "string" }, - "event_properties": { - "type": "object", + "datetime": { "type": "string", "format": "date-time" }, + "attributes": { + "type": ["null", "object"], "properties": { - "$value": { "type": "number" }, - "items": { - "type": "array", - "items": { - "type": "object", - "properties": { - "object": { "type": "string" }, - "name": { "type": "string" }, - "sku": { "type": "string" }, - "price": { "type": "number" }, - "quantity": { "type": "integer" } - } - } - } + "timestamp": { "type": "integer" }, + "event_properties": { + "type": ["null", "object"], + "additionalProperties": true + }, + "datetime": { "type": "string", "format": "date-time" }, + "uuid": { "type": "string" } } }, - "person": { - "type": "object", + "links": { + "type": ["null", "object"], "properties": { - "id": { "type": "string" }, - "object": { "type": "string" }, - "$email": { "type": "string" } + "self": { "type": "string" } } }, - "flow_id": { "type": ["null", "string"] }, - "campaign_id": { "type": ["null", "string"] }, - "flow_message_id": { "type": ["null", "string"] } + "relationships": { + "type": ["null", "object"], + "properties": { + "profile": { + "type": ["null", "object"], + "properties": { + "data": { + "type": ["null", "object"], + "properties": { + "type": { "type": "string" }, + "id": { "type": "string" } + } + }, + "links": { + "type": ["null", "object"], + "additionalProperties": true + } + } + }, + "metric": { + "type": ["null", "object"], + "properties": { + "data": { + "type": ["null", "object"], + "properties": { + "type": { "type": "string" }, + "id": { "type": "string" } + } + }, + "links": { + "type": ["null", "object"], + "additionalProperties": true + } + } + } + } + } } } diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/flows.json b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/flows.json index 54ea3dded05c..e65227382ac1 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/flows.json +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/flows.json @@ -1,13 +1,29 @@ { "type": "object", + "additionalProperties": true, "properties": { - "object": { "type": "string" }, + "type": { "type": "string" }, "id": { "type": "string" }, - "name": { "type": "string" }, - "status": { "type": "string" }, - "created": { "type": "string", "format": "date-time" }, "updated": { "type": "string", "format": "date-time" }, - "customer_filter": { "type": ["null", "object"] }, - "trigger": { "type": "object" } + "attributes": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "name": { "type": "string" }, + "status": { "type": "string" }, + "archived": { "type": "boolean" }, + "created": { "type": "string", "format": "date-time" }, + "updated": { "type": "string", "format": "date-time" }, + "trigger_type": { "type": "string" } + } + }, + "links": { + "type": ["null", "object"], + "additionalProperties": true + }, + "relationships": { + "type": ["null", "object"], + "additionalProperties": true + } } } diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/global_exclusions.json b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/global_exclusions.json index 84e21e29f70e..21f80313fd0a 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/global_exclusions.json +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/global_exclusions.json @@ -1,9 +1,32 @@ { "type": "object", + "additionalProperties": true, "properties": { - "object": { "type": "string" }, - "email": { "type": "string" }, - "reason": { "type": "string" }, - "timestamp": { "type": "string", "format": "date-time" } + "type": { "type": ["null", "string"] }, + "id": { "type": "string" }, + "updated": { "type": ["null", "string"], "format": "date-time" }, + "attributes": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "email": { "type": ["null", "string"] }, + "phone_number": { "type": ["null", "string"] }, + "first_name": { "type": ["null", "string"] }, + "last_name": { "type": ["null", "string"] }, + "properties": { + "type": ["null", "object"], + "additionalProperties": true + }, + "subscriptions": { "type": ["null", "object"] }, + "organization": { "type": ["null", "string"] }, + "title": { "type": ["null", "string"] }, + "created": { "type": ["null", "string"], "format": "date-time" }, + "updated": { "type": ["null", "string"], "format": "date-time" }, + "last_event_date": { "type": ["null", "string"], "format": "date-time" } + } + }, + "links": { "type": ["null", "object"] }, + "relationships": { "type": ["null", "object"] }, + "segments": { "type": ["null", "object"] } } } diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/lists.json b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/lists.json index da721281b4e4..6adea37deef3 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/lists.json +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/lists.json @@ -1,13 +1,25 @@ { "type": "object", + "additionalProperties": true, "properties": { - "object": { "type": "string" }, + "type": { "type": "string" }, "id": { "type": "string" }, - "name": { "type": "string" }, - "created": { "type": "string", "format": "date-time" }, - "updated": { "type": "string", "format": "date-time" }, - "person_count": { "type": "integer" }, - "list_type": { "type": "string" }, - "folder": { "type": ["null", "string"] } + "updated": { "type": ["null", "string"], "format": "date-time" }, + "attributes": { + "type": ["null", "object"], + "properties": { + "name": { "type": "string" }, + "created": { "type": ["null", "string"], "format": "date-time" }, + "updated": { "type": ["null", "string"], "format": "date-time" } + } + }, + "links": { + "type": ["null", "object"], + "additionalProperties": true + }, + "relationships": { + "type": ["null", "object"], + "additionalProperties": true + } } } diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/metrics.json b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/metrics.json index bfdb31a0e45e..1d6984e551f8 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/metrics.json +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/schemas/metrics.json @@ -1,19 +1,25 @@ { "type": "object", + "additionalProperties": true, "properties": { - "object": { "type": "string" }, + "type": { "type": "string" }, "id": { "type": "string" }, - "name": { "type": "string" }, - "created": { "type": "string", "format": "date-time" }, "updated": { "type": "string", "format": "date-time" }, - "integration": { - "type": "object", + "attributes": { + "type": ["null", "object"], "properties": { - "object": { "type": "string" }, - "id": { "type": "string" }, "name": { "type": "string" }, - "category": { "type": "string" } + "created": { "type": "string", "format": "date-time" }, + "updated": { "type": "string", "format": "date-time" }, + "integration": { + "type": ["null", "object"], + "additionalProperties": true + } } + }, + "links": { + "type": ["null", "object"], + "additionalProperties": true } } } diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/source.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/source.py index 4843cd213aa1..5647ca33ea67 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/source.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/source.py @@ -3,11 +3,13 @@ # import re +from http import HTTPStatus from typing import Any, List, Mapping, Tuple from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream +from requests.exceptions import HTTPError from source_klaviyo.streams import Campaigns, EmailTemplates, Events, Flows, GlobalExclusions, Lists, Metrics, Profiles @@ -21,6 +23,12 @@ def check_connection(self, logger, config: Mapping[str, Any]) -> Tuple[bool, Any try: # we use metrics endpoint because it never returns an error _ = list(Metrics(api_key=config["api_key"]).read_records(sync_mode=SyncMode.full_refresh)) + except HTTPError as e: + if e.response.status_code in (HTTPStatus.FORBIDDEN, HTTPStatus.UNAUTHORIZED): + message = "Please provide a valid API key and make sure it has permissions to read specified streams." + else: + message = "Unable to connect to Klaviyo API with provided credentials." + return False, message except Exception as e: original_error_message = repr(e) @@ -39,14 +47,17 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: :param config: A Mapping of the user input configuration as defined in the connector spec. """ api_key = config["api_key"] - start_date = config["start_date"] + start_date = config.get("start_date") return [ - Campaigns(api_key=api_key), + Campaigns(api_key=api_key, start_date=start_date), Events(api_key=api_key, start_date=start_date), GlobalExclusions(api_key=api_key, start_date=start_date), - Lists(api_key=api_key), - Metrics(api_key=api_key), + Lists(api_key=api_key, start_date=start_date), + Metrics(api_key=api_key, start_date=start_date), Flows(api_key=api_key, start_date=start_date), - EmailTemplates(api_key=api_key), + EmailTemplates(api_key=api_key, start_date=start_date), Profiles(api_key=api_key, start_date=start_date), ] + + def continue_sync_on_stream_failure(self) -> bool: + return True diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/spec.json b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/spec.json index e5b1408af651..1f03cf982e63 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/spec.json +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/spec.json @@ -10,17 +10,19 @@ "title": "Api Key", "description": "Klaviyo API Key. See our docs if you need help finding this key.", "airbyte_secret": true, - "type": "string" + "type": "string", + "order": 0 }, "start_date": { "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. This field is optional - if not provided, all data will be replicated.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", "examples": ["2017-01-25T00:00:00Z"], "type": "string", - "format": "date-time" + "format": "date-time", + "order": 1 } }, - "required": ["api_key", "start_date"] + "required": ["api_key"] } } diff --git a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py index f2b1b085449f..10009335765e 100644 --- a/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py +++ b/airbyte-integrations/connectors/source-klaviyo/source_klaviyo/streams.py @@ -2,51 +2,52 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import datetime import urllib.parse from abc import ABC, abstractmethod from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union import pendulum import requests +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy +from airbyte_cdk.sources.streams.core import StreamData from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +from .availability_strategy import KlaviyoAvailabilityStrategy +from .exceptions import KlaviyoBackoffError -class KlaviyoStreamLatest(HttpStream, ABC): - """Base stream for api version v2023-02-22""" + +class KlaviyoStream(HttpStream, ABC): + """Base stream for api version v2023-10-15""" url_base = "https://a.klaviyo.com/api/" primary_key = "id" - page_size = 100 + page_size = None + api_revision = "2023-10-15" - def __init__(self, api_key: str, **kwargs): + def __init__(self, api_key: str, start_date: Optional[str] = None, **kwargs): super().__init__(**kwargs) self._api_key = api_key + self._start_ts = start_date @property - def availability_strategy(self) -> Optional["AvailabilityStrategy"]: - return None + def availability_strategy(self) -> Optional[AvailabilityStrategy]: + return KlaviyoAvailabilityStrategy() def request_headers(self, **kwargs) -> Mapping[str, Any]: - base_headers = super().request_headers(**kwargs) - - headers = { + return { "Accept": "application/json", - "Content-Type": "application/json", - "Revision": "2023-02-22", - "Authorization": "Klaviyo-API-Key " + self._api_key, + "Revision": self.api_revision, + "Authorization": f"Klaviyo-API-Key {self._api_key}", } - return {**base_headers, **headers} - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: """This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. Klaviyo uses cursor-based pagination https://developers.klaviyo.com/en/reference/api_overview#pagination This method returns the params in the pre-constructed url nested in links[next] """ + decoded_response = response.json() links = decoded_response.get("links", {}) @@ -60,31 +61,66 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - # If next_page_token is set, all of the parameters are already provided + # If next_page_token is set, all the parameters are already provided if next_page_token: return next_page_token else: - return {"page[size]": self.page_size} + return {"page[size]": self.page_size} if self.page_size else {} def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: """:return an iterable containing each record in the response""" + response_json = response.json() for record in response_json.get("data", []): # API returns records in a container array "data" record = self.map_record(record) yield record - def map_record(self, record: Mapping): + def map_record(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]: """Subclasses can override this to apply custom mappings to a record""" + + record[self.cursor_field] = record["attributes"][self.cursor_field] return record + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest + record and the current state and picks the 'most' recent cursor. This is how a stream's state is determined. + Required for incremental. + """ -class IncrementalKlaviyoStreamLatest(KlaviyoStreamLatest, ABC): + current_stream_cursor_value = current_stream_state.get(self.cursor_field, self._start_ts) + latest_cursor = pendulum.parse(latest_record[self.cursor_field]) + if current_stream_cursor_value: + latest_cursor = max(latest_cursor, pendulum.parse(current_stream_cursor_value)) + current_stream_state[self.cursor_field] = latest_cursor.isoformat() + return current_stream_state + + def backoff_time(self, response: requests.Response) -> Optional[float]: + if response.status_code == 429: + retry_after = response.headers.get("Retry-After") + retry_after = float(retry_after) if retry_after else None + if retry_after and retry_after >= self.max_time: + raise KlaviyoBackoffError( + f"Stream {self.name} has reached rate limit with 'Retry-After' of {retry_after} seconds, exit from stream." + ) + return retry_after + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: Optional[List[str]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + stream_state: Optional[Mapping[str, Any]] = None, + ) -> Iterable[StreamData]: + try: + yield from super().read_records(sync_mode, cursor_field, stream_slice, stream_state) + except KlaviyoBackoffError as e: + self.logger.warning(repr(e)) + + +class IncrementalKlaviyoStream(KlaviyoStream, ABC): """Base class for all incremental streams, requires cursor_field to be declared""" - def __init__(self, start_date: str, **kwargs): - super().__init__(**kwargs) - self._start_ts = start_date - @property @abstractmethod def cursor_field(self) -> Union[str, List[str]]: @@ -101,104 +137,20 @@ def request_params(self, stream_state: Mapping[str, Any] = None, next_page_token params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) if not params.get("filter"): - latest_cursor = pendulum.parse(self._start_ts) stream_state_cursor_value = stream_state.get(self.cursor_field) - if stream_state_cursor_value: - latest_cursor = max(latest_cursor, pendulum.parse(stream_state[self.cursor_field])) - params["filter"] = "greater-than(" + self.cursor_field + "," + latest_cursor.isoformat() + ")" + latest_cursor = stream_state_cursor_value or self._start_ts + if latest_cursor: + latest_cursor = pendulum.parse(latest_cursor) + if stream_state_cursor_value: + latest_cursor = max(latest_cursor, pendulum.parse(stream_state_cursor_value)) + latest_cursor = min(latest_cursor, pendulum.now()) + params["filter"] = f"greater-than({self.cursor_field},{latest_cursor.isoformat()})" params["sort"] = self.cursor_field return params - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and - the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. - """ - current_stream_cursor_value = current_stream_state.get(self.cursor_field, self._start_ts) - latest_record_cursor_value = latest_record[self.cursor_field] - latest_cursor = max(pendulum.parse(latest_record_cursor_value), pendulum.parse(current_stream_cursor_value)) - return {self.cursor_field: latest_cursor.isoformat()} - - -class Profiles(IncrementalKlaviyoStreamLatest): - """Docs: https://developers.klaviyo.com/en/reference/get_profiles""" - - cursor_field = "updated" - - def path(self, *args, next_page_token: Optional[Mapping[str, Any]] = None, **kwargs) -> str: - return "profiles" - - def map_record(self, record: Mapping): - record[self.cursor_field] = record["attributes"][self.cursor_field] - return record - - -class KlaviyoStreamV1(HttpStream, ABC): - """Base stream for api v1""" - url_base = "https://a.klaviyo.com/api/v1/" - primary_key = "id" - page_size = 100 - - transformer = TypeTransformer(TransformConfig.CustomSchemaNormalization) - - def __init__(self, api_key: str, **kwargs): - super().__init__(**kwargs) - self._api_key = api_key - transform_function = self.get_custom_transform() - self.transformer.registerCustomTransform(transform_function) - - @property - def availability_strategy(self) -> Optional["AvailabilityStrategy"]: - return None - - def get_custom_transform(self): - def custom_transform_date_rfc3339(original_value, field_schema): - if original_value and "format" in field_schema and field_schema["format"] == "date-time": - transformed_value = pendulum.parse(original_value).to_rfc3339_string() - return transformed_value - return original_value - - return custom_transform_date_rfc3339 - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed - to most other methods in this class to help you form headers, request bodies, query params, etc.. - :param response: the most recent response from the API - :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. - If there are no more pages in the result, return None. - """ - decoded_response = response.json() - if decoded_response["end"] < decoded_response["total"] - 1: # end is zero based - return { - "page": decoded_response["page"] + 1, - } - - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - """Usually contains common params e.g. pagination size etc.""" - next_page_token = next_page_token or {} - return {**next_page_token, "api_key": self._api_key, "count": self.page_size} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """:return an iterable containing each record in the response""" - - response_json = response.json() - for record in response_json.get("data", []): # API returns records in a container array "data" - yield record - - -class IncrementalKlaviyoStreamV1(KlaviyoStreamV1, ABC): - """Base class for all incremental streams, requires cursor_field to be declared""" - - def __init__(self, start_date: str, **kwargs): - super().__init__(**kwargs) - self._start_ts = int(pendulum.parse(start_date).timestamp()) - self._start_sync = int(pendulum.now().timestamp()) +class SemiIncrementalKlaviyoStream(KlaviyoStream, ABC): + """Base class for all streams that have a cursor field, but underlying API does not support either sorting or filtering""" @property @abstractmethod @@ -209,192 +161,160 @@ def cursor_field(self) -> Union[str, List[str]]: :return str: The name of the cursor field. """ - def request_params(self, stream_state=None, **kwargs): - """Add incremental filters""" + def read_records( + self, + sync_mode: SyncMode, + cursor_field: Optional[List[str]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + stream_state: Optional[Mapping[str, Any]] = None, + ) -> Iterable[StreamData]: stream_state = stream_state or {} - params = super().request_params(stream_state=stream_state, **kwargs) + starting_point = stream_state.get(self.cursor_field, self._start_ts) + for record in super().read_records( + sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ): + if starting_point and record[self.cursor_field] > starting_point or not starting_point: + yield record + + +class ArchivedRecordsStream(IncrementalKlaviyoStream): + def __init__(self, path: str, cursor_field: str, start_date: Optional[str] = None, api_revision: Optional[str] = None, **kwargs): + super().__init__(start_date=start_date, **kwargs) + self._path = path + self._cursor_field = cursor_field + if api_revision: + self.api_revision = api_revision - if not params.get("since"): # skip state filter if already have one from pagination - state_ts = int(stream_state.get(self.cursor_field, 0)) - params["since"] = max(state_ts, self._start_ts) - params["sort"] = "asc" - - return params - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and - the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. - """ - state_ts = int(current_stream_state.get(self.cursor_field, 0)) - latest_record = latest_record.get(self.cursor_field) - - if isinstance(latest_record, str): - latest_record = datetime.datetime.strptime(latest_record, "%Y-%m-%d %H:%M:%S") - latest_record = datetime.datetime.timestamp(latest_record) - - new_value = max(latest_record, state_ts) - new_value = min(new_value, self._start_sync) - return {self.cursor_field: new_value} - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed - to most other methods in this class to help you form headers, request bodies, query params, etc.. - :param response: the most recent response from the API - :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. - If there are no more pages in the result, return None. - """ - decoded_response = response.json() - if decoded_response.get("next"): - return {"since": decoded_response["next"]} - - return None + @property + def cursor_field(self) -> Union[str, List[str]]: + return self._cursor_field + def path(self, **kwargs) -> str: + return self._path -class ReverseIncrementalKlaviyoStreamV1(KlaviyoStreamV1, ABC): - """Base class for all streams that natively incremental but supports desc & asc order""" + def request_params(self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs): + archived_stream_state = stream_state.get("archived") if stream_state else None + params = super().request_params(archived_stream_state, next_page_token, **kwargs) + archived_filter = "equals(archived,true)" + if "filter" in params and archived_filter not in params["filter"]: + params["filter"] = f"and({params['filter']},{archived_filter})" + elif "filter" not in params: + params["filter"] = archived_filter + return params - def __init__(self, start_date: str, **kwargs): - super().__init__(**kwargs) - self._start_datetime = pendulum.parse(start_date) - self._reversed = False - self._reached_old_records = False - self._low_boundary = None - @property - def state_checkpoint_interval(self) -> Optional[int]: - """How often to checkpoint state (i.e: emit a STATE message). By default return the same value as page_size""" - return None if self._reversed else self.page_size +class ArchivedRecordsMixin(IncrementalKlaviyoStream, ABC): + """A mixin class which should be used when archived records need to be read""" @property - @abstractmethod - def cursor_field(self) -> Union[str, List[str]]: - """ - Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is - usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. - :return str: The name of the cursor field. - """ - - def request_params(self, stream_state=None, **kwargs): - """Add incremental filters""" - stream_state = stream_state or {} - if stream_state: - self._reversed = True - self._low_boundary = max(pendulum.parse(stream_state[self.cursor_field]), self._start_datetime) - params = super().request_params(stream_state=stream_state, **kwargs) - params["sort"] = "desc" if self._reversed else "asc" - - return params + def archived_campaigns(self) -> ArchivedRecordsStream: + return ArchivedRecordsStream(self.path(), self.cursor_field, self._start_ts, self.api_revision, api_key=self._api_key) def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: """ - Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and - the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. + Extend the stream state with `archived` property to store such records' state separately from the stream state """ - latest_cursor = pendulum.parse(latest_record[self.cursor_field]) - if current_stream_state: - latest_cursor = max(pendulum.parse(latest_record[self.cursor_field]), pendulum.parse(current_stream_state[self.cursor_field])) - return {self.cursor_field: latest_cursor.isoformat()} + if latest_record.get("attributes", {}).get("archived", False): + current_archived_stream_cursor_value = current_stream_state.get("archived", {}).get(self.cursor_field, self._start_ts) + latest_archived_cursor = pendulum.parse(latest_record[self.cursor_field]) + if current_archived_stream_cursor_value: + latest_archived_cursor = max(latest_archived_cursor, pendulum.parse(current_archived_stream_cursor_value)) + current_stream_state["archived"] = {self.cursor_field: latest_archived_cursor.isoformat()} + return current_stream_state + else: + return super().get_updated_state(current_stream_state, latest_record) - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed - to most other methods in this class to help you form headers, request bodies, query params, etc.. - :param response: the most recent response from the API - :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. - If there are no more pages in the result, return None. - """ + def read_records( + self, + sync_mode: SyncMode, + cursor_field: Optional[List[str]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + stream_state: Optional[Mapping[str, Any]] = None, + ) -> Iterable[StreamData]: + yield from super().read_records(sync_mode, cursor_field, stream_slice, stream_state) + yield from self.archived_campaigns.read_records(sync_mode, cursor_field, stream_slice, stream_state) - next_page_token = super().next_page_token(response) - if self._reversed and self._reached_old_records: - return None - return next_page_token +class Profiles(IncrementalKlaviyoStream): + """Docs: https://developers.klaviyo.com/en/v2023-02-22/reference/get_profiles""" - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """:return an iterable containing each record in the response""" + cursor_field = "updated" + api_revision = "2023-02-22" + page_size = 100 + state_checkpoint_interval = 100 # API can return maximum 100 records per page - for record in super().parse_response(response=response, **kwargs): - if self._reversed: - if pendulum.parse(record[self.cursor_field]) < self._low_boundary: - self._reached_old_records = True - continue - else: - if pendulum.parse(record[self.cursor_field]) < self._start_datetime: - continue - yield record + def path(self, *args, next_page_token: Optional[Mapping[str, Any]] = None, **kwargs) -> str: + return "profiles" -class Campaigns(KlaviyoStreamV1): - """Docs: https://developers.klaviyo.com/en/reference/get-campaigns""" +class Campaigns(ArchivedRecordsMixin, IncrementalKlaviyoStream): + """Docs: https://developers.klaviyo.com/en/v2023-06-15/reference/get_campaigns""" + + cursor_field = "updated_at" + api_revision = "2023-06-15" def path(self, **kwargs) -> str: return "campaigns" -class Lists(KlaviyoStreamV1): - """Docs: https://developers.klaviyo.com/en/reference/get-lists""" +class Lists(SemiIncrementalKlaviyoStream): + """Docs: https://developers.klaviyo.com/en/reference/get_lists""" max_retries = 10 + cursor_field = "updated" def path(self, **kwargs) -> str: return "lists" -class GlobalExclusions(ReverseIncrementalKlaviyoStreamV1): - """Docs: https://developers.klaviyo.com/en/reference/get-global-exclusions""" +class GlobalExclusions(Profiles): + """ + Docs: https://developers.klaviyo.com/en/v2023-02-22/reference/get_profiles + This stream takes data from 'profiles' endpoint, but suppressed records only + """ - page_size = 5000 # the maximum value allowed by API - cursor_field = "timestamp" - primary_key = "email" + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + for record in super().parse_response(response, **kwargs): + if not record["attributes"].get("subscriptions", {}).get("email", {}).get("marketing", {}).get("suppressions"): + continue + yield record - def path(self, **kwargs) -> str: - return "people/exclusions" +class Metrics(SemiIncrementalKlaviyoStream): + """Docs: https://developers.klaviyo.com/en/reference/get_metrics""" -class Metrics(KlaviyoStreamV1): - """Docs: https://developers.klaviyo.com/en/reference/get-metrics""" + cursor_field = "updated" def path(self, **kwargs) -> str: return "metrics" -class Events(IncrementalKlaviyoStreamV1): - """Docs: https://developers.klaviyo.com/en/reference/metrics-timeline""" +class Events(IncrementalKlaviyoStream): + """Docs: https://developers.klaviyo.com/en/reference/get_events""" - cursor_field = "timestamp" + cursor_field = "datetime" + state_checkpoint_interval = 200 # API can return maximum 200 records per page def path(self, **kwargs) -> str: - return "metrics/timeline" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """:return an iterable containing each record in the response""" - - response_json = response.json() - for record in response_json.get("data", []): - flow = record["event_properties"].get("$flow") - flow_message_id = record["event_properties"].get("$message") - - record["flow_id"] = flow - record["flow_message_id"] = flow_message_id - record["campaign_id"] = flow_message_id if not flow else None + return "events" - yield record +class Flows(ArchivedRecordsMixin, IncrementalKlaviyoStream): + """Docs: https://developers.klaviyo.com/en/reference/get_flows""" -class Flows(ReverseIncrementalKlaviyoStreamV1): - cursor_field = "created" + cursor_field = "updated" + state_checkpoint_interval = 50 # API can return maximum 50 records per page def path(self, **kwargs) -> str: return "flows" -class EmailTemplates(KlaviyoStreamV1): - """ - Docs: https://developers.klaviyo.com/en/v1-2/reference/get-templates - """ +class EmailTemplates(IncrementalKlaviyoStream): + """Docs: https://developers.klaviyo.com/en/reference/get_templates""" + + cursor_field = "updated" + state_checkpoint_interval = 10 # API can return maximum 10 records per page def path(self, **kwargs) -> str: - return "email-templates" + return "templates" diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_latest_streams.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_latest_streams.py deleted file mode 100644 index f4f21d665c48..000000000000 --- a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_latest_streams.py +++ /dev/null @@ -1,128 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest import mock - -import pendulum -import pytest -import requests -from pydantic import BaseModel -from source_klaviyo.streams import IncrementalKlaviyoStreamLatest, Profiles - -START_DATE = pendulum.datetime(2020, 10, 10) - - -class SomeIncrementalStream(IncrementalKlaviyoStreamLatest): - schema = mock.Mock(spec=BaseModel) - cursor_field = "updated" - - def path(self, **kwargs) -> str: - return "sub_path" - - -@pytest.fixture(name="response") -def response_fixture(mocker): - return mocker.Mock(spec=requests.Response) - - -class TestIncrementalKlaviyoStreamLatest: - def test_cursor_field_is_required(self): - with pytest.raises( - TypeError, match="Can't instantiate abstract class IncrementalKlaviyoStreamLatest with abstract methods cursor_field, path" - ): - IncrementalKlaviyoStreamLatest(api_key="some_key", start_date=START_DATE.isoformat()) - - @pytest.mark.parametrize( - ["response_json", "next_page_token"], - [ - ( - { - "data": [ - {"type": "profile", "id": "00AA0A0AA0AA000AAAAAAA0AA0"}, - ], - "links": { - "self": "https://a.klaviyo.com/api/profiles/", - "next": "https://a.klaviyo.com/api/profiles/?page%5Bcursor%5D=aaA0aAo0aAA0AaAaAaa0AaaAAAaaA00AAAa0AA00A0AAAaAa", - "prev": "null" - } - }, - { - "page[cursor]": "aaA0aAo0aAA0AaAaAaa0AaaAAAaaA00AAAa0AA00A0AAAaAa" - } - ) - ], - ) - def test_next_page_token(self, response, response_json, next_page_token): - response.json.return_value = response_json - stream = SomeIncrementalStream(api_key="some_key", start_date=START_DATE.isoformat()) - result = stream.next_page_token(response) - - assert result == next_page_token - - -class TestProfilesStream: - def test_parse_response(self, mocker): - stream = Profiles(api_key="some_key", start_date=START_DATE.isoformat()) - json = { - "data": [ - { - "type": "profile", - "id": "00AA0A0AA0AA000AAAAAAA0AA0", - "attributes": { - "email": "name@airbyte.io", - "phone_number": "+11111111111", - "updated": "2023-03-10T20:36:36+00:00" - }, - "properties": { - "Status": "onboarding_complete" - } - }, - { - "type": "profile", - "id": "AAAA1A1AA1AA111AAAAAAA1AA1", - "attributes": { - "email": "name2@airbyte.io", - "phone_number": "+2222222222", - "updated": "2023-02-10T20:36:36+00:00" - }, - "properties": { - "Status": "onboarding_started" - } - } - ], - "links": { - "self": "https://a.klaviyo.com/api/profiles/", - "next": "https://a.klaviyo.com/api/profiles/?page%5Bcursor%5D=aaA0aAo0aAA0AaAaAaa0AaaAAAaaA00AAAa0AA00A0AAAaAa", - "prev": "null" - } - } - records = list(stream.parse_response(mocker.Mock(json=mocker.Mock(return_value=json)))) - assert records == [ - { - "type": "profile", - "id": "00AA0A0AA0AA000AAAAAAA0AA0", - "updated": "2023-03-10T20:36:36+00:00", - "attributes": { - "email": "name@airbyte.io", - "phone_number": "+11111111111", - "updated": "2023-03-10T20:36:36+00:00" - }, - "properties": { - "Status": "onboarding_complete" - } - }, - { - "type": "profile", - "id": "AAAA1A1AA1AA111AAAAAAA1AA1", - "updated": "2023-02-10T20:36:36+00:00", - "attributes": { - "email": "name2@airbyte.io", - "phone_number": "+2222222222", - "updated": "2023-02-10T20:36:36+00:00" - }, - "properties": { - "Status": "onboarding_started" - } - } - ] diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_source.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_source.py index bc90b7154d08..f8dfa207ad9d 100644 --- a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_source.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import pendulum import pytest from source_klaviyo.source import SourceKlaviyo @@ -14,20 +15,20 @@ 400, "Bad request", False, - "HTTPError('400 Client Error: None for url: https://a.klaviyo.com/api/v1/metrics?api_key=***&count=100')", + "Unable to connect to Klaviyo API with provided credentials.", ), ( 403, "Forbidden", False, - "HTTPError('403 Client Error: None for url: https://a.klaviyo.com/api/v1/metrics?api_key=***&count=100')", + "Please provide a valid API key and make sure it has permissions to read specified streams.", ), ), ) def test_check_connection(requests_mock, status_code, response, is_connection_successful, error_msg): requests_mock.register_uri( "GET", - "https://a.klaviyo.com/api/v1/metrics?api_key=api_key&count=100", + "https://a.klaviyo.com/api/metrics", status_code=status_code, json={"end": 1, "total": 1} if 200 >= status_code < 300 else {}, ) @@ -35,3 +36,26 @@ def test_check_connection(requests_mock, status_code, response, is_connection_su success, error = source.check_connection(logger=None, config={"api_key": "api_key"}) assert success is is_connection_successful assert error == error_msg + + +def test_check_connection_unexpected_error(requests_mock): + requests_mock.register_uri( + "GET", + "https://a.klaviyo.com/api/metrics", + exc=Exception("Something went wrong, api_key=some_api_key"), + ) + source = SourceKlaviyo() + success, error = source.check_connection(logger=None, config={"api_key": "api_key"}) + assert success is False + assert error == "Exception('Something went wrong, api_key=***')" + + +def test_streams(): + source = SourceKlaviyo() + config = {"api_key": "some_key", "start_date": pendulum.datetime(2020, 10, 10).isoformat()} + streams = source.streams(config) + expected_streams_number = 8 + assert len(streams) == expected_streams_number + + # ensure only unique stream names are returned + assert len({stream.name for stream in streams}) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_streams.py index fcee95efe92f..b81bda168e9f 100644 --- a/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-klaviyo/unit_tests/test_streams.py @@ -2,36 +2,48 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from datetime import datetime from unittest import mock import pendulum import pytest import requests +from airbyte_cdk.models import SyncMode from pydantic import BaseModel -from source_klaviyo.streams import EmailTemplates, Events, IncrementalKlaviyoStreamV1, KlaviyoStreamV1, ReverseIncrementalKlaviyoStreamV1 - +from source_klaviyo.availability_strategy import KlaviyoAvailabilityStrategy +from source_klaviyo.exceptions import KlaviyoBackoffError +from source_klaviyo.streams import ( + ArchivedRecordsStream, + Campaigns, + GlobalExclusions, + IncrementalKlaviyoStream, + KlaviyoStream, + Profiles, + SemiIncrementalKlaviyoStream, +) + +API_KEY = "some_key" START_DATE = pendulum.datetime(2020, 10, 10) -class SomeStream(KlaviyoStreamV1): +class SomeStream(KlaviyoStream): schema = mock.Mock(spec=BaseModel) + max_time = 60 * 10 def path(self, **kwargs) -> str: return "sub_path" -class SomeIncrementalStream(IncrementalKlaviyoStreamV1): +class SomeIncrementalStream(IncrementalKlaviyoStream): schema = mock.Mock(spec=BaseModel) - cursor_field = "updated_at" + cursor_field = "updated" def path(self, **kwargs) -> str: return "sub_path" -class SomeReverseIncrementalStream(ReverseIncrementalKlaviyoStreamV1): +class SomeSemiIncrementalStream(SemiIncrementalKlaviyoStream): schema = mock.Mock(spec=BaseModel) - cursor_field = "updated_at" + cursor_field = "updated" def path(self, **kwargs) -> str: return "sub_path" @@ -42,269 +54,436 @@ def response_fixture(mocker): return mocker.Mock(spec=requests.Response) -class TestKlaviyoStreamV1: +class TestKlaviyoStream: + def test_request_headers(self): + stream = SomeStream(api_key=API_KEY) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_headers = { + "Accept": "application/json", + "Revision": stream.api_revision, + "Authorization": f"Klaviyo-API-Key {API_KEY}", + } + assert stream.request_headers(**inputs) == expected_headers + @pytest.mark.parametrize( - ["response_json", "next_page_token"], - [ - ({"end": 108, "total": 110, "page": 0}, {"page": 1}), # first page - ({"end": 108, "total": 110, "page": 9}, {"page": 10}), # has next page - ({"end": 109, "total": 110, "page": 9}, None), # last page - ], + ("next_page_token", "page_size", "expected_params"), + ( + ({"page[cursor]": "aaA0aAo0aAA0A"}, None, {"page[cursor]": "aaA0aAo0aAA0A"}), + ({"page[cursor]": "aaA0aAo0aAA0A"}, 100, {"page[cursor]": "aaA0aAo0aAA0A"}), + (None, None, {}), + (None, 100, {"page[size]": 100}), + ), + ) + def test_request_params(self, next_page_token, page_size, expected_params): + stream = SomeStream(api_key=API_KEY) + stream.page_size = page_size + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": next_page_token} + assert stream.request_params(**inputs) == expected_params + + @pytest.mark.parametrize( + ("response_json", "next_page_token"), + ( + ( + { + "data": [ + {"type": "profile", "id": "00AA0A0AA0AA000AAAAAAA0AA0"}, + ], + "links": { + "self": "https://a.klaviyo.com/api/profiles/", + "next": "https://a.klaviyo.com/api/profiles/?page%5Bcursor%5D=aaA0aAo0aAA0AaAaAaa0AaaAAAaaA00AAAa0AA00A0AAAaAa", + "prev": "null", + }, + }, + {"page[cursor]": "aaA0aAo0aAA0AaAaAaa0AaaAAAaaA00AAAa0AA00A0AAAaAa"}, + ), + ( + { + "data": [ + {"type": "profile", "id": "00AA0A0AA0AA000AAAAAAA0AA0"}, + ], + "links": { + "self": "https://a.klaviyo.com/api/profiles/", + "prev": "null", + }, + }, + None, + ), + ), ) def test_next_page_token(self, response, response_json, next_page_token): response.json.return_value = response_json - stream = SomeStream(api_key="some_key") + stream = SomeStream(api_key=API_KEY) result = stream.next_page_token(response) assert result == next_page_token - @pytest.mark.parametrize( - ["next_page_token", "expected_params"], - [ - ({"page": 10}, {"api_key": "some_key", "count": 100, "page": 10}), - (None, {"api_key": "some_key", "count": 100}), - ], - ) - def test_request_params(self, next_page_token, expected_params): - stream = SomeStream(api_key="some_key") - result = stream.request_params(stream_state={}, next_page_token=next_page_token) - - assert result == expected_params - - def test_parse_response(self, response): - response.json.return_value = {"data": [1, 2, 3, 4, 5]} - stream = SomeStream(api_key="some_key") - result = stream.parse_response(response) + def test_availability_strategy(self): + stream = SomeStream(api_key=API_KEY) + assert isinstance(stream.availability_strategy, KlaviyoAvailabilityStrategy) - assert list(result) == response.json.return_value["data"] + expected_status_code = 401 + expected_message = ( + "This is most likely due to insufficient permissions on the credentials in use. " + "Try to create and use an API key with read permission for the 'some_stream' stream granted" + ) + reasons_for_unavailable_status_codes = stream.availability_strategy.reasons_for_unavailable_status_codes(stream, None, None, None) + assert expected_status_code in reasons_for_unavailable_status_codes + assert reasons_for_unavailable_status_codes[expected_status_code] == expected_message - -class TestIncrementalKlaviyoStreamV1: + @pytest.mark.parametrize( + ("status_code", "retry_after", "expected_time"), + ((429, 30, 30.0), (429, None, None), (200, 30, None), (200, None, None)), + ) + def test_backoff_time(self, status_code, retry_after, expected_time): + stream = SomeStream(api_key=API_KEY) + response_mock = mock.MagicMock() + response_mock.status_code = status_code + response_mock.headers = {"Retry-After": retry_after} + assert stream.backoff_time(response_mock) == expected_time + + def test_backoff_time_large_retry_after(self): + stream = SomeStream(api_key=API_KEY) + response_mock = mock.MagicMock() + response_mock.status_code = 429 + retry_after = stream.max_time + 5 + response_mock.headers = {"Retry-After": retry_after} + with pytest.raises(KlaviyoBackoffError) as e: + stream.backoff_time(response_mock) + error_message = f"Stream some_stream has reached rate limit with 'Retry-After' of {float(retry_after)} seconds, exit from stream." + assert str(e.value) == error_message + + +class TestIncrementalKlaviyoStream: def test_cursor_field_is_required(self): with pytest.raises( - TypeError, match="Can't instantiate abstract class IncrementalKlaviyoStreamV1 with abstract methods cursor_field, path" + TypeError, match="Can't instantiate abstract class IncrementalKlaviyoStream with abstract methods cursor_field, path" ): - IncrementalKlaviyoStreamV1(api_key="some_key", start_date=START_DATE.isoformat()) + IncrementalKlaviyoStream(api_key=API_KEY, start_date=START_DATE.isoformat()) @pytest.mark.parametrize( - ["next_page_token", "stream_state", "expected_params"], - [ - # start with start_date - (None, {}, {"api_key": "some_key", "count": 100, "sort": "asc", "since": START_DATE.int_timestamp}), - # pagination overrule - ({"since": 123}, {}, {"api_key": "some_key", "count": 100, "sort": "asc", "since": 123}), - # start_date overrule state if state < start_date + ("config_start_date", "stream_state_date", "next_page_token", "expected_params"), + ( ( - None, - {"updated_at": START_DATE.int_timestamp - 1}, - {"api_key": "some_key", "count": 100, "sort": "asc", "since": START_DATE.int_timestamp}, + START_DATE.isoformat(), + {"updated": "2023-01-01T00:00:00+00:00"}, + {"page[cursor]": "aaA0aAo0aAA0AaAaAaa0AaaAAAaaA00AAAa0AA00A0AAAaAa"}, + { + "filter": "greater-than(updated,2023-01-01T00:00:00+00:00)", + "page[cursor]": "aaA0aAo0aAA0AaAaAaa0AaaAAAaaA00AAAa0AA00A0AAAaAa", + "sort": "updated", + }, ), - # but pagination still overrule ( - {"since": 123}, - {"updated_at": START_DATE.int_timestamp - 1}, - {"api_key": "some_key", "count": 100, "sort": "asc", "since": 123}, + START_DATE.isoformat(), + None, + {"page[cursor]": "aaA0aAo0aAA0AaAaAaa0AaaAAAaaA00AAAa0AA00A0AAAaAa"}, + { + "filter": "greater-than(updated,2020-10-10T00:00:00+00:00)", + "page[cursor]": "aaA0aAo0aAA0AaAaAaa0AaaAAAaaA00AAAa0AA00A0AAAaAa", + "sort": "updated", + }, ), - # and again ( - {"since": 123}, - {"updated_at": START_DATE.int_timestamp + 1}, - {"api_key": "some_key", "count": 100, "sort": "asc", "since": 123}, + START_DATE.isoformat(), + None, + {"filter": "some_filter"}, + {"filter": "some_filter"}, ), - # finally state > start_date and can be used ( None, - {"updated_at": START_DATE.int_timestamp + 1}, - {"api_key": "some_key", "count": 100, "sort": "asc", "since": START_DATE.int_timestamp + 1}, + None, + {"page[cursor]": "aaA0aAo0aAA0AaAaAaa0AaaAAAaaA00AAAa0AA00A0AAAaAa"}, + { + "page[cursor]": "aaA0aAo0aAA0AaAaAaa0AaaAAAaaA00AAAa0AA00A0AAAaAa", + "sort": "updated", + }, ), - ], - ) - def test_request_params(self, next_page_token, stream_state, expected_params): - stream = SomeIncrementalStream(api_key="some_key", start_date=START_DATE.isoformat()) - result = stream.request_params(stream_state=stream_state, next_page_token=next_page_token) - - assert result == expected_params - - @pytest.mark.parametrize( - ["current_state", "latest_record", "expected_state"], - [ - ({}, {"updated_at": 10, "some_field": 100}, {"updated_at": 10}), - ({"updated_at": 11}, {"updated_at": 10, "some_field": 100}, {"updated_at": 11}), - ({"updated_at": 11}, {"updated_at": 12, "some_field": 100}, {"updated_at": 12}), ( - {"updated_at": 12}, - {"updated_at": "2021-04-03 17:15:12", "some_field": 100}, - {"updated_at": datetime.strptime("2021-04-03 17:15:12", "%Y-%m-%d %H:%M:%S").timestamp()}, + None, + {"updated": "2023-01-01T00:00:00+00:00"}, + {"page[cursor]": "aaA0aAo0aAA0AaAaAaa0AaaAAAaaA00AAAa0AA00A0AAAaAa"}, + { + "filter": "greater-than(updated,2023-01-01T00:00:00+00:00)", + "page[cursor]": "aaA0aAo0aAA0AaAaAaa0AaaAAAaaA00AAAa0AA00A0AAAaAa", + "sort": "updated", + }, ), - ({"updated_at": 12}, {"updated_at": 7998603215}, None), - ({"updated_at": 7998603215}, {"updated_at": 12}, None), - ], + ), ) - def test_get_updated_state(self, current_state, latest_record, expected_state): - stream = SomeIncrementalStream(api_key="some_key", start_date=START_DATE.isoformat()) - result = stream.get_updated_state(current_stream_state=current_state, latest_record=latest_record) - - assert result == (expected_state if expected_state else {stream.cursor_field: stream._start_sync}) + def test_request_params(self, config_start_date, stream_state_date, next_page_token, expected_params): + stream = SomeIncrementalStream(api_key=API_KEY, start_date=config_start_date) + inputs = {"stream_state": stream_state_date, "next_page_token": next_page_token} + assert stream.request_params(**inputs) == expected_params @pytest.mark.parametrize( - ["response_json", "next_page_token"], - [ - ({"next": 10, "total": 110, "page": 9}, {"since": 10}), # has next page - ({"total": 110, "page": 9}, None), # last page - ], + ("config_start_date", "current_cursor", "latest_cursor", "expected_cursor"), + ( + (START_DATE.isoformat(), "2023-01-01T00:00:00+00:00", "2023-01-02T00:00:00+00:00", "2023-01-02T00:00:00+00:00"), + (START_DATE.isoformat(), "2023-01-02T00:00:00+00:00", "2023-01-01T00:00:00+00:00", "2023-01-02T00:00:00+00:00"), + (START_DATE.isoformat(), None, "2019-01-01T00:00:00+00:00", "2020-10-10T00:00:00+00:00"), + (None, "2020-10-10T00:00:00+00:00", "2019-01-01T00:00:00+00:00", "2020-10-10T00:00:00+00:00"), + (None, None, "2019-01-01T00:00:00+00:00", "2019-01-01T00:00:00+00:00"), + ), ) - def test_next_page_token(self, response, response_json, next_page_token): - response.json.return_value = response_json - stream = SomeIncrementalStream(api_key="some_key", start_date=START_DATE.isoformat()) - result = stream.next_page_token(response) - - assert result == next_page_token + def test_get_updated_state(self, config_start_date, current_cursor, latest_cursor, expected_cursor): + stream = SomeIncrementalStream(api_key=API_KEY, start_date=config_start_date) + inputs = { + "current_stream_state": {stream.cursor_field: current_cursor} if current_cursor else {}, + "latest_record": {stream.cursor_field: latest_cursor}, + } + assert stream.get_updated_state(**inputs) == {stream.cursor_field: expected_cursor} -class TestReverseIncrementalKlaviyoStreamV1: +class TestSemiIncrementalKlaviyoStream: def test_cursor_field_is_required(self): with pytest.raises( - TypeError, - match="Can't instantiate abstract class ReverseIncrementalKlaviyoStreamV1 with abstract methods cursor_field, path", + TypeError, match="Can't instantiate abstract class SemiIncrementalKlaviyoStream with abstract methods cursor_field, path" ): - ReverseIncrementalKlaviyoStreamV1(api_key="some_key", start_date=START_DATE.isoformat()) - - def test_state_checkpoint_interval(self): - stream = SomeReverseIncrementalStream(api_key="some_key", start_date=START_DATE.isoformat()) - - assert stream.state_checkpoint_interval == stream.page_size, "reversed stream on the first read commit state for each page" - - stream.request_params(stream_state={"updated_at": START_DATE.isoformat()}) - assert stream.state_checkpoint_interval is None, "reversed stream should commit state only in the end" - - @pytest.mark.parametrize( - ["next_page_token", "stream_state", "expected_params"], - [ - (None, {}, {"api_key": "some_key", "count": 100, "sort": "asc"}), - ({"page": 10}, {}, {"api_key": "some_key", "count": 100, "sort": "asc", "page": 10}), - (None, {"updated_at": START_DATE.isoformat()}, {"api_key": "some_key", "count": 100, "sort": "desc"}), - ({"page": 10}, {"updated_at": START_DATE.isoformat()}, {"api_key": "some_key", "count": 100, "sort": "desc", "page": 10}), - ], - ) - def test_request_params(self, next_page_token, stream_state, expected_params): - stream = SomeReverseIncrementalStream(api_key="some_key", start_date=START_DATE.isoformat()) - result = stream.request_params(stream_state=stream_state, next_page_token=next_page_token) - - assert result == expected_params + SemiIncrementalKlaviyoStream(api_key=API_KEY, start_date=START_DATE.isoformat()) @pytest.mark.parametrize( - ["current_state", "latest_record", "expected_state"], - [ - ({}, {"updated_at": "2021-01-02T12:13:14", "some_field": 100}, {"updated_at": "2021-01-02T12:13:14+00:00"}), + ("start_date", "stream_state", "input_records", "expected_records"), + ( ( - {"updated_at": "2021-02-03T13:14:15"}, - {"updated_at": "2021-01-02T12:13:14", "some_field": 100}, - {"updated_at": "2021-02-03T13:14:15+00:00"}, + "2021-11-08T00:00:00", + "2022-11-07T00:00:00", + [ + {"attributes": {"updated": "2022-11-08T00:00:00"}}, + {"attributes": {"updated": "2023-11-08T00:00:00"}}, + {"attributes": {"updated": "2021-11-08T00:00:00"}}, + ], + [ + {"attributes": {"updated": "2022-11-08T00:00:00"}, "updated": "2022-11-08T00:00:00"}, + {"attributes": {"updated": "2023-11-08T00:00:00"}, "updated": "2023-11-08T00:00:00"}, + ], ), ( - {"updated_at": "2021-02-03T13:14:15"}, - {"updated_at": "2021-03-04T14:15:16", "some_field": 100}, - {"updated_at": "2021-03-04T14:15:16+00:00"}, + "2021-11-08T00:00:00", + None, + [ + {"attributes": {"updated": "2022-11-08T00:00:00"}}, + {"attributes": {"updated": "2023-11-08T00:00:00"}}, + {"attributes": {"updated": "2021-11-08T00:00:00"}}, + ], + [ + {"attributes": {"updated": "2022-11-08T00:00:00"}, "updated": "2022-11-08T00:00:00"}, + {"attributes": {"updated": "2023-11-08T00:00:00"}, "updated": "2023-11-08T00:00:00"}, + ], ), - ], + ( + None, + None, + [ + {"attributes": {"updated": "2022-11-08T00:00:00"}}, + {"attributes": {"updated": "2023-11-08T00:00:00"}}, + {"attributes": {"updated": "2021-11-08T00:00:00"}}, + ], + [ + {"attributes": {"updated": "2022-11-08T00:00:00"}, "updated": "2022-11-08T00:00:00"}, + {"attributes": {"updated": "2023-11-08T00:00:00"}, "updated": "2023-11-08T00:00:00"}, + {"attributes": {"updated": "2021-11-08T00:00:00"}, "updated": "2021-11-08T00:00:00"}, + ], + ), + ( + "2021-11-08T00:00:00", + "2022-11-07T00:00:00", + [], + [], + ), + ), ) - def test_get_updated_state(self, current_state, latest_record, expected_state): - stream = SomeReverseIncrementalStream(api_key="some_key", start_date=START_DATE.isoformat()) - result = stream.get_updated_state(current_stream_state=current_state, latest_record=latest_record) - - assert result == expected_state - - def test_next_page_token(self, response): - ts_below_low_boundary = (START_DATE - pendulum.duration(hours=1)).isoformat() - ts_above_low_boundary = (START_DATE + pendulum.duration(minutes=1)).isoformat() - - response.json.return_value = { - "data": [{"updated_at": ts_below_low_boundary}, {"updated_at": ts_above_low_boundary}], - "end": 108, - "total": 110, - "page": 9, + def test_read_records(self, start_date, stream_state, input_records, expected_records, requests_mock): + stream = SomeSemiIncrementalStream(api_key=API_KEY, start_date=start_date) + requests_mock.register_uri("GET", f"https://a.klaviyo.com/api/{stream.path()}", status_code=200, json={"data": input_records}) + inputs = { + "sync_mode": SyncMode.incremental, + "cursor_field": stream.cursor_field, + "stream_slice": None, + "stream_state": {stream.cursor_field: stream_state} if stream_state else None, } - stream = SomeReverseIncrementalStream(api_key="some_key", start_date=START_DATE.isoformat()) - stream.request_params(stream_state={"updated_at": ts_below_low_boundary}) - next(iter(stream.parse_response(response))) + assert list(stream.read_records(**inputs)) == expected_records - result = stream.next_page_token(response) - - assert result is None - - def test_parse_response_read_backward(self, response): - ts_state = START_DATE + pendulum.duration(minutes=30) - ts_below_low_boundary = (ts_state - pendulum.duration(hours=1)).isoformat() - ts_above_low_boundary = (ts_state + pendulum.duration(minutes=1)).isoformat() - response.json.return_value = { - "data": [{"updated_at": ts_above_low_boundary}, {"updated_at": ts_above_low_boundary}, {"updated_at": ts_below_low_boundary}], - "end": 108, - "total": 110, - "page": 9, - } - stream = SomeReverseIncrementalStream(api_key="some_key", start_date=START_DATE.isoformat()) - stream.request_params(stream_state={"updated_at": ts_state.isoformat()}) - - result = list(stream.parse_response(response)) - assert result == response.json.return_value["data"][:2], "should return all records until low boundary reached" - - def test_parse_response_read_forward(self, response): - ts_below_low_boundary = (START_DATE - pendulum.duration(hours=1)).isoformat() - ts_above_low_boundary = (START_DATE + pendulum.duration(minutes=1)).isoformat() - - response.json.return_value = { - "data": [{"updated_at": ts_below_low_boundary}, {"updated_at": ts_below_low_boundary}, {"updated_at": ts_above_low_boundary}], - "end": 108, - "total": 110, - "page": 9, - } - stream = SomeReverseIncrementalStream(api_key="some_key", start_date=START_DATE.isoformat()) - stream.request_params(stream_state={}) - - result = list(stream.parse_response(response)) - - assert result == response.json.return_value["data"][2:], "should all records younger then start_datetime" - - -class TestEventsStream: +class TestProfilesStream: def test_parse_response(self, mocker): - stream = Events(api_key="some_key", start_date=START_DATE.isoformat()) + stream = Profiles(api_key=API_KEY, start_date=START_DATE.isoformat()) json = { "data": [ - {"event_properties": {"$flow": "ordinary", "$message": "hello"}, "some_key": "some_value"}, - {"event_properties": {"$flow": "advanced", "$message": "nice to meet you"}, "another_key": "another_value"}, - ] + { + "type": "profile", + "id": "00AA0A0AA0AA000AAAAAAA0AA0", + "attributes": {"email": "name@airbyte.io", "phone_number": "+11111111111", "updated": "2023-03-10T20:36:36+00:00"}, + "properties": {"Status": "onboarding_complete"}, + }, + { + "type": "profile", + "id": "AAAA1A1AA1AA111AAAAAAA1AA1", + "attributes": {"email": "name2@airbyte.io", "phone_number": "+2222222222", "updated": "2023-02-10T20:36:36+00:00"}, + "properties": {"Status": "onboarding_started"}, + }, + ], + "links": { + "self": "https://a.klaviyo.com/api/profiles/", + "next": "https://a.klaviyo.com/api/profiles/?page%5Bcursor%5D=aaA0aAo0aAA0AaAaAaa0AaaAAAaaA00AAAa0AA00A0AAAaAa", + "prev": "null", + }, } records = list(stream.parse_response(mocker.Mock(json=mocker.Mock(return_value=json)))) assert records == [ { - "campaign_id": None, - "event_properties": {"$flow": "ordinary", "$message": "hello"}, - "flow_id": "ordinary", - "flow_message_id": "hello", - "some_key": "some_value", + "type": "profile", + "id": "00AA0A0AA0AA000AAAAAAA0AA0", + "updated": "2023-03-10T20:36:36+00:00", + "attributes": {"email": "name@airbyte.io", "phone_number": "+11111111111", "updated": "2023-03-10T20:36:36+00:00"}, + "properties": {"Status": "onboarding_complete"}, }, { - "another_key": "another_value", - "campaign_id": None, - "event_properties": {"$flow": "advanced", "$message": "nice to meet you"}, - "flow_id": "advanced", - "flow_message_id": "nice to meet you", + "type": "profile", + "id": "AAAA1A1AA1AA111AAAAAAA1AA1", + "updated": "2023-02-10T20:36:36+00:00", + "attributes": {"email": "name2@airbyte.io", "phone_number": "+2222222222", "updated": "2023-02-10T20:36:36+00:00"}, + "properties": {"Status": "onboarding_started"}, }, ] -class TestEmailTemplatesStream: +class TestGlobalExclusionsStream: def test_parse_response(self, mocker): - stream = EmailTemplates(api_key="some_key") + stream = GlobalExclusions(api_key=API_KEY, start_date=START_DATE.isoformat()) json = { "data": [ - {"object": "email-template", "id": "id", "name": "Newsletter #1", "html": "", "is_writeable": "true", "created": "2023-02-18T11:18:22+00:00", "updated": "2023-02-18T12:01:12+00:00"}, - ] + { + "type": "profile", + "id": "00AA0A0AA0AA000AAAAAAA0AA0", + "attributes": { + "email": "name@airbyte.io", + "phone_number": "+11111111111", + "updated": "2023-03-10T20:36:36+00:00", + "subscriptions": { + "email": {"marketing": {"suppressions": [{"reason": "SUPPRESSED", "timestamp": "2021-05-18T01:29:51+00:00"}]}}, + }, + }, + }, + { + "type": "profile", + "id": "AAAA1A1AA1AA111AAAAAAA1AA1", + "attributes": {"email": "name2@airbyte.io", "phone_number": "+2222222222", "updated": "2023-02-10T20:36:36+00:00"}, + }, + ], + "links": { + "self": "https://a.klaviyo.com/api/profiles/", + "next": "https://a.klaviyo.com/api/profiles/?page%5Bcursor%5D=aaA0aAo0aAA0AaAaAaa0AaaAAAaaA00AAAa0AA00A0AAAaAa", + "prev": "null", + }, } records = list(stream.parse_response(mocker.Mock(json=mocker.Mock(return_value=json)))) - assert records == [ - {"object": "email-template", "id": "id", "name": "Newsletter #1", "html": "", "is_writeable": "true", "created": "2023-02-18T11:18:22+00:00", "updated": "2023-02-18T12:01:12+00:00"} + { + "type": "profile", + "id": "00AA0A0AA0AA000AAAAAAA0AA0", + "attributes": { + "email": "name@airbyte.io", + "phone_number": "+11111111111", + "updated": "2023-03-10T20:36:36+00:00", + "subscriptions": { + "email": {"marketing": {"suppressions": [{"reason": "SUPPRESSED", "timestamp": "2021-05-18T01:29:51+00:00"}]}}, + }, + }, + "updated": "2023-03-10T20:36:36+00:00", + } + ] + + +class TestCampaignsStream: + def test_read_records(self, requests_mock): + input_records = [ + {"attributes": {"name": "Some name 1", "archived": False, "updated_at": "2021-05-12T20:45:47+00:00"}}, + {"attributes": {"name": "Some name 2", "archived": False, "updated_at": "2021-05-12T20:45:47+00:00"}}, ] + input_records_archived = [ + {"attributes": {"name": "Archived", "archived": True, "updated_at": "2021-05-12T20:45:47+00:00"}}, + ] + + stream = Campaigns(api_key=API_KEY) + requests_mock.register_uri( + "GET", "https://a.klaviyo.com/api/campaigns?sort=updated_at", status_code=200, json={"data": input_records}, complete_qs=True + ) + requests_mock.register_uri( + "GET", + "https://a.klaviyo.com/api/campaigns?sort=updated_at&filter=equals(archived,true)", + status_code=200, + json={"data": input_records_archived}, + complete_qs=True, + ) + + inputs = {"sync_mode": SyncMode.full_refresh, "cursor_field": stream.cursor_field, "stream_slice": None, "stream_state": None} + expected_records = [ + { + "attributes": {"name": "Some name 1", "archived": False, "updated_at": "2021-05-12T20:45:47+00:00"}, + "updated_at": "2021-05-12T20:45:47+00:00", + }, + { + "attributes": {"name": "Some name 2", "archived": False, "updated_at": "2021-05-12T20:45:47+00:00"}, + "updated_at": "2021-05-12T20:45:47+00:00", + }, + { + "attributes": {"name": "Archived", "archived": True, "updated_at": "2021-05-12T20:45:47+00:00"}, + "updated_at": "2021-05-12T20:45:47+00:00", + }, + ] + assert list(stream.read_records(**inputs)) == expected_records + + @pytest.mark.parametrize( + ("latest_record", "current_stream_state", "expected_state"), + ( + ( + {"attributes": {"archived": False, "updated_at": "2023-01-02T00:00:00+00:00"}, "updated_at": "2023-01-02T00:00:00+00:00"}, + {"updated_at": "2023-01-01T00:00:00+00:00", "archived": {"updated_at": "2023-01-01T00:00:00+00:00"}}, + {"updated_at": "2023-01-02T00:00:00+00:00", "archived": {"updated_at": "2023-01-01T00:00:00+00:00"}}, + ), + ( + {"attributes": {"archived": False, "updated_at": "2023-01-02T00:00:00+00:00"}, "updated_at": "2023-01-02T00:00:00+00:00"}, + {"updated_at": "2023-01-01T00:00:00+00:00"}, + {"updated_at": "2023-01-02T00:00:00+00:00"}, + ), + ( + {"attributes": {"archived": True, "updated_at": "2023-01-02T00:00:00+00:00"}, "updated_at": "2023-01-02T00:00:00+00:00"}, + {"updated_at": "2023-01-01T00:00:00+00:00", "archived": {"updated_at": "2023-01-01T00:00:00+00:00"}}, + {"updated_at": "2023-01-01T00:00:00+00:00", "archived": {"updated_at": "2023-01-02T00:00:00+00:00"}}, + ), + ( + {"attributes": {"archived": True, "updated_at": "2023-01-02T00:00:00+00:00"}, "updated_at": "2023-01-02T00:00:00+00:00"}, + {"updated_at": "2023-01-01T00:00:00+00:00"}, + {"updated_at": "2023-01-01T00:00:00+00:00", "archived": {"updated_at": "2023-01-02T00:00:00+00:00"}}, + ), + ( + {"attributes": {"archived": False, "updated_at": "2023-01-02T00:00:00+00:00"}, "updated_at": "2023-01-02T00:00:00+00:00"}, + {}, + {"updated_at": "2023-01-02T00:00:00+00:00"}, + ), + ( + {"attributes": {"archived": True, "updated_at": "2023-01-02T00:00:00+00:00"}, "updated_at": "2023-01-02T00:00:00+00:00"}, + {}, + {"archived": {"updated_at": "2023-01-02T00:00:00+00:00"}}, + ), + ), + ) + def test_get_updated_state(self, latest_record, current_stream_state, expected_state): + stream = Campaigns(api_key=API_KEY) + assert stream.get_updated_state(current_stream_state, latest_record) == expected_state + + +class TestArchivedRecordsStream: + + @pytest.mark.parametrize( + "stream_state, next_page_token, expected_params", + [ + ({}, None, {"filter": "equals(archived,true)", "sort": "updated_at"}), + ({"archived": {"updated_at": "2023-10-10 00:00:00"}}, None, {"filter": "and(greater-than(updated_at,2023-10-10T00:00:00+00:00),equals(archived,true))", "sort": "updated_at"}), + ({"archived": {"updated_at": "2023-10-10 00:00:00"}}, {"filter": "and(greater-than(updated_at,2023-10-10T00:00:00+00:00),equals(archived,true))", "sort": "updated_at", "page[cursor]": "next_page_cursor"}, {"filter": "and(greater-than(updated_at,2023-10-10T00:00:00+00:00),equals(archived,true))", "sort": "updated_at", "page[cursor]": "next_page_cursor"}), + ({}, {"filter": "and(greater-than(updated_at,2023-10-10T00:00:00+00:00),equals(archived,true))", "sort": "updated_at", "page[cursor]": "next_page_cursor"}, {"filter": "and(greater-than(updated_at,2023-10-10T00:00:00+00:00),equals(archived,true))", "sort": "updated_at", "page[cursor]": "next_page_cursor"}) + ], + ) + def test_request_params(self, stream_state, next_page_token, expected_params): + archived_stream = ArchivedRecordsStream(api_key='API_KEY', cursor_field='updated_at', path='path') + assert archived_stream.request_params(stream_state=stream_state, next_page_token=next_page_token) == expected_params diff --git a/airbyte-integrations/connectors/source-kustomer-singer/README.md b/airbyte-integrations/connectors/source-kustomer-singer/README.md index f92a7a9d4d56..b0d3d66e7960 100644 --- a/airbyte-integrations/connectors/source-kustomer-singer/README.md +++ b/airbyte-integrations/connectors/source-kustomer-singer/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-kustomer:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/kustomer) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_kustomer_singer/spec.json` file. @@ -54,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-kustomer-singer:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-kustomer-singer build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-kustomer:airbyteDocker +An image will be built with the tag `airbyte/source-kustomer-singer:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-kustomer-singer:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,44 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-kustomer-singer:dev ch docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-kustomer-singer:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-kustomer-singer:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-kustomer-singer test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-kustomer-singer:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-kustomer-singer:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -122,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-kustomer-singer test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/kustomer-singer.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-kustomer-singer/acceptance-test-config.yml b/airbyte-integrations/connectors/source-kustomer-singer/acceptance-test-config.yml index 4e8535b8f6d6..8da27742f89a 100644 --- a/airbyte-integrations/connectors/source-kustomer-singer/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-kustomer-singer/acceptance-test-config.yml @@ -19,16 +19,6 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" - cursor_paths: - conversations: ["bookmarks", "conversations"] - customers: ["bookmarks", "customers"] - kobjects: ["bookmarks", "kobjects"] - messages: ["bookmarks", "messages"] - notes: ["bookmarks", "notes"] - shortcuts: ["bookmarks", "shortcuts"] - tags: ["bookmarks", "tags"] - teams: ["bookmarks", "teams"] - users: ["bookmarks", "users"] full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-kustomer-singer/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-kustomer-singer/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-kustomer-singer/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-kustomer-singer/build.gradle b/airbyte-integrations/connectors/source-kustomer-singer/build.gradle deleted file mode 100644 index 4722da7271e5..000000000000 --- a/airbyte-integrations/connectors/source-kustomer-singer/build.gradle +++ /dev/null @@ -1,10 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - // TODO -// id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_kustomer_singer' -} diff --git a/airbyte-integrations/connectors/source-kyriba/README.md b/airbyte-integrations/connectors/source-kyriba/README.md index 5474b11a3d65..5fc9f63ccd96 100644 --- a/airbyte-integrations/connectors/source-kyriba/README.md +++ b/airbyte-integrations/connectors/source-kyriba/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-kyriba:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/kyriba) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_kyriba/spec.json` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-kyriba:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-kyriba build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-kyriba:airbyteDocker +An image will be built with the tag `airbyte/source-kyriba:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-kyriba:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-kyriba:dev check --con docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-kyriba:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-kyriba:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-kyriba test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-kyriba:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-kyriba:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-kyriba test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/kyriba.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-kyriba/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-kyriba/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-kyriba/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-kyriba/build.gradle b/airbyte-integrations/connectors/source-kyriba/build.gradle deleted file mode 100644 index a33b161be39e..000000000000 --- a/airbyte-integrations/connectors/source-kyriba/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_kyriba' -} diff --git a/airbyte-integrations/connectors/source-kyve/Dockerfile b/airbyte-integrations/connectors/source-kyve/Dockerfile index 13c92bb1b460..5f1079c2ce22 100644 --- a/airbyte-integrations/connectors/source-kyve/Dockerfile +++ b/airbyte-integrations/connectors/source-kyve/Dockerfile @@ -34,5 +34,5 @@ COPY source_kyve ./source_kyve ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-kyve diff --git a/airbyte-integrations/connectors/source-kyve/README.md b/airbyte-integrations/connectors/source-kyve/README.md index 9c864ea7d8f3..481e5466ffdb 100644 --- a/airbyte-integrations/connectors/source-kyve/README.md +++ b/airbyte-integrations/connectors/source-kyve/README.md @@ -1,141 +1,29 @@ -# Kyve Source +# KYVE -This is the repository for the Kyve source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/kyve). +This page contains the setup guide and reference information for the **KYVE** source connector. -## Local development +The KYVE Data Pipeline enables easy import of KYVE data into any data warehouse or destination +supported by [Airbyte](https://airbyte.com/). With the `ELT` format, data analysts and engineers can now confidently source KYVE data without worrying about its validity or reliability. -### Prerequisites +For information about how to set up an end-to-end pipeline with this connector, see [the documentation](https://docs.kyve.network/data_engineers/accessing_data/elt_pipeline/overview). -**To iterate on this connector, make sure to complete this prerequisites section.** +## Source configuration setup -#### Minimum Python version required `= 3.9.0` +1. In order to create an ELT pipeline with KYVE source you should specify the **`Pool-ID`** of the [KYVE storage pool](https://app.kyve.network/#/pools) from which you want to retrieve data. -#### Build & Activate Virtual Environment and install dependencies +2. You can specify a specific **`Bundle-Start-ID`** in case you want to narrow the records that will be retrieved from the pool. You can find the valid bundles in the KYVE app (e.g. [Cosmos Hub pool](https://app.kyve.network/#/pools/0/bundles)). -From this connector directory, create a virtual environment: +3. In order to extract the validated data from KYVE, you can specify the endpoint which will be requested **`KYVE-API URL Base`**. By default, the official KYVE **`mainnet`** endpoint will be used, providing the data of [these pools](https://app.kyve.network/#/pools). -```sh -python -m venv .venv -``` + ***Note:*** + KYVE Network consists of three individual networks: *Korellia* is the `devnet` used for development purposes, *Kaon* is the `testnet` used for testing purposes, and **`mainnet`** is the official network. Although through Kaon and Korellia validated data can be used for development purposes, it is recommended to only trust the data validated on Mainnet. -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: +## Multiple pools +You can fetch with one source configuration more than one pool simultaneously. You just need to specify the **`Pool-IDs`** and the **`Bundle-Start-IDs`** for the KYVE storage pool you want to archive separated with comma. -```sh -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` +## Changelog -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle - -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: - -```sh -./gradlew :airbyte-integrations:connectors:source-kyve:build -``` - -### Locally running the connector - -```sh -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - -### Locally running the connector docker image - -#### Build - -First, make sure you build the latest Docker image: - -```sh -docker build . -t airbyte/source-kyve:dev -``` - -You can also build the connector image via Gradle: - -```sh -./gradlew :airbyte-integrations:connectors:source-kyve:airbyteDocker -``` - -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. - -#### Run - -Then run any of the connector commands as follows: - -```sh -docker run --rm airbyte/source-kyve:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-kyve:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-kyve:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-kyve:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json -``` - -## Testing - -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: - -```sh -pip install .[tests] -``` - -### Unit Tests - -To run unit tests locally, from the connector directory run: - -```sh -python -m pytest unit_tests -``` - -### Integration Tests - -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). - -#### Custom Integration tests - -Place custom tests inside `integration_tests/` folder, then, from the connector root, run - -```sh -python -m pytest integration_tests -``` - -#### Acceptance Tests - -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. -If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run - -```sh -python -m pytest integration_tests -p integration_tests.acceptance -``` - -To run your integration tests with docker - -### Using gradle to run tests - -All commands should be run from airbyte project root. -To run unit tests: - -```sh -./gradlew :airbyte-integrations:connectors:source-kyve:unitTest -``` - -To run acceptance and custom integration tests: - -```sh -./gradlew :airbyte-integrations:connectors:source-kyve:integrationTest -``` +| Version | Date | Subject | +| :------ |:---------|:-----------------------------------------------------| +| 0.1.0 | 25-05-23 | Initial release of KYVE source connector | +| 0.2.0 | 10-11-23 | Update KYVE source to support to Mainnet and Testnet | \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-kyve/acceptance-test-config.yml b/airbyte-integrations/connectors/source-kyve/acceptance-test-config.yml index d4daf4793898..af31203535c0 100644 --- a/airbyte-integrations/connectors/source-kyve/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-kyve/acceptance-test-config.yml @@ -6,12 +6,12 @@ acceptance_tests: spec: tests: - spec_path: "source_kyve/spec.yaml" - config_path: "secrets/config.json" + config_path: "integration_tests/config.json" connection: tests: - - config_path: "secrets/config.json" + - config_path: "integration_tests/config.json" status: "succeed" - - config_path: "secrets/config_multiple_pools.json" + - config_path: "integration_tests/multiple_pools_config.json" status: "succeed" - config_path: "integration_tests/invalid_config.json" status: "failed" @@ -19,12 +19,12 @@ acceptance_tests: bypass_reason: "Schema tests fail by default as the offset is not present in the data but important for the pagination." basic_read: tests: - - config_path: "secrets/config.json" + - config_path: "integration_tests/config.json" timeout_seconds: 60 - - config_path: "secrets/config_multiple_pools.json" + - config_path: "integration_tests/multiple_pools_config.json" timeout_seconds: 60 full_refresh: tests: - - config_path: "secrets/config.json" + - config_path: "integration_tests/config.json" incremental: bypass_reason: "Schema tests fail by default as as the offset is not present in the data but important for the pagination." diff --git a/airbyte-integrations/connectors/source-kyve/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-kyve/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-kyve/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-kyve/build.gradle b/airbyte-integrations/connectors/source-kyve/build.gradle deleted file mode 100644 index a14286e26086..000000000000 --- a/airbyte-integrations/connectors/source-kyve/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_kyve' -} diff --git a/airbyte-integrations/connectors/source-kyve/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-kyve/integration_tests/acceptance.py index ca7e3877a08f..77f078a20c15 100644 --- a/airbyte-integrations/connectors/source-kyve/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-kyve/integration_tests/acceptance.py @@ -12,7 +12,11 @@ def pytest_collection_modifyitems(config, items): skip_cursor = pytest.mark.skip(reason="MANUALLY SKIPPED: Cursor never in schema") for item in items: - if "test_defined_cursors_exist_in_schema" in item.name or "test_read_sequential_slices" in item.name or "test_two_sequential_reads" in item.name: + if ( + "test_defined_cursors_exist_in_schema" in item.name + or "test_read_sequential_slices" in item.name + or "test_two_sequential_reads" in item.name + ): item.add_marker(skip_cursor) diff --git a/airbyte-integrations/connectors/source-kyve/integration_tests/config.json b/airbyte-integrations/connectors/source-kyve/integration_tests/config.json index 396fb936447e..91029e14957b 100644 --- a/airbyte-integrations/connectors/source-kyve/integration_tests/config.json +++ b/airbyte-integrations/connectors/source-kyve/integration_tests/config.json @@ -1,7 +1,7 @@ { "pool_ids": "0", "start_ids": "0", - "url_base": "https://api.korellia.kyve.network", + "url_base": "https://api-eu-1.kyve.network", "page_size": 1, "max_pages": 2 } diff --git a/airbyte-integrations/connectors/source-kyve/integration_tests/multiple_pools_config.json b/airbyte-integrations/connectors/source-kyve/integration_tests/multiple_pools_config.json index 408a9dd4d401..1cda6d14729b 100644 --- a/airbyte-integrations/connectors/source-kyve/integration_tests/multiple_pools_config.json +++ b/airbyte-integrations/connectors/source-kyve/integration_tests/multiple_pools_config.json @@ -1,7 +1,7 @@ { "pool_ids": "0,1", "start_ids": "0,0", - "url_base": "https://api.korellia.kyve.network", + "url_base": "https://api-eu-1.kyve.network", "page_size": 1, "max_pages": 2 } diff --git a/airbyte-integrations/connectors/source-kyve/metadata.yaml b/airbyte-integrations/connectors/source-kyve/metadata.yaml index f3240f90b9af..6f2b6cfc27ec 100644 --- a/airbyte-integrations/connectors/source-kyve/metadata.yaml +++ b/airbyte-integrations/connectors/source-kyve/metadata.yaml @@ -2,11 +2,11 @@ data: connectorSubtype: api connectorType: source definitionId: 60a1efcc-c31c-4c63-b508-5b48b6a9f4a6 - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 maxSecondsBetweenMessages: 7200 dockerRepository: airbyte/source-kyve githubIssueLabel: source-kyve - icon: kyve.svg + icon: icon.svg license: MIT name: KYVE registries: diff --git a/airbyte-integrations/connectors/source-kyve/setup.py b/airbyte-integrations/connectors/source-kyve/setup.py index 0ed10d850cea..1d6d5bbde662 100644 --- a/airbyte-integrations/connectors/source-kyve/setup.py +++ b/airbyte-integrations/connectors/source-kyve/setup.py @@ -17,12 +17,12 @@ setup( name="source_kyve", - description="Source implementation for Kyve.", - author="Airbyte", - author_email="contact@airbyte.io", + description="Source implementation for KYVE.", + author="KYVE Core Team", + author_email="security@kyve.network", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/bitcoin/block.json b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/bitcoin/block.json deleted file mode 100644 index 735b6990dc29..000000000000 --- a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/bitcoin/block.json +++ /dev/null @@ -1,84 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "hash": { - "type": ["null", "string"] - }, - "height": { - "type": ["null", "integer"] - }, - "version": { - "type": ["null", "integer"] - }, - "versionHex": { - "type": ["null", "string"] - }, - "merkleroot": { - "type": ["null", "string"] - }, - "time": { - "type": ["null", "integer"] - }, - "mediantime": { - "type": ["null", "integer"] - }, - "nonce": { - "type": ["null", "integer"] - }, - "bits": { - "type": ["null", "string"] - }, - "difficulty": { - "type": ["null", "number"] - }, - "chainwork": { - "type": ["null", "string"] - }, - "nTx": { - "type": ["null", "integer"] - }, - "previousblockhash": { - "type": ["null", "string"] - }, - "nextblockhash": { - "type": ["null", "string"] - }, - "strippedsize": { - "type": ["null", "integer"] - }, - "size": { - "type": ["null", "integer"] - }, - "weight": { - "type": ["null", "integer"] - }, - "tx": { - "type": ["null", "array"], - "items": { - "$ref": "bitcoin/transaction.json" - } - } - }, - "required": [ - "hash", - "height", - "version", - "versionHex", - "merkleroot", - "time", - "mediantime", - "nonce", - "bits", - "difficulty", - "chainwork", - "nTx", - "previousblockhash", - "nextblockhash", - "strippedsize", - "size", - "weight", - "tx" - ] -} diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/bitcoin/transaction.json b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/bitcoin/transaction.json deleted file mode 100644 index 817aab822182..000000000000 --- a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/bitcoin/transaction.json +++ /dev/null @@ -1,113 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "txid": { - "type": ["null", "string"] - }, - "hash": { - "type": ["null", "string"] - }, - "version": { - "type": ["null", "integer"] - }, - "size": { - "type": ["null", "integer"] - }, - "vsize": { - "type": ["null", "integer"] - }, - "weight": { - "type": ["null", "integer"] - }, - "locktime": { - "type": ["null", "integer"] - }, - "vin": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "txid": { - "type": ["null", "string"] - }, - "vout": { - "type": ["null", "integer"] - }, - "scriptSig": { - "type": ["null", "object"], - "properties": { - "asm": { - "type": ["null", "string"] - }, - "hex": { - "type": ["null", "string"] - } - }, - "required": ["asm", "hex"] - }, - "sequence": { - "type": ["null", "integer"] - } - }, - "required": ["txid", "vout", "scriptSig", "sequence"] - } - }, - "vout": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "value": { - "type": ["null", "number"] - }, - "n": { - "type": ["null", "integer"] - }, - "scriptPubKey": { - "type": ["null", "object"], - "properties": { - "asm": { - "type": ["null", "string"] - }, - "desc": { - "type": ["null", "string"] - }, - "hex": { - "type": ["null", "string"] - }, - "address": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - } - }, - "required": ["asm", "desc", "hex", "address", "type"] - } - }, - "required": ["value", "n", "scriptPubKey"] - } - }, - "fee": { - "type": ["null", "number"] - }, - "hex": { - "type": ["null", "string"] - } - }, - "required": [ - "txid", - "hash", - "version", - "size", - "vsize", - "weight", - "locktime", - "vin", - "vout", - "fee", - "hex" - ] -} diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/celo/block.json b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/celo/block.json deleted file mode 100644 index a7aa400be894..000000000000 --- a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/celo/block.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "hash": { - "type": ["null", "string"] - }, - "parentHash": { - "type": ["null", "string"] - }, - "number": { - "type": ["null", "integer"] - }, - "timestamp": { - "type": ["null", "integer"] - }, - "nonce": { - "type": ["null", "string"] - }, - "difficulty": { - "type": ["null", "integer"] - }, - "gasLimit": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "hex": { - "type": ["null", "string"] - } - }, - "required": ["type", "hex"] - }, - "gasUsed": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "hex": { - "type": ["null", "string"] - } - }, - "required": ["type", "hex"] - }, - "miner": { - "type": ["null", "string"] - }, - "transactions": { - "type": ["null", "array"], - "items": { - "$ref": "celo/transaction.json" - } - }, - "_difficulty": { - "type": "null" - } - }, - "required": [ - "hash", - "parentHash", - "number", - "timestamp", - "nonce", - "difficulty", - "gasLimit", - "gasUsed", - "miner", - "transactions", - "_difficulty" - ] -} diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/celo/transaction.json b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/celo/transaction.json deleted file mode 100644 index 6a48246b53af..000000000000 --- a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/celo/transaction.json +++ /dev/null @@ -1,108 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "hash": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "integer"] - }, - "accessList": { - "type": "null" - }, - "blockHash": { - "type": ["null", "string"] - }, - "blockNumber": { - "type": ["null", "integer"] - }, - "transactionIndex": { - "type": ["null", "integer"] - }, - "from": { - "type": ["null", "string"] - }, - "gasPrice": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "hex": { - "type": ["null", "string"] - } - }, - "required": ["type", "hex"] - }, - "gasLimit": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "hex": { - "type": ["null", "string"] - } - }, - "required": ["type", "hex"] - }, - "to": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "type": { - "type": ["null", "string"] - }, - "hex": { - "type": ["null", "string"] - } - }, - "required": ["type", "hex"] - }, - "nonce": { - "type": ["null", "integer"] - }, - "data": { - "type": ["null", "string"] - }, - "r": { - "type": ["null", "string"] - }, - "s": { - "type": ["null", "string"] - }, - "v": { - "type": ["null", "integer"] - }, - "creates": { - "type": "null" - }, - "chainId": { - "type": ["null", "integer"] - } - }, - "required": [ - "hash", - "type", - "accessList", - "blockHash", - "blockNumber", - "transactionIndex", - "from", - "gasPrice", - "gasLimit", - "to", - "value", - "nonce", - "data", - "r", - "s", - "v", - "creates", - "chainId" - ] -} diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/cosmos/block.json b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/cosmos/block.json deleted file mode 100644 index 9e64350abfe9..000000000000 --- a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/cosmos/block.json +++ /dev/null @@ -1,232 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "block_id": { - "type": ["null", "object"], - "properties": { - "hash": { - "type": ["null", "string"] - }, - "part_set_header": { - "type": ["null", "object"], - "properties": { - "total": { - "type": ["null", "integer"] - }, - "hash": { - "type": ["null", "string"] - } - }, - "required": ["total", "hash"] - } - }, - "required": ["hash", "part_set_header"] - }, - "block": { - "type": ["null", "object"], - "properties": { - "header": { - "type": ["null", "object"], - "properties": { - "version": { - "type": ["null", "object"], - "properties": { - "block": { - "type": ["null", "string"] - }, - "app": { - "type": ["null", "string"] - } - }, - "required": ["block", "app"] - }, - "chain_id": { - "type": ["null", "string"] - }, - "height": { - "type": ["null", "string"] - }, - "time": { - "type": ["null", "string"] - }, - "last_block_id": { - "type": ["null", "object"], - "properties": { - "hash": { - "type": ["null", "string"] - }, - "part_set_header": { - "type": ["null", "object"], - "properties": { - "total": { - "type": ["null", "integer"] - }, - "hash": { - "type": ["null", "string"] - } - }, - "required": ["total", "hash"] - } - }, - "required": ["hash", "part_set_header"] - }, - "last_commit_hash": { - "type": ["null", "string"] - }, - "data_hash": { - "type": ["null", "string"] - }, - "validators_hash": { - "type": ["null", "string"] - }, - "next_validators_hash": { - "type": ["null", "string"] - }, - "consensus_hash": { - "type": ["null", "string"] - }, - "app_hash": { - "type": ["null", "string"] - }, - "last_results_hash": { - "type": ["null", "string"] - }, - "evidence_hash": { - "type": ["null", "string"] - }, - "proposer_address": { - "type": ["null", "string"] - } - }, - "required": [ - "version", - "chain_id", - "height", - "time", - "last_block_id", - "last_commit_hash", - "data_hash", - "validators_hash", - "next_validators_hash", - "consensus_hash", - "app_hash", - "last_results_hash", - "evidence_hash", - "proposer_address" - ] - }, - "data": { - "type": ["null", "object"], - "properties": { - "txs": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - } - }, - "required": ["txs"] - }, - "evidence": { - "type": ["null", "object"], - "properties": { - "evidence": { - "type": ["null", "array"], - "items": { - "items": {} - } - } - }, - "required": ["evidence"] - }, - "last_commit": { - "type": ["null", "object"], - "properties": { - "height": { - "type": ["null", "string"] - }, - "round": { - "type": ["null", "integer"] - }, - "block_id": { - "type": ["null", "object"], - "properties": { - "hash": { - "type": ["null", "string"] - }, - "part_set_header": { - "type": ["null", "object"], - "properties": { - "total": { - "type": ["null", "integer"] - }, - "hash": { - "type": ["null", "string"] - } - }, - "required": ["total", "hash"] - } - }, - "required": ["hash", "part_set_header"] - }, - "signatures": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "block_id_flag": { - "type": ["null", "string"] - }, - "validator_address": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "signature": { - "type": ["null", "string"] - } - }, - "required": [ - "block_id_flag", - "validator_address", - "timestamp", - "signature" - ] - } - } - }, - "required": ["height", "round", "block_id", "signatures"] - } - }, - "required": ["header", "data", "evidence", "last_commit"] - }, - "__kyve": { - "type": ["null", "object"], - "properties": { - "block": { - "type": ["null", "object"], - "properties": { - "data": { - "type": ["null", "object"], - "properties": { - "parsed_txs": { - "type": ["null", "array"], - "items": { - "$ref": "cosmos/parsedtxs.json" - } - } - }, - "required": ["parsed_txs"] - } - }, - "required": ["data"] - } - }, - "required": ["block"] - } - }, - "required": ["block_id", "block", "__kyve"] -} diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/cosmos/parsedtxs.json b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/cosmos/parsedtxs.json deleted file mode 100644 index 44584e37b678..000000000000 --- a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/cosmos/parsedtxs.json +++ /dev/null @@ -1,132 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "height": { - "type": ["null", "string"] - }, - "txhash": { - "type": ["null", "string"] - }, - "codespace": { - "type": ["null", "string"] - }, - "code": { - "type": ["null", "integer"] - }, - "data": { - "type": ["null", "string"] - }, - "raw_log": { - "type": ["null", "string"] - }, - "logs": { - "type": ["null", "array"], - "items": {} - }, - "info": { - "type": ["null", "string"] - }, - "gas_wanted": { - "type": ["null", "string"] - }, - "gas_used": { - "type": ["null", "string"] - }, - "tx": { - "type": ["null", "object"], - "properties": { - "@type": { - "type": ["null", "string"] - }, - "body": { - "type": ["null", "object"], - "properties": { - "messages": { - "type": ["null", "array"], - "items": {} - }, - "memo": { - "type": ["null", "string"] - }, - "timeout_height": { - "type": ["null", "string"] - }, - "extension_options": { - "type": ["null", "array"], - "items": {} - }, - "non_critical_extension_options": { - "type": ["null", "array"], - "items": {} - } - }, - "required": [ - "messages", - "memo", - "timeout_height", - "extension_options", - "non_critical_extension_options" - ] - }, - "auth_info": { - "type": ["null", "object"], - "properties": { - "signer_infos": { - "type": ["null", "array"], - "items": {} - }, - "fee": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "array"], - "items": {} - }, - "gas_limit": { - "type": ["null", "string"] - }, - "payer": { - "type": ["null", "string"] - }, - "granter": { - "type": ["null", "string"] - } - }, - "required": ["amount", "gas_limit", "payer", "granter"] - } - }, - "required": ["signer_infos", "fee"] - }, - "signatures": { - "type": "array", - "items": {} - } - }, - "required": ["@type", "body", "auth_info", "signatures"] - }, - "timestamp": { - "type": ["null", "string"] - }, - "events": { - "type": ["null", "array"], - "items": {} - } - }, - "required": [ - "height", - "txhash", - "codespace", - "code", - "data", - "raw_log", - "logs", - "info", - "gas_wanted", - "gas_used", - "tx", - "timestamp", - "events" - ] -} diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/evm/block.json b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/evm/block.json deleted file mode 100644 index e3d4b5365385..000000000000 --- a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/evm/block.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "hash": { - "type": ["null", "string"] - }, - "miner": { - "type": ["null", "string"] - }, - "number": { - "type": ["null", "integer"] - }, - "gasUsed": { - "type": ["null", "object"], - "properties": { - "hex": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - } - }, - "required": ["hex", "type"] - }, - "gasLimit": { - "type": ["null", "object"], - "properties": { - "hex": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - } - }, - "required": ["hex", "type"] - }, - "extraData": { - "type": ["null", "string"] - }, - "timestamp": { - "type": ["null", "integer"] - }, - "difficulty": { - "type": ["null", "integer"] - }, - "parentHash": { - "type": ["null", "string"] - }, - "_difficulty": { - "type": ["null", "object"], - "properties": { - "hex": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - } - }, - "required": ["hex", "type"] - }, - "transactions": { - "type": ["null", "array"], - "items": { - "$ref": "evm/transaction.json" - } - } - }, - "required": [ - "hash", - "miner", - "number", - "gasUsed", - "gasLimit", - "extraData", - "timestamp", - "difficulty", - "parentHash", - "_difficulty", - "transactions" - ] -} diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/evm/transaction.json b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/evm/transaction.json deleted file mode 100644 index b0c818fe7240..000000000000 --- a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/evm/transaction.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "additionalProperties": true, - "properties": { - "r": { - "type": ["null", "string"] - }, - "s": { - "type": ["null", "string"] - }, - "v": { - "type": ["null", "integer"] - }, - "to": { - "type": ["string", "null"] - }, - "raw": { - "type": ["null", "string"] - }, - "data": { - "type": ["null", "string"] - }, - "from": { - "type": ["null", "string"] - }, - "hash": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "integer"] - }, - "nonce": { - "type": ["null", "integer"] - }, - "value": { - "type": ["null", "object"], - "properties": { - "hex": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - } - }, - "required": ["hex", "type"] - }, - "chainId": { - "type": ["null", "integer"] - }, - "creates": { - "type": ["null", "string"] - }, - "gasLimit": { - "type": ["null", "object"], - "properties": { - "hex": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - } - }, - "required": ["hex", "type"] - }, - "gasPrice": { - "type": ["null", "object"], - "properties": { - "hex": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - } - }, - "required": ["hex", "type"] - }, - "blockHash": { - "type": ["null", "string"] - }, - "accessList": { - "type": "null" - }, - "blockNumber": { - "type": ["null", "integer"] - }, - "transactionIndex": { - "type": ["null", "integer"] - } - }, - "required": [ - "r", - "s", - "v", - "to", - "data", - "from", - "hash", - "type", - "nonce", - "value", - "chainId", - "creates", - "gasLimit", - "gasPrice", - "blockHash", - "accessList", - "blockNumber", - "transactionIndex" - ] -} diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/uniswap/event.json b/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/uniswap/event.json deleted file mode 100644 index 9d945a06b04f..000000000000 --- a/airbyte-integrations/connectors/source-kyve/source_kyve/schemas/uniswap/event.json +++ /dev/null @@ -1,55 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "array", - "contains": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "blockNumber": { - "type": ["null", "integer"] - }, - "blockHash": { - "type": ["null", "string"] - }, - "transactionIndex": { - "type": ["null", "integer"] - }, - "removed": { - "type": ["null", "boolean"] - }, - "address": { - "type": ["null", "string"] - }, - "data": { - "type": ["null", "string"] - }, - "topics": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "transactionHash": { - "type": ["null", "string"] - }, - "logIndex": { - "type": ["null", "integer"] - }, - "parsedEvent": { - "type": ["null", "object"] - } - }, - "required": [ - "blockNumber", - "blockHash", - "transactionIndex", - "removed", - "address", - "data", - "topics", - "transactionHash", - "logIndex", - "parsedEvent" - ] - } -} diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/spec.yaml b/airbyte-integrations/connectors/source-kyve/source_kyve/spec.yaml index dae37e3ca58c..be0330feef98 100644 --- a/airbyte-integrations/connectors/source-kyve/source_kyve/spec.yaml +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/spec.yaml @@ -6,6 +6,7 @@ connectionSpecification: required: - pool_ids - start_ids + - url_base properties: pool_ids: type: string @@ -20,7 +21,7 @@ connectionSpecification: title: Bundle-Start-IDs description: The start-id defines, from which bundle id the pipeline should - start to extract the data (Comma separated) + start to extract the data. (Comma separated) order: 1 examples: - "0" @@ -29,11 +30,11 @@ connectionSpecification: type: string title: KYVE-API URL Base description: URL to the KYVE Chain API. - default: https://api.korellia.kyve.network + default: https://api.kyve.network order: 2 examples: + - https://api.kaon.kyve.network/ - https://api.korellia.kyve.network/ - - https://api.beta.kyve.network/ max_pages: type: integer description: diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/stream.py b/airbyte-integrations/connectors/source-kyve/source_kyve/stream.py index 31184963c9ac..6688a5832f2f 100644 --- a/airbyte-integrations/connectors/source-kyve/source_kyve/stream.py +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/stream.py @@ -1,28 +1,37 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - import gzip +import hashlib import json import logging from typing import Any, Iterable, Mapping, MutableMapping, Optional import requests from airbyte_cdk.sources.streams import IncrementalMixin -from airbyte_cdk.sources.streams.core import package_name_from_class from airbyte_cdk.sources.streams.http import HttpStream -from source_kyve.util import CustomResourceSchemaLoader +from source_kyve.utils import query_endpoint logger = logging.getLogger("airbyte") -# this mapping handles the schema to runtime relation -# this needs to be updated whenever a new schema is integrated -runtime_to_root_file_mapping = { - "@kyvejs/bitcoin": "bitcoin/block", - "@kyvejs/celo": "celo/block", - "@kyvejs/cosmos": "cosmos/block", - "@kyvejs/evm": "evm/block", - "@kyvejs/uniswap": "uniswap/event", +# 1: Arweave +# 2: Irys +# 3: KYVE Storage-Provider +storage_provider_gateways = { + "1": [ + "arweave.net/", + "arweave.dev/", + "c7fqu7cwmsb7dsibz2pqicn2gjwn35amtazqg642ettzftl3dk2a.arweave.net/", + "hkz3zh4oo432n4pxnvylnqjm7nbyeitajmeiwtkttijgyuvfc3sq.arweave.net/", + ], + "2": [ + "arweave.net/", + "https://gateway.irys.xyz/", + "arweave.dev/", + "c7fqu7cwmsb7dsibz2pqicn2gjwn35amtazqg642ettzftl3dk2a.arweave.net/", + "hkz3zh4oo432n4pxnvylnqjm7nbyeitajmeiwtkttijgyuvfc3sq.arweave.net/", + ], + "3": ["https://storage.kyve.network/"], } @@ -55,27 +64,20 @@ def __init__(self, config: Mapping[str, Any], pool_data: Mapping[str, Any], **kw self._cursor_value = None def get_json_schema(self) -> Mapping[str, Any]: - # this is KYVE's default schema, if a root_schema is defined - # the ResourceSchemaLoader automatically resolves the dependency + # This is KYVE's default schema and won't be changed. schema = { "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", - "properties": {"key": {"type": "integer"}, "value": {"type": "object"}}, + "properties": {"key": {"type": "string"}, "value": {"type": "object"}}, "required": ["key", "value"], } - # in case we have defined a schema file, we can get it from the mapping - schema_root_file = runtime_to_root_file_mapping.get(self.runtime, None) - # we update the default schema in case there is a root_file - if schema_root_file: - inlay_schema = CustomResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema(schema_root_file) - schema["properties"]["value"] = inlay_schema return schema def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: - return f"/kyve/query/v1beta1/finalized_bundles/{self.pool_id}" + return f"/kyve/v1/bundles/{self.pool_id}" def request_params( self, @@ -115,25 +117,41 @@ def parse_response( for bundle in bundles: storage_id = bundle.get("storage_id") - # retrieve file from Arweave - response_from_arweave = requests.get(f"https://arweave.net/{storage_id}") - - if not response.ok: + storage_provider_id = bundle.get("storage_provider_id") + + # Load endpoints for each storage_provider + gateway_endpoints = storage_provider_gateways.get(storage_provider_id) + + # If storage_provider provides gateway_endpoints, query endpoint - otherwise stop syncing. + if gateway_endpoints is not None: + # Try to query each endpoint in the given order and break loop if query was successful + # If no endpoint is successful, skip the bundle + for endpoint in gateway_endpoints: + response_from_storage_provider = query_endpoint(f"{endpoint}{storage_id}") + if response_from_storage_provider is not None: + break + else: + logger.error(f"couldn't query any endpoint successfully with storage_id {storage_id}; skipping bundle...") + continue + else: + logger.error(f"storage provider with id {storage_provider_id} is not supported ") + raise Exception("unsupported storage provider") + + if not response_from_storage_provider.ok: + # TODO: add fallback to different storage provider in case resource is unavailable logger.error(f"Reading bundle {storage_id} with status code {response.status_code}") - # todo future: this is a temporary fix until the bugs with Arweave are solved - continue + try: - decompressed = gzip.decompress(response_from_arweave.content) + decompressed = gzip.decompress(response_from_storage_provider.content) except gzip.BadGzipFile as e: logger.error(f"Decompressing bundle {storage_id} failed with '{e}'") - # todo future: this is a temporary fix until the bugs with Arweave are solved - # todo future: usually this exception should fail continue - # todo future: fail on incorrect hash, enabled after regenesis - # bundle_hash = bundle.get("bundle_hash") - # local_hash = hmac.new(b"", msg=decompressed, digestmod=hashlib.sha256).digest().hex() - # assert local_hash == bundle_hash, print("HASHES DO NOT MATCH") + # Compare hash of the downloaded data from Arweave with the hash from KYVE. + # This is required to make sure, that the Arweave Gateway provided the correct data. + bundle_hash = bundle.get("data_hash") + local_hash = hashlib.sha256(response_from_storage_provider.content).hexdigest() + assert local_hash == bundle_hash, print("HASHES DO NOT MATCH") decompressed_as_json = json.loads(decompressed) # extract the value from the key -> value mapping diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/util.py b/airbyte-integrations/connectors/source-kyve/source_kyve/util.py deleted file mode 100644 index 23ee80e078dd..000000000000 --- a/airbyte-integrations/connectors/source-kyve/source_kyve/util.py +++ /dev/null @@ -1,28 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import importlib -import os - -import jsonref -from airbyte_cdk.sources.utils.schema_helpers import JsonFileLoader, ResourceSchemaLoader, resolve_ref_links - - -# We custom implemented the class to remove the requirement that $ref's have to be resolved from the 'shared folder' -class CustomResourceSchemaLoader(ResourceSchemaLoader): - """JSONSchema loader from package resources""" - - def _resolve_schema_references(self, raw_schema: dict) -> dict: - """ - Resolve links to external references and move it to local "definitions" map. - - :param raw_schema jsonschema to lookup for external links. - :return JSON serializable object with references without external dependencies. - """ - - package = importlib.import_module(self.package_name) - base = os.path.dirname(package.__file__) + "/" - resolved = jsonref.JsonRef.replace_refs(raw_schema, loader=JsonFileLoader(base, "schemas/"), base_uri=base) - resolved = resolve_ref_links(resolved) - return resolved diff --git a/airbyte-integrations/connectors/source-kyve/source_kyve/utils.py b/airbyte-integrations/connectors/source-kyve/source_kyve/utils.py new file mode 100644 index 000000000000..cc5648bbb03c --- /dev/null +++ b/airbyte-integrations/connectors/source-kyve/source_kyve/utils.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import requests + + +def query_endpoint(endpoint): + try: + response = requests.get("https://" + endpoint) + return response + except requests.exceptions.RequestException as e: + print(f"Failed to query {endpoint}: {e}") + return None diff --git a/airbyte-integrations/connectors/source-kyve/unit_tests/test_data.py b/airbyte-integrations/connectors/source-kyve/unit_tests/test_data.py index 102469c49b3c..9b6aeffdc8ac 100644 --- a/airbyte-integrations/connectors/source-kyve/unit_tests/test_data.py +++ b/airbyte-integrations/connectors/source-kyve/unit_tests/test_data.py @@ -2,4 +2,4 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -MOCK_RESPONSE_BINARY = b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03\xed}]\x8fd7r\xe5\x7f\xe9g=\x90\xc1 \x19\x9c\xb7\xcc\xac\xca\xb5\x81\x85\xe1\x97}2\x8c\x05\xc9 =\x82G\x9a\x81Z\xb3\xb0a\xcc\x7f\xdfCf\xb5\xa4\xce\xbe\xb2>"\xe5I\t\xaaBw\xe5\xc7\xbd\xbc\xfc8\x87\x11\xc1\x88 \xff\xe5\xbf\xde\xfd\xfb\xf8\xcfw\x7fx\xe7]\xe2\x1c\xc9\xbd\xfb\xec\xdd\xff\xab\x7f\xfa\xebx\xf7\x87\xffz\xf7\xc7\xfa\xfe\x8f\xf8\xca\xfdG\xa8n\xe4\xe0\xcb\xa4\x16)\xc8\x0c\xa3uWcW\xf6\xa5\x17j\x93\xbaw\x8e\xe3\xf0U\xe6\xa8\x83\x89h\xf6!\xa5IK$\x19\x85\xfe\xa5~5\xbe\xfc\xfa\x1f>\x94\xe8F\xcbS\x1c.\xe8\xadi\xee3\x87\x1aS\x8a<)y\x1e\xc1\x87T\x9d\x06.\xa9\xd7^gvEk#\xad\xd1\xc7\x11\x93L\x94\xf8\xe5_\xbfh\xe3\xabw\x7f\xf8P\xf3\xcf\xde}\xfd\xf9\x17\xe3\xfd\xd7\xf5\x8b\xbf\xe0\xc3\x94c\xcc)\x14\xc1\x85\x7f\xfe\xb2\x8f\xdbS\xef~P\x8a~>\xe7\xe7\xfd\xaf\x7f\xfa\x1a\x9d\x80"\xfe\xad\xbe\xff\xdf\x9f\x7f\xf1\xf9\xd7\xab\xf9_\xff\xe7_\xd6m\xe7\xcf\xff\xed\x9fn\x8f\xfa\xec\xdd\x1f\xc7\x7f\xdc\n\xa2D\xb1\xa2\x80\xbf\xed[\xfe\xcf\xfb\xa1?t\xc7\xbe\xf6\x8b\xcf\xbf\\U\xc6{-\xd2\x82\x8b\xb5(\x870]"e\xee\xfe%2u\xed\x14_/\x17wu\x95W\x15\xc7\x7f|\xfdU}\xa9_\xd7}#>\xc0\xdb/\xdf\xd7\xfe\xf5\xe7\x7f\xfe\xf2\xfd\xbb?\xfc\xcb\xbf~\xf6\xae\xd5\xf7\xe3:\xc6?\x8f\xaf\xfeW}\xffC5\xe1*>w\xb9\xd5\xe8\xff~\xb7\x03~\xb0\x05\x7f\xfb\xdbgw\x80\xf1\x87\x80\xe1\xe45\x8b\xf0djap)\r\xcd\x95)\xc9\xb7>\xda\x18\xd2Z\xf3h\x9e\'J\x03#\x9e<\xad7\x82\x11v\x94\xeb\xf8\x140\x0f\x80\xe0\x1d`\xfc1`\xca\xb3\x00&U4\xf9c\xd0\xa0\x03\xbc\x9bE$\xc5\x1az\x0c\xaf\xfe\xc5\x15\x96V\\o.\x05\x96\xd7\xe4\xc0\x8f\xf8\xec\xa0\xa1C\xd0Hs\xc49\xf5\x18eP\x9e#\x15\x8dqt\xaa\xa3\xb8\xd4\xc9\x8d\xc1\xb1u\xe9\x92\x9d\x14\x99\xbe\x0e\x9a\xa4\x14\x93\x0fcr?\x98e\x1e\x00\xc3;\xd0\xd0\x11h\xd8\xa5g\x01\xcd\xdd,\xc3%\xbc\xcc:\xa3\xa4<\xaf\'w\n\xe7\x97v\xe9\xe9\xd2\xea9\x8c\xd7\xa9\xf3\x94}\x10\x96g\x07L8\x04\x8co:\xb4\xb0\xe6\xc8Rf\xe7\x16ZeI\x01S\xa8\x96X\xc6(\xdd\x0f\xe7\x00\x8e\x92\xd3L@U\x8d\xade\xc1\x17\x1c]\xf5\x9f\x02\xe6\x01\x10\xbc\x03L8\x04\x8c\xa7g\x01L\x8aC\xe2/\x06\x9a\xef\x8a\x84\x9c\xfc\xf4K\xba\xd7>\x8bw\xcdq.\xc9\x85YH]\x9c\x144\xc5\xde\x83\x8c\xac\x1e\x9d\x8a\t\xbc\x84\xa0\x9a\x0b.s\x99\x17\x17o\xed@w\xd4\xde\xc7{\xf4\xc8{t\xc8\x97\x7f\xfd\xd3\x9f\x00\xcd?\xfd\xb9\xff\xfb?<\x10\x18\xbb\xc0\x7f\xfat,\xbfm\xdf?~\xa9\xab\x13Q\x9d\xf9\xd5\x9f\xbf\xb8\t\xf5D\x1eH\xf1\xd7\x94{\xca\x04\xf5&\x9e\x01\xa0k\xc8W\x06\x96:\xf9p\x1d|zy\xb7\x87\xe7\x9f\xbf\xfa\xbc\x8f\x1f\x1c\x9fU\x0e4\x85\x0fc\xfa\xa3`\xd0C\xbc\xdd\xf0\xf5\x9for\x83\x9c\x0b\xb1y\xf6y\x96Y\xf4\xf5\xc4\xae\xe4\x16\xb3hp\x97\x90\xc3\xe5\x05]\xed\xf8\xbb,\xfb\xc1Y\xe6\r\xbf\xec\x1d\x07\xc8R\xfd\x80\x826\xbb\xc6\x9c\xfd=\xaa\x7f\xfc\x0fI\x87\x1c\xfe\xf9\xf7\xaf\x1f\xb1\xdd\xee\x86\xedv\xdf\x8d\xcf\x87L\x89!$\n%\x94\xb4\xfeJ\x82r\x82O\xe2\xfa\x0c\xa2*@G\x96\xf5]p\xf8\xce\xa7%\xbe(\xe0\xd3%\xc6B\xc4\xb7\xb8*$|\x06\xcd p`\xdc\x19\xf0y\xc2?\xc8\xc5\xc4\xbbD\x94\xb9_A\xf7\xc6O\xd8\xdf`6[/m\xf5\xefn\x95\xb2\xeb\x1f\xf1\x84\xb0\x9f\xebnOD\xbdW\xfd#Z\x98W+\xd6\xd3\xf09Z\xb0\xbf[\x17\xad\xda\xfa\xf5!\xfew\xab\xcd\xbb\xee\xb4\xda\x85O\xd6u\xab\x05\xb4{\x88\xf75\xab\xed\x98bP"\xfe\xae\xa7\x86\xf5}\x0e\xb7:0\xee\xa7\xfd\xdd\xea\xbb\xd5;\xb4\xcb\xa4\xb4J-\xab\x07\xf0\xad\xa0\x84\xd5\x9fe?\x1d\xa5\xe2\xfd\xea\xb5Ug\xd4\x065F-WK\xf6\xd8@\xe1\n\xab\xa6\xbb\xe7W\xc3p}\xd8O\r\xbb\x9c\xbc\xfa\x1c\x1fg\xfc\xc6\xd5\xf3k\xd4\xf0\xbb\xfa\x7f\x95C\xebI\xab\x0c|\xb7\xda\xbd\xeaGh\x15\xe4\xc3\xea\xb3\xb4\xeb\xba[\xb7\xfa\xeb\xd6#\x82\xefp\xc5\xea\xc97L\xc8\x1a\xf3\xd5\xb7a\xbd\xf6\xbbgW9\x06\xfe\xdd\xc6o\xa3k\xf54\xfa\x00\xaf\x1d\x9e{\xc3\x1b\xda\x92V\xff\xad\xb6\xa3Q\xabf\xbb^\xab6\x84\x7fy\xf7\t\xda\xb4\xc6f\xdd\x89\xef\xd2\xee\xc9\xb4\xdb\xba\xfas!\x93w/\xc9\x1e\x99\xb4K\x96\x85\xe4\xdd\xd65\xeey\xf7\x9el\xe4\xd2\xae\xc1\xe2\xc4\r!kT6\xf2Q\x98\xbf\xdd\xb3\xc7m\x8d\xbf\xdb\xbd!\xbbWd\xf7\xe3\xfa\x9f\xd6\xd3V\r\xf0\xca\xdfz9\xad^\xe6\x1bo\xd6\xa7\xbbN\xb2_e\\\xb9z\x94\x17\xe7v\xbd\xd0\xb2\xf5\xa47\\\xfap\xab[\xde\xe3\xe77\xe2\xfcn\xed\xdb\xc8\xef_\xd9\xc8\x977\xec\xf0\x1e\xad\xdb\xc8\xf3\xc6\xcb*e\xa1\x847\x87\xc3\x1aw\xb4a\x11\x11\xa5c\xc2\xbd\xc9\xe3Yz-3\xeb\xc8\xb3\xe6\x86);NN\x10`Qjh\r/f\xd5\x12\xa8\xe6\x9e\x99f(\x9a\xdb\xac5W\x82\xae\x82\x81\\\xd2\xec\xfd\x9bX\x9c\x01\xdam\xef\xb5\x01\x83\x90\xc3\x0e\xeaoOP\x82{\x18\xb5M\xc8\xae\xdaU\xa4D\xa1\xe4|\xa1\xc9\x19\xf2q\xe9\x02\x9aZ\xd8\xa6\x1e\x84\xa1\x80\x16\x9f\xbd\xeb_\x8d\xfa\xf5x\xffA\x00\xf7?\xd6\xcf\xbf\xfcG\xa8\x1b\x98\xb7\xfd\xdf\xfe\xae\xca"\x1f*\x8baB2\xb7\xe2=\x86\xb2\xb6\x04-\x8c\xc6\x9a#b\xed\xb1\xad!@g\xf9.\x15\x8d/\xc3\xc3\xd8\x8c\x13\x16\x80\xeb,\x1e}\xdf\x0e\xac\x8b\x07h\x19w\xca"\x1f+\x8b\xfc,\xca\xe2\x9du\xe1\x1d\xf5\xe0\xd8cZ\xd0tu\xe7\xb4\x0cP\x85\x19\x1a\x08f7E\x7fm\x10\xe3\xa1>\xbbu\x11\x0f\x01\x83\xd2\xc1*P\x00\x8a\x91rI\x99u\xaa\xa4\xd2XK\xea\xa9\x005\x91\x93KN|\xac\x83\xc7\xa8C@R\x97G\x04O\xbd;X\xc3\xb0C\xf0\x0e0\xf1\x180\xcfj\x8e.\xf95_24\xff\xe4T\xd3\x85_N\xb3\xa6$\x9cZ>\x9f\x13\xd15\xd0\x05z\xe9\xb3\x03&\x1d\x02\x06#\xef=7\x95\x9cc%\x187U1\x80\xc2\xd5\xcf0%F\xcc\xd0iv\x9f\xb4z\xd8\x8f1\xd5\xecR\x1d]\x99Q-\x1f\xda\xa7\x80y\x00\x04\xef\x00\x93\x8e\x01\x93\x9f\x140\xe9\x14h)\xa3\xf9\xf5E|u\x90M\x1c\xaf]]>\xc7>2\xc5R+\x95kx\xfaU\xd2|\xbc~\xa1\xd4\xe6\xcc\r\x14\x00D\\\x9f\xa0G\x81\xd0\xf0{\xb5\xa1A\xeb\xa1\xea\x92NGJa\xe4\x9c\xa7K%@\xd0$\x98\xcc\xb9\xf5\x83eu;\x04\xef\x00\x93\x0f\x01C\xcf*\x92~+\x80\x91C\xc0\xcc0\x96\xe6?\x16\xeb\x13\x94R\xdf#\x0c\x1f(z\xe8t\xaa\x05\xd3(\xf4g\x9d\x98\t\x82#\x08\x107\x9b\xa2\xbd.\xd7\xd6d\xa6p\xa0\xc3\xd8!x\x07\x189\x06L|R\xc0\x9c\x8b\x933\xb1\xf4\x90O\xa7\xe5[r\x97y!\x8a3B\xd1\xf7\xe5\x02j\xf8\xc6\xee\xe9\x01S\x8eg\x989\xe7\x08\xa1\x972aB@h\xb4N\xadB\xf9\x84\xae\x9fz\r\x93\xa9\x0f\x14\x00)\x8c7\xb5\x177a\x02\xf4\x06Sc(\x0c\x8bO\x01\xf3\x00\x08\xde\x01\xa6\x1c\x03\xe6YE\x92`\xdc\x8a\x17\xaf\xc5\xbd\xf8(\xed\xdcjO\xcc\x97\x97+\xe6\x98\xf3j\xb2\x10\x80\x93\x9e\x1c0\xe1\xd8\xd3;\xe3X\x8b\x0f\x89\xca\xa4\x8c\x86\xf5\xea\xa4\xa3\xb5S\x134{\x8c $\xc2Z\xde\xe8\x13\x86\xce\xcc\xb1\xa9\xd3Qk\xf7\xaa\x82;\x0eD\xd2\x03 \xf81`\xc2\xa1\xa7\x97\xe9i\x1cwyiew\x8ao,\xd3\x05j\xa1\xce\x0c\xe3OZ\t\xe7s\xa9I\xdc+\xbf\xf4SP\xf1P\x82\x9d>;h\x8e\xbd\xbd\x82Y\x80\x07\x9a3\x97i\x93\xfa\x18NJ\xf6}\x84\x1a\x849\rjk\xb1%\x07L\xaf0\x85a$\x13l\x9a\xa8\xbe7J\x18\xba\x83Y\xc6\x0e\xc3;\xd0\x1cz{\xf1\x8cg\x01\xcd\xbd\xa5\xf4\x1b\x01\xcc\xb1\xa77\xe4\x0c\x15m\xa4\xe8hd\x073\x981\xdc\xea\'\xd4W\x99!\x8e\xde\n\x95\x10\x1d\x0fR\x0c-\xf49\xdf2*E\xa1S\xd1D\x07\x8e;;\x04\xef\x00s\xec\xe9\rO#\x96|\xe5\xd1>\x06M+\\|AWq\xea\x91\xdc\xb5\xbf@>\x8350\xba\xa3\x0f\xa7\x1e[\xac\x12\xb6\xf7\xfd\xa9As\xec\xed\x05"rL=\xf4\xe5\x89\xcbn\xd6\xa5x@msA\xd9\xade\x94&\xb5Bq\xad1\xc1\xfc\x95\xc25\xf9\xc1\x12\x9a\x13W\x81\x9d\x83\xf5\x18;\x0c\xef@s\xec\xed\rO#\x9a\xeef\x19\xf7B/\x189\xea\x11}{n)\x9cj\xb8\xa2+\xd1z~q\x058*\xcb7{}v\xc0\x1c\xaf\xf8z\x17j\x89>Ch0\xe6\x85\x04Y\xe2&\xde@`\xb8\x06\x08\x15\x98\xca\x1e\xedKn*Cp@C\x99m\xf8\x06\x83\xb8\xb2K\x07\xca\xef\x03 x\x07\x98\xe3\x15_\xf6O\n\x98tr\xd4\xeb\x84\xa5\xe8]&\x89\x0c\xf3)\x90\\\x840\x8b\xe6$\xc5g\xbe\x005\xcf\x0e\x98\xe3\x15_P\xbf\xb8\xea\xa0wj\x95\xa5_D\xc0\x03\xea\x9c\x1b\x8d\xb9\x00/\x19\xf3@qP\xfd5I\x8f\xd9\x0bl\xe3BPO\x94\x14\xaa\xec\x81\xf2k\x87\xe0\x1d`\x8eW|9<)`\xf8\x82\x0e\x8a\xa7Q\xda\x0b,\xc5kX\x9e\xa5rE\x9f\xe4\x1e\xe5\x12j\xf3\xd2\xae\xe9\xf5\xd9\xcd\xebp\xbc\xe2\xdb:\xcf\x99\x9c\'\xe0\xc5\xd7\xd2\x12Z#m\xc5!B\x94P\xcd\xc2c\xf6\xa5\xb5z*1\xc2\xbc\x11\xc0\xa1-?7\xb4\x93vd^?\x00\x82w\x809^\xf1\xe5\xa7Q|\x03Fh~\x0c\x9a\xeb\xf2oc.\xbd\xc4\x17\x91\x13\xb0\x92\xf0:\x87\xcc\xf2Z\\k\xc5\x9f\xbc\x7f\r\xa5=;h\x8eW}\xdb\xf4%+\x0c]\x1e\t\xa6K\xaf\xe83\x9d\x98\x0b\x8a\x06\xe9\xa3E(\xf9#f\xd5\x00zH[\xeb\t\xd3w\xd7\xa4\x17\x92\xd1\x0e\xf4\x98\x07\xc0\xf0\x0e4\xc7\xab\xbe\xfc\xbb_\xe9\x97\x05\xcc\xf1\xaa/\xa5\xe0\x8a0&\xce6k\x86\xd4 \xc7\xb0\x83\'Io\x10\x12]c\xa3<\x80\x16\x11\xf6,0}\x86/K\xff\x80\x19P\xa7;\x02\x8c\x1d\x82w\x809^\xf5\x8dO#\x96\x0e\xc2\x1c\x1f\x08\x9a\xefz\x8d#\xc6&B\xd6\xe7\x811\xc1\xe4\xdc\x0b\xec\x08\xd6\xees\x9b3\x86\xca\xdc\xc0P\x8c\x14A\xc2c\xf6\x1b-\xc7\x1c[\x9d\x03S\xfb\\\xbe\xf2\x1f\x1d\xe6\xf8\x00`\x1c\x849\xee\xb1\xfc=\xcc\xf1(\xcc\x91\x1e\x1f\xe6X\x7f\xfe\xfd\xeb\xe7\xd7\x1f\xe6\xb8\xc3\xa7h\x87\xc6\xdd\x02\xe4h\x87\xd5\xa5\x1d\xf8\x96w\x90T\xd8\xa1~\xb7\x10\xb9\xb2\xc3\xe9\xfc\x0e\x98\xdaar+\xc4o\x87\xfb\xc9\x0eV\xe3\x1d\xc6\xe7V\x99;\x10\xd0\xa5[t!\xefp\xb4\x15\xd6\xb5B\x1d\xfd\x0e\x04\\AY\xf60G\xda\x01\x95i\x05\xab\xed0\xb3\x15\x02\x98v\xa8\\\xdc5\x0e;`m\x85\x1b\xde\x02\xc6V;\xca\n\x94\xdbW\xc7\x1d\x00\xc7o\x81\x8be\x87\x06\xba]\xd6-@Lv\x18\xe5\xae\xff\n\x85[\xa1\x87\xcb\xbd\xb9\xc2\x0c\xd3\xad\xec]\xf2\n\x98[%\xeeP\xb4\xb8[\xfe\x16\xee\xb8\xc3\x0e\xe9\x16B\x1an\xe1\x8bn\x07\xbd\xed\x9e]\x01\x83+\x90\xee\xd6\x8a]\xe7U\x17\xda\xe5\xa7\xb7\xf0D\xde\xbd\x98n\xc1x\xeb]X!\x94\xbc\xbf\x95\xb7~\xbd\x8d\x9a\xdf\xe1\x88\xb4[\x99w;V\x00fz\x0b\xa9\xf3\xbb\x07V\xd0\xe3-TQ\xc2\x87 \xc9\xb2\xc3:Wp\xa0\xec\xf0\xc7\x15\x9a\x17w\xcbn\xbf\xb7\xd0O~k\xcf\nr\r\xbb\xf5\x14\xc8<~\xbf\x87\xa9\xfe]\xc3T\xdf}\x08s\xf4\tJ,W\x14\x19Y0\xb7\xc3\xa6\x19\x9e\xdb\x8a\xfd\x0fI`/7\x18\x00\xaet\xbc\x05\x9cp\x01\x94\x80\t\r&\xcf\xd6g\xd8)"\xb70\xc70"\x8d\x99\x96\x93\xa9\xc3\x90bR\x9e\xa1\xf5\xd8G]\x05\xa7F\xa8\xcc\xc8Y2\xfb\x15x\x05I\x88B\xab#\xd7\x1b\x86m{R\x9f?\xcc1\x1c{|\xb3\xa3\xd00\xaf\xc7\xee#,\xef*\t\xfa[\xa2RC\xeaiy\xb2\xd9\x8b\x0c\t=\x08\xf9\x9c3\xfad%\xb0,\x87J\x84$?\xc8\xbc{\x80\x96q\xa7,\x1e{|\xe3\xb3\xc6\x94\xc4\xe8\xc1\xef\xad&:w\xc1\xff0\xb0\xda9_}\x17>\xd5\xd8\xf4\xec\xd1\x95\xfc\xec\xd6\x05\x1f{|\x05z}\x8f\x03@\x01%JJ\xae\x0f\xb7r)\xf3\xf0u\xc6\xd1z\xaf2+\xb1\xcc\xb54\xc1i\xa6Q}\xa5\xde\'\xec\xcdR\xd3\xa7\x80y\x00\x04?\x06\x0c\x1f{|\xe3\xb3\x9a\xa3\xbf\x91\xdc^>\xf6\xf6j#\xc2\\I\x98J3\xb4k\xd6\\\xdd\x04z\x94s\x9d\xe2\x94z\xa3*\xb1H\xee\xa5`N/IF!O\rV\xa7\xabZ\x0e\x9cwv\x08\xde\x01\xe6\xd8\xdb\x9b\x9e\xc6\x1c\xfd\x8d\x02\xe6\xd8\xdb\xeb\xa8\xc6\x9a\xaa\xd7\nU\x05\xd2\xba\xb5\x019\xab\xa1\xadx3\x97\x1b\x85\xce\xbe\x8e\xa4\x1d\x06q\x0b*\xb3\xe7\x08k+c\xf6\xc8e\x1e\x84\x07<\x00\x82w\x809\xf6\xf6\xa6g\x15I\'9\x9f\xce\x0e\xfa\x9d\xb2\xf3\xe8\xc1\xd7\xc4+oeB\xb9z\xc9\x1e\xb6\xf8)\x87\xab\xdb+{O\r\x98cO/l\x87\xde\xa6\xf3>\x92j,P1\x82\x962)\xa6\x91|.\xb09\xeb\xf4\x156G\x969\xa1EB\x90T\x1f\xbb\xc4\x04\x9b\x0e\n\xc8\xa7\x80y\x00\x04\xef\x00s\xec\xe9MO#\x92\x98K\xbc\x03M\x893\x00\xe9\xd1\xa3\xd3\xca\na\xe3\x96\xe5\xcc\xa7\xe6ctY.\xab]\xa9\x96gw\xde\xf1\xf7x{y4\xd70n\x90\ra\x9bF\xd0\xef\x05\xff\xf0~{\r89\xcc\x14\xda\x1aT\x90\xd4\xf2(0\x04::\t\x9a\x1d\x8c\x85\x83U\xd2\x07\xc0\xf0\x0e4\xc7\xde\xde\xf4\xac{\x94\xfcFv\x0f\xe0coo\t\xb0\x0b\xfb$\x07\xb1[\xbc\x0b\\{\x8b\x98RG\xcdU\xa7B\xac\x94\n\x8b\'\xc5^3\xacP\xdf\xcb\xd0\x1c\xc7,\xc3ii)\x1ex{\xed\x10\xbc\x03\xcc\xb1\xb77\xbb\'\x05\xcco$6\x96\x8f\xbd\xbd:\xbd\xf4\xe24\xf4\x91\xfb\xec\x92i\x80\xf53v\x0e\xe2\xfc\xda\\d\xad5\x04/\xc3\xc5\xc6\x1e\xaa\x9bV)J-3gv\x07\xb1\xb1\x0f\x80\xe0\x1d`\x8e\xbd\xbd\xf9i\xe2I\x0e\xfc0NN\xa1]r\x9a\xe2Oa\x14\x19z\x8e\xd7\x93\xf7/!\xe2\xf2J\xbe4\'\xd9\xff\x98,\xc2\xefn7\x11\x98\x08\xe6\x82$\x17\xa7\x8c^)\xe8t\xa1y\x9f\xa3J\x87\xc8\x1f$\xb9R\xf6\xa3t\x015\xfbL\xd0\xbc\x87_\x89\x0eu\xafZ\xfch?\xcc\x03\x80q\xe0\x87\xd9c\xf9\xbb\x1f\xe6\xc8\x0f\x13\x1e\xef\x87i?\xff\xfe\xf5\xf3\xeb\xf7\xc3\xec$\xfe\xf8\xcd*\xfe\x07\x8fF\\k\xfb{\x1dx\xa5\xf1\xfb\xb7\x14\xff\xe5\x85\xe0\xb7uf\xb7\xd7\x94\xe9m\xb5\xdc\xefd\xff\xf4\xb6\xee\xcb\xdb3P\xf6\x06\x14y{-\xcaN\xdc\xe7\xbd\x12\xbe\xbc\x07\xb7\x04xZ\x08\xb2\xfc\xecu\xfc\xb5Z\xce7_\xc3^#\xa7\xbd\x02\x8d\xb2\xdf\xb6\x0f\xf0{\xe3\x81\xb5$\x9e\xb6\x7fh\'\xff\xef\xabW;\xd6\'e\xaf\xb3\xcb.+\xbdmi\xe0\xb7\x87\xc0/_\xd1\xf6S\xc5\xede\xd9k\xe3\xdbs\xb1\xd7\xf3\xd7\xfa\xfb\xf2S\xbcmC@{\xab\x81\x0f+\xf8\xe9\xe6wIkM>\xec\r9V-\xdd^\x87\xa7\xed-\xda\xbe\x81\xed3(\xbb7n\xf5\x8b\xdbw\x95\xb7\xb7\x83\xf7/\xbd\xf9\x1f\xe2\xf6\x11\xdcV\xfc\xcb\xaeu\xda\xeb\xfcq\xf7h\xda[~\x84\xbd\xa1\xc2\xcd\xfbp{\xeb\xf6\x16\x05y\xfbS\xde\xbaa\xf7\xc2\xad\x867\x0fL\xd8\xeb\xf4\xe5\xe6\x83\n\xb76\xde\xfe\xf2\x1e\xd3\xbc=\x00o\xde\xa9t\xeb\xcd\xe5\x19`\xf3\xf8\xfd\xeeG\xfb\xbb\xfa\xd1\xbe\xf1\xc3\x10%Y\x8bp\x83|j\x10\x9eN!]\xda\xda\xef\xc3\x03\xa4\xbe\xf6\\g\nc\x14J\x0e\xd2\x9aK\x8ea(z\xcc\xc3\x04\xbae\xfb\xdf\xfc0YZ\xf0\x93\x1bt\x16\xe90\xa3\xb9\xc1>Z\x9b\xd7\xb5\x91\xa0$\x97\x02\x11\xbc\xac\xa6\xd0\xa7@\xd7\x91\xae~\xe8\xe4\xd2\xd0g\xeaR\xfe\xd6\x0f#O\xec\x87\xe1\xe3(/\xe6\xd9`d\x87\x1aG\xa9\x03\xafym\xdb!hj\x86\xb8\x84\xb2\xe1\xb4\x0ffG\xb3\x06\xd7:\xd49\x88\x80\x94\xb4\xc1\x1aXI.\x07\x8b^v-\xe3NY<\x8e\xf2\xca\xcf\xbaJ\xca,\xd7\xfe\x9a\x84\xd2\x85\xaf\xee%\xceX^\xf3\xf5\\\xb3.)\x84\xc9\xf4t\x02\x8b\xf3\xd3\xfba\xbe\'\xb7w2\xe5\xe54[\x89\xa7u6\x85.<2gt\x1a\xa5V\xa2\xc4\x1ce\xedH\x17;5p&\xf9\x11h\xac\xcdPG%\xe2\x03\xc0<\x00\x82w\x809\x8e\xf2\xca\xcf\x9a\xde\x90\xc79\x9e\n\xda#A\x93H\xf4\xa9d\x82\xcd\xf5\xda\xd1\xcd\xe4\x06\xe8\x92\xfc\xcb\xf3\xaf_|\x8f\xa7\xb7\xb9$uE\x93\xa9\x00\x19T\x1cE\xae\xb1\xc2\xf0\xc9\xc5\xf5\x00\x82w\x809\xf6\xf6\x96g\x15K\xbf\x15\xc0\x1c{{\xd3^\xf5\xa8\xe4\xa3\x96\x18\xd9c$Kw\xd4\xa1\xaaB\xcf\x88.z\xa8\x1f\xb3\xeb\x8c\x11\xe6a\xce\xd5;\xf1~\xf4\x90\xfaPr\x07\x1b\xf3=\x00\x82w\x809\xf6\xf6\x96\xa7\x11K\xf7\x96R\x0c\x1a\xaaBy\x93x\x12zq\xd7\xf3\xd9c\x8a%\x98\xe3+h\x02\xbdJq\x9c\xce\xcf\x0e\x98cO\xefpDE|\xcc\xcd\x8b\x0f\xb9\r\\\xad\xa5\x8bw\xcbY\xb9R\n+e\x0c$W\x80\t\xaa,\x04N\x87q\x1c{\nk\xe9\xe1`\xdb\xef\x07@\xf0\x0e0\xc7\x9e\xde\xe79\\\xe0\xb7\x99\xa6\x19\xbf\xc7\xd3+\xa9\xb5RIk\xc5eCF\n\x8eG$u\xbe\x15\x07\xfchS\x85\x12"\x1aK\x0f\xb5\x04?<\xb4\x90\x96\xeb\x8aP\x9e\xc4\x18\x90\xf1\xeb\x8f\'\xd9\xd1\x16i\x1f\xc4p\xcb\xbb\xf5;\xf6!\xbc\xc5\\\xd0>h!\xedx\x91[4\xc2\xce\xed\xddQ\x14\xb7\xdc\xc1\xedkO\xf4\x8d\xc7:oO}\xda\x19\x87\xb7\xa8\x8e\xdb1\x0cn\x97\xb4\xa3\xf5\xdf\x0e\x96X^\xf2G\x1c\x7f\xc1;> \xa5\x9b\xbf~\x1f\xb7\xb2s-\xd3\xdb\x01\x10;\x06c\xbf\x8ao1\x00y\xc7\x06\xf0\xce\xd9t\xfb`\x87\xdd\x82\x9d\x03\xca\xb7\xa8\x8a[\x9e\xf3\x8a\\\xd8\x99\x9b\xb7L\xcax\xcbj\xdeY\xca~\xdf\xb9=\xfe\xbb\xddn=g\x97@;~c\x7f\xb6{\xedvHEN\xb78\x89[d\x89\xdf\xbd\xb7\xa3\x13\xc2-\x1b\xd5\xed\xa7\xa5\xb7\xc8\x9bU\xfa\xb7\x07q\xf8}\xdd-\x92\xa2\xec\xa8\x86\x1du\xb0\x9f\x16o1<;\x0eb\x1d\x80q\x8b\x17\x88\x1fb)\xc2\xed\xd0\x8a\xd5\xe6\xbcG\xf2vP\x08\xbf\x8d\xd1-B!|\x88\xf6y\x8b\xb5\xe1\x1d]\x91v\xbf\xf8\x1d\'\xc3{\xbcn\xd1A\xe1\x96\xd3\xbd3N\x1f\x91\xd7\xfb{<\xd0\xdf1\x1e\xe8\xdd7\xc7\x97\xf0\xdaf<\xb8\xdcZ\xa6ud\xc6\xf2\x1c\xca\x9c.s]\x11 kU\x8eP\rn\x90\xd0sh\x19K\xc5H=B\x99\xd1\xc2K\x9a\xdd\xe2I\\\x85\xe6R\x13\xf0\xab\xec\xb2\xf3\xb5R\x9fN0\x83g\'\xb5s\xa69\'\x14\xc6\xd4\xa0HH\x1bRC\xf4B\r\xcf\xe2\xd2w<\xfe\xaf \x9e$\x1e{{\xc3\xcesv0\x08\xc4uhx\x95J\xca\xbe\x02\xa2=OYk\xda\xc1)\xb4\r C\'\xf4\x13\nM\xd9\xf3l-\x8eX\x8f\xb6\xfe~\x80\x96q\xa7,\x1ez{\xa3\x7fZ\x93\xf4q\x8a\xe2\xdf\x130\xe9\xd8\xdb\x0bU)Q\x17n%\xcc\x91\xa0\xbb\xe1\xda\x8e\x1f\xe5\xb5lI\xcb\x8c\xea\xebh\xd4Rs\x89\xb1\x05\x85|\xd0^X\xab\xf7+\xc3\xf9S\xc0<\x00\x82\x1f\x03&\x1dz{\xa3\x7f\x1a\x93\xf4\xb7\x99\xde\x90\xbe\xe7\xcc^\xc1\xb4\xcc.\x8a\x9f\\\x19\xbav\xaa\xac2\x01\x8d\xbd\xe9\xa5N\xe7sq\x03B\xac\x16\xcdy\xc1\x8aW\n\xf7Z\xd4Z[\x18\x1e\x84\x07\xd8!x\x07\x98COo\xfc\xfd\xbc\x9b_\x180\xc7\x9e^B[0\x84\\ZJM\xc8\x8dR\x0b\xc42D|WJe\xf6\x99\xb2+~\xf4\x9a\xa20\xec\xe5\xb0\xeca\xd1\x0c\x8ep\xf5G\xe7\xf5\xda!x\x07\x98CO\xef>\xc9\xf39\x00\x93\xab\xa7_\x0e4\x1fm\xb9\x1d\x08h(\x1e\xa6\xb3\'\xc9\\j\x84m^\xc8\x85\x1aE \xf5!\xfa\x07\xcf4}_\xb1\x16D\x05w\xb8\xd4C\xab\xaez\xf9)\xeb\x17\x0f\x00\xc6\xc1\xfa\xc5\x1e\xcb\xffv\xfd\xa2\x05i\x1c0\x11\x97\x93\xa43f\x10\x88\xa6S\x89a\\\xa1\x00\xbe\xb6\xec%\xc5\x93\xcc\xf3/\xbd~1\x99\xf8\xbb\xeb\x17\xd1kPth\x8a\xe7F\x973\xa1\xf3\xa7\x94t\x16B\xab\xc6\xb9\xbed\xf1\xd4\xf6"\xffO^\xbf\x80\x86+\xdfY\xbd\xe8\x1c\x86R7.\x1f\x18\x8d\xaf\xff\xe6\xfeY\x03\xb0\x05\xeb\xac4L\xf9\xa0w\xa3\x99G\x1f~\x8c\xd08\x81\xdd\x05\xcaX\xfa\xd6z\xc8\xb9\xc3`\x10\x8aa\x9fp\x0e#\x82]\x9d<\xfc\xca\xe7\xae\xebh\xbde\'\xa4\x98\xf3\x8c\xd9\xad\x8dS\xa9L\xc8f\x86\x80\x16\xc8\x95o\xac\x07R\x1f\xaa\x1fubL\x89\x92\xce\xb1\xb6\xe2\xc3\x1cS\x98\x9d@4A\xf3\r\xa2\x04\x83Fr\xd3V(\xc2\x04\x89\xab\x1c\x12\xd1\xf9\xeeW\xb1+P:\x8e\xb1h\xbd\xc1\xa8\xd7\x08U\xbfS^\xce\xba\xb0\xcf\x88L\x8dy\x1d#\xb06\xd8\xd0\x19\xfa\xc0\xec\x95G-c\xf8\xb9\\\xdf\xdd\xadc\x80\xe4@\x19|\x00\xc7\xef\xa6\xea\xc3\x18\x8b\xf8<\x07\x07\xe5\x1aK\xfdM\xee\xdb\x91\x8e\xe3,\xd2\x14\xc207\x87\x81\x9d\xb3\x89\xf7\xb0\xcf\xbd\x14\xd8\x01\xa3\x8c\x99\xdb\x94u\xd8F\rc\x9d\xdc2j\x96\x918@Z,\xff%\xd4\xbe\x03\x93\xd3\x0e\xc3;\xd0\x1c\xc6Y\xecc\xee\x9e\x034\x1e\x8d\x84$\xfd\xc5\xf2\xea?\xdaf\xa5\x8c\x90gJ\x1e*v\x93\x81~\x0573\xcc0Z\xdbw\xc5\x11 \xe2\xfdR\x94J(.\x94\xa9\xa3\x90\xc22s\xa3\xc6\xe8\xfbO\x91\xf0\x0f\x80\xc6\x91\x84\xe7\x1f\x92\xf0i\xa9\x86/\x99\xce\xde]\xd3\xa9^%r\x91L\xc0\xc6)\xbc\xf6\xf6R\xe4|\xbe\xc0\xae\xff\t\x12\xfe#"\xfd\x0c \xbc\xc9\xf8\xf2\xda\xfd%\xbc\xf4+\xba|B\xa9\x8a\xd73\x0cC\x1ag\xa6\x93\x8f)M\x8d\xact\xce?G\xc6\xfb%r\xbe\x15\xf1\x14\xc5\xc1\x14\xb5\x89\xe8d\xba{\xeb\x92\x96\xfb\x83QC\xf1\xc6\xe3\xe1W\xa8\x8e\xed~\x9b\x8e\xd4\xc5\xb6\xf1(8d\xbc\xdf\x96p\xdb\xc5\x06\xa0.\xd9x\xbf\r@]l\x00\xeab\xdb\xb8\xb7\x8b-\xe1\xbc\x8b\xcdC\x07\xd5\xcdx\xbf\xcd\xc3\xd8e\xda\xee\xb7\xf2\xbf\x18\xf9_\x8c\xfc/F\xfe\x17#\xff\x8b\x91\xff\xc5\xc8\xffb\xe4\x7f1\xf2\xbf\x18\xf9o\xdc\xb8\xbb\x1b7\x9c\xe8\xc6\x00\x83^\x8c\xfc/F\xfe\x17#\xff\x8d\xfb\xa6\xf7j\xe4\x7f5\xf2\xbf\x1a\xf9_\x8d\xfc\xafF\xfeW#\xff\xab\x91\xff\xd5\xc8\xffj\xe4\x7f5\xf2\xbf\x1a\xf9_\x8d\xfc\xafF\xfeW#\xff\xab\x91\xff\xc6\xfdzz3\xf2\xbf\x19\xf9\xdf\x8c\xfcoF\xfe7#\xff\x9b\x91\xff\xcd\xc8\xfff\xe4\x7f3\xf2\xbf\x19\xf9\xdf\x8c\xfcoF\xfe7#\xff\x9b\x91\xff\xcd\xc8\x7fc\x80b\xefF\xfew#\xff\xbb\x91\xff\xdd\xc8\xffn\xe4\x7f7\xf2\xbf\x1b\xf9otqu\xa3\x01\xd9\x8d\nh7\n\xb0n$@\xefF\xfew#\xff\xbb\x91\xff\xb6\xea\xe3~#\xff\xd5\xc8\x7f5\xf2_\x8d\xfcW#\xff\xd5\xc8\x7f5\xf2_\x8d\xfcW#\xff\xd5\xc8\x7f5\xf2_\x8d\xfcW#\xff\xd5\xc8\x7f5\xf2\xdf\x98\xe0\xd0\x87\x91\xff\xc3\xc8\xffa\xe4\xff0\xf2\x7f\x18\xf9?\x8c\xfc\x1fF\xfe\x0f#\xff\x87\x91\xff\xc3\xc8\xffa\xe4\xff0\xf2\x7f\x18\xf9?\x8c\xfc\x1fF\xfe\xdbn\xc7\xfdF\xfeO#\xff\xa7\x91\xff\xd3\xc8\xffi\xe4\xff4\xf2\x7f\x1a\xf9?\x8d\xfc\x9fF\xfeO#\xff\xa7\x91\xff\xd3\xc8\xffi\xe4\xff4\xf2\x7f\xda\x08lT\xffq\xbf\x8d\xffj\x8c\x91Tg\xe3\xbf\x1a\xe3?\xd4\xd9\xf8\xaf\xc6\x00\x12u6\xfe\xab1CV\x9d\x8d\xff\xeal\xfcWg\xe3\xbf:\x1b\xff\xd5\xc8 5f\x18\xab3\xf2\xdf\x98\xdf\xab\xde\xc8\x7fo\xe4\xbf7\xf2\xdf\x1b\xf9\xef\x8d\xfc\xf7F\xfe{#\xff\xbd\x91\xff\xc6\x002\xf5F\xfe{#\xff\x8d\x19\xfa\xea\x8d\xfc\xf7F\xfe{#\xff\x8d)\nJF\xfe\x93\x91\xffd\xe4?\x19\xf9OF\xfe\x93\x91\xffd\xe4\xbf1\x00U\xc9\xc8\x7f2\xf2\x9f\x8c\xfc\'#\xff\xc9\xc8\x7f2\xf2\x9f\x8c\xfc\xb7\xd1\x07\xf7\x1b\xf9o\xdc\xa0B\x83\x91\xff\xc6\x03W\xd4x\xe0\x8e\x06#\xff\x83\x91\xff\xc6\x00r\rF\xfe\x07#\xff\x83\x91\xff\xc1\xc8\xff`\xe4\x7f0\xf2?\x18\xf9oL\x7fP6\xf2\x9f\x8d\xfcg#\xff\xd9\xc8\x7f6\xf2\x9f\x8d\xfcg#\xff\xd9\xc8\x7f6\xf2\x9f\x8d\xfcg#\xff\xd9\xc8\x7f6\xf2\x9f\x8d\xfcg#\xff\x8d\xe7\xc5i4\xf2?\x1a\xf9\x1f\x8d\xfc\x8fF\xfeG#\xff\xa3\x91\xff\xd1\xc8\xffh\xe4\x7f4\xf2?\x1a\xf9\x1f\x8d\xfc\x8fF\xfeG#\xff\xa3\x91\xff\xd1\xc8\x7f\x1b\xfcp\xbf\x91\xff\xc9\xc8\xffd\xe4\x7f2\xf2?\x19\xf9\x9f\x8c\xfcOF\xfe\'#\xff\x93\x91\xff\xc9\xc8\xffd\xe4\x7f2\xf2?\x19\xf9\x9f\x8c\xfcOF\xfe\xdb\xe0\x83\xfb\x8d\xfc\xcfF\xfeg#\xff\xb3\x91\xff\xd9\xc8\xffl\xe4\x7f6\xf2?\x1b\xf9\x9f\x8d\xfc\xcfF\xfeg#\xff\xb3\x91\xff\xd9\xc8\xffl\xe4\x7f6\xf2\xdf\xb8\x7f\x80\x1a\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\xab1\xff_\x8d\xf9\xffj\xcc\xffWc\xfe\xbf\x1a\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\xab1\xff_\x8d\xf9\xffj\xcc\xff7\x86o\xe3~#\xff\x8d\xf9\xffj\xcc\xffWc\xfe\xbf\x1a\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\xab1\xff_\x8d\xf9\xffj\xcc\xffWc\xfe\xbf\x1a\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7fc\xfa\x05\xee7\xf2\xdf\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\xab1\xff_\x8d\xf9\xffj\xcc\xffWc\xfe\xbf\x1a\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\xab1\xff_\x8d\xf9\xffj\xcc\xff7\xa6O\xe1~#\xff\x8d\xf9\xffj\xcc\xffWc\xfe\xbf\x1a\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\xab1\xff_\x8d\xf9\xffj\xcc\xffWc\xfe\xbf\x1a\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7fc\xfa#\xee7\xf2\xdf\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\xab1\xff_\x8d\xf9\xffj\xcc\xffWc\xfe\xbf\x1a\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\xab1\x81^\x8d\t\xb8jL\xe03\xa6/\xe3~#\xff\x8d\x01\xc8j\x0c`Tc\x00\x94\x1a\x03(\xd4\xe8\x80U\xa3\x03G\x8d\x0b\xc0j\\@R\xa3\x01\xaaF\x05V\x8d\x02P\x8d\x04Rc\xfe\xbf\x1a\xf3\xff\x8d\x8f\xc7\xfdF\xfe\x1b\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\xab1\xff_\x8d\xf9\xffj\xcc\xffWc\xfe\xbf\x1a\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\xab1\xff_\x8d\xf9\xff\xc6\xe9\x03\xf7\x1b\xf9o\xcc\xffWc\xfe\xbf\x1a\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\xab1\xff_\x8d\xf9\xffj\xcc\xffWc\xfe\xbf\x1a\xf3\xff\xd5\x98\xff\xaf\xc6\xfc\x7f5\xe6\xff\x1b\xc5?\xee\xb7\xf1\x7f\x18\xf3\xff\x871\xff\x7f\x18\xf3\xff\x871\xff\x7f\x18\xf3\xff\x871\xff\x7f\x18\xf3\xff\x871\xff\x7f\x18\xf3\xff\x871\xff\x7f\x18\xf3\xff\x871\xff\x7f\x18\x198\x8c\xf9\xffF\xf5\xdd\xb9\xce\x9d\x1d\x97\xf0\xfdv\xd4\xdab\xaf\xf3L>\x97XB\x90\xc1yr\x0cu\x1d\x93\\\xa8A\x81\xa83\x1a\x97a\x9d\xf5\x18\x17\xc7\x9c\xc4\xc7u\xe6\xf7\xf7]\xb1\xcf\xcc\xd1L\xcd\xbb\xb9\x8eL\xfb\xf6\xcc\x9c\x1aFo\xeb\x0c\xab\xd6\xa3q=\xdf\x19\xf3\xd9\xff\xe7\x7f\xbcQ\x02\xb8o\x8e\xa8[\xc7\xa9\xfbB^\x96W2\x0e\xf6\xd4\\\xe6\xee\x8aw\xa5\xfbH\xbc\xce\xe1\xc6\xdf2K\xd3\\I\xf3\xa8\xde\x87(\xb5T\xc75~sD]jn\xf6\xe9*y?;\xe5\x91\x99\x12\xe5()\x10\x8d\xc8L\x12\xa5\xf7\x02\x0c\xb2\xafcm\xa1\xd98\xd4\x1a\x94{\xed\xad\x95_\xc7\x01\xd7)\x1e\x1fp\x9d\x1d\x07\t\xda:\x9eU\xbb\x97\x8a92\xe5\x01\x826U\x91\x92\x04:\x0fP\x16\xf7\xe1A3d\xef"\xfar\xce\x9e{\x8b\xf3\xd3\xd3\xc6\x1epH\xd5\xddic\xf1\xf8\xb41y\x96\xd3\xc6\xeeN\x12M\xe1\xac\xa7\x92IB\xa6K\xeb\x92\xeb\x85\xc2I}\x8cte\xe6\xd7\x8b\xab\x85n\xa7\xbf=\xfax\xba\xa8 \x04\x86/=\x020\xe9\x100}5l\xc2\x0e\nUjL.\xb6X\xa97J\xd5\xe7\xc0^\xb9d\xed\xbc\xce\xa6\x9eIi\x0c\xe1\xb1v\xfd\xed\x14f\xd4\xb4\x8f\x83\xbf?\xe0\xda\x0e\xc1;\xc0\xa4C\xc0\x04z\x16\xc0\xa48$\xde\x9d\x8a\xfe\xea\x13\xc7(\xe7q\t\xb10&\x1fN=D\xaa\xe3\xd4S\x08\xce\x0f\xe2\\\xe4\xf2\xd3\x0e\xa7\xa3\xee\x1d\t\xe62\x1f\x05\xe2\xb4*aB\xa4\xc2 d\x86\x80\r\x1a\xf6,\xa9\xd2+\xa6\xbe\x15\xdcF\x81\x92\xe4\xd2\xc0\xd3\\\xdcO9\x9c\xee\x01\xc08:\x9c.\xfd\xd0\xe1t\x9a\xc8\x8f\xe2\xfc5\xe5\xbe\x8e\x8d--\x9e5\xc6k\xc8W\x1e\xa9t\xf2\xe1:\xf8\xf4\xf2K\x1f?\xdbWn\xffw\x8e\xa6[\xa9\n!6\xcf>O\xc8 }=1\xa6\xc4\x16\xb3hp\x97\x90\xc3\xe5%\xa8:\xfe9G\xd3\xb1\x07e\xe2wN\xa7k\xb3k\xcc\x96\xec\n\x92\x1e\x8c\xd1iF\xe3\xc0l^\x1a7\xe7YZap\xa1$N>x\xbc\xe2P0/a\xa6\xc6\xdc\x91\xd2\xfa$\xe2\x93E\x98\xbc(\xb9\x94?|SRH\xb4\xbe\x83\x86\n\x0e\xe3\x15\x855\x9f\x11\xee\xd9W@O\xc9\xb8\x9b1\xe4\x1cp#\xcap\xb8r]\x8d\xfb\xf0O\xf0\x1aO\x00\xe3\x8dji\x87\n\x92\xf6\xd3Q\xe7\x84\x1aA\xdd\xd95G\x8dw\x1d K\xf1\x1aj\xeb\xae\xf3j\x17\x87\xf5*\x07\xd4\xe7\x9b+W\x8d\xf65\x98\x0b\x96\xdaMx\xb5\xda\x00u*\xacr\x05W\xa3\x19\xbbE\x1e\xf7\xa0=\xb8c\xb5p\x95\x82\xb6\xadV\xee\x1a\xac\xcb\x1c\xca\r\xb8\x12\xea;\xbe\xa1]\x8b\xdd\xb3(\x9aV\xb9\xf8\xb7\xee[\xfd\x92q\xf5\xea\xa5\xfd\xbc\xb0\xff\xa0\xff\xd6\x9c\xb4JD\xdd\xf1\'\xee\x1e\xdd\xf7\xecrP?\xd4\x1a\xcf\xdb\xa5\xf8u\xdd.MV\xc9\xfb\xeeu\xc5\xea\xe7\xd5\xc7n\xf5\xfb\xae\xf9\x1aM\xbf\xeb\x8f\xf6\xeek\xca\xad\\\x94\xb9\xeeY\xfdA{\xc4\xdd\x1e\xf3\xf8\xd6jA\x1fPZ}LK\xa6\xe0\xdb5\xea\x19\xdf\x8bqw\x8e=~\x1fp\xb2\xfb\x19m\xc2\xff\xab\x06y\xf7\xc0m|\xc2\xdb\xabU\xff\x80\x7fy\xd7\x7f\xd5j\xa3\x12\xb5\xa1\xddg\xbc\x91X6\x06\xc3n\xf7j\xd9\xc2\xf1\xfan\x8dg\xde#\xc8{d\xdc\xc6\xc3\x1a\r\xde\xa3\x96w\x9f\xd2\xbe\x83o\x9f\xa5\x1b\xf6\xd3\xae\xcf\xc2\xb2l\xf4\xb8\x8d{\xde\xbd\xe66Z\xe2FA\xd9c\x1ev\x1d7?\xc2\xfaf\xbd[\xd7\x95\xdd\xb2\xdb\xf8\x01i\xbbD\xda,\xa2=v\xab?\x17Fp\xfdF\x9e,\xce,\xcem\xac-<\xaf\xe7\x95\xdd\x0eN\x9bm\xbb\xcf\\\x08o\xe8\xdc}\xb1F{\xb3-\xed~A\xad\xf6\xe8\xa5\xcd\x93\xbc{Sv\xbb\xd6H\xd27\xd6\x85\x0fK\x11\x99\xd5W\x8a(\x85\xeb\xb2\x08\xa0\xc4j\xad\x92b\x0b\x91%+\xb4\x10_2D\x19\xfa\xb7\xe1Q\x95{\x9c\xe4D\xf6y\xd07\xeb\x02B{F\x82\x81\xe1\xd3\x0c\xcb*i\xebX\xdd\x01!\xcbnLMu\xcc\x1a\xf0\xae7?[\xa3A\xad\xe1\xfe\xa9\x03\x9aO\xf2\xcc\xbf\xdc\x01\xd8\xd1+\x0c\xd4Y\xe5\x11\xcab>T\x16gV\xef\x84Xg\x97\xa8\n\xeb))S-\xe8\xa7\x1dJ\x92\xa2\x1b9\xf6u\xaeC\x1a\xcb\xa6JM`#\x14\x1d>\'\xa1\x03\xeb\xe2\x01Z\xc6\x9d\xb2\x98\x8f\x95E~\x16e\xf1\xce\xba\x18i\xbc\xc4<\xcf\xc0\xf1\xb5\x9d\x8a\x82\x8e\x97A3^\xaf\'\xcf\xd4\xcf\x94\xca:\x80x>\xfb\xe1\xd7r\x08\x18\xf5I\x8a\x8f^9h\xb2;^\xc3\x98\x05\x8f\xa80\xbfB\x19\x80\x08d\xae\xce\xc1\xbe\xc3\xe6U\x10\xa4+\x8c\xe4\x88Y`\x84\xccY!oj\xd2\xbcVm\xd5\xcb(\xfdS\xd0<\x00\x86\x1f\x83&\xbbc\xd0\xc4g\x01\xcd\xdd,C5\xca\x89\xc3\xe9\x02e\x8a\x95e\xce\xd6e\x86\x8c\x8f\x8b\x1f\xce\x17q\x04+=<;`\xfc\xf1*)\xb3#\xe7}\xed\xd2d\xba\xbc\xc6\x8d\xc4\x95\xe20A\x90\x8fMZ\x0f\x98h\xa1S\xc5\x8cf\x0e.\xb5\xc2$\xa7\xd0\x82\x9b\x85\x0e\xf4\x18;\x04\xef\x00\xe3\x8f\x01\x93\x9f\x140\xeca\xee\xcck\x1a\x98V]}\x15\xa8\xf8\x97\\__<\xecN\'\xbe\xb8H\xe3\n1\xfd\xec\x80\xa1C\xc0$u\xdb\xf4\xa8\xbdaz\x05\x080\x16\x15\xda\xc5(\xb16h#-\xf92\x0b\xda\xd9G\x1f\xa9\xb5\x16 n\x1cc~\xe0QZ>\x10K\x0f\x80\xe0\x1d`\xe8\x180\xe5Y\x00\x13\x18\x16\xca\xc7\xa0q\x97\x1a\xe4\xea\xc2\xa5\x15\x7fy\xcd\xe7\x97\x9c\xdd\xda\xd2\xb4.\xc9t:\x9f\xf8\x1a]\x7f\xa9\xe9\xd9A\x13\x0eA\xe3\xa2\x0e\xf2\xa3\xb5Z{\x84~0\\\xca5e\xc2\x10:/\x14\x01"I\x98G!P\x12A7\x01G`<\x83@\\0=lG\xd4\xbd/\xc6\x0e\xc3;\xd0\x84C\xd0\xc4g\x15K\xbfV\xc0\xfc\xeb\xff\x07`L\xf1{"\x0c\x01\x00' +MOCK_RESPONSE_BINARY = b'\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03\xec\xbdi\x93\xa2\xd8\xb6\xf8\xfd]\xea\xad\xe7\xb6\xb07\x9b\xe1D\xdc\x17\xcc8\x00\x8a(\xc3\x8d\x1b\'\x98\x14\x91I\x01Q\xfeq\xbf\xfb\xb3\xcd\xee>}\xba\xec\xec\xae\x81A\xf1\xdb\x8f\xff\xa5\xfe\x14\xc1\xe5\x1f\xe1\xf5\xe5\xbb_\x10vZ\x96\xbd9\xc4\xed\xbb\xbb\xe0\xf6=\x9ax\xf9\xf5\xe5+\xe7\xba= \xd8%\xff(\xbb\xe2g\x99n\x8f\xa5\x08\x88c\xfa\x97\xa7\xdf~\x1aw\xa7\xa0}\xa9\n_H@p\xbf\x1c\xeb\xe7\x03\xfe\xedw/M\x12\x14\x8b\x18\xfa\xf62\xffT\xe7\xf6:u\x17\xfe\x03\xdb\xe8\x1f\xb73p\xb3\xf3\x97$\x06\x08\xe1\x1aws\xb3\xff\xfb\xcd\x82\xe0[\x8b+bnb\xe0\x8a\x89\xb3\x1cD,C\x91\x14\xc5\x12\x14b$\x91f(\xc8!\x85\x93D\x81\xa1\x15$\xb2\x0c\xc9@\xfc/I@\x01)\x12\x80\xac(\x8b\x7fZ\\EA\x91(\x08x\x02\x1f\x16\xa7N\xc4\x10H"\t\x84\xc3\x99B\n\xafP\x94\x84\xff/\xf2\x8c\xa0p"\x02\x9c\x88\r\x86\xeb9\x12i\x16\'\x04\x96a\xdeV\\_\xf2\xe6KD}C\x99\x05\x7fXf\x01\xf9w\n\x97Y\xf0\x13\x00\x1c\xa2\xb1\x0c\xf0\xcf\n\xee\xe7oL\xfe\xa0\xe0\xd3\x80\x17\xb1\x899@\x0b\x12>\x16!\xd0\x8a\xac\x88\x0c\xcf\xb2\x14\x87\x0f\x0f\x15\x16{-\x96\x12k\x02\x19\x02WAF\x96i\x88_\x85\xc2\xe7\x9e \xc5g\xc1\x7f\xdf\x82\x8f\xbd\x04{\x92\x0c$\x92\x95)^\xa0\x05\x12[\n\x87\x10\xe2Y\xc8\x89\xbc \xb2\x02\x89_KF<#\n\xd8\xb4P\x12hN\x82\x1c\xce\x18\xa2,@\xfeY\xf0\xdfX\xf0\xc9\xdf\n>\xf9\x87\x05\xff\xf3\x87\xf9\xef\x1a\x8e\xdf\xd2\xfc?\xb6y\xb0\xfb\xf2w\xf0\xaf%\xf17C\x03\xe2\xe6\xf6,\x8d=\x95\xe4\t\x82\xbf\xbd\x14\x05D\x89\x16\x05\x9a\'e\x96\x05"\xce\x7f"\xf3K\xa2\xc4e\xbe\xa8\xbf%[\xfeS\x18\xfc\xe0\x82n\x81b4!\x1b\x17Dw\xd8\xb43\x95_\xa9\x82Q$g\xc2^\xae\x8am\xd7L\x90\x9a\x0bI\x1c\\&\xf3>\xac\x17\x8b\x9e\xd5\xc3\xddy\x98\x12NQ_\xf9\xd3\xec\xe0\xfa\xe6\xe2H\x91\xedj\x90*\x14\xba\xfaT\xf8\xb5W\xf865\xbf\xd9\x9f\xfeBMD\xff\xc4A\x12\x11\x90\xe4\xc0\xd7j\xae\xc6\x1bQ./l4>\xef\xfbt5r!\xdc\x1fY\x1e\xa8\x1d\x8a7G\xfe0(\xdc\xa1e\xcc\xb8.+:\xda\xecI\x8eZ*\'\xd1Q\xcc\x8b\x908\xf2\x8a,\x8b\x04\xd6\x19\xbd\x88\x17\xcbKU\xd4{\x10T\x92\xd8\xff\xb1\x9a\xe4\x1f\xab\xf9\x95\xf8\xb8\xa3\xc0\xe2\xdf~\xdb/s\xe3/\xa3\xe3\xbf\x08}k\x89\xfe\xe3\x0e\xff\x8a\x0f\x10P$H\x9a\x17\x18Y\x84@\xe2\x14Vb\x18^\x91\x04\x92#q\xcaBP\xe6\tV\xc2I\xfc/}\x80\xf9\tK\xc624\xe2\xee\\]\x1d\xf8dQ\xae& \xb6N3B?\x00#\xb9\xee*\x86\x9c\xd3\xe3f\xc7\xe6\xd7\xfd\xeelz\xd6\x1e\xe6\x99\x99\xb5D(\x024f\xb8dZ\xa2r2Y\xc9\x15T\x9a\x82\x1b\xa7\xc7\xe3\x99\x95\xe9\x88\xa7\x14\xb4\xe0\x8c7\xba:\xc7q\x10WqD\x11\x8cHp\x02\xa4 \xc2\xd5\x1e\xa7)\xc8\x0b\xb8I\xe4\x91\x08(\x1a@\xf0-\xae\xceQ\x90\xc5\x95\x87\xfaZM\xea\x1am\xa2\xc0\x8e\xf8\xed\xaa\xdb6\n\x13\xf99\xe9\xcb\xecI\xcf\x0eUf\xf9\xdal\xb2\x08f\x96\x9cAs\xbfD\x86\xce\xe9\xd2>0M3]\xba\x94L\xd0k\xbf\xa7\x8d\x9a\\\xe5j\x16\x14\x01q\xddY\x83}\xbc\xf2\xbb\x0fq\xf5W\xf3"\x00\x8a\xcc\x8a"\xc5\xc9\x14KCBdh\x92&d\x92\xc4=\x12\xee\x8c9\\\xbb%@Q\xdfdEH#|\x84\xbb\x84\xb1\xf4\x14\x8f\xe7\xbdY`\x15\xf2dV\x9e\xf9\x03okc\xbf,\xda \x92\xa7\x895\xeey5\xd5w\xe9\xca5w\xd4\xd95\xe5\x91\xb4\x1c\xa9\xd6\xd1\xcd\xf6\xc7\xc8\x1b]\x05\xbdW\xa3ic%m\xbc\x9c\x9f\x83\xa15\xd6\x85\xb8\xfcLV\x84\x12\xc5a\xcdqa\xe3iV\x14D\x11AH\x902!\xcax\xae\x91\x19\x9a\xa5\x05\x01\x8a/\xc3\xd5_[\x11\xf7a\x10\xff\xfa\xda\x8a\xc1ic\xb1\xcbKM2\x99?\x8dM2Y\xc9]_Q\xa7I\x03[\xb9\xd1\xd6\xfa\xde\xa8\xbd\xe1\xc01&k\x8f\xa5\xa4\xa9\x83\xf6\x1c\x91+qC\xdaf\xb7\xb1$j\xd86\xaa0\xc3\x8dp\xb4]\x19M\xdc\xe7`)|\x8c/~P^D\x80\x86,\'\xd0P\xe4y\x86\xa79^\xc2\xe5\x91\xa1EIT\x14\x89\x10qc\'\xd2\xb2\xf4\xd7\xae~\xcb\x8b\x10WT\xdc\xed~}\x92x\xa4MS\xe1\x84\x02j=DZ%\xea\x8a]{;\xa9\xefK\x1e\x06\xc9Ik\xfb\xca;SS?X\xae\x0b\xb6Q\xc7\xc5\xca\xb9\x82c!\x9e\xa73\x12\xae\x07\xd1\x9b\x9ed\xb3\x93\xb4\xc6\x9ds3`u~\x8a\xdb\xc9\x8fq\xf5\xe7\xe1\xdf\xeebx\xd6\xe4i\x88G-\x99U(Yf\xf1\xd4\x05\x11\x87\xfbLA\x06\x88\xc1\xf5\x08\xe2\x99\x88\x93\xbf)\x0f\xb0\x14\xcb\xb08\xa3~\xedb\x1b\xdd8\r\xd7E\xbd\xe1C\x8a\x1b\xa5\xfd\x91[\x83\xd1\xa1\xc0#\xd1\xe4\xc0\x98#I\x026\xe0\x97\x8e\xa5\xd2\xb5\x98\x12\xa3|.\x9dWA>\x19M\xa7\xd3\x1d;\xee4v\x95\xe4\xa1\xb1\x9d\x12\xd5(\xa5P\x9c\xa9N\xdaI\xaf\xb4_\xafE\x12\xa0%Z!\x19\x16\xb7\x142\x9e\x9bEH\xe2)\x80\xe0X\xdcw\xe2 \xa3(\xdc\xe0\x93\x8c\xc4\x7fS$\xe10\xa2\x00}\x17I\xb66R\xd3uA8\xbd\x89\xd2\x89\xeah\xbb\x1e\x0f\xb2\x87zZ\x12\xe5\\Z\xc2\x83\xa0\x0c\xe5V\xb3\x93nv\x1aU\xab*\x14\x8by\xd0\xebm\x1cG\xe6\xdc\x01\xe6\xb9D#%mv#\xb5\t6gq\xb6\xee\xa2\x9d\xf4J\x87\xf1A\xf9HDH$q+&\xdeFp\x81\xc5\xc3\x08\xd6\x10I\x12n\xdc\xb0\xa7@RQh\x1a\xc8\x7f\xdd\xc0`+\x92\x0c\xc9\xe0^\x87\xfb\xda\x8a\xa1N\x0b\xf5\xb1\xd9\xc7\x88\x16\x93M\x13l\x17\x8e\xd5\xfbdg\xd3[&b\xa22^\x9f\xf2\xc3UIa\xb4\x9a\xecNF3m\xec\xc3pE\x93\xd6\xbf\x18~\xe0\xcc\x18G\r\xe6qv\xb1V\xdbDZ0\r\xa1\xfa\xbc\xf8J\xd1x\xad\xc3\x00x\x02\xe1x\x11p$\xcb+<%\x92\x02\xa2%(\xd3\nK\x13,\x94)\xec?\x12.\x9a\xdf\xd6\xa7q\xd8$\xf0NM642\xc9\xbbV\xb8Q\xe9pW8-\x17m\x15\x0b\xdb\xb6\x0e\xce+\xa4\x80Imk\x8ct\xd2\xcb\xf5\xc5_\xb4\xeaq\xba\xde`\xefX^\xc6]\xad\xeaJsv\xb6\n\xea\xdb\x91\xabp;A\x10\xcd\xbd\x17/\xa2\x19\xff\xa9\x9c\x05\x12\x88\xe3xN\xbc5\xf3\x9cB\xe2\x16\x9fgp\xb2\x018\xbdHH\x11\x15Q&e\x8ad\xfez\xb0{)^,\x8b\xd0\x9d\x11\xa35i\xfa\xca\x11\xd2\t\xb7^\x16\xc3:f\xb7I\xc0\xf6G\xb7bv\x8b\xf6T\xad\xc7\xacH\x0e\x8dg\xcc\xe2X\xe0\xf7\xd7m\xcfkPXd\x83a\x9e\x92j\x17y\xee\xd4-\xc2\x95\xa8\xcc\xb6}\xcdr\x9b\xb4\x8eG\xe2\xdbzz\x08\x14\x9e\xe5!\xcb\x0b\x9c\x82n\xca\xd24\x1eW)<\xb3S\x08\n\x10\xf0\x14\t\xb1a\xbfi|e\x19\x0e\xe7\x16\x86\xfeZM\xcfY\xc43y\x84\'QwK"Z\'\xa7\xaaD\x9c\xe7\xfbEX\x19\x8c\x92\x8dm\xfe\xc2\xe6\x07\xc9\x98\xdb\x93\xa9\x846\x92eO\xb6\xfb\x05\xb9\xb7[r\xb7\\\x9c\xcd\xf5r\x96\x07\xd3\xe5q\x1f\x1f\xc8\xf1T\xf0\x85\xd9z\xf1ZO\xff\x8a\x9a\x14\xd6Rdd\x1a\x0f\xe2\xdc\xcbe\x14\x01r4\'I\xa4B\x88\x84\xc4\xe1\xa0\xe7\xa1\xac\x90\xdf4\xba\x90\x0c\r9\xdc\xb8\x7f\xad\xe6X?\xc2\xcb0\'\x81yX\x8f\xf7\x86K\xae\x96\xbeYn)c\x9c\xaf\x92\xc3.\x05|J_\xf83\x1f\xd1\xdcX\xe7\xab\\X\x9f\x93~\xc2*\xcen\x01\xa9\xf0\xe0\xe8\xe9\xd6\xa0\x83\x93{X\xd3\x0e\xdb$\x8b\x04\xeeu\xe9mj"\x99\xc5g\x0f\x97\x08\xfe\xe5\xa2\x1dn\x85YJ\xc2%\x83\xa4\x08\x19\x9f\\Q\xa1\xb1\xca\x04\xfc\xebA\xf4\x16\xf9$\xc0Z2w\xb3\x85\x08\xf3&P\x94juF\x8d3\x8a\xadl6\xcb\xd5\xf3\xf4\xdal\x9a\x8d\xabi\x9be\xaa\x18\x888Pv\'\xd0\xf2\xe2\xdc\x99\x82\xe2\xc4\xdc\xa1_5\x0b\xc1\x90bS\xd7\xb2\x15\x03\xd2\xc5!\x98 \xbd\x8f\'\xd9ul\xbc\xd6\x15\xbf\xa2&\xcd\x13\x92 #\x89\x108 #\x86\xa4h\xc4+\x08\'n\x81ah\x1a\xb1\x90\x15\xf1\xf7_\xae%\xff\xb5\x9a\x04"p\x9b@~\xad&\x9e\xc6\xa4\xb3\x8d\xe2\xe3\xaav\x96g\xfe\xc8s\xbc\x93\xd6\x86@Wa\x93\x84G\xd9d\xbc\xc9\xe945\xe2\xb6\xd8\xb2\xa9C(\xe7\x8e\x0c\xa3"\x9c\xca\xc1\xda<\x16\n3q.\xc6\xba\xf3\xa6\xd6F>Q\x91\x16\x95p\xc1\xffG^sy\xe5$\xb1\xb7K\xb7\x04\x82\xa4,#N\xa0\x08E\x96\x15IF\x00\x07 b \xcd\n\nC\x92\x9c\xfc\xd7\'\t\xe7O\nG,E\xdcW\xa1\xa3m\x0f\x07\xda\x85\xf3\xcd\xe8$\xbdv\xcd\x85\x01x\xc0\xa5\x14\xdc\xc6\x8a\xa2\x0c\xb1\xabs\x02dxN\xa2 \x87\rI\x92H\xc6E\x89\xfa\xa6\xfe\x16\xe0\x86\x85\xe1\xee\xf3352\xcd\xc00M\xab\xdd\xfaC\x9f\xf0\xdc\xe1\xb08\x1f3\xb3R\xd9\xb8\x95\xd8r\xae\x08\xb5\xde\x1c[w\x9e\'u\xac\xcd&\x96VV\xb5\x96j\xc3\xa1\xa8.UoFsi\\\xf6jt\xd4\xf52\x18j\xbd\x12\x96\xd2+#\xd4kjJ\x04\xc0\r,K\xb3\x0c\x90\xa0\x80\x83\x1b\x8a\x88U\x04Y\xe4I\xdc\xb0\xb14\xce\xdf\x80d\xbe)q\xd1\xd4\xad\x93\xbfo\xe3g\x87\xc01I4X\x04\xc5whF_\xb4\xa8\x9a\xb2\xdc:2\x9al\xbc\x88\xbd\xe9\xd5\x8f\x0e\xdc\xa1\x9b\xf8\xd2\xc4=\xb8N\x1b$\x95\x1b+l\xca\xce\xe3\xe8r\x9aK@:/\xa4\xcb\x0eL\x0f\x0b\x9enP\x0b\xf9\xc3kj\xfeGF\xb4D\xb1\xb8\x940\x08\xca\x12\xf69\x89a%E\xe2\x00\xa9 \xfc7\r\x19\x86\xc4\r\x03\x9e\x9f\xd8o\x89h\x00oe\x94\xbd\x1bB\xa8d\xb2\xf5U\xf6\xd2\x93\x87iq5s:O\xd9x\xc9/s\x7f\xebw\xb4\x9d\x12\xe1L]\xd8\xd4\xb1\x14x\x18\x93}\xac\x9f;g\x97\x155\r\x97\x0e\xb4\x8dzM\x19VWL\x9dmE\x1cck\xe4S\x0b\x83\xf9OH\xbb\x7fu\x9b\x07\xf8wn\xf3hOA\xd9l\x93\xd3\xabwW\x80M\xe7\xab\xbf\xbb?"\x84F\x1b\x0e2L4\xa1\x88\x0b\xe5\x14\xda\xd34t"ddS\x14j:\xf0\xb3\x1d\x91d\xc6\xc9w.\x8d\xa9\xead\xa8\x1at\x04\x8d\xf4\xab;0\xa2b\x93\x05\xaeP\xfb\x0e"\xee\x8e\xae\x1c\xe3\xcc#\xa3\xc1(\xa3R)\x8c\xd2\x03A\xd1\x0f\xbe\xc6\x13\xb1\xe4g\xa6\xbbDQ\xd1 \xd31\x0e&v\xc4@\xf3\x1b\xff\xf7G\xf7\x1c\xf2\x1c\xff\xee\xc8\xbf\xd8\xf0\xb7{K\xf0\th\x82]\xf2\xdd4\xff\xd7\xd7\xfe\xe7\xbb\x92\xa7\xa4\x0fN\xf1k2\xe0\x17i\x02\xc7Jc\x95\xbb\x93\x03\xfai\xa8r\xbd\xefN.\x86\xdav\xba\xda\x9e\xf4\xc1\x18\x92R\x80\xb1\xab\xa3\xb0\xd8Qa\xb6\x83~Q\xe7\xa1\x94\x1e\x82r\xd7\'\xb6P\x04\x9a\x95\x06\xc3\xfa\x8d\xb6zyS\xb3y\xb9#\xe2\xb3\x8b\xfa\xb3E\x9bO/\xe7\xc3L\xea\xc3i\x89=\x0e\x8b\x17]\xe3\xcc\x07:\\R\t\x163\x86\xeb!\x82i\x19Kyk\xb8z\xafC\xfc=\xf7p\xd5U\xfdCL\xfa\xfer>\xbd\xf4\x13{\xa9\xbb\x84\x06\xdc\x0c\xb1\x9d^\xf4"M\xa3"\x06\x89\xbb\x19B\xc9\xba$\xe5\x14\xf9\xaeG%RJ\x98n\x9c{`sI\x9c\xe6c\xbc\xf4\xdd\xe5|\x9cI\x9d\xf4\xe4\x17=\x8cT\xa5\x89\xe1\xb4\xc2\xc5\xf2\x1a\xa8\xe9U/\xf3\x02\x8b}\x8d\xec\xf4\xe8\xbbV\x19;\x06a\xa8\xd6!q\xfa\x8f1\xe9\xbb\xcb\xf90\x93\xea\xd2\xf4\x1a\xbb~\x1fg\xd3!*H2,\xfc\xc2/\xd34.-"QQ\xe5\xbbu\x13f~\x11\x16\xc6\xd5W\xeb\x83\x9eE\x1fb\xd2\xf7\x97\xf3q\xb9\x14\x90\xa5\x0f(")\x10\x0e\x9e\xbc\xd4\xb3\r\xf4\xa5\x880q\x80%*\xf6\x8alz\x0c\xd4\x98\xf4qg\xefK\xc4\xa4\xef/\xe7\xc3L\x1a\xa9u\x1b\xe2\\\x14\x03c\xf0\\k\x88\xdd\x1c\x99*1D\xee\x14\x07\x97B\x86\x99\x07#8A\t\xf6\x88\x18\xfb\x89\xaf~Lyz\x7f9\x1f\x17\xf8\x8eA\xc7\xd2\xa6\x0b\xd4K\x87\x9f\x82"W\x1e\x0c\x98b\x11\xf3\xc1ts<\xdd\t\x07\xa3<\\\x8dbI\x06e<\x18\xd2\x075Q\xef.\xe7\xe3L\xeanp\xb5\x9c\x1eb\x8dG\x91c\xe4\x86\xea#o\xd8\x91\xa1\xed\x93\x91\x16\xb7\xa6\x8b\xc5\x96\xf4\xde\x90r\x18Hy\x96\x14\x1f\xd3\xea\xbf\xbf\x9c\x8f\xab\xf8 \x06\x91\x83\x1b\x13\xd7\xe8BG\xe9\x83"\xeeu\xb0\x06>@\'\x13\xe7(}\x98^\xe2\xcc(u\xb8)\rp\xc9b\xe7c\xca\xd3\xfb\xcb\xf90\x93\x86ZZ\xe9\xda\xba7@\x9a\x06\x99G\x18\xa5@\x98\xce\xba\x0f\xa5ichJ\xa6\x83%\x85\xa7\x91"\x91\xb0\x97\x94u\xa5\xab\x1f\xd3D\xbd\xbf\x9c\x8fk\xf5U\xab\xf1\xed\x9c\xf0`\xdd\x98R\\F\xd0o}\xd7Ou\xa0\x13\xa6\x8a\xf2\xb0\xcc;\x7f\xb0:\xc3\xf6Q\x04\x8c4\xd4>h\xc6\x7fw9\x1fgR\xe0\x91\xa1\xea_#hQ\x1e\xf4(\\O\x89D\xd2I}\xf0\xfaX%\x0b#\x9b\x1et5\xcecX\xd3\x11\x8c\x86\xa4\xf8\x98\xf2\xf4\xfer>0\xf0\xeb*TQ\xef9)\xc0m4\xf4K\x8f\x08\n2M\xec)\xe1\x03\xa5\xf2\xec\xcd\t\xf7xd\xe4\xa60t*\x14\x83\x8f1\xe9\xfb\xcb\xf98/\xb5wW}H\x81\x9eY\x17\x0f\x18\xd5\xed"Y\xe4\xe8\xd7x\x10\xae\x86z!#{\t\xf0\x1cM\xc58O\xe1\xf9:\xf7\x1c\xe2c\xbc\xf4\xdd\xe5|\xe0\xf5\xd2C\x7f\xcbCz\x91\x0e\x1eL)\xcf\xcd\x8fz)\x1c\x12w\x8d\xbfN[\x1f\xb7\xcd\xc6\xb0\xe9\xc2\xd2\xaf\xbc\x81\x87\xa6\xfbQ\xd7K\xdf[\xce\xc7\x99\xb4\x88+\x1f\xde\xfeXT\xac\x1aC\xa0V\x84g\xe78\xbd\xf30\xb6o\x8d\x89\xdf\xe9e\\\xf8\xa5r\xf1\x1d\x0f\x1a\xf0\x83L\xfa\xeer>\xcc\xa4\x1e\xb8\xf4\x91;!\xc2rz2\xd4\xb6\xf2AK\x05YD\xf9\x85\x01c\xb0\xc3\x83]\x8f\x87\xbf\x88\xf0]\x9e2U\xaaO\xec\xe5\x87\x98\xf4\xfd\xe5|\x9cIa\x9eG\x9aO\x18N\x9b&\x0e\xae\x96@\x86F\xe1\xdf\xae\xf0\x90\xbe\x94\xb6Q)\x80@2R\x1cl\xc0\x94\x8c>\x86\xde\xc7\x98\xf4\xdd\xe5|\xe0%h\n\xe7 ?\xf55\x03\xb7\xce\x04\xceKq\xa6\x0fq\x16\x00\x1dF\xea&\xf3\x07\xff\x18gS\xda(\xad*Q\xc9C\x00\'\x1ft\t\xfa\xbd\xe5|\xe0\x95(\xbf\xd2\xb3\xe5\xe0\xdb\xeb\xdes\x85\x8b\xae\xd5d,\xf1}\\\xe6\x95/\x19C\xe4lN\xbaZ\x13\x81\xaaS1\xf4\xf1\x9f\x8f\xc9\xa5\xef/\xe7\xc3Lj:\xd55P\x97\xb8A\x11\xf0P\x97\xe2`\xb2\x06SZ\x0e/\x95\xd4i3]Kq\xd2\xb7\x00\xfe?\xe1\xb9\xd3\xc2\x94>\xc6K\xdf_\xce\xc7y\xe9\xa0\xe3I\xc3 t<\x17\x07\xaa\x82\xa7\x10\x812T\x19\xe9\xaa\xd1\xe3\x01\x10E\xee\x06\x9av~\x0c\xf0\xf7Mm\xd9\x9b\xda\xc7\\\x82~\x7f9\x1fw%J\xc3_k\x1b**\xa78\xf5\xc7\xe4\xed\xddE\xc3\xd9\x90\xa6fe~\xb6\x06F\xe9\x9f|\'F\xa1\xda\x0e\x9eSS\x1fu\t\xfa\xfd\xe5|\xdc\x1b%jJ\x04\xd2\xe4j\x80K\x13\x95\nL4?\x8f\xcaMkd\xc6\xd5t7MT \xa6\xe2\xbf\xbf\x9c\x0fl\xa2\xd2\xceP)R\xcfvC\x00,\xe4KV\xaf\x17\x97\x93\xe7\xac\x89\x08\xa0.\x825a\xbau\x86\xcb\xc3\xd5(\xf3\xc2(\xe5\x0fj\xa2\xde[\xce\x87\x994\xb1\x052\x1aR*.\r\x80\x87\xbaK\x00\x1a\xe4\r~\x17K\xb81)s\xd2p\x14\xd2p-\x18\x0f/mv\x13\x96\x1fs\t\xfa\xfd\xe5|\x98Ich\x01]m{|f\x0f\xb8)\x19|\xc7:\x98j\x8b\xa2l\x93F\xe5\x84\xf03\x03\x8b\xbe\xbe\xc6\xa5@%v~\x89\xed\x8f\t\xfc\xf7\x97\xf3q\x15\x1f\xd4M\xa8\x19]T(m\xacF\x83\xef\xc6T\xe4\xc60*\xe3c\xa2\xfa\xa5w\xbb\xac\xabn\n\x9c\xafRS\xdd\x11\xa1\xfa1&}\x7f9\x1fW\x9e\xcaM\x96\x94\x9b\x83\xe9\xa0<\x92x\x10K\xc2)P\xe3*.\x08`HF\xe5g9\x19\xc1\x880\xdc\xc9\x10\x94)\x9eL\xa8\x8f)O\xef.\xe7\xe3\x02_M\x87x0:\xd3\xb6\x9a\xa4Lq\x9f\xd7v\x81f\x0c\xa6C\x91\x86m\xd1\x91\xcb\x13\xbe\x8a\x8a\xa4\xdcAS\xad`\x04?\xa6<\xbd\xbf\x9c\x8f3)\xb8\xddD\x88\xf3\x11\x14\x80^\x1e\xf0\xd4\xcc_#\xb5F1@Y4\xc4T\xe8\xd6\xf8q\x9bC\x80\x8b\x02\x0e\xaaSP~\xcc\xdby\xef/\xe7\xe3L\xea\xe0\x87i\xeb\xc1/\xea\xd2\x1f"\xe8\x0fK\n\xb7\xd2m\xe2\x1a0\x802\xf6\x0c\xe1\x10Iq\x89\xff\x7f\xf0\x80U\xe2\x8a\xfa1&}w9\x1f\xd8\xea\xd7Y,\xf9\x17\x03\xa0>\x82F\x1f\x01\x9c\xfa\xd5\x18$E3D\xc0\xc2\xb3r>D*\xd9D\xaa\x87\xf3\x97\x95\xfa\xf0c\xae\x97\xbe\xbf\x9c\x8f\xbb\xdb\xa4\x88[\x1d\x18 P/$N\xf1\x94_n\xae\x01\xdc\xa1\xc8\xc1\x15\xd5\x8e\xfbP\xb5\x80/-\xa1\xe9\xf4}\x90\xf9t\xf8Ao\x94\xbc\xbf\x9c\x8f\x9b\xf13\xab\xf5\\\xa5\xf5\xa0\x02\xf0\xbc|\x89\xdd\x0342\xeb\x14k\x15\xf4\xe0\xba\xf7\xf1\xd4\x92\x14>\xc4%\x00\xf9\xe5\xb4\x8cJ\xfecf\xfcw\x97\xf3\x81\xb7F\xaca\xe4\xeez]\x13h\xbd\xa8\x80\x0f(\x14\x0c;*,-*\xd1\xf2\xc6w\xf0\xd0\xe7\xd6E0\x18\xb8/L\xa1Q~\xd0%\xe8w\x97\xf3\x81\xf7\x97\xa6\x83)\xc5\xf0\xf6\x89\xa6\xdb\x19\x0f\x8bK\x16\xb9\xd6)\xc4\x83^\xa8\xa2\xcc\x94x2\xca&\x84.-\xaf\x89$\xd0>\xf8\xa0\xcb&\xef.\xe7\x03o,\xdf\xdd\xea\xe7\xd5\x032\x11\xdf\xee\xdeP\xc9>P}`\xba;\xdc6\xc7\x17\xd3Q\x86\xa8\xd4\xf10\xa8\x83\xa8\xf4p\x9e\xfa\x98[#\xde_\xce|\x7fN\xca\xdb\xe7 _\x114P7\xb5\x0f\xd2\x7f\xfdL\xa0\xbe\xfb\xddg\xfbB\x07\x97H\xb09\xb8`\xda\x84\xc08E\xfd\xef\xb7>/\xbf\xfa$\xa0u\x88\x8a\xcd\xf0\xd5\xa3~S\xdc8\x87\xa5\x8eg\x1c\xb9\xc7\xe1W\x06\xd0\xbb&\xce\xaeO2\x1eg\xb9\r\xa5\x03\xb97\x1d\xbf\r\xd5\x14F\xb6\xd0\x07\xa0mc\xc9H\xf1 \xf9\xd5g\xfa>\xa9b\xb1+\x9c\x8c\xdb\xa0V\xf2\xb8\x1a\xae\x89X\x9b\xb6\xb1S\x91\xbe\xb3i\x0c@\xf6~\x99B\xd3\xdeP\x9e\xb3\xb9\xc60\xa7"x\xf81\x143\xed\x18y6O\x1a\x83A\x9a\xee4\x0b\xb5\x8a\xd2\xa5\x94\x88q\xc4\x1b*q\xd5q\xed\xc2y\xe1\xe5\xd2TR.)\xff\xe5\xee\xf2\x1f@\xb1\xd8\xf1{_mo\xf7"\xa2\xdb{\x12>\xa8\x07#\xcb\x9b\xa0\x88o\xd7\xd5RS\x95\x87\xc8\xe9a\xa0YD\xa0\xa6\xa7\xb8\xfcQ\x14s+*\xb47\xd7P\xf5;\xd3\xd1)\xac\x14\n\xc0\xeej\xa8i\xa9K\x16\x918\n\x998$\x1e+\xf2F\x97\xf2\x83\xe1,\x7f\x0c\xc5\x0c\xc9\'\xfc\xb2\x02q\x96\x1e"Wi\xe2a\x03<\xe8#\x1d\x90\xc7\xd8\xf6\x0f\xc9\xedm\x10\xdb\xea\xc3ls0m\xaf\x8f5\xf9\x07Q\x0c\xc4\xb9\x01\x95\x83\x0e\x0f\x83\x0f\x88\x1e\xf7~0\x92\xa6](y}\xe2 \\\x1a*\x88\xdd\x15\x86N{x\xb9\xcfs\xd0\x7f\x0c\xc5\xc22\xbfFE\x8a\xe2ls\t\n\xb2\xd0\xcb\xba\xf3\xed5\xd0q2\xf1U\xe3\xaak\xd3&\xc2\xf5\x0e\x8fd\xb77\xba\xae\x91\xfb\x83\x9c1\\\xb0\x0f\x11Tr\xbf\xacA\xe2\xe2!h\x98\xf6\x06L\xfb\xa0<\x90\xa6\x9d\x1euI\xc0e\xc0\xc7\xc3\x92U\xe0\xe74\xde\x8f\xa2X\xe4\x10\xe0\xf6\x81\xff@\xdbT\xa6\xbb\x84z\x91\x02\xdc\x00\xe2~y\x93&n\x8d\x95\xb32C\xab\x90\xafZ\xb4\xa1\x19D\xfc\xc3\x14h\xdb\xc3\xcd\xbeB%\x99\x7f0\xe0\x12yN\x8b\xc7U\x1e\x1a\x03\x7f\xc5\x1d-\x11e>\x19\xb9\xb7+\x06~\x16\x96\x13\xe4k?HV\x8c4|V\\!3\x8aj\xf0\x06\xe3\x12k)\x1e\x16\xf32*\x14d8\x17\xfc\x82\xd8M\xdd\xf8\x10\x16\xfe1)b\xac<\xffc(\x86\x9b`*\x801H\xb2\x14Dp2\x04E[\x85\x85A\x85\xd2\x9aJ\n\xdcR9\xd6)\xd0\x8cSl[\x83_\xac{S\xfaA\xceX<\xecz\x0f\x90\xd4m7\x87\x0f\xeb\x93\xa9z\xd8-\x97}0\xf8\xc7@\xdb\r\xa1d\x10\x9e\x93\xa6\x864\xcdb\xb5\xb9\x9a\xf6\x0f\xa2\x98.\x19\x07\xbd4p\x0bU_pj\xc7\xcdn\x04b\xd7\x1f\xc2\xb2\x82\x86J\xc2\xd0^\x93\xa1m\xe1VJ9\x86\xee47\x8a\x1f\xa4\x8e\xddj\x17N\x10\x07\x1cWDP\xac\t\x03\x9f\x9d\x18\x1a(.\x8c\x93o\xe7M\xa4M\x8b\xc4M[\\\xe7\x86\x08+\x18\xfd(\x8a\x19\xb6\x95\xe3!\x13\xfa0n\xe3\x82\xc0\x8d\xf0\x1a\xf9C^%\xaa\x87\x02\x9c\x05\x83B\xc1%\xc0\xbf\xfdK\xc4j\x8c\xe3\xed\x07I\xf7\x81d\x95\xb7\x11E\x07\n\xe5K:\x1eSv\xc0\x03:i\xd8K\xac\xec\x06$j\x9cc\xa5q\xfa7\x8e\xbe\x13\x93\xc9\x8f\xa2\x98Q(]\x04ZB\xc7\xee\xe7\x81\n\x84\xae\xdf\x06\x1a\x0fui\r}[\xc0\xf9r}\xd5\xb3\x1d\xb8\xe1\x84\xe2\x97m>?\x88b\xa1\x9d\x9e\xfc2\xc6\xe7\xc4\xcf\xf1\x083\x18\xae\x00\x12m\x8a\xe73\x9d\xf4\xa1\x81\xbb\x8e\xf4h\xde2\xa2kT\x1e\xaciS\xed\xff\xfb\t\x90\xfaK\x80\x14\xfc\xcd\xe4\x7f\x0e\x90R\x08\xc4\xc8\xacH\x08\x02\xc7\xf3\xb4\xc0\x88\x12\'\x89\xb4x[E\xc6#I\xa4\x90 \x03\x85\x85\x8d\xd3UxZ\x10\xb1\xc8\xce\xb4\x9d\xbe\x17D\xa9Q\xed\xf3\xban\xed8\xccw\xeak\x0b\xe9_9\x952\x81\x9dU\x92\x05\x8a\x84,\xe0\x10n&\x18\x11k\x079E\x11h\x9c\x7fp\x84\xf0$\xeec\xbe\xe1T\xd2$I0\x1c\x00wK\xc6w<\x8c\xea\xd4\x9d;\xb4\xadn\x16a;\x02\x8b\xf5\x86J\x8fq\x86\xd6\xa5\x16\xf8\xf9\xce\xdam\x12.\x92\x06\xfe\xe8\xb1\xb1;Tc\xdbL\xccYV\xae\xe7go\xc2g\xa4\x17\x0c\xee0g\x83\xb9\xe2]\x84b`\xa8\xf0s\xf1\x91\x1e\x03\xd3\xfa% \xc1\x90\x04w\xb7\n\xd8\xaa\xb9\xd9\x80hEYoJ\xdd\x9d\x9c&:\x82ue]\xfblR\xf4u\x91\xf1\x193\xb0s%9\t\x91\xb6Z\xf6\xc6H\x89tC\t\xc0LXUs\x84.\r\xaf2\xfd\xda\x99\x14,*GQw\xb0l\xd7\x16\xdeF\xf4x\x0cL\xeb\x97^\x90!\xb8\xdb\xe6\xf3\xbb\x8d\xc7\x1aU\x08\xeb\xa9\xd1:\x84p`\xe4\x10\x98\xf5JpZ_)\xe7\x9db\xed\x16\xfa\xb4\x02\xea\xc9\x9cl\xd0\xec\x1a\xf6\xa9\x97m\x9b\xde?l/\x1co\xeb\xfb\x81\x9e\x99\xba\xbb\x16\xaenSo\x8c\xd5f\x9a\xb1\xba>]\xbf\x11\\"\xe3:+\t@\xc4\x05\x17\xb7*\xb8^\xf0$\x0f\x14\x19\x0f\x0f\xa2\xc8H8\xbd\x01B\xc1\r\xce_\xb0(~>\x9b\x08\xd7\xb4\xbb\x15\xe0]$\xce\xd7\xc1Nmu4\xe3.\t\xd4+\x9f\xf5\xe7\xe2p\xe0\x9b\xd5v\xbd$B\x7fd9\xcb\xc99p\x87\xd2.\xc6\xc4\xca\n\xaa&\xa6\xae!{r\xbba\'\xe8\x9aG#>\x9bq\xfeyt\xe6,\x7f9\xc8\xfc\x1bO\xe5c\x88^wm\xfd\xbfj\x89f\x8b\xc2L4k\x96\xbb9}\xa6\xab3\x91\x96\xa1\xc7Qk\xadJ\x97#\xaaM\xe2X\xe6\xa3n;\x9a\xacl\xd5Js\xe50[\xabu\x97D\xa45?Q6e-\x98\xde\x8a\xe9u+\xcd2R\xde\xe7\x81w\xe5\xdez&\tV\xc1\x15I\x91\xb1\xe72\x04#\xe1\x84F\x0b<\x8eS\xdc\xa4\xe2\xd6\xfb\xd6ZJ\xb8\xcf\xfa\xa6BEP$\xcda\xab}\xad\xa6\xd4-\x86\x02\xca\x9b.UG\xd1\xdc\x13\x18\xa8Y\xb4\x9b\xe3.\xd0g\xdc\xf1|2\x9f\xd0\xe9q\x1b/\xae\xfd\xa5K\xdd\xad\xa0gMR\x01+\tF\x01t\xd75\x04\x93\xfa\x94\x83\x83c\x0b\xa1\xa9DFQ+x\xb6\x9a\x19Y\xfbA\r\xc5+\xad\'\x81\xcb\x0e\x12!\xad\xe0"\x84C\x18\xdd\xd6\xd5C\x1e\xb7\xda2G\xe0*+1\xe4\r\xa3\xc0\xff\xa5\x15I\x88\x1b\n\xec)\xb8\x92\xdd\xad\xad\xdf,\t\xb2\x9c\xf8\xe3&Y/7\xb6\xb7\xd0\x03\x8a\\\xf0\xd3i\x16\x8c6ln\xf3\xcb\xa6\xa3H\xd5\x11\xcemH\x1f\x1b\x8f\x99t\x82V\x13)G3\x82\x96\xd2\xad2\xf5f\x9e\xe1\x95\x8b\x99~\xec\xce\xa5\xb6\xdc\x15\x10\xbe\xc6~\xf9\x91Ak?\x87\x1c\xcb\x92\x14M\xdfe\x7fD$S\xc6\x8a\xb9\xaa[\xa5\xc6rY\xb2\xd6\xeeX\xc2F\tS\x8f\xddo;\xf3\x18\xedbi\xd1f\xbb\x84\x18-\xcd\xd3\x82tO\x06\x07=\x98\xca\xca\xf6Bs\x95\xe6\x18s\x9c\xfb\x1b\xb2\xef\xdbF\x11\t\x81i\xba\xd7\x00d\xef\xcaY\xfb\xb5\x94\xe3\xf4\x0f\xef\\\xa5\xdel\x03\xa3^\x03\xd7\xa9\x96\xfd\xe4p8+\xb67\xdb\x89\x93\x93O\xad\xa8J\x94\xcf9\xa1t#\xdb\x1d\xa5\xb9\xbb\x96\xb81e6;\x9fR\xf1l"\x16V[\xac-\xc0v\xfa\xd0%\x17\xdf\xb4\xcf\xeeT\x1f\xa9\xfa\xe5\xb5a\xecG\xc6\xac\xfd\x9c\xb6H@\xde\xa0\'wV\xf4\x0e]\xe6\xd8\xaa\xd8\xd2\xca\x85OV\xb2\x95l\x98\xb0k\xf5\xc3\xce\xebV]\xc2\x1dG\x97\xc2\xda\xb5\x9b\x1dGii\x9f\xb7\xc2~\xc7\x9a\x97}mo\x86\x11\xef\x9f\x97U\n6\xf3\xbc\x98\xdb\xc8\x1aI\x97\xd1\xd9\x1c\x81\xcb\x1b\xc7\xbd\xc7`\xd6~\xcd\xce8\'\xd1,\xf3\xb5\x9a\x9cS9:\xb19\xe5\x84n7v\x17\xe5\x8c9\xed\x07\xdc\xef\x94W\x9f1\x15y>\xd9\xeb\x86\xb3VNd\xb1BT\xb9\x8c\xc1\xdc\xb6d\x1a\xedK\x13\xcf4\x93\xab4=\x11#\x87Enr\x91!m,\x93i\xed|.\xc0\xcdc0k\xbf:\x0bAs\x1c{W\xe3\xd8\x9c\x143\xb7ppP\xd7a\x14^u\xa9\xb7\'\x93U{`y\xfe\xda\xa2`\x0e\xe6\x99p\xd2\xd7j\xb2\xf5\xc0\xb4;\xcf\x8e\x12Y%P\xed\xb854\xae4k\xaf%Z\x0b\xb8|W&{\xef"\xe2Qk\xdc\xbf6#|\x8c\x15\x1f\xc37\xc35\x0e\xfcDb+\x12\x14$\xeeB\x0e\xae7(J\x13t\x1a\x89\xd6\x8af%4m\xe6\x0e\x88&\x9a\xcc\xd1\xe6v\xc5\xa5\xab\x94\x89\n\x87\xdd\x194\xaezyY7L\x93G\x8e\x19L<\xb4v\xf7a\xad\x1a\xe7QT\xa4\xf3\xbd\xd7\x9249\x9c\xc2\x1c\xbeF6\xfc\x18+>\x86+\xf6K\x8d\xe3 \x9e\xef\xff\x80Bt\xf6sv\xbc\x11\xfdl\x94\x01{0\xe9Q3\xf0\xd9\xb9\x1fC\xefP\xc7\xc3\\\xa1"~!^\x95E\x9az\xedj\xe3L\xc6\xe3#O\x88\x1b\xfd\x08W\xdd\x92\x83eV\xf7\xb3\x02L\xd8\xf64\x1d\x170\xeb2@\xbf\x91\xf9\xc7\x90\x92\x82\xb33\xcbp" \x10\ryJ\x06\xbc K<\x87D\xec1"b$A\xa1\xf9o)r\x08\xb7\xbf\x04\xa4\xc8\xbb\x81\xf5\x8c{_\x96\xf0eT\x83\xda\xa3\x86LSi=/\xc8\x82\x9f\xf8\x97\xf9\x00\tM\x1e\xf3\xd0\xd5G\xfc\xd2\x90\xc6Y\xb8\x8c6z\xa6P\x89`O\xfaumM+\x8f\x98\xd7\x0b?F\xb4q\x99A\x8e\xe9\x8d\x93G~\xae\x90{\x0c\xdf\xecg+\xe2N\x03\xf7\xe7\xf70\xd1\xbe\xa4\ng^^k#U\xbbv`\xe7hW\x1cx\xdb\x08\xae\x9aL2\xc5$H\x19[KFFt\xb4\xe7\xbc\x85\x84\xe9\x9a\x1f(NX\xb0Gnf\xf1S\xcf\x9d\xac\xea\xddz}\xdc\xea\xe4\x9b\xcf\x95\xfe\x1f\xc37\xfb5\xe48\xc8\x11\xcc]\xfa\x1fA\x8f\xd1\xb7\x9bde\xaa4G\x87\xe73P\xd5\xebl\x9fU\xf0\\\x10\xcb0\x86\xee|\xa8-\xb9\x98\x0f9\xddHG7\xdd\x99\xab\xac\xef-\xeft\xaa\xc7ry\x08T1\xe3\xa7\x15y\x9a@A\xa3H\xc5\xddF\xf2\x1b\x91\xac\x8f\xe1\x9b\xfd\xac&n\xb3\x01\x1e\x06\xa9;\xfe\xa4\xcf\x80Q\xbag\xe9\xe5\xa1\x1d\x13Dut\x0b3\x07q\xbf\x9f\x9c6A0\x9a\xea\x9by\x9f\xea\xeb\xb1B\xae\xa8\xf5\x86\xbbVC\xb3\xe1Pi\xb6yD\xaf\xdd\xc5\xc0\xf1-\xcf\x1e\xaaU\x97Y\xd7\xf3\xd2Y.\xaf\xfb\xe3\x07\xf1\xcd^\x99\xe4\x04\t\xddzK\x1aJ\x10\xd2\x80\xc7\xbd%bo\xef:!\xe5\xf6N\x16\x04\x10\xb0\x0c\x14$\xf2\x1b\xaa\x1c\xe4H\xdc\x83\x92\xc4\x9d\xb3\\\x1cb\xa9]/l\xc8\xaezI\x89\xc1a\x87\xa2\x051\xb0\x03\x84=7l/6y\xad\xd3\xd0\xe95<\x0c\xacK(\x9e\xc8(\x08\xc33Y\x0fB\xe4\xc6\x13\x98\xa6\x17\x06-\xe8\x95\xc5Fj\xa3pd\xe5\xaf\x837\x0e!\x8f\xe1\xac\xfd\xd2\x12Q\x0c`\xd0\xfd\x956\x81\xb7\x8e\xe7\x84%\x8b\x89\xe1\xaf\x85\x01\x1dC@\x16Km\xd8)\xa7\xdd\xdc\xa1L\xb4F\xb3b\x9by\x1a\xac\xd6^\x07\x0cl\xdczbv\xeb\xd8\xf1\x0c\x8a9\\\xfc\xcb\x8a^4+<\xc4H\xddn\xeb\x8b\xdd\xc9^\t\x9f\x8a\xdf+\xe1\xdc\xcb1\x90\x13EQ\xc1\xc9\x95\'\x10K\x93\xbc(\xb0<\x00\xb8a\xe7x\x8a\xc1\x9e\xc4}\xd3\x85<\x9cY8D\x12w\x17\xd8\xab\xa5k\xd3\xd3cA\x9c\xdd\xe8|\x91\xa6\xda~$_\x8b\xf4<\x1a\xcd\xe3f\x7f\x98\\\xfa\xb4\x13FC>\xa2\xf2-\xabL\x0c\'\xe5\x19\xdd_f\x1c\xc7\xe8d\x02e\xf9ze\x0f\x1e\x01\xe4\tO\xac\x84\xb6\xb0\xd3q .\x7f\xe6\x85\xfd9\xef\xed\xe5\xf6\x90\'\xef\xed\x9b?x\xf6\xe4\xbd}\xc4g\xc8\x9f$\xad\'\xef\xed\xc9{\xfb\xff\xa1\x97>yoO\xde\xdb\x93\xf7\xf6\xe4\xbd=yoO\xde\xdb\x93\xf7\xf6y\xe5|\xf2\xde\x9e\xbc\xb7\'\xef\xed\xc9{\xfb\xcc\x1c\xb5\'\xef\xed\xc9{{\xf2\xde\x9e\xbc\xb7O+\xe7\x93\xf7\xf6\xe4\xbd=yoO\xde\xdbg\xe6\xa8=yoO\xde\xdb\x93\xf7\xf6\xe4\xbd}Z9\x9f\xbc\xb7\'\xef\xed\xc9{{\xf2\xde>3G\xed\xc9{{\xf2\xde\x9e\xbc\xb7\'\xef\xed\xd3\xca\xf9\xe4\xbd=yoO\xde\xdb\x93\xf7\xf6\x999jO\xde\xdb\x93\xf7\xf6\xe4\xbd=yo\x9fV\xce\'\xef\xed\xc9{{\xf2\xde\x9e\xbc\xb7\xcf\xccQ\xfbO\xe6\xbd\xf5\xbf\xfbl\xdf_-u\xdf}\xf5I\xc0OL\x0f\xfb\x0e\x8a}\x0c\x8b\xea{(\xf6!,\xaa\xef\xa0\xd8\xc7 \x9b\xbe\x83b\x1fC6\xfa\x0e\x8a}\x0c\xd9\xe8{d\xc5\x0f!\x1b\xbdM\xb1\x7f\x8b\xe1\x91hx\xb4\x1a6\xd0t\xad\xces}R\x1f\xac.rPn:>\x8d\xdb\x896\x91\xa6\xc7HK/7F\x8e\xa1Zx\xe2\xdd\xfd\x18\x8a\xddfK\x1f\xbb\x9d.MOF\x19w\xd1`\x91\x89\xedS\xfe\xed\xb3\xf4\xd9\x86\xd4\x01j}\xc9G\x91\x9a6\x91\x9d\x9ft)\xfa1\\\xf1c\x90M\xdfA\xb1\x8fA6}\x8f\x02\xfd!\xc8\xa6\xef\xa1\xd8\xbfI6z\x02\x80\xfe\x14\x00D\xfdf\xf2?\x07\x001$%\xb3\x12A#\x99\xa7nK\xecD\xfa\xc6I\xa0\t\x012\xac\x04x\xc0\n\x14MI\xac\xfc\xc2\x86\x00\x04/#N\x11y^D"\t\x10!\xdd\xb6\xe4\xbc\x8e\xb7@\x14\'Q"#\x13P\x91Y\x1e@\x05A\x82\xb9\xed\x87Q\x04N\x91I\x81\xe5\xf8\x97\x15i\xa4 )\x04\x94d\t\x7f\x97A\x0c\'3\x1c\x89HY~G\x00\xd0\xaf\xcb\x8b\xbf\xfc\xd1\x02#\xf4\x13GR,\x0b\x98\x9fw\xdb\xbfF\x00\xfa\xfc\xf0\xa4? \x00\x89\xb4 \xd2,\x14\xf0\xd1XH I\x82\x9c$\xd3@\x16X|4\x92\xa4\x05\x92eD(!^$\x18$\xcb\x88bDV\xe0(\xc0\x89\x00\x02J\xba9\xea\x93\x00\xf4\x8e\x04 @\xdc\x84\x92\x10u\xdbr\xcaAA\x12E\x81\xe4\x90\x80dZ!\x15\x8e\xa3\xf0A%\x99\x13\x00\xc5\x8b\n\xc7\x00R\xa0Y\x9e!xB\xa2qD\xd1\xe0\xcb\x0fN\x00\xfaf\x94\xc9\xa3\x08@\xb7]O\x7fJ\x00\xfa\xfca\xfe\x81\x04\xa0\x17f\xcck\\\x05\x7fg\x84c\xd5\xcaOt\xd5ic\xd7\xbcx@\xbb\xeef\xab\xd9\xf5\xb2\x84Z,I\xbb\xf3:\t+\x9d\xbd\xd2g\x81\xbcN6f3RZwO\xb1@H\xa8\xa0<\xea\xde\xc1\xdd\x8c\x8e\xeaN\x8f;\xaa\xea\xaaD\xd1\xc4\x8f\x01\x00\xddj\x02>+\xb8\xd8rw{$\xc9E\xbd\xdd%\xfe1"\xfc\xc3a\xdcu\xc1.\xd2fkTek\xf6\x08\x82\xf6\xca\x88\xd7\xe9H\xdf8sN[\x0f\xea\xb1\x9az\x94f\x07\x83\x016\xeb\xbe\xf4\x1adJ\xe1U\x1fBU\xad\xf2\xd6\x07L\xc8-\x05\xe7\xb5mo\xafm@|\x14\x00\x08\xab\xc9\xd2\x1c\xba[O\xeb\xba{;H\x07\xad7\x9a\x910\xdd\x1a\xca\xaa\xacT:\x88\xd7GY\x9cU\x14(\xce\x84\x16v\xf2v3\\\xc7\x86tVv\xe7N\xdckWN^\xd9PTV}RD\x86\xaa\x87\x88\x96cvP\xe8\xe5x$\x817\xee\xeb\x7f\x14\x00\x88\xa4\x7f" v|\xc4\xdd\xaf\x04\xd5\xec\xd5N\x9c\xb6\x85L\xb5\xab1\xa3\xcd\xfc\xc2^z\xbc2\xae\x04x\xd07k\xd3\x99\r\x87z\x94(\xd9 \x9f\'\xba-\xb3\xb4\x1b\x17\xf9\x91\xa1\x91\xa4d\x93\xc1`\xcb>Xl\xa7\xab\x89 \x1a$\xef\x1c\xe3\x91\xef}\xae\xc5\xaa\x8f\x02\x00\xdd<\x050\xb8\x9dC\xec\xdd\xaeL\xa695\x85M\xb7\xf3\xed\xe4\xe0\xb5\x06\x81\xecQ\xd8\xf4ryLG\xac\xe0EkNd8\x08\x9a&\xaciZ\xdf]\x05\x92j\xabR(/\x84\x1cU\x13\x98\x85h\x98\xa6\xa5\xc6\x17\xd3L^Vcr!5\xf4\xf1\xad\x01\xf1 \x00\xd0M\xcd\xdb\xf6XHQw\xcer\xd4\xe0J\x02\x97D/\'\xe3i:\xab\xf7\x0b\xdc\xe2\xba\xb1\xd2({\xc8GF\x18\xe0\x12\xcc\xed\xb9\xb8\x8eEs\xbf6\x83E\xb8\xc0C\x9b\xa8\xf0\xd3MEh\xdb6\xb5/\xf5\xe0\x9e\xb4=ck\x93U1\x83:1{#\xdf\xecQ\x00\xa0[L\xe0\xac\xc828\xbf\xdd\xa5\xb7\x113a\\\xa3\xd0\x9c(o\xc8k\xd1]z\xd4\xefv\x17}\x05\xaf\xab\x12\xae\xa0m\x92\xb9*\x1eN\xd4 \xb6\x03\xb1V\xea\xc5\x8e1\x19\xc7\xaf\xd6{w\xd4\x1f\x0f\xbb\xf3\x10n{\xc1C\xb3M\x16u\xb3\xbe\xad\xfc\xb7.\x83\x7f\x10\x03\xe8v69\x96\x81\xf0\x0fV\x93V\xde\x14\xee\xf6\xfdz?\x02\xf9\x95\x9c\xd0lC+\xeb\x81!\t\xfb(\x1c\xb1\xbf\xae\xf2X\x8b\xa3\x0eL\x96\xc7\xb4W-\x19i\xba7\xcf\x86E\x17\x00\xda!\xf3\xa5\r\xab\xcd\xbc\xcd\x92Q\xbaZ\xcdF\xde\xd9\xdc\x06\xcc\x1b\xf7\xd8>\n\x02\xf4\xa2&\xe2\x10\xcd\x12w5Y]\x17[s\xf0wrMl&\x97\xcdU\xb7\xf6)\xe2<6\xd6\x95x\xeeT\xdb\xd3\x8c\xd8\xcf\x91\xae\x99\xc6\x98\x96\r\x06\xed\x04N\xbb\x08\xacp\xec@\xca\xdaK\xa2\\M\xc8dgp\xfaU\xe3+w\xf0|*\x18\xbfq\x03\xeb\xa3 @_\xcfi\xbf\xdbJ\xbcB\x0brI\xc3\x1d\xcfwr\xdf\x9a\x8e5\xb0\xfb\x15\xce\x94\xf96]^\xf4\\-\xebl\x7f;W\xb3\xb1\x93\x91{QW\xd2v\x82\x96\xd7\xc9d\x1b\xce\x8emf$P^\xd7i\xa2\xcb\'}1\xb1\xecJ\xadv\x1f\x94\xc8_Y\xd7\xab\x00\x08\xf1dM\x03\x8a\xe5o\xf8\x15\x06\x8a$\xa2p\xb4\xdf\x16\x18\x93\x82\x88\'\x07J\x10\x98\xbf.\xf9\xecO\x14\x1e5\x18\x8e\x82w\xa1\x1f6gd\x9e\xbbu\xb7\xbd\x0c\x0bzqP\xa7\xec\xe5(\x9d\xd5K\xd9V\xdd2TV)`\x1b\x03\x9d\xb9\xd8\x95\xaa\x83\x85\x9c\xe9\xb2\t\xd5\xc9\x94\\\x8f\xc054\xb2\xb2\xd5\xf45\xc5\x12\x97\xf0\xbc\xda\x8f\xc3\xd3z\\\x1e\xdfH\x8cz\x14\x8c\xe8\xa5^\xc1\x1b4\n\xdc9K\'\xe0\x87\xdb\x8b\x869\x07\x9c\x838\xca\xafBU\xb9\xb8V{J\xcap\xe4ls\xbff\xbb\xe5:[\x8a\x01\xbf\x1d9\xec\xb6Z\x1aF\xb7\x1cF\xc3~\xe2\x07\xe1\xa8wN@\x0b\xc9*e&\xa4\xdfN\xb257\xde\xbdF\xe9yg\x18\xd1KL \xc0 \xfc\xd7\x9d\x9ac\xaeO\x0f\x86H\x8d\xd9\xeb\x04t\x81\xbd\x11pVV\xe2\x036(\x97\xec\xa4\x91X%3\xc5\xe7wV2\x9e;\x11i\xcev>\x0f\xb7]\x13nO\xdb3\x03\xfb\xd2\x16\x9d\xe9\xc1\x0c\xe2\x8dfS\xe5\xa1\x06\xf6\xf0Z;\xfeA\xcd\xcd\xa3`D\xd8\x8a\x88$\xb0\xc1\xef|\xc5_\xd0\xd0_\x08d\xa0\x1e\x9c\xce\x9a\xf8K\x98z\x87\xa3\xc6\x87n#\xf2B\x99)\x91\xc4\x94\xf5|j\x17\x13\xdds\x9ba\xad\xee\x08\xd3h\xf8}\xc9\xcf\xb4p\xed\xf5LjY\xd5\xa2\xc8\x06\xcf\xd6)Sp6#\xee5_y%\xf2eZ\x90x\x82d\x11O+\x1c\x8dd\x89\xc3M\x9c"\x88\n\x0e\x7fY e\x81%h\xdc\xe6\xfdu#\xcc\xfc\x84 M\x12\x88\xbdG.\x89#\xc5\'\xb54)\x992\xd0f\xe4j\xae2\x03[$C_\x19\x87\xb4\x92.\x80\\\xd3&\xe1\x16\xd6X)\xc8s6&F\xf0\xaaX\xea\xd9\xa0 \x1f\xf6s\xd3\xb2\xc2\xe3\xf1LH\xa4j\xe3x\x00\xcc\xeah\xcc_\xe3\xe8\xbc3\x13\xe9\xd6\xdb\x00\x02O\xe24{W\r\xcb\xd67\'V\x1en\xa5\x03)\xec\xcbe[\xc2kgO@\x0e\xa5J\x99\x15\xb9\xd2n6\xcd|"\x1c\xa8p\xe1\xf3\xc3.\xeb\xc5`\xb4\\\xca+<\x97^2o\xbfp\n\xb7\xa8zS\xf7\x87\xccp\xf69\xb3\xeb&\xc2\xdb"\xffQP\xa4\x97\xc8\')\x80s\xf9]C>Zu\xa2\x16\xd2\xdb\xc9"\x1a\xaf\xe6\xca6\x00\xbc\xb0\x8a\xba\xf5!L\x8e\x9e t\xd3X\x9c\x04gj\xc4r\xf6|6An3\xca\xc0a\x93]T\xa3\xe6K}\xd4k\xe1\x92\x9e$\x0cO\x91MT\xf9\xe79eZ\xda\x07!\xf1\xde\x99\x8a\xf42\x00\x83\x1b\xe7\x86\xbe_^\xdf\x8d\xe7&\xbf\xf3Mu}\xd4\xb2h3:n\xe3\xe9Ql\x8e\x07\xcb$\xd0\xda\x9c\x9fv\x8e\xb8u\x9a\x8c\x18\x96\xbbZ\xab\xfb\x83]R\xbd\xb0T\xa1\xd9@c\\\xd5NMb\x0f\xe6\xa7E\xcfO\xa6\xd7\x13\xd8\xf3\xe3\xd70d\xefLE\xba\xc5\xc4\x8d\x8at\xbb\x04z\x87\x89\xac\xc6;\xd2\x83\x9a\xac\xc5|\x9a\xc5\x96\xed9U\x97uB\xd4MN\xd7\xc8\xc4S\xaf~\x8c\xe7\x07\x0e9\x9bYs\x1e\xd9\xeb\xd5D\x8bG\x9b\xa51\xd7\xa6\xa7\xc1Kg\xe4\x90\x86\x83\xc7ns\x16r\xe3Q\xcdx;\xfb\x8d\xd3\x1bP(\x89f A14\x1e\xe0H\x0e\xf1\n\xeen\x04\x85\x82,\x1e\xf5E\x81\x96\x18\x16\xe1\x14\xff\rj\x92/\xd8\n\x8e\xbb\x8b\t\x83\x1f-\x95\xbe\x9cPqC\x08r\xc2F\xd1\x99\x18C\xc5\xe3\xbb\xd3\xa5\x9c\x91\x93\x92]N\xaaU~\xb5\xcb\x93z\x1c\xe8P\xd3\x0f\xcd\xfa\x80\xb6A\x9b\x18W\xe3\xc6t\xceD=l]z]\x9e\x83\x86\xeb\xe8^\x9e\xbd\xf1\xe2\xd4\xa3\xe8L\xb7\xb3IB\x82\xc0\xc1\x7fw6\xb7\xa3\x99\x83\xc4Mr`*C:oy\xd9\x08;\xfb\xa2\xc5\xc2\nu,%\x90\xba\xa5\xca\xf2\x89$NW\xe72\xab\xc4<\x98\xf7\x8b8\xc3\x13\x9b\\\xc9Tl\xcc\xad\xe5\x14Z\xbcI\x85\x04s\xda\xefj\xee@\x9e\xf3\xcfU\xf4\x1fFgB?\xb1\x1c\x07X\x92\x84w(\x02U\xb1\x89\xc2wS\xf6x\xba\xae\xe3\xb1\xb0A\xc2\x88P\xa5$\x9e\x8c*\r^\x8a\xa5\x9eL\xd7\xbb\x0b=n$\x9fw\x95\x13\xb7\xf5\x8c\nW\x83\xd0\x98i\xf2vp\x17\xc9\xa0o\xd6;^\xbb^d\x93]\x8a\xd3hl|\xae\xebB\x8f\xa23\xbdX\x11`W\xc4\x19\xeak+\x16\xa7\xbd]1~\x98\xacF&n\xd2\x86:_\xeeZi\x15\x8f$\xd7\xd9\x9cwg\xbd\xdff\x83|\xc2E\x08\xcf\x9a\xa3\xab\x10\xceF>\xf4\xe6\xf3\xc5"\x11\xb9MfP\xd6\xda<\xac\x02\xcd\x10\xf3\x89\xbf\xd6\xf8\xd8\x1e\x89\xaf1T\xde\x99\xce\xf42b\xe3\xd2\r\xee\xe1L\x00\xa1\x8cZ5\xc2b\xc1n\xda\x98\xa3\x02\xa8\xec*N9\x95{\x00.\xc2\x81\x984i\xa5\xe0\xee\xf2X\xec\x86i\xbf\xa3\xcf\x07\x9f\xf6\x82\xf9\xfaH\\:6\xe6\xf6\xec\xf4\x1c\rft8\xf6D\x0f\x81L\x99\xd4\x88|\xe3\xd0\xc4\x88\x88\xe7Y\xdc\x00\xe3\x84\x80\x0b"\x0e\x00$\xd2\x14\x96VA\x0c\x1e\x96$\xc0\t<6\xc27\xb5N\x1c\x9e%8\xe6\xbe\x11\x9er\xeaX\xda0\x8b\xe6\xa0\xcfe\x00k*\x8c;\x8d\xb6F\xd3.\xadhj\x8bl\x1d\x9ekf\xdc\x8c\xad\x81\xdc8\'\xbf\x8a\xe6Km\xb5\xc0\xb3\x94-\xea\'3\t\x02\x7fI\xa5B\x0b\xf7\x93B\xdd\xf2\x83\xa5\xb1\xfd2\xb6\x92\xbd\xbfa\x9c\xcbx}e\xdf\xe8,\x8f\x82D\xbd\\*\xbd\x8d\xe4\xec=\n\xd1k\xb6\xac\x14c\xd7\xd7\x1dd$\xe5&\xe4\xe6\xdb}99\xab{\x19\x12\xf1\xf1\xd8\xc4\xd7\xe0 y\x0e\x07\xad\xb5x\x1c\x1c\xed\x10\x8c\xebx9\xab\xb2\xad\xe8\x9e\x8d\xcd\x80\x0e\xad\x92\x18M\xef,X\xba\xa2\x8e:9(\xaf\x85\xfe\xc78\xcb\xc3 Q\xb7\x96\x85\xb9\xdd\xecq\xdf\x99I\x13\x98\xee\xaf\xda\xdaJ\x81x\x99\nL\xa0\xcd\xa7+qAl<\xbd\xd6\xccq\xaf\x86M\xa6\x8f\x93\xf5\x86\x90\xcb\xcb\x99mv\xc9a\xd9\xae/\xbb\x12t\xe3x\'+!C^&\x0b\xb43\xb7\xaeN\xb7;av\xbc\xbd\xd9\xf7!\x90\xa8\x972\x81#\x08\xa7\xa4\xbb\x98\x80\xf3\xfd5\x9bF$\x9d\x81u\x02:;\xe1mT^tj\x16,\xfc\xc6\x9e\xf8\x03\xd7\xb4\xdbv\xb4=\x9b!_\xd9\xcb\xb04\x16\x19y\xdd\x9b\xf3\xad>\x8eDp\xe9\x0b\\3:<\xf6X\x15g\xd6W\xa6\x12F\xdd\x07\xb1\xa1\xdf\x19\x12\xf5s\xb1%(\x12\xdc\xb5\x14\xd6nV\xb6\xb1\x98\xab\x01\xc3\x9b3o\xb5\x9a\xb0P\x91\x8a\xc9t\x9bm\xd0\x01D\xf4bK\xaf\x1a\xbeV\xf6\xd7B\xacIN\xd1\x93\xc9l\xbf8\x01\x15m\xad\xda\xd7\x81\xd0P\x9b\x0e\x94\xee\ttS\xcbB\xa3F\xdd\t\xfc\xb70\xa2^\xae8>\x19Q\xdf\xfca\x95\xff\x1cF\xd4\x0fD\xdfy\x02\x8d\x9e&\xfd\xfc&}2\xa2\x9e\x8c\xa8\'#\xea\xc9\x88z2\xa2\x9e\x8c\xa8\'#\xea\x13\xcb\xf9dD=\x19QOF\xd4\x93\x11\xf5\x99\xd9KOF\xd4\x93\x11\xf5dD=\x19Q\x9fV\xce\'#\xea\xc9\x88z2\xa2\x9e\x8c\xa8\xcf\xcc^z2\xa2\x9e\x8c\xa8\'#\xea\xc9\x88\xfa\xb4r>\x19QOF\xd4\x93\x11\xf5dD}f\xf6\xd2\x93\x11\xf5dD=\x19QOF\xd4\xa7\x95\xf3\xc9\x88z2\xa2\x9e\x8c\xa8\'#\xea3\xb3\x97\x9e\x8c\xa8\'#\xea\xc9\x88z2\xa2>\xad\x9cOF\xd4\x93\x11\xf5dD=\x19Q\x9f\x99\xbd\xf4dD=\x19Q\xdf\xbe\xf7\xdf\xf8\xfdg\xfb\xfej\xef\x7f\xff\xd5\'\x01?/#\xea{(\xf6!\x8c\xa8\xef\xa1\xd8\x87\xa0\x94\xbe\x87b\x1fB\x1cz\xa3b\xff\x16[\xe3C\xc0<\xdfC\xb1\x0f\x01\xf3|\x0fW\xfc\x10~\xcdw\xc9\x8a\x1f\xc1\xaf\xf9.\x8a=\xf95\xef\xc1\xafA\xbf\x99\xfc\xcf\xf954\xc1\xf0$E\xf3\x14\xa4\x05Ed\x15x[\x11I\x91\x00 J\x12\xf0\xf7\x00E\xd3$\x16\xf5\xb6\xa7\x04P\n)\xf1\x10\xf0\x02\x90\x19\x04x\x1e0\xb7Ey\xaf\xc3\x19HD\xd1@$dQ`\tF\xa1DZ\x10\x08\x99\x12\x15\x81eE\x99\x16\x18^\xe4\x15\xfe\xb6/\x9d\xe1XV\xc0\x02\xc8\xe4m\xd7\xb8 K\x12$8\x9e~G~\xcd\xaf;\xe1\xbe\xfc\xc1\xba\x0c\x00\x7f"X\x0e"\x86\x81\xe4\x9f\xf1k>?\xfb\xe7\x0f\xf85\x1c\x07\x19\xec|,-\xb3\xb2\x02i\x85\x93\x14\x11\xab\n\x19\x99\x94$\x0e\x1b\x1e\xab\xc2\x8a\x92 \x93\x0c#@\n\xbf>#\xb2\x9cL\xc82\ri\x9e\xb8\xed\x82z\xf2k\xde\x91_\xa3\xe0\x00cD,\x0c6=\x92\x91\xc8\x08H\xe1YA\xc6B\x10\x80Pp\xac\xcb7/\x93x\x9a )B\x004\x12Y\x8a\xa1\x89\x1bq\x88\x93d\xf2\xcb\x0f\xce\xaf\xf9fJ\xc5\xa3\xf85\xb7=4\x7f\xca\xaf\xf9\xfca\xfe\xb1\xfc\x1a\xfaO\xb6\xc83\x0b"oNY|]h\xf6\x987\x97\xe3c\xee\x0fD=V\x94\\\xe9\x851\x14G\xb2\xc8\x95\xd7\xde\xed\xae\x81\xb2n*\x00W\xb4\xd0./\x0b\xa2\x95\xe9<\x1a\xf3\xa3\xd6\xf3\xfbY\xec\x1fBv\x91\x9e\'\xa2\xd9\xbf\x91z\xf2(\x80\r.\n\x00@\x02\xb0\x04q\xb7\xe6=\x17#\x8f\xdf\x18\xd2!p=\xb0\x18o\x9bz\xc1\xd5\x86\xa3\xabT\x12\xfa\xac\xb2\xe3%\xc7\xdf\xf4\x13Z\xcd\xcf\xa3\xb3G\x95r\xadezn_A\x1b]K\xcd\xa2F\xa3\xb1b\xf1}\xc6\xaf\xac\xcb1\x1c\x96\xb9fDo\xdcX\xf8(\x80\xcd\x8b\x9a8\xa9\xdcm\xdbZ\xcc&\xd3KKN\xafFEPp\x1e\x95\xe5t\xb36bd1\xd6<\xb4\xa2\x93%\xba\x9d\xc05\xdaN\x85p\xd9\xf1\xbc6M\x0bn\x9d"ck\xf7\xc95\xdb$\xd3\xbed\x8d\x80\x98m&\xc7U\x08\xa2=N\x97o\\\xf7\xfa(\x80\xcd\x8b\x8e\xdcm;4d\xbeV\x93R\x02v\xb1s7MK\x8d\xecEk\xc7\'\xd2>3q\x06\xc6\xaaw\xa5\x04oB^8O\xea\xa7"\x14\x8f\xc9\x9eO7\xf9f|\r7\xe1\xd8R\xcc\xed\xea\x94\xc1\xd1(\xe8b2\xad\xcf\xe4\xda=O}W\xdb\x86\xaf-\xda\xfb\xb1\x016\xd8\x8a\xb8\xbf\xa38\x86\xb8_\x0b>\xd1\x18\xb6?"38\xac\xce5s(\xa9=\x9a\xcc\x9aq#\xa4v|(\xecN=\xb2\xaaK\x16\xfc0\xd9\xb2q\x14\xba\xc7\x94%F\xd2,\xdb\xe5\xe5>\x07\x17C]\x1a\xa3\x91\xcfn8\xae;AB\xef\'\xa41\xbc\x11y\xf2(\x80\xcdMM|\x1c\xdc\x173w\xcb\xd3F\xb672+\x8bw\xf3\xb1\xb6c\x05\xd0\x1b\xe6&\x89\x13\x15\xcd\xc8\x8b\xa3\xdb\x17\xb0\x1dV\xf4\xe84W\xceg\x1c\x97l\xbeT\x1aARY\xab\x19v\x81_\x18#\xbb\xdar\x0b\xe0\x06{e\x1a1\xa2\xab\x03{\xb5z\xa3\x9a\x8f\x02\xd8\xdcz^\n\xa1[\x89\xbe\xe3\x1e\xb8\x81\xd0x\xa3\x8c\xce\x0e6\x85\xb8\xcdXg\x8b(\xcc\x81\x82\x9b\x13\xa2\xcc@\x7f-\xf2\xc0\xdcl\\\x05\xb9\xa9\xb4\xd5\xab-a\x146\xaa\xdd\xde\xd3\x0e\xbc\xb2D3\x7fuT\xe0\xbe\xb37\xe7:\xb8\xecL\xb1\x1eb\xf1c\x006/g\x13\xdb\x84%\xc8\xbbb\x15\x8cgAy\xe5\xe8\x81\xc8\xb9Yn(6\xbb\x8eg\xec\x86\x1c\xf5h2r\x1d\xea4\x9c\xc7\x9bj\xcf\xb5}\xbe\x1eKL\xab\x1fDFS\x82x<\xe8(\x13)!\xdf\x1a;*\xf1\xf6kq\x97Tf~\x18FT\xfd\xc6\xb5\x89\x8f\x02\xd8\xbc\xa8\x89\x9bW@\xd3w\xdb!\x13]\x01Z{\xa4\xce\x8c\xbc^\x8e\x04+F\xae\x97\x02nE\xf5\xc5\xc2\xbb\xda\xe1F(\x9a\\\xd2\xd79G1>\x08\xa8\xe4\x1a\xe0\n`\xf3s\xa7\x8a+(\xb8wZ\x95\x81\x87c0,/\x11P\xa6\xe7\xd0\x944\x89\xdcX\x80S\xfd\\W\xf3:\xa6\xd3\xf5\xe6J{\x9bvXj\x94\xa58W\x9b\xd3\xe9\x14%\xd6(\xbd\x92\xf2r\xa3%\x1e?\xf1Bp%\x85XSr&\xa3\x02\xf0\x1a\xb5\xee\xc7\x06\xd8\xdc\xac\x08n\xbb\xa9\xe9\xfb\xcb\x19c\xd9Y\x8f\xf58X\xae\xa7\xf4r\x7f\x89k\x19\x0f\xed\xd4\xfa\xdc,c\x93>\x92\x89\xa3\x88\xeaR\x18\x8cp\x08\xa6\xd1\xae\x1c\xf9\x1b\r\x0f\xe5\xa8\xb9^\x8f\xd4\xea|*\x99\xcb\xa6\xd89[b\xb1\x1bK\xa1U\xf4\xcd\x92\xdd\xbe1\xf4\x1f\x05\xb0\xb9\xa9I\xdd\xf80$\xb8\xeb\xf7O\xb6\xafY55:\xee\xa6\xa3\xf3\x80\x8a\xe4\x90\x1b\xf4h\xe4ua\xb3\x18\x8a)Sf\x97x)D\x87\x06%\xc73\x10O]I\xf0\x1d\xa4*\xb1X\xae\xb2\xc4\xbfZ;[\x11\x04\x9b\x1e\xa4e|\x02\xa5\n\xab\xeaU\xe4\xc9;\x03l~\x19\xf5_`\xb5wE\xff8\xea\x81\xb39G\xa9k\xb5\xb0\x8c\xb3\xa4\x18\x1f\x00\xd3\xba\x8b\x8d6\x13\x07\xcd:\xb7\xf6V\x12\xf8\xfazD-\xdc\xd8\x96\xb0\x1d\xcfG\xb9)\xc7\x93`\xdbLb!W\\a\rwsW\x0f\x0b\xb3\x9c\xd5\xc3\xfe\xf4\xc6\x9d\xfa\x8f\x02\xd8\xbcd8\x88\x00\x8b\xee3\x9c\xab/)\xadd\xcdKyPOEq\x8d\x8e1t\x9d6d\x9a\xe9\x8a\xbaN\xc7\x9d\xa6QWu\x97\xef\xd0Y+\\S83\xc94[u\'M\xe1.b\x95\xb9!\x9a\x84\x91\xb98\x12;\x07\nG"h\xed\xcf\xd6\x08?\n`s\xb3"$I\x1a\xffugE\x8a6\xc6\xf9\xfedL\xc7\xfela\x84Z\xa9\\-X\x13e\x7f\x0c!\xac7\x8blY\xe0\xbe\xa3\x15\xb5&\xa2g\x92\xa2\xdb\x10\x0e#\xeeZ\xf3\xdedK4\xc4\xbe\xe4\xb8\xc1\x14\x96\xc3\x08\x9d\'\x80r.\xba\xb8\xde}\xae\xebB\x8f\x02\xd8\xbc\xb4N\x88c(\x92\xbeK\xa0\x91-UI;\x0fg\x00.b.\xf6\x17u\xda\xae\xa6\x92=\xcb7xH[\xad\xce0\x89`E%\xabB\x12\xf9V\xbd\xd0\xb1p>N\n\xd9\xe4W\xbc\xb1\x9a\xd7\xe2\xf12Y\xd8\xd7\x89\x14\xaa\x8b\xd0\x14\xb8\xa6\x1akoE\xbb<\x08`\xf3\x12r\x80\x03\x0cu\xbfT\x7f6\x17)\xaeO\xf4\xd9Q;\x1f\x1dY\x93<\xc6\xb5\x86,\xac\xf1K]\xe3dDq\xf2\x91,\x92a&\xc2s\x14\x13q7\xea\xf6B\xbe\xcda\xaa\x1f\x08\x86o\xd9\xf8\x1a+\xd2\x8cu\x8a\x85)\xcf\xaf\xf9@\x9a\xe77\xa2p\x1fE\xb0\xb9%P\x1a\x02\x8a!\xef\xaf\x95\n\x12w\x9a\x9f\\8\x97\x12\xde\xe9\xfc\x19aTP\x1bua\x17\xba\tk\xec\xa81\x89]i\x12\\\xfb\x9a\xaf\xc3n\x83K\xdd\xe2\x9c\x88\xb5~\xdcZ\xb5n\xa9\x04{ <\xd7\xb7")<.t\x1f(v\xc7\to\x1c\x0e\x1fE\xb0\xb9\xa9I\x10\x14\x8b\xfb\xc9;5\xc1<\xabuZ\xc4\xc3X\xd8\x8f[_?M\x17\x06\x9fg\xf9\xaa\xee\x94a\xe9\xec\xe0q\x9f\xba\xa3Q\xc0\xf5Ks\xb2\xa0\x97\x97Rb\xa7WF\xe9\xe9~\xea\xcc\xe7\x80\x15$\xff8\xaa\xfc\xb9}\xd6\xa7`\xbe;\xf4\xf9\xf6sAI\x1eE\xb0y\t}\x8eA\x04u\xff\xfeA^]\xc2\xad)h\xcc4\xcbwY\xd8]\x03\xc2\xd7\xe9*KkA\xe9o\xb3\x96_\x12k=<\xb6vD*\xc1D\xb8\xec\xda\xb1\xa8\xf0g\xb6\x9bU\xc0\xa9#V\xc8\xae\x9bQ~\x8c/T0\xdaK\x92:\xad\x16o\xbc\x88\xf8(\x82\xcdK\x9d@\x0c\x85\x0fs\x97\xe1\x1c"*\xc2\xdc\xbe\x9c\xfa\xe3A\xb4\xab\x84\xde\xadc\xd8\xf6\xa9m1Hk\xad\\\xf6R;\xe1\x9a\xf9i<\xe01#\x0c\x8b\x0e\x9d\x02\xbe\x01\xd2H\xec\x16\xd0/\x8fR\xe6\xa4\x9d+\x8e\xa53_\xed\xe6\xb2\xe3_\x8672\x0ey<\x01("\xc3\x11\xf2\xed\x0e\x0e\x04\x15\x02\xab%\x89"\x07\xc4[\x1f\xc4\xa0\xdb;\xb1\x00}\xcb\x0c\x0c\xd0\xed\xfa\xda\x1f0\xba.\x16\xbd\x0b\xdb)\x8f\xa6\xa3=U7+3\xc1\xadD5N\r=[mM\x020\x1e\xcf =>+\x87\xb0Q57S\xe1T\xa7\xad+\xcdw\x841\xc4<\xa1R\xed|\xc8\xa5aJ\xab\x06U[\xcd\x92\xd4\xec7&\xf2G\x91tnj\xe2)\x98\xa6\xe1=v).\xaf\xe3]\xb6PJ\xe9\xda\xbb\xcdb\x9fI\xa5\xaf04\xdfo\xe3I\x91\xe85bqb\xcd}\xe9\x02\x86n2\x9b\xa6r\xbd>\xa9\x16\xd8e\x9d\x10\x0c\x0b\xce\x98\xadU\xb0\x12\xcb\xe48;\x06#{\xd6H\xe7S\xfc\xc6N\xf8Q$\x9d\x97\xb9\x86\xa3HD\xdc\xcf\xfabq\x8a\xd2Y\xacxjM\x1cN@\xf0\xf4z\xc8&yU\xce\xcc\xb5jd-0\x97\\\x8d2\xa72\x92\x85:\xb2\x97+\x9b\xab\x9ai?\xbe\x98\x04\'D]\xb6"O\x9a~\xd1W\xfd\xa8m\x0e\x9a6\xaa\xba\xf0\xfc\xb9\x18\x87\x8f"\xe9\xbc8\x0b\xcb!\n\x82;g\xb9z\xd1\xc5\xca\x0fg\x92.\xe7\x06\xba\x8c\xc6\xfaX\xde\xea\x8b\xd9J\x1f\x9f\'H<\xf3\x07f\xab\xe5t#f\xfd\x99\x88\x97\xc4\x98\xeb\xb8\xcbJl\xa7\xa4%\x81\x8bP\xb3j\xbfrO\xa4s\xb2\xfa\xa3\x03/\xbem\x1eW?[\xf1\xafP:/\xf3\xc1\x13\xa5\xf3\xcd\xf7\xf4\xff\x07\xa1t~ \xfc\xc3\x93\xa8\xf1D\xe9Q:O\x94\xce\x13\xa5\xf3D\xe9|fD\xcd\x13\xa5\xf3D\xe9\xad\x9cO\x94\xce\x13\xa5\xf3D\xe9\x06\xa5\xf3\x1d\\\xf1cP:\xdf#+>\x893\xefA\x9c\xa1\x7f3\xf9\x9f\x13g8\x8a\x90\x08\x81G\xac\xa4\x10$C02\r\x19\xc0Q,#PP\x96)\x91\x00\xa2\xc4\x8b\x88!\x88\xdbr\r\xc4\x91"#\x892\x8d\x10#\xf2\xf8\x99\xb7\xc5L\xaf\xe3\x14\x04F`!\x90\x14\x11 \x9a\x93D\x05H\xb2\xc4*\x12\xcf*,\xcf\x12<>\xb4"q\x88eE\x82\xe5^\x96@\x93$\th\x0eH^\x95,\xc7d9 \x8e9v\x1d\x17\r\xbd\'Bh\xe7(_\xee\x94\x8bV\xf7\xb9b\x87\xfd\xf1R\xde9Jk`)\xe4\xcfY\xda\x0f\xb2\xbb\xabr\xbe\x819|/q\xce\r\xb3\xdb6\x14\xceu\xf7#D\xe4\x17\x07g\xd6\x9c \xd6\x99@0\xec\xe58\x1cQ\xee\x18p\x82\xcb\x14Z\x7f_O\xbb\xd3\xd9\xc0/\x12\xe6Y\x16\x17\x10\xb8B;\n\x97O\xb5\xc3d\x8fA\x88k\x10/\x8e\xf0\t\n\x15\x03\xcf\x99q\x0fDYP)\xb3q\xe1\xa2\x9bi\xcd\xe7\x1a\xcdw!\x92\xbe\xd9g\xfbc\xe2\xcc\x1a&\xbc./\x0c&^[\'\xd7\xd1\x10N\x1d\xab;\xb5\x8e\xeby\xac\xd7\xd7\xfe4U\xb7\xc7\x9dH\x81\xabC\x0f\xf8\xc1\xa1\xfb\x8c{\x98\xedy\xd1k%{\xd0I\xad\x9a\x95\x8caZ\x07\xd7\xc1\xc4?\xbc\xf6b\xef\xbc\x83i\xf41\xb3\x98\xe20\xbfy\x1c~L\x9cYsb\xcd\x88\xe7\x1b\xd5\x97\xda\xe66)\xbbdiJ\xf0\xe1\x89\xb6Q\xa2\xa9;\xf6E\x0f\xa9\x1a`\xee\xaa5C\x0f\xe7\x11?wa5\xbf\xf9\x92\xefc\xee\xcd\xf3n\xb8\x16{\xeb\xb9\xff\x92\xfa\x96\x95\x9b\x85\x9fQ\xe09\x93M\xb2y \xfd\xdd!\xc7\xfb\xdd\xeb\xf8\xd1\xb6\x1f\xae\xc3]J>*5j\xe4\x14\x8fE\xa1\xab\xae\xdd\xe5\x8eA#\x9eT\x8a\xc9#a,\x16b\xceN\x8fC\xe7\xea\xb3\xb7\xdf/\x93\xf1\xe6\xa1\xff1\xf7\xe6Y\xeeS8NA\xf8\xcb\x0b\rC`]@\x96\xc0\xfc\xce{vJ\x82D\xab\xae[\x97 \xce\x9a\xff8h\x91{\xf2\xec;43\x0fNcH|\xfdC\xc9\xdd\t\xc7\xdba\xe8%\xe4\n\xec\xdd\x1d\x19\x82\x05\x9a\x99\x81\xc8\xd3\xdd(\x1cH\x1b\x84\xcfor\t\x1fso\x9e\xc7\xd5\x1a\x01Hb/a*\xc2\xe1B\x9c\xce\xd2\xb5=\xa7\xe9\xfa}0\xff\xe8[\xcan\xb7\x83bQ\x16\xe5\xf6\x96,\x94\xe4[4\x7f\x8c*\xe8F+*\xb5\x97\xa1\x02\xf5\xa1\xeb\xd4\xccgb<\xb9\x98K7-\xde\x8f\xfe\te\x0cx\xado\x947y\x9f\x8f\xb97\xcf\x12\x0e{jM\xc4K\x98j\x90\xc3uB\xa0\xd2\x8ca9\x97\x89\x83IA U\x87\xc3\x1e\x14\xee=\x0b\xc5{\xd4\x7f83\r\xd8\x97\xbd^\x89{C>\x0cPU2E\xa1\x04\x94\xdd\xa6\xc6\xee\xe8\x81T+M\x8d\x84\x1e\xaas~\xaa\xb3\xe4{\x99\x81\x1fso\x9e\xa9\x0f=wr\xf0\xe58\xd4\xa1\x93~9\xee\xd3!\x93\'\xea\xc6[\x0b{\x08\x82V\xd91~\xe3]n\xd1\x02\x0c\xc8\xfcp\xca\xc2\x9d\x17\x07\x91;\xa9\xba\xb3\x0f\xb4`\x8d\xfa"\t\xe2m1\x8d\xa6\xe9\xaa\xf3r\x17I\xd4\xbf\xa3\xa29\xb1!\xf7&0\xf91\xf7\xe6\x99\x13\x04\x89P\xd0\xeb\xe5m\xb4\x00\x81\xf5XY=\x0f#c\x9dDB;M,G\xcd\xb4l\x1e\x04\xf9\x94\x10\xc7\xf0R\x1cw\xfb\xeb\x9d\x92oY\xda\x1c:\xa0\x90X\xa3Qs8\x1b\xe0@<($pB\x00\xf6\xd8\xaa\xf9R\xca\xe5r\xa8\xc77S\xffc\xee\xcdzG\xc5\xa9u5\xa0\xaf\x1f\xbc1\xb83\xd0^\xa8\xb5\xf09\xf6\x91A\x0e\x10\xcc\xa5\xbc\x96\x10\x8b\x08\xc2&\x89\x08\x1d\x8f+$\x13\xd2\xd9\xdb\xad\x13\xcb\x08\x93\xadh\xcf(K@d\xc7Y\x8f\x9cE8"<\x04\x8e\xcdq.=\xd1\x03\x1f\xe0\x03x\xf3\xcd\xf4\xc7\xdc\x9b\xafJ\x15!p\xe8\xb5T\xdd\x9b\xe9-~T\xf3\x80k\t\x93\x826\xd9\x88\rs\t\x10\xdb\x87\xaf\xcbQ,\xeb\x9e\x12\xe7\xde\'*\x83@\\n\xd0\xe6\xdaW\xeb\x84\xd5\xb4x~\xe8\xfa-\x9bO7\x8dL\xd4\xc3\xba\xb2O\xe2,\x02\'\x9e\xcf\xdf\xfd\x14\xf5S\xee\xcd\xd7\xa7Fk\x9d\n\xbf\xf2>%)\xde,H\x9a\x1e\x16%\xcc\x9d,_\xa7\xfd\xc3\x86\xcd\xf9\x02\x97\x84>\x98\xd2\x1d\x97o\x9e\xe1\xc1\xfc\tO5\xd6\xba\xc4GaLF\xf3\x12\x86\x0f\x00\xc7\xc4\xb1\x99\x8eS\xa9\x85fp\x95\x91`\xbaYc*\x1e\xbe\x97\xec\xf51\xf7f\xcd\t\x10F!\x10z\xb5e\xf7\xcb\xe0Y\xc49\xdb\x8f\x83\xad\xcc"\xc3:{\xa5\xe8)\xeah\x1a\xb6\xe9\xd6\x0f\x01p[E\x07n3\x06\xda\x8f>\xbb($}\xba\xce \xd7\x11fI-)R:\x88c\x10p\xb6\x00\xe6\xd1;{\xfb\x1daq\xbf\x15\xfc\xff\xc9\xbd\xf9z\xd3\xbc\xb97\xff\xf5\x0f\xe0\xff?\xe4\xde\xfc:\xb0\xc0f5l\xee\xcd\xe6\xdel\xee\xcd\xe6\xdel{\xe97\xdcK7\xf7fso6\xf7fso6\xf7fso>\xf7\x9c\x9b{\xb3\xb97\x9b{\xb3\xb97\xdf\xd9\x93\xd9\xdc\x9b\xcd\xbd\xd9\xdc\x9b\xcd\xbd\xf9\xb6\xcf\xb9\xb97\x9b{\xb3\xb97\x9b{\xf3\x9d=\x99\xcd\xbd\xd9\xdc\x9b\xcd\xbd\xd9\xdc\x9bo\xfb\x9c\x9b{\xb3\xb97\x9b{\xb3\xb97\xdf\xd9\x93\xd9\xdc\x9b\xcd\xbd\xd9\xdc\x9b\xcd\xbd\xf9\xb6\xcf\xb9\xb97\x9b{\xb3\xb97\x9b{\xf3\x9d=\x99\xcd\xbd\xd9\xdc\x9b\xcd\xbd\xd9\xdc\x9bo\xfb\x9c\x9b{\xb3\xb97\x9b{\xb3\xb97\xff\x15\x84\x90\xfc\xcb\xef\xf6\xfd\'\x08\xc1\xf8\xddo\x02~c\xf7\xe6/\x08\xec\xe7\xb87\x7fA`?\xc7\xbd\xf9\x0b\x02\xfb9\xee\xcd{\x81\xfd\x9f\xb0\x91\x9f\xe3\xde\xfc\x053\xf6sx\x98\xbfb\xf3\xd8x\x98?\x83\x87!\xfeg\xc8\xff=\x0fC\x90\x08\xc2s$\x07\x11\xf4\xb3i\t\x84<{\x93 \x02\xce\x81O\xd9\x80EA\x1c%y\x92g@\x8e\xa6\xa8\'\xea\xf0T\n(\x8e\xa7x\x1c\x17H\xe1\xd9\x95\xf5\xc7\xf6\x01\xc5\xc18M\xd1\x84\x13\x04FOa\xba4@\xd4\xa4\x84\xafg\xc45&\x8b\x13\x81KC6g\x83\xcf_\x97\xfbA=:\xf5\x91q\xbc(\xa3N\xc2\x11\xef\x13/\xa1\x8e\x1a\xc0@d*\x9fw4\xee\x8a\xde\x8f\xba*\xfe\xc9:\x0c\x82\xff\x1dD(h=\x98^\x9b\xef\xda\x17\xc6\xd5\xf2\xe2\xb1oj\x84\xd6\xba\xca\xb6\x81S\x88\x99\xb2a\xcf\xce\xe8\x12F\xb8f\xfe\x03\xd4.\xf0\xc0\xce3@Mk\xd8>PSQ=\xbar\'\x9d\x98\xc3\xb8K\xaa\x9b\xdd\xd8\x17\x83,3P1\x08\xb4~3\xccO\xf10\xcf01\x90\x82_ZTI\xf1u\xa0\xed\x87_\x8c\x94\x9b\xceCc\x93\xc8\xae\x1e\xcb\xc38\x85G\xe1\xee\x8cJ\x86\xba\x9d\xa0\x17)\xa1\x1e\x8d\xab\x96[\xd4\xc3\xa3\xbb\xb6b\x85v\x89\xa8\x82\xbeWC?\x9e\x89\x1c_o\xb7\xc9\xc2\xf4\x19\x04qo\x8a\x1f\x9f\xe2a\x9e1\x12OT\x02|\xe9P\x05\x9do\xa7q\xd9\xdf\xa0]\x86\x9c\xcf\xf9DYS\xb0\xc7\x02\xbb\xd0/\x9d)T>Mp\xb0\x14\x8a\xfa\xee2\x8d\x10K\xb5\xbe\xf4\xb8\xd4\x8a1\xf4d\xb6\x1b\x15\x87\xd6\x11W\x16r\xcd\xe7\x10,ug\x16\xef\xc1r\xfafm\x94?\xa4\xc3<\x07\x91B@\x04\xc1_\x9a\xb6u]\xeb \xb7,\xb0\t\xd8\x9d\xf3\xcb\\\xed j\xe74\x00\x92\x9c\xcdj\xdf_\x95\xa1c\x8f1\xcbk\x84\x7fg\xcd.\x80\x0b\x12~\xd47\xcc\xe6\xce\xc8\xae\xaf4k\xb0\xad\x16\\\x10\x1d\tO\x80\x00\xf3K\xb7\x1e\x88\xef6\xdb\xff\x0c\x0e\xf3\xac\x04\xf1\xf5\xa0\xc6H\xf0us\xb3O}H\x13\x1a\x97\xef$ja\xd9\x8b\x91\xfb\x8dV\x02\xfd\xf1\x0e\x8e\xfdy\x06\xee\x0e\xd6h\x9e>\xdb\x0bRb\xe5\xf5\xc4\x88\xc6\x84\x19\xfb\xd8\xe6\xa9\x1dI\xbb)\x90\x1f\\\xdbl\xa5\xc8\xa1\xca`8\x97d\xf6x\xb3\xc9\xe8\xa7p\x98u2!\xe4\xd9[\x16\xc6_R\xe2\x0c\xb1\xd1Z\x16Vd\xca\x081\xd1\x8d\x1cu\x16\x81]\xc8^I\xf3a\x0c\xc9\x99G\xcb\xebp\xefe\xfd\x90hA\'+\x19q7\x80\xdd`\xdet\xbf3\x9b\xd2\xb3\xfc1\x91T9\xad\xcaS\xa0\x94 \xe8ko\x1b8\x9f\xc2a\xbe67\xf4\xd9K\x15}\xe9\xaa6&\x1d{\x11\xd5v\xa8H#\x8bw\x90j\xd9L\x03\x9ax\x96\xe1\xd3\xb2+O\x0cv\x10\x95j\xc7\t\x04`fh\x0e9\xd1r5\xb4c\x10\x00\xf2:k\xb0\rLzvL\xc5\xa8\x85\xddF\xd8\xbb\x1e\x84\x8c\xb77!\x8cO\xe10\xbf\x85Ib \x81\xbc\x84Y\xc7\x03\x1ei0\x04\xd2\xd1\xcd\xb7s\x7f=V\xcbK\x7fV\x87i\xd8S\xb1\x88\xf47\xcf\x9aF\xd2\x1b\x1b\x7f\x19\xd0\xdd>\xbdG\x03]\xd0\xa4w\xe6\xcb\xf1H\x16E\xde]\xc2\x05\xbe\xdd\xcbC\xe9\xeeD\x95\x128\xff\'\xe10\xbf\xbb\xa5\xfd\xef(\xcd}\xcd?N}\x97\x97v\x9177\x1b;\xc4\xd2Z\x85f\xc3|XPx\x99\xcd\xd2\xf1N,l\x17\xe4)lL\xd1\x02&\x98\xb9\x18\x8b=\x99\x085\x88\x98\xe7\\\x8a\x04\xa6\xda\xd6\xc5\xce\xb7r\xc78\xa6\xba\x9f\x7f\xb4f\x7fm\x1b\xe6\x99\xf9\xf0:\x848\xf8\xd2b\xd4\xc0\x94Eum\xfa,\xea\xc4\x0c!\xd9\x15\x04\xe8I\x08l\x19\x1a\x16&\x9f\xbaG H\x9a\xd0\x0ex&\xce\xa5S-:\rz\xcc\xceO\xe0\xfc\xe1\xc6=x\xa4\x8f\x04\xb3P\xa4sUA\xd8R0\xe1X\xdd\xaa7\xb7\xf1O\xd10\xcf\xa5B\xc1\x04B\xacG\xd6\xef\xc3\xbc+\x17\xae\xba\xde\xec\x93\xa30\x93+dKx\nr\xdbe\x01\x17O\x96\x94i5\xbb\xbb\x94\xf4\xf9\x1eQ\x08\xb7\x87\xec\x92S!O&\xd9\x8c\xf2\xd5\x18\xad\xfaR\x89r\xd5%u\x15\rT\\\xc4\xceG\xe8\x82\x16o\xf6\xde\xfc\x14\r\xf3U\xd8\x80\x14E\xfeQ\'\xd5\xabO\xd1T\x93\xed\xf5\xfb\xd1\xc0E\x8bhNG\xd7\x18v\x8e\x99L\\\xb3\xbbs\xce\x03t\x13\xa4`\x0f^l\xb6w\x0cw\xa3\x0bm\x9dY*\xe5\xa2\x9a\xeb0\xd8X7\xc7C\x96D\xee]>\xf21\xed\x07\xfa\xfeG\rc\x7fm\x1a\xe6\x99\x12\x10\xb9n\xba\xeb\xa9\xff\xfbQ\x8c\xef\x9dr\x95\x0c\xee\xd6\x16\x96\x9b\xb1{\xbf4\xd0\x060%\x91g\x9c\xf8p\xbdP=a\xef:\xdf\xad\x16\xf7\xcc\xc7\xd2|\xbc\xb6w\x88\xf4A/\xa8\x0f\xd0\xa1\x05\xaf\x92 \x91\xfd,V>3X\xa9\x8f\xeew\xfb\xf2]\x10\xeeC4\xcc\xd7\x85\xe6\xf9:\x0f\xc4_R\x9f\x97k7>\xea6(Nzx:p\x0f\xb4\x14\xcc\xf3\x08v\xd6.\x8bp\x00\xe4&\xda\x872\tB\x9c\x14a\x1flC\xd7\x85\xa2\x98\x89\xec\x06m\x98\x1b;8l\x94.v\xf6\x17\x9dId\x03\\\xa6{\xd4Ao\xe6\xc4\xa7h\x98\xaf0\xd7\xaf\xc4\x08\xf2\xc5\xc1\xe8\xab\x08\xd5F\x82\x9cQ\x94\x82\x9a8\xdfM\xfe5\xf4Sa-\xc4\x86\xa4wd\xa3\xd2\xef\xe3\xa1Z\xf7\x9d\x85\xbfp\x87\x86e\xc4A6%B;d\xb8R\xc7\xa1\x13\x8c\xcb\xcc\xc3Ja\xef\xd5\xe6\x01\xecg\xcd\xe7\xdfT\xa1>e\xc3|\x9d\xf9\xc4Z\xb7S\xc4K\x05\x17Q0\xdau\x8d;j\x0f\x88\xde\xf1\xc3N\r\xe4\x1aC\xbb\xc3\x90\x83CCw\xe8\x0e\xb4E&\xdd\x11U(.:\xa0]\xcd\xe0\x98kr4t\xe8i\x8e\xd0\x1fm\xe4\xbf\xb6\r\xf3u\xa9\x01Q\x12F\xb0\x97\x0b\xb0o\xad\xe5^3\x9e\xd94\xe0\x15\xbc\x96\x93\t9&\xf4\x85G\xf0cd\xd0\xe9\xb0\x87\x18O\xb1o\x1e\xc2\xd42Lv\x07\xfb~0k\xbf\x13\xc0\xfb\xa0\x19\x0f\x1b`\xe1\x9e\xc0\x1d\xa0\xf3\x81\xaba\x00\xc6R\xa5\x10\xfa\xe6\x05\xf8S6\xccW\xe5\x04?\xeb\xc3\xd7S\xdf\xbf\xf4\xa7\xa8\x93B\xa9\xbc\xdd\xf1)\xbc\xc2\xb6\xbb\xeb[Aue/@\xee\xe3\x15\xb8\xa77y=\x0e\xaa\xd8\xaa\xb9 I;>.\xca\x0b\xa0\xf6\xdaL\x99\x1c{\xefc(n\x0e2\xa5\xca\xac\xb0\x18W\xa7\x9a)\xe5\xcd\xc6\xe2\x9f\xb2a\x9e\x1b9\xbe~\xfd\xfa\xe7/m\xe2\xad\xea\xa2b\x91\x0f\xa3\x95\xdb\x9f\xe40\x9f\x08\x92s\x91.E\xae\xcc|\x02\x96\xd9\xc9\\\xafc\xf4|\xb8\xea\xa8\x03\xb4\x84\x95\x89B\x9f\xd9s\x1e\xec\x1c9\x9b*]\xb1%\xbcqN\x0e\xdb\xb5\x88p\x92\xce{F}3\xccO\xe10\xcfE\xfb\x9c\x7fh\xbd\xd8\xfc>L\x8c\xbe \xdd\xad\x1c\xcds\xc9\xbbD\xcf\xb7\xf2\xe4\x82f\xc4K#5\x14>W\x15\xba#\xfbD8h64\xd9\xaa\xaa\xa8\x03s\xb2\x0e\'\x10F\xeaZ\x14\x1f\x9c|\xa2\xf0\xdd\xd4\xfa\x93\xc5\xb2\x82\x0e\x95\t\xcbv?\xa2\xcc~m\x1c\xe6\xeb\xa6\x0f\xa2k\xed\xf4j\x88\x89Y:\xa6\xb7|\x16n\xd3\xb5\x96-7j\xe4s55\xa5\xa1\xdfn\xad\xb5\x90\xeb\xedE\xbd\xc6\xd7<\xaa\xd1;\xa7r-k<\xaa(O\xd6c0qE\xc7?\xeb\x8av\xb2\xcc\x13\x00\xb5\xf7\xb0\xb9\xa9\xd2\xf9\x9e\x9f\xdf\xac\x84?\x86\xc3\xac\x8b\x05\xc6\xd0\xb5\xdeG^\xc2\xc4\x0e\xbb\xfea\x9ek\xd61o#J>\xe0\xa0\xc7O\xeaD0\xbe\xf7@\x8cz\xb6x\xc2\x19\xb0\xcc\x0b+(\xb7\xa6\xc3\xbe\x93.Lb\x97#\t\xc5\x82&_8\xf3\x14\x07\xd2\xc0NZ\xd0\x1e5\xe7\x8ap\xa3\xbc\xbcI\'|\n\x87y\xce\xe6\xfa\x7f\xafy\xf1\xfaR\xba\xbd \x16\x95\xa2\x0f\xfe\xb0\xcf\x95\x05\xe1\xe3t\x18\xcc\xbb\xa0Fr\\8W\xadQQ\xed1\xd7\xc2\xd5M\x91\xa0?\xac9X4\xd9}<\xf9s@\xdc\xcbG\x80\xa9:\x87\xdc\xc6 ;)f\x8cy\xa9\xd8P-\xfbf\x98\x9f\xc2a\xbeR\x7f=\xab\x88uA\xfc>Lvi\x9c\x96;\x1d1\x8a\xb8\xb6\x9a\xec\xa4 m\xe2\xbd\xd7\xac\x175\x0e\xd3\x1f0\xed-&\xefI6\n\x9d/R\x98\x15\xbe\x8e\xc6\xe2>\xbf\xe4 \x889\xe3\xfe\xb2\xdem\xa0Q\xe2\xae-\xcew8\x193\xc1\xa8\x03?\t\x87yn\xe4\xe4z\x17\x84\x91W\xf2\x92\xba\xa7ze\xb3\'W\xf6w\xcb\xc1\xbe\xaa\xa9\x88F\x01\xc0 ]\x84\xa1\xc5\xfc\xb0\x1eR\xe3\x14\xd7\x1a\xa1\xda\x1c\xf7z\n\x0fv5\x84_\x07\xbfg\xd9\x9a@\x90\xc1\xea\xbc\xc5\xe8N\x96p\xbc\xb8\x8fY\xd5.c\xf6\xe6\xbb\xf7O\xe10_\xb9\x89\xac;9N\xbe,Z\x98\x85RF\xde\xb5\x17\xf2\xaaf\xb1g\x18g\xd9\x8b\xb0>\xcd\xaa\x90\x0c\xc5\x80\x8eX\x93\x9d\x03L\xc8\xbb\x9c6\x11N\x0eN\xfb\x9dY\x1e"\xff\xbc\'\x98\xe8\xd6_\x1cD\xf4j\x00\xf6#\x1eq\x05\x80\x1f.0\xf3\xbd\xaeo\x9f\xc2a~K\xfd\xb5\xa0\xc7^\xd5\x94>J\x9a\x89;\xb0Q\xae\xf3}c\x9f\x04\xb7.\xcb\xf6\x1a\xe2\xb8\xee\xa5\xf0\x99\x1c\xa3\x84l$\xf6\x94\x17\x11\x9e\xb9\xbd\x97\xa1a\x13\\\x91\n\xd25\x97\xc2=1(\x85p\xa0cF\x0b\x89\x9a`/\xb1\xac)\xdd\x9b9\xf1)\x1c\xe6\xb9X\xa0\xe7>\x0e\xbe\x16\xfc\x99\xba\x13\x10t\xb8\xe7\xe3#\xe1\xcfv}\xb8\xdf\x13\xe7\xec\xb4{\xe4\x92\xea1r\x18\x1d\xf9\n5\xfd\x9d\xce)\xb7\xe2u\xf2p\x83,\xaf\x08\xb24\xc6\xa4\xfc(\xc8\xf7lai%\xad.6\xb0\x86o\xfb\xa0\xc4?\xde\xbc\xbe}\n\x87y\xa6>J!\xd0\xf3\xe3\xd4\xdf\x87I\x16\xc6\t\xc0\x02k\x00JQ\xd9K\xb9\x17"]\x06JM\xc3\xe2\xeb\x95\xa5\xd7\xf6\xa6`H4.\x1a,g\x11Q\xec>R_Ao\x0e\x142\xb7(\xa2CP\x19\x98\xc4\xc5S\xb6\'\xa2\x0e\xbe\x94\xfa\xc5\x01\xd07]\xa8O\xe10_\x1b\xf9\x9a\xfb\x18\xf8\x1a&\x9a\x89\x8b.\xb9\xcd\x9c\xd3\x10\xa7\xe4\xda\x1e\xbeqR\x8d\xa0N\xa3\xc0\xa6\xe6\x11w\xc6\x08K\x8f\x10i\xccc\xe3\xf0h\x90\x8b\xe7\x18\x1eR\x99eg!\xd0dr\xd5\xb1\x86\xc5b\x1c\xbd\xfb\x055U\x8cK\xe7\xe1M\xa7\xf5S8\xcc\xd7\x0b*\x0c\xc2\x11\xe8\x15n\xaa-\xf4\xcaw\x017.d\xe2U\x96^\x9b\xd6\xad4\x8a\xe1\xac!\xb3\xe4\x1c\xf7\x18\xe2\x0f\x1d\x9f\x1d\xf0\xdb\xf10#\x93_\x9fv\xb2\xf28_\xb1[h\x9a\x84\xcb\xf3\x9e\xbbS\xc2S>\x9e\xf1\x0e\xc8\xba\xd0\xdec\xd4\xf7\xba\xa5~\n\x87\xf9\xba\xeb\xaf\xf7|\x82\x00_\n~Z\x89\xc5\xf4\xec\xcb\xa7\xda\xcb=+\xc4\x8a=\xbd_\xf2\xe8X\x8c\x88v.\x14gBwn\xe1\xee\x9d!\xf2\xe7u\xa4\x10*>vM<$\xfb\x88\x86)\x12\xb0R\'\xf0\xb8\xfbn?\x91\xac\xcd\xb3Z\xe7\xb8G\x94\xa5\xff\x1b\x1c\xe6k\xc7\xdap\x98\xff\xfa\xa7\xd4\xff\x1f\xc2a~\x1dvc\x93L\xb6!\xfd\xfeC\xba\xe10\x1b\x0e\xb3\xe10\x1b\x0e\xb3\xe10\x1b\x0e\xb3\xe10\xdf\xf897\x1cf\xc3a6\x1cf\xc3a\xbe3\xba\xb2\xe10\x1b\x0e\xb3\xe10\x1b\x0e\xf3m\x9fs\xc3a6\x1cf\xc3a6\x1c\xe6;\xa3+\x1b\x0e\xb3\xe10\x1b\x0e\xb3\xe10\xdf\xf697\x1cf\xc3a6\x1cf\xc3a\xbe3\xba\xb2\xe10\x1b\x0e\xb3\xe10\x1b\x0e\xf3m\x9fs\xc3a6\x1cf\xc3a6\x1c\xe6;\xa3+\x1b\x0e\xb3\xe10\x1b\x0e\xb3\xe10\xdf\xf697\x1cf\xc3a6\x1cf\xc3a\xbe3\xba\xb2\xe10\x1b\x0e\xf3\x06\x830\xfe\xcb\xef\xf6\xfd\'\x06!\xf9\xddo\x02~c\x1c\xe6/\x08\xec\xe7\xe00\x7fA`?\x07\x87\xf9\x0b\x02\xfb98\xcc{\x81\xfd\x9f\xa8\x91\x9f\x83\xc3\xfc\x053\xf6sp\x98\xbfb\xf3\xd8p\x98?\x03\x87!\xffg\xc8\xff\x03\x0eC`\x0c\x8c\xb1\x0c\rB\xb4\xc0\xf38!<\x9b\xcc"4\x03\x82<\x8f\x08\x08B\x80\x0c\xc9\x82\x10A\xd2\x18\t#<\n\xf2$\xcc\x80,\xc9\x91\xeb\x17\x92\xfc\xdf\xfe\x1d}\xc0C\x1cH"(\x01C0#@8M@\x14\x8dC,I\xaf\x7f\t\x8a>\x1b\xd9\xa1$\x0e\xd2\x14B\xae\xdf\x97\x800\x8e\xe0)\x14\xe2\x11\x9c@!\x96"\x85?\x11\x87\xf9g#\x8b\xbf\xfdA\xf7\x05\x94\xf8;\x06\xe2\x04\xb9>\x16\xf6\xeft\x98\xef\x0f\xeb\xfc\x81\x0e\xb3\xce(-`\x1c\n>[\xb5\x90\x04\xc8\n<\xb8\x8e<\x0c\xa2\x98\xc0\xb3\x18\r\x92\x18H1\x02\xc2P(.\xb0_\x1d\x04i\xee\xd9\xce\x86\xa6\xd7y\x84\x9e\xed\x867\x1d\xe6O\xd4aH\x82\xc1\xd6\xaf\xe3I\x02\xe2\x19\x81E\xb1g\xcb:\x86"\xd6\xcd\x04\xc7)\x96\xc59\x86\x87!b\x1dOz\xfdF \x0e\xc2\x14\x8e\x91$\x89\xa10\x02\xe1,\xf8\xb7M\x87yO\x87yv5\xf9\x87\x0e\x03\xfd\xa1\x0e\xf3\xfd\xd3\xfc\xe7\xea0\xf8\x0f\x9b\xb4\xe7`{\xa8\xf2\x9d\x97\x8f\xd7\xf3\xb1t\xaa\xe8\xd6\xed\xc4\x04bg\xdbQ&\x16\xc0o\x9d}J\x84D\xdc\xef\xce\xe3\xd1\xab\xcc8Z\xeb\x02\xf0!\x86X\xcfV\x06\xe3\x9c\x06e\xa7\xd0\x13\x0b\xa9\x06-\\\xfb\xdcT\x0c\xe7G\xbd)\xffd\x1c\xe6y&`\xebJF_\xfb\xa9\x1e\xef\xc7\xf9\xaa\x8fn\x0f\xd1\x8c\x07\x1f\x1e\xa0\xec\xa6\xe1\xc4xL\xc3\xf7P\xbb`M\x14y\x89\x1dh\xc7Yh\xac\x8b\xe7\xd2\xb7\xbc\xa6\xd6\x81mDCp-%\x9f\xf6\x8dw},aZT\x89\xc5/\xf7!B\xad7\xbb\r~\xca\x86yFI\xa0$\xf5\xd2o\x0c\xc9$\xf2\x01\xeb\x9a\x02\xec\x99\xd9\x1a\xb52\x1c\xe2F%d\xa2\xacF\x1f\xab<\xc0#\x12\xfd^Y(yP\xfa\xb2\xda\x8b\xf1,K\x0f\xf5\xd1Y\xbb\x90\xcd\xafgLs\xd7\x84\x8b\'\xe3Nw\\\xd7.\x15\xc3\x16o\xce\xe4\xa7l\x985\xc6\xb5\xf0Z\xf7 \n~i\xc45\xdbZs\xbez\n\xa8\x15bA\xa8y\x7f\x86\xe5`-\xda\x06\xf2^\x05\xda\x99\x8dY\x8a[\x9f5\x87\xca>a\xfbkw\xe1\xd2\xd3D\x93B\x7f\x9f\x80s\xe0D\x93\x00w\x8cts\xd3\xa1\xa9\xfd\xd6\xf5\x04\x86\x84\x7f\xd4j\xf0\xd7\xc6a\x9e+e\xad\x14\xd1g\xdf\xc3\x97\x06\xc3\xfaH!]zd:x\x8f\xfbP\xd1\x95\xbb\x108\xf1\xdcr\xac\xa4\xca\xcd\x13\x1a\xbch\x14b4@\xa7\xf4\x11\xb9,\x89\xe4cm\x7f(\x81\x03h\x89\xed0\x96k\xd9)\x1fTf\xa1\x8fp\xe4J\xad\x82<\xcc7\x17\xcb\xa7t\x985\xcc5\xe7Qd\xad\x94^rb\x0f\x97\xf1\xe1~\xc3\x88,\x10)b<\x14\n7\xf1\xc9z\xbdQ)t\xaf\xf8D\xa2d\xe6\xbcZ\x08SMh1\x17\xe3\x1eV[\xb5\xea\xcbsu\xf4G\xa1\xb2,\xe9\xd8\x1c]r\xe0\xd9\xb4\xf13\x82>)\xc7\x10\x13\xc7\x9d\xcb\xdf\x1c\x97\xda[c+\xf6Kl\x8a\xe7\x82\x96\r\xcbW\xe8\x19j\xac\xf9\'\xe10\xcf(\xd7j\x02#\xb0\xd7\x06\xc0C\x06\xa4\x99\xd1J\xc5\xde\x12\x85\xcc\xda\x07,_R)<\x12\x97\xc3\x15@\x04\xeaJ\xd5\xf1|r\xb5\x93&\xf1\x96\x1e\xf4\xa3\x93\xa0\x0b\x90tG\x8aN&\x9c>t\xfbQ\xd7Ht\x8f\xb9\xcdR\xa4\xac\xe6\xa4}\xcb\xbe\xd7K\xf5S8\xcc\xefoi\xff;L!\xb7\xfd\xe1\xe0P\xe2"\xd1\xa5\xa9\xb3\xb5\xc8r\xed}\xe4\xbb!W\x8e\xccH\x96Wh!.f:[\xbcF\\\xc1\xea@\x8fu\x169)\x99\xf7\xb93\xf7\xf3t\xad\x9dR?\x971_\x80\xc0\x9e\xf5\xce\x85\xc4\x7f\xaf^\xaa\x9f\xd2a\x9e\x99\x8f\xc2\x10\tB\xd0\xcbb\xa9\x02k\x07\x0en\x06\xc6~\x12tyl\x031Q\xd1\xf5,/8\x8a#\xd4\xad\xb8\xba\x86Q\x8b)x\x85\xe6\xfb\x19\xb8\xf3U\x9a\xc4H#K\xddbW.\r\xb0\xad\x82b\xf1\xc3\r\x15c\x1f\xe7\x07\xaa\xf1F\xff\xcd\xc2\xe6S<\xcc\xd7b!\xd6\xdd\x12|\xadR\xb3~\n\xfb6\xe7\x18N\x8a\x02\xeb\x84\x1b6\x19\xf8\x12\xae\xdb\xf8\x83\xda\xdf\xa5\x82\x8c.\xd11(\x1a\xe3\x942TJ\xdb\x12\x9b\n\xcc\x05\x84z\xfb\x1c\x88\xe6\xd1mo\xd6`\x13\'=\xba\xf66\xbf\x8bN\xa5\xf7\x98\xdel2\xfa)\x1e\xe6\xab\xb6\xc1\xc8\xe7\x8e\xf8\x92\x13\xf7\x14\xe2\x8f\xa82(vj\x807L\xc0\x94\xbe\x1f\x1c\xa4\x90#\xf0\xac\xcd\x8b9qZO22\xba\xcc\xb7\x1b\xb5\xf0w\x85\xce[\xad\x14U\xcb:\\\x15\xd2v\'\t\xcep\xbbk\x90\x12Q\x83CI\xc3u~\xfbQ\x98\xbf6\x0f\xf3\x95\x13\xeb(\x92$\xf2r\x1aR\xb5\xe5\xe6Z\x9a\x8f\r\x99\x82\'E\xa8v\xd7\xdd\x15\x9e\xeeJ1a\xc0\xe9\xc2>\x0e\xbczM4\xfd\xa0\x18R\x88[D\x1db\x17\xd0\x0c\x8d\xf3\x88\xdeH\x9a\xbec\x1a\xe5r\xcbA\x07)\x11$\xf6\x07\xfdl\x01\xf2\xbbH\xe2\x87x\x98g\x98\x08\x02\x13$H\xbd\x1a\x11\xca\xc4\xfba!\xfa\xf3\xdc\xae\x8f\x88\x9b\x89-{H\x06C\xd0\x1c&|\x1f&~\xad\xf0\xca>j\xbb;\x9c\xda\xf7\x853\x95\xc08\xb8\\\xe9W\x9d\x95\xef\x82\x89\xa4{s\xd4\xfc\x96@\xfb\x16\x04\xeb\xb8r\xe87\xe5\xbbO\xf10_\xb5\r\t\x83$\x85\xbc\x9c\xfa\xc4\xba[\xba\xa3\x12\x9d\xe1\x9aA\x18\xaf\x1f\xe7s\x07\x1a\x94\xdb\x95xED\x8a\xb5\xf7\xf9zw\x80]JK\x81\xbd4[\x130\xba\x04\xd5\xfb`\xecYR\xa0q\x99\xe6\xdf\x95+.^\xc0\xdd\xb1\xd6J\x8b\xba\xf9\xce\x9b\xb5\xcd\xa7x\x98\xaf\xda\x86\xc4\xa0\xb5\xea}\r\x93k3\x06K\x0e\xd2~4/e\xb0?\xed<>\x94\xe7(\x13\xf6\xba\xc7\xc4\xdc\x917;+\xed\x9c\x06\xe2\x0e\x04\xec(\xb5qs\xa5\x87qpnJ?s\x0fY\x96MP\x9d\xaf\xcct:\xd1\x8a\x8a\r8\n\xfd\x10\xfb\xf9\xb5y\x98\xaf\xd4_G\x10\x7f\xa5\x84\xee\xd2\xd4\xd7\xc6\xf9\x0e"u\xa1\xceLCe\'\x98\x95h5\xac\xef\xce\x02\xc0\x9a\x84\xe1\xa1S\xdfl\xbf2_\xceZ\x95\xc4Z\x9b=\xaa\x05/\x8a\xb0C\xf7\xbbD:\xaf\xd5\xca,j \xca\x96\x90\xb5\xb7\x82]\xb8\x0fq\xfav\xbc\xeen\xf7\x8bJ\x89\x9d<\x89\xe1\x9e\x84\xa4\xf9\xa1\x1f/n2]\xb5\xf0\x0eY\xda\xe8F\x8c\x9eZo\x86\xf9)\x1d\xe6\xb7\x17\x1a\xebdb\xe4\xcblN\x07\xd5\xe6\x0e\xf7\xfc\xce\x97\x8dz\x14\xc5\xee\xa4\xa7\x15\x15\xa0\x17\x0c?\x1b\x80T1l\xb3\xd3\x01\xa8$\xb8\xbd\x9b\x13\x88J\x9eP\xdc\xc90\xa4CO \x81r;\xd8\xb5\x82\x07\x03BA\x91J\x0fLfDh\x8c\xb8ou\xe8\x7fJ\x87\xf9\xca\t\x0c\x86a\xe2\xf5\xb5\x10\xd8\x12p\xecr\xe7\xc1\xe0\x83\xc8\xd2"\xb9\xdc\x0b5\x15\xa9\x98\x15\xf7W\xffx\xd5\xd0\x9d\x04\xa4\x889S\xe5\xa9.\x00(?&\x05\x90\xab\x00\xd9\x84\xd9\x01\x93\xf6{\xb6\xc3\xd7[\x04&\x14\x96^\xf9\x8f\xc29\xc0\xed\x9b\x02\xc5\xa7t\x98g\x98\xeb\x85\t\'\x91\x17\x80\x82\xc3B\x08\xa7Y\xdb?\xb9\xfa\xfd\xd0\xae\xc5`\xb3c\xfd\x9b\xd7\x8e.\xbd\xd3\x1f\xeb\x93c!uT\xdb\x06\x1dx_\x96\xd3l\xb8\xc6f\x06\xe1\xd3Q\xde\xedr\xdb\x0c\xcc\x9a\x91\xbb\xbd^\x84\xf5t\xf5\x9c\x04\xaaU\xe9\xcd:\xf8S8\xcc\xd7d\xae\xb7|\xfc\x0f\xce\xfc\xb9i\xa2H\x9d\xe8~A\xb8\xc39cRS?\x13\x1d\x92,\xf6>\x16r0\xc9\xf2d\xbf0\xe8p9\x9b\xceZ\x01^\x1dt-\xc6\xd7\xa3\x9f\xc7\x0b\xe6q\xb6\xe6F`T\xa0;\xb2\xc2E\xf7\x07s\x80\xf8\x9cW\x7f\x12\x0e\xf3u\xe6\x83(J\xc0\xafg\xfe-T\xd2R>\xd9\xd2\x92\x9eBh\xd2\xcdGp \x0e\x88\xa6\xa9%\x07\xee\t\x93;\xde\xeekY\x0c\xb13T\x17\xe1tp\xa5\xf1\x9a\xdew\x85\x1e\xda\xb5/5\x91\xab\xc6\x89\xb6\x17.\xb5\xd2\xbb\xdc\x91\x01\xdd[\x1b\xbc)D|\n\x87\xf9\xedV\x03\xa3\x14\x8a\xbf,\xda\x0cM\x95\xdc\xba\xc1\x0f\xf6J\n\xd4\xe8\x11\x13}\xee\x1e\xb2\x98V\x03\xe3\x86\xfep\x90 =s/VW(\xdd\xe9t\xa34\xc8\xc6+\xb9\x83F\xaa\x01\x1f\xa8\xce\x80\xb8j\x94\x98z\x90\x96S~Y\xc2;~.\xeeo\xbe\xd0\xf8\x14\x0e\xf3\xf5\x11\x03\x85"kv\xbe\xec\xe3\x98\x0c\t=]\xb9\x88\xbf\x8c\xb8\x84#\xea\x9934\xcfQK\x17\x025\xc7\x00\xf6\xb3-\xb8{\xa3\xe6\xec\xdd\x85i\xca\x82 #\xe0Q\xba\x95\xbe\xaf-\x7f\xa8\xa9\xb3\xd4\x1d\x8e\x87tAf\t\xa6\x14\xb0e\xae\xea\xe3Ga\xfe\xda8\xcc\xd7G\x18\xd4z\xa9\xf9\x03\xfb]yH\xe0\xb8\xb7\xd6\n\x1ekJ\x06\xaa#jT\xc4"\n`\xc2\x8dwE\xe3\x10\xf1L\x13\x97i\xf0\xd0\xe1$h\xa1Z\xd9\x13\xb1\xbf?\xc2\x94G\\\xdc\x9d\x01\xf4\xe8x\xcd\xeeq\x13r\x98&|\xd5/@$z\xb3\xdc\xff\x14\x0e\xf3uyCa\x12\xc3^\xdfe\xa2\xb1\x0c\xcb2\x01\x1d\x92PV\xdaxa\x06\xad\x8cS\xd0v/\\\x1a\xfa\xb4Q5@s\xc7r5\x83R\xa1\xce\x13\xc9&M&L\xa6\xb9CUP\xedd[\x89\x80\x1b{8\x0c\x0f\x91\x02%\xfe2\xb5J\x8c\xff\xe8\xd0\xff\x93q\x98g\x98\xebI\x87?/D/8\x0c\xd1^\xc8e\xa7\xcf\x03\xb0\x04\\\x16`\xcb)q&`"f\x026\x97\x81r\xfb\x9a\x1e\xe6\xc4\x0f\xda<\x1bM\x19\x01\x8f\x9d\x99\x10\xe5\xc2\xb2sg\xc1x\xd2\x15"|Y\xc8\xf2>_\xc8GZ\x86g\xc5\x9d\xde4\xdb>\x85\xc3<\x17-\xfeD)I\xece6Ys\x80\xfd\x14\x1b!\xef\xe1\x00\xc2\xde\xb4\x8a\x9bw\x01)@[l\xbc5w\x91Q\xda\xb4y\x8d.b|\xd7Q(\t\xc6\xea\x9a\xe7\xf1,\xfa\xbcgt\xb5$"F\x15GAY\xb5\xa5>\xcc\x90R\xde.\x10\xf6&H\xfb)\x1c\xe69\x9b\xe4z\x19#!\xfc\xa5\xc6*\xd5\x19\xd7,\xe40\x82\t5\x9dM&k\xd7\xe3^L\xf6:J\xa7b|\x118l\xda\xb1\x8e]\xf4UU@\x13\xc7\xdfT+(\xa0=\x8a\xde;\xa4pn\x9eh\x8e\x80\x9e\xf4P\x82Wk\xf1\x8e^T\xb1\xbe~\xafW\xb6\x9f\xc2a\xber\x02!\xd0\xe7U\xff\xe5\xfd\xfe\xc9\x1eA{\xbd\x8c\x00\xdc\xe5r9\xa5D\xea\xc0L~^8\xafjr\x82WvH\xf0`\xc9\xb8\xb7\xec\xbd\x8d\x0c\xf6\xd2?`\x83\x97\xd3G\xc9F\x81\x9d\xde\xfc\xb3.\x03\xb7]V\x0c\xb8\x1e\x0c\x17\xef\x81\xd01\xf7\xdb%\xf5?\xe10_\x93\xbf\xe10\xff\xf5O\xa9o8\xccO\xe9\xcc\xb7I&\xdb\x90~\xf7!\xddp\x98\r\x87\xd9p\x98\r\x87\xd9p\x98\r\x87\xd9p\x98o\xfc\x9c\x1b\x0e\xb3\xe10\x1b\x0e\xb3\xe10\xdf\x19]\xd9p\x98\r\x87\xd9p\x98\r\x87\xf9\xb6\xcf\xb9\xe10\x1b\x0e\xb3\xe10\x1b\x0e\xf3\x9d\xd1\x95\r\x87\xd9p\x98\r\x87\xd9p\x98o\xfb\x9c\x1b\x0e\xb3\xe10\x1b\x0e\xb3\xe10\xdf\x19]\xd9p\x98\r\x87\xd9p\x98\r\x87\xf9\xb6\xcf\xb9\xe10\x1b\x0e\xb3\xe10\x1b\x0e\xf3\x9d\xd1\x95\r\x87\xd9p\x98\r\x87\xd9p\x98o\xfb\x9c\x1b\x0e\xb3\xe10\x1b\x0e\xb3\xe10\xdf\x19]\xd9p\x98\r\x87\xf9\xef\x19\x04\xed_\x7f\xb7\xef?1\x08\xe3\xef~\x13\xf0\xfb\xe20\x7fE`?\x05\x87\xf9+\x02\xfb)8\xcc_\x11\xd8O\xc1a\xde\x0c\xec\xffD\x8d\xfc\x14\x1c\xe6\xaf\x98\xb1\x9f\x82\xc3\xfc%\x9b\xc7\x86\xc3\xfc\x198\x0c\xf5?C\xfe\xefq\x18\x9c@p\x88\x83@\x08e@\x96GA\x8a#\x05\x01e\x10\x88%a\x8cfi\x96\xe3x\x18\xc4H\x92\x03Q\x06\xa3\x04\x9c{*\x14\xcf.\x8d4\x05r\xc4\xf3\xfb\xfc\x1b\x1c\x06\xc1\x11\x9c\xc0Q\x9c@y\x16\xfcrGHH qV\x80I\x1ce\xc8\xf5/dI\x08\xe5\x9e\xad\xbcQ\x01\xc2\x05\x1a\xc2\t\x9a"`l\xfdw\x06\xc5\xffD\x1c\xe6\x9f\xad3\xff\xf6\x07\xdd\x170\xe4\xef\x14\x8cB8\x0e!\xd4\xbf\xc5a\xbe=\xac\xf3\x078\x0c\x8c\x93\x02\xfc\\\xd8\x02\xc6\xb0\x0c\x8c\x12,\xb3N\x04\xc9\x938L>{\xc6\x83\x82\x00!\x0cA\xf3\x0cI\xd1$\x8f \xf0\xb35:\x8e\x11\x84\xc0\xa1\xebC\xfcm\xc3a\xfeT\x1c\x06Cx\x16FI\x08f1\xea\xd9\x14\x14!\x9e]\xec\xd7\xc4\x04\t\x90\x10h\x01\xc7\xd6\xd5\xc3\x11O\xdf\x87\xc2\x10\x08\'\x9f\r\x8fA\x02!(\x88\xe5)\xe1\x97\xc7a\xfek\xbd\xe0S8\xcc\xb3\xab\xc9?p\x18\xf0\x8fq\x98o\x9f\xe6?\x15\x87A\xc9\x1f7i7\x9bLt\xcc\xb4\xac\xa8\xd2p\xf4%8\x826p\xd7\xc1k\x19@\xa2\x97\x9cGR``\x85\xe8\xe2\xd3\xa2\xf6&\xd8\x1b\xad\x91\x00%D\xcfDCk^v-\xd9\x9c\x0bv\xa0\xc92;u\xd4uZ\x97\xa6\xe4\xfcf\x87\xbaO\xe90\xcfC\x01_\x83$\xd1W&\x02F\xd3\x1c?\x10\x92\x13\xec0P(\x1eLa\xa6\xb7*)\xa2!1\tAat\xaa\xcc\xf6\xb5r\xb3NXj\xb3\x0c\x06Y\n=w\xeaaw\xca\xb4\xaah\x1av\x04[V+r\xeb\x1e+\xb2\xc0\x0c\xe9\x00\xaao\xf6\x8d\xfd\x14\x0f\x83\xa1\x7f\x87`\x1c\xc6\x90\xdf\xc7XgQab\\\xa6)\xf9UA\n\x0b&\xa0\x0c\xb7N:q\xb2L|\xd93\xec\xe9\xcc\xd2\xc5\xe5\xb2\x8f\x01\xcf\xd0"\x9f?\xf3\xd3n\xfd\xab\xda\xe8BND\xd0\xdd\x8a\x07\xd0S1\x83y\xb9\x8d_\x1d\xfb\xd1\x99\xc0\x9b\xad\xb8>\xc5\xc3\xac1\x82\x18FP\xf8z\x0e\xbetW\xaak\x134*\xef<\xb7\x86\xf8\xe8\x02\x03\xbf\xd9\xbe\xfb8\xdd\xda\x87\x85\x8f\x05]\xb7\xaaN\x85\x15\x8d\xed\x0cT\xe1\x80}\x9d\xda\xc51lO5\xa9^=\x18ft\xbe\x8f\xce\xfb.\x9a&\xa1\xbd\xeb\xe6\xc0\xde\x85\xf9GS\xf9k\xf30\xcf\x84x\x9ex\x04I\xbc\xf4\xa8:\t\x8d\xafdZ\x04\xb73\xe7\x1a\x04C\xa1\xeb\x12\x04\x8b\x0b\xe0\x0b\x92\x8eDu\x81\xe8\x96\xfb\xa0J\xf2\xa41\xb2\xb8\xa3\xb4>\x01\x9c&:\xf0\x1c\xa1\xc3=\x99\xc4x@\xb9\xd9\\\x05\xe7\xfb#\xe9\xc3\x94\xeb\\\xfcM7\xe5S<\xcc3\xccg7\xb3\x97\xb6\x94\xb8\xfc\x98\xbc\x01G\xae\xb9\\\xd9\xb8yK\\\xcdCZa\xdd\xe8\xc3&\xd7\xf3\x85u[\x83\xb2\xf6\xc0)\xf6=\xb4\xa7\xae`\x8d\x1d0`\xc0\xfbc\xc2w\xa2\xedw\xf49\x1cnLq\x14\xa3\xf9^\xf4\xe4\x8d\xc0\xa37\xdb\x0b\x7f\xca\x86y&\x04\xb1\x0e\x06\xf5\xba\x83\xfbL\xa7\x93W`/\x88 \xfb\xf9\xa9\xb3au\x02\x1d\xf1\xaeu\xc8\xf9\xb6\xdf\xeb\x94(\xcaK\xd8\xad\x03\x01X\xa2R\xd9\x8e]\xb2\x81h\xdc\x12\xe7r\x82#\r\xd1\xce\x1cl\xcbY\xb2Xi\xe5\x10\xe7X\\\xff\xfb\x9b-\xb1?D\xc3\xfc\xbe\xac\xff\xdfa\xaa\xf0\xb5\x0b\xf9s/\x1e\xdd\x1d\xd4N\x17\xa3\xe3\xe3\xdd\x0c\x11\xd3\xce\xd6$\x00\xb7u1k\x17\x0f<\x0c\x81\x98"\xa1r<\xd3\xacQ]1\xf2x\xc3\xec\xaaa\xddG\xef(\xd6}\x0f\x0268>\xc0\xfb\xed\xa6\x13\xcd\x9b\x8dq?e\xc3<\'\x13\x82\t\x18\\\x0b\xda\x97\xfe\xbf\x8buP\x96k\xb6\x1b\x0e\x94w\xc4gNE\xda\xc0\xd3\x0e\x04\xe4\x1e\xd2\x8b\xdds{\xf8 N-\x897\x80\x01\x1c\xad\x9dt\xc5\xd02_w\xf1\x83y\xd89\xea\x85\xd6\xe5(\xbdM\'\x9eb\xab\xd8\xbf\x9a\x03~\x8e\xdf\x9c\xcdO\xd90\xcf\xd9D`\n\xc1 \xe4\xe5\xacrK\x1a/\x95\xbd\xbc\xeb\xd1\xf3\xd8!1~\x14\x10S=\x86:/\xb0\x85A=\x9c\x14\x88%\x98\xd7\xa3\xde\x97\xf0\xbb\x84\x1d\xbaI\xbd\x9a\xd6\xde\xd3}\x8d\x9d\x9b\xa0\x14E&\xcb\x9a\xe8\x86x\xd6\x04\xc0>\xc0\xc6\x96\xb5fP\x06o\xe6\xdf\xdc\xc6?E\xc3<\xa3$\x08\nYO\xf2\x97\x94\x80\x0bo\xd0\xaa6\x13\'\xe0\x9a\xbaXN\xcc\x01!y\xb6\x92U\xa9\xbb\xdf\x0f\xe9X/\xeb\xd2?\x8ed\xa6\x8e\xc6\x91\x94\xc1\xda\xdd\xb5\xa5\x91\xa5\xe4\xfe\xbcCR,\xe2\x8e\xc9\x15\xb8{1\xa5\x94@$P\xcd\x91\x0e~\x94\xf9\xbf6\r\xf3\xb5\x7f\xae\xf7\xedu\xb1\xbc\xa4\xc4%\xbbyp).\x13\x1d9\x85u\xbc\xd4\xb1\x00\x0e|#\xa6Yh\xdc\xd2\xb4\xe9j\xdd\x12D\x9c/`\xfa>\xc8mr+\xfc\xe5\xf9c\xd6\xeb\x96\xb98>z\x8d\n\xae[t^\x8aKC\xe6\xefD\x8f\xe3\xfa\xecv\xba\x9c\xc2BV\xebf=\xfa\xf3\xd2\xeb\x17\x01\xf3\xb0#\xd1\'\xb8 \xef\xe6\x91S\xbb"\x15\x1a\xb2Z\xb3\xa9t\xb3\xdb\xee\xea\xdc\xdf\xf4\x8c>e\xc3|\xbd\x92\x86\xbf>1{U`\xe7\xc1>u\x87\xec\xea"\x17z\xe7\x18eo?\xf2j\xf4|\x87\xc0/\xad|\xc3\xf0;\x07Y\xe7\x13o\xf1\x841\xe5\x02\xcc(\x8e\xb6\x0c%\x85\xf5\t\n\x82u\x18\xc2X\x17\xc7\xc99/\xd0\n\x81\t\x93\xee\xe57+\xd5O\xd90_\xb77\x0cCH\x92xY\xb3\xc2d]\x0fn\x9a\xb5`j\xb5\xbd\xdd1im3WI_\xe0\xfe\xc4+^\x80\xa0a\xe1\xb0\x01\x839\xb4\x80\x8e\xa5A\x87u_\x90\xed5\xba\x02\x1c\xdd\xa6\xd9rc\x89k\x9d\xa8\xd7\xbb\xaf\xc6\xa8f\x8dz\x99\x7f\xaf}\xfcS6\xcc\xd7\xebZ\x0c\x87 \x12|y\xb7OGah\x13WA\xe8#f\xe8i\xe3\x86\xe89%\xf4\xb3e\xf8\xa5\xc9\xbb\xe4z\xdd}X\xca\xd9\x11\x12hA\xf6\xb0\xe2g\xfe8\xd15}\x84\x14\\\xb5.\xe1\x81\x9dk@\xafh\x00\xee\x8dT#R\x89Boo\xc2\x9e\x9f\xb2a\xbe>\x8d"\x10\x12\x82\x91\x97\xda\xa6\x91\xddY"\xae\xe0\t\xbe\xdb\xe11\x95Jj\xa7 \x11Xt\xf7k\x9fk\xb52\xa4"\x7f\xa9\xceW\x1bu\x15\x9e\xbf)Pq\x10\x84c.D\r\x01WV\x7f\xb9{\xc1X\xba\xd2\xec\x07ENW\x9dt=\xb7\xf3\x9b\xb5\xcd\xa7l\x98\xaf\xd7S0JR\xd8+\xd6h\xdb\t\xdd\xeb\x83n!E\x1e\\\xc9\x9a\xe2\x12\xc1\x08\xe0\xda\x19\xe7\xa3w \xee\xa5\x9dJ\xa6\xd0\x99\xad1\xcd\xc4\x9d\x85\xa5\xdb\xd1\x13\x80\x01\x7fXW\xdf\x8e\x9a\xb0=\xe0fG\xf0(\x9c\x8f\x82\xd4_\xc4\xbd\xd2\x05o\x86\xf9)\x1b\xe6\xabR\xc5\xa0\xb5\xf6xE\xbeNj\xa9\xd6\\\xa7\xdf\xfd\x10tgat\x13Y\xb9%\xc6\x8e\x80X\xfbb\x18\xfe\x81!\x15\x11?\xed8%\xcfG>v.\x1a\xd1\xdc\xc0\xab\x0b1\xb2\xa6\x02\xdc\xba\xcet@p\xac\xdc(<*y\xb0\xb5\xb8\x97\xefo\x82F\x9f\xb2a\x9ea>\x8dV\x0cy\xfd\x9cA?\xb0>4\xeaL\xa1\xf6~\xe7\x10\x81\xc4a.r$\xefg\xe9TLz\xba\x9c\x1dGa\x8eJ\xbf\xd0"\xa0\x84\xf7Y\xaaF\x17\xc2\xb93"\x88\xb2e\x03\xa0y\x85\x16P\xb5\xefe\xec7\xbd\x11\xb3\xfb3\xe2|\xafW\xb6\x9f\xb2a\xbe>\xbd\x04\xe1\xa7\xff\xfer\xd5\xa7\xca\t\xae\xf8\xaa5\x99\x86\xee\x8c\x80\x82\xe3\xe5&#\x81\xec\xe3\xd6\xc9y\x9c\x1fV\xb2G+\x02\xaa8\x0e\xcfH\x87ypV\x92\xa9u\xaa\x9fM\x1f\xaeF\xb8%\x16d0\x13W?x\xfb,$"[a]\x9a\xfd\xafl\x98\xaf\xf7M\x9b\r\xf3_\xff\x90\xfaf\xc3\xfc\x94\xc6|\x1bd\xb2\r\xe9w\x1f\xd2\xcd\x86\xd9l\x98\xcd\x86\xd9l\x98\xcd\x86\xd9l\x98\xcd\x86\xf9\xc6\xcf\xb9\xd90\x9b\r\xb3\xd90\x9b\r\xf3\x9d\xcd\x95\xcd\x86\xd9l\x98\xcd\x86\xd9l\x98o\xfb\x9c\x9b\r\xb3\xd90\x9b\r\xb3\xd90\xdf\xd9\\\xd9l\x98\xcd\x86\xd9l\x98\xcd\x86\xf9\xb6\xcf\xb9\xd90\x9b\r\xb3\xd90\x9b\r\xf3\x9d\xcd\x95\xcd\x86\xd9l\x98\xcd\x86\xd9l\x98o\xfb\x9c\x9b\r\xb3\xd90\x9b\r\xb3\xd90\xdf\xd9\\\xd9l\x98\xcd\x86\xd9l\x98\xcd\x86\xf9\xb6\xcf\xb9\xd90\x9b\r\xb3\xd90\x9b\r\xf3\x9d\xcd\x95\xcd\x86\xd9l\x987\x14\x04\xe3_~\xb7\xef?(\x08\xda\xef\x7f\x13\xf0\x1b\xdb0\x7fA`?\xc7\x86\xf9\x0b\x02\xfb96\xcc_\x10\xd8\xcf\xb1a\xde\n\xec\xff&\x8d\xfc\x1c\x1b\xe6/\x98\xb1\x9fc\xc3\xfc\x15\x9b\xc7f\xc3\xfc\x196\x0c\xf4\xbf\xe6\xf1\xdf\xe30\xd0\xfa\x94(\x06b(\x0f2\x04\x8aq,\x8fb\x18\xcc"\x8c@\xc3\x08L\x91\x0c\xc2"\x08\r3 A\xf1(J\x930\xca\xf24\xc6\xc1\x1c\x8b\xe14\xc3=\x9bn\xfdX>`)l\x8d\x04\xa5\x90g\xabG\x02g\x89uHq\x96\x85a\x0e\xa5\x11\x94cx\x9cf\x9f=\xbaq\x04\x11h\x88\x07q\xe6I$\x08$\x8e"\x94@\x83\xd0\x9f\x88\xc3@\xffl\xc2\xf0\xb7\x97\xfe\x0b\xf0\xff\x07\x82\x7f\xc7P\x10#!\xf0\xb7~k?\xd2a\xbe\xbf\xac\xf3\x07:\xcc:\x85\x18G\xb2\xd4\xb3\xd9\x16\x05C\x1c\xbd\xfe\x83\xb2\x18\x0b\xd1$\x0f\xf1,\xc6\xb0\x1c\xc9\xf0\x1c\xc5\xb0\xeb\x8a\xe0\x98g\xb3*\xf0\xa9\x950\xb8\xb0N\xa4\xf0\x1c\xc5M\x87\xf9\x13u\x98u\x1ay\x9e\xa3A\n\xc6\xd7,\xe39\x81A`\x9a\xc4ix\xfd\x9bP\x18\xe5\t\x9a\x86\x08\x14\xe4\x9f\r/q\x18\' \x12Y\x87\x98G\x10\x08\x86\x18\xea\xab\xaf\xd0/\xad\xc3\xfc\xd7]\xfd?\xa5\xc3<\xd3\xf0\xdf\xea0\xdf?\xcd\x7f\xaa\x0e\x83\xa1?n\xbao\xe3\x836M\xd7\x94X\x98;\x9d\xa9I\xb7\xdb\x01D\xe1\xeeD\x93\x03\xd0I\x8e\xf7nD*my\xb5\x0fI\x1a\xb2\x99\xd2w\x04?^\x04\xb9"\xb0\x18=\x1d\x91\x8bL\x1eF{\x92\xb8\xb4\xbcs,}\xb8!\x8b\xf9nk\xa3\x8f\xe80_\x87\x02\xbe&\xecs+\xfc}\x94\x88y\xe4\xb3)\xd2\t\xca\xcd\x88S\x90\xfb\x18\x15#\x19\xb6\xecZd\x80B\xb5\x80c\xfdt\x1b\xa0{\xe5d\xd2\xb1BK\xf22\xc5\xc7S\xce\x92D\x8aH)\xe4\xed\x12C\xb32w\'K3\x80\xefow\x07o\x977\xfbT}\x06\x87\xf9\x8a\x92\xc0\xd7E\xf12\x93\xe72\x83(pH\xfc=\xba.\xc5\xba\xad)\xbb\xf2p\xa1\x00\xd3\x9a\xaeb\xdb2u\x9f\x99\xf2\xc7\x90_\xea\xb1\x83\xf96\xaa\x8cP\x8d\xe1\xf0\x01Y \xd6\xf8\xcb\x9e\x12\xa3\x98\xa9i\xdf\xa2\xc9\x84o\xb9&n\xab\x1fuo\xfaSq\x98\xdfb\x84q\xe4)j\xbd4\x903\xa1\x0b\xdc(\xbb\xd1*Z\x1e\xd6c}\x94Csg]4\x15\xbe\x90\xf5\xa9\xe6\x12R\xeemm\xe6\xe3\xc7\xcd\x80\xe6\xe1\x06\x1f\x00\x95\xb1#\x12>`\xa1;_\xdb\xe4\xd0\x1c\xa5B\x82t\x05\x1fOz\xdf,\xf9\x1d\xfcQ\xc3\xe8_\x19\x87\xf9\xadH\xc2)\x1c\x03\xc1\xd7f\x86\xb7\xbc\x19\\\xdd,j\'p(\x0b\xd12\xe1\xce\x92\x8fkh \xb7}\xa5\x1c\x1e\x06b\xb34\xe5GD7\xaeI9PM\xb5C\xee\xf4#\xd8e@\xb1h\xb1\x0c\xc4\x84\xdegc$\xd2HSIj\xce\xe5r\xf0\xa6+\xf0\x19\x1c\xe6\x1f\xb5 \x04\xc2\xebi\xfd\xd2\x8b\x0b\x15<1\xe9\xeebq\xc2\t\xd9\x06u\xeb\xd0Qa\x18B\x84\xeb\xe9\x8f`T\xeeIvf9+\x05{\xa6\tw\xdct\xbc\x15\x0bg\x08\x11\xd4h\x8d\x95\xd37\x00\xe0\xe2} \xfb\x08/\'\xd8\x84@\xe7P@\xdf\xcc\xfb\xcf\xf80\xbf\xedn\x18\x82\x83(\x82\xbe\xfa\tE\x04b\x86\xaaR\xcb\x04\x9f\xb2\xf8\xb4\xa3H\xc7\xba\xc5Y\xf4`\x92\x99,\xc1l\xb1\xf6\x99\xdd\xda\xa9\xa8\xa2\xf9\x19p\xd0\x1eY\x14h\x80\x0f2\x85\xf7\xb8\xaf\xc6\xc8`\x03`n\xb6\x0f\x81\x1a\xf5\xe2hzb\xf9\xe6l~\x06\x88\xf9m6\x91\xe7\x8d\x07"_v\xb8\x1b\xaf&\xa28\x87F\x83\xf5\xf6=\xce\xfa\xf6\xb6\xe8\xee<\xb7\x99\xc1\xdc\x13Wl\x86s\x13\xaf\xd3\xd6\xe5G\x11\xab<\xfbP7\'\x939\xe2\x94\xdd\x180\xcdgg\x9bx\x0c\xc6U-\xbd\x9d\xbfO\x15\xe3\x1a\xb4\xf4\xbb\xb3\xf9\x11 \xe6\xb7\xd9\x840\x0c\xc7\xa8\xd7\xb6\xb1^\xe5\xa3\xf3U\x12\xc0k\x0f\xe5s\x1f\x9f\xcfL\xe4\x99\x82\xc6WTg\x0b\x96\x9a\x8d\xde\xd2\xf9\x82\x18\x84\xf5\x04\x8bvy\xacv\x9c\xac\x873\x19\xd8U\x0bs5\xa1\xd0KT\x0e\xd4\x18\x8c\x03\xe9\\R\xf6xR\xdel\x8e\xfb\x19 \xe6\xb7\xd9$!\x08\xfd\xe7=\xed_\xfa\xa9\x12\x8axX\x00N\x19\xf8#\n\x07\xc7\xa3Y0z\x9d\\m\x17\xb8\xef\x1f\xdc\xf9FF\xe4\xa3\xdf\xdfjN\xf1\xc1\xeb\xc23\xfe\x91]n\r\x18\xd2\xa1.\xdc\x9d6\xb4\xaf$\x06@;f/B`\xed\x0b\x99\x86\xa6?\x9a\xcd_\x19\x88\xf9m\xb1|U\xab\x04\xf8\xb2X\x0eL\x84\x9a\x8b\x9e\x99\x1d\xe4\xd0F\x13\xcfzr&\x81\x85\x86\xce\xb9\xcau\xecX-\xb1z\xf6\xbd\xd8Re\xe6q\x98\xb0\x86e-\x1b\n0K\x02\xcd>\x89\x1d$\x80\xf4\xba\x85\xb8{\xa4\x97\xf5z\xe1\xe2\xca\xc0{\xb3\xc7\xf0g\x80\x98\xd7K\xfd\xbf\x14p\xfc\xe1\x14.N\x95E\xb6G\xd7\xc2\x15W\xd6[\xe0\xed\x8c\x1a\x98\xe2W3\xe5\xb8H\x15Q\xbe\xf7\x80P1c\x83\xb0=\xa3-`\x84\x1a\x82;\x02|\xc9[\xa8\x03\xb5[\xc0\x88\n9\xab\xc3\x90QG\xd9\x83\xce\xcc{e\xeag\x84\x98\x7f\x94\xa9\xebY\xb7\x9e\xcb/\x1b9h\xec\xc8[\\\xc7gB\xa2\x9b\xcb\x00.\xea\x03\xb2\tw$\xf2\xf3y\x97+\x99J\xf5\xeb\xf6\xbdx\xe8\x1d\x1aB{F\xe4DH}G_bV&\xa8\xa6\xe7\x91\xb3\xd1\x0cy?2\xe2u\x99h\t\x8fn\xd3\xf9G\x04\xd6\xaf,\xc4\xfc\xe38\x84\xb1\xf5\x06\x86\xbf\xe4\x84\x9c\xabDB \xa8\xba\x84\xc2E\x00,`-kT&.\x02N\xd5%vA\xe6\xeb\x81\t\x8f\x17\xe7\xdc\xf3:\x9a\x8a-\xe8O\xfbH\x9c\x109\x1b\xf3e\x94,\xe2L\xf8@\x8aX\xfc\x91L\x8a\xa0\x81\x82\xb0:\xbf\xd9\x90\xfa3B\xcco\x950B\x92\x08\x06\xbd\xd6p"\xcb\x01\x1c\xdd0|\xb8\xdb#\x1e\xe1@\xb5\x93\x03W\xbf\t\n\xa3\xc2\x1e\x1cEw*o\xb2\x175\x9c\xc8n\xf4\x1e\xec,\x93\xea\x88\xe5\xa9ts\xcfx>\xe4\xf7\x85\x17\x0e\xce\x04\xfb%\x8f\nA$\xc1G7\x7f\xf3\x9c\xf8\x8c\x10\xf3\x8f\xe3\x10\xc2\xd6\xbd\x92z\t\xd3\xd9\xf9\x8e\x92\x02\xd7\x8a5&\xdb\x1d\xf0\xc9\xcf#\xc9\x8a\x8b\x0e\xb8Si\'\x03\xd3Y\x9c\xf6\xd3\x98_\xb8\\\x10\x15\x08.\x81\xe5hTp}\x1c\xad\x98\x02,\xb5\xb7N\xfd\xbdq\xc3\x87)\x90=\t\xd4\xa7\x04<\xbc\x99\xfa\x9f\x11b\xfeQ\x91\x93\x18\xfa\xac{_\x8a\x1b\xae\x97\xb2+!\x84"q\xc5)\xbd\x0e+\x990\x85x\xbe@\x81S\x8a#\xb8\xaf\x1e\xe4\xbe\x9e\x96\x16z\xac\xf5\x14\x0f\x93\xd45ir\xf8\xc2F\xdd]\xe3\xe9\xbc\xb2\xa1\x87~\xab\xd6\x1b\x90T\xe62\xa2\x19\x11^\xfe\xa81\xf6\xaf,\xc4\xfc\xb6X\xc0\xb5z\x82\x91\x97\xcb\xe1\xaeu\xad+\xbb\xeb\xe7\xb4\xd5\xc8d(3i\xa9R\xe3\xde\x0fC\xc0J\x15\x89\x1c!\r\x8d\'AZ \xc6\xf4I;\xb0n\'\xb4\x8e\x81I\x89i\xe0\x98\x98}f4\x97\x88o\xf4\xd3A\xbf\x8b\xec-\x82E\xf4]\x1b\xea3@\xcc?J\'bM~\xecU\x13:\xdd\xc6s\xef\xb4\xbb\xd3\x95>\x12ZL\xef\x88=\x8agw\xa6\xbdW\x8e\x0f\xa0*\xe8\\\x1f\x83\xaf^y]3\xf5\xc7\t\x97r\x8cU\xe3\xe4\x80Q\xeb=\xa0Y\x8b\xe2k\xc5\x9dv\xe8\xfet\x07,\t\xcf9\xc4\xa5\xf7oJ\x89\x9f\x01b~\x9bL\x02]\x13\x9f|U"\x06\xf4\x00k%x\xd6\xab`LZ\xcfX\x94yR:;\xa3\xf8\x18\xd2\x1c>h\xfas\xee\xb6MM\xa4\xb6\xbd\xbb\xa5\xbez\xd6F\x15To;\x9d}\xf4\xae(\x93<\x12\x07\'K\xe2:\x01\xed3\xcdk\xf5iz3\xcc\xcf\x001\xbf\xcd&\xb4\x1e\xf9\x08\xf5\xea\xe0\xa0\xbbQ\x97}\xee\xb1?\xeb9\x95\xee\xf0\xe3"\x83\xb3=\xb9m\x01gWgz\xa0\x85 \xdf\x95\x1bS\x80l \xe3\xa78\xb8\x08\x18u\xac\xbd\xf9\xc1\x03t\x93\x0b\xc4!8\xc8\x0b\x1e\xef\x92\xfa\x10\xc5\x17\xbc\x9b\xcb\xfb\x8f\xf6\xf1_\x19\x88\xf9g\x85\x88\xc0\x10\xfe\xfa^\x08\xad\x88&:\x9e\xb4\xdee\xbb\x03\x97g\x9a\x83\xf5\xf0\xec\x99\xb1!6]\xcfbBO0\xa1\xbf\\\xb3\xdb\x88X\x85\xbf4\x81[\n\xdd%\xee\x1d:\xc4\xb4\xe1p9\xdc)#\xa7Y\x9d\x8b\xfd\x1a\x8bT;\xb9\xc9?\x05\x88\xf9-L\x98B\xe0\xf5\x0e\xf6\x92\x13\'\xfd\xbaK\xe9\x8bk\xb6\'%.\xf9%E\x81\x08\x90\x05\xfa\xb0\xaf\xf0!\x82@\xed!6\xa7\x04\t\xd3\x8b\x85\x0f"8\xd9\x07\x82K,\x11t\x95\xa3L\xcd!\x1d\xd5B#\xdc0\xe9r\xc3\xe98;\xcc\xa7;f\xbe\x19\xe6g\x80\x98\x7f\xcc\xe6\xba\x83P\xc4\xab\x1e\xaa*\x878\x8dK#\xcd\xc7`\xcf\xac\xe7\xfb2i\xf6\xbd\xf2\xa2\xe9\x0c\xe9\x07;\x89\x90\x19\xe8\r\x81\x0f\x1fB\xc6\xca\xa7c\x82\xb4\x1a\x0c\x88\xd6arkr8\xe7\xa0o\xdc\x90NIvq\x91\x9b\x86\xc4\xd6\x0bP\xbdY\xc2}\x06\x88\xf9m\x87#Q\x12\'(\xe8\xe5Zc\xb0\xa7\xc3\xd1\x8e@!\x99\xba\x9d\x16\\\x10\xf2\xc6\xde\x8a\x8aK\xd0\xbc\x0c\x1a\xeevg\xafk!Z\x89\xbb\x1b\xd5\xea>V\xf0\xc7\xdd\xd1\xb94a>H\t\x7f\xc9\x1d4\x98\xebf\x99\x1b\xb3j\n\xa7T\xf89\xad\xb875\x8c\xcf\x081\xff\xd8\xc8\x9f\xc5\x10\x85\xbf\xbc~\x1f\\\xa2\xdf\x91L\xae\xe9}\x1b\t\x98\xa3u7\x93A\xf6\xad\x16\xe8\xe6y\xd9sZ\x1c\xf5\xa4c\xb0I\xcf\x8c\xa8M@\xb6m\xb5%\x91Zx\x140GX\xea"\xba\xbcv\x85rQ\xb5\x07q\xe7\x82[I\xfa?\x84\x05\xfeT!\xe6\x9f\xb9\xb9\xd6\xb8\xeb\x18\xfd>\xccy\x08Z\xb8\xc5\'\xbc\xdd\xab\xa3\xb3\xbf\xb4\xf89\x9e\xa4b\xf1\xee\x15\x1b\xe00.\x08\x16/i$\xb2G\xd5\xfd\x90\x18cTAi\xcb\xa1\x03\t\x0f}\x8b\x96x\n\xabf"\xb9T\x10P\x15\xe4\xe1u\x9cE?\x14\xb7\x7fe!\xe6\x1fo\xf9\x10\x9cXoH/;\\Q!k\xe9t\x8a\xd5"\xa6\xc2\xb4B\xac\xfe\x80\xc7Z\xcbL\xb5qMk\x80\x92\xc0\xa9i\x91\x1as\xd9\xbd~\x1ekb\x94\xb4K~\t\xa4^oN\x91\x18{e\xcf\x85\xd7\xe4\xa2Z8\xcd\x82\x94\xd1%!\xf6\xae\x08\xf9\x19!\xe6\x1f\x8b\x05\x82\x10\x98x\xad\xf71\xe8~j\'\xcf\xda\xd5G\x1f\x9c\x19\xfe\xd4\x19@Z\x1d\\\x10\xbe\xc1\x17\xdd\xde_\x11\x8aF\xe0\xc3zd\xf9,\xc2\xe7\x95k\xba\xf91q\x06\xc8\xbfjA\x91\xd7\x13Y\xb0\xb7\x04\x9d\xce\x9c\xb2@\xc4\xd9\xebR%x\xf3Z\xf3\x19!\xe6\xb7K\xeaZ\x02\xe2\x08\xfa\xfa\xf9"\x1a\x9f\x86 \xb9\x85Qt\xae\x93T\x97.\xf3#\x1e\xaa*\x9d\x17\xc0\'\x82\xf8\xc4\xf4\xf1\x18\xc1\x15\x01\xf5\xa1\x08\xcf7,\xacR^\xd0G/\x88c\x8f\xa7\x82\x1bg\xb6\n\xeb\xd7\x00$\xf2\x83j\x8f\xc6U\x92\x0fon\xe4\x9f\x11b\xfeq^\xe1\xd8\xf3\x16\xf8\x92\xfa\xf7\xbb\xe5fa(\x9f\xc4&\xecw\x97!*\x97h\xa7\x8b\x862usq\xa6\xaf!{\xa9\xc2;\x8c\x84\xac\xba?\xe5\xd4\xf5JE\x8f#\xd6#\xa42\xa9\xceq\xd6\x8fQ\x90\xdb\x97TQ\x93\x1b$\xda\x87{\xed\xcf\xf4\x9bw\xf1\xcf\x081\xffx\xdb\x08\x92O\xbc\xfc\xf7Q\x929\xa1\xad\x17#:QHq\x97\x0b\xeb\xf1?\x1a\xb4n\xdf\xbd\xc4\xa4\x8e\xa1Qi{\xb4Y\x1e\xca\xd0\x91h\xa2\xd3\xf8\xe2K\r(\xbb\xfe\x1d\x05C\xb7\xbe\x8b\x94\xe4\x8c\xd5\xc5\xba\\\xcat\x9e\xf6f\xa9#\xf1\xc5\xfe^\x8a\xe9g\x80\x98\x7f~\x16\r\xa3(\x06\xbd\xa4\x04}\xe5\x8f\x1d8\x80\x9a\xe7X\xf8\xd1\xa6D\xb4\xde\x83\xa1\xda&\xed\xc39v>uB\x93\x07\x93=$V\x8f/\xad\x88\x87n%\xd6\xe0.\x84ds"\x8d\x82\xe3\x19\xe1\x0e\x1e\xc2\x13Q\xde\xbd\xf0\\\x11\xdd\x9a\xb1)G\xff7@\xcco?S\xb5\t1\xff\xf5\x8f\xaa\xff\xbf#\xc4\xfcB\xcd\xcd\xb7~\xf1\x9b\x10\xb3\t1\x9b\x10\xb3\t1\x9b\x10\xb3\t1\xdf\xf89\xb7\x13\xff\xfb\x9e\xf8\x9b\x10\xb3\t1\x9b\x10\xb3\t1\x9b\x10\xb3\t1\x9b\x10\xf3\x8d\x9fs\x13b6!f\x13b6!\xe6;\xcb+\x9b\x10\xb3\t1\x9b\x10\xb3\t1\xdf\xf697!f\x13b6!f\x13b\xbe\xb3\xbc\xb2\t1\x9b\x10\xb3\t1\x9b\x10\xf3m\x9fs\x13b6!f\x13b6!\xe6;\xcb+\x9b\x10\xb3\t1\x9b\x10\xb3\t1\xdf\xf697!f\x13b6!f\x13b\xbe\xb3\xbc\xb2\t1\x9b\x10\xb3\t1\x9b\x10\xf3m\x9f\xf3\xff`!\x1c\xed\x7f\xd9]\xfe\x93\x85`\xfc\xee7\x01\xbf\xaf\x10\xf3W\x04\xf6S\x84\x98\xbf"\xb0\x9f"\xc4\xfc\x15\x81\xfd\x14!\xe6\xcd\xc0\xfeO\xde\xc8O\x11b\xfe\x8a\x19\xfb)B\xcc_\xb2ylB\xcc\x9f"\xc4@\xff3\xe6\xff^\x88Ap\\@\t\x92%\x04\x98\xa3\x08\x8a"8\x14A\x9fmG0\x82\xc3!R`)\x98\x10\x04\x02\xc20\x1e#X\x94\x07\x05B\xa0@\x9c\xe2y\x0c\xa1Y\xfa\xf9}~\xac\x1f\x80\xeb\xff\xce\xaf\xd1\xc1,\x84c\x04Ec$\xf8\xff\xb3w\xa7K\x8f\x9a\xd9\xa2\xa0\xef\xc5\x7f\xe9(\xe6\xe9\'\xa3\xc4,\xe6\xa1\xa3\xa3\x03\x01\x021\x08\x04\x12 \xfa\xe6\x1b\xa5\xf7>\xa7\xcar\xbaJ\x15\xca\x9d_\xd6!\xa2\x1c\x15\x99\xb6\xf5\xb1\xdeq-R^\x0f\x84b\xeb\xa8 \x0c\x8a\xd3\x1c\x89\xd14\xc5\xd0<.@4\xca2$\xcaB\x84 \xb0\xf4\xfa\xe1,\xc5\xb0\x0c\xf6#\x85\x98\xff\xee\xd8\xf1\xdb\x9f5` \xff\x06c(\x89\x10\x04I\xff\x95\x10\xf3\xf5y\x9d?\x11bh\x96\xe6\x19\x06\xe5i\x01^\xa7\x90\xa6\t\x81gi\x86Y\x83E0\x94\x12\xd6\x87#\x04\x12#i\x91$i\x8c\'\x10\x08ca\x11\xa10Rd\x05\x81C\x9f\x9dx7!\xe6\x07\n14I\xd1\x1c"\x8a\x08\x8b\xae?\x10\xe68\x9a\\\x9f\x8a\x87\x10\x9aea\x14\x16\xd6_\xb1\xebhS\xb0H\xa2\xa4\xc8\xa3\xe4\xbaN\x88\xf5\xef\xa24\xc6p\xec\xb7v\xa9\xbf\xb4\x10\xf3/\xc3%\x9f\x12b\xbe56\xf9K"\xe6\xeb\xef\xf3\x9fH\xc4\xac\xc7%\xfc\xfd\xde\xdb\x18\xc5\x1c\xc6Z\xa1\x89=>\xd3\x83\xee\x97\xc3P\\\x03\xe9Qk\xb2\xb6\x1f\x94iX\xb0\xce\x9f\xae\xe9\xd1\xb5@!IP\xdb4#\xcd:\xb1\xe7\xdd\x95n\xf6\xe7\xb3{;\xf6\xc3\x8d\xa94Fc\xee\x12t\xb2\'Oy\xb3!\xd7\xc7\x88\x18\xf2o\x08\x8c\xd0\x08\xbc^h\x7f\x0c\xb3\xa3\x01+\xbc\xa3=V\x02 r\xa0\x06\xa4a\x9b\xe5A\x12\xfc\xee6\xe2\x88\x04\x1d\xac\xccs\xf9\xcaW\xa3\xbb`D\xd7\xab\n+\x16\x03\x1dZ|9\x86\xed\xf9\x81\xeco\x87b\x9a\'2\x17\x91\xb6\xefc\xeaR)\xfa\xcf2b\xd60\xf1?i\x16\xdf\xe6\xd5\x01,lp\x07\xaa`UL\x13\xe5\x1eX\x11\xf4\x08[\x89\x18o\xba\x8b\x06\xc5\\\xdb(Dg\x1e$\xf6\x12\x98K\xee\x84b\xcc\x91\x85]\xf0R\x1c\x8e\xfb@\n\x806\x11\xd8\xd6\xc3p\xcaJ\x17\x88\x8b\xea\x9fE\xc4\x90\x7fC!\x02\xc1i\x0c\x7f\x99\xc9\xb9\xbc\xe7\x85B\xd4\x95\xcaP12\xa3\xf9\xfa)w\xd0\xbaH\'E\x80\x92\xd6\x97\x8e\xfeyVy\xc8\xcb\xfa]\x16QCL-\x85u\x11\xea\x11\x1d\x84\xf8rt\x04\xbe\x02\x9b\xe2\xac*\xd6\xa1(t\xc8Z\x06\xfc\xfc\xbd\xd6j\xbf8\x11\xb3fI\xeb^\xa21\x84x\xd9\xf6\xa0\xb0\\sj\xdc_e\xb6\xc7\xbd\xa0\x93}c8T\xf9>T\xb3\xaa\xca\x97\x91\xe1\xca\x96\x97\xdc\xde\xb5\x02J\x10\x16\xe7\xc4-\xb8&\x1e\x92\x8a\xb9\x0f\xc5\xbd\xbe{\x9eqx\x90\xb4ro\xf8\x11\xa6\xf6\xd5\xb14\xa5w\xfb\xf0}\x8a\x88Y\xc3$ \x0c\xa2\xa9\xd7f\xf1\xfc\x92;\xbd\xd6Cz\xc1\xe2\xd34\xf0\xcap8\tq!^\x89\xa6\xc5X \x1cG\x14SS\xff\xb4\xbb\xcf:\xbd\xafG;3\\5\xf1\xb3\x01\x94EB[\x8f\x00tB\xd5\x94%\xce\x85\x12\x1e\xc3\xe2\xc6\xdd\x1e\xfe\x9b\x9d\xdb>F\xc4\xfc!\xe7\xfd\x07\xe7\xeb\x88\x967\x85(\xb5p\xa0\xf2\x0c\x18y\xe2\xc4\x8f>\x97\x80\xbb\x1a\xc6\xcb\xf8\xbc\xdfY\x1a\x07\x05R\xe2\xa6\x8a\xdd\x97*.\xa4%v\xf29(\x0cv7\xfa\x88\xdc\x96\xf4\x18MP\x08-\xa0$\x02c\x05\x9c\xbcw\x05\xac\x8f\x111k\x984L\xc0\xeb-\xfb\xd2K\xb9$\x07\xd2JT,\x14\x12\xa9\x14%/\x1aR\xd6r\xe8\xc8c\n\xf5\x82\xfa\xfe\xe1\xd1$\xa0#;\xa1\xa2\xfa\x1d\x8b\n\x13\xc0.W\xc6\xa6\x1eH\xbd\xbfW\xb3\t\xdd\xc3\xd6X\x98}\xec3\x03\xa4\xf1\x10\xa5@\t\xf7\x93\x88\x985\xcc\xf5\xee&p\xe2\xb5\x85\x1c\xfa\x90\xee\\\xa0\x14\xeb\x11V\xcc\x98\xc4f\xcb\xd8H\xac\xa8\xbb\xd3\x15\x13(\xcb:\x87z\x98q!\x90U\xf2#\xa5.\xf8u(\x8fE\xed-\xfd\rA\x1b\xa4|\xec<\xe1\xe6\xdf\xc4\x96\x89j\x85\xd3\xaen\xdaI\xd1\x9b\xea\xc7\xc7\x88\x98\xe7\xdeDH\x04Gi\xe2\x8fa\xde\xc8.\xb9^0\xed\xeaH\xd9\xcd,\x12\x9cU<\xd8\xc6\x9bk\x18\xc7\xf6]i\x976H\xa0\x8b\x19#\xea\xac\xb6\x87\xdb\xa4\x9f\x85\xf6\xe4N\x0f\xc9\x90\x8b\x83Z\xba\x19\xce^\x84\xd0\xbc \xf8Qoe\xcd9\'^\xf1\xb5\x1a\xaa~\x8c\x88Yo|\xf4\xc9C\x91\xaf\xdd\xe2\xfbC\x8av\xcb9u\xdd~M\x9aL\xa5ztwD`=\xe2Q1\xe6Mk\xcc|\xcd{\x1dfL\xb0\xa8J\xacV\xc2\xdb\x91W\xf2\xcb0\xef\xb3\x81:%\xb9_\xc8\x0c\x98t\xa9!\x12\xa7A\x03]7fg\xf6\'\x111\xe4\xdf z-lI\n~i\x00^B{\xf40cT\xc2$\x8d\x89#\x0f3\xe8\xeeT\x98\x91V\x98-"\xff\x93\x88\x98u6\xd7\x0c\x95 \xd7\xd2\xf3\x8fa\x86(\x02\xd5!\t*w81\x8c\x86\xa5\x83\x9d\x95\xedh\x19\x1b\r-\x06\xe2\x1cJ\x9a\xd0\xad\x1d\x0fO\xdbq\xbd\x81KL\x8f\x97\xda2\xad\x99\x99\xe4\xb0\xa6\xf39\xbc\x8c0`e\xd7\xc2\x9c.\xfbe)\x1c\xa1\xffb\xc9\xcd\xa7\x88\x98u\x14\xd7l\x9fZ\xcb\x9a\x97K?\xe0M\x7f\xb6\xa6>8\xfb\xb3p\x03\xa1{\x80\xa5\xf8\xa5\xab\x1d|\x0f\xcbg\xd24\x179\xca\x05\x19-\r]\xbb\x14u\xa2\xcd-\xba\xe8\xae\xe6g\xbe\xa8\xe0m*\x1c0$kJ\xd5v\x17l\xcd\x08\xa6z\x8e\xdf\x94\x05>&\xc4\xacQb\x04M\x90\xc8K\'e\xe5&\xe8\xf6\xa5\xb9\xf1\xfb\xc9\xf4\x92\x12\x12}\x1f\x1a1\xfc\x18e\x1aS/\xded\x90%b\xdf/\xf7\xcce\x11\x0b\x00\xb0\x08\xb91\xc9=\xad\xa5\x1c8GV\x1eUcC.\x03\xca\x1a\xf4\xfd\xaa;\x87=\xd7R\xf0\x9b\xe9\xfe\xc7\x80\x98g\xa2\nQ\x14\x8dB/\x97a\xed\\dt<\x9c\xc1\x91\x05/\xf0\x02\x8cL\xd5\x8e\x875\x9fa\x85\xf3z\xaf\x90\xfe\x98\x99\x0c\x1d\xec\xdd8\x02\x856\xcf\xd74u"(\x92\xbb\xb4k^\xda\xaa\x08^\x9c\xef$-\xd5SV6HL\xe8\tT\xc6o\xde\xf9\x1f\x03b\x9e\xa9\rD\x13\x14\x8c\xbc\\\x86\xfd\x8d\xd6+.\x1ec\xfd\xbe\xc0\xc7n\x86\x92\x13\xbc\xe3z\xcf\x02[\xe9q\x14\x95\x13\x18u\xc2\x8d\xbfd\xad*\xce\xb5\x98\xaaK\x14\xdc\x81\xe5\xce\xd4\xa1\xc5\x0bZ\xfa8]\xce\xf1\x03\xde\xbbP\x07/\x87\x0c\x07\xd4\xe8\xfa\x1f\n\xc4<\x8fO\x94"\x9f6\xc1\xcbb9I\x04\x9f\x94\x8fT\xdd\xc3\\\xdaW\x8f\xd2\x9d\x93{\xec\x89\x96\x80\xdfa\xcbT\x02\xcc\x98M\xc07M\x90\x9d\x93\x9a;\xc4\xfdP\x9cx\x0e\xf1\x1c\xc4\xdb\x13\x98\xa3D\t\x88)\x8a[\xed\x1a\x04\xe9K\xe5>V\xc4\xc0\r\xd0\xefh\xd2?\x15b\xbe-\xf2M\x88\xf9\x97\xbf\xaa\xfe\x9f#\xc4\xfcB\x9d\xb8\xb7\xe6\xe6\x9b\x10\xb3\t1\x9b\x10\xb3\t1\x9b\x10\xb3\t1_\xf897!f\x13b\xfe\x0fJ\xa26!f\x13b6!f\x13b6!f\x13b6!\xe6\x0b?\xe7&\xc4lB\xcc&\xc4lB\xccW\x96W6!f\x13b6!f\x13b\xbe\xecsnB\xcc&\xc4lB\xcc&\xc4|eye\x13b6!f\x13b6!\xe6\xcb>\xe7&\xc4lB\xcc&\xc4lB\xccW\x96W6!f\x13b6!f\x13b\xbe\xecsnB\xcc&\xc4lB\xcc&\xc4|eye\x13b6!f\x13b~\xaa\x10\xf3\x0f\xc9\xef?\xb1\x10\xfe \'|m!\xe6\xc7\x07\xf6\x93\x84\x98\x1f\x1f\xd8O\x12b~|`?I\x88y\'\xb0\x7f\xcf\x1b\xf9IB\xcc\x8f\x9f\xb1\x9f$\xc4\xfc\x0f\x1c\x1e\x9b\x10\xf3C\x84\x18\xe4\x7f\x8f\xf9_\x0b1$D\x10<\xc2@,M\xc3\x1c\xc9r\x10\r\xd3\x02I\xa0\xeb3a4FC(J\x89\x82\x88\xaf\x0f\x03!{\xde\xfc\x05~ B4\xcd\xe2"!`\x08\x872\x18*\xd08\x89\xc30&@$\x85\x90\x02\xb4~&\xcd\xb0\x08B\xd1\x10#\xae?\x05\x82\x11J\xc4x\x9e\x7f\xfe\x18\x82\xff\x91B\xcc\x7f\xf7\x0b\xfb\xedO\x1a0\xc0\xe8\xdf\x08\x18\'Q\x84\x84\xa9\xbf\x12b\xbe>\xaf\xf3\'B\x0c\xf6l\xa6/\xe2"\xcd\x108A3"\xbf\x86\x89\xf1k\x14(\xbd\xfe\xea\xd9\xd6\x08CpD y\x9ca\x04\xe1\xd9dx]\x0e\x02\x87\xa08\x8b\xb3\x08\xfa\xecY\xb5\t1?P\x88YgB\\75/\xae\x1bC\xa4\x18\x11f0\x02\xa61\x8c\x14\xd6u\xc400\x05\xad\x03\x87\xf1\xebx\xaf\x13I0,\x81\xb1\xf0\x93\x13a\x11\x1e^\xb72\xf9\xdb/.\xc4\xfc\xcb-\xef?&\xc4<\xf7\xe1_\n1_\x7f\x9f\xff\\!\x86\xfa>.\x10\xc6w\x9a\x18\xb9\x13y*\x95ZH\x90\xc7\x11fb"<\xec\xf8{\xba;\x1d\xd5\xda\x89d\x9f\x84[g\xf2\x84\x0bn\xcd\xd8\x83$\x1b\xe1<\x80\xbbG\xbb\xe0\x19f\x89\xb9}!\xb3\xd9P\xf5\xcbT:\xd5\xe5\x8c\x9c\xd9\xf7ZU}J\x88y\xde\n4I\x90$L\xbc\xb4ok\xec\xc4\x85\xf5\xa6_\x9f\xe8qv\xd4k\xd6\xb6\xb2\xdau\xca\x81\xca\xa8\x1b\x80\xe3Yng\x08$\xe9\xde\xb5\xd5\x06\xc0\xbb/\xbb}8\x1ec\xa2R\xcb\x0c\x9b\x11vo9s[\xc9\x16\xd1\n\x84\xff\xd8\xf5=\xee\xd9o6\x90\xfc\x94\x10\xb3\x86I!Os\xea\x8f1\x1e\xcf}&\x1d\xb9\x87G\xd6\xe3\xf5\x18\xe5}\xdb6\xd2\xe5\x00\xf6|\x1aY\xf4\xbe\x8c\xc6\xb4\x18Be\xdfX\xed\xa1\x1f\r\xed\xc4\xd29-\x8f\xa1\x812#\x95A\xf7n\xcf5(\x03h\xeb\xb3\xc3e~\x99\xa5\x14\xbc\xbe\xd9\x89\xefSD\xcc\x1a#\xf9l1M\xc1\xc8K\xc3A\'\xb7{\xe0\xd8=\x96\t\xcf$\x16\x8e\xac\xa1\xa7\xf7\x87i}\x04\xcb\x90\xe3\x12H\xbc\x93\x91\xee\x1c:LDx7W\x03q\xcb(\x86+8\xe5,\\\x0e\xed\x8e\xd8\xf17<\x04v\x9a_\x8e\xbdx"\xf6\x16>E\xdfk\x00\xfek\x131\xcfQ\\O\xff5}@_\xba}\xc1\x12r\x13\xc3k;\xdd;\x93\x1e\x99\x1dR\x0cc\x99>L\xb5$*\xe3NN\x8f2\x87B\xe9\x81\xa2\xad\xc1/\xd7\x11&zo\xb1Jt)\x9c8\xcc\xa4\xaboKu\xa6\xb6\na=\xb4\x1d\x0e1\xc1\xe4\xd0\xe6\x9b]\x86?E\xc4\xfc1\x1b\xfc\x87n\xf1M\xe2\x13,\xe5#ID\x18@\x1f\xb8\x93\xd7/`\xc7\xeaw\xb1\xdb\x8bT~e3\x01\xe3\x1e\x8d<\xf0\n]w\xa8\x8e\xeeoE\xcf\x94=\xa92e\xd1\x1d\xfc\x83\x86\xde\xfd\xe3~\x12\x97j\xd2\x87H7C\xe5MY\xe0SD\xcc\xb7=\x81?)\x9cWY\xe0x,l9\xbbu\x11\xec\x04f!kE\x12\x0bYhM\xf2\xc1\xaa&U\x8a}\xd1\x9fJ\xbb\x1dy FG\xf6\x04\x8a\x85ZJzx\xe9\nt>\x80\xa7\xbb\x93\xda#\xe5%\x84I\x1fs\xc5\xa2=<:\xe7o\x86\xf9)"\xe6\x19\xe6zR\xac\xf5\xcck\x98\xd8\r\xc6e\x010\xc9]\xd4f\x8b\xe1U\n\x16 (\x14\x04T\xc5\x91v\xbeWG\x88";\x00\x1a[N\xe3\x9c`-mT\x8a\xf5!\xc3\xb8[}<\xa9 \xd2i1,\xf7\xcc\xa1G\x04\n\x89M\xbe\xe8\xacw{\xff\x7f\x88\x88\xf9\x16\xe6\xb3_0\x85\xbe\xdc\xc9\x05\xe7\xd4\xb0\x817=\x85\x9f\x8b\x1bj3\xb6\x0ctW\x80 |\xadkZ]H\x1c!"\xc9\x03\xd6\x9c\xcc\xd2\xca\\-\xdbK\xee\xbe\x9etG\x90H\xb0\x1a\x0f\x89W\xed.5+\xcc\x07\x97jS\xd1\x9d\xee\xcc\xe9\xcd\x06\xb9\x9f"b\xbe\xedM\x14\xc7\x08\xe2\xb5\x03\xb8\x97\x15\xe3d\x1aW\xc3\xee*\xdf\xb8\x8e\x94\xdf#\xa3\xb6\x0f\x92\xfd:\xae\xfd\xd1!\xe4\x13\x04\x02D\xd0\xd6,xp\xd5\xd0\x9a\x1f\x93\xa1{\xdac\xe0\x8e\xa9\x86\x1f\x07\xc8\xbed\xb8]\x0cRr\xdes\x96g\xc1\xb5\xfe\xbdn\xc7\xbf6\x11\xf3\xbc\xf2!\x12}\x12%/\'\\%W\xbb\xf5\xa2\x14\xc9\xea\xccrM\xaa5\x81\x8d\\\x0e\xbd\x0f^\xfa\x0b\x1d\x8aW\x0f?\xa6`6Z\x8f\xd9M\xed&\xb3\x07<\xc6`\xfc\xe8\xf8]\xa3t\xa7\xc9j\x05[\xa9L\'i\x92\x1a\xf7\x96\xa3\xe68g\xfc\xcd\xee\x94\x9f"b\x9e\x8b\x85 0h\xcdS_n\xfd\xb4\x1e\x8b\xfd\xe9\xd1\x82\xf9\xe03\xba\x9e6\x95\xa3#\xb3.UC\x99z\xdd\xb1\x95\xaf,\xba\x97\x12\x07T\x97\x80\xbfi\xed\x08E\xb5:\xab\xa6\xafT\x11\x1c\x1d\t\xdf\xb3\xb3d\xa7\xbb\xe8\xe9\xb4\x88\xd7\xea\xdaX\xb8\xff=B\xe1\x07\x131\xcf\xad\x8fB8\xb4\xd6e\xaf\x07\xb9L)\x06R\x05`\x8d]\xfa\xf9\x06\xa1~\xf8\xf0.\xa5\xac\xe3\xba\xd0\x8a\xe9\xd0\xb3\x17\xe48\x0e\xd6\xc0>\xce\xbcU\xb48+\x89\x8bs/\x11\x00\x12\x0b\x0fBX\xc8 \x02\x8e\x0cT\xbdk\xea\x86\x9b&\xd6\x05\xbe\x97\xc3\xfd\xdaD\xccs\x14\xe9\',\x80B/\'K\xc1\x83\xea\xb4\xb0\xc0\xa3\xf6\xc7+yp\xa3}C\xde\x83k\xd8U\xc6\x19\x90M\x8a\xe7\x0f\r\xb8&\xfd\x9a0;\x02\xd3e\x8a3k\xbb)\x0bk4\x0e#p\xc7\xf4\xa1\x9eZ\x81\x84\x9c\x89\xf4\xca]\x98\xc9\x9c;\xec\xcd\xc6\xb4\x9f2b\x9eaR\xeb\x05\xba.\xba\x97\x84_\n` Z\x0b%8\x8a\x03J\x9f#\x05\xf7\x83l<\x92\x97#p\xb6\xd9\x8a\xa8\x07\xc4\xe9\xa487\xbaGW\x8d\xca\x94\x1f\xf3"\x9cB/\xbb\x91\xec\xe1<<4\x10/\x03E\x8b\x07\x8b\xd3Q%B`\xb5D\xdf\xac\xdd>\x85\xc4|\xdb\x13\xf4Z\xb9\xa1\xc4K\xaa:"L\x9d\x1c\xc6\xcb\xb5\x98\xfd\xeb\x01\xd6\x11|\x81\x96\x04\x10\x0f\xb2\xc9+-\x97\xc35\x8b\x9e\xe9\x85\xe8m\x04\xcb\n\xff\nY\x0f\xbd-\x8be\xe7\xf0M\x14\xc1\x8c\xd0\xd5\xaa\x04\x91\n\x152g\x86M\n\xb9\x1avo\xa6\xaa\x9fBb\xbe\xdd\xfa\xc4\x9a\xa7\x92\xe4\xcb\xa2%\x10%\x8b\n;>z\x10\xb4\xbb\xeei\x99\xce\xa5\xc4\xcf\xe1\x02\x85\xc9}T\x82\x14\xe65\xfa\x85\xceY~o\\\xab5\xa5\x94\x1c1\x04\xeaQ\xda\xc9\x0c\x97\xa9\x80\x0f\x1fd\xf9\xaaT0D,\x87\xbcD\xc5\xab\xac}\x0f\x86\xfb\xb5\x91\x98\xe7(\x12\xd0\x13PyI\x9df\x15\x9c\xfb\x1eF\xa1\x04\x1a\xfc\x05\x99$\x81#\xb24i\x1fN\xb7\x1cs$\xaf\xd8S\xb6#N\xea\x12\xa3\x97V\x0e\x1d\x8e\xd6\x0c\xa4.Y\xe9\xc4\xea<&;a\xa0\xc9\xf6\xcdY\x88\xe1\xd8\xc78w?\xba\xe5\xf8\xe6m\xf8)#\xe6y\x1b>_@R\xf0k\xd3\xfd\x12\xe9\\$\x16\xbd\xba\xc4\x15\x1f\x13\xed\xd0=\xa1=x\xd4l\xf62\xe0d\x97\xdb\x8f9b\xacKI,\x85\'\x06\xad\x8c\x1ep$?Ty\x13\xcew\xd7\x96\x8at\xd0\xf4\xc9\xcd\xc3\x89)\xbc\xddp\x19\x91,$\xde\x84">e\xc4|{\x9d\xb1\xa6C\x10\xfc\xba%,M\xd6\xc9\\\x07\xbb|\x86\x89\xfc&\xc2\x00`\x88\x18\xa4\x88\xda5\xd0\x91\xa0\x0e\xdc\x87\xaf\x89=\x07+\xc2\xa0\rEVEn\x8av\xc6\x9e\xf7\x81\x83\x8f_\x8b\x0b\xa9\xebNtj\xc7\xc7\xf5F\x9eb\x96\x90\xdc\xf0\xcd>\xea\x9f2b\xbe\x1dp0\xbcf\xd1\xe4K\n\x07*N\x1b\xec\x17\xc7\xa2\x91\x9dmv\x11U\xbb\xfb\x8b^6\x97\xab%\xb3\x83\xa2\xbaE\xbc?\xf1.~\xba\x02V\x9d\x14cXa\x0b$F\x8c\x1a\xed\xc1\xc8\xec\x06)\xbf\x90E\xa8\xd9\x9c]\xfaF\x85k\xa1-<\xbeV"\xfc)#\xe6[9A\xa0\xf8Z\xea\xbft\xa3\xd7\x16r\x9a\xaf\xf2P\xaai\x99\xdev\xd3\xe3\xc0@*\xa0\x80\xfaEf\x92)2\x05~@\x89\xbe<\x8b\x84\xb1\xbb\xb4\xb8\xd6C\x90\xbf\xc0\x15\x00>\x04*\xd3\xa0\x000d\xcc\xe5\xba#\xe3\xef\x10h\x81@)^G\xf1\xbd\xc5\xf2)#\xe6\xdb5\x81\xc0\x10E\xd0/[\x1f\xd7\x9d\xcc\xad\xf9v\xa1\xa9\xd4\to\x80\xa1\xb5F\xe21\xcc\x92\x06\xb4\x17\xe8\xa1\xe3Yu\xc5\x81X]\x0f\x1d4\xf2`\x94\xbd\x81))\x85-\x7fiQ\xab\xc6\x0b\x88\xba\n\xf7n\x1dA\x13\xf5\xe6\x07:\xd8\x84\xbdCa\xff\xcd\x14\xeeSH\xcc\xb7\x177kvC \xaf\xb9Mrf\xaf\xf7N\x98\x12\x9f\xcfm\xa7L\xfb3\x0eWG\x08\x86\xcc l\xc6\x02\x1b\xd5\x0b\xa3Lb\xa7E\xa0\x96\xc9C.\x85\xee\x12\xc2\xe9\x8c\tZ\xb7?C\xd7\x8ai\n\x86ip"H!\xa5\xee\xe1\xdbl\xb8o.\xdaO!1\xdf\x0er\x92@\x10\xe8e2\x858;\x15\x18\xa5\x01\xa1\x0e\xa9\x19\xd6g\x0e\x16\x8b\xcc\xb9\xba\x1fH\xa2\xba\xe1]|\xcdj\xab`=N\xe3\xf3\x9eO\x14d\xbfw%K\x0c\xd4\xab6\xba\xc5hc\x10J\xe7w\x00T\xa4\x0eY\'\x97\xb5/\xd2\xf4\x93\x8c\x98o[\xf3y\x00\xbd\x92\xb7\\\x03\x9d\xed4\x98\xae\xde\x9ar\x14\xc1\x95f\xdaS\x10\xf4\x07J\xad\xbb\xe8Q\xe3\x98\x04\x9ew\xb2sP\x86X;\xd3wy\xdf\x19\x06\x9f\x82\x94Z!si^d$\xc1\'\xf5\xfe \xa6no\x81T\xd5\x95\xec\xa9\xf9Z\t\xdc\xa7\x88\x98o\x1b\x1f\xc5\t\x18}\xb5\x12\xb5\xbb\xc1^`\x8d\x9dE\xb0T\x12HV@\xcf8\xf3I\x025\xb2\xe0\xd6\x80\x91M$\x9f\x07\xd6\xb4\x13\xd5\xab\xfb\xa8\xec1\xaa$\xfd\x8a\x14\xc0\xa9>_\xe8\x08\xcc:\xd5\xe6Rk7W1\xaaH\xbb\x8a\xc4Q\xef]\xbb\xfcCD\xcc3L\x8aBQ\x1c\xc5^\x8e\xf1\x11Lo\xe8\xc0\x00\x86\x81\xc2\xb7\x16\x99\x1f>s\x81\xa0\xf2v\xbe\xd95\xb9\xf7\xa5\xe3\x81\x99\xe7\xc2\xccQ\x00m+\x14\xe0\x1ct\xbe\xb4\x86\xd8\x9e\xe9\x93\xdd\xf2\x10\x9f\xb7\x0f\xbfF\x85$\x8bts\x0f\x93\xd3U\xc6\x9b7_e~\x8a\x88yn|\x1a\xc7\xd6{\x19y\xe10\xc0\xd6\x08X\xec\xa0d\xb0s\x82\x89\x84_H8\xa5FL\xc3s\xc1\x1f\xc8\xc86(\xa6\x8c\xc1\x93\x0f\x12{*n8g\x8cs\xa6B\xf6`~\x85\xf7\x98\xb90vTb\x19\xcb\x03d\xcd^0\x95L2\xd5\x17\xde,Q?E\xc4\xfc\x9e\xc1\xe1\xd0\xab\x9c\xe2\xde\x93\xb9\x14J\xb1\xac\xc4\\,3:W\x86\xc3\x91\xf3\xf2\xa4U\xaeZl\x1e\x06<\xb4 \xb9]K\xf1\xc0\xb8^\x87i\x9f&\xc0\xb8\x1c\xd2\x84\x00\xab\x846R\xb6\xc5\xa4 &\x94\xf0\xa6\x16\xba.\x93\xb7\xc8{w\xc9~\n\x88\xf9\xfd\xb54\xfe\x14\xda_\xd2\xd4\xdbe\x7f8\xe5B\xc9\xe8\xca|\xb1\x00\xfb\xb6\xa7\xb0\x11\xaf\x19\xfd\x008\x87G\xeaC\x84\xcd\x86\xac\xe8 (\xca\x88\xbd(\xabQZ\x00\xfc\x9a\xa1pB\x80>@\xe2\x0c\x80\xb5\xb9\x03\x00!\x90J\xf1\xd2d\xb6;\x7fWi\xfe\xb5\x81\x98o\xef\xa6\x9e\x07\xe2Z\xd5\xbe\xbc\xaf\xedX\xeb\xf4\xa0G\xfb>i\x8f\x93)\x9d\x80\x8br\x11\x10u\xdc5\xa2\x16\xe2E1:uY\xb7|\x87gL\xad[.k1\x134\x9d\xf3\x03\xb8g\xb3\x02@\xbcB&x\x17\r\xd7\xab\xc6\x04cX\x9c\x04F\xf8}\xb1\xfcS \xe6[z\xbd\x011\xff\xf27\xd5\xffs\x80\x98_\x88\xde\xd84\x93mH\xbf\xfe\x90n@\xcc\x06\xc4l@\xcc\x06\xc4l@\xcc\x06\xc4l@\xcc\x17~\xce\r\x88\xd9\x80\x98\r\x88\xd9\x80\x98\xaf\x0c\xafl@\xcc\x06\xc4l@\xcc\x06\xc4|\xd9\xe7\xdc\x80\x98\r\x88\xd9\x80\x98\r\x88\xf9\xca\xf0\xca\x06\xc4l@\xcc\x06\xc4l@\xcc\x97}\xce\r\x88\xd9\x80\x98\r\x88\xd9\x80\x98\xaf\x0c\xafl@\xcc\x06\xc4l@\xcc\x06\xc4|\xd9\xe7\xdc\x80\x98\r\x88\xd9\x80\x98\r\x88\xf9\xca\xf0\xca\x06\xc4l@\xcc\x06\xc4l@\xcc\x97}\xce\r\x88\xd9\x80\x98\r\x88\xd9\x80\x98\xaf\x0c\xafl@\xcc\x06\xc4\xbcC!\xfc\xc3V\xf8\xa7\x14\xc2\x1fR\xe5\xaf\x0c\xc4\xfc\xf8\xc0~\x12\x10\xf3\xe3\x03\xfbI@\xcc\x8f\x0f\xec\'\x011\xef\x05\xf6\xefp#?\t\x88\xf9\xf13\xf6\x93\x80\x98\xff\x81\xc3c\x03b~\x08\x10\x83\xfe\xef1\xffk \x06\x15H\x1cbY\x94\xa1\x19Z\xe0\xa9g[2\x8c\xc5I\x91!\t\xe1\xd9A\x1c!a\x92\xc1x\x8e`\x08\x08eP\x08[\x0f=\x11gI\x98\xc5`\x8a\xfd\xd6g\xf3\xfb\xf6\x01O\xf2\x0c\x84\xa3$\xc1\x088\xce\xae\x83I\xd2"I`\x1c\x85Q\xe2\xf3o\xa0\x0c\xcc>\xdb\x1e\xaf?\x04aE\x0caQD\xa4D\x8c`X\x11\xc1`\x04\xff\x91@\xccs\x84\xbe\x07\xc4 \xd0\xdfp\x0c#1\xe4\xf7\xa6\x99\xdf\xf3a\xbe>\xae\xf3\'>\xcc:\x11\x18\x0e\x89(\xf2\xc4\x11(\x86\x10I\\\x80\xd1u\x11 0\xcd\x91(N\xc2\x04\x8e\xe2\xbc\x08?I\x01q\x9de\x12z6\x92\'8\x9a\xa5h\x0e{6\x81\xf8{\x1f\x86\xe5!\x82\xc1p\x8a]\x17\x12\x87\xe2\xa8@\xad\xcb\x02aq\x1c{\xfe\x8fFX\x96\xc1\x18\x06\x86!\x8e\xa1Q\x1e\x13a\x9ay\xe23\x18\xc5\xd2\x82\x08\x13\xff\x05\x19m>\xcc\xff\xf2aDh\xfd\xc7!\x8aDQ\x9a\x12\x04\x86"\x10\x8e\x120\x02\x85\x9f\x1du(\x96\xe2`\x84 !j\xfd\\\x9a\x13i\x16[\xa7\x9aD9\x8e\xc2\x05\x94`HH\xfc\xed\x17\xf7a\xfee\xce\xe3\xc5\x87\xf9\x8d\xeb$\x96k)\x96+\xa61D\xe8\xe5\xe8\xd3\x8b\xda\xc8\xc5\xb1\x19\xeei)\x9e\xa3\xc0*4\x1b\xd7\x13d\xcdK}\xbc\x12\x1aJQ\xbe\xfds\xd4\xac\x07\xd1Z\xdayknT\\\x8e{aYo\xd5\xf5r\x9ak\x1d\xb5\x1e\xebE\xb4\xa4\x0e{M\xd6\x9c)F\xc3%\x0cj\xc2\xd8i\xf6\x7f\xff\xbbZ)\xc3\xf1>m\xd7|\xe9\x92\xee\xd9\xb5\xc2\xf6\xd0h\xb7\xa6\xf6M\x87D\xbb\xe8\xa2_t"\xd9\x89\x93\xb6\x13\xb1t/\xad\x99r\x15\x0bS+\xa6\xfe\xb7\x7f\xdf\xe6Ji\xd6\xcb|\xd1\x1d\x13\xd7\xf9\xd0\xf6\xf2\xce\xe0\xaaP\xd9?\xe8\xf2\xf7\x0bS\xbb\x87\xa8\x8c\'{kT/z\x1d\xa2\xccCw\xa2~\x8d\xc5L}Y\x8d\x82\xca\x96\xa6\xb6X\x0f\x16\xfc\xc89\x07\x17\xd0\xe4\xba\x0e\xfb\x8c\xe2\xb1\xbb\x08\x9f\x8a\x08\x15\x08\xb5N\'l\xd1\xb4rr\xaf\x8a\xe4\t\xb9\xa90y\xce\xae\xffo\xf6\xf1 \xee*\xe6\xd0\xc9\xbb^\xda-\xe9ui\xf5\xce\xa8\xaf3\x86\xe8\xf3\xb0\xc8\xe0\rfPQ5\x0es\x06(\xb9\x8b\x84g\xc5f\xf5\xf8X\x99\xd5\xe4\r\xcbYo\'\xdf\xa5\x11\xbf|\xd0naez\x1c\xa0\xedU\xe0|\xa1=\x9c\x7f\xfbw\xd5\x9e\xe7\xc9\xf0\x97j\xcf\xd7?}\x7f\xaa\xda\x03c\xdfg-\xaa@`/\xb2v\xf2X\xddJ/\xe5Q\\U\x83\xe3\x94(\xfc\xe2fF\xf9\x88\x92\xc7\xb0;\x9d\x00\xc3\xb2\x86A\xa8$\xa4-\xae](\xe8\x132\xa9\x98u\xec\x8e\xc5\xbb\xbd\xe0>\xa4\xf6<\xc3\xa4h\xf2\xb5\xb1}F\xb2\xc9r<\xf2\xbe\xdb\x99^\xe9\x83\xf1\xe1\x81\x9b\x08\x1ct:[\xe5\x03\x08\x1c\xb5\xc6=\x87\xa5\x7f\x05\xeb\xc5\xa9N\'\xf04xwr\xaa\xa3d]\xb5$\xc7&\xad\x1c\x99\x9e/\x89\xc6^\x86{D\r%\xa8xW(\xf9\x90\xda\xf3\x8c\x91x6\xc4\x86\xb1\x97\x15\xcb\xc8\xfd\xd9\xa0\xe9\x8e|~\xcd\xd3wh\xc9=\xcaj0\x98l\xf3\xa0\xaa=N\xb9\x8f\xf9\x0e/\x99\x1e\x04\xb3\xef\x1da\\\x81\x8d\xfb\x03\xde\xdb\x9e\xa5\xcc\x03\x00Q\xf3A\xc2\x1eF\x85{\xc4\xe1\xbc<:B"\xc0\xf8{\x8d\x99\x7fm\xb5\xe7\x99\xbb\xe2\x18E\xad\xab\xe5\xa5[\xa82\xb5\x975\x95;\x84L\x87\x1e\x83\x98P\x05\x01\xbe\x95\x18\xb7\xa6nEd\x18G\x14\xdeUvM<\xdc\xc8\xcc\xcf\x90\xdf\xba~!\xeb\x05\x0f\xb2.;@i~\x96.\xd1(\xeb\xc2\xdcc{\xf3!\xa0\xb2m\xe9\xcc\x9b\xfb\xe1Ch\xcf3\xca5W%\xd6\xf4\xf4\xa5\x8f\xe6Q$\x90H,A\xf2\x14A\xd0\xf1v6\xecXn1q\xc8O\xd2M\xd8w\xd1\xc8X\x86\x1d\x98\xe8\xf5x\xb1\xd7a8C\xd7\x01\xf0\xd2\x94,&]\xa9D\x82iU\x84h8\x0c\xbf\x99H\xbe\xa0=?\xad\xdb\xf4M\xc7\xeaCf\xcf3J\x12#\xc9u3\xbd4\x82KP\xab\x9f\xa0L\xc9\x96\xce0\x19\xf6j\xc3N\x04F\xd2p\x98T\xb8\xdfq\xa8]\x15\x91I\x1d\x13\rq\x8d[}d\xb2\xcb\x19\x8dKg6Rp\n\xe9\xde*\x90\xd1\x88\x99\xfdq\xd4\xef=\x1d\x14k\x0e!\xf1ov\xb8\xfd\x94\xd9\xf3-L\x18C`\xe8e2\xb5\x03\xae:\x84p\x00P\x1bb\xf9(^\x93\xe6\xc2\xb6\x9c\xd4h%\x11\x02\x87\x83\x904>R+\xf2\xc9)\xdb\xa0!\xd1\x8bw\x0b\xe2*J\xf8\xf3\x19o\x14/\x98I\xf8\xd0_ \xdc!rK\xa0gj\xa92\xeb\xdd\xc9\xfc\x10\xd9\xf3mc\xe2k\x85MR/\x93\xd9\xdd\xf66\xa2@\xbd;\x02=\x8b\xf8\xe3\xa1>{\x8c\t\xe4\xf6\xde\xf0n\x98\x99\x9d\x04\xb0\x0f0\xfdT\\\r7a=\xd1\xe5!5N.Ai\xa2\xc7\x83\x86\x96\xa1I\x84R\xe9\xe8\x07I7M\x99[\xf3\x84C\xff\xe6M\xf5)\xb2\xe7y\x8aC\xdf\xa8\xc3\xd7F\xbe\xf3]\\\xb2F\xe2\xd5^\x19\xce\x86Aq\xfd\xac\x92-YTr\xbck\xecd\x0c\xdc=2\xd1\xd7\xf5\xce=\xa8W\xf5\xc1h\xc9\xe1b\xa7Y\xbch.\xe2\xb5E\x07\xde\xf77\x18\xc1pP\xa8\xaa\xdc\xd5\x8c\xb8\x9d\xf1\xef\xb5\xf3\xfc\xb5\xc9\x9e\xe7(b\x10\xb1&\xaa\xaf\xa3XO\x89\x08\xee,\x06\xeaR\xae\x14\xc0\xd34\xc7\xa8\xb4C\xe9\x1d\x17\x0e\x1c\xef9w\x7f\x99\xac[qY\xf41\xbaK\']dbQ\xf2\xdd#\x00\xb5\x8adG\xd9\xaf`\xbc\xab|\x95\x9co\'~7\x0c\xfe\xf5\xc65\xe1\x9a\xd7X-\xf1=Y\xf2\xd7\x16{\xbem\x89\xf5\xb0\xa5P\xfc\xa5y\xff\xf5aE\xe5\xa3u\x88\xab\xc15\xac\x06\x9d tT\x00\xd8\x8e\x90\xf0\x92\x95\xc5cy\x94\x95\x98>\x1e\x8e\x9b\xd3\xacV\xd1z\xd3ka\x8bV"\xe9:cz\xf1U\x83\xaa\x00u\xda\xa5M\x06\xc5\xd4x\x96\x81\xc5|\x93<\xfb\x94\xd8\xf3\x0c\x13]\x07\x04\'\xa8\x97=\xe1/\xc6\r\x1f\xb4\x068{RR&\x88\xaa1\xf7]T\xc5\xc0\xa9\x87\xef*r\x05\xdd\xd3\x8dU\xa9!\xd7\x88=\x06\xad#\xbb\xdbQ%V\x0e%\xee\xd7\x8eK\xcc\x88\xa5\xdc\xebR\x1d\xce7\xed6\xb2\xcc\xee\x04\xdb\xce\x9ba~J\xecy\xee\t\x9a \xd7<\x81|\xb9\xf4\x1f4Q\x1e0\xe5tn\x82<;\xf0\x83\xc5\xaa\xe01\xc5\xaal\x9f\n\x97!Hv\x14\xdd\xcd\\\x05<\xd6\xb3\xce\xc6\xaf\xb0\x8d\xf1\xfeY\xcc\x8d\xddD\x1b\x9c\x9b1\xd9\xc8\xa8\xc0\xf5zrn\t!]\xbav_\xef\x88wM\xb2\x0f\x89=\xcf0\x89g\xf7~\x1cz\xc1f\xe4\x9c\xd2\xf6\xa9~?A\x00\xf40\xa7\x0es*\x14\x90M\xb8\xbd\x93&\xee\x1f\xab\x91\xcb\t\xbe\xbb\xd8\x9dc\x12\xf1\x02\x9c\x15TK\xe4\x93\x18*\xb1\xe7{\xa3S\xeb\xf6\x19\x02M\xd8l\xaf\x1c-\\\x17\xedz\xd0w_\xac\xa6\xf9\x90\xd8\xf3-\xa7\x80`\x9c"\xb1W\xac\x8b\x01z"^O\xe8R\x8el\xba8\xc6\x11\x9c\xc2\xd0\tD-=\xbe\xe0\xf2A\x8f\x04\xc3\'\xe8\x98\xb8\xab\xfa\x04h\xf1N\x06\t<\x95Ni\x83xT\x1ag\xdc\xb1\xc9\x9b\xfd\xae\xbf\xed\x88\xf9\x9e5h\xc5P\xd0\xbbv\xe5\x87\xc8\x9eoa\xae5\xf4Z\x00\xbd\x9cp\x01^\xed\xbdK\x0b\x94"\x17\xc9\xf0\x05q\x89\x1a\x9b\x9dF\x89\x93\xaaz\xc8rR\xd7\xc2\xee\x91b%\x17z\xb9\xcac-\xa2\x8e\xc1\x1dt-$FF\xaeU\xd6\xda\xd8\x00\xea\xcc\xd2\\-\x1b\xe2\xcb\x1dJ\x0ek\xce\xfa\xde\xa5\xff)\xb2\xe7\x19\xe6\x9a\x03#\x14\xf5\xea\xca\xe2mg\x9c!\xe8\xc6\xd0NM\xde\xf3\xb2\xde\xb7\x85-\xa1G>\x9eU*\r\x97<\x07\xae\xf5\xc2\xbb\xc1\x0c&\x99\xfb(\xf6\xaa6\xdf\xcc\x03v_\xd6\'\xbbs\x89\x80\xa3&@\xf46\x1b\x8eJ>\x9a\x16q&\xf9w\x13\xe1\x0f\x91=\xbf\x17o\xf8\xb3A\xfc\xcb\xeb\x0c\xa1\xec\xb8\x89\xb1t\xedV\xe6\x95U\x15\x1872#\xb43\x84$b\xafW\xbbU\xc4\xf5\x8ec|\xbf,\xd5\x8bT\xb4l\x96\x18\xc90\xdb=\xcf]\xd84U}\x1b\xb4\x9aC\xa8\xd65\x81\x19\x17\xef\xac\xe8N\xd9\x7f\xaf\xac\xf9\xb5\xc9\x9eoU\x13A\xe1k}\xf92\x8a\x13\x83\x1f\xa2F\xc6\xa9\x07\xe2w\xcdu\xf1\xf5\xc6/\x85\xb6\xd3\xb1a\xa9[\xe5\xd2e\xe7\x82\xa8Lg>\xf6\xaeG\xdf\xa2]o1y\x00\nKq$c\xefPP\xf0\xbe\x018\x14\n\x97\xca\x05e_\x0c\xaa zS\xe5\xfe\x14\xd9\xf3\xed\x9eX\xaf\x95\xf5ny\xd9\xfa\xf0H\x8b{\xfc\x1a\xd6\xd7\xd1\xdb\x9f\x10\xd1\x98/\x13|I3\x9f\xd9w\xe0\xcc\x12*A-\x00u\x98n\xe1!\x95\xc5D;\x146&=\xa49\x08\xa9\xa8\xdbUsA\xe6V\x88\x01Q\r\xf8\xec\x83\x1c\x80~\x91\xaa7\xf3\xfdO\x91=\xdf\xf6\x04\xf2\xdc\x14\xaf\xd7\xa1\x81\xb4\xcb\x8d7\xd0\xa3[\x86\x85z\xb8\x81ga_\xf9\xb2\x7f\x97\x82H\xe2.\xd7]\xe4\xf4L\n\x92\xddC\xbcK\xb9\xdf\x16\xc4n\x9f\xa2\xdd\x03\xab\xcf\xfd8\xc8\xcb\xad\xa4\xc8\x91\x07ul\x8e\xcc@\x19\xf5L\xf1\xa4\x9fD\xf6|{)\xbd\xc6Ia\xaf@\tb\x01\x00D\xbb\x89\xd7\x8b"\xee8\'EI\x03Rm\xf1I:\xdc9C\x1d\xaa\xc9\xeb/.\xd0B\xa6\x05\x16\x86\x89\xe6\xc6\x19=\x01\xa4Hr\xd9\xa3\x19\x1e\xd4\xbe6\xcb\x1e\xad\xf5j\xe7\rd`ZF\x13xo\xe6p\x9f"{\xbe\x1d\xe44\x02A\xe4\xab\xce\x0b\\\xef^X\x885\x13,|\xdb[\xb0\xcd\x98\x10\x05\xcc\x90\xa7\xb3K\xe5\x8f\xb8\xb63\x16\x89\xbd&\xe4\x89\x92\x17\xaf\x18\xf2\\\xf5/\xb38\xf5\xb6L\xca\xb2\xc7\x06\x19g0*B\x15\x0c\xedBZ\xf2\xd8k\x94\xf9&P\xf2)\xb3\xe7\xf7\xd7Sk\xfd\x87\xbd\xa2\x16uFH\x0b\xaf\xdf\xe0|=\xd8\xd8\x8c\x18\x82\n\x81\x1flR\x91k\x9e\x9at\xe7\xa4Y\x0f\x02PfA\x86\xeah\xec4\xc3\x0c\xd8b\xa3(\x02\x14\xe6\xed\xf6\x14\x83\x92\x18{\xf1\xcf\x8fNzX]\xca\xdcy:R\xbf\xb7h\x7fm\xb4\xe79\x8a\xd0ZQ\xff\xd9[>\xf2\xc6\xee\x11\x7f\xd2+\xd8\xb1\xed\t \x8e\xb2\xd4u\xd11\xc1\x88\x93j&\xacy\x90\xf9\xb3&\xc9tb\xc9\xcaR\x1e\xbc\xe3T_U\xe6A\x9e\xf2=\x8c2\xc2E\x15@\xae\xe1\xb9\xf3Q\xd6\x1e\x83X\x86\xe9\x11\xbe\xbd\t<}\n\xedy\x86\x89\xc0\xf0\xba\xf0\x88\x97\xf2\r\x11y\xe36\x81\xe0\xc2\x80\xbb\x1e\xaa\x13/\x16\x96\x9d\xc8\x1793p\x8fc\x11\x8a\x8f\xb4\xf5T\xc6n\xdb8\\\xce5\xdc\xaf\x85\x0c\x99\x93\xf6\xce\xa0\x87\x856[\xe2>\xfb0\x14y\xf3\xbd\x12bO\x97\xc1Y\x88\xde|\xa5\xf1)\xb4g\r\x93\xc4\xf1\xe7\xb6@^\xc2\xbcH\xb6\x97\xae\x97\xbd\x0f\xf0\x9awW\x89,\x08\x86l.\xe9\xe8bc\xf7\x02\xc6\xc6\xe9\xccR\xa8\xbc\xb7w^*@\xe4N9b=\xf8X3B\xdd\xa3\xfcPi\xe3\xc1r\xf0\x80L\xa3\xc2\x80\x9cp\xa2\x86\xc3\x05~3#\xff\x14\xda\xf3\xed \'\x90\xf5BG_N8l\x0c\xc4\xd3\x14\xef\xfbe\xdf\xe3\x8f8\xcb\xee\x98\xac\x9e\x92\xa0\xcb\xe8\x99K"M\xd1\xafk\x92KL\x17\xf6\x9e8\x18\xbc\x047\x9c\xbe\xdb\xeb\xa9\xee\xe5\x97\xa9\xd2\x0eW5\x17\xdc\x03x\n\x0e\xa3\xdcKx\x02H{\xfd\xdd?D\xfd\x10\xdb\xf3\x0c\x13\x810\x04\xa2\xb1\x97\xb7\x8d\x9c\xf4\x10\xf5k\xbeD\x1aZ\xb8=\x08h\x01*]\xe9\x8a\xb8,\xfb\xc5rk=\xed\x00\xcd\xc0B\\\x01c+\x07\xb5D\xbaSB\xbe\xef\x82\xddxi\xcf\xa7\xda\xf2T\x88q\xeb \xa5u\x95\x0f\xce\x0b\xe0\xbb\xf8\xe9{a\xfe\xdal\xcf\xb7Q\\gd\xfd\xa8\xd7T\x95\x8dq;i\xba\xeb\xf9vS\xcd\xc71\xc5s \xa7X\xd0ed\xb4U\xca\x8c\x11\xe6F\t\xcfd\xa55)h\xdb\x85\xbd\xb7oH\x9e,\xb7\xaa\xe6\x1b\x01T[t\x86\x16\xb9\xbd\x0cqL+\xada\xe0\xfd\xc9\xe5\xfe5\xb6\xe7\xdb\x17\xdd\xfe\x9e\xed\xf9\xbf\xff\xbf\xdf\x926\xcd\xbe}\xaf\xe2\xf7o\xd0\xfc\xc6\xe5\xa1\xc2\xee\xf5\xfa\xd8\x98\xcfok\x9e/\xa7\xf6\xf7q\xcf\xe3\xe1\xff\x9d\xe2\xcb-{~\x91\xeeY~\xd3\xff\xf5\x9b\xf7\xe1\xdbo\xad\xe3\x82\xc3\xdf\xbe\x1b\xf4\x02\xff\xcc\xdf\xfb\xaai\xe8\xebe\x80\xeaub\xbe~5t\x11\xe0\xb4\xd1\xe74\x10\xa7l\x97\x0e\x89\xa3\xd7F \xc7\x9d\x8b\xa6{\xb9\xd6\xf6:\xac\xed\xba9\xd9\xe9}Z\xeaE\x16\xb4X\xf8\xa0\xa6\x7f\xfc\xae\xeb\xf7\x7ft\x82\xd4\x97c#Bi \xd7\x7f\xf7\xa3->r\xdc\x8b\x88\x9b\x17m\t\x03\r\xb1v\xa2\x9b\xec\xc4%r\xd7\xb4\x1b\x91\xf5\xcc\x0f\xd1H\x14\x8dL\xd4\x89\xb8,\xc2\xb4\xa9G\xab\xc2o\xb6_\xf7\x11l\x05G\xc7\x13\xed\n\xde\x85\x15\x04\xc5\x8d\xa8[\xcdlF\xbby4\xf6\x91o{^\x18\xd7\xe9`8\x91\x1a\x0b\xe2\x10\x0b7A\xdf%\xf4\xc1\xfc\xa3\xd9\xf3O\xbc\xa0u\xd8\xa0\xd8\xa7\xef\x7f\xf7\xdc\xdf\x08\xa1\x17\xfb\xe7\xdf\'\x97\xde\x1f\xfb\xff\xbed\xff\xeb\x93\xff\x8a\\\x12\x1e\xda\xc5kS\xa4\xb8GM\n%\xbb\xa8\xd7\x83\xb4\x8a\xf7\x16\x12_\xa2*j\x12$A\xbb)\x0c\x98\xf5\xaf\x02J\xca\x081\xcc\x7f\xf8\xf4\xd7\xff\xf6\xe37\xad\x14\x10\x83\xd7`\x9d\xaf \xfd\xe2\x8d\t\x02\x8fo\r\xea\xc7\xc7\xe2\x8d\x9f}\xf4\xe9*\xf5\xe7\xbf_\x88a#\xde\xe3\xff\xfa\x9c\xff\xe7\xff\xfa\xb6M\x87.~~Y\xea\xb7\xdf\x9e\xbf\xb1Y[\xdf\xf9\x8f~\xfes\xac\xad_\xa8\x01\xfff\x1al\xd6\xd6fmm\xd6\xd6fmm\xd6\xd6fm}\xe1\xe7\xdc\xac\xad\xcd\xda\xda\xac\xad-/\xfd\xca\x86\xd5fmm\xd6\xd6fmm\xd6\xd6\x97}\xce\xcd\xda\xda\xac\xad\xcd\xda\xda\xac\xad\xaflXm\xd6\xd6fmm\xd6\xd6fm}\xd9\xe7\xdc\xac\xad\xcd\xda\xda\xac\xad\xcd\xda\xfa\xca\x86\xd5fmm\xd6\xd6fmm\xd6\xd6\x97}\xce\xcd\xda\xda\xac\xad\xcd\xda\xda\xac\xad\xaflXm\xd6\xd6fmm\xd6\xd6fm}\xd9\xe7\xdc\xac\xad\xcd\xda\xda\xac\xad\xcd\xda\xfa\xca\x86\xd5fmm\xd6\xd6fmm\xd6\xd6\x97}\xce\x7fK\x95\xf9\x87\x03\xfb\x9f\xaa2\x7f\xd88_\xd9\xda\xfa\xf1\x81\xfd$k\xeb\xc7\x07\xf6\x93\xac\xad\x1f\x1f\xd8O\xb2\xb6\xde\x0b\xec\xdf\x91\x9b~\x92\xb5\xf5\xe3g\xec\'Y[\xff\x03\x87\xc7fm\xfd\x10k\x0b\xfb\xdfc\xfe\xd7\xd6\x16\x0bs\x0c\'\xe2\xb8\x00\x13<#>\xbb\x8b\xf0"C\xc2\x0cK\xf1\x90\x80?\xbb\xa8B\x08\x05\t\x10B\xe2\x1c\xc1b0\x8d\x11\x98Hp\xad$q\x1do\x9cb!\x11"\xd6\x19\'\xd6\x8fCi\x14\x85\x04\x92G\x05\x06\xa1Q\x1e\xc29\x06\xc6\x19\x82\xe38F\x84\x9e\x8d\xe6\xfe\x1e\xdb\xfa\x80\xb8\xf4\xbf\x96\xf8\x86m=\x9fP\x80\x18V\x809\x8c\x84\x9eM\xfe\xd6\xa1Z?\x84\xa6\x98\'\xe0\x80\x0b\xbcHP\x0c\xc1`O\xa8\x05\x86\xe9\xf5\xa0X\x9f\x94[\x17\x06K\x93<\x89\x13\x02\xf9\xdb\x9fc[\x04\x0eA\x8c\xb0\x06\x05Q\x04\x02\xf18\xbe\xeeB\naH\x01}.,\x82cx\x16\x87`H\\\x97/\xce\xa3\xc8\xba\xe9\x19\x06\x81\x91g`\x1c\x8d\xff\x0fb[\xff26\xf4\x82m\xfd\xbb\xac\xd3\xb3\x1b\xd1_\xb2N_\x7f\x9f\xffT\xd6\t\x81\xbf\xdb1\x7f8Mz\x8f\x11\x9756(\x80;\x94\x01\xd5R\xed\x82K\x83]f\xb4\xd9\x81\xfe\x0cdl\x8c\xef\xad\xb0YNM\n\x8a\x96\x8e\x93\x9cu\x82m{\xf0\xef-\xf2\xb8\x08*\xd7\\\xc7e\x17k\x0e\xed\xb6a\x87\x86o\xb6\\\xfc\x98\xeaD\xfe\r^\xf7\xe1:^\xaf\x9d%/zL\xf4\xf6c\xa4\xc6\x0b\xc1\x9a\x176\xceq\xe0\x06W\x04\xd7\x1a\xf4\xfen-=\xaczp\xe8\x00\xe9\xc1\xbfV\xde\x9c\x05\xc3`Z\xa7\xa9\xc5vu>\x99\xa1s\xaf\xc7\xbcc\x03Lv$|b\x1f\x98d\x13\xc5\x9ba~Lu"\xff\x86 4\x0e\xbdv{\xbea\x13\xe1)\xbaf\xc1\xc2Q\xd5\xa0\x16\xb8\x8e\xad(\xe5-\x8f^\xf8\xbbx P\x10\x19\xfb\xe2t\x12\xa4C\xc4\xd9\xfc\xce\x00N\xd3\xb2\xf3\xf7\x18\xc1\xd5\x07\xee\x0c[\xd8\x19lE\x10S\xf7\xfd,K\xa951\xcc\x9b\xdd\x9e?\xa6:\xadSI`kNC\xc1/+V\xbd\x1d\x12\x85\xf7\xee$\xb6&$\xfd`d\xa6\x8d\x98\xd2\xa1+*\xc6\x00|\x8b?\xa1\xfax;\x14\xddI\x99G\xdbB\xad\xba\x1aQ\xfbd\xa1\xfe\xd5\x08#\xdd\x86\xc9a\xc7\xef\xbb3\x7f\x14\xce\x06\'`\x95\xd4\xb3\x11\xf3\xb5\xf0\x83O\xa9N\xeb(\xa2\x18A\xafy\xdd\xcb\x86\xa0\x99Z\xa5\xec\x125\x0b\x97&$\xcd\xcf\x13\xe0\x02\xdc\x8e\x01\xef\xb5\xe4U\xa6\xf7BTuY\xc1\xd9\xe8]\xb2&$\xa9w\xa5\xa2\xee\x95\x88`/\x87+\x13\xceti#RZ\x91\xd7\xb9B\xda\x99\xa7\xe63\xc7jo.\x96\x8f\xb1N\xcfdp\xcd\x14(\x02{\x912$e_\x99vE\x15.\xfa@Q\xfe\xd4?\x93\xe2\xbb9X{[\xa2\x8b(\xc6\x8e\xd4.\x81\xf0}1\xdc\x99\x9cF\x12\xf3\x81#\xdd\xe3\xd0\xd27\x1f\xf2\xcc+#6\xdd\x8c\x02\xed\x03\xbe\xf8\x98KK\xe7\x1e\x8c\xc87[\x83\x7f\xccuZg\x13&H\x8c\\7\xd6\x0b\x054\xe0\x9cR\xf8\'M\x11:\xa2Y2\xb0b<\xf6\xd44\x1c\xd8\x94\xd9\xce\xe9@&\xd2\x0f\x83\x0f0G\xcd>\x07\xd3A\x113\n\xdf\xf7\x00l\xd4\x95\xc4\xec\x17\xcaB\x15w6\x1f\xf7S7s\xd29i\xba;\xfbn?\xfbO\xb9Nk\x98\xebG\xc1k\x8e\xf2\xb2\xf5\xcd\xf6\xbaG#\x89\xea\x12\xf8\xdc\xe1\xcc\xc5>\x0f\xed(#\x13(\xc8\xadX9wT`\xbb\x0c\xb6\xee\xf2\xd8(\x00\x85\xe7G-J\x1d\xd1\xbb\xe6\xfe)\x1c\xac\xe1V\xf4\x17\xc1\xd3Q\xa4\xf4\xb0;\x0b\xb6Zp\xb2\x9c7{\xa1~\x0cv\xfaC\x05\xf3\x0f-_\xc9\xfdE\x9f\xa5\xe3\x95\xa7\xb8\x01\xe0\xee\x86a\x06\xc3A\x15\x8a\x90]\xc23r\xd7\xc2Z3\r\xd12\xe0Q\x89,\x8fH/\xc8d\xc5\x9a\xd2j\xa7\x93\xc6\x92\x0cx\xbc\x8f\xcf\xb6\xaf|\xf1\x00c\xd2\x1b\x8c\xf5bxso~\x0cvz.\xdaggs\x82|m\x83\xbc\xe7\xb19\xaa\x01\x1c\xac\x93\xb1\xc0C\x87\xe0#\x19\x84\xb4q*\x93\x8b\xa1\xde\xef\x1d\xbd\'\xe4\xd4O\x9eu,nY\xe6|\xa6\x14\xf9\xae\x83\xd0!\xb8\xc9\x0bN\xa0\x91\x06\x02\x89\x13\x0c\xc6PX\x00\x00bm\xff\xb5\x0e\xf2\x8f\xc1N\xeb\x95\x8f\xe2\xebz\x83\x91\x97QD\x88\xe8\x02\x97\xa5\rp\xa8\x8e\xc3\xf5\x90\xd1\x80bZa\xf7\x80\x9a\xcb\xfe|\xd4\xc7\xab\xe5\xc1q{G\x92\xa6\xba\xf4\xb6*[b\x82^q\xa0\x91\x10C\xaa\x8eH<-\xb7\xb8\xbc:k\xc5\xe1\x1f\x01\xe4\x8aJG\xed]\xf7\xe4S\xb0\x13\xf1\xb7o\xe78I\x93/\xd8\x19E\xde\x8d@<:L\x1f\xd5\x82}\n\x1d\x8b\xb5\xedst\xc2\xd1\xa0\x08\xcc\xf3i?i\x0e\xbb\x14\xdc\x19\xb1l!\xa6\x0bW<\xc3\xf8 H\xbe9W\xb5\xdb\x9e\xdd\xac\xd9u\x0f\xfe\xb0\xdc-G\xbe\x90cR\xa7\xef\xb6\xed\xff\x98\xec\xf4\xdc\xfa\xf8Z\xed\x91\xe8K?{^P\xa6\xc3H\xab\xb8\xc8f\x91D\x94\xfa\x88h\xc1-s\xefg\xce\x9c\xc0\x1ez\xc4\'\xec\xc1\xfa\x95\xb8>\x0b\xe6iX\x82\xd4,\xcd\x9e\x12 IA\xe5f\x8d\x94s\xd7\x91\xb9\xc0\'P\xb1\xc1H\xf6\x04\xff\xb9\x85\xbf\xd2\x9e\xf8\x98\xec\xb4\x9e,\x14B\xa2\x04\xfd\n\x7f>\xeee\xac<$U\xbe\xe1x~X\xda{\xedV\xb8W\x10\x1e\xdc\xb3VW\x97m.?H,\xce\xc8\x8c\xc9\xf7g\x05\xc6\x0b\xc3\x8c\x13\x02*\x16\x0e\xcc0\xe2\xc0\xc9\xebxZ\xf7GK0\xe9X\xa1\xa9\x9b:\xe77\x17\xcb\xc7d\xa75\xccg\xa2\x84\xc3\xaf\xf7\x04\x13(f|\x94n\x13x "\xa3\x05\xb0\xe6L\x8cI\x93\xdd\xe5V\x13q\x85\xb1\xce8W\xd7=(i\xa9\x12\xa5K]X\x8c)\x1d\xe9\xbb1\x01\x9d\xb6\x07n\x17\x87\xbe\x0b\x13\xe6p\x9c\xaah\x85\xea\xda{\xc0{\xf3:\xfc\x98\xec\xb4\x86\xf9\x94p`\x12{In\x96"\x869\xad\x9eQfd\xe0\x1b\xb9CA\x15\xf6Xz(\x0c\x15\xf0\x87~\xbd\xcc\x05\xdeWe\xe9\xa1y\x8c\x93\\]@\x9e\x0e$)\xfb\xd9\xa3\xcb\xf2]>\x810\xa7\x16\x88m\xef\xfb\x87\x08\xcbv+T\x1a\xfdf\xcf\xec\x8f\xc9N\xeb\xd6\xa7\x9e\xf3\t\xbfR\x16\xcd\xa3\xe0\x83XD\xd4\x12?\xf96\x0e\xdd@F\xca\xc38\rn>\x19\x8a\xc3.INjOL\x89O\xc0\x14KQ\xf4\xfe\xca\x07\xa7y\x7f[\xcbW*W\xe4\xb5\xb8\xe9}\x020\xdc\x86I\xefs\xb3\xec\xd7,\x97\xfe^\xaa\xfa\x8b\xcbN\xcfB\x1fF\x91\xb5\x9e|Q\x01\x16\xc1kNz\x17\xca;\xcd)\xce\xbcw\x13\x10\xaap|o\x94\xad\x87\'\x82,Q\x1e`)\x01\'\x87\x91\xb0\xdb#\xd4\xc2K5\x81Y\xe1kF\xe0\x93\xc3\xcdh\xfaY\xf2\xd9\xdbH\xe3\xcd\xf2\x15\xae\x06\x15\x82)\x0fS\xa4U@\xb5t\x7f \xe1\x9d*\x0b\xa1vh3\x1a:\x82]\xd2X\x08"4\xd9uz|\xaf\x04\xfe\xd1\xae\xd33L\x18Y\x87\x05~\xa9j\x1c\xaeNt\xdaj\x14=\xdd\xab\xf4rGi\xfd\x06k\xaa\x7f\x06t>\xc9x!r\xeb\xa2=\x95\xf1\x9cB\xb3\xcfw\xa3C\x1f\x11\x01\x05}\xe2\x16))#\xb4\xb6<\x1b\xfc\x02\x97x/\x8b\tQ\xb8\x11\xa7\xc5o\xfe\x89\xd4\xc7\\\'\xf2o(\x89\xac\x19\n\x81\xbdT5\x94u\xe3\xce\xa8l+\xe7\xd3\x01N#\xf8\x12\x13y\\\xc7\x17U9\x87d"\x1a\x9e/\xf9\x13z\xcch_\x01)\x9a\xbe\xb4\x08\x8c\x84\ny=`\xf1\xa5\x10v\\\x83\x1d\xc7\x99\x99\xba\x88\x90gg=\xa8\x06\x8c\x85\xdf\x9c\xcd\x8f\xb9N\xcf\xb2\x03\xa7\x9e\x7f>\xff\x92{\xc0\xa1\xc8\xda\xb2\x16\x1f\xbbe\x00/h\xa2\x1c\xe7\xa2\xda\xfb@\x1098|\xc4\xf1\xc9Y\xa6\xb8^\xf3n\xfe$H\xf1\xd5\xe7\x96)\xa9L\xca\xb5/\xe7\xb5\xee\x01\x9e\xaf\xe6+.\r\xd4\x9d\x0b\x14eE-\xbbG\xdf\xbe\x19\xe6\xc7\\\xa7\xdf\xc3\xc4\x08\x84|Y\xb4E\xaa\x1d:Zh\n\x88\xec\x07K\xdd\x01\xa2\x08\xb1\xcc4\xf0\xe3>\x01w\xa6\xa4(\xe5Z\x87\xde\xfc\x07;\xab\xecz\xd6s\xb8\xce\t\x084\xc8\xf9l\xebq[\xcdt\x11\xda\x8d\x80i\xea\xb2\xc8\x8cmPWq\xfa^>\xfe\x8b\xbbN\xcf\xd7S\x18\x89\xe1\x14\xfdr\xc2q\xdc<\x91x\xbd\xf0f\x83\x13\xa7\xfdip\xfd\x9c\x97\x89\x1a\x7fLx\xa8\x17\x98\xa0\xb9\x12\xdf\x9e\xb5\x1d\xb6\x17\xf7k\xf5Qf\x9eP\x9f\x9d\xeb\xc4\xd99\xab0\xd5X\x0f]\x7f\xcb\xac\xda\r\x01\xe5\xa4\x00\xdc\xc5\x0e\xfeE\xd7\xe9\xdb;\xd8\xbfw\x9d~\xff\xe6\xddF\xc4|\xe7\xbb\xea\xff9D\xcc/$El\xf8\xc6F\xc4lD\xcc\xff\x89\xabt#b6"f#b6"f#b6"f#b\xbe\xeesnD\xccF\xc4lD\xccF\xc4|eze#b6"f#b6"\xe6\xcb>\xe7F\xc4lD\xccF\xc4lD\xccW\xa6W6"f#b6"f#b\xbe\xecsnD\xccF\xc4lD\xccF\xc4|eze#b6"f#b6"\xe6\xcb>\xe7F\xc4lD\xccF\xc4lD\xccW\xa6W6"f#b6"f#b\xbe\xecsnD\xccF\xc4lD\xccF\xc4|ez\xe5?\x9a\x88\xf9\x87;\xf0\x9fb\x08\x7f8\xde\xbf2\x11\xf3\xe3\x03\xfbID\xcc\x8f\x0f\xec\'\x111?>\xb0\x9fD\xc4\xbc\x17\xd8\xbf\x03\x8e\xfc$"\xe6\xc7\xcf\xd8O"b\xfe\x07\x0e\x8f\x8d\x88\xf9!D\x0c\xfe\xbf\xc7\xfc\xaf\x89\x98\xf5!\x10\x8aE)\x16!\x18\x12\xa61\x11\x17a\x8ad`\x08&\t\x96$ \x84\x80)\x06aq\x96\xe2X\x81\x83\x11\x8e\xa2\t\x94\xa78\x94"\xb9\xa7\n\xf1\xdb_\xe9\x07\x14\x8eb\x1c/\xe0<\xc5C\x10\x87\n\x10\x8b\xa2\x02\x82\xb1\xa8\xc0\xc3$\x8f\xe10\x03\xa3\x08\x0e\xf34.\x8a\x02\xc3\xa3$\x053\x02,\x88\x1c\xc72\x1c\xc91?\x92\x88\xf9\xef\xa6k\xbf\xfdI\x03\x06\x14\xfd\x1b\x01\xaf\x93\x8f \xbf7%\xf9\x1e\x11\xf3\xf5}\x9d?!b\xd6\xa9\xe5)\x8ab\x08\x9ce\xd0\xf5\xe9x\x9a$\xd6\xd1\x16`\x9a]\x9fL\xa4y\x9e\xe7\x18\x18\x13yL\xc0\xb0\xa7\x1a\x82\xd3\x08\x85\xa2<\x8d\xa1\x8c\x08\x93\xcf&G\x1b\x11\xf3\x03\x89\x18\x9e\xc2)\x1a\'\xd7\x91"\x9f-\xc4`R`I\x1eZ\x7f\xc9\xc3\x0c\xcfq\x10\x07\xd1\x0c, ,B\xa2\x18\xben\'\x84\xa00\x12F\x05b\xdd\x9f\x10\x84\x13\xbf\xfd9\x11\xf3\x81y\xfa\x9f"b\xfe\xc5f\xf0\x1f#b\x9e\x8dM\xfe\x92\x88\xf9\xfa\xfb\xfc\xe7\x121\xd4\xf7\xfb\xd1\xf7:\xd7\x9d\xcc\x19\x86k)\xca\x91\xd9\x95\xfd\x86\x83+\xe0x\x9b\x17d\xba\xe3.\x11\x0b\x8c\x97\x9c\xb9]\xdb@\x99\x11\xf9\xe6\x0eo\x1c\x0bO\xc44:\x19e\xbe\xbb:\x91A\\\x8d\xeeX2\xd6\x0eqR\xf3t\x1f\xdeD\x14>e\xc4\x95\x85\xee\xf0\xb8\xe9$\x8f\x04>\x91\xc2\xd3\xfd.\xa8$v=seB\x85\x04\xb0\x0c\xa8\xb7\xbf#\x1a\xa9\xdb\xaey\x1bK\x05d\n\x97\xb2z[\x13\xb9\xa1\xf2@\xeck!\x9b2+?\xb8\xf9\x84\xe5\xaa\x9e\xe8"\xc4\x19\xf5\x0cu \x89\x0b\xe8\xf5N\x94x\xe4\x9fim\xec[\x89\xe7\xeb\xeb\xe5\xcd\xc6\xdb\x9f"b\x9e\x8b\x85"\xd7\x02\x16y\xbd\':\xc2\xa4\xe3B\x07}\xd3\x15\x8e\x88f<<\xac$O\xd4|\xc0\xee;\t\xe5lR\xe0\xb9\x14\xefmO/{\xe0\x10\xef\xa7\xda\x18\xb9Zj\x02\x9a\xdf\xc7\xa6\xc3\xdd\xd0\x88\xae\xb8z*\n\x1d\xc8\xc6\x01\xee\xc6D\x7f\xb3\'\xfe\xa7\x88\x98\xe7l\xa2$N\x910\xf1r\xc2\xb9\x8bv\xd2\x059\xa5x\xc5\xb5\xd5k\t\xc1pAe9\x86\xa5*\x01\x84}\x98\x81A\xdf\x82\xfa\xa4\xc1\xe1\xcd\xb9g#X\x82\xaeQY\xccc/\x8e3\xc6\x81\xa6\xbb\xcf\xae\xc8\xd1\xc3\xb8Q\xca{3W\x1d\x97\x7f\xb3\x99\xf2\xa7\x88\x98o\t9\x86\xa04E\xbd,Z\x85\x04\xf7\x14\x7f\xed+\x9es\xdb\xc7\xf88\x0b\x11\xdc\xfb\'\xcco)\x9d\xea\xd8\xc3\xd4!\xd6TU\xbbV\x05x\x02;`\x8b\x88\xa4\xcaz)>v(/\xd5\xa1\xc9D\xa2\x12\xd7;\xdb\x02\xdc\xbc\xf5\xed=\xef\x16\xc8\xf7\xaa\xb7_\x9b\x88y\xee\t\x84\xc4 \x88|mb\xdct\x87\xec\x1cV\xd1^\x15i\xf7\x98\xafe0u$K\xcc\xc6\xd2kh\xe4^{r$1\xacH \xb9\xe5r\xe4\xe1\xde)>\xd3\x8dv\xccA\xf8Q\xefs\xa84\x12\xd1&\xac#\x82N\xac\\\xd5\xc7\x03\xf7 \x8a7_g|\x8a\x88y.\x16\x88D)\xe2O\xb6\xbe\x03I\xfa\x08\x89\xa6nSds\xba\x86\xa2\xa9\xf1s\xbeS\xc7vh\x88`\xe2\xe1`\xd1\nSJ#\xd8\xba\xd5\n\xd7\r\x9e\xdet\xceI\xbf\x96{\xc64Ru\x16\x95\x13d:\xf1:\x80\xb5\xc5Au!\x98\xe4\x9b\xd5\xdb\xa7\x88\x98\xe7l\xd20BR\xebQ\xfe\xc70\xe7\x06\xc9#\xb3\x0e\x82\xe1\x90\xc6\x13v\xbf\xec\xf7\x86\x00\x117H\x95\x16X\x17x&\x00`"\xb9\xd0F\xed6GE\x8d\xa9v\xb2\xeb\xd8\xafP\xba\x9d\x1d\x0b(M\xc6\x0b\xa2\x9c\xab\x12\x84\xc4+\xc7\x17\xef~\xa0\xbc9\x9b\x9f"b\xbem}\x08_\xf3\xd9?\xb9\xae\x88\xc8S\x85\xecRqp\x96\xd6\xa2\xee\xddJ)G\x0b\xa3\xe9\xd8\xdc\x15P\xc2\\\x82\x02p\xeb\xd1\x81o\x14w\x10\xc0\x89\x05\x91\xc6\x9c\xb8\x9dy\xda\xdf(\xf2\x98\x0f`\x8c\xebD\xdd\xa9|\xcf\xf8\xfdI\x97\xd4\xef\xe56\xbf\xb6\x10\xf3mK`\x14\x02\xd1\xaf/4\xcc\x06\xbf\xca\x9d~\xde\xc3\x99u\xc9X\x18\'u\xbd\xb6\xefN\x0b\x94\x9a\xd1\xfb;\x90\x94\x005\xbd_!s\x7f,\xb9(\xea!\xc7\xd5\xe3\xdc\x06\x11\xa5\xe2\xee\xc1Q\xb8\xe8\x872\xb0[1q\xa1\x9b7\xcf\xf30\x8d?I\x88\xf9V\x03#\xeb\x08\xfdInsH\xd7\xc5\xc8\xc3I\xd659^\xdb.\x93s\x96C\xba\xb7\xab\x85\x8b\xad\x07\xf1 \xb7\xeel\r\xf4o\xf0\x80\xf7\xe7\x9d\xbe\x8b8\xd6\x9b\r\xca\xa6\x9bTg\xda\xbb\\P>=^\x08#\x15\x8d{!\xda\xbe\x7fY\xde\xcc\xf7?%\xc4|{o\x83R\x18D /\xb3\x19\x0b\x19cQs\r\xd8\x02<\x1c\xc8k\xd1\xb2\x93\xcb\x1fg\xd5C\xe3\xf6j\xdc\xadQ>GS.\x86\xd0z\xb6OW\xb2\xa5v\xf45S\xc9\xf2\x11&\xd5m\x8fj\x10\x03y\x9c\x9bK^\t\x1e\xf7\xd2\xe1\xffg\xef\xce\x9a\x1c\xc5\xb2\x04@\xff\x97z\x95Y\x8b}\x19\xb3y`G\x80\x10\xfb6\x0fc\xec\x08\xb1\t\x81\x04\xcc\x9f\x1f\xe4Y5]\x95\xca\xa8.\x95):\xcf\xaa\xcd\x05\xdeS\x02S\xa6\xc5\x1e\xab\x17J9r^\x17\xdeve\x17\xc4\xd5\xa1\xc3\xa0sY\'3\x99\x03\xb8\x99N^\xb2\x18|\xa4\x18\x02)\x93L\x1c\x96o\xd2\x02\x9f"b\xbe\xd6\xf1\xf5\x98\x8f\xaei\xdc\xcb\x9di&R\xb5\xc3\xc1\xbb\x91\xb6\xac\xfa7\x99N\xee&\xdb\x9do\x07\x16/S\xf7\x12\x1a\xc2\xec\xddM\xf4\xc6\x89(\xe6@gz\x07k\xca \x12K\xe9\x04\xde\xd1c\xa9\xfe\x94\x0c\xe9\xbe\xc6{}\xefin\x8fG\xaa\t\xbey\x7f\xeaSD\xccsn"\xeb\xea\x05\xaf)\xf9\x8bn\xf2(!\x93?\xdf\x89\x87H\xe8\x86\xc0\xce\x04&a\x81\x95\xdd\xdb]\xeb\xf6MH_\x84\x84]\x8a\x9b1YVK\x8d\xfbx\xba\x96\xc5\xa5\x93\xf5\xb1\x03\xed\x12\x15-\x13\x03\x84\xde?=\x96\xe06"A\xea\x1c\xcd\xdd\x7f\xa6\x11\xf35\xf5\x11x\xdd\xf2_\xef\xef\xfb\x97bo-\xaa\x80L\r\xe9\xba\xf5l\xd5\xce)r\xb511\x8e4z\xe6/\x9e\x83Q\xebx\x87EH\x0ee\xfb\x90\xb3\xf9E\x0e\xf8\xb3bc\xe5\xd4/\xa1\x9e\xed\xcb:@,\xc2?\xb4\xd7\xb2\x1b\xf6\xeb\xb1\x01{\xf7\xa8\xff!#\xe6k\xb0\x00\xd8S\xe6x\t\x931{%\x99`\x8b"J\xdf\xbc(\xfc%,\xed\xd8\x17\xc2\x8ehL\xee\x81\xe6aZ\xf8^^8x\xbdOK\xe1Z\x84\xe4\x9e\xbf\xd7pK\xe0>\xb7\x1cP\xe0\x1e\xa9\xbc\xe7a\x91\xd4\x9d9\x0b7\x06E\x80\x837\xf9\xbbO\x191\xcf\x9fH\x01\x18L\xc2\xf8k\xa6Z\xdf\x04\xa8\x07\x8f\x81\xda\xa8\xf4c\x0ex\xf6>&\xe7q\x0e%\xb9\xbe\xe1\xb9\x06\x88\xe7~\nR\xb2\xe6\x87\n\x81\xce]b\xc8\x16u\x0f\xc6\xdb\x8d\xa6\xc1\x80\xbf\xf5\x0f-CCd\xb0C\x10\x83@\xecv\x81ENx\x133\xfc\x94\x11\xf3uHEq\x84\xc0^s\xb8\x01C\xab\x89\xa8K\x1d\xba\\\x83\xd86\x87\x982\xadc\xc1\x8d\xe3\xbe\x8a\xca!\xf2\x02\x1bP\x175-\xbbn\x10\r[\xbd\xc5\xfc\xd9\x1aB\xcf\x90\x84\xfb1\xb8\x9cp\x80\xd8\xdb\xcd\xd8\xc9Z\xa2B\xbbb\xb1-T{\x93\xa6\xfc\x94\x11\xf3\xf5\x83\x06b\x9d\xd9\xc8+f\xb8\x8b[Of&FQ0\xb9:+M\xbd@\x135\xca\x8475\xa1sJ\x84s\x92\xa5\xc7\x11\x0f\xf1[\xe0k\xb7B|\x80\x19\x01P\xb2\xa4\x97\x8c\xd2\x0b\xf6\xb5!\xe0\x93\x1dH-g\x9b\x13Y\xd1q\xc1"\xf2\x8fz\xf3\xcfm\xc4<[\x11\xc3!t]\x14_\xa6~\x99\x1fZJ\x8d;\xc9\xda_\x98 \x87\xce1v\xb6a\xb9?\x8f\xca\xd1J9\xa8Ny\x95\xa2\xb29\xb1\x98\x99\xd0q\xcb\xbahp\x98\xdc\xe0\x83\xbaC\xe5\x19(\x0b\xcd\xbd\xb5v\x9c\xc9\xfa\xce\x05\xd5\xbb}\x98\xae3M\xfdKF\xcc\xd79e3b\xfe\xe5\x87\xd5\xffs\x8c\x98?\x91\xbe\xb1\x81&[\x93~\xff&\xdd\x8c\x98\xcd\x88\xd9\x8c\x98\xcd\x88\xd9\x8c\x98\xcd\x88\xd9\x8c\x98o|\x9d\x9b\x11\xb3\x191\x9b\x11\xb3\x191\xdf\xd9^\xd9\x8c\x98\xcd\x88\xd9\x8c\x98\xcd\x88\xf9\xb6\xd7\xb9\x191\x9b\x11\xb3\x191\x9b\x11\xf3\x9d\xed\x95\xcd\x88\xd9\x8c\x98\xcd\x88\xd9\x8c\x98o{\x9d\x9b\x11\xb3\x191\x9b\x11\xb3\x191\xdf\xd9^\xd9\x8c\x98\xcd\x88\xd9\x8c\x98\xcd\x88\xf9\xb6\xd7\xb9\x191\x9b\x11\xb3\x191\x9b\x11\xf3\x9d\xed\x95\xcd\x88\xd9\x8c\x98\xcd\x88\xd9\x8c\x98o{\x9d\x9b\x11\xb3\x191\x9b\x11\xb3\x191\xdf\xd9^\xd9\x8c\x98\xcd\x88yGC\xf8\x877\xf9\x1f5\x84\xdf\xed\x98\xdf\xd9\x88\xf9\xf9\x81\xfd"#\xe6\xe7\x07\xf6\x8b\x8c\x98\x9f\x1f\xd8/2b\xde\x0b\xec\xdf\x11G~\x91\x11\xf3\xf3{\xec\x17\x191\xff\x0b\x8b\xc7f\xc4\xfc\x14#\x06\xfb\xef6\xff\xe7F\x0c\n\xf34\x03\xc2\x04\xc9@\x10\xca\xa3\x1c\xbd\xfe\xcd\xb2\x04\x05\xb0\x08\x8e\x80$\x0b\x82\x00\xc7\xb3,\x06C8G\xf0\x18\x00R\xeb\xa7a\x02C@\x16a\x98\xaf\xfa\x1e?\xe6\x0f8\x98\x03\x10\x96\xe2\x9fu9I\x0c\x07H\x8a!8\xeaY\x02\x86EI\x96\x879\x06\x07!x}\x1d\x02\x838\x18x\x96E\xe1\x81\'\x9f\x01\x00\x04\x8b!\xdc\xcf4b\xfeV9\xf7/\x7fP\x80\x01\x01\xfe\x0bC\x08\x1c\xc0\x80\xdf\xaau\xfc\xc8\x88\xf9\xfe\xc0\xce\x1f\x181\x04\x0f\xb0\xd0\xfa\xe6\x1c\x8c\x100\xc9\x02\xcc\xda\x91$\x8d\xe1\x10B\xd2\x0f\x1d\xa3VoV\x1a\xfd\x14\x11\xf3\x9c\x13\x04\x80\x93\x18H\xbel\xcb\x1a\xec\xdc\x12y\xda-=\xd0\x9b\xd0\x9ah\xe6P\xad\xe3Va\xd4\xb8\x03\r\xee\x82\\\x14\x0cp,qr\xa9k-Y\x91\xadX:\x85\xa0\x97l\xb7\xd3\x05z\xed8\x86\xb8\xce\x03\xd0\x97\x83\xc0\xd4\xa7\x98_\x13\xbe\xf0G\xdb\xf2\x9f\x9b\x88y&\x88\xe03\xd1G^\x0boK\xc54\xc8\xa3\xe0\x1eF\xc6\xd2w^\x90a\x99\xbf\x80$}\xb1\x19\xd2N\x89\xfb\xd1h\xfb\x88\xa2\xaf\x99l\xcdZ\xe8\xd7(\xa1\x1f(l\xc2U\xc2A\xe9\xe1N\r\xf6\xf9V\xca\xe8\xdc\xb6\xd8\x83\xd2\xf2}q\x1d\x8c7\x93\x9bO\x111\xcf0a\x04\x87\x815{\xf8}\x98G$\x11\x92+\xd7\xfa\x92\xaa\x90\x83\xcd\x19m#^\xe1t\xb9\xc60\x10\x85\xc0\x05(\xa2l6\x11\xcb(\x0e{\x8e\t\xe5\x96\xb9c\x9a\xb0?\x82\xcd\xfdJy\xbeg\xb7\xfd\xcdo\xc5\xe5\xe1\xd5\xce\xfeT\xd4\r\x99\xd1o\xee\xfa\x9f"b\xbe\x92\x1b\x88\x00\xc1?`\x93\xa0\xde\xe1\xf42pX\x8f\xca\x8f\xa3\xf587\'\xe2\x00\x9d\x1bE>_\xb10#\xd6~\xec\xc2\xe22\x15\xd0q\xe4\xeegI\xd5\x93\xa4\xf4\xed\x03]\xe3\xb6\x18\x94\xea\xde;\xa0\x84Y"P\xdb\xd3\x11\x95%\xeb\xb4\xa2\xde<\xa0~\x8a\x88\xf9\xda\xf5\x81\xaf\xb4\xf7%\xb9\x81\xbc\x94a\x92(#\x80\xee\x1e\xa9^LH<\xb1\xcb\xe4\x12L\x91\xf9L\xdc /\xac\xef\xc4\xac\xc0\xf7\xfb\x84\x8a\x17\xe6,\xd3\xf7LQ\xfa{\xc81A\xc5\xba\xd9!"E\xf6\x84\xc2\xaa\xe2#s0\xf7\xf0\x05\xf3~T\xfb\xff\xcfM\xc4|-\xa0\x08\x8e\xae\xa7\xe5\x97\xa4b\x06\x84\x89!\xda\x843\xae\xceE\xd1\xf9\x9b\x10\x05\xfaL\x91\x84\xc2\xc2\xd25\xba\xdd\x0b]\x951K\xbc\x0f\'\xf7\xd8\x17\xf9\xac\x12 \xc2\xb7(\x00\xd3\xd7\xae\xc2\x9d\xfa82g[\xdc\xa3\x8d\xc3\xb8@k\xfb]r}s\xea\x7f\x8a\x88\xf9\x1a,8\x0c\xaf\xa3\xe5e\x85\xcbn\x8f\x128t\xdc\xa4\xe4gW\x81\xb4\xeb\x906\x8e\x17\x05\xf9~I\xae\xd2\xe9\xdeb\xd1\xe3\x8a\xeb\xb7\xec\x81\x05\xfde\xbfkG\xc19\xbb\xd4\xb1\x89\xb2\x0b\x0e\xf3\xd0s%s\xd2\x04?\xa6@\x91\xe6\xbd\x8c\xa5hZ\xb1\x02{\xd6\xc5\xb4\xe7$\xe7\xc0\x1c\xf3\xa1\xc4)\xa9\xe5h\xfd\xb2\x9b\xad\xc0\x1a\x92\xec\x1c\xdc\x1d\xe4\x17\x191_\x87`\x02\x84\x01\xe2\xf5~\xedNe\x8e\xc3\x9epGI\xbc\xb0\x97\x1b\'\xddAh/\xc2!qZ\xff3\xf6\x9dY\x11w7\xa8\xd5\\\x03\xb4\xd4$J\xd4\xab\x95c\x84dx\x1e\xa5\xa5Y3\xe1\xc3u\x95\xc7\xd0\x03,V\x80\x12:(\tS\xbe\xb9\xeb\x7f\xca\x88\xf9\xed\xac\x0f\x82\x10\x8a\xbe\xcc\t]?\xa7\xcd\xbe\xaf{?$Y\xd2\x8f<\xdb\xbb\x8c\xd9\x01j\x03\xe0\xc1N8\x0b/\x978\xe9\xca\x8bz\xbc\xa2\xc6,\xa7$$\x9chTs\xcc\xdb\xf5qi\xd5\xf8 \xd7l\xa1N\x17\xa6\xbf\xe9a\xb2C\xc1E\x8e\xdf\xc5S>d\xc4\xfc\x16\xe6\xda\x99 \xf2\x12\xa6y\x83\xee^?\xa7Xr\x18BT\xdc\x81\xfb\xfb}\xb4v\x11Q\x90a~GF\t\x19w\xa7\xf02_\xa1\x0bx:rA\x0e9\xfa\xcd\x12\x1ck\xac\xe8{;\x85\xde\x15\xaaN.\x937t\xb4\\U\x12\xbc\xaa\x86\xf5\xe6\xa0\xfd\x94\x11\xf3\xb5\x90\xa3 \xb4\xeeY/9\\\x11\r\x12\xe2\xe0\x07\x05;\x8b\xeb9>\x14\xedC?\x8f7k\xde=\xc4\xc3\x12\xb5\xb2\xd1\x87\xfb!\x0e(-\nh)\xe1,\xf1\xb4c!Q\xf4\x9b\x99\x0f\x10\x04\xc9\xcb\xea\xd0\x0exc\xf477\xdd\x93K\xed\n\xf0/2b\xbe\x0e\x1e$\x08A\xc0+Pw\xb2\x18\xc1\xe0\xf82\xe1M4\xe3\xe1\xa3\xdb\x1b\xfc\x01\x08h\xcf\xed\xa7\x87\x85JN\xb2\xdb\xbb\xe9<\xd8\xd3\\ aP\xa2\xdaI=\xd6E\xb9DB\xd3W\xa7\x8c\xbf\xb5\xc4\xbc\\\x8e\xa5q\x99\x14W\x91\x01%\x16~\x14\xe6\x9f\xdb\x88\xf9\x9a\x13\xeb\xd2A\x90\xe0\xcb\x9c\xe0\x90\xdabr&F\xf3dX\xa7\xb7#\xac\xaf\x02\n\xa0]\x8daF\xca\xd8L\xa8\x9a3[\xe9l\xee\x16\xdf\x86\xc1\x82N\x14"\xc4:\xa8\xdf7>wr*a$1\x0f\xdes\xd1Ma\x00\x99\xdb/R\x12\xbd\x89\xa7|\xca\x88y\x86\t\xa1\x18D\xae\xe3\xee\xe5\xa75|3%l\xbc \x88\x91\x0f7[\xeaF\x99\xe2LHsR+N\xfb^\x1cS\xdd\x1d\xc04\xb8V\x97\x0bs\xf4\xf2R\x1f\x9biW\x80SE\x1f`\xcd\xec\x88\x0boF\xd9MCk0C\xfa\xa3\xb2\xa3\xf0\xdb\x9b9\xdc\xa7\x8c\x98\xe7\x9c\x80\x10\x08\xc0\xc8W\xd3\xcc\xb4\xca\x1d\x92(\x99\xd7\x8f&\x0b\xf1\x97\xbb\x17\n\x11\xbf\x9f`\xceN\xa0\x99eR\xb3\xdb\xf1m\x07\xec\xc9f\x8c\xe4`\xb4\x8e\x9c\xaf\x89\xa5\x1b\x97W\x8f#\xaf$Uiag\xa0\'\xb3z\xe4\xbc\x8e\xe9\xbbs}yS\xfc\xf9\x94\x11\xf3\x95}\xe0(\nA\xe4\xcb\xb6\x1c\xe4\xc2\x03:0\xfcD\xb91@\x8ex&\x043~\xe7\xaa3&v\x99m\x1fp\x1a\x0b8\xdb\xf2\xfd\x9b\x02\xf0\xfb\x8b\xfa\x90\xdb\x85\xf7u\x04\xcf\xa1\xf2!\xd0aM\xd6\xdc\x01M\x15\xf7Z?p\'\xd1e\xb7|7\xccO\x191\xcf0Q\x10]\xf3^\xf8e\x85cM\x02\xbe\x82\xf2\xbd\x89\xf6\x04\x868z\x80\xc5\xfcU.\t\xbc\xe3N\x9c\x9c\x1d`\xe0\xa4\x9e\xf6\x00({@wx,H0g\xf2p8x\x94\x13\x91\xd7\x8e-\x0c;\x90y\xcc\xb3:\x948\x96m\xe6H\xfa\xe2}\xaf\x9f\xbd}\xca\x88\xf9\xbas\xb3\xae\x8f\xcf\xa6\xfc}+\xa2\xb8\xf8\xf0\xf0\xbd\x80qD\xb7\x1c\xe7\xec|\xbdR\xf7c\x8d\xd2\xb1\x9e#\x88\x94\x8e\x8a\xe0P>[\xa1K\xec\x0cW\xa9\xbbP\xa3L3\x9c0P\xf3\x00i.\x12\xb5\x06\x0c\xe9K4\xd1\xcc\xecd\xc9\xf5\xac\x98\x7f\x9d\xfa\xff\xa3\x11\xf3u\xb3y3b\xfe\xe5\x87\xd5\xffs\x8c\x98?S\x95\xe3\xadp\xf4f\xc4lF\xccf\xc4lF\xccf\xc4lF\xcc\xf7\xbd\xce\xcd\x88\xd9\x8c\x98\xcd\x88\xd9\x8c\x98\xef\x9cB\xff\xff.\xd5\xdf\x8c\x98\xcd\x88\xd9\x8c\x98\xcd\x88\xd9\x8c\x98\xcd\x88\xd9\x8c\x98o|\x9d\x9b\x11\xb3\x191\x9b\x11\xb3\x191\xdf\xd9^\xd9\x8c\x98\xcd\x88\xd9\x8c\x98\xcd\x88\xf9\xb6\xd7\xb9\x191\x9b\x11\xb3\x191\x9b\x11\xf3\x9d\xed\x95\xcd\x88\xd9\x8c\x98\xcd\x88\xd9\x8c\x98o{\x9d\x9b\x11\xb3\x191\x9b\x11\xb3\x191\xdf\xd9^\xd9\x8c\x98\xcd\x88\xd9\x8c\x98\xcd\x88\xf9\xb6\xd7\xb9\x191\x9b\x11\xb3\x191\x9b\x11\xf3\x9d\xed\x95\xffh#\xe6\x1f\xfa\xed\x7f\xd4\x10~wI\xdf\xd9\x88\xf9\xf9\x81\xfd"#\xe6\xe7\x07\xf6\x8b\x8c\x98\x9f\x1f\xd8/2b\xde\x0b\xec\xdf\x11G~\x91\x11\xf3\xf3{\xec\x17\x191\xff\x0b\x8b\xc7f\xc4\xfc\x14#\x06\xff\xef6\xff\xe7F\x0c\x06\xf1,\x8b@8\xc9\xd3\x10\x083\xcf\x1a\xac\x1c\x8bs\x1c\x8e\x138B|Uc\xc3P\x82b\x10\x80Bi\x18gi\x06di\x88\xc09\x18C \xf8\x8bW\xf8\'F\x0c\x0b\xafM\x08S<\x0e\xa08\x88\xc1\xad\x03\x9e\xe5\xf8\xf5\x1a\x10\x80\xe6\xd9\xbf\xfc\xc9\x8d\x98\x7f\xb9N\xfa\xc7\x8c\x98ga\x93\x7fj\xc4|\xffy\xfeK\x8d\x18\x04\xfc\xb1/p\xa0\x03\xd4\x9a\x1eTl)\xaa\xc0\x10\xf7\xa88\xce\xd9\xce!\x17J\xb5\'<\xeb\x0c\xa2\r\x08F<\xb4\xec\x89\xed\x97\x1d&\xde!\xeanT\x0b\xa3\x07\xfc\xe5p\x19Fx\xefQ4\xcb\xef\xdb]9z#N\x08\x88\xf8f\xa9\xaa\x8f\x191\xbf\xdb\x15\xfeA\x8b\xc0\xb0\x1c\xeb\x1e\xb7\xbd-\x82\xe3\xc3\xb6\x0b$\x04<{\'\x1a\x14(\x1fg\x12\xda\x9b\xa1\x8f-\x14f\x8a\x15\x1d\xd41\x8bR\n\xc0\xa5\x0f$\xe1N\xd0x\x07PR\xd0|\x14K\x0fc\xf2 \x85~\x195N\x89\xdf\xac4\xfc1#\x06\xff\xafup\x81\xe8K\x99a\xa6\x1cY\xcbsj\xa4\xe0\xca\xcbxj\x0b\xae\x9ehK\xb7\xd1\xb4\xbe\xd8\xb4\xbe~\x95\x13\xc1\xe2\xb5\x9e\x11\xfc\x02\xd8\xc2\xe36\xf2\x06\xafr\xc3]\xf6=\xc9\xd7LNsK*h\xaf\x99\xab\xa7";\xd0E\x1eA?*3\xfc\xb3\x8d\x985F\x02\\\x97\x17\x04\x7f)!\x07-p \xf1\x1a\x1b\x1c\xb0Ak\x93\x87P\xa5I\xf9\xb8\xcf \xe3\xe4\tx\xd6\xb2\xd9\x08\xef]q\xbd\xa2r\xc1\x9f\xd1\xf3n.,\xc5_\x8fT6\xd9:\x8f\x10\x9cs\xe4\x96O^\\cMf5\x85\xb2\xa4\x1e\xf3\xad\xcaT}\x8c\x88\xc1\xff\x0b\\\xb3\x04\x0c\xc2\xe1\x97\x8a\xaai\xea2\xb7|\x96\xcd\x13hG{\x86>\x98\x01\x91\xaay\n\x9d\x1dD\xc2`\n\xc4g\xef44X`8\x92A\xf8\xe3\xb1p\xcb\x1c\x8b\x01\x8eU!jw\xe2\xb9\xddQ\xed\xc4\xe3AC\xf3\xbe\xe2\xc5Pa\xe3\x02!\x93\x1b\xd2GK\n\xba }\x81Z\x8c:t\x0f\t\xb8\x8a\x17\xec\xee\x1ch@\xb5\xb2C\x82{\xd7\xcc\xce\x88\x8acs\x07\xdf\x89\x13\x9c\xff\x88\xfc\xf9\x93#1\xeb`\xf9rE^g\xfeY&\xe8\x14]W\xb9),\xa6Nen\x80#\x83\xb2\xa9\x1eI\\=\xecP\xa7\x1e\xa6Hf\x00G\xa5\xca\x07\t,ns\xd7QV\xb5\x8e\x91sj\xe3\t\x05/\'\xdc\x8d\x0b\xd1\x1f\x80K\x07\x06p\x12xR\xf3f\x91\xe1\x8f\x191_Sb\xcd\x11\x80\xd7\xca\xd8\x98\x96\xdbS\xd5\xf0\xfb)X\xee\x14\xd2[\xe3x\xf3m\x0f\x18.\xaa\xd8\x8a\x19\x9e\xf2\xd6R\xe3\x90\x898\x88\x7f!\xce\xcc\xa1\xaf\xdd\t\x94\xafEuF\xc61\x97\x84\x93\xa2x\x95j\xab\xae\xa63\x8bB\xd2\x9a\x8e\xbd\xc9C}\xcc\x88Y\xc3\x84\x08\x94\x84\xf1W\xf1\xc7\xbfCH\x86\x0f3\x0c\xc3\xcc\xa9N\xf2\xa1,\x90\xb6g\xe3G81\xb7^\xb5\x81\xe8X\x15\xcd\xbe\xc44\xc3\xbe\xd3\x1a3\xcc8\x7f\xe7\xd3\xc0?/\xa7b]\xcdw@!\xf6\xa7S\x93\x15w\t\xb3\xc3\x9b\xe9D\xc1\x8f(\x9c?\xb9\x11\xf3\x1c,\x04\xb4\x1e\xe1^7}\xad\xb4\xb0+;\xdc\x14\xc4:\x17\x95\x85\xf1\x82-\'{\xa9\xdc\x0b\x80\xe9j\xacl\x14\x98\xd5\xb3\x86\x14Fy\xfc\xb81}\x7f4\xaa\x83>\xac_\x15\x13V\xce@\x17C\x8e\xed\x83O\x8b\xd6\x19!q?\xb9`y\xf5f\x91\xe1\x8f\x191k\x98\x04\x86A\xd0\x1a\xeaK\nG\x10\x0b\xb7\x8b\\o\xcd}\xf0\xddA\n2\xc7\x11b\xe6|\xab\xa1\xa6=\x8c\xae\xcf\t%?\xe3\x1dy\xbd\x17sJ5u\x02Va{\xbd1\xcb\xf9^\x9feOk\xbb*\xb8\xeb\x8f\x86\xd9\xd1\x9e\xee\xd7\xb6i{o\xb2\x97\x1f3b\x9e\'7\x02Z\xcf\xe3\xc0\xcb\xc9\xad\x18\xe1\x9dt:)~\x86\x9e\xe6z>\x83\xfd]\x8b-lhP\x993\xd6`i\xea\x9cy\x0fz=\xf4\x98M\xec\xef\xab aE\xf2\x8a\x9a\xb2 \x9e\x95\xbb\xbc\x93\xf2}\xb0\x94\x83)-VI6,s\xdc\xb5\xf9\x9b\x07\xd4\x8f\x191\xcfA\x8b \xeb\x11\x15xI\xc8u\\\x1c+\xc2V\xb5}\x81\xc3E\\ea\x19\x8c\x17ZX\xbfuDC:\x13v\xde\xfd\x105\xe1\xcd\xd6\xd6\xb1)\xf0d\xd6N\x01\\\x93C\x98a\xe2\x84\xe8b6x;C\x14\xcc]\xd0\x13\xd6\x81\xa7\x02g\xfc^\xf5\xc5?f\xc4\xac\x83\xe5\xb9\x08#\xf8\xcbn\xb8S1\xaeT\x8c.\xf5U\xf6\x92\xa6N\x7f\xce\x14\xfc^u3\x99s\xda\x94\x9a\xee\x90\xd1\xa3\xe7\xedz2\x17\xd8Yq\x91D\xbb\xe7B\x8f\x83\xc9\x91\x85\xc3*X\xae7\xf1B\xb7^\xd7\xa2^\x103\x94\xea\xd8\x8fw\xb1\xc4O\x111\xcf\xb1\xb2\xae\x89\x18\xf6z\x04\xbe\x01\xb2\x977\xd9I\t\xc2$\xbe]\x1a\xa1\xb1pQ\xf1\xf6 \x00\xf2\xf2\xb1+L g\xd1\xf1\xd0\xf7\x8e\xe7\x98#\xaa\x02\x8f\xc0E.\x8c\xebH\t\xb0$\x04\x14\xc5{w]zI\x05<\xb2B\x03GI\xe6\xc5o\xc2p\x1f#b\xd6\x83\xfe\xfa" \xb2\x1e\xf4~\x1f\xa6m\xb7\xa8\xe4\x9c\x04-p\x86]`\xbbf\xee\xdcu)\xd5\xe0\x16-\xc4{-27\\97\x8a;\xc8\x19;T9_\x1d\rc\x9d\x99\xbb\xb37\xb6~P\x16\xed\xa2\x97\x19\xd2\xb9\xacwo\xca\x8a\xe4";H\xde\xbc\x9f\xf11"\xe6k\xd3\x87 \x0cz\x99\xf8G|0hv\xa4=\xfa\xca\xa4e>H\xfb\xd9\xf3][\x1b\x8f#"\x18\x95\x99-3<\xd8\xbd\xa4\xe2d\x08\xa9\xba\x1f/\x83\xac\xb8ry\xc4O3O\xb8\xc3x:\xdb\xf9\xedv\xf5uv\x90H+?\xbaW\xe7{\xed\xf9\x1f\x13b\xd6\x89\x0f\xe0\xcf.\xc1^\xa6D8k{\xb9\xd6\x119?\x0b:\xc4\x1b`\tS\x9ed\x8e#w\xb8+\xd9\xfe\xd4\x85&DU\xb5\xeb\x13\x91\xb9\x9f\xcf\xd6\xc9d\x12Q\xbf\x87\xfe\x9d\xaf\xfb\xf8F7Eh\xf6n\x19\xed[\x03M\xcdJ7\xaa\xd6xs\x97\xf8\x98\x10\xb3\x8e\x15\x80 H\x04{\x99\x11\xa7\xc13|\xd9=\x99\x9aWx\x8fCG\x03\xe4\x151\x1c\xda\xe6\xc0\xc4\x99G?\xf1\x0891\x0eU0{\n\r_\x99\x88k&\x07<\xd0\xe7\xabR^\xd9tG\x9dNr\x00\x8f\xa5\xc8-\xb1\x0c`\r5\xed\x8a7\'\xfe\xc7\x80\x98gg\xa2(\x02\xc0\xc4Kf\xa3\x08\xd6\xd0\x13\x0e\xe7\x11D\x8a%\xacRD\x86\xae\\\x11O\xee\xc0\xf3\xda\x90\xd6\x00D\x8bv\x89K\x94pP7\x08\'\xfe0\xe8\xfe\xf9:\r\xb7s\xe9\xc0\xc4\x8d\xbf\x02\x9a\x08\x8c{\xc3n\xb4\xfbc\xdf)\xf4){3\x81\xfb\x18\x10\xf3\xecL\x80\x00\x9f\xb7G~\x1f\xa6\xa1Np\xcfO\xe3P\x1f\x14\x1c:_\xa5\xbc\xd0*\xc9$\xf6\xa7T\x0b.\xf2\xce\xbb\xa9rI\x946]\x06;\x15\xeaI:\xd1&\xc3\xc3\xc3\x85\xe0u\xc8\xc8\r\xa3\xae!j^\x8c\xe3I\xa7k\xe9\x91\x9c[\x99~s\xcc~\x0c\x88y.\xe3\xd8\xbaW\xc1\xd0\xcb9\x1f\xc6\x03\xc8\xa2)0#\xbaGE\x85\xad\x92\xc3\x90_\xb2\nx\xd6\x07\xad\\Z3=\x0e`(\xf1~%\xdaGi\x08\x1f\xba:\xf1V\xa4\x10\xcd\x05\x81L7\xe6&\xcd\xa2\x81e\xc00xfN\x0e@\x14X\xf2f\x98\x1f\x03b\x9ey*\xb9\x0eY\xe85\xf7\xa8U\xffp\x03B\xff\x16\x1fU\x82\xf5o\x07a\xaa\xe6$\xd2\xac\xc6=\x9e\xa2\xf9\x06\xa3\xf8\xc3\xe7\x168\xb2\xdd\xe5\xee\x11ME\xd7\x93r\x165l\xc0\x93l*\xfb\xa1\xa0Z9/\xd1\xa5\xbe\xf0Sv\xbb\x9bx\'\x7f/\xe4\xefc@\xccs\xea\xc3\x04\xb2\x9e\x81_\x0e\xfa\x11\x8a\x98T>P\xfe-\xa1y\'\x82\rK\x03b\x885\xe2+\xdf\x1c\x94\xfd\xc1\xbf\x97Sv\xb9tY\x14{\x073\xc25>2\xcb\xdad\xe2K\x88\x11P&P\x84]\xe6}\x10\x14\xfd\x81\x1a5 5\xb2\x9e~s\x85\xfb\x18\x10\xf3\xb5\x8e\x93$\xf9\xba\x8e\xef\n\xfc\x90\xaa\xe3^\xbc\x0bM\xfc\xc8\xdc\x82\xad\xee\xc8^tco\x80\xcf\xfb\n\x11x\xf7@\xd5\x87\xc3\x1c\xa8\x97\xd8\xcbSnj\x13\x1b>I;y\xb4\xcb\x12M\n\xcfD\xc9\xdb\xb8\x1e\x98C\xf2f\x9d\xf6F\x17\xee\xde\xccS?\xe6\xc3\xacQ\x92k\x06\xb4\x9e\x94_\xc2\x8c\x8e\xf3\xec\xf2\xeb2\x1d\xefE\x9e\xbe"X\xbb\xf4\x822\xdf\x8d\x93\x0fP\x0b\xcfR\xcdUU\x974\x9e\xcf\x0bs\xe5{\xae\xc3"dh\r\x1f\xba\xd2\xc9\x8e\xca\xddI\xd5\x92\x1e\x80OE\xaekm\x00\xfa\xc3\x03\'\xdf\xbck\xf31\x1f\xe6\xf9\xb3\x94u\xee\xa3\xd0\xeb\xcd\xa9\x1b\xd1/L\xbf\xd8\xe8\xc9\xf4\xd8\x99\x03\x08\xdf\xc2O]\x97v\xd9z\xc4\xc0\x0bm}Sk\x7f\x13\x8c\x08\xe8t\x91S\xecu\x9d\xab8\x842\x0e\xd3\x12\xa5\x9c\x13E"\xe6\xeb)\xacTl\x18\x90\xa7\x82\x0c\x83\x1e~\x130\xfe\x98\x0f\xf3\x0c\x13E\xe0u\x1f\x7f\x91M\x0c\x90}4\xd7\xfdp\x1e\xe86\xd5H\x15\x8f5q2\x87C\xd7LJ,\x11;\xec\xea\xa9\xa4\x0ci\xb8\x11\xee\x8c\xd2n\xc6\xf2L\x08\x04\xb6\xebM\x9f\xd0\x88\x83\x1b\xc3\x8f\xbb\xe8 3\xb57\xa5\xbb\x1ci\xd76\x85\xfeC}\x98\xaf\x1f\xd5\x80\x08\x0c\xbcR\x86\x97\xf4\x04\x1fw\x11\x16=\xd8=l\x1a2C4SA\xb6\x05\xefG\x9c8\xc6\xd1\xa1g}\xfb\xdc\x0eW%mc\x15N\xc3Z\x1f\xcbP/\xd7|\xb45c\x0e\x803\xa1g\xc1\xb5\x03\xd4\xaehF\x14\xd5r\x1d\xfc\xed\xc7\x97\xff\xa3\x0f\xf3\xb5bm>\xcc\xbf\xfc\xa0\xfa\x7f\x8e\x0f\xf3\'\x9276\xccdk\xd2\xef\xdf\xa4\x9b\x0f\xb3\xf90\x9b\x0f\xb3\xf90\x9b\x0f\xb3\xf90\x9b\x0f\xf3\x8d\xafs\xf3a6\x1ff\xf3a6\x1f\xe6;\xbb+\x9b\x0f\xb3\xf90\x9b\x0f\xb3\xf90\xdf\xf6:7\x1ff\xf3a6\x1ff\xf3a\xbe\xb3\xbb\xb2\xf90\x9b\x0f\xb3\xf90\x9b\x0f\xf3m\xafs\xf3a6\x1ff\xf3a6\x1f\xe6;\xbb+\x9b\x0f\xb3\xf90\x9b\x0f\xb3\xf90\xdf\xf6:7\x1ff\xf3a6\x1ff\xf3a\xbe\xb3\xbb\xb2\xf90\x9b\x0f\xb3\xf90\x9b\x0f\xf3m\xafs\xf3a6\x1ff\xf3a6\x1f\xe6;\xbb+\x9b\x0f\xb3\xf90\xefH\x08\xff\x90\xa9\xfd\x8f\x12\xc2\xefz\xf9;\xfb0??\xb0_\xe4\xc3\xfc\xfc\xc0~\x91\x0f\xf3\xf3\x03\xfbE>\xcc{\x81\xfd;\xda\xc8/\xf2a~~\x8f\xfd"\x1f\xe6\x7fa\xf1\xd8|\x98\x9f\xe2\xc3\x10\xff\xdd\xe6\xff\xdc\x87\xa1h\x9e\xe7!\x06\xc61\n\xa3\x9e\xa5\xa6Q\x8e\x82(\x80a \x90\x80 \x9e\xe4`\xe6Y\x8dj\xfd\xf8\xf3k\xa0u\x93\x02X\x86\x85Y\x8c\xc4P\x92D\x9fu\xe8~L\x1f<+\xb9c8\x83\xf18\x85\x808\x05\x11 \x8d\xa10Mq\x10\xc0\x13 \xc9\xa2,\x0cB O\xe08\x07\x90\x10\x05\xb1\x0c\x06\xd1\x0c\xf2de0\x12Yw\xf9\x9f\xe8\xc3\xfc\xad\x96\xc5_\xfe\xa0\x00\x03\x8a<\x8bHC$\xf2[\xb1\x8e\x1f\xf10\xdf\xdf\xd6\xf9\x03\x1e\x86\xe0\x01\x8c\x04Q\x0ec\x9e\x7f#(D\x03$K\xb3 \x02\xe0\xeb\xa0\xa3\xa9\xa7\x1b\xc3\xd14\x06\xc24I\x93\x18\x0b\xd04\x05\xe2\x04\x00\xc1\xcc\xfa\xca\xc8\x133\xf9{\x1e\x86\xc5\x9f%m1\x84\'\x11\x90_\xbf\x07\'\x18\x16\xa4\xd6.d`\x1a`Y\x9e\xe0\xd7\xa1\x8e\x028\xce\xb3\x10MQ0\xb1\xb6+\x08#\x1cA\xaf\x03\x97g7\x1e\xe6\xf7<\x0c\x83\xf0\xdc\x93\xae\xc0\x11\x9a%\xd6\x11\xf0\xd5\x90<\xc9\x12(\x04\xad\x1fG\xb0u|\x81\xe0\xda\xd9\x08\x83\xe1\xec\xda}\x1c\xccs\x0c\x84\xd1\x18\x8c\xc1\x0cH\xfe\xe5O\xce\xc3\xfc\xcbh\xc9\x0b\x0f\xf3\x17\xa6\xe3h\xa6Eh\xa6x\xdc}\x88\\"\x97\\\x94Z*\xa2\xfa6&%\x7f\x0e<\xa38\x9a\xa8\x1aCk^\xea\xa2\x17\xaeFd\xf9\xeb\xeb\x88)v\x8f`\x08\xa1C\xd4\xb4H\xdap`(L\xd7\xf58\x87\x1e\xc5#\xa8\x8a\xf4\x9a\xa6\x07\x88\x0f\x1b@\xdc\x18h\xd2\xa8\xd5z^1\xff\xf6\xbd\xc7R\x02C1i\xd7|\xa9IDz=a;p \xac\xa9}\xddA\x81\x104j\xa3b\xb1\xc0?\x8e\x02\x8f$\xe2a\xcd\x94/!\x97\xb7|\xe2~}\xbf\xc9X\x1c\xa0Z\xf9\xe3\xeb\xcf$\xb9\xcc\x05\x91\x8d\xbc\xcd\x14H\xbd\xc7\x10x\x8fg\xb4\x8c\x9b\xea\x91\x08\xc4\xbaY:e\xcc\x1e@\xb5\xbeMJE\x83\xfee\xa8R\xe3P2\xe7u\xca\xf9\xe4\xc32\x0e\x16\xe5\xc2\x8c\xf88\xe0\x01\xf9\xc0c3\xd7\xf6\xa3\xd7\x89\x1e\xa7\xefs\x98\xf6\xaf\x17\xbb\xb8\x10&M\xb5\x0cCq&M3;\xe8\xa1\x87:\x9f\xe7K\x8f\xf0\xce\xdc\xcb9\xa4S\xb1\xa3W\x14q8\x1e\x82\xd6\xcb\x19\xe2\xbc\x8b\xc8\xdb\xe2{\xd6\x00\x9a\x8au\xd5wI\xb2\xeb\xc2\xbdi\xb1\x02X\xdf,+ .\xa8+\x9b\xe7\x010"\xd5\x94|}\xa0\x13\xe7<\xad\x9b\xf5\xbf\x8b\xf6<\xab\xcd\xfcS\xb4\xe7\xfb\xaf\xbe\xbf\x16\xed!~\xac\xd9p\x81W\xca\x08\xb9\x8b\x87\xfbE\xb3c\xec4\x1a\rq\x1c\xadT\xb5\x8eh6\xc9fc$\x17\xd8oE|a9\x91\x9e,\xb0\xa4z}9X\x98\x9c\xefd|t\x82F\xb8\x94\x9c\xab\\\xb2\xd4N\xdaj]}\xde\xf5\x1e>\x85\xf6<\xb7jt}\x05\x18D^\xea@\x9ekvB"\x86v\x13\t\xe7\xa2\xd14!?\x89Y\x99\x1d=\xd7;\x1eIx\xc8Nu\x08&\x92\x00\xfa\x0b\xac\xc2\xa7}\x0f4\xf8b\xa8\xbb\xc7y\x9f\xe2\x8c^\xf6\xf4\x95pI\xe2n\xa0n5,w\x83\xcc\xcc\x1fQ/?\x19\xedY\xc3\x84P\x84\x84_b\x8c\x08\xa0\xf0\'\xf3L\x18\xec%\n\xce!V7p\x12\x1c\xf5\xb4z(\xb7\x0e\xd7\xed:\x96\x8a\xc1\x02A\xec\x9e-\x8fr\xddu\x98\xeb\xf1A\xb5\xd5|\x05\x96iD\x1d\xcf\x9d\xa60\xcd\x9b\xa3\x01\x89\xb6\xdb\xe1\xa5\x92\xbdY\xc5\xfbSh\xcf3F\xe8Y-\x13\xc7^j\xb2O`\xc66\xec\xa9:J\x90\xd5\xa4\xc7\x1d\xde\xa9\x1e\nwa+YG\x8c\xe5\xe3\xb1\x9f1Y\x90\x8a\x82\xf54\x96Z$NHA\xd5\xea\x94\x1a\x8a\x81\x07\xab\xa3\xa7\xba:\x9dd\xd9\xc9\xf0\x84\xed]\x89\x9at\x83\xf8^\xd5\x11?\xa5\xf6<\'\x04\x8c\xe0\xe8\x9at\xbd\x94`\xbb\x9c\x9cK\xa5\xf8\x03\xc7\x17\x06\x1d\xf3)\xc0\xd7\xde\xe2\x8b|ek>Y)\x8c\xc4\xb0#\x19\xec\xaa\x80\xbe^Na\xa49\xeb\xf9\xcc\x9d\xa3\xca\x1a\x8c\xb3\xb1O\x86]y\xc0\xfcNOu\xfdt\x9d\x044\xa4\xaf\x81\xf3&g\xf3)\xb5\xe7\x19&@\xe2\xd8z\xaex\xa9\x1b\x88x}=\x0e.:\x89\xaaV\xe6,\x18\xa9\xcap\xbdXF\x85*\xecU?\xe4\xd2\x04\xf1\xd4\x90\x8f\x82\xdeAN\xe6\xb1\x80~[T\xc8\xcb\x02\xef\xb8\x7f\xa8#TI\xfa\xbc\x84v\x91\x15\xb1t\x94\xc6^\x98t\xebM\x00\xe5Sj\xcf3L\x1cFH\x08\x7f\r\x93fO\xa7\xa5\xa8\xfd\x00j\xb4\x0b\xc3R\xa1\xae^\xe5\x876d|2\xa3\xfd\x9ag\xb4s+\xe1\t\xac\x11\xd2yQ\xd3\xfdEW\xcbV\x07\xb0\x19\xb3\x10N\xe0\xf5T\x13\xa9\xeb\t\x06\x9b\xa6\x15-\x1f &\x89\x17\xdf\xaco\xfd)\xb5\xe7k\xd0>\x81\xb9?\xa0\xa6\xda\xbe\x0f1\xba\x83\xb1\xe1\nG\xf2\xcc\x0c\xe8(\xb7=Uw\x8b|G%\xa5g\xedz\x04\x8e\xb0\xd5\xca=\xa4\xe8\x18\xd6\x8e\x9e\xd9\xea\x98\xdd\xcfm8b9\xe4F.d\xd6\x80{\xf0\x1fM\xd4j\xbe,\xf3\xfd\xdb+\xdcg\xd4\x9e\xafs%\x82\xc1\xeb\xb9\xe2eO\xc6L\xf7|\xb2\x03G\x80\x14sG\x91{\xf2z0\x8e\xf1upH\x84\x93\x0c\'}0\r\xdf\xef\x1b\xac.\xbd\x87\x06Kg\x96\xbdu\xdci\x84o\x10ze%n\x19!\xf8@\x01\xa3\xd4\xfa\x0b\xba\x07\xa0\xf9\x8a\xb7\xd9\x9ben?\xa5\xf6\xfc\xb6\x04a8\t\x00/en\x91\xb3}\xe2f\x18\x15h\xa0E$]a\x8d\x90\xa8\x19\x9f.\xeb\x87\xb0\x838Wo\xa3H \xf1~\xaf\xd49\xc6\xb1:\x9ew\x87\x94O\t\xf8\xc4)\nR$Qz\x12$\x19\xd2\xa6\xfb^\xd1\x82l\x8f\x05\xb0\xf5\xa3\xda\xcc\x7fn\xb5\xe7\xb9\x1d\x02\x18\x8a\xe3\x08\xf8\xb2\x1d\xe6QB\xb1\xd2\\IV{\xf4XV\xe9\x12\xadr*^\xcd\x04\xbf\xbb\x1e\xfa\xae\x98\x8a\x9cN\xe0@E\xd5\xd0\xa3{\x82\x1f\xa5\x06\x18O\xca\xcd\xb3O\x17\'\x9e"\xc5\n\x94\nt\xee^q\xbd1\xf1\x99\xf7\x1c\xfcG\xb54\x7f2\xdb\xf3\xbb{-\xffP\xf9Y\x9b\xf6\xfd\x12"\xfb\xda&\xa4T\x83j\xf4r\xde\xf1\xb8J\x17\xbc\xa3xmY\x9f\xb5\xcbu\x07z\x02h\x06\x0f\xb4<\xe5g\x7f\xbf\xe0HhR\x8f\x0b\xef\xb0\xac\x89$\xad;\x02\xd0\x9d\xaaO\xfa)\xbeu\xc5\xd8\xbd\xbd]}F\xedyF\x89\xe1\x08\t\xae\xfb\xc2\xef\xc3\xec=\xf6\xae\t\xce\xf5\n0\xdd\x81w\xb5\xa3?q\xee^\x85v\xfe\x88\xec&\x14\xa0{\xc5\xc3\x1c\x00X@\xf2\x96w\x17\x1c\xaf\n\x7f`\x96\xeb%O\xb5\xce<\xe2#\xae\x8d\x8cX\x95c\xfb\xb0TS=\x9a@\xc9\xfe\xa8f\xf1\x9f[\xedy\xb6\xe2\xba\xcb\x80kV\xf9\xb2MH\xe55\xb4R\xb1N\x81\x02\x9b\xc9kn\x9cn\xe4\x88zM\x86\xf5s\xc2\x9d\x9d\xb4A&\x90\x9fY\xed\xde\x89\x89P@\x93\xb4;A\x8f\xd3-\x93*;?\n\xb8\xa9x\xb6\x91\x1dY\xe9\x11\xec\xf9}\x18\xe9>\xcb\xbd)Y}J\xedy\x86\xb9\xee4(D\x12/)\x9co\xda\xad\xe8\x16J(\xf7*\xc5\xf6|\xed\x86k\xba\x0fA\xafG\xe0i\xe9\xe5\xe7\xbd<\xbd\xbbN#\x9d\xe8"z\xee.\xb51\xc0\x80\xfb8\x9a1\x0bS6\xa4U\x92\x82\x96t\xa9\x19X$\xa2\xd6T\xf0\xae\x8b]z\xb9\xc6\xc5\xf0\xee\x1b\x83v(M\x90\xb3\xa5*\xda\xb9Q\x1b\x06o\x9e\xf4?\xe5\xf6\xaca\x02\x08F\xfe\xe1n\x98R\xbcY-<\x80\r\xa1\x03\xce\x1c\x1d\xf8fb\\\xeb\xb3"\xa5{Dg\xed\x1bX\xb6\x9a\xf1H"\xa4\x8b\xf8\x02\x8b\xe59\xf2\xf8v\xc8Nwfp\xfb1.\xbaVI\xac\x83\xe7cIr\xb1\x01\x82\xc0F\xfc\xcd\xfb\xb5\x9fr{\xbeR8\x08\x7f\xfe\x1c\xfb\xe5\x08\xac\x01NdN\xed\xae\xaf\x80FMheT(\x96\xbc\xa5\rH\xfb\x993K:\xb8\xcb\xef\x8b\x88(Z{p\xafg\xc4=\xb1\x1c-pF\xab\xc9\xa9P\x9b\xeb\x1b\x1e\x93S\x8a\x9e\xe5\xd8S\x9as\x93\xf4;67\xdf\xf4\x1e>\xe5\xf6\xfcv\xf7}\xedJ\xf2u\x85\x93Y\x86\xbcvI\xa7\x03\xdcth`.\xf7\xb2\x98\xe4L\xba\x1f\xab\xf1\xf1\x08a`\xce\xd2\xb9b\x12\xcd\xce\xaeP\x86q=\x18\x89\x8c9Ixr\xd0\x0e$]\x0e\x89q:\x80\x01]\xd3n\xbc/\x01\xe6\xe2g\xf6\x9bz\xc7\xa7\xdc\x9e\xaf\xe4\x03\xc3\xd7\x13\xdf\xcbI\x9cn\xc3V\xbd\n\xbb1\xc6\xba\xd4d\x1b\x02\x93\x15 V%\x04\xf0\xcf\x99\xac\xb1ZA\xad=\xd7\xd7\xc6N_P\x82\xece\xd5\x14T\x81\xecdc\xc9\xbd\x02\xa9\x1b\'\xc4\n\x0f\x18h\xce\x198\xf2\xdc{\x0b\x0b\xfd(\x1f\xffs\xb3=_wl\x11\x08\x86_O\xc0\x144\x93\xe4\t\xb0U+\x91\xfb\x9df\xa2\xe1\xb8?8`\xdfa\xd7\xc9X\x17\x18\xee\x92=T\x1b\x9d\xe7V\xb2\x97IK\xc7F\xf6\x12\x04\x10\x142?\x11ip.T\x95Pi\xa2\xc5:\xfa\xac\x8a\x87G\x88*\xc9\x9b\xcb\xf8\xa7\xd4\x9e\xe7\xfaF\xc0\xc8z\xcc}E\xe6\xba\x11\x8e\x88\x07+\x93\xfc\xc4\x1a\xc0\x85_\x13\x8as1P\xf9\xed\x06e\x82\x93\x02\xbd\xd7<(\x92\xd8\xd1\xbe\x1cz~\xf0\xac^\xc7\xa0fJ\xd1\x87\xc8r\xa9Ty\xde\xdf\xc3{\x7f\n.|\x81\xc8\xea\x9d\xc7o\xef\xf2\x92\x9fb{\xbe&>@B\x04\x0e\xbc\xa4\xe3b8M\\\x14u\xb9.6pFwir\xba\x84\'\x054\x91\xcb~\xf6z\x0e\xdf\xed%\x11\xed\x18c\x0eQ\xda\xd7\x08\xee\xa4Va\xcfGQ\xd2\xdb>\n\xc9\xf6\xa1\xc4\xd7\x83:\x04w*t\x13v9&\xb66\xfd\xde2\xfe)\xb6\xe7y\x0fn\xed~\x00\xc4\xc1\x97M\xf9\\\x89\xf6\x19\x16\xb2=\x9b\x07\xce%O\x06\xaa\x11\x1c`]\xc1\x137\x15\x17bW\xf5\xa0#\x9f=IR\xa9\xcb\xae\x03"\xbd\xb5\x14\xf5\xa1f\x97\x92\xb1Y\xd3\xa3*\xb9\x9a\xba\xa1i\xdc\x93d\xdc)\x13y\xccP\xf7\xe6I\xfcSl\xcfWo\x92\x00\xbe\xe6.\xbf\x8f2\x8b\x0e\x8e\xbf{\xae\x1e$8\x99\xf5\x14b\x98u\x08\x86\xc0\x86f\x06\x86\x15\x90\x8d$\x83\x86\x9az?;JOW\xeb\xc0\xfe\xcfT{\x9ec\x05\xc1\x9f\xb7m^\xef\xd7\xf6)c)\x17\xec \x15\xe4n\xce\xd4s\xed\x84^\xcaA\x10K\xd36_eL!E\x0c\xd1\xb7{\xf7\x11\x8cGf\x1a\xf1\xbb\xe7\xf3\xbbBOm\x1d[\x00[G\x0f\xb7\x015\xa0\x8b0 \xf6XA\xf3:I\x9d\xdff\xfe\xff\xa8\xf6|u\xfe\xdf\xab=\xff\xd7\xff\xf3\x97\xb8M\xd2\xaf\xc7*~{\x80\xe6/L\xee\xcb\xb4\xa8VQ\xad?\x1f\xd6<7Y\xfb[\xbb\xe7\xe1\xed\xff~\x84\xcd\x90&_\xcfNA\x08\xf6\xd7\x0f\x8e\xb7\xaf\x0f=yY\xec\xeb\xd1\xa0\x17\xf7g\xfa\xd1\x93\xa6\xbe\xab\x96\x1e\xacV\xb1\xfe\xfad\xe8\xc2O\xfe\xe2\xf4Q\r\x8ei\xb9fx\x96\xd3FB\xd7\x04\xe5\xe5\x11/6\x10\x8b\x06\xa46E\x99\x88\xc6\x9c\x88\x15\x14C6\x9a2\xc4\xe3\x1f\x1fu\xfd\xf1[\xc7P\xd5D5\x0f$\x9eT\xfd\xdd[;\x80\xcaYp\xd08U\xa1\xc40\xcf\xb9\x17\x9eI\xf9\x80ul\x03~Vs\x8b\xdc\xa2\xd2\xed\t\xb5X\x07:6\xdd)\x14\x0c\xed\x08\x06R\xc2\x91\x0f\x83\xb5\xa7\xe3\x12\xdc\xe2\xb2\x92-\x1b<;\xeb\x11\xd9\xba\xa0c\xea\x187\xd7\xa3\x15sQM\xa39N\xa6;\xb5\xfaB\x0bG\'9\x1auLj\xfa\xef\xc9\x9e\xff\x81\x0bZ\x9b\r\x08]r\xfc\xbb\xeb\xfe\x12\x84^\xe8\x9f\x7f[\\\xfa7\xda\xfe\rq\x89\x9b\x8f\x8d\xd3&P1\x06u\x02\xc4B\xd0\xab^r\t\xd7W\r\x9b\xe0\x12\xd41\x14\xc3\xdd\xc3\xf7\xa8\xf5O\x01\xc4e\x00\x9d\xf4\x7fx\xf5\xd7_\xfd\xf8\xcb\xd1\xd2\xc1\xe7\xef\xe0\xac\x7f\xe0\xbf>L\xf5^\xa3~\xbc-\xdex\xef\xc8%/\x89;\xfd\xfd@\xf4k~\x0c\x1f\xff\xe7\xdf\x9ep~N\xd3[\x17>\x9f\x95\xfa\xcb_\x9e\x1f\xd8\xa8\xad\x1f\xfc\xce\xcf\x7f\x0e\xb5\xf5\'\xaal\xbe\x15\x8b\xdf\xa8\xad\x8d\xda\xda\xa8\xad\x8d\xda\xda\xa8\xad\x8d\xda\xfa\xc6\xd7\xb9Q[\x1b\xb5\xb5Q[\x1b\xb5\xf5\x9d\t\xab\x8d\xda\xdaNO\x1b\xb5\xb5Q[\xdf\xf6:7jk\xa3\xb66jk\xa3\xb6\xbe3a\xb5Q[\x1b\xb5\xb5Q[\x1b\xb5\xf5m\xafs\xa3\xb66jk\xa3\xb66j\xeb;\x13V\x1b\xb5\xb5Q[\x1b\xb5\xb5Q[\xdf\xf6:7jk\xa3\xb66jk\xa3\xb6\xbe3a\xb5Q[\x1b\xb5\xb5Q[\x1b\xb5\xf5m\xafs\xa3\xb66jk\xa3\xb66j\xeb;\x13V\x1b\xb5\xb5Q[\x1b\xb5\xb5Q[\xdf\xf6:\xff-T\xe6\x1fn$\xfc\x8f\xa8\xcc\xef\xf2\xba\xefLm\xfd\xfc\xc0~\x11\xb5\xf5\xf3\x03\xfbE\xd4\xd6\xcf\x0f\xec\x17Q[\xef\x05\xf6\xef\xc0M\xbf\x88\xda\xfa\xf9=\xf6\x8b\xa8\xad\xff\x85\xc5c\xa3\xb6~\n\xb5E\xfew\x9b\xffsj\x8b~\xd6\xc1\xe40\x80B\x00\x10\x04y\x9c\x02P\x14\x00\x9f\x85\xd3\x11\x86&1\x9c\xe6a\x18\xa0\x18\x16Dp\x1c\'x\x90FA\x86\xe2(\x12\x01P\x8eA\xa8g\xa1\xbc\x1f\x83%4NQ4\x04\xc08\x05\x02\x10\xc4\x90\x14\xc2@/Q\xac\x18e\xc7at\x05r\x81<9(\xbb\xb6U\xe9\x1a\'\x1d\xc5X\xc8\x07\x9c\x84\xcc-\xb2 \xa3\xec\xc3\x1f!\x19?\x15u\xfa-F\x14\x04ar\x1d\x06\xbf\x0fsh\x914\x97\xdd\xa3r?i\x93\xa0\x11\xd3\xb8\xc8\xed\xfd\xec\x94\xc4\x80\x9fl\x1d\x9el_\x11\xdb\x8a\xae\xc9]t\x1d\x97\xd0\xae\xe5\xbe\xa9\xa8\xa4sN.\xc9\xaa\t\xad\xa6\x91"1;_f\x04R}\x9c\'-\xb3\xbeW\r\xe4\xcf\xa0N_\xad\x88!\xeb\x02\x81`\xaf\xa8\x13\xa8\xd5E\x13\xe4\x9c\\\xc9\xaa%\xa8\x16^\xe1\x93F\x91\xe7\xe8\\u\x0f*3gy\x0fA{\xabM\x17T\x12f\xbb\x83N\xc5!5\xd5\x01%\xbb\x1dC\x97\xb2\x8a\x98\x05\x9c\xf6c\xf1\xfc\xd5meW\xb6\x8e\xa5\xbd;!>\x82:\xfd\x16&\x88\x00\xd8\xba\n\xbf\xcc\x89\x87\xac\xb9I\x0b\xd2:\xe6\xc0WF\xe4vg\t\xcb\xcbV\x8a\xca\xf1!\xc3\x84\xc5e\xaclf\xeb!\xcd\xe6\x14\xae\x90\x12\xa0?pN\xe9\xc3\xb9\x94\x86\xa9\xea"\x08{\xb7s\x88\x16@^\xea\xfc+\xda\x9f\xa9\xb9~\x93<\xf9\x0c\xea\xf4[\x988\xfcU\x1d\xf8\xa5Fh!U{5\x12\xad\xe5\xbaW\x8d\xd3N)\xdc\xab\xcc\xf5\x87\xe0t9\x85\xb9\x96\xd3Nre\xabY\xb2\xec\x13\xda7\r\x99\x9d\xa1\xaa\x07c\xa9o\xc4\xc3\xd0\x12{\xa7\x05[\xc70\x10r\xaf\x11)\xddi\xec\xa9>d\xbf\x04u\xfakoB\x04\xbe\xa6-/\x05\x11-\x84K\xa2k\x08\xde\xc5\x9bI\t\xf8]\xde\xb97\xba^$\xdd\xf5\x12A\xd8\xc7;\x05\x9a\x85\xe1\x91\xa19|\xd4\xe9\xfcvWz\xfbJ\x9f\xfb\xb0\x90w\x82\x96\xd0Ic[d\xd3`\x9cFr\xc3\xe2>nh|}{\x85\xfb\x04\xea\xf4\xb7A\x8b\x00$\xfeZ=\x93\x1bh\x83[\xae\xc0q\xc1\xb0(9\xfa\xf1\x1d\xed\xc1\xe4\xa4\x8b^\x9d\xad\xdf\xb0\x07\xbb\x1cS\xce\xbe\r8\x17\x17c{\x0c\xce\xc8\xd9\x95\\\x97\xe0\xbc\xbd\x19\xc4#\xd3?\xe6\x9c\x17\xe3\x059\xb7\xfd\xe2\x81P\x92\xdc\xde,\xf5\xfc\x19\xd4\xe9\xb7=\x19Z\xd3M\x88|\x1d\xb4wR7\x04\x03\x98A\x1fl\x8d\xaaj\xe2\x03.M\x98\xa9\xca\x8eP\xf3\x01\x9bj\xb4\x82\\\x9d\xf3\x1a\xa5)_\x0e\xc6\xfe\xe0\xefA\xb6\n\x8c\xda\x84\xee1\xdf\xc8\xb8\xcd\xc0\x11\x07u\xe7"E]\r\x08\x84\xde\x95\x7fT\xd8\xf6\xcf\x8c:}\xb5"\x01=\xf9\x04\x1c}\x81\x0f*\xfe\xca_\xf8\x82\xd6I\xfd\xd1Fq\x0e\x16W-`\xee\xf6U\xf7\x0e\r\x88\x85>&R\xca\xd0\x1e\xaa\x0c\xb0h\xb9\xa9\xa7\x1b^d1r\xed+l\\\x1e=\'\x04=\xba\xd3vjNOt\x84\xce\xbc.\xed\x957U\xce\xcf\xa0N\xbf\xcd\x89\'\x04C\x82/\xeb8\x1f^\x07\xb7\xf3\x8c~\x96x\xad\x9f\xe6\x82\xf2DX<\xf4\x17\x13\x1b8{\xe9u\xc0\xbaQ\x15<=&<$\xdc\xf6\xa0\xf7\xf9\x04\xb5s\xe6\tS\xdb\x15\x0f\xf3\x98\xa75\x03\x1c\x08\xde\xa8F\xd5\xaa\xd3%1\x9a7\x9d\x9e\xcf\xa0N\x7f\x8d\x12~\x96\xbf\x87_:\x13\xee\x080\x0b\xdc#>\xf9\r\xae{\xa0(\xe1%\xc7\x89\xc7\x82\x06\xda\xa6\xcbf\xad\x03\xc5\x9d\x92\xd7\xfd\xa1\xacM\xf6\x0e\xech\xc4\xc5:\xc8\xbc\x9c&\x8fKnf\xe1\xb5\x1a*%(s\xb0y\xb3\x1aLY\xdd\x9f~T\x04\xf9\xcf\x8c:\xfd\xd6\x8a(\x01\x12(H\xbe\xb4\xe2#\x06\xa7\xebx{@$C\xea6\xefU\xa8\x8d\xd4\xce\x08e\xa3c7\xfc\x9e\xbe]\x14\x8c\xba\x86\xf4\x11V\x16Ca\xee\xe2)E$u)\x11m\x1e\xb4\x1b\xc4\x15,\x17\xaau\x10\xcf~g\\0!C\xef\xb5\xfe\xe6\xfa\xf9\x19\xd4\xe9\xb70\xd7C*\xf4\x9c\x17\xbf\x0f\xb3vH\xed\xec\x1e\xbd\xfd\x00\xee\xd4jXN\xa0\xcc\xd4\x80\x9b\x19\x00\xa8[~6W\x8f\xf3%\xb8x\x16\xed\xb8\xb8\x17Q\xd6u\x07\xcawp\xc7\xfb\'2\xaa\xb9%\xc9\xc9\xe3\xe1\xbaP\t\xec1kb\x19EZry\x978\xfc\x0c\xea\xf4\xd79\x01\xa2\x04N\xbc\x82\x95UV7\xb8\xcb\xe7y7\x05.\x19\xe5\x87\xeb\xa3\xe8\x0f\xb7\x10\xf7\xf4\x9a\xb4\x00|\xe7\xf9c\xe1 RR\xe9Z\xd7\xc5\xe0\x8c\xc4]gU\xb3\xbe\xe47B\x8fE0y\xb8\xf6\xa3x\xc4\xb0(kR)\xd0e\x01\xbe\xe9;|\x06u\xfa\xeb\xa6\x0f\xa2\xeb\x91\xfa\xb5\x9a}u=N\x90\x02\xf1\x84\x15\x92\xa1\xd6\x1e\x17\xda\xec\xcb\x02\xde\xdf\xe9\xd0#O\xc9\xdd;\x81\xd0nrg?\xa7\xf0\xc0\xdd\xd1\xa9r\xe4\xddSxO\xe42?\x9e\x1a\'p\x17\xc1^\x8f\x96\x05\x80\x8c\xe2qf\xb4@\xda\xff\xe8 \xfegF\x9d\xfev\xacYW`\xe25u\x02\xa8\x99`BY,N\xe1\xc57\xb5\tI\xe8\x8eC\xc6\x02\xcc\x13y\xca)~8s\x1dwJ\xd7-S\xdc3\x99\x7f\xde\x8f\xac\x8a\xf9\xc6z\xd6\xbd\xd6\x8d\xe0-\x98EL\x8c\x9b\xec\xf0f\xe0\x00\xe5\xd0\x85}\xf9\xc3V\xfc\xa9\xa8\xd3_\xe7\xc4\x9a\x08\xaf\xa3\xee%\x11\xee\xe4\xc1\xce\x05B\xe95\xb3\xe2\r\xb1\xf0Sw\x89\xc6PB\xf5.H\xcf\xa2\xbbW\xb0:\xdaMq]\xd6\x0b>\xf8T\xae:qPI\x94z\x94\x10\xa6U\xcd\x13\x89\x86\xcaA\xcde\x07\xec&\xa7I\xb2\x14\x97\xdf\xdc\x0e?\x83:\xfdv\xd4_\x0f5\xeb\xacz\xc5@nRZR7\xfb\x10\xf4\x11X\xf2\xe5\xc2\x16\xd9\xd9\xcfK\x00?h\x91y\xf1\xd3\xb8\x8f\x04:\x1e\xcei8d\xe9\x02\x1a\xbd\x91\n\x85\xe1\x14N@\xe5\xc0\xd5|\xb41g\xd9\x8c\x1aJ8U\x08W\x19>\xe8\xd1\xfcf\x98\x9fA\x9d\xfe:h\xc1\xf54\xbb\xce\xff\x97\x1b7\xa9\xbd\xab\xe7\x19\x00\xf7\xa3.d\x9dr\nX\x8eo\xdd\xfc\xe0bN\xed@.\x9c_\xeelU\xdcv\x9c\x94\x9e\xa8\xe64\xd2D?;e\xdb\xf9\xfe\xa5\xf0\xaf\xbc\x9c\x1dI\xa9\x94&\xf4H\xf1G\xfd~(\x8d\xb0\xfc\xd1~\xf5gV\x9d\xfe\xb6O\x80\x04\xfcz6\xc4\xafh!{\xf1\xe2\xf2\xed\x99\x02\xf6\xac\\\x8b\x83ZL\xb3\x08\x9f\x9c\xeb\xfd\xa1\x1c\xaf\r\xf1|\xe82=\x8f\x1d\xb5\xbb\xd4\':\xa2\xe6\xaeG\xc9\xd9p)R\xca\xa1L\xf7v\xbb"\x01\xddy\t\xe65\x83\xcc\xaa\xdb\x9b\xb0\xc2gP\xa7\xdf\xa2\x04\x08`Mo\xc0\x97C\x93\xc1\xb5\xb3\x1a\xde\xabA\xa9\x80\xc4\x89\xc6\xae\x1b1\x91\xee\x00u\xd6\x13\x96\xb5\x9d3D\x87J\x98\x9f\xce\x8b\x06t\xa1\t\'\r}_\xa2\xf2\x81I\x0eB\x03\xe9\t\xf71\x14\x80s\x12\xd0{Q\x99\r\xdcL\xa8\xe4\xcdt\xff3\xa8\xd3ogC\x80X\x0f\xc0\xe4\xeb\xfd\xdaC\'\xe1\xbb\xbe"\xb2RR\rm\x11#{\x8e\t\xdd\\FnG\xccUvC&\xb4\xd1\xaa\x86\xc8\x98\xbdUY\xe8`@`g\xd2!\x92h82\xd3\xa36\xc2wf_\x00\xa4\x1c\x0e{\xf7\\&\x0b\x90]\xde\xc4\xf9>\x83:\xfd\xb57\xd7V\x82\xc1W\x89\x0c\xf3B\x8d\xd4b9\xec\xe8\x12Lux\xc8\xa1#\x1a\x13\xde0\x14\xb2\x88\x92\xa1\x00\xd9\xe9)\x04\x1f\xb3|y\x1cw\xdc1\x9e\xbd\xa9\x9a\x16aw\x12\xed\xd3\xc3\\7\xb7\x92\x94\x81\xf1~Ud\x02i1S\nm\xf2\xcd\xbbp\x9fA\x9d\xfe\xba\x8e\x93k\xb2J\xc2/.gh\x197\xac\xe7\x19\xca\xf2XZ\xde\x07{\xbe\x16\xddl\xd4Tv\x9f\x95\x13\x8b^+\xe8"\x9f\xc1\xc7\xdd\xb7\xec\xd06}S\xc4\xf3\x9d\xc1q\xc7E\x0b)9\x86|\xf4N\xdf\x8ch."\xf8\xd1M\x14pn\xf2\xe9\xcd\xbbp\x9fA\x9d\xfe\xbfs\xc7\x9a\x80\xe0/\xbd\t\x9dlO \xa9~\xc6\xea\xc23\x87\x1a\xcc\xd3\xf0\xe6B$$;\x81\xf8\x18\x14l2N\xe6n\x8a\xc4dl\xcb\x19\x08\xeb\x8b?\xb0\xb4t\xeb\xa5\xd2\x9a4\x168\'\\\x81\xc27]\xf7\x15\xa0\x0f\x98"K\x10\xe2{\xdd\x99\xfe\x8c\xea\xf4\xdb\xd4\x071\x94\x84\xd0W\x1b\x0b\n!_\x0b\x14\xaa\xa9B\xae\x8a\xd9\xfd-\xe4ha\x17U`\xb6\x1f\x1e\xd1\xe9v\x13\xf0I\xb4<\xde\x16\'\xbc\xe0\x1dd$]\x03n\xa1\xc30\xdaYci\xe0"\xf9\xa0e`\xfb\x16Zr\x7f\x89\xa6\xbb\x07\xfbo\xaep\x9fa\x9d~\x0b\x13#P\x00\xc3^\xe7\x04`\x9a\xe5\xbd\x1b\xcc\xd3n\\\xf3\xcf\xde\x85c\x91\xe4\x02E\xc8\xd9#~\x91I\x1b\x9a\xa1\xa9\xaf\x80\xb2\xe6\xd59\x06\x9b\x00Ff2\xc0u\xc8\xe7u\xbe?\x90;\xf0\xe8y&\xde\x00\xdd\xd1/\xc7a\x10\xa8\x13\x11\xbeyz\xfb\x0c\xeb\xf4\xdb\xd4_\x8f4\xeb\xd1\x0fx\xdd\x96\xc9\xa3\xf4(\x03\xdc\xdau\xe2]v\xae\x1d\x85\xf4\x01\xe4N!\xdc\xca\xe0\x15\xc7\x0f\xad\xe6\x9a\x01\xa5\x07\xed=\x9a\xc7R\xa12#K\xee\xf4C\x9c\x0c\xa5<37>\xa4x\x82oK\x12;\xdd\xad\x10*\x8dEV\xdf\xec\xcd\xcf\xb0N\xbfM}x\x9d\xf6\xeb\xc8x\xd9\x96\x1f\xfcE-=\x82\xdc\x87\x12\x12\xc4\x89#CDa\x9beB\x90\xf5\x11^3\xd8.\xe4\xc4\xb6xPG\x12\x1ep\xb6\x9brsg\xd3\xc5\xf1\x94\xdd\x1axOd7\xe0\xb47T"\x12w5\x8a]5\x19\xc1$\xad\xfa%\xac\xd3_3U\xec\x99\xb5\xbcn\xcb\x95\xde\xae\x87L,\x9c\t\x1e:\xde\xa93\xcd\x96^\xcd\xc2\xce1x(5s\xadjc\xc0o\x1a\xcc\xee\xdb4RZhB\xdb^\x9dxtQ!\xef\xbc\x1ce\xb9\xa0\x83P-\x07\xfeL\xea(\x1de\xe2\x01\t\xcc\x1f\r\xda?\xb3\xeb\xf4[+\xae\xbb!\x88\x82\xaf0\xa7\nS\xe3\xa3[Z\xca\xcf\xedx\x0e\xcc\xf6\xe6`>j;\xb9\x12\x97\xa7\x81\xa2F~.\xca\xc1\xab\xb2\xc3\xc5\xc0AJ\x1a3\r\xc9\x98nW^\x94;TK\x8f\xfa\x06\xf7\x918A\xb4\xde\x0e$\'\xceH\xc5H\xd4\xe3_r\x9d\xben8\xfd\xbd\xeb\xf4\xdb\x93w\x1b\x11\xf3\x83g\xd5\xffs\x88\x98?\x11\xbe\xb1y&[\x93~\xff&\xdd\x88\x98\x8d\x88\xd9\x88\x98\x8d\x88\xd9\x88\x98\x8d\x88\xd9\x88\x98o|\x9d\x1b\x11\xb3\x111\x1b\x11\xb3\x111\xdf\x99^\xd9\x88\x98\x8d\x88\xd9\x88\x98\x8d\x88\xf9\xb6\xd7\xb9\x111\x1b\x11\xb3\x111\x1b\x11\xf3\x9d\xe9\x95\x8d\x88\xd9\x88\x98\x8d\x88\xd9\x88\x98o{\x9d\x1b\x11\xb3\x111\x1b\x11\xb3\x111\xdf\x99^\xd9\x88\x98\x8d\x88\xd9\x88\x98\x8d\x88\xf9\xb6\xd7\xb9\x111\x1b\x11\xb3\x111\x1b\x11\xf3\x9d\xe9\x95\x8d\x88\xd9\x88\x98\x8d\x88\xd9\x88\x98o{\x9d\x1b\x11\xb3\x111\x1b\x11\xb3\x111\xdf\x99^\xd9\x88\x98\x8d\x88y\x07C\xb8\xbc\x85!\xfc\xee\xb6\xc3w&b~~`\xbf\x88\x88\xf9\xf9\x81\xfd""\xe6\xe7\x07\xf6\x8b\x88\x98\xf7\x02\xfbw\xc0\x91_D\xc4\xfc\xfc\x1e\xfbED\xcc\xff\xc2\xe2\xb1\x111?\x83\x88\x81\xfe\xae\x1f\xff9\x11\xc3\x02\x18D\xf1\x10\xccA(\xcb"(\x85\x90,F\x110\xc9S\x04\x8f#8\x0f\xe3,\xcb\xa1\x1cG\xf18\xc9r8\xbd~\x9a\x83(\x02 \x01\x9a\xc4\x19\xe2\x9f\x131\x04\xfd\xac\xde\xccB(O\xe2\xeb\xfb\xf0\x18\xc2\x108\x080\x04\x07p\xcf\xd2/4\x86\xb0\xdc\xfaE\xec\xb3\x0c"Ga4\xfd|\x07\x08\xe1\xe8\xf5\xbd\xd7w\xfe\x89D\x0c\xf4\xb7*\x0c\x7f\xf9\xa3\x02\x0c\xf8\x7f\xa1\xe4\xb3B\r\x86c\xff\x8c\x88\xf9\xfe\xbe\xce\x1f\x101\xf0\xda\xe8\\\\\xbc\xc1\x85\xa5\'\xbc\xc6p5z\x1d_\x86\xe0\x99R\x8a\xf3]:\x0b\x91I\xe1\xba.Bw\x8eWN\xd8|\xb4\x82\x1f\x15\x18\xff\xd9F\xcc\xb3+\xd7\x97@\xa0\xd7\x11+\x89\xcb\xb1\xa7\x9d\xc6\xef\x17\xad\xe2\x026\x81v&\xcd\x961\xe3\xb9R\xc3\n.Q\xb8\xbc\xd9\xb1\xfb\xeb\x89\x04\x076\xd5\x94\x87j(\x97\xcc\xbeW\x88l[\xfc\xb1=?:i\x87\x04\xb6D\x0cI\x08JJ\x86\xed\xbeW%\xbe\x8f\x191k+\xae\xcb\xff\x9a\xc9\xbe\x165\x93\xcd\xb6\xe9\xce\x8c\xbf,"hx\x0f_2"\xda7[\'jx|\xea\xf4}\xc3\xd2\xfb\xc5{\xd0"p\x181\x19\x04\xd4\x1d\xac\x96\xd5\x15\xd4A-\xc9\x99r>C\xfa\xdd\xe7\x8f\xeb\'\x1b\x87&f\xcf\xb6\x02\xfeG@\xc3\xcf6b\xd60A\x0c\x02\xd6\x14\xf6\xa5\xcap@y\xbeD\xa2\xa0%\xb51 \x9c\x89#\x82\xa3Ep\x15\xd4\x9e\xdd\x95\x18\xce\xe7r\xd7\x90Y\xdcZ\x83\xece\x1d8\x8aS\xd9jtJ\x9e\xc7#\x9c\xaa\x8d~we\xadw\xf6U\xaf\xc8\xfe\x8d\xd1\xba\x83d\xed\xdfUE>e\xc4\xfc.\xe9\xfd\x07U\x04q\xb0\x9b\x03\xcc\xb0\xe5"\xb1m\nge9B\xd1q\xf1\x03<\x9a0\xc7\x9bk+\xa1\t"@A\x99L\xf8\xe8v."\x00\\\x9e\x05\x01s\x0f\x95\xcf\t\xcc\x1d\'\xa0\xe7NN!"S)\x10\xfcia\xdf\xec\xcd\x8f\x191\xcf\xde\x84\xd7FA\x89\x97\x82\x83w\xb4=G\xf8M\xb8\xec\t/,\x9d\xbc\x96\x1fP\x0c\x17\x98\xe2\xeb\xdc\xb1\xc7\xa6\xb0Qf\x9c\x88\xfbJd\xc8T\xec\xe8\xa4G\xeaD@\xd6\xbc\xcb\xda\x81\\E\xe6\xe7\xa8\xa4\xed)H\x03\x13\xd9\x81I\x15EhB\xfd"#\xe6\x19\xe6\xba%\x93\xeb\xc8}Y\xc8%[\xbe\x1d\xd5\x86\t\xfc)\xd4m\xf7~\x87\x8e{\xb8\x94c\xec\xf4\x10\x82]x\xb8\x8dF\x8d\xa0\xbb\x135\xba\x0bF;\xee4\xdc\x90+G\x04\x80\xde\xf4|8\xe3\x1ar7\xb0\xc2\xe0\xc2&\x99\xfb\xfd9\n\xce\xec\xbb\xa5\xb1?e\xc4<\x17r\x88\x80\x11\x04yY\x822\xa7+\xee\xa0\xa8\xca\x0f\xbb\xbd\x96s\xd3=\xab8\'\xd5\x95\x0fR\xb0\x80&5h\xa5\xc1X\xd8\xdc\xb1\xc0|\xd2# \xdf]G\x1a\x1az\x97\xaef|Wrso\xd4\xe2\x85\x1c\xd2G\xfc\x90\xae\xce\xa3\xbe<\r\xbb\xef\xb4\x90\x7f\xcc\x88\xc1\x9f\x85i\xd73=\x81\xbd\xd4\x1a=\xc6Z$f\xa8\x98X\xeb\xf47\x1d\xb6F\xa6\xecA\\\xfb\xa3DW\x8419\xd9\x0c\xa3{kOLd\xa60\xb3\xc1:}DT\xcc\xb1\xd9\xc3!\x7fb\xd9\x0b\x1a\xd5\'\x07\x16%\xf8\x9aV^\x93\xf6\xba\xef\xeao\xd6\xdf\xfd\x98\x11\xb3\xaep\xcf\x13\x18\x89C/+\x1c\xab\xdd\xceG\x95\xb0\x0c\x8f\xe1\xe8\xa9\xa3\xc7\xae\x8c,\xe3|\x8d \xe3V%\xc2)\xba"\xa0rV/\xb7Z(R$u\x8f@\x98\xdb`\x9a\xc5\xbd\x96\x0b5F\xddl\xa6\xca\xc4\xe5p1u\x917G\x9fP}\xa0m\xdb\x02\x82\x81\x109\xd7W\xba\xb8\x06\'\xe9\x1a\x84i:\x1cx\xf5\xba\xe8o\xb6\xa8\x1fCb\x9ea\xae\r.\x8a\xbc\xce\x8b\x7fX\xb1E\x807\xf0\xb46`\x1di\x81\xe10\x87\xcd\x11F\t\xdf(">\x15\xa3\xb5\xe8\xdf\xd1\xb8\xc5\xb8\x05\xb7\xd4\n\x84s\xf3\x98\x88\x1cN\x9b\xa0j\x9f\xcf\xe3\x95\xe5\xfa\xdd\\\x95\x9crm\x10\xe5\x06\x160\xdc\xb2\xdf\xea8\xfc\x18\x12\xf3L\xa0\xc4Z \xa2\xaf\xd4N\x93\xb4\x8d\xf3\xa0\xc90\xd7\xee\xc9(]\x89\x9a\xbd\x96Bub$4\xed\x18\x12\x16Cq\xcc\x97S\xa3w\xc8Q\xaf\xb2+\xbf\xa7nZ\x82\x85\xe9\x80 \x9drb\xe5\x85\xd2\x1fXT`Wl\x80\x06\xa6]0ax\xd3\x87\xfa\x18\x12\xf3\x15&D\xc0\xd8\xeb\xc3B\xb4\xde\xe2\xc2L\x8c\xb7\xe0=\x10T8dw\xe3Y:N\xd4tmc\x1a\xdd]\xd4v\xb82e1\xc2\xae\x05#3\xb3\x9f\xc1\xfd8\xb5^\x0eQ\xb9\x17\x1b\x01Q\x07\x88H\xab;\xc9\x00\xd3\xd4\x841F\xe7\xdf\xcdp\x1fCb\x88\xbf\x130\x88\xae\xcd\x02\xf9\xb2\x9a\xce\xad\xf4\x00\x81:O\x8bm\xe5 \xe6\xce)|\x1es\x96\x12l\xb59\x96\x96}\xcb\x96\xf3\x85\xbe\x05~p\xcc\x13\xff@j\x93WRx\x8a\xdc\xb9\x07\xe2[\xe9\x03j]\\&5\x9d\x08\x05T&\x06\xad\x0c\x12\xe0]-\xf1SH\xcc\xd7+\rh\xed\xe0\xc0\x97\x1a\xeepp\xd0L\x8f\xbd|\xa6\xdc\xcb4\x9d\x9a\x9d{\xb9v7\x90\xdb\x1bdK\x8d\xb5I\x1a\xdeI\x89\xf7\x17\x04\xd9\xeb\xa0\xe3\x0c}\xdbM\xe6\xbd\xc6\xa7\x86g[gZ\xaf\x94\'\xee\xde\xc4cb-\x93\x80\xc9\x84R\xf6\xa3\xb7S\x7fq$\xe6\xb9\'`\x8cZ;\xc4\x97\x17C\xc2\xc8\\3\x95\xaa\xe0B\xb4\x96\xbc=\xde\x82n\xb4.\x9e\xbak\x01\xea\xc8\xd1N\xd2\x10\x12j\xefo\xe5\x9dhp\x0fn\xe2\xe8\xdaw\xfb&;\x1c\xd1 \xb7:Q\'\x82\xc5t"\n|\xa4Zn\xeb\xbb\xe1\x12\xf8o\x82\x1b\x1fSb\x9e\xe7\x04\xb8n\t\x02y\xd9\x13\xcc\x15\xc9\xf49\x92a\xbe\x99\x83\xbdO\x00\xa7\x96\xa5\xb3\xca>\xa5R\x1c\x9de\xc9\x122\xab\xf5;\xe5\x9er7\xd1\x97y\xd2/\x1a\x0b\xbc\x8b\x9d\x99\xd3;\xab\x16\x0f\'\xcc\xa4\x9b3\x04^\xcd9GD0\xb9\xc0\xf3\x9b\x95\xf0\xc7\x94\x985\xcc\xb5\xabA\x89\xf5\x9e\xbcT\xc2\xc5\xc3_\xd2\xc5\x1f\xbb\x9dK\xd6\t\x94\x1a\x99+\xa8\xe2\x8d\xe9\xcd\x8b6\xc3\x08\x05Dz|\x95\x0f\xe1\xa9\x83\x859\xdf\x1d\xeaE\'qK\xe8\x05\x96\xe2E\x10\xf5\x1cT\xbax\x01()N\x90\x81\x07\xddg/\xa77\xd9\xdb\x8f)1_\xc5\xcd\xf3\xed\x16\xfe\x92\xc8\x9d\xc5\xbeQVG?\xf4\xd9\xb7\xea\xe16\xdf]\xf1t\x96\xee\x16E^\xe7\xf5\xbf\xee\xc6%}GZ\x89\xa9\xe8\x01\xbe\xca\xd1 ^g\xf7\xcc;\x18\t\xa8Y\x99\xa5n\x1ekG ;)\x83\xea\x165\xb4>\xc2-\xfb\xa3S\xffg+1k"\x87(\xf0Y\xc0\xbf\xec\xcd\xb5O\x91\xe6\n\xde\xc7\xed\x11\xcc\xa3\xc9dg7\xe0\x15\x99\x80\xa6\xae\xc1\xb9\xa6\xd1#\x05\xf6l\xeeN\xdd\\\xf5\n)\x16tN\x16\x08\xa65\x06\xc2pW\x0b3+p\xf67\x9b\x00ZUu\xf7#\xde\xf3*\xdd\x0e\xa0|<\x1f]\x80A\x19\xe0Pp\x8f{\xa5+\x85\x00]\xd7\xc3\x91\xbd\xe4\x02\xe7\xf0\xc2AI(k/\xd2\xac\x94\xbd\xa9B~L\x89!\xfe\xfe\xfc&k\xdd\xfb\xaf\xc6\xa7\xa6\x92\xf1,F\xb7\x82\xc5\xa8\x80:\x1f\xe0\xfc\x88\xee\xd6\xae\xc5\x84\xcfmk\x86i\xc0_\xf1\x06\x1d\xcf\x1c\x98t.\x8d\xe6\xb7\xb9t\x14\xa5\xd9\xa9\xac\xe8\x14\xec\x03\xbc*\x03f`\x19;]h\xe5^\x99\xee\x99z0o\x9eW\x1fSb\xd6\xd5$\xe0\xb5&\xff\x03\x0c\xe7A\x1c\xa3\x19\xf4\xe3\x9e\x90\xfa\xc8h\xafk\xe1\x06\xef\xd0\x0e\x83\x0e\x82\x11\x02\xb8\x9a\x9b\xac\xbc\x16\xf3\'I\x0c\x02i\xdd\xf9=\xebO\x0f\xae\xe6l*O\x99\xce\xc0\xec\x06\xd8\xe7\xd4"\x00fHHzd:\xa7\x84{\xb3K\xfd\x98\x12\xf3\xac\xe1 \x98\xc4\xd6\x84\xf1\xfb0\x8f\x18\xd2\x13\xf3\xe5(,6FD\x95\'\xef\xd5L@3\xf3t\xf2\r\x97\xf3\xcfr\x1d\x86\x056\xf9\x8f\x94\xc9\x90\x1cQ&\x8c\xc8\xdb6b\x05\xac\x86\xee\xc4\xb2\xf07C\x97\xa5\xf8\xde5\xa8g*K \x9db\xf3\xcd\x8a\xfccJ\xcc\xd7K[\x18#\xe1\xd7\xef\x8d\xaa\x9ej\x1d3\xef\x0e]\xef\x07\xf6`]\xf3;\xa4v\xedd5|\xef\x08\xae~\x8e\x10+\x81\xe2\xd8\xba\x1ck\xb1ALH\xcf\xb5\xd3\x9a5\xd3f\t\xf61\xa2\xb6\x0c\xb3GP\xd4d{\xe0Ga\xfe\xc5\x95\x98\xe7\x9e@P\x10\xc6_\x8b\x1b\xae\xb6x\xfb2\xe4&\xf1\x80\x16\xb0\xec\x05+R\xa2+c\x13Zv\xbc\xb1@B\xd0\xde-" \x05\xa7\x86X<\xbb\x16\x9d\xea\xd8\x0c\x84\x1a\xb43\xef\xae\xe5\xee\xf8\xa3xrG`\xae\xc1p\x97-\xf2\xec\xa1\x14\xf7\x8fs\xe2\x9f)1\xff\xf8\xbb\xaaM\x89\xf9\x97\xff\\\xfd\xbfG\x89\xf9\x0bM\x90\xdd\x86\xf2nJ\xcc\xa6\xc4lJ\xcc\xa6\xc4lJ\xcc\xa6\xc4|\xe3\xeb\xdc\x94\x98M\x89\xd9\x94\x98M\x89\xf9\xce\xfa\xca\xa6\xc4lJ\xcc\xd6\x90~\xbb\x86tSb6%fSb6%fSb6%fSb\xbe\xf1unJ\xcc\xa6\xc4lJ\xcc\xa6\xc4|g}eSb6%fSb6%\xe6\xdb^\xe7\xa6\xc4lJ\xcc\xa6\xc4lJ\xccw\xd6W6%fSb6%fSb\xbe\xedunJ\xcc\xa6\xc4lJ\xcc\xa6\xc4|g}eSb6%fSb6%\xe6\xdb^\xe7\xa6\xc4lJ\xcc\x7f\xa7\x12S\xff&\xbb\xfcS\x0f\xa1\xf9\xed%}c%\xe6?\x10\xd8\xafQb\xfe\x03\x81\xfd\x1a%\xe6?\x10\xd8\xafQb\xde\x0c\xec\xdf1G~\x8d\x12\xf3\x1fX\xb1_\xa3\xc4\xfc\'\x92\xc7\xa6\xc4\xfc\x14%\x06\xfa\xbf{\xfe\xe7J\x0c\x0fQ\x02\x0e#(\x84\xa1\x18M \x10&\x08(M\xd20Br\x14B\xb3\x14\x8d3 I\x08,\xca14#\x08\x18\xc2\xac\xff\xb00M\xa2\x04I@_\xf3\xa8~\x0c \x80\x10M\xf0\x04\x03\xa2\x14\x8c\xc3\x1c+0\xdc\xfa\x0f\xc6q\x1c\xcc\xae\xbf\x1aBH\x1a\xc5@\x01\x84a\x90\xc6i\x18\'!\x98\x80@\x1e\x83Y\x96\xc5@\x96\xfc\xa9J\xcc\xff\x8e\xec\xf8\xdb\x1f\x0c`\x80\x90\xbfS\xc8s\x04\'AP\x7f\xa6\xc4|\x7fb\xe7\x0f\x94\x18\x12f \x90F\x18z\xbd@\x94CH\x90\x80\xa0\xe7\xef\xc1\xd1\xe7hD\x96#\x9f\x04\x06K\x80\x04\xb1\xae\t\xbb\x9eq\x18\x0eA\xb0\xc0\xa1\x04\xcb\x13\x90\xf0\x9cY\xb5)1?Q\x89Y?\xb6\xee\r\x02%i\x9a\x02i\x90B\x9e\x92\x07\x85\x13k\x96`8\x81\x86X\x96\xa01\x1a\xa10\x16\xa3\xd7\x8bgp\x98A\x9eST9\x1a\xe7@\x04\x87\xfe\xf6\x17Wb\xfee\xbb\xe4SJ\xcc\xd7`\x93?Ub\xbe\xff>\xff\xb5J\x0c\xf9c_\xa0\xa3\x822\xdb;\xe6#\x8b\xe16F\x1f g\xdc\x84\xf9\xc6\xd7\x07<\xd8\xf1\x84T9\xc4`\x8a\x12W\x8b.\x13T8#\xd6X\xef\x81\xd9\xe1|\x06/U\xd5\x95*\xd6Y\xe1\xa1\xb2\xfd\x82:\xf3zq>\xa27\xeeMZ\xe4SJ\xcc\xf3T\xc0I\x14{z\x05/\x13\xb9h\xae\x82\xb3\xf6@R\xa7\xdd\x92\x11M%9\xca\rL\x96f\x9f\xf0T2S\xc0\xd5a&rp\xcb\xd0\xf7\x9a\x06\xd1\xc9@]\x0b\xd4\xb4J\xe6\x07(\xb8\x06\xb6\x1bT\x0b\x12\xf5,\x18c\x05\xe5\xd7#\xb05\xa77GU}J\x89\x81\xd0\xbf\x83k\x92|Y\xc9\xa9\xe8n\xca\xec\xdd\x04\x9d\x0ek\xc8\x9c(\x13\xf3\'\xe5@Fay,\x9d\x13O\x86y\x89\xea\xba\xa9J\x8fz\x82\x07\x8d\xba\x97.\xd4\xf0ZfT\xa7\xf1\xe4 \xd2\xde\x03\xa5\xc7)\xb5\x1c\x06\x95\xf2\x08\x00\xc6jys\xa2\xea\xa7\x90\x98g\x88k\xe5\xb1\xfe\x00\xf1\xb2\x92\x01\xb7\x1c;{\x97\x19\xfa\xe3q\xc6\xfb\x93\x101K!\x0e\xbe\x96\xe8\xa4\x02\x0e\x87\xb3\xa9\xf0\r\xbcC\xefn\xff(\xc7\xf8\xd4\x82\xb8\x9dZ^\x98\xa5\xe7=\xc2\tz\xa9\xba\x10\xb4\xd0,\x00\xf9E\x94/\x19\xa0x{\xf3l\\*Y\t\x1f\xe5.\xd1\xd1e\xdc1\x86yE\x0c>\xfd^\xda\xd7\xa7\x90\x98\xf5.\xae\xb5\x1eF\xae\xb7\xfd\x05NR\xc6\xf4\x1a\xb1\t\xc5\x0crI\xe7L\xe0\'\x01\xaf\xc9Sa\xb4\xda\xbd\x99\xe2[X\xea\x02\x0c\xa6\x02\x02\xdfLzj\x19\xd95\xa8^KO\xc0.As\x0e\xf7\x98k\xa6\x1d0\x8d\x0b\x93\xf3\xe4\x1bh\xdf x\xf6\xe6p\xcaO!1\xcf0\xb1\xa7\xbe\xf2\x07p\x12\xa0\xb3\xde\xedz8\xacy\x0b\xd3\xed]\x11W\x98\xdfcbN\xecT\'=\x9e\xab\x1cN\xe9\xebR\xe6)/\xa4\x9e\xa8C\x15\xd5e\x01\x1cs)`\x89\x07>\xd3;\xad\x87FL\x9a\x15\x92\xbcf\xa3_q\x12\xf1\xc3S\xff\'#1_\xab\t\x11\xf0sr\xe7\xef\xc3L\x07awQ\xc3\xa3\x8b\xb4\x198\xa8g3\xbeq\xbc T#\n.\x18?H\x19LzNr9zg\x11f\xab\xfb\x01.y\xda\xa6w\xd0!\x8bA|\xe9\xb9\xc0\xdc\xe3\x99\x18\xcc\xdeXhu\x19\x9e\xfaz\xe9~\x84\x9a\xfd\xb5\x91\x98gf\xa1\x08\x08E\xb1\xd7\x12\xf1d\xdf\t\xc9s\\4\x1ez*\xc3\xa5c\xe1j2x\x08-\x82\xeb\xb2}\xc4\x9f\xebC\xa5\xd5\xfcC\r\xccy\xd2c:\x13\xc4\x08\xb5\xcb\xa3\xe1\xb4\x8b\x95\x8a\xc1\xc4\xf9\x95\xac\x9a\x19Zd\xad\x9a\x1c\xf5\x84\x89\xdc7\x13\xe8\xa7\x90\x98\xaf0\x9f@\xd4\xeb\xbc\xd6\xe8\xde\xad\x8d\x8a\x00r\xecYI.\xfbk\x99\xc7Q\x9e\x85\xd1@/\xcf\xb7\xe8l\x0cG\xd2]\xb0\x14\xe1"\xef\x03eB\xb1\xf8\xe1\xdf#\xf8\x88C\xc1\xd5-\xec\x81\xb9\x9a\x1cs\x12;\x92\x81\xce\x03\x87\xc8\xaeB\x80oF\xf9)#\xe6\x19\xe5\xba\x90\xd0z\xa4\xbe\x8c\x8c\xa6#/\xde\x9d[6\x92]\xfd\x96\x04\x95oJ\xc3\te\xd8 \xb4-\x87\xe7\x94\x1bZ\x85-\xbe\xb7s\xf0\x80Oi\x90\xfb0\xc2\x10\x8b\x87\xc2\x05\x13\x1ac\x8e`\x12\xb80\xfd\x18g\xf4\x82\x9bl@9\x84.\xbd\xab`}\xc8\x88\xf9:\rq\x14C\x91\xd7\x82|d\xbb\x06\xbdem\x19=\x0c\x97_\x82T\x92\xda\xb8\x87NV\x1d\x8f\xe6\xd8\xe6\xf9"Y\x81\xf2\x08\x17\xed\xa8bZ^\xb9t\xe5\xf0\xb2\xad\xd9pf\xc5\xd4i\x89\x9a\x07\x8d<(\xbb\t\xc99w\x1a.d\xd9\xfd7kk>d\xc4<\xef"\x81 \x08N\xa0/\xa7a`\xab\xa6r\x8c]\xdb_OC\xab\xf3`]\xdf\xd5w\xa0\x1c\x07\xfe\xc4V\x08%\xcf\x88/1:\x99\x89p\xa1\x9e\xab\xe3d\xcb\x9d$\x04c$\x12\x0c\xd0y\xa7In\xe7\xfb|Qy\x8a7\xd3\xc2\xf6\xb0\xfa\xe8\xbe9x\xfbSF\xcc\xd7\xdb\x0c\x10!\xd73\xe5\xe5a\x81\xcc\xb5\x9d!\xb4\xa1x\xe4hWJ\xb6\x87>\xf6\xe4#%.\x92]\xe7\xc5\x0eP"}\xea&\xe1\x81f\xdc\xe0\x84P\x1c\xfb:z\x01\x1f\xe4t\x18;\xc0\x86\xa4\xf6>tRi\xc3\xb6\xa4\xf2i*Z\xf1\x80 o\xee\x89O\x191\xcf\xb6\x06^;\xb7\xb5J|\ts\xb0[\xbe\xc8\xf2\xd2 \x97\xb2`b\x13\x81\xda\xb3t\xcb\xccK\x1e\xde\x8f\xc8\x1e O|z\\7\xe4u\xd2\x0eF\xc7\xcc}.\'#\x0c"\xb5%\x90s5P\xf7\x19oz^Q\xceTf\xcd<\x17\xecC\xff\x873\xf7\x7f\xb2\x11\xf3\xb5\xf5)\x08&\xff@7\xd9\xaf\xa7\xa0\x1e\xa1\x06\xd1{\xc0\xe4\xb9\xfe\xe8\xcb,0\x04\xaf\x8f\xca\xeb\xa9\xcfh\x8b\x7f3yE\x1eN\xa10\x99\x04e3\x8c\xe2\xb9\xf4\x1e\xdd\x03\x856(W@\x13/E?f \\-\xdc\xe9\x04\xf87\xba\xbc\x01\x87\xe1R\xddn\x04\xaf\xf4\xba\xb5\xaf\xe0,\xba\x03\x18y\xd7\xd4&\x83\xaf\xe2\x9b\x1e\xec\xa7\x8c\x98g\x98\x10\xb1\xe6q\xe8\xf5\xc5\r\x8c*\xa9b\x0e\xbd{\xe5.\rT\xa2\xfbb6u\xaf4{\xf5\xb3\xa91\xcf9\x9e\x1b\xb6/\xd1\x89\x1d\xe84xX\xab\x7f\xa82|U\xe3\x1e3\xb9\xe4\xf4)\x99wa\x98\x14\xbd\x9f\xdfciO\x8a\xc0\xa8\xe70\xfa\x8b\x88\x98\xafJ\x95$\x11\x8aD^*\xd5\xc8\xbdT\xe3u\xb7G\x98\x1bI\xb5>xmE\x8f\xdd3\x91u\xb0\xaf\x8c\xaa\xf7I\xc6\xf8\xf6\x1e%E\xf1\xee\t\xf8\xc5\r\xfd\x05\x9aG\x968G\xc7\xc3\xc2\xee\x88\x0ea\xd9\x983\x83\x01k\x8aA\x83\xf2j\xc0\xa0\xffN"\xe6+\xc1\x810\x81\x83\xd8K\x82\xbb\xc3l\xd9\x0f\xddU\xdf\'\x8a\x92\xb5\x85\xc7\x04\xc3\xc4\x9c1\x1e\x94\xd4=\x14&G"]\x12V%l\x01*\xf7\x02|\t\x82Z\x14\xeb\xde\xeb\x185\x8e;\xf6\xe8_l\xef\xe0)&u?\x02^\x87\x9d\n\xd60\xdft\xbd?E\xc4\xfc#\xc1a(\x0cB\xaf0\\\r\xe6\x9e@\xf4\xd7\x06\xc0Ke\xdfaT\xabq\xc6\x90\xd6\x864$"2\xee\xb1\x88\xf3\xa5\x117\xa6\x84\r\x1eZ|\xb9\x9f&\xde\x84}\xe4\x86\x97\xceR\x87\xa7\xbb\x7fn\x19\xf1N\xea\x8e\x1a_/\xe3~1\xd57_\xd9~\x8a\x88\xf9\xeaQ\x11\x84\x80@\xec\xa5\x15gM\x1d\x82\xee\xd9\xfe\xac<\xd8\xba\x9a\xda\x0bj\xe9\xf9-\x02c\xd9\x90\xcfI\xc0\xb3\xdc\xa4\xdc\x94\x8bu@\xc9\xea\xba<:\xb0\x89:\xccP+\xe1\xe1\xcb\xd7\xbb\xdf\xc1n\xecpE#;w=:\x9c\x1f\xc1\xfer|\xf7T\xfe\x14\x11\xf3\xccp\x08\x81!0L\xbd\x1cW\xe8\xec\xd5\x15\x02\xa7\x16\x8c\x9fQmN\xcfG\\\xa6\xef>s\x8b[.\xde\xfb\x14\xbc\\\x90\xf6\x0c\x9f`\x15\xbb\xc9\xd1dP\xe8\x89\x10c\xa82\x0f\x01w\x82\xaa\x01\xbf\x12q\x9eK-\r,\xb4\xa2\xb4\xf1p\x8c\xf57\xdfL\x7f\x8a\x88\xf9j\xaf`\x9cXK\xdc\x970+\x7fA\r\xf9\xeaX\xf1\xd8:UM\xf4 \xeb\xb6\x1e\xa2:\x8a\x97X\xe0\x0cY\x12\xdf\x86\x81\x98L\x10\n\xd7&\x0b\xe5Ir\x8f(\xb20/7"\xd1\xfbk\xed]$\x90\x07\xc9\x16>C;\x81O\x11\xa8;\xfdw\x121_\x87\xfe\xfa\xa8\xc0\xd0K\x8fZ\xa6M\xdat)80\x81\xbcv\xc0|\xc2f\xc7\xce\xb8\xe7\xe8A\xa5TZ\xcb\xc7;9,\x1dz5X\x89\x83\xf3\xfbM9/"\xa9\x9a\x11\n\xf4\xd3\x0e3\xefb&1)}\x9a\x1b\xf8\xd6k\xfc\x8c\x00E\x8b\xb1\xff\x9a\x10\xf3\xf5\x8coB\xcc\xbf\xfc\xa7\xea\xff=B\xcc_\x08\x8a\xd8\xec\x8dM\x88\xd9\x84\x98\xff?>\xa5\x9b\x10\xb3\t1\x9b\x10\xb3\t1\x9b\x10\xb3\t1\x9b\x10\xf3}\xafs\x13b6!f\x13b6!\xe6;\xcb+\x9b\x10\xb3\t1\x9b\x10\xb3\t1\xdf\xf6:7!f\x13b6!f\x13b\xbe\xb3\xbc\xb2\t1\x9b\x10\xb3\t1\x9b\x10\xf3m\xafs\x13b6!f\x13b6!\xe6;\xcb+\x9b\x10\xb3\t1\x9b\x10\xb3\t1\xdf\xf6:7!f\x13b6!f\x13b\xbe\xb3\xbc\xb2\t1\x9b\x10\xb3\t1\x9b\x10\xf3m\xafs\x13b6!f\x13b6!\xe6;\xcb+\xff\xd5B\xcco\x8a\xdf\x7ff!\xd4\xbf\xcbE\xdfY\x88\xf9\xf9\x81\xfd"!\xe6\xe7\x07\xf6\x8b\x84\x98\x9f\x1f\xd8/\x12b\xde\n\xec\xdf\xf2F~\x91\x10\xf3\xf3W\xec\x17\t1\xff\x81\xe4\xb1\t1?E\x88\x81\xff\xef\x9e\xff\xb9\x10C\x82 \x03\xd30\x84\xe0\x02J\x11\x0cAp\x10\x85\x92\xcf\x11\xec\x14/\xc00\x83\xb3$\xcb#\x18\n\x11 \x84!_\xffK\x820B\x0b\x10\xc8\xc1\x18\xf7\xd4-~\x8c\x1f k\x86DQ\x88\x16P\x1c\x82y\x06\x87(f\xad\x92h\x8c\x87\x04\x14a1\x96ah\x14z\xcevf \x88\xe59\x84\xc4Y\x8aGi\x96\xc7p\x18\xa18\xf6g\n1\xff;.\xeco\x7f0\x7f\x01\x06\xff\x8eR\x10I\xc2\xff\x18\xba\xf4# \xe6\xfb\xeb:\x7f\x00\xc4\x08\x1c\xcd\x11\x14H?\xc7\xceB\x84\x00\xd24\t\xaf\xd7\xc8\xa2\x0c\x0e\x124\xf3\x1c2\xc8S\x1c\x0c\xa3\x02E\xb0\x1c\x85"(\xf3\x04\x16 \x8a}N[\xfc\xda\x0e\x1b\x10\xf33\x81\x18\x8c\xc3\x08\x98\xe6\t\x1eE`\x84\x17\x10\x88\xa6`\xec9\x07\xfb93\x91CpJ`P\x8c\x03\xd7\x8f\xac7\x9d\xa4\x9e\xa3"\t\xe4)\xc5\xd0\x10F\x7f\r\xc0\xfcK\x031\xff2t\xf11 \xe6\x99\x17\xfe\x14\x88\xf9\xfe\xfb\xfc\x97\x021\x10\xfa\xe3\xa1\xfb\x92\x9f\xf3\xbe\x1e@n\x89:\x11\x0b\xb4\xe9\xa5\xa1\x04a\x0c\x9b\xd6.\xb9\xc1\xbf\x19uL\x02\x0f\xdf\xa2N=\x06>\x0el\xad\xe6\xb7\x9br\x19\xfde\x9f\xdd3\x86\x16\x1e\xfa\x1dfd\xce4\x86\x87O\x93\xe4<\xef\x0eo\xb2"\x9f\x02b\xd6C\x01[\xef=\x84\x92/x\n\xac\xf0P\xe5b\x87\xde\xbd\xabM\x0e\x1a;N$\x0fp6\x01I\xffH, \xa7\xcd\x13\x80\xf7\xbe\x0b\x843q\x9e\x00\xa7\x12yz<\xde\xd1\x18\x8e\xd2\\\xbe0&.M\x1aY\xc9sV\x1a\x0b$^q\xeb\xd0\xbd\x19\xe5\xa7|\x985J\x9c\\\x0f\xff\x97\x91c\xf3B\xd0\x9es\xf3\x86#\xd1_\xc2\xe2\xa8\xe1\xf6^\xd0s\x05m\xb9\xa3\xa9*\x8fa\xcd\x98\x16t.\xb8+\x89\xba\nJ\x94\x8fv\x1c\x80\x14 \xb3\xa0\xf4m\x0fJH\xfd&\xb0\xd2\x1e\x16\xeaG\xb9\xf7vx\xc5Uo\xceT\xfc\x14\x10\xb3\xc6\xf8\x9c9\x0b\xff\x01+\xb0OP\xfa2I\\\xdf\x10\x00e\xcfS\xc3z\x9a\x87\xc5lwl\x13\xc4\xe5\x88\x8e\xb0\xddi\x18t$8G\xf7QF{j@K\xeb\xae1{\xdc\xb9<\xae.NI\x9c\xab9\xb7\xb4\x8b\xe7\xc9\x06r\xe4b\xb9?\x1a\xc5\xf5\xd7\xf6a\x9e\xdb\x81\xc0\xc8\xb5|\x83_\x9e\x953~\x94\xa6\xdb\x80\xc4d\x85\xf0\xb7\x03$\xe5\x1apq\x05*}\x94\xc7\x86f\x8cs\x1a6Ly\x89\xb2\xac\xf2\x1f\x87\xc5\xb9\xd9\xbb\x13\xc8)\xcd\x08\\,s\r\x93\xb7\x1e\xd0\xe3D/\xbbk\xeaa\xf9\x81\xaa\xe1[\xf2\xe6p\xf1O\xf90\xcfR\x90\xa0\xd6\xaa\x81\xc0^\x92\xdbYB\xe1K\x1c\xf0Y\ti*5*\x84)\x96q\x9ds^\x93f\xcbyD\x818\xa2e\x95!\xac\xac\xb1D\xea\x92\x13xs\x92\x11\xce\x01l1Pt\xaf\xa1"\xb9\xe4\x8e\xce\xe3\xa6\xd0FQ\xf9@\x05\xc8\x0b\xfd\xa6\x99\xf4!\x1f\xe6\xb9\xed\tl-2\xb0\xd7\xb9m\xe5\x03\xefP\x82p0\xca\x1f\xdc\\#(\x0fH\x88)Q\xba\xba\xd0\xe9\xeb\rF\x021\xa9\x10\xab\x01w7\xd9f\xcf\xe7\x1e\xba\x1c\x81GN\xc4\xa1"\t8[vg7Sk\'=j\xba\xde\xb0\xf8\xc0\xe5\x1c\xf6f\x98\x9f\xf2a\x9e\x0f-\xba~n-\xca^\xc2$\xf8&e\xa8\x1c\x87\x17\xf7|\xa9K\x1c\xa7\xccP\xc4\xa1\xd4\x91=\x85\xad\x15w9\xcf\xa7\x1b\x11\xb3\'U\xb0Y\'\x83\x8b\x93\xd8\xca\xcd\x9aD\x06X9kC\xcf`\x84\x93\xfaEB\'\x9c\xc0\x1f\x12 \xb8\xc9\xc3/\xf2a\xbe\xf6&\x88\x93\x04\xfa\x9a\xc7\xd3FC=\xb1`X\xc5\x1a\xb9\xf8\x9e\xc3\xf5\xce\x87\x8e\\\x14\x9a\xf9\x11<\x8fEx\xc5\x99\xc5.\xfd\xa5\x94\x03\x03"\xef\x0bC\xe9-$\x17j\x02\x88\xfb\xf3p\x8b\xb9!\xb4N\x1c\xd9\x9cT\xda\xf2j\xbd>:\xef\xce\xc6\xfd\x94\x0f\xf3\xbb6\xed7\xb3q\xd5#\x14\xf5\xcd\x05\x1dZ\x9e\x91\xc7\x1c-BZi\x12\xdeK\xaa\xee*a\xb5\xc8\x8e%\x9a\xdb7O>\xbb\x89}^\xa8b\x7f\x97v\xc7\x8e\xd3j\x94*\r"\xa4\xaa\xa6d\xa5\xf4\xcc\xa6\xfc\xb2\x17\x01b}\x92\x7f4\xe8\xf8\xaf\xcd\xc3|\x1d\xf80\x88\xa08\xf22M\xd5c\x9c\x10\r\x05\xc6\x97u\x0fp;\xd9_\x96\xdd\x98\x0e\x01\xc3,\xc8A\xb156?^\x87{v;M\x03o\xa44\x14\xdf\x04\xba\xc1\xea^m@\x88\x1d\x18\x97\xcd\x12*M\xb0\x9cUzp\xdfS\xdc\xdd\x96\xa37G\xef~\x8a\x87yn\x89\xb5R\xc0(\xf2U1\x8c\xe4Ar\x1f\x0e\xd2\x94\xb8\x01\x1c \xad%.\xd8^\x93;n\xe4\r\x80\x8f\xfb\xe6\xcc\x1cd\xd7\xdfC\x89@\x06\x99\xb9\x00\xb7\xdc\x8e-1;\x0c\x9e\xbb\x08\xf1Z\xeb\x14\xbcqU\xf8\x90\x00\x1fB\xcf\x05g\xa8\x8e\xdf\xe6\xcc>\xc3\xc3<\xc3$\xa9\xb5\x80\x83_\x91\x08\xa9\x0f\xc3\x13)\xf5\x1e#\xb2\x0b\x8b\x89\x0b\x1c.\x8f\x9e\xf6\x17~\xf4\xc2\x1b\xc0\xfb\'\xbdj\xd39G\xd3<\xdf\x0f\x97t\x17\xc2\\,\x17\xce\xc5h.\x11\xee@\x89o\x1c\x19\xf22q\xc6\x9d`a\x15g\xce&\xf3\xa3S\xf9\xaf\xcd\xc3|\xdd\xc55%\xc1\xe0+\xb2#\x16\xb7\xe3\x0eHt\xdfr\x11\x1f3i\x10\xa3\x9488\xcb\xbc(A\xe6$\x83\xe6x\x02j\xb1\x87\xdc\xb5\x14=&\xaa*\x0c\xf1t\x89\x0bf\\O\xf8\x83 \x8fuv\xc7gD\xc7*1\xf1\x80\x0en\r\x9a\x16\xdf\xad\x83?\xc4\xc3|\x9d\x86\x18\x81\xaf\xb5\xf4\xcb\xe8]\xb5\x04\x06\x08\xe5\xea\xe8!\x80\x1d\x15\xd9\\h\xcbzB\x8b&`\xd0\x96\xbc6\xfbn\x1e\x1a\x04w^\xcc\x99\x81a#X\x8f\x14\xff\x84/\x1aX.\xb7hb\xbc\xfd\x12\x8d\xd8\xe2\'\xf1\xd5kr\xed\x84zw\xf4\xcdy\xd1\x9f\xf2a\x9ea\xe2\x18\xb8\x16\xe5\xaf\x13\x86)\xc3\xd1\xe9\x0b\xeb\xd6\xa6qh\xcfGJ\xf6\xfa$a\xb2\xdc\xbf\x8aF\xac\'.V\xc3\xf3\x8e[s\x8c\xb4\x13hSwL\x92(\x11\xc3K\xbd\xfa\xc8\x9d\x07)J/\x1d&\xb4\x14\xde\xcf{\xee\\:\xbeN,\xf9\xf7\x1a\xbd\xfb)\x1f\xe6\xeb.\x82(\x04c\xe0K\xbd\x9f/\x93\x84\x8e\xe7K\xa4W\xa0\xc5\x07W\x11\xea\xd5\xf5\x84\xec\xd0\x07C\xc6\xbcp\xbc\x07Z\xb2wE|8\xe2\xfc\xf9\x9e\x86\x0c\x92t\x99\x9c\xe2\x8f\x82\x02\x91\xf6z\x8a\xbcsy\xac.mxK\x8a\xd8\xbb\xb8FC\x06\xbf\xc8\x87\xf9*\x9dH\x92\x02a\xf2%\xccN\x16/#v\xab#h\x96\xa7C\xc9\x08\xacc\x1d\x16\xadQ!\x0b\x87\x1c\xe9\x9a\x8b\x8bs\x01\xe6\xb0\x9e\xaf\x91~\xd7\xef\xda\xfd\x11\x14\xcaYI\xc2\xabx=F=\x9f\x88\r\xee\x11\xfb{\xe6_\x07\xf2\xea\'\xb8R\xbf\xc9^~\xca\x87y\x167\xf8s\xbc\xf8\xda\n\xbe\x9c\xfa5\x02;\xa7\x06\x03|C\xc1\xef\x02f@Jr\x01N\xb4\xd1\x9b\xbd_OtGr]\x89\xe8\xad\xe3\x96\xf7\xa50I\xedz\xb0\x92\x1d\x11\xe7\x80;\xb1\x91\x9bZ\xb76Z\x08Ek1&\xb0/\x9c\xd4\x86\x8f\xcb/\xf2a\xbe\x1e\xda\xb5\xd8\'0\xf4%\xc3\t\xcd\xd2\x87\xedH3@\xbc\xf3\x1e\xc2ha\xc9\xae\xcc\xb3\x9a\xb6\xd7\x1e@b/\x962f\xa9\x17\x16\x04b\x1c\xaf\x054s~iW\x0f\xb8\xb9u\xd7\xb3\xa3+\xce\xbe\xd73F;\x86{}\xb6\xba\x1e\x12I\xf4\xee\xfc\xa8{\xfbk\xfb0\xcf=A\xaeG\n\xbev /PbH\x01N\rxX\x8d/D-\xe0\xe2$)g\x1bj\x1d\x922@\x89\xd2\xd6\xaem7\x89\xeeuN\xfd\x1d\xd0\x85\xf0x\xd7\xb1\t\xf5\x8f\xf7\x05\xed\xd2x_\x0e7\x1c\x1e<\xb0\x1c\xf2\xd4\x00l\x1b\x08\x97\x80\x7f\xb3\x07\xfe\x94\x0f\xf3U\t?w>\x8a\xbe4\x87{!\x82\x11\x85\xb8\xfa\x1c\x95\x8d\xfd\xee\xfe8\x1cO\x92\n6\xe4\xfa\xc4\xdco\xe6\x94\x00\xeaq\x18\xa3\xdc\xd8q\xb8E\x1d\xf9\x87{F\xdd\xbb\xe4\x9a\xdaY\xbe\xb1\xe7\xab\x96\xa9\xdeA\x99\x1e\x11N&m\xe3j\x08@\x87o\x16\xfc\x9f\xf2a\xbe^\xdc\xac\x05\xff\xba\xfd_\xac\x8d\xd4:\x9d\x8en\xe6\x08W\xce;VDU\xa1W\x85\xba\x1d\xca\xa0\x9a\x19\xe7R\x8a\xe3.\xba\xd8{:\xc5\xe8\x0ba\x082wp\xd7\xe8F=\x07\xadt\xf6\xb9k\xe3a\x81\xa3f\\6\x9fs\x8a3a\x9f\xe3q\xe7\xcdS\xffS>\xcc3\xc3\xc1\x10\xb6\xb6o\xaf\x14l\xbfV\xf2\x99pM\xebc\xe7\xa0|N7\xd6\xed\x86u\x90\xea\xa3\xde\xd9\x854Q\x82\xa9\nr\x08\x8d\x05!(\x159\'t\x07E\x08Xj?\x06Y\x92V\xbe\x03S\x16\x14tE-"\x0c\xd2\xa1B\x13\x00\xc2\x9b\x0f\xed\xa7\x80\x98\xaf7\x1a(\xb96A\xafv\x13v\x07\xab}j5\x9a\xd2/\xfb\x90-\x95G\x92j\x96\x102d\xb4\x97`n\xa1\xec\xddy\xe4\xa1\x84\x10x\x97\xa2\x8d\xf5\x01\x021\x85\xf0,\xfd\xb2k9w\x84\xa7\xbe\xbaEd\xe1DN\x9a\x03\xd7\xe3A\x9a\xd9\xe5\xcdD\xfe) \xe6\xb9\x9a \x84\xa3k6\x7fi<&g4a\xd4\x06\x85\x8bo\xc0\x81\xd61\xb2\xd3Yd\x84\xc4<~\xa1\xc7\xcah\xe0r\'\x04!\xcf\x1fJ\x94\x0cripu\x7f\x99B\xecv`\x98!\xd4O\x85\x03Jv\x8b\xeb\xf9x\xab\xf1\x13w}\xb4\x99\xfa\xbd\x12\xf9\xa7\x80\x98\xaf\xe3\x10!A\x04y}\xa5\xe1\xdc\x98cv\xf1O\xa9\x07\x06=\xcf\x1c\x03\xc9r\r\x89i\x0e\xe3Uq\x88\x18\n=\xff\x0etCZ\xc6\xf5U\x12\x11-\x87\x87\xc1\xe0\xb0\xcc\x1d\x0c\x92a\xe5\xa0I\xc6\x10\x0f\x11\xb6\x10\xc3\x87\xa0Hx\x08\xed\xd2w_M\x7f\x08\x88\xf9\xfaB\nF\xe0g\xc1\xfbB\x19"\x8eo\x9ewN\x92\xd3Kz9\x1b\\z\xc8\xf6,\x0f\xec\xcd\x94\xab\xc8>q\xf0\xd8\xeb=\x8a\xb0\x1f\xa6\x8e\x8fue\xaf\r#e\xc9s\x8c\n\xf5\xee:\r\x8b \x1d.\x0c\xf3\xd0\x8f\xdcrk.\xd7\x8e\xb0\xd47m\xa8O\x011\xcfok\xd6\xd2v\xdd\x13\xaf\xc5M\xae\xdd\xc6\x9a.\t\xe0\xecA\xe9\x99V\x13\xd7\x90\xcfi\xa9\x15\xf5\x08\x1d\x89b\x1a\x80^\xb6\xe0\x9c\xdd\xd74x\xa0%\x94\xb2Q\xfd\xa1\xbb\xed\xec\x18y0\x0f\x92D_\x03*/(ft\xe0\xfb8\xe1\xa9W\xa6o\xbe\xb3\xfd\x14\x10\xf3\xb5\x9a\x08\x8c\xa1\xe4k"\xa7\xcb\xf3\xb9\x01\x05b\x9aQ\xc3z\xe0\xd4\xf0\xd0\x8d}\xea\x86\x07\xae\xad\x13]E\xdbt\x19\xef)\x0c\xcebP\x95L\xe0\x02\xd6\xce\xb3*[\xdf\xd1\x8aq\xb9\x8a\xb5<\xe0\xa7\xd6\xf1){\xb2\xaf\x8a\x98\n\x001\x11o:8\x9f\x02b\x9eab\xebYE\xae\x87\xc0\xef\xc3\xe4\xae>\\8|\x16\x0f\x8a\xeb?\x8e\xfa\xb9Q\x97\xc6\x02\xaed\xc0\x12"k\\\x03\x1b,\xc2E\x10\xaf\xccm~X\xeb\xf1\xb8\xe8\x1d\xc6\x91\x86\xd8/\x8a\xb2\x9c\tl:Y\xb74?c{5\xa2\x804\xbe\xd1\x00\xf0\xbd\xbe|\xfb\x14\x10\xf3\xf5u\r\x88a\x18\x82\xbd\xd4p\xe1\xc9`\x06C\x94M\xd3\xad\xce\xfdM\roqI\\S\xe11\x02nfFx\xe7C\x92{\x06@\xe0fU\x83\xc2\xa4\x87\x8e\xf2\x91B\xe7\x1e\xf7+\nE\xbc\xec\x8a\xc6\xc9\x835(\xeb\x0fh\xde\xba\xa7AC\\\x86\xfe\x97\x84\x98\xaf\x12{\x13b\xfe\xe5?U\xff\xef\x11b\xfeB\xf6\xc6\xc6\x99l\xb7\xf4\xfb\xdf\xd2M\x88\xd9\x84\x98M\x88\xd9\x84\x98M\x88\xd9\x84\x98M\x88\xf9\xc6\xd7\xb9\t1\x9b\x10\xb3\t1\x9b\x10\xf3\x9d\xe5\x95M\x88\xd9\x84\x98M\x88\xd9\x84\x98o{\x9d\x9b\x10\xb3\t1\x9b\x10\xb3\t1\xdfY^\xd9\x84\x98M\x88\xd9\x84\x98M\x88\xf9\xb6\xd7\xb9\t1\x9b\x10\xb3\t1\x9b\x10\xf3\x9d\xe5\x95M\x88\xd9\x84\x98M\x88\xd9\x84\x98o{\x9d\x9b\x10\xb3\t1\x9b\x10\xb3\t1\xdfY^\xd9\x84\x98M\x88\xd9\x84\x98M\x88\xf9\xb6\xd7\xb9\t1\x9b\x10\xb3\t1\x9b\x10\xf3\x9d\xe5\x95M\x88\xd9\x84\x98w,\x84\xdfl\x85\x7fj!\xfc\xaeT\xfe\xceB\xcc\xcf\x0f\xec\x17\t1??\xb0_$\xc4\xfc\xfc\xc0~\x91\x10\xf3^`\xff\x8e7\xf2\x8b\x84\x98\x9f\xbfb\xbfH\x88\xf9\x0f$\x8fM\x88\xf9)B\x0c\xf2\x7f\xf7\xfc\xcf\x85\x18\x86\xe2\xa0\xa7\x02\xc2c\x02\x81\xd1\x04\x88\xe10\x8e\xd1\x08\xcb`\x08L\xa10\x87\xa2\x1cB\xc3\x0c\x87\x83\x04\xc5c8\n\xc3\x1c\x88\xf00\xc2\x08\xc8\xfa\xef\xc9?\xd7\x0f \x90B9\x92\x81a\x9ef\t\x88~\xce.\xc4\x05\x88\xa6H\x1e\xc2\x10\x82\x021\x8a\x06\x9f\xd8\x05\nB<\xc2" O@\x08\xf4\x1cl\r\xd2\x0c\x0c"\xc8\xcf\x14b\x9ew\xe8\x87B\x0c\xf1w\x14\xa6 \x08A\xc9?\x13b\xbe\xbf\xae\xf3\x07B\x0c\x8c2\x10\x07\xad\xbf\x12\xc5H\x1cgq\x06\xe1\xf8uy \x10\x13\x04\x92\x82\x11\x9cei\x01#`\x96\xc0\x18\xee9\xc7\x82dp\x01GqX \t\x9c\xfc\xe246!\xe6\'\n1\x14\x05"\x04\x0c\xd2\x04\xcb\xae\xcf\x14B\xae\xb7\x8f\x05\xd7\x1f%8\x8c\xc3\x85u71\x02D\xd0\x1c\r#0O\xa08\xcd\xa0\xc2\xfa\xf8B4\xc6\xf1$D\xf2\xc8\xdf\xfe\xe2B\xcc\xbf\xec\x96|L\x88y\xce5\xf9S!\xe6\xfb\xef\xf3_*\xc4\xc0\xd0\x0f\x07\xb5\xdbe\xde\xda\xe0\x14\x9e\xba\x9b{\x19Xs"0\xc1m\x1f\x01d\x92\x8b"V\xd7[B"\x14\x8a.\xca\\\xde\xf7\x81}<\x06-\xee\xd7h]:h\xe7\x9a\x87SF\xd9^m\xee\x0e\x9e\xcc\xb1~\xcb\xd9y}\xa7\xdf\x9c\xbd\xfd1 f=\x13\x08x\xfdq\xec%J+\xd7:\xa6\x16!\xfc!(\x0bY\x8d\xe0}Z\xdb\xb9:\xaaM\x90\xb0\xdb\x9b\x89I\xce\xc1\x02\x8b\xf6\xdc\xf6\xfb\xbd\xd7\x9a\x08g\xa2\xc9\\\x9cZc\xb73\xe1\xde8\x9e\x1ey\x14\xdc\xe6\xce\x15\x92\xfc\xd6\x187\x12\xa4\x98_\x04\xc4\x10OB\x01\x03_Fd\xaa\x93_e\x8c\xce\\`\xfd\xeaR\xb6E\x83\xbb\xa2\xaa\x18\xa5\x9a\xd6gd\xd2%~\xd2-\x19\xd8UD\x11#\xd3c\xcc@\xa2\xf7\xdb\xbb\x1e\xec\xc4Jc\x9b\\.\t\xb8\xb0\xe6\xe9\x92F\xa7\x0b\x8e>v\x11qK\xdf\x1c\xaa\xf81 f]\xc9\xe7\xc4h\x0c{}`\xbb\xd812\xf8\x12\xdcu\xd7\xaa\xf6\x87\x9c\xe6\xc6\x0c&}\xc2s\xa7I/\x99\xc5Q\x8e\x96\xd4$\xd2\xd9DNKn\x11\xd1QL$D\xef\x8ab\\\xcc\x93\xab\x9d\x87{m\xd4q\r\x8d\xd3#\xf5$1(\xb1\xc3\x8f\x1e\xd8\xbf\xb8\x10\xf3\xdc\x0f8\xb6f\x8c\xd7\xc1m\x88\xbd\xf7M?\xbe\xb8}O\x8c\x041\x15\xf7l\xeaniI\x10\xd95SI\x17\xb5E-yp6\x84\x08\x03\x0f\x88\xec}\xdf\xc6\x10W\xa7\xc7\x86\x19\xef\xa3`+\x99\xc1\x0c\xb2c\xe8E\xe6\x9fl\\\xc0\xca\x0b\xfd\xe6\x8c\xe1\x8f\t1k\x98\x18N"\x08\xfc:,\x1e\x15\xf5\xf8\x126\xfb;\x82\x0e\x1a\x1a\x1f\xb0\xbb\x8f\xe4\xca\xae\xc4\xb3s\xb0(\'\xd1\x94\xb2\xb8\xd3\x18\xe3^0\x03\xa16\x0e\xe6\xed0\xa8\xb6Q\x85\x0f\xef\xe3%\xb3\x87\x10\x8f\xea$0\xcc[\xedP\xa0\xcf\xa8\xbc\x8e\xbe9\x86\xefcB\xcco+\xde\xdf\xcc\x19M}9\t\x89\x83\xab\x131gA\xb2\xeb\x16\xc8-Y"\xabJ\xec\x8e\x85\xe0\xa0\xb7\x02\xbbM\xc54(\xd9\x1d\x16<\x04\x89\xba7\x1e%t\x91\x1c\x90w\xe9p,\x1e\xf0\xee\xc6\x08\x846\xcd\xf1\x9d\xd8\x01\xcc\x1d{s<\xdd\xc7\x80\x98\xe7b\xae\x95.B\xbc\x8e\xff\xd6\x1ey\xcb\x80W\x11p<\x83iyK\xb9\xd4\xb6\xd9\t\x87\xe1\x18\xec\x0ewG=\x18\xc9-g\xeax\xa6\x93\x0c\x07\xf3\xbbD8\xa3}\n\xaa\x1a\xb7\x89\xf5s\xa9>\xd8\x07\x86\xb0H\xef\x06\x1e\xc7KP\xe4\xc7J~[\xc0\xfa\x10\x10\xf3\x15&\x02Q \xf82\xf9\x7f\x0e\x87\x9c\x9a|\xf8>\xc2\x11\x1fT\xd8\x95\xb7\xcc{\xcc\xc0a\x9c\x1d\x00\xa79\xf6\xe6R\xf0\xfb\xf3=\xb9\x88GE\xb8]\xee*\xa6\xef<\x80V\x05\xd2$\xb0V\x0e\x15\xbd`AqX\x97\xc0\xf0\xb9Q\x08\xad\xa2zw.\xf6\xa7\x80\x98\xaf\x13\x19\x01a\xf2u\xa2\xf3\x02\x05$C\x9e\xc7\xb08=v\x12w\xea\xfc\x05\xa2\xf2SV\xd2\x1a\xa8/a0\xd9\xcd\x0c\x00B\xae\x9c3\xc9\xe0\xbb\xdb>\xcf\xfc\xb3\xa9\xb1\x8f\x1a\xd0\x0c\xa5\xf7n\xb8K\x04 P\x9e\x06=\x93\xe8\xeaj\xdf(\xea{\x8dS\xfd\x98\x10\xb3\x9e\xf8 L\xac5\xe8+\x9d\xb2\xe3\x90E\x06\xa4 v#\xa6\xcaY@\xca\xbc\x91S\x96K\xb8\x94\x81\xack&#>r\xee\x88@f\x02\x0f]\x0e\x8da\xe4y\x12\x0c\xce\x86\xd4\xdc\xf2\xc0=\xeb\xce9\xd2\x8dQ\x9c\x10\xa9\x8a\x91\x12\xe7\x80\xe3\xfcf\xf9\xf61!\x86\xf8;\x82#\xe8\xb3\nz\xc9\xe3z\xf6\xb8\xe7\xe8\r\xe5\xfc\x08\xc4\xfaZ{`l3\xcc\x8b\x96] \xca\x82\x1d\xf1t\x93\x90%D}\x80\xb5\xe1p\xad?\xe2\xd1\x9c\xe7C/\xd2auQ\xd3\x1d\xcb\x85\x99\xde\xb8:\xa7IQh\xcf.\xb2c\xf3\xf1G\x9c\xd0\xcf\x16b\xd6=\x81C\xeb=\xc3\xf0\x97\xb9\xd8\xae,E\x8f\x1d\xb8\x87\x83\x1b#\xcfpL\x1c\x8aB\x0ey\xa4\xc4\xee%"SACb\xee@\x06K^9\x81\xcc\xd4\x93\xacy\xf5|x\xb3\xd8\xff\x98\x10\xb3n\xfd5g\x10\x04\xf1\xea\'`IU\x8b\xba\xd3\xc5\xc4\x11\n\x8dA\xa3\xe3\xcb$\x04\xe2\xe0j\\\x1d\x1d\xf0][\xf0\xceQ\xbdK\x8c\xbc6\xa7\xe5.\x12%\xc81\xc8\xc4\xbaEk\x05\xb0WO\x01\x15k\x89\x93\xde\x8b\n\xe3v\x85\xeew]\xef\xbd9]\xfccB\xcc\xba\x9a(\x0c\xe1\xcf\xbd\xf5\x92\xe1\xe2\xac5\xcf\xc4$\xed\xfc\xaa\x1fn\x9e\xc8\xefk\x83#\xe8\x021\xe0\xbe\xb4}\xfcv@\x80\x9e3\xbc\xae\x96`\x15\xe0\xc5\x1a$\x81\x83\xb64\xa7\xf86*\x06\xcf^w\\i\x07\xa0\xa4\x83q+,\x8e+k\xf5\x9b\xb6\xe7\xc7\x84\x98\xafS\x1f]\xdbp\xe2\xe5\xd4\xd7r\x11\x82L\xab\xd8w\x90\xa9+\xe1\x18\xd7\xb0e\x86\xa6\xdf\xb6\xb8e\x16\xb6\xb4h\xddZ\x9f\x1d\xbbTb-}g\x8c\xd7\xa8\xf2\xe8\xc8\xc4\x8dZ\x12bW\xd4\xcb\xab#\xf8<\x8a\x8f\x0c\xbbt5w\x8e\x16F\xf9Q\xa5\xfa\x17\x17b\x9eE\x05\xf4\x9c\xed\xfe:\xa3\x1d\xdb\x8f\xe4A\xa6\xf8\x10\xc1\xab\x0e\x98k>r\xf4c\x7f*\x1e\xd0\x1e\xbc\xc8\x97JvM\x16\x8er\x99\xef,\xcc\x9b\'\x11_\xda\xa5\xb29\x88vCC\xd9\x19\xfd\xb0\xbfC\xe2d\xc9\x12R\xf34\x1b^\xf1\xf4\xc0\xbd9]\xfccB\xccz\x1cR\x14\x04\x12\x08\x04\xfd>\xcc\x83,\xdd\xaf\xd8-\xf2\xfa\xd3\xf9\xa4\xa0y\x10t\xb2\x1e\x9e=b\x89&O\xd1\x92\\Je"\x92\x90\xc4\xa6\xa5i\x11!\x89\xb0\xb3\x86\x07\xf0\xecX\xba\x17\x8d\xc6\xa7\x85\\OS\xa8\xd4)d\x19E\xad\xdauS\xf2f\x86\xfb\x98\x10\xb3f\xb8\xe7\x10\xe65\x95\xbf4\xa9\x11e5\xe5C\xe91j\xb2\x14\xa0\xbb\xf5\xc6e\xe4\xa0\xfe\xc2v\x0f\xda"i \xb1\xcf\xddU\xcc\x94\x83(\xfbG\xd5\xa1\xbc~2\x00T\xa9w6O."y\xe2\xed\xb8:\x14\xc4\xfe`\x95\x88\xb0\xc3\xfa\x0c#\xc77\xfb\x9a\x8f\t1\xcf\xadOP\x10\x8c\xbfj\x18c\x8dE\x13!5\x97\xab\x1cr\x1d\xb7T@\xa5\x8f7\xc9o\xcfK\x81\xfa\x17\x00\x8c\'\xae\xf1\xbd\xa4 v\'&\x9a\x14\x16\xa7\xf9\xe5\x88\x98\xacr\xd2g\xc9[\xc8\x94\x1ey\xcc8P\xe1\xc9\x9f\x98\xa5f\xe3&\xfe^V\xe2\xc7\x84\x98\xe7]\xc4\xc85\x0f\xbf\xb6MU\xaeuUm-\xa7\xee\xc6-\xa0g\x1a\xfb\xca\x9a\xcc\xde\xba`w\xab\x86\xa7\xb3\xc5L\xe6\x81\xc0hS\'sr\x85`Dy@\xf6\x88I\xae\x07\x1b\x95\x80\xa9\xde\xef\xd6\x02\xfc\x00\xf4\x98\xe9\xb2\xaed\xe1\x13\x17\xcc\xd7\x90igs\x8f\x84<\x11!x{\xad\xa7\xfa\x9e\xd2\xb6\xd8\xed\x827\xc3\xfc\x98\x10\xf3\x15&\xb8\xe6I\xe4\xa5;4\xd8\xba?\xb3\xdd\xa9\x1c\xf5:\xdf{\x15\xf0x\xc0^\x19\xf9\r\x1b\xb13#\xd0\xa2\xc7\t\xe9^\xe4f\xa3LjYT\x9d\xe3\xdc>B\xc7v\x1d\x11\xac\xb2\xfb|\xc1\x06\x13\xa1\x95\xbb4\x9ef\xbd\xc9\x86\x1aj\xc07\xa9\xaf\x8f\t1\xcf0It\xfd\x15\xc4K\x98\x82\xd2\xb9\x88}h\xe2\xa3\x91\xedP\x8e\xb6\xce\r-$\xe5\x19F\xefk\r\xc7\xe1\xb6]!\xa8\xe8\x1b\xe4\xf9q\x9eR\xcf\x86!]\x8a9\xb0\x9fl\xc5\x90\xa5\xdbC\xef\xcf~\x9f\xb2\xed\x9e\xc6\x93\xcc[\x12OB\x847\x13\xf9\xc7\x84\x98g"\xc7 \x04E\xa9\x97.5$\x8acI\xfb\x14\xd59\xde\xd3\xac\xec\xdd\xcb\xda\xf7\xea\xc69k\x02M`p\x8b\xd1\xf49A,\x83KAA;\xf0\x8fkB\xb8&\xd3\x9e&N>\xd8KF<$y2\r[\x914\xdb\xa6\xbc\x88\xe4\x1e\xfa\x9b\xac\xd1\xc7\x84\x98\xaff|m\xea\xc1W\xbd\xa9w\xf9\x86\x0cDP\x9d#w\x8f\x91 t\xda\x1f\xcf\xb7ib\xef\n-j\xd7\x12\x8c\xecy\xe1d\xba1\xd5t\x92\x8c\xd3\xd8k\xbcQ\xe0\xbaC\xa9\xb6\x87+>_\x14]q\x82\x87\x03\xe7\x9d\xd6\x9eE\xe81P\xfa\xd1C\xfb\x17\x17b\x9e\x89\x1c|"X\xaf/\x86\xec\xa1\xc2\x94\xfd\x83\xb9\xd9\x85Ehj\x1e\xba;\xfe\xc26b\xfeH\xf2\xc3\xc1\x8cO\xbbS\xc2\xc0Z\x1b?\xca\x9a\xe0+\xa7df>\x16\x8f{>0<\x1c\x14\xeb\x94\x06 \xc5}\xdc\xa6\xab]\xc9 \xbf\xcf\xae \xf9\xe6\xfb\xaf\x8f\t1k\x98\xeb3\x07\xaf\xc7\xc2\xabhfx\xbbG\xe9\x07\xd5u\x86\xfb$\xcb\xb9`t\xb8\xa0\xd2\xfa\xe2\xecW^\xdd\x8c3\xa7\xa9\x97~\xd7\x96\xf9\xadG\x01"\x9b4\x9cS\xf7\xa6\xd0\xe2\xb1\xf2\xa4\xb3\xd0Z\x8aQ\xca3\xe9\xe5L\x9d\xcf\x9a\xe9\x18\xd6\xaf\x12b\x9e{\x02!\xe1\xb5\x0e|)n\x88\x11Hu\xc6S\xdd\x19\xef\x1d\xd0\xb2N\x87sT\x8f\xaer\xac$\x1d\xd3\xf4L\x06S\x84\'t\xbb;\xaa\rB\xa4\xa1\xe1\x1c\xc4\xd6\x90\xd8e1\t\xff\xa4^\xf3\xbb\x1ci\x8c\xb0\xb0\x9a\xa0#d\xec\x8d\xe4\x03{\xf3\xcd\xcd\xc7\x84\x98\xe7\x9b\x9b\xf5\x91\x05q\xf4\xe5X\xee\x1e\xf7\x07t\xb87P_\xf4\xe2\xcd"\x96Q\xa2,^E!j\xdcKc{\xdf\xef\x81\xc7\xa5<\xc0\xf4\x84\x15\xf9r\x0b\xb5\xa3\xbbk\x03\xe8`T\xa6}\xb9-\xd4\xad\xc0\xe4\xbd\x02\xde\x88"\xbf\xb3\\\x1e\x07\x05\xf8x\xf3\xcb\xb7\x8f\t1_a\xae\xa5\x0b\t\xbe\xf4W\xe0Z{5\x07X\x1b\x88\xd3.\xa0\xb8p\xea\xb0:\n\x1e\x8a\x01Ap\xd9\xe9\x8c\x7f\xefm\x18\xf3\x8f\xbb\xc4\xa9\xfc\x81<\xd4\x97P\xf5\xed\xd3\x8e\xe7\x1b\xed&\x01.\x0c4\x83\xb9#\xe0\x88\xd3\xe8u\x0376\xbeP\xf4\xb7\xeaR?&\xc4<\xdf\xdc\x80$\xb4\xb6\xba/\x0fKU\xfah\x19\xcdm!\x13\x9d\x16\x8e\xe0=)\xf6\xc7t\x19A\x9f+\'E\x97 \x95\x04k\xa8\x13#Z\xbc\x91Hs\x9a\'m\x075.\xda\x18\xf8\xf5z4`]S\xc1\x8b\x9b;\x9cZ\xc5\x9a\xa7\x06\xec\x80\xc9\xdc\xbf&\xc4|\xfdM\xd5&\xc4\xfc\xcb\x7f\xaa\xfe_$\xc4\xfc\x85T\x83\r\x8a\xd8\x84\x98M\x88\xd9\x84\x98m\xe3oB\xcc&\xc4|\xdf\xeb\xdc\x84\x98M\x88\xd9\x84\x98M\x88\xf9\xce\xf2\xca&\xc4lB\xcc&\xc4lB\xcc\xb7\xbd\xceM\x88\xd9\x84\x98M\x88\xd9\x84\x98\xef,\xaflB\xcc&\xc4lB\xcc&\xc4|\xdb\xeb\xdc\x84\x98M\x88\xd9\x84\x98M\x88\xf9\xce\xf2\xca&\xc4lB\xcc&\xc4lB\xcc\xb7\xbd\xceM\x88\xd9\x84\x98M\x88\xd9\x84\x98\xef,\xaflB\xcc&\xc4lB\xcc&\xc4|\xdb\xeb\xdc\x84\x98M\x88\xd9\x84\x98M\x88\xf9\xce\xf2\xca&\xc4lB\xcc&\xc4lB\xcc\xb7\xbd\xce\x7f\xcbB\xf8M\xc2\xfe\xa7\x16\xc2\xef6\xcew\x16b~~`\xbfH\x88\xf9\xf9\x81\xfd"!\xe6\xe7\x07\xf6\x8b\x84\x98\xf7\x02\xfbw\xbc\x91_$\xc4\xfc\xfc\x15\xfbEB\xcc\x7f ylB\xccO\x11b\xd0\xff\xbb\xe7\x7f.\xc4\x90\xcf\xd9\xa4\x14\x01\xa14\xcds\x18\xc9\xc2\xcf!\xda \x06#\x0c\r\xe2\xc8J0\xee`SV\xee\xac\x81k\xf7\x90\xed\xad\x16\x04\xe0\x99~s\xc6\xf8\x87\x8c\x985L\xea9\x89\xf1e<\xee\xbctR\x99A\xd7\x16+\xe33\xac?\\\xdc\x8f\x18\xe4\xa8\x8f\xd7\xc5^l8| 1\xc6\xd4d\x13\xa5!]\x8e\xbe\xaa\xe8\xd5\x0e3\x08U\xa94\x80\xe4\xeb\xc7\xa1\xe5\xef\xb7&\xd5kI\x94\x0eh\xf6\xc8\xcf\xfa\x9b\xd3\xe2?E\xc4<\x8fw\x8a\\OA\x12y\x19\x05J\xc6\x1c\xe7\xa4\xae\xb6\xf0\x83\xad\x82\x0bN.\xf0T\x1a;I]\xcb dl\xf2Ls\x8b\xfd\xe9r\x80\x15\xa0"\xb8\x0e\x84\xf4;7\xd0\x8c2{\x91`^4\r,\xdd\x99\x0b\x19\x11\xbb\xeb\xeb\xda\xda(\xc5\xfa\xf2\xf7\x9aR\xf5)"\xe6\xb9\x1fH\x08Z\x13\x13\xf62\xb9M\xd7v\xc1\x9d\xc9l\x17\xc4\xe5\xb8\x1f\xf6\x9a\xdb\xc1o\xbaF\x9f2b\xbe\xc2\\\x8bM\x10y9\xaa\xac\xe9\xec+\x04\x05\xdc\xbd\x0b\xeb\xcewS\xbc\xdaH\xd5\x8bN\xd3\x94\x06[^\x05\x10&\xf7\x16\x95\xc3w\xa4\x9f8o\xa8j\xc3M\x00Hne5/:\xd6\x1c\x1c\xf88\xeb\xfc\x1e\x0f\xa7\xac\x1d\xd0\xcc\xbe\x86\x89\xfaf\x94\x9f"b~\xdf\xa6\xfd\x7f\xc3\x04t\xdb\x070\xbb\xa6U\x10\x88*\x02\rL\xb3\xe9;\xe0\x9aK\xeb\xc1{\xae\xcfv\x95\x97M\xc4\xc9\xb3J\xd2\xa5\x8f\xe1C\x06\x07\xf0}\x89\xb4\xda\xf3}\x08\xa1J\xedHK\n\xa7a\xc6p\xc0\xaf\x9e\x89\x9fZ\xfbG\xa3\xff\xff\xdaD\xccs\xe7\xa3k[\x8f\xc1\xaf[\xa2\xd4\xca%f\x07W\x14\x1ei\x1f\xedL\x9a \x80\xe5fY\xc3\xdeT\xf1}\xb1\\"\x14\x03"\x89\x8bd}\x8c\xe9\x1e\xc2Z\x9c)\x80\xd6\xc6\xf6-\xe7\t\x97(B\xb2"\x1c\x14\x8c\xbc\xc8\xcb\xe0\xe5\xbd\x82\xed\xa47\xeb\x9aO\x111\xcf-A<\x89\x98?\xd0\xa1\xc6|pDj\x84\xe7\xf6\x10V\xb4\xdc\xf2\x1d\x0e\xa7L\xe2;\xec\xb9\xc11w^\x8f\xaa\xfd\xd5\xd5\xfc\xeeQ\x00\x85\x90\x0e\xce\xd2+\x8a\x7f\xec\xab\x87\x19\x84\x8ev\xec\x96$\xbb\xedmd\xa6\\^mk\nnj\xe2M\x1d\xeaSD\xcc\xd7\xa9\x8c\xad}%\n\xbe\xe4q\xcc3\x946>B\x8a\xd7\xe4\x8acA\xbb\x0bW\x93y\x19X\xe4L\xef\xfd\x13e\xa8\x1a\x13\xab\xa7\x1a\x0b\xb5\x82\x10\xd5.eB\x92v<\x85\xa2\xbc\xd4\x91-\x88A\xf6\xdd\x9e\x85\xaf\xa8u\x85u\xd1#,fI\xbe\x99\x13\xf1!"\xe6\xeb4D\x89\xb5\xb7\xc3_\xf6\x84k=j*e\xf8\x92I\xd3\xb4U0=\xa7\'3\x92\x9c\xd8\n s\x1c\xe2r/a zy\x14\x18`\x9d\xb2\x06~hrk,BN\xe3\n\xb8s\xe2\xb9\xbe\xebQ\xc4$\\\x03\x1a\x805\xf5\xf6%_\x8c7\xc7\x8b\x7f\x8a\x88y\x86\t\xa1\xe4\xb3\x16~)\x84Y\x7f\xaeK\x80\xac\xf3)\xf7N>\xa9H!\x80Ksi\x04Hd\x02\xb5\xbc\xde\xc4\xd6\xda\x01\xcd\xfd\x01w\xbdpD)\xc9\x98\x99\x9aJ\xce\xc3\xd9\x0cn\xdce\xb64\x97\xcf\x84\x9c\xd3\xa3 \xc4\xd3\x83J\xca:\xfe\xe6\xd6\xff\x14\x11\xf3\xdc\x13k\x8cke\x83\xbc\x94pMiN\xa0Y\xe3\x97\x00\t\x9b\x87f\xc7W\x11;\x82D4\x9aZp\x05G\xb1&\x8a\xb3p\xa8=\n\x84+\x05\xb9ke\x86\xb2\xf6\xc1\xbf\xee\xf2\xab\r;<\xb9Ct\xfct\xf2wQ\x82\xab\xb6\n\x9c]\x0f.\xdf\x1c\xbe\xfb)"\xe6\xeb8\\\x7f\x16%\xb0W"\xc6\xdb\xd9\xb0G0\xbaX\x8aW\xb2\xe2)\xceWzy P\xc0\xe8\x15 \xe6\x1d\'S\xb0\x8b\xad\xea\x86\x1a\x15\x0e\xd1k$\x86\xf4^\x1e\xd0\xb3\xc8\xf0\xf4n\x88\x15"w\xaa\xce\x9c8\xda\n\x89k`\xd0\xf3\xf4\xcd\xe4\xcb\x0f\x111_{\x02B\xd6\xcbC^u\xa8\x04Y\x9aA\x95t\x1d\x02t\xdd\xec\xce\x85/\xc5\xa2\x12\x90%\xe3\xb0\xc2\x05\xb7\x826\x0c\xa2Q\xcd\xc0\x00\x8f\x8er\x90 \x0f\x8c\xe7k\xf2f\x16\xc9\x05p\xc2\xc8\xc0!1PB\xcd\xbf\tU"\x1d\x99BD\xde\x9c\xbb\xfd)"\xe6\xeba\x81\xf0g\x83\xf4r\x1c^\x05\xda\xaane\x80\x1d\xcc4\x05\x85~\xbf~\xc8 Np\x91\xd6\xbc\x93\xc7ci\x85\xcdan\xa5D6\xb5\xa5\xbb\xed\xbd\xfbC\xba\x95\xa8b\xf0p\xc4\xa9\x83=\xc5-\x11\xca\xfc\xddh\x18go\xdc@\x8c\xde\xef\xde<\x0e?E\xc4|\xbd\xcd A\x04\x02_\xe7n\xfb-\\\xe9W\x05\x1a\xeaeYK\xfe\nB\x10t]\xc89\xc8\x1aH&)$\xd0\xeb\x04\x05!`Yk\xe1\xf6N.\x97\xc2$\x9b\xfax#k\xbc\xcd\xa5\x87\xc3p\xb5\xbb7\xc6)+\\w\xcaO\xe3\xa9E\xea7\xb7\xfe\xa7\x88\x98g\x86\x03)|\x8d\x80x9\xf5wx\xeb\xa5\x1d\x02\x1cd\x91A\x0f\xd1A\x1d\x12B\x14\x93\xdd\t\x0c\x1f\xc8\xc18\x89\x1d\xb0`\xbd\x11z\x83\x83\xcf9A\x96\x9c?\x1c\xadRW\xbc0\xab\x8bks\x85\x8e\xc9~\x7f\xd9%\xe6~t\x81\xcb}\x7f}<~\xd4\xbd\xfd\xb5\x89\x98\xe7\x9eXS\xc6\xda\x1c\x82/\x9e\x10\x1bDxtb1s"N\xc2\x03*+nx\xf0R\xdf\xa8\x05!y\x87\xe3#\t\xe2\xf5?\xc5\x93cz\xf1C\xbd\xe4\x9aS\xda\x17\xc3\x81\x8b)\xd1-v\xb7\xa28\xa5\x92Mt\x9e\x01@HZ\xae\xf9\xbc*\xd27\xdb\xa6O\x111_\xcd\xe1ZK\xae]\xf0\xcb91\xda\xeaz\x12\x06\xf4@\x9esp\x8f\xf21\xee\x1aG\x08T\xed\xa0\x17|fG\xad\xf9-q\x1e7\xed\x94]|\xa2\xe54Fn\x17\r\x08bn\x9a\xbd\xa9\x90t|}p:\x10\xe5\xf2\xb4\xba[84\x01\xb2\xf1\xee[\xbeO\x111_/n\xd6\'\x02\x83\xff\xa0\t>\x9374\x11\x04\xc2\x91\xe9I\xae\x9a\xcb\xf1\xa0\xfa*\xa7\x94\x97\x846\xa4\xc5;\xcb\x8a(\x8a\xc3\x8d\xbf\x80\t\xccP\xb7\xca\xe4\x1e\x8cJ\x8f\xa7\xe1\x18\x1e\xfc{K\xf2\x89jOK\xd4\xe1GX\x82\x9d\xa6t\xf7\xf8\x9b\xab\xf9)"\xe6k5!b-\x84\xe0\x17TD@\x0c\x84\xab\xc4\xae\x8b\x8aG\xd1\xd7\rS\xc1\xd4\x1e\tG\xda\xacJ2Py\xef"^l!\x9d{\xc1\xf1!5\x1c\x93\xd0J\x0e\x90\x042m\xa6\x80\x13\xa2X\xa3\x82\x81\xf6}O\xa7\n\xba\xbb+\xf5~2\xd17W\xf3SD\xcc\xff\xbc\x96~\xaa\x1a\xbf\x8fR\xbe\xc9\xe9^V\xb2\x03J\xa46wL\xbb\x14\x83\xaa\x96\xe3\xe4\xce\xd8\xb5u^qmAI(ND7&>\xe7\xd6q\xbfk\xe7#\x8b\xc6vg\xb5\x16!D=\x03{M\x85\x90:-\x89X\xa3\xca 8\x81?ze\xfb\x93\x85\x98\xe7b\xc2\xd0\xba\x9c\x08\xfc\x92\x81\xd0\x01\x92/\xd5x\xb8P\\\xeeZ)}\x82\xa9\xd3\xd5=\x81v\x01\xed\x8b]\xe4\x0c\xc509\xc5\xf1t\xee\x96\x8bA\x05z\xf2P5\x03\xc7J\x9d\xd30\xa2$\xf2\x13"\xec\xd0\x96\xb1/\xf7\x9b\x98]H\x9a\xce\xdb\xe2\xbfS\x88\xf9\xc7\xceG\x11\x1c\x87^\xea\xfd\xc1\xe5\xcfd\xd6O{9\x9e\x9dc!\xa8!wr\x91\\g\x8b\xf0\x8e\x90\x1c/3m\xc7p\x94\x90wU}O\x8eQF\xdc\n\x16\x86Tq?2\xf5I\xb6\x90sP\x91k\xcf\xd5\x97g\xfc\xcc\xd4mE\xdd\xb07\xbf\x90\xfa\x94\x10\xf3\xd5\xea\xe3\x18\xfa|s\xf3"$\x0b\xb5)\\c\xee\xd8\x9a\xea\xe1\xf2p`\x18\\\x1bR}\x1f4\xa7$\xba\xdf\x0f\xc2\xfeN2\x8a\ng\xfb;M;\x194\xda\xfc\xa3G\xcb\x9b\xad\xf8\xa7\x80\x98g\x94k\xe2\x87\xb1\xb5s\x7fi;\xcek\x87/\xa0P6A\x8b\'\x94\xf7\xab\x13\xf6\xce\x12\xbb!\xa3\xbb\xc8\xa0\xf0\x9cX]jRh\xb1\xa0g\x80\xab/\x8cJ\xae0\xc0C\xf7n\xf6\x11z\xf4\x83\x12t\xbbN,w\xd1\\L-v\xaa\x12\xa7]~\xc4\xd3\xfd\xb5\x81\x98\xffy\xbd\x0f\xa3 \xf5r\x17;\xb5\xec\x8f\x90}\xb4"G9\x80\xee\x912\x1cv"1\x7f\xf1\x02\xbaWa\xfe\x10\xa3\xb6\x92\x80A2\xeed.0\xb1\xeb\xf1\x8e\x86\x8c\xc4z\x92\xd9L\x16\xcdV\xf9\x94aE\\\xd0\xean6\xa7\xf8p\xbe\xec\xf8\x7f\x946\xff\x14\x88\xf9z\x07\xbb\x011\xff\xf2_\xaa\xff\x17\x011\x7f\x1dzc\xd3L\xb6[\xfa\xfdo\xe9\x06\xc4l@\xcc\x06\xc4l@\xcc\x06\xc4l@\xcc\x06\xc4|\xe3\xeb\xdc\x80\x98\r\x88\xd9\x80\x98\r\x88\xf9\xce\xf0\xca\x06\xc4l@\xcc\x06\xc4l@\xcc\xb7\xbd\xce\r\x88\xd9\x80\x98\r\x88\xd9\x80\x98\xef\x0c\xafl@\xcc\x06\xc4l@\xcc\x06\xc4|\xdb\xeb\xdc\x80\x98\r\x88\xd9\x80\x98\r\x88\xf9\xce\xf0\xca\x06\xc4l@\xcc\x06\xc4l@\xcc\xb7\xbd\xce\r\x88\xd9\x80\x98\r\x88\xd9\x80\x98\xef\x0c\xafl@\xcc\x06\xc4l@\xcc\x06\xc4|\xdb\xeb\xdc\x80\x98\r\x88\xd9\x80\x98\r\x88\xf9\xce\xf0\xca\x06\xc4l@\xcc;\x14\xc2o\xce\xc0\x7fJ!\xfc.\xbd\x7fg \xe6\xe7\x07\xf6\x8b\x80\x98\x9f\x1f\xd8/\x02b~~`\xbf\x08\x88y/\xb0\x7f\x87\x1b\xf9E@\xcc\xcf_\xb1_\x04\xc4\xfc\x07\x92\xc7\x06\xc4\xfc\x14 \x06\xfb\xbf{\xfe\xe7@\x8c\x00Q\xb4\xf0\xe4_8\x98\x87)\x0cEi\x8e\xe51\n\x81X\x82\xa6I\x92\xc6\xd1\xf5\x03$\x8f\xf0\x89F\xdb\xad\xb74}\x14N.\xba5-i\x16W\xd6c\x84\x9bc\xdc\x1e\x91\xe2Hw\xa9*\xd6\x0e_q\xd8^\xb3i\xb5\xd8\xf3\r\x97(M3W\xf1}l\x12\x8fC+\x94\xb4\xd9\xe6\xf8\xf8^\xb2\xc0\xa7\x84\x98\xe7]\x04\xd7\xf4\x8fb\xaf\x9c\x90(7\x16b8708\x1c}Pf\xe2\x19\xf5v\xc2!\xd9-\x0b9\x9b\x06;i\x96\xc9\xc4\xe0\x880j\xae\xde\xfb\x0b Jn\xdd\xa3\x10\xe1\xba\x14\x9d\x8d\xba\x8cD\x1a\xf8\xc8m\xcc\x91\xec.I\x94\x11\n\x88_$\xc4|m{\x18\'\xd6\xf4\xff2\x8d\x8b\xc1|8\xa2\\\xf2\xcc\\\x19\xc0\xc7\x11\xa3<\xbb\xde\xc9=\xf0\x92\xbe\x8b`\r\xe5\x95n\x8f\xb1\xf7\xa3\xa9z\x9dy\x08\x0e\x1c\xac\xde\xd9s\x13\x1cqM]\x93\xdcL\xa5\xb3s\x82\xefN\x91\x122Pg\x87\xaa]\xbc\x1f=,?Y\x88\xf9\xda\xf6\xcfY\x83\xc4\xeb\x1c>\xb8J\xc6\xab\xe55\x8dW_\x87\xe3C&\x1f\xdd\xd5\x1d\xd8|!\xce"0\xddN\xbd\x999\x86\xa2@\x95D\xde-+\xc3\xeck\xc3S]\xe8\xb8\xb7\x03\x06^\xaefb\xde\xc3\x0bDg\xb0z\x13L]O\xce\xbeN\xbc9\xa1\xeeSB\xccs5\xd7\xa5\\?\x8d\xbd\xcc\xff.\x8e"\xdb\x9d\xc7\x02\x9dt{R\xa4>y\xdc\xa0\x138DN^ \x16\x8bIfi\xd2\x9cw\xe3\xa6y\xbc\x18\xca\xd0\x89\x80\xe9^\xc5\xe2\xe4J7^\xbc;.TFz\xeb"\xa5s\xc0\x81To\xf1\x98\xdb\xa3o\x8e\x1b\xfc\x94\x10\xf3\x0c\x93\x84\xd7\xd3\x1cy\x1d\x1e\xe93\xe7\xfaq\xac\xefc>sc\x00\x9em\x14S\xfc)\xb9\xec\xbd}\xa1Q6\x0e\x967\xfd\x8e\xd3\xb0\xee\x90g\xfcx\x93\xe1\tRP\xb2_\x98\xf2\xe2Y=f\x1e\xc8\x06\x06\xc0B\xc7\xb9(\xbf\x83\xc3L\xdf\xf9\xc3\x9b\xc3#?E\xc4<\xc3\xc4\xd6\xfa\xf2Y4\xbd\xa0m\xe3\r\xbe\xba\xea\xd2\xabW\x02W/2\xb0o\xe9\xa3\x8c=\xaa\x03\xc5\x00\x8e\x06\xaa\xe6\xf9x\xdb\xd1G\x94\x8a9u\x8c\xb3\xf94\x8a\xf4\xa4\x92\x01\x19\xbb\x83Ii\xb6\xa2\x1bj\xa0\x1bi\x05\xdaKG\xdc\xb0\x1d\x15|\xaf\x99\xf8\x9f"b\xd6\xbb\xf8$E@\x8a\xc0^f\x0c\xb7\x00\xabT@~\xaa\xbc\xddQ\xf0\x1bl\xaa1\t;!\xe4|\xb9s\x88l0`\x98 \x1c\x9e>\xec\xc8\xb3sZ\x92\xd0\xee\xaa[\xe6\xae\x92\xe9\x87\xb6S\x1c\xb3G\x8e\xa8y\xecZ\xc1-9\xab\xbb\xcek\xd1}z\xf3a\xf9\x14\x11\xf3u^\x91\xc4Z\xc8#/e*\xaa\xff?\xf6\xeec\xc9q\xf4J\x14\xf0\xbbh\x8b\x08\x11\xde\xdc\x1d\x00\x12\x10r\xb2\'r\xad\xe3\x13\x16\xb7J\x01QcAR\xe8;\xef\x1c\xa4>\x1c\xf6\xc6:\xbe\xca`\xa6p\x0e\x13\xf6\x1dq\\J*1\x9f\xd9YQuH&\\\xe7\x18#{\xf3:\xfc\x14\x11\xf3\x0cs\x1d\x11\x94\x84\xe1\x97\x03\xd4\x81\x07J\x1c\xf62\x18k\xb5\x89E\xea\xf9F\x1d"\xd6\x99\x8b\xe6q?\xd1l\xe5\x9e\xadX\xe2\xd6j\xc5H/\xaa\x08Z\xf8a\xcc\xa7\x06\xb3\x0eW\xb4\xd5\x9c\x0b\xd3\xca\x86/\r\xd6\x9d\x0cv\xcc\x0c\xfb\x92\x7f\x8f\x8eo\xbap\x9f"b\xbe\xf6\xc4\x9a\xa9\xae\x85\xcdK\x98\xccI\xcb&)\xba\xc5\xfb\x0bE\xc4g\x1dG\xa8\xdb\x1d\xd5\xf7\xd6\xdd\x8bw\xb8j\xf1g\xb2\x83=\x04\xd1\x03\x9d\xbd\xfaN\x95\xd3.\x1b\xa1\x96\\\x95\x06\xe7\x9c\xfb\xdc\xbb_\xe19\xcd\x89T\xaf\x02\xbaN\xe3\xe8r\x90\x99_C\xc4|\xdd\xfa\xc8S\x89y\xc5\x0c\r\x92\x9eP\x07\xf1\xf7^m.\x888F\xb7\xdaO\xb9\xab\xb0\x8f\xb5[\xa4*\x19 \xe8\t\xf5\xb0\xa1\x83\xfa\xa0H\xd8\xdc\xefr\x97\\\xa4%\xf5&+cxL=\x06\x8f=<\x98\xae\x9d\xca\xd6h\xc1c\xac\xc4\xfb\x1fq\x18\x7fm"\xe69\x8a8\x85\xac\x194\xf9r\x80\x02\x8c\xe7r\xfb\xaaPO\xc0\xc9\x06/\t:\xda\xb3\x17v\x8e\x83a\xf0a\xe6\x92\xe2\xde\xdczq\xb9\xc79\xb8fjhKU\xbc\xe9\r\x98\xadtS\x96S\x11$\n!mt\nP\xd8\x90y\xe3\xb8\xea\x94\xde\xbcw\xed\x94\x0f\x111_\xb9\x13\x08R8E\xbe\xdc\xfaS\x06:\xfc"X\x06\xa7I\xcd4\x1a\x06_\xd1\xa0?\x85{/\x1fO\xe7bA\x06\xb5\xbc*\x8a\x87(\x87F9\x04T\t]\xcc\xf8|\xeaoFM\xdb\x80?P\x8f#\x934\x89SsW\xecreOJ\xe8\xb2?\xea\xd5\xfc\x93\x89\x98\xaf\xba\xe6\xd9x\x99@^\xc2L\xcc\xfepY\xaf\xd5<\x91\xce\xc7\x0b\xa1\xc6Jn\x89\xcbq:F<`\xe1\x08=\xca\xf0\xac)\'0\xdb\x89\xc8\xc5@*X\x7f\x0cw\xd7=0Y\xd0\x80S\x077\x81\xd5\xf8\xf6\xceM\xf8lN\x85:)\xf7\xb7\xfa],\xf1CD\xcc\xd7\t\xb7\x1e\x88\xe8\x9f,Z\xdd\x06\x04\xb5\xe6\xb1y\xb8`\x03q\x19\xbd\x07\xa8\x9e\xbc%\x04T\x86\xb2\xa8\x05\xa0\x02\xa6D@\x19K\xc9\xb8;\x8a\x15!-8CtrqDg\x02&\xc8\xe4p\xc6Q\xebnbB!\xeay\xee\xd4\x12\xfa@\xfe3\x89\x98\xe7("\xe4\xba\xe8\x90W\x18\xae-\x18\x89W\x08\xfe\x96^}=D\x8e\xad\x14\xa4\x95n\\\xd1\xae\xb3\xc0\xbd\x92\xf0\xf7\xab\xba\x9c\x16$Q\xe3N\xc7\x1ew%~\xe0\rs \xf1\xb3l\xb7\xee\xf1 j\x90\xc2Qy\xcf\xcd\x08F$\xb9\x0b\x0b-\xf6\xe6=\xf1)"\xe6\xeb\x9e\x00!l\xbdo\xfe\x18e\x1ei|\x1b\xec\x0e\xdc.\x0e\xfa&#\xe4c\x7fn\t\xd5`\x8fb\n\x0c:\xb1\x07 \xa9d\\\xa0\x03\xc7\x87)\xb7p\xed\xaf\xeb\xfe>Wpx\xd0v\xfb\xec\xf18\xcb\xf9\x05\x0caG\x06\xd52H\xf4\x12,\xc17\xd5\xa4O\t1_\x93\xb9\xde\xa2 \xf2\xfaF\xa3r\x98[!\xdd\x0b\xd9\xb3\xfb\x11\x06\x84\xa2\xf5\xa1c\xa5\x01\x08.\xf6\xd4\x01\xf2H\x1d\xb2\xc2\x0c\x02\x13\xf3\xe0\x1f\xd4\xbbd\x87\xfd\xaeh<\'XR\xa3\x1c\xefB1-I\x8cb\xc2)\x9a<\x15d4:\xad\xda7\'\xf3SB\xcc\xf3\x80\xc31\x8c \xd6\x99\x7f\x99MH_\xb0\xf4<\xf5\xd3`\x8ff\x9b\xf3\xe5\x14\x8d{z\xf1\x1d\xf4.\x18s\x15a]0\x0c;l\xba\x8c0\xd1\x02)\xef\x17\x96<\x1d\x90\xf66/8p\xa5X-\xd0\xb9\xdb~g\x90\xecAP\xaf5%\x90\xe5\x9b\xb3\xf9)!\xe6\xeb\x1c\x07\x91u\xb5\x93/a\n\xc4\xf5z\xba\xd5HPY\xc3A##+\xea,\xfe\x16\xd71\x11qk\x99\x12\x1ct\xa7#\xa2G\x16\x8a\x12u\xb89\x00\x94\xa6\xb4\x8b\xb84\xa4\xdd*\x89s\xc2\x9a\x083\xe3T\x83a%\xa0\xed)!;\xff\x9a\xbc9\x9b\x9f"b\xbe\x12\xf2\xdf^\xaa\xbe,Z\x14\t\xefnq\x04Nib9\xe88\x198\x16u\x06\xed\xe4\xa2`\x95\xcd\x15Oh\xa0Y\x8b\xf2\xc8\x8co\xd4N\xd8Q\x9d\xe9\xecu\xe1RW\xc5\xc5\x97\xb5\xa86\x99\x93,rK^\x10\x0b-uJ\xb57\t\xefG\xb8\xc9_\x9b\x88\xf9\xda\xfa\x04\x08\x11\x10\xf9\xe2\t\r\x10b%(|<\x91H\xbf\xa7v\x0e_\x08;\xd1\xd3tX\n\xf5v9\x91j\xa0\xa6\x18\x04\xc4\xf7\x08n\x870\xc5\xa4G\xbe\x13`\xa9n\x8cGs\xe94\x88\x12)\xcd\xb4\x1eT_r`\x8f\xcc\xe2\xc1\xf3\x17\xee\xcdR\xffCD\xcc\xd7\xe7\x18\x04\xb9\x9e\xf8\xd4\xcbb\xb9\x01\xfa\x11\xf5*\xb7\xd4J\xe4\xe4\x1a\x91c\xab\x93\xe7\x90c\xd1\x11\xd9\xec\x9d9\xe7\xc8@\xa5z\r]^\xb4\x06Z\xedU\xc7+\x86\xd2\'\xc96\x98]\x17D\x05\xb2\x87Ae\ta\xf5\x9aF\x97\xc5 \xe5\xdd\x177\x9f"b\xd60\xd7s\x1f\x84\x90\xb5\x84\xfbc\x98\xdd\xcc\xea`~\x9d1\r\xc9\x97C6\xb6\xca\xad\xbf\xb0ZI\x85\xa3\x0b \xc4\xd1\xca\x10\xaa=>\xc2\xba\xcdO\xbc\xb6<\xe4\xc7D\xa1$\x1bAX\x7f\xe1\x1f\xbc\xf5\xc0\xd4\x93^\xef\x97\xa8\x9d\xdb\xe8rYC>\to~\xbe\xf8)"\xe6\xebV~\xe6\xf0\x18\xf4\xf2y\n\xdf\x9aFK\x1a\x83\xc0\x18\x8d\xa8\xdbm\x05\rx\xd4\xc6\xe8\xf1dX{\xc0\xbb\x02\xa1M\xa9hm\x89~\xd7\xa7\x11\xf0\xf0\x1b\x08\xcf\xf8+J\xa0\xa8\x12\x1elY\xe8R\x08\xe8/\xca\xc8$^\x82t\x81\x8c\xdf\x82_d\xc4|-Z\x12Y\x13r\xf0%\xcck\x9c\x16\x1a\x10\xf8qPxf\x8fv\x9e\xf7P\xed\x98\x94\xa2\xfeq\xb2\xe8\xca`e\x01\x84\x99 <\xca\x80 5\xbb\x8b\x98\xf3\xfb[\xc7\x1e{\xe9\xe8\x06E\xc90W\xf2\xa0f\x0f\x99\x9c\xbc\xean\xf7\xd6Q@\x8a\x1f\x95W\x7fm#\xe6\xeb\xd6\x87\x89\xb5h\x00_\xf6\x84\xef\xa5\x07\x8aP\xaf\xa7l\x96tz\xba\\\x0e\xb3h\x05\x16\x90\xd3\xa5w\x8c\xd4Ck\xfaIGX\x06{\xba.5\x8cL}\x8bPV\xe5zW\x90\x96\xd8\xf1(\x00\xec8j\x9cW\xe6\xed$\x9b\x9d\x92z~&\xfcv\x1d\xfeK#\xe6\xabN\xd9\x8c\x98\x7f\xfb\x97\xd57#\xe6\x974\xe8\xdb@\x93mH\xbf\xfb\x90nF\xccf\xc4lF\xccf\xc4lF\xccf\xc4lF\xcc7~\xce\xcd\x88\xd9\x8c\x98\xcd\x88\xd9\x8c\x98\xefl\xaflF\xccf\xc4lF\xccf\xc4|\xdb\xe7\xdc\x8c\x98\xcd\x88\xd9\x8c\x98\xcd\x88\xf9\xce\xf6\xcaf\xc4lF\xccf\xc4lF\xcc\xb7}\xce\xcd\x88\xd9\x8c\x98\xcd\x88\xd9\x8c\x98\xefl\xaflF\xccf\xc4lF\xccf\xc4|\xdb\xe7\xdc\x8c\x98\xcd\x88\xd9\x8c\x98\xcd\x88\xf9\xce\xf6\xcaf\xc4lF\xccf\xc4lF\xcc\xb7}\xce\xcd\x88\xd9\x8c\x98\xcd\x88\xd9\x8c\x98\xefl\xaflF\xccf\xc4\xbc\xa3!\xfc\x8f\x1f\xf2/5\x84?\xdc\x98\xdf\xd9\x88\xf9\xf9\x81\xfd"#\xe6\xe7\x07\xf6\x8b\x8c\x98\x9f\x1f\xd8/2b\xde\x0b\xec\x7f#\x8e\xfc"#\xe6\xe7\xcf\xd8/2b\xfe\x0f\x0e\x8f\xcd\x88\xf9)F\x0c\xfe_c\xfe\xcf\x8d\x18\x94\xe41\x1a\x15@\x98\xa28\x1aa\t\x94#q\x9e\xc2p\x14cA\x16\xe1A\x82\xc01\x04\xe6\x10\x84\x13\x04\x04\xa2Pv\xfdw\x88!y\x12\x82(\x92\xc5\xff\xb9\x11\xc3\n\x02\x8f\xa1<\x82\t\x98\xc0"$\xc9\x91\x10I\xf2<\xc2r0\x0c!$\xc5s\x0c\xc1P\x02\'\xb0 \xc9\xa04\rr\xd8:.\x1c\x06\xe1\xeb\x7f\xc6@\xe1g\x1a1\xff\xe8\x9c\xfb\xb7?i\xc0\x80A\x7f\xc7\t\x8aB \xe4\xb7\x06\xa1?2b\xbe?\xb0\xf3\'F\xcc:\xf6\x02\x8b\xc2\x04O\x93(\xf5l\xe0\x82\t\xbc\xc0\xac\x93Or8\x8a\x828\xc2\xd1,\x06\xe3\x1cM\xc0<\x03\x93\x02A3\xe8\xb3\x15\x05B2\xeb\x9c\xa1\xcf.G\x9b\x11\xf3\x13\x8d\x18\x14]\xa7\x91\xa5q\x12$I\x9c\xc0!\x12\x7fN\xb4\xc0\xe2<\x01\xe2\x0cH 0\x81\x0b \xc9\xb3\xec\x1a\tHQ\xc2s\xeb\xe0\x10\xca\x0b8\x83\n\x04\xf5\xb7\xbf\xb8\x11\xf3oC\x10\x1f3b\x9e\x0f\xf2\xbb\x11\x03\xfe\xa9\x11\xf3\xfd\xf7\xf9/5bP\xfc\xc7x\x8a\xb5\xf3\xb2\xdc\xe1]m\x7fLN\xc8\xb20\x01\xc0\xf4\x0f\x03\x1almX/\x10\xbf\x15\xd1\x92\x8e\x8e\x0c\xa4S\xd9\x10k\x0c\x88\xeb\x99\x98\xdc5\x7f\x1e\xcf]\xd3\xc4W\x1aA5\xad\x19Oi\x8aK\x13\xe1\xb6\xd4\xd4\xff"#f\xbd\x15\x88gc\xe25\xd6\x97.u\xdd\xadA\xae\xaex\xe5\x0cn\xfd\xbe\x00O\x1d;\xf7reb\xe9X\xda\xec^f\xf5\x87\xe0`\xe9\xfd.\xf0}\x90/\xa2|\x94\xceT\xc1\x89e\x00\x9c\xaf@\xa7KL]\x01\xf1\xd0\nY;-\x85~\x9d\x90\xdey\xb7\x19\xdf\xa7\x8c\x985L\x12\x83\x08\xea\xa55\xb6\x84[ewj\xcf^r\xf64\x93\xf7\x0f@\x17\x0e$\xb8\xe7\xcf\xcd\xe9(]\xcd`\xd7z\x81\x1cV\xeba\xd4\x1br6\\\xc9\xdem\x05\xc65A-1w*\xa6\x9a\xa4\xcb\x87)\xb2p\x9c0\x1e\x00\xa2B\xe2\xdd\x9bS\xf9)$\xe6\x19#H!$N\xbe\xb4\x02U\x1e2\xdca@\xae\x1c\x82n\xdei\'\xb1t\x1a\x8d\xb1\xe0\xc3\xa9x,\xd9\xe3J\xa6\x9e\x92\xd3\x90c\x1f0\\9"\x176w\xba\xc72\x1b\x9a(\xc0X\t\xfae\x12\xda{ow=\\q\x08K\xa02$9\x9a\xf9A\x94\x7fm#\xe6\xb9\x1f(\xf4)u\xfc\t(\x84\xcdr4\xb8\x0e\x85G\x9d\xbd\x1fv*\x01\x1d\xf3\xd6h\xae\x00\x85\xca\xfe`\xd5f<\xdb\x8d\xb0\x94>bj\xd3Q\x85\xd5\x9dn\x81)\xa6\x1d\n\xac\x1a\xdc\x9dG+\xd0\x00\x9eF\xaa\xf0g\xaf\x0b\xe4Z\xe0\x8f\xef\x9aI\x1f2b\xfe\x98\x0c\xfe\xf70\xcf\x87\xccy\xc8\x94\xbd\xc0\x1cV;\xea\x82\xe7\xc5\xfd\xa8:,\xa8\x026p?\xa4\xf3\r\x8c;1SZ\xc8A\x8f<\x10\x13\xc9}*(B0\xd8\x9d\x16_L\xdc\xde\x97\xa6\xa9h\r\xc5\x03\x05R@#\x03\x07\xb7wU\x91\x0f\x191\xcf-\xf1$\x14 \xe4\xb5C]9\xa5\x96\xcb\xcb6\x8fW\n\x1b E\xa6\x9b\xeb\x19\xa8\x9b\x80sR\xdb\x14\x87\xa0\x9b>\xf7\x9dI\x87\x153F\xd9P\n\n\x10A\x9dC\xdc\x169\xbe\x84Ri\xc1\x80\xac^Rn\xb6\x8bG\x90R~[\xe5\xda\x9b\xa7\xdb\xa7\x8c\x98\xe7\xa2]\x8b\x9b\xb5\xc2\xc1_\x16m\xe7\xee\x84\xfaq8\x99\x85\x14\x1f\x04\xb1.\x07\xce\xa1\xf6\xca\xe0\xe9\x076\xac\xf1k\x16q\xf2\xe8V7"\x01X\x87\xc5@\xd1\x05\xbb\xc2\xd6P\xb1\xd4\x11]\xa0\xae6`\xb4E\xac\x96U\xa5\xfa\xc6\xe1\x84\xe6*c\xbc+\xfe|\xc8\x88y\x86I\xc2\x18J\xfdIS\xfc\x0c\xd3\x83P)\xe9\x93N\x17\xe9\xf9ty\x0c\x92*\x9aK\x96\x17\x00\x8f\xda\x00 d\x81\xdca\x8d\x82S\x96c\x06\xe8M5\x85\xe9\x02\xf1\x0f\xb4\x05#\xac\xefw\xcf\xe6\xab\xa9\xe9ZC\xcf\xda\xc29\xbc\xe7&\x16\xbey\x8e\x7f\xca\x88\xf9\xda\x9b\xeb\xa0\xac\xe7\xd0\xcb\xa2\xa5n\x90\xad\xb3\x19p\xd7\x1d\xa4\xe8\x1e(\x95\xf5W\x80\xa1\x8f\xbd\x7fM\xa6\xe8\xa4\x91t\x18\xadW\xff\xb0\xeb\xbd\x96\x0f%\x0b\x9a-t\xbd\xa6U\xb7k\xef\xbe<>\xfau\xd1V\xd9l:5z\xb1\xd4\xe0\x91=r\xf9Gm\x15\xff\xdaF\xcc\xd7\xd6_\xc7\x9f\x80\x89\x97~\x83\xf1\xf5\x04\xbaH\x88\xa8\x0c\xe8j\xb8M#\x84\xaf\x187\xfa\x8a\xda\xd7y)\x17\xdf:\xdfn@T\xe8CIrC\xf4\x88(}t\xb8S\xcb\xed\xc5\xc0}\xc4~\xe9`\xd7\x84\x9d\x14S!\xb0\xb85yW\x8e\x08\xf0\xcd6\xea\x9f2b\x9e{\x02^s \x10zm5J\x157\x7f:O\xf6P\xa9R\x13\x1c\x85\x99\x18\xd9ji\x8a\xbdRU\xba\xbc\xe0P\xe0]RhFfq\xbeND\x1b\xb5>\xa7Y{G\x06\n\xef\x92\xd1\xf8}\xe0\xee\x16\x01R\x11\xc7\xe4X\xb4\xaf4p\xef\x8ao\xf6\xe0\xfc\x94\x11\xf3u-C\xe0z\xf5A/\xd9\xf8\x0c\xe7a\x1c\xb6l\x15\x9d\x9f\x85\xa7o\x9e\xb5\x90\x8a3\x17\x96\r\x94\x1d\xf7\xc9\xe1|*p\x90\xe2\x94\xe4\xe0\xb0\x17\xe0h\xb9\xb9\xbc\x1f\x98\xd0\n\xc6{&\xf1\xb5z\xd1\x03\xe4!T\xd99\x9a\xfc8Ak\xfb<\x7f\xaf.\xc3\x9f2b\x9e{b\xad\xeaH\nzMn*R\xdaw,\x01,\xb8\x1b\xe6F\xa5d\xfc\x18\x1d\xe1v\x7f\xb5\x94&9\x1c\xed\xfbQ1})@/"Y\x17\xc1i1\x84Z\xc4\x92\x83\xc8\x05\xfc`\x0e\x13\x8c\xb7\x18\xd8-\x06X\xe0W@\xed\xf7xB\xee\xcb7A\xa1O\x191\xcf0a\x82"\x9f\xc0\xe7Kr\x03\xe2\xe3\x85 \'\t$\x9c8\xc3\x81\xdc\xb1\xab\x8b\xef\x90\xe7\x98\x03=\t\x0cR\x084l,\xb9z\xa2p2\xc4\x87\x03\xa2gZ\x00\xe7\x89\x93g.\x80\xc0\x8e\xe7\xf0\x84.\xda~<\xaa\xf5\xdc\x10\x87\xbd\xa9Aov\x19\xfe\x94\x11\xf3U\xba\xe1\xe4z{\xbel\x89\x06\xa2\xb0\xf8a\x1c\xaa\xf5\x92\xab5\x80\x15\xfd\xf6,Q\xbdz\xf7U\xe0\xd6\xb6)#\xee\x07.;\xc6I\x127\x91\xa5O\x89~\xd2\xef\xd6%\x82w\x14\xa3\x04^4\x95\xb8\x06\xb3\x10\xea=\xbb\xcb7.L\x1f1\xf4\xcd\x96\xd1\x9f"b~\xcbmp\x18\x83\xf0\x97K\x1f\xf7/\x81\x1b\x0e1/\x8c\x05zfE\x8d\x81\xed\xa4\xedp#\xaf\xfb\x1d\x87N\x13c)\xde\x1e\xabt\xe7\xba\x98\xf5\xe1N\xec\x01\x18\xf6F!\xa1\xae\x9e\x16\xb0\xc2,R\xc3\x85\xeas\xe8!RA\x9a*9a\x81\xdf\x8c\xbe\xfc\x10\x11\xf3\x1c\xc5\xf5\xeb \x0c|],\xdd\xe2\x81\xb9\xe5T\xfbd\x86\xc8\xe8\xd1\x9fb\xf0\xa4\xf5\\\t\xe01\xa7\x11z\x12\x91K\xea#\xfd~bl\xb7?F\xacu\xba^\x0eg\x02\xa1\xd6\x0bt\x06\xd2\x999\x8a\x07\x1a\xa4\xc6|\x94\x85\x99\xea\xc98&\x867\xd9\xa4O\x111\xcf\xd4\x89\x02\x9f\xed\xe8\xe1\x17;e\x99|\xbf\xa8l\x05:v.\x9ft\n\xdf4\xb7\xb0(i4\xdd\xf3\t)\xee\x86\xd1H$\xca\xdd\x01\xc3C9\x96\t\xa0j\xce\x1dJ\x98\x9e\xec.=G\x8e@7\x11\xb5\x8c\xde\x92E\xc7NR\xechtb\x14\xef\xe6\xfb\x1f"b\x9e\x07\xdc\xba\xebI\x92z\xedH\xbd\x10#\x18C\x11\x1d\xea\xcb%\x87\'m7K\x84\x81D\x99py\xdc\xfc\xf5\x87%a_\xe0\xe3\xfe\x88\x9aYcV\xc1I\x9b\x92\xdb.\xa7\xef\x9e7k\xf6\x91u8k\x12\xd9\xdb\xae\xeb9U\xdf\xe9\x0c\xa26\xb3\xff\xe6\xa5\xff)"\xe6k\xd1"\x18\x81\xbc\xdeV\xc2\x81\x01\x94.`\x1fy\xdb\xd9\x12\xd5\xe2\x1d@i=\xf8\xb0"w\x87ar\xc21a\x0c\x1a\xfc>\x8de\xad\xf3\xf1\t\x1e9\xae\xee\xd5\xc3\xa5\x8d\xb8[1\xb1QZ\xdd\xacB?\x9f\x86\x14\x07"\xa9$"\x9f\xfe\xd1\x01\xf7\xd7\x16b\xbe\xaa\x895\x9f&a\xe8%A4\x97+\xdf\xfb\x8f\xe9\xa8C\xea\xb9\xf4\xc3\xd2\x9e\xce\xa4m\xcf\xebAzO0\xc2O\xda{\xd3\x15\x0e\xb8f\x90\xce\x85\xedh\xb3\xd8]\xfd1\xae\xf5J\xe7\xbc[}4Z\xa5Vo\x0c\x971u\xb1N\xb7\xd9&\xb5\xf5\xe6\x0b\x8dO\t1_\xd7\xc4Z^b$\xfe\xf2B\x031o\x96m7\x98\xca\xe9w\xfa~s.H\xe4v\x02.\xd6\x0e\x02\xdc\xc4\xd1\xc6:\x93\xc7f7\xe7\x0fe\xbe\xc3\xc8\xacK\xef\xe0\xf8\xb8\xc6\x8e,\xa0u\xa6\xd0\xa1\x02p\xc4\x14bg\x84u\x9b\xf6qh\xe6\xa2|S\xdb\xf8\x14\x11\xf35\x9b\x18\xbe~\x83W\x03\x8bi9s\xe4\x1b\x1fH\xcc\x01K\xb9\xb6x\\\x8c#|\x97\xc7\xea\x0e\xabGe*\xaei\x8eO}?<\xfc8\x07\xa7]\xd0 \x81A\xb4\xbe\xe0\x97\x80w\x9f\xcfl\x9ay\x173\x80&\x90\x91[\xceb\x96\xae\x16\xde\xdc\xf9\x9f"b\xbefsMi\x11\x92x\xb9\xae\x02\x96\x9b\xbb\x07\xbft\xc3\xa1%\xb16\xbcR1\xad\x0cNA\x1c\x10\xcd^\x17\xf0\xddQ\x86\xca\xf04^\xd4\x81\x1d\x8c\x1f\xc4\xf4\xce,)\xdfg|\xac\x05Y\xbb\xc0~I\x99\xb7Q\xad\x8e{2@%\xa2\xa6\xf8\x1f2\xa9?\x99\x88\xf9\xed\x1c\x87\x11\x10\x81_\x16\xed\x84\xa4\xf82\xb9\xd2\xee\xa2&.q!\xef\xa3\x0e\x9fJ\xe2\x92\xc6\x8c\x89>\xf6\x91qW\xce\xa42Y\x18\x9e\xd9J\x14F&\x05\xb0\xf4=\xba\x1dyG)X\xa3\xb5\xe5\xfa\xaa3\x81i\x1d{\xaf\x11y\xaa\xf2\x15\xf2\xcd\x1a\xf5SD\xccW\x8d\x8a\x81k\x9d\x87\xbd\xbc\x99\xee\x80]idy\x17\x89$\xc4\x97c\xdab\xa7\xf96\xd5{\xfd@\x17&\xcb\xc9\x8dxF!\t\x102]\xcb\xf8R\xbc\x99\x03Mb\xe8\xad\xd8\xa5:\x7f\x97\x96\x14XW\xaa\x0e\x8d\xf5\x9a\xb7\xdc\xf2[\xe2\xb4\x96\x08\xfeho\xfe\xb5\x89\x98\xaf\xad\xbf\xe6\r\x10\x8a\xbdX\x89IQ\x9e\xbb\xb9\xf0\xbb\x9e\x8f\x8f\xac\xd01\xa2\x82\x9c\xee\xfd\xd9N/\x9d\x85\t^\xbbG\xaa\x94\xd50V\xab\xdd\x83\xa9\x83KoaW\xed\xe4\xc3>igxtr\xa9\x06\n$~B\xa7XEC\xbb\xa4tXys\xb1|\x8a\x88\xf9*\xde\x10\x12\x06\xd7\xdc\xe1e\xb1tJ<\x15\x07\xd1O\x1bN\x07\xac\xfd-\xdbe\x03T\x1f\xc54\xe7\xeb\xea\xec\x9e\x9ax\xbd\xe6o\xf5IL[(u{\x03\xccOY\xa1u\xc1\x9aL\x82q\xc3\x18=\x05\x12\x14\x14WuK\x0f\xca\xc2\xec#\xe1\xf1f\x98\x9f"b\xbe\xb6>\x05a\xcf\x9b\xee\xe5\xcdt\xefz\xa9GA\x9e\x9ecx\x91\x9c\xdd\x9e\xa9\x1a\xe5v q6\xb4O\xd3n\xbd\x11-\xc1\xb9\x03\xc5\xf1\xc8\xa5-m#\x0e\xeda\xa39\x90--A\xf7(]\x90^\x91\xa7Ia\x92\x85d\x13;LZ\xf2\xf6f\x98\x9f"b\xbe\xde\xdb\xc0$\xb6\x9e\x17/\'\x9c\xbcP\xd3\x1e|\xcc\xaa\r\x8d\x9a\x0b\xc5\\H\x1dx8\x90\x1d\xb8g\x0fj\xda\x99\x92\x18\xaf\x8e\xf4i\xb13\x89\xa3\xff-"\xe6\xebU\xf3F\xc4\xfc\xdb\xbf\xab\xbe\x111\xbf\xa4?\xdf\xe6\x99lC\xfa\xdd\x87t#b6"f#b6"f#b6"f#b\xbe\xf1snD\xccF\xc4lD\xccF\xc4|gze#b6"f#b6"\xe6\xdb>\xe7F\xc4lD\xccF\xc4lD\xccw\xa6W6"f#b6"f#b\xbe\xedsnD\xccF\xc4lD\xccF\xc4|gze#b6"f#b6"\xe6\xdb>\xe7F\xc4lD\xccF\xc4lD\xccw\xa6W6"f#b6"f#b\xbe\xedsnD\xccF\xc4lD\xccF\xc4|gze#b6"\xe6\x1d\x0c\xe1\x7f\xcc\xdb\xbf\xc4\x10\xfe\xf0H\xdf\x99\x88\xf9\xf9\x81\xfd""\xe6\xe7\x07\xf6\x8b\x88\x98\x9f\x1f\xd8/"b\xde\x0b\xec\x7f\x03\x8e\xfc""\xe6\xe7\xcf\xd8/"b\xfe\x0f\x0e\x8f\x8d\x88\xf9)D\x0c\xf1_c\xfe\xcf\x89\x18\x8a\xa5\t\x8a\xa1P\x98\xa3P\x06\x13\x10\x8e@Y\x1e!`\x01\xe3q\x90\'@\x92\xa09\x9c\xe3\x11\x92\x85\x04\x9a$P\x82]\x9f\x96ciR\x80Y\x84e\x9f\x1d\x8c~\xac\x1f\x80\x18.\x90\x04\xc8p<\x8b\xb1\x18\x89\x124+\xa04\x021\xeb7\xa6X\x0c\xc6\x18\xf0\xa9\xd3\xf00\x88Q0\xce<\x9b\x93#\x04J\xf1\x04\xc33\x14+@?\x93\x88\xf9G\x03\x96\xbf\xfdY\x03\x06\xf2\xd9\xc6\x02\xc1\x9f\xb3\xf4\xcf\x88\x98\xef\xef\xeb\xfc\t\x11\x83\x81\xcf\xd6\x800C\x80,Lq\xeb#1\x02\n\xc34E@\x1c\xc9\xac3\x01\xf3\x18N\x08\x88\xc0\xa2<\x8b0\x02\x03\xe2\x18\r?\xdbY\xb1\x02A\xd3\x18\xff\xb7\x8d\x88\xf9\xa9D\x0c\xf4l\x18\x02Q\x0c\n=\x1b\x1bB<\x0b\n \xc1\xc14N3\xeb\xfcC\x14\x85\x11(\xc3\xac\xdf\x9b\xc3P\x81\xa4\x08z\xdd\xb0\x0c\xccs\xe4\xfa\xa0\xeb\xdf \xff\xf6\x17\'b\xfe\xed\xc6\xd7\x1f#b\x9e\x8dM\xfe)\x11\xf3\xfd\xf7\xf9/%b0\xf8\xc7\x88\xc2 \xb1\x10^A&\x94\xd0A\x01\xcd\xc8Z\x0f\x98\x9d\x81\xe3x\x82\x9c\xa6T[\xff\xd1\x99W\x04\xaf\xe4\xec\n\xfb\xc6\xad\xba)\xbc\xacdL\xf0X\xb8,\x83O-\xdeI\x13\xb3&\r\x02\x10 \xce=\xb5\x16\xccy\xfcH\xdc\xf8\xd9D\xccz+\xac\'\'\tQ\xaf\xad\x15\x0f(\xb7\x0b\x02\x82\x1a\xdd\xab\xd5\xc1\xdcL\xea\xec~\r\xcd\xbb\xca\xf7 6\xd2\xf9tk\xec>\xea\xda\xe9\x1a\xe97\xac\x0e \xbb\x17=\x93\xc5\\\xef\xc8N\xca\xb9\x96(\xf7\xccM\x82x7\xc1\xfczs\x04\x9f\x9d.o\x86\xf91"\x86\xfc;\x85`\xeb\x95\xf0\xaa\xfd\xb8\xe5\xddT\x83L\x9d\x1e\x9e\x8e\x13\xb7\xabz\x83\x03^\xc4\x90\x92%\x01\xbf\xbe\x00:\x13\x04\x8f!\xf8UP\x10\xa1\xf6\x91Si\x05\x1d%\xa2\xbc\xd7\xd4E/E\xda\xc8\xcfZH\x8bsf\xc2\xc2\x1eJ\x04\xe8\xa1\xaf\xdf\xac9\x8c\xea\rr\xf7\xd58\xb9\xb30\x1b\x82J\x8cu\x18s\x91\x9fM\xb5)\xa9\xf1\xb2\xa0\xf1$5\xca\x8f\xfa\xe3\xfe\xc5\x8d\x98g\x9aD\x90\x10\x81@/\xdd\xbeH\xb0\xf1|\xac\xe1n9j\xb6\xadv\xc5\xbd}[\x93\xcfE\x1e\xf2F,^\xcb\xf3p\x87\xb54C\x92"~\xf0\xe7\xf5\x88?:\x1c\x0f\xee\x13:\xd1J\x84\x88\xa2\x8e\xb9\x92=\xa2\xc7I}"\x13o\xa7+\xba\xfcf\xaf\xd1\x8f\x191\xcf0\xd7\xa3\r\xc7_\xf1\x14V\xb5%\x86\x0f\x96"\x07\xad\x9d\xef;a\xdc>\x1cn`\xb3\xe3\xfe"H\x9a?\\!&\xf1X\x88;\x9dO(F\xfaly\x9c{\xe4\xbeG\xd1\xa4\xcb,\xac\xa6O{\xce\x90\xb0\xeeT\xf2\x8bi!]\x99\x11\xc8\x9b\xfb\xfecF\xcc\x1a&\xb1.\x01\x8azm\xc2y\x00P\x1f{\x94%\x84P\xd2\xb83\xed\xda\x16Q\x07\x92j\xab\x9f:\xe1|P\xef\x8a\x9b\x9bwK*\xba\xb8\x06\xfdf\xbf\xafgAZ7\xb9k\xc2\xa8^\xa3\xa7\x9bK"w(\xd1\xadxl\x8a\x19\xd4\xf3NY\xde\xdc\xfa\x1f3b\xfe\x90\xdb\xff\xf70\xf9KMa\xc7\x13\x9a\xf5\xd6\x19\x15\x15\xe3D)\nO\xec\xb0\xf5\xca\xd0M\xb4\xad&L2L#\x1cK\xcb\x91}\xac\xf7F\x91\x0f\x1d\xe3r\x0c"\xb2\xa9\xd7?M|H\xcf\xeb\x0e\xf0\r3\xca \xa3\xc9\x07 |\xb3\x03\xf8\xc7\x8c\x985L\x0cAQ\x94z=\xc8[\x8c\xadA@\xcdF\xca\xa7%\x7f\xb7V|;\xd5A\xa1\xd1n\xf4Y\xcf\x1c\x11\xed\xc5\x8b\xda\xccw\x10(\x18\xa7\x11{\xbf|d\x12\x11\xdc\xf6\xa2n\xf1\xa6O(\x07{$O\xf3\xf5z\xa1\x85\x07\x06x\x9c\xd4\xf4ov\x8e\xfd\x98\x11\xf3\x0c\x13$\xa85\xd6\x97\x06\xb9E\x08J5\xe4L\x07\xdf=\xa4w\xd9\xf3\x1f7f7\r\xbe\xb2F8\\\xaa\n\xbc\'\x80\xe8\x11\x84\xb7O\xa1\xdb\xad;\xd9\t\x9f\x1f\xbaJ\xc1N\x03\x1b\'a\xc3\x9aW"\x1c.t\xe1W\x05F\xa1\x82"$\xc0\x8f(\x9c\xbf\xb8\x11\xb3^\xf9$\x81?\xbbF\xbf4\xa6\x95\xab\x99\x1f\xcbz\xa8\x05S\xb0o\x06~\xf5b\xe8L\xb2\xa3\x96]\xf4\xa9\xb1\xc26OnQd1\x06\xe8\xa0q~\x0c\xf6\xb6r\x80E!7\xcb&6\xcc1P\xa9^\x04O\x9eD\xf9\x8a\xb7\xee\x13\xa06\x96\xe5MP\xe8cF\x0c\xf9w\x82X\xbf\x19\xf9\'\x07\xb9\xe9\xc9l\x94\xc8\xe2#&A\xc2;a#\xd6\xcb\x03\x07k\xa7`\x0f\xb3\rY\xb1\xb5\x1d\x9e\x15\x96\x95\x1f:\xa8\x8eU\xcf\xc9\x87\x01\xa4\xd3\xc1\xb6\xf7\x18b\x95\x07$\x87\xee\x9e\x1c$,:\xa3\xdd\x00tRKEob_\x1f3b\x9e{\x02\x021\x10\xa3^\x92\x1b%td\xe2\x10U\x97I_\x8b\x93\x12\x16\x89\n\x1c\xcd\xa1\xd1$\xd0E\xc3SE\x08\xd1\xa4^c\x7f\x1c\xd4\x03\xa9I\xdaxj\xcdR\x1d\xe3;\xdf\x01\xa1w\xa6oj^Z;[>\x15\xd9\x9a\xfe\r\x0c\x9c\x02\xa7\xef\xe5&}\xcc\x88\xf9\xda\x13\x10\xbef\xc2/\x8b%8\x82\nc\xeb")b\x9c\x01\x84\x12\xbb7\xd4a@o\xd1\xc5l\xa0\x135\xb7\xcc\x85t\xc0}\xe60M:;1\x1d@>\xd7\x0bw)\xc3Z\xa0N\x03\xdd\xce\x91T\xd4o\xda.\xe4\xa3\xe9\xd0i"\xd4\xebo\x1e\xa0\x1f3b\xd60\xd7:\x13\x87)\xe2\xa5\xff.\xe4\xcb\x14\xdc^\x80P\tl\xf6:Qu\xe3\x19\xa2\x19\xe6&F\x84\xcaN\x8bx\x08\xf6\xfc\xac\xb9\xcf\x90\xf7p\xf1\x8c\xd4\xf4\xe1\xd8j \x07\xe6\xae\x9bdY\xa7#\xc7\xd8\xbc\x0eg\x10q\xd5\x94ra\xba\xce\xaeo\xf6\xdf\xfd\x98\x11\xf3\xdc\x13k\x82\xb4V\xcd/\xf7\x84\x1b#.\xc0\x06\x16\x8a\xee\xf0\x8e\x08\xcc\xc35\xb8]\x1dq"N\x05:\xa8\xe5\xf1\x94]P\xe5B.g,\x1c\\\xf7\xd6BW/*\x00\x88\xaa\x17\xfe\x16N\xb5\xc2\xb3R&h\rE\xedB\x00\x84\x02:8\x8f\xd77g\xf3cH\x0c\xf9\x84\x8d\xf0\xf52}Ebvw\x9de\xab\xb2\xeb\x86\x04\x9f\x12A\x85\x90\x14F<\xf1F\xd2\xba\xa9\x99\'\xe6\x06j\x97\x93\t\xdb\x9a\xcb\xa6\xbeZ>H\xbd\xae{\xef\xd8\x08\xf5#\xd7r\xc1?\x15$\x1c\xf6\xfa5\x01e\x11L\xa11\xbb\xdb\xf5\x7f(\x12\xf3\xac\x0e\xd7\xf4\x10^K\xea\x97L\x98\xa1\xfb5\x0bS\x8c`\xf0\x97\x13q\xa2\xccj\x8eB\xae\xac\xd8\xb4\xc6Opz\xb4Kh\xdd\n\xa0{B\x98h>\xab0\xdb\xca\xbe\x7f+\xcds\xbd\xdcy\xa3(\xf9+\xac\xb1\xea\xc5:\x98K\xa3\xfa\xca~4~x\x80\xfel$\xe6\xeb\x9e \x9e*\xc6\x1f\xa3\xbc\x90\xa1\xa1\xdbA#*"}\xf4MA\xedl\xf8\xd8\xf8\xdc\xee~X\xef\xd0a\x0c\xcf;\xf6\x11\x16P,\xf6*\xb2\xf4\xda\x7f(\x12\xf3,\x0ea\x04$P\xec\xe56\xccw\xb2u\xb5+\xc4\xa3\xe2\xb15\xe2#\x07\xc9}\xf7H\xf6\x8d4G\xbc\xea\x01\x81\x1f=\x9b\xf0\x03\x93I?\xb2\x98\xd2\x8f\x9cr\xcf\x0f\x87K\x13\x1cN\xd2\x05\xd54\x8aw\xd9\xd9\x16\x0e\x8b\xb6\xdb\xed\x0e\xa6Hb\xe5\x9b\xb7\xe1\xc7\x90\x98g\x98\x10\xbe&\x9a\xafm\xda-V\x17A\xc7\x10\xf4\xf1qn\xd9\xd0\xbdD5\xdf[C\xfbH\xe0!\x86\x8e\xea\xc3\xb6D\xd9p,\xe9\xcc\x94}\xbf\'S9\x7f\xcc\x15\xe7D\x00s\xca\x83\xe0xJS\x03\xa9\xa0\x1d\xc1\x80-\xd5\xd8\xbc\x0c[\xe1\x9b\x89\xf0\xc7\x90\x18\xf2\xe9\xdf!\x10JR\xaft\xf9q\xb4\x0c\xa87OUd\xdbxBi\x08v\x9addO\xa2\x9d\x1c3 bb6\xaf\x03r&V\xaa\x1a\x89\xe3Q>W\xfd$\x8b\x91\x97\xde\x88\xcc|d\x017_\xfa\xcc\xc9\x0f\xeeB\xc5F\xdcr\xfc:Z\xbf\x08\x89Y\xc3\x84\x08dM\x13^_f\xba\xe7\x01\x9c\x16\x9d]T\x17\xee\x9eES\xed\x95\x93\xe3ck.\x86/B\x801$\x1f\xd3\xad}\x9e\xdb\xa3\x01\xefn\x05\x80bx\xcd\xd7R\x99A\x85O\xaa\x93\xc5\xa4\xd1\xfd\xe6\x98\xf3\x90\xcd2XY)\x1a\x14\xcc/Bb\xbe\x0er\n\x02a\xf2\xe5\xc5\rT\xd89 \x88z\xa0)M\x14\xf7\xaa\xcc\xa7\xc6\xcd\x14NCS\xb3\xa3\xe1\x1fFEjiKD\xee\xb5qY\xcfk\xc5S\xef*\x1f\xf9~\x15@\xf9\xf1\x01\x08\xc6:\x93\xfaY\xd0$\xf5P$W\x03m\n\x9by\xf3\xfd\xd4\xc7\x90\x98\xe7A\xbe\xe6\xee\x04\xf6\'\xf06\x12\x8c\xe7a\x822B<\xd2)}\xbc\x19\xeepkoS\xc9\xeb\x8cly\xee!\x0f\xf9z\xafe\xb8\xb2\xdc\x01F\xbeR\xce8\xc39+\\\x948\xa5p\xce\xb1OKPr\xa7\xae\xe7o5\x11HJ\x81\xa2\xf7\x1f\xa1\x8d\x7fq$\xe6\xb9\xf5\x915\x7f\x83^_f\xaa\xc8Uq\x9a\xe6\x92\xa0I\xb2\xbf;\x82\xe9\xa2\x07\x96\t\xb1\xf0\x8c)eT\xde\x8b\xa9A\xe8\xc6\xbf\x14\x1aXc:\xbdS\xce\r#\x94G\xe48\xbbA\xa9\xef[\nr\xc5;`\x16\n!j\xb0\xac\xca\x99\x02&oJ\xd0\x1fCb\x9e[\x1f\xc2\xd63\x12\x7fy\xcb\xb73\x85+\xdc\xf1$U\x13\xb2h\x84\xd9\xcd%i1\xb7\xa4\x1bNH\x8e\x97\x85k\x0eP\xcbG\xcb\x91 \x0er\xf3\xe8x\xb2U\xc3\x91\x05g\xc6\xb4\x82\xa1\x80<\xc8\xf8\xdc-F8\x9e;X\xde\x07\xbd|\x84\xab_\x85\xc4\xac[\x1f[\xaf>\n|}\xcbw>c\x05\x84\x98\xb8ysz!`c\x98\xc8\x93\x80\x1b\xc0\xaa\x03\xab\xd2\x90\r\xe1\xe4\xba\x07)\xb1\x1fm/Y\xa3>\xd2\x80\x07\xdbK\xb4?8d\xa6Hj\xa9\xa9\x13\x08\xdaJ3$\xea\x9at\xd1\x9ct\xe6\xcd7\xef\xab\x8f!1\xcf\xba\x03z^q\xd8\xcb\xa2\x8d\tK;\xa3{\xdd\xdb\x17y^6\xb8\xce\xb0\xd3A\x0b\xcbk\x9e\xc1U\x15\xa4\xea]\xd6\x8b\x85\x97\x8d\x19\x0bZ\x0c\xb0\x93\x90d\x1b\xa6$R[I9\xb9\xa0\x13\xa1`v\xd0\x91\x91\x13,\xba\x1a\t\x9ei\xfe\xfcf\xaa\xfa1$\xe6Yw\xc0\x10\x86\x93\xe0\xcbl\xda\xb1\xb3\xe0\xae?T}9@\xa4\x17Z\x9e\xda\x1e\xb1\xddZe\xc90\xc3\xe3TS\xd0\x83\x9f\x1e\xdbT\xc9|\xa6\xdckC\xc9\xc5\xbb{\\\x88\xc9\xbe\x87\x14\x89\xeer\x8dx@q\xab\x8c\xf1\xcdO\xc9u_\x1e\x8b\x1f\x15\x1e\x7fq$f\xdd\x13\x10\xb5\x8e!\xf1*\xe0a\x06\x0c\r\x92\x17)>\x0b_L\xecq\x99\xabk\xa0\x04\x9c\x05\xcf\x1alp\xf7\xc9\x97\xca\xdd5 \xb4"0\xe7"H\xf0\x83n\x02\x02kL\x17\x8b\x1eD\x8f\x14;3\xe4\xac\xd3T\xa8\x95}\x92;v\xb7\x07n\xbf%7\xff\x12\x89\xf9:\xb26$\xe6\xdf\xfem\xf5\xff $\xe6\xaf\xd3\x85\x7f\x83\r6$fCb6$fCb\xb6\xb3\xf4\x1b\x9e\xa5\x1b\x12\xb3!1\x1b\x12\xb3!1\x1b\x12\xb3!1\x9f{\xce\r\x89\xd9\x90\x98\r\x89\xd9\x90\x98\xef\x8c\xaflH\xcc\x86\xc4lH\xcc\x86\xc4|\xdb\xe7\xdc\x90\x98\r\x89\xd9\x90\x98\r\x89\xf9\xce\xf8\xca\x86\xc4lH\xcc\x86\xc4lH\xcc\xb7}\xce\r\x89\xd9\x90\x98\r\x89\xd9\x90\x98\xef\x8c\xaflH\xcc\x86\xc4lH\xcc\x86\xc4|\xdb\xe7\xdc\x90\x98\r\x89\xd9\x90\x98\r\x89\xf9\xce\xf8\xca\x86\xc4lH\xcc\x86\xc4lH\xcc\xb7}\xce\r\x89\xd9\x90\x98\r\x89\xd9\x90\x98\x7f\x8fC\xf8\x1f\x99\xda\xbf\xe4\x10\xfe0\xcb\xdf\x19\x89\xf9\xf9\x81\xfd"$\xe6\xe7\x07\xf6\x8b\x90\x98\x9f\x1f\xd8/Bb\xde\x0b\xec\x7fC\x8e\xfc"$\xe6\xe7\xcf\xd8/Bb\xfe\x0f\x0e\x8f\r\x89\xf9)H\x0c\xf9_c\xfe\xcf\x91\x18\x06\x82q\x88g(\x0ec0\x8c\xa4a\x94d\x11\x90\xa4\x9fM @\x1aD\x11L\x00\x19\x96\xc2 \x16$\x9f]\x16!\x90\xe0P\xf8\xd9\xad\t\xa5I\x9eE\x9e\x8d/~\xec\x1f\xd0$K\xe38\xc1\xe0\xa8\x803\x10\x06\x13\x14\n\xe3(\x87\x82\x18\xc31<%\x90\x02\xc8\x12\xcc\xb3\x9d\x06\xc9r$\xbf>\x08L00\x8a\x92\x04\xc6\x804*\xfcL$\xe6\x1f\xbd,\xfe\xf6\xd2\x80\x01\xfd\x7f \xf6w\x1c\xa2\xd0\xf59!\xf8\x9f!1\xdf_\xd8\xf9\x13$\x06~\xf6\xcd\xc2@\x1ccP\x8ay\xaa(\x02\x82\x83\x14\x033\x0c(\x80\x02\xca\xe3$\x83\xac3\xf3\xec\xd6\x8c\x13\x08Opk\xa6F\xae\xb3\x87\xf2(\x87 \xc8s\xa9nH\xccODb\x10\x81`\xd7\xa1\x04q\x0eA\x19\x08\x15X\x1a\xa1\xd7\xffq\x08\x8eP\x10\xbbn\x10\x0e\xe4A\x96$\xd7\x7f\xc2 \x812\xeb*!Q\x9ag\x98\xf5\xc1)\x9e\x86\xff\xf6\x17Gb\xfem\xba\xe4cH\xcc\xb3\xb1\xc9?Eb\xbe\xff>\xff\xb5H\x0c\xf5\xe3\xbe\xfb\x17^\xe6\x81L\xb6F\xa7\x1c\xef\xf7\xb6%J)\xe3\x8d\xa1\x90\x84\xe3\x1e0u\xcf>D\x01\xeeh\xd2\x04\xf6\xa4\x85\xc33(\xd3\x93Tx\x97\xe4\xecRGK\x1d\xe5\x10\xd4\n\x92\x81\xfb\x00\xb8\xf2gP\x00tk|\xb3\x87\xd3g\x90\x98\xdfn\x05\x04F\xa1\xf5\x1e|\xe9\xc3\xc9\xebb]H\x16s\x03\xedH\x9e\xe1\\W\x015\xb34H\xbc\xae\x7fYXl\xbf\x84\xdd\x85\xda\xeb\rS\xa8B=\xfa\xfcB\xe0\xe1\x15\x1b\xa8]\x85I\xe7\xd1s90`\x074T\xc9\xfd\x8el|\x00`\xe1\xee\xcd\x8e\\\x9fAb~\x0bs=\xc9^[\x0f\xf5\x885z\xf0\x10\x1d\x10hY\xc0R\xdds\xf7\xb1\x81\xceC\xd6\x90\\Pw`\xa3z\x86{P\xb3\xc5i\x9c\x9a\xcb;\x8eW\x8d\xa0\xd1\xa4\x8e\x8e\xce\n\n_9Q\xe0\xcb\x1b\xee\xc4\x02\x8e\xfb\x07\xc3\xe2=\'\xffQ\x8b\xba\x9f\x8a\xc4|\xc5H\xac\xd7 \x0e\xffI_EF\x18\xee\x04\x84\x07xS\xd5\xb0\xbfoh\xc0\xa8\xf8\xb32\xfa\n\x96\xddZ99Z\xed\xd9\xb8\x85<\xbf#\xcc\x1b%\xed5N9d\xad&\x04\x9ek=Nwd\xac\x0c?.\xac,s(M\xd4\x95\xf5j\xa8\xc0o\xd6G\xfd#H\xcco+\x05]\x0f)\x0cG^\x9a\x9a9pL$\x1eC\x87\xd0\xc4\x87\xfc\x89\x07E[x\x9c\xd8{\x91\x8c\x12M\x91\x17\xc1\x12\x98\xc2\xc9\xd3\xdd\xecv\x8fZ\xbef;.\x13\x02n:wn\x9a4\x8b\xc849Zt\x0c\x8b\x96my<\x9c`\xa6i\xc27\x9b)\x7f\x06\x89\xf9-\xccg\x07N\x88zm\xa9Z4\xa7eJq\xefl\xd8\x87\xb1\x1d4\xe0\xeaA\x94q\xaa\xc7\xe4\x8c\x05^\xc7\xf52~\x8dt\x91c\x15\xbcKyE\xde\x03Mp\xae3Ut\xb4\xe14\xa3\x11\xb9K\xa9#\xdd\xb9\t\xaaw\x00\xe0\x95\xfd\\\x95o6\x1c\xfc\x0c\x12\xf3\x15&\xf6\xe4VH\xec\xf5\x14g/\x91\xab\xd5Lv\t\xbd\x0e\x9aO\xd6\xe8b\x8f\x92\x82#\x1aHY~\x7f\xae\xe9=\xd8\x16\x1cH\xea\x15CT\xfa\xe96\xb7\xe2\x9eg\xca\xc1O\t\xa5\xb9\xd7\xf8\x1d1j\xc4\xae;QA\x00\xa9\xb8\x1a#\xa6\xc3\xbf\x04\x89y\xcd\xed\xff{\x98\xb0\xaf\x86\xea\x1d\x8e\xfbz\xe1\x96{\xb0X\xd7\xb9R:\xe7a\xee\xc8\xe0\x1e\'2\x90N\xcb8\x00\x92\x89\x91\x16\x8bkaM\xad\xc1x!d\xe1\xc5\x19\x81\xca\xea~\xde\xb9\x95\xc5yz\x7f\xe1\xaa\x1b\x83@\x17\xa3x\xf3\x14\xff\x0c\x12\xf3\xdb\t\xb7\x96vk\xd1\x86\xbc\xcc\xa6\xaf\xc3#R\x9f\xb5\t=\x9583\x93\xeb\xff\x99%SB\xd8\xd1-\x86}\xe5\xda\xa0\x84\x94\x03\xaa\xfa}\x97O%D\x9b\x98\xeb\xe5z\x06>\xf2<2\xdc\x0b\x9aA\x8f:\xe6/\xf81soW7\xdb\xbb\x1c\x1e\xd1\xbf\x02\x89\xf9}ob0\tC\xe8\xcbl\xd2\xdd\xf9\xd1\xc4\x17x\xcf?\x90|z\x80\xd6\x1d\xd0\xf0\xec\x06fR\xccj\xbb\x8c<#\x92/j\r\xe9\xdccD\xdc\x9f\xcbAVPCL\xaemP\x89\xe5\x89s\x1a\xe4\xdci\xbd\n$\xd5\xb4\\\x81\tX\x83,\xbe\xd7A\xfe\x19$\xe6\xb7\xc5\x82\xc1\xcf~\x9d\xd4\xcbAN\xf8vf(\xae\x97\x1d\xcc\x9c\x10\xef\x9asY\xd4\x00\x010\xe8"\xe2S0\xef\xd0\x02 &Bq\x8cL\xc4\x95\xb3I\x11\x8e\xd3T\xc9\x1c\x87\xfe\xc4\xe7\x89\xc6\xa6\xac;\x13\xc9\x087\x18\xdcSD\xb9O\xcc4}S\xfb\xfa\x0c\x12\xf3\x8f\xad\x8f\x13 \x88\xbc,\x16|\xd7\xf5\xcc\xfd\x14\xdeU\\\x9eD*\xbf\x9d\xe6\x82\xd6)\xacZ\xd4R\xa23\x1b\x00\xef\xc5\xe3\xb8\xd4\xd92=\xc6\xc4\xb9\xc3\'\')v\xb1rI\x18\x0e\xb1e\xe5xE%\xd4]\x98\x19F\xc5\xdc\x0f\r\xc8\x81\xfd7O\xb8\xcf 1\xff\xc8S\x11\x9c\x84^O8\\9\xd9\xe5%\x11O\xa5mU\xf8\x90\x87\xd3\x81B\x84\xf5\xe7\x1b\xd8#\x0c\x00\x1b\xe69\xa7j\xd1sp\xea.\xf2c\xaaO\x85\x10 \xe74=\x1d:\xe8\x11K\xf1u>\x9ag\xd1\xd5p\xe8\x91*\xe2\xce>\x14\xcb\xf8\xbd\xf6\xc4g\x90\x98\xdfF\x11\xa7 \x92"\xa8\x97L\x98,\xe3\x08\xcd\x0f\xc5\xb5\xbf\xa1@6\x15v\xd1 \x81\xff`uV\xd8s\x00W\xd0\x17\xdc\xa2\x91ntk\xc5Q\x9c\xeb\xc9\x938\x9c\x92\xaec\xd8\x1a\xe5\xc0\x06\xfe\x1d\x12\x0c\xb3:\xd1G\x85\xbe\x06\xf4\xd9\xd60\x13z\xf3\x00\xfd\x0c\x12\xf3[\x98\xd4ZO>\xe9\xa4\x97\xebpN\x80G4\x9eBS\xbf,\xfb\xfc.\xdac\xed?\xe6\x8b\xe5\x97\\\xda\xc4K\x98\\\xc1E\xe1\xa0\xdb\xa5\xa8v\xa3\xe0\xa9\xa3pP}-\xca1\xd4\xa3wL]\xc5\xd5\x898\xccYx\xa8.\xb9IH\xd7\xecpx\xb3v\xfb\x0c\x12\xf3[\x98 D\xa0\x10A\xbc\xec\x899\xd2\xdd\xe8\xcc\xcd\xb6\x8f\x1f\x172\xe3\x9c\x87\x8c\xed\xdc"\xb1G\xfd\x1c\xe6-f\xc4\x02\xb3Ow;\xd9og7\xa9\x19\xe1\xa8\x8751i@\xdb5B\x12\xe3\xf69Bh}\x01\xef\xd7\x1e\xa8\x06f\xcc\x97>\xfa\x91\n\xf9S\x91\x98\xdf\xb7>\x82`$\xfeZ\xa2\x1e\xdda\xbf\x07b\xf1f\xeb\xeb\x92\x96\xe4\x03)\xd9F8f\xe2\xf11\x82hv\xacX$\xd2\x84S\x8c\x8f*4G\xc6\xe5qiYe~\xc49?\xe9\r\xfe\x90\\\xe2r\xcd\xa3\xdb\xedv\xde\xd19\xce\xb4\xf7\xf3\x05\xfcf\xf8\xe5G\x90\x98\xdf2a\x02\x84`\x82$_\xaaCaf\xee\xfa`8t\x17\xf1\xc9eA\xd4\x9b]h\xba\xb9k\x8ax\xb0\x97\x8e\xc2m\xa6\xbd\xd6\x87\xd1\x11\x01\x159\x8aPt\xc1\xc7^\xe3\xe9F\xd5;O\x9859\x07\xd91\x9cs\xbc)\x84\xb0\xa9\x1a\xe2\x88\x16o\xee\x89\xcf 1\xff\xa8k\x10\xe8y\xed\xbcl}\x1c\xc9\x05\xa6\xd6w0It&"u\xb2YQ\x1dPq5\xeeE\x0f/\x14\xc8s\xb6\x8e\x0f\xbd\x03]\x14\xca\xa7}OA\x18\xa6BjI\x965\xf4\x00\x115C\xa2\xb0\xcb\xdd\x9dQ\xb6\xacq\'tZ\xcb\xdf4\x14>\xa3\xc4\xfc^\xebc\x08\n\x81/\x07\xdc\\\x1ci\xd4\x191\xb1\x8a\x89\x98\x03\xdd\x98\xd5o6\xb0$\xc9\xee\x1a(w\xadZX\xfc|6Dr}\x0c\xeb\xe3\x0c\x8en\x99\x8f\xefQ\x7f\xc2\x08\xe7\xae\x1b\xd8\x05\x0b\x8b\xd3\x0c\xdc\x986\xa7\xbdB\x995+\x9a\x93\x92\x10\xaa\xaa\xf5C\x1e\x92\x17\xe6M6\xe93D\xcc?\x82$1\x92x\xd56\x98\xc9W\xa0\x0bO\xcb\xbe\xd0>R=?k\xd2\xdc\xb3\xbb\x93:\x1d\xb9E\x1a\xaeE \x92\xe5Ic\xdb\x01\x07=r\xe4\x97BQ\xf8p&\x97c\xc0?r\x07c\xb1\xee\xc0\xea\xc9)\x9a\x1c^\xbc\x99\xd3YL\xe57\xef\xc2\xcf\x101\xbf\x87\x89\xad+\r~\xdd\xf8>U\xed1\xf9Xk:\x83\xa4\x8f\x0c\xb6\xc6%gw\x87\x13\x11\x0f\x81\xe3\xc9>\x83Y\xb4\xcbSk\xe9\xe8k\x0e\xd6_\xfb\x81\x9cB\xe4p\xb3\x8cE\x01\x8c\x1du\xe1\xcc\x96\t\xda<\xed\xd3\x8a9Zu\x9f\x17\xc6\x8f\xd8\xa4\x9fJ\xc4\xfc\x9e\xd9\x90\xe8Z\x05#/\xa2\x99\xaa\xc2\x07\x8b\x192\xbf\xce\x8fd\xc3\xb0\xa6\x16(\xbb\x19\x81/y\x1cW\xf1\xd1\xdc3g\xe1R\x95\x94%\x1f\xe7\x0b9k l\x8e!\xc0\xda\xf2\x0e\x14\x04\xd9\xdd\x81\xe3,\xf4\xa4\x10\xa5\xbb\xf8Xx^\xcc&c\xfb\xe6e\xf5\x19"\xe6\xf7S\x1c\'\x08\x14D^2\x9bY\xc0\xb0r\xf1\xcb\xf9\n\x84\x9el\xee|\x98\x8f\xce!3\x8b\x9d\xe9\x0c\xfb\x9e\xbd\xd3\xebb?\x057:N+=R\xf2\x07O\x11<\xa1XT\x16R\x14+w\x91Q\x10\xed\xfa\x17\x08\xfar>\xb8J\xce\\\xbb\xf3\x9b/\xa6?C\xc4\xfc6\x9b\xd8Z\xbd\xaf\xf5\xdb\xabRe\xc0\xa3\xe4-\xb5\xd2D\x8fZ &\xd2\x8bG\xd4\xc8\xd6\x9a\x90\x91\xf0\xdd\xa5\x1a\x0c\xf9Q\x96\xb0\x7fGG\x16\nR\xe9\xba\x03\xe4\xc2Q"\xeeJ\xca\x80\x9e\xad\xcb\xa3\x182\xac\xaaI\x94\x88\x08r\xd7?\x00|\xfeQ%\xfeW&b~\xdf\xfa\xeb\xe6\x7f\xbe\xb6\xfd\xe3(\x82\xdc<\xd2\xa9\xb4\xd8Q\xear\xc29\xceT\xda[\xcf\xf3]\xc5\x02R\x0c\xaaxnT\xeaT\xfa\xdep\xa0\x05\xce\xe6\\\x9c\xead\x8b\xdb\xed\x0b\xdaf\x1eR\xa1d\x13\xb3\xd3\x92t\x88-\n/{\xbf\xf3\x82\xf9\x87\x0c\xc5O%b~/QA\x04\xa5 \xe2\xe5 \x97\x8f\xf1\xee\x11$\xea\xa5V\xf8\xbc\x99\'C\xc4\xfc\xfe1\xc3\x9a\xce\x13\xc8\xeb\xbb\xc6ADF\xcbW;\xd8\xf1\n\x8e^T\xd1\x00\xf2\x01\xe2A"j\x00-X"\xb2g\xa8\x88\x9a)\x8d\xa4\x0e\xfb\x86c\x1c#\x0eg_v\xacS\x98\x97\xa1\xcc\xd6\x85\xee\\\xbck\xa2\xb3\xadXt\xa6\x92T\xbd\xfa\xbd\x1c\xd3\xcf\x101\xbfo\xfdu\xb5\xbc\x8e!\t\xdb!\x81G\xcb]WnU\x8eP\xfb\xdb\x15h\x01\xcd\xc9[\x1cW\xb9C\xd0R\x910Q\x0bo\xa2\x0fg\x7fmZ^\xe8\x97\x0e\xdc+\x1c\xed@h\x98\xd5 =\xf2m\xa8%B\xe9\x1der!z\xa6K~\xd3\x84\xfe%\x10\xf35\xf7\x1b\x10\xf3o\xff\xa6\xfa\x7f\x10\x10\xf3\x17j\xcb\xb9u:\xdd\x80\x98\r\x88\xd9\x80\x98\r\x88\xd9\x80\x98\r\x88\xf9\xbe\xcf\xb9\x011\x1b\x10\xb3\x011\x1b\x10\xf3\x9d\xe1\x95\r\x88\xd9\x80\x98\r\x88\xd9\x80\x98o\xfb\x9c\xdbk\x93o\xfc\xdad\x03b6 f\x03b6 f\x03b6 f\x03b\xbe\xefsn@\xcc\x06\xc4l@\xcc\x06\xc4|gxe\x03b6 f\x03b6 \xe6\xdb>\xe7\x06\xc4l@\xcc\x06\xc4l@\xccw\x86W6 f\x03b6 f\x03b\xbe\xedsn@\xcc\x06\xc4l@\xcc\x06\xc4|gxe\x03b6 f\x03b6 \xe6\xdb>\xe7\xff\x8aB\xf8\x1f/\x12\xfe%\x85\xf0\x87\xbc\xee;\x031??\xb0_\x04\xc4\xfc\xfc\xc0~\x11\x10\xf3\xf3\x03\xfbE@\xcc{\x81\xfdo\xb8\x91_\x04\xc4\xfc\xfc\x19\xfbE@\xcc\xff\xc1\xe1\xb1\x011?\x05\x88\xa1\xfek\xcc\xff9\x10\x032$+\x10,\xcf\n\x0c\xce\x0b\x0c\x85\xe1\x17\xd8\xbeJ\x88\xe0\xd1\x87\x91\xe7\xe0y\xa2\x11\x1e\xea\x18\'MI\x10\xbf:\x80y+\x98d\xcd8\x1eqN\x89{\xf8f\xd42\x83\x14\x00!\x9c%\xb3\x12\xe1\x96(\x887;\x8d~\n\x88y\xee{\x88\xc2\xd7\x83\x12\xc6^\xfa\xc6\xf2\xce\x84=\xcf\xfa\xc5\xb9\xb3"\x13\xa2s\xb5\xf3\x8c\x93\x89\xccn\xc6\x13\xed,\x19\xceE\x08wf\n\x95\n\x997z\xb6s-\x8a<\x85\xe3\xfa%w\xf3\xcc\xb2\xfd5\xd7Z;\xc1\xd5\x81\xec\xa9\x0e\xbd&\xe3\xf5M>\xe1S@\xccs6\x89uO\x808\xf5\xd2\xa0\x8e\x17\xa1\x1bM\x90\t\xda%\x08\r\x98L\x06M\xc1\xcc\xa3{av)T\x92\xb2\x1crZ\xbd\xb9\x9fv.\x80Ct\x80\x89tp\x04O\x8ep\xba\x12\x92t\x0b\xf2\x06?>D\xa3*\x8f\x8e\xafXp\x1c\x16\xa3\x8c\xbey\x8a\x7f\n\x88\xf9Z\xb4\x14Db\xe4+)\xa2\x1d\x8fa\x80\x1fD\xf9\xb8\x98\xc9)\xbb\x979\xd65u0\xaf3\x89\xe5\x81L\xe1\xf9\xf5\xc4zh\xf4\xe0\x8b\xac\xad\xaf\x0e\x1d\x04jR{\x0f\'\x04\xe9%\xc9\x04n8\xb2y\xd8$\xec\xb1ePN\xbd5\xfb\xcc5\x83\xfd\xfc\x9f\t\xc4\xb4~\x0b\x12A_h\x81\x8e]\xe4\x92\xa3\xeec\x1c\xb0\xbdy3.=?\tw\x05H\xdb\xd8d\x17 \xbcU\x06RB\x17\xe7\x82e"\xee\xd2\xa7G\x85>\x86\x9b\xb8\xd7\x91D\x85\x86\xcb\xe8b\xb4T\x07\xfe\xee(\x10\xda\xfd\xb6\x9e\xfc\xc6=z7\x13\xfe\x90\x10\xf3\xb5\xf5\xe1u\xe7\xa3\xe0\xcbAN\x01\x07\x92<\x99z\xe0\xf9\xa1\xe1\xe297<`\xdc\xb3\x02\xef\x91 \'\xdc_\xce\x8c\xa8\xb5\x1cs\x80\xcff\xa4\xd3\x05\x12uRtU\x17\xe72\xf7\x88\x8f\xcd\xb7\xde\xbas\xcb\xf9A\x9c\xd0]\x92\xee8d\x98\xd3o\x96\t\x7fJ\x88\xf9\xad\xd6\x7f\xbe\xb1~}1\x046\xc8\x83>C\xc6\x14\x98N\x03\xb4F;\xedF\xee2\xec\xae\xed\xed\xbe\xd6\x105\xe8\xe07vgv\xb2\x84\x17\x9ds\x83O\x10\xd4\x1d:\xbf\x8a\x1b\xff\xbc\xd4\xd0I\xd6<\xe2q\xba\xee\x93\x878\xf81j\xc1\xce\xd4\xbd\x89&}\xca\x88\xf9\xba\x0e\xd7\xe5\x82P\xafeS\xdb\xb5\x90w\xa5\x86\x04U|G\xe3\xee\xeb:\x19\x96:\x1b+I>!\xfer \xe3\x07f0\xd6!\'\xee\xb4.\x16\xf8\x15\x84\xa4\'\x1d\xael\x07/C\x18"\x1ex\xe8\x00\x8cZ\xc9\x08\xe1u3\xb9\x0fv\x9eEi\x8dPB\xd7\xc3\xc1J\xf2E\xca\xc7\xe0\x02*\x87\xa9\xd4x\x00\xd3\\\xcb\x00{$\x9f\xc1\xb6\xb1E\\-\xd8\x9a\xcdO\x0e\xfe\x8b\x88\x98g\x98\xeb\x89\x08\xe3\x14\xf5\x12\xa6pd\x01\x81\x9e\x95\xb0S\xad\xd9\x89\xc8uIE\x11\x17\x86b\x9f\'\xe7\xe4v\x1b\xd5\xcb\xbc\xc3\xbb\xb5\xda\x80(\x9b0z\x95\xa0\xab\x0e\x90\xf7\x9d+\x1c\x083S\x11\xae.\xb9p\xc1\x01\xd97\r>\x0c\x98a\xd9xN\x1d\xe2\x9b,=\xb4[\xa9\x055_]\xd9\x89\xb4\x93\x1b~\xe8\x1fI\x0e3\x97&\x98\x86!S\xbak\x89Q\x11\xc9\xfd{H\xcc\xd7\x0b\xa7\r\x89\xf9\xb7\x7f[\xfd?\x08\x89\xf9\xeb\xf0\x1b\x9bh\xb2\r\xe9\xf7\x1f\xd2\r\x89\xd9\x90\x98\r\x89\xd9\x90\x98\r\x89\xd9\x90\x98\r\x89\xf9\xc6\xcf\xb9!1\x1b\x12\xb3!1\x1b\x12\xf3\x9d\xf1\x95\r\x89\xd9\x90\x98\r\x89\xd9\x90\x98o\xfb\x9c\x1b\x12\xb3!1\x1b\x12\xb3!1\xdf\x19_\xd9\x90\x98\r\x89\xd9\x90\x98\r\x89\xf9\xb6\xcf\xb9!1\x1b\x12\xb3!1\x1b\x12\xf3\x9d\xf1\x95\r\x89\xd9\x90\x98\r\x89\xd9\x90\x98o\xfb\x9c\x1b\x12\xb3!1\x1b\x12\xb3!1\xdf\x19_\xd9\x90\x98\r\x89\xd9\x90\x98\r\x89\xf9\xb6\xcf\xb9!1\x1b\x12\xb3!1\x1b\x12\xf3\x9d\xf1\x95\r\x89\xd9\x90\x98w8\x84\xfa-\x0e\xe1\x0f\xaf\x1d\xbe3\x12\xf3\xf3\x03\xfbEH\xcc\xcf\x0f\xec\x17!1??\xb0_\x84\xc4\xbc\x17\xd8\xff\x86\x1c\xf9EH\xcc\xcf\x9f\xb1_\x84\xc4\xfc\x1f\x1c\x1e\x1b\x12\xf33\x90\x18\xe4\xbf\xcd\xe3?Gb\x10\x92\x05A\x84E9\xea\xd9y}M\x19!\x92\xc00\xf0\xd9I\x95\xc6\x11\xfcij`\xf4\xb373\xcb\xf1 \xc6 \x14\x8c\x93 \xc7\xb3\x18\x0fQ\x18K"\x7f\xfb\xa7\xfe\x01\x08?\x9b;\x824\'\xac_\xc7"<\xcf\xe0\x04\xba\xfe!\xca\xb2\x02\x8c\xf1 \x8b\xe10L\x80\xb8\x00\x91\x02\x82\xe20\xcbQ\x08\x05R\x08\x8dq,MP?\x11\x89A\xfe\xd1\x85\xe1o\x7f\xd6\x80\x81\xfc;\x85\xc2\x18\x8a"0\xf5\xcf\x90\x98\xef/\xec\xfc\t\x12\x83\xd2<\xc3\xc2\xd8:\xec\x04\xf4\xecA\x85\xe3\xeb,\xb0\x18\x8d\xd1,\x01\xb3<\x82\x10\xbc@\xf1\x9c\x80\xd1\x1cO"\x18\xc3C,Ms$\x82\xa00\x0f\xc1\xecs\xc67$\xe6\'"1\x02\x08\xf2\xeb\x9e\x06Yf]A\xeb B4\xce0(\x83\xc0\x8c\xc0\xa1\xcf\xe6u\xeb\xcfX\xe7\r\xe7\xd6=\x0b\xf3\x14\xb4>4\x0cs0\xb3.:\x82\x04a\xe6o\x7fq$\xe6\xdf\xa6K>\x86\xc4<\x1b\x9b\xfcS$\xe6\xfb\xef\xf3_\x8a\xc4@\xf0\x8f\xf5\x94\xe3\x83(0\xba\n\xea[\x16\xac\x0b\x1b\xee\x11\xd8\xa4\xcf\xb7#\xba;\x9cg9\x82\xef\x0b\xe9vB\xc3\xbbI\x97b\x0b\xac\xb6<\x0b\x8a\x07,\xbeX\xe9\xd2\x0e\xecX99\xf4@&.\x8d\xf4\xfc~\x17\xb1\xd1\xe4\x92_\x85\xc4\xac\xb7\x02E\x10\x14F\xbcv\x1bus\xae\x12\xcd#\xde\xfb\x90\xdb$~o\xcb\x05r\xde\x01\x87\xbd\xa8\x16\xc9\xder\xe1a\x9a\xf2\xa4C\xa6\x0bY\xcc\xfd\x85\x97D\x13\xa7v\x9d\xb9\xe06\xcf\xe9\xa3\xbc_\xea\x1e\xbd-\r\xe2\x8ct\xea\xef\xc7\xc8\x04\xe17\x9b\xf1}\x0c\x89\xa1\xfe\x0e"\xe4\xbaS\xff\x18csVj\x17}\xf8\xba\x91\x9f\xf67\xb6\xab\x87\xd4\x1a\x8dI\x06`\xf8>\x9fU\x1f\x91\xf7\xe4@\x1e0\xa5\x1fB\x93\x9e8\xd8\x10\x90\xde\xccc\xff \xf3\x1eXJ\x96\x87r\xa9\x95W1\xe5\x84w\xd7\xbd\xefX\x9c}\xb3\x99\xf2\xc7\x90\x985F\x0c[\xef\x02\xe2\xd5\xc2\x89dq\x07\x86I\xd9\xedz\x8eq\xf5\\;\x08UzVK\x15\x04b\x9f\xb9\x8b\x9e\x94DB~H\xcf{\x9fGTD\xaf\xe3\xd2\xbf\xa7\xa7\xa9\xd1\xe8~\x99\xe4>\x99\x11\xb4N\xc0\xcc\x8e\xe6\xe3\xee\xdc\xb2L\xbf[\xfeC\x91\x98uC\x10\x14B\x82/]\xd4y$\xbc\x0c\xf6\xa3\x10\xf8q\xb4\n\x0f>^v\x91B\xedsb\x9f^\x929\xd8\xd7E\xd5\x85\xd6\x9c\xdf.k\xe6\xde\x8b(\x14\xe9\x13B\xb5$\xae\x96\xe7 \xc9|mxh\x07\xc5\x9a1\xe0\x8e1y}\x93\xf19|\xb3\xf9\xee\xc7\x88\x98g.\xf8\xd5\xd1\x8e|Y*w\xf8\x02\xc1\'\xc9\xa2\xb9\x18O\xd2\xb4\xea\xa7K2\xd1\xf4\xc4\xde\x80\x9b\x94I\x87.\x81us8c\xcb\xce\x86O\x86b\t\xa0\xba\x0eA~\x0f(\x0c\x94\x90aT\xae\x07\xf5(\r\x92o*$z\xdc\xe3\xbc\xe1\xe4ov\x1a\xfd\x18\x11C>\xbb\xe3B\xd8z_\xbd\x84\xc9 \x82\xa3\xf0|z\xcf\x99Q7G\x0f\xde\x8b\x13v \xb2C*\x94A\x1e\xb7\xfb\xc1\x82\x07\xae{\xa8G\xbb\xd9\x9b\x87Z\x81\x99!:\x05\x11\x90\xf5,\xc1\xefpQ\x96\xdcX\xd5\xdc\xc5\x85\xe2\xa9\x8c\xea\xa2-\xe27\xc3\xfc\x18\x11\xf3\x87\xcc\xfe\xbf\x87\xa97a\xf1\xd8u\xf5\x84\x81\xa5\xd7\xd0\x07Y\xa2\xb0\x1a\xbb\r\tm\x1a\xc3\xd9\x118\xa34\x19\\\xe5\xf1}\xc0#\x8f0\x9fx\x01,\xcb\xc7\xde\x89\xeb\xab\xee\x8e\x8e\xe7\x87(.\xcf(g\x0c\xb6Oe{"\xc2\xc07;\x8d~\x8c\x88\xf9\n\x93\x04\xc15\xcd\x7f\xe94jG#\xc5\x0c"j\x01\xbb\xb9\xa3\x9aY\x95\x0f\xd32b\'\nh\xf2s\xdc$\xae\xe9\xde\xbcp\x0c/\x0b.\xe9\xf5\xd1v\x1e<5\'\x99yI:\x132\xf2\x92\x03\x8a\xc9@s\xc4\xee\xee\xd6\xa4\x99\x0f\xb1F\xdf\xecu\xfc1"\xe6\x19\xe6Z\xb0P0\xf8\xd2nP\x8b\x9a\xff\xcf\xde\x9d-;\x8a^\x89\x02~\x97\xba%\xc2b\x1e\xfa\x8e\x191\x89I\x0c:\x17\'\x98\xc5\x0cB\x08\xc1y\xf9\x832\xcb\xddv\xa9\xd2n9T\xce]\xddr\xb8\x1c\x8e\xcc\xda\x12\x8b\x7fZ\x0bi\xaf\xef\x86\x04\x1ey\xd3\x16\x98\rJ%\x98f\x94\xc92\x9b\xe9\xb5Ub\xc4\x0bJ\t\xbb\xdd\xb6@n\x94v\x80\xc0\x8b\x86\xcb\xf6\x12\xf3\xb2\x1abn\xa2\x89\xbeq\xafr\xa9\x8bx\xb2wg\xa3=\x8b\xbb\xcbP\xa8?J<\xfe\xe4D\xccv\x18\xa2\x0fj\x87\x00\x9f\xf2\x1a\x0fR\\\x12\xda\x07\xedp\xda\xf1\x1df\x95\x13%\xf2\x15\xd3n3\xc1\x9f\x9b\x1d\xd8_\x8a\x90\xdbe9\xe7\x12\xbb\x8e\xcei\x8b\xadB\xa7\xca\xcbqHOA\x88\x14\xecM\x94\x94\xe5Z\xd7Kon\x7fM\x18\x1a\xcc\xbf\xd8P\xf5mD\xcc6Y0\x02z\x98\xa8O-8\t\x1fd\x80\x93S\x9e\xd9=\xccMa\xb3\xf2v\xb8\xdaz\xe1\x13\xfb6\xa8\xa8C\xef\x9eR\xe6\xee\x1cB\xe2\xd0@\x97\xb8\x11pE\xb2\xedVM\xf831\x19\x9e\xdalgKP\x18R~;\xf2\xf5^\xa5\x93FW_\xdc\xe1\xdeF\xc4\x90\x8fN\xa3\xf8v^=u\xfe\x97\x12-\xd6\xcfG\xf2\x14o\xffO\xd2\xf8\xf3\x19\xc9\xc5j\x02k\xea\xde\xd4G\x0f=\x93Su\x82\x90d\xf6\x99:\x18S\x08\xc9GY95\x82\xcf&c\xe2\x0e\xeb\x80\x9em2,\x17\xa0e\xb2c\xd4e\'\xa3\xbd\xfdh\xe5\xff\xc9\x85\x98o\xa9\xfe\xb6\xb2\xd0\xe7&\xea\x17\xae(\xc1K\x92Mq\x18\xd9\xc1I\xb3}I\xc2\x8f\xb6\x82\xb7]\xd9\xda\xc4\x19\xe8w>)\xfb\xea\xb8\x9b\xa8\x03\xb0\x87/\x8a\xba\x9d,\x84\xec,\xd4\x14\xac\xa7\xa3\x9d\xcf\xbd5\xb4G-\xbfX\x8b(\xa7^\xd7y/\xf2wo\x13b\xb6\x95\x0fn\xd5\xe4v~>\xfb\t\x07\xffr\x15\x93\xfd\xe9\x82\'\x98 \x94F\x0c\xee\x97\x9c8\x15\xb5\x19\xdd\xc8;\xc1\r\xa2(\x8b\x15\x1f\x94\xb6{\xee\x046\xf0\x91\x04\x05\xf4\xfb\tL\x12\xdeO\x89P\xcc,\'\x18\x801\xa2\x84\x1b|\x9ct\x8f\xbd\xfe\x88\x13\xfa\xa3\x85\x98m4\xc9\xc7\x8fn\x19\xffo\xc3T\r\x89\xd5$\xf8p\xc92\x8c\x8d\x96\xf4\x9e\x93|\x83\xf2\xf9\xfd\xb4G\x9adgxkF\xa4pJz8~\xd8N\xadmN\xed\'\x97GL\x01N"\xba\xba\xcf\xcc\xcc^\xc6\xa9\x10\xef\xf2\xa2\x99w\xb0fP\x9a|q4\xdf&\xc4\xbf\xed\x95\x1a\xb8\xd6\xcd\xdc0\xc7\xd4\xc0\xae\xc4\x8d\xe8\xc4\x88\xe9v\x82\x13\x86\xc4H\x86\x85\xe3\xaa\xfe\xa9Bv?\x0b\x88y\xa46[\xf5\xf6x\x08\xfb\xc4D\xec&\xc7\xeam\xf2h\x13Ww\x16\xe4;\\\x16\xa6\x89\xa2b\thW\xd5\xa0p\x13n`\x08,8\x92\x9fa\x14\x88\x0e\x82\xa41\xe1\xc4\xed\xf2\x16Ia\xfdh\x10)Q\x99+\xce[\xe7\xc1*\x19s\xacK\xf5\xc5\xd4\xe6m@\xcc\xe3\xcc\xdf\xca!\x1c\x07\x9f\xb9\x9f"9\xf0\xb0\xb8s\xd4\x96\xd4:\x0e..\x91>\xe3q\x95\x87\xf5r\x9f\xd23\xed\x11\xd6H_\xb2\x08\xa1\x0es\'d:vv\x06,\xccD\xd2\xdcc+]\xea\xbaq\xbf\xed\xd3\x16<8R\x15\x0b\xf6P,\xfd\x8f6\xb8?9\x10\xf3H\x10\xb7\\s\x9b+O\xb5\xa1^4\x9e\x18h\xf0t:u\xf9N\x01\x0eW"\x92\x11\xfb\xbc\xfaZKW\x11S\x87\x8b\x9bsl\xd4\x0f\x03\x1c\xad2\xb1.\xadaZ7\xff\x12\xcf\x12\xaa\x90Y\'\xb3py:+\xba\xd2\xb4`y8\xd1\xdd\xe1\xf8b+\xfa\xb7\x011\x8f\xa5\xff\x98*\xc4\xb3\x92j\x9c\xae\xb1\xa2\xd6`\xa2\xd6\xed\xfe6$La\x12D\xa5\xc4^\x0c/\xeb\xb6\xcb\xd5\x84%\xc1\xa3\x9d\x1az\xb1\xf7\xf6\x14\x91\xd7{\xaaT\xec&\x97\\Y\x91\x8f\xa3\xe8\xe8\xbe\xc0\x92\x17\xab\x86\x85\xd8\x01\xfb\xca\x9b\xc6\x17\x9f\xd6\xbe\r\x88\xd9\xc2D\x1eX\x1a\xf4t\xe6_\x93\xfbpl\x94|\xe6\xa0k5{\x88|\xc9\x92\xcb\x1e^\'%\xe24IH|\x91\xcah,#\r47N\x05\x13\xdfQG\xf0\xe5j\xc4\xdah\xc6\x8b=\xd0Qj\xc5R\xc7>\xe2kp8\xa8\xb1-L\xaf\xba\x02o\x03b\xbe\r&\n\xc3\xc4\xb3\x05;@\x01\xa0\x0e\xedu\xd8\x1d\xda\xca\xb9KR\xeb1\x00\x82\xfa\xb7\x8a\x82\xaa#l\xec\x9a>t\xfa\xa5\xf6U\xd5\x06\xc2\x08\x92\xb4l_\xfbf.@\xa8\xd7h\xf2\x05\x1a[-\x0bZ_\xc4\x8e\xec\x99\x8a\x87&M\xe4\x9f\x05\xc4<\xf6q\x1cG\x1f\xec\xed\x93{\xa96k\x9c\x92x\xb2\x14\x18ab\x97Q\xea\xf7\xfa \x07\xc1n\xb0\x85P\xf6\xcf\xb1\xb2$\x11V\xba\xe7{\xe7b\x9ax?\xdf\'o\x7f\xc1\xceYv\xd1e\xebpf\xf0N7\xa8a<\x08J\x13\xd6\xf1M:\x89/\x8e\xe6\xdb\x80\x98\xc7\x03U\x02\xdc\x92\x0f\xf2\xe9\x83\x94\xc6K\xf0]@L~6\xee\xe6eAs\xbdN\x0f\xf0\xe0\x9e\xc2\xd1\x8d\xae\xdc\xedh\xb1\'5\xcb "\xf4\xd1\xfa\xd2+\x97\xf5\x88,\x8b\x9e\xd9Q\xd1\x1b+7\xc2\xc3\r^\xc9\xda\x10\x8br\x07\x0cEs\x10\xa6\xcb\x8fN\xe5?9\x10\xf3X\xf9\x8f\x02\x18yN\xe1\x04\xde\x98\x91\x03y\x11\xc6\x0c\x0f\xe3S\xa2U~n\x19\t\xe5\x10\xbb-y\x9b\x97KW\xcc[\xc2\x83\xb6\xe9uF-}\x88\x08\x1d\x11\x15\',hy\xb6G\xffNN\x13\xb5\xe8Fz\n\xbd\xf1\x00)\x13\x7f\x0b\xe4W\x0b\xfdw\x011\xe4_H\n\xc5\x1f\x9b\xe2\x93\x12\x01g\xd7\x11\xbcR5E\xea\x17g\xe7K\xaau_T\xef\x90\xd1\x92\xe5\x16\x9eu\xd9\x023.\xc7(\x03\xcf2\x197\x15\x81\x0b\xe6N\x96K\xe9\x8c\xcfG\xf4N\xaf\xd4\x0e\x9c&\xb1\x86]\xa7\xf5\xcc\xf3>\x99\xd3\x1e\x7fqM\xbc\r\x88\xd9\x96>\xb1\x9dr \xf6,\xfb.2\x0e$\xc1U\x15f\x8aG:\xa9\xbc\x9c#\xee\nt@i\x9a\x98\xd83\xd4p=M\xfe]0\xf7\\\xdc4+\xd6\x8e\'\xbbhn\xbe1\x1b\xfeI\x9d\x8a\xf2L\xdcD\x8e\xda\xebw\xa7\x1c\n\xedJ\xcd\xc0q\xf7b\xf1\xf66 \xe6\xdb\xd3)\n\xdb\xe2|:\xaf\x12\xa0\xcc3\xae\x91\xe2>\xb4\xd3\x9d\x0c\xec\x9c\xb6\xa9\xa8]u\r)\xa5\xb1\\~\x8c\x90\x99\x9e\x04\x7f\x15\x12\xc2R&\x06"(1\xaft\x9d2\n\xb1\x10 \xa0\xd82\x9a\xe5\x1c\x0e\xa3u\xe1nE\xd0\xb7"\'\xbd8\x9ao\x03b\xb60\xb7\x84\x1e\xd9\xe6\xed\xd3F\x8e\rC\xed\xe2h\xee>p\x12\xa5:\x9f\xa0\x0co\xcf\xac\x82\xd2iA\xe4\x05\xb5e\xe2\xfby\xa7PYW\x14I\x86\xe4\x1a\x04\xc1Z\xef5\xa8\x83e\x87\x0c\x00\x12]\xbe\xcf!\x89k{\xe0\x9c\x96b\xb0\xcd^\xe1k\x01\xc6o\x03b\x1e\xf9\xfeV\x1cA0\xf1\xb4\xf4K\xd38"\xb6\xa7zNU\xd7\xc1A\xd6dj\xbd\xdeN>\xdc\xe9c\x86\xb9G\xb8\x1fLQ\xdcQ\xad\xb5h\x90\\\xa3\xee\xc1\xbf\x00\xab \x8d\xa8Y\x01c\xbe\x80`\xd6E\xc2\xd8\x982\xbb\xeb\xc9\xf1\xec6-\xfd\xfd!\xdf?\x03b\xbe\x7f\xa5\xea\x03\xc4\xfc\xb7\xbf\xa9\xfe?\x07\x88\xf9\x139\x11\x1fz\xe3\x03\xc4|\x80\x98\xff\x8d\xb3\xf4\x03\xc4|\x80\x98\x0f\x10\xf3\x01b>@\xcc\x07\x88\xf9\x001_\xf7:?@\xcc\x07\x88\xf9\x001\x1f \xe6+\xc3+\x1f \xe6\x03\xc4|\x80\x98\x0f\x10\xf3e\xaf\xf3\x03\xc4|\x80\x98\x0f\x10\xf3\x01b\xbe2\xbc\xf2\x01b>@\xcc\x07\x88\xf9\x001_\xf6:?@\xcc\x07\x88\xf9\x001\x1f \xe6+\xc3+\x1f \xe6\x03\xc4|\x80\x98\x0f\x10\xf3e\xaf\xf3\x03\xc4|\x80\x98\x0f\x10\xf3\x01b\xbe2\xbc\xf2\x01b>@\xcc\x07\x88\xf9\x001_\xf6:?@\xcc\x07\x88\xf9\x001\x1f \xe6+\xc3+\xff\x93\x81\x98\xf5\xefv\x97\x7fJ!T\x7f\x7fI_\x18\x88\xf97\x04\xf6s\x80\x98\x7fC`?\x07\x88\xf97\x04\xf6s\x80\x98\x17\x03\xfbW\xb8\x91\x9f\x03\xc4\xfc\x1bF\xec\xe7\x001\xff\x8e\xcd\xe3\x03\xc4\xfc!@\x0c\xf4_\xf7\xfc\x1f\x031\xb8\x80\x10$\x87\x83\x08E\xb0\x1c\xc1\xe20\xff\xe8V\x88\x90\x8fv,\xb4 0\x02\xc9B\x18\x811\x08\x88\xd0\x0c\x03A(\x89\x12\xdf\xfa\xb6\xd0\x10\xcc3\xe0\xe3}~l\x1f\xa0 \xce\xb1$\x88a\xecCR\x80a\x16\x12@\x98\xe49\x8e\x10\x08\x9c&\x10\x82c9\x08\xa2\x18\x9cf!\x9e\xe1P\x8a\x82H\x96\xe50\x1aF\x11\x14\x13\xb8?\x12\x88\xf9k\xcb\x8e_~\xa7\x01\x03\x8c\xfde\xbb+\xdb\xb0 \x10\xf9\x8f\x80\x98\xaf\xaf\xeb\xfc\x0e\x10\xc33\x98\xb0\r#\x0f\xa3\x0c\x84\xe1\xc86K\x91\xedn\xe3(\xc4\xd3<.\xe0\x02C\x10,\x0bQ\x10\n\xc3\x0c\x89P\x08F\xa2\x8c\xc0\xe38$\x08\xf4\xf6\xbf\x8f>k\x1f \xe6\x0f\x04bHN\x80\xa1\xed\xd2\xb6\xe9\x84\x0b\x1c\xba-\x1d\x0c\xe7`p\xbb\n\x02}\xac\x11a\x9b\x05m\xe4b\x91\n\xdb\x11\x81\x9e\xbc\x1b\xe3\xc7\\Z\xb4a\xb5\x8fV_D\xb7#L\xdd\xf1\xe0\xad%\x99\x94\xdc;\xe1\x05\t`\x91/\xd4\xda\xc7\xc4\xbe2\xd4\xe3\xe5\xa2/:@t\xd3\x8ejn\xf6L\xd5\x9d<\x81Ll\xac\xdck\x87\xd5\xbb\x88\x98\xdfVj\x7f\x1b\xa6e\xe5z\xc7\x98;\xe0f\x9d\x9d[;ZN\x95+\x07\xc9X\xda`\x9f!+w&\x8d\xce\xc6\xed\x8348]I\xaeB\xdao\xbb\x10\xd0E=\xc4\x1c \x03F\xb4Z_\x80\xa9g\x99;$\xc44\xce\xd7\xcb\xba\xffZ-U\xdfE\xc4lw\x11\xc7\x1e\xb9\xcd\xef\x9c\xfa\xc4n\xdf\x89\xc2\x04r\xc6\xc1L\x14\x82$\x0e\xdbd\x07\x87\xd1\xa6\xe8\x98G[\x105\xa3\x9a\xe3\xb8\x1c\xbdo\xeb\xa0\xe6\x8e\x8c\x8a_\x88i>\x18j38\x07\xdb\xbe\xb5\xebM_(e\xa4\x82\xadL\xba\x0e8]\x17\xdc\xcf!b\x1e\xe7\xd5\x96\xbf\x81[\xc9\xf4\xb4\xc3\xcd`\x93\x01\xea\xe1\x0eM4\xe6\x99\xd2\xded\xf8\x92:\xdb"d\x87\x8cG]iO\xb3\xe7c$\xfb\x01BC\x16r]\xae*F\xa0p\xcf\xde\x0fts\xa6QZ,\xcfB\x18\xeac4\xb6\x99\xb2\x1ag9a\xf6/6\xff\x7f\x17\x11\xf3\x08\x13|\xe46\xc8s\x98#\xbaTy\xd9D\x00\xc3\xbaM\x1a\xe6\\\x9d\x9dED\x9d\xd08D\x98N\xb2\x06\xe1\xea"\xad \xcd\xf7\xb3Y\x0f\x8ar\xbcE&\x89\x82\xe6\xa9\xd5\xe8\xbeF4lu\xa1\xdeKG\'\xeb\x12\xffAvhp\xf0\xa3\x1d\xee\xcfm\xc4<\xee"\x8e<\xf2\xccgc\xcbO\xe1=\n1R`\x16[)\xd6\x1f\x90K\xef*\xb0\xb5\x10e\xa2\xc8:\xe2\xc69\x17L\x17T\xabA/\xcd\xceYi)\xc8\x11 ;A\x99\r\xd4\\a\x11\'\x1b\x11\x06\x17\x8c\xd2:\xe7z\xd5G{$\xd5\x9fd\xc4<\xce\t\x10%Ah+i\x7f\x1bf\xbf\xdceX\x8e\x8f\xday9u\xa7\xc2\x853\x1a\xca\xd9\xb6\x08\x0e\x17\xca \xa9\x91\xddO\x9e\x04\x83\xe3}\x0c\xe0\xb3N\xacYc\xaaR\xdd\x12\xc65\xde\tS;\xd7\x19\x16\xa6w\x92\xf0\x8f\x03P\xaa{ihv\xc2\x8b\xdd\x94\xdfe\xc4|;\xf5\xf1\xedDD\xd1\xa7\xc6\xb4d\xa0\xaa\xa5\xde\xd1{\xcbr\xf2\xcc\xc0\x8aYl\x10]\x1f\x01LdXK!4\x04\x12\xc5\xa2\xac\xc2{\xdd\x1a\x12P\xf0\xcd\x8d\x80\xc9\xbb\xe2\xf2\x91\x8c\x99\\\t*\xc5\xa2\xf3\xab\x9b\x0b\x97y\x8d\x10\xa1\xe3\xfb\xdb\x8bK\xff]F\xcc\xb7I\x0bn\xb7\x05$\x9eRU\x96\xdd\xddv\x04\xd0\xf8\xd6^\xddrk\x89\xc7.\x01\x0c[3\t$\xcb\x84\xaf9~\xc1v<_\xe2}=E$\xe5W\x908\xe3\xde|s\xef\xce\xec\x82\xca\xe42\x00\xe5\xb1\xc9\x99\x92PZ8\x9d\xfa\x96\x9c\xaf\xc7/\xb6\xf4\xdfd\xc4<\xee\xe2v\xcbI\x94\x00\x9f\xeeb\x8eY\x08\xc7R\x9e\xc3\x96@\xbb\ng\x89(:\xdf\xad\x80\n]\xebKJ"D/C\x07\xeer\xdf\xebmc\xc1\x15 \x15V\xcfC\x86\x19\xc0\xf3\xb0\xec\xf6$\xa6\x98\xbc.\x013{>\x92\xf7\xfbX5%\xd5\xbcH(\xbc\x0b\x89\xf9\x96;A\xdb\xa9\x03b\xcf\x8d\xd4\xef\xac\xa5\xcb{\xb4\x08\xb0\xb3\n\x9aW\xd0\xb0\xab\x11\xbd\xfb.\x82^ J\xbc(\xc9\x1e\x1aB\xde\xb2\x91)\'\xc6\xb5\xa8\xf6\x88\xd6\x05ewN$>\xa7\xcd\xa0\xefE\x98\xc9h\xbe\xbd\x81\xfa\x80&\xf5\xc5%\xab\x17\xeb\x9aw!1\xdf\xaaT\x02\xc5\x10\x9cz\xaaR\xcd\x98\x19z\xde\x82\xa3)Y\x8b\xe5r\x99]\xc0\xbdh\x19X\xe3\xb28\x9a\x91\xdc@\xf7{\xcca\x11\xd1\x03ve]1\x94\xf0\xf0l\xbbDu\x0b\x131\xd3\xe2\xc2\xfa\xf9\xae\x00\x03\x85\x91\xed\xfcP\xb2N\t%/Z8\xefBb\xbe%7[R\x8d\x81\xd8S]\xa3\x8c8\xe3\xdev\xb0`\x80\xbarr\x9b\x1d\xbe\xf8\xee\xca\xa1\xddp\xf1\x13\xb2DY\xcb\xf7\x8f\x9dK\x19\xfd\xbe?\xdeT}\x1d5r2\xab\x18\xbd\x81<\xe2\x9fb\x16\x92\xf4m\xa2\x9c\xc0\x82\xc9\x054\xada\xfa<}\xad\xa5\xff.$\xe6[\xad\xbfm\xbe\x04\xf9|\x1c\x02\x06\x8f\xe1\xd7\xabH\x87\x0bM)\xed\x80\xea\xda|J\xee\x18\xef\xf9\xd6\xc1\xb0\x8e\xc9)\x9eZ\xf6\xe2z\xe0MM\x90\xe0p\xbd#\xc0\x1e\x12\xc8\x84\xb8t\xd9\x10\xe4h};\x1a\xf6\xfe\x1c\xa7\xc9\xce\xbcne\xb6+L/\x9e\x13\xefBb\xbe\x1d\x87 \xf4xd\xf6\xb4&\x0e\x1dp\x91\xe4\x1bsH\xf7\xf3\xc5\x05\xeex\xdb\xf2\x8dj_e]8\xe2\xac\xd4"\xfe\xd8\x84\x16#\x1d\x0f\xf20_\xe2\x92\x9b\x9b\x03\x08\x9e\x8fAu<\xef\xd4E\xcbc\xa2\x1c\x0f{b\x079Rt\xd3Jc\xed\xe0\x9f\x84\xc4<\xc2\xc40|+\xa4\x9f\x19,\xbd\xb4K^,\x87\xe3\xbd(\xd1~\xc0\xcb\xe3.\\q0\x15\xe7\xe2:\x1a\x84\x9c\xebl^tX\xd1\xee\x9a\x18\xd2;\xc5\x8e\x14\xb6\xa2\xd0T\xe3)x\x9dt\xbb\xb6\x92=H\xdc\xe0A\xf1\x91ei*f\x9c\xb8\x17\x1fi\xbcK\x89\xf9\xb6\xc3\x81$L\xfe\x8e+\x12\x90&T\x8f\xa7\xc1\x08\x86>\x85\x87\xbd\xa7\xafY\xe9\n\x19\xd1\xf0P\xe3Q\x15\x9e\x12\x07\x89N\x95\xc1\xc4\x16-\x94\x0bS\r\xc1\xe0\x90\xdc\xb3\xbdpwa\x1eM\x8fy\xcc\x82;\xe3\x14\xf4j\xdd\x05\x17W\xc2\x8a\x17s\xb8w)1\xdf\xc2\xdcb$0\xeaim*\xbewmsX\x93\x8dk4P|\xeciZ\x0bs\xa3/\x80\xa0\x016\x8aT\x15\xc4\x14x,\x14T\xb1\xbfR7\xcdQy8\xad\x96\x00\xe4\xf62\x7f`\xe7\x9e\x1fwJ\xc6\x8b\xf7}2\x85\xd1DF4a\xbd\xb8\x91\xbfK\x89y\x84I\x110EP\xe8\xf3\xe3\xc6\xe1\x16\xc1mF\xf9\xf7\xbb\xc4&s@\\\xfc\xdb\t?8c\x1aa\xa5\x90\x9f/W\x95\xe5\x8d\x87o\xe2.cKx\t\xe0{|\x13\xab\xe5d15\x977P\xc9\x92\x17\xae\xa5\x8ep\xc9\x9e\xe6\xca8\xf6\xbc\xf8#\xc3\xf8\xcf\xad\xc4|[\xfa8\xb4\xed\x16\xcf\xe0u\x0e\xa9\x97\x9c\xf1\xeb\xa4\xd5\x0c&-\xf3\x03\xa6\xe6\x01 \xe1\xcbHXJ\x16\xb8\xc8\xc9@t\xae\xc0+\x0e(\x11\xcb\xeafg\x02\xc3\xdb(\xc7#\x83\xb0y\xb7\xaa\x93T\x91\xe6\x18\x90\xb9 \xaam\x90\xd2\xf50\xbe\x98\xdc\xbcK\x89yL\x16\xf8\xf1\x1c\x94|\xfa\x1c\xe3\xb2\x1e\\\x82\xd2\x94\xc5\xe3S\x8d\xf5}K\xe1\x98\x8e\x08q\xcb\xd5\xa9\xd3\x04d\xc0\xba\x17\xad\x9d\xd6\x94\xaeO\x17\xa7\xbe\xbb\xb0\xc7\x02Z16\x1a\xf5\x1d$\xefsSQr\xff\xec/\x84\xc9u{]\xe8\xd3\xc4{\x95\x15y\x17\x12\xf3(R\x11j\x9b\x0c\xd8\xf3\x07\r`\x92\xc6\x88\x9e\x1e\x02j\x07\xc2\xd8\xe8\xa2H-\xe50\x03\x00\xc0z\t\xf1\xc0\xf2\x8f;Q\x1bf^\x19\xe1\x9c\x99\xb8\xa9\x9b%\xd2\x1f\xe4+A\x1b\xab\xde\x91\xf6\tHJ\x0b\x9d\xd1\xc8O\xcb\xdaQ\xb1\x13q\xa3_\x84L\xdf\x85\xc4|\xab\xde\xb6\xcd\x02\xdd^\xe5\xe9\x99\xadgs\xb72;\x06\xb6\x93\x91w\x15\xd5\x8f\x84\xe7\xac\xe3x\xa1BH(X\x9b\xf2\xd4q\x1fIdXM\xb4\xa6\xaa\xad$wwowC\xab\x83\xe5\xa1_\x9b\xba\xc6\x1b5\xcf]\xbc\xc6\xf1\x0cq5S\x0c\x10SmH\x08\x8c\x12\xb9\xa6\x8e\xe5V\xa3\xe2b9V\xdf3\xd5\x7f\x8a\xc4|\x9b\xe4\x1f$\xe6\xbf\xfdm\xf5\xff9H\xcc\x9f\x88\xdf\xf8\x88&\x9f[\xfa\xf5o\xe9\x07\x89\xf9 1\x1f$\xe6\x83\xc4|\x90\x98\x0f\x12\xf3Ab\xbe\xf0u~\x90\x98\x0f\x12\xf3Ab>H\xccW\xc6W>H\xcc\x07\x89\xf9 1\x1f$\xe6\xcb^\xe7\x07\x89\xf9 1\x1f$\xe6\x83\xc4|e|\xe5\x83\xc4|\x90\x98\x0f\x12\xf3Ab\xbe\xecu~\x90\x98\x0f\x12\xf3Ab>H\xccW\xc6W>H\xcc\x07\x89\xf9 1\x1f$\xe6\xcb^\xe7\x07\x89\xf9 1\x1f$\xe6\x83\xc4|e|\xe5\x83\xc4|\x90\x98\x0f\x12\xf3Ab\xbe\xecu~\x90\x98\x0f\x12\xf3Ab>H\xccW\xc6W>H\xcc\x07\x89y\x85C\xf8\xbb\xe4\xf7\x9fq\x08\xebo\xf6\xa2\xaf\x8c\xc4\xfc\xf1\x81\xfd$$\xe6\x8f\x0f\xec\'!1\x7f|`?\t\x89y)\xb0\x7f\x89\x1c\xf9IH\xcc\x1f?b?\t\x89\xf97l\x1e\x1f$\xe6\x0fAb\xe0\xff\xba\xe7\xff\x18\x89\xa1\x11\x14\xc4\xb6\xb3\x18\xc4P\x04#8\x0e\xa5I\x9e\xe7\x11\x96\x00\x05\x84AXF\xe0h\x1e\xc7@\x18C \x12F\x10\x81\xe7Y\x9acX\x0c\x02\x1f^\xc5\xb7\xf7\xf9\xb1\x7f\x00o?\xbe\xfd\xeb\x08\t\x92\x18B\xc3\x8fv\xae4E\t\xac\x80Q\x04\x8fQ,\x85\xd3\x08\xbbE\xcf\x0b\xdb\x9b\xb0 OC4\x8d\xa2\x18\x8d\x13\x08\x82\xb2\x82\xf0G"1\x7f\xed\x17\xf6\xcbS\x03\x06\xfc?`\xf8/ \x84\x10\x04\x81Q\xd8?Bb\xbe\xbe\xb0\xf3;H\x0cHq(\xcb\x81\xf8\xf6\xaf18\x87\xe2\xd8v\xb1\x10\xc1oS\x00b1\x82"9\x16D\xb6\xa1\x86x\x90\xe0\x1e\xbd\xb9p\x16\xc7\xb7yAq\x14K\xd3\x02\xf1\xe8=\xf8\xb7H\x0c\x81s\x10L\x10\xfcv\xa9\x14F\x900\x89\xe1\x14\x0b3\xf0c\x04\x91\xed]H\x92\xdc"\xa58\x06\x81\x1e\xa0\x06\x082\xf46\x1dx\n\'8\x84\xa3\x84_\'\xea\x07\x89\xf9O$\x06DY\x8c}\x00$\x10\t",\x89\xe0\x18\xc5\xd3\x18LC\xdb\xdbp\xe06$\xdb\x803\x82\x80`\x14\x841<\xc9\xc00\xb3\x8d\x9d\x00Q\xe0\xf6\x17\x04\x8f\xfc\xf2\xbf\x16\x89\xf9\x85\xedI\x86\xedg\x86-\xb4[\x00Sk\xe4Q\xab\xda\xea`\xe0]\xfb\xa8\x89\xa7\xa4\x14\x8a\x93o\x9d5\x1b\xd3c8\xe1\xb7\x92\xb8>\xc1\x02x\xb2d\x88-\xae\xb7\x18\x86n\x9ag\xd5:"l\xb993\xa7\xa5<\xe8M2\x1d\xbc\x11=\x88\x18\x98n\x89R$\xbac\x08\x0b\xf8\xa9\x19\xd7\x83\x97 \'\xd1\xc2\xf8R\xfe\xf6\xb3\x8f\x1a(B\x98:.\x85\xea\xe4$\xf7\x94\xa3\xe7X\xea\x96p\xcb\xea\xa3\xb2\xbe\x1c\xb8\xf3\xb4%[h\xba\xbdo$^\xcfis\xbah\xf0}L\xa5\x18\x8e\xcf\x9d\xcd\xe6.\xf4\xed\x84\xbc\xefe\xdd\xa1g\x8d\xfb\xfe\x0f_\xe7\xca\x91\xeeE\xf6L\xfeW\\\x8d\xbe\xa4>\x03F\x0b\xb6\x9e<}\xd6\xca#\x1c\xae\xfct\x94\xdc\xc2\x86]\x8c/4eo\xea\xf6A\t\x17\x99\xd3\xb4.U\x0b \x12\x9c\xc3\xbd\x0c1\xbc\xab\n\xf4\x80\xa8C{\r.\xb3w\xc8\x80\x10\xb4\xf6<\x9b\xef\xf7\xb4\x953|n\x9a\xb3\xdcib\xc5\x1d\\\xcc:\xd0\xbc\xd6\xc7\x95\xe0\x9a\x873\xb3C\x17\x8b\xb6\xa8<8]bsA\xd7"\xac\x96\x8b\xcc\x1b\xf7\x15&n\xd2h\x01\xf7\xd9\xbb\x00\xa2\xac\x19\x04z\xdc\xd3\x0b@b:\xe9\x12>d\xa6d\x1c;-:!\xfb\xbd\xe1\xac\xc9\nFz\x96Zlx8JIE.\\O\x8f\x12\x0b\x10\x98\x14\x8cD\x04\x83\xba\x9b\x8f\xd7\xc2\x01\xa7\xb5\x84[\xe9U:\xe0=\x9c\xcf\xf7\xf3\x1b\xa3H\xfc\xb9\xc7/^\xce\x8azM`D4\xea\x05t\x8eD\x95u\xc6\x94\xf3;\x17\x8d\xd91\xd99\xf7\xfb\n\xc59\xc8\x97\xa0\xa0tg\xa5\x83\x1d=`H\x87q\x95\x02PU\xaa\xd2\xb0\xa3\x95"r\xe2Z\xd6\xc9\xd9C\x90OI?\x05\xf3\xf9\x16$\xb4\r$\xf6\xd4\x08^;\x84\xe0h&s]@\xde\xb0p\xc8i\x0f\x0e\x99\xea\xed`~\xc1\x9b\x0b\x11\x91\xcb\x8a\xe5\xb8U\x1c(c\'4NO-\x17\xedx=D\xc7\x83\xa6\xdb\xbb\xb4\xcf\x14\xfe\xca1uX\xe3\xb3q\xb5(eM\xd2\xe0G\x9d\x04\xffP\xcc\xe7{\x8c8B\xc0\xdb)\xfa\xd4\xea/X\xa0%PS\x8cl\xa4\x10X\xc0\x868\x91X\x82JWK\xdf\xf6o\xc0\x14\xb7\x03S\xa2D\xbf\xbc\x1eH\xd5\xb9\xf2w[\x86\x0e2?\xed\x95C\xc1\xeeT\x83\xc7\x84\x89\xe0\xee\x06\xddl\xc5\x03\xadTB\x80\xa4XA=aByQP\xda\xd2\xd9{|\x18\xcf-\xe9\xd1\xc5\x91}\xb1\xbb\xe6{0\x9f\xefa>\x8a\x18\x88z\xea\x9av4p\xde\xc9\x05\xc7jt\x8b\x82\xf4\xfd\xba\xcf\xaa8\'\x8f(x&b)9\x9cB\xab\x0e\xc2\x14:\xd9A\x03_(\x02\xa0aU`vKzC\xf2\xb8\x85yk\xba&\xf9\xe4{U\xa6t\x9c#l\x07\xe3L\xbf\xea[\xbd\xc5\xf2y\xaeM\xfe\xae\x87\xa8Kq2\xb1P\xd8}\x19\xb1\x86P\x88\x80\xeb\xc3\x9b\x85`\xd2\xedj\xcf+I\x0f{\xed\x88\xe4&Ot\x0c\xd8z\xe4\x96\x0c\x04jq!\xf8\xdd\xe2W\xae\xe2\xf7\x92\'\r\x86r\xdb\x0e\xd8cA\x0cI\x14\xa3\xfd\x8b\xc8\xcd{,\x9f\xefa"\x04FnK\xff\xa9\xbf/\xb6G\x1b\x06\x80A\rT\xc6\xa52\x9cCji\x15\xac\x9al\x84\x05\x0cr\xbc\x1e\xf9\x8c\xc9/<#\xcb}r\x07EIg\xb7\xf4\xcc.-uny0\x1bjM\xac\xc9H)\x1a\xff|:\x82\xc4"\x82\x95\xe5\xbe\xd8\xf8\xf6=\x96\xcf\xafs\xf6\xd1\xea\x8f|>\x90\x17>\xe6&M\x1d%\x1c\x84vX,\x9b]vFV\xef\xe2\x9d\xf1\x85J\x89]I:\xed\xc86\x85\xcaHi\x17\xaf\xa5\xbb\x17I\xa7\xac}\xa9>{\xc4\x8e`\xd8\xd0\x1aA`\x17\xe8:)\x98\x11\x84\xe4[\xae\xc3\xbc\x16\xe6{,\x9f_\xc3\xc4\xb0\xdf\xdd\xc7\xef\xf6\xe4\x0bx\xc1b\xeey^/\xd2\xe0\x06\xd5\x8e\xcb\x1c\x9c\x0cqb\x92\x896-\xf6{\xe8,\xc2\xdce\xd0\x12m\x9b^Eu\x84\xb4\xcb\xc0\x97\xfc\x99\xbb\xae\xf0\x12\xd8\xd9)\xad\x99\xa1\xf0\xbb#CXB\x0e\xec~\x94w\xfc\x99-\x9f\xefw\x91\xdc\xca@\x02$\x9f,\x9f\x89E2\x94\x94\xacy\x86\xddmgf\xbd\xb4A\x07\x9e\xc3\x95\xf9(\x0c\xbc\xedQux;I\xe6\xb9\xcb\xb3Zd2kj-\x85\x83\xaaS\xd6\xa1\x9d\xd1\xf0Hzw\xed\xc3[\x98\xd0_\x1eu"\xf80@~\x1bf\x84s\xe6.>)\xe9U\xb8Y\x81.[8\x13P:\xa0Y\x02\x13\xa0\x91\xda\xd4\xcc\x15\xd9\xe7\xc8\xcd\xe7\x11/\xbb^\xe1\x88cl\xcf)\xe3" \xa1<\x10/\xf2\xfd6\xe0\x96\xaa g\\qoi\xd9\xeen\xd1\x8b-a\xdfc\xf9\xfcuM\xe0\x14J<3^\rZ)q\xb4T\xe6|)\xce=\x9c\xaeJ\x0fF\xf84f\xdb\x06^\x9c\xe9\xdd\xad\xe9Z\x12\xc9\xa3\xb0h/\xa7X\xc6\xa6Z?v\xe7J\xd9\xc9\xad\x17\x07\xfd\xec\x19\xbb\xaa\x9eI\xc9<]Z\xf3V\x8d\xf7c`}\xad^\xa9\xef\xb1|~]\x130\x84\xc2\x0f\xc4\xe07w\x91\xd8\x9f\xa9S\x1cJ\xe5\x12\x8a\xfd\x84\x89\x81\xcas>\xa8;7\xc9,\xa20\x0f\xd8\xfb\x81\xb80\x9e)\xcc\xc5\x01\xc0\xec\x04p\xad\xd6\xc8XV\xb5\xd1\xb493\xd5\xbe\t\xb1r\xacl\xf7V\xde\xd6\xd2\xf3\x85l`_lw\xff\x1e\xcb\xe7{"\x0c"0\xbem\x7fO\x1b\xa8\x81W\xa1\x7ft\\z\x8a\xc2\xf3\xb5\x9a\x8fC\x02\xdcJ\xed\'\xc6\x0bB\xd3s\xb9\xccl\x96\xc0\xb4*k\xf1^\x05-3"-nm\xc2,\xb80\xf9Y\x10\x8b\x83\xe9\x0590n\xc3{3y<%X\xc6\xa5\xfb\x1d\x17A\x85\xb7O\xc8\\\xd6\x01`\xa6\xa5^\x8bOx\x98#/\x96n\xef\xb1|\xfe\x9a\xdc\x80\x04\x01\xe1O9\x1cl\xa3u)\xe02S\x82$E\x0ev\x0b\xe1{|\x81\xd9\xdb\x01l\x11\x1bP#\x08\x97\xb3\x03l@\x89\xa2\x15\xde9\x9a\x91\x0eX\xec9M\xd1\xd5\rJ,\xc1\x0b\xa8Z\x18~\xcev\x1dx\x19E\xd6\x1d\xae\xc7\x1f\x8d\xe6\x9f\xd9\xf2\xf9~\x17\x1f\xaf\xb0\x95\x97OeM\x94&(c\x85\x15z\xe5\xf6\xa4d+U\xbe\xda\xe2Y,1\xc3eXnb=\x07\xbf\xd2\n\x1e\xde\xf7\xedV\xf2\xe8\xf8.%\xe9\xae\xa8\xe7\x12\x17f\xf7\x9c\xf3U\x9eN\xb3:#\x07\x16YP\x15S\x1b\x85\x7f\x95\xb6{\x8f\xe5\xf3\xeb\xc3\x0c|\xcb\xa2\x9f\xa3d\xe5\x14\x9ef\xf8HN\xd0Xx7\x0b\x10\x9c\xee@\x82\x8dKQ+\xcb1\xb4\xdd\xa7\xf8\xa9\xc9O\x1c\xd2Em\x9e\x10@\xbb4\xe9<\xfb\xa2\x7f\x14\xe6\x8a\x99R\xe1X\xec\xecSm\x83\x0e\xe8\xdd\xb1\xb8;\x13\xafz\x1e\xef\xa1|\xbeop(\xba\x15\x06\xf8s\x86X\x97\xb3.\xf7\xd0\xb1\xee\x87\xa3T\xad\x12\xefM\xd2\x8e\xce\xb8\xc6\x16\x0b\xa1n\x89`\x9a\x03Yq\x873hM\x8d\x19\xf2\xedV\xcb\x8a\'\xb9:\x96\xc7\xf6 A\x86\xa7F\xbe\xe6k\xe40;,\xa5\xb9\xc4\x0e\xe1\xe7\x17\xf3\xfd\xf7P>\x7f\x1dL\x04\xd9\xee\xd8S\xbe\x8f\xee\r$M\x93E\xa0\xc8]\xca\xccG\xc3\xaf,>w]\xab\\+\\\xc1hU\x8c\xf8"\xf5\x84\x12\x8d\x80r>\x05\xecL\x1a\xc3\x08\x17\xa3\xe7\x1e\x03\x0f\xe0R+<\xecg[\xd9\'\x94\x0b\x82E\xa6\x02\xed\xe5k\x1d\xfa\xef\xa1|\xfeZ\xe9\xa38E<\xd5\x86)\xb3\xea0%\x0e|\x13xb#\xf1m\x97\xe3\xc2Q\x9bIy\xda[d\x9e\x8cA\x15k\x82\xae\x97\xcaP\xa0kb\x81{\x10\x98\x10\x01fv\xf0\xcd)\xd6\xac^D\xda+\x8cE\xed\x96C6\rM\xaf\xe1\xe0\x8b\x95\xfe{$\x9f_O\t\x94\xa2H\n~\n3\x83\x14S\xb9\xc9\x0f\xd3\x01\xd6\xe0U\xdf\xb1\xcea\xef\x95\xc7]%\x9b0\xdcBy\x8e^\x9aA%\x86\x1b\xe0\xdbC7Z\x17\xd5\x15\xaf \x00\xfa\x1a\xca\x04\xb9\x99\xda\xd9\xdd\xb81\xb5\x08g\xc5^4\xdc\xdc\x99\x98\x17\xc3|\x8f\xe4\xf3\xd70\x1f\x0fn\xc8\xa7\x95\x8f\xb2\xeb}\xeawj\xb8\x06n&\x8f\x0eV\x06\xa4\x1c\xe1\x02RvG,\xcb\xf1\x93\xa2\x1e\xe64\x10\xa3\xe5:R\xe1\xe5\x04\xdd\x85\x06\xea\x0f\x00tj\x92\xabR\x0c.b\x96\xa0\xa1\xd0\xf4\xe1^N"\xe1R\xf5\xbdy\xf1\xcc\x7f\x8f\xe4\xf3k\x980\x8c\x90[E\xf4\xd4\xd4_\xf5J\x8dCO\x02\xde\x1f\xaa$u\xa6"G#\x89\x87\x1e\x9f\xe3`T\xa0T\x06?\xe2\xb8\x12Y0[d\xde\x99@\xba\xd1\xbe\x1d\xb3#&\xdc\t/\xacvz1$;\\\xb9{\xea\xe9X\xab\x80\xcc\x82\x87\xf3\x8b\x9e\xc7{$\x9f_\xf7q\x1c$\xb6\x8c\xef)\xb5a\xc0\xfd\xf5\x00\x83-|\xb8\xd2$\xc8q\xbe\xa4\x0f\x1d\xe9\xa8(\x18z\x9a\xd7\x05L{/\xd6\xfe\xda\xc5\xd3u\xa8u\x01\x16\x11\xb9E\x8b\xde\xd4\xf5\x84\xda\xb7nj\xb2^\xe7\xd5\x1c\x97u\x04\xe1C\xd4.;\x89\xfa\x8bp\xf8{$\x9f\xbf\x96\xe2$\x0ec\xf8\xd3\xa4\x15t[\tg\xbc\x0f\xa81@\xaf$\xd8]z\x03g\xef\xf7\xa4JUE8\x9c\xf7\xdcaD\xa5S\x80\xa6\xacs\xd5\x84D\xe9\xc7\na\xe7\x8b\xedJ\x05\xa2\xf0\xb8y]\xfc\x8b\xdaS\xfc\xdad!9-\x02\xdf\xf3?\n\xf3\xcf,\xf9\xfc\xba&\x1e\x0fl\xd1g\xb0\x9bJf\x8a\xc2\xbc\xae^\xa2\x86:\xb7\xda\x94\xe3\x88\xd3\xee4y:\xa4\xa4\xaf\xba\x1a\xd3KFa(Q\x92\xb9Ys\x01\x81\xccG\x82\x83zo\xd0\t\x94\xca=\xc9kx\x9dhU\x8f\x9c\xd6\x82\x055JGz\xf6\xc5\x0c\xee=\x92\xcf\xf7\x07\x1a\x8fI\x87\x83\xd4\xd3F\xdeT\xe7\xca:\xab\xab\x19\xae*\xd7\x1c\xf0\xa8\xac\xab\xb0\xed"\xc3\xb9\x1f\xcc\xc3`\xdfv\xa1$\x8f3\xdfb\xfb=\xa5\xe9\xdd\xee\xea\x99q\xda\xb1\xe6-\xe4\xc4\x03*/\xfa\x18x\xcbi\xa9]\x1e$BS\xea\xc4\x83\xfdb\x98\xef\xa1|~\xfd\xb0\x86\xa2`p\x9b\x1eOaz@\x81\xe0K\x8c\xfbGi\x05\xf1D\x0fN;>\xa3\xe1\\I\xb1\x18Jqw\xdc6\xf9\xa0\xaf\xf1\x18;;\xe5\x0e\xb6\xa6]w\x15\xbb&\x12z\xad\xb0\x82L\xb18\xad\x1f\x8f\xfbh\xbd\xfaX`^\xea\x90\xb2_\xfcL\xea=\x94\xcf_?\\D\xa9\xc7\xa3\xb8\xdf\x86)\xdb\x16|\x0c\x1aW\x04\xfa\x8c\r\r\x91\xbcT\x06\xe6\t+k\xf9{ \x9d\xce\x16+7S\x01\x82<\xd4\xae\x83 \x84vvm\x00#\x0eg\x84\xd5m\xb6]Pcln\x02vK\xcc \xb2\xcfn\xaf\xb9\xd8\xf8b)\xfe\x1e\xca\xe7\xd70Q\x08\x86\x90\xe7O\x18\x0f!l\xb7*\xd8l5\xb6\x92\xad\xf5yg\xcc\xd8e8\x8a\\M\x14D\x05fP\x1d\xc5\xad\x1d\x92k\xbd`\xec`\x95\x180\xf9\xa8\xd8\xe6\x96|\xb3hN\x13<\\\x00\x07\xad\xe1l#>\x92\x86\x0c \xe1\x05\xf8Z;\xdc{(\x9f\xefwq[;\x14\x06!O\x8f\xa7\x04\xc2]\xf6\x8ec\xc39;\x1dx\x12\xdd2w\xae\x15\xc8rEm\x14\xd4n,\x0b\xe7\xc3z\x94WuV\xd9\xccFQ)F\xa5p\xa0\x8b%\xe0\x08\x0e\\w*\xbesU\xc9\x8f\x1bg\xd1hd\x881\xe1\x96}\xcf\xf7\xff)\xe5\xf3-\xc3\xfe[\xca\xe7\xff\xfc\xbf_\xe2.\xd9\xae\n\xfc\xeb\x97j~a\xf3NaE\xab\x8eD\xb7\r|\xab\xde~\xa4h\xb3\xee\xfb\xbd\xcf\xc3\xf1\xff\xcea{M\x93oQ\xff\xfa\x15\xc6\xc7\x9fN\xe3\xb7?\x83\xb6Z\x0e\x83\xbf}i\xe8\x89\x04\xba\xff\xe8K\xa8\x81\xa7\x97>\xa2\xd7\xb1\xf9\xfc\xa5\xd1\xf5\x85\xaf\xd4\xacu\x9b\xc0V\x95\xda\xe4\xfd\xef\xbf\x05\xfb\xe3\xb7\x8e\xe1\xba\x8d\x1a\x01L|\xb9\xfe\x9b\xb7N\x9b{\xbdU`\xadYY\xaa\xc7\xf7\xac\xbb\xed\xb3\x11$4\n\x08]\x8eG\xb7\xf1\x8e\x89\x148\xba\xe0\xf8\xae\xae\x8b\xd0\xec\xf8\x02v\xacL4XkHGr\xc4\xf2N\x9dS\x9b\xb7\xb8\x96\xe1\x08N\xc2\x93\x9b\x1c,\x8e\xb9\xea\xc2\x19t\x9b\xd3=\xe6\xf5C\xbc2\xa8\r\x9d\xdd\xe3\xb1\xc6\x1c\x88\xa7\x0c\xf3\xb7\x9a\xcf?\x91\x84\xb6\xdb\xb6eP\xd4\xf47\xd7}\x12\xdd\xf1\xe4%\xe7D<\xfe\xe6\xb5\xfee\x90I\xb8\xea\x9e5\x05\xc8\x96Z\x97\xc9\x98\xb4\x0c\x9cz\xfa\xa2\x95:\x1e6<\xa6\xc1\xa7:@\xdcIk\xaeeR\xee\x17\xbdI\xc6\xf8\xaf\xe7\xd0\xaf\xaf\xfc\x0fA\xa6\xd7G\xf7\xef_\xfd\xf97C~\xd1\x9d|\xd5\x1c\xfa~pr$\xf1\x1e_\x9b"_\xbb\xb1o\xbf\x17\x7f\xf7\xab?uX4\xaf\xbf\xf3O\xb9OIZ\xa7yx\xfd\xe1\x8d\xfaY\xbf\xa2\xf6\xcb\xdf~K\xee\xa5\xa1\x8d<\xaaJ\xb6\xbb\xf57\xaf\x15#\xd69\x84\xeb\xe94\xff\xfd\xb4}\xf7P|\xfb\xf6\xfac\xb7\x1d\xfb\xf0\xf1\x95\xb7_~y\xfc\xc1\x87Q\xfb\xc1\xefs\xfd\x0fb\xd4\xfe<\x9d\x96?\xcd\xab?\x8c\xda\x87Q\xfb0j\x1fF\xed\xc3\xa8}\x18\xb5/|\x9d\x1fF\xed\xc3\xa8}\x18\xb5\x0f\xa3\xf6\x95y\xb2\x0f\xa3\xf6a\xd4>\x8c\xda\x87Q\xfb\xb2\xd7\xf9a\xd4>\x8c\xda\xff\xa2\'Q\x1fF\xed\xc3\xa8}\x18\xb5\x0f\xa3\xf6a\xd4>\x8c\xda\x87Q\xfb\xc2\xd7\xf9a\xd4>\x8c\xda\x87Q\xfb0j_\x99\'\xfb0j\x1fF\xed\xc3\xa8}\x18\xb5/{\x9d\x1fF\xed\xc3\xa8}\x18\xb5\x0f\xa3\xf6\x95y\xb2\x0f\xa3\xf6a\xd4>\x8c\xda\x87Q\xfb\xb2\xd7\xf9a\xd4>\x8c\xda\x87Q\xfb0j_\xe8w\xbf>\x8c\xda\x87Q\xfb0j_\x8bQ\xfb\xbb\xa5\xf0O\xc1\xa0\xdf\xa4\xca_\x99Q\xfb\xe3\x03\xfbI\x8c\xda\x1f\x1f\xd8Ob\xd4\xfe\xf8\xc0~\x12\xa3\xf6Z`\xff\n\xca\xf5\x93\x18\xb5?~\xc4~\x12\xa3\xf6o\xd8<\xfe-\x8c\xda\xff\xf9O\x80\xec\xd1M\xc4\x9e\x9aoJ\xca\xaf\xbf\xa0\xbe\xbdNzi\x8a\xf6\xfa\x97\xf8\xb2\xf4\xd7\xee/\xc6\x14\xd5E\xac\xa4\xcb\xff\xe5\x7fE\xca\xfe\xf3\xfa\xff\xdf\x7f\xbae\xff\xf1\x8b{ \xc1\xfd\x0e<\xd8\xf5\t\xca\xcb\xe4\xee\xd3\x07\x0c\xd9\xa5\xbbh\xa5JfN\x05\xd5-C>f%xx\x14X\x8f\xbe&}7?\x18\xb0_0\x10\xfa\xe5\xfb\xaf\xd2\xff\xefq\xdd\x90\xbf\xbd\x89\xff\xc8uc`\x8ab!\x8e\xe4\x10\x8e\xe4!\x84\x16@\x9e\xa4)\x92\x17\x84\x07\xa8\x82\xc2\x84\x80\xb2\x08I\xa00\xce\xc3\x08\x03\x11\x10\x84c0O\xc2\x04%l\xd7H<.\xfc\xc7\x10\x0e\x82\xc04I\xc08\x84\xe1,I>D\x03\x90\xe3\xf0G[5\x94ex\x8c\xc7Q\x0c!\x1f=\x95P\x12f\t\x90\xc0\x1f\x0e\x00\xc7\xb0\x1a\x18\xfe\xad\xeb\xf6\x06\xdc\xeb?\xa7\xf8\xbf\xc1u\xe3\x91\xc7\xf4\xe5(\x84\xe4Q\x18\x15\xc8\xc7\xbc\x86\t\x04\xdb.\x1c\xa4h\x9c\x7f\x90B\x02\x03n\x0bw\x9b\xe8(\x8bb \x8b\x83\xfcv\xa5\x14\x04o\xb3\xff\xd1\x85\xf7\x0fv\xdd\xb6i\xc2`\x1cMr\x1cG\xd0\x02\x85b\xdb\xe5\xe1<\xc1\xb1\xd0\xb6: H\xc09\x96%q\x86\xa6\xb6\x97ax\x1eD!\x12\xa4\x1f,\x00H\xf2,OQ\xbf\xfc\xbe\xebF\x82\x10\x83\xf0\xf0\xf6\xd6\x1c\x842\x04\x83R$\xcb\xa1\x02Ac4\x89\xb2\x14\xb5\x8d\xa2\xb0\xdd\\\x88\xe6)\x86\xc4)X\x001L\x10\x04\x86\x05aA\xc0\xc1G7\xfe\x7f\x8f\xeb\xf6\xdf6\xac\x9e\\\xb7\x7fU\x0b{\xac\xc3_\xb50\xf0w\xb5\xb0\xaf\xbf\xce\x7f\xa2\x16\xb6m\x97\xc8\x8f\xa9\x99\x13=_\xe7SK^\xd6\x99\x18;\xca\x1b\xa8\x1f\xb5L\xfe\x93{a[\x9a\xb4m\xe3\x08\xb5\xed\x8a\xbf\xbd\x8b\xc3\r\xbc\xc0\x0c\x0fqI8U\x9cO\xee\t\xc4\x8a\xe2!m\xadd\xd2S\x82\xb0\xeeM\x948C\x14hU\x82\xf6|\xda\xdc\x87\xd4\xcbo`\xe9\x8fUn\xba2\x00\xcf\x98{\xba\xa1\x9e\xa2w\r\xa48<\xbd\x18/v\xb8|\x9b\x17\xb6\x85Il\xa7!\x89\xa3O\x1d.\xdde\x14$\x0f\xdc\xb7\xe7\x9e\xe9\xd7s}\x10-G\xd7\x00\xec\xb8d>s\xa7\xd4\x90\xbd\xd6\x9edN\xc3\n\x1f\xfc\x02ce\xc8\r\xee\x0b\xbfR\xf1N\xc6\x1dlM\x83\x01\x1f\xa4\x05\xa7t^\xb9\x8cN\x19^\xc5\xf6Eg\xe6m`\xd8\xb6&\xb6\xb2\x04F\xb6{\xf3dj\xdc\xa0\xe8\xc4\xe9D\xa7\x19M\xcd\x16l\xed\xce\\X\x87\x0e?\x8bP\xdfK\'\\\xcf\xdd\x8a\xf2\x94\xbe\xf1\xbcj\xe4}\xdb\xcc\x89^fp\x9f\x11\xf7I\xcd\x11c\x10!\xd2<(r\xa46\xd8\x0e0\x0fv4\xbf\xa8\x06\xbd\r\x0c\xfbMn\xffw\xa6\x06\xa3H|>\x9c&\xe0h9K\'O\x17\'\xb5vD\x91\xa3H\x17v \xbf\xb7y\x15\x88B\xc9;\x1cc\xb9\x88\xae\xd2I\xd0\x95\xeefBu%\xabsr\xdd;\x0bT1\xac\x0c\xda\'\x17\x85\xe9^\xb8\xf9\xca\xab\xa3\xf9.0\xec\x11&L\x90\x08\xfa\xdc}v\x9a9\xa5\x9a\xecBu\xe3\xca\x81\x84\xdb\x94s\xc6\xb9\x97f\x1b\x07\xe8[8C\x117NS\xbf(\xbb\x15\xed\xd7d={t\x90\xaa\xf3\xb8\xe3o%\x19#g_\xe0\xaf\x0eb\xacG\xbf\x1eT\xd7\xcfY}B\xe6\x17\xf9\xb7\xb7\x81a\x8f-\x08\x86\x90\xdf\x83`\xa8\x12\t\xd9\x11"\x19{*k]\xba\x86\x9c\x7f\xd0Ff/FZw\x02\x8d\xe8\n#\xe7\xd3\x81\n\xce6\xef5;U\x1c2B\xdc\xcb:A\xbb\xb7Y\xe0\xd5{#\xf7\xa3G\xb6\xf4~;\xaaC\x86\xba\xea\xa7mJ\x7f)\'\xe1m`\xd8v\xe4\x83[f\x83c\xc4\xd3\x9aPmQD\xf0\x90\xb0\xad\xcb`u\xf3\xfd\xaa\xec\x0e\xccyp\xb7+9\x9e\xabT\x9b\xb2td\x8c\x049\x80\xf1mq\xef\x19{`i\xdb\x8a\xf7#b\x1fg\xca\xc2\x8e\x93\x13\xe4g\x9e&\xe5\x9dk\x1d\xac\xb4\x9fP\x80{m\xb2\xbc\r\x0c{L\x16\x92z\xecpO\xc9\xcd\x19\x9cQ&9\xe0C\xb92X)\x1b\xe6\xfdd0\x1e\xb9\xcae\xe6\xbb&\xb7\xddh\xdb\xdf\xcf\'f\xca\xb3uB`#DJ\xb5!\xcb\x98\xeeR\xe2\x962\xee\x8e?P2|#=S\xdc\x1d\xc8$9\x13;\xf7GJ\xe8\x1f\r\x86=\xc2|T\xaf\xf0sGf7\xa2\x19\xb5\xd1y\xa0K\xf2\x95H\x91\xd9\x02s\x16=y \xa9\xd6\xd8\xe8\xf6\xc7\xc2\x89\x05ES\xcf\xdc\xa8\x92\xf8\xdeS\x9d\xfb\xa0\xdc\xef\xbd\xc6\x10\\\x88\x13h\x04\x18H\xc9V=u\xec-\xd3\x12\xfc\xa0\xd6\x81\x1f-\xfd?9\x18F\xfe\x05\xdd\xeay\xf0\x91\xde\xfc\xf6.Vc\x17\xaf\xc1\xc5\xd2\xc9R,"\x14\xa9\xddc}\x96\x92\t:3\xee\xfeR\x96\xc7i\xb1\x94`\x89oY\xb0\x1e\xfc\x98o\xef:\xc8\xd7Aq\xedO\x97\\;\x94\xf2\x8c\xc3&\xd6\n\xb0\xc3_\xb8\x0bY\xc9v\xff\xea\x9ax\x1b\x18\xb6\x85\tA$Dl\xf9\xc2\x93\x9f\xe9\xa9|\x90\x1a\xd6\xf9\xb6l\xb3\xb0/\x1a\xd0\x8a\x1d\xee8\xa3\x8d\x9b\x9c\xc8\xdaY\xd5\xaa\xb4/|\x0e9$S\x03U\xc4g\x85\x02\xa0\x80\x84\x89\xac}\xb8\xda\x8e\x89\xb1\xe21\xc7vF\x12m\xc9\xb1vh\x8f\xbc\xf9b\x98o\x03\xc3\xb6\xe4f\x1b\xff\xedD\x85\x9ez\xce\xc7\x97sj\xf3\'<\x9e.p\x84\xf5\x99[\xc1\xbb\xd3y\x99N\x95i\xd1\x1af\x02\xf5Xr\xe2\x96\x1a\x9c)y\xd6R.\x81\xe2\xdct&\xf9.]\x84\nkf~]`\x9c\xf0VG\xf4\x90NLK/\xd5\x8a\x17{\xce\xbf\r\x0c\xdb\xc2$ax\xdb\xe3\x88\xa7\x1d\xce0\x12X7H\xcc\xd0\xe8,\xf6\xf0\xd8T@:#\xe8\x1d \\\xf6w%u\xd8\xf3N\xccb\xf9&\x1e\x8fP\x89x\x05|F\xea\x03\x99z\xae\x16[\xf3\xeaO\xf7\xd2\xe0\xe9\xe3I:\x01#e\xdd\xacLq\x89\xfa\x7f(\x18\xb6\xddE\x0c{\xa0kO\xfb\xe7\xcd\\\xcct4\xcf\xd4~\xd4W\x8bJww1Z@\x19\xd9r\x89\xf6\x8a\xa1GF\xdevD\x81R\xb4c\xc7T]\xa8\xef\x8f\xd0a\xf1\xa5\x0bu[\xfc\x1d:\x1c\x9dn\xe5\x05\x0c\xdb\xdd\xf7\x97\x13\x99P\xe4\xcd3\xe0W\x13\xe1wya\xdf\xca\x1a\x18\x01\x91gR6\xc7wX\x91\xee\xb2J\xd3\x15\xad\xf4\xb0;\x81\xb5(\xbc\xdcz\xbf\xf2\x85=\xbe\xd3\x97\xe1p\x8f\x15\x976*\xe1:\xdd\xd6B-\x07\x86\xbbfa~\x1cqv\x14\x0c}at\x18\xc1/\x99\xd0\x9d\x087\xf1\xf6\xc1\xfeEI\xebm`\xd8\xb6\xc1!0\xb9\x15\xfb\xe0\xd3\xca\xdf\xe9*}%\x10\x919a"\xc0\xb1LF\x08K\xdd\xab\xf6\x98Q\xd8rA#\x89\x0cf\x8c@\xd5\x8c\x8a\xe1\xdd\xf9B\xa1>\x8ex\xa0#\xad\x15*\\Q\x13\x9e\xae\xe7\x1d5\x19{!\x11\xdb\xa45;_\xe4\xe1\x17=\x9d\xb7\x81a\x8fC\x9f@!\x12\'\x9e\x94Ph\xca-\xb3\xa8n\xf0\xddK\xd5\x9bG\r\xc8\x0eT\xe6-\xcf\x91Op\xe7\x04\x85\x97\x81"\x1f\x05\xa7\x81\xb9\tF\xb1:\xbeu1\x8eg@\x1b\xd0%\x90\xe49\xb3@i\'\xa1\xbd\xa8\x9f\x14}\x9b\xaf\xe7\x16\x0f~\xf8\xe0\xe6O\x0e\x86=\xaa\xa6\xad`\xc6!\xf2\xe9\xd0?\x1ddV\xdc\xef\xa6^\x11\x91T:\xef\x9d[\xc8\xf13.\xac\xea\x19\xa8\xcd\xbd\xc4\xddw>\xe5\xcbds<\x1a\xc2\xd9M\xdcl\xe9\xd2\xe1\xa8\x1c\xfa\x9c>; }+\xdb\xce\xa5\xa4\xa9#tZ?\x82\xa2\xa6\x9e\xc2\x17\x05\xf4\xb7\x89a\x8f0\xb7J\x01B\xd0\xa7c"\xf2O\x8b\x8cI\xd4\xf9X/\x8eT\xf3\xf1\xe1\xd2\xc2\xb7\xe6\xb8\x065Dv\xa2n\xcc\xfa\xc0\xde9\xc9\x1e\\\xea\x9a\x89\x04\x92\x1c\xa7\xf6v\xac\xe8\xfb\xc1\':O\xc3;k\xcb^\x15s\x18o\x8d\x90\x92\xa6\x06\x1f\xef/\x16\x87o\x13\xc3\xb60\xb7t\x81\xc4p\xe2i\x87\x83\x84\x86:\xcd\x08\xe0d\xe8d\xea\x14\xbd\x8foRcp\xd0\xb6/O\x00\x9e\xb6z\x03\x08{\xd5\x81\xba\t\xe5\x9a\x81:\x0b\xb7bU#\x850Mm;\x87\xb9\xd3!\x05\xd2\xa5X\xee}\xe7X\t9\xfa\xa5\x0e\r/>\xcc|\x9b\x18\xf68\xae\x10x;\xae\x9e\xb9g\xa6\x9ak\xc2\xe8I\xfd,\xbbxW\x82\xa4\x93ZX\x0c\xcf\x85\xeeT\x1e\x80\xfb\x97!0A\x131\x90i\xdf6@ox\xa2\xc6*\x10\x8c$C{+\x16C\xbd\xf4\x8e~\xd5\t?"O"ichm\xc2\xcd\x8b)\xdc\xdb\xc4\xb0o\x1b9\n\x92[\xfa\xf7\xdb0\x9d\xd2\x978g\x9d0>\x930d\x02\x8f\xd7dR\x0f\x1e\xdd\xc1\xf6\x00uM\xaf\xc8\x02~_\x10~\xbd;[b\xc3\'.@!\x15ckw?m\xcb3\x011c\xe6\x9c\x17\xa5ZpI\x08\xee\xf3\ns\xe0\xfc\xe2\xd3\xc6\xb7\x89a\x8fG\xd3\x14\xba\xd5Z\xf8\xd3\x13\r%\xf2<2lMT\xef\xe7\xe4\xc8\xdd\xe2y1\x04\xba\xf1+\x83p4\x9d\xc4\xc5\x13\xcbFG\xe2J\x0e=\xbeU\xb3\x08\xd1bt\x0c5\xec\xe1n\x03\xba\xe1\xfbi4Pr\x86\xcc\x08\x1cR2\x95v\xbb\x00\xba\x7f\xad\x14\xeemb\xd8\xb6\xf41\x92\x00I\xf49\x87C\xe3\xb0\xb8\r\x111le\xcb\xbd\x94\x85;M\xda\xcc\xed\xea\xa0\xe8\xe3\x1bJ\xb7\xf8\x0e\xddW\xee\x08,\xcc$\xad\x81%\x8b\xfb\x05[P\x18e<\xb0\xd3\x87\x81U\x92\xb8\xc4\xcb)h<\x10?\xda\xb7\x8c\xbdCn\xf1\xa2\x89\xf861l\x9b,(\x08>>\x02y\x9e,\x137\xb2\xbav\x0f\xe2\xf2\x8e4a\x7f\xc3SA\xd7X4\x0f\xa4\xf3i8\xc0\xf8y!\xb3\x82R\xa8l\xd5W->\xc8\xfc!e\xd6\xd3\xe1n\xd2W\xe0\xa0\xd4\xbdR\xd8\xd0\xad/\x1cYt&\x8c\xf0\xeeq\xcf\xbc\x98\xc3\xbdM\x0c#\xff\x82A\xdb\x06\x87P\xcf\xb6-\xbc\xfaG\x1e?\x8e\x08\x8e\x1bc\xc8\xde\r\xa6\xad\xc0\xd5s\xfd\x93\xd9"YD\xf9\x08i\xed\x10\xb0\xc2\x8dKs\xa3\xb8I8\x1d\x19\xe9|pORU\xb5\x17 2\x85<\xce9\x7f\x7f\xed|\xc6\xc8\xa5<;\xb9\xc3\x8b\xa3\xf961\xec\x91\x91o3\x16\x05\x9f\x8feH#\xc6[\x80\xc5\xf6\x1a\\\'o\xb8\xa3\xc1\xbel\x03\\\x99@\x84\xbc\xc0\x9e\xe6\xaf\xc9\xde\\\xa5%m,\xd9\xd2\x19\xc5\x81\xc5|\n\xf0\xd4\x10\xcf\xc1\xc8m\x9bRiT]7VhE\x1e\xf2\xdc\xbb\xc8^\xa1\xbfX\xa4\xbeM\x0c\xdb&-\x0e\xc1\x04\x8c\xa0O\xe7\x15v\xaf\xbd\xbbBc\x0c\xd3z;\xb1\xa7\x0c?4\xf7\xe7h\xaa\xc0v\xf5\xcd\xf1R\xa8\xddu\xa1\x96\xebu\x7f\xb1\xf3\x01b\xda\xe1\xb6R\xca\xae\xdcO7\x8c\x1a\x17\xd0u\x06\xfa\xae\xa6\x08\x93\xb1\xd6\x0e\x82\xd9\xb5\xba\xbb_\xeb\xf9\xd4\xdb\xc4\xb0oE*\xf4x\xc6\xff4Y.})\x86\x9e~\x86\xe4!\x0e\xc2\xd3M\xd4\xc3z(\xcfw\xd56\xb2\xbb\xaf\xc0\x88\xce\x8b0\x1b\xf5\xda\x1c\x08\xa0\nx;`\xe7g\xeb\x1d\xbb\xa3\x06>\xcf\xde%\xa2\xe2Kt\xd3\xc5\x830\xe9\xc7B7\xeb\x9d\xab\x7f?\xf5\xff\xa9\x18\xf6\xedkU\x7f+\x86}\xff\x96\xe0G\xad\xf9\xc1\xd7\xe7?j\xcd\xcf\xf8\x1d\xa3O\xaf\xd0\x8fZ\xf3Qk>j\xcdG\xad\xf9\xa85\x1f\xb5\xe6\xeb^\xe7G\xad\xf9\xa85\x1f\xb5\xe6\xa3\xd6|e\r\xe6\xa3\xd6|\xd4\x9a\x8fZ\xf3Qk\xbe\xecu~\xd4\x9a\x8fZ\xf3\xbf\xe8I\xd4G\xad\xf9\xa85\x1f\xb5\xe6\xa3\xd6|\xd4\x9a\x8fZ\xf3Qk\xbe\xf0u~\xd4\x9a\x8fZ\xf3Qk>j\xcdW\xd6`>j\xcdG\xad\xf9\xa85\x1f\xb5\xe6\xcb^\xe7G\xad\xf9\xa85\x1f\xb5\xe6\xa3\xd6|e\r\xe6\xa3\xd6|\xd4\x9a\x8fZ\xf3Qk\xbe\xecu~\xd4\x9a\x8fZ\xf3Qk>j\xcdW\xd6`>j\xcdG\xad\xf9\xa85?U\xad\xf9\xbb\r\xfb\x9f\xfa\x0c\xbfY8_Y\xad\xf9\xe3\x03\xfbIj\xcd\x1f\x1f\xd8ORk\xfe\xf8\xc0~\x92Z\xf3Z`\xff\x8a\x81\xf2\x93\xd4\x9a?~\xc4~\x92Z\xf3o\xd8<\xfe-j\xcd\xf7\xbf\xf8\xdf\x84\xc4\xa0\xffu\xcf\xff1\x12\xc3r(&\xb0\x04\x07\x920\x8f\xd2\x18\x86\x110\x87\xf1\xfc\xb6\xb5\xd1\x02\xc20\x18\x05\x93\x8f\x067\x14N0\x913\xce\xbc\x16\xe3\xbb\x90\x98\xc7Pn;:\x89<\xb5\x02\xcdh\xb4<,\x87\xb3\r\xf5\xf3`\\\xc8\x12\x9fNJuc\xf4\x99\x86n\xc0\xf9\xa0\x93\x80\x08W\x0cX[\x8a\x0fa\x99y6.\xa3C\x9eH\x0e\xc0\xf3,\xcc\xbaV\x92Y\xa3n\xfb\xb2\xf5R\xa6\xbbu\xaa^\x02?\n\xf2\xcfM\xc4<\xee\xe1\xb6\xe9m?\x86?M\x95\xac"J\x85\x03\x01v \x96{\x9fm%IjUE\\j\x0eD\xe9\xe6\xd1\xe9\xb5`*$\x1dq#\x96\xeba\x98S\xd4HIs\x90\xf3[\x1e\xaa\x95\xf3\x89`\x83\xd4\x89\xda)\x16\x96\x9b\x1c%R\x0b\x9f\xa1\x97[\xee\xbf\x87\x88y\xe4\x82\xc4\x96\t\x12\x18\xfa\xd4E\x9d\xecv\xdc\xca:[\x9e<\x0e`\xa14\xa6\xce[\xdaR\xf6\x8a\xe6\x1dQNcb\x006\x99Ee\xc9`O\x1cp\x83\x19\xcck\xe2\xb6\xab+\xda\xa4Dw\xb6\xb8\xbf\xed\xcfs\xe0\xf4sp\x8eR\x9c\x17\x93\xd0\x1c~\x12\x11\xf3\x18\xcd-\xcb\x03\x1f\xe9\xc53\x9b\xc42{\xef\x84\xc1E\xbf\xcb0\xcb\xbd\xee\xf6=P\'\x04\xa0\xad\x81Y\xabG\x8fEv\xa3\xdb{;\xb6M\xc6\xc6\xd5n\x87\x19\xd0\x81\x11t\x96m\x17\x8c=\xe7\xb0\xaf\xe7\xf4\x82\xfbk\x13\xd7\x00\x8bc>[\xd4/.\xfcw\x111\x8f0\xc1-\xb7\xc7\xb7\xf4\xebI\xc2\xb1[\x89\xf6jBCL\xee\xa0\x9abB\xb8\xee\xfd\xe8\x1cB*\x1a\xd6%\x03\xd6\x1dL\x8c=i\xedhY\xb7F\xe7j\x88\xb7\xc4\x95\r\x87\x10E\xdf\xa1\xa8\xc1\xaa\x01\x03bE*\xa1\xdcTb\xd5s\x03\\\x90\x17\xa1\xafw\x111\xdf\x8e\xaa\xed?\xcf\xfd\xe3\xc6#\xcc\x1d\xd7\x18\xa5\x89\x83\x05q7\xe1.\x14\xdelw\r4y\xf6m\x9f) \xc4\xf2\x9c\xb2\xfa\xeb\x1c\xbb\x08\xbd\xa2LR\xd4\x15\x19O\x93\x14(\xd7\x110\xf7\xd0e/\xe4\x08\x9a\x82\xde=\x10\xd5\x8a\xc5\xd8\xfe\xc5\x83\xea]@\xcco\xab\xb4\xbfk6\xe8\x9a\xf3l_\xae\xa2\xc4\xaeQ\xc3\x8f\x97]r\xc1\xe2\xdbINN\xb6$\xed\xad\xce\x13\xf3\xc1S\xe1\xdcA\xb8\xa8T\xabl\x01E\n\x1cg\xb5\xdb\xedw\xb1\x9b\xb5`\xcc\x878\x14u\xeeR\x85\x90\xab*\xa0\xc3\xfe\xa8C\xe6\x9f\x1b\x88y\x1c\xf7\x18\x06c\xe8\xb6\x84~{\x17\xb9\xc2C\xd4\x0e\xbf\x11\x0b\x1d\xcf\xd3v\xca\x17\xb6\x9c\xde\x9cn\xa9\xabr\xa8\x12f\x0f`\xac\xa8\xef\xfd\xa5\x03D\xef0]Qa\xdb\'\x87B6H:\xf3\xaf\xf9E\xdd{k\xb3\x80\xde\xc5\xe6\xed\xb2\x88*\x81 \xf2\x17[\xe2\xbf\x0b\x88y\xac\x08\x18\x7f\xf4\x18F\x9e\x1aFGV\x8e\\b\xf3\x9ckI\xdffRq\xb5\xbdRn\xd6y\xd6\x11h\x8d\x91\xc89.G\xe9\x16\x12\xa6h\xa2\xa7<\xb6\xfaR\xbe\xc3\xb3\xb6\xe5n *\x975\xb3\xe3\xab\xbdrK\xa8""\x8e\x0b\x0f-\x91;\xf0/6\xe0|\x17\x10\xf3-L\x12\xdc\x16?\xf2\xb4\xbf9\xb1\xbeg\xafD\xc1\xdb\xb0]E\xd7]\x1b\xcc\xfb\xb9\x8a\x06\x00-\x06d\x14k\xed\xa4\xee\xce\xd7\xa2\x12,\xe3\xc4\xe3\xe3$\xdb\x91\x0e\x06\xae\t\x9c\x03\xfe\xb4\x87\x8bH\xa4K\xcf\x81\x12\xc43b\xe7\xa8m\xff\xcc\xc2\x8fN\xab?7\x10\xf3\xfd0\xdcr\x07\xf2yM\xa0\x96\x98eVw\xd7\xf2\xfa\xde\xb1I\xb7\xf7\xb1\x082K\tf\x0e\xb6qm\xc8\xc1v\x0f\xcdI\xacx\xed\x84H\x1eU\xde\xceHk\x94\xf3z\x98\xa1\xe8\x0c\xcb\xf3R\xdbTU^/\xf6\x95\xbc\x12;\xaa_%R\xfeI@\xccc\xe9C\x04\x08m\xa5\xfaS\x1e\x0c\x1e\x95\nN\xa5#\x85\xe1\xe5y\x07R\x971\x973}`V\xecFJ\xf0J\xe9\x13\xe4r\xe6J\xe2\xaa\xe1.\xa3t\xd2<\x80\xbd\xa6F\xdd\x1b\xaa6\xe1\x98\xa7_\x05_96\xe1\xc0t\xc8\xda/g\x13g\xfa\x17S\x9bw\x011\xdf\xd6\x04\x84@\x04\x02>-\xfd\xc5\x07\xccl\x14#h\x8e\x87c\xc1\xf8\xf7\xcb\x9e\x1e\x10\x1epR\x19\xd6\xfc&O\x100\x17\xc5sfK\x8e\xef\x01\x19\'U\x0ey\xc8\x93\xf0\xb2\xee|\xf2\xb6\x03:`6\xf5\n\x92\xc9\xb81\x95:\xf4\xa1\xdd\xb6C\xbd\xb8\xc3\xbd\t\x88\xf9\x16&\xbc\xfd\x1c\xf6\x9c\xc1\xe5x\x1f\x1e+,\xb0\xab\x1d\x91W\xd9\xd1S\xd4\x00\x1a\xb0\xcc \xf6\xc6\x85\x11{\x96\xe3\xa3\x90\xb3\x12\xe2\xcc6(SBx\xc0\xf7\xa0\xba\x9e\xb1\xae\xbeiSU\xbbW\x947\x04\x06n\xb4\xe3\x05G\xce\xf8\xc5p\x81\x1f\xa56\x7fn \xe6\xdb\xd2\'p\x18F~\x07\xdb`\xeb\xbb\x03e\xa8\x11\xd8\x93\xeadY\x9b\xcffb\xfa\xe2\xc2\xfa\xcb2\xeb\x87,8\x04z\xc1\x82\x86\x0b\xddPd\xe7\xa6\xda\x1dBe\xd6FP\x9e\xf0\x94\x1b\x9d\x86\xeb\x10\xaf\x9a\x96x\xfd\x02\x85\x07qu\x05\x94\xfb9B\xcc\xb7\xdc\xe9\xd1\xa4|;]\x9fv8\xe2\xec4\x93c\xb6\xf4\xe1x\xa43B\xf0\xd1\xa1\'\xc8\xee^1\x8c\x08\xf8\x89\x8cI\xadm!g \xa4\xc9\xd6\xbe\xad\x99\xb1\xd4\xbe\xed\xdf\x1f_%%\xa3K\xac\xde\x13C\xa2\xa9\xb2\x10:h\xc81[Ep\xfcE?\xe1]B\xcc\x16&\x05\x82$\t\xe1\xd0S\xf1\xb6&\xf0\xb58w\xc3\xfdXu2\x03E*}T\xe7\xb1\xb3]\xdf\x83\xdah?\xdc\x89p\xa7a\x16\x00\x07\xe1b\x00\xb0(2U\x05\xee\x82\x00-l\xc9\x9a\xf2\xf1\x9eQ\x97sx\x0b\xad\x0cj\x8dc\xb8$\x83\t\xbfJ%\xbeI\x88\xf9V\x8aS\x04\xfe\x80?~\x1b\xa6\x96\xb5:\x03\xf0\xe7\x00\xb3\xab\xfd\x90w\xb1\xdf\x14nzR\x91\xd1K\xb4\x04j\xb5\xc6\xa5\xae8m\xe2\x1d5\xeb\xe7\xe2\xbc\x02R\xed\x93\xf3\xcdj\xe9+b\xcbE\xdcw\x18\xe8\xaf\x1d\xb1\xef\x16A\xd8\x157G#\x7f4i\xff\xdcB\xcc\xb7J\x1f\x061\x90\x02\x9f\xee\xe25\xbcX$:\x1fH\x86YAD8\x8c>\xaf\xc2\x0e\xba\x0e\xf5\xda\xf6\xfe\x99\x810\x99$\x9dz\x0fH\x1a\xb9\x8aS0g\x91\x91\xdd\x05E\xe3\xcb\x18\xa8\x85\t\xbfNB\xdf\x1e}\x816\x14^\x12\xa0\x01\x81\xd4\x17K\xe0w\t1\xdfJ`\x82@a\xf8\xf9\xb9\xcd\xc9\xa2m\xcb\r\x04\xc0Z\xc1\x0cI\xdba\xe6\xe7\xa5\xd0\xf2+^\xa6\xb3\x9fo\xa7\xcbhd\xa3\xb4cO\xa7)/ex`\x18\\\xbe\x8d\'\xa4\x03\x90S\x11\xc4\xd6$\xf8\x08\x82\x88\xe6\xf5\xd4\xef5\x19\xa6\xe8\x1b\xff\xe2\xd2\x7f\x97\x10\xf3\x18M|\xdb\xe1\xd0\xdf\xf1\xd2@\r\xaf\xa3\x1df\xdc\xfc\xcc\x82\xcb\xbad.\xbd\x00\xd2\x0eX\x06!\xc4\xd23K\xb7z\xb0Pe\x1fG7v\x08\x0e\x0b\xd4Y\xbeN^*\xb7\x95\xb2\x19&\xb4e%\xd7\xb5\x19`o."~\xd2\xed3\x9a\xde_\xc4\xa1\xde%\xc4<\xc2\xa4(\x0c& \xf8i\xd2B\xf1\xa1s\xa5\xab>\xf1\xae\xe0\xc3\x18\xb8\xef[ :8\xeda\xc5\x84\xb8\xaf\xe2d_F+\xc6\xd58y5\xf3\x05\xb8\x06\xd6v\xf4\xf5\xd7\xcb\xf9`$`"\x8bH\xc8FM\xa5\xd9\x1c!\xeb\x13y\xe9\x18`\x7fy\x91Ny\x97\x10\xf3HUI\xfcq\xc4=\xebM\x9a\xbf\xc4-\x0csk\x82\xeb\xbb\x82\xd1\xc4c\xc1\xd1g\x95Z\xce\xc8x\xa6\n\rlb{\x9f3Hu\xba0\n\xa4\xd2\x15\x855\xd7\x95;\x9d\xda\xed\xd8\x0e-\xdd\xe9\xe7ng(\x18\xa9\x12U!d\x85w\x8c\x88\x17\xab\xd4w\t1\xdf7r\x84\x82\xe1g\xb2\x91\xbaCew\xe7\xdd\xc6\x80\\%\xb685U1\xec\xf6\xf8\xfd\xa2u\x94\xd3\x8c\xac\x9d\xe6\n\xc6@{\xc8\x19\x16t\xdaL\x14\x97\x12\xe6h\xb2\x01\x8e|\xa7\xc3\xcd\x88^\xcf\xd2\x1a\'\xa4\x97\\\xb0+\xb3\xedh\xa2\xf8\xb5\xa8\xafw\t1\xbfn\xe4\x04\x8eaO\xa6Hx\x0f\x90I\x07.\xc2x\xa1\xefX\'h\xc4H\xe6\x95?\xc2\xe4\x1e\xefl\xf9~\x89\xed\xb8\xf5NBw8\xd4\x88\xc9\x80\xf6\x15s\xf2\x1e\x87\x0c\x96/.w\xa1\x80\xa6\xb0D\xda\xe4\x18v}\x16\xed\r/\xc2Z\xfc\xd5\x0fj\xde$\xc4<\xd6\xc4\x83\x1f\xc1\xe0\xe7\'\xd3il\x1e\xc1b.\x08\xb4G&\x86\xae\x99\xb4\xbf\xceS\x81"[\x9db\x89\x1aq\xe5\xa6\x9a\xd9R\x0e\xc9\xbc\xca\xa9>{.\x9fK\xae\xc2\x89\r\xa1\x9c\x19\xb7+\xd5\xebu\xb8\x9cZ\xea\xc0\xcfu\xb8\x1a5Q\xe3\xf9\x8b\x1b\xf9\xbb\x84\x98G\x988\xb4e7[\x1a\xf8\xf4\xb1\xdb^\xcb\x1b\x18K&\xa67\x02\xf5B\xf2\x89\x06\x9e\'h\xd1N\x90\xa6\x9e\x80\xcbZ7\x1a\x14\x9c\xf60\x00!\x19\xd0\xa4\xc7\x8e\xe5\xb1\x82K=\x1b\x88\x94\xe1@\xb7\xa6\x1dk\x19s&\x14\xfe\x82\xa1\x95\xe3E`\xf6b\x95\xfa.!\xe6\xfb\x93\x1bx\xdbI\x90\xa7Ik\xc5+\x19a R\x02\xb42l\xb9\xb9\x14\x1a\x9c\xd9Y^\xbb\x80\xb7\xbd\x85\x1a\xe9%71S\x16F\x1c$"P\x16\xac\\\xcd\xae\xe6~*\xea0\xa4\xae\xcel;U\x8b\x16Yx\xdd\x93\x0b]\x1ckwI\xd9\x17S\xd5w\t1\x8f0\t\xec\xf1\xbc\x1e}\xaaR\xa1\xb4\x03\x19\x90\xea\xf7\x83\xa9\xc8\x18\xd6C\xa1F\xeb\x82\x1f\xc5\xb7:F`?:a\xf2(\xf0\\j\xf2\xeb\x1a\x11!\x05:\xb86\t\xee\x99\xe6m\x8a\x9c\x15\x80\xba\x00\x87k0\x905F\x8d\xeb\xc9\xdbq\xa3\x95\x7f\xad\x87\xb6\xef\x12b\x1ew\x91B\xb6\n\x89xN\xf8S\n\xc0\xd5UJ\xed\x8c\xf5\xc6\xcbA\xbcU\x18+\xec\xed\x036\x80[\x960\xf0\x1c&\xdda\x10\x05R\x05\x04`G\xe9.\xa0\x989\xf9\x919\x9e\xfa`\xcd;D9\xb2\xf0\xffg\xef\xce\x96\x1c\xb5\xb6Ea\xbfK\xdd\xf2\x87\xe9\xbbsG\x8f@\x02\xd17\'\xfe8A#\x1a\xd1\n\x81\x1a\xce\xcb\x1f\x94e\xef\xede\xb9\xbc\x96V\xa8Veys\xe1\xb0\x9d\x95R2\x98s\x8e9&\xa9\x1a_p\xc81\xac\xa4\xb8s\x1e\xa6\xdc\x06\xde|-\xf8\xff\xa9\x10\xf3\xf1\x14v\x15b\xfe\xe5\x8f\xaa\xff\x8d\x84\x98\x9f\xc7\xdeX9\x93\xf5\x96~\xfe[\xba\n1\xab\x10\xb3\n1\xab\x10\xb3\n1\xab\x10\xb3\n1\x9f\xf8:W!f\x15bV!f\x15b>\xb3\xbc\xb2\n1\xab\x10\xb3\n1\xab\x10\xf3i\xafs\x15bV!f\x15bV!\xe63\xcb+\xab\x10\xb3\n1\xab\x10\xb3\n1\x9f\xf6:W!f\x15bV!f\x15b>\xb3\xbc\xb2\n1\xab\x10\xb3\n1\xab\x10\xf3i\xafs\x15bV!f\x15bV!\xe63\xcb+\xab\x10\xb3\n1\xab\x10\xb3\n1\x9f\xf6:W!f\x15bV!f\x15b>\xb3\xbc\xb2\n1\xab\x10\xf3\x8a\x85\xf0\x0f{\xe0?\xb5\x10\xfe\x90\xde?\xb3\x10\xf3\xfd\x03\xfbAB\xcc\xf7\x0f\xec\x07\t1\xdf?\xb0\x1f$\xc4\xbc\x16\xd8\xbf\xe3\x8d\xfc !\xe6\xfb\x8f\xd8\x0f\x12b\xfe\x03\xc9c\x15b\xbe\x8b\x10\x83\xff\xf7=\xffk!\x06c8\x88\xa7h\x02c(Q$Y\x94^\xfe\x83\xa3)B\xa0q\x8c\x110\x8aa1\x81\xa1Y\x8c[\xbe\xca\x88\xf8r\xcd0O\x12\x10\xc2r\xac\x80\xc2\xf8_\x0b10\n3<-@,\xcb\x10\x08$\xd0,\xca"\x98 \xe2\xc2\xa3m\xe7r\xe1\x90@\x10\x18DS\x08\xcb@\x18\xc3\xa0\x14\xccb"*\xf2\x0c\xc3\x12"J\xc3\xdfS\x88\xf9\xad\xed\xda\x97?i\xc0\x80\xc1\xbf`\xcb\x05C8\xfe\xb5\x01\xc3\xb7\x84\x98\xcf\xcf\xeb\xfc\x89\x10CR\x02\x82b$\xb7\x0c \xc4b\xa8H\xd0\xe8r]\xcb\xe0\xd3$B\xb2\xa4\xc0 \x18\x89\x8a(G\xc1\x0c\xc6\x8a"\x83\x92$\x87\xd30\x8e\xf3"\r\x11\x88\xf0e\x15b\xbe\xab\x10\xc3c\x02$<(\x1e\x84F\x05\x94CQ\x11\'\x1f\xf7\x97\x17x\x08\x7ft\x84e0h\x99f\x14\x8b \x0fT\x84\x81Q\x92\xc5\x97\xc5\x88\xf2\x1c\xb6,!\xe6\xcbO.\xc4\xfc\xcbn\xc9\xdb\x84\x98Gc\x93\xbf\x14b>\xff:\xff\xa1B\x0c\x8a\x7f\xbbU{\x81f,f\xb1\xecL\xc0\xa4\x97\xdc\xd1\xb6F\xa2\x8b\xcaE2\xc1au\x93\x8c\xd6\xc6>$\xa0\xee\x0b\x07r\x0b\x0e\xd2!\xdb\x88\xbc\x85\x1dI\xac\xdb\xdf\xe5\xa2\xa66[wkcZA\x9d[\xc0k\xa9\x1a\xd0\xe3\xeb\x8b\xdd\xb7\xdf%\xc4,\xbb\x02\xbed\tjI\x8fO}\xa3\xcd\x84\n\xcf\xb0\x05Bn\xac\x9e\xd0\x89S;\xabT\x13\xa3\x0bp\x0b`\x05E\x1eI\x83\x17b\xbf\x86\x96R\x9b,\xf7\xc81\xe1\xf9\x1c\xed\x96\xb9\xb1S\x80\xdd.\xeb\x81C\x85\xed\xa6\xd1X\xfe\xc51\xc2u\x12\x85\xf2U\x12\xe3MB\xcc\x12&\x01\xc1$\xf2\xd4\x8e+R.;\x13OM\xf5\xd67[\xc2\xa6e\x9e\x8eZ\x86\xe1#j\xca\x9c!\xbe\x9d\xfc\x99\xc9nX\x17z"1\xc1{\x10\xbf\x85`f\xb14mU\xdb\xcac\xba\xe4\x82\xaa[Hj6\xc8T!\'\x90o\xa4M\xc9\xbd\xd8\x1b\xfbMB\xccc(\x97=\rZ&\xff\xd3\x8c58\xda\x08"\x08\t*\xf6\x00N\xa5\'\xbaj \x0b\x07\x0fI\x12*h\x18\xbet"j\xd3\x8fY\x8c+\xca\xce\x812\xdf\xb2h)\x90\x08|\xc0R\x9c\x1e}}{\xba\xe1\xa3\t6\xdb\xae\xdeV\xcc(\x93X\xfa\xad\xa1\xfc\xb9\x8d\x98G\x99D\xc3\xe8r\'\x9f;\xf1\t\x17\x8b\x0eqB\x90\xc9\xe5\xbc\xd0\xd6rr\x83X\xe0(\xee\x1dt\xb8\x17A^\'\x0eG5\xa1Z\xfar]u\xaczw\t:B\xcd\x04\xb8V\xb4\x9b\x15l\xb1\xd4\xc7\x16\x16\xb7G\x0bL\xac}v\xaam\xf1\x8a\xbe\xd8\x84\xf3]F\xcc\x1f\xab\xc1\xdf\x87\xd9_\xa9\xb3\xd6\x05\xd6Vi\xcf\x00\xcf\xea\xa6\xc1V\x15\x12\xee\xe7\xbbp\xce\xc9\xf1\x80\x02\xe8\xc9\x9bD\xbbK\xb6\x0e\\e\x98\xc6\x85\xae\xdd\xa5e\xec\xdf\xd0\xed\x90\x8b\xfe8f-\x83r\xc8\x1ekw\x88\xa1("\x8a\xfd #\xe6\x11\xe6R\x03R\x14\xfd\xdcp\xb0>\x08\xbe\x11V\x17\xa9\xbeG\xc8}\xc7i\x97\x99\xdbnr\xc7\xd4\x91\xc4+\xdcc\x95\xb0\xfb\xd3\xd6\xce\xa3q.\x81c\xebI\xfb\xb1h\xb5\xd0\xcf.7\x8e\xd4\xd4^\xbe\x89<\xe7B\xb8\x06x\xb94\xef\xcbS[\xcd\xaf6\x8d~\x93\x11\xf3\x11&\xb4Ts$\xfc\xb4\xf4\x11\x9b\xee\xe4\xee\xc2EN\x06\xc9\x19q\xb1J\xcdR\xdcm\xba\xabG\xa5`\x19\x1f\xaf\xcel\x96\x81T\xdc\x1bl\x91\x82w\xe0\xb6\xc1\xee\xd4\xee\xa0\xa4\r1\xe0f~\xca\xb7\xa13\xf8\x8e_\xb0\xce\x88\xbbQ\xa9\xcb\xc8\x0f2b\x1ea\xe2$E\xe3\x14\xfc4iI\x0e\x96\x93\xca\xd4\x187\x0b-\xf9\x90V\xd0\xcdi\x90\x13\xee5\x9e\xbc\xd1\xa9+\xcb\xf2\x10\x0c\xa4\xe2\xe1dHu\xd7C\xfc\xed\xc6\xf03\x8d\xa6\xaa\xdfis\xbb\x17w\xb5\x1e\xc6\x1a\x19\x96\xc2x:\x87\x1c\x86\xee\xcf/\xb6T}W\xc3\xc1G"\'\x90e\xd8i\xec\xa9\xddq}\xd7E\xfe\x84\xcfD\xd3eh\xaa\xf0\xa6\xb9\x856\xe6\xb1\xec%\xa6:m\xcc\xec&IJH\'\x07\x8d\\*\xd3\x13\xefn\x99\xa37"\xb2H\x820\xefm"\xd2\xf2\x01?\x90\xe9\xc6\xde"\x8c\xe9iC\x9c\xda\xc1\x8b\xfb\xd5\xbb0\x9c\x8f\x14D\xa1\x0fA\xe4\xa9\xaf\xa2\xd6K^(&\xde\xd1\xcd\xeda\x18\xab\x81\xb5\xa8M\x9b\x06\\Zc\x07\xa1g\x11_#\x05\x05\xbe\xf3@f\xa8\xb9x\xeey(\xdez\'\xac\xe2\x0e\xf2\x04\x04\xb0\xd0\xf6\xcby\x0f\xa7f\xd4\xd9\xb05p\x967\x13\xf3\xb9\xfa*\xbe\x0b\xc3yT6\xe4R\xaa?\x18\xac?\xdeE\xb0\xecG\xa7\x84\xb9\xab\x98\xe6\x0c\x17\x82B\xa5\x01\xe4v\xd0\xd9Y\xec\xfdJf\x15\xc2qo\x14\x08\xe3~X`\x97L\x01\x99\xcd\xac\x90\xf6\xc1\x983\x18\xa3\x9a\x1aA\xef\xd7=v\xa9\xfaM*\xf6\xaa\x1b\x91\x1e\xa3\xbdX\xc0\xbd\x0b\xc3\xf9\xba_\xd1\xe8R\xac>y\x8d\xed\xdd\x96\xe4\xd0\xae\xbc\x1cm\x8e\xf9A\x8d\xe2\xb2\x9fp\x06\x0c\x80\x03\xddH]\xe1\x94W \xa6gl\xbf\x9d\xc0-\x90\xfbJ\x97\x03\xde\xb2\x02\x9c\x1d\xa2k\x13\x85felk\xad\xedg\xb3\xce\x9dp\x17\xaa\n\xfc\xc5\xed\xea]\x16\xceG\xf1\x81b\xc4R\x8c?\xe5q\xf5\nL9\x03\x82L}\x1a%\x06\x1aL\xbf\xe2y\xa4w\xcey\xeb\xde\xd2\x88\xf1\xb6l#%{\x87\xc4\xf4q7\xd9\xbe\xd1z!/\x96GW\xdf\x8d"S\x0c5.\xa5\xb0\xc2k\x82\xec\xe8\xc2\xa9@\xf5|)\xe9\xff\x96\x16\xce#\x7f\xa2\x04\x8e-%\xf4\xd3d)\x8eG\xeb\x0e0i\xce\xce\xb7\xdb\x16,\xf6@\x88\x1b\xf3\x9d\xb6\x9c\xf1\xca\\\xaa\x11\xcfA\xd8\x8b\xf2+5\x10\xb7+[s,\xea\xd2\xb9\x7f\x90\x04\x17\xc9\xb3\x9e\n/A\xc2\xa1m\x9aa\xc79\x87\xf6qO\x90\xbb\xe2E=\xe5]\x16\xce\xd7m\x02^\x8en\xc4\x13\x12#\xc0\xe9V\xb8\xda\xc0\x00#RC\xd8\xceN4 \xadhj\xf4|\x92Q\x1dlp\xd3Q5~cv\xd1\x86\xdd2C\xe2\x9b\xf5a\x92\xae\x9b\xee\xa4\xc9\xf7\xf3a\x1f\xcbg\xd9\x8a\xeac)\xcd\x15\xb7U\xba\xa00\xe0\x17+\xd5wY8\x1fk\xe2\xd1K\x99\xa2\x9e\xc2\x94\n\xd3j\xa1i\xc4\x07\x7f#{B~\xbc\x0f\xf6I;";\xe94\xd1\xbe`\xa0\xaeB\xa3e]\xcf\x17$\x9f\x98{8\x1cU\xc8\xd7(_%\xb0\xd3A\x85:\x86\xb5\x13\xa1\x10\x82\xd6"[11\x1aN\xa4\x0f/\xf2&\xef\xb2p\x1ea.\xefA\xd0\x7f"\xe0E\xb1\xd5\xe2\x1e\\\xa6\xdeQI\x8c\x91\xce \x1f\x03\xee]35\xda1\x9d\x8dQ\x84)\x15\x02:\xe4lm\x8dp\xe3u\x86u\x0c-\xfd^\x92n\x7f\xbe\xe2\x1d\x04\xde&*L][>\xd2\xe4\xa9<`\x8e\xb4l:\x9fj7|\x97\x85\xf31Y \x12]NE\xcfG}\xc2rA\xe8\x10lm\xe8\x82\xea\x96\x01\xe0\xb7\xba\xc9\x95\xfe\xa8`7\xbeK|\x97\x80\x99\xcd\x1e \xe4q\x1eh\xb6A\xb3\xe8\x06g\x88\x98]\x97\x9a\xb3l\xb1\xb4r\x88\xcd|H\x15\xd6\xbc\xde\xe9\x08cj\xf8\x08\xbeh\x0b\xbc\xcb\xc2\xf9\x98,\x18\xb2\xac\x0b\xe2i\xb2\x90\x88\x8c\xc4\x003\x80\x17XW\xe0\xcc\x9f\xfbc\x17\xa0\xc4\xac\xf3WL\xba\x05\xb99\xd6:\xb8+\xe8\xb0\xed\x14\xa5\xac\xcdR\x95\x91\xa9\x98|\n\xec\x06\x1d/\xae\xd9\r\x02\x8ca\xcf\xc1\xed\x81\xbaJ@\x13\x9c"\xfb\xc5\n\xf1]\x16\xce\xc7\x13\x8d%\xe5SK.\x7fz8\x95e4\x9d+\xd6T\xe8\xde\x98`\x91\x8f\xee!(>\xc2M\xaa\xb9\x81\xa6g\xd8E\xd50M\xd5P"\x96\xfd\xa8\x0b\xdc\xdd\xc9u\xf3r\x8be\x9d\x89\x1d\rdO\x95\x836\xd8\x84\x87;\xbc/\xf3^\xabe\r\xff\xdah\xbe\xcb\xc2\xf98\xd6P0D\xd0\xc4S!<\xa6\xb9g-\xf5\xf6\xa6\xf6\x92\x1d)H\xf3I\xd4\xf6\xb7\xc6(\xeb.%t\xaah\xc8\xc3\x96r\xc9\x13\x15\xb8\xa9\r&\x92?{\xe4]o\xca\x8b\xe6\xc2\xc1\xe5\n\x10\xda\x168\xb4YM\xf2\xe4}\x02\x8c\xce\x8d\xd5\xf2s\x11\n\xef\xb2p>\xd6\xc4\x07\x99F?M\x96q\xce\xd3\xfcp\xa8\xea\xaa=l\x92\x83\x9b\x1e\xcabk\xa4Mx\xa1\xb6\xd8pe\t\xf2^\x1c\x05V\xac\xb2=U\xc8\xae\x048\x88wuB\xcbR\x08\xc9B\xf8H:\xb0G\x08ZNW\x85Mq\x98q\xd6\x19\xf2\xfa"\xa1\xf0.\x0b\xe7\xa3\x10&1dY\x16O\x0fl\x89\x9d\xa0\x95\x0c\xb4[v$\xf8\x9e\xb1\x10\xc9\x86m \xdfG+\x03\xf9\xfb\x9d\xf0\xd8\xbce\xeb\xc4\x88l6\xea\xadm\xaf5\x95v\x02XF>zx\xb6\x99g\x1a\xf7\xce\xd4\x19,\xfd\x86\xa6\xae\xc9u\xa3&\xc9\x8d}1\xccwY8\x1fa\xe2\x10\xb6\xe4\xc5\xa7\xd1l\ta\xa05\x8f\xc9\xb1\xe4\x0e\xc1#\xc6Y\xdd\x06\xbc2\xec\xb2\xa7\x05t"\x1c:T\x01\xe7+\xa2\x94#\x80\xc2\xe5\xe9\xbaG\xb7\x08J\x9ba\xb0\xd5\xbd@O\x19\xd8a\xcf\x1b\xb2sN\xe0\x88\x1f\x86MP\x93\xb4\xf3b\xc1\xff.\x0b\xe7\xeb\xd2\x87\x96#\x1f\xf1tzk\xe3\t\xdf;\xbb\xb2)\xe7s\x90\x9f\xcb\t\xde1\x0e5%D\xb1\x8f\x86\x0b\r\x93\x1b\x93\x8c\x91\xe9f\xdc\xc8$\xb8\x00\x1e\xbf\xe4o51s"8\xcbdm\xb1\xe2\xd8l\x07\x13\x0f\xdcY0\xa2\xf0f\x0e\x86\xf3M\x10\xf6;[8\x1f\x89\x1c\x85)\x04y&\x7fp\x98\t-\x9e\xc6U\r\xe4\xce\xf4\x91"\x95\xb0\x92\x0f\xae\xa5\x85\xe7b\x9b\xe6\xd0r6%\xb80\xa1YY\x98!\xc1\xf6zf\x1f+\x10xR\xeb\xd2\xa7\xab\x03)\xd3\xe6I\xe6\xee{`;&\xbb\x94\x0b\xc4M\xac\xbc(E\xbc\xcb\xc2y\x84\t\x11\x18\xfdg \x06UE"\x7f\xef|\xe1\x84\x19Z\xb1/\x8c\x93\xbf\x1c\xb3\xc5\xed\x92E\xa5\x93{W"#K\xad\xb3\xc9\xddT6k\xce|\x89\xb6w}\x82Z\xba<\\\x1a\x8b\xcb\xb9r)<\xfa\x8b\x91^&\xd5\x93\xd5\x88v/I\xf5\xb9P\xb3wY8_3\x1cE\xc1\xc8Sm\xd3\nX\xacR%U\xce\xb8\xa8\x0f\x18jH\xc2\xd2\x9e%\xaa\xfb)\xbf&YO\x16\xc7\xbbv\xa5\xe7\x17\x97\xc4\xbb(\x9c\xaf\xdb\xd5r_0\xf8)\xcc\xe9\x143\x07\xeb\xe6\xf9E\xd8\xa1\xf7h\xce\x1dK\xe4\t{&\x8a\xd6\xd8)\xb6\xca\xa1F\xec\x10{,\x14\x01\\\xaa\xfd|[\xbbN\xbc-\x0b\x00\xdc\x96\x88\x93B\xb2T\xb9\x16\x1a#\xee| y\xaa\xca7\x93\x19\xbd\xf8\xfb\xc5wQ8\x8f%A\xd1\xe4r\xa4}\xa6\xdb\xbc\xe8Z\xe4Lj(\x87Z\xd9\xe3^\x86)\xac\xc1o\xdd\x8d\xad\x19{\xe3\x8e:\x1b\x18\xc5\x84\xc1f\x03\x00v\xa8\x00\xaa\x87\x1b\xed$e\x10T\xe59\xc4i\xf9\x02\x94\xb4o\x04\xc7>Lk\x9dC\x88l\xaf\x93\xc3\x8b\x87\xd4wQ8\x8f0\x97=\x06\'q\xfc)L\x80L-E\xe6\xe0s\xec\x1a3k\x1d:\xaf\xf45\x02\x0f\xc2\x04\x9eG\x95\xe5\xf5\x10\x1cf\xe4\x02#\xa5\xa6\xb9r\xc7\x11\xfa\xa1\x94\x08Y\x98\xec\x82\x17\xf7;\x87+\nG\x19\x81#m\xee\xeaY\xba\xd58d\x05\xdfz2\xfds\x1b1\x1f\xdb\x04\x8e=>!\xf0\\\xef\x9fk\x0c*#\x08\x8c.\x9b\\\xb0\xdbMR\x9d\xb3J\xa8\xf2\x11\xa9\x04\xeet\x99\xe78\x8b.G2e[\x8fr\x0f0\x96l\xe8\x0e\xcb\xb2\x13Z\xd8\x92kE\xe8m4\xa9K\x0fM\xd0\xfe\xe8XW$n\x1b\xe3_4b>\n\xf8\xd5\x88\xf9\x97?\xac\xfe\xf71b~\xa2\x9e\x87k\x1b\xc9\xd5\x88Y\x8d\x98\xd5\x88Y\x8d\x98\xd5\x88Y\x8d\x98O|\x9d\xab\x11\xb3\x1a1\xab\x11\xb3\x1a1\x9f\xd9^Y\x8d\x98\xd5\x88Y[L|\xba\x16\x13+\xbb\xb3\xb2;+\xbb\xb3\xb2;+\xbb\xb3>/\xfd\x8c\xcfKWvgewVvgewVvgewVv\xe7\xf3^\xe7\xca\xee\xac\xec\xce\xca\xee\xac\xec\xceg\xe6lVvgewVvgew>\xedu\xae\xec\xce\xca\xee\xac\xec\xce\xca\xee|f\xcefewVvgewVv\xe7\xd3^\xe7\xca\xee\xac\xec\xce\xff vg5bV#\xe6s\x191\xff\xf0C\xfe\xa9\x86\xf0\x87\xf4\xfe\x99\x8d\x98\xef\x1f\xd8\x0f2b\xbe\x7f`?\xc8\x88\xf9\xfe\x81\xfd #\xe6\xb5\xc0\xfe\x1dq\xe4\x07\x191\xdf\x7f\xc4~\x90\x11\xf3\x1fH\x1e\xab\x11\xf3]\x8c\x18\xe2\xbf\xef\xf9_\x1b1\x08\xc4\xe20\x07\xf1\x18,\xb2\x10\x89r"\xc3\xa1\x1c\x86\x08\x14\'r8\x851\xa2 \x8a0\x8c2\x1cN"\x8c\x88\xe0\x98@\x88\x84H\xf30B\xf3,\xc7>\x1a\x96~\x9b?\x10\xe9GOS\x91\xc5Y\x12c0\x14\xa7\x19\x8a&(\x81F)\x92DQ\x8c}\x10\x18"\x05\xc38\x8f\xd3\x1c\xfd`\x108\x04fp\x1a\x82Y\x18a\x05\xea{\x1a1\xbfu\x94\xfd\xf2g\r\x18\xc8_\xa8\xe5\xae\x92\x8f\xa6\xbc\x7fe\xc4|~`\xe7O\x8c\x18\x8a\xa6\xe0eL\t\x84\xc1\x04\x94\x80\x97\xff\xe3\x04\x91\x12\x18\x0c"\x05\x8a%\xd9\xe5\xa7\x12\x04\x8cR<\xce\x8b\x08\x02/\xf3T\x10\x1e\xbe\t\x0e\xf3"\xc1\x7f\xf4\xe3\xfc\xbd\x11CP<\xb7\xbc\x06\xe3P\x16\xc3Ir\x99\x19\xcb\xfb2\x0c\x83\xb0(\xca\xf1\x04L\x8a,\xc91\x14F \x1c\xc6#\xcb\xbd\x118A\x80\x08\x16\xe2aZ`\x08\xe6\xcb\xef\xa6\xf8j\xc4,\xef\xc6q"\x81\x08\x02.\xf0\x14\xcf\xb34\x89\x08\x18\x0c\x13\x10\x8b\xe2"E\xa10\x04/\xa3\x87\x91\xf0\xf2\xe7\xbc@a4\xc7>\xba\x83\xd1\x08\x8b\xe0\x14\xc2!(\xfa\xe5\'7b\xfee\xeb\xe2\xc9\x88\xf9\xc2\xf5\x14\xcb\xf5W\x96+w\x97\x00\xa1\xe7\xd8\xa3\xe7m\xabA\x817\xf6q\x93L\xe9Q,C\xdf,v\x16\xae%H*\x84\xde\xad\x0e\x11\x11\nM\x05\xe6\xca\xf1\x92 \xf0e\xe7\x99\xb5\x86\x8aKm\xce^\x0fG\xe5\xa45\xe9\xa4{gL\x97p\xe8\xb0\x14J\xb1\xe4\x9e#D$\xc2\xe6<\xeb^\x8a\x86\x92\x89\x0bG\xe5\xe3\xb5\x8f3P\x8c\xb2ur\x14\xab\xd0No\x07\x9e\xb9&rw\x8f\x96\xaa>>\xd6\x83\xce\x17\xd3Rla\x87\xe5\xe7\xc6\xd2X,\xc7\xf1a\x87\xdc\xce\x079A\x92\xa2\xb3\xb8\xdc\x85?v\xc8\xdbF\xd1l\xe6\xba\xe3\xbf\xfe#\xd4\xb9\xea0\xbd\xc4\x15\xd4\x7f\xc7\xd5h\xf7\x83\xcfB\xf1\x1d\x9fCO\xbb\xee\x8e\x0e\x12\xcd\xc2\xe4\xc8ni!..\x94;uch\x96\xaeFw\x85\xdf\xed\xba\xc3\xb6\x04b\xd1\xd6o\xc7\x08\'\xba\xaa\xc4tt{j\xc7`\xb8zz\x06D\x90\xb9\x11\xb8|\xb3a\xcc\x9c\x13r\xc3\xb8*\xddN\xaa\xb8\xea\xee\x88\r\xe5\xdd\xee.|0\x13\xc0\x9a1\xdaA)4\xf4u\xc0.\x90\xe9\x82\xe3%K37\xa0\x05\xef\xd5\xe6laE\x1d\xf4g\x97:\xce\xa6\x94\xd4v\xd8\xb4->\x8d7\xb0\xb3OGiW_\x94\x93\xc14\x9b/\xff\xae\xe6\xf3\x982\x7f\xa9\xf9|\xfe\x8c\xfcC5\x1f\x0c\xf96w\xb1\xf5\x1ca\xbf\xf5Oco\t\xe6\xe52p\xa1rk!\\\x97\xe6\xb1\xd0\xee*\x95n\xad\xdc\xcbg\x9f\xe7\x82c\x15I;\x18 [\xe4Px\xfa\x9c\xef5U\x8f\xca\x86 \x97r\x9c~"\x1a\xda\x0b\xc3`\xb7`9\xbd\xd5\x11\xba\r\xf1#x\xb9WG\xf5\xa4m\x88\xcd\x06rUJ\xee\xa3C\x0f\xealh\x89\x9bC[\xa4\xe5N\xe3{\x84\xd1\x91\xe6ds\x17\xc43\x06t;B\x1b\x9fnt\'H\xa2F\x9e\x90\xcf\xb5\x17\xbe\x8d\xf2yl\x12\xcblY\xf6\x97?QQ"\x9f#\xe8K\xe1\xdc\xe3\x92\xf3\x05l\xe3\xf3\x8d\xc8d\xc5m3\x9e\x8c\xba\x91\x82\xb2\xb2\x8b\xf4\xc2\x1e\x89#&p{8\xc3G`\xe0\xaa\x10l[\xf9\x92\x04\xb8p\x16\xaf\x8c\x9e\xd9\xb4iX\x1c\xba\x9d\xd2\xad\xf2"l\xf76\xca\xe7\xe3\xecF\x10\x04\xf6\xecy\x04n\x14\xdf\xb7g\x02US\x8c\x93\xfc\x12\x1fS}\x06\x8f\xa6c\xdeO\x86\x98\xe8`\xa0n\xf8x\xe3\xd7\xacy\x15w\x03\xc2OU\x86\x1c\xe6\xb3\x07)I\xe2\xec\xf0\x1a\x1a!3f.\xd8\xc9&\xda[\x8c\xc7M\xdd\xbcxv{\x1b\xe5C\xfd\x02a\x04LC(\xfa|\xa81pk\x8f\x983\xc6\xcd\xb1\xa3\x86\xbb\xbc\x85\x07{\xbf\xd3\xe6\xf6\xa2].\xa8m]\x1d\xb3\xedU\xa8;\x1a\xe9\x9c^\xe33\xb2\x01R\xb6\x8f\x88a\x80\xbc{\xcd\xf3FJ#\x97\xc3\x8c*\xf7\xa4>w\x88^,Q\xbe\x96\xe0\xdeF\xf9<\xf6|\xf2\xf1\x9c\xf1\t\x80p\xa3\xe0$\xd3\xecesC\x98b\xc0Q\x15++t2/\xf8\xb9\x1cFW;\xf9\xfa\x1c\x1c\xac\x1a\xb4\xa4\x83]zb\xeasI\x11E\xe5(\xbb;\xc6OOU\t6\x87"\'Ll&\xc8ad\x98\xb3\xa9\x94\xdf*\xf6\x7fr\xc9gY\x12\xc8R8/\xd5\xd3\xd3\x91I\x8b@>Jd"\x80b56o5\x96[w\xe4`m+\xaa<\xd67\xf8\nL\x16\xc4(]\xe80\xfb\\\x96c!\x91\xa8+\xdc\xcbX\'4\x81\xdc\xab\xac\xde\xc9=2 \x04S\xf0\xbe\xa0\xb0\x06\x90,\x05\xcak+\xffm\x92\xcfc\x9b\xc0I\x8c\xa0\xc8\xa7\x95_E\xa7^\xb7\x14Z>8\x89\x0c^\xf6*\xd2\x14JB\x8f\x11.A\x13\xb7\xe9\x8c\x84\xef\xcf\xa7\x8a\x95\xcf\xc6\x10\xb7%.,{\x82R\x08Uy\x9e\xf9\xab\xd7kT?\x1f\x96\x8dcC\xa7\xd9\xac\x03E\x93R\xe3\xf9E\xd8\xeem\x92\xcfc4\xf1\xe5\xa0\x8f>\xa3\xbd\xf7M\x08nu\xbe\xdb\xb3\xf4\xa0\xed\xf7\x86\xd4\xdbj\x1e{\xad\x12\xbaHQ\x84;\xea|\xdd\x92\xdb3\xa4\t\xda\x81\x1f\xbc\n\xbc\xcb\xd0V\x89\x99\t}\x10\x1c\xdd\xcc\xce\x878\x8a\\0\xce\xeb\xc4\xcc\xae\xf0\xa1=\x13/\x8e\xe6\xdb$\x9f\xc7\x93\xc6e\xce\xa3\xe4s\xb5o\x1aZ\xe3\xb0\xc8)\x0cR%\xe5\xe2\xd1\xe6z~\xda\x88\x18Q\xec\xadT9\xb8\xea\xdc\xa2\x00\x84_p\x12<:\xe2~\x0c\xc9)\x97\xf6\xbc\xe6\x17\x03\xd0t\xe9\xf5>\xfa\xb3\xab\xa1\xfe\xa0\xcc\xe7QP*(\xab\xbaW\xc3|\x97\xe4\xf3\xc8\xe3\x18\xf4`1\x9e\xc2\x0c\xce\xda\xe1\x1c2]\x04`\xd3]\xe7Bt\x0b\x9c\x85\x9d\x90\xa8l\x1c\xc9\x9b\x1b\xab\xb09\xd1\xdd\xe9=\xb6\t\t\xc3\xdeC\xf1U\xd4\xee$\xc3\xde\xa6\xf4\x12\xdd\xc8\xa8ks\xa4\xa9\xc6P\xb9\xf9\x1c)\x83f*q<\xff\x83$\x9fG\xa1JR(I??\x9c"=u?\x1dv\xce\xe5\xbc\xf1\xeff\xec\xabWI\xda\xb2\xd3\xac\xdb\xec\x8f\xa5\x8fS\x14\xf1\'\xc6\x8d\x00\xab\xc4\xe0\x19H;\xcc\x05!\xa5\x9a\xb2\xf1\xd2P\x99\xa4N\xa1\xeax\xc9\x07R\x0fzt\x1arZ_vW\xbf\x96O\xf1\xe6l\x9ef\xec\xe0\x1dki+\x08\x02hUH>r(c\x1fo\xa7\x88\x9a\x8fx\xf7\xe2\xe1\xedm\x94\xcf\xb2\xf4a\x18E\x962\xf0i[\xee\x84\xf3\xb8\x9du\xb0\xe1\x97\xa1\xb2\xb1\xb9\xf7\xd3+\xe2\x88\x04X\xd8\xaa\xeaV\xf8\x92\x01D\xd7\x1eD\xcd\xe5\xc1=\xa4\xdf\xa6\xdb=\xdc\xa0\xbd\x16\xc8A\x1c\x1b\xe26\xa99\xd8\xadGd\x9f\xa7=[\xddQ\x83\x0c\x9a\x17K\xb8\xb7Q>\xcbhb$\x86\xff\x99<\xa7b\xae\xda\xa5w\x84\xcc\xcc3;\xf3\xc0\xe8\x98\x817\xdc\xfbR\xdf\xa7\xc1\xe9\x9a]U\xa7\xd2\x89\xd1\xa6 \xa7\xa0\xe4\x03Cr\xf6#\xd1o\xafM\x9fO\xdcU\x83\x8b\x11\xdd\x0f;\x9a\xd8\x07 TJwG\n\x81\xbf)\xe5\xb3\xdcE\nCq\xe4O\x9e\xee\'\xcbV\x1b\xa1\xb6S\xe8`\x08+v\xd6\xdc6\xb4:gy\x7f\xf1r{\xdf\xe7|\x9bY\xfa)F \xd66CtL<\x08\xf4#\xb6\xd2\x03q\x9b\x917*g\xeb\xd3%V\x87\xa2G\x0f\x8c\xc4\xf4}\xef\xe4\xde\xd7J\xf8\x9fR>\x1fO`\x7fO\xf9\xfc\xef\xff\xfb%\xe9\xd2\xc3\xc7\xc7*\xbe~\xa8\xe6\x0b\x97w*\'\x99u,\xb9m\xe0\x9b\xf5\xf2\x92\xb2\xcd\xba\xaf\xf7>\x8f\xce\xff\xe7\x1a\xb5\xe3!\xfd\x88\xfa\xd7\x8f0>\xbe:\x9d?\xbe\x06#\xcbQ\x89\xfe\xf8\xd0\xd0\x13\tt\xfb\xd6\x87P\x03O;\xfa\xa8V\'\xc6\xf3\x87F\xe7\x17>R3\xd7m\x8a\x98\xd5\xc1\xa2\xee\xff\xf8)\xd8o\xff\xe8\x04\xa9\xdb\xb8\x11\xa1\xd4W\xea\xdf\xfd\xe8D\x80\xcfF\xdd#f\xab\x1dc\xc9\x0c"\xb80"t78b\xa8o!\xec\xee\xb6=\x12\xcdf\x95\xb4\x9b!t\xeb6\xacE-l\x99\x8b.\xb3\xbc-\xb8\xb0\xe5\xf6\xbe\x85(RXU\xd8NT&\xabv\xee^\xcb\xc0*\xa2\xdc\x03$@L;\xbc\x1c\xa4^\r\xda\xba\x88 Q\x8d\x1c\x81\xde\x1b\x7f\xd4|\xfe\x89$\xb4\xdc6(\xf2\xe8\xe9w\xd7\x1dJ\xee9\xf4\xd2"\x95\x9c?\xbcWz\xa8\x0fy4~\xf3\xcd~T\xbb\xe0/\xbf\xff\xc0\xd3K\xe1\xc7\x1e]\xa5\xcb\xf0\xff~\xd8P\xb3\x88\x90z\n\x7f\xab\x0e\x7f\x1b\xe4o\x90K\xff\xc6\xdc\xfau\x90\xfe\xff\xff\xefc\xe1\x9c\xfb\xe8\xf1\xe9\xa5/_\x1e_XE\xaco\xfc\xd5\x9c\xbf\x8f\x88\xf5\x13YC+\xdf\xb4\xde\xd2\xcf\x7fKW\x11k\x15\xb1V\x11k\x15\xb1V\x11k\x15\xb1V\x11\xeb\x13_\xe7*b\xad"\xd6*b\xad"\xd6\'zt\xb4\x8aX\xab\x88\xb5\x8aX\xab\x88\xb5\x8aX\xab\x88\xb5\x8aX\xab\x88\xf5\x89\xafs\x15\xb1V\x11k\x15\xb1V\x11\xeb3KS\xab\x88\xb5\x8aX\xab\x88\xb5\x8aX\x9f\xf6:W\x11k\x15\xb1V\x11k\x15\xb1>\xb34\xb5\x8aX\xab\x88\xb5\x8aX\xab\x88\xf5i\xafs\x15\xb1V\x11k\x15\xb1V\x11\xeb3KS\xab\x88\xb5\x8aX\xab\x88\xb5\x8aX\x9f\xf6:W\x11k\x15\xb1\xfe\xa6"\xd6?\x8c\xdb?\xb5_\xfepI\x9fY\xc4\xfa\xfe\x81\xfd \x11\xeb\xfb\x07\xf6\x83D\xac\xef\x1f\xd8\x0f\x12\xb1^\x0b\xec\xdf\xf1\x95~\x90\x88\xf5\xfdG\xec\x07\x89X\xff\x81\xe4\xf1\x1f\x11\xb1\xfe\xf7\x7fYR\x8f\xc6\x10\xd6\xd4|\x80\x17\xbf\xfe\x05\xf5\xe5}\x0eCS\xb6\xe3/\xc9p\xef\xc7\xee\x97\xfd\x14\xd7e\xa2\x1e\xee\xffG\xf8\xd5\x9b\xfa\xaf\xeb\xff\xbf\xffEP\xfd\xaf/\xaeNA\x1b\x10\xd2\xad:\x84\xf3cz\xf3\x19\x1dG\xc1\x03\x18\xcf\xf4\x91\xbd\x1e\xc4\xad{\x8c\x84\x84\x93\x91\xd3c*?ZT\xf4\xdd\xf5!:}\x81!\x08\xfe\xf2\xf5\xef\xd2\xff\xcf1\xba\xc8\xdf\xdf\xc5\xbf2\xbah\x81\x840\x9c\xa7X\x9c\xa6h\x81}\xf4\x90\xa5i\x8c\x80Y\xf4Ak\x11\x02K\xe0\x1cA\x92\x18\xc6\xe1\x1c\x84s\x82\xb0|\x1b\xcb\x8a\x14G\xb10&\xe2\xc2\x97\xbfDM \x8e\x85I|y5E\xc3\x0cBs\x98(\xe2($\xd28\xce\x08\xcc\xf2u\x08\xa18\x01g\x97\x9f\xc3!\xa8\xc8Q\x08\r\x89\x9c\xc0\x93\xac\x80\t\xb8\xf8]\x8d\xae\xdfZ\x0e}\xf9\x93\xce*8\xfe\x0b\x8c\xc1\xd4r\x99\x10\xfeWF\xd7\xe7\x07\xce\xfe\xc4\xe8\x82\xf0G\xb7\xfcGS<\x92\x83E\x9cEX\x8e\xa2\x19\x92\x17\x96\xf9\xc8\xc2\xc4\xf2&,OS\x8c(>z\x8ar\x0c\x81R(\xb2\xfc\x11M3\xff:\xff\xb1\xf2\x13\xf5m\x98`\x1fz)\x92_\xd2]\xccv>\xc3pw\xb8\xe9\xb7\x81\xde\x0f\xc8\x99\x14m\xd8H\xeb\xa3{fg\xaaKL\r\x13\x1c\xdc-\xa7Z^-\xbe\xd8Z\xfb]\xf2\xd3\x12&\xb2d\x02\xfci(\xeb\x1dT\x1aj\x93o\xe0rF\xfd\x1b6\xc4\xc8\xe9\xdc*\xec|A\x0e\xb7\xc0\xdb#\x1a\x19\x82\xe5\xb4\t\x9a\x9a\xd04\xeb\xb4\xd7\x1f\x1f*\xa8\x9a}h\xc6\xd2\xf960\xfc\xec\x93\xc7\xc6o\xa5\xe14V@\xce\xc9w\xf9E\xff\xe5]\xf2\xd3#F\x1c[r+\x04?\xf5\x85\xd5\xb2D\xbd\x85r\xd8t\r\x82G\xa9\xe8S`\xb6?\x12W\xd2T\xbdH\xb1\x8a\xda\xe6\xfd\xac\xe8\x80g\xd8nO\xa2}\x9be\xb8\x8c\xed\xd0\xed\xbd\xe8F&;o\x00AQ\xb8~"Ma\xeb\x9fb\xd4\x99:*\x8eog\x7f\x9c3\xb0$\xc3\xb0\xc9\x95\xa6\x0fr\xf8\x04\x8ab\xc6\xc3\xfa:/?\xa7 \xbb\x80\xaeI\xd0\x81\x11\x13\x878\x15\x97\x81\x055&\t\x9a\xf7Q\xa9\xa1\xb9.0\xed\x84\x81\x1ba\xcc\x08\xd6\x9a\x19L\xcb6\xd2\x1d\x1c\xa7\x1d\xb9\xa1\xe4\xb46\x1f[\xb8oKK\xe0(\x0c\xd5hKE\'z\xfc\\\xdc\xc5\xbb\xe8\xa7\x8f\xcaf)\x1d\x1e.\xce\x1f\xef\xe2\x81\xef\xe8=\xa8_\xb6\x1bx\xbc(>_\x1b\xbcu\xc8nB"\xe8\xe0M\xb3\xb3\xde\x16\xe3+\x92\x90z\xa1\x0f\xcd^\x0c\x92\t\x1fy\x93e\xcf,t\xd0\xad\xeb>\xbb\x0e\x80{\xa2\x18@\x0b\xea\xe5K\xf4\x99\x05\xe7\x17\x9b\x08\xbf\x8b~\xfa\x98,\xd4\xb2\xf3\x91\xc8S\r\xd7(\xae\xc34GrI\xe0J\xac\xa0\xf2\xcd\xab\xc2\xa1n\xdd\xc3R\x90S\xa4\xdf\xba\x02{F;\x99H\xd8\xd4\xd2\xe1\xcd\xf1\xd0\xa1\x00\x8f\x1f\x005\xd9F\x12;N\x94q\x80\xc8\xb67Z \xbdo;\xf7\x98c~\x8b\xb3\x90\xd9!\x9b\x1d\x8eLGDg`2V\x8c\xee\x96v\x00\r"\x8ex\x17\xf5\xec\xb0\x85\xaa\xb6A \x1f\x8e\xf62\xa1\x9c\xe0\xc11{\x1c:\x8e\\\xeb\xb1\x14i\x93\xda\x8b\xad\xe6\xdfE?=\x16\xfe\xa3\x9f\xfe\x83Hx\xda\xf3c\xa00\xfcHw\xd9\xc0\x88\xb9\xd2g#*\x97\x02\x0e9\xe7=uK\xe4\x06\x8b\x05\xe2*\x1b\xe2\x0co\xf0\xea\x961R\x85\xa5C}6\x82\x19\xc4\x94\x81\xc4\xc1\xc1i\xf1I\xd3y>\xf4H\x89(\xcd\xd3\xac\xbex@}\x17\xfd\xf4\x18K\x8a$p\x08\x86\x9e*\xb8\x02\xdag\xd0\x99\xbe\xc5b\xe8\xc6jc\x16\x11\x18\xe4\x83<\x83-\x10\xef\xeaCo]\x08\x00\xf3\xbb\x84\x18n\xe6Q\xd8ygT\x16AU\xec`\xb6\x91\xa8k3\xf7"7/\xe9\xce\x10\xb7\nq\xaa\x8f\xb7\t?\xdf_\xa5\xed\xded?}\xe47\x88B\x97\xe4\xff\xc7(\xa9\xb1\x93\xf6\x149\xaaw.\x8f\xb3N\xca\xe1\xed%,\xa1\xd00\xeb\xe3\x98\xd4g\x18\xa4o\xcb\x8f\xc5\x0f\x1bF\xdeA\xb6K\xaaQiW\x86\x0e\x06\xb4L\x9f\xe9\x9d>\xf7\xc7y\xcf\x81\x84O\xda\xd4~\xa7\x97\xdb\xf8\xfc\xc9\xd6\xfd\x9b\xe8\xa7\x8f\xb3\xdbr\x07!\xfay/\\\xeeM+\x05\xc0>\xad\xa0]\x818\x89N\xc4\xbd\x8a?0\xc1l\xa7T8\xe0\xe8a)v\x19H\xcb\xcbK\xae\x88s\xba\xec\x1c\xce;\x98\xbc\x13\x98G\x1bM#N\xbe\x9cx\xc19\xc4\xbbr\x08\x10\x19+\xb1\xe1\xc5M\xe2]\xf4\xd3\xc7&\x81-e3\xfd\xb4\x152\xbcV!\xd15\x0f}<\xda\x88\xe6l\xec\xd4\x86\x9ez\xab\x8cm\xc9q\xb0\xb9\x1e\xd8\x804\x82\xeb\x12\xc7\xb2\x0em&m-\xf8De\x8d\x93\x12\t\t\xe6\xaa3j\x8ay\xa0\x89[\x8e\xa7\xb9\xab\xed)?}\xf5\xe8\xf6.\xf9\xe9\x91\xdf0\x02\xc6\t\xe8\xb9\n\x06\xfb\x12\xae\xebC\xbc\x8f\xe9k\x18\xa9\x03,\x86\xcc\x18\x95\xb1\x8b\x0e\xf1\tW)6\x9a\x04\x8fO\x98\x91\xa7|\x08\xc8-\x81\xd8U\x0e\xa1h\xe6\xde\xa6\xa5\xf2d\x8d\x89\x16[^\xb2\xe1\xcd\xe4X\xdf7\n\xb3\x95\xad\x17\x0b\x9bw\xc9O_w|\x94z\x08\x04O\xf9\xcd\x8e\x9a\xe1\xe42\x1d\xaa\xf0E>\x93P\xc0\xc2i\x9d\x16\xad\x803\\\xe7\x06C\n\x08\xf9\x0c\xecZ\x12\xbcK\xf46?m\xee\x1e\xb6\r\x0f\xf2m/E\x15M\xd0w\x9d\x992\xe5@H\x92_\x99\xf72\xb1\x13\xe6[\xd4\xf4\xcfM?\xfdZ\x1e\xe2\xcb\x01\xeci3\xec\xbaCw\xa6">\xc5{\xbc\xdaH\x86\xb9\x97S\xb7\x93\x94k\x0bZ\xbe\xea\x9d\xb8+\x0f\xba\x1b=\xdd9\xb1\xa4&\x86\xb0\xd4\xc3\xb4\xc3\xdf\xa0mpt\xa4[7n\xa2\xe1\xee`\xba;\xb5.\xee\x14p\x003}\xf3"\xf7\xf8.\xfa\xe9\x11&J\xa2K}\xf8L<\xcbW\xcb#\xe5\x08@\x0c\x03\x9f\x931K\xa5\xf1B\xb4\xf4i\xf6&\\\x05\x95\x11\x13,\xb0DLj\x9fM\x10~q\xc6\xad\'\x03\xea\x9di\xcd\xe2\xe8.u\xce\xdd\xdb"6\xe9\xa0\xc2=\xb8\xc5p\xa0\xaa\x00n\x9f_,m\xdeE?}\x84I\xe1\xc4\xc3Dx\xe2\xd7\x8b`4\x92\xee\x18\x13^\xd6\x9f\xfdm\x07\xc1p|\xe4\xeaT\xbb{\xc154\x8f\x01\x08FW\xfe8\xd7\xa3\xa7\x1b^\xe6\xea\xc4\xf1\xb6\x94\xfc\xc3\x99\x17\x1d\xbd\x9dD\x81\xbe\x8c5J\x897\r\x95.@;\x92\xa4\xc9\xbd\xf6\xd4\xe6]\xf4\xd3G\x98\x8f_\x1e\x11\xf4S\xb1\xcfAw;\xf0\x05J+\xaff \xf6\xa9\xbalV\xc58%j\xafQ\x8dTb\x05\xe8\xa2\xdd\xa6\xd6g\x1b\x0f\xc3T\xddy\\J\x80\x92\xe8Kb.Qr\xd4\x05NYI\xca\xc9\xf7F;\x8fs\x7fG\xa7\x97\xf9\xc5\xd2\xe6]\xf4\xd3\xd7DN.\xd5\xcd\xf3\xda\x14\xe9h\x18O\x88~wQ9;Z\x82s\xf2\xe4\x00Y\x82:\xc3\x817\xdf\xc7\xa6h\xf7&\xd8\xb9cn\x15\x02\xef\x97\xa3\x80\xce\x8d\x7f\x16\x90\xa1]Jw\xe4\xd2\xef\xb4f\x1f\xedD\x1b\xe3\xaf\xe2\xf6t7o\x8e\xf0b"\x7f\x17\xfd\xf4Q\xa8\x12KA\x8e\xc0O\x89\xbc\xcb\xe1\xdb$\xa4\x15p]\xbex\xd9\x01\xaa\xc2\xce\xd5=\xdc\x04\xdaY=\xc9\xfa\x15\x1f\x00S\xc97\xe3A\xd9\x05\xa3#\xc7Ey\xd1\x01\x91\xce[\xaf\xf6\r\xe1p\x0c\x11\xbd\x0b\xf3\x84\x9ef8\xbeS\xca\xa4\x86\xe3\xe6\xefI?}\xcdp\x18\xf4x\xd1S\x86\xd3\xb6\x10\xae#\xb0\xc5\x80t\x19L\xba?\xd1FTF>\x8a\x88\xe6\xaegN.r\xa4\tVM\xd0s\xad\xe2Cn\xf1y\x7f)\xfb\x13\xc0U$\xbboO@\x19M\xf9\xf1\x98t\xf5r\xc6\xdb\x13\xbc\x8a@R\xf7\xea9\xffM\xf4\xd3\x12\xe6R\xa4B\x08\x8a\xd6\xf2\xef\xe9\xa7\xaf\x9f\x11\\\xcd\x9ao|x\xfeod\xd6\xfcD\xce\xc2JW\xacf\xcdj\xd6\xacf\xcd\xba\xf0W\xb3f5k>\xefu\xaef\xcdj\xd6\xacf\xcdj\xd6|f\x0bf5kV\xb3f5kV\xb3\xe6\xd3^\xe7j\xd6\xacf\xcdj\xd6\xacf\xcdg\xb6`V\xb3f5kV\xb3f5k>\xedu\xaef\xcdj\xd6\xacf\xcdj\xd6|f\x0bf5kV\xb3f5kV\xb3\xe6\xd3^\xe7j\xd6\xacf\xcdj\xd6\xacf\xcdg\xb6`V\xb3f5kV\xb3f5k>\xedu\xaef\xcdj\xd6\xacf\xcdj\xd6|f\x0bf5kV\xb3f5kV\xb3\xe6\xd3^\xe7\xbf\xa53\xfcCY\xf1Ou\x86?\x8c\xf2g6k\xbe\x7f`?\xc8\xac\xf9\xfe\x81\xfd \xb3\xe6\xfb\x07\xf6\x83\xcc\x9a\xd7\x02\xfbw\x04\x94\x1fd\xd6|\xff\x11\xfbAf\xcd\x7f y\xfcG\xcc\x9a\xaf\x7f\xf0?I\x88\xa1\xfe\xfb\x9e\xff\xb5\x10\x03S"K\xe28\xc6\xd1\x18\x04C\x14\x87\xf1\x18\x07\x89\x04\xce\xb2$\x8cQ<\x06\xb1\x1cKb,\xc2\x90\xa2\x88\xd2<\x81c\x14+\x10\x04\x87\n\x0f1\x03z4g\xfc6~ p(\xff\xe8\x82\x84C\x08\xc9\t\x08\x89"0*\x108\xce\xc3<\x8c\xc0\x04\x83\xd3\xb8\x08\xf1(\x8d?:\xce\x90,\x86\xe1\x8f\xef\xe3EN\xe0\x10\x96\xa6\xc4\xef)\xc4\xfc\xd6|\xf8\xcbS\x03\x06\xf2\x7fA\xf0/\x8fV7\x04\x8c\x7fm\x9f\xf7-!\xe6\xf3\xf3:\x7f"\xc4P\x08\x84\x0b\x8f[\r//A8\x98\xc1\x1e\xfc\tGs\x1c\xb4\xc4\x81\xb2\xcb$\x10\xd0\x07J\x82\xa2"\x051\x0cL\xb30$\x90\x0f\x92\x84\xe28\xf4q\x17\xbf\xbb\x10\xf3\x06\xc7\xe5\xa7\x15b\x08\x96\x12q\x02%\x97\x89\xc4c\x04\x8d#\x1c\xca\xe18\xc5P\x18\x8b2\xcb\xbd\xc4!\x9c\xe1P\x82\x11\x96\xe5\x07#(\xb4\xcc*\x1cf1~Y\x94\x1c\x01\x13\xd8\x97?\x17b\xde0N\xff\x19!\xe6_vK\xde&\xc4<\x1a\x9b\xfc\xa5\x10\xf3\xf9\xd7\xf9\x0f\x15bp\xe2\xdbM\xf7\x87\xbdO\xa9U;\x91I\xda\xc1\x9bK\xe4Ef\xe6\n\x1eY\x1a\x89Q\x9c\x19\xcf\x8c\xcewOT=\xd0\xb9\xf7\x86ol\xb7y\xd8\xc8;\xdd\xe2-D\x9f\x02\x07\xdf\x18q\xa6\x8a\xfd\xc6;\xfbt\x1c23\xc8X\xfb\x17[U\xbdG\x88\xf9\xba+\x100\x8a\xd1\xcb4\xfec\x98s\xdb\xc0\x86:\xdc\x0b\xbb\tKh\x0f\x0c\xe7\xd1$`\xc4\xef=\x9aG\xfa\x99\x95\xb6QC\x82\x00Y\x1c\x92\xa6\xb8Y\xf4\xd4D\x84\x1en\x1c\rf\xe5\r\xce`p\xa2\xe8\xd0.\xb9\x0e\xb7\xbb\x8f\x11\rn\xdf<\xf3\xc5\x1eN\xef\x11b>\xc2\xa4\x96\xc2\xe5\xb9\x15=I7\x88\xb7\xbb\x9e\xf2\xbe\xa3\x98^\x0c\x1c\xfd\xa6\xe8\xae \xe6\x82)u\x14\xda\x97[\x9f\x11\x87IAs\x01\x99x\x13\x9306\xeb1\x9e\xcaj\x94\xcf\xd0\x8a\x9b\x18l\xaf\xf7\x80>_\x0c\xf6\xb0\xcf|\x82N\x0f/\xb6\xa8{\x8f\x10\xf3u()j\xc9\xc2\xd4S\xe3\xd8\x1b~;m(\nm\xce\x8e\xdd\x917\x07\x08\xb0\xab7\xa9dn\xaa\x88\xc3\xb6\xcbi\xb7\xf0dm\x07\x037\xdf\xado^\x00O\xc3\xdd\xe1\xf6S\x07\x1aiJ\xe7\x17\xbc\xda\xd6\x9dxvQ\xde\x07y\x15B\xfdbf>W\x87\xba\xf7\x001_o"\r\xd3\x8f\xfb\xff\xd4\xaa\x19m\xa9;\xe3g\xa2mIb\x9fY"a\x86V<\x99^q\x8e\xc3V\xcal\x1e\xb3o\x98\x03\xab\x86\x7f<\xd9@v\xd8yQ\x1d\xa8.V\xa9Q`\x05\x86\xc1;f\xbb\xa4o\x83\x9b\xf4n\'Cr\x1b\xd2\xf7o\xdd\xc5\xef\n\xc4|\r\x13\xc3\x1e\n\x05\xf4\xd4~\x97\xde\xc2g\x0c:\xa3\x92\xa0j\xf4\x06\xa3\xecr\x0f\xd9n\xd4\x9am@\xc6\xda\x96\x85\xccTa\x9c\x13\x8bOb\xda\x89w\xc3\x83\x88>\xf1\xce\xed\x89\xb8\xddL\xf4J6\xc8>\xe4L\xa7\xa0\xf8\x94\xeb\xfd2\xe9\xf8\x89~\xb1C\xdd{\x80\x98\xe7\x9a\xf7\x1f\xc2$\xcd\n\xbfL\x08@\xa0ez\x95\xef\x07\xd4\xe9\xfb\xb3\xbb7\xb21lt\xbd\x17F\xe3$\xa7\xb1b9F\xb9\xbf\xd2r\x98\x87\xa4K\x1c\xd8\x13C\xa6\xdb\xfaZeD|\xc7\xbd\xb0\x9c\xb0`T\x83\xe0\x14\x10)\xc3\xbe\xd8L\xf9-@\xcc\xaf\xa3\t!\xd0\xb2\xa1?%8TKg\xdf\x9f\xfa\xa1"\x00\xd8\xbd\\3YA=g6\x03\xff\x9cq\tY\xc1QX\xc3\xc2\xe1\x803;(\x08\x0f\x10\x8c\x9eN\x1cW\x95\xa7\xa6]\xf6\x04\x95h\xb1\xac\x81\xfbp\xb6#\n:\xde\xb6\xf2\xa4i\x87\x17\'\xed{\x80\x98_\xc3\x84\x97y\xbb\x14\xb9\x7f\x0c\xf3\x12o\xb3\xf62\r\x0cx\xbd;\x88\xeb\xedET\x02\x88\xde\xdc\xee\xea\xab\xd7\x84@\xaajAz\xdcW\x94&\xefl\xe9n\x98\x15\x8d\xd4V\xcb\xeaW\xeb\xa8\xee\x14\xd6"/8\x01\x9e\x15\xb9\xdfIX\xb3o\x85\xbe5^\xdd\x92\xdf\x02\xc4|\r\xf3\xd1d\x0f\xa1\xc8\xa70\xef\xf3\x01\xcb+\xea\xb2\xe5\xe6\x0b\xb9dpQ\x98/\xbd\xcf\xe0\x1b=*L}\xbc\xd7\xe1\x91:^S\xa3\xa0P\x85M\xa8\x03w\xdbQ\xe4)6\x8b&\xb9\x98\x12\x83\xf6\xd6\xc4\r\x1bkd8sb\x04\x89\x92%L\xfd\x96\x9c\xf2]\x81\x98\xdfFs9] \xcf\r\x01\x9d\xd1Tg:\x8cj\x16fL\xdc\xf6J\xdbc/z`\xcb\xb2\xcc7\x16\xae\xc8\x07\x05\xb8K\xd4.V.\xd3\x84\x9bP^\xef\xda[\x85L\xf9Qu+\x0e`!L\xd9\x97\x87\xce\xb9^\x91`\xb6O\x00{6\xd0\xcf\xd5\x1a\xfb=@\xcc\xaf\x9b>\n\x934\xf9\xdcjTJ\xa4\xf1\xe0\x10\x9cy\xd4\xcd,\'S\xcb\x138n\x146\x9c(\xc7\xad4%\x14\xe52\x04\xb8\xad\xbc\xb1\x91\x1a\xe1\x14\x1a\xe6\xee\xd6\x17#=\x82`\nP\x11H\x1e;\xc3\xeb\nN\xcb\x8fX\x19\x0e\xa5\xd4I\xdd\x8b\x04\xd6{\x80\x98\x8f0\t\x8c\xa0p\x82\x82\x9e\x8a\x1b\n\xbc\xb7\xf4\xa9\xe7\xc9L\xca\x96\xf3H\xa8{\xde>*:\x7f<\xa0\xe7\x04\x94\xaeA{\xdf\x93v\x98\x1c\x19*8\xc2\xc4!\xaf\x8eHDH\xd6\x1e\xbaW\x0e\x97\x0c2\xba\xa7(\xbdN\xfb\x18\xd7\x10\xa8\n\xd03%\xff\x10 \xe6\xd7\xa5\x8fQK\x0ey\xee2l\xc0\x8ay\xd5\xe6\xad\xd5\xe0U\xb4\xd4\x8c2\x97\xed`+\xc8X\'\x82\x85\xc2\xf1\xf9#\xdd\x8d\x99\xe2\xd5\x92zF`o\xda\xa3&\xb6\x1cH\x1c\rD\xa3\xfdLu\xce\xa1T&\x06\xdc/{v\xae\xecc?0\x02\xfd\x93A\x11o\x01b~]\x13\x18\x89\xd0\xc4s\x83\xf1s\x0f;My\xd6\xc3+}Q\xd8\x9b\x15T\x1b\xb3\xcd%\xea0\xa3\x96\xba\xc7\x12\xd1\xbf15\x9c\xb6\x81\xd6oT\xc0e\x8c0Mw\xa3\x1cP\xadx\tx\xef\xaaY)O\xd8\xcda\xaeX0\xb9\xc5\xe5Re6\xaf\x8a\x90o!b\xbe\x9ei h9\x95c\xd0\xd3d9\xc7e\xc1\xe9\xbe\xc9\nN\x19;M\xc04n|\xf7\xd0\x8b\xe3\x96\x15\x17\xb6\x13\xea\xba1<\x84\x9a\x04\x9e\xfc\x1b+\x11\x83\x99@@{\x8c\xef3\xdb^v8\x9a\xf6\xceE\xcb\xb28\x99\x8e\xc8\x15\xe1\xb9\xd3\xadh_l0\xfe\x1e"\xe6\xebh\xa2\xd0r,\xfa\x13\xe4\xaf\x1c\x8fBA\xb7\xc4f\x17\x97\xcb\xd4\x97\xcequd\xeeZ\xd9k\x99\xe8t\xd4|\xe2s\x1d\x08\x12\xa3\xab\x1dD\xe2A\xd0\xab\x0e\xf7\x82\xa2\xc6\xac7\xbd\n\xe4\x073LQ\xa2N\xd9$\x9c\x0eEve\xcfw\xe8\xf4\xe2v\xf8\x1e"\xe6\xd70i\n]\xea\xd5\xa7.\xc3R"BX\x0b\x9b\xa8<]\xae\xe6\x0c\xa9BwD\xb3\x93q\xc9#\xe6V;\x03\xee^\xafJ\xa8\x9c<2\xe0D\x85i\xbb\x8c$\xafK\x1a7H\xda\xa8s\x85\xcd8\x8f\x00\x01\x82\x96\xce\xf9\x8eO}\x92\x82X\xf1[\x87\xd4\x9f\xd9\x88\xf95\x81\xa2\x14\xb9\x1c\x8b\x9e\xd59\xb6\xd3obh\xdb\'\x17W\xa9\x9ewA\x90\xb9\x8e\xb5~\xed\x0fs\xaea\xc2\xe1\x92d\xb1\xe5\x04\x80(\xb8\x84\xe9o\x94\xfc~\xce/\'Y8\xea\xb6\xa4\x11\x9eK\xa7\xb6\xa2\xc4\xb9\x7fD\xdc\xd8\x8d\x0c\xfd\xec\xe6\xfckk\xe2=F\xcc\xafOmP\x12\'\t\xe4\xe9\xf8v\xda\xc1\x17"T\x8b)K\x02\xf3b\x87\xd1\xce\x86\xd2R\xba\xedU\xae2<9\x0b\x82\xfd\xe6n\xdeP\xad\xf6\xa8^!fp\xa6\xad\xca\xa6[\x14\xd0\xc3\x92\x92\xd3nN\xeb\xd67\xbbSt\x86\xf9\xf1|\xb4\x1a\x95y\x95\xc2y\x0b\x12\xf35\xc3a\x08\n/\xf9\xe2i\xd7\x9fTv\x8e6\xdb\x1c\x08/\xc69\x9e\xae\x98]\xe8\xb3\xc5\x9d\x84ILN\xad\x9c\x84\x00f\x87\'\xec^\xb1J;\x1d\x06u\xe8h\xac\xc3\xb6[\x87\xbb\x08\xd3rA\x9d8\xcf\x93\xa2N\x16\x17W\xf0E\xa1.9a\xbeZ\t\xbf\x05\x89\xf9u\xd2"\xcb\x8c\xfd\x93\'7\xa4~QP\xb0\xd1\x98"\xf7&\xda9\x8dd7I\x10\xa0_2\xad\xcb;4 \xb9\x82\xb5\x9b)=+\xd6\xbc\xcd\xe3\rt9\xb5\xd8\x8eeC\x9e\xdf\x17\x86c4\x8d\xb4K\x14\xc7\x11\x9d\xa6\xd9\xcbD\xa5T&\xf9\xad\xfd\xeagFb~=O \xcb\x01\x13\xa3\x9f\xb6\xc32\x80\x96\xba\xba\x07yH\x15\xe3-H\xe7\xac\x94\xd2Gu\xcc\xb0-\x1d\xd8\xe0\x10v\xc3\xe9\xe4\xed\x9d\x94kB\xf7\xd8m%\x95\x99\xe7\x06\x92\xb0\r*\x98|\xa9\x19\xfd=\xb9\xf0\x15\xa5\xf4\xfcqK\x1d\xe28@\xb2\x17\'\xcb{\x90\x98_\xf7\t\x9cZ\xee\xd1\xb3\x98\xb6\xbb\x1f@\xa6\xbc{\xa6"\xc2\xd4\xc0\xf3\x02\x00e\xa9\x11\x17g\xf8\xa8\x0e\x17$\xd9c\xa7\x165\xb8m\xde\xe3\xa0^$\xb2h\xe1\xb1\x945{\x91\x14\x84\x1d\xdaC2b\xf3*\xceOS\xc9y\xe5\xa1\xa2\xd3\xfa\x96\xfe\x10$\xe6\xd70\x973\r\x86\x10Ok\xc2\x16\x05\xab\x9e\x86\xa3#l\x9d#\xbb\xa1\xd5\xad\xb6s4\x996\x0f\xe8p\x01\xb9\xddy\x9b\xbb\x86\xb5O\xcbq{\x18U[L\xfd\x96@M\xe5\x1eC\x9b[W\x12\x8c\xb1\xb1\xafQ\xc5v\xd0\x05\xbbR\x8a\xae^$\xf2\xd5\xe7\xd2\xefAb\xbe\x86I>,\x12\x1a{\x9a\xb4H\xdfF|\x08j\x970\x03&\x02\xd5.6 \x8e\xc7\x10n\xcdi\xf48\xebx\xe1\xfb\x93u\xb3Yi\xdbcE.3\xde\x1e\x0e\x81\xfc\xd28\x95)\xc1C\xb5\xcb\xcd\xad\t\x18\x11\x9f\xd7\xea\xf5\x18D7\xc9)\xbe\xa9z\x7fW$\xe6\xd7D\xbe\x1c\xde\x90\xa5&z\x1aM\xbbM\xc9\x93|\xe8\x8f\xbaVx\x9bF\x11\xfc"\x172I\xa2 \xa3\xb8\xda\x84\x1af\x9bt\x1ec\xd9\x8f\xcb\xbd\xb7\x85u\xf5\xaa\x81\xbbr\x82\x12\xd9\xaa2p\x86\r`\xec\xd1\x0bC\xe4K\xf8\xd2\x99\xc2\xc5\x91y\xf1\x94\xfa\x1e$\xe6\xb7\xd1\xc4(\x04z\xb6\xdb.\xdd\xb01e^,O3\xa5\x86{^\x9f\x80S\xc5\xc9\r\xaa\xf9\xb9\x82x"e\xece&\xd3\xedI\x84n\xee\x92\xc1\xe9\xe1\x0cr\x94\x12\x06\x14\xb5\x9b\xe0\xd4\x1c$\xbd\x92\xbd\x9b\xe1\xa6 ,\xf5\xb0\x14k\xe7\xfcs\xd5p\xefAb~]\xfa\xcb\x8a \x10\xecy;d\xdc-\xbb3\xc2\xda\xde$\x87S\xbeG\x06<\x04\xf8C\xb8Q\xab\xdb\x94\xb6\x8d\xc1\xf7t19I\xb6\x1b\xacZ\xc6\xd5\x9d\x8d*\r\x99\xc4N\xb4\xdf\x99\xa7\xc0\x9f{\xf96\x90\xf9\x1d\xbcd\x0c\xa2\x10M\xbe\xc1\xe0\x17\xe9\xcb\xf7 1_\x1fi,G7\x08{~\x02\xdf\x86\xea\rr\xa7\xf9\xcc\xddq\x06\xdc\x94\x82\x90\x91W\xcf1\x06/\x9c3 \x92|\xe5\x90\x14\xc1\xd5\x84\xe7\xaa\x1f\x1a\xf9\xd0\x9e\xc0\xde\xad+\xc7.\xd0\x12\x1b\x8f3\x8e\xd6\xb8\xaa\x05\x83\\\xb8\xcd\x81d+\xc9\xe0\xc1\x17W\xfe{\x8c\x98\xaf+\x1fY\x8e5\xd8\x92\xcd\xff\x18\xa6\x8b\xf6s\x1e\xcc\xe9\r\x1d\xce\xc7\xda\xb05)\xf1\x87\xb3\x16T\xbc\xa9\x02\x90\xca F\xcc\xd4 [\xfa\xd5p\xde\x93\xb6]\x91N\x1c\xd4\x19}\t\x9a\x0bm86\n;\x99*l\xa4\xa2W\xf7ev\x8d\x12wV^|\x02\xff\x1e#\xe6w\xcfl\xf1\xe7_\x1b\xdd\xd4\x19\xd7\xf2T`:\xa96\xae\xbe\x1e\xa1\x0c\xb7\x97\xa4\r\xc9\x85\x962\x16\x18=\xe2\xc0^V\xe6\xa5\xd8n\xa2\xca\x1a\xc4$\xa4\xec\x8d\xc2\xba\xf6\xa5\xc4\xca4\xc3\x0frv\x04\x8a\x1e\x0e\x0f\x15\x8fB<\xa4\'\xf3\xb7\xb6\xab\x9f\xd9\x88\xf9\xed\x89\x06\xb9\xac\t\xf8i\xe5WS\xbd\xc1z\xcf\xee.=\x90\x1d\xcbId\xf7]\x95\xed\x04\x99\xbeA\xf8Mn\x1a\x8c\xcblG\xc8\xd54\xc6\x8b$\xf4#\x80>\xd1W\x1c36\xd4\xb9\xf5\xbdR\xe3\xd0;\x1e\xe5\xe7\x1d\t\x10\x90h7\x15x\x1b\xbe\xfe\x1e\xe3\x9f\x1a1\x1fO\xecW#\xe6_\xfe\xb0\xfa\xdf\xc7\x88\xf9\x89\xa8\x88U\xdfX\x8d\x98\xd5\x88\xf9\x9f8KW#f5bV#f5bV#f5bV#\xe6\xf3^\xe7j\xc4\xacF\xccj\xc4\xacF\xccg\xb6WV#f5bV#f5b>\xedu\xaeF\xccj\xc4\xacF\xccj\xc4|f{e5bV#f5bV#\xe6\xd3^\xe7j\xc4\xacF\xccj\xc4\xacF\xccg\xb6WV#f5bV#f5b>\xedu\xaeF\xccj\xc4\xacF\xccj\xc4|f{e5bV#f5bV#\xe6\xd3^\xe7j\xc4\xacF\xccj\xc4\xacF\xccg\xb6W\xfe\xd6F\xcc?\x9cz\xff\xa9\x86\xf0\x87"\xe43\x1b1\xdf?\xb0\x1fd\xc4|\xff\xc0~\x90\x11\xf3\xfd\x03\xfbAF\xcck\x81\xfd;\xe2\xc8\x0f2b\xbe\xff\x88\xfd #\xe6?\x90U\xef\x12\x1a\x96\x99\x82\xc2K1\xb1$\xf4?\xdeE\xf8\xecif\x01\x81\xcd\xf6B\xab\xfe}\xa6R\x15\xa5\xa5S7\x19\xd9e3\x90]1\x1c[o@\xa6\xdb\xb6\xec\x8e\xa9}\xb7\xae\x85\x8boL\xa9\xdb \xc3\r\xb9\xa7]\xe8\xe4\xe50 J\n\xa1\xb1f\xfa\x1b\x94\xe4\xf9\xd7\x9a\x9a\xbd\xcd\xc2Y\xc2\\\xca"\x8c@\x9e\xdb\xa8\xef\xf1$V\r\x8f!\xae\xcc0)\x05\xd0Vj\x8b\xa5c\xc0\x1c\x06\x1eS\xe5sye\xc5vJ\x83Rs\xaaTe\xeb\x96;u\xa8&\xe3\xb8\xa8\xf2&W\x17V\x05K\xb7L\x8bA\xb9`\xc7\x82\xa5=\x02\xfaQ\x14\xceR\xf3.\x15\x06\x02C\xc8\xd3`\xf2\xae\x1c\xed\xb7\x93\x0e\xee\xbd.6\x96\xd3\xcd\xd6\xba\xec\x8eY\xc9\xe8\xcc\x1e\x13\xce\x8c\xc6\xc0[\xb7\x1289\xdf\xf5U\xcd\xfb\xd1]\x05BV\xd1\xab\xf35/6\xe2\x04h\x8c\x14\x8d\xb4\xbf\xc3x\xda\xcb\r]\xc2\xa6\xcb\x8b\xed\x06\xdfF\xe1P\xbf \x10\x85P$I<\x8df\r\xe2\x07\xbb\x82\x0e\xad\xbfW")\xb3\xfav\xda\x80\xc9\x06W\x92v{s9*\x89\xae\x9b\xed\xa4\x1d\xef\xe9\x1d\xd4\x9b\xc2\x19\xd5I&\xa8}\xef[AN\xcb\x0e\rH\xa9\xb5\xa9\xed\xcd5B\x03\x93\x15v^{r_\x1c\xcd\xb7Q8\xcb\x9c}\xacr\x88~\x16\x14\xf6\xd1N\xe1\x0b\n\x0f.y\xee\xc2g&Q\x1b\xceQ\xee\xad\xbe\xe5A\x8e\xcc\x0f\x17\x8b\x9abm\x17u\xf1\xe6\xe6\x17\x17\r3\xcb@\x175!tq\x0e\xb62R\xd8\x8bVH\x1bz\xbf\xd1\xcd3sMb\xe7<\xbc\xd8o\xf0m\x14\xce\x12\xe6\xa3\r#LbO\xdb\x15LY\xe1C\x808\x8b\x03\nyx\x8e\x08n( :\x03\xdd\xb6\xed^b\xbcz)\xf6/\xfb\xabZ_\xe7m.e\xb9\xcct,\x0b\xc6GIu\xaaM\xdd\xa9t\x930\x1eY\x06\x165\xf8\x07\xf2v\xb5e\'\xe3^\xdb\xae\xdef\xc4,k\x93x\xe4\x1f\x12\x7fj\x08\xa8\xa6|C\xb4\x98=\xd5;\xa5l\x80\xed\xa0\x01\x8a.\xd6\x16*S\xd7\xd2;\xa4\xed\x99\xcd\xecZ\xb4\\\x84\x05t\xd9\xd9\x1dt\x14\x9eE\xe1v\xe7\xe7l\xb2\x19G\xdc\xb4\xe7\x84\xa8C\xf4*\x17\x83[\xd7\x1bN\xf4>\xd7v\xf56#fY\xfa\xd8r\x02#P\xfa\xa9L\xe5\x03\xcd\x8f\xd0\xc8\xba\xf1l\x8bt\xbd\xb5\x1fv\x14\xd7\x15\xfdTlM\x0b\xba0\xb5\x94nw\x9be\x15\x90\xbd\xea\x1e\xc3]\xa5"h\xd9gIv\xd9\xc2\xfb\x83\xee\x06\x8e\xc5\x82\xc4\xfe\x0c\xf9c\x88\xc9\xc7\xdb\xb5\xca\x89\x17\x97\xfe\xdb\x8c\x98?<\xbc\xf8\x07\x05\x8b\xca\x88\x12\xd6\x0b\xb9g\xb6\x17\xee(\x0b;\xb4"N\x80z\xeb\xb3\xfef\x9cs\x96\xe6\x0c\x86\xe8\xc9\x06\x1e\x8fU\xe2\xb8\xe61\x8d9$\xbd\xb9J\x180^P\x9f\xaf\xbdf\x99\xda=Gid\x8e\xabm`x\xd2\x8b\xbd\xff\xdff\xc4|\xec\xca\x04\x06Q\xcf\x9d\xb1u\xf9\xaev\xc4\x8e\xd7\xae6\xe6\x12u\xb5\x9c\xb3[*\xc6\xf6AB\xa5\x80\xb1\x9c\x9cu[\xcf\xc6\xf1|#\x95\xf6\xd1\xb3>\x9a\x81\\\xbe\x12\xe4\xfd\xb0\x99\x0e-\x17\x80B\x18\x96Q/f\xa3\xae6\xdbi\x7f\xd1\x8e\xc2\xe7\xea\x16\xff6#\xe6Q\xc2\xa1$N\xc1\xcf\x85\xb0>\xb7\xd8M/\xe3\x01\x1d\x99\x82\x96\xd3F\x8e\x1a\x06s\x02\xd3E\xf3\xe0\x10\xd8\x05\x94\xb5\xbd\xdc5$\xbd\xbb\xd85X\x930\x87^\xa7\xf0D\x8cP>)\xe5q\xb3\xab\x0e\xa9Q6[\x88%\xb5\xf9X\xf8\xa0I\xbf\x98@\xdff\xc4,a.\x87\x1f\x02\xa2\xc8\xa7\xedP\xe5\xb9Cv*\xed\xfa\xee;\x9b\xce\xca\x98\xa65G\xfcDjC\x86)\x08\x0b;\xb3;\x93jkM\x17\xc7\xa6$uRi\xb8\xc6\xe3\xddF\xb9\xddw&\x84\x01\x19r\x0b\x8a3\xd2\xef\xec\x8c\xe3\xf5\xfa\xb4l;\xd6\x8bk\xe2mF\xcc\x12&L/[\x0eJ=\x157\x1a\x7f\x0e\xb1\x9e)\xa3\xc6\x9e\xf6`\x87\xcd\\y;"N\x0c\xd6l\x85\xa0pH\xceh_%\xba\xedO"\xbc\xe7n\n\xe7Bdu9\x93\xd3\x05@\xf5\x8e\xcaxt*G\xaa\xba\xc6Nt\xbbiW~\xbe\xda\xe7\x17Ooo3b>\xb6Cj\xc9\xe4\xcf&-q\xa3R\x02dC\xdd\x1a\xa2\xda\x90`f\xc4Qd\x12%6\rc\x82\xafF\xf7J\xc7\x07U\x86\xa7\x83}\x80o\xedU(\xceVX\xa6\xdc\x8d\xe7\x8fb\xa0\x8e\x83Z\x1a\xcem\x12\x98ns\x04T\x1a\x92wW\xf7[a\xfe\xe4F\xcc\xe3\x9c\x0f#\x8f\xdf\x1f\xb2=\xb2\xe9L\x01\x97\xa5\x9c\x94\xf6Zh\x17EeKL\x9d\xd3\xc3\x85d\x87\xd2\xbc\x80\xe0\x18\xe1\t\x94b!\nEN+{nzc\x13I\x8b_<\xbd\xbd\xcd\x88Y&\xcb\x03\x12[\xe6\xf8S\x98\xae\xe2\x12\xbb\xf22\x96fr\xeeeI9\x97)\x08\x1b\xa6m\x13\x01\x91B\x9a\x1bG\xf6\r\xdf\x14\xa2\xb8\xe1\x88\xd3\x12\xea\x0el\xa8M/P`\xc0n\x8e\x9e\x19\xfb\xd3\xf5\xba\x05\xae\xd2\xe9\x0c\x1d\xad\xd8 \x87\xdd~x5\xccw\x191\x8f\x0c\xb7\xac\'\x9c\xc4\x9e\xdc$\xb0\x1b\x94\xcc\xadDo\xd4\xfa\xe4\xb4\x0cO\xef\x07\xd2\xb9\xf0H\x9e\xb8\xa3\xf7$vI&\xa4K6=\nD\xeb)\xfa\xd4\x9ew\r\x1f\r\xf7\x9d\xc6\x86\x07v\x10\xe2\xb4\xa7\xaa\xa0A\x1a\xfb\xff\xb1wg\xcb\x8e\xa2\xd7\xa2\xa8\xdf\xc5\xb7D\x98\xbe\xbb\xa4\x15B4\xa2G\xdc\xd1\x0b$D\x8f\x04O\x7f\xd0\xac\xb3\xcf\xda.U\xda\xd6\n\xa5s\xa6\x0f\x8er8*j\xa6\xcc\xf8\xdb1\xd0\xac\xf15GA\xd8\xb32\xa5\xfd\x90\x11\xfc\xd9F\xcc\xb3\xae!\xe8\xa7\x15\xf1r\xc2\xedg\xf6\x92\xca\x1d\xe2\xe9*\xd9\x9d\x81\xbbN\x86\xf4t\xb8\x9d\xaca?\x9dpB\xd0\x82#u+9\xe7F\x84\t_\x13\xe68\x87\x17HjL\xb6\xd9\x1b\x17\xdc\x93`a\xba\x8d;0\xaa\x0c?\x86`\xec\x94\xf5\x0e\xff\xbdd\xb8\x8f\x191\xeb\x9e\xc0(\xe4\t\xd0\xbf,\x96\x86\xcf\x9b\x04\x06J\x110\x1d\xd8F;/\xbf\xf3m\xaft\x0c\x82\x13z\xe6\x88\xfe\x00\x9e)j\x94+\xed\x00\x06#\x84z\xb1\xe6\x17\x05h\xb7(3\xc5\xb3\x18]\xec\x1e\xdb\xa9\xf1\xa1\x9dX)<\xf8#\x12\x8b\xcb\x9b\xd7\xe1\xc7\x8c\x98\xaf"\x18%\xfe\xc2I\xe5\xda(\xd9\xd7\xd3%%sH\xdf\x9f\x86+d/hP\xd6\xc3\xac"\xa5\xc9\x87\n\xa3\x03\xd3P\xd3\xd8E\x85\xd2\x90\xcc\xcf\xa6\x99\xe7\xb9\xb0\x1ba9\xe8\xac\xba:\x9d\xc3\xe4\xae*Q,\xa8\xfb\xe8\x01\xbb\xf7\x1d\x0e\xbe\xe9\xdf}\x8c\x88yN&\x89\xc3\xd0_\xbc\x96\x9e<4j\xab\x13\x8c\xf3L\xd0\x80\xc7"#\x07\x95\x0b\xcb\x9e\x0eu)\xdd\xca\x83\x02\xceE8\x13\xfe\x15%Gbg5\xd0\xb4\xc7\xd1s\xac>\xd6|D\x05]]\x88\x0fQ\x10g\x16\xb1;\x13z\xd2\xf1\xc9\x9b\xb3\xf91"\xe6\xeb\x1c_o,\xe8\xf5\xbb\x14\xa3\x873\xdf8a\x026\xd1GS\xacH\x99\xda\x0b\x8dv\xc9\xcfr\x9c\xb5\xe3\x94\xd6\x01[\xf2\xc1\xdeS\x1c\x0c\xcev\x08\x85\xde\x914\xf3\xc6s(\x85);\x85\xe0tZ\xb4\x98}pR)ya\x7f\xa3\x90J}\x934\xfb\x18\x11\xf3\xf5]\n\x05!\xeb\x9d\xf5\xe70\x87\x8c&\xeb;A\xd7\xa1}.\xd1v\x8e\x1e8\x00\xd7\x0f\x90:\xd8\x11X\xc9\x15\nu\xa2\xde\x14\xbc\xea\x0ei\xd0\xde\xd9\x84\xf3\x9c\x96\xbfZd}C\x1bEI\xe3\xde\xec\xf2\x19*\xf9\xac\x0c\x84\xcb\xe4\x84\x07\xce\xfeQ\x98\xbf9\x11\xf3|g\xbb\xe6~\xcf\xef\xee_8X\x88\xd8u\x88\xbf\xb4U~os\n\xaa\xd0\xf1\xbaW-\xff\xe4&\xa2i\x0e\xe6\x85\xbap\xdc\x81\xd3\xd95\xf7P\x1e\xf1\xa4\x13\xf0\xfdT YK\xbb\xe0\xa5\xa8\x9cH\x9a!J\x04nk\xd1\x80\xf2\xf2\xa3\x14\xa0\x98ys\xeb\x7f\x8c\x88yn}\x8c\xa2\t\x8c~\xd1\x0c\xe7\xc9\xf7\x82\xe5Vz\x02\x7f\x85I\xe9!\r\x1dj\x0e\x8f\x07L\xf5A\x0b\xaa\xae\x9d#\xcb\xa8\x17\xad\xa6`\xb3u\x85;\xd3[&Z\xc5`\xe7(\x88\xfb\xa6\x02\x10\x85\x80\xf7\xd5#\xc8\xe9\x87d\\\xeb\x08\x19N\xf2\xaf2b\x9e\xef\xa7`\xf4\x19\xeaK\x98\x84\x92\xac\x17P#\x0ef{(w\x92\x15AgL\x14\xd9SLT\\\x17R\xba\x8a\xcf\x0c\xf9p\x9cr\x8f\xd5G\xe4\xf9\x8b|\xcf_5\n\xee\xe7[--\xe2Z87\xed\x153\xeeg\xfeTstA3!\x14\xa8o\xa6p\x1f3b\x9e[\x9f\xc2\x08\x18\x83_\xaa\xb7\xa8\x11N\xa80\xf4\'9\xcb\xdc\xd4G\xd5j\xb8\x1ff\xcc0\xc3\xeb}\x94\x91t\xa7\xed\x8e\xb9\xbc\xc3t\x91\xa9\xac\xc2R\x9a\xda\xe6\x0c\x97.k\n\x85\x1e`\xe3\x83\x17\xfe\xf0\xb09\x15a|>\x1b\xd1Q\xf2\x05\xedG\xaf\xe1~s#f],\xebU\x83\xae?\xfa\xf2\xe6\xdb\x8cL\x1a\x1d\xe7\x13\tN\xd3\xc1\xac\x1a\x19\xe3\xf1\x1d\xfcp|\x12\xc5J\x9e0wWy\x06\x83\xc7\xfe\x94\xcfn\xc1\x04i\xe5\xe6"\xd2\x1a\xd0\x90\x0cAj\xa4\x08\xe4\x8e\'ipFS \x8bs\xe1^UtT\xfe(k\xfe\xa5\x11\xf3\xf5&f3b\xfe\xed_V\xff\xef1b~#}c\x03M\xb6!\xfd\xfeC\xba\x191\x9b\x11\xb3\x191\x9b\x11\xb3\x191[\x13\xfeo\xd8\x84\x7fcw6vgcw6vgcw6v\xe7\x1b\xb2;\x9b\x11\xb3\x191\x9b\x11\xb3\x191\x9b\x11\xb3\x191\x9b\x11\xf3\x8d\x9fs3b6#f3b6#\xe6;\xdb+\x9b\x11\xb3\x191\x9b\x11\xb3\x191\xdf\xf697#f3b6#f3b\xbe\xb3\xbd\xb2\x191\x9b\x11\xb3\x191\x9b\x11\xf3m\x9fs3b6#f3b6#\xe6;\xdb+\x9b\x11\xb3\x191\x9b\x11\xb3\x191\xdf\xf697#f3b\xfeK\x8d\x98\xcb[\x1a\xc2\x9fj\xe4\xefl\xc4\xfc\xfc\xc0~\x91\x11\xf3\xf3\x03\xfbEF\xcc\xcf\x0f\xec\x17\x191\xef\x05\xf6\xbf\x11G~\x91\x11\xf3\xf3g\xec\x17\x191\xff\x81\xc3c3b~\x86\x11\x83\xfd_\xf3\xf8\xcf\x8d\x18\x86CDF\x14h\x04\x83q\x82\xa1E\x18\xe3D^\x84y\x1cc\x9f=opR\xa4\t\x12\x17y^\xa0\x18F\xa0\xd6\x7f*R"\x8a\xa34\r\x89\xb0(<[\xfb\xfc\x13\xfe\x00\xe6(\x14ap\x14%\x9f:\x8c@\x08\x1c\x89=\xbb\x04\xe2\x0cNS\xb0@p0C\xc1\xcf\x86o0\x84\xb3\xa8\xb8\x86\xcd`\x14\xb7\xfe\x7f\xc18*p\xc4O4b\xfe\xbf\x06,\x7f\xfb\x8b\x06\x0c0\xfew\x08\x85i\n\xa7 \xec\x9f\x191\xdf\x1f\xd8\xf9\x0b#\xe6\xd9\xc3\x04_\xe7\x90\x16h\x1e\x12\xd7\x10!\x9cbq\x1e\xc7y\x16FQL`Y\x18\xc3(\x91\x80\x88\xf5SE\x04F\t\x8ac\xd69\xa4\xd7\ta\t\xe69n\x9b\x11\xf3\x13\x8d\x18\x02\x16\x04\x08F\xd6\x9f\xc1\x08\\\xc0x\x82\xc3\xd6\r*b$\x8b\xb18\'\x08$\x822(\x86\x8a\x04#0\x1c,\xb2\x04%\xae;\x91^W\x9a\x88\x13\xcfM\xf3\x9b\x1b1\xff\xb6\\\xf21#\xe6\xd9\xd8\xe4\x9f\x1a1\xdf\x7f\x9f\xffZ#\x86\xfeqWj\x8cT\x1f\xc9\xc5\xcb\xe2\x00\xb8s\x8a{\xad2\xa0\x98\x84\xe2\x91\x9e\\\x7f\x1dL\xa7/\xe6\xa1 \n\x1av\x04%@\x93i\x16\x0f\xbcty\x80\xa4rl\xaf\xcd"\xc7-\x16\x1f1\xd2\xae\xd4\x96\x84\xb4\xde\x12\x9d\xa4b\x7f\x8d\x11\xf3\xbc\x15(\xfa\xc9d\x91/\xcd\x8d\xba\xddp\tD\x12\xe1\x13\x1f\xbeE\xe9\xd1\xbb;\x15\xd85Uh\xdc1\x18E\xce\x89E\xa0)\xe4\xed\xe8\xe8\xd6u\x18$\xc3)KQ$zq\xee\xf9u/u\xd3\xc1\xf0\xfb\x1b\x7f \x9bD\x01\xf5@\x8d\x18\xd4\xf3\xde\xa4p>e\xc4\xaca\xc2\xf0:6/]\xc7R\xf8F?\x1e\x08\x94XS\xa3\xc8\x07\xee\x90`<\xa6\x10\xfe\xf1\x8a:\n\xcf\x19;\x86S\xd0\xf4R\xe0\xe7\xb18\x92\x12]\x02\xa6\xc2\xa3\xb4\x98\xe3{j\xa6\x94\xeb\x014\x8d\xcb\x01\x14za\x17)8\x1f\x8d\x9c\xfan\xd7\xb1O\x191\xcf\x18\xd1\xe7\x9fG\xb1\x970\xad\x01\xbe\\\xd1\x1a=\xe3\x16\xb5\x04\xbcv\xb8u\xce\xbe\xbbH\xa9g\x900\xae\xe5\x81\x1d\x98\x86\x81\xa8G9\xe9\xf91i8\x01w|\x17`\x0f76@[!t\xc45I\xee\xf5+\x1e\x86\x1e0\x9f\xfc\x0e8\xc4\xdf\xab\xe9\xfe\xa7\x8c\x98\xe7\x86 \xd7co\xbd\xf2^\xf6\xfd\x11\xd7\x87\x9c\x15\x14\x04\xde3\xe3\xa1\x0f\x90\x03\x10\x93\xd2\x99\xa5\xa1\xa0U\x02fG\x9c\xf9\x08\x10\x99\x01<\xa87\x99\xd0\xb3\xd4i@\xe6\x18Q\xf7\x08.*Za\xe1GI\x03\xbc\xa2\xeb\xa1\x01d\xf8\xe9Tp\xdd\xe5\xdd\x96\xd4\x1f2b\x9ea\xd2\x10\t\xc1\xf4\xcb\xb67\xc2\xc4\xf0\xb8A-\x90=\xe9\x13;\x90\xcb\xf3\xd3c8^\x1b\xd7Dr\xbc\xba:\xb0F\xd1a\xa2$:&\xa4g+$\x10\xbdJ\xda\xe1dh$/.5\r\x97\xf1\xd5\xd4R\x14|\xb8`i\xcb\x97\x0c\xe8\xb07\xbb6~\xca\x88\xf9\xcay\x11r\xfdQ\xf2e2\xddi\xba\x16\xe7\xde\xce\x94R\xecJ\x8d\x16\x8b\x93m\t\xc6\xe2\x1c\x03\xfc\xfa\x98q\xe6\xbc@\xf1\xf5\x8e\xdcU\xf4\x02E\x1c\x8f\xe6`\xe7\x08\xb70\xdc;\x10\xc5\x05\xd8\x85\xab\x94\x02\xee\x0eQ\xf2\xf0.d\xf1\x180\xaf}3\xccO\x1910\xf6w\x9a\xc6\xd7\x8b\nA_\xda\xe3\x12\xb3\x05\xeaK\x9e\x97\x80\x9f\xc2s^2WDO\r\xedx\xd5\xd5\xc9,\x1f @R-(\x01F\xb4\xd7J&i\xe5\xc9\xc9z\xc6?f\xdc\xe9\xd2\xb5\xc9Cs\xc3et,\xca\xba\xeec\xd4p\xe7\x92k\xce\xcc\x9b\xadF?e\xc4\x0eSp\xe7J_;+J\x05\xc4\xfb\x16\xcd\xd4\xeb\xdeq&\xb9\xae\x95\x8cN\x97\x9c\xbc\x02\x14{g\xd7|\xf9@Tg\xca\xf1z\xd8\x0f\xcd\xb4{\xb3\x07\xe7\xa7\x8c\x98\xe7\t\x87S4\xb9\x96\x8f/=\xa3\x99B\xc7\x17\xd1\xb2\x1c\xc71\x94r\xec\x8c\xabr\xc1\xaf\xa4\xc23m\xd3\x98\xe6\xd1\xa7r\x96\xec"\xef\xea\x1a\x90\x1b\r\x14\'J\x19f\xb9\x1as\xc4\xf7\x8d\x02\x1f\x90\xa1\xb0:\x86*A<\xafxx\xd6\'\xe6.\xbf\xd9\x14\xffSF\xccsO`(F=\r\x81\x97\xa2\x83u\xa4G\x9b\x1f\x851D\x10=.J\x95\xa5q[\xb9\x8eM\xc5\xcb0\xb2\xc8\x17fwk\xc1\xe3\x91\xb8S\x81{\x19\xcfx\xab*\x81\xa6RU\xdc\xfa\x05\xd3\xb0\xbb\x90\x0ek\'\xa2n\xce\x01\xa8$\x8e\xb0\xe3\xfa\xbf\xd3\x88\xf9J\xe1\xb0\xa7\xa1\xf0Z\xba-\xce\x89\x91\n\xb0\xc3\xc4v\xc2&\xfd\xc1W\xb2$\xb8\x0e\t\x0c\xfa\xb2\xe6O\xf7\x10\x1d\xc0\x9e\x95u\x93\xd3\x82:z,t\xa0\x86V\x99-\x8b;\x08\x0bmf\xfbl\xf1UC\xdf\xb9d~-\xab=\xea\xb4\xde\xf9\xcd\xd2\xedSF\xccs\xebC0\x0c\xad\xf9\xcdKK\xea\xf0\x91K\x0c\xc9\x9e\xfa\x93\x11\x16\xc0D\xaei\xcb4\xc7V\x9aRX\xec\xc8\xd0 \xb9\x14\xd8\xef\xe6R@\x12\x95(\x89\xfe\x04\xb3`[*\xc1\xce\xc7\x80\x19\xbe\xd4\x0e\xa7\xd0\x87\xa8\xb8\xceP\x84\xe2\x8a\x169\xa55\xf2\xef\xdd\xfa\x9f2b\xbe\xc2\\\xb3A\x02y\r\xb3\xc2x1\x8c\xd7\x8d%ZYo\xf1n\xa9>\xc0]\xe8\x8f\x08m\xb2s\xd8\xd2\xb8x\x84\xd4\x07pl\xf7\xb2\xc9\xdd\x0c\xfbX\x05Axl\xd3\xc7\xb5\xd0\xdd\xbe;\xc0A^\xb4\xd3\xc8\xee%\x170\x9a\x02\x88\xfb\xfc\xfcfr\xf3)#\xe6\xb9h\xd7\xe5\x8d\xacu\xc7\x0b\x82\x05/-\xe2\xba\x96\x98Ne\xed>j\xd2\x92\x9d\xbe\x9c]\x80\xcf\x10S\xdd\x17\x0f\xc1\xc7\xb0#i\x01\xaa\xa5\xef\xa4`]\xaa\xe6B;\xa0uon\xa00\x17\xf0\xad2m\x89\xeb\xce\xd8\x9e.-?m.\x8f\x07\xf6\xa3\x1a\xf5\xf7&b\xber\n\x8c\x86\x9eY\xe5KYc:\x91\xdb+\x83>G\xfcB\x1f\xf5\xfd)Qk\x96\xc8\xd0\x03\xa5\n\xe5\x8dP\xd0}}\x9d-\xae\'K7P\xcd\xb2\xcd\x87\xd3\xe9<>\xc2\n=F\x9d\x1e\x90\xe2p5G \x07\x1e\x1a\xc4\x9c\xd1.\xd24\xf7\xcd\x0c\xf1SD\xcc\xd75\x81\xc1\xd4z\xc4\xbd\x1cp\xa3\xbf\xbb\x81\xd8E\xef\xee\x91b\xb9\xd9\xbd\x97\x0f\xec\xc3\xbd\xf1\xd7\xc3yX\xea\x1b\xe1\xd0\xdaqdP\x0fS1\xb1:\xf4\xe1Atx\xd0C\x99\xd9\x8c\xa2\xd3\xfd\x94\xc3Ss=J\xb73\xa6\xae\xb9d;\x0c\x8c\x13!o\xd2\x97\x9f"b\xber\x1b\xf2\xa9\x0b\xe0\xaf\x90){\x06\x9c\xeaxW\x92\x18NF\xc4VdH\xb9\xc4\xfdU\xefN\xb7K^Y\xf9\xa96\xf5\ni\xfa\xa5a\xaa]\xc8\xd5*\xaa\xee\xea\xdd8\xe1B\xe5D\x83\x90\xf67\x96\xbbXK"\xef\xbd3F\xeb\xfb1s\xde<\xc7?E\xc4|\xed|\x14\x86\xa0\xbf8\xc7\x8fq\x11\xf8\xd5\x8e\x7f\xcc\x91\x17\x0c\x8d\x93\x97W\x8b\xd0c\xfd\xb6^\x8d8HP\xd0\xbcV\xeb\xc1\xc3Jz\xf0\xc2%\xf0\x81\x97\xd33\xa2\xac\xc5\xae&\xcf\xeb\x9f9\xcf\xd6\x15\xb9\x80\xba\x96\x13H>/}~\xb9+\xd4\x7f\'\x11\xf35\x8a\xf8:\xdc\xaf;\xbf\n\xfd\xe2r\xa2u\x8a\x16g!\x97\xc3=\x86\xf6\xdd\xd1\xd9\x1dq\xaf\x96\xabx\xe0\x04\x91~\xe8\xfc\x19\x96\x0c*B\x872\x9dCiz\x18\xd7\x9d\x972:\x91\xa6\xc2r\x1c9\\):\x83\r\x9d1\xcb\xf7\x0eL\xbf\x99\x07\x7fJ\x88\xf9\xaa\r\xb1\xb5V\xa6^_\xd7\xc6\xb8\xa7\xed\x8a9\xf1\x95\x92\xb7Jz_\x0f\x16\x8b\xda*\xa9"\xadvO\x07\xb8u\xcf\x89\x88\x18\x8b\x01\xdb\x87\x1c\x19U\x15\xc9\xe8\xdc>,\x8c1D\xc3=\xd7g\xb4\xab\xf0\x198\xc8:6\xa3\xf5\t\xe5;\xa3x\x93\xf4\xfe\x14\x11\xf3u\x8e\xa3\xc8z\x1b\xbe\x121V\x1c\xc9\x02\xbf\x17v\x95\\\xf0\xd3\x892\xfb\xcb\xac\x08\x1eK\x8e{&;\xab*#\xeb#^\xcd\xd0\xec\x11\xc8\x1ei\xe8\xaa4\x12\x0b\xd0\xedS\xd6\xcf\xa3\xba\xa3:\x13\xb5\xf4kL\xe6\xc1AU\xc4\x03S\x03\'\xeb\xcdt\xffSD\xcc\xd79N\x12(\x89\xbe\x1epB)\xdd\xda<\x17\xaf\xf0\x9a\xd1\xec\xb3\\\xf7)G\x0f\x9a\xf8\xe8\xcd>\x7f\xbf\xd8y\xb5+\xabc>C^L\x11\x07\xd6A34,,k\xd7C\xa8kF\xb7\x1a\x06\xe1E\xc8\x1d\xa78^b\\awA\x00Uo\xbek\xfc\x14\x11\xf3u\x8eS8\xbd\xe6H/\x98!}:L2\x9d!\xf9\xc0;i6\x9fJ\x15l/\xb7\xa3:\x03mb\xa5\xf0\xd9BY\x10X&\xd7#A\x00\t\xb0f\x86\xebR\x9b\xa4\x05:QU\xa9\x94{\xbfX\xc6\xc6\x96\x86\xc7)\xb9F\xf7\xb3\xb3s\x0b\xf1\xcdD\xf5SD\xcc\xff\xfbU\xca\xbap_\xb9V9\xd9\x11\x8fk#\x0b\xf7i\xf1d;K\x8e\xe7<\x18\x89\xa0\xc2cN-!\xff\x02\xc2\xbe\xa4\x1e\x97\xfd\xcd\xba,\xe4P`\xf8\xd0\x9d\xcfg\x1c:8;\xce:\x1d;\xf2|C\x0e\xc2\x8d\xbbM\xb1<\xab*1\xac{\xe0[\xa5p\x9f"b\xbe\xceq\nG!\xf8\xf5[\x0co\x8f\xb5p\xe1\x1e\x84K\xd6\x8d\xa8\\\xf1\x8a\xd9^&C\x07\xc1(\xb7\xef\x97\x82\xc4\xce)\xbf\x9c\x83\xf1\x9cb5\xb1c\xa2\xc3\xe3a\x8c\xed\x85<\x08a*+^\x11\x13}iK\x14\xdc\x8f\xe8!sM\xbbi\xca7!\xe8O\x111_5*\x84\xad\xb7\xc2\xab\x9dB\x99\x88U)\xa6\xa0\xb8>\xd8[\x02\xf9\xf0\x13\xdc\xba\xa4\xd1\x81x$p\x10\x9d8\xa9\xde9S{g\xf0J\x81@yi\xd6\x0c\xb5?Dq\x98\xa4\x85\x91\xdd\xd0=\xa2\xde/m\xcdC7\xe1\xf1`\x0c\xb7\xf4/\xf2\x9b\x07\xf9\xa7\x88\x98\xe7\xd6\xc7\xb0\xb5p\xa7\x89\x17\xf5\x03AwY\x01`\x81\xbe#\xa2\xbd\x80\xfbD\xa6\xf6\xb3gu\xa7\x98\x9c\x8fY\x06\x14\x0f@\x03\xa1\xeb,\x87:z\xba\xe8\x872T1G\xa0\xd1\xab\x7f\xdd\xc5k\xfa\x16\x90\xfdZ6\x1b\xc1\xc3\x80\xa4C$U\x01\xc2\xceo~\xbd\xf8)"\xe6\xeb\xebE\x12!\xa8\xbf\xb8\x96\x13a\xe7\xd4\x194\\\xc6)\x1a2\xd7\xdd+)SZC\xafGBu\x89(^\xc8\\\x8d\xf0\x96\x8e\xa9\x00\x81\xa8\xf5\x85R\xed\xa9?\n&\xa6\xef\xf7\x03\xcf$\xc4B\xc6\xbbe\x19\xac\x1d\xb3K\xc1\x90w\xbc\xfc\xfa\xcd\xbez\xfb\x10\x11\xf3\xc7Wo\xeb\x99\xbbn\x8a\x97[\xff`\x80d\xd2\x93\x97\x89_\xca\xa3\xad3\x8c\x08\x9c4\xa0Y\x18OWE\x16p#r\xef\xfbs\x1e\x9f \xb3\x0f\xeeGY\x88\x04\xa8\xa1;%\xa8d\x97\x95C,6z,\xc4\xb8XnQ\x95\x07\xd2\xa3\x19>8\xe6\xdf!b\xfe8\xb26"\xe6\xdf\xfe]\xf5\xff\x1e"\xe67j\xc5\xbdu7\xdf\x88\x98\x8d\x88\xd9\x88\x98\x8d\x88\xd9\x88\x98\x8d\x88\xf9\xc6\xcf\xb9\x111\x1b\x11\xf3\xff\xa3$j#b6"f#b6"f#b6"f#b\xbe\xf1snD\xccF\xc4lD\xccF\xc4|gze#b6"f#b6"\xe6\xdb>\xe7F\xc4lD\xccF\xc4lD\xccw\xa6W6"f#b6"f#b\xbe\xedsnD\xccF\xc4lD\xccF\xc4|gze#b6"f#b6"\xe6\xdb>\xe7F\xc4lD\xccF\xc4lD\xccw\xa6W6"f#b6"\xe6\x17\x121\x1a\xff\x0f[\xe1_b\x08\x97\x7f|\xa4\xefK\xc4\xfc\'\x02\xfb%D\xcc\x7f"\xb0_B\xc4\xfc\'\x02\xfb%D\xcc\xbb\x81\xfdo\xc0\x91_B\xc4\xfc\'f\xec\x97\x101\xff\x91\xc3c#b~\n\x11\x03\xff\xcf\x98\xffs"\x86ex\x84\\\x03e1\x9c\xa79\x92\x87D\x94\x170\x91\xe7a\x82&iA\xe4D\x86\x16!\x14\x838\x02\x83p\x86\xe5X\x92\x85\x08\x9a\xe6X\x9a\x81\x19\xe2\xd9\xb0\xf4\xc7\xfa\x01\x810\xcf\x0e\xce\xa8\xc0\xb1\x88 @\x08I\n,\x8a\x91\xeb\'\x8a\xd8\xfa\xc9\x14\xc6\xd18Is\xeb?\xe0\x99\xf5\xc7\x9e\xfdQ1\x88\xc7I\n\xa7h\x9cC\xb8\x9fI\xc4\xfc\x9f^\x16\x7f\xfb\x8b\x06\x0c\x08\xfc\xf7g\x07-\x82\xa6\xfe\xe8`\xf3#"\xe6\xfb\xfb:\x7fA\xc4\xc0"\x02\xc3,\xcf\x89\xcfge\x08\x82\x80\x10\x84\xe0h\n\x83\x05ZX\x9f\x1a\x12I\x18a0\x82^?\n\x85\x9e\xd3\xc7?Y\x0f\x1cY\x7f\x86@\xa1gG\x97\x8d\x88\xf9\x89D\x0c\x86\t\x02/\x8a,I\xf2\x08\xc4c\xb8\xc02\x04-P(GQ(.\xae\x03\xbaN/\x82\xaf\x0bk\x1dW\n\xe1Y\x88\x84\x05\x01\xa5\x9f\rb\x19\nC\xa9\xbf\xfd\xe6D\xcc\xbf\xdd\xf3\xfeSD\xccWc\x93\x7fJ\xc4|\xff}\xfeK\x89\x18\x98\xf8\'\x1d\xcc\x1dn\xa0\xfb\nU\x1copq\xf4L\xdd%\xb4\xf6\x1b8S;\xb2\xbe\xf2Cu\xf2Nw\x0f\x85Ek\xe7K>\xa3\x0c>\xaa\xf6\x08\x0f;\xa9\xdc\xfbQ\xb0\xc8~\x0f\xc8\x85q\xe1\xaen\xe0k\xd9\xd5\x95\x83\xe6\xcd\x1eN\x9f"b\x9e\xb7\x02\x86\xad\xb7:D\xbd\xb4\xe5\xd9\xc5\x05\xa1\xea\xe2\xcd\xa5\xa6J\x11\xcd\xb9\x98e\xfe\x02\xb4\xd5\x05\r\xab\xaa\x06\xd4\xfc\xecrG\xac\x95\xae$#N\xb7b\xe7\\\xc2\x9bC\xfb\x97\xc3e-\xff\x90\xc7\xe3\xb8\x03\x1b\xf5b\xa5u\xf1\xb4\x82:A\xdc\xbb\xc4\x8f\xfaS\xfed"f\r\x93D\xc8u\xd3\xbdL\xe5\x149-\xbf\xd7Y-+#H\xe7\xe1\x82B\x83Xa@L\xebD\x02G39\x91\x1e&\xc3\xa3\x86\xa6:\xa8\x9a\xed\x83\xa8\xea\xe1\x8b]\xa3>"\xa5$\xbe3B\xacU+X\x00Q s\x05\x00\xec2\xf6\xcd\xaec\x9f"b\x9eS\xb9\xfeq\x04\xc3\xf0\x97N|\x84}\xa8\xcf7\x10\xb9\xd3\x10\xe5\xef\xe3{\xa7\x1d\xa4\x16\xd0\x13H\x8d\x85\xae\xe3\xd7?UjuxQ\x02\xcd\x16\r\x8e\xf2\x86\xd2\xb4\xf6\xc1r\xb8\xdd\xd3\xf5\x04\xc4\xe9\xb8U\x04\xdd\xa2T\xb6r\xaf\xde\xb9\x9f;\xed\xf6CA\xe1\xf7&b\x9e\xa3\xf8\xcc\x08`\xf4\xb5\xa9\xd9`\x91*\xe7\x80;\x1c\x8a\x86\xb96\xd8\xe7\x8dz\x91"\xf7\x16Ygi\x11\xa4\xbdw\x81\xb3\xa4\xc2\xe2chB\x0b\xb6\xafAc\xbd$\x86<\xe9\xad\xbbAb%yi\x0b\xbb\x1c\x99\xc14E\xc4\xa7y\xe4r\xe2\xde\xecN\xf9)"\xe6+L\xea\x99\'a/\xfd\x0c\xfdK\xc3I\xb6\x174u{\xd1Nax\x84\xf2\xf4T;\xb6li\xcc\x9aGe\x0f\x94@C\xb2\x1d.\x08\xb8\x07\x88\xfb~-u\x04 tG\xf81\xa0\xcbpT\x84ab\x8f\xfbQx\xbci\xa7|\x8a\x88\xf9\ns\xbd\xd5p\ny\xd9\x9a\xb9\xeb\xcb\xb3M-\xed\x9c\xe5\xcdA\xbe\'\x07y9]mKUn\x9a\xca\x82h\xd0\xeeQI\xdf\xa1\xb9\x9a\x06\x1c\xa91w\xba\xf3\x16\xf0\xbaPI0\xb2\xf5l\xd4\x8d\x9b\x84r1\xfe\xa8\x01\xf8\xefM\xc4<\x17\x0bN\xad\x05.B\xbdr\x18\xca\x81\xd7\x8a\xb0c\x12r\x9e\x16oN\x1d\xd1\x9c\x1a\xd7\xea]\xe8<\'\xcd|\\\xcc\x98\xbbx\x8b\xcf]\xe5\xd8\x83\xdc\xf3C\x11\x08\xc3K\x8fw^\xc1\xd4\x9eW\xa7S"\xcb1\xd0\xc1^\x01]\xe4e(v\xdd\x9b=8?E\xc4|\x85Ib\x08\xb5\x0e\xd9\x8b\x9d\xd2\xe1\x80\x17<&\xe6$\xb3\x1a\x82g\xfd.\x1cyTA\x8f\x87\xf4v\x03q\x11\xf4\x84\xd8m\x8fi\xc1\x12\xbbG?\x87\x1d\xd6\x16nt]\xf2P\xd1\xabk]=\xb0=i\xf2\xee\xfe\xe8a\x8d\xc3\x17\x8a\x94t\xc1\x9b\xd9\xf8\xa7\x88\x98?\xbf\xa3\xf9\x87\xd9\xd4\xb9F\xec#9q\x88\xda\xf4n\x06McG\xad\xaf\x9c\x9b\x9b[G\x0c\xb9\xd8B\xcc\xa3d\xec?\xf8\xb17\x15t\x08\x0f\xf1q\x08\xf4\xa1cS6\x9b\xa9\xf3\x94\x02\xa0\t\x14\xc8\xa1O\xcc|\xb9\x9c\xc1\x83\xb9\xfb\x11i\xf6{\x131\xcfQ$\xd7LzM ^\xf6\xc4=Y\xea@\xc6A:\x82i\xc3<\xf5\xba\xdb,c+\xba\xd3\xc5U\x95@\x15\x08$\xbb\x9b\x19|\xf1 \xc6\xa4q\xf3\xe8/\x80\xb0\x0c\x97\x87D\xc4\xaca\x120\xf5\xd4\x90_\xf7\x04HgH\x18\xa0g\x9a@\xac;\x7fos\x9c\xa4l9\xb1`\xa5i\x94Csol\x03\xa7\x95\x9e\xf4\x9c\xc9\xcf.\xc9=A\xe6\xddbdD\xb5L\x07\x04\xc9n\x18\x8b\xb7\xeb\x0eA\xdc\xaa\x1d\x08%\x90G{\xc6\xde\xb4S>E\xc4e\xc4\x99\xc3p\xb7\xa5!\x9b\n\x19-\xf0\x83\xba+\xe4\x9cc}\x87L\x10\x10q\xa7\xac\x86\x90%+jMp\x8b6Vg\xf5\xaa\xe8\xa9\xfa\xa6\x9a\xf4)"\xe69\x97k!I\xa3\xf0\xebe\x18g\xe0\x94\x83\x90\x04\x83G#\xdd\xf91\xae\x03\x9a\xc7\x9d\xdc\xd4\xf7\x99ZH\xae\xc7\xd4\x04d\xee.\x8c\xf5\xe5\x10\xd4i\x0e\x01-\x82\x1d\xb4\xac\xa2\xa2\x93YJ\xba\xa5\xdc[\xc3*T\xf2|\x84\xc7\x8b\xa7\xd2\x84~z\x93\xf6\xfd\x14\x11\xf3\xc7k\x9b\xb54 \xa1\x970uo"\xfc&\x86&\x86PMmI3\t\x10T\xd4\x01\x92\t\x80\x9bd\xcf\xa5\xdds\xd4Q9\xb1\x16m\x9a>a>\xc3\xd2R\xd1\xc0l\x8bd\x8dX\xb8\x8bc{\x96G>\x10\x8ey\xa4\x8a4\xe1\xfb\x0c\xbc\x8c\xb3\x8d\xec\x00et\xd0\xefu\x8c\x7f\x8a\x88\xf9\xda\xfa\xeb}Ja\xaf\x19\xdc,v\x14\x16wg\xf8x\x1f\xad\x81\xa1z\xab\x05\xf2\x03\x0c\xb4\xb8\x01\x9dK\xeb\xfe\xe0\xf1\xde\x0cb\xfcf\x9a\xe8R\x08\x80\x07\x08v\xfbhO43\xf06\x89\xed\xef\xbe\xacu\xba\xe5"\xdc\xed6\xc0\xdar\xe5\x1a\xe3\xcdw\x99\x9f"b\xbe\x8a7|\xdd\xf7\xd0\xab\xea\x8d.Z8M\xb3st1\xc0v{\xe5j\xa94\x08\x1a7\x81\r\xc6\x89\xa5\xd5+\xe5i\xec\x9c\xe4\x8a\x9dV\x90\xcb\xeb}_\x92\x1a5\xdd\x90F\x15:\xdb\x1e\xac\xbeC\x81\xba\xd5\xa1\x87\\-\x0c\xe8\xb2;\x19z\x133\xfc\x14\x11\xf3\x0c\x13!\xd6\x1a\t\x7fMmLh\xac\xcf\x1eJMB|#\xb4\x8bF\x19\x11e\x822~\x88s\xa3\'\xeb\x05\x9ei\xb45\x80Lt\x92R.\x0cjNw\xc8\xbd\x89\xbbH?\x1e\xdc\xe14\xde\x1d\x8e\xbc\xb0\xc8\x00\xd9\xfc\xce\xe6\xb4\xf9<4\xe2\x9b\xf7\xd5\xa7\x88\x98\xaf\xb2\x03\xc20\xfa\xf5\xcb\x94=\x89Ji$\xdf\xe6\xba\xa4\x8a\xe7\x06\xc4l@\xcc\x06\xc4l@\xccw\x86W6 f\x03b6 f\x03b\xbe\xedsn@\xcc\x06\xc4l@\xcc\x06\xc4|gxe\x03b6 f\x03b6 \xe6\xdb>\xe7\x06\xc4l@\xcc\x06\xc4l@\xccw\x86W6 f\x03b\xde\xa1\x10\xfe!S\xfb\x17\x14\xc2\x9f\xe0\x84\xef\r\xc4\xfc\xfc\xc0~\x11\x10\xf3\xf3\x03\xfbE@\xcc\xcf\x0f\xec\x17\x011\xef\x04\xf6\xbf\xe3F~\x11\x10\xf3\xf3g\xec\x17\x011\xff\x81\xc3c\x03b~\n\x10\x83\xfc\xcf\x98\xffK \x86\xa7HH@9\x94`9\x08\xc31\x98!\xc9g\xa7\x17~}Dj}J\x91B\x11\x94\xc0!\x8c\x86I\x84\x10i\x8c\x81\x05\x04\x85!\x8ea\xd7\xff\xfe\xed\x9f\xd9\x07\xb0@1\x14\xcb\xc2\x04\xf3\xec\x04Jp\x18/\x928\xcc\xf3"\xcb\xaf\x7f/\x884\x83C4\xcc\xb2\xac\x08\xa30K\xf3\x04\xcb"\x08B28Bc<\xc5\xd1?\x13\x88\xf9?]\xb4\xfe\xf6W\xfd\x17\xa8\xbf\xc3$Eb$E\x93\xff\x0c\x88\xf9\xfe\xba\xce_\x001O\x93D qF y\x12\xc20\x8c\xe5hF\xe4PHd0\x82%\xe1\xf5qa\x08a\x05\x01\xa60\x12&0Zdaj]d$\xc904-\xf0\xcf\x1en\x1b\x10\xf3\x13\x81\x18\x1a\x11\xf1g/d\xfc\xb9\xd9P\x92\x81D\x82\x800\x1e\x850\x0eY\xd7\xe3\xb3[(\x0e\x13,\x8a\x90\x18\xc3C\xec\xba\x15\x19N\xc0H\x81Y\xff\x9e\xa2h\xe1o\xbf9\x10\xf3ow\x82\xff\x18\x10\xf3<\x18\xfe)\x10\xf3\xfd\xf7\xf9/\x05b\x10\xe4\xc7\xbd\x9a\xb9\xb6\xf6-\xc5\xb6\x0c\xdf\xc8\xa3)t\xf3\xb0\x81\x1b\xca\'\xf7p\x1b\xd2 v%\xc5\x1e\xe1\x064E\xeel\x85\xad\x03\x9a\xa6I\xbc\xbf\x94\\g\xb6Y\xe4\xf5Q\xfb\x10\x0c\xb1\xeeD\xc0\xb8\xf3\x06\xae`\x06\xca\x81o6o\xfb\x18\x10C\xfd\x1dAQr\xddx\xe8K?\xae\xd2m\x19\xebZE\xc4\xf9$\\2\x91\xeb\xbd\xb9\x0eP\xfd\x11 \x9eK\x81d\xd73\xea\xe8.H$\xbb=e\xe1Vw\x8d\xfd\x8bYt\xbeh\xaa\x0b_Z\x13\x87\xf6\xa0\x81\x02N\x9f\x86\x8b\x02d\x12R\x1f\xb2\x1f\xf56\xfa\xd9@\xcc\x1a&\x86\xc2\xafr\xca\xc4?n%\xcb\x94\x81\xeezpF\xedm\xd7A&\xfc.\xd6S_E5\xc6\xf7p\x0e\x086\xbb\x9b\xaf\x04z8[\r\x89\x84${g\x97e\xa7c\x04\x8d\xe8&m;\x95H\xec\x0f-\xab\x0c\x94o\x85\xb9\xc5\xbc\xd9n\xf0c@\xcc\x1a#\xbe&`\xcfnw\x7f\x0e3\xb3wArj\x12S\xe8jT\x00\xc4\x98\xe3\x19+\xd6\x02\xdf\xa6\xf7j\x8a\xa8Jp\x9b\x85x\x8a\xc1\x84\x94\x90+\xcd\xe4\x96\xac\x92\x8c\xb8h\x99\x93\xdf\xceBd{\xfa\xd1(\x0eK\xca\xf2\xfb\xe4\x8c6\xe9"\x17?rp~s \xe6+M\xa2(\x18\xa7^\xfa\x8c\xdeE\xa8j\x1a*\x90\xb90 5\xfc@\xf6\xe7\x16\xa0O\x18m5\xa9\xcd$7\x06\x01\xd2,\xe3`\x08\xec\xa4&<\xfb\x9a8\xb0\xa3/\x17\xa2x\x8e\x02\x89\xa9\xd0\xaa0\xca\xecr\x9a\x15\xb9F\x9a\x8a\xf5\xa1\xeb\xe1\xcd\xcem\x1f\x03b\xd60ix\xcd\x80 \xe8%\xcc\x0e\x83Lc:\xd8Ja\xf96\x06\xc4\xd7\xd1Q\xa3S\x86\xea\x89\xd6u\xf2~\x1e\xb9={\xe4[\x0e\x0fP\xde\x87\xce\x97\xc3\xd9(\x8f\x14\x14"Q\xd6\x87\x8bq\x88\xe1c\xc1\x07\x13|\x80\xc7\xc6\x89\xdd\xb6\xce\xcd\xf9W\x011\xcf\xe3\rZ\x0f8\xe8\xa5\r\xdf\xedV:8\x95^a\x05>>\xd6\x03\x9b\xf5\xf9\xf8^\xcb\x18D\xe0f\x7fM\x87tn\xf7#`\x8dvU\x04\xe8\xdeL\x84\xc0\xe1&t\x19Xa\xd0,D>\xe4\xee\xa3X\xb2\x82\x8bXH*\x9c\x9b\xcd\x03\x96\xf7f\xb7\xc1\x8f\xf90\xebd\xaes\xb9&\xb9\xf4K\x98\xb1\\[{k\\-N\x97Hd\x86b\x06/*\xba+\x0c\xa0\xf7\xf3\xb3o\xce\x90O\x9a\xea\t\xcf2\xac\x97\x92\xf0Q\'\x80?\x1en\xc6\x03!i\xc0x\xb3\xd3\xf1\xc7\x80\x98?\xd5\xa3\xffw\x985d\xd0\x05}L(>\xb3\x8e\\\x1cz\xb0\x86\tdn?\x1c\xa08\x94\x03\xbe;\xf9\xfc\xc55S\xd8\xf0\x13\xa2\x938\xe6\xd1\xd2N\xa3\x97\xf8\xacH\xe3\xc3;\n t\x8c@\x0c\x91\xeb\xdcg\x0f\x00\xcd\xf7\x9d\xf3\xbd\xfa\xa9~\x0c\x88Y\x17\x0b\xf4\xecVJ\x93/\xa3\xa8\x932\xc4\xf7\xa7\x93\xdeEHX\xdf\x94f0a\x9fh\xadZ\xdd\xe3\x87,R\xe5\x88L0%\xe5m\xca\xce\xc1\xf3\x15H\xa6\x87\xe2\xdb\x15k\xde\xc3cS\xda\xbbe\xd1\xc5\xe0\xaac.!\xa5\x17;r\xa8\x11(\xdf\xe5\x13>\x05\xc4\xac\x8b\x05[\x0bA\x98~\xe5\x1a\xdb\x08Z\xac\x07\xa8\\d\xa7\xef\xf6\x18R\xa4\xac-\r\x86\x03\x01\xfbf\x0fi\n~O\xfa\xfb\xe3"\xdb\xc5\x11\xd6[\x13\x89\xc2\x1d\t\x1fY\\\xb5\xf1\xf4\x8c\x90\'S\xd4c\xaf\xe0,\x89\xf6\x8fdp\xc0\xdd\\\x7f\xbc\x9b\xa6~\n\x88y\xa6\xa9\xf8z\xb3\xad\xc5\xe4\x9f\xc3DB\x03\xd0X\xf0\xe4\xf0\xdd\xc1e\xd7S\xaa\xaff\xbab\x8c[Y\x9c\xb4\x08\xe9\x0b\x0c98\xa5A^\xc6\xb8\xba\xdd\x08u\x07<\xac\xe0P3M"\x9c\xc7AMm\x7f\xf6x\xda\x0c\x9b{\xec\x01P\xe1rA\xdd}/\xe3\xefc@\xcc3\xb7\x81\xd6\x8b\x85x\xed\xd7\n\x1d|\xa6C.\x04\xaa=B\x0c\xd1\xb4\x88\xc9\xc7Ll\xa3\xfd~4\xeb\xec\xbc\x8b\x99\xae\xbc\x1e\xacc\x88ZQ$\x9f\xf2\x98\x06c\x82\x80\xe5\x80\xeey\xf6\x1a\x05\x98\x870\x17w.\x88\xccF;\xe9\x92\x8bC\x84\xbe\xa9\t}\x0c\x88Y\x17\xcb\x931\\\xaf\x9b\x97{b\xea\xec\xdb\x04\x08\xac\x8a\x9f\xa0\xaeK\xc5\x04\n\x1c\xbcb\xd0\x9b\xb9\x07\xeb\xc8\x90\xee\xbaW\xc3\xadh\x8f\x1c\xc7+\x8c\xca\xa5\xfa4\x1f\x185\xd0\x98\x9dZk\xa3\xa0{\xde\xa3g\xc8\x13U\x1fHm\x97\xc64;"o\xba\x97\x1f\x03b\xd6\xd9\xa4 \x84|ve~\xe9\xbe;V\x17\xebq\ny\nb\x9c\x13\xd0\x97P=\x13rnx\xa7\x13O\xb7\xec\xbe0.\x87\xcby7\xdcFKi\x9c\xcb\xce\xa08\x8d\xaaFK;\xc7\x98\xb4\x81\xe8\x07\xff\x90:\xd3"\x1e\xb1\x9e\xe0\xc1bM\x81I\x99\t1w\xf8\xce\xd3\xed\xe8\x80C\xea\x1d\x98\x1e\xf4\x98&\xf4\xb5@\xdf,\x82?\x06\xc4<\xc3|\xde\x9d\xc8K\xbe\xcf\xf0\xeb\xde\xf5\xbb\xb5\x12S1\x13\x87D\xcfcc\x99\xb8\x03\xcd\xf5V84\xb8\xc4;z_*\xb0]\xe2\xc7\xe6x\x9f\xb4\xd2\xd8_\x1d\xe2\xa6\xd9{{\x99\x1e\xc5`T<.\x8dW\xe9\xd4\xbb\x80\x00\x14*\x95\xe1\xdc\x9b\xef2?&\xc4x\xa6\xd1\xf7\x15\xe4a\xb7\x80\x9e\xad\x93\xae\x072=\x1a`\xf3\xf0\\S:A\\\xcb\x14%\x0e\xb1\x00\xfbf\xa9\xff1!\xe6\xeb\xf5\x14\xbc.\x83\x97\xc9T\x88io\x19\xda]\x85\xa3\xeb\x95\x17\x814\xc9\x8d \x19\n\xbdYP\xf0H\xb7\x01a\x8be\xa0\x99\x85v\xf6\xf5%\x96C\xc2Zs\xd8\x99\x93\xc4\x91\xa0\x16EJ\xf2!\xb9\xfb\x01\xdf\xf2\'\xf2^\'\xfd\xa4\x17\xe8\x9b\x19\xdc\xc7\x80\x98\xe71N@\xebU\xf0z)\xef\x12\x19F\x8f\xd2\xbd\x12I+e5`\x0e\xe23\x0b\xe3\x8e}AX4\xa3;\xe1|A\xd3\xeca\x0c}@\xfa\xf2\xd5\xb1F\x9ds\x02\xdd\x1f\xa9\xaa\x89\xf3\x12\t\x97\xf3\xc9\x9bB\xb1;y\x176\xca\x8c}\xd4\xee\xdfT\xdb>\x06\xc4<\x8fq\x08A\xb0\xb5 \xffs\x98\xbd&\x9c\xfa[A\xa8\xd7\x81\xcf@\xbc6\xea\xcbM-\xb0\xe9\x0c\xa8\xb4\xd3\t\x8d1\xb9\x16\xf70\xfa\xcc,:\x8fl\xe9\xfb\xfd\xeaZ\x12\xe4\xe6\x0cnv\xf73v\xdb-B\xef\xbbv\x81\xc9!\x131\xa8\n\xe9\xf7\x1f]\xca\xbf9\x10\xb3\x9eo8\xb6\xe664\xfd\xb2X\xb0K+\xef\x04\xc9N\xfb\xb0\x1d]\tw\xbb\x93\x02J\xc7\x9d\x80\xf7pJ.&\x1fu\xd3yp ,\xbc\xedG\xbc\xb6\xaa\x04\xd6.\xd3]Z\xc0\x83\xc6\x8aPV\x02\x1e0Y\r+\xd5\xdd\xd2\xda\xa5[V\xe8\xe3\xcd\xc5\xf21 \xe6\xabF\x85!\n~\xb9\xf2{\x9c\xdb\x17\x19I\xb1\x16\xe6\xd3L\xb3\xa7\xef\x19o\xea\x86\xc0q\x855\xe6\xa7b\x9c"5\x9e\x94\xb0\xec\xaay\xd2/\xe3\xe1\xc2\xd2\xd7#\xc4k\x01\x08\xc1^z\x0c\x8b\x8c\xde95\x85\xdc\xc8\x8a>^)i\x97\xd8\xa8\xaf\xd2l\xa1\xb6\xfa\x9b\x80\xc2\xa7\x88\x98\xe7\xd6\x87H\x9c\x84\x89W\xef\x87\x18OfRa\xc5E\xc1\xc0t<\\{m\xd8\x957\xb9DZ\x00s\x819\xe0\x08\xd6b\xa7\x80\x17\xb2y\xbeg\x996\x14\xbd\xee\x10su\xab\xfd\xb5\x94\x15tI\xcb\x8eJ\x18r\x86 \x19\xad\r\x91\xed\xbe\x89\xdfl\xab\xf8)"\xe69\x9b\x14\xb9\xa6\x99\x08\xf2\xd21\xba\xe6\xaa\x84\xb72\xb0\x91\xdb\x03\x0ct|\x86{\xc3\xa8\xea\x17hw\x98x\xbaOQq,H\xdce\x13\x9f\xe7\x96\xc1$\x98I\x9ey\xd5\xefOht\xf5\x1c\xdf\xb2\xe6{\xcdg\x96\xdc\xdbg\xebd+pEV\xc4\xbb\xe0\xcf\x87\x88\x98g\x98$E\xad\x19\xe5+\x11\xd3\xe3\xc3\x9e>\xee\x9c\xc9\xbd\xb7|:&\x13\xb7N\x89\x8e:\x11\x8c:\x0f\xd7\x87;\xc58\xb3UH,\x87\xf4p\xd9\xf5\x99\'7\x1c\x9b\xc7\xd4.\xe8\xb0\xbe\x07\xa3Ad\x0fj\xcc\xd9\x0f\x9dJ\xad\x81\x8f.\xfb\xc1\x7f3\xf5\xf8\x14\x11\xf3\xb57\xc9\xf5\xa4E\xf0\x970\x81[\x9b\x8e\xc2:\\;(\xb5H*\x80%\xc8Ss@\xbd\xbb\x0ei\xaaq\x87\x97\x940\xc4\xb0\xa4\xb6F\x1b\xa43z\x8c\xda!\x10\x97\xe9\xc0\x9dqU~\xe0q\x9e\x11\xe8a.\xa0\xaa\x19\xf7;\xf3\xea\xc0\x86\xfd\xa30\x7fo"\xe6\xb9\xf5)\x92X\xef\xb7\xd7\x13.\xb48\x91\xcf\xe9\x12\xbb\xdeZ\x06\x94\xf3\xf8F\x14It\x89\xa0@q\x87C\x05\xf2S\xd8\x9e\x81\x187\xa6\x0b\x16\x1c\xbc\xf5\xec\x86\xac\x0c\xd9\xc1\x97\xa1\xbe\xc80\x01\xb0\xc0\x91\xa3\xe9\x14\xbfb\x12\xa5\xc8\x96\x1et1\xf8f\xaf\xd1O\x111\x7f\x1c\xe4\xf4z\xc6\x91/-U)\x81\xcb\xcd]\x86\xf4\x01\xce\x1dk\n\xf0\x15,=\x8c\xe1\x00\xd4\xa0[\xc7\xc7|MqjGo\xc719\xa0\x1el\xf4\xcd\xbc\xc4\xa6\xa4Y\x16C\xbb\xd9m\x8fF\xcah\x95\x978\xab\xaa\x1b\x96\x19<\x1a\x82\x94\xcb\xbc\xb7\xf5?E\xc4<\xc3\xa41\x82\xfc+\x1dj\xb2\x91\xf8vh\xae\x81T\x02\x95\x06\xa6\x16\x0b\t\xfcH_\x0c\x06\xe3\xd5\x1a\xa2y\x8cK\x91\xdb\x0e(\xf6\xac\xa0\xed%o7\xecR\xa1\x03\'\x90m\x1eP}\x8f\x18\x1cK\xc7\xe9l\x92N\xd6;%KI\x864\xfdh6\x7fo"\xe69\x8a\x18\xf5\xdc[\xaf\xb5[\xd2\x12M\xd2\x95N\xb7\x1e$v\xa6\x81\x0ej\x07 x\x8e\x8c\xdb\x80\x1b\xfc)\x0c\x1ccg\x18\x8e\x9b2\x1d~7\xb4z|T\x8a|\x1c\xaa\x13Jp\x80-yE8\x82\x12\x8c^\xb4\x19 =\xa8\xbbx\xf3\x8e~\x97\x84\xfc\x10\x11\xf3\xdc\xfa\x18\x0c\xc3\x04\xf1*\x0b\x04#\xddJ\x8c\x14\x04\xc5~\xa8\x11\xfe\xee\xc1\xc0\x0c\x8b~\x04\xb9-2\x83E\xc5<\x0c\xfc\x92\x153h\xb9\xcb)\xedO\xfaD\x16\\\xc1\xb0\x0b\\c=\x132wiL+:>\xden\xcc~\xd4\xcb\xd0 0\xee\xcd\xba\xe6SD\xccW\x89\x8a\xae\x17?B\xbc\xf4\x8cF\xcdc\xdf1A\xde\xeeI\x1cL\xb5\xcb)\n\xf7\xb1\xdd\x86Y\xe6\x94\xaaz\x96\x17o\x98\xea\xc2\xd7O\xe0e\x8c\xc9\xebp\x8c\xefn\x10\x9eu@F\xbbx\xbc+\x03i\xa7M\xba_\xc4u\x03El\x1d6@\x01\xbd\xd9f\xf8SD\xcc3L\x18\xc2I\x14~E\xb0p\'?\xd1\xd7\x81\xdcg\xf7\x16:5\xe75\xc5\x7f\xc4\x97\x83\xbd\x16\x8d6dy\x94\x14*\xa7\xd0\x91\x00.\xf2\xec\xc2a,\xcaak\xce\x0ej!\x8f\\\xfe\xd1\xed\xba\x8a\x02\r\xe1\x9a/gK\xf4\xb3\xe3=\xca\x95\x1f\xaa\x1f\xbf7\x11\xf35\x8a\xcf\xea\x81D_j}\x84]*\xb3L:\xebt\xa0\xda6\x04\xcf\xb7\xa0\xaemH6G\x01\x9e\xcf\xa5]\xc9\x8a\xe8 \x80p\xb7\xf9\xc4E\x03D\x9d\xee;"\xb6\'\xde\xc3\xacd\xe6r~\xae\xea\xab<5\xa4\n\xcd\xf7\xe3\x9b\xf7\xc4\xa7\x88\x98g\x98(\xba^\x15\x08\xf1\x12f\n\xa4U\xe7\x00naP\xc9\xd9\xe4-\x8e\x8a\x06R\xc5\n\xed:\xe6\xa4\xdfz\xe2\xdc\xa6\xe8U[\xef\xb9\x16wwk\xd2\xecb;\xd1\xeec\xdc\xd6\x90s?\x944Xu\xb0F/\xf4\x12^\xc9;\x0f\xf3c\xf5\xb6\t\xf9!"\xe6y\xc2\xad\xc5\xf8\xb3#\xfbK\xc2\x7f\x0b\x97\xcb\xfa\xa1\xf3\xa9\xad\xaf3\x18*\x89\x9e\xa8\xf7Pt\x80\x90\x12\xe1\xbaF\xcda~\x08x\xce\xd6\x03#\x1ey\x07\xbfD\x8f\x9b\x8eu\x91\xc28\x99\x0c\xe9B|F\xf7\xdd\xac\x01\xed\x8d\xdb\xc7\xaa\x0ez\xd8\xdd~\xd3\xfa\xfa\x14\x11\xf3U\xbeAk\xe6\x8c\xbc\xdeW\'\xf6~\xf4-;s1\x90x$ns\xd8\xcf\x18V3\x1d%\x9d\xd8\\\xbf\x0cV\x03\xa8\';(\xd6\x04\x1c8\xef\x01{|Xes"\xfb\x0e(3\xf8\x96\xcd\xb4o\xb4\xa2\xb3\xe0jv\xe6\xf8\xbc=\xc1YL\xfdh\xd1\xfe\xdeD\xcc\xd7\x9e\xa0\x08l\x1d\xc4\xd7"\xb8\x87\xc1\xea\xc8\xf47J:\xee\x96\xcc;\x01\x93c\xae5\xc2\xd4B\xc7c\x1e\xdb3\x8f\xce$\xab\t\xa2F\x1b$^I\x81\x99\xc9\xe7\xcao\x87\xb9\xbf3m\x0fTs\xe0y7\x16\x17\x81\xf4\x10aL\xa3\xadY\xd8/"b\xbeN8\x18"\x11\x04y\xb9\'\xf4\tK\x11-\xea2+\xda\x93h\xe6*\x94\xe0\xb9V\xd2\xef\xe1L\x0b:\x96\xc6\x18^in\xe4\xb9\xed\x90\xc0\x9e\x9f\xefe\xb5E\xe4\xb0\xccS\xc9\x87\xd5\x9a9vd\xe39\x1e\x1e\x0f\xd9\xe4\xfbIG\xe9\xbe\xfc\xe1b\xf9\xc9F\xccW\xc2O\xc3\x18N\xbd\xaa\xb7\xb8\xa6\x1dI\xf8\x80\x1e{\x11X\tr\xcf\x8f\xc3\xc1\x9555n@\xf5Tt\xc5\xbe\x020\xa1\xbaS\x08\xacTYb\xea\xbc\x84\x8f\xd8\xc2\xf3\xf8\x8d\xbc\xec\xc7z9\xe8\xa8?\x87=Axp\xdc\xa6\xefn\xfdO\x191\xcf\xadO\x12\x08\x89\x92\xe4\xcblV\xb6\xb4\xdf\x1d\xcf\xf3\t\x15\x85s"C3\x10\xc4)\x06\x0c\xc8\x9ex\xd4\xb1cc\n\x9a\xdcI\x80Op\x8f^\xfc\xec\xc6XG\xa14\xa0x0D\x03\x94\xa9\xb8\xd7\x18s0\xfd\x00\xe0\xcd\xf8\xd1\x94\x90r.\xd47g\xf3SH\xcc\xd7A\x0e\xa3\xf8\x9a\xf1\xbd\x84\tPy\x12\xec\x11f\xbd_\x83\xbd\x0b\x16\x11w\xa3\xa1\xc2o\x0f\x8d5\xc2g\xd6\xbc\xd7:j\xc5;jhfC\xdde\xce\x9cS\x9c\x97{r\xdf\xeb\xe8\x9dc\x0e1\x19tU\x92\x81\xa0\x1b\x8b8S\xf3\n#\x03o\xa6\xaa\x9fBb\xfexs\x83>\x13\xbf\x97#\xc8?\xa6\xba/\xeatK-9\x1a\x9f\x08\xc1\x92\xe1\x873\xd8\xc9\xad\x16\x95y9\xde\xf5K\x15<\n\xcau\x9d\xc1;\xed\x85\xa1\x9d\xf7\xf2cY\x7f\xe0:\xcb)\x9b\xeeN\xb6\xdf\xecO\xb4\xa6\xc4\x94\'\x19]\xc7\xa7\xcc\x8f\xbeM\xf9\xbd\x91\x98?N8\x82\xa4\xff\xe2\xd6w\xec+\xe8s\xa4\xc5\xf3\rpV\xa1$\x8b]\xb6\xe3\xcb\x85\xacl\x99\xaee\xab\xd8\xab\xc8\x8e\x085$K\x06\xeb\xa6\xedL4\x10\x9d\xa3]\xdf8\xc5\x15\xf7\xc9p4\x07\xd62@\x8c\x0f\x13\x0f\x82\xef\xc49\xe8\xb37_\xda~\n\x89\xf9\n\x13\xc3\xd7J\xff\xf5\xbe"\x8f\x07\x14Fns\r\xb58\x0bu\x07\xfd \xf9;\x80gr\xbd\xcd\x01\x92\x08+D\xe1{\xb0\xb6u\xeaV^\x05\xb1\xech\x0b\x8eN\xbaeU\x07\xb0\xa54\x9d\xb9\xa9\x94\xd9\xf4\xc7\x80\x00E\x19p\x8f\x0c\x97\xa0o\xee\x89O)1\xcf\xad\x8f\xae\xc9-\x8d\x12/\xaf4vTc\xeb-K\x1c\x8aGRW\x02v%\xd6\xbd\xdda\xa4\xb8\'}K/-\x1a\x0el}A2\x84\xf7p\xd7\x8c\xc8\xee\x8e@Ry\xc0g\xca>\xddZ\xe1\xda1\x91m\xa3\xd2\xb5\x14]}\xddW\xc4\xae\xc1\xac7\x85\xcfO)1_\xc5\xf8\xba$\xd6J\xf5\x05\xc3\x89\xa3\x98\x80\xcb\x1d~\x82\xa8\x00O\xda\xf3\xd0\x1a3\x17\xd9\x1e\xa9\x98d\xe4\xe4\xd6\xb5\xbf:"\xe4\xc7\'\nH\x8e7\x99\xa4Q\x9b\xcen`\xbc\x87\x166\xda\xd1\x17k\xa9\xa3\xe2Z\x1d\x83\xe1\xe8\xd2M\xd1dK,~/\xe6\xefSJ\xcc:\x8a\xcf\xaf0!\xe4/\x00\x83\xe2\xdd/\xe3\x1b\x86\x05z:0\x19%2\x8cy8\x85\x086j\x0c\xc8\x9a&+.\xf6\t\xc6\xe0=\x91\x08\xd0\x8f|\x93\xdf[\x89y\x8e"\x84\xc3(\x82\xbc\xac\x95\x03t\x88\xed\xeb\xe8\x13\x8av\x85N\xb4C\x1fJ\xc2n\xe9\xb3~\xcd\xba\xe5\x80\xa0\x88&\xf2C\xd2\xe0T\xef\x9d\xceW\xafD\x114\xbf\x86\x96\x9b\xe2G\xe8\x9a\xb7\xf7Z\x11\x1e$\xd0^\xd2\xe0\x8a\xa1;\x15\x1a\xd2\xee\xfa\x8b\x90\x98\xaf(\xd7\xdb\x9aB^\xd7J\x97\x80\x18\x82\xc1\xb3\x00L\x9c\x88\xf5R\xee\x04\xd2\xcd?\xc7\xf7Gv\xd4E\xf8\xa2&\xb4+\t*\xc8\x0cm\xa5\x02\x87\xdc\x15m\xe2\xe6\x0b\xe7`_\x05\x16\x02M\x95:f\x07\xd3\x19\xafAtA\xcb\xe8\xa4\x18\x91\xf9f\xe7\xedO!1\xcf\xd3\r\xa5\xd7\x0b\x87\xc4^\xfa\r^\xb8\xc8\x1e\xebZ0s\x18\x12\x98\x11R\xa1.\xbf\xdc9\xfe2\xf4\xfbC\x1bV\xd9H\x15J\xc6W\xb9\xeac\xaeE\xb7\xe6\t\x97\xef;\xf0\x90\xe9#\xdaXQ15\x97\xdaBNZ\xc9\xee\xc3\xd2\xdfE\xc4\xac\xcc\xef\xb6\x8b\xff\x10\x12\xf3\x9cM\x0c\xc6\xa0u\xb8^\xfa\r\xe2)\x16\x87\x93#\xf5w\xc2\xdc/\x8f"\x93nR\x1dY#\x15\xb04\x02\xed{\x0f2\x95w)\x94|/\xe7\xefSF\xccs\xb1\xc0\x04\x0e\xe1\x14\xfdr\x806\xa7\x84=#\xaa\x1c\xd2 x2\xe8e\x87Y\x1cf\xf6qw\xaa\xc9\xc5\xb5n\x1e\xcdd\xf0\xc8\xe5\'h\x9f7\x92dfM\xbc\x93v\x91\x93\xc3\xe8\xed\xb4\x0e \x96\xb3\x92k\\\xc3cv\x034\xe4\x1e\x19\x13r{\xb3\xf1\xf6\xa7\x8c\x98\xaf\xad\x0f\xafu4\n\xbf\xec\x89S\'\nKw\xa7\x15\x1dvfSK\xfdyfN\xb7kn\xd1\x0c\xd7N{\x0c\x9a-\x0cb\xa2;\x16\x9c\xfc\x9b\xbb\xbf,\xb8\xcc\x12f\x87\xcb)\x90?\xa6\x91v\xbd\xf8\x0c\xd2\x0e1\x83B+\xa7y\x1c\xe4\xed\xe1\xcd\x02\xf5SF\xccW\x81\x8a\xd2k\xfeG\xbed\xaa\xbb"\xedH\xa9\xb1%\x119\xb1\xe6bv\xe4 \xa3\x99\xaf\xd3G\x1c\xdb\xd1qi\xbb\xad\xe3x\xbe\x10]i\xe1\xc2y\xa7\xb1\xb73\xcd\xbe\xc1\x97Z2\xc8[\xa9\x9e\xc4\x03\xa7\xe1\xc9\xe5\x98\x1a\x158\x97\x1e\xa5\x9f\xf07\xb7\xfe\xa7\x8c\x98\xaf0\xd7\xf2\x9d \x89\x97\xad\x9f\xc2\'uww\xd1\xeb\xa3\xf5Q=\xc9\xf6v{7\x11\x1cV\xd0\xae\xd2\x0e\'\x95\x83\xdcir/\x81i\xf3\xd2\x9a\xc5\xd8\xae\x15i\xf6\x19=\xca\x14~\x9c\xea\xf6\xb8@cI%X)\xa1\x12\x92\xa4\x11_\xa8\x93\xfc\xa3\xd9\xfc\xbd\x8d\x98\xaf\xad\x8f!k\x12\xf2\x9a;]\x82I#\x02T\xb3hI\x80+\x16g\x19\x81b)\xaa\x1b\xb4j7\xa5\xd3ti\xae\xc5#@\xc9\x03\xe1\r\xda@\xef\xd9\xab\x8d\xf6{\xb5\x04\xc5r\xbeU%|\xccL\x19qx\xbd\xf4\x8f\x11r\xf1\xb3K\xda\xe2\xefj\x89\x1f2b\xbe\x16\x0b\x04C8\r\xbf\x82B6k6\xa6:\x01\x83\xfc\xff\xb0wg\xddnb\xd7\x02\xa8\xffK^\x19#\xf4\xdd}\xa3\x17\x12\x88\xbe}C \x04\xa2G\x88F\xbf\xfe"W\xe5\x9e\x93R\xb9\x12e\xc8\xc7\xdb\xb9<\xd4\x88\xe3\xb2\xf7^s\xb5s-\xed\x9a\x1f\xa5\x871U\xf6\xf4\x04-\x8d\x89\xd8w(\xbd\x99\':\xc5D:\x96\x94(\x06\xd9\\J\xdc\n\x9fw\xc4\xee<\xdb}\'\x1b\x94\xdd\xee\xe5\x82\xd8\tz]\xd3\x8axr\x86,\xbe*\xee\x9bZ\xe2\xa7\x8c\x98\xe7\x0e\x87=\x89w\n~\xb9\xbd\x1d\x04O\x9dz\xcb\xadH\x00\x06\x8e\xa0gW%%y-\x90us\xc2\xed\xd9rj\xa7\xf4$\x0b\xca\x8d\xbb\xec\xef\xb5r\xbb]\xc1\xfb\xb1\xbb\xa0J\xcb\x9f\xd2#\x82_\x19\x03\x93]\xbbB\x15.\xf2\xb1" \xe0\xa3\xfbn&\xfc!#\xe6\xdb\xa9O"\x10J\xe1/;\x9c8\xdbk\x82<\xc6\xe7\n\t\x80\x9a\xab\xa0\xc4\xf1\xb3\x9c>\xbaj\x18\x927\xf3^\xad\xdb\x85\xd1\xc8\x13V"\n\xbb\x03\x02\xe5\xd2\x86i\x0f\xc6Kq\xdbC\x08kt\x8fp\x81\xc6k\x89\x04\xa0o\x9e\t!\x88M\xfek\xd5\x17\xff\x94\x11\xf3\xec\xc5\xf5\x9a\xbfN\x18\xf2%\x13\xc6\x06\xfa:>Hg\xf1\x19\x14=\x9dL\x05d\x19R\xeaA\x8c\x9a;\x91*\xb0\x83\x89\x0f\xf3T\x1e\xe9{\x0f\xb2\xa4\x932\x80tZ4F7\xa9Y\xd2\x933\x08\x9b1\xc16\x80o MU{\x80\xba\x0bL\xe7M\xee\xfaSF\xcc\xb7;0\x82\xe0\xcf\xa7\x91?\x86\t\xda!]\x18\xc4\xd2O\x1c[\xc7\x96\xc1g8|\x07uL\xc5\xac\x0b\x88F\x0c\x04\xaa\xd0\x0c\xb0\xb9B\xfa\xec\xa5\x1c#9\x82&\t\x06/^\xc6\xeb\xa4\x9efB\xe3\xef9\x02VU\xcd\x0b\xcf5~\x05\x0f \xff\xe6\xd2\xff\x94\x11\xf3\x0c\x13\xa1\x11\x9c\xc2^O\xfda\x98\x17Q9(!\xd2\x01\xde\xc3\x87\x9c\xd2\x84\\\xc0\x0c\xd1\x16\x83\xa8\xd3P\xb7qg,\x85\xb3\x8f\xb2\x85\x87\xe0\x03e\xdd\x8b\xe3\xe1`\x86\xc2y\t\\\x17\xccw\x90qN\x126T\x94\x16\xd5\xb0\xa3J\xde+\xe8\xcd\xd1\xfc\x94\x11\xf3\x0cs\xbd \xc1\xcf\x0f\xd9\xfe\x18\xe6\xae\xbf\xe9\x0c\xcc\xa37\xd1vs\xab\xef\n\x9a\xde{\xe34\xc2Uu\xc7\xf2\xb6P \xbb\xf1\x85`4\x98\xe5\x80K\x81_\xed\xda@r#Z\xaf\xbbz\x96y7\x1a\xb8\\\xc9\x92\x9c\x95}3R\xf5\x8e(ooS8\x1f2b~\xdb\xc8\x9f\xc3\x89\xbf\xdc\xf5I\x80\xbbJ\x9d6@~0"\xb9\x8b\x1f\x0e\xc0\x99f\xe4\x9a\x1eNZy\xd8I\x05\xca\xdd\x89~\x7f\xf6\x83\xf5\xdby\xc9\x83\x84\xd7q\xbc\xf7\xd2\xd5\x0e\x0f\x97Cu\x19\\nv\x1193\xb0\x10\xa9b\xb5\xcb\xe2\x0b\xc6\xfd\x1c#\xe6[\xf6\x81@\xeb%\x10\x7f}\x9f\xe2\x9d\x87\x06+F\xb4\xd7\xe7E\xba\x19\x1eM\xe9W\xc1\xc0\xcc{\xaare0B\x91\xd6(\x03?B\xd4\xed\xc1\xac\xe9j\x13\xa5&H2\n\xe2q\r;\xae-d\xb3\x87C\xdeD)\x01P\xac\xb4\xafgK*\xbe\xb76\x7fm#\xe6\xdb\xc3\x10\x04\xe3\xd0:"/\xf7\x9a\x0b\x85\x99\xf0!\x92\x03^\t\x8a%A\xae\xb4\xe7\x9e\x97\xe8\xf6H\x9aT\xf6\xf4Z>L\xea\xa9*\x11\xe8!"Xr\xb7\x8c\xf6\xb4C&\x0cM\x05\xd1\x99\x87t\xce\xd8\xf4\x04^\xa6\xd3\x1e\xdb\x9f\xe2x\x90\x8cfz\xd7\x83\xfd\x90\x11\xf3\xed\x95\xef\xf9rC!/\x93\xe5\xe4\xf4\xe7\x93\xee\xef\x8eF\xcb?\n\xbd\xba\xa2\x92<\x12\x0f\x80\x81\x81x\x97_\xa6\x8aF0\x9cVw\x0fe\xec(\xcd\xbe1\x19=\xb6m\x96"7\xde\xe2tIM:\xb5\xa0\xf6\x88\n[\xb7A\xba\xe9t\xb0\xa8\xdaO2b~\xbb\xa5\xae\x07\xc1\x9a\xf2\xfd1\xccuG\x90\x97\xf4\xce\x0e\x8do\x1f\x875Y\x12M8w\x8e\x88M\xa7\xc8\xf5H1\x03\x9f\xc0\xe1\x1d\xe5\x01\xec\xcc\xdc\x1c\xab\xc1!\x82\x1e\xf8\xa1v#-\xc3\xd4\xb2;\xd3\x84:\xd9\x19\xbb\xa3\xcf\xec\xf4#\xc3\xc9\x9ctbl\xca\xcb\xc0\xafu}\xfb\x94\x11\xf3\xed\x13L\x98$ \x14}\xb9\xbe\xd9\x14\\\xd21\x03\t\xe1\xa4\\\xdb\x9cx\x10\x13,\xa7~\x97\xd4\xeb\x95\xea\xf0\xc0\xa6\x98}H\xfa\x94\x93T\x91\\\x1e\x835\xc1\xfd$v\xc3p\xc6\xa1\x1e\x01[\x84?\xe3p\x1b\xcfk\xa2s\xef\x8dTm\x04c\xba\xfe\x96\t\xffK#\xe6\xdb\xf3\xe4f\xc4\xfc\xdb?\xac\xfe\xdfc\xc4\xfcB\xfa\xc6\x06\x9al]\xfa\xf5\xbbt3b6#f3b6#f3b6#f3b\xbep;7#f3b6#f3b\xbe\xb2\xbd\xb2\x191\x9b\x11\xb3\x191\x9b\x11\xf3e\xdb\xb9\x191\x9b\x11\xb3\x191\x9b\x11\xf3\x95\xed\x95\xcd\x88\xd9\x8c\x98\xcd\x88\xd9\x8c\x98/\xdb\xce\xcd\x88\xd9\x8c\x98\xcd\x88\xd9\x8c\x98\xafl\xaflF\xccf\xc4lF\xccf\xc4|\xd9vnF\xccf\xc4lF\xccf\xc4|e{e3b6#f3b6#\xe6\xcb\xb6s3b6#f3b6#\xe6+\xdb+\x9b\x11\xb3\x191\xefh\x08\xff\xb4a\xffK\r\xe1\x0f{\xd1W6b~|`?\xc9\x88\xf9\xf1\x81\xfd$#\xe6\xc7\x07\xf6\x93\x8c\x98\xf7\x02\xfbO\xc4\x91\x9fd\xc4\xfc\xf8\x11\xfbIF\xcc\xff\xc1\xe6\xb1\x191?\xc4\x88\xc1\xff\xa7\xcf\xff\x85\x11#R\xa4\x88>\x9d\x18\x0ea\x19\x8e\xc1!\xfaY\xdbB\x109\x81\xc4`T\xe4E\x86\xa50\x9eGD\x96\x82\t\x8e\xa0Y\x18\xe3p\x98\xe5p\x8aF\xc9o\xc5\xcb\xfe\x82?\xc0E\x1a\xa1I\x06\xe5h\x82\xc5y\x86\xa68\x91\xa6XN\xe4X\x92\xc3E\x16\xa19\x94\x14\x05\xea)\\<\xf1\x0bT\xa01\x88G\x05\x92\xc79\x9a\xe3\xb1\x1fi\xc4\xfc\xa3\x1e\xd9\xdf\xfe\xac\x00\x03\xf9w\x12\x81q\x1a\' \xe4\xaf\x8c\x98\xaf\x0f\xec\xfc\x89\x11\x83\xc1\x02-\x92\x0c\x83\x900\x0f\xa3<\x83<\xa9\x18\x12e8h\x9d\t0\t3\xcfzu\x0c\xc6\xe0\x98\xc0\x10\x0cA\xac\xbf\x140\x84\xa7\xd60\xd7f\xa0\xcfbN\x9b\x11\xf3\x03\x8d\x18\x86e\x91u\xf5\xb3\xac\xc8P\xe4\xb3\xbe\xd9:{\x18\n\xe1\x90u5a\x18F\x11<\xc4S<.\x100\xf2,0\x840"\xb6.\xcd\xb5i\x14\x0c\xf18\xff\xcb\x1b1\xff\xb6\\\xf21#\xe6Y\xd8\xe4/\x8d\x98\xaf\xbf\xce\x7f\xaa\x11\x83\xc1\xdf/H\xcf\xe5\xe8%\x9b\xb1G\x14\x03\x1a\x02\x8et\xdb\xf9\x07B\x9d\x9a\xd1\xac\xf8c+\x11\xd5\xc0T@\xe2vG\x04\x8c\x81\xae\x08\x93\xbd\x1f\x88\t\xb8\x04\xb4qNh3\xd5`\xc8\x14G\xf0\xd8\xd8\xf2={\xcc\x14\xef,.\xf3^\x95\xba\x8f\x191\xeb\xa9@b$\t\xad\x8b\xf0\x8fa\x12RR\xf7\x12r\x06\xe3\x1aD\x91av\x88`\x82\xa3\x92\x9e\xf7\xde\xa2\xeb\xd4D\xcas\x90\x93\xa0n]\xd8\xc9\xc9\xc2B\xd6\x0f\xd8\x9a\x8c&gl\x9f\x10\xb5\x04\x9e\x8f\xca\xe8\x8cx}\xecU\x8c\x02\xe1:r4\xfe\xcd\xf2m\x1f3b\xc8\xbf\xaf\x1b\x19N\xbdT\xe2\x13\xc8<`:\x1f6\x17\xc6\x10\xa6\xceI\xdck\xad\xe8;\xb2\xc2.\x0ePF\xa0 \x91\xb8\xe1$\x9d\xee\x9c\x18\xe8\xc2E;\xb9\xc9\xf0\xa2\xc2\xf1\x98T\xc1\xbb\xb2CD\xcc(\xee\x16\xd8\xdf\x841/%\x01:\xa1\xed\x9b\xee\xc7\xc7\x8c\x98\xe7P\x92\xcf\x02\xcc\xf8KIU\x08*\x1f\xf7"\xd6z(\xb2\xefh\x9f\x98i3\x14\xf8Xv\x01v\xc6\x02H\xabr|\x10\xa84\xa3}M\xed\x8e\xbbC\xed\xcc\x18\xbd\xf7\xc8\xd3\x81O[\x08\xefv\xb1\xe8/x\x8834k\xc4\x95\x1ez\xcb\xe5\x8bU\x18\xff\x98\x11\xf3\xecEd]F/}\xa8\xcc9\x9a\xb6\xb5\x8et\x93\xedF\xdaY)\xb8\xf1\xde<\xb8\x1a\x9a\xaa\xf35F\x1fS\x1f\xaa\xeeM\xc1\x87\xa3\xb03zeb\xd2Yf\x86\xf9r8\xd2\x9d4WI\x10I\xb2e\xae\x97\x12\x8f\x87\x87\xfdC;\x17\xfbwk\xee\x7f\x8a\x88\xf9C.\xf8\xbf\xc3D#\x85\x834\x0f\xbc\x1f\xac\xf6\n?\x8a\x11\xc8\x14t\xb9\xdfg\x0f\x1b\x005\x11\x183\\l\x0b\x06L\x03\xc9\'`\x17 \xba Q\xbb\x1bP"\x9c\\\xd1N\x13\xa5\xce1:\xbb\xe3EC.3\x08\xee=kd\xde\xae\xb9\xff!"f\r\x93\x80\xd6A\xc7\xa9\x97\xcd\xcd\x19\xa3\x07I\x93i\x98\x1ciD\xe5UKK\xc3z9\x14\xc7k\x15\x90\xfb{\x7fL\xe8H:\xed\x1f\x1d8\xd9\x1d\xeb\xed%Uuz\xef\x84\xe6\xc2\xcd\x8c\x8b\xc8\xda\x07\xbb\x80T\x9a\xa5\x94+\xc6 \xa8T\t\xa7\xe9]T\xe4SD\xcc\x1a&\x8a>\xef^\xf0\xcbh\xb2\x8e\x0bK\x9a\x16Om\tk\x07_\xad\xcap\x8a\xc9v\x80\xd1\xe58\xc0<1\x87%MfMT\x94\x13p\x15&M\xd9\xb7\x1cW\x17F9\x92\xcdt\xa1\x1b\r\x1b\xe8\xb8*q\x83\xb8\xcb{\xb4p\x10\x12\xa8\xdf4\xb0>F\xc4\x90\x7f\'hx\xfd\xf3\xeb\xb0\xbdLZ<\xe0O5\xeb\x84\xf6\x94+7I\x95\xa4vt\xee\xc9\xf1@h!~C\xbaL\xbf\xdb9\xc2*\x9ar\xec\xfa\xda*\xe6\xde\xb6=\x99\x19%\x95\xd2\x07\xa4\xa1\xed\xee\xd6\xdfx\xfd\xc4\xd9\xf4\xe3\x94g\x97="\xa0o\x9e\xc8\x1f#b\xd6\xd1\xc4\t\x9axV*|)v|\xa34U(\xc1\xa8\xc3\x9cD\x1a\xee\xe7\xc1j\xc9K\xb9(\xbe\t\x8b\xea\x0c\x92\\%\x99m\xdb)\xba\n\xdc\xe0\xf39go\xb5Ks\x18\x97\xd9c\x1d\xb0V\xed\xdc\x8fmh\x0b\xc0\xa1q\xf7\xf4!]R\x84z\xb3\xf6\xff\xc7\x8c\x98o\x89\x07\x81\xe1\xe8k\xb1c[Q3<\x8f\xc0\xfd\xbe\xa8\xce\xc1]n\xd1\x1a\xb8C{\xa9\xaa/\xfb\xe9T\x9a=\xa3\xcd\x1a\x03\x06\x97\x9c\x94\x03\xcb\x12\xce\xe1\x92\xb61\xcd.\x19\xe8r\xd1\xa8\x84\xb9z\xdc/{\xeb\xc4b\x99\xa6e\x83\xb6\x7f\x94_\xab\xaa\xe2\xc7\x8c\x98\xb5\x17i\n&\xd7\x13\xeee\xb2\xe0\xfc \xdf\x9b\xa9\xe6\xb8&\xa1]\xc1\xbf\x8c\xc7\x8e\x04\x80\x92p\xd0\xda\xbd\xd5\xf8\x88]\xf2lQI5$eO\x8d0;\x19x\x87>\xef\xe4\xfbb\x1d\x90\xb08+f\xe9\x87GU\xe3\x16\xbbpn\x9e\\\x9d\xf57\x0b\xaa~\xcc\x88Y\x97>\xbe\xde-\xd7\x04\xee\xa5(~0@\xda\r\xbe\x90\xde\xac\xf1\xba@\xac\x13\x05\x12*\x18A\xeb\xa8\xf0#\xd00\xcb\x0c7-E\x96S\x89\r\x04\xcf\x89\xfa\x19\x1ca\xae>c\xa2>\x13\xf5\xec\xec\xca\xa0?P\xdc\xb0;\xb5\x84\x82\xca\xbb+\xd2\x85\x06\x0e\xce1\xb9o\xe5t\xf9\xaen\xf2\x8b\x1b1\xcf\xa4\x02\x821\x02\xfe\x13\x1e\xaaN\xe8\xae\xbeq\xae\xe7g\xca\xed9W\xfa\x9d\xe1\x9c`\x1ct\xa3c\x1c\x8a\xcc\x8eag\x199\xc7\x9c]@}\x06Y r\xf4l\x8b`\x0e\xa0\xd1t\xc0\x15\xc2\x88\xde\xc6\xc6\xdbUy\x00\xaa?L\xb6t3\xdf|\xcd\xf8\x98\x11\xf3\\\x13\xd0\x13\x84}UE\xb8[\xd2\xa6\xfc\x92\x02\xd4( \xd3^G\xc4\x93\xcf\xee\xc5X]\xf3\xc2h>Vu\xadE\xf0}\\\xf0iF\xb5\xcb\xd9X2\xa1X$\x85\xd1qz\x02\xc9(\xd6}\xfax,|I\x11\x160Y\xf0\xf5r\x89\xd9\xefR8\x9f2b\x9e\x1b9\xbe\xae,\x1c~Y\x13\x8d1\xb8\x12~6\x01\xff\x04\x94\x80\x9e\x1f\xfa\x92i\x8e\x17A\xd87^\xf7`\xb3S\x93O\x92\xcd\xd3\xbc_)\xfa\xd1\xed\xd2\xc9\xef5\x97^ \xa6\x97F\xd2\xdf\xa5\xf7\xc7\x11d\x15\xf4\xaa\x17\x8dq\xd2\x9c\xc3t\xe3\xdf\xbc\xa5~\xcc\x88Y\xc3\x84!\x1c\x86\x08\xe4\xb5\x8cz\xcd\xecv]\\\xb2\x8a\xfd\xb0\x03\xac\x07Rj8\xc1J\xd2\xf5\xd4\x059\xad]\xc3\'\xea\xb9\x89\t\x8f\xcb\x9cr\x9c\xf4\x88\xbb\xb5\xedp\xbf\xe8Py\x03\x00\x0c\x83d\x9c\x8c\xb2\xe3%\xce\xda$%\x81\xd6\xc6\xa2\xe1k-\xfd\x8f\x191\xdf\xee\xfa0\xf4|\x02\xfac/\xce\xe72\xb5U\xb8\x89\xc7\xfa\x0e\x97\xcd\xa25\t\x1e\x1bG.>w\xa5\x9e\x13\x17a\xaeoa>\xc7]\xb0\xe6\xbe\x11\xaa\x1e\xbd\xf38C\xc3\xd2\xa7\x19\xeb`\x00\xc3\x93x\xd5\xf9\x01\xc6V\'%\xec\x13;\xca\xc1\xdb\x9b\xe7\xc4\xc7\x8c\x98\xe79\x01\xa34\x8c\xbc\xa2\x90-\xef\x9bd\xe36\xd5\x18\x9as\\\x8d\x0c\x8d\\\x06\xd9e\x0e-\x89\x9d\xd4\xbbj\x0b\xf8\xb4\xd0.017y\x04G\xae9\\%\x94W\t\xaeBA\xc5\xd5\xf7\xda\x00\x8aG\xedB\xa6\x92:\xb1u\xa5\x9dMEx3\x13\xfe\x98\x11\xf3\xcc\x84\xa9\xe7y\xf8z\t^\x8aPz8\xd2u\x08mM\xae\x9a\xf8\xc8\xddn\xfb\x1dH1\x03\xb3\xf0\x8d\\)c\x8f\xecl\x1d6M[Dw\xc7\xe6\xa0\xca\x0f\xaf\xe3\x97\xbc\x16\x06Y#\x1f\xe8l@UI\xb4\x0f\xb5b9\xe3PC-\xf3\xee\x0e\xf71#\xe6y^Q\xd0z\x11~Mn\x1e\x12{P\r\xe0\x8a&\xf4r\xa1l\x88\xad\xae5di\xe7\xa3l\x9c\x97\xe3\x9eS\x9c\x93>\xc3\xb11\x13\xaf|\x1f"\xca\xb9\xf5\xa1\xec\xcaPE\xc6\xc2\xc9\x12\x08\x92E\xcbe\xa1\x94\x1a\x19\xa1\x82bd\x0e\xf6\x98M\x9d\xe7\xf7\xe8\x85\x9bI\xc6\x0br\xf5\x96\xf5\x8e\xa4?\xac\xd9v\\l\xa4\xf7Q\x8d\xfb.o\xbc\xf9\xcc\xf71#\x86\xfc;\x85\x92\x18\x86\xbddp\xa5\r\x80\x16\xa6qU\x17\x8a@\xe2\xf6A\x1f>|\xbf\xb8\xd3r&GWL^SP\x91\x88\xa2\xf1\xb2+PF\xa4K\x17\x9bj\x12Z\xd2c3\xdc2=\x1a\\\xa0\xa7\xf5<\x95\x06O\xbfD\xf7\xfa\x88\xeb\x9a\xfe\xa6h\xf61!\xe6\xdb\xb5\x03\x83\xa1\xf5\xba\xf7\xf2\xe2\x80\xc8\x13`\xcepv#\x96P-\x12\xe6M>\x94v,\xac9\\\xd0\xcf\x12uP\x86XD\r\xa3\xeco\xf2\xbdOi\x85I\xef\xb2kg\x02\xfb\xef\x011\xdf\xd2\xf7\r\x88\xf9\xb7\x7fR\xfd\xbf\x08\x88\xf9u*\xf0o\xa8\xc1\x06\xc4l@\xcc\x06\xc4l@\xcc\xb6\x97~\xc1\xbdt\x03b6 f\x03b6 f\x03b6 f\x03b\xbep;7 f\x03b6 f\x03b\xbe2\xbc\xb2\x011\x1b\x10\xb3\x011\x1b\x10\xf3e\xdb\xb9\x011\x1b\x10\xb3\x011\x1b\x10\xf3\x95\xe1\x95\r\x88\xd9\x80\x98\r\x88\xd9\x80\x98/\xdb\xce\r\x88\xd9\x80\x98\r\x88\xd9\x80\x98\xaf\x0c\xafl@\xcc\x06\xc4l@\xcc\x06\xc4|\xd9vn@\xcc\x06\xc4l@\xcc\x06\xc4|exe\x03b6 f\x03b6 \xe6\xcb\xb6s\x03b6 \xe6\xbf\x14\x88\xf9\xa7o\xf2/)\x84?l\xef_\x19\x88\xf9\xf1\x81\xfd$ \xe6\xc7\x07\xf6\x93\x80\x98\x1f\x1f\xd8O\x02b\xde\x0b\xec?\xe1F~\x12\x10\xf3\xe3G\xec\'\x011\xff\x07\x9b\xc7\x06\xc4\xfc\x10 \x86\xf8\x9f>\xffk \x86E\tN$XR\xa49\x8e\xe1\xd7_\xe0\x84\x80`\x04\xc4\xb2(\xcd1\x0c\x04\x93\x08+\xc0\x0c)\x08\x1cB\xa1\x18\x0c1(MR0\xc2\x08\x18\xc5!\xf0\xb3^\xe9\xf7\xed\x03A\x14\x10\x91#p\x06\x17h\x06\xe50\x81G\x11\x9c`\x19\x14!!\x8ee\x19\x1e\xe5y\x8e!!\x92\xa6\x9e5\x918\x1a\xe3\x04\nC1\x8a!y\n\x13\xc9\x1f\t\xc4\xfc\xa3\xa0\xec\xdf\xfe\xa4\xfe\x02\x8e\xfd\x1d&\t\x1c#\x08\x14\xfeK \xe6\xcb\xeb:\x7f\x02\xc40\xcf\xdaL\xa4 \xae\x8dE\x89\xb5\xf1\x14\xbf~\xd9\xb5\xe1(Fpp\xa5pkw3\x8b\xd6\n\xb4?\xf3\xfc\x95o\x8f\xdcu\r8\xd7Ol;L`z\x1a\xcd\x13\x04\xd1\xae?#\xbb~\x07%\xfc)?qt\xe9\xa1J\xf5=\x07\xe7\xd7\x06b\xd6^D0\xf8Y\xcc\x12\x7f\xa9\xbb-\xec\x92\x98\x06\xec\xc3qw#S\x17\xd3\xd2\x19\r\xcfg\xe0\xc0\xa27\x83\x7ft\xc6Df\x1epC\xf6\xa4K\xe1Y\x8f\x06\xd0t$\xfd\x88\xb6=\xd5k\x1fsq\xd8]\xf6\xe1y\xbd\x7f\x1b\x14\xed\x1c\xd3f2\x92\xf4\xf1\xe6\x82\xf8\x14\x11\xf3\x0c\x13\xc5I\x98\xa2_k\x9a\x1d\xdc\x12\x03\xc2\xe3y\x9eP\x17\xab\x0cW\xad\xca\xdaj\x9c\xea@\xca\x9dz\xd3s\xa3r\xcfu\xe9,\x17k\xe8\xe3T\x00KX\xb4N\xe3\x91\x1bM\xca\xe7\x9c\xd8\xdc9\x97\x93\xe5\x8a\x14\x12\xa8S!KZ\xea\x0e*\xfb\xe6\xba\xff\x10\x11\xf3\x0c\x13A\xd7\x0b\xcfz\xf3\xf8c\x98\xa2\x1e)m\xe8Z\xb9\x95O,I\x9c\x8c\xc7N<&\xc7\x00X\'\xd0\xedl\xcc\x85\xd5\xe4\x07\x10\xd0\xa2\x90\xb0\x1b\xa0\xdb\xcf\xe1\x12\xfa-}\xe2G\xd5\xa8\xd6\x9d\x0e\xe4\xdb(\x8d\xe3\xca\xcb\xb4`\x90\xa1\x0cg\x87\xf0\xcd\nu\x9f"b\x9ea\xae\x93\x9c^\x13\xbaW\x1b\xea\xc6Z\xacm\x16\xa0\xa7\x1fD\x88 [\xb0\xdb\x83\xa05\xb0\xbcl\x84\xd2\x053\xf4\x99\xd0\x03\x8c\x83\x9cF\x8e4k*t\x86+\xf2;\x16\xeb7\xe1\xe6\xee\x1d\xfc\x8aW\x03\x0f\x9a\x846;v\xfe\xdc\x8a\xd6\xbb\xcf\x9b\x15\xa3?D\xc4<\xc3\x84!\x12]\xf7\xfc\x97\xb2\xb1\x942\x05\xb4\xfaXX\xfd\x0e$a4\t\xa5\xc39\xaa\\\xe7\xf9\x85E.y\xc5\xc4\xb7\xfc\xd8\xb2K\x0e\x80^\xaesF\x90\xc7\x11\x93\xb6\t\x87/w\xdc\xb8\x91W1X\xa2J\xf6\xd8\xb3\x84\xb5\xc4\xb5\xdf\x1d\x98\xe9\xcdz\xaa\x9f"b\xbe\x85\x89A8M\xbdTUt\xdb8a\xc7\xe9\xac\xd7\xe6\x8d\xcf\xf5ka\xc4,\xc8\xc1V\xde\x072\xda\xb7\xd4\xa4\xba\xe0\xf1\x1aw\x1ds\xb2\xfbZ\x03u\x16\xdd!\xae\xe3:\xe7}\x85\xdd\xfa\xc4\r\xf9i\xae\xdap\xc9X\'\x80/\xce\xfe\x829?I\x88yf\x1e\xeb&\x0b\xa1\x18\xf6R\x08\xf4\xd0\xdf\xf89\x1c\xb5\xfd\x9c\xbb\x914\x96\xa3\xb1\x94\xec8\x06R[gE\x9e\xab\x17bv`\xb2\x04f\x8f\xa5\xf8&\xf3\x91\x94SnYg\xef\xf7\x97k\x8b\x81\xf8\x15\x1b\xeeW\x05\xe4/\xf6Q\xa5\xd8\xb3\xc7;\xde\xe1\xbfS\x88y\x1e\xfa\x18\x0cS\x10\xfe\x9a\xdb\xb8\xa0\x03a\x0f\xe7\x8e5\xc1\xb9\x8f52\x80\xa0i\x98\xec\tP\xd0:\x94Z\xc4\x8a\x10\xd3"\xcaB\xeft\t\x97Y\xc2\xee\xdch\xe7\x0f\x12\xd2\xd3\x93\xeb\xc5qR\x1c\xdb\xbd\xc5\'\x07\xb2!\r\x91\xc5\xf3\xc7\x1ez\x13\x87\xfa\x94\x10\xf3m\x1f_\x17\x04\x81\xbc\xd2)kz\ty\xadr\xf3)\xf2\xe1\xd7\xbd\x7f//\xc6Uz\xd4\xfc\xba\xd3\x84\xe6p\xbe\xa52\xb0C\xae\xf6IE\x1d\xbe=<(\xeeF!r\x80\xc6\x81\xb2\x8f/\xa9Bip\xe7\xa5\x8bq\xe7\x89]\xd7\\,R[\x0e\xefV\x8d\xfd\x90\x10\xf3\xedT\x86`\x8c"\x89\x97S\x19\xc9\xfc\xaa\x1b\xed\xb8;w\x0e:g\xfa\xb2KT\xc5\xec=\xdd@e\x12\x98\xfbab\xcf2\x0f\xec\xce\xb4\xda\x96-\xe0\x96D~\xe3l\x003\xc7 8\xf9\xe1\xa2"\xd88\x9e\x08\x16#\x8901\x90\xfd\xe0f\xcb\xd7\xaa4\xfa)!\xe6\xd9\x8b8\x84\x13\xeb\xc5\xe6\x05\xf9\x8bL\xac[\x16S\xd9\xa9x_\xe7\x93u\xe4\xdcG\xbc\\\xb9\x1e\n\x85=|\xd2\xdd=\xdb,8\xad\xdf\xf49/-\xceDKD\x19c\xdd\x8e \xa2F\xa2\xc9\x16\t\xde\x9a\x10\xe0\x86\xb2ITE\x8bKW\xe2\x9bK\xe2S@\xcc3\xca\xe7{\x04L!/s\xc5.G\x81\xe0\xe3\x13]\x90\xfe-\xf4\x90\xb4\xb0\xb19\x96i\xaa\xa5\xa5}\xad^8\xca\x92\xfc\xe5v\x8dw\x89\xb7\x9f\x1f\xbb\xdd^@<\xf8\xb6 A\xe2\xf1P\xd4;n\xdaA*\xd6\x06s);\xa7rN\x9db\xff\xe6\x92\xf8\x14\x10\xf3\x0cs\x9dh(\x89A/\xf9x\xc2\x067cV\xd8\x1a\x0c\x1f\x81t\xb9\xf8\x83^\x1dU\xe8,]\xeeMQ\x17|-6\xc8\xbc4\x97\xc0\xcc\xb7\xd4\x86~V\x9d\x7f\xcd\x83\x1f&\x1d\\\x118\xd5\xe6H\xa9\x03(\x1f\x19\'\x14\xcb\'\xfeaM\x13\x9f\xec\x81=\xd6\'\x84\xdar\x8c\x91\\\x80\x8b\xb3<\xf6\x87\xf3\xa8a\x07u\x17`\x19s\xcad\xa1\xe8M\x881p\x82RrB\x07#P\xf9\xde\x06\xf7k\xfb0\xcf%\xb1N6\x88D^i\xa8\xd8\x99\xe85m\x84.\x8b\x83\xda;He\xad\xdaU\xe1\x9b\xa6\xc6\xe5\xd4\r\x8f\xd8\x1d\xc1\xc3\xa5V\xf9\xf0Z5CC\x97Y&\xea\xe4=*\xf4+\xe2\x82\xc6\xf5\x9c\xa7p\x16\x1e\x97F\x12FL30\xdf\x16\xce\xe0\x9b+\xffS>\xcco7`\x04\xa5I\xf4\xe54\xe4\x9a+Y8wmP\xf1\xcc\xb3N\x85\x17\xb5w\xca\xa2\'\x0b[\xd0\xc3\x91\xa0\x1f`hDm\x07\xc6W\xae#\xf8<\xc49J\xbb\xde0\xdc\xb4@\t\x15\x19\xaf\xa3\x8d\xa9\xce\xf8\xbb\x8c\xdd\xb1\xee\x1aO\x9c\x81\xd0o\x02\xc6\x9f\xf2a\x9e\xa3\x89\x92\x08\x02a\xd4\xcb\x9a\xc0\xb2\xab\xda\xd6\x96n\xd1dd\x1e\x84{P\xc9T\xe2h\xa7[\xcf\x80\x87\xf3b\xd7\xbet\x97\xac\xde\x97<\x01\xe3\xb8\x8c:!\x1883`\xef\xb7"6\xcd\xc3\xbe\x0bj\xc3\xf7\x11\x8eR\x9dB\xa2\r\xf0\xb0g\xd87\xdf3>\xe5\xc3<78d\x1du\xe8O|\x98\xf0$\xf0A\xd3\x9cL\xd2-Bn\x1f6\xd5\xb2\x8c#\xb0\x8b\x011\xed\xe5\xb44j\xf9p\x1e\xd4K4\xb9gz9\xe70U\xa4\xc9\xb5<\xfb\x8dn\xf6\xa9\xb7\x1f\xf7\xcd`\xf6Nq*s\x10\x9f\r7\xe1\xa9\xfb\x9b\x8fp\x9f\xf2a\xbe\xed\xe3\xc4z\xc2\xbd\xc2\xe5\xe3\x1c\'\x12\xe4\x92\xc3^V&\xa17\x0f;\xe5~&[:\xa7\xf4K_4\xc9\xa9>\xc7r96\x18\n[nEF\x1d\x88\xd9\xb2v\x14\x9a;\xa69\xa6\xdb\x9f\xb9\xd3\xe5\x164\x030\x1d1\x86\n\xa1L\x02\xf97\xdf\xe0>\xc5\xc3|{8\xa6\xe9\xb5\xb3\x90\x97g\x9b\x19CT`\x1eFK\x85\xe8\xf2\xe0\xf1\xf2R\xd4g\x0e\x88\x8f\'\x80\xa0s\x14a)\xcb\xd3\xce\xae\x1a\xa7z\xca\x86\x92_\x1d\x03\xeaj\x1d\xf3\xf9:?\xb0\xc1S;\xd0\xb3\x11\x10\'\x134h\x0e\x0e[*"\xf1]\xa6\xf9\xd7\xe6a~[\xf9\x10EA\xc8Kj\xb3\xeck\xa6\x15jW2\xb2:\xccQ4K\xcd\x91L\x17\xb0\x99\xef\xf31Eq\x1b<\x03T\x16\x93T\x9c\xbaN5<.\x92+\xe5$\xe8dh\x13W\x1a\x91\x9e\xf6\xa7k\x17\xc5\xfc\xfa?\xe8)\xa4\x1b\x17\x13\xaa7\'\xcb\xa7x\x98o\x0f\xb6\xeby\x85c/K\xe2F\x9d\xd2\x1c7\x0e\xca\xc8\xa3\xb7\x1d\x97\xf2\x19L#\x13\xc4\x88\xe7\x1d\xaa!B6\n7\xd4\xe4\x94\x93\x0f\xab\xf8\x85P\x13\xfc.\xb0\x01\xca\xb1\xc5z\x93!&\xe3\xe8\x81W\xcd5r\xa7?<\x8e\x0b9\x16Y\x0e"o\xeeo\x9f\xd2a\xbe}"\xb5\xa66\xc4\x9f\xd0w\xb8\x17\x1e\xcciF\x1f\'N1\x94\xda.\xb4$<\x9a\x8f\xf9*\xb4\x84\xc4%-\xe8)7\tW\x89\x85\xf3c\xc1!p\x0f\xaf\xae\x8f\x9d\xafV\xc7\xe4\xd8g\x82k:]\xfb\x84.U\x1b\x9fAi\x8f\xe4G\x1dy\xf3\xee\xf6)\x1f\xe6\xdb\xadc\xdd)`\xf2U\xa4\x1d\x08\'\x053\x8d;W4\xd8E~\x92\x8dQ\x9f\xef\xdd\xb6\xd4\xf2\x9d\xebe\xa7\xe7\nG\xc7\x8a\xd1{\r\x9a\'Y\xda\xf9I\x15\x873\xdck\xc7=,\x06\xdd T{N\xf3\xe1\x01c\xf1\x81\xb8[\x97=\xf3\xdd7\xb8_\xdb\x87yN\x16t\xbd\xd0\xac\xb7\xb7\x97^<7\xc1i"/l\x8cJB\x97\x83\xe0\xa3\x9c\x84\x82P\xd2\x83\x06\xaa8;\x8e\x8f\xe3\x8e\x8f\x99\xb3\xb3O9-\xd8\x89B\x16\x1eJ\xa0\xf1\xda\x00\xb4\xb0s\x86\x14\x0c\x9e^E\xf0\xc25\x10\xb1^\x8a\x86S\x12\xc8\xd5o\x93\xe5_\x021\xdf^`7 \xe6\xdf\xfeI\xf5\xff" \xe6\xd7\xa176\xcdd\xeb\xd2\xaf\xdf\xa5\x1b\x10\xb3\x011\x1b\x10\xb3\x011\x1b\x10\xb3\x011\x1b\x10\xf3\x85\xdb\xb9\x011\x1b\x10\xb3\x011\x1b\x10\xf3\x95\xe1\x95\r\x88\xd9\x80\x98\r\x88\xd9\x80\x98/\xdb\xce\r\x88\xd9\x80\x98\r\x88\xd9\x80\x98\xaf\x0c\xafl@\xcc\x06\xc4l@\xcc\x06\xc4|\xd9vn@\xcc\x06\xc4l@\xcc\x06\xc4|exe\x03b6 f\x03b6 \xe6\xcb\xb6s\x03b6 f\x03b6 \xe6+\xc3+\x1b\x10\xb3\x011\x1b\x10\xb3\x011_\xb6\x9d\x1b\x10\xb3\x011\x1b\x10\xb3\x011_\x19^\xd9\x80\x98\r\x88y\x87B\xf8\xa7q\xfb\x97\x14\xc2\x1f\x9a\xf4\x95\x81\x98\x1f\x1f\xd8O\x02b~|`?\t\x88\xf9\xf1\x81\xfd$ \xe6\xbd\xc0\xfe\x13n\xe4\'\x011?~\xc4~\x12\x10\xf3\x7f\xb0yl@\xcc\x0f\x01b\xc8\xff\xe9\xf3\xbf\x06b\x04A\x10\x19\x9a\xc4E\x06\xa6\x11T\xc0qD\xa0x^ `\x1efD\x8a\xe1Q\x82\x831x\xdd\xe9\x18\x8a%P\xe4Y\x8f\n!q\x14g\x9f\xd5\xcb\xc4o\x05\xda\xbeo\x1fk\x8f4qY\xe8\xa8\x1c\xaa6\x0en\xa1\x8dh\xb26\x1fiLS\xe7\x93\xdb\xde\xdd\xcb#\xbb\x973xsk\x082\x99JC\xd1CW\xddR\x9am\xb8!jqU\xaf\xcd\x9dy\x1e\xf0&\xba\xae\xf9\xc3{1~\x06\x88\xf9-F\x0c\x82\xe8u(_j\xc53u\x88\x04%\x02\x1cQ\xcdJ\xef\xfa\xa1\xbar)\xd8\x93\xac\xc9k\\LK\xe8\x01z\x08\xdc\x02\xed\xf0Xp\xa4["\xb5\xd7\x94E\x1d\xe8\x8cd\xea\x91u$\xe6`\x8d\xe8\x95\xab\xa6\x81\xc1\x9bR\xebE-`\xee\xdf\xab\x18\xfd+\x031\xbf-\x08\n\xc3a\x88z-H]U%ps\xb8\xf4\xe8\x94\x02\x18>.e\xec\xda\xd9m\x04\x10h \x0f\x0f\x17&\xb0\xdd|\x95y\xaa\xc9\x0c\xd6\xc5\xe1\xb6iI\xa7\xca|\x87we\x8f\xa9\x8c\xbb\xbc\xecoj*\xdcs4_\x80\xbbwM\xbbZ~\xb3\xd2\xe8g\x80\x98\xd7l\xf0\x9f\xaa\xef\x1ej\x91]&\xc0!\x06yBt\xcf\xb8\\0\xa9\xa3\x91\n\xbf\xc8dt\xed\xe9\x9b\x7f\xc2@t\x8f\x85\x87\x11\xc9\'\xec\xc1\x82\xa6r\x03\xe3\x13\x93\x8f\xac\x1e\\\x16\xc6?\xceP\xc7\xca\x8e}\xe5g1\xf0\xef@\xc3\xbeW}\xf73@\xccoa\x12$\x81\x11\xf8+, \x08X\xc1QgU\xa5\xfaT\x86,\x812\x11\xc1\xc8\x1f\xbbK\xe8T\x08Z\xf6\x0f\xb2\xbe\n\xfbX*\r}N\xb8\x04\xa2\x03\xec\xa4h\xa7\x92\x12\xea:\x84\x1d\x9f\xe6U\x95+P\xb8%\x82\xe0\x8eL\x8dfZ\x977K\xd4}\x06\x88\xf9-\xccg\x87A\xc4\xeb.\x0e\xf4&\x89\xb1\xc5\xe4\xea\'\x05:\xa9\xc9\xa3\x07\xeb\x07[\xbbg\x0b\x97R\x91wC*\xb6\xf6\xf7\xb0\xe3\xfd\xfb8\x06K\x8a\xc4\x8bb\xb8\xa0\t\x06\xc0!d\xf7xA\x9f\xe1\x93\xbd\xcb\xa9S|>\xf8-\x87\'\xb3\xfafY\xc5\xcf\x001\xbf\x8f\xe6\x9aca\xeb\n}9\x93\xe9\xfb\x04z8qi T=\x8b\xa9\x19P\xb0\xddI\x05K\xea\'\xdd\xcc\xd5\x149^m\xd1\xca5&\xc6N\x99\x07\x07{U\xa2\xeb]\xeaC\x11k\xb2H7\x9fr\xe8x\xa8\x93\xd2\x92\xeb\xa9I\xa7\x9d\xc7@\xec\xcf\x00b~\x0b\x93|n\xe4\x7fR\x1e\xd7\x12\xd9\xe12\xe9@\x83\xa5\r\xd3M\x85\xea\xb1\xe1^x\x80\xf9t\x0c\xe3\x83\xc0\x92\xc8=8dk\xbe*\xea\xfb\xf2B/\xbd\x88\x17\x8f\xe3C\x18\xf0+\xc3Ww\xf9(\xdc\x04\x92\r(\xa95{M\xa9\x1f\x84\x03\x00o\n\x07\x9f\x11b~\x0b\x13\x83\xd0\xf5\x82\xfc\n\x9a\x9d\xf8\x9a\xdeMHt*\xd5\x84\xa2\xfd\xcc\xe6dZI\x96\xf2\x9e*v{\x8a\xd1}\xc1\xdb:q\x84H\x10|\xa8\x94\xe5\xdb\xfb\xa2\xcf\x1d\\C\x88\x06\xab\xefXF\x90&\x8e\xc4F\xec,0\xb3\xa3h\xb70\x15\xffk\x9dW\x9f\x11b\xfe\xb1\xc3\xad\xc95F\xbf\xecp\xf6\x03\x86\x0f\xe6e.T\x93\x8aU#\x97\xfa\x9bq\xb5`f"\xd7\xeb\x8d^\xa1p3\xc6\xba\xb7n\x8f\x9eh\x85LM\xb2(\xb2\xa61\xf4x\xb0\xd5\xbb\xef\xdcdu\xbd2\xa7\xe3]\xe4\x93A\xabb\x94!\x12\xc9zsM|F\x88\xf9\x16&D\xc2\x14I\xe2\xe4\xcb\x0e\xa7\xa5E\x8b\xbaM\x83_1\xa5\xaf.m\xe4j\xc5}\x1f\xec\xa2\xd6Co\xa6k\xe2\x0f3\xbeL\x04\xe9#Y\x91]\x0f*\x98\xda\x87\xfc\x1c\x9f\xfd\x06\'\xef\xa0\xa6/\xd2~\xa4\x88\x05\x94\x85\xb6\xc7\xf8}\xc8)\x0e\xf3\xe6\x9a\xf8\x8c\x10\xf3{\xf6\x81\x7f\x8b\xf2e\x87\xebYm\xbd]M\x08K\xce\x12\xa0\xf2V3s\xd5\x11\xa0\xd0\x1a\xa3T\xeb\x10_N\x97\xf1\xd0=\x8c\x1c\xba\x83\x9d`\xa3 8\x1c\x85\x18\x92{M\x05\x86\xf0D\xc1\x9c~\xc3z\xaf\xed\xd4\x08\xb2 \xf6\xb6\xf4z\x08|\xefX\xfe\x95\x85\x98\xdf2ax=#\xd6\x19\xf3\xd2\x8b\x10Q\x9c\xea\x9d\xd5\x86^N\xb3\xd9\x88@{Y\xb8\xbb\xb4\x94\x1aFs\xf2\xe9\xebeW$\xf2p\x8e\xd0\xeel\x0ft\x89\xcfd\x11s&qt\xbbGU\x94\t?]\x108\xac\x13\xfe\x80\xee\x1f\x14\x84\xc8e\x12\xbd[-\xfe3D\xccoa\xa2\x14\x8e\xfeI\x95a*\xa4M$\xa9\xafZI\x06\xa6\xd2u\xe1\xf5H\x86\xf6`\xee\x19\xe8\xd6\xb8\x07\xd0\x13\xc6\xbc\xd7\x10L98\x02[)\xd5\xd4-\x12\x06\x02^\xb2;$\xd4nT\xd1p\xe7\xba\xb7\x80r\xafS\x01]\xdd\xf5_\x8f\xfd\xf7L\x91\x1f*\xc4\xfc>\x98k\xf6\x07c\xf0K\x95\xe1\x9b\xa8\xf1k[\xe9\xf5[=\x8a\xae*\xaex\x98Q\xdca6ro\xcf\x18\x80\xc4\xa6\xed\xae;9P\xeb\xcb\xa4u\x86\xaf\xa6v\x97\xa0;n\x8d)\xc5\x1f\xa9\xfd\x10\xaa\xd4\xd4\x118\x8c\xf5\xc9\x89\x19x\xc4\x8e8\xe9\xcd\xdb\xdbg\x84\x98\xdf\xf7q\x1a\xa3H\x8c|)\x16\xcf\x10\x841\xec\xd3q\xbe4\x16iM\\x3b\xb8\x82\x97&\xd6\x88X3\x8b\xbe\xcarG={\xbbx\xd6M\xa8\xa6\'\x89\x82\xcd\x87\xbe,\x8e\x14\x9bu\xbd\xa6\xb9\xf6\x1e6H\xb1\xbf\xb8\xceh:c>\x9b\xc6\xd7:\r?#\xc4\xfc\x9e:}\xdb\x8c__m4C!R\xe9\xaaM\xb7\x85T\xa5\xf0~\xf6\x99\x02\xf3O\xb7\x9e\xf5\x84P\xaf\xf4j\xae\xf3RU\xf1\xd9\xb9\xbb$\x88\x89\x84\xc6\x19\xd8\xe34\xce\xd7!S\\\xd4\xdd\xdf\x07\x19\xc8\x0b\x88\x95#wo\xc7\xb4nT\xe6O\x11b~\xbf\xbdQ\xeb\xe4\xa2\xe9\x9751s\x1a\x96&\xd7\xe4\x92\xe5\xc3bj(\xf6@\xfac}\t\x1e\x84\xeaf\x1c,\x11\xc3\t>\xc17\'H\xd98E\x0e^c\xf0Ff\x03\r\x15uI\\\xdaw\'\xf3\x82\x03\x93\xc5\xeaA\xa5\xd8\x05V\x0b\xd6\xcd\xdf\x05!?"\xc4\xfc\xb6\xf4I\x84@h\x14y\xd9\xc7\xd7\x1d\xd9\x04tM\xa9\xe4p$]\x88\xc4\xe7\x18T[e!\x1d\xed \xa9\xc3\xbd\x19\xa0d\xe2;3\xad;pp1\x1c\x94\xcb\xac\xeeD\xff\x01c\x89\xcf\xef\x13[\xa1\xa3*\xb2eG\x9d\xc3\xe0:NY[Lo.\xfd\xcf\x101\xff\x98\xb4k\x82@\xbf\xb2F\xe5rT\'iA \xe5\x1eF\xf6\xc4Rh\x00j\x8b\xa9 \x19\x03\xa6;!\xba^*u\x7f\x07\xba\xd6\x8b\x90b\xf0m\x08c\x94\xf86\x97\xb2?\x8d\xb8\x8d(\xae\x14\xf9\xd5\x08D)>\xb5\xf8\x19\x08R\xf7Z~\xadC\xff3D\xcc?z\x11&\xd7\xa5\xffrk\xe2n\xb5n\xab\x1c}B\x0c\x89\x88{L8\x9e\xcf\x07\xb5Cg\x7f\xb8\x94\xf70=\xb7;d\x8f\xd0k.\xd0\xd3p[\xc8\xa5X%\x92q\xaf\xdc\xe1\x1c\xe7o^k>C\xc4\xfcc#\xc7\xd6\xf3\x8a|9\xaf\x1c0\xccO\xe91\xbaZ\x93X\xdd\xf5\xe0R\xe9\x00\x9eY\xde\xe4S$\xc8U\xb4\n\x05\xe4r}\xb0\xe6\xf5d\x94*\x8f\x9c\xee\xe2\xfe\x88OTYN\xc7+/d\x1e\x81\xd3k2}/f\xa1\xddW\x92EE\r\xfa\xe6\xa4\xfd\x8c\x11\xf3{\xaa\xbaf\xb6O@\xe4\x8fa\xfa\xf04\xa8WA@5\xc4\xd5\x86\xfd)\x98\xe2\xea\xd1\xa1\xb6\x81\x1fg\xce\xf2\xe8\xda9\x04\x1e\x9fZ3*(`\x9f\x97v\xe9&\xb6\x90(\xdd\x03\xe1\xf8\xfe2w17@\xc11\x99`\xb5\xe9\x80\x8a\xae\xe1E\xfdZ9\xdcg\x8c\x98\xdf\xaf\xfa\xeb\x90\x124\xf4"\xed\xa0\xccd\x1f\x03\x97\xa1\xba\x93\x0c\x01V\xeb\x02\xed\\\xd8\xb3\r\x87\x00v@\x02\x04\x8f2\xdf9<\xca0+\x96\xe8\\66%\xef\xf8:)\xb0\xf8\xaa\xc7\x01r@&\xc7\xe6`/9\x82\x82/\xf6\x90\xda\x1cv\xc8\xbb\x1fI}\xc4\x88\xf9}#\x87\xf1u!\xbd>M{\xb5\xb7+\xb2\xa3\x97\xe4\xc7\xdc\x98p2\xda\xed\xbd\xb9\x19I\xb9\t\xa5\xa8C\xb4\x98\xeb\xec*\x07\xaa\xfe\x90]\x1f\xf4\x9dE\xce\xed\xa9\x16\x0e\x10\x11\xa1\x1aZ\x15\x86|&\xad}\x05\xf1\x03\xdb\x06\x9aX\x8b\xb4;\xbe\xfb\x01\xe3g\x90\x98\xdf\xd7\x04\x86?\xc5\x9f\x97\xd1\xac\xce]\xed\xa5\x92\xea]\x87\x82\x99\x03\xcecD`\xa0s\xb7rxWB\x90\xa4\xe8J\x03o\xebS-\x1b\xd25\xd9qW\xaa\xb5\x1f;{\x8e\x1d\xcf?*\xd7s\x15iigB\xb4\xe9\xdd@~wk\xfa\xdb\xd5\x7f\x133\xfc\x0c\x12\xf3\xff}l\x04a\x04\xfc\xb2\xf4\xd3\xeab/W\xc9\xe3Q\xb7p2\x98\x8c\xd2\xd8\x0c\x9bI\x8e\xae\x08\x0ed\x94\xc6!\xfb\xb8\xb5y\xebq\xb0\xdc\xa0T\xed\xdb\xa5\x08|\xc9S/\x0c\xc0\xcc\x0fk\xb6\xcb\xa2j\x85x4\x13\x13\xe7\xccT\x18=u\xff\xbd\xd1\xfc\x95\x91\x98\xdf&\x0bDc8I\xbe\xbe\xf2a\x94R\xfa\xc6\t_Z\xdd\x94\xaf\xceH\xdcUP\xbeC\xd2t\xea\x13h\x80\x08\xe9\xb6\xb8M2\xabj\x1c\x0eI\xc5\x95\xc3|F,\x84\xc3\x128"D\x9c\xbb"\x8c\xd1\xae\x07\xb4\xdb\xde/\x04b\xa7u|\x15\xf9\xdf\x96\xfe\xbfDb\xbe\xad\xe5\r\x89\xf9\xb7\x7fZ\xfd\xbf\x07\x89\xf9\x85\xac\x88\x8d\xdf\xd8\x90\x98\r\x89\xf9\xff\xe3,\xdd\x90\x98\r\x89\xd9\x90\x98\r\x89\xd9\x90\x98\r\x89\xd9\x90\x98\xaf\xdb\xce\r\x89\xd9\x90\x98\r\x89\xd9\x90\x98\xaf\x8c\xaflH\xcc\x86\xc4lH\xcc\x86\xc4|\xd9vnH\xcc\x86\xc4lH\xcc\x86\xc4|e|eCb6$fCb6$\xe6\xcb\xb6sCb6$fCb6$\xe6+\xe3+\x1b\x12\xb3!1\x1b\x12\xb3!1_\xb6\x9d\x1b\x12\xb3!1\x1b\x12\xb3!1_\x19_\xd9\x90\x98\r\x89\xd9\x90\x98\r\x89\xf9\xb2\xed\xdc\x90\x98\r\x89\xd9\x90\x98\r\x89\xf9\xca\xf8\xca\x7f5\x12\xf3Oi\xc5\xbf\xe4\x10\xfe0\xca_\x19\x89\xf9\xf1\x81\xfd$$\xe6\xc7\x07\xf6\x93\x90\x98\x1f\x1f\xd8OBb\xde\x0b\xec?!G~\x12\x12\xf3\xe3G\xec\'!1\xff\x07\x9b\xc7\x86\xc4\xfc\x10$\x86\xfa\x9f>\xffk$\x06&8\x8a \x08\x0c\xc1I^d9\x8a\xc7D\x14\xe5\x89\xdf\n\xb0b\xbc\x88\xb0\x1cB\x11\x14O\x08\x10\x8eP\x1c\x82\xb0\xbc\xc0\x0b0\xcc\x12\x10\r\xd3\xc2_\xfb\x07\x1c\x84q,\x8b\xf2\x88\xc0\x90\x9c \xac_\x97\x15\xd7\xff\x03\xd1,\x8f\xe3\x04\xc4\x89(\x81b,\x8c\x08\x14\x8a\x100G0\x10-\x88\x10\xcbP0%\xe2\x0c\xc5\xfdH$\xe6\x1f%\xd0\xff\xf6g\x05\x18\xc8\xbf\xe3\x14LP\xcf\xd2m\x7f\x85\xc4|}a\xe7O\x90\x18\x91Ay\x01\xc5P\x1a\xe7i\x0c\x13!\x98\x10E\x9c`\x9f\x15X\x90\xe7\xdfFi\x16a\x9f\xbe\x00\xb9\xfe[j\xfd\xa2<\x87\x88\x08\xc2a,M"\xe2\xb7\x02\xfc\x1b\x12\xf3\x03\x91\x18\x9c\x17ET \xf8u,Yd]\x8c$\x8f1\xeb\x9f\xe5q\x82Bp\x01\x83x\x8e\xa1hX$\x11\x96A\xd6\x05\xf5\xb4B JD\x18^\xa4\xd6\xa1\x15\xb1\xbf\xfd\xf2H\xcc\xbfI\x97|\x0c\x89y\x166\xf9K$\xe6\xeb\xaf\xf3\x9f\x88\xc4\xac\xdb%\xf2}E!\x17q\xf0\xbcC\xf7x\xb0\xa3\x1c\x81h\xae\x957\xef\x12\xd0\xd9\x9dt#\xac\xe0\xa3WZl*\xb7\x1e\xcc%\xa6h@\xd1\x82\x1e\x19\x96_\xd3\x8b\xac\x0cZA\x13Mx\xe8\x848\x12\x1e\xc3\xbe0*\xe2\xe6\xc6B\xf0\xb3\x90\x18\xf2\xef\xeb\xe0\xe2\x18L\xd0/\x95\xda\xe7*}\xd8\x04\xe97W%S\xf43y\xafN\xc7\xb3o\x82\x18-\xc9\x01\xccOiP\xe7\x02\x17\xcc\xbb+\xa5@\xa3\xea\xeb\xcb%\xd3\xf3k\xa95:r\\\xa8\x03\x02\x9f\x9f\\\x06\xde3\xbbS\x8a-b\xb6w]\xee\xcd\xe2\xdb\x9fBb\xc8\xbf\x93\xeb\xd9\xfeJ(t\xe9\xc1^l\xfe:`\xe8\xd1\xa2\xb8\xd9@\'\xfe\xe1G\xb8\x96\xb3\xe5\xe1h\xd2\'\x03\x00\xd4\x1dk\x0e\xea\x81J\xad\x8b\x88\xcc\xa6\xa3\xb8\x97\x83S\xe5}\x16\x82,\xde\x9d\x08\xf3P\x03\x8f\xfb\xc9=\x1c\x1el\x0e\x1a\xfe\x9bU\xc7>\x86\xc4\xacC\x89C8A\xa2\xafU\xc7&\xac\xe1\x15\x12(:\x1dG<\n\x13\xb9\x92\x06\xb0]\xcd\x91K]\xa0\xe7\xc3Y7-\xb2~\xdcF\'0\xd3\xc0\xeao*\x1f\xc8\x87{\xe4[\xba\x93\xc5\x18i\x9c\xcf\x81\n{9B\x1eC\x9c\xf0\x15\xbc\xec\x03\xe0{E\xa3\x7fq$f\xedE\x84\xa4!\x1a}\x15\x1a\xa6\x1d\xbf?\xf7\x07\xbc\xa7\t\x94\xbbD\xca\xe8\xbb\xc8\x11B\xe5\x02\x80{\xc2\x9a.\x18%h\xa3\xb7\x0f\xa8\xda\xd2\xe0\xcb\xe5~\x05\x02\x7fX\\\x19\xc2I\x8dmj\x1a)\x1eY\x9e\x0e^\x1d\xd4\xe5\x0e\xb3\xf7\x870\x14\xde,\xbd\xfd1$\xe6\x19&F@\xeb\xf9\xfb\xc7([>\xbeg=qb\x00\x00\xc4\xa1Q4\x07\xc2\x8c\xbc\xca3\xfc!\xb6\x04\x99\xe2\x06qp)\xb8\xa1r\xe3\xd6s\xc4\xb4\\\xcf9~\x11\xd5D\x16r\x11\x1e\x12\xba)3\xbe\xf0\xab\xd6^\x94\xf3Y\xa2S\x7f\xdc\xbfYQ\xf5cF\x0c\xf9wlM3\xd6\xf1G^V>\x99\x18b~\xa2\x8fw\xe2zp\xcf\xa6M \xc0]\x12\xf9\xee:#B|\x1bY\x07\x10\xdbR2\x8f\xc8\xfel\x04\xc7\x90\'\xae\xbaN\x067g\x80"\xa1 \x94\xfa\xaeX\xa3\xd7\xda\x00`\x82\xe1\xce\xeb\xac\xfd\x90Qo\xe2)\x1f3b\xfe\x90\xda\xff\xd3&\x9e\xe8\x8e\x80\x97f\x12\xe4wU\xe3c\x11\xdaq\x8c\x01\xd0X\xdeB7G+\xaeN\x0eh\xf7\x1e\rg\xee\xe0\xd6\x8f\x82.\xa6\xfaZ\\\x0fH\xcd\xc23\xef\x87\xf2z\x9b\xf1"\x0bb\x81\xfdp\xa4\xcb\n6$\xed\xcd\xd1\xfc\x98\x11\xb3\xceY\x08B!\x88\xc2_\xc2\xbc8\xfb.\x03`\n\xe6Z+B\x14+\xc7\xae0\x92\x0bp)vyX/\xda\xa3\xbe-\xc6.#I\xda\xa7p\xec\xaeD!]\xec\x89@p\xae\xdd\x19\xa3\xd4^\xad[ \xaao\xc1\x92\x95<\xaf+\xe4,\x00\xee\x9b\x15U?f\xc4\xac\xa3I\xaf\xf7\x16\x02z=\xae*\xed\xbe\x97\ng\xee\xc7\xeb\xd1\xabZ|][\x00\x86\xba\xfa\xbd\xce)aD\xe6\xa2\xabP5r\x93\xb0Q8\x9e\xadoz@k\x8b\xee@\xfae,28\xbf:@\x8f\xdev\xf2\xc8\x9c\xd8Sw\x8d\xd0\x8c\xb0NoVT\xfd\x98\x11\xf3\x0c\x93\x84!\x18!^\xb6\xa0<\x02\xaf%\xf9\xc8\xa0\xf20\xde\'\xa6)b\xa5-\xd3\xca\xd4oj\x14\xed \x15*N\xf1\xda\xafn\x89&\xcd-\x88\xe0\t>\x9c!*\x07\xf5\xc1bBB8\x1c\x854b\x1eI\x05T|d\x15s}\xcb\x03\xe3{\xba\xc9/n\xc4\xac\x89\r\x8c\xa0\x10\xfa\'\xe5w\xdd\xa5\xf0D\tN\x0f\x15\xc3^@\x95\xbb\xa3\xa5E\x04laf\xc2b\x85\xee\x9d#N~,\xae\xe7\xa2\x97\xcf\x84F\x00\x8csX\x842)\xcdQ\xb6\x8clW\x9e\xafD#\xd6p\xf7p\xd7\xa4=o\xc6\xee\xce%\xcc{k\xe2cF\xcc:Y\xd0\xf5x\x7fVf}\xa9\x1e\x19!\xc0\xc8\x87\xf6\x91\x82\x1a\xc8\x9c\xc2\x89\x01\xaf\x93/{\xf3\x1a\xb7\x0e\x9f\xce\xa7{\xcc\x11\x826g\xb8xd\x17\x89M\xa2\x91\x19\x1f\xebQ\x11R!\xaf\xd0\xcd\xc91J\xea6\xcf\x1e\xc3R$\xc8\x08\xf5\xd5\x87\x91\xb8\xac\x9c\x8cp\x0f3E%6R\xbd\xf96\xf51#\xe6\xf9h\x83\xe00\x89\xc0/K\xdf?\xecH\xdeaj\xd5\xe3d\xb3\x02\x85\x98\xc2\x1e\xe7\xc3u"\xccAC\xd4\x1e@\x87\xc3\x85!\x04A\xbd\xa4ipYO\xe5\xd0w\xa5+\x9bX\xd9\xe8\x9e\x16\xde\xbcvy\xc7\xd8\x9a\x83%\xc1\x9b\xb7\xa6\x8f\x191\xe4\xdf1\x8a$1|\xdd\'_\xae\xfa\x8e\x9f:\xbc\'+\xd0\x8e\x87\xbb\xfd\x03\xecG\xffD\xa3\xa7\x1e\xec\xba\x1bR\xf0\xb0cL\xe6\xa4\xb6P\xf5\xc0.H\xc5i\x91\x94\x9e)\x17\xf2}\x7foS\xe0\xba\x0b\x0cE\xbd\x8b\x8aG\x98?\xd6\x84?\xc7\x0f\xb2\xf7\xe6{\xed\xc7\x8c\x98\xe7\x9a\xa0\xe8\xf5\x1c\xa5^\x9e\xa5\x99jf\x12cLDJ\xf2:\x15\xc6\xc4\xc0/\xda\xfe\xa2+u\x8bu\xd5\xfe*\xe0\x97\x11\x1e\xc5\x0b\xd6h\xb9\xa4\xbb\xe1I`\xad\xdb\xaejQY\xe6\x85j@5\xee<\xa3i.\x19 \xc1\xba\xe6\x0e\x9c\xfd\x07\xf1\xe6F\xfe1#\xe6\xb9\x91\x7fSW\xe0\x970\x1b\xfdv\x85\x0cX\x00\xbd\xd3\x15\x085\x8a\xa6\x8d!\x0e\xd7\xdc\xa6\x11o\xf7(\xa3\xd5[\x02qv:\xf9\xc6U\x1d\xd0\x03\x185\xa5\xd14\x86\x14/\x05\xbe+\xd9\xd8U/~#\xb1\xea\x12"D8\x12\x98\x05\xb3\xfe\xcf2b\x9e\x1b9J>S\xb8\x97I\xbb#\xef\xf2\x83I\xefj^\r\xf6m\xc9\xc1\xa6\x99C\xaa\r\xd3\x10\xcd\xdc\xd6\x03CnR\xb1f\xa6\xe3\x1e\xd0\x08#=[\x90\x19\xa4\x8f\x88=\xbb2^\x0c\x88\xb7\xcf.\xfa\xc3v&E\xee\xc7\xf5\x10\xc3A\xc0\xbf\xbe\xf9\x0c\xf71#\xe6\xdbh\xa2\xd8\xf3\xe1\xe7\xe5\xbc\xaan\x8cD&i\xa9e\x15\x9e ~\xc0(V\xda\x1c.\xae\x1e5\x8fQnq\x08\x9c\xa4\xc0\x19|\xb1>{\xf9\xdd\xe9:\x1d\xa8\x1c\x006vp\x81\x89F5\xc8Qm\xea1&\xf0\xd7T\xf0aC\x13\xb8\xf2k\xe5p\x1f3b\x9e\x8f\x99\xf4\x9aQ\xffI/\xf2\xfd\xbd7\x16W\xec\xc6r\x98\xcc(;PB\xee+6\n5\xb4\xde\x8d\xf8\rnZ>k\x1d\xe3(K!\xcf*\x10\xff\xdb\x88a\x19\x18\x83\xa1up)\x98 x\x94\x86a\x9c\x178\x96\xe1Da\x9dB\xb8\xb8\x0e\xba <\xcb\xbb\x88$\x0es\xcfo\xc9\xd1\x02\xc9r\xcf\x92q\x98\xc0\x12\x7f\xfb_S|3b\xd6\xaf\xb6\xb6\x89\xc3IX\xa4q\x11CD\n[\x07\x8bxV\x10\xc40\x18#x\x91\x87\x98u\x98q\x1a\x13\x9f\x95\xa6E\x8e\x85D\x0c\xe5\x05\x84\x17iz\xfd\xc5\xb7r\x82\xbf\xb4\x11\xf3o\x0b\x06/F\xcc\xdf\xb8\x96a\xb9f\x1d\xb2l\x1a\x03\x84~\x9c<\xfa\xa1T\xfb\xecT\xdd\xee\xc9U\xccC\xdf\xccT\x0b?\xc6\xc8\x9a\x98zx!T\xd0\xe1\xf0\xed\xcfQk\xb6\x17\x16I5\\\xcf\xd5\x84\x1e+h:\xf2\xec\xfa\xfb\xc6CE\xe6F\xf3\xb3\xbb\xe65S\x80\x04\xcb\xb1\x1a\xea\xc4\x17\xa7s-X\xff\xdf\xdf\xe5\xd5%z\x1c\x91d\xf7\xfc\xef\xb5\xe1~\xfd;Ub\x97DT\re\xec\'D\xbc3\x91\xa8\x16\xd1\x18ug\x15\x99\xd6,\xb2\x89\x04\xa3\x11\x13\xef\xdb\xdf\xb78^XT\x9e\x99\x9e\xff\x08ep\xb0/\xad\xc4e\xd4\xff\x8a\xe1\xb8\x9c}\x16:-\xf8#\xf4\x8e\x93zu\x90\xe8!\xdc\x9d\x9d\x9b[\x88\x8b\x0b\xb9z\x90\x8d\xe3~\x86hR\x81\x02g\xf0\xeeh\x1f\x15\xae\xc9P\xbb\x8bL.S~/C\x95n\x95\xce4\xec4e\xc4Y\x16\xb8\x8b,3\xa6,\x08*\x17\xf1l\xcb\xb4\xb0Y/\x98\x19\xa4\xbaT"3;xG\xec gpO\xfaA\x94Q\xeb\xfd\xc4\x1a\xbdj\xd9\x1d\xf7\x9d\x1c\xc2Q\x01_\x8fE\xe4J\x16\x98\r\x85a\xc1\xc7I>\x8c\x924\xa2\x06mt\xfcB\x14\xa9dz\xf8\xf1Y\x80\xec?\x95{\x9e\x1b\xcd_\xca=_\x7f\xf7\xfd\xb9r\x0f\xf5}\r\xe1*Z\xd7r*\x8e\x9dv\xbe\x1c\x0bA\xde\x95A\x05N\xf2\xa9\xcd\xf3^_\x80\xfeQ*\xad\xa1(\x8bo\xc6n\x0b\x07\xe3.\xa85\x14\xbc\x87g\x99\xcbz\xda$s\x05\xb3\xef\x87`\xd9\x93\x8e\xa3\x89\xb0\x95\xbau\xf7f\xb5\xb9O\xc9=\x7f<\xab\xffw\x98\xb0\xa8(\xda\xc9,);\x9e\xb4\xf4\xa4\xb2T9\xdd\x18\x08\xd1A\xa4J@\xf78\xb6U*\xc7\x07#f\xca\xe4\x82\xa8\xf5\xcd \x06}6sT\x17\xcf\x10_3\xd8\xd5g\xb3\xdc\xbb\xb9c\x1ah\x91\x9dY\xc7\xb9\xfb^\xb5\xb9\x1f,\xf7\xaca"\x04Nb/\x85\x03Ek\x8f\xd0\x87\x16\xea\x12$e\x1e*qG\xb5q\x9d\x1d\x16\x88H9r\xbb\xa5\xb3\xbb\x10\x12w\xba\xedz\xc0\xe0\x92\n\x9b\xf7\x10\xa5\xd7{\x10\xaf\x97\xf9^\xdc\xe5\xd3\xce\xe4U\xa1*\x1dP\xa5%"9IK\xb1\x7f3\xc6O\xc9=k\x8c(\x8d!\xeb\x82B_\xca\x16G\x17\x9d6u\xa0\xc9\xe9;\x96\xf1\xba\xe4\xf3\x8f\xde\x06\x1c\xc9\xe6k\x08\xafn\x10;d\xb5\xbb\xaf\xedi\xd4/\x0f\xf9^p\xca\x1a\x91\xd3\xe7s\xb3\xa3\xca<\xbd\xfb\xa3oP\xba9\xb6*\x11\x90n\xb9\xcf\xebC\xf8=\xa0\xe8\xd7\x96{\x9e3\x85\x82\xd6\xdf\xa6^\xabL^\x89\xf5\x9a\xc8S\xf19\xd1\x0c\xaa0\x91\xe3L\xack`\xe7qy\xc2)\x1e?\x1e\x91\xf5\xa4\xf6\xce\xf0\x19\x05G(\xbdhc\xc9+\xf8l\x1fl{\xf0]\xed\xe1REE)8\x9a\xf6\xf2\x05?N}/\xa2~\xf6f\x05\xd8O\xc9=\xcf0I\x18\xa2\x10\x18~)\xfe\xcc\xf8J{=\xef\\\x07\xc9\x94\x12\xb2\xbcJ\x81\xe5=\x1ad\x16m\x83\xca\xf5\nK\xae\xa5\x9e\xee\xb2\r\'9~\xdew\x82\\I3\x85/\xcc\x10K\xd5M\xad\x1du\t\xcey4\x91\x0f<\xb1\xb2\xab\xbd\x94`\x87\xbf\xbb\xee?D\xf7<\xb77\x1c~\xd6\xd3\x83^\xea#v\xf7\x865\x99\x94\x9evG\xf7:\x95\x87r\x90\xda\xee\x16\xf0\xc1MC\x1e\xcb}\x81\xca\x1ck\xe9\xa5\xf2U9\xd3\xad\xbb\x01\x94>\xc7=X^{\x04d~&BX\xc7Q\xa3\x98]\xc9\xa0\x1d`\x84\x05\xa3+\xd37k\\\x7f\x8a\xeey\x8e\xe6s\xe7\xc0\xa9\x97\xc1\xe4\xbck\xaf\x9f\x8c\x91^/\r\xf7q\xe4D\x1f6\xd6\x14\x02\xf0ZmME\xd6\xec\x01\x9c\xd1\xf1l\xae\xbd\xab\xf9\x16u\xdd\x03\xe4\x8e\x92\xae\xdd}\x89\x15SP\x94b:s\xcb\xe0\xcc\xf8\xbe\x1d\x9c\xe7\x98\xd7Q\xf6\xee\x9c\xfd\x94\xdc\xf3\x8cr\xbd4S\xc4\xabk\x81\xd4\xbbc\xbc\xb3\xccn\x80\xa6\xa8l\xcc\xb8\x05B\x1c\x84\xe8\xbd\xc0\xc4\xa6\xa4c \xc9EY\xeb\xd0k\x96c\x04\xa1\xd8\x1b`\xbb7\x00\xcfQ\x1d\xe0p\xa6!*\x97\xae\xc3zQ\xd1\xef;(i\xa7\xb6\xd3\x8b\xcayS\xef\xf8\x14\xdc\xf3\x8cr\xcd9H\x98x\x15\x19@\xa1W\xd9\x08\xe0\xa1\xc2\x04\xb3\x9b\xc5\xc6\xa7s\x12\x83\xd7"\xc0\xfc\xcb\x1dof\xe4\x11\x1c/.\xcc\x9a\xc0\x01\xcd-\xa9\xcf"\xd5\r\xf9\x8b\xa8,\xcd=\xa9\xbd\xf4"\xd0\xf8q\xb8\\\x80\xbcp\xb3\xcc\x0cY\x8aJ\xde\xa4\xc3>\x05\xf7\x99\xfdr\x04v\xbcA\xee\xd8\xdbY&=R\xecB0\xf5U\x88\xa2b\x14g\xceK\x18\x10\xd7\xb1\x14\xda\xf3\x91;\x11\xe1\xf0\xb5N\xabO\xc1=\xcf3\x7f\xcd\xce\xe9\xf5&\xfcr\xe6\x0b\xbex\t\x8e\xa3\xd8=\xc2\xb1l\xbdx\xea\x01\x9c\xb3\x8cT\xe3\xc0\xf9\xc2\xe6\xa2\x93\x01\xa2\xd9\'W\x18j\xd6%\xd0Abc\x9bM$v\x8cV\x8cp\x04a\x995-\x9e\xdc#\xc9\xd1\x99\x0e\x06o\x12\x0f\x1bx\xf7\xb4\xfa\x10\xdc\xf3\\\x13(\xb4.\xa1\xb5\xc3\xfe\x18&\x9b\'\x88\xc2\'L\xe3\xb1\x88\x8f\\4\xd0&\xafr\xcc\xb5Hv=\xea\x8b\x90\xd5\x14}\x19 =\x9c\xf6\xb7`\xe0-\x1b\xc6\x97\xcb\x01n\x85\xd3b\xc2*\xa8\xb1\x9cysNK\xc8\xd9\xff/{w\xda\xe4\xa8\xb5%\n\xfb\xbf\xd4W\xdd0\xf3\xf4~c\x14\x88A\x08\x84\x18:nt0\t\x04\x88A\xcc\xf4\x9f\x7fQ\xd69\xdd\xb6\xe5\xf2i9T\xae\xacsU\xe1\x08\xbb\xd2\x99\xd2\xde\x9b=\xac\x85\xc8\xf5\x98C3\xc0G\xdf\xb89\xf6\x93\xf6\xe2\xab\xe0\x9e{7\xb1\xbb\x01B=\x16\xf2Nn\xfbl\xe7\xba\x81\x18\x87\xc3\xde\xcb\xbdC\xe5\xa4\x11\xc1\x99(\x12\xd2\xa5\xb3\xd7\\N\t\x16\xfb\x80\xf6x\x18p\x90k\xe8}\x87\x89\x9c\xd0\x9bs\xba4\x11N\xb9\xecv\xc3\x18\xad\x19\xa0F\n\xcd\x13\xceVU\xfa\xad\xb2\xec?\xb7\xdc\xf31Y\x08\x14\x85\xd6\x1f\xfb\xfd(n\x89!\x02\t\x17U\xc3-\x03\xa7\xb9\xb5W\xb4d\xab\x93\xad}\x0c@\xc2\xc9\xb4s\xa7\x83\xfb\x90\xbc\x82\x02n\xf2\xc6Q\xcc\x91L z.\xf1\x82\x93?\x16L[`yma\x86\x16\x97\x98s9\x03g]\xc3\xbe)X~g\xb9\xe7#P\x85\xc0{\xc1\xe0\x87\x04\x95\xbf\x01\x9a\x10\x99\xea\x8e\x9f\xa1\x9d\n-\xed\xba\xc3\xc0,t\xeaTb0-\xf8\xb0O\xf9\xed\x14/\xd6!N\x92\x94#\x9c\xa9\x0e$\xc8\xf1\xb6M\xa6,X\x91\xeeJ\xab\xdc:\xaalG\xcb\x8ew\'c\x10nF\xfa\xe4q\xf8*\xb9\xe7k\x82\xba\xee\x1a\xe0\xe3\xa9\xcf\x19\xb3\x81\xee\xad\x98i\xab\xb3\x95Z\xb4\xa3\xf3\xd0lE)V\xc5\xf5e\xee\xd3\xa3\x124\xa1\x17\x05\x8a\xb6\x0b\xc7\xc5&\x0e\xed\x95\xa9\x86\xeb\x16\xda\x18\xd0vXSw=;pR\x86\x12h\xa7\xd8\xc5f\xdc#\xf1\x85{.P}\x95\xdc\xf3\xb1\xf4\xd7\xf3\x00\xc7\x1e\x95\x12[\x02\\\xe5T\x9eMGt\x8a\xbda\x9a\xf1M?*\xe8D\xf1|\x05\\\x07\xb4\xbd\xed\x8fWb\xef\xc1=\xc8\xdfb2$\xc8`G\x94\xfbZ\xe5\xfa\x83\xb4\xcb`:\x16\xe1[\xd8\xb6\x01\xb7n\t;Y\xf4\xbc\xa8\xf8\\\xa5\x9f_%\xf7|,}\x14\x04)\xe2\x11x\x93\x84\xb0&\xbd\x16\x1b\xf8\x94\xcbv\x181\x12UcQ\xdcY^\xf6\xec\xdeg\x8d-w4\xb7\xb2\x8f\xe8\x08\x0bo*A\xf2\x8b\xc4\xaf3\xbc@x\x06\xf1\xfa\x0b\x02\x1a\x19\x19\xa5\x832\x03\x9dI\x91\xb7\x83< \xfa\x93\xae\xc5\xab\xe4\x9e\x8f\xac\x86\xb8\xdf!~\xe8e\xa6\xae\x9bhv]C\x87s"\xf1\x17)\x8b({L\xa1\xc4\x9f%8\x8d\xf3t\xde\xe9n\x19p2\xb4\x89y\xf1r\x1d\xac\xe1ttAU\xdc*\\r\x8171\x81\xdf\x04\x15V\xd8[\x00\x8d\xb9:v\x08|\xea\x9f\x85:_\x04\xf7\xdcc\x1b\x94"\x90u7|\xc8\xc4\x95=\x7f\xf0%x\xbc!\xe2%\x13\x1c\xaa\x02\x80\xad\xaa_\xe7\xeb\xb8\xc6\xd8s\x98[~\x93\xe3\x03\x0b\xf3\xcc\x95c\xae\x87\xddXB\xc4V\xcc\xe7c\x1csAOt\xcc \x89\x94\xdb&\xe5\xa4\x11c\xa2\xa2M$\xcd?\x08\xee\xf9\xc8\xc4\xd7\xa3\n[\xb3\xdd\x87\x12\xfeTa\x8a5\xa6\xf2\xbeT)\xdb#l\x1d\xc2\x00,\xf7\xd0)\xe5-9\x8c\xd2\xedv\xeb\xd7\x9d\xeac\x9d\xc5\xde\xce\xe7\xa3\xdf\x84.\xb9k\xf6\xdd\xd8%\xa7\xf3\xa98\n\x8ep\x1e\xa2\xf5\xbc?\xdevQ\x06\xaf\xdf\xab|\xebj\xfe\xdcp\xcf\xc7\x92XsI\x9cz\xc8&\x9c\x1c\x82\x9a\x05B\x15\xa5\xdazm\x84\xefqR\xbc\xd5\xc7\x02\xd0\x0e\x83\xa1\x84\xd7Q\x02\xccL\x06\x9c\xa8C\xb7&<\xea\x11\x8b\xf8x\xaf\xe9\xfb\x14V\xec\x01\xa9\x17~\xae\xca\xc6p\xe7]\xbf\x05\xbc5\x1c>\x90\xf3\xb7\x18\xd0\xef\xec\xf6\xdc\xe7\n\xb8\xae\x1e\x14|8\x0b\x87z\xcb\x13H\xa6\xb6\x1b\xeb\xcc\':\x1c\xeb\xcbz\x04\x05@bm\xdbv\xe9\xda\x88\x80x%\xdbZ\xae\x876\xa5\x88b\xebAi\x87\xd8hJ\x83\x05#\x0e\xcdl\xce\xfa\x85(\xf01\xce\x8b\xcae\x84\xe4\x18m\xe8\'#\x9bW\xb1=\xf7k\x89\xa2(I\xfe\xc1\xfe\xe6j\x9c\x0e\xed\xd0\xf0\x88m\xec\x8biM\xc3>\x8c\xae)qF\xc0\x8e\x02\x89\x14\x01\xab\x81\x99\xeb<\'\xd4\xbc\x816\xd0\t\x98\xb8\x1e\xed\x0b\xf1\xd80\xb7#\xd8\x05\x83\x84l\xcat\xa9i&\xa56=\xa5\x84\x97\xcb\xb3\x89\xfe\xab\xd8\x9e\xafq*\x89a\xc4\xe3\xc2\x17\xec\xda38Q\xbaZ[\xa5o\xa9b\x8b\x83\x97\x88\xb2j\xdaN\xe2R\x91\xf1\x8a\x1edbw(\x8e\x14\xdaY\x1dv\xca,K\xc5$m.\xb6$\x1e\x9e\x1ctZd%c\x81(\xddQQq\xcdy\xe4X\x91O\xde\x9bz\x15\xdb\xf3\x91\xa2B\x14B@\x8f\xa0\xbcp\r\xf6z\xbc\xe7*=\xf0\xb9p\xd3\xa5ut!Z\x06\xb4\xb2\x1aj\xa0p[!\xf6\x14)\tk!\xe1\x80\xd0\x81y\x82\xf6\x8d-1\xd6Q.\xaa-G[\\u8kJ\'"\x8cUfX\xbc\xe1#E|2w{\x15\xdb\xf3\x11\xc0Ak\x8e\x8aC\x0f\xb7m\xf4\x8ely\xe9\xb4\xaf\x8d\xf4\xe0\x04\xa0;6m\xd1\x8f\r\xca\xf9\x8a\xda\x95$\xd4!Q|\xc8\x88\xacS\x04\xfd\xb8\xb9\x0e\xc4(\xd7P\xa2.t\xd5\xcd\xf3.\xee t\'\xe1\x9c\x9e\xf4mN\xd1\x03.\x1c\x007\xfc\x16Q\xf2s\xb3=\x1fK\x1f\x02\xc1\xfb\xbd\xcc\x07\x98,G\xdc}\x96t\xb9\xef\xd6\'\xa3\xcb\xfcvFs\x95\x04X\x1fT\xb4Y\xee\xddS\xd1\x12\xba\x9c\xbaS\x9b\xf4\x84\x8f%\xb5\xd6\xe9\xedd\xe6Y\x7f\xb8n\x90\xa1\x02\xd3\xd3\xf1\xbc\xc7,{{\xe2SRc\xf6FF?y[\xfaEl\xcfGR\xb3\xc6C \xfa\x18\xda\x00\xe3\xc8\x8b\xc4\x11\xcb\xb7\x14x\x9ae}\xa8\x12tv\x8c\x89\x8e\xac\x13\xb4-\x04/\xc5\xd0\x1a\xaf!\x87q\xa8\xb0\xe6\x02sM[\xcdj\x13\xb7\xfaE\xb2-BW\xe3\x88\xe26\x1d\x18k\r\xc6\x9e\x95\x008\xa2\t\xfd\xdc\xd2\x7f\x15\xdb\xf3\xf1\x89\x14\x89\xa0k\x00\xff\xb0\xc3]\x8bc\xe9l\xf7\xd2|6\'Nk6\xdem7\xf2\xa8\xd3\x9ep\x999\xc2\xfc\xd4mq3\x8b.\x85%\x10\xa0\x163\xca\xe6\x80\xe0\x8cpj\x90T\xe9\xb0\x0e\x18\xfdes\xac\x8cf\xeb5\x05\x06\xec,\x8e\x9c\xcf\xc6\x93\x11\xdc\xab\xd8\x9e\x8f\xab\xb9\xa6|\x18\xfcH/\xd6\xf2B\xf5\x80\xbc\x1c\xcez\xdcm\x8d\xfdE$M\x9f\x05\x83\xc5\xccw\x97\xd1.\'\xbe\xd9]j\x92\x9bJ\x8b;\xef _\x06N\xfb\xf0\xcc\x9f\xf7b\x86\xceRqs\xd5\x8aE\x8d\x98*\xc5k\x13\xc5\xa7\x04\xb6\x81\xee[\xe7\xd5\xcf\xcd\xf6\xdc\'\xcb]^\\\xb3\x9a\x87\xe0fkawRQ\xa5xC\xbb9\xd8\xcd\xeaobCfF\x85u\x18\x1e\xe9\x88K\'\xe1>O\xe0\xae9^\xab\x85\'\x07\xd1g:\x1f3\x17\xc5K\x07+\x9fq\xa6\xa1)\x8b8(u\n\x97\xbe\xe8G\xf3\x8e\xfez\x1c\xfeK\xb6\xe7\xe3>\xcc\xaf\xd9\x9e\xff\xf8\xaf/a\x15\xc5\x1f\x8fU|}\x80\xe6\x0b\x9b\xb82#jEp=\xdc\x9f\xd6\xbc\x94\xe7\xea\xeb\xb8\'~\xfb\x9f\xa3_vq\xf4\xd1\xe3\x7f<\xaax\xffj\xdf~|\rG!\x12\xffx6\xe8A\xfe\x99\xbe\xf5\xac\xa9kk\x99\x83hExx|6t\xe1G\xefj\xc0>\xac\xe1\x01\x17\xc2\xc1\x91^s;\xad\xf38u\t\xb6\t\x16oQ\xcc\xcf\x98\xcc\xcb\xa4\xf5{"(t\x18<4\xc9\xf1\xb7\x0f\xbb~\xfb\xadC\xb8(\x83\xab\x00F\xce\xae\xf8\xd5[\x87\x9c`\x06e\x0eZEq\xb5\xf8\xa8U\xcb\x94\x8d\x84h\xaf\xf1\xdd\xce\xe7\x84Y\x83R\xcf\xb5\x13T\x15\x0b\xcdB<\'p\n\xe9\x98\xd7\x93y\xaa&\xd7n\'?\xc7n\xee\xc9\x13\xad\x99\xaa\xa2mgX\x8b\xb0\x8f\xc0B\x89\xf2H\x8c2\xcd\xd8\x9f\x84\xc9\x10\xf3\xb5\xc9\xeb\xf7\x14!t,xJ?\xfc\x1e\xed\xf9\x17`\xd0:l\xa0oS\xfd\xaf\xdb}7\x84\x1e\xf0\x9f\xbfn.=?\xf6\xff\xdc\x97\xff\xf1\xca\x7ff.\xf1\xa3\xba\xecn*\xe2\xad\xa3/`\x81\xdd\xae\xaf\xeaA{\xa7n|\xf84EH=F\xa2\xd7\x84N\xb4D\x0e\xbf\x04\x1c\x8d\xc4\xc9o^\xfd\xf1\x97?\xbe\xa8Gi\xfc\xfa\xe4\x14\x03}\xbc\xcb\xf8\xe4\xa0\xbe|,\x9ex\xef\xc0\xa6\xf2\xc8\x9e~=\x11\xdd\xab\xd0\xfb\xff\xe8\xc3\xff\xfd?\x1f\xeb\xb4\xad\xfd\xfb\xc3R_\xbe\xdc\xbf\xf0\xc6\xb6\xbe\xf1[?\xff>\xd8\xd6O\xc4\x18\xbde\xa8\xf7\x90~\xfe!}c[ol\xeb\x8dm\xbd\xb1\xad7\xb6\xf5\xc6\xb6\xde\xd8\xd6\'n\xe7\x1b\xdbzc[ol\xeb\x8dm}f\xc4\xea\x8dm\xbd\xb1\xad7\xb6\xf5\xc6\xb6>m;\xdf\xd8\xd6\x1b\xdbzc[ol\xeb3#Vol\xeb\x8dm\xbd\xb1\xad7\xb6\xf5i\xdb\xf9\xc6\xb6\xde\xd8\xd6\x1b\xdbzc[\x9f\x19\xb1zc[ol\xeb\x8dm\xbd\xb1\xadO\xdb\xce7\xb6\xf5\xc6\xb6\xde\xd8\xd6\x1b\xdb\xfa\xcc\x88\xd5\x1b\xdbzc[ol\xeb\x8dm}\xdav\xbe\xb1\xad7\xb6\xf5\xc6\xb6\xde\xd8\xd6gF\xac\xde\xd8\xd6\x1b\xdbz\x86\x95\xc9\x7f\xfd&\xff\x92\x95\xf9]\x8e\xfc\x99\xb1\xad\xef\xdf\xb1\x1f\x84m}\xff\x8e\xfd l\xeb\xfbw\xec\x07a[\xcfu\xec\xaf\xd0M?\x08\xdb\xfa\xfeW\xec\x07a[\x7f\xc3\xe6\xf1\xc6\xb6\xbe\x07\xb6\xf5\xeb\x03\xf2\xcf\xb1-r\xdd\xc4H\x0c\xc6\x18\x81\xa5`\x92\xa3x\x88\xa3 \x90\xbb\x17\x19\xa6\x19\x0e\xa2A\x1cgi\x01\xe1\xd9{\xb9)|\xed\x10G\xd1$H\xc30B\xf1<\x86\xdc+y|[,\xc1`\x9egh\x94\x17(\x96\x02\x11\x06\xe5I\x94\x86\x19\x98\x87)\x9e\xe1\x11\x10\xa2\xd0{\xfd"\x88\x81Q\x90\xa7q\x0e\x160\x08\x01\x85{\xd16\x81\xe6I\x8a\xff\x8e\xd8\x16\xf6\xcf\xda)_\xfe\xa0l\n\x0c\xfdrW\x0f(\x84\xc2\xf0?\xc5\xb6>\xbdT\xf6\x07\xd8\x16\xc5\xad\xb3\x1aA\x19\x1e$xn}\x1df\xbd\x148\x81"w+\n]\x9b\xc9\xd1\x0c\xcd\xd14\x83\t8\x0f\xa1\xdc:\x11@\x0cd\x11\x82 \x9c!\xa8{\t\xb6_c[/\x10\x97\xfe{\x8a\xbf\xb1\xad\xfb\xa4B \x1a[_\xe1^\xc7x\x9d_\x04D\xae\xd3\x13_\x97$\x8b\xf1$\xbc\xce\x03\x16_//\xc2\xe2$!\xe0\xc2\xbd\xa9\xebz$`\x94a`\x81\x04\x05\xfa\xcb\x1fc[\x10\xc3\xb1\x0c\x85\x82\x14\x83\xf1\x08J\xd2\x0c\xc1\x08\\\xb8\x9a\xb0N\x87\x16\x02\xd2\n\x18x\x80Fr\xbf\xb0\xc6\x92\xaf/\xf5\xb7.\xe5\xcf\xcd:}\x8c"\x85\xdf\xe3\xdc\x07Z!\nXdG\x8e\xe6f#\xe4\xb0_\x1fBh\xce\x8b\xfa\xd2\x00{~\x8c-\x18\xc0\xe2\xc9\xec\xd5E,\xd5E\xaft\xbb;\xda\x1bT\x08\x94[\x8f\xfb\x9bK\x03v^\xa3yrh\xce\x01$Bs\x15\xba\'"\x1e\x9f,,\xf9*\xd6\xe9\xde\xcd5V\x85`\x92z\xa8\x9fy\xbb4\xa6\n\x1a\xf1\xb5\xe8\xac+\xdf\xa2\xe1h\xec\x94\x98\x1b\x15\xba\xa7\xb3u\x1b\xb0\x92.\xae\xb0\xc8\xad\xfdv\xb4\xf1\x92\xef\x03\x14\xb0\x8a\x0b\x00\x93&\xb8\x1dp\xe4\x96O\xf1|\x82\x97h\x1611\x9c\x1c/s\xf5\'\xd7\xc4\xabX\xa7\x8f\xa0w\xdd\xc4\xef\xee\xc6Cipg\x87_t\x0fh\x10\xc5vq\x06\xd5:\xab`\x89\xc5q\x11\xd7\xa1\x94\x9aF3<\xf6\x80\x0e\xbd\x95l\x9e\xe7F\x1a\'\x0ew\xdeX\xc6V\xa44\xbd\x0e\xa7v\xde\xc3nj\xe4\x96\x04m\xe8\x05\xc5\xf7\x9b\xab\xf4$\n\xf0*\xd6\xe9\xdeM\n^c<\x18{(,\x19U\t\x10m.\xd1\\\x9bm\x1d\xa4\t+]d\xa8\xda4\xc4\xa0\x81\x892\r\xc2\xa4\x80\xadKW9B\x1e:\xc0\x82"v0\xc7\x08\xabx\xfeZ\xdf\x00\x89\x98\x80*\x19\xc3\xaa\x07\x97\x9cQ\xa0\xa4\xa62\xcdzr\x17\x7f\x95\xeb\xf45\x85A\xc1\xf5\xb0z\xb8\x9a\xd5U;\x18a\xa5\xf0\xa703\xf2sbnLvQ\x17\xf34\xady\x99Y\xcc\xb8V\xe55R\xf5\xd0\x16#&\x9e\x82]\xd5\xc5\xca\x92\x96)\n\xee\xd2\x14\x1e\xd2:aj\xe8t.\xf4>\x04=%\xc6IP\xfcV\xe9\xdc\xef\x0c;}tsM\xa4q\x1c~8\x93\x85\\\xd6w7\xfc\x84\n\xd1\xcdR@\xa3[\xdf\xf1*\xf1Q\x86u\xb27C\x81\x02\xdd\x98Y-\xbc\x03}0J\t\xcb\xfb\to\xe8\x9a\x0c\xce\xbb\xb3r\\\x8eHt$\\\xf9PIL\xc5g,\xa0t\x8c~\xb1\x9e\x9c\xb4\xaf\x82\x9d~\x9f\x90\xfe\xa6\x14;\xad\x01\xfet\xabZ\xbb\x90j\xd3\x0c\xdc\xb3\xce\xa7\'{\xbc\x05\xaa]\xe4\x0e\x1b\xf7H\xbdM\x8d\rT{WA\xb8^\xb7\x9a\xbf0Nr\x1a\x8a\xb3\xc6\xf9\xa1\x9cR\xbb\x99\x06\x92#[DNb\xebL\xe5\x9c\xd2ou\xf3\xe7\x86\x9d\xee\x1b9\x89Q\x04E<*`]\xc9@\x11\xb4$\xfd\xde\x1e\x8b\n\x82\x86+\xd2P\x1bpJ}3\x0f\x97}k\xa0W\x9b\xbc\x82\xe2(0f\xc8\xed\x98\xb0O\xc2\xf3\xd6\xbc\xc0\xcaI\xcb\xb9*+\xba\x82\x96v\xd6i\xbc\x1d\x15z\xeer\x03!\xc8g\xcf\xab\x17\xc1Nk7Ab\r\xfa\xa0\xc7\x95o\x94)\r\xd0\xe9\x14\x9b\xadAhB\x0fn\xe3\xc3d\x1f\xc6\x8c-\x9a\xb61\xe8\xa6XTNV\xe1x\xccT\xc6^\x18\xdf\x80A\xb5\x0c\x02fMy!\x07-@\x94\xa8R\xc4\xcb\xfc#\x06\x0f\xb6\x93\xb2\xe7\xb2z6L}\x91\xebt_\x12\xe4\xbd\n6B=D\xaa\x04\xbb\x9eCD\x7f\x9a@z\r4\xe4,\xcb\xb35\xd3\x97v\xfc1\x0f\x0fB!\xec\xb6\x1a\xee7~\xbf\x9f\x1de\xbf\'{\xb6\xec\xba\xda\xaf\x14[\xd4\xeb\xe0\xe2\xc3\xbb\xabe\x84\xf1\xec\x04\x0b\xe0\xba\xb1\xbel+\xf8\xf2\xad\xa4\xe3\xe7v\x9d>NC\xe4\xce\x9e \x0f\xa38\xb9\xe0\x01e\xb3\x82\xe26\x89%%\rU\xeb7\xdd\x88\xfd)1\x0e\xc3\xa5\xbbAEoy\x8d\x8f\xe3r\xcbZ\xd8\x81:\x00\xdc\xb6D\xc6A\xe6\x97\xb8\x10Z\xf0\xeaz\x9a\xae\x91\x89i\xf6\xe8i\xcb\x9d\x9b\xba\x12\x9e\xc4]^\xe5:\xad\xddD@\x98\xa0 \xe4\x11y\xd8\x0eN$\xc1\xde|\xb2)g7\xb4\xdb\x80\xbc\xde \x9a\x99\xc0\xe62\xc6M\xc4B{\xcd\xb2G\xc5\xaa\xcf\x87.2/\xf5\xb2\x91g5\xa1ZG\xbc\x05\\U[\x05\xa5\xe3]\xa1\xf4\x8a\xbe\xcf\xfd\x19c\xb6\xaa\x13\xfc \xd7\xe9\xbe\xc1!\xeb\xa9\x8f"\xd4Cm\xf0C\x81\x9b\x84\xc2\xa1\x90vR\x0fe\xe3\xdf\x127\x1fN\xe0\\n\x872\xaa\xf4\x8b\xe1]g\xd9tH?\xd7\xa9\xedr\xa2\xce\xcb!\xdc\xba\xb7\xb4\xe0\xe8\x8bN\xd1\xe6\x8d\x88gh#\x03f\x8a\x8c[^\xb72\x11\xc3\x7f\x90\xeb\xf4q\x1a"\xeb\x8b\xe0\x8f\x05\xed\xed\x9e\xb5"x\xa1eD\xb4)\xb4a\x11\xddfL\x8e\x05\x1a\x14q\xb4\xba\x8a=Q\xdab!\x00Mj\x08\x05K\xaf\x13\xdc\x01J;:\x8a\xf8\\\xdd_Y\x9c\xe04\xeb\xe0\x19l\x15\xcc\xf5\x05v\xdd\xbc2\x9dOF\xba\xbd\xc8u\xba\x8f\xe2z\xde\xac\x03O>\x04\xc2\x17\xb31\xc5\xae\xf0l.\xb1v\xc70Q\xc4s\x9aP6=\xe8\tj)x \xb8\x84H\xddFjc6\xea.\x9c\xad\x16:I\xfb\x18myJ\xc8\x06\x8cg\x0e\xd7\xcdW\xc1N\x1f\xb7T\xef\x1a\x19\x85?\xc4\xe3\x86\xd1\x00\x0c\xd8\x88^\xa7\x9a*\xd8\x9b\xe9\xb2c\xb3\xba\xa1\x04Y\x0f\xe9\x1d\xb1\\\x14\x9d\xe8\xc7B\xbc\x9e\xc7y\x13&\x87\x14\x90\x84]"\x1d7\x1a\xa3\xee\xc8\xf2$\xd7\xe7\x98\xd9Z\xb1u\x16\xa3Z\x124J\xf5\xc7ou\xf3\xe7\x86\x9d\xbe.}\x1c\xc3\x1eI7\xadU\\C\nw\xd0\xb0\x1e\x88\xbb\x8bz=O\xeb0\x0c\x03b\x89,B\xec \x06Gb\xc9\x19\xebqSSA!\xe8B\x14\xed\xf7\x90G\xa6.\x1a\x17D7\x94\x8b\x8a\x82\xf41\x1aJ\xb3\xdf\xdb8\xdc;a\xff\x83\\\xa7\x8f\x14u\r\x8c`\x04}\xb8\x9f!\x9e2a\xb6\xc0\x98X\xce\x84 \xd8]=\xd1\xb8\xba\xbb\x01&\x9cP\xca\r\xb7\xf7(`^\xda\xa2\xebNi\x98\xccR6\xe3ri8\x95\x87\xef\xda\nk|\xed8\xe7\xd5\x8d\x06\n\x95\x0b\xb4\xe3H\xaa\xebn\xdc>y\'\xf3U\xae\xd3\xd7;\x99\x08B \x8f\xda"0\xe8\x99\xb6\xc6\xd3\xc3x\x99\x12\xb7U\xd8\xd1!\xf0\xb3s\xee\x89\x01\x96\n&<\x00\xfbC\x17\x8e\xa3Uom\x14\x95\xb7|\xc9e\xf5r\xb4\x0f\xa0z\xf4pU\x0e\xfb\xd4&\xd1\xad\x06T\xb1|\xde\x9f\x89\x8b\xdaAO\x06p\xafr\x9d\xbe\xde\xb6!\xd05cy\xc8\xdd\x80Cm\xf3\x08E\x02\r\x96\xdd\x84\xeb\x88\x16\x89\'\xd2C\xd8_cB\xdb\xfbK\x10\x1bcrqd\xf3v\x8b\x0b\x0c\n\x14x8\xb3\tY\x9f\x1c\'\xa4$X\xac\xaen\x05\xf8\xa9\xcb\xdf\xf6\xe7\x03\xc2\x1a\x87Su\xfa\xd6q\xf5s\xbbN\x1f\x9f\xbc\x91\xeb\xcf\x80\xe8\xc3d\x99\x0fN\x93\x87\x82\xe3\x81\xa4\x80\xd21g\xdb\x04X\x8cgq[\x9d+\x9cR\x0e\xc7\x14\xad\x81\xf1Xk\x07\x84?*.\x0f\x83{P\x8eje\x0c\x9a\xcc\x90F\xc2=\xe0\'!7kw\xbf/+n:\x11!\xb9p\xf4\xff\xc6u\xfa\xfaL\xd5\xaf]\xa7\xaf\x0f\xde\xbd\x85\x98o<\xaa\xfeo$\xc4\xfc<\xc5\xcd\xdf\xf5\xe2\xdfB\xcc[\x88y\x0b1o!\xe6-\xc4\xbc\x85\x98O\xdc\xce\xf7\x89\xffyO\xfc\xb7\x10\xf3\x16b\xdeB\xcc[\x88y\x0b1o!\xe6-\xc4|\xe2v\xbe\x85\x98\xb7\x10\xf3\x16b\xdeB\xccg\x96W\xdeB\xcc[\x88y\x0b1o!\xe6\xd3\xb6\xf3-\xc4\xbc\x85\x98\xb7\x10\xf3\x16b>\xb3\xbc\xf2\x16b\xdeB\xcc[\x88y\x0b1\x9f\xb6\x9do!\xe6-\xc4\xbc\x85\x98\xb7\x10\xf3\x99\xe5\x95\xb7\x10\xf3\x16b\xdeB\xcc[\x88\xf9\xb4\xed|\x0b1o!\xe6-\xc4\xbc\x85\x98\xcf,\xaf\xbc\x85\x98\xb7\x10\xf3\x16b\xdeB\xcc\xa7m\xe7_\xb1\x10\x8e\xbfY\n\xff\xd2B\xc8\x7f\xdb\xa4O,\xc4\xfc\r\x1d\xfb1B\xcc\xdf\xd0\xb1\x1f#\xc4\xfc\r\x1d\xfb1B\xcc\x93\x1d\xfb+\xde\xc8\x8f\x11b\xfe\x86+\xf6c\x84\x98\xbfc\xf3x\x0b1\xdfE\x88\x81\xfeg\xcc\xff\\\x88\xa1\x19\x9c\x05\x19\x01\xa2aN \x19\x8a\xe5\x05\x81\xc3Y\x14\xc2q\x18bh\x04\'\xd8{\xe5\x1b\x06\xa1\x18\x86\x170\x10\xa4HZ\xc08\x14\x168\x1a\xa6q\xfa^\x95\xf5\xdb\xf8\x01\xc2R<\x07\xdf\xebW\x11\x1c\xffQ>\x87\xa3\xd6\xf7\x828\x14#X\x08B \x94\x83\x08\x82Y\xffJ\xb3\x1cCr\x08N\xdeM\x05\x84@0\x10\x83\x04\xe8{\n1\xff\xace\xf1\xe5\x8f\n0\x10\xbf\x10\x10\x88\x11\x04\x05\xff\x19\x10\xf3\xf9u\x9d?\x00bp\x9e\xa7\x08f\xbd~\xcc:\xf8\x1c\x84\xa1\x14\r\xd18C\t\x9c\x00\n\xfc:\xf1\xd6\xab\xc5\x0b\x0c\xb2\xbe\x89@\n\x08N\xb3\xc4\x1d\xff``\x10\xa1\x04\x94\xbe\xd7p\xfb5\x10#\xd0\x1f4\t\x06S\xcc\xfdz\x92\x08\x7f/n\x89\x808A\xe2\x02\xc40\x04!`<\x01\xd1<\x8b\xa3\xec:\x1e\x14N\n\x14\x8c\xe1<\x08s$\xce\xf1_~5\xc3\xdf@\xcc}@I\x9eC\x10dm\x02\xc7\xde\x8b`\xd2\x8c\x00\xf3\x9c\x00\xa1\x0c\xba^\x94{mt\x0cf\x08\x86\x861\x01\xc2)jm.v7~x\x9a@1\x9c#\xf9/\x7f\x0c\xc4\xbc@s\xf9{\x80\x98\xff5t\xf1\x00\xc4|ak\x94a\xeb\x96a/\xea\xe0\xc2\xd4\x12\xd8\xd4\xa2\x94\x1a\xe8\xda]\x1d\\\xc3>\xca\x84\x8b\xe7\x18\xa9jbZ\x08G\xbcgO\x85\xb7nO\x9e\xb1\x03\xd9K7\x8404\xa8\x8e\x07k\xd7t\x8e\x1ca\r\xc3\x190>\x1ah\xb8\xed\x96\xbdS\xf7>\x12\xa2\x1eR,\xda\x9a\xab\x04\xa5v\x7f*\xfe\xe69;\x8c\xcfv\x1f?{O\x81\x02\x84)\xc2\x8c\x07}\x18\x1d}\xb8]\xd6\x83\r\x0f\xc5\x08\n\x97\xa2\xf7\xb9\xb4_\xd3\xc9\xab\x7f=\xb5\xfb\xad\xd1\xac)#\xa6n\xbd*\xda\n7\xcd\xa8\x0c69A\x1f\x07\xe4$Ik\x00\x84\xa9\x1c\x0fkKh\xbaI\xbdgsW\x16g*\xfbz\x80\xaa\xbd\x8b\xec\xb0P4\x86\xb5\x7f\x85\x8b\xd0\xb3v\xf4nk\xbf\x0e\x91\xbdS<\'7\xa5\xb1J\xe9\xa1D&&\xb8\xf2\xe0F\xe5g[\xed\xa7\xf4\x9ae\x17\x07\xdav\xacu\x9a\x07@\xf5\x87\x84\xdc\x87\x86tE\xf9\xe4 \xd3I\n\xf0)-\xb3I%D\xf6}\xfcH\x93>\xd2\x87\\\xe6\xf5m\xce\x01\x19\n\xf0\x1b\xc4\x1a\xdb\x8d\x10\x8d~\x0f4\xa5\x1c\xcb\xb5\x18%\x01\x9b\x9fwG\xc0\xbaF\x18>c\xd4^\xf0\xbb\xed\x11f\xa9\x05\x8f\x88%\x00R%\x82\xe8\xed\xb6\x81\x0e\x88\x0ex\x965\x1c\xcb\x1eP\xd1\xed\xc0\x1b\xd5\x97\xbf\xc8\xfa|\xa4\x80\x7f\xca\xfa|\xfe\xdd\xf9\x87\xb2>0\xfc\xed\xa2\xf3=!\x0e\xda\xd1\xa7U\xfaf9n\xbf\xa6\xe2\xcb\xed\xe0\n\xfcV$\xec\x86;nov\x8b\xdb\x96s\x12\x86\xca\xdchQ"\xce>\xbb(\x0c\x00z\x11q::\x8ea\x9d\x18\xc6\x93\xc9\xc4/\x14N\x80\xc8C\xbd\xa8|\xbf\xb5\xe2\xc9\'\xfc\xc8g w*p>8\xd4\x83\x7f\x10\xd9M\xdd\xd1\xdb\x19\xbf\x19#7\x04\x8b\xc4\xdeL`\xe4\xe5\xdbH\xba9\x13o\xcb\xfe\xd6\x1c9|fU\x93\x08\xc2}d\x1f\xd3\r}(\xdd\xb9Q\x9eE/^\xc6\xfa\xdc\xfb\x88C DR\x0f\x15\xf1\x02-9B\x06\xd9\xb7\x0b\x07\x06\\\x8b6t~b\x9c\x1d\xd2-7j?\xcd\x82\x97\x97\xa0\x91\xf1\xfbm\xdbvR\x98\xdf\x8e\x14p\x82\x1bq\x14de\x9c\xf9\x80\x1a[Q\x1d$T\x15\xae$l\x13\xa7H\x8b\xa0\xcd\xe7*\x0c\xfd2\xd6\x87\xf8\x85\x84\xd6\xf0\x8c \xd0\x87Q\x94:#\x17\xcc\xba\x8c\x8f\xa2\xc2\x0cjF\xa9\xd3-\xb9m\xf8\xb1\xa32\xa1B\xb6\x18\x177\xecv#\x8c\xda^\xdfl\xaa\x99\x00\x99K\x16\xe4\xc8\xc9\xa9\xd8\xa1\xe9\x0e\xa2\xc5\xde\xa0\xaeV\xf5\x9e\xdcUj\t\x8d\x8b\x89\xeb\xa4\x05\xa9\x8f\x82x\x0f\xf5a7\x13z\xd97\xde\xa2\xa54\xbb\x83o\x87\xdb\x15n"\xc4\x97*\xd0\x15\xf3\x1dx\x83\x00\xa8l\x00BIT\x00F\xfa\xa4\xbd\x91R{\x8e\x07\x13\xdc\x85\xe7\xbd\x19(\x9b\xc1\xbab\xf80:#y\xc5Xr\x80\x845\x80\xffA\xac\xcf:iQ\x0cE\x10\xe2a\x1f\xa7\x01\x87+G-\'B\xeaVp"e\x9a\x8b\xda\x8b\x8e\t\x11\xc2E$\xe5v\xb3\x88\xaa\xd6\xef{\x1aS/\xa5\x1d\x9b\x8eru\xae\xe7\x90\xc0\x92\xc1\xa8z\xcc\x9am\xbfE\x01Y\x0c\x12\xaa!9N\xb9\x1e\xa4\xe1\xc99\xfb2\xd5g\xed%\xb6\x1e\x05\xeb~\xf6\xa8\xd0\xc9\xebBopx11j\xb1\xf2\x1c\x8d\x98\xb8\x9aG\xe5r\xe6\x99\xe8\x9a\x01\xe9\xd4f\xdd\xc2%H\xd9D\xf0\xce\xd9\xc9\xeb\xd2\\\xb3?\x94\xfc\x83B\xdc\x84+\x05\xe2\x89\xf3\xd6@K\x05I\xed\xda\x99\x9d\n\x88FZ9\xe5p4#\x17s\x88#3\x84\x18Q]U\xeb\xe0o\xae\xe3&\x15BlH\xec\xe5D\xe9\xbcd\x9fE\x1c\x16\x88\x9d\xc5\x0c\xb7\x03\xa1\xdf\xae\x04\xeb}+\xf2\xf8\xc9U\x9fu\xe5\x13kh\x03\xc3\x8f<\xdb\x99\x98\xab\xc6\xdb\x1a\x01\x10\xe9\x9d4\xc9\x86o\x15\x02}\x9b\xee\x9f\xa7!\x91\xc72\x1b5\x04\xa2\xe8(\xe3\x00\x19;`\xd3\x19\x84>\x9f\xceN#N\xde\r\x96\x88\xf4\xb0\xc9\xdavS\x98\x1c\xcf\xde@\x86\x9f-\x8cx\xb2>\xec\xcbT\x9f\xfb\xca_\xfbHb\x8f\xf5a\xf7\xc7\x11\x1e!3;\x80\xcaYg=Y\r\x1a\x16\x16\x1d\xf8\x88-\xad\x89\xaf\x9d\xf0\xce-\x8a\xba\xcd\x19\xa0\xeb\xe9|)\xb0v\xdcL\xc6\x90\x1fZ\x0e\xbe\xd4\x19\x8d\xcd\xf2p`\xd2\x85\xd9x\xee\xa52\xed\x9e\xf2\x98\'\xa3\xf1\x97\xb1>\xbf\xbd\xb1\xf6\x9bm\\Wa\xd1\x04\xe3y\xf1\xa9\x12\x15bVB\x80\xcd\\Z\x10?\xe1B,\xae\x87\x04\x10\x18b\x96\x07\xf6\x8d\xdeM\x96\x05\x98\xfc\x99\xdb\xa5\xc0\x0e\x90\x8c}\x1e\xef\xd0\xdd"\xab\xebh\xcbG\x01\xbdD\xe2\xd5l\xe5\xf1s\xc1\x8c/S}>\x0e\xc35\x08F\x1e7\x96\xed@\xc1h\xd3\xb5\x94\x90hR0\x81\x88\x9d\xee\x8dD\x02H\x0f\xda^4p\xc1\xe0A\x88r\xa4\xa9\x13PM\x1c\x86q\xfc:\xd6tB\xdfc\x13(P\x80P\xce\xf2fq\xd0\x8d|\xa3)\xd5\xa6\xd0\xd3e\x8e\xbe\xc5$|o\xd5\xe7\xbe$pb=\'\x1e\xfdI\xec\x0co\xdcI\x8c\xb7\xb4E\xca\xfa\x054\x96!j1\x19\xef\xd1\x13\x06G\x82q,\xc2\x9b\xdd\x9d,\xce]\x8eI\x1f\xf8\xdd\x82;\x9d\xdb\xe4C\xaa\x8f\xce\xb9\xcb\xcc]T\x82\xeauoT\x9b\xd3\xd60\x84f.\xb6O\x866/S}\xd6n\x82\xeb\xb2"\x90\x07\xf3\x02/\x0f]\x7f\xe24\xee\x06;s\xb2\xf7\x0b\x83\xef\xf2\xdc2s\xdb\xbe\xff\xba\x11x\xba\xf1\xf0H\xc8\xdd\x8c\x99:d\xefT8\xda\x1a\x9d\x84\xeci-\t\x88uc\xb0\xe398\xf6\xea\xb6\x8d\x81s;\xdfP\';o\x9f\\\xf8/C}\xd6^\xaeg\x01\xb6\xc6\xf6\x0f+\xbf\x890\xf2t!\xcf\xf8a\xb3\x8d\xc5\xfa\xd6\xd3\xe3\x1c\x18\x00\xaa\xe1\x87$\xd6vG\xf7\x04\xd9\x85p1`n\xb7\x9e\xe6mG\\\x8b~g\xa1x\x17\x84\x9bm\x86\\\x115\xb1\x8e\xc7LN\x80\x08\xb2\x95XuQ@\xe0>\xd5a\xf82\xd4\xe7cIP\xe8\x1a??\xac|\x0b\xe8\xc7\x03#+g\x15\xbe\xe1:\x15\xcb\x8c\xd3\xf6\xa7\xa5\xf1\xcc\x05\xd2r@\x80\xc4\xf2RT\xeb\x1e0P\xe0E\x1e\xc5\tY\x03U\xb2\xdc\x05=*\x99\xcey[\x1d}\xd0\x02\xcbc\xc9IbP\x14\xcdB\xe6&\xfb\xdcdy\x19\xea\xb3FN\x04\x06\x11\xf8z\xa6\xfc\xbe\x9b\xaa%l#"j\xbcb(F\x8b\x9a\xd8\xf4z\xaa\x12\xc3\x10\'.\x16\xf6\xc5\xb5\x125\x1c\xf7\xbd\x89\r\xea\xc1c\xec\xe9\xa0\x00\xf0%p(\\\x11o\xb7F\r\xa0\x9a\x8e\x14\xfa\x9c\xa0\t\xac\xb9\x90\xbe\x9b\x16\xcd{2E}\x19\xeas_\x13\x10\x8c\xe0\xeb\xf2z\xd8\xe04F/ \xe9\xb0a\xb9%_\xd7\xfb\xb9\x1a\xfd\xb2Tt\x01\xec\xc9)\x94\x0ed\x9e\xd0\xd4!6\xf7]x\x0c\xb8\xc3\x15q:\x11\x06q\x9d\x11\x07\x85j\xf9)\xe5\xd4%\x086,*\x1eJ\x88\x964\xe2\xda>\xab1\xbf\x0c\xf5\xb9\xdf\x81CP\x14]_\xe6\xa1\x9bT\x97%\x16yS\xe4I.e\xfd\xc63\x8dkP\xfc\x91\xd9\r\xe6\x8c7#\xe0\x83d\x98UX \x1d\xa1R\x86/\x85\x1a\x97\xc2\xb9t\x95h<\x8b\xads\xf3+>\xb5\xaa\x8b\xecg\xa5\xa5\xf2U\x8d\x17H\xfdo\xaa\xfa\xdc\xb3\t\x02\x84\xee\xb4\xed\xc3\xa1_c\xbd!yTBx\x18\x14"\x0b\x8b\x10\xe8|\x96\x8dt\x03k\x04.x\x17~\x9as*\xcb\x99\x10b\x96\x00g%\x7f\xeb\x9f6d\xde\xec2\xafTGPW\x90p\xa7\xa1\xee4\x05\x12\x146D\xc8\xe8\xe2\x93\x80\xc0\xcbX\x9f\xb5\x9b\x14\xb1\xee\x1f\xc8#\xebC3=\x8e\x1f\xeb\xedq{\x19\xae\xd0\xc1s=\x0bpy\xe6\x922\xac\xc4\xea\xe7L\xebr\xeb\xb0a6\x17\xe9*\x9f\x92\x03\x04\xdd\x8a\x1egJ\xbb2\x98\x0c\xe9n\x1bwhvn\xa8\x83\xdeN\xd5\xfc\x03l\xcf\'?z2\xd1\x7f\x19\xebs\xbf\x9a\x18\x82\x91k\xcc\xf0\xe0\xf7:\x06\xa6\\\xf6\xac\xe6\xaay\xe5\xce\x17\x95\xd0\xa6\xebyb\x95\x0e-\x01.!\xd1\xb6\x83\xd4Vn\xf7%\n\xe2\xad\xbeS"\xd9F\x13\xa9\xf0(\'\t\xb7ry^#\xf4d/\xca\xb7\x8d:\xcf|C\x8b\xf9E~2\x05~\x19\xebs?\xaf(\xec\xbeU>\x9cW\xf2u\x87\xa5\x1e\xbe\x97n\xd1.\x1fNI\x89\xec-u\xdd\x0b\x8f\x9b\x92 \xe2v\x03\x06&\x14\xdb\xf2x\xd0\xf0^\xd3\xb8\xa2\x80\xc5P\x8a\xf6\xa3\xbe5Bl\x83l\xca:\x89\xe5t\xee\x8fK\x14\x01\xd5e\x0b\xb2\xf6\xf4\xe4\xd5|\x19\xebs\xbf\xa1AQ\x18\n?\xde\x7f\xdf\x1f\x0f\xa01\x16l\x97,\xad]o/\x1e\xc3\x96\x8e\xd7c\xa7\xe1\x86c\xca\x89\xbd\xf5&\xb0\xdd\x0e6\xdc\x9f[\xbf\x8eNp\xe8t\x9clI\x05\x93\xc8\x00 \x97\x1bu\xdf\x81C\x91\xd0\xe6\xce\x8b\x04\xa0\xd6XD\xfcQ\xac\xcf\xdaMd]\xc4(\x02>|b\x04\xa6<\x81\x94\x87\xa0\xac\x8eP\xcb\x95\x01\xba+c\x1a\xaaj\xb9\x13\xb6#k\xca\x11\xb5h\x11\xe8\xee\xe1\t\x90+\xb8\x97O\xdb^\xef\\wvM\xca\xbd\x82#f\xb5\x9e\xaa\xcb\xb6\x1al\xba\x18\x03bM\x9a+\x9a\xfeT1\xdc\xcbX\x9f\xafK\x7fM\x82\x1f\xa9\xab\xd0\x08\xedsT\'|\x0b\xd7\x12\x18\xf6G\xe3\xb6\x90\xd0\x94Qp\x1d\x868\x00L\x8a\xad\\\x8f\x03\xb7\xefT\xf5\x8e\xa6p\xd4\x11`Ss\x88\x82\x06\xdaS\xd8V\xb8^\x07\xe9vHm\x9dJJ\xb5Y\xf6\xb5\x02<\xb9\xf4_\xe6\xfa\xdc3\xfd\xf5\xc8\xbf\xdf\xeey0L\x16\xab\xc8\x14\xe7\xbcc\xbd\xe8\xb4\xad$q\xe6n\xb2\x96\xb3\xc7\x0c\x87{\xc6W&\x1c\xab\xce\x99D\xf8\xfbZ\xc5\x13\x89\xe9Lo\x0exF%\x81\xac\x1di\xca\xe8F\x1d\xe6\x9bs\x07\x8c\x89\xb8\xd7}\x9c\t\xad\xf2I\xb8\xece\xae\x0f\xf1\x0b\xb5\xeepw\x7f\xf2\xf1\xe3E\r\xe1\x80\x1d\x12\xf6{U\x17}\x19\xba\x06\xe6\x9a\xb8\x8c\x95\x06\xc0\xc7|\xb7\xe1r\xddN\xb7\x94\xba\xd7\x0c\xfe\xa0S\xa5-:\x8e \xdf\xd2\xd4\xd8I\xbd\xa3r\xad?\x03\x1bY\x1b(<\xaa-bC$\x02v\xd0\x9e\x8c\xe1^\xe6\xfa\xac\x93\x96\x84\xd7\xad\xf1\xf1\xb3b\xe2F\x8cV\xaa9~\x1c\x9e\xa9\td\x93\xcb\xad\x89rE\xadr\xa3\xd8\r\x05\xaf\x1c\x05F:\xdc\xe8\x10\xf3\x90\n\x9c\x0e\x07\x01\xf46 ;l\x80\xee\xcc\xd2H\xd78\xa1,\xdbx\xd3W\xe4\xf1\x0c.x(m\xf6\xdfJR\x7fr\xd6\x87\xf8\x85D1\x0cZ\xc7\xff!\xde\xf7&\xee\xc8\xc5\x18\x1c\x9f\xa8\xdaG\xd0Z\x05\xc55nSPG\xd1\x06r\xa2\xf6)\xb4+\xd0B\xdd`\xae%\xcc\xd0r\x19\x83Y\x06\xe7\xbd\xd2l\xc2*E\x10_\x8c\x16\xa4\xb8m\xd3\x04;\x1e.\xa7J\x1f\xd2\xe4\x1f\xd9\xdb\xbfd}>.\xfe\xafY\x9f\xff\xf8\xaf/a\x15\xc5\x1f\xcfU|}\xc2\xe6\x0b\x9bT2\xbb5\x8a`{*]\xc7(\xd6\x1f\xb9\x94\xe7\xea\xeb\xd8\'~\xfb\x9fkv\xd2\xc5\xd1\xc7\x03V\xffx\x9c\xf1\xfe\xd5\xbe\xfd\xf8\xda\x9a\xe3\x90\xc4\xd7\'\x88\x1ex\xa0\xe9[\x0f\xa4\xba\xb6\x969\x88V\x84\x87\xc7\x07H\x17\x01\x8e2\xaf\n\xcb\xd3\x149\xf5\x18\x899\x18\x8b\xcc-\\\n<\xb8v\xc8~\x1ba\xe1rB\xb5-\xb6\x04\xd7\x10\xf4\xe1\xd3\x1c\x9b\xe4\xf8\xdb\'b\xbf\xfd\xd6!\\\x94\xc1U\x00#gW\xfc\xea\xad\x95\xe5p\xb3\xc4\xf4\x10q\x82\xb6N\xea\xca\xdejbX\xbap(\x16\xbd\xef\x08\xa6\n\x15b\x840R|\xb2\x06\xd5\xaa{\xcf>-\x91\xedY\xbe\x10\xcd\xd6\xb5]\xe7\xbf\x97\xc90V\xec9i\xd0\x04u\xf1\x04\x81\xf5E\x01wi\x03\x9b\xca#{\xfa\xf5Z\x0f\x91\xf5\x1d\xe1\xa2\xf7\xc6\xdfN\xdbW_\x8a\x8f\'\xd9\xef\xbbm[\xfb\xf7g\xde\xbe|\xb9\x7f\xe1M\xaa}\xe3w\xbb\xfe\x8dH\xb5\x9f\x07\xabz\xfb_\xef!\xfd\xfcC\xfa&\xd5\xde\xa4\xda\x9bT{\x93joR\xedM\xaa\xbdI\xb5O\xdc\xce7\xa9\xf6&\xd5\xde\xa4\xda\x9bT\xfb\xccT\xd9\x9bT{\x93joR\xedM\xaa}\xdav\xbeI\xb57\xa9\xf6&\xd5\xde\xa4\xdag\xa6\xca\xde\xa4\xda\x9bT{\x93joR\xed\xd3\xb6\xf3M\xaa\xbdI\xb57\xa9\xf6&\xd5>3U\xf6&\xd5\xde\xa4\xda\x9bT{\x93j\x9f\xb6\x9doR\xedM\xaa\xbdI\xb57\xa9\xf6\x99\xa9\xb27\xa9\xf6&\xd5\xde\xa4\xda\x9bT\xfb\xb4\xed|\x93joR\xedM\xaa\xbdI\xb5\xcfL\x95\xbdI\xb57\xa9\xf6\x0c\x1e\xf4\x9bH\xed_\xe1A\xc7\xdf-\x9c\xcfL\xaa}\xff\x8e\xfd R\xed\xfbw\xec\x07\x91j\xdf\xbfc?\x88T{\xaac\x7f\t\xe8\xfaA\xa4\xda\xf7\xbfb?\x88T\xfb\x1b6\x8f\xbf\x85T\xfb\x8f\xff\xc6\xc8\xee\xd5D\xcc\xfe\xfa\xc1\xa4\xfc\xe3\x17\xd4\xd7\xd7\x89o\xd7K\xd9\xfd\x12\xde\xe6\xba\xab~\xd1\xfb\xa0\xb8\x84r<\xff\'\xff\x0f\xb0\xec\xbf\xdb\xff_\xffm\x98\xfd\x7f_H\xd7\xc8y\xd4\x8dn\x8c\xee\x9cd*M.\x8a+\x07\xb7\xa4\xd4,w\xd7n\xb7\xe7\x90\x94#\xa0\xdb\xdb\xfd}#\xbf\xd75\xa9\xab\xf1N\x82}A@\x98\xfc\xf2\xf5w\xe9\xff\xdfA\xde\xe0_\x8f\xe2\x9f!o\x18\x06B(F\xc08\xc8\xb0\x18\x89\xd3\x0cO\x90\x1c\x0e\x12\xc2\xfa\x17\x0c\xc3H\x8e][B1\x04\xcf\x93$\xcb \x08\x86\x934\x02\x82$)p\xcc\x1d\xb3\xfa\xf2g\x14\x0eA\xb3$\x8e"\x04\t\x83\xa4@p(\x0bQ,N0 \x86\xd3\x10\x0es\x18\xc80\xdc\xfa\x164~/\x7f\xca\xb08G\x0b \x8e\xacc#\xc0\xbf\x90\xf7\x07\xca\x1b* \xec\x1a\x0e\xb28\x84\xc18\x0f\x92\xb6\xe5\xbfGy\xfb_+V\x0f\xca\xdb_\xf5\xc2\xee\x1b\xc3\x9fza\x9f\x7f\x9d\xffX/\x8c\xfcf\xe9\xfe\xf0\xe2\x9cy3A\xb9\x98jf`\xdfr7\x88#\xf6\x88\x08[\xc4\xd6\x98\x86\xb8\xb9\xedoJ\x0ez\x87\xcaf$s:\xed\xb6\xe3%\xc5j\x8a9p<\xe8XT\xd1\xd9\xda\x8e\x19y#\xb6A\x03l\x1bU\xd8AO\x96\x9e~\x15\x17\xb6\x1e\n\xe4\xbdF\xf7\xbdV\xed\xef\xbb\x19\xa5\xf3\x8e\x1f\xabyI\xa4\xd3\x92\xa0m\x00t\x1bS\xd6\xb6\x07d\x17\x08\x17\x96\xf5\xb7\x96s\xac\xaa\xab\xb3_\x06\xd3\x16\x83\x86\xb1\xd5)\x81F\xc3\xe2\x12\xc6\x1f\x86\t\x96I{q2\xad\xd3LI\x17i\x0c8\x05O\x96\xd9}\x15\x17\xb6v\x93B\xc8?\x10\xa0D\xf3\x94\xb7\xdb9\xce\x81!+\xa73G\xee\xbd\xbeS\xe1\xb4R\x11\xb3\rbc\xa1\x19\xc9-K\xe4\xdc\x96R\x9al\x97\x8c\x17f0V\xf9\xc0\x80\xf3\xbc\x8dc\x8dVe\x05\x11-\xf8\xd4\xa0u\x9f\xa3\xde\x1e1\x9f\xac@\xf9*.\xec~)\xef\x1b5\x85C\x0fE\x0b\xe5s\xe4C\x83\x06\xeb\x17d+B\xe2\xc0\xa7\t\x0e\xed\xbb\x11i)@\xdeZ\x1e\x9d\xa4\xdc%<\x03vp$\xa3#>\x0c\x02\xba\xdd5J{[\xaeH\xb5U\xc1R\xc8\x84\xbd\x97\x19\xd7\x10:y\xc5^t\xd5L\xfbVU\xd6\x9f\x9b\x0b\xbb\x8f"\x86C\x10D=\x9aDLwA\x1a\x0e\xf1d\xfeb\xc9\x8d\xaf\xde\x92\x06vq\xe5,\xa6\x02\x99\x0e=t\xd5\x8b\xc36@2yJ\x13\x18f\xe0\xa38\xd1ZkE\xd7\x13\n]\x07v7\xdb\x94\xd0\xef\x15\xd0!\xb1\x1b\x9b\xc2$\x11W\xdd\x93\x93\xe5U\\\xd8=\x18\\c\nb\r~\x1e\xd6\xbd\x8b\xca\xfb!\xd7\x19Y\x17#\xa3\x17\x8b\x90vJ\x0f\xf5\nk\x1d\x15\xe0l\x15\xba{\x06,w\x16oh\x07)bk\x1f\xc8 4\xaa8\xe8wq\x01(>\x92\x9dc7]w\xb71\xb9M\xcbr\xc5\xe0\xfd\x91|\x96\t|\x11\x17\xf6\xfb\x98\xf7\xd7\xdd<\xaat&\xd3d\x96\xb3\x06\x8b\xa9\\=\x08\xa6f&\xcd\xad!K\x82u\xe4\xf6j\x14\xce\xc8*\xe5t\xba\xc2\xbb\xcb\xf5:\x91Q\xe5bH\xa4\xd4V\xec\x14\xd6\xb6\xb2\'\x99\'\xa6,\xac\x9c\xf40\xc1\x8d\xdc.\xc7o\x95\xf0\xfd\xce\\\xd8}\xd2\xc2\xeb>\xbe\xa6M\x0f\xddT\xd9=Z/@K\xccA\x8a\xcaM\x1a\xc6\x1d\x17\x1e\xc1\x84N\xe7&X\xca-q\n@\x8e<\x16\xb7\x8dg\x01\x15\xae]H\ra9)\xa0\x8b\\\xea\xa8\xbc\x15\x1cz\xa7\xed\xaf\xaa\x08\xb5,%\xfa\xb0\xc4%\xfe\xb7\xb4\x89\xef\xcc\x85}\\Ml\xcd"\xd7]\xf2\xe1\xb0\xc2\xda\t\x9bz\xa7\r\x83~\xd6v[\x8d1\xd6\xb9w\x12\xa1S\x95\xf8\x13v\x83.\xf4\xa2\xa7gN\xf4\xb7-\xe4\x80-|\xe6\xbc\xca\xc0MT\x93\xcega\x9f\xc3\x8b\x01\x98\xae7Qn\xb3T4=\xd5-R\xd0\xcf\xba\x8f\xaf\xf1\xc2>\xb6\xa0u\x13Bp\xfcam\xf2\xc7+mm\xdd\xe1\xbc\xabl\x97\xe9}\xe9T\x0c\xedP\xd6\x86\x8cl\xcfr\xebg\xd6<\xa1\xc7\xe3\x11\xca/,Q\x12\x89\xe2\xa5\xbb\x03\xe6N\xf1\xad\xb4\xc6\xa2d=\xb34\xe2\xe3\x90\xb0\xfb\r\xd2\xb9B\xb2&\x04\xc8\x93\xde\xcd\xab\xbc\xb0{7A\x1c\\C,\xe4\xa1\x940C\x02\xd1t\x81\xc5}\\\xa0Tm\x02y]\xa7qM\x9a\xccp\xbah\xf1\x85\xee\x0f\xb8R\xa2\x0c]\x88\xd7\xce++.3\x80\xa47\x98\xd6f\x13u\xec\xb2\xa2\x0e\xf0\x8d\xa4\xb9\x86u\xc4/D\x13\x939f}\xb30\xf4\xcf\xed\x85\xdd#\x1b\x90Dq|]G\x0f\x9e\x8e:W\x88\x17!:\x82\xc6\xa2\x1d\x82\x98G\xaa\xc6u\x0b\xe5\xf2\xba\xa3\xca*hK\x01@\x14%\xc1\x1cC\xce\xe9:G\xb2\xe9\x86\x91\xa7D\x82\xfa\xae\x9a\x80\x85\xb3\xaf\xc7%\x98#\xd1)E\xbb\xb0\xaf]0rOn\xe4\xaf\xf2\xc2>\x96>N\x91\x10\xf8h\x87\xb8\xe73\x82\xfa\x9d\xe2\x04A(F$(\xa9\xe0>\xe9iY\xe3\x1d~\xcb3\xd9\xf6\xd2\x89\xc7\xd4\x9f<\x04=\x85!Ny\xb4\x1bB\x1b}:\xc3\xeb\x01\x8cRI=\xce;\x9a>\t\xac$!\x99\xe4\xd7\xfb\xdc\xd9=Yr\xfeU^\xd8\xbd\x9b0\xb8nr$\xf1\xb0\xf4\x99\xdb\x8e\x9f\xfc\xd4-s[\x18!\x12j\xca\x96\xca\x8a\x02\x10-\xc5\xd2\xe8\xed\t\xcd\x17\xb9\x98\xfa[P\xce\xa9\xab3g\x07\nE\xdb\xab\xe3\x92\xee\x13\x06\xaee\x1e\x8f\xaes\xa6k&\xd8\x80\x87\xab$\xbb#\xc2\xfe{\x82a\x1fk\x02\x82`\xfc\x0f\xb4\t\xf8L\x1d\x84\xa4\x02)\x95\x9d\x1a\xc1\xb5r\x1f8\x8bj\xb0e\xdb\xa8\x0fu\'\xbd^\xce\x91\xae\x9f.r\xb1p\xd8\xd8\xca\xa1\x1d\x99\x86\x8e"\xc04\x92\x99\xee \xb9Y^\xbd\x91\xaed\xb12\xea\x8dj\x9a\xdb)}2w{\x15\x18v\xdf@\xd7\xc8h\xcd\x11\x1e\xd2\x1ap\xec\t\xd3\x06\x17\xe2\x88B\xf5\x82\xa9\xc2"^\xb4j\xd3]\x84\x89\xbe\x1e`\x11\xdf\xb5\xf5\xac\x16\xd0\x95\xb9Y\xa5\xdcnE\x82\x18\x05\xcc(o\x07lQ\xa9Q\xce\x9b\x83\xda:\xc7\x1b\xa0\xd3\x9bJ\xc7\x95M\x9a\x8cO\xea\x99\xaf\xf2\xc2\xee\xbd\\\x93\xa05;\x02\x1f\xfc\x00\x8b\x9c\x19Q\xe37>\xe6R^\'L\x92\x1fuv\x8f\x9e\x82\xaa\xe13\\F\xd5\xa4\xd5\xda\xe1\x10\xed\xf7\x15)\x00^\xb6\x9f\xf6j\xd5Q\x16\x8b\xac\x89\xfa~\x8a7\x13\xacN<\x13\xe8\xbb\xa6\xc2\x8f\xa0y\xbd\xda\xd7g7\xb8\x17\x81a\xf7\x95O\xe0 Da\x8f\xf7\x1b\xd2E\xb9\x00\xc2-\xb6\x9a\t\xe0\xd3\xb9\xd7\x8f\x1c\xd8\x91.-\x0b*\xacvg}\x13I\xc4\x06\xec\x83[\xce%\xfan+Sv\x9a#Jb\x1cp\x12N\xbba\x8e\xed\xdbe\xb0\xfd\x81\xd7\xe5M\xebb\xd1l\x15\xf1\xb7\xe8\x90\x9f\x1b\x0c\xfb\xc8\x81\xa1\xfb\xbd\xb9G\x1ex\xcbOW\xb4\x96.Z\x99\x0c\xedAI\x15\xe1\xb8\x17\xd2~N\xa1j#b\xe5\xe4:\x8db\x17\x8c\xdaFuE\x00\xfb\xc5?\xba\xea\x88\xef\x14\x9f\xed\x03z\x80T\\\xa7B\x0c\xe8\xa0m\xa6\xf9\x87"\xc2\xfa&\t\x9f\r\x84_\x04\x86}\x9c\x86\xeb\x92\xb8\xe76\xbf\xeff)\xfb\x8d\x05\x16I8\xf7\x8d\xbb\xf4{\xdc(\xbb\x8c\x8a\x85\xd6\x00E\x87\x97\xfcIe9=\xad\t(\xe4\xe7\xcb\xb5\xdd\xd75\x90\x8e\xa1\xe7\x13i}\x98\x893qF\xeb\x86<\x08\xc1\xc0\n\xe5\xf6R7&|}\x16y\x7f\x15\x18\xf6qs\n%I\n\x81\x1er\xf1\xc9\xc8p\xc5\xdf\x8c4\x9b\xa0\xa673\xd6|\xae\x8cq*6\xfb\x81S\xdb\x0bN\x93\xb7B\xdf\xa4\xa2\xed\xd0\xf1:\x95\x04\xc7\xce\x84\xf6\xe0*\x04#g\xb6\xef\x8bmI\xa3\xbb\x12`vzs\x1e\xf5^\xa46.\xf6,\x9c\xfb"0\xec#\x10^\xb3\xfa5%zLRm\x11RR\x1b\x1ff\x86\xc9\xb5\t\xdd\xdb#\xc3\x86>t\xdc\x1e\xa0\xc2k\xf2Nt\xbb\xa3SH\x1c\xda\x81\xfe(\x99\xc2vC \xd5-\x0e\x85\xb20\xb8M*o\xd4\xc4\x99\xf7\x07\x89\xde\x92\x0e\x89m=M\x17\xbeu\\\xfd\xdc`\xd8\xc7\x06J\xe1\xeb\xb1\x0f=\xac\t*\xde1\x972\t\x1a!8!&\xb0\x8b\xafzY\xc3\xc1y;\xc4^R\xf5\x968\xf3s\xb2+e\xb0\xe5n\xe9\r#\x1b\xc9\x08\xa7\x83\xb3M!\xb2G\xafv|\x85\xd5]vL\x10\xecT,\x1eSR\x90\xd8\x9e\x9e\x84s_\x05\x86},\xfd;0\x01\xe2\x0f\xb1\xcdfI\x86\xe3\xb8\x13{\x08\xb77|\x8eo\x07"\x0c\xcb\xd3\xfeH\x84TJgXrEQB\x12\x8a\xa5\xe0\x1bF\x0f\x8c\x84a\x07\xaeK\x0e.U\x03\xdc!\x05{O\x0b\xf7[\x7f;\x1f\xf9\xaa\xb8\x9d\xb7\xc8\x05~6\xd5\x7f\x15\x18\xf6\xd1M\x90\\\x97\xc4\xe3F\x9e\xf5\x07T\x8b\x17\x88-\t\x8eI\x05 \xc8Fh\xc8\xa5\xfe\xa6\x9cQ%\x15\xb6\x05\xd7\xeaI\x1b\x08&\xde\x1e\xb3~Y$\x1b\xe4\xb0\x0cg\x8c-\x7f\xd9\xa7\x93\x84\xec=$\x8a*&&\xafVT\x8d\xf1P\xb8\xbb\'s\xe0W\x81ak7q\n$`\x0c\x86\x1fN\xfd\xeb\xbcS\xe2\x86pN\x89\x0b\x1fZ\x87p\xe1\x13\x1c986\x18Rx\xf2\xe2\xf3\x12\x82\x10R\x0f-s\xf1\xd8d\xe7!\xd4\x9c\x14\xf8\xf6\xd0\xa5\x12\xbb\xb18C\x1dv-9\x00g\xa5]\xf8\xd0\xd2\x039e\x0b\xfb\xc9;\x1a\xaf\x02\xc3\xbe~\xca\x80# \xf8\xa8=\x1b\xe7\xc3fQ\xe3\x0b\xa6\xabD\xa0\xd6[\xf0\xcc\xe9M\x85\x84cv\xc4a\x05Yw-{X\xd8\xe9\xe2\x9e\x88\xab\x19))\xbf\x869\x1a\xb99\x1b\x10e\xa2Qq\x8d8\x80+J\x98\xda\xe8\xcd\xe4\xb7\x07\xb1\xb0\xbc\xf2\x07\x81a\x1f\x1b\xf9\x9d\xb7\\W\xe8\xef\xbb\xd9\xf3\xe1y\xdf;Hw\xf3.{\xe9\x12bG\xa4\xdc\xd68\xec\x12\xb9\xe0\x94)\xa6\xa5G\xf7z\xc9Z\xcb \xe6\xb8q\x87uC\x82p\xd5\xf0\xf7\xa8F\xdd\xb8\x1d\xd8_\x0e7kMC\xdcH\xb6\xaa\xcd>,\xcc\x0b\xf7\xb9\xee\xc0\xbf\n\x0c\xfb\xba\xc3\xad1\x1c\x89?\xa4\xfa\xf9\xd9\xb7}\xa1\x8cwB\xcd.\x14d\xdcN\xc2\x89:q:s\x1a\xaf\x91\xb1\x98D<\x05\x1d\x03%\xf4\xb5\xaaF\xe1|Y\x86Y\x12\x90i\xa8NF7\xdb\x82\xe3\x11\xc9\x94P\xf6!\xdb\x10j1\xde\xe6.\xce\xa0gi\xf0\x17\x81a\xf7\xc9\xb2n\xf8\x08F\x92\x0f\xa9~\x95lA\xde\xc1\x08\xb6\x8ag\xea\xb6n/\x84vjN\xac\xcerN\xe9\xf1`!G\xf14\xc1q\x86\xd3\x9e\xb3\xd1\xf2j\x06d"\x8eS`Od\xf1\t\xb9H\x08\xe9\x1d\xecn\xd9\x91\xc09fn\x03\xcf\xc0\xc5\xeeII\xebU`\xd8\xbd\x9b\x14\x82\xaf\xe7\x15\xfcp^\xb9\x10\xb1\xc0\xae/$\xa2(\x10n\xc2\xcc\xba\xb6\xb6\x7f\xd7\xf5\\\x98_\x8b`\x91\xa9\x9c`\xdc\xc4uF\xdc\xbdu\xd1v\xdb@\xeb\xce\x13\xcdvI\xefI\xcf\xd1oRN\xa91\xbf\xe6\xe3\n\xbe\xa7\x95\x9d\xc2\xdd\x94\'C\xd5W\x81a\x1f\x89\xc7\x9a\xcf\xc2\x10\xf5 ?:\xf7z8\xfb\xa1\xe8z\xec\xb8\xef\xbdn\x83t;8v\xf9=R\xda[\xc5@\xae\xbdZQ\xb7\x98\xbc\xf05\xba\xbdI0;\xaa\x98s\r\x82-&^\x97%\xc6\x97\xb8@\xeb\xa3\xcfh\x9bY\x03Om\x81\x82\xdd\xf1[\xfc\xdb\xcf-\x86}\xdc\xf9F!\xe8\xfe\xf4\xd6\xefGq\x84\xa6Ej8\xd2p:\x18\xb2}\xb1D\xe2\xba\x8eOKp\x00\xd3:\xc7\x8e\xd6\x91\xba\xcd\x18\xd2ni:_6\rX\xc7\x9e\x1c\xdf\x04x\xc3\xd5-\xb3;\x94\xc3\x89kL\xbd\xc8\x03.\x18\x19;\xb3<\x989\x7f\xbd\xfd\xf5/\xc5\xb0\x8f\xd8\xf3\xd7b\xd8\xd7\xa7\x04\xdfj\xcd7\x1e\x9f\xff7Rk~"i\xe1\x8dW\xbc\xd5\x9a\xb7Z\xf3Vk\xde\x0b\xff\xad\xd6\xbc\xd5\x9a\xcf\xdb\xce\xb7Z\xf3Vk\xdej\xcd[\xad\xf9\xcc\x1a\xcc[\xady\xab5o\xb5\xe6\xad\xd6|\xdav\xbe\xd5\x9a\xb7Z\xf3Vk\xdej\xcdg\xd6`\xdej\xcd[\xady\xab5o\xb5\xe6\xd3\xb6\xf3\xad\xd6\xbc\xd5\x9a\xb7Z\xf3Vk>\xb3\x06\xf3Vk\xdej\xcd[\xady\xab5\x9f\xb6\x9do\xb5\xe6\xad\xd6\xbc\xd5\x9a\xb7Z\xf3\x995\x98\xb7Z\xf3Vk\xdej\xcd[\xad\xf9\xb4\xed|\xab5o\xb5\xe6\xad\xd6\xbc\xd5\x9a\xcf\xac\xc1\xbc\xd5\x9a\xb7Z\xf3Vk\xdej\xcd\xa7m\xe7_\xf2\x19~s\x06\xfeK\x9f\xe1wq\xddgVk\xbe\x7f\xc7~\x90Z\xf3\xfd;\xf6\x83\xd4\x9a\xef\xdf\xb1\x1f\xa4\xd6<\xd7\xb1\xbfb\xa0\xfc \xb5\xe6\xfb_\xb1\x1f\xa4\xd6\xfc\r\x9b\xc7\xdf\xa2\xd6|\xfd\x1f\xff/\x191\xc8\xff\x8c\xf9\x9f\x1b1\x9f\xb3C\x82\xefPs\xc8\x9f\xac5\xfa*$\xe6\xdeM\x10\xc1\xd7=\x9a|X\x13\xa6\x97\xe8-~,n\xc9^\\\xe0\x925\xb1\xcd\xd5$Y\xab?\xdf\x06}}u/\xdbk\xb5|\x1b\xf8Mi\x851z\xe9#c:\x0fj\xc9\x02\xeb@\xd8\xe7\x89\xe1\xa4\xfa\xa4\xf8\x98/\xd9yBuf\xbb\xef\x81\'\xeb\xa8\xbf\n\x89Y\xbb\x89\x90\x04\xb6f<\x8f\xb6@%\xdc\xd0\x06\n\xa0M\n4I\xb5\xb85!\xce\x05\x8cw)F\xea\x1d\x87\x87fF_6\x88|)q\x82\xb2m[\x9f\x90\xa9\x12\xa8S3]=w\x04\xd1s\x8b\xed\xe0\n\xdf$\xc8\x14\xb4\x94\xe3\xe4fN[\xcf\xfaP/Bb\xeeWs\x9d\x01\xeba\xf5Xp\xd0\xeb\x0eG\x0e\xed\xb2\x1e>\n\xfd!A\xe9\x93\xbeET:\x8c5.\xe2\x93M\xbf\xce\x1b\xfe\x966r\x8d\xb6~\xef\x97\xe6\xa9\x8b\xfa\x12E\xb4\xe6\xe2\xdd\x91\t\xfe\x8aZ\xc1N\xc3\xc55\xb1\x99m(#\xf4\x8d\xed<\xd9\xcdW!1\xf7]\x9c\x84\xb15\xcb|\xe8%r\x00\x9b\xddUh$\x92\xbe5\xba{M\xc6\xa8\xa4\x0e2d$\x0e\xa6\xe50j\x0cj\xa0a\x94t\xe3g8T\xb0EAC6\x94b,\x85Y\xe2pYw\xefl*6P\xc1\xba#\x1d\xd0\xa8s\xe6\x0f\x83\xf4\xadB|\xdf\xd9\x88\xf98\xabp\x9c\\\x87\xe6a\xce.\x1dG\xec\xb3\xe00\xb9\x1b\x97t\xe1y\xf6\x02\x9e\xd3T\x82\xd4O\xc7\xbdW\xb9r\x1e\x07\xe85\xcaD+\xe4\xc6\xa6\xb9\x1c\xa1\xeb\xe9<\xbbId-\x950\x81(b\xcbC}\xc1\x1b\xa7\xdc\xefoE`\xe6\xd1\xb8{r\x07z\x95\x11\xf3\xb14!\x82"q\xf2\xa1\xf8\xbf]`\x03\rYM\x82\x9c\xcb|\xb7\xd17\x99;\x92\x80\xa9\xf6\xac\xd9\xe4X\xbb\'fV\xafg\x003\x08\xef\x145\x11\xb0Csr\xba\xe6\xd5\xec\xc4\xda.BH\x0f\xe4\xdak\xa6\x01F,*\xb3~\xe2+E&\xbf\x15y\xfc\xdcF\xcc\xfd\xd0\x87Q\x0c\xc5\x1fkSr\xd2\xd6O\xd7\xd5`0&!K\x06\xc7\xd0\'T\xb1\x99\xedI\xce\xcd\xdd\x02_z5-\x1b\xbd\xd1\xe4l \xaa\xa0\x1a8\xc4\x05\xaa\x0el\x94l\xa1sb6\xf6\xba\x9a\xea\\\xdc\x12\xbar\x92#\xdd\xac7\xb1\xf7\xa4\xf1\xf7*"\xe6c\xe1\xaf\xb1\xd0:]\x1eJF{\\J^B\xe9f&\t\xa2S\xae\x80\x037G\xd3b\x00\xb8\xce\xd3\tHs\xff\xb2\x067\nT\re\xef!d\x83\x05\x9b\xc9&w;i\x18\xd5\xc8\xae\x97\xaa\xda\xa4\'u\x8b\x95&\xa4\xa9;\x9a\xf7o\x8a\xd3=i}\xbd\x8a\x88\xb9ws\xdd:\xfe\xa8X|\r\xf9\xc7FwD\xa0\x1a\xc39\xd8P\x93&[\x91?\x1b\x1e#u:4\xdb\xc8\xfe$\xd5\xbe\x94V\x04\xccF\xac\xdf\xac\xe9\'\x8e\x8bU\x99\xce\xfe$\xe9\x16<\n\xa6\xd8G\xa8\x8b\x97\nV\xa4\x90pC\xba\xc3\xe7R\x93^%\xc4\xdc\xcfB\xe4~H\xfc\x81\x9a\xa4\xb1\x94\xa3\xa5GT\xe1\x0bs(]@\xd06\xdd\x1e\x87\xc4\x92\xd8`\x12u\xec\xc5y\xa6 ~H\xb96\xb9\xf0\x1c4d\xa7\x82/l\x9c\x01\x8f\x15\xe4\xef\xbd\x0br\x80\xb1\xb8Y\xd0\xb3d\x1ag\x97\xbam-8x\xb2\xf8\xee\xab\x84\x98{7Q\x04Z\x03\x1b\xf8\xa1 \xf5&\x88\x85\x9co\x1d\x0b\x0fn\xa2V\xd56\xc5\x1fd\xf3\x04\x01!\xad\x9b\x9c>\x85\xc1\x00A\x17P1\xb0M\x17W\xf1\x90\xc0\r\xc6\x00\x16PeDa3g\x88\x93\xe0\xc5\x03\xdbeg\x01x?9a\x84\\\xb1\'\x93\x9aW\x111\x1f\xf9)\x04\xad\xf9\x10\xf8PE\xddc\xdd\xf0l\xb2\xb9\x00\x98\xc0&\x0feP3\x8e\xbc\xa5;Z,_7\xa17\xb5Ft\xdb\x0f\x0c\x9e\xed\x14yv:\xaa\xd7]\xd4\x8c.\x9e\xb9I\xc2Zh1E\xc1\xf8\x936\xbb[.2\x08\xdc\xado\x10xy\xb2X\xfc\xab\x88\x98\x8f\r\x0e\xfb\x08\x85\x1e\xf6q\x9f\x8e\x03\x84?\xb6|\x9fB\xc6\xb4\x90\x9a~\x94X\\\x8f\x1a\xa4\x15\xa6\xcb"a8?^\xd8\x1d\xb1\xef\x1a\xa3\x15;n9qa\xaa\x19.S"\x82*l\x9d\xdd\xc8b\xb0\x891s\xd1\xfa\xe5\x90\xee\xfd\xcb|\xfe\\\x87\xe1\xab\x88\x98\xfb(\xe2\x18\x85P\xe8\xe3\xcd\x8ceG\x93\xc3\xdc\x0b\xa2,\xb1\xcd$\xb1\xb1\xc5\xab\xf4\xfeR@lzj2\x99\xbczU\xe6e\x90\xa2*\x11\xadF1|8\xbb6,-5\x94y\xf2\x81\x97\xf6\xb7\xcbY%6\x97\xa1<\x07 \xbauR\x16\x89\xf5\'\x97\xfe\xab\x88\x98{\xe4D\x910\x84\xba\xf9\xe0J\xe0\xa8\xc8\xed\xb3,\xdc\xab\x84\x98\x8f\x95O\xadQ3\x04?\\M\x94eT\xecZ\xdf\n\x08\xca\xf6F\x02\x08\xac9e\xb5\xc5\x03\xa4\xa7\x9f\x98\x00\x16\xbb\xda`\x80\xec\xbai\xd0=x\x85$JTg\x04\x05d\x83[T\x81\x93\xce\xad\xdb\xd9\x08L\xf4\xddm\x91\xf0\x1d3\xd9\xa9P\x7f.\x1c\xeaUB\xccG2\x01C8D>*\x14\xb8T]\x1a\xbaA\x18\x8b\x82\x10\x0b8\x9aD\x98\xcd\xb3=m\xcb\xcc\xaf7\xd3\xa2\x9a\xcd\xb1\xa4\x81D\x1e!\x16\x0e\x83\xf3\x85\x17\xe8u\xa8\xeb\xa1\x0f\xbcv\xa2\xf8\x04?k\x8dv\xccT7\xe6\x84\xeb\x11]v\x82\xf9\xe4\xed\x8cW\t1\x1f\xa9!\x8aR\x18\x84=\xa4\x86;\xdf\xda\x1c)j#\xa7\x97\xadR\xc3Ui\xc7-\xce\x18m\xa1\xe9\x89\x9e\xb7\xd8 \x06\xea\x1e\xb81\xbc\x82\xd9;\xbc,\xe5~\x17\x19\x9e\xce\xb5`\xcf\xcbF\xda\x99\x00\xcc\x9e\x98\x82\x0bv\xe16\xd9\x97\xe8q\xab^\xbe5Y\xbe\xb3\x10\xf3q5\xc1\xfb\xdd\x1f\xec!\xdc\xaf\xf7\xe3\xf9\xd2\x8b\xb5$\xc5\xb8\xef\x93>\x03\x02E\x80\xf8\xc5\x8c\xa2\x8c\x85t\xd8~{\x0eB]\x13\xec>\xee\xf2\x99\\\xf2M\xb5a\xa6$\x9f(\xa4\x0c\xe3Q\xb2\xaf\xc5\xe9z8\xf2\xc3\x1e\xab\xb3,s{\xd3.\x9e\xbc\xf7\xfe*!\xe6\xebMi\x12\x83\xa8\xc7\xd8F\x1c\xf0\x00\xa8\x9a\rR\xb6\x9es\xde\x17*\xba\xaek\x94\x16\xa1Ak\x0e\xfd\xf5\x1aP2\x02\xeeN\xe2nJl\xa5\x13\xd9\xcd\x8d\xc0\x06p\xbbI\xcdB\xcbo\x06\xbd\x91\x0er\x88F\xf1\x11\xf7a\xf2\x8c8k<\xd7<\xdb\xcd\x17\t1\x1f\x1b9tO\xf3\x1foN\x8d\xed\x99\x1c\x8f\x95\x15\xec\xce\'\xe3\xda\x80\xfb}\xb5\x9b\x93\xd3\xa6?o\x90\x8b\x91\x1ee\x886\x06\xf0\xa6^\xc8\xd9\xdeud\x7f\xacj`sH\x91(\x04\x16#\r\xb6&\r\x13D\xc3\xa4R\x83\x82\x8c\xbc\xa6\xb0\x9e\xaf=y^\xbdJ\x88\xf9H\xde(\xf2\xfe\xe7a\x0b\xba\xe8\x8a1Y\x14\xd9\xd6Z\x19l\n\x8f\x9d\x8c[\x80C`|\n\x02\xd1Z\xd4$o\x0c\x18\xd8)\x0b\xde\xdb-\xe6\xe2h\xb3/\xc6\x82\xe3\x94\x10\x9b\xb6D>hR?i&\x9c\xa7\xe9)\xd0\x16}\xdb3\x87\xf4s\x85p\xaf\x12b\xeeK\xffn\xec\xc0$\xfc\x90\x03o+t\xb2\xf6\xe6|%w\xf8%\x18\xb5\xfe\xea\x13a\xbe@\x12\x9d\xf5#\xe8Z\xf1"\x91\xa6\xd7\r\xfb+y\xa3\x10|\xe3\x9c4\x88+{\x9c\xbf\xe2\xba\x84\x12\x9a\xd1\xc3\xf3\x1eL\xdd\xfdH\xe9Rwf\xeas\x12=yC\xe3UB\xccGZ\x83#(\x84=\xde\xe3\x9b6\xa69#\xf9a\x1at\x8b\xca\xaf\x1e\xd3\x1b[\xa7\x87\x18\xa8\xcavH\x00Xg\xe7\xd2A*\xb7\x1c\xb9\x80c\xd3v\x19\xd3\x9d\x02Y!\x85\xce6\x14\xc2\x08WV.\x15\x9e\xbb,\x94\x80c:\x18<\xa9\xd9\x98*\x8dPn\x9fX\xd5\xba\x1f@\x07\xf1\xaf\xa3\xde\t\x1a\x14G]\x1f\x8cE\\\xde\x02\xc7\xe6\x0c\xc6\x90\x9b\xe6V\xba\xe7\xbe\xd6\x94\xf3\xf7\xbc\x8d\x1f,\xc4\xac\xa7\x02\xfd\x94\xb5\xd6\x9d\xf2\xa5N{\x96F*\x1b\xde\x9d\x05\x00K\xcc\xb8\'\x17\xf5\xbe\xbb\xc2\xfb\xa3W\xe9\xc6\xbc\xa7<\xa2\x902\x85\x86\xb2\x16\xc8ht\x97\t\xb4e-\xddQ\xb6\xf4\x84\xc8\xb2\x16\xbc\xee\xa0Sx\xbf\xb5\xf1\xad^\xb3,\xf9B\x0e\x0c\xc3\x7f\xb2J\xdd\xbb\x84\x98g\x98\xd4z\x02\xbc\xd6\x17\xc7\xc4S\xbbC\xda\x0bN\xde\xa0~\xdd1\x89\xdbd\xb6\xfcx\xe6\x00E\xbc\xe3\x9d#\\.<:\x0cH;(\xf8\xba\x9b\xe6T\xbc\x17/7r\xb7\xbf\xa2\xe6\x01\x90\x1eeb\xb4\x82\xd2\xdfp\x1b\xbd\xc0v\x89F\xf9\x8d\xfd\xac,\xf0\x1e!\x06G\xff\x02\xe1\x10\x82\xac\x8b\xe8\xa5D\x1d\xc4\x00\x16w\x12\xb8\x13\x1f\x11\x055\xe6-\xa0\xd4\x92\xcb\x17\x9e}FEFM\x14\xba\x82\xc0\xa1\xd4\xd2\x0bK\xa0\x9e/G\x97Q.\xf6W*\xc2#\t@N\tk\xccn|\x9a\xc43q\xbeQ\x9a\x96&C\xf2]\xec\xe7\xcf-\xc4\xe9_\xbd\x0b\x88y\x8e&\r\x91$J\xc0/+\x7f\xec\x1f\x87\x82\x9eOC~\xd70\xca\xb3\xd2\xd4\x82l\xff\x80\xe0\x05\x90#3\x1f\x1c\x8f\xb1\x926DtRy\xa4\xa0D\xfdT\xd5Luh\xd0\x8e&\x04\x17q\xe0\x85\x8de2S\x83\xc3\xb5H\x1f\x92\x07;\xe1\xfc\xf8d!\xbew\x011\xcf0\xd1\x8f\x94\x1e}\xa9\xa8\xda]\x0f\x8a\x87\x1f\x1c\x88\xb1\x1a\x1e\xcdX\xe4\xa1\x90\x83\x8c\xd6<\x8f\x03\xc1\x92\xf8\xfb\xe5\xa8\x8bZ/\x82\x11\xe8b\xe7\x84\xcf\x0f1\xb3G\x98\x89\x07@\xd4\xbd\xe7\xb3\xaa\x1d3\xb3q*\xacn\xcdfP\x0b\xaeL\xc4O:\x11\xef\x12b>\x8edr\xbd\xb3\xbcj\x18\xc5)\x88E\xa8G\xa4\n\xb3\x8eXv\xa5(p@3p\x00\xa12\x1c\x90Z\xdd\xa5\xa8z\xc4\xa6\xa5e\xa3%W\xa8\xe5\x06\xa7\x04f\xcc([\x98\xd8\x0cx\xc9\x03\x1f\x1d\x90\x00\xa7&f+\x9a\xcb\x1eW/\'?[\x19\xfbM@\xcc\xc7\xd2\\\x13}\x1a\x85_ \x87\xf3xDy\xd4\x12X\x19\xda\xa9\xe5\x19\x18\x03\xb7D;G\xeb\xef#\x81\x05\x0bH\xe2W!FSCa\xa0\xc5\xbe\x127\xa0\\\xc0\x12\x9e\x0b\xcd\xae\x0f\xf9$r\x15\x07r:\xa4\xea\'\xec\x8a\x1c\xea\xe3Q\x19n\xc3\xf7\x0e\xe5?7\x10\xf3q\xe6\xaf\x1b!FQ/\xbd\xc8\x9dkx\x8a\xcb\xb1\xbbK\x94\x9a\x89\xa6\xa0\xa5\x02\x05i\x11\x8f\xcd\xd9\x82jy#\xaa\xd8\xb1\xf4\xe8\x1b\xdb\xc3K\x93\xca\xa2[\xa9\xe1\x81\xbf\x1bR$\x05\x85\xdd*\\\xe0\xf3\xa0\x17\x85\xeb\x14\x1a\xce\xc4\xe2\xef\xa5\x07\xfbI@\xe1MB\xcco\x9f]\xfcm\x98\x8dW,\xbbI_\xd7\xda\x99<\x84\x8a\x0b\xed\xadAO:\xff\xee\xd7\xad#\x1d\xe2t\xba\xa5\x80\x08\x82~/\xc4\x8f\xdc\x1f\xca=\x9f\xf9\xf8#\xa8PY? H\xaeD\xed\x9c\xed\xc6\xea\xd8\xdeN\xdcR\xf3\xa6\x99\x9b\x9f,\xa8\xfa.!\xe6\xb9\xf2q\x14#p\xf2\xb5\xc8pn\x1f\xe7\xc6\xc6j\xc0C\xcd\xc2\xb9\xec\xe1\x89D\x11y\xa7Q\xf6>\xdf\xd3l(\xc4@t\xb2\xc7\xa9\x96\x893\x8ad\xf5\x9dQ\x14\xe3\xce\xbb\x87.\x95Gk\xf0k\xcb\xd0\xa4\xbcU\xb9Ga\x03\xce\xa6\xbb\x83\xa4\xb0\x88W\x15\xa6H\x80\x043\xf1|kmR\xd7\x88\xf6,\xdf\x1f_\xab\xbe\xf8\xbb\x8c\x98\xe7\x9a@Q\x12\x83\xa0\x97\xd30\xb0\xc0G,\x0f\x88tT\xf4\x9a1\x8c\x84>\xfb\x0c\\\x95\x0fY\xb0\x11\xb3\x9cA\xe9lD\xebIYv\x14\x9e\xf5\xf5\xa4P;\x1d\xc2\xa0\x8e\x1a\x1eXd\'iU\xde\x83\xd3p\xf3\xd21s\xe5\x82\xd4\xedE\x13>)\x0b\xbc\x8b\x88\xf9\xc8m\x10\x9a\xc6i\xfc%C\xac\x06\xf0\xdc\xcfwR\x9c!\xe0\xfc\x98\x02\x14\x89\x02vM\xf4\xd9\\l\xea\xc6\x8dC\xa9\x07Prgq\xad\x17\xa6\xc6\xf5r\xae\x14S\xe85^\xcbBz\xa4\x0b\xe7\x8a\xa3\xb5\x1aZv\x91+5\x96H\xf09\\\x13\xe1\xcf\x9d\x86\xef"b\x9e\x83\x89P\xe4\xc7m\xff\x85\x88\x99\x9d\xe00\xec\xf6S(\xdaG\x04J\t\xaa\xa4\xab\xf3\x18-p\xf2H\x01\xe9\x96\xba\x17Z\xb08\xc0\x98\xc9\x13i\xc4\xa7ZJ\xd8j\x99#\x17\xf0\xacz\xc2\x02\x01\xc6o\xb7\x00\x11\xf8n?OP\x05&9\xc4}2\xdf\x7f\x17\x11\xf3\xf1\x14\xeeyC\x82\xc9\x97\xdc\xa6\xed3\xf3n\x17\xc9\x88\x80\x8c\xa9\xc3\xfb8\xa3:\xcf\xb2\xe62\xc6\x1bs/\x9b\x8bV\x94\x83\'\xd2\xc0E\xd5\xe0\x8c\xe0a/OG.\xcf\xb1\x14U\xf9i\xf1\xd2\xfcH\x14E_3\x04\xdc%\x16\x9c\x0fJ\xcd}2\xb7y\x17\x11\xf3m\x83\xc3\x11l=\x12^\xae5J\xb7\xefo\x81\x85\xb69\x06zD\xb6\x08z\x82$\x99w\xb7v\x88\xd4)!|\xda\x0fJX\xcdg\xec\x96\x05|t\x0b\xd1KG]\xedG{\x15[&\x1btd1\xd7%L\xe0\xb8\xe1t\x17\xbe\xe5\xc7\x94.?\xf9x\xea]D\xcc\xb7\xe76$\t#\xaf\x7fc\x84\x08a\xa7b\xdc\xa0\xe7\t\x94\xc33\xb7o\xdc{\xbbo-p\x10s\x0f\x06\x1fX\xc9\x85\x0c\x06\xf9;\xe8\x8e\xde\xe6}w\x05\x82\xdc\xb0N\xe2!\x15c\xe7\x9a\xda\xbc|\xbf\x8cW\xd4\xc4\xda+{\x80\xb8f\x1f/\xc9\xf7F\xf3\xcfM\xc4|[\xfak\xbe\x8f\xbfN\x164|0\x921@\xe41\x93k\xdag&.\x96[\xb6\xe6\x062\x89 \x0e\x18\x8d\x16\xc2q\xe1\x06\x9e\xee\xf6M\x05\xae;\x84/O\x15\xe9T\xba|\xe8\xaca\xb1&p\xc2\x9a\x16\xab\x18\x87J\xf6\x17\xa6G=\x9a\xfb\xa4\xf2\xf7."\xe6\xd7\xa5\xbfnq\xf4K\xbe\xaf\x9e\xa5\xf0\xd4\xd0,\xad\x0bs\xbcc\x12\x8f\x1bu\xdd\x8a"s$\x83\xabp\n!\x93C\x02*\xb8%\x97\xb4\xb2F\xad\xab[*\x0c\xf6\xa7P\xcd\xcf:wB\xe1\xd9\x1e3U\xdb;\xc9z\x95\xdbU\x87\x0c\xcf:\xf6\x93O\xa6\xdfE\xc4<\x9fO\x91\x18I!$\xfe\xb2\xc3\xc1i\xa4\xe2\xd7\xaa\x04\x1a\xa0\xa1K\xf5Z\xdfu&\xe3\xe6\n4\x89\xc7y\xa7\x9f\xdc\x02\xbd\xfa\xc0r\xc8\x0e\xbbCF\x8c{\x81\xdb9\x84Y\xcf\xc5\x1d\xa1\xda|\xa7\xcb\xd0\xf9\x1a\x92\xa0\xc2\xaa\x07T\x16\xa9+\xcbQ\xb7O>\x9fz\x17\x11\xf3\x91\xa9R0\xbcN\xfb\x97I\xdb\x9f\xaa*\xa5\xdbD\x07\xe2C\x03y\xfb\x07\'V\xa8n\xecG\xef\xf1\x10\x0b\x17) \xf1,j\x02\xbd<\x1e3\xb0s\xbb\xa3s\x830\xdd)p\xf7:\x83VL\xba\xb8"\xf1K\x08\x98\xael!\xd1\xbc\x985%|\xad\x077\xef"b\x9e\x93\x05\x81(r]]/8T\xef\x9e\x90\x0e\xa0\x83\xf8h\xd0\xb050\x9dQ%B\xe7%XZ\xcd\xa9O\x0b\x87\xf5\xf83\x0b\xe6\xd8\xeaA^\xc2z\xa4+\x90\xe16\x8b\xda&\x14\x1a\xe2\xb5&Wi\xdd\xf3\xd5\xcc\ny\xa81\xb24z\xd9\xb7%\xf1\x0f\x85\x98\x8f\x87\x93\x9b\x10\xf3O\xbf\xaa\xbe\t1?\xa5\x96\xdc\xc6\x99l]\xfa\xd5\xbbt\x13b6!f\x13b6!f\x13b6!f\x13b\xbep;7!f\x13b6!f\x13b\xbe\xb2\xbc\xb2\t1\x9b\x10\xb3\t1\x9b\x10\xf3e\xdb\xb9\t1\x9b\x10\xb3\t1\x9b\x10\xf3\x95\xe5\x95M\x88\xd9\x84\x98M\x88\xd9\x84\x98/\xdb\xceM\x88\xd9\x84\x98M\x88\xd9\x84\x98\xaf,\xaflB\xcc&\xc4lB\xcc&\xc4|\xd9vnB\xcc&\xc4lB\xcc&\xc4|eye\x13b6!f\x13b6!\xe6\xcb\xb6s\x13b6!f\x13b6!\xe6+\xcb+\x9b\x10\xb3\t1\x9f\xb1\x10\xfen\xc3\xfe\x87\x16\xc2o\xf6\xa2\xaf,\xc4\xfc\xf8\xc0~\x92\x10\xf3\xe3\x03\xfbIB\xcc\x8f\x0f\xec\'\t1\x9f\x0b\xec_\xf1F~\x92\x10\xf3\xe3G\xec\'\t1\xff\x86\xcdc\x13b~\x88\x10\x83\xffw\x9f\xff\xb1\x10\xc3#\x04\x8f\xb3<\xc3\x90$\xc5\xa3\x08\xc62\x1c\x82r<\x81\xd0\x10\xc9a\xcf:\xca,\x0cA\x08\x82\x90"\x85\x91\x18E\xa1O\x1a\x01\'\x05\x98`)\x11\x7f\xd6\x82\xf9>~\x80\x11,\x83\xe3,\xc4\x92\x1cK\x08<\xc1\xf0\x02\x01\xc1(\xce\xafy\x13\x81\xb1\xa4\x88<\xeb\xa1@\x02\xc6\xe1\xb8\x88\xc0(\xc9\xe34\xc4`8\x8a\x92\x14\x0c\x0b\xdc\x8f\x14b\xfeZ\x8d\xec\x97\xdf\xab\xbf@\xff\x85\xa2Q\x8aD)\x9a\xfc#!\xe6\xeb\xf3:\xbf#\xc4PkG\x13\x02\x86\x13\x18\xb7\x8e*\xfe\x1c\x01\x84\xc6\xd6\x162\x08\x83Q\x02,\x88\x14Cs\xcc\xfa3 \x92d0\x82\x12\x19\x81\xc2\xa0\xf5\xcb\xd68\x10\xec\xe9\x99lB\xcc\x0f\x14b\xd6Nf\x9f\x92\x05J\xac+\x91\x84p\x98c`\x0eEp\x9a\xa4q\x96Z\x7f\xc1\x82\xc0\xd1\xd0\xda\xdd\xd8:jk\'\xa2\x04\xc1\xae\x7f\x90\x11)\x84\xfaV<\xfcO-\xc4\xfc\xd3n\xc9\xdb\x84\x98g\x97\xfd\xa1\x10\xf3\xf5\xd7\xf9O\x15bp\xf4\xfbE\xa9)\xe5t\xb5\x9b\xdb\xe2,\xd5-\xb5\xbd\x9e\xbe\x13NDN\x92#0R@W\xd4)\xa3\xdb\t\xbf\xd1\xed\xc8\t\xb9\x88+WG;(P\xf2\xd8\xdf\xc0\x86w8\xb4\x85\xa5\xeb\x05\xbfjp\xc29:HX{_\xb8\x7f\xb2\xb6\xd1\xdb\x84\x18\xfa/4\x8c\xa3\x10\n\xc1/e\xcd\x9aX\xee+\n:dl[\x05\x99\x03^9\xe2\x07\x0c\xe8\x10\xe0r\xa9z\x1d\xce\xef\xc3\x1e\xf2},\xb1\xcb\xf3\xd7\xaa\xa8\xfa6#f\xdd\xc8\xd7\xcb\x12\x06\x93\xc4K}\xf1$\xd1\x8e\xec\x02\\\xbd\x13\xdfLr\xcc\x8ec\x17\xdf\x1f\xb6xl\x8a\xe5,\x94^S\x05\xd0\xbd\x91\x05\xe0j\xb4\xfb\xeb\x9c\xc2\x03@\x1a\x8c\x8f\\\xe2\xd2gF\x81\xe8\x85D\x99\xc2\xa2\x80$/\xd8\xe9\x1eM\x80\xf4\xf0\xd9\xf3\xea]F\xcc:Y`\x18%\xd7U\xf42Y\x02\x8fR\xf8Nt\x86^\xba\\<\x86\x1ePd\x94/\x92\xd0\x0c\x80\xbf\x9b1\x83\xf2K\xd6\xd8\xdf9\xeb~\xeb\xf7;\xd9\x9f\xae\xd9\x11\x94\x8aX`\xf1\x19\xc1\x8e\x1d\xc6tD8\xbazg\x81e\xd8\xda\xc4\xcdX\xd0OVT}\x9b\x11\xb3\x8e\xe6\xfa\xb15\x93\xa1^\xb2\x0f;\x9f\xef\xc1\xf4\xc0\xd3=_\x13nQ\x1c\x13*\x85\xed\xdb\x15\xdd\x99\xc1\xba\xe9\xdcK7;\x9b\xf4i\xbaH\xc5q\x08\xf5\x83,\xd4\xe0\xd5\x08\xc8G~98S\x9a\xd4\xb1)p<\xc8LI9\x94\x9c\xae\xc6\xfe\x18\x7f\xad\x1c\xeemF\xcc\xc7\x9a@\xa1\xf5n\xfbB\x0bP\xd3>s\xeaS\xd7\xa9-\x19_\x9dJv\x9d)\xb9\xe7\xe3\x99Q`Q\x0c"dw&\xbd[\x05\x99@\x81@\xed~\xc1\xa3\xa0\x1dG\xc8\x81uL\t\x85j\xea\xd3\xd3\xa5\xd9\t\x92\x92\xf9RQ\x14:\xb0\xd8\xfbO^j\xdef\xc4\xaca\xae\xc9\xd0\xba&\xe0\x97\xa5_\xc9\xeci\xd8\xb5\x8c\xe3\xf0\xbc-\x19\x00\xe1\x85B@\x0e\xcc0\x1a&\xd8V QE\xd2\xde2K\xae\xde\xd5Z-\xc9\xd2}G\xce\xae\xa5\x04~\x89M\x87Y\x01\x8c\xa3x\x7fP\xe9\x99\xabU\x843\xc8B_[\xfd\x93\x8c\x98u\xe9S\x04A<\x15\x96\x970-\x93\x97$\xd1\'5\xc9\x96\xc6\xb1\xa1IE\xdf\xa9m\x05\x89]\xd7\x8ba\x05s\xd4\xf1\xdc5K\xa9\xa7\xfb\x02\xd2\x0cI/\xc8\t\x86\x93\xa5\xe7\xd4\xa3m\xf8\xcb\x98+C\xeb\x87d\xcd\xe4\x9cc\xbb\x9d\xc8}6\xcc\xb7\x191\x1f\xc9\xcd\xba\x8dP\xe4\xcb\xa4\xf5\x0b\xf7\xb2\xcc\xe8\xf5\x10\x1du\xe9\xd2T\xd3\xce\xb9\xd6\xd1\xa3\xb2f\x1a\xf6\xe8\x91\x85\xe5\x85\xe40\xa8\x9b\x9brI\x8f\x0bT\xf7{f^\xa2\x9a\x0b\xa7\xc9\xb2\xef\xa2U\x8b\xd7\x11E$\x85\xbai\x92\xca"\xd5\xc2\xb0\xfc\x7f\xa6\x11\xf3\xdc@\x9f\x890\x8a\xbf\xac\x894\xb1\xef\x17\x1b6;\xfb\xb2{\xdc\x9c!\xc9Yk\xbe\xf5c\x8e_\xa8\xfb\x89\xc0\x82#L\xb9\xa9*zfa\x9c\xee\x04\x98\n\xb5eQ\xebX\x86\x82\xd8\xdd(\xfbL\xedU\xd57\xcd&\x8a\x91\x1d\x01\xb6\xdc<\x8b\x9f,\xba\xff6#\xe6\x19&\tQ\xeb%\xe8%\xe1\xf7\xc1\xe0\x06+\x8e\xe8h\xbb\xe8P\xca\xf3\xb9LTI\xd7s\xfd\xe6\x88CO\xdc\xc9\xd3\x85\xc9b\xfd\x81\xd6\xb9\x80)\xc7P\xdd\xd1\xd5\x13\xd1\xcc\xb3S\xa4\xc9\xd7\xfb\xc4\x15\x1a&\'\xb2\x01\x8cn\xaf\xf9\xd6z[\x9d\xbe7Y~\xb4\x11\xf3|l\xf3\xbc\xbf\xd3\xaf\xf7\x9a\xc6q\x8c~\x97\xdd\xe3\xe6\x96\x87\xee\x01f\xfa\x0b\x95\x9fG\xf3(\x96\xf4Y\x8dNme\xf9\xe8\xd1$\xb2\xac8\xb8M\xb3\x93`n\xaa\xe2\x83\xa3\xd0\xbb\xfb\xd1\x86=\x80\xdb?\xac\x83T\xe1\xaa\x85_\xaff\xd4\xed\x1a\xf9\x93\xe5\xe2\xdff\xc4\x93J\x9d\x93lc\x11&\x97g\x9f|\xa4\xf16$\xe6y\xf1X\xb3\xf7\xe7=\xe5\xb7a\x96\x07v\xdc\x1d\x95\xacZ\xbc+0)\x89\xe9\xacY]}f\'\x14e\xadT\x97\xf6Lbq\xc3P\xa4\x87\x92\xa1\xd8\x18\xe2ZWw\x1c)\xed{\xfdR\x1e\xdc\x9b{\xd2/Y\x0b\xec\x17\xab\xe2\x9d\xa3\x81`\x89\x08|o4\xff\xe4H\xccs\xe9\xafW\x04x\xdd\xcd\x7f\xdb\x8b\x80_\xc88\x8bP87\xb9mXc\xb7\tW\xea\xa1\xdc\xd1\xf5\x89\x86\xeeh\xbdnb\xce|\x11\xca\x0c\n=\xa2WQt\x00\xe4l\xb6\xaeg\xef\xa0\xbbd3\xa7\x94v\xe2\x83\xbb`\xdaC<\xa5x\x08\x18\x10\xf6I\x19\xeemH\x0c\xfdt\xfe\xd0u\xf5\xbc\xdek\x0eG\xb8\xdc]\xac\x87\xdb\x92\xb99\xc7iZL\x82S3!\xc9)\xfdU\xa4\xec;lM\xea0\xbb\xbd\xc4\x1d\x0c;#m\xa5\tj\xa8(\xfc\x89\x06\xfb\xfd\x02\xda\xb2\xd6z.\x1df\xbbN\xbf\xa8\xd12r\xf5\xe1\x93;\xdc\xdb\x90\x98uM`\x10\x8d\xae\xbf^\x8e\xe5\x8e\x82\xb5[\xe5\x9c\x1a \x1fE"\xd3\xaaG\xe0\xe3\x13\xc5\xb7Z5\xdeI\x80\xc4\xeb\xdcCRF\xec\xc7\xfd\xcd\xe7*\xafeP\t\x0e\x0e\xddz.L5\xb2\x1b3@\x92f\xb4\x81dk9\x8e\xc3c\xaf\xda\x9a\xf7\xc9\x1d\xeemH\xcc\xf3/T\x10\xfc\xf9\x8a\xd3\xcb\xa45&c\x7fT\xdd#N\x9d\x17\xa5\xe9u\x88b}%\x03g\xe9\x00\x1c\xbaR\xe3ZZ\x8b\x88\x88\xac\x83\x94\xdc\x8fJ\x07\r4\xeb\xf3\xe2\x05qr\x94\xca\x01\xef\x0c_\xa0\x1b\xa6\\x\x17Pv\x07\x1c\xb7\xf2H\xb1\xbf7\x9a\x7fr$\xe6\xf9Hc]\x14\x18\xf9*^C\xc0~Mej "\xda\xb3+\xbb\xfd\x898\xa3\xa5\x96\x93\x93$\xe11\x91,\x01`\x84\x90\x81\x1f\xefSE{d\x9dV\xa0$\xb3e\xddcw\xed\xc0[s\x8f2\x9e\xeb8\x08\xc5\x94\xa8\x05\xc4\x87\xfb@\xf01\xf7\xcf)1\x1f\x19\xfc\xa6\xc4\xfc\xd3\xaf\xab\xff\xe7(1\x7f",b\xf376%fSb\xfe\'\xce\xd2M\x89\xd9\x94\x98M\x89\xd9\x94\x98M\x89\xd9\x94\x98M\x89\xf9\xba\xed\xdc\x94\x98M\x89\xd9\x94\x98M\x89\xf9\xca\xfa\xca\xa6\xc4lJ\xcc\xa6\xc4lJ\xcc\x97m\xe7\xa6\xc4lJ\xcc\xa6\xc4lJ\xccW\xd6W6%fSb6%fSb\xbel;7%fSb6%fSb\xbe\xb2\xbe\xb2)1\x9b\x12\xb3)1\x9b\x12\xf3e\xdb\xb9)1\x9b\x12\xb3)1\x9b\x12\xf3\x95\xf5\x95M\x89\xd9\x94\x98M\x89\xd9\x94\x98/\xdb\xceM\x89\xd9\x94\x98M\x89\xd9\x94\x98\xaf\xac\xaf\xfcG+1\x7f\xf7C\xfe\xa1\x87\xf0\x9b\xed\xfd++1?>\xb0\x9f\xa4\xc4\xfc\xf8\xc0~\x92\x12\xf3\xe3\x03\xfbIJ\xcc\xe7\x02\xfbW\xcc\x91\x9f\xa4\xc4\xfc\xf8\x11\xfbIJ\xcc\xbfa\xf3\xd8\x94\x98\x1f\xa2\xc4\x10\xff\xdd\xe7\x7f\xac\xc4\xa0\xb4\xc8\xa1\x1c*P\xcf\x12l\x18+\xf24\x8a\xa0"\x0eq\xb0\xb0~\x8abP\x98\xa2\x19\x04\x82I\x0e\xe5\x11Vd \x98c!\x9eAY\x12\xa2h\x9e\xf8\x90\x1e\xbe\x0b \x90\x04\xc1R\x0c\x06\t\x10\x89\xe2\xf4\x13\x11\xa0\x10\x8cf\x9f\x85\x11\x9e%\xa0\x10\x88\xc3\x18\x9cd\x19\xeeY\x06\x99\xa6y\x88\xa0`\x9a\xa7\xc9\'\x84\xb0~B\xf8\x91J\xcc_k\xca\xfe\xf2R\x80\x81\xfe_\x10\xf1\x17\x04Gi\x02C\xbe\x95\xce\xfc\x9e\x12\xf3\xf5\x89\x9d\xdfQbh\x96\xa4\xd7\x01\x85Q\x16\xa2\x11\x9c\xe0Ham$*<\x99\x14\x8eE\x08q\xcdc1\x91Dia\x9d\x08\x18G\xa1,\x8ab4\xf9,\xcf\x8b?\xeb\x9f<\x0br\xfe\xad\x12C"\x02\x8c\xb1,\'\x10\xf8:K\x9e\xf5\xa0x\x06Cq\x01\xa7\x18N\xc4D\x9a\xe7Y\x18\xe5`\x86\\\xbf\x14\x82qr\xfd?4\xcb\xf2(\x8aS\x08I|\xd42\xfc\xaf)\xbe)1\xebwc1^d\xd6\xbd\x81\xa6ql\x1d\x19\xe4Y,\x1cbDv\x9dW\xebp\x10\x1fa\xa2k\xae\xf6\n|\x04R\x9d]U\xcbH\xf7\xfc\xa8y\xaa\x13=\xe4\xc3\xe3\x14\x16\x80nU4\x93A\xddH\x80\x97\xbd\xd7\xf9\x05b\x01\xfb\xbb\xa8\xb6`\x1f\xb0Ch\xde\xd6\x1d\xbbl\x1b\x0f\xdc\x07\xf7\xc6\xdd5E_\x9e\xca\xfdEu/\xf6\'\xaba\xbe\xc7\xf6\xf9\x16#FP\xc4:\x96/\x85\x05\xaf\xc5\x90u\xea\xd5\xcct\x8c2j\xb8\xba\xf4\xd1y\x94\xeb\xbd\xc6\xdc\xfa\xfa\x8c\x9by\x17\x11\x87#i\xeaDZ\xe3S\xa6\xb8\xdfz\x11&\xd1u!\x91\xd8\xcb\x82\xd8\xef\x93\x93\x98\xa23\x1d\xd5\xbb\xb4\x0f\x9d\x83\x95&7K\xa3\xa1r\xea\xf3\xcb)K.\x05z97.\xcdI\xcc\xfe\x9e\x98\xf0uw\xe8\xcc\x845\xe0{\x12\xa2L+\x1e\x0f\x92F;\x85qw\xb0\xe6 DU\xf0\xdd\xea\xfa?\xd4\xf6\xf9u\xb2\xe08\xbefO/\xa5S\xa7\xb9>\xf0\xe8zt\x0b\xad\xa8\x89K\xe4G\xc1\x00\xf2|\xa7\xca\r\xb6\x1b\x81\xb2\xf43PZ\x08\xce\xb1\x83\xc7\xae\x7fXC\x9c\x94\xb8W\xf9\xe9u\x10\xd1^`dm\x98T\x14)\xd9\xfaT\xc6\xa7\x9c\xf5&I\xfe$z\xf3\x1e\xdb\xe7\xdb\xf6\x86\xac\x07\x17I\xa0/\xa5\xbe\x1f\x876\x10\x0c\x96i\xe9v\x0ch\xb8\xa1M\x12\xab\xc1%S%S-X\xc0\xf5\xab\x1e\x15\xa2eB$\x81\xca\x96\xf6\xd2N\xf6Q\xea\x00:\xa7\x06\xdf\xf0\xe4*\x96\x02\xce\x07\x0e>\x92r\x8e\xc4\x95{\xb9-\x84\x9fb\xfb|\x1bM\x92\x86\x9eu\xb0_&\xed\x05W\xb4\x9e\xddA\xfb\x9d\xe6\x05\x8b~\x10\xf2\xaaY\x96\xeb-\xcc\xafv9xW{\xc0\x0e\x8b*\x92\xc0\xed\xe6^\xd8\xa9T\x07\xa1\'\xa5\x89\x99nf`\xc9QR\xf7\xa4\xd7<\xc8\x9d\xeb\x1fd\x04\xbe\\]\xd5\x0f>\x89\x19\xbc\xc7\xf6\xf96\x9a\x14\x82\x928D\xbfL\xda\xe5r\x9d)\xd2\xdc\xe3\xe5\xb51\xfd\x19\x93\xc9\xcbxK\xb0\x89\x9e\x1a\x172T\xc9;9\x16\xa8\xad\x89\xe1D\xf2j\x92\xa1E\xad\x14\x97\x161\xef^\xa1\xe8\xedu\xee\xa9:2\xb0\xe9\xda4\xfbr\x01\xee\xd0\xf1\xbe\xff\xeca\xf5\x1e\xdb\xe7\xd7-\x88\xa2\xe0\'U\xfa"5\x8d\xa4\x10V6\xa9\xf5\xe6i\xba[\x87{5\x9b\x13a\xdc\xe5\xdbYe\xed\xbbzv\x07J\x01v\x0c\xe9\xdfsh\xc9\xaa\x8c\x1a\xe6\xd4\xb8\xf0\xf9\x8e\xa126\x84O\xb1\x06\x9ez\xa5\x82\xbaS\x95e\xce\xe4\x99\xfb\xee\x93\xe7\xd5{l\x9fo\xa3\t\xd3k\xe6BQ/\xc723\x07\x15\xe9\x14\x8f\xa6\xdfw\x1c\x0f\x19a\x0e\x12F\tY1\xf0\xb0\x9a^;u$\xb13m\x8c6L\x02\x00\x9cyT\x1e\xea\xc5\xf3\x17\xf8,_Pmbe\xbb\x80\xef\xbb\x88]\xe8\xf2 \x13\xa0L\xcf\xf8\xec\xb3\xff\x81\xb6\xcf\xafK\x9fZ\xa7\n\x82\xbf\xf4\xe2\xc1?\xc1\xae\xf5\xa8\xcb\x93_\x80\x95\x10\xde\n\xf3 \x9fn\xb50\xc2\xb2\xd9\xba\xf4\x81\xa6{\x80\xf2\xf4\\B=\x9b\xeapF\x92\xf6\xd7\xdd\x840\xed\tP\xae\xb0N\xeb\xb7\xa5?6\xae\xd2\x8b0\xa7_\xf9\xeb\xe1q\xfe$\\\xf8\x1e\xdb\xe7[\x98\x10\x01#\x14\xf9Z5\x19\xb9\x85\xc7Lle\xe4\x04\x115\x02B\xf7\x9eH2\x1b\xbd\x8f-p\xe6\xc5\xc5\x06\xd5\xeb\x9a\xa9\x1b\xc3\x04FI\x84\xde\x8f\x0f\xac;]n\xa0^+\x0e\x03]\xc0\xddY\'j\x9b\xedb\xadl"\xbb\xc0\xc2\xebDf\x9f\xac\x10\xfb\x1e\xdb\xe7\xd7<\x15{\xa6p\xd0+a\x045\x1a6v)\xc1O\x92\x1e:1BB8\xbbS\xa9b\xc2\xae\x11\x0eg{\x9e\xf4\x8b\xb4\xech \xc2\x0f\xeb=\xe3\x01\xa1\x92\x84\xee\xf2\x0e\x80\xf7w\xe5\xe0\x977\xd2\xf7\x03n\x82}\x85\xaa{\x1f\xda\xd5\x85\xf1\xb5r\xb8\xf7\xd8>\xbf\x9e\x13\x10\x85\xe0\xeb\xfd\xf5%\x87\xdb\xd1\xfe\xfd\x88;\xec\xce2\x14\xfcQ\x1c\x8c+\x81.\x97[\xb2\x03\'\xb7+"\xbb\xc43\x80\xd4\xeb\x10-\xc3\xa3\xb2\x90\x94)T}qX\xaf\xb5}s\xea/\x06\xe9\x81\xa8\xb4\x83t8\xd7\n\xeb!\x12>?(\x9f\x84\x0b\xdfc\xfb|\x0b\x93Fq\x92X\xb7\x8c\xdf\x869\x9cq\x82!a%p\xdb>\x8ew\xfe\xed\xe4\x1d\xef\x03\xbfGc\x92\xba4DL\xa6G\xe9\x86\'\x04\xea\x80\x95\x8f\xc7\x9c\x15\xcfG1&ow\x10\xa4f\x9d\xe3\xdb\xa3\xe5\x88PG\te\x1e\x95s\xdd\xc3\x95\xf0\xddr\xc2?\xd4\xf6\xf9\x16&A\x90\xeb\xa1C\xbc,\xfd\xbfo\xef\x0f\xd7\x8e\x1a\xf3\x81B\xd5\xe4\xc3j\x9e\xb6p\xd6\x937x\x11\xdc\xd8;\xc2\xfb\x8a\x1e\xbc\xd8rQ\xd0\xb9}O/\xfb3\xdb>\xbf\xae\tr\xbd\xda\xfc\xce\xd2\xcf;\xd5\x1e\xee\x8b\xee/\x16?\x88R\x97\x1fK\xb1tC;\xc7\xf4\xd9;\x0f\x01Hg\x98\xc8\n0.\xa0\xbb\x81\xf7y+/.l4\x8b\xfb;\x18G\xbe\xb9\x1f\xfc\x1a\x91\n .\xd1\xcb\x82\x9f\xe5\xa3T\x9d\xd5O\x1ap\xef\xb1}\xbe\x85\x89\xc2k~\xf8;\xb6\x0f\xbe\xf3\xec\xa3\xa7\x98\xe6\x85\xde\x1d\xd7\xdb\xd3#\xcb\x01p\xe7\x11\x83n\xc4L56\xddx=\xba\xc3\x05\xa3\xaa\x19\xc3R\xe4 \xf8;\xc5V\xa3\xdd\xd2#\xb2\xbe\xb8}Y\xb44\xb5\xcb\x14m~\x0c=\x06i\xd4| ?\x9b\xf0\xbf\xc5\xf6\xf9\xf5\x96\xba.0\x8az\x05\xb6\xef\x1aX4\x18%\xa3p\xb2\x98\xe0\xd1\xe9\x1d\x08\x02\x06\xc9\xdas\xd0\x91R\xa0\x8b\x12i\xb7\xde\xad\\QF3u\x88r~\xbd\xa7\x1e\tm$o%F\x97\xa0\xad\xe9\xb1\xd7\x14\xf6\xb1\x1b\x08\xfc\xa4\xc8T\x8c\x01\xc4\'G\xf3=\xb6\xcf\xaf\xa7>\x02\x7f(\xf6\xbf\r\x93\xbd\x1c\xe6{\xcd\x82\xd8\x11\xc5\x91z\xc6f\x94h\xf8b\x00\x15\'\xabi\xd0\xb3$)\xcd=G\xb3\xf1\xf92$7l\x0e\xa4\xe3Y\x19\xbdb>b\xe9\x03\x13\xd5\x1c\xd7L\xafp\xce8\xa6=\x9c\xda\x01\xe9\xc6\xb4\xbfwK\xfd3\xdb>\xbf\xde\xf5q\x18z\x1a\x90/\xe8\r\x00x\'\xa9p/\xf4\x99\x1a\xa4\x93\xca\xb9\xf1.\x16\xf2\x01\xed\x157F\xc0)\x9f\x99\x9e\xbdOTn(\nFZ\xc0\xd1\xe6\x07\xef\xf1X\xca\xa5\xe9\xceZ\x80\xc2F\xb7\xd4\x05\xdft\x87uow\xbbYC*\xe3\x93Ol\xdfc\xfb\xfc5\xb9\x81P\x1c}\x95/\x92\xde%\xd6\x89\x00\xdd\xbc3g\xba\xear\x07\xaeF\xc7^)\xbf\xea\x1b[\x89O\xeb\xc5\x9e\xc7!w\xcd\x10\xdbc\xee\x9b\xdeM\x8f.L\x8b\\\xd91b\x96b6d*\xb5\xb5+N\x91\x15\x12\xc4\x80pun\x82\xf3\xc9\xe4\xe6=\xb6\xcf\xaf\xb7\xc3\xf5L\xc5i\xfc%\x13\xae0ew\x8e\xe6\xa5\x1c\x141\x04\x19k\xb03\xcdc\xa6\x06&\x85\xf3\xfdX,H\xcfE@\xb4\x1fwuY\xb0\xd6m>P\x85\x89S}\xd4\x93$\xa9e\x13\xe3\xce{\xaf\x92\xab\xab_\xe5Ye\xa0s\xed\xe7\xf8\xf7n\x87?\xd4\xf6\xf9u\x87\x83\xf15Y}}\xdcX`F\x91,\xc9q\x88N:1\xe97X=?FO\x93\xe2+\xd3u\xc1\x89\xe8e\x08\x12\x06^\n\xc9\xda\xd4\x19]`\x8e"\x95\\y\x05\xd7N\xfdz3\xef\x90\xd4\x84c\xc9n%\x8f\xa5\x0b\xc4x\x9cc\x06\xf8$\xd2\xf6\x1e\xdb\xe7[\x98\xeb\x06\x87>\xa5\x8c\xdf\x86I\x0e\xa7\xe00\x81c\xf5\x10\x07R\x17\x1fi\xb8\xf3\xa6\xa1K\x1f\x07s\x00\x0f3\x9fs\xe4\xb1\xc3qFMzLJt\x0fATk:xWQ\xa3\xc4\x96\x98\xa6S\x82\xef\xa9\xf3l&\xfb\xc2<\xe5x1&\xe7\xfc\x93\xd7\xb7\xf7\xd8>\xbf\xdeR!\x1aB\xf0Wwk\xa9\xdd\x89\x9cG.\x14\x00\x87\xcfv\x07\x98k:\xca\xab\x05c*\xc4+\xa6\x1e\xc6\x87\xcc\xee[&\xce02\xc7\x98\xc5 \xd5\x1d\xde`@\\\xc3{\'i\x04\x89\x13&RT\xf6|\x17\xe7zh\xe2w)M\xb9\xaf\x95\xc3\xbd\xc7\xf6\xf9u\xe9\xaf;\x05\x02C/\xbd\xc8\xed\xf2\xe9\xca\'r\x16\x15\x15mS\xce\x98\x08{\xef\x18\xdde\xe3 ]\xb4\x9c2M\'R\xa7\x16o\x96(\x9fLlv\x83\x84\x9bA\xc7\xef\x8f9_\x1ex\rI\x03\xe1\xe0\xef%\xaf\x8c\xcb\xe5p\t\xa4\xe8$}\xd6\x9d\x7f\x8b\xed\xf3\xeb\xa9O\xe0$\xf4;\x7f-\x95\xc3\x1e\xe2\xefK\xc8\xccZ{\x88\xc6\x86gx\xb2\xf3\xee\xe0@\xdf2\xd4q$\xc0\x12\xd8%\xc5\xf1\xdb\x8e\xc6\xcfKu\x933d_\x106\x1a\xa9\x178\xe5$\x05\xd5\xa5\x87\xed\xf4\xbd-,W\xf2\xbe^a\x99\xcc\xfdd\x0e\xf7\x1e\xdb\xe7\xaf\x0f\xa8hx]A/\xe7U=\xee\xa8\xe1\xae\xfb\x0f\x89\xe7:\xcf)\xe3xI%\x87\xbaO\x85\xa5\xb7\x87}\x08\xde\xd2\xbd940F%=\x8a\xf0\xceB\x083\x80Q&\x9dy\xe7\xc8:E.\xe9a\x91\xac\x18\xb1\xca\xb3\x93\xb0\x8fD\xb0\x80?\xa9m\xbe\xc7\xf6\xf9\xf5\x96\nc0\x81#/\xd9\x87\xc1N\xbe+\x02\x997\xc7\xd4\x00?\xee\x96\xeb\x08\xf5\xc4D\xe1a\xb9\xe2{`\xd2P\x13h\x02\xca\xc5\xc1\x8c#u\xf8R\x17\xdc0B\x9d\x11\x9b\xa0xRwR\xaaL\xccB\xdbG\xae\xd6M\xfa\xc8u\xaa\xa1c\xdf\x9b\xb4\x7ff\xdb\xe7\xd7s\x82\xa6q\n"^\xbc+\xf3b\t\xe4\x1d{$\x07i\xa2\x9d}\x0b\xbaw\xda\xe5Gr\xc0\xcetj\xe2\x08\x83\xe7Z;\xd3\x0f?5=Q\xca\xbb\xfc\xac\x98\xe7\xdd\xb5\xca\xb5\xe9hO\xee\x01 o\x86g\xc3\xb0|!h\xab\xe6]\xc7\x0cY\x9e\xf9\xa7l\x9f\x8f\xa7\xb0\x7fk\xfb\xfc\xef\xff\xfbK\xd2\xa6\xd9\xc7\xbb\x15\xdf\xde\xb2\xf9\xe5\xf9f\x08\'\x99u,\xb9\x97\xc07\xeb\xf5K\xce\x97S\xfb\xad\xef\xf3\xe8\xf6\x7f\xa6\xe82d\xcf7\xee\x9e\xcf\xa5>\xdei|~t\xbc}|l]\x1c\xe4\xc7\xc3\xde\xdf1\x82\xe6\xef\xbd\x95\x1axz\xe9\xa3z\x9d\x1c_\xdf"]D$-\xc36\xb9\xb8s\xea_\xa7T\xae\xa0Lf\xfbd\xa9\x89\xb8\x19PCJ\xf1dq1]\xc2\x97\xb8I\xa0\x08q\x1f\x99EM\x7f\xffZ\xec\xf7\x7ft\x82\xd4\x97\xb8\x11\xa1\xd4W\xea\xbf\xf9\xd1\xa6\x83[GT\x0f\x0cg\xc2bih\r\x9fu5\xdf\xb4\x83Z<\xda2k{\xcf\xd7\xf9\xb9\x81\xb0-\xfa\x1a\xfai\x904\n\x11\xd8\xecQ\xf7\xaf\x85\x03\x85\x8dk\x8b{\xdb\x1a`\xd3\x83\xf9\xb8L\t\xf51\x14\x8el\x16{D\xdfeN>G|\xcd\x1b\x82\xd3kb\xeaxe0\xa5\xb5@\x1f\x8e\xbf\xe5}\xfe\x01-\xb4v\x1b\x14y\xf4\xf87\xed\x0e%\xf7\x16zi\xb1\xee>\xbf\xf9^iVgy4|\xf7\x9b\xfd,*\xe4\x97\xbf\xbe\x01\xf5\xa9\xd0c\x8f\xaeRo\xfe\xdb!K\xd0\xf5\xa7!\xf5\x18\xfeuC\xf9\xeb\x00\x7f\xc7_\xfa\x17\xe6\xd5\xaf\x03\xf4\xff\xff\x7f\x1f\x8b\xe6v\x8d\x9e\xaf/\xfd\xf2\xcb\xf3\x03\x1b\x8f\xf5\x9d\x7f\xa7\xf3\x9f\xc3c\xfd\x89\xe0\xa1\xcdr\xda\xba\xf4\xebw\xe9\xc6cm<\xd6\xc6cm<\xd6\xc6cm<\xd6\xc6c}\xe1vn<\xd6\xc6cm<\xd6\xc6c}evj\xe3\xb16\x1ek\xe3\xb16\x1e\xeb\xcb\xb6s\xe3\xb16\x1ek\xe3\xb16\x1e\xeb+\xb3S\x1b\x8f\xb5\xf1X\x1b\x8f\xb5\xf1X_\xb6\x9d\x1b\x8f\xb5\xf1X\x1b\x8f\xb5\xf1X_\x99\x9d\xdax\xac\x8d\xc7\xdax\xac\x8d\xc7\xfa\xb2\xed\xdcx\xac\x8d\xc7\xdax\xac\x8d\xc7\xfa\xca\xec\xd4\xc6cm<\xd6\xc6cm<\xd6\x97m\xe7\xc6cm<\xd6\xc6cm<\xd6Wf\xa76\x1ek\xe3\xb1>\x03\xc1\xfc\xdd\xb8\xfdC\x08\xe67M\xfa\xca<\xd6\x8f\x0f\xec\'\xf1X?>\xb0\x9f\xc4c\xfd\xf8\xc0~\x12\x8f\xf5\xb9\xc0\xfe\x15l\xe9\'\xf1X?~\xc4~\x12\x8f\xf5o\xd8<\xfe-<\xd6\xff\xfe/X\xeaY\x14\xc2\x1a\x9b\x0f\xf1\xe2\xd7\x7f\xa0\xbe~\x9f\xaco\xce\x97\xe1/I\xff\xb8\x0e\xed_\x0ec\\\x9f\x93}\xf6\xf8?\xc2\xaf\xf8\xd4\x7f\xb5\xff\xff\xfe\x97G\xf5\xbf~\xa1\x02\xb3\x12\xb0 \xed\xd9\x83\xef\xee\xe9"?\xab\xc1>\xee\xf3\x8b\xee\x04\xcaM\x92N\t\xb5O\xc1\xc1\xf0\xc6\xe7F\xfe,Oqm\xa7\'\xef\xf4\x0b\n!\xf4/\xdf\xfe-\xfd\xff\x1c\xb0\x8b\xfc\xdb^\xfc#\xb0K@\x04\x8a\xa2QTDi\x14&iH\xe4(\x1c\xa2 \x12aH\x9a\xa7\x18\x94\xe69\x86c\t\x1a\xc6 \x0c\xc2EZ@P\x8c\xe4p\x8e!`\x82\xa4\xa8?\x06\xbb\x04V\xc4\x11X\xe0\xd6\xc3\x96\x15(\x12\'a\x0e\xa5p\n&\x05\x9a\xe0\t\x12\xe70\x98CX\x1a\x16\x19\x8aGPB\x14\x11\x0e\xe1\x08\x84\xc7P\x91 !\x82\xff\x91`\xd7_\xcb\r\xfd\xf2;UU`\xf4/8\xb66s\xfd\xdf\x1f\x82]__;\xfb\x1d\xb0\x8b\xe5`\x91\xa7\xd6\xf0\x10\x8eY\x1b\xce\xa0\x18\x01\x93\x84\x80\x93\xfc:\xce\x18\x86\xa1,\xcc\xaf\x1f\xc7\x05\x88\xc11\x8e$(\x86\x84XZ\xe0H\x01~\xb2T\xcf\xa9\xfa\xb7`\xd7\x1b\xd4\xa6\xff\x9a\xe2\xff\x06\xb0k]\xd1\xebd\xa7\xa1\xe7W\xf1\xa8\x80a\xa2\x80P,\xc9\xf3O\x0f\x0bA\x10\x0c\x7f\xd6nC`Q\x84\xf9g\xb9A~m\x08\x89\xc2\x88\x08\xb1\x90\x88"\xcf\xdd\xe9\x07\x83]k\x9f\x90,\xfa\xac\x1aG\xe3\xa2 \x12\x94\xc0C\x94\x88\xc1\x08-\xe24F\xac\xa3\xcc\xf3(G\xf2\xc8:\xbdD\x1a\xc7\x04\x04cH\x9c]\'\xc7:\xadH\x8e\xf9\xe5\xf7\xc1.\x82\xa4I\x92#X\xf4\t\x81\x11\x90H\x8b\x0cJ\t\x0cJ\xb2\x82 \xd04\xb4.n\x9ez\x02\x19\x1c\x03\xb1k/\x088\x84\x13\x8c@!(\xbdv\t\xfe\\<\xff\x1e\xb0\xeb\x9f\x06\x89^\xc0\xae\x7f\x95~z\x96\x11\xfaC\xfa\xe9\xeb\xaf\xf3\x9fK?\x91\xdf7\x91BdOh\xf7\x0b\xc0\x19\x90\xd4\xab\xcd\x8e%p\xd9\xd1g\xb1ct\xa0K\xf2\xe3\xee1\xc6\xa8h.{\xc5W\xce\xf8l\xa5\x0f\x83HO\x93F\xe9\xcc0\xe3\x1a\xdd7\x8cR\n\x0b.\xec\xa4vw\xce\xfb+^e\x9f,8\xfd.\xfai=\x15\x08\x04!pr\xdd1_\xea\xa5\x82Y \x17w\xdb\x9e\xddD]\xa8\xbdl\x88e\x1fV]K\x86\\w\x81\xf8\x88\xb5\x072\xa3\xf7\xcd\xa9\x8c\xb5\xe22\xf89\x13\xc9~\x0ef\x97\x91\xdcg\x8d\x9aR\t\x15L\xf2\xe9\xe2\xca\xa6\x93\xdf\x03\x1fl\xe8O\x96\x9e|\x17\xfd\xf4\x0c\x93D\xd7\xc9\xf8\xdb\x18QA\x99\xf5\xdb\x1d\xab\xf7`\xbf\xab\x87\x92;\xa4\xe5\x04\xb6\x84\x83\x8bwi\xcd\xb2\xaa\xbe!\xc8x\x1a\xfc\xfb$!\x02&\x12\xd38)\xf2]z\x94\x17\x83\xaa\xdc\x02%M\xfd\xb8\x14\xa0\x02\xccF3J\xae\xce\xde\xf2O\xc6\xf8.\xfa\xe9\x19\xe3\xfa\xe7\xc9uw{\xa9"zB\x14#a\x13\xb0s\xf3\xaa,\x94n`&0&,\x11\xa0\x07\x85\xf5\xd0v\x1f\xd9\xb5\n\x9b\xc9\xb1\'\xf7\xaa7Z\x08b\x1bgOf\x8e.\xb3\xa7\x06(\xa9\x94\xa8s|\x87m\xcd&\xb3\xee\xad\x90\xfa\xcd\x17+ \xfc.\xfa\xe9\xd9\x8b(\xb2v\xe1\xba\xf8\x7f\xdb\x8b\xe2\x99\x9e\xe9i\xbe\x1f\x9dio\x94\xf2 \xbb\x94\xefV\xc5.\xae\xd0Sz\x93\xd7\xf9\xe0\x13\xb7\xd4\x94\x8d\xc8\x1e\xfd\xf2\xd6\xd7x\xea\x8bG\xc7\x97F`\xb8\xb6\xf1\xa5\x05d\xed\xc8$\xc4\xdd\xbbx\x95{=\xc5\x85P\x7f\xd2]y\x17\xfd\xf4\xcc\x06I\x1c"I\x08{Y\x13\t\xac\x8fp}^\x86\x18\xba\xaf\xc1Bq\xa9\xf3\x17>\xd1\x14\xb1\x00z\xb9o\x83[k\x01\x94\xbeC\x08\xad@\xbd\x00\x8b\xd8\x98\x9b\xf2,\xe5\xaf\xd1H \'\x92\x8au\x1d\x1e\xfbcw\xf5\xeehd\xa8\xe2\xc3\xf8n\x91\xd2\x1fL?=\xc3\xc4a\x1c#\t\xe8eMD\x87\x90\xa4|Ew\xec\x83\xe7q\x1c\x19\'\xaa\x8a\x16\x02<\x14\nO.H"\xa1\x18e\\\xa04\x17/e\xbfKt\r_\xafv\xe9\xa0\xd3\x02\xf8\xd0d]\xb2+N:\xf1"\xa7\xe6\x8bk\x1e\xb5\x88\x0e\xf1\xe3\'\xcbA\xbf\x8b~z\x86I\xe0(\xb4\xdeC^\xc2\xcc\xd2~\xd4F;\x0e\x8d\xb1\xd6\xc1\x03\xd5\x07\xd6\xc9{\x08\x11^\xf7h\xd9\xe87\xaf]hL\xca\x0cV\\\xff3\x1c\xa7\xf6\xc1\x1dD\xa8\x7f\x12\x81x\x17\xfd\xf4\xdb+\xcc\xdf\x99HQ\xf5 \x1f\xac\xef\x9b\xb1\xa1\x86\xd2!\xa1\x1f&c*\x05\x98\x13\x1a\x86\xeds\xc0\x91fKw%\x8e\xc0]\x1f+\xb8(Wm\xa2\xa38\xf7x<\xddwh\x7f~\xa4\xd5\xb0\x93Q\xd8Ky]J/Ey)?\x19\xe6\xbb\xe8\xa7\x8fI\x0bA\xd8\x93 \xfem\x98\xb1\x1c\x0535\x1cy\x06\xca\xbdH\xd8\xf7\x8a\x88\xe83\x14\x07\x0e\xe32\x13!\x997\xbf<\x1b\x92\xcf\xe9\xba\xa0bm\xb6\x00z\xb8\'/d\x92\xc3\xa3\x02\xc2\xed\xcc\xf6A\x9f\xf2\xdacj\x8f\xf8\\\x1cm\xf0A}\xb2N\xf2\xbb\xe8\xa7\x8f\xd1$\x11\x9cZ\xbf\xe2\xb7aB\xbd\xe9\x9c\xc8\xf84\x19\xea\x10\xa9\x8f{-\x8ew\x8f4\xd2l\xe7Pzr\xf0\xe5\x9b\xcb\x802\x89u\x99\xc7d\x8e ;je\x08\x19\x05\x9e\x1f\xe3\xfd&B{\x12\x0e\xca\xb9-\xa7D\xed\xae\xa7A\x99\xef\x10\x15~/\xcc?7\xfd\xf4q\xea?\x85R\x14{\xad6\xcd\xef\xcf;\x922E\xfdb\x0b\x1ef>H\x14\x88\'\xb6i\xa0\x80\x0e\x12~\xeax\x05\x9a\x9c+\x80\x14&\x97\xa8f\xed\xce\x97\x13\x1b6\xf4\xb9\x81\xa7$\x02-y\xe6(O@\xfa\xec|\xd0q\xff\n\xce\x8b\xa2\x7f\x16Fy\x13\xfd\xb4\x86\xb9\x1e\x07\xf0\xba,^w8\x81\xe7\x8e\xb2\x17\x14ji\xba=\x8c\x80"\xa5p\xa5\xd7\x91iJ\x8egufb\xa1p\xaa0$\x8cc\x9d\xa2e\xbb?{\xa7\x94\xb9D\n\xcc\x1c\xe9S\xb16\xe74ZR>M\x11\xcd\xfa\x10\x94.)S\x9a\x9f\x14\xae\xdeE?}l\xe4\xc4\xf3\xd2\x84P/\xe7\xd55\x88O\xb33\xc6\x07X\xf7$"\xd6&HA+\xfd\xfa8=\x9c\x18\x96\xe6\xe3\xae%g\xe8A\x83]\xb0_\xc4\xa3_\xe8\xc7\xd8>B\xad\xb7c\xf7*"\x00\xf9\xf9\xb1\xb8`p\x0c\xee8\x9bvr/\x93\xa5\xc2\x7f\xa95\xf1.\xfa\xe9\xb9&\xd6\x9d\x05\xc3 \xfce\xb2Lg\x17>\xf1\xeb~\x12\x8c\x94*\xc7(\xef\x1f,\xa5$R\xda\x94\x95\xaa\xc4[]\xf1\x845y\x81\x1bnoN\xa8\xaepq\xd4\x9dm\xd7\x9dR\xfc\xa1\x8b\x86\xdd\\\xb2\xbd\x96\xd0\x8668\xfe\xa0\x82\n\xaa\xb0\xc3g\xc1\xc77\xd1O\x1f\xa9\xeaz\xadY\xd7\xfe\xcbd\xb1\xcc\xb6a\x91\x13\'\xc7\x1e\x0e\x1bW?\x05\xa4\xd8\xa7\x85\xb3y\xa2\xce\xa3N\xf3\xeeT!V\xb9\xcb`\x06K<\xa3h"+L\x1c\x92\xda\xc1\xe5,x\x07\xaa\xd3,\xc4RX\x9ck[f>p#\xca\x81\x90\xfa\xc9\xa2\xda\xef\xa2\x9f\xbe]Qq\x04!\xe1\x97\x1d\x8e\xd8\xf5\xd2\xe9\xb9\x1cb\t\x06\xda\xd4\x89\xaf\xb4~\x12\xc8\x13nQ,\x80?\x1a\xaat<\x1e\x97\xf46|\xac\xd9\xe8\x1e\xb8\\\xbc\xc7\xee\xb0\x87\xfb\xc5\x8f\x8b\xab9\xbb\x8d\xb1w\x11\x96\x84g\xae\xc6\xcf\x10\xd5\xe0I\xf7Y\xdc\xeeM\xf4\xd3\xc7qH#\x14\xf9Z\xf4:\xae\xef]M+K\x13\xf4\xb4\x1b4\x95\x122V\xa2\xa6% b\xeeU\xaa ]\xb8\x88\xa9z\xb2\xf5\xc48b\xbcm\xf5hnf\x05\xcf\x96\xfd9~\xe4\xf0\xd9\x19:+Y0\xee\xac\n\n\x87y\x1d<+\xfe\xd7R\x03\xde%?=\xe7\xcaz\x8d^\xd7\xcf\xcb\x8a80\xe6\xe1\x06O\xe7\xca\x03\x86\x9c5\xd5\xb4\x92\xbc\x1a\xeaQ&\x17\xc7~"g\xa3\x1d\x9a\xa4\x072\xd2\x8a<`\x1f\xd1\xedAh\xc2#>\x01\xc3h\xe4\x08\xd6\xb3\x89s\xbb\x10X!\x01\xa6\xa0\xcb\x8b\xa3-\xb2\xf8\xbd\xed\xf3\x07\xc3O\x1f\t"\xfa\xd4\x07\xe8\x970m\x98\xca\x14\xf5\x18\xd3>\xaf1\xd7\xa9\xf2\xab\x06u8\xb2\xdcU6r\x85\xa1e9\xba\x9cQcH\x1e\xef\x9b\xfc\xa4\xf0\xb0L\x1f\x95\xf6&\n\xc1\xd9\x9e3\xe3n\xf7\x86\xdf\'!\xf1\xb8R}\xa5g\x97\xc2b\x80\xcf\xfaVo\x82\x9f\x9e\x83\x89\xc3\xeb\x05\x15}5\xdf\x1e\xc3\xb2 .\x93\xdb,\x13V\x07\x83\xc4\xcf\xa0|"*\xb0\\(\x05\xdc\xf3\x0c&\xf9\x8f\x9e^\x7f\x1fL\xee\xc8b\xc6@\xec\x05\xe2\x02NN\x1e\xe6\xce\xd8`W\xd0\xd5r\xd6;z\x9a3\xddF\xca\xf0q\xb2\xfb\xe4\xfe\xf6.\xf8\xe9\xe3\xcc_\x13A\x84~\x9d\xb4\xb2U]Eq\x18\x94G\x04J8i\x9b\xe2\t= \xa19\n\xddB\xa9\xe7~\xa83\xbb3\x87\xf8\xe8\x1e\x9a\x11\x17\xf4\xf3\x03\xb4K\xd8\xf0n\x89pUC/\xb8\x1a\x9e\x08^ 5\xc7\xf3[uJ\xd8cR\xdc\xbe\xd6s\x9bw\xc1O\x1fk\x02[g\xdbz\xebzI\x10o\x17[\x01\x8a\xb4\xabG\x882\x83\x8b\x8b\xf32\x12\x9e\xbb\x16\xb8\xc4\xb9>`#@\xd1\xf8U&\xd1\xa9\xe5\xda\x98\x89\xb3 l\x11Z\xe7\xb5\xcb\xe9\xca\xec\x8b\x9a+\xd4k\x80\xee]\xfc$Z\xe7\x03\xd0\\m\xf7\xfe\xc9\xe76\xef\x82\x9f>N\t|M\xf6\xa1\xd7\xc30a\\>\x94\x99\x0b\x8e\x97&m6Z\x8c\x17\xcaz\xa6\xb16\x93HK\xaf\xab\xdd\x03\xf5N\x1c)\r\xfa\x8e\x9b\xac+8\x04q;\xf3\xebzW\xe7T87\xe7\xfe\xd2\x1f\xa9"n\x94\xe2bD\xb2\xd5\xf0\x9d\'~\xf2\x81\xc6\xbb\xe0\xa7\x8f0\xd7{!\x89\xbe\x8e\xa6\\\x03\x8f^\xd0\xd0\xe4|\xc7\xc0\x02\xe7\x1a-\xc4\x13x\xaf\x80\xcd\xb9\xba\xedd\x91\x9bn\xf2\x82\x9ayr\x11O\xfbF<\x17]\xecZ\xf7\xe2r\xa5<\xed*!\x9a\x1b+0Ej\xeb\xe2\xdf\xcd\x8f\x11q\x86\xf3:\x14\x9f\xdb\xc8\xdf\x05?}$\xaa\xc4\xbaI \xaf\x0f4\xf61\xc4\x82w\xe6\xe4\x16,41\x86\x07\xd1\xe7\xd0\xde\xe9\xc6\xe1p\xe8[\x01Dw\xf3\x11\x02\xbd\x1e(\x02!A\xdd\x99\xa6\x02\xe4Hjz\x94\xcer+\x89\x0e\xd5\xdby\x1c\xa95\x07O\xb3\x06\xc8\x17I$X\xe3\x937\xfdw\xc1O\xdf6\xf2g\x9aJ\xbf\x84\xf9@\xc6\xc1\x0fL\x8f8\x1bj\x92\x19\xb9\x0b@J\xae\x83\xf0C_\x16\xcc\xa3Q\xc4\x95\x07\xbc\x97\xf6\x04\xdc\xdf\x9c\x07q\x89a\xfa\x12\x80\xfa\xa0\x92]\xbb\xcff\x04\\\x94C\x89\xb3\x9e\x85\xaeC0[\x8e\xe1_\x97O\xdeQ\xdf\x05?}\x96>JS4\xf1\x8a<\xdf\x89\xce<$\x1c~"\x86\xe9v\xb0\x97\x18\xb9Tk\xba\xf30\x00\x01\t\x07\xe3\xacH\xe6\x8c\xdf\'\xd6u\xe5\xd6\xf15\xc2\xb5\xafA\xa0\x0f\x05J\xed\xd5\xbac\xf7\x95r\x06\x94\xf1\xb8\x9f\xafw\x98s\xb2\xfe\xbc\x0f\xdaO\x9ao\xef\x82\x9f\x9ea"4J\xd0\xd8\xebF~F=\xd5.\xce\x0f+\x91\x94.o.\x97K\xc7\xf8\x18\n\x95Y\xa73\xd3\x8d#c<\x1a\t&\xc6J\xaa\xc0\xf3n\x17<\xf8\xc3\t3\x0b\x0f\x88\xf8d\xe2\xb8\xa4=\xdf!\x00f\xa6\xf9\x86\xf2\t\xda\x0f\xd3\xa9\xff\xe4\xad\xe6]\xf0\xd3\xc7\xe5\x8d$!\x1a\x82_\xd6\x04\xd2\x8f\xc7r\xa2\xa6\xb3\xb4\x8b;\xf6\xaa\x03s\xb8\xb74n\x12d\xbeL\xf2\x93!\xac?\xa8\x88h\xaa\x82He\x10\x8eS\xb1\x17\xaf\x81*\xae\xa7\xe6\xfdQ8S\x83\xec;\x04\xc8\x0e\n\x7f\x94\xad\xab#\x116?{\x9f<\x96\xdf\x05?}\xcb\xe1\xd6YK\xfc\x0e;\xa9Y\x86x\xca\x99\x99\xf7\xbb\x9e"J\x12\x92\\FR\xea\xf3\x94\x02\xf9\xee\x06\xe5\xc8\xa3\xc6\xb5\xf3\xe1X\xb1H\x89Fm\x80L7\r\xe3\xc0\xe8\xf8x\x10Gkt\x973DN\xe5xw\xae\\\x06\xa57>\xf5\xf1\xaf\x95\xc3\xbd\x0b~\xfax\xa0Ac\xe8\xefe\xc2\x08\xbaLb\x1e\x90\x0c\xdca\x98\xdf\xef\x9a\xfd\x1e\x9cA\x89 \x00\xcf\xc2\xe6L\xd2\x85\xc1`\xaa\x9b\xd2\xcb\xf29Y\xcf\x04o\x82cP\xa0\x83e\x86v\xe7\xa8#\n\xe5\x11\x8a\x90\x94S\xa7\xe4\x0e\xaf\'\x91\x14\xac[\xd3?\x07?}\xac\xe5\xbf\x85\x9f\xbe\xbd%\xb8\xa95\xdfy}\xfe?G\xad\xf9\x13U\xb4\xdd\x8a\x04oj\xcd\xa6\xd6lj\xcd\xa6\xd6lj\xcd\xa6\xd6|\xe1vnj\xcd\xa6\xd6lj\xcd\xa6\xd6|e\rfSk6\xb5f\xbb\x90~\xb9\x0b\xe9\xa6\xd6lj\xcd\xa6\xd6lj\xcd\xa6\xd6lj\xcd\xa6\xd6|\xe1vnj\xcd\xa6\xd6lj\xcd\xa6\xd6|e\rfSk6\xb5fSk6\xb5\xe6\xcb\xb6sSk6\xb5fSk6\xb5\xe6+k0\x9bZ\xb3\xa95\x9bZ\xb3\xa95_\xb6\x9d\x9bZ\xb3\xa95\x9bZ\xb3\xa95_Y\x83\xd9\xd4\x9aM\xad\xd9\xd4\x9aM\xad\xf9\xb2\xed\xdc\xd4\x9aM\xad\xf9\x0fUk\xfe.\xad\xf8\x87>\xc3oF\xf9+\xab5?>\xb0\x9f\xa4\xd6\xfc\xf8\xc0~\x92Z\xf3\xe3\x03\xfbIj\xcd\xe7\x02\xfbW\x0c\x94\x9f\xa4\xd6\xfc\xf8\x11\xfbIj\xcd\xbfa\xf3\xf8\xb7\xa85\xdf>\xf1?\xc9\x88\xa1\xfe\xbb\xcf\xff\xd8\x88A\xf8g\xb9u\x96\xe11Z\x14)\x91\'8j\xfd]`\x19\x84g0\x81\x86\x11\x0c\x16i\x8c`9\x81\xe4x\x91G9\x08e\tl\xed\x16\x8a\x17\xf8o\xf5=\xbe\xcf\x1f\xb0\xd8\xfa\xb5\xc2\x1a(M\x90\xb0\xc0\x11\x08+\xb0\x10, \xfc\xda\xb5\x14B\x12\xd4\xfa\xad\x08\x12#1\x01\xc2E\x91\x11y\x84G8\x88\xa0!N\xc0\t\x06\xa3~\xa4\x11\xf3\xd7\n\xe8\xbf\xfcN\x01\x06\x04\xfa\x0b\x8cb\x18M\x918\xfaGF\xcc\xd7\x07v~\xc7\x88\xa1\x10\x9c\xc1E\x8e\x10Yv\xedc\x9c\xc6\x05\x9afE\x08C\x05\x91\xc51\n#i\xee9\xb8(\xccQ,D\x8a4\x83\x11\x18\x8f\x13\x14\xc2\x08\xc2\xb3\xe8\xd1/\xff\x06#\xe6\r\x92\xcb\x9f\xd6\x88\xc1p\x1c\x17\x18^$q\x88\xe6Q\x16\x85X\x94_\xfbim\x19\xb1\x0e2\xc7\xaf\xc3\x86\xac\xdf\x15&\x99\xb5c\xd7\xb5$\x10<\x86a\xcfo\x07\xa1\x9c\x08\xa1\xbf\xfc\xbe\x11\xf3\x86q\xfa\xf7\x181\xffti\xff\xb7\x191\xcfM\xec\x0f\x8d\x98\xaf\xbf\xce\x7f\xaa\x11\x03c\xdf\xafG\x0f\x08\xd5\xa0\xf3\xa5r\x0b\xcak\x00\xb9\x87\xd2*\xdc\x8a$\xa5\xd4\x02\xcdah\xc3p4D\x87\xdf\xed*\xf6<\xb1\n\\\x1a=\xa1f\xd2^n\x01\xf0\x84LC]\xc9\x8d\xa5\xd8~\xc2\x87\xfb\xe9\x16\x96ro\xcd\xed\'\xeb\x0c\xbf\xcb\x88y\x9e\n$\x01\xaf\xab\x96~\xadk\xe6\x84\xe0ev2M\x92a$\x07\xc7}\xc5$;\xe4\xa6;\xf3%\xa6\x8d\xb4\xe0-Y\xe01\x90}\xe0\x18\x8c\xda\x9aCa\xeaI\xbe\xb8\xa8\x92T\xfe\xad\xaf\xa1\xc2\x08\xcb)T\x88#\xfa\x00%\xf2\x16\x1c\xba\xf0\xbbu\xcd~\xb0\x11\xb3\x86\xf9\x94BP\xe4\xb71\xc2\x1a\x9b\x16\xc2\x00\xede\xb0\xb7\xa4p\xc7\x01\xb9}\xebsu\x98\x82\xa5-ul/f<;\xa1\x17\xf7P8\xadLPw\xd3\xb8-\xadh\xd4\xf4p\x05\x91\xd4\xf0\x11\x90\x81"\xb9.\xa8F\xee\xcd.l:\x10\xfad\x89\xbaw\x191k\x8c(M \xe4\xfa\xebe(\xaf\x86w\x06\xbcH\xb1&\x03\x07\x15Mr\x95\xdeZsQE\xb8/<5"\xa0\xd3\xce\x91tL\xf8\xe9\xe6\xaf\xdf/\x9ci\xb5\xc924\x9a\x873\x95\'K\xca^\xac\x07\xab\x00`\xcd\x05\x8f}\x93v\x82\xaf\x0b\xdc\xd7\xaa/\xfe.#\xe6\xb9 \x08\x04\xa5`\x02\x7fY\xf7\xcd\xe9z\xaf\xd2[-]\xcd\xee\xc2\xe6\x8df\xf11\xa0\x9e\xb3C\xde\xa1\xc7\xc1\x9f\xa9\x0e\xba\x9el7\x9b\xd2\x1b\xd2\x8fAg\x16A}\xa7\x89@>f\xce\x9e\xb7)~\t\xd6\x8d\xd4\x86\x89V+\xaaA8\x0069}r\xdd\xbf\xcb\x88\xf9\x08\xf3Y\xb2{\xdd\x1c\x7f\x1b\xa6\x9bps\xac\x95V\x12\xd3\xb5]H6\xcc\xdf\xda*1\xf2|\xa4)Y\x17\xefd\xd3\x1a\x93%a\x9d!\xd9N\x03M\xfc\x9d\xbc\xde\x1e\xa7\x07\xb2\x93\x19\x84l\xa2\xdd\xeeBg\xd6Y\xa2\xe3h\x02\xe5\xe2a\xeb\x8e\xf2\xc9\xfa\xbb\xef2b\x9ea\xc2\x10\nc\x08\xfc\x12f\x9c\xc7w\xc1\xef\xb8\\o\x85\x07m\xe8\x8c\n\xd5\xb1\xb3X\xa1ir6Y\x06\x13l\x19\xe2\xd5w\x1e\xe7(\x93O\xdc=\x1cf\xaa\x9f1\xe86\x87\xc4ba\xed#r\x95\xceC!\x00\x11XG\xef%\x13\xf1/?\xc9\x88yno\x10\x84\xaf\x81\xbeNZ\x93\xc5\x06\xb4\xa9h\x05%\xa3\x8c%/\x12z\xe5\xc4\xfa\x08\xd5A\xd0H\x93v?\x8c\x16\xce\x80\x95\x9b\xa8\xfa\x9e\xcf\x86\xbb\x07]\x10\x02\x8e\xa8\x83j\x9bdx\xf5\x19j\x94"|\xca9]\xc3\xd5\x81\t\xa7\xcb\xdea~\x8e\x11\xf3\xdb+\xcc\xdf\x86\xe9y<\xa7\xca\x81E\x91\xd8#.\xce\xf9I\xa3\x02\x1e\x08\xac5\x89\xb3\xd1$Po\xac\x17k\xc3\x01\xbfSG$\xabk\x05\x8b\xd21\xa4\xb0\xb0\n\x02\x9aA\xdaRl=\x85fG\x88%\xf0\xf6\xb6\xbf$\xdcr\x12\x98\xcf\x1dV\xef2b>F\xf3y\xbf\\?\xf3\xdb0G\xa8\xc3\xe0\x16\xca\xc2\xdb\x08%\xc4\xce#\x9c\x8c\xcb\xaf\xf78d&/\x9d\xf1D0\xcf7\x0fmv\x16\xe6\xee\xe2\x1b9\x9d\x8dK\xdd\xcd\r!j3\x94\xd0\xc3\xe3V\xfbl\x85B\x92\x1a\x1a\x05\x19\x1a\xb7\x89\xa1\xc9\xcf\xd6\xc6~\x93\x11\xf3\x1c\xcd\xe7F\xfb\xac!\xfa\xdb0o\xb9*"4\x80\xab\x80\xb5\xee\x93\xd3\x99\xa5Sb\xdf\xf8\x98\xdb;\x8b\xdfs\xad\x87\xa8\xe6Mu\xa2\xf5F\xaf\xef\xb3\xea\x9c+\xac\x81\x9a\x15\\\x9f9\x97#%\xda\xbf\xd3\xd0u\xc7\x81\x11|\x84\x16@\xd4\xe0\x07\x87~\xad\x92\xaa\xef2b>N}\x1aC\x9e%\xe3_N\xfds5\xa1\x81\xc5>\xe6\xac\x94=\xcc\xe6/I\x1c\xa8\xfb}u\x80\x02\xa5Cj[\x87\xf9\x8e;\x82;\x10\xd4Y\xf6\xe1\'uRt\x82=\xdd\x1d\xadM\x02\x84\xd4D?Y\xa0e\xd6\xf4f\x9a<\xfd\xea\xd8\xf5g-\xc3w\x191\xcf\xc9B\xc14\xb2\xde\x1f_\xd6\x84\xbdf\xbag\xd6(p\x07\t\x8f\xcd%d)\xe8\x92\r\x03\x0c\x84utIay\xc9\x1e-\xb2<*\xde\x968\xbb\xed\x1b\xef2\n\x01g\x19\xdd\xbd\xafA\x1e\xeeZ\xb1?\xd1{\x17\xf5K\'\xe1\xe1\x8b<\xc5\x97\xe6\xb3y\xea\x9b\x8c\x98oa\xa2\xe8z\x81y\xa9\x1c\x0b\x98\xc1n\xceK0\xb1\x0e\x95\x13\xdd(\xa6\xdd\xaf)5\xe0\x97\x1dN\xc7F\xa3\x87\xaa-\xb6\x19\x05\xbb\x12\xda\xf6\x82\xdc\x83\x9a\x86=\xb4"-S\xee\xb8\xdcK\xa8\xd3\xed\x84L\xec\x038=v\x88t\xe0\xf8\xb6\x1e\xbfw^\xfd\xb9\x8d\x98\x8f\r\x14\xc3P\x14y!b.\xd1\x94\xc0\xf7\xe3\xdc\xd3\xf79\x1f\xafir\xd0n\xaa\xc6\xd7\x0f\x16\x1bU\xf1\x82\xd6\x95{4\xb8:\xaf\xa3\x13\xe8\xcc4+j\x02\xee\x08\x8b^\x05KA\x9e\x90\x16=\x15\xbe\x85\xe8\xb6\xd1O7\x98\xbe`\xb7\x01\xcc?9W\xdeE\xc4<\xa3\xc4\x9e\x8f\\0\xf8e\xe5c\x17\xc6\x97Ax\xe8\xf4\xb6\x1bv\xf9.\xd7ZRu Dvc\x8f\x05=\n\x8d\xf7\xd3\xa9(\xdb{\x89#a\xac#U\x16\x11\xe8\x08\x0e\x17\x84\xf3\x00\x13\xd6\xdb6J\xe1~\xfd\xac\x8c\xdd\xce=\xc5\\\xc6\xa8\xfb\xe4i\xf8."\xe6#S%Hj\xcd!^(\xb1\xeetP\xbbn`\xce\xc2\xedX\xc9\xf2\x89\x8d\x9db\xa1oB\xb5;\x0fH\x93G\x8e8W\xd7\xe3D\x10\\eU\xe9\xf18\x93\xfd)\x05\x02\xfa\xda\x96U{\xc9<\x9d\x8f=\x98["\n)P\xdc\x93\xad\x1d\xea\xe6\x9f\x0c\xf3]D\xccs4\xe1u\xed\xe3\x10\xf6R\x1a\x1bF\x8f\xf7\xd2\xe9\xe3+z\xf2\x0c\x1a\xf7 Sl}URN\x94\xa8\x06\x0c\x8e\x1a\xfa\xcdWiG?u\xd6q\x84R\xb0;\x8dW\xfbT7\x97nT\x9ah\xb1v\x1a+ZS\xee\x13\x97i\x9dX\xb2\xa0\xc3P\xf9\xbdb\xca\x7fn#\xe6\xa3\x17Ih\xbd\x08\x12/\x19\xe2\xb1\xbd\x12\xfd\xd4\xdf#{\xceMd\x1c5UR\xf5&\xa6\x9b[U\xc3\xb9\xf7\xe8\xc3\x82>\xd9\xa7\xe0\x10$T]\xe2;\x0c4\xc6\xcb0\x9e\x8eg\x8a\xc5\x95\xfdz\x15:\x87\xff\x8f\xbd7\xdbV\x1b\xcb\xd6u\xdf%ni-UW\x17\xe7B5\xaaQ]\x9c\x8b\xd3T\x01\x92\x10\x02!\xa1b\xbf\xfc\x113r\xed\x95\x19\xd8\x91Il\xc8\x89\xd7&"\xdc\xc2\x9e\xf6\x84\xf1\x8f>z5\x90\xfb\xa7j\x1b\x0fV\xb6k6\xecc\x07\xb3\xc3\x07\xeb\xfdgAb\xbe\xda\x1a\x1c\xa5\x96\xbe\xe6.\x1b^P\xa9e\x03p8\x85\xe8q\xb4=|\xa7\xaa\xb5Zi\xa3\xb7+\x8b\x00\x108\xec\x14r\xee\xb9\x02\xe8\x84\n\xd7\xca:/g\x87\xe5O\x07j\xf6\xda\x1d\xb6\xbc\n\x82:\x89\x16\x82[\x0fl\xeb\xfe\nt\xc5j$\xbf\t\x12s\xabm\x96\x9c\xbf\xd4\xc1\xf7\x177\xf5\x91<\xa6\x96@6\xb2Jh\xb4\x848\xf4i\x8d4@\xb1:$[\x9f\xb0\x13-Q\x95q\x0e\xe3,\xbe\xe6Q;`U\xce\x1c\x19\xbcV\xf9Lrh\x0c\xdb\x00\xa8\xb2k4\x05\x90c\rRN\xe0!\xed\xc7\xf3\x83\x03\xc6\x9f\x05\x89\xb9Y\x93BoW=\xf7\x177\x83\xd7\x02F\xc80\n\xe8\x9d\xf1j:\x1a\x90\xa8\xda\x11`\x9fM\x0eXE\xc4N\x1b\xfdp#b\xfa\xd2\xcf\xe4\xfe%+p.\xaba\\\xd8\t\xb0\x1bA\x05o\xa2\xc7\xc3A\xb3\x8d -\x81Qw\x1b\xf0,\xe3\xe6\xcf\xba\xb7_\x1b\x12s\xdb\xc5\xc5 8\xb14Zw\xc8I\xd1\xa4Zo\xbf9FbD\x1f\xd8\x8e\xbf\x1c\xd4\x1e\xc3\xa0\x9a\x92\xf9\xda\t\xf8,\xa2&\xf9\xb4\xf8L"\xd8\x1b\xa1;\x0e\x1c\x11\xa6~\x9d4\xd29\x88\xd8pU\x08\xca\x19\x81r\xb8\xd1\x01lc\x05TN8\xc4\x83y\xe2Y\x90\x98\xdf[}\x90DA\xf2\xae\xd5g\x15W\xf6\x91Y\x8c.\xad\x12l\xf1a_\xa6863\xd2%\xc1\xd7W7c\xc3\xf5pR\xf2V\xb2D\x9cc\xe0\xc87\xbc\xb5y`=H\x96\xb8\r"\xea\xbb\xf5&s\xf3A\xe4\x83\x93\nkx\xd6\xaf\xfa\xc5Y\x1f\xf3\x89gAb\xbe\xac\x89\x90(N\xa2w2\xe5T\xae\xad\xa2\xf1\x8e\xaa\xb9o\x8e\xbcq8\x00\xca\xd8\x1b\xdbUUP\xee\x18\x93z4\xa8E\xb1^\x83\x9b\x88\xd7o\n\x97P\xae\x024\x8c\x03\xa1Q\xa2\xe6\xb4\x876\xac\x1a[\xd2\xfdS\xd1E\x9dY\xa1\xe6!\x1e\x0e\x88\xbc\xfd=\xeb\xffKH\xcc\xd7\x87\x86\x1fH\xcc\xbf\xfd\xb4\xfa\xff\x1cH\xcc/\x84\xdf\xf8\x10M>[\xfa\xfe[\xfa\x81\xc4| 1\x1fH\xcc\x07\x12\xf3\x81\xc4| 1\x1fH\xcc\x1b\xaf\xf3\x03\x89\xf9@b>\x90\x98\x0f$\xe6\x9d\xe1+\x1fH\xcc\x07\x12\xf3\x81\xc4| 1o\xbb\xce\x0f$\xe6\x03\x89\xf9@b>\x90\x98w\x86\xaf| 1\x1fH\xcc\x07\x12\xf3\x81\xc4\xbc\xed:?\x90\x98\x0f$\xe6\x03\x89\xf9@b\xde\x19\xbe\xf2\x81\xc4| 1\x1fH\xcc\x07\x12\xf3\xb6\xeb\xfc@b>\x90\x98\x0f$\xe6\x03\x89yg\xf8\xca\x07\x12\xf3\x81\xc4| 1\x1fH\xcc\xdb\xae\xf3\x03\x89\xf9@b>\x90\x98\x0f$\xe6\x9d\xe1+\x1fH\xcc\x07\x12\xf3\x08\x0e\xe1\x9f\xba\xde\x7f\x89C\xf8C\x11\xf2\xce\x90\x98\xd7\x0b\xfb&H\xcc\xeb\x85}\x13$\xe6\xf5\xc2\xbe\t\x12\xf3\x98\xb0\xbf\x82\x1c\xf9&H\xcc\xeb-\xf6M\x90\x98\xff@\xf0\xf8@b^\x02\x89\xa1\xfe{\xcf\xff\x05$\x06\xa5a\nE\x04\x06\xe20\x06G\x08\xfe6\x91X\x10P\n\x04\x11\x82\xe1A\x01\xe2\x11\x8c\x85q\x02gxR\x80o\x13\x93\x08\x9aD1\x18\x83\x05\x90\xfb\x9a\xc9\xfcs\xfe\x01\xceP\x08\xc3\xb3\x14\x82\x12\x04B\xc3\xec\xf2\x0b\x88\xe60\x90\x00\xc1e\xd9$\n\x12\x18\r\x91,\x8b1\x04C\xdf\x06\xdd!\xc2\xb2\x00\xe2F\xc3\xc0\x10\x10\xc5^\t\x89\xf9\xaf\xc1\x92\xbf\xfdh\x00\x03\xf17\x18\xc5\x10\x84\x80a\xe2\xcf 1\xefO\xd8\xf9\x01$\x06\xe4\xd1\x1b-\x81@\x16k0\x02\xc1`\x18\x86,Vax\x8e\x07Y\x8cEX\x86\xa58\x16\xc3Y\x81\xe4HJ Q\x96\x00\x11\x92D)\x06\x870\x84\xbb!M\xfe\x11\x12\x031<\xbf\xac\x13\x04\x97\xd7\xc2q\x98\xa3 \x8c\xe5Y\x06\x819R\xc0I\x88D9\x1c\x87(\x9c\xa7A\x9a\xe0\t\x1e\xa4X\x12\xbf\rP\x82\x19\x18^L\xfdw\x9a\xd1\x07\x12\xf3\xbf!1\x08\x06\xb34\xb2X\x89Y\xac\xcc/\xf6DY\x16\xb9\xb9\xa5\xc0.\xafB,^\t\xd2\x14/\xb0\x10\xccS\xcb\xef\xe04\xb3\x9c-\x8c$h\x01\xc1\x97\x1d\xa5\x7f\xfb\xc5!1\xff\xf6\xd0\xfb;H\xccolC2l30\xec~\xb8\x8605\'>5\xab\xb5\xbcO\xeaK\x9f\x95B\x11\x05\xd6^\xb31=\x85\x97\xc2\xd4\xc7*\xbe\x1e\x14\xe5\xeb\xcf\x91cv\x9b\x01\xb2\x96\xa14\xf0\xf0tma\xfaz\xa9\xf4\x10\r\xcbk\xac\xcd\xe6\xfd1\x0f4(\xe7\x96z\x02\xc6\x8e\xba\xd8\x1d\xd2ce\xff\x1f|o\xcc\xd3\x8d\x90\xf9_\xdfo33?h\x1c}\xfbaG\xe6\xc9`\xabPYOT\xf9{\xb6\xd4\xfa\x10\x91\xb1\xe5u\xaf\xeaQ?\x84\x08=\xe9N\xd4.:\xcc\xcc\x97\xd5(\xa8lih\xf6\xf4\xf5\x88\x8cLR\xf3\xe0J\xe3\'_\xeb\xc7}]\x96E\x00\x89\x1d\xebz\xd3\x15\xd0\xe2\xeb\x8e4RK\xaaQ~g*\xf4n\x0f\xf0{M\xe1\xcc\x7fX\x07\'\xdd\xd6`\xee6\x8fGz3\x01\xe46\x9c\xb5\xd3$\xe4\x04*2\x1e\xa5\xc3\x16\xc7\x1e6j\x07\xa3\x84\xbd\xb2\x11,f\x8fM\x9a\xbb\x9a1\xa3\xe8\x88\xf9+\xe6\xb1Y\x82OC\xf9,\xb9\x1b\xc7)\xf2\x1e\xe8q\x0c\x9a|\xbf\xa7\xabQ\xc7\x8bK\xab5\xf1\x94\xd3n\xbb\xc7\xe6\xe5\x9cj8\x1c\x98\x9b9R\xd5-Zk\x88\x17\xec\xac5\xcf\xedH\xd7\xb6(^\xa9\xfd+q\xa0\x1a\xa9\xeb)\x06`Q\xd1B\\\xb4\xc8\xd13==8N\xeci \x1f\xe2o_q\xf1n2\xdc\xd1\xaf\xb6\x00\x8d\xc0{V+\xc5\xe3a\xad$3\x96(\x1a\xa5\xed\xb5U\xc0$\xf8%\xe30\x1c\x086\xb2\xcab\xf9y)J\xb1\x8d\xcd\xe9SoT\xc5\xb2\x0e\xbe\xe3\xa4\xd3v\xa2.\xd7\x83\xb7\xa3\xc4A+Z\x1c,\x1e4\xe4\xd3@>\x8b\xc6\xe5$@\x14F\xde\xcd\x83=\xdb\xa7\xe3(\x94>\xe9\xb6\x19\x11\xf1\n\x11zmz\xf2\x12\x98\xf6j1X\xdc\x05\xc4\xf8S\x9f\x07\x9ckK\xea6\x8e\xa4\xc6NT8\xdcR\x05\xdai\x8c\r\xe3a0\x16q\xb6w\xc8\xeeT\x81Qm\x83\xa3\xfa3\x99\xbf8\xc8\xe7\xe6\x0e\xb7\x7f\xf1\xfb\xc1s\xda\xe0\x03\xe3D\xb9\xfd\xc1\x1f\xb6\xc9\x91\xc7\xd3L\xb5\xcbm\x98\x8d}E:i{\xd1.\'\xc64\x87\xf5\xa5\xa3\xe0\x11\x90\x15d3`\xaa\x81\xb9\xd9\x9eN5 \xda"A:+\xcd%0\x03\x14\x06\x18\x8c5B\xf8\xc1Q\xd0O\x03\xf9,2\x972\x15\xc4\x10\xecn\x82(\xcb\x03\x18K\xb3+*K\xcb\xe3*\x0eO\xad\x0e\x18x\xae1\xf8\xea\xe4\xf5\xabX\xe6\xa8&\xa2ggS\xd3\xa0\x82K\x03\x0f7@\x80\x04mJ\xae\xc4}\xb4\xe5\xf4\xe82!\xb8\xe5[\xb7\x91\x9a\xe7\xb5.\x1c\x94\x88{L\xe6\xd3@>\x8bL\x0c^\xaaH\x88\xb8\xf3\t\xc8\xab\x8e\xd4F\xbff\xf2\xb8\xf4j+h[\x94\xe4\x10\x0f\'\xbc\x89\xb6\xe5\xa5\xec\xe2\xa28\n{k\x82\xfb>WWW\x8cN\xe8M\x1e\x8a\x14\xc0\xe0\'i\x13\xa0\x9e\xbawZYq\xec\x1d\x88z\xe5\xb1d\xcf\x04\xfd(\xc3\xebY \x9fE&\x84\xa1\xe8\xd2\x05\xdfE8\x8b\x87\xcf\x84\x0f4\xa1\xcc\x9fk\x9bQr\x0c\xda##55$E\xd8\x02\xbf\xa3\x806\xb4|R4\xd5\x02\xb9\xecH\xeb\xe2\xce\x96\xa9\x9e\xf2\xac\xf0)S2\x06\xb5>\x01.\xd7\x02\xc2)\xb1\x01\t\x08\x99\xa5\xe7ypP\xea\xd3@>\x7fh3\xffQ\xa6\xa1\x9bu`@\xeb\xad2\x88\xc8\xae\xc8\xfb\xde(\xf7\x17\x00\xd0\'\x0f\xd5\x0f\xa6\xd0\xe497^O\x07\xd0n\xa8\xcb\x90\\\x83\xabI\xef\xf8f\x18\xfb\x039\x9f\xb1\x1d\x8c\xec\x89\x83\x15!n4O\x08S{\x17\x96\xd5\xbf\x0b\xe4s\x93y\xe33\xfc \x90\xab=\x08\xc6\x87k\x12\xda\xa1o\x9c\x894M\xd6.\r\xe95\x14\xaf\x14+\x9c\x1aL\x12\x92\xa1\xa8\xf0*\xc7v\xa4\xcam\xd6\xfb\x1b~\xee\x84H\xde&\xa1\xd4\x80j\xb6\xc0nP\xa4vDc\x8e\x15">\x1e\xe2\xed\x83!\xe8i \x9f/\x99K\xfc\xc1\xc9;\x10EB\x9e\xc1\x90QV=\x01v\r>\xd2\xea9\xd9\xc4\x85$4\xf5\xb8\x0f\xf1M\xbc\xbd\xd6xx\xed\xbc\xf18#e*\xca\xc9Z\xef\xac\xf39\x86\x05K\x07*\xa5\x14\xeb<\xa9]3\x8b\x0c\x05\x98\xf6Sa5\x17\xe0g\xbe\xf9\x8b\x83|\x96\xac\x0f\x81K\xcbH\x10wU\xea\x81iXs\x07\xe5H\x03\x01Lr9\x1d\xb8T:\x8b\'\xcb\x8e\x05\x8d\xf4U\xad`\x1cY\xcc\xf49\xd0\xe5\x138\xb2t\xc2\xcdT\xae\xda\xb9\xd9\x06\xb4"\xad\xb2\xce\xa7[\xc2\xe7j\x1f\x15\r6\xe0\xac\xfa\x88n\x1e\xcdW\xcf\x02\xf9\x10\x7f\x830\x18BA\xec\xbe\x18\'8a\x1a9a\xd7Q\xea1\xc5uJ\xa7\xc4~\xc2\xba\xb3\x86d\x9b\x80\x12\xaa\x194\xc35Y\x90\x85\xb9\x81jy#+\'\xdb\xdex\xcd\xa5\x06\xd0%\x08\xf1\xbbP&\xa38\x8f\x8cM\x1dR\xf0e\xab\x06\xc5\x12\x88\x1f\x05\xcf=\t\xe4s\xcbW\x04\x88\x82Kst7\xf6\xd6\xa2J\xbeI(\x94\xdb@\x89"\x87\xd36M\xfa\xd8\xe5\x1d\xcdVm@t\xa68\xd1c\x93dV{\x820&\xadX\xea\xf6y\x051\x81\xc0\x96&\xe7L\xb8\x89\x9d\xed+`\x17*\x1esB\xc6\xa6k\xaa3\xdek\x14\xf4\xd3@>\xcb..\xc5 \xbc\xb4\x06wS\xe0y\x16F\xca\xca\xe8V\xf5e\xcb\xb9-\x82\x9d\xea\xe6d\x9f\xac\x1e1=\x9f\xd5\xf6t[\x9c\xd5\xd3R\xe1\xe1\r\xafauE4\x8b\x9f\xac\x1b\x84\xf4P?\x9d\x84\tLK\x07\xdd&W\xb2:l\xeb\xb4:\xebD\xeb\xfd\x0ci\xf9j\x92\xcf\xe2\xfa\xcb\xb1\xba\xdd\x16\xdf\xc9\\\xaf\xb1u\x14\xa3\xe3\x146\xfc\xc1\xad\x85\x02\xaee\r\x10#\xf9B\xd9V\xae\xecwM5\xa1 \xa3l6\xb2\x1d%\x8c\x16\x0b@\x1aT\x834\xac\xb6\x8c\xdb\xb8\x08\xbc\xd2\x15\x07w\x12D\xd0\xb5\xe1Tc+\xcc\xa5\x1f\x9c\x91\xfc4\x92\xcfbM\x84X\n\xdc\x1fU\xe4H\xa5\xaa\xf9p\x00.v\xae\xedK\xb7\xa8\x8f`+\xd1}\x9e25\xcc{\xc6\xde\t,j\xe9\xd9\x1a\xdb!=m\x8d\x9dq\xe5\xa4\xf4\xb4\xae\x1b\x9b1\x8c\xe0\x01\xe7\xe0|\x10\xb50\xed\x06\x91\xda\xa2\xe3a\xaf\x1f\x1e,U\x9fF\xf2\xf9*n0\x90\x02\xefg\xfacl6\x10I\xe4\x9df\xed\x18\x05I\xacD\xc1\xc5\xf6\xc2z\xba\xaeP\xcb\x84-C/\x9au\xca\xce;.8kD\xbb\xd6\x97\xbacsl.rm\x1cl3\xceQ\xb7]m]\x1anV\\\x1c/\x89\xf5\xacL\xf9{\xb5oO#\xf9|\x05P\xe2\xd6\xe9\xdd\x15\x15\x00\xb4\xc6\x83\xe1\xec\xc3g\x1f\x97\xda-;\xa6\xc7\xcb:\xe9\xadl\xe5\xb7(G\xf6\xe1A\x96\x1dep\x13M\xd9\x13\xf2\xac\xa0X\xf0?\x8d\xe4\xf3%\x13&\x97Lz\'\xb3\t\x1b\x1c\xdeh\xcdiw\x9c\xcd\x08eV\xa3V\xc3\xf9\x18\xbazt\x15\xbdX\xd2\x88Fcs\x85\xed\xabd\xa6\xbbd3\xb8\xa9Tl\xd6\x01i,\xd2s\xe8\\\xce\x82gI.Z\xcb\x04[\x19bv\x0c\x89\xecA\xd7\x7f\x1a\xc9g\x89p\x18\x84 8tO-I\xb9}\xb1\xd9\x89\'4%\xb4\x91_\t.\x94 tg9g\x8f\xc8\xd7\xcais\xd2(g\xa7`\xc7\xb3\xc4\x19\xc3f\x14}\xc4\x06\x9aAe\x8c\xae4\xechm\x0fU\'\xe4\xac\x04\xb2\x9df_ZLO\xfd)z\xb0\xe0\x7f\x1a\xc9\xe7fM\x04DP\x14\xbas}\xdeM\xbc\x11:o\xa0U\x9c\xec\xaa\xa85\xf8\x03:\x13-\xe9\x8a^\xc4*\x80\x9d\x80\x9c\xee\x08\x1aF\x85:\xa0\xdbE\x9f\x10P\xa5z\x86$\xd0k\x8f\xe1\xc2\x9fE\xb8_\x9c\xe4s\xdbE\nG\x97f\xe4\xee*s\xbe^\xfal*\xaa.\xaa(\xa5\x19\xa0\x19\nq\xc39w\xd7DQ\xa4=\x05\xa2p\x98 \x00\x9d\xe4^7R\x8a\x11\n{\x0e+p\xb9\x91\xf2&>\xcd\xf3z\xf4\xd9\xcb\x15\xb6m%Z\xf3\xb8f\x80r\xcfE\x0f\xd2\xad\x9eF\xf2\xb9\xe5\t\x08\xbd\xb1\xd7\xef\\\xdf/\\\x9cX\x9d\xcc\xaa`\xb8\x19\x85\xe6\xc6\xd7\xcdV\xce\xce\xdbJZ4\x91\rG\xe4\xd7&\xa2\x00\x02?\x14\xfe\xa4]|DDtL\x15\x07\xf4<\xeb\xc2%K\xb7r7\xb7e`K\x03/\x01Z\x17\x81\xc0\xfeA\x99O#\xf9\xdcd\xa2\xd8\xf2\x1dw\xad>|\xb5\x8a\xbe+\xe5.g\x9aM\tfBI[\x88\xd7\x15\xe7\xee\x98\xeazw\xf0$\xc8\x1f#\x91\x90d=>\x8e\x96\x19L\xe4\xca\xb7\xb0\x04\x1e2\x952\xb2u\xed\x99\xd8\x16\xa5\x87z\xd0\x14\xb9^\xef]~\xf5(\xbe\xefi \x9f[ms\xbb\xeb"\xf0\xbb3kqN78R\xb2\xb5\xb7AM\x86\x94\xc6\x16S\xc8\xa1\x07A\xdcb\xd1\xe9pA\t\xb2E\xac\x86\x11r\x1e\x12]\x9f\xa82\xc7\xeb\x038\xf6\xc7\xc2NV\x1b\xd1R\x04y\x9b\xe8\x95\\Z\xb59\xe4M5\x1c\xd7\x8f\xf2\x8a\x9e\x05\xf2\xb9\xc5qd\xe9\xfb\xe0{\xd0\xb4\x97\x0e\x9eV\x1d\xcf.\xe0G\xc9\x90\t\xe1P(u\x99W\xd4\x8e\x1d\xb1z\xeek\xa9\xb9P\x0e\xc2\x8f\x1c\x90\xe3\xben\xc7\xce\xf1\\\x85\xeb\xcd\x188Xi\xd1jsY\x8f^\x13\x8c\xab\xb2`\x832:\x9f\x14\x0by\x90\x8e\xfe4\x90\xcf\xef}\x07HP\xd8\xdd\xc5\x8dF\xc2U\x1dH[V\xab\xa4D\x1d3\xda\x02k\x07\x8c\xd7\x05M\xea\xf0\x89\x18\xe8\x0e3\xb9\x93\x84\xe9D\x11k\xce5;\xa0;\xbd\x87\xc5\xb1\xc1\x9d\x89\x9abr\x00]\x03w\x93KA\xef\xf6J\xbb-\x143\x1a~vh\x7fq\x90\x0f\xf1\x05(%1\n\xb9\xeb\x81\x01\t\xa0Wb"\xe5U\x05\xc6\xbe\xc4\xc0\xdeu\xa42\xae\xdcnj\x88D\x82\x89Z\x915:\x81+u\xa9l\xac\xf0B\x94\x0cvZ_\xf1k\xb4F\x0f\xf8\x0e\x95\xed:\xd5\x86\x12\x01\x01\x84\x0ek;a\xd4\x8c\xe4\x1ee\xcc?\x0b\xe4C\xfc\r\xa2H\x04\x01\xef\xef2\xe3\xebH\x0f}\x18]\xa6\x91\\\xc9\xdb\t\xbc\xb8\x14\x90\xb3\xd8v\x89g\xbb\xd5\xb1\'\xc2\x06\xf2\x08Y\x0f\xd2\xb0\x04AI2\xaau\xbc\xa2\xeb\xab\xb9;]\xe5t))\xfb~\xe0\x9bJ;1\xe5\x9e\xd4\xb32\x008\xed\xc1\x00\xf74\x8e\xcfbL\x92@A\x90\x84\xef\xb9s\xb0l\x05\x9b\xbd=\x97\xfb\x88\x10\x05\xbb\x17\xf9\xf2\x82r\xe6,qWj\x8bm\xb4*\xed,\xec2\xaa\xd0q\x04g\x86\xdc\xd6\xf2\x99\xdb\xf8\xbb\xcb\xd1o\xc3\x98=\xf3\x87\xb2\x0bE\xb7\xb0\x9b\x8d\xb7\x9f\xa8$\x9et\xf7\xc1\xeb\xa9\xa7q|n\x9f\x1a-\x85\x07r\x7f\xff~\x8e\xedL\xc7D\x84\xf5\x9a\xb8\'%\x93\xb9l\x1aP\xdcJ\xf8\xce\xcfv\x17:\xf4\xd2s\x904\xbc\x94\x9a8m\xaf\x10\x8a(.\xba\xd1\x1bv\x13^\x01\xb4-6\x8e\xb9\x1a"\xad\x06\x80B\xdeG\xccY\x9cg\xaax/&\xf9\xd30>\xb7\xfb\x8cem\xe4\x0f\x00\xcc}\x87c\nV\xf2\xd6\xea\xd4\xef\x07RU\xf4c\x83\xea\x14\xd3\xe4U?m\x8bj\x05\xeb\x17\xa3\xf3bg\xadX\xc3\x9a\xe8\x97\x9a8\x18K>\xc0fR\xaf\xf4\xaba\xa0\xab\x80\xb27\x85\x92\x1b\xc1>\xdc\xe5\x14Rn\xd9\x7f\x0f\xe3\xf3u\x0f\xf3\x8f\x18\x9f\xff\xf7\x7f\xfd\x966Y\xfe\xf5X\xc5\xef\x0f\xd4\xfc\xc6\xeeB\x85Y\xeb\x87\xa46oOo\x16\xc7m\xf3\xfb\xbe\xef\xe2\xcb\xff7\xc4\xc7.\xbf=XG\xfe\xfd\xc9\xc5\xdb\x17\xfb\xcb\xd7\x97ph\xe9\x93\xbe\x1e\x15\xba\x03\x01\x8d?{\xf44\xf4\xf52@\xf4Cj\xde?*:\x0bpVFMz\xf4\xc6,8\r\xd9\xba\x02\xf35\xd3\xa6\xf3\x01O\xea\x0eY\xda9,\x9d=T\x17\xb19\xa9S0\x86\xbd)\xb7\xc9\xe1\x9f\x9f}\xfd\xf9[\xa7\xf0\xe1\x98\xd4\x02\x98\x05\xf2\xe1\x1f\xde:\xe6\xd1\xd6\x11\x0e\xb6[\xca\xbc\x071^\xea\x99\xb7\xb1\xedR\x08ep\xecY]\xbc\xc4\r\x9f\xbf \x91 \xf4K\x03\x03\xda\x13U\xf9N$\xfa\x07yR\xa0\xa8q\x8e\xfb\xc1\xaeeCCd8-\xb3\x83s<\x14f\xa9\x0b\x99\xef\xf96\x12i\xe1\xcc\xac\xe3*\x8a\x1cn\xf93\xb0\xb7\x8f\x8e)\xb51\xff\xc8\xf0\xf9\xcb\xe8\xa4\xbf\xb0g\xffUG\xfd\xfd\x95\xff\x0c\x9d\xf4\x7f\x0cf\xba\xff;\x1c\xbfi%\xbdl\xb0wMa\xe8\xfa\xcf\x9b\xf0W!J\x7fu\x0f\x1ex\xef\xe5\xdc.I\x9b\xea\xff\xe1\xbd\xbf\x96s\x07czSC\xfe\x1f\xbf\xfa\x8f\x0c\xe9\xd0\xbf?\xc9\xf6\xab\x193\xf1\xa9*\xf3\xc7\x7f\x8c\x02a-\xf4\xf1\xf0\xff\xfc\xd7\xf3\xe6\xb7\x18y9\xc5\xb7\x07\xd5~\xfb\xed\xf6\x85\x0f\xf8\xec\'\x7f\x03\xeb\x7f\x0e\xf8\xec\x17\x02L|\x98\x1d\x1f\xf0\xd9\x07|\xf6\x01\x9f}\xc0g\x1f\xf0\xd9\x07|\xf6\xc6\xeb\xfc\x80\xcf>\xe0\xb3\xff\x8b\x8a\xa8\x0f\xf8\xec\x03>\xfb\x80\xcf>\xe0\xb3\x0f\xf8\xec\x03>\xfb\x80\xcf\xdex\x9d\x1f\xf0\xd9\x07|\xf6\x01\x9f}\xc0g\xef\x0c\x14\xfb\x80\xcf>\xe0\xb3\x0f\xf8\xec\x03>{\xdbu~\xc0g\x1f\xf0\xd9\x07|\xf6\x01\x9f\xbd3P\xec\x03>\xfb\x80\xcf>\xe0\xb3\x0f\xf8\xecm\xd7\xf9\x01\x9f}\xc0g\x1f\xf0\xd9\x07|\xf6\xce@\xb1\x0f\xf8\xec\x03>\xfb\x80\xcf>\xe0\xb3\xb7]\xe7\x07|\xf6\x01\x9f}\xc0g\x1f\xf0\xd9;\x03\xc5>\xe0\xb3\x0f\xf8\xec\x03>\xfbV\xf0Y\xf5\x8fo\xf2/\x11?\x7f\xe8\x91\xdf\x19|\xf6za\xdf\x04>{\xbd\xb0o\x02\x9f\xbd^\xd87\x81\xcf\x1e\x13\xf6W0Z\xdf\x04>{\xbd\xc5\xbe\t|\xf6\x1f\x08\x1e\x1f\xf0\xd9+\xc0g\xf8?\xd8\xf1\xcf\xc1g\x1c\xc9\xb3\x08CR8\xb5\xac\x96\xa4\x19\xfa\xf7\xc9\xe6\x9c\x80\xf0\x04L\t\x04HQ F\x92\x10\x82B$\x89\xc2\x10\x8ab<\x8e\xf2\x02\x8a\xf2\x10\x05\xc3\xb7\t\xcf\x7fB\x8b!\x96\x97Gx\x96\'h\x1cY\xf2\x1b\xc1\xb1$\xc92\x04\x8b\xe2$\r\t\x08\xc1@$\x88q(H\x0b<\xc8B\x0cL0 \t\xa3\xd0\x8d \xc3\xc2\x10\xf1B\xf0\x19\xfe_sk~\xfb\xc1\xc8\x1a\x04\xf9\x1b\x8aR\x14J@\xc4\x9f\x83\xcf\xde\x9e\x1a\xf7\x03\xf0\x99\x80A\xf8\xb2$\x8a\x07\x11\x04\xa2A\x81\x11x\x06\x83\x96\xaf@ N\xb1\x04\xca\xd1\x04\n\x924\x8b/\x82\x96\x9a\x96\xc6qp\xb1\x1eI/\x16\xc7)\x92\xbc\r\xbf\xfbG\xf0\xd9\x13\xe8W\xff\xfb\x88\x7f\xc0g\xbf\xdd\xe68-\x16\xc0I\x81Dy\x9eY\xecJq(D\x80\x1c\x01\x13\xb7\xb3\x80p\x14\xcb\xb0\x02\x04b8\x86p\xd8\xd0\x14Ha\\"%\xdc\x83\x13\xa8\x9f\xc5\xd4\xfa:\xb4K-\r\xde\xa7\xab\xbe\xe7s\x89\x91;e\x9f\xa7\x04x\x86\xce\x07\xd7\xdf\xc1\xd2\x9a\x89\x8a\xb8\xa1Wu\x80\xf4\t\xd1\x1e\x0f=o\x94\x04u9\x8c\x14*\x82W\\H\xa9\x8b\xccC\xe5\x15\xf6\xe5\xcb`\xe5\x19\xef\x18\xec\xe4"\xf1\xa4\xfa\xf0\x83\x0c\x91g!\xb5\xbeT\xe2\xb7\x8a\x05\xb9;\xb3\x94T\xcf\xe7\x11\x9c\xf4z5,\x91\xa8\x8f B\xe7\xf6}\xa1\xd9W\xa2\x93m;8`c\xd0\xf8\xdd\x81\xdb\xab\xfeE\xdbY\xc4\xca\xe8\xe2\x9dI\x15WHT\x03\xdb;uZ\xa6\x85\xa45\x06\x18\x1bVk\x02\x83\xb4\x9fe\xe5_\x1b\xa9\xf5U\xd8 \xcbI\x03\xf1\xbb\xa4\xefz\x90\xc5\xa4&\\o\xcd\xba\xa4\xe3\x08\xb0\xb1)\x97\xf1\x1dP\x1e-8u\xa22H\xc1.\xba\xaaL1\xec\x1d\xa3\xd8]\x8am`\xad\x0fe\xe3\x9b\xbb\xc3f\xe2\xfc\x8bs\xa5\x10\xf9\xc2\xd9\xb4\x1b\xe4{W\xf1\x84+\xf3X\x80{\x16Rk\x91\x89\x90$D\xc1\x08x\x8f\x0f\x9a/=\n\xd2\xca\xd9\xe7\xe3I\xdf\xb7W_q\xec\xcc\xd2\x9b\xde9\xb4T\xb8/\xfb\xde\x08&\xdaO\\Tp\x06\xfc\x928\xf6\x8eU)\xed|p\xf2T\xab\x8b\x00\x1c\xca\x16O\x92k\xbb\x12\x92\x94\x8dD\x9c&\xb9G\xd3\xd5s\x90Z_>A\xe1Kg\x8c\xdf\xf9D\x152\xdb\xb42V\x03&\x15\xe3n\x93\xef\xa1\x10=\xae\x83#\x10\x8fX1\x1d\xc8\x144\xce\x1b\xe6h\x86\x18\x0f\x1d\x0e\x12\xc8W\x1b0]\x8f\xe1\xc9\x1b\x10\xc3\x9b\x92Up1\x8f~!\xed\x8c\xac\x9d\xf4\x82\xa4\xd1C\xf2^%\xdc\xb3\x90Z7\x9f@\x96M$I\xec\xce\'\xa8\x04\xce\xf0-\xd0\x86<\xbb\x8d\x8e\xeb\x0c\xe4\xd9D"\xa6\x8c\xf5S>\xf5\xfd:\x0e\xf2\xe9j\xa3\xe0\x99\x92I|uA\x8e\x87\xf6l\x10\x1d\xac\xdef\xc1\x1dk\x9ep[\xdf\xeb\x93\x1d/_\xd0\x8d\xec\xd2a\xd9\x1a\x0fvn\xcfBj}U\xaaK\xc5\x8bB\xf7p\x8d\xbc\xbaP\xa1\x0e\xaf\x19\x850<\xc5\xd1E\xdd$R\xfa\x00^\x15\xc3\xbf:\x0evmm"I\xa7<\xd2\xb5M\t\x0b\x86e\x12\xc8\x05Sv\x84\xcej\x11\xabk[\xaep\x8a\x83Y\x9d\x9a9\xd6\xc6\xe4<\xb2\xb5\xf1\xe0T\xf6g!\xb5~\xb7\xe6\x17=\xe8\xaeR\xbd.\x1d\x8d\x07\xc58.#\xb2L\\\xdb\x8cGs\xb3\xca\xb8\x15%\x8b\xc3\xb6b\xd7\x19\xb5\xba^\xb6\x93(\xb6\xf0~}&\xd2\x8c4P5\xf7\x04\xa6\x0b\x98\xb2\x0cK\xcd\x88z\x0b\x18(\x81\x1b;\xb5b6\xeeV|\x10\x90\xf6,\xa4\xd6WAN\x11\x04E\xdd\x13a\x90\xec\x90\r\xc4\x01\xd5\x94\x81(\xd7\xb4\xa2\nl)\xae\x92\xc8\xcd\x80\xee\xba\xcfb\xefPn\xdddU\xf3:\x170W\xea$\xe6Vn\xcekH\x05\x89\\\x8fO\xe6\xf2\xa7ez\xcfUS=\x98\xfa^\xdb\x05\x1b\xf6\xf0\xb3\x12\xee\xd7Fj}\xddf\x90\xb7\x02\xee>\x1d\xc2\xf0\x81\xc0\xbd\xab\\Y\x11\x06\xeaf\x05\xf7&\xaf\xba=\xb5\x17\xb6\xdbC$\xf5\xee\xde&\x1b\xc0\xe0\xfd\xd4)\xddC\x19\x85Zy\x11\xb7\x89\xe7Q\xb0\xed\xc96\xda\xe1\xc3\x9ep&\x92g\xe3Uc\x02\xb4\x1fy\xf4\x83\xdd\xdb\xb3\x90Z\xb7\xc3\x82P\x08\xb4\xa4\xd7;zP\xe0\x03\xa5b\xd81Gb\x99\xeeP\x8d~\x9e\xd5n\xc7\x10\x08\\\x97\x8e\x14\x9a-*H\x96rt\xdc\n\xe6\x13\xceN\xce\xe7\x06F\x98(\xed2\x0b\xc68i \xa9M\x19\x16cR\xaa\xdb\x9d\x16\xe4\x0e\xe6\xbb\x0fz\xfe\xb3\x88Z7c\x92(\x0cR\xc8}\x8fz\xd1\x06\xed\xbcu\x1a\x7f\x968w3\xb1`E\xf0\xe7\x98\x96}\x9b\xb2\xf6\xfe\xa8\xcb\xa11\xec4I\xbb\xae\x95\xbew\xea\xbdU,{\x01\xe8 \x14\xae\xd2&\xd3xq\xbf\'N\xb0\xd3\x9e\xfc#\xe1\xadvR\x97\xed\xa9o"j}\x19\x13!\x10|)\x11\xfe(\x13_\x11\x0cBo\xed$\xc0O\x87=\xb3\x0f\xe2\xe3&a7c\xb9\xf1.[\x9co\xe7m\xad(=\xe2\x19\xbd\xc9\x1c`K\x85\xa4\x19Z\x85\xd0\xf6\x18\xe5\xf6$\xad\xf56\x01Q?\xb4\xc6\xe9$\xac5\xf6R\x1e\xf9\x93\xf53\x86\xc8\xafM\xd4\xba\xed"\xf6\xc5\xb3\x01\xefJ\'O\xeaq\xb0/\xe2s\x13T\xb6\xcd\xc4\xdeEMO8=Tr\xb2bv\xd7\xe2t\x1e\x03\\\x16\x1c\xc9\xf7\xce\xdcz`\xa7\x93Im\x8d~\xb8\x82\x94\xeb\xb4|\xe9\x97JL\x11S\x02\x19\x01x$\x9c\x04\xe9q\xe9\xc1\xde\xf0YD\xad\xbf\xdf\xdb`\x18L\xdda{\xb2\x83\xe97%\xa1k\xb91\xaf\xea\x1c[W\xd9 \x8c\xda\xda\x18\xe5\xe2bP\x8c\x7f\x9e{\xbe\x8cTa\x7f86kB\x87\xd5a\x8c\xf6DP\x8cc\xe4\xa9\xfd1cG\xbf\x9d\xb9\xab.uy\x1b\x8bF\xdc\x1e\xab\x9f\x91\x17_L\xd4\xfa*\x84A\x9c$q\xfcN\xa6{l\xaf\x90\x1e\xa9}z\xe0\xa7\x1a\x0b5\x86o\xba\xcb\xd1:\xd8^\x87\x1e\xd6\xbc\xd3\r\x02A\x9d\x0b(e4\xb3\xb1/N\xef\xe2\xc09\x0c\xb8\x89\x90O\x13\x166\x98\xa1\xd9\x9b\xc0\x11RW\xb5\x85\xd1&\xcc\\~0\xe9?\x0b\xa9\xf5\x15\xe1\xa0%\xb3Q\xf7\xc4\x99X\x17\x03T8V\xea\xda"[\xf3J\xbbM\x16\xb6E\x86G\xa1s\xe8\x8b\x9eGjof\x1d\x17\xe0O\xbb&\xa8;\x9e\x8d\xba`-^|\xd1?{\x17\xefpj\xc1\x80\xd5\xa3&\x92!\x9c\xa8\xfa\xb9\xcb\x13Oy\xf0\x16\xeeYH\xad\xbf\x07rj\xa9\x88\xee>d\x88O\'\xae\xd02\xaci]\xbe\xb7\xce\x1b\x81\xa4\xa23\xde_\xf7\xc1\xe5\x82\x85\xa2\xb5\xcf5\xa1\x9e\x96\x80\x16\x1d\x0e\xc8J\xc3\x00l\xebd\x82$[\xd1j\x7f\xdeS\x151\xcb\xfe\xc8a\xae\xd8\xc3\x85I:\xb4\xb0e\x0e\x0f6\xa9\xcfBj\xfd^\xa9.%\xee\x92\xfa\xee\xeeT\xdd%z+\xba\x97\x9d\xbc\xb3g\x9a\x9a\xed\x1b\x07\x1cv\xcaY\x80v\xa4\xb3\xe1s\xef\xe4\xc3\xae\xc2*\x00e\xa09w\xa2\xb0\x89`\xd1\xf3%#+8s\x90m\x9e\xe6\x03\xea\x82\x9e&\xb0\xd1\x14\xb82\x0eu\xea{\xddh<\x0b\xa9\xf5{!\x0c.!\x0e\xbakk\xc0\xbd:\x86#ta\xca\xa0\xd4\xd6M\x0ee\xa7\xb3\'!\xdc\xd5J!\x96\xad\xc6>f\xad\xc8\xcc\xe7r\n\xcb\xa4n\xcdj\xaf\xa5b\x90\xbb\x06\xe7\xa9\xcd0\x05s\xb7\xc4\xf7\xb1f\x0c}\xb5\xb1\x94Cg\x17\xa3\xf5\xe8\xcd\xf4\x93\x90Z_Y\xff\xc6}\xbe\xcb\xf9y7Q\xe1&HO;b_\x95\x0e\xafA\xbd\xef\xcd\xe3j\xb7t\x9d\x06)\xc1\x03\xadH]u\xac\x86\xf9li\x14\xa3\xce\xa3\x95g\x81\xdd\xcf\x882\x9b9_Q\x87\xb5\x18\r\x83G\xb2\xae\xe9\xc8\xda\xe1@\x17\xfa\x83\xf7\x19\xcf"j}y\x04\xbc\xf4\xa7\x14uGB\x87\xe8j\x15_\xb2>\xca\x82\x00\xd6\xa93j\x91\xd8\x01P\x9c\xa2\x1a\x04.\x9b\x0f\xe1\xec\xe8\xfb&\xe4\x9c\xcd\xdek]\xd1;\xda\xf6J\xdbFgqu\x00]`e\xbagF\xd6\xcf\x1d\x02\xe2\t\xdeut\x83\xd0\x17\xf5\xc1\xeb\xf7g\x11\xb5n\xb6\\\xfa\xb6\xe5P@\xf7\xd76\xe7\xb9\x05\xf4\xc9\xa3\x0b\xa2T\xaet\xdb\\\xfd\xe3(\xe0M\xc6\x1a\xd5\x01L\x9d\xa17q\x03\xb1\xc2\x0b\'\x91\x16\xb5\x8a\x8e\xc1Qj\xe90"\xf1f\x94\x02\xe7\x82\xc1Z\xd2P\x85\xb1.\'MO\xc6\xe42\xd3>\xfbV\x8e\xff,\xa4\xd6\xd7}\x06\x89\x13\x04z\xcf\xd2C;\xd9;\x80\x16\r\xae\x8dV#"Q\xe0J\xdf[\xd1:\x056+\xe9\xb8\xcb\xb6\xc6\x00\xc4!0\xa2[\xb2pW+\x1d\x94\xd8\xb2\xe3|5\x1fi\x86RNl\\82\x95]!F\xde\xd7\xb39^\xa2\xd4\xfa;e\xee_!\xb5~\x7f\xa4\xea\x1f\x91Z\xbf?w\xf7\x01\xc4\xfc\xfd\x95\x9f\x8fwz?T\xcc\xd3\xe7G\xfff8f\xaf\x95<\xa8\x97\xd2l8!\xac\x97\xfc\xa0s\xbb\xe9\xc6W\xca|jN|\xf2\x0f\x8a\xdewh\xf7o\xfa\xec\xf6\xda\x9c\x8ezYa\x9a\x93\xceZ\xb9C\xb5y\x07\xe9\xe5\xcf\xc4\xbc\xe9X\xef\xef4\xcb\xd3A/\x8bY\xc2\xfek\xf1e\n\x1b\x0e\x8d\xe9\xceb\x1e.\x84t.D^i\x96\x97(\x01\xd5\xf2\xc7J\xb4\xe0G\xb1\xe7}\xa16K\xc0\x94\xfa\xe5<\xa1\x9a\xe3"\x8b\x90Is\x96w)\xd3\xc5sx\xe8\x95fy\x85\x12mT\xcb\xc5K\xb8\xddM\x01d8\xfc\xa2\xc8]L\x14\xce\xfak\xcd\xf2t\xe0\xcco\x9a\xa3\xdd\xcc2\x1aK\xa0\xd0\xb8t\t\x01\xdahp\xe6\xb0\xfc\x1a}\xa9\xb7\xbcB\xc9\xa0\x96?V\xa2!\xaf5\xcb\xb3\xa157\xe8`\xaf\xcf\xf4r\xae\xf8\xdb\xc4aD\x9b\xcd\xd9\xe0\xc2\xc5[\xdc\x97\xe6\x96\x97(A\x16\xb3\xfcP\xc9\x8b\x83\xd8\xd3\xc17\xbf\xe9\x05\xba\x04.z^RWK\r\xa3M\xbac\xbd\xd4,\xcfW\xf2\xb3\x9aR\xe3v\xf0\xaf\xd7\xb7\x94|\xbf\x9c-\xc8\xe0\x96\xadr$\xe4\xab\n+\xdd\xa5\xda\x0f\xb1_\xabo\xf9F\xb3<\x1d\x04\xf4{\x10\xfb\xa1\x98\xc3K\xbd\xe5\x05J\xbe\xcf,O\x87\t\xdd\x82\xd8-\xcdOKr\x1c\x16aK\xed\xa2aK\x9f\xbc4a\xd1k\x83\xd8\xd3\x95\xfc\xc4\xef\x97\x02Y\xfaI\x07\xf6\xbe\xe0\xa4\xaf\xdcbp\xd5R\xbbh\x88\xb1\xb4\x95K^Y\xf2\xcc\xed\xfa\xe2\xb5\x95\xd8\xf3\x95,\x85q\xbf\xd4\x92Kv\xda--\xaa\xb9d\xc9[\x93\xbcxK)\xbd:\xe5?\x1dj\xf4\x9b\xb6\x14\xc8\x86\xb3\x88(w\xf3R\xb3L\x06\x97b_e\xa6\xa3\xbf\xb6@~\xbe\x92y\xd7\xff\xb8x1_] ?\x1d\x8c\xf4\x9bf\xa3\xa3\xbe\x98ci\xbe\x16\xd7\xd7\x96<\xa3a\xcb\x99[L\xf3\xda \xf6\x02%\xdfg\x96\xa7\xc3\x95nf\x99\x7fx\xf9\xc21/\xee[\x9e\xae\xa4\x0co\xd7HK\xf8MA\xcd1\x97\xbc\x12.M\xf2n9l\xd5\xabs\xcb\xd3\x01M7\xb3,\xfd\xb0\xb9\xa4\xd4j1\x85\xb4DczX\x92\xe7"\xee\xc5fy\xbe\x92\xd2]\xbcE\xbb]\xb6\x8c\x8b\xe7\x0cK^Y\xd2\xfe\x0e\xbb\x151/\x0fbO\x86<\xfd4\x88i\xcek\x0b\xe4\x17()\xa5\xfe\x07Y\xf2\xf6\xe9\xd1\xf4joy6(\xea7\x8dE\x91e\xe1\x8b\x97\xec\x90E\x14\xf6\xf5\xe9D\xb9\x9c\xde\x17\xdf\x89\xbd@\xc9\xed\xd2\xd5Yr\xca\xf2C[\xfc})\x92\x17\x13\x857E\xe0\xcb?\x06{2l\xeaf\x96[\xbf\x02\x1a\xb7\xb9\xf6s\xb84`\xb7IW7\xd7\xf7^k\x96W(Y\x82\x96\x89.^\xb3\xfc\x7f\t\xc9\x8e\xbb\xa8\xa8\x96\x92\xffg\xe1\xf8}\xc1Z\xbf\x9b\xa5\x94\x86\xa57^z\x97\x10\xd2g\r^R>\xa2\xcf/\xbe\xaa|\x85\x92\xef2\xcb\xd3\xa1W_f\xd1n\r\xd8\xed\xd3\xc9\xc5\xddu\xe7\x16\x995d\xa9\x80^j\x96\x97(\xb9u\xf7\xd8b\x1ah\xc9\x92KX\xae\x96\xfa\xf2\x96_^o\x96g\x83\xb3~7\x0b\x97\xfe\xf1^\x0c\xbd\xbd\xe2K\xcd\xf2\x12%\x8f\x85\xe3\xf7\x85\x84\xfd\xdd,\xda\xfd\xe5\xfe\x8b\xaf*_\xa3\xe4\x9b\xcc\xf2t\x80\xd7\xb7y\xcbk\x94|\x93Y\x9e\x0e\x01\xfb6\xb3\xbcF\xc9wy\xcb\xff\xad\xc0\xb3\xef\xdc\xf4\xa7\xa3\xc6\xbe\xcd\x17^\xa3\xe4\x9b\xcc\xf2t\\\xd9\xb7%\xf4\xd7(\xf9&\xb3<\x1dy\xf6\x8d\xe5\xef+\x94|\x93Y\x9e\x8eM\xfb6oy\x8d\x92o2\xcb\xd3\xd1k\xdf\xe6-\xafQ\xf2]u\xd6\xb3\xf1m\xdf\xd7\x95\xbcD\xc9w\xa5\xfcg#\xe0\xbe\xcd,\xafQ\xf2]A\xec\xd9\x18\xb9o\xcb-\xafQ\xf2]fy6\x8a\xee\xfb\xcc\xf2\x12%\xdfe\x96g\xe3\xec\xbe/\xe5\xbfD\xc9\xb7\xb5\x93OF\xe2}_ny\x89\x92o2\xcb\xd3\xb1z\xdf\x16\xc4^\xa3\xe4\xbb\xee\xc4\x9e\x8d\xe6\xfb\xbe;\xb1\x97(\xf9\xae \xf6l\xbc\xdf\xf7\x05\xb1\x97(\xf9.oy6"\xf0\xdb\x82\xd8k\x948\xc4\x97\xe0\x10\xa1\xff\xde\xf3?\xc7!\x12\x0c\xc5"\x0cC#0\x87\t(K\x907\xe8\x16\xc5\xa1\x04t\x1b\xb8\x0e\n\x08D\x93\x08\xca2\x82@\n,\t\xb3\xec\x17\x05\x8c\xa7A\x8e\xa0I\x81\xbe\xa1\xdc~N\xfa"\xc1e\xff@\x90G\x18X@`\x88\x04oc\xbeY\x90\xa2)\xfa\x061\x81P\x16\xc2H\x16\xe1os\xe00\x9c\x80Q\x1e\xbc\xcd\xc8\xe48t\x11\x0f3\xc8+q\x88\xff5\xb9\xed\xb7\x1f\x8d\x1b\xa3\xfeFa\x10\x86@\x04\xf8\xa78\xc4\xf7gI\xfe\x00\x87\xc8\xe1\x14\xcc@4\xc33\x1c\x0eR\x8b\x99Y\x86\xe2H\x92\x10(\x98\xa6\x04\n\x05\x19\x01Z\xd4 \x1c\x8a\x82,\xcb,\xe6\'\x10\x96\xa4A\x86\xc6a\x12\xc3o\x03\xa8?8\xc4\x17\xe2\x10)\xe1F\xf7\xa1\x08\x81#H\x8a\xbc\r \x84)\x14%o\xd6\x17h\x8c\xe7X\x88\xc0(\xecfhD\x009\x18\x86h\x10\x85(\xf6fE\x82\xe1\xe1\x9f\xe1\x10\x9f`\xa7\xff\x0c\x0e\xf1\xdff\x9e=\x0b\x87\xf85\xc6\xefOq\x88\xef\xef\xe7\xdf\x8aCD\xd0\x9f#\xa6\x86%\xc0p\xc5\x81\xcc"\xeajo\xdc\xab\xad\x11\'\xd2\xb8^\x87\xc9\x9f\x03.\xb0\x81\xcc\x8a&\xc6\xd7(\xe1(\xa9\xd9\xf5\x0c$\xea!\xb4\xb0]}\xca\xc2y\x18\xfa\x89\x1a\xa9\xbd(bg\xf00\xf0M\x88w\xbd\xaf\xfel\xc6\xe5\xabq\x88\x7f\xc8\n\xff(s\x1fnY!-\xf9L\x8c\xac\x11\xb0\x04\xa3\xd5\x89\xd5k\xc0\t\xa6u\xd4\xcdK\x04|P\xe6\xb3x\x88(\xf87\x08\xc5\xe1\xbb\xd9\xb3T\xe7o0)\x9c\x88B\x14\x0c\xf0 \x18\xa7\x1d\x98\x03\x94\x8f\xc1\x00K\xb0\xeb\x9dtl\x0e\x83\xac\\\xad\xd4\xdf\x0es}9.\xe5\x1dr4\x15\x10\x02\x8b,9\x88\n:\xc2\xe3\xe9\x94\xf3\xc3\xe5\x90\xab\xf4\xf5\xb0\xee\xe1\x9fAf^\x8cC\xbcI\\\xdc\x07C!\xf8n\x8e\xf0\xda`\xbc\xec\xc0\xf3\\-\xc7\xcd\xa9\xafN\x0ejw;\xc4\x07\x07\xa6:D\xd7\x80\xeb\xd7\xae\xb8&\xf7\xf3\x01\xddo\xd6A\xef\x8b\x9c\x80z\xecY\xeb@\xbeX\xb2D\x0b\xcb\x86\x90\x17YX\xb86\\\xc9\xa2\xed\x00\xeb\xf7b\xe9<\x0b\x87\xb8\xec"\xb8\x84<\xeaV\xb9\xdd\xb13\xe1D\xd6w\x1a\xe0\x08\xa5B\x19s\xdd;;t?\xad\xb6\xab![\xf9\xbdq\t\x14\x04#b\x07\x06\xda\r\xee4\x170\xdeX\xeb\xc4\x8c\xca\xb0\xdc;\xc4.\x92R\x14\xf5\\f\x9d\x9e\xc5\x91\xb2tz\xe057~\xf0\xb0<\x0b\x87x\x93\x89\xdd\xc0M\xe0=\x8c\xac\x9c\x85\xc5\xef\xbd<\xa0es\x12GfX\x0b\xd0u\xda\xf7\xa9\xad\x96\xa3Z\x1f\x8e>r\xaa\xda\xc0\r(n\x1cD:54Q\xac\x1a\xce\x03\xe0>\x00\x1c@k\xe0\xb6R\xe9\xe3\x963\xb4V\x05\x9d\x04\x01\xc2\xd5\x83\xf4\x80\xa7\xe1\x10\x97\xe8v\xc3 \xe3\xc4\xbd5y\n\xf0\xc1=\x93\x9d\x15\x17\x82\xac\xea\xaa\xcc\xa9\x10\xb2)/s\x88\xeb\x85\xb4{\xd8f\'\x80+0-I\x90kg\x976Y\x84\xd1\xae\x95k\xd2uBv\x86*\xc0\xce.D\x8c\xed\xc2\xf3\x15I\xf7\x1e7V\xe0\x83\x9c\xc0g\xe1\x10o\xd6D\xa0\xc5\xa3\xb0{\x99\xbd\x87\x93\x83\xe9\xf7N\x97\xae\x10d\xae\xb7\x07\xa3\x9b\xae\xab\x042\xb8j\xb5\xf1pJ\x0f=\xef\x90\xb7\x05\xec`\xc3\xaa=.\xa5\xe1\xbc=\x8d,{\x80\x137\xa4\x93\xf5ET\xc5%P\xac\xb1\x15s\x143\xde\xce\xda\xf9\x9bp\x887\x990q\x8b\x1c\xd4]\x84C\xd2+\xec\xda\xa5\xd2\x13$\xafd\x07\xd7tZ\xeb\x88\xef-M\xd6@b\x8f^\x99:\xe8\xdc\xcb6<\x8d\x85-\xc1J\x12\xe1\xdc,*E\xce-\x15\xb5\x7f\xc0\x0cD\x8a\xa3\x0c\x98V\xa0g\x10],!\x0e\x19\xd5\x8fZ\xf3Y8\xc4\xe5\xd0\x92\xf8R[\xc2\xf7\x90\x04wlh\xa2\x1f\x8b\x06\xbc\x9c\xad\xc1\xa7s\xee(7\xd3\x9eU\x10)\x91\xf6e\r\xad\xec\xb1\x19\xd0\xa1a"}\xaa\xc5\xc6\x1dwE\x1d\xcb\x01\xce\xad\xafu\xc5\xee\xb5\x96\x05\xddz\xba\xc0]Y\x1e\xd4\x81\xd4\x80\x8axp\x88\xf8\xd3x\x88\xb7\xca\x03&\xa0\xe5\xb4\xdfYS\x9b\x05\xbb\xb0\xa9\x81E\xb6\x83\xb5\xdb0R\xb0\xd4\xd3\xdc>\xf5t@\xc8\xa2\x13\x81\x94sSh3\xce\x12\xce`m\xc6s6\xf2{#\x1e\xad\x1d~(\xb1]v\xa4\x95\x16\x98\x10\xfe`\xea\x97\xab( u\xe3\xa8\xf0\xcf\xd0E\xbf6\x0f\xf1\x96\xf5\x91\xa5\xf9]\xea\xd4{Z\xd8\xa1)1\xa84O\x84\xb0Q\x0c\xee\x84@\xeb\xd1\xaa\x8d6"\x11j\xdc\xd7\xee\x10\xcb[\xc2\xc5B\x80\xc5\x8c]\x1a\x1f7\xcb\tY\xea\xd9\xaa\x07\xaa\xc0\xb1\xf6\x8c\xa8A\xeb\x0c\xd5\xc6\xad\xd2\x07\xe7\xc6\xe6\xf1\xd3a\xf7]<\xc4\x9bO@K\xa1J\xa1w>Q\xf2A\'\xb1i\xbc\xbeB+\xd8E4\xb3\x16x,i\xe3\x9e\xe9\x80\x12\x85\xeb\xd4\x9f\x0c\r\xe6\xb1\xd1\xa1jMl\xa7H\xc6i\x9c19\x9e\xeb\xc3h\x0f_*X\x11\xcc\xd5\xd5j\x9d\xc8G8\xd6Z\xc3\x94\xfd3f\xd0\x8by\x88_i\x19\x84\x96B\x06\xb9\x0b\xe4d\xa0\xc0\xa4\x01+\xfde&\x07Z\x0cp`\x17\x10@\x7f-B\n\xben\xc4r\xcf\x17\xae4o\xf8\x9e?j\xf2V\xd9k"\xac\xad\xcfp\xdc\xa0\x9a\x9cVQs\xed`\xbaW\xf7%\x15\xa8\x11\xb7\xdf\xd5\xfd0a?K\xcb\xbf6\x0f\xf1+\x1d\xc2\xf8\xf2\x0fvW\xdcX\x891\x0f\xcc\xa4\xd2e\x03\x12c\x86\x1e-#\xe6\x8c\xadd\xa6m\xaa\x90\xc4\x05\x92\xcfPJz;\x17\xb7\xa9\x98j\xe7\xf5A\xbc\xe4P\x00\xca\xbd\xe8\xe5\xe6\x883UU\x19R\xbb)K\xe8\x88\xd4\nG:\xc3\xeeQl\xee\x93x\x88_\xe9\x10D\xa9\xa5q\xb8\x93yt*\xdeJ\xc8K4X\x90u\x02l4\xe2\xf4\xa8\xe2}\xac\xc8\x80`-\x90d Z\xf9 \x86\x9a\xb0\x13\xb6i\xe8\x80V\xbbC\x89\xa3\xce\x1c\xfb\x88D/;*T\x8cH\xae\xf61\xadu\x1bh\xcb\xc8\xf4fz\x10\xf3\xfe4\x1e\xe2\xcd\xf5a\x8c\xc2\xc8{\xa4\xb5\x02O\xa47\x8cf\xec\xf5g+\xa2\xa0\x8a\xe1\xaf\x97j\xe4\xab\x8d"-m\'\xb3\xf5\'\xca\xdd\xe13\xd0\xd8=\xe9\x9b\x96\xe2ar\xe1\x1c\x0c\xbf\xa4\xcfz/\xb3\x1b\x12F\xebc{\x1c\xba\xaeO\xd3\xba\x0e Bx\x14P\xfe,\x1e\xe2M&I\x10(\x8a\xdf\xc1t\x82U\x08\x84\x037\xd5\x97k\\\x13\xb1V\xa5\x00QX\xe7 \xb2\x94\x88\x85\x92S;\xaa\xc81?\x16\xbe\xecX\x08\x15\x94\xf4\x0e\xa1p\xad_\x0b\x1b+*\xcbN`\x1b\x82k\x06H\x15\xd5+$\x06\xdb\x04%V\xca\xcf(\xc8\xbf6\x0f\xf1\xab}[\xbe\x8aP\xc4]:d\xe5\xf3\x00H\xb5[\x10jS\x9dp\xad\x90\x15\xa1\xb4\x1d-\xf1\xad\xc3\xd8\xb2\xe7\xf1x\x9c\x82\x8d\xb0\xd93\xfdu\xbf\xb9\xce\x12c\xaeS\xfdZf=\x96\xb8\xc7l\x9dX\xe4e\xbc\x9e\xa2\xb4g\xb1\x94\x10\xc4]|B\x1e%\x05>\x8b\x87\xb8\x1c\x96\xe5t-e\xe6=#\xd4\x8c\xc5\xd9\\m\x98\x81vJ\xf6\x08\x17\xc8,+\x04\xaf\xf4>!\xc4\xd8dH\xe8yC\xf9q\x86\xdb\x86\xb3\xdd\xd1\x93\x06\xe7\x1b\xa1u\x8d\x1d\x12\xcf\\\xd0u\xf3\x010\xa3\xc4\x06\x8f\xb5V\xd7\xd5\xce\xf6\xcc\x06\x0e\x1f\xcc\xfa\xcf\x02"~\x05r\x02!\x16s\xdeQ\xd1\xf8m\xafs\x99\xb8\x02;\x95\rB\x97\xdd\x1c\x8b\xe9\x9c#km\xde\x1bG\x07o\x81\xb0*@\x85\xbaP\tx\xe9\x12aH\xd5Y\xb6\xf3+\xc5_\xd9\xd0\xf6cb\xd3"k\xc6X\xd5A\xae4\xa7h@Bh8>(\xf3Y@\xc4\x9b\xcc\xa5\xa6\xc6\x11\x08\xbe\x0b\xe4~\x9aE\xec\xb6v%z\x7f\x16&`\xdc\x90G\xaeZg2d@1Y\x1f\xe7\x83\xbcFQyT\xb7\xc2\xa5\xb4\xf9V3\x03\xe9\xa0\xba*\x10\xca\xb4\xd8\xef\xe5\x9dr\n\x13x\xc7\xf3\xf2\xcaF\x18\xac=\xc5\x87\xc0\xfc\x19\x01\xea\x17\x07"\xdez\xfd\xe5\xa8,\xdd\xe1=[\xae`\xfc\x88\r\xd5\x86+\xb6\x07T\x8f\xfb\x92\x10\xad\xa4\x02\xa5\xb6\xf3\xc0\x02\x8dcz{B\xc0M^o\xc9\xb3\xe4M\x9b\x9a8l\x0e\x84\xae\xb2\xb1\x01\xd8\xab0\x1fq\xe5l\xe3\xc8\x00#\xa9\xc5I\xcb7\xb3\x99\xf7 \x05\xf9i@\xc4\xaft\x08\xc3\x04\x89\xdc\xc9\xac@\xddZ\xbbB\xc9\x11S\xd9\x02\xa8vl\xb2\x0bY\x85\xad:\x0b\xc7\x8dpZw\xa5\xbdf\xcd\xab\x92s\xbby3k\x96g^+\xc6?;\'\x12M=GP\xdd~\xdd\xe6\xaeo`\x8a\x93\x13\xf9\xc8\xb0\x86=<\xd8\x1d>\r\x88\xb8\xc8\\\xa2$\x82!\xe0\x1dIK@\xfd3l;\x02wU\xa1(:j\x87);z\xf6:\x96 S\x9c\x92\xfc\x900\xda\xb0\x83{\x94,+\xa6?e\xabU\xb2#k\x07\x1d/\xc9\x08G\xe7\xf3PI^\x9e\xcfA.\x15\xc8\x85\xb9\xca|\xa2]\xabGI\x81O\x02"\xde\\\x7f9\x03\xd8\x12-\xef\x9a\xe0\x820\x1cK\x18\xc2\xc6\xce!\xd4\xb9fS\xee\x913\x0c\xc0\x10e\xa6,\xc9L\x973L\x07\xf2ehI\xcaZ+\xdb-i\xb5[\x86\t\xaf\xabm^1\x90\x92jW\r+\xb7\xb3\xbcI\xfa\x8d\xc1\xa4\x00\xa6\xe2\xe8\xcf\xb2\xfe\x8b\x81\x88_\x81\x9c\x04\xe1\xe5\x0c\xdc\x05\xf263\x0bE\xe8\xd2\x06\xd5\'\x9c\x19y\xfd\xaaq\x9dHi\xf3|\xd0\xaf\xf9^\xbb\x82\xdb\x8d\xb0\x87\xb8\xdcL\xb6\xd1z8x;\xb2\x1b\xacK\x04\x9f\xf7\x03|\xe0y"#\xa1%\x08c\x99v64\r\x1f\xb7\x1d\x10<(\xf3Y@\xc4\x9b\xcc\xa5\xe2\xc3\x11\x14\xbd\xa31\x9b\xbb\xd0VA?W\xc66\x82DOvrG\xe5E\x11\x06z\xa6Re|R\x9a=\x14f\x06p:\x0c\xb45E%\x98\xc2U\xcdy#\x88\xca\xf5n\r\xb3[<\n3\xfb\x14"[n0\xaf\xd6\xd5EL\xe6\xbd\xb8hO\x03"~\xb9>\x8aR\xc8\xfd\xe75\xa3\x08M}\x9e\xf8\x1cVP\xc7\x1a\x1e\xebZ\xc8zg\xcf\xda\xb6b\x0e\x86\xd1\xf5\x81p\xbe\xb6\x11&\x14\xf1\x8ee\xe0D\x0e\x93\xe0\x8c\xe3\xdd\x08\x04\x02\x13q\x87B\xed\xb5X0[\xc1\t\x8c\xf6\xa8\xb0\xc1\xb0\xea\x1e\xfdH\xeaI@\xc4\xaf^\x1f\x85(\x84\xa4\xee\x0e\x8b\\\xe9~{\x9e4\xc3\xc3=\xb6Wgj\x7f\xe1/\xd0\xac\x10\xd5&u\x91T\xad\x05&\xees)m\x80\xdd$\xfa\x13\x926\x90\n`\x05\xb9\x06\xeb\xd2\xdex<\xbe\xf6=]\x07i!\x032(i i_\xfa\x0f\x96\xaa\xcfB"\xde.\xa8\xc8%\xa3#\xe8=$\x14\xb6\x0fH4\xf3\xe5\x95\xb4\xe4\x8d\xda\xf9\xda1\x84%Q"c\xe7\xcc\x05\x87=O\xd8\xb0\x84\x80\xb2\x97_\xa5\xac\x1d\xedv\xb7W\xef\xcb\xd3{\xd1\x9c\xa4\xcf\x96\xfe\xdf\xbc\xa5O\x87\xf2\xbdhK_\xbf\xce\xf7E\x03\xbe\xea\x94\xbe|\x9d\xef\x8b\xf5{\xd5\x96\xbe|\x9d\xef\x8b\xe4{\xd1\x96\xbe~\x9d\xef\x8b\xd3{\xd5)}\xf9:\xdf\x17\x85\xf7\xaa\xf4\xf4\xf2u\xbe/\xc6\xeeE[\xfa\xfau\xbe/\x82\xeeU#Q_\xbe\xce\xf7\xc5\xc7\xbd\xca\xf1_\xbe\xce\xf7E\xbf\xbdjK_\xbe\xce\xf7\xc5\xb6\xbd\xaa\x88z\xf9:\xdf\x17\xb9\xf6\xa2-}\xfd:\xdf\x17\x97\xf6\xb2\xba\xf4\xd5\xeb|_\xd4\xd9\xab\xb6\xf4\xe5\xeb|_L\xd9\xcb\x1c\xff\xd5\xeb|_\xc4\xd8\xabN\xe9\xcb\xd7\xf9\xbex\xb0\x97]A\xbfz\x9d\xef\x8b\xf6z\xd5\x96\xbe|\x9d\xef\x8b\xe5z\xd1\x96\xbe~\x9d\xef\x8b\xd4z\xd5\x96\xbe|\x9d\xef\x8b\xc3z\xd9\xad\xfe\xab\xd7\xf9\xbe(\xab\x97\xddD\xbdz\x9d\xef\x8b\xa1z\xd1\x96\xbe~\x9d\x1f\x84\xd4\xd3\xd7\xf9\xbe\x80\xa8W\xddD\xbd|\x9d\xef\x0bwz\xd5\x07%/_\xe7\xfb\x82\x99^VD\xbdz\x9d\xef\x0bUz\xd1\x96\xbe~\x9d\xef\x0bDz\xd1\x96\xbe~\x9d\xef\x0b3zU\xc6\x7f\xf9:\xdf\x17D\xf4\xaa\xf4\xf4\xf2u\xbe/D\xe8U\x8e\xff\xf2u\xbe/\x00\xe8U[\xfa\xf2u\xbe/\xbc\xe7U[\xfa\xf2u\xbe/x\xe7e\xa5\xfe\xab\xd7\xf9\xbe\xd0\x9cW=m\xf2\xf2u\xbe/\xf0\xe6U=\xfe\xcb\xd7\xf9\xbe\xb0\x9a\x97=\x0c\xf9\xeau\xbe/h\xe6e\x0f\x96\xbfz\x9d\x7f\t\x87\xf0O\x95\xda\xbf\xc2!\x94\x7fp\x9cw\x86\xc4\xbc^\xd87Ab^/\xec\x9b 1\xaf\x17\xf6M\x90\x98\x87\x84\xfd%\xe4\xc87Ab^o\xb1o\x82\xc4\xfc\x07\x82\xc7\x07\x12\xf3\x12H\x0c\xfc\xdf{\xfe\xe7\x90\x18\x8a\xe58\x82c\x88\xdb|u\x01Bx\x94Z~N10\x0c\xb1<\xc8\xb3,\xc5\x82$\x87!0\x8d24\xc420\x03\t \xc2p\x18sCa\x80\xe8m\xbe\xc7\xcf\xf9\x07\xb71\x80\x0c\x0e\xdf\x18\x1a\x02\x05\x12(\tA\x08I# \x8a\x10\xecm\xea+N!\x02\x83\xd0$AA$H\xd0 \xc5\x80<\x03a\x18\x8d\x12\x14\xc9\xf0\xe4+!1\xff5I\xeb\xb7\x1f\r`\xc0\xffF\x11$H\xa2\x7fF\x88y\x7f\xbc\xce\x8f\x081\x0b\x10\xf3OM\xda?\xcd\xc4^\x15\x17\xbe\xad\x82L\x06\x89K&\x98\xceF\xb5Ir\xc0\xa7\xadyv\x0e,\xcf\x9c\xc5}C\xcf\n\xee{\xfe\x16 x\xeaz\x12W"\xee\xd4\xf5\x11\xd97\x996]\x18\x01\x8c!w\xb7"c(;\xeb\'R$\x1f\x9c\x8e\xf9,:\xccW\xf8!\x96x\xfc\x83\xaa\x03\xd0}\x96i\xb7!Y\x13\xaak\xe6d\x86F\xa4\r\xd2]L\x88\xd6i\xcd\x9bl\xb2ql\x9a9\xbb\xcaN\xa9\xda\x18\xd2\xb7N\x8c\xa4py\xdd\xe3\xa702S6o\x84\x084\xb1\x8b\xbeI\x8a\xb2K\x82\x93\x902o5O\xf1it\x18\xe2o\x10\xb4\xb4Q\xcb\xeb\xdc\xd55\x19\xbd\xb1{%?\x8a\xddq\x9e\x956\xd4\x03V<\x12U\x84\x19\xfef\xbbV\xd2S\xd1c\xb5\x8b*3c\xe7\x06-6[\x7f`\xb7\xd5\tp\xfdS\xb6\xc9\xb99O%\x91\x178\x9c+\xfb\xc5\xf39\xa9l(\xf9A\x12\xc6\xb3\xe80\xb7\xc3\x02c\x14\x88\x81\xc8\xdd\xb0\xe8\xd1G\xc5\x1d\x14\x9e\xc9\xc6+\x00\xb6\xa3\x86Y\';x?I\xa22\xf5\xc7\x8e\xb1\x85\xd8]\x03\xdd\\\x1fZ\x1d\x8e\xb7\x87\x01YM\xe6\xae\xd8\xf4\x10(n\xfa\xe3\xf6r\r\xec\x9d\xce\x8a\xde\xb9A`\xb9\x8b\xadD\xec\x1e\xf4\xfb\xa7\xd1a\x96(\x0ec\x18\x81-e\xcc\x1fe\xe6\xe4U\xd5V\x93\x9e\xc6tY\x80\x1e\xb8\xd1,\x03\xd0\x93\xfc0N\xc1\x98\x16\xa3HL\x11\xa29s\x8aV\xd0\xdc\xb6C\xd8tM6\x8f\xc7\xb0\xa1\xe3\x83\xc7\xed\xd6\x0c\xa9\xda\'\xb9\x05t\xf4\x02u\xc6\xb6\xd9\x84\xf3\xcf\x92\xd5/N\x87Yvq\x89\xbc\x18B\xde\xb7m\x9b\x9d\x84\x15\xfa\x115\x19\x04_/\x8d\xe1l\xe9\xae\rQ|2\xc2\xd9\xb8!\xe8\xb9\x1a\x951\xf2\xd4R\x80p\xce:\xd7\'\x9d\xae\xd0Z"v\xeb`\xf0eKE\xf1\x90c\xba\x8c\xbe"\x11\x822\x87\xe39\xdf\xf9\x8fR\xc4\x9eE\x87YdR7\xce\n\x82\xde\x8f\xdc\xb64\x80\xba`\xf5yi\xde\xb2\xb58\x12*\'\x91BFt\\E\x9d\x91j\xa9\x14{\xe8\xbaQk\x0b\xd9\x04W5\xb5\xc8&.\xd7\x133[\x17\xc3]\n5\xe6\x02\xb9\x8d\xb9I+\xdc\x92\xd0A<\xc6\xb9\x89\xcc?\x0b\xa0/\xa6\xc3\xdcd\x82\xd4R\t\xc2\xf7\x98V\x81Q\xe4\x8d\xb6Z\xa1\xb2J\x95\x97X\xb7\xe4\x1df\xe3\xab\x06\x08\x04\xb8\x94M\xda\nm\xa8\xbe\x96\xa2\xc2\x9dX\xbf\xd1\xae\xbb\x92m\xe3D\x1aj2\xda\xd8\x80\xe6\xdc(\xb8\x9e\xa8\'M\x83\x1bg\xb5\x02A#/\x1e\xb4\xe6\xb3\xe80_)\x9fX\xeaq\xea\x1e\x82cj\xf38\x87\xccJ\x19\xea\xb0?\xa5\xe7\xa0`r\x0e\x90<\xb9\x16\x07\xc3\xc3W\'\x7f\x82\xce\x1e\xa3\x12#\xa9\r\xc1\x9e\x9d\xb5-\x16\x03\xd5a\x9bYP\x80\xcb\xdc\x114p\xce\x14\x92n\xef\x80E\x0c\x88\xa3"\x84?\x8bp\xbf8\x1d\xe6\x16@\xa1\xe5\x1b\x96.\xf8\x8e\x93\xc8N\x92\x84\xabxW\x9e\x83\x19\x117\xec\xdc\xa5~tNss)\\\xb6\x96{\xad\xaf\xe2^\x05(\xa3\\\xcf\x83\x91\x1b\x16Q\x93\xccI\xb4\xc1\x04\xd5\x8f\xe1\x92\xaf\xaetx8\xc5\xad\xb69K\x02\xc2\xfa]&\xa1\x0f\xa2\x84\x9eE\x87\xb9\x1d\x16\x12"q\x0c\xbb\x87\n\xb0\x8a\x16\x05a\x87\x9b9]\x8d\x92FZL\xeb\x8e\xbc\x029\xbb\xfdn\xb5t\x01AF\x1d\x1d\x94\xd0\xf9\x14\x90,wH\xc7\xf4\xea\xf5z\xe7\x0c\x90\x8d\xec\xc4\xd3y\xe5\x9e\xa7\x0b\x8e\xaf\xd6;\x80\x08g\xf2\xdcj\xd6\x9a{P\xe6\xb3\xe80\x8b5\t\xea6\xa2\x1b\xbfs}\xd5\xa5\x01MJ\x0f\xb6h3\x132\xb0}\xb9\xcd\\\xb5\xeb\x07)\x80&^\x13\x80c+\xb6\x99M\x9fA\xbd\x83\xc0\xb3\t\xec\xec\x9d\xd2\t\x1c\x01\xc6\xec\xbcM)5\xf3f9@JB\x14\x020\x93|\\\x99\xe3\x07{\xb7\xa7\xd1an\x11\x8e\xa4\xf0%Z\xdc\x05r\x9ajh\x9f;\xc9\t\xeeD%oW\xe7]\x11*n\xb9\xa9-\xc88\x97\xd4yP&\x15jYq\xc8[\x81\x9b\x19\x07i\xf4n\x97\xea\xed\xd5\xcf\xe0\x94\xae\xf4\x13}\x14z\x8f\x17\xc1.\xdf\x19\xe4\x98-N$\xfe\xcc\x9a\xbf6\x1d\xe6\xcb\'nG\x0e\xbb\xab\x10\x03I\xaaud\x040\xca<\xdbg\xac\x14\xdb0\x1e\\\xd8>\x89\xdc\xc94\xb1`w\x99\x1e9\x15d \x9f8K\xd7\xe5\xa4Duy\xdeTsv\xe9x\x00\xecI\xbc\xca\x98i\xd3\xa0-\xad\xae\x08\xd4\x0f\xf5&\x1f\x8bf$\xd1\x13:\x84\xe4\x105\x16\xff\xa0O<\r\x0e\xf3\xd5\xea\x13\xd0\xd2\xd5\xdc\x1d\x96\x1c\xef2K\xd5\xe4\xcb8\xf7\x82\xb5x\xfeqo\xc0u\xa4m\xc3F\x98\x87\x8b5\xcc0z\x11\xb5\x15\xc5\xc8\x18\x95\x95\xb9O\xa7\x04\xe3\xb3z\xa0&\x9e\x1f\xbb\x11P:\xb3\xca\xec\xf8\x0e\x1d\x06\x14-\xa9"`\x85\x07/l\x9f\x06\x87!\xfe\x06Q\xe8\x12\xc7\x7f\xd0\xbd\xe5\xa9\xc2\x16D\xa59]\xb8\x99\xc1i\xb3S1}{U|\x9c\x07\xd9\xa6\xdb\'\xa6\\\xf4\xa4Q\xee\x9arZ\xabk\xd5\xac\xd0\x13\xb0u\xab\xc3\xe2.[n\x9fRpB\xbbe\xd2\x9f\x0e\xa5\x8a \x97p3u\xea\xf9\xc1J\xf5ip\x98[\x84[2\x1c\x81\xde[\xf3\xa4\xadX\xa6)\xaf\'\xb2\xba\x1e\xdc\x1e\xe1m|\xd0*\xe1\xba\x11\xbb~_\x0cD\xb6t\xd7\xd9\xe4Gm\xc3"G\xe4<:\xdc>\x9a\x08\xea\xb4\xcf\x14N\xc2\x065\'\xbc\r\x8219k\xa9\xd8x\xa4\x84\\v/?\xfb\xc8\xe8\x17\x87\xc3\xdcvq9-\x14\t\xdf\xdd|\xe7\x15p\xe4\xd0=\x97\xa5\xbb\xbcr\xa9K\xd3\xd2}\\\x9e\xa3\x9a+7!?Yp`\xeaq\'\x07Y\xadn[\xcc\x86RqwB\xd6\x87\x15\xe4\xcap\x98\x0f\x1c\xe6\x03\x87\xf9\xc0a>p\x98\x0f\x1c\xe6\x8d\xd7\xf9\x81\xc3|\xe00\x1f8\xcc\x07\x0e\xf3\xce\xd0\x95\x0f\x1c\xe6\x03\x87\xf9\xc0a>p\x98\xb7]\xe7\x07\x0e\xf3\x81\xc3|\xe00\x1f8\xcc;CW>p\x98\x0f\x1c\xe6\x03\x87\xf9\xc0a\xdev\x9d\x1f8\xcc\x07\x0e\xf3\x81\xc3|\xe00\xef\x0c]\xf9\xc0a>p\x98\x0f\x1c\xe6\x03\x87y\xdbu~\xe00\x1f8\xcc\x07\x0e\xf3\x81\xc3\xbc3t\xe5\x03\x87\xf9\xc0a>p\x98\x0f\x1c\xe6m\xd7\xf9\x81\xc3|\xe00\xffC\xe10\xff\x94\x03\xff%\x06\xe1\x0fu\xdd;\xc3a^/\xec\x9b\xe00\xaf\x17\xf6Mp\x98\xd7\x0b\xfb&8\xccc\xc2\xfe\nj\xe4\x9b\xe00\xaf\xb7\xd87\xc1a\xfe\x03\xc1\xe3\x03\x87y\t\x1c\x06\xf9\xef=\xffs8\x0c\x8c\xb3\x18D\xd2\x18\x8b\xd2$\xc9\xe1\x08\x04\xa18\xc1"\x1c\x8aB\x0c\x03\x13\x1c\xbc\xbc>\xc30 \x8dc\xb8\x00\xd3\x08G\xa3\x0c\x8c\xb0\x14O\xc2\xcb\xb7\xc2\xb7\x89\x10?G\x1f\x084\xcc\x800\xc8\x82\x10\x8a\x820*@4\x8eC\x10B\x13\x14\x04\xf20\x83\xc0\xc4mv\x1b\xc3\n\x04\x84P\x08\xc8\xe34H#\xcb[\x80<\x87\x10\x18\x08\xbf\x12\x0es\xdb\xa1\x9f\xc1a0\xe4o\xd8b\x1f\nG~\x9f\x92\xff3>\xcc\xfb\x93u~\xc0\x87\xa1\x10\x98\x17P\x08\\^\x17\xa1x\xfa\x86\x0e``\x12\x87\x84\xc5\xfa$\x89\x13\x8b\xc5\xc8\xe5\x04\x907K\t\x8b\x89@l\xb1\x89\x80!(\x0b\x13\x8b\xb1nc\xe3?|\x98\x17\xf2a\xb0\xc5\xf3)\x81_\xfe\x83\x04\x98\xa49\x88\xc4\x05R\x80 \x9c_VK"\xb4 \x08\x8bMn\xafCc\xd4mj\x0bJc4\xb5x-\xb6,\x9cb\xe1\xdf~q>\xcc\xbfM-y\x1a\x1f\xe6\xb6e\x7f\xca\x87y\x7f?\xff^>\x0c\xf1\x931\xed\x17n"N\xeb\xd3\xd6\xbd\x90\x8c\xecq\x85\\c\xf0\xc0\x19\x92`\xe1\xa6\x92b\xe7\xb6a$\xba\x9a\xf0b\x83&qA\xb7\xb5?\x86G\xcf\xef\x04\xcd\xba\x06\x97\x10\xb86>\xb0\xe9\xf3\x83k1xW\x9d3\x02"\x7f:\xbf\xfc\xc5p\x98[J A\x9c\x84\x08\xe2n\x1c\x17u\xcc\xdb+\xbe\x8a\xa2q\xa5\x9cj\xb9\xa4K\x1d\xacJ\xb7r\xfe\x7f\xf6\xee\xac\xc9U+K\x18\xf6\x7f\xa9["\x9ay\xbad\x061\x8aQp\x07Hb\x9e\x85\x84\xf8\xf5\x1fJ\xdbow[>\xd5\xa5\x0e\x9d>\xe9\xfa\x14U\x11v\xd9\'S,\xd8{\xad\xbd\xc8\xac\xf5\xc8\xa7K\xc4\xef\xe9\xc1[\x982\\\x05&\xedp\xb7\xcc\xa2\xa8\x88\x86K\xb3\xc0\xbb\xd3|\xdc1\xf6\x02\x9dZ\x1d\xedH\xb0\r\xf4\x90\xa5\xb9\x1ar\x16\xe4U%\xe2M8\xcc\x16\xe6\xa3\n<\xcf\xdeU\x92\x02\':\x0e\xedL\xf0\xba*\x96(\xaaK\xc1\xa5mJ\xf8\x16\xd7p\x8d\xa9\xda\x11[\xcf\x91\x86J\xcaUHmB\xe5\x8e^\x1b\x15\xb5\xe01\x07\xb5r\xf8v\xe7\xf4s\xa6k\xfd\xc4\xad\xb9\xa6\x8f:[\x8e?\x1a\xdc\xf6\x93q\x98\xc7\xa3\xdcV\x00\xb9}\x8f\xe7\xc9j\x80\xd4\xe5\x1d\xbc@\x1a\xc2\xb6|%5\xd5\x0cC\xb3\xea\xab\xbe;\xd6\x88G\xee\xe9sJ\xc8\xc8\x0eq\xcf\n\xc3\x13Iw\x84\xa0\xb4;!\xcb.3x+3\xd9\xd9Uv\xde\x91\xee-\xfb04\x96\xadUg\xf3{M\xdc\x7f\x97\x0e\xf3\xb8\x8b4\x89oO\xe2\xe9&\xce\xc2l8,\xa6\x17\x994.-\xd5\x8d\x0b\x8c\t\x92\xcb8\xa5s\x18v\xd11w\xd9\xfby\n\x9cu\x12\xe2\xeb\xd9+\xf8\x9b\xc3\r\x86Efr\xcbH\x88\xd7\x16g%C\x8f\xc7+1\x03C}N\x9b\xf0\xe09/\x0e4{\x17\x0e\xf3\x88r\xab\xd6\x04\xb9%\xe8?\x879`.\x84\xf7\x92t;\xcdF\x89\xc5Lj\xde\xf7l\xaa\xea\xee\xddhq\x95\xa7\xf5c\x91v\xc2\x8c!n\xbe&\x98\x15\x9e\x02\xd9\xcds\x91CK\xcb\x96P\xd5\xcd\xb8"\xbeZ\xa0t\x01\x83\xe3\xb4s\xcb\xd2\x9c\xc6\x17\xe7\x8c\xbe\x0b\x87y\x84\xf9\xf8Zz\xdbEOO\xb3\xe0\t@\xdf\xca{<@\xca\x81v\x82U\x01Py;\xc0\x88\xb9Wg\xda\xc5\x02\xf9\xd5\x13\xb1f\x96\xda\xe1\xa84<\x8f\x16\'\xc7\xc5\x02\xe5\xc8\x05^\x7f\x97\x8f\x84mQke\xd4\x08\xbb\xaa|\x11\xe6]\xa3\xbf\xcaB\xbdI\x87\xf9\xf3\xb9\xfe\xbf\x86\xa9\xdbH\xce\x9a\xc6\xedN\xc0\x8b\xd8\x05\xa6\x9eh\xe0\xe1\x14*\xbc4\x9c\xa8y$\xafBw_\x8f\xe5\x01\xca\x88\x04Y\x1ez\xc6\xe7a%\x82\xa2\x95@\x04\xbc\x91/:8\xef\x02b\xbe2\x1c\xf5x\xadD?5\x1dq0\xea\xca\xe2U\x88~N\x84\x04H\xfd\x95\x85N\x12\x01\r\xa9(^\x8b\xc3\xd8\r\xf7t\xda>\xaeWE]\xde\xddR\xb1\x80\xed5\xe1\xb7\x15P\xcb\x88]\xca\x8c\xaf\x06\xd3]\\\xae\xb5\xdf#L\xb5\x84l\xd4\xfd\x08\xc3\xf8{\x031_=\r\x8ab\xc8\xb6\xb9\x9e\xce\xfb7j\n\xd4I\xec\x8d\xac\xedw\x80\xc3\xedF\x88\x96YU\xd2\xd0\x18\x87lH\xc2\xf3.\xb6\xf6\xa7\xeb\x11\t\xa2\x8b\x07\x1b\xc7\xbaUj\x03K\xf4B\xcb{&\xf5@f\x16\xe9\x80\x03\xb8\xb8\x93pN\x13%\xce|\xf1\x0c\xf7. \xe6\xb1\xf5!d\xab\x134\xfd\x94@m4\xb1\xf6\x8b\xa1\xab\xe2]/\xc3\xad\x9a\xd5gY\xc0[\xbf:QRy\xb96T\n\x0bs\xe8\x1f\xdc\xca\xbeFT\xd6\xf0\xe0z\xde\x01bOT\xb7\xe2J\x8a\xedn\x06\x95\xd8T\x83\x14\x9d\x1d\\i4C\xb3\xca\x17G\xe2\xbf\x0b\x88yl}\x9c\xa2p\x02\xa2\x9e\xc2\x14\x13D[d\x01\x0e\xeb\x85\x905^)\xef\xba\xa0\x8a\x89\xe6\xb6\xfb\xf1Y\x92\xef\xbb\x05]\xfazw\x98E\xb0\xd78<#\xf1\x98C\xe6\xec\xba\x07\x98\x15#\x0e\xb5\xc3\xd9{j\xccU\x98\x99M#g\x9c\x08_3\xfb\x80\xe9k(\xb9\xcb\xe1J\x7f/\x1a\xea]>\xcc\xd7]\xdc\xb6\x15M"O\x8b%He`\x15\x82\xd5?\xba\t\x83\xf6\xe4E\x85g\x89m\xa3\x9b\x9e\xb0\x97\x98Ibsj\xaa\xdb5\nUp\x12\xc7s6\x9f\xa7\xf1\xbcL\xa1\xe7\x82\xa2\x92\t\xfa\x1e\xf7\x85m\xab\x0f\x92\x02\x9c\x0c\xa99\x97e\x7fz\xb1kz\x17\x10\xf3\x95\xe0\x10l\xab\x13\xf8\xd3b\x99V\x85W\xdb@\xf6\xf7E\n\x8eA~\xf1\x0f"y\xd3\xad\xac\xdb\xd6\xf8\x10U^N!c\xe6$\x00\x82\xf5K\xee\x9e\x8a\x90\xb5\x89\x9av#\xe3\xd4\xdf\xaf;\xa5Q\xc0\xe4DW=\xecV\xd0*\xebA\x8bm\xd7\xf8Z\x98\xef\x02b\xbe\xaa!\x0e\xc1\xf8_X\x1b\xdbi?\xa0\xed\xf9~r\x0b8]\xc4+[\xee\xeb\xc0\x15j\xf3\x9c\xd5\xa7\xf4\xd4d\x89\xea_\x87\xb1\xaf\xa9K6w\xaa\x14\xc4\xa1p\xf4,\xf7"m\x87\xfe\x03\x85^\x0e\xed\x9aXnw\x83o\xba\x1d\xd1\x11\x12Z\xcb\x8bO\xf3]@\xcc#L\x92\xa2q\n\x86\x9eN\xaa\x96\x14\x9ci\xe6d\x81\x1d\xebM\n3u\x89\xbd\x90\xbd\ri\x83W\x97\x8aa[so\xfa\x03\x1e\xaan\x0c\xcb\xa8-\xde\rIX\xbb\x92\x01v\x87\xd0b\xac%_\xa9\xa3\xb2\xe0\xc5$Q\xd7\xea\xea\x84Sr\x7f5\xc3\xbd\x0b\x88\xf9-\x8fC4\xfd\x17mM&\x98\xc1I)R\xc9\x1b%\xadDXX\x02\xd6\xde+\xc3jZ\xc1t\xcbZ\xc2l\xd5\xddB,\xbe[\xcd\x8d2\x8e\x1e|\xc8vi\x00H\xcbn\xbc\xd6V\xe4q\xb0ep(\x7fc\x11\xd5(\x8d\xa4\xc6\x10\xfb\xc5&\xf5]@\xccW\x988\x8a\xd1\xd4\xb3^l\x0e\xd39\xf1\xa3\xf6<6\xd7\x95\t\xe1\xd3\xa9n\xc7 A)P\x8c*^^\xe5R\x8b\xe9\x0bL\xec\xbcI\xbe\x9d\x97\xbc \xd9$\x95\xdc\xe3\x1e\x80+p6\x92\xa0\xb6\x1d\x7f\xbc\t\xb3\x9d\xe3<*Y\x1d\xba\x86\xdf\x8bg\x7f\x17\x10\xf3\xf52\x13\xc6\xa9\xad\xb3y\xda\x13\xb8\xa8\x109\x0b\xcd6\xc3\xa1jh\x1dG\xc6\x9c\xc8*\x9dzz;\xa41\xabv8c\xf0b\xc5P\x87\x16q\xc6\x9b\x1e*t\xc0|\x84\x9bSUn\x9b\xdf\x14I`\xcbm\\\x03\xde\xda\xaba\xd6\\\x00\x1fS\xe0\xc5\xb3\xcd\xbb\x80\x98G"\xa7!\x02&\xe0\xe7\xb6F>\x9e\xcfvr\xac9\xa9>\x8bm\x03\xe8\xc1\x80\x17\xcb\x8e\xd2\x0cy\'hDN\x83d\xef\xde\xa3\xfc~\xc7\x12\xd3\x04\t\t\x16n\xa7{\xb1j\xa5wJ\xdcZ\xc7=\x97\x91\xc2Ke$\t\xe0k\xb1\x1d\xa3\xc8\xf8\x8b\x80\x98\xaf&\x15~\x90f\xcf?_\\w;Yd&\x8f\xaff1I\xc6\xd8<.\xa4\x9f\xedG\xc5\xac\x8cn\xd8J.eh\x80\xae\x13\xa5D\x9fx!\xf7\xa9\x9bdL\xea\x9d\xa2v\xc6P\xee\xc4P\xdf\xdf\xc1v\x08!\xdfN\x07*\x9c\xa0;:\xee~$\xa7\xfcd \xe6\xebgo[\x8a\xa3h\xfa\xa9\xef\x80\xbb\x0bi+\xc7\xba\x92\xc95\\\x140;\xb2\xf7\xfa\xb6\x1f+\xfc>\xee\x83U\xbe\x0c\xd6|^e\xeef\xdf\x9dKw\xa9\xf0\xc1\x1e\xd5+\x94\x86\x00\xab\xe2\x0b\xech(%z-\xa0\xa9<\xef\xc9\x9dh1\x8eD\xff\xe8\xd5\xf4\xdf\x1b\x88y,\x96\xc79\x98\x86\x9f\x99\x9df\xa6\x93\xa3,\x86gG.\n\xb0\xef\x1f\xbft\x82\x12Gg\x1d=\xa1*=\x9e!\x85\x9d\xc1\xdf\xf5x\xf2t\xe1\x86\xad\xfb\xc6vfL\t9\xd3\xab\x87T\xb7.\xa1\xe5\x1a\xe0\xbeI\x92\xdd\xd9\x96L\x87\'E\x8eg\xfe% \xe6\xebW\xaa>@\xcc\xbf\xfc\x9b\xea\xff>@\xcc\xdf\xc8\x89\xf8\xd0\x1b\x1f \xe6\x03\xc4\xfc\xffq\x95~\x80\x98\x0f\x10\xf3\x01b>@\xcc\x07\x88\xf9\x001\x1f \xe6\xfb^\xe7\x07\x88\xf9\x001\x1f \xe6\x03\xc4|gx\xe5\x03\xc4|\x80\x98\x0f\x10\xf3\x01b\xbe\xedu~\x80\x98\x0f\x10\xf3\x01b>@\xccw\x86W>@\xcc\x07\x88\xf9\x001\x1f \xe6\xdb^\xe7\x07\x88\xf9\x001\x1f \xe6\x03\xc4|gx\xe5\x03\xc4|\x80\x98\x0f\x10\xf3\x01b\xbe\xedu~\x80\x98\x0f\x10\xf3\x01b>@\xccw\x86W>@\xcc\x07\x88\xf9\x001\x1f \xe6\xdb^\xe7\x07\x88\xf9\x001\x1f \xe6\x03\xc4|gx\xe5\xdf\x1a\x88\xf9o\xd9\xe5\x7f\xa4\x10\xfeT1\xbf3\x10\xf3\xf3\x03\xfbE@\xcc\xcf\x0f\xec\x17\x011??\xb0_\x04\xc4\xbc\x16\xd8\xff\x86\x1b\xf9E@\xcc\xcf\x7fb\xbf\x08\x88\xf9?H\x1e\x1f \xe6\xa7\x001\xd8\x7f\xde\xf3\x7f\x0e\xc4\x08\x0fp\x00\x17`\x9e#8XDq\x02\xfb\x1a\xf6\xc2\xf1\xbf\r*\x139\x92" \x8cC\x1fS\x8aq\x92\x84x\x88FY\x86b\xd1\xc7@\x93\xed\xef\xff\xf1\xcf\xec\x03\xea1E\x86\x82H\x82\xd8\xbe\x0c\xa7QZdp\x14\xa5XQD\t\n\x7f\x8c>\xa3y\x9a\xe0\x11\x96\x83!\x8a@QV\xdc>\x85\xe1y\x88gD\x98#\x84\x9f\t\xc4\xfc1k\xf5\x1f\x7f\x1e\xc0\x80?\xe6@\xfc\x07L\xa0\x04\x8e\x13\xf8?\xf3a\xbe?\xae\xf3W>\xcc\xf64D\x0eaX\x9a\xc4D\x1e\xc7y\x08\xe2\xa9\xed\xee\xe3\x8c\xc8S\xe4c\xb68\x82R4O1\x10D\xe2[H\xc8c\x84\x15!l\x9ba{\xee4\xfd\x98\xc9\xf7\xf1a~\xa2\x0f\x03\xe1(\x8bn_\xba-)\xe21\x16\x1a!\x18\x02}\xac\xb0\xed\x8en\xbb\x92\xc3\x19\x82\x86p\x86\x13\xb7\x0fB\x10\x86fE\x96x< \x8e\xd8>{\xbb\xf0\x7f\xfc\xcd}\x98\x7fy\x18\xfc\xdb|\x98G\x0e\xfb\xa7>\xcc\xf7\xdf\xe7\xbf\xd4\x87\xc1\xb1\x1f\x8f\xdd/\x15@\xea\x89\xdb\xc5\xb8\xa3\xc7!g\xc8\x18\xb7G\x11-\xe4\xbd\x13\x9e:\xf6\xde>\xfe\xdf\x86\xbal\x0b\x8e\x0f\xe7\xb1\xcb\xdb@\x991\xb8\xee\xb0\xcb\xb9\xd4\xb2\x03c\x02\xee\xf5 \xe5t\xa8\x00\x88\x9eK\xc0"\x06\xda\xf1\xc5!uo!b~/\n\x14\x86o\xbb\xfd\x99\xc4\x18|\xf5\x02\xae\x91\xc0s\x85\x90g\xa1\x06\x1e\xad\x05w\xcd)>H6\x02R\xb4\xeaP\x82\x84\t\x96\xde\x1a\x07\xdeds\xba\xdb\x8f]iE{!\x0b\xac\n\xec\x95"\xd4\xa16\x00\xfb\xc3\xa8\xb6\'Z\x0e\x8f\xcb\xab\xb3\xf8\xdeB\xc4\xfc\x16\xe6\x96\x08\xffbn,\x9c;!\xa6p;\x9a\xbc\x9dh\xb3\x8d\x84R\xef\xce\xb54q\xc4aB-\'\\\xb0\x04\x13\xe2\x810\xa5\x98O\x92\xd9\xf5\xba\xd0\xa8\\\x13\x1c\xc0a\xac\xaf\x95\x9b\\\xf2\x94R\x82\x11\x08\xef\xfbQ\xa4/\x03\xe4!/\xceR~\x0b\x11\xf3[\x8c\xdb\x97\xc38I\xc1O+\xb6+w\xb1D\x98\xbb\xc2\xb7l\xb7\x93\xd53i\xe9>u\xbam%\x92\xb9:K`\xe8\x1do\x1c\x10\x93\xd3\xc4\xb5\xefq!\xb8.[\xc6\x13\x05(\xe7\xd0"\x8drJR@\x12\x0f\x02\xe4\x9ek\xb3\x7f\x0f\xae\xc9\xa8\xffh\xb6\xda\xdf\x98\x88\xf9\xfd.\x92\xd0vB\xd8\xf2\xd0\x9f\xef\xe2\xadV\x89\xa13n\xae(\rvj\xe9}\xcf\xe0\x0e\xbf\x95=p\t\x05\x89d\x0f\x17\x04\xef\xef<\x89\xd2\x8d~\xdb\xd6\xcd\xb6(\xb6\x964\xc6\x9d\xf3\x8e\xaa\xccH*\xb1\x92\x825\x8a;U.Y\xa0\xfb\x04Lb\xebE3\xe9-F\xcc\xef\xfb\x9e\xde\xce\r0I<\x85\t\x1b8\xb3\xeb\xfb\x1aD\x81.^\xc0+t\x9ds)+\xf2-\x8d\xce\x8b\xa8\xc6Ad\xb2\xa1\x15E\xa7\x12\x99\xcfsv\x14\xd8!L4E\xe2v\xb7\xa5\xbd\x88\xf3\x05\x04\xfd\x90tI\x19J\xaf\x18\xe3\xdb\xe8"^^\xdc\x13o1b~\x0f\x13\xde\x8e\xe8\x08\x06\x13\x7f\x0e3\xb5\xd1Ddvr\x97\xb6`\x92n7\x12\xe5\x00\x809\xe7\x82&A"H\xb2\xcaa\xc7\xd1>3w\xf1]9Q\xc4\xb5\xa3\xa5\xd5\xb4\xc1\x83\xb8\xb2wy\x08\x80=\x11\xaf4s\xb7\x8eJ\xdd\xdcl"D\xb4\xecUA\xe1-F\xcc\x1fY|\xab\xcb\x7f\xa5\x8a\x94\x04%[\x941\xc0bE]\x0f\xfa\x85\xa4\xcdP\x8bm?dx\xd8%\xb5I\xc54OF\xf6\xdc\x15\xbb\xa9\x96[\x94h:5e\xbb8%q\x85\xf0\x04\x03\xa0\x80\x17\xda>\xe6\xea\xbc\x9b\xb4;\x19\xb7M\x14\xbe88\xf6-F\xcc\xef{\x13F\xb7\x13\x05\xf4\x9c\xe1V\xe8v?X\xd7\x8bn\xee\xc6\xb2\xa2\xcf\xa3F\xe1\x0bW\xa1PM\xa4\xf9\x1eG$T\xe7\xd0\x04\xe3xg\xa6P!\x13\x94\\\xb3\xb8\x01\x91\xda\xa9\xdc\x8a\t\xec\xba\x80\xa8\x8d\xa8/`\xc2]/\xae\xb8\xc6\x9at\xa4\xf3z\xb8CZ\xa4\xda%\x1e\x91\xf2^D\xdf_\x9cs\xfe\x16#\xe6\xf7E\x8b\x10\xc8c\x8e\xfe\xd3,P\xf4RF\xfe\xb8U\xda\xdd\xce\x80#0%:\xd5\xc9t\x1c\xed\xd6\xa3Q\xc8\xado\xa0\x04\xbc\xd8\xf9r_\xf8YT\xe8x\x02\x12\x8b\x86.Fd2\xc2pC\xf6\xfb\xc9\x8d"v\xab1W\xc9h\xecD6n\xd7\xe6{\x8dU|\x8b\x11\xf3\xfbb\xa1pz;\xc4\xc1O\x89\\\xc7\rc`\x19\xa0\xe7n\xa88\xeeME\xdb\'pV\xb7\xf7\xf3>\xc0\x1805\xa59\x0b\x03O\x19"\x07\xec\x0eP\x1a\xae\xe2\xee\xc69k<\xa0\xb7D\xcfI/\x08\x14\xf9\xca\xcd{\x84.\xe8\xa57\'\n\xbd\xbf8Q\xf5-F\xcc\xf3\xcb\x8b\xff\x1a\xe5e\xce|x9\xb1F\x15+\xb0\xcc\xa9\xecL\x9c\x86\x03\xa7&-U\xea\xec\xfe\x9c\x8e\x17,4\xfa\xe4>9(\'\xe7\xf8\xc9=J\xe7FgB\x8d\xaaOc\r_\x1f/\x1e\xfb\x84\x9f,\x05QUq>\x00\xf8\x0f\xf9\xbb\x9fI\xc4\xfcQ\x95\xa9-[\x92\xd8\x93\xdcv\xeeo!\xd7\xc4\xcd]U\xa0\xf3\xcczR\x12I\xeb\x8a6+1\r>\x13h\tr\xe8R\x9b\xa1F2\xaes~\xa8jo\x18\r\xd8\x87\xc7\xfbm\xb8\x8em\x8f\xc3\x1aX\xaa=}=\x8f\xf9\x81<\x9d\xd7\xaaM~t\x1a\xff\x1b\x131\x7fTC\x12\xc1 \x8a|\xca\x9f{\xd7\x17\xee\xc7\x86\xf3\xc2\x1d\xa4M\xfb\xea>\xc8\x97+\x95L{\xbbfK:\xe2\x00&\n\xa8\xea\x8a\xcc\xc3U\x97\xce\x87%B\x95\\4\x95\x83\xc6\xc4bQ\x1c\xcf\xbb\xd0\xbc \xdb\xb9\xcf\xa3\xd2#\x16]M\xbb`\x82\x1fA;?\x93\x88\xf9}\xe7\xe3$\x0c\x11$\xf2\xd4\xd6\xdcwq\xc6\x99\x13\x08BZ\xe2\xa7\xa8\x86\x8a\xf7}n\xa3\xa6\xd25\xd2\x00\xf9\x19L:\x08\x1f\x1di\xff\x08\xb7\xdevnp\x19\x17\xbf\xc5\xea\xae8\x8b\xac\xc68S\x93S\x1c\xc8U\xb3I\xebT`P\xb1\x82\x9c\xdd\x17\xcb\xc4[\x88\x98\xff\xd7\xa1>\xde\xdf=Sb\xa1F\x0f\xd0\xd0\x8b\xb3@\xefo+HZ\x18\xd4\x11\xc0v\xc8i\xbb\x04>\xe3\xe3z\x8a;\xfb\xda\x9d\xd2xJBf)r1\x11\xf4S)\xf1\x99X4\x03\xef\xfa\xc6\x19#1\xb8O\x1bh\xd5#\x128\xfb5\xf2\xe2,\xe5\xb7\x101\xbf\x87\x89#\xdb\x12\xa7\x9f\xcf6\x821\n@\xb2\x16^(\xe5#gf\xc7\xe8\x94+)B\xecE\xfa\x86n-0\xacO\xd9\xee\xd6k\xa5x\xb1\xaa\x1by\x07U\x84"\xec\xd3~G\xcb\x86\xd2\x85\xed\xa9\xd2\xf6kb\xb4\xc1"\xac\xd2H\x9e j\x8a\xbe\x17\x14\xf1\x16"\xe6\x8f=A\xa3[\xef\xf0\x17=0 #\x08\x10\x9f\x94z\x7f\x95\xe4\x11^IdoJ\xd3\x11\x8d\xa7\xf6\x1cb\xe9Y\x02nw\\9\xe2u\xabK=\xb0\xe6G\xa3&{\x9c\x9e\xc4\x0c[n|Z\x82\x8eA\xe8\xa2\xefj\xcbT\xb8)s\xac\xd9\xec\xc5\xee\xed-D\xcc\xef\x8b\x05\x85\x08\x9c\xc4\x9fNN\xc1N\xa4\xf4`\x9f\xd4\xb2\xb1\xd2\x04t\x83)!p\x8b\x19m\xf1\xa5\xba\r\x10Qt\x0cE\x9b\xc7\x19\x9f\\p\xd6O\x88G\x1d\x9a\xc1$\xefnVA\x06\xcfu\xdd\xd6\xed\x13\xfd\x95\xe9btiG\xd1\xf4=\x1a{qK\xbcE\x88\xf9-J\x14\xdd6\x11N=\x8f\x17\xa7\xa8\xd3\xb1I\xeb\xaa\xd4\x8fxN\x15\xd7\xa6\x0b;\x9b\xb8\xc5\xfb\x10;\xb6z\x7fEh\xa5\xf4\xae{5\xad\xfa\xc9\xb38G\xcb}\xb9n\xd4@\x9fN\xc5\x92\xcel\x9a@\x98\xc0\x8a\x97\xe4\x84#Fo\x1eL\n\xe3^<\xee\xbf\x85\x88\xf9\xe3h\x03\x134F\xce\x14TH\xa2\'\x86\xf4rUD\xb3\xabX\xfb2c!\xba\x1d\x95S[R\xb4,k\xcc\xae\x06\xbd*\n\xc3CX./\xf6\xf9o\x01b\xfe\xe8\x0c\x1f\x95\x10%\x9f\xce\x87\x9ep\x03e\x1d@\x8d\xceoF\xe1\xdc9\xd8\xc5\xb5\x9c\xaeJ\xc8\xb1vY\xa4\xd3W\x946\xc4e\xd4VR\xbc\x19{\tX\xc6\xc1U/T0\x18;\xcaIPA\x00\xc3\xb1kR\xa1\x0b-\x92(\xdca\x95\xf7/"\xa9o\x01b\xfexk\x83B(F OomJ\xb0\n\xc8\x1d\xd9[\x8eS\xf8\x1d+\x05\x94\x8f\xc7:\xab\xf0W\xc7\xc9X\x8c=\x8f"d\xdf\xf3#%\xb6\x8c\xc0NX\xe8yh\xc9\xb8\x87\xab\xb5\xcf\xc3\x8c\xbd\xe6\xf7{\x89k[\x7fZ\x0b\x93\xa4M\xe0n\x07\xb3/\x1a\x7fo\x01b~\x7f\x9a\x10\xf1\x90H\x9ey\xcfj\xe8\x86!\xe4&g\xac#\xde\xcd\\-\xe7\x81\x1c4<\xd3\xc6\xf6\xd4\xbdaQ\xff\xb6g\xaf\xfez\xbbl\x1f\xccE&>\xd0\xb7L\x9e\xb8v\xd4\x82\xa2\xf4e\xf4\xd4\xd5\t-;\xb9\x9b;\xf9\xcc\x02\x92`\xdc^lj\xde\x02\xc4\xfc\x91\xc6ql+\xe7\xf8\xb3\xd3\x0c\xd9\x01q\xbb,\x97a\xee\xb8\x9e\xb1\xf2\x84/\x0ep\xe9\xa8\xc7Q\x02#\xd6\x1e1R$a\x16\x05\r,\xdewh\x94\xc6\xacMt\xe6 \x0fE\xc2cL\xa4\x81FT\x18F\xe4\x0eFB\x18\xf9\xb5\x87\x1a\xfa\xc5\x1f1\xbc\x05\x88\xf9}\xd1\x92\xe8v\xae\xa5\xc9\xa7E+\x96H\x03\x9dR\x93={\x06YI\xf9\xa0u\x87\xb5\xbb8\xa0\xb7\xb3\xc5\x13\x91\xb8\x01KYu{\x18\\\x85\xe23\x84W\xf8P8\xa9\xf2QK\xcfs\x9a9\xac]\x05\x16\xb4\x9a\xeb\x91\xec\x0b\xb9\xcd\x8c\xe3\xb5%\xbf\x97\xd0\xfe\x16 \xe6\x8f\xad\x0fQ\xdb\xf9\xedY\x13\xea!N\x08\xf3R\xd4\x9bri\x022?\xd3\xc7\xad\x01\xbb4>\\\xac\x910\xf1\x91\x7f\xf7c\xab\xe5\xfb\xbeH\x85\xd0q*\xad\xc7\x1aW\x10J\xbb_\x8acu\xc8\xeck%K\xe1Y\xc1\xc4|B\xae\x96N\x9c\xa6W\x1b\xfdw\x001\xffo\xb1\xe0\x10\x84<7\xfa\xf6X\x18\xa9YX"\x04\xa2t\xd5\x1c\x87\x1b\xe8K\xac)$\x80q\xb1\xf2\x95cs\x1e\x83\xb3\xceTY2\x14H\xa0@\x99\x93\xebhH\xc9\x95\x17\xf7\x1c\x82Lj\x853RR\xa3\xe3\x1c)\x12\xe60f\x0ft/\xd6\xab\xb7\x001\xbfo\xfd\xad?\xa5`\x8a~\n3\xe1p\x0foz\xe7\x8e\x87\xda\x12L\xda\xa5\xdae\x87\xc8\x9d=`\x91\x8a\xfd\xb9cE\x9a\'\t\x03\xf0fOR\xcc<\xcf8\xabfk#\\\x05\x88i\x1a\xb9!bo\x90\x15!\x04\x8f\xe5\x0eF\x11\xda\xc2\x81\xf1\xc5\xa7\xf9\x16 \xe6\x8f\x13\x1c\x82m\x91BO\x8b\xf6\xb8\x14$\x7f\x84g\x7f\xad\xee\xcb}\x05\xa5\x80\xf5G\x9e\xa6\x07\xd0u\xb1nX\x157\xbf\xf2<\x1d\x8f<\xe5i{\xa0\xcd8a/\xa2|O.(\xab.\r5l\xa7s1\xf3\x01\xfa\xa6\xb0<\xc0\xe8\x18\x9f\xa3\xdf\x8b\xf7|\x0b\x10\xf3\xc7bA\xa9\xadyC\x9f\xee\xe2a\xd5\xcf\xb6\xdbs\xf2\x922\x95@x\xc6\xcd\xc0eI\xf1\xee o1A]dp\xb9\x94\x19\x93\x958X\xcb\xfe\x02\x8a(LT51h\xd8cfE_,\xe6\x80\xd3\xe9\xe8\xd2\xaa\\\x05\xaa \x05\xf2\xc5\xe3\xfe5 \xe6\xeb\xd5\xe4\x07\x88\xf9\x97\x7fS\xfd\xdf\x07\x88\xf9\x1b\xd1\x1b\x1f\xcd\xe4sK\xbf\xff-\xfd\x001\x1f \xe6\x03\xc4|\x80\x98\x0f\x10\xf3\x01b>@\xcc7\xbe\xce\x0f\x10\xf3\x01b>@\xcc\x07\x88\xf9\xce\xf0\xca\x07\x88\xf9\x001\x1f \xe6\x03\xc4|\xdb\xeb\xfc\x001\x1f \xe6\x03\xc4|\x80\x98\xef\x0c\xaf|\x80\x98\x0f\x10\xf3\x01b>@\xcc\xb7\xbd\xce\x0f\x10\xf3\x01b>@\xcc\x07\x88\xf9\xce\xf0\xca\x07\x88\xf9\x001\x1f \xe6\x03\xc4|\xdb\xeb\xfc\x001\x1f \xe6\x03\xc4|\x80\x98\xef\x0c\xaf|\x80\x98\x0f\x10\xf3\x01b>@\xcc\xb7\xbd\xce\x0f\x10\xf3\x01b>@\xcc\x07\x88\xf9\xce\xf0\xca\x07\x88\xf9\x001\xafP\x08\xff-a\xff\x8f\x14\xc2\x9fr\xd1w\x06b~~`\xbf\x08\x88\xf9\xf9\x81\xfd" \xe6\xe7\x07\xf6\x8b\x80\x98\xd7\x02\xfb\xdfp#\xbf\x08\x88\xf9\xf9O\xec\x17\x011\xff\x07\xc9\xe3\x03\xc4\xfc\x14 \x06\xff\xcf{\xfe\xcf\x81\x18\x0c\xa38\x82\x85E\x1a\x83\x1e\xf3\x98\x08\x11\x13I\x8egX\x91f\xa9\xc7\xf0|\x1c\xc71\x18\xa7\xf0\xc7L*\x14f`\x81&X\x81EqT\xdc\xfeK\xd3\x8fa0?\xc6\x0f\x18\x96\x10\x1e\xe3py\x88"X\x02\xc6\x1f#P\x04\x8c\xe3\x11\x0e{\xfc\x1b\x06\'1\x8e\x85)\x82\xa3\x11\x9e\xe3Y\x06\xc2\x04\x08\xa6D\x18\x15H\x11%`\xeag\x021\x7fL#\xfb\xc7_\r` \xfe\x83\xa6\x11\x04B\xa1\xdf\xa6u\xfcH\x88\xf9\xfe\xba\xce_\x081\xdbU\t,A\xa28\x86\x90,\x8ea0A`\x10.b\xf0\xf6\xe0q\x11\x12\xb6\xc7%l\xdf]\xe0\x10\x01\xc5h\x14\'E\x8a\xc5i|\xfbc\x8f\x89\xdc_s\xe3?B\xccO\x14bpN\xa0\x18\x04\x12\xb7\xcf\xdb>\x1e\xc6\x08n\xdbn\xb0@l\xcf\x05\x16(\x08\xe7Iv{\xcc\xdb}\x84\xb6\x0f\xe5\xb6\xcbb\x04\x14bHa[Z\x18\x01\x7fM\x17\xf9[\x0b1\xff\xb2[\xf26!\xe61\xd8\xe4\x9f\n1\xdf\x7f\x9f\xff:!\xe6\x91.\xe1\x1f\x8e\xa4\x16jM3&E>8j\xd1\n\xfc\xceF\xbd\xb8\x9c\xa02\x0c\\W\xdf\x9do\xed\x89\xa0\xfa\xd8Sc\x1c\xef\x0c\x00\xf4\xf0\x1a\x88\x18\xa8%\x85~\x9c| \x0f\x87\xb0\x83\xd9\xc6\x8e\xdb\xdeVIQ\x1a\x11,\x0f^\x1c5\xfa6 \xe6OE\xe1\xbf\x86\x99\xef\x01q\xc8\x19\x84\xc5\x81K\x81\xdb\x83\xe6\xda\x00\x14\x0em\xd1\x921\xa3\x8b7*8\xe5\x1d,\x99(e\xc5;8"pv\xbfr\xb2\xd3\x9ez\x1f\xf5p{\x17\x0f&\te\xd6N\xac8e\xaa\x10w\xf62\xf0\xc5)\xc3o\x03b\xc8\xff\x80(\x0c~\n\xd1:_\xe8"\xcaE\xad\xb2\xc2i7\xb8\x8c\x04\xdcJW\xbdO\x17\xa9\xc5\xcc\xc9\x9a\x92K\xc0^u\xff\xe8\x8ej\xcb\xad\xf0(\xb3\x97&\x92\xcb\xaa]\xa1\x03F\xc9u\xdf6\xb7\xf9\x04P&\x13\xd6Z:\x87\x89\t\xbd8n\xf0m>\x0c\xf9\x1f_g\x91m\xc5>\xcd\xc8T\xbc\xb1oJ,\xdf\xeb\xc5-\xd8\xedjr\xe6\x06#\xf6\xeb\xd4\xf6%Wq\x8a\xc3-#\xf2\x85\'N\xd3A\xcbiy\x97\x8c\xc9"\xb5\x13\xb8\x8b\xdc\xbeg\xd9\xc2\xdc#\x80w\n\xc9\xab\x89\xfbw\xd9f]\xa2=\xfdh,\xf6\xdf\xdc\x87\xd9\x16\xca\xf6\xe7q\x84\xa0\x9e&\x0c\x07p|,\x11W*\xf6r\xccTJ\x88\x1frK\xb3b\x89p\x92\xf6\x98\x17\xcc\xac\xe2B\x96%6x*.\xd0M\xa2B\x8d\xa8(\x98;\x87\xc5\xe8\x81\xba\x91\xb3\x99f\xd4\xe2\xcdC00Ad\x1a\xc3\xe7\xeb\xfeEj\xe3m>\xcc\x16&\x89\xd1\x8f\xf9\xbdO\x8b%\xf3\x08\xf6&\xdc*\xb0\xf1"NIW\xd16\xaa\xe6\xe84Ss!\xa7\x95\x0bz\xc4\x92\xb8I\xdb\xed\xf5\x1e\x1a\\\x07\xb0\x0f\xa3{\x85\x0e"\xe7\xe2\xc0B\xa8\x07\xc7\x842\xc2Yv\xfdQO\x98u\xcb\xea`<\xbe8b\xf8m>\xcc\x16&F\xd0\x0fe\xeei6%\xe1A\xfdA\xbf\xb5^\x81k\x9d\xae\x1a\xf19*\x88\x81^h#K\x87\xb6\xbaI;\x89\x1fJl\x86\x88AP\xaf\x10\xb2 \xaex\xb0\x08\xb4gx\xea\\1\xf2\xe5\xe4\xeeX\xa3]\xda4\xa9\xae\xf1\xf9f\x97\xb7W\x93\xf8\xdb|\x98-L\x1aBidK\x1b\x7f\x0e\x93\xdf\xb9"\xe4;#\xcc\xcd\x9e\r\x94\xd2U\xe2\x00l\xbbZ\xb3\t\xebx\xce\xd9\xbb\x8d{\xd2T@>\x0e\xe7\x8e\xbd\xf3\xd2=\xda\xed*W\xbcTa\xb6\x93\xcc}\xed{%\x82\xf6\xcaU\x1b\x8a\x94\x81\x8d\x86\xee\x85\xb2\x02\x12]\x07\xba\x84\xce\r\xb4e\xfb"r\xd7\x01\xe8&\xfc\xdcT\xcb\x8bs\xb1\xdf\xc6\xc3|m|h;\xc3aO;\x82;u$\xedSDC\x9dH\xf1\xb2\x95\xa4\xe8&\x92\x9d\n\xd2\xeb\x80\xe1\x16\xe6Y,(\xe6\xf9\xd1Iu\xf3\x8a\x1dR\xa4s\xc5\xbbT\x01\xfea\xb8\x08\x90RQt]@[\xf2\xd3\xad\xb5\xd2w\x12 \xa4.\xba~/\xe1\xefm<\xcccG\x904D\xe1\xcf\xe4%ac\x07"\xa3J\xfd\xc2\xe2}+8\xa4\xaf!\x97\xac2\xa8\xcb\xf1r\xbd\xe1\'\xec\x98\xe9\xb5\xcf$\xc9yl\xc8\x8a5,\xcca\xaf+\xd0q\x9d;E\xa0\xcc\xcc\'S\xef@A\x87\xe3>\xec\xb2]D\xae\xbb\xf1\xc5y\xd4o\xe3a\x1e{b;\xdal\xbb\xffi\x86\xbao\xef}0\x87#\xf1\xbc.\x9a\xd4\x1b\x08\x01&\r\x8a\xd0\x03\xbdgE0C=\xa3,\xaa\xb0\x01i\x17dM?\r\x0b\xbci\x87C\xd0Qw\xd1B\xe3\xd5\xd7p\x03\x17\xe1;NhD|\x10\x81\n)\xa7_\xa5\xc3lQ\x92\x08\x8cm{\xff)\xcc5\xa2\x13\x97\xa1\x98$\xdd\xdd\x8e\x82\xb9\x93\xf9L\xb6\xac#\x85\xcb\x00H\xa2iXi\xceH\x02\xd4\xaa\xa7,\x86\x01{\xec.\x0c\x8d\xbc\xf4*i\x98u\x90%iV\xec\xc6\xc6\xce\xf3\xfe\xa8\xa8in\x97\xb3M\x94\xafr\x8d\xef\xd2a\x1ea\xa2\xdb7\xc0\x9e}\x83d6\xa3\x0b\xeex*\x7f\xcf\xeas\x96\x90iO\xf7\xd7\xad-n\x8b`\x07\xef\xa4\x1d\xdc\x8f\xb5\xa1\xbai\x19\x05\'\x1f\x08\xc7:\xe3\xe5\xe5\x98\xab\x17\x82\xe1\x03\x0e\xb0\xc8\x98\x1dDx\xaf\xb9Y\xc9\xe7\x1d\xe8\x19\xa6\xf9\xa3\x92\xff7\xd7a\x1eM\r\xf5\xe8\xde\xe0\xa7\x81\xfb\x1drH\xf8\x9b8]\xf2\xddE\x84A5\xbd\xdd\x02\xc0 N\x19\x83\xa9\x97@\x04C\xad\x85l{E\n\x877\x83}C\xdc\xeb5\x93\xcb\xec0\x8c\x9a\x11\x92+s\xafU\x10ti\xa5J\x80$\x0e|\x9c5\xa9\xf0\xc5q\xd4o\xd3a\x1e\xafl\x1e?\xc6\x81\xa8\xa7\xf7\x19\xb4\xdf[\xa3N^\x8f\xcbM\x97|qKq\x8e\x8b\x10\xe9\xa2\xaa\xeern\x17\x12\xe8\xce\xddm\xc2\x8e\xf6\x19%z\x06\xe9\xda2Q\xf7\xe9\xed8eG\xa9\xc9\xcd\xbe\xa05\x85\xbb#\xc8x\xdc\xc9\xe14\xd8\xb7\x1a\xd5^\r\xf3]<\xcc\xe3i\xd2\x04\x85\xfc\xc5\x9e\xe0\x82v&\xb8\x99\xa5[\xee&\xd3{K7\x19\xb2\xb9+\xd7\xf2F\x9a^|\x92\x04\x8eL\xf6\x9ew\xc9\xda\xee\xc6\x88F\x9b(\xb1m\x02\xed\\X\xe6\x83\x05Cl\x96\x1d\\\x9fN\xf5\x96\x84B\xd1\xebb\xb4\xa2^\xcc\xe3o\xe3a\x1eM\xcd\xd6\xb9\x91\xe43\x93(\\Im\x91\xeeP\x8e\xdc\xf77\xa1%\xdb\xf6\x8a\xac\x8a\xb7\xeb\xcf\xec\x10\x80@X\xc39\xb8\xf5.\xc8\x8a\x8c\xcb\xb6Q\xc2\xac\x00x\x16\xa5\x9bDP;\xa8\xa8\xe9\xfa\x806\x8e\x92\xea\xf6\xd5\r\xf3\xc6b\xb2\x14P\xbe\xd7k\x9b\xb7\xf10\xdb\x9e\xa0 \x88\xda\x12\xc8\xd3p\xf1\xc4*\x8f\xf2\xed\x0eT\xa9\x96Y^<\x80\xb9\xe9\x95J\xe0l;]<\xb68\x1e\x9a\xbb\xb3:\xee\xa8\x11.S\xc1Vc\xd1/\x1c\x16\x08\xd5\xc1\xce\x01\xd6\x9f \xfbbO\x1d\xe8\xfa\x93Hc\xd1\x1d\xa4*!\xbb\xbe\xf8\x8e\xefm@\xccW\x98\xe8#\xcc\xe7\xc5r2\xb4c\x8e\xb0\x1c\xd5(1Oz\xd5\x8ah\xee\xdc\xba\xae\x03H\xc7\xd3\x1e\x13\x80k\x06\x99U\x99\x05\x96\xab`\xf8x\xe4J~L\x18\x03\x10(\x14\x1f\xd5s\x00#\xce4\xc2\xe9\xf9\xa45\xb48\xee;"\x86^l\r\xdf\x06\xc4<2\x1c\xfd\xb0\xd2\xa0\xa7\xad\x7f\xb0\xd2Z\xe0\xa2\xd1S\x0c\x91\x04\xd3\x16\xe4Q\x93!\xe4\xf9\x96\xd8Q\xa7\xe9\xd9=\xa6\x08A\xb1\x01\xf6\x94C\x9a\xed \xfaUYw.G%h}\x8e\xcf\x93\xaez\x85\xd3g\x15\x10\x91%\x9d\'\x9d\xb6\xcfS\xe9U9\xe5]@\xcc#\xc3m\x99\x7f\xcb\xe5O\xef3j=W\x92K\xcbzB\xac;\x17:\xc2GJ\xed4w\xe5\xc2|\xf5\x8cDm-uw\xe5w\xc8\xe1\x98\xed\xe3\x80\xbc\xa2\xf8R\xec3\x07*\x1a\t\'\r?\xad\xe6\x14[X+\xb6Cg-\xf4\xf9R\xaf\xe4\xf9E\x00\xebm@\x0c\xf9\x1f\xf0vP\xa2\x1f\xf2\xe5\x9f\xc3\x049t\')D\x12\xd1\x9a\xd0\xb2\x8b\x83\xf3\xd7*Y\xc2}\xeab\xc7\xe6j\\R\x009\xd4\xc7\x13}\xb5\xfc\xdc\x9c\xd8Y\xbfQ"\x11\x03\x9e\x19)R=m\xfb\xf3\x16\x0cm\x0c.\xb2\xb55C\xabt\n\xce\x1d\xf3"a\xfa6 \xe6\xd1w\xc0[\n\xdb\xce*O^\xe3\xf98\xc5\xa7\xab\x1cW\xf6\x05\\\x92\x0b\x9e\xd5\nV\x1e\xea+6r\xbas\xb3\xa8\xb3i\xecIP\xd2x\rQ\xc6\xe6z\xbd\xab\xe8t\x94\t\xb7\xdd\x03]1\xa6\\\xa0\x90{\x1d:_M\x9c\xd7F2\xce\xd8Q\xfa^\xce\xd7\xdb\x80\x98\xc7]|p\x90(\xf5\xb4\xf5w{\xaa7\xc0\xac\xf3\xe5\xaa\x0c\x99\xe0t\x91\xbb\xb2\xb6\x89k\x85K~\xab.w\xc9\x93O\xc1\xe4\xf2\xd6)?\xbb\xb3p!\xe3!\x1f\xfbH\xab!*\xa3a\xa0m\xc6\xdc\x10\xb1\xc7x \xdc\x87\xacF\x12\xc5({\xf1\xfd\xfb\xdb\x80\x98\xc7\xd6G(\xe4\xf1\xcd\x9e\xde\xd8F\x96!64\xc83\xf2ejf%BS\x1e\xc1x\xe9\x08\xb1\x07\x1a\x99\xebx\xe8\xa1b.\xab\x94m\x0c\xfaL\xc0\xfdR\x8b\xe5%\xabA\x13\x89\xfb)\xa2\xad!>z\xee~;\x01\xc2.R\x83>\xb4\x95\xff\x17\xfb\x9a\xb7\x011\x8f\xa7\t=xO\xe4\xe9\xc5\xcd\xdd\xd0\x04\n9\xb5\xac\xdfc9\x84\xac\xfb\xdb\x99@\x8d<\xaex\xa3\x03\x1d\xa1\xbf\x13\x129\x81\xd1\xe1:\xf1\xebA\x16\xbb\xb3\xea\x17\x12%\x07\xa6\xac\x01\xa5\xed\xd7\xa5|9\xd8Q|\x8e\xc1vv\x0fB\x89\xb5\x10\xf7\xe2\x8f\x19\xde\x06\xc4<~\xbaHB[-\xc0\x9fN\x1f6S\xea\x87z\xbakfW\xa8\xa3\x1b\xee\xb8\xfd2\x184\x9c\x1c\xecJ\r\xeeb\xd7\x17\x02\xe2\xd2\xd7r(\xbb;\x04\xe4\xf6ya\x10\x88\x87xE!\xef\xfb\xc3\xeet\x18\x1c\xc1\xda\x9f1\xd6(\xcc\xea \xde\xdc\xdb\xe1G\x8b\xf6o\x0e\xc4<~\x80\t=~\x84\xfd\xbc\'&z\xb4\x82@T\xb8"\xf6\xd0Xr\xbbS\x8c;\x8e\x0e"e@\xc6\x98\x823\x82FaC~XB\xb5i\xef\xa1\xa1t\xbdU_H\xd7\xd6\x9a\xc9\xabQeUojTkC\x00-\x04\xe3\xcf\xba\xecI\xe4o\x07\xfe\xff\x11\x88\xf9:\xc1\x7f\x80\x98\x7f\xf97\xd5\xff}\x80\x98\xbf\xd1\x88\xe3\xcf\xd4\xe8\x0f\x10\xf3\x01b>@\xcc\x07\x88\xf9\x001\x1f \xe6\x1b_\xe7\x07\x88\xf9\x001\x1f \xe6\x03\xc4|\xe7#\xf4\xff\xef\x8e\xfa\x1f \xe6\x03\xc4|\x80\x98\x0f\x10\xf3\x01b>@\xcc\x07\x88\xf9\xc6\xd7\xf9\x01b>@\xcc\x07\x88\xf9\x001\xdf\x19^\xf9\x001\x1f \xe6\x03\xc4|\x80\x98o{\x9d\x1f \xe6\x03\xc4|\x80\x98\x0f\x10\xf3\x9d\xe1\x95\x0f\x10\xf3\x01b>@\xcc\x07\x88\xf9\xb6\xd7\xf9\x01b>@\xcc\x07\x88\xf9\x001\xdf\x19^\xf9\x001\x1f \xe6\x03\xc4|\x80\x98o{\x9d\x1f \xe6\x03\xc4|\x80\x98\x0f\x10\xf3\x9d\xe1\x95\x7fk \xe6\xbf}\xc8\xffH!\xfc)\xbd\x7fg \xe6\xe7\x07\xf6\x8b\x80\x98\x9f\x1f\xd8/\x02b~~`\xbf\x08\x88y-\xb0\xff\r7\xf2\x8b\x80\x98\x9f\xff\xc4~\x11\x10\xf3\x7f\x90<>@\xccO\x01b\x88\xff\xbc\xe7\xff\x1c\x88!9\x1e\xc3D\x94$x\x88eaT\x14!\x91CE\x02c \x06Eh\x94\xe79\x9c\xa6q\x02\x151\x1a\xe2\x10B\xc4Y\x02\xc7`\xf4\x81>p\x18J=\xe6{\xfc\xd8>\xc0Q\xfc1\xe4\x18\xc5q\x92\xa5)\x8c\xe5\x10H\x10DL\x14`\x01ahQ\x10P\x88$iZ\xe0\x04\x1c\xe1D\\\x140\x02\xa59\x81@q\x0c\x85`\x0c\xfd\x99@\xcc\x1f\x03e\xff\xf1\x17\x03\x18`\xf4?\xb6G\x87\x92\x18D\xc3\xff\x0c\x88\xf9\xfe\xba\xce_\x001\x10\xc9\x90(\x8elO\x96%(\x9edy\x92\xda\xee6\xcf\x89\xac\x00m\xd7O\xa0\x08\x8b@\x1c\xcd \xcc\xf6\x0cp\x88~|\x12\xc5\xd3k\x83|\x0c\xa3\x8bw\xb7Z?a3e\x99\xf5(G\x18\xc2\xaa\xa3\xbd\xc4\xd7d3[\xf9\x19J\x93\xab\xa5PLVy\r\xdf\x9e\xca\xf6\xb8-\xe2\xab\xe1\x1d0\x1f\xdf\xf9G!\x15<\x8d\xba\x99<\x8e\xda\x1a\xf3\xadF\xd4\xbd\x8b\x88y\xac\x14d\xabw4\xfa,C\x19NM`\xb30BIq\xdeU\xe6zd\xbc\x9c\xa5\x1aN\xa60\x98\xcfo\x85/\x07\x05\x8b\xb5\x0212\x08\x10\xee\x0e\xa6H\xd1\xc4e\xdf\xdc\xaaz\xc0\x01I$\xa3\x93_\xc7uG\xf8w\xf7\x04\x1e4vt.\xaf\xce\xa4~\x13\x11\xf3\xd8\xf7\x18\x8e\xd10\x8d>\xcd\x1a\xad/1\x1c\x03\x1c\xe2\xef\x8e\x92j\x81\'$\xa7\xac\xe9\x90\x8d7B7f\x9f\x1f\xfb1\x1a.\xf9*\x8b\x0e\xc3!\x80\xe6\x1c\xd5\x10\xbb\x19\xc2\xb1\xcbs\x1f\xca@\xb88tF0Z\xd0\xc5L+\xab\xe4\xe6C\x8aE/\x8eT}\x17\x11\xb3\x85\xb9\x1d\x8e0\x82z\x9e\xc1)\xafp=wS\xe8\xe8#\x06_"N\xc6T\xca\x0c\xc6\xc1\xcf\xd7E\xd0\xb9.\xc6\xe5\xe0\x88\x18\x07\x7fL\xee\x16d\x83\x88\x9c\x9d\x06\x0b\xbeud~G\xab\x0b\x8c\x00G\x91F\xa6\xe3\x90Dd(\xd7\x11\xc7\xdd]\xfeES\xe4MB\xcc\xe3an\xa7/r\xbbMO\xac\xd1\x12\xa2\xa3e\xb2\xa7\xe3\xc5?\x9fWW\x91yf\x1f%%\xdd\xd8\x19u\xb60\xf5\xe6-~\x075L\x15\xfbn\x9e\x0fGI\x1b*\x91V\x94)\xdb\x9d\xc0\x92\x1f\xd6\xe18\xde\x06-a\x17#\xb6\x90\x98\xc8@bz1\xccw\t1\x7f\xee`\xfe\x9b\x81E\x0fF{\x19\x81`@\x08\xf8H\x0f\x13\xd2z\xed\x91\x9c\x13 \x93\x86,P\x93\x0bB\x012Sq\x91\x18s\n$\x172]u\xe8\xad\x99\x87\xf8\xc0\x8a#\xa6\x0f\xc4\xa4\xbb\x0b\x90\x86d\xe4k0\xb6t \xfc\x8b\x84\x98G\x98\x14JR\xdbA\xf9i\xac\xe2\x9a6\xa5\x9d\x897\x1fm:\x98\xb3\xeb.q\x9c\x0e\x19u\xdb\x9dR`\xd1\x8e\x0etX\x913\xbb\x0cY\xe6\x9e}\xa0\xf71\x00O"\xe1\xb0HSqQ\xa8\xb2\xcd\xc8\x83\xce\xa2\x9e|T\xcf\x15\x18\x98\x07Q\x97^\xdc\x9a\xef\x12b\x1eaB\x08N\x900\xf2\x94\x81\x00\x7f\x9doW\x96\x08=\x85\xa3\x95\xa2S.\xc1\x0e\x89\xa0\xb2L\xc5\xc2\xb3\xe9st\xf7[\xe1f\x93\x07\xe0\xd0:\xb4\x99\x85\xcdz\xbf\xcb$$\xf4\xfb@\xbc\'\xb7K\xba@{&\xd3\xd4@;\\\xe7k\xe7w\xcd\xf2\xef)\xc4<\x8a>\xb15J\x10\xf5j\x00F>\x9dm\x0e*\xbdTV\xb4\x8bY\x1c\xc1\x98\xabf\x1b@vQ$~\xd0\xd0=\xea\xc19w\x08\xaf\xc0\xad\xa7\xdc\xdd\xb1w}\x02\x98\xb0!"\xa91\x19\xc9\xc6D&C\xdb\xa7a\xd1E\xa8\x8eQ\xfdm\x86!\x16\'\xdd\xfb\xcb\xa2\xd9{\x8c\x98\xc7\xe1\x03\xda\x8e\xbb\x0f%\xe6\xcfa\xf6K\'\x82bB\xd66\n\xce\xf2\x0e6\xccu-"\xfdhU\xd2r\xb1x\xe7\x10\x9dt`\xf5\xbc\x9e\xba\xe6\x17\x9c\rvb&\x98\xd1E_\xfd\xda\x12/s\x92\xadsV4\xca]\x98\xb7\xe5\x82x5X\x0e\xf3\x8ffF\xff\xbd\x8d\x98\xc7]\xc4\x11\x98\xc2\xa1\xe7Q\xa3\xab\xb9U\xbc)$d"\x90\xbc\xa6K\xb8f\xbe\x0f\xdc\xb8s\xd3\xf0\xd2\xed\xef\x97j\x9f\xeau\x0bs\x07\xb92@75v\xa5\x92\xb7{B<&~\x04l\xad\x02qH\xc9\xe5N\x9bS\xde\xd6 VQ\xe3\xde\xcc_u\xe1\xded\xc4<\xc2D\xb7\xe6\x8f \xe8\xa7\xc5\xe2\xf6\xa7}\x02\xdf\x80h\x8a\xb0 &\xd4\xee\xcc\x95\xe0%R\x0f\xda@\xfbM|\\\xa6\xdd\xde\xc19\x1e\xed\x96\xcb\xf1\xe05\\*\x8ft\x82\xd4^\x94\x92l\xba\x82N\x16\x1aT\xa9\x1c\tyT\xa7\x9d\xbf\xb5\x88q\xf2\xe2\xe4\xedw!1_[\x1f\x81(x;d\xff9\xccK\x84\r\xc3~\xaf\\\x84\xd4\xbc\xa5A\x0f\x15\xc0\xde\xd6-\xe5<\xd5\xb5W\x98\x82\xb8o0\xd2\xe4Q\x96;M\x1d\x1b\x87\x8e,\xd9\xfe\xd4xe\xd3&\xbc5\xec\x86\xa2\xe6\xd4\xe30h\xc4\x02La\x07q\xcd\xed\xfc\xe2T\xfcw!1_U\x7fK#\x04\xf9\x8c\x19\xc2\xfc\xfe\x86\xdf\xa6D\xbe\x87\x10j\xdc+u\xdf\x8e:\x10)\xe2Pe*\t\x96\xa8\x1c\x13\xfae\xb8\x87\xdd\xcew\xe7\x13\x16\xe3"t\x90\x8b\xb1\xbc)\xc0\x08\xb9\xb6\xa6b:\xa2\xee\x01\xa2\xa8+d\xebi\x9b\xc5\x81\xbfW\xf7\xf6.$\xe6\xb1\'`\x02\xdd\xd2\xc4\xf3Hj\xf3\x94\xa3\xa6\'\x8f\xd7a\xd2\xf3c\xd8\xe6\x14\n\x87\xa9\x94L\x08\x19\xe5\xc4\x01V\xe7\x11;\xd6\xf1\x11\xcb\xdbn\x97\xc6\x89\xee\xe4A\xe73\xd2\x0e\x98\x00\x93Y\x9cK\xe3YM\xc2HqX\xa4\x96=\xed@\xbe\xf1\x7f\x11\x12\xf3hk\xb6\x9a\xb35p\xd0\xd3[\x1b|/\x9bW\xb7\x9e\xd9\xd3\x89\x89\x8b\x9e+K\xcbU#\xd0\xb1N\xde\xde4d\x08\xde\x81\xb5\xd7\xd4\t\xa19\xe6I\xb9\xdf\xf6\x19\xc3\x8eY%\xec0\xfa\n\xdd\xf4sS(\xec\xc9(*lDV)r\x01\xdaY\xa8\x17\xf7\xc4\xbb\x90\x98\xaf\xb76\x18\t\xc3\xc8S\xd1\xd7)\xc9w\x8b\xec\xb0\xdc\xc3\x1cN\x05c\x18O\x02\xaa \x90\xe1\x95\x94\x0f\xa6\xe7\xa2\xf4O\x05x\xc5\xd4\x13\xa5\xddl\x1b\x18\x14\xf2|+\xc1\x18\xc7\x19k\xb8/\x06s\xe3\xa3\x88W\x0f s\x00O\x0c\xea\x97\xa8\xb2\xbc\x98\xe0\xdee\xc4|\xed\xfc\x07\x9f\xfd\x17\xcc_4,$\x16\xde<\xf2\xba\xa7w\x1c\xd2d\xe8\x89\xe7\xf5\xdd}\xcf\x13\x83nH$,\'\xb5q\x15x \xac\r\xeb\xee\x1d\xaa&\r\xd8\x8bil\x9dm\x13\xcc+x\xe7e+\x15/K\xc9\xc6\xa2<\xa9\xe4~9\x97?:\xdb\xfc\xbd\x8d\x98\xafN\x9f\xc6\xb7\xd3\xe53\xed\x9b4\x8c3y\xa6q\xe0!\xbb;\x90l}H\x9c\xb0\x97h\x17c\xe1\xf8\xec^J\'\x94\x13\x1d)\xfb\xbd80\xd9\\)\x17e&\xb6\x8ea\xe8w\xed\x0c\xd0\x07|\xca\xcf\xa0\x92\xdc\xc8\xd1\xbcP\xf8\xc9\xa2:>|\x91\xdbx\x97\x11\xf3X,[\x9f\x8f\xc0\x7f\x11&\x14\xe4\x1d\x0c\x80E\x87\xcf;wZi%\xea}\x8eLO\xfe\xcc\xdb\xc5\xf1R\xfa\x06\x80Q\x13\x84\xabe\xec\xdcb\x95\x15\xeb\xfdv$f]\x9b9q\xd1\x055\x8f\xf7\xb4R\xf3\x8cS\xaf\x88B\xee|\x1fC\xef\xd6\x8bn\xd2\xbb\x8c\x98\xc7\xd3$\xb7/\xc1\x9e-X\xe0\xecO\xe4\x89\xde\xcf\xe7\x0e\xf0/\x89|\xba\xadY\xd8e\x97\xe6\x1c\xd3*\xb7\x9e\x99a\x15\x001\xda\x1a\x9b\xce\xb9\x1d9o\xd8\x89c\x12\xe0\x1d\xa9\xea\xa5qb\xab\xb9\xe9\xc5\x82\xf6[\xff\xb8NkU_\x13\xdf\xe9\x7fT\x0c\x7f2\x11\xf3U\xad\x1e?5\x7f*U\xad\xa7\x04\x89\xaf\xa6\xbaS\xa0s#;\xc3\xe2hyk\x8e\xc1r\xb8\xf8{\xdd\x9e\x95}lm\xc5\xbf\xad\xc6p6\xbd\xfa\x8e\x80\x06\x15yk\xe4\x97\xb4\xb6\x03A\x066Z-\xb6\xe5\xa1\x99\xb6\xff} v\x92\xfd\xa2\xf1\xf7.\x1f\xe6+\x87\xe3\x0f2\x84x:\x8ck\x84\xab\x8b]\x91\xdc\x014\xef\xe8am\x1c<\xe1\x91\xad9=\xf7\x86\xac\x0bk\x1f\x98\xc1\xc5-4hbP+c\xacX \xef\xb9e\xdaB\xded2o(\xf7VR:\xaa\xa1\xe8\x0c\x1fk\x07\xc5\xec\x1e\xd6_|i\xf3.\x1f\xe6\xeb\xb51N<~\xad\xe0\xe9i^\xb4z\x17\xe3\xb4S\xb6\x88\xa5\xa6%\xa2\xd0\x07\x92\xf6ZV\xabN\xca\xe0\xbb\x9a\xd9\x85\xc8,6\xe4\xcd\x8e(9X\xb4\x10E\x02\x8cH\x99tg\r\xf3\x8a\xb1\xc7\xe9*\xf7\xaa\xdd\x04\x88\xc0_\xb4\xf3\xb5\x9a\xa1\xfb\x8f\xfa\xf0\xbf\xb7\x0f\xf3\x95\xc4\xe1m\x99a\xcfF\xaa_\x96;\x02\x8d\xcfV`\xc9\xa0\x01\x9bt\xbf\xa6\x0b|\x11\xc35\\Z\t\xe0\xcb$\x12\x87\x82\x85"b\xe1B\x7f?\xc7\xd8^?\x93\xbeD\xe2p@D\xbd\x16\x1e\xfa\x13\xe4:7\x9f\\\xd5\x85\xf0\'\xc2S\xf2\x17\x93\xf8\xbb|\x98\xaf\x96\x06\xc3\x1fx\xf9S\xc5G4:\xce\xee\xa4\xa9\x15\xe0\xd2\t\xb6z\xbeG\xe0\x0eQ\x16\x1e\x1f\xd8)\xcf\x05\xd4\xc2\xe3\xdd\x89\xbe\x00\xce\xc9`\x8b$\xf6\xcf\x07\x8fquv\t\xce\xecb\xdeU\t\x15\xc3\xc4\xd3\xcaqG"\x80\\\xdb\xc6Y\x07\x7f\x91\x0f\xf3\xb5\xf5I\n\xa1\xb0g\xe8\xabJ\x9b^\xa1\xc0v@\x909\xaf\x9a\x93\x1fcJ\xd0\x0f\xd8\x90*W\x99\xd9\x91cF\xfb\x99n4\xd3P\x96\xca)\x9c\xfcQ\x9fS\xe8\x18(\x8b\x89j\x9e\xb6\x1a\xa7\x89\x8d\x0eG|M\x14V,\x08\xc5\xaf\xd6\xe1\xc5\x0c\xf7.\x1f\xe6\xf14\xe9\xed?\xdbW>-Z\xd7\xbb\x9d\x9b`t\xf5\xe4p2\xa9\xe1\x02(gM\x0e\x97c\xc5Y\x93\x9a\xde\xe9\xf1\xd8\x91\xb5\xd2\xd3WI\x9c\x9c\x02Z\xccAZR\xda\xb60M\xf24\xbb\xa5\x98A\xbb\x840V\xe3Q\xca*@U\xaf\x9c\xd5\xff\xe8i\xfe\xbd}\x98\xc7by\xbc+{\xfcz\xc8\x9f\xef\xa2jjx\xc6\x1b\xc6\x84(k\x16\xf3\x13q\xb8fu\x14\xde\rl-\xc8\xab\x7f\xadB\x93%\xb0>Z\xda\x12\x88\x85\xbcc\x04 i\xe6\xc6\xb4Z\x13\xd3\xe9\x11\xc1\xd6\x9b\x8e\xa1\t\xc51\xd4\xde*W\xe9^\x06\xfb\xdf~H\xfb?\xfa0_/`?>\xcc\xbf\xfc\x8b\xea\xff>>\xcc\xdfH\xde\xf8`&\x9f[\xfa\xfdo\xe9\xc7\x87\xf9\xf80\x1f\x1f\xe6\xe3\xc3||\x98\x8f\x0f\xf3\xf1a\xbe\xf1u~|\x98\x8f\x0f\xf3\xf1a>>\xccwvW>>\xcc\xc7\x87\xf9\xf80\x1f\x1f\xe6\xdb^\xe7\xc7\x87\xf9\xf80\x1f\x1f\xe6\xe3\xc3|gw\xe5\xe3\xc3||\x98\x8f\x0f\xf3\xf1a\xbe\xedu~|\x98\x8f\x0f\xf3\xf1a>>\xccwvW>>\xcc\xc7\x87\xf9\xf80\x1f\x1f\xe6\xdb^\xe7\xc7\x87\xf9\xf80\x1f\x1f\xe6\xe3\xc3|gw\xe5\xe3\xc3||\x98\x8f\x0f\xf3\xf1a\xbe\xedu~|\x98\x8f\x0f\xf3\xf1a>>\xccwvW>>\xcc\xc7\x87yEB\xf8o\xcf\xed\x7f\x94\x10\xfetI\xdf\xd9\x87\xf9\xf9\x81\xfd"\x1f\xe6\xe7\x07\xf6\x8b|\x98\x9f\x1f\xd8/\xf2a^\x0b\xec\x7f\xa3\x8d\xfc"\x1f\xe6\xe7?\xb1_\xe4\xc3\xfc\x1f$\x8f\x8f\x0f\xf3S|\x18\xf2?\xef\xf9?\xf7ap\x9a\xe3\x11\x96\xe2P\x18\x17D\x14#\x1f\x03\x8b\x1e\x86\x05\xf7\x98T\xc1\xc3\x0cAS_\x14\x05Dp"\x0e3\x18\xc7\xd0\x04\x83\x92\x0c/p\x18\'\x08\x0f\xdb\xe2\xc7\xf4\x01$\xe2\x1cN\x13\xdcc:\x18\'\xf2"\xc9\x91\xb0\x00c\x14\x83\x904\xb1\xfdE\xc4\xb7\xef\xc9a<\xcb#\x14\xfc\x18\x87\xcc\xd2,\x04\xe1\x8f\xe9\xa6\x1c\x01\xe3\xf0\xcf\xf4a\xfe\x98L\xf2\x8f\xbf\x18\xc0\x80@\x8f\x01\x0c\x8fa-\x18\xf6\xcf|\x98\xef\x8f\xeb\xfc\x85\x0f#n\x8f\x03f\t\x81&\x10J\xdc\xbe\x0e\x83!\x04bI\x18C\x18\x81\xdfB\x16a\x81\xa7\x11\x8a\x83a\x02E\tL\xd8\xaa\x1c\x043(G2\x0cF\xa1\xdcc\xd4\xc9\x7f\xf5a\x10\x06\xe1(\x04Aq\x12\x16\x05\n\x17\x10\x02\x87EF$\xe0\xedZ\x85\xed/<\x07\xa1"I\x89\x0cC>\xa6\xf7\xf3,.\x12,I\xc2\x10\xb1];Bc\xff\xf8/K\xfc\xe3\xc3l\xdf\x8d\xda\xae\x87\x820T\xc0a\x02c\xb7\x1b\x86\x92\x1c\x8ea\xdb7"\x10l\xdb>(\xb1m(B \x18z{\x88\x10\xc4oK\x8e!ql\xfbt\x9ac(\xf4o\xef\xc3\xfc\xcbj\xc9\x93\x0f\xf3\x0f\xae\xdf\xb6I?\xb1\\\xa1_C\x84^\x93\x80^\xb5\xd6\x80\xc2\xe0\xd2\'M:\x1fK\xb1\x88\x0ev\xae;\xb8\x91"G!\n\x96:\xda\x1emd\xef \xae\xb8\\S\x04\xbe\xea\x87\xc7\x8ft\x96{\x1a\xc0\xb0\x19\x18\xf7\xb85\x96\xaf\xaa\xc4\xefo\xa7v\x87F\xf2\x0e\x8bQf=\xae\x1e\xb2U\xaf\xed\x94\xd8\xe7B\xb9\xfb\xfa\xdaG\x0f\x94\xa0l\x9d\x96\xe2\xaa\xaf\x06\x11\xa2[_\xd8\xe4]\x18\x84\x88.gP\xe8V\xb7p\xfb\xf7\xe6\xa1\xde\xda\xf3\x147\x83~\xd8\xfe9v\xe4\x19\xec\x94w6\x97\xf9\xf0W\x85\\\x14Ew\xbd\xd5\xd8Z%\xc3\xcd\x9c(\xebM\xae\nU\xf9N\x97\xbfUP}\x0e\xd1\x1d\x9e\xca\xf6u\x8b\xaf\x0eQ\xe6n\xb8\xd1\xb8\xc5\xb5?\x06;-:T\x8er\xebr\xe6\xc6\xca"\xa7WA\x9c\x97\xdb\x05\x87\xfa\xba^\x93^\xc6\xcd\x89;3%\xb5\x82\xf1\xb8:R\x03\\y{\x12\xb2\xbd\xcad\x19+\xe4{\x95\xcf:\xf1\x18<\xee\x1f\xe5\xb0\xae\x80\xean\xb8\n\xe6\x89\xe2\xf2\x9e1m#q\xe4\xb2\xca\xcd\xddb\x87\xf5}\xbd\x9d!h\xcf\xc8\x89\x98$\xd2<\xd99\xe9m\xf9\xeeZ\xcca\xbd\x04w [\xfc\x10G\x90\xf2\xaa\x15x\xdbUY\xff\x80\x06LWw\xa3\xc0\xd5\xe1\xc0\xbf\xe5\x9c\x10\xf4\x8f\xc9z\xff[\xdd\xe71\x96\xe6\x9f\xea>\xdf?K\xffR\xdd\x07\xc6~L\t\xf4\x9cw\x08g\xb2\xc1\xf0c\xba\xde#%Yr\x16\xc9\x05?l\x13APw{)\xafo\x8b\xe2D\x85\xed9\xe7U`\x19\xa7X\xcc\xd0\xd0q\xdf\xb6N\xda}k\xed\x91^\xc4n\xbd\x9d\xe8\x85~\x91x\x07\x17\x9c\x17\xe7\xc4\xbeK\xf7y\xd4t\x8a\xa21\x8a\xc4\x9e&p\xdd x\x0e0\xfa*\x9e\x93zR\xab\xc2\x85Wg\x12\x96\xf6^\x13]y\xf4]w*\x01\xbcC\xfdT\xb50\xb4 \xf7mo\xad+~o!\xd98\x99\xeb\xb5\xc0!\xe9\x8c\xd6\r\xd3\xb1\xa2r.\xd9\x86=\xee\x83\x17\xa7E\xbeK\xf7\xd9\xc2\xdcn\xceVW\xfe\x1c\xe3\x91R\x11\x048Y\xf2p\x9a\xc9yw\x04\xd0\xe5\xc8\xafl\x8c)\xe5\x0e\xcb\x0e\x16Z\x0c\xb0\xac1\xa402\x8e\xdb\xf97\xeb,S\x8c\xbfK\x0e1\xf0\xa0\x04\x1a\x15\x8fm\x85W\xa6z\x9d\x8a3\xdbb\xf2\x10\xb1\xbfH\xf7y\xc4\x88\xe0(\x81\x10\xc4\xd3\xc8_\nC\xf09w\xd9\xa8\xa5\xe4\x90\x17\xe1\xcc\x13\x06l\x8fs\xec\x05\xa7\x1b\x88\xa8\x9b9[\x93\xb0\x1a\x86H;[\xa2qsV\xd4\xf2\xebA\x8d\xce\xb8=@\xe4\xdc\xf2wzo`\x07\x8f\x96\x8aa\xe9t\xe8\x0e\xad?\x1a\x86\xfb\xf7\xd6}\xbe6\xc4c\xc0\xec\xd3\x8c11^\x1d#v\xe4\xf3\x0c\xdf\xc8\xe0t\x95lIn\x95\x1e\xbd\xf53\'\x8a\xbb\xa4(vd"\xbb\xc1\xa1\t#.\x9cR\xe1D\x9fp\xde\x87:\x1f\x9b\xbb\xd8\xa1\xc9\x01:\xe4\xa4\xe8\xa64\xda\x82\xa9{\x11\xfbB\x7fq\xa0\xf0\xbbl\x9f\xc7R\xd9z)\x98\x82\x9f9\x18\xef|\xb8\x16\xac\x81\xbb\xd3\xde\x81j\xb3\x1f\x0f\x06\xa6j4\x9b\xa8\x95\xbc\x0bv\x059\xd2\xb7\x84\xbf\xcd\xabA#\xae\x1f\x87*\r\'^\xb1\x17\xc9X\xe8~IR\xf0\xe4\xcc\x13\x7f\xec\xe3Z\xa5\xc2\x1d,\xb0\xafm\xcdw\xd9>\x7fn\xb2\xff\xdb\x98Xz\xd6\xc1\x9bL\xea\xb6\x04\x00\xcd\xa9\x9bM\xb2H\x8a8T\\#\x98W1\xe6\xaf\xc7lW\xb1\xbe<\x84\xa64\xe6\x07\xcd\xab`.l\x9b\x15\xb3\x97\xab\tc\x92\x7f\x93\n\xf8f\xee+4\x11\x177L[\xf0\xd5i\xb8\xef\xb2}\xbe\xc2\x84)\x04"\x9f\'\xb7\x83l\xbe\xe3\xeb\x93\xea\xdc\xe7\x12\x0c\x94\xc88\x19\x17\x84\\\xaa\xfdqQ\xee&\xac\xc2\x8a\x95M:\xb3TjXY\xa2n\xd3b}H\xb8\x8e5\xbd\x11\xdd)\x92\x1d\xb9i\xb6\xca\\7\xb1w#\x8f`v\x1ca\xfc{9&\xef\xb2}\x1ey\x1cF\xben\xe4S\x1e\x8f\x15mi\xafWg)\x93\x047t\x84\xf29\xae\xd8\xd1"\x9e1\xbbI\xef0\xb8\xe1\xd6Z\x05\x8f\xd9m\xabb:\x98x\x14\xda\xd9z\xbe\x9c\xf2\xea\xda\xaf8\xb5w\xe7\xd1D\xafpz Y\xe7\x9e0\xe7\xf4\x06\xf2/\x96\xab7\xd9>[\x98[\x05\x87\xb0\xed\x98\xfa4\xec\x1bw\x92\x13H:\xfdE\xbd\x81\x17\xc7U\x96\xf6\xc0&\x19CJ@2\xa5\xae\xb4\x0fp \xad5\xb1l\x19\x1aq\xf0+\x14\xf4\xb2\xdb\x03\xa3\xea\xe3>R\x15l\xc8\'\x17\xa8\x1c\t\xcb\x86\x88&\xee:\x04F\x8b\xfc\xd5C\xea\x9bl\x9f\xaf<\xbeeK\x0cz\xae\xca\x97N\xcc*;E\xaf\x99E\x1d$\xbe"\x9a\xe8\xd8\xdd\ril\xaa\xd0K#\xdc\x035\xc7m\xa4\xb6c\xb3\xdc\xcb\x1cBV_\tt+\xa4\xdbI\xf8\xa9N(\x86\xa2\xae\xa1\x90\xd5\xbb\xa6\x89N;.\x1a\xbcy\x15\x88\xa0\xb9\xdcE\xe6L2\x8b\xc7\x05\x1e\xea\xe4\xae\xe5\x15&\\\x15\'T\x0cU\xf6\xc6Vl3\xa3\xbb\xc0Z\x01\x18\x013\x1f\x8c\x8e\xe1\x89\xdf\x9b\x01q\x0f\xb5\x17\x87\xc3\xbf\xcb\xf6y\x84\x89m\t\x0e\x85\xe9\xa7q\xe2\xc7\x08\xce\x8b\n\x8b\xbb\x1b\xb8\x03\x8fj_\x80\xe8\x040\x80@U\x81\x1e53\xb2\xa4c\xeaW#e\x94$X\xd1\x9d+\xd1\xa3&\x8a>y\x02\x8a\x81\xca\xc1\x1d\xcc.\xb2\x97\xb6\x8d\x15(Y&{5"\x02\xca/\xb2}\x1e\x89|\xfb\x93\xe8\x03\xda\xfds\x98\x18\x1b\n\x05\xdf\xb7\xf2\x8c\xed\xbc\xf4\xb2\xec\xd9n@o\x81\x7f\x80\x86\xa6\x8do\xe0b\'\x9a\xc0\x9c\xaf\xa8\xb4\x0b\x93\xed\xf8\r\xb5C\x94\xae\xab3\xcen[\xe8\xcd$\xa4\x95G\xad\xbei_\x17\xa0\xddw@a(\xe6/\xc2}\xbezq\x08\xdd\xba\x86\xa7\x9d\xcf\x1e\xdbB\xbdc\xcd\xf6\xef\xb5D\t\xae\xe1!\x8es<\x1c\xda`\xa7\xa10\x88\x1d\xef\\/&1\x03\x8a\x08$\xb8\xa0\x81\xda\t\xb45\xb0#M\x1b7\xd9\xb0o\xf82\x9b{\x13\xbe6X>\x82\xc8Q\xcd\x19\x99\xfd^\xaa\xd7\xbbl\x9f\xaf\x9dOl\xbd\xe1_\xbc\xc6Lg\x14\x05N|\xc6h\x07\xd8i\x9d\x82\xa2\x18\x93\x14\xe1\xc0\xd7x\xe7\x0c\xea\xe4\xb0T\x01\xe4\xb8p1C\xf4\xa9\xdcCd:\x85\xc5\x19\xd9M\xbe)\xc7\xd1\x95nu\x9c8\x0c\x98\x90\xf9\x8a{8\xeafA\\\x93\x17\xcb\xc4\xbbl\x9f\xdfz\xe0\xadG\xd8\xd2\xc5S\x82\x1b\xd1\xbe\xe4\xb7\xce\xde\xf1s\xde\x92\x89\x91NZ\x8f;\xef\x94!\xb90\xbd\x126B\x15]\xe3\xbd\x83\xb8U\xb0k"R\x90\xcf\xac\xb2\xb2\xc9\xae\xb8I\xed\xc2:Lz\x00\xd6@B\xae\x16\xd4d\'\x1e\xdb_\xc9\xfd\x8b=\xf0\xbbl\x9f\xdf\x0e\xc2(\x04\xd1\xe8S\x98\xf7\x9a\xed\xe6C\xb1\xe5O\x1a\x99E)0\xef\xe9\xc5\xd8\xae\xf9\xd4\x1f\xac\x03b\x81f_S\xb7B\xd8\xeb\xc0N\xbe\xf6\xb8o\xb7\xf8p\xca}\xae\x1c\xef\xa21G{H\x05\x006Mk\xe3T\xaes\x97\x9bl{\x1c^|o\xf3.\xdc\xe7\xeb\x85-Bl\x05\x0e{\xee\x81\xb1\xd3\x999\xf0*\xda\x07\x14\xed\x1aIFa\xc8\x80\x14[];\xc9\x98\x14\xb2M\x05eX\x0fqq\xe8\x80\xd0\xc4\xb1~\xd6\x88<\x9fPy\xb8r\x16@\xf6\',\x0e\xfb\xe2@,\x84u\n\x03I\xafz\x81\xe3^4\x8c\xde\xe4\xfb|\xe5\xf1m\x7f\xc3\xd4s\xf7\x86\xcd;@\xa3\xbcn\xb9HjO\x91\xa7\x03\x06ND}\x1d\x12\x90\xac;.U\xdd\x00\xbb\x1e\xd9\xcb=G\xe7;\xc1\x83\x83d\x82\x87\xb1\xa0\xc5*\x93s\xe8z\x91-\xdc\x84\x91\xe8z\x1e\xf0*/k\xf0\xb0\x07\xd4\xe2\xc5<\xfe.\xdf\xe7\xb1h!hko\xf1\xe7^\xdc\xb1w\x0c\xe0Uv1\xdf8t\'\xdf5\xd0\xe7\x0c\x81\xa4\x1b\xb6\xc1$\xfa2\'C\x06\xa3\x87Z\xed\xc5-\x1fjg\xd0\xdd\x8f\xa7\xd6\xcb\x97\xe4x\x1f\x84+\xda\x8e\xabu\xb0\x83\x89\xd0]\x02f\x1bE\xb9\x8a\x8b\xfb#\xcb\xe4\xef\xed\xfb\xfcv\x10\xc6\t\x9a|\xde\xfau\x7f\xdcS\xcb\x81\xe5\x0e`\xdft\x1c\x8a\xfa\xf7\xe5\xcc"\xecN\xb8\xec\x978\x0fe\x80\xf3\xf3\xeb\xe8c\xfd\xb2\x03\xb2\xe8~ZQ*P\xef\xc61e#\x01\xbfr\x19\xb1\xa6\xb4_\xedO\x08x\xbfF1\xe3]\xce\xed\x8b"\xcc\xbb|\x9fG\x98\xdb?D\x11\x82|jk\x16=\xda)\xbb\x98\xde]\x98+\xc4\\\xdc\xbd\x8a16\x8e\xb4t\xe4\xb5\xcb2\x9dH\xb6BW.\xc0uKG\xcf\xb9*\xf0i?\xb3\xf2\xd1\xb5/\xccm)\xa7\x9dn\x1c\xdb%\xba&w\x8b\x9c\xdc\x91\r\xeb`+\x00\xaf\xed\x89w\xf9>_\xbd8A?\x12\xf9\xf3\xd6\'\x14w\xb0n\xbe\xc8U\xed<`&r\xc4\xca=\x8d:\x8b\xe9Cw\xd0\xb2\xf8s\\\x167\x8c\x8b\xc0\xe4\x16*Y\xc7\x85\xde\xc9\xd2V\xba\xb5\xe3\x12\x83\xd4\xb3\xa2.\xa3\xcd\x834-\x80cU\x02\xb0\xd4oY\xfe\xb5\x0c\xf7.\xdf\xe7\xb7\xbe\xe3\xf1\xc7\x9f\x13y9\xba\xf8)\xbb\x13P\x13\'\xfct\x9d\x0e\xa7Y\x8c\x05\xd6\x8a\xb1s}\x00\xeb\xc1o\xabj;\xfa\x1f\'\x85\x92\xa4\xb9\xe7\xaf\x83\x8f3y(X\tA\x19\xc4\xfe\xce\x1c\x13\x17\xc3\xfd\x8bY\xfc\x7f\xec\xbdW\x8f\xe3\xda\xd5\xae\xfb_\xfa\x96\x07fN\x97\xccILb\x12\xb5\xb1q\xc0\x1c\x94(J\x8c\xfb\xcf\x1f\xaa\x96}>{\xa9{\xd92$\x97\x96\xb7\xd0ht\xa3P%\xcdw\x8e9\xd2\x14k<)\xbaFw<\xd8\x12\xc5\xaf\xac\xf9\xe7\xe6\xfb|}F{\xbb\x18\xc0\xef\xa1\xcc\'\xb4\xef\x9cD\xa45o\x1f6.H*M\xef\x1ev\xf1n\x1b\xac\x99<\x0b\xd2dy\x7f\x9b\x06\xd6\xd8\xe2\xf0\xc9R\xc4\xa8\xd644\x00\xabsS\x97\xc5\x07\xba\xe0WQ{\xc5Ns\x08\xe6s\x02\xa9\x11e\x99\xd7\x7f\x91\xef\xf3\xe5\xcb\x7f\xcf\xf7\xf9_\xff\xe7GrJ\x97U\xe1\x7f{\xd2\xe6\xb7\x07)\xabc~\xfam\xb7\x8b\xe8\xf2\xff\x0e\xd1\xf1\x9a\xdd\x9e\xb3\x83I\x98@\xd0\xbf~\xb5\xbb|}\x8d$(\x14\xfazt\xe8\xaf`\xa0\xff\xfd\xff|\xbd\xe6\xa5\x89nOv\xfc\xb8\xa4\xbb\x1f\xb7\x879?\x10\xa1_\xfc6\xc3\x7f\x11D\xe8O\x04\xbe\xf8\xb0D>\x10\xa1\x0fD\xe8\x03\x11\xfa8\xfe\x07"\xf4\x81\x08\xbd\xef:?\x10\xa1\x0fD\xe8\x03\x11\xfa@\x84\xde\x19\xce\xf3\x81\x08} B\x1f\x88\xd0\x07"\xf4\xb6\xeb\xfc@\x84>\x10\xa1\x0fD\xe8\x03\x11zg8\xcf\x07"\xf4\x81\x08} B\x1f\x88\xd0\xdb\xae\xf3\x03\x11\xfa@\x84>\x10\xa1\x0fD\xe8\x9d\xe1<\x1f\x88\xd0\x07"\xf4\x81\x08} Bo\xbb\xce\x0fD\xe8\x03\x11\xfa@\x84>\x10\xa1w\x86\xf3| B\x1f\x88\xd0\x07"\xf4\x81\x08\xbd\xed:?\x10\xa1\x0fD\xe8\x03\x11\xfa@\x84\xde\x19\xce\xf3\x81\x08} B\x1f\x88\xd0\x07"\xf4\xb6\xeb\xfc\xb7p\x19\xffPV\xfcS\\\xc6\xef\xac\xfc\xce\x10\xa1\xd7\x0b\xfb&\x88\xd0\xeb\x85}\x13D\xe8\xf5\xc2\xbe\t"\xf4\x98\xb0\x7f\x07I\xf3M\x10\xa1\xd7[\xec\x9b B\xff\x81\xe0\xf1\x81\x08\xbd\x04"D\xfd\xcf\x9e\xff1D\x88\xe5x\x9e\x129\x9cAQ\x81ax\x1cb\x08\x98\xc4aV\xa00X\xe4n,\x13\x91eq\x8a\xc41\x1aFH\x12CEQ\xa4\x11\x1a\x15\xa1\x1bZ\x85\x10~\xfc!a\x01FY\x9c$\xd9\xdb\x84\x05\x82ca\x8e\xe29\x91\xc6`\x94\xe7\x19\x81!\x19\x16\xc1\t\x92\xc3\x10\xf2\x8b\x1c\xc4\xd27\xa8\x8b\xc0\x8b0.\xc0\x04-\x10\xec+!B\x7f\x1b\xb2\xfe\xe3gS\x1e\x88\xbfP4\xba,\xee\x8f\x08B\xef\x8f_\xfa\tA\x08\xc3 \x96\xa4I\x04\xa7\xe1\xe5\xbf\x14&\x8a\x8bMY\x92\xc4\x11\xeak84G\xc10\x84\x12$!p,\xca\xd0\xe4\x8d,\xb4\x9cS\x1e\xbbM\xfc\x15\xc9\xdf\x13\x84\x9e\x80\x91\xf9\xff\xcf\xf7\x87 \xb4\xbc\xda\xe2/\xb7Q6\xe8RZ,\xc1\x83&Q\x04\x11\x04\x81"\xf0\x1b\x1fd\xf19\x04\x15q\x86\x12!\x9e]6\x1c\xc29\x0eA9\x8c%\x16\xd7\xbcy*\xc2\xfe\xf89A\x08\xc7\x96\xf3\x043(\x04\xdd\x86\x19\xc3\x84\x88\x91\x04\x8b@,\xc6\x8b\x14\x0e-GB\x14\x97#\xc1\xe1\x08I\xdd\xe6cS\x02\x85\xe0\x10\x0e\xddN\x19\xcd@\xccM\xf8\x7f\x86 \xf4/\x8f\xee\xbe#\x08\xfd\xbb\x0c\x9a\xdb\xe8\x94?d\xd0\xbc\xbf\x9f\x7f+\x83\x06\x81\x7f=\x0b\xfe\xb2\xad\xc1\xf3A\xdb\x11\xc9\xda\xdf\xc7LB\x9f\xd4\xc6hZl\xddgL\xaf\x1f\xc6`\x82\x88F\x87\x98\xa0\xc1\xf5\xa0\xf3\x84L\xe1\xdblF\xb4\xb4:\x14\xb0T\x83~\xe2\xb3Y!\x8b\xf9I\xe0\xdc\n\x08\xe11\xd8>89\xedi\x0c\x1a\xe2/4\x84\xe2$\r\xdd\x8f2\x1e\x1c\xa2-\xc8Qqe`\xd6&X9\x9cV\x05\x8c\x05\xd1x4n\xbf\x89\xe6\xad\xb0} \x9e-k:3\xb9\xe3\xd4\xd1\xe6\xc8\xb8\x9a\x18m\x1b\xb8Paf\x94&>\x92\x8f=\xa9\xe2b\xbc>\xb7\xd7\xe2:\x9f\xe1\x07\'6?\x8dA\xf3\xf7\x99\xef\xef5\x06\x97\x8a\xb0\xda^\x80|qw\x86\xebXL\xa9s\xbb\xda\x8e\x87.fY\x97\x82,\xa3L5\xce\xd5\x14\xa9\xae)f\xd3M\xb3j\xe3\xcej7\x13\xf0|\xea\xc3\x04B]\xa4\xd4\'\xda\x0f9\xa7\x15\xbd:\xaaO\xd4w1h\xc8\xbf@KE\xb3\xfc\xc4\xdd\xbc\xafq\x05J\xbc\x7f \xb4\xf3\xe4\xc2"\xc8\x00\xb0X\xa2Q\x10\xc9\x82\x8e\\\xa8\xde\x9e2\xc4B\xd6X\xce8>\xe1\x1e\x04\xd2>\xa5$s\xf4\xd2\x1d\xd54~x\xec\xa4\xa5*\xd1\x0e\x12\x1f\xe8\x16\xb6A\xc1\x02\xc4\xda\xe3{M\xf5\x7f\x1a\x82f\xf1\x07\x0c!\xc8% \xde\xcd\xc0c\xcd\xc3V\xd89\x03\x04\xa5f\xd7r\xda\x89\xaa\xddk\x1b\xe5F\tC\xa9\x8dl8%EIYgPX\xdc\x9c\x03\x03\xb9\xfa\x003\xd3b\x96\x0b\xe6\xeco\xd48\xa3\x94\xc2\x0f\xbc\xd2N\xdd\xb8u\xc3i}:\xca\x0fNM{\x1a\x84f\x91I.\xa5\xfa-\xf6\xde\x8d1\x8e\x93\xe1\xecB6\xcf\xe5\xebq\xef\x0f\xb3&\n\xe6\xd5\xde\xabIN*2?\xd5pz\xbd\xb8)v\xd9\xa1\xfb\x06#\xae~\x83\xf8\x07\xf8\xa4]Jm\x96\x0eJnk\x95\x04\x19\x8a\x89]\x00\x19\x9f\t\xc3\xa2A\xf4A\x99O\x83\xd0,n\x8f\xdd*\x01\x02\xb9\x1bkf\x1dg\x15\x08\xd6\xb4z>*\xad%\xc60yui}>7\x16\x1c\xef\x13\xc6=E\xc2\xc6\xf3\xeb\xc3em\x8a\x1d\x1e\x94\xe2\xbe\xf3\xb6x\xd5\xcdQ\xa6\xef%P\x05F\xbcG8\xe1\xa2Z:\xdd\x00jzu\x97z\xe9\xb1\xa1\xd4O\x83\xd0,\xd6\\tRKwq\'s\xcf*j\x80\x8f\xe5\x19D\x92\xfad\xd6-\xda\x9e\xd1h\x03+\x97\x19\\m\x07\x1bgOv\xa6DS\x11W\xf3,\xaeV\x01\x0fT\x8b\xc3\x0b{\x18\x8c%\\I\xb9\xd8\xc3\xf9\xf9(n\xab"\xf0*\xb0Mv\x18\xf7 /\xedi\x14\x9aE&|+hQ\xf4\x8e. \x12h\xb9\xae\x1d;!\xc3\xd8O\x1c"\xc8\xa6\xc1p\xc8\x1a\x8c\xcbf\x9d\x12=bl\xac\xb2G\x81 \xd8/\xf1\x1ch\x86b\xbbI7\x01\xee\xe6\\\x101\xab#\x15(1{\xd8\xc4\xe0q\xed\x92\xd4\xde<\xe7\xe5\xe3,\xb1\'Qh\x16\x99(u\xc3\xed\x10w\xd6\x0c\x13\x19\xc0\x98=\xc5\x9cql\xaeAJ\x9eYe\x9b\x9ecW&9\xc7\x02\xe0\x93|8G\xd5\x8a=\xb7\xdb\xdd\x05[\xc7\xc4I\xa3\xc2\x1d\xca\x1f\xac\x8d!Y\xb2r\xb4\xb4 `|\x19\x84vD\xc9h\xc4\xd6\x15\x89\xef\xa2\xd0,\xbey\xa3\x12\xe0\x08}gMw}p\xab\xea\xc8\xb6>P\xb3\xf8ZT\x93\xed\x10Tp#\xc4\xe7M\xe9e\xcd\x91\x1aGN.%83\x14n\xc4lNZi\xdc\xbc\xa7\x199rk\xdf\xc8\x88s\n\xe8\xab6\xa9\xbc\xd3\x18\x02\xa4/`\xcd\xe5W\x95\xc7\x9f\x9cBs;,KoB\xdf\xa3|\x08\xac\xcfv\x1bL^\xe5\xc7\x9c\x04Th\xbb\xa34M\xd1ON\x8d\xe1\xd1 \xcb\xf6\xbco\x13\x95\xd4\x13\xcc\x95\xb96\xb1\xbb\x84\xf4\xc5#`\xf6q\x84\xd2*\xdb)\x00\xaf\xef\xd1\x8b\xb9A\xd4\x8e\xc3\n\x19b\xea\xd3\x83\x01\xeei\x10\x9aE%\x82.\x91\x02\xbb\xf3\x88\x8e\xac\xaeh6Z\xfc\xd2\x18r\xc2\xc1`\xae\x85\xdd\x1dv\x9cY\x879\x1c3\xbcR\xb1\x16D\xb8\xc0\xae\xb7/V\xee\x82-\x99O\x86\xdd\x198l\xbb\xb9\x90\x00\xbc\xa4\x00B,\x932\x9b_\xdanR\x8bm\xc6=\xe8\x11Oc\xd0,*qty1\x12\xbd\x93\t\x06\xb8w\x01\xa5F\x15f\xd6BC>\xcb\xb3l\xa5\xd3\xe7m\'j\xd0rV\xc4\xd63\x88\x96\xafhAPg$\x13Gs@\xc9\x03hZ\xcc:e\x0e\x81F;j\x1e\xeb\xf2u\xads\xb3\xe5\x9cVG)\x8a\xde\xab\x82{\x1a\x83\xe6V\xc1A0\xb6\x9c\x97\xbb\xb8b\x9d=\xb2\xa6)\xfb\x1c\xd6\xe3b\xa4\xe8|\t4^V\xaaie\xaf*\xe9\x02\x19\xa0\x11\xea\x97:\xf6\xaf\xc2z+\x9e\xf0z\xb8\x9c\xae\x99\xa3m\xd5\x9e,\xf7\xf5d+W\x90\xab\x1dJ\xa4\xcd\xc6\x05A\xc8\xdeD\xbb\x07\t\rOc\xd0\xdc*\xb8\xa5\xf7\xa1\x7f\x92\x0c\xa7\xced\xf0&!#N,[`\'\xd3\xa5Kf\xdb\xf3\xaa]C*m\x8cu_NK\xa7\xbdR(\xeb\x12`\xb1-\xa4[\xae"\xf1\xf9\xd4\x95\xdb\xa0:\xf7x\x8dfU\xc5c\xc7C\xb1:\x85\x83g\x8b\xb9+=8\xc5\xf8i\x0c\x9a%K,*\x11\x84\xc2\xef\x865\xb3\xa1\t\xa1\xda\xf1\xa4_\xd2}r\xcaD\xef`I\xdb^TN\xe4\xd0U\x81\x1b\xef1Y\xd1\xc1&C\xa7n\x86\xf2-jq\xf8\xf9r\x80\xae\x00\xb0\x8e\xb6S\xd8\xf3\xbc~\x1e\xb0\x00\x94\xf2\x83\xaej\xe05\x89\xcc\xeeQ\xca\xd6\xb3\x184\xb7\xd2f\xe9\xc1a\xf4>\x19^\x08\x1d\xf7\x8bB#7\xc0\x8e\xb6VAK+\x01\x18\xee\x90\x03\x15et\xb1QwI\x0b\x98\x19^@vqu\xe6]\x84\xa6\x1a\xb8A\xe3\rg\xda\xbeB\xb6\xea\x194*\xdat\xf2\x15H\x07#-ls\xae\t~%\xf3O\xce\xa0\xf9J\x13\x14\x8dP\xd0]\xf3f{\x00\xa3\xa3.{h\xa0\xa4\xe9\xf0\xf4\xec\x8a\xfe:\xdc\xf02\x89&\n\x0e\xb4j"O\x86\xce\xe3\xe7DL\xf8\x93\x85\xa2\x17\x8d\xd4)\x90\x11bV\xe8\xc4\xac\x9aj\xfd\xba\xa9\xd1&\xbf\xd2\x9bY\x87@&\xa5\x94\x07\xefl\x9e\xc6\xa0\xf9\xaa\x9c\x10\x92\xc0\x88;l\t\x1d\x08{\xbc\xcf\xfa\xa1:\x0fB\x14\xa3~\xbe)\x14\x95\xed5\xaf+\xe1\x13\x8fZNE9\xa1\x9b\xd3\x8e\x99E\xdb\xf4\xb2\xeaM$?\x0e\x9dEC\x88\x89\x01u\xc5\x8b\xd3\xeat\x96\'\x15a\x8d\x15\xba;\x1f\x86\x13\xff j\xe7Y\x0c\x1a\xf2/\x10\x86`\xd0OJ\x1b\xb6\'\x89\xda\xda\x99>`(\xaa\xab\\\x04\xb2\x84\xd5\xccV\xcf\xa1\x05hy1\xaeFi]\xa4\x17\xcd1W1\x89\x1e\xb2k6@\xd4Yi\xd8\x18\xa4\xc7\xb3$S\x0e@\x87\xbd\x16\x02pUxh5\xa9\x13\xcb<\xd8\xa2>\rA\xb3\x9cY\x82\xa0al\xc9o\xbf\x979\xc7\xd4I1\x1c\x83\x8eq\x80\x00Nn\x7f\x9d\xdc\x94\x8e\x91\x82\t\x8d\xadik5\xe5l\xd1\xcd\x9e\xd0w\xeb\xc0\x81\x118\x01#\xc9T\x84h\xb7Mu\x80"\x14\x05\x0f,=\xa5X\xb0\x91\xae\xc7\xb0\xf2m\xc5\x1d\x98\xb7J\xfaOc\xd0\xdc\x1a\xfd\xa5\xd1Bh\xfa\xee\x1a\x13\xc7<\x0e\xb1/\xa0B]\x02@(\xf9L\x8ezH\xddl)\xac\x89$\xc7\xd5\xe6A\xcf6\xfd\x91\xf4j\xc0\x0b`\xba\x1a=\xf9\x10k\xfbCM\xe6\xb2\xd0\\\xca\xed\x88g\xdcz\xd3"t!\xd6\xd2NvN\x85\xf1+\x92\xcf\xab\x194\xb7j\x7f)\xf7!\xe8Nen\xe8\xdb\x8b"\x9d\x93\x08\x8b\xaf\xe2(M-d;n\xb8\x83\xf6=\\j\xa6\x12\x92e\xe8\xb5G\x11Sw\xa4/\xe5\xa1*\xe1\xc2\x8e\xa4\xdahw\x8ch\x03\x08\x8b#\x90yg\x9c\x94\xe8d\xe5\x89i\x03\xc4\xbd\xfa`\x1d\xfc4\x04\xcdbL\x94\x82p\x92\x80\xef\x92\xa1\xc6\\5c\t\xe0g\xfd\x92lT\x89\x1d8\xe0\xb3\x10\xa9\xeed\xca\x86s\xf4\xda\x01\xdf;\x02AI\xc2\xe5\xea`\x8aA\xad\xe7\xe3\xa6e<*\xe3\xa8\xa9"\x0e\xaaL\x8b\x85\xbc\xdeG\x98\x06\xf7i8\x1cg\x19\x12\x130\xbc(q\xfe^a\xfci\x04\x9a/\xcfG\x16\xb7 \xee\x89\x93\x1b\xd6\xd4\xdd\\\x9a\x84\xb5\xbfa\xad\x8b@s\xcb\xf9$v{`\xe1N\xe6\nOJE\xdc\xe3\xaa\'\x96 :\x1e\xc8\xbaw\xe7\xd8\x9dz\xbdR)\xb4d\xf7\xe5ibf\x00\xdd\x95k\x05\xa0\xe5\x99G\xc4\xdd\xe0e\x06\x12_mSa<1S<\xb5\x1e)l7\xeb\xd2\xfa\x88\xef\xdc\xdd\xeaA\x94\xd8\xd3\x084\x8b\xe7\x13\xc8\xed&\x93\xb8\x0bpA\xb8\xb9\xee\x880\xf1R}^s\x0e\x8f\x1f\x8d\xb0d\xd2\x84Q\x90\xd5\xee\xa4C\x97"u\x1d\x1d\x95\x1dy{m/\x83\x81/\xff\xab4\x93^z \x8aB\xa2\x03\xc4&\xd5Z\xd7\xfdu\xe6k BVZ\xcd\xcd\x0fz\xfe\xd3\x084\xb7\x0b\x07\x1aC!\xe2>\xc0y\xd1\x98:\n\t\xa9))h\xeeN\x01j\x08\x0c\xd9q\t\xda\xb4\x9a\x95\x82=\xc0\x85\x14\xb2p\xa1\x02\x81\x8d\xa3\xd8\xd91\x0c\xf7xr\xb6F\x8dE\xa8\xdc\xed\xaaM\xeeA\x07\x1a\xa0YSJ+hk\xc1R\xf3\xabr\xfcON\xa0Y\x0e\x0bB!\x14\x8a\xdf\xe3\x8ax\x11\xd7\x03~[\x9d\xbdL\x0e5\x05sQ%\x02\xccM\xb2\xb5\xcd\x92\xd9\xcf\x96@\xc0E\xb0\xab\x87q\x85\xd0>\xd6g3\xb0\x1a\xeb]k\x97\x9d\x1a\x88\r6_7Y\x99\xe4<\x8f_\xa5\\\x92W\xbbEP\xfe\x9b\xeb\xffS\x02\xcd\xd7\x07\x86\x7fO\xa0\xf9\xed\x99\xbb\x0f\x1c\xe6\xaf\xaf\xac\xbb\xc9h\xd4\x06\xfc\xf5\x0e\xc3\xefq-\xffE\xa8\x98gC8~\x98<\x83\xadjo\xd0\xebE\x9d\xeb!\xa6k#\xc6\\\xc0\xba\xab`:\xea\xf7\t\x02\xf7?^\xf3K\x15/\x11\xd3\x99\xfc\xcf\xc5\xa4\x01=\xc7\x01\xf5;\xf3\xbc)\x1b\xe5\xbf\xcb,F\xed!\xabZ\x9f\xf5\xc5\xefMW\xc1M7\x99\x0c^\xc7\xf4\xdb\xc0\xe3\xe3\xcf\xc4\xbc\xabYnJ\xf4\t\xfb\xb9\x92\xda\xffE\xfcy_\xb0\xcd\x128\x8b\xe5\x9c\x15\x88>3\xc3"\x045\x96\xf3\xa6\xd7\x02f\xba\xfam\xce\xca\xcbL\xf3\n%;\xcc\xe4\xb0A\x9f\x05D\xbfy\xce\xa2l1\xd7h\xb8\x0c\xa6\xcf\xec\xabM\xf3t\xf0\xcc"H\x18\x97\x10\xb0\x98\xc4\x83\xf5z\x11\xc1\xef\xe0E\x1c\xa4\xcf6\xa4\xbf\xd2k^\xa2\xc4p\xb0\x9f+q\xcb\x97\x9b\xe6\xd9\x00\x9b\x1f\xa6+,y\xa6@\x8c\xc5\xfd\x17\xcf\x19M7\x1c\r^\x98\x17\xa1\xd3K\xf3\xcc+\x94|\xf9\xff\xcf\x94\xe8\xf2K\xf3\xcc\xd3!8?\x0cW\xef\xf4z\x07-\x01l\x11\xa2/\xc9\xf2\x16\xd0\x04\xdc\x98u\xe4\x95fy\x81\x929\\|_\x87\x8c[\x10\xe3\x93\xdb\x88\xeb\xc5k\x96\xd4\xef.\xea^k\x96\xa7\x83tnI\xb33\xea%:\xf3;|\x11\xb0$NfI\x9a\xc2\xa0\xf3\xc9k\xbd\xe5\x15J\x96B\xe6\xe7J\x8c\x9ff\xcb\xf7\x85\x06\xfd\xd0ge\tb\x02b\xd4K\xaaw\xc3\xe5\x95\x962\xe0\x96\xfay\xef\xa5fy\xbe\x92_\xd4\x97\xa813\xf3\x8b\xcd\xf2t\xa0\xcf\x12\xc4\x14xU3\x8bY\x84\xc9\x98\xedy\x11\x83\x9a\xee\xad\x9e\t\'\xf3\xa7\xae\xff\xa6h\xa2\xef4\xcb\xd3\xa1@\x7f\rb?\x15\x03\xbd\xd2[^\xa2\xe4\xbb\xcc\xf2t\xb0\xd0\x8fE\xc4\x92\xf2\xbdy\xa9]\x96\x1e&Y\xd2~\xb1\xf4.\xc9\x928w\xe3K\x83\xd8\xd3\x95\xfc\xdc\xef\xed%\xcf\x14\xe3\xab\xcd\xf2l8\xd1\x92[\x96Jl\xbe\x99A\x80\x8dYY\xaa\xfd\xdd\xb0\x98f\xbe\x99\xe7\xa5fy\xba\x12\x83\xb7\xa1%\xe5\xe3\xfa\xecA_M\xb2\xbb\xd4{\xf3\xad\xbe\xbcM\xd4xm\x81\xfcl\xc0\xd1\xadG^\x82Xqs\xf9\xa5(N\x16O\xd9\x8dKq\xfc%\xe8\xa5\x05\xf2\xf3\x95\xcc\xca\xd2\xb7\xfc\xb4x\x19\x8d\xd7\xde\x8f=\x1d\x92\xf4\xc3t\xb0\xa5\xca\xb7\xf1\xa5\xac\xbc\x95\x96\x8b\x18\xefvu\xb14e\xdb_\xf4\xc6o\x8a{\xfaN\xb3<\x1d\xb4t\xbb\xb6\xe8t\xf7\'\x170KD~m\xdf\xf2t%\xb52/f\xb9\xad~\t}\x02\xb2\xc4\x81\xc1\xa8\x97vri3\x7f~\x95\xf4\xbeP\xa9\xc5,K\xdf\xe2\xdeLQ,\xe7l\x89\xca\xee\xadO\xb6G\x93W^k\x96\xe7+Y\\\xcc\xa1\x0cKd^\x9a\xb0_=[\xf0\xa6\x18\xae\xef4\xcb\xd3\x01X7\xb3\x8c\x8b\xeb\xc3\xb7\xcf\xf8\x96\xb4\xbf\x08\x11\x96\x86l\xf1\x16\xfe\xb5\x05\xf2\x0b\x948\xd8\xbc\x140\xb7\xc2e\xf9\xb7XL\xb3\x83\x0c~\tj\xfc\xcb?\xa1|:D\xebf\x96\xe1\xeb\x13\xf1Y\x9f\x96\x7fq}\xe9\x93\x97n\x7f6\xea\xfdk\xcd\xf2\n%\x0f\x86\xe3\xf7\x05\x86}\x99\xe5\xd6\x88\xe9n\x88|\xd5,\xf5\xad*[\x92%\xbf~m\x10{\x85\x92\xef2\xcb\xd3a^\xdf\xe6-/Q\xf2]fy:\x10\xec\xdb\xcc\xf2\x12%\xdf\xe6-\xff\xb7\xc2\xcf\xbes\xd3\x9f\x8e\x1d\xfb6_x\x89\x92\xef2\xcb\xd3\xd1e\xdf\x96\xd0_\xa2\xe4\xbb\xcc\xf2t\xfc\xd97\x96\xbf/P\xf2]fy:B\xed\xdb\xbc\xe5%J\xbe\xcb,O\xc7\xb0}\x9b\xb7\xbcD\xc9\xb7\xd5Y\xcfF\xb9}_W\xf2\n%\xdf\x96\xf2\x9f\x8d\x83\xfb6\xb3\xbcD\xc9\xb7\x05\xb1g#\xe5\xbe-\xb7\xbcD\xc9\xb7\x99\xe5\xd9X\xba\xef3\xcb+\x94|\x9bY\x9e\x8d\xb6\xfb\xbe\x94\xff\n%\xdf\xd7N>\x19\x8f\xf7}\xb9\xe5\x15J\xbe\xcb,OG\xec}[\x10{\x89\x92o\xbb\x13{6\xa6\xef\xfb\xee\xc4^\xa1\xe4\xdb\x82\xd8\xb3Q\x7f\xdf\x17\xc4^\xa1\xe4\xdb\xbc\xe5\xd9\xb8\xc0o\x0bb/Q\xf2\xa0Y\xfe-\x08\xd8?\xdc\x01\xfdS\x08\xd8\xef\xca\xc7wF#\xbe^\xd87\xa1\x11_/\xec\x9b\xd0\x88\xaf\x17\xf6Mh\xc4\xc7\x84\xfd;\xa0\xbdoB#\xbe\xdeb\xdf\x84F\xfc\x0f\x04\x8f\x0f\x1a\xf1%hD\xfa\x7f\xf6\xfc\x8f\xd1\x88$\x8c\x8b\xe4\x8dR\x81\xf3\x1c\x0e\xf1\xfcm\xec\x19\x07\xc3\x02\x8f $L\xd2\x02,\x08"\t\xb34\xc7s\x0c\x8fp\x08\xb9hbp\x1ef \n\x16E\xf66\xf6\xed\xd7\xe0/\x1eCX\x92eP\x86A\x18\x96cQ\x9e\x17P\x0ce\x10\x0c\xc5\xb0\x1bq\x8fc\xc4\x1b\x07\x91\xc7`\x1a&\t\n%\xb9\xdb\xfb\xc3\xc4\xf2\x0e\xdc\x8d\xea\xf4J4\xe2\xdff\xaa\xff\xf8\xc9\xf81\x14\xfd\x0b\t\x110\x89Q\xbf!T~EG|\x7f\xae\xe4O\xe8\x880\x87\x930\xbd\x98\x17\xc7I\x8c\x80E\xf66\xdf\x8e\xc6H\x06\x82i\x06\x86\x18F\x84I\xfe\x0b\r\x88\xf2,O10\nS\x10\tc$K`\x08\xc7\xdd\x8e\xea\x87\x8e\xf8B:"B\x8b\x18Eq\x02\xc4\x0b\x08\xc6\xa0\xcbz\x97\x9d\xc2\xd0\xc5M)Z\x84\xe8\xc5\rq\x86GY\xfe\x86t\x83\x97=\xbe\xcdREP\x8a\xc0\xa0%\xa6\xc0"\xff\xe3\xe7t\xc4\'\xd8\xe9?CG\xfc\x97avO\xa3#\xde\xc6\xfa\xfd!\x1d\xf1\xfd\xfd\xfc{\xe9\x88\xe4/xz\xa3Wz\x13\xa6xj\x1clW\xb6K\x99\xdcYl\x82a8\xea\x8c\xb2\xe4\x06\xc5\x13w\x9e8\x89s\x19\xec\xd6\xe3U\x0e51\xad\xf7<;2akD>\xd1o\xb6`|\xda\x9c\xf7\xa1\xa58\xb5W\x16\xb5\xcb\xe4\xdf\x84F\xbc\xa5\x04\xec\x16;\x90\xfb\xc9\xec\xab^\nW\xfe\xe6Rm\x95\xc9\xda_\x19\xed\n\xbb\x1dC\xda\xd7\xfdn\x00#&m\xa5B\xdc\xb1\x90\xc4\x8a\x07\xb36\x19\xb5f\xac\xcb9\xb3\xb6\xf6$\x1fM\rn\xf7\xc7\xb1g\xe9\xab$R\x83}\r\xf4\xb0\x9di\x80yl>\xf3\xb3\xd0\x88\x8bL\n\x86`\xf4\x1e%\x90\r\xd9\x16h\xa2\x01\x06\x8f\xe2\xe6\x8c\xca\xb4,q\xf6q-\xeem\x87\xb5\x84\\\x18%@>\x9e\xe1n\x07\xd4\x07\x80\t\xcaN<\xca\xb0\xd2\x86b-3\xe7\xcbi\xa3y{\x7f\xcd"\xebv\xde\xefV\xd7\x11\xb3\xb7\xd0\x83\xf8\xc7g\xa1\x11\x17\x8d4B\xdc\xe6\xa6\xa3w`\x08\xf2p\\\xed\xc8\xfaT\xd0\x84!\xd2\xa7\xac\xce,\xd7\x88\xcb\x10@*\x14\x91\x05\xf9\xb2\xa5X\xe0\x10\xb5lUj\xaa\x9c\x8f\x8c\xdb\x13ak\x8f`\x12\xaa\xe4\x057\x8b\xe2\x80\xe0Y9C\x88e\x05:~X\x19\xf3\xfc^d\x9dg\xb1\x11o\x0e\x81SKbB\xa8\xbb\xf1\xf3\xc7\xd3\xa9f\xda.\x80\xa7\x83\x90\xcdk\x02\xe0P"\xc1E\xfeJ\xd50\x13\x1ds3\xa1Kn\x88\xb2\xcecC\x1d\x94\xa2mb\x91\xdcP\x1e7\xab\xc4\xcbW\x15D\x87{\x1b\x01\n\x02\x17K~\x87\xa5\x05]\xd6\xf8\x83c\x8b\x9f\xc5F\xbc\xc9D1|\xc9a\xe4\xdd\xf8\xf9\xb4A5*-\x11p\x7f\xe5-\x10\xde\x1f\t\xff\x8aPU\xad\xad\xad\xd5\xce\x19\xfbZ\xf1\x11\xcc(B\xf3\x80\\4`?n\xa3\x15\xcd`\xe1a\x05y\x8d\xa6\x9ey\xc0\xaa0X9\xf8Mk\xd9\xfd.\xaf\xf7-\x8a\x18\x82\x9e\xc5F\xfc\nA0L\xe3?\xa9\xb0\x8eM\x07\x11\xf9\x84\xc9}\xb1\xa9\xb3m\nut\x8fp\xa76G\xad\xf1\x82]\xdcR\xab\xc9~\xbf\xba0\xd7EF\x17\x82SF\x9f\x9bj\xcb\x92k\xff\xbcT\x1c\xf6zW\xb1\xa1\x0cd\xbdP\x8b\xbe\xc1\x94\xad<\xa8\xbfb\xdf\xfc\xb9\xd9\x887\x9fX:\x95%Z\x92w\x87%\xf40\x95+S\x04R.X\x7f>\x0c\x8d\xbeQ\xdc\xc9\xf7\xdb\x14\xf2\x8b\x99\xda\x1d\xd4u\x90\xa6\xfc\xc0j\x92P!\x85\x04%\xeba\xc4\x1a;\xce/\x03_\x14\xb4tq\x1b\xe5p\x15\xa8\xd8\xa6\x0c\xd4i\x81\x8a\xa5\xbe\t\x8ex;,\xf0\x12(p\x82\xba\xa3\t\xc8\x95\x1dv\xda.\xb45\x05\xe9\xe5\xeez;\x1e\xd5u\x12\xe4\xae\x9c\x12\x82\xb2\xe0k\x98\xb4H\xeeA\x06\x11\xc7\x01\x07\xfa\x1d.X\x97\x16\xf5\xa2~\x17\x14\x87\x8bb\xb4\xb3\xd3*\xccQUX#HJ\x83\xdb\xe0\x0fbQ\x9fEG\xfc--/\x8d$q\xcf\x868\xaa\xd3vCz{)\xe7\x87\xd6\x00SX\x90\x14N\x15\x11`\xd6"\xbfGw\xd6\n\xdd7\xd0u\x87\x01\xc3\x16\xab\xa9pcJ\xb97\x88\xf8\n\x0c|\xf2h\xe8+sH\x08\xa70\xba\xb8\x0e6\x16h)\xb2\x8a\xfe\n\x97\xf2\xe7\xa6#~\xa5Cti@\xa9{\xe8L(\xb5\xd9\x04\x06`)\x8f\xb0|\xcc\x85s\xef\x99X\xb7\xb2\xc8\x95\x10\xe7GHt\xa4\xc4]{\xc7\xceeOY[\x99:n\x97\x17\xe5\x9c*\x9eTmc\xc0dK\xfd\xa8,=\x916`\t\xbd\xc3\x148\x81#\xfdA\xc2\xc6\xb3\xe8\x88\x8bL\x82\x82Hl\xc9\xaew\xc5M\x95U\xb4\x90\xad\x86v\xb7\x81\xa3\xd3\x01\x80VN\xbd\xbf\x9e\xa6\xe2\x1a\x1c\x06\xecB\xda;\xca:\xe0\xfcQ\x1c\xc2\x91j\xaa,\xb7\xf83\xb2\x82\x95\xe1\xba\x02Ig\xac\xecSH\xa3\xcc\x86\x10b%\x8b!\xa2\xe9u\xe8\xf0\xa0\xeb?\x8b\x8e\xf8\xd5\xa2\x92\xf8\xd2\xdaRwY\x1f\ndS\xb0\xdaU\xaa\x14\xab\xe5e7x\x0b\xaa\x8a\xdf\r>\xbe.\x05\xe2j\xa2Nw=O\x1d\xd4\xfa\xe9\x8a\x88\xac\xfdFAqE\xa8\xe7\x90?tZ\t1+\xf8\xa0\xc3\x19\xe4\x0eBA)\x93\x0f\xb6\xbe@>Z\x91?\x89\x8e\xf8\xd5x\x10\xcb\xb7\xd2\xf4]:\xac\xf9U\x96o\x077\xba\xd4\xd29`2\x9b7\xf8\xe60\x1e|\xac\xc1\xf2\t\xb9\x0c*5\xac\x03Z4\']g\x01v\x08\xc1\xb8\x95\xb7\x181\x91[\x91\x95D\xbfB\xeb#\xd4H\xf6\x15\xd1u\xf9\xe2\xe0\xd7\xbc?\xfd\xaa\xb8\xf9s\xd3\x11\xbf\x02\xe8\x12<\x96\xaf\xdf\xf9\xc4\xc5\x99\xdc],\xb0\xfa\x96\xf4L\x07\x9e\x99\x13\xc0]\xa0u\xd5\xcb<\xb0/\x14\'\xb3\xe2\nZ\xea\xc8\x83\xb5\x9da\x15\xb2\xa3`w\x92\x98N<\x19Wi\xc9\x9c<<\xf1\x93\x13W\xb5l\xedAPJ\xb1\x1e\x89\xba\x07\xf3\xc4\xb3\xe8\x887\x99\x8b\x13\xa10t_;1\x13\x8d\x8c5|6\xb1\xc4\x93\xa2l\x05\xc7\xc1\xd4\xd1\xd8F9\xd6Z\xd1$W>\x87wh\xcc\x84\x86\xd3\x87\xc1\x00\'$\xeb\xf4~\xbeo\xdd\x03wN\xb2<\x14\xadz\x10\xbb\xbd\x86z\x8a\x8f\xa9\xa1x\x02z\xf6A\xea\xcc\xb3\xe8\x887\x99K\xe7~\xbb\xdc\xbe+n,C\x9a\x8ai`jj]\xa5W\xd0e\x15\xab/rv\xa9\xf8\xd6\xfc\xc6\xe4\xdbN\x06 \x99\xc7\xbc#%\xf3\xae\x91\x9f\xaa\\p\xb6\xb0\xac\xe8Y\x93C4\x02\x808\xcc\r\xf6\xba\xd7\x93\x11D\xeaXA\xb9qz\x90x\xf7,<\xe2Wq\x03--*\x85\xdf]\xc21\xba\x90\xb9\x14i\x0b+\xd8\x987{#M\xe6\x8c\xa8py\x17\xaf\xcd\xe6\\\xf4\xe7^\xf5B\'O\x91\x1dG5\xc4\x01\x93\xaar\xbd\x02\x92\xa5T\xc3\xd7>y\xba:\xda\x15I\xd2\xd6\'\xea\x93\xdc\xc45\xb3\x1c6\xed\xbd(\xe1\xcf\xc2#~\xed"A\xe00vO\'C\xae\x05\xe3\x95\xdc\xa4P\'\xc3\xf6\xd7\xa8\x86+\xa7\xd58\xf7,\x8f\x0bC\x8e[ {\xd9\xedT\xb4,\xc1\xbdd\xec\xb7\xbd\xc3\xadx\xd5\x8c\x9bV$zH\x8f\xfb\xb5\xadiz\x00M{%\xd5+X\xb1\xe0\xbeo\x1fl\x82\x9f\x85G\xbce\xfd\xa5\xffZ<\x02\xb9;,\xc6\xae\x86\xf2\x19 \x10oG\xc3R\x97\x922c$\xddq"\xe0\xaa_Y\x11\x9c@\xae!t\xe3\xc13\xd8U\r[\xf6\x861VY\xb7\x93/\xb3&\x84X\xeelvNh\xc3\xcdu\x05\x91\x89\xb4TE\x87\xda9>\x08F}\x16\x1f\xf1\xab\t\xbe\xb5\xfa\xc8\xfd\x05\xd5\xd2NxK\xfc:&\xc4\x08\xd9\xb8\xbb\x03\x8d\xae\xc5\xa8\x8aQ\x0eVqA\x9d\xdd$\x84\xb1\xd5\x84\xa7v\x83b^=.\xa5\xb4m\x1brx\x1e\x1d0\xddlu\xb2\xf0P@Y\x15{&\xa8UvJ\xa4u?c\x0f2\x91\x9f\xc5G\xbc]\xda.\xa7\x96\xa4!\xec\xaeTu\xf7\x83/\xdd>\xbd\xa6A\x99oX\xfd\xa4\xcc\x92\t\xc3\x08\xa4J@\xbd\xc7\x90\xc9;\xc0M\xeeT\x17\x92\x03\xe1\x15\xaa\x02\xee\x19\xd5R@/}7\xc6\xf6t\xb62\x86\xc6Q\x91\xfax8\xc0\xc0\xf58\x8c\xae\x00\xfe\n\xa9\xf7b>\xe2W G0\x94\x80\xd1;\x99G\xb7\xda\xeaK\x97\xcfh\xeb\x95`\xc8\xcbi\x95H}6\x89M\x03Y\xda\xb2\xa0\xcd\xe1\x0c\x1c\xae>Ft\xe98l7"$\xcbC>$b\xa8\xaeNlM\xc1iz\x11\x81\x8e\x1fvl\x85\x9a\xcb75\x1a\xdc?\x98\x96\x9f\xc5G\xfc\nA\xd4\xe2\xd8\x04qw\x0fg4\xa0\x18\xbbQ\xc1\x9e\xdc\x93q\x00\xaeig\xc5\xc2v\xd8j\x8b\x82\x8c\x94.KM0\x82|\xa4c\xc9mX\xc0\xa6`\xdb\xa5\xdfX\xc9\xc0\xe2\xbar\x9b\xeeU\xe2\xa8L+A\xab\xd2\xbd\xe2\xa6\xeeA\x9c\xa6\xee\xf4+\x99\x7fn@\xe2\x97\xeb#\x14\xb4d\xc3;\xd7\x97\x8aBDy\xd10\x86ql\xc3\x9d0\xd9\x15\xac3\xf8\xd8e\x19\xa5c\xe7\x841B\x98u[\x86\xdf\x0e\x07\xe4r\xc6\xa2\x94 \xb6\xe3\xa9q=\x05%\xc7\x98PgVn\x08\x96\xcd\xa3\xb0\xf2\xf9C\x83\xeb\\\xfb\xe0Go\xcf\x02$\xde\x029\x8e!K\xa7@\xdc}\xd2\xd0\xe3\xd3\xd1sV\x97V\xb4\xbcn\xe0#As\xa0j\x8d\xac\x89+\x98Z\xd6<\xe0\x97\x93\x1dA\xab\xed\x98\xb6\x02\xd9\x93+\x06nw\x00\t]7\x89?\x01\xb9\xbf\xea|\xa3+\xacMnA*\x0b\x95\x83\x1c\x01}\xf1M\x80\xc4\x9bL\x02\xc1o\x9f\xbe\xdd3\xef\x0e\xb9\xee^\x88\xd3\xcah\x90\x8d\xb5\xde\x84q\xc6d{\xd12\xa2\r\xe8\xa4\xe5\xc0\x82$\x9b\x92\x9b&\xd7\xc4c\x12\xd1\xa3\xdfa>\xcaz\xa9\xecQ\xb3\xd9\x9e\x84TX\xad\xa5\xf5\x16\xe3\xc8\xd3$\x96\x8d\x89\xf16\xfb\xa0\xeb?\x0b\x90x;\xb4K\xb3\xb7\x04\xb8\xfbC\x1bL\x15\xbe\xa5fo\x89Q\xf6f\xeel9r\x8ca\xb0\x0eSm\xb2\x93\x84&3\xef\xb2h4]\xa7\xc6\xdb\xf4:i\x18\x94\xea7\xa7Ci\xc7\xa5\x10\xb7$\x8b\xf9\xc6U\xec\xae\x17\xf5\xc0;\n\xb9\xb2)A\xd6~e\xcd?7 \xf1v\x9by{\x14k\xa9\x15\xee\x0e\x8b\xbb-\x0c\xc4\xdb\xb2\xa4\x9a\xd4\xa9\xa8\xf9~Al\x18\x1cp%\xc0\x92\xd05d\x9f\t\xf3\xc4\x92\xe7\x19^\x07\xea\xb9\xb5\xe9CW\x99\xf36\x04F\x1e\x9b\x8f\xa2\x9a\x97D\xc3\xa7\xd5\xfer&6\x08\xa0W\x8c\xc5\xab\'\x8e\xf9\x97\x00\x89_W1\x1f@\xe2\xbf\xd2\xa5\xebsU]\xd4\x15\xca\x14\xaa\xb5\xd6\xac\x00\x9b\x91C\x8bc\xc3q\xa7\x04\x08x\xe0\xcf8\xd7\xd0W\xadUx\xc6\xb4\xbdu\xce\x97\x02\xc0k}\x1d\x0c\xba\t\xf2\xa0\x05\x1a\xde\x10\x1e\xb9\x07\xa7)?\x8b\x12s3%\x86\xc2K\x81F\xdd\x8d\xa8\xd3\x8f\x1eB\x1bT\xbf\xa9\x97%G\\\x13\xa9\n\x0e\x9c\xcd\x01c\xd6G\x8f\xbf\x86\x12#\x08Bg\x84a\xd1l\x85\x13\xd1f\x9b\xe1\x12\xf8\xee\xc9\xec$*f\xa3\x93\xb3:\x86\xf8\xb9uzbo\x1d\x00\xd4\x06\x12>\x7f\xaf\t\xe3\xcf\xa2\xc4|9\x04\x8cC\x10r?\xb1\xd5\xe2\x82\xebF<\xc4\x04q\x0169\t\x94a\x92\x1c8\xdd\x05{p\xd3\xec]\xe3\x1a: bY-&V\x13\x1dB\xe2\xd6j\xf8\xcb|D+\x94\x07;<\xb4\xed\xceJ\xc6+\xc0\xce\xc6$\xea\xe7\x15\x9e\xe2\r\xf3\xe0\xd8\xc6gQb\xbed.)\xebv\\~/3qb\xf6Py\x82\x05\x01v\xee\x92!*o\xed"d;\xc0\xd8\xefc\\\xe2\xb6\xa7P\xdb\xcd\xc5\x11^\x8ag\x8b\x1b\xf1\xba/\xa7\xce<\xea\x13\x8e\xc1H\xec{F(\xc0\'\x88\xf0\xf0\xf9\x18d\xb8\x18\xef\xe6\x93\xf6\xa8\xdf?\x89\x12s\x93\xb9h$\xa1\xe5\xa5~/\xd3q:2\x1c+\xb9QB)KM\xa3\x8a\xcdB\x82F\xa1\x89\xed\x1e7\xa5\nq=\xd2\x04\xf6\xbb\xc6\xcb.\x87sS\x8b\xacq\xd4\x8a\x0b\x89\xcd\xfe\xe1h\x0f\x90\xb7m\x86\x8a\xe0e\xce\xb5\xce\xf3\xb5M\xc8\xc9s\x1e\x05D=\x89\x12\xb3\xc8\x84\xa9\x1b\x8f\x01\xa6\xef\xac\xc9\x8e\x04v\x95\x95\x168\xab\t:\xee\xcd\xe3\xb5\xb9\xa0A![6\x08\xaf\x12\xea\x80\xcbT\x92N\x17\xe0Hy\xb3\x8er\xbe`\xeeb\x16X\x95\xe7N;;\n\x10\x0e8f\x12\xa5\x18\xfa\xbe\x0c\xa9;\xddUOx\xff\xe0\\\xc5gQb\xbeZ\x98\xa5.D\xd1\xfb!\x9c#sF\xa1i\xd8\xc3\xba\x9b\n\x9e\xef\xbai\x06"P}I1\xd6V\xc8\x90\xf1\xc6\xa1\xfa\xa5N\x0f\x8aDhMRl7\xc6\xd0\xf5\x94|\xdd\x04\xe7\x8bU\xa3k\xabt\xba\x8bL\x1e\xc5\xa4l\x94S\x02K\x9d]\xc8\x0e\xe0;\xac>\x8c\x97$G\xdc\x93~f\xa1\x03\xbd\x93\xa44\xe9\xb4}\xb1\x1b\x82\xf3\xa3Y\xffI\x94\x98\x9bL\x92\xa6n7\x95\x1c\x1f,U\x9fE\x89\xf9-\xeb\xc3\xcb\x0fCw\x159\x01\x01\xe4\x85\xd8x\x90\x89y\xda\xee*\x87+_&\xf1\xbc\xc7%A\x06\x08 \xdb\x90I\xbb\x8f\x1d\xb8\xdd\xe1\xbbc\xc1\xd0\xb9\x02\x8db\r^W\xdd\x86\xdb&\x0e\xc0(\xb1\xcd\x9f\xd8\xdacZ\x173\xa1\xf4z\x0c\xb3\x07\xe7\xc5?\x8b\x12\xf3%\x13\xbb]]Sw\xae\x8f5c\xb6q#\x17\x83\xfb\x194\xba\xcd\x99\xa8\x1c:\x82\x9a\x8d\xaf\xdf\xa6\xe6"\x92\x93;;\x9f:X\xfb>\x82\n\x051\xeb\x95\x0c\x8c\xbb\xd1\x1cwm\xb1cu\xbd\x8aG\xe6P\xcc;\xb0\xdb\x1c;*0\x01\xbf\xf9U \xffsSb\xbe\x02(\xbd8?t\xdf\x04\'\xdc!\x02\xa2\xc3h9\xa7\xe3\xe2\xe4`\xed\x0f\xe9u+1\x1b.\xdab\x12\xeb\xee\x95\x1a\x83z\x90n\xae\x94\'C\r\xd3+\xd7\x9e\x03\x91S\x12\r\x1d\x91\x96\xc1\xdc\xb0+\x92R\xca\xc2\xe3\x9a\r4\x9eH\xfc\xa8<\x8aOy\x12%\xe6K&\x8a\xa3\xc4O\xc6\xd1;yZg\xadj\xa2\xe6\xecn\x85x\xf4fiE\xd7\x90m\x9d)\x18\x8ffm\xaap\x11\xb8p\xb3\xcf\x1bq\xc9\xd76\x9bAk\x05>\x8f^\x1f\xebS\xe6\xae\xfbye\x91^\x1e\x1a\x17\x14s\x07\xc7`\x90\t}\x90\x83\xf5,J\xcc\xd7\xb5\ryC5\xdf\xb9\x84w\\\x02\xdb.R\x97\x926\x0b\xc9\xea\x9a\x9fP\xc7\x99\xcf\xc9\x8a\x925y\x0bJ\x16\xb4\xd53mtwC\x81\x88\xf9a\xed\xf4[\xde*\xf2\xed\x1a\xb76%\xbbDm`r\x14c\x1d\x11\x1d\xb7\xd9\x83\x13\x1bw\xdb\xe2\xc1z\xffY\x90\x98/c\x92K\x9d\x84@w\x85\xf0\xferE\xb6\n\x7f\x98[\xd79PU\xb0\x8f7\x04\xd0\xbb\x9c\x9cE\x82%9\xfb\xc1\x13Y:k\xd9\x03,\\\xa5\xbe+\xf9\xceQ\xa39\xecv+\x9fW\x0b\x15\xaa\xf0Y<\\\xf8s\xa6\xd8\xfb\t"Z\x86\x00\xb9\xf7\xe2C=\x0b\x12\xf3[\xabO\xc0\x14y\x0fI\x9e\x03\r\x1d6\xf5n\xea\t\x9c\x158]r\xbb\tf\xa4\x96[160\x16\xbd\x15\xb1\xf6\x99\xf7\xb7\xe7\n7x\xc3w3[6f]\x9d\x06\xbd\xe4d\x9b1x\xfa\xc8\x8d\xbb\x13\xe4\x9bb\x92f\xfc\x0648\xe5\xc1\x1b\x8dgAb\xbe\xd2\x04\x8c\xc2\xcb\xf7\xde\x1d\x16l\xd7P\xa7\x0b*\x19CpQ=\x02\x94\x92\xa6;\x06\xfeA\xb0{\xcf;\xadb\xcc\x14\x92p\x93\x19\x97.\x83\xbc\xb6\xeb.\xc6\xa0\xb0\x83\xaf]\xd3\xa8E\x0er\x959\xe6\xc6!\xe7\r/\xabP\xc2\xacV}\xaa\xef\x1f\xcc\x86\xcf\x82\xc4\xfc&\x13_\xfa\xc3{Rjt\x0c\xd1\xf3\n\xa5\xbdA\x0b\x08t\xf2\x05\xcb\xc8l\xddMD\xef\xb8\x99\xd1\xeeB\x0c\x15\xd5\x04\x90\t\xf1\x06b\xb6S\xb5\xcb\xb7P\xe7\x07\xb9\xc5\x91\xa9\xc0\xeb\r\tm\x0f(\xd9z0\x92\x99\xf4p=\x10dyy\x10\xa1\xf0,H\xccW\xa5\x8a\x10K6\x80\xef\x0e\xed\xd6\xa5\xab\xda\xa2\x9d\xa2\xcf\x10A\xcc\xd9\x0b{T\x83\xb2<\\d\x87D\xf4<\xa5\xf3\x1d\xea\x14\xf4D6\n\xdb\xaf4\xf7r\x8e\xd4x-\x9e]\x17\xc2\xea\x1d;6\x94\x19f\xf0\xb5(\x05\xde\xae\x18\xe8\xc8\xacy\xed\x9b 1\xbf\xc5q|\xb1\xff=\xc0I\xd1"7G}W\xe2\x99\xab\x839ke\xb1\x13\xdf\xb0\xe8t\x1ch\xca\xc0\xbd\x9d\xa8\xd8\xfd \xe2\xe1\xba\xd2\xda\xb9\xd1\x12T]\x85\xdc\xe2\x8e\xbc\x02\xee\xb3\xa0s\x11\r\xa9\xcf\t\x0e\xf2\xc7c\x93\xcc\xd7\xedV\xd9?\x18\xc8\x9f\x05\x89\xf9*\xc8\x974\x8e\xd3\xf0]\x93\x8a\x8e\xc7\x8b\xb3\xce\x82&q\xfa\xb0w\x91\xab<\xbb\x9e\xaez\xc7(\xd5\x99a\xdf\xc4\x11$\xda\xbaJ\x1e\xd3bO\x0b\xb1\x14\x8e\xae\x90\x1dz\x94\xf3\t=\x80P-\x12\xb7\xcd*\x97\xc8\xe8b\xaeN\xce\x1e\x971\xe5\xcdh_\xcf\x82\xc4\xfcv/\x84\xe3\x18\x01\xdf\xd5\xfb!w\\\xebm\'\x9e\xa3}\x98\xac\x95Z\xab/<\xb54\xeb\xbb\xcc\xadO\x9e\x93\xc1[}<\x0f\xc4\xbc\x9b\x8c4\xb2\xe6\xbd\xac\x0c\x99J;\xca\x89\xedyt\x13\x9d\xceE\x91I\xbdmiM%\xe0s&\x92\xe5T>z5\xfd$H\xcc\xedF\x83 \x97\xe6\x8dB\xee\\\x7fWk8\xc0\x14y\xe9\xf2\x9b\x1c\xaf\xd0\xdd\xc1;\x16\xc9\xf5\xda\xda\x9b\xad\xd4\x91\xd9|.`\xb8\x05\xc5\x1e\x05r\xc2T\xb8\x137\xc0\x12K\x9e\xa6c\xbd\xa1JCg\xe1\xe8(\xe7\x1e\x8f7\x16wQ\x06\x0f\x85\x92\xf0\xc1\x8b\x9bgAb\xfez5\xbdl\xcd\xfd\xe7)\xa9\x06n\xf7\x98)\x1e\xbc\xb0\xd8\xf5\xb2\xc1\x00\xb9\x90\xae8\xa6\x1f\xcds$\xcfa\xecv\\ 3d\xaf\x04D\xaf\x05\xb6r\\*U\x12\xf7F\xdc\x15XB\x06y@\x0f4{\xddQddy\xe9q\x8eK\x7f\xfb\xa0\xeb?\x0b\x12\xf3\xf5y\n\xb4\xa4s\xf8\x1eo\x121hk\xc9`k\x1c\x953\x86\x84-\xb4\xd9KNT\x0cF\xb62\x05\xba\xe1\x00\x8fT\xe6\\0\xba8\xc6\x0f]\x91f50\xf2#"\xa8\x08:4m\x8f[\xc44q\xb2~\x89+\x17\xd3\x9b.D\xael\xf9\xab\xb4\xfc\xe7\x86\xc4\xdc\xf2\x04\x84B\xd4\xd25\xdc\xed\xe2\x86\x92\xd0=\xbe\x92&\x10:\xcd\'\xfa\xb0Y\xab\x8e\x9f\x1d\xe8\xb83\xa5\x93z\xe2\xc5nV\xca\xd4n/\xeb\x04S\re\xae\x9df\xa9+\xcf\xf5\xcaXk\x83\x80\xb4b}\xda\xbb\x17\x9e4\x01}i.\x8f\xf6\xccBg\xc6\xfeW 1\xbf=V\xf5\x81\xc4\xfc\xcbO\xab\xff\xf7@b\xfeD\xac\x88\x0f~\xe3\x03\x89\xf9@b\xfeo<\xa5\x1fH\xcc\x07\x12\xf3\x81\xc4| 1\x1fH\xcc\x07\x12\xf3\x81\xc4\xbc\xef:?\x90\x98\x0f$\xe6\x03\x89\xf9@b\xde\x19\xbe\xf2\x81\xc4| 1\x1fH\xcc\x07\x12\xf3\xb6\xeb\xfc@b>\x90\x98\x0f$\xe6\x03\x89yg\xf8\xca\x07\x12\xf3\x81\xc4| 1\x1fH\xcc\xdb\xae\xf3\x03\x89\xf9@b>\x90\x98\x0f$\xe6\x9d\xe1+\x1fH\xcc\x07\x12\xf3\x81\xc4| 1o\xbb\xce\x0f$\xe6\x03\x89\xf9@b>\x90\x98w\x86\xaf| 1\x1fH\xcc\x07\x12\xf3\x81\xc4\xbc\xed:?\x90\x98\x0f$\xe6\x03\x89\xf9@b\xde\x19\xbe\xf2\xdf\x0c\x89\x99\xff\xc1\x15\xfe)\x0ea\xf7\x8fKzcH\xcc\x7f@\xd8\xf7@b\xfe\x03\xc2\xbe\x07\x12\xf3\x1f\x10\xf6=\x90\x98\x07\x85\xfd;\xc8\x91\xef\x81\xc4\xfc\x07,\xf6=\x90\x98\xffD\xf0\xf8@b^\x02\x89\x81\xffg\xcf\xff\x18\x12C@\x0cL\xf0\x04\xcd\x89"\xccQ\x02\x8c\x90"&\xde\xa6\x17\xdc\x06_\xb0\x14\x06C\x14\rq0FP\x18\xc1\xa1\x88@`0\xc3\xc1\xf4m\x1a=\x0e\xfd6\xbb\xec\xd7\xfc\x03\x1a\xc3(\x94\xe4!\x81\x13h\x8e\xc3\x05\x9e#\x10\x12\xa7\x10\x14\x85\x05\x0c\xc7\x11\nfI|y=N\x14\x10\xfe6\xe1\x1eE1\x1aeY\x9c\xe6 \x1c\xe1\xb8WBb\xfe6\xcb\xe2\xc7\xcf\x060\x10\x7f\xa1 \x02\xc6`\x9c\xc6\xfe\x10\x12\xf3\xf6\x84\x9d\x9f@b\x04\x86D`\x88\x13IV\xc4`\x84\xc61\xec6(\x9e\xc7I\x92\x17P^\xa4\x97/\n\x02\xb4\xec?\x04\xf1<\x86\xb1\xcb\x1b3(r\x1bj\xc1\x8b\x04\xca\xdc&\xba| 1/\x84\xc4\xdc6\x85!0\x94\xbb\xb9#!,g\x0b!n\xe3\xd1\x90\xc5\x0fyqY0\xce\x8b"\xc7A\x8b\xf9\xb1\xe5(\x08\x82\xc8,\xafA\xc3\xf8\x8d%\xc2r\xdc\x8f\x0f$\xe61H\xcc\xd7`\x93?\x86\xc4\xbc\xbd\x9f\x7f+$\x06\x83\x7f=\xc2\x9c\']\xe52Y\xc9\xe1\xa0\xa3\xa6r\xde\x9a"\xb7=\x0c\xc5\xe5\xbc\xc5\xc0\x16\x80\x0e\x1a\xb4\xbb\x9au\r\xadOz\x02\x1e\xa0yf\x1bx%\x1c\xd1&\x11N\xb5\xd91\xc9\x05>\xed\xe6v\xd0\xda\xd8M\x8e\xdb\xa8\x15\xb6\xf0\x83\x83\x86\x9f\x06\x89Y\xb2\x02\xb2$m\x02\x82\xef\xe6pn\xc0@\rI\xf0\x12\r+\x07\x9f\xe5\xf2\x14e~\x9d\x94\x0eZ7x$\x81\xfb<\xddW\xf2y="|\x97\xa13a\xc1\xf8)\xdbd\xcc\xc5p&\xa9\x9b\x90\x84\xdf\xc3\x90u:\x15\xdc\x98\xfbG;>HTN=\xc8\xc2y\x1a$\x86\xf8\x0bM\x93\x08u7zh\xad\xd9\x8d&v\xfa\xe8\x9c\x96N\xb6\x99\xb7=E\xaes\xdbN\xddY*\x12\xee\xb4\xa7edO\xccS\x00\x05\x13XlAsL\x0e\xbc\t\x94\xb2hc+\x9a\xaa\xa3ycP*U\xe4\xbe\x83] Y\xc6\xe9\xf0\xf4\xe0\xd0\xfd\xa7Ab\x16\x8d\x14\n->G\xdf\xc9L\xe8\xad\x19\x1d\x0b\\Dp\xd2\x0e\xb1\x1c"\xe6c\xe5\x99\r\\Fz\xad\xa4\xca\xb9rK\xc1\x90\xdd+\x9d\x8b\xa2\x8a\xcf\x90\xec\xd6&v\xcc\n\x1d\xab\x81\xf5 t\xe7\x9dp\x81Nb\t\t\xc5\xc5\xf3-\n=J\xef5t\xffi\x90\x98\xc5!\xf0\x1bN\x0e\xa3\xef&og\xad\x85 \x025W[\xac\x17\xe2\x94:\x12\xfbj\xc5\x95@\xa2\x84\x12{>\x91\x9a\xbe%\x0e\x9b\xc8\x87\xb4\xab\x19\xe6\xc5\xf2\x02\x8a\xe0\x08aP\xc0}\xdf\xc7((\xf8\xe15j\xf5\x04\x8d\xa7\xba\xda\x1b\x9d\xb2\x83\xa9_\xa1v^\r\x89Yd\xa2K\x1e\xc7)\xe8nD\x9d\x86\n#\xe9{\x91x\xe6\xa1\xf6\x82`\xc4\xc1-"\xd7\xc1;\x8buw\x1af\xb0\x9d92)\x07\xa3Z\x88\x00\xcc\x94w\x97u\x07\t\xb2\xbf)y\xecR\x9e.=\xa7\n\x1a\nT32^\xca\xeb\x8a\xdc\x8d\xdb\x0b\xf3\xa0\xdf?\x0b\x12C\xfc\x85\xc4q\x02^\xd2\xd1\x9d\xcc\x95\xa3C\xa0\x966\xd4.\xe8A\xf1\xbaD\x13\xed\xc0\xb9\xbd)*\xebB0/\xa0\xd5d\xdb,\xdbkW\x10\xc5c\x821\xfd\xed\xe1<\xab\x04+Hv\xbfk\xf7Qr8\xefm\x05\x82\x02]\xe7Zx\xa2\xa1\xd4\x9d~5\x7f\xf7\xd5\x90\x98\xc5\x9a4\x86@\xe8\x9d\xe3\xd3\xa3\xb8\x9f\x12\x12\xb8d\xfb\xe9\xd8d\x89\x19c{\xd6\xb3\xf2R\x90\xf7\x17\x87\x10f\x88\xb4\x98\x1e\xd1:\xdft5I\xedW\xee\xb6\x10c\x9d\xdf\xe1\xf0$\xf6\xd0\xae\xbc\xecCy\xd6\xc7]\xb0\x81\x12\x19?\xb8f\xa3?\x18\xc3\x9f\x86\x88YDb4B\x110}g\xcbr\x93\xb6\xf1\xa9\xf0\xfcB\xb4\xb5\xeeR\xd1\xae\xdb\\N\xd1\xba\xb7\xf7g\xed\xb8vQ\xa1\xbe\x1eE\x89\x02&\xa5\xf1\xd0\xd0^Ixo\xa4\xb7\xdf\xbf\nt\xd3\xa8\x93x\x15\xaf& \x03",\xdc^\xbc\x12\xbc\x1c\xb9\x87\x81?\xcfB\xc4\xdcd.y\x00\xc5\xc8\xbb\x81\xaa,\r\x9f\xab\xb3\xc3\x14\xe1\xfa\x00\x8f\xaeu\x9e\x0eq2)\x07\x1d\xf0\x8a\x0eF\x83jw\x90\xc5\x8e\x15+r\xae\xa2\xb2XYzXkezJd\xc7g`\xd5\x16\x86\xb47\x9b\xc1\x9b\xbd\xcaK\xf7\x1b\xebZ\xc0\xfa\x83S\x80\x9f\x86\x88\xb9y&L\x93K\xc5rwh\x91@\xa1w+\xa6\xdc\xebQ\xb2)%j&;\x99e\xf8\x02\xc8V\xc6\xc0\x8c0_\xe1\x10\x0f]\x88!\xd8\x8d\x00s\xaej\xd6L\xdd\x08Qi3!PWt\xdcyE\xd3\x83y\x14\xb6\xb2\xbc\xd9\xefOA\xa4a\xbfJ\xca\x7frD\xcc\x92\xf3q\x92\xa2\t\xfa~\xa0*aDGkCWR\x13dQ\xbcQ\xcc\n\xbfv@\x04\xf9M\xc8\xaf\xaf\xc1\xb0.t%s\x0f\xbbI\x1ar\x7f\xbf\xea\x13;\xd3Q?\x8f\x8aS\'\\7\x94\xcc\xd0\x18\xc3\xa4\x0eW\xc7T\x08\x08\xae\xcfW\x04\xfd]\x88\x98\xdf\xdd]\xfc\xbdLj\x14G_\xdc\xe2\x12\xaeC\xaer\xd4Er+\x17P:\xcb\xda\xb8c\x8c*\x83U\xa9\xcc+\xe5\xb4\x8d\xabN\x90\xb2\xfd\xe6\xe0\x9fv\x93\x9bdg\xa2\x1bgf\x15\x99Ls\x9av\x85\xe5\\Hq5\xef\x8f\x13\xb8\x7fpd\xf4\xd3\x101_\xb5\x07N\x13\xf8\xfd@U\x08\xeb|\x0bMJP\x1e\xc9\x83\xb6t\x19&\xb2]%\xa9}*\x93\xde\xd5a\xf1Z!\xf5N)\x88N\xbfx\x06\xa1\x82\xb9d\x07Pw\x91Z\x16\x15\x83\x96\x0e\xfc-E[\xa2!\xb1J\x12h(g\xadB\xbb{\xafI\xa3OC\xc4,>\x81-G\x05\xa6\xb0\xbb\x96\x068\xe6\xd7\xc8\x9d\x91\x84\xe0\xcf\xc8\xa4u\xf6\xc1\x00\xc2\x8b\x84S{e\x05\xa8\xdc\xa9\xe5\xda:\x12\x13\xb5\x05\xb1\xd0\xc3\r\x0b\x95w\xeeF\x06\xf9i\x9a\x08+\xa5T>\xd9\xaf\x90\x95\xe9\xaea1\xcc:+\x86\x86\xb5\xf7`i\xf34D\xcc-\xe7\x13(\xbcD\xdc;\x99\xb3T\x0e\xa7\xa8\xa3\xc7\xc3\x9c:\x1d \x12\xd4\x91\x13\xcf\x95dHa\xab\x19\xea\x001k\xd3\xed\xb9>\n\xa7\xcaF\x0f\x8a\\{\x85\x11\xb9l\xe6*=\xd5 \xb16jd\xc6\xb8\xe0\xa45"Sb\x83\x8e\xc4\xc7\x07}\xe2i\x88\x98%O,i\x1fEq\xf2\xce\xf5\xb7\xa2g\xce\xd8\x11+\x1c\xf7\x90\xa0\xb2\x1e]v0w\x81[\xde\xf1{[b\xeb\x95tr/\xf5\xa6U\xe80\xcb\xb7]\x12\x04\x87\xf3yL\xce\x06\xbf9P,\xde\xa4\x1b\xc1k)\x01+s\x10La%\xb8\xd2\x9e\xf4\xe0\x90\xe1\xa7!b\xbe\xb2>\xbc\xf4o\xd4]\xdb!\xf9\x92\x7f6Q\xf0|\xcd\xb4I\xa5\x18\x07\x10S\xb5+\xdc\x06\xcaV\xe5E \xce,I\xaaHg\xb5\xcay\xe4\xf2.\xa3\x9ah\xcc\x89x\x9c\x96hP\n\xac\x16nz\x0f\x9b\x96\xca\x00\xef\x82pJ+TK\x90\xf7J\x87OC\xc4\xdc\xba\x1a|9)\xf8=\xd9\xb7\xdb\x04G\xaa\xcd\x85fC\x96~\x9c_I\x0c\x91\x9b1`\xa5\xddR\xeen\xdc\x8b\x19\x84Af.\xfe\xa1\xe0\xcc\xa6\\o\xd3i\x151\x16\xc9\x0b\xe7H\x17\xcf\x194\xdbJ\\\x91\xb6)v\xa8\x06%\x97\xec\xe8\x97\x1b\xe6\xb1\x12\xf1i\x88\x98\xc5\'\x08l\xe9\x83\x96\xdc\xf2{\x990\x8d\xfbW8.3z\xed_\xc3\xeb9\x0c\xb4\xda\x9f\x84\xc0\xf1x\xe0|6\xafp\x83\x1e\xdb\xb8\x831\x1d>\xfa\xda\x91m\xd6\xcez\xa5\xe26\x91b6\xba]\r\x97\x15!l`\xbd\x11V\xe1\x81\xed\xa2\xf0`s\xfe\x83\xc4\xb4\xa7!bn\xc5\r\x86\xe1\x14y\x0f\xf7-\xf5\x19\x8c\xf3\xfdvfv+;\x95\xf4d#Y\\Y\x19\x13qd\x8e\x9bf\xa9X\xc8\xae\xa0\xb8\xe9\x98S\xa7\x93h\x8ce)\x87\xca\xe6:&\xec\xb9Tq\x99C\xf3C\xe1I\x15\x18\x9e\x14A\xcc\x8d\xbd\x02\x95\xd9\x83\x11\xeei\x8c\x98\xe5\xd0\xc2\x10B\x93?A\x0b\x1c.\xec\xb8\x85\xdc\x84\x80\xceY\xa7\x9f\xac\xbd\xe99\xa6\xe2XN\xcc\xce\xd7\x1e\xeac\xa6\xd4\xa91m\xc2C\x17\xe6\xe7Q\xdfj[\x00\x06\x06H@\xd42\x06\x8f\xf0z\xaf\xb4\xcc\xfe\x04\x94\xe3v\x02\xe5\x18\x05\xfd\x83\xf8\xab|\xf5\'g\xc4\xdc\xfa\x89\xdbG\x10\x10|\x97\'\x0c\xa5[\x03\x8a\x12\x9fb\x87\x84\xf8\x04\x15\xae\xf1\xc1\x1f\x15v\xbf\x87w\xeb\xe1\xec\xee=\xf0\xca\x81\x8dKJp\xa1\xae\xe8\x84j\xb7x\x8f"\x85\xc1\x1f\xf8+\x05D\xd7h\x83\xdbg\xdbO7\x00\xd9\x1ey\xc4\xabF\xf8Al\xd2\xd3\x181\x7f\x8dp\xf0O\xeek\x0f8q\xb6\xbd]\x1c\xa54\xe18\x95\x89^b\xa6W\x96\x9a\x9b\xbc\x92B\x80\x9c\xc8,K\x9a\xba]G\x0eQg\x1e\x8cV\x94\xb5\x9e\xd83\x93\xd4\xf9\x1e\xe8a[Z\x953\xb2\xd3kSKw\xa7\nq\xdc\xab\x91,\x8d\x98\x94\xacC\x07=a\xc9|Uu\x0f,\t\xcb\x91\xc8\x16\xea\xd7\xfb\x8b\xd9\xf4\xd8!\xdf\xe2\xa6v\xe9\xd9c\xd2\x182\';\x86y\xb2$D?\xcf\xe1\xf6\xa2\x88\x9a\xa7\xf2\xde\xb8\x84\x11mD\x1e\x0c\xe4Oc\xc4\xdc\x029\x8d \xe8\x92\x15~/\xb3\x06\xf58\xdeZX\xee\xb5\x06\xd8e[R\xd2\x19\x03\xda\nM}l\x10\x1c\xd9w\xe4v\xa3l\x04\xc5OSU\x9f\xe3\x88\x0f\xe9:h\x0e\xa7\x88E\xc7u\xd4\x81\x1an\xc4\x9d7f\xb6Wm\xae\xa4?\xb1\xea\xd5x\xf0N\xf5i\x8c\x98\x9b5\xc9%dA\xf7\x87\x16\xdd\xef\xce\xdb\x93\x97j\xf3Y\xde\xed\x04\xca/\x98\xfd5H\xd6\xc1\x89\xa3\xd1\x8a\x06\xb9\x89PV8\x98\x1d\xcb\tt\xd0z\xdf\x87dkl\\\x14\x18\\P\x84\x07\x81\x9d\xe1\xd48]\xc3\xd8\xa4\xf3<^\xd2\xd5\x18\xcb\xefEi\x7f\x1a#\xe6\xcb\xf5\x97\xc6\x06\xb9O\x87\xf2Y\xa4\xae.\x87\xe5\xa0n\xeck\xa79\xd8|xZ\x8d\t,\x96\xa6`j\xea\xd9\xbf\x08\x92(\xdb\x88\x97\x0f\x16\x8d\xaf\x83\x01;$\x12\x1b\x95 \xe2"\x11\xb8\xa2\x00dN\n\xd1\xb36M\x82\xfb\x80\x12\xf2\xdb\xed\xa3\xbd\xfe\xb3\x181\xcba\xa1nW\x85\xf0}q\xe3\rt\xefR-\xaa\xee\x1d\xf2\x08\xf1\xa8\x1c\x07I\x95&y{\x9d\x1c\'G\xe9\n\xd4wGe\\\xe3+u\x8d\xe9Z\x13\xc2qD\xdaY\x9b\x1eG\x93`J\xaa?\x00WL\xd1\xb7C\xef\xf7\xed\x14z\t"3\x0f\xdeL?\x8d\x11\xf3uA\x85\xa0\x08y\xcf0\x0e\xa6\x9d&h\x03\xba\xc2\xaa\xcc\xf0\xb9\xb3\r\x85\xb3i\xc5\xcb\x0e\x1f"\xa9\n\xb8 \x85\'\xa7pP\xd0\xd4[=\xa5\r\xe8J\xe1\xa5+\x9a\'&"\xa2\xf5Z\xdckWi:\xfb\xe1\xe6\x98\x88:\x05\x83\xc1*\x9a\x1e\x8cpOc\xc4,\xd6$o\x11\xe3Ndx\xe4\x93\xc2\xd1.\xb3_\xaaG\xe9\xbc\xa6c\x01J\xfd\xbe`\xba\x921\xe3\xa8\x83\xd2f\xcd\x86\x9c\x05\xab@\x06\xf0N\xe2\x03\x84j\x1d\x95\xea\xdc\x97+\x7f\xb7S\xa3l\x84\xf3\x84j\xd8\\\xde:\tI\xd4\xbd\xeb\xbeW\x05\xf74B\xccrT\x96\\H\xd0\xe8O.4\xb2@\xd5\x83\xe0\xb41\xbcA9"\x8e\xb7\x8aS/X\x0b\x1c0\x17\xee\x15\xc9\xb6\x01x\xb4\xa7\x96\x8d\x9a\xb14\xd6\x05\x96\x01\x88\xa2Kn\xd1Vs\xc7\xcb;\xf4\x98\xae\x0c\x80\xba\x10\xe7\x08\xdf\x17W\xca\xca\xf9\xb2\xe7\xff5B\xcc\x97\xed?\x84\x98\x7f\xf9Q\xf5\xff\x1eB\xcc\x9f\x88\xbd\xf1\xc1\x99|\xb6\xf4\xfd\xb7\xf4C\x88\xf9\x10b>\x84\x98\x0f!\xe6C\x88\xf9\x10b>\x84\x987^\xe7\x87\x10\xf3!\xc4|\x081\x1fB\xcc;\x93W>\x84\x98\x0f!\xe6C\x88\xf9\x10b\xdev\x9d\x1fB\xcc\x87\x10\xf3!\xc4|\x081\xefL^\xf9\x10b>\x84\x98\x0f!\xe6C\x88y\xdbu~\x081\x1fB\xcc\x87\x10\xf3!\xc4\xbc3y\xe5C\x88\xf9\x10b>\x84\x98\x0f!\xe6m\xd7\xf9!\xc4|\x081\x1fB\xcc\x87\x10\xf3\xce\xe4\x95\x0f!\xe6C\x88\xf9\x10b>\x84\x98\xb7]\xe7\x87\x10\xf3!\xc4|\x081\x1fB\xcc;\x93W>\x84\x98\x0f!\xe6\x11\x16\xc2?Tj\xff\x8c\x850\xff\xceq\xde\x99\x10\xf3za\xdfD\x88y\xbd\xb0o"\xc4\xbc^\xd87\x11b\x1e\x12\xf6o\xf1F\xbe\x89\x10\xf3z\x8b}\x13!\xe6?\x10<>\x84\x98\x97\x10b\x90\xff\xd9\xf3?&\xc4`<\xcd\x898JP\x82\x88S\x0cCq\x14\x8d\x0b\x08F\x8a0\x04s(\x89\x93"A\xb2\x8c\x88\xdc\x06_\xb0\xa8\xc8\xf2\x14\xc1\x88\x04Op\xf8m\xa2!r\x9b\xcb\xfak\xf8\x01)\x10"\xb1\xfc\x9c\x00\x8b\xcb^\xb0\x18\xc3/oA\xdef\x90\x0b\x14\xbe\xfc%X\x01A`\x86E\x10B\x14E\x84F9BD\x04\x12\xc5E\x02# \x8a}%!\xe6os\xb4~\xfcd\x00\x03N\xfd\x05\x85\x10\x1c\xfb\xdb\xe4\x95_\x11b\xde\x1f\xaf\xf3\x13B\xccb\xbdee\x0c\xc2\x93\x02\xb9\xfc8Ds\x08\xc7\xf2\xb0(p\x08\x85a\x10w\x1b\xfc\x83\xdd\xa66\xd2$\xc91\x9cp\x9b\xc7\xc9 \x0c\x87\x11\x08\xc4\x0b\xd8m\x1c\xe7\x87\x10\xf3BB\x0cA\xf2\x04\nS(~{!\\\xc0\x10\x86GY\x9e!\x10\x14f0\x81\xb9\x8d\xeeg\x10\x82a\xd9\xe5\xb4\x91,\xc4\xc3\xcb\x06\xa3(\x85P\xcb\xdbQ\x8b[\xfd\xf8\x10b\x1e$\xc4\xdc\xb6\xec\xaf\x84\x18\xf8\xa7\x84\x98\xf7\xf7\xf3\xef%\xc4\x90\xbf\x1eJ-m\x0f\xf6l\xaf\xdb\xb5\x87\xf1\x97\x0c\xa05p\xc3\x8dc\x95\xae\x84\xab\x1fp\xc6\x88L>\x10 \xb5%4:\x83\xee4\xd7\xf6\xec\xb5\xac\x9e\x10\xbd\x13X\x99A\xa0\x94q\xf3\xfc\x02r9f#\xc0\xb1u\xe8\x9d|}pP\xd5\xb3\x081\xb7\xac\x00\x13\xb7!\x9c\xf7\x03x\xa9\x8d\x97m\x1dm\xb7Cf\xa2GRh\xe7T\x18!\xa2n[\x92\x0e\xb2\xbb\x80\xba!K\x82\x85K\xe8\x01\x80\xa6\xe4p0TW\xd5\\5\xa1\x12\xad\xd0b\x12\x94\x01>W5;\x13\xf9!8Zrv\xaa\xaf\xcd\x83\x13\x07\x9fE\x88Ydb8\x8a\xa2w\xc3\xb8\x98\xab\xb6\xde\x88\xfdP\x14~\x043J\xcc2\xfe\xe0\xc3\xc95G\r\xbe\xa9(\xf2\xca\xea\x97\x19h\xb6$pB-Zi\xf4v`#\x1eI4\xdbEUT\x16\xd6\xf8iOj)E\x18k\x88a\x10`\xdf\x04\xd1\x83\x04\x85g\x11bn\xa6\xa4`\x14"h\xf8N\xa6\xe2\xb2\xe7\xcd\xd8\x19\xdd\x053\x0f>\x08\x86|=\x9a|\xd3!\xec\xa67\x9d\xe3\x19\xcfTu\xc0\x88qPf\x80L\xc1Za\x12\x95\xa5\x9d\xbcB\xc6\xd1-\xa55C\xa7\x0ef6~\x0f\xcc\xcd^V\xe2\xcb9\xb3~ub\xff\xdc\x84\x98\xdb.\xd2\xf0-\xbc\xa3w\x0e\x01\xae\xd8m0l\x8e\x13\xack}\x8c\x0b\xe3\x01\x1e}[\xb3M\xb0i\xf7l\xc0v\x95\xe8\x0f\xb1\xaeY\xa5\xde\x0f\xea\tH\xb9y<\xcdY\xd0\x93\xddT\x8d()\xf6{.C\\i\xe9\x08\xd4\xb6FG\x86=\xa5\x0f\x02\x1a\x9eE\x88\xb9\xc9D0\x08^\xba\x81;\x99Y;&|\xc7\xf1F\xdbkr\x08\\\x925)\x14\xc1\\V\xd5\nQ\xf9*l\xe38t\t\xe4\x10\xa2\x92/@F\xd5\xa5\xd0\x1e?"\xda\xa9\xf7\xb1\x9a\xc7\xbb\xab\t\xb2\xb6\xcc\x81\x96\xbfa\x98\x96)9\xc9\xbc<8P\xf5Y\x84\x98E&\xb2\x9c\x0b\xf26\xd2\xefn\xd2\xa8\xd1\xc9\x82H\x12\r\xef\xe1\x83\x01\xadl\n-,\x99\xea\xcb\xbe.\xd2\xd3X\xab\xf3\xbaB\xb1\xd0\\\x12a`\x10\x04\xb1\x86Jw\xbcN\xacI\x83\xeb\xfdUN$\x07b\xe3\x03\x13\xa0\xe3\xea\xd4u\xed~7\xfb\xfe\x83\xe1\xedY\x84\x98[x\xbb\x1dY\x9a"\xefd\x06\xdaN\x01$\x87\xd5d\xbc\x9c[\xd8\xb1\xe2\xab#pNj\x03 *\xe2\x9cn\x9e\x8f\x1b\xd3\x9dHc>HU\x1d\xc0[\x10\xa0ch\x0bX\x8a\x94\xb8\xe08\xf9W\x18\x1eqf\xdd\\\xa7}o\xf6:\xean\xc4GGF?\x89\x11\xf3\xfb\x16\xe6\x1f\xa7*\xda}\xabo\xebux(\x0f \x95\x98[\x8a\xa9zN\x8fR\xc2K\r~i\xd5\xf4\xa0b\xce{\nU\xbb\xb6\xf0QL\x9b\xe4 \xb6\xcf\xca)`\xc0\xd3e\x92\x08\xd7\xbe\xb0\x90\xa6\x9c\xcc\x9a\xd2UTg\xf7\xde\xc3\xa8\xaf\xe70b~\x0bA\xe8\xad\x84\xba\x03E\xb0z>\xc9\xa0_9\x00u\x96S#\x027\xaa\xce\xb7F\xdc\xf2\xd5Ah\xa3\xd4c\x05\x94\x92u\x11\xce\x98Ff+$&,\x81$\x03dX\xe1\r\xb5V\x0f\x04;->K\t\x1b\xb9\nw\x1a\xb5\xc1]4|\xd07\x9f\xc5\x88\xf9\xb2&\x84\x10K\x81u77\xd6\xa1\x00\x0ct\x0e\xd5\xa6Td\xe3\xd27*\x93o\xcc\xab\xaa\x95\x96n\xf9\xf4x.]\xd8\xde\xcc\x13.\xfb[\xc5\xd2\xf5L,h*\x15\x0b\x07N\x07\x02\x90\xdd`\x0b\x9f\xab\xb6\xbe\x9e\x87\x98\x95\xbc\xf8\xb2\x94\r\xa5\xfa^D\xb3g1b\x96]$\x97]\xc4i\xfc\xde\xf5\x8f\x94\xefO+\xdf\xd8.\xd6\xe0\xf5\xc2&\r\xe7\xca\xaeN\xb9\xb6\x0e\x86\x18\xce\x96\xa6\x86\x90v+/\xd8\xad\x0e\x05{\x95\xd6\xfe\xda\xc6N(\x11\x0b\xfb\t\xd4\x96\x03\x9b#44\x9a\xea@\xa4\xc0l7\xf5\xca\xa1\x9c\xcd\x83>\xf1,F\xcc\xed\xb0\xa0\xd0R\xa6\xe2\xe4\x9dO\x88\x83mE\xa4\x7f\xe2\xdc\x0bkm\x03\x9f\x17\n\xfd\xaa\r\x1c\x1d\x0f\xa7\xa6U\x81\xea"Z\xa5\x1d!N\xd1\xf5Nw\xa1\x81\xcb\x91\xe2l\x91\xebW\xe1R\xb1\xa3\n\xb2n\x13\xe4\x8c\xcc\xb1i\xb0)\x02\x1e\xd4<8>\x18\xe1\x9e\xc5\x88\xf9*\xc7i\x94^\xce\xc5\x1d(\xa2J\xa8\xc3\xd8\xb9\xae\xccG\xadK\xe9\x0c-\xba%X\x0cT\t\x1c\x1b">\xadd\xaa\x9e\x93z\x97\x8dQ]\xd7\xc4j\x04\xcb\x84\x87.vM\xec\xfb\x13H\xd4x`\x00\xad\x8d\xcfk7\xa8\x99&4\xfb"\x19\xba_Q\xb0\xfe\xdc\x8c\x98[:\xa4\xc8\xa5NZ\x8cr\x97\xf5\x0b0l\xb2DlsU\xa4\xcc\x14a!\n\xd7\xf6d+\xcc\x97fG\x8c\xf5\x8a\xc0}Z\x85\xe05x\xc23a\x7f\xe1\xd9\xeb\xa1AwR\xa5\x9f\xa1q\xb7\x9a\xa0y\xf2\x82f\x8bY\xe9\x89;\xaf\x98+\xee&\x9bob\xc4\xdcd\xde.I \x92\xbec\x89Y\xec\x06!\xbc\xc9\xe7g\x03JBY\x1c\xe4\x93\xbd&\xb8,<\xf5T\xd7\xbb\xfe\x9c\x0eUU/\xbd\xa7X\x01\x1a}&w\x12\xd2\x9d\x89\xd9-\xb8m\xd6\xee\x97\x03v\x81E@\x1br\xd3\x1b\xb3\x845P\x90\x00\xc8\x07\x91i\xcfb\xc4|\x95\xaa4D.\x05\xfe\xfd\xfc]P\xb3BD\xc6u\x83>\x9c\xa8\x803{\xf6\x10\xfb\x80\xd5\x9cq\xa9\xf0\xac\xae\xdcz\xc0\xdc\xa7m1@\xaa\xbf\xe3\x02\x9c\x98\xe0\\\xc9\xac\xcd\x0e\xcb\xcc\x16\xdb\xf7\xd2l\x1cJs]\x02e./\xb5\x9c\x13\x9f\xb7\x0fZ\xf3Y\x8c\x98/\xd7Gq\x8a\xa6\xeeKU\xbd,\xc7\r[\xa0y\xbf\x8fg\xf0@6\x0688sY\xaac~T\xd36\x9cJ\xcc\x15MK6S2\x1d\x8f\xa4U\x9b\xd8\xeeD\x90\xa5df6C\xc0}@Z\xe7\xa3A;g\x1b\xb8\xa6\xa5*\xa6\x023\xbeW:|\x16#\xe6\xb6\x8b\xe4\xed\xf6\x8e&\xeef\xee\xd3\x07X\xd0\x9b\xf3\x05B\x1b\x1f\x9b\xbd\xcc\x91L\xe9\x10cr\xe8\x83\xc7\xede\x97_\r\x99i$\x86Y\x1a\xf0\xc8\xe5\x15\xf0\x10\xc3\xde\xb6m\x82\x9d\xeb\xe69\xa0I\xf8\xe5\xa2\xb1\x01w%\x10\x0eG\xe6\r\n\xcc\x19\xf7\xe00\xfag1b~\xab\x9d(\x02\xa6\xee\xc9ptR\xa4$\x86M\xb3`\xdb\x944F\x07\xd3\x0e2\x1c\x87\xf8\x00\t\x8b\xc3\xe4{\x13G\xb8\xa1\xd6\xb7\xfd\xb8\xf6*\x9f.\xbd\xcdR=\x14c\x1eo\x8b\xfa\xd4\xc5\x15\x8e\xe4\x08Fk%\x9b-\xed\xac\xd0)\x14\xa9=\xd8\xd7<\x8b\x11\xf3U\tcK\xf3\x0e\xe3w\xd6D\xe1Y\x83\x06\x88\xa1\xae\xfaF\xf6\xad\xb5\x06\x1eW\xeb\x92\x8fk%c\x9c\xb8\x02GQP\xf2\x0e\xda\xb8s\x90^7z\xdc\x81np:t>o\xe7\xbe\x00\xaf\x0e\xa28e\x14~\x80\xd7\xa9\xe7i\x96S\x0eU\'?\x18\xe1\x9e\xc5\x88\xf9\x8ap\xf8\xd2\x8c\xc3\xe8]\xbe\xfa\xff\xd8\xbb\xb3%G\x91lQ\xd8\xef\xd2\xb7\x985\xf3\xf4\xdf1\x0b!\xc4\x8c@wbF\x8c\x12\x83\x80\xa7?(\xaa\xfa\xef\xd3\xa5\xca\xee\xd66\xe5\xce\xa8cXY\xd5Ee\xa4\x82\xe5\xee\xcb}9\x11\xb6\xbePr\x8fh\x7f\x955\x96\x12a]\x15\xe0G\xdc"\x88e\x93\xbd\xce\xdek\xf4\x1ck\xe3\x914\xdc\xe8\x0e\xf0U\x9bx\x03\x91^H\xe2zk+\x1eu\x81}O\xb4\x0b\xac=\xb8S\x82\xe5\xf1x\x8c\xc0\x91\x82\xf9\xe0G\x05\xff_\xdb\x88\xf9\xba\xeb\x13\xcf:\xeb\xd5\x9cL\x1a\x81\xbc\xdfK%\xdc\xed[\x99\x0e\xd6\xfb\xa7*/\xb0\xaec3\xcb\xe1u#.\xc6T\xb7l"J\x180\xc2\xe5\xb9\xb1\x18co\xedm\xac\xe3\xdb;\xaf[\x13\xd7JvX\xeeR\xca3\xd1\xfba\x1a\xa2^\x9c\xb8\xf7r\xe2SF\xcc3\xccu{\xc3h\xfc\xf5v(L\xf92\x9fqB\x91\x1f\xb9\xce6\x16&\xc0\x13\x19\x06\t]\xf5\xf1\xc3\x00\xabJc\xa5B0\x14}\xbd$\xad\x85\xee\xbdm\x16\xfc@c\xa3E\xc8:\xad\x18\'\xd6\xcc\xca9\xaf\xdd\xfc\xc4\x18\xd6\xac\xec/\x199\xbcI\xa6}\xca\x88\xf9\x9aM\x18E\xd6"\xe1%\xf5C\x86\xb8\xa5A\xc4\x9a\x86\xd3\xc2J\x83\xd6\xfe2K\xb3R\x88W\xf2\x94\xd7\xd2Ugj\x92\x19\xcc\xc7\xd9\xe8\x8ffmbQ\xc6u\xa9\x10\'\xfb\xc6-#\xca_$\x81\x08\xc7\x14\xbf\x9e#\xcf\x12\xa7`\x86\x85\xe8]U\xe4SF\xcc\xd7l\xc2\x04B \xaf\xecm\xaf\xa2\x8eN\xb2\xea\x03`\x92\x92\xbd"\xd3c\x08 \x16\x9c\xf0\\?6\x16-Uu\xae\x86v\xda\xe4\x8a\xc6\xf3e\xdd\xad\xf7\xe2\x1a&\x8a\xa5sQ\xf1\xa1\xee\xcfB\xa4\xe1\xb5\x02\x00\x0fE\xd8\xab\x18\xc5]\xf8\xbe\xfdEF\xcco\x1b9E\xe30\xfaR\xdcL\'\xed\x91rYt\xd4 \x02\xcb\x01\xbf7\x0f\x88\xe3]\xc2V1R[\xc1Ei\xdfO\xa8\xc0N6VMXY=\xe4\x00\x0e\r$:\xc8S\xaf\x8b\xbb5\x89\xed\x91\nS\x93m\xc6\xa3\xa3F\xb00\x1c\xf8\xf2M\xc9\xf4SF\xcc\xd7-\x95\xa6\x08t\xbd\xca\xbf\xccf\x9a\xab\x81M\xdb\x81\x8c\\\xc3\xcb\xb5\x8e\xd7\xba\xed\x01\x9e\xc2\xfa\xe0\x95\xc0n\xc2\x92r\xc9\xf3\xca\x04)\xf7\x14\x01 z=\xa15Qx\xbdT\xfa\x8fp\xca\xbf\xb6\x12\xf3\xf5\x83\x0cl\xbd4\xe0/\xa7atLq\x1b\'\xf1#\x0b\x8b*>\xde#l\x1c\x81ES\x81t\x01\xd5\xa8\x89\xf2;W\x9b`+\x84\xa6\xc6\x8f\x94\xd4\xe3\x98sK&*\xafiZ\xca\xd5\xbdfX\xbc\xb5\x1c\n\xd7nT\x0f@\x05;$\x9a\x8eg\xfe+$\xe6\xab\xf4\xdc\x90\x98\xff\xfa\xb7\xd57$\xe6\x97\xb4\x93\xdbD\x93mH\xbf\xfb\x90nH\xcc\x86\xc4lH\xcc\x86\xc4lH\xcc\x86\xc4lH\xcc7~\xce\r\x89\xd9\x90\x98\r\x89\xd9\x90\x98\xef\x8c\xaflH\xcc\x86\xc4lH\xcc\x86\xc4|\xdb\xe7\xdc\x90\x98\r\x89\xd9\x90\x98\r\x89\xf9\xce\xf8\xca\x86\xc4lH\xcc\x86\xc4lH\xcc\xb7}\xce\r\x89\xd9\x90\x98\r\x89\xd9\x90\x98\xef\x8c\xaflH\xcc\x86\xc4lH\xcc\x86\xc4|\xdb\xe7\xdc\x90\x98\r\x89\xd9\x90\x98\r\x89\xf9\xce\xf8\xca\x86\xc4lH\xcc\x86\xc4lH\xcc\xb7}\xce\r\x89\xd9\x90\x98\r\x89\xd9\x90\x98\xef\x8c\xaflH\xcc\x86\xc4\xbc\xc3!\xfc\xcb\x19\xf8\x1f9\x84?\xd4u\xdf\x19\x89\xf9\xf9\x81\xfd"$\xe6\xe7\x07\xf6\x8b\x90\x98\x9f\x1f\xd8/Bb\xde\x0b\xec\x7fB\x8e\xfc"$\xe6\xe7\xcf\xd8/Bb\xfe\x176\x8f\r\x89\xf9)H\x0c\xfa\xcf1\xff\xf7H\x0cC\x11\xa2\x80\xe2\x14Lc\x0cI0\x0c\x8d\xc2(DQ\x02\xca\xa3\x0cL\xb1O\xd4\x02\x85y\x14cD\x9e\xa2H\x91cE\x92c\x18\x9c\xa5\x08\x9e\x12x\xe1\xf9}~\xec\x1f\xb0\x0c\xc1\xa2<\'\xa0\xe8\xb3U.\x8bB\xa8\xc0\x8b$\xc6@$)\x08\x04\x87`"\t\x91,-2\x88H\xd3\x04\xc7\xb0\x1c\t!\xcfa\x171\x01\xa3q\xf2g"1\xcf\'\xffs$\x06\xfe\xff \xec\xef\x14\x8c\x914\x85#\xc8\xbfCb\xbe\xbf\xb0\xf3\'H\x0cL\xc1"\x8b\x08\x14B \x14\xc4\xb2\xbc\x88s0*\xac\x93\xb3\xce\x16\x8f\xd3\x0c\x8e\x880\x8a>\x01\x15\x92\xe4y\x96&Q\x91\x11\x18z\xfdR\x16\x17\x05\xf6\xd9\xcbiCb~"\x12\x83\x8b$Lp\x1c\xc1`\xdc\xfa\x0f,0\xa4\x00\xb1\xb0\x00\x91\x1c\xcb\xaf\x7f\x83XW\x1c#\xf08\xc7R0\x85P"\xcfQ,\xc2p\xeb\xf7\x10D\x92\xa5 \xfco\x7fq$\xe6\xbf&\x0c>\x86\xc4<\x1b\x9b\xfc\x8e\xc4@\x7f\x8a\xc4|\xff<\xff\xa5H\x0cN\xff\xb8!\xbdp\xbf\xf0\xed\x83\xa1\xfd\xda\xac\xd2\xa90\xcf\x93\x01C\xd4yO\xed\xb1\x83$\x05\xec\x98J\xeb\xde\xbf\xd7\xa2\t]\xb3\xc7\xa4\x81\x8bIN\xa9P\xcaI\x9a=N>\\\xf5\x06\xafB\xed<\xe8\xb9\xb7$z|\xf1\xd5\xfa\xcd\x96\x83\x9fAb~;\x15P\xfa\x89\xc4\xbc6\xa9\xa3)\xbd\xb4\xf5\xe2\xe6\xa3\x9ak\x81\xd7\x9b|\x0f\x8c\xa3\xf5\xa0\xcf\xad\xb7\xa7C\xf2\xe2Y\xb4\x19\x11\xce\xd9>\x15\x13Si|\xe5\x02 \x19\x8e\xd2\x03\x1dB\x7f\xbe!uc\xc7\xa9er\xd5\xfdj\xbb\x0e\x1d\xefb\xa0{\xb3\x17\xdfg\x90\x98\xaf0i\x98\xc2\xe9\x976U\xfc\xd0\x94\xb2h!\x12\xe6?\x0ee\xd4\x1c\xea8\xbc\x9d\xe5X\xbf\xf8&\xacs\xc3\x051m$\x8a\xe6\x86\x87\xf9\xc7\x10\x81~\x9c\xea%q\xca\x06\x11\x08\xa2\xe96\x9e\xa7\x1d{\x18<\x85\xeek\xaaOs\n\x0f\x8f\x877\xdd\x8f\xcf 1\xbf\xc5Hc\x14\x84A\xc4K\xcf\xe8C\xaf\xef4Q8\x1fl\xa6\x17\x8b\xdeH\x0e\xa5\x06\x9c\x0eqr\xc8\x9bk@r%9\xaa\x05\x1f&x\x04\x9f\x9d\x1a\xd7\x18\x07;\xd0]4\x8a\xa5=?\x16\t\x08\xcdN\xa5\xfdK\xbf0\xc0(.F\x10\xaa\xc2\xf5G\xdd#\xff\xcaH\xcco\tAB\xeb(\xd2\xe8k\x97a\x8a|\xf8\x07x\xa7\xc3\xe0\xfd\xb8\x0e\x0e\xc8\xc3\x84W\xab\xeb-:\t(\xc3I\xc53re\x95\xfbC\x00\xd4\x84\x82B\x9a\x84\xcf\xa2\xee6z\x13\xa4\xf9\xc3T=tr\x8e2\xd3\xee\x06O\xc8s\x04\xcfNW\xfb\xfcf\x87\xba\xcf 1\xbf\x85I\x93\x18\x81\xe3\xe8\xcbbA{\x18\xcd\xa7\x8a/XH\xf7\x8f\xc6\x95\x0c\x85\xd9\xbc\xdf\xe3PV\x19\x8f\x9f;hp\x8a\xabZ%\x94Fg\x10\xe1w\xd7F\xc5\x8c\xb4>\x1d[$\x1d\xb2\xc7\x05LM\x03\xd8\xfb\x1d;7\xd3\x1c\xf9x\xca\xca\xc8\xbby\xff\x11$\xe6+\xccu\xea!\x18ym:\x86M\xf7\x04JB\xd0P\xb9\x80\xb1\xe5|y,y\x05.\x08k\xf6-\xc9\x1a\x11gu\x87\xb2\xf7\x86A\xbaI\x0f\xe6l\x9c\xee\x07\xc2\xa9\xb0\x05\x19\xf8\xe6\x0cAVgA\x17\xd7\x19\x0f\x8a?\x13\xfd\\\xaf\xd5\x01\x7f\x7f\xb3\x05\xe7g\x8c\x98\x7f\xecn\x04\x84\xe3/M\x15\xaf\xec5\x80\xe3\xc0p\xd9i\xa9\x02S\r\x1d;\xbb\xf5\xf5p2\xc1\xa5\x9f\x9d\x1d\xb4\xeew\xfe`-\xf1\xe0d\x03B\xb0\x86.\xbb\xe6\xa4\xe1\xfe\xdc\xb1\x19\xd5\xaaX+\xcb\xf9\x01\xbc!9\xb6\x0cI\xbc\xe8\xbe\x9c\xbeyT}\x86\x88\xf9m\xc9\x12$\xb5\x96\xb9\xf4K\x98\x19z\xcc\xe4[\x0f\x8a&pA\xf6i\xde*7*\x1e\x15\x8c\x0bn#\xea9\xb8\xa0\xe6\xe0\x12\x8aj\x0f\x1b\xf9\xa5QS\xebA\x85\xd3\x059\x1e/G\\>\xcf\xc7\xc3\xde\x98OMIW\x1d\x81\xc6\x10\x92T23\xbfi\xa7|\x86\x88\xf9=Ll=\xd0\xffD\xa9\x92k8\x9d\xe1>\xb8\x02\x048\xcekA-\xee\x94I<\x10~d\xc3\x95\xc8\xcc\xe0\xc9\xa0\xe0\x1b\xd6N\xe3\xe5\x8c#(\x9e\xce\xa0\x90P\xb8\x8c$\x84}8\x0b\xfd\xac\xda\x96WP\x15\x06\xcc\xbb`G$\xa7J\xee\xf87\x1bc\x7f\x84\x88\xf9-3\t\x9cXG\n~I\xcd\x8e\xb9\xdc8\x1fS\xa4\xaa\xbbA\x9a\xcfiy\xa4VC\'1\xf7e|\x04\x9e\xe5<\x80G\xec\x1f\x88\xc8zD\x10\x7f\xb9\xa7\x17\x03\xabG]\xca\xc5\x9b}\x11]\xc0d\xe3\xd1\xbd\x91P4\x14\xd9\x99\x98m\x14p\x83\xefE\x9a}\x86\x88YG\x11\xff;NPO\x08\xf5\xb5\x03\'.\xb72\xcdCB\xec\xf3\x8f\xb4\xec#{\x1d\xc1"sRH\x1b2^6z\xf5\x08O\xa0\xa0\xc5D\xd1\x9b\xca\x88\x18\x1d\xa0\xcc\xf7\x87vCT\xa57\xdb\x85\xbd\xd9\xce\x9e\x9cDb\xe2\xae\x81|\xda\xdd\xd7:\xa1\x7f\xb3c\xf4g\x88\x98\xd7w\x17\xffw\x98\xa4\x94\xed\xfdX:\xe1\xf4\x14T\xac\xb9/\x80\x98A\xb2n^\x1e\xf4\xb3r\xd9_\xdc\xce\xb0(\xd5\xc3]\xd8T\xad\x043F\x9e\xd3\xb9>l\x9b\xcb\xed\xd6\x81\rq84\x1cC,\xfb\x91\xd44\xe7 \xc36_\xbe\x19\xe6g\x88\x98\xdf\xc2\xa4\xd6\xbf\x82\xd0\xc4\xcbl\x1e\xa5\xa8\x90\x93\x93M\x9d\x1eP\xd5\x1c\x03\xde\xc9\xd9S\xa6\xc8\xa8oqlC\xe9i4\xfawWbK]\xd6\xf4>c\xc5\xf1~\xb49\xb1n\x84S\xe8!9\x17=\x1d\x15\x84v\xa7\xe7\x91\xc9h\xa3e\xe6\x90\xe7\x87\x96\xcd`6\x1f(*\x9f\xf0\xebA\xb6\xe5\xab\xa2(|\x07\xb4.J\x85G\xdc\x85\x94]\x1e\xd3\xc1D\x9f|\xbc\xc8Kvhk\xf1]\t\xe7#D\xcco\xb3\xb9&\x0f\xb9\xd6\x9b/9\xe1\x9d&h\xb6v\xcc\xa1\xdf\x01\x12\xcfM\x10\x1f\xd0\x1a\x11\x8c\x0c\xbf\x07\x1f<)\x81\xde\x81m\xf7\xe8\x05\xf5\xca^\n\xd2GN\xed\xcf\xe3a\\\xf7\x9cf\xba\xe30\xa6\xe4\xccz\xc7"F\xfd4\xf8\xa6\xa2\x96cH\xa4\xbf\x84\x88\xf9\xfd\xd4\x87\x9f\x9b\x1c\xf4r^]\xb4<\r\xd8aV1I\x0fC\x05L\xf7}~\x91j\xfar\x13\xcf\x05\xb0\x0bGH\xb4K\x8d\xe7J\xfc\xce\x9d\xc1ch\xd4\x90U\x9a\x98\x1f\xe5w\xfe\xd8\xf0\xa1\xc7\x8dP<\xe6\xf9^l\xc2[[Ka\xb1\xc4\xdf\xeb\xbd\xcdg\x88\x98\xdfF\x11CH\xfa\xf9s\x8d?\x8e\xe2\xee\xee\xda\x8f,,\xee\xad\xa9\x16\x8e\xe5=n\xe5\x9dK\xafNe>\x88[s\xbe\x13\x97\xb35\x86ed\xd2;\x81\xe1\x92Pp\x91\x03+\xd5\xcedp\x92\x0f\xdb%8\xe1,OZ\x95\x95zqm\x89Q7\'\xf8\x9b\x94\xd8g\x88\x98\xff\xffv\xb8nr\xe4\xcb\xa9\xdf-k\x11\x1c]\x16\x05\xc5{{\xd0\xd7o\xd1E\xbc\xa1]\x94\xa0|T\x95a\x8d*\xcc\xd3F\x08\x1b}\xdd\xcc\\S\\\x90G\xaa\xc4f\x14\xd7\x06g\xd2\xe8\x03\xafZ\xf2FN\xb3|\xb5\x11\x87\xbd\nx&\xd8o\xeaP\x9f!b~\xbf\x1d"\xc8z\x1a@/\xb3\xe9T\xe8\x98s\x16\x07\xb7\'\xa3\x11\xa4\xbdw\xbf\x91\x86\x82\xdd\xd5\xa2\xea\xd9\x9d\xe1J\xa2\x0f\xbaC\x94\xaaQHWX_c\xc0\xb4W\x1eG\xf1&\xb7.D\x14h\x99\xf9\xcaRL\xa9e\x1a9P\xd97,\x99\xe87e\x81\xcf\x101_\xd7\xb7\xf5\x7fA\xeb\x15\x89|}eK\x1fpm\xe8:\xb3\xc6\xc5\x9bHV\x87\x98l$\x1d\'r\x92j\xa7]M\xb0E\xabt\x10bB\xd7\x07\xef\x01\xad\x82\x1e\xe2\xe5\xdaF\x95W\x01\x1a\xd2\x85}\x89\x8f^\x878\xfc\xf3\x9d\xc4\xa2\xf7\x89q\x9b\xc97\x8b\x9b\xcf\x101\xbfo\xe4\xeb\xeeOA\xd0\xcb\xa2="\x9a\xd0\xeb\xb6\x9b\x03"3"\x98\r\xf6\xdc\xdeP\x9a\xbbT\t\xfe\x90\xf3\xb7K\xb5\x0bN\xaa\x82\xc2\x8d\xad\xe0\r\xa8\xa2\xe2\xe8\xcd\xf5Z\xd6\xac\x07X\xcd\xee\xd1\xaa\xbe\xe6N\xd4L\xfc\t?>.\xe2\x95\xa1\xd1\xf9\x97\x101\xbf\xff,\x85 \xb1\xf5\x96\xfa2\x9b\xfc`\xdfu.`Z\x91*\xdb\x00,\xc2S\xdbU\xcc~\xd2\x9d\xa6\x00JRru\xb8?\xc6n\x9dBe\xab\xceH\xeb\xcf\x02s\xa2\xa0}\xf9\xc8\xe4\xdbR\x82V\xbb$\x83e*\xe2-B\x8f\xbb\x9913\xfd\xf2\xbd\x98\xbf\xcf\x101\xffH}\x1c%\xe0\x17:%\xe6:^\xc3[\xf3\x89\xfd\xc5~\x85\x9b\x1a\x15@\xd6\x00\xe1!\xc3\x92\xb2?\xb2\xa68F\xb7\x04\xb2\xa10k\x80\x9ePk\x98;\x00\xa9\xbc\x13@\xc2\x96\x12\x02\'\xc1d\xc1#\x84M=8\xc9g\x92Ro\xf8\x8f\xb0\x8d\x9f*\xc4\xfc\xfe\xca\x16\xa1\xb1\xa7z\xfb\xc70\tQ\\\xaf\x9a\xe2\xa2<\x9b\x11S\x05x\xc7\xae\x9a}\x95\x03\xb0\xa7\'\x83$@\x0ch\xd2\n\x1f\xd0=I\xc0H\xcbKk\xc5\xa7\x8b\x19\x92QhW\x9b\x8b\x00\x08\x97~\xbda1\xb8\x99\x08\xfd\xd16\xe8\x16\'\xde\xc5\x0c?#\xc4\xfc\xf3\x92\x8a\x12\xd0\x0b\x9d\x12e\xa5\t\x98\xdd\x18\xc4\x90i\xe7E}\x9ew\xb15u5\xef\xa4{CS\x13\xe2\xdc]\x03\xdf\xf0\xac\xe2\xe4TE\x9d\xdbt~\xbc\xda"\'\x95\xb4|\xd4\xeaz\x87*f\xa6\xe8y>\xef\x85\x1b\xc4Vb$\xee\xde|g\xfb\x19!\xe6\x1f/n`\x9cF__\xdc\x90w3\x14m\x1f\xecQ\xecJ\xe2\x9a0;\xe1}\xbf4\xccq*\xc8Rj\xb0\x90Z\xbab\xd4u#\x11 l\t\x1e=+NSJ\x1b\xa5\x97(\xdd\xe2r\x11\x19Z\xf1\x80\x0cW\xdd \x02\x85\xb6P)\xb2~\xf4\x06\xfe\xaf,\xc4\xfc\xb6X\xd6\xbc\'\xf1?!!m\x8c\xd1\xd5\x9a\x13\xf2\x03K\xf1\xc9\xc0gC\x92$\xa0\x1a\xa8g\xc4:\x04\xd3\xe0\xc3%g\xdc\x95\xec\xd0\xcd\x91\xd9L\x90M\xdcs\x14\x93J\x80N\x8c\xa8[\x90\xd2"\x04\xf0P\xd93\xeaR\x17\t\xc1\xf5\x87+\xff\x97D\xcc\xd7/UmD\xcc\x7f\xfd\xbb\xea\x1b\x11\xf3K\x9a\xc9m\x9e\xc96\xa4\xdf}H7"f#b6"f#b6"f#b6"\xe6\x1b?\xe7F\xc4lD\xccF\xc4lD\xccw\xa6W6"f#b6"f#b\xbe\xedsnD\xccF\xc4lD\xccF\xc4|gze#b6"f#b6"\xe6\xdb>\xe7F\xc4lD\xccF\xc4lD\xccw\xa6W6"f#b6"f#b\xbe\xedsnD\xccF\xc4lD\xccF\xc4|gze#b6"f#b6"\xe6\xdb>\xe7F\xc4lD\xccF\xc4lD\xccw\xa6W6"f#b\xde\xc1\x10\xfeew\xf9\x8f\x18\xc2\x1fN\xcc\xefL\xc4\xfc\xfc\xc0~\x11\x11\xf3\xf3\x03\xfbED\xcc\xcf\x0f\xec\x17\x111\xef\x05\xf6?\x01G~\x11\x11\xf3\xf3g\xec\x17\x111\xff\x0b\x9b\xc7F\xc4\xfc\x14"\x06\xfb\xe7\x98\xff\x07"\x86\xe39\x96#8\x12\xc2\x18\x11Ex\x9c\xe10\x02\xc18\x04\xa5\x11\x0e\xe6\x18\x92#)\x9aga\x88{:2\x18+\xb2\xe2\xfa\xb58\x87\xf2\x04\x89\xad\x8f\xf9\xb7\x7f\xa7\x1f`\x14\xcd!\x14\xf3\xfcR\x82@\x10\x1a\xa3\x04\x1e\xc1\x08\x1ac\xd7\x91aI\x0e\x110\x84\xa1\x19v\xfd\xba\xf5\x9b\xa3,\x8e\xe3<\xcf\xb3"\x82\xd3$\xcd\x12\xf0\xcf$b\xfe\xd1k\xf5o\x7f\xd2\x80\x01\x86\xffN\xd2\xf0\x1a\x1d\x8a\xc2\xff\x8e\x88\xf9\xfe\xbe\xce\x9f\x101\xd4\xb3c\xb1(\xe0(N\n\x04\xcd<\xe9\x0e\x0e\xa7\xd7\x81\x87P\x88c\xd7\xd9`P\x84\x86X\n! \x02Cyz\x9d@\x11BI\x96\xe7)\x84!\xc5g\xe3\xf8\x8d\x88\xf9\x89D\xcc:\x87\x02F\xb1\x02\xc9\xa3\xa8 0\xb0\xc8c(.\xb0\x14\xbf\xe6\xc9\x9a;\x04\x01S\x0cG\xad\x13\x88#$O2_\xbd\xee\xd7\xe5\x85\xe1\xcf^h\x08\xc1\xfc\xed/N\xc4\xfc\xd7p\xc9\xc7\x88\x98g\x1e\xfe["\xe6\xfb\xe7\xf9/$b\xbe\xba\xb7\xfd\xb0+\xb5\xd3\x1e\xdcH\xc2\xdc}\xfa\xa8\xf3~\xac\x8dy\xb4\xf4\x91\xbaDB\xeb\x89\xe0\xd9\xa7\xce\xc9\xfc\xb0D\xd3t\x80\xf9\x86$*\xde\x1e\xc3\x1b\xe1U\xa6\xa8\x99I\x9b*\xae\xd2\xcf#!\xce\x12u\rg\\\xec\x82\x13[\xbf\xd9\xa9\xfdSD\xccz*\xfb\xb1R\xd4K\x8b\xba\xc3X\xec}\xe9\xd2\x94@\xa8&\x8f\xfb\xe5\xa0\xde\xd7\x9b\x9c\xeey\xb5\xa3a\xaa\x0e\xe4k\x05B\x1ev\xac)\x1ebXD\x92\xf6\xdc\xb8\xf3u\xe4R\xd0\x06\xb4qi\xa1\xab9\xc6 \xf2\xb0\xebX\xbc\xb4\xbdRZY\xf3fG\xd5O\x191\xcf0q\x1a\xa3a\xe8\x95\xc2\x11\x94\x817\xdbD\xe4\xe5\xb0\x98\x15"\x07%e\xb1\xa2\xb0\xce\x1b\xec\xca\xce\xfd@p:$\xcf\x0c\x1dc\xbe\xc8\xc8g\xb3E\xd8\x9dl\xc8\x1a\xcaY>\xaf\xe5\xe1M\x83\x00\xc4$\x9b\xba\x9au\xcb\xc6\x04\xab\x1doovT\xfd\x14\x12\xf3\xc7+\xcc\xbf\xf4Q\xdf\x19\xd7\xcc\xceN\x89A\xdd}\xb9\xb4\xf3\xe4\xf68\x1c\xd1@/\xe4$\x86\x17-\xe1\xe3\x82\xf78\xaa6\xf0\xe9\x08\xb9D\xd4\xd0\xc4\xc0\xd4g(\xeeTZ\xa6slJ\x9a C\xb0\x82B\xe2\xe3\x88\xcb\x1e\x8a>\xf8w7\xf2\xcf 1_ab\xebQ\x8e\x10/a\x16\xba\xe6u\x8dND,0\xd0\xe1\xd8\x8d\x11s\xc5\x19gn\x06:^\x1e\xd7#\x1ce\xf3\x94\xed/\xe3}v\x8aZ\xa9\xae|\x83t\x92L\x96\x1a\xb6\x87\xf6ua+{\x1b\xbe/*(\xf9\x84TM\r4\xd8\xbb7\xcf\xabO!1\xbf\xed\xb4$F\xe2\xaf\xa5\x87:yVO\xb4\xbaO-\x17\xbew\xe0E\xe4:\xb0\xb5\xd6\x81TIt.n@\xda\x91>\xca\xdda\x91\xb8\xc69\xa55\xd5\x84\xb8\xf5Cw2\xd6\xa0\xd0\xfcq\x0c\xaf\xda\t\xad\xaf5w\xbef\x0f x\xf4\xf0\xeeG\xa5\xc7_\x1b\x89y\x8e\xe2zi\xa2\xd7k\xdc\xcby5k\x8fJk\xad\xa4\x17\xfc\x81\x15\xd7\xd5\xc7\xe2}G)\x02r\x10\xbb\xf4\xac\xa6^\xf4\xa0\xaa\x00uBK\xbf\xa1P4W\x9c\x0e\x8fV\xe9\x1cG?J\x1e&|\x96\x95s-\x1e\x15\xf2\x11\xe2\xae\x9d\x97\xac\xe0^\xdf\xdc\xc8?\x85\xc4^8\xc5\n\x9d\xcc`\x1d\x9f\x06J\x84S\x8e0e\xf2\xf2\x92\xd9x\xe2\xfb]+X\x90-\x14\xf2\x9b\xcd\xff?\x85\xc4|\xa5>\x89\xad\x1fC\xbct\x8eM\xf6\xa8\xe7\xee%\x8a\xc4\xcf\x84\xb3\xc3\x8e3\xba\xcc\xf5\xbd\x90\x1f)\x9aW\xdeZ\x16\x87NE;\x1d3Z\xf9\xe8\x985\x128\xd3\xd5\xd0\x86C\x00\xb3%/U\xc5roKNY\x0f\xac p\x8e\xd3L4.r\xf8^]\x86?\x85\xc4\x87\x1dL\xef\x9bq\xbf\xeeo\x0f\xf0T\x90Q\x9e)u{=\x9dQ\r\xb5\xd1\x8a\xbb77\xb7^\x17\xbcX\xdd\xec\xbci\x82\x92R\xc5T\xbc\xca\x00\x88.\xaa\x12\x95\x0e\xea\xde\xc1\xfd\x90{\xe6\x9b5\xdc\xa7\x90\x98g\x98\xd4\x9a\xdc(\x8c\xbc\xa4>6\xb2\xb3\xdb\x02\xa8\x96\x1b\xf54]\xdb\x13A\xa9\xbdT\xecu\xdavP\x90\x80\x18\x910\xab\x18\xe5\x1a\xfe\xeez\'\xb1\x05t<\xb9\xab\xb7i\x0cI\x8fA\xd3\xe1\xe8\xbbN\x7f\xba\xe5Ej\xc2\x8a\xb6\x18Upc\xbf\xd7\xf5\xedSH\xcc\xd7\xfb\x0c\x08\xa2\xd6k\xf4\xcb9Q\xd6\x05\xed\t\x9e\xa1R\xcc\xf9rV\x93\x0cR\xdb|\x7f\xe9xLt\xd7Ys\xef\x0f\x95\xf0\x8b>\x03\xe25c\xc6\xcc\xce\xcd\xb92\x0fi\x9c\xcc\xb7\xb9\x08k\xc7\x8e\xb3\x03\xd2c9\x9adG\x97T\xfd{\x07\x12obb\x9fBb\xbej\'\x14\xc7I\x12y\xb9\xebW\xa8\xd6\xdd\xdcJ\x02E\x831\xeeXv\xbe&E\xac\xb10u\xd8\xf7\xaa\xec\xc5\xcc\xf9\xc0\x83\x85\xd6\x80\xeas\x9b)\xc8\xf4\x82\xe8\xee\x1ck\xfazu\xdb\x07\xd7r\xc7\x1b\xb7\x1d\xd0\xc5*\xd8$>o\xb1^\xe6\x9e\x7f\x11\x12\xf3\x0cs\xbd\x04\xae\xbb\xdc\xeb\x9b\x9b\x0c\xa1\xeay\xf1\xe72\x19\xae\x949\x176\xaf>\xb4\x11T],\xbc\xab\xb0\xcb\x8d\t\xcb\xdcf\xa6Ic\x0c9h\xe7\x11\xe1g\\\xee\x05\x98;]c\x02\xc0\x94\xd4E"i\xe1\x83\xcesG3\xcb\xe7\x9dC\x18o\xf6\x8b\xff\x14\x12\xf35\x9b\x10\x8c\xa3\x08\xfc\x82\xc4,\xf4\x8d\xe8o\xf4(\xcc\x90}\xc1D\xf3\xd8\x00\xd2\x80\xde\x95&a\xfbq\x82\xca3\xdd\xe8\x17w:\xd6\x81\xa1\x96\x94\xe36#\xec\xd5\xaa\x8c\t\xc7\xcb\x0e\xd7N\xea\xfe\xf6\x80\x8d&n\x16=\x94\xee8}\xed8v\xf9\x7f\x13\x89\xf9\xed\x9c\x80 \x1a\x81^\x8e\xc3\x87|\x88\xecs\x83\x86\xb0tc\xdb\xdd\xc1*I\xc7Huy\xb8\x9by\xe4\xd1\xc2\x03\xd0\x0b\xf3\xbe_\xf4\xacc\xf1<\xb8\x03\xaa\xb3\xc7\x92;\x19\x17D3\x02N&\xe1\xb7\x83\x87\xf0\x9d:Y\xe7f\x9fQ\xd6p\x8d\xdf\xa4\xe1>\x85\xc4|\x9d\x13\x04F\x10\xaf\n\x16\xec\xc3\x99!&e-\xcf\xdab\xefP\xb3\xd7\x05\xf8\x02\rw\x81\xbdp1\x02\x06T\x0c\x8e\xe5\x01\xee\xb0a\xcd\xcf\xd4\'\xd4\xfc,w\xc5|\xd2x80B\xf4\x18\x1a\xf7\xd4$\xc4\\jp\x91\x078\'@\xd57/\x87\x9f2b\xbe\xa2\\\x8b\x84\xe7%\xf8\xe5\x9dm\x7fRe\xd7\x9bnD\x9a\x9bC6\xf5Y\xe5\x90dK\nfypQ\xc5\xac\xefV\xec\x8f\xb9`\xac\xa5\xae\xdfs\xf1l\xc7\xf0\x84W6\x90\x1ca\xbe=\x99\xbd\x03\xcc)\xe7\xb1\x0fN\xd3\xaeU\xce\xcdCW\xbdy\xd5\xff\x94\x11\xf3\xb5\xc1\xad+\x16\x85^78\xecX\xa2%\x06\x97\x92 tX\xc8\xd7\xd4\xdc.\x80\xaa\x9d)\x19,F:\x1f\xcc\xe8\xc2\xf8\xb6\xa0d\xcc\xc0\x0b\xbb\xf2<\xb4"8\xe0\xcc\x9e\x89/\x18\x7f\xe7/\xc2z\x05\x9eg\\\xca\xd0!\xcf\xceX\x89s\xc7wM\xb3O\x191_ab\xcf\x1f\x1b\xd2/x\x8aA\xf9Uj\xb2\x9c\x91\xf8\x0f\xc6z \xca\xbe\x963\x96\xdf9\xa9p\x830\xe7T\xed\x02l@\x1d1\r\x08J\xc1\\\xb1\xb8\xca\x8f#1\xc8@\xa7TT\xbaxb>\xbaK\x85\xaa \x9d\xc1V1\x95\xec\xe3\x98\xbc\xf9R\xf5SF\xccWA\xfet;\xb0\xd7\xbb8r\x98\xab\xeb\xe96\xb6S\x1a\xaaz\x84\xc6\x0b\xec\xdae\xa6\xbb\xa7L\t\xba\xa1\xf1\xd1k\xcc\x82^\xa9\x1fg\xf5\x08\xc7\x8f\x19?\xb9\xfe]\x13\xec<"Pu9\xab~~\x1a\x11\x8e\xbc_\x94\xa1\xdba\xc1\xd5\xe8\x85\x1f\xe5\xe6_\xdb\x88\xf9\xda\xc7\xa1u\xb3@\xc9\x97\xd4o\xe3{\x84\xaf\xd7\\\xf8\xb2\x96\x84\x0c\xd6\xeeb\x9dA\xf8t\x90\x18\x1c\x8f\xd7\x02\xbe\xbe\xaf{7(7q\xfb\x180\xa2\x97\x04\x00Ct\xb1\xc6\xb0\xbd\xb3o$ \x95%[\xda\xa9\x0f\xe9\x9a\xd8F\xaeb\xb1vW\x82_\x84\xc4<\xc3\xc4P\x14\xa2\x88\xd7\x97\x99jJ\xec,\xd9HQ\xf7\xa4\x0f\xbd\x00\x95\xb0\xdc\x81Y\xb6?K\x95DK\x9dz\x8cw\x8d]\x99\xaa\xdd\x90\xc6\x95\xe3\xf6G\xdahoD\xac\x9f;\xcf\x01 \x10\x8e\xd3\xda\x18I09\x83\\-q{o&1\xe9\xcd\x9c\xf8\x14\x12\xf3\x95\x13(\xfc,\x05_\xc2\x8c}\x04\x9e\xa7\xe8\xee4\xee\xc5\xe3yZ\x93\xfc\xb4\xbf\xc1\x1cfEaIvFb\xa6\x1eY\x0e\x1d\xd7\xaa\'\x91C\xd3[\x14Zk\x12\xf0\xc5\x98pT\x01\x13Q\x0fO4`\rc\x00\x9e\x9c9\xb9!\xf7\x86\xfaEH\xcc\xd7\xbd\x03[\xef\xb5\x10\xf1\xb2\x91\x87\xb7\x1b\x99\x85\t\x88\x07M\'\x9dy\xd2\xb1h\x14X\x00^o\x04\xf4\xe6\x8a]\xb6\'\x16\xe2\xa1\xea\x03<\xd4\r;\xa72(\x14\x18W\x88\xec^\x8b\x97\x89\x8df\x8d\xd0\xa8r\xbf\xc7\x1c\xaa6\xf1\x03\xc3\xba\xa4\xf5\xbdR\xffSH\xcco\x1b(\xf5\xfci\xdd\xcb9a\x96\xcd\x95\x8c3\xf6V\x04n\x96\xcd\xb3Z\x12\xcd\xech\x17,\x88\x9d\xa3\xe7^\x82\x93\x11\xdcJ\x9dM`\x9a2\x80&\x06O&\xef\xed\'\x11\xcadYg\x11\xa7C\xea\x89\roE]\xe9\x12\x99\x1c\x10K\xc1\x9c\xdfN\xfd\xff\x88\xc4|\xbd\x9d\xdc\x90\x98\xff\xfa\xb7\xd5\xff\x1fBb\xfe:\xfd\xcd\xb7\x96\xf1\x1b\x12\xb3!1\x1b\x12\xb3!1\x1b\x12\xb3!1\xdf\xf89\xb7\x13\xff\xfb\x9e\xf8\x1b\x12\xb3!1\x1b\x12\xb3!1\x1b\x12\xb3!1\x1b\x12\xf3\x8d\x9fsCb6$fCb6$\xe6;\xe3+\x1b\x12\xb3!1\x1b\x12\xb3!1\xdf\xf697$fCb6$fCb\xbe3\xbe\xb2!1\x1b\x12\xb3!1\x1b\x12\xf3m\x9fsCb6$fCb6$\xe6;\xe3+\x1b\x12\xb3!1\x1b\x12\xb3!1\xdf\xf697$fCb6$fCb\xbe3\xbe\xb2!1\x1b\x12\xb3!1\x1b\x12\xf3m\x9f\xf3\x7f\xc4!\xfc\xcb\x86\xfd\x1f9\x84?\xecE\xdf\x19\x89\xf9\xf9\x81\xfd"$\xe6\xe7\x07\xf6\x8b\x90\x98\x9f\x1f\xd8/Bb\xde\x0b\xec\x7fB\x8e\xfc"$\xe6\xe7\xcf\xd8/Bb\xfe\x176\x8f\r\x89\xf9)H\x0c\xfe\xcf1\xff\xf7H\x0cK\x92\x18\n\xe1\x04!"\x82\xc8B\x02D",F\xc2\x94\xc0\x938\x82#\xac\x00cO\xcf\x81\x86D\x82\xa59\x1a\'`\x9a\x85\x08\x81f\xd6/\xe3(\xf1\xdf#1\xcf\x7f\xa9\x11\x03#?\xeeG/:N\xcf\x9c\xf1{\x07\x93\xa4\x10\x9a\x96\xa7\xce\xcd\xa3tO^\xd2NP\x82H\xc9\rZ\x94\xc0\xbc\xd0xFK\t\xaf\x9cN\x1e\xb6\x87\xd6B\r^\xcc\xf8\x82\xdb^\xc6{\xe6\xc5\xbe\xc0s\xab\x93g\x86\xf1\x0fE\xc6\xbe\xd7\xa4\xeeSF\xccz(\xa00IQ\xf4\xfa\xefK\xefm\x08\n\xab\xbc\xda)`%)QM\x1d\xf9=\xae\xf9\xd8\xcd\x1e\xf5\x08\xbc\xea\xfb\x83$-Bu\x0f\x14=?\x1f \x81\x0c\x1d\xbc\xd0@\xe2d\xd9^\x18h\xe3\x8e\xbc\x98\xa0\'\xcb\x02k\x02\xd7\xb3\xe9\x14\xaa\xb7w\xdel\xe1\xf4)#f\r\x13[\x17!\xf9\xd276\xbf\x1e\x0bH\x9f\xed\xa3\xad].\xa5\x93\x0b\xae"\x1dm\x00\xd7\x1d7\xf1\xa2\x8b\xeb\xec\x1a\x07\xbc\x85\xa7\xee\\\x9d;\xf4\xaaX)!\x97V\xb0\x0c7\'i.0\xbbW\x81lf\\\xa4\x1f\xa1\xe4J\xcd\\\xbf+\xc67\x1b\xaa~\xca\x88y\xc6Hb\x08\xf1\'}\xf8L\x01\x98v\x97u\xc3\x17\xe0\x82\x8f\xc0\xdb\x11\xac\xfc\xcb:Q\xa7sm(&\xd6\xab\xbc\x0f\xa9\x93d$\xb1S*\\A\xa8\xe0\x95H\xe4vw 0\xf7\x00d&*7\x87\xbcs\xedK\xcf]\xdd\xb8\xa4\x80\xe0\xda\xfe\x085\xfak\x131\x7f(\x92\xfe\xa5\xa3\x19.\x0c\reS\x97\xc3Q\x88\xcdkuIleo\xa1h\xcf\x07\xa4%\xe5\x8c\xca\xed#12fQw\xf2\xfb#\x1fq8"\x01\xadd&D\x85*\xd9\x84p\xed\x92&T\x1c1\xf2\xe5~\xde\x03\x88L#\xa77[\x0c\x7fJ\x88yf=\x01\xc1\xe4\xf3\xbc\xfec\x98\'\xac\xcbY\x05\x08/U\xb8\x1b\x1eG\xbd\n\xe1\x87\x12#\x9abe\xdc\xa5\x0c\xb5\xaa\xb7\x83\x04l\x1f\x11\x04,\xf3\x0e-\xc4\x18\xe7\xeb\x93\xd1\x99X\xd3\x82JaFg\xc4u\xb39\xe9t\x06\x86\x84R\x15\x94\xf5\x9cy/#>%\xc4<\'\x13\xa3`\xfa\xb5\xd5 y\xc7\xef\xf5\x99\xaf\x8e\x0cg+\xd7\x1a\xd7)\xb04\x0e$o\tW\xb4\x9a;8\xb6\xa3/\xc4\xed\x94\x93=H\x97\x9ec\x16 +\xec5\xacqii)\x9c\x85\x03S\x99\'\xba@\x080\x1a\x18\xa9\xc9\xa01\xd2x\xac\xa7\xccMt\xf6\x87\xc0G\x1be\xff\xa6e\xf6)\x1e\xe6\x19&\x0eC\x08\x89\xbf6\x8e\x0c(\x83\xa9,\x8f\xc5\x0e\xf0\xb3\rj\xa6\xec\xb3\xdd\x95\x9a\xf2KRt\xbbS$\xcb\xa2\xa1T\xc9\xd5V\xc89\xa8\x96vBZv\xa9|_\xf6\xa9\xebl]\x17\xf8<>BA\x91\xedn\x01\xdc\xc5\xe3\xdb\xfa\x1a\x9ao6\x00\xfe\x14\x0f\xf3\x15\xe6z\x0c\x90\xeb.\xfe\xc70\x1b\xeb\xa6\x12\xe5u\xf1qz\xd2$\x17\r=\xa5(\xc4\xf2\x9a\xd4\xe35\xb5\x88\x0b9,S\x93\xc3T`\x1d\xcc\x1dqx\x1c\r\x1a\xa2\x0c)e\xb3\x03[j\x05g\xd8\x0f\x01E\x06\xa7l\x81]\xd8\xf76\xb2\xec\xaa7\x13\xf3S<\xccWb\x92\x04\xbd^\x1a^\xf6\x9f\x94\x97\xf6X\xb1\xb0S\x08E\\\x000\xb5\x94\xee\xe24\x1d\x8d{d:9\xd8\xb5\x18t\xa7\xb5\x1a\xb9\xf6lr\xb0\x97\x16O\xc7\xf2\xdc;"\x01;\xca\xb2\x03qa\xd9\xc5\x98\xdc\xa6\xd7\xe0|\\\xce6\xc9\xfa\x8f\xf8{5\xc5\xfe\x14\x0f\xf3\\,\x18\x84\xe3\x7f\xb6\xc3\x19\x97)\xe5I\x15\x10\xa3\xfd\x99:E\x8c\xd1)7\xfc.\x8b%z\x1eqNQ0`\xafb\xd9\x11d\xd7M\x1b\xd7\x02\xdd\xe5N\xac$\xc5{o\xe7\xe1\x89\x1e\xcb]6\xf2\xce\xbcL \x1d\xb33<\x04!\xce\xc1\xef\xe2\t\x1f\xe2a~\xdb\xe1(\x1a\xa7^\x0f\xab\x8b-+\xeb\xb1\xce\xa2\xdd\x94\x9e\xac,#\xe5\xaa\xb95\xc2\xc2e\x18\xf8HEu\x17")\xd4&\xd9U>^ri\tQ8\xc6\xc3ij\xe7Z\x8b\xe5&*\xa4\x9a\x02A\x1f\x1e\xf6-\r\xf4\x14\xd9b\xee\x83y\x9b3\xfb\x0c\x0f\xf3\x0c\xf3\xd9\x1b\x93\xf8\x13\xad1\x06\x1a\xec|:sV\xbeW,F\xb1\xb5\xe3a7\xc7\xa2p\x96\xef\xc19N\xb3\xdd>\xc1\xb86\xdc\x9d0\xf7\x82\xf8\xa5\xd9\x17gA7i\xf6\x82\xcd\xc7\x94A\x13P\xe8\x0e\x13$`\\\xd9E\xb7\\\xee\x96\xe3\xe4~3#\xe2C<\xcco\x1b\xe8\xba\x05\xe3\xafU\xb0\x04\xaa\xa0\x86\x9f\x04\xe8\xc4gP\xe1\x1au\x99\xc5\'\xe2hPW\xb5\xa7\xca\xb3-\xc2\xa9u\\:\xa0\xc5\xaf\xd6\xa2\\s\xbc\x91\x0eN\x87\xdd8\x88\xcad\xa4\x06\x1b\x12\t\xb1.\xd9\x97S\xcd]L\x8f\x8e\x87)\xfeE<\xcc\xb3\xd6_W\x1cF@\xaf\x9dw\x03\xebV\x90\xb5\t"\xbc\\cX\xa5\xefH;\xdc;\x87\xfb\xde\x10Aa\xbc^\x1f\xb7\x83;\xabV\x00\x9c.;Tk\xf67\x8e\x9c\x97\xc4"\'\x87\x1a\xa0\xee\xd4\xef\xe8\x044ra$Z\x18+c\xea\\\xfb\xb1\xfef\x0b\xf5O\xf10_\xd58\x84\xadW<\xe2%\'\xc6|P\xf9\xa8\x80I\x17k\x1a\xcbo\x9c\x9c\xf2\xaeXr\xdc\xe9\x14UH\x0b]\xd3\xc4\xc56\xa7\x90?\x9d\xe2\xe0\x94\xe1\x1a\xa3\xf4\x05{\xf0\x19\xe9.UXE\xea\xeb\xf0\xab\xe09`\x12G\xd0\x9c\xbd\x08\x96\xec\xed\xddr\xfcC<\xcc\xd7\xa2]?\x04\'\xa1\x17#\xc2d\xae|\xbcKn\x81\x8b\x04\x0c\x14\x08LD\xdcw\xfd$W\xbb\x00?\xc7\'\x04\xd9C\xcbE;\x82\xd8\xcd\xba\xf3R\x96\xf2\xe1r&\xb0\x8c\r.\xe6\xe2a-\xeb\x9c\n\x0e\x899\x8f\x87\x98k|\x93\xaa\xd3\x05C\xbf\xd7q\xf8)\x1e\xe6\xeb\x9c\x80I\x8cD\xb1\x97\x9c\x00\xa9q\xbe\xf1,\x84\xecX\xde\x1c\xab\x03\x9c\x8c\xc6\xb4\xa8\xba\x94\xf6\x15\xe1^=\x83T\xb3\t\xde\xf1WV\xf7\xf04\xab\x8a\x94\xd4\x1eUa\x9b\xe8`\xb8\xca\x8cX\x01\xa1\x0c\x89\xc1\x0f\xe8U\x9cD\xaf\xba5H\xf7f\xed\xf4)\x1e\xe6\x99\x13\xeb\x99\x8f\xc3\xc4k\x98\x0c<\x10\xc69\xde\xef\x8d\xe8r\xb1`\x9d\xa7\x1fUxp\xdd82H\xc2\xdcg\xfe\x83\xf5\xcb\x0bR\x84z\xe9uh\xe2\xf6\xa6eN&P4\xfb\xbbw\x91uh\xa4R\xf7\xd6\x8a\xa0\xbf\xabT\xbf\xefj|\xef9\xef*8\x1f\xe2a\x9e;\x1c\x8e\xac\x05\x03\xf2\xaa\'\x94\x12n\xe1B\xc2zi\xbf\xbf\x9e\xaas\xeb\x8d\x07\x1d@\xf5\xcbA\xceN\x19Q),\x1e\x1f\xba\x9a\xcd\xd5\x19\x1b\x81\xe3\x81\xf6\xd3qv[\xff(\x1b\xb6!Q\x0cB\x91\xc5R*V\xe91\xf1Z\xafw\xcb\x08\xe4\xbf\x88\x87y.Zj-l0\x8a~\xe5a\xbc\xa3\xcdf\xc3\x02^/\x10\xaf\x1c\x80@\x1ax1Bz%\xa6\xabD\xd1\x0eX\xdc\xaeg\x13J\xe37\xe2\x8c\x1dQ\xc2\x96\xcaFP\x0f\xbd\x06\xea\xb9\xa1\xecUWhS\x1eu\xef\xc7\x9b/\x1f\x1d\x0b\xb5\xb4!\xfe\xd1\xbb\xa9\xbf6\x0f\xf3\xcc\t\n\xc5\t\x94|\xb1\x19\xee;i.\xc1=\xc7\xf0-\x8f\xf8\xe1\x9d\xf5\xb1\xf2\x08$\xcc\xbes\x8e\xe6<\xeb(\xf0\xf0N\x12\xdf\xe5\xba\x1fi\x12\x9c\x10\xc8\x81\t\x12\xe7\x90\x9cu\x08\x90\xeb\xae\xd0r\x0f\x14\xce\x89;/\xd1\xa1\xa5FH;\xc7o\x16\xc2\x9f\xd2a\xbe\x8e\t\x08A1\x94~I\t\xc4\x89n;O\xa7\x97y\x1f\xf1\xd8i\x1e\xbdu\xc1\x9b\xc2\xd19.\x97\xe9d\r\x97\xa9\x9a\xea\xa1\x15\xa1\xc8u\x0e=\xa5HF\xb4+e$\xd8\x83\n\x03pe\x93\xd9\x8f\xc3\x8c\x1e\x15\xf6Z\xb1\xc4\xd5\xe7\x1a\x1cx\xd7\xbf\xfa\x14\x0f\xf3\x9cL\x94FQ\x92&_6\xb8d|\xd0{%\xbe\xa3\xb6\x1d\xfa"}\x8c\xb49\xc6\x00dn\x1f\xbb\xbb\xc2>2\x97\xae\xa1\xcb\xbd\x12\x83\x12p\xf7v!\x06\xa7\xba\x9e\xf5\xb9o\x8aQ\x0e\xba*\xde\xf7\xa9u;\x12\xcd|\xf0/n\x99a\xfc\x88\xbd{\xad\xf9\x14\x0f\xf3\xf5\x0e\x0e\xc3\xa8\x97\xf7\x8c\xc7\x8c\xddW\x07\xc9\xd0f\xef\x01\x8bTK\xb8\xc3x\xc5\xfb\xc8\x92Be\xbd\xf0\xc2w\x12\xee\x84\xf8\xea\xef\x82\x051\x970<\xab\xd7\x18{\xb4\xf6Y\x8c\x82\x93?\x1fYsFf\xc3\xf5\xc9\xe5\x92{gg\xb9[\x8c\xfc\xe6[\x9bO\xd90_\x9b\xf8S\x98\xa0__g\xe4K\x97\x1c\x1e\xbd`\xb0u\xb7sG\x8a\xc9B\x97\xd6\xdb\xaa{\xb8\xbbRP\x9d1\x03jq\xc7\xf4I@*\x85n\x17\\\xc9UC9\x1e\xce5R\xf5$2\xddQG\x0b\xf3\x02\tJ\xf0\xae\xed\xf2Zuc\xe5M\xd0\xe8S6\xcc\xd7\xd5\x8dB\x9f^\xe3k\xe5\x81,\xf8\xc3\xc3R\x1a\x17k\x07 \x92\xa1\xd0-\x80R\x80\xc5\x1e-\x98\x14%Vc\xe7!\xc0\x8a\xcb\xd5\xb1\x96{\xcaA\x9dji\x91\x7f\x0ecy\x87\x9d\xa3\x8b\xf3\xf0\xd3t\xc7B\x17l8\x883(\xf5,\xe0\xfe\xe8\x86\xfa\xd7\xb6a\xbe^\nA\x04\x86C/\x9b\xf8\x91>\xcf\x85\xbb\x04k\xa2\x85\x87\xcb\xb1\xde\xcf\xfaP\xdf\xeb\xa3~\xa3\x86\xabp\x1e\xd3\x86\x98\xe1Y\xbf\x9d\xe3\x85U@\xac1\xf7AV\x08\xb5Q\x1f\xf4=\xae\xdf\x15\'bQ\xd4p\x8bI5\xd6\xbaD\x89\x13\x8a\xe5\xdf\xdc\xdd>E\xc3|\x1d\xf80Jb8\xf6\x92\xf9\xbb8\xbeh^Ou\xd5\xb4(\xb0w>\xd7"\x14f\xa7\xe5\x1e+pq\xe3\xcf\xb7\xb3\x9fN\xbc\x188\xa5L1\xda\xbap\x87\xe8\xca\xb3\x8dT1J\x90\xc3\x19\xea\xb5\xeei`X\xbb\xb8]\x03\x90X\xe4\xca\xb5\xc37/\xa8\x9f\xa2a~{_K`\x04\xfcZ\xd7p\xe2\x05#(\x8f\xc1\x9d\xa1\xc9\x86k\x15\xa1\x17\xc6\xe3iI.\xcc\x05\xf3D,=\x10=\xeb\x1d\x8f\x12/\x9d5\xf3z\xd5\x81G\xdd\x1a\xa7\xbc\x9b\xf6\xb3\x15\x1c\xa60\xee&k\xb1FV\x9c/\x97\xc7<\xdc\x88\xba~\xb3\x18\xff\x14\r\xf3\xb5\x89#\x14\xf4|\xe1\xf0\xc70y\xdf\xceF\x14\xae\xc6\xbd~dg\xd5\x00\x02\xdb:\x1b\xfd\xe1~\xcd\x96\xfd\x02\xa0\xbb\xc9:\xacw\x0c\xe3>\x9f\xe4\x8c\xa1%SUw\\\x86\xa8\xd950k\x9f\x9bLux\x9c\xc0\xe2!\x878t\x83\xbbC\xae\xa3\xfc\x8f\xee\xe1\x7fm\x1a\xe6yL`\xeb~K\xd2\xaf\xee\x1d\xaf\xf6\x89\xcb\x08\x8f\xce\x0e\x86\xa0\x9bN\xe6^\xad\xb1\xa4\x92v\x17\x87\x88\x93\x81kbc\x1f\\\xea\x81{\xac\xa7!\xa8\xccz\xdc\xe9;+V\x91\x13l\xc3\x8f\x11\x08A\xb4:@\xcd\xf1\x16\x97\xf8\x15sy\xb8\xb6\x84\xdf\xb4\xf0\xffH\xc3|\x15\xef\x1b\r\xf3_\xff\x8e\xfa\xffC4\xcc_\x07\xdd\xd8\x1c\x93mH\xbf\xff\x90n4\xccF\xc3l4\xccF\xc3l4\xccF\xc3l4\xcc7~\xce\x8d\x86\xd9h\x98\x8d\x86\xd9h\x98\xefL\xael4\xccF\xc3l4\xccF\xc3|\xdb\xe7\xdch\x98\x8d\x86\xd9h\x98\x8d\x86\xf9\xce\xe4\xcaF\xc3l4\xccF\xc3l4\xcc\xb7}\xce\x8d\x86\xd9h\x98\x8d\x86\xd9h\x98\xefL\xael4\xccF\xc3l4\xccF\xc3|\xdb\xe7\xdch\x98\x8d\x86\xd9h\x98\x8d\x86\xf9\xce\xe4\xcaF\xc3l4\xccF\xc3l4\xcc\xb7}\xce\x8d\x86\xd9h\x98\x8d\x86\xd9h\x98\xefL\xael4\xccF\xc3\xbc\x83 \xfc\xcb7\xf9\x8f\x08\xc2\x1f\xb6\xf7\xefL\xc3\xfc\xfc\xc0~\x11\r\xf3\xf3\x03\xfbE4\xcc\xcf\x0f\xec\x17\xd10\xef\x05\xf6?\x81F~\x11\r\xf3\xf3g\xec\x17\xd10\xff\x0b\x9b\xc7F\xc3\xfc\x14\x1a\x86\xf8\xe7\x98\xff{\x1a\x86\xe6\t\x1a\x81\x04\x86`0\x02\x81\x11\x81\xc5i\x8e\x81\x04\x11\xc7Q\x8e!\t\x88\x87)\x11\xa6\x10\x01%1\x86\x81 \x8eea\xe6\xd9\x06\x83\xc4hH\xa4\xb1g/\x98\x1f\xb3\x07\x9c@ ,/\x12<\xc6\xa0$\xc9b\x18\xf9\xec\xfd\xcb\x12\x1cK\x0b\xcf.\x11\xb8\xc8\x10\xc2\xfa\xe14\xce\xb2\xebH#\xe4\xf3\xfbB\x08\xcc\x13\xb8H\xa2(\xf53i\x98\x7f\xb4\x93\xfd\xdb\x9f5`\xa0\xffN\xc2\xd8\xfa$4\x8e\xfe;\x1b\xe6\xfb\xbb:\x7fb\xc3\xa0<\xbb\x0e9\x81\xf0\x14#\x08,&\x90(\x05\x89\xcf\x0e\x1f"O\xae\xb3\x80\xf3(\xc1S,O\x91,M\xe2\xc4:\xe90D\xb0"\xca\x92\x02\xc6!4\xf4\x1c\xc5\xcd\x86\xf9\x896\x0c,"\xebxA\x08\x0f\x0b$\xc3\xd0"\x84>\xbb\xdc#\x02\x87\xe20K\xa04\xbe\xee\r"\x03\x11\x14\xcb|u\xd5\x83\x9e\x7f\x01Ea\x14[S\x08\xe6\xa1\xbf\xfd\xe5m\x98\xff\xb2E\xfa\xc7l\x98\xe7\x83\xfcn\xc3@\x7fj\xc3|\xff<\xff\xa56\x0c\x82\xfd\x10\x89\x18w\x85\xe8\xa2\x8e\xd6\xa7\x8co\xde\x87j\xea:\x12r\xd9\x0c\x10\xea[\xd8\\gz\x19D.\xbb_=|\x96/\\\xc57Z\x95%\xd8\xf00o%P/\xb0}\xbb]}Z>8\xa4\xf3P\xaa=\t\xd4\xb2zz\xb3\xf9\xee\xc7h\x98\xf5Px\x92)\xcf\xfe\xd6\x7f\x0cSF\x0c7-\xb5\x8a\xf3\x07\x15\xe9\x08\x1a\x10\xc5\xbd[\x1a\xfb\xd6;r\x0b-@\x8f \xca\x97A)\x0e\xddC\xc6\x17;\xe3\xa2Ct\xb9\x18\xe7\x8bx\x82"\x02\x02\x1c\xb9\xd4Y\xd2*\xad\x0b`s\xbb]p\x0b\xed\xe9\xcd>|\x1f\xa3a\xe8\xbfS\xc8\x9fto\xc2\xcaVm\x1d\xb5\xf2%1\xf0\x1a\xeb\xa4\x02\xf2\x9dt\xc7\xc2\x05\x1a1\xbf\x86,i\xcb\x01\x11k;\xb0\xde\xebUq\xc2\xc6\n/\xf0\xcb\xd9\xf3\xd46\xb9 tE\xa8\x10-\x1ez\xe9zC\x00z\x87r5T\xbf\xdb,\xfaS0\x0c\xfdw\x1aZS\x18\xa6\xe8\x97\x894\x1b\xca\x1f\xcah\xd9\xc1V\x97\x1e\x12Ef,\x02\xc8\x1er\xda\xa8\xd5\xe0\xc7~x\x84\x1e\x89\x1d91A+^F\x14\x8a\x11t\x11>a\xfdM\xd3bX9]\x17k>\x0f7\x0b\xa9\x13&\x96\x88\xa0\xa5\x14\xeaG\x13\xf9\x17\x97a\xd6t \x91u\xad\x91\xc4\xcbb1\x8f\x97\xb2v/\xc5\x88(y\x07\xe8\x97u\xa3\xca\xdbi\xa88\x969\xb9-G\xe8\xe7\xabk\x01\'~\xaf\xa0\xf8\x92\xf3\xa5rVO\t1\xbb\x08\xdb\x06\xda\xbc\xd4\xd0\xd5\xf4\xa0\x07~.\xb3B\x85\xad\xbdG6\t\xff\xa6@\xf11\x1afM\x875\xe1I\x02\xa1^\xc2\x0c3\x99\x9cm1\xa0\xf6\xbb\xbaG\x13wN\x07\x8d\xe5\xa8Q\xa5\xfb\xd6<\x93Ww\x86\xe7\xb3m\xcaeL\xa7\'\xdb\xaf\x96\xebhc\xda\xa0\x1a\x16/v\x87\xbbNe\xbd\xdd8\x8bA\xf8\x87Q\xef\xfc\xb9](\xe3\xedn\xfb\x1f\xa2a\xe8\xbf?A@x\xdd\xdd^\xfa\xd3\x9d\xab"\xf1\x01\xb8\xe8v\x1c\xc3.\xcc#\xea`E\xed\x19fG-\r\xae\x91\x0bm\x1b\xa6\xda\x8aN(Hea\xca\x1dw\x14\xc3H\xb9\xf2W\xa2B$o?\xa7Dx\xf6\x9dJ\xc5\x07;\x01,\xffh\xccw\xe5\xcd\x06\x9c\x1f\xd3a\xd6\xd9\x84qh\x1d\x11\xec\xa5e,W\xc7n8`\xe5d\x14r\xb8n&\x0f\xa2\x0f\x9ce\x94Y\xef\xb6\x8f`1\x14Q\x0f\xe1\x97\xc9#K\xc0\x1d\x01\x06q\xf9"\xa2\x1b\xf9z\x9f\xef\x11t\xdd\x0f\xb5\xcc\x03\x9d\x82\xdf\x95\x07\x8f\xee\xfb\x83\xbc\xeb\x05\xe2\xcd.\xca\x1f\xd3a\x9e\xb9I\xac\x13\xbf\xce\xfc\x0b\xd6\x96^\x88A\xeb*\xfb$\xa7\xfe\x95\x84\x17\x1bd)\xb98\x1e\x12\xe3\xaa\x11\\$=\xc2\x1d(\xec\xf2$Gl\xf2x\x1f:\x8eS&6\xdd\x07\xbam\\\xf5\xa6\xb7.l\xbf\x1bp\xda\xd8\x01=3\xc5z\x12\xc8\xef\xf6\x89\xff\x98\x0e\xb3\x86I\xac\xcb\x16\xfa\x93\x16\x99\xeb\xbe\x8b\xeb\xbdL\x84\x15{\x1c\xba\x98\x17\x95\x96\xc1\x90\x1bWIg1\xdae\x95\xe6_\xef\x03\'A7\xfdx\x17\xcfR\x0e\x18\xe7\x84\xf5\x8c\xb1WA-\xa1rq\xb8\xd3\xb1~?\xd4\x95\xe6\xca\x93#\x1e\xd4|\x9c\xde\\\xb4\x1f\xd3a\xd6\xdc\xa4\xb0\xb5D\xc4\xc8\x97\xdc\xa4b\xa4Qv,{;\x83\xf0!-\xce\x8ep;\xef"\xf3"p\xe4bR2-\xf6R\x83\x8cS\x8b\x17\xa3ZF\x95\xe0\xba\xd2\x1d\x0el\x1fU\xee\x8ez\x15\x16w\x86<\xf9\x02\xe9ww\x16.fu|\xd0\xa5\xf8\xbd\xce\xab\x8f\xe90\xeb\xa9\x8f\xaf74\x18\xc2_z\xef\xce"=\x15\x98\xcb\xd2\xa0[\xaa@+\x9dw\x928\\c\xab\xa0:;iY\rl\x92I\x0b\xb2\xbc\xf6\x17\xf6\x8a\x97\x86\xa3"\xea\xad\x8d\x0fe\x96X\xf3\xc5H\x0eN\x89\x85h1\xca\xa5\xad6\xf3\x94\xb5\xc0c|\xb3J\xfd\x98\x0e\xf3\x87W\x17\xffw\x98\xfb\x1b\x91.7\x8e\xc2\x98\x06\xb3\xc8\x8eu\x91\xd9\xc9\xb0Ln\xaf7\x87\xd2qr\x9a\xf7\x95\xe2\xc5\xc0@{~\xea\x97\xd6QY8|:\x08G\xa2\xa6.D\x1b\x05\x16\xb7x5\xd3z\x12\xc0\x15\x87\x1c\xa0\xc6\xe8\xc4\xff"\x1d\x86~r\x948\x8e!\xaf\x12\xc6<]u\xcf 9R\x15\xca\x0c:\xb16\x87$\xb1e;\x96\xa3_vF`,\xe1Y\xd6\x9d\xb8\xe0\t/R=\xa2"\xcd6\xb6\xcb\xf4,E\xaa/\xf5\x16\xea\xa6W?\xb6JT\xcfw\xc7:?(\xb4\xf7\xb8\xb0\xdfJ\xf7\xfb\x98\x0e\xf3<\'\xd6b\x10\xa2\xa9WO\x84\x85h\xa4\xb9\x1f\xcfi~\x0e\xd4%\x00\x8d!=Q\xc1\x10\x17\xbb\x87\x05`w\xd5&\xc4\xc7\x98<\x04C\xc8\x8b\xd1=\x11:\xcc\xb7\x06LubvZj\xf9\x0cR\xe1Y?\x80\x16\xac\xfb\xe7\xa8\x92\x8ai\xb9\xb6o\x167\x1f\xd3a\x9e5\xdcs\xbb ^\xbb\x8bw\x96\x00\xfa\x88\x91i\xeb\x13\x13\x02x_\xac\x1e\xb3u\xd5\xc8\xb2\x893\xd7\xf2%=\xb9\xf3}\xff\xb0w\xf7P\xef"\xe0\x80\xc7\xe7\x85\n\xb8\x8b\xa3r\xf6\x89\x8b\x9b\xae\xb5\x87\xfd\x05@\x86`\x17\xaa\x99\xa6\xe5\'\xccx\xb3\xf9\xee\xc7t\x98g\xea\x13\xeb\xa6H\x91\xaf\xc5M\xecC\xf1\xdc6-\x9f,W\x9f\xe4<\xb0\x94\xd0a\x9f\r~p\xbe\xdeS$3S{\x0c\x16\xe8\xae\xea,\xa3.\\\xd1\x95\xc8\x95\xbd\x1b3\xbb\xe4\xfb\xa5\xa7\xd2T\xf6 \xb1\xf0\x94\xf9DZ\x82oeu\xa9\xbdy\x1c~L\x87\xf9\xba\x87c(A\xa3/\x8b\x16\xb9\x98\xc1r\xdf\xe1\xf3\x01\xba\x91\xa5\x91\xf8I\x0cY=\xdf\xdaa\xc6\xa5\x10~$\xd4\x92\x02{\xe6\xd4\xd2:\xbf\xf0\x11\x15\xef\x9d\x86\xae\x90D\x9co\xc9c\x82\xda\xaa\xabLYk%\x19\xb7q\x99\x1e\xc4\xd8\x14\x8c\x1f5Q\xff\x8b\xeb0\xeb(\xe2O$\x02}\xed\xbb\xdd\xfa\xf5\xc9\xaf\xa1\xf0\x90:\x19\x932l\x0b\xd3"\x94y\'\x01\xcbYAz\xa4\x16O.\'\xc7~\xe4\x97\xfb\xd9-\xa1\xd1\xce\xe4{\xb0\x93\xb2z\x12\tR\xee\xb4\xe1\xc4\x0fw\x91\x1b\xc03j\x93UV\xa4\x073\xfdU:\xccZ;\xd1\xeba\xf8|\x07\xf9\xc70]7=\x8d\'\x19+\xd1\x91Ds_\x07+\xbf\x9b\xd8=\x91\xcd\xc6mz\x9c\xda\x8b\x01\xef\x93\xa2\xca\xc4\xe2p\xbd"\x15\xa6\x97\xb5\x19\xb9\xe4\x113\x85x\x11\xad=\xc6\xee\xe06S\xce\xce\xb1rv\xf0Uv\x14\xce\xf8U:\xcc\xba\xc3\xade3M"\xc8K\xc7\xfd\x86m\xb4\xd3\xfeh\xb4z\x13\xdfC\xec\x92#1?^\x11\x1bA\xf0\xf9Q\x0b\xf6-\xf1`\xb6\x03\x0f6\xaf\xa8,T\xa3{\xb88\x9e]3R\x92s\xc1\xa0\\H\xce0\xe0\x00\xa2\x9a7\xa8k\xc4\x86\xcb\x9aZ\xf7f\x98\x1f\xd3a\x9e\x1b9\xbe.s\x0c~)\xf8\xabT\xa4\xcf\x11\xdbZ\x97j\x8e)Q\xa7\xf9\xf3x\xf1Os q\xbe\xc0\x07\t1\xdbD\x86\xd0\xfets\xe4\xebP:\xd3\x8e\xf1\xe5l\xd4b|wt\xe9\xe9X\xe8\x8b\xef\x83(\x01\xe3\xe5zqM\xce\x00v\x14\x7f\xf4\x82\xea/\xae\xc3<\xef\x13\xebg\xe0\xd4\xabq\xdd\xe7Y\xca^\xf8I\xbf\xb0\xd3\xad\xa8\xe6c\xc3\xe9"w\x1c\xab\x14\n\xe2\x83\xc1u\xacz\x1b\xc8\xb8\xbd^$Zm\xd3\x08m\x90\x1d\x94\xf3\x86\x8a\xdc\x11I\xeb\xa3\xbd\x0eLy\x0f\xf5\x90~\xdb+1\xca\xe3\x97`\xff\xe6\xa9\xff1\x1e\xe6Y\xdc\xa0\x18\xf6\'o4\x90S\xee\'\xde\x1ee\xf6gW\xbeSfB\x90\xd6`&\xc0\xb4\xafB\x95\xc3S\xdc\xd8%\xbb\x01\x8aCd8\xec\xb1\x9e\xf5\xf59Q\xd3Ds\x91\xbcoqy\xad\xb3JJ\xe0\x8a\x9bb)\xe7\x06\xa2#\xcdOo?\xf2\x03\x7f\xb6\x0e\xf3|q\x03A\x14E\xbe\x96pu\xa6wg2\xba7b\x18\xf1\x99@\x9d\xc8zw\x11\xcce\x91K\\\x05\xa8\x18"\xcc\xa0\xae\xcb{\x88\x8c\xc9\xae\'r45o\xca\xd1\xeeyE\xca\x8eVp\x99\xba\x83\x058\r\xcf\xe8\xed\xee\xe4T\xf6\xae\xd1\xea\x1f\x15\xc2?[\x87y\xd6\xfb8N?I\xc8?\x86Y\xc0\x0f\x05\xbb\x1d{Q\xb2\x9a\xb5X\x1c\xcb\xf0z\xbb\x0b\x933\xe7\x06\xc4l@\xcc\x06\xc4l@\xccw\x86W6 f\x03b6 f\x03b\xbe\xedsn@\xcc\x06\xc4l@\xcc\x06\xc4|gxe\x03b6 f\x03b6 \xe6\xdb>\xe7\x06\xc4l@\xcc\x06\xc4l@\xccw\x86W6 f\x03b6 f\x03b\xbe\xedsn@\xcc\x06\xc4l@\xcc\x06\xc4|gxe\x03b6 \xe6\x1d\n\xe1_\xe6\xed?R\x08\x7fx\xa4\xef\x0c\xc4\xfc\xfc\xc0~\x11\x10\xf3\xf3\x03\xfbE@\xcc\xcf\x0f\xec\x17\x011\xef\x05\xf6?\xe1F~\x11\x10\xf3\xf3g\xec\x17\x011\xff\x0b\x9b\xc7\x06\xc4\xfc\x14 \x86\xfc\xe7\x98\xff{ \x06B)\x96\xa1E\x16\xe6\x08\x86\xe7\x08L\xc4I\x12\xe29\x88\x81X\x14\xe3\t\x82\x82(\x8eb(\x18\xa1\t\x12\xc5Q\x86\xc6\t\x11"(d\xfd\x0f\xcf\xc2\xf0\xb3\'\xc0\x8f\xed\x03\x86\x14!\x16\xc6\x11\x9e\x821Q\x14yR\xa4i\x8a\x85yNd\x90\xf5\x13\x18\x18Ga\x18!\x18\x86GDR\x14P\x98#q\x8c\xa7a\x1a\xe1D\x8a\x12\x85\x9f\t\xc4\xfc\xa33\xc9\xdf\xfe\xa4\x01\x03J\xfc\x1d\xff\xb2\x00P\xf2\xdf\xf90\xdf\x1f\xd7\xf9\x13\x1fFdD\x0e\x119\x08\x12\x04\x8c\x10i\x04]\xd7\x19\x8cr8\t!\x08\xcf!\x08\xc1a,E#\x02\xb2~>B@\xf4s\x81\x104\x81\x90\xf4\xfa_\xf1K\xf5\xd8|\x98\x9f\xe8\xc3P$\xb9\x0e\x1b\x8b\xb2\x1c\x81\xe0\x0cA\x11\x14/\xae\xab\x89\xe2H\x82\xe6q\x98\x13\xd6\xc7\xa7D\x92\xc4H\x91\xa1(AD!\x9e\x11\xd6?zv\x88}\xb6\x87\xff\xdb_\xde\x87\xf9/\xd5\x92\x8f\xf90\xcf!\xfb\xb7>\xcc\xf7\xcf\xf3_\xea\xc3\xa0\xd0\x8f{Rg\xad\xb9\xc7{\x9d4\n\xc5\x82\xc3S\xd1\xef<\xee\xa1\x98\xa2R_I>\xa0\xa3\x9d\x96\xe8\xad\xe9\x8b\x19+\xcd\xado\xc8\x14e\xf2{\xc4\xce\xee\xaa3\x9d[a\xbfC\xa9\x82\xb8g\x17e\x90\xc9+\xa28R\x92\x17o6\x1c\xfc\x14\x10\xf3<\x14\xd6S\x01\x83\t\xf8\xa5\xd7h!\xdc\xee\xb7\x02\x99\xf6\xfb\xe3\x9d\xcc\xfc\xeb\xd4\x0e\x17\x85\x04\xec3\xc8\x0b\xca\x1eJ\x01&\x8c!\x80\xd8OG\xed\xd8:$\xd5\x1d\x05\xbe\xbbK\xcd\xe3\x94\xc2\xd3MW\xb2\x8c\xc3\xa3}\x05vW\x07Z\xce\\\x12\x08P\xf2fK\xd5O\x011k\x98\xeb.\xb5&\xf1\xcbT&Ax\xb9\xc10T\x90v\xa6<\xf4\x82\xd9\xf1\x10W\x80\xc5\x89\xbc\xd8H\xd6^\xe3\xd2e\xd0$\x83[\xce\xd2\x1eT\xb0\xf8(\xdbSg\x8c\xe5\xb0\x06c\xb8@\tU\xdf\x85wr\xc7Q\xad\x15\x8e ;\xb8\xfc\x8f\x9a\xef\xfed"\xe6\x19#\x04\x93\x04\x85\xd3/\xdd\xb8r\x05O%\xf9d\xe9V\x12\x05\xde\x9a\xeb\x0f@\x83s/}\x84\xc7\xae]t\t\xcd\xa5\xea8\xe8\\W\x8f\xfa\xb3\xa71\x98\xdeNf`\xe6\r\x9c\xf0\x03\xee\xe9\x9c~L\xc1su\x9c\xd3Gs\xe6\xad\xcb\xe9\xe1\x08\xdf\xacM\xd5\x87\x88\x98gB\x10\xf0Z\x19P\xd8K\x9b*|"R\x18]"\xf3\x02\x1a\x92K\xe1\x14\x9f\xba\xea\xe9\xd6\x0e\x16\x1d\xe3K\xe03\x8epk\xc3tj&\xf4\x90\xeb\xe1rI5\xabN\xf7\x81\xe4\xfa\xc7L=UHE\x10\xf7yO+\x80\x03\xd3KK\x13\xe0D\xbe\xa9m|\x8a\x88y\x86IR\x14\xb6\x16\xec/\x8b\xe5\x1ad\x18\x08-\xach\x9eU[\xc2v\x91c\xafG0\x1c\x05T\xce\x80\xb3T\xed\xf7\xcb\x9d\xb7mo\xd24\xa9FC>H\xf8\x08\n\xa5\x10\xc2\xadz\x1d\xfd|\xbdv\x83\xe3\x05\xdf\x9f\xa0nZ\xd7\x8c\xd7i\x88\xfcf\xdf\xedO\x111_\xdb\x1b\xb4\x96\x10\xeb\x02\xf8c\x98Jr\xc4w\x8df\x1c\x0en\x81\xa3\xfblbG\x0e\x14\xe8\xf9\xb1\x9ezs:\x9c{\ttj\xb4\xf1\x15\xf2^\x9f\xd2\x80D\x19\xa9\xcfG^_\xf8\xb5\xd4\xea\xf0\x18G\x1cC\x10\xeb\x82\xf1E\xa723\xbb\xd7E\x86y\x13\x15\xf9\x10\x11\xf3\x87\xd2\xfe_\x9a\xe3\x9a\xe7\x8b\xd6\x97\xdc>\x17\xe1\xb1=\xb41\x9c2\xbd\x8e\xc2\xc0\x8d\x85\xb1^\xb4\xc0h\xb6S\x19\x99\xca\xd9+\x07\x8e\x1d\xec\x0b+1!y\x06\xdc&\x8e\x92\xd4E%\x01\xbc\xa6\xf8 \xfb\\=]\xd3a\xe7\t\xc4\x9bL\xc4\xa7\x84\x98\xaf(\xd7\xdas]\xb4/-8O\xc2\xe8\t:<\xd73@\xf7\xf4\x11\xee\xefGJ\x8d8\x8b#\x14\xf48\'Lk\xdc<\xfc4\x81}\x9f\xa8\x1e\xff\xc8\x19]FuB\x99a\xed\x86\xd8\t\xa3\x9b\xc5\xa9>\xb3@,\xe6\xe1\x95\x92\x18|\xacP\xfa\xdd\xc9\xfc\x90\x10\xf3\xb5\x03!\xeb\xfa\xc7_;\xe23Fq\xec\xeb\xbc\xba\xf0\xb14\xa6`\xef\xdfB\xa0\xe6\x16\xafC\xcf\x87\xbcn\n~d\x88\x89\xf39u\x1dO\t\xe8\xbc\xfb\xbe\x1fA\x1ah\x8e\xaa\x07 \xf0I8\xaf\xebJ\x134\r\xc9$\r\xc0\x96\xea~\xb3\xde\xc40>\x05\xc4|e\xe6\x136\x81^{\xb9\x1f0\xd9Bu\x88\t\xda\xd9\xdd\x91G9\xe1\xe9\x8e\xc6\xc5d\x9e\x8f\xd4L\x0eY\xab#\x01J\xd1>\xfe\x98U\xc7\x00|\xb1\xef\x87\xd6\xa7\xa7\x1b`\xde\xb1Kxa)\x82\x84\x80K\xf8\xd0kv\xa7\xabfA<\xc0\x1f\xf5\x8e\xfck\x031\xcf3\x7f\xdd\xdfh\x02\x7f\xadR\xb9\xa8\xac\xf9@\x10\x8ae\xbat\x93h\xdc%\xa1\xbe\xc5g7\xc5\xac\x88\xaco\xb0G\xc8\xc5i"S\xe6\xa2N\x89\xe1\x9d\xcdu\x95J\xe0\xf1d\x86\x80f\xe9\xa7\xa5\xb3v\xde@\x96\xc5\xe9\xc6\xb2\x97A.\xfb;\xe3\xbf\xd9I\xf9S@\xcc\x1a&F\xc08\xba\xc6\xf9\xd2\xfd{\xaco\x85\xd4\xc2\xdcNL"\xfbb\xd8\x95|\x14\x90\xa9\x07I5\xb5+\xca\xef\x0e\xf5\xa1\xe7\xfa\xa4\xbb\xe8G\x7fF\xc3\x98\xd7\xfb\xe8\x86\x14\xb9nL\x99m\xe0b^w\xe3\x9e=7\xf7C\x93\x9cn\xd5\xac\x8c\xcb\xe9\xcd\xcc\xff\x14\x10\xf3\x95\x134\xb1\xee\x89\xaf9Q\x01\xb9\x1c\x05\'QIo\x0b\x073\t\xc8\x9e\xfaV5\x8fi\xb2\xcbmvrw\x85\xfd\x90\x1b=:\xc7\xb7CI\x0b\x11\xd1\x877\x90\xe8\xc4\x1a\xc7j\x01\x1dk\x9b\xb5y\xff\xc8\xd4\xe6|\xcf\xe3\xa3\x03\xafi\xca\xfe\xe8P\xfek\x031\xcfQ\xa4\xbe\xc6\xf0\x95O\xc0\x0cg|\x0c7\xd38\xacu\x9cB\tF]\x91\xe2Lh\xdd\x9e9yWB\xa2\xd1\xddL\xe2\xa15t}@D\xf7\x92;\x1d\xd3sF8\xbeJe\xf3\x92\x04Hz&8\x85\x82\xc5\xeaQ\xf3\xf8\xd2\xdf\xcc\x04x\xb7\xdc\xff\x10\x10\xf3\x0c\x93^\x0b\xb8\xf5@|)T!\xf2\xd0z\xa8\xb6\x16\x85\x87F.\x0b\xe0Lf\x0cp\x96AN\x13\xa5R:a\xf4aBI\x0e\xb9\xd6\xb4\xc2\xe4X%\xb7\x96\x86\x9eH&\x13|\x15\x10l\xce[S\xb5\xf3\xf9G\xa6D\xde\xdd\x9a\x85\xe3~\x14\xac7\x9b\xef~\n\x88y\xeep0\x8a#\xe4\xbaY\xbe\xf4\xdd.\t\xd0\xe9\xb8s\x8b\xcc\xdd\x8c\x15\x82\x81\xdd\xaf\xa547\xee\xf1|\x0c\x8f\xa6y\xd0\xf7\xf7[X\x1d\xd0\x8c>\xd8\x9d\x19\x1c\x8f\x97\x0b\xc8\x94x\x958\x17\xc7\xbe\xf4\x0ffb\xf9Y\xb7\xce\xa2\xafI\xaa\x8e\\\x0c\xdc\xe6\xde$\xb0>\x04\xc4\xfc^\xdb`0E\xbe\xecp\xc6\xa9\xa5\x87\xeb\x02f(\xbe\xdf\xd3\xf5\xe4\xc7\xf9\xbd\x9e+&\x08y\xeb\x90\xe8\xfe1\xbdX\x95\xd6\xf0\xc5\xb90R\x94\xeaR\x1a1\x1c\x07\xe2\xf1\xb3\x91\xb8\x8fS\x80v\xf6\x94H\x8en\xa7z\x88\xae\xac\xac>\x86\xb3\xff{\xed\xe56:\xebG,\x89\x02\x87\xee|\x02\xf7\xd9H\t\x0fLym/\xe9\xea<\x98\xc0(\x8d3d\x82.\x19\xb2\xe9\xe1\xca\xa6\xdb\xf8br6\xdb\x19\xa2\xae\x02\xb1qX\xeaV->\x9fgT\x91\x99#\x1a\xa9\x14\xa4\x19sf\xef\x91\xe1\x80\xe23\xdc9\xc9\xaf:\xfd\x17\x03b>.m(\x12_z\xcf;\x9f@\x0fyy\x819\x8b\x97\xa6U\xb7\x12\xc8k\x80\x97\xc73l\xfb\xba\xd6\x06is\x0c\x8d\x9ed|\xaa\xf5\x0e\xc3h\xc7\x1a7\n.\x92\x82\x12|\xca!w3\x98[\xbe\x9f\xe00\x03\xc5V\xf0\x01I\x01db\xa2.\x0f\x0e\x8b\x7f\x16 \xe6G\x8f\xba\xc4\x8b{kz\xe3\xd9\xed`u\xe3\xca\x9d_\xec\xddU\xc7\x02\xd7q\x022c0ZY\x13\xbcJB\xb3\xaa\x85\xba\x03\xa8\x02K\xd3\xda W\xce\x8f\xb3\xce\xdb\x1a.\xa0\x0cS\x9f[\xfa\x00\x146q\xe5\xae\xcd\x19\xc6r\x9c(\x82\xffL@\xcc\xf7~\x02\x85p\x14\xbf\xcb\xfaHqY\x8a_\x13\xb8X\xe7\x19\xab\xbd\x16>S;\xef\xb2\x938W>\x9dB\xe8\xb82\xd1\x03J\xf0\x87\x8d\x9b\xadJ}\xf0p\xbd\xc6\x97\xed\\\xcd\xdb\xfa\xe8\x95\x1dD\x15G\xc2f\xaaS\x88\x90;\x98\xcd\xf6;\xcc\xf0\x1e\xcc\x13\xcf\x02\xc4|\xcf\x130\xb1\x08\xbd;,\x13\x18\x92\x19\x8b\x8d\x9a\xdf\x1cql\xb7\x93k\xe2\xbc\x19\\\xce\x8a\x8e\xdd\x91s+~\xe8gl\x14\xa9\xe3\xa1\xd5\x06eU\xaf\'7\x80\xc5\x8d\xbe\x1fs\xc6\xe8\xfcX\x1fIDu\xaf%y\xf5\xd8S\x9a\xda\x999I\x0f\xca|\x16!\xe6C\xe6\x12\xdeqd\xcahZ\xcb?\xca\xbd|\x16!\xe6\xe3\xac,2\xb1\x9f$\xfd\xda9^9\x10\xbd\x9e;\xc8\x16})IO\xbbj\xe4t2$\x8e5V\xd76V\x80\x9e\xbe\x92@Y\x8f\rR\xc1\x04,a\x04w\x05\x90QM\x93f\x10qh\xb0\xbf\x10\x80v\x11\xf2\x1d\x95kg\x80\xb5\xf5\xdd\x83.\xf1,B\xcc\xf7\xf7\xa4(x\xf1\xaf\xbb\xb7\x19\xfa\xd0c\xf9D\x1b]\xeb\\2|\x8a&\xa9\xa2wL#\x9c\xb5\xf3\x06\x186F\xbebi*= \xf6\x85\x8bgb\xd3\x1d[\xceH]\x0e)\x92\x02\xb0\xc1\xe3.\xcei\xc6<\x0c\x98=\x0e\x9e\x89\xae\xf2\nc\xb5\x07\xad\xf9,B\xccG\x93J,\xb5*v\x8fh\xaf\xcfW\x12R\r0\x12\'\xf4B\xfa\x02\x1f\x93\xfc\xc6\x9b*\xc0\xba\x9a6\xbb\x93\xaf\x83\x0c\x94\xccR\xa7\xd3\xd7\x83\x87\xa5u\xc5\x8e`\rL\xf8\xa4^\x1c\x9e;q=\x88\xc8&\x9b\xb6[0\x075\x15R\x8eW\x01\xffbo\xbd=\x89\x10\xf3\x91&(\x18\xa7`\xe4\xae\x84\x1b\x19J\x11&\xae\xa9\xf6:zrN\xd89JW\xa8\x1b\xa0\x05\x8e\x80)3\xe2\xb6Q\xae\x80\xb0=\x03.\x956\x87\x15j3\xad\xb0kw;s\xdej\xc0\xa9\x8b\xe1\xb9\xb6\xa3\xd2i\xf1\x82\x97\x11\x02\xaau~\r\xfd\x8b\x84\x98\x0f_~\x13b\xfe\xe5G\xd5\xffs\x081\x7f\xa7\x81\xbc\xef\x19\xc7oB\xcc\x9b\x10\xf3&\xc4\xbc\t1oB\xcc\x9b\x10\xf3u\xd7\xf9&\xc4\xbc\t1oB\xcc\x9b\x10\xf3\x95\xc9+oB\xcc\xbb{z\x13b\xde\x84\x98/\xbb\xce7!\xe6M\x88y\x13b\xde\x84\x98\xafL^y\x13b\xde\x84\x987!\xe6M\x88\xf9\xb2\xeb|\x13b\xde\x84\x987!\xe6M\x88\xf9\xca\xe4\x957!\xe6M\x88y\x13b\xde\x84\x98/\xbb\xce7!\xe6M\x88y\x13b\xde\x84\x98\xafL^y\x13b\xde\x84\x987!\xe6M\x88\xf9\xb2\xeb|\x13b\xde\x84\x987!\xe6M\x88\xf9\xca\xe4\x957!\xe6M\x88y\x13b\xde\x84\x98/\xbb\xce\xbf\xc4B\xf8]Y\xf1?\xb2\x10\xfe`\xe5\xafL\x88y\xbd\xb0O"\xc4\xbc^\xd8\'\x11b^/\xec\x93\x081\x8f\t\xfb+\xbc\x91O"\xc4\xbc\xdeb\x9fD\x88\xf97\x04\x8f7!\xe6%\x84\x18\xf2\xbf\xf7\xfc\xcf\t1\x14\x81S\x08\x81\x908\xca\x93\x18\t"\x0c"0 A\xb2$A\x92\x18L\xf2\x1cA28\xca\xa24#\xd0\x1c\xc2B\x02\x83\xf2\x02\r/\x7f@\x9a&\x90\xdb\xc2\xff\x84\x10\xc31\x1c\xfc1\xb5\x9e\x83P\x02\x05q\x18c(\x84\xe2A\x88c\x11\x8a#P\x1c!Q\x1e\xe1i\x0c$1\x06\xe2\x11\x88 \x10\x16\xc49\x96gp\x8a\xa0_I\x88\xf9\xe7\x00\xf4o?\x19\xc0\x80\xc2\xff\xa0P\n\xa2H\x1cF\xff\x0c\x11\xf3\xf5\xf1:?A\xc4\x10\x18(\xf0\x82\x80\xf0(\x8f\xe38L\xf1\x02\xba\xbc0\xba\x98\t\xe4p\x92A)\x92\xe7\xb1\xc5\xba8\xc2\x93\xcc\xf29\x08\'h\x06\x83)\x04\xc6Y\x82\xa3o3j\x7f\x8b\x88A8\x14\x02\t\x98\'\tB@h\x96\xc3\xe9\xdb\xd4aH `\x12fA\x14E\xc1\xdb\x18X\x1cc\x19\x81\xa1\x91e\x97`\x94\x01o\x83PX\x16\x07i\x96\xfe\xf6\x9b#\xfeF\xc4\xdc<\x12\xa2Y\x08bH\x02\xc7y\x08c\x91\xdb\xc8%N\x00\x17\xe3\xa2,\xc4\x138\x0b\xa1\x0c\x0b"<\xbb\xf8#\x8e\xd1\xf8bN\x08\xc4\xe0\xc5\xa0\xcb)\xc3\x08\xf6\xdb\xdf\x1c\x11\xf3/\x13=\xee\x101\xdf\xd8c\xc0\xb1G\x8dc\x0b\xed\x1ayr\xa96\xfa5i\xdc>\x80\xc7\x8f1 \xea!\x18\xd5\n\x9aCP\x9fBO\x00CW_z\x96m\x9f2\xea\x99\xceO\x13m\xb6g\x15\xae\x8b`\xc2\x96\x8f\xa7m*\xfe\xf8\xfcZ\xebS\xf1\xfb\xb8\x8e%\r\xf6)\x8b\xc1\x9a\x8dq\xb1X\xd7\xf1\xc1rRqy-Ka\x97\xefW\x02\xef\xb2O\xe0}\x1f\xfa\xe6E\xdbK\x8b\x99x\x93\x1e[\x9e\xddpl\\,\xff_\xe2vq\xd1r\xac\x9e\xf3ZN\xe3L\x9e\xb3\xbc\xa2\x14[P\n\xd9\xbc\x92h+g$\x9a?\xd1&mst\xcb\xd2\xcb\xf7KV\xceK\xf40\xd0\xee\x9f|\r\xf3\xf1\xb1\xe5k\xdau\xea3\xb7b\xae\n\xdd\xd6\xfc\xaf\xff\xf6,\xd9\xac\xf4\xefk\xdd\xd2\x8cG\xf3\xf6>\xe7\x15\xb5\xa0\x8eA\xa3\xf5\x8b\x96C\xb4\xb6\xca\x1f\x9af\xf5`\xd5qc\xd5I\x03\x1d\xe3\x83\xd9\xa7%\xdf\x9b0\xd5\xdf\xc6\x96$\xc8vv\x91[\xaf\xe7\xda[\xba\xd5X\x1d?]\xa4@\xe4\xf5\x06\xc6$\x97\xe3\x0bI\x91\xf83\xcaU\x80F\xd8\xf0\xfe`w-\x1f\x1c\xa6\xf51\xea\x0ce\xe4\xe9\x030\xd5V\xeas\xb5\x12\x05\x1b!\x17\x0b\xceW\x95\xb5\xedK\n\xc9A\x12\xca0\xfd\x15\x80r\x06\x1c\x8f\x1b\x0e\xad\xe8\xc0\x1c\xfc}\x9b\xb6\x99\xee\x9fI\xab=\xff\x98\x932h\xb3|\xd6\x90\x10L}\x01\x8b\xbdn\x8e\xc5p)l\x8f\xa7\x08\xde\x8e)r\\Z\x8d\xf0\x94\xf8\xe9\x9c\xfa\xfc\x1cs4\x92\xed\xe5\x9a\xadP\xc5\xca\xdb\x9d\n/g\x03\x86\xae\xc9\x84\x95\xc9\xa1\x1eR\x91\xbci+\x13N\x82\xf4\xa6\x1b\xd5\x9a\x81\x82\xeaRg\x96T\xb2\x05\xcf\x19g\xe0l\x05\x8d\x8b\x03\xc9>m\xec\xedi\xdc\xf0V\x94V\x01\xb3\x1e\x08m\x14\xd4u\xe3\rF\xa0 \x15h3\x8bM\x16\xbb\xdb\xfc\xd0\xeal\xbe\x85>\xd6;J\xbcV\x9a\xa8\xbe\xe7\xcc\x82\x0e"\x93A\xd7\xdd\xc97\xf6\xb1\x96w\xfa\xaeF\xd1\xa2\xeb\xea\xac\xb3\xaf\x06\xc7\xda\'nX\x9b\xa8`J\x05\xd2\x85\xf3\x8el\xf0\xe4\xe2\xf7\x94\xd0\x1d\x0e\xfbI&\xfa\xab\x85\xf4X<\xaee\x7f\xbd\r\x93\x8d\xc5\xdbX(\xf11\xb0\xf4\x8c\x7f\x15\x94t\x1b\xef\xf3\xa7\xa0\xa4\xaf\x9f\xed>\x17\x94D\xfc\x92=1\x9dq\xda\xa7\t\xca7hd\x07\xd4F%\x88\x97\xbe\t\xcb\xc1\x92\xe1\xca\xdf\xb4\\+\x89!/;k\x84O\xe2\xa0^\xeb`l\x96q\xde^\r5\xbfb%\xbd\xddz$)[\xbd\x93\xa0g\x99T;\xe1\xc4\x9d\x1f\x9c\xf0\xf5,N\xd2\xad4"\x17\x95\x18y7\xda\x0f\x9b\x84$J\xcb\xdd\xae7\xe5\\\x84\xbdc#h\xe6,\xfa\xe4\x14\x96\xf3!l\xeb\xb2p\x12\x9f\xe0\xd3\xf3\xa1\x9a\xa4a\xdd\x8a\xed\x91\xeb\x15\x9f.h\xf5\xa2\xd4E\xdf\xe6\xd0\x8e-\xf1\xcbN^\xb3\x17G$\xc7\xc3\xf4\xab\x19\xad/\xc6$\xa1\xc8? h\xb1\xe4\x1d+\xa5S\xb6\xae\xdd\x93F\xa5\x8f\x84\x9c\xac\x862\xe7!\xa5\x9c\x92q\xd3)@\xcc9\xb4%WQk\\5\x8c\xb7[\x99)\xb4\xe2\n\x17W\xbc>\x87\x9b3\x98^h\x03ZAD\xeeER\xd0T\x96\xc5\xaf\xa8\xdd\xd9|p\xb4\xe8\xb30I7\x8d\x08\x82\x83\xe4\xe2qw\x98$M\xdf\xa7{\xe4\x88J\t\\\x1a<\x1b\xa2\xda\xb9e`\xf2(\xdb\x96\xc90bG\x08\xe3\x88\xed\x04\xef\xb4\x977fy:\xc1\xfe\xf5\xdaL\x82\xbe\x13\xb3\x18\xdf\x90W\x06v\x8e\x01\xb6\xbfn\x00\x148\xf2\x13\xec\x19\xd4\xd7\x02O<\x0b\x93\xb4\xec"\xb8D<\x08D\xb0{\x16K\xea\x88&\x17{k\xa7\xe9\xe4Y\x11\xb8\xccG\xb8\x84mMJ-\x18*\xad\xcf-\xed\x87\x89%\x13\xc7\x98\xaej\x95U.{\xe5,n\x0f\x1b+\x9di3 Nt,\xd7\x07R\x19Th\xbf\xa3\xda\x96/\n\xf6\xc1\xd9\xcc\xcf\xc2$\xddd"\x04\tB\x18u\xe7\x13l\xdd\x9bB\xc3\xd7\x12\x87G\xcb\t\xc4\xf2S\x91\x8a6\x9a(\xb4\x7f\x18\x15\xd9\x98\xa3\x14\xbf(QvD5\x1d\xb8\x94y\x86\xd8\xd8U\xdf\xfa|\xd8\xd8\xf6\xc8\xe1\xa0s^]\xa3u\xa7G\x84\xd9\x90g`\x90\xcf\xc3\x83c\x1a\x9f\x85I\xbaE\xb7\xa5^_v\x87\xbc\x1b\xd3H\xca\xa3;o\xab\x96\x0eZ\xa19\r\xc3\xfa\x9a\x85Jx@i\xa8\xd0Zt\xc4w\xb5{\xb8\n\x05\x97{USSF_X\xcaV\xce\xaez\x9b(C|\x89\xdc\xf5u)@l\xfa\x88\xf3\xfe|IOv\x19\xed\xa1\x07\x01B\xcf\xc2$\xdd\xac\x89#K\xdb\x0faw\xb9\x8a\x96\xc9\xa2\xaav\x80$\xd0\x18\xbd\xbd\xa4\x8aa\x1eYw\xa3S\xccN\xaf\xd1nt7\x121a<>\x96\x08\xb8\x966W?(\xca\xbd\x02L\xc6n\x1a\x97\xf2\x1e\x89`\x0cu\xc8\x8c\xe8\xce\x9a\xe31P!\xc4\xed\xe1\xd1\xb9\xe9O\xe2$}\xf8&I\xe2\x14~wfsd>2\xfdZ\xca\xfb=\x98\x98\x9d\xcft\xa32\x9b\x9a\r\x9b\xeb\r\xab1\tq\x9d\xd16\r\xfb-\xba\x89\xaf`5\xb7\xeb\\i\xf8k\x0b\xceG\x9bv*\xa0\xc8\xe8\x93!\x8cLHF\xcb+Hf`]\x86\x07\tf\xcf\xc2$\xfd\xf1\xb2\xe2w`\x08>rTl\xe8zw_+N\xd4$\x07\xc2\x9d}\x8f\x8e%\xe44\x02\xfb\xad\x16\xac\xcf9>\xa7`\xc4\xda2\xb0\xb6\x944\xd6`@ \x8e\x05+\x1dm\x08N\xb2dL\x8c\xfdt\x14\x8b\x93\xb4\xb8\xc4\xed\xae\x0f&\xf0{\xbc\x864\xa6\x87\x98\xf5\xe3\xa5^C\xec\xe0\x98\xb4\x061\xa5\x84\x82\xd5 \xdb\x1c7\xdb\xc6\xdbJ\xeep\\]\x97\xea\xa7\x12\xeb\xd8\x13\xf6\x82\x9bgp\xce\xf4\x14\xec\x98\xb3\xbeR`}K\'\xd3\xe1dN\xd7p\xe4\xa6\xbe\xfae\xd2\x7f1\'\xe9vX(\x14\xa70\xe8~x\xb2<8\x94\x88k\xba\x94\x82q\xbb6\xb2\x10%\x04\xc5\xac\x82\xed\xc6\x0bw\xaa\x03\xb2\x82\x9e\x17lp\x81\xaf(4\xaa\xb9\xe1\xb1\r\xad\\\xb5\xabl\x08H!\x81\xab}3\xb6\xda\xe6\x9a\xd0\xe6\xbe\xde\x88\xa3\x96\xe9!\xf8`V~\x16\'\xe9#+C8\x81\xa3\xd8]\x84\x93T\xb6\x85U\x9c?\xf5\xe4\x91\xe1\xe9\x8c7\xb4\x1c\xa6\xe4\xb1XW\xeb\x8bbk@\x9e\xf1T\xadU*\xa8\n\xa3]V\x08\xc4\x11\xa0):\x10\xa9\\\x8f\xf1\xa9\xdf\xda\x07\xee* \x81|\xc8\xcbsu\x10{js\xfaZ>\xf1,N\xd2G\x9a@\t\x12C\xefy\xa1-\x00\x8d\x02\xb9/\x96R\x91\x89\x99:5\xf9\xdd\x8a\t\xd4\xeb\xd6Y\x82!\x81\x85\x98\xcb\x820\xc6\xbb++\xb5V\xe2a\xcat<-*\x80v\xc3\xe3\xc8\xb9\xc0v\x9fc\xfb#\xc9G\xedH\x8dE\xb7\xdb\xe8;\xef\xf2 \x02\xf2Y\x9c\xa4\x9b\xcc\xa5O 0\x18\xb9\xe3$\xed\xa3\xac\xe0\x0e\xeez\xac\x0bs\xc7\xba\x8ai\x9b\x0cy\\q\x95\xed\xee\x0b\xfe\xd4\xb0\xb2\xbbq\x0e\x0c\xbe&E\x89N\xcc\xf1\x82\x83\x80\xe2\x88\x8dR\x85(\xea\xe3#PE\xb8x\xa84\x1d\x83\xc9s\xac\xfb\xfb\xb5\x0f=\xc8\x84y\x16\'\xe9\xe6\xfaKS\x84\xdf.b\xefxz#5]\xe7fG$\xdb\xd9\x1f\x92\xf9\x84\x98W\x06b\x96\x93\xc2\x86V\xa1\x1c\xc1\x93!\xee\xd5\xb0\x13y<\'\xca\xd9ZO\x04\x8f\xa3L\x84\xdbhQ\xa8d\xb1\x1a\xcc\xab*K\x82\xee#\xa2\xe8\xee\xa0r\xde\x9d\x1f\xb4\xe6\xb38I\xdf\x0f\xed\xd2\xe5\x92\xf7\xe9\xd0\xdf\x84%P\x0e\x1at)\x97(\x9a#+\x13\x1f\xe3\xd8\x85\xcb\r|\xac\x92e+gKP\xb1\xb3\x1b\xd1\xe8d\xcb\xea\xe6\x9a\x17R\xcf\x9e5n\x95\xc1\xb3<\xee\xf6\x16\x08 \xf6I\x82\x83<\xf3K\x07$41\xfbZ\xb0\x94gq\x92n\xbb\xb8DP\xf2\xf6\xc6\xd3\x1fw\xd1\xf0N\'\xf7\xbc\x8f\x9d\xfd\x9cyL\xd2\xfbJ{\x96\xb1\xdcqxC\xa9.~v\xeeMo[\xc4}\x96\xa3\xe0\x00q\xb3\xb2r\'\xcd\xf4\x1bl\xaa\x13q)\xf8\x8f\xf2\rP\xb1\x99\xd5\xd3\x9a\xba\xc2>\t\x87Y\xfb`\x9ex\x16\'\xe9\xa3v"og\x0e\xb9\x8bp\x0e|(\xbc:\x86\xd5\x95\x17\xfb\x91zD\nW\xc6=?\xadL\xe6\xba\xda\xe1\x91$)\xfb\x86\x8b\xc3\x8e\xbe\xe2\xe8\x01\n]\xed\xba\xa6\xd3@(\x8c\xed\xecj\x1bM\xde\x03A\x13\xf2\x99k\x08\xa7\x92\xa9\xa8\xf9zV>\x89\x93\xf4qk\x83\xe3\xf8Ra\xdeYs\x84\xd5M\x84v$Sl\xebl\x1f\xf4\xbeE\xd7[\xf2\xeaa\x97\x19g\x8e\x80\x0c\xab\xf19Y"\xb6ip\xc7+\xd4\xef\xce;oj\xea`\xda\x02.\xda\x15\x87\xc8\xda\xf1\x92w\x16\x043A/\x14\xe4\xb4\xde\x89\xab\x1e\x04\xde=\x8b\x93\xf4\xbd\x17\x07A\x14\xbf\xef\xde\xb4\x0c\x1e\xd0\xfez\xd5\x00\xb2\xaa\x87$\xe3N\x0c&\xf6l4*\x832\x92{\x10\xd9\xebI+G\xeb\xfd\xc8[\xa7#BU[\xd0C\x81\xe24\x8d`\x02\xf7\x82\xb16\xf3$\x14\xcf\xde\x1a\xc5\xd7#@\xc4\xeb6\xd4\xbf\x16"\xedY\x9c\xa4\x8fV\x1f\x05a\xe2\x9e\xa9\x05\xb7\xfb\xb8\xdc\xc1,\x1fx%\xe3W\xc89;\xf6I?\x83TA\x86\x05\xd2_\x99-\xbe\xf5\x11.Z\xfe\x8d\xc6ay\x99X\x18\xc97 A7\xa6\xe5\x9dy\xe3\x18\x06\x91"-\x89\x91\xb8\xa2\x06\xa1\x93\xe3\x06{\xd4%\x9e\x85I\xfaPy\xbb\x9b\x82\xf0;\x99,3Q\xd0\x1cL\x1a\xae\xb12\x88\xb61\xc6`D\x92U\xca\x81*\x89~\xd8"K\xc0\xc1"\xc1\x8e\x1c\xfd\xa4S\xc9&\xca\xa0\xb5\x080\x95\x8a\xf5\xa80_\x03f\xd7\x96\x9a\xa5\xe0\xce\x90\r\'q\xdfo\xdar\xcd=&\xf3Y\x98\xa4\x9bLp\xd91h\t\x19\x7f\x94\t\x058\xc5\x17~\x1a\xd5\xdc\xc8\xb4\xe6\xcc\x02-P;\xf2\xd2\x08\x0f2\xea\x94\xf3\xe8\xb4\xf2q\xa6\r\xdd3\x9a\x04Ui\xa3\x9f\xd1\xe4\x10C\x9b\xfa\n\x96M%\xb8\x94]\xc5\xa9\xcf\x8cU\xean%l\x80\xd4#\xf1\xe0\xbd\xcd\xb30I7\xcf\x87)\x8a\xba)\xfd\xa3L^\xa2\xe8\xea\xc8]\x8f\x80 \x13\xc2\xed\x97\xa3\r\x83\x87V\x07c\x1b\x1b\x97\xfe0\xf0p\xc4@E,\xf7\x82\x88\xda]\xd3A\xc8\xfe\xb4>\x03}\xba>\x88\x8d\xb7\xd9"lT{\xae\xb0\xf5\xe7]q\xad\x8a,\xd9\xb9\xbb\x07\x0f\xed\xb30I\x1fq\x1c\xbb\xddM\xdf\xdfL\x1fp\xeaP\x1c\x0e\xdd\x85u\\\xc4\x02\xb5\xd6\x99S\xbbD\xaf\xc0V\xaf\x02\x05q:\xcaf\x04\xaeqE\xae\xb06\xa2\x99\xf19~\xd0R8F1\xeb\xec\xd9Y\x1f;\xb8\x0c\xe8\xf9\xf5z5\xf4i\xbc\x0e\xbe\xdb\x93\x0f\xc6\xf1ga\x92>\xe28\x85b(\x02\xdf5\xa9\xc1\x94^\xd8\xf3t\xd8\xd2\x0eA.]MB\x15\xf0D\x9a\xf1\xb0\xbb\xd2\xe7\x1d~\x8e\x8e\xf0\xe5\x04\xe1;{\x0e\xe9\xbe\x88`\xf4R:\xdb-\xe4\xa2\x8e\x7f\x9eH<\t\x8b)\x91\x95\xac\x07\r\x84\xac\xc2\x1d\xe0\xc9\xd7\xcd\xd7\x8a\xe3\xcf\xc2$}D8\x08\x81\xc9\x9f\x00\x84\xf8\xbc[\xf1hw*\x01\x827\xdb(\xd9*\x9a\x97\xae\x029)\x94m\xedn\x89Q*\xfa}\x8a:\x04\x12\xb8K\xc4[\xc9\xf1\x8a\xd0\xc6\x93\xad\x93\x04!\x8a\x02\xb0\xbap^Q\xd41\xb0\xdeSf\x96\xd6\x81\xaf\x9f\x1f\xbc\xe5{\x16\'\xe9\xe3\xb0 \x14\xf2\xb37\xde\x9a\xdd\x05\xb4V\x12\x0f\x04\'E\xae\xf6\x1d.z\x15\xa4\x1b\x9d-\xf8i\x87\x01\x1cm\xcd\xc0\x05\xe9\xad$\xb1\xab\x1aT!\xc5\xec\xb5r\x05\x19\x06\x07n\xaf\x90gdT\x14\x08\xa48\xac\xf7\xc3\x81\xe7|\xd7\xad\xda\x99}\xcc\xf5\x9f\xc5I\xfap}\nE\x96\xbc|\xcf\xf1\x8e.\xba\x7fj\xd5\x15"\x07p\xcaD\x80=(K\x9f\x12QUJ\xed\xd6\x10\x04)\xdd\xc5\xdb\xcd\x95\xac\x19\x8c7j\xf0\xe92(\xd8\x9eH\xc7pK:l\x0b\x8aB\x9d\xe7\xf8\xca6\xc2.\xe3\xbc(\xb9\xe8\x1b\xed\xc1\xee\xedY\x9c\xa4\x8f@~+\xc81\xf4\xfe\xca\xc1\x86\x99\xb0\xb6\x17\xf7\xb8n\x0c8,\xd7\xf3\x91E\x18B.\xf5,#7$\x10\xa1~\x13\xc7:75\xf0|%\xaa\xde_5)\xa2\xacc\xa5:\xb8\x16 \x8e\x92^\xf2z$[|8\xab\xe9Zn6,\xb1\xf9\xd5\xa1\xfd{s\x92n\x87\x05\\\xb6\x10\xbdG@"\xd4\xd6[\x17$\x82sW\xb2\x91\xe9\x8c\xf4\x15\xf5b\x99k\x90\x96\xeb\xe2\x88\xd9\x95\xc1^\n6\x9a/}h\x06\x9e\x07\x1f;@\xbd\xcc\xbd\x1dcN\x9a\x8f}\xa2\x1aS\xea\xce\xc5|^:\x86\xa3\x90]\xc6\x0b\xc10\xff\x1a&\xe9\xe3-\xc3\xdfb\x92\xfe\xaf\xff\xe7[\xd2\xa6\xd9\xc7c\x15\xdf\x1fX\xfa\xc6\xe6\xa4\xc2y?\x1eB\xdaR?\x1e\xba1oO\xca\x16\x87]\xfb\xdd\x04y\xd4\xfd\xef!:\\\xb2\xf4\xe3\xca\x10\x05\xc9\x1f\x1f\xec\xbb\x8f\x0f\x11$\xf1\xf1\xc6\xddO\xa0K\xe3\xaf\x1e\xf3\r<\xbd\xf4\x11\xbdN\xcc\xfb\xc7r\xff\xca\x035\x059\xfc\xfe9\xe3_\xff\xe8\x04\xae\x0fq#,\xaf-\xd7\xbf\xf9\xd1\xb6\x98\xd6\xd9\xeda\xf4*\x9c3WG4\xfb\x12DM\xadf~\xed\xe9\x87\xbdjp\x96nW\x97\xab\xcb\x0b\x95%\xa6\x9e\xe9\xd5Z\xe8\xef{\xa7\xa0\xb0\xe0\xc0`\x9a\xe7"\x9a\x90b\xa1\xb7\x95l\x0f\x83\xf4\xb2\x96B\'\xed\xd2\xea\xf6;:z\xb5mF\xc4p\xb7\xb6#\x8e\xe7\x10\x92a\xbdI\xa8\x8d\xf9G^\xd2_\xc7T=\xbeg\xff\x8c\xac?^\xf9\xcf0U\xff\x9f!X\xf7\xbf/\xf3\xed\xe3!\xa4\xc3\xf6\xe3\xe1\xa7\xdfo\xc2_\x06V\xfd\xc5=x\xe0g/\xe7\x16\x8c<\xaa\xff\xcd\xcf\x0e\x10yI\xc8V\xed\xc3?\x1e\xd2\xbb\x83`%\xe7,\xbad\xff;\xa9\x8b\xc5A~\xf9\xca\xbfy\xa6\xcd\x87\xf7\xf5\xc7s\x7f\xf4\xef\xbcC+\xc0Q\xe7$L\x9f+\xf0\xb7{\xfb\xe3\xa9B\x1f\xae\xab\xdf~5\x97\\~\xff\xb0 8\xfc\xec\xbb\x10\x0bKD\xf7\xff\xfc\xb3\xef\xbcS\xf4?\xecR\xec-\xb9\xd0\x1b\x7f\xeb^\xb7\xc7 \xff\xb8C\xff\xf7\xff\xf1\x11\x90\xbact{*\xec\xdb\xb7\xdb\x07\xde\x14\xb7_\xfc:\xd9\x7f\x0e\xc5\xedo\xc4\xc7z#\xc7\xde[\xfa\xf5\xb7\xf4Mq{S\xdc\xde\x14\xb77\xc5\xedMq{S\xdc\xde\x14\xb7/\xbc\xce7\xc5\xedMq{S\xdc\xde\x14\xb7\xafLG{S\xdc\xde\x14\xb77\xc5\xedMq\xfb\xb2\xeb|S\xdc\xde\x14\xb77\xc5\xedMq\xfb\xcat\xb47\xc5\xedMq{S\xdc\xde\x14\xb7/\xbb\xce7\xc5\xedMq{S\xdc\xde\x14\xb7\xafLG{S\xdc\xde\x14\xb77\xc5\xedMq\xfb\xb2\xeb|S\xdc\xde\x14\xb77\xc5\xedMq\xfb\xcat\xb47\xc5\xedMq{S\xdc\xde\x14\xb7/\xbb\xce7\xc5\xedMq{S\xdc\xde\x14\xb7\xafLG{S\xdc\xde\x14\xb7GxE\xbf\xebz\xffG^\xd1\x1f\x8a\x90\xafLq{\xbd\xb0O\xa2\xb8\xbd^\xd8\'Q\xdc^/\xec\x93(n\x8f\t\xfb+L\xb0O\xa2\xb8\xbd\xdeb\x9fDq\xfb7\x04\x8f7\xc5\xed%\x147\xea\xbf\xf7\xfc\xcf)n$A\xa24\x84\xdc&_R(%\x10\xcb?(D\xe1\x14\x88\xb0\x02\x0bb\x14KC8\x88 \x04\xcd\xb24Ib\x18\xcf\xf1,Ms4\xc3\x10\xb7\xcf\xfc9\x9a\x85 !\x92d\x97\xaf\xa6\x11\x86C \x86\xe7i\x9c\xe4\x19\x84\xe28\x98&9\x18\x839\x08\xa4X\x16a?Fa\x91<\r\xc1<\x0b\x83\xc2\xc7\xecf\x14{%\xc5\xed\x9fSo\xbf\xfdl<\x0c\xf5\x0f\x12#\t\x1c!\x10\xe4\xcf(n_\x1f\x81\xf7\x13\x8a\x1b\x8c\x130Nb\x02\x8a`4Gq<\xb1\xd8\x85bh\x12^\x0c"\xf0\xb7\xa1\x92\xc1\xd3\x10V7\x997\xc44u\x0f\xd1S\xa3\xa8R\x9dH4\xa6\x9e\xa0U\xddT\xd6S\xad\xd1\xea\x1aS\xf8=\x14\xa01S\xa3\xbb\xbc8\xae\xa2\xd3\x89U\xa0\x06\x0cD\xb7\x8ee\xbeZ\xd7\x03\xeb\xab\xf1.\xd9p.}DY\x8d\xbc\xacVn\xac\xd0\xb3M?8\xdc\xf6i\x08\xab\xe5\xd0\x82\x04\xb4\xb4\xd3\xf7\xe3\xbb%m\x80\xddy\xad\x06\x823\x8b#\xcbt\tO\xb9v\x03X\x0e\x06\x9b\x07\xfc\x90\x11\xfa54j\x02\xb6\xcdz\xb5=\xf7bcZ%\x05\xa4zr\xed\xe0p.\xf4\xeb\xb9\\qT:`)\x14\n]\xc9\xc9\x92\xfd\xb5\xf2\xd5\xd3\x10V7\xd7\x07!j\xe9\x7f\xef\x90\x1dQ9l\xf6\x9d\'\xf1)0\t$\x15q\x85\xd1\x9f\x08\x99\x0eQ9\xf5\xd7\xc4\x11\\\xc5\xfd\xd9)wk%\'\xf5\x9e\xf7\xd7\xdean\xa1\xad\x91\x91\xf1\tU\xd5\x08\xc3\xa8C\xb5fz:f:O\xc9\xd7\xc7\xb3\xf9(p\xf1Y\x04\xab\xc5%\x90\xc5\xef\x11\xe8~\xde3\xd4\xfb\xb1\xaf\xf0H\xbfJ\x14rJX\x1c\xcb\xa1\xd5\xfa\xbam\x0e\xe5Vmu\x97\xc8\x9bk2\x01\xab9=3\xf3Q\xbdl)\xb2d<\x9e\xa5\xbbS^\x99\x1d\x90\xb0\x18;\xa4^\x19c\x94\xa9V\xde\xa9\x98\x8e\xcc\xa3c\xad\x9fE\xb0\xfa\xc3\x1d\xcd\xef\xe6=\x83\xb1[\x12l\x08\xb0\x9chK\x8e\x88^\x97\x1fD$\x85f\xa7;\x0e\xa4h,n\x06\xa6<\x1f\xc5F\x1c\xf0\xde\xd9\x11.\xb6$\xee:\x10\x99\x84\x80\xe4\x08\xf4!\xae\x11\xd6&_{\x14\x9c\x9f\xd5\x16\xa2\xad\xeck\xb9\xc4\xd3\x08V\x8bK \xd4-\xd1\x90w\x85p\xdcpE\x10;\xd0q\x1aIxE\xeey\xa9\x06\x18\x9c\xf7\xac\xf1\xc2X\r8\x10g\x97\xb6\xb4\x9eZ\xfb\x1b\xb3+\x1a1\xbaB\xa1\xc23\xd7T\xda\xf3\x80\x8a"\x17\xc4\x10j\x9f\xa2\xfamc\x9c\xe8\xcb\xd5\x18\x0c\xeeA\xca\xc3\xd3\x08V\x8bL\x1c\x051\n\xc1\xee\x0f\x8b\xe5\x893\xaf\x9a\x1d}\tC\xb4rgJ\xcf\xcf\tD\xaf{\xc1\x0b\xcek\xd0\x16g\xea\xd22\x88}\xe1\x83\x83\xd4\xec\x0ck\xd2t?\xe76B\x14m9\xe4p\x90\xc5\x8e^\x9f\x945\xcf\xc7\x05\x19\'q\xab\x12\x9fE\xb0\xbaeC\x1c\x87\x976\xf7\xce\xf5\x11+9\xc7~\xb4\xf2\x10i\xa5\xa6\x0c\x14\xb6F\xa9\xed)\xb0]Z\xe0"?\xd9|\x98f\x07\xc4Bp\xf6\xdaCi\xb4\xa7\xed\x139Vt\x186+=\xc2\xc3\xe00\\.A\x07G\xf5E\x01/\xfaaX7\x8e\xfb k\xf5i\x04\xab\x8fJ\x15\x82\x97\x98{whY!fK\x04\x9b=i\xc2\xbc]\x902\xe69=zG\x10O\x89\xd8\x9e\xce\xa4c\xb9\xd7\x98\xc8*s+\xe5\xd8t \xd9Cf\xf1\xd0\xd8(\xa6\x89\xc8j\x9e\xe4\x18s\xe1pI\xdc\x97\xe0z\xe0\\\xe1\xb4*\xc6/\xe6\xfa\xcf"X\xddj\n\x8a\xa4(\xf4\x1eVM\xedrsV\x8c\xb1^\xaf\x07\xc70\xc1\x1a\x9e\x8f\x81\xb5\xca\xfbI\x93lV\x18\xcf\xae\xa9@\xe96\xf7\xba\xe8\xb2\xab4\xc4]K\'\x87\x06Nr\xa8\xa4\xe3h\xe0\x17\xc2\xa8\x0c\xb6r\xb6VJi\xf6\x08O\x84\x17>\x98\'\x9eF\xb0\xba\xe5\x89\xa5\xbc\x84\x91\xfb\xd2\xe9r\xb90n\xb4B;\x94\x95i~\xf6\x93\xb0\x13\xeaf\x16\xe1Y9)\xe8\xfe\x00\x14\x1c\xd9I\xe6\xd0\xe4\x0cf6\np\x0c\x1c HM\xc5\xcfs\x14\xaa]\x81\x1e\xe5\xf5\x0eT\x07\x8b*t\x9e$\'c\x1c\xad\x07}\xe2i\x04\xab\xefm\r\x8e\xc0\xe8\x9dLog3\x19w6\x85\xfaB\x1e\xafKV\xe3\x0f\xd74<\xae\xb3NW\xaf\xc0\xfe\xe0\xaf\xbc\xbc\xec7^\xea\xe0\xfeH_\x1d\xbfmv\xbb\x11\x16s3\xcd\xd8\xf6\xb0\xeb:\xf3R\x87\x9edS\xf1\x1e\xe0(\x8b\x82c\xe2\xc1\x8b\x9b\xa7\x11\xac\x16k\x928\x06\x81$x\'\x13g\xe5\xa0n-\x89=5\x17\xc6\x87\xe1\xa2(\x08\xf3BI\xe2\xe9:]\x9ajU{\x19+\t\xe7t\r\xf2\xb4\x9f\xf7\x1b\xc8\x16\xbc\x86\x9a\xf605\x89\xb39\xf8\xb3\x7f\xe6s\'\r\xa8\xa4\xdd\x88\x9d\xc9b\xb5\xa0\xfd*_\xfd\xcd\tV7\x9f \x96\x98\x0b\xddS\xdd(\x8d\xacC{\xda\x95\xf2\x86H\x00\x12\x1b\xed\x1c\xf3\xd5,\x00\xd1v%\x1dD\xbe]#\x83\x86\x1cVa\x8baA}\x8a\x84\xeb\x1c\xac\xf5\x16\x89\xc2\x80njw\xec\x8a`*\x90\xfa\x1ch\xd3\xde0 j\xed\xd6\xf2\x837\x99OCX\xdd\xf2\xc4\x12\x10\t\xf0\x1ew\xb6\xb3\xd5\xec\xcc\xafI\xd8\'\xcc\x1a\x87]y<\x12\xb9\x19\x81\xe19t\xa9K\xc3!\x17q\x0e\x02\xa1I\xdb\x0e?\xaa\xfc\x10\x85>9\x15\xd9\xf2\x8f%\x04\'\xefR$\x87.\xc7\xa5\xeb \\\xdd\xcb \xb5\x9dq\xbd2\x8f\xc9|\x1a\xc2\xeaC&\x84.)\xf1\xce\x9a[\xf6\x14\x9d\x00\x04\xdb\x94\x96\x80\rg\xe5\xea{{Y8\x11\xb9\xc5\x08\xab\xf2\xe2\xac\xb0\x16\x98k\xd1\x05\x8f\xe4\xe6*3\xf6\xdaQ\xd2\x0blO\xa2E\x8c\xab\xecrU\xa9P\xf2\xd9a\xf1\\\x9f\xe0\x8e2\xeb\x88*\xfb\xa0\xccg!\xac0\xf0\x1f \x01\x13\xd8O\xd2\xd5t\xdd\xaf%Z\xdb\x8c*/#\xe0H2\x07\x9a?\xb3d\x91h}\xe7\x14~\xdc\x18\xa0x\x06\x96Nx\xd3\x83\xfb\x12\xb9\x0e\xf8*n;\x81\xca\tiV\xb5\xcde\xbfZ3kY6\xc8\x92\x96\x8c^u\x82\x96Y=xf\x9fF\xb0\xba\xc5\xf1\xa5\xd1\'\xa0{\xf48\x05x2\xa7\x16)\x0bU\xbb[V\x020\x9f\x91\xcdfC\xfaUs\xa2\xd9\x92\xdf\x19\x9b]\x86\\\xe0Q\xb7!#\xaf\xdd\xa0F\xe0\x1d\xbc\xba\xeex\xd8\x92N.R#l\x9bo7\xe6\xae\x8a\xd9p\xc7m\xd9\xa5\xfey,]=\x8d`u\xabT\x97\xbe\x03A\xa9{\xe0b+\xcc\x06<.\xddF|\xbe\x9c\xb7\x9b\xbd\x9f\x1f\xf00G\xd7"\x84_\x1d\nM\x02>\xb6\xd6\xc9j\xe9\xd5 \xaf\xc1\xc7\xfaZ\xad\xa8\x18\x1a\xbd\xd9g\x03\xdf\x15b3>\x08xv\x862\x00\xdb4W\xa7\x868\xf9k\x95pO#X\xdd<\x1fD\xb0\xe5[\xee\xee2\xa5c\xb6c\x92\x8dx\x1c\x19\xdel3\xa0\xa2.\xdb\x90\xa5\x81D\xb5`]PO[\xdb^We\xebn\xf0(\xafr\xfe\x8c\xb3\xc4\xaa\x1a\x81H\xb5\x00\xf3\xc8bL\x06\x10\x16\xe3\x96\xf6&d,\x8c\r-\x0c\xc9\xbd\x07k\x9b\xa7\x11\xacn>q\xeb\xddH\xe4N\xa6?\xaf21\xc1,\xf2\xa8\xa4m\x99\x82\xb0\xa6\x9bR\xab\xc9\xf4&\x01\xa0cQ\xd6}U-\x1d\xec\x96\\)\x9eL\xe5\x12\xd5+\xc5\xbew\x04\x1esFI)U}\xa7\xb5\x8e\xdf):\x92\x1c\xfc\xb7\xc0A8i\xc8Ye\xacS\x8e\x84P\x14LP\x1f\x07\x82\xef\x8b\xb9\xebO\xc9\xbc\x94~\xa73?C\x0f\xa2\x08\x9fF\xb0\xba\xf5\x1d \x86\xa1K%\xf8G\x99\x18\xa0\xef1\xe1\\fj\xd4:\x109\x9dwE\x98\n%\x12F\xbe0c\xfd`\xcek\x00\xa8*es\x8eNZ\xce\xb3\xa9\x85\xda\xa1\n\x81:,\xd0l%\x8b\xf1\xac\x90\xed:<$\xd8\xfa\xe0BWl\xb4\xdc\xffT\x82\xd5\xb2\x8b\x18I\xdd\xae\xf9\xee\xf2D\xdd\x8bv\x8f\x95e\xd60\xd1\xb0r\n\xabF#1\x8d\x15%&\xc0\x862\xe3\xae\xd6\x8f\x9di \x0c\x12V\x96\x1ee\xdcA1K|I\x05\xb2fK\xdd\x9a`A\x18\x14v\xa89\x93c\x08kY\x88\xebH\xc8\xfdk\x08\xab\x8f\x9b\x98\xdf"\xac\xbe?z\xf7f\xc4\xfcx\xe5\x17 \x94\xbe\x1c-\xe6\xe9\xd0\x88o\x9a\xc3#j\xc9\x0f\x06\xb7\xac\x7fv1\x8d3\x17\x95\xf9d8\xfc\xf4\xf3\xdd\xfc\xba\xa4\x8e\x9b\x98^s$H\x9b\xb5y\x114j\\\x85hNu\x1b\x12\x89\xa4\x1e5\xc7\x1e\xf9\x07\xf3|Q\x96\xc7g\x9a\xe5\xe9\xac\x97o\x86\xe3\xf6\x1a\x17 :\x97\x0c\x1f\xe6(\xb5q\xf9\x83js\x02\xbf\xd2,/Q2\xa8\xe5O\x95@\x7f?o)\x93^\x9b]Xs4\xcc\xe0rD\xe3hd\t\xa2\x8b\xc7\x98\xc3\xdf\xcc[\x96\xefT\xcb\xdb\xaf\x8eh\xb0>K\xb3\xe6\x04\x90\xee,\x7f\xcf4\xa4#\xaf\xf5\x96g3g\x16\xd7\x0f\x96 \x16\x8c:\x17`\x8b\xa0\xf9\x16\xbc\x96\xf3\xb5\x9c1\x13}\xa9\xb7\xbcB\xc9\xb8x\xcbO\x95\xe8\xfek\xcd\xf2ln\xcd"F[\xbc\x85\xc6t\xce\x84\x0cNB\x0c\x8eFo\xfc\xc1\xc5\x83\xc0\x97\x9a\xe5\x15Jf\xb5\xfc\xa9\x92\xf9\xc5A\xec\xe9\xec\x9bo\xfa\x84\xdeR<\xa8\xcf\xf9"\x86\x1e\x97P\x80\x1a\\5\xe8\x9c\x05}\xfc\xf4\xe1\x8f\x15\xd9\x17\xa5\xf8\xdcR\xfe\x92[n\xe6\x90`\xdd1\xe7%\xe5/j\x96<\xc3\x05\xb7\xa9W/\xf5\x96g\xf3s\xbe\x196\n-\xee?\xe9\x8e\x86\xe8e\xb0\x98dI\xb3e\x80\xbd\xda,/Pr+^~\xaa\xe4WY\xf2\xeb\xb2\x82\xbe\xe9,\x8a\xe8\x1f\x91X\x83\x96\xe2xqy~\xf1\x1a\x1e5\x9c\xedK\xcd\xf2|%\xbf\xa8)\x11\xc3\xa1\xc7\x17\x9b\xe5\xe9\x1c\x9f\xdb\xafd.b\xdc\xd9p\x92\xa5\x1as\x17s\xd0K\x0fs#\xf8J\xbf\x10\xf3E\x89D\x9fi\x96\xa7\xb3\x80\xbe\x07\xb1\x9f\x8aa^\xea-/P\xf2yfy:O\xe8\x9b6\xa1\xa8\xb6$I\xc3\xc9\x97\xbf\x93%\x12\xbb\x93^\xba\xa0\xce\xbd8\x88=]\xc9\xcf\xfd\x9e\xbe\xfd\xfd\xf2\xdc\xf2l&\xd1-\xb7\xdc*\xfb\xf1\xa3\x9ct*xi\'A\xcd\xb9\xb5\x95\xfak\xcd\xf2|%\xa5\xd9/\xc6\x83\xf4\xd9\x04oxq\xbdL\xa6\xa51^\xb2e\x80\xbd\xd8,O\xe7\x1a}\xd3\nt1\x87\xb4\xd4,K9\xc9\xb9\x98\xee\xe4\x8b\xc7T\xa36\xa7\xaf-\x90\x9f\xaed\t^\xfd\xcf\x8b\x17\xf7\xd5\xde\xf2t6\xd27\xcdF\xc1\xa5\x8c\\\xa2\xb14,\xe6\xf9\xc0\xd8/\xe7\r\xd2\xcb\xfdK\xcd\xf2|%\x9fh\x96\xa7\xf3\x95nf\x81\x7fz\xf9\xe2\x08/\xee[\x9e\xaed\x96\xfa\xe5\xcfd|\xdc\xee\x05\xb7\xb2\x7f9`.\xac\x95\xd2\xfcj\xb3<\x9b\xd1t3\xcb\xd2x\xdd.\xf4nW\xe1\xb73&-\x7f\x07K\x8f\xfcZoy\x81\x92\x99_\xccr\xcb\'\x01\xf4\xfd\x9a\xd2DuNCo\x95\xd9\xcb\x83\xd8\x939O\xbf\nbK[\xf7\xda\xdc\xf2\x02%e\xde\xdfg\xc9|\x89\x01\xe6\xabS\xfe\xd3YQ\xdf4\x16\xc5\x96\xcak\xd6\x1d\x1e\xd1\xcb%\x15/U\xd8\x12\x91\xc1W\xf7-/Pr\xbbtu\xf8[N\x19noQ\xe8\x1c\xbf\xa8\t\x96\xba\xd2\xfc\xc5\xa5\xeb\xd7\xe5b\xdd\xcc2\xdc\x92\xe2RN.^B\x8fK\xff\x02\xdf\x92\xe5\xb2e\xaf5\xcb+\x94,\x1eB/\xdfe.\xe5=\x7f\xcb/\xf3\xd2\xbf\xcc\xc6/o\xf7\xbe.[\xeb\xc3,\xfa\x9c,\xa6\xb9\xbdY\x9c\xcf\x1fgm\xe9\x8b\xf5W\x9b\xe5\x15J>\xcb,O\xe7^}\x98e\xa9U\xd0[5f8\xd2-A.\xc2\xf8\xa5@\xae_j\x96\x97(\x19o\x07ji\x8e\x97\x9cB\xdf\xba\xfdI/\x97\x86t\x0e_n\x96g\xb3\xb3\xbe\x9b\x85\xab\xee\xef\xc5\x9c\xd7^\xec\xbfF\xc9c\xe1\xf8\xebr\xc2~\x98\xc5\xbc\xbb\xdc\xd7_\xdc\xe5\xbfF\xc9\'\x99\xe5\xe9\x0c\xafO\xf3\x96\xd7(\xf9$\xb3<\x9d\x03\xf6ify\x8d\x92\xcf\xf2\x96\xff\xbf2\xcf>s\xd3\x9fN\x1b\xfb4_x\x8d\x92O2\xcb\xd3\x89e\x9f\x96\xd0_\xa3\xe4\x93\xcc\xf2t\xea\xd9\'\x96\xbf\xafP\xf2Ify:9\xed\xd3\xbc\xe55J>\xc9,O\xa7\xaf}\x9a\xb7\xbcF\xc9g\xd5Y\xcf&\xb8}^W\xf2\x12%\x9f\x95\xf2\x9fM\x81\xfb4\xb3\xbcF\xc9g\x05\xb1g\x93\xe4>-\xb7\xbcF\xc9g\x99\xe5\xd94\xba\xcf3\xcbK\x94|\x96Y\x9eM\xb4\xfb\xbc\x94\xff\x12%\x9f\xd6N>\x99\x8a\xf7y\xb9\xe5%J>\xc9,O\'\xeb}Z\x10{\x8d\x92\xcf\xba\x13{6\x9d\xef\xf3\xee\xc4^\xa2\xe4\xb3\x82\xd8\xb3\t\x7f\x9f\x17\xc4^\xa2\xe4\xb3\xbc\xe5\xd9\x94\xc0O\x0bb\xafQ\xf2\x98Y\xfe\x12\xfb\xab\xfa\xed2\xfeG\xf6\xd7\x1fn\x8c\xbe2\x11\xf1\xf5\xc2>\x89\x88\xf8za\x9fDD|\xbd\xb0O"">&\xec\xaf\xf0\xf5>\x89\x88\xf8z\x8b}\x12\x11\xf1\xdf\x10<\xdeD\xc4W\x10\x11\xc9\xdf\xd8\xf1\xcf\x89\x88(\n\t\x0cN\t K\xd1,\tR \xcb0\x98@\xd1\x04F\x90\xb7a\xbc \xcb\xc3\x04\r\xf3\x1cH!\x08\x81\xc0(\xcd\xa3\xd8\r\xa4CS,\x86s\xb7\xe1\xab\xbf\x86}\xf1,\x06.\xaf\nc\x14H0\x8b`\x8e\x86i\x82\xe0I\x0e\x11\x18\xee\xf6Z$F\xf3\x9c@s$\xcd\xd1\x04\xcdR\xcbk\xd3\x02\xbal\x0cEa$N\xbe\x90\x88\xf8_\xb3)\xbf\xfdl\xf8&\xfe\x0f\x0c\xc1\xd0\xc5F\xc4\x9f\x01\x11\xbf>M\xf2\'@D\x84\xa7\x10\x0e#Y\x90\x12ns\xd8\x99eQ8\x83\x83\x10\n\xe2$B3\xcb\xff0\x0cCY\x8a`\x96\x1f\xc8C0r\xe3\xfa\xf1\x02\x03\x93,\x82\xd2\x1f\xb0\xa97\x10\xf1\x85@D\x96@p\x06\x81\xf9\xe5\xeb\x90\xc5\x15\x04\nci\x81 o\xb4\x83\xdb\xc8E\x8e\xa0@\x86\xa3 \x9eY\xb6\x93_\x94p\x02Gp\x0c\x85\n8\x87P$C~\xfb9\x10\xf1\tv\xfa\xf7\x00\x11\xffeL\xdf\xd3\x80\x887\xa3\xfe)\x10\xf1\xeb\xfb\xf9\xa7\x02\x111\xf0\xd7l\x92\x88\xc0Q\xab\x00\xcd\xf3\xe9\xc0\xa0\xbbi\xe7\xedw\x1a\xe3\x95\'s-\')\xa0qm\xda$\x17\x7fG9\x9b\x8ei\x85\x8b\xc1\xbb\xbc_\x1b\xc1\x1e\x1d \xcf \x845\x90\xf2\x9c\x9a+f\xbbgW\xb6\x15\x9ch\xf9\x94?8\x97\xf5Y@\xc4?$\x85\xdf\xb1C\x1c\xc9`\xfd\x90W#[\x02|\xca\xaf\xaa\x93\x9aU+\x8aB\n7:a\x1aR],u\xcb\xa8\x85-5E\xc1\xba$~p\xe43\xeb\x1c\xb7f\x05\xd1\x03\xb0vw\xa1\xb5\xba6\x9d\x9d\x1fa\xfc\xa4\xedP\xb9\x7f\x90\xa7\xf3,\x1e\xe2\xa2\x12\xc7Q\x92\xbc\x1b\xb39#\x10\x1a\x97\xd2QG\xc2\xf2BtY\xe4\x10\xf9\xda\x0c\x01\x8ei)\xa5!\xf7\xd4y\xe7R\xf3\xc0\x9c\xc6N\x1e\xf8\x1dTy\'7g\x8f-\x04)5\xce\x85\xb9`\xf4\xf4\xb5!6\xdbn\x8c\x07e\xd8\\b\x9fxp\xb6\xf6\xb3x\x887\x8d\xe0\r=\x8a\x92\xf7\x07\x16\x9b\xf7M\xbe;\xa6\xcc\xe0\x02G\xb8\x07\x03\\\x04k\xb1\x82\x8ex>O\x88Z\x1c=\x0bH#\xb1a\xc8\xb4\xde\xa2\xd4\x99\n\xbcx\xbd.\xce\x90S\xadj=\x11\xdby\xed\xf9\xe3\xa9dgP\x86\x00\xcf\x16\xf0\xcb\xaf\xe8\x01\x7fo\x1e\xe2\xcd\x1f\xa8%\xbd!\xf8O\x98k\xda\x10\xc4{\xf0\nd\x1b\x9b8\xf7\xd69\xa4\xc8=\x94\xe7\xfc\xb8.\x84~\x0bK\x8a\x07\x0f\xfe\x14\x11A\xd6\x031}\xa8L\x8fp`\xbb\x8f\x8e\xc4\xd1\x8aq\xca%\xecTu]\xf2\x98p\xa0n+\x0c\x97\xb0\xf3\x83\x94\x99g\xf1\x10o21\xeav\xe0\x88;\xbf\xbfv\xb5\xcd\xa8\'\x15\x1c\x91\xb1%\x9d\xcb\xee\xb8W=\x99\xdd\x95en\xc6\x9d\xcd\x8dP9\xa7d\xb0\x13\r`{\x89z6\xd6(i\x85X%\xdc{\x99\x0c\x0c0\x81\xced\xd0`\xa7B\xcfr\r&\xe5k\xa5\x98\x0f2\x83\x9e\xc5C\xbc\xc9\xbc\xcd5\xc7\xa1{\xaa\xad\x95\xa4\xcd\xbcV\xcb5S\x84\xc6\xa0g\xb4C\x054p#\x9d@\xd0jU\xf3+\x1eH\x07$#/\xe0\x0e\x14\x81s\xdd+\xa5[\xe7\'\xa1\xb8\xac\xaa\x89\'\x8axr\xcf\xbbrI\x91\xa5H^O\xeb``\x89\xfcQ\xd7\x7f\x12\x0f\xf1&\x13%@t\xc9\xdewtK\xbf\xdfC\xe3u\xa7\x1a\xc0\x01\xb4\xcf\xf2YD\x9aXkH\xa6Wf\x99\x96/\x1b\xc7\xb8Bt\x98h\xe5\xa0K\xe8\x1e\xde:\x8c\xdf\xfa\x1aPn\xc9\xb0\x0f\x99\x9d\xdb\x87@\x0f&\xa8\x9bh\x8d8\x13cB\x02\xf1\xcc=\xc8Gy\x12\x0f\xf1&s\xc9\xc90\xb5l\xd7] \'Oh9\x9a\xae(_/8\x93\xaf\xd2\xd1P\xfb\x0b\xde\xad\x95\x0b\xbf\x16\xe0=%\xcaVol\xba,[e\xe1\x18J+A\xe8F=;\x993u:V\xc2Z(l\x981\xaa-\xd1\\\xb3=\xb7\x05\xf9\xf6T>\n\xbaz\x12\x0f\xf1C&N\xfd4_\xe1\x1ew\xec6KMUh\xcdA\xad\xfb"S[V5\xcb\xc93\x05\x15\xd8\xb2\xc5\x89\xdaU\\\x80^w\'\xabFg5:\xc5\xe9iM\xa7\xe65w2\x86\xcc\xea\x187\x04\x9dn|r\xe5C\xf2\xc5\x8d\xb5]\xf8 \xdd\xf2Y<\xc4\x9b\xcc[\xc5\xb9H\xbd\x1b\x95\x8eMY\xb7:\xa4{4\xe3m\xf8|\xb67\xe5EMd\xb1\xc9\x82\xc3\xe0\xc0#\x96\x1a\x1d\xee\x91~\'\xf9 e\xec\xd2rk\xc1\xbb\x96__\x0e\xdc\xf5t\r\xf2\xd5~\x15\xe8\x97DYW\xda\x1a )\x0bjK\xd4\xfa\x15\x0c\xe2\xef\xcdC\\v\x91X\xcaw\x1c^r\xdf\x1dk\xe2\xd4\x8bu\t\xebGaM\xa3cs\xdax\xfa\xd9RU:\xbb4\xbc0\x9d\xeb1\xc1\xe1\xf5\x98l.l\xdc\xc9\xa8\xb0\xe5q\\n\x14\xff\n\xc7\xb5>\x9d\x05\xb3\xce\xb9\xd6:v\xfdl\xc9\x84`\x04\x9eje\xbb\x07\xe1\x99\xcf\x02".2\x97J\x08\xa3\x08\x08\xbd\xc3\x14\x0fyju\xbe\x17K\xab2\xa32@4\xab\tD\xd4\xddQ\xec\xce\xc9\xb8\xc5\xcb\xe2\n\x9b\x9a\x0bB+\xfb\x8c\xb1\xcd\x00\xe8\xb2:\xec\xdb\xac4\xf5\xf1d\x11\x9b]\xb4\x9b\xb90\xf1\r\xc1P\xba2\x82\x84\x03Sa\x0f\xa6\xe5g\x01\x11?\x029\x8a\x83K\xb5{\xe7\x12q-\xbbe\x87ME0X\xbd Nq\xebx^8[Y\xb4U\xe4\x94\x8c[p#+*\x03\x82\x9e\xa4\x90`\x94\xf0W)\xe7,\x14%SB\xad=\xa3m\x07\xd9\xe8\x0e\xbbrS\x1f\xfdF\x9f\xf7\xbe}\xf8U\x1c\xff{\xf3\x10\xbf\xd76$\x84S\xf7.\xa1\x92(~\xf2\xd8\xe8prdu\xb3\xdfDa\xe4\x18\xbcD\xf1\xf3\xa5\xd7\x12\x10d\x88\x8b\xa5\x8f\x16}\xb4\xd7\xf8\x9aI\xa1\x86#\x92-\x1b\x839xZ3m|<\xb5\x8eKOx\xc0(\xaa\xba\xaa\x1a\x04\x05\x8bGY\xcf\xcf\xe2!\xde\xea}\x84B\x16\x03\x83w\xf5\xfe^\xde\x94\xd6ff\xe5\x1eV%F5:M\x0e\xd9K\xcd\x1f\xcb\x95$\x81X\x10sas\xf6S\xb4\xbc\xdam\x81\xbaN"\x14\xe9Y6\xb7\xa8ao\xf0\x9c\x13\xcd\xb9\xb0\x06\xac\x03\xa4\xb4B\xec\xbdH\x06 g<(\xf3Y<\xc4\x8f4\x01\xa1 \xb6l\xd9\x1fef\xf9\xde\x85\x0fW\xa3\x8b\x84\xdd\x95\xdbSM\x04\x03\x91w\xa8V\xda\x14\xbb\x93#6Ms\xdcJN`\xd2\xf6\x01\xab65utC\xd5\xbeH\xb5D\xe6m:\x9f\xe2\x86\x12cJr\xe8&W\xe1\x04\xedS\x1b\xaa\x1e\x05\x94?\x89\x87\xf8!s\xe99\x90\x9f\xf0Q,\'^\xef\x1a\xc7R\xca\xf3\xe5 \x82@\xa8v\xc7\x9e\xcfP\x7f/\x18H#\xac|{\xafm\x05\x92h\xe2\xd5A\x9f\x13\x9d\xc2\x8f\xd9YM|d\xcb\xef\xe21\xe0\tK\x1b/\xba\xd3$]\x99\x14\x15P0\xc7m\xf0\xabJ\xf5\xef\xcdC\xfc\xe8\x81\x97\x9e\x06Y\x8a\x8a?\xee"\x9fP\x97\xcb:\xf6(\'%\x9dJhw\x87s\xe3GHK\xe3\xb2\x04e\x12zp\'Y6\xe8"\x18;7\xb8\xa0p\xa3\x1c\xf0\x90?;\x8cq\x16m\x02\xe9\xd6-\xd3I\xe2\xb9\x93\xbc\xa09\x16$\xcf\xd9W\xe0\xd1B\xf8I<\xc4\xdba\x81@\x8c\x00\xc9\xfbV\x9f\x92\x1a\x98w\xaa\x96\xba(\xe8\x99\xe6\xb3d5\xd2\xc4f\xde]\xd8\xfdt^AQ\xba\xceGv\x1b\x05\xab*HR\xb4=\x97\xc0\xb8k}\x84\xea\xa9nvV\xe0\xbcJSR\x99\xf7\x88\xaf\x1b\x87\x8d\'\xb9!\x88=Z\x08?\x8b\x87x\xb3&L-\'\x02\x06\xef\x9a\xd4\x13\x05tL\xac\xdcj\x16\x87\x89\x10b=\xb3\x18/T\xa7\x12\xa4D\x06\xcd@\x1b@,\xd6\x87{\x07\xd9\x18L\xd0h\xdaI\x0f\xe85B&0\x9f\xd3\x97`\x03\x8d|V\xf2\xfb\xb6o\x0c)0n\xd7\xd3W\xfbA\xba\xe5\xb3x\x88\x1fWp\xc8R\x14\xddS`\xa6ZR\x8fv\xbf\x8dO3\xb4g\x8d5Bo#\xc8\xb0["\xcd\xcap\xcb\xf2\x81wE\xd2\x12\x9f\xcc\x13\x93o\xe7\xa3;\xa5qL\x19Ma\xe6\x13\xa6\xc9g\xd2\x1eR8_\x85\xfb\xdc lp\xdf]\x04\xa6K~\x15\xc7\xff\xde8\xc4\x8f\xa6\t\x82\xb1\x1b\xbc\xe9.\xe9\xe3\xc5\x98\xc2\xe8\xba\xd8W\x18\xe6\xe6\x91u@\xb7#/\x0e\x8ck&s\xdfM<\x04\x80\xc7\xc4\xa6\x8eX#]\xd1\xb88\xc6\x83\x05\xd89i.\x8do\xcf\x8a\x1b\x96;\x8b\xd2\n\x98k\xee\x9aQ\x12\x0e\xdb\x07j\xfa$\x1c\xe2M&\xb8\x847\x0c#\xef\x02\\\xe8\xd8\xac\xb4\x9d\x89\x8c\xc8\xa7\x93P#\x80\xd1\xc4\xc5vH\xdd\x02`\xc9a7\xb7dh\xe3\'\xfal"g@\x07X\xe8\xb2\x87\x06J>s\xd3d\x00\xc7f\xe3c9\x88\x85\x07I\xe6\x91\xf9\x14\n\x84$4\xed\x99y\xec\xbe\xf6Y8\xc4\xef\xd9\x10\x03A\x18\xbbK\xfa<\x9cc\x16\x91D\x9b\\\xf59T>\xf1r\x969"\x87\x9d\xfd\xb6\x86V3\x9eV\x12\x14\xb3Y\xe4\xd7G i@\xd1I\xf9\xfd\xf68\xed\xaf\xf9\x06C\x9a\xfe\x1a\xc0\xf89\xd7\x8f!\xb2\xe68\x96\x19\x0e\xa0\xd4u\x0f\xca|\x1a\x0eqi\xde@\n\\^\x07\xbc\xabTC\xb1\n\x9bbGTh\xcc5k"\x9d\x8e\xc6\xc4\x88\x01\xd4\x05&+)\x0c(\x95:\x1d\xed\xd7\rl\x18p]\x1fm{\xb5\x9fKj\xbfM\xed\xc1\xbe\xec\xe7ei*viSej\xa6\xb9\xa3fS[J\xd9\x07\xdfcx\x16\x0f\xf1\xa3R\x85\xb0\xc59\x89\xbb[8\xac\x0f\xf8$\xcf\x1c>9\x92\xe0i\x18\xa2\xe9,\x13\xe91\x08\xf4=\xbb\xa5\xb7\xe2y\xa5\xcb\xdeF`\xc0\xa9E$\x0b\xe5/g\x1b\x15\x82\xd0ct\xe3\xba\xb9\xf0\xe1@\x14i\x0f\x8eX\xe2\x9c\xc4\n\x1e\xdcRT\xc6\x07\xd9o\xcf\xe2!~\\\x1dC\xe0\x92\xf3\xc8;\x99\xba\xa1\x9c7\x1e`\xf1\x84]gG\xb3\xec\xe9|M\x86\x87\xad\x11S\xe2\xb8Z\xfb(\x11\xb5\xdb\x8d\xbd\xca\xdb\x98b\xb5\xa2;\x11\xc3\xd6\x19\xeaS\'0\xa3W\xd7\x9d\xb6\xc6@9\xac\xb6s\x97\x9a\xa567\x8a\x04\xc7$\x8d\xdaT\xc5\x85\x8a\xb5\x1b\x1a\xf0\x82Kr944\xb1$a}@\xd7;`\xc9i\xdd6V\x01\xc3HBbk\x1d\x19}\x9b\x04\xeaNt\x1fT\xf9,\x1c\xe2\x8f7Q\t\n\x07\xefd\xb2Q\x13\xb8\xca\xc6_\x8d\'3>\xed\xb0\x9dN6\x0e\xb6+\x99U;\xd0<\xd4\x15\x91o\'\xce\x81\xde:\xc6.\x8d#d\xaaqv\x9cJ)\xdcNe\xba\t\xe6\xf38u\x9b=@\xdb\xde\xb5,!\x1c\xf4)\xde\xf8ZW\x99\xcf\xc2!~\x9c\x15\x1cB\x10\xea>~\nE\x95]\x8f\xb3\xd6\x86\xc0\x08\x0c\xe3\x06\x13\x15\x9a\xce\x88\xbd\xe3s\rY\x17\x01\xbe\xbe0E\xabCW\x90`x\xfbtY\xa3\x13\xae\x80<\x86\xcd\x95lrAv\x0eE\xe9\xaaV\xbe\xd2-\x99S\xa6\xf7\xc2\xd2;\xffk8\xc4\xef\xe9\xed\x8dC\xfc\x97\x87\xe9\xfc\xe7@\x10\x9fND\xba\xdb\xab\xaf\x8b\xa1z\xd1\x8c\xa4\xbf\xd3\x96>\x1b\xea\xf7\xaa-}\xf9:\xbf.Z\xf0E[\xfa\xfau~],\xe0\xabN\xe9\xcb\xd7\xf9\x8e\xa5_7\x96>\x1db\xf7\xa2-}\xfd:\xbf.J\xefU\xa7\xf4\xe5\xeb\xfc\xba\x18\xbcW\xa5\xa7\x97\xaf\xf3\xeb"\xec^\xb4\xa5\xaf_\xe7\xd7\xc5\xcf\xbdj\x1c\xea\xcb\xd7\xf9u\xd1q\xafr\xfc\x97\xaf\xf3\xebb\xdf^\xb5\xa5/_\xe7\xd7E\xb6\xbd\xaa\x88z\xf9:\xbf.n\xedE[\xfa\xfau~]T\xda\xcb\xea\xd2W\xaf\xf3\xebb\xce^\xb5\xa5/_\xe7\xd7E\x94\xbd\xcc\xf1_\xbd\xce\xaf\x8b\x17{\xd5)}\xf9:\xbf.\x1a\xeceW\xd0\xaf^\xe7\xd7\xc5z\xbdjK_\xbe\xce\xaf\x8b\xe4z\xd1\x96\xbe~\x9d_\x17\xa7\xf5\xaa-}\xf9:\xbf.\n\xebe\xb7\xfa\xaf^\xe7\xd7\xc5X\xbd\xec&\xea\xd5\xeb\xfc\xba\x08\xaa\x17m\xe9\xeb\xd7\xf9\xc6G=}\x9d_\x17\x0e\xf5\xaa\x9b\xa8\x97\xaf\xf3\xeb\x82\x9d^\xf5F\xc9\xcb\xd7\xf9u\xa1L/+\xa2^\xbd\xce\xaf\x0bTz\xd1\x96\xbe~\x9d_\x17\x86\xf4\xa2-}\xfd:\xbf.\xc8\xe8U\x19\xff\xe5\xeb\xfc\xba\x10\xa2W\xa5\xa7\x97\xaf\xf3\xeb\x02\x84^\xe5\xf8/_\xe7\xd7\x85\xff\xbcjK_\xbe\xce\xaf\x0b\xeey\xd5\x96\xbe|\x9d_\x17\xba\xf3\xb2R\xff\xd5\xeb\xfc\xba\xc0\x9cW=m\xf2\xf2u~]\xd8\xcd\xabz\xfc\x97\xaf\xf3\xeb\x82j^\xf60\xe4\xab\xd7\xf9u!3/{\xb0\xfc\xd5\xeb\xfc\x0b(\x84\xc5\x8e\x0f\xa1\x10\xaa\xdf/\xe9\xeb\x02b\xfe\x1d\xc2>\x05\x10\xf3\xef\x10\xf6)\x80\x98\x7f\x87\xb0O\x01\xc4<*\xec\xaf\xe0F>\x05\x10\xf3\xef\xb0\xd8\xa7\x00b\xfe-\xc1\xe3\r\x88y\t \x06\xfa\xef=\xffs@\x0cD\xa2$\xc1\xb00A\xb3$\x03\xd2\x14\x86\x934C"0\x87\xf0\x08\xcdr$\x83\x12\x0c\xce\xb2\x02\' \x14H\xb3\x08\x8fQ4\xb9|\x0fC}L\xbe\xbd\xd1-~\r?\xc0h\x88\xc7y\x9cfy\x94\xa3\x04\x98FY\x02%P\x94\xc7\xb0\x1b\x89\x06\xe6)\x12aQ\x96bP\x81\xc6@\x86\x17P\x08%\x11\x8e\x07I\x88\xe0\x19\x84\xe5_\t\x88\xf9\xe7,\x8bow\x03\x18\xe0\xff\x05\xc2\xff\xa0\x10\x08\xc2\x08\xe8\xfb\xb4\x8e_\x11b\xbe>]\xe7\'\x84\x18\x01\x87y\x06\xe4X\xe66\x94\n\xa5H\x12G`\x86@p\x8a\xa0@\x01^>\x84\xb2\x8c@\x82\x04A\xb1\x10\xce`8\nC$\xc1-:\t\x02G\x19\xee\x8f\x84\x18\x81\xa5\x19pY\x05\xb6\x1cv\x96\xa7\x97-!\x08\x84&x\nB\xe1\xe5\x13\xb73\x05R(\x0f\xa1 \xba\x1c,\x9e\x84P\x9e \xd9\x8fa\xc7\x14M\x0b\xc8\xb7\xdf\x1c\xf17!fy5\x8a\xa7`X\x10\x04\x8e\xa0A\x04\xc4)\x94\\\x96\x08!\x10J!\xb0\xc0r\x08\xb1\x9c9nY\x1b\x01\xf1\xd0\r\x8f\x80A\x18L\x91,A\x100\x85\xb3\x10\xf8\xedoN\x88\xf9\x97Q\x17w\x84\x98o\xec\x11e\xd8c\xc7\xb0\x85v\r`j\x8e=jV\x0f:\x18x\x97c\xdc$}Z\nE\xe8[{\xcd\xc6\xf4\x04N\xf9\xd0\x1b\xebp1mh\xc9 [\\\xae\t\x0c]5\xff\xf6\x96\xce8%\x1e\x04\x19\x9e>E\x07}\xfc\xc8J\x9c9d\x07\x19\t\xd72\x1a!\xf4\x9c\xce.\xbcd\xaf\xa5J<\xee\xf9R\xfe\xf8\xde[\x0f\x14#L\x9d\x94\xc2\xac\xcd:\x1e K_\xd8\xec\xdb\xc0\x0b`m\x9d\x83\x81S\r\xc1\xf2y\xc3\xaf\x97\xf6<\xc1\x0c\xefxZ>\x8e\xa6\x1c\x8df\xfb\xd6b\xf3\xedw\xea\xe5(I\x9a\xe3\x8eK\x06F\xb49\xb1#\xfah\xb2U\xa0\xac\'\xaa\xfc\x9eA\xb5>@d,Y[\xd7E_\x1d \xf4\xa4;\xe1y\xd1e\xa6\x9e\xac\x86~eKC\xbb\xa7\x07f-\xb0Z\xe5E\xfbrYp\xa0\xcd\xf35>\xae1\xa3cwtI\xce\xab\xe8<\xdbb\x03\\9\xab\xe3sS\xa1\xf3\x9c\x11i\xde\x16\xe8\xd6\xf8\xcdZ\x04\xcd\t\x90%\x0b\x9b\x84\xdc*b\xc5P:\x7f2\x1a\xcaKG\trA\xe5Z\xfa8\xa8\xb0J\xbdssX9\xd9\xf5\x8e\xaaw=u\xe8\xe2KQ\x9e5Rl\x15\xa2?\xaa\xaa1\xf6\xc5\xe07\xbb\x089R\x03\xa7l\x80\xd3@4\xabu\x05s\xe4\x948\xcba`\x8f4\xc3\xb6 \xc3\xee\x87\xff6_#\xef\xe3\xa6\xfb\xa3\xe9\x9c\xd0\xc3*\xbe\x01\x15\xe5\xe3\xeb\xc8\x07~\xbbV\xc3B$\xad\xc2ue\xff\xf3{\xb5\xc3\x11I\x9c\xfd\x10\xc0\xd0e\xf9\xdc\x1cs\x15\x94\x1c\xb6u\xe2\xeb\x8d^n\xf7\xb7\x87Ku\xffXF%=\x84\xe5\xf6\x12z|\xc4\x9b\xad\x90z\x1f\xdfo\xb3\x9c;h\x1c\xfd\xf1\x87\xafs\xc5\xa5\x8f"\xbb\'\x7f\xa3A\x9f2\x9f\x01\xe3\t[\x8e\x96>h\xa5\x0bG3\xdf\xbb\xebma\xc3[\x8c/4E2u\xdbP\xa2I\xe64\xad\xcd\xd4\x02\x88\x05\xc7\x18\xcb\x08\xc3\xdb\xaa@\rD=\x1d.\xc1y\xf0\x8c\x1d\x10\x81\x96\xc4\xb3\xb9$\xd1V\xce-f3\x07\xb9\xd5\xc4\x8a\x0bN\x0e\x19\xc0\x00\xd3\x99H\x98\x9dw<+\xdb\xdd4l\xb1\xd4U\xe2\x11\xe6\xac-\xaa@Y\xc6\xfb\x83\x96\xc4\x83\xe3\xec\x9fC]\xfa^i-&\x82a\x02\xbe\x93)]\xe4s\xd9kv\xd4*\xb6\xb9\xfc@\x10\xf3\xb4\x9d\xc5\xbb\x1dN`&}:\xba\xb8Ej[\x04U\xd9%\xb7\xac3J\xc5\x8fh\x90\xc0\xe8\x14zk1+%3\x95\xcd\xc1\x82w\xab\x8d\xb0\xdf\xed\xd2\x9e\xe6G\xfa\xd1q\xf6O\xc1.-2\x91\x7f,\xe5$|?\xce\xab\xa4:\x9fODE\xb9\n\xda\x85\x1e\x8f\xb9z \x81\xc5\xff\xb7\xc2~\x08\x1d|\x8baj9\xd6\xf1\xda\xf2\x92\xde\x19j!\x9e\xf2\x06\xee\xc4=\xd3\t\x98\x0cf\xa5+\xb1@\x15\x82\x94\x9b\x88\xc1I\xbdt\x15rB\x1f\x9c\xe1\xf9\x1c\xec\xd2\x87F\x10]\xbe\x9c\x02\xf1;S\x8a\x16\xefx\xa5j\xd6l\xab\xecz\x18\xd9\xd3cc"\xfdbB\xd0\xe7w&!M\xe7\xadGJA\xaf\x02\x01\xc1\xa0:\xe1\x81\xc9\xc1\xda\x81\xa2\x94"\x8c\xca\xd8\xee\xa0\'U\xb3w\xf0nY5\xe3T~q\xb2\x7f5\xc3\xf3\xef\x8c]\xfa\xbe\x8b\xcb\x07A\n\xbe\x1b\x84\xean\x10N=I\x1d\x1d\x81\x19v=M2\xb0K\x90\x00\'\x1b\xa4\x0eZ\xfa\x8c_h\x94\x19T\x94\xe0\x9a5\xb7i\xa6XK,K\x98Dy{\xd8O\xf3\xc6\xda0u\xcb]t@\x00\xd8\x84h4ww\x956\xec\x83\xd3{\x9fC]\xfa\xaer9m\x04\xb8\xb4\x17\x7f\x94\tZ\x80\xbd\xd96\xea\xa1C8\xa3d\xa9C~B\xfdV\xc6\xf7ny\xcc\xb89\xaa\x84\x94\x15D\xd7Y\x1f\xc5\xf1<_\xf5\xde\xde[\xb4\xaboA\xfbXn\xd7\xb5\xbe\xc92/\xd87\x07a\xa3\x11t\xc7\x8ar\xc1k\x8f\xd2\xd6\x9eB]\xfa\x1e\xdd\xe0\xa5`\'\xc1{\x99\xbb\xf3\xda\xf6\x97r0\xda\xf9\xa0+\x8e;y\xbbs)\xce\xa7\xd2\xd5\x88\x15;\xca\x9c<\x88cF`u\xdc\xb8\xbd\x12\x9cj\xa8%\xaf\x9e\xb7\xf2\xfd\x9d\xc2\xc1\xc9\x94\xad\xf6\x87\xa6 \x96M\x84\xe9\x15\xc7\x9dJ\xff\x12\xa2\x9fB]\xfa.si\xab l\t\x86\x7f\x94\xb9:\xd5\x05\xb5m\x8ehj\xed]F\x11\x1c\xb9\x9c\x1d}[4\x07j\xdb!\x17\xdc(\xa2\xb5\xd2\xce\x94\x01\x1b\x97\xb5\x81\xd4>y\xddu7L\xd3\xbaTK\x05\xdb\xd2g\x1e\xe8\xce\xdd\xc5\xc0Qq\xb5\xd7`\xaa\x8c\x86\x07\xc9\x04\xcf\xa1.\xfdpM\x18\x86\x08\x1c\xb9\x9bO~\x1c\xf9\x1e\xdd\xd8\x0e\x8f\x05\xd3\xfe\x18\xab4_\x92h\xdd\xa1\xc0L\xce\x97\xe1\xe8\x17\xd5z\xefA\xaa\xb9?V\x84k\x82\x1b\xa6\xd4\xb9\xb6\xde\xefd=,\x03\x88\xe8\x8b\xd1%\xddz\xd6\xa84\x1e\x01\x1c\xdf@\xbdyypH\xf1s\xa8K\xf7\x97\x1f\xbf\xc3\xe7]/\x83\xdb!\xf3\x01g\xa6\xd2+\xf7\xe4\x8cSm\x1a\xbb\xc4\xbe(\x05~\xcb\xef`\x9d\x07\xe75\x91d3.\xd5\xd6\xb6\xdand[\x04\xc4\xf5\x04\x12nx1\xba\xd3\xe4d\xac\xcb\xf5;9\xc7\xea=\xb6\x8a\x94\xf4\xc1C\xfb\x1c\xea\xd2\x0f\x99(\x88\xa0\xf0\xbd5G\xe1\x88\xb9+\xc6\x80|/\xca\xd4~\xd4ip\xae\x8eT\xd8\xca\xe6T4U \x04\x1e\xdf\x9c\x8e\x14\xd3\xd09!\x9c\xfc\n\xdc\xb8Q\xb9u\x00\xb8\xf5W\xf5\x12\xf3iM\x967\xdb\xd5\x12\x15\xd59H\xacj\xf4\x0e_+]=\x87\xba\xf4\xe1\x130\xb18\xfe\xf2\t\xfc\x8f\xbb\xd8\x1f\xa3`7\x19\ry\x1c1\x9b\xbe\x80\x82}\x8e\xcdv\xdb\xc7\x1di9\xb9hv3(\x13\x9b\xed\xe8\x0c\xad-\x1ck;\x99\'\x85M\x98^\xe4,=\xd9J\xc5VY\tz2n\x95\xe4\x14\x90k\xddN\xc9\xfe\xf2 \x80\xe19\xd4\xa5\xef\x87\x05[\x1a\x96%\xda\xdeE8\x0c\xcf$\x06\xc2\xb8\xa5\xdf\xbe\x08j\xb1\r\x88N\nlm\xe6k\xd6r\xf7\xc4\x9c\x05\xb3\x87\x83\xa4\xc3\xee}\xcd$\x93\x1a\'\xa6\xc1\xc5\xa9M\xe4R\xa5\xcd\xad\xe5y\xc4\xfb\xa3\xe7\x9b\x17\x8a\x8d\x89=N\xb6\x82_>\x88\xd3x\x0eu\xe9G\x84[\x8a@\x84\xc2\xef|"\xa4W\xe0\xc9\x0b\xb6\xf6~\x04!p\xd5\xb1L\xb0\'\x0f\xfd\xb5p\xb8\xd4\x8b\xcb&\x98;\x00vKzw\r\xcc4@\xfd\xcd\x81\x9e\x97n\xd9C8\x90\xdf2A\xc7\xbbj5\x19\x8c\x18\xe4\x86\xc6]\xa4]\xaao\xbd\xaf5\xbe\xf79\xd8\xa5\xef\xbb\x88/\x850I\x10w\x01t\x8f\x85N\x87\xba\xf4}\x17A\x1c\x87@\xf2>~\x8a\x06\x98Hkcj\x08{\x1f\xe0\x1d\x19[bK\xcdE\xb6TJ9\xef\xf9\x87|\xbf\xeb\x14\xf4\x8aK\xe8tlW\xf4\xb0\xae\xc0\xe6\xbc^\xe2\xa9\n\xafs\x9f\xe2VC\xa36n\x06o\xe6\x15sj\xae\x1d\xd7\xcf\xfe\xafv\xf1\xa5\xd4\xa5\x1f\xa5\x13L\x91\x14\x8c\xde\x1d\x96\xae\xcb\xe24\x1a\x1d\xa8\x08\x04\xd3\x9bM\x17\x9e\xf4z\x86\x98\xbc\xa3\xd8lg\\/c\r7Y\xe0\xccJw\x12\xc3\xe8\xd0\xf0s*\x186\xb4\xd7698\xecI\xfbd\xb0\xf6\x86\x14G\x179\xcbx\x01\x10\xfb\x9c\xfc\x14\xea\xd2\xf7K\x1b\x98\x84\xb0\xa5M\xbd\'\xe9\xa0\xc8\x19\x033>\xbd\xa6\xda\x04\x13V>\xbaz\xa5\xac\x992=\xf7\xc1\xd0\xf4\xd5\x8a\xe9\x11qH\x80^\xedv&\x04O`\x9c\xf4\xe1\x81\x9f\x93!\x18\x92\xb6\xd1\x8d=\x13\xb9\xaa\x8a\xd7Z\xc28\xa7\xd2\xb1\x8b\xe3\x83\xae\xff\x1c\xea\xd2\x8f\x00\x07\x13\x14\x84a\xf7\xc0\xb5]\xbas\xcc\xee\x0cPi\x14I\xdb\x9cUJj\xabGf*\xb3Xj+\xa5`J\xb8F\xcc8\xaap[?\x9e\xd0M8\xef\x80\xd6\xe3\x97\x00\xa8\x13\x0c;\x95\xd6E\x88e\xc1,\xd6\x80\x87SY\x16j\x01\xf3\xab\xeb\xa9\xbf3v\xe9\xbbO@$\xb98\xfe}6\xdc\x1d\x99\xedY\xcd=\x88e\x01\x81\xad\xa3\xd0\x14\xf0S\xe8!\'0\x91Sn(\xe5\xa0f\x8af\'b[.\x92\xdd\xa6\x0f\xc4\xcb\x86_/>\xc4\x13\x8c_\x17\x1dY\xe5Px\x8e\xce\x82\xc5\x9cko\xdf%\xed\xd2\xd1}\x06v\xe9G\x9e\x80!\x14#\xef\xf1\xa3\xb3\x85\x9a\xf3\xbc\x9bHTJuiGk\xa9\xb3\xea\xf7\xac\x94\x93ci\xee\xd2\xfaL\xea\x83\xea\xd35\x027e\x89T\xb5\x88U\xbdS\x80e;\xd8\xb4.k{\xcc\x13\xb8R\xb8\x12\x05\xe8LI]5+\xd8\xd0\x1e\x94\xf9\x1c\xec\xd2\x0fk\x12\x08\t!\xe8]\xd6\xdf\xc0"\xdbI\xfba\xa0\x8e\xe4\xe4\xdb\x11\xabu\x96\x1ac\x02eA\xbb.\x01\x8f\x02d\x8a3\n\x846\xe2\xc7\x9b`g\x1f\xce\x0ee9\xb2,\x13\xc9nF\x1aJ\xcekM\x0f\xa6\x0b\x81\x1b\xfa\x96\x1e\xf7d\x95b\x0f\x96p\xcf\xc1.\xfd\xb0&\tbK\x9bt\x17\xc8\xf7\xc9\xc6\xdeYA8\x9fYP\x0f"\xa6\xaf\xa8\x83\xa5H\xbb\xc3\xd0\x1fh,)E\xd2\x8b\xe1j\xb7\xd2\x82\xa9\x06\x86\xe6\xc05\xaa\xbai\xd1=\xd0\x88\x8a\xc6n\x189YY\xadS\xb4\xf2\xff\xcb\xde{-=\x8e\xa5\xdb\x81\xef\xd2\xb7\x98\x10\xe1\xcdD\xcc\x05\xbc\xf7 \x9cB\xa1\x80#<\x01\x02\x84\xd5\xcb\x0f\xfe\x9fU\x1d\xd9\xea\xee\xe7\xb44YuU\xccJpc\xed\xcf\xacoa\x13\x0b\xf70i0\x8aS<\xfb?X\xe1\xfe9\xb6K\xbf\x15r\xfc\xcaN\xfc\xaf+\x9c\xf1\xae\xb8m\xdd\xb31D\xc1\xfa\x95\xc76T\xf5\xc1\x08Oy\x10\n\x8d\xdeO\xeb\xc6\xce\x8e\xeb\xb5C\x04\xe8\xb2\x8b\xd7z\xe6>\xed\x8b)mx\xfd\x8a\xd8\xe9\xf6\x082\x18\xbc&\xa1W\xa3\x18U\x8cm\x94PH\x7fp7\xff9\xb6K\xbf\xc9p\xf8\xb5\x9b\x08\xf2W\xbb\t\xf4\xfa\xd0m9\xb4n\x1c\xc0\x00\x1d1f\x1ct?\xab\x97?\x145\x1a\xcf\x07\xd3\xa3{\xf6h\x17\xa52\x1fA\xa1\xf4\xe7\xb3\x9d{\xd8\r\x92X>\x87G5\x9eo\xc3\x89\x8eYz\xf6\x1b\xc3\x9b\x11\xeb\x9f\xf3\xdf-A\xff;\xdb.\xfd\x96\xfa(v\xe1\xfeW \xc2\xf0\xbaG\x13\xb8\x06\x8b\x187\xa5w\x18\xaf0\x82\x13i@\x917\x7fP\xaeX\x8e\x19!h\xb6\xac`N\x0c\xac\x0b\xedG\xa5\x80FJ\xd5;\xbd\x8b\x02\xaa\xf287a\xf7\xee\x18\xe0\xbd2\x93>cG\x8e\xd2?\xaaL\xffS\\\x97>wI\xa0\xd7\xf4\x80\xfc\xb5\xaf\\\x06\xd0\xaf\x89K\xf9\xd5s\xfalUR]\x17\xe0k\x14K\xb6+\x9d\x15\xc0\xde-\xd9\xdfO\x90\xde\xcd\xb8j\xb6=\x93\x83{$\x01\xb7Yz\xc3\xddPU\x8b\xed\xe4\xe6\xd0r\x8a\xd3D\xe4\x1d~\x0e~Zw\xc3\x1fL\x89\x7f\x8e\xeb\xd2\'\xf3Q\xe4"\xf5\x7f\xa3+\x9b\xca\x1b\xdc\xc6JI\x1ft+2J\xacSe\x93\x87O\xbd\x1cz\x07/_\x92h\xce\xab\xbc\xb66.Ed\xab0\xdb\xdc\x14\x94\xdf64YG*\xc3\xf6x\xb3\xcb\x0e\'n\x8e\xcd\xdfUj\xd0\x86c\x8d\xdd?\xf8\xe8\xed\x9fc\xbb\xf4g\xdd\xe6[\xca\xfe+\x01~\x8a|m\x81\xee\x05Z\xd0\x0f\xc8\x16\x90\xf2t5\xd3\xb7:\xbb\xeb\xa4\x1e,\xb6\xe5q\xec\x87y\xf8@\xa9\x9f\xd0\x9dP\x1cL\x9by\r\x9d\x10<\xe4\xa6\x95\xc0\xf6=-\xe2\x0e\xd9\xf9\xf7-S;\xca\\6\xf1_\x8b\xc2\xfdsl\x97~\x7f\xf4\x06^s3\xf9W(N{\x17\xf5\x04\xf3:\xa7\xfamfd\x1c\xe6\xc79\xc0\x88\xf1\xe4\x99\\_\xc0\xfb\x16\x9d\x92q3\xb7G<\x8e\xd5j\xc4\x9e\xb6\x96B`\x8f\xee\xee\x9byu\xde]\xa5P\xcf\xf4\x05\xf2\x91\xd6\xbf\x976uT\x97\xfaLo\xff\xa6\xed\xd2\xf7\xee\xffh\xbb\xf4_\xff\xc7\x9f\xb2!/\xbe\xcfU|\x0e@\xfd\x89-\x07\x95\x15\x9d.\x15\xfdg\x14:\xdd\xf5W\xea\xe7c\xf8`_&\xf3\x7f\xdf\x92\xe7\xbb\xf8:\x0cy\xcd\xf7\xf0\xb7\xac\xfb\xf5\xe92\x7f>\xbb\x06B\x90\xfc>\xe0\xf5W\xf6M\xfb\xdf;0\x1c\x05F\x13"F\x97\xd9\x7f}\xc0\xf7\x146\x1d\xf6\xbf^\x8b\xf1\xce\xbd\xae\xc9\xfa\xf1\xcc\x02\x08N\x83\x12\xd4\xa5\xf1\xc8a\xe7(\xc4\xf7\xa6#\x19d4q\x95y\xcc+r\xc9\xfd/O,\xff\xfd\xaf\xce\xe0\xee\x99\xf6\x02\x98\x87J\xf7\xc3W\xc7\x9d\xe3\xba\xf0\xb8\xe5]\xbcD\x90a\xfa-\xc6e\x8d\xbd\xab\x88\xc3\xc5p\xb7\x05\x9c\xa0x}\xb7\'\x08\x13i\xd0\x18\x19H\x9c\x18`\x86\x04\x01\t\x17\x1d\xc3fw\x032\x10\xc1+\xc2\xf1\x8cz\xeat\x1a\x7f\xd1\x9f6\x1c\xf7\x06S\x04\xc3\x9a\x85\x86e\x841j\xdc\xb1\xa7\x06B\x8b\x13f\x94e\xff\xcf\xceK\xff\xb0\xe1\xd5?\x80\xd9\xef\r\xf8\xb7+\xff\xcc\xf0\xea\x7f\xd9N\xeb\xaf\x7fy\xf3\xa7\xcf\xb1\xb0\xeasLl\xfb\x9f\x81\xf8G\xed\xaf\xfeQ\x1c\xfe\xc0w_\xb1\x0b&\x01\xb5\xfc\x18<\xa2?\xc7A^\xe5\xe2\xfd\x9f\xb6\xa1o#p\x96\x08\x89\xfb\xbc\xc9\xe7\xfc\xc9\xc0E`\x1czc\xe0I\xcfc\x17\xf0]\x84\\\x11\xd6\xbf\x9b\xbc\x91\x0f\xa3\xcf\xe7\xec\x0fl\xe8\xffr\xb8\xfc\xcd\r\xbd\x9f\xc6ym\xbe\xf7O\xdf\xd4\x7f\x10\x8b\xbf\xf8\xad\\\x97\xd4\xfd\x7f|Z\xfdC8\xe5EW\x94\xc9\xfb\xef\x02\xf5\x9f\xf5*\xac?\xfdp\xac\xf4\x8fmm\x1aPm\x1e\xec?\x16\xdc\x0cq\xaa\x04\xee\x96x\xfb\xcb\xb0\xfd\xe7g\xf6\x7f\xfb\xbf\xbe[\xde<&_\x07\x0f\xff\xf4\xfde\x7f\xdd\x03/^&\x19]\xda\xdb\xff\xcf\xdf\xef\x7f\xf0\xef?\xb7\xf8\xa1\xffa\x04\x81\xc2\xff\xd4\xf6\xd7\xc6^\xbe\x17\x1c\xbde\xd2p$M\xfcL\x9bn2\xb9j\xc9\xa5\n-.\x14S\xf1]\x15}<\xe9g\xf7\xbcn\xbc-\\\xf2\xfc_l\x7fz\xfb\x9e407\xf3>\xaf\x9d;z\x04\x12\x7f\x16\xbc0\xfa\xa1bd`\xdc\x05B\xcc\xa5^\xd5\xc6\'\x13\xc4w\x03\x0cDlp\x9a\x8a\x8bN!\xf1a\xdf\x8c\xfa\xae\xcf\xcf\xfbn\xb7o\xdb\xab\xa9(\x91\xfc\xc5\xf7\x9ckq\x8a\x91x\xe5\xee\xf3\xb9\xa4s6t\xafI\xcc\xbbw\x91\xde\xf2\x7f\xa3\xfd\xfd\xf1\xca\xfb\x1d.\x7fe`\xf8\x8fW\xdd?\x8e\xfd\x1fi\xa3G\xf1\xccwSb\x9a4\x80\x9e9b\xcc\xa6\xe7\x1fy\xe0\xef\x19\x1c\xc1F \\l!\x83\xae\x94\xfd\xfau`o\x04P\xf7oW\x13\xc3\xa3\x7f;>\xcc\xfc\xb3\xab\xee?\x88\xc5\xffZY\x88zaI\xb6\xbf\x93\xb8\xff\xed\x97a\xe8\xdf\xfd\xe5\xf2\xffA\x86\xa1\xff\xfbX1\xfer\xb7\xfc\x05\xe9\xbf>\xa4\xbf\x0cC\x7f\x19\x86\xfe2\x0c\xfde\x18\xfa\xcb0\xf4\x97a\xe8/\xc3\xd0\x7f\xe1u\xfe2\x0c\xfde\x18\xfa\xcb0\xf4\x97a\xe8\xbf\xb2\x11\xe7/\xc3\xd0_\x86\xa1\xbf\x0cC\x7f\x19\x86\xfe\xcb\xae\xf3\x97a\xe8/\xc3\xd0_\x86\xa1\xbf\x0cC\xff\x95\x8d8\x7f\x19\x86\xfe2\x0c\xfde\x18\xfa\xcb0\xf4_v\x9d\xbf\x0cC\x7f\x19\x86\xfe2\x0c\xfde\x18\xfa/t\xfa\xf0\x97a\xe8/\xc3\xd0_\x86\xa1\xbf\x0cC\x7f\x19\x86\xfe2\x0c\xfde\x18\xfa\xcb0\xf4_x\x9d\xbf\x0cC\x7f\x19\x86\xfe2\x0c\xfde\x18\xfa\xafl\xc4\xf9\xcb0\xf4\x97a\xe8/\xc3\xd0_\x86\xa1\xff\xb2\xeb\xfce\x18\xfa\xcb0\xf4\xffP\xc3\xd0\xbf`j\xff\x865\xde\xffd\xa4\xf7\xafm\x18\xfa\xef\x7fc\xffI\x86\xa1\xff\xfe7\xf6\x9fd\x18\xfa\xef\x7fc\xffI\x86\xa1\x7f\xe4\xc6\xfe1\xfb\xc9\xff$\xc3\xd0\x7f\xff\x1d\xfbO2\x0c\xfd\x0f(\x1e\xff!\x86\xa1\xff\xf5\xcfV\x9b_\xefbr\x97\xfe\xdbd\xea\xb7\x1f\xa8_\xd7)\xa6\xbe~\xbe\xffK6\x1d\xe3{\xf8/\xd6\x92vu\xa6\x16\xc7\x7f\xe7\x7f\xb3\xe3\xfc\xf3\xfa\xff\xc7\x9f\x1d:\xff\xef?\xa1/>)\xbd\x8e%\x16\x85%6\x12=\xd7z"\xde\xcfN=\x11\x90\xd6@NW\x90\x17\xb8^)\xe6~\x05\xd9\xd7[\xa1\xc6a\xfb2\xbc\xfc\x13\x84\xff\xe9\xf3K\xfa\xff\xff\x18\x98\xc2?b\xf83\x03S\x06\xe69\x04!A\x82\xe2\xbe\xac\x8bx\x12\xfb6\x83 X\x84\xc0`\x98c!\x1a\xc6\t\x94\xf9z\xab=\x04\xa1\x02\x86\xe0\x1c\x8f1\x94\x80\x91\xd7\x1f0\xf4\xd7\xab\xb0\xfe\xbe\x8d\x18\x8e3\x1cF1,A\xe14B\xd2\x14\xc3\x814\x0c\xe2\xac\x80 \x02\xcaB<\x0b\xd2\xd4\x97\x8f\xa5@\xf1\x8c@p4G\xd0(\xcc`\x04F\xb1(\xc2\xa0\xf0\xbf\xa7\x81\xe9\xef\xaf\xa4\xfd\xd3\xdfx\x95\x19\x04\xfe\x17\x18\x07Q\x9c\xa4 \xf4g\x06\xa6\xff\xfa\xee\xaf\x7f\xc3\xc0\x14\xc4\x05\x98\x04q\xe2\x02\x1f\xa21\x9c\x83\x04\x94AH\x14G\x05\ney\x18d\xa9/\x17+\x1c\x83X\x12\xc2\xbf\xden\xfd\xf5^k\x9e$\x04\xfa\x8aJ\x0c\xff\n\xd5\x1f\rL)\x18\x83)\x86AP\x94\x01\xaf\xd8\x00Y\x9a&/\\\xae\xdbE)\x96\x85\x08\x0e\xe3)\x9a\xe6`\x81gI\x02\xa5Q\x90!\x10\x86\xe7x\x98\xe6X\xfa\x82\xeaO?\x84\xf8\x7f\x84\x81\xe9ui\x1eF\xaf\x1bghB\xe01\x1e\x07y\x84@I\xf0\nH\x86\xc3i\xec\x8aL\x98D!\x1c\xa3\xaf\xcdG8\x90\xc4hT\x00A\x08\x04Q\x1e\xe4\xf9?\xfd\xbb\x1b\x98"4\xccC\x04\x0e\n\x1c/\x90 B\xa38K\xd2\xec\xb5\xa38\xcb\t\x10A\t\x08IP$\r\xf2\x04\x88\\\xff\xf2\xfc\x95\xa8\x94@\xf3\x14{]\x91\xc1\xf9?\xfdm\x03S\x98 \x05\x92G\xafo\xbc\xee\x11&\x18\x98\xba\xf6\x00b\x10\x01cx\x10\xbe\xa2\x11\xfc\xcaU\x96\xe58\x08\xbdp\xe0P\x04!\x18\x9aF \x86\xa6`\x96\xfaJ\xf4\xff\x18\x03\xd3\xff\xcf\x0e\x80\x7f\xc3\xc0tqe\xda~\xb1\xa9=4\x1a\xdc\xd5\xd1\x815)\xa2tZo\\\xb3\x92\xbf\xe4,\x06\xff\xe6\x80\xe9g\xe2\xd7\xec\xe4s\xa9\xd8u\xe9\xd3v\xe1~S\xb9\x93F\xb4\xd0\xe9\xd2\xde\xe9\xb2\x1e\x1a\xaf\xcf\xdf:\xa3\xc6)=\xf4\xbf]\xef\xbaN>\xe4\xe2o\xd7\x93\xf4\xe5\xba\xc6\xf7kE\xaev\xfd\xfb\xf5\xe58\x10\xdak\xf8r\x89~V\xcb#R\xdbR\xff2\x07\xd5\xf8r\xac\x12X8\x13\x11\xebr\x16\'\x92\xf0\x1aln7_\x829\x88\xf3\xcb\x96\xf4\xbc\x1a\xaeX\xec?\xb1h\xdc L\x95Y\xb9eo\xdd\xa4\xb1=\x98\x00\x98\xbe\x1c}g\x80\xdc\xf8v4N\x11\n<\x0f\x83\xa9\xbcZ\xd26\xc2\xee*\xda\xae\x1e9.X0l]\x1c\xe2t\xf11\xb6R\xcc\'s>\x8e\x18Gl#\x0fnd-i\xe6+\x89h\xb9/e\xba\x92\xefst\x7f\x0b\xb7\xb1\x7f{\xe5\xabA.b\x9d\xed\xa9<\xccP\x0e\xd1\tG\xd7\xca\x8bpE\xa6b\x05X^\xc4\xeb?iT\xc5\x8f\xe4\xe9\xd9@\x9a\x89wN\x03\xfd^\xbcg\xe49\xdd\xce\xa6\xf0\x19\x1b\x82h\xaf\xbb\x1a\xe2\x83\x94Jj\xd5t\x19(\xfbW{U\xec\xc1k\x93\xfd\x8dc\xd5\x18\x94q\x84\'\xe4\x8a\xa3\xa5^\xa1>\xe6\xb0\x8c+[,\xcd\'\xecV\xd3\xed\xd4p\xbdE\xdcn7\x92\xf9}\x8dJj[P\x82\xbe\xb6\xfe\xcd\x95S.R3\x926\x92um\xc9\xfd\xc75V\x90&\xd6_kL\x10\x00\x12^\xaa\rF\xc8\x9c/EW\x1b\x9c~\xbfW\x80\xa6j\x90\xb8Qj\xab\x8c:`\x8c\xb1\xc9\xe7\xadL\x9f\x96\xce\xed\x8a\xad(\xf7\xf9m#58\x1a{\xd4\x88\xa0Kp\xa7p\xc7z\x7fR\x0eb%\x19W\x19XZv\x05\x11\x84\x9c\xa7\x95\xcaP\x0cLn\x08F\xc4\xb5;\xf5\xf6\xd2h)\xd0=\xba$o\xda=a\xfcT|\xa6nU?E\xf9\xf5x\xab\'3\xe0\xe1\xfd\x95\x9b\xb3\xd7\xd6\xdc"\xbc\xbdU~\xd3\xe8\xef\xb1\x94\x97\x16\xcc\x07ru\xe7f\x87\xbe\xa1\x8e}r=>Hw\xb5p~\x88%}\x19\xf0/<[f\xf5\x9f\xe3M\x95\x15\xb22\xa4\x83\x0c\xec\xa6\xf6\x87\x07\xa3\x1a\xe6d\xe2\xf7\xad\x00\x9e\xa607\x81R{AY\x90Z8\x8e)\x9d\x01r-\xd5\x9d\x8cr\x12z&3\x00\xaaf\x08\x8f\x11\x05\'\xd5\x9b\xd241R\x7f\xdf\xa7\x9f\xe4\xd9\x8f\xfbd\xe3\x05\x8b~\xedS\xcf\xbcL\xc0\xa8Gsx\x8a^\x19\x9e\xad\xce\x11q\x81\xe1vD\xf0\xb7\xb4\xbaF%Z\xd4=\xbbF\xe5cJ\x88\x13\xd1\x8d\xadY\x83W\xa0\xafJ*\xfa\x15U\xd8\x075G:\xa2\xe2C\x03\xae\x08.[\x89\xc0\xb1\xbf\xed\x13\xc70*\x9eqr\x94r\xc3$^E\x99~j\x15$\x95\xea\x12\xfd\xb8OMt\x10O\xfa\xda\'%\xa5\x08\xaaAn}\xe7\xbf\x08\xa0\xa7YH\xca\xef\x8bq\x02R\x8e\xa8H\xae_\x91\'\xe9\x14\x11\xdc\x8cXL\x9f{,&\x86\x1e\x0e\x8a\x88:\x82\xbdW\x99\x00=\xf3\xba\x02\xe9\xc8\x7f)\x98\xf1>\x8c{};\xb7\xfb\xef\xfb\xb4R\xcfA0X\xf7yR\x95\n\x89\xa2\x9f\x87\x06\xbfl\xe6\xfe\xde\x7f\xcc\xf9uB\xc6\xf9\xcay&n\xf5\x95D\xaf\xfa\x7f\xaf$\xd1\xc2C\x13d\x10\x9d\xc77\xb6\x16<\xab\'\xf3\x04\xc5\xab"%\xe61fn\xacq!\xcde4\x9a\xd2G\xa2N\x05\xd5\xdb\xe1c2\x86\x06~\xd2<\\i\x8dT\xc8x\xe6\xde\xb5\xdf\xf7\xa9I\x16\xf7\xe9\xb5U\x84\xb1\xe9\x8d\x99\xe7\x1a\x90\xee7h\x16\x1a4\xfaq\x9fv`\xa8\x83\xaf}z\xdd\'\xa1x\x08\xa0\xc4BcU\x1d\x82z[u\xe9\x19\x9a\x8b\xd2jc\xf8J)\xb6\xa0\x8a\ns\x0f\xb0\x1f\xe4\xb0\xbf\xe6j\xf1\xce\x9eI\xc1M\xdaZ\xb6@\xfd\xe8\x0e!\xed\xd3\xd11<\x107\xa59\xe6z\x90\xae\x7f\x9e\xf3\xb6\xbcD\xe5\xf2\xaa\xb0&\xa1\\\xa5\xe7l\xed\x05\xd0\xbe.p\xba\xfe\xc3\x1a9\x10HU\xe9k\x8d\xa39\xa7X\xe1\xe8\xdc\x99gXP\xf7\x8b>\x0b\x88h\xce\x02\xcc\xad\x0bvs\x1c\x8ao$\x07\xc6\xd2Q\x91g\xba)cj\x7f\xf4\xb3\xfd\x86\x01l\xd5\xa3\xcc\xe5\x01#R\\8\xce\xec\xe3T\xa4Iv\xce[\xfao\xad\x11\x98n\xa6\x88\x8f\x82\xf5\x02X?%\xb2piVJt_\x06\xf1c]\xe2\xc6F>\xac\xefx/\xf2\xe3>I\xce\xf2 \x1bE\xf4R\xa9\x11K)\xf3\xed\xd29\xde4\xc1\xbf-z\x13\xf6\xf5\xbc\xd1\xf58\x05\xe7$\xdd\xbb\xae\xaa\xe3\xc7\x90\x03qS\x07\x18\xd1V\xc1\xe0>\xb7\xfb\xcc*41([\x9c\x9c\x1e\xcb\xfe\x9d5\x96\xe4\xd5\x9f\x06\x8d\x95Yw\xd1\xcd\xe8\xb6^\x1fo|\xcdm%#\x96\xb3\\**\xbaa\rp}z\xa3\x7f\xafm\xc6\xad\xc1\xb8*\xb8\xcf\xda\x9e/k\xf7\xe4`\xc0\x8e\x1bJ7^\xf6_\xe4L\xf2\xa8\xbb\xaf\xdaf\xf9\x87\x0b\xa4K\xbe\x8a\xed\xfa\xd8M\xe3\\\x8e;\xcb\x15;\x84\x8aM\xf7\xa8\xca\xa4\xb3\xaf\xc1\xd3\xf6\x9ex\x9cf]N\xbd\x08\xd6",\xff\xe9!\n\x19_H\xe8\x00\xb5\xb8\xd6\x83\x04\x9ff\xf7\xde\xb0\x90\x1a\x84\xfb\x953\x7f{\x8d\x7f\xec>?y\xe7\xf6\xd8@I\xb2\xc5H.M\x12\xc4pgu~Y9\xfc\xf5\x1c\x7f\xcc;\x85\xcd\xe6$\xfb\xea\xb5\xbc\xbd\xbc\xd9A\x15\xf9\x84\xf6\x16\xbc\xa8\xd2\x12h\x17t\xc7qs\x1e\xf4\xf9\x9do,\xe2\x0b\x07\x10\xe2\x13\x9bY\xf6\xc0\xa3\xe6\xdd\xdd\xfc\x92{\xf8\xaePj*\x10\xfb\xb9\xbfV\xe6@\xa6\xd7\xbc\xea\xa9\xe1\xbc8\x9b\xf8\xc7\xf6\xe2\xe71\xf7T.\xb2O`Ad\x9c\x16\x1em\xb0\xb3\xda\xf8\xba\x19\xad\xe7\xffE^H\xc0\x8a\x7f\xe7\xae\x83)\x13\xb0\x87\xb1\xd5\xddv\xe2\x95\xf2\xee\x13\xca\xf9n%\xe1f\x82\x9a6\xef\xee3|\xf5cc\x8b^\xdd\x9d\xb7L?\x08\x8d\x86\xd5\xf8\x8b\xacI<\xa0\xc4\x1c<\xbbIp\xee\xe7\xb5&\xf4\x05[\xb7\xa1\xc17\xee\x0f\xee\xc5?r\x9f\xa5\x8b\xf0k\x90\x85!\xbb\xab\xb9\xca\xe3\x9e\xd8\xdb\xb7\x96\xe7\x8an\xfe\xf1>;b\xfa\xf4\xfc\x9e4\x93J\xa1\xdd\xe3m\xe0\xefN{\x95L\xeb\x9e&\x98\x17.G\xbf\x18\xf1\x9d\xe6(\x8dH\xf2\xcb\xe4\x04\xd9|j\x12\xa4(\xef\xb3\x1d\xe3t)-\xdd\xee\x97G\xa6\xed\xa55\x86\x19\xd9n\x8d\x0e\x16\xbap>\xd9\xe3\xb7\xbc\xd0\x1f\xef\xa6k\xde5\xb1\x93\xdc\xd8:+>\xa0\xfb\x11\xd24j\xb4?\xe6\xc5\xf2J\xca\xf1\xab\x97H\r\xd1\xf42\x87\'\x1eV)\xa9\x11?:?\xa5\x86\xc6\xe2oF\x16!o\xb7\xc6\xd0\xda\xd0\x16\xea\xf9\xf0\xa1\x04\xd7gH\x9cg-\xd8b\xb2&\x87\x86\xd2\xea\xab3\n\xc3]\x1f\xdf\xa8\x89\x0c\x9c\xdb\xae"+\x94\xe5o1\xed\xb8\xce\xe3^\xe0h\xb6K\xa5\xcf\xb3"\xd6"0v\x16\xeb\xc0\xdc\x7f\xe4\x8f\x16\xe1V\xf1WLs\xc3\x8c\xdf\xed\xda\xc7\xb0\xfa\x0e\x11\x9d\xd1Y\x17[fID\x80Z\xbf\n[\x8aV_\'\x04l\xda\x8a+jr\x7f\x13\t<\xe9\xafR\xdb\xb4\xa5,\xd8\xa4\xf6\xa2\xc3%4@\xb7H\xc7y:\x08Z7\x0f\x0c\x8d\xfe\xdcK\x00^Y\xd0\x86!u\'\xce\xce\xea\xe4\xe11\xaf\xa7\xa7\x87z3\xffc/y\x01\xd6\xf2\xf8\xda\'\xb0\xf3\xcbw\xbf\x91\xa5p\x17\xb6\xa7\r\xb6\xe5Y\xf2x\x14\xddj\xc0+\xdc\x94\xe5\xd9J\nd\xe4\x0c\x17\x1fd`}\x81\x9aY\xec\xfb|\r\xf2w\xdf\x93;\n\x00q\xb4j\xd4a\x8d\xdd\x04\x9d\xadJ\xd1\x95\xf8wk\xe0ok\xccX\xb2J\xa9\xab\xe2\xf6^\xbc\xcb\xaf\x054\xbb\xe8\x11\x1c\xc4\x1dn\x7f\x8c%\xe4e\xb2\xda\xd7\x1aS\xa4\x1a\xf3U\x10\x95\xd8\xb7\xa3W\xcf\xe8\xae\x04(\xcc.`\x1b\xe7\x03\xf7\x11\xa3\x11\x06\x86\xe3\xb1\xab\x1bo\xc5\xb1\xa3\xce\x8fl\xbaV$\xd3\x028\xdc\xe6\xf4N\xe5\xa6o\xdd\xf4>\x0f\x9c\xe3Z\xc0\xb4?:\x92\xf9\x9d\x97\xc8\xbdA\x11\xcb\xbe\x17\x8bZ7\x84\x169\x8f z\x06\xb8\x0c\x9a\xe7\x8f\xb1\x84$u\xfe\xcdK8\xc88\xb4\xfaaP\x82\x89"\x9c\x15\xd0\x16\xa9\xcc6\x87\xa4\xd2L2)\x87\xd8Z\xf3\xc4\xd2\x95\xa5\xfa<\xb3\x9b\xe7\x1bO\xfd.N\xdf\x12\xdb\xf1\xb7\x8a\x02\x10L\x16=\xb8\x98\x15X\xe1\xc7\x99x\xadg\xf3\xd6\xed\xdfc\xc9{\x0c\xb7}\x80s\x8b\x91\x8d\x91\x13\xa1\xd2 \xaa\x0c\xbd\x05c\xa5\xfc\x05/)\x94!\xbe_\xb1D\x87-\x93\x1a\xc7\xc0d\xc1!9C}\xcd\xc5:T,K\x04k\xca3\xd4<\xcd\xdd^\x05+\xc9\x18\x7fG=SV\x80\x07-W]\xa1\xd7a\xec\xf4\x8d\x7f\xe7\xe4\xf09\xbf)]\x0c\xad\xbe)L_\x05\xef\xc4&\xff\xbc\x86{\x08|\x1e\xcd\xed\xc1\x0b\x076\xac\xb6\xc8\xd9\x88j\x1cU\xa3]\xe5\xf9/\xd6\x88\xb7X\xfb\xc5\x9d\xa6\xb4\xd52&\x11\xcb|\x9dj#\x18\x1d\xcd\x9eiL\x82A\x91J,\xbd\xe7\x00A@_\xd5b\xb25\x12\x9b\xd6\xdd\xbc\xa6\xa7y2\xa5\x93!6\x9c\x06\xe8\xcd \x07OO\xb98\xb0\x81\xd8\x04}\xb0\xcc\x02\xd9\xfc\xf9\x1am9Z\xae\x81\xf6V\xca\x98\xcf\x85\xc1CI\x80\x10v\xe8\xa6\x06\x03\xe6\x875\x9ax\xb1\xbc\xbef:zC\xbc\x1b\xc1+\x00a@\xe2\x91v)EeB\x86\x1b\xb9\x96\xc16\xfc\x06\xdb\x8b~?&\n?\xc3\xb5a\xddz}\xfa\xc5#\x8a\x82\xc076D\x0c\x1f\xf7ZK\xa4p\xf0\x9a\x14"d\xf8\x86\x89\x89\xda\xb3\xa8\xf2{\xbc\xb3\xd3n\xbb\x07\xc5\xbdo\x0f\x99\xdcV\x0c\xe3\x14\xd0\xac5\xe4\xda\xf8\x1f\xe3]\'\xdf\xf87w\x9a\x95\x03E\xa2t\xf5(\xb3\xbbeol\xe3\'\xc5hqx\x15 E\x1f\xdb\x06\xd2\x92\xf3UV\xc2q\x9e\x89\xc2\xfaj4\xa0\x0fS_\x19\x87\xd6\xee\x9bTjF\xd7r\x1c2\xadx\xcdAEV\x95\xf0k\x14\xd8\xdf9\x85\xe6Q\xdd[L\x1c\x0b\xd3\x92\xed\x8ep\x07m\xd4\xee\xfd\xf1\x1e\xde\xf4\x8f\xf1>\x11^\xd3\x94W\xbc\x9b\xc7x\x80M\x9aU+\xcd\xed>\xe4\xdf-/\x86\xdc663\xd2u\'\x14\xc98\xdd\x87\xe7\xec\xb1\x18\xa0\xe2\xe4\xd5\x01\xe1\xa4\x07\xf3\xfd\xc3\xd3}\xacH\xf1{\xcb\xe8I\x87\x16E\x97\x8e#\xea<\xa47)\xf6\xff`\x0f""\xc9\x94\x9f\xf8\xd5\xf1\x9a\x00\x01\xef\x1a\x9bov\xb2\x8b\xbe\xff\xd8\xfe\x82\'7X\x12\x7f\xe3\xb8\xdeF\x00-$9\xc0\xf6\xd2\xf5\xdf\x92\x95\x9e\xa7\xdbs\xa1\xc7\xee\xdd\x13}:\x01\xe7!R\x91\xebp\xe1\xf1\xa9d\xd7\xc2\x83\x1asL\x1dG\xfdL\xd01\xb3)\xa2\xce]p\x01V\xb5\xe8\x06Ky\x8c\x19\xc8\x1d?_\xa3#\xf0\x02GF\xa0n\xa8\x16\x91\xab\xcd\xe3\\\xaa\x08\xb2$\xac\x1c~\\\xe3\x08\x0fx\xfa\xb5F\xa2Va\xd8x\xda\x11Ov\xb7R\x89\x0c3c@\xba\xbf&\x88u\x0f\t\xceOn|\xea+\'=\xd67\x15\xd76\x83\x95\xfa\xe1\xe1Q\xee\xcbOg=\xdf\x07\xceIE\x8d\xe4C\xa3 c4#t\x8dp\x06\xee\x0fr\xd0\xdf\xf2\x9aSz\xe3\x89<\x94\xcd\x16\x1c\xeap\xaf:r\r\xde\xc7\xaa\xda\xd6\x8f\xb3\xab\xaa6\xc8\xb8}\xe5L\xdf?tK\xce\xfa@?\x81\xf5\x15x\xd3m\xe1\x89\xbe\x0ck\xad\xa9\xce\xa8&\x18\xba\xbe\x19c\xb4\xa1\xa9\xde\x1e\xc9\xabn\n\x0c\x80\xa5\x95\x1c\xb5\x9b\xec\xa9N\xe5\xd4w\x96\x02\xd8\xe0\x05\xeb\xe7\xb0`\x0b\x07\xfa\xed\x9fu\x10\xe6L\xea\x07\x7f\rzQ\xe6U\xc0;-M\xe7x\x0c\xe8Sr\xffb&RN\xf2\x18\xbfp$[\x19\x07\xa1\x02Z$z\x7f\xbc\xf2iH\xd7\xe5u\xf3\xde\\\x11\xc9\x86\xf6\x1a\x16qn\x14\xe5q\xa6\xb6r\x83\xca)\\pM\xb7\x8b\xbd\x1f\xaf\x91\x97\xe5\x81\x00\xb9\xc1K\xa2\xb8\xc6\xf4\xacb\xb7B\xb1\x82\xa1"\xfaw\xbeA\x97\xf2(\xbf<\x00wTaS3\x81\x10^P6g\xa7~\xcd\x1b?\xe4\xcc0\xdf\x84\xe7W\xce\xf0\x99\xb9\xca\xd7`\x99j\x950\x89i}\xcf\xc1\xd6\x10\xf2\xd2,V\xc0\x1c\x96\xd9\',x\x92\xc1\xd3\xac\x1f\x8d7"je""\x866\x9c7\xa9F\xd37\xefJ\xc5\xe8\xa3\xeae\x16\x06\x85m`\xf9W\x9a\xf9\xa5\xfe{m\x83\xba\xa5.\xe2\xdcNq\x96\xd6lx\x1b\xab\xa5\x03\x9e\xb94\x9f?\xd6_3eo\xb2\xfd\xb5O\x82\xc5\x04\xe0C\xca4\xe4!\x03\xd3c\xe0\xa5\xbeA\xca+[O}\x07=\xd5>}\xe5\xaaM\xaa\xfbn\xd0\xda\x92<\xebv\x0f\xed\xc6\x195v!h\x80_\xc7\xb5X$\xe3\xc5\xa6A\xaa\nRe:\xafp\xb9\xff9\xdeem\xca\x0b\x13B\x86A\x00t\x9f\x9d\x99\xfb+\xb4\xbdDc\xd6\xbf\xc8I\xae%\x8f\xfek\x9fX{\xd0\xb3E."\xe8\xedC\xc4\n\x12Z\xee\xecc\x89n\x0f\x0b\xb8\x85\xaeI\x86\xe5\x15\xc9\xc0\xb1\xa1\xc1:\xf2\'\x9d\xfb\xb7\x03\x83\x14\'\t\x19\xf9\xbd\x8f\n\xc50\x05\xfe&\xfdw\x8e)\xba\xe4\xa3_M\x99\xf97\xf8F\xe3\xa3\xb3=\x85.\x90\x97\xa5u\x8c@\x1b\x9a\x90\x11u\xae\x0f\xff\x98\x93\\\xfe$\x83\xef\x9c4%\xf8\xbe\xe1\xe5\xd3\x9b\t\xccTk\xb5=^\x94\x04\xebZ\xa9}\xe5m17T\xe1\xdf\xc0\xc4\xdbCE\xac\xe0k^\xe5\x80\xe1y\x9f8j\xd3\xaf\xde\x8dr\x15\x97\x05\xdd\x8d\xfdz\xb5\x8aJ2\xc3(\xb8\xa5\xf4wg\xd7\x9f\xcfK\xfc\x1b*\xcd\x0e!\x1f \xef\xea\x9e\xf6B\xd5\'\x88\x88^L\xf9\x7f\xa9S<\x81h\xf8\xea\xb5\xbc>\xa0\xeft\xb4\xac\x84\x8c\t\x01~X5\xee\xa0\t\xcc\x12\xd1\xeb\x91\x98\x19Z\xdf\x1a\x80V"\xf4,7SK\xba\x19\xce\x03b-\xe2\'\xa7E\xbc\x1d\x0cS\x84 i\x87\t/\x8cP8\xc5wMfo[\xeew\x1c\xab\x07\x1a\x80+\x1b+\\\xc7\xde\xadW\\\x8e\x14$\xa2\x0c5\x8c\xf2\x8f8\x02\xd8\xe3\xd5~\xe1\x18\xbb\xbe\xf1\xf2\xd9\xaa\x85\xe1qL\x18$\xae\xa2\xd7\x99\xe8\xd0\x98Q\xaf\\\x9d\xee\x19\xa3\x93\xe0\x0b\x88\xa4"\x8b\x11P\xfb\x95\x93\x16c\xb3\xf7\x9c\x1ca\x14;\xd0B\x8c5\x85-\xe2\xc0G\xf5\xfeU\x9e\xa1L\x82\\SwH\xdd\xb4\x8a4\xeb\x01\x00\xe1\xf3\xd2\xb0P\x0e\x10c\xc9>c\x98\x1a\xc42\xee\xc3e*\x1c\xabZ\x18f\xc8\x1d\x0b\xfe\xb3\x9e\xe4R[\xff<\xa8\x1a%\xe8w:1\xca\xea\x0e\xce\xc8\x86\x16zS\x7f\xac\x9d\nn\xddF\xf2k\x06\xe8\xf2j2D\x06\xf1\xdfv\x11\xed\x1d\x15\xdb\xb0\x12\ra\xe0\x87\x18\x8fU]p\xb5\xf5\x8b\x1f?\x016V7M\x9a\x0c!\x89\xbaP3\xd8\xd6\x18\xa4\xba|\xe29\xd4)\xab\x86\xef*y\xcb\r\xe4\xe6k\xc4\xb2Y\xfc\x02\xd6l;\xa8\x02\xfb"]\xc1\xdd\x87,xU\xb4\xc5\xe3\xc3\x1d/\x93\xabNf;\xcc\xd7\xb2*3S>\xcd\xb04\xde\xe0U\xd5\xde\xcf\x95\xbb\x9f\xab\xc4\xa4\x08q/\x993N\x1fA>\x84;\xd4]\xac\xa8\x10u\x97G\xec\xb2$\xd1\xf08o\x9f\x7fh{\xbc\xc6\x02w\xaf\\0\x04\xfa\x042\xd9\x97\x95oYw\ns]\x9f>rU\x87\x9a\xad\xb9\x99\x19\x8d\xb7\x1f\x1b\xaf\xa3\xdbxqk\xb3\xa2\xb9\xba\xec\xc9t\xab\xfb\xee\xde\xe5Z\x87o\xd6a,\x1c\xfdr\xf4D\xd2V\x90\r._\xec\xbb\xea\xd0\xc3\xbd<\x8d\xdbKW\xbbuW\xb912\xf9M?\x17\x96\x11\x9dg9\xb8r9\x94\x82D\x94H\xc0\x8c\xc2\x06\xf6\x87/\xa3X\xd2a\x83\n",|5 7\x1d\x98I@\xf2\xec\x9d\x9b&\x1d\r\x05\x01\xd0\x07[\x11+\xca\xb1\xad}\xe1\x13\x80\x1c\x15,\xea\xacL6\xaa"\xce\x9a\x1a0\xba\x9fgOw\xdf\xf8p"Gb\xd0v\xd6\xe1\xf62\x87+\xe9\xb5\xd6\xcaS\xbc\xb9x\xfe\xa3Z]^=\t\xd7\xcftP\x8cW\xd1\\\xed\xd2\xd6K\x9cb\x19z\xfb\xc2\xc5y\x1b\xf7\xf8\x84\x0f#H\xb8\xfd6\x89b\x18\x0fk\x93\xcaV\xb2\xee_\xb8\xb0H\xc36\xfe\xb8{\xab\xc8\xe3*wT\x0ex\x86\xe8[-B\n8Z4\x9f\x90\xfa\xa2\xfd\xc2\xd5\xff\xa7\xaa\xee\xc7\xf1\xcak5\xe9\xaf\x14\xfc\xc6E\x8a8\xaa\x12j\xd9\xa7\t\xde\xb6\x00\xfe\xaeo\xd8\xf2b \xf6\xd9~\xe3\xb2d\xc1\x9d\x8bv\\H\x9b\xe9\x91 C%\xc7\x96\xec-\x19j/\x8f\x01\x81iK\xf4\xe5Z>c{\x97\xa39\x12]\xbaf\xf72\x96\xd9o\\\x98*H\x04\x18\x9d8KS\xd3~\x07\xa1\x8d\xf3\xf6\xd8\x91\xee\x95\xfd\x8d\x8bY\xb9\x99\xd8W\x18\x01\x00J\xf1Z\'\x1c\x96\x87\x02\xb4\xf4\xf7bz\xe5\x10$q6\x17\x015O\xc1h\x8a\xfamN\x0e}\xd1\xdc\xdcA\xd9\xa9\xff\x8e\x17\xfa\xd0\xa5\xbd\xf6ZS\x95=,\x9c\x84|st\xf2h\xb4y\x0b\xf5\x88O\xbc\xc8a]\nA\x00\x1eG\xf6\xe6\xbd\xbfJF\xbe\xda\xe6\x8b\xe6\xbfq\xd1n1\xcexv;\x92\xb7\xc8};U\x17\xfa\xa7\xa3\xcd\xdc\x9cf\xdf\xb8\xa4\xed5\xee\nj\xc0\x89\xe9\xed4\x8a\xfe\xd9\xdd\xe3\xb5J|\x90\x90\xd9d\x1aH\xddc\xbbS\xad\x14\xc9\xcf"(\t\x12\xda\xe5\xfc1\x1e\x98o\\dxjG\x10\x93\x03\xd3\xeeo[\xaa\xc9k\x05YT\xcaD\x80\xf8\xc1%\xcf\x1f\xe2\xae\n\x15\xe9\xd14\x13p\xb4\xfavo\xef-S\x08\xf6]\x87\x9e\x10\x0e\xf1n\x0fcE\x02\x17.5\xd3\x95|y\xc8b\xf0\xc1\xc5\x89\x96\x87\xe2<\xd2\xab}\xfa\xd9\xed\xc4{\xc1\xba\x91O\xa6\x87E\xc2\xf9\xce#\x1b\x18\x92\x1d-\x8e\xcar\x8c\x8c\xa1\xacL\x89\x04.&n;\xb0\xa2\x1dn\xd4\x92\xaa<\x82AC\xba\x96\xa9M|@\x19Y&^M\xc9\xabv9\xfcL\xb7\xfd\xc6e6\xe2\x04\x0f\xcb\'\x8a\x19\xe2\x1b\xe0D\xd3\x06\x9dh\xe3\xbd\xbeG\xdf}Ug\xde\x9b\xec\x0b6\xaa\x06\x14\x8d\xce\xc8!e\x1d\xe6\x1f\xf4\xc8^\xfb9!ok\x1cvn!\x1eo\xcaK\x05\xa4\x1em\x0f\x1d\xe1p\xfe\xac;\xf2-\xb2\xc4]\x8bRp|\x19\xba&\xac\xd4\x9b\xf6\x8c\x1e\xca\xadi\x9fp\x02%\xa8&\xf4Og{\xe7N\xc3\xa1\xc8\\n\x80A\xb0\xad\xac\n\xd6\xebvV\xcb\xcb\xf1\xf0\xb5\xec\x02\x00\x11\x88\x81\xb89\xed\xab)\xa0o\xbc\xe5\xa5\x8e\xed\xa7\x03S|\xcb2\xebZPh\xd4\xbcq\xfb8\xa6\x82\x83\xa7\x15}`\xbb\xe8\x95\xf42F/\xd1x\xe9Y\xcd%\x8d$\x7f\xe3\xd16\x8buzl\xb8\xd4w\x90\xbd\xe2\x0bPsj\x80\x02\xe2\x10?x\xd8^\xc1H-\xbe\xba\x92\'Am\xeb\xcf\xc3\x16z\xcb\x1b\x17\x81.\xd7\xef;Q\xb0\x1a\x0cx\x84\xe0l\xf7\xa8\xd3\xa0EV\\*-\xbf\xf0p\n[~\xca\xc6<\x10)-u>\xf0dp\x89\x98{\x9b\xe1\x1f\x9fz(\xbc\x9ck \x00\x80\x9eb\xf3\xd4\xbe(\xfe5\xc4\xde\x19*me\xaa\x14\x02\x8f\xd0\xf4\x91CZ\xc9\xdfF\xa7\xae\xd0 +\x87Bx~\xe3A\x8f\x0fT\xaf\xd8>\x82*\x04\x91:\xd1Vi\xd6"\r\x81R\xdao<\xa4\xf7H\xf01\x7f\r\xe0\xd6\xe8\xdfGr\x1f%h\xd9\x19?\xad\xee\xd4\xf3\xe9\xe8fM\xe0\x0bw\x86W\xaf\x14u\xa4\x8ej\xce@\xfb\x0f\x1e\x08`\xb5\xa6\x10\xbb\xdb*\xc08\x10dV\x8a1\xbd\x05n\xf0H\x7f\xe3qq5\xcb\x94\xc9`Y\x02A\xa1\r\x99Wba\xb6\xb5\x9a\x1c_\xc8C[\xcd\x82) \xd05)\xeeAG\xb74\xcfe\x0bFB\xfb\x1b\x0f\x8c\xc0k8\xd4\xa4\xfd\xd5\xb7\x88\xf9\xb6l\t\xb4\xdf\'\tx\xe5\'>\xe4\x80\xea\x86ZMBi_\xb0P\xe5N\xdfJ\x98\x80\xbf\x06\xac0\x17\xc7\xdb\xbc\xcbMO\xf2\x87v/\xf7\x06\xc0\xfd\xad\xbc-]\xf6\x89\x8f\xda\xd3\x8f0,Q\xe0\x91i$\x0e\xbeK\xc1{\xablj\x9e\x9f\xfaM\x0f(?\xfbT\x19\x00\xe0\x00lp5\x9b\'3\x9aF*\xa9\xb7\xb3qk\xddwg\x88\xf1p\x83\x0c\x11Q\xe1\xde\xbe\xcb\xda\x84\x7f\xff\xc6CUoO\x88\x95\xef\xa9\xd9D\xbd\xa1+}0\xe55\xf4\x9a\xcc\x97\xfd\x8dG\x1d\x96\r\xb9\x92\xe9c\x95\x15\x13xZw\x07(\xe3\xd6\xab\x88\x07wq\xf1i\xb0\x84\xb85_\xc7N\x89mD\xce\x90\'[\x9aq\xdf\xbe\xf0\xf8\x99\xf6\xfd\xdd\x17\x0e\x81\xe47k\x0e\x96(\x01l\xeaU?5u\x0c\xdfK\x90\xd9\x01\xe9\xe2\x86\x12 1\xa3\xd9c\x1f\x1dU\x1d/\xd7P\xd2\xa7\x95\xfd\x8d\x07\x07\x90\x10x\x93"m\xedP\x19\x1bX\x87\xe3\x0c\x03V\xbb\x9c\x8e\xbe\xf1\xb8\xa6\xb6 9\x14\xc1\xf2 s\xafe\x9e[\x94\xa2Wu\xc1\xc6\x93\xe0\xdd%\x05\x99\x06\xb8\xf3z\x9d\xfe\x1a\x8b\xaa%x.\xdb\xecw\xf0\x1b\x0f@#\xab\xbb(>R\xff\x0el&\xc6+\x00\x9b`:)\x07\x0b\xf9\x8d\xc7N\x86\xb7\x89UR\xe1\x9e\xaf\x15aS\xb1\x8b\x8a:\xff\n\xc2\xa7\xeb\x80\xd1\x05~\xc2\xf6C\xd9\xa4\x80YF\xd4B\xa9\xb2\x95\xae\xeew\xbe\xd8\x18l\xfa\xcd\x96\x90new\x0f*%\x9a\x0c\x90!\xa8\x01\xf5\xa9\xfa\x8e\x8f\xa6\x04\x16 \xbc%\x14\xe3=\x0f\\\x14\x8d5\x0e\xf30\x11\x12\xfb\x9e\x87aBW\xa1%X\xd6YM\rSo-\x18\x95\xb3b\x88\xdfx(\x87c\xbf\x0b\xc0\xef\xb9\xec\x04z\xddhS\xd0\x9cH.\xe6!\xef\xc3\x7f:\x8dC\x16>\xb0\x16\xdao\xc3\xae\x82\xc1\xf7QB\xcc\xae?\xd4\xc5k\x95<\xaf\x05\xf0 \xd1\xb71\x81\xa2\x82(V\xcd\xe6\x87\x87~\xe31\xfb\xd3\xb2\t3&\x06$\xde\x1dT\xb2h\x8f\xab\x1c\xe9g\xca}x\x89\x89M\xc5\xeb\xa8kl\x8fs\x93\xed\x1f\xa8\x89\xc3\xc0\xf6~-\xcf\x93\xa5\x1fP\xee\xfa\xc1~\xd3\xfc\x19(\xec\xa8\x89m@\x96\xe1\xbb\xf5\x8d\x87[K:\x04\xcb[B(\x9dx\x0c\xed\x89\x0b\t\xe6=\xe4\xbd\xb1\xbe\xe3\x83\xc17\x1c\xa5.>\xd2a\xbc)\xf5\xd0\x96\xea\xd5v\x9b\x851k\x90\x16S\nBt_\xb1\x1d\x8d(\xac0\x8dv;\xc1\xf2\xb8\x91\xd47\x1e|\x01\xb2S\x1a\xb8\xefw\xceF\xd0\x03\x9a\x18\xa2H3\t$c\xea\x1b\x0f\xdd\'v\xc8p\xa675\x85\x1c-\x1d\xac)\xd9y\xc4M\xaek{\x91\x040\xc7zhS\xe9\xb0\xeb\x89\x89\xe6\xb30]nD\x9a\xf6\x1b\x8f\x82?m\x133\xb2\x07\x87Av\x84z\x05Zj\x83\x9b?\xf1q\xa2\xab\xe4\n\x9e\xec\x15\x16\x82\xbb\x903\xc4\xd4\xd6\xdd\xc39*U\xea\x05\xf5\xb4\xe3u\xbd\xf1\x90\\\x85r\x1a\xbc\xba\xf0\x00\x06\xfb\x83\x87\x0147\xb1\x89A\x92/\xe1VO\x03\x1aY\x84\xb3\xa1\x92\x9e\x7f~\xe3\xa1\xc5T\xa57\xdeR4\x82Y^\x05\xe2\xc21\x0fu\x0e\xeeB\x89\xd4\xe4\x92J\xab\xeen\xf1zM\x9d\xbd\xa8\xa5\x86\xc1\xb2\xf0\xe9\xd9\x1f\xfeq\x0fu\xdc\xcc\xcf\x04-\x0c\xaber\xaf\x85\xa7\x1a\x1d\xb3\x9b\x16}\xf0 \xe5$\x89\x0bTj\x8a\xb7\xb3\xacmk\x02J6\xbf\x8cY\xdc\xd1\xd7\n\x13\x94\x9f]\xa5\xaa\xf2\x08\xcc\x8ePr\x06e\xf3j0\xdf|\xecg\xba\xfc7\x1e\xbe\xdf\x1b\x93Il\nYc\xf7\x84\xe7\xf0"\x8f\xeb\xa6I@\xcc\x02\x8c$l\xe2\xf9\xe9v/\x17\xf6\xd4\xbd\xc6\x0e\xcf\xbef\x89\x16\xfc\xc6\x83\xed\xdc\xc8\x01\xa9\xd9\xf0\xf1\x15F\x1f\xd9#p\x1e\x8aQ\x81\x8dZ}\xf0\xd0\xe2w\x82\x9bk\xe2\xacf\xbb\x80\x88\x87,\xe0\xca^\xb3^\xde\xde\x84\xfbA\xbdq\xff\x14\x9d\x14\xde\x89Z\xd4$\xe9\xc2c\xdf\xcf\x0f\x1e\xac\xa8-9m\xa1\xac\xe9\xdb\x10\xf4<\x82\xb5\xa3Z[F\xdfB\xf9\xa9\xa7\xb6\x01RCv.\x81\xe5\x13Q/\xf3\xe6\xde\xb4b\xd9\xc5\x0f\x85\xa6\xf2\x80\x16Bm\xc4\xf1\x95\xb8\xf66\x022\x19\x94\xad\'\xfb\xc1\xc3VW\x81\xd14\xd0D(\xb29\xd9\xdb\xb3\xd5\x9f6.\xaf\x05\xb8\x7f\xe6\r\x12\x18\x96\xa7v\xdfz\xcb\xcfd\xb7\'\xdd\xf91\x9d\xb7\\\xf2\xd9t"\x1e]\xffh3L[A\xe1\xe4\x99Z]\x1b\xbb\xac\'\xf4\x83\xc7O4\xf4\xcf<\x90\xa1\xed\xf8\xca\xa1\x92vT8\xa7n\x8b\xce*\xd4\x86\xa5t\xc2\xad\x932S8\x98yw\xb3\xc0C\xad\x11\xcd\x024X\xee\xf9\xe6>x<37\x05Y\xed\x8930\x89\xbf\xbb\xfev\xf3\xb0\t\xc9\x04\xed\xf8\xf4\x97\xa2\xd2\xccL}\x8c\xb5\xf6\x9c\x9b\x1b\xca\x81\x00$\x05r\xee0\xe9\xba\x07Ka\x84.%N}.\xba+\x18\x11/\xfe\x8a\x0f\xbc\xfa\xe0\xe1.\xa4\xc9\xf9PH\x11\xce{t\xe5\xe7\xd9]\x03\xda\xfe.\x06\r\xf8\xc4\x07\xfa\xa8\xa0f\xbd\xe9\x13\xe4Y@\xed#\xb3\xbb\x05\x9b3/1\xd3O0zd\x81\x0c\xcf\x14\xda\x1b\xef\'\xd3\xac\xd7XR\x12h\xf9\xc1\xc3\x84\x99\xb3V\x80\x01\x98e\xe8M\t\xec\xf6\xccxqv:?$\xbf\xf1\xe0\xd3\x08{h\xfbS\x05(\xf4\x89\x8f\x0fG?x\x12\x9b\xb7#\\\x8cD(\x90\xd5]\x1c}\xdd\x9c\x8d.E\xe5\xc1\xea\x07\x1bn\x8d\xfe\x8d\xc7\xd5x}Ka\xa6\x01G^\x87\xd7\xed1\xdf\xf8\x17\x81\xa5O\xf8\xc3\xc7\x9a\xb5RW\xcc\x9d\xce\xe6m1U\xffn[\x97\x97$\x89OC\x8e\xd1_W\xd7\xd0\x0c\x95\xe6\x1fz\x90e\xd1S\xd5OY9\x1a\xfd\xbb\xbf8\xfc\xdd\xbe[{co\xe0p{\x83/\xe3f\xe2Bs\'\x1f=\xfb\xc1C\xbb\x83\xbcx\x16\xc0\xdb\xbb\xd80Q\x11\xb3\x16nc\x95\xa1\x19g\xad\xf7G\xecT&;Q\xd9\x88\xb0\xcb^\xef\xd4)\x97\xf3#;>x\x80\xd7\xb2\x07%}\xb4\xd0\xfd\xdc\xc6\t \xb8\xd16\xe4\x88\x82\x8f\xbe\xa0#\xd2\x13\x08\xc8\xb7\x9c\x1e&>\xb6\x9b\x85o\xde\xe6\x07\xfeI\x87\xe3\x8c"\xd8\x91o\xbc\x89\xdf\xcc7-\x1a\xe9\xaa\xd7\\@\x9c\x1f\xbeN\xd4/-\xf0\xab(\xe5\xbbTz\xae\x08am\x83\x17\xe9^\xca\xdc\xbf\xf1\xe8\x93f\x99\xdf\xc2\xc5\xc7\xa9\xa9\xf5Ox\xefn\x94\xdfD"BE\xd7\xd8`\xa4MY\xbem+\xa6.\x14\xa2!\xd17Y\x15\x1b\xfdS?~\xa2\xff\x7f\xe7\x0b\xfe\xba\xda1\x1a]sDl\xc7S\xde7g?\xd9cy\xd1C\x94\xbf+\xeaK@\xa08\x979\x8d\x9b\xf6\xbaC\x8f\xad\x1c\x0b}\xfe\xcc/\x19<\xa6\x00\xe5\xda\xec\x89\x9b\xca\x0e\xf6\x1f>\xa62X\x1e1\xeeLe|%e\x14\rS\xc9\xee\xc8\x0fQ\xba\xfe<\xbe\x12I\x9373G\x81\x9c\xe7/<"\xf5\xe0\x02\x98\x9d?\xf5\x83\xae\xdbz\xb4\x88\x91\x1fBz\xdc\xc29\x98\xfcc?\xf5\x93D\xbf\xf18\x0b,\x83\xb9\xf5"S\x118\x96d(\t\x92\xba\xe0\x16\x90\xd7>\xcc\x84\xe5\x82\xef\x92\xf7\xca3h\xdb\x87\xab\x9e\xde\xa7\xab\x9e\n\xeaw|\xfcLg\xff\xc6\x83\xbb\xbb(l+\xb9\xb6w(\xc4\x8d\x92X\xef\xba\x99\x8fD\xd7,\xa4\x95\x987\xecf\xe7\xcd C\xbcX5\xf3\xc0\xb6%\xa1^@\x7f\xd7S\x16G\xc2\xe6\x9d\xe3*\xf3h\x04{\xa2}\xb5\x03y\xb8\xde\xd2O\xbe0\xe3\n{\xc6x\x1fP\xb2\x1b\xf5\x86\xae\xbb\x10\x81W\xdc`|\xc6\xe7F\xf3=r\xe5\x11\xe2\xcfYy\xe8\xa2\xea5\xca\x15uW~~\xe3a0z &r\x1f\xaa\xf8\xba\xeaK^\xeb\xa7\xf2"k\xfb\x16\x7f\xea\x87\\y\xfe\xf3\x991\xa4E,ui\xbb\xceK(\x8e\x86\x1f\x92\x96\x804\xc8\xc7\r@|\x01U\xcd\x1cR\x1bu\xa4\xdd\xc8\xca\x83\x97?|\xcc\x08\xebk\x0fs\r\x83Y\xfa\xa6>q\'#\xeeP\xed+a\xf2\xc9\x97\xd7\xed^\xdb\xb5\xb8\xf7\x8f\xf0\xb9T\xfbTm\xa9\x8aZ\xd4\xc9\xa6"\xb0\x9f\x89-v\x85\xac\xf0~\xb1\xdf\xbf\x9aR-\x97\x070\xd4\xdfx\xc8a;l\xb3E\xbe*t\x9bB\xed\xf1j\xab\x87I-\t\xab[\x9f\xf9V)\x0b\x92\xa6%o\xba\xa66\xc9\x17\xaf\xe1\xcb\x91\xd5\xdd\xec\x9bwK\xb6\xcf\x9a\xcc\xe6\xc5\xea\x9ee\xf1\xeaD\x030e\x97{4\xf5G\xffPZ\x0ba\x11,\n\x07\x00]H"\xac\x905}\xd2\x110\xeb\x9f~\x1b\x1e\xccc\xe3\x9d\xfd\xc40\x91\xc0\x08_\xbaZ\xec[\xc9n8\x8fcC\x17\x18\x06\xf18\x03\xa1\xc7W\x86\xbe\xe6\x97{y\xcd/\x9d\xfc\xa9\xa7?\xd1\xe6\xbf\xe7\x97eP\x1ef\xef\rd\xd1b\x92\xc0\xd8\xfc\xba{\x8c\x9bb\x91\xefr\xcf\x88\x94\xd7P3\x146\x03p\x88\xa9\x0f\x88\x01\xcb\xf9m\xe7\xbf\xe1\x01\x08om\xeb]\x1b\xb3;\x0e\xef#\xf5\x91;\x0b\xfdHT\xf7\x83\xc7\xcb\x89X\xafw:\xaa\xa0\xe4\x02\x05\xaf\xe1Wd\tBJ\xa9Y\xcf\x08\xfd\x06e\xdd\x90c\xb4\r\x99\xd7\xfc\xa2\x19\xa2\xcb\xba\xa7\xf3\xe1\xeb\x14n\xd8\xf7t\xc80\xeb\xe9\xbf9!\xdc\xafN\x10\xe6\xad\xea\xdc?\xfc\xd4(<\xc3x\x80.\x16\x91b\x81^\x9b>\xae\x05\xc4d\xb2\x0e\xc3\xf38\x0b\xf0\xaa]\x94;\x12\xaa+w"P\x8db\xd9`{\xf1\xc3\xc7~\xa2\x81\x7f\xfa\xed\x8c\x84tV\xdf\xee^\x04\xf7Z\xda\xbb\xa86\x834\x8c{\xe2$\x87U]\xbb\x84\xaai\'1/mU\x03\xe3^\x96\xeb+\x8b\xf8\xff\x08=\xdbi\x9c\xdb2\xa0b\x19\xc9t\xc5\xd6C\x97\xd5\x84{1\xef/=`\xdae_\x13\xab=y\xf14\xe9\xbd*\xef\xc5Lj\xcb\x9c\xd2\'~\xc47\xbc\x9cQ\xf1\xbc%\xe4\xb0\xc0\x05ix&\x99\t\xeb\xe3\xed\xd2\x92\xd1\xbd\xa9\x18\xae\x1eRp\xe7\xb8\x1c\xbd\x08\xb9\x97\xb3,wV\x0f\xcd\xfa|3\xf9\xd1\x175Q\xd0\x9c\x07\x1eOwT\x1d(\x849B=8T\xcd\x08\xf3\xdft\xd7Mla\xc8\xbf\xa8\xa2\x1a\xd2v\xca:;\xae;\xf3\x95\xfc\x01\xa5\xc2W)\x04\x94\x02Tx\xfe65\xa7\xf8\xd1\xd0K\xd0\x801\xe6\xa3\xd3_\xdd\x06\x18\xb6Wp\xe3\x07\xb3+Q\x8f>MT\xe6c\xb1\xf8\xf49\xfa\xfeX\xb8<\x12\xdb\x90sS\xe5\xde4\xdd+\xec\xcb\xc9\xcb\xb4\x84)\x14e)\xc59\xcc\xa1\x03\x02Q\x94\xa9\xbf5tY\x95\x96[\xa9\x7f\xeb\xae?9\x0b\xfe\x8d\x8f\x1eY\xe73\xe4\xcc\xab\xad\x0fx\xa9>\xa3\xc5\xa6\xa0\xf5e\xee\xf4Ui(\xd9\x9e\xac\xb7\xe2o\xa7\x1d\xf7d\xf4\xad\xa1\x1f\xdc\x04\xca\xfcG\xa7\xff\xc9\xd9\xee\x0f\xef,t\xd5\x08\x9e\x82IF\xde0X\xaa y\xc6\xa3\x06G\xec\x81\x10\xaaW\xc1k\xa1\xd2\xc8|go\xe2"~4\xf4r|%\t]~\xeb\xd1?9\xab\xfd\xad\xc7\\#\xbcm4\xd2\xe0/\xe65\x18\x175_\xb5debg\xc5b@\xf3\x88h2\xbe\xd7rct\x1c_\xd5\xdf\x1a\xba\xac\x89L\xfb\x9bN\xff\xb3\xdf8|\xe1\x82\x0b\xca\x96$W\x07\'\xd0\x0e\x7f\x01\xd7\xa4vak\x93\x07\x8cb&[\xc5cw\xecX\n\x1f\xef\xa4E\xa3o\r\xdd\xe5,\x01\xd3}\xc4\x93?\x95s\x90\xe4\x12\x0bA\xa2n5\xc5\xc1}\xe9\xe9\x1f\xfd\xee\xef\x9f\x89\xfe\xe8\x99\xb8\xd2\xb1\xb9\x94\x82\xae-\x8b\x00E Um"#\xa7\xa9o\xd4J\x96[u{\xf2\xb1I\xb3xW\x0f\x11<\xec\x87\xfc\xa5\xa7\x7f\xf4*\xd7Do\x86\xae\x14\xb8\xe2\xd9\xf2yj\xe8\x88\xc0\x8d\x9f\xaer\xf4\x89\x13\xc6@\xf9\xc2{Qr\xae?\x02\xd7\t\x8e\x86\xc9M\x1c\x19R\xdc\x07wOz\xa3\xf2\x9ab\xd6\xecK\x8cS\xdf\xd7\x00-\xbf\xf4\xf4o<\x04\x8c\xae\xb3\x12>\x9a\xf7L\x17~\x88g\xd2\xf0X\x8e\x98\xe6\x9d\x0f?\xd2\x83Q\x00_\x16\x81\xa7\xa0\x02\xbd\xd9\xd8\xc8\xd0\x9a\xb1%h\xb1\x1b\x98\xf6ck\x9e\x8f\xd6*\xecxT)\x1c\xd4\xf28f\x0f\x91\xe8\xb2\xc12\xee\xd4 D\x9e\\R9\x98<\x81\x93\xaf\xcdt\xf3\xe4\xb7\xcb\x89\x17U\x0eY\xeeKO\xff\xc4\x87r\x11\xc4\xbc\xe0\x1a\x8c\xda\xd6sr/\xfcLpPf\xc2\xfc\xc4\x07\xdb\x94\xee\x1e\xc2LJ\xb8\r\xc1%\xe2F\xe8b\x8bBsv\xc5\x95\xa1\xca\xbb\xaf\x9ew\xc1\x1a\x90\xfd\x91E\xad\t\xf9\xf2\x97\x9e\xfe\x9d/\xf60\x01\x8f\xbb\\93n\xd4\x11\xb8\xf5q\xf2\x86\xb5\xe05\xe3\xe37\x1e\xacR\xb0\xd6q\xdb\xdf\x00Y\xb6\x04\x9eQ\x8e\x843\xbdcJoR\x7fs\xc9R\x92~o\xe3\x93&\x81\x89\xd3\x1c\xa7\xaf\x97_z\xfa7\x1ej\xda7`V\x92\xfc5n\x9c\xebR\xd9\xf2\xe6\xe2\xa4\xf5z\x15\xbf\xe93r\xb8y4\xb0\'X5a\x1c\xfcR\xb3\xf9\xd5\x150\x16\xefL\xbak\x03\xd8\xa5y\xcb\xbf\xd9>nTQ\xe9\xd1;\xcb~\xe9\xe9\xdfxX\xc4\x19zd^.@X6}\xa9\xf0\xba\x0e"\xd1\xcb\x07\x99\x8f\x1e\x11\xd9c<\xc9\xed\x9d\xb2\xee\x84(\x97\xb6y\x95\xefh\xd8.Z>&\xb0?\x12\xfb;\x1d\xc5\xd7d\x04T\x1b\r\xa4o\xc8_z\xba\xfd\xd1\xef\xd6C\xb0\xa3\x98z\xfb\x1eA\x13\xad\xfdD\x95\xa9=\xb7*!>|Q\xda\x1f\xd7\x00\xdd\xcd\xbe\x03\x03\xd2\x9b\x83:\xad\x8c6w\x8c\x8b\\\x91\x9c\x0b\xfcxP\x93\x9a5V\x14g\x1a\x98\x00\x87\xf2KO\xff\xe8\xff\xcf\x08\x0b,f\xda\xafR\xe2\xdf\xa0u1N\xb4\t\x03\xcb>\x96\x0f\x7f09\x9c\xc0\xae\x89\x10\x14\xcc\x876D\x11Lu9\x94\x0f\xfe\xc88P\x9e\x0f\xac\x90\x91w\xf2*\x14\x8b.j\xac\xe7\\\xb5\xac\x05?z\xe6O\xce\xb7\x7f\xf8\xcf}\xb6\n\xd4\x89\x00\x94\t\x9c"\x90R\x17Q\xb9\xbb\xc4\xef\xe4\x0b3\xed\x0e\x82\xe5\xd0\xd3\xce\x1a\x84W;zf- \x7f\xe9\xe9\x9f\xfa\xb1\x07\x0bA\x07g\x1b\xa7\xe7+\xd4\xa0\x17\xfa&\xcb\x1a=\x8d\xd9\xfc\xf0\x12\xa2\x7f\xe3\x93\xaajO\xa1\x03Q5\xbe\x95\xc4\x94\x12,\xfc\xc2\x1f\xba\xcc\xdc|H\xf1S\x89|\x07\x17sg\xea\xab\x0b\xa3\xe5\x97\x9e\xfe\x89\x0f9;\xc3\xa8a\xa6\xb9\xbf\xa7j\xedY\x83?u\x06\xd9\x88\xfa\x877H\x1375\xf0\x80\xd7ak\xa3E\x1a2\xf3!q\xc0\x16t\x87UxmG\xcc\xb1:vL\xfdl\xa2V\xd4\xd7\x9b\xe5r_z\xfa7\x1e\x0ea\xab\xefw\xac\xc2\x10\xa84\xfe-\x98\xa57\xf6\xde\x1e\xde\xfe\x04?\xcfC\xc2U\xbaC\xe2\x82\xc0\xefL\xa7\x1f\xb2\x8c\xe6\xcc5&\xbc\x15\x851\xca\x07*\xa97yY\x18^\xbd]\xf5\x03\xcfQL\xfe\xd2\xd3?\xfd\x05\xf5\x8c+,\xa0\x9d\xbc\xa89\xd3\xef$4\x0fVi\xe1#\xc9}\xea\x87\x17k\xa4o\xd6{1\x03m\xb0\xc2\x1a\xa7\x10\xeb\xb3\\\xb0I\x81\xaa\xe6H\xf8G\xbd\x9a\x8f\x02\xa0L\xe2\xaa\xa7\x88\x97\x95_z\xfag\x9e\xe0\xdf\xbb\x06=\x05h\x9b\xdd\xa7[\x99\xed>\xb6\x16\xc6\xd0\xc8\x98~\xf4\x99\x80\xcb4\xe9|\x0c\xc8\x13h\x97\xa5C\x16{\x0c\xda!\xbc\xa1\xf9U\x8cD\xae\x9b\xbd\xfa\xad]\xf4\xdd\x14DcA\x8d\x83\xfb\xd2\xd3\xbf\xf1\xd8\x01\xd8\x0cLX[\xa5\xcd\xec\x1d\xf1\xf6j\x1aI\xc9h\xae\xfcM\x9f\x89\x0cY\xe7rs6k\xe9\x06\x13EK\x9a\xe9\x93\xb1\xa8\xc7\xa6\xa6\x8e\xb8s\x0eQu\x17\xbb\xbe\xbdW\xe4\x1e\xed\xe3\xd5\x82\xbf\xf4\xf4\xcf\xf3\xd4\x9f\x9c\xab\xff\x8e\x0f\x8a\x0c\xbf\xe6~\xe5\x06\xaag\xb4\xd2\xd4\\\xde\x9a\xdd|v<\x96\x80\xe5\xfe0\xba\xfdl\xcd;*\xef\xa6S\xb7UsUi9\x82?\xcf\x0f+6].\x8e\xc4\xf1A\xa8__`\xbc\xc2\xe8\x85\x81\x00O|\xea){\xa0\xdd\xba\t\xe4\xaa\'B\xe2l]\xc3\xd7\xc0\xbb\xa6\x91\x01\xec\xd7\x9bZ\x07n\xfb\x98\xa3\t\x84b\x81\x17\xf5p0\\\xeeKO\xff\x89\xdaS1X\xf6KO\xff<_\xf6\x95J\xf2\xe0\r\n\xaf%T\xd4N\xc5\x02\xd9\xc2\x92{\x11\xe6\xcfy\x8cw\x0fM\x03N\x8d"\x89\xbb\x84\x97[\x05\xe1\x92\xca\xfc0`\x8f\xf4=\rF\xc6P\x8f\x855\x95\x169\x9a\xb3\xec\xc2C\x92~\xd3\xbb\x7fr\xe6\xfd;_0\x1a\xf1\xe0\xea4\xb1\x88\n\xdf\x17i\xb1\t\xd1\xb4\x13\xd6\xef\xc8\x96\x83\xa0\x88\xf6\xb3\xe5\xa2\x97\xb7\xf7\x13a\x1alk\xec\xf2KO\xff\xe8w/\xf2\xce\xcf\xabM<\x90\xc3\xd25\xa0\xb0\x8f\x9c\x1bj\x8a\xf0\xf8\xcf\xbcq\xe2\xf5\x15\xe8\x1e\x1bQ\xfeV\xa9\xe9\xae\xee\xc0jQ`~W\x96\xf5\r\x041\xe0\x9c\xedr\xb3C\xfd\xe2\xd5/\xe0\xc2\xe3KO\xff\xc6\xe3m\xc1\x01\x18\x8f\xfc\xad}O\x8c\x97Y2\r+\x8a\x03\xc05\xf8\x99\x93\xb0\xb3v\xfb\x11\th\x86\x9fb\x04\x01\xd0M~"WAb\x06\x13\x98\xfb\x13\x99\x9c\xdd\xb01b\xcc\x96\x8bp\x136(\x7f\xe9\xe9\x9f\xf8\xc8we\xb5\xe5\xe51HW\x83\x81\x83\xf4FY\xfd\x9ao\xf7U\xfd\xe8U8?c\xdb\x08\xf4\xf2R\x1c\xd6\xad\xb4<\xe2\x9eG\x15\xc4\x0b\xcbMH0\x14\xca50Yb0VoN\xb3\x0c\x9e]~\xe9\xe9\x9f\xfeBl\xa8\x11\xfa\x0f\xe4-\xc2\x9d\\?\xcc\xe0\xa9p"\x84\x97\xeb\'>\xa4\n\x90\x02\xf3F\x1e\xab\xaf\xdfp%,v\x8dI\x19\xf1\xc0Z1Y\xd3F\xd9\xa3&9/\xde\x08\xc3\x99h\x02\xb2\xc1r_z\xfaG\xdf\xfd\xfbg\xdc\xbf\xf1p\xdeMG\xaeb6\x11H\xfdxu\x14\x7f\xca\xde\xd9\x9f\x9cR\xa0\xd0\xe0\x13\x05\xa5b\xfcqc$\x9a\xa5\xa3\x8e\x94O\xf9KO\xff\xe8\xdd\x88b\xdc\xc9V\\\x9a\xa9\x98\x143\x14Z\xd7\x08.\x02\xcbY\xf1\x87\x8f\x99\x00\xff\xbau\xeaj\x9a )\x8a\t\xec:\xb2\x9de\x99s52\xfex]]c\x06\x07\xda!\xf7x\xdf\xebl8\xf5\xf2KO\xff74\x864\x18h\xdb\x11\xf4.\'\xf9KO\xff\xe8\xdd2\xab\xbaj{[Z\xb7\xca\xd8\'S\xec\xc9;\xd4$\xfd\xb4>z7g\x10\xdd\x19\x99\xb7\x8bL5n\xcf\xedYv\xcf^\xc4p\xc3`%\x8c\x84\x82%&>7G\xe8\x0c9\xb1j\x9a\xdd\x9d\xcb/=\xfd\xc3O\x83\x00W\xb0\x0eb\xd6A\xa6\xdf\\\xca\xf6"\x1e\xe29X\x9c\xe6\xe7\xbc\x8e\xe1hMLw\xe0*\x86\xcd\xddl\xb3D\xe37\x0c~.\x81Il\xd4@\xdc\x9e\x14\x03\x9bes\xb7\x13\xd1\xe7\x0f&\xcaUX9\x15c\xaa(pI\\$f\x9e\xca\x1b\xa2+sJH\xc6H\'>Z\xb6\x8eO\x08\xee\xd5\x94^r\xcd}\xe9\xe9\xdfxT\x85\xdb\xb0\xdb\xedxI\r\xfb\xbe\xb8\xc7\xcb\x15H\x02\xc6\x06y\xfb\x9c\xd7\xc9z\x86\xd8h\xfabd\xd7\xd4\x96\xdfc\x80\xac\xf8r\xe2\xf1\xc4\x84\xbd\xc3-\xb4\xfd8P (\x18\xfcu\x8f\x10\x14\xad\xe4/=\xfd3\xcf\xf5\xf6-\xd3\x1e]U4DM\x9c\xefB*\xa8%c\xaeJ\xb5\x7f\xea\xe9C\x16n\xb4#I\xd6\xb3M\xd7\xe7\xfb\xfeHW\x18\xec\xce\xdb\xe0M}\xedG \xf2\xbe\x19Q\x90\xbeH\xfe\x9a_0\x97.\xbf\xf4\xf4\x0f\x1e\xeeU\xe0\xf1\xf0\xb1\xdflG\xde\xf5e*_\x0f/\xbbh\x0e\x14~\xe6\x17\xfc\xc2\x0e\x8f\xcdf\xc7\xbd6\xbb\x0b\xb4{\x13tAY\xbb\xca\x97\xcd\xbc\xd9j*\x9b\xc1V9\x89\xe9.j\xda]r\xd9/=\xfd\x83\xc7\xdf?\xb3\xfe\x99\xf7G\xa1T\xccX\x0c!\x02\xac\x88\xc6\xb9\x86\xdfH[\xe7|\x85\xb7\xfd\x9c\xcf\x9b\x7f\x04\x15\xd22\xdc\x9d\xb0\xa3\xab\x00\xc6\xf2\x97\x9e\xfe\x99\xe7\x90\x01b\x9c\xa5:[ \x0f@\xdd\x7f\x08r\x9d\xe7\xb0=\n\xf6\'> L\x07\xa1\x9b\xa3u\xcd\x16\x13\xb5 \xc3=E\xdc\x85\xad<\xd3\xe4\xe8\xb70!W\xe9\xcc\xcb\x80\x17\n\xa6\xb1\xc7\xaa-\xbf\xf4\xf4\xcf\xf3e\x84M\x1e\x1a\xac53\x0b-3\xdfSj%\xb6Jvk\xc8\xf1S?\x8e3\xcf\x98C\xa3\x1c\xa3I\xa2\xf7\x12k\xe5r\xda\\:\xea\xc9\\\x15\xa2\xaah\xf30\xaf\xd6r\xe0\xae\xa8\xe1O\x91e\xbf\xf4t\xf7?B\xcf\x16\r\t\xc0\x9b2Qa\xeaL\x03r\xd79\xfdHN\x03\xce%\x7f/\x02h\xd2\xe1\xbd\xcf\xbd\x0eO\xfaw\x97\x859\x9eI\x0e\x9c<\x05$C\xfc]\x87\xb7\xcd@\x06\x15a\xfe_\xee\xde\xb49qn\xeb\x12\xfc/\xefWW\xb452TD}\xd0\x8c\xc0\x12\x06\x0b\x84TQQ\xc1d\x81\x06\xc0\xcc\xa8\xa3\xff{\xafup\xa6\xc9\xcc7\xab2\x9f\xdbU\xf5F\xc7}\xf2:\xed4H:g\x0fk\xed\xbd\xf6\xe1\xd0{\xb1\xda\xbb\xa4\nN\xa0\xaa\xb7\xf4M\xcfg\x8a~J\xe3P\x9a\xc6\xed\xd3\xcb&\xb9\xbe\x14r\x9dJ\xe1yV\xe9e\xa2\x0ew\xf8\xf7\xd7\xb97>\x8d;]\xdd\xf98\xcc\xac\xacu\to\xb2\x94\xc6z\x91N\xba\xc7i\xacK/\x911\xddgN\xafw\xbb\xbf\xf7\xcc+7SP\xeb\xfb\x99G\xc3\xfae\xf3\xc3YG\xa7E\xee\x9c\x06\xcau\x87\xf7\x90F*\xcfG\x1a\xbd\xbd_\xb6\xdd\xfb)\x82\xe1n~\x93\xaeN\xc6sg\x0c\xdb\xc3\xd7V w}|\xcd\xfa\xbbn/\x0b|\xc5\x18\xdc\x0c{\xebX\x863\x88\n?\xb5\xb2\x02\xbf\x9b\x99\xbe\xe1|\x18\x03\xe3\xcd6\xb6\x96\x81\x7f\xf3\x87\x99\xe3\x1b\x97\x8b1\xfa\x1f\xfc\x8e)~\x86\xdf\xd9v\x16\x13\x93\x87e\x16\xe9h;\xf8\xfe\xf7x\xd8\x1d\x14a\xb4\xf0\\)\x1d\x1bfl8\xeb\x9e\xb1\x1d\x04\xf6\xfc\xf8\xe3\x99L\xd25\xbc\xfa!\xd6\xf54\xab\xc6\xf9\xc2+\xcf\xb3\xb5t\r\xcc\xadke!\xd7\xe5\xe6_\xb6f0\xf4\xc3H\xee:\xc3q\xf7\xfd\xf3\xab;4\xfd\xd7o?\x1b\x8f\xf4\xd7Q1tG\xc5\xd8\xe9\xdf\xfb&\xdbm\xb9\xb0.\x97y\xef|\x19\xf5\xce\xc6\xa8\xe7\xc5?\xecO\xfd\xa2<~\xaf\x1d\x83\xc8x\x1b]~{\x8f\xb9\x959X_\xf9~\x9d\xb2\xfd\xfdzN\xd6\xfe\xf6\xb3qT\xb4\xdf\x86\xce\xf8m8\x1a$@\x8c\xc9\xef\xf6\xdd\x1cl\xedi\xdc\xcd\xbdL\xfb\xbe\xbeffXu\xf0v\xc2\xeb\x0e\xd8\xa3\xb7\x9e\xe1;\xf7\xef\x8d\xd9\xd8~\xebon\xf3\xa4\xd8(\rG\x1bu\xce\xa9?\xfc\xb0j\xef\xf5\x10-\x93@\xeeg\xbb\xd1.\xdd\xf7\x8cw\xe4\x81\x17#\x03o\xcc\x1c\xf3P;\xb0\xaf\xac[\xae\xa4\xd9!6\xdb\xf6\xfe\xb9\xdaO\xe42\x90\xabF\xfb%\\\xcev\xab\xe7M\xff\xb9\xf3v,/\xc7\xcb\xd3\xac%e\xfe\x9a\xf7\xe3w\xccl\x1f\\{\x9a\xe9\x9b\xdb\x95\xdf\xf1\xf4v6\x7fz\xadG\xc5$8\r\xba\xfeR\x9a6v\xad\xf7\xfd\xebb?\xaa\xeb\x83;\xce\x9c\x97\xfd\xfblw\xec}{\xbde\xce\xbe^oMV\xdd\xee\xb9\xb3i\xef\xaf\xe6I\r\xcdx\xb8\x1b\x0f\x1a\xd7\xd9\x9b\xe9\xda\xa7\xc9\xeeye\xbc\xe4vs\xa67\xa6\x93\xef\xaf_\xd7_\xaf\x7f\x1d\xd5\xfdv\xb2p\x9f]KJ\x17\xc0?\xc7\xe80,\x16#\xa9\x17%f\xc3+\x0e\x1f\xa3\xdc\x1c\x1f\x8e\x8bs\xfd\xed\xf5\xf6\xe6\xe1\xfeCgt\xb15\xf3\xe9\xed\xd2\x9aG\xf9\xc9X4/yW\xf1\xcf\xc7[\xd9\xf6\xe7\xef\x1b\'\x1e\xbfn\xeb\xde\xab-\x8d\xfd\xb5\xc4\xd7\xef\x9c\xd00\xefkou\xb2|Y7[R\xd5\xb7\x9e\xeb\xbe{\x99X\xb7\xeb)\x99\xf9\xfb\xa4T\xda\xc7\xad\xf9\xde\xab>\x9e\xd50\xdb\xaeoX\xfb\x0b\xd6>\xcb\xde2\xe3\xf5\xe1\xfeC%y\xaa\xed\x8f\xcb\xe6\x96%#\xed\xf9\xaa\x8c\xbcl\xf3\xda_\xee\xe4\xca\xd8\xce\x97\xd7(\xeeh\x9373\x89\xb7\xfe\x9a\xf3W\xbe\xef\xbe&\xdf\xf6~\xea\xbf\x19/#\xfb5\xea/\xfbes$w\xb6Mca\xab\xab\xd7\x83\xae\xce\x0eO\x88\xdb\xe6\xfb%\xdf\xbc+Me\xe5\xe77\xffL\xdb\xb1L \xe9\xcf\xfd\xf7\xd6NvU6\xedQ>y\x9e7&\xc0;\xda\xa6\x94l\xa5U\x1bx\x98\x8f\x91m\x1d./}\xd5)\xe3\xcd\xd3tR\xac_\x06\xb4\x9fbz\xb3>\xf7\xd0o\xf5Tcb\x1f\x97J\xb2hn\xd5\xeb~i\xd7\xef\x9b\xd1\xb1J\xb3\xb0k%\x97H}jO\xc3\x0fIj\x87\xbe\xe5\x1f\x85\xed\x0eki4\xbf_\xbfW$\xafm\xe5\xa2\xee\x9fz\xcfv\xdfw\xa4\xa5\x96\xa4\x9ej\xec\xbc}\xd4j\xde\xbc\xc6\xeauu1\xb4\xa14q2\xeb\xac\x99\xd6\xf9`~\xfa\xfe\x9b\xff\xd2\xfc\xe8\'\xc1\xd4}i\x1dJk>j]\xc7\x9a\xdfp\x8f\x83\xf7\xf6n3\x1d\x1f\xb5\xdb\xc7\xa2\xdb\xd5\xf6\x8dT\x7f\xf2~\x8cU\xeb\xae\xd8\x0b\xc7\xf4\xd6Fo\xd7\xd4N-0\xd4\xde1G\xc2\xb7\x97\x0b%r\x8d\x97\xd9\xa4\xee\xbf\xec\xb6\x9e:.\xfc\xee\xd3v\xdf\x19\xc7o~\xc9\xfb\xf7\x07N\xe1\xac\xbb\x83\x81\xed\xae\xf5x\xb4\xbe^\xfb\x9e~\x06jS[\xcf\xb5\x9e\xf5^;#\xef\xbcT\x8f\xa7\xe1\xf6:\xb9\xf9\x1b\xbe\xc6\x188S\xbf\xdf\\g\x85QWU5\x0bZ\xe9\xaaYw\xf6vRL\xfa]\xbdY$]#{\x1d9\x8dr=\xaa\x9d\xeb\xd5_\x8f\xb8\xd7+c\x10l{\xd3Fr\x89\xab\x8f\xc6\xf4\xe4u_\xc7\xd7t}\xc9_\xa2\xfc\xd2\x9a\xc9\xf2\\\xc9\xa7\x87i\xd9?\xcen\xf0\xaf9_c\x1a\xc3mf\xb7\xe0v\x0e\xb6\xde\xf6?6\xe1\xed\xcd>5\x06O\xc7a+}\xf3\x9b\xd9~5\xea\xbe5b\xbf\xac\xda\xa7\xba\xec)Y\xd6\x93\xb3\xa4\']\x92^\xf7\xeb|\xbc\xf3\xbf\x1b\xebn\xedo9\xe8\x1e\xbf\x87\xbd\x931\xd8\xee\xff\xe2\x8c=\xfb\xc7\xd7w\x9f\xad\xac<\xcf\x15\x19\x7f\xca\xfa%r\xdeL\xc6(#\x98\x9aF~AXY\x9b\xc6\xdah\x14\xc5\x87q\xc9\x13\xd3\xf6\x8d\xbe\xc8cf\x7f\xb0*z\x16\xec\xd13\x9c\xccx\xdb\x9a\x86\xe9\x07V\xe6\x1b\xc8q+\xcf\x18d\xb6m\x98\xff\x83\xdf\xc9\xc4\xcf"\xc3|\xcb\x16\xf2\xdc[\xdc\x92xX\xbe\xad\xcc\xef\x7fO\x9d\xd2\x1a\xc8\xa1\x94L\x86\xe5\xc8p\x13\xack\x1f{\x89\xd8m\x98\xbd,1\x8c\xbb\x0f\xad_2\xda\xd2\xdc2\x83/\x9f\xaa\xb2\xce{gx\x98L\x93\xde\xb8Ro\xf3\xe1MIFVv\xb0Z\x8d\x8f\xc1\x9b\xa2\xc9G\xedc=\xbc\xf4\xde\xd4\xf9/1\xd9\x7f_d\xe3E\x1a+N\xbc.\x93\xcdez|2_{\x9dy\xe7M\x9fM\xac\xd7\xcd\x04\xd7\xfe\x8ce\xf6\xda\x98\x7f\xad{4\x1b\xd5\xc7\xc1\xd3\xf8\xe6\xb5Z\x1b\xbb\xb5Z\x15I\xa7\xe3\x8f\xd7\x97\xf1\xc7\xac\xd3\x9dj\xe7\xf9\xdb\xe6#\xdb~\xa4r\xf0\xed\xfaoj\xf6\xf2\xf9\xec\xaf\xab\xfc)t\x9f\xce\xd1\xb1\xcck\xf5<\xac\xcfZ\xc78\xec\xf4\xce\xd9\xb1\xc0\xcc\xbd!\xde_\xd1\xaf\xe9[\xf2u\xfd7\xcf|\xb8\xfeA\xcd\x07\xab\x17\xe5\xb5\xabL\xfc\x9eu4\xca\xe28mW\xeb\xd1\xe5\xf0\xbe~v\xcf\xb1\xdb\x99v]Gy\xee\xc7\x97o\xf9 \xbef\x8f\xf9,\\\xdcJ\xb5\xd1\x7fZ:Y\xdeW.\xadN\x17\xfc!~\xb5N\xf3\xba\x1d\xbf\x18\xc6q\xd9]\x18j\xd4~\xf9\xf8\x96\x8f\xbc\xde%\xfd\\\xff\xb2#\x99qg~\x8eV\x85\xff\xde\xf8x\xbe\xca\xc7\xd6\xd6z^\xaf\xc3i\x7fx\xd2\x1bY\xdf\xdc\x04Fq\xf5o\xd6\xf7|4\xdd[\xe7\xaf\xeb\x07\xc8\x8e\xb3\xfd|\x15yIy\xee\x06^2:\xd8/\xbd\xf1\xf8\xc3r_\x95\xed\xf8\xadJ\x00\xbf\x8b\x1at\xfc\xf8-\x1fu^\x87\xc3\x87|t\n\xady8\xcf\xac\xb1\xbb\xacO\xc1\xb6\xac\x17\x83\xf8)\x1d\xed4oi_.\xd9jm\xef\xcf\xe5{\xf11\x9cd\xfb\'c\xb0o\x1a\x9fx\xc7Y[\xc0\xe3\x158\xf9\xdb\xf3Kw\xb0q_\x83\xf1\xd2\xea\xaff\xa3\xce\xf34U\xca\xd92\xf8\xd8\xbc\xbd-{O\xd5\xf2}\xb5\xed~\xf9\xa2c<\xc6>\xebc\xf9\xf4r\xf6^:\xd6>\xa1\x18J\xdd,\x8a\xeb\xc0R\xe4V\xbdnL\xcd\xa3\x9cZ\xd6\xfbG\xef*\x8f\x0bDq\xda\x80cvMD\\\xc7\xb8&\xa7q\xe9\x1f\xe6\xd1\xea4\xd3\xb4\xd6\xf1\xf9\xb5\xf5\x94X\xf9U6\xf7\xea\xb19+\xa6\xc1\xb8\xf5\x90;\xec\x93\xb6\xed\x1a\x81\xbe\xdbU\xf5\xab\x97,[\xd7\xc6E\xe9\xba\xf5z\xb9\xd4=k\xe0\xdb\xb2\xd9\xdf%ep\r\xe6_\xb9\xc3\xeeY\xd5\xa9\n\xdc\xdd\xc7\xa9\xda\xaf\xd6\x91<\x8f\x93\x83\x7f\x93\xb4\xd6\xab\xd7^L\xc2\xad\xb2\xf7v\xd1>n\xad\xd7\x0f\xb9\xc3x\xf6M\xeb\x1a\xf8\xfa\xe5\xd6\xdb\xd5\xf3"k\x9e\x8a\xd7\xfd\xe8\xdd\x1b\xad\xfbZO\x1b[\xa3S\xb9\xee\xa6\xe7\xd9A\x1f\xed\x0b\xdf\x02\xbd\xb4\xf6\xbeg\xad\xe5/\xce\xf0\xef\x9d\x91\x1a\xcd\xcf\xc4\xd7se|\x02\xaf\xa9\xef\xdca\x8c\xdfu\x8ea\x9d\xbd\xad\xc0A^\xd6\x7f\xc3A~~\xaf\xd1\xdb\xd8\xd8\x06V\xbf\xf1q?\x93\xc1z\xaa\x0f\x03[p\xc3\xaewpf+e\xb6\x9a\xd8N\xda|7\xe5a\xd0\xd9\xcc\xcc\xb7\x96\xf1t;\x86\xe5\xae\xb4\xaf\xf8\xe6\xa9\xdc\x8c[\xc8\xd3\xdb\xac~\x1a\x0c\xebf\xa4;0hU\x9d\xcb\x85\xbf\x1e.G\x88\x97\xadE4\xdc$\x9bM\xf0\xd6z\xb1\xeb\xde_\xc4\xe9$w_\xb4Y\xde>\xcf\x8a\xdd\xc5\x1bJ/\xce\xe2\x88[\xb9\xde\x8ay\x1c\xd7\xde\xaa^M\xb3\xfe%\xac\xdc\xcb\xf6{\x9c\xb6>_\xefN\xcbpP\x8e\xb2\xd7V\xbb1\xbe\xbe\x94\x83\xe5\xbbe\xf7\x9a\xb2\xf6\xb6(^\xbcnp\x9a8\x87\xf3\xf3tl\xbe\xb7\xb2oq\xf6{\x9c\xee)\x861\x98K\xcb\xde@\xfe\xd8\xcez\x8b\xc3\xf6t\xd4\x95\x9e\xf34Z\xec\x9b\xfae\xbd\xf5\xdb\x9e\x91\xeb\x8df\xcf\xfb\x8a\xb3\xd9\xf2\x13w\xfa\x1f\xb7\xe6\xac6\xf5\xf4\xe9\xfa\xba\xaaN\xef\xf6d\x9c\xf6\x16\xb3\xfe\xdc\xbf\x0e\xcdJ\x19/\xf7E\xee\xe7\xc5V\xdf\xdf\xbec\xfe\xde\xeb\xe7}+\x1f\xdakS\xd5\xdf\x9f\xe2\xf5G\xb3\xe9\x9f\xf5\xb8qz\x9f\xf5&\xabz\xe3\xa5\xe5\xd1\xd2\xae\x9b\xdd\x87\xbcW\xe3\x19\xd6\xf63\xc6\xd8;\xff\x13\xf3\x9a\xc7M\xd7\xa8\xde\x9d\xf1KV\x0e\x82\xcd\xf3\xfbi79\x96\xb34\x93\x9d\xe7\xa4z\xb9\rW\x89\xf9>+/\xb9.\x7fa\xde\x81\xf6-\xc63\xc68\x8a\x1c4G\xaf\xe5|u~\xed^\x9e} >\xf3zs\x93\x83;\xad\xae\xb7\xf8\xf4\xa2:\xbb\xc6\xb6\xd7\x03\xfb\xfe\x8e\xb9\x8f\xda\'\xff\xb1l-y\xce\x87\xef{\xa9\x99&\xadvS\x9a7\x9f\xcc\xcbKo9l^\x8d\xa2\xdf\xb8\x16\x89C#\x8cG\xd5W\x8c\x1b\xad\xb2\xc9\xd7\xf5_BM\xf2\xdf\xda\xf9mQ\xde\xacl?\xe8\xa6/\xf19\xdd\x96\xc1KG;_g\xf2j\xa8\xac\x87\xc3\xf0\\\x97\x87\xef\x98\xfb\x90\x04_1\xce\xae\'\xf3\xdd\xe2)?oV\xbe_\x1f\x0f\xaf\xe6m\x18\x99\xe3\xfc\xb2\xdf\x04\xcfH6\xc6\xcbd=\x19\x84\xd2\xf3[c\xfd\r\xf3\xae\x0f\xdeg\x9e\xd0\x87mc\xbcR_$-\xb5:O\x1f\xe7\xd6b\xf6\xde3\xce\xda6\x8f\xf3bvZn\xed,\xad\x8dn\xf0\x16d\xd9\xb7\xeb{\x1f\x97\xfd\xc3\xf5\r\xbd\xa14T\x7ft\xad\xba\xb3[dV\xee\x8bq\xbb\x95\xe3i\xb6\x8a\xda\x1f\xeehW\x05W\xa5\x1b4\x93^\xf3\xed[\x9e\xfa\xc2\xdc\xb4\xfb\x97|\xb0\xb1\'\xbeQ\x0e\'\x87f\xde\xeb\xce\xd5\x91{\xae\xc6\xbd\xa5\xb9\xc9Z\x81\xef\'Y\xa3\xa9O\xf4^1\xd6z\xcf\x88\x01O\x17\xa7g\x10\xe3]\xfd\xec|\xda\xea\xd9u;:\xdfn\xa3\xee5\xaa\xddI\x96oc\xd9|V\xaa\xc5.V\xf3\xdeN*6\xeb\xf3n\xb3\xd1\xa6?`/\xeb1\x9e}l\xce\xeb\xe6u}\xbd|(\xfd\xfde\x10\xeaj\x1a\x8c\x8c\xe3\xf1=\xd0NU6[\xc4\x83l\xd3X\x1f\x16\xe3\xf4;\x16~\x1b \x8a\x9aF\x10\xbf\x94\xe9\xdbK\xc7\xd1f\xd5S\xff\xf5\xe5\xbd~~\x05Wj-\x06\xd3vs\xa9\xe8^\xee\xbe?\xe6\x83\xe6\t\x10\xd0\xeeO\xa7\xe9\xa1\xbeV\x9ds\xb0<\x84\xeb\xe1\\\xdblN\xc3\xac\xb0\x8c\xf6 \xfa\x88\xbbQ\x10u\x9c\xef\xf9\xc0\xb8e\xbb\xfd.7\xa7\xd3\xfd\xeeCK\x8a\xf6\xc4\xad^\xac\xe0\xd8\x7f\xb5\xaf\xe7\xc9\xb8\x06\xa5\xb9~\x8c\x1a\xf2\xd9\xdf>r\x89\xd7\x9b18DV\x7f\x1e\xac?\xe6j\x17\x88\x7fg7\xe4\xce\xb5\xf4\xf3\x93ur\x07\xe5\xcb.Y\xc73\xa5\xd9\x1f\x7f\xd6Z*\x7f/j45\x8f\r\xe8\xee\x035\x95\x16\x13W\x9f\xc5\x87z\xe6\xa5r\x7f\xb2\xfb\x98*\xe3\xebB\xdd]\x16\x9d\xf4c>Y\xd4\x8b\x89S\xcflC]\xae\xba[\xab4z\xc3l\xfb\xcez\x85\xc0\xd07=\x9fo\xca\xcb\xc2k1\x0e\xe7s\xdb\x97\xc3\xea\x80\xe8d\xcaIq,\x97C?\xb7\xd6\x8e\xdd\xdf?\xed\x87I5j<\xcdW\x8b\xeam\xfcq}u\x86\xd3E\x91\x98\x9d\x0b"\x94\xfb\xd2\xa9\xe2K?\xe9\xa9\x85\x04<\xbe\xb5D<\x1d\xfa\x80\xb0Z\xcf\x9c\x8c\xc5\xb5\x9clt\r"_\rWvp\xb9\x1e\xa6\x83\x9e\xd7hF\xd3\xbe\xe6\x86\xaashY\xaf\x87\xba\x1d\xedw\xd6\xb3<\x92\xfd\xf9\xa1\xf8\x08\x97OFc\xbeP\xddR\xfb\x1dw\x13N\xc2\xd5\x92T\'*\xa6N\xb6u\x17\xb1x\xfd\x9b\x15a[#\xe3\x12\xd8\xf8cv\x93\x7f\xba]\x17y\xf01O\xba\xe5\xd2+\xf7/\xf1\xa2oY\xb52\xe8\xec\xdf\x94\xc6\xcbi\x9c6\xbaO\xa5\xf5\xa6{a\xdamd\xf7\xed\xea\xb4\xb8]\x8e\x13XS\xdb\xdc\x19\xeb\'\xb3\xff\xba\x9a\r\x9a\xd3\xec\xa6\x9ae\xf2\x96\x19\x9bl78\xe9\xc5\xd3\xb5[\xd4o\xc7A6\\\x1fO\xef\xc3z7?\xadG\xefe\xff\xf4ni\xb7hS.\xa5\x8f\xd6\xab\x01\xf0|\xaam\x16\xfa\xb5\x81\x19\xbdMTw5\xd4\x06\xff\xe5\xbf\xfc\xdb\x7f\xfb\xe9\xe34\xbe\xfe\xfe_\xff\xdb\x8f\x1fbp\xff\x1c\x88o\x9f\xe1\xc0Ot\xd8oO\x9b\xc5\xbf\xfdg\xe9?\xfd{\x9fn\xf1\x1f\xff\x03\x1a\x0e\xebl3=\x9e\xf6\x9f\x1f\x9b\xf2\xed\x19\xfe\xfb{9\xcd\xfe\xed?+\x8f\x1f\xaf\xf2\x0f\x8e\xc8\xbf\x7f\xc4\xc5\xe18\xadv\xff\xde\xe7\\H\xea\xff\xc5\x8f\xf5\xd0\x9b\xb2\xda\xe6\xe7\\|\xbf\x19\xfc\xf2\xc7\xb1\x8cg\xf5\xb8(\x9e\xcen\xff5\x0f\xe5\xd7\xa9\xb5\xdd\xbd\xf8n!\x1f\xf6uzq\xb2\xe9J:\xb6\xbc\x8f\xb9\xbc\xb2\x173-\xcb\xa5\x85\x1c0\x99\xecWS\xf9\xf6\xf1\xbc9<\xb7o\xda@w6\x92\xdf\x87\xedF\x95\xf9w\x8f\xd9n\xb7\xd5\x96\xd6\xd65\xa9iImS\xd5T\xe0q\xc3\xc0C\x1b\xa6\xae\xcb\x86n)ZCQ\x95?\xd9\xcdF\xb3\xa9\xf1Sk~~\xcc\xe2v\x9d\x18\xfd\x837x\xca\xe7\x8b\xee"_\xb9\xa7`_\x16\x93\xee\xe8\xf6\xfcv\x92W3\x10\xea\x95\xb7N\xd4rs1ne\xde?t\xb5n\xeb|\xf88Xa<\xec>G\x83\xb1\xef/\xb3\xeb%\xbf,\xa6\xdd\xd3Z\xca\xc3\xb7\xbf\x8c\xe2\x8e\xa2)\xb6\xa9Xf[i\xdb\rW\xd3\x0c\xd9P\\Gk\xe9\x96\xd5\xb4\x11\xc5\x15\xc9u\r\xc7\xfc\x93\xc7TU]ik\xda/\x8f\x198\xb7\xed\x93\'-\xe6\xb64_\xf6\'\xc3\xd8\xca\x0e\x03\xd5\x99\xd4\xcf\xf2\xd3\xcbkP\xf6_\xa3Jm\xf4Z\xca\xd0}\xf2\xf5\x97\xdd\xeb\xcb\xe9\xc9\xdf\xcc;\xc3\xa5\xf2\xea[V\xa6\xbe\xed\xc3\xe3^\x06,\xaf\xc3u_\x1f\xbe\xda\x85n\x1a\x7f\xf5\x98\xb8_|\xf6\xd4\xe3cn\x9b+\xe3\xf4\x12\xf8\x97A\xe6\x9e\xe7\xbd\xe7\xad\xac\xcc\xb6\xcd\x0f\xcb\xd1\x87\x97\xd7\xa9};\xac\x16\xaa4=4\xf1\xa4\xdez\xfa\x14\xed\x8f\xaf\xef\x8a\xfe\x8c\xe4\x99u;\x97\xe3\xc5S\xae\xc7\xc1t\xa0\x8e\x06\xa3n\xdfSzK\xaf\xbb\xb3\xb2\xbf\x0c\xe4-\x17\xc9\xd7u`\xbdM\xa9i#v7L\x03\xbe\xaa\xc1-M\x85\x9f\x7fd[\xb2\xf5\'9YU\x1aX\x13U\xd1\x7f~\xcc\xd5\xae1k\x9a/\xc8L\xcf\xcd[8\xea\x17\x1fZ4[\x8d\xbb\x81[\xce\xfa\xed4\xb5\x97\xd3\xe3y?h\x8e\xca\xdd\x93\x976\xab\xa3\xf4\x0c<\x95\xafV\xd7\xd3\xea2_\xae\x06gut\x9d\x9b/\xa7\xcdj\x87d\xc1\xc3K\xf6\xc6\xdf\xe5+\xd5\xc6s\t\x0ch4Z\x96\x89\xd4\xac\x02<\xf0\x93\xd1Zz\xd3i6Z\r\x13\x97\x14\x1f\x0e\xf7?\x8f\xb4\x9a,\x01\xb5\xfc\xfc\x94\xd7\xddX\xda\xf4\x8f\xd7I\xfbt\x92\x95}6\x9d\xbf\xa5\xb2\xec\xc7C\xab1\xf0\xcc\xba5\xdc\x99\xd3\xab\x92\\\x07\xdd}\xdf\tT\xefp\x08z\xf2A/_\x97\x95\xdc\x1f\xac\xdf\x95V\xb5\xdb8\xbbR)\xe6\xcf\xd1[c\xb2\x92\xf7\x85\xfd\x1f*]\xb9\xf0WGR\x1b\x8a\xd62\xd4\x86k5UK\xd658{\x13\xe9J6-\xd3\xb64\xd3l\xfe\x11\xb0i(MI\xd2\xe5_\x92\xfe\xe1\xf8l\xba\xcb\xcb\xa9\xd1z\x7f\x1e\xcb\x0b\'\xde\x9e\xc7\xc6\xdb\xed\xda\xd7\xed\x1b\x9c\xaf\xe1\xde\xe6\xd1$\xdf\'\xd3\xe22\xdf\xb4\xdf\xfb\xaf\xa7k\xf3\xd6x2\xcdC(\xf5[e\xeb\xb2N\x9a\xdb\xf7\xd5 \xee>\x85\x03\xfb\xb0\xb4n\x1f\xca_\x068IG\xf2m\xb5\xcd\x86j\x19F\xd3h\xb4\r\x1bH\xb5\xd9\xb0l\xcbum\xc9jq\xfc\xd9\xb1\xff\xc8\xf3U\xb8C\xbb\xa1\xfd\xe2\xf9\x9b\xdd\xb4\x98y\x96\xfb\x1cm\xebU\xd3\x95\xdfZ;\xdd\x9f\x9c\x1b\xfbr\xfe\xeedyY\xcb\xaf\x07m\xd3\x93\xa6\xc3\xc5\xc6[^\xde\xd6~\xe9i\xd9u95%,\xc9^\xfb\x98O?V}\xfbu\xdc*\xd3\xcd\xeb\xce\x0c\x9f\xd9\x01\xfd\xcbt%5T\xb3-\xb75I3\x1bm]\xc26\x1a\rC\x95\x1d\xb5)\xa9\xcd\x96\xebHMI\x91\xec?\npm\xad\xa1\xca\xed_<\xffi\xeb\xb4L\xb3\xf7a\x9e\xfb\xf3C\xb7\xdeJ\x8d\xd5\xb0\xf1\xfel\xee\x1d\xb7\xfd\xec=\xdf\xae\xeb\xf3\x9b5^\'\x8du8UL\xe3\xa2I\xd7mg|\xdd[\xd2\xf9\xe5\xe5=\x9b\xbf\xaa\xd3y\x96\xbe\xd6\xf3\xba\xdb\x08\xb2m\xde\x89\x93\xd3\x7f,\x08\'!\x03\xea\x16\xbd\xc1l#\x92\xe8\x84n\xaa\x01r\xe3\xb4%$|\xbb)\xeb d\xc6\x9f\xac"\x02\x92\xdc\x04\xd7\xfby\x11\x97\xcf\xafK\xc7\xc8\xb6\xce[\xa1\x95\xab\x8e2Z\x94\xd3\xf5a\xb6n\x84J\xb9\xef\xf9~\'TvJ\xc3;\xb9\xbb\xa9\xd4B\xa8\x19\x9d\xda\xca`\x7f\xc9\xae\xc3U\xd30\xbaI\xd5\xf7\xdb\xc9S\x1c&A\xb4:\xef\xfb\x1f\x8d\xf9G\xe3o\xb3D\xc3\xb4\rIn\xe9F\xc3m7t\xc7n\x03\xc3\xb9\xa6\xe5\xc2\xfd\x1dSv\xcc\x96\xd4\x00\xca\xfb#\xb8\xaf5\x1a`\xe4\xea/.q\xf6\x0f\x86\xfa\\\xfa\xb3\xd1\xae\xe5o\xb2\xa9\x9e\x8c\x0e\xe5\xf9\xd5\xb0\xdf\xce\xc3V\xf7P\xe4g\xc7\xd0\xa4\xb7\xdc\x9b\xb8\xbd\xee\x9b;\x8e\xcf\x99\xfb\xf4\x9a,\x94s_z\xa9\x0f\xb3\x89\xdf\xbb\xa9\xad\xc1\xcc\xec\xbcd\xcbt\x91\x8c\x06\xd2\xdc\xf8\xbb\xc7\xb4\x80\xc2\x85O8-Ws\x9c\x96m\xb6\xc1\xc0\xc0\xc4MG\xd1\x9b@ojCk\xb6\x9d?\xca\x12zK\x91\xda\xf2\xafiByu\xcee=oG\x93\xe7\xe3\xd3q\xee\xed&R\xf6r]\x18\xfbN\xba:o6\x83\xbd33\xf5lV.[\xb2\x95o\x8f3}1<\x9e\xd2S\xb0\xe8\xbc\xfb-\xab\xbe\xcc\xb2 \xfd\x88&\xbd\xb67\xd2[\xbe=\x8b\xbc\xb7\xe2/\xc9\x9b\xa4\x80}\xba\xb05\x00p:\xbb\xa5\xca\xaa\xabJ\xed\x16\x989b\x9f\xa6\xa9FKn\xda\xc6\x1f=fSU\x94\xb6\xf4\x8b\xd1\xbe\xac/ipLf\xefv\xf8\xda\xbd\xb6F\xd9U1\xa2\xcb\xe0\xd4Y\xedTy\xb9\xcc\xf7\x9d\xa3/\x9f%\xe9\xc96\x92xty\x8f\x8e\xcf\xbd\xaes\x99\xb5\xd7\xed\xf28u;\x97\xfe\xf5|5\xec\xf7\xd4]\x1c\']\xb3[^o}\xf37\xbb\xf9\x7f\xc8\xf3-]\xb7d\x10\x17\x0b\x06\xdd6[\x16\xd6\xb4\xd9\xd4m\x1b4\x07\x96\xa2\xca\xae\xdbh(\xce\x1f\xc1\xfd\xa6\x02\x9a\xdfP\x7f\x89\x9f\xca-=\xbf^\xa4\xddS\xd2\n>\xa4\xee\xe192\xf4\xc4x\x9a(\xf3\xd7\xa8\x7f{ky\xda\xe2\xdc\xca\xca\xa0_\x85\xeb\xd9h\xac8z\xa0m\x1a\xe6\xa8\xe1\xdcn\xc7H\x1a,\xb2\xde|o\x1bK\xe9\xbd\xe7\xe4\xd1m\xb9\\>\xc9\xd6\xdf\x19\x8b\x82\xb8\xac\xb7\rKi\xcb-\xc354K6\xf5\x86\xcd\xcf\x11m5\xa4\x96\xea\xf0\xd3Dm@\xaa?*\xda4\xe4V\x1b+\xf3\xf3cv\xe2c\xbd\xde\x1co\xbb\xde\xec\xfd\xf4\x96\xe5\x96\xe1\xdd\x9eTi\xbb\xf3\xdb\x9b\xc3p\xe5n\xdb\x88s\xda\xf6\xa5\xdb\x99\x9b\xbd\xdb$\x1a\xf7W\xd3Mwx\xee\xfa\xf1\x82\xd3\xd3\xf1nk\xd5\xaa\x87\xcc\xb2[,\x94\x89\x99\xbd6\x96\xbf\x83N\xbf{LW\xb3YP\xd2\x9a\r\xf07\xa42\xc3\x05\xba1]Mm\x99\r\xd92\x1b`\xe7:B\xfc\x9fD\xb8\xa6\xa4KJ\xf3W\x8ez\x95\xa7\xb7\xe2\xa0\xe9\x95\xf1\xfa\xd6{*\xe5[e7;\xad\xe9h\xa8m\xf7r\xcf2w\x87\xdd{\xfd\xd6\x19\xb6_\xdb\xc3\xdan\x8e\x9b\x03c\xf7\xb2\xf6\x0b\xb3\xb0=\xfb\xb0~\xc9\xf7\xfe\xc7\xe6i\xec\x18\xa7\xe6x\x93\xf7r%\x18\xef\xff\xb26\xa5Jz\xbbm\xb4-R\xf0\xb6+\x83\x98\x1bM\x04=0\xea\xb6\xad\xbb\x96k9\xb2\xa3\xc9\xcd?*\xc1\x01%\xb4\x9b\xca\xaf\x11n;~\t\x9cC\xef\xe3\xcd~\x19*\xee\xe1\xa5w\xbb\x1a\x95\xd2^\x9f\x86\xdd\xa7\xee\xec\xbc\xfch,\x9e\xba\xc7\xb1\xe5\xad\x9f\xeb\xc3*\x08\x82\xe3a\x95;\xad\xf7\x96y\x18L7Se\xa7\xcdn\x97\x8f\x9bl\xacT\xbb[\xf4\xbd<*\xa5\xdf\xc1\xfd\xff3\xae\xaf\xb9F\xcbj:\r\x07\xcb\xd6\xb2%]3\xd5v\xa3m\xdb\xb2+Y\x92\xdd\x86\xd7\x1b\xaa\xe3\xca\x7f\xea\xfaR\xb3\xf9k!\xd3h\xbc\xb5\x8cj~\xa8;\x91\xe3\xbe\x99\xd3|\xff\xda\xbd\xa8\xe7\x8b9i\xbdtL\xcd\xdb\x1f]?\x19^F\xcb`\x12\xe6\xee\xa2:v\xcb\xb7\xf6\xe4r\x1d\xdc\x9e\x0b\xdb>}\x0c7\xfab<]7\xec\xa5r\xdau\xfdp\x12_W\x7f\x99\'t\xa7%\xb5\x1d\xe4\x08\x03\xf8\xd0V\xc1\x94Z\xfcp^Y\xd6$\x07&d\xb9\r<2\xa0\xe2\x1f\xe5\tMFfi\xfc\xf2\x98ik8\x0b\xfd*\x18\x8d;\xf2\xc0\x8c\xdf\x9b\xd6\xfce&\xad\x07\xbe_N\xfc\x9b7\xf6c\xc9\x08"\xc71\x17Q\xb33\x89\xadN5\x1bD\xd5qi\xe8\xa3\x81\xe3\x99E86b-\xb7\xad\x17m\xbe\xd6\xb7\xb3U\x16\xef\x83\xdf\xe5\x89\xdf=\xbd\xb8\xd3J^\x07\xf3\x95r\xed\x1f3\xd3?\xc4\x89%w\xbb^\xd1\xffp\xa6\xbd&\x10\xff\xc6\x1c6\xb3x\xfaZ\xcc\xc3\xbf|\xcc\xa6l\xbb\xc8V\xadf\xdbR\xc0_UCs\x14\xc3tl\xa3\xad[\xb0\\Ko\xda\xa6\xdb0\xfe$\xeb\xabr\xb3\xddh6\xe4_\xf2\x95\xd4p>&An8\xbd\xc1m\xfa>\x99\x97J\xf3zM\x9e\x1c7{u\x0e\xab\x89\xe6\x87j8;\xbb\xfd@\xea7\x01\xdaF\xe1\xee\xed\xa5\xd1\x1dn\x9b\xf1\xfb"o?]\x06\x1b\xfb#_\x8f=\xfd\xc5*F{\xb9\xbf>n^\xad\xbf\xab\xdb4-\xdd0ZM~\x9e\xb4\x8a\xcc\x0cO\xd4\xad\x86\xa64\x14Wo\x82\xb5\xd9J\xdb4\xb0\n\x7f\x82\xe1\xb0\x8fJ\x03;\xfa\x8b\xd1f/\x1d\xe9u\xb8wZO\xf3\xeb\xd3\xe9\xaa\xaf>\x86\xe7\xf7\x95\xf4q\x93\x82R\x0e\xad\xe5fw\xdd\x07\x91n?\x1f\x96\x8b\xddz\xd7:]\xb4[\xa7\xd7\x19h[S\xcbv\xbd\xfeb\xff6\xdd\xb4\x92$Xv.E\xd2\x9a<\x1d\xdf\xa5\xbf,6\xb6\xf89\xcd\x92\x0e\xb6\xe6\xe8mS\x93\\\xc7qm<4\x02\x12 Y\xa3e\xbaMYn;\x7fb\xb4 n\xcdFK\xfe\xb5\x00o\xca\xf5\xe6\xb0j\xabS\xfbI=\xb6\xb3\xfd\xd1\\\x07\xc1\xc2\xdf&O\x8b\xf7\xd5\\\x89\xb4q\xef\xd2\nO\xc6\xa07\xde\xbd\xcf\x0f\xfb\xf7\xa4\xd9_\xf4_\xe2}|\xf4\x9c\xd9v\x196\xe3,\x88O\x86\xe2\x86\xa9!\x15\xbd@m\xaf\xec\xdf\xec\xe6\xff\x99@\x8e\xfce\x80\x8d\xb8\xc0\xf5\x96\xe5\xa8p\xfd\xb6\xa96\x8d\xb6\xad\xa9m,$\x98\x9b\x83\xec\xa8\xfd\x19\xe0o\xca\xaa\xac\xa8\xad\x9fW\xd1}y\x07\xd6\xee\xaa\x81\xe5\x8e\xa5\xa9\x1f\x99\xcffsR\r\xaeQ\xf3i:(+\xc9i\xe1\x8d?\xaa\xdby:_\x800\x8en\xc5\xc6:x\xea\xb4\xdc\xe7q\xb7\xe3\xbc\x8eo\xf9t\xdcu\xffc\xd5ml\xc4\xc6vSm[\x96\xe5J\xbalH:\xc8\x89a\x99-CQ@t\xda\x86\xd6\xc4\n\xb7\xff(M\xc8m\xd0\xbd\xd6\xaf\x9e\xbf\xb8:\xa7y\xf3t\x1b\xbc\xb7\xae\x9a\xbax.vkY\xf5\xab\x86i\xea\xb7H\xd9<\r\x92\xced\xf8._\xe7\x9bu_\x8e;\xd9\xeb\xb4\x9e\xefG\xab\xfdh|\x8e\xcf\xcf\x9by\x1f\x18r)\xed6\xbd\xf6\xc6~nu\x8f\xb7\xa2\x9e\xdf[o\xff\xed\xff\xa1\xd6\xe0\xfe\xa4x\xa0Sy\x974|))\x04\xf4\xbc\x1e\xbe\xfe\xf1\xbf\xfe\xdf\xff6\xdf.\x96BX\x81\xb5\x98\xb2\xda\x91\xb5z\xf6d|I\xc5\xc4\xd57\x05\xff\xa0\xe7\x0e\xb6\xd1\x0f\xd3Ue\xfb\xcc\xc9\xb4\x89:\xbc-y\xf9\xff\xf4o\xeb\xcd\xfb\xf6\xbeG\xd9\xf4\xf0\xdf/\xd3\xcdq\xb9 .\xd6eI\x91>\x7fz:\x88\x9f\xa9\xf8Q\x8b|by^n>\xef\x04L\x8f\xebs\xbc\xe2\xc7\xd3\xe3q\xbf\x9e\x9d\x8e\x9f\x82\x89by\x13p)\xcc\'*\xac\xee~9l\xe0\x89/\xf8\xc7\x9a\xa8u\xeb\xcaE\xfbO\x7fp\xe9\xb9Rnf\x95\x8b\xf7\xee\x96\x8f\x97V\xba\xb7\x85\xa3\x9fCe1\x0e\x940Zn\x16\xdd\x9e\xba\x1b\x04\xa3\xb6<\x95.\xe7q9\x1cO;\xa5>RZ\xd7\xb0\x1e:\xf3NV\'\xf2\x02\xbe\xb08\xa7\xc5\xa0\x1eK\xfa4\x9e\xb8\xc9\xa8\x0e\xcd\xa5\xd3\xce\x17\xee\xb8\x1f\x14C\x7f\x87<\x1f\xae\xbd\x98\x98\x05\xe7E&\xca\xe7\x0c\x8c\xf1\xf3{\x9evp\xca\xe5\x7f\x9f\x97k8\xc9\xef\xdey\xea\x8dW?=U\xe0a\x95\xf2n\x15\xe6\x85\x12\xe4I\x1dT#%\xa9\x13\xac\xff\\\xe9\xdbs)\xcc\x83\xcf\xefGJ\x1a\xe1\xfb:\xb8\x89\xaf\xe2\xfb\x84_\xd5 O\x0b\xbe\x9e?\xc7\xebU\xbc\xe6\x8a\xafR\xdfN\xe40w\x94\x90_k\x1f?\x1b\x97a\xe5_\x92(\xabq-|-n\x81\x1d\xe0\xab\x81\xaf\xd9%\xc9\x9d\x1b\xeeG\t\xa2t\x8d\xd7\xa9A\x9dh!\xee\'\x14\xef;\xc0k\x1c-\x89V\xab@\\\x13\xefS\x1bZ\xdf\x0eWI\xe5\xf3Z\x17\xbc\xe6\xd2\xb7\x07z\xe8\rd\xbc\xd7%\xc8\xbb\xab\x90\xef\x1b\xf9\x974/j\xfc\xb9\xa6\x1e\xdeO\x19\x16\xbc\x8f\xc06\xf3\xc0\x1e\xdc\xfa\x9e\xbb\n\x15\xbc>*\xae\xfd\xd8\xaf\x93x\xb8J\xaba\x9eF]\xfc)\xf0\x8c\xee\xba\x1fcm\xbcq\xde\xf7\x125\x8d\x1d)\xcd}\x85\x16\x16\xe4\x03\\\xd3\xa0b\xf1\xc6\xf7\n\xed\xb4\x08\xa2\xb0\xa4\x140\xb5i-X\xf2*\xb8\xa4|O\xdb\xb8\xa4\xd1@B\xbc,\x92\xc8-\xf1\xbb\xb8\xaeS\x87\xca@\x0b\xbd\xe4\xd2\xf7\x06uh\xfbrP\xf9\xb0\xfbD\x0bl\x17\xfb\xc2\xe7\xf6/\xfdx\xb8N\x14\xbcO\x84\xeb\xd6s9\x8c})\x8c\xcau\x18\x19\xf8\xeah\xa9]\x16}\x1b\xcfT\xa7\xb8W\xec\xa9\x12\xc8A>\xc7\x9a\x8e\xa4\xc0\x9ekA\xcc\tYG\xc5\xb5+\xf1\xf3\xcaQ\x93\x1c~\x94\x9be\xa0p-\x07R\x90\x17\x1a\xee\xf5\x9a\xd4\x8e\xd4\x8f\xdd*\xa9\x12)\xa93\xbd\x1f\xa5U\x92\x17x6_Mb\xbc\x1f\xfe\xe0\xbet\xfc\xac\x0e\xbd\x10k\\\xc8}\xac]\x92\xcf\xb50\xea\xae\xfb\xf6H\xc6\xefb/\x07jP\xe1\x1a1\xd6%\x0e\xb4\xa0\xea\xe6\x81\x97\xe0\xbe\xf0w{pM*\xb7J+_\xe6\xfa\xa56\xae\x1b\r+\xfc\xbb\xd6\x8f\x0c\xb1\x97\xb8\x0f\x15\xfb\xaa\x85\xb4\x83zU\x84\xf1x\x9d\xe6\x8e$\xf6\xd3\x9e\xdf\x12\xc5\xc1\xeb|\x19\xf7xK\xb8\xc6\xf9\xe8\x16\xd4\xd9%\xb4\xddu\x12\xf3\x9eG\\\xc3"\xa8\xb1\xbfX_\xc4\x12=\xb0\r\xec\x1b\xf6=\xc2\xbd\xe6\x99\xca?Xw9\x8d\xbbU\x10\xe1\xf59\xae\r\x9b\xe9Gx\x8fzQ\xa4^p\x83\x8dUa\xbdX\xa7U\xb7\xc2\xcf\x8b\xb0\x0e\xf3\xd0\x1eb\x0fGZR\x87E\x9a\xafV\xfd8\xc4\xefgJX\x0f.a<\xd0\x92|Q\x05\xcapE[\n\xe8_\xd8\x93\xc0^TI\x8d\xf5Ar\xc2\x1a\xe2\xde\x93+l\x0e\xf7\x8f{\x8d2\x15?\xd3\xd2|\x8e\xfd.`k\xb8\x1f\xda|\x0c/\x80M\xf6\xe3\x01\xec5\xadR\x9bv\xb9(\xe0\'\xd8\xaf\xc5\n\xcf\xa4\xd0>\xd2(\\\x07^ a\xdd\xf5\xbe\x97\xe2\xfd\x83kh\xc3\x06q\xcf\xa1\x17`M\xe1\x9f\x95\xaf\xc3^a\x1b\xb4=\xfeI\xb0\xcf\xf8Y=\xcc\x93\x08~\x13\x15z\x10\xfbX\xf3\xd1-\xc939\x8c\xd2\xffu\xf6\x81x\x10\xda\x0e\xbeO\x10g\xc6E\x825\x84\x87\xd5\x89\xe2\xdfxOI\xce}\x1b\xc3?\x03\xf8\x1abH\xc4\xbd\x86\x9fD\x06\xe3\xc8Z\xf8\x84\xc7x\xe0\xe3\xbd\xdc\xef~\x1er\xcd\x11;\xc2\x1a\xf7\x80\xe7KsC\r\xa2\xec\x1a*\xdd\x15|\x12{\x95b\xef\xe0_\xf4\x03/Q\x12\xc4\xa4 \n\xf4\xbem\xae`\x03\x05\xec\x84\xbe\x83;)`\x83n\xce\xfd\x0c\xf2\x9f\xfc\x1c1$\xa8\xe1\xb7\x15\xd607\xe4$\x1e \xf6\x8d\xb0o\x06^\x07.b;2b\x9a\x9aDi\x89Xt\x83}\x94\x88\x95\xb7\x84\x19\xc9\x1bI\xfc{\xa8\xa4%\xec\x0e\xf7\xd1\xad\xf8}\x8a\xeb\x06\xc2\xe7\xb3\xfb\xf5\xa2\xc1-\x89\xbb\xb8W\xec\x83\xe7\xc37p/\xb1\xbbB|\xbb\x04\n\xd6$\xe6\xda\x85\xf8\xde]%,\xdb\xdb.\xa2\xf7\\\tb\xe73.\x96?\xc4\xc5\xd4Ka\x83#\xe1;X;\xec\x95\t_*\x94$\x87O\xe6\x88#v\xbaN\xf9\xefJ\x17k[\xae\xc2(P\xf1;%\xf6B\x0e\x10\x07\x83\x9a\xf1f\xb1N"\xc4\x1e\xc44\xd8\xc7\x05\xcf\xa2\xf6c\xf8G<\xd0\xe9\xdb\xd8\xf3+|\xfb\x8a{D\x0c\x1b\\S\xc4\xa6\x00>\x97\xc2\xf7\xb1V\x15\xec\x0e\xd71WXC\xacA\t\x1b\x85\xefTX{\xf85\xd7\x19>-\x87\x88\x08\xb0)\xf8;bo\xec\xe8I\x94\xc0\xeeq\xbf\xb8\x1e\xd6\x1f\xeb\x02\xbb\xc1z\xd0F\x11_\xd5\xb4\x82m \x86\x04\x8c\x01\xd1\n~:\xbf\xc0\xb6d\xc4J\xe4\x00\xd8\x92\xe7\xd7\xc8\x1f*\xed&\xa1\xdf\xdb\x99\x06\xdb]\'9l)g,04\xfa+~\x8f\xdfc\x0f|\t\xfb{Ib\xda\x18\xe2s=\x84\xaf%\xb0G\xbco>b\xf6\xafB\x1b9$w\xf4\xd0\x9e\xc3^\x11\xebq\xcf\x01\xe2\x13l@\xc2\xbe\xd5x?\xc4\xdca\x19\xf2\xa0\xe1\x18\xebPa\x0f\xea\xac\xc6^\xe7I\x1e\xc0n\xe1W\xdc\x1f\xe4?\xecS\xc98\x06{\xbb&1\xf0\x8e7^\xf7\xa3L\xe9Gc\xfcN"\xf7\xbd\x91\x9e\xc4\xb0\x0f\xd8|\x9f6^g\xf4O\xb5\x1f\xd1\x1f\rY\xf8~n\xc0n\x1c>\xdf\x9a\xcf\x17\xd8\x05\xfcw\x85=\xa3\xbf\x8eqw\xfc\x1f\xd61\xfai\x1d\x19;a\xd7i\x85|\x19\xc3\xe7\xb1\x97XKI\xace\x8e\x1c_\xc1?\x90+\xf9^\x8cy)\xf7"\x1f\xfdf\x1dM\xc6y\t~\x8b\x98\xf1\xb9\x8e\xb0)\xd89r\x94\xa3\xc0\xeet\xe4\x83k\xdf\x1b\x96\xb4S\xc40\xd8X\x86\xbce\xc2nG\xb8\x97Le\xbeM"\xac/b/\xd6\x1c\xb6Z \xef!\xde+\x89\x96z\x0eb\x08\xec\xc8\x9e\xc3\xbf`\xc3\x1e\xee\x07kI_\xc0Z ^\xfb\xf4\x13\xd8\x04\xd6\xbf^\xadC\xc4G\xc4n\xe4\x16\xdc;l\x111\x03H\x06q\x1b~\x8aX\xa4\xc3\x97r\x11\xb3\x11\x1f\xe1/\\G\xc4!\xae#p\nr\x0c^\x03\xbb\x81/#\xbe\xc3\x8e\xea{\x9c\xc2\xeb\x95\x14q*\xb9\xf6#\xdcG\xbeZ\xa7\x11\xf1\x11\xf7\xd6\x85?\xe1\xd9=\xec\xaf\x9da\xcf\xba\xb0u\xc4f\xc4x<\xb7\x82|\xf4\xe9\xa7\xbe\x88%\xa9\x87\x18\x14\x15Z\xc0\xfb\x01.@\xec\xd3`o%\xf2"b*q\x07m\xdf\x01\x15@N\x88\xe0\x87\xf6\x189\x06\xb6\xc8\xf8\x8d\x18\n\xbbG\x8eA\xae\xaa\x99\x83\x86\xdc\x0f\xd8"P\x83\xd8\x1f\x83\xb1\x1b\xfeX\xe2\xde\x91\x7f\x80\x99\xc2\xda\x81\x8d;:r\x80\x94\x88\xfc\xe3\xd4I4F\xbc\xc3\xcf\x88\xa1j\xd8+mG\x81\x1f\xd0\xe7\x95.\xb0\x8e\xf0\x85\x9f\xe3\x12\xe2pF\x9b\xc1^\x8c\x10g\x10\x9f\x10\'D~@\x8c\x11\xbe\x8f\xbd\t\xea\x129\xd8\xe1\xfd!.\xcf\xf5\xdf\xc5%\xc4\x9bU\x1f6\xff\xdd>\x91\x9f\x91\xc7\x81\xd1\xe0\xdb\xb9\x8f\xdfGN\xa93)\x05\x16\n*\xe4K\x1b\xf0\x19\xf0{\xb8\xbf\x089\xbf\xa6/!v\xd5\x88\xef\x8a\x03\xdf\x82\xe5\xc0wS\xc6\xf2\xdc-\x91\xd7$\xf8\x19|\xdb\xad\xfa\x1e\xf6\x1b\xf96E\xbe\xc5{\xeb\xcc\'\xfd\x08\x19\xdaC>\x83_\xa6\xb4\xcb\x9c\xb1\x8e\xf8\xf5g\x7f\xfag\xf9\xec7q\t\xfe2\xd2\xb1\xf7k\xf1\xfe\xc2\x9f`/\x88\xfbi\x0c\x7f\xb0\x819D\xfc/q/C\xd8\xcd\xb8b\xceJj\x13\xb9\x98\xe3$#\xe4\x1f\xfa\xdb8\x0f\xbd\x14\xf7\x93I!\xef\xa3&\xb6\x02>\xaf\xb1\x8e\xd5\x08\xb1\x08{\x9f\xa7\xf0\x03b\x1c`(\xe0\x1d\xec=\xf01\xe2[\x0e\xac\x9f3F\xe0\xef\xc0\'\x88\xcf\xb8\xa7p\x9d\x8a\xf8\x1b\xc0\xcf\xba\xdcc\xfc>r\\5\xac\xee8\x1bX\x8d\xcf\xf3\x93?\x055r\x01\xec\x16\xb6\x80u\xe6\xeb\xb1\x07\x8asc\xbc`\xcc\xc1\xbe\x83\xaduK\xc6\x10\xc4J\xc4_\xe0\x18\xfb7\xfed\xaf\n\xdc7\xd6\n9\xe4\xd3\x9f\x88A\x10;\x81\x8f\x90\x03\x90[\xc1G\x80\x91\xb1\x9e\x15\xec\x01~\x82|\x84\xf7\x9d3F \x96 \xcb\x00\xc7\xa5\x1ep\x0f|\n\xd8\xf3r\x8fy\xc8\xa1J\x02\\L\x9c\x89\xd8A\x1f\xccK\xe0B\xf2\x1c\xae90\x95\xe2\x08l\x08;G>\x04>\x8b2\x9985$\xb6\xc0\x9e\x04\x02\xa7\xc1N\xb1O}\xe4S\xac\x15\xb8J\x00\x1fA\xac\xc0\x1a\x07\xf6/\xfe\x84\xb5Kh\xef\xe0\x1f\x01b\xfa\x10\x9c\x00\xf7gg\xf0S\\C\xac=s.\xb0\x9d\x87\xdcl\x03\xe7\x01\xa3\xfd>\xcfw\x91\x8f\x86\xcc\xdb\x9fy\xde \xf6\xa2\x9f\xe1*\xa3\xfa\x8e\xffq\x0f\xc4\x13\x88\x8c\x88\x019m=\x88\xb1\x16\xc04\xc8\x01x\x1d\xf2o\x04\x9c\t\\\x80X\xcd\xdc\x80X\x91\xc9)q^\x05{\xc6z\x041V\x08\xcfu\xb7i\xb0\xd3\x1a\xb9\x8e{\xea\x05\xf0\x13`\xad\x1cx\xd9\x06\x9e\x84/\xa4\x15\xfc\xd3\xa3\x9d\xe0\xd9h\x7f\x1ec\x12\xf6\xba\xc6\x1eD\xe0[\xd1\x1c6\x828\x91\xff\xecO+\xfa"0k\t\xdf\x01W\x84\x1f\xf6)`\xa8\x88o\xfd[R3\xfe\x02\x8f\x0bl\x03\xec\x87\xfc\x8fu\xf8]~\xc2\xba\xa5+\xda\xd2W\x9e\xc7\xdazX\x07\xc4\xa7\x94\x98\x11\x18\nv\xa20\xff\x06\xc4,"\xce"\'\xd5\xc0y5\xd7\xc9\xd7\xc4ZV\x8e\x8c\xfd\xd2\x89\x7f\x03\xf1\x9c\x0b\xec\xfb\x08\x98\'\\\x13s#f!n\x17\xc05|N\xe2\x04\\\x1b\x1c\x9aq\x1c\xb1\xa3\x04f\x07\x0e$\xb6^\x94\xa1\xb8\x0e\xed\x15x\x90\xdc\xd0\x1e\x02\xaf3\xe6\xbb\x88\xa9x\xff\x9a\xd8L\xdc\xef\xbf\x90\xe7\xe1\x1f\x1e\xfd)\xbb\xf1\xb8?\xdc\xabB\x9e\xc8\xf8\xd3g\x0e\x02\xf6\x02\x17\xc3Z\x1a\xe0#\xc1\x85\xfe\x8b\xd8\xf5\xbbu\xc4\xfd\x11\x0f!O\xd4\xdf\xd7\x111\x18\x18\xb5\x1a\xc9\xe4\xb7\x8cA\xe4\xa8\xb07\xf0\xff\x15\xfd[C\x1c\x04/\x1a\xd2\xe6V\xc2_\xc0+\xb0\xb6\xe0B\xe4\xcb\x03\xc4\x16\xd8\x07\xec\x01v\x08\xfc\xc3\xb5\x80M{\xe4\xe8+\xc46pn\xe4r\xd8#^\x0b\x7f\xf1\x86\xc4\xaa\xc8\x13\xc4\x9a\xb4\x1d\x07\xb1\x9c\xb9\x9e\xdcy\x00>\x82\x9f!>\x08\xcc\xaa \xcf\xdb\xe0\xa1\nc\xe0\xbf\xb8\x8e\xd83\xda^Jn\x08\\\x12\x92\xe7#\x16\xc3ve\xf0\x08\xf8\xc0\x188\x1fX\xbfb\xbc\x00\x0f\x01\x86N~\x8f\x97\xf0\x8c+\xe4\xd8\x00\x9c\xef{|\x87\x8d\xccU\xfa\xa7\xc8\x17\xb0}\xbc\xd7M\x06\x9ca3\x06\x05\xc4\xd9\x97\xfb\xfe\x8eK\xe6a\xe2Q`\x07\xf8"\xd6\xef/\xd71`}\x83\xfc\x89q+g\xdcB~\xa3]\xd4\x01\xf9j\x99~\xfev@;\xc8a?\x8c\x8f\x11\xe3\xbf\xcf\xf8\x8f\x98\x0b[\xf1\x02\xf8\xff\xaaB\x8e\xbf\xffv\x8e\xf7\xfb)\xcf\x80\xe7\x80\xaf\x8e\x88aa{X\xc7\x9a\xbc\x1ex3\xee\x02\x8f#_(\xb0\x1d\xe6E\xbe\x8f\xcd|\'\xb8\xd3o\xf3L\x82u\x02>\x90\xbf\xe36Q{\xe2{\xe0\x9e\xf9\xbe\xcc_^\x97<\x06\x9c&\x80\x9d\x91?\x01K\xe4f\x01\xde\xa3\x82s!.\xafpm\xbc\x0fx(0-\xde\xcb\xa8C\x85\xef\x0f\xcc\xa5\x8c\xc8\x9f\xb1\x9f\x85\x9c\x92\x8f\xc6\x88\xa7\x8c\xcbyX0\xa7\xe0~\x81S\x897G\x9f\xf9)e\xfeEL$wJj\xc4\x1b\xd6\xa8\x80K\xb86Xs\x8fy\x04\\T\xe4\x99\xbfZ\xc7\xff\x85\xfb\xf9\x98\xef`[\xd5H\xe9\x03G\x81\xa7\xf1\x9a\xf8:\xd0\xf0~5\xb98\xb8\r~\x0f6\xecuW)\xee?\x89Y\x9bs~\x9b\xefD>P\xc8\x8f\xe7\xdf\xf2\x1dp)bD\x0c\xbb\xad\xf0\x9a\x08\x98\xcc\xe6\xf5\x87%1b\xca|\x9d\x03\xaf\x83{\xe1\xfb:\x00\xcfK\x80\xa3\x80\xcd/\xc4j\xc08\xb0ss-\xea\x07\xe0\x07)\xd6\x14\xf7\x90\xb3f\x96\x02{\xf6m\x07y\x0e\xb8\xd0\x1eb\x8d\x10G\x91\xfb`\x13\xe0\x0f\x0e\xfe\x8e\\\x11\'\xd8O\xc4A{$j\xd7)qv\xcd\xd8S\xc0NYg\x19\x83\xefa\xff\x90g\xc3\xfc\x7f\xa9_\xfc\x0b\xf9\xa2\xcc\x81}XsU\x19\xd3\x89uS\xf2\xad\n\xb6\x8ak\x01\x83\xe3\xd9\x06\xc0\n\xf0-\x1b\xac\xca\x1e\xc2V\x7f\x9b/\x04g\xe2k\xbf\xe7]\xc4c\xc4w\x05\xb1t\x8dxS\x00\xc7"\x0f\xe2\x19c\xac\ry\x8b\x88?\x03\x999!\x8c\x89[\x90G\x15\x17\xbc\x10~D\xfeG_\xe0\xcf\xc8\x8f\xed\x80q\xe9F\xbf\x02\xfe\xc63co"\\\xab\x063\x84\xbd\x88\x9a2\xfcP\xd4ODm\x119\x955\xa2\x9ax\x9eq\x11\xaf\xaf\xc5:\xd5}p8b"p+`\x15\xfc\x1b\xf6\xf6?P\x9c\xfb\x17\xf6\x13y\xa2B\xeeE\x9c\n\xf3\xf9\x95X&@\xbc\x82\xcf\xe0\x0fy \xf32\xb0\x1a\xee/\x145kps\xe6\xf3\xdf\xe4\xad\x90\xfb\xf4\x03\xcf\'N$\xee\xcb\xc0\xc5Y\x9fcN\xc2\x9a\xc7>\xf6\nx4\xe23\x16\xfc9\xb1\xe6U\xf0\x88\x1c\xef\xed9\x1a\x9f=\xb5S`q\xf8=\xf08b\x02\xf6\x12\\\x985F\xf0R\xd6\x81`\x0b\x12\xed\x056X\x86\xa2\x0e\xc0\xd8\x9bq\xaf\xaf\xac\xa3\xb0\x1ez\x8f\xf3\xe4\xa5\xc0f\xf5\x1c>W \x06\xae\x10/Y\x87bN\xc4kks\xfds\xbeH\xeaE\t\xdc^\xd2\xb6\xf0\x15|\x01\xf7]37\xe3~i{1\xed\x026\x92\x13\xe7\x12\xe3\xd2\xde\x7f\xc7\xf3Yc\x04\xaecm\xf63_\x84\xf5\n\xb1\x1c\xef\x8f\xfdC^\xd2Y\xe3\xee\x8bu\x02\x8e\x80\xfd\xa6\xe4)\x82\xbb\x10\xff\x84\xe4j\xf8\xf9\\\xd4I\x80\x05W\xec\xa7\xe0:\xe0\x1e\xc0\x8d1xB\xbd(\xc2X\xd4k\x88\xa9\xea0b\xfd\xcf\xbf\x88\x1c\x0f\x9cDL\x8cX\x06\xfb%O\x156\x825\x06\xe7E\\\xc5\xcf\x15\xaes\x1a\x83\x1f\x08\xdbF\x0c\xcc\xb1\x07"ve?\xf3\x12\xc44\xf8 \xf0K\x1a\xb3\xf64\x07\xd7\x01\xfe\xb0YW5\xee}\x19\x0fX\x15\xf7G{\xc1\xba_D\xfd\xfcw\xbc\x04\xf8\x08\xbc\x02{\xf7-Nc\x8d=b\xff\x11\xf6\x90|\x9e5\xad\x8c\xbd\x1b\xe2\x10\xe0`\xfc\xc9Em\x19\xb6\x8a<\'8V\x01|\x8a\xbc\x8e\xbde/\x05q\x19\xfe\x0b\xec\x88X*\xea8\xc0\x8cI\x94\xe0=|\x95\x9c6\xac\x98/\x89\x97\x18\xa7\xba\xb4\x19\xd6$i\xa7:\xf0x\xcd<\x0e<^\xd1>\xe07\xd7@\xd4{\x06*r2\xd6b\x85\xfc\xca\xb5O~\xe5%\x88#\xa2\x86\x1c\xb3\xdf`\xa8\xc4\xca\xe0&2k\x81\xc0c9\xf9\r\xebC\x01\xeb\xc35\xf6\xdbf\xcd\xcb\xf9m\xdd\x8c\x18\x92\xb5\xf9\xef8\xd0c\x8d\x07\xb9\xc5\x83\xffT\xbc\xd3\x8c9\x9d\xb8\x0e\x1c\x99="\xe04\xf8\x02c\x16\x9e\xf7\xca\xb8\x825\xc23\xd1\xaf\xc8\xcb\xd8\xfb\x02\xdfg\xfd\x03{\x14\x08\\:\x02\xbf1T\xc1\xe9\xe3\x81\xc2\x9a~\xc8:oN\x0c\x83\xbc\xc8k\x91\xef!\x1e\xe2\xfd\x90\xf7X{D>\xab\x12\xd8~H\x7f@\xbc\x06\x8ef\x8d!F\x0c\x05?\r\xf3\xbf\xc7\x81?\xc6%\xd8\x02\xf1-\xed\xdf\x16\x18\xb7$\xce\xed\xc7\x81\xc4\x9e\x1cy)\xf24b#\xf0z\xee\xcb\xcc\xcd\x9f\xf9\x02~\x81}\xcb]\xec?~\x17\xd7\xa4\r2\xe6\x871m\xc8\x81=\x02\xa7a\x7fCr\x8a\x985\x1b\xf0\xfb\x1a~\xc9Zl\xcc\xde\x19\xb1\xfa\xfc\xf6\xb9F9\xebx\xf0\xcd\x1b\xf7\x839\x1fv\x82\x1c\xc6|D,\x8e\xbd\xcb\xc1\xe3\xf3\x81\xe8]\xe2\xfe\xb1\x9e\xe44\x05\xec\x87u\xf2\x1f\xe3R\x1f~\x0b<\x88\xfc\x0b<\x19\xe3\xf7\xc0=E\xefS\x19\xe7IN[\xc5\x9a\xe4\t\xec\x81\xf8\x05q\x80u\x81\xfawq\x89\xbd\xaaLO\xed\xafz\t\xfc\x86\xf5Q\xac\x15~\x9e\x07\xe4_\x12\xf1\x10b$,\x88}3`\xd0\x1a9\x9a\xfd\x0f\xe5\xce=\x80N\xb1\x87\xac\xdb\x0c\x0b\xfa\x13\xb8i%bqn\n\xae\xcf\xb8E\xbe\x16*>{\x1f\x05{[\xe2\xdf\xc9\xab\xe1\xd3!\xed4f\xff\x941\x90\xf6\x87\xf8\x91/\x90\x8b}\xd8%\xf6\xdf\x03WQ\x90[\x88+j\xc6c\xc6\xf7\x9f\xe3\xd2\x10~\t\xbeW\x05\xc0\x90\x03]p\x1e\xd6\x9d\x18sk\xe4o\xe6\x00\x8f=\x9d9\xe2v\xca:\x96\xc2\xd3d~\x1b\x97\xec9\xec\xc8!\xbe\xfe\x86\x1f\xc9\xdd\xc5s"\xdf\x80\x0b\x192x\x1d\xf2\x06\xf9\x98\xe8\x03\xddD\xdfI\x11=C\xe4\x1e\xc4\xa6\xbc\x0b\xbcT\x16\xc8U\xe0F\xf8S\x05\xb0\xaf9|\x03\xdc\xd3c\xbcd\x8d\x97};\xf6q\x13\xfa9\xf7\x88}m\x8d\xfd\xb3\xb0N\xe4 *a[\xec\x9b\x9a\xb0\xa31\xd7\xf0&j\x8d\xb5\x83\x7fG\xeeb\xe0\x0cp\xf7\x1b{a\x88?:\xebC\xec]\x8a\x9a\x1f\xf69\xa4\x1d\xfck\xf5dj\x8eX\'\x06F`\r\n\xb1\xd6\xf6\xf9\x9a\x02<\x05y\x8c\xda\x17|en\x8c\xd83%\x96\xfd=\xfee\x9d\\\xf4\xb9\xbf\xe3_>\x03\xec\xbb\x1a\x08]S\xc0\xd8H]F\xcc\xfe(\xf5B\xac\xb5\x01;\x8a^K\xa1\x8b\x1e\x04\xf52\xac\xd5`\xfd`k\xd4\t]\xa9}\x12Z\x1bj\xca`=\x823Q\'D\xfeR\xb3N\x0f\xbbU\x90{\xd9\xeb\x8a\x88\xd9\x82+\xf6_\x11\xdcH\xf4O\xd9\x1bf\x0f\x96\xf8\x0b\xf1\x11\xb6\x9c\x10\x7f1\x17\xe6\x03\xfe^\xfe\xbf\xb1nF\r\x06\xb3\x9c\xd0\xafa_\x81U\x91\xa7Y;\xf7\x02\xbc\x87\xc0#\xb0%\xe0i\xe4t\xee{@\x1f\xff]\xdf5\xc2\xfe(\xec\x13\x05\xdf\xf2\xcc\x8d5\x00\xf6\xd4\xf1Z\xf8|\xc1\xfe_\x1d\x88\xda\r{\x81C\xea\x1c\x10\x07\xc09\xc0\x1b\x02[\xec\x11bNF\x9d\xdeM\xf4\xd7\xe9\x07\xac\x8f\x93\xa3\xd3wm\xac9yx\x9e\xae\x04v\xad\x17\xac\xf7\xafC\xfa?p?|\xbc\xe2\xb5\xf0\\:\xfbh\xacK\x0b]\n\xf3w>F\xec\x1ch\xe4T\xe0!x\x9d\xaf!^_E\xbc\xfdY\x0f\x02^\x03\x1f`?%\x07\xdeX\xb3\xcf\'\xfa\x02\xb87\xe0f\xd6\xfb\xa8\xbbS\xd9o\x81\x9d\xc1\xcf\xe0K\xf9\xef\xfaDX\xab\x98\xca\x07\xc4\xcf\xef~\x9dP\xdbT\xe3~jj`\xc2X\xf4>\xc0\x87\x91S\x11\x0f\x93\n{_\x91\x9fS\x03@\xden\x16\xd4J\xd0N\x93\xca]\tmI\xe5\x8b:>|\x10\xd86\x85M#\x17V\xe0a\x9e\xcb|\x0c\xec\xc4>\xd3\x00\\\x1b\xbb\xc7\x9a}\xc4\xda\xa3A]\xa0\xce\xf8\x94VCbM\xad\x1f\xc1\'<\xf2Jr\xe9!{9\x88\x91\xd4\xf7\xb1\x16f\xd4\xbf\xf4]\x19K\x81\x8d\xd9\xfb\xc7^\xae\x99k\x92\xc8\x81\xed\xb0\x17L\r \xae\xc5z\x02k\xa8\xb0.j\x08\xd2\xe8wy\xc6\xb9\xf1\xdf\xc3\xa8\xfc\x9eg\xc8\xe1`w\xcc\xe9\xc4\xd9:\xfb[\xac7\xb0gLn\x81\x7f\'\x06\x81\xcf!fQ7\x10\xb1V\x00\x8cG\x9c\x19c\xef\x80%\xf8lBc\xc1\x1a#y\n\xb0n\xa0P\xc3\xc5\x18\x82\\SS\x7fI\xae\xcd\xd8\xe8\x12\x93\x02\x8f\xc2\x06\x15j0a\x1b\x11\x9f\x1bx\x9a\xbc\xaa"\x8e\xc4\xf3y\x8e\xf0\x9bD\xd4\xd6Y\xa7\r~\xc6m5{!\xe2z\x11y\x1a\xf5\x88\xc0e\xc8\xbb\x89\xc0N\xc4\x08\xc4\x9d\x85\x9a\x8a\xde.\xf9\x0f\xf9\xfdop\x9b\xa8S\xa4\xb9\xd0g\xdd\xfd\xe9\xc2\xd8\x87\x98\x84\xfbY0\x9e_\x92J\xf8m\x9dR\x8bY9\x17\xa1\xc1\xcb\xa9Y\xca\xee\xba\xcf\x98\xeb\xcf<+\xf4x\x88\xeb\x8c_\x027a\xcf\xa8\x1fe\x7f3\x03>\xa6\x96\x80\x9cr\x84\xd8D=\x04{\x96\xc8\xef\xe0\x8a\x88)\xc0\xb8X[\xd8PJ}`$z\xaf\xda]\xa3\x0b\x7f$\xa6\xa6\x9e"\x1a(\xacE\xa7\xe4=\xbf\xea\x82\xd8\x0f,\xc8\x8d\x80\x95U\xf0j\xd6\xbd\xb1\x0e\xc8\xf1\xacW\x00\xcfS7\x82k_\xa9\xd1\xc55\x95\xdf\xe7I\xd8"b\x00\xfd\xe9\xab\xfeKm\x03\xd7q~C.\xbe\n\x0e\x05?\x056#\xdf\xad\x90\xa7\xc1q\x91\xc3\xe0\x0b\xd8\x03\xd6\xdf\x80\xc1B\xe2\x80\x82\xbd\x02\xc6b<\xc3\xf5\x1e\x93\x91\xab+\xc6\x1a\xeal\xf1l\xd4\x9ePG\xc6z2\xb5\xc8\x111\x0b|\x08\x1c\xbb\xef\t\x1ex_\xd3\xc8@\xbc\xa4\x1c\x9a\xbdz_\xe4\xd3>\xeb\x845k>s\x85}j\xac\x87\xf4\xaf\xe5IjV\xd9\x83\x1a\xe0\xf7\x91\xeb+p?\xf2Ypk\xe4\x1c\xac\'\xb0y\r\xfb\x16\xbd\xcbB\xa6\xae9\xac~\x9b\'\xb1\x1f%\xf2\xd1C\x9e\xb4\xc9{\xa8w\x0e\xd8\x9fG\\C\x8cV\xc6\x15uR\xf8\x9e\xb8\xbc\x0e\xd8\xd7\x15:\xe2\xe0\xc28\x88\xfcx\xcf\xb76y7{`s\x9dk\x05\x0c\x88\xf8\x10P\x97)\t]l%\xfa\xab\x17\xc6\x7f\xd8zA\x1c\x8d\\\xa9\xd2\x8f\x04\xef\x16z\x99\x0c\xf8\x9d\x1c\n>\x92\xb3\x0f\x03\xdc\xc8\x18\x12c-\xe0\xdf\xfd\x08\x1c\x8f:\xc1\xfa\xef\xfb\xae\xff\x02\x1f\xd3X\xff\x13u\x00\xf8.\xfbl\x88\xb3\xf0\xb5\x81\xd0I\xb1\xcf\xdd\x07F\xc1\xba\t\x8eC\x0c\x9c\xda\xa3\xff\xa1\x9e\x02\xef\x81k:\xdf\xfc\x1a|\x9ek@\xfd!\xfb\xd4\xf4*\xc4\xd5\x9a\xba\x11\xf6\xb5\x17\xf8~\xc58\x0b;b\xae3\xb1\x87\xac7-\n\xea\x89\x12\xd6\x16+\xf6(y\xcf>\xf5}\xf0\xf3\xac\xe6^&\xd4\x1b\xc6~-\xeaD\x15|\x13\xd8\xaf\xcf\x9ee\xdd\x05nu\x80K\xc7\xc8%n\xc1\x9a^X\x83\xfbQg\x1b36\x82\x8f\xc1fDm\x8b\xdaj\xe6\x10\x9b\xbc\xe4\x97>\x00\xf0,b\x06\xb1!\xf6\x8b=v\xac=\xb9\x0e\xeb\xe6\xf8>\x03\x9fN\x04\xb7\xc4kT\xd6\xa4\xf1;\xbf\xad\xb7\xa5\xc4\xfb\x15\xb5\xb7\xdf\xec\x91\xdan\xf2G\xe2)\xea<\x0bD)\xd8Z|\xd7\xc3\x82\xbb+\xc4\'\xe4\xa0X\x0b\xf8<9+\xfe\xc7>\x04b\xa2\xa8\x07F\xd4LQ\xfb\x9aP\xff\x83<:\x07/\x1d\x8a| \xea>\xd4FP\xa3A,\x97S/\xb8\xa0\x86\\\x12\xb6\xeaQ/\xed\xe8\xe2\xbe\x98\xdf\x85\xbe`D\xcd\xa0D\xed\\\n|#b\xdf\xfd~\x7f\xec\x1b#V"\xde^\xf8\xec\xec\x15\xa5\xac\xd33/R\x7f\x0f\xce\x04\xbf\xe3k\xd4\x10\xef\x83\xb5\xc63\x83\xbb\xfeV\xef\x17PW\x0c\x1e\xc8\xf9\x86o}c\x83}_\x91\x0f\x13\xe6\xb6\ny\x9b\xdc\x95\x88\xa95\xe3\x0fm\x1f_\xb1\xae\xacw9\xf4S\x89=\x12\xd8W-\xb0\xe7\xef\xfa\xb5\xd4\xb8\x0b\x8e\x94|\xef\x03\x84\x82Sb\xcdm\xe4)\xac\xaf\xd0\x1f".!\xee+!q\rs5k\xb7\x9e\xd0C\xc2\xb6\x18\xe3\x9c\x9a\xf9\x19\x1c\x8d\xf3\x02\xf8\xd9g\xfd\x98~\xc5\xf5dO\x805\xcb<\x00W\xa1-\x15\xb2\xd0\x14sMlb|\xd6\xa1\x103\x15\xd6\xbe\x07x\x1f\xce\\\xb0\xdf\x03\xcc\x16\xb3b4\x87\x1d\x0f\x99c%\xf6\xa5\xd9cH\xa9\xb1`\xdd\x0814\xf4DoYb|E|\xa7^j\xcduL\xa8\x8d\x8e\x10\x17\xa9\x15\xf5\xa8\xa5\x10\xda\xa9k\x12\x0bf\x0f\xa6b\xbd\xd5\xafY\xcfC.U\xfb\xb4ir\x00pv\xd6\xd4\x10K\xd8o)\xfb\xa2\xd6I\xbcJ\xcd\x93\xcf\x9a?\xfb\x10W\xda+0F\x95\n|J}4\x9e\xafb\xde\xa4\r\x0e\xee\xf6\x93\x13\x1f"\x96\x12\xbfP7\xc7\x9a;bN_\xf0\x18\xf6\xd9\xc9\xa7\x86k\xd1\x83TX\x07\x06O\x8b\x1e\xec\xa7\x86\xcf(\tu\xe6\xaa\xe8\t\xe5\xac\x81R#\x8a|U1\x1f!\x7f\x81\x07\xc2_+p\xb0R\xd4\xf3\xeb\x05p,\xd6\x9e<\x90\\\'\xe7\xec\x07\xf0\xa9\xc2\xba/\x11>_\xdf-\x84\xbe\x88\xb8\x99\xfa\\\xc6Ja_\xdf\xec\x05\xf7fS\xd7\x08L\xc0:2k\xba\x8c\x1b\n\xf7\x81\xdaB\xe2C\xc4\x1d\xeekM\\L]\xad\xd8\x97o\xf6\x82\xe7C\xbc\xca\x89\x91\xb0\xde\x11x\x13\xaeK\xed\x1b\xf2M\xc5^!l\x9a\xf1\xafb\xed\x8b\x1ab\xf8\xb6\xca\xbaV\xc8\x9a\x9c\x07L\x83x\x18\xb0\xcfL\x9eC\xfe\x1f\xc3\xa7\xa9uF\xec\xe3^\xdd\xd7\x88\xb6L\xbe\xea?\xd8\x0br\xa8X\xf3L\xa2\x06\xbd\xcfXN\x9d\xacM-<{{\xec\x95\x02S\x8a:,\xf2\x16\xe3}]|\xd9\x8bG\xee4"W-\xb8/\xd8G\x99=^\xfac"z\xab\x19\xe3\x9d$\xb8\xad\xe23\xe7_\xf9\x1a\xe6/\xcez\x89\xd8\xc0\xda\x04s\xaa\x17\x16}\xa1#\xf5a/\xd4\xc5\xc2>\x05\xbec](\xbb\xf6E\x1d\xf8\xc1^\xd8\xffd=\x96:9b2DF\xe4\x0e\xeaT\xd7\\/\xfa[P\x0f\xa9Y\x05\xeeeN\xc5:\xda_\xf6B\xfdK f\x19JrG\x95\x1a\x0c`p\x99\xba@\xec[\x95\x88Y\x08\xda-sVpK\xa9\x97\xc5\xde\x88u\xcbY\xeb\x9a\x03\xcfb/\xc4l\x80\xcb\xfe\x8a$\xe2\r\xe7Y\xe2@\x17\xd7\x12\xfaZjH\x1e\xe3\x0b\x9e\xd1\xa6\x0e\xbf\xbb\x12\xdc\x99{\xcd\xdef\xc4\xfe\x16\xf9"k\xab\xc0\xe7\x15\xe7\xa2\x88\x9d\xc03X#\xfbn/|\rb\x89\x02\x0bc\x9c\'^\x8a\xb8\xbe#\x99\x1a?\xcex\x08\xed\xbc\xc2\xbe\xee\n8\xcf\xc7\xf5X\xabA\xde!\xb6\xad\xd93#\x86D>T\x86\x95\xa8c\xc4\\\xa7U\xc9\x9a\xcf]\xa3\xe0\xb37\x0e\xa6\xe1\x10\xa7>\xd8\x0b\xeb\x14\xc0w\xe0\x0c\xd4M\xb37\x88\xe7-\xa8\x93\xc3\xbd\x02\xd30\xb7\xb0\x0f\x8e\xebs\xae\x90\\:\x9f?\xc4\x17\xf6A\x16\xf9\xbd~\xe7\xeb\xc0\x849\xd54)\xf5\x10\xc0\xe1\xd8\x0b\xe4\xc0\x11\xb8\x0c\xb93\xb8\n\xf61!\x97e\xdd\xa4\xa2~\x88zm\xd8\x00\xf9\xa1\xd0\x0e\xb8\xac\xd9_9\xb7\x96D\xd4\x14\xb0.\xca\xbe%\xe2\x1d\xed>\xfa\xb2\x97>k\xd0y\xc1\x19\x9d\x9aX\x0e<@\xa7^C\xf05\xf6\xad\xbc\xbb\xe6\x1av\x0f\x1b6\x90ca[\xf9\xa3\xbd\xcc\x11\xcf\xc0_\x85^\x04\x98\x02\x9c\x0b\xb1\x9b\x1f=\xc0Z%\x9e\xa9\x8b5`/\x80\xf5\xa8\xf9\x85X\x941= \xdf\xa9\x85N\x16\xcf\x15\xc8\x81\xc0\x92\x99*f9D~\x02\xfe\xa7\x16Q\xd8K\xa0\xd1\xee\xc3\x1f\xf2\xd1@\xa6f\x95\xfd\x13\xf6:\x13\xf8%5\x8fxnv *\xf6\x1cD_\xb5b\x0f\x14\xb9\x84\xf3\x08\xd4\x18|\xc5\x17\xe4\rr\x17\xf0C\xd6\xa0\xd9\xaf\x8dB\xfc{ #O\x01\xf3#\x1e\xd5\xac%\xb2o\xc8>+\xf5GB\xa7\x84<\xc7>\ng\x90\xa8\xf7e\xef\xd9!w/E\xed\x893+1q\xb8\xc8}\x97\xbeG\xcd\x12\xeb\xee\xc2^nw{\xf9gz\xdb\x87\xf8rI\x85\xd6\x92:&\x81Q\xb0w\xc01X\xe3\x94z*\xee\xa17\xa0\xe6\xa0&\x9fb-\x04x\x12\x98\x0c>\x1d\x01\x8b\xb2G\xcd\x9aU\xce\x1edBM\x06s\xd8U\xc4\x1a\xf8\x16\x9e\xfb"\xf0\x06mR\xcc\x11\x8a\xf5\xbe\xf1\x19\xb1.\xac\xd3\xd7\xd4\xf9\xe09\x0b\xeek\xcaYG\x91\xdf8s0\xa0\xcf\x89\x99/\xe0\xcf\xeb\xe3z3\x8e\n\xbd\xaa7\xa2>\x91=}\xe0R\xe45\xce\xa3"\x8f\x13{\'1gT8\xab\xba\xc8\xfb\xac\xe7\xd6\xc5\x8d\x1c6\xb9kZ8\x03L\x1dB\x01\x0eC-\x1e\xf6\x1e9\x02\xcf\xcf~(\xec\x84:_5\x11\xf1\xd0\x17\xf6\x9dr\x1d+\xd6e\x18\xab\x07r\xa0\x8cWB_\x1d\x11\x17\xc3\x97\x91s\xfb\x9c\x0b\xc8\x81\x8f\x04\xac\x8bPkP\xb1\x86\xc9\xde\x0c\xeb\xfb\xe4\x91\x88\x1fx\xaf\xfe}\xce\x8f\xf3J\x1a\xeb\x1b\\\x17\xd8\x07uM\x0f\xf6\xc1\xde\x13\xeb\xe5BK\xabs61\xa0\xb6\x01\xd7\x15\xf1\x8b\xb6\x8ex\x86\xf7\xa8\xf9\x1cI\x05\x0c\x93?\xd8\x07\xe3h=\xa7v\xaa$\xc6L\xc9\xc9\xc0m\xb1\x8e\x8a\xe8\xf7\xc5\x9c{\x0bx\x8c\x00\xf6\x90\xbdx\xdaZ!\x8b\xf9\x05\xe6+a\x17]z\x91N=\x01k\xc8\xc0Sx\xce\x02\xf1\x8b\xb3\x8cX\x17\xe2~\xc4\xad\x07\xfb\xa0\xb6\xfb\xc2y\x13j\xa0\xc0a\x10\xa38G\xe32v\xa9\xf7y>p-\xf6\xefk\xea\xfe\xc9\x93\x17\xf9\x83}`/\x88\xbd\xa89\x13=\x1d\xd8,\xf5\xf3sr\xac\x1bu\x1f\x9c\xdb\x81\xcf\x88zn*b\xa6_\xb3fH=\xdf]\xf7\xc0\x9a\x1c\xd6\x9e\xb3\x99\x8c\xb7\x1e{\xc8\x9c\xb9\x00\xa7e\x8f\x8by\x0b1*\xac\xbf\xec\x83\xbfC\xcc\xc5\x19\xa2\x00\xfc\x8f3\x97\xd8g\xbc\x07\xef[\xccc!\xd6\x1b\x1a\xeb\x0eb\x9e\xd0\xf69\xbf\xf9\xdd>`w\nc\x1a{d\xa1\x98\xc7\x01\x7f\xa4\xc6,\xa2\x0f$\xe4\x9b\x12\xd7\x1d\xfc\x00\xb9y$\x8b\xb9Y\x8f:\xad\x92\xb1\x06\xf1\x93\x9a2b;j\xe2\xc7w^)\xf2%y\x1cg>M\xe0;\xc4\x9c\xfc\xc1>\xc8;\xa8\x89$ve\x1e\xaf8\x87C\xce\x08\x9bf\xbc\xe7LZ\xc5YC\xd6M\x06\xc4\xfc\x82[|\xd9\x87\xa8\xab1\x7f#\x97\'\xe0\xfb\xd4)\x8c\x98{X3\x02\xee\xa0n6\xe1<\xd0U\xe88X\x8b\x07\xb7\x0f\xeb\x05\xf2&g\xcfh[\xacss\x9e3\\\x0b\xcc\x9d\x13\xcfR\xc7\xca|\x80\xf7f\x9d\x95\xfd\xab/\xfb\x00>\x1e\x13c\x02\x1bs\xaf\\\xe21\xe4\xb6\x02x\x99z@\xd8:\xebP\xd4\xe6z\x02=\xd0g\x1e\xe3\x07u\x0f\xd4\x91(\xc4EB\xc7U\xf3=X\xdf\xc5\xfb\x89\xf5C|!\xc6\xe4\xb3\x88\x9c\x11\xc2f8\xa7H\xde\xe4\xd7\xe2\xbd8wV#~\xe5\xddB\xf0\x02ak\xe4\xd4\xc4\xb1\xec_p\xde\xfa!~p&\x9c\xec\x01\xf9\x8auX\xac\x17r\x0ek\xb5\t\xeb\xfd\xb7\xbb\xfd1.\xfa\xac\xa1\xe6)u\x84\x8f\xf8\xdb\xa3\x1e\xd8\xb9\xb1W\x9a\xd2v\x81\xa7\x80\x0fj\xd8\x03\x90Y\x06o\xa6.G\xf0"\xe4\x944\xe7\xcc\x94\x88W5\xe7\xc2YOE~V\x803"\xceOQ{8\xae\xc4\xac\x1e\xe3\xaf\xcdz\x0ek\xa5\xb8\xb7h\xf4\x10?\xc0\x0f\x11;\xa9\x8b\xc4}\xe0{\xd6\xecW\x85\xd0\x0c\xd9\xc1E\xcc7\x8a\xfa-\xf6\x02{\xc09\x7f\xce\x82}\xd9G&\xf2J\xc0~z\x95\x00\x07,\x10\xeb\xe8\xf3\xec9\x94\xd4\xff\\\x99\x0bE\xbd\x80=\x7f\xe6\x01\xec\xcf\xbd.\xee\x96b\x0e2\xe2\xbf\xe1}\xd93\x8f9\xefP\xc0>\x10\xfb\x14\xc4`\xa1\xeb&\xd7\x1e\xd5_\xf6\xf1\x0fu\xdf\x0f|\x01\x98Q\x15{\x8b\x1cE\x9e\x9dF\xb8\xf7\x18vD\xdd\xbc\x98q\xa4\xfe\x9a\x98\x83\xb3\x95\xf4\x13pY\xea\xfa*\xe6s~\x04\x12k7\x86\xc4x\xc35\xe2\xec\x9b\x98\x85`?G\x9cAQ\xb0g!\x8b\xba\xc9\xb7\xf8\xc1\x9e\x892P\xa9\xcb\x04\x9f\xa1\xce\x90u7j<\x14Q\x93dm\xac\x0e\xa4\xd4\x1b\xb0\xde\n\xdf0\x88\xe1\xbf\xe2\x87\xe8\xb5q\x86\x973\xf8\xb47\xc4\x1e\xce\xea\x0b\xde\xe1\x10+\x12c\xc0OX\xdbD{\xc1\xd8\xac\xb0\x7f\x9fp~\x0f\xb1\x19\xd9\x01>\xc2\xda:\xcf(I\xc0\xcb\x98C\xe8\x1f\x0f\xf1#B\xce\xa5f\x84s\xb76kc\xccGx\x1fj\xc0"\xd6\xe1E\xbc\xc4:\r\x84F1a\x8c\x16g6\x14\xb8\x17\xc6\xe2\x8c\xf3\xeb\xc4\x0c5y\xb2\xe0\xc56g\x0f\x879u\xf5\xc1]\xd3\x80\xfc\xf9\x10?<\xf6\xdd\\Q\xd3b\xbd\x99s\x8eT\xfb\xb1\xc6\x11p\xad\xa95\xa4N\xc7fMQ\xe8\x0b8\xab\xf9\x10?\xa8\xbd\xe2y"\xd4 \xad\x84F9\x10=4j\xb4\n\x9d\xf9\x96\x9aa\xc6BjX\xc1\xfbWB\x9f\x85\x1c\xc2Y\xf3\x94g\xb4P{/\xb0\xe0\x88\x15`]\xcc\xe4p\xae3\xa2F\x8b\xb8\x0c\xbf\x03?\x7f\x8c\x1f\x81\x98\x7f\x19P+XS\xa3\x01\x8e\xa4\x08\x1dT\xc5s\x15\xc8\t\x1cbH\xac/5F\xec\x11\x86\x0f\xf5\x06\x03X\x94gip\x06R\xd4Gn\xa2~\xce\x1c\xcb\x19\xbb\x98|(\xe0\xac\x00\xf8\x05g\xd5\xa8\xa7`o\x8e\x9a6\xf2\xe3q~\x9f\xe9J\x84~\xb4O\xad$k\xb6\x11\xb9&5\x1d\\\xdb\x90s\xccu\xfa\x10?\x80\x81\xc4\x99\x07\x89\xe8\xbf\x81/z\xac\x0f\x19\xba8s\x04X+\xe1}\xe5\xf3\xfb\xf9\x02\nx\x9f\xe8G|\xd9\x07\xf3\n5\xe9\xec\x11\xb1\xbe!f\x84\xa8E\xe7\x1c\xa5\xc2\xf5\xe6\xfdSS\x0f\xfbR\xd8\xd7\x86\x1f\xda\xd4CQ\xd7j`?\xbb\x15\xfb\xf4\x88kW\xf2Br\'\xc6\x0f\xd1\xd3\x8c\x02\xea\xf0\x81\xf7\xa9\x1d*\x1e\xf2\xcb\x1c\xf9\x95\xf9\x1a\xfb,\xf8\xd2\x82\x18\x1c\xf70\xe0\xf98\xd4C\xf1l\x05\x8d6\xc5\xfe\x10\xe2\x14\xe7\xce\x1e\xec\x83k3\xa4\x06C\x163\xbe5\xe7\x05\xa8!\x06\xcf\x175d\xe6[\xc46\xa1\xaf\xa6.\x1bv\x92\xb3gO5\t9\x0c5H\xac\x11\x96\x15\xb5j\xa2\xa7\xce\xd8BM8\xcf\xc8\xa9\x05>\xe5\xdc\xae\xfa\x10?4\xea\t\xc1}\xa8!\x84=\x02Wrv\x00\\=d\xadO\xa1\xd6\x9f\xf5\x0b\xea\n\x07WQ7\xa2\r~\xd9\x07kA\xa2\xb6\x87|\x06\xbe\x01_\xf4F\xac)\xb2\x87O\xdd\x0cu\xe9\x8c\xe5\xc8\xe5\x9c\x91O\xc5\x0c1m-`\r\x805\xc2\\\xcc\x90\x14\xd4B\xa4<\xfb\xa6\xa6\xa6\x81\xf1\x0fkE.\xe0Qo.\xae\xf1\xdd>\xfe\xd9\xfc\xc1c~\xa1\x8e\x9f\xb3\x14C\xd1\x1b\xe7z\x07\xbc7\xaf\xbbF\xfeD,3K\xa1A\xb2Y\xcb\x06>\xe3\xb5y\x1e\x02\xb1\x9fG\xbd\x01p\xffg\xed\x9a\xfa\x1dj\x04C\xc1\xcf\x13]\xf4\xd9E\xad\x82\xda\xd4\xe4\xc1>\xa8\xefb\x7f\x831\x84Z\x08\xec=\xe7\xdc\x84>.\x13\xe7B\xd1N\xa9)\x0b\x15\xb7J\x04\xcf\xf5\x1f\xe3\x07\xf5\xa6\xea7=\x02l\xa1\xa2\xcf\x90\x93\xde\xcf\xf1\x11\xf5\xc2\x8a\xd8\x95}\x1f1\x17+r\xa2\xe8wT\xa2\xf6\x8d\\@\x8c$j\xd4b\x9e\x9d\xf8\x83\xbai\xeaq\x05\x96&gR\x1e\xe2\x07\xcf&\xb8\xb1\xaf\xf6YG\xaa\xa9S`M\x85=r\xd8M\xc1\xf8\x17\xe6\xc4\xcas\xe9>\xbb\x15<\xe0S\x03\xd8\x86\xb9:\x15\xe7E\xb0W \xf4\xd6\xb8/\xf6\xadxF\x00\x7f\x179\xb3\xa0~M\xf4\xa9l1\xe3T\x89\xf8\xc4\xb3>"\xea\xf4\xb0\xb6\xc8\x81\xc0F\xba\xd0\x13\x92\x1b\x827\x05\xec/q\xc6\x956\x18=\xc6\x0fG\xe7\x99V\xd4\xe2\x93\x07\xb0\xee\xc2\x9a\x17\xf1\x11\xe7v\x88\xd7\x88\xdfXg\x14\x9cR\xcci?\xd4or\xc6\xd6L\x15\xb3\xa6\x115\x10\xb07\xf6\xcc\x15\xf6-\xf1w\xbe\'\xfb\xc7\x15\xe2-\xe7vD_\xc5\xa7\x9e\x04\xb1\x1f\xfe\x12\xb3.$z\n\xea]\xd3\xc6\xf3\xc1\xd8\x13\xa0f\x93gL\x01\x97\xd3\xe6\xeaG|\xfa\xcft\xfd_\xf6Q\x96\xa1\xe0\x85\xf4}\x9f3 \xc4\xd0\x92\xc8\x19\xd4\xcf*\xc4\xf7|\x1f\xea\xf6\xa8m\x1d"O\x83+\xb3\xbf(\xfa\xa8\x9c\x11\xca\x10\x03\xd8\xbbAlS\xc8\xa3X\xff\xa0/\x91SsmX?\x14u\xa9\xef\xf1\x03kT\xa4".Q\xff\xc4\x98\xb5Z\xf5\x05\x96\x0fy\x1e\xd9U\xf0k\xd1\x03$_\x0f\xa8\xcb\xa8\x1f\xf3\x0b\xeb"B\x9b\xc1\xb8\xc7\xf3\xce\xc0\xd1\xef\xe7"\x80+E\xd4\x8b\x899f\xcev\\Y;\x11\xe7\xb9\xc4\x9c\xf9\xe4\xfb\xf2l!Qc\xd2D\xbd\r\xd8\x98kO\xfe"\xfaI\xc2>X\xcf\x1a\xfc`\x1f)\xe7\xc8\xa9\xc9\xc9\x91;Y7$\xb6\xf0\xa8\xe9d>\xa3\x16zH\xcd\x16b\x11{\xfd\xac\x85\xf3T\xbc\x87\xfc\x923\xdfd\x88C\xd4\xdcP+\x08{\x8cB\xd6\xcbU\x9e\xa3s\xb7Gq\xfe\xc0E\xcc\xef\x03#\xf1\xac\xb4\xe4>\xbbH\xbe\xcb\x1e\x1c{c\x171\x8f\xcf\x19-\xda8\xb9\x92\xcd\xf3\x0cL\xe0\x00\xd69\x06\x8f\xf5\x0f\xfc\x01\xa7c\x8ce\xdd\xb6\xbe\xeb%Y{\xe2y\x05)\xe7bE\xddru\xd75\x8asM\x92G\xfb\xe0l\xc25\x11\xda\xcb\xc5Z\xf4\xf28\x83d\x0f\xees\xbe\xd4\x1b\xc5\xd4J\x893\xc2\x10\xc7\x11\'+\xce\xb5\xb0\xc6\x8d\x18\x0c\x8c\xc3:b\xc0X\xccz\xb8\xd0L\x10\xbb\x8ej1Se\x8b\xf3\xb4Xo\x90\x1e\xe3\xc7?\xd2\xcb\x7f\xd9\x07k\x06B\xbf\x87\\\xb8\x16\xfa\xfd\x98\xba9jq\xd9s\x00\xe7\xa1\x0e\x1b\x9c\x8b\xfd]\xae\x898+\x8a\xd8\xd9&\x17\xe6\xfe\xfa\xd4x\xacD\x1d\x05\xfe\x8akP\x83\xba\x16\x18\x94: \xd1\x97 V}\xe0/\x9c]\xe6\xda\xc5\xd4\x86\xb2\xae\x08<\x11\xb3\xf6C]\xf3\xfc\xca\xd9\x1fp}\xa1\x91\xe69m\x81\xd0\\=\xdaG\x97\xd7\x15\xf3f\xec\t%\xec\xfb\xe4\xec\xc1 \xbf\x88\x9c#r\xb3\xd0\x13\xdc\xfbzsEp%\xce #\xd7\xb3n&\xfa\x86\xf9\x80\xfd\xcb5{`\xe2\x1c?q\xb6\xd6\xdd>8\x9f\xf9\xa3}\x18\x1a\xfbV\xe2X\xe3\x06f\xf2\xd8\xf7\xc6>+\xac\xb9\x93;\x8d\x05G&\xef\xa2\xce\xee~\xc6\x17k\x85\x9c\xc3\x13:?\xce\xed\xb2\xf6\xcf^q\xc9s\xa9\xb8V\xc2G\x80oX\x1b\x80\xdd\xdd\xed\x83g\x8f\xfc`\x1f\xb8\x7f\x9e\xd7%\xf8%\xcf\x88B\xccP\xc4<^E\x8do\xca\x1a\xbf}\x9f\xdf\xe2\xb91\x8c3\xa1\xc7\x9a\xd1W\x7f#\xa1\xd6\xc6c\x7fDh\xcco\xe2\x9e#\xe4\x16\xce\xc3\n\x1e\x0b\xecUa\x8di\x7f6{\x00<\x0b\x839e\xc1\xb8\xa1\xf3|C\xf8\x94$4T\xd4\xc0\xf1\xb4J\xea\x80\xd8W\xa0}\xd8+\xf0\x90\xd1\x0f\xf6\xf1\xcft\xe8?\xf4\x07\x80I\xd8_pD\x1f=\x10g\xa4\xc16r\xd6/Yo\x11\xfaR\xf5>\x9f\xcez!{\x84\xc4\xa3\xbe"\xceZ\x01\xaec\xee\xc5\xda\xe8\xa2N\xc33`\x98_X\x7f\xba\xc7\x0f\xce\xc2H?\xdaGY\x88\xf9\xdd\x8aZ;\xeas8O\xc1\xf3\x11\x05oVD\x97;\xa7~\xc3\xbd\x9f\x07\x16\xf1,\xbdG\xfeB\xbf\x14u\xaeK"4\xd1\x9cq(K\xa1\x87b\x7f\x168<\x15\xe7\xd0\xf0|#\xd6\xa0\xd9{\x17\xfabb\x03\x8d\xfc\x91\xf3\x0c\xf7\xf3\xdf\xe6\xe2\xac\x07`\xf9\xeb\xbd\x17\xe0\xdc\xed#\x1e\xe3\x1a?\xc4\x0frl\xac\x97)\xe67\x85mp\x8e\x9a5L\xea \xd8\xdbe\x1f\x94z\x13\xea\xd1\xd8k\xae\xd2\xc7\xf8\x81\xfc\xcb\xbe-\xb9oW\xcc\x93\xe1\xf9e\xa1\x03\xa4n6\xe6\x8c!\xf5\x1b\xd4\xb4\x83W(\xcc\x9b\xec\x8b\xb8\xf4u\xc4:1\xeb\xc2\x99\x90\xfa^od\x9f\xbe\xccE\xff\x8bs\xa2"~\x8c\xa9/\xfc\xc1>\xb0\x0eX[\xf2H\x9f\xe7^b]\x047\x13\xe7 \x85B\xc7\xc5\xbd@,\xac\xd8\x0b\xe0,+1\xe6\x03>\xb5\xd3\x82\xfds\x81\x97\x88Ub\x9e_0\x12\xfa{\xf6V\xa8\xf7\xe1\xf9\x1d\xb0[\x85=W\xa1\x01 \xe7\x00\x97\xe3\x9c*k\x04\xe2L\x11\x853\x9c\xacS\xb3N\xc3\xfa\xd8\x82\xfe\xc73\xf7t`9\xc4\xad\xe0\x11\x7f\xf0\x8c\x86\x82\xfaK\xae\x81\xe8\xa1q}D\x9c,Tq\x9e\x0f\xb5y\xac\xfdP\x87\x06\xeex?\xbb\xee\x9b}\x885\x91\xe8\xfb\xd4\xd8&\xac\xd7)\xd4\x18Q\xf7\xc4s=y\xd6!\xfb\'\x8ca\xf8cS\x97\xc83%\xb1\'\xf1\'\xd7\xc8\xd9/\xe3Y\x92\x8b\\h\xf6j\xea\x98J\xe6\x87\x9a\xf511?\xc8\xba\xe3W\xfc\xe0\x9c\xd8\x95\xba\x06\xb2I\xecGM\r\x82\xd0\xba\xe1+\xf3J\xca\xbc\xc1\x1e\\\x8d<\x05\xde\x10\x8a\xd7}\xf1\x17\x81\xa3l\xee\x05\xf05{Y6\xcf\xd3\xa4\xfej\\\x8a\xd9zq\xfe\x95\xd0\xb5rV\xe8\xca\x19\x8f\xb4b\x9dtA\xde\x8dg\xa3\xde\xc3\xa5&\xf0\xc6\xfa\x01\xf5\xa4\xa2\xfe\x11\x83O\xd5\\\x1br\xbfy\x1d\xe4?\xd8\xc7\xf5^_`M\x99y\x11\xf9\x87\xb1\x9f\xd7\x81\xffQ\xff\x10\xf2\x1c\x1c\xcegs\x165\xe7\x19^\x8f\xfa\x05\xce\x1a\xb9Bc\x87\x18w\x03^\xbc\xb2\xdf!zl\xd4/r\x0e\x918\x9c\xe7\x91D\xf7\\B\xed9\xcfi\xa5\x0e\x8e\xb9/\xe5<\x0e\xe2-{K\xe2\xdc\x07\x9b\xe7\xa1\xb2\xbe\x14\x88\xfe\x0b0\x7f\x15<\xf6_8_ \xceO\xa3\x96(\x93\xa9S\xe7\x19\x95\xd4\x0b\x81o"\xcfS\xc7\xc4s?\xc9\xa3\xc9-\xf0N\x9cS\xfen\x1f\x85\xce\x9e_J\xcd\x95\xe8\x19\x98<\x8b\x03y\x9d\xbd\x97\x90\xa7<)\x82\x1f\xf2|\x08\xf6\x9f\xa9\xdf\x16\xfa\'q\xa6\x98\xd0\nP\x97\x93Ro r\x8b\xb0\x05\xa1\x17\x11\xf6!\xd6\x96\xbd*j\xda\xbe\xc7\x8f\x7f\xa6\xff\x7f\xcc/\x027$2\xe7\x18\xa8\x7f\xe0\xfc\x89\xc0=\xac\xe5\xb0w\xc8\x18\x04\xfe"\xe6%\xc4\xb9n\xd4\x01\x8az3gM\xa89\xa8D\xae\x14\xe7\xcc\x82\x8f\x80\xdb\t\xbd\x05\xcf\xd1\x10<\x02>K\x1fx\xec\xbf \xff\xb2\x0e\x08\xbb\x93\x02\xce\xa3\xd5\x9c\xdfb\xcd|Qrf\xbdO\xfd\x17\xb5E\x02\x0bR\x9f\xcb^\xd0c}\x8c\\\x84\xe7iP\x17\xcaZ\xb9\x98u\xe3\xf9U\x05\xfe.\x91S\xa7b^g \xb1\xa7\xc8\xd9\x82\xfb\xeb\x85\xae\x10x\x81=n\xf6a\xe6\xaa\xa8\xed\xd3?\xee\xf6QPcs\xb7\x0f\xc4\xb8\xca\xff\x01\x7f0W\x86\xf73\xd6\xaaD\xe8\x8a\xa9=3`\xd3\x81\xc0\xee\\\'1/&4\xdb\xecg<\xd8\x87G\xff\x03G\xa3\xd6\x83qKa\xef\x05\xbc\x94\xeb\xca~\x0cc\x069gL\xdd?\xe3\xf7\\\x11\xda_b\xc4\x9a=c\xd6%\\j\x1e9SKM\x07\xfbLw|Z\xf3\x1c\x98;>\xe5,\xe8c\xfd\xf4\x1f\xe9\xd9\xbf\xec\x03{:b\xad\x9c\xb3\x83\xac-\x02G\xd0\x8eG\xe4\x1b\xc8\x91\xd4\xf4 \x16\x88\x9e\xd6\x9c=\xb1{\xcd\x829\x8dg\xa3\xc1\xde\xc5YN6\x9e\x85\xb1\xc0c<#?^\xf1|\xcb\x0b\xcf\x1a\xa0\xc6\x86u\xd6~\xf4\x88O\xcd5\xcf`\x0b\xc5\xbc\xef\\M\xb8\xdf<+G\xd4\xc2\xdd\x9c\xe7\r\x08^n#\xaf\xf2\xfa\xd41=\xd6OcGI\x05\x16\xa6V|t\xbb\xafQ\xa1\x08\xae+t\xc7\xc8I\n\xcf) \x1e\x17=\xba\xfb\x1fE\x9cY\xa5\xb0\x86\xc03\xcb\x98\x8f\xa8\xa7d^!\xbfM\xa8y\xa9yv\x1b\xb5~"\x7f>\xd8\x07\xdf\'`\x1c\xd3\xefg\x00\xe2\xb9\xd8\xe7\xe6Ya\xe4P\xec_\xc1S\x85^\x96\x1a1\x0f\xb6\x16?\xe2\x0f_\xcc1s\x8e\x82\xf8$\xe0\x1aV\xe2\xec\xa8\x95\xe8\xed\xe6\xd4\xe63\xff1.P{\x0f\xde\xe6\xb1\xde\xc2\xb31y\xae\x1fr\x01\xcf}`\x8f\x06\xd8E\x9cU\xc6\xfa\x87\xcdYrb\xf8\xfbs\xb0\x7f\xfa\xc8oS\xce\xecT\xc2\x0fa\x1f\x9c\xbf\x1c\xd0>\x10\xddX\x9f%\x0f\x98\xd7"\x87\x10O\x88su\x7f\xc0\x1f<\xc3\x92=5]\xe8Xh\x1b\xd4\x1f\xd6%\xcfUP\x85\xf6J\xd4Py\xbe/l\x9a:\xfc\\\xa8\xf6\xe9#\xe4\r\x17\x9e\xc9w\xef-c\r\x81\xef\xc4y\x11\xa2~\xca\xb9I\xff^?\x153\xd6\x0f\xf5u\xd6\xc1\xc1\xc1\x85&\xa2bo\x9d\xe75\x07\xd4\xd3\x8b\xba\x878C\x1c\xcf\xca\xb318\x1f\x13\xe4w\x9d\xe6w\xfc\x81\x98y?\xfb\x91\x9a\x15\xf7^#\x13\xbd6\xe2g\x9e\x87\xd9\xe5\xcc\x86$bA\xc4\x19\x05\xdaaA\x1b\xba\xf1\x1cQ\xf6\xf0Y\xf7\xe1Y\xb9\xf7\x99\x03w%\xce\xdb\xf6\xd8\xd3\xf0\x85\xee\x86\xe7\xfb\n-\xddW\x7f\x8e\xbd\xb1J\xe8\x04\x95\x91\xccg`_\x84sL\xc0\xa8\xec3\xaa\xc83\xc5\xfd\xfc\xc2D`*\x9e)\xfae\x1fc\xe2n\xce\x1eR\xb7\x8e\xb5\r\xc5\xac,\xf3\x18\xfc\x0e\xf7ML\xc7\xb3\x9e8OD=\x0c\xfb\xa1\xe3\xfb\xb9\xdf\x11\xebN\xec\xf1s\xce\x88q\x87\xe7prv\xd5\xb8\xf7_\xb8\xe6\xf6\xbd\xff\x12\xde\xfb\x97\xdf\xed\xe3\x1f\xe9\xe6\x1f\xfa/\xc4\xac\xb4\xbb\x94\xe7HS\x1f\xc0sd8\xdb]S\xb76`o\xaf\x12y,"v\xe7\xba\xb0\xbf\x89\xf8\xcf\xaawMN$z\xfc\xd2\xfdL\xa7t\xcd\xf3\x0f\x05\xfe\xa8\xd8\xfb\x10\xfc\x969\xf3\xfaX_\xa7\xae\x95=\xffT\x9c\xf1Q\xf0|k\xf0\x0c\x9e\x9fA>]\x00\xdfQC\xc4Y\x12\xf6U\xa8\xc1\xe7\xd9Y\x8f\xfd}j\x12\xa8Ef\x8dj \xf4\xc6\xecQ!\x97`\xbf\xa8u\xa1\xf6\xd0A~\xe0l\xa1\xcb\xfa\xa8"\xce\xc6\xa3\x1e\x81\xa7\xacE\xd4\xc5:\x82\xc8;\xd4\x93]\xc4y"w\x9d-kE\xfa\x03\xfe\xf8g:\xf0\x1f\xf8-\xf8\x878/r\xce\x19\x88\x82\xe7FQ\x9f\x06\xaeO}\xaf\xce\x9e"u{\x9cy\x17=\xcf\x9a\xbdk\x9e\xa5\xc83\xce\x11w*\xe6\x0f\xde{(\xce\xa1L\x85\xef\xaf\x1838c{\xa3\x868\x11\xf3\x02\xec\x07\xfd\xffD\x9fM\xfds\x9c\xd2\xb7\xeaO\r\xff\xe7<\x07\xd6T\xd4\x95\xb9\x96\xe3\xe2~\xbf\xa2\x9fJ\xbd"\xe7\xee%\xea\xb1\xc49M10\'\xf1\x1b\xcf\t\xe4\x8c\x90\xa8yQk\xcfY\xba\xe2"\xce\xac\xf9!\xfe\xb8\xab\xd4#_\xa5^\x0c>\xcc\xb3\x13c\xce\xc2\x93\xd73\x96\xf2\xdcO\x9ew\x0fnM;\xe3\xf9\xdb\xe2,b\xe2\x05\xea\xc5\x12\x9e\xab\xcf9\xef\xe2~f65\xdb\xac\xbf\xf2\x9e\xd8\xb7\xe1\xb91\xc4\x9f\xfc\x9dn\xfe\xc33\x8b\xf9\x9f/\xfd"\xeb\xe5!\xcf\x89\x88\x16\xf0gbl\x9e\xa5\xc13\x83q\x8d\xda\x10}\x91\xfb\xf9\x1c\xecYr~i\xfe\xa3\xdeU\x9c\xb3\xc69\x14\xf8\x90\x98G\xe13\xd3\x06\x81\x8f\xe8\x9b\xe2l\xb0@\xf4C8C\x12\xf2\xecWj\xaf\xa9\xcb\xe6\xb9&\xa2\xf7G<\x04\xdc\xc8s]r\xd6s\x88\x1f\xbf\xeb\xd0\xf99\x07\x17\xa1\xd1\xa6>\xf4K\xbf\xa8\tn#j:<\x0fD|v\xc1M\x9ci\xc5\x996\x1bq\x90\xf3\x8e\xd4\x12\xd8<\x7f\xc4\xe1\xf9\xa0\x8f\xf9\x16\xfb\xc8\xb9{\xce}\xd1\xb79+\xc6^O\xca9\x1f\x9e#\xc1z\x81\xce9f\xce\x90\x06\x9c/\x14\xda\x98\xb9&\xce\xcaB\x1c\'\'\xc7z\xa8\xa1\x98\xd7\x1f\xd4w\xcd\xe2\xeaA\x87.x\x06\xfb\x01\xe5\xfds8\xbe\xe9]\xff\xd9\xb9\xe0\x0f\xf1\xe9J\x0c\x10\xd4\xa5\xe0\xc5b\r9w\xe6\xf1\x1cf\xd6\xb7\xd8[`=r\xa0P\xb3/\xb4S\xb6\xc04k\xf2?\xe2>\xce\xf4\xe09\xa9i/\x84\x0e\xa5\xe29G\x0f:t\xce\x1bT\xd4\xa39?\xe8\xe9\xff\xd9\xb9\xdd\x8f\xf5N\x9e\xc9\xe2\x13+\xf1\xcclI\xcc\x18\xda\xc3\x92\xf3_\xf4\xfbP\xf4\xfcxn\x8f8c\\\xe5<\xb3M\xcc5=\xd8\xcb?\xfc\x1c\x87\xef|Ih\xd3x\x8e\x9f.>\x17\x85g\x15\x0b\x1d\xeeX`\xdc{OM\xd8-\xcf\xe2\xaf9\xe3\xdb\x17\x9f\xa3\xc0u\xc3u\x85n!P\xd9\x9b\x13=,\x81\x97\x85\xde\xf5K\x87N\x9d]\xc5:L\xf1C|\xf9g\xe7R?\xd6/\xa8E \xef\xe4g\x1a\xd0\xc7\xb8f\xd4\x88q\xbe\x9232\x06gRY\x7f\xd0\xc4g+0OP\xe1b\x0f\xc4\xacL\x10\x113u\xf9\xf9"\x83\xbb\xa8\x02r\x1f\x88\x93\xa1V\rq\x03\xf8\x08\xc2\x1fKz`\xbe\xc2\xdfA\x9f\x9cC\xb0\x10,\xdc#[\xe4\xae\t4\x1e\x00_\x0bs\x08\xbc\x19P\x9f\x84\xb8\x05\xf8 \x81\x97\x0e\xb0`\x80G\x02\x9e~R\xffcpL\x8eH\xef$\xf0\xd5\xcc\xe0\xacYS\xa4\x86\xfb;>\xfd\xa1>P\x908\xb6\xf6\x08?\x16\xdc\xdb`\xdfe\xc2g\x02\x7f\x19\xf1o\x80\xbb\xa3,\x82u\xca\xb3!\xc9+\x1f\xc7\x1b\xf0\xb80\x1fP\xc3\xe8\xe2\xb9\xb9\xeb\xdf@\xbe\x8e}\x16\xf0%\xe18\x0c\x8f\xf1=o\xc7\x9f\x9d\x17w\\\x0c>\x13\xa1\'\x15\xf2Z\xf8\x1c\xb8G\xbf\xf3\x83\x7f\xc6G\x00\x863\xda\x10\xec\xc5\x1d\x9f\xfe\x10\xdf\xff9\x9e\xe8\x87\xfa3p\xbaA\xbc\x08\x9f\x89s\x0eR\x13\xc0\xbe\x0e\xfb*\x07z\x15&\xf9\xbd\x0f\x01\xfb%\xe8\x1d\x00\xcc\x02\xf0:\xa6\xf8\xac\x84\xb5\x02\xd8\xd0\x1ab\n\xd0YR\x80\x97\x0bAl\x0e\xb1\xf6\x1dKB\xf8\xd2\xee\xf8\xf4?\xd6\x07`\xf67C\xc2?\t\x18R\xe0\xf3\x02]\x15\xe8e\x80\xf3\x0b\xd6\xba\x0e\xbd3\x84\xa7\xcc%y\xb3\xfbx\xff\r5\x11\x06b\x19\xe8\x17rI\x0e\x15\x02\xbe%\x03- \xe0A\xb0\x08\x9e.\xa4\x086\x0c\x01g\xc6\x9c\xf4\xba@N\x08\x9cq\x04o\x86w\x11\xe0\xd6\\\xe0\xdf\x05]\x0e\xc2\t\tz\x04\x10W\x7f\xc1\xa7?\xe0#\x80C\n\xfao\x81C\x01\xb4\xa6\x10p\xd6\xc9\x15\x9c]P\xd3\x00\xeev\xa85\xe2\x18\x83#\xbaRd}=\xe2\xab\xa0\xceL0B\xa4\x07\xdd\x05\x1f\x02w\xdf\x90\x8b\xa5\x80\xd3\x80\xfb-\xe8U\x19_\t\xce\x1b\xe6\x98\xf4pA\x8f\xa3\r\\\xbe\xa0\xd7t\x85^m\xc2;\x00k\x97\xf8{\xc0&\xba\xcc\xbd\xef\x8f\xe0\xd3\x1f\xea\x03\xd0\xcb\x03\xd8z\xd03\x81{\x1a|f\xa6p\xee\x87\xf0\xde\x15\xf0\x8e\x00\x97\xe0\x10\xd6\xad\t\xfd\x98P\x13}\xbc\x9f\x8866\xb9\xaf\x1a\x13|.\xe9q\xc6k\xce&\xfd\xdf9\xc1\xa5\xd8\x04[\x01\xb8u\x9d&|%p\x7f\x9d\xc2\xfd2\xd4R\x80\x07\x07\xe7J\x0e\xac\x19\x8b\xbe\xe7\x9f\x19\xe9\x0f!\xfab06_\xf0\xe9\x7f\xe4\x7f\x19\xc9\xe9\xc0\xc7\x01>\x91\xf0M@-\x13\xc7\xc4\x90\x7f\x02n\x0ez\xe5\x01\xafbz\xc0k\x94\x91\xdc\xe2\x11\x1f\x01sHr\x1d\x1cC\xcf \xbf\'5\x00\xc2qF8j\xe1\xee\x0e\xee\xaf\xb0\xfd\x1e\xdc\xd9\x01\xdf{\n\xfd\xd2P7\x01\xac\x18\xf0\x95\x90\x1e\x1f\xc0\xab\x00\xbf\xdb\x95\xe0\xbb\xa1nA\xea\x03_\xf0\xe9\x0f\xeb\x03\xeaS\xa4\xd6\xef\x12\x8e\x13\x88\xc7\xc0\xb7\x01\x96\x14z9\x01\xaf\xebA\xef\x19\xe8\xbd\x11l\x9c\xc5~u\xff\x8ds\x14\x82\x8dI\xa1W\x0f\xfc \xdc\x99C\x1f|FC]\x13\xc6\xef~\xcf\x0c\xbd\xa6\x13\xf0\x19\xb5G8O@\xdb\rp+\xf0,|\x16\x16p\x8f@\xea\x17,\xc9\x1f\x00\xf3\x0cyH\xaa|\xc1\xa7\x7f\x85\xdf\xbd\xf3:\x81\xbf\xc2gm\x8d\xfd\x1f\xdc\xc1@\xfd\x040~\x05Y\x7fp\xf7r\x83\xb3\x14j\x11\xdeW\xf7\x13x\xbc\x01[_\xe0w\x83\xb5J\xf4\xb9&\x84;\xf2\xae\xa9\x87\xff?\xbd\xe7Ep\x97Gj\\\xe4\xbc\x82;+\xc0\xb4\x00\xb7\x14p\xecA\x7f\r\xf4,B\xbc\x0by5\xe0* \x8f\x82\x1a\xdeg|\xfaC\xff\x10`\xf6\x80\x8b\x97h\xbf\x01\xbf\x08\xe1\xa3[\x03\xd6\xb1\x86\xda\x0c\x8e\xcf\x12\x82\x93&\xdc>\xc0\xa7\xa3?\xde\x7f\xb3w\xbf\x02\xdc\x18\xd0\x8b\x05\xf7\x8d.\xec\xf9\x1a\xce\t\xa8\xfb\x03.\xd0\x85;\xcd\x14\xf0`w\x0e1\x97\xdc\r\xe3\xe7\xe2\x9c\nx^l\xc2Km\x90\xfe\x03\xd2\xf7\x87\xd7\x07\xd4_l\xa8\x07\xff\x8eO\x7f\xb8\xff&\xda\x0bI\x01w\x10\xc09A\xb8\x02\xe0n\x04\x11\xbeX\xe0}\xaf\x81\xbf\xce%5ZX;Q\xf25~\x06\xfb\x06\x04s+\x03\xbe\x1a\xf2%\xe8e/H\x9f\n\xac;\x82A\x81\x9a!\xde\xb3\xf7}B\xc1|\x80\xfe\xc5\x90\xf0\xbcz\xa4\xc6c\xc3\xbd:\xf4\xa7@=\x14\xfc%\x8e\xbb\xee\x98x\xf9\x0b>\xfd\xe1\xfc\x80\xfa\x07\xf4>L\n\xa2\xf9e\x12\xccsN\xb87\x81\xefa\x06X\x1c\x9c{\x02/\x1e\xe0\xb3\x81\'\xeb+|\x04\xe1\xf2\xc1\xef\x02\xda\x08\x19\xac7\xbc\xff\xa1\x9e\xa3\x93\xfe\x1b\x82\xfb\x84Z2\xe1\xbf#\x9c/9\xe1p\x80\x9e/G\x07<*\xbas_@\r\x1a\xfa\xf6I\xdf\x06\xc1W\xc1\xb9Mpz_\xf0\xe9\x8f\xf8;X\xbf\x80\x8d\xd2\x80\xb7\xc5%}\xe2dM\xe2\xd8\x05\xfaB \x8e\xc0\xef\x94\x93x\x19\xfbm\x17\xf8\x1c\x1f\xfb\x87`\x1d\x16\x90\xc7uo\xd0\xbb\x02w_6\xf4\xeb\xc2\x1a\x00l\x16h\x8a\x80O\xd2\x88f\x0f\r\xf8\x11\xe8\x95$\x18*\xe8O\x83;u\xa2i\'\xb3\xf7\xd8m\xce\xc2\xf7\xa0\x96j\x93\xb8\xf2\x0b>\xfd!\xfe\xc09,\xe1s\x84\xae;\xc7 }\xe5\xe0\x97\tg\n\xf4\x88\x11\xce\x7f\xc8\xc7\x17\x19\xe05p\x0c\x93|\x85\xbf\x83\xf3\x04A\x0f\x17pe\x80\xde\x1c\xdc\xd9Y\x9c\x97\xe2\xf9\x87q \xf7W\xc0\xfd\x0f\xbc\r\x90\xb3\x91Z)\x03\xb1\xbbw\xe70\x02\x8e\x0c\x98#\xb8\x85d\x89\x96FM\xb4\x9f\x00\xf3\x83H\\v\xc7\xa7?\xe2#\xfe\x14\x7f\xfbC?L\xc0gA\xbf\'\xf4\x80\xc1\xfd\x1d>\xdf\x0b\xc0:)\x19\xb9\x03\x87~\x01d\x00V\x18\xfb\xc0\x8c!zj\x80\x91\xa8\t\x8f+\xe0\x84\xa0G\r\xee\xbfA\xe7\xe0\x8e\xef\xfe\x82O\x7f\xa8/\x12|\x05\xf4\x8f\x82\x0f\x80\xfb\x0c\xd8\'\x08j\x1c\xa0\x9f\x88\x9fo\x12-(\xc0L\xc1\xcd9\xf4\xd6$_\xd5\x17\xb1\xbf\xf0L\xd0\xa6\x03\xae\xdc\x1e\xc4@\x88\xd4=\xc0\xc7\x12\x7f\x0b\x98>8\x0b\x01\x93\x06<\x1fp\xd7\x035>/\xbb\xf3B\x02\x86\x97\xc4\x824\xdc\xe2\x12~J\xa2Q\x86\xe3K\x98\xe3\xdf\xf1\xe9\x8f\xe7\x07\xe1\xcd\xca\xa0\xa6\x01}>P\x97\xb4A\xf3#\x05\x8c \xf8\x14\x9d`\xedm\xd2\xdb\x93\xc0*\xaa\xbf\xba\x9f \x98]\x1c+\x80\xce\x14\xd4GR\xb2\x17\xf0;\x03&|\x01\xf9\x10`\x1d\x01\xad\x01\xd8\x0e\xe0S`I\xac\x01\xf7\x13\xe4N\xdf%<\x80xM\xe2\xb5\x90\xe7\xc0\xc5\x06\xf9\xb8Kr\x07\x9b`\x83?\xe3\xd3\x1f\xfbS\x01{\x02\xb9,\xde\x83\xf0\xfb\x13\xa8\x0f]I\x1c\x02cQ\x90\xbe7f\x08\xf9"\xe1\xc8\x83\\d\xfc\xd8\x1fB\xdb\xa4\xe69\'\xfc\xe3\x04\x87Y\x80\x0e\'\xe4`\xa4o\x00\xde\x9f"X\x15\r\xf2q\xa8\xe5\xc8\x14\xf4GA\xdd\x89\xf4@\xe1g\x01v\x18\xf2_roO\xe2\x0f\xe0\xe2\x80^W\xd87\x9f\xf1\xe9\x8f\xfde\xc0\xa1K\xb4N2\xc8\x97\x18\x88\xc1A\xa7\x01rT\x82\xd9\x86\xbeG\xe8\x19 \x98\x01\xe8\x15R\xd2G\xfc\x1d\xf4_\xb8\x10\xb3\x00\xbe\xb1 \x9c\xca5\xe1`\x02\xbcTJ4\x0e\x19\xd2\xdfC\xee`\xe78G#\xf7!\x90\xad\x13-6\x8f\xf0\xa5\xc2\xde\x8d \x0e\x8786\xbb\xc7\xa7\x93\x8c\xe0\xbb\x7f\xc7\xa7?\xdeO\xe87\x9c\xd3\x91\xb8\x95p\x98;p?\x0e:H\xa0\r\x048:\xb8\xb7\xc4~\x95\xf4\xf3A\xdd\x08\xb8\x04\x1e\xf13\x80\r\x86\xda\x1e`\xeb\x80\xbf\x0e8N\xa0\xbe\x018e\xb8\xa7\x01]=\x1c\xe3A\x8f7\xdc\x85\x90\xfey\x8bpJ\x83\xc6\x0e\xe9\xf1\xab\xf1{k8\xdeC\xf7\x1e\xcb\xfb\xfd\x15\xdc\xd3A.\xa0|\xc1\xa7?\xc4\x1f\x16\xe3\xdd{q\x80\xff\x0bbE8{\xaf\xe4^\x07jt\x05\xc9\xddR\xa2\xb3\x08\xf7j\x10{\x7f\x15\x9fB/%\xc4NP\xbb\x04\xbd\x1f\xc0\n\x83\xd6+\xe0\xdf\xa3\xbb\x16+\xf8\xddZ&}\xe7p\xcf\x05|\x88\xa15\xf0\x19\x809\x07n\x1b\xec\x0bH\xaf$`\xe6\xe6\xc0\xdd]\x11<,^[\xc0\xed\x01\xd8n\x1b\xee\xf7H\xdd\x06\xf8\xe2\xe7\xf7\xfee\xe01!\xf8\xee/\xf8\xf4\xc7\xf3\xe3\xcf\xf0\xc0?\xde\x7f\xdb\xd0\xdf\x0b:\xc9\x15\x89\x81g\x84\xdf?#}-)\xf07\xaf\x01O\x94Bm\xc2"\xb5\x13X\'p\xbf\x0cw7\x90_\x01\xbf\x07\xc4\r\x907Cl\x8c\xc7\x1e\xce\x8f\x02j~w\xfc\xffg|\xfaW\xfd\xcb\x80k\xc1\xb3q\xe7o\x85\xf8\x98\xf4\xc1\xdf\xb1\x8d\xc0\xaf\n}\'p\x16\x91\x1e\xa8\x19\xd4\xf6\x1e\xf82@\x8b\x80hbC\xcf\xacM8\xbc\\\xc0\xbc\xcd\xa0\x16\x1b\xde>\xafG\x86`\xb8\x81\xc7\x01\xfc6p\xce\x9ad\xae\n\x92\xef\x82=&\xe1\xe2\xc2\xe7\x17\xf4I@\xac\x05\xb9\x12\xd1\xd8\xbe\xfe\x8eO\x7f\xac\x7f\xcc\xc6\xf4\x9do\x14\xea\xb6\x80\xd3&\xda\xd0\x90\xf3\x02\xaf?\x1e\x87Iv\xef\x8d\x03\xcc\x13\xdc\x82\x00\xff\xe4\xc3\xfa\x00\x0c\x14\xe1\xe4\xc5\xb13\xe1\xdc\x06\r\x941u\xef\xdb\x1esw|eF4\xb1!\xee\xb3H?0\xac=\x9c\xb7\x16wl\x07\xd4\x89\xef\xf7\xf6v\x8e}"\xdc\x7f\xe3\xd8\x00\xe6\x14\xc6\xf6\x0b>\xfd\xf1\xfc\x882\xb2o\x00G\x04\xf6\x02\xae\x1e\xee\x19\xa0\xb7\x9e\x9c\xc3\xd0\x8f\x08\xbd\x110&\x16\xf0B\x02_\xcd\x1f\xebcF\xe2C\x82\x0b\x85s\x1e\xb0\xf5\x04SK\xfa\x15\xe0\x0eH\xc7q\x00\xe9{`\xc9\x9d\t\xdc\x93\xc3}\x84\t\xf5k\xc0\xc2\xe6\xd0\xbbEA\xcf\xeap\x06\xfb\x15\xb0\x19\x10\x9fB\x0cz\xc7w\x7f\xc6\xa7?\xe2#\x00_\x82\xee\x18\xa7\t\xd4\x15\x81\xc3e\x03\x98\x04\x825$\x9c_\xe0\xbf\xb1\x9f\'\xfd\x14p\x7f\xa6?\xae\x0f\xe0\xba\xd8\x90Z$\xdc\tAM\x9c\xf4/\x82\x7f\x81\xfbE\xc0_\x11\x8c\xee\xf5s\xbf\x1a\x03Z\x03D3\rbI\xd0G\x9c\x11n\x18\n\xee/\xa1\x16\xeb}\xe6\xdf\x81\xbe\xa2\xfb\xfa\xf8\x8cO\x7f\xc0w\xff)\x9e\xf7\x07\xffr\xd7\x93\x80:\x0cp\xe6\xc2\xd9\x08\xbd\x82\xc0yEp\xa8\x15\xe1 "\xfc,\xa4V\x08X\x04\xe0G+\x00/\x0c\xb5\x7f\xb8\xeb\xc5qQN\xb8C\xc8\xfd3`)\xa06\x00X\'\x18\xdb/\xf8\xf4G\xfe\x03\xb8\xff\x9e_\x896.\xc4\xfc)\xdcWA\x0c\t\x9a)\xa4\xef\x12\xb0M\x14\xe9\xe1\x01~x\x1c\xb3>\xf6\x97\x01\x1b-\xf4\xf4C\x1e\x0b\xda\xe3\x16\xb0\xe9C\x9d\x1f\xee\x19\xeey,\xd1U$H\x1f\xe8\'\xd3 \x87\xc3\xb1\t\xe1*\x99C\xff\x10\x07\xf8a\xb8\x83\x80\x1c\x80\xf4\x8a\x91z5pR\xc0\xfa\x90\xbf\xe0\xd3\x1f\xfbS)\x8f\xf4#\x016Q\'\xba\x116\xec]\x12\xc3\x81\x0e\x17\xe0V\xa0_\x078S\x81c\x11\xc6\xfa\xab\xfb$\xe8\xbd\x83\xfb\x05\xa2\x99K\xf4?gp\xa7\xf3\x99\xfb\x07\xf0\x8a\xc0!\x0c\xe3Wg\x88\xf0\x96i\x80=\xc2_\x83\x1b$\xa8\xb1@\xbd\x1a\xe2p\xe0\x0f!\xe3\x0e\xfeeN\xfa\x9f\xef\xfd!\x9f\xf1\xe9\x0f\xe7\x07\xc4y\x04\x07A\xfafA_\x0fj\xb7P\xfb\x85\xfa\x11\xb9\xdb\x06\x9cGr\xe7S\x83^\xa1\xdeW\xf8n\xd8\x97P\xe7\xc2\xcf\x04nP\x96\xf0\xdf@\xbc\x0f|\x91\x80\xe5\x04\xec\x90\x06}\x92k\x86\xf0oC\xafsJz^\xef\x1a\xf6\x1a\xe0\xfe\xa1\x16\x82\xed$q=\xc4T\xe4.\xa0\xfa\x8c\xef\xfe\x8cO\x7f\xc8_\xee\x9a\tpO\x93\xd9\xe4.\xb3K\x01V\xc4&\xef\x05X\x1e\xb8\x07\xed\xde\xb55\xa0fn\x02\x8e\xee\xd1\xbf\x80n3\xf8\x8a{\xef+\xd4\xeb\xb1\x9f!\x98$\xd2k\r=\xcb\xa0\x91\x0e\xfd\x95\x10\xab\xcc\xa0\xe7\x02\xd6\x8bw\xc7\xd7\x01^\xb5\x86~4Ro\x84{z\x8e\xf4\x97A\xff\xceg\xfc\xffg|\xfa#~\xf7O\xf1\xb8?\xf0\xa7 \xd2sI\xe2%\x1d\xc7*\xa0e\x0f\x9fO\xf0d\x0c\xa9\x91B\xff\x16\xc1\xc3B\xc6\x06\x18\x809\xf0U\x02_qFj\x04\x80\xc7\x02\xbc\x1a\xe11\x90\xa1N\xf3\x19\x7f\xd7\xad\xbf\xe0\xef\x08>\xfd!\xfe\x80q\x04N\x04\xb8\x83\x04N\x0er\xd7\xe7\x10}\x02\x8e\xe4\xaa\x1a`\xf4\x88f1\x0b\xdc\xe28\n\xf9\xaa>\x06cb\x93\xfb\xea\x0c\xb0\xe9)\xc4W6\xc1\xfd\xe8\xc0\x9f\x03\xd8\'\xec\x9f\xa1\x9f"\xac\xa06\xeci8\xa6\x07\xbe\x04\x88\xd7I\xae\x01\x1c\xcb8\xee\x85\xfc\x0fYP\x8bb\x08_\x10\xf4\xd3\xd5\xd0[\xf8\x05\x9f\xfe\xd8\x7f8&\xfd\x0f\x16\xf4\xf4@\xbf\x0b\xc1 \x00\xa7\xc6\x04\xf0>\xa0cX\x03\x86\xc9\x86X\x0b\xb8\xdf\xe0\xfe\xedq}\x98\xe4\x0e\x01\xe6"\x05}^\xf0\x87\x1e\x89\x05\x007Hz\xd0\xa1w>%\xf7pp\'\ruPD\xb8\x92\xa0\xbf\x07\xee1\t\xde\xc3\x03\xed!\xd0\xb3\x06\xcd\xe1{\xff\x10\xe8\x15\xdfy\xd2\xee\xf8\xf4\xaf\xd6\x87u\xaf/$\x10\x03c\xbf\xc8Y\xe4\xec\x07L"\xde\x7f\x84[\x15z\xfe\x81W\x05\xf0{\xe0\x8b\x1e\xf1/0\x06P\x8b\x8f\nR\x83\xc0\xf1\xa2E8\xaea\xff\x00\xdf<>\xff\x81\x1b\x81\xf0;\xc3=\x00\xf8\x12\x99\xf6 \x0f\x05\x1dx\xf0}\x80\x9f\xc7\xe7\x07\xe9\xcfA\x0br>\x91\xbe\t\xe0\x8c\x81\xfb\x97/\xf8\xf4?\xea\x1f9\xf0\xff\xe1=\x0f\x1aE,\xe4\x9d\xd0\xe3\x02\xf9\x1b\x9c5\xd0\xab\t}p\xa0\xeb\xe4\x11^\x13\xa8\x7f\x00\xff\xcd\x1f\xf1\xc7\x90hC/6w\x0eG\xa8\x8fCE\x1b0X\xc0k\x00\xf3\x07\xf9!h\x0c\xc0\xfd3\xc4\x80\x90\x9f\x03\x86\x1a\xee\x1f\xbbD;\x89`\xed\x80\xbf\x06M\xeek\x81\xe0E`}\x10\xac\xc2g|\xfac\xff2`\xc2 \xa7\x892\xc29\x05\xf7\xe9\x1a\xd1sI!\x1e :\x11\xa0/\x96\x12^\xe0\x1b\xf8@\xef+\xffb$\xe4\xfe\x1b8\x19\xef\x9an4\xa9\xb7@\x9f\x05\xe1\x91\x00|\t\xe4/\x80G\xc3y\xdc\x0cp;\xd0\xcb\x0e\xec\x8f\xf1}}h\xeb;\xbe\xfb\x0b>\xfd\xab\xf8\x03\xfb\xca\x19\xe1\x19\xc1\xeb\xaf\x97\x80N\xa2E\xf4\xd4\xa0\x8f\x07bw\x18\'\xc2\xbd\x03\x98\xf4\x94\xf0\xcc<\xd4OA\x1b\xc2&\x98.8\xb7l\xe0\\A\xc0C\x03Z;\xf89\xf8\xcc\x80\x9c\x134\xb3A_\t\xf0\xe8\x80\x01\xc6\xe7\x0b\xd4\xe5\x0b\xa8\x9d\x01v\x03tE\xa0\xb7\x1d\xee\x12\xee\xf8n\xe0\x1f\xb1\x80\xdf\xfew|\xfac\xfdt\x0e\xda\xb2\xa0\x85\rsD\x01\xaf\xd6\x10\xfa\x07LR\xb7\xbf\xef{\xb8\x07\xbc\xf3\xb0\x02O[\xfaX?\x05\xac\xe3]+\x1a\xe2\xd9\x0c\xe7\xc0\xa0\xbd\x0es;\x87zD\xe5\x12\xfeqr\xa7\xc5\xc0\x9d\xd8\xbdf\x01>\r\xb8\'\x81\x07$/<\xa2{\x06g\x01\xf6\x11p>B\xfe\xe2\x90\x1eW\xc2\x7f\xf0\x19\x9f\xfe\x18\x9f\xfe)\xfe\xf3G\xfc\x1c\x8e\x86 \x16\x86\xbef\x9a\xf4s"|\xc6\xa7\x19\xe10\xc4?\x87}\x12\xde/\xc0Q\x02g(\xdc\xd1\x91?pF\x03\xee\x0cj\x08\n\xd1\xeb\x00\x0e\x18\xb0\x1b\x9f\xd7\xa4\xbf\x1d\xfa+\xee\\b\x9f\xf1\xe9\x8f\xf8\x7f\xf0[\x90\xe7\x10\x0c5\xdcw\xc1=7\xf0\xd7@\x0e\x05\xbc\x8e\xb0S\x81\x8f\x17\xb8h\x80\xff2\x7f\x8c?n\x84\x07Y\x9b\xdf5\x88!\xd7\x86\xde#\xc8\xbd\x80#\x18\xf8uI-\x130\x0b`+>\x9f\x08g\'\xc41p\xd7\n\xbe\x00\xaf\x1f\x07\xeeh\xe0\x1e\x01\xee\x16\x00\x9f\xb4\xe6H\xef\x07\x9cu\xbf\xe3\xd3\x1f\xf2[\x88[f\xbd\xfb>$}\xa4\xc0\r\x07\xe7\x02\xe1\xcc\xce\t\x17+\x02\x1f\x02~\x11\xf4\x8bC\xf4\x95\x7f!\xfdJ\x10\x0b\x02\x8e\x05\xce8\x97\xe8\xd4{\xa4\xbe`\x10\xfe{\x82)\x01\xcdW\xa8Y\x13M3\xa8\x01\x13^|\xc8\x91\x80o\x19\xde\xad"<\xcfp?\x91\x92\xfai\xee\xde\xc7\xe6\x0b>\xfda}@\x8f=\xdc\x83\x03&\x82\xf4\xf7\xb3\x84\xcb\x14j\xa0\x88\xf4\xc9q\x84\xab\x08-\n\xc2yDp\xf8\x8f\xe7\x07\xcea!\xde+\x08f\xa5\xba\xd7\xc8\xc8]\x1b\xc4\xcf\x84\xd3\x1e\xf8\xcalr\x16\x10\x8d?X\x87\xc0\xcf\x8e\x9f\x1b\x02G\x02C\xea>\xa0\r\x01\\n\x80\x93"\xfd\xedp\xc7BjZ\xd7\xdf\xf1\xe9\x0f\xf7sD\xff\n\xee\xa1I\xbf(\xd8@\xb4)7\x10\xa3\x12\x1cD\x1d\x11ma\x88\x87IL\x05\x1c*\x0f\xf5S\x88\xbbm\xa8\xcd\x80\xfeK\rZ6\xd0\xd3\n~,\x87\xbb\xc0\x8ap\xfa\x10\xfd\x85\x0c\xf00D/\x8a\xe4\xa2)\xe0\x93\xc7\x80\x91`\xe0|\x07\x9d)\x97\xe0r\xf5\xeb\xfd\xfe\x85\xf0\\\xdd\xef_\x08>\xfd\xb1\xfe\x01\x1c\x92sgW\xa2\x0b\x02u\x11\xa8\x8f\x11~1\xf0\x0b\xca\x17|\xfa\xd7\xeb\xe3O\xf0\xac?\xf6\xa7\x02W\n>\x9fH\x8d\n\xfc\xb6\r}t\xe0K@\x1b\x83\xc4}zE\xb8S\xa0\'\x18\xea\xa3\xc0\xa9\xe2\x00\x1e\x81\xe8B\x00.\x16\xfbm\xb0S\xa6H\xbf \xb9\x7f\xc9\xe0\xfe\x93`\xf9~\xc7\xa7\x7f\xd5\xdf\x0e\xb5\'\x1c\xeb\x01\xbe\x8b\xe4l\x80y\x81\xd8\x85`\xa6\xd2{N\x00\xf7Rc\xf2{6\xd8\xfb\x90\xdf\xda\x84O\n8h!\xef\x80\xb1L\n\xa2I\x9a\x12\xde\xd3\x1a\xf4\x9e<\xe02\x83\xded\xc2\xd9\x0c\xb8b|\xe6\x15\xc0U\x08\xe7\x14\xd4\x99\x81\xabx\r\xbd5w\xbfC\xf8\x9f\x01\xa3\x9e\x11\x9c\xed\x1d\x9f\xfe\x10\x7f\x14$\'\x85\xfb4\xa8\xe1C\x9f\xe6\xd5\x02\x8c\x08\xe0\xc6\x80\xe7\x14\xf4\x88f\x80\x87\x80\xbdIx\xf7A\x97\xe01\xbf\xdd\x10\xadz\x82%\xee\x15\x84\x07\x9d\xe8~\x83\xd6\x18`3\xf03\x01?Gx\x93\xe1\xce\x13\xb8\xa9\xe0~\x1bx\xdb\x81\x0f\x04\xea\xea\x90\xb7\x01\x8e\x19\xff\x1c\xe0\xd6\xe1\xfc(\x08~\x14p\xce_\xf0\xe9\x95\x95\xfeo\xc1g\x03\xfe\x99\xf4v\x02\xce\xe0\x9f\x9f\xfe\xfbo\xff\xf5)\x8bo\x9f\xfe\xf1\xc9E\xd29D\x8bs\xc8,\xea%Jr\xdf\x89\xb6\x91\xfc\xcf\x7f~\xfa\xdb\xa7\x8b\x9f\x9fc\xfc#\xd6\x86"\x15 @\xb8|\xf5\x9b\xd5\xces8j\x89\xf2\xec\xf1\xa7\xb5\xf0\x14\x99\x8b3^7\xb7\xc0\xc9\xcf\x91J}\xf7\xb7\x98\t\x17\x9a\xf3\x7f\xfe\xab\xdf\x84\xd7\xf8\xef\xff\x84\xdf=\xddv\xf0\x13E|<\xfa\xeb\x18\xff\x8e\x7f:\x1d6\xc1\xf9\x14\x1f?\xfd\xe3\xff|yv\xe0HY\xe4T\xf9\xc33}\x9c\x05,\x91}\xf4\x9d\xc5\xfb\x9f\x87\x97:\xe5;\xd2\xf9\xe1y0\\A\xb1H#3\xbf\x04\xb9t\t\xf1\xfbbSn\xf1\xf8\xdbg\x87\xdb\xb2\x8c\xc3\xd3f[\xfe\xb6\xdd\xc5\xe5o\xa7\xc3\xed\x87\x9f\x83$:(\'yX*IXN\xb8\xdf_\xd8\x90v\xde7s\xf1vt+\xfb\xfa\xcfof\xf4\xdb\xa7]\x82\x82\xcb]f\xb2\x0b\x10\xb7\xf2\x9d\xf1?\x7fd\x11\x9eiK\xfe\xf6i\x8f\x06\x7f\xfb>\xdf\xfc\xf6\x9f^\x1fO\x99\xe5\x073Y2\x1b\xff\xf97<\rQ|\xdc\xf9!\xfc\x18y9\xf8\xc2\xa7\x7fP\x7f\xfb\x14\xf9\'\x1f\x7fQ]\xbb}\xa5c\xe7AA\x86eS\xae\xb6\xf0\xa3\x7f\xfb\xb4\xf6\x8f\xbf]\xfd\xf2\x14G\xf8\xbf\x11\x05\xff\xfb\xfc\xd5\xf3\x91|\x8d\x13D\x9a\xc3_\x8a/qy\xba\xbf\xe1g\x0bN\xd5\xbfXR\xe9\x92\xb1\xf3p\xfc\xd50\x06\x8c}\nj\xbcAQRF\xf0\xa7\x93Px\xcbo\xfd\x14\x9a\xc1\x8dc\xa8E\xb4U\x1a\xd8U\xb0\xb7\x08\x87\xae\xe1\xcccB\xd3(|U\xac\xbe\x1e\xbc\x1f\x7ft\x88\xf22(\x0c*Z\xf6\xbe\x1a\xbb\xe9I\x9d\xd1\xca\xd6\xcdq\xfc\xe3D\x9cE\xf5\x8e\xce\xc2.\xc7\x0eW\x86\x86A\xdb\xce\xf1\x10\xeb\xbbC\x9c\xdb\xd4\xdc\x89\xa6\xferB{y\x8f\x0fM\x9b\xf6\x17\x8b"\xd0%\x1a{\x0f*^L\xce\x819\xbf\x86\xb3d<^\x1a\x9big\xc1O\x0c\xbb?\xa62j>\xef\xcd\xe7\xf4\xba\x9e\x14\xc9\xd46ti\xf4f\xb7\xbc\x7f\'\xc2\xa1\xe5\xbd\xd9\xd1\xa7\x83_\x1eW\xf1\xe1\xc7\x83p_y\xcf\x18\xfb\xaf7J\x88w\x81\xbfT\xc8\x92\x7f\xfbt8\x9e\x8f\xf1r\x02\xd4\x8e\xbc\x8f\xec[\x04\xb4\t\x1d\xe5\xe0\x15\xa7\xf3\xb0c\xf3\xe0:C\'\xa4Cd\xe0\xd0\x91\xcb\x87\xe3\xaf\xb7\xa1C_\xa2\xaf\x9f\x0c\r\x82\x00\xa0\x86?\x91#\xd5\x81#\xbeoP\x9f>\x16\xffo\x9b\x18\x07jg\xff\xfa\xcf\xefo\\\xfc\x85 ^o\xca\xdf\x82|\x1bf\xbf}g\xcb\xfd\xd9\x89\xd7\x99\xb8\xa3\x14Qa\x1c\x82Y/\t\x9c\x10G\xe8=.\xe8\x00>|M\xc5\xa9}\xf0\x9c\xea\x08\xa8\xf1\xc0\xb4\xf9\x90\xc1\x11\xe9{&~\x1f\xe1\xbc.\xac\xed2,\x8d\xc2.]\xe4\x17\xd7\xda\xeb\xc8T\x84s\x9f\xe1r\xcc\x85\xc5\x91\x1b\x92\xb8@\xe1\xfc\x8ew\xf4\xe4\x9fO<\xa0T\x13\x9a|\xc2\xf5\x9d\x03\xff\xf4qx\xfc\xec\xdda\xbb\xdb\x1e\xe3\xc3o\x87\xf8\xea\x1f\xa2\x1f\xbd\x03\xfe\x10\xec\xdf&IdJo\xde\x83\xf1\x92\xc0\x94\xae\xde\x12G\xc9K\x9c\xc9\xe3\xb0$\x9a%\x95U$IXD(^.\xea@\x9bTq\xd9\xe3\xbc\xa5\xcb\xc6ZB\r\x97Pi[T\xb1s\xfc\xc9\xc8\xd9\xf5\x9a\x1d\x90\x1e\xbd{\xcco\x037\x06\xd4\x96\x80\x8b\xa0\xb3\xb8\x84\x88\xbe|\xeb\xc5\x8bbs\x92=\x9ck\x8e\xd98]3\x113\xafC\x06\x1f\x80Z~\xb2\x978\xebg\xf0\xd7\x96\x90\xd1X?3&\xb5\xb8A\n\xfc_!"uK\xe8\xe3J\xc9\xfd7\xc2\xc3\xf6\x1dc\x9e4-\x8dXbo\xd8\xef[RO~p\xfe\x06\xd0\xea\xc0\x1a\xee6\x9bN>\xdf\x92\x14\\>\x14]\x81\xaf\x04p\xa6\xa0\xb5:\'<\x96v\xd9\xf0!\xb6\xdb\x06&wu\x9d\x04\xe1D\x8c\xf1J\x97\xf2\x0b:\x89g=\xcaC\xc6\xd6\x9d-\x0e8\xc2\xa7\xc3e\xc2\x04\xce\x96\x8b\xd0\xcf\x8c\xf9\xc1!\x06\r\x18\r\x1fb\xcf\xb6\x04{H<-o\xbc$\x0e\x96\xe7T\xd3\x87\x18\x0cX\x9d\x80:8t\xe4o\xe1\x86"t\xac[T+7\xdb\xac\xe8\x10\x18\xee\xcb\x9c\x8d\xf0\xa9\x1c\x02\xa3\xa7C\xfd,~\xd9\xb0\x15\xf4\xf9\x0eI\xecB\xf8\x16\x00\xc7J\xd9\xf5\x8fnK\x9f\xb4[\x9en\t\xf6)g\x8b\xa8\xfef\x9f\xbbw\t\xf3>\rQ\xd9\xf7\xf7\xfd\x13\xaf\xc4\xb2+\x9c\xbaV\x91\xd4.\x93\xb0\xee2\xdf[\xa5\x92\xc5\xcb9\xfe{r\xf2p\xe2e\xd7\x8bsPz[\x00\xe1\r\x97?=\x91U\x16\xa4[\xa0\xe6\xca@U\x89\xc8\xa5\x129-\xa3\xd9iy\xba%x\x81\xd5o\x12c\x10~\xd3~\xb4\xc0\x9e7-E\xb4\xf5\x18\xf83a#\xd3\xae}sK\xb9\xb3\x1c;D\x99\x89f\x10Rzg\xab\x8c\n\xaf4*\xe0\x1c\xb3\x99_\x98\x16`\xeb\x02\xb56<\x1d\xd0\x7f\xd0\x85.i\xea\xc7w{O\x9a\x96\xa7[\xf2\xc2iqQu\r\x97]*({\x07\xdb\x80\x9c\xc5\xc12w\x94oZl\xc4x\xf8\xcf/\xf8\x96\x97\xec\x96F,y\xd5\xb4\x0c\x9d\xed\xcd7I\xfbUj\xa1\x04o\xfbI\r\x17\xa3$~qN\xa9\xd5I\xb0\x9b\x9c \xfc\xdf\x94\xbb\xec\x15C\xedWv\xcb+\xa6\xa5\x11K^\xb6[p\x06\x1e\x176e\x95\xbb\xad\x0f\xcc+\xa9\xc2Bw\xa1e\xda\xc0\x82\xc5\x85\xcb\x05\x8e\xa9\xf2\xbd\x8f\xbf>\xec\x8c\xaf\xc3\xceO.\x8f\xcas\x9e7\xb1\x17\x9e\xfd\x9e\xaf\x1ct\xab\x83\xff\xdeY\xb0a\xd9\xc3\xae0\xa2\x01\x8cc;\x0bz\xd8\x99\xa4\xa4\x9d\xb8\xf4\x0e\x9e\x13q\x81y\xaa]g\xc7\xfe\xfcj\xe8U{\xa1\x11K^5-\xbe\x99P>a\xe6\xa8\x8eai0q\xc7\xcb\xc3rq\xb2S\xfb6\\.\x8ea\xc1\xddBf\x8b\xbf>I\xc3\xa5}p\xeb_\x89\xb3^\xe1\xd0\x1b\xb1\xe4U\xd3\x82\x83\xc1\xb3m\xb2\x10u\xd7>\x9ap\x9e6\xb9ZEu\x00\x12\x8c\x10q\xe7\x90\xd9Q\xc3\xe5.\xc5\xee\xf2f\x97ya\x97\xfa\x07\xdd-\x8dX\xf2\xaai\x89\xf1G\x84u\xc2F\xa5\x8d\xbc\xd2\xab|t\xe4\xdc\xda;G\x1a\x0e\x18\xcb\x9c\xb6\x1d\x83\xb6\x97\x13&\xaaI\x1av\x0c\xca\x9f^\r\xbdh\xb74b\xc9\xab\xa6%b&\xc82OW\xbc~\xb2\x08\xa8\xa2\x1c\xa0\xe1=qa\xbaH\xc2\x12dDm\x0eh\x12\xa2Ra\xe3Y^E\xb3\x8f\x9a,6b\xc9\xcb\xe2,\xb4;\x06\x1d\xfb\x1c\x16\xc6)2\xc3\xda[Fl\xb8\x8c\x98\xb0\x8c\xf6\xb1\xe9\x95.\\\x0b\x01C\xdbr\x91\x0c\xcd5\x15\x98\x1fuZ\x1a\xb1\xe4e.\xbf\\\xa4q\xb9\xc8\x86\x0e\x97\x87\x9a\x8c"M9\xf8f\xb4\x8d\n\nD\xf4\xb6^\x9a\xd3!\x13R\xf6\xb2[\xfbe\x82\xb3_\xf6\x83NK#\x96\xbc\xec\x103\x93:\xaa\xed\xf3p69\xc6e\x82\xa3\xfc\xd3\xd9\xef\xd8\xf5\xd0ai{6\xe1\xc3\xa5Ly&W\xc4%p$n\x99\x90\xf9\x15\x97\xff\n\xdf\xd2\x88%/\x9b\x16\x04\xbd\x1d\xf8\xf4e\x14d\x95Y\x1d\xa5\xf2-4w\\\x84\xb84\xac#6X\xee\xf0\xcf-2\x1f\xbbQ\xbc\xfd\x0f~\xf9S\xd8\xc1\xab\xa6\xa5\tK^6-\x0e\xfe\xb1\x0e(\xff\xecJ\xaf\x0e\x19\x0f\xd4\xfd\x9c\xe8\x14/m\xc6gt\xbc\xfe\x94,\xd4\xa2\x12\xffw\xe6\xa2I\x89\xe3\x98\x0fz\x885b\xc9\xeb\xd2\xc9]\x1ai^e#\xee\x8a?\xfc\x1a"\xec,\xcd\x08\xc5\xc5\xb1\x0e\xd1\xa4\x18jy\x1d\x9a\xf414]|ZO\x12\x8f\xf9\x85\xdb\x94\xd7\xf8\x96&,y\xd5\xb4\xd8Et\x02\xa6\x1e\xdf\xach\xec\x14Y\xaf\\\xdc|f\xcd\x85\x0e\x8ecf\xd150\'\x08\xf8\xff\x86\xce\xf5\xea\xa7\x1e\x1f\xfc\xca\x95\xf0K\x0e\xb1F,yYM,\x9d\x9c\xdc\xa5qr\x19\x03\r;\x93*Zf@\x9b{\x88:[\xc6e\xe6W\x0f\xe7\xceq\xe11\xd8ir^\xd9+\xc3R\xfe\xa0\xbb\xa5\x11K^v\x88\xa5I=\xd4"\xa20\x07\x06\x04E\x95\x86\xcb\xc9!\xc0\xb9q`r)(s\x84i\x17\xd4\x87o\xb1\xa6\xf0\x1e\xfa\xa8yK#\x96\xbcl\xb7hkp\x8f7\x17\xe9T\x04P6\x93\xbe\xfa\xa6\x87\x86\xcb5\x8e\xf0\xa3j\xe8\x18uX\x82@\x81\x85\xc2\xd2\xc5\xdb\xff\xa7\xd8\xa9\x17\x1db\x8dX\xf2\xcei\xc97\x97\xb8\x8c\x8f?4\xc57\x17;\x0f%_c\x9a\xbe\x8a<\x02\x07{@\xb4\xc8\x96\xa8w\x0c\x90}\x08\xaf_Q\xe2\x0c\xb5o\xa2\xfaI\x16\x16\x8b\xfa\x9b\x9f\xfach\xecKPZ\xd5p\x16\x81J+m\xd76=\\\xf6\xd2\xa0\xb3e--\xa1"\xbcJm\x93\xbaY\xf8\x80\xf1\x80%\xb9\xcc\xe9\xb8\x1c\xb3\x1ei\xec\xf9\x0b\x18fk\x1e\xe5\x95[\x14\xa5I\x16.\x8dcT/@x\x81\xb3\x10\xbd\x8ff^\x16\xc3M\xc1lr\rR\x9c\xcf\xce\xdck\xd4\xd1\xff\x1a\x86\xe1\xc5\x9b\x85\x8c\x91{\xe5\x0e\xc5K\xec\x89\xeb\xde\xd5f\x92\xab_fx3%{KS\xb0g\xf6\xb0\xc7\x9e\x14\xf8w\x8e\xee\xf2/bX\xd8\xd9\xe1%\xa8\xa4v\xb1\xad\xdd\xda\xae\xa2N\x82}W^\x86\x85\xc1\xd9N\x85?\x10\x1b\xbd\x8c\xb2\xa0\xf0\xf6q\x11e6#\xff\xcf\x1b&\xff\t\xc3\xe2\x0e\x0eW\xeb\x053\\N\xce\xee\xd2\xa3\xadzr\x0e\x81\xb2\xc8\xf1x\xec)N\xb1\xd6\xdb\x87\x9d\xa4\ng\x93\x83mNp&\xb2\xfek\xcc\x98_\xd0E\x98z\x19\x9e%\xca/\xe6\x94\xed$I\xc4\xd8\\T\xd8\x07o\x96\x1f\xc3N\xaf\x88\x97\xc9)J\xf1\xa3\xb0q!i\xa8\xf9\x0b\x18f\x17\xc69D\'\xca\xc2\xc6\xb8h\x8b\x82\xa5w\xf2;\xa0\xb3;g\xbc\x99\xc2D\xa0\x95\x95\xae\x11N\xe5\xb1\xb1\xc0O\xe4\xfe\xf33\rS\\F\xdfp.\xc1m5\xf9\xa8M\xe4\x9f\xb6\x87\xdf\xce;\xfc\xef\xf8\xf7o\x84\xdb\xf2\x18\x97\xc7\xf3\xf1\xb7\x9d\x7f\xf0\x8b?\xbe\xfd_\x9f\xc8s\xe0/\x85_\xfd\x16\xdc\xc8W?q\x88E\xa2\x08#\x02_]\xfb\xf05\x9e\xba\xd3\xad\xfd7\x90\xabm\xa2\xb8\x04\x16\xa8\xfb\xaf\xf9\xeb\xf8\xb7\xf2\\\xdc\xdf\t~\x96\xa5\x18D}\xf9u\xf8nt>\xf8@\xbb\x87\xbfG#J\xfa\xfc\xac/\xfcm\x8f\x1fMS\xac\xc8\t<|\xcc\xef\xe6\xc0\xe7\xec\xce\xc1ox|\x7f\x83I\x85\xd9\xfb\x14\xc3\xc0\xd0\xd2\xa7\xff\xfco\xf8\xdf\xef\xa3/2\x7f\x8c\xf9\x83u\xf7\xe1\xdaD\xf0\xf7\xc4?&\xf8\'\rQbt\xcaP\x11\xe2%Y\xe64\x99\xa3\x04Eai\x86\x97i\xc9\xe0TZ\xa1u\x9d\x15\x05Fa\x19Z\x92\x19\x81\x91)\x95\xa68\xd1\x108Fbh\xfc9x4Od\x14O\xdb\x93\x9f\x7f\xfa\x07\xfd\xb7/\x0fW5\x9dU\xf1S$\x99eeN\xa2\x90@s\xbc\xa6\xd2*-\xe2\'H\x82* \xca\xd0\x19E\xa2(\x89\x11i\x83\x93yM\xa1E^\x93E\x9e7\x10\xfa\x84M\xfa\xe3\xed\x93\xd8\x8fb2\x0c\x97\xf8@\xc2\xae?l\xfbD\xc3\x9b\xf8\xbb\x1d\xfc\x15\x86-L\xfcMIL\xfd\xb4=\x16\xdb\xe3\xe6\xf8\x1f\xf0\x13I\xbcY\'\xa7/#t\xda\x14\xb0(\x11\x85\xe8\xff\xa0\xf8\xff\xa0\xc5\x19\xa2\xff\xc1\xa1\x7f\xd0\xfc\xdfE\x96F\x12\xcb\xb1\xc8\xc3?\x97\xfb\xc7\xd3o\xdf\x19<\x05\xe9\x1a\xc3\x88\x94 i0 \xba\xc8\xd1:\xa5\xea\x82\xca\x08\x1cB\xd8P\x19\xf1\x02\xab\x88\x08\x0f\x14\xcd\x1a\x1c\xc3k:\xa7\xe0\x01\x11\xf17\x14\x99\xff\x97\x83\xc7\xf3\x8a\xc6I\x8a*H\xbc\xcc\x88\xb2\xa4h\x94\x8c(^5\x18\xc6`UZW)Y\x92\xf0\xd0\x1a\x92\xae\x18\x82&k\x82\xcc"\x85\x138Ie\xf1l\xdd\x07\x8f\xbc9\tTO\xbf}~\xae\x84(\x9aa)\t\xcf\x9f\xce\xa8\x14\x92d\x89\x95$V\xe4\x19\xa4\t\x14\xaba;T^eiIb\xf0\xe3Y\x96eT\x81\xd3y\x0eO\xbb\xc6\xa8\x0c\xbc4\x10\x14~y\x1c\x9e>JeY$\x89\x06L,\x8b_H\xc1\xef\'J\x12o(\x12\xdeB\x82\xac\xb3\xb4\xce\xf2\xac\xa4H\x0c\xab\xca,~\xaaD+\x82\xc8!E\xe4\xb8O\x0fK\xfc\xf8\xe5\xa1\x08o<\x1d\xe15\x83GI0t\xfc\xf1\x94\xce\x08\xacH\xe1\'+\x1a/sx\x08\x10\x9e"\x9e\x93%\x9ae4J\xe4d\xd6\xa0(\x9a\xa2X\x9d\xd2u\xfc\xd02\xaeN\xbf5\xf2\xe4?\xce\x91/3\x85$\x1ao\x1al\r#\x08"\xcb\xaa\x8cF#M\x96T\xfc+QxQ\xd2\xbc\xac\xe0e\x88W$6X\x13\x04\xd9\xc0\xe7\x82D\xe3\xc7\xe3E+S\xa2\x86\x07\x1cv\xf9\x9d)\x13o\x9e\nN\xc3\xff\xfc\xe6T\xfe\xe3\xef\xe4[\x0f{\xe1~\x9c|9\n\x10~\x91\xc3\xf6\\F\x84}\xf3/\xb9\xcf\x8f\x9bu\xe9\x9f\xce\x87\xcf>\xfd\x8b\r\xbf\xadr\x7f\xfd\xe9\x1f\xe8\xd1+\xfe1\xd2\x88\xd2\r\x9d\x1c\xb7\x12-S\x94\xac\nx\xbe\x91\xaa\xf1\xaa\x82O\x7f]\x14\x91\xca\x89x\xf7\x7f>)\x8f\'\xbf\xd8}\xf7\xb8\xa4\xff\x8ex\x8a\xe5\xf12e\xe1\xb8\xfc\xfde \xaep-\xc64v\xf9\xd6\xd8\x9e6\xfb\x96^\xad.\x8b\xb5\xcf3\xb4y\xeb\\\xbb\xfd1;\x1d\xd1\xa5dm\xd7\xec\xb6O\x9d\x16(\xba]\xae\xf1\x0e\x0f\xf1\xeb\xc3?3\x8a\xfa\x07\xf9\xe7\x9b\x97&0\xcd_\xdf\x0f,\xde\xa84\xb8\x7fM\xe1\xf0q"j\x86&!\x1c}\xe1\xff\xe7\xb1\x07\xa3\x19\x85\x11\x19^\xfc\x95\xfd\xf0\x18%=\x8e\xa2|\xd3\x94`\xb4\\\x0bfO\xd6\xa4\xe3t\xce\xa6\xfc\xb6\x9f\x0cG\x97Q\x19;y\xd6\xdb\xd8\xd3b3\xd8\x07\x92\xad\xea\xf6\x84\xdar\x92\xd0_%\x14\x17\xe5C*\xb4\x06g\xab\xb5-\xdc\xce9\xbct\xa8\x00i\\!\x84\xddwn\x88_\xf6#\xbf`&\xdeC\xd8%J\xcc\xb7f\xba.\xeb\x0f\x86\xf6H\xf1\xd1^\xccg2U\xe7\xf3"e/\x92\xa6\xf7\'\x9b\xcb\xd6YW\x9b\xab\x8b\xcf\x0c\x89\x92\x8a\xd5i~\x12\x06\x1dwOe\xba|\x9a\xd9i\xb6\x0c.\xe2v\xa6E=j\xc4\xb7\xb8\xaa7H\x8fl\xa2]\xdf\xb7\xefq\x9c&b\xc7\xcaR\x82JI\n\xc32\x1c\x8fd\x19\x1b-+\xf8\xa5e\x0e;v\x1e1\xe8\x17\xcc\x14\x90@\xe1\x15\xc0\xbf\x99\xcd\xd9\xfa`MXz\xa2\xad\xa5\x8d\x84\xf6\xf4\xb0\x1f\xac\x07\xe1\xac8WA\xe1\xbbr\xccM+f~\xac\xe6rp\x93N4%\x1d\xd6BOq\xb7\xc6\xbe\x9fZ\xa3\xd1r\xb2\xb1\xd6\xe2Y\xae\xeb\xf6R\xe4\x8b\xccQ\x8c8[\xaa\xef3\x13\xc7lHS\x90\x8a\xe3\nI\xe3\r\x9cO`\xa7k\xe0\xcc\x84SUA\xc3\x878\xce$\x0cYW~e\xebc\xff,\n8\xa5\xf9\xd6\xccn\xcc\xdb\t\x1f\xa5,_*\xb7\xe3\x99W\xe8\x8e\xe8\xb2\xda>r\xcf\xd3U\xebf\xb9\xe5M\xb4\x96m]\x11WF\x9e\xf9Y\x18\'\x93\x89\xde\x9e\xab\xbd\xc96,\xcb\t\xdd\x1b\xcct\xa9\x93\xe1\x80\xba\xeaZ\x99.\x18N\xf9\xce\xd9\xc4\xef\x8b\xad\x12U\x95\x95t\x08\xc9)\x15G\x828D\xa5i\x19\xa7\xa2,\x0e\tiIC,\xfbK{\x93g9\x9eEo\\\xf2R\x1e&\xeb,\xd1;|k;q\x03\xbd\\\xf6\xfa\xe5\x8d\xb9];\xa3\x00-]yd\xd2\x13\x13\x89\tSL\x85%ZW\x99>\x8a.\x15\xb3\xdd\xfb\x07\xe5V\xf6\xebq&:\xb4\x1fF!\x1e\x91\xab;\xdd\x8co\xb5\xad\x8e\xdfy\x90\x8b\x06\xf6\xbd\x86\x8eW\xaf@\t\x1a>\xbbq\x9c\x84\xf7*\x8b\xb7\xa5\x82 \x8a\x864\xf1\xd7\\2^\x118\xa1y\xb3h\xc3\xd8c\xda\xdbe\xffD\t\x17\xb3\xbd\xebv\x94\r}9\xceg\xca\xfc\xb4\x9d\xf2\xd4\xe4\xa8\x05\x85~\\\xd5l;\x9a\x14\xe3\xe9N\xb2W\xe9\xed\xc2l\xe4K\x80\xd6b\xe1g\xfb \xce\xabQ<\xed-\xa9\xc4]\xec;\xda\xba\xaf\xbd\xcfLF\xc3v1\x94\xc1\xc82/\xaa\nv\xcd\x0cC\x91\xd0\x11g\xf7\xba\x80\xfd\x8f\xa20*)1\xfc\xd4L\x06/\x03\x91b\xde\xf8\xab\xd9\xfav\x0eP\xdfdck\x8b\x1d\x11b\xdaz\x7f\xb8O\x16W\x1c\xe4\xf7\xfaNpq[\xa6\xdbU\xe2\xdc\xbbd\x13a\x8dJzc\xd8lv\xdb\x1a=N\x9a\x0f\xb1Gm\x8f\x16\xc9\x1eEY\xc41\xd1\x90os\xfa\xf5G\x8b\xf65\xfe\xca@\x0c\xa3S\xf8XbE\x99\xc1\t\x8e\xc0\xa84\xc7\xe2\xdd.`\x7fE+8mSYE\x11~\x1e\xd9\x08\x7f\xc7i\x17\xc7Q\xa2\xc4};\x8a\x81\x10\xcc[\xde\xaa\x9a\xc7\x86|d\x8e\xfd:\x9e\x06\x93J\xdc\xf0(9v\xcb\xc5p \xcc\xcd\x89\xbc\\\xd4Nd\xeb\xb7s\xbf\xb5\xe1l\xb6\x9e\xad\x84\xec\x9c\xae\x92a\x8f\x13+?\xdaS\xd5\xd6w\x16Zh\xce\xaa\xde\xca{\xe7\xd6\xa78|\xf6\x8a8\x01cTY\x16d^\x925\x1c\xa9\n\xbc\xaa\xa9\x86\xa1Q8wSU^\xd7~i\xeb\xf3\x02/\xd0ow>{\x92\x1dw\x1a\r\xfc[\xd2\xbb\xac\xf9\x80\x15U&\xd8\xf7V\xa1\xa4\xa3M\x92O\xd1L\x19\xa9T\xdc\x8d\x0f\xca\xf1\xdc\xee\xac\xc7\xbb\xacs\x1e]Y\xbd\x9c\xfa\xa2\x89\xdc\xc4\x95\xab\xec|\x9cl\xae#\x9b\x16\x8b\xd5r4\xec\xa8\xef\x0cS%\x8ag\x14\x9c\xcf\xb2\x14\xab\xf0\x12G\xe1Y\x94q\xe2C\xe3,\x1c\xafn\xd1\xd0)\xec\x84(\xed\xd7\xc2T\x9e\x93\xf0?\xdf\x9a\xa98\xba\xcb\xcfG\x9dI\x1d\xf6\x06\x0bc\x98\xc5\xe3\x8d\xe9d\x97\x93\x13R\xe1\xc8\xa9v\xcb\xecx\x1d\xcf\xb5\xbe\x94\xb7\xf6\xe7\xc5f3k\'\xf6\xa4\xdf\xbe\x14\xe7\xf6%\xdd\xe5\xb1\xb7\x8f}\xa1\xa2\x95l\xa7\x99m[+\x05\xd01\xf9H[\x82\xc2\x1e\x90Sa3(\x12>I8\x08\xdd\x18\x19\xe76\xbaDa\x87\xaf\t4G\xd38\xb6\xfb\x95Q\xa4(F\xc2A\x03\xfd\xed(\x8a\x85\xda\xde\xf6|v\x1c\x0c\xbc[?\xd3\xf4\xfd\x962SUd\x8d\xfe\xa6S&\xf9*\x91\xc7\xc3\xa5\xba\x13\xdc8\xc7\xa1Tb\xaf\xae\xd7\x8db\xcc[\x83X\xab\xacP\x8d\xe7\xc1tY/\x98\xd3\x9c\xe3\xaf\x97V\xc7<\xf6\xb6\xef\\,:N\x8de\x8a\x169\x997\xf0D\xeb\x9a\x84\x838CQ\r\xbc\xfdu\x85\xc6\x8e\x98\xe2q\x98\xf7K\xf1>v\xa6\x1cO\xa17\xe7gu\\\x99\x19\xddg\x8e\xa2\xab\xf8\x01\xcf\xe3s\x10;\xbb\xd1l\xcb\x0fZj5e\xd9\xcb\xbc,\xf6\xd7\xf5(A\x91\xbc\x89B\x03\x07\x0f\x97\xf9\xd1.\xac]\x97\x1d\xdc\xae\xadh\xd4\xedu\xae4\xaf\x0bfBm\xd7\x19EI\xefLkT\x1c\x86\x93M\xa1\x8b\x06\xab\xeb\xa2\xa6H\x0c\'\xe1L\\\xd1\x11\'\xe0\xf0\x8d\xe1YA\xd2\x7f\xc9M\xb0H\xe4\x18\xc4\xbe9\xe0\xf8\x0c]FG\xcf<\xc482\xef^\xadm<\xee\xec\\\x8d\xceW\xc2\xe1*\xed\x8c\x90\xdbh\xd6\x1ee\xadQ\xe0_\xfc\xee\x8c\xd6.e\x80\xb3\x9d\xad\xdb\xbb\xa5\xc7\xc3h\xefo\xf5\x81\xa1\x0b=\xba(\xb6Q\xab\x90\xa3\xf1\xf4G{\xe2G\x8b\x16\xe1\xf4\xd3\xa0\x05\x11G\xe0\xb0\xdbU\x86f\x0c\x86\x92D\x9c\x99\xe3\xb3\x8fe\x19Y\xa4\x05M\xfe\x153q\xee\x8e\xe3\x1b\xe6\x8d\x99\xd7\x81\xe7\x1d\xae\xfafl\x1dZU\xb9M\x8d\xa5\xa2\xd6\xeb\x90\x9f\x9dU\xe6$\xaf\xcdR\xdd_\xea2\xba\x1d\xadY\xf7\xc8lWjV+\xf6\xcdE\xb7@\xc3\x9f=\x19\x84K\xb6\xddsn\x1c\xdd\x9b.5\xdd\xd8\xf6b5\x92?\x947\xa4T\x8eSi\x9c\xb9\xa8<\x8f\xbd\x85\xa8\xe21\x15\x04N\xd3p\x9e\x83G\x84\xa1\r\x83\xe7\x91\xfe+\xf1>\xce\xa1\x05\x9a\x17\xde\xb8\x89\xf9\x94\xc9\x84\xedb%\xeb\xcb\xf9`\xa8$cg\xc3\xf8\xad\x8a?\x0f\xf6\xad\xf9*Q\xd1\xc0g\xc2\xd3Zns+\x0fy\x1d!\x8d\xf6\\v\xee\xd2\xee\xf0"\xdb\x1d\xfcI\x97\xb0\x8c\xda\xd3\x16\xbf\xf3Q\'\x12\xf7\xe9\xa1\xe8\xc8\xef\x8b\x9c\x10RdN\x92U\x1c\xda\x89\xb2!\xb3*\xadp\xbc\xc6\xe8\xbc!\xe20\x88\xd1Y\xbc|4\x1cR\xfdJV\x83=\r\x87\x0f\x917)\xeaij\x19c\xb6et,\xcd\xc7\xb1_a\n\x9dh{^\xc91\xa5\xf4y\x8a\x12\x85\x91\xb0\xf3\xad\xb9\xbd\xd6\xafB\xd0\x11GRnM\xac\x96\x82ZW\xbd\xe6\xaa\xf1:\xe8\xf7\xf3\xc5z\xcf\x0e{e\xb9\\\x1d\x03sk\xfch\xa9\xfc\xc8J\x83\xd5x\x81\xa1X\x81\xc7\xe9\x1b-q\xb2\x81c\x1b\xc5`\x19Q\xe1iU\xe1qr\xce\xe1\x13\xfe\x97\xea\x198\xdc\x17\x98\xb7\xcep\xb9\x9fi\x9d\xbe)T\xaa{\xb4;\x87i\'\xe7\xcd\xde\x84\xba\x8c6\x97\xb6b\xb4\xa8@\x8eF\x8c964\xe9\xb2f\x8c\x11>!\x13*\xb2\xcf\x8e[\xac\xdd\x01\xcd\xf7\x82V\xdf\xe8\xdd\x96\x06\xcddn:\xb7\xdc`\xb5\xef\xbd\xf3|c(N\x92dI\x85\x0c\\\xc2\x0fR\xb1\xd3\xc7G\x1e\xc2o\xabq\x86j\xa8:\xad\xb3\xb4\xf0K\xd1>\x07\xf5f\x84\xde\xcc\xa6\xcb\xb1\xc3p\xd8\xbaMu\xb3w\xda\x1c\xdau\xe7\xc4\t\xf4M\xf5\xd1a^\x9e\x94\xbe\xd5\xf3\x84D\xd8$\x13\xb1O/\xa5\xc4)\xcfLV\xad\xda\x8a\xb0r\x96N\xde\x9d\x9d\'\x8bK\xa1\xe2\x03\xc30%O\x9bf\xeeM^\xfe\xa8\xce\xf8\x9a\x8d\xcf\x1a\xb2\xa8\n:\xaf\xe3a\x135\x8ac\x15F\xe2%M\xa3\rJ\xa54\t\xefy\x99\xd1\r\xfa\x976>\xc7#$1o\xeb\x19(\x0b\x99\xa2{\xee\xaf\x9d\xaa2\xebnx=\xc7\x99\x94\xcd\xb8\xc5\x99\x17\x13q\xb8G\xd2\xb6n\x0b\xe5\xa2\x16\x8e\xbe\xae\xb4\x909\xcb\xb3\xc0\x0c\xea\xce\xe1p\x9e)\xfd\xb5\xca\x9b\xee5\xecd\xad\x81\xb0n\x15\x8ey\xedU\x8a\xf2\xbe=\xc1\xe9"%\xe9\xd8C\xc88<\xd4\x18\x9c(\xe1\\\x10{\x0c\x9a\xa5t\xbc\x84T\x83\xc7&\xe3H\xf1W\xcc\xc4\xbf\x84\xadd\xdf\xee\t.\xd1f]\x1f\x19#\x9dCek7\xb3\xed\xf1\xb8\x95\xef\xab:\x1b\x9a\xa8e\xd6\xb7\x83\xc0s\xc7\xb8o\xae\x1de\xd0_\x95\xaaxl\xf7nQ[6{\xa1\xc3\xea\xfd\x9d\xbf\xbb\x8d\x92\xac\xb8\xad\xa8=W\x8f&jKygj\xc8\xcb\x94\xa6\xe8\x9cF)8O\xc3+\x95\xe5\xf1\xe6\xe7\xf0\xb9\xad\x08\x02\xcfs"#\xaa\xf8\xeb\xb4\xfaK\xc78\x9eMQ\xa2\xdfd\xc0g;[\xd1a~\xdeqf\xd7\xf7\xb6j\xb7\x1a\xce\xc3bZ\x1b\xf5\xc5\x90;Z{\xd6\xebD{A7w\x8b>\x9f\xd2\xaa\x17\xed\x9d2X:[[Gmq@\xdd\x92*\x98\xb5\xf2\x03s\xeb\xda\xda\xb0\xdb\x1e\xb5n\x13\xf6\x9de\x1b\x81\xd6\x0c\xec\xabDAR\x11\xc5\xf1\x8c\xcc\xeaHVtM\x968\x15\xaf\\\x95\x134\xc5\xe0\xe5_\xf1\xf9\x12\x0e\xf5h\x96\x96\xdeDpa\x8c\xc6\xc3\xbc\xbb\xf1\xe8\xe4\xc68nZ\xed\\\xd3\xbb\xf4Mz[\xf6\xa2U\xbf\xa0\xa2Ue\xb5G\xf5\xcct\xabI\xcbgOU\x81\xe3,n.\xa5\xce\x91f\xabD\xd8\xa5\xeb\xd6\xb0dG\x01\xe53\xdb\xbeP\xf3\xa2\xa7\xbc/\xb4\x11TN\x96E\x1c\x88#\x1c\x8e\xf2<\xde\x89\x9c\xca\xb3x^\x0cN\xc09\x9b\x86$E\xc6\xa3\xf0+\x11\x1c\x1c\xe4\xa2\x88\x93\xc1o\xcd\xa4\n\xa9w8s\xcb\xe4BS\xea\x9cb\x86g~rt\xe5\xf2\xe8u8\xfe\xd2\x9bW\xec$\xa5*\x0f\xcf\xec\xd2(\rox\xce\xdb\x9d\xd5\xc8\xbfP\xe3Bub\xbe\xd4\xbb\x93\xb0\xaa\x19\xcf\x15\xf7uV\xad\xcd\xc3\xf4\xe8\xbcs\xd1\x8ap\xf8R\x1cN\xd6tNRX\xca\xd0uC\xc3F\xe3\x03\x89\x13p\\\xac\x18\x02MK\xfa/-Z^b\xb0\x8b\x95\xde\x04\x1f\x96\xb0\xe0\xcb\xa8\x9c\x06\xd5\x1c\'\xda\x85\x9ft\xfb\xb9X\x87\xfet;Z\x08\xc7\xael\xedJ\xcbV\xa6\xed(,\xdb\xf6\x94\x97\xcc\xb2\xb3\x93\xc2\xe9\x8cg\x9c\xb0\xb7\x8d$u _,-\xa9h\xd9\xb4\xa3\rw\xa4\x86\x8c\xa4\xfc\xc0\xcc\xd7\x1c\xe4\x92\x80d\x1e\xa7Q8\xaaWU\x1d\xafi^R\x18A\x964\x96\x91\xf0@\xe2\xc4M\xc7\xde\x91\xfd\xb5\xaa\x90\xc4\xd1\x92$\xbc\x89\x83\xeb\xce\xe4\xb2\xe0\xb5`|\xe9\xb1L d\xa5\xb7vN=AG\xe7q\xa1\x1b\xeb<\x8a\xec\x01>\xbf\xc6\xd7\x96\xd2\xe6\xc6B\xecX\xc7=:\xad\x96\x9d\x8b\xbeWWQ\x11\xf3\xcau\xd1\x9e\x9d\x16fw^&\xdbi\xa5\x9c\xc7\xef\xcd\xf45\n\xe1\x98Z\xe4E\x01i\x8c\x82\x0f;F\xe5DC\xd1U\x99\xc6\xf1+\xdcf\x8b\x88\x16~\xe5 \x97\xb0\x1b\xc0i\x15\xff\xc6\xcc8\x10\xd4\xee,\x9f\xdaQ\xa98\xd7\xad\x1a.\xdd\xd0\xe8\xc8\x8bM\xab]8\xf1h\xd1^\xcfE\xb6\xae\x16\xfd\xe0hg\x97\x96\x910\xabp%\x0f<\xd9\x08\x06\xc3\xad\x19\xcd\xcf\xabK<\x99\'x_PG#X\x9e\xf8\x81\xfb\xa3\xc5\xf2\x033e\x9c\xa1\x19\xaa Q\xba\xa60<\xc7\x18\x146KSU\t\xa9"\xcf\xe1\xa9\x81ko\xc4\xfd<\x15\x17\xfeN\xe1\x13\x12\x89\xcc\xdb\xe4\xad\x1ck\xcb\x13We\xc5UC|\x92\xb9[I+r\xaa\xee\x16\x92R\xa9\xe3U>\xaa\xe8:(\x99\xc2\xed\xaa\x81\xde_\r\x9c\x1e5\n\xaez@C\x00m\xdd"\xd7\x1d-\xd6\xd1xu\xa4\xd1\xb4\xbd\xd3\x93j=@\xea\xfbN8E\xe3 \xf3\xe0\x19\x8dax$\xe3\xcc\x83\xc3y\x1b\x928\x03aW\xc5 \x06\x01FH\xa3\x7f)G\xe5\xf0\xd2GoO\xb8\tu2\xcf\xc9B\xe0o\xe3\x8ddM;\x87\x1b\xefD-=M\x13K\xbdp\xe6\x84vi\xa6\\\xad\x86y\x959Q\xdcS\x8e\x83u\x91\xa6\xfdK\xca\xf7\x0e\xd7\xdb\xe88qn\xf5\xe6\xd4\x92iN\xe2\xca\x96y1\x82^\xf4\xa3\x83\xfcEWo\xf8p\x94\x04FRU\xd5\xa08Z\xa68\x91\xc7\x93\xa4\x882B8\xd1\x91dV\xc0C,\xfd|\xeb\xc3b\x11y\x9c\xef\xa17\x8b\xc5r\xf3r\xc3\xf6B~ZDTkm,\xc5\x99\xdf3\xe4\xfc\xd2ZD\xe9$\xdc\'\xd6i=e[\xfa\xb5\xe3\x95\x94\x12\xb4\xe7U\x7f\xd2\x11\xb5\xb6\xecr;\xda\xd4\x99Izt\xf6V/\xde\r\xe4\xeb\xe0\x10&\xb7=\xfe\xdeg\x94\xf5\xefx\xac/\xe8\x92\xaf\x81\x14\x04SU\x1d\xff\xf8\xe6\x1dx\xf7o}\xc4\xdf)M\xf5\x9b]\xff\x08\x82\xff\xbfH\x1f\xf1\xf9\x8ao3\x0b8z\x81\xcd\x9a\x1a\x82~w\rJ%k\xce\xae\xe7Ms\xf46b\xcc\x19\x84\xaf\xbfcL\xb3\xbaU\xff\x9e\x96\x7fi\x8c\x06\xc6t\xafv-3v\xdd\xad@5a8\x9b\xd7\x16\x88Z|WV\xe8\xc3N\x0b\xb6\xc4\xda\xb0\xdf\xb5\xc4\xaa\x1bWzkBU\xb0\x0b\xfa\x88\xac5\xcb8+\x9d_\xb1\x81\xb0\xe6n =\xdf\xa4\x1aO\x13\x96XW \x83\xb6R\x17\xef\x16\xf7\xde\x9dU\xeb\xac\x9d\xea\xc8\xd2~\xc4\xd1\xfd\x81\xf5\x11\xed\x1a\x1ff\xb5L\x83\xce\x80\x05\x02I\xf8\x08\x80\xd63[\xcb\x9a=\xcc\x9a\xb0\xa4\xc6\x8b\xec\xbb\x964M\xcf\xfftEA;\x95\xcfd}\x81\xce\x08(\x8a\xe2P\x00\x1bH\xe1u\xd7\xa8PR#\x96\xe0i\xf9\xbe%\rO\xcb\xf3\x15\x05\xadz~\xb64\x9d\x82?v\xear\xc3\x99K\x83,2\xfew\xa3\xbb\xa5\t\x95G\x9c\x1e`oii\xdd+\xf6-\x8c\xa5e\xb5\xa5\xb9\xb4\x9d\xce\xb9\xa6]\xff\xf3\x15\x055\x88\xc8\xdcz\xa8uY[\xebrD\xc8\x82\x88\xf2\xc9\rk\x8c4`\t\x0eb\xbeo\x89\xd5\xec\xb44\xa0\x8d\x98\xeag{\xe6^\xc1\x88\xa1\xe6\xc2\xf6\xc7;\x05\x98\xd4\xc7\x8d\x1eb\rh#~?\xb6\xbc\x81\xf0\xeb\xf7c\xcb\x0f\xac\x8d\x08\xe2\x15\x834Cxj\x905\x0b\x19p\xff\xa0\xfa\x86\x9d%\xfa\x8bi#\xbenZ\x1aP\x14$\x87\xd8\xf7\x8d\xf9\x8bi#\xbenZ\x1a\xd0F\x9c\x85g+\x1d\xb3\xc0\x13jiD\xe4\xf1f\xd7k|\x98Y\xe8\xaf\xa5\x8d\xf8\x83}\x8f\xa7\'\xac\xbe/\x85\xfe\x91\xb5\x11S\xd0\xdd\xed\xa2\xa1\x16r\xf8\x0f\xf8\x14\xa0\xcb`@8\xe9\xaf\xa5\x8dh\xa5\xd9m\x90Z4^P\xd8\x02\x90\xe7p\xb15.v\xfb\x8d\x17a\x9a\xd0F\xec\xe2i\xb1@\x88\x8b\xf8\x13[[s8~\xc1\xc65++\xd6\x80%\xa9N\xe3H\xec{\xc1\x0b\xd3\xf0ny\xbe\xa2\xa0\xbdaq\xbe\xb2\xc6y\x04\xde\xee\xb3\xf5\r\x8c\xc0\xbb\x07\x87\x97\xcd*\xf24\xa0\x8d\xf8\xbaiy\xbe\xa2\xa0}c\xeb\xef\x14_p\xe4\xdfkVc\xa4\x01\x95\xc71\xc2\xd3\x82\x17\xd8\xf8j\xa7]\x1aO\x0f(>\xd6x\xd1\xd5M\xa7\x93OW\x14\x1c\x82~U\x1dr\x96&W\xd8\xed\xb3C\xa8\xc7\x82q\r\x0b%5\xa1\xf28\xc6\xbe\xa5\x0b~\x05\xfb\x13\x1c\xc4\xcc,\x16\xff\xbb\x86\xc2\x7f\xc3\x91\xd8\xf3\x15\x05\x7fp\x88\xe1X\xa6\xd9\xdd\xd2\x846b\x97\xc3.\xff\x8d\x97\xb4\xd2n\xd3*\xd5\rh#\x82\xacX\n\x92\x95r\r\xd5#K\x03\x91G\xf7j7-\xc2\xf7|Kf\x19>\xf2d\x1c \xe3C/\x9d\xe3\xa0e~\xc3!?\xf8\x96\x1fT\xf7>\xb66"\xf8\x12\xd0\xda\xbd\x82\xd2\x1b\xd4\xc7@\xb5\xbaq\xc9\xca\xe7k#\xaal\xfd61\x86\xbf\xff\xe8*\xfeck#\xe2T-c\xf0\xba\x02\xf5s\xbcz\xbbx\x9ap\xecR7\xec\xf2\x9f\xaf\x8d\xf8\xbaiiF\x1b\xd1\xd2p\xf4\xa3\x01W!>\x89k\x9d\xc3;\x86\xb6\x7fx\xdd\xfaa\xb5\x11U\x16\xc4\x9cA\x99\x1a{\xc91\x08\x87#\x0b\xa4\x90g\x8b\xbf\xa86"\x89\xbe\x80S,d,\xb8f\x85\xa9\xf9\xa1\xfe\xe6\x07\xd6F\xfc\xc1q\xdc\xfc\xa5q3\xda\x888\x11#.\x1fBJr\xb7?\xc71L\xc3\x02\xafMX\xf2\xaaiiH\x1b\xf1\x05\xbb\xa5\x11K^5-\ri#\xbe`Z\x1a\xb1\xe4e\xbb\xe5\xffgm\xc4W\rzC\xda\x88/\xd8\x0b\x8dX\xf2\xaaiiH\x1b\xf1\x05\x0e\xbd\x11K^5-\ri#\xbe$\xfcm\xc0\x92WMKC\xda\x88/\xd8-\x8dX\xf2\xaaiiH\x1b\xf1\x05\xbb\xa5\x11K^\x16g5\xa3\x8d\xf8\x8a\xac\xa4\tK^\xe6\xf2\x9b\xd1F|\xc1\xb44b\xc9\xcb\x0e\xb1f\xb4\x11_\xe0[\x1a\xb1\xe4e\xd3\xd2\x8c6\xe2+\xa6\xa5\tK^6-\xcdh#\xbe\xc2\xe57a\xc9\xeb\xd2\xc9F\xb4\x11_\xe1[\x9a\xb0\xe4U\xd3\xd2\x906\xe2\x0b\x0e\xb1F,yYM\xac\x19m\xc4W\xd4\xc4\x9a\xb0\xe4e\x87X3\xda\x88\xaf8\xc4\x9a\xb0\xe4e\xbb\xa5\x19m\xc4\x17\x1cb\x8dX\xf2\xcei\xf9S*`_\x9d\xa5?U\x01\xfb&N\xf9\xc8\xda\x88\xcd\x1b\xf6"m\xc4\xe6\r{\x916b\xf3\x86\xbdH\x1b\xf1}\x86\xfd\x19\xa5\xbd\x17i#6?c/\xd2F\xfc\x1f8<\xfe\xad\x8d\xd8\x886"\xfb\xc7\x98\xffkmD\xc9\x90tI\xd0TY\xe0D$0\x14\xd2\x0c\x91\xa34A\xe6d\x86\x138\x16(T\x15\x8a\x13\x10\xc7h\xf8\x8di\n\xbb)$q4\x02]0\x8a\x01\x01\x9d\x1f\xcb~i\x0cRe\x8aVtMT\x04\x8e7\x10H\xbd\xd1\x92\xa4#U\xd7\x18\x96\xa6tk#\xd2\xdf\xd5F\xfc\xf8\xfb\xfc\xb5\xda\x88\xc2\x8fE\xd2\x8e\xe3\x8b_\xef\x95\xdd\xd4\t\xb7{.\x1ei^2/X\xce\x9c\xae3v\xb9ZI\xfd\x83\xac^\x87-\xf5\xe8w\xfa\xf3V\xadu\x93U\x8a\x8e\xc1\xce8\xa4e\x8b\x9b\x9ej\xbc\x93\xc4\x89\xa7,\xceY\x1a;~;\xa3\x92\xe5;\xd9\xb6\x9f\xa5\x8d\x88\xbd\x82@\xb1<\xfe\xfa[\xf50g\xb1:\x85\xa7`\xb89\r\xab[\xb9c\xe2\xf6FdSf\xe7\xd9\xd2\xe5 \xae\xfd\xbe\xa6\xef\xf4i\x9d\x9ef\x8b\x80\xaa\xc2u\r\n\xa8q+Q\xe3q\xb8\x8b\xe7\xc3\xd6\xcdvcd\xb4\xc6Y;\xebE\x11bTe\xfeN\xf6\xd2g\x89#b3\xf1\x08I\xd4\x1b\xe2\xf4\xddV\xd5\xf6G}\x9c\x17\xd6\xe4\xb8\xe2n\xf6r\xad\x9a\xbe\x9d\x96G\xb9r&Z9\xa2\xf8\xfe\xb1\xcb_\x99\xb24\xce\x17n\xa8\xadDn\xd9\x9eT\xc3\xc1\xd5\x9e\xe0\x85k\x04Et\xbdp\'\xd3\xbd\xb0++i\xc7\xbcC\xbd\x93\x88\xf6Y\xe2\x880\x95\x02\x0e\xcf\xb8\xb7\xac\xc2{\x87[\xb1\xfd\xdb\xf8p\xd9\xa7\x86r\x18\xb1;\xcd\xcc\xf5e=S\xda\x8b\x93,2v7\xe5\xf8\x0bu\xbbNMvu\xe2[\xcb\xa1\xbb\xf0W\x9e]-\xc3i!m\x82u\xee\xa5R/\xf4\xae\xd5"\x9aT\xd4Z7F\xc5\x8f\x16\xec_[\x1b\x91\xec\x07$p\x08\xff\xf2\x1b!\xcdZ\xdd\xa0\xbc\xe7\xcf\xce\xb6\xcb\x84\xde\xa6U\t\xdd\xb3}Y]F\x82Qt\xd6yu\x16{C\x97\xbbDH\xb3\x93[\x7f\x90\x94N\xa59RX\'\x1eg\xa0\xcb\xf2\xb4\xf0\xf7Z\xaeo\xe8\xf0\x90\xe4Ro\xd3]\x16\xfb\xf7jM=I\x1b\x11\x82AQ\x10x\xecz\xbf\xb5rP\xa4\xc8\xbbT\xa8Tv\xad\xda_`?8\x94\'I\xd6\xae\x02\xf3\x1c\xbaK\xbdJ\x0e:\xab\xccV\xe3|\xc0\x84;\xd3\x11\x82ZM\xbb\x93\x16m\xa9\xd7\xe5\xe90[\xb9\xdct\x91\x9b\xa7\xc9\\\xdcxg3[W-\xfbE\xd2\x88\xdf\x86\xbc\x8ffjfwkp\xb4`\xb8\xb3r=9\xb0\xf3\xf0P\xa6\xc3+\x8aG\x83\xa1\xef1\x0c\xa3\xaa\xec(\xdd\x1bN\xd7\xd1\x04\xa7\xbb\x9f-\xf3\xf3\xa2^\xa9\xd5\xa07\x9e\xdbg\xd4B+\x131\xf9\xf6\xbc\x1d\x84\xe3\xd9\xd4\x18\x8b\x83\xc3;uu\x9e%\x8dH\xcc\x14\x05\x1c\xcc\xbdUEuvKI/\xe5\x9d\xbcX\xed\xa2\xf3~^\xae|m\x94/=FsM>\x9d]JA\xf2j\xf5\x188\xe6\xfex`\xc3,\x18\x0e\xdd\xd1\xce\x13\xfc\\r\xc7\x9b"M\x91\xad\xb1\x9e\xbbV\xe7Nrb*g\xbd\xc0a\xd0\xfb\xf8\xc4\x9f%\x8d\x08f\xb2\x02(\x06\xf2o\x84!\x8e\xce\x96\xeb\xeei\xdf\xd3\xae\xf5d1\xe6\xb5\xee:\xd8\xf5\\q\xef_\xc5\xd5\x811\'\x9c\xa5{n\xacI\x1d&\xc8\xd59\xef\tI\xe5\xb7\xbc\xeb\xd6\xa3,\xbf>\xea\xa9\x1d\x8b\xbb~\xe2\xae\xb9QlL\xd6\xb7\x94?\xa7\xef\x9d\xcd\'I#\xde\x17-\xcf\xc1\xa9\xf5\xad\x99\\rk\xab\xfduk\xadum\xaa^w\xc2X\xf4\x86\xd2,\x94\xca\xe9R@Ig\xdd\x1e\xfb\xbd\x9b\xbf\xdc\xf3K\xc9\xf4\xa3\x9ca\xd0\x94-h\x1f\x87\xa8\xf1%\xbe\xb1~\xff\x96qv\xa5\xa4\xa5{1z\x9d\xb9\xc3\xb6\xc5wJ\xdf\x84\xb9b#\x0f\x95b\xa7L[\xef\xdd\xfa\xcf\x92F\xc4fr\x12\xf6\x06x\xc4\xdel\xfd^h\xb7\xf4\xd366\xe3s\x90\x1f\xd3\xce\xb9\xe4\xd4u\xb0r76\x15\xe6\x82W\'\xdbmk\x9b\x88\xbbS\xc1q\xc9\x81J\xeb\xf9V`L|\x86\xdb\x13\xbd\xb07\xbb`\xef\x14k\xec\xdeZ\x95?\\\xeeV4=\xe2\xe2\x17i#\x92\xe0\x83\x13q\xe6\xce\xbdQ\xf1\x16\x03\xae\xeb\x17Vx\xdbx\x87T<\xce\x83\xb6k\xb2\xda\x90\x19\x0e7\x9a)\rR~q\xca\xab~w\xc2\r\xcd\xcaU\'\xd3\xdefaY\x86V\xae-\xfav\xb8\xed\x06\x9e\xd3U\x16\xc9\\\xda\xad\xed\xb5\xe8,|\xba\xee\xdd\xe4\x0f\xb5\'\x9e%\xaa\x05\xa3\xc8#\t\x0e\xd1o\x07q\xbd\xd7\xb7[K\x99\n#_\x16\xccI\xda\xb9\xb6\xdc\x96t4\x14\xaf\x9c\xf5]?_/\x07F\xa7u\xe3\xc6\xf9\xb8oxY<\x9akG\xda5g\xdd\xa43\xae\x96\xe6me\x97g-\xf0\xd2\xb8OM\xd4V\x7fh\x94|\xfdNa\x9dg)@\x82\x95\x0c\xf6\x11\xe8\xcd\xbe\x97/K-\xd5\xbc\xc3v_\x17\xb6\xa79\x83\xf5\x82\xe56\xee(;p\xc2~\x7f\xb8\xf0\xccr\xe9\x19\x97e=\xaev\x85GG\xf6\xb5\xdb>\xcfv\x8b\xf3\xc5\xe5/\xb6\x17\x8cVhlV+#\x1b\xf6\x94t7\x11\xae[W\xff\xd1JiX\xff\x91L\xa5 `\xe7I\xbd\x89S\x8d\x9eOu(d\xcfog\xf3vrs\x86\xc6\xbb.M"e\xbc7\xad$\xaf/\x1c\xaf\xef\xe7\xb1\xd8\xd6\xb8\x8em\xe6\xc5tt4\xa9\xeev\xdf\xefq\xad\xb1\x1f\xe6\xecm#\x04\x81\xc1\xad\xbd}\x18,-u\x99\x1d\x8bwfn\xcf\xd2\x7f\x04_\x88\x83T$\xd2\xf4\x9b\xd9,\xd5\xf9\xf1\xbaToQ\xf7\xe6D\xca\xfa\xd2n\xdd\xf2$Z\r\x86\xf2\xf9@i\xe8\xd8\xef\xf9\xd7QxY\xd2m\xfb\xa4%B\xc7\xea\xcc[3V\xde\xcf\xae\xcb\xe4\x92\\\x95h\xe9(\x16-\xfb\xf2a\x11\xf6\x8d\xc1\xb6\x93\xa3\xf6{\x97\xec\x93\xf4\x1f\xc1LI\xc0i\x07\xf5V\x0b*\x9b\xbb\xabkA\xf1\xf1\xfa:*f\xcbj\xeb\xef\xa2\x1e\xda\xf9\xd3\xcd\xb9\x1a\xa1\x04\xc9g\xc1\xdb\x1fM\xc4\xe9\x9b\xe86mw\xd7\xac\x98V-\xa9\x88\xb8n[.\xafka\xb6\x1dp\xb5\xacN\xc6\xc7\xf0\xb0\x19\xaa\x82 \x0c~\xa4m\xfd\xd7\xd6\x7f$N\x82\x12pR\xf4v\xebO\xedaU\xe4}e\xb9\xa0\x8e\x03q!\xaa\xd5iF_vl\xdc\xed\xcc\x0eV4\xe6\xadb\xb8\x95\xdaeG\xb72\xa1\xbf^\xad#\xd7\x19%A\xe7\xb6*\x1c!A\xfdn\x90\x1ba1\x9d\xf7\x87Jx\x99Hf\xb5\xbdH\xef\xac\xd8{\x98\xcc\xcf\x97\xe3\x91\xdbYG\x9f\x9e\x87=k\x161"\x7fB\xbb\x89\xb2\x17\xbd\x85\xbdK\xae\xd6u5\x1a\xb0\x13\xafS\xf8a\xd7\x7f\xa7\x10\xfa\xb3\xa4\x11\xc9\xd6g\xf1Y!\xbeU\xb8u\xe88U/\xd6\xee:\xbb\xdej\xd6\x11\'lx\xc1\x07\xf3\xadw\xe4\x16[u\xe0\x8e\'\xabB\xb1\xa6\x97\xa1\x95ZG\xce_P5\xbd\xa15\xd5m\t\x9d\xdd\xae\xd5^1FIO.T:\xb8\x99\x9be\x91\xea\xa6~`>V\xd5\xe6Y\xd2\x88d\xb10\xf8\xac\x15\xc57\x89\xfe\xaa\xbe2\xfe\xd6\xd9\xd1#}\x1dE\x99(\xf3&#\xf4\xbb\xa2\x87\xd3\x89\x01\xe7\xb5\x97\x87\xb3\x85.\xd6\x80\xe9\xc4F-w\'F>\x1a\x18s!]U\x97Q\xc4\x0b\xadu\x92\xb1\xc1\xa4\xe2\xf2\xed\x80q\xcbud\xcd\xce\xe8\x9d\x07\xe8\xb3\xa4\x11\xc1L\x8efEF\xe0\xde\xc8\xaf\x89y7\xda\xb0\xbc\xbb(ee\xa5[\xc5r&\x03\xa9\xc9\xa0\x1b\x8d\x82\xdbl\xbe<\xd7c:8\x1d\x8f\xca\x982\xaf\xf6bLo\xaf\x996\x08\xe3nK\xf3\x0f\xb7e>(E\xea$HcG\xb7\xaf\']I\xf3h\xb3}gM\xfaY\xd2\x88\xc4L^\xc0\x81\xf0\xdb\xa4\x86\xdb\xf5\x0fh\xa7\xa0K\xbeN\x8a\xbdEq\xc9U\xa8*\xad\xa5\xf8\x87At\x906\xf3xVU\xc9T\xdeei8\xca+\xadW2\xab\xdd\xb8-\xb0\xd3(\xdb\xaf\xf3a\x92^\xd5p\xab]\xfa(\xd4\xfac\x93\x92\xa5\xab\xf0\xce\x13\xeeY\xd2\x88p\x90\xe3m\xcfP\xdf\x11\xd3\x1b\x18\xeb\xd6\xe6\x1a\x8e\x9dX\xdc&*7\xde\xebG[\xb5\xe5Q\xbbwb\xe5d\xd2\xeft\x9c~Z\tV8t\'\xa2k\xad\xa4K\x1a_\xe7\xc9\xfa\x14\xc8\xa9\x9f*V\xdc\xbe*\xabs{}\x1c\x1csvs\x9a-o\xca\xeaE\xd2\x88Pz\xe7p\xbc\'\xbc\x95\xf3\xd4d\xf38\xbd\xde\xf0\xa9\x1c\xfb\x9b\xe1\xa1\xd8\xcd\xc6\'\xb1\xb5\xbf\r\xac.j\xc7\xe6V\x9f\xef\xbd[Y\xb8\xebI93L\xeb\xba\xd1\x98\x0b3\xb9X\xbd\xa3\xac,\xe5M\xbbep\xf5q\x91\x04\xf1(q\xdb!}\x8a\x97\xe5\xe5E\xca\x88w\xaf\x8c {ys\x8e\xa3so\x81l\xff\xdc\xbd\xf9\xf2n\xad\xedV\xa8\xaa\'\x95\x95j\xfd\xd9f\xb4s\x83\xcd\x98\x8d]g:\t\x90:\x0e\xba\xdc\xcd_\xcd\xbbvO\x8c\xfd<\xbc\x16\xa7\xed\xf4\xe4*\xfd\x93\xdb\xbb\xc53G\\\xde\x16]:\xf7\'\x1f\xab\x9c\xf1,e\xc4\xfb\xce\xc7\xc3\xce\xbe\xf1\xf9\x9b\xd9\xa4\xa0/6\xdae\xdd\xc5e\xc2.{\xeb8q\xcc\xd6%/\xab\xe9\xf9`\xf6\xae\xad\xe1\xd0\x8aZG\xa3\xf2\xdbGk\x99\x1c\xf4\x9b!8G\x9dm_\xf7\xe3\xe8 \x08\x13\xab\xd3.\xe5\x852O\xb5C9\x14.\x9b(}oY\xfaI\xc2\x88`%\x85c@\x9a\x7f\xeb\xad\xda#4\xa7\xdb\xdd\xf6M\xe6\xd8\xa1G\xb7\xa2E\xe6\x86n\x9515\xa5u\xe9\xb4(\xa6\x83\xe1X\xe1\x16s\x9f\xee\xd7\x82\xd0\xba\xb9\xde\xb0^\xe9;Q\x11\xc7\xcb\xae\x97h\x89\xcd\xef\xecv\xb4WXV\xd4\xaa\xc5\xa6\xbb\xd4\xdfy\xbe=K\x18\x116\xbe\x84\xb3\x03(N}k\xe6l1[P\xe5\xfc\xb21\xed!}+\xa8\xab?\x8d\xe5]>\xd4\xfb\x9d\x99\x15\\\xf8U\xb0\xd5\x85\xf5\xeaR\xef<\xc7\xb9\xf2)\xe4\xb1Z@\xa5\xe8\xa2\x1dxz\x1fw\xbc@\xd8Y\xd1q{\xa4\xc5\xb6\xc4\x8dP\xba\xe9\xbe\xf3|{\x960"\xd9\xf9xE\xf08ty\xa3\xe6\xd9\x8b\xed\x91\xe7dcwx\xa1\x99@\xbc\xec\xa3!/l\xb4b\xdf\xed\xb4\x07\x9b\x92\xe1\x06\xf1H\x88.\xf18\xcb\xfb(\x12lo\xccN\x0e\x8e%t[\xb1\xe4g\xad\xbd{q\xe9\x80\xda\xe7vj\xb6-fo\x9f\xab\xee\xc7\xcaj\x9e%\x8cH\x9c!#\x88\x12z\x1b\x07\xb7+\xb4\x95\x9dkW)\xe8\xfd\x89\x1b\xed\'\x9dr\xbb\x0b\x97\xbd\xcd.v2J\x8a\x8f\xad4\xef.\xec\xa8\xe8H\xdbM~jE\xe7\xec\\#.ho\x84"\xd7\x93\xe9e\xe0ey6\xcb[\x82\x14\xb8]\xb3@3eq\xcf\xf4\x7f*\x8cH*\x93\xff\x16F\xfce*\x9d\xff=R\x88O\x97\x12y3V\x1fW\xbf\xa5!\x86\xa4\xbf\xd0\x90>]\xda\xaf\xa1!m\xfe=?\xae\xb8`S\xab\xb4\xf1\xf7\xfc\xb8\xa2\x80M\xad\xd2\xc6\xdf\xf3\xe3\n\xfa55\xa4\x8d\xbf\xe7\xc7\x15\xe3khH\x9b\x7f\xcf\x8f+\xa4\xd7\xd4*m\xfc=\xff\x1dD}\xdc \xea\xe9\xb2o\r\ri\xf3\xef\xf9q\xc5\xe7\x9a"Cm\xfc=?\xaep\\S\x1b\xbf\xf1\xf7\xfc\xb8\xa2oM\ri\xe3\xef\xf9q\x05\xdb\x9a\n\xa2\x1a\x7f\xcf\x8f+\xb6\xd6\xd0\x906\xff\x9e\x1fW(\xad\xb1\xb8\xb4\xe9\xf7\xfc\xb8"gM\ri\xe3\xef\xf9q\x05\xca\x1a\xdb\xf8M\xbf\xe7\xc7\x15\x17kj\x956\xfe\x9e\xff\xd6=x\xfa{~\\\xad\xb5\xc6\xaa\xfaM\xbf\xe7\xc7\xd5IkjH\x1b\x7f\xcf\x8f\xabq\xd6\xd0\x906\xff\x9e\x1fW\x9f\xac\xa9!m\xfc=?\xae\xb6Xc\x17%M\xbf\xe7\xc7\xd5\x05k\xac\xb8\xd7\xf4{~\\M\xaf\x86\x86\xb4\xf9\xf7\xfc\xb8\xd2PMU\xa2\x1a\x7f\xcf\x8f+\xeb\xd4\xd4EI\xe3\xef\xf9q%\x99\x1a\xf3\xf8M\xbf\xe7\xc7\x95SjhH\x9b\x7f\xcf\x8f+\x85\xd4\xd0\x906\xff\x9e\x1fW\xc6\xa8\xa9 \xaa\xf1\xf7\xfc\xb8\x12DM\xb9\xa7\xc6\xdf\xf3\xe3\xca\x075\xb5\xf1\x1b\x7f\xcf\x8f+\xfd\xd3\xd4\x906\xfe\x9e\x1fW\xb6\xa7\xa9!m\xfc=?\xae\xe4Nc\xa1~\xd3\xef\xf9q\xe5r\x9aB\x9b4\xfe\x9e\x1fW\xea\xa6\xa9\x1c\xbf\xf1\xf7\xfc\xb825\x8d\x81!\x9b~\xcf\x8f+1\xd3\x18\xb0\xbc\xe9\xf7\xfcSB\x08_\x1d\xd8?\x15B\xf8\xe6,\xfa\xc8\xf20\xcd\x1b\xf6"y\x98\xe6\r{\x91W+\xcd\x1c\x87\xbd\xe4,_\xb5\xd1j\xc2\x8ds\xfd0kK\xf8(\x8b\x96\x87\xb8\\\xb13\x7f`\rO^w)\xb0\x08uZ\xc5\xdev\xab\xf9\xc2\xeb\x0c\xbb\xab\x9c\xd3\xc3^\'4\x8e#z\xb5\xed1\xcc\xf8\x9dk\xf6Y\xf20\xd8L\x1c\x96\xe1\xf4S\x12\xde\xb0\xc6\x0e\xf54\x9ev\x17\x12\xb5\xbc\x9c\xf4\x0b\xc7\xab\xf4\x14\xa9\xce\x88u\xbd\xd5\xe4\x92\xb4\xd2\x9e\x84\xfcV\xae1\x82\xd4\x1d\xab\xd1b\x15\xcc\xb4\xf3\xa4u\x9el\xbc\xd6i\xed\x0c\x8b\xce\xfex;\x9eoc\xaf\xda3\x83\xebd\xe0\xcf\xdb\xf2;]\xf2\xb3\xe4a\xc8\xd6\x94$\x91F\xc2\x1b\xc1\xb6\xbeR/\xcbx\xbb\xb2\xe6h\x9fJ\xf5\x85a=\xd1\xd2\xa9K\x9f\xbf\x8cy7Yw\x10c]\xb4\xbd=\xd9]ZC\xae\xb6OBw\xa8\xee\xfb5[\xf7\xe8\x81=/\x96\x15\xbf\x8c\x83KL\x8d\xb3\x9d\xc9\xda\xdd!\xc3\xa6\xefd\xfe\x7f\x96<\x0c1\x93\x02\xfeo\xe9\x8d\x99\x874\xf4v\x8e\xbf[3\xfeY\xefd\x9bZ\x1e\xec\xbdv\xd9\xdd\x1c\xf9\xe3\xa5\xba]\x17\xd3\xe3d9u\xd6\xba<\\8\xd1\xb4;v\x94]p\xd0%\xe7\x92\xc9\xf39]3\x0em\x8c\xce\xb5\xb40\x8f\xecj8\xa8\xa4\xedp\xff\xb1\xf8T\x9f%\x0f\x83G\x91\xe3\xb1c\x13\xe8\xb7$\xea\xbb\xbd\xdd\xba\xc8r\xe2Q-\xb5\x1fG7\xcb\xceUqwIDq@\x85\xc1\x91\xf7\x8d\x16s\xbb\x9d\x0e\xd7\xae6\xe8\x96\xfc\xa8\xd3\xd3\x19\xb6\xb0\x92\xe1i\xe3d\xc5~3v\xf2\xc9\xb6Dm\xff\x16\x08-\xcbW\xb7\xc5\xb0\xf3\xce\xd8\xe6Y\xf20d\xebK\x12\xcbJoc\x1bm3R+\x8e\xcfN\x9e98\x8b\xeb\x0b\xea\xf7\xbbY\xbe\xdb\x9fD\xc7\x8eq\x88\xc8o\xe6\xb1`\xaf\xc5\xb0\xa3\x18h-\xcb\x8c\xd9\xa37\xea\x8e\xa7\x03u\xb6U\xdc\xddbf\xf0\x86I\xc7\x81D)3\xf7Je\xd6\xe0\xfaN\x99\x88g\xc9\xc3\xdc\xf7\x04\x1e0\xf1\xad\xc2\x9f\xa8S\x05?g\xa6C\xa657\xd6\xfb\xcb|w\x9dK\xad\x98>\x8b\x88\xcf\xaf\x1b\x85?\x0e}\xde;\x8d\xdb\x93\xfaZl#V\xb6\xe5\xc1\xc0\xd0n\xd5\xa9\xc5\x8d\xdb\x81\xb7\xda\xfbh\xca\xacN\xe7\xe9\xea\x10\xca\x9d\xac\xb8\xe9\xee\xc7\n\xe1\x9e%\x0f\x03\x810\xdeZ\xa2Hso\x98F}V\x9f\xce\x92.~^\xcf\xdaG\x9d\xee|~\xd3\xf2Q`wc5\xa4\x94\xcb\xe4\xeav\xcfq\x1a\xb6\xa3\xdd\xaa\x87\xae\x8e\xd7K\xe7\xf6 \xa1\xb9\x90\xf1\xa6\x1b|Z\xf9q\xaf\xab\x06\x83\xcd\xfa\xdcEZ\xda\xe9]\xb4]\xaa\xbdS2\xe9I\xfa0`&\xc3K\xd8\x93\xbee\x17o-\xb5a\xd2\x19\xa5\xed}-\xdf\xd8\xbd*\xa4^2.\xd2\x96\xd9\xf6$\xde?*\'I\x89\xac\xac\xbd[nv\xecx\xbd\xbb\xd9Y\xa5f\x1d\x97\xe7\xeb\xd6\xe6Vd]\xaf#\x9bc\x871\xc7T\xf7l\xd8L\x8d\xcf\x91\xf4\x9d\xcaP\xcfR\x88\x81=!0,K\xa1\xb7\xbc\xdb}\x7f\xb0<\x1e\n78\xca\xd9\xec\xc8]\x8a\x8d\x97\xf4\x9c\xee\xda7\xda\xb5hK\xe2\xe9\xca;\x898\xa2\xe9\xf3U\x0e\x86\xe3Sy\x1d\x0f\xe7L5W\xa5\x14\x89\xd7j\x95\x1f"\xd9\xa1BN\n\x168:\xba]\x8b\xaa\xb3\x7f\'\xc9\xf0\xb3\x14b\xe0\x84\xa3E\x1c\x8f\xd3\xc2\x9b\x83\xdc\xdf\xb2I\xea;\xdd\xaa+\xcfO\xf6L5\xc5<\xde\xb7V\xc7A6\x88\xfd\xd5h}\xa4\xad\xc2\x99O\x14\xfc\xc2\x9d\x1b\x12\xce\'\xef:Z\xf1\x83u]\x0c\x0f\xed\xac\xdb\x99\xcd\xe7V\xfb\xd0\x13WkaqY\xedh\xcb\xd8\x0f\xdey\xc2=K!\x06f\x93F<%~G\xf4CP\xa7^k~\x198\xab[\xefSGI\xdd[L\xd0\x88g\xcd\x05\xd5z\xc0u\x04\xdbP\xb5\x99\x1d<\xb2!\x85\x81\xd0\xf51\x10\t\xe7\xc5\x1d\xee]B\xcc\xc7\xa9\x0f#(\x82\xc3Oan{f\xbf\x95-\xd2s\xedJ\xad\xe3j/\xee\x8b$\xe4,Rh/\x1b_7\xc9\xf3a2H\x9cr\x0e{\x144\xf7\xb9\x8d\xbb\x07\x0e\r\x1ds\xb9\x99C\xa2\x87\xb1G\x7f\x83\x9d\xb7\x1bh\xda\xed\x87\x83\rmp\xdd\xfd\xde\x0e\xf7\xf7\x16b>\xae\xfa$\n!\x00\xf0\xb4&Fx\x03m\xb5\x93\x97\x87\xfb\\\x06\xfb\x9by\xdb\xb7\xe11\xf3\n\xf9\x0c\x9c\xd1\x9cA\xe6\xab\xed\x07\xf0]\x04}\x81\xa1n\xba\xdfm0\x96\x88&\xa5\x80I\xa2\xb4*\xbfr*R\x01\xb18\xa5\xda*o\xcc\xa6\x87_]\x13\xef\x12b\x1eK\x1fZ\xb6\xe1%Ix:\xf5\x9d\xbb\x1b\xa8=zm\xdcXo#v(f;\xb5=\x89!S\xa7@\x0c\xae\x82c\x059\x857\xd7\xd1\xb7\xb5\x93o\x87M\xcb\x03K\x1b\x06\xee\xeeIqk9\xc7\xc0\xf3\x8e[\xeb\x88\x8dEDU\xc5NA\xf8\xe8\xc5\xe4\xe6]B\xccG\xc2\xbf$\x83\x10\xf9\x0c\xe1l\\\xcdQ5P%\xda$\xd7z\xddQ\x85LJ\xa2\x00\x05N\xb62:\xbdT\xaa\xe7\xb6\xd88\xb7\xb2\x0f<\xbab\xb4\x8dm\xef\xf62QG3\x0e\x85\xfd\xb8\x0b1\xc8g\xaeG\xaau\xd9St.\x08\xb99\x9c\xa9\xcf\x11b\x1ea\x12Kv\x03\x00\xf0S\x98\x18\x89\x10\'\xb9?n\xe5\xb1\xdc\xe0.\xc8`dm\x9e88\xc7\x95-\x04\x1c\x10\xa5\x17n\x83\xc0\x9d\xe3t\xde\xb5\x11\xcd_\x96\x85\x94u\xc7X9\xe8\xe5!g:"\xe5\xec\xd8\x9fC\xd0\x1c\xeb\x80\\67O~\x95\xbf{\x97\x10\xf3\xed\xd4\x7f\xbc=\xf8\xfcfJH\xfb\'\xe4 l\xb4c\xbbo\x82+\xa1\x12v\xe3\x83^\x9f\xe1\xe6\xdd\xea\xcb\n\x8f\x1c\xd6\xb7\x07\xcc\xaa\x0e\xdd\xbdMu\x19\x90\xd8\x08\n\xa8{t\x91\xa6\x9dd\xef\xcb\xa6\xf2\xc5\x10\xb3\r"\xd76\x0e\x80e\xed\xed\xc5[\xea\xbb\x88\x98G\x980A<\xde9}\xda\xc8\xe30\xa8a\xaf\xa3\x99V\xde\xe3\r\xef\xa7(\xa8j\x9b\x93\xc5\xed\x9c\x93\xc0\xf3\'\x1a/\xc9\xf4l;\x08\x15\x05@\x02\xfbis\xdd\xba\xee\x04\x0e\x9av\xf6\x0f\xf9\xdc\xb4\xf45\xc9\xe5C3\xfb(\xa0\xc2;\x10\xd8}wm\xfe\xbd\x89\x98oK\x7f\xb92\x80O\xfb8%e\x14L\x8f\xe1\xd6B\xc5\x89\x8dQ%S\xa1\x00m\xb9,\xdb\xdd=\xa0[n\xe17\xfc\x1e\x07]\x1c5H{\xcb*h\xbb\xa5\xac\xca\xdc\x11\x9c0\xf9\xdc]\xea\xa7\xf9\n6\xa4\xa4\xb2\xa6\xc0Rpe\xe9 \xfc"\x02\xfd."\xe6\xe3\x99\xedr= \x88\xe7\xe3\n\xb1\xfc^\xb4/m\x12\x98\xc2\xecB\xea=\xe46[\xb1\xcc\xcfv\x90[\x85\x85Ou\xcf\xc9\x84\xbd\xe4\xfb\xa7\xaa\x8d\x0c\xa6\x01\xe7\xd0\xe1w\x81\x08\xd9\xa06P\x88B\xba\x8a\x1b\xa4,\xa8\x86\x07,\x89\n\xe8v,^\\\xf9\xef"b>.\xa9\x8f\xe7\xa1\xf0\xf3\xfb\x0c\x98c\r\xfb;\xe1\x85\x1b\xab\x1f1$\xc23\x1eV\xb1\xea\xc4]4\x92\xe1:\xc5\xccwb:\xca~\xedjW\x17\xd5\xf7yw\x82B\x0b\xdd2\x1d\xda\x05\xb4\xbd\xe7\xc4\xd9\x02\x8d\xbd\x1bd\x93\x9b\xfa\xfc>n\x9a\xf0E\xb7\xed]D\xcc\xb7\x14n\xf9n\x14~\n\xd3\xc9\xb6}\x8d\xd0Ty\xf6\\\x82\xb0fR\xa97w\x91\x19\xec\xc0?\xa9\xd9\xcej\xb6V\x05\x1f\x01\xdb\xb5\xc6\xbb|.\xf70\xdaCw\x15\x13C\xd7\x94Bh\x19\xe3P\xd8\x1b\x96\x9b\xa3\xe3q\xa3j\xfca\x13\xe9\xc8\xd7zp\xf3."\xe6\xd1\x8b\x04\x82-\x19?\xfatL\xd8\x82w.7s\x0c\x1d\n\x7f\xd3\x1f\n*\t\xba\xa4\xe6\x86\xc2BQ\xca\xba\xcd\x82\x9a\xd3\xbb\xbe\xa0\x96\x9b\xf8!o\x9c\x96P\r\xb4\xcc\xf8\xeb\xa8\xef\x85\xe5\x9a\xd0\xd5[\x16!\x1cC\xe1\xda\x90m\x8fW|\'\xf1\xf6\xb7\xfd\xf3_\x121\x1f\t\xfcJ\xc4\xfc\xe9\xcf\xaa\xafD\xccg\xfc\x8d\xd4Z\xdd|%bV"f%bV"f%bV"\xe6\xeb\xb6s%bV"\xe6\xbfQ\x12\xb5\x121+\x11\xb3\x121+\x11\xb3\x121+\x11\xb3\x121_\xb8\x9d+\x11\xb3\x121+\x11\xb3\x121_\x99^Y\x89\x98\x95\x88Y\x89\x98\x95\x88\xf9\xb2\xed\\\x89\x98\x95\x88Y\x89\x98\x95\x88\xf9\xca\xf4\xcaJ\xc4\xacD\xccJ\xc4\xacD\xcc\x97m\xe7J\xc4\xacD\xccJ\xc4\xacD\xccW\xa6WV"f%bV"f%b\xbel;W"f%bV"f%b\xbe2\xbd\xb2\x121+\x11\xb3\x121\x9fJ\xc4\xfc\xd3/\xf9\x97\x18\xc2o\xb6\xf7\xafL\xc4\xfc\xf8\xc0>\x89\x88\xf9\xf1\x81}\x12\x11\xf3\xe3\x03\xfb$"\xe6\xb5\xc0\xfe\x1dp\xe4\x93\x88\x98\x1f?b\x9fD\xc4\xfc\x05\x9b\xc7J\xc4\xfc\x10"\x06\xfb\x7f}\xfe\xc7D\x0c\x82\xb2$\x8f\xa3\x1cBb\x14M!4\xc8\x138\x06\x90\x1cNC,\xb94\x91EX\x9eBQ\x9a&\x19\x14\x81!\x0e\xe7\x11\x06\xa7\x08\x80\x82\xa9\xe5+\xff\x82?x\x14q|TA\xe1A\x04&a\x02|\x94\x00\x04\x08\x06{T}\x02a\x04|\xd4\xb6\x04@\x0c\xe6Y\x88\xa1 \x12\xe0`\x0e\xe1p\x8a\x810\x18\xa3 \x9a\xff\x91D\xcc?J\xca\xfe\xf4;\x05\x18\x10\xf0g\x1c{H\x1c\x04\x86\xff\x91\x11\xf3\xf5}\x9d\xdf1bH\x16\xe28\x8c\xc0X\x1afP\x90\x05Q\x92z\x14\xec@)\x8cX\xfa\x1e\xa4\xc0e\xde.\xaf\xc3\xb3$E#$\x89\xe0\x14\x8c\xf0$\x01\x90,H\x918\xf3\xa8\xfe\xb3\x1a1?\xd0\x88\x81P|\x19N\x84fI\x10d\xe8e\xa4\xc1e1\x00\x10\xb0\x0c\x16\xb5\xac\x1e\x8a\xe6\x97\x17c\xa9\xa5\xb3a\x0e\xa4P\x96e@\x1c\xa2\x11\x9e\xa5(v\xf9\xfd\xd8O\x7fs#\xe6O\x93\x1eo3b\x1e\r\xf9C#\xe6\xeb\xaf\xf3O5b`\xec\xbb\xba\x00\xc0\xe5Y\x8dO&\xcdI2\x1cl\x8fr\xdc\x90\xa4\t\x1e\x8bq\x94\xce\xb4\x02\n\x87%\tk\xfc\xdb\xc9\xdf\xd6(\xdd\x13\xfe\xd5o\x8a\xdd\xd1\xe6\xce\xb2\xcfya\xd4\xc4\xfa\xa0U\xd7\x8b>\xfa*\xe5\xa5\x1d\xdf\xcaE\xa2\x07c9\xd9\x9c\xd4\x8d7+\x19\xc9\x93\x9e\xc2\xceU\xde)ijV\xf3,\xdcv,wqz\xf3X!q\xb1\x19o\xed\x12\xe1\x8bU\x15\xdfE\xc4,1\x92\x10\x8a\x90\xc4\x92\x92\xfc6\xcc\x12\xe3\x91\xeb\xd1\xba6&\\\x1e\xd9\x9e\xbb\x17e\xa5g*\x97\n\xceIB\xec\xbd\xd9\xf1;\xcf\xc7\xaa\xd6\x91\xf6\xcd\xb6\xbc\xf8\x85\xbe\xc9\xe7\xf0j/\xf3V\xda\xea\'\x83\x93\xf0\xf2ppp4\x91\xe5\xfb\x0e\x965x\xf3\xb5j\xee\xbf\x8b\x88y\xcc\x14\x00\xc7a\xf8\xb9@]3\x8b\x19L0)a\xc6\xb1\xdb*\xa3V\x9f\x98\x13x\xb5tF\xa4\x14\x1e;p\xf2)S\xe6\xa2\x98\x80Tk/\xfb"\xc3\x8eF\x82r\xb1\x8e[%\xc1\xd9\xdc\xb6\xb1\xb1(:\xd3\xcbE\xc7\x10\xb7j\x95\x12\xa0\xf9jy\xf17\t1\x8f(!\x18},\x9e\xa7\xdd\r\xaa\xad\x8b\xb6\xeb\x07\xc3\xec=\xef\xae\xa8\x19\xd79\x0e\xe4\x9aG\x8b\xb7\xc7[\x90\xa3\xeam\xefUd\x8c\xf9\x1at\x8e\x02\xbeE\xeb\xa2\xbf\n\xbb\xd4\xc8\xf7~\xefXFz\xf2n\xb4|\x14q/\xe8\xe4\x10\xf6O\xb9\xffb\xdd\xedw\t1\x8f\xdd\rX6p\x08$\x9ej\x8e\x8d\x88\xc3\xdc\x85\xe1\xc4\xec\xe5\x93\xd2\x11(\xb2\xf5+\x05\xa5\x99\x99\xb7\xecY\xdb\xc61\xd3\r\x07\xc9t\xce\xfbm\xd1n\xc5\xdbm{\xd8F9p2\xf1|\xdbk\xa0p<\xb7%\xdai\x93R\xef\xa6\x88PhS\xcd\xb9\x17\xed\x94w\x111\x8f0q\x0c\xc2\x00\x08~\xb2SD|\x93Y\x15\x1e\x12\x8d+5\x03e\xa8D\xa0\xd0\x17\x1e\xdd\xc1\xed5\x9f6\x80\xe8\x86\xfb\x16\xd5\xaeGG.!\xffv(5\xa1\x9dzy_\x89t0\x80\xd0\xfe\x84U\x92\x9b\x0fs,\xdc\x13,\xe3\x08:\xb9\xeb/\x96\x1b|\x17\x11\xf3\xb14\x97q_.\x99Oz\x93\x87\x86\xe2\x81\x18\xe2\x06\x04\xafB\x19\r|\xb9U\xeakX\xefZ\x7fh\x19w\xeam^\xbcIx\xdaJ\' \x8c\x9cMC\xdc1\x9bnE\xbb-\xe2\xca\xce\x84\xee\x9e\xa2\xd8|\xb3\xbd\xfd\xc03\xc0\x88U\xea\x1dyu4\xdfD\xc4|\x84I\x82\x00IBOU\xd4+8\xdeh\x08Jw\n/\xdf.\xf4\xb1\xe3\xd9\x93\xd5\r\xd7\x08\xf6\x96\x8c\xcd\xd1\xb5\xec0L\xbb\xd8\x8cy\xd7\xc8\xcc\xe8\x8e\x1b.\xae\x91\x83RP\xa2L\xa4\xe8\x91\xb3&\x9cH\xaa\xeb2\xfa\xfd9s\xaa\xa1?\x18/\xd6:~\x17\x11\xf3\x98\xb4 A\xa2\xcb\\x:\x95\x15;,&\xfc\xdc2\x84\x8c\xa7\x9b\xe0L{\xc3\x04\x15C@\xb2G\xd5\xe2\x04H\xf6\x18\xfeZ\x9a\xf2\xc0]+\xc0:X\xf4\xae\xe1\x1ci\xa3\x86\xc9\x85\xa5j\xd0\xd4qU\x8d\xef{"\xabY26\x0e\x82D\xbb\xda\xf7J\x81\xfe\xbd\x89\x98\xc7dY\xae_\x08\xfc;l\xd2\xed2\xdbvE4a`\xecl\x12\xb6\xe0\x1eQ\xceN5N\x16\x10r\xa9\x06\\\x03\xab\xedOj\xc9\x17\x97$\x05;2\xbcv@t\x86L\xc5\xb6\x05\xc3\xcb\x8aF@\x95\x1cK\xeeNIv\xd6H6\xf7c\x99\xb1\xafM\x96w\x111\x8f\xc9\xb2\\-A\x10\x03\x9f\xbek\x92\xf6x\xd0\xefI\x7f\xb5\xe8\xfb\xf1,\x10M\xdd~\xef\xd2\xf1\xf7&b\x1e\x890\x88<8\x0b\xfc\xf9\xd4\x8f\x99\x9d\x10\xd6\xe7\xc8\xbe\xdbt\x1a\xf5w6\xe8\x04)\xaayO\xd8;#\xb95!\xee\x84\x11\x06q\x19LaC\x0e\xd82!\x80\xd3\xa5\x9am\x86O\xba+\x92]H\ti\x15a\x98z\xb2\x87n\x85\xa9cziB\xce!\xd5\xe02l8\xe3f\xbb\x1a\xd2C,h\xa8h\x0f\x15\xbc8\x1e\xbb\\\x1dNz \x82\xf0\x92\xf9\x82\xa0A\xd3\xeal\xa2\x17%9\xc1;\xc1a\xe46\x90O~\xfa\xaa\xf5\xf5&"\xe6#\xb9Aa\x12B\x9ew8\xa0\xce+L\xd4\xe7\n\r\xdc\xf0&gG\xeca{P\xe0\xc1\x1e\xf0\xec\xaeC\xe0\xd5#:\xc6\xa5\x8e\x8e\xb8\xb3&\xcf\xf7\x03\x191n[\xefz\x10v\x9e)\xcb\xe0\x99\x96\xf7\xd9\xd9l\xee\x9d)\x9e\xe5\xb3\x07\x83\xf1\xd7\xaa\xa2\xfe."\xe6\xd1\x8bK&\r=\xdbP\xee\xb9j\xb0%\xc9\x13-\xbd\xd2%= \xfaa&.@\x9e\x9bU\x83o\xb6\xd8T\xf6\x81\x16w\x8d6zjR\x03\xbc\xa6\x04bL\xdd\xaej\x1b\x9a\x978:\xfaaO\x1cN\xfcN\xcfi\xc2\x04B\x1c\xd8\x83\xd0\x8b\x0b\xff]@\xccG\x82\x08\x12\xf0\xe3\xc1\xea\xd3U\x9c\xa1\x0c\xe7D\xf7\x92\xb7\x17O\xa0\x83\xe8N\x7f\xba\xcf\x13\xd4\xbb\xe6\xb9V\xa4\xbd\x9e\x97G\x9c\xcd\x19r\xce.]\xa7\x19\xb2"\xda\x95\xefx\xb9\t\x03\x8a\x1f\xce\x9eO\x19\x1cU\x11qd\xf2\x8d\xa8&\x99\xd4\x97/\xae\x88w\x011\x1fg>\x82!\x04\xf8\xfc\x04\x0e\xb3\x88\xac\x84\x08J?\xcf\x15\xb4\xdc}Nb\x00\xb4\x10\x03V{^\xd5H\xc9\xc3\x0f\x8e\x9d\'\xf6\x0eo!\xa0\x98);:\x0bW.\xa2\xefA\xc8\x8b>\xb8\x91\xc5\x1d\xe4#:\xaf^\xaaTd\x06W5$\xff\xf4\xb5V\xc4\xbb\x80\x98\x8f\x9b>\xb1\xdcLP\xe4\xe9\x948\x95r2Sc\x16j\xb1\x86\xd1\x81u\\ng\xa0\xa8*\x97\x99\xc6u\x03\x94\t5g\x03N8\x99\xad`\xa1V\xae{0\x90\x9aQ\x89\x1d\x14\x19\xd4\x0c\xd2:\xe9\xfd\x19=\x83\xfb\xa6@\x14\xa5\xf6\xd2\xdc\xed\xe5\x17\xf3\xe0w\x011\x1fy0\xb6d\x88\xc4\xf3\xe3\xa9\x88\x99J\xb2\xe1\x81\x99+\xef]\x14g\xa0I\x0c*\xcd\xfb\xea\x1c\x84 5\xd5\xbaM\xa2\x98p\x05a/\x8d\t\xba\xdak\xa6\x02M\xc0\x11\x9c\xbb\xb6\xdb\xc9\x03J\x8f\xea\xbd\x9d\x03\xa1\xe2i\xd8\x1f\x1c\xaa\x85i\xf1\x93\x80\x98%\xccG\x16\rc\xd83\x9a\x84i\x82z\xc7\x10\xd4\x1c\xc4\xf2\xdc\x96\xad\x15\xab\x89\x96\xa6\xf0\x15c\xce\x12a\x0c,\xdb\x14\x97\xfd\x11\xa9\xc1*\xde\xe0\x80\xbb\xdc\xfd\x0ec\x89\xd5\x1c\xde\xa7fl\xe9\xdb\xcat)l\x9c\xc8{\xeb\x93\x906\xd4\xeaa\xff\xaa\x9c\xf2& \xe6\xe30\x84P\x08\x84\x81\xa7\xcb\x1b\xc12\xc2^\x89j&\x88\xad\xc4\x01\x00\xdf<\xea\x88\xa2\xda\xadU\xd6\xee\x06\xc9\xf7\xc0\xfe\xd4\xc5C\xcaR\xd5F\xb60\xec\x88tA#J\xd2\r\x14\xc7\x8b\x80B|8\x80\xe2\x8eB\xcby\x00<\x9d\xf9P\xbee\xf0\xedXv\x92\xd8v\xdd]!k\xcbG\x06&\xcfv\xca\xf10\x17Y\x92l9\xa7\xb17\x8c\xefC\xe0\x01\x8a`/P\x19M%\x11)KNmzQlC\xf7\x84\xcc\x93N0:E\xb4nn\x14\xefE\xda\xf7]@\xccc4\x97\x19\x8b\xe1\xe03\xd4\xdc)\xdc\xae5.3>\xf3zJ\xaa\xfb|{\x14\x12\xaenI\x99\xf6L\xa5\xd1QI\xe7C\xf5T\xee\xfa\x92N8\xfeP\xeb\xfaib\xc0K\xc4]\xf0\xf4>\xcf~\xb11\x8a\xaa7\xb5:?`\x93\xcf\xed\xb7r\xf6\xb5\x9e\xbf\xbf\x0b\x88\xf9\xb6\xf4A\x80\x84\x9e{\xd1\xa8\xd2\x9d\x12\xcf"\xe0\x1c\xf7\x00P\x17\xc0\x86>\t\xc4\xceL\x88\xe06\x83.\x91l\xb6,\xe38Bt$\x9d\x9a\x94\xb7e\xdf\xb2:\xb5\x85\xc9\xfaL\xef4\xea\xcc\xc0[{\xdf\\\xa5\xf4x\xe0\x98x\x9cI\xaa\xd9\xbfz\xd3\x7f\x93\x10\xf3\xd8\xc8\x1f\x13\x8c@\x9e\xae\xa8\xa4g\xf3\xd7\xa0\xeeO\xed\xae\x8b\xe5\xf4\xa6WB7\x83bfC\x02\xcaE\xfahu\xbe\xb9,\xf6\xa2\xc3\x87DF\x04\t\xc4D\xe7"\xc8\xf2\xee\x92B\x1b\xfb\xcc\x90\x97+\xd5\xf6\xbd\xc75b\nx\xdb\xa0e\xce\xfe\x8b\xc7\xd5\xbb\x80\x98\xc7\xca_\xd6\xc3\x92\xeb>C_\xfb<\xe2\x8b[|\xb3cA\x1a6\xd7>\xa36\xc6E\x98\xf1\x93\xc7\x94 \x8bP\xe4\xb2\xeb\x90\x97@\x11\x99Z\xf1\rA \xa8\xc0\xc6\x8f=b\x80\xfd@c\x84e\x9dD9\x9d\xea\r\xc5\x1d\x9a\xe8\x0c\x1d\xe4\x1c&^|\x94\xf9. \xe6\xdb\xbb)8\xb4\xdcS\x9f\xf6\xf1#\x91\xa7\xf7\\\xd4\xd8i\x90b\x9cv\x8c\r\xb9\xdc\xa4\xb7\xb4D&\x89Kb\xf3\xd0\xde\xc9\xd9s\xafxQ\x03\xe3.\xa7K>\xb0\x0e\xc1\xb6\xc0\xd2M !\x1eh.Wd\x96\xef\x02\xf2 YL\x95\xdf\x1e\xc1\xae@\xcc\x9f\xfe\xa4\xfa\x7f\x0e\x10\xf37\xa27V\xcdd\xed\xd2\xaf\xdf\xa5+\x10\xb3\x021+\x10\xb3\x021+\x10\xb3\x021+\x10\xf3\x85\xdb\xb9\x021+\x10\xb3\x021+\x10\xf3\x95\xe1\x95\x15\x88Y\x81\x98\x15\x88Y\x81\x98/\xdb\xce\x15\x88Y\x81\x98\x15\x88Y\x81\x98\xaf\x0c\xaf\xac@\xcc\n\xc4\xac@\xcc\n\xc4|\xd9v\xae@\xcc\n\xc4\xac@\xcc\n\xc4|exe\x05bV f\x05bV \xe6\xcb\xb6s\x05bV f\x05bV \xe6+\xc3++\x10\xb3\x021+\x10\xb3\x021_\xb6\x9d+\x10\xb3\x021+\x10\xb3\x021_\x19^Y\x81\x98\x15\x88y\x85B\xf8\xa7q\xfb\x97\x14\xc2o\x9a\xf4\x95\x81\x98\x1f\x1f\xd8\'\x011?>\xb0O\x02b~|`\x9f\x04\xc4\xbc\x16\xd8\xbf\xc3\x8d|\x12\x10\xf3\xe3G\xec\x93\x80\x98\xbf`\xf3X\x81\x98\x1f\x02\xc4\xe0\xff\xaf\xcf\xff\x18\x88\x81\x11\x1a\xe5\x19\x08zP\x16\x10\t\xd0\x1c\x8e0\x18\x02\xd2\x10\xc5\x00\x04\x81\xc1\x04\xc5"\x10\xce\xf0<\xc1c0\x86\xd1<\xc0\x83\xec\xa3\xba\x01\xf4\xa8\xd7\x06?\xea\x95~\xdf>\xa09\x0c\xe2x\x02\x03\x19\x82\x00a\x06\xc1)\x02Aa\x92\xe1Y\x14\xc3H\x10\xa5I\x98\x83\x91\xc77\xa0\xcc\xf2\x0b\t\x80\xa7@\x02` \x0e\x02(\x9c\xa7\xa8\x1f\t\xc4\xfc\xa30\xc9O\xbfW\x80\x81\xf8\x19#p\x12\x81~\xa9\xb4\xf8= \xe6\xeb\xeb:\xbf\x03\xc4\xf0\x18@P \x80\xc3\x0f\xc6\x84gp\x14\x83@\x1a\x87\x971\x001\x12")\x9a\x87\xe0e\xa4\x80\xe5\xb5\x97\xd0 \x0e\xa3\xb0e\xecY\x94\xe6A\x8c\xe6\xe8\x07g\xf2k \x06\'Hz\x89\x97G\x00\x1a\x84ybym\x08\xa6a\x96\xc5\x01\x00\xe5\x00\x08Ybf\t\x00\xa1yj\t\x03a8\x1aA`\x94\x82\x18~\xe9\x03\nb\xe1\x9f~5\xc5W \xe61>\xc42\xab\x18\x1eC1\x16\xc0\x90G\xb1p\x0c\xe3`\x8a\x039\xea\xa3\n\r\x01\xf2 \x003<\x8e\x90\x04\x8cr8@@\xcbhA\x1c\x0b\xe2,Gr\x7f{ \xe6O\xb3%O@\xccOL\x8b\xd0L\xdb\xd3L\xa6M.D\xde\x03\x9b\xbc\xab\xb5\x0e\xb8\xf6\xd0\x06U8F9\x9fy\x8e\x95j{T\x0f\xa1\x88\xf3\xec\xb9\xf4 \x1e\xf0,\x19`\xb2a\n!p\xd2l\x1d\xf5\x1c\x1a\x8a\xd8S\xf7\xf8\xaca\\\xeb\xa9~\xf7\xaa\x88-\x90\xc8\xde!\x81#_\xa3*\xb9\xbb\xb6\x8e\xf8\xb0^\xc6\xd5\t\xe0r\xf9\xe3g\x1fw\xa0\x00\xa6\xcb0\xe7\xf3\xd8>]\xa3z\x07\xfa\x10\x8f\xfau{w\x0f!\xe4\xd5;\xd4\x10\x974X\x00o\xa1\xe85\x1a\xb4|\x9f\xe8"\xda\xb2b\xa2\xb4\xb1\x98\xe4\x04~\x9c\x90\xb3$i\x07\xea\xaa\xb1\x1f\xff\xeem\xaa\xdd1\x85\xab\x8872\xffv\x82j\xa3\x0b\xcbh(Z\xd3\x12_\xe9\xc2\xd4M?x\xdd\x12\xd7.\xb2e\xd5s\x8a\xbdtmR\xaa\x03\x92=\xbe\xe3@\xffH\xf25jf%-\xf8\xa6!n\x81\xdcuk\xffpf\xf3\x9bY3j{\x8a{.\xd9)T\x92\xd0\x02\xc5\xedi\x9a\xa5\xaa\xe4\xea\xef\x94@\xe4[\xdc\xe3\x15\xbb\xd1\xb8\x93\x13\x9as\xe8:\xa3s22\x9b\x90\x8e\xbb^\xb2\xf7\xe0M\xb7$*\x8d\xac\xd6fn\x02\xef\xc6\xdd@\xfaJ\x9a\xef\xfdm\x9a\x8b;.4\xf7\x0cG\xde5nS\\\xd1\xd0a\x8f#\xf1?\x7f\xfaw)\x9f\xc7\xe4\xfeC\xca\xe7\xeb\xef\xc8\x9fJ\xf9 \xd0\xf7\x91\x9b\x86\xa0\\\x90\xe6\xb2\xb4\xc9\xb0\xdd\xc6\x93YC\x86\xa3\xf3|r\x8b\x8d\xd1\xb7c\xb8\t\xf1~3\x10\xc3\x99[\xb2\xcb\xc8\x16\xad\xe6"\x8c\xbeO\xdb\r\xafi\x85\x85\xb0\xd8=>\xee\xda\xad\xacR\xf4\x1dp\x19\x19\xd8\xba/\x16\x84~\x9b\xe5C\xfc\x8c\xe3 \x80\x018\xf4T\x86j7\x1f(\xb8)\xec\x0ctn!\xb7\xf3\x92\xbcu6\x95\x08\xdf\x89\x8bE\xe7Z\x01\xcc\xe7(\xbaf#\xa0\xb3\xb2\x97h\x046&\xf0\xbe\xd5e\x11,\xe6\xf04\xd7L5hh\xbd\x0fCi\x0eG\xa2\xb2\xa3c\x80\xbc\x8az\xbc\xcb\xf2!~^N*\x94x\x1a\xca\xf2$\x1b\x06\x9f\x92]\xd7\x99\xa2tr\x8eC\x00B\xe6\xdc\xc63\xcd\xfb5e8\n{\xc2\xf3`:\x0e\x87\xcd\xf1\x82\xc4\x16|\'yX2\x82\xa0\x89f\x0e\x0c\xdc\x1eq\xc8~8:\xecl\x03\'t\x9b\xe7u\xf5\xbd"{?\xda\xf2Y\x86\x92 \xc8G\xe5\xb4\xa7\x02\x98\\\xbc\x91\tD\x19D\xe2\x1a[\x06#\x14f\xdba\xad\\7\xdb\xa4ip\x98\x8c\x06\xddU\x8a)$Xg\xe6\xf8\xbe9\x1d\xa8\x98\x04o\xcd\xde\x9d\x95\xe8t\x9e\xb1\xadL9\xbb\xd1\x17t\xec|\x0e\x0em\xefn\xdb\xef\x95L\xfc\x9b[>\x8f^D\xb0\xe5\xae\xf0\\Q\xec^^\xc5\x98Wz$\x92\xec\xbaF\x1dx\xf2\\\xce>\x0e\x02\xe9\x03\x02>%7\xd7\xd9\x0e\xe7)\x1a\xfctVx"\r.\xe5\x8e\x1a\xb7\x1e\x12+\x99\x0f\x9c\xef\xbdq\xa6p\x05O\xf4C,\x14\xa3\xc1\xc0\xdd\x89{\xb1\xf0\xe4\xdb0\x9f%L\x04\x07@\x88\x00\x9e&\x0b,\x10\xb5\xbaQm]9\xc7\x0e{\x88/\x8c\xbb\xd9m\xf8H\xcb\xcb\x108g\xdau\xc3H;\x19\xa9\xb0k\x13\xf57t\x1c\xf3T\xb8g\x8c)\x855v\x9a\xa6\xb1\xe3A\xef\xce\x15\xe0\xe8\xa4\xa5w\x88\x88=\xe8\x0e\xdf+?\xf7\xa31\x9f%L\x08{\x94KE\x9f\xca\x88\xe2*\xa8\x80\xc8\xa9\xf3T\xa9\xbax\x97\xe5bU+1{/D>b-\xa9\xda\x0bhi\xcb\x8d\xd8\x08\xc5cc\x1f\x04\x91M\x86\x9d0\x81J\xcf\x061\xac\xd3:\x82\x80\xae\xcc\t\xc6m?\x00\xf3\xa6\x87\xf2\x1cz\x91*{\x1b\xe6\xf3\x9b[\xd8\xaf\xc3D\xeb\xc1\xba\xddwq\x1eV\xa6\x9f\xb2\xee6\xba7>\xdc\x97W-8\xab\x82VW\xa3W\xa2,\xad\x06\'\xc3\xf6r\x1c\xf5+g\x04\xef\xc5\x99\x8d\xa8\x11\xbdV\x9d\xd7\xc3\x99\x8f+\xcd\x91\xecR\xef\x96j6\x187/\xeepo\xc3|\x1e\xa3\t=J\x04?\xd7\xbbGL`\xe0\xee\xb5\xd3d\xc3\x90(\n\xe0l\xf59\xdcd\xb9\xe5*ej\x14`#\x8d\x80z\x9a\xe2\\\x99%\xa2`\x9d0q6p\xe3\x1b\x99\xee\xdd\xae\xb0\x9d\xf7\x1d\xae\xee:\xa7\x88\xf3\xac\xd8\x05\xdb\x13\xe3\x12\xe0\x8bk\xf3m\x98\xcfcm\x92K\xa2\x84\x10OE\xcc\xe3\xbdm\x8d\xce\xd5\xbe\x81\x14M\xc0\x9bR\x93D\xe68l\x15\xda\x8c\xf6]yU\x14d\xc7\xca6\xa8\x1ed\xd3le\xfc&o\x88\xd88\x19\rl\xba\x0c\xa4D\xb2\xae\xe7,\xab\x83jK\xd3\x82\x17\x96\xf9\xd63\xe8\x17\xcb\x99\xbe\r\xf3Y&-N>\xeef\xcf\xb5\xdaI\xe3\xe0\xf0\'\xf0$pf\x07\xc2\xfemW#\xa4\x02!\xd9\xb2\x1f\xb4\x8cp.\x94\xc8\xd67\x1e\'\xfa\x94\xebx\xc7*\xdfH\x9a\x10:\xf4\xd5\x16\xd5>\xb8\xb4\xc3\x88\xe2\xd1I\xcf\xd0\x89\x99\xfa3B\xd5\xcd\xcd6\xbe\x17\xe6\xdf\x1c\xf3Y2\x9b\xc73\xb0G\x8a\xfd\x94\xa7:\xe3$\x9dq\xba\xbb1rN\xcc~\x19m\xedks\xa3\xfc\xa8\xbe\xb0r{$\xba\xa6\x83<\x8e\x90\n#o\xbaT\xbe\xb9\x1cd\xe0i\x05\x06\x87\xe1\xdc&\x8c4U\xf7\x1c\x9e\xe4\xa3\xa3\x8c\xd5F\xf4\xaf\xcb\t\xf0=\x12\xe9Gc>\xcbd\x81\xc0e\xdbX6\x80\'\xd1c\x1c\xbc\xab,_=\xa6\x1f\x86\xbd9J\x07\\(\x99\x13\xb3D\xe1\xa5C\x1d%rRL\xa7,\xcc\xb69Gnl\xb9?(\x1e\xcc]\xe1\xd8"\xc1\x8e\xf1\x02M\xf7\xab\x1a\xbcLb\xd15\x06\r\xb4\xf4\x85x\xb9\xc4\xef\xbb0\x9f%L\x94\x00Q\xf2y\x1f\xf7N\x86\xb6\xd5\xf6\xad\xe6\xe2T~\xf1v\xa8\xa8\xf9\xf5\xf9:\x1b\x89\x8cn\xcf\xc6!\xc4\x9ds\x1a\xee\x03\xd4/\xeez\xcct\xbb\x9d\x94\xf2x\x8c\xcc3^\xab;cf\xc5%c\xb72\x87N\x91\x83c\x9a$\x0fD\xdf\xadH\xff7\xb7|\x88\xc7\xd5mIlP\xe4I\t\xcb\x91\xc3V"\xb65\xb5O\xc3\n<\xf0\xd9E\x1a\xfbp\xf2\x15\x8dV\x1d\x07\xf1\xeb\xd9\xc2q\xabG\x90\xf1\x1cs\x0c\x80\xee\x98\x03\x9a4,\x90J\x08\xc9\xd6\xfe\x91\x18\xcfg\xb2?\xec\xdc%\xa5t\xce\xf4<\xa2\xf7\xe9\xb3,\x9f%\xccG\xdd\xdb\xe5*\xf4\xb4$x\xa6\xa7\xd2\x1c\xb7\xaa\xe1\xe0\x17B\xaeQ\x93\xaa(u\xed\x1dAw<\xef\x1c\xe7fm\xd3>\xba\x10u}4\xb26\xb8a\xcce\xb3)\x1c(\xe8\x1dl\xb9\xcdtb\xb9\x0f\x00FUa\xd6\x0e]\x9d\x18#\'\x9b_\x0c\xf3m\x96\xcf\x12&\xba\xfc\xf3`\xdf\x9e\n%\x97WO\xd3\xc3\xa9\xb9\xa2\x9d\x92{\'E\xdb\r\xb0\x91b\x07+\xb9\x9c\x08\x9a\xb9\x1e{\x15\xdeTv\x8b\xe7\xe3\xdc!ey\xbb\x141\xef3\xf4F\x95N{x\x0f\xf3\x1b(l8\x93\xb4\xe2\x84\xbb0i\x8c2\xe4\x8b|\xc1\xdb,\x9fG\x988\n-?\xfd4ie\xcf\xbb\xe6\xd11+s\xadm\x8e`)\xa3\xa7\x90L\xefeqW\x8e\'E\xc0\xa4\xcb\x0e\xdcV\xe2\xf9\xe8\x8e\xb3\x8ai`x;\x1c\xafwUh\xce\xec\x85\xb0o\x023\xe3\xfb\xa8\xcf\xaa\xd2\xdeN\xc8\x08D\xfe\x9d\xc9^L\xc8\xdff\xf9,\x1b\x1c\x82\xc3\xe0\x92\xf6=\x8d\xa6g4\xf2\xa9\xc4\xa7\x9b\x03\xe5\x07\x15\xe7\xd3$Q\xb7\x1b]>\x027\xcd\x00\xfa\xd0J\xb0k\x9d\x9b>\x15B\x1dE\xd2\xb7^\x94\xe9H>\x08|[3\xed9\xb39\x91\x08\x1c^m\xdc\xcaH\xf3\x94\xe47{\x00\xf8Ze\xaf\xdff\xf9<&\x0bA\x92 \xf4\xbc&\x88\xfe\x96Y\xb8\x13Si&\x87(^E\xb6\x92n\x1cPK\x8d\xac\xa0d\x8aV\xfb\x16\xed\xabSUg\xc2\x01\xd7:#\x03\xcaA\xbd8%0\x0f\x03\xb0u\xb0\xdc\xa7\xe7\xbb\x11N\xac\xadjg\xe2B\x14\xe2(\xbex\xady\x9b\xe6\xf3\xc8\x10\x1f\x0f\x0b\x7f\'\xb7\x01r\x0b\t\x86\xe9V\xe79U\x91\x88\xe6\xb4N4d\xe5\r\xb0\xf2\xc9)\x8f\xc7\xa9\xa3\xc1P\x19T\x80rv\xd2\x0c]M\x1b\x00\xed\xdb\xc6EO@\xce5D\xa2xM\x18$\xcb\xc9\\\x14\xc1\x11j\x15\xcc\xb9\xcd/Z\x17o\xd3|\x96\xd1\x040l\xb9\x07=\x97\x82O\x90\xee\x86\'\x87Q\xd6\x8e\x91\x87\\\xa7\xe4\n\x80r\xd4\xcd\x1cnl\x08A9\xe2\xdcN\x92\x05\xc1\x98;\xf5\x14\xa1"}\xb1\x13\xd2\xafe\xec^B\x1c7d\x9d\xa0\xf9i\xa5\xca%-y\xf5\x85v\xa0#,5_\xeb\xc1\xcd\xdb4\x9f\x8f^|lP\xd8\xd3\xe5\xd0\x80z\xbf\xc5\x00/\x9cv\xb7\xa80\x14d\x9c\xf6[\xdf=/]\xdam\xa0d\x18\x93{|\xed\xb2hs\xafJ\x93\x0c"\xb2t!\x8f\x1c\xd9A\xf7Z\x0f\xbe\xec\xf1Luda\x98|PJ\xef\x98j\xf8\xd0\x96~1\x11~\x9b\xe6\xb3\x84\t"(\x0e\xc2\xf0\xd39\xa1!\x1eD\xc5\xfbs\xcfn uRk\x8fg\xf1\xf3\x92R\xf9\xd0m*,I\x10\xfd\x19l7\xc6\x11\xcf\xa1]\xe1\xab-\x08\x1c\xfa\xc6p\x8f&\xc2\x1e\xf2\xd6rG\xa2g\xccT\xf6#~\xd7\x97\xdb#:sl\xff\xe2\x1d\xf8m\x9a\xcfc4\x974\x08\'\xa0\xa7L\xb8\xab\xd1%E\x92\x91\t\x96\xa0\xc0\x9d`zg\xeb\x11aiC\xd1]}\xff\xa4\xdd\xf4A-HX\xf41\x88\x08\x0f\x14\xba\xe1Z\x1cK\x13fLz\xe5\x0c\x00\x889\xda\xdbZ\x90\xb5\xc3V)w\x07\xac\x07\xb3$|\xf1\x89\xc6\xdb4\x9fe\x87#\x1f\x06\x15\xf0\x8ff\x94u -\xa2\xa1Y\xdd\xa0\xfe\xa2\xce\x05\n\\NI=22\xec\x1f\xeb(\x8a\x1c\xe7\xcc\xf5\x19\xbd\x9d\x8faj\x83\x17=\xb86\xecX\xe6\x97Xh\xa5\x00m\xf0\x18\xbeT\x9cN\xf8\xedm\xc8\xb8H\x96\x85\x91\xfdZ,\xdb\xdb4\x9fG\x8a\x88-\xbb%\x8e?\x0103\xafct\xcb\x0be\xa4\xfb\xda\xf5\xaaZ\xd5\xc9\xad}s\xd0\xcd\xde\xda\xc2\xf1-P\x96[\x02\x98\xdc\xb3;X\x1f \xe9\xae$\x870\xae\xbc\xae\xd8\xd6\n*A\xddYi/\x16\x81\xdfn\x8c\xec-;Bv.\x1a\xe3\xc5\x84\xffm\x9a\xcf#\xb9\x81\x96\x0b\xdc\xef\\\xdf\x04\xa1L\xd0\x1b\x91_9\x15\xd0\xb5\xd2\x0fFP\xf2n[\xe1:\xc5\x01Cn\x87@w\xf8\xa0-\xf1\x9191\xda\x95(\xddZX&W"MV\\\x9d\'\x01&\x82~\xd73\x0e\x10fQ\xec\x1e\x03\xfc\x06:\xf3\x8bO4\xde\xc6\xf9<\xde`||\x1e\x8a|\xd6\xe7\xbax\xd4N\xc4\xd9Lo\xdc@E\xcbt?\xc8p\xdfoz\xf5t\'\xf12r\xef\xc7K\xbb;\xec\xc3\x0b\x96\xc9\x97\x81\x13gb\xcf\xa5\x93\xdch\xe4|\xcd\xefQ}5\xf8\x99\x0bxt\x84\xe1Z\xe1\x1a\x97\xa5\x18\xe1\xc5\xa5\xff6\xce\x87\xf8\x99\x80p\x02@\x9f\x1fOi\x01\xa9\xb3e\xb4\xdbH!_A\xc2\xb2\r\x1fi6idn>\xef\xc1\xde*0|s\x06\x82P\x01\xe0h[\xb8Z\x17uwW9\xea\xfa\xad\xdf\xecU\x97W\x03\xcd\xcd\x86Fn[\xe5\x90\xef\x8a\xf0\xcc\x14w\xf8k=\xb8y\x9b\xe6\xb3\xcc\x15\x12Y\xaeA\xe0s\xbe/N\'\xca\xbb\xe3\xb0+\xdd\xf0\x9b\xdb\x87\xa9\x84\x01Wv\xde\xc7\x11\x05\x9b\x08\xb7S\x80\xd1h\x19ywN\xa6+Z5\xcbyy\xd0|\xe3\x10t\x12\x069\xde\xb9\x19\xa2\xc8l\xb7\xfd\x08\xb7\xf0|\xda\xdeC\x08\x03\xee\xec\xeeOi>\x1fK\xf9\xd7\x9a\xcf\xff\xfa??\x85M\x14\x7f|\xb2\xe2\xdb\xe7j~b\x92Fa\x04\xab\x0c\x84S\xed:V\xb9\xfcHV\x9f\x9bo}\x9f\xf8\xfd\xff\xbe\xfa\xf5\x10G\x1fQ\xff\xf2)\xc6\xc7W\xc7\xfe\xe3k\x8f4\t\xc0?>7\xf4\xa4\x02\xcd\xdf\xfb\x1c\xaak\xeb\xb9\x03\xebe\xb8{\xfe\xdc\xe8\xfd\x85O\xd5\x08\xc3\xdds\xda2b\x88\xf9\x9f?\x08\xfb\xfd_\x1dBe\x1dT<\x109r\xf9\xab_\x1d\x8az~\x82P\xc0\x07OW\xd7\x8e\xe0c\xe9\x15\xd1\x8d\x14b!-u\xdb:\xe9E\x9bEP\xba\xdf\xc1Qc\x17\xb3\xbb?\xf0\xbc\x07\xe8\xc2r\xb7\xe9\xed\xe3,\xdb\'\xaf\xd1\xeb\xe3]\xb3\xc9\xd9\x02P%\xcc\x06\xf1`[\xf4\xb1\x0e\xd1\xfd\xb1>\x99\xf5Rg\xbe=\xfe\xf7t\xfe\xb0\xcc\xb6\xd1\x85\xbd*\xca\xa3>\xaai(\xb6\xf5\x9b\x96\xeb\x98_q\xa8\x06y\xa5\x0b\x9fF\xad\x1a\xf2(\x97nz\x15\xf5\xe1\xees;_g\xddyID\x10\x9d\xf5\xbe}\xac\xed\xfa\xe2\xac~{_\xfc\xd3\x9f^\x95~V\xbd\xfe\x9b?\xa5\x9f\xa2\xb8\x8c\x13\x7f\xf8nG}VI\xb5\x9f~\xf5)\xc5\xd7\x866\xb0\xc9"\xb2\xe7\x7f\xdaha+\xf5\xa1r\xf4\xae\xff\x89\xb1\xfb\x0b6\x8f\xbf\x84\xb1\xfb_\xff\x05\xc0=J\xb9\xec\xc7\xeaC\xa9\xf9\xe5\x0f\xd4\x97\xd7\x89\xbb*\xab\x87\x9f\xc3\xee\xd6\x0e\xcd\xcf\xe6\x18\x94Y\xa8\xc4\xb7\xff\xcd\xfd\x82\xc4\xfdW\xfb\xff\xcf\x7f\xb9q\xff\xe3\xa7\n\xaa\xac=\xcd\x8b3V\x80G:\x06\'\xb5\x00\xcb6\xba\xeb4U\xc4\x18\xa8\xd3\t cI\xc2:\x02\xb14\xf9QT\xa6m\xae\x0f\x86\xedA\xae}\xfbK\xfa\xff>\xac\x1e\xf1\xeb>\xfc#V\x8fBp\x0e\xe48\x9a#a\x94\xe6\x00p\xf9o\x1aB`\x9eF8\x90\xe5\x11\x02Yz\x80\xc3y\x1cEa\x16\x02a\x8eC`\x14B\t\x8cE\x01\x92b\xc8\x07\t\xf6}\x87\x88\x009\x16\xc1\x11\x04\xc5H\x16\xc49\x9e\x86\xa0\xa5\xb9<\xc9\xd1\x0cGp\xd8\xd2\xbb(\x82\x010\x02\x10 \x8dq,\xc63K\x9f\xf30M#\x8fj\xd3<\xf6#Y\xbd\x7fP$?\xfdN%$\x14\xfd\xf9Qkw\xf9\xe7\x9b[\xf1=V\xef\xeb\x9b\x84\xbf\xc3\xea\xd1\x00\x8c\xd1KS\x96\xefe(\x00%1\x06\x00ar\x19*\x8a\'\x19\x90C\x11\x14\x85\xe0\xe5+,\x8dc\xdc\xa3\xa2\x16\x0c\xc0\x04\x8fr(\x01\x928\x81\xfd\x96\xd5{\x83\xad\xf6_S\xfc/`\xf5(\x0e_Z\xb4,L\x98\xc4A\n\x85xfyi\x1a\x06 \x02\xa2Y\x9e@i\x9e\\~\xdf\xd2~z\x99\xb9K\xf7b\xe8\xb2\x18\x10\x92d\x11\x1e\xe1X\xe0\xb17\xfd`V\x0f[\x86\x83#\x97V.\xad\xe1\x81\xa5\x9f\x98e\xc0!\x06\x06q\x84\x02\x10`\xf9\xf1\xa5;8\x84\x86q\x94\xc2\x08\n\xc29\x86F\xb9%\x9bE\xc8G\x03\xc1\x9f~\x9f\xd5#\x96\xb5\x0b-+\x10\x87 \x9ag\x10\x10\xc7H\x00 Y\x14\x05\x96I\xbb\xcc\x1c\x88\xc2I\x9aaY\x8e#`\x1a\xc5\x10\x08`A\x06\xa79\x8a\xe2Xh\x99\x1f?\xfdU\xac\xde\x9f\xc6g\x9eX\xbd\x7f\x17k{\x94\xfe\xfaC\xac\xed\xeb\xaf\xf3\xcf\xc5\xda\xc8\xef\xfb7\xb6\xae\xdc\x8a=P\xebT\x85\x1e)\xd3\x8b\x858*\x12\x83U\xdc\x80\x9fw\x86w\xdc7\xd4\xae\x1f\x97\xed\xfav\x96U\xee\xe8\xa0\xcd5\'\xe4R\xa8\xba\x1dD\x05\xd3\xb1t\xc2c\xcb\xc6\xd5\x054\xfat\xcb\xf1\xa35\xbcXF\xf5]X\xdboO\x85\x7f\x12S\xfa\x93\x82y\xa4t\xb9Bh\xc6\xa0\xb5\'\xcc\x83<\xc5\xd4\x03\xf3\xd9\x14i\xe7\xe3\xb6\x84\x1db\x7f\x94\x14n\'\x91<\xa4\x14\xb8!hb\xc2\xa3\xbbR\xd8\xa5\x9cT\xd5\xb4\x11GGV\xb1\xa5\xeb(v\xae1\xb3/"\x18o\xc2\xda\x960\x1f\'2\xf1T\x0c;\xa7\xc3d\x03\xcd\rV\x16!r\xc4)\x11u\x83a\xdb\r\x8dLI\x80x\x05\xee4b\xc1]\x05\xca\x89|\x1d\xcc\xf42\xb6\x13\x04\xden\xcb\x1a\x98\x1a\xf7B\xd4\x07\xad\xe8Z\xa7R\xc9RQ\xf6t\x878p\xb2yq(\xdf\x85\xb5=bD\x81\xe5\x98\x031\xf0\to\x90\xb4\xf2\x805|\xa5\xe23\x00\x99\xb0hZ\xb7M\xb7\xf7\xd4\xa9@\xf5\x98\xc4\x95\x8bM\xb8,\x0f\xef\xcbb\xcb\xe7vc\x14\xd6\x18K\x13\x107j\x9f\xb1\xbe8\xfb\xee>b\x05p_n[\x96\xb9\x93s\rm\xbeX\xc1\xc87amK/B\xcbA\x05\x11\xcf\xf5v\xc1\xc04\x8e\x1b\xef\x98\xd6\xa3Be\xe1>\xbd\x94\xf0!\x90\xca#\x19hA|C\xf3\x18\x05\x0e\xf2\xe0\x9f\xcfs\x15\xcb\xfca\xdb\x01p\x10\xc6\xd4\xacl\x14\xf1B\x89d\xe98\x98IPu\xb1\xa4\x82\xf6\x068\x85\xf4\xab\xec\xd5\xbb\xac\xb6G\x94\x00\xbc\x1c\xac\xe8\xf3\x928\xd5[\xc7/\xa1\x9a\xbe=J\x89\xc2H\x14\x12\xd2\xce8] \xceL\xb5\x0e=\x0eFq\xafo\x83\xd4O2\xc1Gg]\xdb\x06\x16.p\x91\x9fZl1\xcb|te\xa4X\xee\xb7\xc0u\x1cv\xe1\xf5|\xdc\xdfU\xeaE\xa3\xf1MV\xdbcw\x83\x89%)Y^\xef\xa9\xda?\xd5\x02\xec5\t\xee-RO*\xa8\x94Rx\xd8{\xe4\xd6\x8f\xb7\xedI\xb2\'\xb3U\x8b\xf0\x02\xe9%M\xb1|>/#\x1eq0{m\xb8V\xba\xe8G\x06(\x0c`\x0b\xef\xe4\x9d\xc0\xa4\x0e\xed\\\x13\x01F\xfbW=\x937Ym\x8f\xd1\x04A\x08A\xe0gy\xaf\xee\xb2\xfa\x1c\xa0\x01U\xf0D\x13\x9c\xe1\x1b\xa7\xa8\x1bSt\x8b\x88\xb5Iy\xcb\xec\xf1\xac\x02`-\xd8\xe1WAv+\x98aZo\xb7\'\xd3]\x89\xd7\xdb\xbc\x8eH\x826\x8eg\xf2^Mc\x16\x1e\xfa\xcbr\xc9U\xa9\xcf\xb1\xda\x1ea\xc2\xc8\x83\x06C\x9e\x81\xc1:\xdaS\xdd\xd5\x10\xfca\xba\x81n\xda\xa0\xc3\xfd\xc0\xc6$\xec\xc2\xf1\x002\x87\x9c\x8d"9=:\x8a\x85\xeeM\xd8uF\x949x\x95\x01\x1e\xe7\xda\x0b\xd8\xedq\x87\xd0\x9b\x86\xd8\x01^\xca/\xb7F\xcb\x0e\x0b\xf7\xf0\xbd\r\xee\x07[m\x1fa\xe20F"\xf0\xd3\x16\xd4Z\xc5F\x0e\x19B[\xce\xdb=\x0fO\xe7.V\xdbzGm\x964\xf7L_Fowk\x0f\x0fO\x19\xd1s\xb7W\xac\xee\xae\xe0\xb7Yp\x87\x0c\xbd\xba\xc4E\xee\xb3\xf6v\xa0\xd4D\xcf\xeb\xc19\x9a~V\xd4\x15\xfdZ\x98\xef\xb2\xda\x1eks\xf9\xd1%\x05\xc4\x9e\xea\x1b_\xac>\x03Un\xf2E\xb0eD\xc4\xa4p\xdf\xbb\x9b\'\xb2:*\xa5\x93\xcbxC`\xdd\xe6`\xeb\xe2i\x0bC\xc8\x00\x180\x97\xb6\r\xa03zd(9\xdb(\xf7\xebeN\xad\x1d~\xdb\t\xf5)Qk\x01m\xbfw*\xff\xbd\xad\xb6\xa5\x17\xd1\xe5\xfa\x03\x12\xcb\x95\xe7\xb7\xbd\xe8\xd7Dq\xea\xdc\xdeg\x0f5t\x9e\x02\xc0U\xe9\xa4\x85N)\x88l\x11\xcd\xb3\xf6\x11\xba|k\xaff\x87I\xbb\x92\x92\xaf\xe3w@\xef\x0e\xa1X\xf9\xf6-\xce\xe5R\xdb]y\x13\xe6\xa7:1j\x04\x04|\xc0\xbd\xbe\xb8&\xdee\xb5}\xecp\x10\xbe\x1c\xf0\xe0\x13Q\xb1e\xaf\xd6i/\xb6\xa1\xb5d\'\x91\x9c\x15\xd4\x99\xea\xd5\xfe\x82\xf5\x88y\x1cK}sg\xa9\x8cR\xb1\xa0\x07\xb5}\xa7\xf3!n\xb5\x8aB\xedf\xbb\xf1t!\xad\xdc9;0\xc7v\xa3\x11\x07\x96\x80\xe5\xcb\xed*\xdd_\x948\xdee\xb5=\xc2D\x81\xe5\xea\xfd;\xe7\x95 -\'k~i\x98,\xb4I\xe8&\xd6\xd6\xae\x0b=\xa5H\x12=\x88xa@M\xe3\xaav,b&\x1b\xcbE\xd4$\x9b\xc4Lq\x9a\xc63+N\x86\xd4\xa3d\x1f0\xe3\x02O\xb9\xbe\xcd\xb6"\xa3\x1d(\xff\xbb\xe7\xd5\xdf\x1bk{$\xc20\x86\x02Kr\xf3tu;&\xc5\xc0\xf68s\xe1l\xfd2\x8e^\xd1\xe7\xbb\xd8\xe1p\xb5D\xc7}\x0b\t\x97\x8d\x19\x9c\xb6W\x99\xb8\x9em\xe8\x84J\x97.\xc0l+\x93\x96\xdb\x06\xda[\x93\xcd\xcf\x1d\x82L\x17\xc4$\t\'\x908\x12T.\xdbW\xd7\xc4\x9b\xb0\xb6G\x98\xe0\xb2\xfd\x11\xe03+|\xba9\x031\x96U\xcc\xedr4n\x152\xf6]l<\xeb\x88w\xaf\xd9bIi\xa0\xcd>/\xbb<\xd8\x1f\xabz27\x01\xd3\x19uVR\xfcpL.\xb6\xd8h\xc3\x1c\xdc\xeb\x8b,N\x1c/%\xc0m\xc9[w\xcc\xe7`m\x1f\t9\x8a\xa2$\xf2|C\x05\xf2M\x84\x81&-\x0f\x07\xed\x0c\xe4-@O\x1d\xac*\xbcz\xf7\xa4\x0e9/}\\\xc0\x84\x9c\x0b\x9a\x0f_\x1cNO\xe9\x1d}\xeb,\x0b\xde\\\xa4I\x10\xf7S\xaa\x8bgr\x9f\xf6\x8e\xc1\x99\xfa\xee\xcc\xfbE\xa3\xbe8\x9a\xef\xc2\xda>.\xe28\xb2\xfc\x08\xf8\xb4\xf4;C\x02H&\xdf9=\xcbw\x9e\xda\xb9bu\x07-B!)\xba\x1c\xe4C\xb3\xad\x88Ktc7\x01\xc8;\x07\nG\xa0\x84\xc1\xc2$`\x1d`\xdcf\xe8\x85\xa4\x02x\xc7\xce\xb7\xc2&\xab\xb3\x8e\xde\xf67\xd3\xee_E7\xdf\x84\xb5=\xc2$\x00\x98\x04\xb1\xe7\xc7*\x98;\xf6a,\xc3\xc1\xd0\xea3W\x1c(\xc8?\x0b\x1b =\x0bW^\xbe\xdc\x1d\xdd\xc5\xab\xdeWvU|dcB\xd4\x8ef\x04\xb2\xb9P\x18&?\xee\xf1\x00\xdd\x1d\xea\x08\x82;/7\xc4\x80\xed\xa8Z\x93\xcf\xc7\xafuI}\x17\xd6\xf6X\x13\x10\x04\xe0\xcf\t"\xed\xeb\xf4^\x104#k4\xe8ze\xe3\xc3\x86C\xe1@\xc6"\x1eN!\xb6m\xca\x1c>\xdd\x10X\x14|\x8ff\xa0\x81\xb0\xdbK\xe3c\x05J\x1f\xbcN\xc4\xe7\xb1\xe7,vc]\xc9e)Hw\x19\xa3\xb7\x15\xff\xe2\xc2\x7f\x17\xd5\xf6\x91 \x82(@\xa2\xd0\xd3\xf3\x0c8o\xd2\xa5\xa5^\xaf\xcc\x06\xcf\xed\x8e\xf1&\xb5\xa3\x93\xca\xdc/\xf8r\x9bS\x0e.\x14Z\xba\xaf\xbay&\xc0\x9b\x91f\xf6\x94\xca\x1a\x92]\xda)Z\xa1N>NX\xe6l\xf6\x06\x18Cgu\x12<\xf6\xdck\x9b\x17\xc3|\x17\xd5\xf6q\x15G\x97\xad\x10%\x9fn5\xed@\xd6\x02\x930\x00?m\x9f\x1b\x88\x84\xaf\xc9\xc5j\xc6\x9eL\xeb\xda\xcc`g\xb0\x13\x05\xdfV\xdb\xb8\xa8\xcf\xba\xaf\x17\xd5I\x80\x8f\xbcb\xd9H\xa1E\x9a\xb9\x1fA\x90L\x99[\\\xeac\xc7\x99|\x94d[\xa2^.\x87Se\xc0\xca\xed\xda\xde^|n\xf3.\xaa\xed\xe3n\x08=\xde\xe6x\x86\x8c\xd0=\x06W|u\x8f\xd4ec\xa4\xc6a\x7f\xd1\xee\r\xbc\xec\'\xd6,\r;\x89\x17\xab\x8chg\x01F;\xc8\'\xe3\xf6\x84\x1fmj\xdc\xfb\xdct\x8f:\'\xeb\xd0\x1aA.=\xdb\xb1=\n\x10\xf35\xc8Q\xc7\x9c_\xbc\xe9\xbf\x8bj\xfb\x18M\x90 \x97\x9b\xc2\xd3)\x01\x0f\xf79\xe9\xd4\xbb\xac^M\xc5&\xb4\xcd\x00\x06,\x0f\xd3z\x84\x8d\xd0\xee\xc0(\xc7\x0b0_5\x8c.F\xde\x19T\xcc\xd9\x9d\x83\xc8\'\xe04\xdf\xe2\xbc\x9b\x1e8\xd9\xe0\x02\xd59t\x1btC\xf6\x00pn\xb5\xcd\x8b^\xd3\xbb\xa8\xb6\x8fm\x1c["]\xae\x08O\xbe\xe0\x1d\xdfC\\\xa40\x01\x9e\x9bd/\xe4a\xd2\x14i\xa3$lZ\x01\x91\x0f\x96\x80\x84t}\x0f6\xf7{`\xd3\xc8\xd0\xd2{fJUE\xbe\x08u#R\xd9\x12\x97p\xc1C\x08\xd5=\xaa\x08\x11#*v\xf9\x8b^\xd3\xbb\xa8\xb6\x8f|\x1c\'\x97d\x01~JT\xdb\x83\xb2\xe5\x87\xc9 \x0e\x13<\xe7-\xd7e}\xa7m)\x19\xcaf\'\xc9\xec]"\xcdn\x95\xa2\x88O\x19\xe1\x1d\xcb:c\xd6g&\xde\xb4\xbd\xb2\x03\xcfn\xd2c\xe2N:e\x1e\xe8\x13\x9eJ\x1c\x8c\xfe.\xa9\xed\xf1t\n&\x96\xc5\xff|(\x87bb\x16\x16\x8d#M\xbaG/\xa8\x0b^m\x90\xae}:\x80\xec\x1a\xb9z\x07\x13\xa3\xa9[kq\xc3\tg\xbd\xf9\\\x94\xa7\xc6+1\xd0\x05\x94\xa3\xe3\x08)\xd2\xdd\xfbf\xd3\xde<\xae\xb9mhH\xc7\xacZv_\xdc\xc6\xdf\x05\xb5}<\xb5\x01?\x1e7<\r&\x0bx\xd3\xdc@\xe4\xa9\xa7\x81\xf0&\xb5-T\x93\x91\xb2$\x1d[2\xb5\x0f\xfb0\xa31wSO[\xa7\xdd\xa1\\\x1bp\x97\xae\xda\xed\xeaM\xc1\xee=\xeeVs\xea.Iw\xfa5I\xcap<\xecMVV\rn\xf8Z\x00\xf5\xbb\xa4\xb6\xc7)\xb1\\\x97\x97N$\x9e\x1fg\xccfA\x16iP\x1a\x8a-\x94\xc7\xa3\x92@\xee0\x02g\xae\xd9\xd0\xe2\x96;u\x89\x03\xb5\x17\x0c\xbe\x99\x9a\x16\x99\xa8d\x9c,/\xec\xeb\x82\xc9e\xab<\xa4\x98\xa2\xf8\xb7\xa3\x169BA\xd9q\xb1\xc3\xb5#\x8fR\xd4\x9f\x92\xda>\xde0\xfc\xb5\xd4\xf6\xed\x03\x82+X\xf3\x9dO\xce\xff\xe7\x805\x7f#\nd\xd5U\xd6.\xfd\xfa]\xba\x825+X\xb3\x825+X\xb3\x825+X\xb3\x825_\xb8\x9d+X\xb3\x825+X\xb3\x825_\x19\x82Y\xc1\x9a\x15\xacY\xc1\x9a\x15\xac\xf9\xb2\xed\\\xc1\x9a\x15\xacY\xc1\x9a\x15\xac\xf9\xca\x10\xcc\n\xd6\xac`\xcd\n\xd6\xac`\xcd\x97m\xe7\n\xd6\xac`\xcd\n\xd6\xac`\xcdW\x86`V\xb0f\x05kV\xb0f\x05k\xbel;W\xb0f\x05kV\xb0f\x05k\xbe2\x04\xb3\x825+X\xb3\x825+X\xf3e\xdb\xb9\x825+X\xb3\x825+X\xf3\x95!\x98\x15\xacY\xc1\x9aWh\x86\x7f\xba\xf5\xfeK\x9a\xe17I\xc8W\x06k~|`\x9f\x04\xd6\xfc\xf8\xc0>\t\xac\xf9\xf1\x81}\x12X\xf3Z`\xff\x0e\x7f\xf2I`\xcd\x8f\x1f\xb1O\x02k\xfe\x82\xcd\xe3/\x01k\xbe\xfd\x8f\xffN@\xcc?!;\x7f\x04\xc4\xa0\x0c\xbd\xc4\xc9\x81\x0c\x0b\x00\xd8\x83\x04\x00I\x9e\x86x\x96\x06X\x8ce\x10\x0cx\xc01\x14D\x02\x04\xcer\x04\x00A\x00\x82\xe0 \x86\x924\xc7p\xccGM\xaf\xef\xdb\x07\x1c\x8c\x12<\xbc\xc4B\xb3\x04\x83\x02\x8f\x1a\x13\x18I\xb2\x1c\xcfC\x8fZ\xa0\x08J\xa0\xe4\x83\xa0\xe1I\x1e\x079\x86\xa5\x96\xfe\x82\x19\x8a\x868\x9c\xc6p\x12\xff\x91@\xcc?\xaa-\xfe\xf4T\x80\x01\xfe\x1f\x00\xf43\x88\xc3\x18\x8e\x12\x10\xf9G@\xcc\xd7\xd7u~\x07\x88YF\x99\xc6!\x0c\\\xbe\x07E\x1eZ\xc9\x03U\xc10\x9aai\x12B9\x9cA\xf1\xe5U\x18\x14a!\x92|\x14\xe0\xc48\x8aca\x9c\xe2 \x86Z\x9a\xff\xd3_\x00\xc4\xbc\x81q\xf9\xdb\x021\x08C\x02\xcb\xea\x030\x90\x81I\x80|\x94\x9d\xa1I\x8eg)\x1a&@\x88\x81\x00\x06^\x86\x9a\xe0h\x16Z\xd6\xe9\xa3\xf9\x14D\x91\x1cE 0@\xb1\x1fu\'\x7f\x0f\x88y\xc38\xfd5@\xcc\x9f.\x05\xff6 \xe6Q\xd8\xe4\x0f\x81\x98\xaf\xbf\xce?\x15\x88A\xb1\xef\xcb)==\xea\xeaN\x9e\xd4\x1bC(\xbb\r`\'f\xc1\xba&v\xc6\x81\xbbOo4@w\xafLrp\xb2\x0c\x10\xd3Kt\xeb6\xa6\x0b\xe8}%N\xbcn\xb6\xc9\x01O%\xcbm\xceGo\xd8\xf5\n\x0f\x87[\x95\x15\xa8\xd7*U\xbd\x07\x88\xf98\x15 \x80\x000\x12~\xaaj\x96\x13}p\xee\x95\xf6\xd4\x03\xd4P\xa0\x98\xd7O\xd7T\xebq\x17\xd1\x86\x0b/\x1a\xd0F\xd8\xb7\x89j\x06It12\xb3\xea\x92\x02\xce\xe5N\x18u3\x92\xa6D>q\xf1V\x9f\xaf\xa6\x7f\x16\x8b\n\xd3]\xca\x9f\x8e/\xd6U|\x8f\x0f\xf3-J\xe2Q2\xfa\xb71R\'\xe2\x90\xf8\xe1\xe6\xa0\rsC\xd9\x11\xba)\xa6\xf2*w\xbb\x06*\xf8QV\xe1\xbc\xda\x8d}s"q\xb83\x1a:\xb9\xeevZ\x9d30P\x00\xe0i\xe3\xc1\xe7\x03\x02\xf8~o\xd5\x85\xe92\xc2q\x17\xc4F\xf8b\x8c\xef\xf1a\xbe\xc5\x08\x82\x10\x08\x80\xcfC\xa9\x1e\xc7hr\x02Kb2W-\x01\x18\xe8o\xfb\xe1=@\xcc\xb75A\x82\x10\nb\xf0S\xc9}s\xa3\x1d\xd0\x11QJ\xa9\xa1\xafX\xd5\x04*\xe9@\xf2\xe92t\x10\x1dN\x96\n\rI^\xe9\xb1\x13\x86z"\xc1\x19\xd9\xd6\x95\xcewJLNT^h\xc7\xd4;\x96\xd4\xad\x18\x1c-*\x8c\x86\xd2\xda\xfa\xba\xe4\xe6\xaf%7\xef\x01b~Y\xfa\xcb\x18\xc0 \xf1\x84&\x91\xae\xc9\xa5\x82J\xa6\xa8\xa9\x0eyH\xe5\x82@o\xf7\xb8\x1e^X{\x86\xfb\x81\x84\xf9\xcd\\\x1fz\x8a\xd7\xe6\xab\x8eV\x85\xa4)\xfb\xa3\xe3\xdf\r\xbd\x95\xe2\xe2\xb4\r%\x0b\xb3J\xbav\xe3\xd24\xf9\xb3";\xf9\xab\xf7\x9a\xb7\x001\xdf\xc2D\xc8e\x83C\x89\xa7s\x028\xf5K\xabw\x88\xce\xd9\xec\xee\x16\xe2\xb8Q\xa6\xec5\xd59>n\xf7.d\xe0-D3K\xf2\x16\xeb\xaa\xe0\x1d\xb7\x80\xb0\xdc\xaa\xf2\xb6\x1cZ8 \xed\x0c\x15\x04\xa6\xef\x01L3\x0e=X\xb4\x05!\xc1s\xf0]\xf7\xf2\x87\x021\xdf\x96>\x01b$\xbedDO6\x14W\xdc"\xb5&[wt\xfcQE\x97\\\xa5L\xba^\xa17%V\xdb\xa2/\xec\xedQ\xd0\x95h\x97D\xbc\x04\x9e\x8d[(\x9e/\xe1V\xaf\xcfm\x02\x812|K\xa5m\x92D\xa2\xc7[-EMgA\x1d6/\xeep\xef\x01b~\t\x13C\x11\x02\x02\x9e\x1e\xab\xf8@k\x94\xa5\x06\xf4[\xf0"\xa2\x012y\x17\x9bq\xae\xd6N\xdf\xde\xe7lj\x1a\xb5\x1ciD\xd2JD\xabo\xd7\xc3Ym\xb9\xba\xa4\r\x05\xd2"\x0cg\xf7\xa2\x1aS3\x89\xd7i\xa4^CT\x99\xa6\xde\xab\xae_\xeb\x96\xfa\x1e \xe6\x97\xa76\x00\x01"8\xf8\xd4\x8b\xb7y\xc2\x12:H)\xeb\x06L\xb5Vw\xb4;\xcbcr\r\xca\xce\xd2\xd8\x8c\xc2\x01\xf0\x9a\x14\x03\xa8I\x16/\'\xf0\xaeCE,\xb5\x0e\xb8\x08\xc41\xb3\x85\xe2}C-\xbf\x0f\xb0\xd8\xfb!52\x15\xa6\xc1N|\x91\xbc}\x0f\x11\xf3\x8f\xeb\x1b\xf9\xb8\xa9>\xedpH9\x01\xec\x85\xd2\xa1\x93f\xf1\x15\x9an\x8b\xdb\xd5t\xe9\xed!\x94\xc48>-\x19\x01}"\xbd\x9b\xcdD\t{).\xc3|\x08\x8dzCw%a\x8eQO-\xc7\xec\xc5u\x8eJ\x8a\x0b\xfaT\x9c[;k\xa0\xfe\xc5\x1d\xee=D\xcc?N\xfd%\x85@\x9f5\x8ck\x7f\xdc\xd3\x98q\xcd\xce\xb9\x9c\xf2wiO\xfa$"l\r\xee\xa4\xe7\xdd-\xe1\xdd+\xc5\x92\xdcr4\xaa\xfe!=\xc2\x8cL\x19\xc3!\n\x14H\xa1I\x03(\x90\xd2D7\x94\xa4\x81[v?\xfb\x98\xeba\xb4\xf4\xb5\xea\x8b\xbfG\x88\xf9eI,\x9dM>\xc3\x02\xa5\\\xb8>\x0cY}\x9bK\xce\x18TF\x82\xa5^\x1e\x82\x17\xa8\x99=\xedD\x98>j\x9b]\x14)\xadw\xd1\x00\xbe\x17\xcb[\xb4?Zxd\'\xfc\xa8\xa7W\xcfU\xb5"N\x94\xec\x0e\xe2\x12\xb8\x81\'\xde\xdb\xbe\x98\x07\xbf\x07\x88\xf96U\x96\xcc\x06[\xee\xcfO\xa9M \xed\x8e6t\xb0v\x8cT!\xf8\xf9~\x98\xf2\x92\xb2\x85l\xa6b\x80T\xf9\xc9b\x15\x03\x84\x15\xb8\x18\tw\xe0\xeb{diH\xe53\x0ec\x80\xde~\xe7\xe0\x97\xd9\xb9\xf1"9\x9c\x98\x13P\x88NZ\xb5N\xfc\xe2\xe3\xa9\xf7\x001\xdf\xc2\x84\x96[&\x8a>\x13X\x04\xb7\x15\xcb\xf8\xacd.w)\xacj\x04Y\x81a\xeam\x8f^<\xbdaG\xccw\xb1a\xb9\xf7\xc4\xd4=\x924\xf4^\xe4\xb9\xc9R\x17\xe7\\\x13\xfey\x17\x0c\xaa\x85\xc9\x04{<\xebi4\x1a9h\x19\xb9\x93%/\xb2\xe5\xef\x01b~9\x0c\x1fy\xear\'z\xaa\x89o0pW\xa7]\xbbM\x8c\xc8\xf2\xe9}O\xf2\xf6-\xdf\x8a3r\x1e\x96-F\x8f\xf5d\xc9\x8f\x83\xec\xce\xf5\x00q\xac\xac\xbe\xcc\rws\x90\x0bD\x86R\xf0\xe6R\xf9D\xc0\xe1&\xd3\xa7\xa3\x7f\xe2+M\x17\xce/N\xda\xf7\x001\xbf\xe4\xe3\xe8\x92\x05\xa1\xcf\xb7\x1aB\xbb\x18\x90I <\x1d_b*\xd1 H\x91\x84;\'o\x94\x93\xe1\xb4\xbb\xc3\x85LN\xa0\x8d\xe9\xe2\xf9\xaak\x82\xd8l\xd4\xf8>\xea\x08\xdb\x99\xc3\xc9\xb3\xf0K\x98\xb9\xb73Q\x9c\x80eS\xbc\xa9\xf7\xe9\x10f\xe3\x8b\xdb\xf8{\x80\x98_\x9e\xa9>\xde6#\xf0\'\'\x02W\xa9j\xbb)k\xc3iz&We~D\xcd\xac%\x91;\r\xd0\rMH\xd6\xf9\xc89\xedE_6\xe9\x82\xb2\x97\x91\xdc\xa3N\xe3\x10\xb45\xa9\xd2m\x80Y1\xe8\xe8:\x8cCED\xec(\x85\n\x18\xaa\xed\xffD!\xe6\x975\x81\x02\xc42\x1eOK_vI@\xd4\x120c\xc7\x0eQ\t\xa5\xce\x0e\xb0\xda^}@\xd1\xed\x93\xef\xe0\xa0\x14\xe2[A\xe8*\xdc\xd2z\xf4V\xa1v\'"\x80\x8b\xdf\x8dK\x86\xc5\x91\x98A\xd6@\xef\xd1}\x88r\xd7t:\xc0\t`\xe2\xccg\x081\xdf\xc2\x84I\x10!\x96u\xf1\xdb0\x0f\x87\x8c9\xd7\x03y!\x8a\x8df\xd4G\x13\xf1Z- d\x91V\x1a\xe4\\H\xfb+_B\x8c\x0eo\xb4\xd3v\xd2\xd9\x91;\xfbEv\xf7O\xcc\x85\xdbwS\x94\xed\x8e\x94\xed\xee\xf2vIF\xeen\xdfO\xa4\xd5\x1a/\n\xed\xef\x11b~\xb9\xbc\x11\xcb\xa5~Yd\xbf\r\xd3-0\xe7T\x9d.\xcb\xa9\x80\x1e\nYHz\xe3h@\xb0M\xcd|\x01\x1e\xfaC\xcb\x94\xa8\xcd\xcd{\xb8,\xe4\x06\xe3\x1a\xdc\'\x97\xbc\xad\xe8G\xdfnXO\xba\x14\xc4\xfe\xa4\x9b\x93 N#\xbdY\xd2\xbd8\xd8\xe3\x9fB\xc4\xfc\x92|\x89\x88\xf9\xf1\x81}\x12\x11\xf3\xe3\x03\xfb$"\xe6\xb5\xc0\xfe\x1dp\xe4\x93\x88\x98\x1f?b\x9fD\xc4\xfc\x05\x9b\xc7J\xc4\xfc\x08"\x86\xfc\xd58\xfe1\x11\x03\x00\x18\x8d\x10\xcbN\x86\xf2X\x89\xba\xbd\x0f\xda1\xd6\xcd<\x83\xeb>\xe0\xe0\xfbP\x05\xbdE\xb9crA\xf2&\xef\xa3f&\xf9|b\x04q\x0b\x12\xd6i\xc7\xbfX\xaa\xeamF\x0c\xf1\xf32\x980\xf0T\x8e\xebb\xb6\xaa\xdf\xd6N\xaa\xf9\xec\xc1;\x9d\x80\xa8\xb3\xac&\r\x0f\x88\n\x9co\x13\x02\xa3\xbb \x0f[e\xa0\x9ar3\xa4{\x8e\xb4\xe9{K\xca\x85N\xa5\xc0\xc9\x82\xc5\xb2ALu\xb06\x03!\x1edp\xcc\xf4\xfe(:\x827\xa9\x07yW\x8d8D\x9f3\xc6\x8d\xcfL\xc1\x8b\x17#\x85\xaa\xed\xf7\xd8\x8f\xbf\xb9\x11\xb3,\x08\xec\xe1\x8a`\xcf@\xc3\xb2\xedo&\x11\x9b\x80\xc2q\x8bm\x05N\xe7X\xbd\xd8\xf8\x81O\xae\\_\x93T\x08]\xcfz. q\x9c\x1d\xa9\x94\x98\xee\xc9\xac\xf0\x18tX\xce\xcd\xeb8\x88\xe5\x89\xb5{\xbcW\xd49\x10A\xd3 7T6\\_-\xd5\xfc.#\xe6\x11\xe6C\xc0\x82\x88\xa70\xbd\xa2t\xfdZ\xb8\x94wFt\x0f\xf9\xf1\xd4\x9eIQ\x97\xb0\xf0\x82Vw\xc5\x04\xd4\x96\xd8\xd5\xf2\x85\xf0\'d\x7f?\xc7\x94\x8f\x9f\xce\xb5\x14\xbb\xb1Dn\x10\xc0j6Y\xef\x1d\xdd;\x9a\xdb\xa1\nL\x83x\x89\xac\xf4\xff\xb2wg\xcd\xae\x1a\xd9\xe2\xe0\xbfK\xbd\xd2Q\xcc\xd3#3\x08\x04b\x12\xa0\x8e\x8e\x0eF\x01\x02\x81\x18%\xfa\xcb7\xda\xbe7\xee-\xcb\xa7\xaaT\xa1Sg\xbb\xfe\xbc8\x1c\xdb>\xdb\xb9\x92\xcc\\+\x91\xbc~\xcc\x9b6\xd4\xa7\x8c\x985\xcc5\x15\xd1\xc4\x9ak^\xba\x8eQ\xb1z\xdc\x15x\'\xa6(\x88\xea<\xc7\xe9\x1d\x0b\xc3^\xdc\x10\xc8\xe4>\xa4\xdb\xdc\x8fA2\r\xda\xe8H\xfb\xcb|\xd0\xd2R\x11\x86p\xb7\xc0Px\x14\xb5p\xe9]\x1b\x9eD\xde\xcc:\xff\xa1\xea\xd4\xee\x8a\xdc\xde\xa4p>f\xc4\xaca>\xfb:C\x14\xf5r\xc2\xed\xb2\xf5\x8e\xe2\xd20wm\xeeClNJ\xbf?\x11\xa5o\x93\x81Vb2\x15L\t6S\t\xb8\x87L\xab\x183w\xd7.U(\xdc\xadT\xbex\x91]8!b\x9f\x0fYm\xec\xcck[\x9c\x1c\xc6\xdc\xc7X\xfbf\'\xbe\x8f\x191\xeb\t\xb7\x1e\x8c8\x8c\xbd6\x8e%\xa5]Iq\xc7\xe6\xbe8\'b\t#\x86y\xdcT\x95uc\x04\x99k\x87\xf2\x8cL\xb5x\x90\xb6\x15$\x97\x0c\xfb\x00 \x1e\xbc\\\xe4z\x1f,\xf6\x9em\x87\xddn\x0c\xb4\n\xf5\xc2\xdb\xe9\x94\xe0}\x84t\xe4\xfcf\xe3\xd8\x8f\x111\xcf5\x8bR\xf0Z\x84\xbev;^\xbaG\x06\x8a0\x83!\x9e\xd4\x99W\xcc\x18\x18O\x18i{\xbe\x9f\xe9\xb9m\xbd"\xc0\x8d\x94\x13\xbcy\xb8\xcb;q\x12Z+\xe1P-\x16\xc0\x80h\x19Y\xea|[;t\xbdr\n=\xb5\x11\x93\x07\xd6\x07\xf9\xaf"b\x9e\x95\x07J\xe2\x10A\xbc\x84\xd90\xe4z\x15oA\xd9\xbd\xcbB#\xd2\x01\xd4\x8aG\xd9\xbe\xd8\x00\x8fT\xea\xa0_\x12\x99\x9a\x8d\x82C\x15\xa8\x17\xae,S\xab\xc6,),c\x05\xf9\xfdq\x89:*\xd2\x0f\xcb\x12\xdc\x1f\xd7[\x86\'\x91G\x87)\xc5~+\x0e\xe3cD\xccZ\xd8P\xf0:\x8d\xd8\xab\x7f\xb7\x1f1\x0c-}\x1d\x96\x99\xfb\xa3&\xba\xe3=\t\xe9\xd3\xce\x9cY"\x18FD\xa9\x89\xfa\x91\x97qq\xaf%\x18\x92\xcd(}\xb0\x02|\xc7\x8cS\xe9\xa9\xb2\xc6\x11\x80X\x1d[dn\xda\xcb~L5\x0fC"\x88\x7f\x13\xc1\xfa\x18\x11\xb3\xee\xfc\xf5\x84\xa3\xd0?\xd8\x13\x02\xa5\x9e\xeez0\x93\xf9\xb0\x84\xc7\xe9TK7q1\xcb5M\xdf\xb9B\xc9+\x8d\xc4\xaf\x96\xeb\x1ez\xb9\xb9\xcb\xf0@(\xfaM&\xf2\xa3\x95\xd2\xa4~W\xf7-\xff(!\xdd\'\x88\xf9h\xd7\xc7\xe4\xe4\x8fQL\xbe\xd9L\xf9cD\xcc\x1a\xe6\x9a\n\xd6\x19B_\xba\x0c3r\x04\xe7\xae}\xab=\x0e:1\xae\xb9\x87\xa3\xae\xe2c\xe2h\x04\x98\x10,\x91y\xc7\x8f9x%g\xbc\xb0\xfc\x92\xbb\x97\xee\xfe\x0cy\xb73%\x9bw\x1d\xc4\x84\xcb\x92C\x13a.\xe1>\xb53\xb3\xa2\xcfI\x0c\xfe\xe8\x84\xfb\x93\x131\xeb\xc9B\xc1\x14Ic\xafy\xe2\xd0\x9c\x8c\xe6<\xe3\x84nq\x8e8\xe7\xf2b\x8c\xf1\x8e\x97\xf7t\xbd&"\xe0Jp\xbb\xb4ex\xdf\xbf\xa4\x97\xab3\xc8\xc9 \xe3\x04w\xe5COn`"\x94GS)5\x11\xcby\xe6\xdd=\xf1)mc\r\x13\x87)t\xdd\x15/\x80\xc2\x18\x1c=\xb7\xc0\x85\xfc\x1aC\xf8C\x8a]|\xd8_\xd9\xa3J\x8b\x85\xe3\\\xc6\x81\xa1\x0f\x90\xed\x1c#\xedxA\xd2\x14\xc6!KW\x81\xc5\xd4\xa6\xc3\xa3\xcc8\x18\xc5\xae\xc2i\xa4y\x0b\x9a\x90\xacy\xdc\xb4\xa0\x9f\xd47\xd3\xe1\xc7$\x9cg\x9eX\xb7=\x89Q/{\xc2\xbd\xd2\x0e\xe0,\x84%/gg\x9c\xaf\x87\xd3q\xcf\x84U\x99/A\xdc\xd4\x08\x85\xb1.\n\xe9\xda\x1ce\xa7\xbe\x0bb\x19\xb9\xe75Z1\xccA7s\xf8\x08P,/\x06\x90\x9e\xd3\x87lD`\x03r\xef\xb7z\xfaU\x12\xce\xf3i"(F\x10\xaf\xed\xe2\xaf\xc2Cq\xd9\xa1\x05T\x01\xd8\x03\xa3\xdfD\xb7\x91\x93$\x07\x91c\x9e\xbf\xa2\xc9\xad\x9d\xd0\x9a8Nn\xe0\xc8\xc5:2z1\x82\x03\xbe\x80\xdaEbtfd\xb8\x1c\x05\xb9z\xe8\xf6\x0e\xbb\xde\x97\xb9\xcc\n\xce\xd8\x9b\xae\xd1\xc7$\x9c\xe7\xd3$\x90u{\xbez\x18\xc0\x1d\x08\x988\xda\x13\x93\x99\x85iWZ%@,\xb2\xcdr5Nyk\xa9\x1d\xc26/\xce\xc09\t\x94\x83\xd1\xcc\xa1w\t\x99\xd4(,p \xa4 \xd7/\xfa\x9e\x15\xd6*\xf7\xa6\xa1\xcd:\x01\xf2|\x89\xe0\xe5G\xc6\xc1\x9f\x9c\x88y\xde\xdeH\x1c\xa7\t\xe4e\xeb\xdb\xbc\xd6\xd9\xe4z\x03\xf3\xe3H\xb7\xf4l\x9838k;3\xf02T\xe4\x00\xff\x91(\x8d&\xb0\xa6\xeb\xcb5\xbb\xd6n-\xc8\x9e\xac\\\xbb\xecZP%\x01\xe7`\xd4-\xd7\xb3\x97\x100*6i4\x13\xf2\xd3\xcb\x9bom>F\xc4<\xc3\x84(\x08\xc1\xd1\x97\xaco\x9f\xe1\xa9\x05\xf9H\xcd\xf8Z\xe2\xe2[\xbe\xbf\x86.T\t\xbb\xd8\x10\x12\xc6\x8b2(\xac\xa2\xcaD\xeaf/\x05\x8d[#\x1dd\x1c\x87\xc1\x19%jt\xf0\xa0\xb2\x04\xa9\xbe\xe5M`\xeb@f\xaa\xf4T\xee0\xf7\xcdK\xea\xc7\x88\x98gq\x83\xd2\xd8\xfa[^\x1a\x8cg\xa9\xa5\x07\xa6f\x81\x8a\x97\xe8\xc1\xe0^[y\xbd\xb2\x02\xd9\x08\x88c\x8a\xd1\xe9aH4\xc1\xc2kp\xaf$.\x16\x06W\xc9q\xb3\xc7\xd9\xbd\xa2 ;\x0f}\nB\x90wu\xa0+p\xb3\x13\xb8gk6\x8c\xa6\x1f\xdd\xde\xfe\xe4F\xcc\xf3\xdaD\xaf\xd5\x13\xf2\xfa\x96\xcfS\\7pQ\xd4\xdaa1\xde\xa57\x1eM4g>\xca\xf1\xf5\xa1\xe6\t\x05u\xe0\xb8S\x88V\xe9\x01K+\xe2FmL\xf2\x01g\n\xa9\xa2\x81\x07\x8a\xa4F\xe8=\x96\x15\x8f\xe3#d\xcad\x98Y\xd18+o.\x96\x8f!1k\x98\xeb\x1e")\x1ay)n\x02\xce\x99x\xda\x10\xa5c"#\x06r\xa6\x15q\x0f \x8c\x9f\x8b\x11*\xedo\xedZ\xe1\xcb\xa0)^\x13\xbc)b\xef\x86\xc4<\xf7\xc4ZU\x93(\xfc\x92\xf5\xaf\x96\x91/\xcd\x08\xee\xea\xc3\xd1\xbb\x1d\xa8\xb0\xaa\x06\xa3\x0c\x8c\x0e\x85+\x0c`\ne\xb8\xbb|;8\x8f~\x9f\x1e\xe6\xc3\xa1,\x99}\x8aQIP\x9b\xf8cBd\xcb\x9e,j\x1a\x9b\xfd\x94kGC\xb3\xab\xab\x17\xbf\x99\x0e?\x86\xc4<\xd3!\x8e\xaf5\x1c\xf9r}\xbbd\xe7\xee.E\x9c\x8f\x86l1\xa7p\xa3w\x17\x92\x89\xf3\xa4q!oghAGx\xa8\xd0$\x8a\xa1\xb8#"\xbbE\xc2\xf1\x10b\xf7-\xb0ku\xb0H\x8f\x11+$\x15\x92\x8e\xa9\x94\x85F\x15\xb9\x93\x9a\xbd\xb9h?\x86\xc4\xaca\x92\x14D>\xcf\xb8\x97{\xcd\x8e\x04\xd0K\xea\xee\xb3~\xde\xa9\x91|\xbd\xd6goZ\xfc\x02@8/\x17\xd4\xfdH\x9c\'\xa9\t\xee9v\x98\xd72\xa3\xf6LU\xc9t\x0e\xc5\xb1[\xd5\x8d\xbb\x89 \x9b\xac0\xc4B\xaa\x18\xb5T\x1f6!&?\xba\xeb\xffl$\xe6\x19&\xfe\xfc\xa4\x0c~\t3Y\xd8\x83\xce/,4\xd7\xcb8Kw\xb1\x12`\xc37/\xc7\xc7\x8d+\x9b\x16\xe6\x15\xf3V\xee\xce\xd4\x89\x17"OL\x85\xd2\xa1\x18Ua\x9d\xfdL\x08\xb7\xa2\xab\xef\xc5\x15\x8d3,\x8av\x99O\x9b\xf8\xc5\x8f\xdc\xc3\xf7\x92">\x86\xc4<\x0frh\xad\x83(\xf2\xe5\x84SNWli\x04\xac\xd3;K/\x99\xdb\xf9B\x1c\x93k:\x9f\x99\x93\xf88\xb0}\xd6\xbb\xb1\x06\x1ch#\xaaeo\t\xfc\xab=\xef\xd2\xf0\xa4\x16\xe2\xad?8E\xb0\xf0{\x95F\x8fg\x06\xe7=L\xbb\x94*\xbb>\x97_\x84\xc4\xac\'\x1cJ\xc3$\x0c\xbd\x94p\xd2>\xbe\xb8\xf0\xfe\x0056\xe9;2\xe4\xd4\xfd\xcd\x94G\x0b\xae\xfdj&\x81\\&\xaejy\xc3\xdbZ\xd9S.e\x86\t\xd0\x1d\x10\xf7v6\x90H=\xb02?^\x9c3\xc9\x15\xc7+/g\x9a\xc7>4\xd7i\xde\xbc\xbd}\xcc\x88\xa1\xfe\xba\xd6\xa9\x04LA\xf0\xcb\x87R3\x83\x0e\x16i\x85W)\xa9\xefMXw\xa6D\xdeR\t.\xbc\x13\xca\x16J\xa9\xe8\x8a|3\x10+h\x0fp\x8f\xd7\x0fn\x7f* \xba\xe0\x8c\xe9\xe2_\x83\xc8\xba\x9e\xe4\xc8\x9c\xe6\xbb<\xa1\x05y\xa6\xdc\xda\xc9\xdf\xa5\xdb>f\xc4\x8d\xadG9\xfcr\x8e\xd7\xc5C\x7f\x98\xfa\xd91\x93Cu\xc9\xe0\x06\xd0lj\x0fK^c\x90\xedTL\x92\xdb\x00g;\x98/\xd8S\x85\xea\xf8\x0b\x85A\xc5n\x0f\x9e\x1f\x97:\x98O4\xe6\xa3~}\xc7U\xd4h\xba)\xe2\xda\xba\x8e\xc3\x1f\xa5\xab?\xb9\x11\xf3\xac\xf7\xd7\x7f\xf3\xf9\xdd\x80\xdf\xcf\xe2\xa2Vs\xaek\xbc\xeaV\x08\x14\x19\xc61\xa9\n\xae\x82\xd3~\xc9\xef\x8b\x13\x10;\xc0G\x90\xf0\xe2\x9c\xc8\xd2\xb9Sq\x06\xea\x9c\xa63S9w\xc3\x19\x88\x1b(\x81k\xf8t\xb9(*s\x8eD\x0f\x06\xc7\xb8z\xfc\xa6\xc1\xfe##\xe6\xb7qoF\xcc?\xfde\xf5\xff\x1c#\xe6O\xd4\xe5xk\x1c\xbd\x191\x9b\x11\xb3\x191\x9b\x11\xb3\x191\x9b\x11\xf3\x8d\xc7\xb9\x191\x9b\x11\xb3\x191\x9b\x11\xf3\x9dK\xe8\xff\xe3J\xfd\xcd\x88\xd9\x8c\x98\xcd\x88\xd9\x8c\x98\xcd\x88\xd9\x8c\x98\xcd\x88\xf9\xc6\xe3\xdc\x8c\x98\xcd\x88\xd9\x8c\x98\xcd\x88\xf9\xce\x04\xc3\xffyT\xc4\xc6\xeel\xec\xce\xc6\xee\xfc\x9f\xc7\xeelF\xccf\xc4lF\xccf\xc4lF\xccf\xc4lF\xcc7\x1e\xe7f\xc4lF\xccf\xc4lF\xccw\xb6W6#f3b6#f3b\xbe\xed87#f3b6#f3b\xbe\xb3\xbd\xb2\x191\x9b\x11\xf3\x86\x86\xe0\xfc\xcdV\xf8\x87\x1a\xc2\xe5o\x87\xf4\x8d\x8d\x98\x7fC`\xbf\xc6\x88\xf97\x04\xf6k\x8c\x98\x7fC`\xbf\xc6\x88y3\xb0\x7fE\x1c\xf95F\xcc\xbf\xe1\x89\xfd\x1a#\xe6\xdfqxlF\xccO1b\xe0\xff\x99\xf3\xbfo\xc4\xd0\x14\x841$.0\x14\xc7\xa1\x04\x01\xf3$C\xf2\x84\x88\xf2,\xc7\xc0\x08G\x92\x14N\xf3\x9c\x88R\x9c\xc0s4\xcaC\x10G`\x02\x8e\x89\x0cO\x118\xf4\xf7\xf9\x03H\xc0\xd0g\xcb\x0c\x94\xc7q\x1e\xc3\x08\x14FI\x01\xa10D\xa0\x9e--1\x8abp\x84\x13\xd7XQ\x02\xc7p\x01Fi\x11c\x18\x02%E|\x9dy\xf4g\x1a1\xff\xdd\xcb\xe2/\x7f\xd0\x80\x01\xc6\xff\n\xe1\xcff\xdb\xe8o\x9d\xc8~d\xc4|\x7f`\xe7\x0f\x8c\x18\x18bh\x91X\x07\x06\xe1\x1c$b\xe8sP4\x86@\x14F\x89\x08Aa\xc2\x1a\x11\xb1\x8e\x19\xc6\x05\x9a\xe71\x81b\x05\x98\xa6a\x9a\xc7)\x1cf\xa1gC\x97\xcd\x88\xf9\x89F\x0cM1\x04\x8d\xe2\x18KQ\xeb&[\x8f\xa0ul\xdc\xfa\xb4\x05n}\x1c\xdc\xba\x96p\x01\xe2\xc8\xf5Q\xb0\xb0 \xf2\xcf\xfeZ")\xb2\x08E\x90\x98H\xb18\xf1\x97?\xb9\x11\xf3O\xf7\x0f\xff\x94\x11\xf3\xd5\xd8\xe4\xef\x1a1\xdf\x7f\x9f\xffZ#\x86\xfe\xb1\xb8\xe1\x92X\x12e\\\xcaV\xf9}\xd1X)<\xb8\x97\x01\xabT\x0e\x04\xc3\xebc\xd0b\x0bs-c\xdcY\xa6\xb4d\x1c:h\'a\xdc\x8787{\xa1\\\x89\x89\x86\xe9\xf8\xdc\x9a\xda`\xee\rfYpQ\x1b\xd8\xe9M]\xe0SF\xcc\xef\xb3\xc2\xdf\xb4\x1cL\xc0\x01\xe0\xbcD?P\xb0?\x07\xc6\xd1\x84\x85\xca\x06!CG\x81K\xacyq\x1f\x89\x95M\x8b\xfe\x8e\xe0\x94\xb8bi0\x83;;\xcb\xfc<\x95\x95\xd2\x05\x1fRd\x9fG\xe0\xe4\xd3wa\xb8\xca\x00\x93\xc0\xe0\xbbV\xc4\x87\x8c\x985L\x98\xc6\x08\xf8%\xc6\x93\xbe\x00\x8f$\x82d*K\xa3\xeb\xa2Q\x11i\xa0\xa7\xd3\x85\x8f+S\x90dC\xb9-\x1aG\xcej\xe9\x9a\xceh=Nn\xbc>\xaaQd\xc4<\x9f&\x04Q\x04L\xa1/az\'%\xee\xab\x9dP\xf0K\x92V\xd0\xe2zV\xbdcv\x9d\x84=\xe2\x899{s\x7f\x87Z\xbc"\x1e\xa8i -\xc5\xc4i;\xddO\xec.\x8dS\xc9\xa1\xef\x12\x95\x13\xb7\xd2\x1b\xa2\xc2\xf5\xcd=o\x80\xc3-\xd2\xdf<\xc5?e\xc4<\xc3DQ\x12\xc5\x11\xe4\xe5 \xb7\xba\x88\x95\xe5\xe3\xcd\x05\xb9\xa1A\x1c\t\xa1\xea\xc7r\xdeGi\x08\x1c/\x8d\x94a\xde\xa3\xda\xa7\xf0\xfe\x02\xe7\x19b\xebU\xd4\xd1\'\x10&\x86\x9bv(|\x89\xb3\x80\x9b\x05-\xd7\x16?\xf0I\xa56jQS\xb6\xf2fk\xecO!1_a\xae%\x0cI\x91/\xed\xe2]?\xb4v)`\xf0\x91SS\xe1\x037\x18\x9ea\xda\xc70\x9fj\xff\xb4\xf4\xd5\xe3\xd2\xeeo\x17\xafq\xca\x87\x8b&)"\x08\xb3\xdcN\xa7b\xdc\xe760\x0e\xac\xb9\xde\xc2\x8d\xb6\xcb\x1b\x84\'\xce\x16]Fg\xee\xf4\xe6\xde\xfc\x14\x12\xf3[\xe9\x01\xd3(\xfa\xda>\x92\x80\xea\xe5\xe4h\x0b_\xb2\t\x0b\xe9\xc0\x99\x86\rTSbT\x8aO\x01\t)X"\xd0\xc2\xc9\xdc\xcd\x81\xae\x1e\xe6}u#\x99\x93\xc2_\x84E\xd6\x8b\xa8\xb9\x8d`;7\xf2H\x04q\x96\xb7j~%\xe4=\x89\xf3\xff\x91H\xccs\xb1\xacY\x0f]\x13\xff\xcbb\xd1SY8\x96>\xdf!pv\xf4\xe9\x01\xa1\xd5D\xbd\x0cG\x12p\x0b\x83\x1f\x08\xb7\x1d\xda\x9b1\x12X$\x8a\xf0R\xecZHz<\xbf7[_\xaa\xd3\xc5\xf0.\xc72\x05\xe6\x93uF\xa9c\xe5\xec.g+\xa6\xe77\x9b)\x7f\n\x89y.\x16j\xadt\xd7\xe3\xe3\xa5\xfd.\xde\xc39R.\\\x00v\xf62\xe2\xc5\xac\xd2\xd62\xed\x0c\x9e\xbc\xc6\xd9\xec\xdd\xc5]\xb6^`-\xe0\xb0\x83\x14\x85\xd4\x84\xeb)N\xa0\xe48\xa4\n\x9b\x08g(jb\xa0\x0e\xeeN(d\xb9<\xdd\x0e\xc9,\x9b(\xf3^\x98\x9fBb\x9eO\xf3\th\xa2\xd0k\xf5a\xe8\xba\x19\x1a"}\x04P\r\xc5\x99\xd2\xf4\xcev\xd2\x1d\x1fik\x86\xf8\x80wB\x0b*\xc7\xf8\xdeg\xf1R\xda\x01\x80?\xd8x\x7f\xb4\x8e\x97\xd8\xd9\xdf\x19\x07\x194~\x8f\x9aY\xea\x08\x02\xd1`\x91\xad\x1d\xe3\xf8\xf0\xbd\x9c\xbfO!1_\x07(N\xa2(\xf2Z\t\x03m\x15\xaekd!\x0fK\x95\xecyo\xbd\x12\x06\xc9X3}\xd9C\x967,X\x85k\x16\xda=\xc7\xe1\xce\xc5\xde\xb4\x15k0\x1eQH\xf9\xf1\x84\xc0C\xd9\x1b\xac\x0fr\xc0\x89\xf6\x15\x01U\x11\xb7\xb8\x04\xdc\x9b\xe9\xf0SH\xccW\x98\x10\x84c\x04\xf4\x12\xe6c\xef\xd7-\xd8\x8b\x83e\xa6\x90=\xeeP@t\xe9\x07\x10\x88\xc4\x1eU\xef\x8f\xf64\x11X\xdf\xda\xbc4%\xf9#\xb7\xcd\x05\xd2\xe1\xc7\x05\xf3\xfa\xf9$(\xe6\xa2\x0f\x9e\x1c&\x17\\\x7f\x1c5\xc8\x819\xd4 \xcf\xf77\xf7\xc4\xa7\x90\x98\xaf\x13\x8ez\xd2I\xc4\x0b+\x82M\xb3Y\x99;\xf0\xc1\xe74\xbc?\x05\xa1\xd5\x0f\xee\x11\x92\xd1\x1a\xbb\xf3\x9dP,\x13y\xe8O\x82x\xbf\x17\xfbl\xd0@\xb1\x85\xbcr\xc4#\xac\x05\xb5G\xae\xe3\x8e2\xed\xd3\xd9\x9c\xf1\x00\t\xee\xbe\x83\xf1\xe2\xf1]\x1f\xeaSH\xccW\x98k=H\xe0\xd0\xeb\x0b\x07\x139Kp\xf2l\x17o\xf7\xe1\xdc\x04\xa4f\xa8\xda\xee8\x8f\xc9r\x1cN\xc3\x08\xa7jv\x02r\x10\x06\xb3\x1b\xc5\xc6-a_`\xfa\xb6\x0c\x8fz\xdf\rJ\xa2A\xd41\x1a\xb5\x0b\xc5$EC\x06\x88`\xdb\xf4\x9b8\xe5\xa7\x90\x98\xafRu\xadn\xd0?\xa0\xe1n\xcbZ?\x14\xb1\x1c\x8c>\x96T\x11U\xf8\xddH\x1b\xb2}\x9d)=\x947_\xdc|\xca\x88\xf9\xba5\xd1kF\xa1\xe9\x97\n\xb1\xc4G\xe1q%uiv\xb4\x9b\xef\xec\xe6\x90\xba\x07,5h\xe9\\\xbat\x85\xc5\x86\x92H\x18h\xf2}\xa3f\x14a\xd52_\x14\xb5\x88\xc0\xf7\xc5\xa0\x89\x1b\x990\xd5\x94\xa8\x87\xb3\x19Y\xfa\x11!\'\xe1|\xb3~\x84l\xfdd#\xe6\x19&\xb2VC(L\xbc\xec\x89\xbd\xc6\xf6B\x03\x84\xf7N\xa4\x05\xd5[\xda\x98\x12\xda\x9c\xa6\xf0%+X\xd5\xb0R\x10\xee\xbcs3\xae\xf9\x03\x81s@h\x83\xfc\x0c\xf3\xd6\x98\xf8\xca\xf9:tt_,L\xaf\x1e\xe2\x10Vx\xfd\xd0\xd1\xe5\x99\xf1\xde\x94\xe1>e\xc4|m}\x98\x84H\xfa\x15\x85t8\xe6\x06x$\xc8\xd3\xa4r\x882V\x110\xdf\xa3e\xe5\x12\\\xe7[\x9a\x12w\x0e`5\xbdHO5#\xce\x87\xe5bRT)\xddD\xcc\xb9C\x82}Aq\xe1\xac\x01Ni\xa3\xc34\xab\x81C\x08\x88\xd2\x10o.\xdaO\x191\xff]\x90\xaf\x15\xdc\xcb\x8b\x9b<\x16\xaeUx=Bgg9,\x01iZ\x92+\x15j\xd0\x03A\xeb\x9ayo\xd6\x02}\x03n\xd7{G.3=\x08$U\xd3 \xa1\xe5\xde-\xcb3\xcb\n=\xef(wB\x86\x1b\xc5\xf9\xcc\xdfA\tP\xe8\xe8\xcd\x12\xeeSF\xccWm\xf3\x04\xcd\xe8\xd7{\x07\xf1\xb0m;\xb6\xc5\x0c\rn\xb3+6"\x8e\x13LJ\xebz\x04\x1ef\x9d\xc0\xc4\x1dq/&\xb9\xf42;:\x15R\x02_/\xfa\xae+}\xae\xcf\x16\xd0\xbdJ\xd1dKN\xa9_\x8f\xaa1;\x92\x1c\x00K\x8b~\xaf\x83\xfcSF\xcc\xd7\xd6\xa7\xd7G\x02S/\xf5~\xe9g7k\xf4l\xe1\x91\xa4\x18"\xb4.\x98F\xd0M\xcf\xf1\xbd\xb1L.\xdde;\xf5\xe0\xee\x8c\x9e\x96\xcf\x94\xe9NU\xdd\xf2\n\xb4\x10\xfe\xa4_x@\xd6\r~\xe4B>:@\x03\x9d^\x1c#\xbd\x9e\xc3\x00~\xf7\xd5\xf4\x87\x8c\x98\xaf7\x1a$\xfc|-\xf8\xb2X\x84\x8c\x92\xae\xecnw\xec\x0f\x83\x1f\x1e\x8e\xfaN>\x9d\xc2.uw$\xda\xd9\xf29A<\xc3+bF!\x14\xbe\xbc\x16\x86uI]\xeb\x94\xc9\'\x9a\x94\x90\x07\xc0\xd1~\xa7\x1dk\xa4J\xa3\x9c\xa4\xc1\x1b!g\xd7\x1ff\xfd\x9f\x8c\xc4|}\xf2\xb6\xc6H@\xaf\xe2\xcf\xed\xc1[\xa6\xe7\xdd\x0e\xf7\x05,2^\x1c\xb1\xfb\xe3\x0c\x1a\x1d\xd2V\x1e\x1eN\xbcr8\xb7\xeb\\V<\'\xe9!y\x18\x11\x1bE\xb8\xe2\x88\xb9\xcd$;\xa3:\x9f\xaf`uC:4\xad\x9cj\xb6\x1b\xc5\xa7\x1e\xfb7\xdfO}\n\x89\xf9zq\x03Q\xeb)\xf2\xba\xf5\xcd\xc3L\xd7\x03\x1bI\x9e\x81paqH\x87\x04\xc9-\xa6\x99K\x8aQS%sE\x06\xcf\x9b\xe3h\x8c\xb4C\xb0\xe4\x90*b\xe3\xa2\xbbki\xce\xb9\xe8\x16\x93\xd1O\xe3\r\xd2\xdb6\xf0\'n\xe7*\xb6\xa8I\xdf\xebZ\xf3)$f\x9dE\xe4\xf96s\xfd\xb7_\xe0\xa4S0\xf0\xb8\x06fj{\x96\x91Z`:S\\0l\x07\x80\xb2B\xe7\xe1z\xd8\xa0\x8bR\x1ckY\xe2\x8b#\xe8)\xa2\xa8\x02\x13\xa7\xf9%|!\x1a\x83\t\xb2\xd6\tJ\xa3Z\xff\xc6\xf0\xe7\\\xea\xa0\xd3\xc5\xb6\xd8\x7f\x0e\x89\xf9z\xfa\x1b\x12\xf3O\x7f[\xfd?\x07\x89\xf9\x13\xf1\x1b\x9bh\xb2M\xe9\xf7\x9f\xd2\r\x89\xd9\x90\x98\r\x89\xd9\x90\x98\r\x89\xd9\x90\x98\r\x89\xf9\xc6\xe3\xdc\x90\x98\r\x89\xd9\x90\x98\r\x89\xf9\xce\xf8\xca\x86\xc4lH\xcc\x86\xc4lH\xcc\xb7\x1d\xe7\x86\xc4lH\xcc\x86\xc4lH\xccw\xc6W6$fCb6$fCb\xbe\xed87$fCb6$fCb\xbe3\xbe\xb2!1\x1b\x12\xb3!1\x1b\x12\xf3m\xc7\xb9!1\x1b\x12\xb3!1\x1b\x12\xf3\x9d\xf1\x95\r\x89\xd9\x90\x98\r\x89\xd9\x90\x98o;\xce\r\x89\xd9\x90\x98\r\x89\xd9\x90\x98\xef\x8c\xaflH\xcc\x86\xc4\xbc\xc3!\xfcM\xa5\xf6\x8f8\x04\xe7w\x1b\xe7;#1??\xb0_\x84\xc4\xfc\xfc\xc0~\x11\x12\xf3\xf3\x03\xfbEH\xcc[\x81\xfdK\xe4\xc8/Bb~\xfe\x13\xfbEH\xcc\xbf\xe1\xf0\xd8\x90\x98\x9f\x82\xc4 \xff3\xe7\x7f\x1f\x89AY\x02Fp\x88Bp\x0c\xe2\x08N\xc0H\xe1\xd9\x93TXC_\x07\xc0"$\xcf\xc10\xf1$\x11\xd6\x113\x08\xca\xe0\x08\xcb?\xfbz\xc24*p\xc2\xb3\xf1\xc5\x8f\xfd\x03\x82$\x11\x8a\xe3E\x82DD\x06\xa5\x19\x8a\xc19R@\x19\x1e\xa7\t\x1e\'\x18\x98\xa3I\x18a\x08^$\x19\x06^+\x02\x08\xa6\x10^\xc0h\x08\x178b\xcd\xf2?\x11\x89\xf9\xefNZ\x7f\xf9\x83\x06\x0c\x08\xf2W\x88$\xa9\xe7\xe8\xe1\xbf\x87\xc4|\x7fa\xe7\x0f\x90\x18\x12bD\x8e\x10\x10\x96\xa6hF\x10HJ\x84\x04\x1c%\x19\x9c\x80(\x9eA\x08B\xe4\x10\x0e\x11\x9f\xdd\xe6\x04v\x1d7\xc9\t$\xcca\xd4\xfa\xfb1\x8c$\x9e\x83\xde\x90\x98\x9f\x88\xc4\x90\x14\xcc\t\x14!\xf2$G\xc3\x02AA\x04\x82\x920\x81R\x04L\xaf\xbb\x10\x87q\x1e\xa1\xd79\xa6 \x12\xa1i\x1c\x85\xd9\xf5\x173,&\x08\xeb\xfa\x80\xfe\xfcH\xcc?M\x97|\x0c\x89y\x1e\x0c\x7f\x17\x89\xf9\xfe\xfb\xfc\x97"10\xf1c=\xa5+l}\x96\xef\xbd\x0b\xc1\x1es\xb5\xf7\x14\x9d\xdbwP\xb8\xc44\xeazA \xc1\xbb\xa3\xe3\x94\xae\x96$\xf7b\x84\x02\xb1B\xddn\x01\xb1\x82\xb9\xfa\r\x8c\n\x85\xdb^[\xea\xac#\xd19\xcax\xcc\x7f\x8c~+\xbd)\x8b|\n\x89Y\xb3\xc2\x9a*)x\x9d\xf5\x97\xd6\xb4u\xc0\xd8Ty\xb0\x1e7\xc9\xea\xdb\xcbxO\x0c\xe4\x82\x82\x90H\x1c\x03\xf3\x94\xf7\x83\x08A,\x19_\x14E\xf6\xcc\xd3-\xc4\xe8\xfd^dodN\x85-W7eZ\x1a7\xb8\t\xce\xe8\xc3\xd9e\x81\xc3\xef\x12\x11x\xb3}\xdb\xa7\x90\x985\xccu\x9a\xa0\xd7\x86\x83\xf6md\x92"T\x0bW\x93\x82\x87\x1df\xd0>\x1a]>.\xfbT \xbc\xab\xe6\xe8$a*\xb5KGU\xba\xb1\xa0\xd0\xedM\xb4R\x9ay\x04v\x13\xc7i\rp\x1d\x85\xfeX^\x16\x05\xdaY\x99\x80\t\x9a\xf2f\xd7\xb1O!1\xcfGI\xd2\x18\x8dB\xafM\xf7s\x9f=\xd7\x94\xad\xab6\x8cH\x16\xe6^F\x99\xd8\xd6ly\xf8\xa8\xe5\x9a\xb4?\xdd:T\x07\t\x1b\xc8\xce\xc9\xe5\xcd\xc5\xf2)$\xe6\xb9X\xe0\xf5\x1cF\xe9W[\xa0\r\x984 wG\xc1ni\xc8g\x16L"\xf7u\xe1\xa9{\xc59J4\x18\xb01\x0cN\x98\x11#Z\x07\x99@F\xd8(\xc5\xe8\x91G\xdc\x8e\x19#\xb8\x95\x17\xa35_\xf8\x84b\xdf\x14O\x18Y\x9cb\x8f\n\xf7\xde\xf1\xf6)$\xe6\xf94)tM\xda\xe4\x1f\xf4\x1au\xc1\xc3\xb9\x8f\xabF\x84\xd3\xce\xf1\xe4k{\xbb\x1d\x05\x97\x01\xfbl\xba\xb5\xd6\xd0\xd1\xc5!W\xaf\x97\xc1\x03\x07\x9cp\xa3\xabw}t-;\xaa\xce\x9a\xe7[\x02\xe5\xf8\xd3\x05\x81\xc4\xc2\xea\xef\x0f\xdb\xcb\x81\xd3A\xba\xbe\xd9\xa2\xeeSH\xcc\xefk\xfb\xff\x1d\xa6\x8f\xc9\\\x92\xdd\xf0\xd6w\xd9F\xdds\xc7:\xca\x95\xb1W,\x0b\x8dv\xedm?t0\xa8\x9f!Y\x05J\xb0\xf3\xe6IvN\x07#\xb9 W\xa2\xc4\xfa\xd8A\xfd]hZ=\xa9\xef\xebhvI\xb5?\xba\xfb\xf1\xcdE\xfb)$\xe6+L\x04Z\xcbG\xe8\xa5\x0f\xf0\xb3I{w\xf1\xa8<\x9e\xe1\x90\xc0}H8\xa8\xdcAAU5\x02\xe8\xa2\xc2#@]R4\x9d=\xa13#|\x02&\xc0vB\xf1\xa8\xc2c\x88\x96\x0fr\x8f\xaa\x14pl\x1f\x18 \x9c\x02-\x87\xdc\x13\xb0#\xdf\xec\xa3\xfe)$\xe6+L\x08%\t\xfa%\xca\xc4\xf2vg\x02=\xe5^\xb6\'\x0b6\xce\xad\xe9tS\xb5\x89Ke\xca\xb8\x88\xb6\xbf\x00i\x7f\xc0\xd5\xeb\xb9`\x06^\xaa%Z\xb9\xe9\x9e\ta~,$\x15\xa0vL3\xf3\xf8\xe9\xba\x1f\x0c\xfcj/\x8e\x16c\xb77\xd7\xec\xa7\x8c\x98\xe7\t\x04\xad\xf9j-\xfe_\xbaG\x921x\x11\xb9#\x88\x01\xf1%`Y\x9b/\x04O\x8eYP\x1d\xaa\xc6=\'\x0f\xf0N\xa5\x1dr\x07\xba\xe4\x91^,\x833L\x03\xbbh\x16\xa0%\xd2\x15Cj\x15[z1Kt\x96\xef/))\xd5\x01\xd7\\\xa1\xef\xd5Q\xf5SF\xcc\xb3\xb0\x81Q\x04Aq\xec%\xe9\xd3\xf2\xd2\x1d\x9bE\xe6j\x14\xf3\xd7"\xfe\xe2\x05g\xc3\xd3\xa29\x97\xd62\xbc\xeeS\x8a\x0e\xed\x02W\xfb\xcb\xbeJ2\x05Be\xc3\x97\xda\xab\xdc\x1ev\xcc\x10\xca-\xdbh\xa4\xe7P=\x15\xc4\'\xaf\x14\xcf\x84\xa1\xe0\xef\xa6\xab\x0f\x191_\xe78\x86\xad\xf7F\xe2\xe5\x80[\xc2\x98\xa5\x10r!\x9c\xf3\t\xcb\x85\x13\xa5qb?\xe9w\xc9\xcb\xae\xe5P\xa91\xcd\xca\x11\xb8\x14\x99\xfb\x80B\xaf\xc6g\x06e\xce\xe7\xb2\xcc\xfb<\x1e\x19\x89v\xfb*\xd0\xeb.\xb9\xc9\n\x10\xcf\xe7\xa0\xe8\xebw\xd9\x8fO\x191_\xc5\xc7zUZO\x8c\x97t\xc5\x10g,Mk\xafG\xf0ND#\x86\r\xd2\x87e\xdd\xbc\x89MN\x86\x1c\xdb H\xd9A\xba;\x8d\x18\x87\x95\x88\xad"\x0fu\xc4\xef\x88\xa0\x0e4%\x1b\xcei\x02\x9a]i4y&\xd5\x93\xe0\x9bq\xa3]\x9a\x1f\x85\xf9\xe76b\x9e\'\x0b\x05\xa38L\xbf\x12\nl^c\xd7\xfb\xb8\x8b\xe9\xc3\xa3\xee\xee\xc5\x81H\xf1\xdc?\xe02\x9aFqb*\x8d\x00\x87\xc2\xd91\x10_\xc2\x8e\x99\xdf\\\xab\xb5\xf20\xe5\xab-\xf2~\x7f7\x91{\xc3\x9e]\xc8\x0ea\x9d\xb7\x04b\xcd\xa7Wo\xff\xe6\x9d\xe6SF\xcco\xb5\xcd\xfakp\xfc\x95\x87R\x07\xbcj\x82\xfd\xfe\xc0\x00\xacu\x1f|\xdf@\x97=\xc8B\xc0}=\xf2[\x0c\x1a\xccp\x8a\x06f\x8e\x04\xcf\x8e\xc5&\xd3 g\xbd\xca"\x12|\x8d\x9bC\x08\x98>\xe7e5U\xec |\xafu\xd4\xa1Lr\xec]\x1e\xeaCF\xcco\x95\xea\xf3u\xee\xab\x9b\x14(\xf9N\x1f\x87\x1c\x12\x0fp#E`x\xc1\xe7\x9a\xa7\x95\xf1\xc2\xd0{aq9o\x0c\x8f7\x15\xee\xfa\xfa\xee\xef\xcc\xdb\xac\xac\xf7o\xba=\xdf\x01\xd8\x99oN\xdf\x1f:C}\xa0\xe7\xb2`\xda\xd2\xa2\xcfH\xea\xd5\xbf\xc8\x88\xf9:\xc8\xd7\x14\xb0n\xfe\x97\xd6\xd8]s\xc3\xcf\\\\\xf0\xc4\x15\x98O\xa2{r"9\xa4Tc\x88\xa1\x0e\xc1Aiw4\x99\n\x9d8*\x85\x8a\x9dG\xd5\x07-Q\\\x93\x99}\x8f(\x8d\x10\x17c\xffp\xbf\xdf\xee\x87\x06\x00e\xdb\x95\x1e8\xb8\x1c\xde\x04\xf0>e\xc4|\xd56\x04\xb6\xd6G/\x0fs\x9ew\xb1G\x1d\x1a\xb8o3\x10\xc2\x84e\x0fL\x98\xa5\xdf\xcb\xc3\xb5\xf5(\xb4Z\xb4"\xb2\x02\xbf \x1fx\xb0\xe3\xca]I\x9e\x8c\x91}<\x90\x1eb\x94p\x90\xd5\n\xae\xfb\xf9xQT\xebz\xbf\x1b\x80\xb4\xde\xc8\xbfW/\xe5O\x111_[\x02\xa7\x08\x1cA^f\x11vr\x14\x9b \xfeD\xd5\x14G0\xba\x9ff\xd4yO,hz\xed\xac\x13`\xe0X\x87\xfa\xba\'\xfb\x9d\xeeh\xca\xfe~H\xf29\'2!v\xfdz\xc7ig\x7fZ\xc2\x0c+\xa4Iw\xfa\xc7\rd\x1a$g\xdf\xa4/?e\xc4|eC\x8cX\x93!\xf9*|\xe2\'ki\x05\xc0\xd7\x9cb\x8a\xe7\x19;\xedE\xac<\n85\xa0 r8\xcc\x90\x99\xedX\xad\x8d\xe3G8T7\xf5\x98[\xe9\xc8#\xf7\xe9\x9c\x9fN\x94\x00\x94\'o\xf0|\x16\x8fD\xbfG\x1ceP\xb0]\xf1\xe6\x1d\xf5SF\xccW\x98\x10\xbe\x960\xd0K!|\x9a\xf5\xc1y\xb8D\xec\xebG\xfda\x8b\\\xe0\x90\x12fJ%\xc0\x04z\xc0\xa8\xdd\xbe\xaf\xf6\x8b\xea]n\xe7\xd3\x10\x80K\x03b\x07\x8a\xb1\x93\xb3?*\x8b\xdbT-\xe5\x9b\ty\xc7\x1fM\xd5{w\xfd\xde\xf0\xc8\xe9G\xb7\x9a?\xb7\x11\xf3[6\xc4Qj\xcd\xfc\xbf\x9f\xc5\x867F\x07R\n\xb3o\xe0\xa0\x85\x085\xa7\xe8$\xd4.\xa2\xa7\xea\xf8=\x9c\x914K\xb9\x08\xc1\xe8F\x88\to\xae\x11\xab\xbd]\xa3N\xdd\x9f\x0fc\x1d\x87W\x8c\xb2v\xe8L5g\x95\xf2L\t\x8d\xcd\xab?\xbe\xd9s\xffSF\xcc3L\xe2\xe9\x13\xfc\x01-P\xfb2}\x1c\xd3;\x93\x042\xde\xcd\x8f!\x08\t\xe0\xbe\xaf\x10\xf2\xa0\xf7\xb8\xb3\x16\x8dN` W\xcc8%|F\xe6\xaaY\xc2\xe5\xa9cf\x85\xec\x0e\x86\x8f_\xba\xdd\x01\xbc\xe6\x88!,\x94!]\xcb\xf2\xee\xa5\xbf\xca\x88\xf9z\x9a\x14\xba\xd6C\xaf\xd9\xd0\xbd\xd1\xc7\xdd\xf9\x91\x94\xd8\x01\x13M\xcc*NE\n9FS\xd2\xb7\xb4f\xe3\x0c-\xf9\xe9\x12\x9d\xd9R\xd2R^\x8fx\xc5\xda\xe3\x82P)\x87n\xde\x8d\xc7\x83\xaa=n\xb4>\x17w"5\xdd\xa0\xac\xc1\xcb\x95\xca\xde\xc5S>d\xc4<\x0f\xf2u%\xacq\xa2\xafo\xe1D\x9f\x92r8\xe4G5\xc9\xad9\x17F\xac\xcc-Tf\xa6x\xaf\x19\xeb\xa9uK\x01\xb2+\x1a^\xbd\x87&\xe5\x90\x05=r}\x9a`(\xb7pP_\xa5\x8b\xac\xb4-\xaa\xa67\xf9\x84\x1f=\xe16\x19;\xf0\xcd\x17\x1a\x9f2b\xbe\n\xf2\xf5jD\xc3\xaf\xcc\x9f\x973e#\xe2\x14\x9b\x83\x00F\xf4\x03\xe3\x9c\xdd+\xcdF\x141\xe8}\x8b\xe8}\x1fGH\x16G\x8a\x99\xc13]\xc2\xd2q\xd2\xae\x05\n\x9e\x0e\xe4\x98sx*\xe9\x97v\xdd\xd6\x0f7\x1b\x0e\xde\xd1\xe2\xf5\x8bW\xbfyI\xfd\x94\x11\xf3\xf54\xf1\xf5\xcf`\xf0\xcb\xdet9b\x0f\x98\xb8lq\n+,\x0c\xdf\x85\xb8\xc2B\xd2i\xba\xb2F\x81\x08\x90\x1c\xd99\x9cO\xa39\xf0\x8c\xa9\xb5G\x1b\x80\xb4+:\xa4g\xbc\xa0\x99\xc5\xe9l\xd2b0\x00\x05a\xa4,\xc2\xee!\xcbi\x8e\xffh\xd1\xfe\xb9\x8d\x98\xaf\n\x91F\x11\x82|UE\xa6\xc1\x99\xda\xc2\xb8\x9f\xb5\x87h\x0fb\x14\xe9Hz2.G\xa3\xcb\x17\xae@,[=\xd4\x85\xe6\xc7\x99O=\xee\xc78<\xe5\xb0\xdb:\xb3\t\xd2\xfc0\x89w\xf0\x1e\x92\x17+\xdb\x05\x98+\xa6j\'\xbb\x05}F\xdf}3\xfd!#\xe6\x19\xe6Z\xf1\xa1\x04\x84\xbf,\x16\xd4\xd4\xe8\xe0>9G\xa9Cc|\xe6\xae&\xa1Y\xb4}\x1d\xa0G\x9e\xf6\x91!\x94\xf5\t/\x14\x0f\xafj\xf0*\\P9M+\xe9\xf4\x08\xae\xe1T\x16UP\\g\xfd\x92W\xb1\xec`\xb6\xfe\xa0<\x1dp0\xeb\xcd7\x1a\x9f2b\x9e\xd7\x9a\xf5Q"\xf4z\xca\xbd@\xa6\xec\xf5t@\x06\x0e\x0e\xca1\xb6\xe1\xe8\xa0\x11\xf5m\x98$\x0e\xe9=\xce0\xd4\xd4\x02\x08\xbb\x1b\xea\x12\x91\x9a[\x87\xab{2e\x81f\x8c\x1fs=\xe2\xa7h\x1c\x06\x1a\xedg\xca\xa1\xa6)\x0f\xcd2\x1e\xb5\xb9\xffEF\xcc\xd7\xd6\x87\xd7b\x95\xa0_\x16m\x16\xcd\xf1\xf1\x91\x9f\xc8\x9c\x9cGqg\xec%\x07\x80\xee-\xc6\xf2\xf7]P\'X#\xabs,\x8e\xd5T3\xb9\x9dk\xc3~h\x9b\xa1\x93yEB|%\xeb\xcb[\xfd\x10\x8e#\x0fP\x86e\x1f\x8f\xc7\xbe\x18\x19\xed{\xddk>e\xc4|-\x16\x92\\W\x0b\xf2\xb2\'\xb2\x9d\x14/\xd6x\xd9yd\x13D\xe7\xce\x10M\x12T\x1a\xf5\x80J\xb5<\x95!\xf3P\xaf\xf5\x15y\xd0\xc8\x8d\xb6A\xf4\xbc\x8cX%&it\xcc\xe0X\x98\xe9\xb0\xe1\xca\xccl\x92\x93\xa2\xcd$\xaaC~!t\x0c\xff\xcf\x191_\xb5\xe7f\xc4\xfc\xd3_V\xff\xcf1b\xfeD\xfdc\xb7\x96\xbc\x9b\x11\xb3\x191\x9b\x11\xb3\x191\x9b\x11\xb3\x191\xdfx\x9c\x9b\x11\xb3\x191\x9b\x11\xb3\x191\xdf\xd9^\xd9\x8c\x98\xcd\x88\xd9.\xa4\xdf\xeeB\xba\x191\x9b\x11\xb3\x191\x9b\x11\xb3\x191\x9b\x11\xb3\x191\xdfx\x9c\x9b\x11\xb3\x191\x9b\x11\xb3\x191\xdf\xd9^\xd9\x8c\x98\xcd\x88\xd9\x8c\x98\xcd\x88\xf9\xb6\xe3\xdc\x8c\x98\xcd\x88\xd9\x8c\x98\xcd\x88\xf9\xce\xf6\xcaf\xc4lF\xccf\xc4lF\xcc\xb7\x1d\xe7f\xc4lF\xccf\xc4lF\xccw\xb6W6#f3b6#f3b\xbe\xed87#f3b\xfeC\x8d\x98\xbf\xc9\x81\xffPC\xf8]]\xf7\x9d\x8d\x98\x9f\x1f\xd8/2b~~`\xbf\xc8\x88\xf9\xf9\x81\xfd"#\xe6\xbd\xc0\xfe\x15q\xe4\x17\x191?\xff\x89\xfd"#\xe6\xdfpxlF\xccO1b\xd0\xff\x99\xf3\xbfo\xc4\xe00\x8c0(\xcc\xd2\xc2\xd3ha0\x9cf!\x11y\xf6\xfb\x7fv\xf4\xa00\x98\x85`\x8a\xc3P\x86\x87\x10\x92\x11Y\x81&P\x0e\xe1!\x98!!\x92a\x9f\xfd=~\xcc\x1f\xc0\xc4\xd3H\xa0PV \x10D|vA\xc2a\x91D\x18\x9aa \x88\x13)\x01\xe3`\x84Dp\x0c\xa3PD\xe0\xd6\xff\xac\xc8q\xcf^P\x10\xcb2\x02A\xfcL#\xe69C?4b\xa8\xbf\x120\x85B8L\x91\x7f\xcf\x88\xf9\xfe\xc0\xce\x1f\x181\xec\xfa\x94)\x82&8\x14\xc7Dr\xfd\x8d$\x02\xa1"\xcc!\x08\xc6\xe3\x0cFr\xa2@\x910\x0b\xa3\x14%@0&`0\x07\xa3\x10\xc5\xae\xcbU\xc4Y\xea9\x8b\x9b\x11\xf3\x13\x8d\x18\x82\'I\x8e\x11D\x96\xa2h\x18\x15\tb\xdd5\x1c\x83\xaf\xfbH\xe0a\x8eda\x81\x81\xd7\xd13$\x811\x1c\xbd\xfe\xb2\xf5g,\x8cS0\xcf\xf3\xc4\xfa\xf8\xff\xf2\'7b\xfei\xeb\xe2cF\xcc\xb3\xb1\xc9\xdf5b\xbe\xff>\xff\xa5F\x0c\x82\xfe\xb8\xed\xbe\xc9\xb3\x91M\x86\xceNCQ\xe8\xa8k7\x17\x87<\xed\xa1\x86!\xcb\x9dB\x0c\x99\x15\x02\xe0]\xa1\x0e\x91\x0bs\xef\xe0J>B\xeb\xdc.\xea\xd8\xb9H\x0e\xd2\x82u\xab]\x1a\x1e\x93\xa98t\xc7\xccB\x02\xcd\x0b\x7f$n\xfcl#f\xcd\n(\xb1\xa6f\x04y!1,;"/\x9c\x82\x99\xa7\xab\xd1\x98\xb0e\x01\x11GT&\xdd\xb1\x96\xc5\xd5%\x86\xb4\xeb\x04y\x95y\x94]\xa2\xa5x]\x95j\x1b\x8b\x13\xc5>\xba>\xd1\xae\x85\xd0}\x10|\xec:\x15y\x7f8\xe9\xf7\x9a\xbc<\xfc7[\xd3~\xcc\x88\xa1\xfeJ>\x17\xfbK\x9f\xaa\xcbz@\xb2\x0b\xa0\x1dr\xb56\xb5\xa4\x83\xe9\x07\x8ajZ\xad\xdf\x15g\x1d{\xe4\xc6\xa18\x96j\x03\x16\x08r=\x83jw\xf5\xd6"\xc1\'\x9c\xdb\x8e8N\x08&\xa4jt\xef8[\xf7\xacn\xceZp\xe6\xd5\x8e\xfdEF\xcc\x1a#Ma\x08\x06\x11/}\xaa@\xf7p\x92h.\xcd\x02S\xde\xbb\x0bs\xa2\xcd!\x9a4Rvb\xce\x81\xdd\x84\x96B25\xaewD\xa1\xc1\xfbe\xb7\xd3z&;\xc6\xeaU4\xda\x94EM\x1f\xab)\x1e\xb4\xe5z/\x96\x8f\xbd/\x00\n\x02v\xdf\xab\xc1\xf8\xc7\x8c\x98uC\x90\x10\x8c\x93$\xf6bC\x9d)\xf1b\x03y\t\xa4Uf\x91\xc7\x9d\x9d\xfbm\xd4f\x1e\xecR\xf5r\x81y\\\x0b\x08n\xb21G\xec\xcbu\x8f\x00\x8b\x0c\xad\x13k:\xa70\x0e\xdd\xc4\xbd\xf1\x01*\xb5\xbd\x8d\xfbBQ5\xf8\xfdJ\xeb\xd0\x8d\x7f\xafE\xdd\xc7\x8c\x985L\x1a\xc2 \x9c&_ \n\xe4\xd9\xad\xdcW\xd2\x9dUH\x91\xceVUz\xd8a\xd7\x83\x08\xdfmDF\x9d\x18\xd6\x97l\xa4\x07\x9e:8~\xcc\x1cBk:\xd2\xb2H\xb4\xaen!\xe7u\x0b\x05WQ@A\xd5\x19\xa4\x14\x8f\xf9|\xad4L\xf4\xed\xa6\xfb\x1f2b\xa8\xbf\xe2\x14D\xaf\xf7\x92W\xe9+\xb4\x80x\xb9\xf7\x03\x05\\(\xd0\x92{Y)/\x19`\x83\xae\xdd\xa0q\xe2\xb7=E\xd4\t\xe6\'8cV\x9auiaz\xa1\xacf&\xa1\xfbC\xc2\xf4\x93\xe4Ad\xe8B\'\x13\xd9\x07\xc6m\xf1n\xac\x8a*o\xb6T\xfd\x98\x11\xb3n}\x08[\x0bZ\xea\x95\x87\xa2\xceh$[\x19v#\xa8\xe0L0j\xe5\x18\x13\xe1%\xa2@\xde\x8dz\xb0\x15\x8b>\xdc\x14\xce)\xa7\xf6X\xaf\x0b\xeb\xd6\x95\x07\xda\xde\xd3\x8b\x12z\xe0\x03\xcf\xf3c*\x87\xb3\x0e\xda\xac\x9a\xd6\xb55\xb4\xb5\x1e\xc4\xec{a~\xcc\x88Y\x17\xedZ\x1c\xa0k\x99\xf5\xd2C\xae\x97\xaf\xfd\xf1\x1c\t\xdd\xa5\x8a.J\xa0\xd5#AF\'\rUk\x91\xc6\x0f\x9a\x89\xc7\xfd\xc9,\xce\x8f\xab\x9a\xa0P\xe9\xde\x1c\x1c\x1b`\xc3\x00\xbb\x0b\x1bHdX\xf6K0_\x1a\xfcp\xe0I\xa7\xd4\xd2\xc3x\xb3\x8eo\xee\xcd\x8f\x191\xcf#\x88\x86\xc8\xb5\xa0\x7fY\xb4\xa7\x96\x13\x0c\x07|\x80"\xe4\xb5\x94\xfc\x182\x88\x89\xa83N\x0c\xb6q\xba;\xfbJ\xb2\x95\x9d<\x93\xa6\x93\xe2\xedQ\xac\xc13F\x9e\x07-n*R\xce\x0c\xce\xe9\xe2\x9d*9\xe3)]|l\x8a(\tL\xcbw\xdb\x1d\x7f\x0c\x89Y\xf7\xe6\xd7\xce\x84\x88\x97Ek"\xd9\x05\xa0\xd0\t\xe5\xda\xc7Q\xa7bZ\x134"\x18\x9dk\x9b\x04\xd5\xce\x99CjgQ,\x00\xf9i\xcfG\xdc\xd5\n1:\xd9\xb3\xd8\x03\\o\xdcPu\xd2\xcfD\x135\xa1\xfa\xb8\xb3}\xc3\x83w\xaa\xdc\xcf\xe5\x8f:\x9d\xff\xc9\x91\x98u\xebck\xe9\x00A\xc8\xcb,\n\xc3\xd2\x12\'\tk8\x82,\x0fG\xe7l\xd4\x9e\xeb\xd0!\xa1\xc9\xde\xdc\xa3\x0eY\xb6\x02b\xccH\x92M\xd4\xa3)\xce`?\xcd\x85\x04\xd8\x19\xc5Y\xfc\x15b\xa4\xfd:\x8f7\xe8H\xe3\x8d\xe7z\xe7Qd8\xf7\xcd:\xf5cH\xcc\xef\xde^\xfc\xef0\x0fM:*\xb03\xa5\xf1^9\x1d\xe2\xcb=+\x9b\x06\xbb\x00j;G\\\x14\xb5\xe5\xd5\xf4\x17\xa3\xf1c"I\r\x9a0\xe2\x18l\x82\x8c\xda\x95\xf5\x12\xd9\xe9-\x9b\xeaE\x99\x0e\xd9h.\xd7\x81\x843^*u\xea\xcd6\xc3\x1fCb\xd60\xa9\xf5\xda\xb5\x9e /[\xbf.\x92\x87 EFw\xed\x1a\xf2x\xa5\x98\x84\xbb^&\xba\xf7%{\x8e\x1e\x18\x7f\xbb/\x1eu\xb9\x9dw\xd982\x8d\xaf\xc8\xde\xd9I%\x96\xdbaN~\xb8\x06\x92z\xbf\xfb\xcd\xa1\xed\xe3=p\xac}\xd2q\x93\xf6\xf6\xbd\xf6\xc4\xc7\x90\x98uO\x10\x18\xb2\xde5_\xf5\x14\xa5\'\xf1\xae\x87\x06\xb8\xf7\xae\xdeQ\xa9\xaa\x1d\x88C\xa4\xea\xdd\x8f}pa-\x90;S\x8e2v\xe5\xc3\x19`9\xbd\xf9r\xdar;\x9c\r\xd6$\xaf\x96\xb4P^y\x15\xb2\x0f\xbb\xbd\xc0\xca\x12;\x96\x86[\x90U\xd6\xd16K\x10\xc7oR\x05\xce\x8f\x19\xebm\xe4\xacu\xbe}\x0f\xbc$\xd5\x8a\xd2\x12}\xdb\x87=\xca\x7f\x1c\xdb\x81\xf7!\xe1MK\xeccH\xcc3\xe9#\xd8\x13\xafx\t\xb3\xf0,\xe3v\xb9)N\xbf\xe6\xef\x9dT\xc7u\x1c\xa5\x9c\x02\xaa\xb5\x00\xdd\xd7D\xec2\x9e\x8f\x03a\xaa\x00\x95.\xb0\x11\xc3\x1c9\xf86\xfb\n\x90\xa9k\x19\xa4\x8b\xc8r\xa82<\xbc\xdc\xbb\xfe\x1e9\xf4\xcd\xef3\xff\xcd\xda\xe6cH\xcc\xfa4\xc9\xf5\x0fc\x10\xfdr\xc2aIZ\x06QU\xdb\x9d\xeda\x0b\x14\xed\x16\xc5w\xf4\xe3-\x0e\x14\x98\xd8\x07k\xe5W\xd6\xd7~\x01\xaa\xd8\xce\x8f>xJrtP\xfd\xe9$\xb5\x81 f\x92R\x12y\xd9\x9825\xb2\x80\xca\\\xf5\xe5~\xa8\xab7k\x9b\x8f!1\xcf\xad\x8fb4\x82\xa0/\xf5\xbe\xaf\x0e!\x08\xb0\x91\'\xc6\x07\x80C\x0e\xc6\x8d\x11\xbb\xa8\x85\x0f\xa5d\xfb\xb8d#\xf0t\x93\x81v\xae\x0c\x83v\xaeVF\x9e\x86\xcc2\\\x9d\xe9\x98]r\x0c+\xcb+\xe1\xe9P\x0e\xc9-\xcf\x9b%\x91\xd1\xdb\x0c\xbcI(|\x0c\x89y\x16\xe48E\xc3\xd0\xcb\x1d\xf5\x14;\xb3\x8a\xe0L\x96T\xde\x1d\xcfF\xe0p\t$\xdftt\xfa,pd\x0f/\x84T[\xa0\xd43P}\xc6\xac\xb8\x01g\x1c\x8e\xf1|\xaf\xdf\xdb\x12\xa6{z\xbe\x81\xba\xe7\xc3\xd7\x92GXd\x1c\xa7j-\xdb\xdf<\xc7?f\xc4|Up\xeb-\x1e~\xfd,%<\x03J\xc8;c\x0e%I\xfbH*\x97Q\x0f\xae\x91G\x80)Lv\x0e\xd1\x95K\xcdjrI\xbd\x91\xa4\x0b\xfdf\x1a\xb7Hu\x976\x8f\xc9\xeb\xac7\xfe\xa0\xcc\x8c\xa3J\x05\x1b>@5j\x9d\xe4\xc2c\xd4\xf7\xba\xbc}\xcc\x88y\xee\xfc\'\x10\x85\xbc\x92i\x17y\x06v4(Ty2^F\xfb\xdc\x84\xde\x849N\xc5M\x14\xe4 \xfe\x0e%\x8c=?\x02\x8f\x87\x88\xdeT\xa3E\'\xa3\x05\xdc\x08Z\x92\x190*\xa0\xc0(<>T\xc4\x90\xae{\xa6\xf2!)\x8f\xef8eK\xd7\xb8\x15\xcc\xb5\xba\x02\x9bu\x10\xfb\xb8)a\xf0\x90\xb3\xbb\x1a\x95.cB\xbcy\x8c\x7f\x8c\x88Y\xa3$\x9fi\x00{-\xf7\xb1s\x1e\xe8\xd1\x14\x85\x92U]\x81\xc22\xe1\xd6\x1bJ\xedx\x01\x1bCa\xe2|\xd0B\xb4h\x98\xd0\xcd\x8c0\x8f\xf6*\x9b&,\x02\r\x82=\xc1\xa4\xd1\xec\xf8\x85\xea\x99\xe4\xd0\x9c\xa1\x936\xf9\xdc`\xd6]\xf7x3\xcc\x8f\x111\xcfOS\xd6\x87\xb9\x16\xf5/O\xb3\x15|H?\xa2}\x07H9\xe8\xa2\xca\xa4\xc5\x9cke0\x1cF\xe3q?!%b3p\x80\xb9\x1c\x12\x05\x82\xd1\xc0\xd8\xb9\xac\xa3),\xc3Hl\xa6\x9db\xeb\xc8\x82U\x8c\xfe\xb8\xdb\xc5\xd5\x04\x8b\xfb\xc32\xb3o\xf6\xd1\xdb\xa7\x88\x98\xe7\xab\xcc\xf5\x86@R\xaf\xafm\xba\xd2\xbe\x03\xbb\x0e%%g\xfd\x87d\x01\x8c\xb4\x105\x0f\xc5\x9a/\xd8\xba\xc45\xe7\x91\xdbuj@\x0c\xb7,\x05\x1d\x08\xf3\xfa\x03b:\xee\x87\xd1\x11[\xb5\xd7\xef\x0f\x84\x18{B\xb2\xf7l\xdcC\xd5\xbe7\xd8\xff2!\xff!\x11\xf3\xf5\xa5\xaa\x8d\x88\xf9\xa7\xbf\xab\xfe\x9fC\xc4\xfc\x89\xf0\x8d\xcd3\xd9\xa6\xf4\xfbO\xe9F\xc4lD\xccF\xc4lD\xccF\xc4lD\xccF\xc4|\xe3qnD\xccF\xc4lD\xccF\xc4|gze#b6"f#b6"\xe6\xdb\x8es#b6"f#b6"\xe6;\xd3+\x1b\x11\xb3\x111\x1b\x11\xb3\x111\xdfv\x9c\x1b\x11\xb3\x111\x1b\x11\xb3\x111\xdf\x99^\xd9\x88\x98\x8d\x88\xd9\x88\x98\x8d\x88\xf9\xb6\xe3\xdc\x88\x98\x8d\x88\xd9\x88\x98\x8d\x88\xf9\xce\xf4\xcaF\xc4lD\xccF\xc4lD\xcc\xb7\x1d\xe7F\xc4lD\xccF\xc4lD\xccw\xa6W6"f#b\xde\xc1\x10\xfe\xe6t\xf9\x87\x18\xc2\xef2\xe6w&b~~`\xbf\x88\x88\xf9\xf9\x81\xfd""\xe6\xe7\x07\xf6\x8b\x88\x98\xf7\x02\xfbW\xc0\x91_D\xc4\xfc\xfc\'\xf6\x8b\x88\x98\x7f\xc3\xe1\xb1\x111?\x85\x88\xc1\xfeg\xce\xff>\x11#\x92\x10J\xb0\x02\xc6\x89,\xcb\xd3\x14\x01\x89(\x81\xd04\x8b\xe2\x1cM\xa3,IA(\xca\xe3\x08/\xa0,\x8e\xa2\x18\'\xc0\xbc\xc8\x11\xeb.\xa0 \x8e \xd7\xb3\x07]\x9f&\x81\x92\x82\x00\xad\x8f\x89X\x7f\x03\x8cC\x08\x8fq\xeb\xe4!8\x89\xaf\xdb\x95_\xe7\x1c\xc6\x18z}\xe8\xa4@#,\x8b\xfd\xe5ON\xc4\xfc\xd3\xa2\xc7\x0b\x11\xf3\x17\xae\xc5X\xae\xedY\xae\xd8O\x01B/\x91G/\xdaU\x87\x02oh\xa3:\x1e\x93R,N\xbe\x95\xefm\\\x8f\x91D8y\xf7\xea\x84\x88\xd0\xc9\xdaA\\1L1\x02O{Oo\xd6\x92\xf6\x1e"b\x1f,\xbb\xbb\x8e\xee1C\x12;\xfd\x9a`k\x82\xbcEN\xd2\xc5|N$\xdeq\xd4\xeb\xa4\x8e\x1c\xab\x12\xca\xdd\xd7\x9f}\xde\x81"\x94\xad\xe2R\\\xf6\x8bN\x04\xe8z/\xac\xf3&\xf0\x02d/\x9f\xa1\xc0\xb9\xcc\xc1\xfa\xcf\r\xbfZ\xaf\xe71nx\xedm\xfd9\x96\xf0\x0c\x96\xe6\x8d\xc5\x9d\x8f\xf0W\x86\xbc+\xca\xdeq\xef\xeb\xfd\x12\xdd/\xb1}:\xb7\x06w\tT\xf9A\x97\xbfe\xd0\xfd\x18\xa0;<\x96\xadi\x8d\xaf\nP\xe6\xa1;\xa7n\x8d\xcbL\xbc\x9dv\xf2/\xb6279\x83\xeb8+\x90\xa6h\xf7\xc7\x0b\x19\x8b\xe4\xd9/\xf6v\xdf-\xbb:\x94\x90\xc3e\xb8\xab\xbev)\xdcIm/\xc2\xd9T\x99\xf3\x99\x15rS\xe5\xcf\x8d\x98x\xcf\xf9\xa3l\xd6\x11\xd0}y\x99\x05\x1d\x94\xb8\xbce\xee\xd7\xbej$&n\r\xd5\xae\x0e\xbbqR(\xfa8\n\xc1z\xb1\xb1\x13\xd5\xe1\xe5\xdd\xce\x8dom\x0f\x1ap\x1bfB\xcd\x14\x1eb\xba\x83\xab\x15\xde}(.\x0c\xad\x01\xa8/?\xd8\xab\xa6rw\t\x99\xe3\xc8\xb0\xbcg\x03\xaf\x7f\x15\xf8y\xb6\xa5\xf9\xbb\xc0\xcf\xf7?\xa5\x7f-\xf0C\xff\xb8\xa3\xb8\xa5\xc7,Q\x88\x90\x18\xe1\x87{B\xdeN\xee\x8d0s%\x1e\x97\t\x8fPw\xe1"\x83\xb6\x8fD\xf2\xe0\xc6\xc3\x894\x85\x001\xa4\xb0*\xe1\xb8\xd7\xcbn\xb4Q\x83\xd3\xf1\xfd\xc4L\xea\xfe6\x84\xdd\xc9/\xce \xcd\xbd\xd7\x94\xeeS\xc0\xcf\x9a\xd3i\x08\xc2`\x1c!\xe9\xdf\x87\x99\xdf\xe2@\x0e\xbd\xbe\xa0 Dmu\xf0\xcc\x8fR\x06\xb6\xd7\xcb\xad\xbc\xf6^\x11\\k;\'\xday\xde\xdb\\v\x88q{V\x82\x82\xe1n\xc0\x8ek\xcb\xb66\xf6\xd2x\xbd\x97h\xb1\x9bs-\xe4\xc5\\AX\xf2\xf0f#\xc5O\x01?\xcf0\xa95K\xbftNN\x81\x93\x88\x9b\x0e\xff\x18\x1e\xf9e\xea\xa2\xa6\xa4o\x19\x05(\xb9\xba\xd7\xa3*\x1a\xf7\xa4\xbc\x13M\xf8\xc2BG\xb01\xe4+\r\x18\x8d}\x1c\xe1\x00\x00A\xdfm\xc1E\xa1+\x9b]\xc6\xc5\x02\xb4\x94\x85\xce\xb5E\xbf\xd99\xf9S\xbe\xcf3D\x18Z3\xe9k\x13\xc5\x9c(N\x91\x1a5\x0f\x12\x13\x9d\x18\xd9\x1f*\xf8\xce6\xe4aX\xb8Xxx\x1e\x1fp\xd6>\x8b\x19\xda\x1f\x8bk\x8aWDJ1\xe9t\x07A\x97L\xc9\xcbCfu\x9fU\xdb\xbbV%\'\xaf#\xa3d\xc0\xad\xeaG\xad\xd4\xfe\xdc\xbc\xcf\xb3\xc4}.\x14\x14\x7f\xe9\xb7\xb90-\xa3\xc8\xfa\x8d\xb2!\x8e\xcd\xe3\x87{\x02\xb1#t\xdf;\xf7\x87\x94P\xeaz\x18\xd4\x88zi\x0e\x97kAkHrB\x92\x8b\x1e\x9a\xcb.\xec\x152\xa6\x81\xf6\xb1\x8f\xac\xea\xccg\xa2\n\xac\xbfJP\xc6\x99W\xdfdo>\xa5\xfb<\xa3\xa4\xd0u\xad\xc1\xaf\x1e\x0c$\xf4T\x04\xa0@\x0f\x93\x03\xbcK\x08q\xec\x04\xf8\xf2\x88z90\xa1\x13\x8c\xe8z\xec\xd3E\xa8\xd2%\x06sQ)L\x81h\x1fc\x8a*\x90\xa0\x9bn\xeb\n;\x04\x8e\x86\x13\xbd\x02U\xa5\xddU\x80\x04\x0f\xcc\x9bP\xca\xa7t\x9f\xaf0I\x1cAp\xe4\xe5l[$\xc4\x8d\xe9\xc1\x08.v#\xed\x96\x01\'\xe4\x98\xf1\xcc\xc0\xb6EoY(\xd4\x16\x84Si\xf0\x95^\xedz\xf8\x06e\xa1un\x1d\xc5*K\t;\x1c\x80\xfe\xee\xaa\xb0\xce\xafc\xaf4\xae!\xf5z=\x01\xc0\xfeM\xbc\xecS\xba\xcf3L\x9c\xa00\x02\x85_\x9a(B\x11\xc6\xe9\xe3\xbe\xcb;r\xe0\xfa\x9d\xeft^J\xb3\x8c\x7ft%B?"0\x01\x12!\x80\xf6\x8c{\nO;q<\x9c%\xdd\x10\x0e\x1d\x1e4"q\x90\x8ft:\x02Z)\x1f\x9a\xec\xc6\x96\xb1gj\x87\xe0\x11\xbd\xd9D\xf1S\xba\xcf\xefo\x9f\x7f\xd3\x1a>\x04\x863\x00\x12z\xa0\xb7)\xd8\xdc/j6=\xfa\xb5\xa4\xa2\xc5\xd2\xb8)G\xae\x0e\xcfw9\x1aa\x8c;\x85\xe1\xb1L\tX\x9e=B\xccR\xd9\xe9\xc8C\x1a\x07\x0f\x8aZj\xb1\x00i\x9d\xd3\xbc\x9a)\xb5\xe0\xcd^\xb8\x9f\xd2}\xbe\x16-M\xa1\xf8\x1f45\x8f\xf7"\xbf^\xce<\xa48\x19\xd3\xd8MW\xfcl\x14\xfal\xcf\xbc\xa8y\x92\xa3\xe0\xbb\x84\xbfjY\xd1a\xfau\x8c\xe4\xc0\x07\x07+\xd7K\x81\xeb\x97\xa8s\x12\xf3\\/\x19U\xc6\xca\xbd\xcd\xbd\xb5\xca5\xe5\x8e\xd5\xdf\xccV\x9f\xd2}\xbe\xea\x0e\x84D\t\xf2U2\xf1t~?\x1aF\xbd7\xc9\xe4Pu\x0e\xd1\xa5\x96\x93\x1d\xd4\xdb\x0c\x8e\xf9\xc8\xde|U\xce\xef\xe0z\xad|\x04\xa4Y\x1f\x1f\xa1\x89CKE\x11^\xf3h\xa6\x0c\\\x93\xb5\xda\x84\xd0\r\xe1"\x9d\x9d\xf8\xa9\xb1\x0e\x17\xfd{5\xfa\xff\x94\xee\xf3\x9cE\x04A\xa8\xf5//m\xb0\xd3\xcb\xc3\xec\x81\x11\xc1*\xd2\x83\xcb}-2\x8f$\x9e\x1f\x87\x86\xactQC4Ghz\x06}4\x9cY\x110\xbf\xbf^(I\xd6PI\xac+\xd6V\xcd\xeb\x0e;N\xd7n\x80\xf8\xa1\xdf\x1bt\x88\xe7p6\xb2\xefjt\x1f\xd2}\xd60\xc9\xe7}\xfe\x19\xe9\xef\xc3\x94\xfaK\xc2\x99\x8c\xac\x0c=#\xcd\xebE\xb49j\t\xec\x1e\x83C\x14\xde\x0b\xa9H\x83{w\xa3\xbc\x82U\xaa\xea\xdc\x1d\xe6\x16\x95\xb4\xf8\x18\x01\xb8\xd6\xf7\x819\xc7\x86(\xb72Y\x10\xa9I\xb2-\x80:\\\xd1\x1d\xdeU(?\xa4\xfb|m\xfd\xf5L\x84\xfe\xa0P\xa5Dl\xc7\x9b\x87\xb3^\x8b-\x96\xe0k5\xed\xee\xcfY>\x8cSdR\xeb%\xe5\xe8\xc0\xea\t\xd7\x0c\x05\xb2k\x8c\xf3a\xe3\xd4_\xae\xa4kW\x9e\x88\r\x87\xc1r\xf0\xdb\xf5\x90G\xfdN0\xd5se\xa8\r\xabB?z\x9a\x7fn\xdd\xe7\xb9\'\xd6K\'\xb5\xfe\xae\x976\xb1\xe3N_\xe4$\x93\xc2\xc4\xbcEA*\x97\x94\x08\xdb\x1c/c\x13\xd6>\xf8\x9d\xb1\x8c\xae\xe0\xee\x19\xac\x95dC\xe49\x9dq\x8f\t\xafJ7R\x8b$\x93W<\x1f[\\\xe8hK0\xab[@\xa7\xee\x1d\x8b\x05\xdel\x10\xfd)\xdd\xe7\xb9XH\x92\xc40\xfc\xd53\xa8Es&\x9d\xb3"b\x17\xf4N\xcbT\xd8\xf3v\x98\xcc\xc0\xe5\xd4\xb3Al\x84\x1c8\x95q\xad\xa8\xe6eD\x8c\xa3J\xeb\xdd}\x18\x0e\x0eRj\xb5+I\xe4\xb9\x9c.\xf0\xf5\xba\x17\xf8\x93\x11\xf4#f>\xc0\xc2hX\xee\xbd\x83\xfcS\xba\xcfs\xd1B$B\x93\xc4\xeb\tG\xaa\x0f\xd87\x8b}\x1bt\xf3ly\xf7c\xed\n\x8e\x93\xe2h\x08d\xe4z\xf4\xd9\xb5\xca\xfbF3\xf3wR\x90Qe<\x16\xa4\xf3\x98\xe6Cw\xb7l\xb4\xb1w\xb2\x07\xa2\x90\x91\xa07\xaf\x04\rVu\xce\xe2\xf9\xfc\xa3\xbd\xf9\xe7\xe6}\x9e\xb3\x08?\xf7\xfd\xebM\xdf\xa5\xb3\x03\xc9K\x00:\xcb\x8d\xe6\x8d7\x9c\xe1\x18\x8b\x84=\xe5\x11{;i\xc63I\x13\xfa\xba\xa0:U*<\xa3-\x81\xa4\xc9\xaa\xc2\x00%2|\x18\xa0\xde\xdf0r\x16.\t\xef^5\xfe,U\xcc\xa4b\xa77\x0b\xe1O\xe9>\xff\x95\r)\xfc\xf5V\xb3s\x88\xa9\xe6ZT?`W\xd22\x02\x7f\x07\x12\xd9~_\xf7P\x12-\xbc\x1f\x90\xf8@8\xd7\x938\\\xc4\x83\x1b\x0b\x19Y\xb4;\xbcN\x97\xd8\xc2jmr`\xa9\xa2\xc2\x107\xf5r`\xc3\xd1\xeeC\xca\xdeWo^\xc5?\x85\xfb|\xbdp@\x9fY\x0fz9\xc6\xcfF\xd8\xd8\xb3W\xfa|"RE!8A\x81T\xe6I\x7f4E;\x86\xa9\xbb\xc3o(\x86`\x17!\x9dY"\x944[\xa8\xfb9\xaf\xd6\x9ctIcR\xef\xe7l\x18v;\xe4A\xf6\xa1\r\xc5\x93\xeeK\x85\xfc\xa3\x8d\xff\xe7\xc6}~[+\xe4Z5@/u\xf0=t\xd8\xc1r\x91\x9c\xe6\x94Y&\x8c\xfcZ\x0e]}fH\xce1\x8c\x86\xe1"@\x0cl\xc2\xbb\xd3\xd8q\x0e=j9u\xf2\xe8R\xf7\xde\xa9\xaf=\xb8$\xb5\xdc\x89\xca\taNf\xd7]\x16\x04\xc9q,h\xde\xac\x83?\x85\xfb|\xdd\xf4i\x14&\xd7\xd3\xf7\x05\xb8\xdd\xc1%5\xd6Wr\x88\x9a\x9c\xd8c\xce>\xf1I<]\xaf\x9e\x93\x0e\x08\x98\xecq\x9a8\x16\xf0\x91\x82\xbb\xb67.\xf0\xc0\x9e\xd8}\xd6Q\x85\x8aw\x87\xcaO\xdb\xfc\x1e\x1e\xcc\x1d\xeb\xdd1\x8b\xe9\xc1+\xea\xf8\xf77K\x9bO\xe1>\xbf\x9568\xfc\\\xdc\xbf\x0f\xb3\x01\xd8)ca<\xc4\xfb\xae\x82\xf3\x0e>q]\xd4<\xc0\xeb\xb9G\x1e\\\x1ed,\x7f\x03G\xa8\r\x89#t\xa1\\n\x97\xe7\xc2\t+\xa2\xd9Bgr)\xf6\xec\x8es%\xd3\xa2\x15\xcc\xc7\x8a\xf3\xe1\xcd\xa7\xf9)\xdd\xe7\xbf\nU\x0cA\xe1\x97;\xaa\xbdD\x0f\xb1\xa8u\xb4\'\xad\xb2H\x13\xbfZ\x88T\x9cO\x07\xc1\x8c\xcd%\x84at\xa8\x95\xeb\xf9\xd8f\xa0N\xb8\xdd\xdeiHz!\x85\x89\x07\xf2j\x8e\xd1\x8b\xd8S\x91\xd0Q\xa1J\x85\xac{\xdb\x11\xb0\x87i?\xf2\xc5\xfe\xdc\xba\xcfo\xef2!j-n^\xd2\xe1p\nkf\xef\xc6\xd6n-\x15\xa3\xa6j\x87,8Y\xf7#\x16H\xe2\xa0L\xd2\xd4\xcfP\xa08\xe9=\x8c\x1a\x80t}\xa4\tI$u8\xbd\xbcr\xf3~\x12\xa6)1\x81\xc5\xd2\xce\x08\x8f8\xac%\xd27\xdcz\xf7\xe3\xa8\x0f\xe9>\xcf0Q\x18\x86!\xfa5\xeb\xef\x86\x1e`\xd1\xbb\xbeV\xd7\xa8T\x1a\xc2\x8d\xf6\xa1\xc1>N\x1d\xa8\x9fJ\xe6\xbeS,\xa04\x0f@\xab\xdb^h8x 7\x01\xa3\x13\xfe\xe8\xc5\x914\xb9\xd9`\xc7\xa5\x11\x86\xd0qP\x9dZ\xabb\xd9g)\xeaM\xa0\xf1S\xbc\xcf\xd7\xd6_\x8b}\xf2\x0f^\xd9>\x02\x05\xb5p\x96\x1b\x9b^\x84\xa19\xf3\xce\xb9\xc2\xd3\xd3x=\xb3-^\xf5\x95]\x9f\x06\xb8\x94\xda\xfbn\xea\xd0\x10\xcf\x8d\x8b\xb6\x87\xe9\\\x85\xee\xa4\xc1A\xc153\xb1\xa4\xbe^\x8cJ\xeb#\x10\x94\xe9\xf4\x9a\xbf\xfb\xe1\xe2\xa7x\x9f\xdf>4\xa2a\x8c|M\xcb\xb3\x03N0"\xeeO\xb1\x7f\xc3\x1fU\x84Zk\x9d\x91\xe6\'\x94\x16&\xf0\xecW\xb4\xca\x06\x8b\x9d\xa5\xd5t1\'\xbe\n\xe0\x19\x12w\x88?\xd7\xb7\xaak\x83\x02\xbe\x10\xd0>;\xf0G\x83=\xdf8\x14\xdb\r\xc5\x18}/\xd8\xebS\xbc\xcf\xd7,\xae?\xc4\x90\xd7kMg\xdc\xc9\xee\xe1\xf1\x8a\xd6\xfa\xe7I\xa3\x9a\x90\x8d\x04\xd2\xa9L\xb7\xca\xf8;\xaf\xa4\xc9\xe9\xa4V\x1dm\xec\xf0.~\x88\xc7\xa6\x9b\x92\x86\xbbIyG\x81\t\x8eu\x10\xb0\xa0\xca^?H\x18\x92\xe5lF\x90%\x7f\xf8\xad\xe0\xff\x87\xbc\xcf\xd7\xcb\xc9\xff\xcd\xfb\xfc\xdf\xff\xdf_\xe2&YG\x85\xff\xf7\x17m~\xfb\x1eeq\xcd\x9a\xdff\xfb\x1c\xf6\xff\xef\x1c^\x87\xf4\xf95;\x98D\xa8\xaf\x97V\xcf\x9f\x8e\xfd\xd7\xcfH\x92\x84\x88\xafo\x0e\xfd\x97\x0b\xf4\xff\xfc__\xbf\xb3o\xc3\xf8\xab\x96K.\x7fy~\x97s3\x84~\xf0?3\xfc\x07\x19B\x7f"\xf7b\xa3D6Ch3\x846Ch\xdb\xf8\x9b!\xb4\x19B\xdfw\x9c\x9b!\xb4\x19B\x9b!\xb4\x19B\xdf\xd9\xe6\xd9\x0c\xa1\xcd\x10\xda\x0c\xa1\xcd\x10\xfa\xb6\xe3\xdc\x0c\xa1\xcd\x10\xda\x0c\xa1\xcd\x10\xfa\xce6\xcff\x08m\x86\xd0f\x08m\x86\xd0\xb7\x1d\xe7f\x08m\x86\xd0f\x08m\x86\xd0w\xb6y6Ch3\x846Ch3\x84\xbe\xed87Ch3\x846Ch3\x84\xbe\xb3\xcd\xb3\x19B\x9b!\xb4\x19B\x9b!\xf4m\xc7\xb9\x19B\x9b!\xb4\x19B\x9b!\xf4\x9dm\x9e\xcd\x10\xda\x0c\xa1\xcd\x10\xda\x0c\xa1o;\xce\x7fI\xcb\xf8\x9b\x03\xfb\x1fj\x19\xbf;\x8b\xbe\xb3!\xf4\xf3\x03\xfbE\x86\xd0\xcf\x0f\xec\x17\x19B??\xb0_d\x08\xbd\x17\xd8\xbf"\xd2\xfc"C\xe8\xe7?\xb1_d\x08\xfd\x1b\x0e\x8f\xcd\x10\xfa)\x86\x10\xfe?s\xfe\xf7\r!\x16\x17\x10\n\x82I\x94@)\x92\xe5\x18\x04\xc2`\n\xa2\x18\x16\xc2\x19\x06GQ\x01\xa7P\x84\x87\x19\x04\x83\x04J`E\x8a\x15Y\x16ci\x06\x87\x10\x81F\xff\xbe!\xc4\xd14D\xd2\x90\x08\xc3\x1c\xc4\x08\x08\x86\xe1\xff?{\x7f\xd6\xad(\x96\xee\xf1\xc2\xdf%o\x19#\xe9\xbbsG\x0f*\x9d\x82\x80wt\xd2+"\x88\xf0\xe9\xcftE\xd6\xbb\xab\xd2\x88\xdce\xbdZ\xcb\xdc\xc7\xaa\x91#sd\xb3b\xfe\xe7\xd3O\x8d\xe7G\xcb$\xb8T\x84\x04\x8d\x13\xc2\t\x12+\x12"B\t\xbc\x00\xfe\x1d\\\x12QA@1\x14\xe5)V\x12\x18\x86\xe1^\xc9\x10\xfa\xc7\xd2\xb3\xdf~\xb2\xe5\x81\xc0~Gi\x92`h\xe4\xc7\xc6\xda_1\x84\xde\x1f\xc0\xf4\x13\x86\x90\xc8\x8a8J\x92\x08M\xf2\x82\xc4 \x02\x83\x898\xc6H\x12\x82\xf17\xcc\x0f8\xf7mU\x0c\xc2\n\xf4\x8d1$\xde\x96_S,~\xa3\xe8\xe0<#|\x11o\xfe\x99!\xf4\x04\x90\xcc\xff\xcf\xc5?\x0c\xa1[\xda\xe1\x81\xd3\xd0(\xca!\x0c\x85\x83_\x9b@Y\x9a\xe5XN\x94\x81\xd9\x80q\x18\x9c\x15\x81\xb7!\x04\xc1\xd2\xb7-\x8d"\x8a"(\xf0\x04\x0c\xfcP\x96\xc1\xa9\xdf~\xce\x10\xa2d\x8aGIY\xe2Y\x92\x17\x19D\x14o+\x7fE\xec\xb6\xda\x19D\xbc\xccH\x08%\xe3\xb7\xd5\xfc\x12\x05\xac#\x92\x9c\xc8\x83_\x0b\xfcm\x1e\xc5E\xfckk\xe6\x7f\x87!\xf4o\xb3Q\xee\x18B\xff)\x85\xe6\xb6=\xe5/)4\xef\x1f\xe7\xdfJ\xa1\xc1\xa9_/\xbd7\xd8"2\xab\x05\x14\xe9\xcd\x8e !#+\x02\xa4\xd9\x1d\xf4e8\x9a\xfee\x15\xc9g7KM$C;\x9a\xa52\xec\xbc6\x8dy\x08p\xbc\xb17\xd6q\xca\x16\xc3H\xa1uj\x9c\x93\xd3\xf6\xb4Z\x95\xae,\xf2\xcbo\xa2\xd0\x80\xaa\x80\xe18}[\xfbu\'\xb3^\xbb\xe8hK\xd6\xe9T\xcaT\xa9v\x9dd\t\x8b\x03\xafq\xa5\x1f0\x88\xda\xb4\xc5Q\x81\x13D\xf3)\xd0\x8d\x9e0\xa4n\x8f8C^k\xe6\x12\x80:u\xf4kQ\x8d\x9d\xd0`\xa9\r\x81\xe8=\x05\xc6\xdf\xebB\xfc\x1e\n\r\x90\x89\xa3@\xe6\x9dF\x86)\xbc1\\\xfa\xc3x\xbe\x1c1\xe8\xacl\x92\xb3\xd9m\xb0\xe3\xf5\xc0\x1a\xeef\xef\x85\xd6\t\xbd\xee\xa4\x8bC\xf3\xd5&\x11\x0c\x86b\xe4\xdd\x82\xb8\x13a\xdb\xa3`\x95Q\xdeN\xa3r\xf5v\x1e\xa3\x90\xa3\xdc%\x10\xa6\xa3\xc257O\xe6`W\x04\xa6za\xd1\x8f\xc7\x01\x99\xd6\xa3\xab#*rr\xcb\xcb\xbc4$\x97\x88\xa8F\x10\xd2-\xb1\xd2c\xe9\xb2\x93\xa4\xee\xc1\x80x\x16\x88\xe6+\xee\x91\xdbZ\xe0;\x08\x04\xca\x9e\xc5\xe9\x14\x12\x11a%r\xb7h\xd71g\xc5\'Qt;\xac\xd8{\xa8Xn\x10\xf2\xec\xc9\xec\xce_\x98L\xdcw\xe8\xc8\x87\x11\xa8\xda\xbep\xee\x04B\xcf\xcbi\xbbW(\xfc$\x9fN\xd7\x91V\x96\xa1*b\xbf\xda\x99\xf8b\x0e\xcd\x9f{\xde\x7f\xd9g\xda\xb8\x07\xb5G\nOA\x08\x8d="\xd4\x8c\xac1\xac\xde\xb5\xa7\x98\xe0\x12A\x0f\x86z\xad%\x1a\xa3\xb1\x92\xe2Y\x15k\x07\xea\xa9\xd9H\x88\xb5\xacO0\xcbn\x92\xbdX\'g\xd4\xd2\x08\x08\xfcOk\x0b\x14=?\n\xa0z\x12\x87\xe6&\x93\xc1(\x1aD\xff\xdd\x0e\xf3\xaa\xda\x1b5\x99\xa5\x92\xd6\x92\x97\x1ewX\xf6n"\xf2\xe3\xd6\xbe\x9c\x1ft\x96gqhn\xce\x82\xd1(\x86\x10\xf7\xf8\x82II\x16\xc7\xa1\xb10~\ts\x95\xd6\xa9v\xbb\xe9\xc6\xbc\x91\x9b\x8d:\x95iC\x14\xa7B(K*ZI\x17\xa4M\xf5\x8d]\x88\x1er\x89<.\xaf\x89kJ\xb1\xfe|!i\r\x87\xb7\xd2\xf5\x18\x94\xab%\xd9\xa8\x0ff\xb8gqh\xbeZ8\xe26J\x92w\x19\xeeR\xa4A\x9ekBZ\r\xf8\x0e\xbfn\x16%\xed\x9bVZ9\xb2X\x88\x15f\xb0\xd5\xcca\xe4y\x93\t\xbe\xbd_\xa1^i\xd7\xe1B\x92\xac\xc3\xe9pF\xceF\x94&s\xabnB\xca(\xba\xde\x17\x8e6O\x99\xd8\xaf\x86\x8e\xbf7\x87\xe6v\x8b4\xf8A\x04\x83\xdc\xc5D1\xecMGK\x88\x99\x1f\x96\xdb\xbd\xa4w^\xe9\x98\xb9\xb2#!m^\xc6\xc2%B\xa6\xdc\x9cL\xf6\x80{W\xa1\x12TbW\x9f\xa7mXs{z\x1a\x17\xf9\xc6\x83\x93K\xafz\xc6\x88v\xa1.\x97y\xac\x8d\xf0\x83\xcbo\x9f\xc5\xa1\xf9\xaa\x137L\x03y?\xbam\xa0C|h\xc5\xd5\xa5\x81\xa0\xe3\xbc\xecKwS1P\x8f,\xdc\xa4HY\xb8c\x0f\xfb\n\x8a\xd9e\xbf-\x9a\xa6]\x87L3\xb3GJ(\xd9\x06\xf5\xd6\x01\xba\x94\xd2\x8c\xb3M\x15?\x04)wTr{\x7f\x81\x95GC\xffI\x1c\x9a\x9b5\x11\x94\xc5n[\xed\xff,\x13\xdf\xac55[\x9c\xf7\x16\x8f\\\x0e+e>O7< \xd2\xa3G\xa9\x92\xe5E\xbe\xb3\xfc\xcd\x8a\x0b\x03\x9f\xd8\t\xbb\xca[\xb7\x11\xa68\xaecpvvY\xeb\xd4V3\xbdM\xbc\xd9\xb7T\xb7`\xbc\xcd\xd8\xfa\x82\xff`C\xfe,\x0e\xcdW\xe8\x03\xa7\x05m\xc2]\xab\x8a^\xab\x95\xde\x141v\x00\x83D0]\x10\x89R\xc6m\x81\x1e\x0b\xc1I\x0e\xcc\x0c\t6#\x96e1\xf2\x97\xd9A\x97\xa6\x14\xacm\xf0\x8f\xd93\xdeY\x97\xb1[\xf7H\xd5\x1aa|\x91|\x05\xdeT\xb8\x10\xfa\xdb\xf6\xc1e\xed\xcf\xe2\xd0|Y\x93&1\x16\x8cX\x7f\x96)T\xd9n9J}A\xc4"2\xeb\xb8\x84/\xb9. \x98\xdev\x05\x8fR\xd6\x8b]\xa7\xea\xcc\xd8\xa1&7\xea\x89\xab\x0b\x8b\xeb\x8aS\xa3\xcd~\xdfT\xf3%\xe6\x8e\xa1\xdf\'\xf6X\xae\x03\xd1StS\xa0Pi\xfe\xd5\xfe\xed\xbf7\x87\xe6k\xae\xa1h\x84\x04\xcd\xf0\xdd\x90ZO\xb3\xc8l\xb6\x95\xd2\xf4n\xcc\xc4!;\xe5\xeby%U\xda\x14\xa6[\xfc\xd2\x98 \xcb\x95\x06\xeb\x06\xa6\x9aY\xd7\x10\x82H\xbfa\x0cWX\xf4\x86aZ\x11B.w+\x84\x99MK\xa5\xba\xdd\x89AV\xf6\xf1\xc1\x17\x8dg\x81h\xbeZD\x86D(\xe6\x1e\x91\xecRD\xe3\x1f\xa9%\x1f\xe5\x82\x85\xad\xa86=\x03\xb1\xb5\x0c\xe6\x99P[\xd9\xae\x8f\x95\xde\x9e%\xd5\x8b\xd4%\xb6\xce\x07\xaa$v}\x8ds\xa0)\x06\xb9\x7f\x87\xb8\xa3\xa0\xed79\x967\x1b\x9b\xd0\xdb\xddU\xe5\x83\x07\xe1"\xcf"\xd1|Y\x13e(\x94\xf9\tp\'\xc8\x15z\x99]#;\x84\x9a|\x95m\xa4\xcbnu>\xe9K\x04I\xd3\x9e\xf1\xe9T\x8a%\x069\xf2|?.\xadhi\x15\xb1\xd5%$\x0b7=\x14\x13\x12\x1f\xe9\xbc\xee\xec\xb7^bd3M\xb4\x95\xea\xc1\xea\xafH[\x7fo\x12\xcd\x8f!\x98&q\xe6~\x9e\xa0\x0e\x83\xea\xbb`\xb4\xd7\x91\xbe4\x08\xe3\xa8g\xa7\xdd\xbe\x11\xce\xc2y\x95\xa5\x07\xcb:\xa4X\x18\x06v\xb4\xcf\xc92\t3e\xb5\xbez\xbd\x91\xa2\xae\xa8\xba!2*3f]Z\xea\xc4\xb6d]\xf4\xe5h\x99\xe6\xfc \xaa\xf0Y$\x9a/\x99\x08Kc\xec\xfdt\xa8A*e\x9c\xccV`\x17\x9d6\xa3\xb8\xc8C\xeb\x9d\n&\xdd\xcb\xc18\xf3\xeb\x9cZ6Vj\xce%\x9a\xafa\x88Y\x10\xc1D\xf6\xbbM\xd5\xa9\xf3\xe68\xd9\x9a\x94\xbb\xf6\xd2\xeeKV\x0e\x08\x9f\xe8XiQb\xce7\x91h\xbe\xea\x04A\xdcP w\xcdM\xe3+k\x1d\xcb\x04\xd1\x1f\xae\xb3\xd5\xae\xb9\xa9P\xe8`\x1e\xfa\n\x8f"\xd5\xd3O\x02\xce\xa4\xc2\xde\x87l\xba^\xeeD\xd3I\xc9\xa3\x95(\\V\x1c\x88\\-K\x9e\x88\xe5\xa8Z\x95\x17\xccG\x8d\xd5x\xde_\xfb_\xc6\xc4\x8bI4?\xca!\xc3\xb0?ym\x8c\x97\xf9l^\x82\xbeS\xdb\x81h\xae\xfd\xbe\x96\'\x1e\xe3:\xa1?\x86-t=\xa9\xaa4\x18\xbe\xa4\xd6}\xe1^(\xdb8W\xa1\xa5"\xe7\xfc\x92 |\xc7\xf6{H\xe1=\xcf\xc9\xf4:v\x08x\xb8VW\x8dL\x1e\xfc\x94\xe1Y$\x9a\xaf\x8e\x9c\x05\x83\xfeO\xa8\x1b\xa7X\x97\x14\xb4[#=\x15l\xb93|\xf2\x1c\xb7:M\xe7$\xd0U\xb5`\xcc\xf6\xacD=\xe9]\xda>f\xb6\xd8\xf1\x9a\xd3~\xb9\x8b\x025p\x9bk\x19i\xbd\x11\xec\x11\xf1\x9a\xa7\x0bU\xd9\xc5\xaeb\x9f\xa4u\xfa\xa0\xccg\x91h\xbe\x1eU\t\x8a\x06c\xe0]Yn\x0c\x0c\xcce\xceQ \xc7\\m\xd6\x1cE\xec\x90|a\x9d\xae\x0b\xa9c`6\xab\xd7V\x0c\x0c\xc9\xd8\th\x18=\xdb>\xed\x97\xd9f\x89oN\x12Z\x04$}\xec8\xe3x\xbc$\xae5J\x91&U1\x96v\xe9\xafR\xd0\xdf\x9bDs\xcbp$\xf02\x94\xbdon\xf0\xa0\x13\x8eW\xfc\xe2\x07\x18-\xaf\xf4\xac\x10\xc3\xd4"\n\x85\r\xb7LN\x959\x07\x9c$Q\xa9CC\xd7\xc1\x05\xbe\xd6scCP\x85\xb0Z\xbd\xdf\xb3\x1b\xd2<\xda\xbb\xb3>\x9c\'\xeeP\x1e\xc9!\x82\x03s0\x1e\xec\x84\x9fE\xa2\xb9\xc9\xa4(\x16\x017s7\xeb\x1fuc\x9bd\xb0\x7fH\x9a#|$\xe7\xf32\xdf\xed\xd6\x81\xaf\x12\x02\x1b+~\x12\xd2\xe9\xa1\xbb\x1eq\x05\xe1\xe7\x06=v\xd1%\x97\x8f\x8axh\xea\xf1\xbc\x1d\x830\x94\xd61U\xe3\xe6\xce\x86\xf9\xeb\xe1TeFi?\x18\x13\xcf"\xd1\xdcb\x02\x84\x17\xc2\x82\xe9\xe6\xcf2\xcb1^JQ6o\xf4\xbe\t(l\x8bI4\xbb*\xd5]\xdc\xe4*_\xc8\x0bw\x1f\xdb8\xa2\xed1R\x9e\x9a\x08\xb1\xa0\xae\x08\xa1*\xab\xcf\xb6\xcc\xc0\x12c\x10\xd3\xd6d\x82\xed:\x1d\xe1\xd5\x91\xc6\x13\x029\x11\x0f2\xd2\x9fE\xa2\xf9\x1a\xdf\x18\x1c\x0c\xb5\xcc]Y\x1e\x95\xac`q\x16\xdf\x16\x85\xa5\xf7\xe1\xe2<\xc5\xd2\x10\x06ASIB\t\xab\xdd\xbaS\x93v\x1f,\xa6\x8a!\xe5\xd8\xae:\xb2\x9cXe\xb7\x9d\xbc\xfd"`\xa7=\xac\x87n\xed\x05n\xe1\xebg\xfc0\xe0\x83i+\xef\x15\xfa\xcf"\xd1\xdc^3\x91\xdbG\xd1\xe8=\x8c\x15\nZ\x9c%8\nr\xe6bD]iZ\xf2\x83\x98:\xf9\xe2\\\xa1\x12\xe6iRR\xd6C5-\xf6\xab\xfa,&\x911\xea\xdaYI\x16\xee\x8a\x0f\xfd\x84 \x13\xcb\xe6\xca\xc1\xa7\x83]\xc6m\xa0\xa8\xe9r\xfc\xc2\xfc\x9b$\x9a\xaf\x0e\xfe\x9fI4?\xbe{\xf7\x81\xc4\xfc\xf1\x93u\'\x9eL\x87G\xbf~\x85\xf1\xcf\xd8\x96\xff;\xc8\x98\xa7\x93#~3E\tY\x95\xeel8.\xaa\xcf\x1c\xa6\x8b\x15n:\xdc\xd5(]DW\xb7\x97\x18C/\xbf\xbd\xe8wU=]\x8c1\xc7\x83>g\xa3)\x02c8\xf6h\xcc\xd5\xac\x8b\x1cf8\xf15\xf1\xd89\xf2\x98?\x99\xe7M\x81\x1e\xdfi\x96\xa7\x03_~3\xca`\x04b&Ctq}\xaeF\xa3\x04&)\x03\\\x175\xdc\xfc\xa9\x987E\xd7|)16\xc4O\x95\x18\xf3\xf6\x17\xf9\xe7\x8d#Fw*`\x1a\x0e3\x1d\r\x01Q\x83\x19\xa2\x8e\x001WC\xacf\xc3\x7f\x9di^\xa0\xa4\xe4F] p\xbd\xccH}\xb6g\xa3\x94\x10cvIC\x0c0S\xdc\xbd\xda4O\x07\xd0\x00\xd3H\xd3\xaa\xd4\x08\xd3\x91\x80o\xb9\xc04\xdc\xac;\xfa\xac\x97@\xd4\xe1\x85Q\xf3\x12%zA\xfc\\I)\xbf\xdc4\xcf\x06\xd9\xdc~\x1f\r\x98\xda@\x0b#\xda#\xf05\xe0_6\xf09\xf0\x7fGB^Yg^\xa2\x04\xd4\x99\x9f*\x99\xcd\x9f\xc6\xff\xfbB{~3\x1c}0J\x1d1\x9c\n\x05>v\xbdu\x85 \r\x00q\xf1\xf8J\xb3\xbc@\xc9\x1c` \xf6A#\xe3\x8e\xa0y\x01\xe6\xe0@\xa2\x04\xc9\xd2\xd1\x08\xfd\xb5fy:P\xe7V4\x07\xe0O\x84\xe1p\xa4y\xfb\xdd\xe7N\x05\xb2t\xfc\xd5\xdb\xbc4Z^\xa1\xe4\xba*\x7f\xae\xc4xq\xb3\xfcl(\xcfo \xc8\x07P\xea1\x1d\xd4\x18\xa3\xacnm\xe6\xfc\x15\xfa\xa2\xf6\xd2hy\xbe\x92_\xf4\x97\xa0i\xd6I\xfd\xa7\x95\xf2}\x01D\xbf\xddB|U\xdaW\xd3\tPP0\x81\xa8\x0c\x98F\x9fLQ\xbf\xfe\xdc\xc7\xde\x14Q\xf4\x9dfy:\x1c\xe8G\x12\x13\x7f.\xe6\x95\xd1\xf2\x12%\xdfe\x96\xa7\x03\x86@\xd7\x1f\x80N\x0c\x14J\'\x9bL\xc7\x05\xc9\xcc\x9e\xbe\xc4\x94\xeek\'\xfe\xa7+\xf9U\xdc\x83Zs\xfd\xf9\x90\xfc\xbe0\xa5\xaf\xdab\x80$a\x941\xe8\xc62\xd0\xbfh\xe0\n2\xf26V\xbe\xd4,OW\x02\x1ac|Uf\xa0\xc4\x03\xa7r\xa4\xf1\x96\x86\xf5Y\x02M\xb2\x86\xbc\xb8\x13{:\xe8\xe86\xed\x0f\xa6(\xa1\xa0\x95\x04}\x8b\x84\xebN\x06\xfc\xad\x02\xbd\x8c4\xbd\xb4A~\xbe\x92Y\x07I\xecg\xcd\x8b\xf4\x8b\xe1\xf8}\xa1N\xbf\x99\x1b\x82\xd4\xc5\x18\x08\xc9@\x16\xce@\xf8\x83\xf6\x12\x98E\xff\xe5\xb3\xc5\x9bb\x9f\xbe\xd3,O\x07.\xdd\x9e-\x06\xfd\xfe\x01f\x02\x85\xf8\xc5s\xcb\xd3\x95\x94\xb7)\xdf\xc5A\xf2B\rP[L\x91\x03&\n\xc0_\x83T\x88\xbf\xd6,\xcf\x866\x01\xb3\xdc\xe6\x96\x18$\x8d\x0c\x03Y\x19x\x97=\x02S\x91\xba\x13\xbc\xf6\xf1\xe5\xf9J@\n^\x95`\x84\x9cA]qt\xe0h\x1cPR\x11_\x1d\xd9k\xcd\xf2t\xf0\xd3\xaf\x92\x18n\x88\xbfz\xe0{S\x84\xd5\xadJ\x82N\xec\xbeJ\x82\x06fzq\x83\xfctx\xd4oFA\xcc\xb7p\x07\x9d\x18\xf8\xb3~\xeb[n\x9d\xfe\xa8\x97\xaf\xad-\xcfWb\xde\xd2\xb1\xe3\x82\x9fZ]\xf5\xaf\xea\xa8\xdfL\xf3\xe5l?\x8f\xfb\x0f(\xeb\xff\xdfs\x82X\xe0@\x9d\t@\xb2\xad@\xed\xcb\xc0TR\xdd\xb6\x08!\xa0\xb0\x13/\xbe\xf4\xa7#\xaa~\xfb\xf1\xf1\x10({s<\x82\x9c\x0b\x06F\xe0\x99\xb3\x06\xae\xc4xi,<_\x891\x11`\xccuA[rkI4\xf2\xeb3\xc82&\x7f\x1d\xd5\xef\x8b\xe3\xba\x99\x05\xa4\'\x0e\x05\xb3\xfb-\xe7\x82\xc0\x0eF\xe3\xf6@1\xbf8E=]\tH\xb6\xd8\xdd#\x91\xa3\xdd\x16\x86\xbe\xdc,\xcfFe\xdd\xcc\x02z\x1b\x17x$\x08\xf7\xb2\x02!\x0fz.\xe7\xf6\xe1j\xfdZ\xb3<]\xc97\x9a\xe5\xe9\xb8\xad\x9bY\xae\x86\x93\x81hqA\xc3\x18\x03A\xb7\'\x16\x90\xcb\xc5_}\xfd\xe0M\xc1a\xbf\xe9\x1bb\x06\xfd\xd5\x08j\xca\xd5\xb85\xf4\x8ev\xfb\xc8n6\xe6\xc5\xeb?\x12~2\xb2\xebf\x96\xf16\xfd\x82\xfa\xf2\xb5Z\xcft\xdc\xdb\') Z~\xe5co\n\x1f\xfby:\x06}\xd6\x7f\xe1K\x14O\xc7~}\x99\xe5\xebK;\x8e\x86\xdc>H\x05S\xc9-\x15\x80)\xe5\xb5I\xec%J\xbe\xcb,OG\x87}\x9bY^\xa2\xe4\xbb\xcc\xf2t\xfc\xd8\xb7\x99\xe5%J\xbe\xcb,OG\x98}[my\x89\x92\xef2\xcb\xd31h\xdf\x16-/Q\xf2]fy:J\xed\xdb\xa2\xe5%J\xbe\xcb,O\xc7\xb1}[\xb4\xbcD\xc9\xb75\xc8\xcfF\xba}_\x83\xfc\n%\xdfV\xf2\x9f\x8d\x85\xfb6\xb3\xbcD\xc9\xb7%\xb1g\xa3\xe5\xbe\xad\xb6\xbcD\xc9\xb7\x99\xe5\xd9x\xba\xef3\xcb+\x94|\x9bY\x9e\x8d\xb8\xfb\xbe\x92\xff\n%\xdf7N>\x19\x93\xf7}\xb5\xe5\x15J\xbe\xcb,OG\xed}[\x12{\x89\x92o{\x13{6\xae\xef\xfb\xde\xc4^\xa1\xe4\xdb\x92\xd8\xb3\x91\x7f\xdf\x97\xc4^\xa1\xe4\xdb\xa2\xe5\xd9\xd8\xc0oKb/Q\xf2\xa0Y\xfe#\x18\xd8\xbf\x1c\xe3\x7f\x85\x81\xfd\xa9 \xbe3"\xf1\xf5\xc2\xbe\t\x91\xf8za\xdf\x84H|\xbd\xb0oB$>&\xec?\x01\xee}\x13"\xf1\xf5\x16\xfb&D\xe2\x7f!y|\x10\x89/A$R\xffs\xe7\x7f\x8dH\x94o\xe85Y\xa4h\x84A)\x9e$d\x8aC9\x9c\xe5X\x94\xc5DDBy\x96\x969V\xa4$\x01\xe7)\x84d9D\x129\x8e\x05\x87\xbf\xe1I\xbeJ\xf5\xaf\xe9_,\xc107\xec\x1e\xc2\x01=\x0c\xca\xf3\x08\x86\x13$\x8a\xd1\x94 Q\xb7=\xa84+\x08"\xcb\x908\xcf\xf1(\x8b\x80_]dEp\x10\x01\x15d\x8e&_\x89H\xfc\x07Q\xe1\xb7\x9f\xad\x1fc~\xa7H\x9c\xc1\x80\x03\xe0\x7f\x85H|\x7f\xbe\xe4O\x10\x89\x82$\x124-\x8b\x08A\xb1,J\x02\x8b\xc8\x84$\xdc6\xd12\xe2m\xdf\xfc\x8d\xbe\x86\x01\xc3\xf3\x0cAR\x8c@",\x8eq,\xc7\xe3\x18\xf0\x0e\xeck\x07\xda\x07\x91\xf8BD"A#2-\x13\x94\xc414\x8b\x01k#\x02\xc3I,\x8a\x08\x02-\xa0\x0c\xc1\x0b\x18C\x83p\xc2d\x82\x13q\x1a\xfc\xf7,\xc1\x8b\x12\x01,\xc9\x93\x02\x86\xfc\xf6sD\xe2\x13\xec\xf4\x7f\x15\x91x;\xc8_"\x12\xdf?\xce\xbf\x15\x91H\xe0\xbf\xa6km\xbc+L\xae\xbb9]$\x85\x8ap\xec\xc8\x9e\xdbs\x82\x1d-&\xb3|C\x1a#\x84\x9e\xc2~\t\xcb**c\'\x96\xe7\xaf\x8arf\xc8\x9a8\xef\x8e|\x8d\'\xe6\n>Zj\xb9\xdd\xa2\x1b\xba\xcf*4\xc8\xe5\xc3\x83t\xad\xa7!\x12\xffT\x15\xfeY\xe6\xaa\x8c\xe0Dc\x93\xbc\xf7\xfb\xa9V|\x9a\x0f7\tEc\x9dt\xa6=c.\xa7\x8eTT\x1c\xea\xfd\xcd4\xcc\xfcQ\xb4\x16\x96HJ\xa2\x05\xf1^8\xa4(\xa5\xfbD\xcd\x1e\xdc\xc5\x16a\xb9J\\\xafW\xacg?\xba\x8f\xf6Y\x88D\xe6w\x9a\xc0Q\xf2n\xdd6\\\xf7+x\'\x95:\xa2\xba#\xd5-\x05o\x89\xc1\x83\xe9L\xd5\xd0O\xc6\n\x1b\x84\xb1>!D1,k\xc3\x85\x12\x97o&\xa4\x89M\xb3\x17\x03\xf3:\x9f\\3\xeeM\x1e.\x1bb\x93\xe1\x1df\xa4q\xc8\xb9\x0f\xae\xdb~\x1a"\x91\xf9\xfd\xab\xfb\xb8\xc1\xd2\xfe,\xb3\xc5\x08\xa9l.g^\xc2\xe2\xe39\xa2m\x1a\xdb\xc7\x82p\xc2K\xbc\xa9cw\x19P\x9a\xb01\x8a0@\x1b\xebH\xaa\xb1\x135\xb5\xd2\x90\x9cg/O\x84*`\xc9\xdcm\x04t\xb5\x8f\xf6(\xbb\t.\xa7F\xe1\x1b\x9e8\x9d=hO\xc4\x856.%\xf1\x94\x1a\xdbq\x9a\xe2\xa4\xab\\(\xc8\xd4\x07\xad\xf94F\xe2\x0f\xa7%@@\xdf\xc9\xbc\xf0k\x83\xb6\xe2\x18\xb4\'g\xd7I\xb6\xf4Z\x12\xd70Dhb\xb4\xe8Oh\x01R\xbc6\xc3\x16t^{{\x05\x1a\x88\xb9\xf4\xfd\x1eY\n\xf9\x96K\x98\x1b\x18J\x83E\xf60`y\x0f\xf2\x07\xdd\x8cmb\xc3\xdf\xc5H\xbc\xc9\xa4\x80\xce{b\n\x89[\xcd`\x9a|\xbb\xf0\xfby\xb5\xd9\xb8\xf9~U\xac\x12\xdb\xd0&\x9ch\xa3E.\x9d\xca\xac5|\xc4^b\xba\xb7\x1aB\xbb\xba\xda\x95\x1b\x07\xd7\xd9\x19\xcd\xca\x8a%{\xe7Q\x04.\x08\x97\x92?-H\x05\xb1\x92_Q\x84^\x8dHd~\'A7\x86\xa3`\xd2\xf8\xb3L;M\x18zBxW\xc8\xd6\xcc\xce.\x1d\xd15D\x07\x8f\xf8i\x8c/\x08\xbd \x14\xfe\xc0\x93\x97 T]\x93\xb1\x85\xa0@"\xb7\xac\xdd4-\xa82\xcb\\\xed`b\x83\x1a\xa0UFH\x12\xc5\xc1\xe1\x88\x04\xfb\xf7"\xfa>\r\x91\x08\x8a>M\xe1 \x82\xee\xdbT\rJ\xc3v>\xc8\xbcAn\x94\x88\x0fYH\x1a\xf7D\xb4\xf1F\xb5B\xe8\xc1\xc1\x91\x01$\x1c?\xdfb\xea\x9e<\x1cK\xcc:\xe9\xad\x89\x10G\xeb\x1a\x0f\xe6\xb29\xe7\x04\xd9\xf9v\xa3\x0c\x08\xebhrB\xb3\\\xf2`\x82{\x1a"\x11\x84\x04\x8db7\x0e\xd7]\xe4\x9b[P\x19Ln\xdb\x0fM\\k#UqGr\xbbt\x16\x1d+\xb73\xe9\xda\xb8\xa7\x16m&\xf8Cp\xf0p\x05\xd5/\x84\x17$\xaaQ\xd9\xfe\x86\xf6\x8e8\xb4l\xa1\xb2"\x9c\xb8\xe7\x10>tT\x96\xe4\x97\xbb\x07\xa9wOC$\x82<\x8e\xdc\xe6i\x84\xb8k>\x04\x94\xae:\x0f\x8bNC\xaa\x1c\x8d!h\x94\x9a8!\x16\xbaK\x9ap!\xd9\x17tM\xf5\xe8f\xd6\xad\x8a\xe6\xd1\x0bb_\xa0\xf5i\xc1\x0e\xb8\x11nQ2\xdb\xac\xfbX\xccb%\\:y\x18\x1f\xecuj\xf3y\xfe+\x1c\xdc\xdf\x1c\x91\x08n\x114\xc1\xb7\x17\x89\xbb\xcc"iC\xac#\xe7\x0b!\x12\xebh\x84\xdc\x98\xf7\xce\x94%\x90\xf9f\xb8\xc2\xf8\xc6\x86\x0f9\x85\xa4\xb0GH\\\xcc\x12\xcaEQ\xb3\xa5A\xa3W\x19q\n\xeb\x18,\xd6\x0b\x91ku\xe3\xd4\x81.\xf0\xb4\xec\xa5\t\xbd\xb6\x0f\x96\x89\xa7!\x12oe\x02\xb4z`2\xba\x93\xa9\xa4MW\x17\x11\xd3(\xfe\xc2\x8b\x84\x9d\x0eU\'S\xba$\xfe\x11_h\xc8\xeaD_\xe8 \xec\xc9\x93\x1b\xa4\x10<;\x95\x7f\x1c+\xafB\x89\x0c\xbe\x1e\xa8s~\xec*jt\x88\x1daxg%5\x8a\x92&\xc2\xfc\xd1\xd0\x7f\x16"\xf1fM\xe2\xcb+\xeed\xca\xf6tp\xd5d/\x84d\x1b\xcb\xce\n\xe9\x17\x98\x00\xebc\x05\x1dUo.\x8fd\xe1m\'G_\xa1\x11\xbfX\x84n\xb4:\xc3\x84\x11\xb3\x91\xb5\x84M\xcb\xdb\xc1\x95\x12b|\x8c\xb0\xb3\x82,$\x7fZ*\xba%=\x8a\xb9~\x16"\xf1kB\xbd1\x12\xc9\xbb\xd0\x17\xa3d\xcf\x1a\x17|\x08\x89\x8dyU\xd7\x94pY\xbaml\xf3\x01\x9d\x85\x10\xc4\x8a\xf1V\xa6\xfc\x8c4Ec\xe8*\xd9\xa6\x11\x92\x8b\x13u\xe7\xc4\xc8\x0e\xbeh\x0b\xb2\xdf\x86\x92\xc0\xe8Z\x19-\xa3\xe4\xba8H\xcb\xe8A\xa7}\x1a"\x118-\x06\xfa#\xd0|\xdf\xcd\x1dpw\x94\xda\x00\x85\x90\x0e\xcd\xb5EL\xc2\xcbE\xa8V\xc7\xa4\xd9m\xf6Q\xbe1=\\\xcb\xeci\xa2\x1cs)m\xb7\x96h\xc5\x892\xbb\x03Q\xb3\x17o&`8<\x1e\xa4\xd4AI5\x19\x82\xa1\x9d\xe3\x12\xd1\xf6\xbf"\xa6\xfc\xcd\x11\x89_\xe5\x90\xc4\x81\xc7\xdc9K0%\xa9~\x9e\xe0\x95\'\xa7\x8e:\x0f\xe4\xce\xb9x{\x95\x9c\xce\xebr\x92\xcf\x1de\x91\x93\xb49!\xac\x9dCd\x0b&\xbd\x19R\x9b\x0e\x89\x16bR&\xab6\'\x80\x7f\xa1\x13\x04#~\x81\x0c\xc3Y[\xb7D\xfe(\x13\xfdY\x88\xc4[\x8b\xc8\x82\t\x01\'\xeeB\xdf\xd7Ta\xe4g\xd6]\x85N\xe9\xe95\xe3\x910|\x00\x15~J\xce\xe1P.g\xbd.5\x8b\x17\xda&T\x1a\x7f#NT\xbb\xd0O\x12[\xae\x15\xbe\x85T\xb1e \xb8I\xe5\x90\n\n5Ng\xc8\x9dk\xe7A\x8a\xd0\xd3\x10\x89\xc0\x9a\x04\x98j@<\xdd?\xdc,\x04\xad\xf4zbp\xe0~\xbb\x88\xfc\x03\x1f.\xaf.&\x1c\xf9\xba\xe4/\xf0\xec]B\x0c\xa3\x0e\xe9(W6!\x82\xc2)+)m\x85V\x1c\x16\xe5vb=\x89\x99\xcc#$\xc9\x99\xd9F\xae\xc5\xb6]\x14\x15\xefU\xf5\x9f\x86H\xbc9\x0b\x18\x82q\x92\xbak\x11\xd3\x1d\x8fLK\xe2\xd4+\xf6>Z\xe7Qt\xbdL\xab\xe5$\x9c\x98D\x11\xdd\xa5\x9f\xdb\x9a\x9a\xc6\x9a\xb1\n\xe4\x13/\x94\x8e|T\xfc\xe88\xcc\x07\xe1\xc8!fP\xaf\x99\x921\x10\xcb\xebv\xd8\x86\xe1;\xb5<-\x99\x07_\xf9\x9e\x86H\xbcU}\x8c\xa5\x1806\xdf=\xd8vp\xc5\xf8b\xe23\xf9Z\xed\xb8-\x13\xef\xf6\xb6\xa3\x05&\x9b\xa3\xcd:\x81X$\rf\x95\x1f\xb7L\x8b\xe5;\x1a6\x89S[\x8a+V_Y|R^cC\xa8#\x9b\xcdFwC\xe5\x0e\xbf\xd3)i1\xe7\xfb2\xb7^u\xa5/?\xe7\xfb\xf2\xb2^\x15\xf8/?\xe7\xfb\xb2\xae^\xf6A\xc9\xab\xcf\xf9\xbe\x9c\xaa\x97=\xee\xbd\xfa\x9c\xef\xcb\x98z\xd1\x95\xbe\xfe\x9c\xef\xcb\x87z\xd5K\xd4\xcb\xcf\xf9\xbel\xa7W}P\xf2\xf2s\xbe/\x97\xe9e\xad\xfe\xab\xcf\xf9\xbeL\xa5\x17]\xe9\xeb\xcf\xf9\xbe<\xa4\x17]\xe9\xeb\xcf\xf9\xbe,\xa3W5Q/?\xe7\xfbr\x88^U\x9e^~\xce\xf7e\x08\xbd*\xf0_~\xce\xf7\xe5\xff\xbc\xeaJ_~\xce\xf7e\xf7\xbc\xeaJ_~\xce\xf7\xe5\xee\xbc\xac\xd5\x7f\xf59\xdf\x97\x99\xf3\xaao\x9b\xbc\xfc\x9c\xef\xcb\xbby\xd5\x8c\xff\xf2s\xbe/\xab\xe6e_\x86|\xf59\xdf\x973\xf3\xb2/\x96\xbf\xfa\x9c\xff\x11\r\xe1_\xec\xf6\xbf\xd2\x10\xfet\xa4wf\xc4\xbc^\xd871b^/\xec\x9b\x181\xaf\x17\xf6M\x8c\x98\xc7\x84\xfd\'\xc4\x91ob\xc4\xbc\xdeb\xdf\xc4\x88\xf9/$\x8f\x0f#\xe6%\x8c\x18\xfa\x7f\xee\xfc\xaf\x191\x12#K\x12)I\x02\xc6\n\x1c&\x10(!r\xd8m\xd7\r\x83\xf0\x18/\xd0,\xc9\n\x14-\xf2\xb7\x15\xd9(*\xf0\x92H\xe3\xb7}&\xb2pC?\x10\x7f\xcd\x88\xa1$L\x90\t\x1e\xa3PZ\x92o\xbb\x828\x91\xe3o\xeb\x9dH\x0c\x91E\n\'h\x81\x171\x06\x93\x04\xf0\x8b\x90\x84\x8c\xe38&s\x08\x83\x8b,A!2\xfdJF\xcc?6\x93\xfcv\xb7\x80\x81\xf8\x7f\x10\xe4w\x0c#\xd1\xdbfm\xec\xaf\x181\xef\x0f\xd8\xf9\x19#\x06\x17P\x06\xd8\x9bCi\xe0b\xe0\xc6E\x0cEHT\xa0D\x19\x18\x86\xe7o\xdb\x89\x10\x94\x06\x86\xb9\xb1R8ZDI\x06#\x80\x8bH88\n#\xfd\xf6a\xc4\xbc\x94\x11#\x88\x84H`\x14\x85\x922\x03n\x96\xe19\x82\xa4A8H\x1c\xce\xa3\x18\xb0\x00"\x13\x14\x07\x82\x9e\xa5\t\\dX\x0e\xa7q\x19\x84\x0e"#\x98 |\xf9\xf5\x87\x11\x93>\xc2\x88\xb9-6\xf9\x83\x11\x83\xfe\x94\x11\xf3\xfeq\xfe\xbd\x8c\x18\xf6\xd7\xf0\x94qT`+\xd2\xdc#R:\xd5\x84s\xda*mV\xc2\xce\xc5\xe9I\xf3\x9a\xb1I\x0b\x15)\x04\xbb\xa3\xb2\x01\x8e\x07"\xca\xdbp\xde"\x07\xdd\xd1\xf9%\x95\x8c\x8e\x02w\xea\xb2\xf1(\xb8\x9f-e\xcb\xa7C\x9a#\x0fn\xe0}\x0e#\xe6\xbe*\xfc\xb3\xcc\xd9\xc40\xcb\xdd\x86\xbaa\xaf\x88\xb3T\x87\xfc\xa2\xa6\xf6\xb6L\ne\x9alb\xceZ\xae\xc9C\xb2\x90x\x9fC\x8aesa\xfcu^\xc6M/ynd1\xd2\xb2\xe1\xaa \x82\xcd\n\x8e\x995\xc5{\xa8<\xf6\xc5\x83\x9b\x15\x9f\xc3\x88\xf9\x92\x893\x14u\xafQ\xf5\x95v\xdb\xacZ\tv\x98X#\t-BO\x99\x0c\x05\xa3\xba:\xd0\xc6\x96\xd8{\xc4\x96L\x85\xbaW\xcf\xbd+\x14\xc2\x92\xcdq\x0cO\xa72Z\xa6\x83\xa2\xa9\xbb\x95p\xc6\xf09F;\x85\xa5\xdci6\xf9\xd8}p\xa3\xeas\x181?4\xa2\x04\xc5\xd0 #\xdd-\x8e]K|K\xdb.T/\x92\x95\xb50\xe1Mg\x1fdY\x08ffy\xa0\xd0\xbc\x91fv\x93\xee\xd4\xa2\x17p\x0b\xef\x96\x8c\xaae\xc5\x91\x82Bc\x1a\xb9iG\x99R\xb9=^$vh\xac\x13+r;dA*\xbfZ8\xf8wf\xc4\xfc\x08\x08\x92Fi\x86\xbd\xe7P\xecw\xfa\xe92\x1e\xb9zV\xcd\x94h\x89p\xa7\xea)7\xca\xc6\x95\x17+ek\x8e\xbcO\x12\xb0\x177\xe6F\'\xd71=p\x93?\xe8z`\xd1\x96\xbc\x96*\x89\x8e6\x94\xaeW\xa4\x0b-\xea^\xde\xc52;\x97\x0f.5{\x0e#\xe6\x87L\x06\xbf\x15v\xf6n\xa9\xd9\x82?\x11\xeb\x9c\xe4\x0e\'_\x89\x9b\x15\x81\xd8\xd7t\x93\n\x8b\xc1\x10\xd4\x1aJ\x0f\xce2C\xba\xc5f\x140\xeax\xddF\xe7\xde\x9ah% \n\xa8\xbc\\\x13\xb1uj\xb6\x98\x93q\x8fI\x89\xc5u\xfa\xb4\xf0\x8e\xc3\xf6A\x99\xcfa\xc4|\xc9Do\xd6\xa4I\xe4nE\x1d[\xd9TK\'\xb8\xb5\xd4\xa4r\xa4\xe2\xfcT\xe54V{N\xae\xb0\xd6<\xebl/W\xf3\xde\xdf\xa7\xc4X\x1d\xb8v1\xd54U\xd2\xb5\x12\xede\x7f\x81\xcf[\xd8\xa4\x1b=\x8fq\x97$\xbb\x8d\xc6q+]\xeb\x1e$\xfe<\x87\x11\xf3\xc3\x9a`\xba\x00M\x13}_\xac\xb2#\x81\x1eDy\x01]\xa5\xc3\x1a\xbf\\\x0e\xab\x93pq\xbc\r~\x88\x96N\xb7\xcd\xf8e\xc7\x13\xbe#\xf8\x9e\xa8;\xecq 5y{$u\x15\xbf\xae\x0cX&\xe1P\xc45gU\xc6\xcbr}\x88\xc6`h\xf6\xed\x83\xbbF\x9f\xc3\x88\xf9#6Y\xd0\xb0\xfcYc\x96m\xb5\xba\xdd\x9e[q\xd0N\xca"\xd2\xcd\xd9RylC\xa3\x8b\x05\xb5p/>\xad4\x17\xb1\x12\xfb\xddP\x87V-Z\xda\xa5\xa5\xdd=\xa7NJ\xe0u\x04\xb3rNu\x96HD\xe22\xbb\x99\x1a\xf8\xfeb\xb2\xfc\x83\xbb\xe2\x9f\x02\x88\xf9C#\xc9\xd0\x08H\xfcw\xb8\x1f\x1f\x02\x93&M,\xa8\xa2\xdb\x8c\xe4"\x80\x90\xde\x1c\x16\xee<\xc0\xa2O\x9c\x92!\x8fz\x0f*D\x8bZH\xb0\\]\xaf\xc6\x9a\xd0\xf0}\x7f\xe81W\x80\xa11\xe7\xcdi\xd3\xa6\xab\xb9\x12\x90CG\\91m\x1e4\xe5s\x081?\x02\xf3\xb6L\x95\xfe\t\xd5(q7\xfe\x86\xbb\x94\x93\x86na\xa8G\x16\xd8\xb26\xca~\x0cVJ\\\xa9\xa7pE\xedR\xbf:\xa8\xeeaf\xc8E\x86\xb3x\xbe)\x9c\xcb\xaem\xcaIA\x8d}z\xd1\xd9\x90N\xca~\xb5\xa1-\xe7\xc4\x88\xd7\xb6\xf8\x15\xf4\xe3\xefL\x88\x01\xb7\x88\xfe\x8e\x80I\x04\xcct\xf8]\xdcKV\xb8.\x833[\xfaK\xf3\xd4\x9bZn\xf7\xc8\xb8\xdb\xee\xac\xa4H\xf7\x87Ux\x8e|\x7f\xbf\x14\x9c`\xa9\xee\xb2\xc0>\xc65\xc9^\xd9F\xc9\r\xd5\xdeTCF\xf8\xe7I\xdb\xaf\xc8\xbe\xce\xa9q\x933k\xd5\xd8>\n4{\n!\xe6GL\xe0$K\xa1`\x86\xb8\xc3\t\xa9\xca\xb18\xa6\x1bo\xbf\xf1k\x8a\x818\x98\xb4\t\x8eh\x0bow\xc4\x0b\xe5b\xdb\xe7\xdd\xc2!\x8a\xdd\xcc]\xb2&\xef\xec\xfe\x8a\xef\xeb\xda\x90\xaf\xe9\xf5\xb2[ib\x11\x0c[\xa3\xe5\xc5\xabEY\xb6\xc0e=\x1aP\x18\x04\x07\xec!\xd5r\xbf<\xee\x0f\xfb6U#E \x16$\x02\xef\xd6\x87}\xcdnmw\x9bm\xf9x\xa1\x95\xc5\x9e\x90\xb3\xf4\xbd\x16\x8d>\x87\x10\xf3\xa3\r&i\x92%\xd8\xfb\x92\x9fg\xe1V\xd8\xaa\x02\xb6\xb5\x19\x96\xf0\xc5-\xbb4,\xb6\xa9\x99\xb5IZ>\x92^\x0e\xfdE\x83\x15\xe00;\xe1\xe2\x19ex[\x81\xdd\x90|Z\xe1\xb0\x1a\x9bB\x91N\x1boD\xbcrL}\x85%\x8f\xf1\x9c\xc0\x0f\xe2\x84\x9eC\x88\xf9cp\x03M\x12N\xa3w\xe5\xf0\xc4oa\xe2\xaab\xf4J"\x1d\xd7pM\xcch;h\xc2\xe0m\x10\x9f\xdb\x88\xdceE\x04K\x05\xd2E\x9e\x8aZ\x9a\x9cO\xdb\x84\xea\xd7\xa4A\x89\xb7\xc62\x89Q\xa3\xbd\xa0\xf0\x9a\xb7Z\x01\xd6\xa1\x1d\xd5n\xbaGA8O!\xc4\xfc!\x93\xc6n\x0bx\xefb\xc2\xdc\xf6\xae\xe2\x9f\n\x1d]\x80~\xf5bI\xb2*\xe0U\xe7\xc6(z\xe2Y\x16\xea\x9a\xc1:\x0e\xd7\x9c\xf57;_\x17v\xd8J\xf1s\xaa\xa2PP]\xf9B\xf0\x18\x15\x85\x0f\xa9\xa0a\xf3\xec{\x9c]\x8b}#>h\xcd\xe7\x10b~8-r{\x97\xc6\xeeg\xb7\xdcZ\xc2k\x0b\xd1\xb7\xd8\xc1!\xf8\xc1\xde\tqG\xec\xb6\xfbb)`#S\x0b\x92cy\x1a\x89f(\xdf\x8c\x9cg\x8a\x8e\x0bUK\x9a1d\xca/\xae\xde\x11m\x90`\x17\xda\x81Nz\x85\xb1_\xdd\x9e\x0b\xc2\xf1\xc1\xe6\xe69\x84\x98\x1f\xd6D\x10\x12X\x00\xb9\xdb\x89\x8f"A\xe7\x05#5LE\xdd\xc7\xd9\x8e6x\x10\xabB\xe7\xad\x18\xd7t\xb8\x06\xd5{q\x1d\x0fG<2\xd9\x9a\xae\xac\x1a\x194\xf4\x00\xd3G\x9a\xa3S\xaa\xb2\xe7\xa3\xb7\'\x031Uj\xaf\xe7\xab\x95Y\x96$\xf3\xab\xfd\xdf\x7fgB\xcc?b\x02\xcc\xb2\xf8=\x03\xab\xc7{\xe8\x9a\xc5Lgf\xfd\xe6\xb0\x95\x88\x96_\xa9,\xca\x1d#\x99\xb6\xd7\x8d\\!\x8b\x1d\xbb%6n\x81\x9fD\xf8\x92d\xc9\x96\xe9\xe5\x82_t0\x1c_fD!\x92\xfd\x9e\xdeh\xe4"\xc5b\xeaL\x8d\xc6r|\x90\x9f\xf0\x1cB\xcc\x1f-"\x89\xe14\xc8\xe6w\xcd\rr\xaa\xb8\xd9.LJ\xbaNc3\xa7\x05\xd1\xa7\x17a\xd0\x91\xd4\x11\xba\xc1\x9eV5W\xee\x96\x17=\xce&\x95\xdd\x8e\x1bF\x9a\x1b\xc9\x8c\xdb\xab\x8d,\x0e\xdbj\xdd]\x17\xd0\xca\x9dB\x01\x81\xbdJ0[\xd06?\x18\xfa\xcf!\xc4\xfc\xd1\xf0#\x18\x0e\x92\xc8]\x0f\x87+\xd6\xa2PMZ\x96\xb7\xcc^\x83\xa6\x99/\xe8\xc9+7\x9c\x838Rs<(r\xe7\x16\x98\xee;\xcc\x0e2\xd1\x85\x12\xf92H;\x90m\x82\xc0\x08\x16\x06!\xc8F\x07-\xbbPY\xc3\x91\x86K\x12Z_\x9d\xf7b%>\x87\x10\xf3\xc3Y(\xea\xf6\xf8u\x9fY\x0eB\x06\x86\xd9\xd6\xa0\x8a5g/\xf8\xf5r\xbf\x86\xae\x9c*\x91\xd7\xea\x94ul\x95\x8fb\xa6\xc2xd\xe9\xc8A\xacwrS-\x82\xeb\xac&\x03\xc2\xf6\x1c\xbe\x1bZ\xae\xeas\xbd\\\xcc\x0cb\x9d\x03\';\xf8\xac\xfe\xa0\xb3<\x87\x10\xf3\x87\xb3\xdcv\xa8\x13\xd4]s\xd3f\xbb\xb0ug\xee\xec;\xaa1\xa8\xfb\x84"5\x9f[w\xf1\xac\xae\x86\xa9\xba\xf00~:\xb4\xc9\xc4"\xfbS,\xad\xab\xc5\x81\n\x17}k g\xc5"\xdbj\xd1\xef\x10|u\x99%:\x11ps\x81 m\xb7\xa8\x1f|\x95~\x0e!\xe6\x8fg\x1b\x86\x06-\xf5=;\x85\xbe\xd0%\xd9jq\x0c\xea\x02\xdf\xc2\xb4\xe5\x9f\xa6\x00\x12D\xa9r\xf1p`t/2!\x88%\\\x0c\xaf\xae8\xcd*\xe6^\x0e\x83\xfa\xa2\xb0\x11\\\x0c\xa6]\xb2\x93TU!zl\x0f\xf1\x81\xa1\xdd\xc6\xe9\xa9\xe9[\x081\x7f4\xfc \xef\x93\xf8\xfd#\xdcr\x9d\xea\xe4H\xfa\xe2\x92B\xf5\xd0\xe9\n\xdcrhi\xd9\xf8\xf3\xa5\x16\xe0\x1cN\xa5\xb5\x11(\xe4\xb1\x08M\x9e\x1e\xf0sq\xc6J\x08\xda[\x976jO*D\xf1X\xa2\x97\x9aV\xf6\xf6 \x1b\xd4p\xc227~p\xaey\x0e"\xe6\x8f\x8e\x9c\xa1H\x12\xbd\x7f\x7f\xaf\x98\x1a\xd2\xdd)1\xf7\xd6\xd2\xbc\xd4\xac\xb1& g\xeb\x88\xcc\x81\xb9\x9e\xdck\xab5M\x1d/\xe7\xb5\\\xacr\x98\xdd\xea\x1d\xe8q\x94-\xc4\x08H\xbf\x86\xd5\xbe\x97\xd0TTvc\xac\x91\xa6(6\xce\xce\xc3\xc5\xe2\xc1\xe6\xe69\x88\x98?b\x13\x0c\xe2\xa0\xbd\xfc\xb3J\xd0w3\x8aW\xb8t\t\xd9U\xb6\xd7\xd2>*\x9c\x86\xc0\xfa\x10=\xaf\xa8$\x948\xd2\xa5!MJ\xcc"Y+b\xdf\x95k\xd8,Z\xfe\x8c\xd7\xf5\x8a,\x8dA\xc811\x00v<\x9e\xa6\xa9\x10\x8e\x04^1\xefE\xbf{\x0e!\xe6\x1f\x0f\xb6\xe4m\n\xbc\xf3\x95\x02LG\xf1x\x96\n\xcb\n\xa0\xd4P\x84\x12\xd5\xbcf\x12VG\x1cr\x8aX\x1c\xf2\x80\x19\xf7\x0bH\xadY\x8d\xe8\x85m+\x9d\xd0U\x02\x06:=\xd0v+e\x0eg\x15\x87S\xfb\xe8j\xcbM\x81*\xabFB\x0e\x8f\xc2`\x9fB\x88\xf9\xe3\xc1\x16$\x7f\xec~\xd2\xef\xa1\xa5\xb7>\\k\xcab\xa2\x89\xd67u\xa8\xe4\xe3Rt*h\xc8\xf6\xb9\xc3&\xdavm\'1\x04/\xa0\xddf\x85#\x99aq\xaef`bg\xe7v{:\x0b\x0e\xe1\xa0\xe4\xa1w\xea\xb4\xaa\xedj\xb1\x0cz\xf6A\x0e\xces\x001?\x02\x1f(\xbc\x95\x81\xbb\xd6&\xda\x89D\xa0]\xf7v\xd8K\xf8\x95\x8e\xd7N\x14^\xd1\xdc\xc97\xc2$\x0e\x90\x00\xb7\xc3\xccOc\x16m\xc8k\xb1T\x9cR\xd8W\x19\x14\xee[2\xa9pe\x01\x8f\x88\xa6\x0c\xa4!B\xc6\x91P.\x9eP\x91\xfb\xfcA\x0e\xces\x001?\x8cI\xb1\xc0\xcf\xef\xb1\x97C\xb6f\x90\xa6^\x17vE\xce\xac\x9c9X\x13\xb4\x92\x8f\xe0\xe7\x9d\xbd\x17\xa9FD,|\xf4=\xc3g\x8a\xcdu&\xe1\xc9\xf7\x0e\xb6h\xec\x823\xbb9\xf7\x04\xe6d\xa8\x1f3\xba\xacy\xeb\xfe\xc4\xfb3\x06\xb3\xfd{=\xdb<\x87\x0f\xf3\xc7\xa7\x974\x8a\x03\x87\xb9\xbfE\xd7\xe1\xf3\x8a\xefE\xd6\x9eDk\x90`\xe8X\xd9tc\x18\x12\xb4\xd5\x8f\xa9E\n\x03\x1b\xdaM\xe86\xa2\x14\xe6va@\xf5\x19#\x88\xb3b\xabR\x85O\x93i\xe1\xfa\xb4=\x1c\xb7\xc3\x91\xc5Mj\xcc#\x8d\xe1\xb9\x7f\x8b\x0f\xf3\x15\xc9\x1f>\xcc\xbf\xfdE\xf5\x0f\x1f\xe6[vt}`&\x9f+}\xf7+\xfd\xf0a>|\x98\x0f\x1f\xe6\xc3\x87\xf9\xf0a>|\x98\x0f\x1f\xe6\x8d\xcf\xf9\xe1\xc3|\xf80\x1f>\xcc\x87\x0f\xf3\xce\xdc\x95\x0f\x1f\xe6\xc3\x87\xf9\xf0a>|\x98\xb7=\xe7\x87\x0f\xf3\xe1\xc3|\xf80\x1f>\xcc;sW>|\x98\x0f\x1f\xe6\xc3\x87\xf9\xf0a\xde\xf6\x9c\x1f>\xcc\x87\x0f\xf3\xe1\xc3|\xf80\xef\xcc]\xf9\xf0a>|\x98\x0f\x1f\xe6\xc3\x87y\xdbs~\xf80\x1f>\xcc\x87\x0f\xf3\xe1\xc3\xbc3w\xe5\xc3\x87\xf9\xf0a>|\x98\x0f\x1f\xe6m\xcf\xf9\xe1\xc3|\xf80\x1f>\xcc\x87\x0f\xf3\xce\xdc\x95\x0f\x1f\xe6\xc3\x87y\x84\x84\xf0/m\xc5\xffJB\xf8\x93\x95\xdf\x99\x0f\xf3za\xdf\xc4\x87y\xbd\xb0o\xe2\xc3\xbc^\xd87\xf1a\x1e\x13\xf6\x9f\xd0F\xbe\x89\x0f\xf3z\x8b}\x13\x1f\xe6\xbf\x90<>|\x98\x97\xf0a\x98\xff\xb9\xf3\xbf\xe6\xc3\xb08I\xe34\x89\xf1\x88t\xe3\xb3p\x02Br\x8cHP8\x83\x88\x8cL\xa1"K\x90\x02%\xb0,O\xb3\x18\xce\x10<\x86\xc8\x12%\xca\xa0\xdba\x19\x86\xc1\x7f\xfb+\xf4\x01J\xc9\x1c\xf8\xa9\xb2$\xf1\x04-P2\x8b\x11\x04F\x13\x0c\x82\x134\x85c\xac\xc8\xb3\x1cFS$\xc7J\xa8\xc0r\x94\x8c\xf2\x18*s<\xcd\xf3\x8c\xc0\xb3\xd2+\xf90\xff\xd8~\xfe\xdb\xcf\x160P\xbf\xd38\xce0\x04B\xb3\x7f\xc5\x87y\x7f\xb8\xceO\xf80\x12\x86\xdc\xa00\xb7=&\x82L"\x1c\x8f\xe2,\t\x0e\xcf\n\x12\xc3\xa0\x1c\'\xc84E\x01;a\xa2\x8c\xe2\x04%p\x0c\x85\xb0\x12B"$*\xa3\x94t\xa3\x99|\xf80/\xe4\xc3\x88\x1cAa\x1c\x82\xdcX\x0e\x128\xa1@K\xb8\x80\xb32+\xf08\x8b\xa1\x12Is\xa2\xc42\xa2,\xcb$O\x8a\x92\x84\x89\x0c\xf0?\x92\xa4\x18\x91\x96h\xfc\xb7\xbf9\x1f\xe6\xdff@<\x8d\x0fs[l\xf2\x07\x1f\x06\xf9)\x1f\xe6\xfd\xe3\xfc\x1b\xf90_\xab\xb7\x7f\tN\xb1r>>\xadtv\xbd\xde\xee\xc9Ss\xbc\x1aXq\xed\xaa\xd9\xb4\xea}w0\xednmP\xc6\xa4\xac#\x84\xa9\xe5q\xce\x849\x9a\x8a(+6\x92\x16\xf5^}\xecJ2\xa5Wvw\xe8\xd4\xc4\xd9^O\x0bU\n\x1f\\\xbd\xfd4>\x0c\xa8\n4\xc1\xe2\xa0p\xdf\xad\xe5\xe9\xd6I\xe4jL\xaa\xad\x9d\xd2&B2\x9f\xbd\x8d\xdb^H\xb2\x81\xa1\xb5pH\x14\x83:\x86\xec^\xbej\xc2)]\xc1\xdb\xad0^{\x83\xb7\xe0Y\x95\xcb\x93\x1av;\xc6>F\xe5\x81p\x13d\xbfw\xe6}\xce\x84\xbfZn\xf4j>\x0c\xf5;\xcb\xa0\x14s\xb7OU\x915\xff\xc4\x06H:\xdb+I\xad\xb1r!\xdaW7\x823\xe2\x80W\'\xdb\xb8\x9a\xb8\x9a\xf6\xae|@\xbb\x93\x94\x07\xf1\xec4\x1c\xc7H\x9a\xe5\xe3u\x0ccZC\x8eJ\x87A\x9c\xb5\xc1\x06X\x14\xf433\x1e\x1f\\\xaa\xf84>\x0c\xd0H\x82`b\x18\xf6\xcec\xc3\xb4\xcd\xbd\x8b\xd7(U[`\xa6\xb4;\x96\xcde\xe2\x04\xce\xdb\xef\x94r\xefg\r\\B\xfe\xbe=\xc0\xd4\xea\xc8\x13E&.\xf1M\xe5\x8c\n\xcc\xba8\x8dL\xfd"\xd6\x9b\xb3\xcd\xee\\\x9b#\x83+D\x85\'9\xfa\x15\xea\xe7o\xce\x87\xa1~gP\x86\x00\x91t\xbf|wuV\x8f\x1a\xb4\xdc#(\x8a\xd6<\xcb\xae\xe6\xb3\x8f){\x91\x86\x0f\x88\x98G4\x16\x13\xc2\xa6W\x9cT\xa1\xaa\xa3\xc6M\xa7\xa3\x8f*\xcb\xa6\xcbS\xb2\xa6\xa3L\xb1\x16Wt1\xd6\xb1*9Y\xa7\x17\xec\x84\xb8\xc0\x9b\xbe\x87\x0f\x03d\xe2(\x8d\xdd\x9a\xd7\xbb\x95\xfbKF\x1f\xaf\xe8V\x95\xeck\xe1#\x1c;\xcd\xe4r\xb7"\xd3\xe2b\x16\xb4N\xe6\x94e\xed\xe2\xf5\x8c\x91\xa2w%\xafb\x835\xa4\x03\xa7\xb2\xbfn\x97\x97\xc1[\'\xaeb\xd6\x17\xa5\x1f(t\xe3\xc6\xb9Yq\xfb\xe4\xc1\xf5\xe2O\xe3\xc3\xfc\xa9\xe9\xfd\x97u\x83~u,\x8f\x8bk\xb00\x17\xcdr\xe5gV\xbe\x8c\xebA+]yXDq\xe1u\xa7m:M[\xb9vkk\xd8\x05 }\xac\xae\x8b\xfax\x8d"\xe3p:\\\x88\xc4Q6\x94\xcf\x17\x87nQ\xed\x85\xfdI\xde\xe2\x0f\xca|\x1a\x1f\x06\xc8\x04\xb1\x0f\xc6\x8e{>\x8c\x8dU\xaa$\x12\xd6>\xbb$\x8a\xeet\xb4p=\x12MC\x9f\xdd\xf0\xbc\xbc\xeee\xe9\xb0\xa5\x03\x93\xc6}{\x92\xa5\x19W\xa2\x06\xee\xbb\xd3\xa08+\xac:\x04\xb0\xb27\xa6f\xe8\xa0tV\xd8\x1ajV,\xba\xf1\xc7\x07e>\x8d\x0fs\x93\x89b \'\x12wk\x15\x9b5a\xac\xd7\xd0\xaa\xf5\xcb\xca\xed\x0ft.{\x9bU\x9evvh\x9e\x964\x96\xe6\xfd\xb0\xd1\xf9\xaemYGB\xf6G\xe6bH\xd16\r\xc6I\'\x1b9\\\x9ax\xe4\xae@\xab\x1c\xf7\xccaEW\xcbr\x9b\xab\xd4\x830\xb3\xa7!bnN\x0b\x1c\x16T\xac\xbb%\xc0M\x8c+\xb0\xa01\xdcTi.\xa1t\xdb\xedi\xb9\x92\xf8FY\xf6D\x80\x9c\xb6\xd4%\x16[\x95\x11l\x99.\x0e%\\\x96hQ\xabe_\xb9\xee\x06Iw+\xe6\xaa\xc1\x88B\x9d\x0fNA\x90n\xed\xba\xbcg\xee\xbf\x0b\x11\x03d\x12\xc4m\xe0\xa6\xeew\xe2g)\xbeK\x19\xe2\\\xac:\xd8,C{\xe6\x9c\xa1a\xfc\xddy\xb2\xf8\xa8F\xcd\x014\xc4\xa3\xb0\xb5\x9b8GB\x8f\xd7\xae\xc9\x96\x92hhu\x9c\xfa\xb6?\xa72^ \x87\xb3\xdd\x8d\x9c\x80m\xd6\xfcL\xfe*\x8f\xff\xcd\t1\xa0\x11\x06\xb9\x17\xdc8{\xb7]\xbc`/bv0\xae\x0e.\xad\xeb`\x91\xdag\xd9\xb2#\xa8\tL\xcc\xe4G{+m\xdc\xc5\x86\xba\xb2vJ\x1c\xa45k/\x13lC&\xac\xc1\x05\x8e)\x04z=\x1di(\xdc\x185:l\x8d\xc3\xe6t\xc9=Zy\x14\x9d\xf2,B\xcc\xcdY\x18\xfa\xe60w11QI\xd5\x12q:.GK]\xc0[\x9ft\xb4\xc52\xcd\x18\x1an\xf4d\xcf\xad\x96\x1cDO\xd6t\x14\x90\x0bd\x04\xe4\xa2\'\xb7>k\x8a\x05\xb9\x0e\xe6\xd6 \xcf\x89\xe9_6\xac\x7fB\x89\x15\xcab\xce\x06\xdfw\x0fb/\x9fF\x88\x0121\x82d\xb0\xfb\x1d\xc3\xd4$\x80\xd8>\xef\x94\xcd"\xea\xf3Bi\xb2\xe5\xb2\xd6mt\x1a\xd1\xd5\x9a!WE^8m\xbb\x85\x0cK\xa2\x0c6"\xc4\x19\xa5\x08\x91\x10\x04\x01?\xb6M\xe6\xd2\xcb\x12;\x9e\xabCe\x11\xac`-\xe0\xce\x10\xdaGk\xfe\xb3\x001\xb7\x04\x87a\xcc\xad\xb4\xde\xcd\xa8\xa2\xe9\x90\xa8\x03\xc6\n\x179sH\xae\xe8\xfbT_5\xca\x0c\x1fE\xafi-{\xec\xa0\r\xb4\xe0\xac\xc5h\xc5\x95s<\xa0\xe5\xbc\xce\xe95\xb3\xc9\x8fa\x1d\xa2\xaeJ\x84-\x15lR\xd2\x9b\x0f\x03\xeap\xa1zzp\x95\xf2\xd3\x001\xb7\xe7\x06\x86e(\xe6\x0ej\xa4\x07\xdd5<\x9bQ\x12\xad\xcbM\xcb\x8c\x84\x17S\xea\x00\xe3\xbc\xce\x86\x07\x12\xb7\xc3\x10\xd7*\n\xa2]6\xaae\xc1=\x91$l\xfb\x973\x89]\xedc\xaf_\x94\x82\xc1\x1bd<\xc3W\xef \xe9\x01W,\xf8\x15\xf5^\xab\x94\x9f\xc6\x87\xb9\xf9\n\x8d\x81\xda\xfa\xe7;\x84\xd0\x13\xe5\xdb\xd7\xde@\x95f\x9c\xb0q2\x0e\xec\xb8`y\xf7\xb2\x10\xb4\x99\xcaB\'\xe3\x92T\xd3HE\xe8\xb6\xdd\n\xdb\xb6\x93P/@U\x98L\xa2v\x08-mM|7\x14\xd7\x91\xee&cq-\xa0ac\x7f\x17\x1d\x06x\n\x85\x924\x85\xdd\x07\xc4\xda\x1fG4\xc6\xafi\xbaK\xa1\x8d\xe43&\x1bO$\x9e\xec\xa5(k\xadvj\x07\xa9o\xc63\x8f\xab{\x06\x153-\x8atz\xdd\xf5Q\x08\xa7\x84=]i\x08\xde]\x9cF\x97\xaf\x01\x03\xfbn[\xb7\xf3\xa9z\xf0\xfd\xedit\x98\xdb\x1c~{P\xc5\xee\x93\xf8\x16\xc1WG\x8c\xcb\xf6Z\xe9\xb4\xab`\xad\xce\xa2\xd6\xae\xf9~\xd9U{\x9cTUR\xb9\xba\x17\xed\x8c\xd7\xe3zh\x0b\xff\xa2\\\xfd1K\xcf5)\xef\xeb\xca\xafw\x03\x12\\\x8f\xfa>+\xf5\xe3\x024\xfe\xd8\xc6\xdf\xf1\xef\xf5h\xf34:\xccmb\xa2n\x80\x18\xfa\x8e6y\xd9\xca\xa9\xe4\xf2\xf26\xc9\xab\xb3\xc3\x97s.\xb3\x19\xeb\x99s\x93u\xdczV\xac\x9d\xb0u\xa2K\x0e\xd3\x03\x9a\x86\xe2\x8a E\xe9t\x81\xa6\x95\xb9M\xc1\xa0\x9a]:\xf5l\x16\x89\x9b\xa1\xebn\x974\x91A\xd5\xf1\x83\xa5\xf0it\x18 \x93\x01\xa1\xcf2\xf7\xecP\xaci\xdd\x89\xf0\x9a\xad2\x94&\x1e\x8cku\x83x\xa8\xda\x86\x12-\xd2[~\xb1f\xeb\x8e8^\xe7\xccn\x99\xe0:.\x88\xe3\xa2$\xa8\xa9,\x03z\xbb\x8d8\x8c<\x80\x89i\xef\xf4\x8b\xb4\xba\x1ew\x9de\x07m\xf3 O\xe4it\x18 \x13L\xcb\x04s\x0f\x87is|,\xc5\x9c\xe2\x07\xea\x84\x93v]y\xa4\xbdr\xca`#\x81\x01\x7f\xc4l\x0e%\x92|\xc1t\xfcb\xbb\xc5/\xe4~\xb5\x96f\x19Wx:\xd9x\xf2B+\xddn\xc3AF\xefI\x8aY\xa2h\xce\xe9,\xe3>JMy\x16\x1c\xe6fL\x84A\x11\xe2~rSm\xc7\x9f4\xc9>\xed%=\xc6\x89\x98\x92\xf7\xabY\x95d\x83\xef\x9aK\xb9\xeb\x88\xd1\xb8\x0c\xfe~\xde\x9d\x1b\xd6rT\xdc\xa3\x96\x96QS\x99J.\xc3|\x8f\x9d)7Z&\xb0\xe6m9\x7f\x08i\xa8\xd7SH}\xb0\xe2?\r\x0esKp`\x04D\x19\xfa\xee1\x83?\xc9;\xc3\x97|\xf5\x90:EKW\x1aV\xd7\x9c\xb5\xa2\xa1l}\xc2/\xc6\x85\x16\xafdTjC\xebf}\x1c\xc9\x16BN4\xab\xe6R%\xae\xeb\xa6$\xc5\xcd&\xdf^\x8eI\xa0\xca\xdd\x9e\xb7\x99\xbdd\xf8\xc5\xaf\xa6\xfcW\xc3an\xd6DI0\xe4\xdd\x97\xab\x1d\xa9\x8d+\n\xe2\x0eB\x92m\x9b\xa8[Z\xdbf\x1dq\x1c\x96\xc4\x86\xdbjB\xbc\x9a\xa7fE\xe3\xb9{\xf0\x96\x0c\xa2\xae\xa6\xcdf\x11\x0c\xfc\x00e\xfb\xed\xa5\xa2x"\xf5\x16z\n\xa3\x18z\\\xf5\xa6/l\x89\xee\xbd\xf2\xf8\xd3\xe80\xb7\xc8\'0\x9a\x02?\xf0\xeeC7)?\x0f2v\xce\x96\x02\x13%\xaa3\x89\xa0\x86_\x8a6\x00"\xea^\xae\xb7\xeb\xc0 \xf6+\xf4T\x0ef\xa9V\xac\xa9*s\xe0S\xc4\xa6\xbd\xd4\xf1\x90\x88i\xa3\xaf\x82jW\x04{"\xf4\x9a\x88@\xb6J\xf9\xe0\xcb\xd7\xd3\xe80_\xa1O\x11$r?\xa0\xf2\xdb\x1eO\xb1e\xd2YA\xa9\x9f\xc5\xa6\xa2\x99>\xf3\xc8\xd5a/\x18#\xb2\xad\xb5`)\xc1S\xde\x1d\xa2\xf5\xe2*\xe8\xb5o\xed\xd2z\xad\xad\xcb\xbc_r\xb5\x87\xf4\x82\xbc\x83\xc9\x88\x80pJ\xe4.y\xac\r\xb5\xf8`\x0b\xf74<\x0c\x98\xc3q\n4\xfb(sg\xcd\xa44\xa1\x9d\xb2P\xd6\xfc\x19/\x8b\xe2\xac\x9c\x02\x0c-\xc5f\x13\xe7\xb2\xbeF\xfc\x12\x84\xc6J/\xcb}\x8d:\xe5\xb5\xad)\xd7JX\x82\x95\xf3s\x87\xf6\xb2\xb7\xdc!\x82\xe9i\x85uFsq%-|\xd0\xd0\x15\x0f\xb6pO\xc3\xc3\xdc\xac\x89!(\xf9\x93\xa1\xa6\xd7\xaf\xd9\xd1^\xf9G\x98\xde\xea\x85U\x0e5\xbf\x13\x96\xaeF\x1f\xd3i\xce\x86\x00"\xb9\xd8*\x06\x8d\xddF0\xa4\xb7TTA\xfb$X\x9e\x91v\xcc\xd7\xe7\x85\x1f>\xcc\xbf\xfdE\xf5\x0f\x1f\xe6[vt}`&\x9f+}\xf7+\xfd\xf0a>|\x98\x0f\x1f\xe6\xc3\x87\xf9\xf0a>|\x98\x0f\x1f\xe6\x8d\xcf\xf9\xe1\xc3|\xf80\x1f>\xcc\x87\x0f\xf3\xce\xdc\x95\x0f\x1f\xe6\xc3\x87\xf9\xf0a>|\x98\xb7=\xe7\x87\x0f\xf3\xe1\xc3|\xf80\x1f>\xcc;sW>|\x98\x0f\x1f\xe6\xc3\x87\xf9\xf0a\xde\xf6\x9c\x1f>\xcc\x87\x0f\xf3\xe1\xc3|\xf80\xef\xcc]\xf9\xf0a>|\x98\x0f\x1f\xe6\xc3\x87y\xdbs~\xf80\x1f>\xcc\x87\x0f\xf3\xe1\xc3\xbc3w\xe5\xc3\x87\xf9\xf0a>|\x98\x0f\x1f\xe6m\xcf\xf9\xe1\xc3|\xf80\x1f>\xcc\x87\x0f\xf3\xce\xdc\x95\x0f\x1f\xe6\xc3\x87y\x84\x84\xf0/S\xef\xffJB\xf8S\x13\xf2\xce|\x98\xd7\x0b\xfb&>\xcc\xeb\x85}\x13\x1f\xe6\xf5\xc2\xbe\x89\x0f\xf3\x98\xb0\xff\x846\xf2M|\x98\xd7[\xec\x9b\xf80\xff\x85\xe4\xf1\xe1\xc3\xbc\x84\x0f\xc3\xfe\xcf\x9d\xff5\x1f\x06\xe7\t\t\x17(\x82&\x11\x82\x141\x86\xa71Z\xa0)\x9c\xe6\x04\x9a\x03\x7fFq\x16\xa5e\x9a\x92\x10\x9e\x15X\xea\xb6`\x9e\x17E\\\xa6y\x8e\x94p\xea\x8b\xf1\xf0K\xf4\x01\xc7S\x12\xc1a2\x83c\x18JI\x8cL\xdc,F\x13\x08"s8AH\x12\xc9\x8b\x0cp&\x86\xa3D\t\xa3\x19F\xbaq2p\tH\xa0\x80j\\\x02?\x8d\xc6\x11\x0e\xdc\x14\x83\xd1\xbc\xfc\xdb?\xb9\xf8\x87\x0f\x03~\x9a\x04~\x18!1\x12\xc7a(J!\x1c\'P\xb8\x84\x83\xcb\x02\xb6\xe1\x81\x89\x19QDH\x06\xe7Q\x06D.K\xf0\x82\xc4"\x0c\x86\xdf\x16\xf5c,\xc9\xd1\xbf\xfd\xcd\xf90\xff6\xbd\xe0\x8e\x0f\xf3\x9b\xd0\x82\xebh\xcf\xbcP\xe8\x97\x00c\xe7\xc8c\xe7\xd5\xc1@\x02\xafo\xa3&\x1e\x92R.v\xfe:\xd77\xa4\x11c\x89\xb4\xf3\xae\xf5\x0e\x93\x91\xddz\x81\x08E\x7f\x891\xf4\xa2{\xf9\x10\xceI\x15\xe2\xb7\xcf\x1a\xb65\x18T\xb0p\x8e\xc9\x18O\x86\x10\xeb\xbbX\xb9}T\xbe\x9bn_\xeeH\xd5\x96\x8a\xe7\x1a\x95\xca\xc5\xd7\x7f{\x9b\x81"\x9c\xaf\xe3R\x9e\xf5\xd9\xa0\x02\x1c\xcc\x85M~\x0c\xbc\x00\xd3\xd5\x0c\t\x9cj\x0c\xc0?7\xfd\x1a\x8c\xe71iz\xed\t\xfc}"\x119"\xcd\x8fk!\xdb\xa2_\x15\xf2\xaai\xbac_\xc1|\x89\xebs\xbc\xd9e\xad)T\xc1R\x9d\xd8\xf2G\x05\xd5\x87\x00_\x90\xb1\xba\xbe\x00}u\x80s\x93\xe1\xec:\xa0\xcbN\xbc\xc5j\xe7W\x1bm<\xe6\xdc|\x11\xaa\xab\xb5\xee*\x1f\x1f\xfc"Z\x1c\x14\x13\xdb\xa4[\xbe\x08)\xdb\xd7\xd8\xed9\x84\xf9\xdaX\xb0>QJ\x99\xbd\xe4\xb2\x8c\x97r{)fG9\xf1n\xf7\xc7lxG\x02g\xd0p\xc9P\x17B\xder\xbe\xd7\x07\x12\xccc\xe6\x10\xc30\xde\x88\xd6nt\xf4\xbd\xecJ\xa4\x06I\xf9\x99\xc4\xf3\x82\xc2\x0f\xb7\xd5o\xc1\xf6\x82\x9c\x87\xfd^\xf4\xf7\xd8\x81\xba +\x8d4\x05,:\xf7\xe1yYm\xeb}\x1e\xafM\x8fge\xc2\noM\xc2\x7fJ\xf7\xb9\xad\xa5\xf9K\xba\xcf\xfbg\xe9\xef\xa5\xfb\xd0\xbf\xe6B\xe8i2\x1f&a\xbc\x1e5\xfc\xacnrN\x19RiM\xd3\xd3\x81\xab\xcf\xb4\xb8j\xc4\x03O\xf9&.\x1a\x8b\xd0\xcd\x06\xe2\xc4\x94\xa1]\xc7\r\x93\x17[2\x00\x1e\xa3\xb62O U\xa9\x9d7\xf6\xf1z\xf5u_\xe7\x1e[\xbe\xf7,\xba\x0f\xa8\xe9\x0c~3\tsO\xbe\xe1\x94\x8dhk\xb9\xee\x9ck\xf3\x02\xba\x1e\xa2\x81\x98\xaeH\x99H,\x96\x8eX\xa4p\x082./x\x99(f\xaee\x85\xcaJ\x0c\xd8\x03\x1c\xafs\xd4\xd0\xdc\x95\xb7\xf6X\xca\xd1\x02c;\x1d\xab\x80\xdc\x96\x90\x8e\xd6\xdfD\xf7\x012Y\x16$\xd0\xbb-c\xbaujX\xba\xbfJ\xddXu\x9d&HR\xa6c\xa9\xc9-\x1f\r\xa3W\xaa[\xc5\xdf\xefwgD\xb26\xb8\xd3\xaa\x8a|\x15t\xd1\x0e\x86\x9a\xd9&Z\x89\x83\x041\xf7<\xef\xa3Z\xc9\x90\xca\x83\xcb\xd4\x9eE\xf7\xb9i\xbc\xd5O\x86\xbd\x0755\xba\x19\xaf\xadi_Ks\x91\xdfBp\x99\xe5\x89\x94\xac\xae\xeb\x8b`\x9b\xc1mi#\xa3\xf9\xb0\xd3\xb9c\xc8\xab,]^\xc2!N\x83\xd0\xaf\xd7\xc7,\xc6\x1f\x8c\xfbg\xd1}n2\x11\x16\xa4\x7f\xea~\xe9\xa6\xd7/\xf5\x90\rxE:\xa9\xadS\xdb\x0c\xce,\x80\x07\xb2Y\xb4\xbd\xc4\xd98\x85iX\xeec:\xaf\xc1\xbf\xb6$`\xf6\x88\x92\x8e\x10\x10\xed\xda\xadF6i\xf5\x88h\xb1,M\x9dN\xec\x8f\'O\xda\xe4\xc3\x1e~p\x1f\xee\xb3\xe8>_#\x0b\xf898}\x9f\xde\xf6"\xd2\x90\xabz\x1dm\xc4\xd5\x9c@\xf1A\xb8b\xe9\x89\x86\xdd\x83\x08/\x15\x98:\xe6\x86=\x96}\x98\xf4A\x88-\x99\x9d\x84\xfa\xf6I\xd7B\xa9[ Ye\xe0\xc5a\xb3r\xeaj\xe3\xe0\x0b5\x04\xddZ\xb8\xe7\xcdo\xa2\xfb\xdcd\x82\xfc\xcc\xb24{G\x85\xd0\x19\x1f#\xdb\xc39H\xa0}\xb5\xcf\x9a\xc2\xc3%=\xd5\xf9\x1aZ\xa0\x17\n9\xed\xa7\xabq\xc2\xd6+:;\x85\x1e}\xad\xa85-\x91\x91k\\\x8a\xf3\xa5\x08\x8e53G\x84;{\xfa\xb4\x8d\t\xf4,\xf5\xd4\x9a\xeb\x1f\\\x81\xff,\xba\xcf\x975i0\'\x80?\xeeV\xa8\xf6+\xeb<\xf3\x97\x16\xb6#QQ/\xb1\xa9_:\xa5\xf1\x16\xc7\xcd2\x8d\x04\x9biIv\x0e\x19\xcc`\xe8\xd5\x99\xd9\x95\xb52\xefU+\x81\x0egz\xf2\xf7\xb0\xefh\xa9\xcbI:\x9a\xb4\xab\x06\x9b\xd3\x98=\r\xd6\x83{\x7f\x9fE\xf7\xf91gS`\x8a\xb9\xdf\x8ay\n\x17\xd7r\xcd-Hu\xb4\x84HY\xf5^u\x81\x89\r\x1f\x18\t\x1aM\x97MK\x98Mcn\xf7\x03]w\x1c\xa5\xd8\x11\x93\x12c\xb2\xdb\xe8\x86z\xf2\x95qE!\x87\xa1\xe1\xa4\xf5\x81T.r\xcaONr\xb4\x1e\xacW\xcf\xa2\xfb\xfc\xf99\xe1_V\xb9\x9e)x\xa1\x1bG\x9a=\xade\xa0\x17\x9f\xb7\xe8N\x8b\xf7q\x11\x9f\t\x9f\x00\xee\xbb\x8d\x0f~^\x96{C\xb0]\x8f\xb3v\n\x14\xa9\x07\xa4TP\xcc\xcdO\x8d\x88\x9c!c\x16a^\xb0\x0e\xe08|wQ\xaf\xbfZ\xe5\xfa\xf7\xa6\xfb\xa0\xc4\xef\x08N"\xa0\xb9\xbe\xcb\xe3\x9bk\x14\xf6\xb4\x91\\B\x85X*\x19\xadG\x81\x81^a;Y\x15\xf9\xaa3\xb7\xfeA\xaf\xf6\xe3~`\x8d`}\r[\x92YrN"\x16\x87juIO\xcb,\x8av\xb9\xd0y\xad\t\xb1\xa5\x00\x9d\xe4X\x90\xda\xeb\xa3\xe5\xeaIt\x9f\x9b\xaf`(A\x83ru\x97\xc7\xd15F\xa5\xb9\xafk\x91\xafm\xf6\xe7\xf2\xe4.\x86\x84/\x85\xe6\xb0!\xe8\xed\xe1Z\x8f\xecZ\x0b\xd7cN\x9e\x0f4\x9e\xf7\xd7\x95\xd2\xa5\x8bE^\xec\xc9\xde\xe4WW\xc7\xa8I]V(q\xb2\xd7M\x0f\xf9\'"4\x1e\x94\xf9,\xba\xcf\xad*\x83\xaa\x06\xf2\xe1=R\xd4\x10\x1b)q\x9a\xb5\x19]\ne\\l\xd9\x11;v\xd6\x1a\x83N\xb5\x89\xcc\xaa\xad\xf7\x1d\x16O\x19\xb2g\xe1\xc1\xd8\xae">\xef\x08c\x17\'5\xc5\xba\xfc\x16B\xc1\xd8\xee7\x97\xcbR1\x1bE\x90\x05U\xae\xf2\xfcW\x91\xff\xf7\xa6\xfb\xdcn\x91$q`\x12\xf6\x9e!\x022\xbc>\x16\x82\x19\xc3\xa5\xb3W\x8b\xe0,\x85\xf8\x9c\xda0E]\xaa\xeb\xf1\x1cHs\x7f9\x8dJpf\x16\xb9\x908\xa1\xd3$[h;\xab\x9b\xedh*W\x879E\xc7\xc5\xbe\xabal\x18k\x1fY\\h\xfe\xf2Mt\x9f[L\x80\xc1\x8df\x81\x19\xfe,3M=\xb3:\xa1\x1b?Q\xf61\xdb\xe1^C&\x1b\xa58\xf5bxQ\x8f\xc7%\x97\x8d\x9bf\xdb\xc4\n\x84\xef\xce+\xa6\t\xe1\xc6\x8f\xce{{g\x97A\xa22\xd8\x18\xf6\xd0\x89b\xd2`dWf2\xa1\x9eU\xd9\x0fBK\x9fE\xf7\xf9\xeamH\nt\x0c\xd8]\xd1\xb7\xc9$\xaf\x921\xc2\x176c\xea\x9a\xa8\xed\xd6\x82\xac\x9c\x8e\xa34R\xd3Ub\xd1.\xd93\xfc0\xb4\xc8:QN~\xacEU;\xf3\xc7Cl\xc3\x1di\t\xf3.(D^\x18|\xa1dq#\x1a\x84\xcd\xbc_?\xb8\x05\xffYx\x9f\x9b\xd3\xd27"\x02\x8d\xdd\xad\x87o\x8b\x13\x12\xf3\xce.\xb9x\xa0\xaf1D\xa8wyn\xb9\xdfA\xbb\xee\xb0>n\x08\x98\n{)\xa5\x18GvO2O\xc4\xec\xaa\x87\x97\xd7\xed\x14+\xd6\xd6:\xd1\x95\xb9\xf58\xa8]\xaf\x02\xcd^sB\xcc\x9b\n\x89=\xd8\x90?\x0b\xefs\x93\t\x0c\x7f+\xfbw\x19n5(:H\xc5\xc5Y:\xa5a\x88!\x9e\xe9&yzB\xe82*\xa9P]\x9d:13jj\xddHW\xbc\xb9\xf2\x97\xe3\xe8\x89\xecnO\x9f\xb3\xc6\xf1\x07I\xad\xdbf\x00\x99\x18\x15\x8a\xde\x19V\xfaR9O\xfc\xaf\x12\xf9\xdf\x9b\xef\xf3U\'X\xf46\xbd\xfd\xf9\x12\x03\xe6\xf2\xff\xb2\xf7\xa6]\xaa\x1bY\x1a\xee\x7f9_\xb9\xab\xd0<\xdco\x9a\x85\x84$\x04\x9a{\xdd\xd5K\xb3\xd0\x84\x00!\t\xf5\x9f\xbfA\x9e\xean\x97q\xba\x0b78\xd3\xd5,\x97\xcb>\xe9L\x887v\xec)P\xeeg\x83\xda&\x05\xc7\x862\x9c\xb9\xc5U\xddQ\xca\xc8\x1b\xe5\x84\xc0\xe6%\xd1p\xca7UM\xd4\x8c\xd5fi\xee.<\xebB3)\xceY\xb8\xea\xc8\x95x\xa9\xaceb%\\\x11i\x05|\x89\xbc\xb0r\x95d\xfb`6|\x16\xe0\xe7\xe6\xf94\n\x03\xc7\xc7\xee\x86\xe03\xb6S*\xca\xba\xe3\xe8\xc3F\xd9qV\xad \x85A\'\xce\xca\xef\x13\xb7\xa8\x82%J\x0b\x1b\x91\xcfd\xa9\xdd\x99+\x85\xd1\xaef\xdc\xea\xa7\xbdz\x91\xc3\xb5@\xd7\xeb\xc3y\xd5F\xda^\xd8\xae\xe6\x1c\x04\xa5\x96*/\x0fz\xfe\xb3\x00?\x1f2\x81\xff \x08z/s\xbd\x8c\x1c\x07AW\xb9hxv\x9b+\x01Bj-\xab\x18\xe2r\xa3\xea\xda\xfa\xd0\x04\xcc\xb6Q\xf0\x8d\xab\xef\x04r\xd4W\x01\x125\x04f\x9f\x96\xc7\t\xe4\xcb\xc5e//V\xc8\x85\xd9\x98\xd8\xdc_\xa8^]"\xd6\xf7r\x89g\x01~>.4\x10\x04\xa2I\xf4\xee\xf6\xcb\xd1\xa4\x91sw\xce\xce\xbe\x90F}4\xe7\xb4A\'\x89\xbdl\xb3\xdd\x90\x97\xfe\xbc\x81\xd8l\xf6\xf3\x90o\xa32\x8c\xceX\n\xaf\xf1\r\xdc*\xc3\xe4\xef\x99\xa18J\xa8\xb1*1\xf9t\x91,\x91\x99\xb8x\x96\x9a\xf9A0\xc4\xb3\x00??;}\x18Ep\xfc\xae\xb6\xd9.H\x9d-\xae\xc8Tin\x89\x9bEd\x18\x89\xc1OWe\xb3K\x0e\xb0t\\\x14\'\x82\xde9y:\xaf\xa55\x1eyZ|\xee\x15jmGf\xe2\x87\x8aY\xb4<}\x0e\x9d.\xc5\xcd%e%{\xec\x98\x98\xd8\x83I\xffY\x80\x9f\x0f\x99\x10EQw\x1e\xe1\x0b\x9e}XW\x88\xbaJ\x8fS\xc9\x9f\xbbC \xf3\x9a\xc42\x9a\x16\x193\x19\x8c\xc9(\xbb\xe8\x02\xc4\xfe\xdd \xdf\xc6\xc3\xcb\xec\x9eg\xec\xd1\x08\xc2\n\xca\x0e#\xdaA\xed\xd2:M\x1beah\xb1\xe9\x1e\xc0.R>\x86\x90(|\x7f\x07\xe7G\xc2neN\x85y<\xccS/oN"\xdbk~\xd0E5\xef \x96\xb72\x05\x8f\x98Z[8g\xbb\x83d\x91\xd9\x1a\xd9\x1dZ5\xb2\xd5U;\xfb(\x83/W\xfa\xba\xea\x06\xcc\xa3\x85\t\xea\x8eB\xea;\xfb\x07S\xfe\xb3\xf8>\x1f\xe5\xf8\xed\x07\xe1\xdf(\xc7\x97e\xed\xb2\xd3^\x92\xb9\xf5\x11=\xb7\xf4:\x94\xce\x9aN\xec7\x9f\x00]\rD\xd2\xc8\x9dO,\x16\xeb\xc4;\xd9e\xa3,\xc5\xf5!\xdf\xc1\xc4ZM\xf0\x15S\xaa\xd4A5\x9b\xaa\x88\xa3\xc3e\x0b1\xeei\x83jSH7\x94Um\xd7\x9bna\xef\xd4r\x85\xca\xc4\xd5\x99\xa1,\xbdh\x94F\xac\xe4\x94H\x1a=x\xb0\x82{\x16\xdf\xe7C\xe6\xad\x84\x03i\xf9\xd72\xc7\xb2c\xc4\xec\xdc\x1f1KZ\xbb;Y\xcfj\x839\xb5\xf5\xa6\xedyX\xb5\xae\xe1^\x19\xf7\xc45-\xaf\xce)\xd9\x1a"\x97\xd9\x1d7\x18k\xa5Q\xf3r\xc1!\xa5\xe9\x8a\x9c\xb5\xe0\xa9\xb3e\xf0\xc6\xda\xb9\x80.\xfd{AK\x9f\xc5\xf7\xf9H\x870\x86b\xf8\xfda\xe9\xcc\xca\xadS\x83\xe1\xc30\xa6\xf6\x9a\x95\xea3?\x82J*\xd33s\x8e\xe5\x90\xdd\xd2\xad\xc4\xd2T\xc9qx\xa6\xcf\xdb\x90p\x08K8o\xc75*\xc4G\x92)\xd0p!\x06\xb4\xcf\xad!zY\xfa\xfa\x12\xe6\xd4\x9f\x87\xe5\x7f\xe4\xfb|t\xee\xbf\xe4\xfb\xfc\xdb\x7f\xfc\x88\x0fI\xfa\xf1\\\xc5\xcf\'m~p\xf9A\xe5\xa4m\x1dIN\xeb{\xdb\x1a\xfc\xc8\xbe\xcd\x0e?\xf7>\x0f\xcf\xff>\x86m\x9f\xde\x9e\xba\x83I\x14\xf9x<\xf0\xf6\xd5\xcb\xf9\xe7\xd7@\xd0\xfc\xfb\x93Dw\x9c\xa0\xe9\xb3\'S}W/=T\xafc\xf3\xfeI\xd2Y\xc8\xc8\x8e}l>\xee\xb3\xa0m@&E\xd0$r7\x1c\xd7\x91Gx<\x04\xbe\xda\x11\xa4\x8b\xafQ\xe9\xba\x82\xaba\xb3\xb3{=W=\xef\x144\xc2\xf6\x04\xa7\x04\x91 \r\xb4\xcb\xc9\xf9t\x1eh\xb1\xe9\x0e\x05H\xf6\xdb\x0bz\xde_\xeb\x8b\x84\xd4>\xe1\x1f\x151V\xf9=\xbf|p\x10\xe8\xb3\xa0m7S\xd2(\x82\xd3\x10t7\x1c\xd7\x1d\xd1\xbe\xd6\x83\x82J\xe1\xd4\xd7,\xb95\xb93\x17H\xd7\x88Op\x8b)\xa4\xb5m\xceC*\x98\x86\xe6n\x97\xb9N,8\xf3\xb4\x92\xc5k\x9f\xc7}\xd1\xa1{g\xdd\xda\xc9\xa1\\ai\xcf\xef\xecF\x16\x90\xf4\xf8\xd9\x84\xcc\xbf6\xb4\xed\xc3!0\xb0\x8d(}OL:\x07\xccd\xc5*3\xf7\x15\x89\x9aQs\x91OJf\xf5\xbdS5\xb8 \xd2W\x08W\xa3\xaa\x94#CF\xf2Jj\xcf\xeb\xe5i\x12\xc3\x0b\xd1\xf6\xa0\xc9\xea\xf5\xf1\x12\xda\xf3\x01\xaa\xd8\x92U\x824\xd7\x14mF\x1e%~<\t\xda\xf6qXp\x10-\xeef\xc6^\xd2\xaavM\xc3]m\xf0\xab\xce/e\x04\xddEU\xaa\xef\x8ax\\RQ\x17\xc0Xe;hh\n%\xecq\xd2\xf9\xb8\xbcF\x81\x94e\xd3\xd1j\\g7\xef\xfc\xc3\xc9p\xca)\xb7\xcfq\xb9\x14TGr\xa4\xe1\xc1\xd1\xdf\xcfB\xb6}\x94\xbc\xa0\xf4F(\xf8n\x98*\xb13\xe7\xeb\xf9\xd0-\xd59\xcb\xb5\xee2/<#?-.\xfb\xd6\xd2\xd0\xe1Q\xc8\xd7\x93\x90m7\x99\xe8m\xea?}\x0f4\x12\x19\xc9\x80\x1a\xbdV\x08h\xd3\xaep\x14V\xbcj\x87\x85\xd9X\xb5$\x9c\xaf\xae\xc18\xf8g&\t\xe5\x8eL\xe1 \x0b\xe6U\x88oL3\xe9C\xdaQ\x9b#\x89\xa6k3\x0cc\xa7\xa6H\xeb\xb0U\xb9\xa8\xcd\x1eT\xf9,b\xdb\xaf\xfb\x97_\xcad\r\xc3\x1e\x08\x08\x95[=9w{A\x1e\xd7K\x17;,r\x01:`\xeb\xca8\xa15\xb1K\xcf\xd7R\x98\xf0\xb8\xe4\x8cD\x9f\xbb0\xf0;u\x95.\x9c@\xea\xb2\x89\xd0\xd1x\xa9\xe2\x9c\x81\n\xcd\x94\xb0-B<8\xe2\xfcY\xc4\xb6\x9f2i\xd0\x16\xdc\xa37\xf7J\xd9N\xda\xde\xc1\xd2hf\xc5Q\xdbH3\x95\x0e(\x8e\xe5M\xbf\x89\x0b\xc4k[%hk\x19\x82}\x93YU\xd5\xea\xc2\\\xda:\x8c\x16\xd3ZN)m\xbc\x9e2\xf8\x82\x93\xe9\x92\x98\x9ce\x91\x1c\xd1\xc3\x89f\xbe\x86\xd8\xf6\xf7\xba\x03\xa3A\xd6\xfb\xb5\xcc\xa8\x05\x19\xf6\x04\xd9+e\xaf\x9d;\xf5t\x9e5\x87r\xbd>\x8aO\xcb\xe9\xb8=O\xcd)\xa3\xa4\xaby\xdc\xa7%\xdd\x90\xbb\x13r^abw\xd2\xd3\n\xde(J\x03\x0e{dr\x9d\xa4\x13M\xcc\x84\xce\x9a\xc7{\xfc\xb3\x08\xf4\xd7&\xb6\xdd\xca\x1aP6\xc3 R\xdc\xedb@c\xe7\\\xb9vQ\xb4\xf7\xcc\xf3TF\xfb\xede\xc2B{P.\xeb+\xe4Y\xa7\x99\x0fG\x98e|yn\x0c\xc5\xd8;\xf9\x90\xa8\xf3\xfa0S\x914\x10\t\xebh\xfaI\xd5\xa6C\xd0{\x03\xdc\x93\x06\xe5\x18\xde\x17!\xdb\x80L\x02\x9c\x16p\xd2\xf0\xbbZ6\xd3y\x10\xe2\xb3d\xcbW_\xf0W\xf6\x01=\xedZ7\xb1\xf2\xdd\x8a\xd4\x94\xf5\xa5g\xdbac\xa4c\xd4T\xed\xb5(\x16\x99\x12\'\x0f6\xc1\xcf\xa2\xb6\xfdL\x878\x01z\xbe\xbb\xe2f\x13\tZ\xac\xa9;kZ\x89\x91\x7f$\t\x08a\x15\xcdv5H\x8d\x8e\x8e\xc4k\xfe\xa1,\xa7\x18\xd4n\x11\xdf\xd7\xacT\xcd^\xae\xebyH]}|\x05\x19\xe8!\x11\x0c{M\xef\xc9u\xecw\xe1\x12\xc7\xab\xcax\xb0}{\x16\xb5\xedgEN"\xd0\xbdk\xaaTX_\xdd\xa3\xaf\x93\x9d\xe8\x0f\xc8\xd1\xe6\x88\xebX\x99\xfaq\xc1\x9f\x16\x85J^L)\xb4\xd4\xd5(@\xc1v\xb5 \x1b\xbd\xa4\x13\x92\x8cIq\xc1z{3a\x14\xe1p\xc6\xe5\xb3]^\xbb\xd4\xdfB\xa49\xd3\xf5\x83%\xdc\xb3\xa0m\x1f\xb7p\xf4G\x10\xba\xabm\xecF\x8e\xbbp\xe1\xe4Q\xbc\xac\x17u\x9d)\xb7:T\xcd%\'S\x1a\xbe9\xb7N\x8c`\xd4\x92\xa6\xcf\xa5\x91\x05\xb1{\xe5#5\xa2\xd7\x86]hN\xbf\\\xbb+P\xd4\x88XK\xb3\xe6\xb2\xf6!}[\xf6\x1e\xf2\xd95\xdc_\x1b\xda\xf6\xe1\x12(\xc8\x9b\xd4=\xb9)\xd46(;\xae\x12\xf6`6v}-\\\x0b\xc4iG\xc5\x17\x93\xc0\xb2a\xd4/\x14xW\xc4X\x11jk\x03\x9fvh7q\xfe\x9eYUD\xac\xed\xaf\x86\x85Nl?\xe5<\xa6`&\xd6\xa4\x8d\xa4\xd0z\x98\x7f\xc6\xbfz1\xb4\xed&\x13\xa6H\xd0\xeb\xdf\x7f$e\x148\xeb{\x91d\xec\xe5\x8boe\x17\x9e\x97\xf8\xb1\xa1\xaf\x06\x15\xcd\x91\x16\xc7\xc4\x86a\xf4\n\xdf\xa6+\xaftP\x98=\xaff\xd4\xdac\r\x97\xad\xd8\xf3,\x90$\xe2\xed\xe3\xfd<\xec \xadTv\xfd\xc2f\xc2\xdd\x83%\xdc\xb3\xa0m\x1fm\r\x01\xb22\xf8\xa1\xbb\xcbL\xd5\x0e\x84b\xb1?\xc6\xc9VF\xbc!\x11q`[b\xe4{\x15_\x81\x9af8\xb8\xc1n\xab\xa5WW\x93\xaf\x9e\xeb\xb5\xd6\xb2\x92\xd8HB\x8bk\xc6)\x97y\xb3c\x07a\x97\xe0\xc7\xe1\x1a\x9d\xb6T\x92\xc1\x17\xe7\xc1\xab\xe9gA\xdb~\x16\xe4\x04\x06\xbe\xff\xaeR\xcdH\xb3\x1f\x15\xba\x8c\xc8\xf2\xb8\xf6s\x0b\xae\xba6\x8e\xa0\x81\x19\x8f\xf1\xea\x9c\xcc+\x8df|U)\xd6\x8b\xc2m\x15\xdd[\x18\xabt\xcf9\xda\x1e\x9b\x08\tY\x8c\x9d`\xef\xa0\xad\x08e*Q5\xe5`\xcb\xe3\xdc|\x16\xe1\xfe\xda\xd0\xb6\xdba\x81P\n\x06\xff\xfb\xf5&\xc6(}\xc6\xf6\xfe\x10wl^\xd5\x8dx\x8dr=X{\x97\xcdR\xd2xa\x9f\xac\x90d@\xe5\xf3R\xac\xbc\xdd\x8c\xa6\t\xc1fr\xeb;\xe7\x14[nG\xaek\xa4:2\xe4L4\x0e\'*\xac\xcek\xd0m\xf2\xae6\xaff\r}\xa1\xd7\xbcD\t0\xcdo+\xf94k\x7f_\xfe\xcf\x0fp\xa6@\x9e\x89\'p\xbe\x80\x08\x06\x9c9\x1f\x06bF\xbd\xac\xb0\x97\xe6\x99W(\x01y\xe67\x95\xa0\x86\xfc\xd2<\xf3t6\xcf\x0f\xdd\xd2.\xfal\xe2\xda\x1cO\xba\x05\x92%_\xc1\x06\x9f# J\xe3\xaf4\xcb\x0b\x94\xcc>\xba\x065\xe6\xad\x0c\x00\xaa@nY]\x81I \x90\x7f\xf0\x17\x9b\xe5\xf9\xb5\x0cX\xf4\xe5\xa3n\xe1\x99\t\x9c/ B\xc0@\xb4\x06\xff\x8c_j\x96\x97(\x01!\xf97\x95`\xbf\x9d-\xbf/\xcb\xe8\x07p\xf9\x0b\xf0\x10\xc4\xb0\x98\xf9C@y\xa3b\x00\xcf\xe1\xcd\xeb+\xcd\xf2|%\x9f\xd4\x97W\xe0\xfb\xf3\x8b\xbd\xe5\xe9\x9c!\x10\xc4V8H\xfb\xb0\xc6\x9b\xb7\xe0\x05\xfef0}\xb6A\xc2\x04\xe2~S\xcc7%&}\xa5Y\x9e\xce*\xfa\x19\xc4\xac\xdf\x14\x03\xbd\xd2[^\xa2\xe4\xab\xcc\xf2t\xde\x11\xa8\xfa\xfd\x0b\x08Z\xa0\x7f\xb9\xf5,@\xc4\xad\xfa\x9fc\x10\xd4\xf2\x97Vb\xcfW\xf2\x89\xdf\xcf\x86\x05\xd4\xbc\xd8,\xcff&}\xe4\x16\xbd\xb4q\xdd\xb2Aud\xde\x92%\xaaY9\x0e\xcc\xf5Z\xb3<]\t\xc8\x86\x18(\x90q\xfd\x96\xf2A\xf1\xa0}4\xc7\x02h\x96W\xd0\x8b/b\x9e\xce]\xbau\xfb\x17\x83\xb71\rx\n\xa8[\x80Ib (\x07\xae\xef\xc3/-\x90\x9f\xaf\x04\xf4+ \x88\xfdV\xf12\x19\xbfy\xa5\xf4}\x19S?\x8c\x1d\x06<\xa3\x02}\xb1\x00\xbcE\x03\xae_\x01!\xb7\xeb\x0b\xfd\x93\xde\xf8\x9bR\xa8\xbe\xd2,O\xe7?\xdd\xae-@\xdf\xf2\xeb\x0b\x18\x1b7\xf8\xd5k\xbb\xfc\xe7+)W\xf0\xbadn\x07\x0b\xd2-\x06\x98\xe3\x06s\x12p\x10\xa2\xe1W\x9b\xe5\xd9\x0c)`\x16\xd0\xb7X\x0c\x02\xd2<\xa8alP\x17i(\xa8i`\x90(_k\x96\xe7+\x01\xed\xe3\xba4A\x8a\x07*\xf8\xdbe\x7f\x85\x00e\xb8a\xf9\xd7\x17\xa7\xfc\xa7s\xa8>\x0bbW\xfd\xd3\x0b\xbeoJ\xd4\xbaeI`\x96\xfb,y\xbb\x88y\xb5\xb7<\x9be\xf5C\xdfc\xf3\xed\x13\np\xd6P\x10\x95\x81\x18\x90Wx\x7fz\xb5Y\x9e\xaf\xc4\xb8\x85\xe3\xd9DA)\ri\x96=\x83\nl\xbc}\\\x01\xafn\x9fC\x82\xc2Qym\x88z\xba\x12\x10l\x91\xbbK"\x9e\x01\xe1\xea\xe5\x1f\r?\x9d\xdcu3\xcb\xad\xb6\x01\xde\x01N\xeb\x0c\n\xcd\xf2\xa3\xde\x9a\xb4\xf2\xc5fy\xba\x92/4\xcb\xd3\xe9_7\xb3L\xbau\xfb\x94\x1e\x98\xe2f\x96\xdb\x07]\xa0\x88\xd4y\xe7\xb5A\xec\xf9Jv\x18(K\xb4[!=\xfdl\x12\xed+H\xea\xe0\xcf\x9f)\xf9\xbe\xa4\xb3\x9bY\xc6[\xf7\x0bj\xf7\x8f\x07)\xf49Gn\x1f\xa8\xea\xf3\x8b\xbd\xe5\x15J~#\x1c\xdfn#^\x9e[\x9eN!\xfb0\xcb\xc7C;\x96\x06\x12e<\x03o\x01\x1d\xc9\xad\x03\xde\xbe\xd4,/Q\xf2Ufy:\xc9\xec\xcb\xcc\xf2\x12%_e\x96\xa7\xd3\xd0\xbe\xcc,/Q\xf2Ufy:Q\xed\xcbr\xcbK\x94|\x95Y\x9eNe\xfb2oy\x89\x92\xaf2\xcb\xd3\xc9n_\xe6-/Q\xf2Ufy:\x1d\xee\xcb\xbc\xe5%J\xbe\xac@~6a\xee\xeb\n\xe4W(\xf9\xb2\x94\xfflJ\xdd\x97\x99\xe5%J\xbe,\x88=\x9bt\xf7e\xb9\xe5%J\xbe\xcc,\xcf\xa6\xe5}\x9dY^\xa1\xe4\xcb\xcc\xf2l\xe2\xde\xd7\xa5\xfcW(\xf9\xbav\xf2\xc9\xd4\xbe\xaf\xcb-\xafP\xf2Ufy:\xf9\xef\xcb\x82\xd8K\x94|\xd9\x9d\xd8\xb3\xe9\x81_w\'\xf6\n%_\x16\xc4\x9eM \xfc\xba \xf6\n%_\xe6-\xcf\xa6\x18~Y\x10{\x89\x92\x07\xcd\xf2\x07\xd8d\x9a\xc5\x8c\x8f\xb0\xc9\xaa\x7f\\\xf4\xf7%6\xfe\x19\xc2\xbe\x84\xd8\xf8g\x08\xfb\x12b\xe3\x9f!\xecK\x88\x8d\x8f\n\xfb#\xfc\xbf/!6\xfe\x19\x16\xfb\x12b\xe3\x9f\x12<\xfe\x14b\xe3\xcf\xff\xf0\x7f\x8a\x90\x08\xff\xf7\xa6\xff>!\x91D\t\x81\xa1n3\x9cQ\xf66}\x98$h\x9c\x80D\x8cDX\x0c\xe7X\x8aD\x10\x84\x81n\xe4/\x82\xc4)\x96b!\x04E\xa1\x1b\xed\x8d\x15nd\xba\x1f\xbf\xc7\xfe\x82\x04\x9e\x82i\x9eDa\x0c\x11X\x14\xe3n\xec5T\x84Y\x91\xa3X\x9a\xe7p\x98\xe30J\xe0q\x92B\x05\x8e\x82H\xf0\x92\x10\x85#\x10\x863\xa4\x08\xa3/%$\xfe\xe7,\xb7\x1f\xbf5\x80\x8c\xfc\x1b\x8a\xd0$\x8dC\xd8\xef\x01\x12\xbf?^\xf27\x00\x89\x14)\xd2<\x82s(G\x93\x9cH\n4\xc9\x90\x14\x0c^\xe5\x86\x0eCxA \x81\xd1 \x02c\xe1\x1b.\x89\xbd\xb1\xc1N\x7f\x0e \xf1\x9f&\xa2<\x0b\x90\xf8s\xb0\xdf\xef\x12\x12\xbf\xbf\xa3\x7f)!\x11\x81?\x072\xc5Fl:\x17a\x8eK\xcf^\r\x0b\x87\xe0\t\xd9t5Bj\xf1\xd8\x1f\x15m\x19AD~\xd5\xf1\xba\xd1`\x8b\x9e\xb9\x0e\t&\xf8l\x15\xbc\x13T\xcbbky\x8b\xe9\xa8\x18\xfc\xb9kr\xe7Tj\x91;\x10\xeaW\x11\x12\xff1+\xfcRe\xc9\x9f\xe0\x8d\x85E\xdc\xf1\xc2&\x86.\xd3\xac-MZ\xed\x83/\x86\xee\x896\x92\x8d\xd1\x0b\xc7I\xbb\x90n;\xee\xecT\xea\xa2=t\xc2v\xcaVY\xc8\xc7%\xdc\xb3\xbePtR92"\xc1\xc9kG\x9fU\xfb3(\xcb\xab\x01\x89\xe4\xdf0\x1a\x06e\xc0\xdd\xb8\xed\xdd\xfe\xa0\xd4V\x1c\x14\xd8D\xfb\xa5\x81Trh:4sEQ\x84\xf2\x8d\x92\xd1\x85\xe9\x90$l\xa24\xee)\x82\x84\xd4\xa4\xe0\x8b"@6Ye+\xb4%P[\xca\xa7\xb5M\xefJ1\x12h?\x87tf\xcb>\xc8I{\x16 \x11h\x84(\x82\x84\xf1{\xb4\x9f\xb2\xe7\xe4z\xa6\x11\x85\xa7\xd6\xf3H\xee\x9aij\xdbB\x93\x8f\xd2\xa2%7\x85\xbe\xde\xe5\xa7\xfd\t\xc23t\xb5\x8a\xe2<\x98\t\xb1Z.[S\xcd(\'\n\x0f0\xdcg\xe2z\xc5\xe8\xe8j\xce\xc9\x8d\xe0\x0be\x9c|\xaf\t\xcdO\x03$\x02\x7f\x00\x85*\xa8\xd9\xb0\xbb\x99\xbeu\x93]\xa2b\xb9\x83\xb6\xe6E\xbf\x94\xc4Qf\x8bb\xc9_IGoiV\xb1\xb7$\x0f\xedJ\\\xd1r\x1b\xe9)\xe1d\xef[\x1a\xa9O\x90"\xc5\xfai#v\xc7\x03\xcee\xb9\x7f)\xc3\xc9\xe4N\xfc\n=\x8c\xe3\xa3|\x9dg\x01\x12\x81L\xb0\x1f(I\xdf\xcf\xf4\xa5\x9b6\xaaR\xa1\xc6`\xa3=m\x9c\xd9\xe1Cm\xc9\xac\x02l(=\xf9\x88\x9fqx\xb1\x15,\xab`\x17\x91\x0f\xf1\xd73\x971\x08Rw\x17+Z\xd5T\xbf3\xc4\xfd\x90\x17<\xady\xd0\xaa\xcc\x96\xd9\xbe8\xed\xdb\x07\'\xed?\r\x91H\xfe\r\xf4;\x04\x05\xfe\xef\x8e%\x92\x15\xa1\xc2\x18\xa7t\x19saVe\x8d~Xr\xed\xfa\xbc\x9a\xaf\x94\xb8\xcf\xdau\x1e\x04\x13\xad\xd1\x81\xb2\xf0\x89x_O\x17\x84\xef\x8d\xe8\xc8\xc49\xe3aH\xa0t\xfc^\x1e\x1c\x9c;C[\xd4\xb3s\xa5\x95\xc3\xcd\xa3T\xbdg!\x12\x815)P\x87\xc1\xf8\xfdpah\xbf\x10\x13\xa4YJ\x91B6\x81\xdcQ\n\xc9\xe6a\xa2;\xdbq2v\xcbpZ\xd7K*\x1c\xa0Yp\x9a\xe8x\xeeU\xb1\xea@\xe5\'\x9e\x8cu\xec\xe56\x1f\xf7k\x89\x1e\xd6\x87\xed!\xa4\xa8\xe9\x00\xd2\xbb\xe3>\x18\xe1\x9e\xc6H\xbc\xc9\xc4o\xaf@\xdc\x8d\xdaw#\xc4eW+OJ\x8f\xdc\xe2B\x19s\xbf\xd6A\x84\x8eq\xb5\x83bN\x9d\xcbUPa\xd7\xe3v\xafw\x93\xa0\x18Knuj\xd6\xf1r!\x8b\xe6!M=\x1b\x85\x96\xe5I]\xb6\x90\x0c\xa9>\xa8H\xecA\x1c\x1e\x04\xa5=\x8d\x91x\x93\t\xa30\x89\xa2w2\x0f\xdc\x9a\x96"\xc3\xb8\xeeN\xcc\xf6\xb2[\x18\x13A\xb6\xe8js\x8cy\xb0l\xc4\xcaf\xc9s\x8b\xf5\x12>\xc9\xf1$tK\xcer\xd3}\xc8\xf4\x86\xb7\xdc\xdax\x07\xbeS*vt0\x95\xea\x11#\xabE\x01G\x87\xf0A\x99Oc$\xde|\x13\x03\xfd\x02\xa8\xb0~-\x93\x19\xb0\x9a\xcc K\xd9\x1c)1\x12\xb3\xa5\xb5\xe33W\x96\x10\xfb\xba]\x94\xc2\xaa^\xe7\xf6I_N\xd0\xca\xcd.\xe7u\x98-\x90\xbdrR\xcb\x13\xc1\x85\xdcd\xc9\xb8i[\x94\xaa.\xfb\xc9\t\xd5\x1e\xdd\x1dY\x15\xfa\xcc7\xff\xe2\x8cD\x90\xf5\t\x08CI\xe4\x9e\x96\x14\x1d\xe6E\x9a\x95\tq]#\xa4\x99\xa3\xc4\x8a\x80U\xff\xd0\xf6\x08^)\xd6\xaa\xd8\x0e\x98r\xe4/\xd7\x8b`\x84\xc6\x04\x1a\xc4\xa5o\xb8\xdcn\x89\xe7+\x9d\xec}\xd4\x88\xcfYol\xaf\x1b\xdb\xab\x17\xa9\x0e\xfb\xf3~x\x90(\xf04F\xe2-_!\x04u\xcb\xcdw\xae_l4G0\xd0x\x85Y\x9ce\x8f\xdbK\x9d\xa7=\xc1\xb4^\x93:\x08\xc2\xe2\x89]t\xb3\xd9gLu\xecE\x19\xc2\xe7\xe1\xbc\xe6\xd6U\x8c\xc8j\x8c\x1dd\x8f\x83\xf7\x05\xbc#\x97\xb8\xa4\x81jO\xd9\x0f\x96\xff(8\xe1Y\x8c\xc4[\r\x07\xd1\x04h4\xefd\nG\x87\xc0B(\xa1\xfd)t\x13ztS\x1b\xde\xa1s\xb4\x1d\xf3\x01\x0e\x14\xbeC\x8e#\x83\xa9G\\4\x15C\x1bg\x93\xd0h\xef\xb8@\x8e\xac\x04:\x04\xe4\x8c\xabc\xddW\xa3\\@rp\xda(\xb3>\x86\xfd\xf7"\x88=\x8d\x91\x08v\x91\xfa\xa8\x19\xee!\xb3y\xe8\xd9\xd7\xce\n\x8f\xe3A<,z\x87\xceA\xf16-h\x16\x9a,\xbaZ`b\xbe\xf0\xa1\xc3\x9al\xc3R\x0e6u\x88\xeb\n\xefhMp\xb6\xf4UY\x1c\xb7y\xbd\x16\xb6\x18\xa2\x1d\xb2\xee\x02\xbb\xc1\xa2t\xcf\x9f\xa2\xc2_\xcdH\xbc\xe5\t\x92Do\xf0\x9c;\x9fHX,\xebrw[\r\xe6ia*\xd7\xa2\x1b\xedh\xb9\x146T\xd1\x16aj{\x08\x02*\xfd6\xa6R\x96\xae\xb1\xf6\xe8\xce\xe0\x14\x99\x9d\xbc\x89=\x8557\xee\x19\xb9H+\xfe\xaa@\x171U\x94]\xd2\xf3\xe9\xa3,\xefg1\x12?d"\xc8\xad\x88\xbf\xab\xc8K\x90\xbd\xb6Ky\x1a\xbc\xb2\xdb\x15\xa1\x15\xf8\x0eU+\x9ax\\_+k\x9b\xee\xf6\xc4iy&\xa4\xc6\xd3\x1d\x01\xa1\xa3\x08\xd8\xda\xe7\x8a\x86Q/W\xfab5 jX\x076!\x1ad\x87\x95\x9bE?\x17\xbd\xcf>f\xcd\xa71\x12o\x81\x9c\x00\x15/t_\xc3M\xf6\xce\xde4\x90\x03\xfa\xdd\xb2\x0e\xafc\xe8\xc5R\xd0\xf2iz-\xdb\xec\x1a\xeb\x04\x03\xc3\xa6\x82\xe4\xa3Po\xb7\x1e|\x12=\xa8X\x86\x9b\x05\xb6\xc2\xbb"\rw\x9bSA9\x9c\x11.z\xbc^\xe3\xa7+d[\xab\xafb$\x02k\x82F\x9c\x84I\xe8\xae\xbf\xc2\xfc\xfd\x0eC\xc6v\xd2-\x06S*\x0c\x16\n1\x99\xdb\x04\x9f\x91\x13\x19\xa5\x0bt\xb56\x8c\x8bgH\xf0\xbc\xf64\x95\xc3M\r\xf5\xb6\x04\x8e\xa7\x9b\xdd&\x12u|\xb3\x87\xf4\xb3\x98\x97{\x03^U\x17\x0e\x8e\xae\xd6g\x81\xfc/\xceH\x04\xbbH\xdc>V\xc0\xe1;\xd7\xe7\xcf\xc4X\x04\x92\xb8\xf4\xbc]\x95\x9e\xbc\x10t\xfb^\x8c\xed\xafG]p\t\xdeh\x8f\xfc\xb4\x92\x08*M\x98\x86\xceVb"\xa2\xf0,\xad\xfb\xc3\x80\x8b\x9a\xbbE\xcd\xd0R\nmg\xecN,\xa2+Ft\x1a\xf2\xf3\x83<\xb8\xa71\x12A\x89H\xe3\xa0E@\xee\xaf4\x0e\xdc\xb5U\x85\xa6\xde\xacvGI\xa0\xf1uz\xcaf\xc2\xd0\xf2\xe3\xa8]{\xa4\xd8\x1d\x16\xf2\x14\xe2\x0b\xb5\x0e\xd2\x89D\xa0#T\xf5\x1b:\x842\xbd\xacc\x1a\x13\xae|\xbd0\xc4\xb8\x10\x97\xc2\xbe\\\x195\xeb\x1d\x8b\x07#\xdc\xd3\x18\x897k\x12\xe0\x1b\xa9\xfb@N\x1e\x84=\xe1\x8ej\xcfq\x14\xca\xfb\x97\xd1tg3\x00\xbd\xb6\xd8yG\x16s\x8b\xd8\xe4\xf0\xces\x14\xc2:\x98\xb9k`\x97\xd1\x1d`\xa9\x82\xfd\xc2\x16\xeb\xb1\x80.\x83\xb0\x80\x95\xad6VLV\xb6G\xd4\xf2\xec\xef\xe5\x13Oc$\xde\xfa\t\x14\xc5p\xf4>\xeb_\x19\xdc\xcf\n\xbe\xf4\xdb\xe3I\x9a\x10mPl?\xc5\x14\xe9hx\xeb\x19[\r;l&w>\xd9\xaf\xe9\xf3\xb6\xc0\xa3\xac*\xcbk\x8b^\xd1nW^3|\x14u\x99\x9d\x10s4\x8c\xe5q{$\xa8\xbd4\x96\xcd\x83W\x99Oc$\xde\x02(\x0e~\x9a\xbew\xfd\xbe\x90\x83]\xeb2\xe3\x9e\xdb]\xe4\xadI\xc7\x90\x1db\x0b\xbdC\xf7\xa4~\xd6\x96[\xb1g\x8ek\x85\xd3<\xff|k\x07PE]\xba\x83B&N_2\x82\xbb\x87\x0fW\x90?\xa7q{\xb6\xd1\x13&Si[\x95\x0f\xc2\x03\x9f\xc6H\x04\xd6$A\xa1@\x90\xf7\x17T\x13\xb59^\x8b|\xbb\xad<|\x1d\xc6\xf2\xd1\x8a\x97\n+I6\xcbz\x9b\xb0\xb4\xed\x85mF*RO\x8cx\xecgfP\xe6\x11\x91\xce\xa4\x0cQ\x1a\xc6\x18n\x93\x91\xd5&E\xc7C$\x16\xd9Zr\xd7\xec\x1e\xa6\x1f,\xf8\x9f\xc6H\xbcY\x13\xc1A\xf9p\xdf\x04c\xa9N\x0cb\x7f>\xf5\x81\xbf\xafU\xc9\x9e\x84%\x99-\xba\x15\xec\xa0\xc4\xd5\x96\xbc\x93}\xbeh\x8bS\xd9]\x97\xf4\xa2\x0b\x0f-w`\x926P\xdb6m\x82|/\xd6\xc6)XJ+**\xb9\x95(h;\xe1\x9cs_\xc4H\xbcU\xe44h\xf4\xa1\xfb\x9b\x1b-\xd97n2\x08V\xc2\x18\xd8\x82%F\xf8\xca7\xfd\xda\xc4\xdaZ\xd8\x94Fr\xc9\xf7\x84\x05!\xd3:\x12`s&O\xea\xa0_\xfah\x86\xd0n\xb95t\x91\xf6\xb7\xf8vO\x1d\r\xaa#\xf5\xde"\xcd\xcc\xa0\xdc\x07k\xb8\xa7A\x12o\x81\x1c\x82\t\x92@\xee\xaci\xaf\xdc\x08]F\x10\xccN\xa7\x02\x19\xd3Cu!\xf9\xa9\xed\xb4\x00\xe3J\xef\xd4\xa2\xe2\x1a\x87G\xc9\xa2\xb9l\xd3\xc8\xbdx\x9d\x04\x1cC\x05i\x96\xf3(\r\x86\x15\x81g\xa3\x9ez\x8c\x83\xf1\x08\xb3[/\xa6\xabx\xf9\xac\x86\xfb\x8bC\x12?\\\x1f\xc3!\xea\xbeD\xdc\xb3\x81\x80\x18\xde\xb64\xb5p\xd5\x88\xd3\x9e2\x8eK\xac\'\x10F#\xc6\xb3\x84\xccA\xed\xba\xc7+\xb5\x06\x1d\xfc%\x10\x1c\x13\xc1\xd7y\x90\xd7\xcbe>\xb2\xc8~b7\xcb\x18\x8b\x89\x19\x97T(\xd2\xd4\xf5e\xb1\xdd>z7\xfd,H"8,4\x84\x82\x82\x1f\xbb\xf3\x89\xf4\xb2\xc9\xd0\xc5i\x1fYl\x8c\x08\x17yWS1\xe6al\x90\xaf3\xc9\xca\x92\xc6p\xf3e"o\xedF5\xbc\xc7_\xd1\xa8\xde\xa9\xc4Uc\xb1\xc8\xa0\n=F\x90\x1d~^X\x83\x02\xb1\xff$%\xf1\xc3\xfcoJ\xe2?=t\xf2_\x88\x8b\xf8l\xd2\xdb\xdd^}_\xbc\xde\x8bf\x89\xbe\xb7\xf4\xff\xf2\x96>\x9d\xcf\xf7\xa2-}\xfd:\xbf/!\xf0U\xa7\xf4\xe5\xeb\xfc\xbed\xbfWm\xe9\xcb\xd7\xf9}\xa9|/\xda\xd2\xd7\xaf\xf3\xfb\x12\xf5^uJ_\xbe\xce\xefK\xc3{Uzz\xf9:\xbf/\xc9\xeeE[\xfa\xfau~_\n\xdd\xab\xb0\x01/_\xe7\xf7%\xc8\xbd\xca\xf1_\xbe\xce\xefK\x7f{\xd5\x96\xbe|\x9d\xdf\x97\xdc\xf6\xaa"\xea\xe5\xeb\xfc\xbe\xd4\xb5\x17m\xe9\xeb\xd7\xf9}\x89i/\xabK_\xbd\xce\xefK;{\xd5\x96\xbe|\x9d\xdf\x97T\xf62\xc7\x7f\xf5:\xbf/e\xecU\xa7\xf4\xe5\xeb|\x13\xc2\x9e\xbe\xce\xef\xcb\xffz\xd1\x96\xbe~\x9d\xdf\x97\xdd\xf5\xb2\x0fJ^\xbd\xce\xef\xcb\xddz\xd5\x96\xbe|\x9d\xdf\x97\x99\xf5*\xc7\x7f\xf9:\xbf/\xef\xeae\x1f\x94\xbcz\x9d\xdf\x97U\xf5\xb2\xcb\xbdW\xaf\xf3\xfbr\xa6^\xb4\xa5\xaf_\xe7\xf7eD\xbd\xea&\xea\xe5\xeb\xfc\xbe|\xa7W}P\xf2\xf2u~_6\xd3\xcbJ\xfdW\xaf\xf3\xfbr\x95^\xb4\xa5\xaf_\xe7\xf7e"\xbdhK_\xbf\xce\xef\xcb3zU\x11\xf5\xf2u~_\x16\xd1\xab\xd2\xd3\xcb\xd7\xf9}9B\xafr\xfc\x97\xaf\xf3\xfb2\x80^\xb5\xa5/_\xe7\xf7\xe5\xf7\xbcjK_\xbe\xce\xef\xcb\xdeyY\xa9\xff\xeau~_n\xce\xab\x9e6y\xf9:\xbf/\xf3\xe6U=\xfe\xcb\xd7\xf9}y5/{\x18\xf2\xd5\xeb\xfc\xbe\xac\x99\x97=X\xfe\xeau\xfe1"\xc2\xf4\xcf\x13\x11~\xc5O\xf8\xe6\x9c\x98\x97\x0b\xfb*N\xcc\xcb\x85}\x15\'\xe6\xe5\xc2\xbe\x8a\x13\xf3\x80\xb0?F\x1d\xf9*N\xcc\xcb-\xf6U\x9c\x98\xd7\x07\x8f7\'\xe65\x9c\x18\xe4\xbf7\xfd\xf791,\xc3 \x9c\x88\xe04\xca\xb0\x08\x8d\xb1\xf4m@\xb1\x80\x89\x10~\x9b\xde\x05\xe1\x1c\xcf\xd0,\xcbS<\xc4\x08\x10\n\x0b\x1c\xc3\xf3\x88\x08\x13\xb0\x08\xf1\x0c\xf31\xa0\xf1s\x02\x02/\x00\x9d,\x87\x13\x08\x87q@0\nq\xb8\x08\xf38\xcb\x8a$\xc7q7\xd6\x02\xc2q(\x83\xc0\xfc\x8dYBp\xb7Y\xe5\x0cBb\x1c\t\x96\x85\xc3/\xe5\xc4\xfc\xe70\xad\x1f\xbf1\x83\x01\xc5\xfe\x86\xd0$\x0e\x114E\xfc\x1e(\xe6\xfbSv~\x03\x14\x03\xb6\x1b\xc7X\x96\x12E\x92\xc78\x02\x17\x18\x86\xa0!\xf660\x9e\xe5a\x84Ea\x96\x85\x08`^\n\xe21\x96\xe4a\x88%q\x81\xa51Ld`\x86\xbc\x1d\xd67(\xe6\x85\xa0\x18\xf0"\x18\rL\x0es"\xc6\x83EQ,\x81\x08\x18K\x80\xd5p\xec\xcd\x10,\xca\x92\xe0\xbfS"\n\x93"\xcd\xdd\x06R!\x10\xc7!@\x0b\xc9~D\xac\xbf4(\xe6\x9f\xa6\x97<\x0f\x14s\x0b\r\xbf\x0b\x8a\xf9\xfe\x8e\xfe\xb5\xa0\x18\xeaS\x82\xcaZa/by\xce\xe4\xa0s\xdc>\x96\x13\x9bh\xf7n6-\xa2\xa6\xef\x03E<\xc1P\x8fOh\xb3\x10HURzY\x9f\xc4G\xeaH\x9e\x8f\x04v\xecU\xe7\x88\xb7~"\x1fq(\x9b{k\xbd\x11\xda\xdeZr\xa6!Vg\x07\x99M?\xdcw\xbbE\\\xd5\x17\x9a\xd5B\x8d8s\xfe\xe6:\xb0\xa7V.\x1e\x1cR\xf7,N\xcc\xafk\xde\x7f\x98\xbb\xbf/\xf4ZD\xc2\xfa`\x1cF\xa5G\xd9\xfdz\x1b\xa02\xc6.S\xba8\x9c\xf4\xa0m\xa6Cj\xb9:\xb3\xdap\xf8b\xb5I\xb3l\x00\'+\xf4j\x12\xa5\x12\xaa6{\x98j\xab\xab\x1d\xb0\xa9U\xd3\xa1\xc7\x0f\xd0\x83A\xfcY\x9c\x98\x9b5\t\x98\xc0p\xfa>W\xf1\xa9G\xe1ns\xdc\xc78\x01\xbc\xb9\xe7z\xde*\xb6\xe6uy\xc2\xa8\r\xddwm\xd3\xa3\xfd\xc1\xb9\xf2zq07\x0bJZH\xb6\xb69\xb7\x0b%r\x8b\x9d\x82\xe1\x17\x97\xb7\xaf\xe7r<\xcb\xf6\xd0\xaf\x8al\x1b_\x1e\x1c\xc3\xf9,N\xccM&\x84\x7f\xcc]\xbf\x9f1\x8e/\x0e9\xf0\xa9\x82\x97\x11\xf7\x1a\xdb\xcd\x81l\xbb\x11\x91}\x1c\xc6I8"\xe6K\xbd\xef\xe2\x13\xa1\xcc"\xd2\xe9\x8bCT\xe6\xedD\xb1\xcbHm\xf9\xc3ek\x94*A\x19\x9a\xb4\xad\xd8\x1c#\xaf\xd69\xb3\xa23\xfb5\x9c\x98\x8fC\x8b\xa2$\xc8\xe6w\xe8\x8fAo\xd6\xf9\xc9\xd2\x17\x04\x8f\xef{\xfd\xd08\xe3\x91\xd6\xf8:\xa1J\\.\xa6\x08&\xb7f\xcf9G\xc4f\x12G"\xcd\x14\xdb\x04\x14%\xeb[\x84:\x90q\xab\xab\xd3\x82\x9c\xa1,\x00\x05Q\x07\x9de\x94\xd8\x19\xeb\x07S\xf2\xb381\x1f!\x08\xc1n\xf3\xce\xef\x86\xaa\x0e\x14\xca\xf8[\x7f\xc0s\xcc8\xed5\xbcO(\xb5\x8d\xe6\xc9\x9c\xf7\x1bm\x8d\x06\x90f\xe7\xf0\t[\x9f\x14:\x8dgi\xa9nZ\xe1lblh\xc5\x11k\x8e{\x03..;8=\x110\xa2C9+\xe4\xf4R\xf9l\xae\xf3_\x9b\x13sK\xfa4\xa8vQ\x90\xfa\x7f\xbd\x8b\xf0\xc9:\xfa\x1d\x8e\xd0Hh{K\xb2\xb8\xae\x02\xd5^\x8c\xb4v\xcc\x86\xde\xcd\x1aI\xc0N\t\xd50\xc75\x83^\xfa\x85\xca\x0f\xbb\x82\x9f\x90\x1aT\xce\xabP\xe0\xdd\xb9l\x17N\xd7\xb1^\xbaXRe24+\xc9}p:\xf6\xb381\x1f\xaeOB\x18A\x10w\x11./:uQC\x8ez$\x8cm\xb7T=\xecxYi\xe1qI\xb4\xb0\x9a\x1bu\xd4{0\x0b\xb9{z\xa7\xa2\x89\xd95\xe4u\x93[\xcaNp\xe6\x89]\x04\x0b\x1a\xb2\xe4\x8bl\xc2B\xbaf\xd3l\xdc\x07\xdc!\x17\x1e\x0c\xe4\xcf\xe2\xc4|T\x1f$\x85\x10\xbfQ\xa9\xc2\x95\xb1!\xeb\xc6\xd8\xa89\x1cx\x10w\xb2\x9c\r\x96\x9c\xf01laB\xd4\xd9\x91\xf6\xb8c\x9e\x1c\xaf\x97f\xb3\xceV\xab\x91\x8d.\xeb\x81\x80\xac\x9a\x81\xb5\rJ\x1b\'}y\\\x99\xcd\x04\xbc\x131\x1c\x1f\x82\x97\x9b\x7fMN\xccm\x17\x11\xd0y\xd2\x04qW\xdc\xe4~(\t\xdez]\xc5\x96\xe2\xeeEo\x87\xe3\x0e\xc7;\x94\xb6\x9e`\xfcp\xf6.x\xb1\x07!m7\xc1M\x0f\xcf\x06\xc6\xca\x01\xd2\x94\x90\xcb\x0e\x9d\x10\x18\x9dz\x1d|n\x12\xcb\x1e\xddAY\x84\xcdy\xb7\xb4\xd1\x07\x03\xe8\xb3817\x994\nA\xc4\xfdY1\xb7\xf0nt\x91\x04_\xf2p\x92\xa3\xabSI\x0f\x1ec`\xea5\xee\x8f\xf4\xa1\xa0Bi\x8f*\xf9x\xe4\xc5\\\x98\xd6t\xb7PF#\xba\x16W5\x8a\x92]\x15\xa1\xc7\xe4pV\xa1\\!zR\x9a\xf9K\xb0t\xd5\x07\xbb\x9agab>j\x1bP\x90\xdf.\x97\xee\xe6\xe2;b\xb8o\x80\x80\xd2\x1c\xf8\x11\x8a\xe7\n\x82.\x184\x0f\xce\x89\x8f\xebC\x16\x8ci\xc7{\x89V\xefS\xc6\xf4\xd6\x8b\xa1\x08O\xe7\x0c\x1d\xd9\x04Il\xda\xc77\xc9\xc07Qvf\xba\x99Yi\xd7m\xbc\xd9\x05\x0f\xb2"\x9e\x85\x89\xf9\x90\x89\xdf\x92\xfe=\x12#\xad\xd6\xe8\x82\xe3\xbb\xeb\xe1\xa4u\x01_\xf8\x14.n\x96\xad\x1b[\xa6w\xbc\xc2TI\x9b6\xc2\x8f\\\xb6\xf6z\x88e\x9b\xeeXt\x88/\xa1\xcb>\xc47B\xb5(k\xd6\xa51\xa5\x81\xab\xedzNT?\x9d\xb2\xe9A\x99\xcf\xc2\xc4|\x048\xe0\xe0\x18D\xdc%\xfd\x13\xa9\xa9\xcc\xe8\xc8\xdcj\x81\xeb\xc7\xdc\x90\x9bEl\x86\x94\x05\x1a\x8c\xb6_\xed`W\x9aD\xcf.\xa35m\xab\xc3qq\x86\xd7\xb5\x12\xf4\xabpsU\x97P\x81\'r\x8a\x10\xf9nW^{\xb3G\x0b\xf9R\xf9\xa7\xf9{\x05\xb8gabn\xbb\x88\xc2\xb7\xcf\x1e\xee\xd1I\xb6o\xaeFq\xca\x82\xb0\x9e\xae\xeb\r\x9b\xf3\xae\x9cX\xe1\x88\xa2\xad\xafx\xe5\xec\x0f*\xb5<\x1b\xbeB\xaa\xe9fhb\x0f"\xac\x93\x9f\xa9\x87\n\xf7J,\xd9\x8c&\xb1F3\xabo\x90z}\xc42\x07\xbe\xb6\xec\x830\xd3gab~vo\x14\x02\xe1\xf7\xb3\xb7/\x9aS\xb0\x07&\\\x1fR\x96b\\\xc1\xb3\xe2\xcaY\xdb\xb8\xd9\x9fk\xba>U\xb2-`C\xc4\xe3\xd6\xa6\xf9\xa8L\xcf\xeaU\xd5\xb9\xf9\x98\x92\\\xe0\xe6G~*]C:\xc5\xd85?\xed\x98\x80\x18.R\xc98Nz\xba\xc0\xa0\xa3\xff\x97\xa4\xc4\xfc,\x10Q\x94&\xd1\xbb]\xdc\xc1\xf0\xd9YSXZ\x85\xe2\x14\xa6\x97\xf3a\xd1\xa2B\x1bz1hj9\xbf\xe8\x1d\x83\x97&\xe6\xecq\xab\x06^o\x16E\xc1a\xf0i\x9a\xb7\xaa\xb3R4yu^\x93R\xcc\xd8\xe11[\xfbM\x80\xf3\xa7\xddF\x7f0~>\x8b\x12ss\t\x8a\xc2@\xdbp\x0f\xc3\x91\xd2^\xef\xed\xfc(\x85b\x04\xa3\xa7\x15o\x1b\xf9\xde&\x0e\xbdC\xed\x87<`\x9a\xe60\x95\xe9\xe9t6z\x91\x85\xa8\xf6|\xb0P\x97\xdc\x91\xed\x1ej\x82\xda8*=\xa9\xc5\x1d\xe2-\xb731\x0c\x1a\xbd\x1ca\xe4\x8b(1\x1f2q\n\xec\x08z\x17\xe0\xdcp\x9b\x8du\x1b\xebA\xdc_L\x9e\xea\x94\xd1C\x08\x1b\xd9@:\xbf\xca.5\x04\x9dL\x7f\xb9\xac\x8a\xf8<],)%\xe8\xabY\r\xf0<\x87\x8a\xb2Z\x88\xec\xb6)2\xe3R\x9c8\xb9SK\x8dT\x9b3{x\xb0\x82{\x16%\xe6\xa3\x82\xc3\x10\xf0\xd7]\xa3\x9fre\xb8\x0c\xa8U\'Ug\x90\xea\x88\x11\x12\xd6k\xfcL\x1f\x90\xeby\xbd\xc5\x0em\x9cg\xf2\x8e\xda\x08\xae\xb2\xc7\xb6\xb9w\x91\x14\xce\x1c\x98sj\xe3\xb6\x97\x99\xe0\x85\xa8\xd8m\xda^f0\xdd\x85\xc5\x18\xcf\xf3\xe1\x8b 1\xb7\x16\x15F\x10\n\x81\xef\x8d\xe9Wk\xa1R\x1b\t\xd1`\xf8\x10\xf6\x93E\xaf\xb7\x84i&#\xdc\xc2\x02d_pS\xad\xdc\xc4/& L\xe3\xae\x9bE.\x04\x1c\xac\xee\xbe\x17\xf7\xf6Y\x90\x98\x0f\xcf\'\x81g\x11\xf7\x9fG%S\xe1\xf6c\xa7\xf5\x89W\xa5\xa8t\xb6\x9d\xc0\xf7\xad"\xcf\x92\xc45\x90-\xc5\x96\xc1VskB\x95\xe0\xed\x80Jim\x0c\x04\x9b\xc4fE\x98\xda.\xbaRxAm\xe1\xc2\x8f6vf\xe5\x0b(h\xf9\x95\xfd .\xf1Y\x90\x98\x9bL\x0c\x870\x10\xf2\xef\x9a\xb74\x17\xad\xeex\x1c\xfc\x1c\xb2gy\xb0\xdai\xd3\xe6k\xfaX\xadl\x91\x10\xb7\x95@%\xccH\xb9\xac\xb7\x837Y\xbb\x90\x94n!:=\xae\x0e\xe10.\xa4dH\xfd2;\xeds,TZb\xb1\xd82+g\xdf~\x11$\x06\xc8\x04\xdf\r\xce\x02\xe8\x16\xeeh_\xb3/\xa4\xbbZ\xd41\xaa`\x16A\x03\x17\x9cz\xf1U\xbdG\xcf}\xa97\x91\x19Z\xa9\xc9\xe5tv\xcdG;\x8c\\R\x08k\xe4bn6\x1b8\x17b\xe7\xb4fx\xc6Xa\x8b\x06\xa2\nu\x89(\x81{\xa4\x1f\xbc\x9dz\x16$\xe6\xe3\xc2\x01\xd4-$}\x8f7i\x06\xb4\xde\xc1\xed\x10\xfb\xb1\xb94\xa9^\xee\xb1-\xa9\xeaV\x08\xf9m\xb5qE\xf1@\r\xbe9Cm\'\x82\xce\xdc\xb8\x98V\xb8\xe7qtq\xc1\x186:\xa3\xf6\x19\xc3u2\x1dBH\xa8\x1b\\E\xb1\xca\xe6\xa6\xcf\x02\xf9_\x1b\x12\xf3\xf1\x19-\x81\xe2\xe0\xec\xdd\x05\xd0\r$\xb11\x1d\xe5\x96\x99\x07\xe1\x9a&.zjy\xd6\xf6dz\xce\xbc\x82\xfb\xae\x9b\x03s\xcb\x95\x87#\xee\n:\xdcf\xbb^\x9f\xcf[\xd7\xdf\x10\xa8ZV\x8c\xe0\xd3\xe9\x1eF+\xa1\x1aX\xa1\xbc\xfa)\xaem5\x96\xf9\xe7 1\x1f\xb5\xe7\x1b\x12\xf3O?\xad\xfe/\x04\x89\xf9\x0b\xcd\x91|\x8f\xe6|Cb\xde\x90\x987$\xe6\r\x89yCb\xde\x90\x98\xef\xbb\xce7$\xe6\r\x89yCb\xde\x90\x98\xef\x0c_yCb\xde\x90\x987$\xe6\r\x89\xf9\xb6\xeb|Cb\xde\x90\x98\xffK7QoH\xcc\x1b\x12\xf3\x86\xc4\xbc!1oH\xcc\x1b\x12\xf3\x86\xc4|\xdfu\xbe!1oH\xcc\x1b\x12\xf3\x86\xc4|g\xf8\xca\x1b\x12\xf3\x86\xc4\xbc!1oH\xcc\xb7]\xe7\x1b\x12\xf3\x86\xc4\xbc!1oH\xccw\x86\xaf\xbc!1oH\xcc\x1b\x12\xf3\x86\xc4|\xdbu\xbe!1oH\xcc\x1b\x12\xf3\x86\xc4|g\xf8\xca\x1b\x12\xf3\x86\xc4\xbc!1_\x0b\x89\xb9>\x84C\x98\xfeaI\xdf\x1a\x12\xf3ra_\x05\x89y\xb9\xb0\xaf\x82\xc4\xbc\\\xd8WAb\x1e\x13\xf6\x07\x90#_\x05\x89y\xb9\xc5\xbe\n\x12\xf3\xfa\xe0\xf1\x86\xc4\xbc\x06\x12\x83\xfe\xf7\xa6\xff>$\x06#1\xf2\xc6\nApB\xa0\xb8\x1b\xa6\x85\x82X\x08\xc6P\x84FE\x8a \x08\x96b\x11\x04\xe2h\x8c\xa0h\x9c\xc19\x1a|I\x04\xcaDD \xb9\x8f\xa1\x00\x9f\xd3\x0f\x04\x8cCaX\xc4x\x1c\x11`\x8c\x83\x11\x9cca\x8ad\x05\x18\xc5\x11\x0e\x838\x11\xe6 \x9e\xc3\xc1+\x0b\x10\x8cb\x02/p\xf8\r\xd0\x01\x89(K\xe2\xf4K!1\xb7-\xfa\x0c\x12\x83A\x7f#\x80`\x84\xc4q\xf2\xf7 1\xdf\x1f\xb1\xf3\x1b\x90\x18\x92Dq\nBX\x94%P\x98\x00\xef r\x10K\x89\xb7\xa1\xad\x04X\x1e\x02C\xbc\xc0"\x0c\xc3b\xd0mF\'E\x118C\xa10\x02\xe3$Ka\x1f\x83\xdc~\t\x89A\xd1\x1bb\x86g\x05p\x94x\x9eB\xc0B\x08\x8e"\x19\x08\xc59\x86@\x85\xdb\xa2!\x1a\xc7\x08L\x10 \xe86\xf6\x8f\xa4\x04X\xe4H\x04bD\x1a\xe1\x7f\xfc\xe2\x90\xbf!17\xbb\xb3\xc0\xefX\x88E\t\xe0\xdc\x02A\x8b\x0c\x8d\x92\xc0\x06<-\n`M0E\x90\x02&\xdc6\x16\xc7\x18\x92\xe6I\x88EP\x01\x85I\x9c\xc2!\x86A\x7f\xfc\xc5!1\xff4\xd3\xe3\x0e\x12\xf3\x83\xeb0\x96\xeb\xce,\xb7\xd7\x06\x1f\xa1\xe7\xc8\xa5\xe7u\xabC\xbe\xdbwQ\x13_\x92R\xdc\x07\xde\xb6\xd0v\xb8\x1e#\x89\x10\xb8S\x1d "\x14l\x15\x88\xdb\xf7C\x8c\xc0\x83\xe6\x1dpCrZ\xad\xed\xa0@f\xae\x012\xe1\x91$ Z\xe9_\r\x17\xbfFrP\xa6M\x8c\xc5s\xd2\x18\xb2\xde\xc6\x12\x04\x0b\xa5\xf2\xf1\xb3\xb7.(B\xd9:.EP\xe1\xb3S\\\x82\n\xa49C!R\xb4!Z\x1c\xf5\xb9h|4AC>\xa8"^\x87\x93y[\xc7\xfc\xed\x17\xeaY\\g\x0f[.w\xe0\x8f\x1c9\xadV:\x1f\x8f\x1a\xcf\xdc\xfe\xde\x05fgp\x95\xaf\xcaW\xba\xfc\x99C\xb5\x8b\x8f*x,o\x07\xa0\xaf\xf6Q\xe6\xaa[\xc1\t\xe82\x13WY\x07^\xb5[\x8d\x87\x82!\xc2.&\xf7V\x11m\xab1:\xa8\xc7\xcd\x95H\xbdi2\x07K^@\xb5^y\xc1p\xc9y\x8dv\xb0\\\xc8M\x95\xc9\x8b\xa5Ph*o\x1e\xc4\xc4\xbd\xed\x1f\xb5cy\x1f\xbc6cV\xaa\xb0\x91*~\'\x04(+\xb1\xfb\xa5O\xa6\xda\xe9\xd2\x1b\xa9\x0e\xe3\xf6i\xdf\x96\xba\x19[$d\r\xe7\xac\xdd\xe2\xa6\xe2Y+Q&\x059JK&\xbc`\x16\x88V\x0c\xc9/\x92(T}m\x99#TSU\xd8\x1e\xf8&\xe6T?\xfe0\xde\xe7\x96\x80~\x17\xef\xf3\xfdC\xf4\x97\xe2}P\xfcs4\x04\xe8\xe4\xd0=\xb6(\xcf"\xed\xcf{\xa3\xaa\xdbn\x05\xfa\x1f\xd4\x84\xed\x1c;3ka\x96\x99\xb1w\x0e\x8c\x87\x83l|\x1e}\xd8\x8c\t\xb6%\xb0E\xeah\x0bx\xc6\x0fT\x9bm\xad\x95\xc5\xeeX\xe4\xd4\xa7\x19\x99\x93\xdcc#\x06\x9f\xc5\xf7\xf9uB\xff\x87Ac\x81\xb6\x12\xe1\x84\xb3\xf5<\x0c\x93\x0e\xa2\xb2\xf4\xbad\x05\xc5\x19\xf6\x83\xde\x9f\xb7\xd6\xb9\\-\xf6:>\xd0\xbc.J\xb8$\xc3\xd4I\xca\x9c\xc2\x8a\xf8\xa3\xda\xe8UM\xe5%U\xdbhx&\x90\x80\x05\x11\x05\x9b\xaa\x07\x87\xef=\x8b\xef\x03dR(\x02\x12\xf6\x1d\xcc\xc4\xe7\x0b\xbaV\xadK?B\xb2\x87j\xcan\xd9P\xd8Fi\xafB\xb1\x9fr^\xaaD\xae\xb8\xa4\xc7\x10\xe9L\x1e\x84,\xd9_\xb8\xdc5\xe0\x8a\x81a\xf1J\xf3W\xd7\xe4\x82Ac\x86;j._\x9da\x95^\xc7/\xe2\xfb\x00\x8d$\x02\xaaP\x0c\x82\xef\xe6(\xc6\x84\xde\x12W\xdd\x1d\x8a\x9c.\n"M\x84La\xbdF\x02\xad\x1e99\xe5\xbcP\x0b\xf5\xa4.l!\xa6"P\xeb\xb0\xeeu\xd1\x15~8_\xb3}\x97\\\xb8\x0eA\x91\xf2p\xdd\xe0\xa1-RkB\x1d\xaf\xc7Q\xfdl\xc4\xe9_\x1b\xf0ss\x08\xe26\xd8\x118\xd3\xafw\x91\x92\x97\xa7\xfc\xaaI+CGW\x96!T\xf6\xaa\xdd\x9f\xcdS\xbd\x92C\x18>55u\xd6\x86\xd3\x10\xb7"\xa7\xc2,\xcc)\xe7\xd0\xd0da\xb0j\xdc\xec\x87\x05\xdd]\xb6\xb3\xbf\xe1V\xfb\xc1\x8aN^J\x1b>\xe6?:\x1c\xfeI\x80\x9f\xdbaA\x11\x0c\x03\r\xcd\x9d\xdf\xcf\xe7\xf2 L\xe5I\x1c\x9b\xa6\xb4\xf3\xbe`\xcf\xeb\x12\x85\x83\xc4$"\x83\x9b\xf8\xad~\x9a\xb7+\xb3\xefp\xd7\xdc\xe9\xc7\xb5\xdbQ\x9d\x99\x8f,\x12&B\xc3\xe7\x05\xee\xd8>\xdb\xc2[bs\xea\xf6\xc1\xbf&\xe0\xe7\xa3\xb2\x01\x9e\x0fN\xcc]\xbe\xaa[\xc5\xa5@\xed\xb9L\xa6\x8b\x8c\xe6#\x8a\xb1g\xa9BZ\xf5b\xc2uhp\xce\xa6\x94\xf7\xc4n\xa7\x8d\xb96i\xd7\xf2\x1at\x8bK\x87\xaf\xa8\xe3q\xbfOZ\xee\x94!\xa7Z%\x17W\x1b\xc5\xb6\xdbh\xbbs\x8a\xfe\xc1\x02\xeeY\x80\x9f\x0f\x9f\x801\x1c\xc7\xef0F\x8cW\x0eZX\x8f\'\xd8\x9d\x92E\x81\xd2\xa5\xd4\xc4\x1d"\xfb\xcc*\t\xb6P\xe5\xfb\xeb\xd8a\xc4\rd\x9f\x89\xccm\xb3\xd86gT\x16vA\x1f\xd4"\xb2Z\xd0\xc7\xdd\nb\xf0\x93\x07\xca\xaei\x1b\xb4\xea\xbcy\x14\xd4\xf6,\xbe\xcfG\t\x07\x83\xfa\x03\xc3\xef\x8cy\xd4*\x81\xef7\xf6>8\xc7\xect\xec;\x16\x11\xc8\x011bs\xef\rdn8\xb3)\x89aBJV\'\xab\xca\x81\x1d7c$\xc3\x8b\xf3\x89\xdb\x9d\xd9\xcb\x89\xb6\xc3M\xac\x11\x91\xc7*\xc5UH/b\xb8H\xc6\xef5#\xfaY|\x9f\xdb.\x82TM\x82\xf0y\xb7\x8b\t\xaaj\x08\xa3]\xect-\x9cK\xaa\xd5\xe4\x9e\xe0"Q\x1d\x9d\xf1\xa4U\x9ce\xeev\x0b\xd1\xccm7lC\x1d\xa2\r+\xebScs=XP]_i.\xda\x1f\xfc\xf3\x1c(\x08\xb6\xb0\x15\xb8\xf5F\xd4\xe3\xd9\x07[\xb7g\xf1}>d\x12\xe8m\xa2\xf8]m\x83y\r\x16\xf1q\xbf\t\xf4d\xd3\x05;\xa3>\x97\xf8\x96G\xb6\x18\x93]7\xfe\xa4\xec\x19o\r\x9b\xbd\t\xb7~\xc2^\x852\x1fw(\x1be\xd4\xaa\xd9\xe0r\xbc\x0b$\x81\x98\xe2\xc1\xbb\xe8\xebC\xba+\xa45\xc6\xea\x0fR>\x9e\x05\xf8\xb9\xc9\x84\xc16\x91$vgMgN\x96\x8a\xd9\xbaC\xb9\x1f\xf5\xfc\xe2v\x03\tZTz\xa2\xb7\xa4}l\xa9\xa3\x95\x92[A\xc1-\x7fMa\xd8\xe9\xdc7\xbcZ3)E\xe79\x84\xd4\xbch\x8e\x99\xc1\xeb\xa4r\x99\x1a\xb7\xcaL\x9fa\x07\xb5\x7f4\xe9?\t\xf0s\x93I\x82\xb6\x03C\xb1;k\xca\x1bdpW\xa9\xcf\xf5\t,[\x1b\x95Oc\x9a\x90\xd8\t?l\x0b\x13W\xe2\x94M\x0e8v\x14\xf6\xeb}\'\xf1\xe2tt\xd1\xa4X\x17\xa5\x82\xe2\x8bD\xd9+\xca\xdc\x10\'\xa3%-K-@6H\x97IO>\x08/|\x16\xe0\xe7\x16\xc7A\x17\x86\x81\xb3\x7f\x87k\x92\xc9\xcbt\xcd\xe6HD<\x92_T\xb1vB\x97\xe2(M\xd0pl\xc6\xf4\xa2\x84\x99H\x82\xba\xcd\x1c@\x1bu\x1d\x06~\xd2\x96\xb1vP\xe6^.\xf8,\xcew^\xd5\xd6c\xa2]\xf6\xb6\xc1\x8f\xa1\xa0%\xfa\xda\xfc\xec\xd0\xfe\xb5\x01?\x1fy\x82D\x10\x18\xa7\xee\xea}Cp\rN\xab\xcf\xbc\xa1R\xf6\x82sO\x9d\xb7\t\xb5\xa8\xa7\xc7\x05\xef\xa1^\xe4\xec\xe3\xc8\xd8\xf2:\xb7PNr\xc0\xe7TP\xe8\xf5\xa9\xb7LN\xb1\xb6\xa9^7\xf9\xa6&\x98$2z\xddZ\xc9i\xcd\xef\xe0\xf8\xc1\x1b\x8dg\x01~n\x87\x05\x01-\x12z\xef\xf9\xdeHG\x9e\xedhN\xe7\xa2\xa4/\x91\xad\xd3\xc6Z=\xf0L\xd7\x87uqX\xb8\xf4\xf1\xbaW\xd7;q\xb3\xd8\x86\xc2v]Z\xd8n\x15\xabg\xceW\x99\x18v6\xf5\x8c\x8d\xa9\xe3\xdbXo\x978M!\xa7\x91u\x1f\x05]>\x89\xef\xf3aL\x04\xf4\x0c\xd0=\xb3e\xe4\x13\xa5\x161c\x0b\x85\x87\xfc\xe4\x1e\x07[>p\x8bn\x86\xca\x029v\xcb\x0en\tF\xde\xa3\x88\xc7\xac4\xc4KzVF0\xd2\xe3\x05\x84\xa3\xd7\xfe6%Js\x82\x98\xbaw\xc4"\xa6\x90\xf3b\xa6\x03Ve\xbfU\xd2\x7f\x16\xe0\xe7\xa3\xd3G\xc0i\x01\x89\xf4\xd7\xbb\x88\xb8\x92\x1fBh\xa8\xea\xbc\xd8om\x1b\xeb\x9a8\xce:\xca\x19Bo\x98\xd2h1\x9e\xc4\xae\xca,\x81=\xac$\x87\xa8\x88)#\xf5\x11\x12WM`\x0f\x9a\xd6\xa4p\xdclO\x90v\xe0\xa6\ry1Xt\x8f\x92\x0f6M\xcf\x02\xfc|\xd4\xc18\x04\x8a\xa3{\xaa\xdf\x9c\\\x13|\x83\x11\x93h_D\xcc\r"\x96h\xc2\xbdB\xda}\x0cK\xba\x9d\n\xb0\xc8\xa0+-\xda\x1f\xa5f[\xceGo\x16E\x0e\xc4\x80Y\xa3\xbbrD\xc4\xf5R\xd8\xb7\xa4\xaa\xf4\xdb\xcc\xd5\xd7.\x19\x1f\xf9\xe5\x83>\xf1,\xc0\x0f\x90\t\x8e\x03P\n\xd2\xe1\x1d\x18\xe2\xba\x98Jh!\xe5\xd1IbSg\xaau\x99H\x1c\x8b\xe6\xfc\xd8`\x0e\x9a\xd9`\xb4\xc4\xd9v\xa15\xe4\xe5(\xeb\xce\xfe\xbc\x83\x97~|\x9a|*\x16cd\x90]\x86\xc8\t1^\xcenY\xd3+k/:\xf9g\x85\xf0\x8b\x01?\x1f\xae\x8f\xe1\x18\x08qw\x9d~\x16i1\xb2\xb29r\xdf\x15h\xd08\xdbE\x05\x8d\xa7\x1a\xe9\x97\xf3\x8a\xe0\xcf\xce>\x87\xeaK/:\xc4iyl\x97;D#\x82b1\xf8\xdbV:\x96\x88\x88\x16\xf9\x18\x0e\xc51\xde\xb5B\xcc\x9c\xac\xf5ZX\x8c\xe8\x83q\xfcY\x84\x9f\x8fJ\x15\xc5)\x10\xc8\xefdn\x065\x86\xab\xb1\x9d\x03\xb3\x0b\x8fZ\xb2A\x9c\x80J\x9a\x0e\xf2\x12\x8eK\xbc\x8cjBO/f.\xe0\x92\xe2h\x8bV\xaa,\xc6c\x91\x9b-=\xb2K\xcbZ\xad\xe9\x90O7\x96\xdddi6\x9f\x0b\x07\xb9\xce\xcd\x17\x11~n\xbey\xeb\xdc\xa0\xdf\xe05\r\xeda\x02\xf9\x87\x81\xac\x88\xcaF\x81\x91\xd2\x95G\xa8%\x97\x96\x92\xeaz\xae\xbd\x10\xaeYV\xc3ce\xf2X\x89#=\xb0,\xa4\x97\xf4elCS?\x9c\xce\xd2\xe4\x82\xd0>s\xb0\xaa\x06\x04n\xb6\xdbt#~f\xcd\xbf6\xe1\xe7\xef\xae\x0f\xd1\x10yW\x08_\xd10\xdd\x91\xae\xb0_h\x0e)mF\xf3\xbcbV\xfc\xc1K\xbd`\xb1\x16\x85I\x81\xdd\x80\xc2P5\x8b\x18j+#\xe6N\xdf\xeeM2X\xd8\xc5ry\x8c\x0el\x86\xef\x939d7\'\x88\xc7\xcb\x96\xb0\xa4\xd3b\xfb`\x84{\x16\xe1\xe7\xe3\xb0\xe0\xb7\xfbw\xe2\xeeFcw\x99\xb2\r\x9b9\x077\xac1O\xdd\xd2a\xa4J\xddY<\xd0z9\x9a\xa5\xc9\x8c\xf2I\xeb\xce\xe9:\xaa\xf9\xe1\xb2\xf4D/l\xda\xb9u\xd6\x99w\xb1\xdc\x0c[v\x97\xad\x1d6\xd3\xd5\x9e\xb6\xd7\x04\x0bK+\xda=\x88\xbey\x16\xe1\x07\xc8\xa4\x81)\xc1\x9e\xdd\xd7p\xa2\x951\xeaj\xf6\xb3\x19\x0e\xc4\x94\x1d\x15\xcd\xe2\xb4d\x9b\xc2\xc6>*\x14y^(\xd71\xa0\xe1\x8du\xe6:\x1d\x8b\xf5\xceKd]\x89i\xe12\x00\xf7\x08\xb7W\xbf\x18\xcdS\x80Q\xd9Q\xb8\xc4\xc5\xe1\x1ci\xf8\x83\x177\xcf"\xfc|4\xa9$\x0e\xd1 ^\xfcZ&\xa1A{[\xc4:u[\xd8\x07mJR\x89\xa5v\x87y\xbdW[u]\r\xfc\x02v\x16~\xda\x8e\xeb(\xd8W,zR\xa1b\xd88\xf3<\x0el\xb5\xc5\xab\x93\xce\x8e\xb1\xeef\xa7\x925\x8b\xaa\xba.\x16\xf5\xf1\x9bA\x1a\x9fE\xf8\xb9\xed"\xd8\xef[cs\xd7\x1bz\x9b\xb9\xee\xd9\xcc\xda\r\x886\x1a\xf3\xd89}#en\xce(\xb8\x17A\xd2\\\x94\xeb.\xa8k3\x15\xf0Lg\xe7s\xe2\xd1\x887\xb5\xe3,w4c\xaf9x\x0fw3\x81/\xf5k*\x8c\x08w\xd9;\xce\xcfB\xf8\x7f\x06\xfc|<\x10\xf7K\xc0\xcf\xbf\xfd\xc7\x8f\xf8\x90\xa4\x1fOV\xfc|\xd0\xe6\x07\x97\x1fTN\xda\xd6\x91\xe4\xb4\xbe\xb7\xad\xc1\x8f\xec\xdb\xec\xf0s\xef\xf3\xf0\xfc\xefc\xd8\xf6\xe9\xed\xa9;\x04\xff\xfb\x83\x8d\xb7\xaf^\xce\x1f_\x03U \x86\xa3\x1f\x0f\x12\xdd\x81\x82\xa6\xcf\x1eM\xf5]\xbd\xf4P\xbd\x8e\xcd\xfbGIg\x910\xac\xa2\x0ef\x85H\xa4\xdb\xb4\x80\xe4\x9c\xbaS\xa1\x97+D+\xebK\xdcL\x88\x8fv\xad!khP\x16s\x80\xb2\xbd\xbe\xa3\xc6\x7f|6\xf6\xf3\xb7\x8e\x91\xba\x8d\x1a\x11J<\xa5\xfe\xe5\xb3\xaeR!\x07r\xb2\x0b\xfe\x7f\xf6\xde\xb4Iq,[\xd7\xfc/\xf1\x95\xb6B\xf3p\xbfi\x96\xd0\x84\x064\xd0\xd6\xd6\xa6\x19\t\t\x01\x92\x10R\xff\xf9\xdexf\x9dS\x95D\xe4)\xea\xc2q\xf2\\,-2\xd2<\xdd\xf1\xfd\xee\xb5\xd7\xb4\x05\xeb\x81kD\xdfSs\xb4\xef\xd0\xa4\xb6\x87\xa8\xb1ko\xdfA\to\x13\xdef\'\xaas-\x1as\xaa\x18.[{\x1b\xc3\xd7\xe5\x9a\xdb\x94=fU\xca\x1c\xa3\xa9\x9c\x1c<1\x85Dh\x0b\xd1\xa8\x86\xe0\x81\xde\xd4\xbd-\x82\xdcy\xd8\xaa\x89p\\;><\x85\x1bZ\xf2\xf7\x02\xbd\xb6\xfe\xc8\xf8\xf9\xb7\xd1J\xff\xc6\x9e\xfd=\xda\xfc\xfe\xca\x7f\x86V\xfa\xdf\x067\xdd\x7f\xc6\xe3\x07X\x1b\xac\xcb\xde\xd7[\xac\xfey\x13\xfe]\xc8\xd2\xbf\xbb\x07\x0f\xfcnpn\xa1\xc8\xa7\x87\x7f\xf8\xdd`)\xdd\xd6Ow\xa9\xb4y\x9a1{\xc3\xb7\x87\x10T\xcei\x95v\xe9\x81E2\xdf\x98\xf4\xca \xa2F\xc0\xc1\xa6\xd7!\xea\rz\xd3Wi\xa5L\x06\x10\x9a<`\xcc\xff\xed\xa3\xf2\x13c\xea\xae\x85\x9a|x5]\xe6\xfa\xfb{\xcc\x1e\xdb\xd8\xa7\xef\xc5?}"\xab\x8e\xca\xe6\xbf\xdf\xa5\xfe\xad}J\xb3:+\xa2\xfe\x97\x1b\xf5]\x93\xd6~\xfc\xc3[\x17\x1f3m\xec\xd3\xfb\xd4\xbf\xfec\xb0MP{\x17!\xf5\xb0\x1d\xff\xf9\xd8>\xdf\xb3\xff\x9f\xff\xeb+\xddu\xc7\xe8\xf6\xbe\xc3\x1f?n_\xf8\xd0\xed~\xf11\xbb\xffAt\xbb\xbf\x0e7\xec\x83b\xfbl\xe9\xfbo\xe9\x87n\xf7\xa1\xdb}\xe8v\x1f\xba\xdd\x87n\xf7\xa1\xdb}\xe8vo\xbc\xce\x0f\xdd\xeeC\xb7\xfb\xd0\xed>t\xbbw\xa6\xc6}\xe8v\x1f\xba\xdd\x87n\xf7\xa1\xdb\xbd\xed:?t\xbb\x0f\xdd\xeeC\xb7\xfb\xd0\xed\xde\x99\x1a\xf7\xa1\xdb}\xe8v\x1f\xba\xdd\x87n\xf7\xb6\xeb\xfc\xd0\xed>t\xbb\x0f\xdd\xeeC\xb7{gj\xdc\x87n\xf7\xa1\xdb}\xe8v\x1f\xba\xdd\xdb\xae\xf3C\xb7\xfb\xd0\xed>t\xbb\x0f\xdd\xee\x9d\xa9q\x1f\xba\xdd\x87n\xf7\xa1\xdb}\xe8vo\xbb\xce\x0f\xdd\xeeC\xb7\xfb\xd0\xed>t\xbbw\xa6\xc6}\xe8v\x1f\xba\xddC\x1c\xa7\xf9\x1f~\xc9\x7f\xcdq\x9a\xfeiIoM\xb7{\xb9\xb0\xef\xa2\xdb\xbd\\\xd8w\xd1\xed^.\xec\xbb\xe8v\x8f\t\xfb7Xi\xdfE\xb7{\xb9\xc5\xbe\x8bn\xf7\xfa\xe0\xf1\xdfB\xb7\xfb\xbf\xff\x83\x0bw\x9b\xe7\xe2\x0c\xcd\x17\xa9\xe6\xf7\x0f\xa8\x83\xd7\xc9\xceMy\xe8\xff\x96\x9c\xa7c\xdf\xfem=\xc4u\x99\xa8\xd9\xf4\xff\n\xbf\xb3\xe3\xfec\xfd\xff\xdf\x7f\xe0\xe4\xfe\xd7\x0f\xc2\x98!:\x94\xe4y\xf4AU0K\x90\xb2S1\xe5\x02\x9dBDI\x02h\xbd\xc4\xd5\xb2\xbf\xd8\x81\xbb\xb9\xf5\xac\xb7\xc92\xc7v\xbc\xc1\xd9~`\xf4\x8f\xdf>I\xff\x7f\x10m\x0f\xfb\xc7M\xfc3\xda\x1e\x81@\x14I\xa2\x08\x02\xdd\x86\xde\x130\t\xa1\x10\xc6\xe0$K\xa3\x90\xc8\xc2(\x8a\xc27\x80\x1a\xc9\x8a\x04\xcdS\x90 \x92\x04C\xd0\x18\xc5B,A\x91_#\xa1\x7f\r#\xc2P^\x149\x0cc\xb1\x1bo\x8d\xbbq\xb7H\x88GD\x0c\x15 \x8c\xa5\x05\x1c\xa1\x05\x91Cq\x9e\x83\x10\x1c\x13)\x8a`\xc1\xaeQ,\x02C\x02\xc4\x93\xdcKi{\x7f\x9f|\xfe\xe3g\xf3\x90\xc8\xbf\xe10IS(\xf2\xdbT\xa9_\xd1\xf6\xde\x9fU\xf8\x13\xda\x1e/@"\xcc\x00\xab \x04\xc7\x812\x1a\xec\xba@\x89\x08\xb0\x0b-p\x08\r\xce\x01"\x80_\x05\xc1\x08OQ\xac\xc0\x89\xb71\xa0\x04I\x08,\xcc\x12\x18\'\xfc\xf8g\xda\xde\x13\x90k\xffq\xc8\xff\x1bh{4\xf8v\x9a\xc3\x08\x02\xc1y\x91\x118\x0e\xc7\xe8\x1b\x0c\x8b\x83q\x8a%\x85\xdb\xfc\\\x12\xec1/\x88<\xcdp\x10\xcbs\xb7\x91\x93\xe0\xb8 `[\xc0\xbf~\xbc\x9c\xb6w\x9b\xd4\xcb\x88`\xbb0\x01B\xc0F\xdc\xa8+\xac\x88\xdf\x06z\xf2\x10p Z@H\x1c\x1c\x1e\x88\xe4`\x86\x00\x9b\xcf\xb2\x1cp\x19\x96\'E\x11\x02b~\xfc\x9c\xb6\x07\x8c!@\x04C\xc3h\xcc\xa7!\xdb\x80L\x8a\xc2iP\xa5\xdcYSZ\x85\xfa\x05[\xb7\x06\x04:\xcad\x93N\xb0\xb7\xf2\x02\n2\x17\x8e5\xb9X\xb6;\xdb\xab\xf0\\\xe5\x86 K\x0c\xb6lm\xb1e\x05D\xa7]\xaa\xd4ri\xe53J\xbaQ\xfaq{f\x93\xcd\x81]+\xa4\x0c}\x17\xb2\xed\x0f-\xcc?\xcaT\x87\xca_ea\x16\x15\xcd\xa1-\xcb\x19\xc6\xc9\x88\x915v*7\xeb\xf5\xb8:\xd2\xc3\xa2;\x8b\x08\xc5\x87X\xa5)q\x0f\'\xbe\xb3Be\xdd;k\x07\x813\xd7R\xdc\x8c\xfb\xae\xe8\xe5\xb8\xb4\xa4\x05\x95\x88T\xe8\xf0\xdf\x84l\x032\x81cR\x10y?\xb4\xba\xa3O\x87\xe9\x80\xc1D\xce/\xec\xc5\xf98X\x83\xba\xa0G\xdc\xb5\x96\xce\x8e>\x81\xdarD\x13\xac\\\xe6m~X\xa7\xfd!\x17\xf6\xf4\xe1\xeaxe\xe5\xf4\xcd5\xc5\xc5\xac\xe6q\xd4\x1f\xeax2\xa3\xe5F\xf4\xfc\x84\x7f0]=\r\xd9v\xab

      K\xd1q\xd7wex\x9eYn^\x0b\xac\xfd\xab\x08\xf7\x17g\xb6\xddj\x1b\x12\x06%\x1fr\x07\xc1Y\x0c\x91\x1a\xc4\x82g\x83\x1a\xed\xdc\xd7\x0bM\xd8l1\xe6Z\x0cz\x80r\xc4\xb5j\x14ae\x15~L\xc9\xc2\xd2\xa7\xd3L\xf6\xeb\x19\x8e\x9bxU\x87\xec\xda\x8a\xab\xa1\xa2\xb7G5\xc9\xa7r\xbd\xc8O\xac:\t\x9c\xf7 \xfe\xeai\xcc6\x10@!\x1a\xc4\x10\xec^f\xb9\xda\xca\xdb\xce\xdeQ\x16\x94S\xdeFc+Q6\x85NZ\xb7\xc5\xaa<\x0ei}:\xc9\x84\xa9\xac\xe4>\xcc\x1b\x8c\xe1\x14|\x9f\xf5/\xa88n\\y`\xadE\x10m\xfa\x83\xbc\x9d\xf1".\xcd\xbd\xc3#\x9b\xedx\xe0/j\xe0\xc6\xea\xf5l\x8f\x85}i\xf4\x05%\xee\xc4\xfe\x88\\\n-\xa0\xb6\x08B\xd1\xa9\xbb\xe0\xb9\xd5\xce\xcb\x9d\xf9\x98e8\xbaF\x1e\xa4R>\x8d\xd9\x06\x0e\x0b\x08\x12\x10H\xafw}\xcd\xa8/\xb4eW\xc5V\xe0\xceX\x86\xb0V\xb4\xc4CMW\xce\xce\xf9j\xb7\x07\xe7 \x9a\x08z\x90N\xf2\xda\xd1\t\xe0\rM\xb8$h\xf6rV\x18\x1c\xaeh\xfcT\xc49\x0f\xf7\x1dF\x1f\x90Vg \xd7\xcd\xd2G9\x86O\x83\xb6\x01\x99\x18\x05\xba \xf8\x9e\xd4j\xd4E\x94\xb7\xc8\xc1\xa3\xc5em\xa0\x96\xca\x18\x96\xbc\r\x90\xaaL\xba\x1e\xd9@\x17\xaag\xba\xce\xdbE]\xdaZ\x88\x872\xdca\xbbt.V\x03\xfe\xab]t\x13\xa7a[\x7f\x05\xcf\xe3\xb5\x17\x8f\xca\x02\xfc\xdf\xe8\x97\xcd\xf8_\x1c\xdavk\x9b P#\x12?!]\x8f\x8d\x89SG\x155\xd2\x92<\xd7.\xde\xd0\x85\x14\xcb\xac\x1a\x9e\xa5\xf1\xb8\x9f\xc0\xd1\xf7c\xfahF\xbe\xb46Y\xfc\x18Q\xe4\x9c\xd9\x8a,\xadNC\x9f\xb9\x87m\xb9\x18\xfd\xb5Rod\xad\x93\x02i\xd9\x18\xdb\xf6\xc1&\xf8i\xd0\xb6\xaf\xac\x8f\xd1\x10( \xee\xbaC5\xb5\x17\xb3S\xb0\x8c\xe3\xee\xfbc\x9a\xb4\x07\xd6\xe8\x10M\xb9\xa8\xb5\xd7\x19\xe8\x18\xf6p{\xcc@\x86I\xd7\xab4\x8f\xe5\x8c\xa0N\xc6~\x19\x89r\x12\xcd\x13\x9b\x17\xb0\xb5\x88VFFs8W1\x01$\x16\xc7\x07Y?O\x83\xb6\xdd\xbaC\xe0;\x18u\x9f\x0e\xe3~"\x98r:\x1928$+\x8e\x1b\xc4N;o+{\xbd\x18\xd4\xfex\x0e\x99\xbc8R\xb8\xb1\xdb&\xd9F\xdb\xc5\x9a)\tI$^\xf6[-\xad\'\x93\xdd\xccq\xa9J\xbd\x10^vH\xb1\xe2\n\xefr{>\xfc\x985\x9f\x06m\xbb\xdd\xdc\x90 \xbca\xf7\xd6\x9c/\xb7\xc7~\xfd\xae\xd3\x0e\x96\xb2K\xe9l\xc4\x89\x85\x7f\x8e`\xd1e\xcf\xfe\x81\xddS\x8a\xed\x95\x8a%g\x9afl\xe4k!H G\x0e\x8a\xeeG\xa7h\xa7H\x13[\xc8\xdc\xa8O\x06<\xf0\x19Yg\x93-\x10\x0f\xb6oO\x83\xb6\xdd*r\x9c\xc61\x9a\xb8\xb3f\xb5\xec\xecNBTR\xddk\x1b\xb6\x9ep\xc91.}x6\x86#\xb7*\xd7j0.\xc80D\xeb\xb5it\xb0\x06\x85\xf9\x99!*\xd7\x99\x0f\x8d\x9c]"\xff\xca\xe2k\xaf3Y\x17_\x13\xc1\xf1\xbc\x8e Z\xbc/\xb6r\x89Z\x1b>^\x0b\x9d\x16-5j\x19\n"tp\x96\xd3\xf9\xb0&\x8f\x1e$\xdb\x10\xc7\x86\x86\x96\xb3\x82__\x92\x85\xd6\xd3\xc5n\r\x1f\xe0\x8e\xf3\x9d\xc35\xa5\xdc\xf18\x84\xa6\xdb\xb5\x97\xfd\xc0\xfe\x8a\x1b\xf9\x17\x87\xb6}]\xf3\xa1\x18H\xaaw\x81|\x99A(\n\x85\x112e\xb9\x82A\xb8\x96\xba\xe4\x8e\xc4\xd2\xc6\nl\xdd\x8a\xb6\xc4\xd1>\x1a\xad\x1c\x1aH[\x0b\xa6a\x95\x15\xb3O\x99a)\xa7\x11\x85$\xe9\xad\x1c\xef\x8d0\x0bc#?\x0e\x15\\\xe5\xd2ex0\xeb?\r\xdav;,\x04\x0c\x110q\x97\xf5\xe9\x1d}\xd8[Fzi\x88)\xbevk\xfa\x84\xa5\x19n\xee\\\xd9\xa2\x16\x16T\x17\xf8&\x85YM\x89\xb5\xd3\x06\x1cM\xd3;\xc9U\xa0k\x9b\x95\xc4\xc6\xb9\x1bo\x91H\x9532\xe7\x85d+\x1af \xd9\xc7v\xf9\xab\x07\x8c\xe8\xcfe>\r\xdaF\xfe\r\x1c\x0c\n\xf4\xb5\xf7X\xca\xee\x12]b\n^\x1a\x8ef_}mI,j\x01\n\x0c\x8a\x95\x12\xb9\x85\x15\xac\xdf,\xea\xf9\xe0\x1c)\xa7\xd4\xf5+\x1a\x94\x1c\xe7\x13\x9exA6\xb5\x1d7 \xf0\xad\x82r\x18\xd5\xb9?\xcc$^\xef\xf6\x8e\xa3\xf0\x0f>G}\x1a\xb4\xed\xab"\x071\x84\xbc\x0bp\\sH\x93Fe\xa0\xb5\xaf^t\xf7b\xa1\x90dR|uL\x02i\x13\xda\x86L\x0fs\xecDLe\xf4\xf1\xf2\xba\xb2s\x14\xf4yU\x81\xe0\xcd1\xea\x0eT\xcc,\xb0\xfdNc\xd1v\xcf\xd6\x9b\x93*\xc2Y}y\xaf\xcb\xcc\xa71\xdb\xc0&\xde\xdeC\x06\x93\xc4]\xabO\xf2E\xa1Ek\n\x91M\x918.\xf83\x82\t\x85\x8c\xef\x8c0\xcf\xa3\x90\xddH#E\xf3zU\xe5\xa5zb\xca\xf9\xb0\xa4\xd58\x88\xebAO\xb8\x16w\xac}&wZ\x0b\xf3\x98O\x8f\xe6,#\xdei\xbcJ\xbf\xb5\xfa\xff5\xb4\xed\xebz\xf2\x1f\xa1m\xbf\xbdO\xf0\xc3\xad\xf9\xfd\x95\x9f\x8f\x0e{?\x82\xcd\xd3A\x16?\xf4j\x0fi\x95\x05\x99\xfc\x1e\xd6+\x017\xe6\x04\xd3y\x01\xd3\xdd_\xed\xe6\xfb\xd2C\x80\x98d0\xddd2\xaa\xcdh\xf0\x1b\xcc\x98\x0b\\w\x05\xf4&\xec\xe7`\xad7\xe5\x8b|\xa7Y\x9e\xce\x9f\x01btL\xab\xc2Y\xaf,\x0c\x98\x07\x07\xa6\x99\rW\xc7\xf5y\x83\x99?\x15\xf3\xa6$\x9d/%F\x89\xfd\\\t\xef\xc1_+\x18\xefHs\xef\xeb1FU\x0cF\x15\xa2F\x05\xe4\xbb\xcch\xf2\n\x02\x04]\x8d\x99\x81\xffZ\x1ec\xcc\xca\xa4U\x05Xyq\xd5\xdd\xcd\xd7\xb0Z\xe05#\xf0\x1e\xd8\x08^\xeb1\xcff\xe1\xfc0xf0\\`\x16W\x01g-\x84u>\xb9\x02\xf7G\x8c\xd9\x9a_i\x96\x97(\x81\xb5\xea\xe7Jt\xf4\xb5fy6O\xe7\x87>\xeb\x03\x10\x84\x19\xae\x05\xfe\x00!\x95\x0e\x99\xee\x06\xba\xe5\x9b\x97\x9a\xe5\x15J@~\xf9\x99\x12\x0b7^k\x96\xa73y~\xe8\xee~\xd0\xe7\x04\x01\x02@\x82\x04bf\x01\x01?5\xdf\xc4\xbc\xd2,/PR%\xc8-\x88\xe9s\x08\xf2\tP\xe1\x86\xe0\x0fs+\x01f\xfd\xc5A\xec\xd9\\\x9f[\xb2\x1cL\xbe@A\x056\xe9\x95\x82\x19\xbc>\x99\xee~\x025\xcdk\x83\xd8+\x94\x80\x02\xe6\xe7J\xf4\xc3k\x8b\xe4g\xb3\x81\x80\xb7\x08\xa0H\x06)\xfe\xf6\x89]\xbe\x00\xa5\xa5r#\xba\xe0\xc6\xedC\xad\xafL\xf9\xcfW\xf2\xab\xba\xd2UF\xe3\xb5fy:_\xe8\x16\xc4F\xad\xda\x8f\x86\xbb\x07\x02\n $\x04\xe7K\x07\x91y\x8f\xfc\xbc~ySR\xd2w\x9a\xe5\xe9\x8c\xa2\xdf\x83\xd8O\xc5\xfc\x82\xd5\xfc\xa6\xb4\xa5\xef4\xcb\xd39G?L\x07\x83\xc1w\xdd\xa20l\x82\x0e\x1f$K\x08T\xfd Q\xd6\xbfh\xc2\xde\x94\xd8\xf4k\xbf\xe7A1\xf3b\xb3<\x9b\x95\x04\xc4(\x83\xee\xea \xaf\x84\xb7v\x12\x01"\xc0\xdf \xcf\xb8\xaf\xed[^\xa1$\xbcjU\x82\xebU\x01\xcc\xc1`:\x0f2$\x0fL4\xef\xc7\x17\xf7-O\xe7-\xfd0&\xecj\xb8\xb7To\x81"Y\xc7n\x00\x17\xa3\x12P}\xde\xbe\xd4[\x9e\xaf\x04\x14/\xb8V\xfd\xbcxy\xb1Y\x9e\xcel\xfa\xa1O \x88\xcd{\xc8\xe4A\xaf\x02\xce\x17p\xffQ\x9fASV\xed^j\x96\x17(\xf9>\xb3<\x9d\xfb\xf4\xc3\xe0\xb0\xf1\'\x97/\xd0\xab\xbd\xe5\xf9J@V\xbc\x85c\xa0\xc0\xba\xdd\xea\x8d\xbakM:\x0f\xfa\x97Y\x99~\x1e\x8e\xdf\x97qu3\x0b\nj\x17\x10\x91o\xcd\x17\x88\xcc3\x08 \xa0O\xd0\xdd\xf4\xb5fy\xbe\x92\xb9\x18\x8c\xdb<\xc3\x19d.\xa0\x06\x94\x00\x88Q%\xf0\xd7\x84\x90\xd7\x9a\xe5\xe9\xfc\xa9_\x051\x14\xa4\xda\x17\x07\xb1g+1fa\xf8I\x96\x84n\x87\xed\xd5\xde\xf2l\x86\xd5\x0f\xfd\xf6\x94\x82\xdf\\M~\x8f\x19\xeeWq\x0c\xce\x99\x00B\xc1\xea\xb5\xde\xf2|%\xb3\xf2u_q\x1b\xa7\x0e\x8ad\xe01\x16\x0c\x0e\x1b\xc8\x92\xcc\xab\x83\xd8\xff\xb1\xbc\xae_T\x8c\x0c8R\xbf\xca\x81\xefK\xf4\xfa\xa1s\xd8t\xabLLw\x03J\xde\x025f\x1d7\xf9\x02\xc4[\xf6\xa5\xbe\xf0\x02%%\x86\xdcr\xa0\xe1&3\xf0\x01\xa0f\x8f\x99\xb7\x1e\xeb\xf5\xcf\x1e\x9fN\xdb\xfa2\x0b0\x07\x02\x1c\xfc\xf6\x9c\x1b\x9cP\x0b$\xd5\r\xc8"\xf6kC\xd4\xf3\x95\x80`\xfb\x93K"P1\xbe\xde,\xcf&v\xdd\xccr5oE\xe3\xcc\xdc:\xde\t\xd4[\xb3q{\xb0Z\xbd\xd8,\xcfW\xf2}fy:\xf5\xebf\x16\xd0\x95$\x13\xa8R\x80\xa7$\xb0\xe12 h\x80\x94\xc8\xbf8\x88\xbdB\xc9\xed\xc9\xfc\xf4\xf5\xacq\xde\x83d\xbe\x07\xb9\x05T\x90\xee\xcbs\xcb\xd3\xc9a_f\xb9u\xbf\xbf=s\xfcz\xc3\x0ezk\x1aM\xfe\xb5=\xfcK\x94\xfc$\x1c\x83\x8c5\xff\xea\x80\xbd/%\xed7\xb3\xdc\xde\xac\x03\xda\xab\xdf.$\xf6\xd0o\xefkym\xb3\xf8\x12%\xdfe\x96\xa7\x13\xcc\xbe\xcd,/Q\xf2]fy:\x05\xed\xdb\xcc\xf2\x12%\xdfe\x96\xa7\x93\xd4\xbe-\xb7\xbcD\xc9w\x99\xe5\xe94\xb6o\xf3\x96\x97(\xf9.\xb3<\x9d\xe8\xf6m\xde\xf2\x12%\xdfe\x96\xa7S\xe1\xbe\xcd[^\xa2\xe4\xdb\n\xe4g\x93\xe5\xbe\xaf@~\x85\x92oK\xf9\xcf\xa6\xd3}\x9bY^\xa2\xe4\xdb\x82\xd8\xb3\tw\xdf\x96[^\xa2\xe4\xdb\xcc\xf2lJ\xde\xf7\x99\xe5\x15J\xbe\xcd,\xcf&\xed}_\xca\x7f\x85\x92\xefk\'\x9fL\xeb\xfb\xbe\xdc\xf2\n%\xdfe\x96\xa7\x13\xff\xbe-\x88\xbdD\xc9\xb7\xdd\x89=\x9b\x1a\xf8}wb\xafP\xf2mA\xec\xd9\xe4\xc1\xef\x0bb\xafP\xf2m\xde\xf2lz\xe1\xb7\x05\xb1\x97(y\xd0,\xff\x1e\x93\x0cz\x88I6\xff\xd3\xa2\xdf\x9a\xd4\xf8ra\xdfEj|\xb9\xb0\xef"5\xbe\\\xd8w\x91\x1a\x1f\x13\xf6op\xff\xbe\x8b\xd4\xf8r\x8b}\x17\xa9\xf1\xf5\xc1\xe3\xbf\x85\xd4\xf8\xdb\xff\xf8?\x8a\x8c\x88\xff\xe7\xa6\xff9\x19\x91Ai\x9c\xc4D\x84\'\x19\x16\'Y\x88\xc28\x96\xa5a\x11"0Q\x84)\x98\xe0H\x0e\x16\x11\x8e$nhD\x9e\x818\xf0u\xf0\x9d0ER\x08F\xdc`\x04\xbfF~!0. \xb7a\xc6\x14\x8db8\xc3\xf04\xc2p\x0c\x8e@\x02\x0b\xdf\xd8\x10\x04C\x12\x04\x85\xf1,\x8b\x90\x1c\x82c\x94@`(\xc3\x8b\x18\x8f\xb0\x02\x03\xb1/%#\xfe}\x06\xef\x8f\x9fL\x1d\xc3\xd1\xbf\x9180\x01\x86\xe1\x7fJF|\x7f\xae\xe4O\xc8\x88\xc4\x8d\x84H\xa3_\xf3\ni\xf8f\x18\xf0\xd24\x89#\x14"p\x8c(\xa0\xa4\x08\x0e\'\x8c0\x0c\'\xf2\xb8\x00\x8e\x83\xc0\x03)\x1c\x10H\xdf\xf8\x83?\xfe\x1b\xc8\x88O\xe0\x17\xfee\xc9\x88,\x0e\xe1\x10\x8c\xd30\xc3\xdcV)@\x14\x0cc\x028]<\xc7#\x14#\xe0\x14\xc9R\xc0\x93\x80\xc5Q\x04\xb8\x13D\x82m\xc5(\x92Gi\x8cE\t\xe1\xc7\xcf\xc9\x88O\xb0\xd3\x7f\x0f\x19\xf1_\xc6\xf5=\x8f\x8cx\x9b\xe6\xf7\xa7d\xc4\xf7w\xf4\xef%#R\x7f\x82aZ\xb4\x14bz;\xcf.C\x18iY\xf6\x08\xed\x97\x87\xe6\x14\x17\xa7,\x85f\xba\xdcl0\xa6kv\xfb\xa5\xddk\x97\x8b\xab%\xe9@\x81~\xfe:\x98Cz\xddi\x16\xe6P\xa7cu8/\x8c\x1c\xd2\xcc-u\xda\xa2\xdcc\xf3Y\x9fEF\xbc\xa5\x05\n\xc5a\xe0\x9ew\xc3|\xe1@8\xaef;8\xcb\xa6\x8eP\xfbRs\x17\xd7=A\xf8\x9b\x86o\xb5,\x1bv=\xbatN\xc2F\x16\xd1\\?o\xfc&ZS\x11?\x8c\x8e\xd5\x8eV\x88\xda\xbbD\xda\xb4>\xac\xe5\xfe\xd4\xed\xd4>sI\xac{t\x0c\xed\x93\xc8\x887\x99`o\xee\x08"\x14\x07\x1d\x9d\x85\xc4.[{\xeeze\x13\xe3\'\xb2\xc3\xabEp:\xc4K\x10\x9bHw\x10\xd5k\xa7\xd1\xb2R\xe1\x12\x0e\x9fe}1^\xc9Cw\x1c\xf5\xf4\xe2f\xc2v\xf4V\x90N\xf5\t\x83Rra\xc8\x05o>8\x9b\xf5Y`D \x91\xa20\x12\'\xa9{\x8c\x00\xba\xcc\x956/\xb9s\x0f\x91\xb2\xe8\x84\x84i\xe4\x0e6\x0e\x9c\xdeL\x08\x93\xc1\xf2\xb0\x91\xe4\x15U\xa7\xfdb+P\xe5\xd1\x86J\xaa]\xd6\x8c\xcaM1&U8\x9b%\x12\xe3B4{]\xf0\x86\xbf\xb8.\r\x1f\x1c\xb3\xb7\x1a\xce\xfa$0\xe2m\x17\t\x88\xa0I\xe2\x8e\x87JP\x03\xe2\xb9YY\x9a\x0eF\x06\xb2t)L\xaa\xd4!\x90P)\xb9\x13\xce\x8ca\xfb\x1c]8*S+^\x12\xa1\xb1\x8a\x8bH\xd1\x91\xec\x85nBdOJ\xb8\xa1N[C1\x9d\x9d1, \xb8\x83\xa4\xd6\xfe\xd5l\xeb\x17s\x11\xbf\xbc\x1e\x14H\xf4OFx\xab\x97\xa1\xf0m\xe9\xa0\xbb\x18F(\xed!\xa6\xa7h\xe9\x99u\x98"\xd1\xd1M\x8eJ\x1a\x9b\x8aFDG\xcaY\x11\x914\xa5\xe3|\x85/\xac\x86"\x16\x9en\xe0\x1c\xd1\x92\x98/c\xd9.\xbc\xc6/cL\xd5\x12\xe1a\xd0\xd4s\xc0\x88_^\x8f\x81\x93F\xc0w\x9e\xbf\x1a\xfa\xe0\xac\n\xa4ga\xe8\x8aCaZ\x98\x8e\xc19:\x97\xc3\x11\x15\xaf\x94X\x12y\x83\x99\xab\xd4\xef\xdd\x93\xbcW\xf5\xd6\xdb`\xbc\x8a\x18KX\xdd\xf4\xe1\x04\xd5\xb3\xc5\xaf\xf7\xbe%\xe7\xbc\xd7\x16\xe2\xde;\xae\x83G\xa1:O\x02#\xde\xce,Ba0|\xc7}u,m\x15s]s\x98\\5\x84w\x0e%\xees\x1e\xd5\xc2m\x1emv\xc5\x95\xbch\xc7\xa4\n\x8f\xa7\x8b}^\x1a\xe7=\x910\x17\xcc\r\xcf\x07Z!\xad1\xe6C\xcd\xbeHH\x9c\xe3\x17.\xf6\xa9\xf3\xd4\xacDK|\xd0\x96\xcf\xc2"\xdel\x89\xe1\x10\x82\xa2\xf8\xdd\xec\xe9\x8b)\xee\xb2\xab\xd3{Q\x7f \xc5f2\x80\x9e\x85\x9eg~\x8f\xab\xc1\xd5j\xdbK2S\xe2BOu=\xa6\xa1k\x1dpJ:AW\xaa5\xeb&\xa8\xcfC\xe1\x9e\xd7=\xa8~\xbd\x8c\xd1XH%gvuz\x90!\xf0,,\xe2\x1f\xdb\xb4\x7f\x82\xc1pI\xde\x87\x82,\xcf\xd4\x92"#+\xc9)\x95\x80\x85\xe6Pjrrr\x8c3];\x87\x15\xe7\xabi\xc2\xefv\xfcY\x8fw\x94C\xad\xd4a\x03\xed\xad]8\xe1\xeb.\xad$vG\x1b\x0bt\x7f\x19\x82\xcaC\xcc_\xe1m_\x8cE\xbc\xc9\xa41\x02\xf4\x01\xd8]\x9c\xed\xfc9\xdc\x85\x07\x17\xde\xed\x0e\xdcI\x11+\xe0\x99\xa7y\xc9\xa4\xd5\x1eToFL\x9dxJE\xfa\x15ji\x17\xb3\'F\x9a\xe3\x03ay\xe8\x13\xcd\xd9\xa5s\x00\x15\x1d\x97\xca\xe5\x01\xe3\xd3\x83T\x17\x95\xb7\xa6\xcb\xf6WT\xaf\xbf6\x16\xf1+\x8c\xd3$FA\xc8\xdda\xc9@?F8R\xcc\x08\x07\x95\x1dL/\x13\xe1\x8c\x0b\xd6\xd8)\xa8U\x1e\x85\xa4~\x90Ca!\x13\xe25bUlj!)\x98{\xd8\xef\xb3\xeb>\xcd\xa9\x08\x9e\xf1\xfdE\xf0\xa4\xcbm\xce\xfe\xf1\x025\x02=\xf6\x0f\xc6\xb7ga\x11\xbf|\x82DA7\x8b\xde\x1d\x96 \xb4\x86}]\xad\xecVPv\xb8\x90\xc0\xc1\x84\xec\x97\xc7\xe5\xba;ljd""+\xe0\xae\x10\x97t\xfa\x1aU4\x0c/\xfdM\xa9\x9e\x11\xaf]\xa2%d\x97IB_\xc8*Nl\xdf\x8f\xa5 \x08w\x98\xa45\x0fr\xa0\x9e\x85E\xbc\x85q\xe0\xf8\xe0\x87\xee#\x9c\x88iq\xef\xd9Bq-\xd5k\xa0\xb2$t\x04=M\xbe;\\\xc4\x93C\xa2Z\xb9\xe1`\xe6\xb8m\xe9]4E\xdc6\xaaL?\xa3\x8c\x96\xda\x8b\xb1\xc7,\xcf6\xbb\xb6\xaf\xf5qq\x11\xdb,\xe7\xf3Zu\xe1c\xf3^\x05\xdc\xb3\xb0\x88\xb7]\x04\xdb\x8e\x900~7]\x7f\xd7`\x12\x0cq\xf0&G+\x0bYq\x9c&\xaf\x8d\xb3u\x99\xf8\xb6T\x07\xc8\xdfX\\;f\x12\x9f\xd5\x07\xf2\x14\x11\xd7\xaa]\x9b\xed\xcc\xf7\xea\x11w8\x07]\xea=\x06\x1d4/\x1f3\xc2V\xe7\x98\x9bT\xa8x\xb0o{\x16\x16\xf1+\x80B\xd4-}\xde\xc9\xdc\xce3\xc4aZt\xe1\xe7\x0cH^\x15\xe2V]\xaf\x05O\x88P+Hx\xc8V\xce\x9e\xaf\xd0\xedju\xd6\xb5\xac\xc3Zu\xc7\x1e\'?\x85\xac\tg\'\xfe\x02\xad}\xf9\xc0\xd8\'\xbc0\x8ego\xa3\xeb\xbbc\xf0 9\xe8YX\xc4\x9bL\x12\xa7q\xe2\xbe\t\xdfk\x14\n7\xbe9]\x96\x81\xc3\xca\xda\x90\xb6%\xe8\xdd\x16\xce\x8a\x93\xfd\x85G\xac\xbcyb\xd0jU\x8f\x06)\x9b"\x94_R\xb5\xf58xMTv)\xc4[\xbc\xa0`\xdc\xb5i)\x87=\x13\xa3Lz\x9b.\x1eT\xf9,*\xe2\x97\xe7c\x08\x8aQ\xe4]\x80\xd3\xe6 \x1c\xd9N\xcaW9{\x986\x9dT\xa0tbw\xc3\x1aF\x96\xdb\xabF\xca=\xdf\n\xa3\xd7J\xf1\xb9[%\xd2V\x9f\xdbS{L/\x9as,\x16;\xc1"\xc8\xee:\x97V\xe4Mf\x84\xef.\'TG\x8a\x07k\x9bgQ\x11\xbf\xcaqP\xd8\x93\x18r\xcfG\xea\xd8Nf\x96\xc2y\xc1.\xfc,L\xa0@6\x13\xb6\x84\xa4\x9dc\xe8\xb9G\xb5]\xd2\x16\xed\xee\xb8@.C9\xb5q\xe00{GP\xc5\xcd(\xee\xb7\xda\x82(\xe3\x1cq\xe8\x99\xae\x06v\xe3c\xa5MX\x92\xf4\xab\xda\xe6\xafME\xfc\xda\xc5\x1bg\n\xa5\xef\xea}X\xa8B\x85\xce\xb2\xacw-\xc2F\xe7kz\x99\x1d\xbb!\xedq]\x9c\xf39p\x8c\x11w\r\x19\n\xbc\x85=\xe6\xdb\x8b,O\xb0\x8a5\x89zY\xe2\x06\xbeh\x898bY\xfch_\xc6\xaa\\\xbaG\x15\xe5\xb7\xc2\x83h\xb4gQ\x11\xbfd\x92\x14\x01\xc2\xe2\x1dQGa\xb3\x1dz\xb4Mu!\x84{\x01\x13\x0c\xbe]I\x9e\xd9\xb4l\xb3Ct\x0b\x96\x97\xb8\xddK\xc8*\xe9\xaf\xb8\xb5\xdd\xee\xb6\xa7rN\xed\xbc\xf0\x06\x10`\xa7u\x167\x103\xe7\xb0\x99e\x94\xce\xed\xed$`\x0f\xec\x832\x9fEE\xfc\xbao\x80\xe1\xaf\xeb\xf6\xbb\xb6\xc6J\x18\xbc\xb6\xeb\xbe\xa8}Vu\x14\xf2(\xe9VW\xc7\xe7\x91\x1a\x19\xf9\xac\xee8\xcd*\xc8Zd\x1d\xc5S\x1c\xda\xe2\x0cV-\x17\xa8}v\xac\x89G\x94">\n\x9ceL\xe6\xba\xbe\xa8z\xb2Z\x95\x8b>ZR\xbb\x91)\x17\xd71v\xbc\xa2*O\xba\xb1\x92\xb3\xfa\x981\x95f\x1aU\t\xebWe\xd9\x93\xa0\x05\x97W\x99&\xfb\xbd\x18\x16C\xba\xc4d\xdd\xe2P\xa3]\xec\x1ft\xfdgQ\x11o\xaeO\xd30\xc8\xa0\xf7\x95\xea9\xccF~Y]\xb0\xdd\x12\xd4c\xe3\x80\xcf\xdc\xb0\xdc\r\xf3\x8e\xa8\xf4m\x7f\xceT\x01\xca\x849\xdee*\xac\x0f8ML}\x1a\xaf]X\xb6\xd3$YI\x95\x8f\x90P\xea_a\x9al\x8e\x0b\x19=\xe1Ua\x8d\x0fZ\xf3YT\xc4\xaf&\x15\xa2\t\n\xb9sM\xb6\tEeqZ\x9d\xcec\xed5\xd9\t\xa9w-E \xbb\x88\'\x91 \xa7\xf8}\xd6\x08xg\x9c\xc7b\x8bn\xf9\x03\xdc\x9fb\x85\x10\xf3\x03L\xc5>\x95CM\xb4s\xc2\xe2\x00\n\x11\xa1S\xfb\x0em\x08\xcf,\x1e4\xe6\xb3\xa0\x887\x957\xa6=\x8e\xdd\x97p\x13k\x10\x07oQ\xd6J\xa9f\xd9\xde\x8c\t\xfa\xbc\x16\xe2b\x19\rWN\x0c\t~\xc6\xcd\xe9\xc0\'B\xc4\xee\xc4N\x9eX\x0c\xf7X\'\xa6=\x04K|\xab\xed\x17\xdc\xda!\x03G\xde\x93t\xc6\xc7;\xd7p\xbd\xf4\xbd\xe2\xf8\xb3\xa0\x88_\x9e\x0fa7\x1c\xea\xdd\x85mD\xc3\xd6b\xae=P\x19n\xba\xcb\xd6NTK\x162\xa1\xa9\x82\x14\x91V\xcc\xc9 b\x08\xc6&\xb4\xb1\x0b\x81\xc0\xce\x16\xde^\xa0\xbc\x9f\x8b(Y\xc1\xd7)a\x90\ryM$\xa4X\x95l\xe83\x056E\x9b\xfc\x9b\xa0\x88_I\x1f\x14\x81\x04\x8d\xdey\xfe`\xc6\xd1e\xd1\xc6\x18\x7f\xc5\xcd\xea\xa4C\xbe\x13\xa0\x1b^\x85\xd5sm\xaf\xc4ht\xdb\xb4c\xec~\x1a\x12?WT\x84S\xf9\xd8\xafi\x98QQ\xb9o\xca\x82\xcd\xd6N[F4\xc6\r\xb8X\t\x1d\x15\xf8\xf2\x83=\xea\xb3\xa0\x88_=*M\x82o\xa5\xeed\x96W\x84\xbb\xc6*\xbc_\xc0\xe3~\xbf\x88 %c\xdbC\x8b{\x81;6=\xb1\x9b\x97\x19\x17\x87\xa1\xd1nz\xbe\x82@\'\xc5\xb5\xa1\xbc\xa6\xd5}H\x85\xc4BY\x0c\x89^i\xd2@\x8d\xeb\x99\xce\xc2\xeb\xd1\x15H\xe3\xc1{\x9bgA\x11\xbfn\x1c@E\x0e\xca\x8f\xbb\x82\xbc\xe3\xd4\xfd\xaa\xee\xdc\x18t6\x1b\xb5#v\xd5\xc6$Kj\xb5\x99Z\xe8\xb4g\x1bYD\xec\x92\x94\xb8<\xc6q?\\\xd0D\x9b\xf0m{\xd2\x1a\xf3\x82\x1c\xae\x91]\xc38.@\x87ma\xe3W\xd3)W\xd1\x014\x11ouo\xf3,*\xe2\xd7.\xde\xdes\x00\x13w\xf5\xbe\x88\x90\xfe\x91\xc6d\xa6\xa7"\x8a\x9a \x98\xf30\x1dZR\x9b\xccfi\xe3\xb0\x15\x8f\xfa^1\xed\xfdp\x14\x07*\x9b*\xdc"\x88h?4\xc8%\xf3\xf5M\xc76\xda\x12\x95X\x988\x1a\xa7\xf8\x0cw\x8a\x12\x19\xc2\xbfJE\xfc\xaa\xe0?T\xc4\x7fy\xe0\xe4\xff\x1c\x16\xe2\xd3\xe7\xf2\xde\xed\xd5\xfb\x0eC~\xd1\x1c\xd1\xbf\xd2\x96>\x9b\xed\xf7\xaa-}\xf9:\xdf\x97.\xf8\xa2-}\xfd:\xdf\x97\x0c\xf8\xaaS\xfa\xf2u\xbe/\xd5\xefU[\xfa\xf2u\xbe/\x91\xefE[\xfa\xfau\xbe/M\xefU\xa7\xf4\xe5\xeb|_\x12\xde\xab\xd2\xd3\xcb\xd7\xf9\xbe\x14\xbb\x17m\xe9\xeb\xd7\xf9\xbe\x04\xbaW!\x03^\xbe\xce\xf7\xa5\xc7\xbd\xca\xf1_\xbe\xce\xf7%\xbf\xbdjK_\xbe\xce\xf7\xa5\xb6\xbd\xaa\x88z\xf9:\xdf\x97\xb8\xf6\xa2-}\xfd:\xdf\x97\x96\xf6\xb2\xba\xf4\xd5\xeb\xfc\\\xee\xbd\xef\xe5\xde\xd3\xd9^/s\xfcW\xaf\xf3}\tc\xaf:\xa5/_\xe7\x87\x0e\xf6\xf4u\xbe/\xfb\xebE[\xfa\xfau\xbe/\xb7\xebe\x0fJ^\xbd\xce\xf7en\xbdjK_\xbe\xce\xf7\xe5e\xbd\xca\xf1_\xbe\xce\xf7e]\xbd\xecA\xc9\xab\xd7\xf9\xbe\x9c\xaa\x97]\xee\xbdz\x9d\xef\xcb\x98z\xd1\x96\xbe~\x9d\xef\xcb\x87z\xd5M\xd4\xcb\xd7\xf9\xbel\xa7W=(y\xf9:\xdf\x97\xcb\xf4\xb2R\xff\xd5\xeb|_\xa6\xd2\x8b\xb6\xf4\xf5\xeb|_\x1e\xd2\x8b\xb6\xf4\xf5\xeb|_\x96\xd1\xab\x8a\xa8\x97\xaf\xf3}9D\xafJO/_\xe7\xfb2\x84^\xe5\xf8/_\xe7\xfb\xf2\x7f^\xb5\xa5/_\xe7\xfb\xb2{^\xb5\xa5/_\xe7\xfbrw^V\xea\xbfz\x9d\xef\xcb\xccy\xd5\xbbM^\xbe\xce\xf7\xe5\xdd\xbc\xaa\xc7\x7f\xf9:\xdf\x97U\xf3\xb27C\xbez\x9d\xef\xcb\x99y\xd9\x1b\xcb_\xbd\xce\x7f\x8f\x86\x00?DC\x80\xfeiIo\xcd\x88y\xb9\xb0\xefb\xc4\xbc\\\xd8w1b^.\xec\xbb\x181\x8f\t\xfb7\x88#\xdf\xc5\x88y\xb9\xc5\xbe\x8b\x11\xf3\xfa\xe0\xf1a\xc4\xbc\x86\x11C\xfc\xe7\xa6\xff9#\x86d\x11\x1afP\x82\x14h\x18\x83Q\x9cD(Hd1\\$H\x8c p\x91\xe31\x04\xe2\x10\x8caP\x0e\xba\xe1-`\x82\xc7)\x14\xb9\x114(\x82C\x7f\xfc\x19\xfc@\xa4)\x81\xa61\x98\x14E\x0c!y\x11"\xb8\x1by\x80`i\x1c\x16`\x1al\x0f{\xc3]\xf0\x10"\xf0\x02\x02a"q\x9bl-"\x04\xcc\t0\xc1\xa2/e\xc4\xfc}\xa8\xec\x8f\xbb\x19\x0c\xf8\xff\x82\xa0\xbf!\x18L`\x14\xf1\xdb\xcc\xb5_1b\xde\x9f\xb0\xf3\x13F\x0c\x8c\xc0\xac\xc8\x88\x10.\x08\xc2\x8d\x9c@\xf0$\x81\x884\x04s\x10\xca\xf2\x0c#\x88\x04\x8e\xe3,C1\xf4m\xc4\t\x0b\x81u@\x10\x8f\xf04u\x83\xaa\xdcL\xfea\xc4\xbc\x90\x11C\xa38p\x11A\x84q\x1a\x17oc\x83P\n\xbb\xcd\xc8!\x11\x84&\x18\x08\x05\xbbI\xa3e\x16\xb9\xb4s\xe9L\xcb\x9b7\x9bU\xf5\x14H\xcc\xd7.\xa2\x10\x8e\x83\x9d\xfc\xe3\x1e\xf6\xfa\xfe\xd0\x11[F\x92\x133\x0c\xdd\x98\xb7\x12\x83\xc5\xae\x95\xc7j\x08\xe2\xe6qSns\xce7hE\xb1:\xf3\xe2q\xd4&\x12\x0eu4\x8fN\x00\xd2\xc5\x9cK\x97\xbd\xa8\x1b\xae\x199v\xd3\xb9S\xbe5\xc9\xeaW\xec\x82\x972b~\xf7z\xe0\xf30N\xde\xf1\x84\x8aQ(\xd7\xad\x13\xb2U&t1\x1e^\x84\x8d*\xe1\xdaQ\x8f{y\x894](\xf4\xa6\xa0\xb0\xa7\xe8\xb4H\x18\xa2\xc5$U\x91j\xc34X\x82d\x8a\x08\xdb\x94\x83E\xc8\x81"/Q\x85\x8e*i8J\xc1\x8a\x7ft\xea\xfe3\x181\xbf\xc9\x840\x82\xbc\xcd0\xfb\xa3\xcc+s\x99\xd0\x1b\x1d\xc6\xe9\xb8vw\xbc\xacv\xa8\xe5nzdf\xd7\\\xdf\x1f\x12\x16\x1aUfg\xaf\x96\xa6\xe2\xcb\x07*\xdfN%\x97@\x11m\xc2E\xc9y\xa5\xb4\xb7\xbd)\x19\xe2vK\xd8\xcd\t\xa3)r\xbb\xbd\xf2\xdf\xc1\x88\xf9\xed\xc8\xc2$JA\xe4\xfd\x90\xba\x9e0\xe3\xf4\x9aG~\x15:V\x9b\xb6y\xe2R=C\xb8\xd4\x91\x95Oh\x8f\np\x89B\xfa\xecK\xfdv\xdd\xdb6znP\xabaz\x93\\\xeaK\xb3\xf7\x9d\x12s\xd7A\xe8@\xa9`#\x81\xa4\xefQ\xfb4?\x18\xc3\x9fC\x89\xf9]&(\x19)\xfc\x9e\xf83\xf4\xf0\xc4\x1aN\x93x\x14.\x1dGykFd\xb4\xbf.\xa0-\x9c\xc5\xfd\xc0+\x11\xde%x#\x8c\xb0\x91\x1a\xb4\xeaUs\xa2\x10\xd7u.\xeb\x9d\x0c\xb7hj!\x17\x1f\x9d\xb1qG\x14\x83X\xd4\xfdi\xd9\xffr\xc4\xf8K)1\xbf\xfb\xe6\xad\xca\x02y\xf9\x8f2\xf3}\xa2\xcc^\xbe\xcb\x8bi\n\x9c\x9aA\xd6\xcc"\xcf\xaf\x02\xce\xf4W&\xd5\x19h\xc3\x9fMu\xc5\x98[&\x1f\x9dZ\xc5l\xea@\x1d\xc8\xc2\xf3V\xb9\xb0Y+\n\xca1\xc4\x91K6\x1a\xe6@\noz\xd6tx0[=\x87\x12\xf3\x9b\xcc\xdbP|\x1c\'\xee\n\x0f%%\x87\xa61Tb5\x95\x93b\xacc\x97\x98\xd7\xe9)\xebfIA\x16\x86-G\xdb\xda\x99U\x0f\xd5\xfc\x839\xae\xd6\xed\\\x8b\xb5\xed\xb6cB*\xae,.\x82\xca\x0b\x18\xbeGp\x91\xd7"\x8f\xd7\xf13\x7f\xfeU\x08\xfa+Sb~\xf3\t\nFq\x08\x83\xee\x0e\x0b\xe7\xce\xd8\xde\x96\x19\x1d\xfaChT\x1a\x94_\xb8\x96\xd5\x0e\x89o(\x17\xdcL*)\xd2\x9a\xb1\xca\xe2\x8bz\xdan\xab|\xb1\xae@\xa9\xbd\x1d\x9aBJ\x903\x91[\x90\xd7\'\x10.q\xf5\x86\xd2\xd5M\xbf\xf3#\xb2\xd2\x1e\x84(<\x87\x12\xf3\x9b5\x11\x9c\x02\x8e\x04\xdf\xe5\xab\x94\xda\x89\xcb\x95J\xf1\xce\x16&7\x9cAZDbv\xb0J\xda\x13:\x8ck\xd5U{\x9b\x1b\xbb\x03\x1c\xady\xdcYn\x84\xe9\x92\xa3\x82\x96\x065\xb9\xa4\xd9\rw\x81D\xb8\x8b\xe8\xdb}\x00\xec\xc8\xa2\xa47\xc1\xf1WS\xa3\xbf\xc7\'\x9e\x83\xa2\xf8=\xb2\x10\x04\xf8\xd9{h\xdaq\x1b\xa4\xf49M\xbc\xba\xe8G\xcad1\xc9\x88\xb6gH\xb9\xba\xeb\xb8\x1a\xcf{j\xad)\xaa\xe9$\xfb\xb4\xc4\x16\x1bsB\xb9\xad.0q\x97]\x92\xe8\xda\x18\x1d\x9fo\xe4\xaa\x95P\xeb\xa0%\xe5u=m\xa8\xa2~\xb0\x86{\x0e\x0c\xe7\xf7\x96\x06$\n\x1a\xb4\xd9w\xc3\xb7\xa1\x14\xe7\xe3\x9a\xbe,x\xa3\xc0\x8e\x83<4\xd6\x84Z\x90\xb6\xa6\x8fd\xa9\xe5\xce\xa9"\xa8\xa3\xa5\xda\xfabr\xe8\xa0q\xb6b\xb1q\xbc\xca\xea\xcd\xd10\xfcn5\xcfW\xff\xe4\xaew!\xcc\x19vG\xd4-\xc2~\x0b\x0c\xe7\xf7z\x1c# \x88\xbe\x9f\x1d\x8b\x1diga\xcb\x87SXS9\xe2F\xcd!,\xec\xc0\x99;\r2&X\x9cO|\xbe?\xf3G\xd9o\x8b\xbe\'K\xe1L\n\xe2xdH\x8d\x86!\xdb\x9c\xa0\xd82\xcf\xd9\xa1[\xd2\xd0~\x11\xf8\x12\xbf-\x97\xe4\x83\xe9\xf090\x9c\xdfe\x92(\xf8\xfa\xfd\xc8{$\x1f\xf8M\r\xedN\xcb\x8bs`W\xe4\x96e)\xa9[]w,d`>N6\xed\xb5U&8\xc30\'\xa7\x1c\x1bR2\xcb\x8e\xeb_\x01\xa2^J\xc3\xf9\xcd7)\x1cT\xe4\xf7\x15\x9c@4k\x16]\x99\xcb\x01\xe9;\xf8\x1a\xef\xed+\x82\x0b\x85\x93\x8b\xcd\xb1r6g\xf0\x8aW\x83;\x1bMf\x04\xb7\xcfy\xa6\xfbH\x13wy\xa3&x\xc8;\xc8\xb8&\xf5|H5Y\xd0\xcav\xe2R\xf9\x02G\xbfB\xfe\xa0\xbf\xb0%\xc5R8$\xa08\xc8\x91\x08\x02\xe14E\xd1\xe0o\x92Gns\xb2o@e\xd0H\x10\xc4\x7f-\x12\xfb\x1b\x90\x88\xe1 \x06\xdd\x1dYI\xc9r7]\x11\xdcqI\x0c\x0e\xc3\xd5a\xb2\xc0Dd\xd1\xf5\xfeE\xdaw\xaey5x\x87\xe4\xdbt\x88\xd0\x96t;\xee*\x8a\x15\x1c\xc4\xb4\xbe\xde\xb8\xd2\xa6\x9aHy\x1f(\xae\xbf\xea\xb6\xa5\n\x8f\x88\xef\x13\xc9\x83\xdd\xd5s`5\xbf\xdb\xf2F\x99\x86\x89;\xd0(N\xc7\x06\xa5\x05\xb9_\xad\xfc\xc8\xa4d\x08\xf2\x1cj\xd2\xb7\xb0\x8br\xe9@V\xad\xb1\xdc\xb9\xe8>&s\x98ezyC\x9d\xe0\x85\x88R\x14+-:.\xee\x8d\xddi\xb7\xbc\xce\xfai\xb1\xd1\xaf\xb1\x8bh\x969\x04\x0f&\xe5\xe7\xc0j~\x93\t\xc3$M\x10\xf0\x9d5\x8f:\xbb]r\x1a\x15\xb8\xd4\x80k\x17\x04>\xb2Yk\xa4Q\xe7\x85\xe3iG\xa2f\x15{\xf8\xaaP\xfc\x06\nY&\xb5Q\x8b\x98\x0b\t[I\xa0\xdf\xb2\xf9\x8d\xb0Q\xdc\xcc\xe8\xd1\xe0\xec\x1a\xdd6\xd6\x16%Z\x9f\x1f\x9ct\xfe\x1cX\xcdo2\xc1Na(|\x8f\x18n\x89\xd5\xf2Xn\x87\xa5\x9a.#\xf7j\x04\xc1!\xdd\xaa;\xfdL\xac\xa3\xde,\xc9\xd90\xdd\xa5\xa0\x17\x07vD\txM\xf6\x1cX\xcd\xef\x05\xf3\x8d\xb1D`w}\x81$qgh\x87\xb2\x17A\xf4\x86\x19\xdd\x1c\x96\x1e}v\xd3\x9e\x9d\xec\x06\xf4\x87\xabi\xb0\x143*\x8d(\xd8\x13\xce\xba\xbct\x1e\x9a\xa3S\x7f\xf5k\x18\xac\xc8\x8f<~\xceHwK\xc6\x96r\xdeV\xe6N\x1d\xe4\xf8\xc1|\xf5\x1cZ\xcd\xdf\xd32\x8e\xc3?\xb9X\xc1\xd5\xa3>,\xf1\xee\xe8\xa7Wa\x195\xe4\xb0\xab\xd5yC\x8am\x83X\x1c\xad\xf9\xe7\x03~q\xf0xw\xe2=yw\xc4\xcc\xe3f\xad\xa6;3^^\x8cE\xaa\xbaH\x03;M)\xa3\xed\xbe\xf6-\x03"\xf3\xc3\xb6\xf9U\x91\xf5W\xa6\xd5\xfc\xee\xfa\x14\x8d\xe1\xa0\xf9\xb8\xc36\xb66\x86^\x18k\x01\xe5\xe7\x99\x98s=\x97\xf0\xf2d\x12bW$\x82MPg\x8e\x90Q\x84J\xa30B\xb5y\x81\xfb\xdc%T\x97\xc7t>]\xe0\x1a\xf8zH\'\x92\x1b3\xfe|\xe0\x82\x1a\xe9dUc\xa5o\xa1\xd5\xfc\xe6\xfa8\x04\xda!\xe4\xde\'\x04H\x83\xd9j\xe7\xae\xf5M\x96{\x17\xd6CO\x88\xbe\x13{5\xce\xf5Z\x1d\x1avy>\x96\xceA\x15p\xd9Y\x1e\xaemj\xd1-\xa9\x8et[\x9f\x8f\xd5\xb98l\x89\x13\xa9\xb4c\nrV"k\x9c\xe946\xf2`w\xf5\x1cZ\xcd\x97L\xfc\xf6H\x81\xfaI\r\xb7\xc2\x1b\x8b\x9d\x82\xb5*e\x07\xf4\x94n\x03\xb3\xb9\x9a\xf9^\x9f\x8ac\xb9\x1f\t\xaaZ\\Kn\x90\xdd\x81^\xb4\xd8)\x0f\x90\x93/\xe1l\x96!+\xf1HE\xdbb}p\x90\xd8h<\xa5O\\)/.H2\'\x0f^\x1d?\x87V\xf3\xfb\r9NQ$A\xdf=\xb3\xb6\xf3\xe3%9,\xb7\xf1\xb5\x1d\x92\xa6\xf6!L\x87\x1c\x82uH\xc3T.+o\xeb%[\xa8Tf\xd3\x81.E\xde6\x04q`\xec\xc4\x0b@\xea"qq\x15\x9e\x92\x03\xa7\xae]{F\xf7u\xbc\xc8N\xe4\xb2]\x90\xbf\xba\x12\xf8+\xd3j~\xbfS%a\x94\x80\xb0\xbb\xc3\x12\x1e\x84>\x8bM\xd1TR2\xf48!\x0e\xb7\xe3V\xe3\x12\x03\xe7)\xf8x>C\x94d\xf8\xbe^@\xfa\xee\x02\xcd#Zj=J\xe9\xfb&o!r\x1b\x16\xf5<\tt\x94) ]u\xa9\x02\xbaKC\xddK\xfc\xbfH\xab\xf9\xba\x0c\xfe\xd0j\xfe\xe5\xb7\xcd\xff\x0f\xa2\xd5\xfcu8 \x1f\xb4\xcagK\xdf\x7fK?\xb4\x9a\x0f\xad\xe6C\xab\xf9\xd0j>\xb4\x9a\x0f\xad\xe6C\xaby\xe3u~h5\x1fZ\xcd\x87V\xf3\xa1\xd5\xbc3\x05\xe6C\xab\xf9\xd0j>\xb4\x9a\x0f\xad\xe6m\xd7\xf9\xa1\xd5|h5\x1fZ\xcd\x87V\xf3\xce\xe4\x82\xff\xe3\x08\x0b\x1f\x00\xd0\x07\x00\xf4\x01\x00}\x00@\x1f\x00\xd0\x87V\xf3\xa1\xd5\xbc\xf1:?\xb4\x9a\x0f\xad\xe6C\xab\xf9\xd0j\xde\x99\x02\xf3\xa1\xd5|h5\x1fZ\xcd\x87V\xf3\xb6\xeb\xfc\xd0j>\xb4\x9a\x0f\xad\xe6C\xabyg\n\xcc\x87V\xf3\xa1\xd5|h5\x1fZ\xcd\xdb\xae\xf3C\xab\xf9\xd0j\xfe\xa7\xd2j\x90\x87\xb8\x0c\xf0?-\xe9\xadi5/\x17\xf6]\xb4\x9a\x97\x0b\xfb.Z\xcdc\xc2\xfe\rD\xc8wA]^n\xb1\xef\x82\xba\xbc\xde\xc7>P\x97\xd7@]\xc8\xff\xdc\xf4?\x87\xba\xa0\x14\x89#"\x02\x93\xb4@A\x18&\x82\x1fEP\x98d \xfc\xc6\x9e@h\x14FP\x1aB)Qdy\x18F\x04\x92\x81QD\x10)\x8c\xa7!\x16\xfc\xeb\xc7\x9f\xd1\n\x08\x11\x87E\x96"i\x9c\xa3q\x1cePH\xc0X\x94\xf9\x1aR,\x880\xf1E\xd4\xc0)\x81\xa2`\n\xa3x\x86\x84qQ\x10E\x92\x12\x10F\x84y\xfc\xa5P\x97\xbf\xcf\xf0\xf8\xf1\xb3Q\x05\xe4\xdfn\x13NH\x9a\xc0\xe9?\x83\xba\xbc?\x12\xe7\'P\x17\x08\xe7\x19\n\xa2I\x9edI\x8ae \x9e\xe2P\x8e\xe7q\x11|\x9d\x13\x11\x08\xbf!$\x18\x1c\xac\x18\x12\tX`\xc1\xef\x12\x18\x92#y`|\x84dn\xfb\xf6\x81\xba\xbc\x10\xea"\x90<\xc1\xa0$\xcf\x80}\x15 \x81\xa1`\x86\'x\x1e\xc5H\x01b \x11\x98\x1d\x18\x8dB1\x06\xe3\x81\xed\x10\x8a\x06\xce*R\x82\xc0\xe3\x02N\xd2_\x8e\xf0\x17\x87\xba\xfc\x8b\xa4\x91\xe7A]n#@\xfe\x14\xea\xf2\xfe\x8e\xfe\x8dP\x17\x10/\xe1_\xcf\x91>u\xf2\xbe\xd3W\x97\x83&:MY\xa0\xb4aj\x82\x1e\xf1\xae@i\x1c\xcf8R\xa9\x89\xd5z\x84-\x82\xdaV\x97#=b\xccI\xca\xd0b\'3\x17\x9f\x81\xf9l\xc3\xd5ew^\xa1p\xd7\x87f\x19\xecwe\xf7\xe0\xdc\xba\xa7A]\xc8\xbf!0\x85\xdff9\xde\xc9\xccJ\xf7\xb2\xb4\x96:O\xd4\x07{\xed[!\xa8\xcd\xe2vX\xe4#\x8d)>\xb2\x10Z\xe9\xba\xc5\x8d\xd3E\xc4NX=gA\x00\xd1r5]\x18\x9d\xba`\rb^\xb7\x183B\xa1;\x01\x83Z\xb1\x80e\x97m\xcb<8.\xfbYP\x17 \x93\x80\xa9\xfb\xc9\xe7\n(;\xd5\xb68\xb7\xf0\xca5\xb4\xa0\rLE\xbb\x88\x91\xe9\x90\x17;"\xf0\xb3\xbb\xc4\xd6\xb0\xbf\xd8\xf4\xac\xe9\xba\xeeJY\x1dd~\xe22\xb9\xaf\xf3\xe5\xe4\xd5\xb8\x9f\x18\xd2\xae$\x86\xd3\xc2\x90\n\xbcfRF\x16\x1f\x9c\xe6\xf64\xa8\x0by\x1b\x98K\x93(B\xdd\x8dt*\xd1\xc5"\xa1s\x10\xc8\x05\'&\x08#C\x03n\xa1:\xdd2h\xed\xf0\xc0\xaf\xd2^L\x11CGh\x17\xe9\xb1\xcd(B\xb9 \x11\xfb\xb9g\x92\xb5a\xefEJ\xbb\xc64\x99)\xc3\xbc8.-T\x0c\x92\xa9\xde\xff\xea\xc4\xfe\xc5\xa1.`\x17\x11P\xab\xa2\xf0\xfd.\x1e\xd5J\x9dO\xf10$\xfd\x91/\x19&\xc5i\x9c\xdc\x9d\x8dY\xdc\x9e\x16k\xf3RK\xc3\x166\x84\xf6l\x1b\xc4Zb\xd4v\xb9\x92\xc5\x00\xa5;a\xa7\x81Sz\x8d\xb6k\xda\x915yw\xda&\x14i\x9d\xceb\xdb>\xe8\xf7O\xc3\xba\x80r\x10l\nF\xc3\xc8\xdd\xf4\xd10\xdb\xa4\xc7\x84\x1bqSX\x89l\x17z##m\x8be\xca\xb6\xf9\xd8u\x01\x99(\x87`=^\x16c\x01\xed2\x0f\xb6$w\x01\xeal}o\xd9\xb1!.b\x8a\x1b\x8f\xa9p\x846\xf3\xfa`\xc7TU\x96\x02\x7f|p\x90\xe3\xd3\xb0.\xb7\xaa\x17\xa3Aix7\x95\xf3L\x18\xea\xaa\xceRt\x9d-\x95\x15\xd9j)\x84\xef\xf6E\xab\xcfT\x8bc\xde\xd1i*\xd9\xca\x1d\xb8t\x93\x15\xb5\xda\x80\xee\xe6T\xd7\xbc\xb4\xa5\xe1\x91u\xbdP\xb7GmEfq\x83l=\x83\xebL\xa5h\x16\xcc\x83\x03y\x9fFu\x01*\t\xea\xc6-D\xeff\xacv\x98*\xf4\x16\x16\xf5\xe8!\xe6\'\x11\x9b7\x84\xecXi\xeaQ\xc3\xf5\xeaqQ8\xc7\x95\xd4/\xb5\xfd\xf1H\xd1\x951_\xd7\x96d\xb2\x0b|\xbb\xb8\xe6\xadH\xcfH\xd4\xef\x0b+:\x99g\x97\xc1\xdb\xa8\xed\xd0z\xfd\xa0\xcc\xa7Q]\xfe\xd0\xc2\xfc\xa3\xcc\xa4\x95s\xf1t\\$$\x97\xf0\xd3n\x1f\x11\xf1\x02\x96c\r\x9b\xf6Lp\x10p\x8d\x8c\xf7\x16\xd9/\\\xec\xa2\xefc\x85\x8c\x99\x85[\xad\x8f\xcbz\x7f$|\x9f8/\x88\xd0\xd7\xab\xa1\x1e\xc8\x19\x0ek.\xaa\xc6\x13\xfb]T\x17 \x13\x87@\x15\x06\xc1w\x87\xf6r\xbe\x0e)C\x1e\xda\x901\xe7\xe5\xe9\x80^m\xee\xb2:\x9a\xa4-FV\xd4\x11\xab\xab\x94-\xaf\xe1\xa9X\x1ax\xc9Bg\x0bH\\\x1dv{\xf7\xe8\xee\x0e\xb6\xb8\x8fez\xe9\xce\x15{\xda\x84\xc8)\xe3gT\x8f\xe2_M\xe5|5\xd5\xe5vhA\xaaB\xb1\xfbA\x8b\x1a\xa4\xec\xd7\xcb\x16\xba.\xf7\x8b=\xec\xfa\xc3\xce\x96K\x10?\x0b\xa6\xe0\xddzm\x98\xec\x8c\xe0RMoV\x1c\x99\xe8\xd7n\x1d1A\xbeh\x0bm\x97h\xfd!\xf4\xd7\x16\x8dygn\xb9\xd2@\\&\xcf\xf6:\x9a\xd3_\x8d\xcd\xfc\x8bS]\xbe\n\x1b\x90\x87`\xea\x9e`\xe1Y(\xbe\x91\xcf\xfb\x9a \xf5}\xb2\xecwa\xb6\xf50\x19\x1a\x93T?y\x8e5\xf5\xa7\xd3^\xb9"a\x94\xd1\xeb\xa3\x86\xd6\x9eZ\xef\xa0\x8e26]\'\xcb\x95\x1b\x05v*\xd6*\xa6\xf9\xf2j\x81m\xb7\x81\xc6=\x08\x03x\x1a\xd5\x05\x1c\x16\x94D\x10\x18\xf4\x0e\x7f\x94\x89\xeeg\x8c\x1c\xa9\xab2s\x13\n\x8b\xd8\xb8@\xc8b\x83\x12l\xb9r\xf1\xfd\xd9J\x9c$\x12\x03\xdc\xb3\xc2yo#\x8b:\xddF\x88d\x0c\xad\x17\xad\xf6-\xba\xb5\x89\xfc\xba\xdb{\xc7\x85\xa9\xf8\xac\x0b\xad\xb9\xb2\xe1\xad\x07}\xe2iT\x97\x9bO\xd0\xa0L\xc5\xee\xd1\x0e\xfbMp\xda$\xa8\xe3N\x9b\xe3,\x1e\xb4\xde\xb2*K\xa6\xc58\xd8\xf9\x92\x1a\xc3\xa5%\x93\x0e\x1f1\x97`\xa4\x86\xa3\xb7\xb2\xf3\xd5e\x9e\xb1\x8cC\x89\xba#:G\x97\x0f\x1a\x11\xefC\xdb]l\xcf\x17\xbc\x04a\xbc|\xaf\x12\xeeiT\x97[OC\xa0 \xad\xa0w\x01\x94s\x0e\xd7\xb0\xc7\xabFE\x8b\xe5ac\xcd\xcbe\xea\x97g\xd6n\xec\x83b\xe7\xc9\x9a\x89\xbd\x00K\x97&\xb4\xd2\xd7\xb7\x06j\x9dc\xea\xc9\xb3\xbc\xab\xec"B\x0b\x9f\xc3\xab\xdd\x8e,\x8c\xed\x18\x12>!\x18\xa9\xc9\xe4wQ]\x80\xcc\xdb\\uP\xd4\xdf\x05P\xc4\xe9\x9b\xf8\x1a\x9a\xa2\x8c\x88\x04\x17\xc68\x9d\x18\xcb*p/Aw\x98\x18\x96\x82\x05|6\xb2|?`\x0e\xa2\x91\xb6\xd70\xb9y\x1e\xa6\xf6\xd00\x86\xb8\\s\'\xff \x88LP.6\xd7\x93[r\xcc\xe2\n*\x94\x07\x01\x84\xcf\xa2\xba\xdc\xb2>N\xe1\x10\x10\xfaG\x99\xc15\xa3+\x7f\xeeqG\x86)&\x87\xdb\xc9\x9c\xd2m<&\x1b\xd5\xf1\xb8T\xd6!\x15\x9ek\xd8<7\xfa\x0es[ZN\x84\x8b\xe7\x1e\x86U\x99\x04\x04*\x1e\xd7\xa7\xa5L\xa0\xe7K\xa9\xf6\xdd\x9a\x1d\xcepK6\x0f\x167O\xa3\xba|\xb9>\x01\xbeL\xde\x05r2OB\xfe\x84u\xf6l\xce\x15\xe1\x8c\x869\xd3\x9c\xc3g\x9d\xb2\xc7V\xf5\xa0t\x06\xbf s1\\\xe1Y\xb3;\xc9\x8eP\n\xe1\x91g\xcfu\xb2\x1e\x94\x89\xac\x83\r\xe9\x96\xbd\xb9\xb4\xf3\x96;\xcc\xdai\xbe"\xcb\x07\xfb\x8e\xa7Q]\xc8\x1b\xf5\x00\x18\x93\xb8G\xd6\xd5<\xb4\xc0\x1a:\x8e\xb4\xd6\xf1\xe5\xa0\x82\xeb\xc67a28/5.\xa1PW\xce\xf0K\xd0\xc4.&\x98k\x11\xf5S\x0f\x0e\xf8\xf5\xd8\xba4\x82\x9f+#\\\xec\xbdmU\x85\xabj\xb0\x84\xfd\x19\xd1\x1d\xf3x\xae\x1e\xbcox\x1a\xd7\x85\xfc\x1bDa\x04N\xa0\xf7\x1c\xd9\xed,d\xe8:\x19/6\xbe\x95\xac\xd3r\xa3\xaf)\x97\x9b\x047=i\xfc\xf1(:\xb4\x83\xcb\xf1:K\x12\x93\xcd\xc6\xdd\xde\xef\xab\x9aRqi\xe1x\x82\xd6\xb5YPD\x9c\xcc\xba\xc3\xc2\xd9\xae9QX\x08\x81\xdd>\x98\xaf\x9e\xc6u\xb9Y\x93\xa4(P\xaa\xdeu\x91\xf2%\xe7\xb2\x85\x01\x9aL\x81\x19\xb83\xa3\x0e\xbd\xd7\xd3k\xc6\xbe\x9c\x86\x9c\xf6afX\x1a\xfe\x18x\xd7\x89Y\xea#w\xb6\xfb~\x9c\x8bJ\x12W\xb3O\x96\x81\xbb\xa2viQ@\xf1)\xe6\xe1`k\x18\xa7\x0b(\xc9\x1f;\xb4O\xe3\xba\xdc*r\x92\xb8\xa5\xa0\xbbH[\xf9\xdeu#\xd3\xbc\xd3\xf5\x83\xc3\x88\xeb\xe3I\x92\rN\xe9\xfb\x0b\x9c\x04\x1b[\xd4\xd7\xde\xf6d.6=\x13\\8\x17C\xaae\xb4\xa7\x16\xec\x91\\\x1b\xd1\x1a=^\xa4\x91O\x95\xd3\xc4\xd5+\x04\x0ejs-;U\xfc`\xe3\xf14\xae\xcb-\x04\x81\xacD\xc2\xf7\xf7d\x17\xcd\xdb1u}\x8d\x128\xb9\x9a\xa9cdPR.\xa8l\x9c\xc5\x80\x8a#\t\xb4 \x88\x03\xed\xdd\x86\x98\xce\xfa0\x9d\xb2\x1d\x8b\xda\x1b\x92H\xcf\xd0RZ\xa5\xa7\x85\x03Ad\'\xcf\xd7t\xec ;)\xc0\xda\xcc\xf7\xaa\xc8\x9f\xc6u\xb9\xe5+\x90\x95A\x9bz\xd7\xa5n\xd8t\x9aw\x8b\xf8\x12[P\xba\xbe\x1c"\x0c\xea\x96*\xa2a\xb6o\xcfS@\x06\xfb\xab4n\xafH\x99\xf2-\xb2#dz\xce\xedKG\x1d\x96\xa5B\x8dH6-\xe1`\t\x9a\xf5\x84\xd7\xcfS\x94b\xc5V](\x0f\x06\xf2\xa7q]n\x159\x86C (\xde\xdd9\xe4\xb3D\xe8\xc1\xa1EM\x82\xd9\x1e\xfa\xeb8\xae5;\x12"\x9cT\xd8Bb\xf7\xd16Ewx\xea\xe0\xf4\xe6\x98\xc6\xbb1\xde\xcd\xa6\xbd\xe1J1!\n*]z\xda\xf9\x0cm=t\xbc\x1afK\x9b\x8cLw\xed\xf2\xc1\x0b\xa4\xa7q]\x80L\x98\xa2\x10\xf0\nw\x81\x9c5\x8a\x03\xe5\xbb\x8b0\xa6\xb7\xad\xb8\xad V\x08\xfa\x14T\xdb\xfc\\I\x08\xa8y\x9cqqa\xd6Nc\x11V\xbe\xd1[\xe9\x1c\x91]#"\x9b\x06\xb5\x99\xf6\xe2\x87\xa3\x13\x85%=\x90Q\xb7\xde\xf5\xbd\xd9\x0e\xf3\xf8`\x7f\xf54\xae\xcb\xad\x96\x84p\nX\xfe\x8f*\x19z\x93\xc0\x01))\xde:\xda,\xeb\xbe.\xaa\xee\x84\x90dH\x8e\xa8R\xd2\xd2\n\x01\x85zP\xaa\x82\x9c\xbb\xb5\x12x+\xd2\x8c\xd5\x1e\xc2\xc7\xeb%\xbb\xae\x08b\x13\xcf\xd7\xbc/\x91u;-\xd3\xf5H.}\x84{\x90\xed\xf04\xac\xcbM%F\x83\xea\x13\xb9\x8b\xe3N\xa5\xa6\x909t\xdb\xfc|\xcd\xb5\r\xec\r\x9e\xbd\xa1\x95N\x17\xb2C\xe9\x9c\x9bq\xa5\xb1\xebv\x0c\xc5\x0e_\x0f\xd9\xa2Jfc\xa2\xf0\xca\xe0|\xf4\x12j\x15\x924M\x9cP\xd6\xd1\x88\x8c\x08\x99\x14N\x97N\xe2\x832\x9f\x86u\xb9\xdd\xed\x82\x96\x94\xc6\xef\xd9\x0e#\xaf\x1bf\x93\xedl\x87!\xe4\x05\x1b\xe7\xdc\xca<\xa0\xb8\x01\xea\x91\x13\x85\x06J\x11\xd6\xf1id\xb3S\xb2\x89\x84\xc2_\x93E\x0f\xa1\x15\x07\x19\xe0\x1fi]\x11\x0cGn\xd94o\xf98\xf5\ns\x99\xeb\xc5Yd\xde\xaa\x8b|\x1a\xd6\x05\x94p\xb7\xe7\xd9\xa0\xc7\xbb\xdb\xc5xDV\xe1\xa9\x1fVRuU\x96}.\xb8\xd3 x[\x1e\xe1)&\xc6/\x83\x94&P\xbfE4}\xdc\xa7\x90n5\x08\xb9;\x93$R\xd8G\xe4TK\xd2\xd5@\xc8\xfa\x9c\xac:\xe2Jo\x17L\x14\x0b\x92\x1d<\n>\x7f\x16\xd6\x05\x1c\x16\x04\xa1\xe9\xdb\x89\xb9{\x00\xc8;\'H\xf1\x9d\xe9\xac\xed\xd1\x8e)\xb7\xc8z\xec\xa6\xd0G\xaaY\x18x\x98\xba\xb2\x8b4\xa4\x91\xeb\xe2*\x17\xc6\x19O\xc8\x06\xcf\x89~Z\xf8\xda\xa8\xd6R~@\x88a\xc4I\x87G\x93<+\n\xbb\x17W\xc8\xe9\xc1\x87cO\xc3\xba\x00\xd7\x07\x81\x1f\x86\xe0{\xd7\xf7}n\xf4R\xfdzi\xea=2\xf0]\x8b\x0f\xb6\xe3\xd2KP\x8b\x1e|\x86\xd8\x0fm\xd1\xc2\xb4\x89O\x1e\x95\x16[\x035\xd1\x9d\x8eNd\xd8\xf4{fC\xd8\xc7cW#\x97r\'\x9b\xc8@\x9b\x1c\xe5\xa1\xdd\x14<\xd8w<\r\xebr\xabTi\x84D\xf1{\xa0-nn\xfa\xd55\xd3\xaf\xd7^\xe9Y\xc8\x07\xa9x\x9cT\x95\xb1\x98\xa4a\xacep,I[[X\x1a\xaa\x9e\x89s\xab\xe1)-%\x89\xdb\x84\xad\xe7f\xbe5\xe4C,\x171z`7\xab\x03\xcda\xdb\xe2\x1c\x15\xdc{a\x81\x9f\x86u\xb9]\xaa"$p\xfd\xfb\xc3\xb2\xde7\xfe\xe5D3}\xaf\xd4-r>\xc8\xc7(\xa0\x96\xb3\xa4\xed\xf6\xc5\xd9v\xd3\xa6\xd6\xece:\xd7e\x13Uk\xf5\xb4\xe5\xd0%\xe6Bb\\\xec\xd7\xd8\xa2\x8e-\xdf\xbb,V\xa8&\xec7\xfc\xc2\x95\xe5\x8dn\xab\xfao\xf5\xfe\x7f\x8du\xf9r\xe6\x0f\xd6\xe5_~\x7f\xf9\xff\x1c\xac\xcb_i\x8a\xeeg0\xf1\x07\xeb\xf2\xc1\xba|\xb0.\x1f\xac\xcb\x07\xeb\xf2\xc1\xba\xbc\xef:?X\x97\x0f\xd6\xe5\x83u\xf9`]\xde\x19\x97\xf2\xc1\xba|\xba\xa7\x0f\xd6\xe5\x83uy\xdbu~\xb0.\x1f\xac\xcb\x07\xeb\xf2\xc1\xba\xbc3.\xe5\x83u\xf9`]>X\x97\x0f\xd6\xe5m\xd7\xf9\xc1\xba|\xb0.\x1f\xac\xcb\x07\xeb\xf2\xce\xb8\x94\x0f\xd6\xe5\x83u\xf9`]>X\x97\xb7]\xe7\x07\xeb\xf2\xc1\xba|\xb0.\x1f\xac\xcb;\xe3R>X\x97\x0f\xd6\xe5\x83u\xf9`]\xdev\x9d\x1f\xac\xcb\x07\xeb\xf2\xc1\xba|\xb0.\xef\x8cK\xf9`]>X\x97\x0f\xd6\xe5\x83uy\xdbu\xfe{d\x06\xf4\x1f~\xc9\x7fMf@\xfeiIo\x8duy\xb9\xb0\xef\xc2\xba\xbc\\\xd8wa]\x1e\x13\xf6o@B\xbe\x0b\xeb\xf2r\x8b}\x17\xd6\xe5\xf5>\xf6\xc1\xba\xbc\x06\xebB\xfd\xe7\xa6\xff9\xd6\x05\x83oc\xd8H\x98\xc5)\x98!\x19\x0e\xc6\x19\x04\x11Y\x08G\x05\x18\xc6\x08\x96dxJ$Q\x0e\xa5I\x8e\x15\x04\x1e\xc2\t\x1eFX\x01\xe5X\x8e\xa6\xc9\x1b\x91\xe2\xd7\xbc\x02\x12\x151\x18\xa6P\x18\x01\x9b\xc1\xa3\x04%"\xb7\xc9\xd3\x02\x84A"*\x8a\x04\x86\xe1\x04\xcd\x8b,\x89R$\x0b\xdd\x86\xb3 \x02\x0c\xfe\x9b\x03\xbbCB\x14\xfbR\xac\xcb\xdfg\x96\xff\xf8\xc9\xa8\x02\x18\xfd\x1b\x01\xc18N\xe3\xc8\x9fb]\xde\x1f\x8a\xf3\x13\xac\x0b\x0bc\x14\xc1B$F\xde\xa6\x19\xc18\xc9\xd0$\x8a\xa2\x18\x04\x16\xc4@\x0cM\x81\xc5\xe2\x02.\x08\x02D \x18K\x8b\x14\xc500L\xf3\x1c\xc9r\x08{\x9b*\xfb\x8fX\x17\x82%p\x0e"0\x14\xe79\x9c\x85\x11\x86\x06k\xa0Q\x0capL\x80\x11pHP\x06\x9cQ\x14\x06\xafM\xd0`\xe5\x0c\x8e\xde\xc68\x03\x8bC\x18J\xfd~R?X\x97\xff\xc0\xba\x90\x18\x82\xf00\xc9\x81\x97`I\x04|\x13\x0f\x01\xb7!0\x8aF\x19H` \x14\xa5a\x86\x17\xc0w@(\x0b\xfc\x84\xc7\x81?\xa1\xf8\r1\xc204\xc6\xfc\xf8\x8bc]\xfee:\xc5\x1d\xd6\xe5\x07w\xa4X\xee8\xb2\\\xa9_B\x84\x9ec\x9f\x9e\xb5\x83\x01\x85~\x7f\x8c\x9bdH+\xb1\xdc\x06\xf6Nwp#ARa\xeb_\xeb-"B[{\x05se\x7fI\x10\xf8\xa2\xbb[P\xc0\x1a\xa0\xb7df\x03\xe91\xc3\xdf`[\x7fWE\xc8\x11K\x0f\xb7\x0f\xe0wxT\xb1\x15\xa8,\xda\xd8\xdd\xf5:Z\xe3B\xb5\xfa\xfa\xd9[\xb3\x10\xa3l\x9dT\x02\x9c\x06\xab\xab\xde\xd8P\xc2\xd7\xd7\x10\xd9\x80Z\xb2\x06\x8d\x95\xd7\x1b\x07oL\x11\x08\xcfd\xaf\xd3g\xe6\xf6\x81\xe8\xd6\xf4\r"\xbc\xb6\x0eWx\xf0W\x8e\xbc*+cf\x10\x9d\x17&\xf0g\x16\xeaPu\x8b\xa3\xc4\xed\xa8\xff\xd4\xd5\x18S\x16\xb0P<\xe1\xf3\xd67F\xbd\xda \xd1,\x0c\x1b\xd9+\x1d\xc4\xc3\x85RW\x15K)N\xf3Z[\x9b\xfe\xa0F\x95\xa4\x15\xa4\xca\x9c\x0c\xeaR\x1c.v\x0b\xf1\x1d\xddm\x0b\xb18\xa6\x87\x8aD\xae\x8a\xc0\x15\x8a\x92\x83\xbf\x05\x9d\x8bx\xf6\xc8X\xf2I\xe6\xabD1\x06\xfc\xc4\x9d\xe3\xd4UiX[;\xb4\xc2)\xe7\xeb\x82\xa2qog\x1c\x0eg\xe4,\x1a\x93\x96\x84\x14\xd3P\xd2\x9c\xc1l\x19\x9f\x105E\x88\xb2\xc8\xb4\xb5;\x9c\xe4\x8d}I\x98\xe64\x13\xad|\xba\r\xb7\xf9\xb7\x01<\xb7a-\x7f\n\xe0y\xff\x90\xfc\xbd\x00\x1e\xea\xd7\xd3\xfe5\xe3\xbc[\x9f\xb6\xc5\xa6\x8b\x16\xc5\x05W:\xb6^\xc3\xe1*\x8f`e\x165\x1a]n\xac\x8d\xca\x19\xc5\x11]r\xd7\xd9\\\xe2\xb3\x04\x17\xb3!\xefD\xae3\xb0\xd5%\x88\x8b&\x9b\xb7\xf5IW\x98m\xc9\xb4\xfc\xd5@\x1e\x1cL\xf5,\x00\xcf\x1f\x13\xf8?\xcaL\x0fA\x93$\xe7|\xbf\x08R\x01\xf6\xbd\xa8\x96\xe43\xd6F~\xe9\xd0\xa8\x91\x9d\xce\x8b\xb5\x1fQ\xc7\xdd\xde&\xf7\xc7r\x1f\xea&\xd2`a\x03:y\xf1\xcc\xf5\x96\xb2Zc\xc1\xa8\xaf\x8d\xd4r\xd8\xae\xac\x85\xd8?\xe5\x0f\xce\xa4{\x16\x80\x07\xc8\xa4\x10\n\x83\xee\xc6\xc4\x9a\x19\x0c\xbaR\xc4\x8bbe\\df\xd0q\xd1.3\x12\'\xccB\xc4+\xfaK\x95\x1e\'\x06\x14\x9bC\xb5\xd9\xc9\\\x8f\'\xcez\xea\xd7\x97\x0b5\x15\xd6\x82\xe0\x05\x82\x1d\xf7\xba\xa78A\x19;\x96\x14\x95c5\x04\xdf\x04\xe0\x01\x1aI\x9a\xc6a\x18x\xf2\x1fe\xf6\x9a\x83\xc7\xc69"\xa0s+\xd8\xa8\xa3\xc4\xae\xc3\xa6\xd9\xb1\x91W~\xee\xecl+\xba\xa2\xeb\xf6\x14h%V7K\x1es\t\x17\x9b\xc4\xe6\xca\x00\xc7\x01\xc1YUL\x8a\xd2\xfb\xc2\xa2\xd7E\xcf\xf4\xee\xc4r\x81\xca\xbd\xd5\xfc\xd4g\x01xn\x0eA\xe2\xa0L\xc1\x89\xbb\xc3BNL\xb1\xdf\x9d\xb6\xa3\xc1\xc7\xd8\x99O#h\xb5\nk\x1b\x0f\x19<\xd9\xce\x0e\x19\x8a8\xa5-<\xd1[\xfa\x19]\x13A\xbf\x85\xd9\xf9\\8W=\x97N\xf0\xd9\r\tS\xddg\xe3<\xf40\x92\xfa\x07"<-\xddG\xe7}?\t\xc0s\x93y\x1b.\x8dc\xd0\xddxA\xdf\xf7\n\xb8[\xd1\x15\x8a\x9edvh\x02\x0c\x19\xf7N\xa1.\x80\xb7\xbb9\x82\xaaii\xb0\x1a+\xf1\xcb,C+\x98\xb5\xa3\x95,\xb4\x13\xe7VA8\xaf\xda\x13\x1cy\xbc\x0c\xdb\xaa\x11\x06Y\xaa\xebs+\xb0\xfc\x833\x85\x9f\x05\xe0\x012q\x12\x07\xf5\x15\x8c\xdfMo\xd7\xb6\xd1\xc5[\x8a\xebK}"\xeb\x950\x18\x02\xdbn\xcdFX\xda\xcc\xf1\xb0\xf7O\xb5\x88\xb3\xa7\xd8$\xadL3\x03\xdd\xcd%\xd7\x17}\xad\x06uz\xd4H\x99\x99K\xfbR\x9f\xce\xdc\x8ewi+\xd0"\xff\xe84\xdb\x07\'D?\x8b\xc0ss}\x04\xec\x07\x8cbw2\xb1\xe5l\x1f`\xfddSR\xeb\n\xe9\xaa 63\xa4\x84\'\x1b\x17\x97u\x17k\x17\x08\xdb\xfb\xcd\xb0\xcf\xce}\x8e\xc9\x9e^\xd8\xf8*^\xbb\xd6^\xa8S\xfe\xd2\x9c\xb3\x91\x89\x9b\xf8\xb2KF\xec\xd2\xa9\x96\x82\xcdL\xf3(r\xe0I\x04\x9e\xdb\xa1\x05\x9d\x11\x0cr\xc1\x1fU\xf2\x86.\xad\xbd\xcc\xce\x8f\x9az\x85\xcdS\xdb\xd5X\x92he\xd8\x05Kz\xe5&\xc8b\x14&\xe1\xbc%\xb8\x04\x13Y\'+\xc93\x065H\x02\x83D.\x9b\x01H\x0e9\xbd\x82T\x81\xa0/\x07<\xae\x83\x89B\xa2G\x8d\xf9$\x00\xcfW\x04\x82\xc0\x91\xc7\xee\x99*\x07\xa6\xdd{\xa4~\xf2l\xb4\xc8\xca\x04\xf7jNK\xe83<*\xe2"\xc8\x17\xa0[\xdbm\x8d\x91\xebA\xf6:\xed]QV19\xa7\x8aK\xd8\x1e\x14U\xf5\xe4 C\x974i\x17Bo]\xaa!\xdf\xf2LuX<\x18\x81\x9e\x05\xe0\xb9\xb9&\xb0%qC\xb4\xdc!\xc0R\x11\x9a(\xd2eVRZ\n\xf5\xa0\x87J\xd6*\xd4\xe6\xd2\xf8\xadb\xd3$\xb1\xea\xad1\xd5}la\xa2\xadbM\xec\x12\x94\x16T\x98\x91r\n\xed\xa0"\xe7v>\xbe\x8e\x8ex_\xafW\xd1\xa9i\x11e\x7f~\xafY\x91\xcf\x02\xf0\xdc\n\x1b\x98\xc2\t\x94 \xef\xa1T\xfcV\xff\xff\xd9\xbb\xb3%W\xb1,A\xd8\xef\x12\xb7\x98%\xf3\xf4\xdf1#\x04B\xcc\x82\xbehc\x14\x83$f\x81\xe8\x97\xff\x91GUwf*NV*M\xa7\x8eG\x16\x117a\x1e\xe7\xb8\xef\xb5\xc7\xb5\x11\xbe>\xb9\xe8\x1f\xd8\xb4;\xb0^#\x93\xbb#`\x9fv\xc9\x99\xf1\xe1\xa3z\xa8!K\xf6\xb1\\\xd6\xcd\xa185\xb1\xc4\xa4\x03\xecc\xdc\x82IN\x1d%\x11bxZ]_*s4G\x92\xb8\x89|G^\x16\xf2\xdd\xe3\xeaC\x00\xcfsM\xac7\x15x\xbdC\xbflpk\xda\xa5]\x12\xd0v\xd59\xc4\x12\xbeu\x8f\x08\xedpn\x98\xdd\xbd\x83\x84^\xc2\x8e\x06\x05f:Eh\x81H\x8d\x08\x9e\xe6\xdcc[U\xa2\n{^l\x10s&\xa3\xe32 \xbbi\xe6\x1a\xca\x15\x8e\xd3F:\xfe(\xb7\xf9\xc9\x00\xcf\xd7\xa9\x8c\xae\x0b\x8a"_*\x7fB\x92\xda?L\x00\xd7D[\x8b\xaai\x063\xb9\x92\x06\xb5\x8bv\xd79\xba\xaf\xa9\\\xdf.\xbb(\xaf\xc43S7Cp\xafA\xcc\xc7\xc5\xa1\xd19\x9et\xa8\xfbm>\x14Ry\x90\xb3~<\xa4\xc3\xba\xc5\xb7\x87\x89\xfa^k\xe2S\x00\xcf\xb3\x17\xd1\xf5\x9c\xa0\t\xeaeM<\xdc\x08]\xc7fu\x8fK\xf1E\x00\xa3\xf5\xf87\x11}\x1c\x1a\xe2`P\x0bsr%\xa3\x16\xe7(\xab\xc1\x81t\xbc\tv\xf5\x00\x10o\xca\x19\xd4\xce\xae\xf8f\xa5\xffO\x01<_\xa3\x89\x91\x08\x84B/\xa0\x01\nG\x825957s:\x93A\xbd\xc6\xdd\xcc\xe1\x80\x85\xd9\xbe\xc8\n\xce-\n\xdf?\xed\xa7\x02\x93\x94}\xd0\xa0>\x0f\x8f\xea\xd22\x18\xb4\x84\xb1J"\x1c\x8dq\xa6\xcb\x0e\x8f\x1eu\xf6Q\x00\xd4\xc4x]v?\xd2/~2\xc0\xf3\xccT\xd15\xb3\xc1\xf0\xd7\xbb\xf8\xb5\xc7\xd3\x05=\x9e@\x8e\x1a\xa3\xfb \ndbQ\x17\x92>\x19i%\x9c\xaa\xb4\x17Nw\xc2@\x17\xfaV\xfag\xe3a\xf8\xea\x8a{\xe7K\xbe\x8c\x9d\xd8Z{\xb4p\x96u\xb0\x8e:)\xe1\xd2\xadK\x1cW\x92\x85\xf5,\xbb\xc0\xea\x15$E\xb0e\xa0C1\x0b\xdeU\x9e\x8f\xd9D\x11E\xc6a\xb95\xfe\xa8\x04\xfeO\xf6w\x9es\x16[3xtM\xe2^7\xda\xa5\xf5c\xea\x84\xeb\xca\xdcQ{I\xbb8\x1a\xcc\xe3\xf88\xf5\xe1\xce\xe7*])\xd4\xf1q\xaa\x89s{E#y*\xe9GR\x8a\xaa\x83\xc0\x9d~w\x16\xe4\x0eG\xba\x12BTi\xcbx{\x8a\x96\xb0\xc8\x06\xe6\xbd9\xfb)\x7f\xe7\x99\xa9R\xcf)\x81\xd3/s\x96\x1a-\x9c\xf3\xa3>\xa0nl\xd48;9\xb2P\xfa~\x1c\xf8kL\xe6\x08z\xd6\xb8jJ\x17\xe2~"\x82f\xc6\xcf\x14.\x9a=\x04a\x9av\xdd\t\xfe\xad\xad\x13\xf1>\xebR\xa3\xc4\xc7u=\xc52\xe5\x9c\xc9\xf0\xcd\xa5\xf9)\x7f\xe79\x9a4J"\x14\xf9jq\x1bt|N\x0f\x94q\xcen\xf7\xd3eX\xaa\xb3\xf1\xd0`\xda`YhH\xf7g\xee\xb0\x93\x8f\xa9Q\x9a\x0f#)\xa3\x07\xe0gp\x95\x0f`F\xed\x05\xd7\xedzD\xc3\xee2f\x9c:{D&t\x7f\xee\xf8\x8a\xb0\xb1\x1f=\n\xfcs\xfb;_O\x1c\x10\x82Bp\xfa\xe5T\xd6\xbbC\xde=\x1e\x93\x06\x02c\xdf\xef\\\xd7\xc6<\x89tz\xf1(\x8c\xc1\x81\xe5\x15\xbc\x94\xc2E\xaf\xb2)1y6\xdeu\xa1\x1f.\x16\x86\x1c\xc3Yn\xa4\x0b\n\x1eH+\xcag`>Q6\xb6OU\xfc\xcc\xe8\xbf\xc8\xdf\xf9:\x95\xd7\x1c\x86\x80\x88\x97\x84\x1c\xa8\xa6\x16m\xa8\x1e/\xf7*\xb5WH1\xe4\xaf\x8d\xc2\xa4ah\x05Eu\x81\xe45\x15\x96\x8f\xee\x83\xe1|`\x89O\xa6t6\xda\x9eK\xdds}\xd6\x13\xd1t\x10\xc0\x95\xe0\xe0x\xdfg\x034_\xd8\x88\xb8\x95\xed\x9b\xcf\x8d?\xe5\xef\xe0\xa9EM\x7f\xfbN\xee\x8f\x7ft\x8c\\n\xd1U\x84\x92\x93r\xf9\xab\x1f\xed8\xf9l\tf\xebC\x17=\xb1\xdd\xd9@\x95"\x10MU\xb7E\xcd\x115\xdcr\x0eJ|;w:_\xc1\xee5\xd7\xa3+\xfe\xd0n\x8ad\xdf.\x9a\x0fW\x98\xe1A\x98\x896\x17\xcdQ\x1a\xff&<,\xc4x\x1c\xae\x97[\xe0\xcc\x86+\xbb\xb3%\xbaVr=\xb0\xd1I$\x0eWz\x17\xbb\x02}4\xfe\x9e\xe0\xf9/\xf8\x9f\xb5\xdb\xa0\xd0\xa3\xc7\xbfjw \xb9}\xe0%y"9\x7f\xf7\xbd\xfeeEI\x1c\x0e\x9e9\xfahpM\xca\xa4On,\x92z\x87\x87V\x1e\x88\xf0*\xe0\x1a\x12\\|\xd4\x1d\xb5\xebP&\xe5n\r2\xe9\xe3\xff<\x87\xfe\xe3;\xff#E\xe9_\x18\xdd\xbf\xfd\xee\xaf\xbf\xce\xf1\xdb\xf3u\xae\xc3\x7f\xbc\xce\x95x\xcfW\xb8\xa8\xf7:\xf6\xe3}\xf17\xbf\xafs\t\x8b\xeb\xdb?\xf9\xd7\xf4S\x92^\xd2s8\xfc\xb0\xa3~Ua\xe8\xdf\xfe\xfa\x8d\xbd\xb7\x866\xf2\xe8*\xf1\xe6\xbf^\xeb1j\xe6!r\x19\x83\xe9o\xa7\xed\xa7\x87\xe2\xeb]\xfa\xe7n\xdb7\xe1\xf3\xf5\xbb\xdf~{~a\xb3\xcf~\xf0KX\xff>\xf6\xd9\x9fH\x95\xda\xa0\xae\xadK\xbf\x7f\x97n\xf6\xd9f\x9fm\xf6\xd9f\x9fm\xf6\xd9f\x9fm\xf6\xd97n\xe7f\x9fm\xf6\xd9f\x9fm\xf6\xd97zt\xb4\xd9g\x9b}\xb6\xd9g\x9b}\xb6\xd9g\x9b}\xb6\xd9g\x9b}\xf6\x8d\xdb\xb9\xd9g\x9b}\xb6\xd9g\x9b}\xf6\x9dM\xb1\xcd>\xdb\xec\xb3\xcd>\xdb\xec\xb3o\xdb\xce\xcd>\xdb\xec\xb3\xcd>\xdb\xec\xb3\xefl\x8am\xf6\xd9f\x9fm\xf6\xd9f\x9f}\xdbvn\xf6\xd9f\x9fm\xf6\xd9f\x9f}gSl\xb3\xcf6\xfbl\xb3\xcf6\xfb\xec\xdb\xb6s\xb3\xcf6\xfb\xec\xdf\xd5>\xc3\xfe\xea\x87\xfc\xd7|\x11\xfa7M\xfa\xd6\xf6\xd9O\x0f\xecW\xd9g?=\xb0_e\x9f\xbd\x17\xd8\xbf i\xfd*\xfb\xec\xa7\x8f\xd8\xaf\xb2\xcf~\xfe\x1a\xfbo\xb1\xcf\xfe\xd7\xffU\xc3\x9eU7\xac\xf1\xfa\xe5\x9a\xfc\xc7\xefq\xaf\xdf\'\xed\xae\xc5m\xf8K\xdc=\x9a\xa1\xfe\xcbq\x8c.E\xbcO\x1f\xff[\xf8\x0fY\xec\xff\xb6\xff\xff\xfc_l\xec\xff\xfbmM\xea\xf6\x1c\x91)\x88\xef\x9b\xf5\x01\xa7\xd0\xeb>\xe9\x07\xb5\xc0\xd3\xf34 \\\xf50\xa1G\xe1\xed\xcc\x8b\xf1\xfc\xa5\xacg\xfd\x8f\xa6\x9e\x9et\xd7o0Ja\xbf\xfd\xfe+\xe7\xff\x834\xb6\xbf\xe9\xc6\x7f\xa4\xb1\xa1<\x03\x11\x04\x8cB\xa4\x00C\xf0\xfa\x9f\x1cF04\xca\xe30+>\x8b \xf1O\x12\x8a\xc2\x08\x12\x16QN$\x10\x06\xe1 \n\x11Q\x14\xe5Y\x88\xc0\xfe1^\xc3c"\xcb\xf1\x10E3\x16\x04\x8eC\x04\x11\x87\x98\xf5\xfb2\x14/\x88\x02O\xa3\x08A\xfcT\x8d\xed?K\x8f\xfe\xf6\x07\x85k\x10\xe8YC\x16\x87I\x08\x87\xff\x91\xc6\xf6\xfd-\xbb?\xd0\xd8\x04\n\x12\x19T\xa4E\x84\xc2\x11\x84f`\x9e\xa1\x18\x84_C\xc0\xf0\')DA\xeb\x97D\x8e\xa4\x04\x12\'H\xe19C\x18\x9a\xe3\x18\x8ebD\x1e\xfb2\x14\xfeZc\xfb\x00\xc9\xf5\x7f\'\xf9\x7f\x83\xc6\x86\xf1\xa8\x08\xa1O\xe1\x04e`\x16e\x19\x18\x13\xa8\'\x0c\xc3 4\xcd\xd1\x10\x8ea\x10\'\x08\x0cM\xf08\x8a\tk\x07\x89\xebOE`f\xfd\x1a\xcf`\xc2o?]c[#\x16\xe1uu \x02\rQ\xe2:\xbd\xe8\xb5ca\x9c\xa6\xd6\xd6\x914\x8dQ\x02\xb6\xc6\x89<\xe7\x05\xc3\xaf\x7f\x18\x83E\x1aCXTd\x08\\\xc0I\xf4\xb7?\xd6\xd8\xb0ur\xc0$F\xf2\x18\xb3\xf6\x1a\x843\xd8\xda\x14\x81]\x07D\x10h|\xedgb\xfd~\xdc\xdaP\x92b)\x88#H\x91d\x11~\xdd\x01I\xf1\t(\xfc^\xcc\xe8\xbfCc\xfb\xa7e\x96\x17\x8d\xed_6\xbe\x9e\x05\xa1\xfe\xa1\xf1\xf5\xfd\x17\xfa/5\xbe`\xec\xc7\xf8U\xbd\xfe\xe9\xfad\xbbS;\xd3y\xe8&\x18Q\x86^\x94\xa4U.j\xc7\xc5\xc0A\x89\x18|\xac\x10\x94\xa5\xc3v\x06\x90s\xb8\x06\x9do,\x81+\xe6\xce\xf3f\xe9\xda\xf9\x8dp?\xe1\x8cNu\x8e\x11\xf5\xe5\xbe\xf1\xdf,~\xf7)\xe3\xeby,\xd04J\xac\x8b\xfc\xa5\xb4\xf8\xc4\xbb\xd0\x0eMK\x13L2\x1a[\x13c\xa5\xd8\x0f\xea\x19G3\x81lT\xfd\x14\x9d\xc9\xfbP l\x9b\xbb\x10}\x80\xa6=\x01]\xd6!F\x9bG\x12=N\xe8\xd9S\x01\xad\x1e\x0e\xa1\xba&\xd5\xb7\xde\xdf\xcf\x12\x91\xf0\xefU\xf7\xfc\x94\xf1\xb5\x86I\xe1\x14\xfdj\xc3h\xd5\xc8\x9c\xbd#r\x0e\xc4\x08d\xf7\x0eu\xb82\xad\x1d\x1dP}\xf4\x84\x81\xcb\x97\xc8\xf3\xbad\xbe3\x0f\xee\xceC\xb6P\xc6\xd7d\xb0\x0f)\xc4\xb9\xb8\xa8\x15\x80!\x1c\xe0\x14\x97Q\xa8\x86\x12n)\x8f\xb5%+?\xaa|\xf7\x93\x8d\xaf5F\x12\x87\xd6\x1c\x03\x87_J{z{S\x87x\x1e\xf1\xce\x16s\xa3\x92\xfab\x04%\x03\xbb~R\x96\x17\x0c\xd1\x85\x99JO(\x80X0Y\xde\x96\xd0\xf6\xd4L\x01\x0f6\x06\xd0\x18p=\x13\x13\x85\xc3\xc6Y\xce\x0b\xb2;\x11\x11\x1a\x07\x10\x8d\xe7\xc6\x8f\xca\xa7\xff\xb9\x8d\xafg/\xc2\xeb\xee\x8e\x91\xc8K\xd1K&\x13\x14\xb6\x83U\xba\xaea\x1a\n\xcbI\xa2\x06J\x1aO\x0e\x81\x05\x91U\xc0Y>\x8d\x0f\xa3`\xee\xd1\xbd*\'\xec$`\xe1BMHq\xe9x\x95\xb0\xdcB\xe3T\xce\x04F\xd4*\xf6B\x00\x1e\x19\xb9\xbe\xef\xde-B\xff!\xe3\xebk\xb2\xe0\xc8\xda#\xaf\xe5n\xd5\x12\xf24l\x07\xeb\xec~x\xcc\xe8\x8d\xb5\xd23\x0f\xc2\xd6\x14\xa8\nl\x0f\xb4\xea\xcf\xbe@\xa2\xb5+\xa2W\xd1<3\xfc\xcd\x85=E\xda\x17h\xd5\xe0\xfa\x92\xdb<\x19\x9a@\xe5\x98\xe7\x9dt\xcd@\x8f\xd6\xf7\xc8\x9b\xc5\x8b?e|=\xb7\xb7\xf5|\xa3\xd6[\xc0\xcb\xf6&B\xa0\xa1\xee\x9b\x02m I\x98\xfb\xc3c\xd6A\x88\x85,\x0e\x18X\xd9L\xd04\x8a3\xb1\xb6\x10\xf0\xb8\x84\xd1=\xdf\xc3P5\x98\x8c\xab\xef|\x8d\'\x1d\x97\x81\xd8\xbe\xeeB\x08\x12\x01\xdda\xfa\xbb*\xa4\xac\xf8&\x81\xf3)\xe3\xebk\xd2\xae\xb9"\xfe\n\xb6\xf1\xf9\xde\r\xacl0\xb5\x03\x16\xc8\x0b\xc0c\xe0\x8e0\x15c\xd1e\xf5\xe1\xb5\x90~\xbd\xed\xb8\xdck\xaf\x18r:Qc~\r8\xbf\xa7\x9c\x13\x05H@\xd8\xd5\x83\x80\x89\x1c6V\xda\xb5?\xdf\x813\xaf\xcc\xd4-x\xb7\xd6\xfe\x87\x88\xafg\x94(\xfa$\x0c\xd1\x97Z\xfb\x83N\xdf\xc8H\xeb\x8bq\x14\x1e!\xe1\xa0nz\x01\x12\xdf\x1cm\x18\xde\x11\xc8=\x8d\x9c\xa3\xef\x92\xebO\xda\xfb\rM\xa1*s\x98\x96\xd0\xb5,\x97\xdaIw\rm\xd2\x18\xec\x93\xa4\xba\x9f\n\xe5z\xb5N\x07\xf7\xc8\xbfY\x89\xfaS\xc6\xd73L\x82&\xbe\xeam\xbe\xccY\xd3\xf0$\x1b?\x91^\x89\x1a\xde\x989\xb8r=\xfb\xc7\x03\x95k\x80\x9bB{(8=\xd4\x8c\x05\xf0\xce5\x17\x0b].\xbb<\xd2I4]\x8f\xaa\xe6\x91j\xbe\xe8\')\xd6\xdd\xf9\xfa\x88@Eda\x96M\x84o\xee@\x9f2\xbe\x9eK\xf3Y?y\xbd\x92\xbe\x9c\xca\xe8\xe32\xed\x01[#&\x86H\xc5\x13Yh\xd0#\xae]\x0f\xd5\xfb\nB5\x95/\x86\xc7\x01?\xaf\xf1\r\xc5\xe1\xec\xb6\xb96,\xbbx<\x80\x8b\xb4G[E\x82\x89\xc5\xd5!\xb6\xbd\x81\x97;.\xcbP\x90\xeb\xdf\xac\x1e\xed\xa7\x8c\xaf\xe7d\xa1I\x14C \xeae\xb2\x9c\x0fX\x05\x81\x18S\xd1\xa8%\xf2^I\xf9p\x7f\x01\x07\xbfG\xd7[`Z\x89\xa7\xb1\x08\x86uW\xa84[\xc1\x9d\xc7a\xff\x10\xfc)\xde\xe5\x19\x04^\r6\xef\x86,\x0c\xe1\xa2\xd4\x15MU9\xbe\x16\xc1\x9d^\xbfYk\xffS\xc6\xd7\xdf?\xbd\xf8\xeb0{\xed\x10\xf0\x1eE\xf4~\x14\xde(1M\xef\x99H\xdd\xd1\x89\x84;&\x13O\x8f\xb1\x01\xae1\x96\xce)/U]\x9b\x14\x19\xe8\xf6\xcc!\n9.X\x86T\xef\xb5\xa3\xe1Q\xa8\x12\x18)\xa1a\xd1\xa0\xf4\xe0\xb9|\x97\xa2\xfd\x90\xf1\xf5u*Sk\xa0\x04\xf4\x92\xc2\xf1\x8aK\xd0@\xcd\n\xfe\\@\x07\xde\xe4\x1e\xe5\xbd<\x9c\x067X\xb3T\x02\t-\xf8\x9a\x98\x9a\xb14\x18IH!\xa5\xf9\xc0\xd5i,\xe9Q\xefJ\x17\x9b\x8bT\x92\x0e\xb5,\xda\x99\xa7,t\xc4\xf6 v\t\xf7?\xaa+\xfe\xe76\xbe\xbe6Pr\x9dw\x10\xf9\xd2\x8b\xb0\xb7&\xbe\xe54N]Y>|\x8bd\xf3\x9d:\xdf<\x81\xef\xf6Z\x83g\x89\xae)\xd3"\x9e\xb2y\x877\xe9\x9d\x05\xf2\x91\xef\xf8\x1dhj\x15\xb2\xcb@\x8d(\x19%i4\x17l\xcds\x0b\x1fy\xf3\x049\xcc\x9b\xd5\xd9?e|}\x1d\x87\xf0\xb3\x94?\xf4R\x8az\x00\x83n]\xcb\xb2\xc7w\x8a6\xa8.0\xd2\xd0\x1dh.\x98\x93\xc0\xa5Z\x96\x87\xc4m\xcf\x13\xa1\'\xf4\xa2\xdd\xed\x83\xc4\xb9\x14k\x9bm\x91\x91\x94\x8cuC\xa3TL\x96^t\xd6\xc2\xf0\x02V\x14s\x04\x0f\xf0\x9b\xc7\xe1\xa7\x8c\xafg\x98\xd8zN`8\xf9\x92\xc2u\xee\xbe\xee]e\xac\x97\x0c\x1f\x12Y\r\xd7\x1b\xf5\x10\xc7\x08\x06\xec\x8c]-\xb2\x08\x18\xd9\xae\x7f1\xed\xfc\xd6`S\xbf\xab/\x03\x87\x81}\x167\xb0}\xb9h\xfeu\x1f\xdf\xc5\xb3\xc1\xdd\xd5\x92"\x94\xb8\x88\x01\x15zW1\xfc\x90\xf1\xf5\x15&\x8e\xa34\xfa\xcal\xd4\x18t\x15HO\xc2\xcb\xe5\\\xaa4A\xf4\xdd\xc1$\nc:\x9a\x03\xb1\x1f0\x9aR\x11\n\x1c,btm@\xd7\xb8\x13\xcc\xe5\xa8\xb8\xafMQL\x1f\x19\x1e\xdf`\xcdK\x8d!\xc82\x95\xd2\xe0\xd84\xa4\xd2{7\xb9\xf9\x90\xf1\xf5\x0c\x93ZW\xf2\xba\x93\xbf\x9eWK]`\xe0\x15\x9b\x01\x03\x87\xd0\xb9\xbd\xe8\xebJ\xc8\xcf\x8f\xae\x90\xbb9\xd8\xe7hj\xd3\xe3\x02\xeeR\x9c\xcfS$,\xfd\xb9\x8b\xa3\x84\x88\x02O\'\x17!\xeb\xe8\xbc,\xe4+\x8d\x16\x1d\xab\xb4\x97B\xcb\x012yWl\xfb\x90\xf1\xb5\x86\xb9^\xe41\xe2\x15N\x80\xfd\xb3\x96\xcc\xfd\t\xc8\xf3\xc3U\x1f@\xc2\xd8g-\x0b\x18:\x1b6`j\xf97\xcf\x0e\xd5Q\xe9w\x94\x8a\xcaW\x8ah\x93\xa4ni_&u\xd7\x07\x84\x16pJ!\x88O\x17\x00\x83\xccT\x16\x9di]\xe2\xfbw\x0f\xe5\x0f\x11_\xcf\xb1\\\xd73\xb1\x1eX/\x87r`\x95\x84wL\xaf\xb5\x1cr;4\xb2{D<5\xcb!\x8f\xdd\xabaP\x0f\x11\xbb\xaa}\x95\xa5\xd7]\x1dY\xb1\xb7\x082A\xed\x95B\xcaKO\xf2\xe0\xc0%\xf7Q-)\xb1\xdd\x94\x12\x0c\x18f\xe7\xd8\xb4qx\x13\x19\xfc\x14\xf1\xf5\xfb\xca\xa41\x8cx\xddg\x15\xc4\xbf\x9e97;\x9c)`=X%\xa8\x1a\xa7\xc4\xb2\x07\x17\x8e\xa8\x1cv\x9d\x93l\n\x9eZG\xac(\x07LV\xefnNNy\x92\xd2\x0c \x02\xa3c\xe4\xd8}\xe2\x1f8\xbe\x04\x81\xc1|\x94e|=\xe3\xbb7\xa7\xec\xa7\x88\xaf\xaf\xdc\x03\x85)\x84z}\xf0\xa1\x9b\x97Z!\x16E\xb3E\xc6\x0b.\x05\xae\x0b\xbd\xc1\xa6\x8cZL\xe8Am\x92\x98\xbb:\xd2\xd9>\x98t\xcc\x88\x0f\x98q\x90\x8b\xe8\xc0\xd7@@{\xcb\xe0\xe7B\x9a\xd9\xf5\xcaf\x1c\xea=a@j\xa4\xed\xae\xa1\xf6#\x9c\xea\xcfM|=\x13U\x84^/\xb3\xf0\xebd\x811,S\xa8\x00M\x94\x0b\xa7\x1d\xf9\x91\xbb6\xe6xFp\x170\xe6\xe8<\xc5\xa7\xf3\xd9\xab\x05\xf1X^Gf\x9f\xa7j\x82$v\xa3\xe9zYs\x87!\x94T\x8b\xd5v\xb7\xdd\xbd|\xb8\x8eF\xefm\xb1k\x89\xcb\x9b\x93\xe5S\xc4\xd7W\x98\x18\xb4\x1e\xca\xaf"%\x06\xdfn\x08\x90\x82t\x7f\xc8\xd998\xea\x18\x01\xec\xec\x86e\x12J7\xa98\xbfS\xa0\xe9yW\xcaC\xe5Z\xcc\x17\xe1\xac\xb4*G\x98\xe0<\xeaVr\x15\x1e\xca\xa9;\xe2\xc7\xd3\xfd\xc0_b4\x11\x0fae\xb2o\x86\xf9)\xe2\xeb\xb9&\x10\x02\x81\xe9?0S\x1a0\n\xf0\t\xd4\x19\xb1\xf0.bn\xe2\xd2\xa3\x95,5\xc2\xc4\xde\xd5\xf1z\xd9\r\xe7I\x9b\x8d\x0ei\xa1\x1a\xa6y\x9f\x0b"\xe7\xd4\x9cmh]\x8c\x1e\xc6\xec\xee(\x8a\x0fmm\xa4\xd3\x91\xbdf#\x12\xa20\xfd\xae}\xf5!\xe2\xeb+a\xa6 \x8c\xc6\x88\x97\xdc\xa3+\xb1K\x1f\xc1\xda^\xd0=\x18\xb7\xb0\xe2q\xcelv\xbc\rRm\x1e5\xe2\x10-m\x06!G\xae\xd1\xc7\xaaT\x80\xbb\xa8\x00Mv\xa7\xdaLm\x0b\xee\xa0\xc6\xaa,\xaaW*\x0b\xc6S\x9e\x85\xbd&Ub\x91\xff"\xe2\xeb\xf7\x84\x99^\xfb\xff\xf5#\x1d=E\xf7\xbaE\x92#\xe9Ss\x0b\xa1\x91%\xa1\x13{\xbf\x95\x1d\x05\x98\x07\x19\x1a\xea\x1d\xca\x1f\x1e\xd0}\'\xb7|RM5\xd9\x88\xaa\xdc\x11;\x10\x9av{\x15\xa2\x16\xf6r\x07xE\xcb\x98\xe1V\xc5\xf9\x85\xce\x947\x1f\x93}\x8a\xf8\xfa\xba+\xa3\xebM\t{\x957\xd7;\xfcxC\x83\x19Z\xc6\x9e\x8c\xe0,\x8e\xe0\x81\xb3[\x0ey\x18u\xcf]\xe1\xc3\\\xe0\xb5\x9e\xd5\xa7\xa0\xd9\x91f\xc0J7\xec|m\xa5\x19\xcb\xaf\xd4\xd2\xe9)+\xc4\x98\xec\xb3\x15{\xbc\xa4\xd2\t5\x13E\xad~4\x9a\x7fn\xe2\xebk\xe9#8\x8cc\xafW\xf1\x9e\xc48\x02\xa7\xc1\x12\xc2\xe6\xb8\x07\xfcCwU\xc5\xa8b}\xf5\xc1\xa3\xe4\xdd\x0c\xd4\xca\x81h>q\xaa\x9d|Xo\xe7\x9d\xb4\xa3\xaa\xb0.\xc8\xd8\xc6u\xcc\xbf&2H\xc4A\xa4\xd3\x9dU\x18\x03\t\x8ap1\xbe\xf9\x14\xeeS\xc4\xd7W\x0e\x87\x92k6\xf22Wj\xfa\xe1\xa9\xf1\xc34\xbc9\xce\xafqq\xbd&\xb3\n%\x97\xc7\xa3\xab\x1f\x80B\x9er+\x84\xe8\xc2`w\xc7\x91\xe3h{\xae\x97\xfd\x02\x81q\x9d\xe9d\'#\x8d\xce\x94\xe3\xb4\x17L\xc2\xf2\xe9\xcc\xbc]`\xa0\xac\xdf\xfc\xf8\xefS\xc4\xd7\xf3\xe3?\x08[\x87\x12\xa1_r\x9b\x8eh1Uc\xc2\xf2\x1c\xc5g\xf6~t\xfa+\x7f\xca\xe2\x04\\PI\x0f\x9d\x193\xd0\xd6\x9bQ\xf9\x90\xef\xfc2\x86\xd7\xf3\xcb\xb6\x8d\x99~\xf8\xfc\xa9;\xb7\xb8A\xaf\xf3\x8a\xef\x85Kp\x805-{\x0cw\x18\xc9\xdfTZ?E|\xfd\xfe\t \xf1|\t\xe3\xe5\x01\xb9~\xd7\x8bYE{?\xaf\xc3\x99u\x15v\xff\xd8\xc1\xd3\xc2\x1c\x95\xbe\xd0\xce%\x86\x94\x15bE\x80\x87\x18\xe5>l\xcf\xf6\xae\xaa.\x8c\xb3\x83Kd\x9d\x02\x8cx{\xf0E\x07Z\xca\xc1>\xc8\'W\xe3\xc1\x9bY\x97?\x1a\xcd?7\xf1\xf5\x9c,\xc8\xfaW\xa1?\xf8h\x8cja;\xc2!\xe5\xce]\x86\x10\xc2u\xbd3\xc5D\xc4\xb4\xfe\x11\\\x86\xb3\xc9\xabuq;\x9c\x91&\x15l^\x9d\x8e;\xa0/\x9ba\xc1\x1e&\x96<\x08\x11\xe5:\x84\x85S\xb0Ok\xca\xde\x89\x07\x9f\x9bL\r\xfd}\xb2\xfc\xd7\xc4\xd7\xd7\xa5\xf3\xaf\x89\xaf\xdf\xdfW\xdc\x98\x99\x1f\xbc\xef\xfeo\xc4\xcc\xfc\x89h\x84M\x9b\xd8\x98\x99\x8d\x99\xd9\x98\x99m\xe1o\xcc\xcc\xc6\xcc|\xdfvn\xcc\xcc\xc6\xccl\xcc\xcc\xc6\xcc|g\xbeecf6ffcf6f\xe6\xdb\xb6scf6ffcf6f\xe6;\xf3-\x1b3\xb313\x1b3\xb313\xdf\xb6\x9d\x1b3\xb313\x1b3\xb313\xdf\x99o\xd9\x98\x99\x8d\x99\xd9\x98\x99\x8d\x99\xf9\xb6\xed\xdc\x98\x99\x8d\x99\xd9\x98\x99\x8d\x99\xf9\xce|\xcb\xc6\xccl\xcc\xcc\xc6\xccl\xcc\xcc\xb7m\xe7\xc6\xccl\xcc\xcc\xc6\xccl\xcc\xccw\xe6[6ffcf6ffcf\xbem;\xff5)\x02\x7fK\x8a\xc0\xfe\xa6I\xdf\x9a\x99\xf9\xe9\x81\xfd*f\xe6\xa7\x07\xf6\xab\x98\x99\xf7\x02\xfb\x17\xd0\x92_\xc5\xcc\xfc\xf4\x11\xfbU\xcc\xcc\xcf_c\xff-\xcc\xcc\xef\xff\xe3\x7f\x12\xea\x02\xff\xd5@\xfec\xd4\x85\x128\x11\xe7\t\x9eB(\x8e\xa5h\x8e\xe31\x11&D\x1e\x85Q\x8ae\x05\x1eb\x10\x08\xc59\x11f\x11\x86\x87p\x9e\xe2P\x9a`(\x8c\xe4D\x0e\xa7\xc9g\x81\xd1\x1fk\x05\xb8(\x8a$J\xe3\xdc\xb3&\x1a\xc2\xa0(\xc4\x8a\x0c\x84\xc2\x9c(\n\x18\xc1\xf3(\'R\xcf\x8a\xa5\x08\xc9\x10\x1c\xcd\t4I\x8a\xa8H\xd1\x98\xc8b$,0?\x13u\x81\xff\xb3^\xc1o\x7fT\xaa\x80\xfc\x0b\x8a\xe2(B\xac\x8d\xfdG\xa8\xcb\xf7\'q\xfe\x00u\xe1E\x8a\xe5E\x04c9\x88a1\x9c\x11Xb\xfdn\xd8:\xb28\xc4\xc2$\xb2\x8e\x12\x82\t0.\xb2\xb0 B\x18\xf6,}\x0e?\xeb\x83\xf04\xbcN\xb6\'\x1a\xf2\xd3Q\x97\x0f\xd0+\x7fZ\xd4E$\x08JX\x9b%\xae\xe3-`\xcf\x1a\xbf\xcf\xc2i"\x05\xa3\x08IQ8\xf7\xac\xd3.\xd0\x18\x02\xad\x8b\x06F\xd6\xa6\x0b\xb4@}Q\x15\x10\x06\xe1\xb8\xf0\xdb\x1f\xa3.\x1f\x18\xa7\xff\x1e\xd4\xe5\x9f\x96F>\x87\xba<\x17\xe2?D]\xbe\xffB\xff\xa5\xa8\x0b\x02\xff\xb8\x8c4C5\xaa\\"\xc9\xcd\xd0\x10\xea\xfe\x18\xfb\xc4I\x81\xd0&.\x87\xc2\x86\x04\xc4\xa5\xd2\xe3z0<\x1a*n\x9c\xab\xb6+\xc9\xeb\x1d\xf7\xacS+\x9d\x9a9\xbe_\x9d\xb9\xa0\xf8\xb6A\xa2\xfbu@\xae9\x17\x9c\x1fM\xc5\xe7w\t(\xfbq\xc7B\xf9D-\x93\xd0]\xf0#Q\xb0\xedaG\xb1\xd8H59\xe1\xdd\x1d\xce\x91\x93\xfe\x1cw\xb3ut\xd9w\xab\xe4\x7f\x08uy\x86\xb9\xfe\x0b\xe1\xaf\x95j\xf3\x0b\x81\x8c\xf89j/\xf6q\x1f`\xc4m\x94\xaf\xae\x15\x8b\x87\xd9\xa9\x8e,\xb9\x03)\x93\x92N\xd8B\xe4\xc0\x10\xd6w\xfe\xb0\xdf\'{\xd4\xbb\xb0\xd9T\xeat\xe2\'<\x0e\x8d\xe7\xa6h\xfdZ|8\xed\x11V\x98\xf9\xcdjn\x1fC]\xd60\xf1u\x0f_3\xc0W\xd5\xa5\xc4RDJ\xe7@\xef)3\xa0z\xe0N\xb4\x015\xa9{\xec.[*\x96\xd2\x0e\x9eJ`\xc4v\x17o?8\xe75\xcf+v\xd2U\xb8Q\xbb\xfb#\x91\x14>\x13\xbcy\xe6\x1a\xdb\x9e\x07) \xd5\xd3C\xd7\x8b7\xd9\xaa\x8f\xa9.k\x98\xcf\xf2\x83$\xfa:\x9aGlR\x07\xfd\xe4\xd3\xd2x<\xee\x8a$a\x02)VXO\x1d\xe5\x0bp\xbe\x80Zv\xedks?B\xec\xa1\xaa\xc0\xe31\xc6}\x8e\xbfG>\xdbQ\xf1|\xad\xddJ\xec\xe7;\x1f\xd5\xbb\xd4\x90\x93#\xdb\x8f\xe9qx\xb3\x04\xe1\xc7T\x97\xe7h\xaei\x10\xf2ZIV\xe3(\xd0\xa6b\x91\xbcA\x07,\x8c\xf6H\x0e\x9e4\xf9\x0e\x0b\xfa\x0e"\x02\xfb\xce\xed\xb9C\xc9\xed\xcd\xfd\x1c\x1b$)x5\xe1\x03Q\xf8\x90\xa0\x04\xd1\x17 \x8cr[\x88\xf7\xfb\xfc2Q8=T\x110\x13\x1d\xfaf\xed\xd1\x8f\xa1.k\x94$\xb5^\xc5\xa9\xd70e\xd2)T\xfbQ\xc7X\xcb\xec\xec\xb8\x8f\xbb\xb1q\xf5G\xa6\x94\xc73\x81\x86U/\x9c#\xbf\x8coL`+"\x17Kw\x93\xa6"\xc6\xc1\xe42\xa1\x86\xbb\xe5\xd0\xa5\xb4\x97\x86\x8b\xe0\x97bv9I\xe9I3\x80\xefUD\xfac\xa8\xcb\xda\x8b\xd4\xba\x8b\x10\xeb\xdd\xe4\xb5\x1e\xefN\x95F\xe2\xe4\x1cg\xcfEvX~*Z\xb0\x8e\xccs\xa1\xb7\xfd\xb1\xe3\x95\xb9\xf5\x00Ro\xea\x9bE\x06.L\xda\xe8qI\xddYH\xaf)\xdf\xb7p\x9c\xe0 \xa2\x97\x1d/>\xe2\xb4g\xd2\xe3:\xc4\xf0\xbb\x06\xd9\xa7P\x17\xf2/\x08\x89\x90\xd8z\x8f|\xd9\xe0|\xb1\x14\x8bt\x10i\xf8\x06\x17\xed\xd8\xe4\xa2\xe8\xa5\xfeL\x0e\xe0\x81\xb5-\xae\'\xcex\xbe\x97n\x84\xd7JZ\x8f\xe2T\x06 D\xbe^\xd9h\xc7\xaa\xf5\xce\xd5o\x8cqK(\xda\xfb\xf6\xcd\x14\xeec\xa8\xcb\x1a&\x8d\xc3(\xbd\xe6+\x7f\x1f\xe6\xc5\x8b\x1b7H\xa1VP\x9d\xe1\xa0\xd4\xf0\xb0s\xc8\x82f\xeb\x05@\xd9\x01\x12\r\x89oN8\x918\x06\x14\xd3\xd0N\x85z"6\xa4D\x81j/\xa5\xc5\xfez\'\x0fLT\xc3\xe2A\x86\xfc\xd6\x93.\xa4\x8f\xa5\xdf\xab\x1e\xef\xc7P\x97\xe7\xfe\t\xd1$\xbc\xe6}\x7f\xdf\x8b\xf3\x03\x81\xc9\xd6\xca\xaa\x93\x9a\x00\xa7#\xd3\x81a\x02Ywl)\xb8\xeb\x9d\xbb\xeb\'\xf3\xda(CC\x96\xfe5!\x9a4\xae\xf7\xc7\xde\x96\x8d\xd8S \x1f)]\xb6qi%\xa5\rU=\xde\x10\x1b8^\xa1I\x06\xdf<\xf4?\x86\xba\xac\xf9>\x06\xaf\x1b(\xf9Z|4\xb9\xa0\xbb\x94\x13\xd7\x0b\xf5r\x81P\xbd\xf5\xef\x97\xben\x8b\xb4\x86\xe12\xc0\x86\xa48\xdd\xd6k\x9d\x92\xc2\t\xd1F\xec\xe1\xae[]\x12Z\x82w{\xc0\xc7\xc6!\xcf\xea\xdc,\x16\x15d\xc0\x08\xef\xdb\x8b\x94$\xd6]x\x93\x1d\xfd\x18\xea\xf2L\xc8\xd7\xfeZ\xefo/\xd7\x1a\xfex\x0cXZ\t\xc4\x10\x93\x11\xb4I\x02\x9b\xe8\x8en\xdd\x85\x16\xbbwB\xdb\xbc0\xf8\x91K1\x9d\x0fCUP\xed[/\x88F\xb7\xd0\x93~G/\x97Di\xec\x02i\xb3\xd0\xdf\x9f\xdd0\xad\x07\xec.t#\xfd\xe6q\xf81\xd4\xe59i\xa9u\x03Y\xcf\xc3\xbf\x0f\xd3\xc1\xb2#1\x9f\x03\xcc\xba\xf9\x04\x83\x95\xb0\xc5\x17\xc9\x12*\xc3\xb1\x83\x0b\xeb\xaa4\x1a\x17\xa3\x8c\xe52*\x91\xb7X\x988j\x03\xbbBd\x98\x18(\x83\xd3\x1c\x05!\x18\x1e\xb0\xce\x8a\x96\xeb\xe2\xda\x92\xa9\xea\xb7w\xb5\x93\x8f\xa1.\xeb\xa4\x85H\x8c\xc40\xea\xa5\x8at\xae\xcd3B{0s\xa3\x85\x19l\x89iZ\xf8@\xa2\xccI \xcf%\x80\xc0\xd90\x85^ugKb^P&}x\x92P\x05\x1ds\t\x1d\xa6\x16\xe4\x9dws\xe5a\xc0\x1f9\xd5xKd\xf4\xa9\x1c\x12\xde\x9bk\xf3c\xa8\xcb\xba\x91\xe3\xd8\x9a\x8e\xe3\xafb]\xc2-\xa9&\x08\xa0[n\xebHFm~\x8b\xd9\x9e>\xa3\xf8NA\x80\x07@\xff\xc8A\xfc\x93\xb3.\xebd\x81ih\xbda\xbd\x1a@\xcb\x99\xad\xf6g__NY\x04K\x9c9\xa9)=$\x97^l\xd85\xa5\xc3\x81\xca\xf7\x0f\x07G%b\x01\xcc\xd1\x98\x8f\xc6\x86\xec\x8b1\x89\x1fx\xd3\xee\x06`\xe7\xf5g\xd8\xa8\xb2\xe9\x06\x8bm\xe45\xd3#%\xe4w?\x1a\xfb\x14\xeb\xf2Ln(\x0c"\x90W\xbd\xa6v.P\x0e\xb5N\x0f\xb3\xc2D\x05Z\x0c{\x16\\/\x97\xe3\x85\x19\xf6\x18\xdc"#9\x90\xbbX\x8c9\x00IK}\xce\xbd\x91<\x8f\xc2\x03\xe0\x93\x0bh\xca}D\x96\xe7\x93\x13\x04\x8c^\x10\\\xbe\xc0\n\x8d_\xdf\xcc\xe1>\xe6\xba\x7f\x97\x9f~VM\xf91X\xc0\xe2$- 8,\x8a\xebO\xc11\x8c\xa2\x08\x81\xe0\xe0\xf5o?\x0b\xbea(I\n8\x85\xd1"\x83=\xeb\xb1P(C\xf2$C\x0b\x9c@\x11\xcf\xb2\xed\xd0Ou]\xfe\xb3\xea\xc3o\x7fP\xaa\x00E\xffB\x11(N\xe2\xf0?v]\xbe\xbf\x8a\xf3\x07\xae\x0b\x83\xad\xad\x14E\x84\xe5YDx\x96\xc7^\xbf\xef\x93p D\x02fi\x06\xc2q\x86[\x1bH\xe3$\x03\xf1\x14K\xa2\xe4:\xf6\x0c\x85\xaf\x03\xc5\x11\xeb\xf8\xfc\xb6\xb9.?\xd5u\xa1)\x8agx\x02\x121\x94\xc5\x9f\x15>E\x9a\xa0\xe0gEAlm\x15\xc2\xa1\xc4\xb3F$\x8a\x8b\xeb\xb7}R<(\xb4\x8e\x0b\x0cQ\x02\xcf\xf2\xd4\xfa]\x7f\xfb\x93\xbb.\xff\xb4\xc2\xf11\xd7\xe5\xab\x04\xc8?t]\xbe\xffB\xff\xb5\xae\x0b\xf5c\xf0Dhqe\x17\x99\xe1>\xe4)\xdb\xb8{\xe5\x83r\xf2\xe5\xba\xa0\xd8_ t\xed\xa3\x97*=}3/\xb0[,\x13K\x96I\xbf$\x00\x9dK\xca\xa0\xb7\xbc\x1a\x1ds\x1d9\xca}\xbc\xcf\x8a\xa3\xbc\'\xb1\x07s\xbdab\x93jx\x83P\xbaZ\xe6\x84U\xb5\xb2 P\x08:\xa0\xb1\xab\nR\xc5\x9cQ\xc2f\xde,\xcb\xf9)\xd7\xe59\x94\x18\x8e\x91(B\xbdT\xaejl\xf6\xde\xd3D\xad\\\xb8\x83Yk\t\x07\xc1\x14h\xb9;1\x9ev\xc8Y:s\xf5\xc1\x9a\xf5\xae:\xd3\x805\x85\xcc%\xacQ~6\x8a\xb0P\xab\x9b\xa1\xbb\xc7\x02\x11\xf7\x93\xcd\r\x8bp\xe5]\xab6<\xe7\x01\xbcIu|J=x\x86\tS(\x82\xc2\xaf\xb8\x83ek\x0f\x06z\xc4YF\x9f\xefw>\xb8c\x90\x08w:\xa8\xe1d\x01\t\xd8\t\x8b&S\x04]\x0c\xc6\xf7\xda98\x99P\xe8\xfc\xc8\xb0\xf8s\xf35_\x93\x85z\x96>\'^\xeb+_M\xab\xa9\x8ef\xad\xa7\x80-V\x93\xccS\x16\x1bA~[\x1c\x8f{\x9a\xea\x82}\x0e\x12\xbbu\xbfI\xf4p\xa2A\xbe\xcc\x83\xce2\x8a%\xde\xcfw\xbaX.-\xa6\xbax\xd5[Qi\xcd\xfe5\xbb\xde\xa6\x02{\xb7(\xf8\x87\xf8\x9ag\xd6\x8b\xc3\xebf\x01\x11/\xb5\xf9Z>!D\xea\xda]\x96\x81:\xd6\xc7Nm.\x9cy\x93\x1c\x18n\xdc\t7\xef\xac\t\xb5S\x92p\xd1\xe9L\xb1R]*g\xf9\x91\xa0\xce\xe3\x88\rRBu\x80\x8f\xda\xf4^\xe9T_\xd0\x06\xcb\xa0\x04{\xdf\xef\xdf\xf6\x00>\xc3\xd7\xfc}r\xff7EV\xed\xc7q\xe6\xc6T\x9b\x17\xd1\xde\xc32g \xe7\x80\x9f\xb1\xae\x96Q\xbd\x8e\x82P\xbdp\xc8\xccD\x9e\x98\xb1\x02#\xf7\x8bj\x04\xd7\xab\x7f\n"c\t\x17p\x94u\xf8\xce\x1a\xa9\x00\x92\':\x87\xd4J>\xf4\xfe/\xe2k\xbe&-\xbc\xde\xed\xd6\xbc\xf6\xef\xc3\x0c\\\xf8l\xe2mj\xa0]\xba\xb4\xd6\xe9tB\x80\xb2p@\'\xaa\xf6\xa7*\xc3\r\xf98f\xf51\xbd\x91\xf8\x98D\xb5\xb0N\xd8\x81\x0e\xe4\xc7L,\x8dt\xf1vh\xe5%!\xf5\xb8H\x01[\x02\x8a\xe9\x9ev\xe3\xf4\x8b\xf8\x9ag\x98\x08\xbc^[\xd1W\xa5\xe7~\x1e\xea\xe3\xa5\xbc\xeag\xb3H\x19\r\n\xe0\xd2WlNW\xfd\x16\xef\x19\xbd\xcbp\x15i\x9c\x83\x1b\xefw\x8c7\xb6\x03|\rU\xff.;\x1c\x19 W*U\xca\xb0\xe7\xf6\x94\x06^\xfa\x06\xa8\xf70\x9bL\xe9\xed\xcd\x8d\xfcS\xb0\xcbs\xd2\x92\x08\xbc^I\xd0\x97\xd1\xacm;F\xae7G\xf4\xae\xf6\x0e`$\x1d\xa79\x11`\xeaX\xa3nX\x859\xb7!\xd9\xd5\xbb\xde;\xebB\x01\xae;\xa2\xe6\xf3\xa6l\xf5G\xb3\x94\x1f\x8d\x95\xb7\xbc\x9c"9*\tK\xa8\x19<\xea\x0c"\xee\xf5?*\x80\xfc\xe7\x86]\x9e\x99\xcd\x9a !\x08\x85\xbd\xb0\x07\xa9\xbe\x07\x13Y\x06\x8e\x04+\x98\xa8\x97t\xea-B\xcf\xfe\xcei\xbbH\x8d$}\xac$ \xf2\xbc\xc3h\xa67\xc3\xc4`q\xcdZO5}?\x0b6\xeeJ\xe7]\xd9(\xc7\x88H\x92\x93\t\xac\xd7A\x89\x90\x11\xb7|\xd3\x1f\xfc\x14\xec\xf2\\\x13\x10\xfd|4\x04\xbd\x9cWU\x15(F\x10(%\x03)\'\xdd\xcdq\xd9\x17\x17\xf4r\x9a\x8e\xcdL\xb1>z\xcd\x1eE%\xee1v\xa7\x80\x8eJ\xe8\x15^\xeejZuI+\xa9\xe2\x070rx\xc8_=\xca\xa4o\xfb[xQ\xe6\xf5b\xfanY\xceO\xc1.\xcf0\xd1\xf5\xb6\x8c\xd0\xd8K\xaa:T\xae0f\xfc\x95\x83"\xa7 \x9cT=\xcec\xc4X7S\x1c\x0f\x9a\xa9b\x9e\x7f\xb0\x17>\x1d\xdd#\xa0N;\x17\xf3\xa3Y\xb5\x110\x82t\x15\xae2\x99m\xf5\x9c+p\x1d\x9c\x08&\x1b\x86\xa0\xddW\xda\xfeG\xf5\x95\xff\xdc\xb0\xcbsg\xa1`\x04\x82\xc9W%\x01\xa7\xa1;+ {\x89\n\xd4=\x7f\xe3\xc4\x83\xd5\xdf"<\x8e\xc9\xf3\xc3\xb0\xeb\xc7\xd9\xee\xfa\x1b\xae\xdf\x0c`6;9\x0cr\xbd\xbcU\xcdz\xa7\xa3\xf0\x1d\xa1\xa3\x07\xa0\xc4\xaa7+\xc8\x7f\nv\xf9\xda\xc8\x11\x12%1\xf2e4a\xb9\xb6k\xaa\x9c8pVbyw\x1e\x0e\x83h\x85\x10\xea\x12\xe1\x00\x84U\xda\xee\x92\xfb\xac\x86\x9c@D\xe8lkN~\xcfD+\xd7\xf7\x169\t\xa8\xe6\xb9\xeaz-\x1fj\xe1\x86f\xc4\xa4\x94\xb8\xc8z\x8e\xf4\xee\xb1\xfc!\xd8\xe5\x19&\xben\xd14\x05\xbf\xacM\x88#\xc2\x98.t\x17\xb2\x0f\xa36{~J\xe8\xdc\x1e\xbc\x8e%\xe5\x8d\xfa9\xf4"`\'9\xcd\xc5\xaa\xcb\xf0$a\x8a{H\xafc|\xba\x17\x9d\xf38\xf5\x052G\xa7\xd6\xae\x0cC%\xb4\xe5\x9c\xefn0\xdb5o\xb2r\x9f\x82]\x9e\x07\n\x06\xe1\xeb\x97_/\x1e;\x93\x06\xe9A\x1e\x9b\x9cf\xee\x87\xf0\x9cug\x9f\x02\x93Zr}\xaa\xd2n\x1di\x14\xa0\r\x0cG\xc9 \x03\x0b\xdfu\xddnN\xd8xO\\\xe4=\x0c\x90\xf04h7\x1b\xc3\xa4s\x03LJ\xc2\xb9\xe6=\x86\x10\xe6\xcd\x8c\xfcC\xb0\xcb\xd7\x16DC$BB/\xd7\xc8\xfd\xb1\x16\x95\x9dk"Bs\x82\xfa].\x11\xf0]e\xfa\xeb\x92\xdc\t2\xb4/\xe1\x00r\xdd\x94\x9c\xe0\x94\xa2y\xb6\x11\x88\x01B\x00A\xf6\x10\x184\xaf\xd2\x9c\xf6\xa6\xc9\x02w-\xc7\xe9v\xc2\xe3\x96\xeb\\S\xff\xd1C\x81?7\xec\xf2u\x19\xc7\x90\xf5\xbeG\xbc\xecp h\xb5)YE\xb7\xc4\x1dE7m\x89e\xf6\xb1\xda\xa6\x0eJ\xadZ\xfd\x9d&i\xcdE\x8f\x16\xafy\x98w\xc3\xce\x83y\xc0\xaf\x8a3t~\xe3\xe2\xebB\xd7\xe4E\xb3CJE\xbc\x19OS1a\xcb\x96l\x7fTZ\xfd\'\xc3._\x93\x05\xa5Q\x8cxe\x0fT}\xdf]\x0f\xf2\xb1\x12\xb9\xbc\xa7]\x06\x9aI\x8c\x19\x92#u\xbcu\x0bp\xdd\xcb\x96#\xf2\x87(\xb2p\xb6\x9b\x8e\xe0L\xf0Nl\xcaZ\xf3\xd0\xc2\x88\xdf\xa7\x95*\rm\xe6\xa3\xb4+\x90%i1Tyc\xf273\xf2O\xc1._K\x1f\xa7\xa1uR\xbc\x18\x19C\xeb\x11\'$\xea\xbd\xb0\xca\x8e \xed\x06\xa7\xb1\xb8X\xed1\x85\xcb\xd1>&,\x08\x1a\\\xeb\x13S`\xdaF\xd0\xdf\xf4\xa1\x02\x93\xfb\x94\x81\xe4!6\x8f7M\x83zm\xc7a\xfb\x83\x154\x0bC\x85\xe5z9\xa3\xde\xa4@>\x05\xbb\xfc~\xf1 q\x88~\xbd\x8c\xa7@\xbf\xb8\rkX\x05\xbb\xfc~\xf1X{\x06z\x85\xc8\x10\xf4\xc0\x8a\xb7\xf3\x15\r\x81\xd6\x86\xefV\x8e\x15\xa7V}V\x8e\xc2\xc9\xce\xd2T\xc3\xac\xd0\x1e\'\x87\x9b\x81\x1eO\xb1\xf9\x188\xde\xb3g\x91\xcc\xa4v*\xb2\xda\xe4\xd0\xe9\xd4\xd9\x82i\xc5>\xba\x0e\xb3\xc1W\xa9\xfd\xcd>\x1e\xfb\x10\xec\xf2<\' \x92\xc0\xe0?H\xf8\xad\xd6\x92\xea\t8(d\xc3\xa1\x87c\xab\x0bCO\x13k\x8a\xfb\x88\xe3&\x945_B4*F\xc4\x00s\x0f\x80+\xec;\xa1\xbb\x0eI\x1cf\x85\xc2\xa8\xdc]i\x12\x19c\x91:\x84\x08\x1a\xe4N \xee\xf9\xcc\xfc\xfb\xc7\xe2\xff5\xec\xf25\xfc\x1b\xec\xf2O\xbfa\xfe\xef\x03\xbb\xfc\x89\xc8\x8cM!\xd9\xba\xf4\xfbw\xe9\x06\xbbl\xb0\xcb\x06\xbb\xfc\x0f\x84]\xb62\xef\x9b\x95\xb3Y9\x9b\x95\xb3Y9\x9b\x95\xb3Y9\xdf\xb8\x9d\x9b\x95\xf3}\xad\x9c\rv\xd9`\x97\rv\xd9`\x97\rv\xd9`\x97\rv\xf9\xc6\xed\xdc`\x97\rv\xd9`\x97\rv\xf9\xce`\xca\x06\xbbl\xb0\xcb\x06\xbbl\xb0\xcb\xb7m\xe7\x06\xbbl\xb0\xcb\x06\xbbl\xb0\xcbw\x06S6\xd8e\x83]6\xd8e\x83]\xbem;7\xd8e\x83]6\xd8e\x83]\xbe3\x98\xb2\xc1.\x1b\xec\xb2\xc1.\x1b\xec\xf2m\xdb\xb9\xc1.\x1b\xec\xf2\xef\n\xbb\xcc\xef\xd8\x0c\x7f#9|s\xd8\xe5\xa7\x07\xf6\xab`\x97\x9f\x1e\xd8\xaf\x82]\xde\n\xec_aB~\x15\xec\xf2\xd3G\xecW\xc1.?\x7f\x8dm\xb0\xcb\xcf\x81]\x90\xff\xd7\xe9\xff\x18v\x81\x9e\x14\x05"\xb2,\xcb\xb1\x04E\xf1\x14\r\x8b\x10M\xb1\x10O!\x10\xb9\xfe\x83\xc2\x08\x83\xd1\x14\xc5\xa2\x18,r\x02)\x8a8"\x8a\x14\xc4\xd0\x04E\n\xcf\x9a\xd8?\x16\x0b\x10\x8e\x16\x08\xe4Y\xbf\x16G\x11\x94\x14x\x1c\x87XA`\xa8\xa7f\x01\x11(\x8fa\xe2\xb3\'8\x06\x11\x08\x94\xc6h\x84\xe3\xe0\xf5\x8fR8\x86\x13"\xcd\xffT\xd8\xe5?kN\xfd\xf6\x07\xa5\n0\xf8/\x10\x84\x134\x02#\xd8?\x84]\xbe=\x8b\xf3\x07\xb0\x0b\xcb\x120A\xd3(\x82\xa3\x14\xc9\xf0(\xff,9\x8c\xa1\x0cG\xd0\x14\xc7\xafq\xa3\xeb\x08?+5\xe0\x08J#\x90\x80B(+\x92$\x8bQ,B\x13\xf4\xb3\x1b7\xd8\xe5\'\xc2.(%\n\x0c\xf9\x04\'`\x98"0\x82\xc0\x9f%\\1\x9e\xa5PN\xc0i\x01\xe2E\x9a\xe4YFD\x10\x8a!\x04nm\x1bK\xc1\xbc@\xa2\xec\xb3\x80\x13\xf6\xdb\x9f\x1cv\xf9\xa7\x0b\xb8\x7f\x0evy\xce\xe9\x7f\x0c\xbb|\xfb\x85\xfeKa\x17\x14\xfbq\xad\xfc,\xadgQ\xa3\x0c@\x1eN@\xd7X\xac\x1c\x97\xa2\x0e\xc0%\tL\xa0\xaf%\x8e\xe8\x97\xda\x05J\xae\xcd\xf5\x1c<\x04\xb5\x92<\xe9|2.\xb6\xa8\xb6\xa9\xc7\xd1>\x7f\xda\xcfx\xda\x97\xfcA\xcc\x1f\xaaso\xc2\xf3\x9b\x05\xdd>\x05\xbb<\x8f\x05\x04B!\n\xc1_j \xf7d}\xad\xdc\x13\xe52\xc0P\x0e~\xd44\xbc\xde\xdc\xf0N\x993\xc2\x1da\x7f7h4+\x85#\x81\x82\xd7Y\xb2\x92\x91U\xd9s\x85hq\xcb\xb7\xa7\x05g\x11\x05\xdfA\n\x13H\xb5-\xeb\xc5\xae\x95$E|\xb3\n\xe1\xa7`\x975L\x18\x82\x11\xeae(k^\xc5\x1e\xd8\x89\xf7\xae\xce`\xeeE\x90\xbb\x07\xb6\x00\x0e\x0f\xcfK\xd0\xe0\xbc\x0f\x10\xd2\xb43\xee\x0e\xd0;q\x0f\x17\xf3\xfa\x07i\xb0T\x98C>\xefl\x84\x1d\x9c\x18\x08\xae\xadM\xb7\n\xefR\xf7\x03\x15\x80\t\xc9\xbfY\xb9\xeaS\xb0\xcbs(\xd7\x95\xfc\xac\x98\xfbR\xe7\xd9#k\xfb\xe0\xde0\xa3\xca\'P\xe3:\x11\r\x8d]\x1f\xa2\xd3\xe5\x9e\x04\t"\x1c\x83R\x1d\x17>\xe2\xe4\xd3\xdetq\x8d%;\x82)\x9d#\xbfSR\x14\xdd?FI\x8b\x12\x88a(\xc4\x91\x8a\x81\xbb\xbb\x00\x87\xbdI\x11}\nv\xf9JdH\x08[S\xab\x97\x9aK\xae\xcc\x84\x06vp\x1c\xaf\x90\x15\xd5S5\x12\x92\x91\xe5\x16\x14y\x0f\xb3\x8f\n\xcb\xddL\xba\tV\xd5\xc6\xbeD\x1b\\@\x9a\xfe\x1e\xac\xb1zb\xca(\x0b\x1e%\xce\xcc\x0f\xa0\xcdD\xdc\x03\xc6\x91\x85\xab\xd0\xac\xcc\x1f\x15@\xfes\xc3.\xbf\xaf{\n\xa2\xc9\xd7\xfa_cH\xd6g7\xbd\x8d%\x15\xda\xa7\x1e\xf5\xf6\xe4)\x16\x9f\xbfG\xac\x10&\xb0\x10j:\x10J_\xe4\x80S? \xc3\xb6\x1d\x0b\xcfl\xd8R\x80\x1drV\xc5\\_NP\x07\x1584\x9c\xef\xd8M\xf5\x00\x12\xb2\xd97\x8b\xd6}\nvy\x86\x89\x12\x08\x81P\xe4\xcb\xf6\x86F\xc7\xfd`]"\x9b0\x08\xad6\x0f\x12u\xac\xd0\x1d\x10\xb0\x14u\x1a\x0f\xbb\x81\x892\xb3\x90\xbb\xe2R\xa1\xec8\xe1\x9c\x93\xa9\xb7=\x82C\xd1\xa9\x11k\xe8$\xb83\n\x19\xa0pK\xea\xa8\xbb\x0eWuY\x82\xdd\x9b\xd5\xdc>\x05\xbb<\xc3\x84!\x82$\xa0\xd7j\xd9\xc9\xa5V\x08\x8c"\xbc\xba\xd0\xf9\xa8\xc7;\xbfV[\xfa\x8ej\xec`)S\xad\xe2\x8e\xbd\xd0D[\xa22\xb1\x8en\xd7\x9d\xb3\x02\xdc\xe1\x8f\xea\xd0\x8c\t=D\x16\xa7\xce\x84\x94^t\x7f\x06\xaf\x85O#\xe9\xce\xe6\xdf,\xb2\xfa)\xd8\x05\x83\xfeB\x93\x08\xb2\xf6\xd2k\xb5\xec*\xc5N\xb9\xbf&"\xf8\xedqu\x0c\xa9s\xaf\xe7\xdb\x80\xe6\xaex\xaf+l\x97"\xa0\xcd\x0f\xa9\xaf\xba\xa6\xed\xf05\xfb|\x13\xac\xc88\x1e\xdb\x05\x84\xe0\x80x;\xd1%\xe8J\xc6\xd9BQ\xd40\x00\xd4\xf0\x9a\xf4\x17\xc1.\x7f\x7fU\xfb\xeb0\xc3\x01p\xe7\\"eY\x9d\xf0l\xdaG\xb1#\x9d\x8cV\x87\xa5\x96\n\xfdu\xc6\x1e\xefbLPsq\xb7\x1f\xf3H+D;\xaa-\x97\x8d~\x0b/\x02\x90\xc6\xe7GL\x1f\xee\xec\xe9T\x1c\xe7\xf2\x98(\x0b\xe8\xeao\x96x\xff\x14\xec\xf2\x15\xe6\x9aC\xe34\xf1Rd\xd5\x07\xef|\xc8\xccrz\xd3DO\x04\x99y?\x15E\xe8C\'`\xca\xd2-|\xc4\xf8^%\x08?\x05\xbb<{\x91B!b\xcd\x14^z\x11m\xe8S\xa6\xa2W\x0f<$\xae\x89\x94W\xb6\x10\x8c^\xc9&\x1e\xd6d\xafR\xc0\x19\xef\xb8\xfd\x11\x95\xec\xc1:\x08\xa7\x1e\xc6\xd6\x1c\xdf\xb0#\xc4\xb0AN\xc4IY\x08:\x8d\x83w\xa9\xa7\x85\x18\xd1B\xa7k\x8e\xbd9Y>\x05\xbb<\x97>\xfc\\\x15$\xf6\x92\xc3A\xd6\xcd\xba\xe9y\x1c\xf2\xe4\\r9\xe7\x14)\x16\xf1W\xd3\x149\xed@J\xf7\xc0\nle"G\xa4\xef\xf9\xa4\xa5\xaa=%\xf6\xfb\xd9Z\xe0\x80\xa9\xccfBf\xbd\xbd\xcc\xb3\xaf<*\x1e\x1c\x95\xfev\x143\xf2\xdd<\xf5C\xb0\xcb\xd7F\xbeN\x8a\xf5\xf0{\t\x13\x8d\x84R,\xa0\xc4\xed+\x82\xbe\xd1>\x968\x06b\x94d5\xd3\t[\x9f\x0f\xf7VUDQu/gq\xe8\x06\x15\x02G\xccq\xc8\xfa\x016\xae!\xc8F\xb3\xc4\xf7\xa3\x9f\xbb\xee\x05\xe8\xab\xb1q\xe8{\xad}\xb35\xf1)\xd8\xe59Y\xe8\'q\x88\xd1/\x1b\xe8ufT\x19]\x144\x02\x15Z6-P \xd0Y\x05\xd4`\x18\xb9\x85\xa1\x0f\n\x05\x1e\x08\xf6\xe4\x17\xf4\xec\xe1\x99\xd9\x963\x11j\x08\rb^\xc7\x97\xd7\xe3\x98;\xcc\x05\xdbU\xbd\xb8\xa6\xd0\x01\x0c\xeaU\x01\xe8oj\x95\x9f\x82]\xbe.5\x04\x05!$\xf22Y\x94^\xee\x9c\x91!\x1d\x0fv8\x00o|\\Bu\x83\xca.USM.\x1c\xed;N\xb5\xf2\x81\x9dj_\x93\xed\x90\x15f\xbcFT\x16$\\\xdbl\xb3\x83\xbc\xaf8\x1d\xd7\x14\x9c\xb9\xc3E\xec\xef\xc6\xd9w\xac7U\x85O\xc1._\xe7\xc4\xba\'\xe28\xf2BW\x81\xaaz\'\xf8\xab\xaaT4\xe4\x19\x9c\xa8JS\x1cv\xa4\xb2\x00\xf5r\x0fx\x8d\x00\x94\xaer\xa9!\x1c\x04\xd6\xd8\x8d\xe6\xcd\xd3\\\x87\xc8\\\x13m.\xa5\x8b\xd4\x06(#\xf2\xb1-\x83\xd8\xcf\xd5Q\xed\x12q\xac\xde5\x08?\x04\xbb<\'-\xba^\x86`\xf4\xb5\x82|k\x17\x16f)\x99\x00g>\x99\xd3{d\xd4\xae\x8e\x0c\x1cS\xc7/\x1a\xc3\xb7Qe\xd6\xf0\xc4\xec\xa2\x98\x03!\x0c\x8a\xee\xc8\xf1\xda\x1d\x0b\xd2E\xef\x15\xe5\xe4\xb7\xa3\xe09\x13\x9c\xd3\x17Zv\x106\x8e\x89\xbd\x07\xbd\x99\xdc|\nvy\x8e\xe6zQY\x0f\x81\xd7j\xd9;\xed\x92\xa6ze\x07\xf5Y\xa8\x02g\xae\x933\x0fw\x1e\x17A\xb98\x8c\n-\xeaG+\xdbSg(\xba\xe0H\xd0\xd7N\xea!y~\x0eE\xf6\xc6h\x97=\x91L\n,>,\xb4O\x00\x12@\x08\xeb\xbc\x13\xab73\xf2O\xc1.\xebhR4\xb5\x9e\x06\x04\xfd2i\xd1>6\x16\x13\xe5\xc9E\xd4\xf8\x19s\x02\xda\x0c\xa4.\xedD\xdc\xec\x9b\x11\xa0\xb9y\xb9O\xfe,]\xb8c\x94)\xcd\x1e\xbb0\xd4\xa2L\xf0\x81I/}\x92\x1a\xaey\xedK\xa1W\xe4\x89^v\xb9\x12^\x11\xcc}\x17]\xfa\x10\xec\xf2u^!4A\xfe\xc1\xc5\x83kA\x1a4\x1f=H<\xd2\x1b\x85\x9f\xbb\x83Hu\x821\xc3\x14\xb1\xec\xc5\xe3\xc38\x1bA\xe7\x07\x93x\xc6\xc8~p\x01\x83\xb5|\x98t\nN\xeex\xfc\xc0j3\x82A\xb3|u!v\x7f\nzF\x91F\xb6~\x93\xe9\xf9\x14\xec\xf2{\x98$\xf6|\xc6\xfd2i\x83u.k\xa6D\xd9\x87\xf6\x16\x9d#\xd2d\xb0\xbd\xc2\xf1\x88\x00\xde\xf2:\xe2\xb3{\x92#hf\x80\xcd1\x9d]\xa8\xb8\xc4\xf2\xfd\xb4\x1c\x901\xf6\x06\xa6\x90\x10G\xd8Wk~\x84\xed\xa0C,\xb3\xf3X-\xf1\xf0\x8b`\x97\xdf\x1f\n \xd8z\xd6\xbe\x9a\xb9Q\xf88T\xb2\x0c3\xd2\x85~\x00d\xae\x01\xe15\x8fr\xdde5\xe6|\x80\xa3\x07\x9c\xec\x83E\x99\x0b\\\xec\x12YI\xc6\xc9s.\xddi?\x16\xe3!\xae\x18\xf9H@;\xbd\xb1\x04#qx4\xf08a\x97\xff\xe8\x99\xe7\x9f\x1bv\xf9\xfd^C\xae\xb9\x07\xfa\xb2&\x84\xdd8\xde\x00\xb9y$\xadH^\x85\xe7\xf3[\x8e\x97\xef\xdd\xb0d\x13M#\r\xb0 \x91\xd2\xd9g\xf5\xfc\x18\xd6{k\x17\x95Grd\xd9\xd6u\x05\xe3\xeaS\xcbrj*\x19\x8d\x19\x15\xee\xbb\x05S\x9br"\xba73\xf2O\xc1._a\xae\xe9=\x82\xbf\x1a\x19\x9cx\x8fU\xf3\x16\x19~j\xf1\xd4)\x02\t2\xac-g\xbf\x83\x0fW\x19[\xe7O\xd2\xa4\xe5\xeer\x05\xd8\x02U\xd5\xcbYGDbG\xeb*\xcc\x87\xde\x8e\x0e\t\xdf\xc5\xca0\x1bm\xc5\xf2I_\x00K&8\x19\xbf\x08v\xf9\n\x13\xc3\xd7C\xe15\xfb\xd8cWRQ\x03\xc0\x90\x8f\xc1\xcdD\xc2\x9bN\x1e\xcb\nap\x9d\xebp\xc8\xc3\xc1=\xd0\xdb\x04P\xc6\x94\xf1`\xa1S\xbc\xec\xa6ZI99\xab\x0c\xcb=\xf0s>.b1\xeaU\xb2\xeb3`2\x8d\x0bR\xff\xd0\x91\xfe\xc9\xb0\xcb3Lz\xbd\xbb\x93\xeb\xbe\xf8\xf2\x04\tJ\xcdK\xa2\xa8I\xbe(g\x0c@\xdc\xcb\\I6\x08\x97(\xd1\x1a\xd550!M\xa7\xe8Cl\xef\xce\x9d\xa4\xa6\xda\xae{\\8j\xc7/#x\xd0\xf8K8\xb4\'\xe9xs\x16\xf9F\x96\x90\x81\xb1\xe4T\xd4\xef\xfa5\x1f\x82]\xbeR\xe6\'\xd4A\xbfz\xd6\x13\x93\\\tpf \x80Y:\xa5\x99:\x9d>Sc6\xfaHuM\xe8a\xb9f\x8e6\x8f\xd4>\x8f\xc7\x98\x9d`CK\xf9\x11\x1f\xd5\xc1\x9f}\xd6\xbd\x84y@\x0f2eg\xea\xa3\xa5\xee\xe3\x12](\x89\xf6\xde\xf4\xd6>\x05\xbb|=(\x83\xd7\x9b\'\xfe\xb24\x91\xe8\x1e\x1dD\xf9\xc8\xd9\x1a\xf4\x10\xe6D\xc1N\x17}\xe2\xc7PV\x0cB\x11\x0b[\xd2n\xbd\'\x8dX\xc5\x13\xe0c\xe2\xda\x82\xf6\xefW\xb3\x11\xc5\xb8\xc0\x85\xfb\xd4\xfb\x83\xe6U\x00&V\x08\xe8\x967\xddH\'\xf9G\x0fw\xff\xdc\xae\xcb\xef\x1b\xdcz\x04\xac\xb7\xb1\xbf\xefEl\x7f5\xe1\xdb\x85UN0$$g[\x0b#.\xc8S[l\xa0\xd3\xb8L8Y\\\xa1{\xb4^9\xb8r\x07\xb8\xf6\xc5z \x87uG\xb4OSD\xe8:\x12\xc3;.\x85N7w\\z]\xe4\xe1\x9d\x91\x0b\xcb\x9bs\xe5S\xae\xcbs\xae\xe0\x04A\xd2$\xfe\x92\xdb\x1c\xcf@\xbe\x1ff`:\x9d\xb2#Y\xddo-\xef,\xfa\xb57\x89\xb1\x19\x91\xae\xb7\x93\x9dZ\xcc{\xfbb\xce\xde2w\xb7:\x96\xa3b\x8d\xcf2\xe2\xbe\r\'\xb11\xea\xec\x84\t\xcc-`\xf9\xbcI\xb2\x98O\xe57W\xfe\xa7\\\x97\xe7h\xe2\x18\x85\xe3\xeb\x12\xfb\xfb0\x89I\xc0\x86li\x15\xc2\xe0\r!T\xee\xbd\xd4\xb1\x81 \xf0\xf8\x15\tyt\xec\x11\x0e\xc4\xf6$\x9a\x05\xda9\xe3N\xf7\xe9,\xedJk$x\x95C\xae\x88\xca\x9d\xe2\xb2\xa6O\xf9\x95O\x9d}i\x01<\n\xb3\xf1\x9b\x99\xea\xa7\\\x97\xaf\x14\x0eYOx\xe4\xf5\x01\xd2\x83\xd6ZT\xf1L\xe3D\x95\x1c\x9b\xa0d\xcb\x1d\x9aS,\xcf\x90\xef\x1a\x8e\x9dj\xa4\xd9\xd2\xeah\xd7\xfa\x04\x0fSh\xea\xf1\xe9\xb1\xf0\xa3-%|Y@\x98\xcd\x9c\x83{\n\xa4j6\x07\xfb\xb3\xd5\xab\xbe\x8a\xee\xbe\xd7\x03\xa4O\xb9._\x1f\x1b=_\xdbZW\xc6\xdf\xf7";\xca\xeb\xad\xc5\x06\x05R<\x1c\x82\xbb\t0;)\xd6\xa3\x8c5\x86\xd6\x94F\x18\xdf%N~(&;D P(Q\x8b1\xf70(G\x80\xc3\x99\xd4\x10\xee\xfa>C\x98N\xd5\xe6\xe1\xb0\xbf^\x0cHv%\xed\xf7\'+\xff\xb5\xeb\xf2\x95{n\xae\xcb?\xfd\x82\xf9\xbf\x8f\xeb\xf2\'\xe2\x1d61cs]6\xd7\xe5\x7f\xe2,\xdd\\\x97\xcdu\xd9\\\x97\xcdu\xd9\\\x97\xcdu\xd9\\\x97\xef\xdb\xce\xcdu\xd9\\\x97\xcdu\xd9\\\x97\xef\xec\xa5l\xae\xcb\xe6\xbal\xae\xcb\xe6\xba|\xdbvn\xae\xcb\xe6\xbal\xae\xcb\xe6\xba|g/es]6\xd7es]6\xd7\xe5\xdb\xb6ss]6\xd7es]6\xd7\xe5;{)\x9b\xeb\xb2\xb9.\x9b\xeb\xb2\xb9.\xdf\xb6\x9d\x9b\xeb\xb2\xb9.\x9b\xeb\xb2\xb9.\xdf\xd9K\xd9\\\x97\xcdu\xd9\\\x97\xcdu\xf9\xb6\xed\xdc\\\x97\xcdu\xd9\\\x97\xcdu\xf9\xce^\xca\xbf\xb7\xeb\xf2x\x8bf\x98\xff\xa6I\xdf\xdau\xf9\xe9\x81\xfd*\xd7\xe5\xa7\x07\xf6\xab\\\x97\xf7\x02\xfb\x17\x94\x90_\xe5\xba\xfc\xf4\x11\xfbU\xae\xcb\xcf_c\x9b\xeb\xf2s\\\x17\xf4\xffu\xfa?v]\x04\x91\x80X\x1aby\x82\x86\x11\xfeK\xfc\xe09\x86\x81\x08\x1a\xc1E\x9c\xc3q\x91\xfd\xff\xd9{\xd3&G\xcd-Q\xf7\xbf\xd4WnX\xcc\xc3\x89\xb8\x1f\x18\xc5\x8c\x10\x93\xe0\xc4\x8d\x13LB\x0cB @\x08\xce\x9f\xbf(\xcb\xbb\xdb\xb6\xaa\xdc[nigz\xb7\x1c\x91aW\xba2\xb5\xd6\xbb\xe6Wh=\x14\xc7\x80\x1c\n\x13\x08\x89\xa34\x04\xf20\x8ac /P$E\x92\xb7\xcf\xce\xff\x1cX\xc0\xf10\x82S<$`<\xcdS\x04\x83\x0b\x18G3\xeeK\xdf\x92\xb7\x9d\xbe\x0b\xab#\x0fl\x8fN\x8eM\xeb\xf2\xec\xcd\x1bn\n\x84\xa4\x13{\xccc\x87\xb4\x8a\xc3\xdd\x81\x0b\x08\xd5\x14\x81\x0b\xc2\xce\xca\xbe \xc368\x93\xfd\xd8\xf1\x8fb\x0f\x9eEu!~Y\x9c\x0c&\xee,\x89Ay\xac`\x83\n\x1b\x9e?c\xf6h\xe5Wa\xb4\xc7bwn\xaaM[\xb7lOY8\xae\xadHt\x95\xaf\xd1l\x8f\xd5\x8anl!E8\xaf/\xc7n?\x86\x1c\xd2\xf6\x9ct\xd8c[\xce\x8d\xf1c\xe8\xd2\xfa\x83\x04\x8b\xa7Q]\x16\x1dI\x14"(\xe2n\x99d\xd0\x84\xb9\x10\xb0f\xa89Y\xbb>\xdb\x92\x08y\x8dc\xf1\x11\xb1\xebS\xba\x8e\xa5\xf5!+\xc8\x89\xbf$c\xa9\xa0\xeb=\x91\x05\t\xc0\x01\xa1\x02\xe9\x95\xeeG^\x049M\t\xb7xm.ECi\xe7\x01B\x8a\x87\xb5|\x12\xd4e\xf1\xd7%\xa7@(|\x1f\x96\xa6\xd9d\xf6\x1e]\x91x\x86\xa8\xc0h\x00i\xba\xae\xf9&O\x06#\xd7\n\x9aSh\xc5 \xcb\x16\xa7\x06\xe6\x1c_J\x1c(\xcd5\xb3\x12\xafq\x89L\x80\xa1:\xa3\xa0P\xc1\xda\xcb\xfa\x15pIf\xf9J1+\xedkm\x1e}\x1a\xd4e\xf1\x95%\xac\x96C\'\xee\xc2~w9 \x81\xcb\xec\x1bE\xaf\x148\xf1g\xc0\x8b}`mb\x95\x0b\x9b\x073/\xb3\xd6\xba\xe4\x87\xd3!\x91F\x91 ]\x99$\xa8+\xcf[\x06^\x04\xc7\x10=\xe5\x89\x05\x1cN|S\x0088\x94*3\x856\xf1\xe8n\xe5gA]\x16g!Q\x18B\xd0{4W\xa1\xab\xaa\xb2\xb5\xc7\xf0\xbcn4"\xa8/E\x12\x00\xf4\xcc\xec\xae\xbe\x7flm\xb6\x12m\x89\x95u$\x8d\xb6>\x07\x8f\x03\x98\x90W]\x0b}\x96\x04vJ\xe5cY\xc3TG\x1a26e\xa2\xc1\x0e\x92U\x9f\xbc\xe4R\xac\xf9\xf4\xc1R\xf54\xa8\xcb\xe2\xb4\xcb\xfc\x89c\xe4=\xd9A\x10W`~T;3\xbav\x9aO\xc6{\xbfbu\xb2\xcfb{\x13c\xea\x95\xacz\xc5\xb3\xb4\xd4m\xbd\xb0\xbb\xee\xc8\xa2*\xaf\xe0\x11>[[\x07\x98\xe6\x8e\x95\xbbD\xdbP\xb7\xc7\x96\xd6B\xc4\xa1\x91\xaa\x8b8\xf6 \x9d\xebiP\x97\xc5\x9aKsN\xe0\x8b\xe9\xff\xa8f\xd4\x92\xd8\xa1\xf2\x11}\xa7\xd1Wn\xb4\xdb}\x84\xc9\xb4x\xf6\xc4USU\'\xa1\xcc\xec\xc3\x9a\xdfWg\xb0\x95L\xd8\xb3\xcd\xfc`\xcd\xb6\x9a]\x14\xb4u\x85\xbdV\xa2\xa8wTiem*\xe3\xea<\x8d\x16v\xb8>\xb8\xac\xffiP\x97\x9b\xd3"\x08~S\xf5\x8fj^\x1d\xbb$}\x84(\xfd\xeb%\x1b\xd4\xb4\xa8\xdb\xa3DP\xb3\x10\n\x80\x12\xa2}\xa4#\xa6\x8b\xc6M@\xa9\x83f_X\x94\xa6M\nI\xb5._\xefv\x1c\xc9BB/\x88\xf8v\xf4{\x80\x9b\xf7\xe0\xa1_\x85\xbb\x9f9\xed\xdf\x1c\xea\xb2\xc4\xc4m\x0b\xfb\xd2\xef\xdf\xc5D\x84\xec\xa2m\xcb\xe4\xf4\xd4O\xa51iP\x131[@\x18\n\xdbE\xea\xda\t\xf2\x16\xd3v\xbe\x86lN\x08m\xf7\x83\x9eb\x16\x15_6\x9b\xbc\x9c\xb8s\xcd\x1fC\xe7\xb8\x8d\n\x04\xbd\x14\x04@\x0bgV\xa9j\xeb\xb3\xa0.\xb7\x98@I\xf0\x86\x80\xfc\xa3\x9ag\x7f.\x18\x97]m\xb7@\xeeP\xbb\xb4\\&3\x9b\x9d\x08\xff\xdc#EH\xed-\xabt5j\xbaP\xc7\x8d\xd1\xef\x1c\xc0\xe0 ?\x05\x86\x0bs\x85\xda\x81\xf1\x8eh\n\xe4+\x95U\xc9\xc4gg$\xf4.\xc5\xd1{p\xb7\xf2\xd3\xa0.\x8b\x9aKg\xbb|!w\x8bd3U\x84,\xc1c\xf2\xc1-\xe9h\xbf\x07\xb0\x94o\t\xc7EaQ\xe1\xb0Z\xde\x91\x82\x8eI*\xb9\xabb\x87m\xf6\x91:"\xae\x0f\xca\x1e\x88@xC\xa3\xebL\xdf\x9e\x86\x81\xbc*\xe7\xb6\x10{\x08*t\xa7:\xc1\xcb\xa0;\xb9(\xd9\xd1zQ<\xda\xc3=\x0b\xea\xf2\xd1\xdc\x10(\t\xe3w\x14\xc7\xf9$\xf2\x96\xbfQ\xdd\xb9f\x94&W\xad\n\xbd\xc8\x91\xc3\xf2\xad\x1b\xc9\xc4U\x89\xf06=\x9b\xed\x86+\xb6\xfa\xe6\xe8\x8bq@\xabZ\x0fY\x94\xc6\xce\xd0\x05v\xf6\xa7k\xd8-sI\x1b#>9\x1c\\\xd4\xb8T\x0f\xf2\x9c\x9e\x06u\xb957K\xd3\xb78\xee]\xe8\x17\xca\x15\xaf\xc3\xd6\x9e\xf43-\x8cC\xed\xea\xd7\xed^=\x89\x8a\x1bY\xfc\xb6H\x9b0\x8c\xf4\xab\x89\xea\x90\xef\xa3\x93\x11$\xb9A\t\xc1\xaa\x1b\xb6-]A\xc2q\xbd\x96\xa1\x8d\xab\xa4\x17\x98?\x0cDg\xa6\xa3\x13>\xc8\xaey\x1a\xd4eQ\x13\xc1\xd0\xa5\x87\xbbg\x01\\\xd2\xf5\x05\x8e\xcd\x9a\xf5\x88R\xce\xf5\x83\xad\x9fQ&\xb1$\xdf9\n!\xc3\x1ds\x9f\x1b\x84v\xe5\xb0\x9bx\xd0\xe6\xe0\x88rr\xce\xee\xf7k\xec2\xc1\x8a\xb7\x1d=\x0b]\xafk\xf9j\xa9.\x88\xaeP99\xe5\xc5\xa3\x88\x9egA]\x16\xa7\x05\x11\x10$)\xf4.6\xb9T\xc0j\xef`\xd1\x0c7\xe4\xb3\xe2\x00|\xe0\xf8\x04\x10N\x1b\xd3&\xcb\xf3./7\x8c\xb6\xa29\x1d$\x0eg\xc5\xae\x03\xb8;4\xbeg\xea \xce\xc1A\x10TK\'Nj\xa1\xe5\x15\xf3\x06d\xb3\x0b\xdb\x82\xecgA]n\xd6D?0\tw\xdd\x07\x11\xd8\x93\xb4DZQ3\xeb\xddt=K\x11\x00t\x97\x89M\xc8#\xb4\xd19\xaf\x16\x96\xd0-\xeaId7Q8\xce\x80h\x02\xd0)\x10\r\xaeVz\xf8t <\x0c\x02\xfcV_g\xc9\xed\t\x04R\xa0a\x1f{\xb0\xfbx\x1a\xd4e\xb1\xe6\xe2\x05\x14\xb4\x0c\xccw\\\xa9j\xe7\xc5qq\xb8l\xea\x83\x8f\x9b2n\xccS#\xc5\xad\x97m\x02e\xd3\x16\xe9*c/\xf2\xc0t\x0c\xc6\xce\x15ky\xd9ISe\x8au\x81\xdahA\xbf\xdf\x89\xf9\xacb\xa8"\x03\x0e\x92\xd0&\xc9\x9f\xc1\xe3\x83\xfb\xdd\x9f\x06u\xb9\x15\x14t9\x19\n\xbas\xda\xd5%+t o&,\xa6\x13\x16l6\xf1I\x94\xc5\x0em@e\xca\x80\xcc%\xe9\xf3<\x8dbr\x9d\x1c\x10@\x1c\xf9\xa8\xaf*0k\x8e\xe3\xda\xd7\xc6\xc2U\xd66\xb2\xd7\xb321\xe5\x88\xa6\xe7\x0e\x89\xe4\x15\xf4\xb3L\xfb7\x87\xba\xdcB\x7fI\x140y\x8fs\x9a\xc7\x12UF\xa2]M\xcc\xd4f\xa7H\x8c\xe1\x9a\xa9Fh\x9b\xd0\xb64]t\xbdi\x10vKI\x91Q\x9c\xf5\n@\x8e\xe4z\xa0\xd8\x13\x95\xcbb\x89,c\x12\xb8\x83\x8b\xdd\xb4\x1f \xa2\x0f\x8a\x83\xcdb\xab\xcb\x83\xd3\xdb\xd3\x98.\x8b\x96\x18\xb9\xe4J\x02\xba\xbbr\xc8L,\x91\xcf\x87\r\x89\x96y\x1d\xec\xaa0\xd5U8\x0b\xd6\xed\x1a\x8b\xe5\xd0Z\xa9\x9c\x938\x80\xd2\xe7ay\xd2\xaa\x8c\x01\xd5\xabs\xd0Zv\x9d\x97;\x0bP\xaeR\xb6\xdev}\xb0\xb2\xe2\x8d\x01\ts\xcb\xb5\xa8\xfd 8\xefiL\x97EM\x04\xbb\r1\xc8]+\xa90\xe5yOQS_tj\x81\x0bf\xda\xd7NBhu\xaeY\xca \xcau\x7fX\xef\x98\xabR\x94c7\x83+9\xf0\xf4\xa5\x03;N\xa5*\x81#Q([-.i!\x99\'\x93\xd9\x188E\xa5\xc0\xbc\xd4\xd3\xc7\xd4|\x1a\xd3\xe5\xe3f\x05B\x17\x8b\xde\xe5q\x1e\x01vic\x9fP\x8d,\xf2\x13r\xe4#\xdcE\xe0\xc3pZ\xe9\xabe\xa03y\x1fWYtJ\x1a\xddH\x91\xbc\x84d\xbd\xc0\xf0\xf1\xc0\x96\xb0\tS\x8cb\x94\xa89X\x8b\xe9rKp\xf0\xd2q\x83\xe8]lJF\xba\xb1\xd9j\x1d\xd3\xd7vw=\xe9)\x91\xe3\xaa\xba\x1e\x8f-o$Z\xbd;\\.\xca\xbe?A\x12\xb4)\xdbK\x1c\x9cO\x87\xae\x19k\xda\xcey\xafp\xd3\xcc\xd0\xce:\x95\x8bc\x84\x0e\x8c\x81\x14\x96\x08h\xe8\x83U\xf9iL\x97\xdb\xad\'\t\x83\x10B\xdc\xf5X\xc4\xc6;v\xd8N_\xf7\xec\x85\xd5eeu\xcc2\x8fW\xedV\xcc\xc8S\xebg\xe9*\xb1\xe8F\xdb\\\x07}oLY>i\xab\x9a\xd9\x93\x19\xcak\xd1F.\xe5\xb9\xc1\xe0E/\x02\x064\xc59\xb2\xf5VW\xf2\xafuE\xfe4\xa8\xcb-\xc3A(\x8c\xe1\xf7\xd5PasY2\xcb\x18:\xaa\xcc\xa5JK\xb8\x04\xbd\xa3\xd1\x80\x90W\xc1\xde\xec\xa7J\x8bM`T\x08\x12\x9c\x94\xa9v\xaeE\x95\x1a\x1bw\xcd\xc9Zd\x9b\xa2\x0f\xfb\x9cR7\xf4\xe94\xf9\x1d\x16/=\xb9)_\x9b\x07\xdfOy\x1a\xd4\xe5\xa6&\x05a\xb7\xf9\xea\x0e\x97\xcb6m\x9b\xaa\xf9v\x1bl\xd2ps%\xd4\xa1\x18\xf4za\xce\xe1\xda\xc7\x92\x9d\x8e\xe8\x9c;%\\\xb5\xcc\x9d\x1a\xc4\x17\xf2\xc7\xcf\xde\x86\x85\x08a\xaa\xb8\x10\xda\xe4\xf6\xc1\x13\xcf\xc4\xa2\xa3;jK/\xa9}<7y\x80#\xdb\x075\xf8p\xbb\xf4\x9f\x93\xd9\\\x06-\xbdM\xd7\x074:\x9c\xb6l\xe6B\x1f5\xf2*I:gb\xba\xed#ZaZ!\xdd\x98l\xe9+\xe2D\x15\xdfk\xa86\xf8\x88\x8c\xc5\xe2\xf6\xb2\xe8W\xf9\x08=\xe9vp^\xf42\x13OV\x83]iI\xe3\xe9@7\x99+&.V\x94\n\x86\xd2Hg\x0b3\xd5.}ys\x14\xd3\x93O_#;\x95\xe0\xec\x901\xe7\xb5\xc8g\xa6Bg\x19\xb3\xa6yK\xa0O\xc6od\x11\xb4\xdb08;&\x94d\xfa\xbad\xe3\xb3\xd6\x95\x96\xd6\x9a\x8e\xd5yG\xf1TY.\x8a5\xa2\xae5Yu\x9c\xb2\xad\xb1\xa95\x10\xdd\xce\xa7\x0b\xbc\xdf\xe9\xe9A=l\xbc\x9d\x0b8\xf2\xda)\xe1\xedQ= \\\xc5\xf5\x82b\xedHY*\xa5n\xf2h\x0e\xfa\xf6\x97\x91<\xb7\x05.\x7f\x8a\xe4\xf9\xfai\xfas\x91<\xe4OY5\tP\xa2i\xc6\xf5\xa8\x0epW\xbc\x12\xc0\xb9sTG\\s@)\xcf\x14\xca\x03)\x81\xd5\x9b\xf6\xb2\xce\x8f\xc6\xc9\x17\xba\xa9\xdf\x89)iw\xfb\xf3i{\x19/>q\x95\x07\xd7;P\x08Lh\x95g\x125\xb3?0\x8f-<~\x16\x91\xe7\x8f5\xfdw\x9b\x80E\xdd\xa6\x89\\\x90\xc9\xa2\xdb\\\x8aQ\xba\xe8\xa8\xc7\r\x1dO\x94lte\xfb\xfd\xc6\x8d\xeaIDt\x04\xacN8!\xf7,r\xd5%\x03\x1a\xce\x01\xc8\xd7\xbb4\x18p\x0f\x13h\x1c\xf4w3P]\xf74W\xee\x1f\\U\xf5,$\xcfMMb)\x1bw:jv\x90^\xd4\x16\x90W\x93\x9f\xb14n\xb5r\x94;%\xe3_7\xf4U9\xdc2\x1c\r\x08U%\x01p-o\x14%#%\xd7\x82\xe0M\x9b\xcc\xdbm\xd9\x92X\xa0\x82:\xba\x820\xb8\xc8\xc2(UhY\xec\x1f\\8\xf8,$\xcf\xa2#\xb9\x94Y\x10\xc3\xb1\xbbM|)\x94\xd9\x08\xe1\xeb\x0e\x7fD\x13 >\xa5\xd2*Q\x04\x9a\xf4\xa6\x00\xc3\x05\x9dv`\xb32\xbb\x13^:x\x86\xe3@\x9e\xec\xbb\xa5\xfd\xa9\xb5$\xc2\xd6\xc60\x86\xc6\xb9\xb4\x10\xd5\x9d$??\x85\x1a\xbe\x87A\xdf}\xd0\x94\xcfb\xf2\xdcL\x89\xc3$x\xe3\x97\xdc\x05\xa6\xb3I\x0cy\x7fn\x99]\xaaUH\x88I\xf9\xb8\xa2J\'\xa8\x81\x80\xf0.\x90\x01b\xad\xb7E\x80\xa6\rUj=\xb4\xe8\xfa\xa8wQC\xc5i\x9f{s\x99\xb5\xf6FW\xcaa\xde\xc7n;\xb5\xcdT\xac\xe1\x84\xfbI`\xfe\xbd\x99<\xb7S\xc4\x08\x98\x840\xe2n\xd1\xf9e\xdea\xe25\xea\x8e\xd8\xd1V\x0e\x97\x9d\xea\x05^\x99\x8a\x8e|\x92Gwd\xaf\xfb\xe2X\xf3\xc5e\x0e\'\x13q\x10a\xaeC\xe5DO\xc0$A\xb8\x05\xc3\x19\x92\xcd\xd2\x04W\xfb#\xc5 k\x86b}O\xbb\x18\x0f\xe2M\x9e\xc5\xe4\xb9\xa9I\xdeV\x0c\xe3$|G\x1c\x8b\xfc\x029h\xa73\xeaH\xd1I3\xfc\x02\xb3\xfcXT(q\xcfK\x15/I\xb3\xbfU\xb6\xc8\x9c*\xbe\x958\xe8\x19)\xaf\xa3#\xab\xce\x96-P\x94\xdf\x00\xe1Q>\xafJvH:^A}%N\xb7QI?\xb8W\xf1IL\x9eE\xcd%\xf5c(D\xdd/\x1c\xa4\xb9\xc2\xd6\xac\xf3\xca\xbb\x86\xebs\xb3L\x92\x82\t\xc3FL\x81{z\x83\xf6\xfbzXEW{\xdfs)\x08o\xd0u\x07dR\x00\xcc\xb3\x85[1A\x88\xb6\xbf\xf2\x85~\x952h\x92o\xcf\x9d\xa60\xe0I,\xa5G3\xdc\x93\x98<\x1fY\x9c\x00A\x12A\xee!\x04\x84!\x80]\x8b7i\x8fk\xaeD\x85\xe0\xde\x08\x9ck.3\x9b\x06\xe4;\xd7m\x9b3\x07\x93\xe7\x8d\xc3\xe4\xe7\xc3R\xac\xa9\xa2m\x1c\x0e\x844KC(d\x18%*\xb3\xb6H\x0f\xc2`\x94\xcdk"\x92\x85\xe9\xc1\xf5\x91\xcfb\xf2|d8\x1c\xc5\x10\x10\xbc\xb3\xa6~ut2W\xce\xdc\x068V\xb2\xc6\xed\xe7\x0b\xb5r\x86iG\xa6^\xeeO{\xa2\xeb&Coq\xbaE\x98\x08u|\xc3>d0\xb0\r\xd6\t\x94\xa5B\x82\x1c\x07\xba\x07\xd2\xc6\xd6\xa5z\xb6}\xcf.\n\xb9\x7f\x90\xb5\xf0,&\xcf\xcdi\xb1\xa5\xf9@\t\xf0\xae\xc3\x92\xe8cc\xae\xd8\x03\xd8K\x88T\x9c!c\n#D\x02hROV\xa8\xb3W\x9b\xeeX\x90WU$\xddAVr\xb9\x01\xec\x86\xd0\xae\x0e\'\xa9\x82m\xa8!y\xcaF\xa3w\xb6s\xe2\xea\x06\xb6\x1bbP\xe9\x81\xaf\xb5>\xf2YL\x9e\xe5\x14)\xe2f\x00\x98\xbc+\x87\xdc\xaaS/ib\x94"T\xacd\xe7 H\xc11\xcc\x95$q\x9c"\xefB?m\xea|\xde\x01\x14\xa7\x8e\x15ir\xaa%\x93\x00\xb6>\xf8g\xd2\xf2\xe0\xfap\xb80\xd1H\xcb\xd1\x06\x95\\\xa1s\xe5aMj\xddJ"2\xa9c\x85\xb45\xd8Rjh8w\x9cu\x7f\xb8\x96a\xd1\xa8\x1b4}\xb4O}\x12\x93\xe7\xa6&\xb84\xf1$|\x8fr\xd8d\xc46\xe0\'\xc5NM\r<\xefx\x91\xdf!\xd0\x8a\xdeo\x8a\xf3\x19\t+\xa8<\xc8\xca\x91\xc8\t7[\x99v\x97\xf1\xe3@\x8b,]\x05\x19G\xd5\xaa\xe9G\xa3BE\xa7\x96[\xafBj\x97^\xc2\xf3jN\xfb\x9fY\xf3\xef\xcd\xe4\xb9\x9d"\xb4\xfc(\x84\x93w\xcdM7$V/\x86,\x989\x07]=\x8e\x02t\x15\x9bv\x1c1f\x1b;\n\x7f\x19\xf2^;\xce[#\x97\xd1\xa2k.\xcc\xcc\xb3\xdc\xaa/\x9c=\xecK\xf3)\xdaw\x17\x95\xdc\x95\xa7\xf3\xbe\xaec2\xddmr\xd7\xcf\xb8\xc7b\xe2YL\x9e[\xc3\x8f\xe3 \xb5L\xd3wu"`\xb5\x92w0`\r\xef\xc2\x19(\xf5K\xea0\xfc|0\xad<\x88v\x91\x18\x86+\xe6bjg\'\xf3\x19kHQ,Q\x1d\x12\xde\x1e\xd1\x95\xd3\xe4;\xae\x886,\x14f\x86\xc0\x9d\xa3\xf5\xb5G\x920\xf4v\xed\x83\xe8\xa1g1y~-\x87\x14D\xdes?\x0e\xb1\xe7\xa9\xc9X\x03U\xbe\xdb\x00P\xd9M\xd7\xd4<(\xb1\xe2\x141\x8d\xcc\xfd\x1ao\xaa\xc9\xe1R3O\xb1V\xcf\x889\xdeO\xb9{\xbe\x9e\x8f\x1a\xc7$\x94\xe3\xc3\xe3&\xda:\xdd\x91\x050\x8f\xdc\xee\xae>q\xf9Y\xc3\xffb&\xcf\x87\x9a$\x8c\xe1\xd4=%\x93@-k\x9a\x95\xac\xa9\x93\xac\x8b\xc7N\xd4\xa7d%\xf6\xb1+\xea\x92\x0f\x02\xed<\x8a\xe7\x9a\xdeTD\xca\x96S\xb3fsP4\xfa\xeeA\xf4\xd0\xb3\x98<7\xa7\xbd\x11\xe4\xc8\x1f\xac\x00G\x11p\xdd_\xbd\xac!\xbaeB\xb4\xe3tQ\x06\x0c\r%RC\x94\x95w\xfd\xd0\xc7\x8c+\xfaN\xa7\x01\x07B\x034\x88\x05\xf6\xbb\x90\xb1V\x888\x89(\xbb\xaa\xd5\x84*E`\xcc\xf7ir\xd1\x07\xaas\x1c\xfdQk>\x89\xc9\xf3\xd1\xaa.\xdd\r\xb8\x8c\xb7w \xa9\xeb\xd5H.\xc0lo\xfa\xbc9hkz)=Ws\xe3\x8d\x9b\xcd\xc6\x1d\xce\xac\xc3\x9e\xf4U\xb0\x86\xa4\xa0n\xc7}\xe9\x08\x96\x92\xc0\xe5v\x85\xe2[\xf6\x10\xaf\xa0BS\xe3j\x8bTt\x0e\xadGe\x8cND\xea=\n\x92z\x12\x93\xe7fM\x08\x86H\xf4\x07\xdd\x87\x16\x97\x8c2\xcf\xc5p\xf6\x81\r\xbc\x89\xf5\xf1$\xac\x0ba \xba\x84O\x95\xb5\xc4\x89\xab\xae\xa8v\x08\x92\xa4\x0e\xde\x91j\xb8\xd6Q\xd5]\xdb\x13\xd4\xb7\x0c\x03\xcb\xc9\xce`\r\xf0T\x9b\xe7\x0bq:\x9d\xf6Q\xe4\x90\x8fv\xe4Ob\xf2\xdcZ\xd5\xc5\x90\x14\x88\xe2\xf7\xccS\xba\xdb\x1b\xfb\x81\x1e\xbd\xe2\x80\x87\xcd\xaeU\xedu\xeb\x0c;8\xc2Yd\'\x02\xd3H\xab\x1e>\'\x85}\xea\x03\x99]:\x14\xcd\r\x8a\x82o\xa1\x18\xbb\x90\xba\x92j\x073\xdc\xcap\xd2\xa9\xfbF\x8bw\xda\xaa \x1f\xbc\x0c|\x16\x93\xe7\xe6\xb4\xcb\x88\xb6$\xac{\x94\x83j\x86X\n\x08\xb9pY\x83\xf46K\x8a\r{mW\xd2\xc8\xc2Yi6\x94\xebno+\xde\x8dC \x895y\x1a\x04U*\xbdy\x8d\xd2\x88\x93e\xa1\x1f7h\xa0\x84\xf4.w\xc1\x00\xf5\x06W\xa5v@\xf2\xb3a\xf9\xef\x8d\xe4\xf9\x98\xc5\x97\x81\x17\x02\xc1\xbb\xc8\xf7\xd6\xb9L\xac\x8f\x96\xe8\xa2(d\xaf\xa9#^\xc7E\x89v\xfc\x88\xc2\xab*Ym\xa1\x95$\xe5\xc9d\\\x86\n\xaf\xf6\xab4\x0c\xda\xb3\xddd)\x1fX8\x11\x89\x8a\xb6\x85\xc4"\xb3R\x98\x1f\xd6\x8dC\xa9\xaa\xba{\xb0\xf9x\x16\x93\xe7\xa3\\-\x15\x0e\xfa\x01BN\xe5g\xb4\xd7\xe58\x806\x95\xcb\x97a\x8bTd[e$$n\xb0\x82\xbb\xe4\xa3\xad\xcap\xc8\x96\xf1\x08o\xc3\xcb\xb8\x8e\x90\xba\xd9\xc7^c\xcc\x85\rN\xa8G\x86\xd5(\'m\xc2\xb42\xc1\xccL\x8b3=\xf6`\xf3\xf1,&\xcf\xc7\x90\x8a-\x89\x9c\x04\xef.\x90\xe6\xcc\xd8\xb8$\xb6\x1a\xce\xa8aGiA\ra\xdf\x8d\xc2\\\xa2\xc9\xb1t\xf6}\x7f\xbe@mQ\xac\x07\xd4\xd0`J\x9cw\xca\x10\xb4\xa5\x02\xc5\xdb\x14\xf6\xd7AU\x95T\xd9\xd2\xcb\xe8u(YCmB~5\xfc4&^\xcc\xe4\xf9\xb8\'\xc3\xc1\x1bv\xec\xce\x9a\xf6\x00i\xcd\xf684kNk\xd6\xa4\xd5\xaeL\x8b\x0c\x91\xad\x14HU\x7f\xd2\x8e\xa3\xba\x8eAd\xa6-7\xd8\xf0\xa1\n\xb8\xda\xc8\x9cP\xd0\x894\x9e\x83\x11\xdf\xda\x1f+\xa3\xbdX\xa9\x96a\x83\x96\xf0FB\x16\xea\x83\xd7\x81\xcfb\xf2|t\xcc\x04\xb8x\xc3\xfdx\xb5?\xd1n\x9c\x9b\xae\x08l\xaf\x19a\xf33\rd\r&\xe3dk\xf16\r\x00\xfa\xe0Ut1\x87\xac\x81\xb2\xe1\x89\xdc!`H\xf5\xfb\x1a3\xcd\xa4\x00\x04\x9b\x07f\xca\x8cs\xf9| \x81t\xcb6\x9e)\x9d\x86\x07\xa7\xc8g1ynj\x82\x08\n\xa1\xc8=M\xa6\xf4A5%\xd6\x92>\xd4B\xb7\x0b\xfd\xbe\x07\xe5N6\xd7\x1b\x98\xef"|\x93\n\'6j\xa1&o\x86\xa9\xc8\x07\xde\xdb\xaf\xf7\xbd\xb6[M\x07=\xb2\xc4\xb1I\xc3\t\x90\xa8\x93\x95\xe2\xce^\x16u\x7f\xea\xfd\x16\xfa\x19\xf7\xf0\xef\xcd\xe4\xf9\x08}\x04\xa7n\xcf/\xdc\xcd\x1dp\xeb\xb5\xeb\x8e\xc0a\xb6Nf-\x0bp\xc7\'\xb4P\x1br\x10\x10V\xadQ\xba\x00\xc6\x9d\x8c"i\xf5\x9d\x19a\x93(\xec" Xg\x99nj\xb1s\x1e\xf5\x15jS[\x16\xdfS1\x9c\xd1p\xd3\\3\xe0\xc1\x0c\xf7,&\xcf\x87\xb3` N"w\xe5J\xc2\x8f.\xac\xf5\xab=\x81\x8d\x8d\x19\x0e\x00f\xaa\x87\xa6tf\xb7\x9b\xdd5\xb2\xdd\x85z\x1c\xa2@kp]\x97^\x9c}U\xe8j\xe6\x8b\xb3\x7fQ7`^\x83\x13\xafo&\xd7\x9epm\xd5YF\x97a\x86\xa1<\xc8\xc8|\x16\x92\xe7#\xf21\x08\'0\xfc.$\xae\xcb\xd4?\x03\x87\xf8\x8a\x1f\xf8\t\x83.\x14\x15\xd2BFe\xf3j\xc5\xd2Qp\x81\xeaN=\xeb\xf6\x85@\xebC\x89\xcdi\xe3\x1d\x91v\x1a!\xd8\xeeM\x1c\xce\'G\xef\xa7\x95\x0f\x0f\xd6\xd1\xa8\xa2\xc3\xb0\xd6\xb6\xa4\xff\xa01\x9f\x85\xe4\xf90&\xbc\xa4\x8a{\x9a\xbb8\xb7\xe7\x83\'\x9e)\x93A\xd0\xce\xeb\xb8\xa2!\xf6&\xa9\xf9\xe1f\xad\\\xb6\x17\x15\x1d\x14\xa6\x0f\x1b\x14nJU\xcaHNv\x96\xe4\xca\xf0\xeb\x10\x17\xd2\xb9\xbe\xba\xce\xd6\xf3\x85\x96\xb6\x88\x9eM\xd3x\xeb\x17\xf6\xfag\xcc\xb1\xbf7\x91\xe7v\x88\x18\x04.\x91\x85\xde\x15C\xd0*2\xb5\x1f\x89L0P\x1a\xf5q5\xb0g\xceD\xdd\xb0\xb4\xb12fL$\xaa\x08\xfc\x12G\xe39\xf2i\x949v\x94\xb9\xc5l\xb0\xdc\x15\xc4\xa8\xb0U\x91\x9c\xaf\x8c-\xd2+\x0b\xba\xda\x15\xbf\x19\xaf\x9e\xa1s\xff$\x92\xe7\xe3\x92\xf4\xb7H\x9e\xff\xfd\x7f\xbf\xc5\xa7$\xfdx\xc0\xe3\xfb3?\xdf\xd8\xec\xa4\xb0\xebm\x15\xad\xdd\xda\xdfm\xab\xe5G\xf2z\x7f\xfa~\xf8Y\xd8\xfd\x9f1\xac\xfb4\xf9\x98\x8d\xd0\xa5D\xfc\xfa\xdd\xa1\xfb\xf8\x1et\x83s!\x1f\xcf4\xdd\xa1}\xae?{J\xd6\xf7\xf4b\x87\xe8Ul\xde?\xd5:?\xf0\xc4Omb\xc9\xec\xcc\xbaE^\x7f\xff\x98\xee\xcf_:\x86\xab::\n`\xb2\x93\xab\xdf\xbet\xd9\xe36\xdf\x08a\xd9\x9f\xb7\xa5;mk\xe6\xa4\x97\x8d\xbb\x1c\xca\xc6we\xdc\xe5\xfb\xc0\xaa\x9a9\x02]\',\\\xdc.)\xc1\x9f\x03\xcd\xf0\xd0kZ\x1d.\xce\xbcu4o\xbc\x06\xc7+\xe7\x1c\xaf\x8a}\xec\xc3\xd0\xde\x96\xc9\xecnuH\xb6Cx\xbc\x98\x90\xae\x98\xe0U\x8f\xd6\tc\xb8<\xb51\xffH\xe5\xf9\xcb0\xa4\xbfpf\xff\xe8\x1a\x7f\xfd\xcd\x7f\x06C\xfao\xa3\x96\xee?\x95\xf1\xed\xfb\x93P\xbf>\x195\xfe\xf1 \xfe*\x1a\xe9\xaf\x9e\xc3\x03\xaf\xbd\xf8.\x18z\xd4\xf0\x9b\xd7^D\xe9\x02/9$k\xe7\x89\x06\xfd\xef\xf1\xa7\xfe+\xba\xd5\x7f\xcf]~dP\xc4\xfdxt\xefi\x86\xfcK\xfa?\xe7\xf0{\xdd\xdb\x0e>\x12\x1c\x93"\xe9\x92\x9a\x81SO\x9f\xb4B\xc7\xc3#\x8f-^_\xf9\x88;h\xc7\xbeH\ni\xd2\x8fI\x17?\x10M\xaf8|\x9d\xf3\x7f}\xc6q\xfb\xec\x88\xfa\x8bg\xf1\xbb\x0f\xb1Ua~\xfc\xd7\xe7\xb4\xbftNIZ\xa5Y\xd8\xff\xf4\xa0>k\xbd\xc2\xb7\xdf<\xc6\xfa\x98i#\x8f*\x13\xef\xfa\xdbj\x17#\xdbC\x08WC0\xfe\xdem\x9f\x9fV\xff\xbf\xff\xe7\xa3\xdf\xe8\x9a\xf0\xf6\xfc\xe9\xb7o\xb7o\xbc\x81\x80?\xf9d\xe2\xbf\x0f\x10\xf0o\x84Z{\xd3\xeb\xdeG\xfa\xf5\x8f\xf4\r\x04|\x03\x01\xdf@\xc07\x10\xf0\r\x04|\x03\x01\xdf@\xc0/,\xe7\x1b\x08\xf8\x06\x02\xbe\x81\x80o \xe0W\x06\xed\xbd\x81\x80o \xe0\x1b\x08\xf8\x06\x02~Y9\xdf@\xc07\x10\xf0\r\x04|\x03\x01\xbf2h\xef\r\x04|\x03\x01\xdf@\xc07\x10\xf0\xcb\xca\xf9\x06\x02\xbe\x81\x80o \xe0\x1b\x08\xf8\x95A{o \xe0\x1b\x08\xf8\x06\x02\xbe\x81\x80_V\xce7\x10\xf0\r\x04|\x03\x01\xdf@\xc0/\xf4$\xf8\x1b\x08\xf8\x06\x02\xbe\x81\x80o \xe0\x1b\x08\xf8\x06\x02\xbe\x81\x80o \xe0\x17\x96\xf3\r\x04|\x03\x01\xff]\x81\x80\xe0o^\xe4\xbffz\xcd\xbf\x13\xe9K\x03\x01_\xae\xd8g\x01\x01_\xae\xd8g\x01\x01\x1fS\xec/\xe0\xe5>\x0b\x08\xf8r\x8b}\x16\x10\xf0\xf51\xf6/\x01\x02\xfe\xef\xff@\xe9\xdd\xf6\xceX\xc3\xf1\x03\xec\xf3\xeb\xe7\xb8\x97\xdf\x93\x9e\x8fy\xdd\xff\x12\x9f\xa7\xa6?\xfd\xb2\x19\xa2*\x8f\x95t\xfa?\xfc\xaf\xb8\xbd\xff\x90\xff\xff\xfe\x07\x81o\xd1p\xa6\xc9i\x12\xad\xcbtl\xd9\xa6:\x1f\xc03\x06\xda\xd0\xd2&Wz\x17\x1f\xaf\nB\x9a\xa8\r:\xe8\xe96\x87\xdc6\xe04\xa7\xf1\xc6\xb3\xfb\x86b\xdf\xbe\x7f\xe0\xfc\x7f\x10\xa0\x10\xfb\xed!\xfe\x19\xa0\x90eP\x96$9\x1cB1\x1a& L\xe0p\x0cg\x04\x82\x04A\x0e!@\x82\x15p\x96\x03q\x88\xc6n\xff\x13dY\x16\xe2ot\x02D\xc0\x08\x98\xa6o\x92\xff\x9c\xdd\x04r$\x8a2\xbb\xad\xa6\x05\x97\xdf\x06!\xe0mm)\x89rK\x80r\xa0\xc0p<\x88\xd2\x08N10\x88\xa3 Dq,\x03.S\t\x88-\xbf\x19\x83q\x18\xfe\xd7\x01\n\xffi\xee\xda\x1d\xa0\xf0/#\xeen\x0b\xd1\xfe\x14q\xf7\xf5\x03\xfdS\x11w\x18\xfas\xf8\xdb\x916m\x19\xe6P\x00mB\xb9V\x0bS\xe6."JU\xae\xbe\xf3l\x073\x0e\xf2\xbe\x99\xfa1\xf3\xd40\x85\x911=\x15\xb8\xd1\xef\xd0\xa5\xc5\xd8[!\xccT\xabh\x93o1KM\xe2\xb1\xaae\xa1\xce\xad\xdc~p\x17\xfbs\x18w\xbf\x96\x85\xc5\x1a\x10\x8c\xdfmE\xec\r\x9a\xec/}\xd5e\x14\x8f\n\tvPk=\xc793\xd27\x8a\xdd\xca\xc8\xa6\xa7\xb7\x07?d\xb8f\xd5\x86\x01F\xeb\x18\x97\xb2m\xbd\x95\xd4\x8bh\xd0\x84J\xe2;\x0ff\x10w\x16\xae\xbbr\x90\r\xac\xc8\xbaG\x17y>\x85q\xb7\xa8\t\xfd\x02B \x0e\xdf\xd1\x03\xcaX\x058\xef\x1ao\xbd\xc0v\xa5\t@\xbaCr\x02\xe3\x16@\xa0\x01R\xb7\xca\x05G7\x03-Fa\xcb\x8b\xdb\x81j\x11Q\xed\xf0\xb5*\x91\x02\x83z\xd7)"G\xa7\xadGc2$\xe7\xa2t8\x9d\x8a\x1d\xfb)\x8c\xbb\xef\xa6D!\x14\x06!\x9c\xfc\xa3\x9aXq]\x11+\x188\xd2\xf1\xa6\x9eP\xa4\x0b\xb9"\xc6WL\xc4&\xae7\xe4UT\xac\xfaIo\xc6D\x1b\x0euE\x9a\x83\xbf\xea\x8fiNDr\x1e$\'\xaf\xad\xa5\xed.\x0e6\xc2\x05\x05@\xe2p\x1aV@\xc5<\xb8\xa7\xf89\x8c\xbb\x7fx,\x81,\xce~\xb7\xadt*\xa4\x13u\x96\xd29\x0bI\x7f\xec\x8b\x18\x8a!\x14\xf1S\x9c\xdfn\x10\xd08\xd4\xd3:\xcch(\x9d0\xb14\xf6\x14\x13\x8b\xa8O\xe4&\xd74\x0e\xc7\xa6\xed\xb8\xaeM\x02\x8e\xa79\xe9XK?\xaf\x93\xd5zs\xf8Zh\xa4\xe70\xee\xbe\x9f"\x82b\xd4b\x84\xbb=\x9ep\xc6\xc1\x12\x03\x1e/\xa5O\x93\xf6\x15\xe2c\xee$\x1c\x8b\xd6\xf1H \xab\x8eR\x07\xb3\xb6\x96H\xea\xca\xd5%\x97\xdb\x16\xe4\x1a\xa7Y\x08\xb7\x8e5PTdN\nI\xbe\x011Olc\x8av\xc0\xcd\xf1\xb4S\xc8G\x19\x0cOa\xdc\xfd\x1a\x13\x08\x89\x81\xcb\x89\xfdQM\x19\xe7{aK\t\x07\x02t\xb8\n4\x19\x18Zo\xebYG\xf05\xeb\x9ag\xef|,\xa8\xcd\x96\x17m\nsS\xdc\xb8\xb8{\x84\xae3\xa2R\xba!\x96\xe2z\x9c\xa1\xc3a\x14}x\xeb\xc3\x1a\xd7V~\x91\x1d\x03\xe615\x9f\xc3\xb8\xfb5&\x08rI\x1e\xd4\x9d59\'kv\x87\xab\xe9\xf6H\xb9\xf5\x1cJ\x87\r\x0eb\x95\xab)\x93\x0e9\x1d*\xa74\xfbi:\x05\x13 \xac\xe7(\x80\x01U\xa4J\xf9\xea\xd1%&m\x84\xee\xea\xbb\xc2.\x0c\xe4\xc6\xdc\xe5\xcb\xd8\xe422w8<\x1a\xfaOa\xdc}\xa8\x89\x937l\x08\x8c\xdee8\xa1\xd3\xbc\x06\x98\xc0q\x19COd\x08\xe7\x91\xb3\xdbY\xfa\x12\xd9R/\x84x\xa1\x07\xeb\x16\xb43\x8bl\x85\x83\xa4c}\xb4g\x0f\x9bQ\xd49Eu4\xb6\x0f\x86\xf5\x18Ztz%lE\x0c\x8b(\x9fV\xfa\xe5Qv\xd1S\x18w\xf7\xa3\xda\xef\xf6\xea\xc3\xd2\xb9\x0c\xa3\xf0z\x05\xe5=fa\xdb\xaa\x05$\xc0\x03\r\xdb4V\x97\x8d\xab\xc6l\x93\xae\xd7\xf1V\xb0Z\x1cPO\xdb\x0c\xd8\x92\x1e\x82\x10S\x8d\xd4\xb4G\x82V[\x84\x93cdH\x0bMkQ\n]\x01~06\x9f\xc3\xb8\xfb\xae\xe6\xc7\x85\x04\x8a\xdf\xed\x107\x9dV@\x9c\xa4=\xd5\xa3\xe0\xed\x99n\x95\x8d\xa3\x12\x9c\x9a\x15p5\x9am\x83\xb6\xf8\xb1E\x929\x195\xf6\x12\x07b\xd59\xb8{\x89\xb4\x8a\xd9\xc43*\xa1Y\t/\xfez="M\t\xc8\'M\x80\x88\nS\x7f\xd6z\xfc\x9d\x19w\xdfO\x11C\xf1\xdb\xfe\xee;g\xe9\xf0$`\x8e\x93\xb6\x1fG\xc7k\x1a\xc8HN\x0e\xb4Q\xaf\xa4s-\xd6\xadX;\xf6\x1e\xd9jaC\xc4GA\x9a\xa4]\xa1\x8e\x05\x19\xe3\\Aot\xfa|\r\xed\xce\xe9z\x9d3U\xb7\x9d`2\x82\x94\xc8\xab\x1e\x85\x95>\x85q\xf7=\xf4a\n\xc1\xf1\xfb\xad\xfdS\xaa\xc1\xa8$\x10\xaat\x9e\xd0z\x7f\xf1\xe9\x92?\r!y=L-R\xefN\x18\x17\xb4\xd6e\xdb\xefD\xb9\xdc\x0b\xfc\xd1\x8e\xfc\xa3t\x96\xa4\x16\xef0f\xab:2\xa7\xc2\xd2\xfe8\xa5\xc3\xdaTgb\x1bl\xc6Cc\xd4\xd9\x00\xee\x19\x0b6\xd03\xdc\x90\x88T\xf4-\xf3X\xf7\xf1\x1c\xc4\xdd\xafw\x02\xf0\xed\x8d*\xf2\xce\x9a\x0cZ\x15L\xca\x8b !\xf9\x9d\xa3x\xa0\xc2\xf7@\x02\r\x97\xf5n\x94\xe1z\xa7\xf5t\xed\xa4\x91h8\xd3z7o\x98\xb8\xf5PZ\xcc\xd90\xc9\x8a!\x81\xa9\x90\xd8g\x9b*\xa5c\xab6;\x82\xea\xa1Kx\xfeY+\xf9wf\xdc\xfd:\xbd\x91\xc42\xd6\xdc\'\xf2\xfdv\x86\xcf\xb5L\xd9\xe1\xbe\x12\x81\xc9\x12\xd6lC\xe6G\xaa(Oyy\xad\xccXJ8\x03\xe2\xdd*\x8fHx\xb3=\xe8\xe4\xd4\xd9\xd5\xceWLs\x1ap\x80\xf7\x8fU\xbe?\x1b%\xeb\xc5\xd8\xf1\xaa\xb2\x8ef\xae\x1e\x84\x06=\x87q\xf7\xab\x9a\x10\xb6\xcc\xe2\xd8\x9d\xb3 \xe0\xa8[\x84\xaar3H\xd8H^9\xa8\x9b\xfb\xa9\x07\xe3\xb9\x1b\x06\xf9y[\xa1cJ\xd4\xfe\x0e\xb5\x98\xd2N\x0e\xa8-\xc6Y\xb5\xc5/DFS\x92\x08\x90*r^\x07*\x1d\xfb\xe0v\x07\x10\xa4w\x80\xb4\xe4\xc1\x9b\x95\xe70\xee~Us)m8v\x8f\x80:\xe7 \xd7\x1b\xad?n\xf5N\xb0\xbauv\x96\x07~\xc2f\xc2\xf68G2Yc\xeb\xb7*\xb9q\nR^[\xe3L\xd1\xc6\xda\xd3/:\x88\xebr\xb6\xd1\x04\x06\xb9\xd4\x96r\xbc\\\xb6\xe7\xae:n\xd6f\x7f\x8d\xc7\x07g\xf1\xe70\xee\xfe\x11\xfa\x08\xb5\xf4\x15wj\xbaf\xd2\x1dv\xd7\xb3\xbe_\xd3;I\xd9\r\xfb3S\x97z\xb8\xc8k\x08\x19\x1b\x0b\xd5Z\xc0\x8b\xda\xbe\xe4\x86E\xa2b\xe0\xcc\xa75$.\x93\x88tN\x93\xd3\x84\xc3\x860g;\xc8\xe1\x84\xd2\x1c\xd74Z\x13d\xfd`\x86{\x0e\xe3\xee\xd7\x96y\xa9l\xcbxp\xd72s\x84[D\x104\xda\x17\xe0\xc4SY\xcaj\xdb\xd2\x85.\x8c,P\x81\xed\xab\xe1\xde0vX\xba67\xb2\xbc\x1a2:]\xcdM\xc6\xdb\xc7\x9a\xda\xc6\x9d[B$\xbfK\xbb\tt\x8ed\xbd\x19\xc6\x03:o\xd1\xd6\xf8\xd9|\xf5R\xc6\xdd\xaf\xf7d\xcb?K\xa7uw\xb3B\x95\xec\xca0s\xfa\xb8\x115\xef\x8c\x0b\x8c\xee\xdbz\x7f\x1c\x83\xddz8.\xe9D\xf4\r^&\xb5\xa6O\x97\xe2\xbd\xc6\xd2\x81\xd4\xf0\xa3N\x93=\xc3\x8f\xb2\x95\x0b\xa3\xb1\x1c\x8d\xb5U\xdc\xd8\x9c\xdccJ\xb8d\xcf\x7f\xad+\xf2\xe70\xee\xfe\x11\xfaK\xdd$\xa8\xbbS\x9c\x0eh3\x02"j\x13Ms\xad\xd8\x94\xf3\xb6R\xbc\x0c\x01A<\xaaL\x9e\xecU?sS\x97\xa93?\x88\x88y5^V\xdd\xb4Q:qiZ\xdbe$s\xa0\x03)Y\xc0\x99\xdc\xae\xc4u\xb4\x1a\xd1\xf3zF\x1f\xe4\xf7>\x87q\xf7\xddY\x96.\xf0\xf6\xc8\xca]"7\xe6\xcd\xbcF\x1a[\xdbq\'\xaa8\xc9r\x91\x90\xf2\xa6\xa7\xae{ZW\x85\x8d~@\xd3h\x7f\xf5g\x9a\xc8\x10l\x7f<\xf2^qb %;\xd7\xf9\xa4\xa1\x9d{\xe5\xcajF\xae\x91.Z\x10\x91jc/l\xa3\x07C\xff9\x90\xbb\xef\xd6$\x96D\x89\xfd\xe0\x8d\xcez\x00K[\xc5\xc6\x8b\xd8\x949k\xe5\xb8\xa1\x97\xbe\x9b_/\xf5\xd6\x92\xb8a\xb7W\xb4\x99\xbe^\xb8(\xec\x92\xde\x95\x90\xae\xb1l\x8d\xb6\x107\xe5\x8e\x91\xb9\xab}a,\x14\x05\xdfF\xfb\xd0\xdd\xa4\x05=\xc9\xe5\xfc \xde\xf29\x90\xbb_[\xd5e\x88\\\x86\xe5\xbb\x0c\xc7\xf2\x03\x00)\xc4\x84\x89\xd5\xb10\x0c-W\xf0\xd9\x9f\xec\x9c\xbe\xd2\x95\xe3i\xa4r\xd6F\x05\xf6\xce\x17\xf2\x04\x1e\x8d3J\xf6\xcbt\xb9\xa8\xb6E\x12yD\x8em\x98\x1aa\xc5\xe5%P\xe8\xbe\xa5\xea\xc0:\xd6\xab\xaf\x85\xb7|\x0e\xe5\xee\x1f\xef\x8e-\xd1\x8faw\xf7p\x9b\xa3\xd2\x00"\x80f\x92/\x91\xc7d\xc51\x94\xb0\x11\x0cLM\x92*A)\xc3wQ"\x06\xf4\xcas9\xe00\xa1\xe9e/2\xdb\xc0a\xcaU\\4\nX\xb6\x86\xd4\xb9\xa0p\xf6F:v\xc1\xabb\xb1`B\xd3\xff\x1c\xe5\xee\xa3\x85\xff-\xe5\xee\xfb\x03\x8bo\xce\xcc?\x1e!}\tk\xed\xebQg\x9e\xbe\xe4\xf7\x9bf\xd3\x90\xc1\xa2\xa8\xc1\xf9\xb0Vh\xcb\xbfyD\xb7\xb3q\xf9B\xb4\x99\xf9\xc9\x89~\xdd\xed\xca\x1f\n\xa9E\xf6c\x85~Hp\xfb\xa2\xfb\x97?\xdb4Og\xc7|38\xed\xaa\x16\xe6\xac\xd9\x19l\xd8\xe5m\x83\r\xa2\x17\xe5\xac\x15&\xa8\xef^g\x9a\xd7h\xa2\xb3\xe8\x8f5\xb1\xe5W\x9b\xe6\xe9\x0c\x9a%\x81j\x98\xba$\xd1\xc5\xd7\x10}^\x923\xc7\x83zaN\x8bb\xa8V\xbf\xce4\xaf\xd0\xc4\xb9.Q\x03j\\|\xd5fs\xd49\r\xd6o\x11d;\x90a\x07/\x8f\x9ag\xb3l\x964`.Q\xa3-\xf5\xcd\xbc\xea\xf3\x92\xd8l\x07\\R\xc3b&\x1a\xd5\xc5\x17F\xcdK4\xd1&\xf4\x87\x9a\xfc\xbcz\x7f]v\xcf\xed\x93+\xf0\x92\xd0F\x8d3\x17\xff\x8a1\xcd\xd6P\xbd\x88\x11\x8d[\xda\x91\x17\xd6\x9a\xd7h\xb2$\xb4\x1fk\xf2\xfaZ\xf3t\xb6\xce\x92\xa1\xfda\x19\xde&\xbd\x90P\x8d+\x97\xce\x8d\x9f\r\x9b^\x12\x9c3\'\x1e5G\x1e\xf9\xff\xbe$\xa1=]\x93\xc54\x90\xc6\xa2\x8b)\xf8\xe5+\xc6\x0c\xdbDu[\x1a\x97\xf4\xbc\xfc\xf9\xe5Q\xf3tF\xcf\xb7\xc5\xaf\x06\xdd\xe6\'m\xe9\x88\x96B\x8ai\x1c\xbd\xf8\x98\x04\xe9\xb6\t\xbe\xd24\xcf\xd7\xc4\xb0}p\x89\x7fxq\xb0\xa5\r\xe0\x97j\xe9\xcf\x8b&\x8bf4\xa4\xff\xb0j~]\x1e\xd1\xad\xa7\x19\x969\n\xd6\xe6\xa5\x13Z\xda\xcc\xa5\xf4_u[[\xfc\xcd\x87^i\x96\x97h2\xa9?\xd6d\xa9V/5\xcb\xd3YA\xdf\x96Nl\x89\x96xQ\xa0\\\xba\xb2\x9b\x9fI\xb7v\x13^"\xe8\xa5fy\xbe&?m\xffo\xe9\xf0\x87\xd5\xf2\xebr\x91\x96$\xe6/c\xe6\xd2$\xdb7hq\t-\x99\x19]ZLP_\n\xe6\x8fC\xff\x8b\x92\x93>\xd3,\xcf\x9f\x99?\x92\x98\xfdCe\xae\xaf\x8c\x96\x97h\xf2Yfy:\xf7\xe8\xdb\x92\x89\x97\xda\xe2 KI\x05\xf5\xb9Dn\xd9x\x89\x98\xe5\xcf\xfc\xf4\xd2$\xf6tM\x1e\x8f\xfb\xaf\xcbxZfesX\xda\xc9i\x19\xc4n=\xcc\x87"K\xd3\xbc\x98\xc5G_\xda\x89=]\x93\xa5J.f\xb9\xab\x92\x90~\xab\x94\xaf-\xf9O\xe7/-\x1d\xbf4\xe8\xdc\xa2\xcc\xcd\xb78\xf3{\xa4\xd8\xce\xac\x17\xdaK\x1b\xe4\xe7k\xf2\x89fy:\xc3\xe9v}\xb1\xd4\x96\xfb\x8b\x18\x8d\xf3\x91\xd76\xc8O\xd7\xa4\xc8\x96\x06\xd9_&\xfcl^zKd\x19\x8c\xd1%\x15\x83\x1f\xda\xbd\xd8,\xcf\xe6@-f\xf1\x97h\xf1\'mv\xae\xb7\xeb\x8a\xc54\xd7\xdbn\xf4\xe5\xbf_\x9a\xc4^\xa0\xc9R\x1d\xd5BCnW\x98\xfa\x8d\x88a\xfb\x98>kK:\x8c\xb1\x97\'\xb1\'\xb3\xa4~\x9e\xc48i|m\x12{\xba&\x85\x89\xa9\xc5]\x95\x84\x17\'\x1b\x7f|\x01\xfbu\xb9Y\xdf\x8c\xdb%\xdf\xcc/\xb31\xbf\x84\xff\x92B8gQd1S\xf1\xb3w-\xbe(Y\xebc\xc9\xd0R[f}v\xb0\xdb\xd6\xb9\xc5,\xa3ag\x8b&\x0e\xf6\xe2N\xec\xe9L\xabo\xfa\xc7\xfb|\xd9\xe2[K\xa7o;\xcbQe\x93\xce-m\xf3\x9c\xbc\xd4,/\xd0\x84\xa3\x07}6\xc7%-/\xa6\x90\x96\x92\x9f\xddR\xe2\xf5v\xf9\xff\xe3\xb8\x7f\xf3\xbb\xfe\xbbr\xfe\xb8\x91/JP\xe3\xf8\x9ft\x8c_\x97\xf0\xf5McQl\xf1\x16h\t\xecQ_J\xa06K\xf3\xedb\xd2\xe0\xaa\xd7\xc6\xc2\xd351,t\xa9\x7f\xce2\xee\xd2\xd0\xf7\xb7\x87\xb5\xdb\xad\xf7\x12\xe9\x87\xd7\xbf{\xffd\xfa\xd6\xcd,K\'\xb24\x89\x1f\x95\xa3\\\xcc\x92-\xbd\xd6M\xb9\x9f\xbd\x07\xf1E9b\xb7\x1a8\xdf]\x12\xd9\xf12\x95\xbc\xfe\xed\xe1g\x13\xbcnfY\xd4\xbf\x85>\xbfT\x0c\x1f\xd2\nzi\x83oM\xe3k+\xc7\xf35\xf9L\xb3<\x9b\x02v3\xcbx\x9b~\xf5\x99\x87\x8ce\xac\xba\xf5\xf2K\xf8\x83z\xf1b\xb3\xbcB\x93\xfb\xb8/n\xf3\xbc\xf0\xfa\xa7\xc3\x9eL\x12\xfb0\xcb\xc7\xc3;\xcb\x97\xc1\xf9\x98q{\x9f\x9b\xbb\xd5\x98\xedK\xcd\xf2\x12M>\xcb,O\xa7\x91}\x9aY^\xa2\xc9g\x99\xe5\xe9D\xb3O3\xcbK4\xf9,\xb3<\x9d\x8a\xf6i\xb5\xe5%\x9a|\x96Y\x9eNV\xfb\xb4hy\x89&\x9fe\x96\xa7\xd3\xd9>-Z^\xa2\xc9g\x99\xe5\xe9\x84\xb7O\x8b\x96\x97h\xf2i\r\xf2\xb3)q\x9f\xd7 \xbfB\x93O+\xf9\xcf&\xcd}\x9aY^\xa2\xc9\xa7%\xb1g\xd3\xea>\xad\xb6\xbcD\x93O3\xcb\xb3\x89w\x9fg\x96Wh\xf2ify65\xef\xf3J\xfe+4\xf9\xbcq\xf2\xc9\xe4\xbd\xcf\xab-\xaf\xd0\xe4\xb3\xcc\xf2tz\xdf\xa7%\xb1\x97h\xf2iwb\xcf&\x00~\xde\x9d\xd8+4\xf9\xb4$\xf6l\x8a\xe0\xe7%\xb1Wh\xf2i\xd1\xf2l\x12\xe1\xa7%\xb1\x97h\xf2\xa0Y\xfe\x1a8\rz\x08\x9c\x06\xfeN\xe8/M]|\xb9b\x9fE]|\xb9b\x9fE]|L\xb1\xbf\xc0\xf0\xfb,\xea\xe2\xcb-\xf6Y\xd4\xc5\xd7\xc7\xd8\xbf\x84\xba\xf8\xfd\x7f\xfc\x8f\xa2\x1c\xe2\xffy\xe8\xff\x05\xe5\x90\xe3\x04\x04$\xf1\xdb\x86b\n\xe4q\x86\x02I\x9e& D\xc08\x96$`\x18\x83x\x04\x82`\x94\xe6\x10\x94\xc0\t\x84\x85i\x16\xe5P\x8c\xc4@\x9c\xe7n\x1b\x8c\x7f\x8e\xef\x12\x96\xbfD\xe2\x0c\x02\x93,\xc5`\x02\x0f\x92\x10$\x907\xf8\x19\x04\xd1\x02\xc8/\xc7+P7\xcc\x14\x83B$D\xe0\x10\x0f\x91(\x01\xe38\xcbQ\x02\x87C/\xa5\x1c\xfe\x03\x89\xf0\xedG\x8b\xbb\x88_ \x84\x821\x14B\xb0?\xa3\x1c~}F\xe4\x0f(\x877D\x04N\x93\x0c%0\xd4\xed?1\x90\xc5q\x01\x84 p14I\xa2\x14\xc81\x8b\xd0\x02\x8dB + <\x85a0\x84R<\x7f\xdba\x88\x107&\xdf\xcb)\x87O`\x11\xfem)\x87\x04\x05-g\xbf\x1c9\x0e\xc2\x0c\xcc\x82\x14\x81\xd0 \xb7X\x85\x85\xb8\xdb\xca]\x02d1\x8agy\x01^\x9c\xf8\xb6m\x15[\xe2\x96Z"\x0c\x02q\x16f\x85o?\xa6\x1c>\xc1N\xff\x1a\xca\xe1?\r2{\x1e\xe5\xf0&\xc9\x9fR\x0e\xbf~\xa0\x7f"\xe5\xf0\x83\x8b\xf7S\xd4\x10\xa4\xa6\n\x88\x9bL\xee\x1d\xd5Q\xa8\xcb\x1d}\xdd*\x9b\xb9\x1b\xa2\xb9\xd1\xa3I>\xab\x82{\x9a\x03W52"\x9a\xa9\xe6X \x1b\xadKE\x8e\x92\xc7\x15\xe9\x85\xab\xb5HrA\x00\xe6<\x0f\\5%T\xdb.|p\xf9\xe7\xd3(\x87\xc4/0\x01\x13\x08J\x12w\xe0(5\x83{f\x91\xfe\xd0\xba\xba\xbfu\xcd\xb98\xd2 \xbc_\xf3\x8e\xb8\xed\xce\xac\xeb\xe7)\xe5\xd9\x1c\x1e_0v\xee ^\xa8\xa1\x11\xdcH\xca\x89#\xf8\xcdZ\xb5\xf3\xd2\xa3\xf8\xfc\xaa\x05{\x03\x89\xf7A\xbb\xf3J\xe8Ap\xd4\xd3(\x87\x8b\x9a\xd4-o\xfdQ\xc7|w\x04\xa9\xb2p\x08\x94\x91\x93\x98\xb1\xf5\xbc^\xef\x86](iL\xd7\xc6\xc8\xfe\x80#$\xa2\xe7\xf6\x0e\xe3m]<\xe7\xab\xa4\xcb\x93\xbd\x96r\xd5\xd41=\x1f\xe9\x17\x143f\x89\xdeM\x87\x02%L\x04H.\xf6\x83\x0bN\x9fF9\\t\\\xf2\xd2\xf2\x93\xe4\xdd.\xfe^8\x12ytZoz\xbe7w)I\x9c/\xa5\xa6\xe3U\xaa\xcbA<\x1d/\xc8\xb9\xbd\xcc=\x1c\x9e\xf8 f\xcc\x0bJWF\xbb\xcd\x1b\x8e\xb5\x03\x06\x94\xdc\x0bP\x8cm\xc6\x81\xe4u\xbd\x15X\xd97\xd5\xc0\x13>\x8br\xb8427L*\x89\xdec\x9d\xce\xd1e\x8f\xc8\xa6r9\\j\xed\xc2\x0cD\x14\xf8\xd5\x95\x1cu&4\xc9\xab8\xda\x01\xe8\xed\xfb\x8e\xd9\xe7#\xb623\x91\xeb\x9dp\x07\xe6G\\3\x94\xa3\x9ax\x92\x14^\x8d\xec\xa0!\xfb\x82\xc6\'\x1a\xca\xb5f\xf5\xb5H@O\xa3\x1c.\xce\x02bK\xffKbw\xce\xb2"#S3\xd2a(\xaf\xc1\t5\xe2]1\x1exq\xef\x9c!\x8f\xd1\xd5\xf59\xd0N[\xcd\xc5\x0f\\\xb6\x85\x8a\xc3\x80dJ\x8f\xf1]5\xf7\xc7Q\xac\xb7\xa3\xce\x06\xb9\xb5\xc7J{\xad!\x19\x8eB\xf2~0\xc7\x07\xd3\xdb\xd3(\x87\x8b\xb3\xe0\x08v\x03\x96\xdc\xed6F\xa9\xa6\x11-\x13\xed@\xec\x94\x85\xd0l\xd2\x01\xba\x17\x93\xfeP\xac\x96\xa6q\xaf$\xa9+\xf6\x00\xb3\xea|\x82\xdex\xdc\xee*_\xdc\xfd\x06\xd9`+\x87\x92\xdd\x00/DY\xde\xe9\xae\x05T\x1e\x00m\xb2\xa8\x08\xfb\xe3\xa3\x10\xd7gQ\x0eo\xcd=J"\x08\x0e\xdd-\xe4\x8e0\xe5p\x15O\xe0\xe5\xa4\xd9\xd4\x05\xe9\xca\x0eO\x85NV\xcb\x95\x11\xe1p\xc7H\x80\xb8\x8f\x07\xf2\xd8\x1f(\xeb2\xc2ZU\xd6\x1d\xb9\xc9\x0b\xb3^\x15\x0cK\xc8&k)<\xc4\xc3\x08t\xd2bY\x90\x13\xb0a\x1f$+<\x8dr\xb88-\x0e.\xa3\xe4R\x16\xee\x00\x12js\xe2\xe4s\xee\xba 2\xe1j`.]\xc7~\r\xaew\xdc\x95:\xb5\xd8\x1e\x8a\xcc]\xd6\x11\x14sL"\xb6\r\xf3\x13\xdd\x07\x17\xcc\xb0\xf0H\x10W#\xaa\xf2N\x90\n3l\xa9\x17d\xab\x9d.s9 \x17\xfa\x93(\x877k.QLB\xf7;\x96\xd5-3UZ\x11CX\x8a]\x12)\xd5h\xc6\xdf\xe9\x8cPd\t*\xe9\tM\xf8D\xca\xd8\xe6\x89\x8f\xb6\xab\xfe\xac\x1cfh\xb0\x9aK\xd9\xd7U.\x15\x99^5\x05\xc2%I\xe29\xa65\x99\xbe\xcb\xf8"\xf4("\xe7i\x94\xc3?L\xa4\xbf\x8bM\xbc[\n)\xe0\xa8\xe7m\x0fX\xd0\xe8\xaa\nWtZ\x8c\x93%\xb5\xcb.\xce&\xd8EEG\x00\xad9q;EV\xb6g~\x96:\xf0 \xc2N9\xcd\xe7Cp\x15(&\'\xaff)\xc2\xd21\xec\xd3\xa9#~\xb6E\xfeoN9$~A\xa0e\xf0\x00\xc9\xfb\xed\xed\xc16\x1c\x81Y\xeb\xb26\xcd\xa7\xb0\x80\x88\x04\xdb4\xb8\xe2\xf9\xfa\x16<\xa7\xcc\xb5n\xe5\x1eA\x813?\xcd+v\x02vf\xefd\x1cpP\xb0\x13H\x17\xfbr\x8f\x01Bp\xc9\xc0\xf6`\xed\xe2\xf4 \x8aj\x89\xd8\x0f\xf2\xff\x9eF9\xbcU}\x8a\x02I\xf0\xbe\xea\xaf\xb7\xa1\x90\x98\xbd,*\xd9\xcc\x10\xd6E\xbd\xca\x06\xe1\xec\x9b\xdd!\xb66\x88\xb0\xa64]\x95\xb95\xb8\xdf/\xb3\xa3\xd0^\xe4&`\xa4\x18\xc6yT\xa1\xc9\xae\x80\xb6@\xc1\x9fCB\x92\x0bBD\xd2B\xdf\x89S\xf4 \xeb\xeci\x98\xc3EM\x0c"\x10\x0c\xbc_R/\xd8\xe2\xe5\xcaK\x07@ \xca\xcb$\xe7\xfb9\x9c\xad\xb5=\x91\xa8\xbc\xd6\x12 \xf5\xf3\xecBe\xad9[i\xbd\xf3\xd2\xd6l#\xd6$\'\x91\x9cA\xe8,\xf1\xe8\x81\x80l-\xf0;2w\xa9\x81\xc3G\x0b\xc4\x86\xaf\xb5\xa4\xfei\x98\xc3\xa5N \x8b\x15\x10\xe2\xbe\x1c\xee\xe3\xad\x9a\xbaU\xebK\x08\xa5p\xf3\xca\x80\xe9\x13\xb9\xbb\xfa`\xd2V\xb5\xad\xe6\x8c\xacl\x12d\xf4\x8cV0\xc9\xe1z\xb2\x10\xac\xed{\x9b\x88-\x197\x02\xce\xdaJ\x17\xca\xc0\x04\xa7\x0b\xe4}\xe0\x1fj\xa4-\x8e\x0f\xc6\xc4\xd30\x87\x8b\x9a\x10\x88PK\x16\xb9\xab\x13\r\x03k!\x18\xfaL(\x81\xbe\xe2F\xb1\x19\x10}=o\x19\x95h\xe8\xa6\x85\xc8\x9a\x16\x8e\xe4\x86E=\xb9\xaf\x92\x9d\x9b\xfb\xd7.!W\xb0\xd0X-\x8fz\xb3s\x82\x86\x1e,cS\xd9\xa5Xk\xe8\'\xa1Q\x1e,\x87O\xc3\x1c\xdeB\x7f\xc9\x197\xc6\xdc\x1f\xd5\xac\x9c\x13bm\xc6\xb5Svp\xac\xe7"\x10\x0f\x8ct\xe4\x11\x0f\x8aK\xac\xee\x96B\xd1\xaf\xecH3wa0\x9d\x08\xa9\xa0v3f\x92\xdc\x1cql\x13\xa7E<\xd7\x12\xa6!\xe7\x8a\xe7\xf8\xe2\x88\x81Q\xa3\xe1\xed\xa3j>\x0bsx\xeb\xc8)\x14\xa5\x96\x06\xff\x0eW[9t\x02q\xc6\x8a\x0b\xd05d\xfb\xa3\xb3\x1b\xf7\xe3\x11\xef]\xaar\xdaCc\x16I\x04\xccEE\x08\x98yif\xce*\xb3\xe6\xb2\xf6\xca\xa1\xd5N\xdd\x19\xa2O\xa7\x8d2\x90\xd1\x81\xd3\xea\x13i\n\xc9\xca\x11\xca\x07\xab\xfe\xd30\x87\x1f\xb1\x89`$\x02\xdfA\xc0N\xad\x1cpaHv@D\x9f6\x1ci\xae\xe4\xb3\x0eq\x9c\xea\xadvB\x9d"\x8cC\xae\xb5\x80\xdb\xbb\xb6\x06T6\n\xaf\xd2\xe0|\xf4\x16M\xd7\xa6\nD\xeb\xe4\xb6|\xc8 \xf4`R\x02\xfa\xb8\n\xeb:\xdc\xcb\x8f\xe2\xd5\x9f\x859\xfc\x1e\x9b\x08\x8eaw\xf5\xaa\xb5Y5\xe9|~O%]\x1auPa\xfbu\x88I~-R\x14]-\xa5\xe4\x8c\xc4\xca\x19\xd8\xd1\xab\x9a(\x80\x82\xebv\r\\d\x07QQ\xf4\xbe\xcd\x99Zj\xe1m\x9e\xb1\t\xba^\xe5\x97\x1cR\xc7\xa1l\x1fU\xf3Y\x98\xc3\x8f\x8e\x1c\x83@\x98\xb8\xb3\xa6N\x91\xd0\x8eN5\x9eXu\xf6\x99\x10\x9ckru\xe5\xe4\xaa\x10\xf1\x85\xa1\xb2=\x1f\xf8g1\x0e\x0f\x0e\xd7\xec\xa0u\x1eFZ\xe7\xb3\xb5oa\x03\t\xc9\xe8\xf6\xb2\x05\xed\xca+$\xd1l\x95+\xa7\x81\x19c\x90\xe2\x83j>\rsxkUq\x04\xc6\t\xfcnZ\x06\x99\xc8\xdc\xa0\xdb]\xb8\xf6d@<7\xe7\x95\x06G\xdbi4\x89\x19\xb0w\xb8\x18:u\x04\x0eY\x82\xe7)\xbfQ\xa6fU*\x18\xc4\xcb+v\xab\xe9\xe9\xe5B\xae\xe2#2N|E\x03R\x1b\x19\xe9UO\x98\xe1\xc1\xab\x95\xa7a\x0e\x175\x976\x85D\x97\xaf?\xaay\x98\x00\xba\x89\x10p&OH\x18lJ\x83\xb8\xe0\xa0\xcdj\xf5\x86\x8f\xf8b{\xde\x8f\xc5aP=\xc8$G\xb6R-Kj\xae\xa2_\r\xdcnH\xa3}\xda\x8a\xa3\x14\xaf\xc8Ud9\xb9\xcan6)Q\x07\xb4\xff\xb3&\xebo\x8e9\\N\x11Z\x86\xbd\xc5W\xeeb\x82\\\x1d:w\xd8]Q*G\x84\xf1p\xc1x\x06\x08-1\xc9\x10\xea\x9a\xf2\xb9\n]\x9b\xc3\xa1\xd9\xef\xa3\xb2Ux2\xd0\x13\n\x94\\F\xe92\xb8\x1d\xc5n\xf2\x941\xb3e\x1a\xbb\x94\x9e\xd4\xb6\x06t\t e\xfd`"\x7f\x1a\xe6pQ\x13\xa6(r\xf1\xb2\xbb\x98\x00"\xdb\x17\xd2P\x834e\xaaS\x05\x95:{\x8c\x10\xdbG\xc5X\xa0\\>\x82w\\\xd4\x83\'1\x99\x98\xf5qfNVLY^S\xf9\xc0F\xf1b7\xec\xccm\xe6M\x84j\xa0\xa6{\xd4\xcfV-\xcd9\xf8`L<\rsH\xfc\x02R\x14\x04\x820u\x17\x13"\xe6\xa0N\xe8\x02\xe2x\x0e\xf9@\xea{\x88\xd5\xb1)\xb5\x0b\xd5\x1b\x87\xe8\xc0@[ne\x1d\xd8\x88?Xl\xcf\x80\xc5\xde=\x9aq\xe0-\x8d\xb8\xa5y\xac\x01\nI\'\xaf\xbd)K\x1a\xaef\x18?\x8apOy\xb0\x97|\x1a\xe6pI\xe40\xfc\xc1\x1f\xbe\x1b<\xe6\xc11\xb4c\xcdVYc\xad\xf8\xfej#de9C7\x1f\x8aH\xc0\xb7\x97\xd4\x1aa\x8cv\x0f\xab\x89k\xf2\xa3]h*uJ\x9c\x8bsv\xc75\x1c\x1f\xc8U/)\x1cQ\xfb\xf0\tk\xaa\x94\x82\xaf\xad$<\xc8\xab}\x1a\xe6\xf0\xa3\xc9\xc2\x08\x90\xa0\xeezI\xe1|\xb5\x13-\x18km\xa4\xb7\xe7^\x14OWQ1\xf4#9%\xba\x1c\x9d\xc7\x1c\xddT\x12\xb1\xf4\xfb\xb7m\xec\x9b\xd5&\x1e{p\xc5\x90\x83\xd5[l\xa8\xae\xd3U\x8eS\x1aY[}\xaat\xd6ZZ\xcbD\xe5=\xf8V\xc0\xd30\x87\xb7\xeec)\xe0\x10v\xdfd]Vm\x14\xc7\'\x062O+U\xdd@t3\x9b\x1e\r\xef\xda@\x9a\x839\xdd\x12*W\xb9eX%\xf0v\x84S\x92`\xf7G\x96\xdb\x85z\xb2\x01\x0f\xe3\xb9\r\xd2B9eqv*s\xda\n\x98\xed\xae\x14\x84\xe1k\xdd\x91?\rsx\xab\xfaKWK,Nw\xf7NC\xb2\xdb\xf4C\x0cI\x12%\xc4\x02\x00n\\\x1a\x02`\xb5\xa2\xd5"ZG(\x1fC\x9e\xef\x8bky\x8d\x18\x0cr\x1a\xba\xb6\xf7\xd4\xc8\xddWh\xa3\x08La\xa8\x01\xebl"b8\x92\x04\x8a\xeb-\x9d\xb7p\xa7>z\xe7\xf0,\xcc\xe1\xed-@\n\xa4\x96ru7x\x90\x07\xe4:\xa2\xfb\xa0\x17\x1a^\x13t#\x89\x03\x10.Mo$-\xd4\xbf\xa6u\x87\xc4cz\xa8u\xa7\x1e\xc2\xd2\x16\xf0\x0ch\xb0Jo.\xd7\x0bA\xed`\x7f\x05C\xa7Z\x90\x8f\xa7\xaca\xc0(\xf7\x10\x91\xef\xda\x073\xdc\xd30\x87\xb7\x8b2\x18_r\xe4=\x96W\x8f-\x9f\x1b 2\x1e\xf1\x06w!\xd2\xa4\xd0Z\xcb\xf1\nL\xf6\x80\xe2r\x1b\x0b\xc0\xb0|\x9d\xc3\xa5f\xf1;p\x14\x98\xee\xcc\x9b\xc6\xb0K7V\x1b\x1d\xcb\xf5n\xaf\x86 \xe9\xa7\xa8Y\x1ftA\xa2\x19\xe8\xb4\x7f\xb0^=\rs\xf81_-g\x86\xdd\x95\xab\xd0:\xf0\xc8\x99\x15\xc8:\xb8\xaez^\xe5\xb6\xe3A\xe7\xfcN\xd8\x84,\x14\xd1S\\\xcb)\xba\xa3\xbc\xedI]Q\x9e\xbde\x13\xcb\x16\xf3\x83\xeeC\x0827\xfe\x98%\xfc\x96\x0f.\xa3/\xc1\xa8\xb4\xcb\xb0\xdaP\xc1\x9f5\xe4\x7fs\xca\xe1r\x88KKM\x90 |\x17\xf96\xad{\x81\xa8\x80\xf5x\x89\x04\xa5c\rI\x1f\x0b\x94w\xceBbiX\xb327\x82\xcb\xc3\\\xa4s\xa5\x9c\xcdbr\xe4]\xd3M6\xe8\xb8i\x813\xbb\x95\r\xf8\xd2H\xeb4\x87\x08\x84\xcf\xa9\xcb\x95v\xb1-\xf7OR\x0e?n\x83\xdf\x94\xc3\x7fz\x1b\xe1\xbf\x0f\xd7\xf0\xe9\x84\xb6\xbb\xb3\xfa\xbaX\xbc\x17-\x99|\x1f\xe9\xff\xe4#}:W\xefEG\xfaz9\xbf.\xd9\xefU^\xfar9\xbf.\x91\xefUG\xfar9\xbf.I\xefU\x81\xffr9\xbf.\x01\xefEG\xfaz9\xbf.\xbd\xeeU\x81\xffr9\xbf.y\xeeU\x81\xffr9\xbf.5\xeeEG\xfaz9\xbf.\xf1\xedU+\xfa_.\xe7\xd7\xa5\xb5\xbd*\xf0_.\xe7\xd7%\xad\xbd\xaa\xe2\xbf\\\xce\xafKI{\xd1\x91\xbe^\xce\xafK8{Y\x13\xf5j9\xbf.\x9d\xecUG\xfar9\xbf.Y\xece\x81\xffj9\xbf.\x15\xecU^\xfar9\xbf.\xd1\xebEG\xfaz9\xdf4\xae\xa7\xcb\xf9uY[\xaf\xf2\xd2\x97\xcb\xf9u9Y/{\xef\xe9\xd5r~]\xc6\xd5\xab\x8e\xf4\xe5r~]>\xd5\xcb\xde(y\xb5\x9c_\x97-\xf5\xb2\x9b\xa8W\xcb\xf9u\xb9P/:\xd2\xd7\xcb\xf9u\x99N\xaf\xba\x89z\xb9\x9c_\x97\xc7\xf4\xaa[\xfd\x97\xcb\xf9uYJ/\xebK_-\xe7\xd7\xe5 \xbd\xe8H_/\xe7\xd7e\x18\xbd\xe8H_/\xe7\xd7\xe5\x0f\xbd\xaa\x89z\xb9\x9c_\x97\x1d\xf4\xaa\xf2\xf4r9\xbf.\xf7\xe7U\x81\xffr9\xbf.\xb3\xe7UG\xfar9\xbf.o\xe7UG\xfar9\xbf.+\xe7e\xad\xfe\xab\xe5\xfc\xba\x9c\x9bW=m\xf2r9\xbf.\xa3\xe6U3\xfe\xcb\xe5\xfc\xba|\x99\x97=\xb9\xf7j9\xbf.\x1b\xe6eOA\xbfZ\xce\xbf\x86f\x80\x1fB3@\xbf\x13\xe9Ks]^\xae\xd8gq]^\xae\xd8gq]\x1eS\xec/PB>\x8b\xeb\xf2r\x8b}\x16\xd7\xe5\xf51\xf6\xe6\xba\xbc\x86\xebB\xfc\xe7\xa1\xff9\xd7\x05\xa7A\x02\x07a\x9cg)\x1eA\x10\x12\xa3`\x94\x80\x08\x1c#h\x94Y\xbe+P\x14\xcer\x08K\xe2 \x86\xe14s\x03M\x108\xcbC4\x8a\x91\x88p[\x89\xfds`\x01\xc7\x08\x1f\x8b40X\x10\xb0\xdb\xf6]\x82`aF\xe0\xc0\xdb\x8e;\x02G1\x06\xa1o\xe8\x18\x0eanD\x0c\x98\x82@\x92\xa4!\x84\x84\x18\x82X~\xe8\xa5\\\x97\x7f\xac\xf0\xf8\xf6\x83U\x05\x10\xfa\x0b\x04\x13(\x8c\xa0\x04\xfa\xa7\\\x97/O\xc5\xf9\x01\xd7\x05\xa7X\x01\x03y\x98\xe7\xe8\xe5\xa7y\x96!\x89\xc5\xd9\x16\x91n\xab\xa3IL\xc0\x11\x8e\x14h\x1c\xa3Y\x86\xe6\x17c\t0D\x93\x14BS\x02\xc2p rs\xd67\xd7\xe5\x85\\\x17\xe4\x16\x1f J\xf1$A/GN\xb2\xe8b\x1fl\x91bI(\x1c\xc2\xa3\x14\xce\xd1\xb7e\xd00\x0f\xb38\xcc\x10\x0c\xb4\x84\x0fL\xd2\xcb\x8b/\x92\xd2\xd8\xb7\xbf9\xd7\xe5\x9f\x86\x8d<\x8f\xebr[\x01\xf2\xe7\\\x97/\x1f\xe8\x9f\xcbu!\x7f\xbeu|\x93\n\xdb\x12\xd4\x92\xb3\x8bE\xad\xe7\xb6!\x1e\xc5\x11F\xa4U\x0b\x86\xba$\xcc\x8e\x1d{U\x8f\x04\xfc*\x98\xbc\xb5\xd8\x85i\x9b\xf2mP\xab\xfa%U%E2\xd4\xf6\x0c\xac\x83\x9d\x08{\x8eG\x95I\xbe4\xe3\xdcc\x1b\xc0\x9e\xc5u\xb9\x95\x05\n#\x10\x98\xba\xdf\xad.\xd7\xa8|\xed\xecj\x0f\x90GY\x17\x90z\x7f\xae\xc0\xdc\xa4\x13m\xe5(2w&\x02\xa5\x8edUC*\x15\x00\x06\xd7\xe6\x0eu\xa82]Kwr=eb\xd7\\0\xc8\xb1|\xfdZ:},\x08\xd5Y\x00\x7f\xb6\x00\xec\xc5X\x97EK\x98D)\xe8noU\x7fZ:\xa9\xebU\x0f\xe5Rp\x02K\xd8\xad\xa2\xc1\x07\xc9$\xc6\x8dq]\xe4\xcdq\xc0\xac\xcc\xe6.5\x93\xa9CCu\xb2k\xae/p\x7f\x85M,[k\xf6\xb8\xd63x\x1f\xfb\x97\xbdi\x0b[3B\xd0U\xdc=\xa8\xe3\xb3\xb0.\x7f,\xf0\xbfUSj\xcfk\x1d\x89\xd0\x04\x85\xc0\xb1Xf\xbc\xc9\xb5\x0b\xd8\xb0\x04Jsh)\'E\x10\x1a\x1a\x84\x93\xe4P*W+w/\xf3k\xa6R\r\xf1\xea\x1d\x87\x9dQov:x\x99\x9c}\xb1\xca\x9d8\x8d+C\x8eT\xe6A\x84\xc5\xb3\xb0.75\xb1%\xe9"K\xda\xfd\xa3\x9a\xb8\xd8\x14\'\xc0\xf1\x8f\xec\x809\x00Z\xd8c\xe7\xf1m\x9a7\xf6 V\x97\xe9\x88/\xd9nG\xc8D\xe4\x96\xd0A\xb8"gs\xa7\xec\x0b\x7f\xe4b\xf5:\x1dl3*\x8b\xbd\xdb\xf0S\xa5\xb3y\xe9\xcdx\xc2\x97\xf3\xcf@D\x7fo\xac\xcb- @\x8c\\\xfe\xea\xfdJp\xec\xb0bt\x1e\x19\x08~o\x94\xa0\x91Y\n\rd\xc4u\xbfU\x8a6B\x92- \x9e\xed\x08M\x17\x0fBI\xa0I\xba\xc4\xf2\x88b\xd8ss\xdbod\x88\xab\xe0\x0c,\x8a\x81\xa3\x06M\x88|\x84\xb4\xd8`\xae\x1e\x04\x81<\x0b\xeb\xf2\x91\xdd\x96\x02\xb3\x84\xd5\x9d\x9a\xf3\xe5\xacU\xde\xaa\xdb\xcf\x00\xc3\xa0\xc9\xc1\xa7\x18\x1f\xcf\xc0u\xa0\x9d7\x87H\x10\xe4\xda W\xb8$mDq\xcb\x1a\xaa\x16\xa5\xb0\xe7\xce\xdbR:\xd9\xcei/bY\xdcdP\xb3\xf7\xf4\nY]EFK\xd3\xf9\xfc\xe0\xb6\xcaga]\xbe\xc7\xc4R\xc9)\xe2n)\xa7B\xd6\x17\xc1M\x94&I\xf9M\xb3\x9dV\r\x08(@\x82sQa\xc4aAw\xab\xb9-gx\x8b\x97\xed\xe6\x1c\x96\xa1y,$\xf9t\x85FT\x0f\xf8\xd0V\x02\x1cm#v\xdc]\xfb\x13\xb3\xf5\xdbh7\x9f\xeaO\xc2\xba\xdc\xd4\x84P\x18\xfc\x11\x9f\x0b%W\xd0\xa4\x14`&\x0c\x12 \xb9\x96$\x9b\xfcH\xd1+\xf2p\xdd\xcf\x9bL>\xd0\x95>Mc\x13\xedOvw\x88\x83\x8e\xf1\xc0\x82-\x0b\x92\x0b\x98\xabETtX\x1e\xf9\x00\x0f\xd3\xfe\xffg\xef\xbe\x9a\x1d\xc5\xb2D\x01\xff\x97~%\xa2\xf1\xee\x11\xef\x84@\xc2\t\xde\x84\x13H\xc2\x830\xbf\xfe\xa2S31\xb7K\x955\xa5\x0e\xe5\xcdS}\xa9\xa8\xa7\xcc\x8c<{\xb1\xddZH\xb9\xbesP\xf8v?\x1b\x16\xf1f\xc3\xdcO\xb1.\xcf01\x82\x80P\x9c|Y\xb4\x16%*\x0c\xa8\xb2\xde\x14[,y\t\xd1zM\xddT\xb7\xdd\x11\x0f\xeb2j8x\xca\xca\xa97\x0f\xc7=*N\xc4]\x94(\xca<\xd1\xca\xad4\xc0\x02\xa0\xc0\xd49v4,\x8bF^\xdfX \x01\x96~\xa4\xbc73\x8fO\xb1.\xcf0I\nF(\xe8\xd5^\x10\x1ainm\xf2\xae\xa6\x87r\xbc\xe97{/u\xf2\xd9.\xc5\xfcV\xacU\x89J\xeb\xbaf\x81\xf0\xad\xa9I\xae\x9aEe\x81\'\x19\x882\xa9\x12\x9ce\x94\x97\x0c\x16\xa5A\xbf\x9c\xa6e\xc7\x1cM\x10\xca$H\xfb\xd1\xd6\xfc{\xab.\xcfs|=\xf0\xd7\xd2\xee\xb5\x87\xf4\x94>\xb4S\xe6\x84|\x16xZ#\xa07\xb6?\xd7K\xa9\xb8\x03\x9e\xba\xd5\xc8ZO\x90\x8b\xe3\xc3\xca\x15D\xfa\xae\xa7\xb5\xa0\xdf\xf2>\x8e\xec\x94j\x19\xc6\xf1\x1f\xaa\x8ev=r\x86\xe6\xb8\xa2\x0eJ\xa1F\xf0\xf9M\x96\xefS\xaa\xcb\x1a\xe6ZAPO\xa8\xf5e\xb1\xb4m\x0f\x8d\x8f\xce<\x93\xfd\xb4\xf3\xdb"\xa8ky\x81B\x84>R]9\xa2 \xa8,\xe5\xf1\x8a\xee\xa3%\x99\x02@?"\xee\xb5\xb8y\x84\x9f\xe0\t\xef!\x10\xcd<\xcek\xa2\xfb\x08\xf0\x14\x95\xcf\xb1\xa4\xf2\xc3\xa5}s\xe7\x7fJu\xf9:\xc7\xd7\xff0\xf4\xb5}|\n\xd9r\x92\x97j\x16^\x0f\xba\'Y\xe4#\xac{\x07\xf5\xce\x8e!y\xb0\xff\x80(\xcdE\x02\xc7\xd7\x13\xf7\xe52\xec\xd4}"\xa0\x96 r\xf80\x13e\xd8\xb5UX#\xe5\xbah\xdc\n\xafE\xdf\xf7\xee\x8d:9H\xf1\x80\xa7\t\xa6\x17\xc4Y\xfa\x90\xd1\x99\xa3\xac\x8d\x97+\xa5\x10TX\xd9B0\x96LvVq\xaa\xb2\xa8&x3\xdd\xffT\x87\xea\xe7\xce\xc7q\xf4\xa9\x9c\xbd\xb8\x11\xa1\xb8^\x83\x8d\xb3\xdf\xf7\xd7\xa3\xc2\xac?f]\xe4\xf7\xec\xae\t\xa2\xe6\x08`]^\xa0\xf3\xad\xe2\xdb\x83s\xac\xb1\xc3|&\xa038\x8fDs(\x9d#\x80\xed\x02\x03_\xaf\xcc\xd4\xf6\x1a>$Z\xd3\xe0\x1b\x15\xe4\xdf\xccl>E\xd7|E\xb9&6\x18\xfa\xda`\x15\x0f\xe5\x1b.9q\x0cw]u\xac\xe6[,Z\x8d\xdd\xa5z\xaf\xb1{\x1c \x84\xfcp\xa69\x1a\xabR\x89\x90\xee\xf7\xab\x80\xaa0\xd6p*\x9dZ\xf7\xbb \xd6\x00(\x13\xe7\x07L\xef\xf7\tsH\x1a[\xb1z\xfc\xcd\xd6\xca\x9f\xa2k\xbe\xae\xfc\xe7\xdbg\x02~9\xc6U:\x19\x1e\xd1\xc0.;\x1b\xd1\xe8\xe3!\xc3E\x0f\xe9\x8e\xd2\t(B\x95\x86O\xec\xbdU\x98\xddN\xb6,B[H\x18\xbe\xb3\x08~\xbf\x02\xc1p\xd0h\xd6M\x95\xf3\x84PA:M\xdc\x0e?p\x9du,\xa4\xc4xWY\xfc\x10]\xf3\x15&\x04\xd1$L\xbd$pb\x90\x97T\x8c\xa4\x80P\x80\xbc\x93\xf7\x1a\xdd\xb1\x0e]\x1d\x02nj/\x80u\xd2\x18\xb9^0\xd3\xbe9\xb0\xc8_\xb3\x10\x02\xe9\xbc]\xe4\xbd\x1a\xa4\xa7;x<\x05W/:\x1dw\xc4\xb0\xa6\xd4,w\xd4:;\xaf\xde\xccl>E\xd7|\x15W4\x89\xc2\xe4\xabA\x96\x9eh\xee\x18YsR%\x8f\xdd\xdd<\xec[6\xf1rG\xdfG\xbb\x81\x11\xa9\x03o\x80Hj\xb77}\x1e\x17\xa7\n\xf6!\\\xdb3\xa1\x9c\x84\x98hY2WOU\x92s\xb20\x9e\xdc\xde\xeb\xa3Q}~\xdb\xee]j\xedCt\xcd\xf3R~\xeeq\xf2\xb5\xb6b\x1e\xe0:o\xfa\x15\xe3\xaeX$S\xa8j\xcc:\x10\x0f\xe5\x15c\xc9p\xe9\xdb\xe2D\xe6\x00+,,\xb6\xcb\x89\x11>Pit4\xabSo\x8c\xf9\xc3\x8e.I~B\x90y\xbe\xb4M^\xce\xbe\xaf\xe9\x90L\x86\xefF\xf9!\xb9\xe6\xb9f\xa9\xf5i \x7f\xb0f\xb3\t\xd6\xae\xfa\\\x83i\xb3f\rP\xb6#\x07\x87hG\xd1\x9dz^\xc8\x8fir:bmF\x87\xd7\x9b@U\xb9\xc4\x11\xe5\xdc4q\x87r:\x86\x9f\xdd\xa1)\xaa\xdd@k\x1aPF\x1d\xea\x1a\xf3\xd4v\x03\xf0\xeeA\xfb!\xb9\xe6\x19&\x8a\x11kM\xf6\xda\xc3^\xcf\x0e\xdc8\x10\x93\xa8\x06\xd0\x01tY^\x94\xd22\xc6\xeb\x0b\x98L\x91*\xe3\x85\x1ft>q-\x02\xcciY\xef\xae\xfb^O\x06\xec\xcd4OcE\\H\\&Q\xff\x045\x03Y\xf0\x92\xa8\xc4\xc4\x1d\x91\xa7_$\xd7|\x15\x1d(\x01\xad\x87\xc4\xcb\xa25\xbd\xa5\x9bi\xf7r\xc5\xdd{E,\x8d\xa2\xc9\x168\x8b\x14\xc5\xacU1\xabH.C\x9f\xd1\xb0\x91\x97\x8a>F-VOz6\xce\xc6\xfeaT\x9cr-\x01\x18\x88\xd1\x12;\x1cD\x19Lv\x99\xe3\xf4 \xd7\xa4\xff\x99r\xcdW\x9eJ\xe28\x02#/O\xf1\x11\x1c\x06]\x1b\xae\xacS\x0b\xc9\rO\x8f;\xcc\x9dT|\x97(\xaa\xd4\xddPU\x92n,eVG\xf9\xaea-\x9av\xb6\xbf80\xd7\xa0.\x8d^\xf8\x14\x86\x00\x9b\xac\x04d\x1c\x84L\xe3M\x03\xec\x07|\xef\xbd)\xad}J\xae\xf9\xba\xae\x9e\xfd\xdd)\xe4\xa5\xea\xc0\x91j\x04 q\xcaP\xd0NP\xfd\x06\xd4~qI\xfb\x84X\xac\xa5\xbf\x01vR\xefq\xe2H\xe8\xf4\x94e\x966\xb8\xa2\nk\x07\xa0L\xe3\x8a\xa7H\x17\xa7<2\x19\x8dCR8\x8f\xf8>5\x11L\x89\x08\xf7f\x98\x9f\x92k\xbe\xc2$q\x14\xc3\xc8\x970\x8dS\xadWq\xe2\xd6\xc7\xe0j\x1d-\xe7<\xee/RmiU\x0cx\x0c\xb4\'\xafs<\xf8V\xda\xc0X\xad"\xd9\xfd\x10\x8f\x8a\xe90\x8dN\xe8\x11_\xd3)\xe2\xa2\xb7\xf2\x9c,\x17\x92\xf1\xd2\xae\xba\xb7!R\x0c?\xf2U\x7f\xb2\\\xf3U\x16\x10\x04\xb4\xa6\x92/\xb72\xca\x9f\xd5\x13\x9c\x98\x15Za{C\x19\xa0\xb2\x17.SP\xf2\x12\xeew\x80y\x8fs\x7f\x0c#K\x93\xf6B\x02\x9c\xceG\xfb\xce\xae\xe5\xd3\x084\xbc\x9c\xf5dA\xc8\x0fj\'\xee\xcc\xf6\xda(!\x12\x8cSS\xd7\xe5\x9b5\xe4\xa7H\x97\xff:\xe1`\x8c\x82_\xca\x9f\xaa\xf6<\xcf\x86\xc7;\x90\xd4\xbb\xa2\xc5\t\x9c\xeb\xe7\xbdq\x04\xae\xd6y\x1fi\x01\xec\xae\x8f\x19wS2a\x833C\xcd\x8b\x95\xc4\x9d+><\x9eY\x18\x17Tm\xccx@I\xe5\xe8{i\xb9\x9f"]\xfe{O@\x04\xfdR\x89\xdf\xef\xcd\xb4<*w\x86\xa6\xe1\xfa\xa8\xee\x1d\xe4cP\xbb(\xc7\xa0\x14+\n.\xd8\x90\x90\x93bg\xdc\x92\xbc\xa3\x1ex\xd1c\x04\xd7\x95\x1a-\x82\'"\xbf\x99\x93\x82\xa1c\xdd\x9ev\xcc\xaci\x9a\xa2\x96>\x11\xccozu\x9f"]\x9eaB\x10JQ\xc4+s\xe4\x8a\xadE\x915\xaeU\xed\xa5\xb7(NyhUv\x02k4r{\x8c\xd3\x8f\x0b.\xeaX\x0f\xbbE\x00\xb0\xc6\x84\xa0\xf5^\xc3\xdc\xbb\xddZ\x0f\x101Mt2\xed\x8bx\x87f"\x0b\x97\x9bCaw@\xb3\xf87\xeb\x8eO\x91.k\x98\xeb\xe9\xb6>/\xf2\xd5\xe6\xcav\xe2\x831\x1f\x07\xe0.\xc5B\xf80\xd2(\xbe\x06\xca\x81Z3\xd1\x86l\xaf\x16\x9e\x18Z\x7f6Ut\xa7c\xba \xe5Nt\xf4n\xe3\xe5\xbao L\x95{&\x17kO\n\xc7\xf3\xd8D\xa0"\xa9\xb6(\xa3\xc0\x9b\'\xdc\xa7H\x97\xdf>\xedX\xb3\xa05Yz\x01z`\xf6\xce\xc3XZ\xb1=\x06\xb4{}\x90\xd9K\xc0\xe4\x1d\x9a\x85!\xf5\xfc\x04z\xfd\x8ff\xf3\xefm\xba|\xbdR\x85\x9e\xef-\xa8W\xd3\xc5V\xd0\x9b\x0f\xb7\xf7\x07e\x10\x87\xd4\xa5\xceN\xa9\x1b\xaaP\x00t\xb3G\x04\x85\x0fu$(\x88n\x980\x8fH\xd7\xb4C\xb2\x80\x86\x18\xe9;"\x81\xed\xc1\xab\xd1\x93G\x00%\x05\xa5\xa0\xfe\xb8\x04\xda\xb9"\x99\xc7o\xf4\xd7\xffn\xba|m\xe6\xcdt\xf9\xcb_.\xff\x0f2]\xfe>M\xf37\x87`3]6\xd3e3]6\xd3e;K\xbf\xe1Y\xba\x99.\x9b\xe9\xb2\x99.\x9b\xe9\xb2\x99.\x9b\xe9\xb2\x99.\xdfx\x9c\x9b\xe9\xb2\x99.\x9b\xe9\xb2\x99.\xdf\xd9J\xd9L\x97\xcdt\xd9L\x97\xcdt\xf9\xb6\xe3\xdcL\x97\xefk\xbal\xcdI7&gcr6&gcr6&gcr\xbe\xf187&gcr6&gcr\xbe3?\xb319\x1b\x93\xb319\x1b\x93\xf3m\xc7\xb919\xdf\x97\xc9\xd9L\x97\xcdt\xd9L\x97\xcdt\xd9L\x97\xcdt\xd9L\x97o<\xce\xcdt\xd9L\x97\xffT\xd3\x05}\x8be@\xfeeH\xdf\xdat\xf9\xe9\x81\xfd*\xd3\xe5\xa7\x07\xf6\xabL\x97\xf7\x02\xfb7\x84\x90_e\xba\xfc\xf4\x19\xfbU\xa6\xcb\xcf\xdfc\x9b\xe9\xf2sL\x17\xea\x7f\x1e\xfa\x9f\x9b.$A\x10\x02\x01\xf18MC\xb4\xc0 <\x02\xd3O\xf1\x82D8\x16\x82H\x08\x13\x10\x9c`\x10\x82\xc5\x18\x12\x16h\x91\x87q\x81\xe4\x11\x1a#y\x8e@\xf0g\x83\xd1\x1fc\x05\x14\x82"\x88\xc8\xc0()r4\x05S8\xc6\x8b\x10\x0e!,\x8a\xf2\xa2@\xf1\x1c\x86\n\x18\xcc\xd3\x0cI\xa14\xbd\xfe\xc0\'\x1e\x83@\x04\x8b\xf0\x90\xc8s\xf8O5]\xfe\xbb\x95\xf7?\xfe\xa0U\x01\x82\xff\xf3\xd9g\x98Di\x88\xfa3\xd3\xe5\xfb\x8b8\x7f`\xba\xf0\xeb\xea#h\x86%\x18\x9c\xe5\x9e]>\x18\x9a\xe1X\x14~\xf6V\x15\x10\x02aY\x0c\x128\x1e\x11aB\xa0q\x82\x14\xd6a\x134N\x134)\xc20\xf6l\x94\xb3\x99.?\xd1t\x11!\x02\x17\xd7\xadE\xfd\xd6\xc0\x9b\xe0)\x01\xc7E\x11\xc5\x04\x92\xc5Dr]@,\xc4\xb0\xcf\xae\xfe,"\x12\x04+\xac\xbb\x0b\x17\x19b\xdd0\xeb\x0f\xfcj\xc1\xb3\x99.\xc9[\xa6\xcb\xb3\x05\xc8\x7f\x99.\xf0\x1f\x9a.\xdf\x7f\xa3\xffR\xd3\x05\xc6\x7fLdP\x0f\x97\x9d\xef\xbd\x7f\xeb\x9a\x94\xa9N\r;\xc8\xac\xc4\x852\xde\xceE!kR\x18\xc2\xe5\xde\xd5\xe7\x9b\x1dE\xf8\xb1swv\xa8\xfa\xfb\xfd\xa3\xcdXel\xb2\x96n\x05\xa18`\xee\xb5\xc0\xd9A!\xa1\x8c\x9f\x967\xfb\xe4\x7f\xcatY\xaf\x05\x9a\x841\x88\xc6\xf0\x970\x134\x01\x86\x83\xa3e\x82\xb1\x0f\x90=\x05\x14da\xd5\r\xb8\xab\xd7\xe0\x18\xd9\xb4&x\x18\xcb\x01\xd6u\n\xbe\x0e\xc4\xcdF\xfa#\xd66\xc9B\xa9^\x19\x80\xda"\xec:\xff\xbc\xa7:\xd8\xd8\xf5\x0f\x8c\xbd\x1fR\xed\x17\xa1.\x08\xf1O\x08C\t\xe4\xa5A\xd7\x83dA\xdbK\xe1\xab6\xe2g&\x84\xc9.r\xf3;{DU\xd1\xb7j\xaaU\xeb\x91\x96#\x06\xd9\xe9f\x00\x9a\x02s\xecc\x96>^\xe5,I\xf6.=w\x98\x1b\x1f\x18\xd5\x93\x81\xe5\xe4\r\x9d\x9eh}\xe2\xbd\xd9\xf8\xfcS\xa8\xcb3F\x92\xa0\xf0\xf5x~\x99\xca\x01\xd73\x86\xb9\xa3\xa2\x85\xcaR+9&X\t\x18I\x15\xf2-R"\xf61)\x97#\x1f\'`\x077\xe7\x83\xb2\xc7\xe10Q\xdaP\xf4\xe5\x9c?\x1f\xd4\xb6\n\x82\x13\xa0f\xe7}\xb8\xf0\x18l\x9d\x0c\x8f\xc3Y\xf0\xcd&\xab\x9fB]\x9e+\x16!\x89\xf5\xc0}\xe9@X"t\xc0\x16"o\x88\xd7\xc4\x1c\x9c\x87\x8b\x05E\x0be\x89G\xca\xf6\x99n\x18\x07\x9e#:\xe7\x19\x10\xf5}4\x92!\xba\xea\xf8\xf6\xc8\x1f\xd6\x8a\xca\xd1\xd31\xce\xebF\xcbc\xc2\x89|\x1e \xf7\xc7\xfc~\x00\xd9o\xd6\xb8\xeaC\xa6\xcb\xd7CD\xd6\x03\x1aB^:\xffI\x12\xa4\xdb\xba\x942\x1e\xcd$\xb0\xc2\x81H\xcd^\x8a\x19K\xd8\xe4V\x9c\xf1H\xcb:\xca\xa1\xcby\x02\x08\xcd#\xed\xb2\xad\xccL\x9b,\xbd\x08\x9a\xab-\xf9\x8f\x9c\x07\xc1 Q\x0e\xe8\xf9Q\x1e\xd7\xe3\xb0\xe5\x86\xf1\xdd\x86\xbc\x1f2]~\x9f\xf4\xfe\x0b\xe7\xa4_\xf4y*\x12\x19T\xa1\xba\x8a\xbaKh\x0b"P\x9e\x8b*\x93c\x9f\xc7\xdc\xc3\xa3c\x0f\x1c\x99\x0e\x8eh\xc29\xb3w\xbc\xd9\xf5S?\xaej\x03\xb6\x89\x89\xa2\x00\xbf\x91\xfa2z\\\xa4(\xe0\xcc\xf8\x11\xa7\xec{\x87\xf8\xa7L\x97g\x98\x18D\xac\x97\xf7\xab?6KF\xd2\xd7\x1d\x15\n\xb6>\x1bg\xa4P-\x15.#\x19\xe8\xc3\xf0\xa4QaH\xcc4\xf7\x88\x1f%.\x06\x03\xbf\xb0Y\x0ei\xdap\r[\x9e\'jy\x0c\'\x87\xbe\xcb*$\x1edI\xdbc\xa8&P\x81\xf8\xee\x01\xf7!\xd3\xe5\xeb\xaeZ\xd7\xf9\x9a\xcc\xbe\x1cp\x98\xd7\xec\x11&\xce@GUe\xb0\x8c\xf72\x0f\xd0\xaa\xa2\xad\x13)\xb4w\xc9\x90\t\xf3<\x9bIU"\x0fN<{\xe3\r\xf1\x85r\xb1\x1cJ\xb6n\x12\xa2\t\xd2\xec\xe3\x1d\xec9\n@:IM\x12\x8e\xe9\x81\xef\xb6W\xfe\x90\xe9\xf2\xb57\xd7e\xb1\xde\xeb\xaf\xb3\x89q{\xd0\x11\xf7I\x94\xb9\x0e\x0b:\nA\xdc]\xdd]x\xff4\x04\x8f\xe0\xa83\xb0v\xe0b\xe3\xc4\x1f\x0f}l\x1c\x96\x1d}\xc0%\x87\xd0\x86\xb0\xe9\x1a\xc7@|\x8f\xe1n\xe8\xc5D\xc0\xf3#\'\xea\x86R\xa07\xf7\xe6\xa7L\x97\xe7\xa2\xc5\xc9\xa7\xf0\xf1\xeat0~_]\x9f\xdfu\x1c\x13\x8evk\x07\x03/\t\xa2\x0b\x12\xc1\x057l\xd6\x07\xc8h\xf6\xc2\x04\t\x88/\xab"\x12rBd\xb6\xe1\xa1\x1a\xaf\xfaH\xe2>C\xa8\xf8\x198\xdb\xf7\x1b\xa0\xb0\x1d\xe9>Z\x97\xcd\xa5\x1f\x19d\x7fo\xd4\xe5y\xe9\xaf\x87\x04\x81\xc1\xaf\x82\x05\x016\xa6TX\x05z\xcf\\\xc5\xd4,\x88\xe0\x90IU\xb8\x80$e\x94f+\xc9\xb1\xaa%i\x10H\xcbF\xc8I\x8c\x96\xe9l\xfe\x06\x83J1A\x8b\r\xed\x1d;\n\x08\x19\x98\x95\x94\xbc;^\xb87\x0fb\xfbf\xf3\xd1O\xa1._{b-eQ\xec\xd5 +\x17\xf3\xd4\x18\xa5I\xef\x0fH|1\xcf{G\x8b\x19(\xf6\x92\xee \x0e\xa4\xe4\x93A\xd5\x92\x07n\x92.}l-\xbe\xad\xe6\xc9\x8d\x90\x8e\xcd9v\xc5y\xa7\xec#\xb1\xc3\x9b\x89\x94R\x08\xef\xd8\x02\x04\x8f\xdd\x8ex\x1b\xe7\xfa\x0c\xea\xf2\x0c\x13{\xe2\x0e\xd8k\x8fUB[\x93\xdeD\xd4Tq\xdc\xdd\xaeL%\xe6T6k\xfc\x8c\'\xfcIa\xa7\x9b-\x01{\xc8f\x8e\x87\xce\x01\xbb\xaeKu\xb0S\xe9QT\x9cDKO\x84[\x0e\x03\x9d\x89H\xa6\xdd\xbc\xb5\x0e\xea\xed\x0c\xbf@\xc3\x8ff\xf3\xef\x8d\xba$\xe0\xb2+\xc2]U\x0f\xbbC\x1d\xf8-c\x11\x11\x8a+\x11\xa9\x1fnN .\xfd\xae0\xc8|\xe8\x9d\xb0\xc2\x9a\xb1\xb31y\xaf\xe5\x02|\xc6]\x88\x92\xb9wS\xd5\x0f\xb1._a>\x17\x05\xfc\xdaY\xbd\xb5\xd0N\xde\x9b\x04\x18\xb8\xcb\xdeli\xb6\xa4\x99\xb2\xbf.2\x0bG\xd2\xd9\t\x98SI\xe4@\xe0\xf7\x06\x059Rq(\xef\xdc8\xf0\xbal\xd7\xc4\x82\x1cn\xb6\xedM\xf8,\xdbA\x91\x1c\xee\x17\xd5N\xf0\xa9D\xde\xb5\xd6>\xc4\xba|\x15\x1e(\xb2\xde~\xd4K\xaa\x9a\x1a\xb8\xc6\xd9\x0f5\xbd\x18\x99\x00\x9cy\xbd\xd3\xb0\xd0\xba\xf0\xfe\xe58\xf3|\xbbtvu$(s7\x993\x9czn\xed\x1d3\x17<\xab\xd3d\xd3\x1d\x14g3\xa1\x11f;\xa7&\xd1\xef\x82R[&e9\x18o^\xcb\x9fr]\x9e\xb3\xf9Lc\x90W\xd6\xc5I\xae\x19\x97\xcdL\xde"\xa5\xec\x06\x9cG\xe913\xab0}\xb8T\xe5Ii\xe4c8\xa8\xf5\x92Zww"\xa4\xd0\xad\x1f\x9d\xeaIGx\x82\xcfQ\xb3H\xf7\x9e\xb9\xc0j\x86\xee\x8c\xa2P\xcf70\xb5\x904|s2?\xc5\xba|M&\x82\x90\xcf\xb5\xf0\xfb0/ma\x03\xe3u_z\x8b*\xe9w\xeb\x9a\xc5sN\x1b!\x8c\xfa\xd7\x87g\xdd\x91n\x94\x0f\xaa\xc1]n\x0f\xe4f1\r\xe2[\x0c{\x01g\x1d\xa3n\xc6]\x95%h\xaf8LT\xc6\x89[\xd6\x0eUw\x9e\xb5{\xb3\xc1\xfb\xa7X\x97\xafw\x024\x8e\xae\xbf\xfej\x11\xe1W\xf9.\x98\x9dO\xd4\x10\xbac\xca]K\xec\xec\xafF\xe7\xc5\xc8\xdcv\x15\x8e\xc1z{8\x9f\xd1\xdc\xce\x05r\\\x08\xbd0\xa6#\x11\xc2\xf2\x15 0\xca\xf0$\xfcz\x11\xda&(!!\xe9\x00\xa7\xe1O$\xfb\xadr\xacO\xb1._\x8b\xe5\xe9V\xe1\xaf\xe2\xb9\xaf\x12\x0f~ \t\xc0\r\n\xa1(\x04J\xad\x98p\xafX*\xaek\x0f\x90\x84\x8be\x97\xc1\xe5tp\xd12?\x16\xa4\xf9x\x8c\x8a\xd6\xcd\xe1\x94\x06\x86\x828!\x90\xdf\xe0\x0bS\xc6\xbau\xf7mYa\xe7\xa1~\xd3Z\xfb\x94\xea\xf2Uv\x1c\xce"\xcc\xec\xa7h\xcf-\xfa\xa1JA\x1bat\xa6K\xab\x83\xcf\xed0\xbf\tm\x1eiE\xff\x08\x84\x08\xd4\xfa9\xfap\x18*\xc5vX\x13\x97l\xe4\x01\x98\x9e\xba.u\x99\xde|\x7f\xf4)\xd5\xe59\x99\xe8\xf3\x1d\x19\x82\xbc\xe4\x1e\x8d\xa8\xeeY\xd7\xaf\xed\xcb\x82\xc4\xe9c\tw\xd7\x8c\xc9\xfa\x94+\x12\x9e*\x08r\xd4\xadD\xd9\x01\xb9\xe4\\\xbcn\xc8\xa7(:\t\xfb\xec"\xa1~\xe3]\x97\xe8z\xa84X{\xac\'\xca\xa9`N\x84UhA\xa9\xbf\xb9\xf3?\xa5\xba|\xd5\xca\x08\x86\xa3\x08\xf1\x92{T\xa4fF\xc5\xd5dlU\xd4\xe0\xf5@\x01\xcf\xcd#\nM\\\xabG\xf0\xec\x8a\x17%\xbb\xb3@\xa8\xde\xbc\xcc\x0c\x8fG~\xf1\xcc\xd8\x88\x1a\x07s\xc8\x8e\xb9\xc9{\xde\x86nz \xed\x91k\x17\xc6\x14\x97\\s\xb0~3\xc5\xfa\x94\xea\xf2\xdb\x0br\x8aX\xd7\xc3K\x98]\x94\xa7\xda\x19\x9a\x1e\xa3\xbfH\xa5\xc1\x15r\x85\x90&J\x07\x8c\x05\xc1\xa63\xba\x07\xc2\xdf]b\xd5M\xaf\x97K\xcc\x8e7T=\x9d@\xecP\xbb\x87V\x93\x17R\xba:\xd4\xc92\xba\x1a!:t$\xd1=#\x84?\xba\x94\xd1\x1f\xd4\xca\x04\x8f\xa0\xfc\x9az\xac\xf9$\x8b\xe2\x0c.2\xc2\x9a\'\xd1kU\x80\xb1k\xf2\xc1#\x18J\xf0\xf4\xff\xfeJ\x80\xf8\'\rc\xeb\xb2\xa5_\xa1\x8e\x1384\x84eIs\x16K\xde\x94\x0cx\x9c\xdd\xf1A\x0c\t\x92lnfy^\xb3\xf2N\x10c]+\x888\x10\x0bsi\xa6\xdb\x9dwb\xd3\xf7J\xa1\x95@xj\xab\xdd\xec\x01Dk\x00\x90e\x0f5\xeb`o\xce\xe6\xa7t\x99\xaf\xbd\x89!\xeb\xee|}S\x1f/r\x1f\x90\xfa\xed\xa8I\xd7\x03\xb9/"k\x10\x9bK0\xdds\xe1\x9a\x9dp\x8c\xd2G\xc7\xc7\xd1]W\x18\xe7\xecZ\xb0G>#\xe9\x1e\xe9\xae\t\xbf\x80\xcc\x1e\x0e"U\x8dc\x0c\x93\x14\x0c.\x91\x13r\xd5\xda\xe8\xddW\xd8\x1f\xd2e\x9e\x8b\x96Z\x0bb\x1a}]\xb4WM\xf7\xd8\x8b\xc1r\xdc\xfc(\xf5\xf6r\xa4$\x9a`+q\nn\xa6\xcd\xf6\x1e+I\x08\xb3\xa4\xa3\xea\\:\x11 \xb2\x19\xa2UT\xe6\x15\x91,\xcb\xab\xe2-\xd3x@\x1a\xd5\x9d\x95\x87\xd3\xb1\xbbc\x12\x0c\x0c\xf4f&\xf9)]\xe6y\x04\xc1\x18\xfc|\x8f\xf1\xb2h\x91\xc3\x03\xefx0;\xc7A]9v\x93\xf8\xf1mT\xef\xacK\x97cF\xe3)ZxW\xb6\x96\xd0\x1b\x92G\xad \xa7`\xac\x8b;\xa8a-\x80\xc7T_\x1d\x9bew\x87\x16\xf6\xa0\x18\x03\x1a]\xee\xb6\x82\xf2\xfd\x9bt\xde\xa7t\x99\xe7l\xe2\x04B\xd1\xd8\xebG\x91\x9dv\xcfs\xf7\xe2\x8c\x05\x1b\x966\xed\xda\x9c\xa0\xd6~/\x1d\xdb58\xda[\xc8y\x8f\xd6\xe9\xe0\xa5M\x03\x86\xb7\n\xcfh\xbb\xc3\xd6\xd3F1\x10\x9d\x8dB\xa3\xac\xc6\xb3\x9a\xdd\x97\xa6\x89\x9b|\xadj\xb9z\xcf\xfc\x08\xef\xfc{\xeb2_{b-\xad\xe8?\xc8>\x8c\xf2r\xb2\xaeT\x06\x89\xbd9[c\xdd\xf6&\x1e\xdf\xc3\xf2X\xaa$G[\xdc\x11\x9b\xd6y\xbbe19\xc8\xc5\xc8\xa5\x19S\tP\xc3\x03\x1aP\x88k\xf2\xea\x95GI&\x97K\xb4\x8f<\xd5\x10\xf6\x86\x19<\x00\xee/\xea2_\x9f.n\xba\xcc_\xfe\x9a\xfb\x7f\x90.\xf3\xf7q;6\ne{\xa4\xdf\xff\x91n\xba\xcc\xa6\xcbl\xba\xcc\xa6\xcbl\xba\xcc\xa6\xcbl\xba\xcc7\x1e\xe7\xa6\xcbl\xba\xcc\xa6\xcbl\xba\xccwV[6]f\xd3e6]f\xd3e\xbe\xed87]f\xd3e6]f\xd3e\xbe\xb3\xda\xb2\xe92\x9b.\xb3\xe92\x9b.\xf3m\xc7\xb9\xe92\x9b.\xb3\xe92\x9b.\xf3\x9d\xd5\x96M\x97\xd9t\x99M\x97\xd9t\x99o;\xceM\x97\xd9t\x99M\x97\xd9t\x99\xef\xac\xb6l\xba\xcc\xa6\xcbl\xba\xcc\xa6\xcb|\xdbqn\xba\xcc\xa6\xcbl\xba\xcc\xa6\xcb|g\xb5e\xd3e6]\xe6- \x02{\x0b\x88@\xffeH\xdfZ\x97\xf9\xe9\x81\xfd*]\xe6\xa7\x07\xf6\xabt\x99\xf7\x02\xfb7\xac\x92_\xa5\xcb\xfc\xfc\xa5\xb8!,?\x07a\xa1\xff\xe7\xa1\xff9\xc2"\xf0\x0cF\xa2<&\xe0\xa8\x88\x93,\x02#,\xf9\xd5`\x86\xa4QZ EZ\x14\xc4\xaf\x16\x180- 0\xcd\x88\xa2(\x08(\x8c>\x1b\xe0\x11\x84\xf8l\xa4\xf0c]@@\t\x0e\xc1\x9f-\xa0h\x94\xe7X\x18[\xffV\x9c\xe6hJ@9\x1a\xe2\x11\x9aA\xd7gL\xc3\x10\x8a\t\x1c\xce\x13\x04\xf3\xd4\x1aD\x9c\x80\x19\x9c\xe4\x11\xe8\xa7",\xff\xdd4\xed\x1f\x7f\xf0/\xfaQ\xe4\x9f\xd8\xb3\xe9\xeb\xfa\xff\x9f\x19,\xdf_\xb0\xf9\x03\x83\x85\x12Y\x8a\x85\x08^\\G\xc1\xa38\xc4\x91\x14\xf1\x84% \x02\x87E\x04A1\x9e\x80\tz\x9d^\x88\x83i\x92\x100\x84\x80i\x96GDVD)\x82z>\xc5\xcd`\xf9\x89\x06\x0bI\xc3\x0c\x81@\x8c\x80c\x04\tS\xa2\xb0\xee>\x01\x85\x04b\x9d,V`y\x81f\x10\x86\x12\xd6\xdf"\xb0uu\x11\xc4\xfa\xb3!\x8a\xa2El]\x87\x1c\xcb\xff\xe3on\xb0\xfce\x7f\xe0s\x06\xcb\xb3Q\xc6\x7f\x19,\xd0\x1f\x1a,\xdf\x7f\xa3\xffR\x83\x05!~\xdc\xbe\x1fD\x0fL5\\)\xf5\xa1\xcb^\xb0\x10\xb7c\xea2\xbcu$O\x021\xc4\x90i\xaa\xa4\xe3Vy\xc6\xc6\xe1\xb5\xe6\xad>f\x07\xf2:\xd1\x14\xb1\x1b\xee{\xcfg\x8c\xfd\xc5O\xb8\x82\xb6r\xe4\x96P\xac[\xdf\xe4\xf6\xcd\xc6\x8b\x9f2X\x9e\xb7\x02\x8c\xa1\xebm\x85\xbe\x84\xd9q\xa4\xb0\xb8\xddi`\xce\xbb\x1b<\xdf\x114$\xcf\xbe\xfb@\x1as\xd6\xec\xd4tN\x16\xe3N\xd5eW\x8djCg\xc1\xce\x08D\xf8\xe1\xf9G\x92b\x1b\x95\xb1\xe2\x90\xba\xc6\xcfw\xd1\xc3\x10\x8d\x9e\xa5`\xe5\xf1\xf0\xa3n9?\xd9`Y\xc3\\\x93\n\x84z\xe9e3I\xa1sQ\x1d\xa3o\xda11\xc9&\xc8\xae\x963q\xe0\x95\xcf\xccA\xed)\xdb\xf4\xb12\x9b\x17p\xd6\x99@\xc6]\xe7\x08\x92\x18v\xc3j1\x94\x9dj\xd7=$]\x9e\xc1\xf08\x13\xb8a\xd5aU]\xe3u"~\x89\xc1\xf2\x8c\x11\xc5`\x82\xa2_\x9b\xda\xeb=\xe9\xcc&r\xf1\xe8\xe8v\xba]\x8b\xdb\x9e@\xe2\x11+\x8af9\x16\xa8\xdf\xea\x9dSV\xea^\xc9\xb8D\x12\xe0Nl\x84\x1b\x01]\x1c\xcd\x93\n\xcc\xbf\x9a\x15\xb7\xa7v\xeaD65\xe8M\xa5\xe8\x18Y\xe2\xe5\xbf\xc8`y\xaeX\xf4\xd9\xa7|\xbd\xd1\x7f\x1ff\xe4\xb9\xb5Z+\xa4\xeas\x19\x8a\xd6\xd01n\x8c\xe35\xbc]\xaf\x93\x8b\x8f\xd8\xd5,Y\x82\xd5v\xcd\x94\xe8\x88\x11\x8b\xaa\xb3\x07k\xa0\xea\x15\x90\xb8^s\x88\xd21\xc9\x9elu7\x9a!:\xc4A\xc7%\xc9\x80\xfd\xa8)\xd9\xdf\x1ba\xf9z\x8a\x08\xbe&\t\xe4K\'\xd2\xe34\xe1\xd3\x81\x05\t=\xc7\\0r.sg7\xe3C\xda\x93\xc0\x04\x1dz\xcc7*C\x16\xe6Z\xa0\xb8p\x1aeb\x9f\x0e7\xae\x1f\xe1>\xb9\x1f\xa4\x8a$\xac\xb1;(#r\xa5\'=;H\xe8H<\xf6\xfa\x9b\xcd\xc0>\x85\xb0\xfc.\xe9\xfd\xbf\xa3\xb4zT\xbe7l\xe3\x1a\x8a\xe4\x11\xc76\x8c\x1b\xd5\xd3\xe9k\xee\\\x17\xa9I\xc2+\x7f0\x01=\xc7y\x12\xba\x17\xc2m\xe7M6s\x82\xa0+\xfd\xd8\x9fZ\x033\x14\x17\xb7\xc0#\x12\xdc\xb4\xa5 \x03JY\x13\x9b\xfb\x9b\xadB?e\xb0\xacQ\xa2\x04J\xe0\x18\xf5\xda\n\x19\xeb\xe06\x08\xb4\x18c\xcf~wj\x07~\x7f:<\xce7\xa3\x0f\xe6G`\xe1\x93\xcf\xd8e\xc8$\xe1\xb9!g<\xae\xd0\xd0\x87\x12\xf8zDv\xca\x11\xdc\x91\xc5M\xdd\x85\xbb\x93Y^\xea\xa9s|\xa7p9\x84\xf4\xde=\xe0>d\xb0<\'s}\xca$\xf9\x07B\xc1\xd5Q\x8e\\p9\x14qz\x14d"-*H\xc1nd\x1f\xb57>\xc0o7"\x0b\xa7\x0c\xae\xa9@\xbe\xc3\xf5e\xa7rg\xbb\xa3\xf7\x94A7\xfe\xdd>^w\x08\xbb8\xe3\xae%u\xd2\xbeyM\x1b\x9f\x1e\x97\xe3\x9b\xbd\xf5?e\xb0|]\xc9\x08F\xad\x1b\xe0\xe5J\xf6yA\x03\x17eh\x82\xde\x81,\xcb:\x93\x13\x8e\x1e\xd0\xd2\xce\x13\'\xc8\x8f\x0cIG\xf51N\x0e\x1a\xb0\x93\x13\x196\xd0\xaa\xb7\xf6\xdc\x18\xc5\xa0u?Z4n&\x19C3U\x9eO{\x07c\xf0\x14M\xf4\xf8\xcd\xad\xf9)\x83\xe5\xb9hI\x8c\x80\x91\xf5\xd1\xbc,Z\xc0\xaa\x96\x01h\xd8\x92y\xa8\xb7\xb9 \x9a\xd8\x9aX\x9f\xd7\xcf\xf4\xa3\xec=U)\xa5\x9eK\xee\x0f\xce\x85\xe5\xf9Z;\xfc\x8eC\x98\x04I\xbac\x02 DRZ$x\xeb\xbbb=\xe1\xef\x81\xed\xebZ\x18\x98\xee\x7f\xa6\xc1\xf2u\xe9\x13\x10\x81\x90\xf4\xcbmh\x1b3\xb0\x8f33\xf3\x1dZz\xec\xb2G\x10^\nA\xbf\x98g\x1e\xc22Q\xdf\xdds\xff\x06\x1d=\x19\x1e\xfd\xf16\x87%\x03*\xac_\x95\xe9.\xea\xcfD\xfdH\x99^K\xc9$\x0f]Q&\xd8JR\x0cV{\x17\xd3\xfa\x90\xc1\xf2\\,\xeb\xe1\x06\xe1\x7f\xd0\x1f\xdcLE\xe7R\x9cs\xb7\xda\x11v+%#\x94\xddP\x10\xee\xae\xe29\xd9\xdd\x90\xc1\x13\x05q?^}\xaf\x83\x97a\xee\'5\xf2%n\x1f\xden\x13\x7f;\x04\xc4\xd2\x95\x97\xe3\xc8\x95\x9d\'\x1e\x19\x06>\x0f~/?\xdem\xb8\xfa!\x83\xe5\xeb\x84\xa30\x9c\x84^g3\x01J\xce/\x9d\xdd\x8di\x85!\xb8\\\x08\x80\xa0}c\xca2#\xb6f\x95\x98,\x0c\xe2-$\xcc\xe6.m\xa2G\xae\xef\xbc\xa1\x0e/\xe3\x83\x89\x90\xc2\x05\x0e\x13p\xc6\xad\x80sb$B\xbd1\x87\t\xc5\xe1\x1f?:\xc8\xff\xde\x06\xcb\xd7b\xa1I|-<_\x16\x0b\xa4F]\x80\xda\xfa"hF}\xb80\xe4\xa3EQ\x19\xc8\xd4R=\xf8\xc6\x85\xca.\x92*\xec\xd9y\xcc\xaa\xa1i\x1d\x8eb\xb8\xd2\xeb\xc0\xb4\x97\xc5\xe1\x10\x9f\x13X\\L\xed\x1a\xce\\\xda5c\x8b\xf1\x08\x9e\\\xdel[\xfb)\x83\xe5\xb9\xf5\x9f\xb7\x04M\xd2/e\x8d\x1e\xf0\x04\xebFcc\xe0\xcb\x0e\x06\xc5\xa9\x00:\x8a\xf3\x15\nw|\xab"v\x82+IW#6\xd5X\x8b\xa2\x9a\x86\x89\xeav\x9f\xefD\x88\xb7\xd6.f\xa5\x03<\xb2\x02\xf6\xe0\xa3\x86\xcd\xc3\xfe$\x14j\xc3\xbc\xab\x84~\xca`\xf9\n\x13_sU\x9c|)kNF^\x05\xda\xbeo\xe6\xc1\x0f\x8f\n\xa4$\r\x95\x164\x0e\xe8\x8f\xc3\x9d\xa1\xce\xe7\x00\x8f\xd1\xb8\xd1g<\xd9\x95\x93\xc1\xcce<\\d\xd0Z\xea2L\xa9\x01\xe0\xc4\x13\n\x8cE\xe4\x88\x07\xe1\xe1F\xcc20\xc1\x9b9\xdc\xa7\x0c\x96\xaf[\x9fX\x97,\xf1\xdak\x19A\x16\x1e\x81\xef{KYO\xd2\x9c\xb0\xe7\x0c\xa5@g>3K\x81dg\x90\xc9\x0e\xc2@\xc6\x00\xee\xda23\xde\x97\xbbV\x14\x97\x9b;\xf6\x02\x92\x8dhy\xe4\x0b)\xcfD.\xd6\xe5\xa5\xde\x8dw\x0c\xc7\x08\x97|3\xb9\xf9\x94\xc1\xf2\x9cM\x18\x83\x9f\xdd\xc6_\x92\x1b\xd4?\x1bE8\x11\n\xb1\xcb\x9d\x94\xcfJ\x92+\xeeh\xd3y}\xe3\x97\x92cV\x97Q<\x88\xf2\x8e\x8c\xc5ua\x9d]6\xb9b\xddA\x9e\x84\xee\x04\xa9\x97\xae\x9cd"Vi\xcf\xe8\xaa\xbb\x915\xc3\xb2x\x1d\xf6\xe6l~\xca`y\xce\xe6\x9a\xf0=?\xc9xIn21\x8c\xcc\xdb]\xaa\xd83c\xc3\na\xe3l\xa8\xcd\xf8\xbd*\x8aJx\xb0\x9e\xc0\x95\xba\xae\xedT\xc3\x11\x8b\x96\xb9\x8f\t\xe1\x12E\xc8\xd9\x9a\xddF\x95R68&\x90\xccx\xc2X\xffhW\xf95\xe4.\xb7\xe1\xcd\xbd\xf9)\x83\xe5+L\x08\xc60\xea\xf5\xed\xd15\xc5\xa0\\\xc0\xeeT\xaaT\xc1#i\x0bW\x81KP\xdb\x91v\xcd\xb5\xea\xc3\xf5\xd1\x8b\x0b\xaf7\xf6\xadDb\xe3R\xb3\x95[\x8c\xf7\x16#\x06\xbe+\xa6\xea\xb6k\n\xa3E\x8aJ\x05e\x1f":\xc2\x00\x11\x95y\x17\x0e\xfa\x90\xc1\xf2\x0c\x13\xc3\x10\x98"\xe9W\xdd\x964\xc2\xc8+\x9b\x94\xce\x9d\xde3x\xc6\xe2-\xc5\x93n\xe3\xa1a\x90k#\x81w o\xfc\xe6\x1cg\x8b\xef7\x95\x13\x99\x9e\xb9\x84\xa4h\xde%%\xe8\x98\xe0\xc8\x1a.\x97\xc2!\xe8\x9f\xfb\xbb\x9aib\xe2x\xef\x16\x1e\x1fBX\xbe2r\x98Z\x93\xd2\xd7\x0bE5Y6\x87ZY\xde\x83br\xd7H\xc0\xcaO\xa4\x9e\x1b\x19Gc\xe6,8\x85B\xe8\x82\xd9\xf3\x93Y\xd3\xb6\x93[\xd5=\n\x8f\xe0\xe9\xe2*ns\x9f\xa4\xf1\xa6\x1b\xd0}2=\x0c\x1d\xa8\t{L\x1e\xa0\xf0\xbf\x08ay\xce\xe6z\xfc\xa04\x04\xbd\x84\xc91\xb5\xa2t\x81\x08q\x17\xceS\xc0\x07\x8e\x9e\x1a_|\xa0#:\x11\xce|\xa4\t\x9d\xc0\xeb3\xfb@\x1c\xdc\xe5\x1a[\xb45\xc2Q\xe4\xf94\x15\xc7\xc5\xba!\x06Z\x03|\xa3\xd0\xf5\xdd^\xe6\xa5\x11\xa0\x06\x0f\xe1\xef\xa5\xf8~\na\xf9Z,k\n\xb3f\xf3//\x90\x02z\xc7\xe0\xc0\xd5^\xab.dg\xd6z\x1f\x80\xae\x995F\xda{\xbc\x91\x1e\x06wO\x84\xfaM\xbc\xd9\x1c^G{\x93\xb9\x8a6NYi{\xabQ\n\xc8\xef\x80&\xacgE\xaee\xc3\xcdb=\x1f\xeeE[\xbe\xbe\x99d}Ja\xf9z\x81\xb4\xe6\xf08F\xbe\x1c\xe4\xaaQ\xa9\xfe\xfdb\x9a\xb7\xd6\xd7\xa1@\x94\xed(\xb52\xf8p\x98\x1e^\xb7\x9e:\xb1\xa7\x9fT\xb5\xbe]h\xe4\xe08\x15\x89\x14\x84n\xe5\xb1;B\x91(\x9f\x94\xd9\xbd\x96\x05\xc5\x07<\xd9\x88\x16F\xccH\xd74\xde\x9b\xd0\xdd\xa7\x14\x96\xe7l\xe2\xc8\x9a\xc5P\xaf\x02\\\x8d\x93\xc9\xc1\xcd\xa3Cn\x80u56\xb8\xa1\xa8\'\xedn\xd7\xc6\xe3l\xdf"\xf7\xdc\x1dx\xc8\x8fq\x9f\x87\xa3\x0el\xa3\xf2\xd6\x9f\xa2G\xd4tn7\xaa\xf4\x02\x84x\x93\x1e\x871\x97\xcd3.S\xd1E1(\xfaM\x02\xe1S\n\xcbs6\xd7\x13\x0e"\xe0W\x01\xcentG\x0f\xc1|7<\xdc\xd9\nL\xabP\xd1\xbd\xec\xbaF\x14\xd0*\xc3\xf3\xbd\xc1\x94{\xec\x8a\x996\xd8\x8f\x87\x94\xbf.\xfaMR\xf3B\xbcC}\x9f\xdc]\x86$\x81\x1b,\xa1\x8c\xfe\x90\x118n\xf2<\xc9\xdf|\xb5\xf2)\x85\xe5\xebZF\x9f\xbfL\xbd\xcc&\xc5\xd9\xf6\xbc\xf82\x17\x1b\xe0\x896\x1d\xaf\x9d&\x97\xc6g\xbd\xd3v\x07\xfaHD\xce\xfd\xbe\x9c\xab\x0c\xa7/\xea\xd0\xa9\xa5%j\xb5\xe4Jnt\xb6\x8e\xa7\xb6\xec\x00\xe2\xdc\xda\xe1\xdeL3\xb0*\t\xa6j\xb1iy\xf7C\x9d\x0f),\xcf\\r-0\x9e\xa5\xe1\xcbl\x02i\x8c\xab\xc9\x00F`\x0c\xe8\x91\xcfB\xe51\x03\xe5a\x9f\x97f{\xbe\xde\xa8\xae\nv\x8d:\x15\x919\xed\x02\xa2k`\x88E\xc2\xd3\xc3]\x16\xce\xea\xf7>\xb4\xdb\xf1\x9ds-bp\x02\x8e\xf6\x10\x95\xaa"Uo~D\xf7)\x85\xe5\xb97Q\x1aCa\x82z\xa9\x0c\xf8\xfc\xc6\x1e&ab\xdc\x8b}6\x92\xc7E)\xf9\xd4\xbc\xd2\xaaA\x88\xe311\xf1\xf9\xd8\xd4g* \xfb\xaa8+\xc5C\xa1\x0f HG\xfe\x05\x12\xe2G\xfd\x08\xba\x89\x8c\x87\xea:\xe3\xf5L8\x81\x86\xea\xb4\xad\xbcy-\x7fJay.\xda\xe7\xbb\xa1\xb5\x10~I\xb2\xc4\xc8#\x01\xd8\xb8\xddh\xe4jw\xa0}tN\xbcV\xcc\xe6ZG+\x006\xf0\xa7\x9eV\xec\x98T)h\xde\x81K\xc3\xa8!f\xca\x97\x1b|\xf7\xcf\xaa\xc0\xd7v\xf3p\xcfU{o\xf0J\x91J\xe7P\x05{\x0b\xe0~\x8d\xc2\xf2\xf5I\xe4\xba`1\xe2\x95\'\xa9wg\x8f6\xe6\x9e\x91\x0f\xb9\xb4\xdb\xa3\x8e_\xbb\xc7\xe9Z1J?+\xfd0\xfbg\xab>\xed\xb9\xf9t\x15s\x89\xef\x12\n>\xe6\x11\xefG"F\xdc\xf9\xcbe\xd9\xcfc\x89[L\xd6,\xcca\x7f\xf1\xa9\xf6\x81\x16\xcc\xafQX\xbe\xeeM\x84\\\x93f\xe2%\xc9\xca\x06Bq\x1c\xbf1\xbb\x8c$\x85v1\xc8\xd9\x9d\xc8\xa4\x1f$\x90~\xb0\xd1\x8c\xdev\x060\x9d\xf0\xc8 /\x8f6:\xed\x1b\x13\xe8\xf9=@\xb8\xb7+\xbb\'\xc6s*\x81\xbdj\xb3\x17\xde\x12*CO\xb3G\xdb\xfd\xa8\x00\xfa{+,\xcf\xa7\xb8\x96\xdb\xe4\xba\xff_^d\xd1gSM\xdc\x03\x10@^\x8eD\x8aB\x9d\x81%\xbe,C\xa9\x9ak\xc2\xd1\xe6\x9e\xd0s%\xdc\t}\x17z\xc6\xce\xb7/\x12%\x88\x10 \x8cAh\x88\x15!\x9e\xc3a\x12\x15h\x14E~&\xc2\x82\xfc\xf7\xbf\xea\xff\xc7\x1f\xfd\x83~\xfa\x9f\x18\t\xd1\x10\x8c\xe1\xf4\x9f),\xdf\x9f\xb0\xf9\x03\x85\x85\xc6YQ\xa0X\x86\xe5\x10\x1e#\x9e\x1d\xd3\x04\x92}\xf6\xf4\xc4hN`y\x8e\xc7a\x98c!\x04a8\x0c\x13\t\x08\xe69Rx\xfe\xdd$\xb6\x86+<;\x84l\n\xcbOTX\xd6\x1d\x86\xf24\x86Q\x10$\x10\xeb|#4\xc1p\x08J1<\xc1\xa0\x8cH3\xb8\x003\xec:=\x0cE\x0b\xd4\xba\xd2\xd6I\xe7\x10\x8e\x87y\x84\xa7h\x16\xf9\xc7\xdf\\a\xf9\xcbf\xc6\xe7\x14\x96\xe71\xf6\xa7\n\xcb\xf7\xdf\xe8\xbfTaA\xd1\x1f\xf6\xef\xe7\x11\xa7\x07F\x80\x91:\xf9\x06\x9b&\x1a\x88\xa1xM\xa3\x9a\xbe\x1cQV\xd8=L\xa1\xd9\x1d]\xe9\xd9z\xa3\xb9#\xa7\xbcX\xc8k\x9cz\xb1\x9c\x04X(\x94b\x08\xdd\x84\x13^\xfa\t\xbc@\xf2\x1c\xb11\x993\xec{=\x81>\x86\xb0\xd0\xff$ \x12\xfb\xea\x81\xf5\x12\xe6N\x80L\xba\x19N\x1a\xc2\xdd\x9c(\xb7\xb5\xa3\x0fzB\x14&Vs\xbe\x1ezl\xaa\xdc"3x\xb6U\x9bq(&hv\xaac\xbb\xcea\x8cs\x83\xc4[\xc8\xf1\x98\x12!\\\x9b\x8b/aQQ/\x9ai\xbd\xd9\x96\xecc\x08\xcb\x1a&J\xa2\xe8K#\xbd\xdb\xe1\xf2\xe0\xeb\x8b\xec\x88u\x89h\x82&N\x00S\xdc\x03;\xaen\xfeH\xed\x81C\xb0\x86\x9c\xc5\xa7\xe1\xd1%\xc6\x9acB{&E\xb3\x90W\xfdPA\xea\xab-h\xb5f\xc4|\x1c\xb3\x87>\x8c\xce\x1e\x1e\x06\xfe\x9b\xadB?\x86\xb0\xac1\xc2\xf4z\xe2S\xf0K\xdf#\x07%\xb0\x03B$L\x93)L\x06^p\':9\xaa\nJ\x82\x87\x1b\xd0@\xc2\x0c\xa2\xe8-\xcf\x80\xd5\xbd\x1b\xa2\x0c\x88\xf6|6\xc4\xa2=<\xae\xba,\xa7E\xaa\\w\x90F\x04\xbd\xadxU\xdc\xc8\xcd\xd9\xb0\xdel\xef\xf41\x84\x85\xfe\xa2\x18\x9e-\xb6_V\xec\xb9V.\x9cy\xf3\xfa\xab\xdf\xdfJ\x80c\x88p\xe2\xb4\xe3\x81\xb1)`\xbf\xf3\xf9q\xff\xe8Z\x90\xaa\x010\xd5\xca\xbb\xf4\x10\xe6C6\xfa\xfbv\x7f\xdc\x97\xe1|\xbb\xb34\x9b]\x013\xea\xcd\x9c\x03"\x9c\xe8\xbdH\xf9^\x8d\xca?\x86\xb0\xfc\xb6X\xa0g\x8b\xf8\xdf?E\xe9\xc1[\xa7\xc1\xe0\x8f\x08O\x1f;\x02)H\x04\xac\xd4\xc7\x02O;6\xf0}\x90:\x1e\xa1\xc7T\x89\x8dZ\xecj\x1bH\xa2D~\x10\xfc\xad\xc2\x10\x85\xb4oMYF\xc2\xf9\xe1\x0e0\xcd\xd5@B\x06g\x12\xec\xc67;\xbb}\x0ca\xa1\xff\xb9\xe6/\xf8z\xcb\xbct<\xeb\xf0\x1e\x1b\xed\xb3b\xe0\xb0\x97i\x8a\xec0v\xda\x03\xb0+\x1a<\xd9Xfc\xb1L\xdb\xec.\x81\x91e\x0f\x87\xce\xd4\xfb\x8e\x07;\xec\xb6\x87\xeaLi\xad\xf30\xd0\xc4U\x07\n?\xafwnd6\x89uR\x887\x1b\xbb}\x0ca\xf9]j\xff/\x1d\xcfd\xc9\xc9\xef\xf9\x11tp\xd86v\x16\xaaF\xf3la\xa6\xd2t7\x1dP\xc4\xa3\x86\xa4\n\xban\xfaGR+\x10[\xf7GL\x9e\x92\\\xdf\xdf2\x80\xbe\x04sF\xb4;O|\xb0l\xcb\x8e\xedy\xba\xe3\xc3)\xfaU\x08\xcb:\x99\x18\x84\xaf\xab\x96zi\xba(>z\xa8\xc3\xb2\xdbu\x1a\xae9\xc4\x0f\\W\xefn\x04\xa5\x83\n\xed\xe5\xd2\xcd+LSb% ;M\xae\x99%\xc0i\'\x1c\xe5\x1e\xd4\xef\x05\x1f\x8b#\xe8\x0f\xd28\x88C\x81\x17s\xd80\xf7\xabT\xe7\x11\xa8\xbfy\xc0}\x0caY\xc3\\\xabS\x1c\xa3_\x17\xedb\xd1h\x0f\xdc\xe6\xe3\xfe\x0e\xb5`\xd0_\xf6\x8f\x83\x90\t{NMD">r\xc0\xb5/\xe8t\x18\xf8{{\xd7h\xc18)7\x94<\xf7\x9d#\x8c\xf6PT\xc6\xe1\xd9\x16>H\xf20\xdbk\x8a&\xf5\x97\x1b\xae\xbf9\x9b\x1fCX\xd6E\x8b\xe3(\xb6N\xe9\x0b\xa9C8\xfe\x1e\xd9\x1d\xae\xedi\x1e\x13P\xce\xf6F\xcf\x04`\x9b\x8a9\xfe(:\x7f\xc4\xc8\xa13\x9dGs\x10\xc4\xeanj\xb4\xedT\x88ha\xca\xf1\x82\xc0\xc6=\x0b\xd3\xb9\xdd[\xec\x83; \x0cr\xd9[\xc59a\xe4\x1f\xb5\xd0\xfc\x9b#,\xeb9\xbe\xee\t\x04YS\xf9\x97~\xabl+a\xfc\x12M\xf9\xe4\x9dQy<&\xf1z8\x1f\x994\xf6\xbc\x10\xd02\xc2\xdf\x1d\xb3\xe9\xe1\xcb\xd8\xcem\xb0\xec\xaa\xec\xf7\x1a\xa8.\'\xd2\xad\xae\x9d\xb8d\x97\xc7\r\xbbE\xb3\xb6\xde\xaa\xc5\xe5L\xdb\xb5\xa3\x9bo\xf6B\xfe\x18\xc2\xb2.\x16\n!p\n\xc3_\xc24\xa8\xdd\x1e\x19\x82\x94[\xae\xd9\x90\x06y.\x15\x8ey\xd0\xe5\xc4\x82\xf0d\x99S\xc0 \xca \xea/\xdd\xe4\xa1dP\xd6\xba4\xf0=\xa9\xb9\xce\xa1\x99\xcc\x99\x16\x04\x93b\xd7?\n\xbbC{^jT\xa2\x97\xa4|\xb3\r\xfa\xc7\x10\x96u\xeb\xe3\x14E<\xab\xa5\xdf\x87\x89_\xa9<"\xb8\x01\xe8 b\x08)\x97b\xc8=\\#\x11v\x07\x1eD\xe72\x07\x0f\xb1\x07DS\xbb\x83\xf6\x88\xd3>\xb8\xa9\x85|P\x04\xd1\x94%\x056\xb0\xa9\xd7\x01\xa9\x8b\xf7\xc6\x19\x08\xc8\xd6H\x8b\xa0\x91\xd2\x1f\xf5B\xfe\x9b#,\xebS\\K\xf3\xf5\xcfC/OQ\x1d\x1e\x10}\x08\xfb(\xa78d\x84.\xc7\xc2#\xa4\xe9\xecEW\xd8\xca\x159\xdc/\xda\xd13\xa8\xc9\xbb\xe8\xc9i\xfd\x1d\xf7D\xcd\xf2\x91#\x1fz\xea\x05\x81fq\xde\xd18P;?"\x93\x01\x97N\xa6\x8bK\xf8\x9b\xcdy?\x86\xb0\x96\xcc\xa1\xe93\xf3\x9b\xb3\xf91\x84\xe5\xb7\x1c\x8e\x84\xfe\x80^\x83\x1c^\xf5z\xb35y%\xd1\xd1\x96Nd\n\xeb\x94\x1d;\xd4\xe1\xf0\x98\x0eL\xc1j\xf5\xe5\xac{i\x93\xed)\x13\xae\nb\xa9\x98\x8a\x9e\x87\xc8\xae\xf7\xcc\x99\x0eg&\x1f/&\xa5\xb6\xbb\xab\x89\x8e\xe0P\xd2\xca\xafBX\xd60\t\x84Di\x14}\xd9\x9b\xda\x1dC\x8e\x08\x9a\x10\x93\xa2\xf6d#\x1c\x1bU\x95T\x87\x9d\r8bFW#\xa0C7\xed\xe50[\x92\x045\xb1\x02TQ\xea\xa1\xe0\x8f2\xa9\xe1\x9c\xe8]\x12\xc53j\xbf\x00\x05\x86$7\xa4\x94\x87\xa9\x8b\xdf-\xaf>\x85\xb0\xac\xd72M"8\xb4\xdeZ\xbf\x0f3>\xe1\x02\xe5E\xe1\xcd\x1b\x97\xdb=\xbd-\xfb\xbc\xb1\xd3,\xeb\x8f\xc0^\x1cp\x9a\xac(\x94u\xa3\x98\x9ek\xab\xf7\xf7\x05\xa9\xd9\x08\xce\xb3\xfb\xd0z$g\x15\xbb\x94ip>g\xc8\x99:\x9fz\xa0\xd3\x8erx\xb7\xde\xb5f>\x85\xb0<3\xf2uO\xd3$\xf6\xf2.p\xbaG\xe4nT\xd2\x80\x11\x95P\td\xc3\xb8TRi\x1a\x16k0X\x1b\x11\x90MU\xf79\x17\x13RZ\x0eE\xc8\xdc\x8d\x94\xcc\xcf\x0b\xc7>\x8e\x8atgowl>\xb1\x8b[\xba\xae-\xc3\n\x973e\xa8\xbdy\xa1|\x0caYg\xf3y\xd0\xaee\xdaK\x98\xfe\x00P&\x1e\xe3woLR\xb5\x8e\x03\xd4D\xd1\xdc|\xecl>\xe9$OH3g\x9f\x89)\x97\xc5\xd5 \x17\xa6\x041\x82DtuG\x1e\xc5\x8b*\xec\xcf\x8du\xa3A\x94\xbb\xe9m\xc5\xbb\x88"p\xeb!\xfc\xe6\xde\xfc\x18\xc2\xf2|\'\x80\xae\x89\xf7\x1f\x9c\xb4y.\xaa\xa2\x0f\xdb!\xd5x\xf2-\xed\x08\xa3j\xab(j\xc3x\xcc\xba\x98\xd4\xc2C\xa1\'P\xbeSy\xe6\xc4V\xa3\x9d^\xaa\n$\x81\xc3\xd9\x02\xfd\x00\xe3\x11y\xbfx\xfd\xac\xe2\t\xe71s\xa0_\xa7\xd1U\x7f\xf4\xca\xf3o\x8e\xb0<\xf7\x04\x8dP\x18\xf2\n\x13\xf9\xca\x95\x07\xb8\xc5\xae\x1e\xba\x80v\x15\'\xcc\xa6p\x19d\x9c\x0ci\x19\xb5\x96\xd1\xc1\xcf\x80<\x81*\x89\x8c\x12\x01\x98(\xae\xd4\x99w@\xa9\xe6\xa0t\xc7\x1e\xeb*W\xc4\x90h\x0ei\xb7\x9e\x98\x1d\x17\xc0F}\xc3\xae\x83\x9f\xabP\xb9\x84\xb9x\xbb>\xf4\xc74\x82B\xba\x17\x98L\xbf;\xf8b\x8e\xca-_.x\xadv\r\xc8\x14\x17!oo\xf8\xa8\xde\xa4\x18\xac\xbd\x05\x91\xe0\x92i\xfbC\x03\xbf\xf9i\xc7\xc7\x10\x96\xafO;\xf0\xb5\xc4~\xfdP\'\x86\x99\x93P\xb7}\x08\xea7\x9b\xba\x0e\xb5K\x90i\xe4L*\xfbp\xe9\xc0\xed\xf1n\x17\xef\x83I\x83\xcfL\xd1\xd5S\x03\x82\xc0Y\xdfEl\x82\xd8\xb5\xe5JT\xa9\x1e\xa1\xe0rbO;E\x05a\x15\n\x84\x89\x7fS\'\xf9\x18\xc2\xb2\xce&\x89\xc3\x18E\xbf\xeeMo\x87\x9f\xe4\x88\x93\xd3\xae\x9c\x8fDV\xcc\xb7\x0c\xc8\x8a\xc9\xcc\xf6\x8d;`\xb7*\xd7\x0fb\x98\x80&\x840\xd3\x0e7\x84 :j\xd7\x8e`1\xccK\x0f\x89\xaf;\xb0\x1c\\\xeeK\x86\xc4\xba\xc1\xf7\r\x9b\rt\xfb\xe6\x1b\xa4\x8f!,k\x98\x14\xbcf\xd9\xf0+\x03%q\xdd\xa0\x8d\xb5\xb1\xa8\xb3\xee\'W\xcf\x91&\x00i\xba\x04\x10:\xc2\xbc\xdd\x03[<\xc8\x04zO\xab\xc9\xbe\xf0@@\x8d\x1eg\x9e\x95\xdc\xb4\x92\xfd\xado\x1a\xb47B\xc8e\x1e\xd0m\x01\xf6\x8e\x8c"\x81\x8c\xbf\xf9:\xf0c\x06\xcb\xf3\xf3\x0e\x18\x81\xd6\n\xeae\xcd\xa2\x8e}h+\xb2\xb7\'w\xc2w\xfcu\x9d\x8c\xa1\x15\xc6%\xa3#XL\xe2d\xc4\xd6<\xeb\x1eS\xf2\xee@\x82\xd6-\xbb\xd8{\xa8R\xefg\xc6\xaf\xecG\nU9\xf6\xb8\xdbq\xae=\xe0\xfd95\xaf]\xcf\xe6\xf2\xbb\xa2\xce\xa7\x0c\x96u2\x91\xe7\x17\r`\xe4e6\xe7xy\x8c\x93\xd2Z\x81\xaf\\\x90\x90\xd0\x1e\xc9\x10\xd0\xd7z\xee\x86\x91\x1d\xaa\xc3,\x1ck\x8f\xed\x99\xf2q\xb6\xeeQ\x96\x9b k\xafu\n\x1b?J\xc1\x08D\\;\xe8\xa7dA\x8cT\xf3c4:])I5\xdf\xac\x7f>f\xb0\xd0\xff$\t\x04\xa2\x10\xec\xf5C:\xe9\xe4\xe8\xa0X\x98\xc7B\'$\xfa\x80\xd5{\xcb\xd6+\x17\xb2\xaet\xcc[(r\x12O}\x1e\xab\xd4\xc9Q\x9d\xc4\xf7B\xe2*[\xb1\x07\xd1#\xdah\xd2\x03\xa4\xef\xa3\x1c \x8d\xb4c\xe4\xa6\xc5\xaa\x19\xf6\x94Tz3c\xfe\x98\xc1\xf2|\x91\x05=?\xbdD^r,\x99~\x1c\xc0\x19\xc8\'5\x18I\xf0\xb8C\xfb\x89/d\xb5\x12\xc8\xb1\xd0\xb1\xf1$\x81W\x87\x94\xe6\xa1\xdew\x1e\xa3\xba&\xab\x81\x9e\x11\xe5^\x01\xd2\xeb=t\xed\xa7VO\x99\xaa\x87w\xfd-\xd7\x84\xa6%\xfa\xf8\xf2\xbd^\xee~\xcc`y~j\xbd\xee\n\x92~],\x13$\x1c\xb33\xaa\xa1\xb3\x9e\xc3\x91d\x96\x04)\xeflq\x88\xf7\xa2\x89\x1d\xaf\xc1\xd9=\x93\xc7k\x14\x8dY\xcd\xab\xfed\xe5\xe7x\xbd@\xc0s\x08I\x88\x16XI\x0b;$?Z\xf7u\xd1.z~\x98\xa2zi~[,\xff+\xc2\xf2\xdb\xd7\xcc6\x84\xe5/\x7f\x1b\xfc?\x07a\xf9\x1b\xb5d\xdf\xba\xdco\x08\xcb\x86\xb0l\x08\xcb\x86\xb0l\x08\xcb\x86\xb0|\xe3qn\x08\xcb\x86\xb0l\x08\xcb\x86\xb0|g\xdcdCX6\x84eCX6\x84\xe5\xdb\x8esCX6\x84eCX6\x84\xe5;\xe3&\x1b\xc2\xb2!,\x1b\xc2\xb2!,\xdfv\x9c\x1b\xc2\xb2!,\x1b\xc2\xb2!,\xdf\x197\xd9\x10\x96\ra\xd9\x10\x96\ra\xf9\xb6\xe3\xdc\x10\x96\ra\xd9\x10\x96\ra\xf9\xce\xb8\xc9\x86\xb0l\x08\xcb\x86\xb0l\x08\xcb\xb7\x1d\xe7\x86\xb0l\x08\xcb\x86\xb0l\x08\xcbw\xc6M6\x84eCX6\x84eCX\xbe\xed8\xff- B\x19\xdf\x02"\xf0\x7f\x19\xd2wFX~~`\xbf\x08a\xf9\xf9\x81\xfd"\x84\xe5\xcd\xc0\xfe\r\xab\xe4\x17!,\xff\x0f\x96\xe2\x86\xb0\xfc\x1c\x84\x05\xfe\x9f\x87\xfe\xe7\x08\x0b\x850\x08\xc4\x920\xce\x92\x02B\x93\x08I\x124+ \x1cA\xf3\xf0\xfa\xcb\x10\xc1\x89\x02\x8a\x0bk\xb9\xc7\x10\x10"\x90\x90HQ"\x03\xf1\xb8@\n\xb0\x08?\xfd\x88?AX0\x12\x83!\x11F\x11\x92\xc5)\x04\xa6!\x86#\x18\x96bh\x11B\t\x82\xc2(\x86\xc7i\x82% \x91\xa5\x08\x06\xa5)\x91\x85!\x88DI\x9eG\xe9\'\xb3\xf0\x13\x11\x96\xffn\x8e\xf0\x8f?\xf8\x17\xfd\x18\xf1O\x08\xa6i\x1a\xc7(\xea\xcf\x10\x96\xefO\xd8\xfc\x01\xc2\xb2\x8e\x96\xc5I\x91\x82h\x8c\xc59\x11]\x97\x00#\xc2\x9c\xc0\x11<\'"\x82\xc0\xa5\xb0<\xaf\x05\x1c\xa1\x88u\xae_z<\xb9\x12.Wj,\xb1\xf2)\xb1\xc0$\xf6)\xd0\xbcv\xc7\x01\xb6;\x13l\x0en\xb9\xc0\xe9\x18\xa7\xd1q0\x86S\xb7g%\xaf:fu\xd8q\x979=\xf0\x89\xa6\xeeP\xe7\xba\xe4;Z\x161\x96\xca\xc9D,\x05\xfeM\xa7\xe0C\n\xcb\x1a&\xbc\x9e@\xaf]\x99\xdb\xe9*\xa6\xca\x1c\xc9\xc0\xa5:\x90\x92\xe1\x10\x95~yhm\xb3w\xe4e/\x8b18\xc4G\xeb\xa6x@\xa6p\xeaA6\xac\x96\xc1/\xe50\xaa\xc1\xe8\xe4\xa6x\xf71\xe1\x9ckL\x13\xaa.\xc5@\x0c\xa4\xa5\xdc\xbb\x9d_?\xa4\xb0Ln2}\x03?\x88\x91\xb7\x1bB\x89\xe3\xc7K|\xbd\x01}\xc3 l|r\x9bY\xdb\xcb\x93\x9a\x9f3>\xbb\x9b\xb5 \x8bJ z\xc9Z\xf3 \xacM\x8bU\x05\xdc\xc26h\xc5\xd2\xdf3\xd6b)jT\x04>\xf6\xe6\x9d\xff)\x81\xe5\xeb\xce\'\xc8\'\xf7\xf1\xeagJ\xf5\x00\x1d5\x13`\x94\x0cUB\x0c\xf0\x90]\xcf\xfbK\xa6\x07PH\xf8\xe7\xb4\xe0\x81\x98\xa4\xd9\xddp\xe4\x8e\xb4\xa8MN]\x99X\xedK\xaa\x15Z\xd6\x1e\x15p;]\xefA\xb5\xb2\xb8\xc3\xfdNkfZ\xa7?\xba\xf3\x7f\xb2\xc0\xf2\x9cM\xfa\x99\xb5\xaf\xd5\xd8\xcb9n[u*`\x17\xb4\xc7\x80\xc4\x82\xc7^\xf10\xdaB\x882C\x8a\xcbb\x85\xb2\'\x1636\x1c:@[$K\xbc8\x17\x8f\xab\xbc\xeaJ\xc1H\xb4\xeb\x0f\x98\xc6\xbb\xd9\x02\xca}\xe0d\x82\x16\x87\xe7D0\xac7\xcd\x8eO\t,_\x8b\x96 it=n\x7f\x1f&}\x0fH\x86\xf6\xc1\xf3I\xb8\xdf\xce\x82Z9H\xe37^\x0e\xc8\x1dP\xdc\x83\xa0\x03.\xf5]\xd2\xc5\xf2&\xd9\x80l\x1eCUGO\x9c6\x90pT\x84c\xea\xb9\xb3q\xa8\xefU6W\x9a\x91_"f\x88\x91\xfe]h\xe6C\x02\xcbW\x8a\x05S\xc4\xbap_:!c=}\x8b\xefU?\xc0\xc2\x8e\x0e\xf8\xe1\xfa\xb8xWuit\xb5\n\x8f\xd3^q\xe4\xb1\xbc\xbb\'|h\xe1\xce\x8f[\xa0\xa3\x92\xe2\xd83\xf7\x1eN(\x95\xf7X+\xef\xbc]\xd7]u\x18\xce\xbd\xca:^;\xe0\xdd&\xe8\x9f\x12X\x9eab\xeb\xdc#\xe4\xabO@\xf9p\x8bX\xf4"\xd7\x9a2\xb73\xc9zgLH\xd7\xdftL3\xaf\xd4\x12\xcde\x1eF\x94l\xbd\x1e\xf1\x85\x80C\xcb\x0e\x86"\xa9\x17\xdcBnY\xbc\x1c\xe4\x185p\x07\xb6\xf7\x90\x83\xe9\xf9\xf9\x16\xc9\xfe\xf4f&\xf9)\x81\xe5+\xccu\x91Sk\xaa\xf6r\xa1\x84\xae\xe3*\x9e\xeb\xe8\'\x9b\xa2\x147*\xad\x9c:Q\xb9\x9a\x00\xc2i\xa0\x0b)\xce1\xc2\xc4|@\x01\xbc\xa3\xd3\xa7\xc1\\\xab\x8b]\x14K\x15\x12\x17x\xadg\x19F3\xces\xbd\xeb\x8cT0\xe8{\xb6V\xa4\xdf\xeb\xe5\xd1\xa7\x04\x96\xaf\xa7\x88\xad\xa3\xa3^\xe9\xeex\xc1j\xfd\xa0\x19\xa2\x9f\xbaL\xc0\xdd3\x00\xe8\x13\xe3\x01\\t\x9b%\xca3I\xca\x98r\x91\x99\xda(0\x0f\xc7*\x92\x02N\x00\x8e\x0b\x94\xe3\xf9\xa8Pp]\x88\x97\x87\x93\xae\x16\x08\x17@\'\xdd\xeev\xc6\xa3}\x13\xb7\xfd\x94\xc0\xf2\xdb\t\x07\xe3\x14\xf2\xfaZ\xe5\xe2\x15"X\xfbH\x9e\x15{\xb6*\xc7\xc8\xd15@\x9c`_[\x0f\xf1\xeb\x98\xc0w3E2%V\x16\x97\xf4\xd0\xda%\x80Y\xcd\xe9\xfc\x02\xa8L\x12\'W+\xf6PMt\xae\xcc\xc1\x9c\t4\xba\xe5\x80\x80<\xde\xbc\xaf>%\xb0\xfc\xf6\xc6\x01{\x12\x97/\xb3\x99F< 4\xa0z\\\x1fXs\n\xb4\x08\x11\xb2\n\xcb\xb8\xec"[xwl\\&1/an\x9e\xa2\xf1!\x9a\x15U\x15<\xe6\xed} \xd5\x9b\xe2\x0c\xceU`Ew;\x1b\r\xb3\xc4\xee\xfe\xf1v9Be\xf0\xe6l~J`yf\x1f8\x8a\x11\xeb\r\xff\xb2\xf5\xd1\x9d(\xd9h",\xc6C\x88\x1c\xe8XuDg\x93\xea\xb8ns\xfeI1\xee\xce\x82\xd2N\xea\x91l\xda[\xcc\x94\x92I\xdf\x0e\xdc^\x821\x00\xae*l\x87\x835`p\x84\x83\x1c$+%i\x97\xacq\x82}\xd3\xd3\xf9\x94\xc0\xf2\xf5b\x17\xc71l]\x10\xbf\x0fs\x90\xe3\x1a\xbd\xf8\xfa|\x98\x08Z3b\xb2j}\x93J\xa2\xd0\x1a{\xeb\xd1\x0c\xc7\xb9\x06\xd5[\xd0\x9f\\Q\'K\xd4\xb5P\xc4\xba\x85\xdc\x1d&\x85\x08\xefBRj\xc2\xe6\xd8\xa7\x92p\xcfZ\xa3\x0f\xaah:\x8do~n\xf5)\x81\xe5\xf9a\xc7\xba\xe2!\x12y%(k\xb4O\xae\xed\xf9L\xe5\xa3\x18\x84} \xbaY+z\'H\x15\xca\x87\xec\xb9\xe7\xbc\xb4\x99\xfb\xf9<\x19\xb2\xd4Q\xd0\xa1\x84\xb0\\\xa9\x1f8\xb8p\xbb<\x8aE\x9e\xba\xb6\xe0\xb2gJB;\xb8\xb4_\x9eA\xda\xff\xe1\xfb\xa3\x9fL\xb0|\xedM\x98\x860\xfa5\xfb\x08\xd2\xe0~"n2\xf3@\xfd\xbd d\xf2@\xc4\x8fG\xbb\xbbTI\xd8\xb5\xa6\xe6\x92\xf4\xe5\xa1\x88\x88x\x00\x149\x87o\xb8\x866\x12bM\xd37\xe9H\x18\x96;\xb1\xe7+\xc9^\x1e\t\x00[r\xb9@\x07\x93\xf5\\\rI\x89\x80\x1exwT\x17\xd6\x11\xe4\x8bp\xca\x0eLE\xa7dZ\x81x \r\xbc\xa9\x16\xe7b\xe1\x11\xc9,Q\xe3{\xe5X\x9f\x12X~{\xc1\xb3>x\x04{y\xe7\xa9^\xc6\x8b\xc0\\m~:\xd6\xc5\x95P=\xc5\x8c[\xf31\x86\xec\xac\xa1\x12\x98y\xf4\x99:5\x86\x00\xd5\xbd(\xc4\x8e\xff\xc8\xc2~\x7f\x07=Kl\x05Z\xbbO\xc8\x0e\x88\xf7\x8d\xdf)~qq\xa7Y\x97\xbc\xc9\xfdm\xad\xfc\xef\x02\xcb\xd7\xeco\x02\xcb_\xfe*\xf8\x7f\x90\xc0\xf2\xf7\x81\x186\xdbb\x13X6\x81\xe5\xff\xc7U\xba\t,\x9b\xc0\xb2\t,\x9b\xc0\xb2\t,\x9b\xc0\xb2\t,\xdfw\x9c\x9b\xc0\xb2\t,\x9b\xc0\xb2\t,\xdfY6\xd9\x04\x96M`\xd9\x04\x96M`\xf9\xb6\xe3\xdc\x04\x96M`\xd9\x04\x96M`\xf9\xce\xb2\xc9&\xb0l\x02\xcb&\xb0l\x02\xcb\xb7\x1d\xe7&\xb0l\x02\xcb&\xb0l\x02\xcbw\x96M6\x81e\x13X6\x81e\x13X\xbe\xed87\x81e\x13X6\x81e\x13X\xbe\xb3l\xb2\t,\x9b\xc0\xb2\t,\x9b\xc0\xf2m\xc7\xb9\t,\x9b\xc0\xb2\t,\x9b\xc0\xf2\x9de\x93\xffl\x81ezG\x87\xf8\x17K\xe2\x9b\x0b,?=\xb0_%\xb0\xfc\xf4\xc0~\x95\xc0\xf2V`\xff\x0eT\xf2\xab\x04\x96\x9f\xbf\x147\x81\xe5\xe7\x08,\xff\x87\xbd;[v\x13\xcd\x16E\xfd.uKD\x89\xbe\xd9w\xf4\x12=H\xb4w\xa2\x07\x81@\x80\x00\xf1\xf4\x07\xd9Yg\xadJ\xa5s\x97V\xc8\xe1\xe9\xb5q\xded\xa4s\xce\xc9\xf8\xdb1\x90=>\xf8\xbf\x06\xfd\xef\x05\x16\x8a\x83P\x96\xc4\x05\x06gA\x81\x85Q\x9abQ\x98\xc2\x9f\r\x80\x9f\xbd"@\x8e\xe2@\x16\xe7@\x01\xa5!\x10\x07\xf9\'K\x81\xe1\x04HC(N2\x04\xff\x8f\xbf\xa3\x05P\xfc9\x088!0\x04\xcc\xb0\x88\x80q(Hr\x02\xc6#(D#,\xc6\xf0\xdd\xa1\xfcp9^\x9e\x85\xe9u\x81\x0b\xcc\x85\xd0\xde\xa0\xa0\xfc\x8a\x82\xa2s\x9e\xe9w[<}H`Y\xc3\xc4!\xf8/\x80\x02\xc2\x98\xb4\xa9\xe0\xce]\x0e:)Hc\x0f\xe6\x92\xb1vQI\xdcm\xa6\x93\x8b|9\x9b\xf4\xe5\x00-\xf9d<\xac\x00X\x82+\xb0;WS\x0f,\xf7\xb4\xa8\x0eI\xe6\xf1\xdc\xd9?\x16{nP\x1f\xc1\xee\x11M\x8f\xe1\xcd\xc6G\x9f\x12X\x9e1\xa2\xd8\xba\x06\xc8\xd7v\xc5\xc2\x83\xc2l\xbanj\xa4-4\xc7\x93\xa1\x19\xbbG\x81y\xc6a\xc7\x8d\xa9\x07\xce\xfb\x14i\xf7!\x06;\xf8\xa0@=f\x0e\x1a+?\x98\xd2`\x16\xd8\x8c\xf0b\x88\xef\xea]\xb1\x1f\x139\xba\xc9\x111U\x19`\xdf\xec\xa4\xf7)\x81e\r\x13\x03\x89u\xc1\xbev\xd8\x86\xf3)\xee\xd0L\x1b\xfdp\xee\x1fv\xa8\x11TX\x13\xba0\xdd\xaej\x83\x93\x87\xe0|\xc2|t.\xcf\xb3\x16.0)\x02\x07x\x1a3O\xd1\t\xc5\xbe\xd3!k\xed%\xdb:\x9dE\xf5\xee\xe8k\xe1\xb1\x9f*2\xfdQ\xb3\xae\xdf\x1b`y\x0e"\xb4^\xc1\xd4\x9a\xbd\xfey\x14-\x99\x92]Q\xe5\x91GjPYWqI1\xdbK\x13\xfb\xb4\xa7\xdeMQ\x89\x874G\x17\xe3h,\xe7\x0e\xc9&\xd2\xa1(\x13\xccSA\xa4\xcc\xbd\x87]\xf3\xe6\x1aB\xba\xe2\xbb\xd7\x1d\xc0\x0b\xd8\\`\xaaw}\xb7\xe7\xf3\x87\x00\x96\xe7\xe9\x86c\x04EB\xd8\xcb\xe9vc\xe2\xf9(\x18C\xa2\xd0lmT\xad\xa9\xa9\xb4\x9a+\xce\x8d\x17\xdd\xe9J\xc2\x08 3\x87=\xc6\x1b\x93y\xa5\xdcG\x15R\xb0\x17\xca\xcc\xd1\x9c\xcb\xf2<\xca5\xdd"\xe4\x9a\xbc+\x88\x04*u4\x04\xb0<\xc3\x84\xd7\xa3\x83\xc0^\xb5\xb0lB\xe0]CECsp\x8a\x11D\x00\xeb\xc0\xdc9S^\xfaq\xbd\xabR\xa0\x89w\xae\xc4\xb6\x11\n\xdf3\xc6\xb8\xa6~d\\\x00\x1emSv\xc1@\xc1\xbc\x16\xb8i\x9c.\x082\x93\xe5\x91VR\xf2\xe2\xab\xe0\xbb\xf6\xd2\x87\x04\x96\xe7\xa2E\x9f\xbd%\xd7X_f\x13\xeb\xb0\xfa\x1a\xcej\xd5\x15\xd79\x0bw\x8c\x94\xed\x9a\xe1\x06\x0c3\xea\x12F[\x99,\xb6\xb7\xa4\xf5r\xb6\xf4\xddx\x8f\xfb\xe5\xd2\x05\x12(^B\xcf>%\xa3\xee?\xf0\x87Z\x0c{\x90"4_\xd1C\x86+M\xb1?\x1bwq/\xef\xd0\xba=\xe0\x17\x829\xb9o\xf6\xd0\xfc\x94\xc1\xf2-\xccgML\xbcv\xf0&f\xc1\\\xc2E\x872?Y\xe8\xc3\x0149\xf2\xc1\x1fO.\xdf\xe1\x99\x18\x1f\xb5S\xad\xc8E\x9f\x0e\xf0\x9ed\xc1\xe3\xa1\xc1Aj\x12\x16\x00\x1e\xe9&\x0b\xda\xbb@\xc4V\xc0\xa2W\x1f\xc5\xf4FC\x0fy;"\xf4\x97:\xc8?e\xb0|Kl\x88\xf5J\xc0_q\x92\xf0\xae\xc0M8;\xcd\xe3\x08\xb79\xd9VT\xd5\x0c\xa35\x01P[\x9c;\xab\xeb\x9a\x94\xef\x02^\x82;Wi`\xd2\x15\xe3t\xd6\x08Ge\xc2|\xaa\xda\xe9Pz@\xdb\xdb \xa0\x02\xc8\x15\x8fZ9)\x84\xfaG\xa3\xf8\x93\r\x96\xe7by\x9e\x95\x08\xf1\n\x05\xce\x01\x87T\xcc16\xc9\xc8)\xaf\t\xef!\xe9\xc0\x11\x98=\x1e\xf7\x97%\xeb\xa1A2\xa9\xb2\xc2\xa6\x16\xcbk\xa6\xeb\xb2\xe0\x84\xb9\rt\xaa\xf5\xd3\x9a\xae\x8e\xe9\x88(u\xdf\x9f2\xc8\xa8\xbaQ\x98\xfaa-\xe4\xaa7\xf9\xa5O\x19,\xdf\xee+\x90\x84\t\x08\x7fi^\x19\xb9a\xe0^aQ\x8f1\xa2>\xc5\x8bd\x99\x98Hys\xd27ZU\xc3\xe1\xf1N\xab\x07GL\x0c(\xaa@\xc3\xee#2\xe2\x97GZ1\xb0\x1a\xee\xa4Gq\xb9{K\xdew\xae\xb6g\xd6tG\xec\xbc\xe1T\xfd\x08\x0c\xfb\xbd\x11\x96o\'\x0b\x06\xa1\xd0Z\xbf\xfdy\x14\x13w0!gD\x0c[0\xe7\x16\x00\x91\x05J\xa9\xf2~|@\x83k\xf5\xd4\x18\x18\x07\xff\xd0\xef\x87\x13\xa1\xe1\x85\xdf\xc9\x83{\tG\xcb\xbc\xe3\x00\x0f\xf3\xbd\xbcW\xcb@ ;\xf9n\x07@\xe1\xf0\xdc\xa2/$\xf3f\xe9\xf6)\x84\xe5y\x1d\xc2\xe8:>\xf8\xeb\xad\x9f\xab!58<\x8fes\xd5\xf3\xf3Z\x08rsM\xab31\x03K\xa2\xa7Q\x81\x1c\x1a\xe5\x84\x00ky&\xdcw\xad^,\xa0X\x84!~\xb8\xa4TdT\xe7#\xb7\x90\xca(\xa3\rP\x18\xa6\xef\xf3Ep\xe9\xdf\xcc\xe1>\xa5\xb0|\x0bs\xfd\x06$\x05\xbf\x9cprx\x05z\x0f\x1c"\xee6\xa9@\x92\xe2\x80TK\xf2,\xe1\xbb\xbd\xadvM\x14\xab\x96\xd4k\xed|;\xda\x02\xdf\xa4b\x11\xe1x8\x08\x0e>t\x8f\xcf\xe0\xa44Q\xd5\x14\xbd\xa6\xdc\xa6\x1b\xa6\tg\x06k\xaf\xe4\xdd\xc2\x06\x1c\n\xd9\xf2\x11\x85\x1e\x0e\x1cq\xc8\xa9\x9d0\x99\xd9\x9f\xb9\x19(o\x8fR\xb4\xb2\xc8.\x907\x8d\x82O),\xdf\xcaH\x10E\xd6\xa1yy\x83d\xab\x806\x18\x0bj\x10\xb7\xe1"#l\x9fI\x17\x9b\xbe\xdcO\x07\xe5t\xaa\\\xa4\xf1!\xabR2\xf6r\xf5\xc0\xf3\rMrl\x07\x07\xfc\xd5\x82\xc2\x88\xef\x14\xb3\xba\xecL3\xd1\xce;\x16\xea\x88p\x1cp\x1bv\xb2\x1fU\xcb\xbf\xb7\xc2\xf2\\,\x10\xf8\xfc\xe0\x06|\xb1\x01*\x99\x8fF\xaf\n\xf9D5\x15M\xee\xcf\x94\xabIW_\xbf\xba\x15\x8a,\xacN\x05\xa57\xc9\x99\x8a\xb7I|\xd0}\xf6\xbaf\xe9\xb7P\xbb\xa5\xe6}\xe2\xc2H?\xa2\x13{\xf5\xf3\xdb\x1d1\x07Z\xea\x00&s\xc77\xb7\xfe\xa7\x14\x96g\x98\xc8z)\x83\xd8+\x8bx\xcd\xdb\xda+\xee~\x8427;\x8d-\xc0Hlt(\x1e\xd1\xf5\x84\x03\x8dG\xa6E\xce3t\x1c\xae\xc7\xf3\xa3\x04bR#\xab\xc4\xa6\xe3\xbbD\xda\x10CJ\xda4\x17\'\xecr\xde\x05\xf4%\xba\xf4i\x80,c\xf7f\x98\x9fRX\x9ea\x828\x01\x120\xfc\x12\xa6\xc2\xcaEr[\xb8n\x9f\xae\x17\x93K\x867K\xacn\x00\xdd\x07\xa7}\xcb\x8a\xe8p=^\x0e\xc8\x95#\xc3\x1c,\xd8\x1d\xb4W\xcf.\x88\xba\x9c\xb7\xee\x1dz\x91\\/\xabz\x8dDoa\x90Sk\x1a\xed\xd7\xcc}z3\x97\xfc\x94\xc2\xf2\xdc\xfa\xeb1\xb2^W\xaf\xef\x1c\x0c\x9a.\xb0qV\x1d\xfc\x08\xfa\xa8^\xe6\xd0\xc1\xaf\x80\x84\xbe\xc4\'\x89\x10\xaf\x9dr\xb1\x12\xbd\x1b\xcd\xe3>q){$Dj\xf7\xd0(O\xb6\xc9\xaa1\x91[c\':\xdfp\xd0\xe3\x11\xf5|S\xbaN\t\xe9\xe8\x9bI\xd6\xa7\x14\x96o\xb9$\x08#$\xf4r+w\xfds\xb6\xae\xd6cw\xb9\x82\x00\x11R\xfd)\xf0}=\xf2\xfb\x8eD\xeaAu\xb1\xbap\xd8\x9c\x8c\t\xb9\x18"{\xb8\xe7\t\x06y}H\xdf\xa6\xabEa\xa7\x14Uf\xb4\x9eD\xb8\x8c\xe6\xb3\xdb\xc1\x98\xa4\xde\xde-\x96?\x84\xb0|\x9fL\x90\xc0^\xa9\x19\xd2\xc2\xa8i\xbc\xb5\xb3\x11\xb5\xbb\xc0\xc8\xebX7\xf7|\xde\x91\xb3~\xe0\xc7Ix\xf0E\xac\x8e\t3\xe6^\xcf5\x87\x04p]\\V\xae\xd0\x04\x0c\xb3\xdd\xa9\xf0\x8e;\x15\xf4\xa2\xa3\x93/Ez\x88\x9a\xfc\x91\x92\xdf\x8d\xf2C\x06\xcb\xb7s\x16B\xd7d\xe5UdV\x16\xf1\xf4\x08{\x02t\x1b\xa9\xac\xf6C.\xaa\xa5\x9f\xd5\xd9\xf1pj\xe3AQu\xa3\xd3v\xa1\xf78c\xa0? \xbb\xcc\xe6\xba\xd4\xb2\xca\x10\x9cKK\xc85\x9d/g\x16\'\x1e\x8d\x90\xb1\xfa\xfd&\x11\xc9=R\xdf\xfc\xf0\xeaS\x06\xcb\xb7\xc9\\\xc7\x05\x06_?n\xad\x1b\x9d\x93\x0b\xc7\x15%\xcb\xbcR\xf9R\x17\x87\x99f\x91\xdb\xc5\xf0B\xff\xc4\xeb\x0eU\xcdfy\xcb\xa3\xc2\xe2\xa5\xb9\xe8.7\x91\x84\x1d\x02\xb4\xa3\xf5\xdeH\x9aK\x10\x07\x97\xe8\xa4\xe0\xe5y\x04\xd2|9\'\x17\x8c}\xf3\xa5\xe7\xa7\x0c\x96\xef\xc5,\x8e>\x0f\xea\x97w\xbb\xbcUb\xd7\xfdR\xaf\x17\x1fu\x08$\x1c\xcb\xf6\xe8\x92\x9d\xf6\n\x01\xb5\x8f3\x989s\xbb8J\xa1\x89\x87|\x86x\xdf_\x06Mm\xd2*-\\\xf9\x08\x1ax\xadq\x00DV8\xe0h\xba+\x17\xcd\xee\x12\xe5\xbf\xc8`\xf9\x16&\xf2\xfc\xa0\xef\xb5f\xd7R\x17P\xc6+\xd4K\x9d\xaeP\x05u`\xebq\x9c\xc1\x1b\x11\xef\x80\xb2\xb2z\x7f_A\xbd\x05\xd0\x92V\x10&\x8ca\x8f\xb8wSJ\xe8\xf1\xbdL\xd4\xed\x04\x8a\']\x01<\x81\x18\x07\xd2\xc9\xd8\x06\xb3)N\xfdZ\xef\xb1>\x85\xb0a\x94\x00a\x82E\xd9\xf5\x8bI\x14AA\n}\xb6>f\x10\x9c\xe2\xd8\xf5\xbbq\x14Ic\x02N"\xcf\x8eE\x1c\xc6\x82 \x0f\xa2\xdf\x06zCX~"\xc2\xf2\xec\',\xf0\x14\xc6\n0\x86B\x14M0<\x08\x0b\x10\x08\xae\x8f\xf8\x9c\x17\x84Y7\x0e\xb9.3\x12\xa18^\x00\xd7\x95\x01\x11\xeb\xbc\x084\'\xf0\xfc\xb7\xd6\x8e\xbf5\xc2\xf2\x1f\xcb \x9fCX\x9e\x9d2\xfe\x16a\xf9\xfa\x1b\xfd\x97",\x18\xf2c\x9dD\xdbW\x87[\xa1\xc0hb\xcd\xe3x\xb3rV\x19\x8c.\xb0\x04\x1d\xabPN1\x9b+0\xb3k\x0e\xa3\xd9E=@\x82\x04-a\xca\x9bm\xa4\xc7\xbb\x12\x81 \xb7\xb7@\x8b\x88 \xean\x1fQ\t\xb1i\xe4j\x1d\x7f\xd4G\xe6g#,\xeb\xb5\xb0n\xefu\xef\xbd\x02%\x07@\xe6.\x1a.P\xc8\x03\xce\x9a\xf5\xf4`q\x00\xb3.\x0e\xd3\xc2\x171S\xf0\x8e[\x0e\xd1\xfe>\xc3\x94\xe3\xc3\xda\x12\x05^{\x8b\xce@D$>n\xe0E\x9a\xe0\xbbs\xde\x03p\xf1p\x03L\x1b\x1fnUZo6E\xfd\x18\xc2B\xfds\x1d\x1c\n\x7f\x99J\x9f\xd5\xeb\x86\xbf\xa1\x05\xb3\x8f%\xac?\x9d\xe8\x03\xdf\x00\x05\x01d\xb7Z\x1eH>m\xd8\xba\xe0sqg>\xc8VL#\n\x12\xf7\xf2\xb5\x15\xb3\xc8\x1c\x1fi\xc6\xa46uX\xa3\xcc/NU;\x99\x8b\x9aV\xb0\x9e\xaf\xbf\x06aY\xa7\x92\xa2\xd6\xfb}=\xb2_\xa6\x92\xb03\xab5,\x15\xd5Cg_w\xe4\xd9b\xfb\xc8G*\xce\xbb*z\xb7G\xc9:v\xc5\x9e\xb0M\x07.\xbc\x18\x11\xcd\xb0\x12\xf03\x1f\x85\x0f\xb1\xf4\x93\xc3\x02\xcc\x06\xaa\xdf\xc4\x14y\xe8\xb5\xee\xa44\xc9\xf2o\xae\xd8\x8f!,\xcfD\x06%\xf05\xaf{\xe9c\x15G\x1aX\x8d\x86E\xce{\xe5\xde\xf6=\xe7\x157\xd1\xd6C\xef8\x92j|\xa9\xb4sOf\xa0\r\xb7@\xbaf\x00\x8d\x00\x8d\xc0\xc5k\xac`\x800X\x85\x84\x05\x19\xd8`\xbe\x1dgf\x0e\x84S\xc6\xec\xf0\xc4u\x7f\xd4\xc7\xea7WX\x9e\xfb\x1e\\\xf3\x82\xbf\xe8\xe0=T\xb7\xbc)\\G\xcd\xa1\x99\x0e\xe7\xcc\x08\x87Q(A/\xbbW\xde\xce\x1b3\x08\x9d\x0cH\xb5\xe6\xb2>K\xc1\xf9\xe0j\xa6bh\xea\xba\x82\xfd\xb9\x91\x80pO\x0f\x91Bde\x9e\x06\xda\xbd\xa0\xef7\x9c\t\xd37\xfb\xca~Lay\x86I\xae\xe3\x85\x93/\xcd%\x97\x1bt\x80(\xf6\x98g\x98\x90]\tX(\xc0D\x97Sp/\x98\xd3Y\x12\xe3\xab\x9a\xad\xd5\x8b\x92\xecN@\xed\x02\x8au\x93>\x1a\xbd\xc1\xee\x05\xdb9\xfaQ\xddQ\xd3\x9ba~Lay\x86\x89\xad\xc9\x1e\x08\xbd\x94o\x8e~\x85\xfdiM\x95l\x99\xd0\xebA\xd8\x97V`\xed\xf7\xb8-=\xae\x04\xa6\xf7t7\xaa\xd3\xc1~$\xc3\x01,i\xda\x85\x10[\xb8S\xfa\xcd-\x9a\xbd\xce\xc2\xdc\x03u\xce\xb0\xb7\xcfwJy\rs\x16\x07\xd5\x8b\xf3f\x0e\xf71\x85\xe5\x8f[\x1f$\xc9\x97\x8c\xdc\xa9\x8f\xcb\xad\x15\x06\xda\xc5\xd7\x04\xe3\x8e\xb6\xfch\x81\x89\x8d\xdcC\xaa\xc9\xe2\xa1\xbd\x94\xe2>\x91\xc9B\x9a\xdd\xa3X\xecxy\xfd]\xf5\x8ar|\xe8DNZt\xcb\x01\xef\x19\n\xf7T"\x04\x00\x87@\x0e\x04W\xbeI\xcc}Lay\xe6p\x14\x8a\xe0\xc8K\n\'\x91hy\xbb\xce\x18\xaa\xe3\x1a]\xc8Rl\xb2v\xa4SJ\xdf u\xc2\xb4\xaeR\x0bh{\x94\x0f\xd7\xa2D\xf6\xc9\x9e\x8d\xd01e|\xcfx\x98OO{\xbd\xe3\x91\xfd\x15\x8e\x8e\xfb2\x1b\xf9RH\xb4\xf2FK\xefz\x81\x9fBX\xd65\x0b\xaf\x972\x0c\xbd\x9e@\xfd\xdd\x8f\xef\x0c`\xdcDM\xdeQ7j\xe1w)"\x9b\x0e\xb3\xbf\\=s\xa8s\xe0\xceD\xb7\x8e*\x03\x05\xb8?+\x9dGv\xbc\xef\xcf\x80\x0e\x14\r\xd7\xc27\xc3\x9a4l\x08n\xfcQ\x0e\xef\xa9\xdc\xdf\x0f\xed\xf4\xab\x10\x96\xf5V\xa6p\x12&^\xeed\x08\xf1\xe0Y\x11\x8d\xc3\x01\xc9\xec\x0c\xafp\x1b`\xe9\x04\xf1(\xe1\x80\x9a\xf1y\xa1)-\xb62d\xca \x02\'\x12\xd4\xb1\xad\x0b4wR\xa2\x9e\x08)\xc3\xdcB\xae\xdd\x94\xcd\xef\x89J\x0b\xa38a\xddi*p2\x1f\xce\xb4\xea\x07\\ \x82\xfc>}n\x11v$\xef\'\xf8|\xb8\xdc[*\xbf\xf1\xe6\x84\tG\xe1\xf0\xa6\xf7\xfc1\x81\xe5\x99G\x92\xd4z\x1e\x12/\xa78!PN\x8f^\xe2\xa59)\xc7\xf4\xb8\x87\x1a1V\x1e\xaeyyp\x86o!W&5\xdb=\xcc\x86\x1a\xccP\x80-c\xd9e\x92\xd7\x04#-\x8f)2[|\xc3\xec4\xae`\x17\xac|\x94<\x1e\x1e\xd8\xa8\x0b\xa37\xaf\xe4\x8f\t,\xcf\xd9\x04\xd7+\x19~u\xad\xe7\\.\x86<\xe2\xa3\xaet\xdb\x1b\xa8\xad\t\x88\xa3-\xd81\x12\xa9\xea^\xa1S\x99r\xb1\x10]X&Q\xb8A>\x14\xe9\xd14\xe3\xc8l\xa1\xcc\x98"W\x91\xef\xae@\x9e\x0c\xccp#_\xaf\x94|r*\xadz\xf3}\xc3\xc7\x04\x96\xe7l\x12\xf0\xfa\xeb\xf5c\x80\xf8rSi\xa2\xb7\x0c\xc6\x89\xd9\xc6\xc7\xf3\xd4\'\x89{\xe9\x92`i\r\x04\x8c\x9f\xdc(\xbf\xc6\xfd\xac,;{\xe2Z\xd9\xcd+\xbdn\xd5\xb3P\xb4E\xd4\xafE\x0b\xd1\x9e\x91Y\xbf\x8d\xbe\xe9\xb9Q\xc3\xb16y|\x13\x9a\xf9\x98\xc0\xf2\xfd] \xb8.\xdc\x97\xeb\xea\x12\xdfO\xb2\x8d\xaa\x18a9u\xcfqHn5 l\xb1\xb5l\\\xael\xae\x1d\xe3\xc1\x91w);\xe0\xe1.\x88l\xdc\xf7I\xb6\xc1\xe9\xab\x17\xd87\x87.\x03\xc1?\xebk\xedR]N\x17\xa2\x00\xda\xc6\x973\xe3M\xce\xe2c\x04\x0b\xf5d\xc4\x89\xe7?/{S\x18d\x85\xae\x94\xeaF\x94\xbb{8\xe2\x8b\xb1`\xe6\x9e\xce\xdc{ex\rS6\x96X4\x89\x0c\xc9\x19?\n\x07O\x0f\xafd\nF\xf7\xe5\xf8\xe8\xcc\xe5\x80\xc0\xe6\xbe5\xd6\x83>9\x9f.\xeeM)o\x86K\x9d\x7f\x95\xc1\xf2mo\xe2\xf0\xba=_NZ4\xd7\xcf\x03\x9a\xc4\x1dv[\xb2\x07|\xa70=\xba\x13\xfd\xe9\xc1\xd5\xf8\xeeA\xc9\xaa\xb63\x99~^L\xf8\xf1\xe8\xb0yRi\x87\x1e\xa7\xd3\x89\xe5\xc4\xdc2\xc24=[rZ\x9d\xa4\xd1 \x1f\x02b\x03q\xab\xed\xdf\r\xf3S\x06\xcb\x1a&\xb4\x8e\xd2Z\xe6\xbf\x84\xe9\xd5\x96\xd0\x1e\x9d\xe8~\xd2\x97\xc2\x99tX\n\x8e\xd7\xf8\xd1#\x93\x8a\xa0\x82\xd4\x05\xa0\xfb\xb8\x16\x94\x03\xf1\x13\x980\xe64\xc6=\x98\xce2\xe9\xa2\xad\x10\xdcz\xc3u\xd0Gq\xbdY\xba\xbdS\xd8\xb3qm\x05\xacx\xf3\xf5\xce\xc7\x0c\x96u\xd1\xae\'\x10\x89\xac\xcb\xf6%\xfbXsS#\x9f\xb8\xa3\x8b\xc3;\xf4d\\\xf0\x96\x93\x1c+\xd5\xe1\x10\xce\xa7\x8a\xa45\xc0\x16v\xc9`Hd\xb9@~\xcdB\xd4\xfe\xc0\x15T\x98\x94\xcc\r\xe6w\xa3\x1c\x9a4\x15\xb4\xfe\xa5#\x0f\x08J\xdd\xb8\xc7\xf8f*\xf91\x83\xe5y\x04\x910\x82\xaf\x97\xec\x9f\xc3\x1c\xa9\x02\xebs\xdd\x17\xbaCB\x86\xbc\xb2\xe4Dx\x0b\xc3\x9aQ\x9b\x1e\x96\xb4\xdb\xfe8E\xfe\x9a`\xb9\xba=\x17\xd3\xe25~W\xb6\xc9\tP,C\xf4\x1a\xd1$\x99\x0c\xa2\xcf\xbcvO\xad\xe2a\xabn)\x86\xb7\x1f\x9d\xb4\xbf\xb9\xc1\xb2.\x16\x84@I\x04A_F\xb1qIw(\\\x07\x97\x0b\xddf1\xf7\xa2+\xc7\t\xa3\x17\xe6\x99n\x0f{\xd50Z\x98^dF^\xd2\xde\xbe\xef\xe0\x02;{\td{\x0c\x8bpLqaA\xdc\xde+\x9e4g\xe4x4\xe4I\x85\x10\x8b\xfdO\r\x96o\x7f\xcal3X\xfe\xe3?\x0c\xfe\xbf\xc8`\xf9}z]o\xed\xc37\x83e3X6\x83e3X6\x83e3X\xbe\xf0sn\x06\xcbf\xb0\xfc?\x94Dm\x06\xcbf\xb0l\x06\xcbf\xb0l\x06\xcbf\xb0l\x06\xcb\x17~\xce\xcd`\xd9\x0c\x96\xcd`\xd9\x0c\x96\xafl\x9bl\x06\xcbf\xb0l\x06\xcbf\xb0|\xd9\xe7\xdc\x0c\x96\xcd`\xd9\x0c\x96\xcd`\xf9\xca\xb6\xc9f\xb0l\x06\xcbf\xb0l\x06\xcb\x97}\xce\xcd`\xd9\x0c\x96\xcd`\xd9\x0c\x96\xafl\x9bl\x06\xcbf\xb0l\x06\xcbf\xb0|\xd9\xe7\xdc\x0c\x96\xcd`\xd9\x0c\x96\xcd`\xf9\xca\xb6\xc9f\xb0l\x06\xcbf\xb0\xfcZ\x83ey\xcb\x87x\xfc\xdb#}i\x83\xe5\xa7\x07\xf6\xab\x0c\x96\x9f\x1e\xd8\xaf2X\xde\x0b\xec\x7f@\x95\xfc*\x83\xe5\xe7/\xc5\xcd`\xf99\x06\x0b\xfa_\x83\xfe\xf7\x06\x0b\x88\xd1\x0c\xcd\x83\x10\xc2\t\x04\x8c\x91$B\x91\xa8\x80!\x0c\'\xf0\xb0\xc0\xb08\xfbl\xad\xcf2\x02\x84\x81\xb8 P\x04\x83\xc1\x1c\x8f\xa18H\x938\xcb\xd2\x7fo\xb0P,\xca\xd2\x02F\x0b\x1c\xc7\xf0\xfc\xfa\xff\xc3\x10\xcb\xf10\x8f\xb2\xeb\x8f%\x05\x82D\x04J@IP\xe0@\x92$ ~\xfd\x0f\x14\x81\xa3\xa4\x80P\x10\x072\xf8O5X\xfe\xd5\xb8\xe4\x1f/\x7f\xa3\x9f\xf8? \xf6\xcf\xf5\x81@\x10\x01\xbf7\x9f\xfa\x91\xc1\xf2\xf5\x05\x9b\xbf2Xh|\x9dZ^\xe0\t\x82\xe5\x05\x1caqz\xfd\xae \xc5\xc0 \x81\xf1(I\xa3\x14\x0e\xe1\x94\x00\xe3\x08\xcd\xf3\x04\x84!0L`\x08N <\xceA\xdc\xb3\xaf\xcdf\xb0\xfcD\x83\x85fH\x10\x13X\xea\xd9{\x14fqNX\xd7\x10\x83\x110\x01\xa14\xb5\x0e)\xcfq\xc2:\x80(\xcd\t(J\xf2 \x05\xe2\xebo20\xbb\xae\x13\x02\x06\x99\x7f\xfc\xe6\x06\xcb\x7fLI|\xce`yv\xca\xf8[\x83\xe5\xebo\xf4_h\xb0\xac\xe7%\xf8\xe3F\xe8WY\xbb\t\xcaH\xf8-\x16\x9fF\xaf-[\x185\xe4el\xf4\xe3\x05#\xca\xc2\x95\xa4k\x05\xdf+2\xf0\x9d\xf4\xa2-\x8e\xba\xab(\xe4>\x00\xc7\xd1\xdc\xe5\xf6\xe4?\xc8\x96\xb9\x85$\x92\x8c\xebsb%~\xb8Gov~\xfd\x8c\xc1\xf2\xedZ\xa0\x90\xf5V\x87\xd7\xf1\xffs\x98p\x9dW\x15\x0cI\xb9\x1f\x8b\x11\xcc\xa9\xb7[]\x0e\x9d\x97^`\xd9Q\x97\xbd\xaa2"\xd9\xf6\xd4\xdd\xdc\xef\xe0\xc7\xce:P\xe9\xc9;\xe6\xd3i=dO\xebs\xc5\x83\xe5\x8c\x84\xc6X=\xcdt\x81b6\x13g\xa8o\xf6\xd1\xfc\x8c\xc1\xf2=L\x12\xa2\xf0\x97\xceG\x9d\xe1\xde\x8b#\xee\xdd\x1c\x83#w\xfb|w\xce\xc63\x8a\x860 \xe0c82\xf8\xb2\x90EdN\x0e\x92N\xf9\xf1\xb03\x1e\xfb\xcb\x9d\x90;)\xf4 \xf2\xd8\xc9\x93\x03\xa5U|\xbdM\xd3h1\xf7\xfe\x80\\Z\xe4\xcd\xaed\x9f1X\xd6\x18\xf1\x7f\x82\xf0s*\xc1\xbf\x98JAC\xaeA\x9e/\xa2\xae\xa5\x02{r\xd5\xf2qG5\xe9\x9c\x1d\x9a%\xba\xf9!v,\xb0\xe3`\x0e\xea\xbd\rv3}\xed\xb5\x19R}\xd4t\x1f\xa3\xd2\xa0\xcfv\xe2\xa0\t\xc2I\x91\x87\xd9\x81\xc1p\xb8G\x927\xa7\xf23\x06\xcbk"\xf3o\xed\xba\xa6\xfc\xa0\xd4\x8e\xe2\x88\\\x88p\r\x8a\xee\xab\xfe"\x0c-N\xcaa&\xa2\xbb\xfe~\xa9\x1fg:\x17\xacx\x04hf\x82\x8f&\xaf{\xe1>\xeeR}"rv\x80|"\xd3\xe0d\xad\x185\xbb\xb3n\xb8?\xfd\xb0\x95\xde\xefl\xb0|\xdf\x10\xe0z]#kZ\xfe\xe2<\xd8\xb0\xd3\x1f0?0\xb5\xbaZ\xf6\xc7#H\xc0\xc2\xe8d\xb7,\xc5\x9aCO\x1d\x0fD\x8d\xab\xcbyB\x14\xa2g\xa5\x878\xb0uip\x1d\x05Ybv\x1bs\x1d\xbb\xc9H\xa3\xa7*\xa0\xc0A\xe2*\t\xe4\xeao\xb6\\\xfd\x8c\xc1\xf2\xaf01\x94"\xa8\x97N}\xd3\xbe\xcbm\x96\rG_8\x17s\x8e\xa9;GYj\xbe\xd6\xb2{\xfbP\xe2v\x0cgI6\x91~\xf0Z\xce\xd9cD"\xe7\x92\xe1\xc7\x92\xd5[\xe3\x18[\xd0@\xcc\xa7\x86E\xa8\xa1\xbb\xdf\xc7\x87\x08\x8f\xa4[\xbe\xa9v|\xc6`\xf9\xbe\'\x10\x14\xc2A\xf4\xb5Myp\xdeM\xfa\x19\xa3\xd3E\xa6\xeb\x18\x87C\x80\\\xaa\xa1<:e\x98\xc5s}\xd7a\x94\x0b\xdbi\x87\x8b&\x9a\x93\x03z\xca;\xa2\x8b\x8f\x8b\xc2N\xb3\x88\x04p\xec\x8a\xfal\x02Hy\x85\x8f1\x16\xcbYw\xe6\xde\x0c\xf33\x06\xcb\x1f\x97\x15A\xae\x19&\xf42\x9b\x06%\x18\xbe\xe6\x89\xf2.\xd7\xddP\xe6\x16\xc1\ry0V\xc5\x85\xa7h\xbb5d=\x91\xea\xae\xb3\xf8FQ+\xd2)\x93\xec\\\xd4\x14\x1e:K?\x99*\xbc\xbb\xf8\xeam\x0f\xf8I?WG\xbd\xd0\x0fAT\xbe\xdb\xf4\xf93\x06\xcb\xf7\xd9\xfcv)C\xc8\xeb}U\x91\xe0\xc9\xe6\xa6.\xa3\x16p\xc8\xac\xbdg\x8c\xfc\x9eG\xe3R3\x17CI-\xc9Mn\x033\x99!\xad\x0f\xe8\xfe\xba\x0fwPg\x17\x14\xd0\x1d-d\xbc\xd5\xed\xf0\x80z=I\x91\x11\x88\xab\x03\xb1\x8eXF\xbd9\x9b\x9f1X\xbe\x87\tC\xeb\xc0\xa0\xd8K\x86\x05\xaf9w\x86\xb5\xbc\x91\xe9\x92\xa5p\x06\x9c^Q\rX/\xe4\x96\xacT\xfcR&\xc5\x00Twe?\x14\xf9\xc3\xe5\x8f\xd9E\x9f\xcf\x8b\x8e\x87H7\xc0c\xe6\x1et\x9d:{\xce\xf9\x88\x1c-\x1a\xd4\x0b+\x11\xa2\xe5G}\x17\x7fg\x83\xe5\xfb\x9e\xc0\x89\xb5.\x81\xc8\x97=\x91*F\x1c\xce\xe7\xf4R\xcb\\\xcd\x10\xce1Q\xb2\xfa1\xf9\x82\xc3\xc4\'v-\xca\x1a\xad\xa4d\xd1\x8bu\xff,8\xd6\x03\xa8\x17\xfdv\xb1\x1al\xc1q\x82O\x16&\xbf\xa5\xb5\x84\x85\x94\xebI%(<\x9c\xe0\x9e\xbc\xdb\xed\xfd#\x06\xcb\xf7\xc5B\xac\xf5%\t\xbf\xa6\xe3|vv\xd1:i\x95\x93\t\xdf\xcei\xec\xc2EQ\xd6\xc48\x9f\xb2*\xb4\xa5\xa6\xa5\x8b\xdb\xa2\xdc\x08\xacT\xcd\x0b\xd4I\x08\x1a\x1f|#\xcf\x03\xb37K\x1a\x10\xf7@9MG\xc3>\xc6\xc18\xd5*\xdb\x1e\xf7\xe0\x9b\xbd\xfb?c\xb0|\x9f\xcd\xb54\x02I\x9cx\tS\xee\xe5\xe2\x04y\x02v\xe2\xf0\\\xe6\xc5\x01\x02.=U\xdc\xb10/S\x8f\x1ak|6*^\x15U\xe0\xdc\xfb-\x13\xb3M:Lb\xcf\xf5\xb7\xf4:A\xed\x9d\x19`\xd5\x01\xe4c_t\xd7\xc4Ix&\xcc\xca\x1f\xb5C\xfe\x9d\r\x96\x7f\x155\xeb\x17\x82\xf8\xcb\x9eP\xc6\xdeuI\xa9\x17Qso6\\/\xc5PQ\xe7K\x13\x8b\x01\x94\xd7\x10\xb3\x88\x17\xb8\x90\xb8\x01\xdf\xd1\x90siL\xb8,p\rF\xc4AW\xce\xa1\xae\xee\x169*r\x80\x95\xb4]\x07*\xe6\x1d\xed\xb0\xd6~\x13\x98\xfb\x8c\xc1\xf2\xc7=\x81c\xeb\xb0\xa0\xaf-\xc2\xafmH3\xbc\x9ac\xf3\x1d\'\r\xc2\x1b\x98\xcb!T\x8e\x8e\x87T\xb9+dR\xea\x9e<\xb1\xf0R3\xd9\xf5\xb7\xbc\xa3\xa2\xe3c\x1f\x8eq\xca\xdc\xc5\xe4p\xd4f2\xdf\x87\x93\x7f\xc8\x00~\xf4\xd8\x1d\xb2_\x8a\xe0\xcd\x16\xe1\x9f1X\xfe\x08s-p\xd7!zIUy\xd9\xeb:\x87q\x0e\xc2\xa5\x06\x01\x14\xba\xb8=\x1dE!\xe6\x06\x0b=\xd2\x8c\xe7x\xc4\xc2C\xf7}\x05\xf7\xad\x82\x067P\x0f\xa4\x11\x91%\x83\xa1\x1c-\x8e\x99\x92\xb2\x0e\xd1xK\xfa\xd4\x05\xc0]gg\xf7\x00\xfb%\x06\xcb\xbf\xc2\\/\x04\xe2\x15\xd3\xb2\xacr\x1ao\xc7\xb1\x86\xc5\x810\x86\xd6s\x85h\x08\x9bGKGp]1\x1d,\xef-\x89\xef\x94\xfblq\xd8\xa9\xd3\xc3\x9d\x13\x8d,\xc0\'\xbah\x8cVI\xfb\xb4\x89\x10\xae6iF\xe7\xa9\xd1yWF\xcf%\xf8\x0b\x0c\x96\xef{\x93\xc0@bM\xde_\xd1W\xdb2\xa1\xbc\xd1\xfbSv9\x92\xf7>Vg\x90\xd2\x15\xdd\xb4*|_;=\x16\xec\'i\xeaR\xdc$\xe1\x83\x8ce\x8bt\x0c\xc0\x19\x8e\xfaiW\xb5I\xbe\xae4\x8a5Gg>\x00\xb3T\x94\x96\xd8\xe7\x1a\xf9.\x88\xfa\x11\x84\xe5\x8fky\xcd\xf7P\x0czi*]\xde\xf7Z\x97\t\x8d\x8cL\xf4\x99\xbf\xdd\xd8|G*\xf0d\xd3L\xae]\xc3\xb2\x1d\xe7k\xaeL\xa5i\xb6\x8bT\xb8\xf0r\xd5hZ!2\x9e*N\x16u\xdb\xa7\x87\x10\x89QO\xe9x\xfa\xbc\xd3\xf9\xd0\x80D\x9fz\xb3\xe1\xfbg\x10\x96?\x16-\x81\xae\x85$\xfe\x12fmN\xf9T\x1c\xcf\xf2\x81l\x8b\x8b\x8d,\xd9\xce\x9f\xc5\x1a\x9ee>;\x1c\x0b\x16\xab%C\xe8\xf1\x9e,\xa7\x0e;\x886\\\x1a\xf5\xb1\xb7\x88\x99?`\xc2N\xd4\xaa\x85\xda\x8d\xb0U\xe4Q\x8a\xc5N\x0c\x84\x8f\xab\xfb.\x1c\xf4\x11\x86\xe5\xfblb\xeb\x08\xad\xa9\xd9\xcb;\x07\x9fG\x16\x94\xb0\x0f\x0e\xde\xdc\xb5\x06\xbe\x91\x88\xab\xd5\x82e\rAS\xe6N&\x14\xc4\x83\xb3\x11\xbc\xc3\x08\xd2iR\x06\xdb)\x92\x1c\xb4`fD\'y\x97Z\xb1\n\xa1\x0ba\xe5\xc2r\x1c\xae\x90N)\r\'\x11\x13~\xb6\xcfr\xb0k\x07\x10\xcd\xe8\n\xac\xf4\xccE\xf5#_\x9d\x0f\x0f\xe54Ar\xf8\xa6\xcd\xf8\x19\x85\xe5\x8fZ\x19_\xab<\xea5L\x11\xbe%\x11T\x1b6\xd5\xa3\xa0\x9f\x0e%&\xc2\xb0r`h\xb6k\xf4{\xe3\xd1;\x0e1\x9b\x10\x0c\xf1S\xd2\xe3\xea\x8d\xe7\x8eml\xd9;\xfc\x9c\xd4\xc3#t\xaa\xc2p\xf6\xce\xe2_\xae\xa2\xd4\x1a@?t\xda\xf1\xddW\x02\x1fQX\xbe\xcf\xe6\xf3\x1d\x19\x05\x92/\x8b6\x81]\x87hK\xab\xbd\xa8Gd\x9f\xed\xcd\x07 \x96\x80(\xef\xbd\x1c\xe3\x94SYDg\xec:=\xe2\x84\xc11c\xa2\xa9\x14\xc8U\xac\xbc\xb9\xa02T\x87\xe0\x12\x0f\xfbi\xc2\x07\xcdU\xa0\x01\x92\xa3\xe6$\xd8"\xfdf1\xfb\x19\x85\xe5\xff\x7f\xc1\x83 \xe8\xeb\x0b\x9e\xbda\x04\xbb\x93\xd4\x97E\x08^U\xa3\x89\x04+94\xe7\xe5\x02\x13S\xc9\xd0s\x10\xf48\x1af{\x00\xa5\x85\xe1\x86\x0e\x96o\x9dw;8\x93\x98&}\x8c\x1d\xc8E\xc8\xc5Y\xf4^\xca\xfd\x87\xdc\x0e\x0e\x9f\xe1\xd8\x9b\xef\xb1>\xa3\xb0|\xfb\x90n\x1d\x965\xf3\xc4^?\x8b<\x1art\xf3\x9c=}:;\r\x81\x96\x9a1y\x85\n\xb8\xea\xc2&y9P\x17\xa1i\xaf\xd6i\xb4\xce1V\xc0\xa7\xdb\xf5x\xa6Gs\xe8\xd3\xa2\xb8u\x15vi\xf4\xc5\x9f\xf9;t\xb0A\xba\xc2\xa0b9^\xf973\xc9\xcf(,\xff\xda\x9b \x01\xaeE\xe2\x9f\xc3,\n`^T\x02\xbb&\x05\xa2b\xf4\x920\xda\xa1\xa5p\xa9:\xe1K@6\x04;\xbb\xc8\xf9t\xd1@\xbe\xa8\x88\x08\xbf`{8\xc4\xfd~\x1f\xce\x02\xecCJ\xa8N\xb2\x12/\xb6E\xd5]\xee\x8c\xb5\x10\xa7\t\xf1\xa3\xba\xe0wVX\xfe\xa8\x95I\x90\x80\xe0WNK\x9a\'?\x1f\xf0\xa6*\x904.\xc5\x92J0\'\x90|\x9b.Y\x93\xab\xf2\xcb\xce\r9\xba\x0b\x93(\xb2\x04:F\xd1\xcb\xc8w\x81O\x06\x91\xaep\x86\xff\x98pWj$\x88\xc7Ng\x1e \x82\xf1*\xc6\xfa\xe1{\x8a\xf5\x7fWX\xbe\x1d\xcd\x9b\xc2\xf2\x1f\xffq\xf0\xffE\n\xcb\xef\xe3[ld\xc86\xa4_\x7fH7\x85eSX6\x85eSX6\x85eSX6\x85\xe5\x0b?\xe7\xa6\xb0l\n\xcb\xa6\xb0l\n\xcbW\xd6M6\x85eSX6\x85eSX\xbe\xecsn\n\xcb\xa6\xb0l\n\xcb\xa6\xb0|e\xdddSX6\x85eSX6\x85\xe5\xcb>\xe7\xa6\xb0l\n\xcb\xa6\xb0l\n\xcbW\xd6M6\x85eSX6\x85eSX\xbe\xecsn\n\xcb\xa6\xb0l\n\xcb\xa6\xb0|e\xdddSX6\x85eSX6\x85\xe5\xcb>\xe7\xa6\xb0l\n\xcb\xa6\xb0l\n\xcbW\xd6M6\x85eSX\xde\x12"\xc0\xb7\x84\x88\xe5\xdf\x1e\xe9K+,?=\xb0_\xa5\xb0\xfc\xf4\xc0~\x95\xc2\xf2^`\xff\x03\xac\xe4W),?\x7f)n\n\xcb\xcfQX\xb0\xff\x1a\xf4\xbfWX\x04\x0c\xc1\x08f\xfd\x81\x02M#\x14\x82\x930\x89a\x0c\x063\xdc\xb3\xb3)\x06\xb1\x10\xcd\t\x08A\xe3\x08\x0b\xb24N\x118F\x92(H\x08\x14F?!\x89\x7f\xfc\x1d/\xc0\x820Ap8\xc4\x918\xcd\n \x08\x0b\xe8\xfa/ \xc7b,\x81\x804\x8338\xc2\x83\x0c\x07B8\xcb?\x1b\xaa#\xdc\xb3\xa1\xea\xfa\xf8<\x8e\xd2\x08\xf4S\x15\x96\x7f\xf5\x9f\xfa\xc7_\xfc\x8d~\x08\xf9\'\x88\xa2\x10\x84@ \xf6w\n\xcb\xd77l\xfeBa!8\x1e\xa2\x90uBX\x9c\x06Y\x84b\x10\x0e\x11\x04\x96]\xa3\x00\x05\x1ebP\x0e\x16X\x9a^\x7f\x1c$P\x10/\x90\x0cC\x82\x14\xc3\x80\xeb\xacq\xb4\xf04C6\x85\xe5\'*,\xeb\xd4\x81,\xb5\xee\x1a\x06E@\x10\x11x\x9c\x03\xf9\'\x19\x00\x83\x10G18\x8d={8# \x82\x80\xa4\x00\xc3\xebF$(N\xa0Y\x16Y\x7f(\x0b\xd2\xff\xf8\xcd\x15\x96\xff\x98\x06\xf9\x9c\xc2\xf2\xec\x94\xf1\xb7\n\xcb\xd7\xdf\xe8\xbfVa\xc1\x7f\x8c=\xc8\xe4hbC1\x8d=J\x1d\xca[\xb9\xa3P\xbc\r\x93u\xd8\\F-\xb5B\n\x1c\xb6\xbb\xc6e\xf2\xc8\xd2)&[\xfd\xb2t\xe4\xd9C\xef\x98\xd2\xda\xe8\xd4\x9c0\xb1\xa3\xa7=1f\x8f\xf0A\xf0\xce@X\xce\x9b\x1d\xbb>\xa5\xb0<\xaf\x05l]\xc8\xebF|\t\xf3@\xcb\x17\xc3#\x16\x14\x0f\xe5[C\x16\xfc\x99\x80\x90\xd4\x8ae\x9e\x93\xfc1&\xf8Sh\x8c\xce\x08\xb5G\xa5\xbe\xab\x90\xd6\x80\xe9\x9d\xb0w\xf9\xa5\x02*LGvy\x0f\xf9\xc9\x1d\x1c\xa5\xec\xe0#m\x18\x04\xdd\x0e\xfaE\n\xcb\x1a&\x84 \xd8+\xc5p\xec*\xb2\x82C!\xf7u\xbb\xbcg{\xfd\x82\x1d\x1dYNb\xbb\xda\xdf#\x80?\x9f.jB\x9e|\x7fR\x95b\xe1C\xa1\xc6\x1aTp\x1b\x8dq\x97eo\xcc\x05O\xdb\xa6\xee\x88\xdd\x19\x19\xd0\x07=\x9f\xf9\xdb\xfe\xcdNz\x9fRX\x9e1\xa20\x86\x92\xf8k\xef\xe9\xba\x05\xfb0^J\xb5\x14d/\xeb\x0fy\xe3\x10\x0f\xe1\x12\x15\x05q\xe7\x88\x00gv\xb1\xe7\xe6#f\xd4D=\xee\xcf\x83\x80\xb4\x1d\xc1:\x04\xef\t;tP\xb9D/\xc1\xc1"\x16\xfe\x1c\x04\'\xdb\xefo\xbb\x96E\xbd\x84\xd5MC\xe4\xa1PX\xefXMtO\x1f\xfd\xcb\x90\x12g\xde\x0e\xaeA\xe9\x9evq\xef0ur&\x07\x97G\x87T\x0c\xf9\x83-\x9e\x03\x05\xefm\xbd\x91z\xdf\x02\xc9ooZ\xdfl\xb8\xfa)\x85\xe5\x19&\x05\x13\xeb\xe1\x81\xbe\xf4\x95\xad\\\xc4\x865O\xc4\xaevH\xe0\n\x1c\x80\xd6\x95:\xfb\x9c}\x82\x12.V\xc2\xe0\xb0\x83\xf3\x96\'\xefv\xaf=h\x07\xc6 \xe0x\xa0w8p\xeb\x0b\xe3q~\xbey?\xefIMH\'\xaf\xdd\xed\xf5\xccJ\xeb\xeb\x8f\xba\xda\xffd\x85\xe5\xcf\xc9\xfd\xbf\xf5|\xcep\xa0\xb9\';d\x0f\x1d4\xe7T\xe7\xa7\xa6\x9f\x0fJ\xb4#\xe7\xba\x1f\xb2\xb4RP\xdd\xd3\xa5J+}L\xb1e\xf4\n\x80\xa4xM\x11=\xb4\xc3\xa5?96\x02\xd84\xa5\xe3\xfc\x9a\n^\x89\xcb\x91\xc2s\xe1\xdd\x13\xeeC\n\xcb\xb7EK\x10\xc8\xfa\xeb\xa5S_\x13jd\xb0\xe6\xfd\xb1t1t\x99 v\xda\x0c\xb6\xb4F\x16\xb6\x98\xf2\xbd\x1e\xa5\xf7\xf2\xee0\x97\xb44\xc0B\xd4/\xb3\xef\x9c\xcb\xbef\to\xbe\xc2\xb9\xb4\xb3\r\xe1Ta\xd2\xb2\xf4Ql\x85\xbe\x0b7#\xc0\xbd\xb9h?\xa5\xb0@\xf0?\xd7\xa4\x92\xa4\xd05c\xffs\x98s]\x14\xedm\xb8\x11\xfb\x1e\xf1\x1e\x13\x18\xc3D2\xed#\x058MA\xdfV\xd7\x1bH\xe2\xcd\xed\xfc|\x1b\x84\x06<|\xf4\xad\x02\xc4Z\xcfS\xbbAXJ\x1d\x8c\xf6\xa6ET^\xbf\xa8\x08\xe8\x15\x11\x0c\xa71\xf1\xe6l~Ja\xf9\xb6hI\x88\x00\xff\xe2\xbe\x92\xba\xc3\x0eW\xbb4\x01\x18\x0b\x81\x0b\xd6\x99[\xfc\x926\xa0o(2\x91E\xf4=\xb1\x11m\xf0\x84(\xd8\x8b\x068\xa7]\xd7zk\xc1?\x83B\x17\x8d4}\x97\x9098\xb81\x02\xee\xef\x86\xd2M\x94x^S\xc1/%N|Jay\x8e\xe2\x9a9\xaf\xb5\x1d\xf6\xd2\xa9\x8f\x85\xc06\x01\xd9^+\xcc\x99\xd2:\xee\x98\x1cg\xa9\x7fTx\xe1\x8a\xf7p=XZ\x02\x85\xd1\xb9\n\xd4}\xd5\xacU\x19\xa3\xd4\x10\xe2H\x1c\xb9\x0b\x8fC\xed\xb55\xd4\x94\xca\x14\x1d&\xda\xbd\x03\x14-\xb5\x19P\xbfk\x14|Hay\x86\t\x93 \xfc\xac2\xff\x1c\xa6\xed{8\x1adV\x04\xd3hW\x84D5\xdc*\x08\xc9\xca\xc9;\x12\x1c\xa9\xa7lCJ\x1aB@\xce\x05\xf7\xef\xca\xec\x8fJ\x0f\xbb\x1a\x87\xa3\x1e\x80\xed\xa2b\x18E%@I\xbe!FH\xa9\x11K\x9eYA~7O\xfd\x90\xc2\xf2\xed\x84#\x9f\xdd\xc6\xff\x1c\xa46\xe8\xb28\x8b\x1e\x95{S\t\x97\xd9\x11S\x9b ?\xf9\x84\x03\x8b\n\x02\x0f\xde\x90\xa3\xb1\x93;\xd7\xea\x1e\x82\x1cR0\xf7\xd1/h\xa8H\xba\x14\xa1.\xc7\xd1!\x88B7l\xdf\x14\xe4\x86\n!\xe1\x96\t\xf8\x8fZt\xfe\xde\x06\xcb\xf7s\x85\\\xbf\xe2\x15\'\x99\xc2T\x87\xb8\xa8\xd9G\xd9\xc3\xe5\xaa\x1d]\xe2\xed\t;\xa85\xdaR\x84\xbbC\xc5\xda7X 8\xf0\xc9\xa8\x0fn\xd5I~H\x81\x80\xce^\x86\xd8>\xcb.\x9f\x8e\xa4\x13&=\x1a\x13\x81\x16:\x1c\x7f\xf7\xa5\xf9\xdd\x8e\xd2\x1f2X\xbe\xa5\xfb8\x88! \xf8\x12&\x8c\xefw:x\xaf\xee1w\x0f\xdc\xd1\xb1\xd7\x0b\xff@\\u\x8b\x185\xc1.;\xd9\xe7\xb3\xa9\xb7\xe74\'\xb1.\xed\x10\x0b\x06H\\\x99\x89\xb2\xbeW\xd7\xf8N6\'Z\xdf\xb5\xbb\x9e\x11\xac~i\x95\x07j\xb9\xee\x9b\xddy?e\xb0|\x0bs\xbdaH\xfc5\xb5\tk\x8a\x8f\x89ND\x86\x19\x08\x85\xfd\xf5\x1e\xb4\xda\x05#\xecT\x86\x9a\x1b\x82J\x9e\x87\xe4\xe8(&S\xa9\x8f\x076Z\x0cVD\x18e7\xaa\n\xd5\x18^w\xbd\x9c2\n\xb8\x9c\xe1\xe98,I\xe6K\x1d|b\xdf\xec\xb5\xfc)\x83\xe5\xdb\xa2\xc5\x08\x04C\xa9\x97\xf3\xadX\xafw\x82`z\xf3.],\x82\xd6\xbc\x898)C\x0fR]\x17\x95e\xb3\x04\xc55\xaa(\xdb>\xed\xc8\x93p\x0b\xb3|\xee\xb0~\x8e\x86\x87\x961\x06\xdd\x17\xaez\xd1\x96\x87:\x0en\x07\xfbe\xe7 C\x82\xbe{\x8c\x7f\xc8`y\xa66\xeb/\x14C\xd1\x97\x0cN\x95\x121\xc7\xc4\xa5\x06\xa7\x04*\xc3\xf2\xa0?\xba\xfc\x94>2S\xe4\xc1\xe3\x9e\xd8\t\xa6\xb2\xf0\xac\x06&\x19\xc75\xb4(\x01\xbdv\xac\xcb\xf0z\x1c\xf6\x07\xd3\x9el0\x1c\xb3\xfa\x82\xdd\x83\xc6\xa9\xe7J\xea\xb4\x13\xf5\xa6\xda\xf1)\x83\xe5[\xd9\xb1\x8e\x10\x89\xc1/{\xf3\xd0\xf1g\xc5\xecw\x94\xa8J\xfd\xb1xp\xddu\x0e]\xaa\x98\x196\x99\r\x9f\xbextb/s\xcb\xb9\xcbp\xf5D\xfb~\xa4T\xc2(\xaf\x13\x11?v\x89\r\x99`\xb8\x0f\x83\xc7\x85\x90\nO\xb9>\xae\xa60<\xde\xdc\x9b\x9f2X\x9eaB\x08\t!\xf8\xcb\xd6\xbc$\x0e\x1c\xe7\x97^\xe2\xefP4\x86\xe1(\x01z\x1d\xdb\xc7\x82jOZ\xd3\xa5\x93\x04\r\xb0\xdb\xd4W%\xe4a\xbc^\x93!\x97\x90<\xc6Gp\xafW\xab#\xf3\xe8\x05\x1b\r\xf5 \x1c\xd8,\xc4y\xc6\xe1\x82\xdd\xbb\x00\xdc\x87\x08\x96\xe7\x9a%`\x92X\xc7\xe8%\xc3:\x9dXE\xb0|K$\xd7\xd3\x07\x85_\xedG!\xac59\x94\xa7\x84\xc3G\xa0\xf2\xcd\xecz\x84\xca\xc3\xc0O-\xa3M\x0b\x02wW\xb4\xab\xe4\xca@k3\x19J\xab\x1b\xce\xe3z5\xa9\x9do\x01KcU\x90\xc5i(\x90\x86\x90\xcf\xa2\xe0\x92\x1eOW;\xf1\xde\xec\xdd\xff)\x82\xe5{\xa9L\x82\xcf>\xd8/a\x1eB\xb0aD\xf9\xdc\xed\xdb\xa6\x14\xcb\xe2P5\xb6\x00O\'\xab\xcf\xf6\xfe\x19\x88\x0e\xed\xce\x9bwx)\xde\xe5\x05\xc4/\x9d\x80\xc9\x17ZCb\xacq\xc0\xc6\xb1\xad>G\xd46\xe1\x98CT>\x8e\x17\xfap\xaa\xec\xaf\x95c}\x8a`\xf9\xf6\xda\x18"\xbf\x1d\xe5/%\xeaC(\x8cA?\x83\xc9\xa4\xf0\xe2\xde\xe1\xe2[\xc4\x95\xb0\xa6\xdfN{{-\x91y\x9d\xf1\x9d\xa1/a\xdf\xcf\xfd\x1e\xe2)\x84\x96F\x1f\xb0\x08\x89P\x1c;\x15/\xdax\xd1\x17\xaf%ot\x9dO(\xa6\xd2\xfc\xe9\xcd\xc5\xf2)\x82\xe5\x19&\x02S\x08\xf5\x17\xd0\x03\x1c\x9e\x1b\xaa\xc0k\xc6V }QK\x91M!*4\x1f\x90\x11\xd5\xbe\x02\x8baB{\'\xecj\xb3}\x12\xdc\x0eqq\xe9X\xb0\xa2\xa7\x00\x053\x85\xcaL\xcd\xc45M\t\xc5\x0eS\xe7IUD\xcbJ\xfc\xcb\x9b\xa9\xe4\xa7\x0c\x96oa\xa2O,\xe1\xf5Vfx\xaf\xa0I\xe0\x08\xc4\x94S>X\xa4Q,LIBs\xa7\xf9G`\xf3\x17\x04\x0b\xc5\x91,\n\t$\xcd1$I\n<\x86\xd3\xd4\xf3\x0bA\x90 \xc9u]\xa2\x18\xcf\xb30\xc1C4\x88\xd3 \xc8\x83\x1cFS(!\xb0 \xcfa\x10\xfclO\xb6\x11,?\x91`\xc1\xd7)d\xe0\xf5\xa7\x82\xd4:\xa6<\xc1\x0b\x14\xcc"8\xc5\xf24\x8d\x82$,\xd0\x1c"\x90\x10\x87#$G\n\x02\xc8r<\x82",K\x0b\x02\xccQ4\xf3\x8f\xdf\x9c`\xf9\x8f\xbbV\x7f\x8e`y>\xc9\xdf\x12,_\x7f\xa3\xffR\x82\x05B\x7f\xdc\xd5>\x87\x16\xf6\x0eP\xecBK|\xa6\xea~/\x9d\xf7\xa5\xe8\x0f\x84B\x0e\xa84\xd6\xca\xb5\x11\tY\xbd7D\x07\xa2\nu\x03gCV,\x0f\x80]\xdcY\x8c`\x7f\xc4\xc1;\xb2o\x8b\x0b\xd8[\xb6\xe5\xd8\xfe\xd9\xa6\xe8\xf7\xda\x92}\x8c`\xf9\xf7[\xe1\xbfG\xc9%\xce5\xd4\x14\x95\xbdf\xd5\xc5\xc9\xe8{%\x05\xf2T/\xb0\x1d\xed\x07\x8d\xf724Q\xd7\xdf\xbf\x8b;5\xbbU\xa8}\x05*\x96\xe2\xfd\xc3`I\xf1\x1e\xc1\x1a\xba\xa68"\x93\x9d\xce\x90\x04\x8cQ\x07B\x18\xdfm\xe0\xfd1\x81e\x8d\x92\x840\xf8\xa5\xa3=\xc0P\xfb\x1b\xc6\xdf,\xff|\xc3\xf7\xf5d.4\x03\x0e\xcc\xac[\x8b\x04j2\xc5\xa5X!\x8e\xf1\x94\xf7f\xe0xZT\xec\x80Ifl\xe1H\xf0\x15\xe9\xf2\xe3\xe8H\xa7\x93ZJ0\x1c\xf0~\xc2\xcbB\x865ov\n\xfd\x98\xc0B\xfd\x13\x7f6\xd0$@\xece\xc1J\xb4\x00\x18\xc5RI\x8d\x82\xd1.\x0f\xa7\x0fF\x0eK\xbf%v\xa0\x8dg\x146p@N\x08w\xbc*[\xc1\xab\xaf\xca|Xf}\xbeO\x87k\x81\xc4KO\xb9V\x95\x94\x15A\xab\xedm\xf0\xd8\x14\xefB\x97\xfeQ\xdf\xa3\x9f-\xb0\xacSI`\x10H\x12\xaf\x9dBK\xbb\xe6z<\xf0\xcfg\xceXr\x19\xef\xcdy\xc7W\x17RCv\xd2M\xc1\xfdx\xa9\xaf\x90\x7fH\xe2D\xc2\xe9\xfe\x90\x1c\xd28#\xa8k\x8dVw)\xc0\r\xce\x10dc\xc7\xdfw\r\x19\x10\xe5\x8dT 8\xc4\x7f\xd4.\xf07\x17X\x9e\xa3\x08\x81 E\x12/\x9d\xdd\x86L\x8f\xa9V\xabN\x86\xdd\xcc\xb2Y%\x94u\x9e\xcb\xd8\xe1\x0f\x065\xdaV\x83Fg\x11t\xd54\xe4\xd2\xc0\x1d)\x0c\x9f\':Dc\x8c\xbc\xb1\xc0\xc9\x8aT\xbe\xcdC\xa8\x8c2\xb6bN\x05u\xce\xfd\x87\x1a\xbf\xd9\xd9\xedc\x02\xcb\x1a\xe63D\x02!_\xb6>\xf3\x80\xba\xe5t\xf5\'\xb2\xd9\x13\xa0X2T\xa2e\'\x96\x9e\\\xfd\x923\xcd|\xb4\xf6\x88*\x15\xd9\xc4$\'Uu]g^T\xfc\xa2\x0e\xfd\xe3\x94\'\x0b\x8f\xc6\x88=\x1d=\x9eJ\x93\xfa\xd4\x84\x13O=n\xa7\xb7\x1b\xf7\x7fH`Y\xc3D\x10\x1c\x870\xf4%L{*\xce X\'\r\xc8\xb4%\xde.\x10\xacw\x14)\x05\x8f\x13+\x02\x0f\x11=J\xf2::c5q"H\xe6\xa9\xda\xdd[\x19)F\xdetR\x076\x19\xae;v\xf6\xae\xb21\xbaKXe\xac\xf9s\xdd0\xcc{\x9d\xdd>&\xb0\xac\'\x1c\xb4\x1e\xf8\x04\xfaz\x90\xdf\x88~]0\xe8\xb3\x85\xbam\x9d0\x1eV\xf4\x18:\xc6\xa7N=]4\xa7\xec\x06#$\xb2\xc9\xc4\xaf\x14\'^k/\xddI\xbbS;\xbb\xf6\x19\x13\x104\xe0 \xe9\xa8#G\xfb\x8e\x1fOUu\x0f\x18o\x0f\xe9\xe4\x9b\'\xdc\xc7\x04\x96\xe7\xde$\xd6[\x1d\x7f\x85f\x14g\xc6\x96\xbb\x0fZ8\x96\\\x05\x8d\x050\x8a\x96%\xc0o\xec\xf3\xf5\x8e\xdf\x9a\xde}\xe0\xac\x90\xd4\x89x;\xa2g\xe6r\x06\xf2\x8b\x8e4B\xd9\x1d\x8f!\xed\x0eN]\x1d\xe1\xebY\xb9\xdf\xaaq\xbe\x00$ZI\xf3\xbb\xbd\x90?%\xb0\xaca\x82\x04\x82b\x7f\xd1\xf7U\xd2\x98R\x97\x15\xec\xc2\xed\xf8\xb6dr\xf9\x86-\tt;U\x0b\xc6\xef\r\xdc\x81\xa3<\x93%\xf0|\xe8\xda\xd3a\xa9\xdbS\x1d3IjG\xe8\x92Q\x8b\xc9\x9d\x85\xd6\x18\x1bc"\xe2\xab\xd0N$\x8e\xf3*u\xfdQ\xea\xf1\x9b\x0b,\xeb\x9e@\xd7\xfb\x94"_y9O\xc8u5\x00\xba&c\xfc\x01"\xd8.I\xaa\x1a5Jy\xe0\xb0(#\xdbX\xdcE\xd6A\x8e\xb05u\xb5\x97\x19|\xe09\x02\xfa\xa6rx\xc0\x0b\xad:W\x1b\x07S\xd5\x08\xdb\x13\xa8\xdc\x06:.\xe5\xe48\xbc\xb9\xf5?&\xb0<\xd3T\x08EQ\xea\xb5\xa7\xbd\x0b\x1e\xf2\x93\xc9\xc8\r\xcb\x98\x1e\xe2\x8e\xa7\xcb!\xd6\xe2\xecJ4\xbd~\xab\xcc\xd2\xad\xee@zj\xb1\x9a\x0fc\x11\x19\x12DK\x98\xbe\x0c\xba\xeb8\xce\xf6\xc0\tM\xb4\x96!r\xdf\xd0L\xc2\xcd\x13;(\xb2\xd4\xbe\x99\x8d\x7fL`y\xdeW\x10\xb9\x16\x1d\xaf\xf8RM\xc2\x03\xdd\x1c\x8c\x0bPI6_\xf7\xa4KIG6\x13\x1b\xa4=g\xddcV\x12)\xee\x056!\xc0t\xc7@\x85U\\\x91\xe9r\x11\xe6\x13XP\xd6p\xb0\xda\x0e\xa9\x13_8\xa0B{e\xe1\xb8PD\xdb\xfdQ\x98\xbf\xb9\xc1\xb2\x8e\xe2:\x15\x04\x84\xe3\xaf\xacdu\xbb\x15\x97=O\x16\xc9M\x06A_\x0e\xb9\xd9\xf5\xa5\xba\xd5:O8{.\xda\xb2\xb0\xbb?\\L\t\x8d]\xd0\x17\xc3\n\xd8i~q)\xafaO\x004Q\x17I\x1e\x00\x00[j1\xe1\xb1\xc7\xfa\xe0D\xf1\x9bj\xc7\xc7\x0c\x96g\xc2\x0f\xafe;\xf4\xda\xa2s:\x9e\x9c\x02\xa8Ow\x052\xe9\xae[\xf6*~\xafB\xf7z\xc0\';\xf6\xe1A\x00\xcf\x141\x1a\x0f\xfcQ-\xd6qd\x12\xb7\xecw\x8e\xdc\xe7L\x1b\xefP\xef\xc4\x05\xfb\x12w\xe5P\xe2\x10j\xcd~\xec#\xd6\x98\xef\x82d\x9f2X\x9ea"\x10A`\xe0K\xaa\x1a\xf8\x88\x06\x17\xf0\xe9\xc6C\xf98\xc7\xeaz\'B\xa8r\xb1|\x06\xb8\xe3j\xb8~_\xcf\xed\xea\x96\x1f\x1d\xe5"\x91\x19\xabx7p\xac\xa1\x999\x94\xcd\x18e\xd4\xa5)\x07&\x1f\xbb\x18\xa1}\r\xabk\xf1T\xc1\xbf\xca`y\x96\xa8 \x04\x91\xf0\xcbmx\xf7\xe4\xf5z?\x9d\x07\r\x94eq?\xb1PY\x14\x88\x9f\x91w\xc3\xd4`\xe8\x80\x89*\x0c9\x0f\xba\xac\xcfz\x83e\x86h\x0c\x1d\xfa\xd8\xb9\xc7^\xbf\xc4\x88\x15\xed\xdbL\xadi\xa1 \xaf\xfa\x08I d\xa8\xfb\xfb\x9b\x99\xea\xc7\x08\x96u2\xd7J\x1c\x07\xd7\xcc\xf7\xcfa&\xfat\xe7Y,\xbea\xbbj)t\xceJI[\x95u\xfcj-\xca`\x86\x174\xb9Ad\xe5\x88\x8a\x0f\xca\xc7p\xd8\xdb\x17\xfeh\xddmx\xe7\x91\xde\x00\x91\xf8m\x8f\x8c\xe0#\xc1]c\x1f\x10\r\xf4\xd0\x0fd\xfen\xa6\xfa)\x82e\rs\xcd\xf8\x10\x08~eB\x83@\xac2]3\x00\xbbS[\xf2vu\xa2Z\xc9\xefSC!\xb0*\x83\xf5x\x1aE$\x18\xe2\xe5\xba\x8c\x8cf\xbb\xd5\xd0C\x1dY\xcauU.\x1e\x97\xeb\xba;i\x11x\x07\xf4&(8\x7f\xff\x08\x98\xd4"\xdeL\xe1>F\xb0{\x99\x7f:\x93K\xca\xd11r\xb4l\x059\x95\xd2\x82\xa2\xaa\xf28\xd9.\xee5\x97\x92\xdd\x81\xe5\x1c\x16c%\x07"\xee\xde\xf6\x9d\\,\x94\xb7\xd31\x07\xc1\xfd\x9a[g\xb7\x9a\x15\x18\xcb\xdd\xf6\xcdwd\x1f#X\x9e\x93\x89\x93\xe8Zx\xff9\xcatD {\xa8\x8b\xbaZw\xc0\x89-A\xc4\xd8\x05dl\xc2{1\x15tW3.\x8d\x1e\x1f/\xd9\xf9\x10O\xca\xed\xb0\x17\x07@^\\4!\x1aI\xca\x1cq\xb1\xd9x\xbb;\xe9=\xf6\xa4\xc6F6\x0c\x9c\xc5\x13\xb4\x93\xe8=\xd4Gt\x9b6\xb8s\x829\xeca\xf8Q\x07\xc0\xd1\xf3\xb3E\xae#\xb0\xbb\x01\x19\x0f\x84O;\x08\xaa\x7fti\xfe\xe6\x02\xcb:\x8a(\xf4\xb4\x01\xd0\x97-q\x96\xa8\xb5\xe0\xbf\x94eK\x83\xcdy\xa6\xd9`\x16\xff?\xf6\xee\xac\xd9U\xebZ\x14\xf0\x7f\xf1+U\xa1\xef\x1e\xe9{$@\x88\xe6\x8dV\x80\x00\xd1\x08!\xf4\xeb/Z\xbe\xa9\x93D\xde>QJ\xfb\xee\xe5\\\xaa\\q\xb6\xed\xa55\xfb9\x06\x92\xc67\x0b\xb2\xd9\xed\xe4\x86\xc1t\xd9ol\xe3\x02\xc8Nes\xbb\x8c\x8c\xf4 \xed\xae\x91\xdf\xbbEZ\x1dP\xa5\x92\x81\xcbM\xad\xaeB\x1bN\x05\xce\x90\xd1`\xfa\x9a\xfc\xe6\x9d\xfc1\x81\xe5\x99\x89\xa3\xeb\x10a\xaf\xcc\xc3\x0c\x14\xd7\xc0\xe5\xb4\xaeH\x88LK\xfca\xaf*\xdd\xc52M\x8e\xa8\x9bG\x8b\x10\xc6\x9d\x1e\xba\x11a!\xda\xb8;\x96\xafh\x1d|\xda\xdf\x93@V\\\xbakBz\x00C\x0el\x87\x9eK\xd6\xdc\x03=\xd7j\xdd\xbcy\xbe}L`\xa1\xff\xb6\xc6\xd5$\xf5\x04Q_\xe2\xe5=%\xde\xa6\x85=Z`\xc3\x84\xdazJ\xc1aF\x1e\x86\x8bg\x91\xc7\xde}t\xa7@\xea\xfbf\x01\x01\xc82DD\xc7\xa0\xf3\xde\x11\xd3j\\8\x94\xdew>\xd5\xf3\xf0\xd9\xa9\xd8V\xb2X\xdd\x8c[\x01\xbf\x1e\xde<\xe0>&\xb0\xac\xc7\xf8z"\xae))\xf5\x12{\xa0<\xab\x98\x85\xea\xaa\xb6\x0ex\x8b5\x1d\xae\x98\x02\xa9\xaa\xbd\xb4\x92oDM\xe0\x8a\xf0\x85\xb3jo\xf4\xcc\x8b\x182R=\xb9\xc1\xd4_<\x18\xb6\xaf\xfap>!\x00.\x02\x83\x0c\xdd\xd1^3|\xccJ\x0f\x14\xf7f\x0e\xf91\x81\xe5\x19{ \xeb\xb2\xc5\xa8\x97E\xbb\xc0\x1aW%f\xe5\x92%\x9b\xdd\xcc\xba\xba\x9aK\xbeP\x17\xec\x16\xec3#\x8a\xd6\x15\xcd\xde\x91B\xef\xc5>a\xf9\xd3\xb8W\xef\xa7\x84\xaa\xecc$\xd5\xeb\xe1\xda\x81\xa8s\xd9QY2\x87\x11/2Y\xcf\x9d\\\xf4\xcdE\xfb1\x81\xe5\x99\x16\x10\xf0sF_f\x93\xd5\xd2\x82\xc6\r\xaf_\xaa\x02\xcdz\xfc\x80\x81\xba\xd3\x027w\xc2U!\xa5l\x8f\x1eI\\rK\xc4\xab\xb5\xb4S\xac\xc9\xdbS\xbb\x04\xd3\xa5{\xd7\x97]\x93\x98\xc8|\x80\xd9\x82\x88\x05\xd0\x10\xb2\x0b\xebg\xc8\xe3\xcd#\xe8c\x04\xcbso\xae\x13\x8f\xc1\xaf\xcc\x1d\x89\xfbhHR\xf7^TX\xf1\xa8Z\xa7\x89\xf5\xbc{\x99\x82\xce\xed\xc4\x15\x9c\xdb\x9aE\x03Z\xe2i&v\x07\x92\xc5:\xac\x88\x9dqv\x98\x04\xdc3G%\xd3v\x16~:\xd7Y\xda\xca\xae\xb67kew\xc23\xf6\xcdn~\x8a`\xf9J\xf2\xa0gT\xf6r\xa1\x88 u\x80\x92\xfb\xad\xc32\xac\xbd](\xa2R\xabx\xcc\xdasbyN\xe1Gt\xe9\xc6\xb8K\xb4\x13-R\x11\xcf\x02\xe3\xd4\xea"e\x8f\xc61\t\x85\x89\x02\x9cG$\xc5\xe3\x95\xb9\xe9\xdc\x91\x97\xcf\xd4`j\xea\x9b\xd1\xc7\xc7\x08\x16\xfao\xe4\xdaI\x88~Meoz\xb6\x97\x83\xd2:\x9eS\xb0%]l\xe1\x98\x07v\x16\xbd\x80\xcf<2\xccN\xac\x1dX\xf5\xc1\xb2\xa9H\xb5c\x94L\x85\xa3\x8e@E!_\xaa\xd29\x96\xb7i\xbc\x92;\xca\xd2\xe0S\x86\xa0\xb3\xd1\xc0\xe2\xfa\xc3o\xbe\xeb\xf21\x81\xe5y\x02\xad\x816\n\x13/\xf7\t|\x83\x11\r\xe23$\xeaL\xdf*C\x19\x13#\xb1\xe0\x04\x10N\x8e\xbe3\xa6r\x8d\x82J\xb0\x97\x96\x9ed\xe5,\x16\xf6(\x9d\x88\x9d\x9c-\xae\x81\xc3\x9dA\xd0{\xfeT\x99e*\x9c{~\xc4\xeav(\xb3\xf3\xf7\x8a\xb1>&\xb0<\xaf+\x84\x84\xa8?\xb8\xae\\^n\x89\x12Y\xb8\x87k`\x12\xf5\xf0\x0c$\xbf\xd8\xb1\x0e\xec\xb0>\xd3\xc7\xf3\xa5\x11\xf7^~\x14\xf32\na\xb7\xb9\xf6\x07\xc3\x02@\xa1\xad\x8d3\xd6\x8ej\xe17\xf0\x8e\xe5ADs#3#\xb8\xd3nw\xe4~\x7f"\xf0\xbf\x0b,_\x8f\xa47\x81\xe5\xdf\xfe(\xf8\x7f\x91\xc0\xf2\xd7\xb1-6.d\x1b\xd2\xef?\xa4\x9b\xc0\xb2\t,\x9b\xc0\xb2\t,\x9b\xc0\xb2\t,\x9b\xc0\xf2\x8d\xdb\xb9\t,\x9b\xc0\xb2\t,\x9b\xc0\xf2\x9de\x93M`\xd9\x04\x96M`\xd9\x04\x96o\xdb\xceM`\xd9\x04\x96M`\xd9\x04\x96\xef,\x9bl\x02\xcb&\xb0l\x02\xcb&\xb0|\xdbvn\x02\xcb&\xb0l\x02\xcb&\xb0|g\xd9d\x13X6\x81e\x13X6\x81\xe5\xdb\xb6s\x13X6\x81e\x13X6\x81\xe5;\xcb&\x9b\xc0\xb2\t,\x9b\xc0\xb2\t,\xdf\xb6\x9d\x9b\xc0\xb2\t,\x9b\xc0\xb2\t,\xdfY6\xd9\x04\x96M`yK\x87@\xde\xd2!\xe0\x7fj\xd2\xb7\x16X~z\xc7~\x95\xc0\xf2\xd3;\xf6\xab\x04\x96\xf7:\xf6\x1f@%\xbfJ`\xf9\xf9Kq\x13X~\x8e\xc0B\xfe\xcf\xa0\xff\xb9\xc0\xc23\x10%\x90$F=\x99\x07\x16%x\x91&8\x86A\x05\x92\xe3p\x06\xe5\x9fe\x8d\x05\x9a\x10\xd7\x91\xa0i\x08\x17`\x9a\xc5\t\x01\xa6D\x86\x11y\x98\xfa_\x04\x16L\xa0\x05\x96G\xf9g\t(Q``\x9e\xe1y\x01aa\x88\xe7\x10\n\xe7!a\xfd;\x8f\xaf\xb9\xa4\xc0#\xeb?\x13)\x98]\x7f!)\x08(\nC\x04\xf6S\x05\x96\xbfW\xba\xf8\xed\x0f\xbe\xd1\x8f\x10\x7fCi\x1c%a\xea\xf7"\x17?"X\xbe\xbf_\xf3\x07\x04\xcb\xfa*8\xc7\xc2$\x82P,\xc2\xac\xed\xe3q\x91!1a}e\x8a\xe4\xb9u\xb2\xd8\xf5\xc7Q\x92\xa7D\x0c\x16`\x81d\x05\x02EDJ\xe08Q$\xc4gY\x9b\x8d`\xf9\x89\x04\x8b\xc0\xb1\xe2:\x91\x02\x0b\xc1,\xc5r\xeb\x1f\xa1\xb5!\x02/\x92\xf0\xb3T\r\x8a\xa08\xbb\xfe"z\x1dc\\\x14D\x98C\x04\n\xe6)D\\W\x9fH\x11\xc4o\x7fq\x82\xe5\xdfvA>G\xb0<\xc7\xecO\t\x96\xef\xbf\xd1\x7f)\xc1\x82@?\xb4If\x88\xdeu p\xa2\x1d\x15T+;?PH\xac\xd6\xbbD\x00,\xbd\xc7OK\x82\xc8<<\x8b\xd7Z}\xe4(g\xc7\xe5\xd9G\xad\xc5Lm\x01\xefq\x9e\x90w\xd9\x03=HS\x1f\xa4`?`\xd3d\x9e[\xe3\xcdj9\x9f\x12X\xd6[\x01_o\'\x14G\x91\x97\nOGD\xda-\'\xa6\x08\x8e\xbb$:t\x84\xa4\xb1s\x7f\x1d\x8bQ\xd9\x9d#\x93\x03\xfbC\x16\xe9\xc7\x1b!\x98n4\x8d\xde\xf5pGZ\xb3\x01UA\x17\xe9F\x9e\xf52\x96F\xa8\xc4w\xf6\x89\x8at#"\x0e]H\xbf\xc9\x93|\x8a`Y\xbb\xf9,\xde\xff\xda\xc7{D\xe6\x0fK \xe84G\xc3}f\x8df2\xfa\xfd,!\xd2!>TV\xad2\xddIB%\xa9\xb9G\x89.\xb9\x83\xf5\xc0\x14\x0fX\xeak\xda\x17\xb2\xe8\xec\x82R\xbb\x9dBh*\xb5\x04\x98{]\x1a\xe8\x1b\xff\xb6M\xf2\x19\x82\xe59\x95\xd0\x1a\xd1\xac\xfb\xfa\xb5\x18{\x845\x064V\x00=_\xb2\x93\xad\\\xf3bH\xa1\x1d\x8eHy?\xef\xa0\xe8\xec_\x95\x10\xac\xef\x9d\tU\xfb\x03\xd7.TD\xd7\xbeHz\xb62J\x9cE\x05\xa3\xe0;\xe2|\x18%}\xae\x81\x0b\xd6\x13l\xfefE\xe6O\x11,k7\xb1u\xb9c(\xf4Zx:\xef\xa3\xf2\x01\xa1c\r\x9b\xce\x89\xa1}L\x0c\xf1\xce\x8d\x9eE\xd1M\xe5NB\xd0\xf9\x16\xf8zy\x85\xb4\xf5\xe6\x8f\n\x19\x0c)\xdd\xa4\x1a\xcf\xdd\xc9G\x02\x92OP\n\x8e\x87\x96\x8a\x80\xf2rg\x08$\xd6\x1e\xbb\xf0G\xc5\xba\xfe\xda\x04\xcbs\xb1\xa0\x04L\xaf\xb3\xf0Z\xd9-hR\xb4\x0e\xe7\x8b{\x19d"!\xb4\x956\xd2\x12Fh\x8di\xb2t\xcb=\x8e\xcb\xa2c\x7f/Kf\xaa\xcf\xc3Ds\xbep:\x9ar\x8b\xd0\xe5#nwo\xee\xcdO\x11,\xcf\xbd\t\xa38\xb4\xfe\xf8K\xa1\xbe\xb6\x18xp\x9a\x85\xca\xf6\x0fc\xa7Bw\x14\xbd\x93E\xdbX= zGE\xdf\xf9\xfac\xa6\xccI[\xd06\x8f\x95\xf8\x00\xa73\xa2;>\xca\xcah\x87/C\x9edg\xa9\xcb\xae\x8a\x1et|y\xa5\xcf\xbe\x11\xfc\xa8\xe8\xfc_\x9b`YGqM\xfb(\x94\x82\xe1\x97=Q\xee\xea\xcb\xf9F\x1a\xc5\xddu\xaa\x9a\xbb\x82\xa6G\x87\x13\xd9f5q\xb9U\x01\xc5\xfb\xde\x92\xec\xb2S\xadX\xd1p:=.\xe6\xa3\xc2GP\xedl&\xcf\x0f\xc4\x8e\x18Z\xca\xb2Q\x9f\xf0\x9d\xa3\xeb\x9f\xcf\xd4%p\xde\xb5\xb4>D\xb0<\xf7\x04\x81\xaf\xb1\xc2\x1f\xf8r\x88)[M?\x94\xb0\x1b`\xf49\x88\xb8>\x84\x16\xbc\xcd\xbcq80\x8b\xa3\n&R\x94\xa8\xdf\x9d\'yj3\xf1\x88Gl\x03?`\xe8\x16\xe1\x08:q\xe2\x89\x8c\x91\x1d\xae\x04\xb4\x81\'\'\xc3nJ\x9cw\xdf,\xdd\xff)\x82\xe5\xb9\'\xd6\xb4\x89F\xa8\xd7\xfb\xea\xc44\x1a}\xa4\xb2G\xb6\xb7n\x99\x90\x8bx\xc0\x9c\xd6@\x07\x13)\x0e\x1a\xdc\xd3y9G`t\x88\xcbc\x9c:z=)K\x17)\x00\xdeJG\xd0\xe1.p?_\xc21\x1a*\xdf\x80TMlz/\xe3\xd1\xf4G\xa5\xfb\xff\xda\x04\xcbs\x14\xd7P\x18\x83!\xf8\xe5:d\xeaB\xa5\x1d\x8es\x03\xea\xf1\xa8\xe4\xc0>\x88\n-97L\xed+\xcc\xf7Nr\x80kar\x18B\xe4b\x0eI$\xcd\x98\xa2\xfb\xaek/\xf6\xf2\xe0n\x97\xb3\xb1\x13/\x89\xf28X\xf2-:i\xa4N\x9d\xcc\xf8M\xce\xe2S\x04\xcb\xd7u\x88S8\xf2Z\x1f\xdc#c1N3^O8=\xe3\xe7\xc8\x8e\xd4=\x80\xd3\x84\xd4\x05\xeb\xbe\xb8\xdfiI\xce\xb1\x19l!\xe2j\xef\x17\xbd\xb8$&\x08a"\x82\t&#\xa8\x06Fp\x1d\x84\xef5\x9a\xbd\x9e8a\xbex";\xd9o\xd6 \xfe\x94\xc0\xf2\xd5K\x02\xc3a\xfa\xb5\x9e+\n\xc3T{n\xb4)\xcc\x98\x1c\xcc\xf4Bj\x06\xf5\xbc7N\x1c8\xde\x01\x8d\xa5L\x04;\xf6\x8d\x06v\xe79\xe2\x14\xd3\x04t\xa1\xce\xe7\xfeQ\xbbJ\xba\xdf-\xa3F\xee&H\xf1n\x00\rA\x13\xd2v;\x0fz3\xb6\xf9\x94\xc0\xf2<\xe0H\x9aD\xd6h\xefe\xcd\xc27uL`#\xe7/\xe4\xa5\x05\x99\x07\x92\x86*\x1d\xa8&)\x83{\x02\x89B\xbe\x14%R\xc8g[4zj\x88Y*\x1c\xd55\x0b\x05\x11\x12\xa4\x8cC\x8d\x08\xa5yV\x80\xecZM-\x08\xe2\xd5\\\xdef\x92y\x13\x9a\xf9\x10\xc1\xf2{l\x03\xc1\x14\x8d\xbf\xcc\xe6\xd1>\xd3r\x1d\xc4\x19\\\x1f;\xf2Q\x0e\x044\x1a&\xbb\x93\x9e\xef\xcex\xf7\x87_\xcc\xe1\xc4\x99.72\xb7{zX\x7f;t\x87\xf6\xed.\xed9\x98\xc9\xa6\xbc\xba\xe4\x1d\x05I\x88\xd9\x06\xf7\xca\x83\xf9*\xab\x8d7\xbb\xf9)\x82\xe5y\x02\xd1\xcf*\xe3\xd4+\xa5\x85\x80\x88\xe7z\x96y-\xb9\xbd\xad\x8b\xec\xa9\xbaWCeSc\x14M\x0e\x91\tfq\x91\x0e\x99\x92&\xd6%\x1c"\xe3<\x80\xfa\x1a5\x13\xa6\x08\t\x11\xe3\x01`\xc7%\xfd)^O\xf4h\x1a+#\xdc\x15{\xc6\xe0\xde\x9c\xcd\x0f\x11,_\x8b\x16\xc7\xd6{\x0fy\xb9\xaeHn\xdcU\xa9)\t\xc4b\x05QW,\xf1=%`d\xbe\xfa(#\x97\x8b\xc1\xd9\x9c%\\\xc0\x83>\x92h\xba\\\xa8\xd9\xa6\xb2\xfc\x98Ub\xbf\x10s\xac\xa6\x91YY{\'\xbdW\xf04<\x92\x9a=\xa3\xfe\xfd\xcd\xc2\xd9\x9f"X\xbe\x92\xe5u9`\xc8kQ{\xdek\xfdg\x02\x85\rb\x98\xde\x11/ K\xd7\x16\xb3k\xbd\xdc\xdb\xd0\x16\x88\x8e\x05X\xba\xc7;?1\x1f\x1d\x0f\x1e\xdb\xa6\xc6\x8e>\x91=\x9a{\x080\xf2\xbe\x1b\xcd]\xe3\x89\x1e\x81iZ\xb6\xde?\xd6i*\xdf\x0c>>e\xb0<\x17-\x82C\xebM\xf4\xdaMD\x86o\xeb\x04^zk\x87\x9bN|;D\r\x15-\xf7\xb36\xdd\xa9\xf9\xd2\xf0\xf9\x99?\xcd\xb5\xb7L\x17\xf8F@\xbcdG4\x89\xadG\xd0c\x0cQB(\x11}\xb7\x05\xfe|\x84<\xa7)\xf6\x16\x15\x9e\x02\xd8\xa7\xdc\xe24u\xa4C\x1e\xcfY\x15\xa2E\x10pe`\xc4\xcc\xc9\x1fS\xe6b\xf2\xbcb\x1aR\xfd\xa3k\xf3\xafm\xb0<\xd7\n\x01\xafW;\xfd\xaav\xf4\xa6QbS[\xf2\x0f\xf7\xe8\x9d\xaf\xb9\xe7x~&!\xa7\x16Z\xa4C\xc2H;\xda\xb8a\xf9#\x18\x18\x97\xc6\x13\xa6\x1d\xc4\xc9\t\x18\xb8\x15\xccT\xd0\xa83\xc5I\xd2N/\x12#\xa8\xfb:\xeeCj\x11\xd4\xc7\x9b[\xe2S\x06\xcbs\xadP\x14I \xc4\x0bK\x14\x8c\xd2i/%\xe5p\xb7&\x0fm\xbc\x96\xc5\xb8\t\xd5S\x8e\xa1\xcc"\xcd\x97\xfaB\xe9\xf4\x05\xb8\xd7M.A\x0f\x02\x9f\xa7(\xeaw\x07\x07a\x00\xa9>\x81\xf2\xd0v\x0c x\xb1\xd1\xe2\x15\x01\x1e\xb4P{\x00o\xa2\x1d\x9f"X\xbe\xb2\x0e\x92\x86I\x9a~\x99\xcc\x8a\xc8 \xd3\x838\xc3LJ\xc2\xef\x8ct\x1f\xe9.?v\xf1\x08\x9f\x0fZ\x7f\xb6\xe4\xeb\xc5[\x0e`\x95HU\xa6\x88\xac\xb0\x06\xd6\xf0\xfev\xda\xe5\x0cR\xd2\xc4\x81\xcd\x86Cd\x87J\xd2k\xcd\xb1\x11P\x9f\x9c\xccw\xc9\xc2\x0f\x11,_O\x8e\xf15\xa1@_\x9f\xab\xc0w\x82*\x1e\xae\x9a;\x07\x7f\xa1[\xc2;1\xd2\xdd\x04\xdd\xa2\xc9\x9bN?\xf8kn\xd0;\xfd\xe8\x01S\x06\xf5\xe0c\x0f\xef\xb1E\xc40pqR\x86\xe7\r-{\xa0B|\xbe.\xb8\xdd\xcc\xe2\x81\x9f\xbc]\\\xbd\xf90\xf0S\x04\xcb\xd73O\nG!\xe2\x95\xb6\x9d\xcc\xf0*\xc0\xbeL\x9d\xe6\x1d\xe8\xc6\xcb\xa1,\xf6\xd2\xeexCk\x82n\xf2\x14Yw\xcdL]\x1e\xc8\x9a\x03\xc1\x95l^\x03\xf3F\xa3n\xb9\xf7\xb4\x0b\xa0\xa5\xe6>\x12..\xc7\xee\xac\xdd\xe5\x96\x86\x146\x06\xe3\x18\xbe\xfb\x96\xce\x87\x08\x96\xafw\xae\xd6\xb1\xa1\xd1W\xfe\r\xb7\xd1\x0e"\x80V\x94\x87\xa0U$I\xa3qi\xdf\xd5\x07\x19\x9c\x1f\xa99yG^:\xe6Z\xfdxXIHO\xecB\x07\xc1T\x1d4\xc8\x96\xcb\xca8\xa4\xc7cP\xd7\xe7\xa4\x9c\x88iH;\xcfH\xfcs\xbc\x7fW\x9a\xf9\x10\xc1\xf2u)\xa3\x14\xba\xae\x80\x97\xbd\x99\x97\xbb\xc5P\xe0\x89G\xdc\x90\xf5\x99\xf0&\xe9\xca.#\x98j\x9a=\x9b\xf1\xaf\xa6tT\xd8\xbd\xd1?.\xc1=$\xafam\xa5|\r\xb5j\x08\xca.\xb8\x8b\xec|\x94TH\xea\xab(R\xd0\xe9\x81\x90J\xd8\xabo\x06\xcc\x9f"X\xbe\x8e \x94\xa0\t\xf4\x95f\x8c\x95\xf5\xeao\xa9(\xaa\x06;\xa3\xe44g\xfc\x83Oz\x80\xd5W\xfbb>\xb6z\x87\x93\xea\xb8k\x05\xf5t\x043\xa6\x9d&x&DP\x88K\x83\xb7\x0e\x8b\xff(\xae\x11\xa9\xc8\x19\xe2aJ\xc0\xf6\x14}\xd3\xc17\x1f\xed~\x8a`y.\xda5\xce^g\xf3\xf5i\x1d\xa4\xcd\x9a\x0c\x06\xf3\xc8\x0f\x81}\x9e\xae\xa3\r\xd6\x16\x95d\xda\xc4\xa1\x9a\xe0V\xc8\xd2u\xfbD\xeds\xae\t\x96\xd0\x0f\x1fz\xdd\x9ez\x0b\xe8\x1f\x16\xe8#4\xf6\x08Sc\x7f\x83}\xddG;\xbcH\x1a\x08\xbd\x9e\xdf|\xb4\xfb)\x83\xe5k\xd1R\xf0\xbad\xa1\x97E[zid)\x17\xb1\xe1\x8a\xf4\xc8\xcb(\xac5\xf0\xc3\xa3\xcb\x80)\xee\x05\xba\x13\x8b<\xcc\xa23\x00\xb3\xbaVHE|\xc49QDv\x93^\xf3\xa4rvIH\xc7\x16\xa6\xba-\x87\x944\xa7\x9b|\xe6z\xeaR}\xb3\xf7\xe8>d\xb0|%\x91\xe4\xf3\x035\xc8\xcb(\xfa\xd8B\n\x02p^\x06_\xc7\xd8|i\xd7\x16\xc1i\xd4\nc.j2=\x9d\xf5\xc4\xb1\xef37\xebi\x86\xd5\xa5\x0c\x81{\xf7"\xe0\xbd\x8dpH\x16\x93Q\x1dG@\xd3BW\x83\x0e\x06\xe0\x0c\xeenG\xd8d\x98\x7f\xcf`\xf9\xda\xcc\x9b\xc1\xf2o\x7f\x18\xfc\xbf\xc7`\xf9\x0b\x15\x13\xdd\xea\xb3n\x06\xcbf\xb0l\x06\xcbf\xb0l\x06\xcbf\xb0|\xe3vn\x06\xcbf\xb0l\x06\xcbf\xb0|g\xdbd3X6\x83e3X6\x83\xe5\xdb\xb6s3X6\x83e3X6\x83\xe5;\xdb&\x9b\xc1\xb2\x19,\x9b\xc1\xb2\x19,\xdf\xb6\x9d\x9b\xc1\xb2\x19,\x9b\xc1\xb2\x19,\xdf\xd96\xd9\x0c\x96\xcd`\xd9\x0c\x96\xcd`\xf9\xb6\xed\xdc\x0c\x96\xcd`\xd9\x0c\x96\xcd`\xf9\xce\xb6\xc9f\xb0l\x06\xcbf\xb0l\x06\xcb\xb7m\xe7f\xb0l\x06\xcbf\xb0l\x06\xcbw\xb6M6\x83e3X6\x83e3X\xbem;\xff3\x1f\x02}\xcb\x87@\xfe\xa9I\xdf\xda`\xf9\xe9\x1d\xfbU\x06\xcbO\xef\xd8\xaf2X\xde\xeb\xd8\x7f@\x95\xfc*\x83\xe5\xe7/\xc5\xcd`\xf99\x06\x0b\xf5?\x83\xfe\xe7\x06\x0b\x82\x8a\x02M\xd2\x14\xc6`$G<+\xd4#\x84\xc8\xe14\xc2 $\x8cB\x98\x88\x92,\xc3s\x88\xc8\xc1\xedi\x89\xf4|sX\x04)\x05f\x10\xbb\xd3T\xd7CF\xdd\x008\x13\xfa6y\xad\xa2Q\xaf\xce\xe9T\xf3\x08\xad^g\xa9\xd3+\x9b\xf1\xce\x8f\xaaE+\xff\x1e!\x0b\x15\xe9)b\x91\x15\xb5 4\xf4f\x05\xbbO\x19,\xff\x12\xf4\xfe\x93\xbe\x84\x02,\xe9\xaa\xe3\x98F\x8f\xfd\x85R\xb0\xddz\x98N\xe5iF5cP\xcd\xda)k\x17}`t\xb1\xa8uN`\x07\x01\xa4\xb2\x1a"{\x94\xaa;G9k\x88Djz\x7fp\x01\x87\xbd\xb2N\xd6\xa7p\xef\xbeY\n\xf9S\x04\x0bJ\xfe\x8d^s\xab5hA_\x0e\xf1{\xbd\xccG\x0f\xe3\x8fi\xa8\x1e\xd5\xd1\xa0q\xcd=\xb9\xf1\x1cFCm\xb1\x9au\x86;\x0b\xef\xf6b1QS"_\xee\x92\xad\x8c\x9e+$\x04\x9a\xa4]\x1b\x96\x8b\x9d\xb1U\xcc\xdee\xac\xbe\x12jR\x16\xd82\xfd\xa8r\xffO&X\xbev>\xf9U\xed\xfb\xd5&IG.\x8d\xe3\xcebCn~\x18\xe3-\xc8\x15\x1a\xb4!\xb19\xfa\xba0\x0f\xfd \xb6\x80\\\x11\x0b\xc1y\x02\x1d\xde\xee\xf1\x9a\xf4{\xfc\xe1\xd17\xec\x15\x87=\xf0\xaex\xbd\xb9L\xd99\x86\xd9\xe5\x91\xeb\xca5zs\xcd~\x8a`\xf9Z\xb3\xcfr\xa8\xafU\xcawp\x022\xe7\x1a\xf0\x02\x8a\xd9\x8f\xd6\xc4v\xf7[\x02OBZ\xc6:^\x02\x88\x03\xa9H\xd0\x14b{V\x84\xb5k4\xadA\xf5\xee\\\x85\xce\xe2_\xe1\xd3\xe0\xd6\xc4\xbe\xb6w\x97\xbb\xca\xda]\xd4\xc5\xfct\xd0\xe67\xeb\xbe~J`y\xaeY\x82 \xb1\xf5({\xd9\x9a\x1c\x92\x16g0\x07n\xc0Y\x9c\x06\xa8\x9f\x01J\xb4\x14\xaa\xca1\xbb3|\\\x98\x84\xca\xaf\x0b\xc9\x06p\x02\xbd\x03\xf2\x95\x9d"4o)\x9e\xa7\x16\x12\t\xca\xc9\x1e\xcb,\xaf\x91\x00\x06M\xc2\xb3\xd7\xa3\x0b\xd2O\xff\x9d\x02\xcb\xbaV\x10\x08\xc7\xd6\xcc\x1f{\tmr\x96`yW\xa1\x89\xf9\x01\x951 \\r>H{\x98\x82\x97\xdb\xb1Cg\xb6\x12\xfbk\x8b\xc4\xbc\x1b\x11a|\x82Thq\xe5HUo\x18}\x80\xd8\xb6f\x12\x12>\x0f\x11\xed@\x15@[)_^3\xceys\xe7\x7fJ`y.\x16\x1a[#\x1b\x92|\xe9fpQ\x9b#{\xc8}]=*\x0b5\x91&\x13\xd5\xedi\r\xd9\xc6\xf8\xc8\xcb\x18Q\x90\x07\xf4\n\xf6\xf2\x12\xd6\xea\xb5\xde\x03\xd8\xc9\x08\x9cBN\xfc\xb2\xf4\xae,1\xdb0\xb4\x84w\r\xcf\xd3\xe2Q\xe7\x9c~\xbb\x1f\xdf\x8c\xe0>%\xb0|\x05\xe3\x08\x8e\xe34\xf1\x02\xcd\x9c\nf\xe7\xc4\xac,\x88\x89\xe5\xdf\x8a\x0bR\x8caMk\x86\x00\x05\xa0\x1dNyL\xef\xc7L\x8bE\x85,\x8c\x03%\\|\xfap\xe2\x1b\xc1\x04\xce1\xbf\xbf1\xed\xe5\xe8\xc0J\xa20\xfc\xae\n\xc2\x9d\x1b\xf3G_\xfaQ\xce\xf1\xd7\x16X\xbeF\x11[SV\xfc5\xdc\xa7\xcd\xab\x96\x1e.\x15\xadO\x18\xd1\xde\xa1\xd6Y\xcf\x13\xb9\xd2\xd3\x920\xf2s\xa2@\xf7^\xf7\xc1\x9e\xab\x0e\x88\xc7F\xbcK\x9e\xb3\xec\xde\x83\xbb6\x9f\x94\x0bXt\xea\xd5\x1a\x1e\x11\xa9\xa4H2\xec\xb5=F\xb52\xfcf\xd5\xdaO\t,\xbf\xc7\xc1\x18\t\xaf\xe3\xf2\xaf\xddt&\x18\x14/\xd1O\xd6\x7f\x8aL\xe5\x9b\xd0\xcc\xa7\x08\x96\xafn\xd2\xf0\x9a\x0e\xbd\xc66b\x15t>\xeb\xf0P3\xb4\x81\x81\x84vg\xf9\xbcr\x89\xe4\x80D\xc5\n\xc7\t\x03\x90\xf2B\xea\xc7\x85G\xc1|\xe7a\xa7\xc7\x92Hap\xe3\xf2\x14\x89\xc1\x9bX\x8e\x8f\xbb\x90fCZ\xebk\xe2\n7\x89\x93\xbd\x1b\xa8~\x88`\xf9Z\xb4\xf0\x1a/\xa0\xaf\'\\/\xc5\xae\x04%\xe0^v$\xe1\xdc\xf1{\x0b\xa2\xeb\x92\xbe\x1136z\x0en\x17\x12s!\xba\xaa&\xfb\xf5\x9eC\x04}\x9f\x1b\xb4\xd9\xef\xe2\xfb\xc0\x1f\xba>R\xcd\x16\x19L\x0f\x88\x18\xdfX\x08\\\xe2\x9b\xcb\xc8\xbf{\x90\x7f\x88`\xf9\xcaQqx\r|\x89\x17\x00\xa1\x94\x9d\x9b6\x98`B\xeaH\xa5\'v\x14\x80\x86\xceX!\xec\xcev\xbf\x97\xa8\xde\x08`U\x92\xec\xd0\xa4\xee\xc2\xfd@\x10\x1d\x92T\xc3\xd0t\'\xc1\x7f\xc8\x17\xb8e\x03\r\xe1\x19]\xdf\x1f@\xb4\x96R\xff\xba\xde.\xbf\x86`y\xce&F\xa1\xf8z\xcb\xbftS\xbe\x0fH{\xed\\~\xe0@\x8e\xce\xf6\xb3\xd6i&\xd2\xcdY}agP\xa2\xb4Zd\xe4\xf9\xd4&^KL\t\xc3\xa8\x0b\xcf a\xb7\xc4\xb6\x8e\x01\x8f\xfb\xee\x1eO\xacp@\xad\x888;\x85\xe6\xd4\xb3\x19\xc9\xef\xda$\x1f"X\xbe\xf2\x0e\xea\xa9#a/\x01\xb9\x84P3\x19\x17!\xda\xee\x0e\r\x80\xb2\xda\x83\xc9\x8a\x8c\xdd\x9f\xc0\x83%\ra\xacx\x8f\xfc\x04\x86M\r\xfa\xa9\x059\x17G\xc9\x99\xb9\xc4\x87\x14\r\x9b^g\xae\xca\x993\x8e\xd7L\xee\x11c\x87\x84\x89&B\x8f\xea]\xff\xedC\x04\xcb\xd7\xa2]\x17,M\xbe>X\xc1\x1e3\xb9;\xc5\x98\xeb\xdbw\x83\xc2\xafI\x8a\xc9N\xa5\xe8\x88/\n\xd5\x19\x19\x1e\'\x93=\xc5\xe1\x00\xd8\r\x9b\x1d\xf8bG\x15\xcbdAapFv\xa8\x11\t\x98\xa9\\\x8e^\xeakf\xeb\x93\x15\'\x9d\xac3\xf8\xe6\xde\xfc\x14\xc1\xf2\x15\x91?\x1f\xedS\xd4K}\xf0\xd0*u\xa9q/\x14n)\x1e\x9a\xf0N\n\xde\xec\x00\xc5\xd5"\x1a2W\xf3\xe6\xe2\xe8\xa9V4a\x9c\xaa\xef\xf6\x87\xd8\x0e\xac\xc6\xc6\x87\tt\xb1\x9d\xad.\xb1{+\xe2\xdb~\x0cN\xfc(\xc2K\xbd\xd3|\x8d\x7f\xf3\xf9\xd1\xa7\x08\x96\xe7\xde\\\x133j\x8d(^.\x14\xcc\xb2\xc8k\xda_\xa1\x01\xc1\x04\'\x90s\xb2=Ka\xa3\xd3\xf9\x83t\x0c\x93$\xd8\xb3\xa0]\xe2\xf3\xd5nw7i\x1f\xa7\xa1\x87\n\x89\\\xd3\xf2\x11\xe3{&\x11\xf6\xb8\x15\xbb~\x9b\x070\x92\xf1\x1e\x055\xc8\xe9G\xe1\xc1_\xdb`\xf9\xda\xfa\x18\xb6n|\xe8e\xb1L\xad\\;\xfc\xed\xe8\xe1>\x84\xde\x0c\x9d\xca\x13\xe1p3{`v\x13\x85h8vo\x10\x0fk?Q\xa3\x8e4H\xb5\xa4{\x92R\x99\x87v\xa4\xf8\x1e\x18\xef\xcc!\xae\xcd\xc6\xb0\x17\x85\x9f<\xb1\xd7M\r\xcf\x927\xc1\xe7O\x19,\xcfnR\xd8\x1ag\xadi\xd9K\xe2\xe1h\x83\x81\xee)\xb1\x90\x0e\x94\xda\xba\xc2e\xbd[\xc3\xa1N\x92\xa982\xf41\xd2n\x0e\xce\xc3\x8f\xbd|\xad.\x1dH\xaaU\xf0 \xba\x19\x0b\x13\xe1\xec\xb4\x0b\xc5\x1bi\x8bT\xc6X_\xa5r\xaf9\xa7\x1b\xeaqov\xf3S\x08\xcb\xb3\x9b\x18\x85\x10$\xfe\xcav\xd0\xb2\x91\xf2A\xdeb\xba\x1e\xca\xd8\x0e\xd5/\xa1wx\x18c\x81\\\xcfU"\xeb2\xbef\xe0\x918\xaa\x9aZ\x18\x82-O\xae\xa7\xc4g1;\x02F\x94Q\xf9\xf39 wh)5\xb0\xee\x05I\x04\x8bv,\x997\xaf\xe5O!,_\xb1$\xbc\xc6\xa3\xd0+\r`\x9f\xc7p\x7f\xa1|\xac\x9d\x80d\x90\x95pFT\xcf\x07\xdd\xa2\x93\xe9\\MTt\x06\x1c\x97\x0c\xe0\xba\x158\xb2\x9b\xa5\xf3\xad\xe9\xb1\x11\xd2\x07.\x9e\x8b\x91M\x1c4\xba/\x103\\\xf4 \xe7\xd3\xd8r\xfa|~\xf3 \xff\x14\xc2\xf2\xec\xe6\xba\xe4)\x12~}\xb6;!u\x02k\x92Q\x8aH\xad\x91!34\xae|>G;\xb8\xa2$\xa8\xebREJ\x10\xb6\xaf\xf7\x95\xc2\xbb\xbc\xa1\x03\xbc\x88Z\xb2x:t\xb0\xdf\xc8\x04OQz\x8a\xf8\xa87\xd5\x9a\x85\xb5\xd6\x1e\xb3\xeb5]y3[\xfe\x10\xc2\xf2u\x90\xaf\xff)\x8a\xd1/\x07y\xd6@\xa2g\x9f\xf5\x03\x16\xc6\xbd\x0c\x99tv%\xc2(\xb6\x96\xdb\xfd\xb2\xc7\x05t$\x98v\xbd\xcd\xf3\xaa\x83\xc9\xb1\x844C#,\xbc\xc2\xa3.\xd2\xdc;j\x1d\x0c\xd6\x88\x1c\xfdN\xb9\xa3\xa6\xed\xf3H@\xa6\xe2\xfe\xa6\x1c\xf4)\x84\xe5\xeb\xa4}\xbe\x0b\xb9.\x82\x7f\xed\xe6A\x9bR\xf8\xc4\x86\x17$6\xc7\xbc\xc55\xbc\xcf\x0b\xf8\xa1\x80\x8d\x9e\n\x07\x06\x1a\x18\xc9\xab\x9b@\xf1\xf5\x1e$u\x8bP\x07\xca:8RYb\'[\xc1\x1c)\xc9\xaf\x03\xa4\xea\xcb\xb1\x9e\xef0\xa2\xaa\x81\x8f\xb7\xef\x92:\x1fBX\xbef\x93|z&\xf0+\xea\xa5\xf6\x17\xfa\x80\'0\xdfE"r\xaf{\x9e\xd6\xb9\xde\xad\x82\xde|(\xc7QC{\xa0\t]\xc0\x1b\xe1\xfd\x15\xd9\x89\xf4\xc5\xe7.h~0\xf9\xe3"[\xb9\x90*\x94\x83K{\x1f\xda\xe5\xa9b\x88\x87\x9bh\x9e\xb27\xbb\xf9)\x84\xe5\xeb}\x17j\xed)\xfd*\xc0\xed(\x12\x9d1\xfc\x0c?\x12\x9ea\xe9{v4\x0bq_{\xb7\xe6\x90\xaf\xa9\x9bw\xe9\xf0\xb8[\x0f\x82n\xe7\xf2-{\xd7r\x0f\xba\xc4\xd1q\x1aG+/\x00\xf3\xe6\x01\x05\xdcd;\xbb\xbb\x92\x0c;.\x80\xd0x\xe9\x9bA\xd6\xa7\x10\x96\xe7lR\xd0\x9a\xb5c\xaf\xe9\xec\xe0Ll>H\x9a\xda\x08\x0c\xb9\xb3\x93\xc7(JQ{\x81\xa0*\xb8\xdd\xcc\x8e\xefe\xf4z\xdaK\x086\x99N\x086\xc7\x01\xab\xe6C\xbb\xe4;\xbc\x93\xc5#}q=\x11\x86\xdcj=\xf5\x93;J\xdc\xa4\xe8\xec\xe5?\xea\xe6_\x1bay.\x96\xf5J\x87\x08\x8c~y\x1e(\xf4\'8\xcb\x8d|M\x90\xd6\xdb\x17Q\xb3\xf9\x0c\xe3\xe8\xbd@\xf50\xb0\xf7\x0b\xe3\x894p=\xb2PM \xa3\xc7\xcf\x8aU\xcfr\xdc(\xd5cM\x0bP\x83\xd3N\xc4%\x1e\xc3\x94]\xc3\xf0Nq\xd8\xea\x12S\xfb\xdf\xf3\xab\xff\x1da\xf9zwqCX\xfe\xedO\x83\xff\xf7 ,\x7f!\xdeb\x13C\xb6!\xfd\xfeC\xba!,\x1b\xc2\xb2!,\x1b\xc2\xb2!,\x1b\xc2\xb2!,\xdf\xb8\x9d\x1b\xc2\xb2!,\x1b\xc2\xb2!,\xdf\x197\xd9\x10\x96\ra\xd9\x10\x96\ra\xf9\xb6\xed\xdc\x10\x96\ra\xd9\x10\x96\ra\xf9\xce\xb8\xc9\x86\xb0l\x08\xcb\x86\xb0l\x08\xcb\xb7m\xe7\x86\xb0l\x08\xcb\x86\xb0l\x08\xcbw\xc6M6\x84eCX6\x84eCX\xbem;7\x84eCX6\x84eCX\xbe3n\xb2!,\x1b\xc2\xb2!,\x1b\xc2\xf2m\xdb\xb9!,\x1b\xc2\xb2!,\x1b\xc2\xf2\x9dq\x93\ra\xd9\x10\x96\xb7\x80\x08\xec- \x02\xfd\xa7&}k\x84\xe5\xa7w\xecW!,?\xbdc\xbf\nay\xafc\xff\x81U\xf2\xab\x10\x96\x9f\xbf\x147\x84\xe5\xe7 ,\xf4\xff\x0c\xfa\x9f#,0\xca#\x1c\xc6#0\x8c\x8b"\t\x0b\x9c(\xac\xed\xa6i\x98\x11h\x9a\xa2\x19R\x80\xb0\xf5\xa2dI^\x84H\x8c\xa5p\x8c\x12)\x11a`\x12Bi\x0c~V\xab\xfa1/ B\x04&\xc0\x10\x82@\x02\x89\xd0,\xcbr\x08\xcb?\x8b\xa8\xb28\xcaB\x0c*\xf0\x14Ab\x1c\x82 \x02\xff\xf4?`\x96y\x96\x8b\xc7X\x86\x13\x04\x9c\xe5\x7f*\xc2\xf2\xf7\xb2i\xbf\xfd\xc17\xfa1\xeco\x04\x89C\xf0\xb3\xae\xfb\x9f),\xdf\x9f\xb0\xf9\x03\x85em\n\xf4\xac\x07\x0cq\x10%0\x82HR\x0b\r\x8b\x04\x863\x94\x80\xd0\x08\xc9\xad\xd3\xb0\xfe\x1f\x9aAa\x9aC\xe9g\xb9u\x88C9\x9ex2\x1e,O\x88\xa2\xc0\xfc\xf6\x17WX\xfem3\xe3s\n\xcb\xb3R\xc6\xffUX\xa0?TX\xbe\xffF\xff\xa5\n\x0bJ\xff\xb0~\xffy\x82G\x9bt\xd0\x8eD\x9b0\xde\x9f\x08\xbf\xede\x94\xbf\\\x80N\xd7\xd8pN\x1b\xbd\x91\xd4\x81%Ji\xe7\x14\x99\r>n\xbb\xa4G\xc8\xb1\xf5\x8d$\xdf\x07\xa5\xc3\xc6\xd4\x15\x16J\x81\x91\xa4\xf1*\x8c1X\xbfY`\xf2S\x08\xcb\xf3V\xa0(\x8c^_\xe4\xa5\xce\x8b\x01\xab\x07\x7f\x00K\xd4\x96<\xf3^\x86\xf49\xd2\xaav\xaf\x1aM\x0c\xb4\xc1\xe2g\x8e\x9c\xe2{7\xa9\xbd\xac\x17\xe6\xb7*\xa2\xf9)\x84\xe59\x8a\x10NRO\x05\xe4_G1\xe6\x0b=U\x12\x02\xe2\xd5\x19\x08v\x03|\x9d\x84.\xf7P\x00\xb4X1\xc7.\x8d\xbc\xa7\xe5\x93oy\x8e|%\x8d\xd8\n\xd3p\x87b\x9a\x82\xc1~U\x93QR\xabE\x80\xb4\xd8\x99>\xd4F\x13\xa1\x07X>\x96\xef\x96\t\xfe\x10\xc2\xf2\xec\xe6z\xe5\xd0\x10\x86\xbdn}_\xb5o\x172E\xdb]MN\x07\xfc\x02\x049\xa2M\xf0\xfaz\r\x87&\x04\x85N`|@\rkH\xe3\x9d\xa6\xec%\xba-\xd5k\x00\xfb\x85F\xe1\xae\x99\x14\xf5\x8d\xccoE<\xf8\xf7\xe35\x0e4f\xb2\xcd7\x0b\xb2\x7fJay\x9e\xe20\xfe\xac\x16J\xbc\x8aa\x05\x92\x16\x11{\xbd\xc5j\xb1\xc3;\xb2\x10\x81\xe2*\xc8V\x13\xa56\xd8g\xfez\xccTn\xedD}\xe3\x15\xeb1f?\x16\xaf\xd7\x8c\x87\xc0X\xec0A\x8eyYX\xfcj\xd6y\xbf \xb2P\x87<\x11;\xe1\x9bp\xc7\xa7\x14\x96\xaf\xd9\\\x93\xcc5Q{)\xd4\x17s\xa7\x05j\xfd;6\x92\xdc\xee~\xba\xf5\x80\x06\xb1y*\xdc\xa32\xa5\x8e\xbb\xc9\x16\xd5\xfc \x95\x0b\x08\xd7\x9e6\x05q\xab\t\xeb$\xa6t\xb0O\xd1\xe8\x11,\xc7}\xd0\x1d\xa1\x91\x9dG\xe6lt\x89xy\xd8\xec\x9b\'\xdc\xa7\x14\x96\xaf\x13n\x9dJ\xec\xf5JF\x0f\x80x\xdc\xa7n\x14/m\x92\x84g\xd8\xde_\r34G=\xba\xef\xe8y<\xaa\xfb\xe4\xd6\xf0\xb0dt\xfd\xbc6\xef|-\x9f\x90\xd2\x0cU\xfd\x98y\\p\xaaLc\x12\x0f\xa5\xba?\xa96P\x06\xd5Q6\xde\xdc\x9a\x9fRX\x9ek\x16\xa2\xb15\x06|\xad\xba(\x07\xfey\x00\xedu\xd5q\xe29\xc5|\xfd\xca\x02\x9cza\xeb\xa1\x88\xf6\xcb\xe2A\x84V\x9a \xdf/(\xd6\xf8`\x1fS\x88\x95\t\xc4a\x19X%Q<\x90\xbb\x17\xb3R\xe4\xdd\x89\xbbgFp\xc9\x87Z\t\x83\x1f\x15~\xfdk+,\xcf\xb5B\x91O\xb8\x90~\xb9\rc,\xca `\x04qS\xb9\x9b\x94"\x98Hi\xd9\xfcn\xe4\xfa\x14\xe04\xe7\xc7p\xc1\xf1\xc8\xdd\x13\xe2\xdd\x8f\xbc\x14>K\x06\x9f0L\x98\xb4\x88cuw\xde\xa1\xe2\xcb,B\x15\xe4`\xf9\x8fj>\xff\xb5\x15\x96\xe7(\xe2\xf8\x9a:`\xd0KIi\x15I\xa6\xdd!\xb1\xe6\x0e\xeb\x93\xdaC\x99T\xd4\x8e\xc1\x05\x07t\x84\x98[2\xb3\xea\xbb?\xed\xf2\x11\xbe4*`\xdd\xb4\xb8\xa2\xf5\xc6+\x01f4\x16\xa0>\x97\xb0A\x1d`\xcb\x92F0\xa7:\x9a\xe9n\xcb\r\xfaQ\x84\xf8\x93\x15\x96g\xbc\x0f\xaf\xaf\xf1|h\xf9\xb2\xf5c\xfb\xce\\\x1a-\x98\x18\xa0\x80\t\x0c\x848\xe2>=\x94P\x05\xfd&`\x06V\x1f\xa7\xac.R-\x99R\x120\x8e\x89\xd3\xd5AI\xe1\x1a\xad\xb9\xea}x<\x82\x01^s\xda$\xba\xd5\xfc\x19\xdb\xc1P\xa4\xc9o\x16\x08\xff\x94\xc2\xf2\xd5M\x18F\xd6^\xbet3jN%\xb6\xbb\xc4\x8dnu\x17\x89\xb3A\x9fE\xe62\x17\x0f\xaeV\xc4\x96/\xf82k\xea\xb8\x8b\x857N\xb7n\xbb\x99\x12n\x81\x1aG{\xa4\xc5\x85\x8b\x12tA\xa9SC\x1d\xdd\xfa\xa3\xd6\xd4tA\xd8Pv|\x13\x9b\xf9\x94\xc2\xf2u\xe9c\x04\x85\x90\xaf\xf6\x9a\x80SD\x17\re\x06\xd7\xfc\x95\xa3&\xb7\x0f\x84\xc8\x80\x15Z!\xd4d\x8a\x08\xf2\xae\x9d\x08\xdc\x0f\xe6\x03r\xd5\x918S\xee\x92\x859BxP\xba\x89\xcb\xb0\x1cpO\xb5\x15=\xa6Vf\x97\x90\x18\xe8\x05\xe2\xde\x85\xf4>\xa5\xb0<\xf7&\x81#\x08\x8e\xbe\xa2\xaftd1\xb2\xceb=\xb2\x03\xf5\x16\xbd\x0f\x8b\x066{\xcf\x02\xbc9\xa4\xa6\xa0)\x06\xa3\xe9b\x16\xc6\xe3A\x13\x81\xb3\x1a2\x90k\x85\x1eS\xca~p\x9b\xe9q\x99\xa5\x12+o,\x82\xe8\x0f\x1a\xf4\x91"F\xf0w\xbd\xc0\x0f),_\x91*D\xe0$\xfc\x1a\x90G\xa7\xb64\xa2\xab|\ned\xd9\xdd\xd1[\xa9J\xde\xfeXOw\x89\xdc5\xb7\xdbR\x99\xc8I\xd8E\x83\xe3H\xc3\r\x03\xcd\xe2,\xec\\\x9d\xc8)\xb0\x87\x8ft\xa2\x03j]s-\xd9\xf4X\xe3\x8e\xb9I*\xd8\xbby\xc7\xa7\x14\x96\xaf\xbc\x03ZS\x98?\xd0m\xd7%W\xb7\x97@\xea[9\x97\x8f\xbd\xef]\xeb47\xe7\xdc\x0eA\x0c\x009\xad\xbdaG\x12j%\xff\xec\xcd\n\x97\xc2\xd1\xe4\x01%Z\xe5\x05y=\xc3Nw\xdd\xd1\xda\xd2\xc9v\x9b\xf5\x8f\xb6e/w\xc8\x0f\xdf\xac(\xfd)\x84\xe5kk\xc2\xd0\x1ab\xc1/\xf7Ir\x9d%\xc2\t--\xe5\x91\xdb~\xf2\x1aK\xb7\x04\x1f\xbaa\xe4#\xb1\xf5\xa8#Nl3\x12\x8f@H*\x01\xb5\x8b\x91\x8d0w\x06[\x8b\xe3\x9d\x1b\xa5\xc3B\x8f\xa6\x92%j\x19\xcfR\xc9\xde\x80c\xd4f\xebw\xb9\xab\x0f!,\xcf\xc9D\x10\x1aZ{\xf9\x92w\x8czy\xd9-\x1dW\xae\xcb3\x1b\\\xd5\x8d\x00\xc6k:VQ\x92\xe6\xb1\x0c=\x9e$c\x0bk\x920\xe9\xb9\x11e\xa4?/\x07\xfbxu\x03\x84\xb9Lm9)\xfe\x1a\x9e\xe9\x80u\x9e;\xa6:\xfb*w3\xf17\x8b\xda\x7f\na\xf9=\x89\x84\xbe\x9e\xec\xbe\xe0e\xdd\x92p\xbbs\x823\xac\xb7\xf7\xd8}\xd59\'\x1d\xc6\xc9\x93\x14\xa1\x99S\x0e(\x08\xb0YQ#\x90\x11\xea\x9a`\x9d\xe5d\x0eSz\xb7PyE\x9e\xd1\xf9v\x8ar\xca3\xe8\xd0\x9e\xea\x12rp\tp\x81\xe6GI\xe4_\x1ba\xf9Z,kPKA\xaf\xc5\xe4\xe9\x01^\x13n\x06\xb7\xa6\x08i\x9d\xfbe.\xe83 3\x81DfjP\'\xa09\x86\xbd}\xf4\xa4S\xce\xa5zb\x0c\xf5\xbbo]}\x08ay\xce&\x86\xaf\x01\x0c\xf6\x07\xcf\xc9\x84\xe9\x00\xb8\x07Ki\xcf\x9a\xa1_\x1a\x11\x06\x06\x02\xbd1\x8f\x12\xa2\x96t\xa7\xd6\xb0/\xeco\xa5RHrn\xda\x82eG&\x9b\x89bS\x80Pw\x92o\x91\xaa\x86\'J\x82U\xfe\xdeO\xe5\x92C1\xec \xef\x92:\x1fBX~O\x0c\xa85\x86!_/\x94\xfb\xb2\xd7\x9csr\xdb\xbb5\xb97\x8d\x199\x0f\xb2(:\xac\xa7\x11\xd6:\x89FRZ\xa4\x89UK\xec\xd4\xa7\xe8\x04\xf8}QH\xf0\x98\x11\xfb\xf6\x92\x1b\x85\xb7\xa6T\xd7a6l\xf3\xe4\xddn3r\xeb\x96ly3\x96\xfc\x14\xc2\xf2<\x82\x9e\xefD\xae\x17\xcaK7\xa1\xd1\xae\xf5\xe1\xce\x8d5\xb0\x9b\xf4\xe8z\x85\xd5J\x1d\xf5\xecq\x91\x1a\xb4\xb8\xfb\xe2\xf9\x01\xad\x87\xebD-\x00\xe1\xef\x8e)\xaf]]F6\xe2i6\xc6\x1b\x1b\x07Z\xb9\xe0#\x14\x02\xaa{\xb5bkWQ\xc5\xd9x31\xf8\x14\xc2\xf2\xec&B\xa1\x7f\xf4\x1ck\xaf\x898`eP\xdf\xdd\xfc\x8bL\xb1\xfb\xdd}\x8f\x1f07$\xe1\xab[\x15\xccp\x9d\xe6j1\x16\x14\x05\xfc\xa3HJ\x94\x145\xed\x8eU\x99\x0be5\xce\xce\xbau\x1d\xcb\t\xec>\xd8\x1f\x16#B\x17PJ\x8b\xffN\x83\xe59\x888\xb6.\xae\xf5n\xfe\xd7QtM\x9fD\xc5C]D^|\x8d\x07\xeb1a\x0f\x91\x1b\\Eo\xf3\xa8g\\t4\x16\xdc\\\x83\xff\x93\x8dZ\xd9h\xe2\x02|\x9b$5=\xe6\x16\x8a\x9ed]k\t\xd0\xe7\xb0\x9b\xb4\x06&\xd0\x10\n\xd1\x11q\x7f\xf7g\xffw\x83\xe5+-\xdc\x0c\x96\x7f\xfb\xc3\xe0\x9b\xc1\xf2K*\x0cm`\xc86\xa4\xdf}H7\x83e3X6\x83e3X6\x83e3X6\x83\xe5\x1b\xb7s3X6\x83e3X6\x83\xe5;\xdb&\x9b\xc1\xb2\x19,\x9b\xc1\xb2\x19,\xdf\xb6\x9d\x9b\xc1\xb2\x19,\x9b\xc1\xb2\x19,\xdf\xd96\xd9\x0c\x96\xcd`\xd9\x0c\x96\xcd`\xf9\xb6\xed\xdc\x0c\x96\xcd`\xd9\x0c\x96\xcd`\xf9\xce\xb6\xc9f\xb0l\x06\xcbf\xb0l\x06\xcb\xb7m\xe7f\xb0l\x06\xcbf\xb0l\x06\xcbw\xb6M6\x83e3X6\x83e3X\xbem;7\x83e3X6\x83e3X\xbe\xb3m\xb2\x19,\x9b\xc1\xf2\x96\x0f\x81\xbf\xe5C`\xff\xd4\xa4om\xb0\xfc\xf4\x8e\xfd*\x83\xe5\xa7w\xecW\x19,\xefu\xec?\xa0J~\x95\xc1\xf2\xf3\x97\xe2f\xb0\xfc\x14\x83\x05\xfd\x87\x89\xfcs\x83\x05g \x14\xc2 J\x10a\x08\x12\xb8\xf5O\x9c\x08\xa3$A\x8b8\x86?\x8b\xf9AkC(\x08\x11x\x1a\xc7D\x84\x15a\x81\x11\t\x88x\xd6^\x83Q\xfe\xf9\x15\xf3\x1f\xe3\x02,M\xd2,F\x88\x10O\xa3<\x83\xad\xff+ \x18\x03\x11,\xcc\xc34\x8a\xd1\x0cG\xf14B\xb1$M2$Ms,\xcb\x0b4\r#\x1c\xc3\xe0\x02N\xc1?\xd3`A\xff\xfe\xb5\xfe\xdf\xfe\xe0\x1b\xfd8\xfc7\x14")\x8c\xa4P\xe4\xcf\x0c\x96\xef/\xd8\xfc\x81\xc1\xc2\xe3\x10L\x0b\x04\x8b\x91\x90@\x0b\x98\x88"8\x89\xa1"\x87\xaf\x93\x03\x91\x1c\x8f\xf2\x10I3\x0c\xcf\xf1\x08!\xe0\xfc:R\x1b\xb7\xaboCB}\xba9\x91\x91\xdc\x81\xfd\x9d\x9da\xaf&\xdd\x07T\x89\x8b\x01\xedO\xdc\xb7*\xa2\xf9)\x84\xe59\x8a\xf0zZ\xc0\x10\xfdR\xe0i\xef;{\x8a\xe2\x995?\x81\x0b\x85cE\xf4b_\x0fKx\xf3O7\xba\x88\x10#\x90%\xb8\xbb\x82\xbb5\xdft\x04\xf6\xca$<\xdc\xd3W\xa4O\'\xd9u\xabL\xd1F\x1c\xb1\xa3\xbd\x8a\x19\xc2~\x84\xf8A!\xc2wK[\x7f\x08ayv\x13\xa5\xd7\xf3\x8d~\xady&\xcf\xa0]:\xce\xfe\x1a\x9e\xd8N-\x15\x11\xc1\xb0\xd8^\x16\xca\x95\x11\xddBo\xd0\x18\x89\x1d\xc2Tf\xef3\r\x8d\x1f\x10\x81"\x06\x05uv"\xad\x1a\x1e\x9c\x84sF\x07]\xc0Y{\x00w\xd5\x8c\x96\x88\x87\xf7n\x05\xbb\x0f!,k7\xd7-\x01\x11\xc8\xba+^j\x85\xc6\xf4>\xb7\xdc9\xf7L6\x8a\x01\t\xd2\x10\xee\x84\x0fWlY\xef\xe6Lq\x87G\x11\x83\xccm\xb2\xdc>\x13\xca\xa7_\xb0\xdeV\x18_\x10\x96\x83^\x18\xa6)\xed\xc2\xba\xcc}\xb3\xfeI\xe9T\x8b\xd9Y\xc5\xf8f=\xf6O!,\xcfS\xfcY\n\x19\xc2\xb0\x97E[gvw\xe1\xa6\xc7\x1aR\x19"\xa8j\x03\x1du\xb2\xc9_@"\x88J\x9d\xaf\x0e\x19\xb3`\x07\xd9\x8f`?\xc5\x89\xadi\xf2\x1d\xfaE\x08\xcb\xbf\xa6j\xff\xd8\xcdN%\xaexMY0Y&\xd2\x8dwZ\x0c\xdd\xc5D\xa2\xf6`\xed\xe1;\x9b9]y32IAU0%\xb2\xbc3\xc0\xe1\xb5\xc3\x88X\x96\x01Hx1\x0e\xe2\xdc]\x1ct\x06nWMF\x87\x87+\x0fZ\xfdn5\xe4\x0f),\xcfEK\x93\xc8\x1a\x0e\x93/\xb3\xe9\x1d\xef\xfe)\xa9\x9a\x87rC\x0e\xc1>P\x1d/`se\n\x97\x85Q*\xd9\x19%O\xd7b\x02\xd2.j\x95\xa97\xd9$\x02P\xbe\xc1P\x11r4\xe3\x8ar\x15\xc5m\xd3\x9cg\x0f{\x94,\x16\x95BK\x91\xdf\xab\x1a\xf2\xa7\x14\x96\xe7bY\x8f}\x98\xa4_\xcb\x04\x1fb\x14\xb8\xa1\x88\xe2I\x1e\x1d\xf9\xd6%;\xc5\x9cA*\xc0X\xe5\xb6}\xef2\x02\xbbF\xeanp\x8e&\x17\r\xb2\x1a1\xad%F&T\x0b$\xbe\xd3\xef\x85\xa1t\x17\xa0\xec\x1c\x8f\x86\xa3\x18kI\xfd\xee\xc6\xf3\x9b\x8b\xe5S\n\xcb\xd7bA\xd6W\xc3^\x0b\xcbv\xbbS\x8b76\xba\xef\x8e\t\x91\x86x[,\xd6\x9cA`\xde\xd8^\\\x027\xbe\xc8Nd\x0f\x89\xa6c\xc8g\xa3=D\xfdy\xb8\xb02l\xcfs\x1eW\x17o9\x05\x15\xe5#\xe0i\xa2vp\x81\x1a\xa8\xa3\xc5\xfc\x9bq\xea\x87\x14\x96\xdfc\xb8u2\xa1\xd7\xc2\xb2\xe6\xb5W\x07.\xd0gmVj\x1c;\x05\x00\xdb\x1c\x84\xc2\x08\xbb\x90O\x8b\xc1\x9c\xac\x9b\x08\x87\xa7nAR2\x96$\x95}\xd8\x12\xebN\x81v\x1dd\xd0P\x81\xd3!\x83[\xf5\x14^\x94\x193{\xe3a\xe0\xba\xf6\xa3k\xf9\xaf\xad\xb0<\x17\xcb\x13\xfe\xc0\x90\xd7\xeb0_\n\xf7q8\xd8tj\xf3|{\x84\x01P\x1b\x0c\x97\xf6\x813\x1e\nG$k\xe1\xf2\xeas\x8ez?H;\xb98\x8dN\xe5)\xdc\x14qY\x1a7q%#t\xc0\x16d\xe3{(\xce\xc7\xda\xc5W\x1e\xb2\xa5\xbe[S\xfaC\n\xcb\xf3:D\xd7\xac\x01\xa7\xb0\x97\x80\xdfx\xc0rX\x90\xc2=-\xd5>\xeaj\x19pz\x87*\xca\xb8}\x88\xdc\xe3\xea6W)\xb1\xf7Zv\x87s`\x9a\xdc|g\xf0\xe78\x1b\xa0\x80$N\x87\x9d\x13\xdfR\x90\x875d\xe7&g\xd4Wx\xd4!{\xfdG\x8c\xd5OVX\xbe\xba\x89\xe2\xeb\x9dO\xbf\\\x87\xac.r;\xe6\xf6\xe0\x18\x12V\x8d\xc8\xc9vD\xa5_\\:w\x9b\xb6\x88\xd1\xd0\xf2M\xc5:\xb7\x0c\'\xcf\x18%J\xad\xb0<\x98c\x8b;\xe2\x81\xceN\x9eX\x08\xde\xee4\x17Ds\xc8\x1e\xfbFK\xd55\x01P\xdf\xe5I>\xa4\xb0|\xc5p\x14\x8c\xa0\x7fp\xc29\x0e\xd0\x057\xb3C[\xf5`\\\x90H!\xcc\xe9:J{\xb0x\xec\xfa\xc4\x18N.\xa6\x16\xb8}\x1f\xc9\x07gxNm\x9c\xba#\xfa /\xa5\xa7\x18W/I\xee\xd3#\xa7\x15v\x90 \x0e2\x82S\xe9"\xe3\xbb\xe5y?\xa5\xb0|\x9dp\x14B\xadA\xef\xcb}\xc5\xe7\xc4\x8e\x1da\x16\xab\xa9.X.5d\x8b-#\xc0\xf9H\x1f\xaf\xe4\x816/\xa6r\x133\x10\xebq!\xbb\xe5\xe4\xa2\xe9\x872I\x13\x13\xce)\xbd\xc1\xf0p\xbd\xe8\xd1\xf8\n\x1bG\x89\xd0\x00\xae\xef\xba\x01\x98\xde,\x9d\xfd)\x85\xe5\xd9\xcdu\x0bC8\x89\xbf\x88\x164-\x00\xb0wK\x92\x1b\xce\xa6vU4\xca\xed\x12\xc7\xe6\xb1"9\x92\xb2\xa5\xb0-`l\x99\xc3\x9b{\xe5\x8f$\x14[\xd0\x9c\xe9\tK\xcd#\xd26\xbb:\xed\x12o\xff`*7\x1f\x97B\xcd\xaf\x82\xea\xce\x04\xf4\xaeO\xf2!\x85\xe5\xb9h\x89uw\xc2\xc8k\xf5~\xaa\x18Uk\x96E\xfb\x16&\xf9 q\xbb\\\xc7\x95\xaa\x89C\x8f.\x1d\xf0\xd6\xc3M\x05\xb0}\x10\xc8\xb5H\xd6\x8e\x9da\x85\\\x00\xad\xc5\xb0<\xb7>I!\x18\n\xd3/[\xdf\x84\xd2\xe1>\xf7\xed\xdc\x995\xc7\xb19\xaev7\xa4\xb8\xf4uJ\xa0U\x82\xf3\xd3\x01-\xfdK$\xcd\xde\xfd.4{Z\xbe\xd5\xc3\xdcp\xf7\xbe\x8a\x1e\xf6\xc1#\x01\x80<\xcce\xbd\x0c\xed\xbd\xe5G\xa2\x9d\x1a\xbb}\xf3\x01\xf9\xa7\x18\x96\xaf\xa7\xaa8Lb$\xfcr-\xdf`I\xb6\xf8\x8a\xa7\xce\x97#\x03!l*\xf9ze\x95\x99\x91G\x96\xeb!\xeby\xb5\xc0\x0eI\xfb\x0b\xc6\x1b04\xd6\xbal\xca\xe7\xe0"\x0f\xf1\xc9\xd5\xa4cU\xda\x87\x98\xdf\xc7K\x02\xc8G^\xaf9\x9ee\xa171\xcfO1,_\xd9\xf2\xf3}z\xfa\xf5\xed\x0e\x82\x8e\x00\xcaQ\xda\x08\xbe\x03c;\xc8:o\x18\xe5\t9\x1f\xa6h\xc94\xbb\xba\xa3\xc7\x1bP$\x9a\xd9U\xc4\xce\x1em\xf5~\'\xebb?\xb3\xa3z\x80D\x9a\xb9\xed9\xc8\x9b\xdc\xaa>\x8d\x02$D\x168\xed~(x\xffd\x86\xe5+\xc8"\x11\x82\xa4_E?\xa8\x00\xfa\xc7\xe9\xe0\xec\x891[\xc0=\x89\x89%\xcd\xfb\x070t8\xef.\xe6\x14u\x90\xd5\xdd\x84?\xb8\xca\xafP2\x00(\xf6\xe2\t\r\\\xec\x1f.\x8c\xed&:B\xe0\xe81y\x04\x90\xb6\xd4h\xa0\xb7\xc6,\xc4w\xdf\xd5\xf9\x10\xc3\xf2\xf5\xae\x0eM\x93\xc8\xba\x91_\xee+\xd7e\xe3\xd3Hh\xd2\xa5\xb6)\x9bOA\xf2\xaa\xaa\x88\xe7\xdb^=\x98\xe1RW\xed\x08\xb0Dzn\xd6T!\x81\xc2\xfeF\x98\x96~\x98\xce\x1a\xa8\x03\xcc\xd2\xeam\xc5\x9d\x0caau\x8fdx\xbe\x1e|\xe2\xfa\xe6\xa2\xfd\x14\xc3\xf2u\x04\x11\xeb\xb2 _&\x13Dr\xf7\xc8Sk\xba\xbc\xab\x1e\xa1}\xab\x94\x1b\x95\r0F2\xe5\x83h\xb0R\xdd\xed\r\xe0\xba\x9c\x1fNP\xc5&q\x16\x01\xb4\x19Z0\xa6\xcb\xb3\xcf\xae\xaf9b\xb1\xd3S7gm\x10}\xac\xb5!\xd5\xd7\xc0\xe1\xcd\xc9\xfc\x90\xc2\xf2\x151\xaf\xb9\x13\x8d\xbf\x86\x92vY\x95x\xc8\xb7J/\x8a$\x11C\x10\xc8gF&I\xc7{-\xa0\xf7\xdc\xa2\xa0\x1a4\xf2f\x9f{\xa9\x1d)Ws7\xb1\x1c\x92\xe0IC<\x1e\x0c\xa5=\xe0t(\xdd%\x1dP\xbe\x8dm\xf0XC\n\xea\xfd\xe8}\x9d\x9f\xac\xb0<\xbbI\xadg\x10DQ/\'Pf\xee\x14jwX\x8e\xcaz\x0b\xe7\x1a\xd1\xeduk\xb9\xed\xc9\xd3\xf1\x8a\xf3L8\xeeM\xebz\x8dO9\x94\xb9\x1d\x95\xd9\'\xe5\x92\xec=(\xcev\xa9\x04\x8a\x13:\x86g\xa4\xedB\xc7\xe9\xb88\xa0\xa1\xc7\xeeF\xe9\xfb7\xaf\xcdO),_\xb3\x89`kl\xf6zk\xdeQ|`i\x85;\x85&_\xc3zQ\x9e\xa2\xf8\x9625\xc1\xa8\xd12\x9f\x12\xd5wS\xbc\xcd\x85\x11\x07!\x07F\xae\xaa\x1c%\xc6\x83(!\xd6!9\xe7|\xf3\xcb3\x94\\\x8dT\xbf\xbaI"\xd4VW\x9e~\x94\xe5\xfd\xb5\x15\x96\xe7\xf9\x86 (\xb4\x0e\xff\xcbZ\xb9^\x15O9\x87X29&\x8c6m\x13%\xd6\x95\xe8\x1dzM\x94\xa8;K\x8a\\\'\x05]\xc7\x0f9ptN\xe1\x12\x9aB\x05b{\xbb<\xc6\xfa!r\xf0\x07`\x034\xbdSO\x1d\x8c\xefh\x17\x86J\xfb\xfe\xfb\xf9\xf6\xbf*,\xbf\x7f\xcelSX\xfe\xed\x8f\x83\xff\xf7(,\x7f!\x8ca\xf3-6\x85eSX\xfe\x7f\\\xa5\x9b\xc2\xb2),\x9b\xc2\xb2),\x9b\xc2\xb2),\x9b\xc2\xf2}\xdb\xb9),\x9b\xc2\xb2),\x9b\xc2\xf2\x9du\x93Ma\xd9\x14\x96Ma\xd9\x14\x96o\xdb\xceMa\xd9\x14\x96Ma\xd9\x14\x96\xef\xac\x9bl\n\xcb\xa6\xb0l\n\xcb\xa6\xb0|\xdbvn\n\xcb\xa6\xb0l\n\xcb\xa6\xb0|g\xdddSX6\x85eSX6\x85\xe5\xdb\xb6sSX6\x85eSX6\x85\xe5;\xeb&\x9b\xc2\xb2),\x9b\xc2\xb2),\xdf\xb6\x9d\x9b\xc2\xb2),\x9b\xc2\xb2),\xdfY7\xf9\xafVX\x8c\xf9-!\x02\xff\xa7&}g\x85\xe5\xe7w\xec\x17),?\xbfc\xbfHay\xb3c\xff\x01V\xf2\x8b\x14\x96\xff\x07KqSX~\x8e\xc2\x02\xff\xcf\xa0\xff\xb9\xc2\xc2\x904D@\x0c\xcc\xd0<\xfb\x05\xb2\xc0\x04\x05s\x04&\x88\x14L\xe30\xc4\x8a\x08\xf5490\x16\xe7p\x1eG\tQ\xa4X\xf6Y\x00\x95\x80`T\xfcs^\x80\x81\x90\xe7w\xeb1\x9c\x15\x9e\x05\xcfI\x94\xa4x\x18ci\x16\xe5qQ\xa0\xe1\xafBz\xb0@\t,K!\x10/\x90\x18\xca\x0b\x0c\xbf\xb6\x84x\x1a-\xf8OUX\xfe^\x1b\xe1\xb7?\xfaF?\xf97\x92\xc6h\x0c\'\xc8?CX\xbe?a\xf3\x07\x08\x8b@\xf0\xdc\xbahP\x0e\x158\x94\xe2`\x9e\xc7\x10\x18a0\x94%1\xe8Y\xaa\xffY\xa4\x87d\x04\x08f)R\xc4\xd0g\x85\x1a\xe4Y\xb0\xe8\xe9\x91\xc0\xf8\xb3\xa8\xcd?",\xb8\xb0v\x15\x83\x18\x92g\xc4u\x08h\x06\xe5\xd7\xc6a,\x8b\xac\xaf\x82\x93kSQ\n\xc5E\x0c\xe3\xd6i\xe5q\x86\xe4\xd6\xd5\xcd\x90\xc2\xfa\xefXj}\xd9\xdf\xfea\x8do\x08\xcb\xfaj"\x06s\xfc\xb3\x16\xc5\xb3\xc2\xb0\x80\x10\xd4\xda2\x9e}.1\xecY\xa2b\x1dh\x08\x87E\x8a^_F\xc0H\x91\xa2\xe8\xe7\xe43\x04\x8bQ\x02E\xa3\xf4o\x7fq\x84\xe5\xdf\x96A^\x10\x96\xdf\xb8\x0ec\xb9nd\xb9\xd2\xb8\x05\x08\xfd\x88=\xfa\xa1\xb7&\x14x\xd7.n\x92)\xad\xc42\xf4\xed\xc2pp3AR!\xf4\xeeu\x88\x88Ph\xab\x10W^o\t\x02\xdf\x0c\xaf\x83\x03\xb4\x83C\xbe\x9eB\x9f\x99\xcd\xd6\x9a#\xa4h#9\xbc\x9a\x95\xf5\x88\xa4\x00O}\x135\xf9\xe3\x92\xf2\xf5\x9a\x9d\x19\xb0P\xa9_?\xfb\x0c\xa9c\x94\xad\x93J\xec\xd3\xe7\xd73<\x0b\x8f\x9b\xe3l\xac\x11\x97\xf1\xf5\xe9\xc2\x02\x89\x0f\x01d \xc5\xf39\xee#}Xk:b\xf6\x99T`qq\xb1\xb9\xd3\x11\xfe\xba"\xef\x8ab\xf2\x0cn\x1e\x02\xd4\xa8,\'b:\x8b;\x07\x9a\xbc\xd0\xd5\xefW\xa81\x05\xa8\x8a\'\xb2}[\xfbW\x07(\xb3\x98\x87pX\xfbe\xa5\x9e\xaa\x87\xfe\xd9Q\xe6K\xc1t\xa7\xa3\x9c\x1e\xf1\xea\xac\xe1\x18\x83\x8e\x07\xf1A\xf7k\xf4\xda5rv\t\x98{|\xc8\x14\xe4T\x9c\xd8A\x92\x85\x93\xa51\xa7\x13+1\x8a#2\x97\xdd?\xb4E4\x0e\xa7\x87\xc1\x1b\x16\x15\xba\xbatf\xd5{\xa3\x98\xd5\xb9\xab\xb2i7x\x99\xdcF\x8f\xd0e\x95\xf0\xa2\x1do\xdc\x9a\xb2F%K\x8a@\xcc\x9a\xc7\xa3ul\x027\x19<\x8fI\x19\xe3\x11\xc9,^\x02Y\xda\xe2\x01\x8c\xb8\x84\x12\x99\xbb\x02gA\x0e\x15\xb3\xec\xb7\xff\x94\xd0\xf9*s\xf2\xa7\x84\xce\xf7?\xa6\x7f)\xa1\x83#?\xaec\xaf=X\xe9\x02\xc3\xc7\xc8\xeb\x95\x14\x12\x83\xcc\xe3\xfd\x9d:591\xa4f\x1de\x10\x0f\xdc\x18H\x06\xe2\xf8t\xd3\xb9C\x96s\x03\xc5\xefv\x9e*w\x0b\x96\xdfS\x93\xc7kv\xe7()\x8aAA\xde9\xf3%\xc2\xde,\xb6\xf61B\x87\xfc\x1b\x05\xa3\x08F\x10\xd8k7}\xbda\x8a\x1d0\x0f\x08tb\xb8(;;>Y\xf2\xea\x1d\xbf\xeeb\x1b\xb3\xe6y\xb1vc\x9fA^\xed\x82\x13?\xf4\xa8\xba\xdb\xd7.R\xc1\xd3!\x1aNP\x7f\xb3\x97\x8e\xcd\x8e\xa5\x9e\x9d\x94\x10\x1d\xa5{.\xbf\x89/|\x8c\xd0Y\xbb\xb9\x86.\xe8K=\'\x99\x9f\xb80\xd5\xee\xa5\x10\x0c\xa6\xce:\x85pr\'m\xda\xa7\x8fp\x97\xe7\x13}\xcc1\xec\xb1\x9f(\xef\x0c\x8e8T\xaf\'zY\\\xaa\xd9\xbe\xb6\x97\xecV\x8d\xd5\x9e\xcc\t\x95\x05\xf7\xf6\te\xf9)q\xb5)W\xdf,(\xf71B\xe79\x95\xeb}\x08!\xe4K\xc1\xa5=|\xe0\x01^@\xcb=\x9d;\xb8zW\xb1\xbe\xe9\xa0\xfc\xa8\x1d\xb5\x0c\xb3\xdc\x1b\xb4P&b1\xc5\xe1\x90K\xc6\xb4O.\x8aWf\xa1Qr\xb8J\x1a\xee\xcd\x99\x17`Z\x0e\x96\xe5\xf6\x8dn\xcf\xd1L\x81b\xf7\xab\x08\x9dg\x14J\xac\x01\xd8k\t\xe6q"`\xab\x1a\x83\x024\xce;(\x0f\xf7s\xde\xa8QC\x18=\xa0\x0e\xa7\xc2\xc6\x10h=\xb3on\xc9\x02\xd9!\x14\xcf\xed<\\\xf1J\xe4\x0f\xa1\x90\xf1\x01ms\xa3f\xa1 \xa9\x8dCN\\\xa3Q\xf6Q\xe6\xbb\x15\xe7\xfa\x94\xa0\xf3\\+\xd0\xfa\x17\xf2Z\xaf\xfbl\x03v{\xce!l7\xe06e%\xc1\x98\xc8\xd9C\x89\xd9d?f\x8d\\\xa2WF:\x03\x8eF\\\xd2\xcb\x10\xf7\x92\x1a\xb5\xa1\xe7\xc7c\xa5.\x91sm\xcb(\x8f\xb4C,\xa4\x0c\xa2*<\x1e\xad\xabl\xcf\xbdY}\xfdc\x82\xce\xdaMd\xbd\xb0\xe8W\'h=$#\x02\xbc\x10\xa0\xd4=\xee\x81\xdb\x0cn\x9aV\xea^srl=\x0bB\xcay\xf0\xbb\xb3"D\x18Z\x9cj\x1ag\x0f7\xf8\xd83(hW\x19\x80\xd0&\xcf7\xf4\x92-\xfb\xbd\x93\xee\'Xc\x1fw"\x04\xdf,\t\xfc1@\xe7\xb9#\xd6-A\xa2\xafuA\x07#\x1e\x02\xa1\xee\xf2\xc3\xd9\xa6\x81$\xf0g\x97\xbd\xee\xcf\xe05\x8e.\xda\x9eK\xa2\xc7>\xb2\xe4\xf5\xa4.\x81\xc4\x88\xc9:\xa0\xfd\xcc\xb5<\xd8:\xe7\xb4l\xc8\x98\xaa\xd9\xf5\x91\xbb\x17\xe3\x95\x13\x15\xbb\t\xabao\xbcYI\xf2c\x80\xce\xdaM\x02\xfe?\xec\xbd\xd9\x96\xe2H\xb6\xa0\xfd.yK\xaf\xd2<\xf5\x9dF\x90@\x13\x9a\xd5\xabW/M\x08\x84&\x84\x84\x86~\xf9\xdf\xf0\xc8:\xa7*\x89\xa8\xbf\xa8\x86\xe3\x9eu\xfc"\xd2#=@\xd8\xb6=\x9b\xc4\xfe\x10\xd0 \x0fS\xc9y\xcc\xad\xad\xb5u\xf0\xd7\x92\x85\x1c\xc3m\xa4\r\xc4V\x0b\'\xd1\xc5CA\xdd,iL\xe1y.D2\x12\xe3\xb3x\xdcI\xd1\x89\xe1)D\xf6\xf1\xe1v\xd5WK\x15\xc9\xe5\xe8\x14R0\xcb\xf1\xd0\xdb\\/.\xe7\'\xc5|\x19@\xe7\xef\xbb\xec\xbf\x95\xb2\xa0\xcd|8]vIw[\xe0\xd3\x0c\xea\xee\xcbY\xa7\'|\xa1\xf6\x0b\x11\x94\xbc;\x1f\x15\xc2\x11\xc21\xb0:D\xb3#\xd9\xc1\xb6\xdbf\x93\x9f+\xbf9\xa8M\nw[{c\x04\xfc\x1a!\xa1\xb9\xbdRG\xa6J\x9e\x1c1\xff2~\x0e\xf0L\x18l\x0b\t\xd3\x0fb\xc2\xf9J\x1eR\xaa8\xc8\xaa\xcc\xb4y%\xe1\x83\x9aGh\x98\x07\x9a\x9c\xb9A\xbalv\x0e\xec\x9a\x9ei`\xc0\x80;;#m\xb0(\xb7\xda\xf6\xb2E\xa6\x16\xe3\x1c\xcb\xa3w\xa53\xf8\x10\xd5 5\x8b\xa3\x92\x9c\xf8/\x15\xc6_\xc6\xcf\x01\xbb\x88a8\x85\xc2\xf4Ce#\xa7#\xad\x12\xe3p\xc8\xe9\x8e\xa0.!\xd1y7#\x9e\xc2~\\/\x83s;A\x19\xc1_c\x9e9\xe1\xc4\x82d\xe5\xa5lk\xd9\xe0\xcf\x86\xd2B\xd5.\xb5[\x05\xee\xf4\xe3)\xec@A\xc7\x89+Y\xd3\x13g~r\x1e\xe7\xcb\xf89\xc0%@\x93B\x81\x88\xff`,#\x0c\x95r\xe2\x84\x0e\xbc\x8f%\xd0\xacb>\xe4\x07\x81\xa3\x96\x84\x91\xdanp\xe82\xba\xaewB\x8f\x95\x98\x99\xef\xa9`W\xa7yE\xa9%\x82\xee6\xc3I\xa9\x11\x92\x08&W\xb9A(\x9fb+\x18\r\xbb\xf3\xf1i\x10\xda\x8b\xf89w\x9f\x00\xc9\x1d\xc1\x1f\xc9/Gv\xd3\x85\x06\x04-\x15|\xe5\x9aUm\xe2\xd4^HW\xa4\xb1O\x99\xac/k\x8f6/\xa3\xd8\x84\xfa\xc9\'\x04\x13\xbda2\xda\x9f.D\x86d\x87\xa8\x9cj\xc5\x85;w\x90\xd4\xe4\xa0\x19^9pK\xef\xea\xd0\xd7\x82\x03\xbe\x8c\x9f\x03\x8c\x85\xa6\x98\xfb\x99\xd5C6t@\xf3\x99\xba\r\xc9E=u\xe8\xa0\xb6\xf7U\xc2\xcf\x8e\xd8j\x16\x9a\x13?\x18\x92\x8fm\xc4\xdbF\xb3\xe0\x02\x9f\xa9\xe3~(6+\x9dL\xb5j\xad\x9c\xca\x8d\xb0\xa6\xbbj\xc7\xf5\x19\x99\xd7\x17\\_\x19\x0b\xb2\xc3/\x9f\xc5\xcf\xb9w44\x03\xa3\xf8#XFS\x0fts&J\x97B\xc5c\xa0H~\x80u\xb3$\x9b\xa2\x1fC1\xbe\xc9cU\x82v\xa9\xd2\xa3s},\xc9\x95\xce\x19\xb4\xd1\xf8[w4\xb2\\\xb4"\xdd\xd0Sh\'v\x8d#\xecJ\x91>fUv\xc8\x9fdt\xbc\x8c\x9fs\x17\x93\xa4\x11\x86A\x1eJ8\xd0\x03\xabUU\x14\xe3\xc0\xf5\xc9\x8a\xdfx\xb6\x97\xc6\xe8:\nOfk\xed\xf2j\x1fg~\xb3!Pn\xbf\x92o\xd1\xb4E]K\x86\xfd\n:\xba\xd9\x85n\xc8\x1d\xae\x8c\xe1&[\xaf\x8bP*\xfa\x10\xe1\x83h\xdc?9?\xfae\xfc\x9c{\xd2G\t\x02c\x1e\xca\xf1Q@Y\x8b\xd7\xcd$V#l\x1f\xb3\x07\x01\xebo#z\xcc\xc8\xb8U\x9a\r\xb2\x8b\xe2\xb5>r\xe7\xde\x93\xd1)b\x84\xe9rnk\xab\xc1\xa2\xa4\x10D\x0f\xad\xa2H\xdf+L\xa7\xca\xf1r\x0e@b\xa9\t\xfa\xd9\x01\xf6/\xc3\xe7\x00)1\x02!1\xfcq\xe4\xb9@\xa3\x05\xea\xc3\xd7~J\x08\xdesp\xd1\xdc7\x0b)\x95S\x8f\x9c\x03k)\x92i\xb6V\xf1q\xbd\xecqo\xdenI\xcb\x0f\xd6\xad\x0cI\x01\xba3oH\xaa\xd5Cq*iJ\xed\x18\xf8v0\x95\xd5>r\xadg\x1b\xf1W\xe1s\x80\xcd\x12\xf8\x1d\xbb\xf0\xd8\xa2\xda\xf0\xae\xb2)\xddb\xd1\x0b\xbf\xa2\x89\xf8\xa4\xd9\xa8\xc1\xed\x16{\x10\xbc\x98\x02\x86\xa3"\x96\xbd\xcb#\x89\xd9\xa1\xa4"9\x16{\x8bv.\xe3\xc9\t\xaat\x9b\xe3y(C$\xe5\tc!\xf5>i\xaf\xeeY\x99\xd2\'\xc5|\x19>\xe7\x1ehA\x00B\x7f\x02\\a\xe8y\xbf\xe77>T\xa3\x17e\xbczu\xe2r+&\x04y\xd2\x95\xefc\xd6q\\qX\xf6p\xd5\x85i v\x81i\xc4\x88/\x98\xb7\xd5\xb1\x17(OZX\x81\xdd\xc0%s\xcd8\x81!\xca\xdd\xf64w\x1b\xe7\xb3\xf09\x1f\x95*\x8a\xa0\xa0\\\xfb\xa3\x98\xc4\x9271\x0c\x11\x9e9\xcd\x17uo\x97\th\x98\x0e\xf9\x11\x9e\x88\x9b\'\x96*GO\'\xbdc\xf6\xdcx\xda\x93\xf5\x1e\xb27\xc1\xb6\xe51?2Y\xdeo\x04Z[%se&\xea\xc6\x8b\t+\xf0O&\xa3\xa2O\x8ev\x7f\x19>\xe7.&\x85\xdcO\x10\x1f\x02-t9@\xfc\xd0\xde\xdc\xeb4\xbb\xa5i!x\x13\xdd\xb8\x88-\xd5\xe3\x8e\xd9\xad\xd4\x8b\xde\xe5\xb4\x8fp\xa4\x04K\xeaz\x86+=\xf4\x8fk\x8f\xc1\xed4\xe1\xafR \x8b\xab\xdd\xc9V\x8eX\xb7s\xe5r-\xa3K\x1d\xfd\xaa \x7f7>\x07\x18-~\xef\xba)\xfc\xa1\x89\xbc\r\xc2\xc4\x85\xec,j\xcb\x01\xa5\xdc\xe5@F\xfc\x11\xa3\xbc-\xcd\xc6YS17[\x82\rsj\xcb\x06J\x13\xe6\xea\\q\x89Y\x8dtc\x91\xdb\xd5\x12\xaeh\x8d:\x93\xde&\xcfJo\xa7X\x8e|\x14\x96\x1a\xfbU\xda\xfc\x93\xe3s>N\x1c@\x13\xc3\x10\x0f\xf9Jm\xe2<\xbbR.\x92\x1b\x19\xdds;TV}_[\xb3f\xab\xdaW\xf1j\xe3\x96\xd0\x95\r\x83T3\xf8\xe0\xd5\xf5\xb2\xb9\x19\xf3%"W\nPK|\xbe\xf4U\xb1g\x1b+;\x06\xb2L\xe3\xb2v\x95\xb7\xf9\xe6Icy\x19>\x07\x88y\x7f\xda\x01G\xd0\x87\xbec\x1bY\xc7d\x7f\x13\xca\t[m\x0f2+\xe5\xfd%l3Imd\xef:\x0c\x97\xdcq\x19\xae\xbf\\\x1b\xfeJ\xeb\xe7s\x96\xce\xb2\xc3\xf0\xb8\xeb\n\xe5M\xbf\x1e\xa9\xb4\xdc\xd4\xc3MA\x18e\xd4\xda\xa5\xb1\xf7\x92\xce\x7f\x16>\xe7\xaeM\x84F\xc1\x9b\x1e\x029u\xdd\xae\x8e\xf6\xc6\xd8fb\xa3\x97\xd4j\x8bo\xa1\xb6\x84\x93\n\x97\xf6\xf8uc\xe8\x96\xc9i\x8a3e\xdbzC"\x8cq5(\xcf\xd8\x98\xed\xc5;J\xb7\x96\xed\xc3\x1ds\xa1\xec5^\x0e\xcet\x9c\x97@\xc3\xf8\xe8\xc9R\xf2e\xf8\x9c{\xbe\xa2\x11p\x05\xfcA\x9b\xac\xd3\xb8W\x16D\x85x\x81&\x07\xdf2*\xadi\xca\xd9 |\xa8X\xa1T\x96\xf3DF\x1c=\x7f?\xa1\xde~\xed\x1b\t\x9c\x16\xa5\xa2\xe6x\xb9=\xd9\xdc\xd5\xaaEi\x8c\x14\xf3\x92\xd1J>\xae\x99M\x1c+\xcf6\xcb/\xc3\xe7\xdc\x8f\xc9h\x06\xc1(\xe4!_Y\x96\x9c_\x89\x90(\xc6u_\xa9\xf9\xce\\ba\xb3\xc6\x84\xe3\xbe\t\n\xec6L\xdc06|C\xf0Y\xe3\x11\xf6ZN\tl\xe0w\xd0H\x06\x07kw"W\xf2&\rL\xf6\xaa\xf1\xdd\x08\'\x07\x9eUN\x02\xf2\xec\x1d\x9dW\xe1s\xee\xf9\x8a\x84\xef-\xd0\x83\xd1f\xdeN\x0bw\xa4q\xc0n\xf0.\xd7L\xd6\x11"\xe0ngq\t\x12\xc2:4$\xb6\x9c\xba`S\xd0\xe1pNU\xca\x8c7)\x94\xf0\xc1\xb1\xa4\x87\x92kb\xb9\x92\x91\x95\x1e\xc9\r5\x93)>\xb14\xdb\xc3\xf2\xb3b\xbe\n\x9f\xf3a\xb4\x0c~\xa7\xa3\xffQ\xcc5\xe6Y\xd9\x1a[\xa6[T{\x84\x8cJ\xd5r\x9c\x0f\xcc\x18\x8b\x19D\x0b\x96Q\xec\x8c\x9c\xa3xO\xe67![m\xe7>\xefL\xbd\\\x86J\x19]\x99Q\xed\xe3jK\xcb\x1a\xebl\x15R\xaa{_3:5x\xd27_\xc6\xcf\xb9\xa7e\x06Gh\xe6\xc15\x07\xcd\x0f\x97[\x02G\xa1b_\x8f\xe8\xb9d\x8dK\xdb\xe0L{\xc0\x85\xb5gLe\x0e\xa9\xd2x\xf2\x18\xcdV\xfa@Z\x05\x13\x8b\x1ff\x91H\xc2H\xbdl\xf6\xb4e\x9b\x87J2t,\xf0\xae\x855\xaf\xdd\xed\xb8z\xd25_\x86\xcf\xa1\xfe\xc20w\xdc5\xf6\x98Ov\xf9\xee\xd0m\xf3\x84D\x923G\xe6KOq\xc5n\x0f#\x9a\x82\xa23\x95\xc8\xa7iU\x85=\x99m5\x99H\xe2)?\xac7\xea -M\xe4\r\x15\x87m\xb8h<\x9f\xe9\xe6\xb6b\xba\xd2\xbf\x9c\xae\xde\x06\xb9\xd5O*\xf3e\xf8\x9c\x8f\x1b\x91w\x14#\xf2 \xa6oB\x8d)xy\xd2V!c\x93\xea&l\x14iP\xa85+\x92E/\t\r\xc5\xb0B\xa6\x9fUF\xb2\xe3]\xd0\xd4GR?"\x04\x1c7\xd3\x9e>"\'\xa1\xbb\xce\xd6\x89\x1fw,\xean\xfd\xa6#\xe3J\xfaU7\xfb\'\xe7\xe7\xdc\xcfva\x86!A\xaf\xf2\x10\xc7\xf79G*G\xb9\xdaCe~C\x0e\x14\xb1\xaa\x87\x8e\x93\xac~px.O\x8f;\x8e\xd4\xaf|9!qEFk\x8b\xc1d\xd9\xa6\x99St\xc0V\xd5\xd6D\x82\x8d\xe58\x02\x92\x0f\x8aX\x9e\xf0\xc6\xaf\x13PZ\xb3\xff\x1c?\xe7C\xfd\x7f\xcb\xcf\xf9_\xff\xf7\xb7\xa4I\xb3\x8f\xe7L~E\xf6\x9e[\xbak\xafB\xd6\x96x\x9d\xe2u*;\xce^S\xab#\x16\xba%\x1aWD\xb4ER=\\OF\xe4\xa4\x17w\xcd\x94\x8e\x17b\xe9\xe6\x18\xc4U\xc2\x18\xe6\x1f\x11:\xff2\xb9\xe8_\xd8\xb3\xbf\x86\x9b\xdf\xaf\xfc\x8f\xc8E\xff\xcf\\\xa4\xc7\xafP\xfc\xf6\xe3\x81,\xed\xc7\x03Z\xe3\x1f7\xe2_\xe5\x18\xfd\xab\xfb\xf0\xc4g\x03\xdb\x85#\x8f\x19\xfe\xe6\xb3\xc1R\xae\xa1\x97\x1e\xd3\xb5\xf3\x87k\xa5Y\x99\xe5Q\xff\xcb\x8b}\xd6@\xba\xdf\xfe\xe6\xd9\xbc\xe7\xc4\x8f=\xe6\x9cz\xd3\xdf\xfaN\x82\xed\x8f\x11Z\x0e\xe1\xf8\xf7&\xf5z%\xfd\xef\xff\xf1\x11\xbd\xaemt\x7f\xa8\xee\xb7\xdf\xee\xbf\xf8f\x81\xfd\xe2KI\xff>,\xb0?\xd1 \xd6\xef\xd9\xb6\xdf,\xb0o\x16\xd87\x0b\xec\x9b\x05\xf6\xcd\x02\xfbf\x81}\xe1u~\xb3\xc0\xbeY`\xdf,\xb0o\x16\xd8Wfl}\xb3\xc0\xbeY`\xdf,\xb0o\x16\xd8\x97]\xe77\x0b\xec\x9b\x05\xf6\xcd\x02\xfbf\x81}\xe5s\xc8\xff~\xe7\xa5\xdf,\xb0o\x16\xd87\x0b\xec\x9b\x05\xf6\xcd\x02\xfbf\x81}\xb3\xc0\xbe\xee:\xbfY`\xdf,\xb0o\x16\xd87\x0b\xec+3\xb6\xbeY`\xdf,\xb0o\x16\xd87\x0b\xec\xcb\xae\xf3\x9b\x05\xf6\xcd\x02\xfbf\x81}\xb3\xc0\xbe2c\xeb\x9b\x05\xf6\xcd\x02\xfbf\x81}\xb3\xc0\xbe\xec:\xbfY`\xdf,\xb0o\x16\xd87\x0b\xec+3\xb6\xfe\xbdY`\xd3\xdf|\xc8\xff/\xa7\xe8\xef\xa8F_\x9c\x05\xf6v\xc1>\x8b\x05\xf6v\xc1>\x8b\x05\xf6\x94`\xff\n2\xeb\xb3X`\xef7\xc5\xff\x12\x16\xd8\xff\xfa\x0f\x8a\xd6}\xd8\x835T\x1fP\x8f\xdf\xbf\xee\x0c\xae\x93u\xd5\xa9\xee\xff\x92ts\xdb7\x7f1\x86\xb8<%\xdbl\xfe?\xe2\xef\xa4\xad\xffX\xff\xff\xfd\x0f\xf8\x16\x90pa\xe9y\xdeX\xb7\xb9\xba\xf0m\xd9\x1d\xe1\x8e\x80m\x04T\x93\xa5vM\xaai\x8b\xd1&n\xc3\x0e\xde\xdc\xcb\xf5\xfb\xd8\x89\xb6\x19\xef(\xab\xdfh\xf2\xb7\x1f\xdf\xcb\xfeo\xc4&C\xffv\x13\xff\x11\x9b\x8c\x86\x11\x1eE\x05Z`9\xe2>c\x95\xa0`\xee>\xfe]\x00+\xc0\xb9\x8f!y\xa8\x881""\xf0\xbc\x84\xf1\x12\x8e\x0b\x08\xcc\xd1\xd2}\xec\x1d\xca~\x8c\xef\xf85\xb7Ed\x08Q\x92\xee\xb3\xd2p\x9c%y\x0e\xc3Y\tA\x11\x94\x908\x1e\x13hN\xa4P\x02G1\x1a\x91D\x02|\x10\t.N\xb3\x04\x0c\xfe\xf0\x0c.P\xdc[\xd9d\x7f\x1dK\xf7\xdb\xc3\xb4\x14\xfa\x7f\xc2\xf8_P\x12\')\x12E\x98\x7f\x04\'\xfb\xfad\xb7\x9f\xc0\xc9p\x82\x04\xdb\x0c\xc3,\x8bb(AI\xa4\x00\x13"\xcc\x91\x18G\xdd\xa7\xcb"8K\xa0\x06\x99&\x81A\xf0\x12\x07\x8b\xc0\x94\x10^\xe0D\xe0<\xb0$\xc2\x92@\xfc\xf6s8\x19\xb0A\x96\xa3\x04\t\xa7\xb9\xfb\x9bi\xf0\x03\x13\xc1\xc5D\x14\x05ZEIA\xbcS\xd0p\xf0\n\x94\xc4x\xfc>a\x9d%(D\xc2A0 )\xe1>)\xee\xbf\x06N\xf6O\x13l\x1e\xe0d\xff2\xde\xean/\xff\x10o\xf5\xf5\x1d\xfds\xf1V\xf4/)\x13\xebb\xbc\xf4\xce\x81\xaa\xd2\xf9T\x86\xd5\x88\x1c\x91\x10\xdf\xf5Q\x1b\xd32_\xf4\xd7\xabYj\x96\x890\xf1\xbc\x9c\x83\xa2\xf6[.\xa3X\xc1H\xaf\xc7I\xb1;l\x8a<\x89\n\xba\x1a\x0e\xfb-\xdf\xa4p\xae_\x1a\xeb\xc9\xd9\xbd\xaf\xa1[\xfd\xc8\n\x14L! w<\x0c\xb5]\xaf\xf8\xa8\xceI\x87\x8bV#t$OD\x03J\xc5\xd4\x93\xce\xa7\r\xdf\xdf\xb6\x05\'\xeb\xb7\x8db\x1e\x95\xd6\x81\xaa}\xd8\x18,1\xd2}k;F\xc2\xedl\x8b \x8dM\x05W\xb0-A\x9d\xa0\xb7\xfd\n"e\xf4Yp\xc0K\xe8V\x1fb\xe2\xa0\x02!\x1e\x06!V,\xb6\x93\xd6\x07\xbaB\x08h\xc2\xe7\x9d\xe3\x0f\x82\xda\x86\x8e\xa6\xd6X\x89D\x98\xd4\xee+\xf3\x96\xfaC\xe8\x8f\x87&\xa3\xfc\xa2\xed\xdb\xa2\xde\x8b\xdc2\xfb\xa2\x8f\xb9\xfb\xd6\x97\xc4\xf6\xc0\x9d{\xf6<\x84l\xa3\xe5\xc7\xa7\xb1O\xaf\xa0[\xfd\x90\x11\x87a\x1c\xa7\x1f!^y\xaa\x1d\x18\xcf\x88\x97\x12"Y\xe8\x16\xec{Q\x0f\x90S\xb6Fb\x87\xd2\xb5\xe2\xaccV\x92\x1c\xa8\xf1B\xf5\x81\x1246NL\x04[\ng5\xbb\xacg8:\xa8\xfb\x9b\tsv\xb4:\xaf+9\xd8\x88T\x8f?I,z\r\xdd\xeaCL\x0cFq\x98d\xe0\x87\xf9\xc4\x82\x7fm\xb5\x989\x84S\xbe\x81<\xa9\xcf\xc6\x8dhZ\x06\x837T|\xb5\xbb\r:\x87t8\x9f\xcb$\xd9\xf6\x1b\xad\xd9\xba\xd7\x9b\xe0O9AuW\x85\xcc\x8d\xae\xa0]\xc784\xa0\x12?n\xb5@\xce\x8a\\\xca~e\xb1\x7ff\xbc\xd5\xef\xbb\x08\x8aX\x0c\'\x1e\xc2[\xe7\x9c\xe1\x1dt\x12\xbc\x93\xebV\xb1cAX\xd6\x9c\xdd\xf6\\\xf3\x98X\x1d\xf7\x16Z\'\xb65\xab\\6\x86\x06c\x19\xa9q\xeb\xfb\x90\xb11eo\x94\x07\x1cv4\xab\x91\xab.h\x05*\xe2\xa4\xa5\xce\xa0\xb0\xda=\xc9\x80x\r\xde\xea\x87\x98(\xc5`4\xf3\x08\x0c\xf1\x84\x96\xb1\xf3>0\xf2\xd0\xdb\xe5\xa7>TK\xed,\xaenq9\x06\xd4\xfaJ\xee\xfbq\x0c\xf2l\xef\x91JH\xdc\xb6\xf8\x82\x17\x9d#\xee\xf5u\x04s\x93\xed\xe1\x07\x88e\xe3t\xa3\x17;\xfa\xe6\xce\xe2\xe6x\xb5\xd7\xcf\xce@}\t\xdf\xeaG\x14GP\n\xc3\x11\xea!\x8a\xfb\x81E\xef\xe8j5\xf9\xa6\xd5\xce\xb0$I*\xae\xf6:\xe3\xe6\xca\x08\x8f\xc1dS\xc1\xe16\xcdV\xbcd\x8e\x17^\xd6\xe3\x9av\xdcF^\xd0\x96\xeb,\xb5gz\x97\xc1\x9db\x87\xf8\x89\xa9u\xbbj\xdaaH\xf7d\x14\x7f\r\xdf\xea\x876\x11\xe4>\xe1\x18{\x18\xaam\x8dmv\x0b\x86\x8bc\x0c\xd1\xd9\x1e\x84u\x93-A\xc39\xebq\x0e\x85\x8dK\xe6\xf5\x81w\x87\x06\x0b!D\x99\xb8-d\x8aX)\xf6==\x89Wy/,6\xb4\x92\xcf\x06\xa2\'\xb88D\xebh\xad\xc7\x12\x04=)\xe6k\xf8V\x8f\x9d\xda\xdf\x8a\xa9\xd4\xfe\xb6k\x15\x87\x0ck\x0c; \x9e\xd8v\xd9\x1a\xdd\xaaG\xa1\xc7]\x9d2\xd0K\xae\x92\x10\xe5\x15\xe7\x06\xdf\xd9\xfc\x8a1\x8f\x11\x15\xf1<\xb4V\xd1\xca\xd6@x\xdf\xf3r\xa1\\w[\xdc8\xe6M\xd6\xda\x1d\xfb\xec\xa0\xf9\x97\x00\xae~7Z\x02\xa5I\n}\x10\xf3\x08\xc5~\xe5bv>\x93VX\x1b\x15\x1bQW\xba\xa1\xe4hO(<\xa4CY\xe5dv\xc2\x80J\xf9\x1a\x1f\xc6fI\x9drk\xfa\x97k\xcf!\xa8\xe9\xf6\x18\x95n\x13N\x95\xd0\x16N\xd9H\xef\x7f\x05.\xf93\x03\xae~d}\x82\x02\xf6\x82>F8\xdaE\x1b\t3\xc82\xc6\xd7\xab\xc6\xcb\xdc\x130s\xb7\x9bC\xfft\x8e|\x95X\x110\xc4C\xc9\xc6V/g8B;\xb5\xed\xc9\xecf\xf7\xcb\xb0\xb7h\xb1<\xa7\x17\xd1Wls\xb2\xd1\xad\'\xc8\x17\xd0\xb1\xc6\x05\xfb$\xa7\xf0%\x80\xab\x1f\xc6B\x80\x9e\x0b\xa7\x1e\xe7\xe9[\xd1\x81 \xd7j\xee\xc5\x98\x85\xad\xe4J\x89\xe5,\xdc\xae\xf6\xc8\xa6UX\xcb\xe7\x86\xb4B+?\xa7\x91\xe0$[\xdd\x8a\x8e\x9dy\xcf]\xc7)\\u1*r\x1b>\xefP}\x7f%\x10C\xf1|\xe1\xaaLS\xa1?\xe9\x13\xaf\x01\\\xfd\x9e\xafP\x064L\xf0#\xf4\xad\t\x9bh#]\xbd\xe5\xaa\x1e\xb0\xdc\x9b \xed\xc2F\x0c\xd7\xc7m\xc5\xdb\xf62(\xa1\xbdi1i\x9d\xe2\x10\xa8\xf6D\xf8\xc0\x1c\xad\xc0\r5\xf7d\xf4\xec\xce\xb8\x8a\xf5\xa6\x0b\xd6X\xbd\x17R\xe1\x96\xb0\xf4\xd1\x8a\x84/U\xdc\xbc\x06p\xf5{\x00%>0\xaf\x0f\x83\x815\x1a\xbbn\xa6!_\xda,\xbaV\x1e\x91\xe2bE5\xf5\xa58l/\x84\xb2\xe1\xf7\xfa^\xcb\xa4\x13\x153\'ve9\xeee\xbd\xdd\x9ek#o\x1d\xf7\x98Qp\x9b\xc9\xe1\xe8\xe6\x9d\xd8d\x96\x84\xebP\x0e2"\xf5$\x99\xf85\x80\xab\x1f\xae\x8f\xd2\x18\x01\xbc\xe2A\xccPlS\xac4\x19\xda5\xa3\xd1.\x8f\xc1\x958\x9b]\xb25\xe7\xa9\\\x9a\xdc>\x19T\'-$o@\x1by}\xcd\x91\x0cJ\x90\xb3\xd6B\xc9\xca\x19\'S\x97\x8a\xa0\xd1\x13\xb5Y\xf2c\x86z\x89^Itpx\xb2\x86{\r\xe0\xeaw1A\xbfG1\x8f \xc4\xaa\xde\xb2\xe51\xc7/\xf04\x17\x9ew\xcc\xc9\x03l\xf5\xb4\xe8\\\xa4u(\x1f\xe7\xc3mO\xf4\xb6\xd6\x85\xdbx\x19\xb0Bm\x93\xa3\xec\x05\xe5\x1asJ\x9b\x0b\x9b\x1dS\x9cn \xe1\x877\xd0\xe3\xf1c\xde\xd2\xf8|{\xd2\xf5_\x03\xb8\xfa\xbd\xb8\x01m\rL=\x167\x06lj\x84*\xa0\x13\xd4].\xb6\x01\xc3\x97\xc3L\xd8\x18\xe1\x1f\x1a#\x12\xac\xa6\xcd\xb7<\x93\n\xe7L<\x85\x0e\xb9N\xe1\xe2\xa4\x8bA\xbd\x1f\xf5\xdcj\xa6zX\x04X\xe2P\xce6-\xb6_\xeb\x08_\xae\xe2\xf0S\x08W?\xc4$\x18\x18\xf4\x1d\x8f\xdaD\xfa0\xaf/e;\xda\xd0\x18\xee\x1b#Q\x8dF\x12\xb9\xa3I\xc1\x193\xe5\xae\xeb^Zn\xb4g\x85\x91\xab\xd9\xd9\xed\x84\x93\x11$\x8e\'\xac\xfa\xf3\xac\x92\xc5\xb9\x9d\xc8\x8d\xcf\xf57\x16WE\xda>leon\x7f\x15\xe1\xdeJ\xb8\xfa!&IQ\x0c\x82\xa3\x0fi\x199\xfb\xad\xb5\x0b*\xf88(\xf1\x95<\xa2\xe3\x86(qqb\xb8\xd8\xac\x84`\xe3\x9d9\xeal[\xeb\xebUSG\r\xb1\rv\xa5\xa4I\xbd\x1c7=\xea\xd1#jQ\xc2f5$Jx"\x86 \xee\x90\xf8|=<9\xc8\xfe5\x84\xab\xbf6\x1e\xe0\xfd0\xfep\xe6\xa0\x99\xb0\xb4\xcfR.\xbc\x8d+\xb1O\xcbE\xa1\x95\xda\xde\x18\xa0\xe8\xadG\xb6\xef\x8eOb\x92_C\xb8\xfa]\x9b40Z\xf8\x91\x8f~\xc1k6Y\xdax^\x91I\xe3\'G\x1e\xdd\x9c\xbb]\x86\xf2\xf5\x96H2\xe5h\xf2\x17\xc1k\xf3<\xbb\xd8\'M\xc9\xf4\x95l\x92sy#\x82\x13\xb5\x11I\xde\xe6\x08\xaa\xa6tw\xd7\xec\\b=qdP\x83\xb4\xfe\x9c6_C\xb8\xfa=\xd2\x92\x04C\xa2\x0f6\xbbu\x0e\x01f\xb9\xb9_\xdf\xb2\x1c\x9e\xd69\x89\xd6\xbd\xd1q\xd31\xaf)\xcb\xb8z{\xfa"\xba\xf0\x85\xb3\x8e\xb0g\xb2\xab\xc2\xa1j\xbbt\x1d\xde\x1eV\x07\xb5\xa8\xe4\xc561e\x7fH\xfc\x13?ta\xde\x0f[\xefW\x15\xf3\x9f\x19p\xf5\xc3V`\x064\xb8\xd4#k:\xedd\xd6\x9c\x93\xd3Z\xa4\xc6\xc3\x1e\xda6\xac\x82\x8c\xae\x87/\xd6\x8e\'(l\x05\xb3\x0b\xb9=\x9dv\xb7n2\xda\xe4$\xba\xa72\xc6\xae0\x8c\x87\x0b\xef\x1e\x1d\xea:-\x9at\x0c\xf0\x8e\x02\xa9s\xeb5\xab\xed0>Ia~\r\xe0\xea\x87\x98\x0c\xc60\xc0Z\x1e\x03\x9c\xed\xf9{\xbb\xb1j\xe1V\x05S\x16_\xaeT\xc5\xb1\xe1H"\x9e\xd0^\xe7&dy\xa9\xa0!\x95\xc4b\xe9\xe2+\xe5\x8d\x97\x0f\xcb\xde\xe6\x12\xd1\xe8\xdd}\xc5\xb0\x07^\xb6NY\rZ\x9a\xab\xa9p\xdb~\xcf\x1ff\xde=y\xe6\x88\xe1=\xb2\twK\xd5\x0b5y\xdc0eE\x9f@\xe4E8\xd0\xee=\'\xe5k\xf8V\xbfg+\x02\x861\x98z\xa8$o[\x985\xc2\xa6\xd7\xb6\xfa6S\x10\x0c\x14\xca\xab\x83\xc3\x04\x0c\xc4qd\xbbl \xdd_\xaeR\xabC\x16w\xd4\x17\xb9\x0b\xd0\xb9jt\xab\x80\x8dI\xd5\xbb\x8c4\x86AX\xd1D5\x1dqk\xf0\xe03\xbf\xab\xea\'K\xac\xd7\xf0\xad~\xf4\x05$F\x81\xf0\xff\x98\xad\x90\xf5\xd0\x1b\xdb\x059r\xd9\x01\x94\xe1\xce\x9d\x9dC,tA\xc7\xe8j96;\xef*x\xbb1\xd3p\x97\xf3\xd6=\xbc?\xae\xc6\r\x14w\x97=\x9cq\xadr\xc4\xf1\xb2\xafE\xf0\x89\x98\xa3\xb1\x1b\xd9H\xcc6\xca\x9e\x04\xdf\xbe\x86o\xf5W\xd7\xa4I\x04&\x1e\xcev\xa7\xb5s\xc8Y\xca4\x1c\xe4fJ\xbat\xb5\x8d\x1b\r\x9b\xf5l\xdd\xfc\x15\xb6\x9f\xcc\x88\xb8\xf4\x97\x06\x1e7|\xc0\xc0\r\xc8O<\x1b\x8c8&\xba\xe2\x14\x07\x07\x98_v\xec\xf62\xba\x92\x89\x94Ms\xb0\xaa\xd9[}\n\xdf\xea\x87\xd12 \xfdP0\xf9\xd0\xcc\xcef\\i;\xbfi\xe8\x95\xb5\xa2\x1d\xe5"\\\xfa`\x95\xf2\xc2(\xfb\x86\xa1\xaduW8\xa0\x141-sp\x9b\xcdv\x05\x9a\xa3\xe6\x1c\x04gF\xaa-\xa2\xb8noi\xc1M\x04J\xad\x0f\xfb\xc9m\xe5\xad\rI\xee\x93=\xfbk\x00W\xbf\x1b-\x08?0F<\xa4Me\xcc\x16\x994\\\xff\x1co\xb5\x13qX\rb\x8a\xa4\xa7\xda\xe1\xce\x89f\xaf\xcb%\x86e\x13\x9a\xeb\x03\x96r(\x9f\xb4p}\xc0\x89\xec\x16U\x9ewE\xeb\xd6\xd1\xf6\x98\xa7\xc0H~ltC\xdf\xd1\xab\xab\x99\xb5Ovy\xaf\x01\\\xfd\x1eh\t\x144\xfd\x8f\xc0\xc4\xf4\xd0\xea\xb8\xec*\xcda\xb3wLA2\x86\xa5\xedy\x91r\tO\xd9\xa07\xb2\xdbn\xb6\xd6\xb0\xdf\xd0\xb6\xb5\x8f\xf5\x9b\x97nAC\xd8\x97\x99\xa0\x98\xdd\xce-F6\x9c\xb6\xfa\xb1ja\xd2\xe2\xeb\xf3\r>\xa1\xb8\xff\xab|\xf2g\x06\\\xfd\xee\x13\x04\n\xf6\xfc1\xc2I\\r\xea\xf7\xbcj\x9a\xcdJ\xc1\x96\r\x0e)u\xb5\xa4A\x08\t\xb3~\xae\xa1\x0e\xcbSug\xf7\x90\x95&\x84qi\xd8Y\xf3\xd7:\x17\xba!\x86:\xa7\xd0^\x07\x8c$b\xbe\x9f\xc4\xb1\x04\xf9rXT\xd9\x915\xff9\xc0\xd5Gq\xf8\xb7\x80\xab\x1f\x8fM~Ca\xfe\xe3A\xd6w`\x96\xbe\x1e"\xe6\xe5\xf0\x8d\xdft!Xv\xc5\x19W\x85`\xd6\x05\x19Qm\xf0\xae\x82\x05\xbby\x06\xf6\xeb\xde\x12\x14\xb9\xfd\xf6\x9e/S\xbcE\x98A]~.L\xea1K\xec\xd1\x7fP\xcf\x17e\xa2\xfc{\xa9E+\x12lW\x88\x0b\x10\x04\xd7\x04y\xd2\x96\xf3\xf4!\x88\xa0.\xfa\xe6g\xc2|U\xb5\xdc%Q-\xfc\xa7\x92hK\xf9\x8b\xf8\xf3u\x816 p\x9e\xe1\x1d\x08f:\xb0/\xad8c\x9a-\x02a\xce\x88\xb6\xe4\x93\x8a\xbdO5o\x90\xa4`\x11\xa0\x9a\x11H\x80k\x85\x8c\xdf\xbf\x04\xa3\xd9\xf2\xa4\x0b\t\xf0\x1e\xee\xdd\xaay9p\x06\xa8FD\x80\xd7L\xea\xe2\x8c\xba\xa0\x02[caM\xc8\x11 \xdc\xa2\xfe4\x04|Qt\xce\x87$@5?\x97D\x08\xdf\xae\x9aW\x83k~\xd3ms\x00\xaeO\x00\x81\x08\xf0\x13\xd8\x19\x0b\xbf$\x84j\xdfS\xbes\xff\x89\xbf=\x88\xbd\x18\xf4\xf4\xcb \x06J\x81\xb7\x9e\x89\xbdA\x92\x82\x05jy\xcc\x92\xf7@\xa6\xbe\xf7\xf0\xe5\xe5\xb0\xa8\xdf\xb4\x13\x8eh\x82\x08zc\x07\x07\xbd1\xf8;\xe8\x8f\xed\x9c\xf8\xf5\xdd\xd2/\x8a\xbd\xba\x9f\x89\x01\xbf7\x91\xfbl3\xe09304\x10\xcc\x80\xa7\x14\xe2/j\xca\xaf\x0b\xc6\xfaM\x9dqB\x17D\x04\x94\x94\x13\xc83@\x18\x11\x94\x94\xc1\xa4\t\xca[\xd5\xf2\x06I\x16\x15\xf8}BhK\x82\xe8wo\xf98\x13S\xe1\xfb}\x987\xab\xe5\xe5\xd0\xaa\xbbZFM\xb8\'\xc6{\xb2\xcc\x89\xfby\xabn\xdf\x7f\xbeW-o\x90da\x87\x9fK"\xa2oV\xcb\x7f[@\xd7/\xfb\xab{\xe6xw\x88z5\x1a\xeb7\x95\xc7A\xb7\xfb!\x08\x08Q2\xaa.g\x90\xdcE\xf0\xff\xe9{C\xd4\xcb%\xd1f\xfc\xa3pW\xefwSAI\x02J\xdf{\x0e\x04\xc1\xf7\xfd\xb7\x85_\x8d\xd7\xba\xab\x05\xbc#\xb9\x9f\x0b\xcd\xba`\xce\xf7\xd8\x0bj\xafE\x15\xf6\xefM\xe8/\x97\x04\x94&\xd8\xe3\xd9\x9d:\xaa\x8b\xfbv\xb5\xbc\x1a\xd1uW\x0b\xa8\xad\xce\xa0\xce\x02\x7f\xee\xb6v\xefP\ng\x06M\xda{\xd5\xf2rI>S-\xaf\xc6|\xdd\xd52\xde\x1bE\xed~\xb7\xee~{\xc8\xbew\xbe&(\x7f\x7f%\xcc\x17\x05\x96\xfd\xcc\xef\x81$\xf9\xa8\xff\xd2\xc0\xbe.\xd2\xecC-\x1f\xcfR\xd9\xa0\x87/\xf2\xfbY\x11\xa6\x0bg\xfc\xd7\x0f\xea|Q\xe8\xd9g\xaa\xe5\xe5\xb8\xb1OS\xcb[$\xf9,\xb5\xbc\x1cY\xf6ijy\x8b$\x9f\xa5\x96\x97c\xcf>-\xb7\xbcE\x92\xcfR\xcb\xcb\xd1i\x9f\xe6-o\x91\xe4\xb3\xd4\xf2r\xfc\xda\xa7y\xcb[$\xf9,\xb5\xbc\x1c\xe1\xf6i\xde\xf2\x16I>\xad@~5\x06\xee\xf3\n\xe4wH\xf2i\xde\xf2j\x94\xdc\xa7\x05\xb1\xb7H\xf2ijy5\x8e\xee\xf3\xd4\xf2\x0eI>M-\xafF\xda}^ny\x87$\x9f\xd7\xb7\xbc\x18\x8b\xf7ijy\x8b$\x9f\xa5\x96\x97\xa3\xf5>-\x88\xbdE\x92O;|y5\x9e\xef\xf3\x0e_\xde!\xc9\xa7\x05\xb1W#\xfe>/\x88\xbdC\x92O\xf3\x96Wc\x02?-\x88\xbdE\x92\'\xd5\xf2\xaf\xe1\xda\xe6\xa7pm\xd3\xdf-\xfaK#\x11\xdf.\xd8g!\x11\xdf.\xd8g!\x11\x9f\x13\xec_ \x07~\x16\x12\xf1\xfd\xa6\xf8_\x82D\xfc\xf1\x0f\xff\xad\x10\x84\xd8\x7fn\xfa?F\x10R\x0c\x87!\x08C\xdeiS\xf7\xb9\x9a,.\xf1\xf8\x1dOFJ\x18\xce\xf1\x14I\x10\x02)\xf1\x0cX.\xca\n4)b0\x82!(\x8b\x8a\x0c\x8br\xe2}\xea\xff\xaf\xd9Z\x0cL\x8b\x18\x8aQ0\'a"r\x9f\x14\x8e\xe28\xc1q\x08\xf8\x8d\x801\x18\xc7\xc1wv\x1b\xc2\x92\x14\x02\xfeC\xd0\x0c\xc5\xd1$\xf8x\x96\x95\x18\x02\xa7\xde\x8a \xbco\xd1\xaf\x10\x84\x08\xf2\x17\x0cC\x18\x18\x06\xff\xf2\x8f\x10\x84_\x1f\xe0\xf8\x13\x04!!\xf1<\xca\x93,L\xd3\xd2}\xf2\xbf@\xe0\x18BK\x18B\n\x12%\x90\x04\x10G\x10I\x94\xc3H`\x10(\x89\xa0\x18\xc6\xe3\xa2\x80\xf2\x12r\xff\xbc\xfbT\xc7\xf7#\x08\xff\xdfA\x81\x7fZ\x04\xe1\x1d\x17\xc6 \x02\x8e\xe1\x02J \x14\xce\xf38\x87\xf1\x0c\x0f\x9c\x08\x81y`\\\x18\xd0\x1c~\xd7\t\xc3\xd2\xb4\xc8\x89w\x90\x12\xc1P\xc0\xc4x\x02\xc7\xe9\xdf~\x8e |\x81\x9e\xfek\x10\x84\xff4\x17\xefu\x08\xc2\xfb\x9c\xb8\x7f\x88 \xfc\xfa\x8e\xfe\x89\x08B\xfa\x7f\xc2\xc4\xafA@\xf4\xb6\xd6\xb6\xa0\xb7\xdb2\x94\xc4Oq\x15]\xa0\x13\xea\xcd\xbdU\x9fR\x9aEl< \x88\x0c\x9e\xb8\xe9\xd4\x9fV\xd7\x93\x15\xd0\xd2\x86F\xd6$n\xe0\xb0n\xd7b\xbb\xde\x96\xd7I6\xd4p\xf0h\xf8\xbaQ\xd4z\xff\xe4\x84\xd3W1\x08AZ\x00\xa1\x92d\xe0\xc7\xa9\xbc\xb5\xb7\xd0r\xe1\xc1\xd3\xd2\xe3s\'\xd11\xbe\xc0\xd2\t\x89\xcc\x05\xa8o\x1au\xf5f\x1cg\xed\xc0e;ki\xd6\x13~E\x82a\\\xb3un\xc3JC\x9b\xbe\xb3\x0eO\x1b\xed\x0c\x8f\xd4\x1eF\xf2\xbag\xe5&}r*\xef\xab\x10\x84@J\x02\xbc\xf2q\xber\xad{\xc4\xd5:v\xbc}\xe3b\x8efx\x81\xaa)n\xc7\n\xa1\xe1\xe7\xf6N`\xd2z\x97\x9eK\xbf8\x0e-\x17i|@\xcf\xa7\xb1\xbc\xf9\x9b\xb0\x91\xa1\xc2^Q\x88\xbd\xaa\x98\x85\xdeO\xcem\xe9\xc9%\xbeA\xee\x93\x94\x9eW!\x08\xef\x9a\x04\xa1\x1a&\x19\xfaa\xba\xe9\x12m\x95\xcb\xb8\x11\xcf\x02\xbe#\xa80\xd4\xe9)-#N\xdf\x0f!\xa9a\x98\xc8n\x0f\xa1s\xda\x8a;?K\xf4x\x13i\xe6y%\xc6\xfa\xea\xe49\x90\x9f\xb9q^!=?\xd6R\xabIcR\x08aE\x86\xce\x93\xd3\xb2_\x85 \xbc\x8b\x89\xc0$\xcc\x10\xd4\xc3\xac\xda\x15\x833\xbdn\xf07}T\xb8\xfdI\xe2n\x87<\xb0\xa9+\xdco]j\xd7\x1a\r\xad\xf4!\xafK\x06>]\xdav\xc0O\t\x1b"\xc1\xe5H7E\x16A\xe8m3n\xdbm\xca\xb8\x86|[$\xfeX\xcb\x99\x83\xfejV\xed\x9f\x1bAx\xdfE\x14\x04r\x12\xc1\x1e\xa6\x9bZ\x13\xc1\xe8h\xbaw\\!\xee\xd8\x14\xad,\x14c\x8ca\xa3_\xa9\xcau\xfa\xb3\xc6\xdf\xd6Pv\xf3\x91\x84\x98\xea\x8c$%N\x9b\x93\x82\x9e\xaa"B\x11\xad:\xb3}%x\xab}@v\xb7n}l\xab\r\xeb\x1d\x19r\x17E&\xbbi\x84\r\xb1\xf2\x18\xb6>\x17n\x0f\x9b\x88\xf0\xa4O\xbc\nA\xf8\xe1\x13\xa0RC\xa9\x87 ~\xec\xe7\xc6#\xb954\xad$\xde\xee\xb7\xed\xca\x91\xd7\xd5\xc1YDe\x7fL\x11\xb4\xbf\xb9\xd3\x14\xc2\xea\xc5\x99\xb3y{\x9dhz\xc1J\xfbz>b\x86\x9d\x9d\xf3\x12\x9e.H\x94b<\x82\x96\xb7\x1d\xb44\xcam2\x9eLU\xaf"\x10\x02)Q\x9ad\xd0G\xcc\xe2\t\x95\xf1\xdcg\x03\x9f\xde\xc5\xd9M\xc1\xae\xdc\xbe+Q\x16\xd9\x9d\xc9I\xcf\xf0\xd3l\xb7\xd9\xc9Z\x0fn\xe0\xf7WK \xbcb\x07\xeb~3P\x8e\xd9\x11\x92\xe9\x16\x82(ob\x9b\xbb\xf4\x0bM_0f\x8bO\xdc\'\xf1\x07\xef\xaa\x84i\x8aF@u\xf4G1\xc5\xcd\x903\xbb\xd3\xf9\x02\xeaL\xf2v\x8a\xa8\xe6\xcazm~\x15\xc2"Pe\xa3\xa6\xe5\xd3M\xdcM\x9d\x02\x81\xc8Z\xcc\xe4\xd2\\\xcc)c7\x07\xf20\xb9kv\xb6\x07l\x1a\x05\x92\xc6\xa7\xa9f\xd4\xcc\xda\xa8Y\xfe\xa4\x98\xaf\xe2\x0f\xfe\xb1\x1b\xfd\xbb\xe9\xca\xb5\x07\xd2\xf2\x94\x89Z\xeb\xeb\xb4dC2\x1f\xcf\x82\xd2h\xe1\xe2\xa2qi\xaf\xd6\x91\xaa)\xe2\xf5\xea\x98\xca\xea&\x10{\xeeRU\xd5\xb6\x0e\x92\x1d\xd6\x1b\x81\xac\xb6v\xe5\xea\xeaF\xca\xceW4\x12\x0e\x17\xddJ\xfe=\xf9\x83wc\xb9\x13\xbe@\xb7\xfb\xe0\x13{K\xae*}\xa0\xfdd\x93\x04\xeaa\x1d\x12\xb9|\xeck\x8c>\x1c\xe2u\xc5\xcc\xdc\xfa\x92\xf7\x9d6\xd3\xf3\x18\xed\xb8\xb0\xe2\x91\xae\xdf[z\xe3\xcf\xacB\xe7A.f\xe8\xe6\xd6\x95\x83U\xad\xc3\xc8Jqv\x02\xe43O2\xa4_\xc5\x1f\xbc\x8b\x89R\x0c\x85\x83\xeb\xfdQL\x88\xdc\x1e,\xa1\x13\x8b\xed%\xe1\xac\xc3\xf9\xb0\x99o\xb2f\x1b\xa6\x0b\xfb\x9e\x1c&\xcc\xf6\x00\xbb\x92\xb0E\xe6\xd5\xe1&l\x9a\xfd\xed\x84`\x85vdc\xd7\x1ahF\xbaX\xa6vDPM\xcf\xec\x86\x17\xd4Z\xcb"\xe3\xdf\x93?\xf8\x11@\x11\x04\xa5p\xfc\xd1\'\x0el\x08\x0cE\x84j\xae\xb5\xe5\x15\x7f\xdb\xef\x11\xcc\x83\xd6y\xed\xd29LqCQ\xc3\xa2\x10\x8f\xbb\x94\n-\x0eI\xcd.\x88k\x155\x1a!\x1cK\x99+\xa5k\x81&\x9a> \xe1\xf9\xcc\x18\xaeR(\xb1\xfa$\x80\xe3U\xfc\xc1\x8f\xca\x06\x86A)\xfd\x90&`t\xbf\xaaN;\xd2_\x99\x08\xc1\xac\x9b\xcc\x8aB\xe8\xa0\xa0Z[\x96\xc0\xe2\xfb\xca\xc0\x0c&\xb4\x9c}\x13#\x9ce\x19TrXz\'\xe0\xdb\x91\xa6:O\'\xf0\x86\x8e\xaf\x136\xac#z\x85\x94\x90\xd4\x93\x07\xfdI\x97x\x15~\xf0\xa3L\x05\xfd\x02\x8d\xe3\x8f4\x95\xd1\x1b\xfdzm\x1e\xcd\x82LK:\x11\xf3\xe0\xe8\x06\xe5\xe9jj\x8a\xc2/X\xaab\xd7\xd3fg\xbaKk\x8f\x90\\\xc6\xb79\xdd\x8c|>\\\x0e[\xb3\x1fp\xd40V\x1d\xceIXy\xb1%\xd8\x9e\n\x82\xc0\x9f\xacl^\x85\x1f\xfc\xb0Y\xe0\xf7\xa0\x88{\xa8\xc6A\x1d\x06\xdb\x99\xbc_/\xb2\xb7\x9b\x934`\xf2\x1dDo\x90\x05/,\x8d\x88\xe9:2\xe3\xc8\xadQ\x9c\xb5\xcc%*\xa5_O\\\xef\x8cWd\xdedz\xed\n\xf0IE`\x89\x17Vj\x88\xd5G\xfeI*\xf0\x8b\xf0\x83wm\xc2\x04\xce\xd0\xd4\xe3\xd1\x919\xa0\x01\x9fq\xaex\xc0\x0c\x89@\xe6\xa5\xa8\xe8H\xef\xdb\x9d-\r\xd7*j\x91\x02:\xacT\'<\r<\xc9e\xa7\xeb\x11]\xc7\xe2\xad\x11\x14\xc3\xde47\xab\x8b\x8a\n\xcaW\x04v,l\xa5`\xf6a\xabY\xe2\xb3\xc5\xc7\x8b\xf0\x83\x1f!\x08\xe4vP\xd4?\x84 )=\xf0*\xd4\x04\xea\xa2\nkz\xe6\x1b\xd2\xd9D\xcb\xa5\xc0e\xbe\x83N+\x01\x1a\xea\xa3\rmQ\x86\xa4\x81x7t?R\xa2f\xef\x06\xa1r@\xeb\x97R\xc7\xbd\xaa\xb1ks1\xa34w\xcd\xdd&3&\xffI\xdf|\x15~\xf0\xaeMP\xba\xa0\xe8\x83\xc9\xee\xf2\x8b\x1c\xa9U@\xa1\x8a&/;G\xe0\x8e4\xc2\xae\xd2\xb64\xdbb93z\x12\x08\x9eX\xa7\xb6\x85\xd9\xe1\xfa\xb0\x84g\x05?2h\xed\xc5\xd5\xac\xe5\xd0\xd5Qvm\xde\x05\xa1\xcb\x9d\xf7v\xcaG\xcb\xad\x88\x9e4\xd9W\xc1\x07?\xfad\x92bp\n\x7f\xe4\xe6\x11\xf6(\xcb\x83\xdcm\xa2A9Rh\xd9\xd1\xce\xc9\xdb\xb1\x9e\xd2\xb2g\x9e\n\xba\xbe\x10v\xfdMv3*d.bh\xe4\xb6\xc4\x9dw\x95j\xa9\x98\x06\xdbj\x9b\x9fL}\xad\xb3\xa3W\xd1\x07\xef\xa6B\xc34hi\xe1\x07ci9\xa6\xc1\xbd\x92v\x89\xa3\xe2\x1d/\x8e\xae\xf0\xd9\xae\x8b\x9a\n(}\xcbm\xady\xe1\\\x8d\xday\xab\xf5Fk\x8c\x15\x03\xd1\x839\x97\x8b$b]\xbf\xea\x96^3K\xabNn\xb4P\xd4D&\x06k\x16\xc9\xb7Ob\x94^E\x1f\xfc\x91\x94)\x82\x86\x1f\xbb\x0e\x15=17?W\xb6\xc7}S+|\x022C\x82\xad*G\x10\x07\xf4\xba\xe7\x8dl\x8e\x8fe\x98\xaf\xe6&\xccI\x14\xcaZr}M\x95\x0e\xd8\xec\x9a\x0e\x96\xa3\xc5\x18\xd9\x0ef\'\x1f\xc4B7\xbc\xd4\xfc\n\x13\x9e=O}\x15}\xf0CL\x14\xb8>E\xf6\xa3X\xb8\xa0\xf5\x93\x88\xb3W\xd1\x07?\xc4\xc4H\x82\xa0\x89\x87\xda\xa3\x1e*\xba\x18u\x1f\xd1R\x9d@\xf7{_\xcd\xeb=\x94\x1e\xce\xb8\xa9\xba\xe3\x8e\xd3r\xabjP\x84nE\x0fG\xaa\x95\xbb!6\xb45\xb0\xda\x89W\xcf\x18\xeb\xb6\x86\xd1/5\x84\x8f\x87=\xdd\x86\x8bn+\x07/\xe3\x9f\xab=^\x85\x1f\xfc8\x11\xa0i\x1aA\x1e\x8f\x02=*\x05Mx\xb3\xdf\x9e\x82\x809\xb50\xe8\x1b\xfd>\x13\xd4%\x90-\xf7Z@\xf3ejR\x11\x14\xc7\x17^9\xac\x97CDN6\x03\x05\xcc:\xba\x8eA\x90\x8f}s\xdeJ\x96\ri7\xfcr\xe2\xbc\xce\xf4\xe5\xf6Y`\xdd\x8b\xf0\x83w1\t\n\x06;\xf6xJ\x863\xceI\xbc\xc6\x13\x85\xa4\'%M\xe0h\xd4\x17h?\x1e\xafS\xa4\x14\x0b>\xe5(sC2)\xc6\x8b\xda5\xee\xdcs\x1a4\x08z\xcd\xf6\x1b\xd2u\xf0\xd8)*\xbc\xaf\xea5|\x19\xf6\x86\xd2\x1f\x1637x\xcf\xd5.\xdd\xd9\xa2$\xb8\xde\x1f\xa9A\xad\x96th\xbb\x80`\xa0\x13\xcfn\xd8\xf3\xd9\x82\x93\'\xbb\x9fW\xd1\x07?\xeae\xb0c\x08\x8a=\xe4\x93s\x95\x1d\xe5]6\xd2q\xdf\xd9\x19\x8c\xfa\xd8\xac\xc5\xabL>hQm\x80\xc2\xbc\xe4\xe0\xa2\x89\xe0\x1d\x86\x8b\xeay\xa0\x9a\xbe\x17\x0f\xca\xb9\x15\xcf\xee5,\x1dH\x9d\xe3"\x9c\xc5\xbaN\xda\xddZE\xf2\x19\xee\xd5\xcd\xaf\x0e\xb0\xff\xdc\xf4\xc1\xfb.\x92\x18N\xfc\xec\xfe\xdc\x81c\xa5D\xeeg6\x98ln\xa3^\x0f\xe73\x93\x84v"b\xca\x99\xaf\xfc\xe1\x14\x18E\xa6\x8d\xa5X^5L<\xacq\x81\xa7\xab\xed\xa1\xf68\x85>$\x04\x8ct\xae\x00v\x99\x9e\x8f\xb9\x98@\x8d5\xd2\xf4\x9e\x1f\xff9\xfa\xe0\xc7\x13f\xdf\xf4\xc1\x7fz<\xdd\xbf\x11o\xf0\xd5\xc0\xa1\x87\xbd\xfa\xba\x94\xa77M\x1d\xfc3m\xe9\xab\xf9y\xef\xda\xd2\xb7\xaf\xf3\xeb\x12\xfc\xde\xb4\xa5\xef_\xe7\xd7%\xef\xbd\xcbJ\xdf\xbe\xce\xefX\xfauc\xe9\xcb\x19q\xefr\xfc\xb7\xaf\xf3\xeb\x92\xea\xde\xb4\xa5\xef_\xe7\xd7\xa5\xcc\xbd\xcb\xf1\xdf\xbe\xce\xafK\x88{\x97\xe3\xbf}\x9d_\x97\xee\xf6\xa6-}\xff:\xbf.\x99\xed]3\xdb\xdf\xbe\xce\xafKU{\x97\xe3\xbf}\x9d_\x97\x88\xf6\xae\x8c\xff\xf6u~]\x9a\xd9\x9b\xb6\xf4\xfd\xeb\xfc\xba$\xb2\xb7\x15Q\xef^\xe7\xd7\xa5\x88\xbdkK\xdf\xbe\xce\xafK\x00{\x9b\xe3\xbf{\x9d_\x97\xde\xf5.+}\xfb:\xbf.y\xebM[\xfa\xfeu~]j\xd6\xbb\xba\xa7\xb7\xaf\xf3\x9bx\xf5\xf2u~]\x9e\xd5\xbb\x1c\xff\xed\xeb\xfc\xba,\xaa\xb7\xdd\xce{\xf7:\xbf.G\xea][\xfa\xf6u~]\x06\xd4\xdb\xee=\xbd{\x9d_\x97\xdf\xf4\xb6\xc3\xbdw\xaf\xf3\xeb\xb2\x97\xde\xb4\xa5\xef_\xe7\xd7\xe5&\xbd\xebp\xef\xed\xeb\xfc\xba\xcc\xa3w\x95\xfao_\xe7\xd7\xe5\x15\xbd\xad.}\xf7:\xbf.k\xe8M[\xfa\xfeu~]N\xd0\x9b\xb6\xf4\xfd\xeb\xfc\xba\x8c\x9fw\x15Qo_\xe7\xd7\xe5\xf3\xbc\xcbJ\xdf\xbe\xce\xaf\xcb\xd6y\xd7\x96\xbe}\x9d_\x97\x8b\xf3\xae-}\xfb:\xbf.\xd3\xe6mu\xe9\xbb\xd7\xf9uy4\xefz\xda\xe4\xed\xeb\xfc\xba,\x99w5\xa4o_\xe7\xd7\xe5\xc0\xbc\xed\xc9\xbdw\xaf\xf3\xeb2\\\xde\xf6\x14\xf4\xbb\xd7\xf9\xaf\xb1!\x96\xa7\xd8\x10\xf3\xdf-\xe9K\xf3W\xde.\xd8g\xf1W\xde.\xd8g\xf1W\x9e\x13\xec_\xc0\x94|\x16\x7f\xe5\xfd\xa6\xf8\xcd_y\x0f\x7f\x05\xff\xcfM\xff\xc7\xfc\x15\x1e\x13Y\x8a\x90PZ\xe2a\x9a\xa6\xee\x93\xcci\x98\xc3aB\xe2\tT\xe4X\x98\x17\x08\x92\xa3\t\x86#\xee\xf3\x7fy\x14D=\x86aQ\x1a\xa51\x84\xfc\x98\x03\xfck\xb0\x00\x8eS\x12\x8e`\x12E\t\x08I\x02\xa1$\x8aA\xc1\xb5Y\x18|(\xc9`4+"\x04/p\x0c\x87\xa18+\xa0\xcc}d\xcc\x1d\xdb@H4\x03\xc3\xf8[\xf9+\x7f\x9d[\xf2\xdb\xcf\xbe\xd1O\xfd\x05l\x06\t\x138\x8d\xfc#\xfe\xca\xd7\xa7\xd7\xfc\x84\xbfB!,\xb0M\x02G\x04^\xa4qD$q\x96\x84\x81\x11P\x92\x04\xd6\x84\x8b\xa8 \x01;\xe0\x19^\x14>f_b4\x8eb,\xc6\x8b,\t\xb4\xc5\xdf\x8d\xf5\x9b\xbf\xf2N\xfe\n\xd8%\x8a"\xc0\x8bYRd\t\x96\xe4\x18\xb0\xaf"\xc5\xc14FJ$&I$p*\xf6c\x081I\xe3\xacDH\xe0%8x\x8d$\x00##~\xfb\x93\xf3W\xfe\xe9y\xf2\xaf\xe3\xaf\xdc\'e\xfcC\xfe\xca\xd7w\xf4O\xe5\xaf \xe8\xaf\'\x84\'\\!\x1a\xb4\xa9\xad\x959\x0f\xcf\xbe\xa9\x9f1\xe6\xe2H\xf0\xa09\x02\x1fb\x0c[B\x16]\xc4\xfdf\x9c\xbb\x95~.f\xd0\xa4\xe9h\xc1\xb3\xcc|Z\xf9\x92\xcf\xcc\x81\xe7\x06\xcc&\xc3\xb2\xa0\xae.R\x89\x04\xc1\x93#\t_\xc6_\xf9CZ\xf8;1s\x07\x91[\xd3\xa3\x1bqT&\xe8\\Q\xbb\xc3\x81\xf0|)\xcd$\x9c\xe8\x0c&\xbc)\x94\x82m@\xe6S.~>\xf0>b)\xe3\xa2\xee\xd3\xb1\x9aR\xe6V\xa0\x82\xa5B\xcbf\xae\xaa\xec\xb2_\x9f6\x92\xaf?9\xe1\xf6e\x00\x16\xfa/\xf7Q;\xccO\xf8\x048\xdaP\x84p3\xb2\x16:d\x16\x90\xd4&\\Y\xd8\xa4\xf86\xc5\xc3D\xa0\xdd\x88v\xa1]E\x9a\x9a\xec\x18)oz\xee\xe2\x978"S\x125\xe7\xb9\xb3\xc2\n\x94C\xc3\x8c\xd9Y\xd4:(\xc4\x96\xbb\xfej\xec\xe2\xbb\x01,@\xc6\xfb\xec$\x8ax\x1c\x18(\xaa>\x85\x1c\xe8m\xa4\xac\xf6\x9c\xbf\x9e\xf3\x858\xd6 o\xc0\x90yb4\xc1\xba\x01G!\x17nY*\x05\xber[&\'\x8e\xd3\xb4\xa5\x99\xabl\xd4\x97|\xd6\x82".\xd8B\x80B^\xa5\x9b\xc4co\x9b\xf3\xf1\xc9\xd1\xfd/\x03\xb0P\x7f\xb9#Hp\x10?\x1e\xc6?\xa2\xb9\xdb*Ew\xe6\x82\xba\xb0cIE\xae\xb4K\x86\x04&\xcd=\xc6\x16\x0bQ\x16~p\x89\xbc\xf8\xc4\x1d\xcch5y\xc8\xd5\xdb\xe4\x95sI\xdb\xfb\xd4@\xb2\xbb2\xfaf\xb4]\xb3\x8d+1\n\xfa\xc5\xb1\xe2\xe1W\x8e\xf9\'\x07\xb0\x80]$\x19\n\xd4\xf3\xcc\xc3XY\xe4\x14\xb1\xa85\x04K\x12\xcb3\x83\xf9\xd0(v\x96\xc9\x8d\x16\t5\xb0\x80L\xa4\x89\x9d\xa3]n\xf4i\x05\xd9\t2\xe2\xb2h\x96\xb9\x97\xb4\xc2\x01\x9a\xa0\xbd\xecc\x0c\x19\xcc\xa7\x91\xe4\xb4}\x17mE\x116NO\x0e\xf0~\x19\x80\x05\x88\x89Q\x0c\xf8\xf5\x1f\x85\xe4\xdd\xbe\x9aKu\x92\xf7\xc1h\xb0gAX\xf9\xe7\x1dT\x86M\xbfI\x88\xa6\xc06\x12w\xe8\xe1\xb5M\xd7r\xe4\x8a\xfdy\x9b\xa1nK\x8c1{u\xf9ZN|k\x83A\xd7\x85\xdf\xe1\xb9\x9e!r"Qn^<;\xbf\xeeU\xf8\x15\x10\xc3\t\x1c\xc7`\xe2\xd1#\x8c\xd0\xd9\xb2\xe6>\xa0\xd0a\x17\xf1\xb0\x98\xcd\x9el\xf4%/\x0bx\x16i\x02\xa5\xc3\xb5\xb0\xb1\x99 \xa0g\xf7\x1a\xaf$O+\rU\xf3\xf3\xdc\x1a\xf83\x14\x08\xf9\xea\xd4\xdd(\x1d\x8dh\xb6\xb7\x06\xa7\xe1P\xe7\xfaY\xfc\x95{\xaa\x02\xda\xa4\xc8G\xf0\x92%\xae6q6e\x07\x7f\xb7\xb2\x87\xd9Yt3\x15\x0b\x0b\xa3\x9b\xc3M\'\xa0\xa9\x9d|y8\xb3\xad\xdd\\\x1b\xf3\x80\x89\x97\xb0\xf3\xc6\x83\x83\xef\xf4\xf5N\xb99\x87MK\xe9]\x8e\xad\n\x9c\x0c\x18\xde@\t\x05}r\xb4\xe4\xcb\x00,\x1f\xe1\r\xd4\xd1\xd8\xc3\x88`\xc6\x13Yo\x12p\xc7\xb1\xb2\x19\xdd%3\xb5(\x8e\x05\xc5\'z\x9dbw\xbeD@H\xdaMl+\xa1\xe0q\xb6\x1c-l$r\x92\xd0g\xc2inb\x17!\xee1\xc59\x92V\x10\xd9`\xd7\xa5\r\xfe\xcdxr\xb2\xe4\xcb\xf8+@\x97\x08\xfe1"\xfaa\xbe\xdc\xc9\xdc\xb8\xe6\xae+S\xab\xe1/-\xac\xd8]\x93\x0e\xb8\x1d\xd7\xd7\x83Q\xcd\x8dG\x17V\x8e\x11c\x19\x9c}\xe3\xcc\xcc\x04Q\xa6X4X\xd4\xcd\xb98\xba\xe0"\x17\xd6*\xf3\x93p\xb8\xdd\xfa`]\x86\x14\xd2\xae\x97\xaf5\t\xf9e\xfc\x95{\x10\'pP/3\x0f\xbb(e\xac\xd1\xdffEm\x8f8\xa3$\xc3zK\x87\x87\xbd\x85\xab(\xeeQ\xb2\xbb#\xa1\x89\xa7*\x19o<\xae\x85\xc8\x0bV\x96\xc5\x8d\xba\x8d\xcc\xa6\xef\xb5*nN\xf0\\;\xf0!\xa5K[+\x91-\xbb\xb5\r\x14y\x16O\xf0*\xfe\n0\x16\x06\x88\x08\x12\xdbCa\x93\x9f/\xa2J\xf1~`vG\x9c\xd7\xd3u\xec\x87\xeb\xf5\x95\x99\xd6}\x8f\xdeJH\xbb65\xcf\xe5\x84\xe0\x8b\xd2\x95\xb9\x04\xdb=\xe1A\xd1\xda1\xdd\r\x17\xd6\x8e\xd1BTJ\xca\xed>\xcf\xae\x9e\xd5\x8b\x85\xe8\x94\xfe\xb35\xea\xab\xf8+\xf7\\\x85`0\xd8\xb6\x870~&\xf5pb\xd9\xb9^\xd9{\xd3J2"c\xd8N\xa0y\r?\xe0g,\xd8\x99\x89\x146r\xab\xf9Gv\xd2\xb6\xd0\xd2\xe2\xed\x11\xf6J\x82\xe5\xa4\xa0\x8aE>\xa2F\x96\xc6\xa6\x86\xd9^\x83\xc8S\xd8\x1a\xcf\x88\xafU\xd8\xbc\x0cL\x02\xaa`\x84\xc4\xefH\xc9\x87\x00\nI\xc3i\xa6Kn\xce\x98\x85\xcd.\xf2\x04\xf7{fJ\xa7\x1d}\xa8\xc8z\xbe\xa8\x94\xaa\xa5y\x13\x90\xa2^\xadi\xc4\xc62\xb4N\xf1v\xe9\xfd\x04\xdd#i\x9b\xebD\xe9\xce\xbc\xe6\x10\xb8o\xc9\\\xee\x96\xf2\x96{.\xe7\xbf\x0c3s\xcf\xf9\x04H\x12\xa0\x8f\xfe\xa3\x98\r\x08-\xd5\xecC0U\xad+\xd2\x8a\xb9\x19r2\x012\xac\\E\xd6\xac\x81\xa2!\xa6i\xb5\xdf\xa7)s\xedv\xcarM:1Z\xc2\x85lM\xf4\xd6G\x8c\\G\xee\xe1\x96\x11\xfbxbP{=\x06LX>\x8b\x05x\x15\x80\xe5\xaeM\x90c(\x02}\x88p\x17\xbaF \x05i=\x96\xbf\x19g$\x96\xd2\x92\x1a\xd1\xe2\xc6\xef\xb5\xac\xbe\xb0\xabzW\xd0(\x840[\xf6\x96\xe2\xd0ys&\xf3r\r?\x8b\xd3y\x15\x80\xe5^\xc3\x01\xff\xbe\x93\x1f\xff(f4\x8d\x8a\xb5;\x81\xccX\xb4\x10\xba\xa5\x0e4t`\xe0")\x11\x8e%b}T|\xf3\x1a\x8e\x03O\xe8ir\xb9J\xe1\x86\xc4\xdb\xf8\xbc?\xd1\xb7\xcd\x18U6u\x91\xdd\x15\xae\xd3[\xe0\xc1\x14\x15\x95\xe8\x8e7\xf2\'#\xed\xcb\x00,w\xdf\xa4I\x02d\x94\x87\x84B\xf3\xe1i\xdc\r\\L\xc0\x99\x1fM\n]%\x9a\xa1Xg\xf7\xbc\x1d\xa5\xc8\xbc\xac\xa8x\x85\x8c\xeb\xa6\x85\x14z\xbb\xe5\x16\xe86[\x13\x96\r{I2\xfa\xf3\x8a\x17\x83.?\x1f\x85\xcb\xb8\xaa.\xe5\xacvrI\xae\x9fE]\xbd\n\xc0\x02\xb4\x89\x01\xfb\x06%\xe8C\xf5\xc1\xcc\x1b\xb3`\x0fT\x1aMz\xde\xd0\xd8ew\xd9\x89\x82\x14E\xe7\xcda\x17\xdcZmI0G\\h\xd0\xad\xdaG\xceS\xcc\xbd\xb5x\xa92\xb7\xcb\xc9\xed\xa9\xfa\x90\x90\x92 \xa1};\x81fK\x1a\'\x1af\xaf\xda\x93E\xd6\xcb\x10,\xf7"\x0b\xc6\x90{\xdd\xfcp\x16\xe8\xd1b\x15\xce\xd1v4\xd0\x05\x8e(\x89a\xe4=v]r\xd0\xe0\\\xc8\xabT\x9e]\t\x1d\xd1\xd2@.\xca\x99\xd9\xf7\xf9v\xcdf\xd3U\xafO7\xce\xf2\xda\x13B\xda\xbel\x16\x9bx\x8c\xe2\xf6\x9a\'\xc9q\xf7K\x8c\xe7\x9f\x1c\xc1rw}\x102P\xe2\x81\x14h\x9b@vb\xd8\xaer\xfc\xd2\xaa\xe4!\xb5w\xeb\x9e\xc8\xcb\x8d\x84\x9e\xb8\x9a\t\x87\x0e^+\xb4\x16\x1dm6\x1b\xf8.\x1c\x8e\x90N\xad\'.\x1d,L\xb0\xf7\xb7\x81"\x8f(j\xcd\xb8@\xf9\x83\xe7/XH\xe6O\xc6\xf1\x97\x11X\x80\xad\xc04\xc8\xed \xbf=de\xd1\x14\xd3\x19\xdfk~y\x9c\xecR\x14(rw\x91\xed\x1eI\x05\xe0\xfad\xa1\x9d\xb4\x8a\xe8o\xb2\xdf`\xe7\x8eg\x83\xfd\x9c-\xa6sr1\x8f\x86\x86T\x9f\x85eIv\xd1\xc1\x17\xd7\xce\x94\xdd\xdc\x80\xd1\r\xfcI\xc2\xd4\xcb\x08,@\x990\x8db\xf7\x93\xf4\x07\x90\x16\xdeDK\xef\x12\x8eIgR\xadn\xe3\xb52j\xeej\x14\x05\r\xbd\x92\xfa\xbcp$\xba\xbeD\xc18j\xc4q\xde\x84\x93\xb3\xad\xd7\xdb&\xd6e\xa1p/\x89\x13\xc3\xb3\x10$\xa3bvN\xb6>\xc9\x1a\x16\xd9\xd2\x93<\x9d\x97\x11X\xee\xb7\x00P\x90\xea0\xe4\x91\xfeV\x1d\xf0#\xcaY\xb1\xa3\xaa\x9d[\x90\xee\xd0K\xfdU\xae\xbd]\xca\xc3\xa1\xda^\x8a\xa0\xde\r#\x82\x9e\x03\xb7\xac\xb1XG\r=WV\xb3\xc5\xed\x88\xb0\xb8\x1a4#\xd1\x93\x98\xf6\xae0i\x91\xa3\xf5\xab\x15\xe4?i\xb4/#\xb0\x00mR8E\xc0\xf0\x03\xfe`\x96xkNs\xd0\x1a`|\xb7\n\xa4cr\x0e\xa4\xc3\xc5A\xf1\xc3\xce\xdc\x0f\xcbR\xd7m\x0f\xa1R=`U7t#\xceLD\xe4\xf8d\xdc\x87\xa7\xbdX\x19\xcd\xba#TM\xabV\xe5X#e\x07\xb3B\x85?\t\x9b\x7f\x19\x80\x05(\x93$\x08\x02\xc7\x1f)\xec\xa5\x19h}7\x13\x861\xb7%s\xa5\xb0\xe4\x94\xbb\x81\xef\xf3R\xabW\x91Qx\r\xb59\xd8\xf9\xe6F\xc6v~\xc1\x1d\x97J+\x02v\xc4\xc8\x9a\xe0\x82\x94\xcd%1.\xaa\\W\x87~Y#\xe3A\x1e\x10\xb8~\xf6d\xf7U\x00\x96\xbbkR\x0cyo\xa2\xfe(fV\xf9\xbc\xd2%\xa7R\xea\xc8\xfd\x90u\xcb\xd0O\x8d\xb4fl!\x87\xb8\xe3EY.\x96m\xc2\x07\xc7qC\xbec\xc5\xad\xe7\xd6V\x9a6[a/\x08\x1a\xc8\xd2H\xac\');K\x81\xd5O\xc8\xcd\x8e*\xbc\xf0\x9et\xcd\x97\x01X~\xd8,\xe8\x0b\x1e{v\x03\xedo\x83M\xcf\xb3(M4\xbe%F\x95T\xcb\xcdM\xf6\xb6\xe4\x88\xf0z\xacf\t]\xda\x97\xb0\xe4\x0e\xd9Y\xdd\x88\xed\xae\xaa\x0c\xb1=y (\x19z\xd3\x9f\xa4\x119\x94\x14\x87\x87\xe3:uv\xc7+\t\x05\x9f\x05`\xb9\xf7\x05\x14q\'\xc3>\xb4?\xbbK4\xc4b\x7f\x18\xb5\x88\x1dZ\xa1)d\xb8\xd8\xc1\xde\x90\x06\xe4\xd6\xa7\xe0j}\x9aj\xf4@\x1e1\xbd\x10h\x9a,}H\x89[ic\xd7\x13\xbeh0\x9b\xc5k\x8f\x0bIq\xf6\x82\xeev&\xa6\x9b\xe0w\xd1\x93\x05\xf3\xcb\x08,@\xcc\xfbA\x06F>\x1ea7\x8by\xb9\r\x02\x83\x90\x98\x7f\x82\xfaqF\xa4n\x0b\xb2dr\x06\xed\x0c\xc2F>\x13\xa1\x8e\xc5\x96\xa2\x8e.t\xc1\x90\xdb\x93\x11pe\x11n\xf1\xd8\x9c\xa1\xa5\xdb\x07,\xac\x18\xdc\xf9\xa2@\x86y\xec-\x13w\xfa\xafUb\xbd\x8c\xc0\xf2q\xc7\x9a\x80\t\xe2\x91\xc6\xbc\x9bsC\x0f\x8d\xe1\xc6\x96\xb6,\xc5{\x89\x81Q\x93\xe3]h}\xech\x1e\x12\xaa\xb5/z}\xb7C\xb7\xce\xb8\x8d\xf8\xc2.\xad\xc4\xbf\x95|\x06\x89\xb6j\xb5\xf4U\x10\xf3P.6\xea\x85\xe9C\xbd\x19\xf4\x1d\xe8y\xffI\x02\xcbGh\xfe&\xb0\xfc\xd3\x8f\x82\xff\x1b\x11X\xfeA\x01\xdb\x02\xb5\xc9\xc3\xd8E\x88.\x17R\x0eT\xcdD1\xe7Z\x1f\xd7\x8d\x93_m,T\xd9\xae\xc5\x98\xd3U\x9cq\xd0\x8a\x8e\xb0\xb1Z\x8f~\xba:\x94]6\xfb\xe1\xcd0\x8b\t\x81\xb6-\x97\x1dO];\x98S\x9f+\x1dz\xae\xca\xd2\x8f\x9f\x9c\xed\xf6*\x06\xcb\x1f\x8b\xfb\xbf\x15\xd3K\x03\x1c\x9d\x14\xdd\xa3\xb7\xa8VKjt\x08\xf0\xa0\x81\x0c\xb6;3\xd0\x9a!4*\xbe\xac\x96\x95\xb4\x91\x94\xc0f\x07\x84\xe5\xd71q\x16}\xad\xb9\x92\xe4j\xdfj\xcex1\x9a\x05\xba@rXlS!\xc8\xe0\xe6I1_\xc5`\xb9\x8b\x89\x91(I\x82\xd2\xf6\x01\x1b\x94\x85\xa8\xb6\x11\xf6\xdb\xb6\x0eZ\x19\x85\xeam,\x0c{\xb1\x80\x89\x93\xe1\xf2>=X~\xb2\xbff\xaa\x12%\xdaa\xd1\x90\xa9\x1f\xf7\x8d\x96\xeeSa\xe9\xad\xb6\xf7l\xa4\xccO\xc3e\n}\x01M\xb6\xe7\x82\xe4|\xf6\xc9\x9c\xfc*\x08\xcb\x0fmR\xa0jb\x1e|SS\xf3\xd88#\xe6\xb0\xce\xfcu\xd8\x95\x91\xd7\x9c\xb2\x0b\xb3\x95\xd9\xbc\x9c\xa3\x84\xa3\x92\xd5\xacn"n$\xe3b}\xc5\xdb\x828_\x8fs\xbf\xcbO\xfe\x94\x18\xbaO^\xce\xa7C\xde\x8eI@\xe7J1\x05\xf6&\xa2\x9f\x9c\xbb\xf8*\n\xcb\x87\x988\x06v\x06{\xd0&\xe6\xa1\xc62*a\xe5U\t\x83z\xa1\xaa!\xe1E\x04\xe9\xb1T\x8d\x9dFD\xe0\x86O\xa2\xb0|\xf8\x04\x03ZJ\xf8q\xeeb\xe7\x14d\xb2\xef\x83~eG\'\xb6\xee{\x07G\x1aC\xba\x8c\x19\x8d\xb2[\xcd\xf1\xbdN\xf3\x08i\xe2\xd5\x0e1\xd1\x83\x93\x905\x06\x0f\x8a=\xcaj\x7f\x1d\xd2\xf4\x14\xf6\x92\xb3\xe4\xf0^P\x9b\xc9j\xa31\x1c\xa9_\r\xd0\xfdsSX\xee>Aa\x14\xf28K>\xda\xa0z\xccb5\xba\x90\x03\xaf\xefz\xa7\xd7\xe15s\xe5\x12\'\xbc\xe2\xc7}zv5?\x14\xccuzL\xba\xabx=\xe1\xc5:\xd9\x9c\x87\x00i\xaby}\xa0\xcf\xceY\x0ck\x87`\x8e\x05u\xd8h\x17\x0b[\x84\xfa\xc9r\xffU\x0c\x96\xbb\xa9\x90\x08H\'\x8f4\x8b\xb3a\xf9\xb3q\xcc\x0cWDn\x13\x9c\xcf\xdb2h\xd7\x13\xa1l\xe7\xc6\x0c<\xda\xd0\x07%\x9e\xa5\x9df\x98\x86\xc7B\xf7\xa9\xd4\x1czc\xcfR\xc5\x15#\xa3\xef\xd9\xd2\x9e\x06LY\xc5\xf5\x05\x03\x1d\x83\xb3]g\xf6\x93\xa8\xaeW!X~\xa8\x12\x01y\xf4qn\xf6v\x7f\x94\xe9\xba\x8e\x8ef\x7f\xec\xa25e\xe6\xf2\xda\xdf\xdb\xa9B\xe1n\xb8\xe9\xb3\xa0Ve\x98\x88\xa0&9Z\xea1\xa5\x16\xa9\xc0#\x86\xe9Mb$\xb0\x93\x93F\xd7<\xb0\xd2K\x06{\xc1I\xbe\xde\xa6\xad=\x9c\x9f\x85L\xbd\x08\xc1\xf2\xa1L\n\xa1\x91\x9f\x8c\x07\x17;\xfd\x08\x03w\xc8\xe6c\xbal\xb3\xd1\x85\xfb\x8e8AI\x159\x97\xcb\xb9C+\'\xc7\xe0\x03?P\xc5\xa6v\xdb!\xb8\x90\xee\xb6\x03m^3HT\xee\x0c\x08\x93^\x0e\x8egq\xa5(Y\xfdI\x97\xd6\x07\xaby\xd2f_\x85`\xb9\x8b\t\x1av\x84 \x1el\xb6\xb4\t\x10\xa8\xfb\xbd\x87e\xd1@\xea\xac\x9a\x04\xabu\x91r\xfd*\xf5\xdaJ\xba\x9c\xc3\xad\xacE\x14kw\xe4q\x97\xe5\x84O@z\x91jN\x8c\x9d9]\xa3\x95\x8eZ\x1f\x99\xbd\x1c\x81Bh\xe2\xf1v\xec\xb7\xa7\xab\xf0d#\xfe"\x02\xcb\xddfQP\xa2\xd2@\xff\x7f\x14sZ\x06\xbcYv\xd5\r\xb2\xec\xd1\xd8\x93\x05*\xdf\xd8\x8bNuJ\x96*8\x99\xb4\xe7\xd3F\x1a\x10\xd8,\x92\x9dc\x94\x9e\x852\xe4\x0c\x1f\x91\x0e\xa2*&X>\xb6\xbc\x97\x95\xc7#\xec/\xfb#p\xbfS\xc3\x1a\xe8<\xbc\xbb\xe5izJ\xd9\xd7\x06\xcb\xbb\x08\x96\xaf]\x19\xf9:\x8a?\r\x96\x07\x12\xd6A>A\xd5\xae\xc7\\\x145\r\xf6\xb5\xc1\xab`s\x8c(\xdaM\xa2\x08\xba\xdb\x19\xbdJ\xc2 J\x9c8qS\xed\xd2%v\x9a\xa3\xca\x07Qq!\xe5\xea\xb0G\x96\n\xba:L\x05\xdfp\x19\xd8\x96\xca\x17\x17\xf2w\x11,_a\x92 \x84\x90\xcf\xf7\xc7\x8f\x83\xecB\xf0}N\xe7\xe6H3R\xa4\xdd{\x9ej\x08i\xd1\xd7\x93\xcf\xa8\xb95\xb7i\xc3\xaaw%\x8b=\xef\xd8\xe5\xf2N\xd4\x19\xc1iH[U\xfa\x83\xdb\xdf\xb4\xdb\xc1\x0cp\xe6\x8a\xdf0\x07n\x17\x97\x07\x84\x17\xc3|\x17\xc1\xf2\x15\xe6\xf6\xd3\xed\xb0\xfct.0\x18s\x17\xa4n\xe4n\xa7eOU\xe1A\xd2\xa7\xe3\xe1\x1e\xca\xd3\x85\xcf\xd9\x9dr8\xe91\xd2\xcf#\x15\xe6g\xe0T\xfa\xf1R\x93\xc7\xb1o\xbcB\x88\xa9\x9d.\xb6Nlx\x08O\x1d\x0e\x07\xc0d\x00\xb1\x1cQ\xedEJ\xeb]\x04\xcb#L\x14\x86\xa8\xed\x0f\x9e\xc2\xe4\xa3c\xed{\xba\xea\x15\x08\x87\x9f\xdb\xb3z\x0b!\xf8\xa8\x01wPV\x1bY\x9e\x9c\x88G\xe3\xc9@\x923x\xad\x07C\xd0AxH\xdd\xa5\x8e\xe2\xd2`\xbb[L$\xd8\xf6\x13\xb5\x1fd\x8dTwk\xc2%\xa4\xfa\xea\':o2X\x1eI\x16\x84\x12$\x8e=\x1b\xbe\xf9\xfe~\x96\x08\xab\xc8\x0eW\x11\xca\xab}m_\x98x\xdb\xbcL\x93\x10=7\xcc\xe8\x1c\x90\xea\xfb\xf9\x94\xa3\xd5n\x85\x05d\xd1\xc5\x001@\xca\xa5\xb4~\'d\xadwXp\xdd\x97\x0f\t}8W\xb1\'[.O\xbdJ\xcd\xbc\xc9`\xf9\x1a\xb48H\xe1\xd03N2\xdb\x12\x06\x8bG\xaaq\xf80\xb9\xee\xd7"\xdf\xf2\x90\xd0\xbc$\x8a^\xb85\x89S\x8e+6H/\xf1\xf6\r/\r\xbf\xd4e\xc5M\x8e*9\x1c\x1a\xean\xf0 \xb8\x1c\xc93F\x1cnEp]\xdd\xe6\xb8\xcc\xc7\xfc\xc5m\xf9]\x06\xcb#\xcc\x07\x06\x8f\x04\xcb\x87`\xf9\xb6\xd7\xf9!X>\x04\xcb\x87`\xf9\x10,\xdf\x996\xf9\x10,\x1f\x82\xe5C\xb0|\x08\x96o{\x9d\x1f\x82\xe5C\xb0|\x08\x96\x0f\xc1\xf2\x9di\x93\x0f\xc1\xf2!X>\x04\xcb\x87`\xf9\xb6\xd7\xf9!X>\x04\xcb\x87`\xf9\x10,\xdf\x996\xf9\x10,\x1f\x82\xe5C\xb0|\x08\x96o{\x9d\x1f\x82\xe5C\xb0|\x08\x96\x0f\xc1\xf2\x9di\x93\x0f\xc1\xf2!X>\x04\xcb\x87`\xf9\xb6\xd7\xf9\xaf\xf1\x10\xd0K<\x04\xf8w\x97\xf4\xad\t\x96\x9f\x1e\xd8\x1fE\xb0\xfc\xf4\xc0\xfe(\x82\xe5\xb5\xc0\xfe\x05\xa9\xe4\x8f"X~\xfeP\xfc\x10,?\x87`\xc1\xffw\xa3\xffc\x82\x85\x83\x18\x1a\xe3\xb0G]\x7f|\xfb\x7f\x92\xa5)\x98\x80\x19\ne\x05\x96\x83\x99G\xc1_\x1c\xc3\x11\x96\x071\x06fY\x96z\xd4w\xa4\x04\x9a\xc6A\x8e\xa4\x85\xc7\x95\xff\xd8\x16\x10\x18\x06\xc6X\x16fp\x88\xc3A\x0c\xdb\xfe\x85b4D\xd3\x0c\x8d\xb2\x14\x84A\x0cD\xf0\x0f\xc5\x81g\x10\x9e\x13 \x02\x87(\x92`0\x0c\'y\x86\xa4\x7f.\xc1\xf2\x1f\xf5Z\x7f\xf9\x9do\xf4#\xd0_P\x0c$(\x18A\xc8\x7fD\xb0|\x7f\xc0\xe6w\x08\x16\x0e\x82\x11\x10\xd9\xae\x8a\x05\x05\x96\x06\x19\x0e\xe7\x1eU\xe6\xf8-\x06\x9ad(\x8cG\x11\x8a\x03y\x12\x16(\x10\xdb\xfa\x81\x85(\x96\xa1X\x98 !\x12\x15\xbe*\xa3~\x08\x96\x9fI\xb0p4J\x82\x1c\x8f\xc2\x8f\xea_4\xb6\x8d\x00\x90y\xd49\x84iakT\xe6Q\xaf\xe3\xd1S($\xf0\xd0\xd7\xf4e \x12\x11\xb0m\x90\xf1\x04\xf7UD\xeaOM\xb0\xfc\xd3`\xc6\xfb\x08\x96\xc7\x95\xfcC\x82\xe5\xfbO\xf4?\x94`\x81\xf1\x1fW\xb5\xf7b\xf6 \xd8\'\x14P\x8d\x9c\x085\xdde\xce\xabS\xed\x03\xcaj\xeay\x05\xe8\xa6\xbbY\xae\xa1\xe1n5\x9f1\xd04\xa6\xe8N5Y\xa7\xce\xbc\xefM;/\xde\xaf\r\xbb\x16\x02\x90s\xabn-\xf7K\xa4\x9d\xab\x17\xeb\xa0\xbf\x8b`\xf9\xed\xb6\xf0w%\t\xafQ\x0c\x1a\xf7S\xb0\x92v,a\xad\xe1T\x08\xd1/\xfduBr\xbd\xf19D_#b\xcf\xc0\xdc\xb6\xce\xa6H\xed\x05\xb9\xbf\x0fQ\xd5\x1e\xa3\x1b\xdd\xde%\x06\xf4P\x1f\xe8\xf0\xbd\x8e_\x82S\x99\x1d\xfa\x9dO\xbdX\x97\xec]\x04\xcb\x16&N\x92\xc4sQ\xb2[d\n\xe6zh\x03g\xb8u6\xc3\xf1^\xc6\xdc\xc0\x13\xa1\xd1\xf9\rtM\xf1\xf1Mn\xbc\x92\xe5c\x15\x04\x16\xa9NC\xd7\xe7\xa9)\xa6;k\x7f;\xe5\x1c\x13f\xd7\x9b\xd8\xdc\x00\xafe\x18o\x00Z\x05\xa9\xb8\x17c|\x17\xc1\xf2\x88\x11z\x94\x98\x03\x9f\xabK\xd6\x9d,\x14l\x96\x02\r\xa8u=fU\x90%B\x8b~\xbe\x8f\xa7^\xde6Ij\xed"\xe9h\xdbv\x17\x9f\xbd,\xcb\x92\x93\xd6\xc5\xe2\xa2\x87\xbd\x96\xb1\xb4\x9b\xd2\xa7\x88\r\x17\xaeC\x90b\xddr(\xfeXY\x0c\xf7\xc7\x10,\x8f\x11\xbbu$\xb2\xcd\xea\xa7\xf2\x8f\x84x\xba\x83\xc3:w\x82V\xcb5P\x10G|\xd9Ajo\xd9\x97\x8a\x85\r@B\x9d\xa6\xd8-\x9a\x14Hy+d\xa7\x1d\xeeYb\xa07\xbb+\x7fdCN\x9e\xe9\xde\x1b\x92\x11sJ\xfe\xc2\'\xb9^\xf0\xc2\xfc\xa3\xa2d\x7fn\x82ekEl\x1bo\x14L=\xb7b\x0c[\xc6v\x92\x1c\rng_l=JU\xa9\x04\xcdd\xa8\xae\x03\xac&Fx\x02=\xa4\xc4\xb5\xb1\'\xb8\xbcV\x8c\x0b{k\xc0\xba\x96\xc4C\xd5a\x160\xc6\r\x84\x1f\xfcP\xe1;@M\x8a\xfb1\xa6\x14\x17y\xb1\x1a\xd8\xbb\x08\x96\xc7`\xa1H\x0c\xc5\xb6\xe1\xf2\xdb0\xe9\xa5U\x99\x9c\x17\xf6(\xd3\xf6\xfe\xd4\xee\xf5\x9e\xda\xe3\xbdcB\xbc\x9bkrv\xc6"&\xf4\x1a\xb3n\x02\xee|$S\'\xdf\x99s\xc1:\'\xfa\xe0\x04\x88\xea\x10\xd8\xaa1\xad\x07\xe70\x19\x01\xfe\xed,\x97Y\xfa\xa3j`?\x99`\xd9\xc2\xdc^\x87|\x94\xcf}*\xa2I;\xe0d\xf2b|g\x16\x9a\x1clR\x17#\x03\xd2\xd4\x93e\xe7b\xcd\xaf\x8e\x9c\xc1cy\x99\xb0\x16\x00\xec\xf0\xe2jT\\\xa1\xd6tG:\'\xbd\x8a!\t\x136zp0\xd6<\xa3\xeb@\x8eg\xeep\'\xcc\x17\xeb\xb1\xbf\x8b`y\x0cZ\x18\xdbNY\xf8\xb3M\x02\x96\xa9\xcf\x18\xa99/\x04q\xdaY\xee\xd1X\xb22\xdd\xb3\xc6v6\xbf\xe0R\x17\xe7i82YY\xe5\x93\x0b\x1a\xad\x8f\xecP\xa5K\xd1;6\xb8\x13\xddp\xfb\x01\x05PZ\xf0\x8f\x8d|T\x02S8\xc3\xed\xc5}\xd1\rz\x17\xc1\xf2\xb5\xc2Q8\x86!\xd4S\xa5r(p\xca\xe1\x00\x8a\xa4\x0e\xdf\xf7\xdc\xc1fP+\x94q;\xe8\xe5f\x90\xb6\xfd\x02\xbd\xf4\xa8\xd9\\\xf7\xcc\n`\x179\xa3\x02\xd1\xb8\xc1\xe4]\xd0\xf3\xac\xa9O\x06X\x8f\xc4r\x1eZ\xda\x17Odp\xf0\xf7\xe2Y\x10_\xacT\xfe.\x82\xe5\x11\xe6\xb6:b$\x8c?W\xd7\xc7%w\xad\x9d\x02\xec\x04\xd8\x02 \xab\xb4v\xc6\x85\xe2dU\xc3\xecvf\x02fE\xea\x0c\xd4x\x98\x01@iAL\xbd\x88v\x84\xbe\xce2\xe8\xbb\xc9x\xf1\xf3(\xdd\xab\xd0>\x19\xe3\x16&\xc9k\x07E5\xf0\xefI\xb0<\xe6\xc4c\xa0\xfc\x1e\xc5\x10\xc2\xacn\'\x88a\xd3\xe5|\x11\x10\xdc\xba_\x0f\xd6<\xcd\xa7y\xa4\x05\xaf\xbc\xa3\xe6\xbeq\x00\x1dO`\x05:\xed\xb3\x9b\xbc\xdcK4c\xb2\xf3\xce\xea(\xf1\xb4\xeb\x0b\x83\x08\xb3\xcd1s\xed\xc3\xa9\xa3\x8e\xee]\xd9\xdd\x17I\xa8\xa3TD\xfd\xe3q\xe6\xc6x\xcdMh\xb8\xa0%\xe9*Q!H+\x14\'\x065\xa4M \xfa\x03\xab\xd1+p\xb0\x95\x1dfR%^\x06\xb1\x029\xee\x8b\xe5y\xdf\x85\xb0<\x06\x0b\xbcm\x88\x8f[5\xbf\r\x93\xeaD\x97%\xcc[}\xda\xa7\xdc%\x1bD\xbed\x9bp{\x97\xf9\x1e\x1a\x07\xb0\xaaA\x03\xc9E\xca\xacL\xc2\xd7/\xac\x8d\x87\xab\x10u3\xb9\xa53\xb6+\xdc\xcc&\xba\xef\x96\x1b\x0c\xdfJWE\xc3{\xad\x9b\xa0\xf8br\xf3.\x85\xe5\xd1\x9b\xc4\xa3Z(\xf4\\\x8b\x14\x04\xd0\x1aW\xb4\xa3^\x81p]OP\x16z\xe6a6<\xce\xb88\xd7\x86,\xc8@XV\xed\x1c\xc3\xec\tB\xc2\xc2R\x93-E\x93P{=Y\x89\xd5\xad;(\x85\xd6\xb64]\xcd\xab\x1b\xbe\x08\xa6\xb1\xdfG\xaf\x8a\x16oRX\xbeV\xb8\xed\xd4G\xe2\xf8\xd3\xa0\xdd\x0bGb\xef\x14S6O\x8c\xa1\xed\x07\xc5O\xaf\xf05\xc4]1\xe2\xcd\xb5\xa8VX:\x99\x10\x88\xa2zg\xe7\x9d f\xc6\xb1nb\xa0\x90\xa3\xfd*\xde\xec\xc8p\xa1\xa1\xcbI]\xb5967\xe4\x14\xb3S\xed\xc5\x1c\xee]\n\xcb\xa37Q\n\x86\xa9\xe7\xed*\xe5\xc3v\xc5\x12F\x92\x19_\x13\xc6\xba\xb2\xaf\xea\xa4c\x17\xfb\xdc\x04\x97=\xd1\xefM\x84\xb4\x80\xaa/&\xba\x95\xd1\xa0\xd9\x03\xc6\xd9BE\xc7\x96\xcb\x92\x16`\xca\xeaTsh\xd8\xb4w{K\xc7\xba\xee\x1c\x9a\xc4\x8b\xeb\xf8\xbb\x14\x96G\x94\xe06-\t\xe4\xb93\xd9\xab\x85M\xf5\x8a\x14K\xa5\x80\x8d\xaa\xed!\xae\x1c\xc2\xe1$\xb3\x9d@LYyt")\xb9\x96\x08S(\xe6v\xea\xda\x11\xf9\xb5S\x98C\xb3m\n\xb0\xdc\xb8\xc8\xd56\x08\xc2dE\xbaw\x01!S&\xd8\xbb\x1e^\xed\xcc7),\x8fs\x07A\x81\x8f\xd9\xf9\xb4]il\xd3\xa4\x9dL\x85\xe9\x11\xdfYAa9\xf5a\x9a\xf1\xbe\xbb$%U\xed\xa0\x8bf\x8d\x85\x146\xa8\xc6\xa4\xbbs\x08H\x8ef\x03F\x8d\xb3\x137\xb9M\x15z\x973a\x0c\x87\x92,\xaew\xd3\x1a=%\xd4\xb2W\xfd\xb77),\x8f\xa9\x89l\xe9\x07\xb5\xe5\xbd\xbf\r\x13\x9e\x14;\xf7\xaeT\xdc\'\xcd\xbd\xe7O\x16\x8e\xd0M\xcb\xa7K\x1e^\x07\xc6i/\xe1\x8e\xcb\xc5,$\xe6=\x9e\x0c\xae\x1af\xad\xc4\x01r\x7f!n\xa9u@U\xc9\x11\xad\xcb-\x83\'\xf1P\x16\x9cv\x99P\xb7}\x95~|\x93\xc2\xf2\xe8\xcd\xad3!\x98\x04\x9f\xf6\x93s\x1c\x12\xd8hYF\xca\xec\x8bS\x9d]5aY\xa4\x05\xcaz\xb76O\xba\\VUn"\x83q=&<1\x10\x1d\xd7\x1fH\x16\xb1<\x1a_]=\xef\x94\x15#q\xa0\x05\xb0X<[\xd02H\xa5\xd2\x12\xaf\xd2\x8foRX\xbe\x16Z\x9c\x82\xb6d\xf2in\x9e\x0b\xae\xdf]IW\xc0\xf7\tx\xd1\xaf\x83\x97\xa6\x16t\xa2%)\xf6z\x1c6Q\x19\x9aq\xdcS\x91X/\x88\xf546\xa8\x08\xb5\x13H(\xec\x8e\xc0*\xc4\x08\x83\x99^\xb2\xb3\xb2^\x13CP\x16\xa8<\xf7\xc7\xefu\x03\xe9]\n\xcb\xd7-\x07\x08\xdc6+\xe2iN\xe8w\xe1\x860\xb4\'\xe7\x05\xac\xdf\x1a\x8b\xdc-K\x99\xe0\xa7\x18\x8b(\xbf\x0el\xcb\xdf\xc7\xe2\x12\xd4\xe7q0\x1dM\xcd\xc4b\x1ff\xd1Q\xbe\x89\xd8\xe9An\xf1n\x12\xd1e\x16\xc8E\x9e\xbaPP\x1fM\xc9yq\xb0\xbcKa\xf9:\x8bc\xd8\x96}PO\x83\xc5\x82<\xe5\x18\xbb\xa7\xb0\x1f/\xa7\xd4P\xb4\x81B\xcb\xfc&\xdd\x02\xb1\xe0\xb5\xces\x1aG\xe5\xaf\xb7\x03z)\xd7\x937v\xf7\x8c\xdee\xe0*\x03\xe7\x03\x92\x9c\x8c\xa4Dy4\xdf\x85\xec\xec\xc5\xe4\x04\n\xdcY\\\x7fXN\xfe\'+,\xbf\xf6&N\x12\xd8\xf3\xed@\xf1\n\x03\xb7\xfc`^\xf6\x00B\xf3e>\x93\xf3\x0c\xd8mA\xf7\xf2\x10\xbb)\x83\xc0\xf2\xa9\x8e\xe0\xfe\xa2`\x97\x84\xd5\xaf\x98R\\\x12lT<\xfe\x0e\x03\x14\xa0\x90\xc3"#\xe6\x0e\xef\x1a\x7f7f0j\xed\x14\xf5G\xae\xf5OVX\x1e\xbd\x89?N\x05\xe8\xb3\x81\x00\xf6\xdc\x1a\x05\xc7\x9d\x83\xc6-DQ\x1e\x07JD}\xb3\xfc\x01?\xd1\x11H\xf2\xcd\xc03\xfb*\xbc\xdfG\xf5T\xc5M\xd6\xebxlN\xf8\xe1\x92*\x8a!\x9f\xe7\xba}\x94\xd4\xe5{j;)\xcf\xadk\x82r\xe7\xbc\x98}\xbcKa\xf9\xca>\xb0-\x07\xc5\x88\xa70\xb9\x9a"\x14+M\xef\r\xcaH\xda\x1dH\xdc\xd0\x10\xe2#\x13\x91\xd2<\xac\xbd\xdd\x94,nL\xa6\\\xa9{3\xb9\x9e\xcc\xd6#\xdal\xe0\xc6:\xc5\x82q\xb2\r$^\x00\x98?\xdf\xcejSA@\\\xb4\xd3\x84\xbd\nO\xbfIay|\xe0\x01\xc2\xe4vfz\xfe\x88nB\xf8\xd5e\x8e\xd6\xc8\xdc\xd5\x93\x99jM\x88\xf6SH8{\xb2\x81\x93D\x90\xcdeWM\xa1}[\xaa\xc0\x92r\x03\x9fy\'Y(\xb1\xed\xac\xc1\x1d\xaf\xa6\xdf\xa9h\xe6\x05y\x85\xecm\x97B\xa5\xed\\a8/\xc2\x1d\xefRX\xbe\x92\xac\xad\xb1p\xfc\x99\xba\xdbI\xb5t>\xae\xf0q\xd7\x85\xd1n\xd7\x19\x03J\x19\x99e\xa1l\xe7J\xfb\xea\x1aD\xc3\xfd2$\x97q\'/\xc8L\xc7ew%\xc4x^v!\x9e\x1eZ\x17.\x8f\xf4!\xc3\x84\x95>\xd6(mC\x12y\xef\x81W\xefa\xbfIa\xf9\xba\xb9\xbb-\xda$B<\x85\xc9\xd2\x84y\x19\xa7\x9d\xc4\x8c\xf4\x05\x80\x07T\x1c\xfa\xf2*T;c7\x1e\xc5\x90o\x16=\xbc\n#\xbbd\x06\x91\x92\xbe\x18\xe2\xc2!\x8c\x82sH\x9d2\x8bN{\x86\x9e\xb8\x0c\x85]bo85\x1a\x13\xe1\x85\xa2_<\xe6\xbd\x8ba\xd9\xc2$(\x14\xdf\xc6>\xfa\xb4\xd2\x1e\xf6\xe31\xbfj\x8eE\xde\xef\xf5t,\xa9%1\xb8J\xf3\xa8\xb4>l\x8bi\x84\xdad[\xad\xf5b\xaf\xfb\x0c8\xf2\x81\xdb\x9f\xc9\xdd\xd4\xcc\x07\xc5\x9c\xaeX\xd3\x06\x17VYn\x08eh\xea\xee\xcc\xb1\x9e\x9b\x9f_\xf5\xb5\xdf\xc5\xb0\xfc\xfa\xc1\xcb\xb6\xff\xa0\xcf\xf7&\xcabtU\x177\xd7\x1dU8zTE\xb3\x16\xb3\xaeF\xf5\xd5b\xf1~\xd6\xae\x94:"\xf3Y6G~\x87$\x13\x86\xa2\xb4\xba?\'\x81\x16\xdb\xde\x19\xcc\xf8YK\xe4\x1d\xdf\x02\xf5\xb9\xf4\xf1\xca\x98\xca\xd1.\xbe\xd7\x8d\xacw1,\x8f9\x81\xa3\x0f\x18\x11{\x9a\x13\xd5\xf6\xc2}v9\xf10\x0e\xc47l\xd1s\xda\xe2\xb0\xa1*\x0e\\\xcc\xacFBo9V\x8bd=uHFf*w\x11\xa1p\x88\x0c\xd5S\\c\'\x0f,\xc7\x02\x88/.\xd3\xc9\x99\x80\xeat14*W\xfez\xbe\xfa\xaf\x19\x96\xaf\xbb\xd2\x1f\x86\xe5\x9f~\x1e\xfc\xdf\x87a\xf9\x13\x01\x17\x1f3\xe4\xd3\xa4\xdf\xbfI?\x0c\xcb\x87a\xf90,\x1f\x86\xe5\xc3\xb0|\x18\x96\x0f\xc3\xf2\x8d\xaf\xf3\xc3\xb0|\x18\x96\x0f\xc3\xf2aX\xbe3o\xf2aX>\x0c\xcb\x87a\xf90,\xdf\xf6:?\x0c\xcb\x87a\xf90,\x1f\x86\xe5;\xf3&\x1f\x86\xe5\xc3\xb0|\x18\x96\x0f\xc3\xf2m\xaf\xf3\xc3\xb0|\x18\x96\x0f\xc3\xf2aX\xbe3o\xf2aX>\x0c\xcb\x87a\xf90,\xdf\xf6:?\x0c\xcb\x87a\xf90,\x1f\x86\xe5;\xf3&\x1f\x86\xe5\xc3\xb0|\x18\x96\x0f\xc3\xf2m\xaf\xf3\xc3\xb0|\x18\x96\x0f\xc3\xf2aX\xbe3o\xf2aX>\x0c\xcbKD\x04\xfc\x12\x11\x01\xfd\xdd%}k\x86\xe5\xa7\x07\xf6G1,?=\xb0?\x8aay-\xb0\x7fA+\xf9\xa3\x18\x96\x9f?\x14?\x0c\xcb\xcfaX\x88\xff\xdd\xe8\xff\x98a!H\x12\'P\x8c\xc5\x19\x1eEH\x9c\xa2\t\x0e\xe4\x10^\x80Q\x01\xe2\x19V@i\x0egI\x0e\x87Q\x1e\xc1p\nG\x05\x14\x819\x0ce\xe1\xedo0\xe8\xa3\x82\xf5\x8f}\x01\x0cb`\x90\xc5Ql\xfb%\x1a\x86\x19\x98\'\x11\x96#\x19\x10aP\x02\xe7y\x1cex\x0c\xc4\xb6f&i\x8c\x01!\x9a\xa6Y\n"\x1f\x15QxV\xa0\x99\x9f\xca\xb0\xfcG\xa9\x8b_~\xef\x1b\xfd\xe4_\x10\x02EP\x10D\xf0\x7f\xc4\xb0|\x7f\xc4\xe6w\x18\x16\x98\xa3P\x9ccI\x92#\x1e\xaf\xfd\xc0CH\x94x\xa8\x0e\x10Ap\x18O\xb1\xd86\x18\x04\x90\xe20\x96\xd9\xfe\x18\xe6P\x1c\xa4Hz{Y\x8c\xa5\x98GU\x81\x0f\xc3\xf2\x13\x19\x16n\x9b-\x18J!\xb8\xb0\x8dK\x86\xc2\xb6\x17b\xb6\xbf\xce\x12\xc2\xd6o$\xb6\x8d,\x0e\x839\x10\xa5q\x81\xdb\xf2YH@X\x18\x84\x18d\x0be\x1b\x108\xf8\xcb\x9f\x9ca\xf9\xa7\t\x82\xf71,\x8f1\xfd\x0f\x19\x96\xef?\xd1\xffP\x86\x05\x81\x7f\xec\x93\x10LX\x88\xd7\\\xf6\xc5qD\xeeRA\xcd\x93)\x81;\xf5h\xdf\xe3\x94\xbc\xd5\xe0\x89\x15\xac\x81\xe1\x94\x053\x8f \xcc+)\xdb\xca\xe6%-\t\xd3\x8ew\xd7l^e\x0e\x11\xd1\x16\x84.c\x92\x8d\xe3\xf76\x89\x87(v\xe0\x8c\x85b\x0cYL\x8dA}\x16\x07\x0ee\x97r\x95\xbd\xbf\xa2\xddX\xe7\xd3\\A\x0ebRP\xf4b\x81\xa7\xb71,_\x89\xcc6`\xf1gmF\r\x8e\xd5\xbdA\xbds\x11\xb7<=U\x05\xc68\xcc\xbd \xf0\x04\xd4!\x1f\x12L_\xf5Cc\xb2h/\xb72P9\xb3-\xb1[D@<\x91\x97\xdd \x8c\x98vD\x16AYqU\x08\x922>\xde\xb4\x01\xaa\x7fTw\xfeO\xce\xb0<\xe6=\xb8\xa5\xe4\xd0\xd3\xb4\'-\xdc\xb9c\xbb#\xd8\xf4\xd0n\xb2\x95\x9d\x0c\xd5\xb46WZO\xe6hr\x9d\'E\xccC?\xb2\xb9\x14\xb3\x05^+8S\n\x86\xbb\xde;\x00\xb2+\xf9[\xc1\xc2w\xce\xde\xed\x90hQh\xa2\x8cs~^\xacW\x8b\xf7\xbfKay\x8c\x15\x1cA`\x8c|\n\xd3=\x9e\xc1\xc6Z\xe2\x11\x0fB\xefjx5\xcd\xda\xac$\xf5\xce\x00\x1f\xa5\x18\xf3W\xcd<\xefm\xa6\xf5}\xf2T\xe6\xbc\x98\xe1\xc59\x98\xfb\xa0\x0f\xd1\xce\xd5rb\n\x17\x16\x9a\xbck\xa9\x1f\xfd\xaa\x14\x0e1)\xdb\xaf\x16\xea{\x97\xc2\xb2\x85\tn\x87\x9emh<-pde4W\x13\xad.\xa0\x9b3\x03\xe0\xec\xf8\xe0\xa25b\xce\xd8YB\xe4\x90Q\xaaF)\x97,(B\xd3\xccT\x11\x85\xd3\x9a\x01]\xa5\xf0T\xa2\x98dD1p\xd2a\xf7Bd\xf4\xa4b\x1e)\x82\x0b\xea]_\xac\x86\xfc6\x85\xe57G\x98\xbf+\x89z\x80b\xe7|\xba\xf1\xd7\x16\x97\xb0z\xc9X\xec\xbc\xec\xccv\xa9<=\xadw\x11\x8a\x82\xba\x92\x92\xc7\xaa\xd4\xa3\x1b\xdc\xc4z\x88\xcd\xe3Y\xa9\xe18\xa4N\x97;\x0c\xc7\x17k\x00Pi]d\xd0\x9f\x95\xddID\xc8\x17y\x92\xb7),[\x98\xdb\xf1q;5>\xd7^c\xcc!\x8c\xec\x11\xb1\x1c\xbc\x0e\xe5\xa8Gq"\xf1\xdc\xd9\x8b\x97\x12C\xe4\xf0\x0cp\x83y\x8dX!v\x8d\x1a\x1cY\xbaI\x00\xb4\x92\xae#Ve`XA\xd5r\x0c\x03w\xc7\x1b"ja\xc0\xb8\x1a\xe7`\xaa_\xdc\xae\xde\xa6\xb0l+\x10H\x81\xf8\xb6l=\xf5&\x03^cKhwe\x01\x9fw\x83\xda\xf2z\xd1d\xe7\xb8\x87#\xcb/\xafs\xa1\x04d\\\x88\x85\x9c\xb7\xfd\x14\x96\xb0\xd6\xf1\x9a;\x08\x89\x15\x1a\xd6\x98N\x04pY\xba\xdb\xe4m\xb9\x01W\x90u\x17EW\x0e\xd3~47\xff\xe4\n\xcb6X\xf0-5\'\xd0\xe7\xfc\xedP\xcd\xb2\xc6\x1e\xd9;\xdf\xa7\xd0,z)~\xccn\xe5T\xba;a^P\xb0\xf3 =\xf3\tV*\x0c\xa7<_\xcf\xc4vI\x03\x996:\xae\xbb\x00{\xa2\xfadw\x08c\xc6\xbdE\xc0\xe8\x80A.\xea\xb9\xa0\xbfXX\xf6m\n\xcbcN\x10 \x05\x93\xcf\xd9\xf8\xe54\xc6G\x0e6\xfd\xa1\xf0\xa2SEI>vvrn)\x17\x07_2\xb1\xd5\xfb\x88\x87\x92K\x03v\x93\xb3,\x89\x1f\xf6\xe0|\xea/\x99\x1a\x9d}2;\x18@\xa7\xd0\x9a\xb0\xe3zO\xf3\xc0J\xa4C\xb5T^^\xc8\xdf\xa4\xb0laR[_n\x99\xea\xd3\xd4W:E\x84\x88,\x10\x89\x10\x9a\x10\xba1\xa3r\x81\x8fW\xcc\x1f\x9d+\xba\n+M\xb4\xce\x95\x88\xf9\x88\x1d\x9b\xf0\x0c\'\x05z?\x1c\xf4\xab\x01\xa9s\xea\xa4\xa1\xbdg\x03\x19\x8a\'\x0b\xea9m?\xf2%\xbaz\xde\x8f\x16\xf2?\xb9\xc2\xb2\xad,\xdbN\xb8]\x1d\xf2\xd4\x8a\xa4rI\xea\x951\xa8\x82\xbf\x8b\x95{\x89\xe2\x80\xeb\x90\xa0\xc14\xae\x02\x10\xf04\x1f\xc6\xbb{\x18=\xee\xb8k\x99\xe5~\xb8H7\x90\x96\xd7b=t\xc2\x11\xbe\xed\x9c\x08sK\xee\xe6\x13C<\xea\x10#\x16\xbe\x91\xbe\xb8O\xbcMa\xd9\x06\xcbv^ `\xfc\x99\xec\xa1"\x1a\x90\x18\xd0\xa8V\x8emb\x90=m\xa9Xpu%d\xd1\xcf\x9a@\x1f\x92\xab4\xdaD\x93df\x81\xc7[\xd6\xd1\x8d)23\xb33\xf9\xd5\xad4\xaeT\x80\x88\xeem\x1fUJ/Y\x14\x18\x92j\x8d\xac/\xe6poSX\xb6\xde\xdc\x06\x01\x84\xa3\xcf\x85\xd0Q/\xb1ez\xc8\x85\xab\xb7d\x1a\xc2bpF\xd7\x93p\xecQ\xc7Zw\xcc]hW\xbe\x8a\x13fK\xf2\x00\x86\x12\xce\xd6q\xe8U,q\xf7\xd0Dd\xa1\x9d1\x97\xf5\xb0\xf81\x0e\xaeWC\x91\xe2\x81\xc7\x8b\xfcE{\xedm\n\xcbc\x85\xa3\x90\xc7\xeb7_\xdc\xf5\xdf\xa6\xb0<\xf6+\x02\xdcN\x1e\xe831\xa7\xf0d\xa6Mn\xeb]b_R\x0c\x0c\x02\xb3\xd58%7\xbd\x86\xac\x0b\x7f\x82\xd1\xf5&OjKi\x13\x96\xf4\xeal&\x94\x1c\xe2\xd8tX$r\x1c\r\xde\xc6\x04=\x7f1\xf9x\x9b\xc2\xf2\xb8\xe5\x00m\xcb\xe4\xb6\xe9\xfd6\xcc|b\x08Z\xec\xcb\xcb\xcd\x1a\xb6\x8c\xb5\x88gM\xde)\xd9x\xa2/\xcc\xfe\xe2\xc2;{txG\x9fM&\xbc\x96\xa8bE\x98\x1f\xe0s\xcf\xc8\xd8=SE\x94\x1a\xe6\xf2\x96\x93;\xc9$\x84\x80h\t\xd4\xea\xec\xcb\x8b\xbb\xf2\xdb\x14\x96G\xf2\xb1\xfd\x14$\x9e\x93\x0f\xe6b\x1e\xfb\xd9\x1d\xb7\xbd\xb6v\x11\xc7\x9d,4\x8a5\xc7\xc7/\x01\x06\xac\xd4D\x18\ro\xca\xf6B\xcc\xa5\x87v\xd2\xbe\xe6\x9a<0tb\xd7/\x1eN\xf5\xc4.\x83\xf8\xb9\xd5\xf8k\r\'\xe0q\xb6\x9ci\xa4_\xec\xcd\xb7),\x8f\x85\x9c"\xd1\xedT\xfa\xb4\xc2I\xad\x0b8M\xc0\x1d\xe5\xd9/\x9a\xfd\xbd\x99\x88N\xcb]\xb1W\t\xd0\xe0c\xa4\xce\xba#\xee8\x9d)\x0b\x12s\xe8Z\x17\xaa\xde"\'?\x04\xc9{~\x88\xb4\xa8cK\x07g\x99\xf3N\x17]%\x15\x8c?Jay\xdc@\x02\xb1m\xa2\x13O\xe7\x9fC\x99\x911j\xa4 o.\xa57\xdc\xdb\xbd\xc5\xc6\x8bd\xcf\x04y\xbe\xb6\xfeA\x89\xbd\xeaV\xe3\x13\x85T1xQA\xb4\rO5\xcd\xbb\xb7kN\x9f\'\x885\xc5.UMae\x1d\xf7\xbc\x17\xdd\\\x07\xcf\xc5\x8b\xfb\xd5\xdb\x14\x16\xf2\x8b\xba\x83(\xf4\xd9\xd4I\x10\xc7\'\x0fiH\xa3J&@\xbb\xben\x89\x83\x1e\x9a\x03\xe5\xb4\x924\x95\xf9\x85\xf2\xaeQ\x9e\xf3\x19z\xcb\xaf&@3\xdb\xc1\xe7L\xf6=`\xc5\nw_\xae\x8c"\xe2G\x97Or\xc4\x85\xcc\xda8d\xe7\xf6\xe5O\xe8\xde\xa5\xb0\x08\xcb\xff\x8d\xa3\xf4\x83\xb0|\x10\x96\x0f\xc2\xf2AX>\x08\xcb\x07a\xf9 ,\xdf\xf7:?\x08\xcb\x07a\xf9 ,\x1f\x84\xe5;\xe3&\x1f\x84\xe5\x83\xb0|\x10\x96\x0f\xc2\xf2m\xaf\xf3\x83\xb0|\x10\x96\x0f\xc2\xf2AX\xbe3n\xf2AX>\x08\xcb\x07a\xf9 ,\xdf\xf6:?\x08\xcb\x07a\xf9 ,\x1f\x84\xe5;\xe3&\x1f\x84\xe5\x83\xb0|\x10\x96\x0f\xc2\xf2m\xaf\xf3\x83\xb0|\x10\x96\x0f\xc2\xf2AX\xbe3n\xf2AX>\x08\xcb\x07a\xf9 ,\xdf\xf6:?\x08\xcb\x07a\xf9 ,\x1f\x84\xe5;\xe3&\xff\xde\x08\x0b\xf2\x12\x10\x01\xff\xdd%}k\x84\xe5\xa7\x07\xf6G!,?=\xb0?\nay-\xb0\x7f\xc1*\xf9\xa3\x10\x96\x9f?\x14?\x08\xcb\xcfAX\xc8\xff\xdd\xe8\xff\x18a\x81\t\x88\x86h\x8afx\x14g9\xee\xc1LlQo\x17\x07\xc3\x0c\xc2A4\x01\xc2\x08\xcb\xb3\x02\x83\xe1\x88@\x80\x08\xce\xf3\x14\xces\x04\x0e\x114\xc9\xd0\x8f\xe2\x1f?\xd6\x05X\x02\x16`\x8c\x17`\x08C9\x1ca\xa0\xed\rh\x88\x7f\x94\xa6\xc3\x19\x14\xdd^\x0c\x810\x08cX\x0eB\x11\x84cX\x92\x02Q\x10\xa2\xb6\xb7\xa3\x11\x82@~*\xc2\xf2\x1f\xa5\xb7\x7f\xf9\x9d/\xf4\xa3\xd8_\xb6\xbf\x80C(I!\xff\x08a\xf9\xfe\x84\xcd\xef ,\xc8\xd6\xce\x04\x84\xf2\x1c\x8e\xd2(\x84\xe0\xdbks\x08\xfe\xa8\xbfDQ<\r#\x0c\xc1\xb10\x06\x11\x14\x85\xb08\x86@,N\xb0\xcc\xa3Z\x11O\xd1$\xce\xf1\xbf\xfc=\xc2B\xf1\x0c*p\x1c\x0e\x12\x0c\x8c!\x04I\x80 \x8er0F\x81\x0cH\xa0\x14\x8foC\x80\xc5\x18\x8a \xd8G%M\x98b\xd0\xed\x95)j\xfb%v\xeb\xed\xbf.9\x1f\x84\xe5?\x11\x96m\xfcS\x18\xcf\xf0\x10Ks\x0c\x8f\xd3\x02\' 4\x86?**q\x8fj\xf2\x08\xc6a\x02M1[\x04\x08\x87\x92\x0c\x8e\xb1\x0c\xc7\xf1\x84@\x82\xec\xa3F\xf7/\x7fr\x84\xe5\x9f\x96A\x9e\x10\x96_\xd8\x1ee\xd8\xfe\xca\xb0\xa5r\xf7aj\x8dN\xd4z<\xab\xa0\x7f\xba\xf5Q\x1b\x8fI%\x94\x81g\x16\x8a\x85\xa91\x9c\xf0\xc1in\x02X\x00\x03\xf3\x00\xb2\xe5\xed\x1e\xc3\xd0]\xf1\xfam\xdf\xe1\xe1`\x0f\xc1\xa9\xd8\x9f\xa3s\xb3\xfa\xb0\xb9Db\xb0\xe5\x81\xc1\x94\xecM,\xdd\x17\xe7-E*ToK\x8d=\xb5\xe2\xab\xc3\xd7\xef>r\xea\x08a\x9a\xb8\x12\xd6\xb0\x9d\xaf1\x8c-[\x06\x0c\x06\xab\xda$H1\'\xc8\xe3+0[z\xd5&\x95\xbaG\xc1\xe0\xe4c\xc9\x89GC\xce\xbc\x06Eg\xb2\xb9\x0b}\xed\x91\xb3$)+=)\xdc\xd7?V`\xf4\x1a[\xfb\xb2\xb8P\xd5\xaf{\xa82\xfa\xc8\x01\x8bE\xf3\xbe\xc5\xd7\xf8\x08\xbd\xa8vp\xd9\xe22\x92\xd3\xe1\x18x\xb5%M]A\xf7\x96\x10h\xae\xa7\x1d\x1b]#\x12b>\xf4\x89U\xac(T\x0f\xe0\x9dOxN\x1a\xbd\xfd\xfd.tJ\xb4\xf0\xb9!\xd3y\x01\xf0\x85"sF\'$\xa7G\xfb\x91\x16\xc3\xf9\xdbk\xd3F-\xf3\xfa\xbe\xa6\x8f\x0e\x8307\xd6\xbe\xae=\xb6\x1ct\xa2^\x13,\x0f\x8f^\xd1A\xf8\xf5\xb8\x10\xa3\x80\xe3\xa4hHs\xb5\xb7w\xf0\x00\x9b\x94P\xef\x00\x8f\xf1\x9b\xa3i\x99\\\x17\x82\x9d\xb3M \x1d-\x0ct\xa9a\x0e\xe2,\xd6\xfe\xe5_\xe6s\x1e\xf3\xf1\x1f\xf29\xdf\x7f\x89\xfec\xf9\x1c\xea\xc7\xb5\xfa\xf5\xe5\x0029\x06r\xa7\xe3\x90\x88\x92\x86EE\xae\xe9}\x01\x05)sRi\xa1\xdb\xdf8\x0e\xf0{b:j>3I^\xc6\xd4\xb6\xca\xee[\x13\x1c*p\xbd\xc4W\xf5\x9c\xac\x98\xa6*\xd3\x85`\xb8\x1b\x9d\xe7\xed\x8bu\xc9\xdf\xc5\xe7<6t\x1cA\xb7\x85\xf7\xb9\xfc\xba\x8b\x18\x87U_\xd2\xbb+0k2\xaa\xd4\xc0Q\x9c\x99\x1ap\xd6\x18\x12#\x19;"e\xee\x92G\xb2\xe5u\xa2K+\x83\x17\xb5\x16\x15\xc4Fz5\xd1\xaf,\xaf\x92\xc3\x15\x84\x85jt\x05\xb7\xa0o\xc7\xc15\xd0\x17\xc3|\x17\x9f\xb3\x85\xf9p\x90\xa8\xa7\x18\xf9\xd2/\xf6\x98Z\xf8t6\x16b\xe9\x94\xb2w\x00#\x9c9\xd2K\x89\x8e]\xc7\xcf\xb5e\xba\xb3ZH\xf7\x8c\xd4D\xa0\xe7\xf3\xf5H\x16\xfd\xb0\x83\x14\xa6\x8cE\xd5V\xf7v\x1ev\x16\x8fm\xb9\xb3q\xd0\xeef\xf1\xb2+\xf3\x1e>\xe7\x11\xe3\x96&B\x14\x81\xe7\xd1\x8a\xe8\xd6\x8c\x04\xf2\\\xc6\xedb\x07\xf2<\xe9G\xa6\xe9S\xf0\xd4\xdc\xe2\x92;\xcbJ\x1ami\x19)\xba3\x92\xcc\xf7\x94\x91\xfc\x03\xcc\x05z\xa8\xf8\x0e\xb2O{\xdao\xdd\xea2\xca\x81\x9f\xf5R[\x88{\n\x85.\x84\xd2\x1es\xbf7/\xda\xf5\x0f\xf2s\xbe\x967\xea1\x89\x9eK\x02\xeb2\xd58t\x86\x9cw\xe3\xd1\xcb\x90Z=i\xe6\x9d\x99u\xbe5\x88d\x9d\x1a\x1d\xbf\xd9QZ\xb5$*!9\x00Tuq\x91\xee\xc5~D{T5-\xa7/\x95\x089\x1dv}\xe2W\x11\xe6$\xb7\x8b{\xe4^,\x98\xf9.?\xe7\xb7\xc7\xb2\xbf\r3\xcc\x0f&0z\x07u]}p\x90\xddc\x98\x84e{\xcb\xc4;\xdd\x1b\xb8\xc0\xa2\x97\xf2\x9e\xde\x00\x1f\xc4\xac\xa9\x86w\xe6nE\xafE\xb8e\xe1\xaa\x84z\xe1A\x80\\\x01\x1fR\xf9:\x1cF\x15\x1b\xba\xf3\x12r\xf1\xab+\xdc\x9b\xfc\x9c\xc7\n\x07\x12\xd06\xfe\x9f\xd6q\xe5>A\xdd\x8e\xa7\xe5\x83\x19j\xd7\xb3\xc7"\xde\xb4o\x84\x9a\xe8\x84%o\xfcno\xa5\x81R\x19v\xc4\x1e\r\xf1~Ef\x1e\x12\xa4\t-U\xad\x1cW1Tyi\x18\xf4]\xc2\xcc\x9d6\x9e\xe7{rM\xf3\xfb\x8b{\xd5\xbb\xf8\x9c\xaf\xce\xc4\xa1m\x07 \x9e2\x8fcU\xe1\xc4\xee\xee\xf4^\xee\xc1\rs\x92\xc8K\x94\x97\xaa\xc8\xa78si\x18c\'D\x16\xbb\x84U\xbd\x839\xb0\r4\xd1F\xfb\x08?\x13\xad\xef\xb9\x8b\xd1I\x9d\xe8\xdb\x060\xaa3\x9f"N6\xfb\xd9e\xbc\xbf\xd8\x99\xef\xe2s\xb60A\x1c&0\x14\xc7\x9fj\xaf;\xabz#\x15i\x82\xbd\xc3!\xc7\x95r;\xc3I\x86\xd64\x9d\x8d\xed\xef\xf1\xd1\x1b\xfc\x9aP\x04\xc0\x9b\x91\xa3\x0c\x9e1v\x06x\xb3\xf4q\xc5\xf2N\xd7d\xecBepz,\xbf\x1f\x9b\xd6\x07\xda\x95\xd9\xcb\xdci\xfc\x91O\xf8\xe7\xe6s\xb6VD\x1e\xf5\xceI\xec\xb9\xf0\xf1\x8d\xe0}T\xd4\xe7\xbc\xf3\xbc[,N)}\xceY8Fp\x91W\x16\xf1r\x06\xb90\xa7B\xc2G\xaa\xb9Qp\x91\xcd\xfb}"]\x99\x19\xbcT{\x90\xe8{\xcc!\xd5\xe3xo"\xd0\xcb\xee\x034\xadU\xae\xbd\xba\x8e\xbf\x89\xcfy\xcc\x89\x87\xb5\x84a\xe0\xb3\x9c\xb5`\t9\xf8\xc31\xd1\xa8\xca`\xed\x8e\x97|\xfax\xbel\',\xdcj!m\x7f\xb6B/5\xe5SN\x87\x99\x19\xd6}l\x18*`\xdc\x1c\xed>X\xc3\xc2,\xf4\xeevP\xf9\x1a\x11V\xfa\x18\x13\x08\x8a\xd7\xc2\x8ba\xbe\x8b\xcf\xf9\xcam \x1c\xc3I\xe4\xa9`fY#\xf6\x1c\xb7\xaa\xc1\x97U\xaa\x87Q-\xf2\xfb\x82\xd4\xf7\xd9\x15\xba\xaf\x17\xcb\xdbR\x16\xbavu\xcd\xa2\x8a\xb0\x86\xc6\xda\xce\xcd\x96\xdd\xb2\xda\xe3\xaa\x90\xcd\xe1\x9c[D\x8b\xb7\xd4%\xbc\xf8\xa1\x0b2\x90\xe2\x19\x90\xff#\xf8\xe5\xcf\xcd\xe7<\xb6\t\xf4\xb1\xe7\xe3\xcfsB?\xdc\xbc\xd1,P{=\xede\xaa\x8a\xceZ\xb7?\xa0{\x1fH\xd2\xf1\xa0\x83\x16?X\xb04\xdf\xa7L\x959\xefV\xb6\xdc\x9f\xa8s[\\\xea\xaa\x9f\xb3\xddA\xf0\xaf\xa6\x0e\x9bg\xf7\xc6\x0e\xd4\x92\xdd\x0c\xe6&\x1e\x12U\xba\xf1\xee\x98JU\xc4z\xe6\t\xbb\xd3\x1d\xc7\x83\x1d\xa7\xdbJv\xf74\r+S\xde\xc6\x00MP^\xc4\x90\xde\xc5\xe7|\xed\xfa[F\x8bn\x9d\xff\xd4\x9b]u\x1f\x9a\xb6G\xf8\x03\x07\xe1\x87[Tv-\x7f\xea\xaad\x94\x0e=\xde\xb9\xbe\xce8\xf7!G\x85\xd5eB(\xa4m\xda\x93\x86\x19\xcd\xb7\xb4F\x80\xcf\x9ac*}\xae\xad%e\xc8\xa5\x11h\xadq\x87\x0f\xea\x8b\xc5\xc0\xdf\xc5\xe7<\xc2$\xb7\xbe|\x94\xb0\xffm\x98-\x8fv;fG\xb2qs\x17e\x13\x13\xf6\xc1n\xda\xf3\x07r\x80\x15\xd3\xd4\xd7\xd9\x07\xd3I\xb8\xb4\xfc\x19^/\\\xbd\xac\xf5\xa2\xd6\xb7\x05\xb4\xaeR4d2\xae\x1c\xcaqk\x0b\n\x1e\x9d,\xb5C\xa5\x1eo\x17\xf6\x8f\xe1s\x1e\x83\x16\x82\xb7\x89\x0c>\x83\x96"\xa8\xf7rs\x16\x1c\xaa\xd8\xd6\x9f\xec~\xd2\x92m\xa8,\x8e\x9e\xdd\\\xab\xa6"\xf1F\xed\x9aD?c\xd8e\xc8\xdb3,\xa4;\xe6\xba\xb6\x11\x93\xb4\x83-\xf3>\xd5\xbb\x1aLXa5\xe2\xed\xf9\xa2\xd4\xf4v\x14~\x15Cz\x13\x9f\xf3\xc8\xe1\xb6=\x0fA \xfc)\xcc\x9d\xa9\xdc\xad[\x84\xc9\x1e\x97\xf8\xbc\xee\xce!\x93K\xb7\x8b`\x0cn\\\xd1\xfa\x89r\xf8s5\x0eE\xb0\xd8\xa4\xa5\xba\x14Y\xef\x8c$\r\xc5\xa6\xd7i\xd7\x8d\xc5\x0c\x85G\x8dK]\xf0d/\x9d=^\xc4x\x16~D\x91\xfcd?\xe7k[&)\x10~\x1e\xb3%\xe9\xba\xbb-1E;\xd7$\xc4cc!\x86xTl\x06\xcdW\xd0a#%\x9c\x1a\\\xe1E\x10\xd1\xfb\x8b\x16t1p\x14\x1c\tPnS\xd6\xdexd\xd6\xb7\xd7\xb9\xc1%\xe1\x9c\xacl\xbc\x9f\xae\x9a-H\xf2\xabN\xd9\x9b\xf8\x9cGgn\xf9\x15\x04\xa2\xcf\x05\xec\xf7\xe4=\xd6\x99\x15\xbd\x8b\xe8\xc1\x97n\xc55\x05\xc2\xeb\xadl\x1a2\x0f\xd1\x82\x13\xb1S\\\x0fF\x7f\xdd\xe1\xd9\\\xb4m\xdc\xce\xd5\x88b|\x9c\xa77UZD\xda\xee\xb8b\x01\x93\xd8v\xdd\xb3t\xcd\x9cc\xa3h/n\x9b\xef\xe2s\xbe\xee|\xe0\x18\x89\xa2OQ\x9a\xa8Np\xa9\xef\xe5\xdc1\xa8s\xe7\xb4:\xc7\xd2B\x02\x97&\xf8\xd9\x819*\xcf\x1cP\x07\x92\x9d}\xbc\x0f\xf7\xcb\xca\xd8\x1c~\xdb!\xf9\x19\xef\xc7c~\x0c\x99\xfd\x8e\xb8De1(\x01:_\x8f\x9cZ\xf3\xf7\xeb\xf7*\xec\xfe.=\xe7\xab\x11\t\x10\xa2@\xe8\xd9t\xa8\xcf\x8e\x8c\xf5\xb85p\x8c\xb9\x9a\xfb\x99G#\xba\x96\x8e\x14?\x11\xad\x8e\x03\x13F\x8c.\xcbc\xe9\x89AP\x81\x8dj\xd5om\xc0\xeb1\xf7\xa0e!n\\\xdd\xa4\xdb\xcb&z\xe7eO\x04\xcb:\tl\xe7E\xec\xe0]z\xce\xd7\xa6\x8c\xc0\x18\n=\x87I:\xaa9\x84\xdd\xfd\xd6\x9cd\x9f?\x03\xe2\x12s\xd2H\xf4\xcav\xba\x82\x90`\x002\x86\xd3A\xc3\xc5N\x8cy,\x03\xd3\xc8\x16\xb1\xe5\xb2Ppn\x17\xac\xd7\xcaSE\x1f<\x1e[O\xcc,\xf5`\x9c\x9f\xb5\xdc~q}{\x97\x9e\xf3\x15&\x8a\xe0\x14\x8c=\xcd\x89.\xe4l\xc10wh_:\xe7\xa3\x17\xd78s\xa3\xf4\x92>\xb0\x08\xcc4\xe9a\xec`\xeaz\x0eI+\xe0m\xf2<_\xf7L Z\x0e7W\x17l*\xf9\xec\xec\xa69\x9f)ju\x03T,\xdb\xe3cp\x0e\x9c\x17\xb1\xb7w\xe99\x8fMy\xdb\x911\xecwR\xac\xcb\xbd\xad2L\xcd\xf2]D\xe6\x04.Z&K]&\xee\x00#\xc6~\xe4\xd7Q\xeb\x999I\x1d\x88d\xa84\xc1<\xd84\x8d\xe8\xbc\xac\'\xb8Dt\x00=\xf0y\x8b\xe9\xc9IM\x90\x95O\xb5\xc8\x18\x05g=\xff\xe8\x8e\xc3O\xd6s\xbe\xee\x92A\x8f\xdf\xc0\x9f\xc2\x0cV~\xf5\x84\x1b\x93\xa7g\xf0~\xb2\xceA\xb6\x1a\x9aF\x9ed/\xae2\xe0\xbc\x1e\x12\xa1j\xbc\xc6\xb9\xd3N`L\xe7\x91\x16C0\xd7v^\xe5C\xb0\x1f\x94\xdad\xa37\xdc\xbf\'\x91\x0c\x02EV\xc36\x00\x9d_\xbd\xe7\xf9&=\xe7\x11&\xf9@(@\xf8I\xcf\t\xb94i|L\x9a\xae\xa6\xd4\x1c\x86\xde\xa1O\x1d6\xcc\x06\xd4\xd0\x9d\xbf\x82s\x98\xd9\x03\xbc\x1c\xcfQ\x1b\x94)\x1c;C\t\xcf\xfd.\x9f:\x1bC\x9d"v\xa9\x83vd\xafu\nQID\x88\x15\x06\x17k\xf0\xea\'Wo\xd2s\xbe\xe6&\x88\xa3\xdb\x86\xf5\xd4\x9b\xb7bK\x1e\xfa\xab)\xdc\xf0\xdd\xca#pm\xdeLx\xdf\xa3C=\x8eGvJx\xd4\x1c\xfdI\x03\t\xed\xc6\x97\x17\x7f\x9dN\x1c#I-\xaf9\xde\x00) \x16]R\xa5\x02\xee\xec\xe1\x06Um\xb0\x9f\x93\xbc\xe7^\x85\x83\xdf\xa4\xe7<\xc2\xa4\x1e\x10\x12\x8a>\xad\xb4\xdd\x02R\xd6\x15E\xb5f>\x91*\x0emG\x8e% .\x17\x97r\xf6s\xb7\x00\xb7@\xaf\x8f\x00l\x8f"\xe7\xf4`~\xde\x1f\x89\xe9\xa8\xf3\xb1\xbb\x08\xc2\xfd\xa2\xcf\xb4|k\x02\xb7\x94Vd\xd7M\x87\x0bb\xee\xaf\xc8\x8b7=\xdf\xa5\xe7<\x06-\x81m\x995\x8a>\xdd\xf4dT\xe3\xd2mcd/\xab\x1d\xa9\xcd\xae\xd8\xf91?\xeb\xa27\x1c\x13\n3\x18(\xbdz\xb1\x8c\xf1\x0br3\xdd9\xd3\x8d\x9a\x0b$\xfa\x8a\x11!\xdc)\x02~\xf7\xdbb\xbe\xe3W\xb15#\x90v|\xaf \x06\xe5\xc50\xdf\xa5\xe7|%\xcc\x8f\xd5l;\x15\xff6L\x7fe\xa2\x81\x81/\x1d\x8e\xe8Q\x1aZ\x02\xb9R\xb8;\xedVI\xe8f\tV\xdd\xea\x94\xf8\xe0=\xea\xdc\x9c\xb8\xd9N\xdb\\\xa1\x11\xb0w\xba\xa9\x19\xc0EK%\xd0\xed\x86C\xc2\x9d\xaf7\x15=\x8f\xf5\x99\xbd\xb7\xe0\x8f>\x8a\xfcs\xeb9_\xf7\xb1\x1e\xd4#\x81>\xadp7\x9a\x95\xfb3\x9e{\xf1\xbcuN_\xb6eP\x8b="\xe2",\x87\x18\'\x19\xf3\xd5%\xe7n\xc2\xeeM\x8f\xedj\xe7\xb2c3\xe0\xd8\x1f\xb5\x03X\\1\xb7\x90\xa2CS\xe5\xa5\x11I\xf5\x1ed;\xea\xbc\x8f\x92_\xb7\xe5\xff\x9a\xcf\xf9\xfap\xf1o\xf9\x9c\xff\xf1\xbf~\x89\xbb$\xfdz\xd2\xe4\xd7\x07\x8f~a\xf3Nf\xf7f\x13\xed\xdd\xb3\xef\x99\xcd\xf6+\xe59\xeb~m\xfc<\xbc\xfe\xcf)<\xdf\xd2\xc7S\x880\xf6\xd7\x07=\x1f?\x1d\xaf_?\x83P\x08E\x91\xaf\x07\xab\x9e\x18\x9e\xf9G\x8f\xea\xfa\'\xb5\xf2\x10\xb5\x89\x8d\xe7GkW\x01O\x90\xa4P[\xf3\x96\x9c\x0b<\x80\xb1G\x05\xc8z;\xa7\xc0\x89\xdd\xc0\xb1h\xd6\xa9Wt\x01\xec\x80\xbe\xed"*\xec\xae\xfeBN\x7f\xff\xac\xf0\x8f\xdf:\x86\x9bm/\x11\xc0\xc4;4\x7f\xf3\xd6\xc6*l\xd9PR:\xa7+b\xdbA\xe3\xc3\x8d#\xaf\xae\xaa\xc0n`\xc2\xf5\xac\xee\x05-\xf0\x0e\x9c&:H\xb8w\xa1\xa4\x96&\x03\x84\x82t\x7f\xbd\'\xa2\x00\x86p`\x9d\x1a\x06V\xbc\xc4\xb3\x1aerO7S\xad\x13\xd0\x02\x03\xc9\x02{\\\x06\x1b\xd7\x06\xa1\xc9nc\xcc\xb1\x99s\x88\xc4\x94n\xfcV\xd0\xf9\x97\xe1\xa2\x7f\xa1\xcd\xfec\x8f\xfc\xeb+\xff#\xb8\xe8\xff\x98Ez\xfe\x06\xc5/j%A\x8a\xe8~=r\xf6\xf7\x8d\xf0\xaf\x12F\xffj\x1b\xbc\xf0\xde\xdb\xb8\x05\xc3\x135\xfe\xcd{o\x97r\rNI\x91\xec\x9d\xb7u\xe6M=m\xd9\x00\xf2x\xc2.\xb9&g\x06NO\xea\xa2T*\x1e\xb6<\xb65z\xe3#\xee\xa8\xb4\xb7*\xa9\xa4Em\x93kl\xfc\xf3\x9d\xf9\x7fv\xdc\xc7\x8e\xfb\xd8q\x1f;\xee\x1b_\xe7\xc7\x8e\xfb\xd8q\x1f;\xeec\xc7}g\x93\xedc\xc7}\xec\xb8\x8f\x1d\xf7\xb1\xe3\xbe\xedu~\xec\xb8\x8f\x1d\xf7\xb1\xe3>v\xdcw6\xd9>v\xdc\xc7\x8e\xfb\xd8q\x1f;\xee\xdb^\xe7\xc7\x8e\xfb\xd8q\x1f;\xeec\xc7}g\x93\xedc\xc7}\xec\xb8\x8f\x1d\xf7\xb1\xe3\xbe\xedu~\xec\xb8\x8f\x1d\xf7\xb1\xe3>v\xdcw6\xd9>v\xdc\xc7\x8e\xfb\xd8q\x1f;\xee\xdb^\xe7\xc7\x8e\xfb\xd8q\x1f;\xeec\xc7}g\x93\xedc\xc7}\xec\xb8\x8f\x1d\xf7\xb1\xe3\xbe\xedu\xfek\xae\x15\xfa7o\xf2_\xbbV\xc8\xdf]\xd2\xb7\xb6\xe3~z`\x7f\x94\x1d\xf7\xd3\x03\xfb\xa3\xec\xb8\xd7\x02\xfb\x17\x88\xb5?\xca\x8e\xfb\xf9C\xf1\xbf\xc5\x8e\xfb\x1f\xff\xa9\xae=\xaa\x83Xc\xfb\xe5\xc0\xfc\xf5\xeb\xce\xdb\xeb\xa4\x97\xb6<\xdf\xfe\x12_\x96\xfe\xd6\xfdE\x1f\xa3\xa6\x8c\xe5t\xf9\x9f\xfc_e\xb6\xff\xbc\xfe\xff\xf5\x9fX\xdb\xff\xfb\x0b\xc7\x1e\n\n\xd2\x86\x82\xd2\t<\xf7Y\x02\xea\x05kM\xc0x>\xdf\x07\x93[\x02\x1c\xd0\x8fB\x90\xe5\xc7\xc7 {\xd4)\xe9\xbb\xe9A\x9f\xfd\x82@\xbf\xfc\xfa\xbd\xec\xff\x8b,\xbb\xbfk\xc4\x7fd\xd9A\x0f\x95\xee\x81w\xa1\x18\x8c\xa1,A\xb0\x0c\x8c\xd2\x04\x88\x08\x02\x06\xa1$#`\x04\x08\x81\x10\x81B<\xcc\xd1,\xbe\xb5\n\x8dS \xcfP\x04\xc6\xc1\xff\x85eGa\x1c\x01~9f0\x83\x0b4\x03\xe2\xb8\x80\xd0<\x86\xe00/\x08 \x07\xd2\x18\xc1\x91[3\xa3 \x85\xd1\xd9\x9a)\xab\x9c\x02@\xbc\xb3\xb9\x865\x17wlG\xd5\xa9h\xa8\xcf.=q\xbc4\xf0\xa4\xcb\xd8\x94\x9al\xb5c\xf1\x17\xcb\x93\xbfKD{\xc4\x88m+\x1d\x85\xc2O\xd5\x16E\xf4t,\x96\xbb\xd9\x9d&\xc3\xe8\xc6>\xb8\x81\xf6\xd5\xb90\xdc\x0cb\x8be/\nI\xcb\x18\xd6Z\x08H\x88!1\x1c\xbb~\xe8\xefz5\xe7s\xb3\x00g\xf1H\x0f\xb7A-\xd8x\xcf\xb0uO\x93\x11C\xdc\xae/\x16|}\x97\x88\xf6\x18\xb1\xf0\xb6\x05\xe1\xd4s\xb1\xf9\x998\xb9Q\x08\x9a\x85\x96\xc0\x8dE\x13f\xb0\xd3\xf3(9- a9{\x87#\xb1[sYdN8\xc6\x14\xc0\x0f\xb6w\xc5\xe7\x16\xb1\x1b\x142\xd8\x88\xd1lf\x05m\x131D\x1d\x0c3\x8dL\x17\xa0\xf5\xd9\xef\xa5\x86\xbcKD{\xb4\xe26\xe71\x82z\xd6\xa5LX_O;\xa3\xbe\xdc\xa5\xa3\x90\xedv8s\\n+(y\xdch:5\xbc\x0c\xe3N]\xdc\x9d7\x9d\xf6XKK\x82>\x1d\xb4\x80#\x98\xc9\xb3\xf1eW\xba\x07$\xdc\x99\xc5\xdd\xe7\xce\x8c`J=\xd6\nJ\xf2b\xa1\xd5w\x89h\x8f0\xb7}\x17\x02a\xf2\xa9\xd0j\xca\x96\x98\x1a\xa8\x8cW{\xeb\xc0\x80y\\\xb6\xb6\xbc3pD9\xde\xaa\x95\x94\xc8!9G@u<\xe9\xe5\x9d\xe4\xab\xceV\xd9C\xd2\xac\x9e<6\xb1\xdds\xfc]G<\x96\xdd\r*\x86\xef\xe5Q\xb4\xfcS4\xbf,\xe9\xbcGD{$\xf7_8\n\xf5\\;\xd3K\x96\x89\xb1K\x7fN\xb0\x9d\x82i7\xb0(\xb4\xdb\x84\n\xf5-\x0e\xd7\xb6\xf3\xe4jp=\xdd\xc5\x81\xc3j\x12\xf4\xa1\xaac\xb1\x83\xc8*\x02\xcc\x0c\x1b3\x81+\x8b\xc5\xd7}u8\x10\x07\xf6\xa4mK\xd4~\x8e^\xc4Q\xde%\xa2}\rZ\x9c\xa0(\x92x\xa2\t\xac4\x98\x96\xb0KE\xb7\xbdU\x0b\x94^w\xb3)\xee\x81\xf5\\e\xd2\xb1\x89J\xb2S,z\xd0Q\\\xbb\xbe\xd8X\xb9\x1dS\x7f\x1b\xe6\xe1x\xa7\xfd\xd3\xa4\xe2\x1dB\\\xac1\xd5D\xef\xde\x89\x98\xd2t{\xfc\xd4y\xb9Y0\xddi\x97\x1e&\x01[m\x93\xad\xc93TR{\x83_syo\xae\xae\x92T\n\x06\xdf\xe4\x04\x87\x8a\x1dlz\xe8\xc2&\xe3\x8b\xbe\xdd\xbbD\xb4\xc7\xdc\xfc:\x90a\xcf\x80\xe7\x18`\xfa\x0e\x98\x94\xfdz\xcc\xd79\x1d[\xd6\xd6\xfa\xf0\xc8\\\x12\xaf\x96\xf6\xae\xcb\xb0\xd8\xbdB\xa3\xe9|Y\xc2\x01X\xe1\x928]5\x91Zv\x8aBQ\xc7\x9c\xc6|\x89DZ\xfd.\x96\x89\x1a\xf6\xb5\x0f\x0c?\\\xc7\x7f2\x88\xf6\x95\xa8>\x00-\x9cxZ\x81p\xcd\xf0{8m\x8c\x04dG\xd6\x91Z!\x96+\xf0\x04LK_\xce\xb5\x11\x83\x97J\xe5\xcd\x93/"\xb8\x03nC\x99D\xacv\xc5A\xfd\x0c\xed\xa7\xc32\xc6C\xa4+\xdem\xa7\x9c\x8b\x08\xdf\xeb\x08cf~\xfa\xe2B\xfb.\x10\xed\xeb\x1e\x19\x8c\x91\xf8\xef\xb0\xdaX%\x0b\x8c\xc1j\x1c\xb3G\xe7\x14`\xe1\xd5\xea\xcd1\xbb\xe6]\x95\x0f\xcb)3\x14A\xb4\n\x99Uy\x96\x811+\xc7\xb9\xddJ\x8b^\xa7\xc7\xfb\xe3rr\xe2a\xac\xdd\xb5\xe9K>\xc0\x93\x94\xe1\x80\xc3\xf98\xbe\xb8\x02\xbdKD\xfb\xeaM\x02\x81\x08\xf8\xf9V\xe0\x05`\x8a>\xa9\xb7\xc4\r\xb9\xf0T\xbd\xd4R\xec\x85\xed^\xbe(\xdciXb\x0b4-3=YUk\x994_7\xfd\xce\x18\x91\xbe\xf5O\xc3\r\\\x11\x8f1\xd0\x19\x1a\xba\xa5\xd1\xe7\xb3h\xe91 \\\xc5\xdb\xe9\xc5\xde|\x97\x88\xf6\xe8M\x1c\xdc\xceV[2\xf9\xdb0\xf3\xc0\r\xb3\x06\x040\x06\xdf\xf6\xc8\x90\xd9\xbbA=q&Z/Gh\x1e\xf9\x1b\xba\x03\x94\xfb\xc08.\x15\x87\xe4|\x8e,\xb1\xd2\x16\xc1\xdd\xe5\xcbb!\x93\x87:\x88/\xbbG\xf4\xce\xec\xe7\xbb\xe1z\xfd\xb0-\x96?\n\xf3\xcfM\xa2}\xddq \xc9Gv\xf6t\xc7\xe1\xce\n\xe09\x00&D\x10S\xf9\xea\xf7\xe2\x92\xc0w\x03\xadz\xdaW\xe6\x81\xc4F\xc4h\t\xc6\x94\x0eV\x86\xb9\xfaj\xf2\x1cKW\t]\xa4``\xb5\x03\x94\xc1\x013\xd4f}?X\xa9D\xb8\xe6i\xe2\xdd\xf0\xc5\xdb\xe3\xef"\xd1\xbe\xc2\xc4 \x14#\x9eo\x1d\x93VS\x1e"\xc2\x8c\xa3H\x10z>\xaf\xc6\xe2v\xe9<\xde]\xeeL\xbc\x8d\x96\xd5\xb6\xb8\x12u\x84\x1e/\xf1\x96\xa7\xa7Q\xa9\xb8hQ\xcb\x12\xd4\x98}\x85\x19\x00j\x0c:\xe0B\x8b\xb5\x83+t\r\t}\xca_\xcc\xb1\xdeE\xa2\xfd\x1a&\xb1e,\xc8\xd3\xae\x8cc\xca\xf5J;\x92\xd9\x84\xcdDH\\H\x8b\xfb\xd8\x9f\xfcGm\x90\xb1\xd2\xcdx\xcex\x0f\x92-\xd4\rU\x00\x80\x80\xa9\xedT\x90\x1aCO\xcf\x92r^\xbb\x85\x99\xf6\n}\xa5\xc4#\xe1\xab\x19_\x99\x8e||\xb17\xdfE\xa2=\xa6\xfe\xe3\xa6\xca\x16\xe6o\xa3\xbc\x9e\xe4[\x87\x92\x84\x92x\xfbh\xa4\xd0$\xcd\xa2\xac\xbdIz(\xc3\x1c\x98\x87\xa8\x1b;L\xb6\x18\x17\xa2Nk0\xeb\xed@\x1b\x8e9_#6\xb2(\x01\xe0^\x98\xfaF#\x13gw=\n\\\x19\xf0<\xb7\xec\x1f$\xa2}\xe5\x1e\x04E\xa2\xdbq\xf9\xe9\xa8|\x929\xf2\xact\xb5F\xf0B\xe4w\x073\xa6\x00\xc4e\xe5\x83\xb2\x9dZ\';?J\xfc\xd1r\xb7-\xf3\xaa\xe8#\xb6\x03\x12>\x8b\xd8\x9e#\x86J\xbe\xd9\x91e\xe1\xac}\xe8`N\x19\x12\xda\xc9\xb4\x93M\x06\xd2\x8b7\x03\xdf%\xa2}}\xa4\xb3\rs\x04%\x9eV >\xe5$\xaf<5Vb\x85\xfa\xbc\x1a\xb7!\xf7v\x9a1\xaey\x1b\xee\xeaz$\x9cz>\xf8\x97 \xe2\xe3f\xbdu^\x16"\xc2\x8e"\xdb\xbd\xbf\x98\xf3\xc1\x9c9\x0e\x14\xce\xb7\xe0\xd8\x1b\x0c\xd6v\x00\x04\xb6\xd0\xf9\xc51\xfb.\x11\xedkjB\xe8v\x90z\xbe\x83\xbd\xab4\x0cp\xb5\xaa\xed9\xb1\xa3y\x1e\x87\xc4cj\xeb2$\xcd2/6\xc0\x94\x0b\xf99\x86\x18\x03\xb1\xc6[GM\x95X\xc3VuI#"\xdb\x0b\x9dz\xe1o\x82@\x19\x02\xb3\x14\x01F\xb3\xa3m\xb5\'\xe0\xc5T\xf2]"\xda\xa37A\x04\xddNS\xd0So2&rK\x95\xd4\x02\xb5S\x98a\xedb.\x98\xc1\xd3>\xafq\xdd\xed\\\xd0\xc5)7\xd1\xe2,;\xf2\x059\xc6\xb6\xad!\xa2\'-V\x03q\x9a\xed\n\x00]\x04\xdbOo\xd6}\xc8\xea\t9\x13mI_\xe1\xf0\xc50\xdf%\xa2man\xe7&\x10\'\x9f\xcf\xb2\x84:7zy\x1f\x06\xc7c\xd5I7\xaf\xf2`\xf5AO\xcf\xa6\xcf^\xc5\xfdjk\xdbFx\xec\x99Z\xf3\x9b\xac\x86\x86:\xa0\x1f\x15/O\xecl\x95=\x95_\xa4R\x1f\xf1R\xa3F\x89>q\x89\xe5\\3\xef\xf0\xe2\xe9\xe7] \xda\xaf\x1f.\x81\xdb\xb6\xfb4d\x87\xab$\xe4UMb\x80\xe1\xc7\x0c\x8f;x\xab\\\x03\x8a\xed\x180\xa0\x00\xddZ\x80\x99\xa7\x8b]\xb8\x17\xfcaT\xc8c\x03\xf1Z\xd5\xcb\xf7s\xca)$\xee\x9c\x19\xa9\xd7c\x88O.\x00\x8c\xdc+\xe7\xd0\x8c\xc7\xf0{\xa1\xb3\xef\xf2\xd0\x1e\x8dHm\x13\x7f\xdb\xaf\x9eZ\xd1\tf\xba\x9d\xd5\xa9\x0e\xf7l\xb9mP=\xe6\x01:4k\x06|\xe6|(\xbc{HX\xec\x02#Cig\xaf\x99=\x84z\xe4\xfeZ\xee\x1df\x16\x89\x18\xaf\xaf\xe9\xbd\x1ab\xec\xae\xda\xfc\xc5\x0e\xd8r8\x08\xfb;G\xffs\x1e\xda\xd7\xa9\xf0o=\xb4_\x1f\x9a\xfc\x900\x7f}\xe5\xf7\xab\\\xdf\x0f\x87y;\xbb\xf1\x8bR\xd5\xd0\xb1\xa2!\x95\x8b!\x8d\xf3\x11\x95\xe3a\xc5\xf6\xa1\x1f\xb7\xe6\xf7\xb5N\x1e\xc1\x8c\xaa\xfd\xfb\xc1$\'j\x8dN\xe4o\xba\xe7\x9bj(\xffn\xdd\xa2L\xc7*\xdf.\xdeA\x14.\x9eT\xce_\xd4\xca\x98\x95u\x0b\xec\xfc{\xc1|\xdfnQ&\x8dE\x7f?\x92\xeaG0\xda\xf7\xa5l\xb6E\xd3\x1fUnk\x02[Y\xb6\x99\xb3*6\x8d\xa9\xeb\xb60W\xf1\xfc3g\xcc\xcf\x88\xa4\xc6\x8e\x95\x04\xab\x9c\x03)\x95\xbf\xfdW\xd9\xba\xc5\x98\xd4\x95\x86\x7f\x7f\x90}_\x0e\xe7\x17e\xadG\xa52\xb6q\xc6o\xe3\x8c\xdf\xba$\xdf\x02\xca1\xcdv\xa6\x9f\xba\x90\xfd\x8cH\x96\xe3\xefFb`\xdb;\xfd\xd4ny7U\xf30\x10Ge\xe5\xb7\x8b\xdf\xfe\xe1jt\x9b\xfa\xdb\xb4\x7f|\xf5\xd3\xf8\xa9\xb3\xe5\xa7D\xb2\x1e\xab\xdf\x8dd\xfa\xc9\xdd\xf2v\xee\xe6\xb1*\x8f\x9a\xed/[0\x8b\xc2)\x98b\x1b\x93Rm\xef\xc4)\xc8O]\xc4\xde\x1f\xc96\xb8\x8e\x953\xfd\xff\xec\xbdY\xb7\xe2V\xb2\xae\xfd_|\xab1J}\xf7\xdd\xa9G\x08\xf5-\xbaC\x02\xd47\x08\t5\xbf\xfe\x88\xb4\xeb\xec]&\xd3_Q\x03\xf6Z\xde\x87\xe1\x0b\xdb\x99k\xc1\x8c\x193"\xde\x98\x82x\xb45:\xd4\xf5\x954G\x1e\xefR`\xcd\x05\xb8\xfa\xde$\xf6rd\xceo\xaa\x93\x0c\xda\xb2\x86\xbe\xe3.\xabA\x93\x9a\x0bk\x1a\xd8C\xf7\xf0\x7f\xa7[\xde`\xc9\xda\xf0\xef\xf2xu\xc9~Vsf\xd2\x16\x15Y-Y\xadZk\xcd\x9bk\xcb\xab\xb1;?\xa2e-\xf3\xd8\xea\x9aY[dTs\xd4{\xd9_\xf4Ub\xbe5\x89\xbd\xc3\x92i\x15\xc9?\xb5\xe4\xcd"\xf9\xe5\xe8\x9e5Z\x84\xb5w1g\x9d\x97\xb1\xd5\x10\xf8n\xdc\x9d\xb9\xab\xbd\xd9-o\xb0\xe4\xe7r\x7f\xd4WE\xf6f%\xf6r\xfc\xcf=\x89A\xbb|\xbf\xbaD\xbe\x1b\x84\xa8K\xbc\x8aea5h\x0f\xab?-\x94\xdf\x14d\xf4\x95ny9B\xe8\xf7$\xc6\xff\xd4\x98\xf9\x9d\xd1\xf2\x16K\xbe\xca-/\xc7\x10\xfd\xa6\xdb\xd8\xac\xde\xb3q.cj\x9eL:\x7f\xd72k\xa48\xbf\xea\x8d\xbf)P\xe9Wq\xbf\xa8\xf7\x99\x99oVb\xafF\x19\xfd\xa6e\x18\xae\xaeF\xe8N|Wak]IP}\x95\x96\xda\xf2^\xb7\xbc\xde\x92\xd5\x05\xf0\xdaN>T\xc9\xf5\xcf\xb17\x97\xfc\x97\xe3\x90~Sg\x0c\xbaG\x8a\xf6#\x91\xad\xa55/\x96\xfb\xd4\x0eu\xf1\xde\xea\x967X\xf2uny9R\xe9\xee\x16\xfc\xe7\x97/\xefu\xcb\xeb-YW>h|\xb1\xc6\xbc\n\xdd\x1bc\xcd\x89\xd7&9\xbew\xfc\xbf\xe8\xc0\xbe/>\xea7\xed~\xef\xba0\xf8\x1a)\xa3\xee\x14\xab1\xab\xca\xe7\xe5\xb5\x10[\xefu\xcb\xeb-Y\xf6k\x07\x96\xdc5\xe4\x9a\x02\xef\x17\xafw+\x84\xbb\x9b~q\x8d\xf4}\x11T\xbfJb\x93\xe6\xbc\xd7-\xaf\xb7d]\xf9\xf0\x93*\xb9\x96\xfc\xe2\x17\x97\xae\xdf\x17c\xf5\x9b\x9a\xdd\x95\x18\xb3\x96x\x13\xd2\x16uT\xf9\xfdj\xd8\x1a=|\xfa\xdehy\xbd%\x8b0\xac:r\xd5\x93\xcc\x1a%\xabH\xce\xd5\xf5\xb0\x99\xb3\xba\n\x987\xbb\xe5\xe5\x88\xa9\xdfT\x1b[\xb3\xaf\xbb\xfch\x1f\xf9d\x15\xc7\x05\xaa\xf3{L\xe7\xc3\xb7\xba\xe5\r\x968\xc5\xf0sK\xf6\xd0\x9b\xdd\xf2rL\xd5\xdd-\xcbZ\x1c\xa15\xa7\xafJ\xdfE\xb5\xb5\x8d\\k\xcb_<\xd3\xfb\xa6\xc0\xad\x1f\xd7Hk\xc4L\xf7$\xb6\xa6a\xe8Gm\xc9\x93\xb5j\xc6\xd8\x9b\xdd\xf2\xff,\x92\xeb\xe7\xfd\x15\x9fL\x1a\xcf\xbe\xfb\x91\xf0\xcbaX\xbf\xa9\x1c\xb6*\x92U,\xae\x9d\xae\x9a\xc7\xe8\x9as\xc7\xd5\x98\xf5\x14\xfd\xca\x98o\x8a\xf5\xba\xd7\xc0U\xa5\x15\x90\x9a\xaf=\xfb\xb2Z\xe20\xf7\xf4\xb4\xaa\xc6\xed\xbb\xdd\xf2r\xa0\xd6\x0f\xb7\xac\x05\x1d\xbb_\xa7\xac2x\xedL\xd6B\x9a\xef\xe7w\xf7\xf0o\xb0$\xc3\x96\xc7\xbb;\x19U\x1d\xed\xedny5\x94\xeb\xee\x96I\xe7\xe35R\xccu\x1b\x92U\x99\x08k\xce]\x93\xc9\x9b\x0b\xfa\x1b,\xf9B\xb7\xbc\x1a\xecuw\xcb\xfd\xf2\x01^\x93\xd8\xfd"rM\x03\xee\xfc{Y|s\xb3\xf8\x0eK~\x12\xf7\xea\x9a\xc8\xcaw\xbb\xe5\xe5p\xb0\xdf\xddr\xff\x0c\x15\xef"\xab\xde\xba\x7f8\xe7\xde<.\xbf6\xe6\x9bb\xce\xbe\xd2-/\x07\x8c}\x99[\xdeb\xc9W\xb9\xe5\xe5\x90\xb2/s\xcb[,\xf9*\xb7\xbc\x1ct\xf6e\xb5\xe5-\x96|\x95[^\x0eK\xfb\xb2hy\x8b%_\xe5\x96\x97\x03\xd7\xbe,Z\xdeb\xc9W\xb9\xe5\xe5\xd0\xb6/\x8b\x96\xb7X\xf2e\x02\xf9\xd5\xe0\xb7\xaf\x13\xc8\xef\xb0\xe4\xcb\xa2\xe5\xd5\xf0\xb8/Kbo\xb1\xe4\xcb\xdc\xf2j\x00\xdd\xd7\xb9\xe5\x1d\x96|\x99[^\r\xb1\xfb\xba\xda\xf2\x0eK\xbe\xaeoy1\x08\xef\xcb\xdc\xf2\x16K\xbe\xca-/\x87\xe9}Y\x12{\x8b%_v\xf9\xf2j \xdf\xd7]\xbe\xbc\xc3\x92/Kb\xaf\x86\xfa}]\x12{\x87%_\x16-\xaf\x06\x03~Y\x12{\x8b%O\xba\xe5?\x03\xb4\xe1O\x01\xda\xb0\x7fY\xf4\xb7\x86 \xbe\xdd\xb0\xaf\x82 \xbe\xdd\xb0\xaf\x82 >g\xd8\x7f\xc0\n\xfc*\x08\xe2\xfb\x8f\xe2\xff\x08\x04\xf1\xf7\xbf\xf8\x7f\t:\x88\xfd7G\xfe5tP@`\x94\xc5\tQ\xc4\xee\xbc\x07\x92\xc7 R$\x08\x98\xa7QH`\x19\x88g\x18\x1e\xc2YR\x14 \x8a\xc4h\x11a9\x9c h\x01f\x10V\xa4\xd6\x9f\xfb\xed\xafhZ\x18Ns\x10\xcc\x88\x18\r30\x0cA,\x84\x92\xc8}\x9c<.\xb2\x02\x8f\xf18\x03\x930C0\x18\x8c#\x14\xca\xde\xa7\xe7\xe1\x1c\x87\xf38F\x12\x08\x84\x89\xef\x84\x0e\xfe\xdf\tv\xbf\xfdl\x86\x15\xf5\x0f\x02\'!\x14\xc3\xd0\xbf\x84\x0e~\x7fd\xe3O\xa0\x830\xc2\xf2\x9c "\xe2}\xc6\x17\xc3\xe3\x02\x84R\x10\x8c\x93\x08\n\xaf\xce\xe5y\x14C!\x91\xc6\x89\xd5\x0e\x9c\xc1)Z$\x85\xd5\xfb\x10O\xe04M\x90\xe2\x1d\xd1\xf3v\xe8\xe0\x0b\xd0\x80\x7f[\xe8 \x05\x8b\x0c\xc5\xb2\x10\xbd\xc6\x89\xb0.\x95\\\x97Ip\x08*\xf0k$\xdd\x7f\x91Y\xe3\x11!)\x81\x17\x18\x8c\x17 \x1aF\xb0\xd5s\xd4z\x08iA\xa4\xf8\xdf~\x0e\x1d|\x81\x9f\xfeg\xa0\x83\xff6u\xe7u\xd0\xc1\xbbW\xff\x12:\xf8\xfd\x03\xfdK\xa1\x838\xf2k$\x0eW\x1ejQ\x9a\x8b>\x95#L\xdd\xec\x03j\xd3\xeez\xcb\x07q\x0c\xcd[\x83\x90\'0\xdf\xc1Y\xe6#a\xa7cQ\x190\x07\xc8\xc0\x1d\xbc\xde\xbb\x14t\x90\xae\xf8a\x06{\xbfM\x86\xabE\x89\xa4\x7f\xf5\xce\xb3\xfd$\xa9\xeee\xd0\xc1\xb5,\x90\x18\xb5F/\xf40\x82\xbb\xe3\xb6\xbe\xe3;\xb7\x05\x85}\x900\xaei \x1f\xd0K\x9b\xcd\x1a\'\xc3\x85\xbc\x91,\xcf\xe0\x8b\xae w\xd1\x0c{!r\xda\xf9\xf1r\xc8:\xc5(a\xa1\x9f8\xd5wpI\x88\x19\xe0x\x1dv\x9bT\xbb\x9d8\xee\xd9\xd1\xad\xaf\x82\x0eR\xff \xd7\xac\xf5\xc0\xab!q\xad\xddo,s\x89\xed\xba\xea\x0f\xa0\xeb\x05\xf0\x96\x81ER\xbf\x1eJT\x81\xd4Z@\x01(\xd3\xec\x83\xa3\x1dn\xdd\xb6\xaf\x93\xe3\xa9\xd8\xd3\\-o+\xa2\x80@\xcb\xa4|\x7f\t\xcf\xb8\xea\x8eD\r\\"W{\x92\xe1\xf42\xe6\xe0\xea\xc9\xb5\xe8a\xf7\x1a\xffg3+\x99\xbby\xb2\xcfJ\xf2!\x91\xbd\x9a\x03\x8b\x06\x17r\xa5\xc7\x94\x89\xda\x96)NG\x8d1\x86\x1d\xa4"\xfdq\xe1\x8f\xc9\xa1\xe0I>\x1f\xcf\xb2{\xd8\x80\xb6Mow\xe6>\xc5\x03\xc0T\x988\xf5\xa9!qS\xe3\xc9\x99\xf1/c\x0e\xfeI\xc7\xfcw3\x0b\x92?\x86\xf8\xd0\x166\x90\x1dGj\xe7\xf5\x8c4\xa5D\xd7[R@#\xd3\xad\x0f{\xea\\X \xe9\xcc\xb0\xa8\xc7\x87\x9dT\x81\xf8\xc9\x0e#m(\x81\xe1\xd0\x8c\x12\x7f\xa3\r\x83\x1bj\xb6\x15\xee\x04+l\xcf\xfd\n#\xf77g\x0e\xde\xc3\xfe.])\xf4a0\xf5\xaeE\x13X\xc9\x00\x1d\x05\xe1\x14\xa0C\x1f\x8f)C\xdf\xc6e\xbdc\xce|!1\x85\xb2\'\xf3\xf5|\xf2mE(>\x87\x11Io\xb9\x96\x0e[x\xb4#!\xe0J\xe5\xb1\xa0\x9e\xafC*3\x99$PW\x94\x9d\x9e\x9c~\xfb2\xe6\xe0\x8f\x98@Qd-\xd1\x7f6\xb3.\x13\x9d\x97\xd2\x8c\xddZ\xaae4|g\xdb\xa7\x9d\x81\x80,{\x851\xe3b@\xb5[\x85\xe3l\x19\x82\x93\x9c\x02\x9c>\x8a\xea\x96\x9a\x9a|D\x13\xd9\xda\x0e\x1b&\x14GTG\xd6RI\xc6\x02\x8d\x83;\x14\x9e\x9fFU\xbd\x889x\x8f\t\x9a^\xcf\x06\xf1`\xe6>6H\x1c\xac\x86)\xe6G\xa6\xf7N\xd9\xf9\\n\xc8\xfaD\x9eJ\xd2\xd4m\xbd\xad@\x06C7:f\xf7\x17\xbb\xf5-\x999\r\xd7\x1b\x11\xba\x100\xc8\xc16\xe1E\xd9\x93!\xf6\xe0\x81T\x93\x92\x17\xdcig\xfe\xab\x98\x83wo\xc2\xd4*K\x1eA\x99\xd8\xa2D\xdd\xc5\xd3\r!9\xf7^\x07\xd0;\xf9\xa0\xb3J\xc3\xc7T\x02\xa8\xc0)\xce\x0e\xba}\xd5G\xce\xe2\xc2iQ\\\xa0YB\x13\xde0\xb7=\x16s\xb66Q\xbd\x9fc\x8c n"U\xd8mTs=\x0b\xf3\x93\x19\xeee\xcc\xc1\xbb7\xb1\xb55\xc5\x1e\xe7o\xf7\x88\xb3\xcf\xe6\xda\t1M\xd3\t\x04\xb8*1\x9dWt\x11\x01\x90\xed\xdb\'\xf2\xdc\xef\xce\x1b\x018x\xbb\xf5\x90\xea\xa6\xe9\xd8)1\xa9W\x03\xa5\xb7\xa7\xe0\xd4B\xca\x054ti\n\xb7}\xc6\xfb\x8d\xca@R5=\x0b\xffx\x15s\xf0\x87\x994\x82\xd2\x8ff\x16\x04\xb3\xd9\x9f\x8c\r\x1et\xf19\xed\x81}~\xac\xcf\xf0\xf9\x88\xad\xe2\xc64\x87t<\xde\x18\xd7\xda\xf40\xa8\x93\xa4\xd3j\x85~U\xc4\xe3\x11\x1f\x91X\xa1\\zh\x11\xd3\x0f\x82I\xd8\x01\xde\xa8b\xda\xb4\xf8G\xed{\xc1c_\xc6\x1c\\w\x91^\x1dBC\x8f\xcc\xc1J T\x00\x16\xf2\xfc\xc08\x99C\xc7\x8a\xdf\x1amw\x15\xd1\xb4\xb1\xf8\xb6\xc3k\xbdwo&\x91\x8e\xe8`mC\xee\x88\xe0\x19m\x9f\x87\xbe\x96%e\xdbz>\xc5i\x12x\xe2\xb7\xbd\xe2O"4\x9dP\x17x\x9aU\xf5*\xe6 \xf5\x8f\x1f<\xb6\xb5\xa1\x7f\x90\xa9,\x06\xcdm\xee\xd2\x88\xcc\xc1\xf0f\x17\xe0"\'{-\xc1\xd2\xc1H\xd0\x8a\xc8\'\xf1\xd5\xecRn\x03\xebX|\xce\xb3-\xb2\x07\xb7`\xca\x07\x8a\xe6\\\x93F\xa0A\x1emvbw\xa96\x85\x8dT9W\xa40\xf5,H\xe1U\xcc\xc1{Y\xbe\xd3u(\xe4a\xd0x\xe1\x80\ty\xf0\x92P\xdenj0Ql\xd8\x83\x86j\xa3\x137\xfc\x96\x17\xf0\x9a\xd5\xa2\xdb\xd1\x98\x83vs;\x9d\x1d\x85\xca\xe8b\xe9\x1d/%\x0eH"\xaa\xb6\xae\xb5W\xd6\xb9 \xa6P\x9c\x92(p\xd1a\xf5\xe5\xf7\x8a\x89\x97A\x07\xefb\x9fF\xd7\x97y\xac\x13\xa6\x08\xceg\xaa\x11\xb3V\xd9\x06\xb3\x92\xa8\xe7\xe3e\x03\xbb[0\xc9\xf0\xb29\x1ej\xcd;\xd9\xc4y\x18\xf3D\xddU\n\x10\x01\xa7\xeerNvE\xe7hS:\xc9)\xd6\x19\xf4P\xf5\x18\x8e\xc1\xfd\x85<\xd0\xbc\xca\xf2_\x04\x1d\\\x0f\x0bFAk\x03M=\x08\xfe\x8b#\xe8$\x84\x1f\x1b\xae\x97\xbb\xec*\xa92\x11\xb9\xf9\xf1\xea\xf0@\x80\xd2\xc7\x81\x87\xeb\x88g\x8d}y\xf3\xdc6\x1f\xd9\x03\xe8\x81^hfMaeTx\xc5\xfc@\xc7\xc8s\n\xb6\x07\xbbniK\xdc#\x83\xf1d\x87\xfa2\xe8\xe0\x0foB\x10\x8ea\x0f\xed\x9bp\xda\x9f\xb7,\x8c"\xcb\x18\x9eC\x9c\x17\xf7\x15B\xad{\x92\x9e\xe7\x9b?\\\xf8\xad\xd7\xd8\x87\xbdW\'\xfb\xec\xe2\x02\x05\x8aN\x94F\xd4P\xd1\xdbZ\xee;\xf6\x90\x05-o\xf6\x16\xe0s\xa1G\xeb|\xc6\xd8]\xfc,\x1e\xfbU\xd0\xc1{9\xc4W1I<*rb\xb7\x8d\x17P\xd6\xbbz\xa3yh~\xb9D\x14\xe9\x9d\x1b/p\xf7\xbc\x89\xd3<\xeb\x1f\xb1\x9dx\xad\xce26\xa01\xc6\xa5\xc3I5\\\xd7\xde\xd7\xf6\x9a\xd9\xe7~\xeb\xdd \xb7?\xf6\xf5q\x8f\x9e\xd7\xa6\x8eI\x0e\x0e\xf7E\xd0\xc15\x91\xafY\x12\xb9c\xa0\x1e(N\xc6\xac\n\x12\xd7)\x97\xb2>\x8a!M_\xcciO\xf0\x9b)\xa6\x85\x13\x9b\xf9\x1c\x0ev<\xd6(\xfe\x19j[\xe4\x8a\xb6U*\xb4\xfd\xd5U\xe5\x8byP\x8be\'\x01\xb7\xa5\x19\xc2D\x05\xdceh\xd0\xea6\xb3\xcfv\xa9\xaf\x82\x0e\xae\x87\xf6N\x8a\xb93\x92\xfelf$\\\xb5P\x8aL\x90/\x80\x9b\x94\xa5\xb7\xf2V\xf8x\x9aL\xdb\xc6\xdc\xd6G+%OF6\x18\xc3r\xc6\xe4\xabK\\\xa9\xbd\xaa \x12\x89\xc53A\xc7\xd7!0r\xa2)&\xd1Qk\xa8\xc9\xaa\x80uF\xd0x\xb6\xbfz\x15up\xf5&F\xdf\xb13\xf4\x03\xc6\xa98\xc9[e\x11\xf0*B\x14 \x00\xb2\xf3\xee\xa6\x85N\xb3\xcfn\xf5\xbc\x9c\xd2+\xb2-t\x14\x19\xce\x14~,\x93\x13u\xe4=\xc7c\xf7&k\xb5Gv\xab{J\x80\x9d\'\xce\x84\xa7\xaa\x8a\x12\'s=<\xec\xecg\xe1\x8a\xaf\xa2\x0e\xae\xb1\xb9\xaa,h-C\x0fe9\xf5\xcf\x87\xb1\x0c\xd0:\x067\x87\xb2\x9aEH\x11\x11%\x11\x14\x19d5t\xb3\xf1t\x0c\xc0\xcf\xc7\xe1h\xc1;\x9a\xbd\xb2\xfae\x17\xa6\xe2F\xd9\'ZlH:|\xa7\x15\xb2~_\xe7=w\x8d\xe7d\'\xde\xec\xcb\xb3\x99\xf6U\xd4\xc1\xbb\x990I\xac\xb6>x3%*\x1f\x11\xd7nI\xdf\x9f\xe4DiE\x04\x86]\xf1J7\x00\x82Nc\x0f\x185nd\xe3%\xb7B\xd0U\\\xe68$\xc7\xaa\xdb\xedO\xbb\x05\x0c\xb1\xf08\xec\xf2C\x06\x1b\x87\xdb\xa4"I{yh_\x86\x1d\\\xcdD\xd6N\x02_\x85\xd6C\x03\x14m\n\xd8K\xa4I,\xf7mY\x01\x19x\xc6\n\x93\xab\xfac\x9e\xe6\xd5i\x0bu #\x9f[\rC\xf1\xe0\xdc\xd3\xb5\x98\xcd\xe6I\x1a\xa8Cr\xcc\xe0\xc9DXXC\xab9w\x11\xf2\x80G\x13\x03\xb6\xed\xf1\xfcd"\x7f\x19v\xf0G\xbd\x82\x08\x84z\x04\x9f\xf2G!\xa1\x95\x03\x1a\x1f\xbd\xcbH\xba\xa1\xb5\x9eb>\xaa\x86\x00\xb8T\x1e \xaf\x89\xe8\xda\xe6\xc9\xe4\xb0\xe80\xa2\x94\xa0\xc1\x8bM\xea{\x13Dv7.\t\x0e\xe3\xb8\xb91\x8e.\xc4\xf6\xdaC_\xafD:+=\xf5,\x8f\xefU\xd8\xc1{l"\xf4\xfar\x8f\x876\x84\x15UI\xb0\x9b\xc5\x01\x9c\xcb\x93\x927K\x04|@\xed\xa4\x97obFGl8\xb2[jc\x0b3\xe2\x8f\x02E\xea^\xe8u\xe0U\x05i#\x01\xec<\xb6\x94s\x8a\x86\x19k\xb8\x8d\x1bE\xb7\x96C\xe2\xe3\xb3w\x1f\xaf\xc2\x0e\xaefB8E\xaf\x82\xf2A}h9\xb9?\x88\xa7\xe3|\xb5\xae$\x9d\x81\xb7+\x1e\xb1\xae\x06\xc2\xe7kgJ5\x0e\x03\xb3\x8e\xda\xd2\xd4\x00\xc6\t\xde\x08\x1b`\x14\xbb\xe2&\xdd\xe4\x938\\\xa0C\xb9\xcd;w\x1cfz\xe6\xc3-\xc4\xcb\xd7\xc3\x085_\x85\x1d\xa4\xee\x04k\x02E(\xe2A}\x0c>\xc1^Nj\x8a\x9d\xd6FN\x0f\xa2\x9an\xd9b\xd6\n\xf0\xcey\x86\xe7q4\x92qt}\xb1V\xcf\xbe\xe7\\DX\xb1/R\x8d\xf5\xed\xd1\'aP\x8a\xd5f\x9b\x82\xddnH\xbd\xd0\x8a\x85A\xefM\x14x\xb2\xa0\xbc\x8c;\xf8\xe3&k\xcd\xb3k\x11zx|\xdeO\xe0U*\\\x9e\xb1j>\xb2\xbc1Gr\x07\xeb10\xac\x17\xbcX:\xe8\x00\x8ezBfZ9\xc5\xbbNf\xaf\xc4\x05\xa8\xb8\xcd\x05\xf1\x15\x1f\'"L\x0bZ\xeah!D\xd27\x8d\xbcQN\x03\xb8\x93~\xf5\xd0\xf5o\x0e\x1e\xa4~g\xd7S\xf8#?\xd6h\xf7"\xdda\x02\xcea\xb5\xdf\x19C!Ht\xb2\x8ap\x1c\xd9\xe1Kt\x9ew\x14\x80\xb7.\xc7\x9f\xa3>\n\x14\xdb\xde\xd5 \xd2\xaf\rg$\x8aN\x1a\x9e\xaa\xc9\xe7\x83I\x1b\x1a\xd3\xba@m\x14\xd8b\'\xb2\xbf\xc7\xc4\xff/x\xf0\xf7\x9c\xf5\x01\x0f\xfe\xdbS\xea\xfe\xf7\xe0\x06_\x0e!x\xd8\xab\xefK~x\xd3\xf0\xc1\xbf\xd1\x96\xbe\x1c\x9f\xf7\xa6-}\xff:\xbf/\xc0\xefM[\xfa\xfeu~_\xf8\xde\xbbN\xe9\xdb\xd7\xf9}\xc1y\xef\xda\xd2\xb7\xaf\xf3\xfbB\xef\xde\x15\xf8o_\xe7\xf7\x05\xd6\xbdiK\xdf\xbf\xce\xef\x0b\x9b{W\xe0\xbf}\x9d\xdf\x17\x14\xf7\xae\xc0\x7f\xfb:?R\xff\xfbJ\xfd\x97c\xcd\xde5\xba\xfd\xed\xeb\xfc\xbep\xb5w\x05\xfe\xdb\xd7\xf9}\xc1h\xef\xaa\xf8o_\xe7\xf7\x85\x9a\xbdiK\xdf\xbf\xce\xef\x0b${\x9b\x88z\xf7:\xbf/L\xec][\xfa\xf6u~_\x10\xd8\xdb\x02\xff\xdd\xeb\xfc\xbe\x10\xafw\x9d\xd2\xb7\xaf\xf3\xfb\x02\xb8\xde%\xf5\xdf\xbe\xce\xef\x0b\xcfz\xd3\x96\xbe\x7f\x9d\x1f\xf0\xd5\xcb\xd7\xf9}\xb1V\xef:\xa5o_\xe7\xf7ER\xbd\xedq\xde\xbb\xd7\xf9}qR\xef\xda\xd2\xb7\xaf\xf3\xfb\xa2\xa0\xde\xf6\xec\xe9\xdd\xeb\xfc\xbe\x18\xa7\xb7]\xee\xbd{\x9d\xdf\x17\xc1\xf4\xa6-}\xff:\xbf/>\xe9]\x97{o_\xe7\xf7E\x1f\xbd\xab{z\xfb:\xbf/\xb6\xe8m\xba\xf4\xdd\xeb\xfc\xbe\xc8\xa17m\xe9\xfb\xd7\xf9}qAo\xda\xd2\xf7\xaf\xf3\xfb\xa2~\xde%\xa2\xde\xbe\xce\xef\x8b\xe9y\xd7)}\xfb:\xbf/b\xe7][\xfa\xf6u~_<\xce\xbb\xb6\xf4\xed\xeb\xfc\xbeh\x9b\xb7\xe9\xd2w\xaf\xf3\xfbbi\xde\xf5i\x93\xb7\xaf\xf3\xfb"e\xde\xd5\x90\xbe}\x9d\xdf\x17\x07\xf3\xb6O\xee\xbd{\x9d\xdf\x17\xe5\xf2\xb6OA\xbf{\x9d\xff\x11"\xc2\x1c\x9fBD\xe0\xff\xb2\xa4\xef\x8cay\xbfa_\x84ay\xbfa_\x84ay\xd2\xb0\xff\x80V\xf2E\x18\x96\xff\x81\xa3\xf8\xc1\xb0\xbc\x07\xc3\x02\xff\xd7\xa6\xff5\x86\x85\x14)\x0eeD\x96\xfa1`\x0c\xa7)\x06\xa2P\x92\x81H\x82\xa6a\x14\xc58TdE\x0e\xbb\xe3\x05X\x96\xc5a\x1c\xe6X\x81\xc5\xd1\xbbm\x10\xca\xfd5\x86\x05!\x04\x0cbI\x88\x16(r]\'\x84\xf08\xc2r9\x15\xe85\x18\x96\xdf\xcb\x02}\x9f1\x07=\x0eC\xa7g\xb9>1\xbb\xf3\xa6s\xe9\xc3N,\xd8z\xa6\xabpge\xb3\x95\xda\xc0\x91\xa5\n\xb8\x02Q]\rL!\xda*\x9b\x96\xdd\xb7\xce \xaf\xcdU\xe8\xce\xcc\xd5\x10/D\xa6\xd7\xc1Y\xf6\x19\xb5\x1e2Q\xf0\xc5\xe8\xfa\xe4\xbc\xd0\xd7`XV3\xf1\x7f@8\x05?\xceD\xed\'\xcf\xb1\xa9#i/\xba\x0f\xde\x80\xf9J,s\x1fC)\xc2{\x9a\xb5Q\x15^//\x94A:\x0c\x99\\\xe0.C\x8f\xe3\\_\x04\xca\x80\x90\xd1\x06\xb7\x9dwV$5\x05:\x9a>\x90\x01\x0bT\x12\xb1E\xad/\xe1\xb0\xfc\xb0q\xd528\xb1\x06\xda\xc3\xcc\x9e4\xd3hg\xd8\xa1\x19\x7f\x86\xe7\x83\x1a\x01\x15P\xcb\'{Co\xcdn\xc0$^>\x05g\xe8\xaa\xa1>\x0bb\x02\x83\x83\xbc\x06\xb7$~I\xf0C\xdb\xedq\xd8\x15\xe9\x94,\xc1Z\x05.\x9b\x8c\xb1\x9d:\x00\x97\xf0W\x80\x92\xb7rX\xfe\x102(\x84\x11\x10\xf10c\x0e\xeb$cw2\xc7\xc4\xd31\xeeh6\xae\xef\xc7\xd7\xdd\xd69^(\xfd\x94A\x16\xc8\x1aW\xe1\xa4\rN\x04\xed\xedL\xb3\x13\x126\xae\xc1T\x1a\xb2\x03%\xd5E\xa9\x94\xdc\x90F\x1a\xec35T\x94\x91\x03\xd3n]\xe2\xb7\x1a\xf0\xf4\x12\x0e\xcb\xa3\x1c\xfc\x97\xb8\xa7O\xfa\x04-\x8d\x1f\xa9\xdd\x85\xa2\xd5n\xe2G^5\xf7C\x8f\x8e|}\x852M\xce\x83n\xc7\xc8\xb6\x9d\x129o5a\x8alq\xb7!ZF\xd4\xa2L+;\x8b\x8e*3u<\xc9VID\xf4\x97\t\xf8\xd5.\xbe\x95\xc3\xf2{\xdc\xaf\xc5\xe0\x9e\x16\x1ff\xbb\x05U\xbf?\xa1\xc8\xb9n\xa4:w\xf1\xb4\xad\xa7\x02\xd5\xb7\xc7\xadBJ\xfb\xf0\xa4\xb0\xbe\x0e]\x95\x91?\xf01#\x10\xa7\x18.\x0ej\x82\x07v\xdaW\xf38`\x84%\x8d\xf0\xbe\xbd\xc0\x9aI+j(\x1c\x17dG<9w\xf15\x1c\x96?\xb2\xf8\xda\xbc\xa0?\x19z\x86B\x90\xb2\xc6\xeen\xe0K\xf8\x80\xad\x02\x8b\xe0\x18wA\xc3\xb0\'\x84\x0b]\x18h\xc8\x8f\xc0m\x14\xed\xb4uy\x9e\x0ctBg\xd8\xb9tA\xa5\xc3\x08\xad3\xc4p<30\xd8\xed\x13aJ:\xd2=\x1dk\xf6\xc9\xb9\x8b\xaf\xe1\xb0\xfc\xeeM\x82\x84\x11\n~\xc8\xe3\xa7r\x8af\x8d\x92\xa6^u\x95\xdd\xcdw\x140\x9cn\x91G\xe4J\xca\xb7\x11u\xda\\\xd5\x0c? \x97\xb3\xba\xb8X\x84\x1a\xda\xa4\xc8\x97\r\xa5\xa7\xae\x17\x9e/"\x99o\xf28\xbcU\x87\x81\xeb\xa9\xe5\x94nC\xd2y2\xc1\xbd\x06\xc3\xf2\xbb\x95(L\xd2\xab\x95\x0ff\xee;\x87\x86\xc7Z\x1b"\xf9\x90\x8a\xed\xc2\xda\xc1&\xcc\x8e]L\xcf\xcbp]\x03\x82\xb7a\xa3\x02)\x11\xebdO\x9du\xcf\xd5\x1b\xee|\xe0\x037\x9b\xa5\xeel\x96M\xc3\xe2\xec\xd4.\x18\xa5\xb7$;\xdd|\xed\xf8$;\xe85\x18\x96?\xce,\x82\xae\x85\xfcQy\x18\xb5\xc0\xb5\xa1\x1b2\x17\xb7\xb7wZ\xd2G4\xd9\xcaY\x93n\xddQ\x9b\x04;\x99|\x14J\x02\xab\xc2\xf04\xda\xdb\xe4\xdeRTm\xc24\xb0\x0c75\xd0_\xed\xcaD7\x9b\xa0 \xe4\x84\xd9\x92\x111\x05,\xc9\xfdB`\xfd\x9d1,\xbf\x1f\x16\n%\xb1\xb5\xe5}\x98\x10.\x1cO\xd92\xcc<\x99\x9cAi\xcf\xa2)~\xda\xa31\\\th\x8e\xf0\x86\x9fg\xfc\x10\x9d\x08,\xcc)\xbb\xa6QrO\xb5e\x86\x8a\xb8x6s\x9f\xec\x07\xd1\x06\x9c\xfc\xe0S\x16\xda\x1cY\xa3N\x0b\x84\xf4\xa4/\xc1\xb0\xfc~X\xee\xc8T\xe2\'\xf3\xdeI(\x0e\xe4\x94\xe1\xb8\xe2V\xc9\xc9\xc2\x96\x8do)R\x92\xb8sI\x80\xdb\x1d\xdeS(\xec"Q~=f\x95\x00X6Wo\xf8jF\x06:\xa6R\xfe\x12\x9fua\xde\x8d\xdbi\xb6\xec5\x7f\xa6d\x8a\x987\xe1\xe9<\xfe\n\x0c\xcb\x1f\t\x8e\x82\xb1\xf5\x85\x1e\xe7\xe7nu\xc1\xf3\xcc\xf1\xe6O\x12\x1fUdx\xf0\xf6\x96\xb3\xd9\x1c6\xfeU\xc5k~\x0f\x83H\xe4-\xa9[[\x03\'E{\xed\x92_\x95\xed-"\x84\x11v(\xa1\x9al\x06$\x9c\xe1(FP{l6S\x18\x1c\xcf\xbf\x9a\xd1\xf9w\xc6\xb0\xfc\xbe\x8b\xf4z\xf2 \xf8\x91\xd9\x93\xa0\x81\x14\x92\xfa\x8d\x87\xe2\xe3\x85\xde\xbb\xbc\xab\x08e\xc7m[\xfa\xca\x02\x91)CP/\x10i\t\x1e\xe1\r\xb9-7T3\x9e}Y0\xb4\xab\x90h99\x84c\xbf3\x95s\x02\xc1\x11\x1dn\xaa\x03\x00b{\xefI\xd4\xc3k0,\xbf\xc7\x04\tc\x18\x89?\x124\xd3\xd3\x19\x87\xcf\x07\xa7\x1d\xc9\x1b2[;l[\x85\xa8\x9f\x11b\x91\xdf,\xdf\x07\xc0\xd9\x02\xa0J\xee\x8e\xf0\x80Q\xe3y\x16\xf0\xbd\xa4\x0f\xd5*\xff{\xd5\x92\xac\xb2\xd4l\x97\xc4\x98\xc8\xd7\xb7\xd5\xfdK\x03\xa7\x82\x18\x9e\x04w\xbcft\xf6\xefm\r\x8c\xde\x13\xdcC\x99\xb8\xe95z\xbdu\x1a\xeb\x808\xad\x92\xc9\xcc\x91z\xc4e\x99\x1dM\x92\x8b\x96\x90\x82\x9a\xa6#X\x89WG\x81\x12\xddR\xf7\x92\x047\xc1\xc4\x89\x06M\xd3\xfap\xee\x9c\x08>\xa8g\x82\xe0Q\xaeW\x89t\x04K\xee\xc9\x04\xf7\x1a\xd8\xcc\x1fy\x9c^\x0b\x02\xf58p\x95Z\xae\x1a\x1c\xe2\xae\x1d\xe8\'{G\xef\xb0\xa06\x0e\xb1\xde\xbb\xa4\x89\xe1 \xd6\xdd\xd2\x8aZ.>\xb59\xed\x9d\x9a\xb4(e\xefR\xe3Mb\xa5\xb6\xbaD\xf1\xce\xf4\xd2\xdc1\xb6\x16^\xd2\xedH\xc4\xee\x06j\x19\xecIm\xf3\x1a\xd8\xcc\xeffB\x04\x86\xaf"\xee\xe1V\xe5\xe8C\xf3E.\xe3\x98\xa8\xe4S$\xc3\xa7}\x9cD\x17}j.\xc1@%Rv\xe9\xa6*B\xb0\x05\xba\x86[\x1e\x86\xab\xa0\x03P\xea\xc2\x99l\x0c\x9b\xa6g\xf3\xe8\xe4\t\xb2U\x86\x11\xc4\xe9I>7B\xb1\x1f\x9f5\xf3%\xb0\x99\x7fj\x1b\x82$\xd6~\xfc\xcfffK=\x1eg \xca\xf4\xd6+\x95\xa9\xbam\xf6\xd3t`\xc5tBY]\x92n\xe7\x9b\xb7\x1bv3jV\x17\x97\xeb\xeb\x1e\xbf\x11\x99K[\xb3\xd9\xd3\xa6\xaf\x9c\x99\xc8\xac\xae\xa0\x84Qal#T|\xe3\xd6\xc6\xfd\xf2\xe4\xad\xcak`3\xbf\x87\xe6\x9a\x9e\xef4\xcf\x87r5b\x83_\xec\x0f\xe5\xe1`^]\x8d00\xbf\xc8\x80\xdd\x01\xf5tvh*6\x8b(\xf1x\xa9\x8dMf\xd6\x12\xa5l\x83#\x9c\xdd\x1c\xc5\x0e\x8f\xb7.d\xc3@\xa2\xd6\x0c\xb8\xbde[+\x89\x8dQ\x07\x1bRY\xea/\x81\xcd\xfc\xee\xcd\xfb\x833\x94\xc4\x1f\x0e-K\xd4\xc9U>\x0c\x01\x8e\xf6\x14Y\x92\\q(D\xb8G\xbbU\x85\xdf\x10"5\xea\xab\xdc\x0e\x93\xce]\xfc\xc4\xf3\x90\x90H|)\xde\xdbq8\xd5E\xd1+\xe6\xe5\xea\x04\xe7D\x8b\xa1"\xdc\x12\xab\xeb\x8b\xb2:?y\x7f\xf4\x1a\xd8\xcc?\xaf\x04\x88\x9f\xf2\xaeBoj\xb43\'r`\r\x04\xec\xb1a\xb5\xfa\xb6\xe4D\xbck\x815\x01$)y\xc4\xfd\xc3\xb0\x8aDf\xb3K\x03\xd4:N\x8a\x12)\xc8 \\\xaa\xb4\xc0\xf0\x13$t\x1c8(iasc\x90\x932q\r\xd9/\x81\xcd\xfc\x11\x9b0Nb\xf4cl\x92\x81\x13(\xa0\xba\xdd\xb1\xfe|\xbb\xe9Y\xa9\x0b\'\xc0=g>\xe4\t\xb37\xef\xbc9\xea|\xe7\xe0/\xf1RFW\xcf\x81\x96L`.5\x87\xb0\xa1\xceD7\xb3\xa9\xf0p\xa3\x1fK\x029n!R`Ye\x03?)%_\x03\x9b\xf9g\xa6\xc5Qh=\xf4\x7f6\xd3\xbblIIO\xe5}\xdb(ls\xb0s\x0b\xbc.0\x04]\xf3>\xf1HaY;+\xc6\x93\xadm\xdd\xb2\xfc\xee\xb8\xdd\xad=\xcdf\xba\x90QB\xde\nt\xa9\x8c\xcei\xa8\xbc\x88\xdc\xc4\x07ge\xdco\xe0$\x87\x8f\x0b="\'\xbfaA\x14=\xa3\x08w83\xd8r\xd0z\xd4D,\xd9\x8e\x9e,\xcb\xaf\xa1\xb0\xfc\x11\x9b\x04\x8c\xc2\x10\xf4\x90\x82v\xca&\xdd\x15S\xd5\x94\x93\x1e \xe5e\xbbd\x13a]|\xee\xc0\x03\xd6\x06F\xddzOg[\x93\x12\xfc\xcc\xf0B\x1c\x91\xaf\xb9\x0f\xf6\'\xbb(:\x18H\xa5}\xc9\x1d\xd6\xc8\xd1\x8etY\xb2\xfb="\xc9\x00\xfd\xcb{\xb2\xb7RX\xfe(\xcbk\xfa\xa2Q\xe4\xa1\x9b%q\x12Lm)\x8e\x92\x8c\x891\x1e$\x01\xcc\x9e\x9a\xa0\xb5s\xc5Rw\xc1^Duy\xb9\x95\xe0\xd6\x10G\xa1MGg\x13:.\xbd\xb9\x9a\xfc5\x92\xed\xcc\xb3\xb1\xa1\xd1<\x9d\x94l\xc7\x9d\x08\xba\x18\xf54{\xf2\xd6\xf35\x14\x96?$3\xbe\xa6\xad\xb5\xb1\xf8\xb3\x99\x8e\x18,\xd7+\r\x1dv\xe1PLN\xa5Zh\x8b\x91\xe9\xc9Uj\xcd\x84wV\x02$`\x0f\x19\x9c\xad\xa4@J&\xe7$M\xe709k\xa7Sii\xc8\x051\xd2\xdaRd~\xe7\xd5\x10\x16D\xa2\xa4\xf3\x89\xf6\xe4c\xe5\xd7PX\xfe\xb8\xc8\x82Q\x82\x86\x1f\x0fm\xc4&g\xf3,\x04\xfcU\xf6\xc2Ag\xf9\x8b\x0c\xf4\xe3\xb1\xd167\r\xae\x04\x9b\xaaS%Rc\xb6\t\xbbm[\x8c\x8a\xb8\xd7\xc0\x1a\x03\x9b\x03/\xc9\xf5\x16\x8d"#:\xc0P\xda\xa9\xf0f9\xef\x973\xbfvf\xbf\xea\xf3\xfe\xce\x14\x96\xdfw\x11\x83\xd6<\x0e\xc3\x0f\x89\xfc\x14\xec\xd4!\xbaB\xbc\xaa{\xbbq\x8c\xd4\x12\x03\x1d\'\xb1\xcdSx\x8dC\r\x13C\xb3B\x85\xdc\xd26\xc8\x1eh\xf7,\xbe\xc3\xb5=_\x00\xe4r\xb9\x9eO\t5\xb9\xd2V\x87\xe6\xf9l\xa2\xc0Y\xcc\r\x9d\xd9\xa7\xec\xbfIa\xf9\xe1\xfe\x0f\x85\xe5\xdf\xfe8\xf8\xff\x1e\n\xcb\xdfh\xe8\xedg\x8e\xf0\x87\xc2\xf2\xa1\xb0|(,\x1f\n\xcb\x87\xc2\xf2\xa1\xb0|\xe3u~(,\x1f\n\xcb\x87\xc2\xf2\xa1\xb0|g\xba\xc9\x87\xc2\xf2\xe9\x9e>\x14\x96\x0f\x85\xe5\xdb\xae\xf3Ca\xf9PX>\x14\x96\x0f\x85\xe5;\xd3M>\x14\x96\x0f\x85\xe5Ca\xf9PX\xbe\xed:?\x13Z\xbf\xef\x84\xd6\x0f\xd8\xe6\x03\xb6\xf9\x80m>`\x9b\x0f\xd8\xe6\x03\xb6\xf9\x80m\xbe\xf1:?`\x9b\x0f\xd8\xe6\x03\xb6\xf9\x80m\xbe30\xe6\x03\xb6\xf9\x80m>`\x9bo\x07\xb6\xf9PX>\x14\x96\x0f\x85\xe5Ca\xf9PX>\x14\x96\x0f\x85\xe5\x1b\xaf\xf3Ca\xf9PX\xfe\xb7RX\xa6g\x08\x11\xff\xc2\x93\xf8\xe6\x14\x96\xb7\x1b\xf6U\x14\x96\xb7\x1b\xf6U\x14\x96\xa7\x0c\xfbO`%_Eay\xffQ\xfcPX\xdeCaA\xfek\xd3\xff\x9a\xc2\x82\xe1\x14\x8e\xdd\x07\xfb"\x02w\x1f\xc4\xcf\xa10K\x92\xdf\xc1\'gm\x9dp\x99N\x84[\xe9\x9c\xed"z\x1aO\xf2\x1a\n\xcbj#\x81\xac?\x8eB\x8f\x83|\xcdr\x11\x0f\xa2\x87f\x99@\xcc4\x13bk6n\xcf\x84\r\x1e"Q\x0b.\xd8\xad\x8e=O\xcf#\x91+\xfc\xd4D\xb6\x18=\xe4\x18\xe6\'8\xbed\xb6\xe7l\xf8}\x1a]\x18B\rC\xbd\xf5uqV\xf4\xa9\x7f\xf2\xc4\xbe\x8a\xc2r?\xb1\xf7]\xc2\xe0\xc7\xe1k\x05\xec\x98\xa2\x80\x1dF\xea\xec\xa1\xf1\xc1\x88\xca|V\xe5\xa3\tA\x98U,PbQlm\x8e\x91j\xed\x91\xeda67\x1cb\xd9<)(\x95d\x1e/\x11\x84\xec\xb5\xd4L\x80\xbcfv\xc6\xd9\xdfY\xd7\x13\xd43\xdfkR\xf9\xab(,?vq\x8d\xae\xb5\xcc?\xc4\x04\x9d -5g\xf9\x19\xc5\x19\xf1x\x15;\xdcc`\xeb\xd2;\xb1\x95\xe3\xa5\x83\x9fTss\x85\xf6\x00\xde\xc4qp\xbb\xaa"\rE\x13\xec\x97\x165E`e\x13\xbe\xa2f\x98\xdbY\xc7I\xa2S\x02.\x00\xcb\xae\x99\xe7\xa6\x81\xbd\x8a\xc2\xf2g\xd5\xfb/\xe3\x90\xf3\\\xb7\x14c\xd8*\x91y;\x1c\xeda\xb28\x14\xbbp^|m\x00\x90ig\xaa\x1e\xae\xf1\x1e\xab\x9d\x04\xcf\x02R&\x95\xfcp$\x90-\x9b\xedn<\xd4\xee\xf8h\xb1\xf4\xf3\x98\x8a\x08\xef\xba\x83\x00\xa0\xad~{\x126\xf3*\n\xcb\xddL\x08Y\xb3?\xf6H&\xb2N\x05x\n\xfb-\xaf\xde\xba6>\xb3ldnq,\xb2\xd6-\tk\x0e\xaf\xe9\xdc5\xae\xe7\xaeK\xae\x0c\xa5\x95\x8e\x17\x13\xebKI\xb3OI;\xd290\x049\xec\x95\xa0\x915\xed\x06\xc4k\x93V\xed\xd4}\xff\xa47_Ea\xb9g8\x02Z\xb5\r\x81=\x14\xab%\x95\x8b\xb5\x83r5\xac\x9aOu\'\xb9zF0j\xc2\x0b\xe5zL4\xcc\xe8r\xccs\x13\xc3\xa7\xc5\xbdX\x9f\xa463M\xac\xaa\xc6\xc2\xe0\x99\x90Po(ax\xa11\x9f\xd5Jf\x04\xa6\xa1\xcd6\x89\xcb\'\xe7\xce\xbf\n\xc3\xf2#6\t\xfa\x07\xbf\xe2\xcffV\xbc\x198\xbb\xb1u\\\xc4\xbe\xa1\xee\x08V~(\x14{\xdd\xa8\xe2]\x05N\x15}\x0e\x10\xf3z8]\xaa\xc3&U+V\xd0V\xb5.q\xaa|\x0b\x90\xd6\xf2\x0349\xdc\xdcV;K\xc7dnH\xfbT\x9d\x8b\xe9\xc9\xf1\x92\xaf\xc2\xb0\xdc\xcd\\{\xd6\xfb\x1c\xcd\x87C\xebiJ\xb2\tD\xd7=\xe4\x1bS>\xf1\xe3\x82\xb8e\x99\t\xa1\xec8\xc7\x90\xba\x9db\x83\x90\xaf\xac~\xb6\xd4\xd0\x97z^%Zws\x06\xb0q\t\x17C\x07@-\xb3\xb5J\x87n\xbei\x95\xa4|h\x066\x98\xbeW"\x7f\x15\x86\xe5\xael\xee\xb7\x18\xc4*\xa1\xff\xbc\x8b\x8c\x10\x9c\x85C\xc6kLO\xed\tP\x9d#\xe76A]\rQ\xac\x90\x9fD\x08\x96\x8d\xa1p\x16u\xc4t\x1f\xee\x94[g\x13\xbbU\xa8B4\x94\xdba-\x07\x9a\xcd\x19\x14\xcd\x95-G\x08\xdb\x1cdC\xee*>;\xd7\xfeE\x18\x96\x1f\x89|\x95\x0ck\xd1\x7f\x88\x89$#\xc4\xa9\x9500\xe9\xad\xa6l#a\x0f\xd9\x97\xb6\xbc\x8dr\rmZF\x8dPVp\xf7\xc8\xc9\xc6\xcf\x8c\x12\xd7\xd50\x1a\x00X\x90\xed\xba=\xa9Y\xba\xe8\x91GX)\xb8\xd2\xe2\xe0\x96\xbb\x01\x91R9\x91\x9f\x1c\x87\xfc*\x0c\xcb=\xc3\xad\xce\xa4\xef\xb7H\x7f6SlT\x9b\xdc\x98\xae?\x9c\xb2.\xbfn&%r\x93S\x0cC\xf5F>\xc2Qp\xde\xf3J\xc9\xea\x93hTN\x96I\xbb\xa0\x92\x0f\xf6&\xa6|\xf0\x82\x8a\xbd\x07%\xa1u\xb4\xeb*VF\xa8\xe2\xbb3\x10\x8d\x01\xf2\xab\xd0\xff{cX~\xec"\x8e!\xd0O8Vn\x9a\x90K\x89-S\xdfl\xa3\x80\xba\x18\xe1\x84\xad\x8dplfK\xb4x\x11\x86\xe5\xf7k\x15\x9a$\x89\xc7)\xc4\x81\x04\xdc\xef\xc6N%qjhf8\x0b\x81"\xaeb\xecT\x90\xeb\xc9\xc5\x91\xdc\xc7.\x91E\x8e\xc1\xa5/\x14\xf2r\xe6\x9a\xa6\x03\xccQ\xb3\t\xfd\xc2i\x1a\x9cgGf\xc2\x11\xbb`Y5cm\x8c>\\\x94\xe4\xc9A\xe8\xaf\xc2\xb0\xdc\xcd\xc4W\xed\x81\x93\xd8\x83\x99\xb7\x03\xa5u\xa6w\x85v|L\xc0\x90\xd5\x93s9\x8c\x17\x9b\xdah$\xc6;`\xc0\xe7M\xaa2NQ\xb7X\xaeP.Z%0w\x92\xfc6 =@\xde\x9cw\x81\x01\xd5\xf8\xc6&\xac+\xbbk\xb1\x05\xf1T\xf2Yo\xbe\x08\xc3\xb2\x9a\x89\xd1\x14\x0c\xafB\xf5\xc1\xcc\x01\xb1\x1b\x89\xe2\xb2\x13\xad\xdd\x04O\x13\xc0\xa3\x8a\x0bA\xd2\xc1&\x0c\x98\x1b`\x12\xd4-\n\x9b\xf5\xb1\n\x90&8\'\xa3`\'"\xd3\xeaU\x08Ur\xb1\xa7-\x99\x8bh\xc1bXOX\x98\x18`(\t\x05vO\xa6\xa0WaX~\x1c\xda{\xaa\xfe\tm\xe6\x9a\xfa\xf1`\xf3\xf3\xc9g\x9c\x12\xeb\n\xbc\xea\xd2Pg\xc3\xd9\xe4\x9a\x96\x12\xba\xf0D\xec\x86&\x1a\x0e\xd7\x9d\xd9\x1f\x14\xca\x82\xd6\xacw\xab\x93EX\xc6v\x1f\x9e\xb5\xbb\x88\xc7\x0b\xf0\x98\xc4\x97c{\x83\xaf\x8dk<\t)x\x15\x86\xe5n&B\xd0\xab\x8e@\x1f\xbc\xd9\xf4\xde \xc4&\x81t;\x87q\xf0\xe1\xe0\xa1rY\xcc\xd1N\xd5\x02rc\x9c\xd4m\xedok\xe46\xc2\xe6\xa9\xc2F`\n\xcdl\x0f8\x14e\xd4\xa3al\x92\xc8\xde\xe6\xfe>\x18\x99\xce\x86\x1b\xb3q\x8f\xf4\xdey\xb2Y~\x15\x86\xe5GAA0\x08\x86\xd1\x07\x11\xc4lRl`hA\x86n\xae\x8d\x9b:3w\x02\xa23\x95l\x035\xb7t\xaaw\xb2J\x8b\xdf\x15\xe6e0EMR\x0f#r\xadpk]\x98\xbfu\x11\xf3@\xec/\n<\xcbp\xcd\xe0\x87\xd9\xa0H\xa4T\x0f\xdf\x8bV\xfc*\x0c\xcb\x8f\xc3\xb2&C\x84~\x84\xb1\xe7\x90w\x1bB\xb2:\xc5H\xbd\x80\xa2\x8d\xe3\xc0E\xa3a\x9e\x8a\x83\xed5\xe1\xc1\x1e;\xae\xbb[-H\x17\xd8\x01\xe4\xde\xc4\xc4\xda\xa8\x97S\xe9\x9e\xf0\x83 \xa15\xba\x11\xe3=\xd9\xf3\xdd~\xbf\xbe\xeaf:\\\xdc\xed\x93\x0c\x84WaX~\xb4Wk\xe8\xe3$\xfe\xa0>\x1a"\xdf\x0b\x1d_\xa4\xd8~/\xb9\x97\x8b1\x0cH~\xd9\x80\xa7[k@\xb7*\xd7N\x04\xbc)\xf0\x86\x06#\x1dJ\xae\x19N\x8bA\x90gY\xb8\xd0eZE\x0eYT\xd3\xa9]\nZ\xab(o\x8f\x10\xa1R\xb8Oj\xc9WaX~$r\xf8\x9e\xf5\xf1G\xd0\x14\x89\x15\x1d]2\xe7\xcdV:\x9f\x93\xab\'\xd5\x83\x91\x86\t0l\xf1\xcc$\xe8#T+:\xa5\x99\x82W\xd1\xe9E\xc4E\xcb2\xb7\xfbk_\x0bEH\xb5E\xa7\xfb\xbe\x8d\xd0}>^\xf5\xe1$q\xe7k\x9a\xb6Oz\xf3U\x18\x96\x1f\x8d\x01\x86\xac}\xf6\x83b\x16{\xe3p%\xd6~\x94\xbcA!a6Vm\x00\xe26\xaeg\xc7\xb8\n7\xc2\x95R\xe7E\x17v\x85(;\xe6\x08\x14\xd3\x84#\xcf\xc7\xc9py@\xd8\xef\x8745[ \xbb\x1c8%\x0f,\xf9\x96\x1a\xfd49e\xa9z\xf8U\xdas\x01s\xc80\xa8\x10\xb3\xf8\xa6\x1c\xfb.u\x89\x1c\x14<>>f\xcf\xe2I^Da\xf9qfq\x14"\xe9\xc7\xbb\xdd\x86\xb4\x17\xad\xa7w\xb6\xd4\xa0\x066K\x08\xa5\x01\xa8\x16t\x13\xa0\x80V\xab\n\x81\xed\xf6s\x9a@\xd4\xa0\x90\x91\xd7\x07\xd8\xed\xbc\x14\xed\xce\x8b\x186\xb3\xc2\xd1M\xc5\x16)\xf7\'5\xd0\xdc\x9d2n\x0e\x8c\xbc\xc8O\xe2I^Ea\xf9\xf1\xf4\n\x85\x11\x1a{\x0c\xcdm\x02\xecm|\x04\xe4Xi\xd4J/\x91\xd1\x0fg\xa5\x81\xcf\xe8\xb1\xd6\xa0\x9a\xbb\xe5]\x7f,,\xe3\xc0\x81\x8a\xa7F\x8e\x1cg\x87}\xe2\xa2\xc0\xe2F\x16\x1f\xc2N\xd1T\xce\x84\x1f4\x81\xf0\xae;\xa5=\xc7\xb1\xf5\xe4\xd3\xabWQX\xee\x87\x16Z\x0f-\x82\xd2\x0f\xfd\x8f\xa74-\xb0\x16\\J\x1c:\xa6\x80\xeb\x9a\xd0\xd1\x9e\xdf\xf6\xd6\xce$\xdd\xe3\xbe\xdb\xd5WZ\xb0\xcb\x99\x11=T\xb1@[)\xe3\xb1\xf6\x06\xf2\xccC\n\x06\xeb\x15-\x08\x0e\x8d\x9e\xb8bY\x88l\xaf\xeb\xe0\x9de\xf2\\l\xbe\x8a\xc2\xf2\xa3l\xae?L`\x8f\x1aK\xd8\xce\xa1\x15\xebI\x1c\xd7\xe5\x06HE\x19\xab\nW\xb9^\xb4\xe9X\x87\x8d=_+>\xbe\x8e\xf3q\x83\x96\xbd\xc4\x1d\x10\x03\xba\xf3G+\xa7\xf7\x17\x15\x1eI\xbc\x99\xccj\xeaNYcU\x88\xc5\xb3\xd75\r\xf9\xbf\xc2z\xfd\xbd),?\x9ew\x90\x10F\xfd\x84\xa4GL\xc5`\x9f \x91H"\xde\xc8\x03\xdd\xa3\t\x10;\xdc0\xb8\xc6t6\xc3qg\x1b\xed\x1b\x11\xd1\t\x9d\xde\x15\t\xadbg\x16\x11\xae\xe8\x94\xcbL#\xa1;\xdc\x1c,\x14\xbe.\xbe\xa5\xc3\x83\x824\x0e\x9cd$\xffoRX~\xa8\xa6\x0f\x85\xe5\xdf\xfe8\xf8\xff\x1e\n\xcb\xdf\x88o\xf1A\x86|\xb6\xf4\xfbo\xe9\x87\xc2\xf2\xa1\xb0|(,\x1f\n\xcb\x87\xc2\xf2\xa1\xb0|(,\xdfx\x9d\x1f\n\xcb\x87\xc2\xf2\xa1\xb0|(,\xdf\x99n\xf2\xa1\xb0|(,\x1f\n\xcb\x87\xc2\xf2m\xd7\xf9\xa1\xb0|(,\x1f\n\xcb\x87\xc2\xf2\x9d\xe9&\x1f\n\xcb\x87\xc2\xf2\xa1\xb0|(,\xdfv\x9d\x1f\n\xcb\x87\xc2\xf2\xa1\xb0|(,\xdf\x99n\xf2\xa1\xb0|(,\x1f\n\xcb\x87\xc2\xf2m\xd7\xf9\xa1\xb0|(,\x1f\n\xcb\x87\xc2\xf2\x9d\xe9&\x1f\n\xcb\x87\xc2\xf2\xa1\xb0|(,\xdfv\x9d\x1f\n\xcb\x87\xc2\xf2\xa1\xb0|(,\xdf\x99n\xf2\xa1\xb0|(,O\x11"\xe6\xa7\x08\x11\xd3\xbf,\xe9[SX\xden\xd8WQX\xden\xd8WQX\x9e3\xec?\x80\x95|\x15\x85\xe5\xfdG\xf1Cay\x0f\x85\x05\xfd\xafM\xffk\n\x0b\x05\xb3\x14\xce\x0b\x10\xc3\xd0(Na"\xc6\xf3\x90\x00\xdf\xff!0\x8a\x14)\x86\x10\x11\x1aCD\x92\xc0a\xe2N\xea`)v-\x9c(\x03\xf1\x1c,\x88\xc2o\x7f\x85\x17\x80\x85\xf55E\x8a\xc5\x04\x08&i\x94\xbfOlC\xa8\xf5\xbd\x08R 8\xe2>iX\xb8\xd3\n8\x06\xc7\x19\x8e\xc1\x04Qd\x10\x88\xe0\x85u\xdfy\x9cB\xdeJa\xb9o\xd1/),\xd4\xfd\x1b\xfd\x04N\x90\x18\xfdW\x14\x96\xef\xcf\xb0\xf9\t\x85E\x80\x19\x84\xe1\x04\x8a\xa4E\x9eCpF\\=\r\xaf\xbf\xc6\xd1\x1cI0\xbc\x88\xb2\xeb\xffa(\xb2\xd67\x14^\xdf\x9cB9\x8a\xe4q\x96\x800\x06Z\xfd\xf3\xdb\x87\xc2\xf2V\n\xcb\xba\x9d\xebI\xe28\x88\xe3\xd7\x9f\xe6!\x92\x17H\x11&PJ\x10\x04\xf6\x87\xcb9\x1c\x81Q\x88\xe5\x04\x84\x141\x8cEX\x9c\xa0I\x08[\x7f\x9c\xc3\x18\xf6\xb7\xbf9\x85\xe5\xdfF\x83\xbc\x8e\xc2r\x9f\x94\xf1\x97\x14\x96\xef\x1f\xe8_Ja\x81\x91_\xcf\xef/\x8e\xaa\xb5\xe7\x10\xc2\x0f\x1a\xf0v\x08k\xbb\x11\x93=k\xf19\xb5\xad\x81\x05\x00sS\xaf\x86-\xad\t\xdd~A\x06\xda\x08]\xb0\xe2bo:\'-\xe0\xc8Z\\\xda\xfapt\xf9\xcc\xbbA\xc3\x19\x15\x19\x1em\xec\'\x87\x1f\xbd\x8c\xc2\xf2\xa7\xb2\xf0\xdf\xcd\xc4\xf0\xf60\x8ev\xc3\xe4\xd8\x85\xbf4F* \xbb\xe0rT\x8c]*\x02-o\xef\xeb\xa0\x8c\xfc\xedq\x86\x0f\x19\x97k\xa8HW\x17\xd3\xdfeVqF\xbd\x8d\x94\xd1r\xc7\x9d}\x88\xd7\xc1p;\x97\xcd\xb2\xa1+\xf1\xd9\x89]\xaf\xa2\xb0P\xff I\x02\xa1\x1e\xe6\x11&\xa2e\xd3\xbb\xf2F\xd5\xfc\xd8\xa5;\xcf9\xb21\x8fB\x08\x162DyM9\x0f\x91\xce\xa12^e$3D\x0f\xd8C\x88\xc1^<\xdf=\x85~v8\xd7fq\x1fs\x7f*\xf6z8\x02\xcd\x01\xf4y\x97\x10\x9e\xc6\x93\xbc\x88\xc2\xb2\xba\x92\xc6\xd6\xf4N\xc1\xf8\x9f\xcdlO\xbc0\xcdq\xcf\xee4X\xb3\x02\xb6B\t\xb7\x8f\x81\xc4\x8b\xadR\xf5\xf9\x118*8\t!\x9b\xc4\xce\xb0\xb4\x99\x84\xf4h\xb1>7\\TpD\x8e\x85\x06\xa7\xeb\x01C\x9c\xfc\x14Z\xd3E\x02\xa6\xcbP\xa5\xcf\x8eD}\x19\x85e5\x93"\xees\xca\x1ff\xaf\xb9t\x80\xb2\x07j\xe7\x19\xf3\xf9F\x10,\x9e\x80\xa1\x83B\\\x91\x85R\xdc\x13\x9aP\x95\xca\x08\x0f\x1d\\\xd0a,\x88\x93T)\xc6\xed\\\xda\x17\x041m\xeaTu]8\x07{\xd8`[~\x98E\xe9\x94\xfa\xc9\xe5W\xce\xfc\x9bCX\xd6x\x80\xee\x13:a\xeca\xea\xa2\xc2M\xfa\xa9>\xc0\xf0\xa0\xedP\x0e\xb6J\xfc\x90\x8d\x8a\x14\x0c\x8a$e\x9e\x05\xf7G\xd0o\xce\x8e%g\x00h[\x04s\xf6\rf\x12\xbd\x03\x03\x84\xbc*\r[\xd8\x86\xb9q\'\xf7\x91Nx=\xaar\xcdI,\x9e\x1c\xb8\xfa2\x08\xcb=\xbbQ4\x89\xe1\x0f\xb3\xc0\xb6a\xc5\xa3\x1bu\xaf\x1e{<\xe2\x94\xb1\xe7Z\xfc\xd0\x03\xa8\x05uE\xa1\x08\x06R\xaa\xd4MQ\xaf\x14@\x1a)]\xb7\xb1\x99\xc1\xdd\xcd\xb2Ig\xb7\xdb\xa4\x82;\x9fJ\x85\x9a|~\xabB\xa1\xc4p\xd1"[\xd8\xb3s\xfa^\xc5`Y\xad$i\x92Z;\xb0\x07\xc8\xd4\xf5\x16\xac\x95o\n\x949\x061\xc1\xaeC\x19\xf3%\x82%n6(2\xd0(\xc7\xc9V\xbc\x8d\xf3\xb2\x01\xb0\x83B\x10!\xe7V\xf1\xf9\xb2l\xe0>\xbdE\xec8\xeb\x85\xbe\xd1\x15\xcb\x8b\x81\x8b\xd2o\xdakN\x16ON\xd0|\x19\x82\xe5n%\x84\xad\xc5\xea1\xf0\xf5h\xb9\x897\xc0\xe8\xaf`D\xdf\x94Bl;\x1e,\xf2\x9a\xb8P\xa7\xda\xd3Qq+\xca\x18\x9f\x1e&Qg%\x1c\x00\xc4\xbd\xa8V\xfdp\xf5\'\xd21\xacD\r\xb2\x9bg\x1c\xfb\x9e\'\xe2k{\x9c/\'\x86\xc0\x9ft\xe6\xcb\x10,\xab\x99\x10F\xde\xc7\x9d?\x14+\xd5\xd2\x94\x165\x9d\xf5\x97<.`\x0bo\xd8\xb0\xa7\x83\x9fG\xb7\xcd1\xb9\xd0\x94oU./\xee\xa4\xeb(,\x0c\x9a\xf0Ua9\x91tSt>d\xb8\xc6r\x13\x82/\x97\x9d\xdb\'W+\x02\x8f\xdd\x08\x1e\x05\xf9\xc9\xc8|\x19\x82e5\x13\xbd\xcf)\xa7\x1e\xd4\xd5\xbc8\xb7\xa5\x85O0\x13]\xa2@\xd4\xa3\x92\xf7+*\x95\x07\xe3b\x0bF\xb5Q\xc1L\xbfl\x1du<\xbb\x87\x9bn\x15X|\xf2MhZ\xe4C\xe0\xd8L\x7f\xc5FO\xf7\x12\xa9YR\xe8X\xc4`\xbf\x96.\x82\xffVY\xfce\x04\x965\x8b\xc3\x14\xb9\xf6\xb3\xd8\xc3\xe4J\xe8\xc2\xc8Z\xdd\xe6\xa5>\xc8\x8e\xef8\xa8\xb0c\xd94\xc9\x16\xbd\x15QU+\xe2\xbd\xd3\x1e\xa3}\xb9\x04\xa4\xb6\xf5{\x08S\xfa\xc64\x01hc\x172\xe8o$\xbe\x9be}"\xeb\xe5\xe0\xb6\x1a\xcdH\x99\xa8\x00\xbf\x9au\xf8n\x02\x0b\xf5\x0f\xfc>\x9c\x16!\x1e\xa7\x83\xe3\xaa\xad\n\xac\xd2\r\x1c"&J\xcd\xc1\xa3\x7f\xa9\x1a\xebX\xa0\xe9\xe2I\xf4\\\x92C\xa0\x01\xe8\xb0\xd5\xe5\xbc\xa9N\x87c\x07\x9a\xf0\xa1\x11L\xa414\xfeXbS\xd9\t\xb0\x869`}\x8b\x81.\xdeB;\xee\xc9\x04\xf72\x02\xcb]\xbf\xc18\x8a\xad-\xf7\x9f\xcd,\x85F\x927\xfb\xa2;\x92 \xde\xdd\xdb\xbc\x88!\xdd\x0c\\\x9b\xb7I\xb3\x0c\x04\xdc$\xa5\x17@K\x01N\xed\x9c.\x8db\xea\x9e%e\xd3\xd5\xbd!%,\x9d\x1a\xb4_(\xb4\x01\xdb.*\\k\xee\xcc\xae\xe0\xff\x97\x12X\xd6\x98\xc0\x10\x14\xba\xf7\xe5\x0f4\x8b\xf3\x19\xbb\xfa\x91\xcbz;t\xa6\xf2\xda\xbem\x95\x98+\xa1\x0c\xa1\x909\x844\xadrG]\xc1X3\xb8\xb1^&:\x98\xc3\xdd\xe8]=\xc1\xb8\xd6\x92\x806\x95\x87\xf3\xbc\xb0\x91\xa8p\xbb\x9dSCVw\x9c\x89\'i\x16/#\xb0\xdc\x95\r\r\xad\x02\x82|8,S\x99\xb9\x8b\xe1aq\xc4( \x99\xe8\xfc\xb2\xe3\x0f\x8aW\x14\xd2\x9e\x89]9\x05l*\xc8+\x0e;\x16\xce\xd1\xb8\x95@B*[X\xb6\xe23Z\xf33\xb5\xc5-\xf2|r:\xb4u\x89\x1d\x05w\'-<5\xa7\'\xc5\xfe\xcb\x10,\xf7\xa2\xbf\xca\xddG\xce\x0c\xb7W\n\xc5\x12\xcd+\xec\x92\xf5\xecBgO\xb7\xb3\xfa\xca\xa7B\x8c\x97\x9d\n\xe7\x15#sq\xb5-\xfc\xd8\x1dOJ\xa6\x85\xa0ZN\x88\xbdtQ\x13n\x85h\xd3O\x88\x07\x03\xa3\x08\xd7\xf9\xc9N\xb7\x9d\\@\xcfR\xd7^\x05`Y\x8f,zG\x93\xac\xea\xe6a\x9c\xf4e\xe73\xda,4\xe7\nA\xb9\xc1\xd3\xe9Y\x06N\xc8\x0eK\xed:\x08\x95}d\xd7\xf9,\x9fH.2\x8f{1\x8f\x0e\xc6\x9a\xde\x82\x93v\x086\xfb\xb3\x0c\n\xc6\xa0\x88\xbb\xbc\xcf\xe5\xad#H\xd8\\\x19\xdb\xa9\xfe*\x00\xcb\xdd\x97\xebF\x91\x14\xf4`\xa6\xbd;\x96\x88\x0cI\xf9\xb2uS\xb2\xf3\xf1\x9c\xf2\x02I]\x15\xba\x81\x18\x8d\x87\xc3\x17:\x1b\xd5\x02#\xb3\x94\xd7E\xbd\xa0[\x82\x13\xa1\xe9J67"\xd6P\xac\xdeZ\x9b\xb1\x95S\xc3\xf6D\x16C\x8a\xdb\xc1\xcd\x9f\x14p/\x03\xb0\xacf\xde;4\x04\x81\x1etj\x90\xa2\xd3 _wm`L\x90V\x17\xa3\x17\x81\x06\x08S@p\xf4\x17\x00\x8f\xac\x83\xb3\x0c{]\x1d=\xcc\xbd\x99\x8c5\xe0\x1b`[\xac?\x07\xc0xZxQ\xed\xb4k\xd6\xb75\x88\xc4\xbc\xa8(\x8f\xb1\x8e\x1d\x9f\xbdmx\x15\x80\xe5\x9eg\xa1u\xa7\x88G\x05\x07;p\x0624\x99\x9f`\xab\xdf\xfbr\x85u\xd3\xa0\x8e\nT\xb6\x9a\xa2\x91\xee\x19\xd1\x96\x85.\x1d\x1b\xc1\xb6\xc35,\x81\xa36\xf2%\x00\xcc\x97N\x8b<\xed\xe2(cj\xc1\x9c\x12%\xb3\xa9\x11\x9av\xbb\xb0\xcf2__\x06`Y\xb5\xc7\xaa\xb1\xa8\xf5\xb8?xsW\x80@\xa9\xae\xb9\x07\xeb\x02\x9d\x18\xc9y\x0c\x0b]\x92au K=\x03m\xc7\x1f\x04\x81la\xa0;\x8bko\x95\x1b6\x19l\x844\xe3\xa4\xf3\x90kEWw&\x9f{\x9e\xc3\x92\xea5.\xaf0L)\xe0\x93S\xb3_\x06`\xb9k\x0f\x04\xc6\xa1\xf5\xe4>\xdc\x1d\x15\xf0\xf9\x14\x1a\x14\xe3_\xa4\xb3\xab\xf6E:\xb4\x99\x85\xcd\t\x9f\x93\x11\xd8 ;ys\xe0\xca\xa3,\x03[\xb0\x9a\x03\x07\xc2%l\x82\xbb*\xcc\xb5\xeeF\xad\r\x9e\xdeZ\x1b\xd5k\xfc(\xf5JQC\xd4\x16\xb3\x9fT\x92/\x03\xb0\xacf\xe28\x82\xc2\x14\xf4`&\x0c+\xa4%\xb8\xdd\x92g\x8c\xa4\xb8\x07\xc4\xda\xd2r\xbf7\xabK;\xe9\xd92l\xe2n\xd9\xb6U,\x94)\x8b*!\x0c\xa1\x98\xbe\xb3\xf6\x98,\xf0\x00\xe2l\xc6\xdc\x8f2c\xe8@D\xbc\xf6\xe3\x0e6k\x81>=i\xe6\xcb\x00,\xab\x99\xab\x06B)\xea\x11T|Y\xed\xcaS\xf5\x9c\x8d\xac`\xe7"\xe1s\xa6,\xaeJ\xdc\x13u\xf5\x90z\x12\xd2\x1f\xb7=\xa2\xd8\xa0\x8d\xc7\xfd\xe1\xb2G\xe1\x18\xa8\x97\x05\xe4\xa1\xb4\x99$\xdc\x85\xda\x1eo\xa4Kfv\'4\xe0L\xe6\x08m\x85_\x91\x19\xff\xe6\x00\x96\xbb\xc4Z\x9bT\x1a\xa3\x1f\xa1D[\x96\xc5N\xe8\x9cYx|;\x03\x828\x15\x10bt\xb6`\xb1\xd0\xb2\x050\xe7<\xe8\xfd\xf9rS\xb09\x02\x0c\xd3\xe4\xce\xb0\xb2\x8dS\x93\xa4T\xb2\xee\xe7^\x16\xb4+\xcaG\nCX\x01\xd6{\xa6\x03\x85\xd8\x93\x12\xebe\x00\x96{Y\x86it\x95 \x8f\x03\xf3\x13\xabcoBwT0L\xf1\xc6\xc1R\x8eM\xd8\x0f\x82\xc6U;\xd2\x04\x88-\xec\x18Z\xd8_\xa3\xbd\xbdcL\xd2k\xdc\xb9E\x8f]\x9d\x9d![B\x08k\xda\x03t\x9e\xcfc\xe3\x1d6f\x0f\xd0\xc4y\x1b<\xf9\x04\xe0e\x00\x96\x1f\xeac\x8d/\xf4Qd\x01u9S\rN@[M\xb32w\xbb\x05\x0f\xfc|\xb3Eq\'\xef\x00L>"\xeeU[\xa4\x13\xd9\x0f5\x08\x08|_^K\x1d\xf5\r\xc5\x81I\xfb\xda*\xed\xde@\x87i\xc7t\xfefsAkX\xec\xebVx\xf2y\xce\xcb\x00,\xf7\xd0\xa7h\n\xfb\x89\xc8rv\xe3\x06\x1a\xfa\xc0\xe0N\xe0\xa18\x80)\xbd)\x01l\xa7q$QJ(_\x87}\xac\xec\xcf\xe2v\x1cz1\x03\x8f\xf6\xe9\xd8*r\xb9f7\xc1\xa7A\x8d\xb8I\xa9O8-[6*\x7f;@@\xb8m\xe7\xf3\x93\x87\xf6e\x04\x96\xfbC\x00\x14\x82W\x8d\xf5P\x96\x1dY\xa1\xcf*~\xf4nv\xe0\xe0\x93D\xe5\xd7\xaeI\xb0\xb2\xe3\x83\x0e\x997\xf0m\xd2\xb8\x83 ]H\xd2\x1aj\x17\xc3n$\xcc\x15\xe7\r\xeav\x1a\xea\x86\x86\xb2\x1c\x9d^\x86\r>;f|WnA\x88=m\x9e\xec\x0c^F`\xb9\x1f\xda\xf5E(\xe2\x114\xe3\xe0=\x01W\x01\x16r\xfb\xeb\xb8\xb3\t\xf9\xe0vq\x9f\xfaE\x01.\xf2\x86\xd2A\x88f\xb0K\x0e3!\xaf`#\xbe\xa0 w\xf1X\xbee\xe2\xad\xachm\x83\xf4\xf15\xc0\x08\x99\n\t\xb2\nb7\x1f\x9a\xe8I`\xd8\xcb\x08,\xab\x99\x08M\xae\xaa\xf4\'\xcfZ\xcf \n"\x87k\x86D\xa1sb(s\x1b\x9e\xafpt\xc3;\x10\x12\xb1\xcb\xe5\xecW\xa8\x19V\xf0\xe5$\x07\xd7\xbd\rJ\xc9<\xe1T\x93s3\x1aK\xf1\x9e\x13HP\x83\x87\xf3Nh\xc1N\x96\xe3%\xf1\xd2\xe0\xc9\x0b\x9e\x97\x11XV\xc9\xbc\xd6\xf2\xf5w\x1ecs\x8f\xb6\xd4P\xba\x80\xe3\\\x86\xb9\x906\x94\xd2\xdd\xfc\x1a\xa5\xd0\xbaqzQS\xf7-\xa9\n\x9c\x16M\x1a\xecBC#\x97\xdd\x96\xb3}\x87+\x90%\xd6\x8c9\r\x1b\xa5^\x02bkKq\x85\xf2d\xd5\\=\xfeI\xc9\xfc2\x02\xcb]K\xae\xcd,\x0eS\x8f\xact\xa1Q\xa5-\xc1\xe1\xc9\xf9\x18\xba\x02\xd6\xdd\x10\xa0\xbe\x01b\xad8\xc1\xb2a\x86\xa2\xd0\xb3s\xdep\xdd\xacg\xc3)\t\xe3\x96\x01Y9\x8b\x0f\xf51\xd3\'\x9b\x17w0\x0c\x81-\x92;L\x8e\xb2\x99\xc7\\\xd9\xe8W}\xde\xdf\x9c\xc0r\xbf\xdb\xc5P\x04\xc7\x1e\xefv\xbd\xd0\xca\xedSA,\x10\x18\x07\xd4b\xbb\xb4/\x82\xcc\x1e\xb9 \xd0\xb6\x062\xd4(\x03\xd3\xf2\xa5RV\xb8@\x18\xa8\x05cy\xd5/jW\xe0\xcf\x13\xa0\xe8T\x92\xe3\x00\xc8\xb4}i\xdcbK\x8b\xf5\x0e\x10\x03\xee\xdf$\xb0\xfc\xf8\x8c\xd9\x87\xc0\xf2o\x7f\x14\xfc\x7f\x11\x81\xe5oD\r\xf8\x80\x18>\x04\x96\x0f\x81\xe5C`\xf9\x04\xfe\x87\xc0\xf2!\xb0|\xdfu~\x08,\x1f\x02\xcb\x87\xc0\xf2!\xb0|g\xb2\xc9\x87\xc0\xf2!\xb0|\x08,\x1f\x02\xcb\xb7]\xe7\x87\xc0\xf2!\xb0|\x08,\x1f\x02\xcbw&\x9b|\x08,\x1f\x02\xcb\x87\xc0\xf2!\xb0|\xdbu~\x08,\x1f\x02\xcb\x87\xc0\xf2!\xb0|g\xb2\xc9\x87\xc0\xf2!\xb0|\x08,\x1f\x02\xcb\xb7]\xe7\x87\xc0\xf2!\xb0|\x08,\x1f\x02\xcbw&\x9b|\x08,\x1f\x02\xcb\x87\xc0\xf2!\xb0|\xdbu~\x08,\x1f\x02\xcb\x87\xc0\xf2!\xb0|g\xb2\xc9\x87\xc0\xf2!\xb0|\x08,\x1f\x02\xcb\xb7]\xe7\x7fF\x87X\x9e\xa2C\xcc\xff\xb2\xa4oM`y\xbba_E`y\xbba_E`y\xce\xb0\xff\x00T\xf2U\x04\x96\xf7\x1f\xc5\x0f\x81\xe5=\x04\x16\xec\xbf6\xfd\xaf\t,"\x82c\x02-@\x04\x02\t\x0c\xb4\xfe\x8b%E\x02\xbb\x8f\xdf\'\x11\x81!)\x12C9\x9c%\x19\x8a\xe6Q\x9ec\x08\x91\xa2x\x92\xc5X\x01\x86x\x8c\x13\xee_1\xff5Z\x80\xc4\x04\x86\x80I\xf2\x8e\xb7 9\xe4\xce+\xc0X\x0e#`\n\xe5I\x84G(\x94\xc5x\x16\x86(J@X\x8cb8\x88\xa1\x19\x96\xc2\x18\x01\xba\x8f\xcd&\xdfJ`\xf9\xe7\xe0\x92\xdf~\xf2\x8d~\x14\xfa\x07D@\x04\x8d\xd20\xf1W\x04\x96\xef\xcf\xaf\xf9\t\x81\x85\x11P\x1aG1\x82cy\x91\x15\xees\xf9E\x9a\xe3\x18D\xc01\xe8\x0ec\xa1Y\x1c\x16x\x98\xc1E\xe1>\xb2\x1cBV\xcf\xaf\xef"\x8a\x18\x82b4{\x9fk\xf3\xdf\t,\x18\x86\xc14Db0\xc7\x130\x8cs\x1c\xc4\n\x9c\x80\xe0\x1cA\xf1\xf2\xe5\xda\x9e\xa9\xb0\x90o\x7f\xfc\xee]SG([\xc6\xb9x9\xde\xbf\x9f\xe1\x9bxTy\xa3\xbaJ.\xf5\xc7g\xe1R$r\xf6\x90\x8a\xa4\xf7\x0fk-\xc7\xc5\\\xfb\x11\xedr\x92R,J\x1b\x93K<\xf8G\x8d\x9c\xe4\x8d\x96\x0b\xabj\x91q5\xdd6\\\xc9(V\xd2\x9cw\x88\xf6\xe3}\xe2\x19\xcf\xe3\xba\x1c\x8f\x12\xb5\xd6O/\x8fy\x19\xd6\xaa\xeb\xb4+Yx_\xf4\xe5\xc9\x92s.\x13\xb8\x8au\x8fpP\xe9fW\'\xfciR\x03\x03ir\x13\xa9\xf6G"Mx\xb1\xd7\xb0\xfc\xc0I\x8ct\xdd\xc76\xcb4k\x16\x10\xf6\xcc$\xbb\\\x82)l\xe0\xfdx/!qW-\\\xcc\x9a\xc5\xeb%3\x1eL\xa32\x82\xcb!!/E\x8a\xb6\x94\xc5a;\xe56U\x07HI\xb7\xf4(,\x81{\xaa\xb6\x17#\xec\xe5T\xd3\x8f\xd2\xce\xb3\xa0\\\xc1J\xe3pi\x99-.\xab\x9d\x8c\xde\xac\r[mP)[`[\xd5dz\xeb\xdd[\xdb\xff\x98\x9fs/B\x7f\xf0s\xe0\x9f\xf2s\xbe\x7f\x9a\xfeZ~\x0e\xfdk\xb0LW\x9chm\xc4\x9cb\x8f\x18\xe1\xd1\xa1\xd2z\xe3\xf5\x99_\tKr\xbe\xa8zL"\xe3\x1c yO\xc7>\x17\xef\xf5q\xae\xd6%\xb3\xf4<\xaa[8\x15\xcb\x9a\xe17=VC\x94d\x95\x12m\xee\xa0\xa3\xa3\xc3O\x0e"{\x15?\xe7GQ\xa7WG\x93\x8fl\x19\xb1\x10\x92\x06]\x18\xd2\x026[\xb8\xd9\x0b\x92\xe4d}\x1a\xdb\xa5che!\xeb\xc8)\xa5\xe3\xdc\xa1\r\x9f\xd7\xd4\x90\x19oN\xe3\x98\x99\xed,\xce\x08\n\xfe\xb6/\x86[\x1c;\x11\x08{\xe5\xb1b\x97\xfe$n\x9e\x9c\x0f\xfa*~\xcej\xe6}\xd4\x1a\xf20\x8a\xe8\xe8/\xaa\x86\xc4m\x17C\x9b\x1b\x8a\xe4M T20\xb3\r\x91\xd7\x97M\xdb#\xbb\xdd\x90v\x07\xce\xdfO\xe4\xd4\xcdC\xca\x120\xc8c\xba\xa6\xe3j]l\x84\xe4d\xe1&#\xcet\xb2\xb9\x95\xa4^j@f=9\x85\xecU\xfc\x9c\xbb\x8d\xf7\xa2KP\x8f3\xe5"\xe9\x88RRB\x9fM\x12\xd5\x11\xbe6\x10\x9e\xaf\xe2\xf3\xf6p\x9aw\x8b\x7f\xb8\x82\xeb\x92\n\xe1$\\D\xdb\rF\x98\xf7\x1b\xd4\'\xed\x8d\xa0\xce\x19\x0e%fv\x90\x1b}\xc0\x1bm\xc7h\x98_R;\xd4.\xd5_M\xcd|3?g5\x13^\xbb\x03\x12\xc2\xf0\x87ak]\xcb\xf3\x80\xdc\xd7ShVFt\x80\xab[\xa4\xc5\x95\x0bR\x9b \x15m\xe7\x84 6\xcbZ}~\xb9\xb4`\xb7\xc5\xf4\x8a\x8f\xb6n\x12\xc3Y\x1b\xd9\xfb\x8by8\x9e\x13\x87)\x96\xc6\x979\n^T\xce\xa8M\xfaW\xc3\xd6\xfe\xde\x00\x9d\xfb.\xaeM\x12\x8c\xd0\xc8\xc3h\xf2\x93\xdd\xea;,Q`\xad\xa4/\x17\x83Dc\xb3N\xe2\xa2\xeaf\xb3O\x08\xb4\xdd\x915\x12\xf1x\x9e\xd2\xccP(\xecNg\xe4\xc5\x03\x83\xdb\xa0\xd9\xe7]\x92e\xbb<\x9c\x9a9\x8e[9J(XO\xa2%\xba\xb2O\x0e\xb3~\x11@\xe7n&L\xe1kh=\x9a\xe9u\xa7\xcb\x18\x98\x00(\r\x87T\xb8v\xa6Xr\xa8\'\x10!,\xd5\x04\xb5E9\xd9\xb7\xc9\t\xf1\x99\xae\xdd\x16z4\xc6\x18E@\xa0Z\x15\xbd\xe7\x15J\x9d\xd6\xce\x85\xf5\xe5\xb5\x02\x02\x95:\x84\xad\xed\tR\xf6\xe4h\xf2W\x11t\x10\xfa\x1f4M\xa1\x08\xba\xd6\xee\x87\x19\xa8\xa5\xb9F\x16\xb8\x8f\x8d-\x89\xc8\x8ee\xc1\x06\xc9(s\x19\xdd\xecf\xf6i\xa3!\x83Q\x1a\xce\x90o*Y\xe1\x9f\x8f\xf0\xad\x19\xcf\xaah\x0b\x91TT\xd0\xc5\x83\xd7\xca\x1f\xc4\x93\x17\xe7\xb5\xc5\xc1&,a\xd3\xd3\x19\xeeE\x08\x9d?w\xa0\xff\xc2C:\xdd\x96<\xc2\x8194\xfb\xee\x96\x10\xfe\xd5u\xe3\xd2j\r\xe7T\xd6\x03\xd6\x0c\xec\x01\xab\x0b\x9f\xb5\xa09\xe0ZfwH\x9d\xe1\x94j\xbbZ\xca}l\xa3\xb7(\xb78\xa0$\x9aJ\xbb\xb9\x8623\xb7l\xdc=;O\xffE\x08\x9d\xbb\x994\n\xa1\x14\x0c=x\xd3\xa2\xb6*R\xa0\xa1\x0cf\x1b\x81\xf5\x1bfvI\x90\x0f\xbd\x8c5\xda)\xd2\xe0\xe5B\x9a\xa2\nM"tCO\x8b>\x15^\x7f\x03\x88\xe3mX\x1b\xdae\x93\xed\xaf\x8bvp\x98S\x0e\xb1}x6\x08C\xf4t\xe3I3_\x85\xd0\xf9qh\xd7\xee\x18[e\xcb\x03)\xc8\xaf\xbd\x9dV\xa0\xe6\xb9o\xbc-\x94\xe8\x14\x93\xd4\x0e\xde\xd0\x08R\x89\xe08\xcc\xd2RS\xce\xee\xeay\x84\x84\x99e\xbd-M\xd4\'\xa4\xf9\x12a\xe9\xfa_\x85z\x85\xbbXF\'O,7jS\x99\xaaM\x80\xdf\x8b\x17\xf2*\x86\xce\xbd\xeac(\x89\xd2\x10\xf10\x12\xbc\xc1\x91\xecd\xfa\x80\x020\x87\x1d\x08\x9d\x83\xccW\x99\xb5K\xcbe-<\x84^b\xde4\x1d\x1bI\xf4\xb0K\x91\\+3\xd4\xe8\x873\xc7-j\xe5\x94\xfalq\xbc\xb5\r\xec5\x9dC\xe0f\x94\x02\x95\xddD\xd1\xf8\xe4\xf8\xe3W1t\xee1AA0\x82\xd2\x0f!\xa1\x1b\xf5\x89\x9b5\xaf;\x8f\x87\xbcAz\x01-\xce\xc1\x92yf\x92\x80\xbeyq\xc9\xdcc\x89\xeb\xe0.VDDd\x18e\x87\xff\xc3\xde\x9bv+\x8a\xa5\t\xb8\xff%\xbezW2O\xf7\x1b\xa3\x80\x8c2\x89\xbd\xee\xea\xc5$\xca \xa8 B\xff\xf9\xbb9\x99U\x1d\x95Fd\x95\xd9Z\xe7d\xb7\x91\xe9\x8a\x13\'\xe2\xc8~\xf7;o\xe1} Hb(w\xc0&\xa2\xed\xf4\x93n\xd9y\xbf\x92t\xce\\\xf9\xa6}\x10B\xe6F=:/\xf7I\x08\x9d\x8ft5\x8f0\xc5\xeeQh\xb8|\x9c<\x86B\x0c\x8e]k\xd7kR]\x89#\'\xc2\xf9%\xa4G\xbbl\xd1\x9e\x90 \x853T\tQ\xae\xc2Q\xd7\xe1+\n\x9dn\xca\x95\x07\xe5\xc0\x86\xbf\xc0>\xe6\x84\x19D\x14\xb6\x01\x8b|\x1fI\xce\xb51~\x16\xc7\xff\xda\x08\x9d\x8f\n\x91\x81\x81J\x90\xbb]dzjW\xc8\noyg\xa2\x81\xf6u\xb2\xb6\r\xb3\x1bn\x02\xb0\x7fC\xe8\x90\x90<\x1f\t\x8e\x13\x8b\xe8z\xdc\xe6\xbbk\xdd\x967\xbe\xbeBI\xd9\xa0!\xbe\xbc\xb4\xc5\xb0\xc4\x02\\QA\x03+(\xfe~\x84\x03\xe8\x93\x10:\xb3K\xe0\x08M\xcf=\xc2]m\x13i\x91\xb6Bop\xb7\x10\xd3\x85\xb1\x11\x14\xfa\xa0G\xb7sb\xe1\xc1r5\x19\xa7\xf1\xecFZjY\nLd\x84\xa5\xda1G\xb6]\xd712\x17\xca]^[l\xdb\xed\xeb1S\x03\xdf\xf6\xed\xf3\x8a%F\xf6g\x0c\xb2\x17#t>\xb2!\xf2\xd1\xd8\xdc\x95p\xf5\xb4W\xa5\x93\x16S\xb7\xd3\x8a4l\xb2t\xbbQ\xdeA\'4u\x92\x86\xbdT\xa7Z\x1a\xf8u\x8f\x9f\xe4\xac3eC\x9e\xaa\xad\xef(-\x9b\xf4\xd5\xa6\xac\x9c\x8b;\xe4\x14O\x9eQ\xcbq\x17\xa0Z\xed\x8e\x9b\xee\xf4 P\xeeY\x10\x9d\x8f\x0e\x15\xa6@3\x7f\x8f\x07,\xc5\xfd\xb6b\xb0IhV\x98\x96G}b\xc3Z\xbf\xe4}\x14\xa5B\xf9\xdc\xd7\x85\x04\nO\xfa\x1c\xf6\xb1\x9f\xa8\x82u\xad\xaf\xb2oi\xec)t\xf2v\xeb]\xacU\x95\xda\xc4*S\xf7\xe5*\x18\xa8\x12\xae#\xe8\xf0(\x8f\xe4I\x10\x9d\x0fm\xc2s\x0bs_\xdb\x94\xc2q3\xc0\xb7z\xcd1r\xdf\x1eQ\x1d\x12\x8b\x15g\xdb3\x16\x04\x8d\x8a\n\xca\x12_Q\x9dI\x8f\xa3\xdd\xf4\xa0\xd1>\x0b\xa23\x1b-\xa8\x0fH\n\x18\xee\x9do\x1e:\xee\x98\xa1Y\xe9\xdd\xe2\xe9\xec\xad\xa7\xf6\xe6\x15\x8b#yHO\x81c\x17\x05\x05\xe5A\x83k\xfb\xc5E\x8ck\xb5\x92 \xab\x10U\x19.WJ\xb6\x92\x19s5\x9e(\xae\x93\xd4*/C\x8d\xdb\xdc\xa2\xc5EV\x1e\x14\xf3Y\x10\x9d\x8f*\x08\xa1g\x1c\xe3\xdd\x18\xfb\x9c\x84\xce$^\x98\x9d\x91\x16J\x02\xe1\x96sI\xeae\xccS\xcdqKF\xad~\x93\xb7ml\\u9Q\xb7i[kB\x92K\xf8%l\xa4\x04\xd6\n\xcd:\x18\xea\xa6\x8e\x83<\x95\x87\x91:\xab^(\xa3\x89\xf83\xa3\xfdkCtfc\x81q\x0c\x05\x85\xd6\xef7\x91\x8c/jr[\x17\x0b#91\x9d\xbbgm\x91H\xdd\x0b>\xf9\xb7j\xc7\x1c\xf9\x13Zq\xc3$!\xfd\xdeH\x13\xdf\xd98\x9a\x87@m\x84\xba\xd5\x8a\xca\xae\xb0\xda\x81v\xe8 \x80\xad/\xe8E\x17*\x06sA\xdb\x07\x81K\xcfb\xe8|\xc4q\x92B\xc0\xaf;[\x89x/\xc7\x87.=\xc2=o\\{,9r\x1a\x9c\xaf\xba\x15\xd7\r\xe1\x10\xac&#\x93\xb8\xd7|\xda\xd8K\xf6\xcaW\\7p\xa3\xfd\xe0,\xf3\xcb\xee\x82\xe2\x03\xa6\xac\x8c\x9bA\xdb%\x8fM\x9c\x15Z\x04\xe4\xd5\xfe5\xbb:\xe6\xe0*,{F\x14\x96\x91\xaf\x8a|\xf4t;\x7f\xf0d\xe2Y\x04\x9d\x8fc,\x94\x02\x85\x1ar\xe7\x9a\xcb\x8bhM\xc6\xc6\x89v-\x06\x0f\xd11;\xa9\x19\x8c\xcc\xa0\'\x86\xa8\xbb]\x9d\xdb\xda\xc5\x1cQ~\x95\xca\xb1r,\xf5\x05\x96\x93\x9e\x160\xba\xe3r\xa9\x88+\xe6R!\xc7\xf5\xce;\xc2\r\x95\x93:^.N\xc8\xd7\xe2\xf7=\x8b\xa03\xef"\x89Q\x08\x86aw\xe7;\xdb\xdb\xc2\\\xa8\x99\xd44\x91\x12@\xabEz\x92-s=5\xc6"[2\xdbk\'\x82\xf4+Vp\xb2\x07\xcdW3%=smc8\x18\x9d\x1b\xb7\x87\x95t\x93\xdfL\xdd^!$Q\x9e\x1aeT\x95\xf5\x14\xa6\xb6\xc0\xfek\x04\x9d\x8f\xc8\xfc=A\xe7?\xfe\xeb[\xd2\xa4`Y\xf0\xdf\xee=\xfa\xc6\xe7\xcd\x8a_\xae\xabx\xe9\x1f\xc3\xcd\xba\x02?r8\xee\x9a_7?\x8f.\xff\t4\xdfe\xf3\x8d\x88 \xb4\xa18\xf9\xdbw\xfb\xcb\xaf\xdf\xc3Q\x1c\xc5>\xee\xad\xba#\xf1\xdc~v\xb7n\x18\x18\xc5\x063\xaa\xc4\xbe\xbf\xbbvz\xe0\xce\xa3\xa3M\xa4\x937\x19\x0e=\xfd\xe3\xed\xc2?\xbft\x82V\xc7\xb8\x96\xe0t\xa3V\xdf]z\x85\x94\xc4v\xc3\xdd\xd6\xb2\x7f\xf3\xe0}\x15\x1fA\x8f\xb4L\xa7\xc4a\xe4\xe0\xa8\xae\x13\xcf\xefuX\xf2\xdcz\xed\x18U\x05\x96Yu\xfa\xb2\x92t\xc1\x1b\x82\xcd:p$\xc3H\xea\xca77\xd2-\x82\xa5>BS"A\xdb\xca\xf4\xfcb\x8dTu\x88J\xba\'m\xd5t\xe2N\xfa\xf2\xb6\tA\x04\xb4\xec\xdfCt\xfe4\xbb\xe8O\xec\xd9\xdf\xfc\xf0\xb7w\xfe#v\xd1\xff\x98\x8ct\xff\x10\xc57\xdd-\t\xbd\xf8\xedn\xb1\xe1\xf7\x1b\xf1gIF\x7fv\x1f\x1e\xb86\xb0]8\n\x98\xfe\xbbk\x83\xa5\\\xb6A\xbaO\x97\xde\x13\x15\xfa?\xc3E\xfd3\x18\xd5\xff\xcc\\~\xa0PCx\x812\xff\xd4\x1e|\x7f\xed4\xab\xb2<\xea~z\xf1\xcf\x1a\xb3\xf8\xed\xefwG\xe6\x0fnW\x1c0e\x1a\xdc\xbe\x0f^\t\xb6\xdeGh\xd5o\x87\x7f4\x81\xe7{\xc9\xff\xf7\xff|\xa4\x8fK\x1b\xcd\xb75~\xfb6\x7f\xe3\x8dc\xfb\xc9sa\xff\x8bpl\x7f\x1d\xd0\xd5\x9b\x1d\xf6\xde\xd2\xaf\xbf\xa5o\x1c\xdb\x1b\xc7\xf6\xc6\xb1\xbdqlo\x1c\xdb\x1b\xc7\xf6\xc6\xb1}\xe1u\xbeqlo\x1c\xdb\x1b\xc7\xf6\xc6\xb1}e\xcc\xd9\x1b\xc7\xf6\xc6\xb1\xbdqlo\x1c\xdb\x97]\xe7\x1b\xc7\xf6\xc6\xb1\xbdqlo\x1c\xdb\x17\xfa\xfc\xed\x8dc{\xe3\xd8\xde8\xb67\x8e\xed\x8dc{\xe3\xd8\xde8\xb67\x8e\xed\x0b\xaf\xf3\x8dc{\xe3\xd8\xde8\xb67\x8e\xed+c\xce\xde8\xb67\x8e\xed\x8dc{\xe3\xd8\xbe\xec:\xdf8\xb67\x8e\xed\x8dc{\xe3\xd8\xbe2\xe6\xec\x8dc{\xe3\xd8\xde8\xb67\x8e\xed\xcb\xae\xf3\x8dc{\xe3\xd8\xfe\xb7\xe2\xd8\xe0\xef.\xf2\xcfQQ\xd3?,\xe9K\xe3\xd8^.\xd8g\xe1\xd8^.\xd8g\xe1\xd8\x1e\x13\xecOP\xcb>\x0b\xc7\xf6zS\xfc\xb7\xe0\xd8\xfe\xe3\xef \xb3y\xda\x86\xd3\xd7\x1fX\x95\xdf\x1ew\x06\xef\x93\x9d\xeb\xc3\xb1\xfb%9\x8fm\xd7\xfcb\xf5quHV\xd9\xf8\x9f\xe2o\xb0\xb3\xbf\xaf\xff\xbf\xfe\xce?\x03\x12N,=\x8e\xb2s\x1d\xeb\x13\xdfV\xe7=|&`\x17\x01\xd5de\\\x92\xfa\xb6\xc2h\x1bwa\x0fo\xe6r}\x9e\xfb\xd16\xc3L\x13\xfb\xc6`\xdf~}.\xfb\xff\x10\x1e\x8e\xf8~\x13\xff\x08\x0fG"\x18"\xe0\xb0\xc8\x91\x04#Q4\x82\xe1\xa2\xc0!\xf3\xa4w\x91&\x08\x86\xe3iJ\xe4i\x12\x13x\x94\x150v\x9e\xfd\xc2\xe3\x02\x8b2\x8c\x84\xb1\xfc\x07z\xe6\x0f\xf0p\x12\x81q\x04\x0e2\x19\x85J(*R\x18N ,\xf8#+\x11\x82(!\x14\x8f\x93\x12/\xceW\x14%\x9c\xc4iXb\xa4\x99\x1a\x87I\x12\x03#\xecK\xf1p\x7f\x1b:\xf8\xedG\xe3j\xc8_\x08\x18!1\x86!\x90?\xc2\xc3}}\xb8\xde\x0f\xf0p\x0cP1J!\x08\xd0\x07\x8eb\x12\xc6P0M\xc1,\xccs4\xc914\xc7\x01\xe5\xc08\xca\xcc\x93\x9dY\x81\xc1\x11X\x9c)e\x02#\xe2\xbc@\x12\xf3|\x9b\xef\xf1pO`\x84\xfd\xdd\xc8\xff\rx8\x1c\xe7\xc9\x19\xdc \x80\x9f\xc0E\x8ca\xe7\xa1_\xe0-\xc1U\x08\tl\x86 p\xb0@\x83\x8d\xc1\x04F X\x02(\x8a#g\x16\r%\xd1$!\xcc0\x92\x17\xe3\xe1\x10F\x149\x14#\x10\x91\x01N\x00l\x85 `\x94\xe0i\x01\x99\xc9\x0b"\xf8\x03\xf0W\x1c\x88\xceS,\x86\xb24&1\x9c \x12\x14\xc6r"G\xc0\xc4\xb7\x1f\xe3\xe10\x84\xc5\x10\x8a\xe51\x1c\'\xc0Ra\x96$\x05\x86\x11(`6\x12%\xd0\xb33#\xb4(\xc04\x8b\xb1\x12C\x00\xab\xa2\x05\x89gQ\xb0]"\xf0\xcbyK\xff]x\xb8\x7f\x11=q\x87\x87\xfb\xd3\x80\xb1y\x0c\xd4o\x801\xf8\x87\x80\xb1\xaf\xef\xe8\x9f\n\x18\xc3\x90\x9f\xc3L\\}i\x1f\xa2\xed\x9a\xc4\x90\xf0,\x94\x8c\xb5\xc7\xb4\x8d\xe9\x18\x95\xca\xa01\xb1\x11\x8c\xd6\x9a\x1cq\x0cI\xad\x1e7\xa8\x99\xea\xa7>\xcb\x02\xfe`\x91\x81\xb1O9P{%\xd9\x00i\x90d\xc1\xcd\xb9\xd3\xb1|\x15\xe2\x0f\x0e\xf6{\x1a`\x0c\xa4\x05\x02E1\x8aF\xef&\xa8b+V\xf1[o\xb7-!\x98\xde\xdc,\xb9\xdd\xc7\xdb\xd3\x02\x08ZSLWv\xbd\x17]5\xaaG\x96E\xce,+\tQ\x9b\xb2\x9e2\x8a\x809\xcf\x0fq\x8eZ\x92J0`\x91\xc0B\xbeNltl\x17\xe8\xf5\xcfpM\xaf\x06\x8c\x91\xbfP\x08\xd0\xfb\xdd\xc4\xcd#K\xc6\xf4e\xcc\x8c&Y[\xd4\x05\xb59\xb8\x88\xf0\xcb\xcdq\xa0\xb5\x8ev\xd9\xf2\xb8\xf3\xf8V\xa1\xaai:&NCAzGB\xc3!h\x1c\xc1^6\x18\x81\xe9\x88\xc7\x94\x01\xb4P\xfba\xdd\xf5\xd4\xcda\xa9\x07\xc7`?\r0F\xfe\x02\xd2;Fc`K~/&\xab\xc8\xe5\xb0$\xe2\xdc\xdam\x8f\x17A\'j\xc7\x1c\xa2\x05U\xa7iO\x0e\xde\xe8\x9f1\xb5\xbep\x17\x9ak\x99p\x19m\xb1\xf2r\xde*\xce\x90\x08qe\x86\x83|\x0e\xa0\xa5\xb7\xb1T\x13\xf5\xb8\x06S&\xe4T\xd8\x0f\x0e\xc3}\x1a`\x0cX,\xa8\xf0\x08\x14$\xd4\xbb\xe9\x85<\x9c\xa1\xa7\x95\xa4\x91\xa9\xb5F\xe2\xfd\xa2dF\xc3\x86\x0e\x86\xe3\xafL\xe5\xe4\xeb(\x9e_<\xc7\xbe\xb5\xd2%\xd2\xa4bS@\xe7d\xd1\xf9>;\xc6\x8a\xca\xf4\x1a[f\xa2Jt\xe8\x9e\x0f\xb9$\xf2\x19\x10\xe5\xbf\x16\x97\xe6i\x80\xb1\xdf\x95\x83\xdf\xef\xe2i\xf0cx\x8aY~Z\xa1p\xb0i\x13\xa1H\xdb\xed.\xf0\x89\xde\\\xaaG\xca0L_\xd3\xca\x8c\xcd%\xfe\xbcc\x0b\xcd\x92\x8e\xcdvs&\x11\xe0\x8dE\xbbA\x99\xa5p"\x96"\xa1\xec\x90\x1dW\'\xc5\x99\xf7\x1e\xa4\xd1=\r0\x06\xc4\xc4a\x84\xc6\t\xf2\xceX\x02)\xb2\xe8\x8bf\x9a\xc9F\xd5\xca\x9a\xb8:\x8db\xdfb\xd9c\xf6\xfb\x9b\xe5\\\r\xe5d\x05!\xc6e-\xac]\xcb\x10q\xa4AK%\x81_f\xe1\xf6&X\x8e\x10g\x19\x144I\x9a\xa7\x98\xd7\x92\xd5Z\xbc\x9d\x1ft\xfdg\x01\xc6\x80\x98\xa0\\\xa4Q\x04\xbf\x9f),\xb8+f[W\x16tk\x17q\x1c\xb0X\xee\xb1#\xb6=q\xcbj\xc9\xb1\xd3\xa4mX\x9c\xe8FA\xc3o\xfa\x021\x97}\x89\xbb\xa75G^\xa3sY\x93\x93,XJ[D\xba\x86\xa9V\xba\xd5\x86\x82p\xcf\xe8\xa3\xae\xff,\xc0\xd8\x9c\xach\n\x03\xb5\xd3\xdd\x1c\xec\xa4\x14h\xf9\x90uyw\x88q\xdf79N7\xa1\xa0\xbc\x1d\x1c\xa8)]\xd4\xb5\xf6cKx\xae\x1ej\xf9-\xdf\xe0\xda\t^\xd6\xc7z\xd8\x177\xf9\x10\x1b\x16\x88\xef\x93\x0eb\x05\xb6\xa6\xc7\xdd\xde\xdcg\xad"M\x0f\xd2)\x9e\x06\x18\xfb5\xc2\xe1 @\xde\xcd\x14n\xf7\'\xdf\xf2\xb8\xcdQ\xe4\x16T\x15b]jT\xfd\xa0\xa0\x8d\xafv\r\x84\xdd\x04\xcf\xd8\xad\xa3S\xbc\xd3\xf7\xc2>e\xf6\xf9\xc1\x84\x8e\xaak!\xbcp\x15-uO\x08\xec\xe4\xf7\x0e\x84\x145\xcbD\xb8\x1b9u\xfa\xa0o>\x0b06\x1b-\x82S(\xf3\x83\xb4\xec.\xc7IP6~\x8c\x85\xdc\xa2\x1a\x8f\xe7%\x84\x12\xab\xd1\xef\x86&m\xf6\x90\xbc\xd9_\x8f\xda\xa62\xe2\xcd\x16\x1e\x1d\x89\xcd,\x01f\xf3\xf8,\xb4\xa6(\xacW^\xeb\xa3^\xd3\x8cE\x8fO7\xa1\xf2\x98*]\x9f\xd8/\x15\xc8\x9f\x06\x18\x03Y\x9f\x02\xfd\x0f\xd8\xc8;c\tG\xa4T\x82\xd0o\xd9\x83\x10i\xce\xb6[Yb\x8e\xd0\xf2\xca\x83\xbaP\x9dZP\r\x94cL\xa1\xc1\xe2&$P\xbd\xb3\xa4\x96=_\xc8S\xbd>\x0c\'\xad:9\x17\x98\x9cn\xd0\xda\xdf{\xe7\xca\xcd\x0e\xf8@\xec~\xb6\x8b\xaf\x06\x8c\x01c\xc1AuC\xde\x03[v\x1b\xca\x10\xdb\x01\x92\xe5~Y\x17Q\x1c//j\xc5\xef\x14\x07\xeb\xcd\x1c\xb7\xbav[\xee\xa4\x0b\x14\xe8\x96\x1bg\x86jh\x97\xeb"-\xcf\xb8\xc2\xe7B\x86\xe3\x06\xbf\x9bv\xa8x\xd4\xbc\xb2\xdeW\xb6\xb9V\x16m\xf7`\x80{\x1a`\x0cx>J3\x14\xfd\x83\x12\xee"\x1b\xe7\xa3\xbc\xdacn\xb0\xb3\x0fN\x1c\xd6\x05\x9e\xca["\x92=8Ot\x1b\xf4\xcd\xa7t\x04I\xb1\x14w\x90\x87\xa71\xc3(\xe7\xf1\xbc:w\x9a\xa2\x9d\xe5,c\xc9\x86X-\x0f\x88\xa8\xa6\xf2-)\xf6;\xcb\xfd\x19\xee\xf7/\x0e\x18\x9bwq.\xf7\xc1\x8f\xfd~\x17Qf\xa5\r\xe8\t:/\xbb\x0e\xaf\r\x8e\x91\x1b-`X\xe3(\xfb\x17\xfaH\x1f\xd70\xd4\xd1\x1a".\xa1\x9c\xc1\x87d\x1b\xf9\xfa\xde\xdf^\xa2P\xbb2\xfc%\xee\xaf\xf4\x82\xd8\xael\xd7\r\xd4\xc1E\x1a\xbe\xf1\xe1\xe2\xc1\xa4\xff4\xc0\xd8,&p\n\ng\xee\xb2\xe1\x8d\xbf\x86]qa\xc7\x0b\xcb\xdcb\xbe= \xdb\xd5\x8dg\xbdCW\xe0Q\xa3\xc6$*\xa8\x98\xdb-\xa9\xb4g\xbd\xd1j\x97\xd8ZM\xd9z\x8bh\xa7\xca\xdc\xd6\'\x16\x8f\xa6=\\\x051\x9cK\x1a\r\x9d\x03\xf6f>\xc8\x86~\x1a`l\xaeTg|*\xa8|\xef:T\xa5\xdd\x90\x98\xa7\xcbg\xa4\x0e\xea+$\xe1a\xecT\xcayji\xcc\xd3\xd8-s\xcc\x92k\x8aG\xfd1E\x13\xdf\xdbr\x98\'\xef8\x93%\xb7\x81b\xa3\xde\xd0\x08\xa0\xf8\xc7\x9c\xa9\xb9\xf5}\xb4\xb2E\xcbb\x0e\x8fbo\x9e\x05\x18\x9b\xb5\x89\xcd\xa7\xe4\xc4\x9d\x98\xee&\x08\re\xb0MN\xb7\x80\xb9\x1e\x9a\x8b\xab/\xb6Bs\x10\x10\x03o\xae\x96L\xd6t\xaf^\xac\x1e*\xa5\xe5X&\xb7\xa2\xc6\xb3E\xc8\xb8Q\xb8_r\xa6\x0b)A\xcdl\xa3\x10Kj\xdd\x13\x99\xad\xc2E\xd3\xa3\x10\xa3g\x01\xc6@\x1c\xc7`\x98$\xf0;\xd7\xb4Py\x8b\x1a\x9b\xa3t /\x8dLy\x11\xba^^Rd\x8d\xe8\xbc\xabT:\x84\xeay&\x85+\xae\xc9\x97\x9e.\xb6\xeb\xc0\xcf\xbaa\xb2\x89r\x8bj\xaeU\xebJ\x89\xdf\xa4\x94(\xaeN8\xaa\x9d(\xca2\xdf<\x9a\xad\x9e\xc5\x17\x03R\xceu*\x8c\x12w\xa5\xcd\xd6\x88N\xbea\x8b\xc7.\'\x02\xf2\xb8\xeb[\xee\xa2\x80\xfc\x85`[\x0er\xcdP*\r+_w\x03\x94\xdb\x12\xb3O\x05\xbe\x87(\xcb\x00\x15[l\xd7\xc2u\xeb\xdfvpp\xa1\xd7l\xbb\x1fQ\x9f^66\x1b\xf9\x8f\x9e8<\x8b/6wW(\x8a\xc0\xf7\xcd\x95\xaec\xe1\xc5X\xed\xec\xe3@XJ\x85a\xc05\xfd\xd3j\xe7\x86W\xb5\xacdI\xec\xad\xc4\x8c\x91\xebj\xe9e\xf9\x11\xdd\xc2C\xa1\xef\x1dCb\xd3I9\xb0\xf4\xbe;X\xcbC\xa9\xc7X%\xb3\x89%\x96\xa1R\xf9\x0f\xe2\x12\x9e\x85\x17\xfbh\xae\x10\x82BP\xf2.)wV45\xb3\xdf\x9eH\xb9\xd0\x92N\xdc\x86hNAj\xe9*^\x12\x14N\xdb8D\xb0$\xd8]\x03yeZ\xea\x02B\x8a9t\xd8\'{q;\xae#\xb3t\xb91\xb3\xfb\xb8&\x04l:\x86\x1bQR\xf4\x9f\x1d8\xbc\x1a/6w\x1d8\xe8S\x90\xfb\x1e\xf2\xb8\xa6@\xde\x84\x9aPKk\xed\xba\xf54\xaa?Qy:\x15.C\xa0\xf6\x88nO\xc7\x91\xddL\xfb\xc9\xe6\xf8[*F\xd3\x96+\xb1\xd0\xc0\xb7\xe31\x13\xcc\xf2\x9cH!\xae\xe5q\xccK\x82j`\x923\xe05\xf9hs\xf5,\xbc\xd8\xacM\x8c\xa4\x18\xe2^\x9bb\t\x92\xe2\x9e2\xad\x10\xa2\xe4\xad\xed\x939\x91\x0cG$\xc9\xf0\x03q\xe8s!\xa4\x14\xbf\x94\xe1B\n\xaf;i\xec\rc\xb1\xdfUt\xbc\x81NG\xf1\xd2\x8c6\xad-\x86\xf0zV\x9a\xe9\x9cRe\x91E\xe7\x9b\xf4`\x9c}\x1a^lN\' \x86\x11\xc8}\xc1,\xae\x17BW;\xa7\xed\x06\x85\x0cA\r\x8a>\x90\x99\xe3r9M\xc4n\x11-\xcf\xa9o\xf3H/\xae&t\xb5X\xe7Y\xa89\x9ap\x10q\xd2\xbf\xf8\xaa\xa2\x9c\xcf\xd4J\xac\x8a\x06\xdf\xeed\x01\xbe\xd2\x84<\xe0\xc2O\x8b\x83\xbf8^l\xceV\x08:\xfb\xc5\x9dO\xdc\xb2\x05\xcb\xd4\xac\x9c\xed\xe0\xed\x05\xb8B\x7f\x18\x0fzCiW\xa2\xf3\x98|\xcb\x8e\x88$@\xe9e\x7f\x08\xe5KOJ\x85%5\xc9\xb6DG\xd75%H\xbaTl\x1b+\xbb\x82\xad\x97\xbc\xb1D\x83\xce7\xf3${0\xc2=\x8d/\x06\xc4d\x18\x0c\xc3A\x88\xbb\x03\xb7\xb7\xe9\xc2\xdfq\x97Tu\x99 d\x0f\xf1\xb2b\xa3Pq\t\xf2fm\xb2\xee\xc6\x8b\xb0\xb3\xbd)\n1\xb5\xcd\xde\xad/\xb7\xab\xbb\xb1\xac\x95\xe2H9\x83q\\\x1f\x9aaF\xd4\xa7\x93\xb5\xeb\x86\x15q\xd3!~E>\x98\xae\x9e\xc5\x17\x9b\xc5\x04\xbbAR\xf0=-\xee(\xd2\xfb\xa0?\xc9\xe7`\x0b\xab\xd5qg\xa4-\xd4\xea\xe7\xba-uw\xc4\xcdci\xdbB\'*l\xb8N\xd7\xfa\xd1\xa1[\xb7\x1b\x17\xfb\x85\xc1\xd5\x1b\xd4\xe4\x18h\xb9\xa2\x8f\xean\xc1\x85\t\xdbZV\xe2\xabQ<<\xf8Y\xc7\xd3\xf8bs \xa7I\x14\xa1\xee\x8f\x8f\x94P\xdd^\x96d\x87\x9cN\xdd\x81\xe7q\xdb\xa98.I)*\x85\n\xe8r\xf5\x0cm\n\x17+=\x82]=\x97\xe8\x86\x1d\xd2k\xe1\xa5\xd2\xae\x18\x04x8\xb2Q\xb6\xaaA\\#7\xf1\xeaT\xe6\xd62\xdf\x04\xeb\xf4\xd13\xcfg\x01\xc6\x80\x98\x1f\xd1\x92\xb9\xaf\xb1:k\xb7Y\xc7[mB[\xcaLV\x0b\xdb\x88\x8bP\tbE8\xa7V\x7f\x86\xfa\xcc\xb9\x99|RhGy\xed\x84\xb2D\xfa\xc7^\xce\x0c\xcb\x88\xad,\\\x9fT\xc9\xd5\x0e\xe2I\xf6\xe2\xf8\x84\xae\xae\x91\xac3\x1a\xfc`)\xf9,\xc0\xd8\xc7)\x19\x0c:B\x92\xbe\xf3\xcd\x8bu\t\x86\xc3\xb4fS)O\xc3\xa1!\x95\xb0\xab\x1a9\xe2\xeavy,.\x0b\xd6\xdb\xecz\xedp\xa26\xba7\xed\xb5|\xe5\\\x99r o\xcaED\xf6\xb4\x99GS\x1b;>/zr\x0b\xd27\x0e\x0b\xd5b\xf7(Z\xfdY\x801 &AQ4H\xcd\xf7!("Q\x19N\xda\xe6\x82\xa3e|\xce\x0c\xd4Zz\x9b\x93\xaa\xa1\x81\x94\x87W\x03j`\xe7t]\n\xb2\xd8\xe1\x1e\xe8z\xae!I\xaf\xd3Z\x12\xb6*>(\xf5\n\xa6l\xb7\xcc\x8f\x863\xa4\xdc\xaeZ\x80\xaa\x93\xab\x1e\xfc\xb8\xf5i\x80\xb1\xf93:\x0c\xfc\x8f\xdfS?\xb5\xbd\xbf\xcd\xd7\xe7U\xef\x19m\xe8H\x08l/0|\xc3P\xc8\xce\x92\x94\xd3r\xdd\xe8Z~\xb0\xfd$\x08V\x84dD\xb1L\xd2Y\x1b\x94\xd7\xa3\xbd\xc7|a\xcf1Y\xe0t\xc4\xae\nY\xaf\xcb\x15\r\'\xb3\xf5\xa3\x1c\xb5\xa7\x11\xc6>\x9aY|\xa6\x1c\xdeq\xd4XJ\xd7-,\x9evjt\xd9\xacP\xe8\xd4\xc4\xd7\xbd\xa4\x90Y#\xaf\x8dh\x85\xef\x97\xc7\x95\xdb\x9c\xdc\x90^\xec\xab\xa5u\xeb\x04\x8b,6p\xee$R\'\xea\xfc\xd0K|\x80\xb2K\x89?\x89M\xb6\xf6\x1a\x06\xa7\xbfV\xf1\xf14\xc0\x18\xd8D\x8aAq\x98\xbc;\x10\x18o\x99\x8dd\n\x17k\xe7\xc8\xa0\\\x8d]i\x9e\x84/=_\xb8h\xd8\x8e\xc0\xf9\xc0\xa0\x85[\xb27e\x97\xca%\x8d*\xc4\x80\x11\xb7\xc6\x84\x9d\xb5C\x87\xaa\xf6\x99\\\t\r\x1e\x0et}Q\x12h\x7f!\xa0\xab!\x0c\xff\x1a_\xec\xa3\xf4\xfc\x9e/\xf6\xebM\x93o$\xcc\xdfoc}\x05\xe5\xea\xff\x00 \xe6\x9b\xe9\xda\x83V\x84\x84\xe9\x8a\x981\x89\x88>\x81\x9f\x12\xc2\x9bQ$\x83.\xfb\xd7\x04E\xae\xdf^\xf3(\xc5K\x84\xe9\xc1\xe2\x7f(L\x1a0S\x1c\xd0\xbfS\xcf\x17%\xa2\xfc\xefR\x8b1\xe9\x88V\x887}R\x08CP\x06\xc3e1\xc3\r\x81`:l\x1c\x7f$\xccWU\xcb,\x89\xee\xe0?\x96\xc4\xfdY\xfc\xf9\xba8\x9bo:P\x8bVx\x93^x\xa3\xe1\x96\x18\xb03\\w=\x04\xd8\x1ab\xfe\xd0\xce\xbe(\xf0\x06H\xa2\x8c\xc6\x01G\r7\x07^\x02^Br3&{\xd4\xdd\x04\xa8&}\xb5j\x9e\x8e\x9b\x019M\x81\xb5\xa2DMWAu\x81\x05\x02\xe5\xe8\xfcP\x85!x\xc0b_\xe85/\x91\xc4t\xf0\x1fK2\xa9/W\xcd\xb3\xb15@ v\x04\xaa\xb9\x01[CuW\xc4M`k\xba\xe0\xcd!\x810^\xa9\x9a\x97H\x02T\xf3cI\x8a\xfd\xcb\x03\xda\xb3\xf17 B\x87\xbd>\xb1\xb01y y\xea\xf8lk\xa6\xe0\xc1z\xa1\xdf^Y\x02<_\x12\xa0\x9a\x9b1\xe2\x93\t\xf2\x8b\xe9\x02c\x9bCsac@\x92\xc1\x98^\xae\x9a\xa7ct\xbe\x81\xfc\xd2\x1b\xc0k\xe6b\x1d\xa8\x05\xbc\xbcq~Z\r\xd8\x1a\xfaJ\xd5<_\x12`P\xc0\xff\x93i~,\x10\x047\x90\xfes\xdc\x00%\x050\xba\x9b\xb9ymu\xf6l\x14\xcf\\\xd3\xf4\xc0\xfdaS\xb0\x81]\xd9\xc0k\x14\\\x17@\x7f\x06\xca\x80\x97\x16\xcd\xaf\x90\x04\xd4\x99?\x96\xe4\xc7\x19\xf3\xebb\x87\xbe\xe9 \x90\x01{B\x8c\x99\xbe0\x89\x03\xa8\xd0@\xad\x94O\xa0\x03\xc0^\x1a\xc8\x9e.\xc9\xcf\xca\x7f}\xf2\xe0\x1f\xd7\x98_\x17]\x04\x82X\x8ek\x05K\x80\x02\x19\xd8\x95\x0ej\x18\x16\xd44\xded\n\xf9\xf4\xca\x82\xf9\x05\x92|\x9eZ\x9e\x8e\x15\xfa-\x88\xfdP\x98\x97\x06\xb1\x97H\xf2Yjy:\x9a\x084e \xe5\x0b\xa0\x1asuP\xbf\x84\xa0~a\'\x1d\xa4{\x13$\xce\x97\x06\xb1\xa7K\xf2\xb8\xdf\x7f]\x0c\xd3\xdc+\xf7\xf3\xe2\xe7n\xdf\x04\xea\x00\x1d\xff\xa8\xcf\'K\x05\xfb\xdaJ\xec\xf9\x92L!\xa6\x15?\xce\x92\xc6k\xd5\xf2tD\x12\xa8\xf8\xc5~\xf6\x0eP$\x03\xd7\x17\x81\xa7$@5:\x02:\x81\xe9\x95jy\x81$\x9f\xa7\x96\xa7c\x96\xe6\xe3\x0b\x10\xc4\xee\x0eb\x90\xb9\x97ym\x81\xfctI\n\x9b\xd0\x8ar\x9a\xfb\x17 \xc9h\xba\xc9h\xb8\xa0\xe3\x9f\xf2\x97\xf7-\xcfF5\xcd\xa3K\xfa\x8f\xca\xbe\xb0\'\x10\xd0\xc0\xef!\xe8\xae\xe7Y\\\xdeK\xbd\xe5\x05\x92\x146h\'=\xf0N:x\xb1\xf3;b\xb3\xc1\xe9B\x8e\xbf\xf8\xb0\xff\xe9\xb8\xa7O\x0cbO\x97\xa4P\x86\x8f\xb3\xf1\x1fd\xc9W\x07\xb1g#\xa3\xbe\x19#~\xd3\x05\xf1\x06*/x\x16L/@T\x9eX\xe05?;I\xfa\xa2\xf0\xab\x8fs1\xad\xb0AM\t*1!\x1ctA\x07/\x1b\xe4\x16q>\x85{i;\xf9l\xec\xd47\xe3\x80\x83\xe0%\x12\xc6\x04\x82\xd8\x04rK1\x97\x96%\x08d\xdb\x97\xaa\xe5\x05\x92\x14z\xffcI<\xe2\xc7~\xffu\x11[\xdf\x0c~V\x8b\x02\x83\xfa\xe56\x072}\x8e\xce\xc2\x9cg\x8c\x97\xaa\xe5\xf9\x92\xcc\xc7H\xfaG\x1b\xe9\x81\xc2E\x049\xc5\x9es\xcch\x149\xfeb\xb5<\x1d\x7f\xf5M?\xe0\xe3\xc7Y\xab+\xce\xe7\xae \x80y\xa0/\xd6\tS\xe0^\xaa\x96\x17H\x02:\xb0\x1fK\xc2\xfe\xe4\xbc\xe2\x8d\xfa\xfa\x9f\xae\xf3\'m\xaf\x07\xbe\x16_\xbd\xe9O\x87l}\xd3y\x1c\xd7?:\x11\x90@A\xac\x05\x82 @\x10\xf0\x8e\xeb\xd7\x86\xa8\xa7Kb\xf2\xf8`\n\xe1\x08\x929(\x15\xc0\xef\xae\x87\x83\xe2\x1eT\x8f?\x0b\xb6_\x17(6\xabe>\xaa\xbb\x81\x92w\x9cO\xb9\x80\xbd\r\xc6\xe4\x8d\xafV\xcb\xf3%\x99?{\xbc?RU\x08\xfd\xf5\x9f=>\x1d\xf65\xab\x051\xdd\xf0\xe3\x10\xd2\xfcH\x83\xec\xa4\xbb\xa0a|qB\x7f\xbe$\x9f\xa9\x96g\x03\xc3f\xb5\x0c\xf3Y\x11py\xd00\xda\xd8|;\xc2|V\xac\xbf8\xa1\xbfD\x92{\xbf\x9f\xfb,\xe1g~\xffu\xe1h\x1fj\xf9\xb8\xd5\xcd\x05\xfd{Q\x82\x86\x91\x9d\xef\x08\x1b_\xad\x96\x97H\xf2Yjy:\xb8\xec\xd3\xd4\xf2\x12I>K-O\x87\x9f}\x9aZ^"\xc9g\xa9\xe5\xe9\x00\xb5O\xcb-/\x91\xe4\xb3\xd4\xf2t\x08\xdb\xa7y\xcbK$\xf9,\xb5<\x1d\xe4\xf6i\xde\xf2\x12I>K-O\x87\xc1}\x9a\xb7\xbcD\x92OS\xcb\xb3\x81r\x9f\xe6-/\x91\xe4\xd3\xd4\xf2l(\xdd\xe7\xa9\xe5\x15\x92|\x9aZ\x9e\r\xb6\xfb\xbc \xf6\nI>\xaf@~2\x1c\xef\xd3\xd4\xf2\x12I>K-O\x07\xec}Z\x10{\x89$\x9f\xd6\xe5?\x1b\xd2\xf7y]\xfe+$\xf9\xb4 \xf6l\xd0\xdf\xe7\x05\xb1WH\xf2i\xde\xf2lX\xe0\xa7\x05\xb1\x97H\xf2\xa0Z\xfe\x1c\xb4\ry\x08\xda\x06\xff\xc3\xa2\xbf4\x18\xf1\xe5\x82}\x16\x18\xf1\xe5\x82}\x16\x18\xf11\xc1\xfe\x04?\xf0\xb3\xc0\x88\xaf7\xc5\x7f\x0b\x18\xf1\xd7\xbf\xf8?\x05"$\xff{\xd3\xff\t\x88\x10\xe6\x04\x8c\xc2\x10\x11\xbc`V\xe4x\x82\xa1H\x89\xa7\x11\x9a\x82qn\x1e\xbaK\xe3\x02\xc6\x11"\xca`$I\x93\x0c\xce\xa3\x04"\x91\xac \x8a4\xffA\xa5\xfb9a\x8b\xa40\x0e\xe3DF\x14\x18\x06\x11P\x92`\x04\n\xe6P\x82\xe3P\x16\x9f\xd1\x1a\xa8@34\xc3P\x12\xf71U\x1d\xe5EJ\x98\xa7^2\x04\x8c\x108\xf1R\x10\xe1\xdf\xa8\x05\xdf~0\xd6\n\xc7~\xc1\x11\x82\x01\xff\x11\xd4\x1f\x81\x08\xbf>\xc6\xf1\x07 B\x89\xa7\x08\x0e\x13PI\xa2I^\x10Q\x9a\x84\x11\x81!H\x8c\x84i\x14C%\x1afyA\xe09\x1a\x03J\x960\x84\xa6\xc1\xc5\x04\x89\x80\x19\x91d\x18n\x9e\xad\xf5=\x88\x90bQ\x96\x94(^B\x05|\x9e|\x0b\x0b\x0c\x8e#\x02/\xb1\x085\xa3\xa1D\x8c\x93f\xea\x08"P\x04\xc5J\x129\xd3\x02\x19\x06\x98\x1c\x0e4-\n\xdf\xbe3\xf2g\xe2\x02\xff\xb2 B\x1agf\x9a\x1f-\x89\xa8H\xc238i\x06O\x01\xdd\xc2\x1c\x07\xac\x8d\x85\x89\xd9.y\x18c\xc0\x1bJ$\xcf\x00q8^\xe2PXb1\n\x15\xbf\xfd\x18D\xf8\x04j\xe0\xbf\x0bD\xf8/\xd2\xf1\xee@\x84\xdf\xf8\x96\xe6\xf8v\xe0\xf8\x83~\r\xd1\xf9\x0eEf\xd2\x8e\x06\x1c\x06]\x1b\xd7I\x9f\x16\xd2a\xbbY\xefu\x870\x124\x15\xb7\xc1\xad\xda\xa2\x12\xbc]\xab\x08\x7f\xe8>\xee\x85\xd7\x03\x03\x8f\x8b\xf91!\x05\xd7\xd1\x0e\x94C6\x9c\x08\xea%B\xa5\xca\x90\x8d>>\xce\xcfJVd\x88q\xf56X\xa3zm\xe3b\xa1~\xfc\xec\\u\xc7\x18W%\x85TdK\x1c\x8e\x0bo4\x05\xe3\xbc\x95\xab\xd2\x90\xd91^v{PZMqML\xa0f$\xb2\xda\x18\xb6G\x05\xcd\x82\x10M\xd6\x8d\xc3\xe7\xbf\xcdD\xba)*h\x01@\xd1\xfc\xebK\xac\xf2\x95\xc7\xb6K~O\xff\xb7\\\xb51f\x1b\x0e\x8eGb\xda\x06\xc6\xfc\xcc\x18\x1aMb\xef\xc9\xfe\xc1A}B<\xe8+\xc5\xd6]h!/\xb3\x031\xa2\xf0\xc2\xddi\xe8\x12".\x84\xcc@\xfb\x8a?\xac\x89S\x90u\t\xa7\xac\'A\x19\xd7\x8a\xc8\xe7\x8a\xc2\xaesN\xccm{P\x1b}Yr\x12\xcbK\xfe>lN+\xbbr\x87\xc3\x88*\x9e\x13\xa3\xe4\xe9\xd4\xda\xabj\xedE\n\xd7eX\x83\xd6\x15\xeco\xe9e9*\x9e\xa8\xa0\xb7\xd1"o\xd0\x92v`\xe8"4\xdb\xc2\xf4\xae\xfb\xf5z\xf2\xaa)?\xd6fU}\xfb\xd3\xc8\xc8\xd9f\xfe\x10\x19\xf9\xf5C\xf2\xe7"#\xa9\x9f3\xd5\x8ch\x17\r\nEoN,\xbdDd\x1aR\x92\xa6r2BF"\x13\x98\xf0\xa20\x18r\xb3\n\xd4\x1b\xb3\xa8\xf7k\x04][\xf4\xe9\xd6o\xd8aK\xd07\x9f\x82jFp\x0fK\x9e\xae\xe3\xdc\x1a\x90\xb3zY\xac\xfb\x07\x81\x1c\xcfBF\xce\t\x1cCi\x04\xa7\x98\xbbQ\xadb\xdf\tK+T\xc8\xc6W,x\xe3\x1cq\xac\xaa\x88\xb8S\x99]\xbfG\xb5\xf8\xb8\xa5[b\xb7[\x11\x9cX\xe1\xb2f_\x97\xb7z{\x83nk\x8c\xdb_\xd7h\xe8\xb5S8\xa0!\x95\t\x83\xda\x91u\x850\n\xfd\xe0\xa8\xd6g!#\x81\x98\x04\x8a\x90\xc8\xdd\xf8t\xab5\xae\xce\x98\xdd\xec\xcbjw\xf1\x83R\\\x17<\xb1\xb9\xf1QC\x9f\xae(/\xef7\xc5\xe24p\x16\x16\xd4\x86S\xef\x87[{\xb4\xafB\xef\x979U\x9f\xd2=h\x0f\x06%\x1f\xf8r\x12\x8e\xfe\xc2\x8b\xa30\xe6\xb7\xdcc\x83\xbf\x9f\x85\x8c\x9cU\x89\xe3\x18\xf08\xe4nT\xb4z\xaa\x17\x15\x94\xe2\xe5\xe56-\xd5\x16\xd7{\xaa\x89\x10\xcfCi"\xf3p\x9f\xf4%\x9a\x1f.r\xb0Uc\xc81\x8c\xdd~\x7f\xe9\x06\xf9\x9c\xba\x0e\x94\x9e\'\x1e=\xd9\xf2\x99\xd9\x10u\x8f2E\xb0\x86\xcf\xedT\xbb\x0f\xb2U\x9e\x85\x8c\xfc\xd5b\xe7\xc1\xdf\xd4\xdd\xa8\xe8\xda\xca\x11R\xf2N\x89\x8di\xf1\xd0\xa0\x9b4\x98B\xaaZp\xa9\xe5l\x18\xd7\xa2[\x9f3m\xd8\xe5euR\xd5\xf0\x10bB(\x15U\xb0\x0b\x19\xf9\xe1\x9b\x14\tS\x0cu7,\xba\xe7\xd1\x9c\xbd\x1a1\xcc\xe9\x974L\xcf(\xddN.\x1b\x1a\xac\x08*j\xc54\xaer\xc47:\xbb\x13\xdc\xd0\x9a\x0c\xdcG\n=\xd2k\xb6\xb9vG[\xcc\x97\xf1\xd4\x85\xdc(\x9a\xab\xb5w\x99\x1a.^\x90lr{\xb0\xf4x\x162r\xd6&5\x073\xf2~\xb0\xb8\xbe\xa4+V\t\x82\xbeh\xd1\x08/\xb6\xc4:C\xd0\x98\x11;\x15*\xbcb\xda2qD\x8eC,\xaa\xe2\x1a\xf5\xd3\x89\xb6\x95!\x8dt\x8c\xafw\xd8\x14\xb4\xb1\xb9\xc6\xc6XWV\xb0qX\xf7\xe7\x85\xbb\xdaK\x82\xcf~\xa9@\xfe,d\xe4\\\xd9`3\x81\x0b\xc3\xee8\x18\x84\x98\xc4;\x99^\x94"\x13\xe7\x92\x97\x9bX\xca\xd7\xee\x04\xads\x8a$\r\xd2\x95\xcb\xadeG%\xbbK\xddSo\xac\xad\xaa\xbf6\xf6\xb89D\t\xee\xdf\xec\xb2\x94\x9b\x93\xb1\xd8\xe6\x13g\x04\xfe\x12w\x06\xca\xea\xccG\x81FOBF~\xb8>\n\xf2\x1e\x03\xdf\xf9\x04\xc4"\x01\xd6\x16lz:D\xfb\xc6\xce\x95\xf4 \xb3\xf8*?\xe3\xa1 \x1a\xbd\x8b\\\xdb3\xbf[a\xf5\x92\xdb\x06y&\x95\xa5\xad\x89\x97\xb8A\xb6\x97\x9d\xda/\xd2Ks\xa1\x1a\xf4\xdc\xd6y\xda]n\x17wR\xf7\xd8\xa3\x80\x88\'1#?\x8a\x1b\x1a\x06RR\xc4\xdd\xb0\xfd\x1b*e\x1bJm\xab\x92_g\xdaB#c#\xc7\x146\xd6nft1\xba}v\xf3#t-\xb7\x1b\x11$,C\xb5h\x8c\xed\xdc\xb2]\xa3\xc3\xd5\xe0V*\xd5\xe3Y\xbc%FX\x9e\xe0\xed\xbe\x90\x16\xba\xb5\xfeY\xbe\xfak3#?\xaa}\x8c\xa0i\xec\x9e\xbcy\xac\xdcU\xdc\xf7\x86\xb4Cz\xedZ3\xa0\xc0\x97\xf6\x8e\x10\xee\x15\xd2\xd5\x8e\x0b\xb3k\xf6\xdd\x9e\xa3\xd9\xc4\xdcc\x1e\x1b\\\x92\xda\x1a\x13\xd4\x87\x8e\xcaNXD:\xb9r\xd2\xf1\x8a\xa9\xb7\xbc\xd7y$K\xc33\xc2T\x0f\xa6\xc3g1#?\xb2>L\x02\x9f\xb8\x17S\xaerZ\x87su\xb7\x0e\xa2+\x95q5\x1b\xb6\xc31_\x92\xf6m\xb5\xd0\x19z\xc9\x07\xf9\xda\xc6D\xc4\xdb\xc5\xb8\xac7\xc7]\xb0o\x10\xe3:,\xe8`\xcc+_9u\xa6\x1b,7M\xe6-2\x13\xf5\xd1S\xcc?\xe8\xfa\xcfbF~\xa4C\x1c\x06Q\x91\xbc\xeb\xc4\x8f\xb0\x1f\x10{\x071\x18\xde\xda\xa4\x1c)\x15\xf5\x95\xdd\xae.\x9c\xa3\x9e;>n@\xe9/o\xf0\x9b\xd6\xf2\xc6\xd0.N\xa6Lf\x87\xb5}\x16\xb7DV\xc2\x9b\x05\x8b\xabU\xaa\xb1d%.L\xd9F\xf1\x8baP\xcbG\xb3\xfe\x93\x98\x91\xbf\xb6\xa83\xf7\t\xbf\xd3\xa6\xe4\x1f\\\x1f\xed\xe9\xf5\xd9P.\xb6\xe0\xf8\x8a\xe4\\u\xf3\x1c\x0e\xd0Z\xa5\xd0#\xeesY[\x9f}\xa8\x99\xbc \xbb\xaa~\xe1PY\x16\xdd\xc6C\x1d\x04\x0b\xc9\xae\x9a\xab\xce>6V\x1b\xb9V\x04m1\xb1\xab5\x0c\x15\xfb\xf0\xe2"\x8b\xeb\xd5\x85L|i{\xbb0y\x94U\xff$h\xe4,\xe6\xec\xd3\x08|\x8fl\x837R\x99\x11\x18\xac\xd7\x03]V\x19\xb4&\x0f\xcb\xe8\xe4\x9c\x12\xe6Fvm\x1e\x0bN\xc2\x9e\xc9\xd0j\xd1r\'U\x1a~t\xcf\xd8\xb5h\xeaK"\x9e\xe4m\x9e\x1e\xc2%Y\x06A\xd7\x9a{+U\x17(\x15\x06\xd5\x83\xa4\x9fgA#g\xdf\xa4\x18\x1a\xc7\xefm\xb6\xc2\x1c\xb8\xdd\xe0\x9bl\x15e\xd7\x9d\x05\x1fd\xb8?\xad\xf7S\x9d\xdbt\x9c\xd3[!\xd7.9\xbc\xa3{\xff0\xae\x87\xc0\xc0\xfa\xde\xef\x97S\xbf\x1b\xce\x1b\xae:\xc2\xe3"9$\xeb\xb2R\xa4\xdb\x11\xbd\xee\xf1\xf5E\xf4\x1e\xed"\x9f\x04\x8d\x9c\x95\x89\xa2\xa0\xb2\xa5\xef\x03m\xa3\xae\xb3\xf3\x0e\x1f&\xca\xee)\xb48c\x98\xd4\xae\xf3\xbe\xc8\xb0\xe5\xce\xdfd\xb4\xb0\xe577\xa4\xdb\xc9\x9e\xa1\xc5\xe6\x81\xb1\xdd\xeby%\'\xd8\x86H[A<{\x96[\xa1C- :\x93,K\xbcg5(\xf3\x1f\xb5\xd9\'A#?\x8a\x0f\x92\x06u\nq\'\xe6\xaaX\xb9\tVQ\x8e\xd7rP\x18\xf9\xee\xb2q\xcc\xcet\xa6h!\xd7\x857y\x98\x9f\xc5\xd0y\x83\x0f\'v\xdb\xe0\x9a\x95_\xa1H\xcc\xa8\x85\x93\xca\xad#k\xa8f\xd30\xb3\xcco\xfb\xb6\xaa\xe9I\xe4\x84(\xfdYA\xfebh\xe4G\xda\x04\x06\x0b\xfa\xc8;\xd7<\xc3\xc8Ho{\xc3\xdfzu\xb3\x13W\xfb\x8a\xa7o\xc3JDp\xbe\x08\x16SX\xa9PdB\x9c\x05m\xf8t\x01\xc9L\x17\xd8\\j\xaf\xfa\xcb"\xbc\x84Pz\x8c\xfcEn8\x9e\xd0\xb7\x02\xad\xacwM\xb2\x95\xf6\x0f\xe2\xc7\x9f\x05\x8d\xfc\xd0&\x8c\xe2\x08\xc6\xdc\xa5\xcd\xa2\xb8"\x97\x9b\x86\x1a\xea\xd8Ke!d\xb7\xd5I%\x95n\x07\xdbL\x1b\xb7\xd06k!\xbe\xda"\xac\xc6o@:9\r]X_\x81?o\x08B3\xd4\xdaj\xd6\xea\x99\x05Z>-\x8fc\x16\xab\xd7 n\xc9\x9fE\xa0\xbf64\xf2\xc3X\x10\x86\x02\r\xd6]\xba\xda\xc3X\x9e\xa5,{D\xa4Ua"\xb7\xe5\xb9Lk\xad\xca"\x08\xa2\xc6k\x11\x8e\xb0\x8f\xbby\x80$\x9cFA\x8b\xeef\x08\x84\xc9\xa7\xfci\n\xa9\xb3=\x86\x83z\x84\xa0\x89\x081>v5\xd4\x1au\x85\x1c\xf0\x07{\xf1gA#\xe7\x08\x87\x93(L#\xf7G\xe4\xc2\x90\xef\xe7I\xf1!\xcc\x1b\x16G\r\xf6F\xd9\xa6\rg\xf6\xa6\xde@p\x11\xee\xce\xb7\xcc\x1a2.woh\xa7\xedz\x1b=\xa0W\xad\x9d\x0e\xd9u\xc2\n\x8f\xbe\xd0W\xf3\x0c\x1f\x06\xae\x0bc\x0e\xe7\x15\xa8\x87\xba\x07\xd3\xd5\xb3\xa0\x91\x1f\xa5$\x8a\x82\x12\x1b\xbf\xf3\x89~A\x05\xc5pjJ9XV\x05N]\xf0\xc3\x05\x19\x08zI\xeb\xc5T\t\xca\xe2\\,O\x92\x0c\x97N\x18\x93"t\xb0\xba>I\xa5e\x10k\xcc\xfe\x14*]\x19u\xd6\x11\xa2\x9d\xcb"\xa1\xdcZ5\xc9\x9a\xd5\x1et\xfdgA#gm\x82N\x14\x06\x8d\xf6]\x17\xb9q\x94C\x1a\x1f\x87\xd5\xc5;j\xdbB\x13\xccT\x93\xa5\x1aK\xcec\x88\x97+\xbf2\x83\xe8\xbc!v4!n\xcafsk\xae\x8ca\xd3\x97\xde\xad\xa6=\xa5\xded\xcb\x17\x9d\xd3\x8a\x90N\x98\xc4m4*\xf4\xf83\xf4`\xff\xf3,h\xe4\xaf\x1fx 4\x06\xc3w\xf9*\xc7\xe1\x1dL{>\xdb(\x10u\xca\x8bd\x87s\x06&\xa0t\x93\xe3\x04\x0b\x01_\x15\xce\xb5}6a\xe5|\x8a\xed\xb5[\x98\xf10p\xe7\xdcD\x9a\xbe\xa5\x08\xab>&\x14\x8f\x16\x15\xdb;y\xd1\xae\xc5\xe9$\xe8\x0f\xfa\xe6\xb3\xa0\x91\x1fFK\xd2\xa0\xfa\x80\xef\x8c\xb6\xd3\xa5\xf4\x00\x91\xf9%I\xda\xfeD\x8e\x9bv\xeaI\x9bw\x10$\xb7\xb5dr\xbb*d\xe4\x1bt6\x8f\x98\x84Tjd\x15\xc5a_\xf3\xeb\x00\xdf\xc90\x1b-\x89\xed\xed\xa0\x1c"\xf5\xc4\x05\x10\x91\x12GI\xb7\xd1G\x8f>\x9e\x04\x8d\xfc8\x0eD\x81\x8c\xa07\xb8cc^\x12\xd9\xe1+\x8c\x0b\xcd\x91\xf4\x85\xf1\xd8d\xba\x9c\xe2\xadO\x16\xf6f\x8d\xe9\xe3m\x8cC\xa3\xe3b!?\x80f\xd3\xbb\x8a*\xa3\x91[6*x\xabH\x8e\xf4\r\x1a\xda1\x87\xbb\xc9\xecU\x17\xd2\xb9\xd5\xba\xce\x1f<\xaa\x7f\x164\xf2\xa3b\x06m\xfe|\x1f\xd1]\xc9|\x9e\xd2\xfa\\bG\\\xa1[\xc4p}\xcd\n{\x0e\x16\xb3\x18U\x9ak\x16&\xe1\t_\x8f\xa7\r\x04-\r\x10n\xb9H\xa7\xedMQ\x1d\x12\xf8\xc4g\'\xf5\xd0{\xa6\x11\xed\x11N\x1b.\x8b\xa5\x14\xe41\xc3\xe8\x0fR\x88\x9f\x05\x8d\xfc-\xd2\x82\xb6\x9d\xb9\xff0r*}a!\xd4vEy\x08g\x1a\xd0Z\x18\xac\\0\xca*m\x1dw\xc1\x9d\xce\xcb\xb5\xd2\xe6\x8b\xd0\x19\xbb\xabz\xaeU$\x832b-\xf7S\x12\x84+\xc9\x97!\x1b\xb1J\x19/\xf5\x8d\xdd\xe3\x0b\x82\xd8I\xfb\x9fu\xb3\x7fmj\xe4\xbc\x8b\x04\xe8\xd2`\xec\x1eY\xdd\xb5\x17\xa7?Fpq(\xf9C\xd22[\x97jO\xe4b\xd4\xea\x13~R\x07o\xc9\xf3\xda\xa6\xd9\xa7\xa1RO\x9c\x80\xc9A\xcb\xb6\xa2r\xdb\xed\x11\xdb\xb6\xfc\x0b\xcf\xf5\x0bx\'@\x1a\xbc\x94FK2\xb1\xf6\xc0\x88\xbf\x1a\xcb?\xc7F~\x9cJ\x7f\x8f\x8d\xfc\x8f\xff\xfa\x964i\xf6q\xaf\xc9\xaf\xb7\x1a}\xe3\xf3f\xc5/\xd7U\xbc\xf4\x8f\xe1f]\x81\x1f9\x1cw\xcd\xaf\x9b\x9fG\x97\xff\x1c\xa2c\x97\xa5\x1fb\xffvk\xe7\xfc\xdd\xfe\xf2\xf1=p\r\xfa\xe3S\xcc\x1f\xe0\'o?\xbb97\x0c\x8cb\x83\x19Ub\xdf\xdfL;=p\xa3Q\xed\x97i\xa1\x96&\xa8\xfd\xff\xf1\xee\xe0\x9f_:A\xabc\\Kp\xbaQ\xab\xef.\xedU\x12\x17I\xdb\xad\x0fwZ\x14t+\xdf\xdd\xab\t\xb6n\xd7\xa5\xdf\x99\x1b\xff\x96l$\xdf\xc68\x1f\\\xb0\xb5\x035L\x85\xad\x99\xc8\\\xb9\xae\xd2\xda>rF\xe4my\x1b&\xa6\x10m\xce\t\xba\xc5\xd3\xd2\x87=!\xe5\x8c2\xddf\xf5M7$\xdf\x0f\'i\x15N\xeb\x95\x11\xb4CP\x89\x8ce?\xc8\xa9\x04\xdb\x06G\x01\xd3\x7f\xb7\xee\xed\xd2\xbfl\x83t\x9f.\xbd\xdf\xbd\xd7\x9f\x86\x7f\xfe\x8f\xf1\x9c\x7f\x0c\xff|\\\xbb\xff\x14\xfe\xf9\xe3\xa1\xaf\x7f\x16\xfa\xf9g\xe5\x7f\xce\xe6wF\xb0\xeeCl[\xa7EzI\x8f\x1c\x9a\x05\xc6\xa8\x17\x06\x19\xd5"\xa1\xa3\xdb*\xc4\xfc^\xaf\xbb"\x9dqwuzI\xec\xcf\xdd|}\x06T\n\xe1\xc73W?\x9e\xa1\xf8\xa7\x15\xf1\'\xf7\xe2\x1f\x9e\x90\xaa\xa2C\xfd\xf8\x95?e\x9f\xd2\xac\xca\xf2\xa8\xfb\xe9F}\xde\x9c\x93\xefn\xdc|H\xb5q\xc0\x94ip\xfb>\xd0\x82\xa0\xba\x8f\xd0\xaa\xdf\x0e\xffh\xb6\xcfV\xc5\xc7#\x15s\xaa\xbb\xb4\xd1|\x17\xe6\xb7o\xf37\xde\xbc\xe4\x9fLl\xfd\xdfCH~:\xf9\xe5n\xaf\xbe.n\xe7E\x83x\xffB[\xfat\xe2\xef\x8b\xb6\xf4\xf5\xeb\xfc\xba\xcc\xe1WY\xe9\xcb\xd7\xf9uY\xc1\xaf\xb2\xd2\x97\xaf\xf3\xeb2~_\xb5\xa5/_\xe7\xd7e\xf3\xbe\xca\xf1_\xbe\xce\xaf\xcb\xd4}\xd1\x96\xbe~\x9d_\x97\x87\xfb*\xc7\x7f\xf9:\xbf.\xcb\xf6U\x8e\xff\xf2u~]\x0e\xed\x8b\xb6\xf4\xf5\xeb\xfc\xba\x0c\xd9WaL^\xbe\xcewC\xfau\x1b\xd2\xa7\x13O_\x95\xf1_\xbe\xce\xaf\xcb]}\xd1\x96\xbe~\x9d_\x97\x99\xfa\xb2"\xea\xd5\xeb\xfc\xba\xbc\xd3Wm\xe9\xcb\xd7\xf9uY\xa5/s\xfcW\xaf\xf3\xebrF_e\xa5/_\xe7\xd7e\x84\xbe\xaa\xd4\x7f\xf9:\xbf.\xdf\xf3E[\xfa\xfau~]6\xe7\xab\xba\xa7\x97\xaf\xf3\xcd\xd5|\xfa:\xbf.5\xf3U\x8e\xff\xf2u~]\xe2\xe5\xcb>!}\xf5:\xbf.\xad\xf2U[\xfa\xf2u~]\xd2\xe4\xcb>\xce{\xf5:\xbf.%\xf2e\xe7\xa5\xaf^\xe7\xd7%<\xbehK_\xbf\xce\xafKg|\xd5y\xe9\xcb\xd7\xf9u\xc9\x8a\xafjH_\xbe\xce\xafKE|Y]\xfa\xeau~]\xa2\xe1\x8b\xb6\xf4\xf5\xeb\xfc\xba4\xc2\x17m\xe9\xeb\xd7\xf9uI\x82\xaf\xda\xd2\x97\xaf\xf3\xebR\x00_\xb5\xa5/_\xe7\xd7%\xf8\xbdjK_\xbe\xce\xafK\xdf{Y\x11\xf5\xeau~]r\xde\xab\xee6y\xf9:\xbf.\xf5\xeeU\xdd\xd3\xcb\xd7\xf9u\x89u/\xbbs\xef\xd5\xeb\xfc\xba\xb4\xb9\x97\xdd\x05\xfd\xeau\xfe9\x8a\x15\xfa\xddE\xfe9\xc5\n\xf9\x87%}iR\xdc\xcb\x05\xfb,R\xdc\xcb\x05\xfb,R\xdcc\x82\xfd\t\xa0\xdag\x91\xe2^o\x8a\xff\x16R\xdc\x7f\xfc\x9d\xb16O\x06q\xfa\xfa\x83\x02\xf3\xdb\xe3\xce\xe0}\xb2s}8v\xbf$\xe7\xb1\xed\x9a_\xac>\xae\x0e\xc9*\x1b\xffS\xfc\x8d\xc3\xf6\xf7\xf5\xff\xd7\xdf\xd1l\xff\xef7\xbf\\\xae\xd6\x11[\xca\x11,\x9f\xd8X\x98\x08\x7f\x99\x98\x99\xb2]r\xb2\x9e\xa91\xf8\xd5)\xe6ac"s!4\xcf(i\x9ba\x06\x9d}C`\xe4\xdb\xaf\x0ff\xff\x1fB\xd7Q\xdf\xef\xe2\x1f\xa1\xebp\x96GIq\x06\x95I\x1c\'\x10\x04\x8e\xe3\x84 q\x98\x84\xe1\x08C\xe1\x1c"P0F\x880\x82\x88\xf0<-\x12\xe79\x0c\xe7EZ\x101\x92\xa2?\xa6\xaa\xfd\x9c\xf4\xc3\xa3\x14I2$\x86c\x14\x83K\xa4\x84#<*I\x14.\xf2\x02#\xa0\xd2<\x1c\x8a\xe0\x11\x0e\xa7\x18B`8\x9a`\x04\x98\x16\x05\x8cA\xc1\xfe\x80+\x91/E\xd7\xfdm\xe8\xd4\xb7\x1f\xcd\xd6a~\xa11\x94\xc2p\x18\xfect\xdd\x97\x07\xff\xfd\x00]\x87p\xa4\x84\xe0\x12\x86\xb1,L\xd3\xc0\x80`\x8a\xe6)\t\x111\x81\xa6x\x91#\x11\x04\xe1i\x81\x90`A\xa4\x18\x14Gh\x01\x17)\x8a\xe4)\x02\xe8\x1d\x9b\x07\xef|\x8f\xae{\x02\xbf\xec\xefF\xfeo@\xd7\xcd#P\x81Rh\x82\xe5y\xa0\x1f\x89\'\x11\n\xa8Pd1\x96\xe0\t\x98\x9c\x07yIb"\xa9\xfb\x89\xfd4v5\xf7hf\xb1;\x05D\xe0\xb5m\x1e\xf7\x81n\xa9\x85\xb8\xbbN\x9dvAS\xa3\xdb\x1bc7\xad6G\xa5@\xae\x95\x8f\xe4Kt\xadjd\x94\xf6\x05\xe3Z\x91\xdc-YSa\xa3\x93\xbb\x91\n\x1bs\x15\xe1A\xf8\xd3\xd3\x80h@L\nD|\xe6\x07x;\xde\xde\xba\xfb,TCR\x00\xb1\xf3\xba\xc2.\x85\xec7,\xb1\xc9\xb9\xbc\xb8\xc4\xa2\x9b5f}\xf0\xb5\xccl\x1bfg\x9c\xd3\x89\xd3\xaf\xca6\x86\xb5\xfc\x88g\xd8\xb2\x9c?~\x90WW\xb8\xd9k\xc4\x19\x14?\x18\x8c\x7f\xad9\xd2O\x03\xa2\x01\x9f\xa0@\xcaGi\xfc.\xb2\x18\xcb\x05\xd2]\x17S\xc0\x97\xd2\t\x0b\x1d2&\xd71}\x95V\xa2i\x8eZ\xba\x11\xcb19\t\xb9\xe1\x0e\xe9p\xc9r;\xa9,\xce6I\xff\xdc)\x17N\xa6\x0e{\xd0\xfc\xf96NbD\xceT\xbe\xb44\xd5\xe8\xf8`%\xfc4 \x1a0\x96\x994\x86\xa2\xf7\xe0\xcc\x02\xf6l>\xb8\xa1\xb4\x86\\Q/\ni\xc9$5\x04\xe3=\xc4\x1fV\xdcq\xd9\x8an\xbbYn\xd1\x82]\xb8\x19~\xe8m\x1f\x8f\xf8|c\t\xd2\r\t\xbc\xe3\xb2\xbf\xadv\xfb\xb1E\x02\xff\x1aX\xbb\xedxB\x0e\xc7\x07k\xb8\xa7\x01\xd1f\xd7\'AP\xc4\xee\xa9K(\xb5\x9f\x12\xfd|.[|+F\x16\xb9\xb7\xc5\x95e\xef\xb3\xa3\xb7\xdfoX\x1c\x8b\x04P+\xf6\x93\x95#\x8c\xe5\x92\x13l\x04&\xb4Nz\x9b\xdc\x9d\xd6\xf4">\xdb\ts\xf6\xb4\xbcG\xf6\xc7.\x1dx\x8f:\x92\x8f\xe2\x88\x9e\x06D\x9b\x8dv>\xa1\'\xe8\xbb.U\xdf\x98\x10\x89T]\x0b\xa9\xa7\x98!\xd5d/\xa8\xf5\xc8\x91\xbbhs\xe9\xbb\xb3\x13l\xd0\xa4\xa4\xa4\xa3\xbd@/T\x95\xae\xb1\x05\x0c\xba\xf4\xae\xca\x96\x96\xc2\xe6r\xb9\x86\xf6k\x8eU`\xbd\xe3\x18M\x8b9\xaeV\x93\x07\xb3\xfe\xd3\x80hs\xd6G@\xa1K\x10w5\x1c\xe4\x84\xa3{\xe8N\xf1\xc6\xac-\xbe\xf1\xf7\xab\x9b"rb`bli\xf4\xf0\xb1\xa1\xf6\x8e6,\x90\xb1>\x1c\xa9\x05jQ\xc8\xd5^\x97H!\xd0<]\xe6\xf0\xc5\x84t\xe8\xea\xa4kr\x80az\xe0\x9d\xa5\xbd\xeb\x1e\xac\xc8\x9f\x06D\x03\xda\xc4q\x1aE\xd1\xfbQ\xf6\xa4\n\x07\x9b\x84H\x0e\xeem\xd12\x05\x16\xac\xa7\t\x8a\x1c\xcf\xafhT\xe1\xa2`\xf0\x8eE/\xeb\xbd\xd4\x05YY\x11\xd2\x84\xda\xcbRoEB\x0eH\xc9B\x90\x1bL\xb6D\x19.\xa6>\x87\xaf\x0b\xbe\x11\xa9h\xf7\xe0\xf1\xd1\xd3\x80hs\xbe\x02%9\xc3\x90\xf7\xbey,\x97\x06\xc8\xf2\x15\t\xc5\x89\xa8h\xb9\x9e\xa9\xd0`\x9f\xac\x1b\xd4{2q\xdb\x1f\xd9nu\xa4\xb0\x15Z\xef[\xb8T\xca$\x08\x18\xb8u\x87\x9d\xee\x07\xd3\xce\xca!o`\x02\xa1c\r\x1b\x16\x16\x1bC\xddk\xe4\xa38\xd6g\x11\xd1\x80\xd1R0I\xcel\xa4\xdf\x8b\xb9K\xd3\xc8^/\x17u\xdc\xde\xd6\xeb\xb0<,\xf8p\x8c\x97\x0e\xb1a\x93B\xf2o\xdbm\xbd\x18`\xb4\x95\x98rk\x08S\x8f\xc6\xd4\xb4\xd5\xd6\xf5\x81;\x9d\xb7f*\xf6\xba\xcf\xda\xc7\x89\xd2\xd4\xa8\x00e\xbcS\xc9>\xf6\xe0\t\xd2\xd3\x88hs\xa4\x85A\x05\x8f\xdec\x91\xa0\xbd\xd1[\xd4f{"\'I?\x97:[\x97\x8e\x19\x08\xe2v\x9a,H\xd0\xec3\x14\x07\\\xdaf\xf6&>\xf2\x85\t\xdb\xbe\x94M4q.\x16f\x8d\x9e\xce\xe3.v\xb6\xae\xbb\x93v\x97\xcd\x01\xe7\x1a\xd3\x15V\xccg\x11\xd1>\xbae\x9a\xc2\xc9{\xbe]\xbb\x17\x17\xb4\xc7\xfa,\xdc\x86+F&\x15\xc3\x10 \x9a\x1c=x\xbf\x03\x99\xf4\xbcD\n\xbdq\x85\xf5\x16)G/\xe1\xf0\x8d"\x9dhQJW\xa4\xb4\xda\xae"\xeb\xa0-\xc65\xb8b\'m/\x16\x8d\xb8\xb1\x11\xf9\x0fj\xf3iD\xb4\xd97Q\x0cA\x7f\xa0M\xbc\xa4\xa1\xa9\xdc\xc8i\xc6U\x870\x8d\x979\xaa\xabK\xb9\xe7F\x97\xa7\xfcx\x7fu\xf4\xe3\xa99bN6\xdd6\x89\x7f\x94\xb34\'\x16\xc9\x11C\x03"j\x0e\xbe\xae\x1f\x19\xc2S\x85+\xae\xfb\x96\x98\xd6Z\xa2\x07?K(\x7fq"\xda\\d\xc1\x1fy\xef\xeeT\xb5 \x17f\x92\xd7\xdb\xfe\xca\'\xb7c\xda\\\x1ar\xbf\xb8\\\xda\xd5\x9e\xc2<`\x00-\xe5\x8d\xfe\xea\xd4\xa9\xe7\xf5I\xa3\xc5\xb6"\x8f\xc7\xf8\xaa\x0bc\x18\xb1\x14\xaf\x88\xae\x88_N$\xad\xa9\xf2u\x93\xd0\xbb\xcckS\xf9Acy\x1a\x11mv}f\xa6\xaf\xdc\x17Y\xbd)1\xceJ\xe2\xd4\xe5\xa5;vE\x16\xecc\x87\xae\t\xd4l\xce\xab\xd3\x81do\x95\xe5]\xb8\xc1\xb7{\xff&p\x15\xbb\x8b\xaf7t/\xed\xbd\x86\xbb^\x1a\xda\xd0\xf5\x04\xf4>\x89\x1f\x87\xd6&N\xc3}\xbf\xe3\xad\x07\x03\xf9\xd3\x88h\xc0\xf5I\x18&`\xfc\x9e\xa1e\x8fz\x7f\xb9\xb9 \xbc\x05\x08\xd1Y\xb2\xcc\x1a$\x85.\xe8\x90;$\xc2~\x1bEA\xdbu\x8c\xc78\xc6\x86o9^e\x0e\xf5\xb5c\x07[a\xec\xbd\x7f\xe9o\xcd\xa9\x17\x07\xc4\xcb\x1dk\x7f+\x0f\xf6\xe2\xa4u\xee\x83\xd5\xc7\xd3\x88h@\x9b(\x06"\x1cu\x8f\xf1s1s}\\\xa8\x83\xb3\xc3\xaeqr\xcd\x9c\xc8\\\x1cbVc\x8bj#O\xfb\x06\x94\xcd\'\x88\xbaRX<\xf4$\xa9\xe8\x06\xbc\xaa\x1dv\x1d\xf9\xebD\xad\x8c\x02\'W\x87\xf8\xe6OR\x14c\\c\x17\x97c\x94k\x0f6@O#\xa2\xcd\x9f\xea\xc0$\n2\xf3\x9d\x98iZ\xe2.\xcad\x12"DI\xb1<\xa9G\x0b\xdbM\xc9\xd6DA\x83\xbf8\xc2\xd1!8-3\xf74\xe2\x0b\xb6\xeddV\xd1.\xb1p>i\x01\xae\xaf\x8e#\xbc\xce7\xcax\xa4\xb6\xc9u\xab\x1e\n\x9d\xba4UG>X2?\x8d\x886\x1b-\xce\x104}\x7f\xec\t\xec1\xa5\xe5\xa5\xc82k\xfcpHKxO\xc9e\x08\x87c\xe9\\\xf2\xfe\x16fN\x18%\x8a\x80\x9bK\x91\xee\xb6\x1c9\xd4\xbb\xc8\xa8,\xad\xf4\x97\x17\xa3\xd7q?k\x9aX\xd2\xeb\x8b\n\x9f\no\xd7w[\xef\xf4\xe8\xd9\xc7\xb3\x88h\xb3\xd1\xc2\xc8\xfc\xfd\xbb\x92\xb97\xf1MK \xbe\x1fE\xa3\xee\x17\x0b\xa8\xa2l!7/\x92\x8b,\xf11\xd4b\xd3\r\xce\xcb\x8d\xbf\xde\x81nZ\xab\x03\xb28\xc4+\x98k\xfb\xd3\xea\x14\xb4\x868,i):6\x87\x9d\xb2\xdc2\xae\x90zt==\xe8\x9bO#\xa2\xcd}\x1e\x85\x83\xaa\x1b\xbf;\xdd-K\x97:^#\x9fQ]\xb4\xd4\xc9\xe9l\xb9,\x16\x10\xeaHo\xe8\x03Z\n\x8d\xbf\xf2\xa1\xf5\xc1_8\xf9\xad\xde\x07\xa2\x1b\x1d\xb3\x9d(\xc4\xda\x90\xedx9\xcc\xc8|\xe7L\xed\x9ea\x1cj\xac\x96\x8aM\x9bD\xf4\xe0!\xf6\xd3\x88hs\xf5\x01\x83\x0e\x08\xec\xd5]\x03\x84\xb5\xde\xee`^\x03\x12\x96\x13\xb0\xc8E\x12T\xd9\xaaK\xeb\x8bw\x10\x8bc}\xe3Wy\xb8m\x94z<\x8b\r\x9fa\x0b\xef0\x8a\xc3Q\xe2\xe3\xd8\xa9\xce\xe8\x89^(\x02uN\xe8\x80YlVbrc\xc6-\xfc\xc5>\xa6{\x16\x11mn#I\x18\xe8\x83\xb8;\x14\x88\xb3\xe6\x98\xe5\xf4N\x8a\xe5\xe3E\x82\x94\xba\xb4\n:\x98\xc66\xa9Si\xdb\xb9\xd5\xad!\x16&\xb4\xf1\xae\xc9N\x8b\x86\xd8\x8c\x8a\xd1hv\xc4\x98\xc9n\xb2\xc1Z\xb7\xccZ9\xc1\x16\x87\x9bz\xaa\xb4kx\xb1/gn\xf8\xd7\x88h\x1f\xce\xfc=\x11\xed\xd7[\'\xdf`\x98\x9f\xdc\xa1\xfe\xbf\x07\x0c\xf3\x17Bn\xbc)&\xef-\xfd\xfa[\xfa\x06\xc3\xbc\xc10o0\xcc\x1b\x0c\xf3\x06\xc3\xbc\xc10o0\xcc\x17^\xe7\x1b\x0c\xf3\x06\xc3\xbc\xc10o0\xccW\x06\xae\xbc\xc10o0\xcc\x1b\x0c\xf3\x06\xc3|\xd9u\xbe\xc10o0\xcc\x1b\x0c\xf3\x06\xc3|e\xe0\xca\x1b\x0c\xf3\x06\xc3\xbc\xc10o0\xcc\x97]\xe7\x1b\x0c\xf3\x06\xc3\xbc\xc10o0\xccW\x06\xae\xbc\xc10o0\xcc\x1b\x0c\xf3\x06\xc3|\xd9u\xbe\xc10o0\xcc\x1b\x0c\xf3\x06\xc3|e\xe0\xca\x1b\x0c\xf3\x06\xc3\xbc\xc10o0\xcc\x97]\xe7\x1b\x0c\xf3\x06\xc3\xbc\xc10o0\xccW\x06\xae\xbc\xc10o0\xccC\xd0\n\xec!h\x05\xfa\x0fK\xfa\xd2`\x98\x97\x0b\xf6Y`\x98\x97\x0b\xf6Y`\x98\xc7\x04\xfb\x13\xfc\x94\xcf\x02\xc3\xbc\xde\x14\xff-`\x98_\xff\xe2\xff\x14\x87\x85\xfe\xefM\xffc\x0e\x0b"R(\xcfP\x14\xc5\xb0\x14\x8b\xe1\x94\xc0J\x1cJ\x928\x82\x080\xc5R\xe0K\t\x9f\xc1\x03\x14M\x100M\x12\x08,\n8OS"\xc6\x12\x1c\xff1\xa1\xff\x0f\x00\x03$\xcb`8As\x04\x85\xd0\xb4\xc8!<\xccI,\x8a\n,\x87\x88\xb8\x04^\x9cD\x10\xe0z\x02\xcd\x13\xe2\xccb\x81Y\x0c\xa11\x8eD)\x1e&\xd9\x97rX\xfe6}\xfb\xdb\x0f\x9e\xe8\'\xc8_\x80~0\x94\xc0(\xe2\x8f8,_\x9fb\xf3\x03\x0e\x0b\xcf\xb14\x82\xf3\xf3|\x9dY\xe3,\x86r\x12\xc9\xe0\x18L\xf3\x0cL!\xacD\xa3\x94$q\x92\xc4\xb2\x0c\xce\xcc\xac\x07\x01\x178\x8a\x94P\x94\xc3\x80X\xdf\xfe\r\x1c\x96\'\xd0R\xfe\xb2\x1c\x16\x8eC\x11\x0c\xec\x1a\xd0\x8cH\xe1$,\x01e\xe0\x12#\xd1,\x8f\x03\xf5\x02\xad\x89@\xdd,\x01T&\xb0`a\x1c;cY$\x8c\x00\xaa\x03/\xe1\xdb\x8f9,O\xd0\xd3\xbf\x89\xc3\xf2\xafN\xcf\x7f\x1e\x87e\xb6\xe9?\xe4\xb0|}G\xffT\x0e\x0b\x01\xff\x1cP\x12\'\xa2\x17\xb7\x96\xd5\xa1\xcb\xfe\xd2R\xf8b\xd5\x9cv\xd59\xd6O+\xb7\xb5b8\xbd\xac\x92#!W\xe8%\xdaN\x01\x1c\x027S;\xa9\xc9\xb9\xa4k\xaf\xc5\x98]\x86\x12\'\x02f\xe3\x84"\x96\x06m\x81e\xae\xf4\xe0h\xfbgqX\x08\xea\x17\x18\x05\xb9\x99\x84\xefg\x12*\x95\x00\xf3\xf9!Go\xaa\xe3&\\\xa0\x85\xd3mc\xe9\x1e\x151\xf8\x05\x91\xa4\xb5\x12lN\xfb$<\x0e\x1b\xb6\x97s\xb9\xd8\x1c7\x04$\xaaM\xcc\x8f\x03;\xc1\x8cE,\xd3\xb6\xf5\xa4r\x11LI\x1c.\xc8\x05\xf6\xe0\xe8\xc5gqXf1\x19\x84!\xee&\x1f\xddd\x05\xedw\xc2\x8dl\x83\xb1?f\xd3e\x89\xcb*|3u\xee\xca\xc8\x13EG\x94`\x9f\xa7s\xa0L\xc9\xce\xc6L\xd5\x19\x88k\x89\x9a\xcd\xb5\xcb\x8f(\x01\xe9\xf4\xe1\xba\xe0\xc4dh\x82F\x89;\xa3=\x8f\xfe)\xcf\xd6\x1e"\xda\xcdp\x1a\x900\xc8\x84x\x156\xa9\x9a\x8a\xea\x18p\xcd\xb2\x83\xfc\xa1\xbfA:\xd3\x0e\xf2\x8d\xb49C\xc1\xea\xe5u\xe4H\xafa\nfAT\xc3M\xd9\xad\x1a\xed\xa2l}\xec\xcay\xcd\xfagF\xfb\xd7\xe6\xb0\x80]D\x90y\xca\x1fr\x1f\xc8IW\xa3\xd7[\xdcu\xf6\xbc\'\xa7e\xcf\x96\xa5p\x19V\xe4\xba\xd1\xb9S\x84\xa5\xbd\x10\x0c\x1a\x1bR\'\x8d\x10\xe8\x9b\x8b\xec\x95\x89\xc3\xa6\xf5U\xd9\xc1}t\xde\xf9&]e\x15\x89\x12\x1c\x0c\x9b\xda\x18\xa1\xab\x05\xa5r\x0f\xce\xb5\x7f\x12\x87e\x8ep\xa0\xab%g\xe0\xc9]\xd6\xd7\xe0@0\x1c\xb7P\xb6\xae\xb2Y\xf1\xa5 3\x8b\x15\xa7%q\xa6"\xb1Tl\x8f\xa8\x1aA\xea "]\x88\xbb\xb0\xddumAMZ\xab\x14"U`{\xfadY\x19%\xedc\xb6\x08\x03m\xbb\x94\xd7\xd7\xda|t\xe4\xea\x938,\xbf\x1678L\x90\xf7,\xbd\x11\x1b\xc7m\xbc\\\x1a\xfb`o\xa0\x9b\xddm\'\xa3\x8e\xd0\x94\x95\xb2\xc9\x03.\xed\xa0\xf1\x08\x97\x0c3\x10\x89?\x98\xb2t\x1b\xa4\xc3A\xea\x15\x9e\xba%\x06/\x9d\xb5\x13,\xed\x83\xc8]S\xa6\xb8\xe7c+\x1c\xec\xb2\xfc\xd9\x00\xdd\xbf6\x87e\xf6\t\x98A\t\x1c\xbbg\x13\xf1=\xe2FgH\x11\x8f\x9c\xba\xd8\x9csF9.\xdc\xfa0\x0e\xfb\x96P\xf96\'\xaf\x97}\x17\x92\xdc\xb0K\x17\xa9\xd8\xedN\xc5J\'\xb4E\x12o\xc7\x0b\xb1^\xf9\xa24,\xc7\xc5\xee\x9a\x9cd\x0b\xfa\xff\xdb\xbb\xcffW\xad,a\xc0\xff\xc5_y\xabE\x0e\x1f\xc9I\x02\t\x91\xa7\xa6\xa6\xc8A\x04!\x11\x04\xf3\xe7_t\xdd\xed\xebn\xd9\xdd\xad\x19\xb9}\xdcC\x95\xef\xbd\xaeSG\xd2\xde\xb0\xc3Z\x08\xd63_\xf4#l{oF\xc2\x9frX\xfe\x9c\xbb\xad\xcb\x06\xfcR\x9f\x97\xbdEW`4|\x90w\xf3|2\xa7\x0b\xa6M(\x13$\xc7I\xb4\xbc\xc4\x93F\x0fI"g\x8f\xdf5a\xefqLq\x95\xd4\xe4\xec\xf8B+\x9f\x86>cp\x06?_r\xbb\xd2\x9b{\xac\xf5\xa0\xb6\xb7\x93\xc0x3w\xfb\x94\xc3\xf2m\x9fx^\xcbC\xa1\x97\x145\xb0\x83\x9e.]\x01\xf1\\\xaa\x1e\x07YK\x9cA+\xe1]\xcc\xb4\xde\x91\xc5o7\xb2\xe0\xd8[lY\xe2r\xac\x8e\xca\xe2\xd3\xe6}\x94c\xf3J+\xf5\xe3X2v\xb167\x84\xf7\xe5\x9a\x02\x87\xe6~w\x12\xee\xea\x9bT\xe8\xa7\x1c\x96o\x83\x96 \xd6\x99\x8f\xbf\x047v\x00a#r\xe2-\x15\xf1\x06\xd9?\xca!\xa6\xee\x89]N\xec13\x90P!\x88\xab\xf2\xa2\x02\x1c\x02\xd2\xc7k\xf3\xd0\xdb\x13\x7f"TS\xd9On\xb7\x06%\xbdK\x00B\x11\xad\xb1\x08\xd0J7`D\xc4B}\x17\x99\xfb\x94\xc3\xf2\xedlB$D\xc2\xe4\xeb\n\x07\xe9yKd$1s]\xa5exh\xd2a\xfa\x90\xf6\xc12r\x1ao\xfb\xe2\x03\xd4\xe5:\xe8\xaa\xbag\xf6s\r\xba\xa0\xe5&\x8a\x82\x98S\x8a\xf2"6\x96\xe8> Ln\xa4\xa8\xe2\xb2\x00@\xb2k\x0c\xe5\xcd\x88\xfcS\x0e\xcb\xb7\xfc\n\x870\x82|\xdd\x96g\xb4mG\xa7\xf2\x05\x9d\xf6I\x9e\xb7\xe1\xab`iB\xd4\x9e\xab\xf2Zw\xfbt\xf4\x1a\x9f<\x96\x80Nf\x9aq\xcdO&S\x06nz?\xd9L6p\xe1|H\x13%\xbeec\x89\xb9\x85\x80\xab\xa0x\x1f]\xf0\xcdm\xf9S\x0e\xcb\xb3\x9bk\x82\xb2\x9e\xd1W\xe2\x16\x15\xee\xfc\xc3\x0b\xee.\xa5\xddD\x19e\x19\xa6D\x16i\xc7\x1b>\xf3\x80\x90\x12e\x0cTH\x93\xd6l\xc5\xcbE\xdd\xc7P\xf5\xbc\x9f\x1e\x9c\x98\x19\x08\xe1\x87\x089\xb9\xb3\xe6\xb6;\x8e\x0f\xd0\xc5\x96\xba`\xc2\xc0\x07\xf9\xae\x02\xf7!\x87e\x8d>\xc8\xf5\xb5\xc4\xfa\xe7%T\x953!;CU\xcd\nTO\\\'\xc2\xc4[\xb2\x1ag\xbdp8\xf8rVn\xce\xa3\xf2\x0f\x9a)\xf7\xa7\xf5\x83\xaa=q\x88D$\x98\x05\x00Qa0\x10.\xad\xe5\r\xa7^\x8f$\ti\xcc|\xda\x99F\x93_\xde\x9d\x9b\x1frX\xbe\x9dM\xe2y\t\xff\xd5\x9cpt\xd7L\x9a\x83\x1a\x15\xe7\xe4(4\x02\x1c\xe3\x81\x1e\x8b;\x02\xefM\x97>\x0e\xa3#R\x94\xd8\xac\x0b\xcd\x9dF5\x17\xba\xc7\xdd\xed\n\xeclL\xbcWM!\xc3\x0bQ\x188\xdb\xb8p"\x0c\x8ctU\xcf;\x8c|\xd3\x9c\xf8\x94\xc3\xf2m\tB\xd05W&^\x06-\x86\xa4\xc7\xe3]\x96\x10\xd8\x86l\xf5\xae#`wB\xd8\xe1\xe8 Ag-\x83\xa1\xbb\xf5\xae\xc1\x1fP\xe0\xa0]\xc2Ma\xa0W\'\xa9\xbb\x90\xf6U9\xd2*\x86\xdd\x03\xb2O\xf7\xb4\xc0\x13V\xdc\xa5\x80)[\xb4\xce\xbc\x99_}\xc8ay\x86\xcc\xeb\xc0_SO\xe2%<\x18\xf9\xc5\x8e\xceIW\xed\xce\x17t\xd8\xd7rX\xeb\x05\xe2\xe2!\xaaA\x89\xb8\x04\xe7\xf3EK-aq\x00\xaa\xaf\x08.\xf1\x95\x93\xa3\xea\xecn\xf0\r\xc4J\xd3\x12F\xe0\xd9\xe5\x88L\xcb\x8f\x00\xd9z\xca.l\xf5_\xabk\xff\xc7vX\x9eGq\r\xe1\xe1u\x1d\x7f\x19,\x12\xed$Ho\x82w\xbc\x88\x81\x07\x99\x97~\xad\xdb\xdc.\x9f\x8e\xf6\x03i\x16\x00\xefh2\rb-\x9b\xcb\x8b\x9c?\x06M\xca\x12\xcd?\xdd\xc3)>\x0f\xce2\x84\xad\x0fZ\t2;Yh\xa3x\x83\xed\xa1\xc8zs\xb0|\xcaayvs=>\xc4S+\x7f\x89%]\x9a\xa7/\xa2\xc3\x85\xee\xe0i@\x0b\x1f\x12n\xdf\x9a\x19p\xcb\x10O\xd1Q\xd3\xec\xe7 -\x90\x1d\xb6\xd3\'\x97T@\x0b\x9d\xcd\x92J\xfa\xd1\x04<\xf0:\xa7\xb3\xcd\x15\t\xb2\xceM\xbe\x13\x1cUT:\xf4\xf4;9,?^ZYg\x0f\xf1z\xf18\xbeR\xe8qw>^2\x834\x00\xbf$C\xf1\xa1\x95\xb5\x90\x98;\xe5\xb6\xcc\xfe\xd2\x8d\xc9\x11\xcd\x8b\xea\x8a-j\xe9I\xf4\xa19\xf2\xfd\x03\x16o\xc0c::\t\xbc\xc6@b\x87\xae\x1b\xcb\x12\xd4\x16r\x801\xfe\xf6f\x1a\xf9)\x87\xe5[\xf4A`\xeb x5\xfd\x82+shJ\xd1\x90\xcd\xf0v\xe7\x8c()\x83\xbelY\x13\xd7\n\xfcf\xca\xfb\xf9|\x8f\xc9<\xb2\x05W_\xba%S\xca\xd4Vi\x1aIH$\x0e\xb1q\x9enz6\xec\x89\xbe\xe4L\xf1h\x90\x9a\xe85\x9e\xfc\xe6\xb6\xfc)\x87\xe5\xdb\xd9\xc4\xd6\xd7\xe1\xf0K7;Z\x8a\xac\xa2|\x84mq\xb2\xc2:\xe9\xe03b\xa3\xd4\xf5\xa4\x1asj\x05K:\x1d\xef\x92y\xa9\x1fM\xed"IH1A\x0f\x9a\x1a~\x91\xb1\xbb|\xcd\x0f\x05xutK\x08\xeb\xd49\xb8\xad \x81\xc8\xee\xb2\xbc\x0b\x94|\xc8a\xf9v\xed\x03\\3\x03\xecu\tJ\xb5;cf\x9a\x8dG\xcbQZ\xc0=6\x18\xf0\x94\x17\x94s@Z\xc3\x86\nH\r]\x7f$c\xf1\xa8\xb9\x88p\x0c\x05\xec\x8c\n1xB\xafu\xe3YC\xa6\x11\x1c\xa6\xb8\xe6\xe5\xf9\xe5.\x0e \t\xff(\xf6o&@\x9frX\xbe\rZ\x18\x86\xc0_\x10\xaf\x8a\x8b\x0eB=G\xba\xbc\x8d\xdf\xcf\xd7Vk)\x91K\x8b\xc1m\x02\xa5\x9dY\x91\t$\x0f\xa9\x9d\xb8/s\xf6D\x87\xfb\x92*\xf7\xe8\x80\xf1\xa8\x1b\x1f\\U\xbc\xb7k\xda~\x19\x85y\xeaO\xa7\xc6\x19\xbc\xe3\xc0Lof\x06\x9frX\xbe]\xb0#0\x14\xa2^\x16\xda\x9e\xd3\xfc}\xce:\xbad\xea\x07/\x03`|\x1d\x89<\xef.\xed\x91\xb4\xd5\xd3\xbai\xdc=\xafW\x82}Wh1\x81\x1c\xd5;\xb9@{\x18\x14\x8b\x87\x15V\xc7>\xf7\r\xac\x19\xcfl\x107I>\x8cs\xdd\xdd\xfd7\x89\xa4O1,?^\x9b\xa0\xd0uz\xbeD\xccT.\x08\xe3\x89\xa1\x14\xbd\x18\x15\x93d*\xb3wm\n\x1b\xaeBE\x01\x01G\xdc\xb1\xd2\x87\x1d\x9d90g0v\x90Y\xf3\n\xae\xb5\xdat\\\xac\x9aE\xb2\xb1Qs"VF\x9b>a\xb0\xdb\xd6a^\xfb\xca\xd7\xba\x90\xf5)\x86\xe5\xdb\xd7\xd6\xf8\x1a\xc1P\xaf:Z\x1a\xd7\xb4W\x92\xfc\x80\x02\xa9i\x8c28\x19#\x9e\xc9\xb7\x86\xdd\x0f\xd6\x11\x87Tw\\\x83e\x88\xe1\\1\x8b\xf3;\x81\x14\xe2\xc0t\xde`\x81\x9e\xcf)Y\x8a_\x11\'\xa6\x86\x03]\xe0N\x81\xf1\x9dmv\x87\x1f\xd7\xf1\x7f\xcc\xb0|\xfbzqcX\xfe\xe9\xfb\xc1\xff}\x18\x96?\x90\xc6\xb0\x01\x17\x1b\xc3\xb21,\xff\x17G\xe9\xc6\xb0l\x0c\xcb\xc6\xb0l\x0c\xcb\xc6\xb0l\x0c\xcb\xc6\xb0|\xddvn\x0c\xcb\xc6\xb0l\x0c\xcb\xc6\xb0|e\xdedcX6\x86ecX6\x86\xe5\xcb\xb6scX6\x86ecX6\x86\xe5+\xf3&\x1b\xc3\xb21,\x1b\xc3\xb21,_\xb6\x9d\x1b\xc3\xb21,\x1b\xc3\xb21,_\x997\xd9\x18\x96\x8da\xd9\x18\x96\x8da\xf9\xb2\xed\xdc\x18\x96\x8da\xd9\x18\x96\x8da\xf9\xca\xbc\xc9\xc6\xb0l\x0c\xcb\xc6\xb0l\x0c\xcb\x97m\xe7\xc6\xb0l\x0c\xcb\xc6\xb0l\x0c\xcbW\xe6M\xfe\xbd\x19\x16\xf4-"\x02\xf9\xab&}i\x86\xe57\xef\xd8\xef\xc5\xb0\xfc\xe6\x1d\xfb\xbd\x18\x96\xf7:\xf6?\xd0J~/\x86\xe5\xb7\x1f\x8a\x1b\xc3\xf2\xdb0,\xd4\xf7\x83\xfe\xf7\x19\x16\x86g\x08\x90c0\x86\xe6i\x96\xe6h\x1e\xe2\x11\x08\x84!\x1eC\x05\x1ac8\x12\x15H\x94f0\x02#\xf8\xf5h@\x0cM\x82$\x0es8.\xf0\x1c\xcf>\x1f1\xffu_\x80@ \x1aEy\x84\xe3)\x01\x01i\x8eA\x10\x81z\x96n\xe3H\x9ea\x89u\x01\xc59\x84"(a\xfd \x02\xc7\x04\x94\xa0Q\x9c\xc7\x08B\x80\xa9\xb5Q\xeco\xca\xb0\xfc\xa5\x9c\xd8\x0f\x7f\xfdD?\xfc\xad\xb0\x00\xf2\'\x04$H\x90\x001\xf2\xef1,_\x1f\xb1\xf9\x05\x86\x85`H\x84Da\x9e\xa7P\x94@)\x88\xa1 \x8e\xa2y\x82Z\xff\xc3`\x1a\xa2q\x1c\x11h\x0ef@\x08Bq\x08&@\x82\x00\xd7N\xa2\x1cM\xae\xbf\xf6\xac\xf7\xb21,\xbf!\xc3"\xf00\x041\x08L\xf2\x08L\xa0\x04\xcbp \x07\x91\x1c+@\x98@\xa1\xeb\xe9\xc2p\n\x83\x04\x0e\xa6\t\x81\x83h\x02\x83q\x8c!\x88g-v\x94"p\xe1\x87?8\xc3\xf2O\xdb \x9fcX\x9e\xcb\xd8\xdfeX\xbe\xfeD\xff}\x19\x16\xe2\xd7\x0b\xf8\xe3\xe1!\x82\xf9q\x1c\xe2\xe1\xcaX\xe7\xbd\x17<\xa4\xa3\x00\xd3\xdd\xc0\xd7\x0f\xdb\xadOB!-\xd5L\xbag\xa2\xe1e\x1f\xd4\xca4\x1d\x13u\xa1\xcb\xf3\xd8\xddL\xb1\x82v-\xb4\x88BP\xf4!\xad\xfa\x13i\x996\xf3^\xb9\x9c\x0f0,?m\x0b8\xb8.\x9a$\xfcR\xf2})\xe9.\xb6\xf8\xf35\xbc\x18d\n\x12\x11\x0f\xc9\x068\xe2\xf1]\'\x896\x99EzV\xce\x0f\xc78[59\xaf\t\xe8%I\x9a\xf6"\tx\x0b\x1c\x08\xda&<\x83\xbf\x97M\xd5\xca\xa9a\xdd\xef\xc6\x95Q/\xfd\x9bU\x81>\xc0\xb0\xfc\xa5\x9b8HA\xf0\xcb\xa9d\xefK\x8a\x04\xb7\xbb];l\xbc\x8b,1\x85\x1e\xd0\xe8\xec\xdc\x83mPv\x08\\\xabz\x9c\x92\xf5\xefC\x17y\xe5`\xde\x80aZ \xee\xdeFv$Cg\x98uxYH\xf31:8\x19OL\x1c\xd3\x14\xc8\x9b\xa7\xf2\x03\x0c\xcb_\xfa\x88B\x14F\xae\x8b\xf0k\xed5\xcc\x90\xf5N;e\xd0i\xd7\x1417s\xfc\xdd3\x94\x0b\x15\xe5\xf0|1\xcd\x93n\x95\x8b}?\x13%\xe4!\xe5r\x9d\x0e\x175\xdf\xd7F\xc0\xd8\xd9\xf1\xa8\x91>\xd4\xdd\xe3n\x8c\xc8i \x07\xab\x9f\x86\x16M\xde,\xa2\xf9\x01\x86\xe5\xa7\x11\x0b\xe30\xb4n_/\xa5\x89D\x196\x8fy\xdd\x87\t\xb3\xdb#\xdc\x81tS\xcc!\xc0\x18\xc5\x0f\xde\xad\xdf\xb9\x06gc\xaa\xc6\xdc\xd3v\xca\xcejpg3M\xde\x85 \xa6\x85fh\x94\xeeU\x17k\r\xbc\x90}s(\xa9Kg\x9a\xf1]\xc3\x7f\xad\x92\xde\x1f\x96a\xf9\x1e\x0e\x92\x18\xb4\x06\x03/\xf3\x9e\xf5\xa6\xca\x86IGI&\xa8\xb1%\xf5\x8aqM\xd0=T\t\xa9S\x15\xc02O\x90\xd7\xf3\xd8\xf9\xa2a;\xe9\xc12\xc9\x1d\x99G\'\x8es\xc0)\x99n\xbc\xca\xf1\xfbq\x18\x89\x85\xdd\xd3\x12\x17\x91=x<\xbe[\xa8\xef\x03\x0c\xcbO\xddD@\x1c\x81\xd0\x97\xaa\x8b\xbc\x11\xc3\xa9 \xf7;\x92\xafu\xcb>\xf84sH4\x00\xa7;m]\x02@N\xbb\x02\xaeK\x90J0\x1d/\x13\xe1\xdfy\xa2/\x94\xcb\xee\xb6\xa7,\x07\xac\xa1\xf6\xa0\xa9\xf1\xee\xc1@\x18\x83\xf7\xc1E\xd7\r\xff\xd1\xbf9%>\xa0\xb0\xfc,\xb6GP\x14\x7f\xdd\xab\x82\xcb\xb5\xdeefG\x92\xb7\x85(\x15\xbeY\x8e\x90\xd4\xab\xa5\xa2S\xd7G\x14\xc0M\x1a\x92\x17a\x91\x1f|\x81pQ\xeaJX\x8ffNu\xba\xfb\x0f6\x13/\x88(\xa6\x19\xb0\xb7"\xad<\xa7\xc6p\xcf\xfdu\x91|\xb3N\xdf\x07\x14\x96_La~\xde\xcd\x8b\x12\xee\n\xaf\xda{~A\x98TfJ\xb7v\x04\xca\xdd\xe2\xd6\xe2r\xe3h\x8f=\x18\x0b\x01]"%\x92\x8dI\xe6\xcc\xae8\x95\xe3\xd4\xd4){"=v\xed\xeb\xd5\xb3\x10\xa9\x0bq\xfdH\x88\x95d1\x07Q\xa4\xdf+m\xfd\x01\x85\xe5\xa7nB\x18\xb2\xbe\x04}\xa9\xe0]\xe3e\x9bx\xday\x9f\xe3)C\xa1\x0f8\x9b\xa8\x14j\xbb\xbd\n\xa8\xa5\xd0\xc75I2q\xc4\xed!\x9f>\x97,o\xd6fp\xc8\xaf]\x9ba\xe8Y\xde\x8b\xfbe\x12\xf4\x10\x02\x92\xf2\xee\\\xe3\x9d\xb0~\xf4^x\xb3\xb8\xe4\x07\x14\x96\xef\xeb\xf8\xba\xcf\xad\xc9\xf2\xcb\xdc\xa4\x8d$Q%\xc26\xabK\xd8\x15cn\xde\xc8\xc7\xb1\x03MwA\xb1Z\xc2!N\xc22z\xd7\x0c\xa4\xd8\xd9\'2G\xed8N9\xa3\xeee\xb6Tw\x07 H\x1e9\xbe#\x05\xc0\xbe\x9a\xd6\xf3Rq\x89\xcb^\xcb\x19\xe7,;\x9f\x01\x8f\xbc&c\x0e\x01\xbc\x08<\xac\xf1\xa0\x98\xcd\x9b!\xf3\x07\x14\x96\x9f\xe6\x04\xfa\x9c\x15\x10\xf9r6\x1b\x1d\x08\xfab\x8f\x05\xbe#?\xaaT9\x0e\xa5\xdd\xf8{\x9f=\xc1\xd18\xd0\x13\xa0z\x9a\x8e/\xdc\x05e\x83\xa5\x95\x12\x90=\xde\xafx`\xee\xd4u\x9b\x17\xc1#\x03\xe9W\x93\xd7\xc5\xb1\t\xc5\xd3\xb1+\x1bVd~\r`\xfa\xed\x14\x96\x9f\xce&IB$\x84\xbd^(\xcbot>_\xee\x16J\xe7U{D\x1f9\x18\xd4\x8c\x88\xee!\xe2:\'V\x8b\xc5\x07\xb1H,\xbf\xe4YP\x9eo\x88\xf4\xe8\xd6|\xdd\xbc\xc6\xa8Xa\x0b\x90_\xee\xf1\x19\xc2:\xd6\xaf\xc5C\x00Y\x9a\x0eC\xe5\xf9]c\xfb\x7f\xaf\xb0\xfc\xd5\xa0\x05\t\xf8e\xd0\x16\xd5l\xa7yt\xb0\xb1\xfb\xac5Nn\x1b\xf7\x8c[\xb3.\xc2\xf30iO\x99ly\x81H\xdc\xcb%\x0et\x82\xf4\x9c\xfa9eA\x12\xe0\xc8\xa8c\xc3\xda\xfe|\xbd\xab\xd0\xe9\xb2\x06\xf8>0\xbdcnzkBh\xe8Mj\x914G\x9c\xf4\xf4k\tp\x1f@X~Z\xc6\x89\xe7=5\xeb_\x7f{\x14\rfb!.\xe2Y{\x18\x87pR\x17\xd9\xef\xad\xeb\x03\xd9Q$\xed.\xddR\xdc\x0b\xb4\xa7\xa5\xf4\xca\x1f\xc0\xf3\xf1\xdcK\x08t;Ja.zr\xa5kG\x15\xb3\xad\xc0\xde\xef\xa4\xbeqn\xb7\xda\x1c\x9c\xe3D\xfe\x98D\xfec\x84\xe5["\xb1!,\xff\xf4\xdd\xe0\xff>\x08\xcb\x1f\xa8\xd8\xf5V?|CX6\x84eCX6\x84eCX6\x84\xe5\x0b\xb7sCX6\x84\xe5\xffP\x10\xb5!,\x1b\xc2\xb2!,\x1b\xc2\xb2!,\x1b\xc2\xb2!,_\xb8\x9d\x1b\xc2\xb2!,\x1b\xc2\xb2!,_\x197\xf9?\x87\xb0l\x05Z7\xd7fsm6\xd7fsm6\xd7fsm\xbep;7\xd7fsm6\xd7fsm\xbe\xb2\x17\xb3\xb96\x9bk\xb3\xb96\x9bk\xf3e\xdb\xb9!,\x1b\xc2\xb2!,\x1b\xc2\xf2\x95q\x93\ra\xd9\x10\x96\ra\xd9\x10\x96/\xdb\xce\ra\xd9\x10\x96\ra\xd9\x10\x96\x7f\x12\x88\xc0\xde\x02"\xd0\xbfj\xd2\x97FX~\xf3\x8e\xfd^\x08\xcbo\xde\xb1\xdf\x0bay\xafc\xff\x03\xab\xe4\xf7BX~\xfb\xa1\xb8!,\xbf\t\xc2\xf2\xf3\xf5\xf6\xef#,\xc4\xb3(\x1cE\x11(M\xb0\x0c\x83\xb34Dp8\x06#\xac\xc0\xd0\x18\x8f\xc2(K\x80\x14\r\xa2\x18\xcf3$N\x08,\xcfA \xc7\x92\xcf\n\xcf\xeb\x7f\xc8\x0f\x7fO\x17`\x10\x81\xa5\x91gy~\x92g@\x12\xc6A\x1c\x01\x19\xf2\xa9\xba\xd00\xc7Q\x08\xcf\xf0,E\xe3\xe4\xfa\xee\x9c\x80>\xcb\xaa\x08\x0cD"\x14F=\x1f\xbd\xe7\x7fK\x84\x05\xfb\xcbc\xfd?\xfc\xd2\x13\xfd\xd4\x9fp\x94\xa2(\xe8\xc7\xe2w\xbff\xb0|}\xc1\xe6\x17\x0c\x16\x0e"\t\x02\x87\t\x82\xa7\x08\xe4\xa9\xb1\xac\r\x82h\x81C\xd7\xd8\x08\xe4`\x18\x15P\xf0\xf9#\x14\'\t\x88\x07a\x8c\xa7x\x14\xc3\x19\x1c\xe1\x18\x12\xc3\x9fg\xfc\xe7\x06\x0b)\xac\'\x8b}\x16\x84\xc70\x16!\xa1ga\xe1gQ\x05l}G\x88\x85 \x94\xe5i\x8cc\x05\x8a\xe6\x08\x82Ch\x16\xe6\x04\x8e\xc69\x9e\xa7(\x14\xe7\xc8\x1f~6\xc67\x83\xe59\x18\t\x1c_\x0f\x12O\x934D\xadM\xc3\x9eN\x06E\xc00-\xb0\x0c\x063(\x01\xb3\xc8:ki\xf0Y|\x0c^\x0f=\xbe\x1eu\x1e\xa2\xd7\xcf \x08\xec\x0fo\xb0\xfc\xd3\xcc\xc2\x8b\xc1\xf2\x03;x<;\x1cx\xb6hG\x0f\xa6\x96\xd0\xa1\x96}\xa3\x81\x9e\xd3_\xc3:\x1a\xe2R(|\xd7\xc8\x0fgL\x8b\xd6\xcc?z\xd6\x16\x15m\xc7s\x1eW_\x14\xc0\x10\x91\xcf=}R\x07ZV\xd7\xf1\xd9\x87u%\x86b\x85\xca\x7fn}\xfby\x16\xb8\x87\xcc;3\xd3\xa1\xa1\xb3\xf5w\xc0\x18\xa6\xe6`f\xea\xb5\r\x99\x87(\x95\xe7\x1a\xe3\x1aW\xdfeQ\x83\xa2\xfa\xd96e\x91Ea\xf0Y\x06|\xee\xd6Q\xc1,\xebVZ\xad}Xwn4[\xfb\xdb\x87\xae=\x04\xaeq\xf5\xd7\xf7\x8eE2[\xf7\xec!f\xa7\xf5_a\xf0\x9c\xb8\xfa\xf3\xeb\xfb\x10^\xfb#V\xb8\x7ff\xf2\x08\xd1\xd6\xbe\x1d\x06\x997\xaa\xb8\xb6\xef!\xc2T\xeb\xefp\xcf\xad:p\x0e\xd9i}\xdfH\\c\xfcy\xcab\xb8z\xf6g\x96E\xaa\x96%\xa3]_\xcf\xfa\xae\x06\xca\xfc\xdaF\xc4\x18\xc33\x13<\xdf{m\xf3\xf3\xb86k\x9b1Y\xac\xd6\xf7\xd6\xc6\x08\x86\xc6h\xa6[\x03\xa4\x02u\xfd\xdc\xb0>eQc\x0f\xb2\x88\x8d\xbeh\xaf}\xa3\x9e\xff\xff\xe7\xe3\xdc/\xcf\xf7\xf7\xdc\xf5w\xe0\xe7#\xd7V\x16"\xeb\xe7J\xfe5\x82\xab1,\xd0\xb3>\xb5\x86\xc6\xd1\xd3\xe1\xf5\xcf\x99/\xad_\xfa\xf9t`\xda\xf3\xc1\xfc\xc5\xd7Lb\xc6?\xe4\xa2\xff\xd6\xce\x83\xa9\xe01\xc2\xa3\x91\xb8\x86+N\x8cD\xf0\x84i\xae\x02\xf9\xae\xb0\xf8\xa5\x07\xad1X\xbd\xc6c\xf8\x1a\xb2L\x07\xd1\x83B\xc7\xce\xd5R\xf9\xf6\xdagZ\xf5<\x86Q\xc9\xcfk\x8c\xf2\xd0%\xa6\x0c\x1d\xa8\x89\x11\xed\xae\x9b\xf6\x1c;\xf6#\x82=Xs\xd6\xf4\xa1\x8e\xa0\xa4\xd6\xd6\xc0\xf4\x84\xc4\x8e\xb6f\xff\n\xc7\xe6\xe0\xf7\xb1[ks\xe22`8c\x95\xcf\xc9\x90f\xf2\xd8\xbeb \xef\xd2W\x89!\x17l\xc1\x99\x86\x15`\x17\x8b\xc1\xf0$\xa5\xaa\xdd\xbc\x06\x18p\xc9)\xe7\xf8x\x00\xcaT\x9b\t\xf2\xde\xd5\xa0WD\xb1\xd9\x1alfC\xdfB\xb0\x87,\x1f\x96\xef\xc7\xcb\xce\xae:{\xf1Ti\xa6\xca\x1fC\xb4\xc3\xb0\x8eA,\x92\x8cq\x9d?\x95\x87\xd0\xb3f\xfa\xb7u\xde\x9cbG\xd9\xfb\xee\xe5,OmN\xef\\1;\xf5\xa1\x1f\xd7\x19\xe2\xab\x0f\xc3<\xf1\xf1\t\x10\xcc%a\x8b\xa5\xb9\xa3\x023$\t\x88=t\xbf\xe6\xb3\x93Jg\x19\xb3\xfe{\x9a\x94\xf6 ^\xe8uME\x88}\x87\xf0\x87\x14\x9e`5\xf1\xe0\\\xf1\xd9T\xaf\x8a\xf6\x9eQ61:\x1d6\xed/\xa7$\x93\xd6}\t\x1bU\xec\x1c\xb2\xf3\xc9\xde\xdf\xcfG49[\xe4\xb10\x1e\xb7\x87\xd9\x15\x15"\'\xfaa\xc7>Z\xe0\x87\xff\xb1\xc4\xf4,\x97\xf3w%\xa6\xaf\xbf\xdd\xff^\x12\xd3\x8fA\x13\xfa\xeb\xecC\xc6`\x97!\x99\xd7yI\xd8\xaf\x8d\xd8?8\xc4\xb4\x1e\xc5\xf5\xc8\xaf\xb9\x13\xf9\xa2v\x81\xf8\xdd\x9b\xae)\xc4\x9f\x13\xde-F\xc6 \xec\x1c\xe4NR\x92w\xf3\xa3\xe3\xda\x9b)\x94w\x06\x9bv\xf2t\xb9\x8e\xf6^\xe8l\x86I\x14T\xb8\xd9\xb9\xcdS;\xc8\x9dt\xabx\xcc!Q\xeb\xda1\xd5\xa94\xeb\x7f/\x88\xe99X(\x08\x85\xe0\xd7Z\x9d\x8b\x95\xb8p\xc0k\x1a\xc9`\x15>\xec\x1bO.2\xb9\r\x08\xb0p\xaf\xbbQ\x06\xcb\xfa\xd8\xf7~iVn+\xcf\x1c/\xb9\x16l@\'V\x89\x0e\x93\xb7\xb7\x86I\x028}\x94i\xfb\x18h<\xf8`\x96K\xaf2o\x96$\xfd\x94\xc4D\xfdi\xdd\xd30\x04Z\xff~\x91\x98R\x9c+w\'%C\xe1\x99\xc9\xb3>\x859Nr\x98\xae:\x1a\x08\xack\x91\xcf\xe98\xb2\x93J|1\x14B\xa9\xaep\xe0^\x8a\x98\n\x80\x1d:\xf7wU\xf0\x9b\xde\x93O;\xcb\x13<\xb9\x8e`w\xb8P\x18\xfb\xde^\xf51\x89i]\xc5a\x04Ba\x12}\x19\xb4U\x83.\xb8\xd7\xdc\xce\xfb\x1b\x97\xb8\x84\x16\x02\x18!\x91\xedaO]U,S+Pm\xcbK:\xec\xf3\xc9\xa6r\xfe\xa2\xd9\x92P\xb1\xea\xf0@2\xc5\x0f\x0e\xe7=92\xc3\x1e\x80\x97\xec\\\xf8\xec\xc2\xa1\xd3\x12*o\x8az\x1f\x93\x98\x9es\x13\'\xb055~\xad\x14n\xbb\xfec]\x99d\xf1\xa1\x11\x1cW\xf94\xaa4Q\x0f\xc7k"X\x0ef\x8bF\xa6\xd9+\xa7\x8a\xb3bR\xbd>\x90|`<\x85\xa8\x01\x18\xb6B\xbe\xc6\x17\xbe\x01F\xf9\xc6\x1f\x81\xdb\xbcO\xd2\xc4\\\x13I\xef\xdd\x12\xda\x9f\x92\x98\xd6AK\x10\xeb\xae\x07\xbfn\xcb\x12\x95\xee\xd52\xd7\x92\xd3\x1a\x07GV;M\x80\x9c.]\x87\xcf\xeb.kE\xe5\x058s\x19H\xde\xe0\xf2f\x86\xcc\xe1\x04\xa9\xf7\x94\x04E\x92:IB\'\'\xfdy\x12\xf7\x1aC^/\x04\xdb\x9c\xf58\xb0\xeb\xc3\xed\xd7\xb6\xe5?\xb8\xc4\xf4\xdc\xf5a\xfcY\xd7\xf7e\xb0\xd4W\x8d\xec \x1b\x0c\x16\xee\x98\x9f\xec\xa6\x9d\xc4sv\xf0\xa4nY\xa3\xfbk\xf8p\x80S\x9f\x9a\x93\xb7SG\x9d\x85\xad\xc0\xbbg#^\xdb9|+\x8e\x97\x93\xe9\xef\xfc,"U>N\xa5\x0cy\xd4\x92$\x13\xcf<\xe1\xf7\x91\x98\xd69A\xae?\xc5~\x81c\x81\xc4*D\x92*\x8e\x1d\xb9an\'\x81-\xb2\xe3\xc2\xfaB\xba/\x9cG\x9e&\xe8\xe2\xa4\xc4=0\\\x1dS\xe1\x0e\xeeiC\xd8s\x8a\xbf\xa3j\xc9\x05v\xcd\xee\xc8\xd6\x16\x110\x82\x1b\xec;U\xa5\xd0\xce\xaa\xde-\xa1\xfd1\x89\xe9\xb9_=\xf5W\x8ax\tn\nyrEi>\x10\xa4\x12>\xb2\x99\xb1@\'\x12\xc7l\xc7\xd7w\xd3\xe5H\t\x88\x0eQH\x04\xca\x85\xa6\x96\xe1\x08\x14\x1eu#\x8e\xae*\xf3>\xa7\xb9\xb1s\xc5\xdb\xae\xf1\x04\xbee\xbd\x0e\xd1NG\x08?\x8e\xbf\x8a\xeb\xfc\xc1%\xa6\xf5(B(\t\xe1\xd4\xebv\x08Z\x15\x00\x8a\xc4\x82\x91]\x1eE\x08~j\x91\xee\xbc\x0b\xc0vi/\xf6d\x95\r\xbfwAa\xe2e;\x9c\x98\xe9\xa6\xdei\xc7u\xa1\x13x\xb3\x14\x10\x14\xc2\xee@<\x0e\xd0\x99\x1a(\xb3\x1e\xd0\x8a#Rdv2\xf0\x9a\x08\xc7\xe2\x9c\x02\x9a\x0f@J\xf4\x80\x9cb\xc2\x1b\x061\xed\x1c\xad\xa40\xbeT8b\x15V\x96\xeao\xee\xfa\x1f\x93\x98\x9e\x19*\n\xa3\x04\x82\xbdt\xb3\x08$\xef\x91^\x11@x~=Zq{h\xcd\xa5{\x8e\xb0!\x9ePX\x9f\x94=^\xdbI\xb08[DYr\x03\x16\x9e&\xe4\x0e\x9e@\x13cO\xa1\t\xe2h\xed\x8dJ.\x88\x93\x08\x9a\xcd\xa0t&\xcdpo\xd6[\xff\x98\xc4\xf4\xdc\xf5\xd7\x10\x01"_\xf90\xcc\xce\xc6\x06\xa9\xeb\xf6\xa0\xc0\xb9\xaf\xf6\xb6\xf9\xb8\t8\xd0\x8e\xf1m\x90\x01\xfc\x98\xdb\x83\x93_E\x1eI\x14Q(\x98\x85\xf3\x93\x98\x80b%\xe3\xb5c@0\xb1\x00\x1e\xbd\xb2\xd2\x87F0Sl\x0f\x8f\xc9q\xac\xdf\xccR?&1\xad\x83\x96|^H"^\x1d\xdd{O\xaeY0\xed\x10p\xb7\x0e\x17\x8a\xb9\xde\xcb\xfe\xbaXA\x89\x91\xfc\xc4\x93G\xfe\xdc\xa8\xa3\xeb-\x89\r\xd5b\x00\xdc\xeeJ)z\xf8\xf5\xd1;<\x87\xac\x0by\x7f\xe6#\x11\xa0\xe7$\xe1\xd5]\x05\xa4my2\xde\x1c\xb4\x1f\x93\x98\x9egs=\x93\x08\xf2\x9a\xa5\xee\xc7\xd4\x9d\x8f\xf8\xdc\xdeIr\x0f<\x9c\x92\xd2\x1a\x085\x85F\xc5\'\xads\xa4`~h\x90\xedg\xc6\x82wH\x14_\xb1f\xb0\x1f\xb9\xd9\xc2\xf3\xc0\xb1\xe9L\xf3\x071\x0c\xcf\xc6.x\x9c\xb3\xf0L\xe0]\xa5Po\x0e\xda\x8fIL\xcf\xb3\t\x92\x14\x82`/\xd5\xf3\xdd\xa3\xd0\xd4\xa6\xb1\xcb\xf0.\xd2Z\xa5\t\xa8B\x15o\xa7\xcc\x15\x83\xa2\x9btl\xf70\x04\xdc\x07\x0bb\xc2U\n\x1c\x91\x80\x1eRe\'\xc6\x05\n\x0b\x10W\xcd\xdd\x8dM\xcf\xc5A\xd2\x0f\x90q\r\x97\x90\x10\xc2\xf0\xcd\x8bd\x1f\x93\x98\xd6n\xa2\xeb\xb6\x0e\xff\x02\xe5\r\xa4\x86\r{\'M\x073C\x9a=\xe6\x965\xe1~\x99\x02\xbf\xe4#m2)\x0f \xacb\xd7Q\x80\x8a\xf1\x01\x81k!><\xe0\xd3\xc1e\xa2\x82Z\x80\x05\x89\xaf\xf2a\x1c\xef\xb7\xf3\xc9\\4\xd4\xd2M\xec\xb8{3\x8d\xfc\x98\xc4\xf4-\x8dD0b]\x97_\x82,\x9dH\xe1\xb3\x95N%\xeeD\xe9\x85\x80\xe8{_\x0bl\xd7\xe5\x12\x9cP\x1e[\x9b"\xab[\x13F\xa7yT\xdbj\xa1rG\xc9A\xa5\x02\xf4\x03\x91F\xed\xc7\x99\x01}\xf5\xa8f P#\xed!\xd6\x92+`\xbc\xabz~JbZ\xcf&\x06\xafk\xd9/\xa8\x9e\xb8\xb1;+\x1cZY&@\x88A\x04\x1c\xe7l\x01\xc95d\xe0\xa0\x93vmZ\xe5De\xd8\xbd<\xd1i\x1a\x92\xd9Q\xb6\xa0\x0b\xcb\xa9t\'\xec\xcb\xa8\xbe\xdf\xd3\x13\x14\xce\x08p\x1b\x94\xdb}\xc0\x81\x01\xb2%\xad}7\x8d\xfc\x94\xc4\xb4\x9eM\n\x01\xd7\xe5\n|Yi\xf3\x058\x14#WZp\x8b\xe3\xd9nI[\x98\xce\xd6\x900\xf2=\xc3\xe5\x0f\x873\\\x14%v\xe9h\x13\xd5\xd9N\xf4\xf7\xbar\xd6\xd6=\xee\x80!\xb9\x7f\xf7\x89\x8bt\xdb\xebpy\xac(\x02\xbdW\x047y\xcc\xbbK\xd0\xc7$\xa6gf@\xae\xc7\xea\x17<\xcaYP0@\xbc\xf7\xfc2W%\x03\xd6\xbb\xe3\x94)!m+w\x8b\x16X\x87\xa5\x12\xf7l\x08\x19\x16j\xeb\x86z\x1e}@\x89-+2\xd3\xc8x0\xb6\xdc%\xac\xd4\x8e\x86>Tba\xe2S/\xd3#\xe7\xe1_\x0b}\xf8\x98\xc4\xf4\xdc\xaf`\x10Z\x13\xfb\x97\xc1\xe2&tM=\xae\x03sE\x9apD\'\x18\x95Q\xe1DpA\xdc\xf8\xdc\x88\xc1\xd7\xea\xbc\xcf-\xec1\x01\xc1\xfc\xfc\x0e\x90lm\xf9q\xb9\xd1\x0b\xbc\xab\xb2l\xee:\xcb\x8cF \xd1\xca\x19\xc4/l[\xb6\xedP\xbd\x192\x7fLbz^s@qx]\x0c_Q\x9b\xe7s\xb4\x9cL\xea\xa1~[Tg\x1a\xc6\x0c\xba.\x8a?\x9f\xdb\xd2]$m\x9d\x0c\x8a\x8bfT[t*{\xa4f\x7f\x8f\xae\xdb\x9a\xf0 \xec\xc0\xa2\xa0@\xe7}W\x96rb\xe9\x07s\xa0\xebL1\x00Xz\x13B\xf9\x98\xc4\xb4\xaep\x10\x86\xe1k\xdc\xf2\x12}\xb8\xa1]\x86L\x96\xf9;\'\x85\xc3\xba\xf2\x96Kxw\xf6\xbd$+\x1aA\xd9\xb1n\'\x0f.\x91w\xde\x88\xf4\xae7\xf0\xe2\xf1\x86\x00\xc4p\xa5:\x9c\xde\x0b\xd6P\x81L\xdd\xa4%O2\x08\xc1\xe9#\xc3\x85#\xf9\xe6\xb6\xfc1\x89\x89\xfa\xd3\xfa2\x84$\xf0\xd7\xe8#\x17ii\xff\x90\xaenf\xcc\xa72\x91\xb1\xe2@\x1aR(\x06D\x02(\x0cYdrl\x82:\xc8K\xbd\xa3\x91b\xdcZ\x08\xc0\xcb}\xa7\xdf\xe6\x8bvF-\xe7\x88=\xfaNX\xea\xa1m|M\xaa\xef\xa7\xd0\xc2\xdeT=?&1=\x07-\xb6\xe6\xb3\xc4K6k\x11\xb0z"\x0b\xfbz6\x9c\xbd\x97\x1b\x06o$\x1d\xe0{\x85\x0c55\'\x85>\xde\x9d\x90\\ukW0\x98c\x91\xcd\xc1xXg\x91$_\xbc\x92\x82\x89\xc6\xb9\x1a=\xab\xd6\x08V\\\xd9\xf5\x08\xe1\xb1-Fo^\xaa\xff\x18\xc4\xb4\x8eY\x10\xc5\xc15l~\xe9&L\x8c\xf7\xb9\xf0o\x07\x01v\xa3\x93\xe1\x1a\x97\x83B\xdb|\xd6\x18r`\xdf\x02C\'F\xa5\x07\x87\xc3\r\xaa\xc8\xfa\x80*!E[T\xe7\x0e\xf4\x85\xbb\xa3\xfbJ4\x90{\x9f?v\x87\xcb\xee*p}p\x14D7|sj~\x0cbZ\xc7,H\x11\xeb\x9e\xfcj\x14\xc10\xc7%\xb6P\xe9\xf1\xfa\xc6\xd2\x99O#\xd9R\x19\xc3W\xe4{~\xc0\xcb\x01L\xcf\xcb\xc1s\xcd\xf3\xd2\xe8\xc7kf\xabSSBz\xee\x16\xad\xf0H\xc28\xdfA\xda\xd5\xc4\xea#\'\xddC\nBz\x9a\xd9\xed\xde\x0c>>F1}\xdb\x95\xd7\xe8\x1az\xcd\x7f*\xc0\x16\xa1\xf1|k\xa8\xdcR\xab\xa4\xe9\xe3\xc6\xdb-\xbb\xb3\x0fq\xb727\xb2\xfep\x8f\xcd\x85=\xc8j\x904\xd9\xfa\xcb\xd7\xdd\xf9P\xed\x12OA:\x0b\xaa\xdd@\xbc\xf0\x85\x8a\x91\xa8\xbe\xcc;\r\x9e\xc8\xb3\xab~-R\xfcc\x14\xd3:\'p\x88@a\x14|\xf9\xfa\xcax\xb4#Z*\x8f\xe9\x96\x0e\xbd\x81\xe9\xfbR\x9f\t\xc78\xe8\xeeC\x1f\x97\xe1\x9au\x81p:\xe7(\xd0a=cA\xc1\xae7\x12\xf3\x18b\xa9\xe0\xf1\xe6\xc9\x80\x8b\x9b9\x12\x86\x18\x08;SK%\xbb\xbc\xdd\x06\xf2\xc7\xa9\xff\x0f)\xa6\x1fo6\xfd9\xc5\xf4\x1f\xff\xfdC\xd4\xc6\xc9\xb7[M~\xbc\x8b\xed\x076\x97U^\xd4f\xdf\x11@\xdf\xa6\xe0\x9f\xdd\x95\xf6\xbc\x87\xbah\xd2\xf6\xc7\x13\x91\x05\xf7\xff\x9a\x82\xa6O\xe2o\x87\xe0\xcf7\x10?\x7f:\xdc\xbf\xfd\x0c\xc2\xd78\x90\xfcv\xc7\xde\x0b\xef\xf4\xf8\xb5[\xc0=G+]D\xab\xa2\xd3\xeb-\xdb\xcb;\xf721\xb5\xe6@\x95w&\xa7\xbf\xbe\x07\xfd\xd7?:\x82\xab&\xac\x050v\x95\xeag\x1fm\xf1(f\x16=\xa65\x06j\xf0\x19\x16\x81\xd85\xac!=\x844\xd7r0<\x12\xaa\x87\xc9W\'\xbb\xb6\x90\x83+@\x1e\xb8&\xbeV>\x9e\x9a\x08\x8b%\xbbH\\\xdb\x0e\xddx\t@\xdf\xd4@\xbbS\xc1\xfc\x16\xae\xeb\xac&\xf6\xa5g\xe6\xb6VS\x93/U\xf2:\x96Nk\x97rK\x8c\xa8\xe3\xe9oe\xa6\x7f\xa0B\xad\x87\r\x0c\x1cj\xf8Y\xbb\xff|\xfb^\xe5~\xbb\xd3\xbd\xba|\xbb\x95/\xfb\xdb\xf7\x8dnI\xd0\'\xdfoQ\xfd\xb5\x0f\xf8g\x9f\xd3i\xaeHd\xe6\x93\x07C\xbd\x8f\xc4K\xc8]\xa0\xa8\xb1\xab\xc8\xd5j\xad\xb4\xf3g\xf9\x1d\xcd\xbd\x96AI\x83\x1ab\x97\xf1?\xaar\xf0\xc3\xcf\xee\x10{\xef\x90\x84\x0eu\x89\x9d\xc7\xcfOe\x84\x18y\x00W\x83\xff\x97U\xe7/\'\xfe\xd7H\xad\xf7\xc7\xdb\x9fO\xdc\x7f\xfe\xbfo\x13\xeb~\r\x9e\xb7|\xfd\xf0\xc3\xf3\x07\x9bx\xf6+\x8f^\xfd\x1b\x89g\x7f\x1cKj\xe3\xb9\xb6C\xfa\xf5\x0f\xe9&\x9em\xe2\xd9&\x9em\xe2\xd9&\x9em\xe2\xd9&\x9e}\xe1vn\xe2\xd9&\x9em\xe2\xd9&\x9e}eIl\x13\xcf6\xf1l\x13\xcf6\xf1\xec\xcb\xb6s\x13\xcf6\xf1l\x13\xcf6\xf1\xec+Kb\x9bx\xb6\x89g\x9bx\xb6\x89g_\xb6\x9d\x9bx\xb6\x89g\x9bx\xb6\x89g_Y\x12\xdb\xc4\xb3M<\xdb\xc4\xb3M<\xfb\xb2\xed\xdc\xc4\xb3M<\xdb\xc4\xb3M<\xfb\xca\x92\xd8&\x9em\xe2\xd9&\x9em\xe2\xd9\x97m\xe7&\x9em\xe2\xd9&\x9em\xe2\xd9W\x96\xc46\xf1l\x13\xcf\xde\xd1\x98\xac\xe9g\x1f\xf2\x8f5&\xec\xaf\x9a\xf4\x95\xc5\xb3\xdf\xbec\xbf\x93x\xf6\xdbw\xecw\x12\xcf\xde\xec\xd8\xff\x00\x06\xfb\x9d\xc4\xb3\x7f\xc1P\xfc\xa4x\xf6\x81g\x98\x050Y#\xefh\xd1\xd0`\xcd\x1d\x0fH\x05k\\\x86Db\x8e\x1e`\x03y~\x99\xa99\xd7ACNH(\xfa\xebi\x89\xab\xc3\xf4\xc63\xccu\xc8\x9d\x9e\xd7L\x86p\x8d\xe5\x93Z\x83u\xeeyu\xef4\xaf\'\x0e\xd7\x1c\xa1\xf3\xc5<\xd7\xa5\n:\xc0\xc2zl\xa6I?\xfd\x83g\x98\x7f\xf6X:\xf3#f2\xbd\xf9\xc4\xfe\xc7\x8f\xc5\xb7\xb33\xbe\xbas\xff\xf1\x93\xd8\xf6\xac\x06q\x1e\xeao\xf0\xc7\x9fO\xd8z6\x93[]4\xfd\x9f\xa2\xdb|\xed\xdb?\x1d\x87\xb0*"5\x99\xff\x8b\xff\xb3\xea\xf6S\xfb\xfe\xfb\'\xe8\xedYy\xc3\x16\x07_\xa0\x93\xa4iv~J\xde\xbcNk\xa7\xd9j\xcc\xe5\xd8"K4\xec\xf6\xea\xb5\x17\xe5Fz\xeeJ\xcf\xba\x14\xd7vz\xb2i? \xe0\x0f?>\x1d\xff\xef\xee\xe0\xfd\xe7\xff\x07\x96J \xa4|\x8a=\x00' diff --git a/airbyte-integrations/connectors/source-kyve/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-kyve/unit_tests/test_streams.py index b294187d9112..773e73087357 100644 --- a/airbyte-integrations/connectors/source-kyve/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-kyve/unit_tests/test_streams.py @@ -22,32 +22,29 @@ def patch_base_class(mocker): @pytest.mark.parametrize( - 'stream_offset,stream_offset_context,next_page_token_value', + "stream_offset,stream_offset_context,next_page_token_value", [ - (None, None, 'next_page_token'), - (None, 200, 'next_page_token'), + (None, None, "next_page_token"), + (None, 200, "next_page_token"), (None, None, None), (None, 200, None), (100, None, None), (100, 200, None), - (100, None, 'next_page_token'), - (100, 200, 'next_page_token'), - ] + (100, None, "next_page_token"), + (100, 200, "next_page_token"), + ], ) -def test_request_params(patch_base_class, stream_offset, stream_offset_context,next_page_token_value): +def test_request_params(patch_base_class, stream_offset, stream_offset_context, next_page_token_value): stream = KyveStream(config, pool_data) if stream_offset: stream._offset = 100 - expected_params = { - 'pagination.limit': 100, - 'pagination.offset': stream_offset_context or stream_offset or 0 - } + expected_params = {"pagination.limit": 100, "pagination.offset": stream_offset_context or stream_offset or 0} inputs = { "stream_slice": None, - "stream_state": {'offset': stream_offset_context} if stream_offset_context else {}, - "next_page_token": next_page_token_value + "stream_state": {"offset": stream_offset_context} if stream_offset_context else {}, + "next_page_token": next_page_token_value, } if next_page_token_value: @@ -67,7 +64,7 @@ def test_next_page_token_max_pages_set(patch_base_class): def test_next_page_token(patch_base_class): stream = KyveStream(config, pool_data) inputs = {"response": MagicMock()} - expected_token = {'pagination.offset': 100} + expected_token = {"pagination.offset": 100} assert stream.next_page_token(**inputs) == expected_token @@ -77,49 +74,104 @@ def test_parse_response(patch_base_class, monkeypatch): input_request = requests.Response() inputs = {"response": input_request, "stream_state": {}} - mock_input_request_response_json = {'finalized_bundles': [{"storage_id": 10}], 'pagination': {'next_key': None, 'total': '0'}} + mock_input_request_response_json = { + "finalized_bundles": [ + { + "pool_id": "1", + "id": "0", + "storage_id": "c-24-Ik7KGaB2WJyrW_2fsAjoJkaAD6xfk30qlqEpCI", + "uploader": "kyve199403h5jgfr64r9ewv83zx7q4xphhc4wyv8mhp", + "from_index": "0", + "to_index": "150", + "from_key": "1", + "to_key": "150", + "bundle_summary": "150", + "data_hash": "18446d6b0988bab5cf946482df579ebd6bd32cb289b26cb5515cfb6883269ef9", + "finalized_at": {"height": "2355416", "timestamp": "2023-08-21T13:10:45Z"}, + "storage_provider_id": "2", + "compression_id": "1", + "stake_security": {"valid_vote_power": "957401304506", "total_vote_power": "1140107500247"}, + } + ], + "pagination": {"next_key": "AAAAAAAAAAE=", "total": "36767"}, + } mock_finalized_bundles_request = MagicMock(return_value=mock_input_request_response_json) - monkeypatch.setattr('requests.Response.json', mock_finalized_bundles_request) + monkeypatch.setattr("requests.Response.json", mock_finalized_bundles_request) class _MockContentResponse: def __init__(self): self.content = MOCK_RESPONSE_BINARY + self.ok = True + mock_get_content = MagicMock(return_value=_MockContentResponse()) - monkeypatch.setattr('requests.get', mock_get_content) - mock_response_ok = MagicMock(return_value=True) - monkeypatch.setattr('requests.Response.ok', mock_response_ok) + monkeypatch.setattr("requests.get", mock_get_content) expected_parsed_object = { - 'key': '10647520', - 'value': { - 'hash': '0x3a0e7319f2b5238f3ebc0a5cd419c92bf2c10045e1a8feae4222fce89b8b6287', - 'parentHash': '0x0eb7f809b8cbbd7cf73a56654f2614e3136a0d3496cacaf709dab2da515e568f', - 'number': 10647520, - 'timestamp': 1675576398, - 'nonce': '0x0000000000000000', - 'difficulty': 0, - 'gasLimit': { - 'type': 'BigNumber', - 'hex': '0x02625a00' - }, - 'gasUsed': { - 'type': 'BigNumber', - 'hex': '0x00' + "key": "1", + "value": { + "block": { + "block_id": { + "hash": "C8DC787FAAE0941EF05C75C3AECCF04B85DFB1D4A8D054A463F323B0D9459719", + "parts": {"total": 1, "hash": "B60226F3A84CA6215464AF2983D36C8C7F0CBB12D87F8A933E22DB288EA48E31"}, + }, + "block": { + "header": { + "version": {"block": "11"}, + "chain_id": "osmosis-1", + "height": "1", + "time": "2021-06-18T17:00:00Z", + "last_block_id": {"hash": "", "parts": {"total": 0, "hash": ""}}, + "last_commit_hash": "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855", + "data_hash": "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855", + "validators_hash": "7730A5F777BDB9143E53FAEFE19399C64C75A6689F9A0F7289D66CFD9D1D5EAF", + "next_validators_hash": "7730A5F777BDB9143E53FAEFE19399C64C75A6689F9A0F7289D66CFD9D1D5EAF", + "consensus_hash": "62917BBB85377844C3D12DA9C04E65188A959D13AD2647AB1663219822006E2F", + "app_hash": "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855", + "last_results_hash": "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855", + "evidence_hash": "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855", + "proposer_address": "D8A6C54C54A236D4843BA566520BA03F60F09E35", + }, + "data": {"txs": []}, + "evidence": {"evidence": []}, + "last_commit": { + "height": "0", + "round": 0, + "block_id": {"hash": "", "parts": {"total": 0, "hash": ""}}, + "signatures": [], + }, + }, }, - 'miner': '0xd98b305a9d433f062d44c1D542cdc25ECC0F0a40', - 'extraData': '0x', - 'transactions': [], - 'baseFeePerGas': { - 'type': 'BigNumber', - 'hex': '0x04a817c800' + "block_results": { + "height": "1", + "txs_results": None, + "begin_block_events": [ + { + "type": "epoch_start", + "attributes": [ + {"key": "c3RhcnRfdGltZQ==", "value": "MTYyNDAzNTYwMA=="}, + {"key": "ZXBvY2hfbnVtYmVy", "value": "MA=="}, + ], + }, + { + "type": "epoch_start", + "attributes": [ + {"key": "c3RhcnRfdGltZQ==", "value": "MTYyNDAzNTYwMA=="}, + {"key": "ZXBvY2hfbnVtYmVy", "value": "MA=="}, + ], + }, + ], + "end_block_events": None, + "validator_updates": None, + "consensus_param_updates": { + "block": {"max_bytes": "5242880", "max_gas": "6000000"}, + "evidence": {"max_age_num_blocks": "403200", "max_age_duration": "1209600000000000", "max_bytes": "1048576"}, + "validator": {"pub_key_types": ["ed25519"]}, + }, }, - '_difficulty': { - 'type': 'BigNumber', - 'hex': '0x00' - } - } + }, } + assert next(stream.parse_response(**inputs)) == expected_parsed_object @@ -130,7 +182,7 @@ def test_parse_response_error_on_finalized_bundle_fetching(patch_base_class, mon inputs = {"response": input_request, "stream_state": {}} mock_finalized_bundles_request = MagicMock(side_effect=IndexError) - monkeypatch.setattr('requests.Response.json', mock_finalized_bundles_request) + monkeypatch.setattr("requests.Response.json", mock_finalized_bundles_request) with pytest.raises(StopIteration): next(stream.parse_response(**inputs)) diff --git a/airbyte-integrations/connectors/source-kyve/unit_tests/test_util.py b/airbyte-integrations/connectors/source-kyve/unit_tests/test_util.py deleted file mode 100644 index a5e5bcb8e0f2..000000000000 --- a/airbyte-integrations/connectors/source-kyve/unit_tests/test_util.py +++ /dev/null @@ -1,132 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from source_kyve.util import CustomResourceSchemaLoader - - -def test_custom_loader(): - custom_loader = CustomResourceSchemaLoader("source_kyve") - schema = custom_loader.get_schema("evm/block") - - assert schema == { - "$schema": "http://json-schema.org/draft-04/schema#", - "additionalProperties": True, - "properties": { - "_difficulty": { - "properties": { - "hex": {"type": ["null", "string"]}, - "type": {"type": ["null", "string"]}, - }, - "required": ["hex", "type"], - "type": ["null", "object"], - }, - "difficulty": {"type": ["null", "integer"]}, - "extraData": {"type": ["null", "string"]}, - "gasLimit": { - "properties": { - "hex": {"type": ["null", "string"]}, - "type": {"type": ["null", "string"]}, - }, - "required": ["hex", "type"], - "type": ["null", "object"], - }, - "gasUsed": { - "properties": { - "hex": {"type": ["null", "string"]}, - "type": {"type": ["null", "string"]}, - }, - "required": ["hex", "type"], - "type": ["null", "object"], - }, - "hash": {"type": ["null", "string"]}, - "miner": {"type": ["null", "string"]}, - "number": {"type": ["null", "integer"]}, - "parentHash": {"type": ["null", "string"]}, - "timestamp": {"type": ["null", "integer"]}, - "transactions": { - "items": { - "$schema": "http://json-schema.org/draft-04/schema#", - "additionalProperties": True, - "properties": { - "accessList": {"type": "null"}, - "blockHash": {"type": ["null", "string"]}, - "blockNumber": {"type": ["null", "integer"]}, - "chainId": {"type": ["null", "integer"]}, - "creates": {"type": ["null", "string"]}, - "data": {"type": ["null", "string"]}, - "from": {"type": ["null", "string"]}, - "gasLimit": { - "properties": { - "hex": {"type": ["null", "string"]}, - "type": {"type": ["null", "string"]}, - }, - "required": ["hex", "type"], - "type": ["null", "object"], - }, - "gasPrice": { - "properties": { - "hex": {"type": ["null", "string"]}, - "type": {"type": ["null", "string"]}, - }, - "required": ["hex", "type"], - "type": ["null", "object"], - }, - "hash": {"type": ["null", "string"]}, - "nonce": {"type": ["null", "integer"]}, - "r": {"type": ["null", "string"]}, - "raw": {"type": ["null", "string"]}, - "s": {"type": ["null", "string"]}, - "to": {"type": ["string", "null"]}, - "transactionIndex": {"type": ["null", "integer"]}, - "type": {"type": ["null", "integer"]}, - "v": {"type": ["null", "integer"]}, - "value": { - "properties": { - "hex": {"type": ["null", "string"]}, - "type": {"type": ["null", "string"]}, - }, - "required": ["hex", "type"], - "type": ["null", "object"], - }, - }, - "required": [ - "r", - "s", - "v", - "to", - "data", - "from", - "hash", - "type", - "nonce", - "value", - "chainId", - "creates", - "gasLimit", - "gasPrice", - "blockHash", - "accessList", - "blockNumber", - "transactionIndex", - ], - "type": "object", - }, - "type": ["null", "array"], - }, - }, - "required": [ - "hash", - "miner", - "number", - "gasUsed", - "gasLimit", - "extraData", - "timestamp", - "difficulty", - "parentHash", - "_difficulty", - "transactions", - ], - "type": "object", - } diff --git a/airbyte-integrations/connectors/source-launchdarkly/README.md b/airbyte-integrations/connectors/source-launchdarkly/README.md index a6ba931922ba..3fc1a43a35a9 100644 --- a/airbyte-integrations/connectors/source-launchdarkly/README.md +++ b/airbyte-integrations/connectors/source-launchdarkly/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-launchdarkly:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/launchdarkly) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_launchdarkly/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-launchdarkly:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-launchdarkly build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-launchdarkly:airbyteDocker +An image will be built with the tag `airbyte/source-launchdarkly:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-launchdarkly:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-launchdarkly:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-launchdarkly:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-launchdarkly:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-launchdarkly test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-launchdarkly:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-launchdarkly:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-launchdarkly test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/launchdarkly.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-launchdarkly/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-launchdarkly/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-launchdarkly/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-launchdarkly/build.gradle b/airbyte-integrations/connectors/source-launchdarkly/build.gradle deleted file mode 100644 index c8e2d48f076d..000000000000 --- a/airbyte-integrations/connectors/source-launchdarkly/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_launchdarkly' -} diff --git a/airbyte-integrations/connectors/source-lemlist/.dockerignore b/airbyte-integrations/connectors/source-lemlist/.dockerignore index 880323ffc755..ea2ed092f247 100644 --- a/airbyte-integrations/connectors/source-lemlist/.dockerignore +++ b/airbyte-integrations/connectors/source-lemlist/.dockerignore @@ -1,6 +1,5 @@ * !Dockerfile -!Dockerfile.test !main.py !source_lemlist !setup.py diff --git a/airbyte-integrations/connectors/source-lemlist/Dockerfile b/airbyte-integrations/connectors/source-lemlist/Dockerfile index 6519fa94ff94..82459ff84719 100644 --- a/airbyte-integrations/connectors/source-lemlist/Dockerfile +++ b/airbyte-integrations/connectors/source-lemlist/Dockerfile @@ -34,5 +34,5 @@ COPY source_lemlist ./source_lemlist ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-lemlist diff --git a/airbyte-integrations/connectors/source-lemlist/README.md b/airbyte-integrations/connectors/source-lemlist/README.md index c1d6724ba94d..049a94cddb93 100644 --- a/airbyte-integrations/connectors/source-lemlist/README.md +++ b/airbyte-integrations/connectors/source-lemlist/README.md @@ -1,74 +1,34 @@ # Lemlist Source -This is the repository for the Lemlist source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/lemlist). +This is the repository for the Lemlist configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/lemlist). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-lemlist:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/lemlist) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_lemlist/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/lemlist) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_lemlist/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source lemlist test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-lemlist:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-lemlist build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-lemlist:airbyteDocker +An image will be built with the tag `airbyte/source-lemlist:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-lemlist:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-lemlist:dev check --co docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-lemlist:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-lemlist:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-lemlist test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-lemlist:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-lemlist:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-lemlist test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/lemlist.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-lemlist/__init__.py b/airbyte-integrations/connectors/source-lemlist/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-lemlist/acceptance-test-config.yml b/airbyte-integrations/connectors/source-lemlist/acceptance-test-config.yml index 39155271a6b6..dee191d6552e 100644 --- a/airbyte-integrations/connectors/source-lemlist/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-lemlist/acceptance-test-config.yml @@ -1,18 +1,30 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests connector_image: airbyte/source-lemlist:dev -tests: +acceptance_tests: spec: - - spec_path: "source_lemlist/spec.json" + tests: + - spec_path: "source_lemlist/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: ["campaigns", "activities", "unsubscribes"] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: campaigns + - name: activities + - name: unsubscribes + incremental: + bypass_reason: "This connector does not implement incremental sync" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-lemlist/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-lemlist/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-lemlist/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-lemlist/bootstrap.md b/airbyte-integrations/connectors/source-lemlist/bootstrap.md deleted file mode 100644 index fde382598176..000000000000 --- a/airbyte-integrations/connectors/source-lemlist/bootstrap.md +++ /dev/null @@ -1,26 +0,0 @@ -# Lemlist - -## API Reference - -API Docs: https://developer.lemlist.com/#introduction - -## Overview - -Lemlist is your sales automation and cold email software. Using its API you can retrieve information about campaigns, activities and unsubscribes. - -- Lemlist API uses Basic Authentication. -- Pagination is offset-based. -- It uses fixed-window rate limiting strategy. - -## Endpoints - -Lemlist API consists of four endpoints which can be extracted data from: - - 1. **Team**: This endpoint retrieves information of your team. - 2. **Campaigns**: This endpoint retrieves the list of all campaigns. - 3. **Activities**: This endpoint retrieves the last 100 activities. - 4. **Unsubscribes**: This endpoint retrieves the list of all people who are unsubscribed. - -## Notes - -- The API doesn't have any way to filter information so it doesn't support incremental syncs. diff --git a/airbyte-integrations/connectors/source-lemlist/build.gradle b/airbyte-integrations/connectors/source-lemlist/build.gradle deleted file mode 100644 index d94ebf669746..000000000000 --- a/airbyte-integrations/connectors/source-lemlist/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_lemlist' -} diff --git a/airbyte-integrations/connectors/source-lemlist/integration_tests/__init__.py b/airbyte-integrations/connectors/source-lemlist/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-lemlist/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-lemlist/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-lemlist/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-lemlist/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-lemlist/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-lemlist/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-lemlist/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-lemlist/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-lemlist/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-lemlist/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-lemlist/metadata.yaml b/airbyte-integrations/connectors/source-lemlist/metadata.yaml index 5585511fa412..cf4bd7c33ff8 100644 --- a/airbyte-integrations/connectors/source-lemlist/metadata.yaml +++ b/airbyte-integrations/connectors/source-lemlist/metadata.yaml @@ -1,24 +1,25 @@ data: + allowedHosts: + hosts: + - api.lemlist.com + registries: + cloud: + enabled: true + oss: + enabled: true connectorSubtype: api connectorType: source definitionId: 789f8e7a-2d28-11ec-8d3d-0242ac130003 - dockerImageTag: 0.1.1 - icon: lemlist.svg + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-lemlist githubIssueLabel: source-lemlist + icon: lemlist.svg license: MIT name: Lemlist - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: "2021-10-14" releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/lemlist tags: - - language:python - ab_internal: - sl: 100 - ql: 100 - supportLevel: community + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-lemlist/requirements.txt b/airbyte-integrations/connectors/source-lemlist/requirements.txt index d6e1198b1ab1..cf563bcab685 100644 --- a/airbyte-integrations/connectors/source-lemlist/requirements.txt +++ b/airbyte-integrations/connectors/source-lemlist/requirements.txt @@ -1 +1,2 @@ -e . +-e ../../bases/connector-acceptance-test \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-lemlist/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-lemlist/sample_files/configured_catalog.json deleted file mode 100644 index 6fdac40ccc0f..000000000000 --- a/airbyte-integrations/connectors/source-lemlist/sample_files/configured_catalog.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "team", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "campaigns", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "activities", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "unsubscribes", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-lemlist/setup.py b/airbyte-integrations/connectors/source-lemlist/setup.py index 56192197bbfd..b202001b47e2 100644 --- a/airbyte-integrations/connectors/source-lemlist/setup.py +++ b/airbyte-integrations/connectors/source-lemlist/setup.py @@ -6,10 +6,14 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", + "airbyte-cdk~=0.1", ] -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2.5", "pytest-mock~=3.6.1", "responses~=0.14.0"] +TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest~=6.2", + "pytest-mock~=3.6.1", +] setup( name="source_lemlist", @@ -18,7 +22,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-lemlist/source_lemlist/__init__.py b/airbyte-integrations/connectors/source-lemlist/source_lemlist/__init__.py index e7567c6c38da..49e5d2c93d11 100644 --- a/airbyte-integrations/connectors/source-lemlist/source_lemlist/__init__.py +++ b/airbyte-integrations/connectors/source-lemlist/source_lemlist/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-lemlist/source_lemlist/auth.py b/airbyte-integrations/connectors/source-lemlist/source_lemlist/auth.py deleted file mode 100644 index 3937b9d3ec13..000000000000 --- a/airbyte-integrations/connectors/source-lemlist/source_lemlist/auth.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import base64 -from typing import Tuple - -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator - - -class HttpBasicAuthenticator(TokenAuthenticator): - def __init__(self, auth: Tuple[str, str], auth_method: str = "Basic", **kwargs): - auth_string = f"{auth[0]}:{auth[1]}".encode("utf8") - b64_encoded = base64.b64encode(auth_string).decode("utf8") - super().__init__(token=b64_encoded, auth_method=auth_method, **kwargs) diff --git a/airbyte-integrations/connectors/source-lemlist/source_lemlist/manifest.yaml b/airbyte-integrations/connectors/source-lemlist/source_lemlist/manifest.yaml new file mode 100644 index 000000000000..6b71e2c41425 --- /dev/null +++ b/airbyte-integrations/connectors/source-lemlist/source_lemlist/manifest.yaml @@ -0,0 +1,89 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + requester: + type: HttpRequester + url_base: "https://api.lemlist.com/api/" + http_method: "GET" + authenticator: + type: BasicHttpAuthenticator + username: "" + password: "{{ config['api_key'] }}" + request_params: + limit: 100 + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "OffsetIncrement" + page_size: 100 + page_token_option: + type: "RequestOption" + field_name: "offset" + inject_into: "request_parameter" + requester: + $ref: "#/definitions/requester" + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + team_stream: + $ref: "#/definitions/base_stream" + name: "team" + primary_key: "_id" + $parameters: + path: "/team" + campaigns_stream: + $ref: "#/definitions/base_stream" + name: "campaigns" + primary_key: "_id" + $parameters: + path: "/campaigns" + activities_stream: + $ref: "#/definitions/base_stream" + name: "activities" + primary_key: "_id" + $parameters: + path: "/activities" + unsubscribes_stream: + $ref: "#/definitions/base_stream" + name: "unsubscribes" + primary_key: "_id" + $parameters: + path: "/unsubscribes" + +streams: + - "#/definitions/team_stream" + - "#/definitions/campaigns_stream" + - "#/definitions/activities_stream" + - "#/definitions/unsubscribes_stream" + +check: + type: CheckStream + stream_names: + - "team" + +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/lemlist + connection_specification: + title: Lemlist Spec + type: object + required: + - api_key + additionalProperties: true + properties: + api_key: + type: string + title": API key + description: Lemlist API key, + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-lemlist/source_lemlist/schemas/team.json b/airbyte-integrations/connectors/source-lemlist/source_lemlist/schemas/team.json index 5fe12ac770c5..495ff6d53874 100644 --- a/airbyte-integrations/connectors/source-lemlist/source_lemlist/schemas/team.json +++ b/airbyte-integrations/connectors/source-lemlist/source_lemlist/schemas/team.json @@ -1,15 +1,16 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "_id": { - "type": "string" + "type": ["null", "string"] }, "name": { "type": ["null", "string"] }, "userIds": { - "type": ["null", "array"], + "type": "array", "items": { "type": "string" } @@ -18,24 +19,127 @@ "type": ["null", "string"] }, "createdAt": { + "format": "%Y-%m-%dT%H:%M:%S.%fZ", "type": ["null", "string"] }, - "apiKey": { - "type": ["null", "string"] + "beta": { + "type": "array", + "items": { + "type": "string" + } }, "billing": { - "type": ["null", "object"], + "type": "object", + "properties": { + "products": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "stripeSubscriptionId": { + "type": "string" + }, + "plan": { + "type": "string" + }, + "quantity": { + "type": "integer" + }, + "okUpdatedAt": { + "format": "%Y-%m-%dT%H:%M:%S.%fZ", + "type": ["null", "string"] + }, + "ok": { + "type": ["null", "boolean"] + }, + "freetrialExpiresAt": { + "format": "%Y-%m-%dT%H:%M:%S.%fZ", + "type": ["null", "string"] + } + } + } + } + } + }, + "revenueVisualization": { + "type": "object", + "additionalProperties": true, "properties": { - "quantity": { - "type": ["null", "integer"] + "enabled": { + "type": "boolean" }, - "ok": { - "type": ["null", "boolean"] + "averageContractValue": { + "type": "integer" }, - "plan": { + "averageContractValueCurrency": { + "type": "string" + }, + "conversionRate": { + "type": "integer" + } + } + }, + "_updatedAt": { + "format": "%Y-%m-%dT%H:%M:%S.%fZ", + "type": ["null", "string"] + }, + "dataIntegrityChecksCron": { + "type": "object", + "properties": { + "errors": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": true, + "properties": { + "text": { + "type": "string" + } + } + } + }, + "lastAt": { + "format": "%Y-%m-%dT%H:%M:%S.%fZ", + "type": ["null", "string"] + } + } + }, + "linkedinLastScannedAt": { + "format": "%Y-%m-%dT%H:%M:%S.%fZ", + "type": ["null", "string"] + }, + "ctdCheck": { + "type": "object", + "properties": { + "lastAt": { + "format": "%Y-%m-%dT%H:%M:%S.%fZ", "type": ["null", "string"] } } + }, + "campaignCron": { + "type": "object", + "properties": { + "lastAt": { + "format": "%Y-%m-%dT%H:%M:%S.%fZ", + "type": ["null", "string"] + } + } + }, + "sequencesSafeCheckCron": { + "type": "object", + "properties": { + "lastAt": { + "format": "%Y-%m-%dT%H:%M:%S.%fZ", + "type": ["null", "string"] + } + } + }, + "apiKey": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-lemlist/source_lemlist/source.py b/airbyte-integrations/connectors/source-lemlist/source_lemlist/source.py index fa9192cb40ba..40d485fc7eef 100644 --- a/airbyte-integrations/connectors/source-lemlist/source_lemlist/source.py +++ b/airbyte-integrations/connectors/source-lemlist/source_lemlist/source.py @@ -2,40 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from typing import Any, List, Mapping, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from source_lemlist.auth import HttpBasicAuthenticator +WARNING: Do not modify this file. +""" -from .streams import Activities, Campaigns, Team, Unsubscribes - -class SourceLemlist(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - try: - auth = HttpBasicAuthenticator( - ( - "", - config["api_key"], - ), - ) - - team_stream = Team(authenticator=auth) - team_gen = team_stream.read_records(sync_mode=SyncMode.full_refresh) - - next(team_gen) - return True, None - except Exception as error: - return False, f"The provided API key {config['api_key']} is invalid. - {repr(error)}" - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - auth = HttpBasicAuthenticator( - ( - "", - config["api_key"], - ), - ) - return [Team(authenticator=auth), Campaigns(authenticator=auth), Activities(authenticator=auth), Unsubscribes(authenticator=auth)] +# Declarative Source +class SourceLemlist(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-lemlist/source_lemlist/spec.json b/airbyte-integrations/connectors/source-lemlist/source_lemlist/spec.json deleted file mode 100644 index 7887c9d90f7d..000000000000 --- a/airbyte-integrations/connectors/source-lemlist/source_lemlist/spec.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/lemlist", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Lemlist Spec", - "type": "object", - "required": ["api_key"], - "additionalProperties": false, - "properties": { - "api_key": { - "type": "string", - "title": "API key", - "description": "Lemlist API key.", - "airbyte_secret": true - } - } - } -} diff --git a/airbyte-integrations/connectors/source-lemlist/source_lemlist/streams.py b/airbyte-integrations/connectors/source-lemlist/source_lemlist/streams.py deleted file mode 100644 index e9588e6e1bd3..000000000000 --- a/airbyte-integrations/connectors/source-lemlist/source_lemlist/streams.py +++ /dev/null @@ -1,88 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from typing import Any, Iterable, Mapping, MutableMapping, Optional - -import requests -from airbyte_cdk.sources.streams.http import HttpStream - - -class LemlistStream(HttpStream): - """Default and max value page_size can have is 100""" - - url_base = "https://api.lemlist.com/api/" - primary_key = "_id" - page_size = 100 - initial_offset = 0 - offset = initial_offset - - def __init__(self, **kwargs): - super().__init__(**kwargs) - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """Pagination is offset-based and response doesn't contain a next_page_token - Thus, the only way to know if there are any more pages is to check if the - number of items in current page is equal to the page_size limit""" - - if len(response.json()) == self.page_size: - self.offset += self.page_size - next_page_params = {"offset": self.offset} - return next_page_params - return None - - def path(self, **kwargs) -> str: - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) - params["offset"] = self.initial_offset - params["limit"] = self.page_size - if next_page_token: - params.update(**next_page_token) - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - records = response.json() - yield from records - - def backoff_time(self, response: requests.Response): - if "Retry-After" in response.headers: - return int(response.headers["Retry-After"]) - else: - self.logger.info("Retry-after header not found. Using default backoff value") - return 2 - - -class Team(LemlistStream): - """https://developer.lemlist.com/#get-team-information""" - - def path(self, **kwargs) -> str: - return "team" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - records = response.json() - yield from [records] - - -class Campaigns(LemlistStream): - """https://developer.lemlist.com/#campaigns""" - - def path(self, **kwargs) -> str: - return "campaigns" - - -class Activities(LemlistStream): - """https://developer.lemlist.com/#activities""" - - def path(self, **kwargs) -> str: - return "activities" - - -class Unsubscribes(LemlistStream): - """https://developer.lemlist.com/#unsubscribes""" - - def path(self, **kwargs) -> str: - return "unsubscribes" diff --git a/airbyte-integrations/connectors/source-lemlist/unit_tests/__init__.py b/airbyte-integrations/connectors/source-lemlist/unit_tests/__init__.py deleted file mode 100644 index 46b7376756ec..000000000000 --- a/airbyte-integrations/connectors/source-lemlist/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-lemlist/unit_tests/test_source.py b/airbyte-integrations/connectors/source-lemlist/unit_tests/test_source.py deleted file mode 100644 index aeb669e79eeb..000000000000 --- a/airbyte-integrations/connectors/source-lemlist/unit_tests/test_source.py +++ /dev/null @@ -1,26 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock, patch - -from source_lemlist.source import SourceLemlist - - -@patch("source_lemlist.source.Team.read_records", return_value=iter(["item"])) -def test_check_connection(_): - test_config = {"api_key": "test-api-key"} - logger_mock = MagicMock() - - source = SourceLemlist() - valid_connection, error = source.check_connection(logger_mock, test_config) - - assert valid_connection - - -def test_streams(): - source = SourceLemlist() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 4 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-lemlist/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-lemlist/unit_tests/test_streams.py deleted file mode 100644 index 3de89805735c..000000000000 --- a/airbyte-integrations/connectors/source-lemlist/unit_tests/test_streams.py +++ /dev/null @@ -1,94 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -import requests -import responses -from source_lemlist.streams import LemlistStream - - -def setup_responses(): - responses.add( - responses.GET, - "https://api.lemlist.com/api/example_endpoint", - json=[ - {"_id": "cam_aaWL92T22Sei3Bz6v", "name": "Campaign1", "labels": ["label 1", "label 2"]}, - {"_id": "cam_aaXwBiebA8pWPKqpK", "name": "Campaign2"}, - ], - ) - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(LemlistStream, "path", "v0/example_endpoint") - mocker.patch.object(LemlistStream, "primary_key", "test_primary_key") - mocker.patch.object(LemlistStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = LemlistStream() - inputs = { - "stream_slice": None, - "stream_state": None, - "next_page_token": {"offset": 100}, - } - expected_params = {"limit": stream.page_size, "offset": 100} - assert stream.request_params(**inputs) == expected_params - - -@responses.activate -def test_next_page_token(patch_base_class): - setup_responses() - stream = LemlistStream() - inputs = {"response": requests.get("https://api.lemlist.com/api/example_endpoint")} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -@responses.activate -def test_parse_response(patch_base_class): - setup_responses() - stream = LemlistStream() - inputs = {"response": requests.get("https://api.lemlist.com/api/example_endpoint")} - expected_parsed_object = {"_id": "cam_aaWL92T22Sei3Bz6v", "name": "Campaign1", "labels": ["label 1", "label 2"]} - assert next(stream.parse_response(**inputs)) == expected_parsed_object - - -def test_request_headers(patch_base_class): - stream = LemlistStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": {"offset": 100}} - assert stream.request_headers(**inputs) == {} - - -def test_http_method(patch_base_class): - stream = LemlistStream() - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = LemlistStream() - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = LemlistStream() - expected_backoff_time = 2 - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-lever-hiring/README.md b/airbyte-integrations/connectors/source-lever-hiring/README.md index b1857d9bb975..168feee4a8cd 100644 --- a/airbyte-integrations/connectors/source-lever-hiring/README.md +++ b/airbyte-integrations/connectors/source-lever-hiring/README.md @@ -28,14 +28,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-lever-hiring:build -``` - #### Create credentials **If you are a community contributor**, get the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_lever_hiring/spec.json` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. @@ -54,18 +46,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-lever-hiring:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-lever-hiring build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-lever-hiring:airbyteDocker +An image will be built with the tag `airbyte/source-lever-hiring:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-lever-hiring:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,44 +68,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-lever-hiring:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-lever-hiring:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-lever-hiring:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-lever-hiring test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-lever-hiring:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-lever-hiring:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -122,8 +87,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-lever-hiring test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/lever-hiring.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-lever-hiring/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-lever-hiring/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-lever-hiring/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-lever-hiring/build.gradle b/airbyte-integrations/connectors/source-lever-hiring/build.gradle deleted file mode 100644 index aa222dcc2fb0..000000000000 --- a/airbyte-integrations/connectors/source-lever-hiring/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_lever_hiring' -} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/.coveragerc b/airbyte-integrations/connectors/source-linkedin-ads/.coveragerc new file mode 100644 index 000000000000..6b0b0af5e2ce --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + source_linkedin_ads/run.py \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile b/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile deleted file mode 100644 index 6790451bf0b1..000000000000 --- a/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_linkedin_ads ./source_linkedin_ads - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.6.1 -LABEL io.airbyte.name=airbyte/source-linkedin-ads diff --git a/airbyte-integrations/connectors/source-linkedin-ads/README.md b/airbyte-integrations/connectors/source-linkedin-ads/README.md index 6b17f4d5a616..09a8052cdf6a 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/README.md +++ b/airbyte-integrations/connectors/source-linkedin-ads/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-linkedin-ads:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/linkedin-ads) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_linkedin_ads/spec.json` file. @@ -56,19 +48,70 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-linkedin-ads:dev + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-linkedin-ads build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-linkedin-ads:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations -You can also build the connector image via Gradle: +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew clean :airbyte-integrations:connectors:source-linkedin-ads:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-linkedin-ads:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-linkedin-ads:dev . +# Running the spec command against your patched connector +docker run airbyte/source-linkedin-ads:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -77,50 +120,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-linkedin-ads:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-linkedin-ads:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-linkedin-ads:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install ".[tests]" -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-linkedin-ads test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-linkedin-ads:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` - -To run your acceptance-tests with pre-build connector docker image: -From `.venv` of the connector, run: -``` -python -m pytest -p connector_acceptance_test.plugin -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew clean :airbyte-integrations:connectors:source-linkedin-ads:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew clean :airbyte-integrations:connectors:source-linkedin-ads:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -130,8 +139,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-linkedin-ads test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/linkedin-ads.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-linkedin-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-linkedin-ads/acceptance-test-config.yml index b303c49e4115..64a8fb921408 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-linkedin-ads/acceptance-test-config.yml @@ -5,62 +5,54 @@ test_strictness_level: high acceptance_tests: spec: tests: - - spec_path: "source_linkedin_ads/spec.json" - config_path: "secrets/config_oauth.json" + - spec_path: "source_linkedin_ads/spec.json" + config_path: "secrets/config_oauth.json" connection: tests: - - config_path: "secrets/config_oauth.json" - status: "succeed" - timeout_seconds: 60 - - config_path: "integration_tests/invalid_config.json" - status: "failed" - - config_path: "integration_tests/invalid_config_custom_report.json" - status: "failed" + - config_path: "secrets/config_oauth.json" + status: "succeed" + timeout_seconds: 60 + - config_path: "integration_tests/invalid_config.json" + status: "failed" + - config_path: "integration_tests/invalid_config_custom_report.json" + status: "failed" discovery: tests: - - config_path: "secrets/config_oauth.json" - timeout_seconds: 60 + - config_path: "secrets/config_oauth.json" + timeout_seconds: 60 basic_read: tests: - - config_path: "secrets/config_oauth.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - fail_on_extra_columns: true - timeout_seconds: 3600 - ignored_fields: - campaign_groups: - - name: "lastModified" - bypass_reason: "Volatile data" - empty_streams: - - name: ad_member_company_size_analytics - bypass_reason: "Empty stream; Retention period is 2y" - - name: ad_member_country_analytics - bypass_reason: "Empty stream; Retention period is 2y" - - name: ad_member_job_function_analytics - bypass_reason: "Empty stream; Retention period is 2y" - - name: ad_member_job_title_analytics - bypass_reason: "Empty stream; Retention period is 2y" - - name: ad_member_industry_analytics - bypass_reason: "Empty stream; Retention period is 2y" - - name: ad_member_seniority_analytics - bypass_reason: "Empty stream; Retention period is 2y" - - name: ad_member_region_analytics - bypass_reason: "Empty stream; Retention period is 2y" - - name: ad_member_company_analytics - bypass_reason: "Empty stream; Retention period is 2y" + - config_path: "secrets/config_oauth.json" + expect_records: + path: "integration_tests/expected_records.jsonl" + fail_on_extra_columns: true + timeout_seconds: 3600 + ignored_fields: + campaign_groups: + - name: "lastModified" + bypass_reason: "Volatile data" + ad_campaign_analytics: + - name: "costInLocalCurrency" + bypass_reason: "Data changes too often" + - name: "costInUsd" + bypass_reason: "Data changes too often" + ad_creative_analytics: + - name: "costInLocalCurrency" + bypass_reason: "Data changes too often" + - name: "costInUsd" + bypass_reason: "Data changes too often" incremental: tests: - - config_path: "secrets/config_oauth.json" - configured_catalog_path: "integration_tests/incremental_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" - missing_streams: - - name: accounts - bypass_reason: "This stream is Full-Refresh only" - timeout_seconds: 3600 + - config_path: "secrets/config_oauth.json" + configured_catalog_path: "integration_tests/incremental_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + missing_streams: + - name: accounts + bypass_reason: "This stream is Full-Refresh only" + timeout_seconds: 3600 full_refresh: tests: - - config_path: "secrets/config_oauth.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 3600 - + - config_path: "secrets/config_oauth.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 3600 diff --git a/airbyte-integrations/connectors/source-linkedin-ads/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-linkedin-ads/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-linkedin-ads/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-linkedin-ads/build.gradle b/airbyte-integrations/connectors/source-linkedin-ads/build.gradle deleted file mode 100644 index 6551089257fd..000000000000 --- a/airbyte-integrations/connectors/source-linkedin-ads/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_linkedin_ads' -} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/expected_records.jsonl index 9a55a6a2d76c..2ccd9635d7f7 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-linkedin-ads/integration_tests/expected_records.jsonl @@ -1,50 +1,32 @@ -{"stream":"accounts","data":{"test":false,"notifiedOnCreativeRejection":false,"notifiedOnNewFeaturesEnabled":false,"notifiedOnEndOfCampaign":false,"servingStatuses":["RUNNABLE"],"notifiedOnCampaignOptimization":false,"type":"BUSINESS","version":{"versionTag":"6"},"reference":"urn:li:organization:64265083","notifiedOnCreativeApproval":false,"name":"Jean’s Ad Account","currency":"USD","id":508720451,"status":"ACTIVE","created":"2021-06-14T10:09:22+00:00","lastModified":"2021-08-27T23:40:10+00:00"},"emitted_at":1691579980301} -{"stream":"accounts","data":{"test":false,"notifiedOnCreativeRejection":false,"notifiedOnNewFeaturesEnabled":false,"notifiedOnEndOfCampaign":false,"servingStatuses":["BILLING_HOLD"],"notifiedOnCampaignOptimization":false,"type":"BUSINESS","version":{"versionTag":"4"},"reference":"urn:li:organization:64265083","notifiedOnCreativeApproval":false,"name":"Test Account 2","currency":"USD","id":508774356,"status":"ACTIVE","created":"2021-08-21T21:28:19+00:00","lastModified":"2021-08-25T10:52:25+00:00"},"emitted_at":1691579980306} -{"stream":"accounts","data":{"test":false,"notifiedOnCreativeRejection":false,"notifiedOnNewFeaturesEnabled":false,"notifiedOnEndOfCampaign":false,"servingStatuses":["BILLING_HOLD"],"notifiedOnCampaignOptimization":false,"type":"BUSINESS","version":{"versionTag":"4"},"reference":"urn:li:organization:64265083","notifiedOnCreativeApproval":false,"name":"Test Account 1","currency":"USD","id":508777244,"status":"ACTIVE","created":"2021-08-21T21:27:55+00:00","lastModified":"2021-08-22T20:35:44+00:00"},"emitted_at":1691579980310} -{"stream":"accounts","data":{"test":false,"notifiedOnCreativeRejection":false,"notifiedOnNewFeaturesEnabled":false,"notifiedOnEndOfCampaign":false,"servingStatuses":["BILLING_HOLD"],"notifiedOnCampaignOptimization":false,"type":"BUSINESS","version":{"versionTag":"3"},"reference":"urn:li:person:HRnXB4kIO7","notifiedOnCreativeApproval":false,"name":"Test Account 3","currency":"NOK","id":510426150,"status":"ACTIVE","created":"2022-10-07T16:41:09+00:00","lastModified":"2022-10-07T16:41:09+00:00"},"emitted_at":1691579980314} -{"stream":"account_users","data":{"role":"ACCOUNT_BILLING_ADMIN","user":"urn:li:person:HRnXB4kIO7","account":"urn:li:sponsoredAccount:508720451","created":"2021-06-14T10:09:22+00:00","lastModified":"2021-06-14T10:09:22+00:00"},"emitted_at":1691579983649} -{"stream":"account_users","data":{"role":"ACCOUNT_BILLING_ADMIN","user":"urn:li:person:HRnXB4kIO7","account":"urn:li:sponsoredAccount:508774356","created":"2021-08-21T21:28:19+00:00","lastModified":"2021-08-21T21:28:19+00:00"},"emitted_at":1691579983951} -{"stream":"account_users","data":{"role":"ACCOUNT_BILLING_ADMIN","user":"urn:li:person:HRnXB4kIO7","account":"urn:li:sponsoredAccount:508777244","created":"2021-08-21T21:27:55+00:00","lastModified":"2021-08-21T21:27:55+00:00"},"emitted_at":1691579984340} -{"stream":"account_users","data":{"role":"ACCOUNT_BILLING_ADMIN","user":"urn:li:person:HRnXB4kIO7","account":"urn:li:sponsoredAccount:510426150","created":"2022-10-07T16:41:09+00:00","lastModified":"2022-10-07T16:41:09+00:00"},"emitted_at":1691579984764} -{"stream": "campaign_groups", "data": {"runSchedule": {"start": 1623665362312}, "test": false, "name": "Default Campaign Group", "servingStatuses": ["RUNNABLE"], "backfilled": true, "id": 615492066, "account": "urn:li:sponsoredAccount:508720451", "status": "ACTIVE", "created": "2021-06-14T10:09:22+00:00", "lastModified": "2021-06-14T10:09:22+00:00"}, "emitted_at": 1692699331827} -{"stream": "campaign_groups", "data": {"runSchedule": {"start": 1628229693058, "end": 1695253500000}, "test": false, "totalBudget": {"currencyCode": "USD", "amount": "200"}, "name": "Airbyte Test", "servingStatuses": ["RUNNABLE"], "backfilled": false, "id": 616471656, "account": "urn:li:sponsoredAccount:508720451", "status": "ACTIVE", "created": "2021-08-06T06:01:33+00:00", "lastModified": "2023-08-21T10:08:54+00:00"}, "emitted_at": 1692699331830} -{"stream": "campaign_groups", "data": {"runSchedule": {"start": 1629581299760}, "test": false, "name": "Test Campaign Group 2", "servingStatuses": ["STOPPED", "BILLING_HOLD"], "backfilled": false, "id": 616749096, "account": "urn:li:sponsoredAccount:508774356", "status": "PAUSED", "created": "2021-08-21T21:28:19+00:00", "lastModified": "2021-08-21T21:29:27+00:00"}, "emitted_at": 1692699332065} -{"stream": "campaign_groups", "data": {"runSchedule": {"start": 1629581275652}, "test": false, "name": "Test Campaign Group 1", "servingStatuses": ["STOPPED", "BILLING_HOLD"], "backfilled": false, "id": 616749086, "account": "urn:li:sponsoredAccount:508777244", "status": "PAUSED", "created": "2021-08-21T21:27:55+00:00", "lastModified": "2021-08-22T20:29:09+00:00"}, "emitted_at": 1692699332293} -{"stream": "campaign_groups", "data": {"runSchedule": {"start": 1665160869034}, "test": false, "name": "New Campaign Group", "servingStatuses": ["BILLING_HOLD"], "backfilled": false, "id": 628297234, "account": "urn:li:sponsoredAccount:510426150", "status": "ACTIVE", "created": "2022-10-07T16:41:09+00:00", "lastModified": "2022-10-07T19:16:09+00:00"}, "emitted_at": 1692699332617} -{"stream": "campaigns", "data": {"storyDeliveryEnabled": false, "targetingCriteria": {"include": {"and": [{"type": "urn:li:adTargetingFacet:interfaceLocales", "values": ["urn:li:locale:en_US"]}, {"type": "urn:li:adTargetingFacet:locations", "values": ["urn:li:geo:103644278"]}]}}, "pacingStrategy": "LIFETIME", "locale": {"country": "US", "language": "en"}, "type": "SPONSORED_UPDATES", "optimizationTargetType": "MAX_REACH", "runSchedule": {"start": 1628230144426, "end": 1630971900000}, "costType": "CPM", "creativeSelection": "OPTIMIZED", "offsiteDeliveryEnabled": true, "id": 168387646, "audienceExpansionEnabled": true, "test": false, "format": "STANDARD_UPDATE", "servingStatuses": ["CAMPAIGN_END_DATE_HOLD", "STOPPED"], "version": {"versionTag": "6"}, "objectiveType": "BRAND_AWARENESS", "associatedEntity": "urn:li:organization:64265083", "offsitePreferences": {"iabCategories": {"exclude": []}, "publisherRestrictionFiles": {"include": [], "exclude": []}}, "campaignGroup": "urn:li:sponsoredCampaignGroup:616471656", "dailyBudget": {"currencyCode": "USD", "amount": "10"}, "unitCost": {"currencyCode": "USD", "amount": "62.73"}, "name": "Brand awareness - Aug 6, 2021", "account": "urn:li:sponsoredAccount:508720451", "status": "COMPLETED", "created": "2021-08-06T06:03:52+00:00", "lastModified": "2023-08-21T10:08:58+00:00"}, "emitted_at": 1692704799180} -{"stream": "campaigns", "data": {"storyDeliveryEnabled": false, "targetingCriteria": {"include": {"and": [{"type": "urn:li:adTargetingFacet:titles", "values": ["urn:li:title:100", "urn:li:title:10326", "urn:li:title:10457", "urn:li:title:10738", "urn:li:title:10966", "urn:li:title:11349", "urn:li:title:1159", "urn:li:title:11622", "urn:li:title:1176", "urn:li:title:11886", "urn:li:title:1211", "urn:li:title:12490", "urn:li:title:13499", "urn:li:title:1359", "urn:li:title:1399", "urn:li:title:1414", "urn:li:title:14642", "urn:li:title:14893", "urn:li:title:1586", "urn:li:title:160", "urn:li:title:16432", "urn:li:title:1685", "urn:li:title:17134", "urn:li:title:17265", "urn:li:title:1845", "urn:li:title:189", "urn:li:title:1890", "urn:li:title:18930", "urn:li:title:1897", "urn:li:title:191", "urn:li:title:2105", "urn:li:title:2189", "urn:li:title:219", "urn:li:title:23347", "urn:li:title:23484", "urn:li:title:24", "urn:li:title:25166", "urn:li:title:25169", "urn:li:title:25170", "urn:li:title:25194", "urn:li:title:25201", "urn:li:title:25203", "urn:li:title:25204", "urn:li:title:253", "urn:li:title:266", "urn:li:title:2740", "urn:li:title:3172", "urn:li:title:318", "urn:li:title:328", "urn:li:title:332", "urn:li:title:3516", "urn:li:title:3549", "urn:li:title:3598", "urn:li:title:39", "urn:li:title:3927", "urn:li:title:424", "urn:li:title:4327", "urn:li:title:4384", "urn:li:title:4403", "urn:li:title:4484", "urn:li:title:4677", "urn:li:title:4691", "urn:li:title:5316", "urn:li:title:539", "urn:li:title:556", "urn:li:title:5762", "urn:li:title:599", "urn:li:title:6058", "urn:li:title:607", "urn:li:title:659", "urn:li:title:661", "urn:li:title:67", "urn:li:title:7000", "urn:li:title:7110", "urn:li:title:7176", "urn:li:title:7555", "urn:li:title:761", "urn:li:title:7732", "urn:li:title:9", "urn:li:title:932", "urn:li:title:940", "urn:li:title:9540", "urn:li:title:9633", "urn:li:title:971", "urn:li:title:9715", "urn:li:title:9763"]}, {"type": "urn:li:adTargetingFacet:locations", "values": ["urn:li:geo:103644278"]}, {"type": "urn:li:adTargetingFacet:interfaceLocales", "values": ["urn:li:locale:en_US"]}]}}, "pacingStrategy": "LIFETIME", "locale": {"country": "US", "language": "en"}, "type": "SPONSORED_UPDATES", "optimizationTargetType": "MAX_CLICK", "runSchedule": {"start": 1629849600000}, "costType": "CPM", "creativeSelection": "OPTIMIZED", "offsiteDeliveryEnabled": true, "id": 169185036, "audienceExpansionEnabled": true, "test": false, "format": "STANDARD_UPDATE", "servingStatuses": ["STOPPED", "ACCOUNT_SERVING_HOLD", "CAMPAIGN_GROUP_STATUS_HOLD"], "version": {"versionTag": "3"}, "objectiveType": "WEBSITE_VISIT", "associatedEntity": "urn:li:organization:64265083", "offsitePreferences": {"iabCategories": {"exclude": []}, "publisherRestrictionFiles": {"include": [], "exclude": []}}, "campaignGroup": "urn:li:sponsoredCampaignGroup:616749096", "dailyBudget": {"currencyCode": "USD", "amount": "75"}, "unitCost": {"currencyCode": "USD", "amount": "16.41"}, "name": "Website visits - Aug 25, 2021", "account": "urn:li:sponsoredAccount:508774356", "status": "DRAFT", "created": "2021-08-25T10:52:29+00:00", "lastModified": "2021-11-07T12:41:09+00:00"}, "emitted_at": 1692704799423} -{"stream": "campaigns", "data": {"storyDeliveryEnabled": false, "targetingCriteria": {"include": {"and": [{"type": "urn:li:adTargetingFacet:interfaceLocales", "values": ["urn:li:locale:en_US"]}, {"type": "urn:li:adTargetingFacet:locations", "values": ["urn:li:geo:103644278"]}]}}, "pacingStrategy": "LIFETIME", "locale": {"country": "US", "language": "en"}, "type": "SPONSORED_UPDATES", "optimizationTargetType": "MAX_REACH", "runSchedule": {"start": 1629590400000}, "costType": "CPM", "creativeSelection": "OPTIMIZED", "offsiteDeliveryEnabled": true, "id": 169037246, "audienceExpansionEnabled": true, "test": false, "format": "SINGLE_VIDEO", "servingStatuses": ["STOPPED", "ACCOUNT_SERVING_HOLD", "CAMPAIGN_GROUP_STATUS_HOLD"], "version": {"versionTag": "3"}, "objectiveType": "BRAND_AWARENESS", "associatedEntity": "urn:li:organization:64265083", "offsitePreferences": {"iabCategories": {"exclude": []}, "publisherRestrictionFiles": {"include": [], "exclude": []}}, "campaignGroup": "urn:li:sponsoredCampaignGroup:616749086", "dailyBudget": {"currencyCode": "USD", "amount": "100"}, "unitCost": {"currencyCode": "USD", "amount": "61.02"}, "name": "Brand awareness - Aug 22, 2021", "account": "urn:li:sponsoredAccount:508777244", "status": "DRAFT", "created": "2021-08-22T20:37:17+00:00", "lastModified": "2021-11-07T12:20:05+00:00"}, "emitted_at": 1692704799664} -{"stream": "creatives", "data": {"servingHoldReasons": ["CAMPAIGN_END_DATE_HOLD", "CAMPAIGN_STOPPED"], "lastModifiedAt": 1656599327000, "lastModifiedBy": "urn:li:system:0", "content": {"reference": "urn:li:share:6823991265126957056"}, "createdAt": 1628229937000, "isTest": false, "createdBy": "urn:li:person:HRnXB4kIO7", "review": {"status": "APPROVED"}, "isServing": false, "campaign": "urn:li:sponsoredCampaign:168387646", "id": "urn:li:sponsoredCreative:133813726", "intendedStatus": "ACTIVE", "account": "urn:li:sponsoredAccount:508720451"}, "emitted_at": 1692707187268} -{"stream": "creatives", "data": {"servingHoldReasons": ["UNDER_REVIEW", "CAMPAIGN_STOPPED", "ACCOUNT_SERVING_HOLD", "CAMPAIGN_GROUP_STATUS_HOLD"], "lastModifiedAt": 1656631421000, "lastModifiedBy": "urn:li:system:0", "content": {"reference": "urn:li:share:6836249289476456448"}, "createdAt": 1629888842000, "isTest": false, "createdBy": "urn:li:person:HRnXB4kIO7", "review": {"status": "PENDING"}, "isServing": false, "campaign": "urn:li:sponsoredCampaign:169185036", "id": "urn:li:sponsoredCreative:136324456", "intendedStatus": "ACTIVE", "account": "urn:li:sponsoredAccount:508774356"}, "emitted_at": 1692707187573} -{"stream": "creatives", "data": {"servingHoldReasons": ["UNDER_REVIEW", "CAMPAIGN_STOPPED", "ACCOUNT_SERVING_HOLD", "CAMPAIGN_GROUP_STATUS_HOLD"], "lastModifiedAt": 1631289063000, "lastModifiedBy": "urn:li:person:HRnXB4kIO7", "content": {"reference": "urn:li:ugcPost:6835311566041284608"}, "createdAt": 1629665365000, "isTest": false, "createdBy": "urn:li:person:HRnXB4kIO7", "review": {"status": "PENDING"}, "isServing": false, "campaign": "urn:li:sponsoredCampaign:169037246", "id": "urn:li:sponsoredCreative:135841046", "intendedStatus": "ACTIVE", "account": "urn:li:sponsoredAccount:508777244"}, "emitted_at": 1692707188048} -{"stream":"conversions","data":{"postClickAttributionWindowSize":30,"viewThroughAttributionWindowSize":7,"created":1692168056678,"imagePixelTag":"\"\"","type":"AD_CLICK","enabled":true,"associatedCampaigns":[{"associatedAt":1692609636804,"campaign":"urn:li:sponsoredCampaign:252074216","conversion":"urn:lla:llaPartnerConversion:13703588"},{"associatedAt":1692168067977,"campaign":"urn:li:sponsoredCampaign:251861596","conversion":"urn:lla:llaPartnerConversion:13703588"}],"campaigns":["urn:li:sponsoredCampaign:252074216","urn:li:sponsoredCampaign:251861596"],"name":"Airbyte","urlMatchRuleExpression":[[{"matchValue":"https://airbyte.com/","matchType":"STARTS_WITH"}]],"lastModified":1692168056678,"id":13703588,"attributionType":"LAST_TOUCH_BY_CAMPAIGN","urlRules":[{"type":"STARTS_WITH","matchValue":"https://airbyte.com/"}],"value":{"currencyCode":"USD","amount":"1"},"account":"urn:li:sponsoredAccount:508720451"},"emitted_at":1692726263419} -{"stream":"conversions","data":{"postClickAttributionWindowSize":30,"viewThroughAttributionWindowSize":7,"created":1629376903467,"imagePixelTag":"\"\"","type":"AD_VIEW","enabled":true,"associatedCampaigns":[{"associatedAt":1692167555159,"campaign":"urn:li:sponsoredCampaign:251861596","conversion":"urn:lla:llaPartnerConversion:4677476"},{"associatedAt":1629376986791,"campaign":"urn:li:sponsoredCampaign:168387646","conversion":"urn:lla:llaPartnerConversion:4677476"}],"campaigns":["urn:li:sponsoredCampaign:251861596","urn:li:sponsoredCampaign:168387646"],"name":"Test Conversion","urlMatchRuleExpression":[[{"matchValue":"www.aibyte.io","matchType":"STARTS_WITH"}]],"lastModified":1629380909048,"id":4677476,"attributionType":"LAST_TOUCH_BY_CAMPAIGN","urlRules":[],"value":{"currencyCode":"USD","amount":"0"},"account":"urn:li:sponsoredAccount:508720451"},"emitted_at":1692726263420} -{"stream":"conversions","data":{"postClickAttributionWindowSize":1,"viewThroughAttributionWindowSize":1,"created":1629888666093,"imagePixelTag":"\"\"","type":"SIGN_UP","enabled":true,"associatedCampaigns":[{"associatedAt":1629888749778,"campaign":"urn:li:sponsoredCampaign:169185036","conversion":"urn:lla:llaPartnerConversion:4620028"}],"campaigns":["urn:li:sponsoredCampaign:169185036"],"name":"Test Conversion 3","urlMatchRuleExpression":[[{"matchValue":"https://airbyte.io","matchType":"STARTS_WITH"}]],"lastModified":1629888698401,"id":4620028,"attributionType":"LAST_TOUCH_BY_CAMPAIGN","urlRules":[],"value":{"currencyCode":"USD","amount":"15"},"account":"urn:li:sponsoredAccount:508774356"},"emitted_at":1692726264005} -{"stream":"conversions","data":{"postClickAttributionWindowSize":1,"viewThroughAttributionWindowSize":1,"created":1629664605296,"imagePixelTag":"\"\"","type":"KEY_PAGE_VIEW","enabled":true,"associatedCampaigns":[{"associatedAt":1629664638873,"campaign":"urn:li:sponsoredCampaign:169037246","conversion":"urn:lla:llaPartnerConversion:4604364"}],"campaigns":["urn:li:sponsoredCampaign:169037246"],"name":"Test Conversion 2","urlMatchRuleExpression":[[{"matchValue":"https://airbyte.io","matchType":"STARTS_WITH"}]],"lastModified":1629664630274,"id":4604364,"attributionType":"LAST_TOUCH_BY_CAMPAIGN","urlRules":[],"value":{"currencyCode":"USD","amount":"15"},"account":"urn:li:sponsoredAccount:508777244"},"emitted_at":1692726264514} -{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":14.67,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":14.67,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":830,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-12","end_date":"2021-08-12","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":887,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580070057} -{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":9.230000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":9.230000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":2,"cardClicks":0,"approximateUniqueImpressions":552,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-07","end_date":"2021-08-07","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":2,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":552,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":2,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580070089} -{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":12.470000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":12.470000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":3,"cardClicks":0,"approximateUniqueImpressions":902,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-11","end_date":"2021-08-11","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":3,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":994,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":3,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580070112} -{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":15.000000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":15.000000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":9,"cardClicks":0,"approximateUniqueImpressions":1546,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-09","end_date":"2021-08-09","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":9,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1560,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":9,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580070136} -{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":14.39,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":14.39,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":1241,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-13","end_date":"2021-08-13","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1241,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580070162} -{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":6.460000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":6.460000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":5,"cardClicks":0,"approximateUniqueImpressions":371,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-08","end_date":"2021-08-08","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":5,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":403,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":5,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580070188} -{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":12.360000000000001,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":12.360000000000001,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":1017,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-10","end_date":"2021-08-10","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1017,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580070211} -{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":14.999999999999998,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":14.999999999999998,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":3,"cardClicks":0,"approximateUniqueImpressions":1279,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-06","end_date":"2021-08-06","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":3,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1606,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":3,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580070250} -{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":0.4199999999999997,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":0.4199999999999997,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":116,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-14","end_date":"2021-08-14","pivotValue":"urn:li:sponsoredCampaign:168387646","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":116,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCampaign:168387646"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580070269} -{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":14.999999999999998,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":14.999999999999998,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":3,"cardClicks":0,"approximateUniqueImpressions":1279,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-06","end_date":"2021-08-06","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":3,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1606,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":3,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580220592} -{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":9.230000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":9.230000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":2,"cardClicks":0,"approximateUniqueImpressions":552,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-07","end_date":"2021-08-07","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":2,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":552,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":2,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580220619} -{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":14.67,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":14.67,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":830,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-12","end_date":"2021-08-12","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":887,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580220638} -{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":12.360000000000001,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":12.360000000000001,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":1017,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-10","end_date":"2021-08-10","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1017,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580220656} -{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":15.000000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":15.000000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":9,"cardClicks":0,"approximateUniqueImpressions":1546,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-09","end_date":"2021-08-09","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":9,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1560,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":9,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580220677} -{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":0.4199999999999997,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":0.4199999999999997,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":116,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-14","end_date":"2021-08-14","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":116,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580220697} -{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":14.39,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":14.39,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"approximateUniqueImpressions":1241,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-13","end_date":"2021-08-13","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":1241,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580220720} -{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":6.460000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":6.460000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":5,"cardClicks":0,"approximateUniqueImpressions":371,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-08","end_date":"2021-08-08","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":5,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":403,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":5,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580220745} -{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":12.470000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":12.470000000000002,"conversionValueInLocalCurrency":0,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":3,"cardClicks":0,"approximateUniqueImpressions":902,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-11","end_date":"2021-08-11","pivotValue":"urn:li:sponsoredCreative:133813726","externalWebsitePostClickConversions":0,"externalWebsitePostViewConversions":0,"postClickJobApplyClicks":0,"oneClickLeads":0,"landingPageClicks":3,"fullScreenPlays":0,"follows":0,"oneClickLeadFormOpens":0,"impressions":994,"postClickJobApplications":0,"otherEngagements":0,"jobApplyClicks":0,"jobApplications":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["urn:li:sponsoredCreative:133813726"],"likes":0,"postClickRegistrations":0,"videoCompletions":0,"registrations":0,"talentLeads":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"postViewJobApplyClicks":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"postViewRegistrations":0,"videoThirdQuartileCompletions":0,"totalEngagements":3,"reactions":0,"postViewJobApplications":0,"videoViews":0},"emitted_at":1691580220787} -{"stream":"ad_impression_device_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":0.48000000000000004,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":0.48000000000000004,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":0,"cardClicks":0,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-10","end_date":"2021-08-10","pivotValue":"urn:li:sponsoredCampaign:168387646","oneClickLeads":0,"landingPageClicks":0,"fullScreenPlays":0,"oneClickLeadFormOpens":0,"follows":0,"impressions":36,"otherEngagements":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["MOBILE_APP"],"likes":0,"videoCompletions":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"videoViews":0},"emitted_at":1691580377453} -{"stream":"ad_impression_device_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":3.69,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":3.69,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":3,"cardClicks":0,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-06","end_date":"2021-08-06","pivotValue":"urn:li:sponsoredCampaign:168387646","oneClickLeads":0,"landingPageClicks":3,"fullScreenPlays":0,"oneClickLeadFormOpens":0,"follows":0,"impressions":371,"otherEngagements":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["MOBILE_APP"],"likes":0,"videoCompletions":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"videoThirdQuartileCompletions":0,"totalEngagements":3,"reactions":0,"videoViews":0},"emitted_at":1691580377480} -{"stream":"ad_impression_device_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":11.11,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":11.11,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-12","end_date":"2021-08-12","pivotValue":"urn:li:sponsoredCampaign:168387646","oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"oneClickLeadFormOpens":0,"follows":0,"impressions":674,"otherEngagements":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["DESKTOP_WEB"],"likes":0,"videoCompletions":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"videoViews":0},"emitted_at":1691580377509} -{"stream":"ad_impression_device_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":6.550000000000001,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":6.550000000000001,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-07","end_date":"2021-08-07","pivotValue":"urn:li:sponsoredCampaign:168387646","oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"oneClickLeadFormOpens":0,"follows":0,"impressions":393,"otherEngagements":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["DESKTOP_WEB"],"likes":0,"videoCompletions":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"videoViews":0},"emitted_at":1691580377539} -{"stream":"ad_impression_device_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":0.76,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":0.76,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":0,"cardClicks":0,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-09","end_date":"2021-08-09","pivotValue":"urn:li:sponsoredCampaign:168387646","oneClickLeads":0,"landingPageClicks":0,"fullScreenPlays":0,"oneClickLeadFormOpens":0,"follows":0,"impressions":75,"otherEngagements":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["MOBILE_APP"],"likes":0,"videoCompletions":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"videoThirdQuartileCompletions":0,"totalEngagements":5,"reactions":0,"videoViews":0},"emitted_at":1691580377562} -{"stream":"ad_impression_device_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":0.75,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":0.75,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":0,"cardClicks":0,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-13","end_date":"2021-08-13","pivotValue":"urn:li:sponsoredCampaign:168387646","oneClickLeads":0,"landingPageClicks":0,"fullScreenPlays":0,"oneClickLeadFormOpens":0,"follows":0,"impressions":62,"otherEngagements":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["MOBILE_APP"],"likes":0,"videoCompletions":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"videoViews":0},"emitted_at":1691580377591} -{"stream":"ad_impression_device_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":9.760000000000002,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":9.760000000000002,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":2,"cardClicks":0,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-11","end_date":"2021-08-11","pivotValue":"urn:li:sponsoredCampaign:168387646","oneClickLeads":0,"landingPageClicks":2,"fullScreenPlays":0,"oneClickLeadFormOpens":0,"follows":0,"impressions":797,"otherEngagements":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["DESKTOP_WEB"],"likes":0,"videoCompletions":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"videoThirdQuartileCompletions":0,"totalEngagements":2,"reactions":0,"videoViews":0},"emitted_at":1691580377616} -{"stream":"ad_impression_device_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":0.09,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":0.09,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":0,"cardClicks":0,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-14","end_date":"2021-08-14","pivotValue":"urn:li:sponsoredCampaign:168387646","oneClickLeads":0,"landingPageClicks":0,"fullScreenPlays":0,"oneClickLeadFormOpens":0,"follows":0,"impressions":25,"otherEngagements":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["MOBILE_WEB"],"likes":0,"videoCompletions":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"videoViews":0},"emitted_at":1691580377643} -{"stream":"ad_impression_device_analytics","data":{"documentFirstQuartileCompletions":0,"actionClicks":0,"comments":0,"costInUsd":0.57,"commentLikes":0,"adUnitClicks":0,"companyPageClicks":0,"costInLocalCurrency":0.57,"documentThirdQuartileCompletions":0,"externalWebsiteConversions":0,"cardImpressions":0,"documentCompletions":0,"clicks":1,"cardClicks":0,"documentMidpointCompletions":0,"downloadClicks":0,"start_date":"2021-08-08","end_date":"2021-08-08","pivotValue":"urn:li:sponsoredCampaign:168387646","oneClickLeads":0,"landingPageClicks":1,"fullScreenPlays":0,"oneClickLeadFormOpens":0,"follows":0,"impressions":35,"otherEngagements":0,"leadGenerationMailContactInfoShares":0,"opens":0,"leadGenerationMailInterestedClicks":0,"pivotValues":["MOBILE_APP"],"likes":0,"videoCompletions":0,"viralCardImpressions":0,"videoFirstQuartileCompletions":0,"textUrlClicks":0,"videoStarts":0,"sends":0,"shares":0,"videoMidpointCompletions":0,"validWorkEmailLeads":0,"viralCardClicks":0,"videoThirdQuartileCompletions":0,"totalEngagements":1,"reactions":0,"videoViews":0},"emitted_at":1691580377665} +{"stream": "accounts", "data": {"test": false, "notifiedOnCreativeRejection": false, "notifiedOnNewFeaturesEnabled": false, "notifiedOnEndOfCampaign": false, "servingStatuses": ["RUNNABLE"], "notifiedOnCampaignOptimization": false, "type": "BUSINESS", "version": {"versionTag": "6"}, "reference": "urn:li:organization:64265083", "notifiedOnCreativeApproval": false, "name": "Jean\u2019s Ad Account", "currency": "USD", "id": 508720451, "status": "ACTIVE", "created": "2021-06-14T10:09:22+00:00", "lastModified": "2021-08-27T23:40:10+00:00"}, "emitted_at": 1697196557238} +{"stream": "accounts", "data": {"test": false, "notifiedOnCreativeRejection": false, "notifiedOnNewFeaturesEnabled": false, "notifiedOnEndOfCampaign": false, "servingStatuses": ["BILLING_HOLD"], "notifiedOnCampaignOptimization": false, "type": "BUSINESS", "version": {"versionTag": "4"}, "reference": "urn:li:organization:64265083", "notifiedOnCreativeApproval": false, "name": "Test Account 2", "currency": "USD", "id": 508774356, "status": "ACTIVE", "created": "2021-08-21T21:28:19+00:00", "lastModified": "2021-08-25T10:52:25+00:00"}, "emitted_at": 1697196557239} +{"stream": "accounts", "data": {"test": false, "notifiedOnCreativeRejection": false, "notifiedOnNewFeaturesEnabled": false, "notifiedOnEndOfCampaign": false, "servingStatuses": ["BILLING_HOLD"], "notifiedOnCampaignOptimization": false, "type": "BUSINESS", "version": {"versionTag": "4"}, "reference": "urn:li:organization:64265083", "notifiedOnCreativeApproval": false, "name": "Test Account 1", "currency": "USD", "id": 508777244, "status": "ACTIVE", "created": "2021-08-21T21:27:55+00:00", "lastModified": "2021-08-22T20:35:44+00:00"}, "emitted_at": 1697196557240} +{"stream": "account_users", "data": {"role": "ACCOUNT_BILLING_ADMIN", "user": "urn:li:person:HRnXB4kIO7", "account": "urn:li:sponsoredAccount:508720451", "created": "2021-06-14T10:09:22+00:00", "lastModified": "2021-06-14T10:09:22+00:00"}, "emitted_at": 1697196559364} +{"stream": "account_users", "data": {"role": "ACCOUNT_BILLING_ADMIN", "user": "urn:li:person:HRnXB4kIO7", "account": "urn:li:sponsoredAccount:508774356", "created": "2021-08-21T21:28:19+00:00", "lastModified": "2021-08-21T21:28:19+00:00"}, "emitted_at": 1697196559760} +{"stream": "account_users", "data": {"role": "ACCOUNT_BILLING_ADMIN", "user": "urn:li:person:HRnXB4kIO7", "account": "urn:li:sponsoredAccount:508777244", "created": "2021-08-21T21:27:55+00:00", "lastModified": "2021-08-21T21:27:55+00:00"}, "emitted_at": 1697196560036} +{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0.0,"actionClicks":0.0,"comments":0.0,"costInUsd":-2E-18,"commentLikes":0.0,"adUnitClicks":0.0,"companyPageClicks":0.0,"costInLocalCurrency":-2E-18,"documentThirdQuartileCompletions":0.0,"externalWebsiteConversions":0.0,"cardImpressions":0.0,"documentCompletions":0.0,"clicks":0.0,"cardClicks":0.0,"approximateUniqueImpressions":0.0,"documentMidpointCompletions":0.0,"downloadClicks":0.0,"start_date":"2023-08-26","end_date":"2023-08-26","pivotValue":"urn:li:sponsoredCampaign:252074216","oneClickLeads":0.0,"landingPageClicks":0.0,"fullScreenPlays":0.0,"oneClickLeadFormOpens":0.0,"follows":0.0,"impressions":1.0,"otherEngagements":0.0,"leadGenerationMailContactInfoShares":0.0,"opens":0.0,"leadGenerationMailInterestedClicks":0.0,"pivotValues":["urn:li:sponsoredCampaign:252074216"],"likes":0.0},"emitted_at":1702655286996} +{"stream":"ad_campaign_analytics","data":{"documentFirstQuartileCompletions":0.0,"actionClicks":0.0,"comments":0.0,"costInUsd":100.00000000000004,"commentLikes":0.0,"adUnitClicks":0.0,"companyPageClicks":0.0,"costInLocalCurrency":100.00000000000004,"documentThirdQuartileCompletions":0.0,"externalWebsiteConversions":0.0,"cardImpressions":0.0,"documentCompletions":0.0,"clicks":106.0,"cardClicks":0.0,"approximateUniqueImpressions":17392.0,"documentMidpointCompletions":0.0,"downloadClicks":0.0,"start_date":"2023-08-25","end_date":"2023-08-25","pivotValue":"urn:li:sponsoredCampaign:252074216","oneClickLeads":0.0,"landingPageClicks":106.0,"fullScreenPlays":0.0,"oneClickLeadFormOpens":0.0,"follows":0.0,"impressions":19464.0,"otherEngagements":0.0,"leadGenerationMailContactInfoShares":0.0,"opens":0.0,"leadGenerationMailInterestedClicks":0.0,"pivotValues":["urn:li:sponsoredCampaign:252074216"],"likes":0.0,"videoCompletions":0.0,"viralCardImpressions":0.0,"videoFirstQuartileCompletions":0.0,"textUrlClicks":0.0,"videoStarts":0.0,"sends":0.0,"shares":0.0,"videoMidpointCompletions":0.0,"validWorkEmailLeads":0.0,"viralCardClicks":0.0,"videoThirdQuartileCompletions":0.0,"totalEngagements":106.0,"reactions":0.0,"videoViews":0.0},"emitted_at":1702655287003} +{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0.0,"actionClicks":0.0,"comments":0.0,"costInUsd":-2E-18,"commentLikes":0.0,"adUnitClicks":0.0,"companyPageClicks":0.0,"costInLocalCurrency":-2E-18,"documentThirdQuartileCompletions":0.0,"externalWebsiteConversions":0.0,"cardImpressions":0.0,"documentCompletions":0.0,"clicks":0.0,"cardClicks":0.0,"approximateUniqueImpressions":0.0,"documentMidpointCompletions":0.0,"downloadClicks":0.0,"start_date":"2023-08-26","end_date":"2023-08-26","pivotValue":"urn:li:sponsoredCreative:287513206","oneClickLeads":0.0,"landingPageClicks":0.0,"fullScreenPlays":0.0,"oneClickLeadFormOpens":0.0,"follows":0.0,"impressions":1.0,"otherEngagements":0.0,"leadGenerationMailContactInfoShares":0.0,"opens":0.0,"leadGenerationMailInterestedClicks":0.0,"pivotValues":["urn:li:sponsoredCreative:287513206"],"likes":0.0},"emitted_at":1702656821471} +{"stream":"ad_creative_analytics","data":{"documentFirstQuartileCompletions":0.0,"actionClicks":0.0,"comments":0.0,"costInUsd":100.00000000000004,"commentLikes":0.0,"adUnitClicks":0.0,"companyPageClicks":0.0,"costInLocalCurrency":100.00000000000004,"documentThirdQuartileCompletions":0.0,"externalWebsiteConversions":0.0,"cardImpressions":0.0,"documentCompletions":0.0,"clicks":106.0,"cardClicks":0.0,"approximateUniqueImpressions":17392.0,"documentMidpointCompletions":0.0,"downloadClicks":0.0,"start_date":"2023-08-25","end_date":"2023-08-25","pivotValue":"urn:li:sponsoredCreative:287513206","oneClickLeads":0.0,"landingPageClicks":106.0,"fullScreenPlays":0.0,"oneClickLeadFormOpens":0.0,"follows":0.0,"impressions":19464.0,"otherEngagements":0.0,"leadGenerationMailContactInfoShares":0.0,"opens":0.0,"leadGenerationMailInterestedClicks":0.0,"pivotValues":["urn:li:sponsoredCreative:287513206"],"likes":0.0,"videoCompletions":0.0,"viralCardImpressions":0.0,"videoFirstQuartileCompletions":0.0,"textUrlClicks":0.0,"videoStarts":0.0,"sends":0.0,"shares":0.0,"videoMidpointCompletions":0.0,"validWorkEmailLeads":0.0,"viralCardClicks":0.0,"videoThirdQuartileCompletions":0.0,"totalEngagements":106.0,"reactions":0.0,"videoViews":0.0},"emitted_at":1702656821475} +{"stream": "ad_impression_device_analytics", "data": {"documentFirstQuartileCompletions": 0.0, "actionClicks": 0.0, "comments": 0.0, "costInUsd": 2.29, "commentLikes": 0.0, "adUnitClicks": 0.0, "companyPageClicks": 0.0, "costInLocalCurrency": 2.29, "documentThirdQuartileCompletions": 0.0, "externalWebsiteConversions": 0.0, "cardImpressions": 0.0, "documentCompletions": 0.0, "clicks": 0.0, "cardClicks": 0.0, "documentMidpointCompletions": 0.0, "downloadClicks": 0.0, "start_date": "2023-08-25", "end_date": "2023-08-25", "pivotValue": "urn:li:sponsoredCampaign:252074216", "oneClickLeads": 0.0, "landingPageClicks": 0.0, "fullScreenPlays": 0.0, "oneClickLeadFormOpens": 0.0, "follows": 0.0, "impressions": 498.0, "otherEngagements": 0.0, "leadGenerationMailContactInfoShares": 0.0, "opens": 0.0, "leadGenerationMailInterestedClicks": 0.0, "pivotValues": ["UNDETECTED"], "likes": 0.0, "videoCompletions": 0.0, "viralCardImpressions": 0.0, "videoFirstQuartileCompletions": 0.0, "textUrlClicks": 0.0, "videoStarts": 0.0, "sends": 0.0, "shares": 0.0, "videoMidpointCompletions": 0.0, "validWorkEmailLeads": 0.0, "viralCardClicks": 0.0, "videoThirdQuartileCompletions": 0.0, "totalEngagements": 20.0, "reactions": 0.0, "videoViews": 0.0}, "emitted_at": 1697196622374} +{"stream": "ad_impression_device_analytics", "data": {"documentFirstQuartileCompletions": 0.0, "actionClicks": 0.0, "comments": 0.0, "costInUsd": -2e-18, "commentLikes": 0.0, "adUnitClicks": 0.0, "companyPageClicks": 0.0, "costInLocalCurrency": -2e-18, "documentThirdQuartileCompletions": 0.0, "externalWebsiteConversions": 0.0, "cardImpressions": 0.0, "documentCompletions": 0.0, "clicks": 0.0, "cardClicks": 0.0, "documentMidpointCompletions": 0.0, "downloadClicks": 0.0, "start_date": "2023-08-26", "end_date": "2023-08-26", "pivotValue": "urn:li:sponsoredCampaign:252074216", "oneClickLeads": 0.0, "landingPageClicks": 0.0, "fullScreenPlays": 0.0, "oneClickLeadFormOpens": 0.0, "follows": 0.0, "impressions": 1.0, "otherEngagements": 0.0, "leadGenerationMailContactInfoShares": 0.0, "opens": 0.0, "leadGenerationMailInterestedClicks": 0.0, "pivotValues": ["MOBILE_WEB"], "likes": 0.0}, "emitted_at": 1697196622395} +{"stream": "ad_member_company_size_analytics", "data": {"documentFirstQuartileCompletions": 0.0, "actionClicks": 0.0, "comments": 0.0, "costInUsd": 24.457317520310493, "commentLikes": 0.0, "adUnitClicks": 0.0, "companyPageClicks": 0.0, "costInLocalCurrency": 24.457317520310493, "documentThirdQuartileCompletions": 0.0, "externalWebsiteConversions": 0.0, "documentCompletions": 0.0, "clicks": 9.0, "documentMidpointCompletions": 0.0, "downloadClicks": 0.0, "start_date": "2023-08-25", "end_date": "2023-08-25", "pivotValue": "urn:li:sponsoredCampaign:252074216", "externalWebsitePostClickConversions": 0.0, "externalWebsitePostViewConversions": 0.0, "oneClickLeads": 0.0, "landingPageClicks": 9.0, "fullScreenPlays": 0.0, "follows": 0.0, "oneClickLeadFormOpens": 0.0, "impressions": 1480.0, "otherEngagements": 0.0, "leadGenerationMailContactInfoShares": 0.0, "opens": 0.0, "leadGenerationMailInterestedClicks": 0.0, "pivotValues": ["SIZE_2_TO_10"], "likes": 0.0, "videoCompletions": 0.0, "talentLeads": 0.0, "videoFirstQuartileCompletions": 0.0, "textUrlClicks": 0.0, "videoStarts": 0.0, "sends": 0.0, "shares": 0.0, "videoMidpointCompletions": 0.0, "validWorkEmailLeads": 0.0, "videoThirdQuartileCompletions": 0.0, "totalEngagements": 8.0, "reactions": 0.0, "videoViews": 0.0}, "emitted_at": 1697196644434} +{"stream": "ad_member_country_analytics", "data": {"documentFirstQuartileCompletions": 0.0, "actionClicks": 0.0, "comments": 0.0, "costInUsd": 317.93414846943944, "commentLikes": 0.0, "adUnitClicks": 0.0, "companyPageClicks": 0.0, "costInLocalCurrency": 318.93414846943944, "documentThirdQuartileCompletions": 0.0, "externalWebsiteConversions": 0.0, "documentCompletions": 0.0, "clicks": 110.0, "documentMidpointCompletions": 0.0, "downloadClicks": 0.0, "start_date": "2023-08-25", "end_date": "2023-08-25", "pivotValue": "urn:li:sponsoredCampaign:252074216", "externalWebsitePostClickConversions": 0.0, "externalWebsitePostViewConversions": 0.0, "oneClickLeads": 0.0, "landingPageClicks": 107.0, "fullScreenPlays": 0.0, "follows": 0.0, "oneClickLeadFormOpens": 0.0, "impressions": 19464.0, "otherEngagements": 0.0, "leadGenerationMailContactInfoShares": 0.0, "opens": 0.0, "leadGenerationMailInterestedClicks": 0.0, "pivotValues": ["urn:li:geo:103644278"], "likes": 0.0, "videoCompletions": 0.0, "talentLeads": 0.0, "videoFirstQuartileCompletions": 0.0, "textUrlClicks": 0.0, "videoStarts": 0.0, "sends": 0.0, "shares": 0.0, "videoMidpointCompletions": 0.0, "validWorkEmailLeads": 0.0, "videoThirdQuartileCompletions": 0.0, "totalEngagements": 109.0, "reactions": 0.0, "videoViews": 0.0}, "emitted_at": 1697196666347} +{"stream": "ad_member_job_function_analytics", "data": {"documentFirstQuartileCompletions": 0.0, "actionClicks": 0.0, "comments": 0.0, "costInUsd": 16.428626738541787, "commentLikes": 0.0, "adUnitClicks": 0.0, "companyPageClicks": 0.0, "costInLocalCurrency": 16.428626738541787, "documentThirdQuartileCompletions": 0.0, "externalWebsiteConversions": 0.0, "documentCompletions": 0.0, "clicks": 7.0, "documentMidpointCompletions": 0.0, "downloadClicks": 0.0, "start_date": "2023-08-25", "end_date": "2023-08-25", "pivotValue": "urn:li:sponsoredCampaign:252074216", "externalWebsitePostClickConversions": 0.0, "externalWebsitePostViewConversions": 0.0, "oneClickLeads": 0.0, "landingPageClicks": 4.0, "fullScreenPlays": 0.0, "follows": 0.0, "oneClickLeadFormOpens": 0.0, "impressions": 1064.0, "otherEngagements": 0.0, "leadGenerationMailContactInfoShares": 0.0, "opens": 0.0, "leadGenerationMailInterestedClicks": 0.0, "pivotValues": ["urn:li:function:7"], "likes": 0.0, "videoCompletions": 0.0, "talentLeads": 0.0, "videoFirstQuartileCompletions": 0.0, "textUrlClicks": 0.0, "videoStarts": 0.0, "sends": 0.0, "shares": 0.0, "videoMidpointCompletions": 0.0, "validWorkEmailLeads": 0.0, "videoThirdQuartileCompletions": 0.0, "totalEngagements": 4.0, "reactions": 0.0, "videoViews": 0.0}, "emitted_at": 1697196688970} +{"stream": "ad_member_job_title_analytics", "data": {"documentFirstQuartileCompletions": 0.0, "actionClicks": 0.0, "comments": 0.0, "costInUsd": 13.656450854809513, "commentLikes": 0.0, "adUnitClicks": 0.0, "companyPageClicks": 0.0, "costInLocalCurrency": 11.656450854809513, "documentThirdQuartileCompletions": 0.0, "externalWebsiteConversions": 0.0, "documentCompletions": 0.0, "clicks": 6.0, "documentMidpointCompletions": 0.0, "downloadClicks": 0.0, "start_date": "2023-08-25", "end_date": "2023-08-25", "pivotValue": "urn:li:sponsoredCampaign:252074216", "externalWebsitePostClickConversions": 0.0, "externalWebsitePostViewConversions": 0.0, "oneClickLeads": 0.0, "landingPageClicks": 0.0, "fullScreenPlays": 0.0, "follows": 0.0, "oneClickLeadFormOpens": 0.0, "impressions": 39.0, "otherEngagements": 0.0, "leadGenerationMailContactInfoShares": 0.0, "opens": 0.0, "leadGenerationMailInterestedClicks": 0.0, "pivotValues": ["urn:li:title:68"], "likes": 0.0, "videoCompletions": 0.0, "talentLeads": 0.0, "videoFirstQuartileCompletions": 0.0, "textUrlClicks": 0.0, "videoStarts": 0.0, "sends": 0.0, "shares": 0.0, "videoMidpointCompletions": 0.0, "validWorkEmailLeads": 0.0, "videoThirdQuartileCompletions": 0.0, "totalEngagements": 0.0, "reactions": 0.0, "videoViews": 0.0}, "emitted_at": 1697196712131} +{"stream": "ad_member_industry_analytics", "data": {"documentFirstQuartileCompletions": 0.0, "actionClicks": 0.0, "comments": 0.0, "costInUsd": 7.4596902377485215, "commentLikes": 0.0, "adUnitClicks": 0.0, "companyPageClicks": 0.0, "costInLocalCurrency": 7.4596902377485215, "documentThirdQuartileCompletions": 0.0, "externalWebsiteConversions": 0.0, "documentCompletions": 0.0, "clicks": 3.0, "documentMidpointCompletions": 0.0, "downloadClicks": 0.0, "start_date": "2023-08-25", "end_date": "2023-08-25", "pivotValue": "urn:li:sponsoredCampaign:252074216", "externalWebsitePostClickConversions": 0.0, "externalWebsitePostViewConversions": 0.0, "oneClickLeads": 0.0, "landingPageClicks": 0.0, "fullScreenPlays": 0.0, "follows": 0.0, "oneClickLeadFormOpens": 0.0, "impressions": 165.0, "otherEngagements": 0.0, "leadGenerationMailContactInfoShares": 0.0, "opens": 0.0, "leadGenerationMailInterestedClicks": 0.0, "pivotValues": ["urn:li:industry:99"], "likes": 0.0, "videoCompletions": 0.0, "talentLeads": 0.0, "videoFirstQuartileCompletions": 0.0, "textUrlClicks": 0.0, "videoStarts": 0.0, "sends": 0.0, "shares": 0.0, "videoMidpointCompletions": 0.0, "validWorkEmailLeads": 0.0, "videoThirdQuartileCompletions": 0.0, "totalEngagements": 0.0, "reactions": 0.0, "videoViews": 0.0}, "emitted_at": 1697196734580} +{"stream": "ad_member_seniority_analytics", "data": {"documentFirstQuartileCompletions": 0.0, "actionClicks": 0.0, "comments": 0.0, "costInUsd": 61.54563022857992, "commentLikes": 0.0, "adUnitClicks": 0.0, "companyPageClicks": 0.0, "costInLocalCurrency": 61.54563022857992, "documentThirdQuartileCompletions": 0.0, "externalWebsiteConversions": 0.0, "documentCompletions": 0.0, "clicks": 25.0, "documentMidpointCompletions": 0.0, "downloadClicks": 0.0, "start_date": "2023-08-25", "end_date": "2023-08-25", "pivotValue": "urn:li:sponsoredCampaign:252074216", "externalWebsitePostClickConversions": 0.0, "externalWebsitePostViewConversions": 0.0, "oneClickLeads": 0.0, "landingPageClicks": 28.0, "fullScreenPlays": 0.0, "follows": 0.0, "oneClickLeadFormOpens": 0.0, "impressions": 3762.0, "otherEngagements": 0.0, "leadGenerationMailContactInfoShares": 0.0, "opens": 0.0, "leadGenerationMailInterestedClicks": 0.0, "pivotValues": ["urn:li:seniority:4"], "likes": 0.0, "videoCompletions": 0.0, "talentLeads": 0.0, "videoFirstQuartileCompletions": 0.0, "textUrlClicks": 0.0, "videoStarts": 0.0, "sends": 0.0, "shares": 0.0, "videoMidpointCompletions": 0.0, "validWorkEmailLeads": 0.0, "videoThirdQuartileCompletions": 0.0, "totalEngagements": 28.0, "reactions": 0.0, "videoViews": 0.0}, "emitted_at": 1697196756108} +{"stream": "ad_member_region_analytics", "data": {"documentFirstQuartileCompletions": 0.0, "actionClicks": 0.0, "comments": 0.0, "costInUsd": 12.261068694077421, "commentLikes": 0.0, "adUnitClicks": 0.0, "companyPageClicks": 0.0, "costInLocalCurrency": 11.261068694077421, "documentThirdQuartileCompletions": 0.0, "externalWebsiteConversions": 0.0, "documentCompletions": 0.0, "clicks": 8.0, "documentMidpointCompletions": 0.0, "downloadClicks": 0.0, "start_date": "2023-08-25", "end_date": "2023-08-25", "pivotValue": "urn:li:sponsoredCampaign:252074216", "externalWebsitePostClickConversions": 0.0, "externalWebsitePostViewConversions": 0.0, "oneClickLeads": 0.0, "landingPageClicks": 0.0, "fullScreenPlays": 0.0, "follows": 0.0, "oneClickLeadFormOpens": 0.0, "impressions": 90.0, "otherEngagements": 0.0, "leadGenerationMailContactInfoShares": 0.0, "opens": 0.0, "leadGenerationMailInterestedClicks": 0.0, "pivotValues": ["urn:li:geo:90009446"], "likes": 0.0, "videoCompletions": 0.0, "talentLeads": 0.0, "videoFirstQuartileCompletions": 0.0, "textUrlClicks": 0.0, "videoStarts": 0.0, "sends": 0.0, "shares": 0.0, "videoMidpointCompletions": 0.0, "validWorkEmailLeads": 0.0, "videoThirdQuartileCompletions": 0.0, "totalEngagements": 0.0, "reactions": 0.0, "videoViews": 0.0}, "emitted_at": 1697196779059} +{"stream": "ad_member_company_analytics", "data": {"externalWebsitePostClickConversions": 0.0, "externalWebsitePostViewConversions": 0.0, "oneClickLeads": 0.0, "landingPageClicks": 0.0, "fullScreenPlays": 0.0, "follows": 0.0, "oneClickLeadFormOpens": 0.0, "impressions": 34.0, "otherEngagements": 0.0, "leadGenerationMailContactInfoShares": 0.0, "opens": 0.0, "leadGenerationMailInterestedClicks": 0.0, "pivotValues": ["urn:li:organization:33200573"], "likes": 0.0, "start_date": "2023-08-25", "end_date": "2023-08-25", "pivotValue": "urn:li:sponsoredCampaign:252074216", "videoCompletions": 0.0, "talentLeads": 0.0, "videoFirstQuartileCompletions": 0.0, "textUrlClicks": 0.0, "videoStarts": 0.0, "sends": 0.0, "shares": 0.0, "videoMidpointCompletions": 0.0, "validWorkEmailLeads": 0.0, "videoThirdQuartileCompletions": 0.0, "totalEngagements": 0.0, "reactions": 0.0, "videoViews": 0.0}, "emitted_at": 1697196801205} +{"stream": "campaign_groups", "data": {"runSchedule": {"start": 1623665362312}, "test": false, "name": "Default Campaign Group", "servingStatuses": ["RUNNABLE"], "backfilled": true, "id": 615492066, "account": "urn:li:sponsoredAccount:508720451", "status": "ACTIVE", "created": "2021-06-14T10:09:22+00:00", "lastModified": "2021-06-14T10:09:22+00:00"}, "emitted_at": 1697196810514} +{"stream": "campaign_groups", "data": {"runSchedule": {"start": 1628229693058, "end": 1695253500000}, "test": false, "totalBudget": {"currencyCode": "USD", "amount": "200"}, "name": "Airbyte Test", "servingStatuses": ["CAMPAIGN_GROUP_END_DATE_HOLD", "CAMPAIGN_GROUP_TOTAL_BUDGET_HOLD"], "backfilled": false, "id": 616471656, "account": "urn:li:sponsoredAccount:508720451", "status": "ACTIVE", "created": "2021-08-06T06:01:33+00:00", "lastModified": "2023-09-20T23:33:45+00:00"}, "emitted_at": 1697196810515} +{"stream": "campaign_groups", "data": {"runSchedule": {"start": 1629581299760}, "test": false, "name": "Test Campaign Group 2", "servingStatuses": ["STOPPED", "BILLING_HOLD"], "backfilled": false, "id": 616749096, "account": "urn:li:sponsoredAccount:508774356", "status": "PAUSED", "created": "2021-08-21T21:28:19+00:00", "lastModified": "2021-08-21T21:29:27+00:00"}, "emitted_at": 1697196810793} +{"stream": "campaigns", "data": {"storyDeliveryEnabled": false, "targetingCriteria": {"include": {"and": [{"type": "urn:li:adTargetingFacet:interfaceLocales", "values": ["urn:li:locale:en_US"]}, {"type": "urn:li:adTargetingFacet:locations", "values": ["urn:li:geo:103644278"]}]}}, "pacingStrategy": "LIFETIME", "locale": {"country": "US", "language": "en"}, "type": "SPONSORED_UPDATES", "optimizationTargetType": "MAX_REACH", "runSchedule": {"start": 1628230144426, "end": 1630971900000}, "costType": "CPM", "creativeSelection": "OPTIMIZED", "offsiteDeliveryEnabled": true, "id": 168387646, "audienceExpansionEnabled": true, "test": false, "format": "STANDARD_UPDATE", "servingStatuses": ["CAMPAIGN_END_DATE_HOLD", "STOPPED", "CAMPAIGN_GROUP_END_DATE_HOLD", "CAMPAIGN_GROUP_TOTAL_BUDGET_HOLD"], "version": {"versionTag": "7"}, "objectiveType": "BRAND_AWARENESS", "associatedEntity": "urn:li:organization:64265083", "offsitePreferences": {"iabCategories": {"exclude": []}, "publisherRestrictionFiles": {"include": [], "exclude": []}}, "campaignGroup": "urn:li:sponsoredCampaignGroup:616471656", "dailyBudget": {"currencyCode": "USD", "amount": "10"}, "unitCost": {"currencyCode": "USD", "amount": "62.73"}, "name": "Brand awareness - Aug 6, 2021", "account": "urn:li:sponsoredAccount:508720451", "status": "COMPLETED", "created": "2021-08-06T06:03:52+00:00", "lastModified": "2023-09-20T23:33:56+00:00"}, "emitted_at": 1697196812607} +{"stream": "campaigns", "data": {"storyDeliveryEnabled": false, "targetingCriteria": {"include": {"and": [{"type": "urn:li:adTargetingFacet:interfaceLocales", "values": ["urn:li:locale:en_US"]}, {"type": "urn:li:adTargetingFacet:locations", "values": ["urn:li:geo:103644278"]}]}}, "pacingStrategy": "LIFETIME", "locale": {"country": "US", "language": "en"}, "type": "SPONSORED_UPDATES", "optimizationTargetType": "MAX_REACH", "runSchedule": {"start": 1692612446473, "end": 1695253500000}, "costType": "CPM", "creativeSelection": "OPTIMIZED", "offsiteDeliveryEnabled": true, "id": 252074216, "audienceExpansionEnabled": true, "test": false, "format": "STANDARD_UPDATE", "servingStatuses": ["CAMPAIGN_END_DATE_HOLD", "STOPPED", "CAMPAIGN_GROUP_END_DATE_HOLD", "CAMPAIGN_GROUP_TOTAL_BUDGET_HOLD"], "version": {"versionTag": "15"}, "objectiveType": "BRAND_AWARENESS", "associatedEntity": "urn:li:organization:64265083", "offsitePreferences": {"iabCategories": {"exclude": []}, "publisherRestrictionFiles": {"include": [], "exclude": []}}, "campaignGroup": "urn:li:sponsoredCampaignGroup:616471656", "dailyBudget": {"currencyCode": "USD", "amount": "100"}, "unitCost": {"currencyCode": "USD", "amount": "49.85"}, "name": "Brand awareness - Aug 21, 2023", "account": "urn:li:sponsoredAccount:508720451", "status": "COMPLETED", "created": "2023-08-18T12:05:38+00:00", "lastModified": "2023-09-20T23:33:56+00:00"}, "emitted_at": 1697196812609} +{"stream": "campaigns", "data": {"storyDeliveryEnabled": false, "targetingCriteria": {"include": {"and": [{"type": "urn:li:adTargetingFacet:titles", "values": ["urn:li:title:100", "urn:li:title:10326", "urn:li:title:10457", "urn:li:title:10738", "urn:li:title:10966", "urn:li:title:11349", "urn:li:title:1159", "urn:li:title:11622", "urn:li:title:1176", "urn:li:title:11886", "urn:li:title:1211", "urn:li:title:12490", "urn:li:title:13499", "urn:li:title:1359", "urn:li:title:1399", "urn:li:title:1414", "urn:li:title:14642", "urn:li:title:14893", "urn:li:title:1586", "urn:li:title:160", "urn:li:title:16432", "urn:li:title:1685", "urn:li:title:17134", "urn:li:title:17265", "urn:li:title:1845", "urn:li:title:189", "urn:li:title:1890", "urn:li:title:18930", "urn:li:title:1897", "urn:li:title:191", "urn:li:title:2105", "urn:li:title:2189", "urn:li:title:219", "urn:li:title:23347", "urn:li:title:23484", "urn:li:title:24", "urn:li:title:25166", "urn:li:title:25169", "urn:li:title:25170", "urn:li:title:25194", "urn:li:title:25201", "urn:li:title:25203", "urn:li:title:25204", "urn:li:title:253", "urn:li:title:266", "urn:li:title:2740", "urn:li:title:3172", "urn:li:title:318", "urn:li:title:328", "urn:li:title:332", "urn:li:title:3516", "urn:li:title:3549", "urn:li:title:3598", "urn:li:title:39", "urn:li:title:3927", "urn:li:title:424", "urn:li:title:4327", "urn:li:title:4384", "urn:li:title:4403", "urn:li:title:4484", "urn:li:title:4677", "urn:li:title:4691", "urn:li:title:5316", "urn:li:title:539", "urn:li:title:556", "urn:li:title:5762", "urn:li:title:599", "urn:li:title:6058", "urn:li:title:607", "urn:li:title:659", "urn:li:title:661", "urn:li:title:67", "urn:li:title:7000", "urn:li:title:7110", "urn:li:title:7176", "urn:li:title:7555", "urn:li:title:761", "urn:li:title:7732", "urn:li:title:9", "urn:li:title:932", "urn:li:title:940", "urn:li:title:9540", "urn:li:title:9633", "urn:li:title:971", "urn:li:title:9715", "urn:li:title:9763"]}, {"type": "urn:li:adTargetingFacet:locations", "values": ["urn:li:geo:103644278"]}, {"type": "urn:li:adTargetingFacet:interfaceLocales", "values": ["urn:li:locale:en_US"]}]}}, "pacingStrategy": "LIFETIME", "locale": {"country": "US", "language": "en"}, "type": "SPONSORED_UPDATES", "optimizationTargetType": "MAX_CLICK", "runSchedule": {"start": 1629849600000}, "costType": "CPM", "creativeSelection": "OPTIMIZED", "offsiteDeliveryEnabled": true, "id": 169185036, "audienceExpansionEnabled": true, "test": false, "format": "STANDARD_UPDATE", "servingStatuses": ["STOPPED", "ACCOUNT_SERVING_HOLD", "CAMPAIGN_GROUP_STATUS_HOLD"], "version": {"versionTag": "3"}, "objectiveType": "WEBSITE_VISIT", "associatedEntity": "urn:li:organization:64265083", "offsitePreferences": {"iabCategories": {"exclude": []}, "publisherRestrictionFiles": {"include": [], "exclude": []}}, "campaignGroup": "urn:li:sponsoredCampaignGroup:616749096", "dailyBudget": {"currencyCode": "USD", "amount": "75"}, "unitCost": {"currencyCode": "USD", "amount": "16.41"}, "name": "Website visits - Aug 25, 2021", "account": "urn:li:sponsoredAccount:508774356", "status": "DRAFT", "created": "2021-08-25T10:52:29+00:00", "lastModified": "2021-11-07T12:41:09+00:00"}, "emitted_at": 1697196812612} +{"stream": "creatives", "data": {"servingHoldReasons": ["CAMPAIGN_END_DATE_HOLD", "CAMPAIGN_STOPPED", "CAMPAIGN_GROUP_END_DATE_HOLD", "CAMPAIGN_GROUP_TOTAL_BUDGET_HOLD"], "lastModifiedAt": 1656599327000, "lastModifiedBy": "urn:li:system:0", "content": {"reference": "urn:li:share:6823991265126957056"}, "createdAt": 1628229937000, "isTest": false, "createdBy": "urn:li:person:HRnXB4kIO7", "review": {"status": "APPROVED"}, "isServing": false, "campaign": "urn:li:sponsoredCampaign:168387646", "id": "urn:li:sponsoredCreative:133813726", "intendedStatus": "ACTIVE", "account": "urn:li:sponsoredAccount:508720451"}, "emitted_at": 1697196814776} +{"stream": "creatives", "data": {"servingHoldReasons": ["CAMPAIGN_END_DATE_HOLD", "CAMPAIGN_STOPPED", "CAMPAIGN_GROUP_END_DATE_HOLD", "CAMPAIGN_GROUP_TOTAL_BUDGET_HOLD"], "lastModifiedAt": 1692926398000, "lastModifiedBy": "urn:li:system:0", "content": {"reference": "urn:li:share:6823991265126957056"}, "createdAt": 1692360339000, "isTest": false, "createdBy": "urn:li:person:HRnXB4kIO7", "review": {"status": "APPROVED"}, "isServing": false, "campaign": "urn:li:sponsoredCampaign:252074216", "id": "urn:li:sponsoredCreative:287513206", "intendedStatus": "ACTIVE", "account": "urn:li:sponsoredAccount:508720451"}, "emitted_at": 1697196814778} +{"stream": "creatives", "data": {"servingHoldReasons": ["UNDER_REVIEW", "CAMPAIGN_STOPPED", "ACCOUNT_SERVING_HOLD", "CAMPAIGN_GROUP_STATUS_HOLD"], "lastModifiedAt": 1656631421000, "lastModifiedBy": "urn:li:system:0", "content": {"reference": "urn:li:share:6836249289476456448"}, "createdAt": 1629888842000, "isTest": false, "createdBy": "urn:li:person:HRnXB4kIO7", "review": {"status": "PENDING"}, "isServing": false, "campaign": "urn:li:sponsoredCampaign:169185036", "id": "urn:li:sponsoredCreative:136324456", "intendedStatus": "ACTIVE", "account": "urn:li:sponsoredAccount:508774356"}, "emitted_at": 1697196815100} +{"stream": "conversions", "data": {"postClickAttributionWindowSize": 30, "viewThroughAttributionWindowSize": 7, "created": 1692168056678, "imagePixelTag": "\"\"", "type": "AD_CLICK", "enabled": true, "associatedCampaigns": [{"associatedAt": 1692609636804, "campaign": "urn:li:sponsoredCampaign:252074216", "conversion": "urn:lla:llaPartnerConversion:13703588"}, {"associatedAt": 1692168067977, "campaign": "urn:li:sponsoredCampaign:251861596", "conversion": "urn:lla:llaPartnerConversion:13703588"}], "campaigns": ["urn:li:sponsoredCampaign:252074216", "urn:li:sponsoredCampaign:251861596"], "name": "Airbyte", "urlMatchRuleExpression": [[{"matchValue": "https://airbyte.com/", "matchType": "STARTS_WITH"}]], "lastModified": 1692168056678, "id": 13703588, "attributionType": "LAST_TOUCH_BY_CAMPAIGN", "urlRules": [{"type": "STARTS_WITH", "matchValue": "https://airbyte.com/"}], "value": {"currencyCode": "USD", "amount": "1"}, "account": "urn:li:sponsoredAccount:508720451"}, "emitted_at": 1697196818200} +{"stream": "conversions", "data": {"postClickAttributionWindowSize": 30, "viewThroughAttributionWindowSize": 7, "created": 1629376903467, "imagePixelTag": "\"\"", "type": "AD_VIEW", "enabled": true, "associatedCampaigns": [{"associatedAt": 1692167555159, "campaign": "urn:li:sponsoredCampaign:251861596", "conversion": "urn:lla:llaPartnerConversion:4677476"}, {"associatedAt": 1629376986791, "campaign": "urn:li:sponsoredCampaign:168387646", "conversion": "urn:lla:llaPartnerConversion:4677476"}], "campaigns": ["urn:li:sponsoredCampaign:251861596", "urn:li:sponsoredCampaign:168387646"], "name": "Test Conversion", "urlMatchRuleExpression": [[{"matchValue": "www.aibyte.io", "matchType": "STARTS_WITH"}]], "lastModified": 1629380909048, "id": 4677476, "attributionType": "LAST_TOUCH_BY_CAMPAIGN", "urlRules": [], "value": {"currencyCode": "USD", "amount": "0"}, "account": "urn:li:sponsoredAccount:508720451"}, "emitted_at": 1697196818201} +{"stream": "conversions", "data": {"postClickAttributionWindowSize": 1, "viewThroughAttributionWindowSize": 1, "created": 1629888666093, "imagePixelTag": "\"\"", "type": "SIGN_UP", "enabled": true, "associatedCampaigns": [{"associatedAt": 1629888749778, "campaign": "urn:li:sponsoredCampaign:169185036", "conversion": "urn:lla:llaPartnerConversion:4620028"}], "campaigns": ["urn:li:sponsoredCampaign:169185036"], "name": "Test Conversion 3", "urlMatchRuleExpression": [[{"matchValue": "https://airbyte.io", "matchType": "STARTS_WITH"}]], "lastModified": 1629888698401, "id": 4620028, "attributionType": "LAST_TOUCH_BY_CAMPAIGN", "urlRules": [], "value": {"currencyCode": "USD", "amount": "15"}, "account": "urn:li:sponsoredAccount:508774356"}, "emitted_at": 1697196818777} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/main.py b/airbyte-integrations/connectors/source-linkedin-ads/main.py index c51fcd1a5cc5..899a7e8614a4 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/main.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_linkedin_ads import SourceLinkedinAds +from source_linkedin_ads.run import run if __name__ == "__main__": - source = SourceLinkedinAds() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-linkedin-ads/metadata.yaml b/airbyte-integrations/connectors/source-linkedin-ads/metadata.yaml index b74e5c81af1d..eef04a5cd95b 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/metadata.yaml +++ b/airbyte-integrations/connectors/source-linkedin-ads/metadata.yaml @@ -1,17 +1,23 @@ data: + ab_internal: + ql: 400 + sl: 300 allowedHosts: hosts: - linkedin.com - api.linkedin.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 137ece28-5434-455c-8f34-69dc3782f451 - maxSecondsBetweenMessages: 21600 - dockerImageTag: 0.6.1 + dockerImageTag: 0.6.7 dockerRepository: airbyte/source-linkedin-ads + documentationUrl: https://docs.airbyte.com/integrations/sources/linkedin-ads githubIssueLabel: source-linkedin-ads icon: linkedin.svg license: MIT + maxSecondsBetweenMessages: 86400 name: LinkedIn Ads registries: cloud: @@ -28,11 +34,7 @@ data: - campaigns - campaign_groups - creatives - documentationUrl: https://docs.airbyte.com/integrations/sources/linkedin-ads + supportLevel: certified tags: - language:python - ab_internal: - sl: 300 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-linkedin-ads/setup.py b/airbyte-integrations/connectors/source-linkedin-ads/setup.py index 1c15f41abf5f..ceff2ed3bf72 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/setup.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/setup.py @@ -5,9 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.50", -] +MAIN_REQUIREMENTS = ["airbyte-cdk"] TEST_REQUIREMENTS = [ "pytest-mock~=3.6.1", @@ -16,6 +14,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-linkedin-ads=source_linkedin_ads.run:run", + ], + }, name="source_linkedin_ads", description="Source implementation for Linkedin Ads.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics.py deleted file mode 100644 index 6963ecf1ac9f..000000000000 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics.py +++ /dev/null @@ -1,207 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from collections import defaultdict -from typing import Any, Iterable, List, Mapping - -import pendulum as pdm - -from .utils import get_parent_stream_values - -# LinkedIn has a max of 20 fields per request. We make chunks by size of 19 fields -# to have the `dateRange` be included as well. -FIELDS_CHUNK_SIZE = 19 -# Number of days ahead for date slices, from start date. -WINDOW_IN_DAYS = 30 -# List of Reporting Metrics fields available for fetch -ANALYTICS_FIELDS_V2: List = [ - "actionClicks", - "adUnitClicks", - "approximateUniqueImpressions", - "cardClicks", - "cardImpressions", - "clicks", - "commentLikes", - "comments", - "companyPageClicks", - "conversionValueInLocalCurrency", - "costInLocalCurrency", - "costInUsd", - "dateRange", - "documentCompletions", - "documentFirstQuartileCompletions", - "documentMidpointCompletions", - "documentThirdQuartileCompletions", - "downloadClicks", - "externalWebsiteConversions", - "externalWebsitePostClickConversions", - "externalWebsitePostViewConversions", - "follows", - "fullScreenPlays", - "impressions", - "jobApplications", - "jobApplyClicks", - "landingPageClicks", - "leadGenerationMailContactInfoShares", - "leadGenerationMailInterestedClicks", - "likes", - "oneClickLeadFormOpens", - "oneClickLeads", - "opens", - "otherEngagements", - "pivotValues", - "postClickJobApplications", - "postClickJobApplyClicks", - "postClickRegistrations", - "postViewJobApplications", - "postViewJobApplyClicks", - "postViewRegistrations", - "reactions", - "registrations", - "sends", - "shares", - "talentLeads", - "textUrlClicks", - "totalEngagements", - "validWorkEmailLeads", - "videoCompletions", - "videoFirstQuartileCompletions", - "videoMidpointCompletions", - "videoStarts", - "videoThirdQuartileCompletions", - "videoViews", - "viralCardClicks", - "viralCardImpressions", - "viralClicks", - "viralCommentLikes", - "viralComments", - "viralCompanyPageClicks", - "viralDocumentCompletions", - "viralDocumentFirstQuartileCompletions", - "viralDocumentMidpointCompletions", - "viralDocumentThirdQuartileCompletions", - "viralDownloadClicks", - "viralExternalWebsiteConversions", - "viralExternalWebsitePostClickConversions", - "viralExternalWebsitePostViewConversions", - "viralFollows", - "viralFullScreenPlays", - "viralImpressions", - "viralJobApplications", - "viralJobApplyClicks", - "viralLandingPageClicks", - "viralLikes", - "viralOneClickLeadFormOpens", - "viralOneClickLeads", - "viralOtherEngagements", - "viralPostClickJobApplications", - "viralPostClickJobApplyClicks", - "viralPostClickRegistrations", - "viralPostViewJobApplications", - "viralPostViewJobApplyClicks", - "viralPostViewRegistrations", - "viralReactions", - "viralRegistrations", - "viralShares", - "viralTotalEngagements", - "viralVideoCompletions", - "viralVideoFirstQuartileCompletions", - "viralVideoMidpointCompletions", - "viralVideoStarts", - "viralVideoThirdQuartileCompletions", - "viralVideoViews", -] -# Fields that are always present in fields_set chunks -BASE_ANALLYTICS_FIELDS = ["dateRange"] - - -def chunk_analytics_fields( - fields: List = ANALYTICS_FIELDS_V2, - base_fields: List = BASE_ANALLYTICS_FIELDS, - fields_chunk_size: int = FIELDS_CHUNK_SIZE, -) -> Iterable[List]: - """ - Chunks the list of available fields into the chunks of equal size. - """ - # Make chunks - chunks = list((fields[f : f + fields_chunk_size] for f in range(0, len(fields), fields_chunk_size))) - # Make sure base_fields are within the chunks - for chunk in chunks: - for field in base_fields: - if field not in chunk: - chunk.append(field) - yield from chunks - - -def make_date_slices(start_date: str, end_date: str = None, window_in_days: int = WINDOW_IN_DAYS) -> Iterable[List]: - """ - Produces date slices from start_date to end_date (if specified), - otherwise end_date will be present time. - """ - start = pdm.parse(start_date) - end = pdm.parse(end_date) if end_date else pdm.now() - date_slices = [] - while start < end: - slice_end_date = start.add(days=window_in_days) - date_slice = { - "start.day": start.day, - "start.month": start.month, - "start.year": start.year, - "end.day": slice_end_date.day, - "end.month": slice_end_date.month, - "end.year": slice_end_date.year, - } - date_slices.append({"dateRange": date_slice}) - start = slice_end_date - yield from date_slices - - -def make_analytics_slices( - record: Mapping[str, Any], key_value_map: Mapping[str, Any], start_date: str, end_date: str = None -) -> Iterable[Mapping[str, Any]]: - """ - We drive the ability to directly pass the prepared parameters inside the stream_slice. - The output of this method is ready slices for analytics streams: - """ - # define the base_slice - base_slice = get_parent_stream_values(record, key_value_map) - # add chunked fields, date_slices to the base_slice - analytics_slices = [] - for fields_set in chunk_analytics_fields(): - base_slice["fields"] = ",".join(map(str, fields_set)) - for date_slice in make_date_slices(start_date, end_date): - base_slice.update(**date_slice) - analytics_slices.append(base_slice.copy()) - yield from analytics_slices - - -def update_analytics_params(stream_slice: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Produces the date range parameters from input stream_slice - """ - date_range = stream_slice["dateRange"] - return { - "dateRange": f"(start:(year:{date_range['start.year']},month:{date_range['start.month']},day:{date_range['start.day']})," - f"end:(year:{date_range['end.year']},month:{date_range['end.month']},day:{date_range['end.day']}))", - # Chunk of fields - "fields": stream_slice["fields"], - } - - -def merge_chunks(chunked_result: Iterable[Mapping[str, Any]], merge_by_key: str) -> Iterable[Mapping[str, Any]]: - """ - We need to merge the chunked API responses - into the single structure using any available unique field. - """ - # Merge the pieces together - merged = defaultdict(dict) - for chunk in chunked_result: - for item in chunk: - merged[item[merge_by_key]].update(item) - # Clean up the result by getting out the values of the merged keys - result = [] - for item in merged: - result.append(merged.get(item)) - yield from result diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics_streams.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics_streams.py new file mode 100644 index 000000000000..f58da0e25c8b --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/analytics_streams.py @@ -0,0 +1,373 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from abc import ABC, abstractmethod +from collections import defaultdict +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional +from urllib.parse import urlencode + +import pendulum +import requests +from airbyte_cdk.sources.streams.core import package_name_from_class +from airbyte_cdk.sources.utils import casing +from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader +from airbyte_protocol.models import SyncMode +from source_linkedin_ads.streams import Campaigns, Creatives, IncrementalLinkedinAdsStream + +from .utils import get_parent_stream_values, transform_data + +# Number of days ahead for date slices, from start date. +WINDOW_IN_DAYS = 30 +# List of Reporting Metrics fields available for fetch +ANALYTICS_FIELDS_V2: List = [ + "actionClicks", + "adUnitClicks", + "approximateUniqueImpressions", + "cardClicks", + "cardImpressions", + "clicks", + "commentLikes", + "comments", + "companyPageClicks", + "conversionValueInLocalCurrency", + "costInLocalCurrency", + "costInUsd", + "dateRange", + "documentCompletions", + "documentFirstQuartileCompletions", + "documentMidpointCompletions", + "documentThirdQuartileCompletions", + "downloadClicks", + "externalWebsiteConversions", + "externalWebsitePostClickConversions", + "externalWebsitePostViewConversions", + "follows", + "fullScreenPlays", + "impressions", + "jobApplications", + "jobApplyClicks", + "landingPageClicks", + "leadGenerationMailContactInfoShares", + "leadGenerationMailInterestedClicks", + "likes", + "oneClickLeadFormOpens", + "oneClickLeads", + "opens", + "otherEngagements", + "pivotValues", + "postClickJobApplications", + "postClickJobApplyClicks", + "postClickRegistrations", + "postViewJobApplications", + "postViewJobApplyClicks", + "postViewRegistrations", + "reactions", + "registrations", + "sends", + "shares", + "talentLeads", + "textUrlClicks", + "totalEngagements", + "validWorkEmailLeads", + "videoCompletions", + "videoFirstQuartileCompletions", + "videoMidpointCompletions", + "videoStarts", + "videoThirdQuartileCompletions", + "videoViews", + "viralCardClicks", + "viralCardImpressions", + "viralClicks", + "viralCommentLikes", + "viralComments", + "viralCompanyPageClicks", + "viralDocumentCompletions", + "viralDocumentFirstQuartileCompletions", + "viralDocumentMidpointCompletions", + "viralDocumentThirdQuartileCompletions", + "viralDownloadClicks", + "viralExternalWebsiteConversions", + "viralExternalWebsitePostClickConversions", + "viralExternalWebsitePostViewConversions", + "viralFollows", + "viralFullScreenPlays", + "viralImpressions", + "viralJobApplications", + "viralJobApplyClicks", + "viralLandingPageClicks", + "viralLikes", + "viralOneClickLeadFormOpens", + "viralOneClickLeads", + "viralOtherEngagements", + "viralPostClickJobApplications", + "viralPostClickJobApplyClicks", + "viralPostClickRegistrations", + "viralPostViewJobApplications", + "viralPostViewJobApplyClicks", + "viralPostViewRegistrations", + "viralReactions", + "viralRegistrations", + "viralShares", + "viralTotalEngagements", + "viralVideoCompletions", + "viralVideoFirstQuartileCompletions", + "viralVideoMidpointCompletions", + "viralVideoStarts", + "viralVideoThirdQuartileCompletions", + "viralVideoViews", +] + + +class LinkedInAdsAnalyticsStream(IncrementalLinkedinAdsStream, ABC): + """ + AdAnalytics Streams more info: + https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?tabs=curl&view=li-lms-2023-05#analytics-finder + """ + + endpoint = "adAnalytics" + # For Analytics streams, the primary_key is the entity of the pivot [Campaign URN, Creative URN, etc.] + `end_date` + primary_key = ["pivotValue", "end_date"] + cursor_field = "end_date" + records_limit = 15000 + FIELDS_CHUNK_SIZE = 19 + + def get_json_schema(self) -> Mapping[str, Any]: + return ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("ad_analytics") + + def __init__(self, name: str = None, pivot_by: str = None, time_granularity: str = None, **kwargs): + self.user_stream_name = name + if pivot_by: + self.pivot_by = pivot_by + if time_granularity: + self.time_granularity = time_granularity + super().__init__(**kwargs) + + @property + @abstractmethod + def search_param(self) -> str: + """ + :return: Search parameters for the request + """ + + @property + @abstractmethod + def search_param_value(self) -> str: + """ + :return: Name field to filter by + """ + + @property + @abstractmethod + def parent_values_map(self) -> Mapping[str, str]: + """ + :return: Mapping for parent child relation + """ + + @property + def name(self) -> str: + """We override the stream name to let the user change it via configuration.""" + name = self.user_stream_name or self.__class__.__name__ + return casing.camel_to_snake(name) + + @property + def base_analytics_params(self) -> MutableMapping[str, Any]: + """Define the base parameters for analytics streams""" + return {"q": "analytics", "pivot": f"(value:{self.pivot_by})", "timeGranularity": f"(value:{self.time_granularity})"} + + def request_headers( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> Mapping[str, Any]: + headers = super().request_headers(stream_state, stream_slice, next_page_token) + return headers | {"X-Restli-Protocol-Version": "2.0.0"} + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + params = self.base_analytics_params + params.update(**self.update_analytics_params(stream_slice)) + params[self.search_param] = f"List(urn%3Ali%3A{self.search_param_value}%3A{self.get_primary_key_from_slice(stream_slice)})" + return urlencode(params, safe="():,%") + + @staticmethod + def update_analytics_params(stream_slice: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Produces the date range parameters from input stream_slice + """ + date_range = stream_slice["dateRange"] + return { + "dateRange": f"(start:(year:{date_range['start.year']},month:{date_range['start.month']},day:{date_range['start.day']})," + f"end:(year:{date_range['end.year']},month:{date_range['end.month']},day:{date_range['end.day']}))", + # Chunk of fields + "fields": stream_slice["fields"], + } + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + Pagination is not supported + (See Restrictions: https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?view=li-lms-2023-09&tabs=http#restrictions) + """ + parsed_response = response.json() + if len(parsed_response.get("elements")) < self.records_limit: + return None + raise Exception( + f"Limit {self.records_limit} elements exceeded. " + f"Try to request your data in more granular pieces. " + f"(For example switch `Time Granularity` from MONTHLY to DAILY)" + ) + + def get_primary_key_from_slice(self, stream_slice) -> str: + return stream_slice.get(self.primary_slice_key) + + def stream_slices( + self, *, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None + ) -> Iterable[List[Mapping[str, Any]]]: + """ + LinkedIn has a max of 20 fields per request. We make chunks by size of 19 fields to have the `dateRange` be included as well. + https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?view=li-lms-2023-05&tabs=http#requesting-specific-metrics-in-the-analytics-finder + + :param sync_mode: + :param cursor_field: + :param stream_state: + :return: Iterable with List of stream slices within the same date range and chunked fields, example + [{'campaign_id': 123, 'fields': 'field_1,field_2,dateRange', 'dateRange': {'start.day': 1, 'start.month': 1, 'start.year': 2020, 'end.day': 30, 'end.month': 1, 'end.year': 2020}}, + {'campaign_id': 123, 'fields': 'field_2,field_3,dateRange', 'dateRange': {'start.day': 1, 'start.month': 1, 'start.year': 2020, 'end.day': 30, 'end.month': 1, 'end.year': 2020}}, + {'campaign_id': 123, 'fields': 'field_4,field_5,dateRange', 'dateRange': {'start.day': 1, 'start.month': 1, 'start.year': 2020, 'end.day': 30, 'end.month': 1, 'end.year': 2020}}] + + """ + parent_stream = self.parent_stream(config=self.config) + stream_state = stream_state or {self.cursor_field: self.config.get("start_date")} + for record in parent_stream.read_records(sync_mode=sync_mode): + base_slice = get_parent_stream_values(record, self.parent_values_map) + for date_slice in self.get_date_slices(stream_state.get(self.cursor_field), self.config.get("end_date")): + date_slice_with_fields: List = [] + for fields_set in self.chunk_analytics_fields(): + base_slice["fields"] = ",".join(fields_set) + date_slice_with_fields.append(base_slice | date_slice) + yield date_slice_with_fields + + @staticmethod + def get_date_slices(start_date: str, end_date: str = None, window_in_days: int = WINDOW_IN_DAYS) -> Iterable[Mapping[str, Any]]: + """ + Produces date slices from start_date to end_date (if specified), + otherwise end_date will be present time. + """ + start = pendulum.parse(start_date) + end = pendulum.parse(end_date) if end_date else pendulum.now() + date_slices = [] + while start < end: + slice_end_date = start.add(days=window_in_days) + date_slice = { + "start.day": start.day, + "start.month": start.month, + "start.year": start.year, + "end.day": slice_end_date.day, + "end.month": slice_end_date.month, + "end.year": slice_end_date.year, + } + date_slices.append({"dateRange": date_slice}) + start = slice_end_date + yield from date_slices + + @staticmethod + def chunk_analytics_fields( + fields: List = ANALYTICS_FIELDS_V2, + fields_chunk_size: int = FIELDS_CHUNK_SIZE, + ) -> Iterable[List]: + """ + Chunks the list of available fields into the chunks of equal size. + """ + # Make chunks + chunks = list((fields[f : f + fields_chunk_size] for f in range(0, len(fields), fields_chunk_size))) + # Make sure base_fields are within the chunks + for chunk in chunks: + if "dateRange" not in chunk: + chunk.append("dateRange") + yield from chunks + + def read_records( + self, stream_state: Mapping[str, Any] = None, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs + ) -> Iterable[Mapping[str, Any]]: + merged_records = defaultdict(dict) + for field_slice in stream_slice: + for rec in super().read_records(stream_slice=field_slice, **kwargs): + merged_records[rec[self.cursor_field]].update(rec) + yield from merged_records.values() + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + We need to get out the nested complex data structures for further normalization, so the transform_data method is applied. + """ + for rec in transform_data(response.json().get("elements")): + yield rec | {"pivotValue": f"urn:li:{self.search_param_value}:{self.get_primary_key_from_slice(kwargs.get('stream_slice'))}"} + + +class AdCampaignAnalytics(LinkedInAdsAnalyticsStream): + """ + Campaign Analytics stream. + """ + + endpoint = "adAnalytics" + + parent_stream = Campaigns + parent_values_map = {"campaign_id": "id"} + search_param = "campaigns" + search_param_value = "sponsoredCampaign" + pivot_by = "CAMPAIGN" + time_granularity = "DAILY" + + +class AdCreativeAnalytics(LinkedInAdsAnalyticsStream): + """ + Creative Analytics stream. + """ + + parent_stream = Creatives + parent_values_map = {"creative_id": "id"} + search_param = "creatives" + search_param_value = "sponsoredCreative" + pivot_by = "CREATIVE" + time_granularity = "DAILY" + + def get_primary_key_from_slice(self, stream_slice) -> str: + creative_id = stream_slice.get(self.primary_slice_key).split(":")[-1] + return creative_id + + +class AdImpressionDeviceAnalytics(AdCampaignAnalytics): + pivot_by = "IMPRESSION_DEVICE_TYPE" + + +class AdMemberCompanySizeAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_COMPANY_SIZE" + + +class AdMemberIndustryAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_INDUSTRY" + + +class AdMemberSeniorityAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_SENIORITY" + + +class AdMemberJobTitleAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_JOB_TITLE" + + +class AdMemberJobFunctionAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_JOB_FUNCTION" + + +class AdMemberCountryAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_COUNTRY_V2" + + +class AdMemberRegionAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_REGION_V2" + + +class AdMemberCompanyAnalytics(AdCampaignAnalytics): + pivot_by = "MEMBER_COMPANY" diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/run.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/run.py new file mode 100644 index 000000000000..e37dbe66f17f --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_linkedin_ads import SourceLinkedinAds + + +def run(): + source = SourceLinkedinAds() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py index 8a89a468eaab..4716fe245093 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py @@ -11,9 +11,7 @@ from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator, TokenAuthenticator from airbyte_cdk.utils import AirbyteTracedException from airbyte_protocol.models import FailureType -from source_linkedin_ads.streams import ( - Accounts, - AccountUsers, +from source_linkedin_ads.analytics_streams import ( AdCampaignAnalytics, AdCreativeAnalytics, AdImpressionDeviceAnalytics, @@ -25,11 +23,8 @@ AdMemberJobTitleAnalytics, AdMemberRegionAnalytics, AdMemberSeniorityAnalytics, - CampaignGroups, - Campaigns, - Conversions, - Creatives, ) +from source_linkedin_ads.streams import Accounts, AccountUsers, CampaignGroups, Campaigns, Conversions, Creatives logger = logging.getLogger("airbyte") diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/spec.json b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/spec.json index e130c0a073de..8bfe86125ad1 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/spec.json +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/spec.json @@ -111,9 +111,9 @@ "MEMBER_COMPANY_SIZE", "MEMBER_INDUSTRY", "MEMBER_SENIORITY", - "MEMBER_JOB_TITLE ", - "MEMBER_JOB_FUNCTION ", - "MEMBER_COUNTRY_V2 ", + "MEMBER_JOB_TITLE", + "MEMBER_JOB_FUNCTION", + "MEMBER_COUNTRY_V2", "MEMBER_REGION_V2", "MEMBER_COMPANY", "PLACEMENT_NAME", diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py index 08b0418ab808..5151d52a961d 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/streams.py @@ -10,13 +10,9 @@ import pendulum import requests -from airbyte_cdk.sources.streams.core import package_name_from_class from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.utils import casing -from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from .analytics import make_analytics_slices, merge_chunks, update_analytics_params from .utils import get_parent_stream_values, transform_data logger = logging.getLogger("airbyte") @@ -32,7 +28,6 @@ class LinkedinAdsStream(HttpStream, ABC): url_base = "https://api.linkedin.com/rest/" primary_key = "id" records_limit = 500 - endpoint = None transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) def __init__(self, config: Dict): @@ -52,6 +47,11 @@ def accounts(self): """Property to return the list of the user Account Ids from input""" return ",".join(map(str, self.config.get("account_ids", []))) + @property + @abstractmethod + def endpoint(self) -> str: + """Endpoint associated with the current stream""" + def path( self, *, @@ -92,7 +92,7 @@ def request_params( def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: """ - We need to get out the nested complex data structures for further normalisation, so the transform_data method is applied. + We need to get out the nested complex data structures for further normalization, so the transform_data method is applied. """ for record in transform_data(response.json().get("elements")): yield self._date_time_to_rfc3339(record) @@ -126,6 +126,7 @@ class Accounts(LinkedinAdsStream): """ endpoint = "adAccounts" + use_cache = True def request_headers(self, stream_state: Mapping[str, Any], **kwargs) -> Mapping[str, Any]: """ @@ -169,12 +170,12 @@ def primary_slice_key(self) -> str: @property @abstractmethod - def parent_stream(self) -> object: - """Defines the parrent stream for slicing, the class object should be provided.""" + def parent_stream(self) -> LinkedinAdsStream: + """Defines the parent stream for slicing, the class object should be provided.""" @property def state_checkpoint_interval(self) -> Optional[int]: - """Define the checkpoint from the records output size.""" + """Define the checkpoint from the record output size.""" return 100 def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: @@ -182,11 +183,11 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late return {self.cursor_field: max(latest_record.get(self.cursor_field), current_stream_state.get(self.cursor_field))} -class LinkedInAdsStreamSlicing(IncrementalLinkedinAdsStream): +class LinkedInAdsStreamSlicing(IncrementalLinkedinAdsStream, ABC): """ This class stands for provide stream slicing for other dependent streams. :: `parent_stream` - the reference to the parent stream class, - by default it's referenced to the Accounts stream class, as far as majority of streams are using it. + by default it's referenced to the Accounts stream class, as far as a majority of streams are using it. :: `parent_values_map` - key_value map for stream slices in a format: {: } :: `search_param` - the query param to pass with request_params """ @@ -315,7 +316,7 @@ class Creatives(LinkedInAdsStreamSlicing): endpoint = "creatives" parent_stream = Accounts cursor_field = "lastModifiedAt" - # standard records_limit=500 returns error 400: Request would return too many entities; https://github.com/airbytehq/oncall/issues/2159 + # standard records_limit=500 returns error 400: Request would return too many entities; https://github.com/airbytehq/oncall/issues/2159 records_limit = 100 def path( @@ -388,146 +389,3 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late else current_stream_state ) return {self.cursor_field: max(latest_record.get(self.cursor_field), int(current_stream_state.get(self.cursor_field)))} - - -class LinkedInAdsAnalyticsStream(IncrementalLinkedinAdsStream, ABC): - """ - AdAnalytics Streams more info: - https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/ads-reporting?tabs=curl&view=li-lms-2023-05#analytics-finder - """ - - endpoint = "adAnalytics" - # For Analytics streams the primary_key is the entity of the pivot [Campaign URN, Creative URN, etc] + `end_date` - primary_key = ["pivotValue", "end_date"] - cursor_field = "end_date" - - def get_json_schema(self) -> Mapping[str, Any]: - return ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("ad_analytics") - - def __init__(self, name: str = None, pivot_by: str = None, time_granularity: str = None, **kwargs): - self.user_stream_name = name - if pivot_by: - self.pivot_by = pivot_by - if time_granularity: - self.time_granularity = time_granularity - super().__init__(**kwargs) - - @property - def name(self) -> str: - """We override stream name to let the user change it via configuration.""" - name = self.user_stream_name or self.__class__.__name__ - return casing.camel_to_snake(name) - - @property - def base_analytics_params(self) -> MutableMapping[str, Any]: - """Define the base parameters for analytics streams""" - return {"q": "analytics", "pivot": f"(value:{self.pivot_by})", "timeGranularity": f"(value:{self.time_granularity})"} - - def request_headers( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> Mapping[str, Any]: - headers = super().request_headers(stream_state, stream_slice, next_page_token) - return headers | {"X-Restli-Protocol-Version": "2.0.0"} - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - params = self.base_analytics_params - params.update(**update_analytics_params(stream_slice)) - params[self.search_param] = f"List(urn%3Ali%3A{self.search_param_value}%3A{self.get_primary_key_from_slice(stream_slice)})" - return urlencode(params, safe="():,%") - - def get_primary_key_from_slice(self, stream_slice) -> str: - return stream_slice.get(self.primary_slice_key) - - def read_records( - self, stream_state: Mapping[str, Any] = None, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs - ) -> Iterable[Mapping[str, Any]]: - stream_state = stream_state or {self.cursor_field: self.config.get("start_date")} - parent_stream = self.parent_stream(config=self.config) - for record in parent_stream.read_records(**kwargs): - result_chunks = [] - for analytics_slice in make_analytics_slices( - record, self.parent_values_map, stream_state.get(self.cursor_field), self.config.get("end_date") - ): - child_stream_slice = super().read_records(stream_slice=analytics_slice, **kwargs) - result_chunks.append(child_stream_slice) - yield from merge_chunks(result_chunks, self.cursor_field) - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - We need to get out the nested complex data structures for further normalisation, so the transform_data method is applied. - """ - for rec in transform_data(response.json().get("elements")): - yield rec | {"pivotValue": f"urn:li:{self.search_param_value}:{self.get_primary_key_from_slice(kwargs.get('stream_slice'))}"} - - -class AdCampaignAnalytics(LinkedInAdsAnalyticsStream): - """ - Campaign Analytics stream. - """ - - endpoint = "adAnalytics" - - parent_stream = Campaigns - parent_values_map = {"campaign_id": "id"} - search_param = "campaigns" - search_param_value = "sponsoredCampaign" - pivot_by = "CAMPAIGN" - time_granularity = "DAILY" - - -class AdCreativeAnalytics(LinkedInAdsAnalyticsStream): - """ - Creative Analytics stream. - """ - - parent_stream = Creatives - parent_values_map = {"creative_id": "id"} - search_param = "creatives" - search_param_value = "sponsoredCreative" - pivot_by = "CREATIVE" - time_granularity = "DAILY" - - def get_primary_key_from_slice(self, stream_slice) -> str: - creative_id = stream_slice.get(self.primary_slice_key).split(":")[-1] - return creative_id - - -class AdImpressionDeviceAnalytics(AdCampaignAnalytics): - pivot_by = "IMPRESSION_DEVICE_TYPE" - - -class AdMemberCompanySizeAnalytics(AdCampaignAnalytics): - pivot_by = "MEMBER_COMPANY_SIZE" - - -class AdMemberIndustryAnalytics(AdCampaignAnalytics): - pivot_by = "MEMBER_INDUSTRY" - - -class AdMemberSeniorityAnalytics(AdCampaignAnalytics): - pivot_by = "MEMBER_SENIORITY" - - -class AdMemberJobTitleAnalytics(AdCampaignAnalytics): - pivot_by = "MEMBER_JOB_TITLE" - - -class AdMemberJobFunctionAnalytics(AdCampaignAnalytics): - pivot_by = "MEMBER_JOB_FUNCTION" - - -class AdMemberCountryAnalytics(AdCampaignAnalytics): - pivot_by = "MEMBER_COUNTRY_V2" - - -class AdMemberRegionAnalytics(AdCampaignAnalytics): - pivot_by = "MEMBER_REGION_V2" - - -class AdMemberCompanyAnalytics(AdCampaignAnalytics): - pivot_by = "MEMBER_COMPANY" diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py index f000f88f343a..9872ea0055b7 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/utils.py @@ -14,18 +14,11 @@ DESTINATION_RESERVED_KEYWORDS: list = ["pivot"] -def get_parent_stream_values(record: Dict, key_value_map: Dict) -> Dict: +def get_parent_stream_values(record: Mapping[str, Any], key_value_map: Mapping[str, str]) -> Mapping[str, Any]: """ - Outputs the Dict with key:value slices for the stream. - :: EXAMPLE: - Input: - records = [{dict}, {dict}, ...], - key_value_map = {: } - - Output: - { - : records..value, - } + :param record: Mapping[str, Any] + :param key_value_map: Mapping[str, str] {: } + :return: Mapping[str, str] { : records..value} """ result = {} for key in key_value_map: diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/__init__.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/samples/test_data_for_analytics.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/samples/test_data_for_analytics.py deleted file mode 100644 index 3f6e207a7376..000000000000 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/samples/test_data_for_analytics.py +++ /dev/null @@ -1,239 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from typing import Dict, List - -""" -This is the example of input record for the test_make_analytics_slices. -""" -test_input_record: Dict = { - "id": 123, - "audienceExpansionEnabled": True, - "test": False, - "format": "STANDARD_UPDATE", - "servingStatuses": ["CAMPAIGN_GROUP_TOTAL_BUDGET_HOLD"], - "version": {"versionTag": "2"}, - "objectiveType": "TEST_TEST", - "associatedEntity": "urn:li:organization:456", - "offsitePreferences": { - "iabCategories": {"exclude": []}, - "publisherRestrictionFiles": {"exclude": []}, - }, - "campaignGroup": "urn:li:sponsoredCampaignGroup:1234567", - "account": "urn:li:sponsoredAccount:123456", - "status": "ACTIVE", - "created": "2021-08-06 06:03:52", - "lastModified": "2021-08-06 06:09:04", -} - -""" -This is the expected output from the `make_analytics_slices` method. -VALID PARAMETERS FOR THE OUTPUT ARE: -: TEST_KEY_VALUE_MAP = {"campaign_id": "id"} -: TEST_START_DATE = "2021-08-01" -: TEST_END_DATE = "2021-09-30" - -Change the input parameters inside of test_make_analytics_slices.py unit test. -Make sure for valid KEY_VALUE_MAP references inside of the `test_input_record` -""" -test_output_slices: List = [ - { - "camp_id": 123, - "fields": "actionClicks,adUnitClicks,approximateUniqueImpressions,cardClicks,cardImpressions,clicks,commentLikes,comments,companyPageClicks,conversionValueInLocalCurrency,costInLocalCurrency,costInUsd,dateRange,documentCompletions,documentFirstQuartileCompletions,documentMidpointCompletions,documentThirdQuartileCompletions,downloadClicks,externalWebsiteConversions", - "dateRange": { - "start.day": 1, - "start.month": 8, - "start.year": 2021, - "end.day": 31, - "end.month": 8, - "end.year": 2021, - }, - }, - { - "camp_id": 123, - "fields": "actionClicks,adUnitClicks,approximateUniqueImpressions,cardClicks,cardImpressions,clicks,commentLikes,comments,companyPageClicks,conversionValueInLocalCurrency,costInLocalCurrency,costInUsd,dateRange,documentCompletions,documentFirstQuartileCompletions,documentMidpointCompletions,documentThirdQuartileCompletions,downloadClicks,externalWebsiteConversions", - "dateRange": { - "start.day": 31, - "start.month": 8, - "start.year": 2021, - "end.day": 30, - "end.month": 9, - "end.year": 2021 - } - }, - {"camp_id": 123, - "fields": "externalWebsitePostClickConversions,externalWebsitePostViewConversions,follows,fullScreenPlays,impressions,jobApplications,jobApplyClicks,landingPageClicks,leadGenerationMailContactInfoShares,leadGenerationMailInterestedClicks,likes,oneClickLeadFormOpens,oneClickLeads,opens,otherEngagements,pivotValues,postClickJobApplications,postClickJobApplyClicks,postClickRegistrations,dateRange", - "dateRange": { - "start.day": 1, - "start.month": 8, - "start.year": 2021, - "end.day": 31, - "end.month": 8, - "end.year": 2021 - } - }, - {"camp_id": 123, - "fields": "externalWebsitePostClickConversions,externalWebsitePostViewConversions,follows,fullScreenPlays,impressions,jobApplications,jobApplyClicks,landingPageClicks,leadGenerationMailContactInfoShares,leadGenerationMailInterestedClicks,likes,oneClickLeadFormOpens,oneClickLeads,opens,otherEngagements,pivotValues,postClickJobApplications,postClickJobApplyClicks,postClickRegistrations,dateRange", - "dateRange": { - "start.day": 31, - "start.month": 8, - "start.year": 2021, - "end.day": 30, - "end.month": 9, - "end.year": 2021 - } - }, - { - "camp_id": 123, - "fields": "postViewJobApplications,postViewJobApplyClicks,postViewRegistrations,reactions,registrations,sends,shares,talentLeads,textUrlClicks,totalEngagements,validWorkEmailLeads,videoCompletions,videoFirstQuartileCompletions,videoMidpointCompletions,videoStarts,videoThirdQuartileCompletions,videoViews,viralCardClicks,viralCardImpressions,dateRange", - "dateRange": { - "start.day": 1, - "start.month": 8, - "start.year": 2021, - "end.day": 31, - "end.month": 8, - "end.year": 2021 - } - }, - { - "camp_id": 123, - "fields": "postViewJobApplications,postViewJobApplyClicks,postViewRegistrations,reactions,registrations,sends,shares,talentLeads,textUrlClicks,totalEngagements,validWorkEmailLeads,videoCompletions,videoFirstQuartileCompletions,videoMidpointCompletions,videoStarts,videoThirdQuartileCompletions,videoViews,viralCardClicks,viralCardImpressions,dateRange", - "dateRange": { - "start.day": 31, - "start.month": 8, - "start.year": 2021, - "end.day": 30, - "end.month": 9, - "end.year": 2021 - } - }, - { - "camp_id": 123, - "fields": "viralClicks,viralCommentLikes,viralComments,viralCompanyPageClicks,viralDocumentCompletions,viralDocumentFirstQuartileCompletions,viralDocumentMidpointCompletions,viralDocumentThirdQuartileCompletions,viralDownloadClicks,viralExternalWebsiteConversions,viralExternalWebsitePostClickConversions,viralExternalWebsitePostViewConversions,viralFollows,viralFullScreenPlays,viralImpressions,viralJobApplications,viralJobApplyClicks,viralLandingPageClicks,viralLikes,dateRange", - "dateRange": { - "start.day": 1, - "start.month": 8, - "start.year": 2021, - "end.day": 31, - "end.month": 8, - "end.year": 2021 - } - }, - { - "camp_id": 123, - "fields": "viralClicks,viralCommentLikes,viralComments,viralCompanyPageClicks,viralDocumentCompletions,viralDocumentFirstQuartileCompletions,viralDocumentMidpointCompletions,viralDocumentThirdQuartileCompletions,viralDownloadClicks,viralExternalWebsiteConversions,viralExternalWebsitePostClickConversions,viralExternalWebsitePostViewConversions,viralFollows,viralFullScreenPlays,viralImpressions,viralJobApplications,viralJobApplyClicks,viralLandingPageClicks,viralLikes,dateRange", - "dateRange": { - "start.day": 31, - "start.month": 8, - "start.year": 2021, - "end.day": 30, - "end.month": 9, - "end.year": 2021 - } - }, - { - "camp_id": 123, - "fields": "viralOneClickLeadFormOpens,viralOneClickLeads,viralOtherEngagements,viralPostClickJobApplications,viralPostClickJobApplyClicks,viralPostClickRegistrations,viralPostViewJobApplications,viralPostViewJobApplyClicks,viralPostViewRegistrations,viralReactions,viralRegistrations,viralShares,viralTotalEngagements,viralVideoCompletions,viralVideoFirstQuartileCompletions,viralVideoMidpointCompletions,viralVideoStarts,viralVideoThirdQuartileCompletions,viralVideoViews,dateRange", - "dateRange": { - "start.day": 1, - "start.month": 8, - "start.year": 2021, - "end.day": 31, - "end.month": 8, - "end.year": 2021 - } - }, - { - "camp_id": 123, - "fields": "viralOneClickLeadFormOpens,viralOneClickLeads,viralOtherEngagements,viralPostClickJobApplications,viralPostClickJobApplyClicks,viralPostClickRegistrations,viralPostViewJobApplications,viralPostViewJobApplyClicks,viralPostViewRegistrations,viralReactions,viralRegistrations,viralShares,viralTotalEngagements,viralVideoCompletions,viralVideoFirstQuartileCompletions,viralVideoMidpointCompletions,viralVideoStarts,viralVideoThirdQuartileCompletions,viralVideoViews,dateRange", - "dateRange": { - "start.day": 31, - "start.month": 8, - "start.year": 2021, - "end.day": 30, - "end.month": 9, - "end.year": 2021 - } - } -] - -""" This is the example of the input chunks for the `test_merge_chunks` """ -test_input_result_record_chunks = [ - [ - { - "field_1": "test1", - "start_date": "2021-08-06", - "end_date": "2021-08-06", - }, - { - "field_1": "test2", - "start_date": "2021-08-07", - "end_date": "2021-08-07", - }, - { - "field_1": "test3", - "start_date": "2021-08-08", - "end_date": "2021-08-08", - }, - ], - [ - { - "field_2": "test1", - "start_date": "2021-08-06", - "end_date": "2021-08-06", - }, - { - "field_2": "test2", - "start_date": "2021-08-07", - "end_date": "2021-08-07", - }, - { - "field_2": "test3", - "start_date": "2021-08-08", - "end_date": "2021-08-08", - }, - ], - [ - { - "field_3": "test1", - "start_date": "2021-08-06", - "end_date": "2021-08-06", - }, - { - "field_3": "test2", - "start_date": "2021-08-07", - "end_date": "2021-08-07", - }, - { - "field_3": "test3", - "start_date": "2021-08-08", - "end_date": "2021-08-08", - }, - ], -] - -""" This is the expected test ouptput from the `merge_chunks` method from analytics module """ -test_output_merged_chunks = [ - { - "field_1": "test1", - "start_date": "2021-08-06", - "end_date": "2021-08-06", - "field_2": "test1", - "field_3": "test1", - }, - { - "field_1": "test2", - "start_date": "2021-08-07", - "end_date": "2021-08-07", - "field_2": "test2", - "field_3": "test2", - }, - { - "field_1": "test3", - "start_date": "2021-08-08", - "end_date": "2021-08-08", - "field_2": "test3", - "field_3": "test3", - }, -] diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_chunk_analytics_fields.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_chunk_analytics_fields.py deleted file mode 100644 index c360c249159f..000000000000 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_chunk_analytics_fields.py +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from source_linkedin_ads.analytics import chunk_analytics_fields - -# Test chunk size for each field set -TEST_FIELDS_CHUNK_SIZE = 3 -# Test fields assuming they are really available for the fetch -TEST_ANALYTICS_FIELDS = [ - "field_1", - "base_field_1", - "field_2", - "base_field_2", - "field_3", - "field_4", - "field_5", - "field_6", - "field_7", - "field_8", -] -# Fields that are always present in fields_set chunks -TEST_BASE_ANALLYTICS_FIELDS = ["base_field_1", "base_field_2"] - - -def test_chunk_analytics_fields(): - """ - We expect to truncate the fields list into the chunks of equal size, - with TEST_BASE_ANALLYTICS_FIELDS presence in each chunk, - order is not matter. - """ - expected_output = [ - ["field_1", "base_field_1", "field_2", "base_field_2"], - ["base_field_2", "field_3", "field_4", "base_field_1"], - ["field_5", "field_6", "field_7", "base_field_1", "base_field_2"], - ["field_8", "base_field_1", "base_field_2"], - ] - - assert list(chunk_analytics_fields(TEST_ANALYTICS_FIELDS, TEST_BASE_ANALLYTICS_FIELDS, TEST_FIELDS_CHUNK_SIZE)) == expected_output diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_make_analytics_slices.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_make_analytics_slices.py deleted file mode 100644 index f2579852d87c..000000000000 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_make_analytics_slices.py +++ /dev/null @@ -1,18 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from samples.test_data_for_analytics import test_input_record, test_output_slices -from source_linkedin_ads.analytics import make_analytics_slices - -# Test input arguments for the `make_analytics_slices` -TEST_KEY_VALUE_MAP = {"camp_id": "id"} -TEST_START_DATE = "2021-08-01" -TEST_END_DATE = "2021-09-30" - -# This is the mock of the request_params -TEST_REQUEST_PRAMS = {} - - -def test_make_analytics_slices(): - assert list(make_analytics_slices(test_input_record, TEST_KEY_VALUE_MAP, TEST_START_DATE, TEST_END_DATE)) == test_output_slices diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_make_date_slices.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_make_date_slices.py deleted file mode 100644 index ec5e5c9d5d85..000000000000 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_make_date_slices.py +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from source_linkedin_ads.analytics import make_date_slices - -TEST_START_DATE = "2021-08-01" -TEST_END_DATE = "2021-10-01" - - -def test_make_date_slices(): - """ - : By default we use the `WINDOW_SIZE = 30`, as it set in the analytics module - : This value could be changed by setting the corresponding argument in the method. - : The `end_date` is not specified by default, but for this test it was specified to have the test static. - """ - - expected_output = [ - {"dateRange": {"start.day": 1, "start.month": 8, "start.year": 2021, "end.day": 31, "end.month": 8, "end.year": 2021}}, - {"dateRange": {"start.day": 31, "start.month": 8, "start.year": 2021, "end.day": 30, "end.month": 9, "end.year": 2021}}, - {"dateRange": {"start.day": 30, "start.month": 9, "start.year": 2021, "end.day": 30, "end.month": 10, "end.year": 2021}}, - ] - - assert list(make_date_slices(TEST_START_DATE, TEST_END_DATE)) == expected_output diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_merge_chunks.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_merge_chunks.py deleted file mode 100644 index 65036b99d06a..000000000000 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/analytics_tests/test_merge_chunks.py +++ /dev/null @@ -1,13 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from samples.test_data_for_analytics import test_input_result_record_chunks, test_output_merged_chunks -from source_linkedin_ads.analytics import merge_chunks - -TEST_MERGE_BY_KEY = "end_date" - - -def test_merge_chunks(): - """`merge_chunks` is the generator object, to get the output the list() function is applied""" - assert list(merge_chunks(test_input_result_record_chunks, TEST_MERGE_BY_KEY)) == test_output_merged_chunks diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py new file mode 100644 index 000000000000..c3d9c1c98188 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/conftest.py @@ -0,0 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import os + +os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/output_slices.json b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/output_slices.json new file mode 100644 index 000000000000..edab65234171 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/output_slices.json @@ -0,0 +1,126 @@ +[ + [ + { + "campaign_id": 123, + "dateRange": { + "end.day": 31, + "end.month": 1, + "end.year": 2021, + "start.day": 1, + "start.month": 1, + "start.year": 2021 + }, + "fields": "actionClicks,adUnitClicks,approximateUniqueImpressions,cardClicks,cardImpressions,clicks,commentLikes,comments,companyPageClicks,conversionValueInLocalCurrency,costInLocalCurrency,costInUsd,dateRange,documentCompletions,documentFirstQuartileCompletions,documentMidpointCompletions,documentThirdQuartileCompletions,downloadClicks,externalWebsiteConversions" + }, + { + "campaign_id": 123, + "dateRange": { + "end.day": 31, + "end.month": 1, + "end.year": 2021, + "start.day": 1, + "start.month": 1, + "start.year": 2021 + }, + "fields": "externalWebsitePostClickConversions,externalWebsitePostViewConversions,follows,fullScreenPlays,impressions,jobApplications,jobApplyClicks,landingPageClicks,leadGenerationMailContactInfoShares,leadGenerationMailInterestedClicks,likes,oneClickLeadFormOpens,oneClickLeads,opens,otherEngagements,pivotValues,postClickJobApplications,postClickJobApplyClicks,postClickRegistrations,dateRange" + }, + { + "campaign_id": 123, + "dateRange": { + "end.day": 31, + "end.month": 1, + "end.year": 2021, + "start.day": 1, + "start.month": 1, + "start.year": 2021 + }, + "fields": "postViewJobApplications,postViewJobApplyClicks,postViewRegistrations,reactions,registrations,sends,shares,talentLeads,textUrlClicks,totalEngagements,validWorkEmailLeads,videoCompletions,videoFirstQuartileCompletions,videoMidpointCompletions,videoStarts,videoThirdQuartileCompletions,videoViews,viralCardClicks,viralCardImpressions,dateRange" + }, + { + "campaign_id": 123, + "dateRange": { + "end.day": 31, + "end.month": 1, + "end.year": 2021, + "start.day": 1, + "start.month": 1, + "start.year": 2021 + }, + "fields": "viralClicks,viralCommentLikes,viralComments,viralCompanyPageClicks,viralDocumentCompletions,viralDocumentFirstQuartileCompletions,viralDocumentMidpointCompletions,viralDocumentThirdQuartileCompletions,viralDownloadClicks,viralExternalWebsiteConversions,viralExternalWebsitePostClickConversions,viralExternalWebsitePostViewConversions,viralFollows,viralFullScreenPlays,viralImpressions,viralJobApplications,viralJobApplyClicks,viralLandingPageClicks,viralLikes,dateRange" + }, + { + "campaign_id": 123, + "dateRange": { + "end.day": 31, + "end.month": 1, + "end.year": 2021, + "start.day": 1, + "start.month": 1, + "start.year": 2021 + }, + "fields": "viralOneClickLeadFormOpens,viralOneClickLeads,viralOtherEngagements,viralPostClickJobApplications,viralPostClickJobApplyClicks,viralPostClickRegistrations,viralPostViewJobApplications,viralPostViewJobApplyClicks,viralPostViewRegistrations,viralReactions,viralRegistrations,viralShares,viralTotalEngagements,viralVideoCompletions,viralVideoFirstQuartileCompletions,viralVideoMidpointCompletions,viralVideoStarts,viralVideoThirdQuartileCompletions,viralVideoViews,dateRange" + } + ], + [ + { + "campaign_id": 123, + "dateRange": { + "end.day": 2, + "end.month": 3, + "end.year": 2021, + "start.day": 31, + "start.month": 1, + "start.year": 2021 + }, + "fields": "actionClicks,adUnitClicks,approximateUniqueImpressions,cardClicks,cardImpressions,clicks,commentLikes,comments,companyPageClicks,conversionValueInLocalCurrency,costInLocalCurrency,costInUsd,dateRange,documentCompletions,documentFirstQuartileCompletions,documentMidpointCompletions,documentThirdQuartileCompletions,downloadClicks,externalWebsiteConversions" + }, + { + "campaign_id": 123, + "dateRange": { + "end.day": 2, + "end.month": 3, + "end.year": 2021, + "start.day": 31, + "start.month": 1, + "start.year": 2021 + }, + "fields": "externalWebsitePostClickConversions,externalWebsitePostViewConversions,follows,fullScreenPlays,impressions,jobApplications,jobApplyClicks,landingPageClicks,leadGenerationMailContactInfoShares,leadGenerationMailInterestedClicks,likes,oneClickLeadFormOpens,oneClickLeads,opens,otherEngagements,pivotValues,postClickJobApplications,postClickJobApplyClicks,postClickRegistrations,dateRange" + }, + { + "campaign_id": 123, + "dateRange": { + "end.day": 2, + "end.month": 3, + "end.year": 2021, + "start.day": 31, + "start.month": 1, + "start.year": 2021 + }, + "fields": "postViewJobApplications,postViewJobApplyClicks,postViewRegistrations,reactions,registrations,sends,shares,talentLeads,textUrlClicks,totalEngagements,validWorkEmailLeads,videoCompletions,videoFirstQuartileCompletions,videoMidpointCompletions,videoStarts,videoThirdQuartileCompletions,videoViews,viralCardClicks,viralCardImpressions,dateRange" + }, + { + "campaign_id": 123, + "dateRange": { + "end.day": 2, + "end.month": 3, + "end.year": 2021, + "start.day": 31, + "start.month": 1, + "start.year": 2021 + }, + "fields": "viralClicks,viralCommentLikes,viralComments,viralCompanyPageClicks,viralDocumentCompletions,viralDocumentFirstQuartileCompletions,viralDocumentMidpointCompletions,viralDocumentThirdQuartileCompletions,viralDownloadClicks,viralExternalWebsiteConversions,viralExternalWebsitePostClickConversions,viralExternalWebsitePostViewConversions,viralFollows,viralFullScreenPlays,viralImpressions,viralJobApplications,viralJobApplyClicks,viralLandingPageClicks,viralLikes,dateRange" + }, + { + "campaign_id": 123, + "dateRange": { + "end.day": 2, + "end.month": 3, + "end.year": 2021, + "start.day": 31, + "start.month": 1, + "start.year": 2021 + }, + "fields": "viralOneClickLeadFormOpens,viralOneClickLeads,viralOtherEngagements,viralPostClickJobApplications,viralPostClickJobApplyClicks,viralPostClickRegistrations,viralPostViewJobApplications,viralPostViewJobApplyClicks,viralPostViewRegistrations,viralReactions,viralRegistrations,viralShares,viralTotalEngagements,viralVideoCompletions,viralVideoFirstQuartileCompletions,viralVideoMidpointCompletions,viralVideoStarts,viralVideoThirdQuartileCompletions,viralVideoViews,dateRange" + } + ] +] diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/responses/ad_member_country_analytics/response_1.json b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/responses/ad_member_country_analytics/response_1.json new file mode 100644 index 000000000000..a68474329ce0 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/responses/ad_member_country_analytics/response_1.json @@ -0,0 +1,71 @@ +{ + "paging": { + "start": 0, + "count": 10, + "links": [] + }, + "elements": [ + { + "documentFirstQuartileCompletions": 0, + "actionClicks": 0, + "comments": 0, + "costInUsd": "-2E-18", + "dateRange": { + "start": { + "month": 1, + "day": 2, + "year": 2023 + }, + "end": { + "month": 1, + "day": 2, + "year": 2023 + } + }, + "commentLikes": 0, + "adUnitClicks": 0, + "companyPageClicks": 0, + "costInLocalCurrency": "-2E-18", + "documentThirdQuartileCompletions": 0, + "externalWebsiteConversions": 0, + "cardImpressions": 0, + "documentCompletions": 0, + "clicks": 0, + "cardClicks": 0, + "approximateUniqueImpressions": 0, + "documentMidpointCompletions": 0, + "downloadClicks": 0 + }, + { + "documentFirstQuartileCompletions": 0, + "actionClicks": 0, + "comments": 0, + "costInUsd": "100", + "dateRange": { + "start": { + "month": 1, + "day": 2, + "year": 2023 + }, + "end": { + "month": 1, + "day": 2, + "year": 2023 + } + }, + "commentLikes": 0, + "adUnitClicks": 0, + "companyPageClicks": 0, + "costInLocalCurrency": "100", + "documentThirdQuartileCompletions": 0, + "externalWebsiteConversions": 0, + "cardImpressions": 0, + "documentCompletions": 0, + "clicks": 106, + "cardClicks": 0, + "approximateUniqueImpressions": 17392, + "documentMidpointCompletions": 0, + "downloadClicks": 0 + } + ] +} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/responses/ad_member_country_analytics/response_2.json b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/responses/ad_member_country_analytics/response_2.json new file mode 100644 index 000000000000..ac6433682403 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/responses/ad_member_country_analytics/response_2.json @@ -0,0 +1,61 @@ +{ + "paging": { + "start": 0, + "count": 10, + "links": [] + }, + "elements": [ + { + "oneClickLeads": 0, + "dateRange": { + "start": { + "month": 1, + "day": 2, + "year": 2021 + }, + "end": { + "month": 1, + "day": 2, + "year": 2023 + } + }, + "landingPageClicks": 0, + "fullScreenPlays": 0, + "oneClickLeadFormOpens": 0, + "follows": 0, + "impressions": 1, + "otherEngagements": 0, + "leadGenerationMailContactInfoShares": 0, + "opens": 0, + "leadGenerationMailInterestedClicks": 0, + "pivotValues": ["urn:li:sponsoredCreative:1"], + "likes": 0 + }, + { + "oneClickLeads": 0, + "dateRange": { + "start": { + "month": 1, + "day": 1, + "year": 2021 + }, + "end": { + "month": 1, + "day": 1, + "year": 2021 + } + }, + "landingPageClicks": 106, + "fullScreenPlays": 0, + "oneClickLeadFormOpens": 0, + "follows": 0, + "impressions": 19464, + "otherEngagements": 0, + "leadGenerationMailContactInfoShares": 0, + "opens": 0, + "leadGenerationMailInterestedClicks": 0, + "pivotValues": ["urn:li:sponsoredCreative:1"], + "likes": 0 + } + ] +} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/responses/ad_member_country_analytics/response_3.json b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/responses/ad_member_country_analytics/response_3.json new file mode 100644 index 000000000000..5e128840ccb4 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/responses/ad_member_country_analytics/response_3.json @@ -0,0 +1,37 @@ +{ + "paging": { + "start": 0, + "count": 10, + "links": [] + }, + "elements": [ + { + "videoCompletions": 0, + "dateRange": { + "start": { + "month": 1, + "day": 2, + "year": 2023 + }, + "end": { + "month": 1, + "day": 2, + "year": 2023 + } + }, + "viralCardImpressions": 0, + "videoFirstQuartileCompletions": 0, + "textUrlClicks": 0, + "videoStarts": 0, + "sends": 0, + "shares": 0, + "videoMidpointCompletions": 0, + "validWorkEmailLeads": 0, + "viralCardClicks": 0, + "videoThirdQuartileCompletions": 0, + "totalEngagements": 105, + "reactions": 0, + "videoViews": 0 + } + ] +} diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/source_tests/test_source.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/source_tests/test_source.py deleted file mode 100644 index adda86e5f619..000000000000 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/source_tests/test_source.py +++ /dev/null @@ -1,342 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import pytest -import requests -from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator, TokenAuthenticator -from airbyte_cdk.utils import AirbyteTracedException -from source_linkedin_ads.source import ( - Accounts, - AccountUsers, - AdCampaignAnalytics, - AdCreativeAnalytics, - CampaignGroups, - Campaigns, - Creatives, - SourceLinkedinAds, -) -from source_linkedin_ads.streams import LINKEDIN_VERSION_API - -TEST_OAUTH_CONFIG: dict = { - "start_date": "2021-08-01", - "account_ids": [], - "credentials": { - "auth_method": "oAuth2.0", - "client_id": "client_id", - "client_secret": "client_secret", - "refresh_token": "refresh_token", - }, -} - -TEST_CONFIG: dict = { - "start_date": "2021-08-01", - "account_ids": [1, 2], - "credentials": { - "auth_method": "access_token", - "access_token": "access_token", - "authenticator": TokenAuthenticator(token="123"), - }, -} - -TEST_CONFIG_DUPLICATE_CUSTOM_AD_ANALYTICS_REPORTS: dict = { - "start_date": "2021-01-01", - "account_ids": [], - "credentials": { - "auth_method": "oAuth2.0", - "client_id": "client_id", - "client_secret": "client_secret", - "refresh_token": "refresh_token" - }, - "ad_analytics_reports": [ - { - "name": "ShareAdByMonth", - "pivot_by": "COMPANY", - "time_granularity": "MONTHLY" - }, - { - "name": "ShareAdByMonth", - "pivot_by": "COMPANY", - "time_granularity": "MONTHLY" - } - ] -} - - -class TestAllStreams: - _instance: SourceLinkedinAds = SourceLinkedinAds() - - @pytest.mark.parametrize( - "config", - [ - (TEST_CONFIG), - (TEST_OAUTH_CONFIG), - ], - ids=[ - "access_token", - "oauth2.0", - ], - ) - def test_get_authenticator(self, config: dict): - test = self._instance.get_authenticator(config) - assert isinstance(test, (Oauth2Authenticator, TokenAuthenticator)) - - @pytest.mark.parametrize( - "stream_cls", - [ - Accounts, - AccountUsers, - CampaignGroups, - Campaigns, - Creatives, - AdCampaignAnalytics, - AdCreativeAnalytics, - ], - ids=[ - "Accounts", - "AccountUsers", - "CampaignGroups", - "Campaigns", - "Creatives", - "AdCampaignAnalytics", - "AdCreativeAnalytics", - ], - ) - def test_streams(self, stream_cls): - streams = self._instance.streams(config=TEST_CONFIG) - for stream in streams: - if stream_cls in streams: - assert isinstance(stream, stream_cls) - - @pytest.mark.parametrize( - "stream_cls, stream_slice, expected", - [ - (Accounts, None, "adAccounts"), - (AccountUsers, None, "adAccountUsers"), - (CampaignGroups, {"account_id": 123}, "adAccounts/123/adCampaignGroups"), - (Campaigns, {"account_id": 123}, "adAccounts/123/adCampaigns"), - (Creatives, {"account_id": 123}, "adAccounts/123/creatives"), - (AdCampaignAnalytics, None, "adAnalytics"), - (AdCreativeAnalytics, None, "adAnalytics"), - ], - ids=[ - "Accounts", - "AccountUsers", - "CampaignGroups", - "Campaigns", - "Creatives", - "AdCampaignAnalytics", - "AdCreativeAnalytics", - ], - ) - def test_path(self, stream_cls, stream_slice, expected): - stream = stream_cls(config=TEST_CONFIG) - result = stream.path(stream_slice=stream_slice) - assert result == expected - - -class TestLinkedinAdsStream: - stream: Accounts = Accounts(TEST_CONFIG) - url = f"{stream.url_base}/{stream.path()}" - - def test_accounts(self): - result = self.stream.accounts - assert result == ",".join(map(str, TEST_CONFIG["account_ids"])) - - def test_next_page_token(self, requests_mock): - requests_mock.get(self.url, json={"elements": []}) - test_response = requests.get(self.url) - - expected = None - result = self.stream.next_page_token(test_response) - assert expected == result - - def test_request_params(self): - expected = "count=500&q=search&search=(id:(values:List(1,2)))" - result = self.stream.request_params(stream_state={}, stream_slice={"account_id": 123}) - assert expected == result - - def test_parse_response(self, requests_mock): - requests_mock.get(self.url, json={"elements": [{"test": "test"}]}) - test_response = requests.get(self.url) - - expected = {"test": "test"} - result = list(self.stream.parse_response(test_response)) - assert result[0] == expected - - def test_should_retry(self, requests_mock): - requests_mock.get(self.url, json={}, status_code=429) - test_response = requests.get(self.url) - result = self.stream.should_retry(test_response) - assert result is True - - def test_request_headers(self): - expected = {"X-RestLi-Protocol-Version": "2.0.0", "Linkedin-Version": LINKEDIN_VERSION_API} - result = self.stream.request_headers(stream_state={}) - assert result == expected - - -class TestAccountUsers: - stream: AccountUsers = AccountUsers(TEST_CONFIG) - - def test_state_checkpoint_interval(self): - assert self.stream.state_checkpoint_interval == 100 - - def test_get_updated_state(self): - state = self.stream.get_updated_state( - current_stream_state={"lastModified": "2021-01-01"}, latest_record={"lastModified": "2021-08-01"} - ) - assert state == {"lastModified": "2021-08-01"} - - -class TestLinkedInAdsStreamSlicing: - @pytest.mark.parametrize( - "stream_cls, slice, expected", - [ - ( - AccountUsers, - {"account_id": 123}, - "count=500&q=accounts&accounts=urn:li:sponsoredAccount:123", - ), - ( - CampaignGroups, - {"account_id": 123}, - "count=500&q=search&search=(status:(values:List(ACTIVE,ARCHIVED,CANCELED,DRAFT,PAUSED,PENDING_DELETION,REMOVED)))", - ), - ( - Campaigns, - {"account_id": 123}, - "count=500&q=search&search=(status:(values:List(ACTIVE,PAUSED,ARCHIVED,COMPLETED,CANCELED,DRAFT,PENDING_DELETION,REMOVED)))", - ), - ( - Creatives, - {"campaign_id": 123}, - "count=100&q=criteria", - ) - ], - ids=["AccountUsers", "CampaignGroups", "Campaigns", "Creatives"], - ) - def test_request_params(self, stream_cls, slice, expected): - stream = stream_cls(TEST_CONFIG) - result = stream.request_params(stream_state={}, stream_slice=slice) - assert expected == result - - @pytest.mark.parametrize( - "stream_cls, state, records_slice, expected", - [ - (AccountUsers, {"lastModified": 1}, [{"lastModified": 2}], [{"lastModified": 2}]), - (CampaignGroups, {"lastModified": 3}, [{"lastModified": 3}], [{"lastModified": 3}]), - (Campaigns, {}, [], []), - (Creatives, {}, [], []), - ], - ids=[ - "AccountUsers", - "CampaignGroups", - "Campaigns", - "Creatives", - ], - ) - def test_filter_records_newer_than_state(self, stream_cls, state, records_slice, expected): - stream = stream_cls(TEST_CONFIG) - result = list(stream.filter_records_newer_than_state(state, records_slice)) - assert result == expected - - -class TestLinkedInAdsAnalyticsStream: - @pytest.mark.parametrize( - "stream_cls, expected", - [ - (AdCampaignAnalytics, {"pivot": "(value:CAMPAIGN)", "q": "analytics", "timeGranularity": "(value:DAILY)"}), - (AdCreativeAnalytics, {"pivot": "(value:CREATIVE)", "q": "analytics", "timeGranularity": "(value:DAILY)"}), - ], - ids=[ - "AdCampaignAnalytics", - "AdCreativeAnalytics", - ], - ) - def test_base_analytics_params(self, stream_cls, expected): - stream = stream_cls(config=TEST_CONFIG) - result = stream.base_analytics_params - assert result == expected - - @pytest.mark.parametrize( - "stream_cls, slice, expected", - [ - ( - AdCampaignAnalytics, - { - "dateRange": {"start.day": 1, "start.month": 1, "start.year": 1, "end.day": 2, "end.month": 2, "end.year": 2}, - "fields": ["field1", "field2"], - }, - "q=analytics&pivot=(value:CAMPAIGN)&timeGranularity=(value:DAILY)&dateRange=(start:(year:1,month:1,day:1),end:(year:2,month:2,day:2))&fields=%5B%27field1%27,+%27field2%27%5D&campaigns=List(urn%3Ali%3AsponsoredCampaign%3ANone)", - ), - ( - AdCreativeAnalytics, - { - "dateRange": { - "start.day": 1, - "start.month": 1, - "start.year": 1, - "end.day": 2, - "end.month": 2, - "end.year": 2, - }, - "fields": [ - "field1", - "field2", - ], - "creative_id": "urn:li:sponsoredCreative:1234" - }, - "q=analytics&pivot=(value:CREATIVE)&timeGranularity=(value:DAILY)&dateRange=(start:(year:1,month:1,day:1),end:(year:2,month:2,day:2))&fields=%5B%27field1%27,+%27field2%27%5D&creatives=List(urn%3Ali%3AsponsoredCreative%3A1234)", - ), - ], - ids=[ - "AdCampaignAnalytics", - "AdCreativeAnalytics", - ], - ) - def test_request_params(self, stream_cls, slice, expected): - stream = stream_cls(config=TEST_CONFIG) - result = stream.request_params(stream_state={}, stream_slice=slice) - assert expected == result - - -def test_retry_get_access_token(requests_mock): - requests_mock.register_uri( - "POST", - "https://www.linkedin.com/oauth/v2/accessToken", - [{"status_code": 429}, {"status_code": 429}, {"status_code": 200, "json": {"access_token": "token", "expires_in": 3600}}], - ) - auth = Oauth2Authenticator( - token_refresh_endpoint="https://www.linkedin.com/oauth/v2/accessToken", - client_id="client_id", - client_secret="client_secret", - refresh_token="refresh_token", - ) - token = auth.get_access_token() - assert len(requests_mock.request_history) == 3 - assert token == "token" - - -@pytest.mark.parametrize( - "record, expected", - [ - ({}, {}), - ({"lastModified": "2021-05-27 11:59:53.710000"}, {"lastModified": "2021-05-27T11:59:53.710000+00:00"}), - ({"lastModified": None}, {"lastModified": None}), - ({"lastModified": ""}, {"lastModified": ""}), - ], - ids=["empty_record", "transformed_record", "null_value", "empty_value"], -) -def test_date_time_to_rfc3339(record, expected): - stream = Accounts(TEST_CONFIG) - result = stream._date_time_to_rfc3339(record) - assert result == expected - - -def test_duplicated_custom_ad_analytics_report(): - with pytest.raises(AirbyteTracedException) as e: - SourceLinkedinAds().streams(TEST_CONFIG_DUPLICATE_CUSTOM_AD_ANALYTICS_REPORTS) - expected_message = "Stream names for Custom Ad Analytics reports should be unique, duplicated streams: {'ShareAdByMonth'}" - assert e.value.message == expected_message diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_analytics_streams.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_analytics_streams.py new file mode 100644 index 000000000000..3936c7fad7e6 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_analytics_streams.py @@ -0,0 +1,109 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import json +import os +from typing import Any, Mapping + +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +from source_linkedin_ads.analytics_streams import AdMemberCountryAnalytics, LinkedInAdsAnalyticsStream + +# Test input arguments for the `make_analytics_slices` +TEST_KEY_VALUE_MAP = {"camp_id": "id"} +TEST_START_DATE = "2021-08-01" +TEST_END_DATE = "2021-09-30" + +# This is the mock of the request_params +TEST_REQUEST_PRAMS = {} + + +TEST_CONFIG: dict = { + "start_date": "2021-01-01", + "end_date": "2021-02-01", + "account_ids": [1, 2], + "credentials": { + "auth_method": "access_token", + "access_token": "access_token", + "authenticator": TokenAuthenticator(token="123"), + }, +} + +# Test chunk size for each field set +TEST_FIELDS_CHUNK_SIZE = 3 +# Test fields assuming they are really available for the fetch +TEST_ANALYTICS_FIELDS = [ + "field_1", + "base_field_1", + "field_2", + "base_field_2", + "field_3", + "field_4", + "field_5", + "field_6", + "field_7", + "field_8", +] + + +# HELPERS +def load_json_file(file_name: str) -> Mapping[str, Any]: + with open(f"{os.path.dirname(__file__)}/{file_name}", "r") as data: + return json.load(data) + + +def test_analytics_stream_slices(requests_mock): + requests_mock.get("https://api.linkedin.com/rest/adAccounts", json={"elements": [{"id": 1}]}) + requests_mock.get("https://api.linkedin.com/rest/adAccounts/1/adCampaigns", json={"elements": [{"id": 123}]}) + assert list( + AdMemberCountryAnalytics(config=TEST_CONFIG).stream_slices( + sync_mode=None, + ) + ) == load_json_file("output_slices.json") + + +def test_read_records(requests_mock): + requests_mock.get( + "https://api.linkedin.com/rest/adAnalytics", + [ + {"json": load_json_file("responses/ad_member_country_analytics/response_1.json")}, + {"json": load_json_file("responses/ad_member_country_analytics/response_2.json")}, + {"json": load_json_file("responses/ad_member_country_analytics/response_3.json")}, + ], + ) + stream_slice = load_json_file("output_slices.json")[0] + records = list(AdMemberCountryAnalytics(config=TEST_CONFIG).read_records(stream_slice=stream_slice, sync_mode=None)) + assert len(records) == 2 + + +def test_chunk_analytics_fields(): + """ + We expect to truncate the field list into the chunks of equal size, + with "dateRange" field presented in each chunk. + """ + expected_output = [ + ["field_1", "base_field_1", "field_2", "dateRange"], + ["base_field_2", "field_3", "field_4", "dateRange"], + ["field_5", "field_6", "field_7", "dateRange"], + ["field_8", "dateRange"], + ] + + assert list(LinkedInAdsAnalyticsStream.chunk_analytics_fields(TEST_ANALYTICS_FIELDS, TEST_FIELDS_CHUNK_SIZE)) == expected_output + + +def test_get_date_slices(): + """ + By default, we use the `WINDOW_SIZE = 30`, as it set in the analytics module + This value could be changed by setting the corresponding argument in the method. + The `end_date` is not specified by default, but for this test it was specified to have the test static. + """ + + test_start_date = "2021-08-01" + test_end_date = "2021-10-01" + + expected_output = [ + {"dateRange": {"start.day": 1, "start.month": 8, "start.year": 2021, "end.day": 31, "end.month": 8, "end.year": 2021}}, + {"dateRange": {"start.day": 31, "start.month": 8, "start.year": 2021, "end.day": 30, "end.month": 9, "end.year": 2021}}, + {"dateRange": {"start.day": 30, "start.month": 9, "start.year": 2021, "end.day": 30, "end.month": 10, "end.year": 2021}}, + ] + + assert list(LinkedInAdsAnalyticsStream.get_date_slices(test_start_date, test_end_date)) == expected_output diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py new file mode 100644 index 000000000000..56097007da27 --- /dev/null +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/test_source.py @@ -0,0 +1,361 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from unittest.mock import Mock, patch + +import pytest +import requests +from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator, TokenAuthenticator +from airbyte_cdk.utils import AirbyteTracedException +from source_linkedin_ads.source import ( + Accounts, + AccountUsers, + AdCampaignAnalytics, + AdCreativeAnalytics, + CampaignGroups, + Campaigns, + Creatives, + SourceLinkedinAds, +) +from source_linkedin_ads.streams import LINKEDIN_VERSION_API + +TEST_OAUTH_CONFIG: dict = { + "start_date": "2021-08-01", + "account_ids": [], + "credentials": { + "auth_method": "oAuth2.0", + "client_id": "client_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token", + }, +} + +TEST_CONFIG: dict = { + "start_date": "2021-08-01", + "account_ids": [1, 2], + "credentials": { + "auth_method": "access_token", + "access_token": "access_token", + "authenticator": TokenAuthenticator(token="123"), + }, +} + +TEST_CONFIG_DUPLICATE_CUSTOM_AD_ANALYTICS_REPORTS: dict = { + "start_date": "2021-01-01", + "account_ids": [], + "credentials": { + "auth_method": "oAuth2.0", + "client_id": "client_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token", + }, + "ad_analytics_reports": [ + {"name": "ShareAdByMonth", "pivot_by": "COMPANY", "time_granularity": "MONTHLY"}, + {"name": "ShareAdByMonth", "pivot_by": "COMPANY", "time_granularity": "MONTHLY"}, + ], +} + + +class TestAllStreams: + _instance: SourceLinkedinAds = SourceLinkedinAds() + + @pytest.mark.parametrize( + "config", + [ + (TEST_CONFIG), + (TEST_OAUTH_CONFIG), + ], + ids=[ + "access_token", + "oauth2.0", + ], + ) + def test_get_authenticator(self, config: dict): + test = self._instance.get_authenticator(config) + assert isinstance(test, (Oauth2Authenticator, TokenAuthenticator)) + + @pytest.mark.parametrize( + "stream_cls", + [ + Accounts, + AccountUsers, + CampaignGroups, + Campaigns, + Creatives, + AdCampaignAnalytics, + AdCreativeAnalytics, + ], + ids=[ + "Accounts", + "AccountUsers", + "CampaignGroups", + "Campaigns", + "Creatives", + "AdCampaignAnalytics", + "AdCreativeAnalytics", + ], + ) + def test_streams(self, stream_cls): + streams = self._instance.streams(config=TEST_CONFIG) + for stream in streams: + if stream_cls in streams: + assert isinstance(stream, stream_cls) + + def test_custom_streams(self): + config = {"ad_analytics_reports": [{"name": "ShareAdByMonth", "pivot_by": "COMPANY", "time_granularity": "MONTHLY"}], **TEST_CONFIG} + for stream in self._instance.get_custom_ad_analytics_reports(config=config): + assert isinstance(stream, AdCampaignAnalytics) + + @patch("source_linkedin_ads.source.Accounts.check_availability") + def test_check_connection(self, check_availability_mock): + check_availability_mock.return_value = (True, None) + is_available, error = self._instance.check_connection(logger=Mock(), config=TEST_CONFIG) + assert is_available + assert not error + + @patch("source_linkedin_ads.source.Accounts.check_availability") + def test_check_connection_failure(self, check_availability_mock): + check_availability_mock.side_effect = Exception("Not available") + is_available, error = self._instance.check_connection(logger=Mock(), config=TEST_CONFIG) + assert not is_available + assert str(error) == "Not available" + + @pytest.mark.parametrize( + "stream_cls, stream_slice, expected", + [ + (Accounts, None, "adAccounts"), + (AccountUsers, None, "adAccountUsers"), + (CampaignGroups, {"account_id": 123}, "adAccounts/123/adCampaignGroups"), + (Campaigns, {"account_id": 123}, "adAccounts/123/adCampaigns"), + (Creatives, {"account_id": 123}, "adAccounts/123/creatives"), + (AdCampaignAnalytics, None, "adAnalytics"), + (AdCreativeAnalytics, None, "adAnalytics"), + ], + ids=[ + "Accounts", + "AccountUsers", + "CampaignGroups", + "Campaigns", + "Creatives", + "AdCampaignAnalytics", + "AdCreativeAnalytics", + ], + ) + def test_path(self, stream_cls, stream_slice, expected): + stream = stream_cls(config=TEST_CONFIG) + result = stream.path(stream_slice=stream_slice) + assert result == expected + + +class TestLinkedinAdsStream: + stream: Accounts = Accounts(TEST_CONFIG) + url = f"{stream.url_base}/{stream.path()}" + + def test_accounts(self): + result = self.stream.accounts + assert result == ",".join(map(str, TEST_CONFIG["account_ids"])) + + @pytest.mark.parametrize( + "response_json, expected", + ( + ({"elements": []}, None), + ({"elements": [{"data": []}] * 500, "paging": {"start": 0}}, {"start": 500}), + ), + ) + def test_next_page_token(self, requests_mock, response_json, expected): + requests_mock.get(self.url, json=response_json) + test_response = requests.get(self.url) + + result = self.stream.next_page_token(test_response) + assert expected == result + + def test_request_params(self): + expected = "count=500&q=search&search=(id:(values:List(1,2)))" + result = self.stream.request_params(stream_state={}, stream_slice={"account_id": 123}) + assert expected == result + + def test_parse_response(self, requests_mock): + requests_mock.get(self.url, json={"elements": [{"test": "test"}]}) + test_response = requests.get(self.url) + + expected = {"test": "test"} + result = list(self.stream.parse_response(test_response)) + assert result[0] == expected + + def test_should_retry(self, requests_mock): + requests_mock.get(self.url, json={}, status_code=429) + test_response = requests.get(self.url) + result = self.stream.should_retry(test_response) + assert result is True + + def test_request_headers(self): + expected = {"X-RestLi-Protocol-Version": "2.0.0", "Linkedin-Version": LINKEDIN_VERSION_API} + result = self.stream.request_headers(stream_state={}) + assert result == expected + + +class TestAccountUsers: + stream: AccountUsers = AccountUsers(TEST_CONFIG) + + def test_state_checkpoint_interval(self): + assert self.stream.state_checkpoint_interval == 100 + + def test_get_updated_state(self): + state = self.stream.get_updated_state( + current_stream_state={"lastModified": "2021-01-01"}, latest_record={"lastModified": "2021-08-01"} + ) + assert state == {"lastModified": "2021-08-01"} + + +class TestLinkedInAdsStreamSlicing: + @pytest.mark.parametrize( + "stream_cls, slice, expected", + [ + ( + AccountUsers, + {"account_id": 123}, + "count=500&q=accounts&accounts=urn:li:sponsoredAccount:123", + ), + ( + CampaignGroups, + {"account_id": 123}, + "count=500&q=search&search=(status:(values:List(ACTIVE,ARCHIVED,CANCELED,DRAFT,PAUSED,PENDING_DELETION,REMOVED)))", + ), + ( + Campaigns, + {"account_id": 123}, + "count=500&q=search&search=(status:(values:List(ACTIVE,PAUSED,ARCHIVED,COMPLETED,CANCELED,DRAFT,PENDING_DELETION,REMOVED)))", + ), + ( + Creatives, + {"campaign_id": 123}, + "count=100&q=criteria", + ), + ], + ids=["AccountUsers", "CampaignGroups", "Campaigns", "Creatives"], + ) + def test_request_params(self, stream_cls, slice, expected): + stream = stream_cls(TEST_CONFIG) + result = stream.request_params(stream_state={}, stream_slice=slice) + assert expected == result + + @pytest.mark.parametrize( + "stream_cls, state, records_slice, expected", + [ + (AccountUsers, {"lastModified": 1}, [{"lastModified": 2}], [{"lastModified": 2}]), + (CampaignGroups, {"lastModified": 3}, [{"lastModified": 3}], [{"lastModified": 3}]), + (Campaigns, {}, [], []), + (Creatives, {}, [], []), + ], + ids=[ + "AccountUsers", + "CampaignGroups", + "Campaigns", + "Creatives", + ], + ) + def test_filter_records_newer_than_state(self, stream_cls, state, records_slice, expected): + stream = stream_cls(TEST_CONFIG) + result = list(stream.filter_records_newer_than_state(state, records_slice)) + assert result == expected + + +class TestLinkedInAdsAnalyticsStream: + @pytest.mark.parametrize( + "stream_cls, expected", + [ + (AdCampaignAnalytics, {"pivot": "(value:CAMPAIGN)", "q": "analytics", "timeGranularity": "(value:DAILY)"}), + (AdCreativeAnalytics, {"pivot": "(value:CREATIVE)", "q": "analytics", "timeGranularity": "(value:DAILY)"}), + ], + ids=[ + "AdCampaignAnalytics", + "AdCreativeAnalytics", + ], + ) + def test_base_analytics_params(self, stream_cls, expected): + stream = stream_cls(config=TEST_CONFIG) + result = stream.base_analytics_params + assert result == expected + + @pytest.mark.parametrize( + "stream_cls, slice, expected", + [ + ( + AdCampaignAnalytics, + { + "dateRange": {"start.day": 1, "start.month": 1, "start.year": 1, "end.day": 2, "end.month": 2, "end.year": 2}, + "fields": ["field1", "field2"], + }, + "q=analytics&pivot=(value:CAMPAIGN)&timeGranularity=(value:DAILY)&dateRange=(start:(year:1,month:1,day:1),end:(year:2,month:2,day:2))&fields=%5B%27field1%27,+%27field2%27%5D&campaigns=List(urn%3Ali%3AsponsoredCampaign%3ANone)", + ), + ( + AdCreativeAnalytics, + { + "dateRange": { + "start.day": 1, + "start.month": 1, + "start.year": 1, + "end.day": 2, + "end.month": 2, + "end.year": 2, + }, + "fields": [ + "field1", + "field2", + ], + "creative_id": "urn:li:sponsoredCreative:1234", + }, + "q=analytics&pivot=(value:CREATIVE)&timeGranularity=(value:DAILY)&dateRange=(start:(year:1,month:1,day:1),end:(year:2,month:2,day:2))&fields=%5B%27field1%27,+%27field2%27%5D&creatives=List(urn%3Ali%3AsponsoredCreative%3A1234)", + ), + ], + ids=[ + "AdCampaignAnalytics", + "AdCreativeAnalytics", + ], + ) + def test_request_params(self, stream_cls, slice, expected): + stream = stream_cls(config=TEST_CONFIG) + result = stream.request_params(stream_state={}, stream_slice=slice) + assert expected == result + + +def test_retry_get_access_token(requests_mock): + requests_mock.register_uri( + "POST", + "https://www.linkedin.com/oauth/v2/accessToken", + [{"status_code": 429}, {"status_code": 429}, {"status_code": 200, "json": {"access_token": "token", "expires_in": 3600}}], + ) + auth = Oauth2Authenticator( + token_refresh_endpoint="https://www.linkedin.com/oauth/v2/accessToken", + client_id="client_id", + client_secret="client_secret", + refresh_token="refresh_token", + ) + token = auth.get_access_token() + assert len(requests_mock.request_history) == 3 + assert token == "token" + + +@pytest.mark.parametrize( + "record, expected", + [ + ({}, {}), + ({"lastModified": "2021-05-27 11:59:53.710000"}, {"lastModified": "2021-05-27T11:59:53.710000+00:00"}), + ({"lastModified": None}, {"lastModified": None}), + ({"lastModified": ""}, {"lastModified": ""}), + ], + ids=["empty_record", "transformed_record", "null_value", "empty_value"], +) +def test_date_time_to_rfc3339(record, expected): + stream = Accounts(TEST_CONFIG) + result = stream._date_time_to_rfc3339(record) + assert result == expected + + +def test_duplicated_custom_ad_analytics_report(): + with pytest.raises(AirbyteTracedException) as e: + SourceLinkedinAds().streams(TEST_CONFIG_DUPLICATE_CUSTOM_AD_ANALYTICS_REPORTS) + expected_message = "Stream names for Custom Ad Analytics reports should be unique, duplicated streams: {'ShareAdByMonth'}" + assert e.value.message == expected_message diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py index 048b726343a5..57212a2d2a31 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/utils_tests/samples/test_data_for_tranform.py @@ -23,6 +23,8 @@ }, {"or": {"urn:li:adTargetingFacet:locations": ["urn:li:geo:103644278"]}}, {"or": {"urn:li:adTargetingFacet:interfaceLocales": ["urn:li:locale:en_US"]}}, + {"or": {"empty_dict_with_empty_list_of_dicts": [{"empty_dict_value": "the value"}]}}, + {"or": {"empty_dict_with_empty_dict": {"empty_dict_value": "the value"}}}, {"or": {"empty_dict_with_empty_list": []}}, # dict is present, but list is empty {"or": {}}, # empty dict ] @@ -92,6 +94,14 @@ "type": "urn:li:adTargetingFacet:interfaceLocales", "values": ["urn:li:locale:en_US"], }, + { + "type": "empty_dict_with_empty_list_of_dicts", + "values": [{"empty_dict_value": "the value"}], + }, + { + "type": "empty_dict_with_empty_dict", + "values": [{"empty_dict_value": "the value"}], + }, { "type": "empty_dict_with_empty_list", "values": [], diff --git a/airbyte-integrations/connectors/source-linkedin-pages/README.md b/airbyte-integrations/connectors/source-linkedin-pages/README.md index a6089345655e..aa8934c3e366 100644 --- a/airbyte-integrations/connectors/source-linkedin-pages/README.md +++ b/airbyte-integrations/connectors/source-linkedin-pages/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-linkedin-pages:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/linkedin-pages) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_linkedin_pages/spec.json` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-linkedin-pages:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-linkedin-pages build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-linkedin-pages:airbyteDocker +An image will be built with the tag `airbyte/source-linkedin-pages:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-linkedin-pages:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-linkedin-pages:dev che docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-linkedin-pages:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-linkedin-pages:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-linkedin-pages test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-linkedin-pages:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-linkedin-pages:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-linkedin-pages test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/linkedin-pages.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-linkedin-pages/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-linkedin-pages/build.gradle b/airbyte-integrations/connectors/source-linkedin-pages/build.gradle deleted file mode 100644 index 34323a22dff2..000000000000 --- a/airbyte-integrations/connectors/source-linkedin-pages/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_linkedin_pages' -} diff --git a/airbyte-integrations/connectors/source-linnworks/README.md b/airbyte-integrations/connectors/source-linnworks/README.md index 276bd3258f5c..b5395f00515c 100644 --- a/airbyte-integrations/connectors/source-linnworks/README.md +++ b/airbyte-integrations/connectors/source-linnworks/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-linnworks:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/linnworks) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_linnworks/spec.json` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-linnworks:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-linnworks build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-linnworks:airbyteDocker +An image will be built with the tag `airbyte/source-linnworks:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-linnworks:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-linnworks:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-linnworks:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-linnworks:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-linnworks test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-linnworks:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-linnworks:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-linnworks test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/linnworks.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-linnworks/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-linnworks/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-linnworks/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-linnworks/build.gradle b/airbyte-integrations/connectors/source-linnworks/build.gradle deleted file mode 100644 index b667594d0e18..000000000000 --- a/airbyte-integrations/connectors/source-linnworks/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_linnworks' -} diff --git a/airbyte-integrations/connectors/source-lokalise/README.md b/airbyte-integrations/connectors/source-lokalise/README.md index 2196a40cd774..fbeee5a12a5e 100644 --- a/airbyte-integrations/connectors/source-lokalise/README.md +++ b/airbyte-integrations/connectors/source-lokalise/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-lokalise:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/lokalise) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_lokalise/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-lokalise:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-lokalise build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-lokalise:airbyteDocker +An image will be built with the tag `airbyte/source-lokalise:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-lokalise:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-lokalise:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-lokalise:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-lokalise:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-lokalise test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-lokalise:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-lokalise:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-lokalise test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/lokalise.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-lokalise/acceptance-test-config.yml b/airbyte-integrations/connectors/source-lokalise/acceptance-test-config.yml index e7262b855051..dd9741f472e3 100644 --- a/airbyte-integrations/connectors/source-lokalise/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-lokalise/acceptance-test-config.yml @@ -19,7 +19,7 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - incremental: + incremental: bypass_reason: "This connector does not implement incremental sync" full_refresh: tests: diff --git a/airbyte-integrations/connectors/source-lokalise/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-lokalise/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-lokalise/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-lokalise/build.gradle b/airbyte-integrations/connectors/source-lokalise/build.gradle deleted file mode 100644 index 8a231283342f..000000000000 --- a/airbyte-integrations/connectors/source-lokalise/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_lokalise' -} diff --git a/airbyte-integrations/connectors/source-looker/README.md b/airbyte-integrations/connectors/source-looker/README.md index 9aca9eb29917..d1b02c9ad5f4 100644 --- a/airbyte-integrations/connectors/source-looker/README.md +++ b/airbyte-integrations/connectors/source-looker/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-looker:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/looker) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_looker/spec.json` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-looker:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-looker build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-looker:airbyteDocker +An image will be built with the tag `airbyte/source-looker:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-looker:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -79,52 +72,27 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-looker:dev discover -- docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-looker:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .'[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-looker test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-looker:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-looker:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-looker test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/looker.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-looker/acceptance-test-config.yml b/airbyte-integrations/connectors/source-looker/acceptance-test-config.yml index e6e1c025815f..32746a26bce7 100644 --- a/airbyte-integrations/connectors/source-looker/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-looker/acceptance-test-config.yml @@ -16,18 +16,18 @@ tests: configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [ - "scheduled_plans", - "user_attribute_group_values", - "user_login_lockouts", - "user_sessions", + "scheduled_plans", + "user_attribute_group_values", + "user_login_lockouts", + "user_sessions", ] full_refresh: # test streams except "run_looks" - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" ignored_fields: - datagroups: [ "properties", "trigger_check_at" ] - looks: [ "properties", "last_accessed_at" ] + datagroups: ["properties", "trigger_check_at"] + looks: ["properties", "last_accessed_at"] # test the stream "run_looks" separately - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_run_looks_catalog.json" @@ -36,4 +36,3 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" - diff --git a/airbyte-integrations/connectors/source-looker/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-looker/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-looker/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-looker/build.gradle b/airbyte-integrations/connectors/source-looker/build.gradle deleted file mode 100644 index c8741eae92c0..000000000000 --- a/airbyte-integrations/connectors/source-looker/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_looker' -} diff --git a/airbyte-integrations/connectors/source-looker/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-looker/unit_tests/unit_test.py index ecb92bf7feb6..5276251b3fca 100644 --- a/airbyte-integrations/connectors/source-looker/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-looker/unit_tests/unit_test.py @@ -6,7 +6,10 @@ def test_format_null_in_schema(): - schema = {'type': ['null', 'object'], 'properties': {'field': {'type': 'object', 'properties': {'field': {'type': 'number'}}}}} + schema = {"type": ["null", "object"], "properties": {"field": {"type": "object", "properties": {"field": {"type": "number"}}}}} output = LookerStream.format_null_in_schema(schema) - expected = {'type': ['null', 'object'], 'properties': {'field': {'type': ['null', 'object'], 'properties': {'field': {'type': ['null', 'number']}}}}} + expected = { + "type": ["null", "object"], + "properties": {"field": {"type": ["null", "object"], "properties": {"field": {"type": ["null", "number"]}}}}, + } assert output == expected diff --git a/airbyte-integrations/connectors/source-mailchimp/Dockerfile b/airbyte-integrations/connectors/source-mailchimp/Dockerfile deleted file mode 100644 index 24c0b8946fd4..000000000000 --- a/airbyte-integrations/connectors/source-mailchimp/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY setup.py ./ -RUN pip install . -COPY source_mailchimp ./source_mailchimp -COPY main.py ./ - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.4.1 -LABEL io.airbyte.name=airbyte/source-mailchimp diff --git a/airbyte-integrations/connectors/source-mailchimp/README.md b/airbyte-integrations/connectors/source-mailchimp/README.md index 0c6692dad28b..7f8ca7aeaa31 100644 --- a/airbyte-integrations/connectors/source-mailchimp/README.md +++ b/airbyte-integrations/connectors/source-mailchimp/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-mailchimp:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/mailchimp) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_mailchimp/spec.json` file. @@ -56,18 +48,19 @@ python main.py read --config secrets/config.json --catalog sample_files/configur ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-mailchimp:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-mailchimp build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-mailchimp:airbyteDocker +An image will be built with the tag `airbyte/source-mailchimp:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-mailchimp:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -77,44 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailchimp:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailchimp:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-mailchimp:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json ``` + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-mailchimp test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unittest run: -``` -./gradlew :airbyte-integrations:connectors:source-mailchimp:unitTest -``` -To run acceptance and custom integration tests run: -``` -./gradlew :airbyte-integrations:connectors:source-mailchimp:IntegrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -124,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-mailchimp test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/mailchimp.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-mailchimp/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mailchimp/acceptance-test-config.yml index 12461751e83a..c7b7594f3190 100644 --- a/airbyte-integrations/connectors/source-mailchimp/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-mailchimp/acceptance-test-config.yml @@ -3,58 +3,55 @@ test_strictness_level: high acceptance_tests: spec: tests: - - spec_path: "source_mailchimp/spec.json" + - spec_path: "source_mailchimp/spec.json" connection: tests: - # for old spec config (without oneOf) - - config_path: "secrets/config.json" - status: "succeed" # for auth with API token - - config_path: "secrets/config_apikey.json" - status: "succeed" - # for auth with oauth2 token - - config_path: "secrets/config_oauth.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" - - config_path: "integration_tests/invalid_config_apikey.json" - status: "failed" - - config_path: "integration_tests/invalid_config_oauth.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + # for auth with oauth2 token + - config_path: "secrets/config_oauth.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + - config_path: "integration_tests/invalid_config_apikey.json" + status: "failed" + - config_path: "integration_tests/invalid_config_oauth.json" + status: "failed" discovery: tests: - # for old spec config (without oneOf) - - config_path: "secrets/config.json" # for auth with API token - - config_path: "secrets/config_apikey.json" - # for auth with oauth2 token - - config_path: "secrets/config_oauth.json" + - config_path: "secrets/config.json" + # for auth with oauth2 token + - config_path: "secrets/config_oauth.json" basic_read: tests: - - config_path: "secrets/config.json" - expect_records: - bypass_reason: "Risk to disclose internal data. Need to set up a sandbox account - https://github.com/airbytehq/airbyte/issues/20726" - fail_on_extra_columns: false - - config_path: "secrets/config_oauth.json" - expect_records: - bypass_reason: "Risk to disclose internal data. Need to set up a sandbox account - https://github.com/airbytehq/airbyte/issues/20726" - fail_on_extra_columns: false + - config_path: "secrets/config.json" + expect_records: + path: "integration_tests/expected_records.jsonl" + fail_on_extra_columns: false + empty_streams: + - name: "automations" + bypass_reason: "Cannot seed in free sandbox account, need to upgrade to paid account." + - config_path: "secrets/config_oauth.json" + expect_records: + path: "integration_tests/expected_records.jsonl" + extra_records: True + fail_on_extra_columns: false + empty_streams: + - name: "automations" + bypass_reason: "Cannot seed in free sandbox account, need to upgrade to paid account." incremental: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" future_state: future_state_path: "integration_tests/state.json" - cursor_paths: - automations: ["create_time"] - lists: ["date_created"] - campaigns: ["create_time"] - email_activity: ["49d68626f3", "timestamp"] # Email activities stream has working campaigns with email newsletters. # Due to this sequential_reads test could be failed. full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_without_email_activities.json" - - config_path: "secrets/config_oauth.json" - configured_catalog_path: "integration_tests/configured_catalog_without_email_activities.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_without_email_activities.json" + - config_path: "secrets/config_oauth.json" + configured_catalog_path: "integration_tests/configured_catalog_without_email_activities.json" diff --git a/airbyte-integrations/connectors/source-mailchimp/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-mailchimp/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-mailchimp/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-mailchimp/build.gradle b/airbyte-integrations/connectors/source-mailchimp/build.gradle deleted file mode 100644 index c1f370e1873b..000000000000 --- a/airbyte-integrations/connectors/source-mailchimp/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_mailchimp' -} diff --git a/airbyte-integrations/connectors/source-mailchimp/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-mailchimp/integration_tests/configured_catalog.json index 8baade2f96af..458ab841ad88 100644 --- a/airbyte-integrations/connectors/source-mailchimp/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-mailchimp/integration_tests/configured_catalog.json @@ -28,6 +28,28 @@ "primary_key": [["id"]], "destination_sync_mode": "append" }, + { + "stream": { + "name": "interest_categories", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "interests", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "destination_sync_mode": "append" + }, { "stream": { "name": "lists", @@ -56,6 +78,20 @@ "primary_key": [["timestamp"], ["email_id"], ["action"]], "destination_sync_mode": "append" }, + { + "stream": { + "name": "list_members", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["last_changed"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "cursor_field": ["last_changed"], + "primary_key": [["id"]], + "destination_sync_mode": "append" + }, { "stream": { "name": "reports", @@ -69,6 +105,63 @@ "cursor_field": ["send_time"], "primary_key": [["id"]], "destination_sync_mode": "append" + }, + { + "stream": { + "name": "segment_members", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["last_changed"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "cursor_field": ["last_changed"], + "primary_key": [["id"]], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "segments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "primary_key": [["id"]], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "tags", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "unsubscribes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["timestamp"], + "source_defined_primary_key": [ + ["campaign_id"], + ["email_id"], + ["timestamp"] + ] + }, + "sync_mode": "incremental", + "cursor_field": ["timestamp"], + "primary_key": [["campaign_id"], ["email_id"], ["timestamp"]], + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-mailchimp/integration_tests/configured_catalog_without_email_activities.json b/airbyte-integrations/connectors/source-mailchimp/integration_tests/configured_catalog_without_email_activities.json index 2fcb0f2f2274..befee3dcfc63 100644 --- a/airbyte-integrations/connectors/source-mailchimp/integration_tests/configured_catalog_without_email_activities.json +++ b/airbyte-integrations/connectors/source-mailchimp/integration_tests/configured_catalog_without_email_activities.json @@ -28,6 +28,28 @@ "primary_key": [["id"]], "destination_sync_mode": "append" }, + { + "stream": { + "name": "interest_categories", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "interests", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "destination_sync_mode": "append" + }, { "stream": { "name": "lists", @@ -42,6 +64,20 @@ "primary_key": [["id"]], "destination_sync_mode": "append" }, + { + "stream": { + "name": "list_members", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["last_changed"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "cursor_field": ["last_changed"], + "primary_key": [["id"]], + "destination_sync_mode": "append" + }, { "stream": { "name": "reports", @@ -55,6 +91,63 @@ "cursor_field": ["send_time"], "primary_key": [["id"]], "destination_sync_mode": "append" + }, + { + "stream": { + "name": "segment_members", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["last_changed"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "cursor_field": ["last_changed"], + "primary_key": [["id"]], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "segments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "primary_key": [["id"]], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "tags", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "primary_key": [["id"]], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "unsubscribes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["timestamp"], + "source_defined_primary_key": [ + ["campaign_id"], + ["email_id"], + ["timestamp"] + ] + }, + "sync_mode": "incremental", + "cursor_field": ["timestamp"], + "primary_key": [["campaign_id"], ["email_id"], ["timestamp"]], + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-mailchimp/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-mailchimp/integration_tests/expected_records.jsonl new file mode 100644 index 000000000000..f9a4f79caa5f --- /dev/null +++ b/airbyte-integrations/connectors/source-mailchimp/integration_tests/expected_records.jsonl @@ -0,0 +1,17 @@ +{"stream": "campaigns", "data": {"id": "324b8a398e", "web_id": 13531140, "type": "regular", "create_time": "2022-12-27T08:12:59+00:00", "archive_url": "http://eepurl.com/ig7RxP", "long_archive_url": "https://us10.campaign-archive.com/?u=caf9055242d41edd9215d1898&id=324b8a398e", "status": "save", "emails_sent": 0, "send_time": null, "content_type": "multichannel", "needs_block_refresh": false, "resendable": false, "recipients": {"list_id": "16d6ec4ffc", "list_is_active": true, "list_name": "Airbyte", "segment_text": null, "recipient_count": 47}, "settings": {"title": "Untitled", "use_conversation": false, "to_name": null, "folder_id": null, "authenticate": true, "auto_footer": false, "inline_css": false, "auto_tweet": false, "fb_comments": true, "timewarp": false, "template_id": 13, "drag_and_drop": false}, "tracking": {"opens": true, "html_clicks": true, "text_clicks": false, "goal_tracking": false, "ecomm360": false, "google_analytics": null, "clicktale": null}, "social_card": {"image_url": "https://cdn-images.mailchimp.com/monkey_rewards/grow-business-banner-2.png", "description": null, "title": null}, "delivery_status": {"enabled": false}}, "emitted_at": 1701638872240} +{"stream": "campaigns", "data": {"id": "3cbed9a0fc", "web_id": 13531144, "type": "regular", "create_time": "2022-12-27T08:21:01+00:00", "archive_url": "http://eepurl.com/ig7SKH", "long_archive_url": "https://us10.campaign-archive.com/?u=caf9055242d41edd9215d1898&id=3cbed9a0fc", "status": "save", "emails_sent": 0, "send_time": null, "content_type": "template", "needs_block_refresh": false, "resendable": false, "recipients": {"list_id": "16d6ec4ffc", "list_is_active": true, "list_name": "Airbyte", "segment_text": null, "recipient_count": 47}, "settings": {"title": "Untitled", "use_conversation": false, "to_name": null, "folder_id": null, "authenticate": true, "auto_footer": false, "inline_css": false, "auto_tweet": false, "fb_comments": true, "timewarp": false, "template_id": 145, "drag_and_drop": true}, "tracking": {"opens": true, "html_clicks": true, "text_clicks": false, "goal_tracking": false, "ecomm360": false, "google_analytics": null, "clicktale": null}, "social_card": {"image_url": "https://cdn-images.mailchimp.com/monkey_rewards/grow-business-banner-2.png", "description": null, "title": null}, "delivery_status": {"enabled": false}}, "emitted_at": 1701638872242} +{"stream": "email_activity", "data": {"campaign_id": "7847cdaeff", "list_id": "16d6ec4ffc", "list_is_active": true, "email_id": "11273c9a5dc6ae6c5aaccfb77b2addfb", "email_address": "AirbyteMailchimpUser@gmail.com", "action": "open", "timestamp": "2023-11-06T20:17:57+00:00", "ip": "74.125.212.231"}, "emitted_at": 1701638876052} +{"stream": "email_activity", "data": {"campaign_id": "7847cdaeff", "list_id": "16d6ec4ffc", "list_is_active": true, "email_id": "11273c9a5dc6ae6c5aaccfb77b2addfb", "email_address": "AirbyteMailchimpUser@gmail.com", "action": "open", "timestamp": "2023-11-07T19:49:10+00:00", "ip": "74.125.215.162"}, "emitted_at": 1701638876053} +{"stream": "interests", "data": {"category_id": "a194ba131d", "list_id": "16d6ec4ffc", "id": "bbbb369575", "name": "Donating", "subscriber_count": "0", "display_order": 1}, "emitted_at": 1699963797987} +{"stream": "interest_categories", "data": {"list_id": "16d6ec4ffc", "id": "1bcbe8ba9b", "title": "Product Preferences", "display_order": 0, "type": "checkboxes"}, "emitted_at": 1699963796751} +{"stream": "list_members", "data": {"id": "87ed95f658a2efab665957871270de69", "email_address": "integration-test+yurii@airbyte.io", "unique_email_id": "38619c0ffd", "contact_id": "6ad2f9e1b25a09421fd8df87662b1634", "full_name": "yurii cherniaiev", "web_id": 546044412, "email_type": "html", "status": "subscribed", "consents_to_one_to_one_messaging": true, "merge_fields": {"FNAME": "yurii", "LNAME": "cherniaiev", "ADDRESS": {"addr1": "Airbyte\nkyiv\nKiev 04200\nUkraine", "addr2": null, "city": null, "state": null, "zip": null, "country": "US"}, "PHONE": null, "BIRTHDAY": null}, "interests": {"bbbb369575": false, "97bbc1227a": false, "d802d794f8": false, "b35e48738e": false, "44d2c158e3": false, "29f73b8209": false, "2010f3c101": false, "75f1cb79fd": false, "aa2fd02c59": false, "f7b60a3c3d": false, "7733d60f61": false, "cc454d76d6": false, "797533254b": false, "9ea08b864b": false, "e2e5fdcac9": false, "8eccc648d6": false, "a7c814599e": false, "20ef45c5d3": false, "1824f5d1a5": false, "644f34517f": false, "c57e1a9ff6": false, "b97fee61c8": false, "b9d16768e3": false, "810348679c": false, "43ebb04472": false, "73ee7c1d1b": false, "045738fa17": false, "0a7cbd4449": false, "fef00a4695": false, "4a19201dc9": false, "571a80ed60": false}, "stats": {"avg_open_rate": 1, "avg_click_rate": 1}, "ip_signup": null, "timestamp_signup": null, "ip_opt": "93.73.161.112", "timestamp_opt": "2022-12-27T07:56:47+00:00", "member_rating": 2, "last_changed": "2022-12-27T07:56:47+00:00", "language": null, "vip": false, "email_client": null, "location": {"latitude": 0, "longitude": 0, "gmtoff": 0, "dstoff": 0, "country_code": null, "timezone": null, "region": null}, "source": "Admin Add", "tags_count": 1, "tags": [{"id": 14351504, "name": "Overlord"}], "list_id": "16d6ec4ffc"}, "emitted_at": 1701638878091} +{"stream": "list_members", "data": {"id": "65a02406e7dc3b786af6b94489721e46", "email_address": "integration-test+Jesse@airbyte.io", "unique_email_id": "475570cf37", "contact_id": "6639385c4bdffe110d88044782e37c12", "full_name": "Jesse", "web_id": 546044440, "email_type": "html", "status": "subscribed", "consents_to_one_to_one_messaging": true, "merge_fields": {"FNAME": "Jesse", "LNAME": null, "ADDRESS": null, "PHONE": null, "BIRTHDAY": null}, "interests": {"bbbb369575": false, "97bbc1227a": false, "d802d794f8": false, "b35e48738e": false, "44d2c158e3": false, "29f73b8209": false, "2010f3c101": false, "75f1cb79fd": false, "aa2fd02c59": false, "f7b60a3c3d": false, "7733d60f61": false, "cc454d76d6": false, "797533254b": false, "9ea08b864b": false, "e2e5fdcac9": false, "8eccc648d6": false, "a7c814599e": false, "20ef45c5d3": false, "1824f5d1a5": false, "644f34517f": false, "c57e1a9ff6": false, "b97fee61c8": false, "b9d16768e3": false, "810348679c": false, "43ebb04472": false, "73ee7c1d1b": false, "045738fa17": false, "0a7cbd4449": false, "fef00a4695": false, "4a19201dc9": false, "571a80ed60": false}, "stats": {"avg_open_rate": 1, "avg_click_rate": 0}, "ip_signup": null, "timestamp_signup": null, "ip_opt": "93.73.161.112", "timestamp_opt": "2022-12-27T08:34:38+00:00", "member_rating": 2, "last_changed": "2022-12-27T08:34:38+00:00", "language": null, "vip": false, "email_client": null, "location": {"latitude": 0, "longitude": 0, "gmtoff": 0, "dstoff": 0, "country_code": null, "timezone": null, "region": null}, "source": "Import", "tags_count": 0, "tags": [], "list_id": "16d6ec4ffc"}, "emitted_at": 1701638878092} +{"stream": "lists", "data": {"id": "16d6ec4ffc", "web_id": 903380, "name": "Airbyte", "contact": {"company": "Airbyte", "address1": "kyiv", "address2": null, "city": "Kiev", "state": "30", "zip": "04200", "country": "UA", "phone": null}, "permission_reminder": "You are receiving this email because you opted in via our website.", "use_archive_bar": true, "campaign_defaults": {"from_name": "yurii", "from_email": "integration-test+yurii@airbyte.io", "subject": null, "language": "en"}, "notify_on_subscribe": null, "notify_on_unsubscribe": null, "date_created": "2022-12-27T07:56:47+00:00", "list_rating": 0, "email_type_option": false, "subscribe_url_short": "http://eepurl.com/ihg3RD", "subscribe_url_long": "https://airbyte.us10.list-manage.com/subscribe?u=caf9055242d41edd9215d1898&id=16d6ec4ffc", "beamer_address": "us10-d527bd96ba-6d1a9988db@inbound.mailchimp.com", "visibility": "prv", "double_optin": false, "has_welcome": false, "marketing_permissions": false, "modules": [], "stats": {"member_count": 47, "unsubscribe_count": 4, "cleaned_count": 0, "member_count_since_send": 0, "unsubscribe_count_since_send": 1, "cleaned_count_since_send": 0, "campaign_count": 6, "campaign_last_sent": "2022-12-27T08:37:53+00:00", "merge_field_count": 5, "avg_sub_rate": 0, "avg_unsub_rate": 0, "target_sub_rate": 0, "open_rate": 100, "click_rate": 64.70588235294117, "last_sub_date": "2022-12-27T08:34:39+00:00", "last_unsub_date": "2023-11-06T20:18:01+00:00"}}, "emitted_at": 1701638875717} +{"stream": "reports", "data": {"id": "a79651273b", "campaign_title": "Untitled", "type": "regular", "list_id": "16d6ec4ffc", "list_is_active": true, "list_name": "Airbyte", "subject_line": "Airbyte Test", "preview_text": null, "emails_sent": 50, "abuse_reports": 0, "unsubscribed": 0, "send_time": "2022-12-27T08:36:55+00:00", "bounces": {"hard_bounces": 0, "soft_bounces": 0, "syntax_errors": 0}, "forwards": {"forwards_count": 0, "forwards_opens": 0}, "opens": {"opens_total": 412, "unique_opens": 50, "open_rate": 1, "last_open": "2023-01-09T10:07:54+00:00"}, "clicks": {"clicks_total": 48, "unique_clicks": 47, "unique_subscriber_clicks": 33, "click_rate": 0.66, "last_click": "2022-12-27T15:28:11+00:00"}, "facebook_likes": {"recipient_likes": 0, "unique_likes": 0, "facebook_likes": 0}, "list_stats": {"sub_rate": 0, "unsub_rate": 0, "open_rate": 100, "click_rate": 64.70588235294117}, "timeseries": [{"timestamp": "2022-12-27T08:00:00+00:00", "emails_sent": 50, "unique_opens": 6, "recipients_clicks": 1}, {"timestamp": "2022-12-27T09:00:00+00:00", "emails_sent": 0, "unique_opens": 43, "recipients_clicks": 0}, {"timestamp": "2022-12-27T10:00:00+00:00", "emails_sent": 0, "unique_opens": 1, "recipients_clicks": 3}, {"timestamp": "2022-12-27T11:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 11}, {"timestamp": "2022-12-27T12:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 10}, {"timestamp": "2022-12-27T13:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 3}, {"timestamp": "2022-12-27T14:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 2}, {"timestamp": "2022-12-27T15:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 3}, {"timestamp": "2022-12-27T16:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T17:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T18:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T19:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T20:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T21:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T22:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-27T23:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T00:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T01:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T02:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T03:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T04:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T05:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T06:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2022-12-28T07:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}], "ecommerce": {"total_orders": 0, "total_spent": 0, "total_revenue": 0, "currency_code": "USD"}, "delivery_status": {"enabled": false}}, "emitted_at": 1701638878519} +{"stream": "reports", "data": {"id": "7847cdaeff", "campaign_title": "Invitation to unsubscribe", "type": "regular", "list_id": "16d6ec4ffc", "list_is_active": true, "list_name": "Airbyte", "subject_line": "Invitation to Unsubscribe", "preview_text": null, "emails_sent": 1, "abuse_reports": 0, "unsubscribed": 1, "send_time": "2023-11-06T20:17:44+00:00", "bounces": {"hard_bounces": 0, "soft_bounces": 0, "syntax_errors": 0}, "forwards": {"forwards_count": 0, "forwards_opens": 0}, "opens": {"opens_total": 2, "unique_opens": 1, "open_rate": 1, "last_open": "2023-11-07T19:49:10+00:00"}, "clicks": {"clicks_total": 0, "unique_clicks": 0, "unique_subscriber_clicks": 0, "click_rate": 0, "last_click": null}, "facebook_likes": {"recipient_likes": 0, "unique_likes": 0, "facebook_likes": 0}, "list_stats": {"sub_rate": 0, "unsub_rate": 0, "open_rate": 100, "click_rate": 64.70588235294117}, "timeseries": [{"timestamp": "2023-11-06T20:00:00+00:00", "emails_sent": 1, "unique_opens": 1, "recipients_clicks": 0}, {"timestamp": "2023-11-06T21:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-06T22:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-06T23:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T00:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T01:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T02:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T03:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T04:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T05:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T06:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T07:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T08:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T09:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T10:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T11:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T12:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T13:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T14:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T15:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T16:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T17:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T18:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}, {"timestamp": "2023-11-07T19:00:00+00:00", "emails_sent": 0, "unique_opens": 0, "recipients_clicks": 0}], "ecommerce": {"total_orders": 0, "total_spent": 0, "total_revenue": 0, "currency_code": "USD"}, "delivery_status": {"enabled": false}}, "emitted_at": 1701638878520} +{"stream": "segment_members", "data": {"id": "1dd067951f91190b65b43305b9166bc7", "email_address": "integration-test+Michael@airbyte.io", "unique_email_id": "904643439a", "email_type": "html", "status": "subscribed", "merge_fields": {"FNAME": "Michael", "LNAME": null, "ADDRESS": null, "PHONE": null, "BIRTHDAY": null}, "interests": {"bbbb369575": false, "97bbc1227a": false, "d802d794f8": false, "b35e48738e": false, "44d2c158e3": false, "29f73b8209": false, "2010f3c101": false, "75f1cb79fd": false, "aa2fd02c59": false, "f7b60a3c3d": false, "7733d60f61": false, "cc454d76d6": false, "797533254b": false, "9ea08b864b": false, "e2e5fdcac9": false, "8eccc648d6": false, "a7c814599e": false, "20ef45c5d3": false, "1824f5d1a5": false, "644f34517f": false, "c57e1a9ff6": false, "b97fee61c8": false, "b9d16768e3": false, "810348679c": false, "43ebb04472": false, "73ee7c1d1b": false, "045738fa17": false, "0a7cbd4449": false, "fef00a4695": false, "4a19201dc9": false, "571a80ed60": false}, "stats": {"avg_open_rate": 1, "avg_click_rate": 0}, "ip_signup": null, "timestamp_signup": null, "ip_opt": "93.73.161.112", "timestamp_opt": "2022-12-27T08:34:39+00:00", "member_rating": 2, "last_changed": "2022-12-27T08:34:39+00:00", "language": null, "vip": false, "email_client": null, "location": {"latitude": 0, "longitude": 0, "gmtoff": 0, "dstoff": 0, "country_code": null, "timezone": null}, "list_id": "16d6ec4ffc", "segment_id": 13506120}, "emitted_at": 1701638879995} +{"stream": "segment_members", "data": {"id": "802cb9cc84d031ca07cbf9efa3dcdc2c", "email_address": "integration-test+Carlos@airbyte.io", "unique_email_id": "41ec088075", "email_type": "html", "status": "subscribed", "merge_fields": {"FNAME": "Carlos", "LNAME": null, "ADDRESS": null, "PHONE": null, "BIRTHDAY": null}, "interests": {"bbbb369575": false, "97bbc1227a": false, "d802d794f8": false, "b35e48738e": false, "44d2c158e3": false, "29f73b8209": false, "2010f3c101": false, "75f1cb79fd": false, "aa2fd02c59": false, "f7b60a3c3d": false, "7733d60f61": false, "cc454d76d6": false, "797533254b": false, "9ea08b864b": false, "e2e5fdcac9": false, "8eccc648d6": false, "a7c814599e": false, "20ef45c5d3": false, "1824f5d1a5": false, "644f34517f": false, "c57e1a9ff6": false, "b97fee61c8": false, "b9d16768e3": false, "810348679c": false, "43ebb04472": false, "73ee7c1d1b": false, "045738fa17": false, "0a7cbd4449": false, "fef00a4695": false, "4a19201dc9": false, "571a80ed60": false}, "stats": {"avg_open_rate": 1, "avg_click_rate": 0}, "ip_signup": null, "timestamp_signup": null, "ip_opt": "93.73.161.112", "timestamp_opt": "2022-12-27T08:34:39+00:00", "member_rating": 2, "last_changed": "2022-12-27T08:34:39+00:00", "language": null, "vip": false, "email_client": null, "location": {"latitude": 0, "longitude": 0, "gmtoff": 0, "dstoff": 0, "country_code": null, "timezone": null}, "list_id": "16d6ec4ffc", "segment_id": 13506120}, "emitted_at": 1701638879996} +{"stream": "segments", "data": {"id": 13506120, "name": "Customer", "member_count": 2, "type": "static", "created_at": "2022-12-27T08:12:06+00:00", "updated_at": "2022-12-27T08:12:55+00:00", "list_id": "16d6ec4ffc"}, "emitted_at": 1701638883128} +{"stream": "segments", "data": {"id": 13506124, "name": "Member", "member_count": 0, "type": "static", "created_at": "2022-12-27T08:12:06+00:00", "updated_at": "2022-12-27T08:28:44+00:00", "list_id": "16d6ec4ffc"}, "emitted_at": 1701638883129} +{"stream": "tags", "data": {"id": 13506128, "name": "2022", "list_id": "16d6ec4ffc"}, "emitted_at": 1699963804499} +{"stream": "unsubscribes", "data": {"email_id": "11273c9a5dc6ae6c5aaccfb77b2addfb", "email_address": "AirbyteMailchimpUser@gmail.com", "merge_fields": {"FNAME": "Joe", "LNAME": "Barry", "ADDRESS": {"addr1": "109 Barry St", "addr2": null, "city": "Gary", "state": "IN", "zip": "46401", "country": "US"}, "PHONE": null, "BIRTHDAY": null}, "vip": false, "timestamp": "2023-11-06T20:18:01+00:00", "reason": "Did not signup for list", "campaign_id": "7847cdaeff", "list_id": "16d6ec4ffc", "list_is_active": true}, "emitted_at": 1701638884243} diff --git a/airbyte-integrations/connectors/source-mailchimp/integration_tests/state.json b/airbyte-integrations/connectors/source-mailchimp/integration_tests/state.json index 77791e908b74..26b656926fd5 100644 --- a/airbyte-integrations/connectors/source-mailchimp/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-mailchimp/integration_tests/state.json @@ -24,16 +24,57 @@ "type": "STREAM", "stream": { "stream_state": { - "49d68626f3": { "timestamp": "2220-11-23T05:42:10+00:00" } + "7847cdaeff": { "timestamp": "2230-11-23T05:42:10+00:00" } }, "stream_descriptor": { "name": "email_activity" } } }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "16d6ec4ffc": { "last_changed": "2230-02-26T05:42:10+00:00" } + }, + "stream_descriptor": { "name": "list_members" } + } + }, { "type": "STREAM", "stream": { "stream_state": { "send_time": "2230-02-26T05:42:10+00:00" }, "stream_descriptor": { "name": "reports" } } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "13506120": { "last_changed": "2222-12-27T08:34:39+00:00" }, + "13506136": { "last_changed": "2222-12-27T08:34:39+00:00" }, + "14351124": { "last_changed": "2222-12-27T08:34:39+00:00" }, + "14351504": { "last_changed": "2222-12-27T07:56:47+00:00" }, + "14351128": { "last_changed": "2222-12-27T08:34:39+00:00" }, + "13506132": { "last_changed": "2222-12-27T08:34:39+00:00" } + }, + "stream_descriptor": { "name": "segment_members" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "16d6ec4ffc": { "updated_at": "2230-02-26T05:42:10+00:00" } + }, + "stream_descriptor": { "name": "segments" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "7847cdaeff": { "timestamp": "2231-09-26T05:42:10+00:00" } + }, + "stream_descriptor": { "name": "unsubscribes" } + } } ] diff --git a/airbyte-integrations/connectors/source-mailchimp/main.py b/airbyte-integrations/connectors/source-mailchimp/main.py index b95b566e6b8a..c61875fb7a72 100644 --- a/airbyte-integrations/connectors/source-mailchimp/main.py +++ b/airbyte-integrations/connectors/source-mailchimp/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_mailchimp import SourceMailchimp +from source_mailchimp.run import run if __name__ == "__main__": - source = SourceMailchimp() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-mailchimp/metadata.yaml b/airbyte-integrations/connectors/source-mailchimp/metadata.yaml index c99eeee394a8..4753d72e21da 100644 --- a/airbyte-integrations/connectors/source-mailchimp/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailchimp/metadata.yaml @@ -1,12 +1,18 @@ data: + ab_internal: + ql: 200 + sl: 200 allowedHosts: hosts: - "*.api.mailchimp.com" + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: api connectorType: source definitionId: b03a9f3e-22a5-11eb-adc1-0242ac120002 - dockerImageTag: 0.4.1 + dockerImageTag: 1.1.1 dockerRepository: airbyte/source-mailchimp + documentationUrl: https://docs.airbyte.com/integrations/sources/mailchimp githubIssueLabel: source-mailchimp icon: mailchimp.svg license: MIT @@ -16,12 +22,22 @@ data: enabled: true oss: enabled: true + releases: + breakingChanges: + 1.0.0: + message: + Version 1.0.0 introduces schema changes to all incremental streams. + A full schema refresh and data reset are required to upgrade to this version. + For more details, see our migration guide. + upgradeDeadline: "2024-01-10" releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/mailchimp + suggestedStreams: + streams: + - email_activity + - campaigns + - lists + - reports + supportLevel: certified tags: - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-mailchimp/setup.py b/airbyte-integrations/connectors/source-mailchimp/setup.py index f2973669a61e..0773da084484 100644 --- a/airbyte-integrations/connectors/source-mailchimp/setup.py +++ b/airbyte-integrations/connectors/source-mailchimp/setup.py @@ -9,6 +9,11 @@ setup( + entry_points={ + "console_scripts": [ + "source-mailchimp=source_mailchimp.run:run", + ], + }, name="source_mailchimp", description="Source implementation for Mailchimp.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/run.py b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/run.py new file mode 100644 index 000000000000..15226fdfeebd --- /dev/null +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_mailchimp import SourceMailchimp + + +def run(): + source = SourceMailchimp() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/automations.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/automations.json index 234d6e6965a5..27e691cf22d9 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/automations.json +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/automations.json @@ -8,10 +8,14 @@ "type": ["null", "string"] }, "create_time": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time", + "airbyte-type": "timestamp_with_timezone" }, "start_time": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "status": { "type": ["null", "string"] @@ -42,7 +46,10 @@ }, "conditions": { "type": ["null", "array"], - "items": {} + "items": { + "type": ["null", "object"], + "additionalProperties": true + } } } }, @@ -180,29 +187,6 @@ "type": ["null", "number"] } } - }, - "_links": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "rel": { - "type": ["null", "string"] - }, - "href": { - "type": ["null", "string"] - }, - "method": { - "type": ["null", "string"] - }, - "targetSchema": { - "type": ["null", "string"] - }, - "schema": { - "type": ["null", "string"] - } - } - } } } } diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/campaigns.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/campaigns.json index 453e53807a22..8d058b78e9e2 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/campaigns.json @@ -16,7 +16,7 @@ "readOnly": true }, "parent_campaign_id": { - "type": "string", + "type": ["null", "string"], "title": "Parent Campaign ID", "description": "If this campaign is the child of another campaign, this identifies the parent campaign. For Example, for RSS or Automation children.", "readOnly": true @@ -28,16 +28,18 @@ "type": "string", "title": "Create Time", "description": "The date and time the campaign was created in ISO 8601 format.", - "readOnly": true + "readOnly": true, + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "archive_url": { - "type": "string", + "type": ["null", "string"], "title": "Archive URL", "description": "The link to the campaign's archive version in ISO 8601 format.", "readOnly": true }, "long_archive_url": { - "type": "string", + "type": ["null", "string"], "title": "Long Archive URL", "description": "The original link to the campaign's archive version.", "readOnly": true @@ -52,13 +54,15 @@ "readOnly": true }, "send_time": { - "type": "string", + "type": ["null", "string"], "title": "Send Time", "description": "The date and time a campaign was sent.", - "readOnly": true + "readOnly": true, + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "content_type": { - "type": "string", + "type": ["null", "string"], "title": "Content Type", "description": "How the campaign's content is put together.", "enum": ["template", "html", "url", "multichannel"] @@ -81,7 +85,7 @@ "description": "List settings for the campaign.", "properties": { "list_id": { - "type": "string", + "type": ["null", "string"], "title": "List ID", "description": "The unique list id." }, @@ -92,13 +96,13 @@ "readOnly": true }, "list_name": { - "type": "string", + "type": ["null", "string"], "title": "List Name", "description": "The name of the list.", "readOnly": true }, "segment_text": { - "type": "string", + "type": ["null", "string"], "title": "Segment Text", "description": "A description of the [segment](https://mailchimp.com/help/create-and-send-to-a-segment/) used for the campaign. Formatted as a string marked up with HTML.", "readOnly": true @@ -120,27 +124,27 @@ "description": "The settings for your campaign, including subject, from name, reply-to address, and more.", "properties": { "subject_line": { - "type": "string", + "type": ["null", "string"], "title": "Campaign Subject Line", "description": "The subject line for the campaign." }, "preview_text": { - "type": "string", + "type": ["null", "string"], "title": "Campaign Preview Text", "description": "The preview text for the campaign." }, "title": { - "type": "string", + "type": ["null", "string"], "title": "Campaign Title", "description": "The title of the campaign." }, "from_name": { - "type": "string", + "type": ["null", "string"], "title": "From Name", "description": "The 'from' name on the campaign (not an email address)." }, "reply_to": { - "type": "string", + "type": ["null", "string"], "title": "Reply To Address", "description": "The reply-to email address for the campaign." }, @@ -150,12 +154,12 @@ "description": "Use Mailchimp Conversation feature to manage out-of-office replies." }, "to_name": { - "type": "string", + "type": ["null", "string"], "title": "To Name", "description": "The campaign's custom 'To' name. Typically the first name [merge field](https://mailchimp.com/help/getting-started-with-merge-tags/)." }, "folder_id": { - "type": "string", + "type": ["null", "string"], "title": "Folder ID", "description": "If the campaign is listed in a folder, the id for that folder." }, @@ -184,7 +188,7 @@ "title": "Auto Post to Facebook", "description": "An array of [Facebook](https://mailchimp.com/help/connect-or-disconnect-the-facebook-integration/) page ids to auto-post to.", "items": { - "type": "string" + "type": ["null", "string"] } }, "fb_comments": { @@ -218,19 +222,19 @@ "description": "The settings specific to A/B test campaigns.", "properties": { "winning_combination_id": { - "type": "string", + "type": ["null", "string"], "title": "Winning Combination ID", "description": "ID for the winning combination.", "readOnly": true }, "winning_campaign_id": { - "type": "string", + "type": ["null", "string"], "title": "Winning Campaign ID", "description": "ID of the campaign that was sent to the remaining recipients based on the winning combination.", "readOnly": true }, "winner_criteria": { - "type": "string", + "type": ["null", "string"], "title": "Winning Criteria", "description": "The combination that performs the best. This may be determined automatically by click rate, open rate, or total revenue -- or you may choose manually based on the reporting data you find the most valuable. For Multivariate Campaigns testing send_time, winner_criteria is ignored. For Multivariate Campaigns with 'manual' as the winner_criteria, the winner must be chosen in the Mailchimp web application.", "enum": ["opens", "clicks", "manual", "total_revenue"] @@ -250,7 +254,7 @@ "title": "Subject Lines", "description": "The possible subject lines to test. If no subject lines are provided, settings.subject_line will be used.", "items": { - "type": "string" + "type": ["null", "string"] } }, "send_times": { @@ -258,7 +262,9 @@ "title": "Send Times", "description": "The possible send times to test. The times provided should be in the format YYYY-MM-DD HH:MM:SS. If send_times are provided to test, the test_size will be set to 100% and winner_criteria will be ignored.", "items": { - "type": "string" + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" } }, "from_names": { @@ -266,7 +272,7 @@ "title": "From Names", "description": "The possible from names. The number of from_names provided must match the number of reply_to_addresses. If no from_names are provided, settings.from_name will be used.", "items": { - "type": "string" + "type": ["null", "string"] } }, "reply_to_addresses": { @@ -274,7 +280,7 @@ "title": "Reply To Addresses", "description": "The possible reply-to addresses. The number of reply_to_addresses provided must match the number of from_names. If no reply_to_addresses are provided, settings.reply_to will be used.", "items": { - "type": "string" + "type": ["null", "string"] } }, "contents": { @@ -282,7 +288,7 @@ "title": "Content Descriptions", "description": "Descriptions of possible email contents. To set campaign contents, make a PUT request to /campaigns/{campaign_id}/content with the field 'variate_contents'.", "items": { - "type": "string" + "type": ["null", "string"] }, "readOnly": true }, @@ -295,7 +301,7 @@ "type": "object", "properties": { "id": { - "type": "string", + "type": ["null", "string"], "title": "ID", "description": "Unique ID for the combination." }, @@ -365,12 +371,12 @@ "description": "Whether to enable [eCommerce360](https://mailchimp.com/help/connect-your-online-store-to-mailchimp/) tracking." }, "google_analytics": { - "type": "string", + "type": ["null", "string"], "title": "Google Analytics Tracking", "description": "The custom slug for [Google Analytics](https://mailchimp.com/help/integrate-google-analytics-with-mailchimp/) tracking (max of 50 bytes)." }, "clicktale": { - "type": "string", + "type": ["null", "string"], "title": "ClickTale Analytics Tracking", "description": "The custom slug for [ClickTale](https://mailchimp.com/help/additional-tracking-options-for-campaigns/) tracking (max of 50 bytes)." }, @@ -411,13 +417,13 @@ "description": "[RSS](https://mailchimp.com/help/share-your-blog-posts-with-mailchimp/) options for a campaign.", "properties": { "feed_url": { - "type": "string", + "type": ["null", "string"], "title": "Feed URL", "format": "uri", "description": "The URL for the RSS feed." }, "frequency": { - "type": "string", + "type": ["null", "string"], "title": "Frequency", "description": "The frequency of the RSS Campaign.", "enum": ["daily", "weekly", "monthly"] @@ -477,7 +483,7 @@ } }, "weekly_send_day": { - "type": "string", + "type": ["null", "string"], "enum": [ "sunday", "monday", @@ -500,10 +506,12 @@ } }, "last_sent": { - "type": "string", + "type": ["null", "string"], "title": "Last Sent", "description": "The date the campaign was last sent.", - "readOnly": true + "readOnly": true, + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "constrain_rss_img": { "type": "boolean", @@ -519,19 +527,19 @@ "readOnly": true, "properties": { "split_test": { - "type": "string", + "type": ["null", "string"], "title": "Split Test", "description": "The type of AB split to run.", "enum": ["subject", "from_name", "schedule"] }, "pick_winner": { - "type": "string", + "type": ["null", "string"], "title": "Pick Winner", "description": "How we should evaluate a winner. Based on 'opens', 'clicks', or 'manual'.", "enum": ["opens", "clicks", "manual"] }, "wait_units": { - "type": "string", + "type": ["null", "string"], "title": "Wait Time", "description": "How unit of time for measuring the winner ('hours' or 'days'). This cannot be changed after a campaign is sent.", "enum": ["hours", "days"] @@ -549,47 +557,51 @@ "description": "The size of the split groups. Campaigns split based on 'schedule' are forced to have a 50/50 split. Valid split integers are between 1-50." }, "from_name_a": { - "type": "string", + "type": ["null", "string"], "title": "From Name Group A", "description": "For campaigns split on 'From Name', the name for Group A." }, "from_name_b": { - "type": "string", + "type": ["null", "string"], "title": "From Name Group B", "description": "For campaigns split on 'From Name', the name for Group B." }, "reply_email_a": { - "type": "string", + "type": ["null", "string"], "title": "Reply Email Group A", "description": "For campaigns split on 'From Name', the reply-to address for Group A." }, "reply_email_b": { - "type": "string", + "type": ["null", "string"], "title": "Reply Email Group B", "description": "For campaigns split on 'From Name', the reply-to address for Group B." }, "subject_a": { - "type": "string", + "type": ["null", "string"], "title": "Subject Line Group A", "description": "For campaigns split on 'Subject Line', the subject line for Group A." }, "subject_b": { - "type": "string", + "type": ["null", "string"], "title": "Subject Line Group B", "description": "For campaigns split on 'Subject Line', the subject line for Group B." }, "send_time_a": { - "type": "string", + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone", "title": "Send Time Group A", "description": "The send time for Group A." }, "send_time_b": { - "type": "string", + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone", "title": "Send Time Group B", "description": "The send time for Group B." }, "send_time_winner": { - "type": "string", + "type": ["null", "string"], "title": "Send Time Winner", "description": "The send time for the winning version." } @@ -601,17 +613,17 @@ "description": "The preview for the campaign, rendered by social networks like Facebook and Twitter. [Learn more](https://mailchimp.com/help/enable-and-customize-social-cards/).", "properties": { "image_url": { - "type": "string", + "type": ["null", "string"], "title": "Image URL", "description": "The url for the header image for the card." }, "description": { - "type": "string", + "type": ["null", "string"], "title": "Campaign Description", "description": "A short summary of the campaign to display." }, "title": { - "type": "string", + "type": ["null", "string"], "title": "Title", "description": "The title for the card. Typically the subject line of the campaign." } @@ -703,7 +715,7 @@ "readOnly": true }, "status": { - "type": "string", + "type": ["null", "string"], "title": "Campaign Delivery Status", "description": "The current state of a campaign delivery.", "enum": ["delivering", "delivered", "canceling", "canceled"], diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/email_activity.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/email_activity.json index 4c4f978fc423..b416956c5427 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/email_activity.json +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/email_activity.json @@ -44,7 +44,8 @@ "type": ["string", "null"], "title": "Action date and time", "description": "The date and time recorded for the action in ISO 8601 format.", - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "url": { "type": ["string", "null"], diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/interest_categories.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/interest_categories.json new file mode 100644 index 000000000000..7d808ecd6ab2 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/interest_categories.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "list_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "display_order": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/interests.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/interests.json new file mode 100644 index 000000000000..b936326faa09 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/interests.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "category_id": { + "type": ["null", "string"] + }, + "list_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "subscriber_count": { + "type": ["null", "string"] + }, + "display_order": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/list_members.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/list_members.json new file mode 100644 index 000000000000..50376c80b74a --- /dev/null +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/list_members.json @@ -0,0 +1,185 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "string"] + }, + "email_address": { + "type": ["null", "string"] + }, + "unique_email_id": { + "type": ["null", "string"] + }, + "contact_id": { + "type": ["null", "string"] + }, + "full_name": { + "type": ["null", "string"] + }, + "web_id": { + "type": ["null", "integer"] + }, + "email_type": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "unsubscribe_reason": { + "type": ["null", "string"] + }, + "consents_to_one_to_one_messaging": { + "type": ["null", "boolean"] + }, + "merge_fields": { + "type": ["null", "object"], + "additionalProperties": true + }, + "interests": { + "type": ["null", "object"], + "additionalProperties": true + }, + "stats": { + "type": ["null", "object"], + "properties": { + "avg_open_rate": { + "type": ["null", "number"] + }, + "avg_click_rate": { + "type": ["null", "number"] + }, + "ecommerce_data": { + "type": ["null", "object"], + "properties": { + "total_revenue": { + "type": ["null", "number"] + }, + "number_of_orders": { + "type": ["null", "number"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "ip_signup": { + "type": ["null", "string"] + }, + "timestamp_signup": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "ip_opt": { + "type": ["null", "string"] + }, + "timestamp_opt": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "member_rating": { + "type": ["null", "integer"] + }, + "last_changed": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "language": { + "type": ["null", "string"] + }, + "vip": { + "type": ["null", "boolean"] + }, + "email_client": { + "type": ["null", "string"] + }, + "location": { + "type": ["null", "object"], + "properties": { + "latitude": { + "type": ["null", "number"] + }, + "longitude": { + "type": ["null", "number"] + }, + "gmtoff": { + "type": ["null", "integer"] + }, + "dstoff": { + "type": ["null", "integer"] + }, + "country_code": { + "type": ["null", "string"] + }, + "timezone": { + "type": ["null", "string"] + }, + "region": { + "type": ["null", "string"] + } + } + }, + "marketing_permissions": { + "type": ["null", "object"], + "properties": { + "marketing_permission_id": { + "type": ["null", "string"] + }, + "text": { + "type": ["null", "string"] + }, + "enabled": { + "type": ["null", "boolean"] + } + } + }, + "last_note": { + "type": ["null", "object"], + "properties": { + "note_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "created_by": { + "type": ["null", "string"] + }, + "note": { + "type": ["null", "string"] + } + } + }, + "source": { + "type": ["null", "string"] + }, + "tags_count": { + "type": ["null", "integer"] + }, + "tags": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + } + }, + "list_id": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/lists.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/lists.json index f0a21ce1a299..01cd5b3e1881 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/lists.json +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/lists.json @@ -16,7 +16,7 @@ "readOnly": true }, "name": { - "type": "string", + "type": ["null", "string"], "title": "List Name", "description": "The name of the list." }, @@ -26,49 +26,49 @@ "description": "[Contact information displayed in campaign footers](https://mailchimp.com/help/about-campaign-footers/) to comply with international spam laws.", "properties": { "company": { - "type": "string", + "type": ["null", "string"], "title": "Company Name", "description": "The company name for the list." }, "address1": { - "type": "string", + "type": ["null", "string"], "title": "Address", "description": "The street address for the list contact." }, "address2": { - "type": "string", + "type": ["null", "string"], "title": "Address", "description": "The street address for the list contact." }, "city": { - "type": "string", + "type": ["null", "string"], "title": "City", "description": "The city for the list contact." }, "state": { - "type": "string", + "type": ["null", "string"], "title": "State", "description": "The state for the list contact." }, "zip": { - "type": "string", + "type": ["null", "string"], "title": "Postal Code", "description": "The postal or zip code for the list contact." }, "country": { - "type": "string", + "type": ["null", "string"], "title": "Country Code", "description": "A two-character ISO3166 country code. Defaults to US if invalid." }, "phone": { - "type": "string", + "type": ["null", "string"], "title": "Phone Number", "description": "The phone number for the list contact." } } }, "permission_reminder": { - "type": "string", + "type": ["null", "string"], "title": "Permission Reminder", "description": "The [permission reminder](https://mailchimp.com/help/edit-the-permission-reminder/) for the list." }, @@ -84,35 +84,35 @@ "description": "[Default values for campaigns](https://mailchimp.com/help/edit-your-emails-subject-preview-text-from-name-or-from-email-address/) created for this list.", "properties": { "from_name": { - "type": "string", + "type": ["null", "string"], "title": "Sender's Name", "description": "The default from name for campaigns sent to this list." }, "from_email": { - "type": "string", + "type": ["null", "string"], "title": "Sender's Email Address", "description": "The default from email for campaigns sent to this list." }, "subject": { - "type": "string", + "type": ["null", "string"], "title": "Subject", "description": "The default subject line for campaigns sent to this list." }, "language": { - "type": "string", + "type": ["null", "string"], "title": "Language", "description": "The default language for this lists's forms." } } }, "notify_on_subscribe": { - "type": "string", + "type": ["null", "string"], "title": "Notify on Subscribe", "description": "The email address to send [subscribe notifications](https://mailchimp.com/help/change-subscribe-and-unsubscribe-notifications/) to.", "default": false }, "notify_on_unsubscribe": { - "type": "string", + "type": ["null", "string"], "title": "Notify on Unsubscribe", "description": "The email address to send [unsubscribe notifications](https://mailchimp.com/help/change-subscribe-and-unsubscribe-notifications/) to.", "default": false @@ -122,6 +122,7 @@ "title": "Creation Date", "description": "The date and time that this list was created in ISO 8601 format.", "format": "date-time", + "airbyte_type": "timestamp_with_timezone", "readOnly": true }, "list_rating": { @@ -136,25 +137,25 @@ "description": "Whether the list supports [multiple formats for emails](https://mailchimp.com/help/change-list-name-and-defaults/). When set to `true`, subscribers can choose whether they want to receive HTML or plain-text emails. When set to `false`, subscribers will receive HTML emails, with a plain-text alternative backup." }, "subscribe_url_short": { - "type": "string", + "type": ["null", "string"], "title": "Subscribe URL Short", "description": "Our [EepURL shortened](https://mailchimp.com/help/share-your-signup-form/) version of this list's subscribe form.", "readOnly": true }, "subscribe_url_long": { - "type": "string", + "type": ["null", "string"], "title": "Subscribe URL Long", "description": "The full version of this list's subscribe form (host will vary).", "readOnly": true }, "beamer_address": { - "type": "string", + "type": ["null", "string"], "title": "Beamer Address", "description": "The list's [Email Beamer](https://mailchimp.com/help/use-email-beamer-to-create-a-campaign/) address.", "readOnly": true }, "visibility": { - "type": "string", + "type": ["null", "string"], "title": "Visibility", "enum": ["pub", "prv"], "description": "Whether this list is [public or private](https://mailchimp.com/help/about-list-publicity/)." @@ -183,7 +184,7 @@ "title": "Modules", "description": "Any list-specific modules installed for this list.", "items": { - "type": "string" + "type": ["null", "string"] }, "readOnly": true }, @@ -242,10 +243,12 @@ "readOnly": true }, "campaign_last_sent": { - "type": "string", + "type": ["null", "string"], "title": "Campaign Last Sent", "description": "The date and time the last campaign was sent to this list in ISO 8601 format. This is updated when a campaign is sent to 10 or more recipients.", - "readOnly": true + "readOnly": true, + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "merge_field_count": { "type": "integer", @@ -284,16 +287,20 @@ "readOnly": true }, "last_sub_date": { - "type": "string", + "type": ["null", "string"], "title": "Date of Last List Subscribe", "description": "The date and time of the last time someone subscribed to this list in ISO 8601 format.", - "readOnly": true + "readOnly": true, + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "last_unsub_date": { - "type": "string", + "type": ["null", "string"], "title": "Date of Last List Unsubscribe", "description": "The date and time of the last time someone unsubscribed from this list in ISO 8601 format.", - "readOnly": true + "readOnly": true, + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" } } } diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/reports.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/reports.json index d94cb73ad0df..940b0a83202c 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/reports.json +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/reports.json @@ -9,13 +9,13 @@ "description": "A string that uniquely identifies this campaign." }, "campaign_title": { - "type": "string", + "type": ["null", "string"], "title": "Campaign Title", "description": "The title of the campaign.", "readOnly": true }, "type": { - "type": "string", + "type": ["null", "string"], "title": "Campaign Type", "description": "The type of campaign (regular, plain-text, ab_split, rss, automation, variate, or auto)." }, @@ -32,19 +32,19 @@ "readOnly": true }, "list_name": { - "type": "string", + "type": ["null", "string"], "title": "List Name", "description": "The name of the list.", "readOnly": true }, "subject_line": { - "type": "string", + "type": ["null", "string"], "title": "Campaign Subject Line", "description": "The subject line for the campaign.", "readOnly": true }, "preview_text": { - "type": "string", + "type": ["null", "string"], "title": "Campaign Preview Text", "description": "The preview text for the campaign." }, @@ -65,15 +65,17 @@ "readOnly": true }, "send_time": { - "type": "string", + "type": ["null", "string"], "format": "date-time", + "airbyte_type": "timestamp_with_timezone", "title": "Send Time", "description": "The date and time a campaign was sent in ISO 8601 format.", "readOnly": true }, "rss_last_send": { - "type": "string", + "type": ["null", "string"], "format": "date-time", + "airbyte_type": "timestamp_with_timezone", "title": "RSS Last Send", "description": "For RSS campaigns, the date and time of the last send in ISO 8601 format.", "readOnly": true @@ -138,8 +140,9 @@ "description": "The number of unique opens divided by the total number of successful deliveries." }, "last_open": { - "type": "string", + "type": ["null", "string"], "format": "date-time", + "airbyte_type": "timestamp_with_timezone", "title": "Last Open", "description": "The date and time of the last recorded open in ISO 8601 format." } @@ -171,8 +174,9 @@ "description": "The number of unique clicks divided by the total number of successful deliveries." }, "last_click": { - "type": "string", + "type": ["null", "string"], "format": "date-time", + "airbyte_type": "timestamp_with_timezone", "title": "Last Click", "description": "The date and time of the last recorded click for the campaign in ISO 8601 format." } @@ -206,7 +210,7 @@ "description": "The average campaign statistics for your industry.", "properties": { "type": { - "type": "string", + "type": ["null", "string"], "title": "Industry Type", "description": "The type of business industry associated with your account. For example: retail, education, etc." }, @@ -319,9 +323,11 @@ "description": "Opens for Campaign A." }, "last_open": { - "type": "string", + "type": ["null", "string"], "title": "Last Open", - "description": "The last open for Campaign A." + "description": "The last open for Campaign A.", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "unique_opens": { "type": "integer", @@ -371,9 +377,11 @@ "description": "Opens for Campaign B." }, "last_open": { - "type": "string", + "type": ["null", "string"], "title": "Last Open", - "description": "The last open for Campaign B." + "description": "The last open for Campaign B.", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "unique_opens": { "type": "integer", @@ -402,8 +410,9 @@ "description": "The number of opens." }, "last_open": { - "type": "string", + "type": ["null", "string"], "format": "date-time", + "airbyte_type": "timestamp_with_timezone", "title": "Last Open", "description": "The date and time of the last open in ISO 8601 format." }, @@ -418,8 +427,9 @@ "description": "The number of clicks." }, "last_click": { - "type": "string", + "type": ["null", "string"], "format": "date-time", + "airbyte_type": "timestamp_with_timezone", "title": "Last Click", "description": "The date and time of the last click in ISO 8601 format." }, @@ -444,8 +454,9 @@ "type": "object", "properties": { "timestamp": { - "type": "string", + "type": ["null", "string"], "format": "date-time", + "airbyte_type": "timestamp_with_timezone", "title": "Timestamp", "description": "The date and time for the series in ISO 8601 format." }, @@ -473,13 +484,13 @@ "description": "The url and password for the [VIP report](https://mailchimp.com/help/share-a-campaign-report/).", "properties": { "share_url": { - "type": "string", + "type": ["null", "string"], "title": "Report URL", "description": "The URL for the VIP report.", "readOnly": true }, "share_password": { - "type": "string", + "type": ["null", "string"], "title": "Report Password", "description": "If password protected, the password for the VIP report.", "readOnly": true @@ -510,7 +521,7 @@ "readOnly": true }, "currency_code": { - "type": "string", + "type": ["null", "string"], "title": "Three letter currency code for this user", "readOnly": true, "example": "USD" @@ -535,7 +546,7 @@ "readOnly": true }, "status": { - "type": "string", + "type": ["null", "string"], "title": "Campaign Delivery Status", "description": "The current state of a campaign delivery.", "enum": ["delivering", "delivered", "canceling", "canceled"], diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/segment_members.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/segment_members.json new file mode 100644 index 000000000000..8766876fd2b7 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/segment_members.json @@ -0,0 +1,122 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "string"] + }, + "email_address": { + "type": ["null", "string"] + }, + "unique_email_id": { + "type": ["null", "string"] + }, + "email_type": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "merge_fields": { + "type": ["null", "object"], + "additionalProperties": true + }, + "interests": { + "type": ["null", "object"], + "additionalProperties": true + }, + "stats": { + "type": ["null", "object"], + "properties": { + "avg_open_rate": { + "type": ["null", "number"] + }, + "avg_click_rate": { + "type": ["null", "number"] + } + } + }, + "ip_signup": { + "type": ["null", "string"] + }, + "timestamp_signup": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "ip_opt": { + "type": ["null", "string"] + }, + "timestamp_opt": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "member_rating": { + "type": ["null", "integer"] + }, + "last_changed": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "language": { + "type": ["null", "string"] + }, + "vip": { + "type": ["null", "boolean"] + }, + "email_client": { + "type": ["null", "string"] + }, + "location": { + "type": ["null", "object"], + "properties": { + "latitude": { + "type": ["null", "number"] + }, + "longitude": { + "type": ["null", "number"] + }, + "gmtoff": { + "type": ["null", "integer"] + }, + "dstoff": { + "type": ["null", "integer"] + }, + "country_code": { + "type": ["null", "string"] + }, + "timezone": { + "type": ["null", "string"] + } + } + }, + "last_note": { + "type": ["null", "object"], + "properties": { + "note_id": { + "type": ["null", "integer"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "created_by": { + "type": ["null", "string"] + }, + "note": { + "type": ["null", "string"] + } + } + }, + "list_id": { + "type": ["null", "string"] + }, + "segment_id": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/segments.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/segments.json new file mode 100644 index 000000000000..8840817de2e9 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/segments.json @@ -0,0 +1,58 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "member_count": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "options": { + "type": ["null", "object"], + "properties": { + "match": { + "type": ["null", "string"] + }, + "conditions": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "condition_type": { + "type": ["null", "string"] + }, + "field": { + "type": ["null", "string"] + }, + "op": { + "type": ["null", "string"] + } + } + } + } + } + }, + "list_id": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/shared/campaignType.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/shared/campaignType.json index 0171d498babc..4e7ad8a5978f 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/shared/campaignType.json +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/shared/campaignType.json @@ -2,5 +2,12 @@ "type": "string", "title": "Campaign Type", "description": "There are four types of [campaigns](https://mailchimp.com/help/getting-started-with-campaigns/) you can create in Mailchimp. A/B Split campaigns have been deprecated and variate campaigns should be used instead.", - "enum": ["regular", "plaintext", "absplit", "rss", "variate"] + "enum": [ + "automation-email", + "regular", + "plaintext", + "absplit", + "rss", + "variate" + ] } diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/tags.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/tags.json new file mode 100644 index 000000000000..93e81d28f940 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/tags.json @@ -0,0 +1,15 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "list_id": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/unsubscribes.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/unsubscribes.json new file mode 100644 index 000000000000..ead264a0c180 --- /dev/null +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/schemas/unsubscribes.json @@ -0,0 +1,37 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "email_id": { + "type": ["null", "string"] + }, + "email_address": { + "type": ["null", "string"] + }, + "merge_fields": { + "type": ["null", "object"], + "additionalProperties": true + }, + "vip": { + "type": ["null", "boolean"] + }, + "timestamp": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" + }, + "reason": { + "type": ["null", "string"] + }, + "campaign_id": { + "type": ["null", "string"] + }, + "list_id": { + "type": ["null", "string"] + }, + "list_is_active": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py index 223f64577890..0edf00993e5f 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py @@ -4,28 +4,56 @@ import base64 +import re from typing import Any, List, Mapping, Tuple +import pendulum import requests from airbyte_cdk import AirbyteLogger from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from pendulum.parsing.exceptions import ParserError from requests.auth import AuthBase -from .streams import Automations, Campaigns, EmailActivity, Lists, Reports +from .streams import ( + Automations, + Campaigns, + EmailActivity, + InterestCategories, + Interests, + ListMembers, + Lists, + Reports, + SegmentMembers, + Segments, + Tags, + Unsubscribes, +) class MailChimpAuthenticator: @staticmethod - def get_server_prefix(access_token: str) -> str: + def get_oauth_data_center(access_token: str) -> str: + """ + Every Mailchimp API request must be sent to a specific data center. + The data center is already embedded in API keys, but not OAuth access tokens. + This method retrieves the data center for OAuth credentials. + """ try: response = requests.get( "https://login.mailchimp.com/oauth2/metadata", headers={"Authorization": "OAuth {}".format(access_token)} ) + + # Requests to this endpoint will return a 200 status code even if the access token is invalid. + error = response.json().get("error") + if error == "invalid_token": + raise ValueError("The access token you provided was invalid. Please check your credentials and try again.") return response.json()["dc"] + + # Handle any other exceptions that may occur. except Exception as e: - raise Exception(f"Cannot retrieve server_prefix for you account. \n {repr(e)}") + raise Exception(f"An error occured while retrieving the data center for your account. \n {repr(e)}") def get_auth(self, config: Mapping[str, Any]) -> AuthBase: authorization = config.get("credentials", {}) @@ -35,7 +63,7 @@ def get_auth(self, config: Mapping[str, Any]) -> AuthBase: # See https://mailchimp.com/developer/marketing/docs/fundamentals/#api-structure apikey = authorization.get("apikey") or config.get("apikey") if not apikey: - raise Exception("No apikey in creds") + raise Exception("Please provide a valid API key for authentication.") auth_string = f"anystring:{apikey}".encode("utf8") b64_encoded = base64.b64encode(auth_string).decode("utf8") auth = TokenAuthenticator(token=b64_encoded, auth_method="Basic") @@ -44,7 +72,7 @@ def get_auth(self, config: Mapping[str, Any]) -> AuthBase: elif auth_type == "oauth2.0": access_token = authorization["access_token"] auth = TokenAuthenticator(token=access_token, auth_method="Bearer") - auth.data_center = self.get_server_prefix(access_token) + auth.data_center = self.get_oauth_data_center(access_token) else: raise Exception(f"Invalid auth type: {auth_type}") @@ -53,21 +81,69 @@ def get_auth(self, config: Mapping[str, Any]) -> AuthBase: class SourceMailchimp(AbstractSource): + def _validate_start_date(self, config: Mapping[str, Any]): + start_date = config.get("start_date") + + if start_date: + pattern = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z") + if not pattern.match(start_date): # Compare against the pattern descriptor. + return "Please check the format of the start date against the pattern descriptor." + + try: # Handle invalid dates. + parsed_start_date = pendulum.parse(start_date) + except ParserError: + return "The provided start date is not a valid date. Please check the date you input and try again." + + if parsed_start_date > pendulum.now("UTC"): # Handle future start date. + return "The start date cannot be greater than the current date." + + return None + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: + # First, check for a valid start date if it is provided + start_date_validation_error = self._validate_start_date(config) + if start_date_validation_error: + return False, start_date_validation_error + try: authenticator = MailChimpAuthenticator().get_auth(config) - requests.get(f"https://{authenticator.data_center}.api.mailchimp.com/3.0/ping", headers=authenticator.get_auth_header()) + response = requests.get( + f"https://{authenticator.data_center}.api.mailchimp.com/3.0/ping", headers=authenticator.get_auth_header() + ) + + # A successful response will return a simple JSON object with a single key: health_status. + # Otherwise, errors are returned as a JSON object with keys: + # {type, title, status, detail, instance} + + if not response.json().get("health_status"): + error_title = response.json().get("title", "Unknown Error") + error_details = response.json().get("details", "An unknown error occurred. Please verify your credentials and try again.") + return False, f"Encountered an error while connecting to Mailchimp. Type: {error_title}. Details: {error_details}" return True, None + + # Handle any other exceptions that may occur. except Exception as e: return False, repr(e) def streams(self, config: Mapping[str, Any]) -> List[Stream]: authenticator = MailChimpAuthenticator().get_auth(config) campaign_id = config.get("campaign_id") + start_date = config.get("start_date") + + lists = Lists(authenticator=authenticator, start_date=start_date) + interest_categories = InterestCategories(authenticator=authenticator, parent=lists) + return [ - Lists(authenticator=authenticator), - Campaigns(authenticator=authenticator), - Automations(authenticator=authenticator), - EmailActivity(authenticator=authenticator, campaign_id=campaign_id), - Reports(authenticator=authenticator), + Automations(authenticator=authenticator, start_date=start_date), + Campaigns(authenticator=authenticator, start_date=start_date), + EmailActivity(authenticator=authenticator, start_date=start_date, campaign_id=campaign_id), + interest_categories, + Interests(authenticator=authenticator, parent=interest_categories), + lists, + ListMembers(authenticator=authenticator, start_date=start_date), + Reports(authenticator=authenticator, start_date=start_date), + SegmentMembers(authenticator=authenticator, start_date=start_date), + Segments(authenticator=authenticator, start_date=start_date), + Tags(authenticator=authenticator, parent=lists), + Unsubscribes(authenticator=authenticator, start_date=start_date, campaign_id=campaign_id), ] diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/spec.json b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/spec.json index c18777fcd36b..f88649faa153 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/spec.json +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/spec.json @@ -61,6 +61,15 @@ } ] }, + "start_date": { + "title": "Incremental Sync Start Date", + "description": "The date from which you want to start syncing data for Incremental streams. Only records that have been created or modified since this date will be synced. If left blank, all data will by synced.", + "type": "string", + "format": "date-time", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z$", + "pattern_descriptor": "YYYY-MM-DDTHH:MM:SS.000Z", + "examples": ["2020-01-01T00:00:00.000Z"] + }, "campaign_id": { "type": "string", "title": "ID of a campaign to sync email activities", diff --git a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py index fa5672a6da91..158eaf1e8b47 100644 --- a/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py +++ b/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/streams.py @@ -7,11 +7,11 @@ from abc import ABC, abstractmethod from typing import Any, Iterable, List, Mapping, MutableMapping, Optional +import pendulum import requests from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.core import StreamData -from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream logger = logging.getLogger("airbyte") @@ -29,10 +29,6 @@ def __init__(self, **kwargs): def url_base(self) -> str: return f"https://{self.data_center}.api.mailchimp.com/3.0/" - @property - def availability_strategy(self) -> Optional["AvailabilityStrategy"]: - return None - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: decoded_response = response.json() api_data = decoded_response[self.data_field] @@ -50,7 +46,8 @@ def request_params( next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - params = {"count": self.page_size} + # The ._links field is returned by most Mailchimp endpoints and contains non-relevant schema metadata. + params = {"count": self.page_size, "exclude_fields": f"{self.data_field}._links"} # Handle pagination by inserting the next page's token in the request parameters if next_page_token: @@ -64,7 +61,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp @property @abstractmethod def data_field(self) -> str: - """The responce entry that contains useful data""" + """The response entry that contains useful data""" pass def read_records( @@ -85,6 +82,10 @@ def read_records( class IncrementalMailChimpStream(MailChimpStream, ABC): state_checkpoint_interval = math.inf + def __init__(self, **kwargs): + self.start_date = kwargs.pop("start_date", None) + super().__init__(**kwargs) + @property @abstractmethod def cursor_field(self) -> str: @@ -102,6 +103,23 @@ def filter_field(self): def sort_field(self): return self.cursor_field + def filter_empty_fields(self, element: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Many Mailchimp endpoints return empty strings instead of null values. + This causes validation errors on datetime columns, so for safety, we need to check for empty strings and set their value to None/null. + This method recursively traverses each element in a record and replaces any "" values with None, based on three conditions: + + 1. If the element is a dictionary, apply the method recursively to each value in the dictionary. + 2. If the element is a list, apply the method recursively to each item in the list. + 3. If the element is a string, check if it is an empty string. If so, replace it with None. + """ + + if isinstance(element, dict): + element = {k: self.filter_empty_fields(v) if v != "" else None for k, v in element.items()} + elif isinstance(element, list): + element = [self.filter_empty_fields(v) for v in element] + return element + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: """ Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object @@ -116,11 +134,37 @@ def stream_slices( ) -> Iterable[Optional[Mapping[str, Any]]]: slice_ = {} stream_state = stream_state or {} - cursor_value = stream_state.get(self.cursor_field) + cursor_value = self.get_filter_date(self.start_date, stream_state.get(self.cursor_field)) if cursor_value: slice_[self.filter_field] = cursor_value yield slice_ + @staticmethod + def get_filter_date(start_date: str, state_date: str) -> str: + """ + Calculate the filter date to pass in the request parameters by comparing the start_date + with the value of state obtained from the stream_slice. + If only one value exists, use it by default. Otherwise, return None. + If no filter_date is provided, the API will fetch all available records. + """ + + start_date_parsed = pendulum.parse(start_date).to_iso8601_string() if start_date else None + state_date_parsed = pendulum.parse(state_date).to_iso8601_string() if state_date else None + + # Return the max of the two dates if both are present. Otherwise return whichever is present, or None. + if start_date_parsed or state_date_parsed: + return max(filter(None, [start_date_parsed, state_date_parsed]), default=None) + + def filter_old_records(self, records: Iterable, filter_date) -> Iterable: + """ + Filters out records with older cursor_values than the filter_date. + This can be used to enforce the filter for incremental streams that do not support sorting/filtering via query params. + """ + for record in records: + record_cursor_value = record.get(self.cursor_field) + if not filter_date or record_cursor_value >= filter_date: + yield record + def request_params(self, stream_state=None, stream_slice=None, **kwargs): stream_state = stream_state or {} stream_slice = stream_slice or {} @@ -129,6 +173,57 @@ def request_params(self, stream_state=None, stream_slice=None, **kwargs): params.update(default_params) return params + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + response = super().parse_response(response, **kwargs) + for record in response: + yield self.filter_empty_fields(record) + + +class MailChimpListSubStream(IncrementalMailChimpStream): + """ + Base class for incremental Mailchimp streams that are children of the Lists stream. + """ + + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + stream_state = stream_state or {} + parent = Lists(authenticator=self.authenticator).read_records(sync_mode=SyncMode.full_refresh) + for parent_record in parent: + slice = {"list_id": parent_record["id"]} + cursor_value = self.get_filter_date(self.start_date, stream_state.get(parent_record["id"], {}).get(self.cursor_field)) + if cursor_value: + slice[self.filter_field] = cursor_value + yield slice + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + list_id = stream_slice.get("list_id") + return f"lists/{list_id}/{self.data_field}" + + def request_params(self, stream_state=None, stream_slice=None, **kwargs) -> MutableMapping[str, Any]: + params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, **kwargs) + + # Get the current state value for this list_id, if it exists + # Then, use the value in state to filter the request + current_slice = stream_slice.get("list_id") + filter_date = stream_state.get(current_slice) + if filter_date: + params[self.filter_field] = filter_date.get(self.cursor_field) + return params + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + current_stream_state = current_stream_state or {} + list_id = latest_record.get("list_id") + latest_cursor_value = latest_record.get(self.cursor_field) + + # Get the current state value for this list, if it exists + list_state = current_stream_state.get(list_id, {}) + current_cursor_value = list_state.get(self.cursor_field, latest_cursor_value) + + # Update the cursor value and set it in state + updated_cursor_value = max(current_cursor_value, latest_cursor_value) + current_stream_state[list_id] = {self.cursor_field: updated_cursor_value} + + return current_stream_state + class Lists(IncrementalMailChimpStream): cursor_field = "date_created" @@ -178,7 +273,8 @@ def stream_slices( campaigns = Campaigns(authenticator=self.authenticator).read_records(sync_mode=SyncMode.full_refresh) for campaign in campaigns: slice_ = {"campaign_id": campaign["id"]} - cursor_value = stream_state.get(campaign["id"], {}).get(self.cursor_field) + state_value = stream_state.get(campaign["id"], {}).get(self.cursor_field) + cursor_value = self.get_filter_date(self.start_date, state_value) if cursor_value: slice_[self.filter_field] = cursor_value yield slice_ @@ -220,9 +316,203 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp yield {**item, **activity_item} +class InterestCategories(MailChimpStream, HttpSubStream): + """ + Get information about interest categories for a specific list. + Docs link: https://mailchimp.com/developer/marketing/api/interest-categories/list-interest-categories/ + """ + + data_field = "categories" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + """ + Get the list_id from the parent stream slice and use it to construct the path. + """ + list_id = stream_slice.get("parent").get("id") + return f"lists/{list_id}/interest-categories" + + +class Interests(MailChimpStream, HttpSubStream): + """ + Get a list of interests for a specific interest category. + Docs link: https://mailchimp.com/developer/marketing/api/interests/list-interests-in-category/ + """ + + data_field = "interests" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + """ + Get the list_id from the parent stream slice and use it to construct the path. + """ + list_id = stream_slice.get("parent").get("list_id") + category_id = stream_slice.get("parent").get("id") + return f"lists/{list_id}/interest-categories/{category_id}/interests" + + +class ListMembers(MailChimpListSubStream): + """ + Get information about members in a specific Mailchimp list. + Docs link: https://mailchimp.com/developer/marketing/api/list-members/list-members-info/ + """ + + cursor_field = "last_changed" + data_field = "members" + + class Reports(IncrementalMailChimpStream): cursor_field = "send_time" data_field = "reports" def path(self, **kwargs) -> str: return "reports" + + +class SegmentMembers(MailChimpListSubStream): + """ + Get information about members in a specific segment. + Docs link: https://mailchimp.com/developer/marketing/api/list-segment-members/list-members-in-segment/ + """ + + cursor_field = "last_changed" + data_field = "members" + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + """ + Each slice consists of a list_id and segment_id pair + """ + segments_slices = Segments(authenticator=self.authenticator).stream_slices(sync_mode=SyncMode.full_refresh) + + for slice in segments_slices: + segment_records = Segments(authenticator=self.authenticator).read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice) + + for segment in segment_records: + yield {"list_id": segment["list_id"], "segment_id": segment["id"]} + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + list_id = stream_slice.get("list_id") + segment_id = stream_slice.get("segment_id") + return f"lists/{list_id}/segments/{segment_id}/members" + + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], stream_slice, **kwargs) -> Iterable[Mapping]: + """ + The SegmentMembers endpoint does not support sorting or filtering, + so we need to apply our own filtering logic before reading. + The foreign key "segment_id" is also added to each record before being read. + """ + response = super().parse_response(response, **kwargs) + + # Calculate the filter date to compare all records against in this slice + slice_cursor_value = stream_state.get(str(stream_slice.get("segment_id")), {}).get(self.cursor_field) + filter_date = self.get_filter_date(self.start_date, slice_cursor_value) + + for record in self.filter_old_records(response, filter_date): + # Add the segment_id foreign_key to each record + record["segment_id"] = stream_slice.get("segment_id") + yield record + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + current_stream_state = current_stream_state or {} + segment_id = str(latest_record.get("segment_id")) + latest_cursor_value = latest_record.get(self.cursor_field) + + # Get the current state value for this list, if it exists + segment_state = current_stream_state.get(segment_id, {}) + current_cursor_value = segment_state.get(self.cursor_field, latest_cursor_value) + + # Update the cursor value and set it in state + updated_cursor_value = max(current_cursor_value, latest_cursor_value) + current_stream_state[segment_id] = {self.cursor_field: updated_cursor_value} + return current_stream_state + + +class Segments(MailChimpListSubStream): + """ + Get information about all available segments for a specific list. + Docs link: https://mailchimp.com/developer/marketing/api/list-segments/list-segments/ + """ + + cursor_field = "updated_at" + data_field = "segments" + + +class Tags(MailChimpStream, HttpSubStream): + """ + Get information about tags for a specific list. + Docs link: https://mailchimp.com/developer/marketing/api/list-tags/list-tags-for-list/ + """ + + data_field = "tags" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + list_id = stream_slice.get("parent").get("id") + return f"lists/{list_id}/tag-search" + + def parse_response(self, response: requests.Response, stream_slice, **kwargs) -> Iterable[Mapping]: + """ + Tags do not reference parent_ids, so we need to add the list_id to each record. + """ + response = super().parse_response(response, **kwargs) + + for record in response: + record["list_id"] = stream_slice.get("parent").get("id") + yield record + + +class Unsubscribes(IncrementalMailChimpStream): + """ + List of members who have unsubscribed from a specific campaign. + Docs link: https://mailchimp.com/developer/marketing/api/unsub-reports/list-unsubscribed-members/ + """ + + cursor_field = "timestamp" + data_field = "unsubscribes" + # There is no unique identifier for unsubscribes, so we use a composite key + # consisting of the campaign_id, email_id, and timestamp. + primary_key = ["campaign_id", "email_id", "timestamp"] + + def __init__(self, campaign_id: Optional[str] = None, **kwargs): + super().__init__(**kwargs) + self.campaign_id = campaign_id + + def stream_slices( + self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + + if self.campaign_id: + # Similar to EmailActivity stream, this is a workaround to speed up SATs + # and enable incremental tests by reading from a single campaign + campaigns = [{"id": self.campaign_id}] + else: + campaigns = Campaigns(authenticator=self.authenticator).read_records(sync_mode=SyncMode.full_refresh) + for campaign in campaigns: + yield {"campaign_id": campaign["id"]} + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + campaign_id = stream_slice.get("campaign_id") + return f"reports/{campaign_id}/unsubscribed" + + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], stream_slice, **kwargs) -> Iterable[Mapping]: + """ + The Unsubscribes endpoint does not support sorting or filtering, + so we need to apply our own filtering logic before reading. + """ + + response = super().parse_response(response, **kwargs) + + slice_cursor_value = stream_state.get(stream_slice.get("campaign_id", {}), {}).get(self.cursor_field) + filter_date = self.get_filter_date(self.start_date, slice_cursor_value) + yield from self.filter_old_records(response, filter_date) + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + current_stream_state = current_stream_state or {} + campaign_id = latest_record.get("campaign_id") + latest_cursor_value = latest_record.get(self.cursor_field) + + # Get the current state value for this campaign, if it exists + campaign_state = current_stream_state.get(campaign_id, {}) + current_cursor_value = campaign_state.get(self.cursor_field, latest_cursor_value) + + # Update the cursor value and set it in state + updated_cursor_value = max(current_cursor_value, latest_cursor_value) + current_stream_state[campaign_id] = {self.cursor_field: updated_cursor_value} + return current_stream_state diff --git a/airbyte-integrations/connectors/source-mailchimp/unit_tests/conftest.py b/airbyte-integrations/connectors/source-mailchimp/unit_tests/conftest.py index f16d3ce2785f..5305f0dadab4 100644 --- a/airbyte-integrations/connectors/source-mailchimp/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-mailchimp/unit_tests/conftest.py @@ -4,6 +4,7 @@ from pytest import fixture from source_mailchimp.source import MailChimpAuthenticator +from source_mailchimp.streams import Campaigns, Unsubscribes @fixture(name="data_center") @@ -13,7 +14,7 @@ def data_center_fixture(): @fixture(name="config") def config_fixture(data_center): - return {"apikey": f"API_KEY-{data_center}"} + return {"apikey": f"API_KEY-{data_center}", "start_date": "2022-01-01T00:00:00.000Z"} @fixture(name="access_token") @@ -46,3 +47,31 @@ def wrong_config_fixture(): @fixture(name="auth") def authenticator_fixture(apikey_config): return MailChimpAuthenticator().get_auth(apikey_config) + + +@fixture(name="campaigns_stream") +def campaigns_stream_fixture(auth): + return Campaigns(authenticator=auth) + + +@fixture(name="unsubscribes_stream") +def unsubscribes_stream_fixture(auth): + return Unsubscribes(authenticator=auth) + + +@fixture(name="mock_campaigns_response") +def mock_campaigns_response_fixture(): + return [ + {"id": "campaign_1", "web_id": 1, "type": "regular", "create_time": "2022-01-01T00:00:00Z"}, + {"id": "campaign_2", "web_id": 2, "type": "plaintext", "create_time": "2022-01-02T00:00:00Z"}, + {"id": "campaign_3", "web_id": 3, "type": "variate", "create_time": "2022-01-03T00:00:00Z"}, + ] + + +@fixture(name="mock_unsubscribes_state") +def mock_unsubscribes_state_fixture(): + return { + "campaign_1": {"timestamp": "2022-01-01T00:00:00Z"}, + "campaign_2": {"timestamp": "2022-01-02T00:00:00Z"}, + "campaign_3": {"timestamp": "2022-01-03T00:00:00Z"}, + } diff --git a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_source.py b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_source.py index 22625f877a66..b1ccfcddac6a 100644 --- a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_source.py @@ -2,17 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging + import pytest -import requests -from airbyte_cdk.logger import AirbyteLogger from source_mailchimp.source import MailChimpAuthenticator, SourceMailchimp -logger = AirbyteLogger() +logger = logging.getLogger("airbyte") def test_check_connection_ok(requests_mock, config, data_center): responses = [ - {"json": [], "status_code": 200}, + {"json": {"health_status": "Everything's Chimpy!"}}, ] requests_mock.register_uri("GET", f"https://{data_center}.api.mailchimp.com/3.0/ping", responses) ok, error_msg = SourceMailchimp().check_connection(logger, config=config) @@ -21,30 +21,54 @@ def test_check_connection_ok(requests_mock, config, data_center): assert not error_msg -def test_check_connection_error(requests_mock, config, data_center): - requests_mock.register_uri("GET", f"https://{data_center}.api.mailchimp.com/3.0/ping", body=requests.ConnectionError()) +@pytest.mark.parametrize( + "response, expected_message", + [ + ( + { + "json": { + "title": "API Key Invalid", + "details": "Your API key may be invalid, or you've attempted to access the wrong datacenter.", + } + }, + "Encountered an error while connecting to Mailchimp. Type: API Key Invalid. Details: Your API key may be invalid, or you've attempted to access the wrong datacenter.", + ), + ( + {"json": {"title": "Forbidden", "details": "You don't have permission to access this resource."}}, + "Encountered an error while connecting to Mailchimp. Type: Forbidden. Details: You don't have permission to access this resource.", + ), + ( + {"json": {}}, + "Encountered an error while connecting to Mailchimp. Type: Unknown Error. Details: An unknown error occurred. Please verify your credentials and try again.", + ), + ], + ids=["API Key Invalid", "Forbidden", "Unknown Error"], +) +def test_check_connection_error(requests_mock, config, data_center, response, expected_message): + requests_mock.register_uri("GET", f"https://{data_center}.api.mailchimp.com/3.0/ping", json=response["json"]) ok, error_msg = SourceMailchimp().check_connection(logger, config=config) assert not ok - assert error_msg + assert error_msg == expected_message -def test_get_server_prefix_ok(requests_mock, access_token, data_center): +def test_get_oauth_data_center_ok(requests_mock, access_token, data_center): responses = [ {"json": {"dc": data_center}, "status_code": 200}, ] requests_mock.register_uri("GET", "https://login.mailchimp.com/oauth2/metadata", responses) - assert MailChimpAuthenticator().get_server_prefix(access_token) == data_center + assert MailChimpAuthenticator().get_oauth_data_center(access_token) == data_center -def test_get_server_prefix_exception(requests_mock, access_token, data_center): +def test_get_oauth_data_center_exception(requests_mock, access_token): responses = [ {"json": {}, "status_code": 200}, + {"json": {"error": "invalid_token"}, "status_code": 200}, {"status_code": 403}, ] requests_mock.register_uri("GET", "https://login.mailchimp.com/oauth2/metadata", responses) with pytest.raises(Exception): - MailChimpAuthenticator().get_server_prefix(access_token) + MailChimpAuthenticator().get_oauth_data_center(access_token) def test_oauth_config(requests_mock, oauth_config, data_center): @@ -64,6 +88,29 @@ def test_wrong_config(wrong_config): MailChimpAuthenticator().get_auth(wrong_config) +@pytest.mark.parametrize( + "config, expected_return", + [ + ({}, None), + ({"start_date": "2021-01-01T00:00:00.000Z"}, None), + ({"start_date": "2021-99-99T79:89:99.123Z"}, "The provided start date is not a valid date. Please check the date you input and try again."), + ({"start_date": "2021-01-01T00:00:00.000"}, "Please check the format of the start date against the pattern descriptor."), + ({"start_date": "2025-01-25T00:00:00.000Z"}, "The start date cannot be greater than the current date."), + ], + ids=[ + "No start date", + "Valid start date", + "Invalid start date", + "Invalid format", + "Future start date", + ] +) +def test_validate_start_date(config, expected_return): + source = SourceMailchimp() + result = source._validate_start_date(config) + assert result == expected_return + + def test_streams_count(config): streams = SourceMailchimp().streams(config) - assert len(streams) == 5 + assert len(streams) == 12 diff --git a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_streams.py index ad0618959ed7..b441fe26f7b3 100644 --- a/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-mailchimp/unit_tests/test_streams.py @@ -2,11 +2,28 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging from unittest.mock import MagicMock import pytest +import requests import responses -from source_mailchimp.streams import Campaigns, EmailActivity, Lists +from airbyte_cdk.models import SyncMode +from requests.exceptions import HTTPError +from source_mailchimp.streams import ( + Automations, + Campaigns, + EmailActivity, + InterestCategories, + Interests, + ListMembers, + Lists, + Reports, + SegmentMembers, + Segments, + Tags, + Unsubscribes, +) from utils import read_full_refresh, read_incremental @@ -15,6 +32,7 @@ [ (Lists, "lists"), (Campaigns, "campaigns"), + (Segments, "lists/123/segments"), ], ) def test_stream_read(requests_mock, auth, stream, endpoint): @@ -29,6 +47,11 @@ def test_stream_read(requests_mock, auth, stream, endpoint): ] stream_url = stream.url_base + endpoint requests_mock.register_uri("GET", stream_url, stream_responses) + + # Mock the 'lists' endpoint as Segments stream_slice + lists_url = stream.url_base + "lists" + lists_response = {"json": {"lists": [{"id": "123"}]}} + requests_mock.register_uri("GET", lists_url, [lists_response]) records = read_full_refresh(stream) assert records @@ -48,21 +71,43 @@ def test_next_page_token(auth): @pytest.mark.parametrize( - "inputs, expected_params", + "stream, inputs, expected_params", [ ( + Lists, {"stream_slice": None, "stream_state": None, "next_page_token": None}, - {"count": 1000, "sort_dir": "ASC", "sort_field": "date_created"}, + {"count": 1000, "sort_dir": "ASC", "sort_field": "date_created", "exclude_fields": "lists._links"}, ), ( + Lists, {"stream_slice": None, "stream_state": None, "next_page_token": {"offset": 1000}}, - {"count": 1000, "sort_dir": "ASC", "sort_field": "date_created", "offset": 1000}, + {"count": 1000, "sort_dir": "ASC", "sort_field": "date_created", "offset": 1000, "exclude_fields": "lists._links"}, + ), + ( + InterestCategories, + {"stream_slice": {"parent": {"id": "123"}}, "stream_state": None, "next_page_token": None}, + {"count": 1000, "exclude_fields": "categories._links"}, + ), + ( + Interests, + {"stream_slice": {"parent": {"id": "123"}}, "stream_state": None, "next_page_token": {"offset": 2000}}, + {"count": 1000, "exclude_fields": "interests._links", "offset": 2000}, ), ], + ids=[ + "Lists: no next_page_token or state to add to request params", + "Lists: next_page_token added to request params", + "InterestCategories: no next_page_token to add to request params", + "Interests: next_page_token added to request params", + ], ) -def test_request_params(auth, inputs, expected_params): +def test_request_params(auth, stream, inputs, expected_params): args = {"authenticator": auth} - stream = Lists(**args) + if stream == InterestCategories: + args["parent"] = Lists(**args) + elif stream == Interests: + args["parent"] = InterestCategories(authenticator=auth, parent=Lists(authenticator=auth)) + stream = stream(**args) assert stream.request_params(**inputs) == expected_params @@ -109,3 +154,543 @@ def test_stream_parse_json_error(auth, caplog): responses.add("GET", stream_url, body="not_valid_json") read_incremental(stream, {}) assert "response.content=b'not_valid_json'" in caplog.text + + +@pytest.mark.parametrize( + "stream_class, stream_slice, stream_state, next_page_token, expected_params", + [ + # Test case 1: no state, no next_page_token + ( + Segments, + {"list_id": "123"}, + {}, + None, + {"count": 1000, "sort_dir": "ASC", "sort_field": "updated_at", "list_id": "123", "exclude_fields": "segments._links"}, + ), + # Test case 2: state and next_page_token + ( + ListMembers, + {"list_id": "123", "since_last_changed": "2023-10-15T00:00:00Z"}, + {"123": {"last_changed": "2023-10-15T00:00:00Z"}}, + {"offset": 1000}, + { + "count": 1000, + "sort_dir": "ASC", + "sort_field": "last_changed", + "list_id": "123", + "offset": 1000, + "exclude_fields": "members._links", + "since_last_changed": "2023-10-15T00:00:00Z", + }, + ), + ], + ids=[ + "Segments: no next_page_token or state to add to request params", + "ListMembers: next_page_token and state filter added to request params", + ], +) +def test_list_child_request_params(auth, stream_class, stream_slice, stream_state, next_page_token, expected_params): + """ + Tests the request_params method for the shared MailChimpListSubStream class. + """ + stream = stream_class(authenticator=auth) + params = stream.request_params(stream_slice=stream_slice, stream_state=stream_state, next_page_token=next_page_token) + assert params == expected_params + + +@pytest.mark.parametrize( + "stream_class, current_stream_state,latest_record,expected_state", + [ + # Test case 1: current_stream_state is empty + (Segments, {}, {"list_id": "list_1", "updated_at": "2023-10-15T00:00:00Z"}, {"list_1": {"updated_at": "2023-10-15T00:00:00Z"}}), + # Test case 2: latest_record's cursor is higher than current_stream_state for list_1 and updates it + ( + Segments, + {"list_1": {"updated_at": "2023-10-14T00:00:00Z"}, "list_2": {"updated_at": "2023-10-15T00:00:00Z"}}, + {"list_id": "list_1", "updated_at": "2023-10-15T00:00:00Z"}, + {"list_1": {"updated_at": "2023-10-15T00:00:00Z"}, "list_2": {"updated_at": "2023-10-15T00:00:00Z"}}, + ), + # Test case 3: latest_record's cursor is lower than current_stream_state for list_2, no state update + ( + ListMembers, + {"list_1": {"last_changed": "2023-10-15T00:00:00Z"}, "list_2": {"last_changed": "2023-10-15T00:00:00Z"}}, + {"list_id": "list_2", "last_changed": "2023-10-14T00:00:00Z"}, + {"list_1": {"last_changed": "2023-10-15T00:00:00Z"}, "list_2": {"last_changed": "2023-10-15T00:00:00Z"}}, + ), + ( + SegmentMembers, + {"segment_1": {"last_changed": "2023-10-15T00:00:00Z"}, "segment_2": {"last_changed": "2023-10-15T00:00:00Z"}}, + {"segment_id": "segment_1", "last_changed": "2023-10-16T00:00:00Z"}, + {"segment_1": {"last_changed": "2023-10-16T00:00:00Z"}, "segment_2": {"last_changed": "2023-10-15T00:00:00Z"}}, + ), + ( + SegmentMembers, + {"segment_1": {"last_changed": "2023-10-15T00:00:00Z"}}, + {"segment_id": "segment_2", "last_changed": "2023-10-16T00:00:00Z"}, + {"segment_1": {"last_changed": "2023-10-15T00:00:00Z"}, "segment_2": {"last_changed": "2023-10-16T00:00:00Z"}}, + ) + ], + ids=[ + "Segments: no current_stream_state", + "Segments: latest_record's cursor > than current_stream_state for list_1", + "ListMembers: latest_record's cursor < current_stream_state for list_2", + "SegmentMembers: latest_record's cursor > current_stream_state for segment_1", + "SegmentMembers: no stream_state for current slice, new slice added to state" + ], +) +def test_list_child_get_updated_state(auth, stream_class, current_stream_state, latest_record, expected_state): + """ + Tests that the get_updated_state method for the shared MailChimpListSubStream class + correctly updates state only for its slice. + """ + segments_stream = stream_class(authenticator=auth) + updated_state = segments_stream.get_updated_state(current_stream_state, latest_record) + assert updated_state == expected_state + + +@pytest.mark.parametrize( + "stream_state, records, expected", + [ + # Test case 1: No stream state, all records should be yielded + ( + {}, + {"members": [ + {"id": 1, "segment_id": "segment_1", "last_changed": "2021-01-01T00:00:00Z"}, + {"id": 2, "segment_id": "segment_1", "last_changed": "2021-01-02T00:00:00Z"} + ]}, + [ + {"id": 1, "segment_id": "segment_1", "last_changed": "2021-01-01T00:00:00Z"}, + {"id": 2, "segment_id": "segment_1", "last_changed": "2021-01-02T00:00:00Z"} + ] + ), + + # Test case 2: Records older than stream state should be filtered out + ( + {"segment_1": {"last_changed": "2021-02-01T00:00:00Z"}}, + {"members": [ + {"id": 1, "segment_id": "segment_1", "last_changed": "2021-01-01T00:00:00Z"}, + {"id": 2, "segment_id": "segment_1", "last_changed": "2021-03-01T00:00:00Z"} + ]}, + [{"id": 2, "segment_id": "segment_1", "last_changed": "2021-03-01T00:00:00Z"}] + ), + + # Test case 3: Two lists in stream state, only state for segment_id_1 determines filtering + ( + {"segment_1": {"last_changed": "2021-01-02T00:00:00Z"}, "segment_2": {"last_changed": "2022-01-01T00:00:00Z"}}, + {"members": [ + {"id": 1, "segment_id": "segment_1", "last_changed": "2021-01-01T00:00:00Z"}, + {"id": 2, "segment_id": "segment_1", "last_changed": "2021-03-01T00:00:00Z"} + ]}, + [{"id": 2, "segment_id": "segment_1", "last_changed": "2021-03-01T00:00:00Z"}] + ), + ], + ids=[ + "No stream state, all records should be yielded", + "Record < stream state, should be filtered out", + "Record >= stream state, should be yielded", + ] +) +def test_segment_members_parse_response(auth, stream_state, records, expected): + segment_members_stream = SegmentMembers(authenticator=auth) + response = MagicMock() + response.json.return_value = records + parsed_records = list(segment_members_stream.parse_response(response, stream_state, stream_slice={"segment_id": "segment_1"})) + assert parsed_records == expected, f"Expected: {expected}, Actual: {parsed_records}" + + +@pytest.mark.parametrize( + "stream, record, expected_record", + [ + ( + SegmentMembers, + {"id": 1, "email_address": "a@gmail.com", "email_type": "html", "opt_timestamp": ""}, + {"id": 1, "email_address": "a@gmail.com", "email_type": "html", "opt_timestamp": None} + ), + ( + SegmentMembers, + {"id": 1, "email_address": "a@gmail.com", "email_type": "html", "opt_timestamp": "2022-01-01T00:00:00.000Z", "merge_fields": {"FNAME": "Bob", "LNAME": "", "ADDRESS": "", "PHONE": ""}}, + {"id": 1, "email_address": "a@gmail.com", "email_type": "html", "opt_timestamp": "2022-01-01T00:00:00.000Z", "merge_fields": {"FNAME": "Bob", "LNAME": None, "ADDRESS": None, "PHONE": None}} + ), + ( + Campaigns, + {"id": "1", "web_id": 2, "email_type": "html", "create_time": "2022-01-01T00:00:00.000Z", "send_time": ""}, + {"id": "1", "web_id": 2, "email_type": "html", "create_time": "2022-01-01T00:00:00.000Z", "send_time": None} + ), + ( + Reports, + {"id": "1", "type": "rss", "clicks": {"clicks_total": 1, "last_click": "2022-01-01T00:00:00Z"}, "opens": {"opens_total": 0, "last_open": ""}}, + {"id": "1", "type": "rss", "clicks": {"clicks_total": 1, "last_click": "2022-01-01T00:00:00Z"}, "opens": {"opens_total": 0, "last_open": None}} + ), + ( + Lists, + {"id": "1", "name": "Santa's List", "stats": {"last_sub_date": "2022-01-01T00:00:00Z", "last_unsub_date": ""}}, + {"id": "1", "name": "Santa's List", "stats": {"last_sub_date": "2022-01-01T00:00:00Z", "last_unsub_date": None}} + ) + ], + ids=[ + "segment_members: opt_timestamp nullified", + "segment_members: nested merge_fields nullified", + "campaigns: send_time nullified", + "reports: nested opens.last_open nullified", + "lists: stats.last_unsub_date nullified" + ] +) +def test_filter_empty_fields(auth, stream, record, expected_record): + """ + Tests that empty string values are converted to None. + """ + stream = stream(authenticator=auth) + assert stream.filter_empty_fields(record) == expected_record + + +def test_unsubscribes_stream_slices(requests_mock, unsubscribes_stream, campaigns_stream, mock_campaigns_response): + campaigns_url = campaigns_stream.url_base + campaigns_stream.path() + requests_mock.register_uri("GET", campaigns_url, json={"campaigns": mock_campaigns_response}) + + expected_slices = [{"campaign_id": "campaign_1"}, {"campaign_id": "campaign_2"}, {"campaign_id": "campaign_3"}] + slices = list(unsubscribes_stream.stream_slices(sync_mode=SyncMode.incremental)) + assert slices == expected_slices + + +@pytest.mark.parametrize( + "stream_state, expected_records", + [ + ( # Test case 1: all records >= state + {"campaign_1": {"timestamp": "2022-01-01T00:00:00Z"}}, + [ + {"campaign_id": "campaign_1", "email_id": "email_1", "timestamp": "2022-01-02T00:00:00Z"}, + {"campaign_id": "campaign_1", "email_id": "email_2", "timestamp": "2022-01-02T00:00:00Z"}, + {"campaign_id": "campaign_1", "email_id": "email_3", "timestamp": "2022-01-01T00:00:00Z"}, + {"campaign_id": "campaign_1", "email_id": "email_4", "timestamp": "2022-01-03T00:00:00Z"}, + ], + ), + ( # Test case 2: one record < state + {"campaign_1": {"timestamp": "2022-01-02T00:00:00Z"}}, + [ + {"campaign_id": "campaign_1", "email_id": "email_1", "timestamp": "2022-01-02T00:00:00Z"}, + {"campaign_id": "campaign_1", "email_id": "email_2", "timestamp": "2022-01-02T00:00:00Z"}, + {"campaign_id": "campaign_1", "email_id": "email_4", "timestamp": "2022-01-03T00:00:00Z"}, + ], + ), + ( # Test case 3: one record >= state + {"campaign_1": {"timestamp": "2022-01-03T00:00:00Z"}}, + [ + {"campaign_id": "campaign_1", "email_id": "email_4", "timestamp": "2022-01-03T00:00:00Z"}, + ], + ), + ( # Test case 4: no state, all records returned + {}, + [ + {"campaign_id": "campaign_1", "email_id": "email_1", "timestamp": "2022-01-02T00:00:00Z"}, + {"campaign_id": "campaign_1", "email_id": "email_2", "timestamp": "2022-01-02T00:00:00Z"}, + {"campaign_id": "campaign_1", "email_id": "email_3", "timestamp": "2022-01-01T00:00:00Z"}, + {"campaign_id": "campaign_1", "email_id": "email_4", "timestamp": "2022-01-03T00:00:00Z"}, + ], + ), + ], + ids=[ + "all records >= state", + "one record < state", + "one record >= state", + "no state, all records returned", + ], +) +def test_parse_response(stream_state, expected_records, unsubscribes_stream): + mock_response = MagicMock(spec=requests.Response) + mock_response.json.return_value = { + "unsubscribes": [ + {"campaign_id": "campaign_1", "email_id": "email_1", "timestamp": "2022-01-02T00:00:00Z"}, + {"campaign_id": "campaign_1", "email_id": "email_2", "timestamp": "2022-01-02T00:00:00Z"}, + {"campaign_id": "campaign_1", "email_id": "email_3", "timestamp": "2022-01-01T00:00:00Z"}, + {"campaign_id": "campaign_1", "email_id": "email_4", "timestamp": "2022-01-03T00:00:00Z"}, + ] + } + stream_slice = {"campaign_id": "campaign_1"} + records = list(unsubscribes_stream.parse_response(response=mock_response, stream_slice=stream_slice, stream_state=stream_state)) + assert records == expected_records + + +@pytest.mark.parametrize( + "latest_record, expected_updated_state", + [ + # Test case 1: latest_record > and updates the state of campaign_1 + ( + { + "email_id": "email_1", + "email_address": "address1@email.io", + "reason": "None given", + "timestamp": "2022-01-05T00:00:00Z", + "campaign_id": "campaign_1", + }, + { + "campaign_1": {"timestamp": "2022-01-05T00:00:00Z"}, + "campaign_2": {"timestamp": "2022-01-02T00:00:00Z"}, + "campaign_3": {"timestamp": "2022-01-03T00:00:00Z"}, + }, + ), + # Test case 2: latest_record > and updates the state of campaign_2 + ( + { + "email_id": "email_2", + "email_address": "address2@email.io", + "reason": "Inappropriate content", + "timestamp": "2022-01-05T00:00:00Z", + "campaign_id": "campaign_2", + }, + { + "campaign_1": {"timestamp": "2022-01-01T00:00:00Z"}, + "campaign_2": {"timestamp": "2022-01-05T00:00:00Z"}, + "campaign_3": {"timestamp": "2022-01-03T00:00:00Z"}, + }, + ), + # Test case 3: latest_record < and does not update the state of campaign_3 + ( + { + "email_id": "email_3", + "email_address": "address3@email.io", + "reason": "No longer interested", + "timestamp": "2021-01-01T00:00:00Z", + "campaign_id": "campaign_3", + }, + { + "campaign_1": {"timestamp": "2022-01-01T00:00:00Z"}, + "campaign_2": {"timestamp": "2022-01-02T00:00:00Z"}, + "campaign_3": {"timestamp": "2022-01-03T00:00:00Z"}, + }, + ), + # Test case 4: latest_record sets state campaign_4 + ( + { + "email_id": "email_4", + "email_address": "address4@email.io", + "reason": "No longer interested", + "timestamp": "2022-01-04T00:00:00Z", + "campaign_id": "campaign_4", + }, + { + "campaign_1": {"timestamp": "2022-01-01T00:00:00Z"}, + "campaign_2": {"timestamp": "2022-01-02T00:00:00Z"}, + "campaign_3": {"timestamp": "2022-01-03T00:00:00Z"}, + "campaign_4": {"timestamp": "2022-01-04T00:00:00Z"}, + }, + ), + ], + ids=[ + "latest_record > and updates the state of campaign_1", + "latest_record > and updates the state of campaign_2", + "latest_record < and does not update the state of campaign_3", + "latest_record sets state of campaign_4", + ], +) +def test_unsubscribes_get_updated_state(unsubscribes_stream, mock_unsubscribes_state, latest_record, expected_updated_state): + updated_state = unsubscribes_stream.get_updated_state(mock_unsubscribes_state, latest_record) + assert updated_state == expected_updated_state + + +@pytest.mark.parametrize( + "stream,url,status_code,response_content,expected_availability,expected_reason_substring", + [ + ( + Campaigns, + "https://some_dc.api.mailchimp.com/3.0/campaigns", + 403, + b'{"object": "error", "status": 403, "code": "restricted_resource"}', + False, + "Unable to read campaigns stream", + ), + ( + EmailActivity, + "https://some_dc.api.mailchimp.com/3.0/reports/123/email-activity", + 403, + b'{"object": "error", "status": 403, "code": "restricted_resource"}', + False, + "Unable to read email_activity stream", + ), + ( + Lists, + "https://some_dc.api.mailchimp.com/3.0/lists", + 200, + b'{ "lists": [{"id": "123", "date_created": "2022-01-01T00:00:00+000"}]}', + True, + None, + ), + ( + Lists, + "https://some_dc.api.mailchimp.com/3.0/lists", + 400, + b'{ "object": "error", "status": 404, "code": "invalid_action"}', + False, + None, + ), + ], + ids=[ + "Campaigns 403 error", + "EmailActivity 403 error", + "Lists 200 success", + "Lists 400 error", + ], +) +def test_403_error_handling( + auth, requests_mock, stream, url, status_code, response_content, expected_availability, expected_reason_substring +): + """ + Test that availability strategy flags streams with 403 error as unavailable + and returns appropriate message. + """ + + requests_mock.get(url=url, status_code=status_code, content=response_content) + + stream = stream(authenticator=auth) + + if stream.__class__.__name__ == "EmailActivity": + stream.stream_slices = MagicMock(return_value=[{"campaign_id": "123"}]) + + try: + is_available, reason = stream.check_availability(logger=logging.Logger, source=MagicMock()) + + assert is_available is expected_availability + + if expected_reason_substring: + assert expected_reason_substring in reason + else: + assert reason is None + + # Handle non-403 error + except HTTPError as e: + assert e.response.status_code == status_code + + +@pytest.mark.parametrize( + "stream, stream_slice, expected_endpoint", + [ + (Automations, {}, "automations"), + (Lists, {}, "lists"), + (Campaigns, {}, "campaigns"), + (EmailActivity, {"campaign_id": "123"}, "reports/123/email-activity"), + (InterestCategories, {"parent": {"id": "123"}}, "lists/123/interest-categories"), + (Interests, {"parent": {"list_id": "123", "id": "456"}}, "lists/123/interest-categories/456/interests"), + (ListMembers, {"list_id": "123"}, "lists/123/members"), + (Reports, {}, "reports"), + (SegmentMembers, {"list_id": "123", "segment_id": "456"}, "lists/123/segments/456/members"), + (Segments, {"list_id": "123"}, "lists/123/segments"), + (Tags, {"parent": {"id": "123"}}, "lists/123/tag-search"), + (Unsubscribes, {"campaign_id": "123"}, "reports/123/unsubscribed"), + ], + ids=[ + "Automations", + "Lists", + "Campaigns", + "EmailActivity", + "InterestCategories", + "Interests", + "ListMembers", + "Reports", + "SegmentMembers", + "Segments", + "Tags", + "Unsubscribes", + ], +) +def test_path(auth, stream, stream_slice, expected_endpoint): + """ + Test the path method for each stream. + """ + + # Add parent stream where necessary + if stream is InterestCategories or stream is Tags: + stream = stream(authenticator=auth, parent=Lists(authenticator=auth)) + elif stream is Interests: + stream = stream(authenticator=auth, parent=InterestCategories(authenticator=auth, parent=Lists(authenticator=auth))) + else: + stream = stream(authenticator=auth) + + endpoint = stream.path(stream_slice=stream_slice) + + assert endpoint == expected_endpoint, f"Stream {stream}: expected path '{expected_endpoint}', got '{endpoint}'" + + +@pytest.mark.parametrize( + "start_date, state_date, expected_return_value", + [ + ( + "2021-01-01T00:00:00.000Z", + "2020-01-01T00:00:00+00:00", + "2021-01-01T00:00:00Z" + ), + ( + "2021-01-01T00:00:00.000Z", + "2023-10-05T00:00:00+00:00", + "2023-10-05T00:00:00+00:00" + ), + ( + None, + "2022-01-01T00:00:00+00:00", + "2022-01-01T00:00:00+00:00" + ), + ( + "2020-01-01T00:00:00.000Z", + None, + "2020-01-01T00:00:00Z" + ), + ( + None, + None, + None + ) + ] +) +def test_get_filter_date(auth, start_date, state_date, expected_return_value): + """ + Tests that the get_filter_date method returns the correct date string + """ + stream = Campaigns(authenticator=auth, start_date=start_date) + result = stream.get_filter_date(start_date, state_date) + assert result == expected_return_value, f"Expected: {expected_return_value}, Actual: {result}" + + +@pytest.mark.parametrize( + "stream_class, records, filter_date, expected_return_value", + [ + ( + Unsubscribes, + [ + {"campaign_id": "campaign_1", "email_id": "email_1", "timestamp": "2022-01-02T00:00:00Z"}, + {"campaign_id": "campaign_1", "email_id": "email_2", "timestamp": "2022-01-04T00:00:00Z"}, + {"campaign_id": "campaign_1", "email_id": "email_3", "timestamp": "2022-01-03T00:00:00Z"}, + {"campaign_id": "campaign_1", "email_id": "email_4", "timestamp": "2022-01-01T00:00:00Z"}, + ], + "2022-01-02T12:00:00+00:00", + [ + {"campaign_id": "campaign_1", "email_id": "email_2", "timestamp": "2022-01-04T00:00:00Z"}, + {"campaign_id": "campaign_1", "email_id": "email_3", "timestamp": "2022-01-03T00:00:00Z"}, + ], + ), + ( + SegmentMembers, + [ + {"id": 1, "segment_id": "segment_1", "last_changed": "2021-01-04T00:00:00Z"}, + {"id": 2, "segment_id": "segment_1", "last_changed": "2021-01-01T00:00:00Z"}, + {"id": 3, "segment_id": "segment_1", "last_changed": "2021-01-03T00:00:00Z"}, + {"id": 4, "segment_id": "segment_1", "last_changed": "2021-01-02T00:00:00Z"}, + ], + None, + [ + {"id": 1, "segment_id": "segment_1", "last_changed": "2021-01-04T00:00:00Z"}, + {"id": 2, "segment_id": "segment_1", "last_changed": "2021-01-01T00:00:00Z"}, + {"id": 3, "segment_id": "segment_1", "last_changed": "2021-01-03T00:00:00Z"}, + {"id": 4, "segment_id": "segment_1", "last_changed": "2021-01-02T00:00:00Z"}, + ], + ) + ], + ids=[ + "Unsubscribes: filter_date is set, records filtered", + "SegmentMembers: filter_date is None, all records returned" + ] +) +def test_filter_old_records(auth, stream_class, records, filter_date, expected_return_value): + """ + Tests the logic for filtering old records in streams that do not support query_param filtering. + """ + stream = stream_class(authenticator=auth) + filtered_records = list(stream.filter_old_records(records, filter_date)) + assert filtered_records == expected_return_value diff --git a/airbyte-integrations/connectors/source-mailerlite/README.md b/airbyte-integrations/connectors/source-mailerlite/README.md index e313ce44fab5..f7ea62d35eb1 100644 --- a/airbyte-integrations/connectors/source-mailerlite/README.md +++ b/airbyte-integrations/connectors/source-mailerlite/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-mailerlite:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/mailerlite) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_mailerlite/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-mailerlite:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-mailerlite build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-mailerlite:airbyteDocker +An image will be built with the tag `airbyte/source-mailerlite:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-mailerlite:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailerlite:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailerlite:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-mailerlite:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-mailerlite test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-mailerlite:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-mailerlite:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-mailerlite test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/mailerlite.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-mailerlite/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-mailerlite/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-mailerlite/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-mailerlite/build.gradle b/airbyte-integrations/connectors/source-mailerlite/build.gradle deleted file mode 100644 index e8d38bfa9364..000000000000 --- a/airbyte-integrations/connectors/source-mailerlite/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_mailerlite' -} diff --git a/airbyte-integrations/connectors/source-mailersend/README.md b/airbyte-integrations/connectors/source-mailersend/README.md index 4680ddcb3bdc..a9fbcee8c347 100644 --- a/airbyte-integrations/connectors/source-mailersend/README.md +++ b/airbyte-integrations/connectors/source-mailersend/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-mailersend:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/mailersend) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_mailersend/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-mailersend:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-mailersend build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-mailersend:airbyteDocker +An image will be built with the tag `airbyte/source-mailersend:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-mailersend:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailersend:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailersend:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-mailersend:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-mailersend test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-mailersend:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-mailersend:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-mailersend test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/mailersend.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-mailersend/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-mailersend/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-mailersend/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-mailersend/build.gradle b/airbyte-integrations/connectors/source-mailersend/build.gradle deleted file mode 100644 index d19e16b945c0..000000000000 --- a/airbyte-integrations/connectors/source-mailersend/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_mailersend' -} diff --git a/airbyte-integrations/connectors/source-mailgun/Dockerfile b/airbyte-integrations/connectors/source-mailgun/Dockerfile index 0f9bc9f710fd..3542dbe0bb33 100644 --- a/airbyte-integrations/connectors/source-mailgun/Dockerfile +++ b/airbyte-integrations/connectors/source-mailgun/Dockerfile @@ -34,5 +34,5 @@ COPY source_mailgun ./source_mailgun ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.version=0.2.1 LABEL io.airbyte.name=airbyte/source-mailgun diff --git a/airbyte-integrations/connectors/source-mailgun/README.md b/airbyte-integrations/connectors/source-mailgun/README.md index c3198f50139a..3550d669d3d8 100644 --- a/airbyte-integrations/connectors/source-mailgun/README.md +++ b/airbyte-integrations/connectors/source-mailgun/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-mailgun:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/mailgun) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_mailgun/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-mailgun:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-mailgun build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-mailgun:airbyteDocker +An image will be built with the tag `airbyte/source-mailgun:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-mailgun:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailgun:dev check --co docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailgun:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-mailgun:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-mailgun test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-mailgun:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-mailgun:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-mailgun test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/mailgun.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-mailgun/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mailgun/acceptance-test-config.yml index 3d87f163752c..cfe77380a94b 100644 --- a/airbyte-integrations/connectors/source-mailgun/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-mailgun/acceptance-test-config.yml @@ -23,18 +23,18 @@ acceptance_tests: empty_streams: - name: events bypass_reason: "Sandbox account can't seed this stream" -# expect_records: -# path: "integration_tests/expected_records.jsonl" -# extra_fields: no -# exact_order: no -# extra_records: yes - incremental: + # expect_records: + # path: "integration_tests/expected_records.jsonl" + # extra_fields: no + # exact_order: no + # extra_records: yes + incremental: bypass_reason: "This connector does not implement incremental sync" -# tests: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# future_state: -# future_state_path: "integration_tests/abnormal_state.json" + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-mailgun/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-mailgun/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-mailgun/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-mailgun/build.gradle b/airbyte-integrations/connectors/source-mailgun/build.gradle deleted file mode 100644 index 31173743e8e3..000000000000 --- a/airbyte-integrations/connectors/source-mailgun/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_mailgun' -} diff --git a/airbyte-integrations/connectors/source-mailgun/metadata.yaml b/airbyte-integrations/connectors/source-mailgun/metadata.yaml index e83c02b1cee6..7d3e7d30ee15 100644 --- a/airbyte-integrations/connectors/source-mailgun/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailgun/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: 5b9cb09e-1003-4f9c-983d-5779d1b2cd51 - dockerImageTag: 0.2.0 + dockerImageTag: 0.2.1 dockerRepository: airbyte/source-mailgun githubIssueLabel: source-mailgun icon: mailgun.svg @@ -22,7 +22,7 @@ data: tags: - language:low-code ab_internal: - sl: 200 + sl: 100 ql: 200 supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-mailgun/setup.py b/airbyte-integrations/connectors/source-mailgun/setup.py index 3ede43d33d0b..f7661245f2fd 100644 --- a/airbyte-integrations/connectors/source-mailgun/setup.py +++ b/airbyte-integrations/connectors/source-mailgun/setup.py @@ -6,13 +6,12 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( diff --git a/airbyte-integrations/connectors/source-mailgun/source_mailgun/manifest.yaml b/airbyte-integrations/connectors/source-mailgun/source_mailgun/manifest.yaml index 92ef0a7abfed..ec95d47218d0 100644 --- a/airbyte-integrations/connectors/source-mailgun/source_mailgun/manifest.yaml +++ b/airbyte-integrations/connectors/source-mailgun/source_mailgun/manifest.yaml @@ -52,7 +52,7 @@ definitions: cursor_granularity: "PT0.000001S" lookback_window: "P31D" start_datetime: - datetime: "{{ config['start_date'] }}" + datetime: "{{ config.get('start_date', day_delta(-90, format='%Y-%m-%dT%H:%M:%SZ')) }}" datetime_format: "%Y-%m-%dT%H:%M:%SZ" end_datetime: datetime: "{{ today_utc() }}" diff --git a/airbyte-integrations/connectors/source-mailjet-mail/Dockerfile b/airbyte-integrations/connectors/source-mailjet-mail/Dockerfile index db4a76403c77..a05924e3dc82 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/Dockerfile +++ b/airbyte-integrations/connectors/source-mailjet-mail/Dockerfile @@ -34,5 +34,5 @@ COPY source_mailjet_mail ./source_mailjet_mail ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/source-mailjet-mail diff --git a/airbyte-integrations/connectors/source-mailjet-mail/README.md b/airbyte-integrations/connectors/source-mailjet-mail/README.md index 8669749eeaf7..4bba003682d3 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/README.md +++ b/airbyte-integrations/connectors/source-mailjet-mail/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-mailjet-mail:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/mailjet-mail) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_mailjet_mail/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-mailjet-mail:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-mailjet-mail build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-mailjet-mail:airbyteDocker +An image will be built with the tag `airbyte/source-mailjet-mail:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-mailjet-mail:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailjet-mail:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailjet-mail:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-mailjet-mail:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-mailjet-mail test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-mailjet-mail:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-mailjet-mail:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-mailjet-mail test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/mailjet-mail.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-mailjet-mail/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-mailjet-mail/build.gradle b/airbyte-integrations/connectors/source-mailjet-mail/build.gradle deleted file mode 100644 index 3fbd4c02d8f9..000000000000 --- a/airbyte-integrations/connectors/source-mailjet-mail/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_mailjet_mail' -} diff --git a/airbyte-integrations/connectors/source-mailjet-mail/metadata.yaml b/airbyte-integrations/connectors/source-mailjet-mail/metadata.yaml index 9a5b49f50b99..3a0ae99fd35a 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/metadata.yaml +++ b/airbyte-integrations/connectors/source-mailjet-mail/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: 56582331-5de2-476b-b913-5798de77bbdf - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 dockerRepository: airbyte/source-mailjet-mail githubIssueLabel: source-mailjet-mail icon: mailjetmail.svg diff --git a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/manifest.yaml b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/manifest.yaml index 222189652d69..dfad8159a782 100644 --- a/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/manifest.yaml +++ b/airbyte-integrations/connectors/source-mailjet-mail/source_mailjet_mail/manifest.yaml @@ -83,7 +83,7 @@ definitions: $parameters: name: "message" primary_key: "ID" - path: "/message" + path: "/message?ShowSubject=true" listrecipient_stream: $ref: "#/definitions/base_stream" retriever: diff --git a/airbyte-integrations/connectors/source-mailjet-sms/README.md b/airbyte-integrations/connectors/source-mailjet-sms/README.md index fd5b3e2b5ca3..34b56157fd07 100644 --- a/airbyte-integrations/connectors/source-mailjet-sms/README.md +++ b/airbyte-integrations/connectors/source-mailjet-sms/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-mailjet-sms:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/mailjet-sms) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_mailjet_sms/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-mailjet-sms:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-mailjet-sms build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-mailjet-sms:airbyteDocker +An image will be built with the tag `airbyte/source-mailjet-sms:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-mailjet-sms:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailjet-sms:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mailjet-sms:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-mailjet-sms:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-mailjet-sms test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-mailjet-sms:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-mailjet-sms:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-mailjet-sms test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/mailjet-sms.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-mailjet-sms/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-mailjet-sms/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-mailjet-sms/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-mailjet-sms/build.gradle b/airbyte-integrations/connectors/source-mailjet-sms/build.gradle deleted file mode 100644 index a196522dde6b..000000000000 --- a/airbyte-integrations/connectors/source-mailjet-sms/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_mailjet_sms' -} diff --git a/airbyte-integrations/connectors/source-marketo/Dockerfile b/airbyte-integrations/connectors/source-marketo/Dockerfile deleted file mode 100644 index d4276e9ecc21..000000000000 --- a/airbyte-integrations/connectors/source-marketo/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata \ - && apk --no-cache add build-base - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_marketo ./source_marketo - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=1.2.0 -LABEL io.airbyte.name=airbyte/source-marketo diff --git a/airbyte-integrations/connectors/source-marketo/README.md b/airbyte-integrations/connectors/source-marketo/README.md index b7d14aed736f..50676f5b93e4 100644 --- a/airbyte-integrations/connectors/source-marketo/README.md +++ b/airbyte-integrations/connectors/source-marketo/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-marketo:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/marketo) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_marketo/spec.json` file. @@ -56,19 +48,70 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-marketo:dev + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-marketo build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-marketo:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. -You can also build the connector image via Gradle: +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-marketo:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-marketo:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-marketo:dev . +# Running the spec command against your patched connector +docker run airbyte/source-marketo:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -77,45 +120,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-marketo:dev check --co docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-marketo:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-marketo:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .'[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-marketo test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-marketo:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-marketo:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-marketo:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +139,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-marketo test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/marketo.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-marketo/acceptance-test-config.yml b/airbyte-integrations/connectors/source-marketo/acceptance-test-config.yml index f5fa3a1faf4a..7fe08d8c848a 100644 --- a/airbyte-integrations/connectors/source-marketo/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-marketo/acceptance-test-config.yml @@ -89,6 +89,18 @@ acceptance_tests: bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction" - name: "activities_scheduled_meetingin_dialogue" bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction" + - name: "activities_engagedwithan_agentin_dialogue" + bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction" + - name: "activities_reached_conversational_flow_goal" + bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction" + - name: "activities_engagedwithan_agentin_conversational_flow" + bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction" + - name: "activities_engagedwitha_conversational_flow" + bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction" + - name: "activities_scheduled_meetingin_conversational_flow" + bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction" + - name: "activities_interactedwith_documentin_conversational_flow" + bypass_reason: "Marketo does not provide a way to populate this stream without outside interaction" # 52 streams, most of them use BULK API therefore it takes much time to run a sync timeout_seconds: 9000 fail_on_extra_columns: false diff --git a/airbyte-integrations/connectors/source-marketo/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-marketo/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-marketo/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-marketo/build.gradle b/airbyte-integrations/connectors/source-marketo/build.gradle deleted file mode 100644 index bc438cffd1fd..000000000000 --- a/airbyte-integrations/connectors/source-marketo/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_marketo' -} diff --git a/airbyte-integrations/connectors/source-marketo/main.py b/airbyte-integrations/connectors/source-marketo/main.py index 127c4d2c05ad..4b7b8e8d1708 100644 --- a/airbyte-integrations/connectors/source-marketo/main.py +++ b/airbyte-integrations/connectors/source-marketo/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_marketo import SourceMarketo +from source_marketo.run import run if __name__ == "__main__": - source = SourceMarketo() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-marketo/metadata.yaml b/airbyte-integrations/connectors/source-marketo/metadata.yaml index d0fc6429c308..6fb5daa2c2a9 100644 --- a/airbyte-integrations/connectors/source-marketo/metadata.yaml +++ b/airbyte-integrations/connectors/source-marketo/metadata.yaml @@ -1,12 +1,18 @@ data: + ab_internal: + ql: 200 + sl: 200 allowedHosts: hosts: - "*.mktorest.com" + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 9e0556f4-69df-4522-a3fb-03264d36b348 - dockerImageTag: 1.2.0 + dockerImageTag: 1.2.5 dockerRepository: airbyte/source-marketo + documentationUrl: https://docs.airbyte.com/integrations/sources/marketo githubIssueLabel: source-marketo icon: marketo.svg license: ELv2 @@ -17,11 +23,7 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/marketo + supportLevel: certified tags: - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-marketo/setup.py b/airbyte-integrations/connectors/source-marketo/setup.py index 1588bd2fc2a5..b8dfcde912ad 100644 --- a/airbyte-integrations/connectors/source-marketo/setup.py +++ b/airbyte-integrations/connectors/source-marketo/setup.py @@ -17,6 +17,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-marketo=source_marketo.run:run", + ], + }, name="source_marketo", description="Source implementation for Marketo.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-marketo/source_marketo/run.py b/airbyte-integrations/connectors/source-marketo/source_marketo/run.py new file mode 100644 index 000000000000..0831c3167f5f --- /dev/null +++ b/airbyte-integrations/connectors/source-marketo/source_marketo/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_marketo import SourceMarketo + + +def run(): + source = SourceMarketo() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py index 06ad97f67491..62d4ded15196 100644 --- a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py +++ b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py @@ -5,6 +5,7 @@ import csv import datetime import json +import re from abc import ABC from time import sleep from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple @@ -17,6 +18,8 @@ from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator +from airbyte_cdk.utils import AirbyteTracedException +from airbyte_protocol.models import FailureType from .utils import STRING_TYPES, clean_string, format_value, to_datetime_str @@ -231,7 +234,10 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp schema = self.get_json_schema()["properties"] response.encoding = "utf-8" - reader = csv.DictReader(response.iter_lines(chunk_size=1024, decode_unicode=True)) + response_lines = response.iter_lines(chunk_size=1024, decode_unicode=True) + filtered_response_lines = self.filter_null_bytes(response_lines) + reader = self.csv_rows(filtered_response_lines) + for record in reader: new_record = {**record} attributes = json.loads(new_record.pop("attributes", "{}")) @@ -257,6 +263,23 @@ def read_records( self.sleep_till_export_completed(stream_slice) return super().read_records(sync_mode, cursor_field, stream_slice, stream_state) + def filter_null_bytes(self, response_lines: Iterable[str]) -> Iterable[str]: + for line in response_lines: + res = line.replace("\x00", "") + if len(res) < len(line): + self.logger.warning("Filter 'null' bytes from string, size reduced %d -> %d chars", len(line), len(res)) + yield res + + @staticmethod + def csv_rows(lines: Iterable[str]) -> Iterable[Mapping]: + reader = csv.reader(lines) + headers = None + for row in reader: + if headers is None: + headers = row + else: + yield dict(zip(headers, row)) + class MarketoExportCreate(MarketoStream): """ @@ -280,8 +303,12 @@ def path(self, **kwargs) -> str: def should_retry(self, response: requests.Response) -> bool: if response.status_code == 429 or 500 <= response.status_code < 600: return True - record = next(self.parse_response(response, {}), {}) - status, export_id = record.get("status", "").lower(), record.get("exportId") + if errors := response.json().get("errors"): + if errors[0].get("code") == "1029" and re.match("Export daily quota \d+MB exceeded", errors[0].get("message")): + message = "Daily limit for job extractions has been reached (resets daily at 12:00AM CST)." + raise AirbyteTracedException(internal_message=response.text, message=message, failure_type=FailureType.config_error) + result = response.json().get("result")[0] + status, export_id = result.get("status", "").lower(), result.get("exportId") if status != "created" or not export_id: self.logger.warning(f"Failed to create export job! Status is {status}!") return True @@ -342,7 +369,15 @@ def __init__(self, config: Mapping[str, Any]): @property def stream_fields(self): - return list(self.get_json_schema()["properties"].keys()) + standard_properties = set(self.get_json_schema()["properties"]) + resp = self._session.get(f"{self._url_base}rest/v1/leads/describe.json", headers=self.authenticator.get_auth_header()) + available_fields = set(x.get("rest").get("name") for x in resp.json().get("result")) + return list(standard_properties & available_fields) + + def get_json_schema(self) -> Mapping[str, Any]: + # TODO: make schema truly dynamic like in stream Activities + # now blocked by https://github.com/airbytehq/airbyte/issues/30530 due to potentially > 500 fields in schema (can cause OOM) + return super().get_json_schema() class Activities(MarketoExportBase): diff --git a/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py b/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py index 12f59073cb49..806f39da100d 100644 --- a/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-marketo/unit_tests/test_source.py @@ -6,12 +6,24 @@ import os import tracemalloc from functools import partial -from unittest.mock import ANY, Mock, patch +from unittest.mock import ANY, MagicMock, Mock, patch import pendulum import pytest +import requests from airbyte_cdk.models.airbyte_protocol import SyncMode -from source_marketo.source import Activities, Campaigns, Leads, MarketoStream, Programs, SourceMarketo +from airbyte_cdk.utils import AirbyteTracedException +from source_marketo.source import ( + Activities, + Campaigns, + IncrementalMarketoStream, + Leads, + MarketoExportCreate, + MarketoStream, + Programs, + Segmentations, + SourceMarketo, +) def test_create_export_job(mocker, send_email_stream, caplog): @@ -26,6 +38,28 @@ def test_create_export_job(mocker, send_email_stream, caplog): assert "Failed to create export job! Status is failed!" in caplog.records[-1].message +def test_should_retry_quota_exceeded(config, requests_mock): + create_job_url = "https://602-euo-598.mktorest.com/rest/v1/leads/export/create.json?batchSize=300" + response_json = { + "requestId": "d2ca#18c0b9833bf", + "success": False, + "errors": [ + { + "code": "1029", + "message": "Export daily quota 500MB exceeded." + } + ] + } + requests_mock.register_uri("GET", create_job_url, status_code=200, json=response_json) + + response = requests.get(create_job_url) + with pytest.raises(AirbyteTracedException) as e: + MarketoExportCreate(config).should_retry(response) + + assert e.value.message == "Daily limit for job extractions has been reached (resets daily at 12:00AM CST)." + + + @pytest.mark.parametrize( "activity, expected_schema", ( @@ -284,10 +318,40 @@ def test_check_connection(config, requests_mock, status_code, response, is_conne ("2020-08-01", "%Y-%m-%dT%H:%M:%SZ%z", "2020-08-01"), ), ) -def test_normalize_datetime(config, input, format, expected_result): +def test_programs_normalize_datetime(config, input, format, expected_result): stream = Programs(config) assert stream.normalize_datetime(input, format) == expected_result +def test_programs_next_page_token(config): + mock_json = MagicMock() + mock_json.return_value = {"result": [{"test": 'testValue'}]} + mocked_response = MagicMock() + mocked_response.json = mock_json + stream = Programs(config) + result = stream.next_page_token(mocked_response) + assert result == {"offset": 201} + +@pytest.mark.parametrize("input, stream_state, expected_result",[( + {"result": [{"id": "1", "createdAt": "2020-07-01T00:00:00Z+0000", "updatedAt": "2020-07-01T00:00:00Z+0000"}]}, + {"updatedAt": "2020-06-01T00:00:00Z"}, + [{"id": "1", "createdAt": "2020-07-01T00:00:00Z", "updatedAt": "2020-07-01T00:00:00Z"}], + )], +) +def test_programs_parse_response(mocker, config, input, stream_state, expected_result): + response = requests.Response() + mocker.patch.object(response, "json", return_value=input) + stream = Programs(config) + result = stream.parse_response(response, stream_state) + assert list(result) == expected_result + +def test_segmentations_next_page_token(config): + mock_json = MagicMock() + mock_json.return_value = {"result": [{"test": 'testValue'}]} + mocked_response = MagicMock() + mocked_response.json = mock_json + stream = Segmentations(config) + result = stream.next_page_token(mocked_response) + assert result == {"offset": 201} today = pendulum.now() yesterday = pendulum.now().subtract(days=1).strftime("%Y-%m-%dT%H:%M:%SZ") @@ -306,11 +370,64 @@ def test_normalize_datetime(config, input, format, expected_result): ({"updatedAt": today}, {"updatedAt": None}, {"updatedAt": today}), ({"updatedAt": today}, {}, {"updatedAt": today}), ({"updatedAt": yesterday}, {"updatedAt": today}, {"updatedAt": today}), - ({"updatedAt": today}, {"updatedAt": yesterday}, {"updatedAt": today}) - ) + ({"updatedAt": today}, {"updatedAt": yesterday}, {"updatedAt": today}), + ), ) def test_get_updated_state(config, latest_record, current_state, expected_state): stream = Leads(config) if expected_state == "start_date": expected_state = {"updatedAt": config["start_date"]} assert stream.get_updated_state(latest_record, current_state) == expected_state + + +def test_filter_null_bytes(config): + stream = Leads(config) + + test_lines = [ + "Hello\x00World\n", + "Name,Email\n", + "John\x00Doe,john.doe@example.com\n" + ] + expected_lines = [ + "HelloWorld\n", + "Name,Email\n", + "JohnDoe,john.doe@example.com\n" + ] + filtered_lines = stream.filter_null_bytes(test_lines) + for expected_line, filtered_line in zip(expected_lines, filtered_lines): + assert expected_line == filtered_line + + +def test_csv_rows(config): + stream = Leads(config) + + test_lines = [ + "Name,Email\n", + "John Doe,john.doe@example.com\n", + "Jane Doe,jane.doe@example.com\n" + ] + expected_records = [ + {"Name": "John Doe", "Email": "john.doe@example.com"}, + {"Name": "Jane Doe", "Email": "jane.doe@example.com"} + ] + records = stream.csv_rows(test_lines) + for expected_record, record in zip(expected_records, records): + assert expected_record == record + +def test_availablity_strategy(config): + stream = Leads(config) + assert stream.availability_strategy == None + +def test_path(config): + stream = MarketoStream(config) + assert stream.path() == "rest/v1/marketo_stream.json" + +def test_get_state(config): + stream = IncrementalMarketoStream(config) + assert stream.state == {} + +def test_set_tate(config): + stream = IncrementalMarketoStream(config) + expected_state = {"id": 1} + stream.state = expected_state + assert stream._state == expected_state diff --git a/airbyte-integrations/connectors/source-marketo/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-marketo/unit_tests/test_utils.py index 946885db1554..df3638db9614 100644 --- a/airbyte-integrations/connectors/source-marketo/unit_tests/test_utils.py +++ b/airbyte-integrations/connectors/source-marketo/unit_tests/test_utils.py @@ -3,8 +3,10 @@ # +from datetime import datetime + import pytest -from source_marketo.utils import clean_string, format_value +from source_marketo.utils import clean_string, format_value, to_datetime_str test_data = [ (1, {"type": "integer"}, int), @@ -15,11 +17,12 @@ ("1.5", {"type": "integer"}, int), ("15", {"type": "integer"}, int), ("true", {"type": "boolean"}, bool), + ("test_custom", {"type": "custom_type"}, str), ] @pytest.mark.parametrize("value,schema,expected_output_type", test_data) -def test_fromat_value(value, schema, expected_output_type): +def test_format_value(value, schema, expected_output_type): test = format_value(value, schema) assert isinstance(test, expected_output_type) @@ -55,3 +58,9 @@ def test_clean_string(value, expected): test = clean_string(value) assert test == expected + +def test_to_datetime_str(): + input = datetime(2023, 1, 1) + expected = "2023-01-01T00:00:00Z" + + assert to_datetime_str(input) == expected diff --git a/airbyte-integrations/connectors/source-merge/README.md b/airbyte-integrations/connectors/source-merge/README.md index 7c156f7e27dc..24901415147a 100644 --- a/airbyte-integrations/connectors/source-merge/README.md +++ b/airbyte-integrations/connectors/source-merge/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-merge:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/merge) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_merge/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-merge:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-merge build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-merge:airbyteDocker +An image will be built with the tag `airbyte/source-merge:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-merge:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-merge:dev check --conf docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-merge:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-merge:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-merge test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-merge:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-merge:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-merge test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/merge.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-merge/acceptance-test-config.yml b/airbyte-integrations/connectors/source-merge/acceptance-test-config.yml index 09a99e17613c..fbe2b8b3336b 100644 --- a/airbyte-integrations/connectors/source-merge/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-merge/acceptance-test-config.yml @@ -18,7 +18,7 @@ acceptance_tests: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: + empty_streams: - name: "activities" bypass_reason: "Merge is under beta stage and thus endpoint cannot be seeded manually" - name: "applications" @@ -41,20 +41,20 @@ acceptance_tests: bypass_reason: "Merge is under beta stage and thus endpoint cannot be seeded manually" - name: "offices" bypass_reason: "Merge is under beta stage and thus endpoint cannot be seeded manually" -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file - # expect_records: - # path: "integration_tests/expected_records.jsonl" - # extra_fields: no - # exact_order: no - # extra_records: yes - incremental: + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.jsonl" + # extra_fields: no + # exact_order: no + # extra_records: yes + incremental: bypass_reason: "This connector does not implement incremental sync" -# TODO uncomment this block this block if your connector implements incremental sync: - # tests: - # - config_path: "secrets/config.json" - # configured_catalog_path: "integration_tests/configured_catalog.json" - # future_state: - # future_state_path: "integration_tests/abnormal_state.json" + # TODO uncomment this block this block if your connector implements incremental sync: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-merge/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-merge/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-merge/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-merge/build.gradle b/airbyte-integrations/connectors/source-merge/build.gradle deleted file mode 100644 index d0251510596d..000000000000 --- a/airbyte-integrations/connectors/source-merge/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_merge' -} diff --git a/airbyte-integrations/connectors/source-metabase/Dockerfile b/airbyte-integrations/connectors/source-metabase/Dockerfile index 0cff57643073..f076026e7910 100644 --- a/airbyte-integrations/connectors/source-metabase/Dockerfile +++ b/airbyte-integrations/connectors/source-metabase/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.1 +LABEL io.airbyte.version=1.1.0 LABEL io.airbyte.name=airbyte/source-metabase diff --git a/airbyte-integrations/connectors/source-metabase/README.md b/airbyte-integrations/connectors/source-metabase/README.md index e093d4073774..e6519fbb23f8 100644 --- a/airbyte-integrations/connectors/source-metabase/README.md +++ b/airbyte-integrations/connectors/source-metabase/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-metabase:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/metabase) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_metabase/spec.yaml` file. @@ -69,61 +61,16 @@ To run your integration tests with acceptance tests, from the connector root, ru python -m pytest integration_tests -p integration_tests.acceptance ``` -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-metabase:unitTest -``` - -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-metabase:integrationTest -``` - -#### Build -To run your integration tests with docker localy - -First, make sure you build the latest Docker image: -``` -docker build --no-cache . -t airbyte/source-metabase:dev -``` - -You can also build the connector image via Gradle: -``` -./gradlew clean :airbyte-integrations:connectors:source-metabase:airbyteDocker -``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. - -#### Run -Then run any of the connector commands as follows: -``` -docker run --rm airbyte/source-metabase:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-metabase:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-metabase:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-metabase:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json -``` - -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-metabase:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -2. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -3. Create a Pull Request -4. Pat yourself on the back for being an awesome contributor -5. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master - - -### additional connector/streams properties of note +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-metabase test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/metabase.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. -Some metabase streams are mutable, meaning that after an incremental update, new data items could appear *before* -the latest update date. To work around that, define the lookback_window_days to define a window in days to fetch results -before the latest state date, in order to capture "delayed" data items. diff --git a/airbyte-integrations/connectors/source-metabase/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-metabase/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-metabase/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-metabase/build.gradle b/airbyte-integrations/connectors/source-metabase/build.gradle deleted file mode 100644 index bdf84a8909ba..000000000000 --- a/airbyte-integrations/connectors/source-metabase/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_metabase' -} diff --git a/airbyte-integrations/connectors/source-metabase/metadata.yaml b/airbyte-integrations/connectors/source-metabase/metadata.yaml index 3da269c3938c..c6289bf09e4f 100644 --- a/airbyte-integrations/connectors/source-metabase/metadata.yaml +++ b/airbyte-integrations/connectors/source-metabase/metadata.yaml @@ -1,12 +1,16 @@ data: + ab_internal: + ql: 300 + sl: 100 allowedHosts: hosts: - "*" connectorSubtype: api connectorType: source definitionId: c7cb421b-942e-4468-99ee-e369bcabaec5 - dockerImageTag: 1.0.1 + dockerImageTag: 1.1.0 dockerRepository: airbyte/source-metabase + documentationUrl: https://docs.airbyte.com/integrations/sources/metabase githubIssueLabel: source-metabase icon: metabase.svg license: MIT @@ -17,12 +21,8 @@ data: oss: enabled: true releaseStage: beta - documentationUrl: https://docs.airbyte.com/integrations/sources/metabase + supportLevel: community tags: - language:low-code - language:python - ab_internal: - sl: 200 - ql: 300 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-metabase/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-metabase/sample_files/configured_catalog.json index c5d9714a6ea6..dbe5b6a4db0a 100644 --- a/airbyte-integrations/connectors/source-metabase/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-metabase/sample_files/configured_catalog.json @@ -406,6 +406,115 @@ "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, + + { + "stream": { + "name": "databases", + "json_schema": { + "type": ["null", "object"], + "properties": { + "description": { "type": "string" }, + "features": { + "type": ["null", "array"], + "items": { "type": ["null", "string"] } + }, + "cache_field_values_schedule": { "type": ["null", "string"] }, + "timezone": { "type": ["null", "string"] }, + "auto_run_queries": { "type": ["null", "boolean"] }, + "metadata_sync_schedule": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "settings": { "type": ["null", "string"] }, + "caveats": { "type": ["null", "string"] }, + "creator_id": { "type": ["null", "integer"] }, + "is_full_sync": { "type": ["null", "boolean"] }, + "updated_at": { "type": ["null", "string"] }, + "native_permissions": { "type": ["null", "string"] }, + "cache_ttl": { "type": ["null", "integer"] }, + "details": { + "type": "object", + "properties": { + "project-id": { "type": ["null", "string"] }, + "service-account-json": { "type": ["null", "string"] }, + "dataset-filters-type": { "type": ["null", "string"] }, + "dataset-filters-patterns": { "type": ["null", "string"] }, + "cloud-ip-address-info": { "type": ["null", "string"] }, + "advanced-options": { "type": ["null", "boolean"] }, + "project-id-from-credentials": { "type": ["null", "string"] }, + "db": { "type": ["null", "string"] }, + "ssl": { "type": ["null", "boolean"] }, + "let-user-control-scheduling": { "type": ["null", "boolean"] }, + "use-jvm-timezone": { "type": ["null", "boolean"] }, + "include-user-id-and-hash": { "type": ["null", "boolean"] } + } + }, + "is_sample": { "type": ["null", "boolean"] }, + "id": { "type": "integer" }, + "is_on_demand": { "type": ["null", "boolean"] }, + "options": { "type": ["null", "string"] }, + "engine": { "type": ["null", "string"] }, + "initial_sync_status": { "type": ["null", "string"] }, + "is_audit": { "type": ["null", "boolean"] }, + "dbms_version": { + "type": "object", + "properties": { + "flavor": { "type": ["null", "string"] }, + "version": { "type": ["null", "string"] }, + "semantic-version": { + "type": "array", + "items": { "type": "integer" } + } + } + }, + "refingerprint": { "type": ["null", "boolean"] }, + "created_at": { "type": ["null", "string"] }, + "points_of_interest": { "type": ["null", "string"] }, + "can_upload": { "type": ["null", "boolean"] } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "native_query_snippets", + "json_schema": { + "type": ["null", "object"], + "properties": { + "description": { "type": ["null", "string"] }, + "archived": { "type": ["null", "boolean"] }, + "creator": { + "type": ["null", "object"], + "properties": { + "email": { "type": ["null", "string"] }, + "first_name": { "type": ["null", "string"] }, + "last_login": { "type": ["null", "string"] }, + "is_qbnewb": { "type": ["null", "boolean"] }, + "is_superuser": { "type": ["null", "boolean"] }, + "id": { "type": "integer" }, + "last_name": { "type": ["null", "string"] }, + "date_joined": { "type": ["null", "string"] }, + "common_name": { "type": ["null", "string"] } + } + }, + "content": { "type": ["null", "string"] }, + "collection_id": { "type": ["null", "integer"] }, + "name": { "type": ["null", "string"] }, + "creator_id": { "type": ["null", "integer"] }, + "updated_at": { "type": ["null", "string"] }, + "id": { "type": "integer" }, + "entity_id": { "type": ["null", "string"] }, + "created_at": { "type": ["null", "string"] } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "users", diff --git a/airbyte-integrations/connectors/source-metabase/source_metabase/manifest.yaml b/airbyte-integrations/connectors/source-metabase/source_metabase/manifest.yaml index 6147da2559aa..10fa59aef347 100644 --- a/airbyte-integrations/connectors/source-metabase/source_metabase/manifest.yaml +++ b/airbyte-integrations/connectors/source-metabase/source_metabase/manifest.yaml @@ -54,6 +54,21 @@ definitions: $parameters: name: "dashboards" path: "dashboard" + + databases_stream: + primary_key: "id" + retriever: + $ref: "#/definitions/data_field_retriever" + $parameters: + name: "databases" + path: "database" + + native_query_snippets_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "native_query_snippets" + path: "native-query-snippet" + users_stream: primary_key: "id" retriever: @@ -66,6 +81,8 @@ streams: - "#/definitions/collections_stream" - "#/definitions/dashboards_stream" - "#/definitions/users_stream" + - "#/definitions/databases_stream" + - "#/definitions/native_query_snippets_stream" check: stream_names: diff --git a/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/cards.json b/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/cards.json index 159f3eaa7021..0c5fec00b40b 100644 --- a/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/cards.json +++ b/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/cards.json @@ -1,5 +1,7 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "description": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/collections.json b/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/collections.json index e07cdcd885cf..5327b9d9ab1d 100644 --- a/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/collections.json +++ b/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/collections.json @@ -1,5 +1,7 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "authority_level": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/dashboards.json b/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/dashboards.json index c077a18cedb7..e04926b38aa2 100644 --- a/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/dashboards.json +++ b/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/dashboards.json @@ -1,5 +1,7 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "description": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/databases.json b/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/databases.json new file mode 100644 index 000000000000..b7a330d045f9 --- /dev/null +++ b/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/databases.json @@ -0,0 +1,143 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "description": { + "type": "string" + }, + "features": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "cache_field_values_schedule": { + "type": ["null", "string"] + }, + "timezone": { + "type": ["null", "string"] + }, + "auto_run_queries": { + "type": ["null", "boolean"] + }, + "metadata_sync_schedule": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "settings": { + "type": ["null", "string"] + }, + "caveats": { + "type": ["null", "string"] + }, + "creator_id": { + "type": ["null", "integer"] + }, + "is_full_sync": { + "type": ["null", "boolean"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "native_permissions": { + "type": ["null", "string"] + }, + "cache_ttl": { + "type": ["null", "integer"] + }, + "details": { + "type": "object", + "properties": { + "project-id": { + "type": ["null", "string"] + }, + "service-account-json": { + "type": ["null", "string"] + }, + "dataset-filters-type": { + "type": ["null", "string"] + }, + "dataset-filters-patterns": { + "type": ["null", "string"] + }, + "cloud-ip-address-info": { + "type": ["null", "string"] + }, + "advanced-options": { + "type": ["null", "boolean"] + }, + "project-id-from-credentials": { + "type": ["null", "string"] + }, + "db": { + "type": ["null", "string"] + }, + "ssl": { + "type": ["null", "boolean"] + }, + "let-user-control-scheduling": { + "type": ["null", "boolean"] + }, + "use-jvm-timezone": { + "type": ["null", "boolean"] + }, + "include-user-id-and-hash": { + "type": ["null", "boolean"] + } + } + }, + "is_sample": { + "type": ["null", "boolean"] + }, + "id": { + "type": "integer" + }, + "is_on_demand": { + "type": ["null", "boolean"] + }, + "options": { + "type": ["null", "string"] + }, + "engine": { + "type": ["null", "string"] + }, + "initial_sync_status": { + "type": ["null", "string"] + }, + "is_audit": { + "type": ["null", "boolean"] + }, + "dbms_version": { + "type": "object", + "properties": { + "flavor": { + "type": ["null", "string"] + }, + "version": { + "type": ["null", "string"] + }, + "semantic-version": { + "type": "array", + "items": { + "type": "integer" + } + } + } + }, + "refingerprint": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"] + }, + "points_of_interest": { + "type": ["null", "string"] + }, + "can_upload": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/native_query_snippets.json b/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/native_query_snippets.json new file mode 100644 index 000000000000..4ff870c7c48d --- /dev/null +++ b/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/native_query_snippets.json @@ -0,0 +1,69 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "description": { + "type": ["null", "string"] + }, + "archived": { + "type": ["null", "boolean"] + }, + "creator": { + "type": ["null", "object"], + "properties": { + "email": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "last_login": { + "type": ["null", "string"] + }, + "is_qbnewb": { + "type": ["null", "boolean"] + }, + "is_superuser": { + "type": ["null", "boolean"] + }, + "id": { + "type": "integer" + }, + "last_name": { + "type": ["null", "string"] + }, + "date_joined": { + "type": ["null", "string"] + }, + "common_name": { + "type": ["null", "string"] + } + } + }, + "content": { + "type": ["null", "string"] + }, + "collection_id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "creator_id": { + "type": ["null", "integer"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "id": { + "type": "integer" + }, + "entity_id": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/users.json b/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/users.json index ba05d3c307b6..e0dbd4e2ec9b 100644 --- a/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/users.json +++ b/airbyte-integrations/connectors/source-metabase/source_metabase/schemas/users.json @@ -1,5 +1,7 @@ { + "$schema": "http://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/README.md b/airbyte-integrations/connectors/source-microsoft-dataverse/README.md index b74c48998813..26a4fcff9b32 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/README.md +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-microsoft-dataverse:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/microsoft-dataverse) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_microsoft_dataverse/spec.yaml` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-microsoft-dataverse:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-microsoft-dataverse build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-microsoft-dataverse:airbyteDocker +An image will be built with the tag `airbyte/source-microsoft-dataverse:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-microsoft-dataverse:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-microsoft-dataverse:de docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-microsoft-dataverse:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-microsoft-dataverse:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-microsoft-dataverse test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-microsoft-dataverse:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-microsoft-dataverse:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-microsoft-dataverse test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/microsoft-dataverse.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-microsoft-dataverse/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/build.gradle b/airbyte-integrations/connectors/source-microsoft-dataverse/build.gradle deleted file mode 100644 index ad3ce5ecf70e..000000000000 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_microsoft_dataverse' -} diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_dataverse.py b/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_dataverse.py index e9ab1bac5577..d9f2cb436021 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_dataverse.py +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_dataverse.py @@ -6,12 +6,10 @@ from source_microsoft_dataverse.dataverse import AirbyteType, convert_dataverse_type -@pytest.mark.parametrize("dataverse_type,expected_result", [ - ("String", AirbyteType.String.value), - ("Integer", AirbyteType.Integer.value), - ("Virtual", None), - ("Random", AirbyteType.String.value) -]) +@pytest.mark.parametrize( + "dataverse_type,expected_result", + [("String", AirbyteType.String.value), ("Integer", AirbyteType.Integer.value), ("Virtual", None), ("Random", AirbyteType.String.value)], +) def test_convert_dataverse_type(dataverse_type, expected_result): result = convert_dataverse_type(dataverse_type) assert result == expected_result diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_incremental_streams.py index 5cfcebc5b0f9..84d904af6b64 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_incremental_streams.py @@ -15,12 +15,10 @@ def incremental_config(): "stream_name": "test_stream", "stream_path": "test_path", "primary_key": [["test_primary_key"]], - "schema": { - - }, + "schema": {}, "odata_maxpagesize": 100, "config_cursor_field": ["test_cursor_field"], - "authenticator": MagicMock() + "authenticator": MagicMock(), } @@ -29,16 +27,9 @@ def incremental_response(incremental_config): return { "@odata.deltaLink": f"{incremental_config['url']}?$deltatoken=12644418993%2110%2F06%2F2022%2020%3A06%3A12", "value": [ - { - "test_primary_key": "pk", - "test_cursor_field": "test-date" - }, - { - "id": "pk2", - "@odata.context": "context", - "reason": "deleted" - } - ] + {"test_primary_key": "pk", "test_cursor_field": "test-date"}, + {"id": "pk2", "@odata.context": "context", "reason": "deleted"}, + ], } diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_source.py b/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_source.py index dcdf044caac2..0e93f16521ab 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_source.py @@ -68,7 +68,8 @@ def test_streams_full_refresh(mock_get_auth, mock_request): @mock.patch("source_microsoft_dataverse.source.do_request") def test_discover_incremental(mock_request): - result_json = json.loads(''' + result_json = json.loads( + """ { "value": [ { @@ -91,7 +92,8 @@ def test_discover_incremental(mock_request): } ] } - ''') + """ + ) mock_request.return_value.status.return_value = 200 mock_request.return_value.json.return_value = result_json @@ -101,15 +103,16 @@ def test_discover_incremental(mock_request): catalog = source.discover(logger_mock, config_mock) - assert not {'modifiedon'} ^ set(catalog.streams[0].default_cursor_field) + assert not {"modifiedon"} ^ set(catalog.streams[0].default_cursor_field) assert not {SyncMode.full_refresh, SyncMode.incremental} ^ set(catalog.streams[0].supported_sync_modes) - assert not {'primary'} ^ set(catalog.streams[0].source_defined_primary_key[0]) + assert not {"primary"} ^ set(catalog.streams[0].source_defined_primary_key[0]) assert catalog.streams[0].json_schema["properties"]["test"] == AirbyteType.String.value @mock.patch("source_microsoft_dataverse.source.do_request") def test_discover_full_refresh(mock_request): - result_json = json.loads(''' + result_json = json.loads( + """ { "value": [ { @@ -128,7 +131,8 @@ def test_discover_full_refresh(mock_request): } ] } - ''') + """ + ) mock_request.return_value.status.return_value = 200 mock_request.return_value.json.return_value = result_json @@ -140,5 +144,5 @@ def test_discover_full_refresh(mock_request): assert catalog.streams[0].default_cursor_field is None or len(catalog.streams[0].default_cursor_field) == 0 assert not {SyncMode.full_refresh} ^ set(catalog.streams[0].supported_sync_modes) - assert not {'primary'} ^ set(catalog.streams[0].source_defined_primary_key[0]) + assert not {"primary"} ^ set(catalog.streams[0].source_defined_primary_key[0]) assert catalog.streams[0].json_schema["properties"]["test"] == AirbyteType.String.value diff --git a/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_streams.py index a310f7e86c3b..f272edea6688 100644 --- a/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-microsoft-dataverse/unit_tests/test_streams.py @@ -17,11 +17,9 @@ def incremental_config(): "stream_name": "test_stream", "stream_path": "test_path", "primary_key": [["test_primary_key"]], - "schema": { - - }, + "schema": {}, "odata_maxpagesize": 100, - "authenticator": MagicMock() + "authenticator": MagicMock(), } @@ -30,7 +28,7 @@ def incremental_config(): [ ({"stream_slice": None, "stream_state": {}, "next_page_token": None}, {}), ({"stream_slice": None, "stream_state": {}, "next_page_token": {"$skiptoken": "skiptoken"}}, {"$skiptoken": "skiptoken"}), - ({"stream_slice": None, "stream_state": {"$deltatoken": "delta"}, "next_page_token": None}, {"$deltatoken": "delta"}) + ({"stream_slice": None, "stream_state": {"$deltatoken": "delta"}, "next_page_token": None}, {"$deltatoken": "delta"}), ], ) def test_request_params(inputs, expected_params, incremental_config): @@ -41,8 +39,12 @@ def test_request_params(inputs, expected_params, incremental_config): @pytest.mark.parametrize( ("response_json", "next_page_token"), [ - ({"@odata.nextLink": "https://url?$skiptoken=oEBwdSP6uehIAxQOWq_3Ksh_TLol6KIm3stvdc6hGhZRi1hQ7Spe__dpvm3U4zReE4CYXC2zOtaKdi7KHlUtC2CbRiBIUwOxPKLa"}, - {"$skiptoken": "oEBwdSP6uehIAxQOWq_3Ksh_TLol6KIm3stvdc6hGhZRi1hQ7Spe__dpvm3U4zReE4CYXC2zOtaKdi7KHlUtC2CbRiBIUwOxPKLa"}), + ( + { + "@odata.nextLink": "https://url?$skiptoken=oEBwdSP6uehIAxQOWq_3Ksh_TLol6KIm3stvdc6hGhZRi1hQ7Spe__dpvm3U4zReE4CYXC2zOtaKdi7KHlUtC2CbRiBIUwOxPKLa" + }, + {"$skiptoken": "oEBwdSP6uehIAxQOWq_3Ksh_TLol6KIm3stvdc6hGhZRi1hQ7Spe__dpvm3U4zReE4CYXC2zOtaKdi7KHlUtC2CbRiBIUwOxPKLa"}, + ), ({"value": []}, None), ], ) @@ -58,17 +60,9 @@ def test_next_page_token(response_json, next_page_token, incremental_config): def test_parse_response(incremental_config): stream = MicrosoftDataverseStream(**incremental_config) response = MagicMock() - response.json.return_value = { - "value": [ - { - "test-key": "test-value" - } - ] - } + response.json.return_value = {"value": [{"test-key": "test-value"}]} inputs = {"response": response} - expected_parsed_object = { - "test-key": "test-value" - } + expected_parsed_object = {"test-key": "test-value"} assert next(stream.parse_response(**inputs)) == expected_parsed_object @@ -79,7 +73,7 @@ def test_request_headers(incremental_config): "Cache-Control": "no-cache", "OData-Version": "4.0", "Content-Type": "application/json", - "Prefer": "odata.maxpagesize=100" + "Prefer": "odata.maxpagesize=100", } assert stream.request_headers(**inputs) == expected_headers diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/README.md b/airbyte-integrations/connectors/source-microsoft-onedrive/README.md new file mode 100644 index 000000000000..ee6043e14b38 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/README.md @@ -0,0 +1,166 @@ +# Microsoft Onedrive Source + +This is the repository for the Microsoft Onedrive source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/microsoft-onedrive). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/microsoft-onedrive) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_microsoft_onedrive/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source microsoft-onedrive test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name source-microsoft-onedrive build +``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-microsoft-onedrive:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") +``` + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-microsoft-onedrive:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] +``` +Please use this as an example. This is not optimized. + +2. Build your image: +```bash +docker build -t airbyte/source-microsoft-onedrive:dev . +# Running the spec command against your patched connector +docker run airbyte/source-microsoft-onedrive:dev spec +```` + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-microsoft-onedrive:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-microsoft-onedrive:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-microsoft-onedrive:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-microsoft-onedrive:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` + +### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +Please run acceptance tests via [airbyte-ci](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#connectors-test-command): +```bash +airbyte-ci connectors --name source-microsoft-onedrive test +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-microsoft-onedrive test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/microsoft-onedrive.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/acceptance-test-config.yml b/airbyte-integrations/connectors/source-microsoft-onedrive/acceptance-test-config.yml new file mode 100644 index 000000000000..3966afb65b00 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/acceptance-test-config.yml @@ -0,0 +1,31 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-microsoft-onedrive:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_microsoft_onedrive/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + expect_trace_message_on_failure: false + incremental: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/icon.svg b/airbyte-integrations/connectors/source-microsoft-onedrive/icon.svg new file mode 100644 index 000000000000..eaf2ddb94622 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/icon.svg @@ -0,0 +1 @@ +OneDrive_64x \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/__init__.py b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..1c86c5e27bee --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/abnormal_state.json @@ -0,0 +1,30 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "test_csv" + }, + "stream_state": { + "_ab_source_file_last_modified": "2023-12-23T06:49:25.000000Z_Test_folder_2/TestFileOneDrive.csv", + "history": { + "Test_folder_2/TestFileOneDrive.csv": "2023-12-23T06:49:25.000000Z" + } + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "test_unstructured" + }, + "stream_state": { + "_ab_source_file_last_modified": "2023-12-23T06:49:25.000000Z_simple_pdf_file.pdf", + "history": { + "simple_pdf_file.pdf": "2023-12-23T06:49:25.000000Z" + } + } + } + } +] diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/acceptance.py similarity index 100% rename from airbyte-integrations/connectors/source-mongodb-internal-poc/integration_tests/acceptance.py rename to airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/acceptance.py diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..1bda267ec914 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/configured_catalog.json @@ -0,0 +1,26 @@ +{ + "streams": [ + { + "stream": { + "name": "test_csv", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["_ab_source_file_last_modified"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "test_unstructured", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["_ab_source_file_last_modified"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/invalid_config.json new file mode 100644 index 000000000000..68af1008c0ac --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/invalid_config.json @@ -0,0 +1,21 @@ +{ + "credentials": { + "auth_type": "Client", + "client_id": "client_id", + "tenant_id": "tenant_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token" + }, + "drive_name": "drive_name", + "folder_path": "folder_path", + "streams": [ + { + "name": "test_stream", + "globs": ["*.csv"], + "validation_policy": "Emit Record", + "format": { + "filetype": "csv" + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/spec.json b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/spec.json new file mode 100644 index 000000000000..24ee82201f5a --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/integration_tests/spec.json @@ -0,0 +1,455 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/microsoft-onedrive", + "connectionSpecification": { + "title": "Microsoft OneDrive Source Spec", + "description": "SourceMicrosoftOneDriveSpec class for Microsoft OneDrive Source Specification.\nThis class combines the authentication details with additional configuration for the OneDrive API.", + "type": "object", + "properties": { + "start_date": { + "title": "Start Date", + "description": "UTC date and time in the format 2017-01-25T00:00:00.000000Z. Any file modified before this date will not be replicated.", + "examples": ["2021-01-01T00:00:00.000000Z"], + "format": "date-time", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{6}Z$", + "pattern_descriptor": "YYYY-MM-DDTHH:mm:ss.SSSSSSZ", + "order": 1, + "type": "string" + }, + "streams": { + "title": "The list of streams to sync", + "description": "Each instance of this configuration defines a stream. Use this to define which files belong in the stream, their format, and how they should be parsed and validated. When sending data to warehouse destination such as Snowflake or BigQuery, each stream is a separate table.", + "order": 10, + "type": "array", + "items": { + "title": "FileBasedStreamConfig", + "type": "object", + "properties": { + "name": { + "title": "Name", + "description": "The name of the stream.", + "type": "string" + }, + "globs": { + "title": "Globs", + "description": "The pattern used to specify which files should be selected from the file system. For more information on glob pattern matching look here.", + "default": ["**"], + "order": 1, + "type": "array", + "items": { + "type": "string" + } + }, + "validation_policy": { + "title": "Validation Policy", + "description": "The name of the validation policy that dictates sync behavior when a record does not adhere to the stream schema.", + "default": "Emit Record", + "enum": ["Emit Record", "Skip Record", "Wait for Discover"] + }, + "input_schema": { + "title": "Input Schema", + "description": "The schema that will be used to validate records extracted from the file. This will override the stream schema that is auto-detected from incoming files.", + "type": "string" + }, + "primary_key": { + "title": "Primary Key", + "description": "The column or columns (for a composite key) that serves as the unique identifier of a record.", + "type": "string" + }, + "days_to_sync_if_history_is_full": { + "title": "Days To Sync If History Is Full", + "description": "When the state history of the file store is full, syncs will only read files that were last modified in the provided day range.", + "default": 3, + "type": "integer" + }, + "format": { + "title": "Format", + "description": "The configuration options that are used to alter how to read incoming files that deviate from the standard formatting.", + "type": "object", + "oneOf": [ + { + "title": "Avro Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "avro", + "const": "avro", + "type": "string" + }, + "double_as_string": { + "title": "Convert Double Fields to Strings", + "description": "Whether to convert double fields to strings. This is recommended if you have decimal numbers with a high degree of precision because there can be a loss precision when handling floating point numbers.", + "default": false, + "type": "boolean" + } + }, + "required": ["filetype"] + }, + { + "title": "CSV Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "csv", + "const": "csv", + "type": "string" + }, + "delimiter": { + "title": "Delimiter", + "description": "The character delimiting individual cells in the CSV data. This may only be a 1-character string. For tab-delimited data enter '\\t'.", + "default": ",", + "type": "string" + }, + "quote_char": { + "title": "Quote Character", + "description": "The character used for quoting CSV values. To disallow quoting, make this field blank.", + "default": "\"", + "type": "string" + }, + "escape_char": { + "title": "Escape Character", + "description": "The character used for escaping special characters. To disallow escaping, leave this field blank.", + "type": "string" + }, + "encoding": { + "title": "Encoding", + "description": "The character encoding of the CSV data. Leave blank to default to UTF8. See list of python encodings for allowable options.", + "default": "utf8", + "type": "string" + }, + "double_quote": { + "title": "Double Quote", + "description": "Whether two quotes in a quoted CSV value denote a single quote in the data.", + "default": true, + "type": "boolean" + }, + "null_values": { + "title": "Null Values", + "description": "A set of case-sensitive strings that should be interpreted as null values. For example, if the value 'NA' should be interpreted as null, enter 'NA' in this field.", + "default": [], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "strings_can_be_null": { + "title": "Strings Can Be Null", + "description": "Whether strings can be interpreted as null values. If true, strings that match the null_values set will be interpreted as null. If false, strings that match the null_values set will be interpreted as the string itself.", + "default": true, + "type": "boolean" + }, + "skip_rows_before_header": { + "title": "Skip Rows Before Header", + "description": "The number of rows to skip before the header row. For example, if the header row is on the 3rd row, enter 2 in this field.", + "default": 0, + "type": "integer" + }, + "skip_rows_after_header": { + "title": "Skip Rows After Header", + "description": "The number of rows to skip after the header row.", + "default": 0, + "type": "integer" + }, + "header_definition": { + "title": "CSV Header Definition", + "description": "How headers will be defined. `User Provided` assumes the CSV does not have a header row and uses the headers provided and `Autogenerated` assumes the CSV does not have a header row and the CDK will generate headers using for `f{i}` where `i` is the index starting from 0. Else, the default behavior is to use the header from the CSV file. If a user wants to autogenerate or provide column names for a CSV having headers, they can skip rows.", + "default": { + "header_definition_type": "From CSV" + }, + "oneOf": [ + { + "title": "From CSV", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "From CSV", + "const": "From CSV", + "type": "string" + } + }, + "required": ["header_definition_type"] + }, + { + "title": "Autogenerated", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "Autogenerated", + "const": "Autogenerated", + "type": "string" + } + }, + "required": ["header_definition_type"] + }, + { + "title": "User Provided", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "User Provided", + "const": "User Provided", + "type": "string" + }, + "column_names": { + "title": "Column Names", + "description": "The column names that will be used while emitting the CSV records", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": ["column_names", "header_definition_type"] + } + ], + "type": "object" + }, + "true_values": { + "title": "True Values", + "description": "A set of case-sensitive strings that should be interpreted as true values.", + "default": ["y", "yes", "t", "true", "on", "1"], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "false_values": { + "title": "False Values", + "description": "A set of case-sensitive strings that should be interpreted as false values.", + "default": ["n", "no", "f", "false", "off", "0"], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + } + }, + "required": ["filetype"] + }, + { + "title": "Jsonl Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "jsonl", + "const": "jsonl", + "type": "string" + } + }, + "required": ["filetype"] + }, + { + "title": "Parquet Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "parquet", + "const": "parquet", + "type": "string" + }, + "decimal_as_float": { + "title": "Convert Decimal Fields to Floats", + "description": "Whether to convert decimal fields to floats. There is a loss of precision when converting decimals to floats, so this is not recommended.", + "default": false, + "type": "boolean" + } + }, + "required": ["filetype"] + }, + { + "title": "Document File Type Format (Experimental)", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "unstructured", + "const": "unstructured", + "type": "string" + }, + "skip_unprocessable_file_types": { + "title": "Skip Unprocessable File Types", + "description": "If true, skip files that cannot be parsed because of their file type and log a warning. If false, fail the sync. Corrupted files with valid file types will still result in a failed sync.", + "default": true, + "always_show": true, + "type": "boolean" + } + }, + "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file.", + "required": ["filetype"] + } + ] + }, + "schemaless": { + "title": "Schemaless", + "description": "When enabled, syncs will not validate or structure records against the stream's schema.", + "default": false, + "type": "boolean" + } + }, + "required": ["name", "format"] + } + }, + "credentials": { + "title": "Authentication", + "description": "Credentials for connecting to the One Drive API", + "type": "object", + "order": 0, + "oneOf": [ + { + "title": "Authenticate via Microsoft (OAuth)", + "description": "OAuthCredentials class to hold authentication details for Microsoft OAuth authentication.\nThis class uses pydantic for data validation and settings management.", + "type": "object", + "properties": { + "auth_type": { + "title": "Auth Type", + "default": "Client", + "const": "Client", + "enum": ["Client"], + "type": "string" + }, + "tenant_id": { + "title": "Tenant ID", + "description": "Tenant ID of the Microsoft OneDrive user", + "airbyte_secret": true, + "type": "string" + }, + "client_id": { + "title": "Client ID", + "description": "Client ID of your Microsoft developer application", + "airbyte_secret": true, + "type": "string" + }, + "client_secret": { + "title": "Client Secret", + "description": "Client Secret of your Microsoft developer application", + "airbyte_secret": true, + "type": "string" + }, + "refresh_token": { + "title": "Refresh Token", + "description": "Refresh Token of your Microsoft developer application", + "airbyte_secret": true, + "type": "string" + } + }, + "required": [ + "tenant_id", + "client_id", + "client_secret", + "refresh_token" + ] + }, + { + "title": "Service Key Authentication", + "description": "ServiceCredentials class for service key authentication.\nThis class is structured similarly to OAuthCredentials but for a different authentication method.", + "type": "object", + "properties": { + "auth_type": { + "title": "Auth Type", + "default": "Service", + "const": "Service", + "enum": ["Service"], + "type": "string" + }, + "tenant_id": { + "title": "Tenant ID", + "description": "Tenant ID of the Microsoft OneDrive user", + "airbyte_secret": true, + "type": "string" + }, + "user_principal_name": { + "title": "User Principal Name", + "description": "Special characters such as a period, comma, space, and the at sign (@) are converted to underscores (_). More details: https://learn.microsoft.com/en-us/sharepoint/list-onedrive-urls", + "airbyte_secret": true, + "type": "string" + }, + "client_id": { + "title": "Client ID", + "description": "Client ID of your Microsoft developer application", + "airbyte_secret": true, + "type": "string" + }, + "client_secret": { + "title": "Client Secret", + "description": "Client Secret of your Microsoft developer application", + "airbyte_secret": true, + "type": "string" + } + }, + "required": [ + "tenant_id", + "user_principal_name", + "client_id", + "client_secret" + ] + } + ] + }, + "drive_name": { + "title": "Drive Name", + "description": "Name of the Microsoft OneDrive drive where the file(s) exist.", + "default": "OneDrive", + "order": 2, + "type": "string" + }, + "folder_path": { + "title": "Folder Path", + "description": "Path to folder of the Microsoft OneDrive drive where the file(s) exist.", + "order": 3, + "type": "string" + } + }, + "required": ["streams", "credentials", "folder_path"] + }, + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "Client", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "refresh_token": { + "type": "string", + "path_in_connector_config": ["credentials", "refresh_token"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["credentials", "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["credentials", "client_secret"] + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/main.py b/airbyte-integrations/connectors/source-microsoft-onedrive/main.py new file mode 100644 index 000000000000..205effefe600 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/main.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk import AirbyteEntrypoint +from airbyte_cdk.entrypoint import launch +from source_microsoft_onedrive import SourceMicrosoftOneDrive + +if __name__ == "__main__": + args = sys.argv[1:] + catalog_path = AirbyteEntrypoint.extract_catalog(args) + source = SourceMicrosoftOneDrive(catalog_path) + launch(source, args) diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/metadata.yaml b/airbyte-integrations/connectors/source-microsoft-onedrive/metadata.yaml new file mode 100644 index 000000000000..0bab747ccf4d --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/metadata.yaml @@ -0,0 +1,30 @@ +data: + ab_internal: + ql: 200 + sl: 100 + allowedHosts: + hosts: + - graph.microsoft.com + - login.microsoftonline.com + registries: + oss: + enabled: true + cloud: + enabled: false # We need to either implement OAuth for cloud or remove OAuth from the config for cloud + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 + connectorSubtype: api + connectorType: source + definitionId: 01d1c685-fd4a-4837-8f4c-93fe5a0d2188 + dockerImageTag: 0.1.0 + dockerRepository: airbyte/source-microsoft-onedrive + githubIssueLabel: source-microsoft-onedrive + icon: microsoft-onedrive.svg + license: MIT + name: Microsoft OneDrive + supportLevel: community + releaseStage: alpha + documentationUrl: https://docs.airbyte.com/integrations/sources/microsoft-onedrive + tags: + - language:python +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/requirements.txt b/airbyte-integrations/connectors/source-microsoft-onedrive/requirements.txt new file mode 100644 index 000000000000..d6e1198b1ab1 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/requirements.txt @@ -0,0 +1 @@ +-e . diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/setup.py b/airbyte-integrations/connectors/source-microsoft-onedrive/setup.py new file mode 100644 index 000000000000..c8ed1601ac4e --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/setup.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk[file-based]>=0.57.5", + "msal~=1.25.0", + "Office365-REST-Python-Client~=2.5.2", + "smart-open~=6.4.0", +] + +TEST_REQUIREMENTS = [ + "pytest-mock~=3.6.1", + "pytest~=6.1", + "requests-mock~=1.11.0", +] + +setup( + name="source_microsoft_onedrive", + description="Source implementation for Microsoft OneDrive.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/__init__.py b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/__init__.py new file mode 100644 index 000000000000..c8e3b2178fbb --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceMicrosoftOneDrive + +__all__ = ["SourceMicrosoftOneDrive"] diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/source.py b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/source.py new file mode 100644 index 000000000000..e58d75d8625c --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/source.py @@ -0,0 +1,55 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from typing import Any + +from airbyte_cdk.models import AdvancedAuth, ConnectorSpecification, OAuthConfigSpecification +from airbyte_cdk.sources.file_based.file_based_source import FileBasedSource +from airbyte_cdk.sources.file_based.stream.cursor.default_file_based_cursor import DefaultFileBasedCursor +from source_microsoft_onedrive.spec import SourceMicrosoftOneDriveSpec +from source_microsoft_onedrive.stream_reader import SourceMicrosoftOneDriveStreamReader + + +class SourceMicrosoftOneDrive(FileBasedSource): + def __init__(self, catalog_path: str): + super().__init__( + stream_reader=SourceMicrosoftOneDriveStreamReader(), + spec_class=SourceMicrosoftOneDriveSpec, + catalog_path=catalog_path, + cursor_cls=DefaultFileBasedCursor, + ) + + def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: + """ + Returns the specification describing what fields can be configured by a user when setting up a file-based source. + """ + + return ConnectorSpecification( + documentationUrl=self.spec_class.documentation_url(), + connectionSpecification=self.spec_class.schema(), + advanced_auth=AdvancedAuth( + auth_flow_type="oauth2.0", + predicate_key=["credentials", "auth_type"], + predicate_value="Client", + oauth_config_specification=OAuthConfigSpecification( + complete_oauth_output_specification={ + "type": "object", + "additionalProperties": False, + "properties": {"refresh_token": {"type": "string", "path_in_connector_config": ["credentials", "refresh_token"]}}, + }, + complete_oauth_server_input_specification={ + "type": "object", + "additionalProperties": False, + "properties": {"client_id": {"type": "string"}, "client_secret": {"type": "string"}}, + }, + complete_oauth_server_output_specification={ + "type": "object", + "additionalProperties": False, + "properties": { + "client_id": {"type": "string", "path_in_connector_config": ["credentials", "client_id"]}, + "client_secret": {"type": "string", "path_in_connector_config": ["credentials", "client_secret"]}, + }, + }, + ), + ), + ) diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/spec.py b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/spec.py new file mode 100644 index 000000000000..180993a685c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/spec.py @@ -0,0 +1,117 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from typing import Any, Dict, Literal, Optional, Union + +import dpath.util +from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec +from pydantic import BaseModel, Field + + +class OAuthCredentials(BaseModel): + """ + OAuthCredentials class to hold authentication details for Microsoft OAuth authentication. + This class uses pydantic for data validation and settings management. + """ + + class Config: + title = "Authenticate via Microsoft (OAuth)" + + # Fields for the OAuth authentication, including tenant_id, client_id, client_secret, and refresh_token + auth_type: Literal["Client"] = Field("Client", const=True) + tenant_id: str = Field(title="Tenant ID", description="Tenant ID of the Microsoft OneDrive user", airbyte_secret=True) + client_id: str = Field( + title="Client ID", + description="Client ID of your Microsoft developer application", + airbyte_secret=True, + ) + client_secret: str = Field( + title="Client Secret", + description="Client Secret of your Microsoft developer application", + airbyte_secret=True, + ) + refresh_token: str = Field( + title="Refresh Token", + description="Refresh Token of your Microsoft developer application", + airbyte_secret=True, + ) + + +class ServiceCredentials(BaseModel): + """ + ServiceCredentials class for service key authentication. + This class is structured similarly to OAuthCredentials but for a different authentication method. + """ + + class Config: + title = "Service Key Authentication" + + # Fields for the Service authentication, similar to OAuthCredentials + auth_type: Literal["Service"] = Field("Service", const=True) + tenant_id: str = Field(title="Tenant ID", description="Tenant ID of the Microsoft OneDrive user", airbyte_secret=True) + user_principal_name: str = Field( + title="User Principal Name", + description="Special characters such as a period, comma, space, and the at sign (@) are converted to underscores (_). More details: https://learn.microsoft.com/en-us/sharepoint/list-onedrive-urls", + airbyte_secret=True, + ) + client_id: str = Field( + title="Client ID", + description="Client ID of your Microsoft developer application", + airbyte_secret=True, + ) + client_secret: str = Field( + title="Client Secret", + description="Client Secret of your Microsoft developer application", + airbyte_secret=True, + ) + + +class SourceMicrosoftOneDriveSpec(AbstractFileBasedSpec, BaseModel): + """ + SourceMicrosoftOneDriveSpec class for Microsoft OneDrive Source Specification. + This class combines the authentication details with additional configuration for the OneDrive API. + """ + + class Config: + title = "Microsoft OneDrive Source Spec" + + # Union type for credentials, allowing for either OAuth or Service Key authentication + credentials: Union[OAuthCredentials, ServiceCredentials] = Field( + title="Authentication", + description="Credentials for connecting to the One Drive API", + discriminator="auth_type", + type="object", + order=0, + ) + + drive_name: Optional[str] = Field( + title="Drive Name", description="Name of the Microsoft OneDrive drive where the file(s) exist.", default="OneDrive", order=2 + ) + folder_path: str = Field( + title="Folder Path", description="Path to folder of the Microsoft OneDrive drive where the file(s) exist.", order=3 + ) + + @classmethod + def documentation_url(cls) -> str: + """Provides the URL to the documentation for this specific source.""" + return "https://docs.airbyte.com/integrations/sources/one-drive" + + @classmethod + def schema(cls, *args: Any, **kwargs: Any) -> Dict[str, Any]: + """ + Generates the schema mapping for configuration fields. + It also cleans up the schema by removing legacy settings and discriminators. + """ + schema = super().schema(*args, **kwargs) + + # Remove legacy settings related to streams + dpath.util.delete(schema, "properties/streams/items/properties/legacy_prefix") + dpath.util.delete(schema, "properties/streams/items/properties/format/oneOf/*/properties/inference_type") + + # Hide API processing option until https://github.com/airbytehq/airbyte-platform-internal/issues/10354 is fixed + processing_options = dpath.util.get(schema, "properties/streams/items/properties/format/oneOf/4/properties/processing/oneOf") + dpath.util.set(schema, "properties/streams/items/properties/format/oneOf/4/properties/processing/oneOf", processing_options[:1]) + + return schema diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/stream_reader.py b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/stream_reader.py new file mode 100644 index 000000000000..1fbd5d665bb9 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/source_microsoft_onedrive/stream_reader.py @@ -0,0 +1,176 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from functools import lru_cache +from io import IOBase +from typing import Iterable, List, Optional + +import smart_open +from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from airbyte_cdk.utils.traced_exception import AirbyteTracedException, FailureType +from msal import ConfidentialClientApplication +from msal.exceptions import MsalServiceError +from office365.graph_client import GraphClient +from source_microsoft_onedrive.spec import SourceMicrosoftOneDriveSpec + + +class MicrosoftOneDriveRemoteFile(RemoteFile): + download_url: str + + +class SourceMicrosoftOneDriveClient: + """ + Client to interact with Microsoft OneDrive. + """ + + def __init__(self, config: SourceMicrosoftOneDriveSpec): + self.config = config + self._client = None + + @property + @lru_cache(maxsize=None) + def msal_app(self): + """Returns an MSAL app instance for authentication.""" + return ConfidentialClientApplication( + self.config.credentials.client_id, + authority=f"https://login.microsoftonline.com/{self.config.credentials.tenant_id}", + client_credential=self.config.credentials.client_secret, + ) + + @property + def client(self): + """Initializes and returns a GraphClient instance.""" + if not self.config: + raise ValueError("Configuration is missing; cannot create the Office365 graph client.") + if not self._client: + self._client = GraphClient(self._get_access_token) + return self._client + + def _get_access_token(self): + """Retrieves an access token for OneDrive access.""" + scope = ["https://graph.microsoft.com/.default"] + refresh_token = self.config.credentials.refresh_token if hasattr(self.config.credentials, "refresh_token") else None + + if refresh_token: + result = self.msal_app.acquire_token_by_refresh_token(refresh_token, scopes=scope) + else: + result = self.msal_app.acquire_token_for_client(scopes=scope) + + if "access_token" not in result: + error_description = result.get("error_description", "No error description provided.") + raise MsalServiceError(error=result.get("error"), error_description=error_description) + + return result + + +class SourceMicrosoftOneDriveStreamReader(AbstractFileBasedStreamReader): + """ + A stream reader for Microsoft OneDrive. Handles file enumeration and reading from OneDrive. + """ + + ROOT_PATH = [".", "/"] + + def __init__(self): + super().__init__() + + @property + def config(self) -> SourceMicrosoftOneDriveSpec: + return self._config + + @property + def one_drive_client(self) -> SourceMicrosoftOneDriveSpec: + return SourceMicrosoftOneDriveClient(self._config).client + + @config.setter + def config(self, value: SourceMicrosoftOneDriveSpec): + """ + The FileBasedSource reads and parses configuration from a file, then sets this configuration in its StreamReader. While it only + uses keys from its abstract configuration, concrete StreamReader implementations may need additional keys for third-party + authentication. Therefore, subclasses of AbstractFileBasedStreamReader should verify that the value in their config setter + matches the expected config type for their StreamReader. + """ + assert isinstance(value, SourceMicrosoftOneDriveSpec) + self._config = value + + def list_directories_and_files(self, root_folder, path=None): + """Enumerates folders and files starting from a root folder.""" + drive_items = root_folder.children.get().execute_query() + found_items = [] + for item in drive_items: + item_path = path + "/" + item.name if path else item.name + if item.is_file: + found_items.append((item, item_path)) + else: + found_items.extend(self.list_directories_and_files(item, item_path)) + return found_items + + def get_files_by_drive_name(self, drives, drive_name, folder_path): + """Yields files from the specified drive.""" + path_levels = [level for level in folder_path.split("/") if level] + folder_path = "/".join(path_levels) + + for drive in drives: + is_onedrive = drive.drive_type in ["personal", "business"] + if drive.name == drive_name and is_onedrive: + folder = drive.root if folder_path in self.ROOT_PATH else drive.root.get_by_path(folder_path).get().execute_query() + yield from self.list_directories_and_files(folder) + + def get_matching_files(self, globs: List[str], prefix: Optional[str], logger: logging.Logger) -> Iterable[RemoteFile]: + """ + Retrieve all files matching the specified glob patterns in OneDrive. + """ + drives = self.one_drive_client.drives.get().execute_query() + + if self.config.credentials.auth_type == "Client": + my_drive = self.one_drive_client.me.drive.get().execute_query() + else: + my_drive = ( + self.one_drive_client.users.get_by_principal_name(self.config.credentials.user_principal_name).drive.get().execute_query() + ) + + drives.add_child(my_drive) + + files = self.get_files_by_drive_name(drives, self.config.drive_name, self.config.folder_path) + + try: + first_file, path = next(files) + + yield from self.filter_files_by_globs_and_start_date( + [ + MicrosoftOneDriveRemoteFile( + uri=path, + download_url=first_file.properties["@microsoft.graph.downloadUrl"], + last_modified=first_file.properties["lastModifiedDateTime"], + ) + ], + globs, + ) + + except StopIteration as e: + raise AirbyteTracedException( + internal_message=str(e), + message=f"Drive '{self.config.drive_name}' is empty or does not exist.", + failure_type=FailureType.config_error, + exception=e, + ) + + yield from self.filter_files_by_globs_and_start_date( + [ + MicrosoftOneDriveRemoteFile( + uri=path, + download_url=file.properties["@microsoft.graph.downloadUrl"], + last_modified=file.properties["lastModifiedDateTime"], + ) + for file, path in files + ], + globs, + ) + + def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger) -> IOBase: + try: + return smart_open.open(file.download_url, mode=mode.value, encoding=encoding) + except Exception as e: + logger.exception(f"Error opening file {file.uri}: {e}") diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/unit_tests/__init__.py b/airbyte-integrations/connectors/source-microsoft-onedrive/unit_tests/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-microsoft-onedrive/unit_tests/unit_tests.py b/airbyte-integrations/connectors/source-microsoft-onedrive/unit_tests/unit_tests.py new file mode 100644 index 000000000000..f610ad67a646 --- /dev/null +++ b/airbyte-integrations/connectors/source-microsoft-onedrive/unit_tests/unit_tests.py @@ -0,0 +1,100 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import Mock, patch + +from source_microsoft_onedrive.spec import SourceMicrosoftOneDriveSpec +from source_microsoft_onedrive.stream_reader import FileReadMode, SourceMicrosoftOneDriveClient, SourceMicrosoftOneDriveStreamReader + + +def create_mock_drive_item(is_file, name): + """Helper function to create a mock drive item.""" + mock_item = Mock() + mock_item.is_file = is_file + mock_item.name = name + return mock_item + + +@patch("smart_open.open") +def test_open_file(mock_smart_open): + """Test the open_file method in SourceMicrosoftOneDriveStreamReader.""" + mock_file = Mock(download_url="http://example.com/file.txt") + mock_logger = Mock() + + stream_reader = SourceMicrosoftOneDriveStreamReader() + stream_reader._config = Mock() # Assuming _config is required + + with stream_reader.open_file(mock_file, FileReadMode.READ, "utf-8", mock_logger) as result: + pass + + mock_smart_open.assert_called_once_with(mock_file.download_url, mode='r', encoding='utf-8') + assert result is not None + + +def test_microsoft_onedrive_client_initialization(requests_mock): + """Test the initialization of SourceMicrosoftOneDriveClient.""" + config = { + "credentials": { + "auth_type": "Client", + "client_id": "client_id", + "tenant_id": "tenant_id", + "client_secret": "client_secret", + "refresh_token": "refresh_token" + }, + "drive_name": "drive_name", + "folder_path": "folder_path", + "streams": [{"name": "test_stream", "globs": ["*.csv"], "validation_policy": "Emit Record", "format": {"filetype": "csv"}}] + } + + authority_url = 'https://login.microsoftonline.com/tenant_id/v2.0/.well-known/openid-configuration' + mock_response = {'authorization_endpoint': 'https://login.microsoftonline.com/tenant_id/oauth2/v2.0/authorize', 'token_endpoint': 'https://login.microsoftonline.com/tenant_id/oauth2/v2.0/token'} + requests_mock.get(authority_url, json=mock_response, status_code=200) + + client = SourceMicrosoftOneDriveClient(SourceMicrosoftOneDriveSpec(**config)) + + assert client.config == SourceMicrosoftOneDriveSpec(**config) + assert client.msal_app is not None + + +@patch("source_microsoft_onedrive.stream_reader.SourceMicrosoftOneDriveStreamReader.list_directories_and_files") +def test_list_directories_and_files(mock_list_directories_and_files): + """Test the list_directories_and_files method in SourceMicrosoftOneDriveStreamReader.""" + mock_root_folder = create_mock_drive_item(False, "root") + mock_child_file = create_mock_drive_item(True, "file1.txt") + mock_child_folder = create_mock_drive_item(False, "folder1") + mock_child_folder.children.get().execute_query.return_value = [mock_child_file] + mock_root_folder.children.get().execute_query.return_value = [mock_child_folder, mock_child_file] + + mock_list_directories_and_files.return_value = [mock_child_folder, mock_child_file] + + stream_reader = SourceMicrosoftOneDriveStreamReader() + result = stream_reader.list_directories_and_files(mock_root_folder) + + assert len(result) == 2 + assert result[0].name == "folder1" + assert result[1].name == "file1.txt" + + +@patch("source_microsoft_onedrive.stream_reader.SourceMicrosoftOneDriveStreamReader.list_directories_and_files") +def test_get_files_by_drive_name(mock_list_directories_and_files): + # Helper function usage + mock_drive = Mock() + mock_drive.name = "testDrive" + mock_drive.drive_type = "business" + mock_drive.root.get_by_path.return_value.get().execute_query.return_value = create_mock_drive_item(is_file=False, name="root") + + # Mock files + mock_file = create_mock_drive_item(is_file=True, name="testFile.txt") + mock_list_directories_and_files.return_value = [mock_file] + + # Create stream reader instance + stream_reader = SourceMicrosoftOneDriveStreamReader() + stream_reader._config = Mock() + + # Call the method + files = list(stream_reader.get_files_by_drive_name([mock_drive], "testDrive", "/test/path")) + + # Assertions + assert len(files) == 1 + assert files[0].name == "testFile.txt" diff --git a/airbyte-integrations/connectors/source-microsoft-teams/Dockerfile b/airbyte-integrations/connectors/source-microsoft-teams/Dockerfile index 3cdb20113e74..4b206258d0b3 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/Dockerfile +++ b/airbyte-integrations/connectors/source-microsoft-teams/Dockerfile @@ -34,5 +34,5 @@ COPY source_microsoft_teams ./source_microsoft_teams ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.5 +LABEL io.airbyte.version=1.0.0 LABEL io.airbyte.name=airbyte/source-microsoft-teams diff --git a/airbyte-integrations/connectors/source-microsoft-teams/README.md b/airbyte-integrations/connectors/source-microsoft-teams/README.md index e6a88cfebfbf..de8db4cc28b2 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/README.md +++ b/airbyte-integrations/connectors/source-microsoft-teams/README.md @@ -1,7 +1,7 @@ -# Microsoft Teams Source +# Rabbitmq Destination -This is the repository for the Microsoft Teams source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/microsoft-teams). +This is the repository for the Rabbitmq destination connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/destinations/rabbitmq). ## Local development @@ -29,72 +29,71 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-microsoft-teams:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/microsoft-teams) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_microsoft_teams/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/destinations/rabbitmq) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_rabbitmq/spec.json` file. Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. +See `integration_tests/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source microsoft-teams test creds` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `destination rabbitmq test creds` and place them into `secrets/config.json`. - ### Locally running the connector ``` -python main_dev.py spec -python main_dev.py check --config secrets/config.json -python main_dev.py discover --config secrets/config.json -python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json ``` ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-microsoft-teams:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name destination-rabbitmq build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-microsoft-teams:airbyteDocker +An image will be built with the tag `airbyte/destination-rabbitmq:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/destination-rabbitmq:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-microsoft-teams:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-microsoft-teams:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-microsoft-teams:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-microsoft-teams:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm airbyte/destination-rabbitmq:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-rabbitmq:dev check --config /secrets/config.json +# messages.jsonl is a file containing line-separated JSON representing AirbyteMessages +cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-rabbitmq:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-microsoft-teams:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-microsoft-teams test +``` + +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-microsoft-teams test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/microsoft-teams.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-microsoft-teams/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-microsoft-teams/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-microsoft-teams/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-microsoft-teams/build.gradle b/airbyte-integrations/connectors/source-microsoft-teams/build.gradle deleted file mode 100644 index 2b91c3d20d33..000000000000 --- a/airbyte-integrations/connectors/source-microsoft-teams/build.gradle +++ /dev/null @@ -1,10 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_microsoft_teams' -} - diff --git a/airbyte-integrations/connectors/source-microsoft-teams/metadata.yaml b/airbyte-integrations/connectors/source-microsoft-teams/metadata.yaml index cf90d7d46549..a554cf83e32e 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/metadata.yaml +++ b/airbyte-integrations/connectors/source-microsoft-teams/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: eaf50f04-21dd-4620-913b-2a83f5635227 - dockerImageTag: 0.2.5 + dockerImageTag: 1.0.0 dockerRepository: airbyte/source-microsoft-teams githubIssueLabel: source-microsoft-teams icon: microsoft-teams.svg @@ -13,6 +13,14 @@ data: enabled: true oss: enabled: true + releases: + breakingChanges: + 1.0.0: + message: + Version 1.0.0 introduces breaking schema changes to all streams. + A full schema refresh is required to upgrade to this version. + For more details, see our migration guide. + upgradeDeadline: "2024-01-24" releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/microsoft-teams tags: diff --git a/airbyte-integrations/connectors/source-microsoft-teams/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-microsoft-teams/sample_files/configured_catalog.json index ca4a5fc9076c..a395f1f9b46a 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/sample_files/configured_catalog.json @@ -5,55 +5,7 @@ "name": "users", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "business_phones": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] - }, - "display_name": { - "type": ["null", "string"] - }, - "given_name": { - "type": ["null", "string"] - }, - "job_title": { - "type": ["null", "string"] - }, - "mail": { - "type": ["null", "string"] - }, - "mobile_phone": { - "type": ["null", "string"] - }, - "office_location": { - "type": ["null", "string"] - }, - "preferred_language": { - "type": ["null", "string"] - }, - "surname": { - "type": ["null", "string"] - }, - "user_principal_name": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -63,159 +15,7 @@ "name": "groups", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "deleted_date_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "classification": { - "type": ["null", "string"] - }, - "created_date_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "creation_options": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] - }, - "description": { - "type": ["null", "string"] - }, - "display_name": { - "type": ["null", "string"] - }, - "expiration_date_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "group_types": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] - }, - "is_assignable_to_role": { - "type": ["null", "boolean"] - }, - "mail": { - "type": ["null", "string"] - }, - "mail_enabled": { - "type": ["null", "boolean"] - }, - "mail_nickname": { - "type": ["null", "string"] - }, - "membership_rule": { - "type": ["null", "string"] - }, - "membership_rule_processing_state": { - "type": ["null", "string"] - }, - "onPremises_domain_name": { - "type": ["null", "string"] - }, - "on_premises_last_sync_date_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "on_premises_net_bios_name": { - "type": ["null", "string"] - }, - "on_premises_sam_account_name": { - "type": ["null", "string"] - }, - "on_premises_security_identifier": { - "type": ["null", "string"] - }, - "on_premises_sync_enabled": { - "type": ["null", "boolean"] - }, - "preferred_data_location": { - "type": ["null", "string"] - }, - "preferred_language": { - "type": ["null", "string"] - }, - "proxy_addresses": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] - }, - "renewed_date_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "resource_behavior_options": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "resource_provisioning_options": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "security_enabled": { - "type": ["null", "boolean"] - }, - "security_edentifier": { - "type": ["null", "string"] - }, - "theme": { - "type": ["null", "string"] - }, - "visibility": { - "type": ["null", "string"] - }, - "on_premises_provisioning_errors": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -225,55 +25,7 @@ "name": "group_members", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "business_phones": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] - }, - "display_name": { - "type": ["null", "string"] - }, - "given_name": { - "type": ["null", "string"] - }, - "job_title": { - "type": ["null", "string"] - }, - "mail": { - "type": ["null", "string"] - }, - "mobile_phone": { - "type": ["null", "string"] - }, - "office_location": { - "type": ["null", "string"] - }, - "preferred_language": { - "type": ["null", "string"] - }, - "surname": { - "type": ["null", "string"] - }, - "user_principal_name": { - "type": ["null", "string"] - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -283,58 +35,7 @@ "name": "group_owners", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "group_id": { - "type": ["null", "string"] - }, - "business_phones": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] - }, - "display_name": { - "type": ["null", "string"] - }, - "given_name": { - "type": ["null", "string"] - }, - "job_title": { - "type": ["null", "string"] - }, - "mail": { - "type": ["null", "string"] - }, - "mobile_phone": { - "type": ["null", "string"] - }, - "office_location": { - "type": ["null", "string"] - }, - "preferred_language": { - "type": ["null", "string"] - }, - "surname": { - "type": ["null", "string"] - }, - "user_principal_name": { - "type": ["null", "string"] - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -344,27 +45,7 @@ "name": "channels", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "display_name": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "web_url": { - "type": ["null", "string"] - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -374,40 +55,7 @@ "name": "channel_members", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "display_name": { - "type": ["null", "string"] - }, - "roles": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] - }, - "user_id": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - }, - "channel_id": { - "type": ["null", "string"] - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -417,72 +65,7 @@ "name": "channel_tabs", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "group_id": { - "type": ["null", "string"] - }, - "channel_id": { - "type": ["null", "string"] - }, - "display_name": { - "type": ["null", "string"] - }, - "web_url": { - "type": ["null", "string"] - }, - "sort_order_index": { - "type": ["null", "string"] - }, - "teams_app": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "id": { - "type": ["null", "string"] - }, - "display_name": { - "type": ["null", "string"] - }, - "distribution_method": { - "type": ["null", "string"] - } - } - }, - "configuration": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "entity_id": { - "type": ["null", "string"] - }, - "content_url": { - "type": ["null", "string"] - }, - "remove_url": { - "type": ["null", "string"] - }, - "website_url": { - "type": ["null", "string"] - }, - "wiki_tab_id": { - "type": ["null", "integer"] - }, - "wiki_default_tab": { - "type": ["null", "boolean"] - }, - "has_content": { - "type": ["null", "boolean"] - } - } - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -492,72 +75,7 @@ "name": "conversations", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "group_id": { - "type": ["null", "string"] - }, - "channel_id": { - "type": ["null", "string"] - }, - "display_name": { - "type": ["null", "string"] - }, - "web_url": { - "type": ["null", "string"] - }, - "sort_order_index": { - "type": ["null", "string"] - }, - "teams_app": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "id": { - "type": ["null", "string"] - }, - "display_name": { - "type": ["null", "string"] - }, - "distribution_method": { - "type": ["null", "string"] - } - } - }, - "configuration": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "entity_id": { - "type": ["null", "string"] - }, - "content_url": { - "type": ["null", "string"] - }, - "remove_url": { - "type": ["null", "string"] - }, - "website_url": { - "type": ["null", "string"] - }, - "wiki_tab_id": { - "type": ["null", "integer"] - }, - "wiki_default_tab": { - "type": ["null", "boolean"] - }, - "has_content": { - "type": ["null", "boolean"] - } - } - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -567,40 +85,7 @@ "name": "conversation_threads", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "group_id": { - "type": ["null", "string"] - }, - "conversation_id": { - "type": ["null", "string"] - }, - "topic": { - "type": ["null", "string"] - }, - "has_attachments": { - "type": ["null", "boolean"] - }, - "last_delivered_date_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "unique_senders": { - "type": ["null", "string"] - }, - "preview": { - "type": ["null", "string"] - }, - "is_locked": { - "type": ["null", "boolean"] - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -610,100 +95,7 @@ "name": "conversation_posts", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "thread_id": { - "type": ["null", "string"] - }, - "conversation_id": { - "type": ["null", "string"] - }, - "created_date_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "last_modified_date_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "change_key": { - "type": ["null", "string"] - }, - "categories": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] - }, - "received_date_time": { - "type": ["null", "string"], - "format": "date-time" - }, - "has_attachments": { - "type": ["null", "boolean"] - }, - "body": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "content_type": { - "type": ["null", "string"] - }, - "content": { - "type": ["null", "string"] - } - } - }, - "from": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "emailAddress": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "name": { - "type": ["null", "string"] - }, - "address": { - "type": ["null", "string"] - } - } - } - } - }, - "sender": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "emailAddress": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "name": { - "type": ["null", "string"] - }, - "address": { - "type": ["null", "string"] - } - } - } - } - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -713,84 +105,7 @@ "name": "team_drives", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "last_modified_date_time": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "web_url": { - "type": ["null", "string"] - }, - "drive_type": { - "type": ["null", "string"] - }, - "created_by": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "user": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "display_name": { - "type": ["null", "string"] - } - } - } - } - }, - "owner": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "group": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "email": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "string"] - }, - "display_name": { - "type": ["null", "string"] - } - } - } - } - }, - "quota": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "deleted": { - "type": ["null", "integer"] - }, - "remaining": { - "type": ["null", "number"] - }, - "state": { - "type": ["null", "string"] - }, - "total": { - "type": ["null", "number"] - }, - "used": { - "type": ["null", "integer"] - } - } - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -800,48 +115,7 @@ "name": "team_device_usage_report", "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "report_refresh_date": { - "type": ["null", "string"] - }, - "user_principal_name": { - "type": ["null", "string"] - }, - "last_activity_date": { - "type": ["null", "string"] - }, - "is_deleted": { - "type": ["null", "string"] - }, - "deleted_date": { - "type": ["null", "string"] - }, - "used_web": { - "type": ["null", "string"] - }, - "used_windows_phone": { - "type": ["null", "string"] - }, - "used_i_os": { - "type": ["null", "string"] - }, - "used_mac": { - "type": ["null", "string"] - }, - "used_android_phone": { - "type": ["null", "string"] - }, - "used_windows": { - "type": ["null", "string"] - }, - "report_period": { - "type": ["null", "string"] - } - } - } + "json_schema": {} }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" diff --git a/airbyte-integrations/connectors/source-microsoft-teams/setup.py b/airbyte-integrations/connectors/source-microsoft-teams/setup.py index 1867013845c2..6cc04d3f3b07 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/setup.py +++ b/airbyte-integrations/connectors/source-microsoft-teams/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", "requests", "msal==1.7.0", "backoff", diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/client.py b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/client.py index 8a3d4893fa77..c16459025f98 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/client.py +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/client.py @@ -243,19 +243,24 @@ def get_team_device_usage_report(self): csv_response.readline() with io.TextIOWrapper(csv_response, encoding="utf-8-sig") as text_file: field_names = [ - "report_refresh_date", - "user_principal_name", - "last_activity_date", - "is_deleted", - "deleted_date", - "used_web", - "used_windows_phone", - "used_i_os", - "used_mac", - "used_android_phone", - "used_windows", - "report_period", + "reportRefreshDate", + "userId", + "userPrincipalName", + "lastActivityDate", + "isDeleted", + "deletedDate", + "usedWeb", + "usedWindowsPhone", + "usedIOs", + "usedMac", + "usedAndroidPhone", + "usedWindows", + "usedChromeOS", + "usedLinux", + "isLisenced", + "reportPeriod", ] + reader = csv.DictReader(text_file, fieldnames=field_names) for row in reader: yield [ diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel.json index 536c1efc8116..b99d57b03940 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel.json @@ -6,21 +6,14 @@ "id": { "type": ["null", "string"] }, - "display_name": { + "displayName": { "type": ["null", "string"] }, "roles": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, "description": { "type": ["null", "string"] @@ -28,7 +21,7 @@ "email": { "type": ["null", "string"] }, - "web_url": { + "webUrl": { "type": ["null", "string"] } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_members.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_members.json index 0f72d22d63c5..3c236063c1c6 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_members.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_members.json @@ -6,30 +6,34 @@ "id": { "type": ["null", "string"] }, - "display_name": { + "@odata.type": { + "type": ["null", "string"] + }, + "displayName": { "type": ["null", "string"] }, "roles": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] - }, - "user_id": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "userId": { "type": ["null", "string"] }, "email": { "type": ["null", "string"] }, - "channel_id": { + "channelId": { "type": ["null", "string"] + }, + "tenantId": { + "type": ["null", "string"] + }, + "visibleHistoryStartDateTime": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" } } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_message_replies.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_message_replies.json index 9e58d5f02fbe..ac7cbb06d002 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_message_replies.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_message_replies.json @@ -6,26 +6,34 @@ "id": { "type": ["null", "string"] }, - "reply_to_id": { + "replyToId": { "type": ["null", "string"] }, "etag": { "type": ["null", "string"] }, - "message_type": { + "messageType": { "type": ["null", "string"] }, - "created_date_time": { + "createdDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "last_modified_date_time": { + "lastModifiedDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "deleted_date_time": { + "lastEditedDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "deletedDateTime": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, "subject": { "type": ["null", "string"] @@ -33,7 +41,7 @@ "summary": { "type": ["null", "string"] }, - "chat_id": { + "chatId": { "type": ["null", "string"] }, "importance": { @@ -42,10 +50,10 @@ "locale": { "type": ["null", "string"] }, - "web_url": { + "webUrl": { "type": ["null", "string"] }, - "policy_violation": { + "policyViolation": { "type": ["null", "string"] }, "from": { @@ -68,10 +76,13 @@ "id": { "type": ["null", "string"] }, - "display_name": { + "displayName": { "type": ["null", "string"] }, - "user_identity_type": { + "userIdentityType": { + "type": ["null", "string"] + }, + "tenantId": { "type": ["null", "string"] } } @@ -82,7 +93,7 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "content_type": { + "contentType": { "type": ["null", "string"] }, "content": { @@ -90,156 +101,136 @@ } } }, - "channel_identity": { + "channelIdentity": { "type": ["null", "object"], "additionalProperties": true, "properties": { "teamId": { "type": ["null", "string"] }, - "channel_id": { + "channelId": { "type": ["null", "string"] } } }, "attachments": { - "anyOf": [ - { - "type": "array", - "items": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "id": { - "type": ["null", "string"] - }, - "content_type": { - "type": ["null", "string"] - }, - "content_url": { - "type": ["null", "string"] - }, - "content": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "thumbnail_url": { - "type": ["null", "string"] - } - } + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "string"] + }, + "contentType": { + "type": ["null", "string"] + }, + "contentUrl": { + "type": ["null", "string"] + }, + "content": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "thumbnailUrl": { + "type": ["null", "string"] } - }, - { - "type": "null" } - ] + } }, "mentions": { - "anyOf": [ - { - "type": "array", - "items": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "mentionText": { + "type": ["null", "string"] + }, + "mentioned": { "type": ["null", "object"], "additionalProperties": true, "properties": { - "id": { - "type": ["null", "integer"] + "application": { + "type": ["null", "string"] }, - "mention_text": { + "device": { "type": ["null", "string"] }, - "mentioned": { + "conversation": { + "type": ["null", "string"] + }, + "user": { "type": ["null", "object"], "additionalProperties": true, "properties": { - "application": { + "id": { "type": ["null", "string"] }, - "device": { + "displayName": { "type": ["null", "string"] }, - "conversation": { + "userIdentityType": { "type": ["null", "string"] - }, - "user": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "id": { - "type": ["null", "string"] - }, - "display_name": { - "type": ["null", "string"] - }, - "user_identity_type": { - "type": ["null", "string"] - } - } } } } } } - }, - { - "type": "null" } - ] + } }, "reactions": { - "anyOf": [ - { - "type": "array", - "items": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "reactionType": { + "type": ["null", "string"] + }, + "createdDateTime": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "user": { "type": ["null", "object"], "additionalProperties": true, "properties": { - "reaction_type": { + "application": { + "type": ["null", "string"] + }, + "device": { "type": ["null", "string"] }, - "created_date_time": { - "type": ["null", "string"], - "format": "date-time" + "conversation": { + "type": ["null", "string"] }, "user": { "type": ["null", "object"], "additionalProperties": true, "properties": { - "application": { + "id": { "type": ["null", "string"] }, - "device": { + "displayName": { "type": ["null", "string"] }, - "conversation": { + "userIdentityType": { "type": ["null", "string"] - }, - "user": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "id": { - "type": ["null", "string"] - }, - "display_name": { - "type": ["null", "string"] - }, - "user_identity_type": { - "type": ["null", "string"] - } - } } } } } } - }, - { - "type": "null" } - ] + } } } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_messages.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_messages.json index 3b72078dfbb9..ced93bf8121b 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_messages.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_messages.json @@ -6,26 +6,29 @@ "id": { "type": ["null", "string"] }, - "reply_to_id": { + "replyToId": { "type": ["null", "string"] }, "etag": { "type": ["null", "string"] }, - "message_type": { + "messageType": { "type": ["null", "string"] }, - "created_date_time": { + "createdDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "lastModified_date_time": { + "lastModifiedDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "deleted_date_time": { + "deletedDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, "subject": { "type": ["null", "string"] @@ -33,7 +36,7 @@ "summary": { "type": ["null", "string"] }, - "chat_id": { + "chatId": { "type": ["null", "string"] }, "importance": { @@ -42,10 +45,10 @@ "locale": { "type": ["null", "string"] }, - "web_url": { + "webUrl": { "type": ["null", "string"] }, - "policy_violation": { + "policyViolation": { "type": ["null", "string"] }, "from": { @@ -68,10 +71,10 @@ "id": { "type": ["null", "string"] }, - "display_name": { + "displayName": { "type": ["null", "string"] }, - "user_identity_type": { + "userIdentityType": { "type": ["null", "string"] } } @@ -82,7 +85,7 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "content_type": { + "contentType": { "type": ["null", "string"] }, "content": { @@ -90,14 +93,14 @@ } } }, - "channel_identity": { + "channelIdentity": { "type": ["null", "object"], "additionalProperties": true, "properties": { "teamId": { "type": ["null", "string"] }, - "channel_id": { + "channelId": { "type": ["null", "string"] } } @@ -113,10 +116,10 @@ "id": { "type": ["null", "string"] }, - "content_type": { + "contentType": { "type": ["null", "string"] }, - "content_url": { + "contentUrl": { "type": ["null", "string"] }, "content": { @@ -125,7 +128,7 @@ "name": { "type": ["null", "string"] }, - "thumbnail_url": { + "thumbnailUrl": { "type": ["null", "string"] } } @@ -147,7 +150,7 @@ "id": { "type": ["null", "integer"] }, - "mention_text": { + "mentionText": { "type": ["null", "string"] }, "mentioned": { @@ -170,10 +173,10 @@ "id": { "type": ["null", "string"] }, - "display_name": { + "displayName": { "type": ["null", "string"] }, - "userIdentity_type": { + "userIdentityType": { "type": ["null", "string"] } } @@ -196,10 +199,10 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "reaction_type": { + "reactionType": { "type": ["null", "string"] }, - "created_date_time": { + "createdDateTime": { "type": ["null", "string"], "format": "date-time" }, @@ -223,10 +226,10 @@ "id": { "type": ["null", "string"] }, - "display_name": { + "displayName": { "type": ["null", "string"] }, - "user_identity_type": { + "userIdentityType": { "type": ["null", "string"] } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_tabs.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_tabs.json index c66b4bf72179..ed0867b1e4c7 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_tabs.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channel_tabs.json @@ -6,32 +6,32 @@ "id": { "type": ["null", "string"] }, - "group_id": { + "groupId": { "type": ["null", "string"] }, - "channel_id": { + "channelId": { "type": ["null", "string"] }, - "display_name": { + "displayName": { "type": ["null", "string"] }, - "web_url": { + "webUrl": { "type": ["null", "string"] }, - "sort_order_index": { + "sortOrderIndex": { "type": ["null", "string"] }, - "teams_app": { + "teamsApp": { "type": ["null", "object"], "additionalProperties": true, "properties": { "id": { "type": ["null", "string"] }, - "display_name": { + "displayName": { "type": ["null", "string"] }, - "distribution_method": { + "distributionMethod": { "type": ["null", "string"] } } @@ -40,25 +40,25 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "entity_id": { + "entityId": { "type": ["null", "string"] }, - "content_url": { + "contentUrl": { "type": ["null", "string"] }, - "remove_url": { + "removeUrl": { "type": ["null", "string"] }, - "website_url": { + "websiteUrl": { "type": ["null", "string"] }, - "wiki_tab_id": { + "wikiTabId": { "type": ["null", "integer"] }, - "wiki_default_tab": { + "wikiDefaultTab": { "type": ["null", "boolean"] }, - "has_content": { + "hasContent": { "type": ["null", "boolean"] } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channels.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channels.json index 156390fc505e..999eae607c23 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channels.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/channels.json @@ -6,7 +6,12 @@ "id": { "type": ["null", "string"] }, - "display_name": { + "createdDateTime": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "displayName": { "type": ["null", "string"] }, "description": { @@ -15,7 +20,16 @@ "email": { "type": ["null", "string"] }, - "web_url": { + "isFavoriteByDefault": { + "type": ["null", "boolean"] + }, + "membershipType": { + "type": ["null", "string"] + }, + "tenantId": { + "type": ["null", "string"] + }, + "webUrl": { "type": ["null", "string"] } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversation_posts.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversation_posts.json index 24bf5cd4268e..4389b581962a 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversation_posts.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversation_posts.json @@ -6,48 +6,47 @@ "id": { "type": ["null", "string"] }, - "thread_id": { + "threadId": { "type": ["null", "string"] }, - "conversation_id": { + "conversationId": { "type": ["null", "string"] }, - "created_date_time": { + "createdDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "last_modified_date_time": { + "lastModifiedDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "change_key": { + "@odata.etag": { + "type": ["null", "string"] + }, + "changeKey": { "type": ["null", "string"] }, "categories": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, - "received_date_time": { + "receivedDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "has_attachments": { + "hasAttachments": { "type": ["null", "boolean"] }, "body": { "type": ["null", "object"], "additionalProperties": true, "properties": { - "content_type": { + "contentType": { "type": ["null", "string"] }, "content": { @@ -61,7 +60,6 @@ "properties": { "emailAddress": { "type": ["null", "object"], - "additionalProperties": false, "properties": { "name": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversation_threads.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversation_threads.json index 35078c5f4d89..54c27157062e 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversation_threads.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversation_threads.json @@ -6,29 +6,33 @@ "id": { "type": ["null", "string"] }, - "group_id": { + "groupId": { "type": ["null", "string"] }, - "conversation_id": { + "conversationId": { "type": ["null", "string"] }, "topic": { "type": ["null", "string"] }, - "has_attachments": { + "hasAttachments": { "type": ["null", "boolean"] }, - "last_delivered_date_time": { + "lastDeliveredDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "unique_senders": { - "type": ["null", "string"] + "uniqueSenders": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, "preview": { "type": ["null", "string"] }, - "is_locked": { + "isLocked": { "type": ["null", "boolean"] } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversations.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversations.json index 58a88cffff10..e9045284dd5c 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversations.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/conversations.json @@ -6,31 +6,25 @@ "id": { "type": ["null", "string"] }, - "group_id": { + "groupId": { "type": ["null", "string"] }, "topic": { "type": ["null", "string"] }, - "has_attachments": { + "hasAttachments": { "type": ["null", "boolean"] }, - "last_delivered_date_time": { + "lastDeliveredDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "unique_senders": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] + "uniqueSenders": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, "preview": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/group_members.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/group_members.json index ef6a3bb26028..2bf02fd72977 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/group_members.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/group_members.json @@ -6,44 +6,40 @@ "id": { "type": ["null", "string"] }, - "business_phones": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] + "@odata.type": { + "type": ["null", "string"] + }, + "businessPhones": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, - "display_name": { + "displayName": { "type": ["null", "string"] }, - "given_name": { + "givenName": { "type": ["null", "string"] }, - "job_title": { + "jobTitle": { "type": ["null", "string"] }, "mail": { "type": ["null", "string"] }, - "mobile_phone": { + "mobilePhone": { "type": ["null", "string"] }, - "office_location": { + "officeLocation": { "type": ["null", "string"] }, - "preferred_language": { + "preferredLanguage": { "type": ["null", "string"] }, "surname": { "type": ["null", "string"] }, - "user_principal_name": { + "userPrincipalName": { "type": ["null", "string"] } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/group_owners.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/group_owners.json index dc222c183354..aa1b8915682d 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/group_owners.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/group_owners.json @@ -6,47 +6,43 @@ "id": { "type": ["null", "string"] }, - "group_id": { + "groupId": { "type": ["null", "string"] }, - "business_phones": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] + "@odata.type": { + "type": ["null", "string"] + }, + "businessPhones": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, - "display_name": { + "displayName": { "type": ["null", "string"] }, - "given_name": { + "givenName": { "type": ["null", "string"] }, - "job_title": { + "jobTitle": { "type": ["null", "string"] }, "mail": { "type": ["null", "string"] }, - "mobile_phone": { + "mobilePhone": { "type": ["null", "string"] }, - "office_location": { + "officeLocation": { "type": ["null", "string"] }, - "preferred_language": { + "preferredLanguage": { "type": ["null", "string"] }, "surname": { "type": ["null", "string"] }, - "user_principal_name": { + "userPrincipalName": { "type": ["null", "string"] } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/groups.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/groups.json index a348b7743603..2876585b6fad 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/groups.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/groups.json @@ -6,149 +6,144 @@ "id": { "type": ["null", "string"] }, - "deleted_date_time": { + "deletedDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, "classification": { "type": ["null", "string"] }, - "created_date_time": { + "createdDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "creation_options": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] + "creationOptions": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, "description": { "type": ["null", "string"] }, - "display_name": { + "displayName": { "type": ["null", "string"] }, - "expiration_date_time": { + "expirationDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "group_types": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] + "groupTypes": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, - "is_assignable_to_role": { + "isAssignableToRole": { "type": ["null", "boolean"] }, "mail": { "type": ["null", "string"] }, - "mail_enabled": { + "mailEnabled": { "type": ["null", "boolean"] }, - "mail_nickname": { + "mailNickname": { "type": ["null", "string"] }, - "membership_rule": { + "membershipRule": { "type": ["null", "string"] }, - "membership_rule_processing_state": { + "membershipRuleProcessingState": { "type": ["null", "string"] }, - "onPremises_domain_name": { + "onPremisesDomainName": { "type": ["null", "string"] }, - "on_premises_last_sync_date_time": { + "onPremisesLastSyncDateTime": { "type": ["null", "string"], "format": "date-time" }, - "on_premises_net_bios_name": { + "onPremisesNetBiosName": { "type": ["null", "string"] }, - "on_premises_sam_account_name": { + "onPremisesSamAccountName": { "type": ["null", "string"] }, - "on_premises_security_identifier": { + "onPremisesSecurityIdentifier": { "type": ["null", "string"] }, - "on_premises_sync_enabled": { + "onPremisesSyncEnabled": { "type": ["null", "boolean"] }, - "preferred_data_location": { + "preferredDataLocation": { "type": ["null", "string"] }, - "preferred_language": { + "preferredLanguage": { "type": ["null", "string"] }, - "proxy_addresses": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] + "proxyAddresses": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, - "renewed_date_time": { + "renewedDateTime": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, - "resource_behavior_options": { + "resourceBehaviorOptions": { "type": ["null", "array"], "items": { "type": ["null", "string"] } }, - "resource_provisioning_options": { + "resourceProvisioningOptions": { "type": ["null", "array"], "items": { "type": ["null", "string"] } }, - "security_enabled": { + "securityEnabled": { "type": ["null", "boolean"] }, - "security_edentifier": { + "securityIdentifier": { "type": ["null", "string"] }, + "serviceProvisioningErrors": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "createdDateTime": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "isResolved": { + "type": ["null", "boolean"] + }, + "serviceInstance": { + "type": ["null", "string"] + } + } + } + }, "theme": { "type": ["null", "string"] }, "visibility": { "type": ["null", "string"] }, - "on_premises_provisioning_errors": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] + "onPremisesProvisioningErrors": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } } } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/team_device_usage_report.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/team_device_usage_report.json index 40066ae4fc4d..8ae6a571f5d3 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/team_device_usage_report.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/team_device_usage_report.json @@ -3,40 +3,53 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "report_refresh_date": { + "reportRefreshDate": { + "type": ["null", "string"], + "format": "date" + }, + "userId": { + "type": ["null", "string"] + }, + "userPrincipalName": { + "type": ["null", "string"] + }, + "lastActivityDate": { + "type": ["null", "string"] + }, + "isDeleted": { "type": ["null", "string"] }, - "user_principal_name": { + "deletedDate": { "type": ["null", "string"] }, - "last_activity_date": { + "usedWeb": { "type": ["null", "string"] }, - "is_deleted": { + "usedWindowsPhone": { "type": ["null", "string"] }, - "deleted_date": { + "usedIOs": { "type": ["null", "string"] }, - "used_web": { + "usedMac": { "type": ["null", "string"] }, - "used_windows_phone": { + "usedAndroidPhone": { "type": ["null", "string"] }, - "used_i_os": { + "usedWindows": { "type": ["null", "string"] }, - "used_mac": { + "usedChromeOS": { "type": ["null", "string"] }, - "used_android_phone": { + "usedLinux": { "type": ["null", "string"] }, - "used_windows": { + "isLisenced": { "type": ["null", "string"] }, - "report_period": { + "reportPeriod": { "type": ["null", "string"] } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/team_drives.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/team_drives.json index 0b39515c620e..fbb40f7dcf97 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/team_drives.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/team_drives.json @@ -6,19 +6,41 @@ "id": { "type": ["null", "string"] }, - "last_modified_date_time": { + "createdDateTime": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "description": { "type": ["null", "string"] }, + "lastModifiedBy": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "displayName": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + } + } + }, + "lastModifiedDateTime": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, "name": { "type": ["null", "string"] }, - "web_url": { + "webUrl": { "type": ["null", "string"] }, - "drive_type": { + "driveType": { "type": ["null", "string"] }, - "created_by": { + "createdBy": { "type": ["null", "object"], "additionalProperties": true, "properties": { @@ -26,7 +48,7 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "display_name": { + "displayName": { "type": ["null", "string"] } } @@ -47,7 +69,7 @@ "id": { "type": ["null", "string"] }, - "display_name": { + "displayName": { "type": ["null", "string"] } } diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/users.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/users.json index b5853c0d396e..e02d86a53106 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/users.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/schemas/users.json @@ -3,44 +3,37 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "business_phones": { - "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, - { - "type": "null" - } - ] + "businessPhones": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, - "display_name": { + "displayName": { "type": ["null", "string"] }, - "given_name": { + "givenName": { "type": ["null", "string"] }, - "job_title": { + "jobTitle": { "type": ["null", "string"] }, "mail": { "type": ["null", "string"] }, - "mobile_phone": { + "mobilePhone": { "type": ["null", "string"] }, - "office_location": { + "officeLocation": { "type": ["null", "string"] }, - "preferred_language": { + "preferredLanguage": { "type": ["null", "string"] }, "surname": { "type": ["null", "string"] }, - "user_principal_name": { + "userPrincipalName": { "type": ["null", "string"] }, "id": { diff --git a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/spec.json b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/spec.json index ab4af0e7d1ed..39de5a8b8a96 100644 --- a/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/spec.json +++ b/airbyte-integrations/connectors/source-microsoft-teams/source_microsoft_teams/spec.json @@ -27,7 +27,6 @@ "client_secret", "refresh_token" ], - "additionalProperties": false, "properties": { "auth_type": { "type": "string", @@ -39,7 +38,8 @@ "tenant_id": { "title": "Directory (tenant) ID", "type": "string", - "description": "A globally unique identifier (GUID) that is different than your organization name or domain. Follow these steps to obtain: open one of the Teams where you belong inside the Teams Application -> Click on the … next to the Team title -> Click on Get link to team -> Copy the link to the team and grab the tenant ID form the URL" + "description": "A globally unique identifier (GUID) that is different than your organization name or domain. Follow these steps to obtain: open one of the Teams where you belong inside the Teams Application -> Click on the … next to the Team title -> Click on Get link to team -> Copy the link to the team and grab the tenant ID form the URL", + "airbyte_secret": true }, "client_id": { "title": "Client ID", @@ -64,7 +64,6 @@ "type": "object", "title": "Authenticate via Microsoft", "required": ["tenant_id", "client_id", "client_secret"], - "additionalProperties": false, "properties": { "auth_type": { "type": "string", @@ -76,7 +75,8 @@ "tenant_id": { "title": "Directory (tenant) ID", "type": "string", - "description": "A globally unique identifier (GUID) that is different than your organization name or domain. Follow these steps to obtain: open one of the Teams where you belong inside the Teams Application -> Click on the … next to the Team title -> Click on Get link to team -> Copy the link to the team and grab the tenant ID form the URL" + "description": "A globally unique identifier (GUID) that is different than your organization name or domain. Follow these steps to obtain: open one of the Teams where you belong inside the Teams Application -> Click on the … next to the Team title -> Click on Get link to team -> Copy the link to the team and grab the tenant ID form the URL", + "airbyte_secret": true }, "client_id": { "title": "Client ID", diff --git a/airbyte-integrations/connectors/source-mixpanel/Dockerfile b/airbyte-integrations/connectors/source-mixpanel/Dockerfile deleted file mode 100644 index 4c1d31e45794..000000000000 --- a/airbyte-integrations/connectors/source-mixpanel/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY setup.py ./ -RUN pip install . -COPY source_mixpanel ./source_mixpanel -COPY main.py ./ - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - - -LABEL io.airbyte.version=0.1.37 -LABEL io.airbyte.name=airbyte/source-mixpanel diff --git a/airbyte-integrations/connectors/source-mixpanel/README.md b/airbyte-integrations/connectors/source-mixpanel/README.md index 3b5924076c0e..0867d06cb7c0 100644 --- a/airbyte-integrations/connectors/source-mixpanel/README.md +++ b/airbyte-integrations/connectors/source-mixpanel/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-mixpanel:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/mixpanel) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_mixpanel/spec.json` file. @@ -56,19 +48,70 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-mixpanel:dev + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-mixpanel build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-mixpanel:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. -You can also build the connector image via Gradle: +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-mixpanel:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-mixpanel:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-mixpanel:dev . +# Running the spec command against your patched connector +docker run airbyte/source-mixpanel:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -77,44 +120,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mixpanel:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-mixpanel:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-mixpanel:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .'[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-mixpanel test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-mixpanel:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-mixpanel:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -124,8 +139,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-mixpanel test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/mixpanel.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-mixpanel/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mixpanel/acceptance-test-config.yml index 141b8e19e4b0..1e733552628e 100644 --- a/airbyte-integrations/connectors/source-mixpanel/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-mixpanel/acceptance-test-config.yml @@ -1,6 +1,7 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-mixpanel:dev +# custom configuration is used for tests to speed up testing and avoid hitting rate limits custom_environment_variables: REQS_PER_HOUR_LIMIT: 0 AVAILABLE_TESTING_RANGE_DAYS: 10 @@ -9,58 +10,58 @@ test_strictness_level: "high" acceptance_tests: spec: tests: - - spec_path: "source_mixpanel/spec.json" + - spec_path: "source_mixpanel/spec.json" + backward_compatibility_tests_config: + # credentials became required field; project_id changed path + # migration is implemented in source_mixpanel/config_migrations.py + disable_for_version: "0.1.40" connection: tests: - - config_path: "secrets/config_old.json" - status: "succeed" - - config_path: "secrets/config_project_secret.json" - status: "succeed" - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + - config_path: "secrets/config_project_secret.json" + status: "succeed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: tests: - - config_path: "secrets/config_incremental.json" - timeout_seconds: 900 + - config_path: "secrets/config_incremental.json" + timeout_seconds: 900 basic_read: tests: - - config_path: "secrets/config.json" - timeout_seconds: 9000 - expect_records: - path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: no - extra_records: yes - empty_streams: - - name: export - bypass_reason: "Data expired too often" - - name: annotations - bypass_reason: "Data expired too often" - ignored_fields: - funnels: - - name: date - bypass_reason: "Data changes too often" - revenue: - - name: date - bypass_reason: "Data changes too often" + - config_path: "secrets/config.json" + timeout_seconds: 9000 + expect_records: + path: "integration_tests/expected_records.jsonl" + extra_fields: no + exact_order: no + extra_records: yes + empty_streams: + - name: export + bypass_reason: "Data expired too often" + - name: annotations + bypass_reason: "Data expired too often" + ignored_fields: + funnels: + - name: date + bypass_reason: "Data changes too often" + revenue: + - name: date + bypass_reason: "Data changes too often" full_refresh: tests: - - config_path: "secrets/config_old.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 9000 + - config_path: "secrets/config_project_secret.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 9000 incremental: tests: - - config_path: "secrets/config_incremental.json" - configured_catalog_path: "integration_tests/configured_catalog_incremental.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" - cursor_paths: - cohorts: ["created"] - export: ["time"] - funnels: ["41833532", "date"] - revenue: ["date"] - engage: [ "last_seen" ] - cohort_members: [ "last_seen" ] - timeout_seconds: 9000 + - config_path: "secrets/config_incremental.json" + # The `Engage` and `CohortMembers` streams are not part of incremental catalog as they are semi-incremental, + # so cursor filter is not inside request, but results are filtered based on the cursor value. + # Also, these streams can produce records without cursor field, so abnormal state test would fail. + configured_catalog_path: "integration_tests/configured_catalog_incremental.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + timeout_seconds: 9000 + # skip incremental tests as cursor granularity is day, so records for stream state day are duplicated + skip_comprehensive_incremental_tests: true diff --git a/airbyte-integrations/connectors/source-mixpanel/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-mixpanel/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-mixpanel/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-mixpanel/build.gradle b/airbyte-integrations/connectors/source-mixpanel/build.gradle deleted file mode 100644 index c94ff8d6d8bc..000000000000 --- a/airbyte-integrations/connectors/source-mixpanel/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_mixpanel' -} diff --git a/airbyte-integrations/connectors/source-mixpanel/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-mixpanel/integration_tests/abnormal_state.json index 89a95990ac33..828816502f30 100644 --- a/airbyte-integrations/connectors/source-mixpanel/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-mixpanel/integration_tests/abnormal_state.json @@ -2,7 +2,10 @@ { "type": "STREAM", "stream": { - "stream_state": { "41833532": { "date": "2030-01-01" } }, + "stream_state": { + "41833532": { "date": "2030-01-01" }, + "36152117": { "date": "2030-01-01" } + }, "stream_descriptor": { "name": "funnels" } } }, diff --git a/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_incremental.json b/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_incremental.json index d3ae77abd5a6..6606fd41203e 100644 --- a/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_incremental.json +++ b/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog_incremental.json @@ -10,7 +10,9 @@ "source_defined_primary_key": [["funnel_id"], ["date"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["date"], + "primary_key": [["funnel_id"], ["date"]] }, { "stream": { @@ -21,7 +23,8 @@ "default_cursor_field": ["time"] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["time"] }, { "stream": { @@ -33,20 +36,23 @@ "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "append", + "cursor_field": ["created"], + "primary_key": [["id"]] }, { "stream": { - "name": "cohort_members", + "name": "revenue", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "default_cursor_field": ["last_seen"], - "source_defined_primary_key": [["distinct_id"]] + "source_defined_cursor": true, + "default_cursor_field": ["date"], + "source_defined_primary_key": [["date"]] }, "sync_mode": "incremental", "destination_sync_mode": "append", - "cursor_field": ["last_seen"] + "cursor_field": ["date"], + "primary_key": [["date"]] } ] } diff --git a/airbyte-integrations/connectors/source-mixpanel/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-mixpanel/integration_tests/expected_records.jsonl index 02e36881e9c1..1dc048c3757e 100644 --- a/airbyte-integrations/connectors/source-mixpanel/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-mixpanel/integration_tests/expected_records.jsonl @@ -1,12 +1,12 @@ -{"stream": "funnels", "data": {"funnel_id": 36152117, "name": "test", "date": "2023-06-25", "steps": [{"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "Purchase", "goal": "Purchase", "step_label": "Purchase", "overall_conv_ratio": 1, "step_conv_ratio": 1}, {"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "$custom_event:1305068", "goal": "$custom_event:1305068", "step_label": "111", "custom_event": true, "custom_event_id": 1305068, "overall_conv_ratio": 0, "step_conv_ratio": 0}], "analysis": {"completion": 0, "starting_amount": 0, "steps": 2, "worst": 1}}, "emitted_at": 1687889775303} -{"stream": "funnels", "data": {"funnel_id": 36152117, "name": "test", "date": "2023-06-26", "steps": [{"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "Purchase", "goal": "Purchase", "step_label": "Purchase", "overall_conv_ratio": 1, "step_conv_ratio": 1}, {"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "$custom_event:1305068", "goal": "$custom_event:1305068", "step_label": "111", "custom_event": true, "custom_event_id": 1305068, "overall_conv_ratio": 0, "step_conv_ratio": 0}], "analysis": {"completion": 0, "starting_amount": 0, "steps": 2, "worst": 1}}, "emitted_at": 1687889775303} -{"stream": "funnels", "data": {"funnel_id": 36152117, "name": "test", "date": "2023-06-27", "steps": [{"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "Purchase", "goal": "Purchase", "step_label": "Purchase", "overall_conv_ratio": 1, "step_conv_ratio": 1}, {"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "$custom_event:1305068", "goal": "$custom_event:1305068", "step_label": "111", "custom_event": true, "custom_event_id": 1305068, "overall_conv_ratio": 0, "step_conv_ratio": 0}], "analysis": {"completion": 0, "starting_amount": 0, "steps": 2, "worst": 1}}, "emitted_at": 1687889775303} -{"stream": "engage", "data": {"distinct_id": "integration-test@airbyte.io", "name": "Integration Test1", "test": "test", "email": "integration-test@airbyte.io", "last_seen": "2023-01-01T00:00:00"}, "emitted_at": 1687889778985} -{"stream": "engage", "data": {"distinct_id": "integration-test.db4415.mp-service-account", "name": "test", "test": "test", "last_seen": "2023-01-01T00:00:00"}, "emitted_at": 1687889778988} -{"stream": "engage", "data": {"distinct_id": "123@gmail.com", "email": "123@gmail.com", "name": "123", "123": "123456", "last_seen": "2023-01-01T00:00:00"}, "emitted_at": 1687889778988} -{"stream": "cohorts", "data": {"id": 1478097, "project_id": 2529987, "name": "Cohort1", "description": "", "data_group_id": null, "count": 2, "is_visible": 1, "created": "2021-09-14 15:57:43"}, "emitted_at": 1687889787689} -{"stream": "cohort_members", "data": {"distinct_id": "integration-test.db4415.mp-service-account", "name": "test", "test": "test", "last_seen": "2023-01-01T00:00:00", "cohort_id": 1478097}, "emitted_at": 1687889914154} -{"stream": "cohort_members", "data": {"distinct_id": "integration-test@airbyte.io", "name": "Integration Test1", "test": "test", "email": "integration-test@airbyte.io", "last_seen": "2023-01-01T00:00:00", "cohort_id": 1478097}, "emitted_at": 1687889914156} -{"stream": "revenue", "data": {"date": "2023-06-25", "amount": 0.0, "count": 3, "paid_count": 0}, "emitted_at": 1687889918052} -{"stream": "revenue", "data": {"date": "2023-06-26", "amount": 0.0, "count": 3, "paid_count": 0}, "emitted_at": 1687889918052} -{"stream": "revenue", "data": {"date": "2023-06-27", "amount": 0.0, "count": 3, "paid_count": 0}, "emitted_at": 1687889918052} +{"stream": "cohorts", "data": {"id": 1478097, "project_id": 2529987, "name": "Cohort1", "description": "", "data_group_id": null, "count": 2, "is_visible": 1, "created": "2021-09-14T15:57:43"}, "emitted_at": 1695644145072} +{"stream": "engage", "data": {"distinct_id": "123@gmail.com", "email": "123@gmail.com", "name": "123", "123": "123456", "last_seen": "2023-01-01T00:00:00", "how are you": "just fine"}, "emitted_at": 1695642956746} +{"stream": "engage", "data": {"distinct_id": "integration-test@airbyte.io", "name": "Integration Test1", "test": "test", "email": "integration-test@airbyte.io", "last_seen": "2023-01-01T00:00:00"}, "emitted_at": 1695642956748} +{"stream": "engage", "data": {"distinct_id": "integration-test.db4415.mp-service-account", "name": "test", "test": "test", "last_seen": "2023-01-01T00:00:00"}, "emitted_at": 1695642956749} +{"stream": "funnels", "data": {"funnel_id": 36152117, "name": "test", "date": "2023-01-01", "steps": [{"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "Purchase", "goal": "Purchase", "step_label": "Purchase", "overall_conv_ratio": 1, "step_conv_ratio": 1}, {"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "$custom_event:1305068", "goal": "$custom_event:1305068", "step_label": "111", "custom_event": true, "custom_event_id": 1305068, "overall_conv_ratio": 0, "step_conv_ratio": 0}], "analysis": {"completion": 0, "starting_amount": 0, "steps": 2, "worst": 1}}, "emitted_at": 1695642317451} +{"stream": "funnels", "data": {"funnel_id": 36152117, "name": "test", "date": "2023-01-02", "steps": [{"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "Purchase", "goal": "Purchase", "step_label": "Purchase", "overall_conv_ratio": 1, "step_conv_ratio": 1}, {"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "$custom_event:1305068", "goal": "$custom_event:1305068", "step_label": "111", "custom_event": true, "custom_event_id": 1305068, "overall_conv_ratio": 0, "step_conv_ratio": 0}], "analysis": {"completion": 0, "starting_amount": 0, "steps": 2, "worst": 1}}, "emitted_at": 1695642317453} +{"stream": "funnels", "data": {"funnel_id": 36152117, "name": "test", "date": "2023-01-03", "steps": [{"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "Purchase", "goal": "Purchase", "step_label": "Purchase", "overall_conv_ratio": 1, "step_conv_ratio": 1}, {"count": 0, "avg_time": null, "avg_time_from_start": null, "event": "$custom_event:1305068", "goal": "$custom_event:1305068", "step_label": "111", "custom_event": true, "custom_event_id": 1305068, "overall_conv_ratio": 0, "step_conv_ratio": 0}], "analysis": {"completion": 0, "starting_amount": 0, "steps": 2, "worst": 1}}, "emitted_at": 1695642317453} +{"stream": "revenue", "data": {"date": "2023-01-01", "amount": 0.0, "count": 3, "paid_count": 0}, "emitted_at": 1695644343316} +{"stream": "revenue", "data": {"date": "2023-01-02", "amount": 0.0, "count": 3, "paid_count": 0}, "emitted_at": 1695644343317} +{"stream": "revenue", "data": {"date": "2023-01-03", "amount": 0.0, "count": 3, "paid_count": 0}, "emitted_at": 1695644343317} +{"stream": "cohort_members", "data": {"distinct_id": "integration-test@airbyte.io", "name": "Integration Test1", "test": "test", "email": "integration-test@airbyte.io", "last_seen": "2023-01-01T00:00:00", "cohort_id": 1478097}, "emitted_at": 1695644214153} +{"stream": "cohort_members", "data": {"distinct_id": "integration-test.db4415.mp-service-account", "name": "test", "test": "test", "last_seen": "2023-01-01T00:00:00", "cohort_id": 1478097}, "emitted_at": 1695644214154} diff --git a/airbyte-integrations/connectors/source-mixpanel/main.py b/airbyte-integrations/connectors/source-mixpanel/main.py index 38819642f937..df8cb33fc826 100644 --- a/airbyte-integrations/connectors/source-mixpanel/main.py +++ b/airbyte-integrations/connectors/source-mixpanel/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_mixpanel import SourceMixpanel +from source_mixpanel.run import run if __name__ == "__main__": - source = SourceMixpanel() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-mixpanel/metadata.yaml b/airbyte-integrations/connectors/source-mixpanel/metadata.yaml index 4a49e272acc5..e40a80cfc905 100644 --- a/airbyte-integrations/connectors/source-mixpanel/metadata.yaml +++ b/airbyte-integrations/connectors/source-mixpanel/metadata.yaml @@ -1,13 +1,19 @@ data: + ab_internal: + ql: 400 + sl: 200 allowedHosts: hosts: - mixpanel.com - eu.mixpanel.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: api connectorType: source definitionId: 12928b32-bf0a-4f1e-964f-07e12e37153a - dockerImageTag: 0.1.37 + dockerImageTag: 2.0.1 dockerRepository: airbyte/source-mixpanel + documentationUrl: https://docs.airbyte.com/integrations/sources/mixpanel githubIssueLabel: source-mixpanel icon: mixpanel.svg license: MIT @@ -18,11 +24,33 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/mixpanel + releases: + breakingChanges: + 2.0.0: + message: + In this release, the default primary key for stream Export has been deleted, + allowing users to select the key that best fits their data. + Refreshing the source schema and resetting affected streams is necessary + only if new primary keys are to be applied following the upgrade. + upgradeDeadline: "2023-11-30" + 1.0.0: + message: + In this release, the datetime field of stream engage has had its + type changed from date-time to string due to inconsistent data from Mixpanel. + Additionally, the primary key for stream export has been fixed to uniquely + identify records. Users will need to refresh the source schema and reset + affected streams after upgrading. + upgradeDeadline: "2023-10-31" + suggestedStreams: + streams: + - export + - cohorts + - cohort_members + - engage + - annotations + - revenue + - funnels + supportLevel: certified tags: - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-mixpanel/setup.py b/airbyte-integrations/connectors/source-mixpanel/setup.py index 574a1130d6f9..b89f8d01fbd7 100644 --- a/airbyte-integrations/connectors/source-mixpanel/setup.py +++ b/airbyte-integrations/connectors/source-mixpanel/setup.py @@ -6,12 +6,17 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", + "airbyte-cdk", ] TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6", "requests_mock~=1.8"] setup( + entry_points={ + "console_scripts": [ + "source-mixpanel=source_mixpanel.run:run", + ], + }, name="source_mixpanel", description="Source implementation for Mixpanel.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/config_migrations.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/config_migrations.py new file mode 100644 index 000000000000..628cd46dcbda --- /dev/null +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/config_migrations.py @@ -0,0 +1,87 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import logging +from typing import Any, List, Mapping + +from airbyte_cdk.config_observation import create_connector_config_control_message +from airbyte_cdk.entrypoint import AirbyteEntrypoint +from airbyte_cdk.sources import Source +from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository + +logger = logging.getLogger("airbyte_logger") + + +class MigrateProjectId: + """ + This class stands for migrating the config at runtime. + Specifically, starting from `0.1.41`, "credentials" block is required and username and secret or api_secret should be inside it; + the property `project_id` should be inside credentials block as it is only used for Service Account. + """ + + message_repository: MessageRepository = InMemoryMessageRepository() + + @classmethod + def should_migrate(cls, config: Mapping[str, Any]) -> bool: + """ + This method determines if the config should be migrated. + Returns: + > True, if the transformation is necessary + > False, otherwise. + """ + is_project = "project_id" in config + is_api_secret = "api_secret" in config + return is_project or is_api_secret + + @staticmethod + def transform_config(config: Mapping[str, Any]) -> Mapping[str, Any]: + # add credentials dict if doesnt exist + if not isinstance(config.get("credentials", 0), dict): + config["credentials"] = dict() + + # move api_secret inside credentials block + if "api_secret" in config: + config["credentials"]["api_secret"] = config["api_secret"] + config.pop("api_secret") + + if "project_id" in config: + config["credentials"]["project_id"] = config["project_id"] + config.pop("project_id") + + return config + + @classmethod + def modify_and_save(cls, config_path: str, source: Source, config: Mapping[str, Any]) -> Mapping[str, Any]: + # modify the config + migrated_config = cls.transform_config(config) + # save the config + source.write_config(migrated_config, config_path) + # return modified config + return migrated_config + + @classmethod + def emit_control_message(cls, migrated_config: Mapping[str, Any]) -> None: + # add the Airbyte Control Message to message repo + cls.message_repository.emit_message(create_connector_config_control_message(migrated_config)) + # emit the Airbyte Control Message from message queue to stdout + for message in cls.message_repository._message_queue: + print(message.json(exclude_unset=True)) + + @classmethod + def migrate(cls, args: List[str], source: Source) -> None: + """ + This method checks the input args, should the config be migrated, + transform if neccessary and emit the CONTROL message. + """ + # get config path + config_path = AirbyteEntrypoint(source).extract_config(args) + # proceed only if `--config` arg is provided + if config_path: + # read the existing config + config = source.read_config(config_path) + # migration check + if cls.should_migrate(config): + cls.emit_control_message( + cls.modify_and_save(config_path, source, config), + ) diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/run.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/run.py new file mode 100644 index 000000000000..1d512c472c84 --- /dev/null +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/run.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_mixpanel import SourceMixpanel +from source_mixpanel.config_migrations import MigrateProjectId + + +def run(): + source = SourceMixpanel() + MigrateProjectId.migrate(sys.argv[1:], source) + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py index e6f1dc6edf58..f90a0699bdd8 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py @@ -6,20 +6,29 @@ import json import logging import os -from typing import Any, List, Mapping, Tuple +from typing import Any, List, Mapping, MutableMapping, Optional, Tuple import pendulum import requests from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.models import FailureType from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.auth import BasicHttpAuthenticator, TokenAuthenticator +from airbyte_cdk.utils import AirbyteTracedException from .streams import Annotations, CohortMembers, Cohorts, Engage, Export, Funnels, Revenue from .testing import adapt_streams_if_testing, adapt_validate_if_testing from .utils import read_full_refresh +def raise_config_error(message: str, original_error: Optional[Exception] = None): + config_error = AirbyteTracedException(message=message, internal_message=message, failure_type=FailureType.config_error) + if original_error: + raise config_error from original_error + raise config_error + + class TokenAuthenticatorBase64(TokenAuthenticator): def __init__(self, token: str): token = base64.b64encode(token.encode("utf8")).decode("utf8") @@ -29,45 +38,67 @@ def __init__(self, token: str): class SourceMixpanel(AbstractSource): STREAMS = [Cohorts, CohortMembers, Funnels, Revenue, Export, Annotations, Engage] - def get_authenticator(self, config: Mapping[str, Any]) -> TokenAuthenticator: - credentials = config.get("credentials") - if credentials: - username = credentials.get("username") - secret = credentials.get("secret") - if username and secret: - return BasicHttpAuthenticator(username=username, password=secret) - return TokenAuthenticatorBase64(token=credentials["api_secret"]) - return TokenAuthenticatorBase64(token=config["api_secret"]) + @staticmethod + def get_authenticator(config: Mapping[str, Any]) -> TokenAuthenticator: + credentials = config["credentials"] + username = credentials.get("username") + secret = credentials.get("secret") + if username and secret: + return BasicHttpAuthenticator(username=username, password=secret) + return TokenAuthenticatorBase64(token=credentials["api_secret"]) + + @staticmethod + def validate_date(name: str, date_str: str, default: pendulum.date) -> pendulum.date: + if not date_str: + return default + try: + return pendulum.parse(date_str).date() + except pendulum.parsing.exceptions.ParserError as e: + raise_config_error(f"Could not parse {name}: {date_str}. Please enter a valid {name}.", e) @adapt_validate_if_testing - def _validate_and_transform(self, config: Mapping[str, Any]): - logger = logging.getLogger("airbyte") - source_spec = self.spec(logger) - default_project_timezone = source_spec.connectionSpecification["properties"]["project_timezone"]["default"] - config["project_timezone"] = pendulum.timezone(config.get("project_timezone", default_project_timezone)) - - today = pendulum.today(tz=config["project_timezone"]).date() - start_date = config.get("start_date") - if start_date: - config["start_date"] = pendulum.parse(start_date).date() - else: - config["start_date"] = today.subtract(days=365) - - end_date = config.get("end_date") - if end_date: - config["end_date"] = pendulum.parse(end_date).date() - else: - config["end_date"] = today - - for k in ["attribution_window", "select_properties_by_default", "region", "date_window_size"]: - if k not in config: - config[k] = source_spec.connectionSpecification["properties"][k]["default"] + def _validate_and_transform(self, config: MutableMapping[str, Any]): + project_timezone, start_date, end_date, attribution_window, select_properties_by_default, region, date_window_size, project_id = ( + config.get("project_timezone", "US/Pacific"), + config.get("start_date"), + config.get("end_date"), + config.get("attribution_window", 5), + config.get("select_properties_by_default", True), + config.get("region", "US"), + config.get("date_window_size", 30), + config.get("credentials", dict()).get("project_id"), + ) + try: + project_timezone = pendulum.timezone(project_timezone) + except pendulum.tz.zoneinfo.exceptions.InvalidTimezone as e: + raise_config_error(f"Could not parse time zone: {project_timezone}, please enter a valid timezone.", e) + + if region not in ("US", "EU"): + raise_config_error("Region must be either EU or US.") + + if select_properties_by_default not in (True, False, "", None): + raise_config_error("Please provide a valid True/False value for the `Select properties by default` parameter.") + + if not isinstance(attribution_window, int) or attribution_window < 0: + raise_config_error("Please provide a valid integer for the `Attribution window` parameter.") + if not isinstance(date_window_size, int) or date_window_size < 1: + raise_config_error("Please provide a valid integer for the `Date slicing window` parameter.") auth = self.get_authenticator(config) - if isinstance(auth, TokenAuthenticatorBase64) and "project_id" in config: - config.pop("project_id") - elif isinstance(auth, BasicHttpAuthenticator) and "project_id" not in config: - raise ValueError("missing required parameter 'project_id'") + if isinstance(auth, TokenAuthenticatorBase64) and project_id: + config.get("credentials").pop("project_id") + if isinstance(auth, BasicHttpAuthenticator) and not isinstance(project_id, int): + raise_config_error("Required parameter 'project_id' missing or malformed. Please provide a valid project ID.") + + today = pendulum.today(tz=project_timezone).date() + config["project_timezone"] = project_timezone + config["start_date"] = self.validate_date("start date", start_date, today.subtract(days=365)) + config["end_date"] = self.validate_date("end date", end_date, today) + config["attribution_window"] = attribution_window + config["select_properties_by_default"] = select_properties_by_default + config["region"] = region + config["date_window_size"] = date_window_size + config["project_id"] = project_id return config @@ -80,11 +111,8 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> :param logger: logger object :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ - try: - config = self._validate_and_transform(config) - auth = self.get_authenticator(config) - except Exception as e: - return False, e + config = self._validate_and_transform(config) + auth = self.get_authenticator(config) # https://github.com/airbytehq/airbyte/pull/27252#discussion_r1228356872 # temporary solution, testing access for all streams to avoid 402 error diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/spec.json b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/spec.json index 7e2ea3f591bf..4dbb4511fcbb 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/spec.json +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/spec.json @@ -3,6 +3,7 @@ "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Source Mixpanel Spec", + "required": ["credentials"], "type": "object", "properties": { "credentials": { @@ -14,7 +15,7 @@ { "type": "object", "title": "Service Account", - "required": ["username", "secret"], + "required": ["username", "secret", "project_id"], "properties": { "option_title": { "type": "string", @@ -33,6 +34,12 @@ "type": "string", "description": "Mixpanel Service Account Secret. See the docs for more information on how to obtain this.", "airbyte_secret": true + }, + "project_id": { + "order": 3, + "title": "Project ID", + "description": "Your project ID number. See the docs for more information on how to obtain this.", + "type": "integer" } } }, @@ -57,17 +64,11 @@ } ] }, - "project_id": { - "order": 1, - "title": "Project ID", - "description": "Your project ID number. See the docs for more information on how to obtain this.", - "type": "integer" - }, "attribution_window": { "order": 2, "title": "Attribution Window", "type": "integer", - "description": " A period of time for attributing results to ads and the lookback period after those actions occur during which ad results are counted. Default attribution window is 5 days.", + "description": "A period of time for attributing results to ads and the lookback period after those actions occur during which ad results are counted. Default attribution window is 5 days. (This value should be non-negative integer)", "default": 5 }, "project_timezone": { @@ -114,7 +115,7 @@ "date_window_size": { "order": 8, "title": "Date slicing window", - "description": "Defines window size in days, that used to slice through data. You can reduce it, if amount of data in each window is too big for your environment.", + "description": "Defines window size in days, that used to slice through data. You can reduce it, if amount of data in each window is too big for your environment. (This value should be positive integer)", "type": "integer", "minimum": 1, "default": 30 diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py index 3ea5d4cd41b8..472749f09862 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/base.py @@ -5,13 +5,16 @@ import time from abc import ABC from datetime import timedelta -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union import pendulum import requests +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator +from airbyte_cdk.utils import AirbyteTracedException from pendulum import Date +from source_mixpanel.utils import fix_date_time class MixpanelStream(HttpStream, ABC): @@ -23,6 +26,13 @@ class MixpanelStream(HttpStream, ABC): DEFAULT_REQS_PER_HOUR_LIMIT = 60 + @property + def state_checkpoint_interval(self) -> int: + # to meet the requirement of emitting state at least once per 15 minutes, + # we assume there's at least 1 record per request returned. Given that each request is followed by a 60 seconds sleep + # we'll have to emit state every 15 records + return 15 + @property def url_base(self): prefix = "eu." if self.region == "EU" else "" @@ -61,7 +71,6 @@ def __init__( self.project_id = project_id self.retries = 0 self._reqs_per_hour_limit = reqs_per_hour_limit - super().__init__(authenticator=authenticator) def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: @@ -83,6 +92,7 @@ def process_response(self, response: requests.Response, **kwargs) -> Iterable[Ma data = [json_response] for record in data: + fix_date_time(record) yield record def parse_response( @@ -91,7 +101,6 @@ def parse_response( stream_state: Mapping[str, Any], **kwargs, ) -> Iterable[Mapping]: - # parse the whole response yield from self.process_response(response, stream_state=stream_state, **kwargs) @@ -101,10 +110,14 @@ def parse_response( self.logger.info(f"Sleep for {3600 / self.reqs_per_hour_limit} seconds to match API limitations after reading from {self.name}") time.sleep(3600 / self.reqs_per_hour_limit) + @property + def max_retries(self) -> Union[int, None]: + # we want to limit the max sleeping time by 2^3 * 60 = 8 minutes + return 3 + def backoff_time(self, response: requests.Response) -> float: """ Some API endpoints do not return "Retry-After" header. - https://developer.mixpanel.com/reference/import-events#rate-limits (exponential backoff) """ retry_after = response.headers.get("Retry-After") @@ -119,6 +132,12 @@ def should_retry(self, response: requests.Response) -> bool: if response.status_code == 402: self.logger.warning(f"Unable to perform a request. Payment Required: {response.json()['error']}") return False + if response.status_code == 400 and "Unable to authenticate request" in response.text: + message = ( + f"Your credentials might have expired. Please update your config with valid credentials." + f" See more details: {response.text}" + ) + raise AirbyteTracedException(message=message, internal_message=message, failure_type=FailureType.config_error) return super().should_retry(response) def get_stream_params(self) -> Mapping[str, Any]: @@ -147,6 +166,29 @@ def request_params( class DateSlicesMixin: + raise_on_http_errors = True + + def should_retry(self, response: requests.Response) -> bool: + if response.status_code == requests.codes.bad_request: + if "to_date cannot be later than today" in response.text: + self._timezone_mismatch = True + self.logger.warning( + "Your project timezone must be misconfigured. Please set it to the one defined in your Mixpanel project settings. " + "Stopping current stream sync." + ) + setattr(self, "raise_on_http_errors", False) + return False + return super().should_retry(response) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._timezone_mismatch = False + + def parse_response(self, *args, **kwargs): + if self._timezone_mismatch: + return [] + yield from super().parse_response(*args, **kwargs) + def stream_slices( self, sync_mode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: @@ -168,6 +210,8 @@ def stream_slices( end_date = min(self.end_date, pendulum.today(tz=self.project_timezone).date()) while start_date <= end_date: + if self._timezone_mismatch: + return current_end_date = start_date + timedelta(days=self.date_window_size - 1) # -1 is needed because dates are inclusive stream_slice = { "start_date": str(start_date), diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/engage.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/engage.py index 46aa7593d949..9a52b847f09a 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/engage.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/streams/engage.py @@ -184,7 +184,8 @@ def get_json_schema(self) -> Mapping[str, Any]: types = { "boolean": {"type": ["null", "boolean"]}, "number": {"type": ["null", "number"], "multipleOf": 1e-20}, - "datetime": {"type": ["null", "string"], "format": "date-time"}, + # no format specified as values can be "2021-12-16T00:00:00", "1638298874", "15/08/53895" + "datetime": {"type": ["null", "string"]}, "object": {"type": ["null", "object"], "additionalProperties": True}, "list": {"type": ["null", "array"], "required": False, "items": {}}, "string": {"type": ["null", "string"]}, diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/utils.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/utils.py index cea753dbee29..cc1d1402647d 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/utils.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/utils.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import re from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams import Stream @@ -13,3 +14,50 @@ def read_full_refresh(stream_instance: Stream): records = stream_instance.read_records(stream_slice=_slice, sync_mode=SyncMode.full_refresh) for record in records: yield record + + +# Precompile the regex pattern. +ISO_FORMAT_PATTERN = re.compile(r"^(\d{4}-\d{2}-\d{2})[ t](\d{2}:\d{2}:\d{2})$") + + +def to_iso_format(s: str) -> str: + """ + Convert a date string to ISO format if it matches recognized patterns. + + Args: + - s (str): Input string to be converted. + + Returns: + - str: Converted string in ISO format or the original string if no recognized pattern is found. + """ + # Use the precompiled regex pattern to match the date format. + match = ISO_FORMAT_PATTERN.match(s) + if match: + return match.group(1) + "T" + match.group(2) + + return s + + +def fix_date_time(record): + """ + Recursively process a data structure to fix date and time formats. + + Args: + - record (dict or list): The input data structure, which can be a dictionary or a list. + + Returns: + - None: The function modifies the input data structure in place. + """ + # Define the list of fields that might contain date and time values. + date_time_fields = {"last_seen", "created", "last_authenticated"} + + if isinstance(record, dict): + for field, value in list(record.items()): # Convert to list to avoid runtime errors during iteration. + if field in date_time_fields and isinstance(value, str): + record[field] = to_iso_format(value) + elif isinstance(value, (dict, list)): + fix_date_time(value) + + elif isinstance(record, list): + for entry in record: + fix_date_time(entry) diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py index a1814f08f501..534683c7b2ab 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/conftest.py @@ -14,7 +14,7 @@ def start_date(): @pytest.fixture def config(start_date): return { - "api_secret": "unexisting-secret", + "credentials": {"api_secret": "unexisting-secret"}, "attribution_window": 5, "project_timezone": pendulum.timezone("UTC"), "select_properties_by_default": True, @@ -41,8 +41,4 @@ def patch_time(mocker): @pytest.fixture(autouse=True) def disable_cache(mocker): - mocker.patch( - "source_mixpanel.streams.cohorts.Cohorts.use_cache", - new_callable=mocker.PropertyMock, - return_value=False - ) + mocker.patch("source_mixpanel.streams.cohorts.Cohorts.use_cache", new_callable=mocker.PropertyMock, return_value=False) diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_migration.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_migration.py new file mode 100644 index 000000000000..a2132148baa2 --- /dev/null +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_migration.py @@ -0,0 +1,43 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import patch + +import pytest +from airbyte_cdk.entrypoint import AirbyteEntrypoint +from source_mixpanel.config_migrations import MigrateProjectId +from source_mixpanel.source import SourceMixpanel + +# Test data for parametrized test +test_data = [ + # Test when only api_secret is present + ({"api_secret": "secret_value1"}, {"credentials": {"api_secret": "secret_value1"}}), + # Test when only project_id is present + ({"project_id": "project_value1"}, {"credentials": {"project_id": "project_value1"}}), + # Test when both api_secret and project_id are present + ( + {"api_secret": "secret_value2", "project_id": "project_value2"}, + {"credentials": {"api_secret": "secret_value2", "project_id": "project_value2"}}, + ), + # Test when neither api_secret nor project_id are present + ({"other_key": "value"}, {"other_key": "value"}), +] + + +@pytest.mark.parametrize("test_config, expected", test_data) +@patch.object(AirbyteEntrypoint, "extract_config") +@patch.object(SourceMixpanel, "write_config") +@patch.object(SourceMixpanel, "read_config") +def test_transform_config(source_read_config_mock, source_write_config_mock, ab_entrypoint_extract_config_mock, test_config, expected): + source = SourceMixpanel() + + source_read_config_mock.return_value = test_config + ab_entrypoint_extract_config_mock.return_value = "/path/to/config.json" + + def check_migrated_value(new_config, path): + assert path == "/path/to/config.json" + assert new_config == expected + + source_write_config_mock.side_effect = check_migrated_value + MigrateProjectId.migrate(["--config", "/path/to/config.json"], source) diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py index 0cafc3ce2d62..226f7442b669 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_source.py @@ -6,7 +6,7 @@ import pytest from airbyte_cdk import AirbyteLogger -from airbyte_cdk.models import AirbyteConnectionStatus, Status +from airbyte_cdk.utils import AirbyteTracedException from source_mixpanel.source import SourceMixpanel, TokenAuthenticatorBase64 from source_mixpanel.streams import Annotations, CohortMembers, Cohorts, Engage, Export, Funnels, FunnelsList, Revenue @@ -17,17 +17,14 @@ @pytest.fixture def check_connection_url(config): - auth = TokenAuthenticatorBase64(token=config["api_secret"]) + auth = TokenAuthenticatorBase64(token=config["credentials"]["api_secret"]) annotations = Cohorts(authenticator=auth, **config) return get_url_to_mock(annotations) @pytest.mark.parametrize( "response_code,expect_success,response_json", - [ - (200, True, {}), - (400, False, {"error": "Request error"}) - ], + [(200, True, {}), (400, False, {"error": "Request error"})], ) def test_check_connection(requests_mock, check_connection_url, config_raw, response_code, expect_success, response_json): requests_mock.register_uri("GET", check_connection_url, setup_response(response_code, response_json)) @@ -39,24 +36,42 @@ def test_check_connection(requests_mock, check_connection_url, config_raw, respo def test_check_connection_all_streams_402_error(requests_mock, check_connection_url, config_raw, config): - auth = TokenAuthenticatorBase64(token=config["api_secret"]) - requests_mock.register_uri("GET", get_url_to_mock(Cohorts(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"})) - requests_mock.register_uri("GET", get_url_to_mock(Annotations(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"})) - requests_mock.register_uri("POST", get_url_to_mock(Engage(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"})) - requests_mock.register_uri("GET", get_url_to_mock(Export(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"})) - requests_mock.register_uri("GET", get_url_to_mock(Revenue(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"})) - requests_mock.register_uri("GET", get_url_to_mock(Funnels(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"})) - requests_mock.register_uri("GET", get_url_to_mock(FunnelsList(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"})) - requests_mock.register_uri("GET", get_url_to_mock(CohortMembers(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"})) + auth = TokenAuthenticatorBase64(token=config["credentials"]["api_secret"]) + requests_mock.register_uri( + "GET", get_url_to_mock(Cohorts(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"}) + ) + requests_mock.register_uri( + "GET", get_url_to_mock(Annotations(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"}) + ) + requests_mock.register_uri( + "POST", get_url_to_mock(Engage(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"}) + ) + requests_mock.register_uri( + "GET", get_url_to_mock(Export(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"}) + ) + requests_mock.register_uri( + "GET", get_url_to_mock(Revenue(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"}) + ) + requests_mock.register_uri( + "GET", get_url_to_mock(Funnels(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"}) + ) + requests_mock.register_uri( + "GET", get_url_to_mock(FunnelsList(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"}) + ) + requests_mock.register_uri( + "GET", get_url_to_mock(CohortMembers(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"}) + ) ok, error = SourceMixpanel().check_connection(logger, config_raw) assert ok is False and error == "Payment required" def test_check_connection_402_error_on_first_stream(requests_mock, check_connection_url, config, config_raw): - auth = TokenAuthenticatorBase64(token=config["api_secret"]) + auth = TokenAuthenticatorBase64(token=config["credentials"]["api_secret"]) requests_mock.register_uri("GET", get_url_to_mock(Cohorts(authenticator=auth, **config)), setup_response(200, {})) - requests_mock.register_uri("GET", get_url_to_mock(Annotations(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"})) + requests_mock.register_uri( + "GET", get_url_to_mock(Annotations(authenticator=auth, **config)), setup_response(402, {"error": "Payment required"}) + ) ok, error = SourceMixpanel().check_connection(logger, config_raw) # assert ok is True @@ -66,13 +81,15 @@ def test_check_connection_402_error_on_first_stream(requests_mock, check_connect def test_check_connection_bad_config(): config = {} source = SourceMixpanel() - assert command_check(source, config) == AirbyteConnectionStatus(status=Status.FAILED, message="KeyError('api_secret')") + with pytest.raises(AirbyteTracedException): + command_check(source, config) def test_check_connection_incomplete(config_raw): - config_raw.pop("api_secret") + config_raw.pop("credentials") source = SourceMixpanel() - assert command_check(source, config_raw) == AirbyteConnectionStatus(status=Status.FAILED, message="KeyError('api_secret')") + with pytest.raises(AirbyteTracedException): + command_check(source, config_raw) def test_streams(requests_mock, config_raw): @@ -84,12 +101,11 @@ def test_streams(requests_mock, config_raw): requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/cohorts/list", setup_response(200, {"id": 123})) requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/engage/revenue", setup_response(200, {})) requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/funnels", setup_response(200, {})) + requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/funnels/list", setup_response(200, {"funnel_id": 123, "name": "name"})) requests_mock.register_uri( - "GET", "https://mixpanel.com/api/2.0/funnels/list", setup_response(200, {"funnel_id": 123, "name": "name"}) - ) - requests_mock.register_uri( - "GET", "https://data.mixpanel.com/api/2.0/export", - setup_response(200, {"event": "some event", "properties": {"event": 124, "time": 124124}}) + "GET", + "https://data.mixpanel.com/api/2.0/export", + setup_response(200, {"event": "some event", "properties": {"event": 124, "time": 124124}}), ) streams = SourceMixpanel().streams(config_raw) @@ -105,8 +121,9 @@ def test_streams_string_date(requests_mock, config_raw): requests_mock.register_uri("POST", "https://mixpanel.com/api/2.0/engage", setup_response(200, {})) requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/funnels/list", setup_response(402, {"error": "Payment required"})) requests_mock.register_uri( - "GET", "https://data.mixpanel.com/api/2.0/export", - setup_response(200, {"event": "some event", "properties": {"event": 124, "time": 124124}}) + "GET", + "https://data.mixpanel.com/api/2.0/export", + setup_response(200, {"event": "some event", "properties": {"event": 124, "time": 124124}}), ) config = copy.deepcopy(config_raw) config["start_date"] = "2020-01-01" @@ -125,6 +142,78 @@ def test_streams_disabled_402(requests_mock, config_raw): requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/cohorts/list", setup_response(402, json_response)) requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/engage/revenue", setup_response(200, {})) requests_mock.register_uri("GET", "https://mixpanel.com/api/2.0/funnels/list", setup_response(402, json_response)) - requests_mock.register_uri("GET", "https://data.mixpanel.com/api/2.0/export?from_date=2017-01-20&to_date=2017-02-18", setup_response(402, json_response)) + requests_mock.register_uri( + "GET", "https://data.mixpanel.com/api/2.0/export?from_date=2017-01-20&to_date=2017-02-18", setup_response(402, json_response) + ) streams = SourceMixpanel().streams(config_raw) - assert {s.name for s in streams} == {'annotations', 'engage', 'revenue'} + assert {s.name for s in streams} == {"annotations", "engage", "revenue"} + + +@pytest.mark.parametrize( + "config, success, expected_error_message", + ( + ( + {"credentials": {"api_secret": "secret"}, "project_timezone": "Miami"}, + False, + "Could not parse time zone: Miami, please enter a valid timezone.", + ), + ( + {"credentials": {"api_secret": "secret"}, "start_date": "20 Jan 2021"}, + False, + "Could not parse start date: 20 Jan 2021. Please enter a valid start date.", + ), + ( + {"credentials": {"api_secret": "secret"}, "end_date": "20 Jan 2021"}, + False, + "Could not parse end date: 20 Jan 2021. Please enter a valid end date.", + ), + ( + {"credentials": {"api_secret": "secret"}, "attribution_window": "20 days"}, + False, + "Please provide a valid integer for the `Attribution window` parameter.", + ), + ( + {"credentials": {"api_secret": "secret"}, "select_properties_by_default": "Yes"}, + False, + "Please provide a valid True/False value for the `Select properties by default` parameter.", + ), + ({"credentials": {"api_secret": "secret"}, "region": "UK"}, False, "Region must be either EU or US."), + ( + {"credentials": {"api_secret": "secret"}, "date_window_size": "month"}, + False, + "Please provide a valid integer for the `Date slicing window` parameter.", + ), + ( + {"credentials": {"username": "user", "secret": "secret"}}, + False, + "Required parameter 'project_id' missing or malformed. Please provide a valid project ID.", + ), + ({"credentials": {"api_secret": "secret"}}, True, None), + ( + { + "credentials": {"username": "user", "secret": "secret", "project_id": 2397709}, + "project_timezone": "US/Pacific", + "start_date": "2021-02-01T00:00:00Z", + "end_date": "2023-02-01T00:00:00Z", + "attribution_window": 10, + "select_properties_by_default": True, + "region": "EU", + "date_window_size": 10, + }, + True, + None, + ), + ), +) +def test_config_validation(config, success, expected_error_message, requests_mock): + requests_mock.get("https://mixpanel.com/api/2.0/cohorts/list", status_code=200, json={}) + requests_mock.get("https://eu.mixpanel.com/api/2.0/cohorts/list", status_code=200, json={}) + try: + is_success, message = SourceMixpanel().check_connection(None, config) + except AirbyteTracedException as e: + is_success = False + message = e.message + + assert is_success is success + if not is_success: + assert message == expected_error_message diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py index d519bc8d124b..de54d1c6b89f 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_streams.py @@ -10,6 +10,7 @@ import pytest from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import SyncMode +from airbyte_cdk.utils import AirbyteTracedException from source_mixpanel.streams import ( Annotations, CohortMembers, @@ -278,23 +279,47 @@ def engage_schema_response(): 200, { "results": { - "$browser": {"count": 124, "type": "string"}, - "$browser_version": {"count": 124, "type": "string"}, "$created": {"count": 124, "type": "string"}, + "$is_active": {"count": 412, "type": "boolean"}, + "$CreatedDateTimestamp": {"count": 300, "type": "number"}, + "$CreatedDate": {"count": 11, "type": "datetime"}, + "$properties": {"count": 2, "type": "object"}, + "$tags": {"count": 131, "type": "list"}, } }, ) def test_engage_schema(requests_mock, engage_schema_response, config): - - stream = EngageSchema(authenticator=MagicMock(), **config) - requests_mock.register_uri("GET", get_url_to_mock(stream), engage_schema_response) - - records = stream.read_records(sync_mode=SyncMode.full_refresh) - - records_length = sum(1 for _ in records) - assert records_length == 3 + stream = Engage(authenticator=MagicMock(), **config) + requests_mock.register_uri("GET", get_url_to_mock(EngageSchema(authenticator=MagicMock(), **config)), engage_schema_response) + assert stream.get_json_schema() == { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": True, + "properties": { + "CreatedDate": {"type": ["null", "string"]}, + "CreatedDateTimestamp": {"multipleOf": 1e-20, "type": ["null", "number"]}, + "browser": {"type": ["null", "string"]}, + "browser_version": {"type": ["null", "string"]}, + "city": {"type": ["null", "string"]}, + "country_code": {"type": ["null", "string"]}, + "created": {"type": ["null", "string"]}, + "distinct_id": {"type": ["null", "string"]}, + "email": {"type": ["null", "string"]}, + "first_name": {"type": ["null", "string"]}, + "id": {"type": ["null", "string"]}, + "is_active": {"type": ["null", "boolean"]}, + "last_name": {"type": ["null", "string"]}, + "last_seen": {"format": "date-time", "type": ["null", "string"]}, + "name": {"type": ["null", "string"]}, + "properties": {"additionalProperties": True, "type": ["null", "object"]}, + "region": {"type": ["null", "string"]}, + "tags": {"items": {}, "required": False, "type": ["null", "array"]}, + "timezone": {"type": ["null", "string"]}, + "unblocked": {"type": ["null", "string"]}, + }, + "type": "object", + } def test_update_engage_schema(requests_mock, config): @@ -436,6 +461,19 @@ def test_export_stream(requests_mock, export_response, config): assert records_length == 1 +def test_handle_time_zone_mismatch(requests_mock, config, caplog): + stream = Export(authenticator=MagicMock(), **config) + requests_mock.register_uri("GET", get_url_to_mock(stream), status_code=400, text="to_date cannot be later than today") + records = [] + for slice_ in stream.stream_slices(sync_mode=SyncMode.full_refresh): + records.extend(stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice_)) + assert list(records) == [] + assert ( + "Your project timezone must be misconfigured. Please set it to the one defined in your Mixpanel project settings. " + "Stopping current stream sync." in caplog.text + ) + + def test_export_stream_request_params(config): stream = Export(authenticator=MagicMock(), **config) stream_slice = {"start_date": "2017-01-25T00:00:00Z", "end_date": "2017-02-25T00:00:00Z"} @@ -482,3 +520,22 @@ def test_should_retry_payment_required(http_status_code, should_retry, log_messa stream = stream_class(authenticator=MagicMock(), **config) assert stream.should_retry(response_mock) == should_retry assert log_message in caplog.text + + +def test_raise_config_error_on_creds_expiration(config, caplog, requests_mock): + streams = [] + for cls in [Annotations, CohortMembers, Cohorts, Engage, EngageSchema, Export, ExportSchema, Funnels, FunnelsList, Revenue]: + stream = cls(authenticator=MagicMock(), **config) + requests_mock.register_uri(stream.http_method, get_url_to_mock(stream), status_code=400, text="Unable to authenticate request") + streams.append(stream) + + for stream in streams: + records = [] + with pytest.raises(AirbyteTracedException) as e: + for slice_ in stream.stream_slices(sync_mode="full_refresh"): + records.extend(stream.read_records("full_refresh", stream_slice=slice_)) + assert records == [] + assert ( + str(e.value) == "Your credentials might have expired. Please update your config with valid credentials. " + "See more details: Unable to authenticate request" + ) diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_utils.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_utils.py new file mode 100644 index 000000000000..bda29ca92b90 --- /dev/null +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/test_utils.py @@ -0,0 +1,46 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import pytest +from source_mixpanel.utils import fix_date_time + + +@pytest.mark.parametrize( + "input_record, expected_record", + [ + # Test with a dictionary containing recognized date formats. + ( + {"last_seen": "2022-09-27 12:34:56", "created": "2022-09-27T12:34:56"}, + {"last_seen": "2022-09-27T12:34:56", "created": "2022-09-27T12:34:56"}, + ), + # Test with a dictionary containing unrecognized date formats. + ( + {"last_seen": "09/27/2022 12:34:56", "created": "September 27, 2022"}, + {"last_seen": "09/27/2022 12:34:56", "created": "September 27, 2022"}, + ), + # Test with nested dictionaries. + ( + {"user": {"last_seen": "2022-09-27 12:34:56", "created": "2022-09-27T12:34:56"}}, + {"user": {"last_seen": "2022-09-27T12:34:56", "created": "2022-09-27T12:34:56"}}, + ), + # Test with a list of dictionaries. + ( + [{"last_seen": "2022-09-27 12:34:56"}, {"created": "2022-09-27T12:34:56"}], + [{"last_seen": "2022-09-27T12:34:56"}, {"created": "2022-09-27T12:34:56"}], + ), + # Test with mixed data structures. + ( + {"users": [{"last_seen": "2022-09-27 12:34:56"}, {"created": "2022-09-27T12:34:56"}]}, + {"users": [{"last_seen": "2022-09-27T12:34:56"}, {"created": "2022-09-27T12:34:56"}]}, + ), + # Test with a dictionary containing ISO strings with offsets. + ( + {"last_seen": "2022-09-27 12:34:56+05:00", "created": "2022-09-27T12:34:56-03:00"}, + {"last_seen": "2022-09-27 12:34:56+05:00", "created": "2022-09-27T12:34:56-03:00"}, + ), + ], +) +def test_fix_date_time(input_record, expected_record): + fix_date_time(input_record) + assert input_record == expected_record diff --git a/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py index 65ffdad8b74b..2a46806b2197 100644 --- a/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-mixpanel/unit_tests/unit_test.py @@ -79,7 +79,9 @@ def test_date_slices(): region="EU", project_timezone="US/Pacific", ).stream_slices(sync_mode="any") - assert [{"start_date": "2021-07-01", "end_date": "2021-07-01"}, {"start_date": "2021-07-02", "end_date": "2021-07-02"}] == list(stream_slices) + assert [{"start_date": "2021-07-01", "end_date": "2021-07-01"}, {"start_date": "2021-07-02", "end_date": "2021-07-02"}] == list( + stream_slices + ) stream_slices = Annotations( authenticator=NoAuth(), @@ -103,7 +105,9 @@ def test_date_slices(): region="US", project_timezone="US/Pacific", ).stream_slices(sync_mode="any") - assert [{"start_date": "2021-07-01", "end_date": "2021-07-02"}, {"start_date": "2021-07-03", "end_date": "2021-07-03"}] == list(stream_slices) + assert [{"start_date": "2021-07-01", "end_date": "2021-07-02"}, {"start_date": "2021-07-03", "end_date": "2021-07-03"}] == list( + stream_slices + ) # test with stream_state stream_slices = Export( @@ -116,5 +120,5 @@ def test_date_slices(): ).stream_slices(sync_mode="any", stream_state={"time": "2021-07-02T00:00:00Z"}) assert [ {"start_date": "2021-07-02", "end_date": "2021-07-02", "time": "2021-07-02T00:00:00Z"}, - {"start_date": "2021-07-03", "end_date": "2021-07-03", "time": "2021-07-02T00:00:00Z"} + {"start_date": "2021-07-03", "end_date": "2021-07-03", "time": "2021-07-02T00:00:00Z"}, ] == list(stream_slices) diff --git a/airbyte-integrations/connectors/source-monday/Dockerfile b/airbyte-integrations/connectors/source-monday/Dockerfile deleted file mode 100644 index 592a1a6d20c7..000000000000 --- a/airbyte-integrations/connectors/source-monday/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_monday ./source_monday - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=1.1.2 -LABEL io.airbyte.name=airbyte/source-monday diff --git a/airbyte-integrations/connectors/source-monday/README.md b/airbyte-integrations/connectors/source-monday/README.md index 9fb9c7cc4e57..b0e9c8b4ffc4 100644 --- a/airbyte-integrations/connectors/source-monday/README.md +++ b/airbyte-integrations/connectors/source-monday/README.md @@ -6,23 +6,28 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development ### Prerequisites + **To iterate on this connector, make sure to complete this prerequisites section.** #### Minimum Python version required `= 3.7.0` #### Build & Activate Virtual Environment and install dependencies + From this connector directory, create a virtual environment: -``` + +```bash python -m venv .venv ``` This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your development environment of choice. To activate it from the terminal, run: -``` + +```bash source .venv/bin/activate pip install -r requirements.txt pip install '.[tests]' ``` + If you are in an IDE, follow your IDE's instructions to activate the virtualenv. Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is @@ -30,15 +35,8 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-monday:build -``` - #### Create credentials + **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/monday) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_monday/spec.json` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. @@ -48,7 +46,8 @@ See `integration_tests/sample_config.json` for a sample config file. and place them into `secrets/config.json`. ### Locally running the connector -``` + +```bash python main.py spec python main.py check --config secrets/config.json python main.py discover --config secrets/config.json @@ -57,76 +56,117 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: +#### Use `airbyte-ci` to build your connector + +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-monday build ``` -docker build . -t airbyte/source-monday:dev + +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-monday:dev`. + +##### Customizing our build process + +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: + +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -You can also build the connector image via Gradle: +#### Build your own connector image + +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. + +```Dockerfile +FROM airbyte/source-monday:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -./gradlew :airbyte-integrations:connectors:source-monday:airbyteDocker + +Please use this as an example. This is not optimized. + +2. Build your image: + +```bash +docker build -t airbyte/source-monday:dev . +# Running the spec command against your patched connector +docker run airbyte/source-monday:dev spec ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run + Then run any of the connector commands as follows: -``` + +```bash docker run --rm airbyte/source-monday:dev spec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-monday:dev check --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-monday:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-monday:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. -If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-monday:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-monday:integrationTest +```bash +airbyte-ci connectors --name=source-monday test ``` +### Customizing acceptance Tests + +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + ## Dependency Management + All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: + * required for your connector to work need to go to `MAIN_REQUIREMENTS` list. * required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector + You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-monday test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/monday.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-monday/acceptance-test-config.yml b/airbyte-integrations/connectors/source-monday/acceptance-test-config.yml index 16ea6d61a9d4..23f80f6c22ce 100644 --- a/airbyte-integrations/connectors/source-monday/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-monday/acceptance-test-config.yml @@ -22,13 +22,13 @@ acceptance_tests: # `boards`, `items`, `updates` streams schemas were modified. PR: https://github.com/airbytehq/airbyte/pull/27410 # Changes applies to all configs backward_compatibility_tests_config: - disable_for_version: "0.2.6" + disable_for_version: "2.0.0" - config_path: "secrets/config_api_token.json" backward_compatibility_tests_config: - disable_for_version: "0.2.6" + disable_for_version: "2.0.0" - config_path: "secrets/config_oauth.json" backward_compatibility_tests_config: - disable_for_version: "0.2.6" + disable_for_version: "2.0.0" basic_read: tests: - config_path: "secrets/config_api_token.json" @@ -68,11 +68,7 @@ acceptance_tests: - config_path: "secrets/config_api_token.json" incremental: tests: - - config_path: "secrets/config_api_token.json" - configured_catalog_path: "integration_tests/incremental_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" - cursor_paths: - items: [ "updated_at_int" ] - boards: [ "updated_at_int" ] - activity_logs: [ "created_at_int" ] + - config_path: "secrets/config_api_token.json" + configured_catalog_path: "integration_tests/incremental_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" diff --git a/airbyte-integrations/connectors/source-monday/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-monday/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-monday/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-monday/build.gradle b/airbyte-integrations/connectors/source-monday/build.gradle deleted file mode 100644 index afe5583358a0..000000000000 --- a/airbyte-integrations/connectors/source-monday/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_monday' -} diff --git a/airbyte-integrations/connectors/source-monday/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-monday/integration_tests/expected_records.jsonl index 319adbe5ebcb..8c9370bc4be5 100644 --- a/airbyte-integrations/connectors/source-monday/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-monday/integration_tests/expected_records.jsonl @@ -1,17 +1,8 @@ -{"stream": "items", "data": {"assets": [], "board": {"id": "4635211873"}, "column_values": [{"additional_info": null, "description": null, "id": "person", "text": "", "title": "Person", "type": "multiple-person", "value": null}, {"additional_info": "{\"label\":\"Working on it\",\"color\":\"#fdab3d\",\"changed_at\":\"2019-03-01T17:24:57.321Z\"}", "description": null, "id": "status", "text": "Working on it", "title": "Status", "type": "color", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2019-03-01T17:24:57.321Z\"}"}, {"additional_info": null, "description": null, "id": "date4", "text": "2023-06-11", "title": "Date", "type": "date", "value": "{\"date\":\"2023-06-11\",\"icon\":null,\"changed_at\":\"2023-06-13T13:58:25.871Z\"}"}, {"additional_info": null, "description": null, "id": "tags", "text": "open", "title": "Tags", "type": "tag", "value": "{\"tag_ids\":[19038090]}"}], "created_at": "2023-06-13T13:58:24Z", "creator_id": "36694549", "group": {"id": "topics"}, "id": "4635211945", "name": "Item 1", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2023-06-15T16:19:37Z", "updates": [{"id": "2223820299"}, {"id": "2223818363"}], "updated_at_int": 1686845977}, "emitted_at": 1690884054247} -{"stream": "items", "data": {"assets": [], "board": {"id": "4635211873"}, "column_values": [{"additional_info": null, "description": null, "id": "person", "text": "", "title": "Person", "type": "multiple-person", "value": null}, {"additional_info": "{\"label\":\"Done\",\"color\":\"#00c875\",\"changed_at\":\"2019-03-01T17:28:23.178Z\"}", "description": null, "id": "status", "text": "Done", "title": "Status", "type": "color", "value": "{\"index\":1,\"post_id\":null,\"changed_at\":\"2019-03-01T17:28:23.178Z\"}"}, {"additional_info": null, "description": null, "id": "date4", "text": "2023-06-11", "title": "Date", "type": "date", "value": "{\"date\":\"2023-06-11\",\"icon\":null,\"changed_at\":\"2023-06-13T13:58:25.871Z\"}"}, {"additional_info": null, "description": null, "id": "tags", "text": "closed", "title": "Tags", "type": "tag", "value": "{\"tag_ids\":[19038091]}"}], "created_at": "2023-06-13T13:58:24Z", "creator_id": "36694549", "group": {"id": "topics"}, "id": "4635211964", "name": "Item 2", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2023-06-13T13:59:36Z", "updates": [], "updated_at_int": 1686664776}, "emitted_at": 1690884054254} -{"stream": "items", "data": {"assets": [], "board": {"id": "4635211873"}, "column_values": [{"additional_info": null, "description": null, "id": "person", "text": "", "title": "Person", "type": "multiple-person", "value": null}, {"additional_info": "{\"label\":null,\"color\":\"#c4c4c4\",\"changed_at\":\"2019-03-01T17:25:02.248Z\"}", "description": null, "id": "status", "text": null, "title": "Status", "type": "color", "value": "{\"index\":5,\"post_id\":null,\"changed_at\":\"2019-03-01T17:25:02.248Z\"}"}, {"additional_info": null, "description": null, "id": "date4", "text": "2023-06-13", "title": "Date", "type": "date", "value": "{\"date\":\"2023-06-13\",\"icon\":null,\"changed_at\":\"2023-06-13T13:58:26.291Z\"}"}, {"additional_info": null, "description": null, "id": "tags", "text": "", "title": "Tags", "type": "tag", "value": null}], "created_at": "2023-06-13T13:58:24Z", "creator_id": "36694549", "group": {"id": "topics"}, "id": "4635211977", "name": "Item 3", "parent_item": null, "state": "active", "subscribers": [{"id": 36694549}], "updated_at": "2023-06-13T13:58:26Z", "updates": [], "updated_at_int": 1686664706}, "emitted_at": 1690884054258} -{"stream": "boards", "data": {"board_kind": "public", "columns": [{"archived": false, "description": null, "id": "name", "settings_str": "{}", "title": "Name", "type": "name", "width": 400}, {"archived": false, "description": null, "id": "person", "settings_str": "{}", "title": "Person", "type": "multiple-person", "width": null}, {"archived": false, "description": null, "id": "status", "settings_str": "{\"labels\":{\"0\":\"Working on it\",\"1\":\"Done\",\"2\":\"Stuck\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Status", "type": "color", "width": null}, {"archived": false, "description": null, "id": "date4", "settings_str": "{}", "title": "Date", "type": "date", "width": null}, {"archived": false, "description": null, "id": "tags", "settings_str": "{\"hide_footer\":false}", "title": "Tags", "type": "tag", "width": null}], "communication": null, "description": null, "groups": [{"archived": false, "color": "#579bfc", "deleted": false, "id": "topics", "position": "65536", "title": "Group Title"}, {"archived": false, "color": "#a25ddc", "deleted": false, "id": "group_title", "position": "98304", "title": "Group Title"}, {"archived": false, "color": "#808080", "deleted": false, "id": "new_group", "position": "163840.0", "title": "New Group unit board"}], "id": "4635211873", "name": "New Board", "owners": [{"id": 36694549}], "creator": {"id": 36694549}, "permissions": "everyone", "pos": null, "state": "active", "subscribers": [{"id": 36694549}], "tags": [], "top_group": {"id": "topics"}, "updated_at": "2023-06-20T12:12:46Z", "updates": [{"id": "2223820299"}, {"id": "2223818363"}], "views": [], "workspace": {"id": 2845647, "name": "Test workspace", "kind": "open", "description": null}, "updated_at_int": 1687263166}, "emitted_at": 1690884065399} -{"stream": "boards", "data": {"board_kind": "public", "columns": [{"archived": false, "description": null, "id": "name", "settings_str": "{}", "title": "Name", "type": "name", "width": 400}, {"archived": false, "description": null, "id": "files", "settings_str": "{\"hide_footer\":false}", "title": "Files", "type": "file", "width": null}], "communication": null, "description": null, "groups": [{"archived": false, "color": "#579bfc", "deleted": false, "id": "topics", "position": "65536", "title": "Group Title"}], "id": "4634950289", "name": "test doc", "owners": [{"id": 36694549}], "creator": {"id": 36694549}, "permissions": "everyone", "pos": null, "state": "active", "subscribers": [{"id": 36694549}], "tags": [], "top_group": {"id": "topics"}, "updated_at": "2023-06-13T13:28:31Z", "updates": [], "views": [{"id": "103920755", "name": "Table", "settings_str": "{}", "type": "FeatureBoardView", "view_specific_data_str": "{}"}], "workspace": {"id": 2845647, "name": "Test workspace", "kind": "open", "description": null}, "updated_at_int": 1686662911}, "emitted_at": 1690884065405} -{"stream": "boards", "data": {"board_kind": "public", "columns": [{"archived": false, "description": null, "id": "name", "settings_str": "{}", "title": "Name", "type": "name", "width": 380}, {"archived": false, "description": null, "id": "manager1", "settings_str": "{}", "title": "Owner", "type": "multiple-person", "width": 80}, {"archived": false, "description": null, "id": "date4", "settings_str": "{}", "title": "Request date", "type": "date", "width": null}, {"archived": false, "description": null, "id": "status1", "settings_str": "{\"labels\":{\"0\":\"Evaluating\",\"1\":\"Done\",\"2\":\"Denied\",\"3\":\"Waiting for legal\",\"6\":\"Approved for POC\",\"11\":\"On hold\",\"14\":\"Waiting for vendor\",\"15\":\"Negotiation\",\"108\":\"Approved for use\"},\"labels_positions_v2\":{\"0\":0,\"1\":1,\"2\":7,\"3\":8,\"5\":9,\"6\":3,\"11\":6,\"14\":5,\"15\":4,\"108\":2},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"},\"3\":{\"color\":\"#0086c0\",\"border\":\"#3DB0DF\",\"var_name\":\"blue-links\"},\"6\":{\"color\":\"#037f4c\",\"border\":\"#006B38\",\"var_name\":\"grass-green\"},\"11\":{\"color\":\"#BB3354\",\"border\":\"#A42D4A\",\"var_name\":\"dark-red\"},\"14\":{\"color\":\"#784BD1\",\"border\":\"#8F4DC4\",\"var_name\":\"dark-purple\"},\"15\":{\"color\":\"#9CD326\",\"border\":\"#89B921\",\"var_name\":\"lime-green\"},\"108\":{\"color\":\"#4eccc6\",\"border\":\"#4eccc6\",\"var_name\":\"australia\"}}}", "title": "Procurement status", "type": "color", "width": null}, {"archived": false, "description": null, "id": "person", "settings_str": "{}", "title": "Manager", "type": "multiple-person", "width": 80}, {"archived": false, "description": null, "id": "status", "settings_str": "{\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Manager approval", "type": "color", "width": null}, {"archived": false, "description": null, "id": "budget_owner", "settings_str": "{}", "title": "POC owner", "type": "multiple-person", "width": 80}, {"archived": false, "description": null, "id": "budget_owner_approval4", "settings_str": "{\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "POC status", "type": "color", "width": null}, {"archived": false, "description": null, "id": "manager", "settings_str": "{}", "title": "Budget owner", "type": "multiple-person", "width": 80}, {"archived": false, "description": null, "id": "status4", "settings_str": "{\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Budget owner approval", "type": "color", "width": 185}, {"archived": false, "description": null, "id": "people", "settings_str": "{}", "title": "Procurement team", "type": "multiple-person", "width": null}, {"archived": false, "description": null, "id": "budget_owner_approval", "settings_str": "{\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Procurement approval", "type": "color", "width": null}, {"archived": false, "description": null, "id": "procurement_team", "settings_str": "{}", "title": "Finance", "type": "multiple-person", "width": null}, {"archived": false, "description": null, "id": "procurement_approval", "settings_str": "{\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Finance approval", "type": "color", "width": null}, {"archived": false, "description": null, "id": "finance", "settings_str": "{}", "title": "Legal", "type": "multiple-person", "width": null}, {"archived": false, "description": null, "id": "finance_approval", "settings_str": "{\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Redlines\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Legal approval", "type": "color", "width": null}, {"archived": false, "description": null, "id": "file", "settings_str": "{}", "title": "File", "type": "file", "width": null}, {"archived": false, "description": null, "id": "legal", "settings_str": "{}", "title": "Security", "type": "multiple-person", "width": null}, {"archived": false, "description": null, "id": "legal_approval", "settings_str": "{\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Security approval", "type": "color", "width": null}, {"archived": false, "description": null, "id": "date", "settings_str": "{\"hide_footer\":false}", "title": "Renewal date", "type": "date", "width": null}, {"archived": false, "description": null, "id": "last_updated", "settings_str": "{}", "title": "Last updated", "type": "pulse-updated", "width": 129}], "communication": null, "description": "Many IT departments need to handle the procurement process for new services. The essence of this board is to streamline this process by providing an intuitive structure that supports collaboration and efficiency.", "groups": [{"archived": false, "color": "#579bfc", "deleted": false, "id": "topics", "position": "65536", "title": "Reviewing"}, {"archived": false, "color": "#FF642E", "deleted": false, "id": "new_group", "position": "98304.0", "title": "Corporate IT"}, {"archived": false, "color": "#037f4c", "deleted": false, "id": "new_group2816", "position": "114688.0", "title": "Finance"}], "id": "3555407826", "name": "Procurement process", "owners": [{"id": 36694549}], "creator": {"id": 36694549}, "permissions": "everyone", "pos": null, "state": "active", "subscribers": [{"id": 36694549}], "tags": [], "top_group": {"id": "topics"}, "updated_at": "2022-11-21T14:36:50Z", "updates": [], "views": [{"id": "80969928", "name": "Chart", "settings_str": "{\"x_axis_columns\":{\"status1\":true},\"y_axis_columns\":{\"default-label-count\":true},\"z_axis_columns\":{},\"guideline_base\":{},\"graph_type\":\"column\",\"empty_values\":false,\"group_by\":\"month\"}", "type": "GraphBoardView", "view_specific_data_str": "{}"}], "workspace": null, "updated_at_int": 1669041410}, "emitted_at": 1690884065408} -{"stream": "tags", "data": {"color": "#00c875", "id": 19038090, "name": "open"}, "emitted_at": 1690884065804} -{"stream": "tags", "data": {"color": "#fdab3d", "id": 19038091, "name": "closed"}, "emitted_at": 1690884065806} -{"stream": "updates", "data": {"assets": [{"created_at": "2023-06-15T16:19:31Z", "file_extension": ".jpg", "file_size": 116107, "id": "919077184", "name": "black_cat.jpg", "original_geometry": "473x600", "public_url": "https://files-monday-com.s3.amazonaws.com/14202902/resources/919077184/black_cat.jpg?response-content-disposition=attachment&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA4MPVJMFXGWGLJTLY%2F20230801%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230801T100107Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=5d2d3ca95375589e620f89630d58ff0f7417f1ddd8968ceb57af854657718564", "uploaded_by": {"id": 36694549}, "url": "https://airbyte-unit.monday.com/protected_static/14202902/resources/919077184/black_cat.jpg", "url_thumbnail": "https://airbyte-unit.monday.com/protected_static/14202902/resources/919077184/thumb_small-black_cat.jpg"}], "body": "", "created_at": "2023-06-15T16:19:36Z", "creator_id": "36694549", "id": "2223820299", "item_id": "4635211945", "replies": [], "text_body": "", "updated_at": "2023-06-15T16:19:36Z"}, "emitted_at": 1690884067025} -{"stream": "updates", "data": {"assets": [], "body": "



      ", "created_at": "2023-06-15T16:18:50Z", "creator_id": "36694549", "id": "2223818363", "item_id": "4635211945", "replies": [], "text_body": "", "updated_at": "2023-06-15T16:18:50Z"}, "emitted_at": 1690884067027} -{"stream": "updates", "data": {"assets": [], "body": "

      \ufeffTest

      ", "created_at": "2022-11-21T14:41:21Z", "creator_id": "36694549", "id": "1825302913", "item_id": "3555437747", "replies": [{"id": "1825303266", "creator_id": "36694549", "created_at": "2022-11-21T14:41:29Z", "text_body": "Test test", "updated_at": "2022-11-21T14:41:29Z", "body": "

      \ufeffTest test

      "}, {"id": "2223806079", "creator_id": "36694549", "created_at": "2023-06-15T16:14:13Z", "text_body": "", "updated_at": "2023-06-15T16:14:13Z", "body": "



      "}], "text_body": "Test", "updated_at": "2023-06-15T16:14:13Z"}, "emitted_at": 1690884067029} -{"stream": "users", "data": {"birthday": null, "country_code": "UA", "created_at": "2022-11-21T14:03:00Z", "join_date": null, "email": "integration-test@airbyte.io", "enabled": true, "id": 36694549, "is_admin": true, "is_guest": false, "is_pending": false, "is_view_only": false, "is_verified": true, "location": null, "mobile_phone": null, "name": "Airbyte Team", "phone": "", "photo_original": "https://files.monday.com/use1/photos/36694549/original/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_small": "https://files.monday.com/use1/photos/36694549/small/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_thumb": "https://files.monday.com/use1/photos/36694549/thumb/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_thumb_small": "https://files.monday.com/use1/photos/36694549/thumb_small/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_tiny": "https://files.monday.com/use1/photos/36694549/tiny/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "time_zone_identifier": "Europe/Kiev", "title": "Airbyte Developer Account", "url": "https://airbyte-unit.monday.com/users/36694549", "utc_hours_diff": 3}, "emitted_at": 1690884067354} -{"stream": "users", "data": {"birthday": null, "country_code": "UA", "created_at": "2022-11-21T14:33:18Z", "join_date": null, "email": "iryna.grankova@airbyte.io", "enabled": true, "id": 36695702, "is_admin": false, "is_guest": false, "is_pending": false, "is_view_only": false, "is_verified": true, "location": null, "mobile_phone": null, "name": "Iryna Grankova", "phone": null, "photo_original": "https://files.monday.com/use1/photos/36695702/original/36695702-user_photo_initials_2022_11_21_14_34_12.png?1669041252", "photo_small": "https://files.monday.com/use1/photos/36695702/small/36695702-user_photo_initials_2022_11_21_14_34_12.png?1669041252", "photo_thumb": "https://files.monday.com/use1/photos/36695702/thumb/36695702-user_photo_initials_2022_11_21_14_34_12.png?1669041252", "photo_thumb_small": "https://files.monday.com/use1/photos/36695702/thumb_small/36695702-user_photo_initials_2022_11_21_14_34_12.png?1669041252", "photo_tiny": "https://files.monday.com/use1/photos/36695702/tiny/36695702-user_photo_initials_2022_11_21_14_34_12.png?1669041252", "time_zone_identifier": "Europe/Athens", "title": null, "url": "https://airbyte-unit.monday.com/users/36695702", "utc_hours_diff": 3}, "emitted_at": 1690884067356} -{"stream": "workspaces", "data": {"created_at": "2023-06-08T11:26:44Z", "description": null, "id": 2845647, "kind": "open", "name": "Test workspace", "state": "active", "account_product": {"id": 2248222, "kind": "core"}, "owners_subscribers": [{"id": 36694549}], "settings": {"icon": {"color": "#FDAB3D", "image": null}}, "team_owners_subscribers": [], "teams_subscribers": [], "users_subscribers": [{"id": 36694549}]}, "emitted_at": 1690884067856} -{"stream": "activity_logs", "data": {"id": "81d07d4d-414d-458e-b44c-fef36e44c424", "event": "create_pulse", "data": "{\"board_id\":4635211873,\"group_id\":\"new_group\",\"group_name\":\"New Group unit board\",\"group_color\":\"#808080\",\"is_top_group\":false,\"pulse_id\":4672924165,\"pulse_name\":\"Item 7\",\"column_values_json\":\"{}\"}", "entity": "pulse", "created_at": "16872631837419768", "created_at_int": 1687263183, "pulse_id": 4672924165}, "emitted_at": 1690884068262} -{"stream": "activity_logs", "data": {"id": "c0aa4bab-d3a4-4f13-8942-934178c0238a", "event": "update_column_value", "data": "{\"board_id\":4635211873,\"group_id\":\"group_title\",\"is_top_group\":false,\"pulse_id\":4672922929,\"pulse_name\":\"Item 6\",\"column_id\":\"status\",\"column_type\":\"color\",\"column_title\":\"Status\",\"value\":{\"label\":{\"index\":1,\"text\":\"Done\",\"style\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"is_done\":true},\"post_id\":null},\"previous_value\":null,\"is_column_with_hide_permissions\":false}", "entity": "pulse", "created_at": "16872631743009674", "created_at_int": 1687263174, "pulse_id": 4672922929}, "emitted_at": 1690884068266} -{"stream": "activity_logs", "data": {"id": "4e8f926c-1b4e-43d8-a3de-09af876ccb9e", "event": "create_pulse", "data": "{\"board_id\":4635211873,\"group_id\":\"group_title\",\"group_name\":\"Group Title\",\"group_color\":\"#a25ddc\",\"is_top_group\":false,\"pulse_id\":4672922929,\"pulse_name\":\"Item 6\",\"column_values_json\":\"{}\"}", "entity": "pulse", "created_at": "16872631712788820", "created_at_int": 1687263171, "pulse_id": 4672922929}, "emitted_at": 1690884068269} +{"stream": "items", "data": {"id": "4635211945", "name": "Item 1", "assets": [], "board": {"id": "4635211873"}, "column_values": [{"id": "person", "text": "", "type": "people", "value": null}, {"id": "status", "text": "Working on it", "type": "status", "value": "{\"index\":0,\"post_id\":null,\"changed_at\":\"2019-03-01T17:24:57.321Z\"}"}, {"id": "date4", "text": "2023-06-11", "type": "date", "value": "{\"date\":\"2023-06-11\",\"icon\":null,\"changed_at\":\"2023-06-13T13:58:25.871Z\"}"}, {"id": "tags", "text": "open", "type": "tags", "value": "{\"tag_ids\":[19038090]}"}], "created_at": "2023-06-13T13:58:24Z", "creator_id": "36694549", "group": {"id": "topics"}, "parent_item": null, "state": "active", "subscribers": [{"id": "36694549"}], "updated_at": "2023-06-15T16:19:37Z", "updates": [{"id": "2223820299"}, {"id": "2223818363"}], "updated_at_int": 1686845977}, "emitted_at": 1705072697006} +{"stream": "boards", "data": {"id": "3555407826", "name": "Procurement process", "board_kind": "public", "type": "board", "columns": [{"archived": false, "description": null, "id": "name", "settings_str": "{}", "title": "Name", "type": "name", "width": 380}, {"archived": false, "description": null, "id": "manager1", "settings_str": "{}", "title": "Owner", "type": "people", "width": 80}, {"archived": false, "description": null, "id": "date4", "settings_str": "{}", "title": "Request date", "type": "date", "width": null}, {"archived": false, "description": null, "id": "status1", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"Evaluating\",\"1\":\"Done\",\"2\":\"Denied\",\"3\":\"Waiting for legal\",\"6\":\"Approved for POC\",\"11\":\"On hold\",\"14\":\"Waiting for vendor\",\"15\":\"Negotiation\",\"108\":\"Approved for use\"},\"labels_positions_v2\":{\"0\":0,\"1\":1,\"2\":7,\"3\":8,\"5\":9,\"6\":3,\"11\":6,\"14\":5,\"15\":4,\"108\":2},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"},\"3\":{\"color\":\"#0086c0\",\"border\":\"#3DB0DF\",\"var_name\":\"blue-links\"},\"6\":{\"color\":\"#037f4c\",\"border\":\"#006B38\",\"var_name\":\"grass-green\"},\"11\":{\"color\":\"#BB3354\",\"border\":\"#A42D4A\",\"var_name\":\"dark-red\"},\"14\":{\"color\":\"#784BD1\",\"border\":\"#8F4DC4\",\"var_name\":\"dark-purple\"},\"15\":{\"color\":\"#9CD326\",\"border\":\"#89B921\",\"var_name\":\"lime-green\"},\"108\":{\"color\":\"#4eccc6\",\"border\":\"#4eccc6\",\"var_name\":\"australia\"}}}", "title": "Procurement status", "type": "status", "width": null}, {"archived": false, "description": null, "id": "person", "settings_str": "{}", "title": "Manager", "type": "people", "width": 80}, {"archived": false, "description": null, "id": "status", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Manager approval", "type": "status", "width": null}, {"archived": false, "description": null, "id": "budget_owner", "settings_str": "{}", "title": "POC owner", "type": "people", "width": 80}, {"archived": false, "description": null, "id": "budget_owner_approval4", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "POC status", "type": "status", "width": null}, {"archived": false, "description": null, "id": "manager", "settings_str": "{}", "title": "Budget owner", "type": "people", "width": 80}, {"archived": false, "description": null, "id": "status4", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Budget owner approval", "type": "status", "width": 185}, {"archived": false, "description": null, "id": "people", "settings_str": "{}", "title": "Procurement team", "type": "people", "width": null}, {"archived": false, "description": null, "id": "budget_owner_approval", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Procurement approval", "type": "status", "width": null}, {"archived": false, "description": null, "id": "procurement_team", "settings_str": "{}", "title": "Finance", "type": "people", "width": null}, {"archived": false, "description": null, "id": "procurement_approval", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Finance approval", "type": "status", "width": null}, {"archived": false, "description": null, "id": "finance", "settings_str": "{}", "title": "Legal", "type": "people", "width": null}, {"archived": false, "description": null, "id": "finance_approval", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Redlines\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Legal approval", "type": "status", "width": null}, {"archived": false, "description": null, "id": "file", "settings_str": "{}", "title": "File", "type": "file", "width": null}, {"archived": false, "description": null, "id": "legal", "settings_str": "{}", "title": "Security", "type": "people", "width": null}, {"archived": false, "description": null, "id": "legal_approval", "settings_str": "{\"done_colors\":[1],\"labels\":{\"0\":\"On Hold\",\"1\":\"Approved\",\"2\":\"Declined\"},\"labels_positions_v2\":{\"0\":0,\"1\":2,\"2\":1,\"5\":3},\"labels_colors\":{\"0\":{\"color\":\"#fdab3d\",\"border\":\"#E99729\",\"var_name\":\"orange\"},\"1\":{\"color\":\"#00c875\",\"border\":\"#00B461\",\"var_name\":\"green-shadow\"},\"2\":{\"color\":\"#e2445c\",\"border\":\"#CE3048\",\"var_name\":\"red-shadow\"}}}", "title": "Security approval", "type": "status", "width": null}, {"archived": false, "description": null, "id": "date", "settings_str": "{\"hide_footer\":false}", "title": "Renewal date", "type": "date", "width": null}, {"archived": false, "description": null, "id": "last_updated", "settings_str": "{}", "title": "Last updated", "type": "last_updated", "width": 129}], "communication": null, "description": "Many IT departments need to handle the procurement process for new services. The essence of this board is to streamline this process by providing an intuitive structure that supports collaboration and efficiency.", "groups": [{"archived": false, "color": "#579bfc", "deleted": false, "id": "topics", "position": "65536", "title": "Reviewing"}, {"archived": false, "color": "#FF642E", "deleted": false, "id": "new_group", "position": "98304.0", "title": "Corporate IT"}, {"archived": false, "color": "#037f4c", "deleted": false, "id": "new_group2816", "position": "114688.0", "title": "Finance"}], "owners": [{"id": "36694549"}], "creator": {"id": "36694549"}, "permissions": "everyone", "state": "active", "subscribers": [{"id": "36694549"}], "tags": [], "top_group": {"id": "topics"}, "updated_at": "2022-11-21T14:36:50Z", "updates": [], "views": [], "workspace": null, "updated_at_int": 1669041410}, "emitted_at": 1705073472066} +{"stream": "tags", "data": {"color": "#00c875", "id": "19038090", "name": "open"}, "emitted_at": 1690884065804} +{"stream": "tags", "data": {"color": "#fdab3d", "id": "19038091", "name": "closed"}, "emitted_at": 1690884065806} +{"stream": "updates", "data": {"assets": [{"created_at": "2023-06-15T16:19:31Z", "file_extension": ".jpg", "file_size": 116107, "id": "919077184", "name": "black_cat.jpg", "original_geometry": "473x600", "public_url": "https://files-monday-com.s3.amazonaws.com/14202902/resources/919077184/black_cat.jpg?response-content-disposition=attachment&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA4MPVJMFXILAOBJXD%2F20240112%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20240112T154009Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=b4f14a9dd800d70520f428ff7f4a29aa1b6a259d761f3b073fe83c41010c729a", "uploaded_by": {"id": "36694549"}, "url": "https://airbyte-unit.monday.com/protected_static/14202902/resources/919077184/black_cat.jpg", "url_thumbnail": "https://airbyte-unit.monday.com/protected_static/14202902/resources/919077184/thumb_small-black_cat.jpg"}], "body": "", "created_at": "2023-06-15T16:19:36Z", "creator_id": "36694549", "id": "2223820299", "item_id": "4635211945", "replies": [], "text_body": "", "updated_at": "2023-06-15T16:19:36Z"}, "emitted_at": 1705074009909} +{"stream": "users", "data": {"birthday": null, "country_code": "UA", "created_at": "2022-11-21T14:03:00Z", "join_date": null, "email": "integration-test@airbyte.io", "enabled": true, "id": "36694549", "is_admin": true, "is_guest": false, "is_pending": false, "is_view_only": false, "is_verified": true, "location": null, "mobile_phone": null, "name": "Airbyte Team", "phone": "", "photo_original": "https://files.monday.com/use1/photos/36694549/original/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_small": "https://files.monday.com/use1/photos/36694549/small/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_thumb": "https://files.monday.com/use1/photos/36694549/thumb/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_thumb_small": "https://files.monday.com/use1/photos/36694549/thumb_small/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "photo_tiny": "https://files.monday.com/use1/photos/36694549/tiny/36694549-user_photo_2022_11_21_14_10_42.png?1669039842", "time_zone_identifier": "Europe/Kiev", "title": "Airbyte Developer Account", "url": "https://airbyte-unit.monday.com/users/36694549", "utc_hours_diff": 2}, "emitted_at": 1702496564648} +{"stream": "workspaces", "data": {"created_at": "2023-06-08T11:26:44Z", "description": null, "id": "2845647", "kind": "open", "name": "Test workspace", "state": "active", "account_product": {"id": "2248222", "kind": "core"}, "owners_subscribers": [{"id": "36694549"}], "settings": {"icon": {"color": "#FDAB3D", "image": null}}, "team_owners_subscribers": [], "teams_subscribers": [], "users_subscribers": [{"id": "36694549"}]}, "emitted_at": 1705074164892} +{"stream": "activity_logs", "data": {"id": "81d07d4d-414d-458e-b44c-fef36e44c424", "event": "create_pulse", "data": "{\"board_id\":4635211873,\"group_id\":\"new_group\",\"group_name\":\"New Group unit board\",\"group_color\":\"#808080\",\"is_top_group\":false,\"pulse_id\":4672924165,\"pulse_name\":\"Item 7\",\"column_values_json\":\"{}\"}", "entity": "pulse", "created_at": "16872631837419768", "created_at_int": 1687263183, "pulse_id": 4672924165}, "emitted_at": 1705074202226} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-monday/metadata.yaml b/airbyte-integrations/connectors/source-monday/metadata.yaml index 2c85a0d98294..d5a3a9f0f41c 100644 --- a/airbyte-integrations/connectors/source-monday/metadata.yaml +++ b/airbyte-integrations/connectors/source-monday/metadata.yaml @@ -1,12 +1,35 @@ data: + ab_internal: + ql: 200 + sl: 200 allowedHosts: hosts: - api.monday.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 80a54ea2-9959-4040-aac1-eee42423ec9b - dockerImageTag: 1.1.2 + dockerImageTag: 2.0.0 + releases: + breakingChanges: + 2.0.0: + message: "Source Monday has deprecated API version 2023-07. We have upgraded the connector to the latest API version 2024-01. In this new version, the Id field has changed from an integer to a string in the streams Boards, Items, Tags, Teams, Updates, Users and Workspaces. Please reset affected streams." + upgradeDeadline: "2024-01-15" + scopedImpact: + - scopeType: stream + impactedScopes: + [ + "boards", + "items", + "tags", + "teams", + "updates", + "users", + "workspaces", + ] dockerRepository: airbyte/source-monday + documentationUrl: https://docs.airbyte.com/integrations/sources/monday githubIssueLabel: source-monday icon: monday.svg license: MIT @@ -17,12 +40,8 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/monday + supportLevel: certified tags: - language:low-code - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-monday/source_monday/extractor.py b/airbyte-integrations/connectors/source-monday/source_monday/extractor.py index bd8524044024..830aafc5cf9a 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/extractor.py +++ b/airbyte-integrations/connectors/source-monday/source_monday/extractor.py @@ -72,11 +72,12 @@ class MondayIncrementalItemsExtractor(RecordExtractor): field_path: List[Union[InterpolatedString, str]] config: Config parameters: InitVar[Mapping[str, Any]] - additional_field_path: List[Union[InterpolatedString, str]] = field(default_factory=list) + field_path_pagination: List[Union[InterpolatedString, str]] = field(default_factory=list) + field_path_incremental: List[Union[InterpolatedString, str]] = field(default_factory=list) decoder: Decoder = JsonDecoder(parameters={}) def __post_init__(self, parameters: Mapping[str, Any]): - for field_list in (self.field_path, self.additional_field_path): + for field_list in (self.field_path, self.field_path_pagination, self.field_path_incremental): for path_index in range(len(field_list)): if isinstance(field_list[path_index], str): field_list[path_index] = InterpolatedString.create(field_list[path_index], parameters=parameters) @@ -100,8 +101,10 @@ def try_extract_records(self, response: requests.Response, field_path: List[Unio def extract_records(self, response: requests.Response) -> List[Record]: result = self.try_extract_records(response, field_path=self.field_path) - if not result and self.additional_field_path: - result = self.try_extract_records(response, self.additional_field_path) + if not result and self.field_path_pagination: + result = self.try_extract_records(response, self.field_path_pagination) + if not result and self.field_path_incremental: + result = self.try_extract_records(response, self.field_path_incremental) for item_index in range(len(result)): if "updated_at" in result[item_index]: diff --git a/airbyte-integrations/connectors/source-monday/source_monday/graphql_requester.py b/airbyte-integrations/connectors/source-monday/source_monday/graphql_requester.py index 0e5fd049583c..fec0d06cbc3e 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/graphql_requester.py +++ b/airbyte-integrations/connectors/source-monday/source_monday/graphql_requester.py @@ -79,18 +79,37 @@ def _build_query(self, object_name: str, field_schema: dict, **object_arguments) arguments = f"({arguments})" if arguments else "" fields = ",".join(fields) - return f"{object_name}{arguments}{{{fields}}}" + if object_name in ["items_page", "next_items_page"]: + query = f"{object_name}{arguments}{{cursor,items{{{fields}}}}}" + else: + query = f"{object_name}{arguments}{{{fields}}}" + return query def _build_items_query(self, object_name: str, field_schema: dict, sub_page: Optional[int], **object_arguments) -> str: """ Special optimization needed for items stream. Starting October 3rd, 2022 items can only be reached through boards. See https://developer.monday.com/api-reference/docs/items-queries#items-queries + + Comparison of different APIs queries: + 2023-07: + boards(limit: 1) { items(limit: 20) { field1, field2, ... }} + boards(limit: 1, page:2) { items(limit: 20, page:2) { field1, field2, ... }} boards and items paginations + 2024_01: + boards(limit: 1) { items_page(limit: 20) {cursor, items{field1, field2, ...} }} + boards(limit: 1, page:2) { items_page(limit: 20) {cursor, items{field1, field2, ...} }} - boards pagination + next_items_page(limit: 20, cursor: "blaa") {cursor, items{field1, field2, ...} } - items pagination + """ nested_limit = self.nested_limit.eval(self.config) - query = self._build_query("items", field_schema, limit=nested_limit, page=sub_page) - arguments = self._get_object_arguments(**object_arguments) - return f"boards({arguments}){{{query}}}" + if sub_page: + query = self._build_query("next_items_page", field_schema, limit=nested_limit, cursor=f'"{sub_page}"') + else: + query = self._build_query("items_page", field_schema, limit=nested_limit) + arguments = self._get_object_arguments(**object_arguments) + query = f"boards({arguments}){{{query}}}" + + return query def _build_items_incremental_query(self, object_name: str, field_schema: dict, stream_slice: dict, **object_arguments) -> str: """ @@ -133,6 +152,17 @@ def _build_activity_query(self, object_name: str, field_schema: dict, sub_page: arguments = self._get_object_arguments(**object_arguments) return f"boards({arguments}){{{query}}}" + def get_request_headers( + self, + *, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Mapping[str, Any]: + headers = super().get_request_headers(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + headers["API-Version"] = "2024-01" + return headers + def get_request_params( self, *, diff --git a/airbyte-integrations/connectors/source-monday/source_monday/item_pagination_strategy.py b/airbyte-integrations/connectors/source-monday/source_monday/item_pagination_strategy.py index 5b18cb4b37b7..a6276416d2e5 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/item_pagination_strategy.py +++ b/airbyte-integrations/connectors/source-monday/source_monday/item_pagination_strategy.py @@ -6,6 +6,10 @@ from airbyte_cdk.sources.declarative.requesters.paginators.strategies.page_increment import PageIncrement +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + class ItemPaginationStrategy(PageIncrement): """ @@ -45,3 +49,53 @@ def next_page_token(self, response, last_records: List[Mapping[str, Any]]) -> Op return None return self._page, self._sub_page + + +class ItemCursorPaginationStrategy(PageIncrement): + """ + Page increment strategy with subpages for the `items` stream. + + From the `items` documentation https://developer.monday.com/api-reference/docs/items: + Please note that you cannot return more than 100 items per query when using items at the root. + To adjust your query, try only returning items on a specific board, nesting items inside a boards query, + looping through the boards on your account, or querying less than 100 items at a time. + + This pagination strategy supports nested loop through `boards` on the top level and `items` on the second. + See boards documentation for more details: https://developer.monday.com/api-reference/docs/boards#queries. + """ + + def __post_init__(self, parameters: Mapping[str, Any]): + # `self._page` corresponds to board page number + # `self._sub_page` corresponds to item page number within its board + self.start_from_page = 1 + self._page: Optional[int] = self.start_from_page + self._sub_page: Optional[int] = self.start_from_page + + def next_page_token(self, response, last_records: List[Mapping[str, Any]]) -> Optional[Tuple[Optional[int], Optional[int]]]: + """ + `items` stream use a separate 2 level pagination strategy where: + 1st level `boards` - incremental pagination + 2nd level `items_page` - cursor pagination + + Attributes: + response: Contains `boards` and corresponding lists of `items` for each `board` + last_records: Parsed `items` from the response + """ + data = response.json()["data"] + boards = data.get("boards", []) + next_items_page = data.get("next_items_page", {}) + if boards: + # there is always only one board due to limit=1, so in one request we extract all 'items_page' for one board only + board = boards[0] + cursor = board.get("items_page", {}).get("cursor", None) + elif next_items_page: + cursor = next_items_page.get("cursor", None) + else: + # Finish pagination if there is no more data + return None + + if cursor: + return self._page, cursor + else: + self._page += 1 + return self._page, None diff --git a/airbyte-integrations/connectors/source-monday/source_monday/manifest.yaml b/airbyte-integrations/connectors/source-monday/source_monday/manifest.yaml index dc482bae0f68..658c635cf206 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/manifest.yaml +++ b/airbyte-integrations/connectors/source-monday/source_monday/manifest.yaml @@ -11,11 +11,11 @@ definitions: field_path: - "data" - "{{ parameters['name'] }}" + requester: type: CustomRequester class_name: "source_monday.MondayGraphqlRequester" url_base: "https://api.monday.com/v2" - http_method: "GET" authenticator: type: BearerAuthenticator api_token: "{{ config.get('credentials', {}).get('api_token') if config.get('credentials', {}).get('auth_type') == 'api_token' else config.get('credentials', {}).get('access_token') if config.get('credentials', {}).get('auth_type') == 'oauth2.0' else config.get('api_token', '') }}" @@ -49,6 +49,7 @@ definitions: action: RETRY backoff_strategies: - type: ExponentialBackoffStrategy + default_paginator: type: "DefaultPaginator" pagination_strategy: @@ -62,17 +63,20 @@ definitions: $ref: "#/definitions/requester" paginator: $ref: "#/definitions/default_paginator" + base_stream: retriever: $ref: "#/definitions/retriever" schema_loader: $ref: "#/definitions/schema_loader" primary_key: "id" + base_nopagination_stream: retriever: $ref: "#/definitions/retriever" paginator: type: NoPagination + tags_stream: $ref: "#/definitions/base_nopagination_stream" $parameters: @@ -105,6 +109,12 @@ definitions: class_name: "source_monday.item_pagination_strategy.ItemPaginationStrategy" type: "CustomPaginationStrategy" + cursor_paginator: + $ref: "#/definitions/default_paginator" + pagination_strategy: + class_name: "source_monday.item_pagination_strategy.ItemCursorPaginationStrategy" + type: "CustomPaginationStrategy" + activity_logs_stream: description: "https://developers.intercom.com/intercom-api-reference/reference/scroll-over-all-companies" incremental_sync: @@ -173,12 +183,13 @@ definitions: page_size: 20 nested_items_per_page: 20 parent_key: "pulse_id" - field_path: ["data", "items", "*"] - additional_field_path: ["data", "boards", "*", "items", "*"] + field_path: ["data", "boards", "*", "items_page", "items", "*"] # for first and further incremental pagination responses + field_path_pagination: ["data", "next_items_page", "items", "*"] # for cursor pagination responses + field_path_incremental: ["data", "items", "*"] # for incremental sync responses retriever: $ref: "#/definitions/base_stream/retriever" paginator: - $ref: "#/definitions/double_paginator" + $ref: "#/definitions/cursor_paginator" record_selector: $ref: "#/definitions/selector" extractor: diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/boards.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/boards.json index e976ea5e54f4..992636621db7 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/schemas/boards.json +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/boards.json @@ -2,7 +2,10 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, "board_kind": { "type": ["null", "string"] }, + "type": { "type": ["null", "string"] }, "columns": { "type": ["null", "array"], "items": { @@ -36,26 +39,23 @@ } } }, - "id": { "type": ["null", "string"] }, - "name": { "type": ["null", "string"] }, "owners": { "type": ["null", "array"], "items": { "type": ["null", "object"], "additionalProperties": true, "properties": { - "id": { "type": ["null", "integer"] } + "id": { "type": ["null", "string"] } } } }, "creator": { "type": ["null", "object"], "properties": { - "id": { "type": ["null", "integer"] } + "id": { "type": ["null", "string"] } } }, "permissions": { "type": ["null", "string"] }, - "pos": { "type": ["null", "string"] }, "state": { "type": ["null", "string"] }, "subscribers": { "type": ["null", "array"], @@ -63,7 +63,7 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "id": { "type": ["null", "integer"] } + "id": { "type": ["null", "string"] } } } }, @@ -112,7 +112,7 @@ "workspace": { "type": ["null", "object"], "properties": { - "id": { "type": ["null", "integer"] }, + "id": { "type": ["null", "string"] }, "name": { "type": ["null", "string"] }, "kind": { "type": ["null", "string"] }, "description": { "type": ["null", "string"] } diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/items.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/items.json index 27f02d52354f..a793a5b281b3 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/schemas/items.json +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/items.json @@ -2,6 +2,8 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { + "id": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, "assets": { "type": ["null", "array"], "items": { @@ -18,7 +20,7 @@ "uploaded_by": { "type": ["null", "object"], "properties": { - "id": { "type": ["null", "integer"] } + "id": { "type": ["null", "string"] } } }, "url": { "type": ["null", "string"] }, @@ -38,11 +40,8 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "additional_info": { "type": ["null", "string"] }, - "description": { "type": ["null", "string"] }, "id": { "type": ["null", "string"] }, "text": { "type": ["null", "string"] }, - "title": { "type": ["null", "string"] }, "type": { "type": ["null", "string"] }, "value": { "type": ["null", "string"] } } @@ -56,8 +55,6 @@ "id": { "type": ["null", "string"] } } }, - "id": { "type": ["null", "string"] }, - "name": { "type": ["null", "string"] }, "parent_item": { "type": ["null", "object"], "properties": { @@ -71,7 +68,7 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "id": { "type": ["null", "integer"] } + "id": { "type": ["null", "string"] } } } }, diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/tags.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/tags.json index c96a58d1d42a..e1a4faeb63db 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/schemas/tags.json +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/tags.json @@ -3,7 +3,7 @@ "type": "object", "properties": { "color": { "type": ["null", "string"] }, - "id": { "type": ["null", "integer"] }, + "id": { "type": ["null", "string"] }, "name": { "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/teams.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/teams.json index 16cb865fcc92..0bccac5fd4fa 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/schemas/teams.json +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/teams.json @@ -11,7 +11,7 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "id": { "type": ["null", "integer"] } + "id": { "type": ["null", "string"] } } } } diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/updates.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/updates.json index d0e004a69982..8dc809329358 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/schemas/updates.json +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/updates.json @@ -18,7 +18,7 @@ "uploaded_by": { "type": ["null", "object"], "properties": { - "id": { "type": ["null", "integer"] } + "id": { "type": ["null", "string"] } } }, "url": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/users.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/users.json index a064bdc3f4bc..bd2347a4fc2b 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/schemas/users.json +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/users.json @@ -8,7 +8,7 @@ "join_date": { "type": ["null", "string"], "format": "date" }, "email": { "type": ["null", "string"] }, "enabled": { "type": ["null", "boolean"] }, - "id": { "type": ["null", "integer"] }, + "id": { "type": ["null", "string"] }, "is_admin": { "type": ["null", "boolean"] }, "is_guest": { "type": ["null", "boolean"] }, "is_pending": { "type": ["null", "boolean"] }, diff --git a/airbyte-integrations/connectors/source-monday/source_monday/schemas/workspaces.json b/airbyte-integrations/connectors/source-monday/source_monday/schemas/workspaces.json index af7bf79b2d97..3f8439055873 100644 --- a/airbyte-integrations/connectors/source-monday/source_monday/schemas/workspaces.json +++ b/airbyte-integrations/connectors/source-monday/source_monday/schemas/workspaces.json @@ -4,14 +4,14 @@ "properties": { "created_at": { "type": ["null", "string"], "format": "date-time" }, "description": { "type": ["null", "string"] }, - "id": { "type": ["null", "integer"] }, + "id": { "type": ["null", "string"] }, "kind": { "type": ["null", "string"] }, "name": { "type": ["null", "string"] }, "state": { "type": ["null", "string"] }, "account_product": { "type": ["null", "object"], "properties": { - "id": { "type": ["null", "integer"] }, + "id": { "type": ["null", "string"] }, "kind": { "type": ["null", "string"] } } }, @@ -21,7 +21,7 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "id": { "type": ["null", "integer"] } + "id": { "type": ["null", "string"] } } } }, @@ -43,7 +43,7 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "id": { "type": ["null", "integer"] }, + "id": { "type": ["null", "string"] }, "name": { "type": ["null", "string"] } } } @@ -54,7 +54,7 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "id": { "type": ["null", "integer"] }, + "id": { "type": ["null", "string"] }, "name": { "type": ["null", "string"] } } } @@ -65,7 +65,7 @@ "type": ["null", "object"], "additionalProperties": true, "properties": { - "id": { "type": ["null", "integer"] } + "id": { "type": ["null", "string"] } } } } diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/test_components.py b/airbyte-integrations/connectors/source-monday/unit_tests/test_components.py index 8b6c19fb6617..670aff5e4e44 100644 --- a/airbyte-integrations/connectors/source-monday/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-monday/unit_tests/test_components.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock, Mock import pytest +from airbyte_cdk.models import AirbyteMessage, SyncMode, Type from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig from airbyte_cdk.sources.streams import Stream from requests import Response @@ -34,9 +35,9 @@ def test_slicer(): "last_record, expected, records", [ ( - {"first_stream_cursor": 1662459010}, - {'parent_stream_name': {'parent_cursor_field': 1662459010}, 'first_stream_cursor': 1662459010}, - [{"first_stream_cursor": 1662459010}], + {"first_stream_cursor": 1662459010}, + {"parent_stream_name": {"parent_cursor_field": 1662459010}, "first_stream_cursor": 1662459010}, + [{"first_stream_cursor": 1662459010}], ), (None, {}, []), ], @@ -71,8 +72,8 @@ def test_null_records(caplog): parameters={}, ) content = { - "data": - {"boards": [ + "data": { + "boards": [ {"board_kind": "private", "id": "1234561", "updated_at": "2023-08-15T10:30:49Z"}, {"board_kind": "private", "id": "1234562", "updated_at": "2023-08-15T10:30:50Z"}, {"board_kind": "private", "id": "1234563", "updated_at": "2023-08-15T10:30:51Z"}, @@ -80,10 +81,12 @@ def test_null_records(caplog): {"board_kind": "private", "id": "1234565", "updated_at": "2023-08-15T10:30:43Z"}, {"board_kind": "private", "id": "1234566", "updated_at": "2023-08-15T10:30:54Z"}, None, - None - ]}, + None, + ] + }, "errors": [{"message": "Cannot return null for non-nullable field Board.creator"}], - "account_id": 123456} + "account_id": 123456, + } response = _create_response(content) records = extractor.extract_records(response) warning_message = "Record with null value received; errors: [{'message': 'Cannot return null for non-nullable field Board.creator'}]" @@ -94,6 +97,90 @@ def test_null_records(caplog): {"board_kind": "private", "id": "1234563", "updated_at": "2023-08-15T10:30:51Z", "updated_at_int": 1692095451}, {"board_kind": "private", "id": "1234564", "updated_at": "2023-08-15T10:30:52Z", "updated_at_int": 1692095452}, {"board_kind": "private", "id": "1234565", "updated_at": "2023-08-15T10:30:43Z", "updated_at_int": 1692095443}, - {"board_kind": "private", "id": "1234566", "updated_at": "2023-08-15T10:30:54Z", "updated_at_int": 1692095454} + {"board_kind": "private", "id": "1234566", "updated_at": "2023-08-15T10:30:54Z", "updated_at_int": 1692095454}, ] assert records == expected_records + + +@pytest.fixture +def mock_parent_stream(): + + def mock_parent_stream_slices(*args, **kwargs): + return iter([{"ids": [123]}]) + + mock_stream = MagicMock(spec=Stream) + mock_stream.primary_key = "id" # Example primary key + mock_stream.stream_slices = mock_parent_stream_slices + mock_stream.parent_config = ParentStreamConfig( + stream=mock_stream, + parent_key="id", + partition_field="parent_stream_id", + parameters={}, + config={}, + ) + + return mock_stream + +@pytest.mark.parametrize("stream_state, parent_records, expected_slices", + [ + ({}, [], [{}]), + ( + {"updated_at": "2022-01-01T00:00:00Z"}, + [AirbyteMessage( + type=Type.RECORD, + record={ "data": {"id": 123, "name": "Sample Record", "updated_at": "2023-01-01T00:00:00Z"}, "stream": "projects", "emitted_at": 1632095449} + )], + [{'parent_stream_id': [123]}] + ), + ( + {"updated_at": "2022-01-01T00:00:00Z"}, + AirbyteMessage(type=Type.LOG), + [] + ) + ], + ids=[ + "no stream state", + "successfully read parent record", + "skip non_record AirbyteMessage" + ] +) +def test_read_parent_stream(mock_parent_stream, stream_state, parent_records, expected_slices): + + slicer = IncrementalSubstreamSlicer( + config={}, + parameters={}, + cursor_field="updated_at", + parent_stream_configs=[mock_parent_stream.parent_config], + nested_items_per_page=10 + ) + + mock_parent_stream.read_records = MagicMock(return_value=parent_records) + slicer.parent_cursor_field = "updated_at" + + slices = list(slicer.read_parent_stream( + sync_mode=SyncMode.full_refresh, + cursor_field="updated_at", + stream_state=stream_state + )) + + assert slices == expected_slices + + +def test_set_initial_state(): + + slicer = IncrementalSubstreamSlicer( + config={}, + parameters={}, + cursor_field="updated_at_int", + parent_stream_configs=[MagicMock(parent_stream_name="parent_stream")], + nested_items_per_page=10 + ) + + initial_stream_state = { + "updated_at_int": 1662459010, + "parent_stream": {"parent_cursor_field": 1662459011} + } + + expected_state = { "updated_at_int": 1662459010 } + slicer.set_initial_state(initial_stream_state) + assert slicer._state == expected_state diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/test_extractor.py b/airbyte-integrations/connectors/source-monday/unit_tests/test_extractor.py new file mode 100644 index 000000000000..869c7ab7bbca --- /dev/null +++ b/airbyte-integrations/connectors/source-monday/unit_tests/test_extractor.py @@ -0,0 +1,63 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from source_monday.extractor import MondayActivityExtractor, MondayIncrementalItemsExtractor + + +def test_extract_records(): + # Mock the response + response = MagicMock() + response_body = { + "data": { + "boards": [ + { + "activity_logs": [ + { + "data": "{\"pulse_id\": 123}", + "entity": "pulse", + "created_at": "16367386880000000" + } + ] + } + ] + } + } + + response.json.return_value = response_body + extractor = MondayActivityExtractor(parameters={}) + records = extractor.extract_records(response) + + # Assertions + assert len(records) == 1 + assert records[0]["pulse_id"] == 123 + assert records[0]["created_at_int"] == 1636738688 + + +def test_extract_records_incremental(): + # Mock the response + response = MagicMock() + response_body = { + "data": { + "boards": [ + { + "id": 1 + } + ] + } + } + + response.json.return_value = response_body + extractor = MondayIncrementalItemsExtractor( + parameters={}, + field_path=["data", "ccccc"], + config=MagicMock(), + field_path_pagination=["data", "bbbb"], + field_path_incremental=["data", "boards", "*"] + ) + records = extractor.extract_records(response) + + # Assertions + assert records == [{'id': 1}] diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py b/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py index ed79a1bce6bb..b4f46146b6bc 100644 --- a/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py +++ b/airbyte-integrations/connectors/source-monday/unit_tests/test_graphql_requester.py @@ -5,22 +5,17 @@ from unittest.mock import MagicMock import pytest +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod +from airbyte_cdk.sources.declarative.schema.json_file_schema_loader import JsonFileSchemaLoader from source_monday import MondayGraphqlRequester nested_object_schema = { "root": { "type": ["null", "array"], - "properties": { - "nested": { - "type": ["null", "object"], - "properties": { - "nested_of_nested": {"type": ["null", "string"]} - } - } - } + "properties": {"nested": {"type": ["null", "object"], "properties": {"nested_of_nested": {"type": ["null", "string"]}}}}, }, - "sibling": {"type": ["null", "string"]} + "sibling": {"type": ["null", "string"]}, } nested_array_schema = { @@ -28,17 +23,10 @@ "type": ["null", "array"], "items": { "type": ["null", "array"], - "properties": { - "nested": { - "type": ["null", "object"], - "properties": { - "nested_of_nested": {"type": ["null", "string"]} - } - } - } - } + "properties": {"nested": {"type": ["null", "object"], "properties": {"nested_of_nested": {"type": ["null", "string"]}}}}, + }, }, - "sibling": {"type": ["null", "string"]} + "sibling": {"type": ["null", "string"]}, } @@ -51,7 +39,7 @@ {}, {"query": "query{test_stream(limit:100,page:2){root{nested{nested_of_nested}},sibling}}"}, {"next_page_token": 2}, - id="test_get_request_params_produces_graphql_query_for_object_items" + id="test_get_request_params_produces_graphql_query_for_object_items", ), pytest.param( nested_array_schema, @@ -59,33 +47,33 @@ {}, {"query": "query{test_stream(limit:100,page:2){root{nested{nested_of_nested}},sibling}}"}, {"next_page_token": 2}, - id="test_get_request_params_produces_graphql_query_for_array_items" + id="test_get_request_params_produces_graphql_query_for_array_items", ), pytest.param( nested_array_schema, "items", {}, - {"query": "query{boards(limit:100,page:2){items(limit:100,page:1){root{nested{nested_of_nested}},sibling}}}"}, - {"next_page_token": (2, 1)}, - id="test_get_request_params_produces_graphql_query_for_items_stream" + {"query": 'query{next_items_page(limit:100,cursor:"cursor_bla"){cursor,items{root{nested{nested_of_nested}},sibling}}}'}, + {"next_page_token": (2, "cursor_bla")}, + id="test_get_request_params_produces_graphql_query_for_items_stream", ), pytest.param( nested_array_schema, "teams", {"teams_limit": 100}, - {'query': 'query{teams(limit:100,page:2){id,name,picture_url,users(limit:100){id}}}'}, + {"query": "query{teams(limit:100,page:2){id,name,picture_url,users(limit:100){id}}}"}, {"next_page_token": 2}, - id="test_get_request_params_produces_graphql_query_for_teams_optimized_stream" + id="test_get_request_params_produces_graphql_query_for_teams_optimized_stream", ), pytest.param( nested_array_schema, "teams", {}, - {'query': 'query{teams(limit:100,page:2){root{nested{nested_of_nested}},sibling}}'}, + {"query": "query{teams(limit:100,page:2){root{nested{nested_of_nested}},sibling}}"}, {"next_page_token": 2}, - id="test_get_request_params_produces_graphql_query_for_teams_stream" - ) - ] + id="test_get_request_params_produces_graphql_query_for_teams_stream", + ), + ], ) def test_get_request_params(mocker, input_schema, graphql_query, stream_name, config, next_page_token): mocker.patch.object(MondayGraphqlRequester, "_get_schema_root_properties", return_value=input_schema) @@ -100,10 +88,72 @@ def test_get_request_params(mocker, input_schema, graphql_query, stream_name, co limit="{{ parameters['items_per_page'] }}", nested_limit="{{ parameters.get('nested_items_per_page', 1) }}", parameters={"name": stream_name, "items_per_page": 100, "nested_items_per_page": 100}, - config=config + config=config, + ) + assert requester.get_request_params(stream_state={}, stream_slice={}, next_page_token=next_page_token) == graphql_query + + +@pytest.fixture +def monday_requester(): + return MondayGraphqlRequester( + name="a name", + url_base="https://api.monday.com/v2", + path="a-path", + config={}, + parameters={"name": "activity_logs"}, + limit=InterpolatedString.create("100", parameters={"name": "activity_logs"}), + nested_limit=InterpolatedString.create("100", parameters={"name": "activity_logs"}), ) - assert requester.get_request_params( - stream_state={}, - stream_slice={}, - next_page_token=next_page_token - ) == graphql_query + +def test_get_schema_root_properties(mocker, monday_requester): + mock_schema = { + "properties": { + "updated_at_int": {"type": "integer"}, + "created_at_int": {"type": "integer"}, + "pulse_id": {"type": "integer"}, + "board_id": {"type": "integer"}, + "other_field": {"type": "string"}, + "yet_another_field": {"type": "boolean"} + } + } + + mocker.patch.object(JsonFileSchemaLoader, 'get_json_schema', return_value=mock_schema) + requester = monday_requester + result_schema = requester._get_schema_root_properties() + + assert result_schema == { + "other_field": { "type": "string" }, + "yet_another_field": { "type": "boolean" } + } + + +def test_build_activity_query(mocker, monday_requester): + + mock_stream_state = { "updated_at_int": 1636738688 } + object_arguments = { "stream_state": mock_stream_state } + mocker.patch.object(MondayGraphqlRequester, '_get_object_arguments', return_value="stream_state:{{ stream_state['updated_at_int'] }}") + requester = monday_requester + + result = requester._build_activity_query(object_name="activity_logs", field_schema={}, sub_page=None, **object_arguments) + assert result == "boards(stream_state:{{ stream_state['updated_at_int'] }}){activity_logs(stream_state:{{ stream_state['updated_at_int'] }}){}}" + + +def test_build_items_incremental_query(monday_requester): + + object_name = "test_items" + field_schema = { + "id": {"type": "integer"}, + "name": {"type": "string"}, + } + stream_slice = {"ids": [1, 2, 3]} + + built_query = monday_requester._build_items_incremental_query(object_name, field_schema, stream_slice) + + assert built_query == 'items(limit:100,ids:[1, 2, 3]){id,name}' + + +def test_get_request_headers(monday_requester): + + headers = monday_requester.get_request_headers() + + assert headers == {'API-Version': '2024-01'} diff --git a/airbyte-integrations/connectors/source-monday/unit_tests/test_item_pagination_strategy.py b/airbyte-integrations/connectors/source-monday/unit_tests/test_item_pagination_strategy.py index b2ba067e4a56..979c72284713 100644 --- a/airbyte-integrations/connectors/source-monday/unit_tests/test_item_pagination_strategy.py +++ b/airbyte-integrations/connectors/source-monday/unit_tests/test_item_pagination_strategy.py @@ -5,7 +5,7 @@ from unittest.mock import MagicMock import pytest -from source_monday.item_pagination_strategy import ItemPaginationStrategy +from source_monday.item_pagination_strategy import ItemCursorPaginationStrategy, ItemPaginationStrategy @pytest.mark.parametrize( @@ -28,8 +28,8 @@ [], None, id="test_end_pagination", - ) - ] + ), + ], ) def test_item_pagination_strategy(response_json, last_records, expected): strategy = ItemPaginationStrategy( @@ -40,3 +40,42 @@ def test_item_pagination_strategy(response_json, last_records, expected): response.json.return_value = response_json assert strategy.next_page_token(response, last_records) == expected + +@pytest.mark.parametrize( + ("response_json", "last_records", "expected"), + [ + pytest.param( + {"data": {"boards": [{"items_page": {"cursor": "bla", "items":[{"id": "1"}]}}]}}, + [], + (1, 'bla'), + id="test_cursor_in_first_request", + ), + pytest.param( + {"data": {"next_items_page": {"cursor": "bla2", "items":[{"id": "1"}]}}}, + [], + (1, 'bla2'), + id="test_cursor_in_next_page", + ), + pytest.param( + {"data": {"next_items_page": {"items": [{"id": "1"}]}}}, + [], + (2, None), + id="test_next_board_page", + ), + pytest.param( + {"data": {"boards": []}}, + [], + None, + id="test_end_pagination", + ), + ], +) +def test_item_cursor_pagination_strategy(response_json, last_records, expected): + strategy = ItemCursorPaginationStrategy( + page_size=1, + parameters={"items_per_page": 1}, + ) + response = MagicMock() + response.json.return_value = response_json + + assert strategy.next_page_token(response, last_records) == expected diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/.dockerignore b/airbyte-integrations/connectors/source-mongodb-internal-poc/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/Dockerfile b/airbyte-integrations/connectors/source-mongodb-internal-poc/Dockerfile deleted file mode 100644 index 7b1aec14b1eb..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-mongodb-internal-poc - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-mongodb-internal-poc - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.0.1 -LABEL io.airbyte.name=airbyte/source-mongodb-internal-poc diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/README.md b/airbyte-integrations/connectors/source-mongodb-internal-poc/README.md deleted file mode 100644 index 8ec72f9f4466..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/README.md +++ /dev/null @@ -1,60 +0,0 @@ -# MongoDb Source (Internal POC) - -## Documentation -This is the repository for the MongoDb source connector in Java. -For information about how to use this connector within Airbyte, see [User Documentation](https://docs.airbyte.io/integrations/sources/mongodb-internal-poc) - -## Local development - -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-mongodb-internal-poc:build -``` - -### Locally running the connector docker image - -#### Build -Build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-mongodb-internal-poc:airbyteDocker -``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. - -## Testing -We use `JUnit` for Java tests. - -### Test Configuration - -No specific configuration needed for testing Standalone MongoDb instance, MongoDb Test Container is used. -In order to test the MongoDb Atlas or Replica set, you need to provide configuration parameters. - -## Community Contributor - -As a community contributor, you will need to have an Atlas cluster to test MongoDb source. - -1. Create `secrets/credentials.json` file - 1. Insert below json to the file with your configuration - ``` - { - "database": "database_name", - "user": "username", - "password": "password", - "connection_string": "mongodb+srv://cluster0.abcd1.mongodb.net/", - "replica_set": "atlas-abcdefg-shard-0", - "auth_source": "auth_database" - } - ``` - -## Airbyte Employee - -1. Access the `MONGODB_TEST_CREDS` secret on LastPass -1. Create a file with the contents at `secrets/credentials.json` - - -#### Acceptance Tests -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-mongodb-internal-poc:integrationTest -``` diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mongodb-internal-poc/acceptance-test-config.yml deleted file mode 100644 index 5db8fa612d87..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/acceptance-test-config.yml +++ /dev/null @@ -1,38 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-mongodb-internal-poc:dev -acceptance_tests: - spec: - tests: - - spec_path: "integration_tests/expected_spec.json" - backward_compatibility_tests_config: - disable_for_version: "0.0.1" - config_path: "secrets/credentials.json" - timeout_seconds: 60 - connection: - tests: - - config_path: "secrets/credentials.json" - status: "succeed" - timeout_seconds: 60 - discovery: - bypass_reason: "The first version of this connector returns null when discovery is called, which causes the test to fail. We can stop bypassing this test once a new version of the connector is released." - #TODO: remove bypass_reason once a version that supports discovery is released + uncomment the lines below -# tests: -# - config_path: "secrets/credentials.json" -# backward_compatibility_tests_config: -# disable_for_version: "0.0.1" -# timeout_seconds: 60 - basic_read: - bypass_reason: "Full refresh syncs are not supported on this connector." - full_refresh: - bypass_reason: "Full refresh syncs are not supported on this connector." - incremental: - bypass_reason: "Incremental syncs are not yet supported by this connector." - #TODO: remove bypass_reason once a version that supports incremental syncs is released + uncomment the lines below -# tests: -# - config_path: "secrets/credentials.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# cursor_paths: -# listingsAndReviews: ["id"] -# timeout_seconds: 180 - diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-mongodb-internal-poc/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/build.gradle b/airbyte-integrations/connectors/source-mongodb-internal-poc/build.gradle deleted file mode 100644 index 29bc652075ba..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/build.gradle +++ /dev/null @@ -1,68 +0,0 @@ -plugins { - id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' - id 'airbyte-connector-acceptance-test' - id 'org.jetbrains.kotlin.jvm' version '1.9.0' -} - -application { - mainClass = 'io.airbyte.integrations.source.mongodb.internal.MongoDbSource' - applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] -} - -dependencies { - implementation libs.slf4j.api - implementation libs.jackson.databind - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:bases:debezium') - implementation libs.airbyte.protocol - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - - implementation 'org.mongodb:mongodb-driver-sync:4.10.2' - - testImplementation testFixtures(project(':airbyte-integrations:bases:debezium')) - testImplementation "org.jetbrains.kotlinx:kotlinx-cli:0.3.5" - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-mongodb-internal-poc') - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) -} - -/* - * Executes the script that generates test data and inserts it into the provided database/collection. - * - * To execute this task, use the following command: - * - * ./gradlew :airbyte-integrations:connectors:source-mongodb-internal-poc:generateTestData -PconnectionString= -PreplicaSet= -PdatabaseName= -PcollectionName= -Pusername= - * - * Optionally, you can provide -PnumberOfDocuments to change the number of generated documents from the default (10,000). - */ -tasks.register('generateTestData', JavaExec) { - def arguments = [] - - if(project.hasProperty('collectionName')) { - arguments.addAll(['--collection-name', collectionName]) - } - if(project.hasProperty('connectionString')) { - arguments.addAll(['--connection-string', connectionString]) - } - if(project.hasProperty('databaseName')) { - arguments.addAll(['--database-name', databaseName]) - } - if (project.hasProperty('numberOfDocuments')) { - arguments.addAll(['--number', numberOfDocuments]) - } - if(project.hasProperty('replicaSet')) { - arguments.addAll(['--replica-set', replicaSet]) - } - if(project.hasProperty('username')) { - arguments.addAll(['--username', username]) - } - - classpath = sourceSets.test.runtimeClasspath - main 'io.airbyte.integrations.source.mongodb.internal.MongoDbInsertClient' - standardInput = System.in - args arguments -} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/icon.svg b/airbyte-integrations/connectors/source-mongodb-internal-poc/icon.svg deleted file mode 100644 index 66b68e75556d..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/icon.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-mongodb-internal-poc/integration_tests/configured_catalog.json deleted file mode 100644 index 32f5f5691b81..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/integration_tests/configured_catalog.json +++ /dev/null @@ -1,281 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "listingsAndReviews", - "json_schema": { - "type": "object", - "properties": { - "amenities": { - "type": "array" - }, - "notes": { - "type": "string" - }, - "access": { - "type": "string" - }, - "house_rules": { - "type": "string" - }, - "first_review": { - "type": "string" - }, - "calendar_last_scraped": { - "type": "string" - }, - "description": { - "type": "string" - }, - "neighborhood_overview": { - "type": "string" - }, - "_id_aibyte_transform": { - "type": "string" - }, - "availability": { - "type": "object", - "properties": { - "availability_365": { - "type": "number" - }, - "availability_30": { - "type": "number" - }, - "availability_60": { - "type": "number" - }, - "availability_90": { - "type": "number" - } - } - }, - "number_of_reviews": { - "type": "number" - }, - "space": { - "type": "string" - }, - "review_scores": { - "type": "object", - "properties": { - "review_scores_checkin": { - "type": "number" - }, - "review_scores_communication": { - "type": "number" - }, - "review_scores_rating": { - "type": "number" - }, - "review_scores_accuracy": { - "type": "number" - }, - "review_scores_location": { - "type": "number" - }, - "review_scores_value": { - "type": "number" - }, - "review_scores_cleanliness": { - "type": "number" - } - } - }, - "cleaning_fee": { - "type": "number" - }, - "reviews": { - "type": "array" - }, - "price": { - "type": "number" - }, - "reviews_per_month": { - "type": "number" - }, - "host": { - "type": "object", - "properties": { - "host_verifications": { - "type": "array" - }, - "host_url": { - "type": "string" - }, - "host_response_time": { - "type": "string" - }, - "host_has_profile_pic": { - "type": "boolean" - }, - "host_about": { - "type": "string" - }, - "host_picture_url": { - "type": "string" - }, - "host_id": { - "type": "string" - }, - "host_listings_count": { - "type": "number" - }, - "host_total_listings_count": { - "type": "number" - }, - "host_location": { - "type": "string" - }, - "host_is_superhost": { - "type": "boolean" - }, - "host_neighbourhood": { - "type": "string" - }, - "host_thumbnail_url": { - "type": "string" - }, - "host_response_rate": { - "type": "number" - }, - "host_name": { - "type": "string" - }, - "host_identity_verified": { - "type": "boolean" - } - } - }, - "property_type": { - "type": "string" - }, - "summary": { - "type": "string" - }, - "monthly_price": { - "type": "number" - }, - "security_deposit": { - "type": "number" - }, - "images": { - "type": "object", - "properties": { - "picture_url": { - "type": "string" - }, - "xl_picture_url": { - "type": "string" - }, - "medium_url": { - "type": "string" - }, - "thumbnail_url": { - "type": "string" - } - } - }, - "address": { - "type": "object", - "properties": { - "market": { - "type": "string" - }, - "country_code": { - "type": "string" - }, - "country": { - "type": "string" - }, - "street": { - "type": "string" - }, - "suburb": { - "type": "string" - }, - "location": { - "type": "object", - "properties": { - "coordinates": { - "type": "array" - }, - "type": { - "type": "string" - }, - "is_location_exact": { - "type": "boolean" - } - } - }, - "government_area": { - "type": "string" - } - } - }, - "weekly_price": { - "type": "number" - }, - "bed_type": { - "type": "string" - }, - "listing_url": { - "type": "string" - }, - "guests_included": { - "type": "number" - }, - "maximum_nights": { - "type": "string" - }, - "bathrooms": { - "type": "number" - }, - "extra_people": { - "type": "number" - }, - "bedrooms": { - "type": "number" - }, - "minimum_nights": { - "type": "string" - }, - "last_review": { - "type": "string" - }, - "transit": { - "type": "string" - }, - "accommodates": { - "type": "number" - }, - "interaction": { - "type": "string" - }, - "name": { - "type": "string" - }, - "cancellation_policy": { - "type": "string" - }, - "beds": { - "type": "number" - }, - "last_scraped": { - "type": "string" - }, - "room_type": { - "type": "string" - } - } - }, - "supported_sync_modes": ["incremental"], - "default_cursor_field": [], - "source_defined_primary_key": [], - "namespace": "sample_airbnb" - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/integration_tests/expected_spec.json b/airbyte-integrations/connectors/source-mongodb-internal-poc/integration_tests/expected_spec.json deleted file mode 100644 index 4ad8a5adc2f3..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/integration_tests/expected_spec.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/mongodb-internal-poc", - "changelogUrl": "https://docs.airbyte.com/integrations/sources/mongodb-internal-poc", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MongoDb Source Spec", - "type": "object", - "required": ["connection_string", "database", "replica_set"], - "additionalProperties": true, - "properties": { - "connection_string": { - "title": "Connection String", - "type": "string", - "description": "The connection string of the database that you want to replicate..", - "examples": [ - "mongodb+srv://example.mongodb.net/", - "mongodb://example1.host.com:27017,example2.host.com:27017,example3.host.com:27017/", - "mongodb://example.host.com:27017/" - ], - "order": 1 - }, - "database": { - "title": "Database Name", - "type": "string", - "description": "The database you want to replicate.", - "order": 2 - }, - "user": { - "title": "User", - "type": "string", - "description": "The username which is used to access the database.", - "order": 3 - }, - "password": { - "title": "Password", - "type": "string", - "description": "The password associated with this username.", - "airbyte_secret": true, - "order": 4 - }, - "auth_source": { - "title": "Authentication Source", - "type": "string", - "description": "The authentication source where the user information is stored.", - "default": "admin", - "examples": ["admin"], - "order": 5 - }, - "replica_set": { - "title": "Replica Set", - "type": "string", - "description": "The name of the replica set to be replicated.", - "order": 6 - } - } - }, - "supported_destination_sync_modes": [] -} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/metadata.yaml b/airbyte-integrations/connectors/source-mongodb-internal-poc/metadata.yaml deleted file mode 100644 index fdcc68d8f2ab..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -data: - connectorSubtype: database - connectorType: source - definitionId: 5ac5a7e5-43f5-4e7a-bf53-70961b0307bc - dockerImageTag: 0.0.1 - dockerRepository: airbyte/source-mongodb-internal-poc - githubIssueLabel: source-mongodb-internal-poc - icon: mongodb.svg - license: ELv2 - name: MongoDb POC - registries: - cloud: - enabled: true - oss: - enabled: true - releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/mongodb-v2 - tags: - - language:java -metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoCatalogHelper.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoCatalogHelper.java deleted file mode 100644 index de8626fd4b41..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoCatalogHelper.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mongodb.internal; - -import io.airbyte.integrations.debezium.internals.DebeziumEventUtils; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.AirbyteCatalog; -import io.airbyte.protocol.models.v0.AirbyteStream; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.SyncMode; -import java.util.ArrayList; -import java.util.List; - -/** - * Collection of utility methods for generating the {@link AirbyteCatalog}. - */ -public class MongoCatalogHelper { - - /** - * The default cursor field name. - */ - public static final String DEFAULT_CURSOR_FIELD = "_id"; - - /** - * The list of supported sync modes for a given stream. - */ - public static final List SUPPORTED_SYNC_MODES = List.of(SyncMode.INCREMENTAL); - - /** - * Builds an {@link AirbyteStream} with the correct configuration for this source. - * - * @param streamName The name of the stream. - * @param streamNamespace The namespace of the stream. - * @param fields The fields associated with the stream. - * @return The configured {@link AirbyteStream} for this source. - */ - public static AirbyteStream buildAirbyteStream(final String streamName, final String streamNamespace, final List fields) { - return CatalogHelpers.createAirbyteStream(streamName, streamNamespace, addCdcMetadataColumns(fields)) - .withSupportedSyncModes(SUPPORTED_SYNC_MODES) - .withSourceDefinedCursor(true) - .withDefaultCursorField(List.of(DEFAULT_CURSOR_FIELD)) - .withSourceDefinedPrimaryKey(List.of(List.of(DEFAULT_CURSOR_FIELD))); - } - - /** - * Adds the metadata columns required to use CDC to the list of discovered fields. - * - * @param fields The list of discovered fields. - * @return The modified list of discovered fields that includes the required CDC metadata columns. - */ - public static List addCdcMetadataColumns(final List fields) { - final List modifiedFields = new ArrayList<>(fields); - modifiedFields.add(new Field(DebeziumEventUtils.CDC_LSN, JsonSchemaType.NUMBER)); - modifiedFields.add(new Field(DebeziumEventUtils.CDC_UPDATED_AT, JsonSchemaType.STRING)); - modifiedFields.add(new Field(DebeziumEventUtils.CDC_DELETED_AT, JsonSchemaType.STRING)); - return modifiedFields; - } - -} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoConnectionUtils.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoConnectionUtils.java deleted file mode 100644 index b5f387e3a712..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoConnectionUtils.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mongodb.internal; - -import static io.airbyte.integrations.source.mongodb.internal.MongoConstants.AUTH_SOURCE_CONFIGURATION_KEY; -import static io.airbyte.integrations.source.mongodb.internal.MongoConstants.CONNECTION_STRING_CONFIGURATION_KEY; -import static io.airbyte.integrations.source.mongodb.internal.MongoConstants.PASSWORD_CONFIGURATION_KEY; -import static io.airbyte.integrations.source.mongodb.internal.MongoConstants.REPLICA_SET_CONFIGURATION_KEY; -import static io.airbyte.integrations.source.mongodb.internal.MongoConstants.USER_CONFIGURATION_KEY; - -import com.fasterxml.jackson.databind.JsonNode; -import com.mongodb.ConnectionString; -import com.mongodb.MongoClientSettings; -import com.mongodb.MongoCredential; -import com.mongodb.MongoDriverInformation; -import com.mongodb.ReadPreference; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; - -/** - * Helper utility for building a {@link MongoClient}. - */ -public class MongoConnectionUtils { - - /** - * Creates a new {@link MongoClient} from the source configuration. - * - * @param config The source's configuration. - * @return The configured {@link MongoClient}. - */ - public static MongoClient createMongoClient(final JsonNode config) { - final String authSource = config.get(AUTH_SOURCE_CONFIGURATION_KEY).asText(); - - final ConnectionString mongoConnectionString = new ConnectionString(buildConnectionString(config)); - - final MongoDriverInformation mongoDriverInformation = MongoDriverInformation.builder() - .driverName("Airbyte") - .build(); - - final MongoClientSettings.Builder mongoClientSettingsBuilder = MongoClientSettings.builder() - .applyConnectionString(mongoConnectionString) - .readPreference(ReadPreference.secondaryPreferred()); - - if (config.has(USER_CONFIGURATION_KEY) && config.has(PASSWORD_CONFIGURATION_KEY)) { - final String user = config.get(USER_CONFIGURATION_KEY).asText(); - final String password = config.get(PASSWORD_CONFIGURATION_KEY).asText(); - mongoClientSettingsBuilder.credential(MongoCredential.createCredential(user, authSource, password.toCharArray())); - } - - return MongoClients.create(mongoClientSettingsBuilder.build(), mongoDriverInformation); - } - - private static String buildConnectionString(final JsonNode config) { - final String connectionString = config.get(CONNECTION_STRING_CONFIGURATION_KEY).asText(); - final String replicaSet = config.get(REPLICA_SET_CONFIGURATION_KEY).asText(); - final StringBuilder builder = new StringBuilder(); - builder.append(connectionString); - builder.append("?replicaSet="); - builder.append(replicaSet); - builder.append("&retryWrites=false"); - builder.append("&provider=airbyte"); - builder.append("&tls=true"); - return builder.toString(); - } - -} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoConstants.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoConstants.java deleted file mode 100644 index fd0e1f485407..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoConstants.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mongodb.internal; - -public class MongoConstants { - - public static final String AUTH_SOURCE_CONFIGURATION_KEY = "auth_source"; - public static final Integer CHECKPOINT_INTERVAL = 1000; - public static final String CONNECTION_STRING_CONFIGURATION_KEY = "connection_string"; - public static final String DATABASE_CONFIGURATION_KEY = "database"; - public static final String ID_FIELD = "_id"; - public static final String PASSWORD_CONFIGURATION_KEY = "password"; - public static final String REPLICA_SET_CONFIGURATION_KEY = "replica_set"; - public static final String USER_CONFIGURATION_KEY = "user"; - - private MongoConstants() {} - -} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoDbSource.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoDbSource.java deleted file mode 100644 index 47f371d539c8..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoDbSource.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mongodb.internal; - -import static io.airbyte.integrations.source.mongodb.internal.MongoConstants.CHECKPOINT_INTERVAL; -import static io.airbyte.integrations.source.mongodb.internal.MongoConstants.DATABASE_CONFIGURATION_KEY; -import static io.airbyte.integrations.source.mongodb.internal.MongoConstants.ID_FIELD; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.annotations.VisibleForTesting; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoDatabase; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Projections; -import com.mongodb.client.model.Sorts; -import com.mongodb.connection.ClusterType; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.util.AutoCloseableIterator; -import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteTraceMessageUtility; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.protocol.models.v0.AirbyteCatalog; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteStateMessage; -import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; -import io.airbyte.protocol.models.v0.AirbyteStream; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.StreamDescriptor; -import io.airbyte.protocol.models.v0.SyncMode; -import java.time.Instant; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Optional; -import java.util.stream.Collectors; -import org.bson.BsonDocument; -import org.bson.conversions.Bson; -import org.bson.types.ObjectId; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class MongoDbSource extends BaseConnector implements Source { - - private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbSource.class); - - /** Helper class for holding a collection-name and stream state together */ - private record CollectionNameState(Optional name, Optional state) {} - - public static void main(final String[] args) throws Exception { - final Source source = new MongoDbSource(); - LOGGER.info("starting source: {}", MongoDbSource.class); - new IntegrationRunner(source).run(args); - LOGGER.info("completed source: {}", MongoDbSource.class); - } - - @Override - public AirbyteConnectionStatus check(final JsonNode config) { - try (final MongoClient mongoClient = createMongoClient(config)) { - final String databaseName = config.get(DATABASE_CONFIGURATION_KEY).asText(); - - /* - * Perform the authorized collections check before the cluster type check. The MongoDB Java driver - * needs to actually execute a command in order to fetch the cluster description. Querying for the - * authorized collections guarantees that the cluster description will be available to the driver. - */ - if (MongoUtil.getAuthorizedCollections(mongoClient, databaseName).isEmpty()) { - return new AirbyteConnectionStatus() - .withMessage("Target MongoDB database does not contain any authorized collections.") - .withStatus(AirbyteConnectionStatus.Status.FAILED); - } - if (!ClusterType.REPLICA_SET.equals(mongoClient.getClusterDescription().getType())) { - return new AirbyteConnectionStatus() - .withMessage("Target MongoDB instance is not a replica set cluster.") - .withStatus(AirbyteConnectionStatus.Status.FAILED); - } - } catch (final Exception e) { - LOGGER.error("Unable to perform source check operation.", e); - return new AirbyteConnectionStatus() - .withMessage(e.getMessage()) - .withStatus(AirbyteConnectionStatus.Status.FAILED); - } - - LOGGER.info("The source passed the check operation test!"); - return new AirbyteConnectionStatus().withStatus(AirbyteConnectionStatus.Status.SUCCEEDED); - } - - @Override - public AirbyteCatalog discover(final JsonNode config) { - try (final MongoClient mongoClient = createMongoClient(config)) { - final String databaseName = config.get(DATABASE_CONFIGURATION_KEY).asText(); - final List streams = MongoUtil.getAirbyteStreams(mongoClient, databaseName); - return new AirbyteCatalog().withStreams(streams); - } - } - - @Override - public AutoCloseableIterator read(final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final JsonNode state) { - final var databaseName = config.get(DATABASE_CONFIGURATION_KEY).asText(); - final var emittedAt = Instant.now(); - - final var states = convertState(state); - final MongoClient mongoClient = MongoConnectionUtils.createMongoClient(config); - - try { - final var database = mongoClient.getDatabase(databaseName); - // TODO treat INCREMENTAL and FULL_REFRESH differently? - return AutoCloseableIterators.appendOnClose(AutoCloseableIterators.concatWithEagerClose( - convertCatalogToIterators(catalog, states, database, emittedAt), - AirbyteTraceMessageUtility::emitStreamStatusTrace), - mongoClient::close); - } catch (final Exception e) { - mongoClient.close(); - throw e; - } - } - - /** - * Converts the JsonNode into a map of mongodb collection names to stream states. - */ - @VisibleForTesting - protected Map convertState(final JsonNode state) { - // I'm unsure if the JsonNode data is going to be a singular AirbyteStateMessage or an array of - // AirbyteStateMessages. - // So this currently handles both cases, converting the singular message into a list of messages, - // leaving the list of messages - // as a list of messages, or returning an empty list. - final List states = Jsons.tryObject(state, AirbyteStateMessage.class) - .map(List::of) - .orElseGet(() -> Jsons.tryObject(state, AirbyteStateMessage[].class) - .map(Arrays::asList) - .orElse(List.of())); - - // TODO add namespace support? - return states.stream() - .filter(s -> s.getType() == AirbyteStateType.STREAM) - .map(s -> new CollectionNameState( - Optional.ofNullable(s.getStream().getStreamDescriptor()).map(StreamDescriptor::getName), - Jsons.tryObject(s.getStream().getStreamState(), MongodbStreamState.class))) - // only keep states that could be parsed - .filter(p -> p.name.isPresent() && p.state.isPresent()) - .collect(Collectors.toMap( - p -> p.name.orElseThrow(), - p -> p.state.orElseThrow())); - } - - /** - * Converts the streams in the catalog into a list of AutoCloseableIterators. - */ - private List> convertCatalogToIterators( - final ConfiguredAirbyteCatalog catalog, - final Map states, - final MongoDatabase database, - final Instant emittedAt) { - return catalog.getStreams() - .stream() - .peek(airbyteStream -> { - if (!airbyteStream.getSyncMode().equals(SyncMode.INCREMENTAL)) - LOGGER.warn("Stream {} configured with unsupported sync mode: {}", airbyteStream.getStream().getName(), airbyteStream.getSyncMode()); - }) - .filter(airbyteStream -> airbyteStream.getSyncMode().equals(SyncMode.INCREMENTAL)) - .map(airbyteStream -> { - final var collectionName = airbyteStream.getStream().getName(); - final var collection = database.getCollection(collectionName); - // TODO verify that if all fields are selected that all fields are returned here - // (or should this check and ignore them if all fields are selected) - final var fields = Projections.fields(Projections.include(CatalogHelpers.getTopLevelFieldNames(airbyteStream).stream().toList())); - - // find the existing state, if there is one, for this steam - final Optional existingState = states.entrySet().stream() - // look only for states that match this stream's name - // TODO add namespace support - .filter(state -> state.getKey().equals(airbyteStream.getStream().getName())) - .map(Entry::getValue) - .findFirst(); - - // The filter determines the starting point of this iterator based on the state of this collection. - // If a state exists, it will use that state to create a query akin to - // "where _id > [last saved state] order by _id ASC". - // If no state exists, it will create a query akin to "where 1=1 order by _id ASC" - final Bson filter = existingState - // TODO add type support here when we add support for _id fields that are not ObjectId types - .map(state -> Filters.gt(ID_FIELD, new ObjectId(state.id()))) - // if nothing was found, return a new BsonDocument - .orElseGet(BsonDocument::new); - - final var cursor = collection.find() - .filter(filter) - .projection(fields) - .sort(Sorts.ascending(ID_FIELD)) - .cursor(); - - final var stateIterator = new MongoDbStateIterator(cursor, airbyteStream, existingState, emittedAt, CHECKPOINT_INTERVAL); - return AutoCloseableIterators.fromIterator(stateIterator, cursor::close, null); - }) - .toList(); - } - - protected MongoClient createMongoClient(final JsonNode config) { - return MongoConnectionUtils.createMongoClient(config); - } - -} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoDbStateIterator.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoDbStateIterator.java deleted file mode 100644 index 4dce1c39aadf..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoDbStateIterator.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mongodb.internal; - -import com.mongodb.MongoException; -import com.mongodb.client.MongoCursor; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.mongodb.MongoUtils; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteMessage.Type; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import io.airbyte.protocol.models.v0.AirbyteStateMessage; -import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; -import io.airbyte.protocol.models.v0.AirbyteStreamState; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.StreamDescriptor; -import java.time.Instant; -import java.util.Iterator; -import java.util.List; -import java.util.Optional; -import org.bson.Document; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * A state-emitting iterator that emits a state message every checkpointInterval messages when - * iterating over a MongoCursor. - * - * Will also output a state message as the last message after the wrapper iterator has completed. - */ -class MongoDbStateIterator implements Iterator { - - private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbStateIterator.class); - - private final MongoCursor iter; - private final ConfiguredAirbyteStream stream; - private final List fields; - private final Instant emittedAt; - private final Integer checkpointInterval; - - /** - * Counts the number of records seen in this batch, resets when a state-message has been generated. - */ - private int count = 0; - - /** - * Pointer to the last document _id seen by this iterator, necessary to track for state messages. - */ - private String lastId = null; - - /** - * This iterator outputs a final state when the wrapped `iter` has concluded. When this is true, the - * final message will be returned. - */ - private boolean finalStateNext = false; - - /** - * Constructor. - * - * @param iter MongoCursor that iterates over Mongo documents - * @param stream the stream that this iterator represents - * @param state the initial state of this stream - * @param emittedAt when this iterator was started - * @param checkpointInterval how often a state message should be emitted. - */ - MongoDbStateIterator(final MongoCursor iter, - final ConfiguredAirbyteStream stream, - Optional state, - final Instant emittedAt, - final int checkpointInterval) { - this.iter = iter; - this.stream = stream; - this.checkpointInterval = checkpointInterval; - this.emittedAt = emittedAt; - fields = CatalogHelpers.getTopLevelFieldNames(stream).stream().toList(); - lastId = state.map(MongodbStreamState::id).orElse(null); - } - - @Override - public boolean hasNext() { - try { - if (iter.hasNext()) { - return true; - } - } catch (MongoException e) { - // If hasNext throws an exception, log it and then treat it as if hasNext returned false. - LOGGER.info("hasNext threw an exception: {}", e.getMessage(), e); - } - - if (!finalStateNext) { - finalStateNext = true; - return true; - } - - return false; - } - - @Override - public AirbyteMessage next() { - if ((count > 0 && count % checkpointInterval == 0) || finalStateNext) { - count = 0; - - final var streamState = new AirbyteStreamState() - .withStreamDescriptor(new StreamDescriptor() - .withName(stream.getStream().getName()) - .withNamespace(stream.getStream().getNamespace())); - if (lastId != null) { - // TODO add type support in here once more than ObjectId fields are supported - streamState.withStreamState(Jsons.jsonNode(new MongodbStreamState(lastId))); - } - - final var stateMessage = new AirbyteStateMessage() - .withType(AirbyteStateType.STREAM) - .withStream(streamState); - - return new AirbyteMessage().withType(Type.STATE).withState(stateMessage); - } - - count++; - final var document = iter.next(); - final var jsonNode = MongoUtils.toJsonNode(document, fields); - - lastId = document.getObjectId("_id").toString(); - - return new AirbyteMessage() - .withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage() - .withStream(stream.getStream().getName()) - .withNamespace(stream.getStream().getNamespace()) - .withEmittedAt(emittedAt.toEpochMilli()) - .withData(jsonNode)); - } - -} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoUtil.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoUtil.java deleted file mode 100644 index 21c326be72b5..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoUtil.java +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mongodb.internal; - -import com.mongodb.MongoCommandException; -import com.mongodb.MongoException; -import com.mongodb.MongoSecurityException; -import com.mongodb.client.AggregateIterable; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoCursor; -import com.mongodb.client.model.Aggregates; -import io.airbyte.commons.exceptions.ConnectionErrorException; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.AirbyteStream; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; -import org.bson.Document; -import org.bson.conversions.Bson; - -public class MongoUtil { - - /** - * The maximum number of documents to sample when attempting to discover the unique keys/types for a - * collection. Inspired by the - * sampling method - * utilized by the MongoDB Compass client. - */ - private static final Integer DISCOVERY_SAMPLE_SIZE = 1000; - - /** - * Set of collection prefixes that should be ignored when performing operations, such as discover to - * avoid access issues. - */ - private static final Set IGNORED_COLLECTIONS = Set.of("system.", "replset.", "oplog."); - - /** - * Returns the set of collections that the current credentials are authorized to access. - * - * @param mongoClient The {@link MongoClient} used to query the MongoDB server for authorized - * collections. - * @param databaseName The name of the database to query for authorized collections. - * @return The set of authorized collection names (may be empty). - * @throws ConnectionErrorException if unable to perform the authorized collection query. - */ - public static Set getAuthorizedCollections(final MongoClient mongoClient, final String databaseName) { - /* - * db.runCommand ({listCollections: 1.0, authorizedCollections: true, nameOnly: true }) the command - * returns only those collections for which the user has privileges. For example, if a user has find - * action on specific collections, the command returns only those collections; or, if a user has - * find or any other action, on the database resource, the command lists all collections in the - * database. - */ - try { - final Document document = mongoClient.getDatabase(databaseName).runCommand(new Document("listCollections", 1) - .append("authorizedCollections", true) - .append("nameOnly", true)) - .append("filter", "{ 'type': 'collection' }"); - return document.toBsonDocument() - .get("cursor").asDocument() - .getArray("firstBatch") - .stream() - .map(bsonValue -> bsonValue.asDocument().getString("name").getValue()) - .filter(MongoUtil::isSupportedCollection) - .collect(Collectors.toSet()); - } catch (final MongoSecurityException e) { - final MongoCommandException exception = (MongoCommandException) e.getCause(); - throw new ConnectionErrorException(String.valueOf(exception.getCode()), e); - } catch (final MongoException e) { - throw new ConnectionErrorException(String.valueOf(e.getCode()), e); - } - } - - /** - * Retrieves the {@link AirbyteStream}s available to the source by querying the MongoDB server. - * - * @param mongoClient The {@link MongoClient} used to query the MongoDB server. - * @param databaseName The name of the database to query for collections. - * @return The list of {@link AirbyteStream}s that map to the available collections in the provided - * database. - */ - public static List getAirbyteStreams(final MongoClient mongoClient, final String databaseName) { - final Set authorizedCollections = getAuthorizedCollections(mongoClient, databaseName); - return authorizedCollections.parallelStream().map(collectionName -> { - /* - * Fetch the keys/types from the first N documents and the last N documents from the collection. - * This is an attempt to "survey" the documents in the collection for variance in the schema keys. - */ - final Set discoveredFields = new HashSet<>(); - final MongoCollection mongoCollection = mongoClient.getDatabase(databaseName).getCollection(collectionName); - discoveredFields.addAll(getFieldsInCollection(mongoCollection)); - return createAirbyteStream(collectionName, databaseName, new ArrayList<>(discoveredFields)); - }).collect(Collectors.toList()); - } - - private static AirbyteStream createAirbyteStream(final String collectionName, final String databaseName, final List fields) { - return MongoCatalogHelper.buildAirbyteStream(collectionName, databaseName, fields); - } - - private static Set getFieldsInCollection(final MongoCollection collection) { - final Set discoveredFields = new HashSet<>(); - final Map fieldsMap = Map.of("input", Map.of("$objectToArray", "$$ROOT"), - "as", "each", - "in", Map.of("k", "$$each.k", "v", Map.of("$type", "$$each.v"))); - - final Document mapFunction = new Document("$map", fieldsMap); - final Document arrayToObjectAggregation = new Document("$arrayToObject", mapFunction); - - final Map groupMap = new HashMap<>(); - groupMap.put("_id", null); - groupMap.put("fields", Map.of("$addToSet", "$fields")); - - final List aggregateList = new ArrayList<>(); - aggregateList.add(Aggregates.sample(DISCOVERY_SAMPLE_SIZE)); - aggregateList.add(Aggregates.project(new Document("fields", arrayToObjectAggregation))); - aggregateList.add(Aggregates.unwind("$fields")); - aggregateList.add(new Document("$group", groupMap)); - - final AggregateIterable output = collection.aggregate(aggregateList); - - try (final MongoCursor cursor = output.cursor()) { - while (cursor.hasNext()) { - final Map fields = ((List>) cursor.next().get("fields")).get(0); - discoveredFields.addAll(fields.entrySet().stream() - .map(e -> new MongoField(e.getKey(), convertToSchemaType(e.getValue()))) - .collect(Collectors.toSet())); - } - } - - return discoveredFields; - } - - private static JsonSchemaType convertToSchemaType(final String type) { - return switch (type) { - case "boolean" -> JsonSchemaType.BOOLEAN; - case "int", "long", "double", "decimal" -> JsonSchemaType.NUMBER; - case "array" -> JsonSchemaType.ARRAY; - case "object", "javascriptWithScope" -> JsonSchemaType.OBJECT; - default -> JsonSchemaType.STRING; - }; - } - - private static boolean isSupportedCollection(final String collectionName) { - return !IGNORED_COLLECTIONS.stream().anyMatch(s -> collectionName.startsWith(s)); - } - -} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongodbStreamState.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongodbStreamState.java deleted file mode 100644 index 12b8dad1f151..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongodbStreamState.java +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mongodb.internal; - -/* - * TODO replace `isObjectId` with _id enum (ObjectId, String, etc.) - */ -public record MongodbStreamState(String id) { // , boolean isObjectId) { - -} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/resources/spec.json deleted file mode 100644 index 4a38d5ee89d2..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/resources/spec.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/mongodb-internal-poc", - "changelogUrl": "https://docs.airbyte.com/integrations/sources/mongodb-internal-poc", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MongoDb Source Spec", - "type": "object", - "required": ["connection_string", "database", "replica_set"], - "additionalProperties": true, - "properties": { - "connection_string": { - "title": "Connection String", - "type": "string", - "description": "The connection string of the database that you want to replicate..", - "examples": [ - "mongodb+srv://example.mongodb.net/", - "mongodb://example1.host.com:27017,example2.host.com:27017,example3.host.com:27017/", - "mongodb://example.host.com:27017/" - ], - "order": 1 - }, - "database": { - "title": "Database Name", - "type": "string", - "description": "The database you want to replicate.", - "order": 2 - }, - "user": { - "title": "User", - "type": "string", - "description": "The username which is used to access the database.", - "order": 3 - }, - "password": { - "title": "Password", - "type": "string", - "description": "The password associated with this username.", - "airbyte_secret": true, - "order": 4 - }, - "auth_source": { - "title": "Authentication Source", - "type": "string", - "description": "The authentication source where the user information is stored.", - "default": "admin", - "examples": ["admin"], - "order": 5 - }, - "replica_set": { - "title": "Replica Set", - "type": "string", - "description": "The name of the replica set to be replicated.", - "order": 6 - } - } - } -} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test-integration/java/io/airbyte/integrations/source/mongodb/internal/MongoDbSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test-integration/java/io/airbyte/integrations/source/mongodb/internal/MongoDbSourceAcceptanceTest.java deleted file mode 100644 index 71f1b0ec4646..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test-integration/java/io/airbyte/integrations/source/mongodb/internal/MongoDbSourceAcceptanceTest.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mongodb.internal; - -import static io.airbyte.integrations.source.mongodb.internal.MongoCatalogHelper.DEFAULT_CURSOR_FIELD; -import static io.airbyte.integrations.source.mongodb.internal.MongoConstants.DATABASE_CONFIGURATION_KEY; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoCollection; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.AirbyteStream; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.SyncMode; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.List; -import org.bson.BsonArray; -import org.bson.BsonString; -import org.bson.Document; -import org.bson.types.ObjectId; - -public class MongoDbSourceAcceptanceTest extends SourceAcceptanceTest { - - private static final String DATABASE_NAME = "test"; - private static final String COLLECTION_NAME = "acceptance_test1"; - private static final Path CREDENTIALS_PATH = Path.of("secrets/credentials.json"); - - private JsonNode config; - private MongoClient mongoClient; - - @Override - protected void setupEnvironment(final TestDestinationEnv testEnv) throws IOException { - if (!Files.exists(CREDENTIALS_PATH)) { - throw new IllegalStateException( - "Must provide path to a MongoDB credentials file. By default {module-root}/" + CREDENTIALS_PATH - + ". Override by setting setting path with the CREDENTIALS_PATH constant."); - } - - config = Jsons.deserialize(Files.readString(CREDENTIALS_PATH)); - ((ObjectNode) config).put(DATABASE_CONFIGURATION_KEY, DATABASE_NAME); - - mongoClient = MongoConnectionUtils.createMongoClient(config); - - insertTestData(mongoClient); - } - - private void insertTestData(final MongoClient mongoClient) { - mongoClient.getDatabase(DATABASE_NAME).createCollection(COLLECTION_NAME); - final MongoCollection collection = mongoClient.getDatabase(DATABASE_NAME).getCollection(COLLECTION_NAME); - final var objectDocument = new Document("testObject", new Document("name", "subName").append("testField1", "testField1").append("testInt", 10) - .append("thirdLevelDocument", new Document("data", "someData").append("intData", 1))); - - final var doc1 = new Document("_id", new ObjectId("64c0029d95ad260d69ef28a0")) - .append("id", "0001").append("name", "Test") - .append("test", 10).append("test_array", new BsonArray(List.of(new BsonString("test"), new BsonString("mongo")))) - .append("double_test", 100.12).append("int_test", 100).append("object_test", objectDocument); - - final var doc2 = new Document("_id", new ObjectId("64c0029d95ad260d69ef28a1")) - .append("id", "0002").append("name", "Mongo").append("test", "test_value").append("int_test", 201).append("object_test", objectDocument); - - final var doc3 = new Document("_id", new ObjectId("64c0029d95ad260d69ef28a2")) - .append("id", "0003").append("name", "Source").append("test", null) - .append("double_test", 212.11).append("int_test", 302).append("object_test", objectDocument); - - collection.insertMany(List.of(doc1, doc2, doc3)); - } - - @Override - protected void tearDown(final TestDestinationEnv testEnv) { - mongoClient.getDatabase(DATABASE_NAME).getCollection(COLLECTION_NAME).drop(); - mongoClient.close(); - } - - @Override - protected String getImageName() { - return "airbyte/source-mongodb-internal-poc:dev"; - } - - @Override - protected ConnectorSpecification getSpec() throws Exception { - return Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class); - } - - @Override - protected JsonNode getConfig() { - return config; - } - - @Override - protected ConfiguredAirbyteCatalog getConfiguredCatalog() { - final List fields = List.of( - Field.of(DEFAULT_CURSOR_FIELD, JsonSchemaType.STRING), - Field.of("id", JsonSchemaType.STRING), - Field.of("name", JsonSchemaType.STRING), - Field.of("test", JsonSchemaType.STRING), - Field.of("test_array", JsonSchemaType.ARRAY), - Field.of("empty_test", JsonSchemaType.STRING), - Field.of("double_test", JsonSchemaType.NUMBER), - Field.of("int_test", JsonSchemaType.NUMBER), - Field.of("object_test", JsonSchemaType.OBJECT)); - - final AirbyteStream airbyteStream = MongoCatalogHelper.buildAirbyteStream(COLLECTION_NAME, DATABASE_NAME, fields); - final ConfiguredAirbyteStream configuredIncrementalAirbyteStream = convertToConfiguredAirbyteStream(airbyteStream, SyncMode.INCREMENTAL); - - return new ConfiguredAirbyteCatalog().withStreams(List.of(configuredIncrementalAirbyteStream)); - } - - @Override - protected JsonNode getState() { - return Jsons.jsonNode(new HashMap<>()); - } - - private ConfiguredAirbyteStream convertToConfiguredAirbyteStream(final AirbyteStream airbyteStream, final SyncMode syncMode) { - return new ConfiguredAirbyteStream() - .withSyncMode(syncMode) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withCursorField(List.of(DEFAULT_CURSOR_FIELD)) - .withStream(airbyteStream); - } - -} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoCatalogHelperTest.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoCatalogHelperTest.java deleted file mode 100644 index f67e0e7f1645..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoCatalogHelperTest.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mongodb.internal; - -import static io.airbyte.integrations.source.mongodb.internal.MongoCatalogHelper.DEFAULT_CURSOR_FIELD; -import static io.airbyte.integrations.source.mongodb.internal.MongoCatalogHelper.SUPPORTED_SYNC_MODES; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import io.airbyte.integrations.debezium.internals.DebeziumEventUtils; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.AirbyteStream; -import java.util.List; -import org.junit.jupiter.api.Test; - -class MongoCatalogHelperTest { - - @Test - void testBuildingAirbyteStream() { - final String streamName = "name"; - final String streamNamespace = "namespace"; - final List discoveredFields = List.of(new Field("field1", JsonSchemaType.STRING), - new Field("field2", JsonSchemaType.NUMBER)); - - final AirbyteStream airbyteStream = MongoCatalogHelper.buildAirbyteStream(streamName, streamNamespace, discoveredFields); - - assertNotNull(airbyteStream); - assertEquals(streamNamespace, airbyteStream.getNamespace()); - assertEquals(streamName, airbyteStream.getName()); - assertEquals(List.of(DEFAULT_CURSOR_FIELD), airbyteStream.getDefaultCursorField()); - assertEquals(true, airbyteStream.getSourceDefinedCursor()); - assertEquals(List.of(List.of(DEFAULT_CURSOR_FIELD)), airbyteStream.getSourceDefinedPrimaryKey()); - assertEquals(SUPPORTED_SYNC_MODES, airbyteStream.getSupportedSyncModes()); - assertEquals(5, airbyteStream.getJsonSchema().get("properties").size()); - - discoveredFields.forEach(f -> assertTrue(airbyteStream.getJsonSchema().get("properties").has(f.getName()))); - assertTrue(airbyteStream.getJsonSchema().get("properties").has(DebeziumEventUtils.CDC_LSN)); - assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get("type"), - airbyteStream.getJsonSchema().get("properties").get(DebeziumEventUtils.CDC_LSN).get("type").asText()); - assertTrue(airbyteStream.getJsonSchema().get("properties").has(DebeziumEventUtils.CDC_DELETED_AT)); - assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), - airbyteStream.getJsonSchema().get("properties").get(DebeziumEventUtils.CDC_DELETED_AT).get("type").asText()); - assertTrue(airbyteStream.getJsonSchema().get("properties").has(DebeziumEventUtils.CDC_UPDATED_AT)); - assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), - airbyteStream.getJsonSchema().get("properties").get(DebeziumEventUtils.CDC_UPDATED_AT).get("type").asText()); - - } - -} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoDbSourceTest.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoDbSourceTest.java deleted file mode 100644 index c4a02f426d83..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoDbSourceTest.java +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mongodb.internal; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.JsonNode; -import com.mongodb.client.AggregateIterable; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoCursor; -import com.mongodb.client.MongoDatabase; -import com.mongodb.connection.ClusterDescription; -import com.mongodb.connection.ClusterType; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.debezium.internals.DebeziumEventUtils; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.AirbyteCatalog; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import io.airbyte.protocol.models.v0.AirbyteStream; -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.stream.Collectors; -import org.bson.Document; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class MongoDbSourceTest { - - private static final String DB_NAME = "airbyte_test"; - - private JsonNode airbyteSourceConfig; - private MongoClient mongoClient; - private MongoDbSource source; - - @BeforeEach - void setup() { - airbyteSourceConfig = createConfiguration(Optional.empty(), Optional.empty()); - mongoClient = mock(MongoClient.class); - source = spy(new MongoDbSource()); - doReturn(mongoClient).when(source).createMongoClient(airbyteSourceConfig); - } - - @Test - void testCheckOperation() throws IOException { - final ClusterDescription clusterDescription = mock(ClusterDescription.class); - final Document response = Document.parse(MoreResources.readResource("authorized_collections_response.json")); - final MongoDatabase mongoDatabase = mock(MongoDatabase.class); - - when(clusterDescription.getType()).thenReturn(ClusterType.REPLICA_SET); - when(mongoDatabase.runCommand(any())).thenReturn(response); - when(mongoClient.getDatabase(any())).thenReturn(mongoDatabase); - when(mongoClient.getClusterDescription()).thenReturn(clusterDescription); - - final AirbyteConnectionStatus airbyteConnectionStatus = source.check(airbyteSourceConfig); - assertNotNull(airbyteConnectionStatus); - assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, airbyteConnectionStatus.getStatus()); - } - - @Test - void testCheckOperationNoAuthorizedCollections() throws IOException { - final ClusterDescription clusterDescription = mock(ClusterDescription.class); - final Document response = Document.parse(MoreResources.readResource("no_authorized_collections_response.json")); - final MongoDatabase mongoDatabase = mock(MongoDatabase.class); - - when(clusterDescription.getType()).thenReturn(ClusterType.REPLICA_SET); - when(mongoDatabase.runCommand(any())).thenReturn(response); - when(mongoClient.getDatabase(any())).thenReturn(mongoDatabase); - when(mongoClient.getClusterDescription()).thenReturn(clusterDescription); - - final AirbyteConnectionStatus airbyteConnectionStatus = source.check(airbyteSourceConfig); - assertNotNull(airbyteConnectionStatus); - assertEquals(AirbyteConnectionStatus.Status.FAILED, airbyteConnectionStatus.getStatus()); - assertEquals("Target MongoDB database does not contain any authorized collections.", airbyteConnectionStatus.getMessage()); - } - - @Test - void testCheckOperationInvalidClusterType() throws IOException { - final ClusterDescription clusterDescription = mock(ClusterDescription.class); - final Document response = Document.parse(MoreResources.readResource("authorized_collections_response.json")); - final MongoDatabase mongoDatabase = mock(MongoDatabase.class); - - when(clusterDescription.getType()).thenReturn(ClusterType.STANDALONE); - when(mongoDatabase.runCommand(any())).thenReturn(response); - when(mongoClient.getDatabase(any())).thenReturn(mongoDatabase); - when(mongoClient.getClusterDescription()).thenReturn(clusterDescription); - - final AirbyteConnectionStatus airbyteConnectionStatus = source.check(airbyteSourceConfig); - assertNotNull(airbyteConnectionStatus); - assertEquals(AirbyteConnectionStatus.Status.FAILED, airbyteConnectionStatus.getStatus()); - assertEquals("Target MongoDB instance is not a replica set cluster.", airbyteConnectionStatus.getMessage()); - } - - @Test - void testCheckOperationUnexpectedException() { - final String expectedMessage = "This is just a test failure."; - when(mongoClient.getDatabase(any())).thenThrow(new IllegalArgumentException(expectedMessage)); - - final AirbyteConnectionStatus airbyteConnectionStatus = source.check(airbyteSourceConfig); - assertNotNull(airbyteConnectionStatus); - assertEquals(AirbyteConnectionStatus.Status.FAILED, airbyteConnectionStatus.getStatus()); - assertEquals(expectedMessage, airbyteConnectionStatus.getMessage()); - } - - @Test - void testDiscoverOperation() throws IOException { - final AggregateIterable aggregateIterable = mock(AggregateIterable.class); - final List> schemaDiscoveryJsonResponses = - Jsons.deserialize(MoreResources.readResource("schema_discovery_response.json"), new TypeReference<>() {}); - final List schemaDiscoveryResponses = schemaDiscoveryJsonResponses.stream().map(s -> new Document(s)).collect(Collectors.toList()); - final Document authorizedCollectionsResponse = Document.parse(MoreResources.readResource("authorized_collections_response.json")); - final MongoCollection mongoCollection = mock(MongoCollection.class); - final MongoCursor cursor = mock(MongoCursor.class); - final MongoDatabase mongoDatabase = mock(MongoDatabase.class); - - when(cursor.hasNext()).thenReturn(true, true, false); - when(cursor.next()).thenReturn(schemaDiscoveryResponses.get(0), schemaDiscoveryResponses.get(1)); - when(aggregateIterable.cursor()).thenReturn(cursor); - when(mongoCollection.aggregate(any())).thenReturn(aggregateIterable); - when(mongoDatabase.getCollection(any())).thenReturn(mongoCollection); - when(mongoDatabase.runCommand(any())).thenReturn(authorizedCollectionsResponse); - when(mongoClient.getDatabase(any())).thenReturn(mongoDatabase); - - final AirbyteCatalog airbyteCatalog = source.discover(airbyteSourceConfig); - - assertNotNull(airbyteCatalog); - assertEquals(1, airbyteCatalog.getStreams().size()); - - final Optional stream = airbyteCatalog.getStreams().stream().findFirst(); - assertTrue(stream.isPresent()); - assertEquals(DB_NAME, stream.get().getNamespace()); - assertEquals("testCollection", stream.get().getName()); - assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), - stream.get().getJsonSchema().get("properties").get("_id").get("type").asText()); - assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), - stream.get().getJsonSchema().get("properties").get("name").get("type").asText()); - assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), - stream.get().getJsonSchema().get("properties").get("last_updated").get("type").asText()); - assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get("type"), - stream.get().getJsonSchema().get("properties").get("total").get("type").asText()); - assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get("type"), - stream.get().getJsonSchema().get("properties").get("price").get("type").asText()); - assertEquals(JsonSchemaType.ARRAY.getJsonSchemaTypeMap().get("type"), - stream.get().getJsonSchema().get("properties").get("items").get("type").asText()); - assertEquals(JsonSchemaType.OBJECT.getJsonSchemaTypeMap().get("type"), - stream.get().getJsonSchema().get("properties").get("owners").get("type").asText()); - assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), - stream.get().getJsonSchema().get("properties").get("other").get("type").asText()); - assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get("type"), - stream.get().getJsonSchema().get("properties").get(DebeziumEventUtils.CDC_LSN).get("type").asText()); - assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), - stream.get().getJsonSchema().get("properties").get(DebeziumEventUtils.CDC_DELETED_AT).get("type").asText()); - assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), - stream.get().getJsonSchema().get("properties").get(DebeziumEventUtils.CDC_UPDATED_AT).get("type").asText()); - assertEquals(true, stream.get().getSourceDefinedCursor()); - assertEquals(List.of(MongoCatalogHelper.DEFAULT_CURSOR_FIELD), stream.get().getDefaultCursorField()); - assertEquals(List.of(List.of(MongoCatalogHelper.DEFAULT_CURSOR_FIELD)), stream.get().getSourceDefinedPrimaryKey()); - assertEquals(MongoCatalogHelper.SUPPORTED_SYNC_MODES, stream.get().getSupportedSyncModes()); - } - - @Test - void testDiscoverOperationWithUnexpectedFailure() throws IOException { - final String expectedMessage = "This is just a test failure."; - when(mongoClient.getDatabase(any())).thenThrow(new IllegalArgumentException(expectedMessage)); - - assertThrows(IllegalArgumentException.class, () -> source.discover(airbyteSourceConfig)); - } - - @Test - void testFullRefresh() throws Exception { - // TODO implement - } - - @Test - void testIncrementalRefresh() throws Exception { - // TODO implement - } - - @Test - void testConvertState() { - final var state1 = Jsons.deserialize( - "[{\"type\":\"STREAM\",\"stream\":{\"stream_descriptor\":{\"name\":\"test.acceptance_test1\"},\"stream_state\":{\"id\":\"64c0029d95ad260d69ef28a2\"}}}]"); - final var actual = source.convertState(state1); - assertTrue(actual.containsKey("test.acceptance_test1"), "missing test.acceptance_test1"); - assertEquals("64c0029d95ad260d69ef28a2", actual.get("test.acceptance_test1").id(), "id value does not match"); - - } - - private static JsonNode createConfiguration(final Optional username, final Optional password) { - final Map config = new HashMap<>(); - final Map baseConfig = Map.of( - MongoConstants.DATABASE_CONFIGURATION_KEY, DB_NAME, - MongoConstants.CONNECTION_STRING_CONFIGURATION_KEY, "mongodb://localhost:27017/", - MongoConstants.AUTH_SOURCE_CONFIGURATION_KEY, "admin", - MongoConstants.REPLICA_SET_CONFIGURATION_KEY, "replica-set"); - - config.putAll(baseConfig); - username.ifPresent(u -> config.put(MongoConstants.USER_CONFIGURATION_KEY, u)); - password.ifPresent(p -> config.put(MongoConstants.PASSWORD_CONFIGURATION_KEY, p)); - return Jsons.deserialize(Jsons.serialize(config)); - } - -} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoDbStateIteratorTest.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoDbStateIteratorTest.java deleted file mode 100644 index 040081f2b8b3..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoDbStateIteratorTest.java +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mongodb.internal; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.when; - -import com.mongodb.MongoException; -import com.mongodb.client.MongoCursor; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteMessage.Type; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.SyncMode; -import java.time.Instant; -import java.util.List; -import java.util.Optional; -import org.bson.Document; -import org.bson.types.ObjectId; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; - -class MongoDbStateIteratorTest { - - private static final int CHECKPOINT_INTERVAL = 2; - @Mock - private MongoCursor mongoCursor; - private AutoCloseable closeable; - - @BeforeEach - public void setup() { - closeable = MockitoAnnotations.openMocks(this); - } - - @AfterEach - public void teardown() throws Exception { - closeable.close(); - } - - @Test - void happyPath() { - final var docs = docs(); - - when(mongoCursor.hasNext()).thenAnswer(new Answer() { - - private int count = 0; - - @Override - public Boolean answer(InvocationOnMock invocation) throws Throwable { - count++; - // hasNext will be called for each doc plus for each state message - return count <= (docs.size() + (docs.size() % CHECKPOINT_INTERVAL)); - } - - }); - - when(mongoCursor.next()).thenAnswer(new Answer() { - - private int offset = 0; - - @Override - public Document answer(InvocationOnMock invocation) throws Throwable { - final var doc = docs.get(offset); - offset++; - return doc; - } - - }); - - final var stream = catalog().getStreams().stream().findFirst().orElseThrow(); - - final var iter = new MongoDbStateIterator(mongoCursor, stream, Optional.empty(), Instant.now(), CHECKPOINT_INTERVAL); - - // with a batch size of 2, the MongoDbStateIterator should return the following after each - // `hasNext`/`next` call: - // true, record Air Force Blue - // true, record Alice Blue - // true, state (with Alice Blue as the state) - // true, record Alizarin Crimson - // true, state (with Alizarin Crimson) - // false - AirbyteMessage message; - assertTrue(iter.hasNext(), "air force blue should be next"); - message = iter.next(); - assertEquals(Type.RECORD, message.getType()); - assertEquals(docs.get(0).get("_id").toString(), message.getRecord().getData().get("_id").asText()); - - assertTrue(iter.hasNext(), "alice blue should be next"); - message = iter.next(); - assertEquals(Type.RECORD, message.getType()); - assertEquals(docs.get(1).get("_id").toString(), message.getRecord().getData().get("_id").asText()); - - assertTrue(iter.hasNext(), "state should be next"); - message = iter.next(); - assertEquals(Type.STATE, message.getType()); - assertEquals( - docs.get(1).get("_id").toString(), - message.getState().getStream().getStreamState().get("id").asText(), - "state id should match last record id"); - - assertTrue(iter.hasNext(), "alizarin crimson should be next"); - message = iter.next(); - assertEquals(Type.RECORD, message.getType()); - assertEquals(docs.get(2).get("_id").toString(), message.getRecord().getData().get("_id").asText()); - - assertTrue(iter.hasNext(), "state should be next"); - message = iter.next(); - assertEquals(Type.STATE, message.getType()); - assertEquals( - docs.get(2).get("_id").toString(), - message.getState().getStream().getStreamState().get("id").asText(), - "state id should match last record id"); - - assertFalse(iter.hasNext(), "should have no more records"); - } - - @Test - void treatHasNextExceptionAsFalse() { - final var docs = docs(); - - // on the second hasNext call, throw an exception - when(mongoCursor.hasNext()) - .thenReturn(true) - .thenThrow(new MongoException("test exception")); - - when(mongoCursor.next()).thenReturn(docs.get(0)); - - final var stream = catalog().getStreams().stream().findFirst().orElseThrow(); - - final var iter = new MongoDbStateIterator(mongoCursor, stream, Optional.empty(), Instant.now(), CHECKPOINT_INTERVAL); - - // with a batch size of 2, the MongoDbStateIterator should return the following after each - // `hasNext`/`next` call: - // true, record Air Force Blue - // true (exception thrown), state (with Air Force Blue as the state) - // false - AirbyteMessage message; - assertTrue(iter.hasNext(), "air force blue should be next"); - message = iter.next(); - assertEquals(Type.RECORD, message.getType()); - assertEquals(docs.get(0).get("_id").toString(), message.getRecord().getData().get("_id").asText()); - - assertTrue(iter.hasNext(), "state should be next"); - message = iter.next(); - assertEquals(Type.STATE, message.getType()); - assertEquals( - docs.get(0).get("_id").toString(), - message.getState().getStream().getStreamState().get("id").asText(), - "state id should match last record id"); - - assertFalse(iter.hasNext(), "should have no more records"); - } - - @Test - void initialStateIsReturnedIfUnderlyingIteratorIsEmpty() { - final var docs = docs(); - - // on the second hasNext call, throw an exception - when(mongoCursor.hasNext()).thenReturn(false); - - final var stream = catalog().getStreams().stream().findFirst().orElseThrow(); - final var objectId = "64dfb6a7bb3c3458c30801f4"; - final var iter = new MongoDbStateIterator(mongoCursor, stream, Optional.of(new MongodbStreamState(objectId)), Instant.now(), CHECKPOINT_INTERVAL); - - // the MongoDbStateIterator should return the following after each - // `hasNext`/`next` call: - // false - // then the generated state message should have the same id as the initial state - assertTrue(iter.hasNext(), "state should be next"); - - final AirbyteMessage message = iter.next(); - assertEquals(Type.STATE, message.getType()); - assertEquals( - objectId, - message.getState().getStream().getStreamState().get("id").asText(), - "state id should match initial state "); - - assertFalse(iter.hasNext(), "should have no more records"); - } - - private ConfiguredAirbyteCatalog catalog() { - return new ConfiguredAirbyteCatalog().withStreams(List.of( - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(List.of("_id")) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withCursorField(List.of("_id")) - .withStream(CatalogHelpers.createAirbyteStream( - "test.unit", - Field.of("_id", JsonSchemaType.STRING), - Field.of("name", JsonSchemaType.STRING), - Field.of("hex", JsonSchemaType.STRING)) - .withSupportedSyncModes(List.of(SyncMode.INCREMENTAL)) - .withDefaultCursorField(List.of("_id"))))); - } - - private List docs() { - return List.of( - new Document("_id", new ObjectId("64c0029d95ad260d69ef28a0")) - .append("name", "Air Force Blue").append("hex", "#5d8aa8"), - new Document("_id", new ObjectId("64c0029d95ad260d69ef28a1")) - .append("name", "Alice Blue").append("hex", "#f0f8ff"), - new Document("_id", new ObjectId("64c0029d95ad260d69ef28a2")) - .append("name", "Alizarin Crimson").append("hex", "#e32636")); - } - -} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoUtilTest.java b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoUtilTest.java deleted file mode 100644 index de2e80e82e4f..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/java/io/airbyte/integrations/source/mongodb/internal/MongoUtilTest.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mongodb.internal; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.mongodb.client.AggregateIterable; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoCursor; -import com.mongodb.client.MongoDatabase; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.AirbyteStream; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import org.bson.Document; -import org.junit.jupiter.api.Test; - -public class MongoUtilTest { - - @Test - void testGetAirbyteStreams() throws IOException { - final AggregateIterable aggregateIterable = mock(AggregateIterable.class); - final MongoCursor cursor = mock(MongoCursor.class); - final String databaseName = "database"; - final Document authorizedCollectionsResponse = Document.parse(MoreResources.readResource("authorized_collections_response.json")); - final MongoClient mongoClient = mock(MongoClient.class); - final MongoCollection mongoCollection = mock(MongoCollection.class); - final MongoDatabase mongoDatabase = mock(MongoDatabase.class); - final List> schemaDiscoveryJsonResponses = - Jsons.deserialize(MoreResources.readResource("schema_discovery_response.json"), new TypeReference<>() {}); - final List schemaDiscoveryResponses = schemaDiscoveryJsonResponses.stream().map(s -> new Document(s)).collect(Collectors.toList()); - - when(cursor.hasNext()).thenReturn(true, true, false); - when(cursor.next()).thenReturn(schemaDiscoveryResponses.get(0), schemaDiscoveryResponses.get(1)); - when(aggregateIterable.cursor()).thenReturn(cursor); - when(mongoCollection.aggregate(any())).thenReturn(aggregateIterable); - when(mongoDatabase.getCollection(any())).thenReturn(mongoCollection); - when(mongoDatabase.runCommand(any())).thenReturn(authorizedCollectionsResponse); - when(mongoClient.getDatabase(databaseName)).thenReturn(mongoDatabase); - - final List streams = MongoUtil.getAirbyteStreams(mongoClient, databaseName); - assertNotNull(streams); - assertEquals(1, streams.size()); - assertEquals(11, streams.get(0).getJsonSchema().get("properties").size()); - } - - @Test - void testGetAirbyteStreamsDifferentDataTypes() throws IOException { - final AggregateIterable aggregateIterable = mock(AggregateIterable.class); - final MongoCursor cursor = mock(MongoCursor.class); - final String databaseName = "database"; - final Document authorizedCollectionsResponse = Document.parse(MoreResources.readResource("authorized_collections_response.json")); - final MongoClient mongoClient = mock(MongoClient.class); - final MongoCollection mongoCollection = mock(MongoCollection.class); - final MongoDatabase mongoDatabase = mock(MongoDatabase.class); - final List> schemaDiscoveryJsonResponses = - Jsons.deserialize(MoreResources.readResource("schema_discovery_response_different_datatypes.json"), new TypeReference<>() {}); - final List schemaDiscoveryResponses = schemaDiscoveryJsonResponses.stream().map(s -> new Document(s)).collect(Collectors.toList()); - - when(cursor.hasNext()).thenReturn(true, true, false); - when(cursor.next()).thenReturn(schemaDiscoveryResponses.get(0), schemaDiscoveryResponses.get(1)); - when(aggregateIterable.cursor()).thenReturn(cursor); - when(mongoCollection.aggregate(any())).thenReturn(aggregateIterable); - when(mongoDatabase.getCollection(any())).thenReturn(mongoCollection); - when(mongoDatabase.runCommand(any())).thenReturn(authorizedCollectionsResponse); - when(mongoClient.getDatabase(databaseName)).thenReturn(mongoDatabase); - - final List streams = MongoUtil.getAirbyteStreams(mongoClient, databaseName); - assertNotNull(streams); - assertEquals(1, streams.size()); - assertEquals(11, streams.get(0).getJsonSchema().get("properties").size()); - assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get("type"), - streams.get(0).getJsonSchema().get("properties").get("total").get("type").asText()); - } - -} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/kotlin/MongoDbInsertClient.kt b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/kotlin/MongoDbInsertClient.kt deleted file mode 100644 index a944983fa008..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/kotlin/MongoDbInsertClient.kt +++ /dev/null @@ -1,52 +0,0 @@ -package io.airbyte.integrations.source.mongodb.internal - -import io.airbyte.commons.json.Jsons -import kotlinx.cli.ArgParser -import kotlinx.cli.ArgType -import kotlinx.cli.default -import kotlinx.cli.required -import org.bson.BsonTimestamp -import org.bson.Document -import java.lang.System.currentTimeMillis - -object MongoDbInsertClient { - - @JvmStatic - fun main(args: Array) { - val parser = ArgParser("MongoDb Insert Client") - val connectionString by parser.option(ArgType.String, fullName = "connection-string", shortName = "cs", description = "MongoDb Connection String").required() - val databaseName by parser.option(ArgType.String, fullName = "database-name", shortName = "d", description = "Database Name").required() - val collectionName by parser.option(ArgType.String, fullName = "collection-name", shortName = "cn", description = "Collection Name").required() - val replicaSet by parser.option(ArgType.String, fullName = "replica-set", shortName = "r", description = "Replica Set").required() - val username by parser.option(ArgType.String, fullName = "username", shortName = "u", description = "Username").required() - val numberOfDocuments by parser.option(ArgType.Int, fullName = "number", shortName = "n", description = "Number of documents to generate").default(10000) - - parser.parse(args) - - println("Enter password: ") - val password = readln() - - var config = mapOf(MongoConstants.DATABASE_CONFIGURATION_KEY to databaseName, - MongoConstants.CONNECTION_STRING_CONFIGURATION_KEY to connectionString, - MongoConstants.AUTH_SOURCE_CONFIGURATION_KEY to "admin", - MongoConstants.REPLICA_SET_CONFIGURATION_KEY to replicaSet, - MongoConstants.USER_CONFIGURATION_KEY to username, - MongoConstants.PASSWORD_CONFIGURATION_KEY to password) - - MongoConnectionUtils.createMongoClient(Jsons.deserialize(Jsons.serialize(config))).use { mongoClient -> - val documents = mutableListOf() - for (i in 0..numberOfDocuments) { - documents += Document().append("name", "Document $i") - .append("description", "This is document #$i") - .append("doubleField", i.toDouble()) - .append("intField", i) - .append("objectField", mapOf("key" to "value")) - .append("timestamp", BsonTimestamp(currentTimeMillis())) - } - - mongoClient.getDatabase(databaseName).getCollection(collectionName).insertMany(documents) - } - - println("Inserted $numberOfDocuments document(s) to $databaseName.$collectionName") - } -} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/resources/schema_discovery_response.json b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/resources/schema_discovery_response.json deleted file mode 100644 index 55c665524244..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/resources/schema_discovery_response.json +++ /dev/null @@ -1,31 +0,0 @@ -[ - { - "_id": null, - "fields": [ - { - "_id": "string", - "name": "string", - "last_updated": "date", - "total": "int", - "price": "decimal", - "items": "array", - "owners": "object" - } - ] - }, - { - "_id": null, - "fields": [ - { - "_id": "string", - "name": "string", - "last_updated": "date", - "total": "int", - "price": "decimal", - "items": "array", - "owners": "object", - "other": "string" - } - ] - } -] diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/resources/schema_discovery_response_different_datatypes.json b/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/resources/schema_discovery_response_different_datatypes.json deleted file mode 100644 index f487d4d80404..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/resources/schema_discovery_response_different_datatypes.json +++ /dev/null @@ -1,31 +0,0 @@ -[ - { - "_id": null, - "fields": [ - { - "_id": "string", - "name": "string", - "last_updated": "date", - "total": "int", - "price": "decimal", - "items": "array", - "owners": "object" - } - ] - }, - { - "_id": null, - "fields": [ - { - "_id": "string", - "name": "string", - "last_updated": "date", - "total": "string", - "price": "decimal", - "items": "array", - "owners": "object", - "other": "string" - } - ] - } -] diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/.dockerignore b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/Dockerfile deleted file mode 100644 index 437a991592c6..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-mongodb-strict-encrypt - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-mongodb-strict-encrypt - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.2.5 -LABEL io.airbyte.name=airbyte/source-mongodb-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/README.md b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/README.md deleted file mode 100644 index 792f6343877b..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/README.md +++ /dev/null @@ -1,56 +0,0 @@ -# MongoDb Source - -## Documentation -This is the repository for the MongoDb secure only source connector in Java. -For information about how to use this connector within Airbyte, see [User Documentation](https://docs.airbyte.io/integrations/sources/mongodb-v2) - -## Local development - -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-mongodb-strict-encrypt:build -``` - -### Locally running the connector docker image - -#### Build -Build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-mongodb-strict-encrypt:airbyteDocker -``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. - -## Testing -We use `JUnit` for Java tests. - -### Test Configuration - -## Community Contributor - -As a community contributor, you will need to have MongoDb instance to test MongoDb source. - -1. Create `secrets/credentials.json` file - 1. Insert below json to the file with your configuration - ``` - { - "database": "database_name", - "user": "user", - "password": "password", - "host": "host", - "port": "port" - } - ``` - -## Airbyte Employee - -1. Access the `MONGODB_TEST_CREDS` secret on LastPass -1. Create a file with the contents at `secrets/credentials.json` - - -#### Acceptance Tests -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-mongodb-strict-encrypt:integrationTest -``` diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/build.gradle index 6adacf948ad8..7d8895e4570f 100644 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/build.gradle @@ -1,24 +1,33 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.7.7' + features = ['db-sources'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.source.mongodb.MongodbSourceStrictEncrypt' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - implementation project(':airbyte-integrations:connectors:source-relational-db') implementation project(':airbyte-integrations:connectors:source-mongodb-v2') - implementation 'org.mongodb:mongodb-driver-sync:4.3.0' + implementation project(':airbyte-integrations:connectors:destination-mongodb') + integrationTestJavaImplementation project(':airbyte-integrations:connectors:destination-mongodb') - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-mongodb-strict-encrypt') - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + + implementation 'org.mongodb:mongodb-driver-sync:4.3.0' } diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/icon.svg b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/icon.svg deleted file mode 100644 index 66b68e75556d..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/icon.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/metadata.yaml deleted file mode 100644 index 791eed372111..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/metadata.yaml +++ /dev/null @@ -1,20 +0,0 @@ -data: - registries: - cloud: - enabled: false # strict encrypt connectors are deployed to Cloud by their non strict encrypt sibling. - oss: - enabled: false # strict encrypt connectors are not used on OSS. - connectorSubtype: database - connectorType: source - definitionId: b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e - dockerImageTag: 0.2.5 - dockerRepository: airbyte/source-mongodb-strict-encrypt - githubIssueLabel: source-mongodb-v2 - icon: mongodb.svg - license: ELv2 - name: MongoDb - releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/mongodb-v2 - tags: - - language:java -metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.source.mongodb/MongodbSourceStrictEncrypt.java b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.source.mongodb/MongodbSourceStrictEncrypt.java index e01d6fd787e2..834c97308143 100644 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.source.mongodb/MongodbSourceStrictEncrypt.java +++ b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/main/java/io.airbyte.integrations.source.mongodb/MongodbSourceStrictEncrypt.java @@ -6,13 +6,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.base.spec_modification.SpecModifyingSource; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.mongodb.MongoUtils; -import io.airbyte.db.mongodb.MongoUtils.MongoInstanceType; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.base.spec_modification.SpecModifyingSource; +import io.airbyte.integrations.destination.mongodb.MongoUtils; +import io.airbyte.integrations.destination.mongodb.MongoUtils.MongoInstanceType; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.ConnectorSpecification; import org.slf4j.Logger; diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongodbSourceStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongodbSourceStrictEncryptAcceptanceTest.java index 048a5ff9e59d..6e47cc8693bd 100644 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongodbSourceStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongodbSourceStrictEncryptAcceptanceTest.java @@ -13,14 +13,14 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.mongodb.client.MongoCollection; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.mongodb.MongoDatabase; -import io.airbyte.db.mongodb.MongoUtils.MongoInstanceType; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.integrations.destination.mongodb.MongoDatabase; +import io.airbyte.integrations.destination.mongodb.MongoUtils.MongoInstanceType; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.CatalogHelpers; diff --git a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/resources/expected_spec.json b/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/resources/expected_spec.json deleted file mode 100644 index 48be1e68bb2f..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-strict-encrypt/src/test-integration/resources/expected_spec.json +++ /dev/null @@ -1,114 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/mongodb-v2", - "changelogUrl": "https://docs.airbyte.com/integrations/sources/mongodb-v2", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MongoDb Source Spec", - "type": "object", - "required": ["database"], - "additionalProperties": true, - "properties": { - "instance_type": { - "type": "object", - "title": "MongoDb Instance Type", - "description": "The MongoDb instance to connect to. For MongoDB Atlas and Replica Set TLS connection is used by default.", - "order": 0, - "oneOf": [ - { - "title": "Standalone MongoDb Instance", - "required": ["instance", "host", "port"], - "properties": { - "instance": { - "type": "string", - "const": "standalone" - }, - "host": { - "title": "Host", - "type": "string", - "description": "The host name of the Mongo database.", - "order": 0 - }, - "port": { - "title": "Port", - "type": "integer", - "description": "The port of the Mongo database.", - "minimum": 0, - "maximum": 65536, - "default": 27017, - "examples": ["27017"], - "order": 1 - } - } - }, - { - "title": "Replica Set", - "required": ["instance", "server_addresses"], - "properties": { - "instance": { - "type": "string", - "const": "replica" - }, - "server_addresses": { - "title": "Server Addresses", - "type": "string", - "description": "The members of a replica set. Please specify `host`:`port` of each member separated by comma.", - "examples": ["host1:27017,host2:27017,host3:27017"], - "order": 0 - }, - "replica_set": { - "title": "Replica Set", - "type": "string", - "description": "A replica set in MongoDB is a group of mongod processes that maintain the same data set.", - "order": 1 - } - } - }, - { - "title": "MongoDB Atlas", - "additionalProperties": true, - "required": ["instance", "cluster_url"], - "properties": { - "instance": { - "type": "string", - "const": "atlas" - }, - "cluster_url": { - "title": "Cluster URL", - "type": "string", - "description": "The URL of a cluster to connect to.", - "order": 0 - } - } - } - ] - }, - "database": { - "title": "Database Name", - "type": "string", - "description": "The database you want to replicate.", - "order": 1 - }, - "user": { - "title": "User", - "type": "string", - "description": "The username which is used to access the database.", - "order": 2 - }, - "password": { - "title": "Password", - "type": "string", - "description": "The password associated with this username.", - "airbyte_secret": true, - "order": 3 - }, - "auth_source": { - "title": "Authentication Source", - "type": "string", - "description": "The authentication source where the user information is stored.", - "default": "admin", - "examples": ["admin"], - "order": 4 - } - } - } -} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/.dockerignore b/airbyte-integrations/connectors/source-mongodb-v2/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-v2/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-mongodb-v2/Dockerfile b/airbyte-integrations/connectors/source-mongodb-v2/Dockerfile deleted file mode 100644 index c59d6f6d0d1f..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-v2/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-mongodb-v2 - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-mongodb-v2 - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.2.5 -LABEL io.airbyte.name=airbyte/source-mongodb-v2 diff --git a/airbyte-integrations/connectors/source-mongodb-v2/README.md b/airbyte-integrations/connectors/source-mongodb-v2/README.md index 45570207bfbc..0b648be99cdf 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/README.md +++ b/airbyte-integrations/connectors/source-mongodb-v2/README.md @@ -16,10 +16,11 @@ From the Airbyte repository root, run: #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:source-mongodb-v2:airbyteDocker +./gradlew :airbyte-integrations:connectors:source-mongodb-v2:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/source-mongodb-v2:dev`. the Dockerfile. ## Testing @@ -37,13 +38,16 @@ As a community contributor, you will need to have an Atlas cluster to test Mongo 1. Create `secrets/credentials.json` file 1. Insert below json to the file with your configuration ``` - { - "database": "database_name", - "user": "user", - "password": "password", - "cluster_url": "cluster_url" + { + "cluster_type": "ATLAS_REPLICA_SET" + "database": "database_name", + "username": "username", + "password": "password", + "connection_string": "mongodb+srv://cluster0.abcd1.mongodb.net/", + "auth_source": "auth_database", } ``` + where `installation_type` is one of `ATLAS_REPLICA_SET` or `SELF_HOSTED_REPLICA_SET` depending on the location of the target cluster. ## Airbyte Employee diff --git a/airbyte-integrations/connectors/source-mongodb-v2/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mongodb-v2/acceptance-test-config.yml index b9fbb0c58c78..74f7ca1a60a7 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-mongodb-v2/acceptance-test-config.yml @@ -5,6 +5,8 @@ acceptance_tests: spec: tests: - spec_path: "integration_tests/expected_spec.json" + backward_compatibility_tests_config: + disable_for_version: "0.2.5" config_path: "secrets/credentials.json" timeout_seconds: 60 connection: @@ -15,15 +17,17 @@ acceptance_tests: discovery: tests: - config_path: "secrets/credentials.json" + backward_compatibility_tests_config: + disable_for_version: "0.2.5" timeout_seconds: 60 basic_read: - tests: - - config_path: "secrets/credentials.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 120 + bypass_reason: "Full refresh syncs are not supported on this connector." full_refresh: + bypass_reason: "Full refresh syncs are not supported on this connector." + incremental: + # bypass_reason: "Incremental syncs are not yet supported by this connector." + # #TODO: remove bypass_reason once a version that supports incremental syncs is released + uncomment the lines below tests: - config_path: "secrets/credentials.json" configured_catalog_path: "integration_tests/configured_catalog.json" timeout_seconds: 180 - diff --git a/airbyte-integrations/connectors/source-mongodb-v2/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-mongodb-v2/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-v2/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-mongodb-v2/build.gradle b/airbyte-integrations/connectors/source-mongodb-v2/build.gradle index 896e88e74958..ed08fabcc683 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/build.gradle +++ b/airbyte-integrations/connectors/source-mongodb-v2/build.gradle @@ -1,27 +1,134 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' - id 'airbyte-connector-acceptance-test' + id 'airbyte-java-connector' + alias(libs.plugins.kotlin.jvm) } +airbyteJavaConnector { + cdkVersionRequired = '0.13.2' + features = ['db-sources'] + useLocalCdk = false +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.source.mongodb.MongoDbSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } -dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - implementation project(':airbyte-integrations:connectors:source-relational-db') +configurations { + dataGenerator.extendsFrom testImplementation + debeziumTest.extendsFrom testImplementation +} - implementation 'org.mongodb:mongodb-driver-sync:4.4.0' +sourceSets { + dataGenerator { + kotlin { + srcDirs('src/test/generator') + } + } + debeziumTest { + kotlin { + srcDirs('src/test/debezium') + } + } +} + +dependencies { + implementation libs.mongo.driver.sync - testImplementation libs.connectors.testcontainers.mongodb + testImplementation libs.testcontainers.mongodb - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') + integrationTestJavaImplementation libs.apache.commons.lang integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-mongodb-v2') - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + + dataGeneratorImplementation project(':airbyte-cdk:java:airbyte-cdk:airbyte-commons') + + dataGeneratorImplementation project(':airbyte-integrations:connectors:source-mongodb-v2') + dataGeneratorImplementation libs.mongo.driver.sync + dataGeneratorImplementation libs.kotlin.logging + dataGeneratorImplementation libs.kotlinx.cli + dataGeneratorImplementation (libs.java.faker) { + exclude module: 'snakeyaml' + } + dataGeneratorImplementation libs.jackson.databind + dataGeneratorImplementation libs.bundles.slf4j + dataGeneratorImplementation libs.slf4j.simple + dataGeneratorImplementation libs.kotlinx.cli.jvm + dataGeneratorImplementation 'org.yaml:snakeyaml:2.2' + + debeziumTestImplementation libs.debezium.api + debeziumTestImplementation libs.debezium.embedded + debeziumTestImplementation libs.debezium.sqlserver + debeziumTestImplementation libs.debezium.mysql + debeziumTestImplementation libs.debezium.postgres + debeziumTestImplementation libs.debezium.mongodb + debeziumTestImplementation libs.bundles.slf4j + debeziumTestImplementation libs.slf4j.simple + debeziumTestImplementation libs.kotlinx.cli.jvm + debeziumTestImplementation libs.spotbugs.annotations +} + +/* + * Executes the script that generates test data and inserts it into the provided database/collection. + * + * To execute this task, use the following command: + * + * ./gradlew :airbyte-integrations:connectors:source-mongodb-v2:generateTestData -PconnectionString= -PdatabaseName= -PcollectionName= -Pusername= + * + * Optionally, you can provide -PnumberOfDocuments to change the number of generated documents from the default (10,000). + */ +tasks.register('generateTestData', JavaExec) { + def arguments = [] + + if(project.hasProperty('collectionName')) { + arguments.addAll(['--collection-name', collectionName]) + } + if(project.hasProperty('connectionString')) { + arguments.addAll(['--connection-string', connectionString]) + } + if(project.hasProperty('databaseName')) { + arguments.addAll(['--database-name', databaseName]) + } + if (project.hasProperty('numberOfDocuments')) { + arguments.addAll(['--number', numberOfDocuments]) + } + if(project.hasProperty('username')) { + arguments.addAll(['--username', username]) + } + + classpath = sourceSets.dataGenerator.runtimeClasspath + main 'io.airbyte.integrations.source.mongodb.MongoDbInsertClient' + standardInput = System.in + args arguments +} + +/** + * Executes the Debezium MongoDB Connector test harness. + * + * To execute this task, use the following command: + * + * ./gradlew :airbyte-integrations:connectors:source-mongodb-v2:debeziumTest -PconnectionString= -PdatabaseName= -PcollectionName= -Pusername= + */ +tasks.register('debeziumTest', JavaExec) { + def arguments = [] + + if(project.hasProperty('collectionName')) { + arguments.addAll(['--collection-name', collectionName]) + } + if(project.hasProperty('connectionString')) { + arguments.addAll(['--connection-string', connectionString]) + } + if(project.hasProperty('databaseName')) { + arguments.addAll(['--database-name', databaseName]) + } + if(project.hasProperty('username')) { + arguments.addAll(['--username', username]) + } + + classpath = sourceSets.debeziumTest.runtimeClasspath + main 'io.airbyte.integrations.source.mongodb.DebeziumMongoDbConnectorTest' + standardInput = System.in + args arguments } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/icon.svg b/airbyte-integrations/connectors/source-mongodb-v2/icon.svg index 66b68e75556d..c5ec7a168e4d 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/icon.svg +++ b/airbyte-integrations/connectors/source-mongodb-v2/icon.svg @@ -1 +1,3 @@ - \ No newline at end of file + + + diff --git a/airbyte-integrations/connectors/source-mongodb-v2/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-mongodb-v2/integration_tests/configured_catalog.json index d8637ff7bdfa..32f5f5691b81 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-mongodb-v2/integration_tests/configured_catalog.json @@ -269,12 +269,12 @@ } } }, - "supported_sync_modes": ["full_refresh", "incremental"], + "supported_sync_modes": ["incremental"], "default_cursor_field": [], "source_defined_primary_key": [], "namespace": "sample_airbnb" }, - "sync_mode": "full_refresh", + "sync_mode": "incremental", "destination_sync_mode": "overwrite" } ] diff --git a/airbyte-integrations/connectors/source-mongodb-v2/integration_tests/expected_spec.json b/airbyte-integrations/connectors/source-mongodb-v2/integration_tests/expected_spec.json index 516813e0c3d1..e77e9d632780 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/integration_tests/expected_spec.json +++ b/airbyte-integrations/connectors/source-mongodb-v2/integration_tests/expected_spec.json @@ -5,118 +5,179 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "MongoDb Source Spec", "type": "object", - "required": ["database"], + "required": ["database_config"], "additionalProperties": true, "properties": { - "instance_type": { + "database_config": { "type": "object", - "title": "MongoDb Instance Type", - "description": "The MongoDb instance to connect to. For MongoDB Atlas and Replica Set TLS connection is used by default.", - "order": 0, + "title": "Cluster Type", + "description": "Configures the MongoDB cluster type.", + "order": 1, + "group": "connection", + "display_type": "radio", "oneOf": [ { - "title": "Standalone MongoDb Instance", - "required": ["instance", "host", "port"], + "title": "MongoDB Atlas Replica Set", + "description": "MongoDB Atlas-hosted cluster configured as a replica set", + "required": [ + "cluster_type", + "connection_string", + "database", + "username", + "password", + "auth_source" + ], + "additionalProperties": true, "properties": { - "instance": { + "cluster_type": { "type": "string", - "const": "standalone" + "const": "ATLAS_REPLICA_SET", + "order": 1 }, - "host": { - "title": "Host", + "connection_string": { + "title": "Connection String", "type": "string", - "description": "The host name of the Mongo database.", - "order": 0 + "description": "The connection string of the cluster that you want to replicate.", + "examples": ["mongodb+srv://cluster0.abcd1.mongodb.net/"], + "order": 2 }, - "port": { - "title": "Port", - "type": "integer", - "description": "The port of the Mongo database.", - "minimum": 0, - "maximum": 65536, - "default": 27017, - "examples": ["27017"], - "order": 1 + "database": { + "title": "Database Name", + "type": "string", + "description": "The name of the MongoDB database that contains the collection(s) to replicate.", + "order": 3 }, - "tls": { - "title": "TLS Connection", - "type": "boolean", - "description": "Indicates whether TLS encryption protocol will be used to connect to MongoDB. It is recommended to use TLS connection if possible. For more information see documentation.", - "default": false, - "order": 2 - } - } - }, - { - "title": "Replica Set", - "required": ["instance", "server_addresses"], - "properties": { - "instance": { + "username": { + "title": "Username", "type": "string", - "const": "replica" + "description": "The username which is used to access the database.", + "order": 4 }, - "server_addresses": { - "title": "Server Addresses", + "password": { + "title": "Password", "type": "string", - "description": "The members of a replica set. Please specify `host`:`port` of each member separated by comma.", - "examples": ["host1:27017,host2:27017,host3:27017"], - "order": 0 + "description": "The password associated with this username.", + "airbyte_secret": true, + "order": 5 }, - "replica_set": { - "title": "Replica Set", + "auth_source": { + "title": "Authentication Source", "type": "string", - "description": "A replica set in MongoDB is a group of mongod processes that maintain the same data set.", - "order": 1 + "description": "The authentication source where the user information is stored. See https://www.mongodb.com/docs/manual/reference/connection-string/#mongodb-urioption-urioption.authSource for more details.", + "default": "admin", + "examples": ["admin"], + "order": 6 + }, + "schema_enforced": { + "title": "Schema Enforced", + "description": "When enabled, syncs will validate and structure records against the stream's schema.", + "default": true, + "type": "boolean", + "always_show": true, + "order": 7 } } }, { - "title": "MongoDB Atlas", + "title": "Self-Managed Replica Set", + "description": "MongoDB self-hosted cluster configured as a replica set", + "required": ["cluster_type", "connection_string", "database"], "additionalProperties": true, - "required": ["instance", "cluster_url"], "properties": { - "instance": { + "cluster_type": { + "type": "string", + "const": "SELF_MANAGED_REPLICA_SET", + "order": 1 + }, + "connection_string": { + "title": "Connection String", + "type": "string", + "description": "The connection string of the cluster that you want to replicate. https://www.mongodb.com/docs/manual/reference/connection-string/#find-your-self-hosted-deployment-s-connection-string for more information.", + "examples": [ + "mongodb://example1.host.com:27017,example2.host.com:27017,example3.host.com:27017/", + "mongodb://example.host.com:27017/" + ], + "order": 2 + }, + "database": { + "title": "Database Name", "type": "string", - "const": "atlas" + "description": "The name of the MongoDB database that contains the collection(s) to replicate.", + "order": 3 }, - "cluster_url": { - "title": "Cluster URL", + "username": { + "title": "Username", "type": "string", - "description": "The URL of a cluster to connect to.", - "order": 0 + "description": "The username which is used to access the database.", + "order": 4 + }, + "password": { + "title": "Password", + "type": "string", + "description": "The password associated with this username.", + "airbyte_secret": true, + "order": 5 + }, + "auth_source": { + "title": "Authentication Source", + "type": "string", + "description": "The authentication source where the user information is stored.", + "default": "admin", + "examples": ["admin"], + "order": 6 + }, + "schema_enforced": { + "title": "Schema Enforced", + "description": "When enabled, syncs will validate and structure records against the stream's schema.", + "default": true, + "type": "boolean", + "always_show": true, + "order": 7 } } } ] }, - "database": { - "title": "Database Name", - "type": "string", - "description": "The database you want to replicate.", - "order": 1 + "initial_waiting_seconds": { + "type": "integer", + "title": "Initial Waiting Time in Seconds (Advanced)", + "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds.", + "default": 300, + "order": 8, + "min": 120, + "max": 1200, + "group": "advanced" }, - "user": { - "title": "User", - "type": "string", - "description": "The username which is used to access the database.", - "order": 2 + "queue_size": { + "type": "integer", + "title": "Size of the queue (Advanced)", + "description": "The size of the internal queue. This may interfere with memory consumption and efficiency of the connector, please be careful.", + "default": 10000, + "order": 9, + "min": 1000, + "max": 10000, + "group": "advanced" }, - "password": { - "title": "Password", - "type": "string", - "description": "The password associated with this username.", - "airbyte_secret": true, - "order": 3 + "discover_sample_size": { + "type": "integer", + "title": "Document discovery sample size (Advanced)", + "description": "The maximum number of documents to sample when attempting to discover the unique fields for a collection.", + "default": 10000, + "order": 10, + "minimum": 10, + "maximum": 100000, + "group": "advanced" + } + }, + "groups": [ + { + "id": "connection" }, - "auth_source": { - "title": "Authentication Source", - "type": "string", - "description": "The authentication source where the user information is stored.", - "default": "admin", - "examples": ["admin"], - "order": 4 + { + "id": "advanced", + "title": "Advanced" } - } + ] }, "supported_destination_sync_modes": [] } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml b/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml index b489335c356c..3bba900cd503 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml +++ b/airbyte-integrations/connectors/source-mongodb-v2/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: source definitionId: b2e713cd-cc36-4c0a-b5bd-b47cb8a0561e - dockerImageTag: 0.2.5 + dockerImageTag: 1.2.3 dockerRepository: airbyte/source-mongodb-v2 documentationUrl: https://docs.airbyte.com/integrations/sources/mongodb-v2 githubIssueLabel: source-mongodb-v2 @@ -14,14 +14,17 @@ data: name: MongoDb registries: cloud: - dockerImageTag: 0.2.5 - dockerRepository: airbyte/source-mongodb-strict-encrypt enabled: true oss: enabled: true - releaseStage: alpha - supportLevel: community + releaseStage: generally_available + supportLevel: certified tags: - language:java - - language:python + releases: + breakingChanges: + 1.0.0: + message: > + **We advise against upgrading until you have run a test upgrade as outlined [here](https://docs.airbyte.com/integrations/sources/mongodb-v2-migrations).** This version brings a host of updates to the MongoDB source connector, significantly increasing its scalability and reliability, especially for large collections. As of this version with checkpointing, [CDC incremental updates](https://docs.airbyte.com/understanding-airbyte/cdc) and improved schema discovery, this connector is also now [certified](https://docs.airbyte.com/integrations/). Selecting `Upgrade` will upgrade **all** connections using this source, require you to reconfigure the source, then run a full reset on **all** of your connections. + upgradeDeadline: "2023-12-01" metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io.airbyte.integrations.source.mongodb/MongoDbSource.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io.airbyte.integrations.source.mongodb/MongoDbSource.java deleted file mode 100644 index f0d3ab7ba877..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io.airbyte.integrations.source.mongodb/MongoDbSource.java +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mongodb; - -import static com.mongodb.client.model.Filters.gt; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import com.mongodb.MongoCommandException; -import com.mongodb.MongoException; -import com.mongodb.MongoSecurityException; -import com.mongodb.client.MongoCollection; -import io.airbyte.commons.exceptions.ConnectionErrorException; -import io.airbyte.commons.functional.CheckedConsumer; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.stream.AirbyteStreamUtils; -import io.airbyte.commons.util.AutoCloseableIterator; -import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.mongodb.MongoDatabase; -import io.airbyte.db.mongodb.MongoUtils; -import io.airbyte.db.mongodb.MongoUtils.MongoInstanceType; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.source.relationaldb.AbstractDbSource; -import io.airbyte.integrations.source.relationaldb.CursorInfo; -import io.airbyte.integrations.source.relationaldb.TableInfo; -import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; -import io.airbyte.protocol.models.CommonField; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.SyncMode; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.commons.lang3.StringUtils; -import org.bson.BsonType; -import org.bson.Document; -import org.bson.conversions.Bson; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class MongoDbSource extends AbstractDbSource { - - private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbSource.class); - - public static void main(final String[] args) throws Exception { - final Source source = new MongoDbSource(); - LOGGER.info("starting source: {}", MongoDbSource.class); - new IntegrationRunner(source).run(args); - LOGGER.info("completed source: {}", MongoDbSource.class); - } - - @Override - public JsonNode toDatabaseConfig(final JsonNode config) { - final var credentials = config.has(MongoUtils.USER) && config.has(JdbcUtils.PASSWORD_KEY) - ? String.format("%s:%s@", config.get(MongoUtils.USER).asText(), config.get(JdbcUtils.PASSWORD_KEY).asText()) - : StringUtils.EMPTY; - - return Jsons.jsonNode(ImmutableMap.builder() - .put("connectionString", buildConnectionString(config, credentials)) - .put(JdbcUtils.DATABASE_KEY, config.get(JdbcUtils.DATABASE_KEY).asText()) - .build()); - } - - @Override - protected MongoDatabase createDatabase(final JsonNode sourceConfig) throws Exception { - final var dbConfig = toDatabaseConfig(sourceConfig); - final MongoDatabase database = new MongoDatabase(dbConfig.get("connectionString").asText(), - dbConfig.get(JdbcUtils.DATABASE_KEY).asText()); - database.setSourceConfig(sourceConfig); - database.setDatabaseConfig(toDatabaseConfig(sourceConfig)); - return database; - } - - @Override - public List> getCheckOperations(final JsonNode config) { - final List> checkList = new ArrayList<>(); - checkList.add(database -> { - if (getAuthorizedCollections(database).isEmpty()) { - throw new ConnectionErrorException("Unable to execute any operation on the source!"); - } else { - LOGGER.info("The source passed the basic operation test!"); - } - }); - return checkList; - } - - @Override - protected JsonSchemaType getAirbyteType(final BsonType fieldType) { - return MongoUtils.getType(fieldType); - } - - @Override - public Set getExcludedInternalNameSpaces() { - return Collections.emptySet(); - } - - @Override - protected List>> discoverInternal(final MongoDatabase database) - throws Exception { - final List>> tableInfos = new ArrayList<>(); - - final Set authorizedCollections = getAuthorizedCollections(database); - authorizedCollections.parallelStream().forEach(collectionName -> { - final MongoCollection collection = database.getCollection(collectionName); - final List> fields = MongoUtils.getUniqueFields(collection).stream().map(MongoUtils::nodeToCommonField).toList(); - - // The field name _id is reserved for use as a primary key; - final TableInfo> tableInfo = TableInfo.>builder() - .nameSpace(database.getName()) - .name(collectionName) - .fields(fields) - .primaryKeys(List.of(MongoUtils.PRIMARY_KEY)) - .build(); - - tableInfos.add(tableInfo); - }); - return tableInfos; - } - - private Set getAuthorizedCollections(final MongoDatabase database) { - /* - * db.runCommand ({listCollections: 1.0, authorizedCollections: true, nameOnly: true }) the command - * returns only those collections for which the user has privileges. For example, if a user has find - * action on specific collections, the command returns only those collections; or, if a user has - * find or any other action, on the database resource, the command lists all collections in the - * database. - */ - try { - final Document document = database.getDatabase().runCommand(new Document("listCollections", 1) - .append("authorizedCollections", true) - .append("nameOnly", true)) - .append("filter", "{ 'type': 'collection' }"); - return document.toBsonDocument() - .get("cursor").asDocument() - .getArray("firstBatch") - .stream() - .map(bsonValue -> bsonValue.asDocument().getString("name").getValue()) - .collect(Collectors.toSet()); - - } catch (final MongoSecurityException e) { - final MongoCommandException exception = (MongoCommandException) e.getCause(); - throw new ConnectionErrorException(String.valueOf(exception.getCode()), e); - } catch (final MongoException e) { - throw new ConnectionErrorException(String.valueOf(e.getCode()), e); - } - } - - @Override - protected List>> discoverInternal(final MongoDatabase database, final String schema) throws Exception { - // MondoDb doesn't support schemas - return discoverInternal(database); - } - - @Override - protected Map> discoverPrimaryKeys(final MongoDatabase database, - final List>> tableInfos) { - return tableInfos.stream() - .collect(Collectors.toMap( - TableInfo::getName, - TableInfo::getPrimaryKeys)); - } - - @Override - protected String getQuoteString() { - return ""; - } - - @Override - public AutoCloseableIterator queryTableFullRefresh(final MongoDatabase database, - final List columnNames, - final String schemaName, - final String tableName, - final SyncMode syncMode, - final Optional cursorField) { - return queryTable(database, columnNames, tableName, null); - } - - @Override - public AutoCloseableIterator queryTableIncremental(final MongoDatabase database, - final List columnNames, - final String schemaName, - final String tableName, - final CursorInfo cursorInfo, - final BsonType cursorFieldType) { - final Bson greaterComparison = gt(cursorInfo.getCursorField(), MongoUtils.getBsonValue(cursorFieldType, cursorInfo.getCursor())); - return queryTable(database, columnNames, tableName, greaterComparison); - } - - @Override - public boolean isCursorType(final BsonType bsonType) { - // while reading from mongo primary key "id" is always added, so there will be no situation - // when we have no cursor field here, at least id could be used as cursor here. - // This logic will be used feather when we will implement part which will show only list of possible - // cursor fields on UI - return MongoUtils.ALLOWED_CURSOR_TYPES.contains(bsonType); - } - - private AutoCloseableIterator queryTable(final MongoDatabase database, - final List columnNames, - final String tableName, - final Bson filter) { - final AirbyteStreamNameNamespacePair airbyteStream = AirbyteStreamUtils.convertFromNameAndNamespace(tableName, null); - return AutoCloseableIterators.lazyIterator(() -> { - try { - final Stream stream = database.read(tableName, columnNames, Optional.ofNullable(filter)); - return AutoCloseableIterators.fromStream(stream, airbyteStream); - } catch (final Exception e) { - throw new RuntimeException(e); - } - }, airbyteStream); - } - - private String buildConnectionString(final JsonNode config, final String credentials) { - final StringBuilder connectionStrBuilder = new StringBuilder(); - - final JsonNode instanceConfig = config.get(MongoUtils.INSTANCE_TYPE); - final MongoInstanceType instance = MongoInstanceType.fromValue(instanceConfig.get(MongoUtils.INSTANCE).asText()); - switch (instance) { - case STANDALONE -> { - connectionStrBuilder.append( - String.format(MongoUtils.MONGODB_SERVER_URL, credentials, instanceConfig.get(JdbcUtils.HOST_KEY).asText(), - instanceConfig.get(JdbcUtils.PORT_KEY).asText(), - config.get(JdbcUtils.DATABASE_KEY).asText(), - config.get(MongoUtils.AUTH_SOURCE).asText(), MongoUtils.tlsEnabledForStandaloneInstance(config, instanceConfig))); - } - case REPLICA -> { - connectionStrBuilder.append( - String.format(MongoUtils.MONGODB_REPLICA_URL, credentials, instanceConfig.get(MongoUtils.SERVER_ADDRESSES).asText(), - config.get(JdbcUtils.DATABASE_KEY).asText(), - config.get(MongoUtils.AUTH_SOURCE).asText())); - if (instanceConfig.has(MongoUtils.REPLICA_SET)) { - connectionStrBuilder.append(String.format("&replicaSet=%s", instanceConfig.get(MongoUtils.REPLICA_SET).asText())); - } - } - case ATLAS -> { - connectionStrBuilder.append( - String.format(MongoUtils.MONGODB_CLUSTER_URL, credentials, - instanceConfig.get(MongoUtils.CLUSTER_URL).asText(), config.get(JdbcUtils.DATABASE_KEY).asText(), - config.get(MongoUtils.AUTH_SOURCE).asText())); - } - default -> throw new IllegalArgumentException("Unsupported instance type: " + instance); - } - return connectionStrBuilder.toString(); - } - - @Override - public void close() throws Exception {} - -} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/InitialSnapshotHandler.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/InitialSnapshotHandler.java new file mode 100644 index 000000000000..5d7d9ad72587 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/InitialSnapshotHandler.java @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb; + +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.Accumulators; +import com.mongodb.client.model.Aggregates; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; +import com.mongodb.client.model.Sorts; +import io.airbyte.commons.exceptions.ConfigErrorException; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.commons.util.AutoCloseableIterators; +import io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcConnectorMetadataInjector; +import io.airbyte.integrations.source.mongodb.state.IdType; +import io.airbyte.integrations.source.mongodb.state.MongoDbStateManager; +import io.airbyte.integrations.source.mongodb.state.MongoDbStreamState; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.SyncMode; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import org.bson.BsonDocument; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.bson.types.ObjectId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Retrieves iterators used for the initial snapshot + */ +public class InitialSnapshotHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(InitialSnapshotHandler.class); + + /** + * For each given stream configured as incremental sync it will output an iterator that will + * retrieve documents from the given database. Each iterator will start after the last checkpointed + * document, if any, or from the beginning of the stream otherwise. + */ + public List> getIterators( + final List streams, + final MongoDbStateManager stateManager, + final MongoDatabase database, + final MongoDbCdcConnectorMetadataInjector cdcConnectorMetadataInjector, + final Instant emittedAt, + final int checkpointInterval, + final boolean isEnforceSchema) { + return streams + .stream() + .peek(airbyteStream -> { + if (!airbyteStream.getSyncMode().equals(SyncMode.INCREMENTAL)) + LOGGER.warn("Stream {} configured with unsupported sync mode: {}", airbyteStream.getStream().getName(), airbyteStream.getSyncMode()); + }) + .filter(airbyteStream -> airbyteStream.getSyncMode().equals(SyncMode.INCREMENTAL)) + .map(airbyteStream -> { + final var collectionName = airbyteStream.getStream().getName(); + final var collection = database.getCollection(collectionName); + final var fields = Projections.fields(Projections.include(CatalogHelpers.getTopLevelFieldNames(airbyteStream).stream().toList())); + + final var idTypes = aggregateIdField(collection); + if (idTypes.size() > 1) { + throw new ConfigErrorException("The _id fields in a collection must be consistently typed (collection = " + collectionName + ")."); + } + + idTypes.stream().findFirst().ifPresent(idType -> { + if (IdType.findByBsonType(idType).isEmpty()) { + throw new ConfigErrorException("Only _id fields with the following types are currently supported: " + IdType.SUPPORTED + + " (collection = " + collectionName + ")."); + } + }); + + // find the existing state, if there is one, for this stream + final Optional existingState = + stateManager.getStreamState(airbyteStream.getStream().getName(), airbyteStream.getStream().getNamespace()); + + // The filter determines the starting point of this iterator based on the state of this collection. + // If a state exists, it will use that state to create a query akin to + // "where _id > [last saved state] order by _id ASC". + // If no state exists, it will create a query akin to "where 1=1 order by _id ASC" + final Bson filter = existingState + // TODO add type support here when we add support for _id fields that are not ObjectId types + .map(state -> Filters.gt(MongoConstants.ID_FIELD, new ObjectId(state.id()))) + // if nothing was found, return a new BsonDocument + .orElseGet(BsonDocument::new); + + // When schema is enforced we query for the selected fields + // Otherwise we retreive the entire set of fields + final var cursor = isEnforceSchema ? collection.find() + .filter(filter) + .projection(fields) + .sort(Sorts.ascending(MongoConstants.ID_FIELD)) + .allowDiskUse(true) + .cursor() + : collection.find() + .filter(filter) + .sort(Sorts.ascending(MongoConstants.ID_FIELD)) + .allowDiskUse(true) + .cursor(); + + final var stateIterator = + new MongoDbStateIterator(cursor, stateManager, Optional.ofNullable(cdcConnectorMetadataInjector), + airbyteStream, emittedAt, checkpointInterval, MongoConstants.CHECKPOINT_DURATION, isEnforceSchema); + return AutoCloseableIterators.fromIterator(stateIterator, cursor::close, null); + }) + .toList(); + } + + /** + * Returns a list of types (as strings) that the _id field has for the provided collection. + * + * @param collection Collection to aggregate the _id types of. + * @return List of bson types (as strings) that the _id field contains. + */ + private List aggregateIdField(final MongoCollection collection) { + final List idTypes = new ArrayList<>(); + /* + * Sanity check that all ID_FIELD values are of the same type for this collection. + * db.collection.aggregate([{ $group : { _id : { $type : "$_id" }, count : { $sum : 1 } } }]) + */ + collection.aggregate(List.of( + Aggregates.group( + new Document(MongoConstants.ID_FIELD, new Document("$type", "$_id")), + Accumulators.sum("count", 1)))) + .forEach(document -> { + // the document will be in the structure of + // {"_id": {"_id": "[TYPE]"}, "count": [COUNT]} + // where [TYPE] is the bson type (objectId, string, etc.) and [COUNT] is the number of documents of + // that type + final Document innerDocument = document.get(MongoConstants.ID_FIELD, Document.class); + idTypes.add(innerDocument.get(MongoConstants.ID_FIELD).toString()); + }); + + return idTypes; + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoCatalogHelper.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoCatalogHelper.java new file mode 100644 index 000000000000..37f0c51dd1ba --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoCatalogHelper.java @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcConnectorMetadataInjector; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.v0.AirbyteCatalog; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.SyncMode; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Collection of utility methods for generating the {@link AirbyteCatalog}. + */ +public class MongoCatalogHelper { + + /** + * The default cursor field name. + */ + public static final String DEFAULT_CURSOR_FIELD = MongoDbCdcConnectorMetadataInjector.CDC_DEFAULT_CURSOR; + + /** + * The default primary key field name. + */ + public static final String DEFAULT_PRIMARY_KEY = MongoConstants.ID_FIELD; + + /** + * The list of supported sync modes for a given stream. + */ + public static final List SUPPORTED_SYNC_MODES = List.of(SyncMode.INCREMENTAL); + + /** + * Name of the property in the JSON representation of an Airbyte stream that contains the discovered + * fields. + */ + public static final String AIRBYTE_STREAM_PROPERTIES = "properties"; + + /** + * Builds an {@link AirbyteStream} with the correct configuration for this source. + * + * @param streamName The name of the stream. + * @param streamNamespace The namespace of the stream. + * @param fields The fields associated with the stream. + * @return The configured {@link AirbyteStream} for this source. + */ + public static AirbyteStream buildAirbyteStream(final String streamName, final String streamNamespace, final List fields) { + return addCdcMetadataColumns(CatalogHelpers.createAirbyteStream(streamName, streamNamespace, fields) + .withSupportedSyncModes(SUPPORTED_SYNC_MODES) + .withSourceDefinedCursor(true) + .withDefaultCursorField(List.of(DEFAULT_CURSOR_FIELD)) + .withSourceDefinedPrimaryKey(List.of(List.of(DEFAULT_PRIMARY_KEY)))); + } + + /** + * Builds an {@link AirbyteStream} with the correct configuration for this source, in schemaless + * mode. All fields are stripped out and the only fields kept are _id, _data, and the CDC fields. + * + * @param streamName The name of the stream. + * @param streamNamespace The namespace of the stream. + * @param fields The fields associated with the stream. + * @return The configured {@link AirbyteStream} for this source. + */ + public static AirbyteStream buildSchemalessAirbyteStream(final String streamName, final String streamNamespace, final List fields) { + // The packed airbyte catalog should only contain the _id field. + final List idFieldList = fields.stream().filter(field -> field.getName().equals(MongoConstants.ID_FIELD)).collect(Collectors.toList()); + return addDataMetadataColumn(buildAirbyteStream(streamName, streamNamespace, idFieldList)); + } + + /** + * Adds CDC metadata columns to the stream. + * + * @param stream An {@link AirbyteStream}. + * @return The modified {@link AirbyteStream}. + */ + private static AirbyteStream addCdcMetadataColumns(final AirbyteStream stream) { + final ObjectNode jsonSchema = (ObjectNode) stream.getJsonSchema(); + final ObjectNode properties = (ObjectNode) jsonSchema.get(AIRBYTE_STREAM_PROPERTIES); + MongoDbCdcConnectorMetadataInjector.addCdcMetadataColumns(properties); + return stream; + } + + /** + * Adds the data metadata columns to the stream, for schemaless (packed) mode. + * + * @param stream An {@link AirbyteStream}. + * @return The modified {@link AirbyteStream}. + */ + private static AirbyteStream addDataMetadataColumn(final AirbyteStream stream) { + final ObjectNode jsonSchema = (ObjectNode) stream.getJsonSchema(); + final ObjectNode properties = (ObjectNode) jsonSchema.get(AIRBYTE_STREAM_PROPERTIES); + addSchemalessModeDataColumn(properties); + return stream; + } + + private static ObjectNode addSchemalessModeDataColumn(final ObjectNode properties) { + final JsonNode objectType = Jsons.jsonNode(ImmutableMap.of("type", "object")); + properties.set(MongoConstants.SCHEMALESS_MODE_DATA_FIELD, objectType); + return properties; + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoConnectionUtils.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoConnectionUtils.java new file mode 100644 index 000000000000..c3a363adc63e --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoConnectionUtils.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb; + +import static io.airbyte.integrations.source.mongodb.MongoConstants.DRIVER_NAME; + +import com.mongodb.ConnectionString; +import com.mongodb.MongoClientSettings; +import com.mongodb.MongoCredential; +import com.mongodb.MongoDriverInformation; +import com.mongodb.ReadPreference; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumPropertiesManager; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; + +/** + * Helper utility for building a {@link MongoClient}. + */ +public class MongoConnectionUtils { + + /** + * Creates a new {@link MongoClient} from the source configuration. + * + * @param config The source's configuration. + * @return The configured {@link MongoClient}. + */ + public static MongoClient createMongoClient(final MongoDbSourceConfig config) { + final ConnectionString mongoConnectionString = new ConnectionString(buildConnectionString(config)); + + final MongoDriverInformation mongoDriverInformation = MongoDriverInformation.builder() + .driverName(DRIVER_NAME) + .build(); + + final MongoClientSettings.Builder mongoClientSettingsBuilder = MongoClientSettings.builder() + .applyConnectionString(mongoConnectionString) + .readPreference(ReadPreference.secondaryPreferred()); + + if (config.hasAuthCredentials()) { + final String authSource = config.getAuthSource(); + final String user = URLEncoder.encode(config.getUsername(), StandardCharsets.UTF_8); + final String password = config.getPassword(); + mongoClientSettingsBuilder.credential(MongoCredential.createCredential(user, authSource, password.toCharArray())); + } + + return MongoClients.create(mongoClientSettingsBuilder.build(), mongoDriverInformation); + } + + private static String buildConnectionString(final MongoDbSourceConfig config) { + return MongoDbDebeziumPropertiesManager.buildConnectionString(config.rawConfig(), true); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoConstants.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoConstants.java new file mode 100644 index 000000000000..efbcc319b75e --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoConstants.java @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb; + +import io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants; +import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants; +import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.Configuration; +import java.time.Duration; + +public class MongoConstants { + + public static final String AUTH_SOURCE_CONFIGURATION_KEY = MongoDbDebeziumConstants.Configuration.AUTH_SOURCE_CONFIGURATION_KEY; + public static final Integer CHECKPOINT_INTERVAL = DebeziumIteratorConstants.SYNC_CHECKPOINT_RECORDS; + public static final String CHECKPOINT_INTERVAL_CONFIGURATION_KEY = "sync_checkpoint_interval"; + public static final Duration CHECKPOINT_DURATION = DebeziumIteratorConstants.SYNC_CHECKPOINT_DURATION; + public static final String COLLECTION_STATISTICS_COUNT_KEY = "count"; + public static final String COLLECTION_STATISTICS_STORAGE_SIZE_KEY = "size"; + public static final String CONNECTION_STRING_CONFIGURATION_KEY = MongoDbDebeziumConstants.Configuration.CONNECTION_STRING_CONFIGURATION_KEY; + public static final String COUNT_KEY = "count"; + public static final String CREDENTIALS_PLACEHOLDER = MongoDbDebeziumConstants.Configuration.CREDENTIALS_PLACEHOLDER; + public static final String DATABASE_CONFIGURATION_KEY = MongoDbDebeziumConstants.Configuration.DATABASE_CONFIGURATION_KEY; + public static final String DATABASE_CONFIG_CONFIGURATION_KEY = MongoDbDebeziumConstants.Configuration.DATABASE_CONFIG_CONFIGURATION_KEY; + public static final String DEFAULT_AUTH_SOURCE = "admin"; + public static final Integer DEFAULT_DISCOVER_SAMPLE_SIZE = 10000; + public static final String DISCOVER_SAMPLE_SIZE_CONFIGURATION_KEY = "discover_sample_size"; + public static final String DRIVER_NAME = "Airbyte"; + public static final String ID_FIELD = "_id"; + public static final String IS_TEST_CONFIGURATION_KEY = "is_test"; + public static final String PASSWORD_CONFIGURATION_KEY = MongoDbDebeziumConstants.Configuration.PASSWORD_CONFIGURATION_KEY; + public static final String QUEUE_SIZE_CONFIGURATION_KEY = "queue_size"; + public static final String STORAGE_STATS_KEY = "storageStats"; + public static final String USERNAME_CONFIGURATION_KEY = MongoDbDebeziumConstants.Configuration.USERNAME_CONFIGURATION_KEY; + public static final String SCHEMA_ENFORCED_CONFIGURATION_KEY = MongoDbDebeziumConstants.Configuration.SCHEMA_ENFORCED_CONFIGURATION_KEY; + public static final String SCHEMALESS_MODE_DATA_FIELD = Configuration.SCHEMALESS_MODE_DATA_FIELD; + + private MongoConstants() {} + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoDbSource.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoDbSource.java new file mode 100644 index 000000000000..2a6c15c28677 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoDbSource.java @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.mongodb.MongoSecurityException; +import com.mongodb.client.MongoClient; +import com.mongodb.connection.ClusterType; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.commons.util.AutoCloseableIterators; +import io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcConnectorMetadataInjector; +import io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcInitializer; +import io.airbyte.integrations.source.mongodb.state.MongoDbStateManager; +import io.airbyte.protocol.models.v0.*; +import java.time.Instant; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MongoDbSource extends BaseConnector implements Source { + + private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbSource.class); + + private final MongoDbCdcInitializer cdcInitializer; + + public MongoDbSource() { + this(new MongoDbCdcInitializer()); + } + + @VisibleForTesting + MongoDbSource(final MongoDbCdcInitializer cdcInitializer) { + this.cdcInitializer = cdcInitializer; + } + + public static void main(final String[] args) throws Exception { + final Source source = new MongoDbSource(); + LOGGER.info("starting source: {}", MongoDbSource.class); + new IntegrationRunner(source).run(args); + LOGGER.info("completed source: {}", MongoDbSource.class); + } + + @Override + public AirbyteConnectionStatus check(final JsonNode config) { + try { + final MongoDbSourceConfig sourceConfig = new MongoDbSourceConfig(config); + try (final MongoClient mongoClient = createMongoClient(sourceConfig)) { + final String databaseName = sourceConfig.getDatabaseName(); + + if (MongoUtil.checkDatabaseExists(mongoClient, databaseName)) { + /* + * Perform the authorized collections check before the cluster type check. The MongoDB Java driver + * needs to actually execute a command in order to fetch the cluster description. Querying for the + * authorized collections guarantees that the cluster description will be available to the driver. + */ + if (MongoUtil.getAuthorizedCollections(mongoClient, databaseName).isEmpty()) { + return new AirbyteConnectionStatus() + .withMessage("Target MongoDB database does not contain any authorized collections.") + .withStatus(AirbyteConnectionStatus.Status.FAILED); + } + if (!ClusterType.REPLICA_SET.equals(mongoClient.getClusterDescription().getType())) { + LOGGER.error("Target MongoDB instance is not a replica set cluster."); + return new AirbyteConnectionStatus() + .withMessage("Target MongoDB instance is not a replica set cluster.") + .withStatus(AirbyteConnectionStatus.Status.FAILED); + } + } else { + LOGGER.error("Unable to perform connection check. Database '" + databaseName + "' does not exist."); + return new AirbyteConnectionStatus().withStatus(AirbyteConnectionStatus.Status.FAILED) + .withMessage("Database does not exist. Please check the source's configured database name."); + } + } catch (final MongoSecurityException e) { + LOGGER.error("Unable to perform source check operation.", e); + return new AirbyteConnectionStatus() + .withMessage("Authentication failed. Please check the source's configured credentials.") + .withStatus(AirbyteConnectionStatus.Status.FAILED); + } catch (final Exception e) { + LOGGER.error("Unable to perform source check operation.", e); + return new AirbyteConnectionStatus() + .withMessage(e.getMessage()) + .withStatus(AirbyteConnectionStatus.Status.FAILED); + } + + LOGGER.info("The source passed the check operation test!"); + return new AirbyteConnectionStatus().withStatus(AirbyteConnectionStatus.Status.SUCCEEDED); + } catch (final IllegalArgumentException e) { + LOGGER.error("Unable to perform connection check operation.", e); + return new AirbyteConnectionStatus().withStatus(AirbyteConnectionStatus.Status.FAILED) + .withMessage("Unable to perform connection check operation: " + e.getMessage()); + } + } + + @Override + public AirbyteCatalog discover(final JsonNode config) { + try { + final MongoDbSourceConfig sourceConfig = new MongoDbSourceConfig(config); + try (final MongoClient mongoClient = createMongoClient(sourceConfig)) { + final String databaseName = sourceConfig.getDatabaseName(); + final Integer sampleSize = sourceConfig.getSampleSize(); + final boolean isSchemaEnforced = sourceConfig.getEnforceSchema(); + final List streams = MongoUtil.getAirbyteStreams(mongoClient, databaseName, sampleSize, isSchemaEnforced); + return new AirbyteCatalog().withStreams(streams); + } + } catch (final IllegalArgumentException e) { + LOGGER.error("Unable to perform schema discovery operation.", e); + throw e; + } + } + + @Override + public AutoCloseableIterator read(final JsonNode config, + final ConfiguredAirbyteCatalog catalog, + final JsonNode state) { + final var emittedAt = Instant.now(); + final var cdcMetadataInjector = MongoDbCdcConnectorMetadataInjector.getInstance(emittedAt); + final var stateManager = MongoDbStateManager.createStateManager(state); + final MongoDbSourceConfig sourceConfig = new MongoDbSourceConfig(config); + if (catalog != null) { + MongoUtil.checkSchemaModeMismatch(sourceConfig.getEnforceSchema(), + stateManager.getCdcState() != null ? stateManager.getCdcState().schema_enforced() : sourceConfig.getEnforceSchema(), catalog); + } + + try { + // WARNING: do not close the client here since it needs to be used by the iterator + final MongoClient mongoClient = createMongoClient(sourceConfig); + + try { + final var iteratorList = + cdcInitializer.createCdcIterators(mongoClient, cdcMetadataInjector, catalog, + stateManager, emittedAt, sourceConfig); + return AutoCloseableIterators.concatWithEagerClose(iteratorList, AirbyteTraceMessageUtility::emitStreamStatusTrace); + } catch (final Exception e) { + mongoClient.close(); + throw e; + } + } catch (final Exception e) { + LOGGER.error("Unable to perform sync read operation.", e); + throw e; + } + } + + protected MongoClient createMongoClient(final MongoDbSourceConfig config) { + return MongoConnectionUtils.createMongoClient(config); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoDbSourceConfig.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoDbSourceConfig.java new file mode 100644 index 000000000000..3591286490b0 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoDbSourceConfig.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb; + +import static io.airbyte.integrations.source.mongodb.MongoConstants.AUTH_SOURCE_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.CHECKPOINT_INTERVAL; +import static io.airbyte.integrations.source.mongodb.MongoConstants.CHECKPOINT_INTERVAL_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.DATABASE_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.DATABASE_CONFIG_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.DEFAULT_AUTH_SOURCE; +import static io.airbyte.integrations.source.mongodb.MongoConstants.DEFAULT_DISCOVER_SAMPLE_SIZE; +import static io.airbyte.integrations.source.mongodb.MongoConstants.DISCOVER_SAMPLE_SIZE_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.PASSWORD_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.SCHEMA_ENFORCED_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.USERNAME_CONFIGURATION_KEY; + +import com.fasterxml.jackson.databind.JsonNode; +import java.util.OptionalInt; + +/** + * Represents the source's configuration, hiding the details of how the underlying JSON + * configuration is constructed. + * + * @param rawConfig The underlying JSON configuration provided by the connector framework. + */ +public record MongoDbSourceConfig(JsonNode rawConfig) { + + /** + * Constructs a new {@link MongoDbSourceConfig} from the provided raw configuration. + * + * @param rawConfig The underlying JSON configuration provided by the connector framework. + * @throws IllegalArgumentException if the raw configuration does not contain the + * {@link MongoConstants#DATABASE_CONFIG_CONFIGURATION_KEY} key. + */ + public MongoDbSourceConfig(final JsonNode rawConfig) { + if (rawConfig.has(DATABASE_CONFIG_CONFIGURATION_KEY)) { + this.rawConfig = rawConfig.get(DATABASE_CONFIG_CONFIGURATION_KEY); + } else { + throw new IllegalArgumentException("Database configuration is missing required '" + DATABASE_CONFIG_CONFIGURATION_KEY + "' property."); + } + } + + public String getAuthSource() { + return rawConfig.has(AUTH_SOURCE_CONFIGURATION_KEY) ? rawConfig.get(AUTH_SOURCE_CONFIGURATION_KEY).asText(DEFAULT_AUTH_SOURCE) + : DEFAULT_AUTH_SOURCE; + } + + public Integer getCheckpointInterval() { + return rawConfig.has(CHECKPOINT_INTERVAL_CONFIGURATION_KEY) ? rawConfig.get(CHECKPOINT_INTERVAL_CONFIGURATION_KEY).asInt(CHECKPOINT_INTERVAL) + : CHECKPOINT_INTERVAL; + } + + public String getDatabaseName() { + return rawConfig.has(DATABASE_CONFIGURATION_KEY) ? rawConfig.get(DATABASE_CONFIGURATION_KEY).asText() : null; + } + + public OptionalInt getQueueSize() { + return rawConfig.has(MongoConstants.QUEUE_SIZE_CONFIGURATION_KEY) + ? OptionalInt.of(rawConfig.get(MongoConstants.QUEUE_SIZE_CONFIGURATION_KEY).asInt()) + : OptionalInt.empty(); + } + + public String getPassword() { + return rawConfig.has(PASSWORD_CONFIGURATION_KEY) ? rawConfig.get(PASSWORD_CONFIGURATION_KEY).asText() : null; + } + + public String getUsername() { + return rawConfig.has(USERNAME_CONFIGURATION_KEY) ? rawConfig.get(USERNAME_CONFIGURATION_KEY).asText() : null; + } + + public boolean hasAuthCredentials() { + return rawConfig.has(USERNAME_CONFIGURATION_KEY) && rawConfig.has(PASSWORD_CONFIGURATION_KEY); + } + + public Integer getSampleSize() { + if (rawConfig.has(DISCOVER_SAMPLE_SIZE_CONFIGURATION_KEY)) { + return rawConfig.get(DISCOVER_SAMPLE_SIZE_CONFIGURATION_KEY).asInt(DEFAULT_DISCOVER_SAMPLE_SIZE); + } else { + return DEFAULT_DISCOVER_SAMPLE_SIZE; + } + } + + public boolean getEnforceSchema() { + return rawConfig.has(SCHEMA_ENFORCED_CONFIGURATION_KEY) ? rawConfig.get(SCHEMA_ENFORCED_CONFIGURATION_KEY).asBoolean(true) + : true; + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoDbStateIterator.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoDbStateIterator.java new file mode 100644 index 000000000000..e956e12ddbcd --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoDbStateIterator.java @@ -0,0 +1,203 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.mongodb.MongoException; +import com.mongodb.client.MongoCursor; +import io.airbyte.cdk.integrations.debezium.CdcMetadataInjector; +import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcEventUtils; +import io.airbyte.commons.exceptions.ConfigErrorException; +import io.airbyte.integrations.source.mongodb.state.IdType; +import io.airbyte.integrations.source.mongodb.state.InitialSnapshotStatus; +import io.airbyte.integrations.source.mongodb.state.MongoDbStateManager; +import io.airbyte.integrations.source.mongodb.state.MongoDbStreamState; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import java.time.Duration; +import java.time.Instant; +import java.util.Iterator; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.bson.Document; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A state-emitting iterator that emits a state message every checkpointInterval messages when + * iterating over a MongoCursor. + *

      + * Will also output a state message as the last message after the wrapper iterator has completed. + */ +public class MongoDbStateIterator implements Iterator { + + private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbStateIterator.class); + + private final MongoCursor iter; + private final Optional> cdcMetadataInjector; + private final MongoDbStateManager stateManager; + private final ConfiguredAirbyteStream stream; + private final Set fields; + private final Instant emittedAt; + private Instant lastCheckpoint = Instant.now(); + private final Integer checkpointInterval; + private final Duration checkpointDuration; + private final boolean isEnforceSchema; + + /** + * Counts the number of records seen in this batch, resets when a state-message has been generated. + */ + private int count = 0; + + /** + * Pointer to the last document _id seen by this iterator, necessary to track for state messages. + */ + private Object lastId; + + /** + * This iterator outputs a final state when the wrapped `iter` has concluded. When this is true, the + * final message will be returned. + */ + private boolean finalStateNext = false; + + /** + * Tracks if the underlying iterator threw an exception. This helps to determine the final state + * status emitted from the final next call. + */ + private boolean iterThrewException = false; + + /** + * Constructor. + * + * @param iter {@link MongoCursor} that iterates over Mongo documents + * @param stateManager {@link MongoDbStateManager} that manages global and per-stream state + * @param cdcMetadataInjector The {@link CdcMetadataInjector} used to add metadata to a published + * record. + * @param stream the stream that this iterator represents + * @param emittedAt when this iterator was started + * @param checkpointInterval how often a state message should be emitted based on number of + * messages. + * @param checkpointDuration how often a state message should be emitted based on time. + */ + public MongoDbStateIterator(final MongoCursor iter, + final MongoDbStateManager stateManager, + final Optional> cdcMetadataInjector, + final ConfiguredAirbyteStream stream, + final Instant emittedAt, + final int checkpointInterval, + final Duration checkpointDuration, + final boolean isEnforceSchema) { + this.iter = iter; + this.stateManager = stateManager; + this.stream = stream; + this.checkpointInterval = checkpointInterval; + this.checkpointDuration = checkpointDuration; + this.emittedAt = emittedAt; + this.fields = CatalogHelpers.getTopLevelFieldNames(stream).stream().collect(Collectors.toSet()); + this.lastId = + stateManager.getStreamState(stream.getStream().getName(), stream.getStream().getNamespace()).map(MongoDbStreamState::id).orElse(null); + this.cdcMetadataInjector = cdcMetadataInjector; + this.isEnforceSchema = isEnforceSchema; + } + + @Override + public boolean hasNext() { + LOGGER.debug("Checking hasNext() for stream {}...", getStream()); + try { + if (iter.hasNext()) { + return true; + } + } catch (final MongoException e) { + // If hasNext throws an exception, log it and then treat it as if hasNext returned false. + iterThrewException = true; + LOGGER.info("hasNext threw an exception for stream {}: {}", getStream(), e.getMessage(), e); + } + + // no more records in cursor + no record messages have been emitted => collection is empty + if (lastId == null) { + return false; + } + + // no more records in cursor + record messages have been emitted => we should emit a final state + // message. + if (!finalStateNext) { + finalStateNext = true; + LOGGER.debug("Final state is now true for stream {}...", getStream()); + return true; + } + + return false; + } + + @Override + public AirbyteMessage next() { + LOGGER.debug("Getting next message from stream {}...", getStream()); + // Should a state message be emitted based on the number of messages we've seen? + final var emitStateDueToMessageCount = count > 0 && count % checkpointInterval == 0; + // Should a state message be emitted based on then last time a state message was emitted? + final var emitStateDueToDuration = count > 0 && Duration.between(lastCheckpoint, Instant.now()).compareTo(checkpointDuration) > 0; + + if (finalStateNext) { + LOGGER.debug("Emitting final state status for stream {}:{}...", stream.getStream().getNamespace(), stream.getStream().getName()); + final var finalStateStatus = iterThrewException ? InitialSnapshotStatus.IN_PROGRESS : InitialSnapshotStatus.COMPLETE; + final var idType = IdType.findByJavaType(lastId.getClass().getSimpleName()) + .orElseThrow(() -> new ConfigErrorException("Unsupported _id type " + lastId.getClass().getSimpleName())); + final var state = new MongoDbStreamState(lastId.toString(), finalStateStatus, idType); + + stateManager.updateStreamState(stream.getStream().getName(), stream.getStream().getNamespace(), state); + + return new AirbyteMessage() + .withType(Type.STATE) + .withState(stateManager.toState()); + } else if (emitStateDueToMessageCount || emitStateDueToDuration) { + count = 0; + lastCheckpoint = Instant.now(); + + if (lastId != null) { + final var idType = IdType.findByJavaType(lastId.getClass().getSimpleName()) + .orElseThrow(() -> new ConfigErrorException("Unsupported _id type " + lastId.getClass().getSimpleName())); + final var state = new MongoDbStreamState(lastId.toString(), InitialSnapshotStatus.IN_PROGRESS, idType); + stateManager.updateStreamState(stream.getStream().getName(), stream.getStream().getNamespace(), state); + } + + return new AirbyteMessage() + .withType(Type.STATE) + .withState(stateManager.toState()); + } + + count++; + final var document = iter.next(); + final var jsonNode = isEnforceSchema ? MongoDbCdcEventUtils.toJsonNode(document, fields) : MongoDbCdcEventUtils.toJsonNodeNoSchema(document); + + lastId = document.get(MongoConstants.ID_FIELD); + + return new AirbyteMessage() + .withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage() + .withStream(stream.getStream().getName()) + .withNamespace(stream.getStream().getNamespace()) + .withEmittedAt(emittedAt.toEpochMilli()) + .withData(injectMetadata(jsonNode))); + } + + private JsonNode injectMetadata(final JsonNode jsonNode) { + if (Objects.nonNull(cdcMetadataInjector) && cdcMetadataInjector.isPresent() && jsonNode instanceof ObjectNode) { + cdcMetadataInjector.get().addMetaDataToRowsFetchedOutsideDebezium((ObjectNode) jsonNode, emittedAt.toString(), null); + } + + return jsonNode; + } + + private String getStream() { + return String.format("%s:%s", stream.getStream().getNamespace(), stream.getStream().getName()); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoField.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoField.java similarity index 82% rename from airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoField.java rename to airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoField.java index a63f50a976f5..1238d42ba0f8 100644 --- a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/main/java/io/airbyte/integrations/source/mongodb/internal/MongoField.java +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoField.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.integrations.source.mongodb.internal; +package io.airbyte.integrations.source.mongodb; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; @@ -15,7 +15,7 @@ */ public class MongoField extends Field { - public MongoField(String name, JsonSchemaType type) { + public MongoField(final String name, final JsonSchemaType type) { super(name, type); } @@ -31,7 +31,7 @@ public boolean equals(Object o) { } public int hashCode() { - return Objects.hash(new Object[] {this.getName()}); + return Objects.hash(this.getName()); } } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoUtil.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoUtil.java new file mode 100644 index 000000000000..08ecbb83e04f --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/MongoUtil.java @@ -0,0 +1,384 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb; + +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.integrations.source.mongodb.MongoCatalogHelper.AIRBYTE_STREAM_PROPERTIES; +import static io.airbyte.integrations.source.mongodb.MongoCatalogHelper.DEFAULT_CURSOR_FIELD; +import static io.airbyte.integrations.source.mongodb.MongoCatalogHelper.DEFAULT_PRIMARY_KEY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.SCHEMALESS_MODE_DATA_FIELD; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.mongodb.client.AggregateIterable; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoCursor; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.MongoIterable; +import com.mongodb.client.model.Aggregates; +import com.mongodb.client.model.Projections; +import io.airbyte.commons.exceptions.ConfigErrorException; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; +import org.bson.Document; +import org.bson.conversions.Bson; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MongoUtil { + + private static final Logger LOGGER = LoggerFactory.getLogger(MongoUtil.class); + + /** + * Set of collection prefixes that should be ignored when performing operations, such as discover to + * avoid access issues. + */ + private static final Set IGNORED_COLLECTIONS = Set.of("system.", "replset.", "oplog."); + + /** + * The minimum size of the Debezium event queue. This value will be selected if the provided + * configuration value for the queue size is less than this value + */ + @VisibleForTesting + static final int MIN_QUEUE_SIZE = 1000; + + /** + * The maximum size of the Debezium event queue. This value will be selected if the provided + * configuration value for the queue size is greater than this value OR if no value is provided. + */ + @VisibleForTesting + static final int MAX_QUEUE_SIZE = 10000; + + static final Set SCHEMALESS_FIELDS = + Set.of(CDC_UPDATED_AT, CDC_DELETED_AT, DEFAULT_CURSOR_FIELD, DEFAULT_PRIMARY_KEY, SCHEMALESS_MODE_DATA_FIELD); + + /** + * Tests whether the database exists in target MongoDB instance. + * + * @param mongoClient The {@link MongoClient} used to query the MongoDB server for the database + * names. + * @param databaseName The database name from the source's configuration. + * @return {@code true} if the database exists, {@code false} otherwise. + */ + public static boolean checkDatabaseExists(final MongoClient mongoClient, final String databaseName) { + final MongoIterable databaseNames = mongoClient.listDatabaseNames(); + return StreamSupport.stream(databaseNames.spliterator(), false) + .anyMatch(name -> name.equalsIgnoreCase(databaseName)); + } + + /** + * Returns the set of collections that the current credentials are authorized to access. + * + * @param mongoClient The {@link MongoClient} used to query the MongoDB server for authorized + * collections. + * @param databaseName The name of the database to query for authorized collections. + * @return The set of authorized collection names (may be empty). + */ + public static Set getAuthorizedCollections(final MongoClient mongoClient, final String databaseName) { + /* + * db.runCommand ({listCollections: 1.0, authorizedCollections: true, nameOnly: true }) the command + * returns only those collections for which the user has privileges. For example, if a user has find + * action on specific collections, the command returns only those collections; or, if a user has + * find or any other action, on the database resource, the command lists all collections in the + * database. + */ + final Document document = mongoClient.getDatabase(databaseName).runCommand(new Document("listCollections", 1) + .append("authorizedCollections", true) + .append("nameOnly", true)) + .append("filter", "{ 'type': 'collection' }"); + return document.toBsonDocument() + .get("cursor").asDocument() + .getArray("firstBatch") + .stream() + .map(bsonValue -> bsonValue.asDocument().getString("name").getValue()) + .filter(MongoUtil::isSupportedCollection) + .collect(Collectors.toSet()); + } + + /** + * Retrieves the {@link AirbyteStream}s available to the source by querying the MongoDB server. + * + * @param mongoClient The {@link MongoClient} used to query the MongoDB server. + * @param databaseName The name of the database to query for collections. + * @param sampleSize The maximum number of documents to sample when attempting to discover the + * unique fields for a collection. + * @param isSchemaEnforced True if the connector is running in schema mode, false if running in + * schemaless (packed) mode + * @return The list of {@link AirbyteStream}s that map to the available collections in the provided + * database. + */ + public static List getAirbyteStreams(final MongoClient mongoClient, + final String databaseName, + final Integer sampleSize, + final boolean isSchemaEnforced) { + final Set authorizedCollections = getAuthorizedCollections(mongoClient, databaseName); + return authorizedCollections.parallelStream() + .map(collectionName -> discoverFields(collectionName, mongoClient, databaseName, sampleSize, isSchemaEnforced)) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + + /** + * Computes the size of the queue that will contain events generated by Debezium. If a queue size + * value is provided as part of the source's configuration, this method guarantees that the selected + * queue size is within the {@link #MIN_QUEUE_SIZE} and {@link #MAX_QUEUE_SIZE} values. If the value + * is not present in the configuration, the {@link #MAX_QUEUE_SIZE} is selected. + * + * @param config The source connector's configuration. + * @return The size of the Debezium event queue. + */ + public static OptionalInt getDebeziumEventQueueSize(final MongoDbSourceConfig config) { + final OptionalInt sizeFromConfig = config.getQueueSize(); + + if (sizeFromConfig.isPresent()) { + final int size = sizeFromConfig.getAsInt(); + if (size < MIN_QUEUE_SIZE) { + LOGGER.warn("Queue size is overridden to {} , which is the min allowed for safety.", + MIN_QUEUE_SIZE); + return OptionalInt.of(MIN_QUEUE_SIZE); + } else if (size > MAX_QUEUE_SIZE) { + LOGGER.warn("Queue size is overridden to {} , which is the max allowed for safety.", + MAX_QUEUE_SIZE); + return OptionalInt.of(MAX_QUEUE_SIZE); + } + return OptionalInt.of(size); + } + return OptionalInt.of(MAX_QUEUE_SIZE); + } + + /** + * Retrieves the statistics for the collection represented by the provided stream. + * + * @param mongoClient The {@link MongoClient} used to retrieve statistics from MongoDB. + * @param stream The stream that represents the collection. + * @return The {@link CollectionStatistics} of the collection or an empty {@link Optional} if the + * statistics cannot be retrieved. + */ + public static Optional getCollectionStatistics(final MongoClient mongoClient, final ConfiguredAirbyteStream stream) { + try { + final Map collStats = Map.of(MongoConstants.STORAGE_STATS_KEY, Map.of(), MongoConstants.COUNT_KEY, Map.of()); + final MongoDatabase mongoDatabase = mongoClient.getDatabase(stream.getStream().getNamespace()); + final MongoCollection collection = mongoDatabase.getCollection(stream.getStream().getName()); + final AggregateIterable output = collection.aggregate(List.of(new Document("$collStats", collStats))); + + try (final MongoCursor cursor = output.allowDiskUse(true).cursor()) { + if (cursor.hasNext()) { + final Document stats = cursor.next(); + @SuppressWarnings("unchecked") + final Map storageStats = (Map) stats.get(MongoConstants.STORAGE_STATS_KEY); + if (storageStats != null && !storageStats.isEmpty()) { + return Optional.of(new CollectionStatistics((Number) storageStats.get(MongoConstants.COLLECTION_STATISTICS_COUNT_KEY), + (Number) storageStats.get(MongoConstants.COLLECTION_STATISTICS_STORAGE_SIZE_KEY))); + } else { + LOGGER.warn("Unable to estimate sync size: statistics for {}.{} are missing.", stream.getStream().getNamespace(), + stream.getStream().getName()); + } + } else { + LOGGER.warn("Unable to estimate sync size: statistics for {}.{} are missing.", stream.getStream().getNamespace(), + stream.getStream().getName()); + } + } + } catch (final Exception e) { + LOGGER.warn("Error occurred while attempting to estimate sync size", e); + } + + return Optional.empty(); + } + + /** + * Checks whether the user's config + catalog does not match. This can happen in the following cases + * : 1. User is in schemaless mode + catalog corresponds to schema enabled mode. 2. User is in + * schema enabled mode + catalog corresponds to schemaless mode + * + * @param isConfigSchemaEnforced true if schema is enforced in configuration, false if in schemaless + * mode. + * @param isStateSchemaEnforced true if schema is enforced in saved state, false if in schemaless + * mode. + * @param catalog User's configured catalog. + */ + public static void checkSchemaModeMismatch(final boolean isConfigSchemaEnforced, + final boolean isStateSchemaEnforced, + final ConfiguredAirbyteCatalog catalog) { + final boolean isCatalogSchemaEnforcing = !catalog.getStreams().stream() + .allMatch(stream -> verifySchemaless(stream.getStream().getJsonSchema())); + + final String remedy = isConfigSchemaEnforced == isCatalogSchemaEnforcing + ? "Please reset your data." + : "Please refresh source schema and reset streams."; + if (Stream.of(isConfigSchemaEnforced, isStateSchemaEnforced, isCatalogSchemaEnforcing).distinct().count() > 1) { + throw new ConfigErrorException("Mismatch between schema enforcing mode in sync configuration (%b), catalog (%b) and saved state (%b). %s" + .formatted(isConfigSchemaEnforced, isCatalogSchemaEnforcing, isStateSchemaEnforced, remedy)); + } + } + + private static boolean verifySchemaless(final JsonNode jsonSchema) { + final JsonNode airbyteStreamProperties = jsonSchema.get(AIRBYTE_STREAM_PROPERTIES); + return airbyteStreamProperties.size() == SCHEMALESS_FIELDS.size() && + SCHEMALESS_FIELDS.stream().allMatch(field -> airbyteStreamProperties.get(field) != null); + } + + /** + * Creates an {@link AirbyteStream} from the provided data. + * + * @param collectionName The name of the collection represented by the stream (stream name). + * @param databaseName The name of the database represented by the stream (stream namespace). + * @param fields The fields available to the stream. + * @param isSchemaEnforced True if the connector is running in schema mode, false if running in + * schemaless (packed) mode + * @return A {@link AirbyteStream} object representing the stream. + */ + private static AirbyteStream createAirbyteStream(final String collectionName, + final String databaseName, + final List fields, + final boolean isSchemaEnforced) { + if (isSchemaEnforced) { + return MongoCatalogHelper.buildAirbyteStream(collectionName, databaseName, fields); + } else { + return MongoCatalogHelper.buildSchemalessAirbyteStream(collectionName, databaseName, fields); + } + } + + /** + * Discovers the fields available to the stream. + * + * @param collectionName The name of the collection associated with the stream (stream name). + * @param mongoClient The {@link MongoClient} used to access the fields. + * @param databaseName The name of the database associated with the stream (stream namespace). + * @param sampleSize The maximum number of documents to sample when attempting to discover the + * unique fields for a collection + * @param isSchemaEnforced True if the connector is running in schema mode, false if running in + * schemaless (packed) mode + * @return The {@link AirbyteStream} that contains the discovered fields or an empty + * {@link Optional} if the underlying collection is empty. + */ + private static Optional discoverFields(final String collectionName, + final MongoClient mongoClient, + final String databaseName, + final Integer sampleSize, + final boolean isSchemaEnforced) { + /* + * Fetch the keys/types from the first N documents and the last N documents from the collection. + * This is an attempt to "survey" the documents in the collection for variance in the schema keys. + */ + final Set discoveredFields; + final MongoCollection mongoCollection = mongoClient.getDatabase(databaseName).getCollection(collectionName); + if (isSchemaEnforced) { + discoveredFields = new HashSet<>(getFieldsInCollection(mongoCollection, sampleSize)); + } else { + // In schemaless mode, we only sample one record as we're only interested in the _id field (which + // exists on every record). + discoveredFields = new HashSet<>(getFieldsForSchemaless(mongoCollection)); + } + return Optional + .ofNullable( + !discoveredFields.isEmpty() ? createAirbyteStream(collectionName, databaseName, new ArrayList<>(discoveredFields), isSchemaEnforced) + : null); + } + + private static Set getFieldsInCollection(final MongoCollection collection, final Integer sampleSize) { + final Set discoveredFields = new HashSet<>(); + final Map fieldsMap = Map.of("input", Map.of("$objectToArray", "$$ROOT"), + "as", "each", + "in", Map.of("k", "$$each.k", "v", Map.of("$type", "$$each.v"))); + + final Document mapFunction = new Document("$map", fieldsMap); + final Document arrayToObjectAggregation = new Document("$arrayToObject", mapFunction); + + final Map groupMap = new HashMap<>(); + groupMap.put("_id", "$fields"); + + final List aggregateList = new ArrayList<>(); + /* + * Use sampling to reduce the time it takes to discover fields. Inspired by + * https://www.mongodb.com/docs/compass/current/sampling/#sampling-method. + */ + aggregateList.add(Aggregates.sample(sampleSize)); + aggregateList.add(Aggregates.project(new Document("fields", arrayToObjectAggregation))); + aggregateList.add(Aggregates.unwind("$fields")); + aggregateList.add(new Document("$group", groupMap)); + + /* + * Runs the following aggregation query: db..aggregate( [ { "$sample": { "size" : + * 10000 } }, { "$project" : { "fields" : { "$arrayToObject": { "$map" : { "input" : { + * "$objectToArray" : "$$ROOT" }, "as" : "each", "in" : { "k" : "$$each.k", "v" : { "$type" : + * "$$each.v" } } } } } } }, { "$unwind" : "$fields" }, { "$group" : { "_id" : $fields } } ] ) + */ + final AggregateIterable output = collection.aggregate(aggregateList); + + try (final MongoCursor cursor = output.allowDiskUse(true).cursor()) { + while (cursor.hasNext()) { + @SuppressWarnings("unchecked") + final Map fields = (Map) cursor.next().get("_id"); + discoveredFields.addAll(fields.entrySet().stream() + .map(e -> new MongoField(e.getKey(), convertToSchemaType(e.getValue()))) + .collect(Collectors.toSet())); + } + } + + return discoveredFields; + } + + private static Set getFieldsForSchemaless(final MongoCollection collection) { + final Set discoveredFields = new HashSet<>(); + + final AggregateIterable output = collection.aggregate(Arrays.asList( + Aggregates.sample(1), // Selects one random document + Aggregates.project(Projections.fields( + Projections.excludeId(), // Excludes the _id field from the result + Projections.computed("_idType", new Document("$type", "$_id")) // Gets the type of the _id field + )))); + + try (final MongoCursor cursor = output.allowDiskUse(true).cursor()) { + while (cursor.hasNext()) { + final JsonSchemaType schemaType = convertToSchemaType((String) cursor.next().get("_idType")); + discoveredFields.add(new MongoField(MongoConstants.ID_FIELD, schemaType)); + } + } + + return discoveredFields; + } + + private static JsonSchemaType convertToSchemaType(final String type) { + return switch (type) { + case "boolean" -> JsonSchemaType.BOOLEAN; + case "int", "long", "double", "decimal" -> JsonSchemaType.NUMBER; + case "array" -> JsonSchemaType.ARRAY; + case "object", "javascriptWithScope" -> JsonSchemaType.OBJECT; + case "null" -> JsonSchemaType.NULL; + default -> JsonSchemaType.STRING; + }; + } + + private static boolean isSupportedCollection(final String collectionName) { + return IGNORED_COLLECTIONS.stream().noneMatch(collectionName::startsWith); + } + + /** + * Represents statistics of a MongoDB collection. + * + * @param count The number of documents in the collection. + * @param size The size (in bytes) of the collection. + */ + public record CollectionStatistics(Number count, Number size) {} + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcConnectorMetadataInjector.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcConnectorMetadataInjector.java new file mode 100644 index 000000000000..3a6d9a39b489 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcConnectorMetadataInjector.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.cdc; + +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.debezium.CdcMetadataInjector; +import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants; +import io.airbyte.commons.json.Jsons; +import java.time.Instant; +import java.util.concurrent.atomic.AtomicLong; + +/** + * MongoDB specific implementation of the {@link CdcMetadataInjector} that stores cursor information + * for MongoDB source event data. + */ +public class MongoDbCdcConnectorMetadataInjector implements CdcMetadataInjector { + + public static final String CDC_DEFAULT_CURSOR = "_ab_cdc_cursor"; + + private final long emittedAtConverted; + + // This now makes this class stateful. Please make sure to use the same instance within a sync + private final AtomicLong recordCounter = new AtomicLong(1); + private static final long ONE_HUNDRED_MILLION = 100_000_000; + private static MongoDbCdcConnectorMetadataInjector mongoDbCdcConnectorMetadataInjector; + + private MongoDbCdcConnectorMetadataInjector(final Instant emittedAt) { + this.emittedAtConverted = emittedAt.getEpochSecond() * ONE_HUNDRED_MILLION; + } + + public static MongoDbCdcConnectorMetadataInjector getInstance(final Instant emittedAt) { + if (mongoDbCdcConnectorMetadataInjector == null) { + mongoDbCdcConnectorMetadataInjector = new MongoDbCdcConnectorMetadataInjector(emittedAt); + } + + return mongoDbCdcConnectorMetadataInjector; + } + + /** + * Adds the metadata columns injected into records by this implementation to the discovered field + * information. + * + * @param properties An {@link ObjectNode} representing the fields in an + * {@link io.airbyte.protocol.models.v0.AirbyteStream}. + * @return The modified {@link ObjectNode} with the CDC metadata columns added. + */ + public static ObjectNode addCdcMetadataColumns(final ObjectNode properties) { + final JsonNode stringType = Jsons.jsonNode(ImmutableMap.of("type", "string")); + final JsonNode airbyteIntegerType = Jsons.jsonNode(ImmutableMap.of("type", "number", "airbyte_type", "integer")); + properties.set(CDC_UPDATED_AT, stringType); + properties.set(CDC_DELETED_AT, stringType); + properties.set(CDC_DEFAULT_CURSOR, airbyteIntegerType); + return properties; + } + + @Override + public void addMetaData(final ObjectNode event, final JsonNode source) { + event.put(CDC_DEFAULT_CURSOR, getCdcDefaultCursor()); + } + + @Override + public void addMetaDataToRowsFetchedOutsideDebezium(final ObjectNode record, final String transactionTimestamp, final Object ignored) { + record.put(CDC_UPDATED_AT, transactionTimestamp); + record.put(CDC_DELETED_AT, (String) null); + record.put(CDC_DEFAULT_CURSOR, getCdcDefaultCursor()); + } + + public String namespace(final JsonNode source) { + return source.get(MongoDbDebeziumConstants.ChangeEvent.SOURCE_DB).asText(); + } + + @Override + public String name(final JsonNode source) { + return source.get(MongoDbDebeziumConstants.ChangeEvent.SOURCE_COLLECTION).asText(); + } + + private Long getCdcDefaultCursor() { + return this.emittedAtConverted + this.recordCounter.getAndIncrement(); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitialSnapshotUtils.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitialSnapshotUtils.java new file mode 100644 index 000000000000..1e844f4949ca --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitialSnapshotUtils.java @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.cdc; + +import static io.airbyte.cdk.db.jdbc.JdbcUtils.PLATFORM_DATA_INCREASE_FACTOR; + +import com.google.common.collect.Sets; +import com.mongodb.client.MongoClient; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.mongodb.MongoUtil; +import io.airbyte.integrations.source.mongodb.state.InitialSnapshotStatus; +import io.airbyte.integrations.source.mongodb.state.MongoDbStateManager; +import io.airbyte.protocol.models.v0.AirbyteEstimateTraceMessage; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.SyncMode; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utilities for determining the configured streams that should take part in the initial snapshot + * portion of a CDC sync for MongoDB. + */ +public class MongoDbCdcInitialSnapshotUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbCdcInitialSnapshotUtils.class); + + private static final Predicate SYNC_MODE_FILTER = c -> SyncMode.INCREMENTAL.equals(c.getSyncMode()); + + /** + * Returns the list of configured Airbyte streams that need to perform the initial snapshot portion + * of a CDC sync. This includes streams that: + *
        + *
      1. Did not complete a successful initial snapshot sync during the last execution
      2. + *
      3. Have been added to the catalog since the last sync
      4. + *
      + * + * In addition, if the saved offset is no longer present in the server, all streams are used in the + * initial snapshot in order to restore the offset to an existing value. + * + * @param mongoClient The {@link MongoClient} used to retrieve estimated trace statistics. + * @param stateManager The {@link MongoDbStateManager} that contains information about each stream's + * progress. + * @param fullCatalog The fully configured Airbyte catalog. + * @param savedOffsetIsValid Boolean value that indicates whether the offset exists on the server. + * If it does not exist, all streams will perform an initial sync. + * @return The list of Airbyte streams to be used in the initial snapshot sync. + */ + public static List getStreamsForInitialSnapshot( + final MongoClient mongoClient, + final MongoDbStateManager stateManager, + final ConfiguredAirbyteCatalog fullCatalog, + final boolean savedOffsetIsValid) { + + final List initialSnapshotStreams = new ArrayList<>(); + + if (!savedOffsetIsValid) { + LOGGER.info("Offset state is invalid. Add all {} stream(s) from the configured catalog to perform an initial snapshot.", + fullCatalog.getStreams().size()); + + /* + * If the saved offset does not exist on the server, re-sync everything via initial snapshot as we + * have lost track of which changes have been processed already. This occurs when the oplog cycles + * faster than a sync interval, resulting in the stored offset in our state being removed from the + * oplog. + */ + initialSnapshotStreams.addAll(fullCatalog.getStreams() + .stream() + .filter(SYNC_MODE_FILTER) + .toList()); + } else { + // Find and filter out streams that have completed the initial snapshot + final Set streamsStillInInitialSnapshot = stateManager.getStreamStates().entrySet().stream() + .filter(e -> InitialSnapshotStatus.IN_PROGRESS.equals(e.getValue().status())) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); + + LOGGER.info("There are {} stream(s) that are still in progress of an initial snapshot sync.", streamsStillInInitialSnapshot.size()); + + // Fetch the streams from the catalog that still need to complete the initial snapshot sync + initialSnapshotStreams.addAll(fullCatalog.getStreams().stream() + .filter(stream -> streamsStillInInitialSnapshot.contains(AirbyteStreamNameNamespacePair.fromAirbyteStream(stream.getStream()))) + .map(Jsons::clone) + .toList()); + + // Fetch the streams added to the catalog since the last sync + final List newStreams = identifyStreamsToSnapshot(fullCatalog, + new HashSet<>(stateManager.getStreamStates().keySet())); + LOGGER.info("There are {} stream(s) that have been added to the catalog since the last sync.", newStreams.size()); + initialSnapshotStreams.addAll(newStreams); + } + + // Emit estimated trace message for each stream that will perform an initial snapshot sync + initialSnapshotStreams.forEach(s -> estimateInitialSnapshotSyncSize(mongoClient, s)); + + return initialSnapshotStreams; + } + + private static List identifyStreamsToSnapshot(final ConfiguredAirbyteCatalog catalog, + final Set alreadySyncedStreams) { + final Set allStreams = AirbyteStreamNameNamespacePair.fromConfiguredCatalog(catalog); + final Set newlyAddedStreams = new HashSet<>(Sets.difference(allStreams, alreadySyncedStreams)); + return catalog.getStreams().stream() + .filter(SYNC_MODE_FILTER) + .filter(stream -> newlyAddedStreams.contains(AirbyteStreamNameNamespacePair.fromAirbyteStream(stream.getStream()))).map(Jsons::clone) + .toList(); + } + + private static void estimateInitialSnapshotSyncSize(final MongoClient mongoClient, final ConfiguredAirbyteStream stream) { + final Optional collectionStatistics = MongoUtil.getCollectionStatistics(mongoClient, stream); + collectionStatistics.ifPresent(c -> { + AirbyteTraceMessageUtility.emitEstimateTrace(PLATFORM_DATA_INCREASE_FACTOR * c.size().longValue(), + AirbyteEstimateTraceMessage.Type.STREAM, c.count().longValue(), stream.getStream().getName(), stream.getStream().getNamespace()); + LOGGER + .info(String.format( + "Estimate for table: %s.%s : {sync_row_count: %s, sync_bytes: %s, total_table_row_count: %s, total_table_bytes: %s}", + stream.getStream().getNamespace(), stream.getStream().getName(), c.count(), c.size(), c.count(), c.size())); + }); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitializer.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitializer.java new file mode 100644 index 000000000000..bc8957c98179 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitializer.java @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.cdc; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoDatabase; +import io.airbyte.cdk.integrations.debezium.AirbyteDebeziumHandler; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager; +import io.airbyte.cdk.integrations.debezium.internals.RecordWaitTimeUtil; +import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcTargetPosition; +import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumStateUtil; +import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbResumeTokenHelper; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.commons.util.AutoCloseableIterators; +import io.airbyte.integrations.source.mongodb.InitialSnapshotHandler; +import io.airbyte.integrations.source.mongodb.MongoDbSourceConfig; +import io.airbyte.integrations.source.mongodb.MongoUtil; +import io.airbyte.integrations.source.mongodb.state.MongoDbStateManager; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.Properties; +import java.util.function.Supplier; +import org.bson.BsonDocument; +import org.bson.BsonTimestamp; +import org.bson.Document; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Provides methods to initialize the stream iterators based on the state of each stream in the + * configured catalog. + *

      + *

      + * For more information on the iterator selection logic, see + * {@link MongoDbCdcInitialSnapshotUtils#getStreamsForInitialSnapshot(MongoClient, MongoDbStateManager, ConfiguredAirbyteCatalog, boolean)} + * and {@link AirbyteDebeziumHandler#getIncrementalIterators} + */ +public class MongoDbCdcInitializer { + + private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbCdcInitializer.class); + + private final MongoDbDebeziumStateUtil mongoDbDebeziumStateUtil; + + @VisibleForTesting + MongoDbCdcInitializer(final MongoDbDebeziumStateUtil mongoDbDebeziumStateUtil) { + this.mongoDbDebeziumStateUtil = mongoDbDebeziumStateUtil; + } + + public MongoDbCdcInitializer() { + this(new MongoDbDebeziumStateUtil()); + } + + /** + * Generates the list of stream iterators based on the configured catalog and stream state. This + * list will include any initial snapshot iterators, followed by incremental iterators, where + * applicable. + * + * @param mongoClient The {@link MongoClient} used to interact with the target MongoDB server. + * @param cdcMetadataInjector The {@link MongoDbCdcConnectorMetadataInjector} used to add metadata + * to generated records. + * @param catalog The configured Airbyte catalog of streams for the source. + * @param stateManager The {@link MongoDbStateManager} that provides state information used for + * iterator selection. + * @param emittedAt The timestamp of the sync. + * @param config The configuration of the source. + * @return The list of stream iterators with initial snapshot iterators before any incremental + * iterators. + */ + public List> createCdcIterators( + final MongoClient mongoClient, + final MongoDbCdcConnectorMetadataInjector cdcMetadataInjector, + final ConfiguredAirbyteCatalog catalog, + final MongoDbStateManager stateManager, + final Instant emittedAt, + final MongoDbSourceConfig config) { + + final Duration firstRecordWaitTime = RecordWaitTimeUtil.getFirstRecordWaitTime(config.rawConfig()); + final Duration subsequentRecordWaitTime = RecordWaitTimeUtil.getSubsequentRecordWaitTime(config.rawConfig()); + final OptionalInt queueSize = MongoUtil.getDebeziumEventQueueSize(config); + final String databaseName = config.getDatabaseName(); + final boolean isEnforceSchema = config.getEnforceSchema(); + final Properties defaultDebeziumProperties = MongoDbCdcProperties.getDebeziumProperties(); + logOplogInfo(mongoClient); + final BsonDocument initialResumeToken = MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient); + final JsonNode initialDebeziumState = + mongoDbDebeziumStateUtil.constructInitialDebeziumState(initialResumeToken, mongoClient, databaseName); + final MongoDbCdcState cdcState = (stateManager.getCdcState() == null || stateManager.getCdcState().state() == null) + ? new MongoDbCdcState(initialDebeziumState, isEnforceSchema) + : new MongoDbCdcState(Jsons.clone(stateManager.getCdcState().state()), stateManager.getCdcState().schema_enforced()); + final Optional optSavedOffset = mongoDbDebeziumStateUtil.savedOffset( + Jsons.clone(defaultDebeziumProperties), + catalog, + cdcState.state(), + config.rawConfig(), + mongoClient); + + // We should always be able to extract offset out of state if it's not null + if (cdcState.state() != null && optSavedOffset.isEmpty()) { + throw new RuntimeException( + "Unable extract the offset out of state, State mutation might not be working. " + cdcState.state()); + } + + final boolean savedOffsetIsValid = + optSavedOffset.filter(savedOffset -> mongoDbDebeziumStateUtil.isValidResumeToken(savedOffset, mongoClient)).isPresent(); + + if (!savedOffsetIsValid) { + LOGGER.info("Saved offset is not valid. Airbyte will trigger a full refresh."); + // If the offset in the state is invalid, reset the state to the initial STATE + stateManager.resetState(new MongoDbCdcState(initialDebeziumState, config.getEnforceSchema())); + } else { + LOGGER.info("Valid offset state discovered. Updating state manager with retrieved CDC state {} {}...", cdcState.state(), + cdcState.schema_enforced()); + stateManager.updateCdcState(new MongoDbCdcState(cdcState.state(), cdcState.schema_enforced())); + } + + final MongoDbCdcState stateToBeUsed = + (!savedOffsetIsValid || stateManager.getCdcState() == null || stateManager.getCdcState().state() == null) + ? new MongoDbCdcState(initialDebeziumState, config.getEnforceSchema()) + : stateManager.getCdcState(); + + final List initialSnapshotStreams = + MongoDbCdcInitialSnapshotUtils.getStreamsForInitialSnapshot(mongoClient, stateManager, catalog, savedOffsetIsValid); + final InitialSnapshotHandler initialSnapshotHandler = new InitialSnapshotHandler(); + final List> initialSnapshotIterators = + initialSnapshotHandler.getIterators(initialSnapshotStreams, stateManager, mongoClient.getDatabase(databaseName), cdcMetadataInjector, + emittedAt, config.getCheckpointInterval(), isEnforceSchema); + + final AirbyteDebeziumHandler handler = new AirbyteDebeziumHandler<>(config.rawConfig(), + new MongoDbCdcTargetPosition(initialResumeToken), false, firstRecordWaitTime, subsequentRecordWaitTime, queueSize); + final MongoDbCdcStateHandler mongoDbCdcStateHandler = new MongoDbCdcStateHandler(stateManager); + final MongoDbCdcSavedInfoFetcher cdcSavedInfoFetcher = new MongoDbCdcSavedInfoFetcher(stateToBeUsed); + + final Supplier> incrementalIteratorSupplier = () -> handler.getIncrementalIterators(catalog, + cdcSavedInfoFetcher, + mongoDbCdcStateHandler, + cdcMetadataInjector, + defaultDebeziumProperties, + DebeziumPropertiesManager.DebeziumConnectorType.MONGODB, + emittedAt, + false); + + // We can close the client after the initial snapshot is complete, incremental + // iterator does not make use of the client. + final AutoCloseableIterator initialSnapshotIterator = AutoCloseableIterators.appendOnClose( + AutoCloseableIterators.concatWithEagerClose(initialSnapshotIterators), mongoClient::close); + + return List.of(initialSnapshotIterator, AutoCloseableIterators.lazyIterator(incrementalIteratorSupplier, null)); + } + + private void logOplogInfo(final MongoClient mongoClient) { + try { + final MongoDatabase localDatabase = mongoClient.getDatabase("local"); + final Document command = new Document("collStats", "oplog.rs"); + final Document result = localDatabase.runCommand(command); + if (result != null) { + LOGGER.info("Max oplog size is {} bytes", result.getLong("maxSize")); + LOGGER.info("Free space in oplog is {} bytes", result.getLong("freeStorageSize")); + } + } catch (final Exception e) { + LOGGER.warn("Unable to query for op log stats, exception: {}" + e.getMessage()); + } + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcProperties.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcProperties.java new file mode 100644 index 000000000000..e567d190da32 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcProperties.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.cdc; + +import java.time.Duration; +import java.util.Properties; + +/** + * Defines MongoDB specific CDC configuration properties for Debezium. + */ +public class MongoDbCdcProperties { + + static final String CAPTURE_MODE_KEY = "capture.mode"; + static final String CAPTURE_MODE_VALUE = "change_streams_update_full_with_pre_image"; + static final String CONNECTOR_CLASS_KEY = "connector.class"; + static final String CONNECTOR_CLASS_VALUE = "io.debezium.connector.mongodb.MongoDbConnector"; + static final String HEARTBEAT_FREQUENCY_MS = Long.toString(Duration.ofSeconds(10).toMillis()); + static final String HEARTBEAT_INTERVAL_KEY = "heartbeat.interval.ms"; + static final String SNAPSHOT_MODE_KEY = "snapshot.mode"; + static final String SNAPSHOT_MODE_VALUE = "never"; + static final String TOMBSTONE_ON_DELETE_KEY = "tombstones.on.delete"; + static final String TOMBSTONE_ON_DELETE_VALUE = Boolean.FALSE.toString(); + + /** + * Returns the common properties required to configure the Debezium MongoDB connector. + * + * @return The common Debezium CDC properties for the Debezium MongoDB connector. + */ + public static Properties getDebeziumProperties() { + final Properties props = new Properties(); + + props.setProperty(CONNECTOR_CLASS_KEY, CONNECTOR_CLASS_VALUE); + props.setProperty(SNAPSHOT_MODE_KEY, SNAPSHOT_MODE_VALUE); + props.setProperty(CAPTURE_MODE_KEY, CAPTURE_MODE_VALUE); + props.setProperty(HEARTBEAT_INTERVAL_KEY, HEARTBEAT_FREQUENCY_MS); + props.setProperty(TOMBSTONE_ON_DELETE_KEY, TOMBSTONE_ON_DELETE_VALUE); + + return props; + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcSavedInfoFetcher.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcSavedInfoFetcher.java new file mode 100644 index 000000000000..1d7004494b07 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcSavedInfoFetcher.java @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.cdc; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.debezium.CdcSavedInfoFetcher; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteSchemaHistoryStorage; +import java.util.Optional; + +/** + * Implementation of the {@link CdcSavedInfoFetcher} interface for MongoDB. + */ +public class MongoDbCdcSavedInfoFetcher implements CdcSavedInfoFetcher { + + private final MongoDbCdcState savedState; + + public MongoDbCdcSavedInfoFetcher(final MongoDbCdcState savedState) { + this.savedState = savedState; + } + + @Override + public JsonNode getSavedOffset() { + return savedState.state(); + } + + @Override + public AirbyteSchemaHistoryStorage.SchemaHistory> getSavedSchemaHistory() { + throw new RuntimeException("Schema history is not relevant for MongoDb"); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcState.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcState.java new file mode 100644 index 000000000000..1d94fdbd3546 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcState.java @@ -0,0 +1,25 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.cdc; + +import com.fasterxml.jackson.databind.JsonNode; + +/** + * Represents the global CDC state that is used by Debezium as an offset. + * + * @param state The Debezium offset state as a {@link JsonNode}. + */ +public record MongoDbCdcState(JsonNode state, Boolean schema_enforced) { + + public MongoDbCdcState { + // Ensure that previously saved state with no schema_enforced will migrate to schema_enforced = true + schema_enforced = schema_enforced == null || schema_enforced; + } + + public MongoDbCdcState(final JsonNode state) { + this(state, true); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcStateHandler.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcStateHandler.java new file mode 100644 index 000000000000..81fd7cb417ab --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcStateHandler.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.cdc; + +import io.airbyte.cdk.integrations.debezium.CdcStateHandler; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteSchemaHistoryStorage; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.mongodb.state.MongoDbStateManager; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Implementation of the {@link CdcStateHandler} that handles saving the CDC offset as Airbyte state + * for MongoDB. + */ +public class MongoDbCdcStateHandler implements CdcStateHandler { + + private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbCdcStateHandler.class); + + private final MongoDbStateManager stateManager; + + public MongoDbCdcStateHandler(final MongoDbStateManager stateManager) { + this.stateManager = stateManager; + } + + @Override + public AirbyteMessage saveState(final Map offset, final AirbyteSchemaHistoryStorage.SchemaHistory ignored) { + final Boolean previousStateSchemaEnforced = stateManager.getCdcState() != null ? stateManager.getCdcState().schema_enforced() : null; + final MongoDbCdcState cdcState = new MongoDbCdcState(Jsons.jsonNode(offset), previousStateSchemaEnforced); + + LOGGER.info("Saving Debezium state {}...", cdcState); + stateManager.updateCdcState(cdcState); + + final AirbyteStateMessage stateMessage = stateManager.toState(); + return new AirbyteMessage().withType(AirbyteMessage.Type.STATE).withState(stateMessage); + } + + @Override + public AirbyteMessage saveStateAfterCompletionOfSnapshotOfNewStreams() { + throw new RuntimeException("Debezium is not used to carry out the snapshot of tables."); + } + + @Override + public boolean isCdcCheckpointEnabled() { + return true; + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/state/IdType.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/state/IdType.java new file mode 100644 index 000000000000..ed620edce098 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/state/IdType.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.state; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.bson.types.ObjectId; + +/** + * _id field types that are currently supported, potential types are defined here + */ +public enum IdType { + + OBJECT_ID("objectId", "ObjectId", ObjectId::new), + STRING("string", "String", s -> s), + INT("int", "Integer", Integer::valueOf), + LONG("long", "Long", Long::valueOf); + + private static final Map byBsonType = new HashMap<>(); + static { + for (final var idType : IdType.values()) { + byBsonType.put(idType.bsonType.toLowerCase(), idType); + } + } + + private static final Map byJavaType = new HashMap<>(); + static { + for (final var idType : IdType.values()) { + byJavaType.put(idType.javaType.toLowerCase(), idType); + } + } + + /** A comma-separated, human-readable list of supported _id types. */ + public static final String SUPPORTED; + static { + SUPPORTED = Arrays.stream(IdType.values()) + .map(e -> e.bsonType) + .collect(Collectors.joining(", ")); + } + + /** MongoDb BSON type */ + private final String bsonType; + /** Java class name type */ + private final String javaType; + /** Converter for converting a string value into an appropriate MongoDb type. */ + private final Function converter; + + IdType(final String bsonType, final String javaType, final Function converter) { + this.bsonType = bsonType; + this.javaType = javaType; + this.converter = converter; + } + + public Object convert(final String t) { + return converter.apply(t); + } + + public static Optional findByBsonType(final String bsonType) { + if (bsonType == null) { + return Optional.empty(); + } + return Optional.ofNullable(byBsonType.get(bsonType.toLowerCase())); + } + + public static Optional findByJavaType(final String javaType) { + if (javaType == null) { + return Optional.empty(); + } + return Optional.ofNullable(byJavaType.get(javaType.toLowerCase())); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/state/InitialSnapshotStatus.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/state/InitialSnapshotStatus.java new file mode 100644 index 000000000000..192ef6607e81 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/state/InitialSnapshotStatus.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.state; + +/** + * Enumerates the potential status values for the initial snapshot of streams. This information is + * used to determine if a stream has successfully completed its initial snapshot when building the + * list of stream iterators for a sync. + */ +public enum InitialSnapshotStatus { + + IN_PROGRESS, + COMPLETE +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/state/MongoDbStateManager.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/state/MongoDbStateManager.java new file mode 100644 index 000000000000..3c5c473eaeeb --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/state/MongoDbStateManager.java @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.state; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcState; +import io.airbyte.protocol.models.v0.AirbyteGlobalState; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteStreamState; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A state manager for MongoDB CDC syncs. + */ +public class MongoDbStateManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(MongoDbStateManager.class); + + /** + * The global CDC state. + */ + private MongoDbCdcState cdcState; + + /** + * Map of streams (name/namespace tuple) to the current stream state information stored in the + * state. + */ + private final Map pairToStreamState = new HashMap<>(); + + /** + * Creates a new {@link MongoDbStateManager} primed with the provided initial state. + * + * @param initialState The initial state to be stored in the state manager. + * @return A new {@link MongoDbStateManager} + */ + public static MongoDbStateManager createStateManager(final JsonNode initialState) { + final MongoDbStateManager stateManager = new MongoDbStateManager(); + + if (initialState == null) { + return stateManager; + } + + LOGGER.info("Initial state {}", initialState); + final List stateMessages = deserializeState(initialState); + + if (!stateMessages.isEmpty()) { + if (stateMessages.size() == 1) { + final AirbyteStateMessage stateMessage = stateMessages.get(0); + stateManager.updateCdcState(Jsons.object(stateMessage.getGlobal().getSharedState(), MongoDbCdcState.class)); + stateMessage.getGlobal().getStreamStates() + .forEach(s -> stateManager.updateStreamState(s.getStreamDescriptor().getName(), s.getStreamDescriptor().getNamespace(), + Jsons.object(s.getStreamState(), MongoDbStreamState.class))); + } else { + throw new IllegalStateException("The state contains multiple message, but only 1 is expected."); + } + } + + return stateManager; + } + + private static List deserializeState(final JsonNode initialState) { + try { + return Jsons.object(initialState, new TypeReference<>() {}); + } catch (final IllegalArgumentException e) { + LOGGER.debug("Failed to deserialize initial state {}.", initialState, e); + return List.of(); + } + } + + /** + * Creates a new {@link MongoDbStateManager} instance. This constructor should not be called + * directly. Instead, use {@link #createStateManager(JsonNode)}. + */ + private MongoDbStateManager() {} + + /** + * Returns the global, CDC state stored by the manager. + * + * @return A {@link MongoDbCdcState} instance that represents the global state. + */ + public MongoDbCdcState getCdcState() { + return cdcState; + } + + /** + * Returns the current stream state for the given stream. + * + * @param streamName The name of the stream. + * @param streamNamespace The namespace of the stream. + * @return The {@link MongoDbStreamState} associated with the stream or an empty {@link Optional} if + * the stream is not currently tracked by the manager. + */ + public Optional getStreamState(final String streamName, final String streamNamespace) { + final AirbyteStreamNameNamespacePair airbyteStreamNameNamespacePair = new AirbyteStreamNameNamespacePair(streamName, streamNamespace); + return Optional.ofNullable(pairToStreamState.get(airbyteStreamNameNamespacePair)); + } + + public Map getStreamStates() { + return Map.copyOf(pairToStreamState); + } + + /** + * Updates the global, CDC state tracked by the manager. + * + * @param cdcState The new global, CDC state as an {@link MongoDbCdcState} instance. + */ + public void updateCdcState(final MongoDbCdcState cdcState) { + LOGGER.debug("Updating CDC state to {}...", cdcState); + this.cdcState = cdcState; + } + + /** + * Updates the state associated with a stream. + * + * @param streamName The name of the stream. + * @param streamNamespace The namespace of the stream. + * @param streamState The new stream state. + */ + public void updateStreamState(final String streamName, final String streamNamespace, final MongoDbStreamState streamState) { + final AirbyteStreamNameNamespacePair airbyteStreamNameNamespacePair = new AirbyteStreamNameNamespacePair(streamName, streamNamespace); + LOGGER.debug("Updating stream state for stream {}:{} to {}...", streamNamespace, streamName, streamState); + pairToStreamState.put(airbyteStreamNameNamespacePair, streamState); + } + + /** + * Resets the state stored in this manager by overwriting the CDC state and clearing the stream + * state. + * + * @param cdcState The new CDC state. + */ + public void resetState(final MongoDbCdcState cdcState) { + LOGGER.debug("Resetting state with CDC state {}...", cdcState); + updateCdcState(cdcState); + pairToStreamState.clear(); + } + + /** + * Generates an {@link AirbyteStateMessage} from the state tracked by this manager. The resulting + * state message contains a global state object with the CDC state as the "shared state" and the + * individual stream states as the "stream states". + * + * @return An {@link AirbyteStateMessage} that represents the state stored by the manager. + */ + public AirbyteStateMessage toState() { + // Populate global state + final AirbyteGlobalState globalState = new AirbyteGlobalState(); + // TODO For now, handle the null cdc state case. Once integrated with Debezium, we should + // never hit this scenario, as we should set the cdc state to the initial offset retrieved at start + // of the sync. + final MongoDbCdcState cdcState = getCdcState(); + globalState.setSharedState(cdcState != null ? Jsons.jsonNode(cdcState) : Jsons.emptyObject()); + globalState.setStreamStates(generateStreamStateList(pairToStreamState)); + + return new AirbyteStateMessage() + .withType(AirbyteStateMessage.AirbyteStateType.GLOBAL) + .withGlobal(globalState); + } + + private List generateStreamStateList(final Map pairToCursorInfoMap) { + return pairToCursorInfoMap.entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .map(e -> generateStreamState(e.getKey(), e.getValue())) + .filter(s -> isValidStreamDescriptor(s.getStreamDescriptor())) + .collect(Collectors.toList()); + } + + private AirbyteStreamState generateStreamState(final AirbyteStreamNameNamespacePair airbyteStreamNameNamespacePair, + final MongoDbStreamState streamState) { + return new AirbyteStreamState() + .withStreamDescriptor( + new StreamDescriptor() + .withName(airbyteStreamNameNamespacePair.getName()) + .withNamespace(airbyteStreamNameNamespacePair.getNamespace())) + .withStreamState(Jsons.jsonNode(streamState)); + } + + /** + * Tests whether the provided {@link StreamDescriptor} is valid. A valid descriptor is defined as + * one that has a non-{@code null} name. + *

      + * See the + * Airbyte protocol for more details + * + * @param streamDescriptor A {@link StreamDescriptor} to be validated. + * @return {@code true} if the provided {@link StreamDescriptor} is valid or {@code false} if it is + * invalid. + */ + private boolean isValidStreamDescriptor(final StreamDescriptor streamDescriptor) { + if (streamDescriptor != null) { + return streamDescriptor.getName() != null; + } else { + return false; + } + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/state/MongoDbStreamState.java b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/state/MongoDbStreamState.java new file mode 100644 index 000000000000..2dc8b74eee9b --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/java/io/airbyte/integrations/source/mongodb/state/MongoDbStreamState.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.state; + +public record MongoDbStreamState(String id, InitialSnapshotStatus status, IdType idType) { + + /** + * Takes a value converting it to the appropriate MongoDb type based on the IdType of this record. + * + * @param value the value to convert + * @return a converted value. + */ + public Object idTypeAsMongoDbType(final String value) { + return idType.convert(value); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json index 2f535f07687a..9c4af4f046c9 100644 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/main/resources/spec.json @@ -5,117 +5,178 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "MongoDb Source Spec", "type": "object", - "required": ["database"], + "required": ["database_config"], "additionalProperties": true, "properties": { - "instance_type": { + "database_config": { "type": "object", - "title": "MongoDb Instance Type", - "description": "The MongoDb instance to connect to. For MongoDB Atlas and Replica Set TLS connection is used by default.", - "order": 0, + "title": "Cluster Type", + "description": "Configures the MongoDB cluster type.", + "order": 1, + "group": "connection", + "display_type": "radio", "oneOf": [ { - "title": "Standalone MongoDb Instance", - "required": ["instance", "host", "port"], + "title": "MongoDB Atlas Replica Set", + "description": "MongoDB Atlas-hosted cluster configured as a replica set", + "required": [ + "cluster_type", + "connection_string", + "database", + "username", + "password", + "auth_source" + ], + "additionalProperties": true, "properties": { - "instance": { + "cluster_type": { "type": "string", - "const": "standalone" + "const": "ATLAS_REPLICA_SET", + "order": 1 }, - "host": { - "title": "Host", + "connection_string": { + "title": "Connection String", "type": "string", - "description": "The host name of the Mongo database.", - "order": 0 + "description": "The connection string of the cluster that you want to replicate.", + "examples": ["mongodb+srv://cluster0.abcd1.mongodb.net/"], + "order": 2 }, - "port": { - "title": "Port", - "type": "integer", - "description": "The port of the Mongo database.", - "minimum": 0, - "maximum": 65536, - "default": 27017, - "examples": ["27017"], - "order": 1 + "database": { + "title": "Database Name", + "type": "string", + "description": "The name of the MongoDB database that contains the collection(s) to replicate.", + "order": 3 }, - "tls": { - "title": "TLS Connection", - "type": "boolean", - "description": "Indicates whether TLS encryption protocol will be used to connect to MongoDB. It is recommended to use TLS connection if possible. For more information see documentation.", - "default": false, - "order": 2 - } - } - }, - { - "title": "Replica Set", - "required": ["instance", "server_addresses"], - "properties": { - "instance": { + "username": { + "title": "Username", "type": "string", - "const": "replica" + "description": "The username which is used to access the database.", + "order": 4 }, - "server_addresses": { - "title": "Server Addresses", + "password": { + "title": "Password", "type": "string", - "description": "The members of a replica set. Please specify `host`:`port` of each member separated by comma.", - "examples": ["host1:27017,host2:27017,host3:27017"], - "order": 0 + "description": "The password associated with this username.", + "airbyte_secret": true, + "order": 5 }, - "replica_set": { - "title": "Replica Set", + "auth_source": { + "title": "Authentication Source", "type": "string", - "description": "A replica set in MongoDB is a group of mongod processes that maintain the same data set.", - "order": 1 + "description": "The authentication source where the user information is stored. See https://www.mongodb.com/docs/manual/reference/connection-string/#mongodb-urioption-urioption.authSource for more details.", + "default": "admin", + "examples": ["admin"], + "order": 6 + }, + "schema_enforced": { + "title": "Schema Enforced", + "description": "When enabled, syncs will validate and structure records against the stream's schema.", + "default": true, + "type": "boolean", + "always_show": true, + "order": 7 } } }, { - "title": "MongoDB Atlas", + "title": "Self-Managed Replica Set", + "description": "MongoDB self-hosted cluster configured as a replica set", + "required": ["cluster_type", "connection_string", "database"], "additionalProperties": true, - "required": ["instance", "cluster_url"], "properties": { - "instance": { + "cluster_type": { + "type": "string", + "const": "SELF_MANAGED_REPLICA_SET", + "order": 1 + }, + "connection_string": { + "title": "Connection String", + "type": "string", + "description": "The connection string of the cluster that you want to replicate. https://www.mongodb.com/docs/manual/reference/connection-string/#find-your-self-hosted-deployment-s-connection-string for more information.", + "examples": [ + "mongodb://example1.host.com:27017,example2.host.com:27017,example3.host.com:27017/", + "mongodb://example.host.com:27017/" + ], + "order": 2 + }, + "database": { + "title": "Database Name", "type": "string", - "const": "atlas" + "description": "The name of the MongoDB database that contains the collection(s) to replicate.", + "order": 3 }, - "cluster_url": { - "title": "Cluster URL", + "username": { + "title": "Username", "type": "string", - "description": "The URL of a cluster to connect to.", - "order": 0 + "description": "The username which is used to access the database.", + "order": 4 + }, + "password": { + "title": "Password", + "type": "string", + "description": "The password associated with this username.", + "airbyte_secret": true, + "order": 5 + }, + "auth_source": { + "title": "Authentication Source", + "type": "string", + "description": "The authentication source where the user information is stored.", + "default": "admin", + "examples": ["admin"], + "order": 6 + }, + "schema_enforced": { + "title": "Schema Enforced", + "description": "When enabled, syncs will validate and structure records against the stream's schema.", + "default": true, + "type": "boolean", + "always_show": true, + "order": 7 } } } ] }, - "database": { - "title": "Database Name", - "type": "string", - "description": "The database you want to replicate.", - "order": 1 + "initial_waiting_seconds": { + "type": "integer", + "title": "Initial Waiting Time in Seconds (Advanced)", + "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds.", + "default": 300, + "order": 8, + "min": 120, + "max": 1200, + "group": "advanced" }, - "user": { - "title": "User", - "type": "string", - "description": "The username which is used to access the database.", - "order": 2 + "queue_size": { + "type": "integer", + "title": "Size of the queue (Advanced)", + "description": "The size of the internal queue. This may interfere with memory consumption and efficiency of the connector, please be careful.", + "default": 10000, + "order": 9, + "min": 1000, + "max": 10000, + "group": "advanced" }, - "password": { - "title": "Password", - "type": "string", - "description": "The password associated with this username.", - "airbyte_secret": true, - "order": 3 + "discover_sample_size": { + "type": "integer", + "title": "Document discovery sample size (Advanced)", + "description": "The maximum number of documents to sample when attempting to discover the unique fields for a collection.", + "default": 10000, + "order": 10, + "minimum": 10, + "maximum": 100000, + "group": "advanced" + } + }, + "groups": [ + { + "id": "connection" }, - "auth_source": { - "title": "Authentication Source", - "type": "string", - "description": "The authentication source where the user information is stored.", - "default": "admin", - "examples": ["admin"], - "order": 4 + { + "id": "advanced", + "title": "Advanced" } - } + ] } } diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAbstractAcceptanceTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAbstractAcceptanceTest.java deleted file mode 100644 index 30f8a1c0d71a..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAbstractAcceptanceTest.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.io.airbyte.integration_tests.sources; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.Lists; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.db.mongodb.MongoDatabase; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.SyncMode; -import java.util.HashMap; -import java.util.List; - -public abstract class MongoDbSourceAbstractAcceptanceTest extends SourceAcceptanceTest { - - protected static final String DATABASE_NAME = "test"; - protected static final String COLLECTION_NAME = "acceptance_test1"; - - protected JsonNode config; - protected MongoDatabase database; - - @Override - protected String getImageName() { - return "airbyte/source-mongodb-v2:dev"; - } - - @Override - protected JsonNode getConfig() throws Exception { - return config; - } - - @Override - protected ConnectorSpecification getSpec() throws Exception { - return Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class); - } - - @Override - protected ConfiguredAirbyteCatalog getConfiguredCatalog() throws Exception { - return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(Lists.newArrayList("_id")) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withCursorField(List.of("_id")) - .withStream(CatalogHelpers.createAirbyteStream( - DATABASE_NAME + "." + COLLECTION_NAME, - Field.of("_id", JsonSchemaType.STRING), - Field.of("id", JsonSchemaType.STRING), - Field.of("name", JsonSchemaType.STRING), - Field.of("test", JsonSchemaType.STRING), - Field.of("test_array", JsonSchemaType.ARRAY), - Field.of("empty_test", JsonSchemaType.STRING), - Field.of("double_test", JsonSchemaType.NUMBER), - Field.of("int_test", JsonSchemaType.NUMBER), - Field.of("object_test", JsonSchemaType.OBJECT)) - .withSupportedSyncModes(Lists.newArrayList(SyncMode.INCREMENTAL)) - .withDefaultCursorField(List.of("_id"))))); - } - - @Override - protected JsonNode getState() throws Exception { - return Jsons.jsonNode(new HashMap<>()); - } - -} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAtlasAcceptanceTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAtlasAcceptanceTest.java deleted file mode 100644 index 6cae561d5527..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceAtlasAcceptanceTest.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.io.airbyte.integration_tests.sources; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.mongodb.client.MongoCollection; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.mongodb.MongoDatabase; -import io.airbyte.integrations.source.mongodb.MongoDbSource; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.AirbyteCatalog; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import io.airbyte.protocol.models.v0.AirbyteStream; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import org.bson.BsonArray; -import org.bson.BsonString; -import org.bson.Document; -import org.junit.jupiter.api.Test; - -public class MongoDbSourceAtlasAcceptanceTest extends MongoDbSourceAbstractAcceptanceTest { - - private static final Path CREDENTIALS_PATH = Path.of("secrets/credentials.json"); - - protected static final List SUB_FIELDS = List.of( - Field.of("testObject", JsonSchemaType.OBJECT, List.of( - Field.of("name", JsonSchemaType.STRING), - Field.of("testField1", JsonSchemaType.STRING), - Field.of("testInt", JsonSchemaType.NUMBER), - Field.of("thirdLevelDocument", JsonSchemaType.OBJECT, List.of( - Field.of("data", JsonSchemaType.STRING), - Field.of("intData", JsonSchemaType.NUMBER)))))); - - protected static final List FIELDS = List.of( - Field.of("id", JsonSchemaType.STRING), - Field.of("_id", JsonSchemaType.STRING), - Field.of("name", JsonSchemaType.STRING), - Field.of("test_aibyte_transform", JsonSchemaType.STRING), - Field.of("test_array", JsonSchemaType.ARRAY), - Field.of("int_test", JsonSchemaType.NUMBER), - Field.of("double_test", JsonSchemaType.NUMBER), - Field.of("object_test", JsonSchemaType.OBJECT, SUB_FIELDS)); - - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - if (!Files.exists(CREDENTIALS_PATH)) { - throw new IllegalStateException( - "Must provide path to a MongoDB credentials file. By default {module-root}/" + CREDENTIALS_PATH - + ". Override by setting setting path with the CREDENTIALS_PATH constant."); - } - - config = Jsons.deserialize(Files.readString(CREDENTIALS_PATH)); - ((ObjectNode) config).put(JdbcUtils.DATABASE_KEY, DATABASE_NAME); - - final String connectionString = String.format("mongodb+srv://%s:%s@%s/%s?authSource=admin&retryWrites=true&w=majority&tls=true", - config.get("user").asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - config.get("instance_type").get("cluster_url").asText(), - config.get(JdbcUtils.DATABASE_KEY).asText()); - - database = new MongoDatabase(connectionString, config.get(JdbcUtils.DATABASE_KEY).asText()); - - final MongoCollection collection = database.createCollection(COLLECTION_NAME); - final var objectDocument = new Document("testObject", new Document("name", "subName").append("testField1", "testField1").append("testInt", 10) - .append("thirdLevelDocument", new Document("data", "someData").append("intData", 1))); - final var doc1 = new Document("id", "0001").append("name", "Test") - .append("test", 10).append("test_array", new BsonArray(List.of(new BsonString("test"), new BsonString("mongo")))) - .append("double_test", 100.12).append("int_test", 100).append("object_test", objectDocument); - final var doc2 = - new Document("id", "0002").append("name", "Mongo").append("test", "test_value").append("int_test", 201).append("object_test", objectDocument); - final var doc3 = new Document("id", "0003").append("name", "Source").append("test", null) - .append("double_test", 212.11).append("int_test", 302).append("object_test", objectDocument); - - collection.insertMany(List.of(doc1, doc2, doc3)); - } - - @Override - protected void tearDown(final TestDestinationEnv testEnv) throws Exception { - database.getDatabase().getCollection(COLLECTION_NAME).drop(); - database.close(); - } - - @Override - protected void verifyCatalog(final AirbyteCatalog catalog) { - final List streams = catalog.getStreams(); - // only one stream is expected; the schema that should be ignored - // must not be included in the retrieved catalog - assertEquals(1, streams.size()); - final AirbyteStream actualStream = streams.get(0); - assertEquals(CatalogHelpers.fieldsToJsonSchema(FIELDS), actualStream.getJsonSchema()); - } - - @Test - public void testCheckIncorrectUsername() throws Exception { - ((ObjectNode) config).put("user", "fake"); - final AirbyteConnectionStatus status = new MongoDbSource().check(config); - assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 18")); - } - - @Test - public void testCheckIncorrectPassword() throws Exception { - ((ObjectNode) config).put("password", "fake"); - final AirbyteConnectionStatus status = new MongoDbSource().check(config); - assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 18")); - } - - @Test - public void testCheckIncorrectCluster() throws Exception { - final String badClusterUrl = "cluster0.iqgf8.mongodb.netfail"; - config.withObject("/instance_type").put("cluster_url", badClusterUrl); - final AirbyteConnectionStatus status = new MongoDbSource().check(config); - assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().matches("State code: -\\d+.*")); - } - - @Test - public void testCheckIncorrectAccessToDataBase() throws Exception { - ((ObjectNode) config).put("user", "test_user_without_access") - .put("password", "test12321"); - final AirbyteConnectionStatus status = new MongoDbSource().check(config); - assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 13")); - } - -} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceDataTypeTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceDataTypeTest.java deleted file mode 100644 index e1f4f9a88822..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceDataTypeTest.java +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.io.airbyte.integration_tests.sources; - -import static io.airbyte.db.mongodb.MongoUtils.MongoInstanceType.STANDALONE; -import static java.lang.Double.NaN; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.util.MoreIterators; -import io.airbyte.db.DataTypeUtils; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.mongodb.MongoDatabase; -import io.airbyte.integrations.source.mongodb.MongoDbSource; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteMessage.Type; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.SyncMode; -import java.time.LocalDate; -import java.time.ZoneId; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.UUID; -import org.bson.BsonArray; -import org.bson.BsonBinary; -import org.bson.BsonBoolean; -import org.bson.BsonDateTime; -import org.bson.BsonDecimal128; -import org.bson.BsonDocument; -import org.bson.BsonDouble; -import org.bson.BsonInt32; -import org.bson.BsonInt64; -import org.bson.BsonJavaScript; -import org.bson.BsonJavaScriptWithScope; -import org.bson.BsonObjectId; -import org.bson.BsonString; -import org.bson.BsonSymbol; -import org.bson.BsonTimestamp; -import org.bson.types.Decimal128; -import org.bson.types.ObjectId; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.MongoDBContainer; -import org.testcontainers.utility.DockerImageName; - -public class MongoDbSourceDataTypeTest { - - private static final String STREAM_NAME = "test.acceptance_test"; - private static final long MILLI = LocalDate.now().atStartOfDay(ZoneId.systemDefault()).toInstant().toEpochMilli(); - - private MongoDBContainer mongoDBContainer; - private MongoDatabase database; - private JsonNode config; - - @BeforeEach - public void setup() { - mongoDBContainer = new MongoDBContainer(DockerImageName.parse("mongo:4.0.10")); - mongoDBContainer.start(); - - final String connectionString = String.format("mongodb://%s:%s/", - mongoDBContainer.getHost(), - mongoDBContainer.getFirstMappedPort()); - - final JsonNode instanceConfig = Jsons.jsonNode(ImmutableMap.builder() - .put("instance", STANDALONE.getType()) - .put(JdbcUtils.HOST_KEY, mongoDBContainer.getHost()) - .put(JdbcUtils.PORT_KEY, mongoDBContainer.getFirstMappedPort()) - .put(JdbcUtils.TLS_KEY, false) - .build()); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put("instance_type", instanceConfig) - .put(JdbcUtils.DATABASE_KEY, "test") - .put("auth_source", "admin") - .build()); - - database = new MongoDatabase(connectionString, "test"); - database.createCollection("acceptance_test"); - - final BsonDocument bsonDocument = new BsonDocument() - .append("_id", new BsonObjectId(new ObjectId("61703280f3ca180ab088b574"))) - .append("boolean", BsonBoolean.TRUE) - .append("int32", new BsonInt32(Integer.MAX_VALUE)) - .append("int64", new BsonInt64(Long.MAX_VALUE)) - .append("double", new BsonDouble(Double.MAX_VALUE)) - .append("decimal", new BsonDecimal128(Decimal128.NaN)) - .append("tms", new BsonTimestamp(MILLI)) - .append("dateTime", new BsonDateTime(MILLI)) - .append("binary", new BsonBinary(new UUID(10, 15))) - .append("symbol", new BsonSymbol("s")) - .append("string", new BsonString("test mongo db")) - .append("objectId", new BsonObjectId(new ObjectId("6035210f35bd203721c3eab8"))) - .append("javaScript", - new BsonJavaScript("var str = \"The best things in life are free\";\nvar patt = new RegExp(\"e\");\nvar res = patt.test(str);")) - .append("javaScriptWithScope", new BsonJavaScriptWithScope("function (x){ return ++x; }", - new BsonDocument().append("x1", new BsonInt32(256)).append("x2", new BsonInt32(142)))) - .append("document", new BsonDocument("test", new BsonString("let's test!")).append("number", new BsonInt32(1352))) - .append("arrayWithDocs", new BsonArray(Arrays.asList( - new BsonDocument().append("title", new BsonString("One Hundred Years of Solitude")).append("yearPublished", new BsonInt32(1967)), - new BsonDocument().append("title", new BsonString("Chronicle of a Death Foretold")).append("yearPublished", new BsonInt32(1981)), - new BsonDocument().append("title", new BsonString("Love in the Time of Cholera")).append("yearPublished", new BsonInt32(1985))))) - .append("arrayWithStrings", new BsonArray(List.of(new BsonString("test"), new BsonString("mongo"), new BsonString("types")))); - database.getDatabase().getCollection("acceptance_test", BsonDocument.class).insertOne(bsonDocument); - } - - @AfterEach - public void tearDown() throws Exception { - database.close(); - mongoDBContainer.close(); - } - - @Test - public void run() throws Exception { - final List actualMessages = - MoreIterators.toList( - new MongoDbSource().read(config, getConfiguredCatalog(), Jsons.jsonNode(new HashMap<>()))); - - setEmittedAtToNull(actualMessages); - final List expectedMessages = getExpectedMessages(); - - assertEquals(expectedMessages, actualMessages); - } - - private ConfiguredAirbyteCatalog getConfiguredCatalog() { - return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.FULL_REFRESH) - .withCursorField(Lists.newArrayList()) - .withDestinationSyncMode(DestinationSyncMode.OVERWRITE) - .withPrimaryKey(Lists.newArrayList()) - .withStream(CatalogHelpers.createAirbyteStream( - "test.acceptance_test", - Field.of("_id", JsonSchemaType.STRING), - Field.of("boolean", JsonSchemaType.BOOLEAN), - Field.of("int32", JsonSchemaType.NUMBER), - Field.of("int64", JsonSchemaType.NUMBER), - Field.of("double", JsonSchemaType.NUMBER), - Field.of("decimal", JsonSchemaType.NUMBER), - Field.of("tms", JsonSchemaType.STRING), - Field.of("dateTime", JsonSchemaType.STRING), - Field.of("binary", JsonSchemaType.STRING), - Field.of("symbol", JsonSchemaType.STRING), - Field.of("string", JsonSchemaType.STRING), - Field.of("objectId", JsonSchemaType.STRING), - Field.of("javaScript", JsonSchemaType.STRING), - Field.of("javaScriptWithScope", JsonSchemaType.OBJECT), - Field.of("document", JsonSchemaType.OBJECT), - Field.of("arrayWithDocs", JsonSchemaType.ARRAY), - Field.of("arrayWithStrings", JsonSchemaType.ARRAY)) - .withSupportedSyncModes(Lists.newArrayList(SyncMode.INCREMENTAL, SyncMode.FULL_REFRESH)) - .withSourceDefinedPrimaryKey(List.of(List.of("_id")))))); - } - - private List getExpectedMessages() { - return Lists.newArrayList( - new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(STREAM_NAME) - .withData(Jsons.jsonNode(ImmutableMap.builder() - .put("_id", "61703280f3ca180ab088b574") - .put("boolean", true) - .put("int32", 2147483647) - .put("int64", 9223372036854775807L) - .put("double", 1.7976931348623157E308) - .put("decimal", NaN) - .put("tms", DataTypeUtils.toISO8601StringWithMilliseconds(MILLI)) - .put("dateTime", DataTypeUtils.toISO8601StringWithMilliseconds(MILLI)) - .put("binary", new BsonBinary(new UUID(10, 15)).getData()) - .put("symbol", "s") - .put("string", "test mongo db") - .put("objectId", "6035210f35bd203721c3eab8") - .put("javaScript", "var str = \"The best things in life are free\";\nvar patt = new RegExp(\"e\");\nvar res = patt.test(str);") - .put("javaScriptWithScope", Jsons.jsonNode(ImmutableMap.of( - "code", "function (x){ return ++x; }", - "scope", Jsons.jsonNode(ImmutableMap.of("x1", 256, "x2", 142))))) - .put("document", Jsons.jsonNode(ImmutableMap.of("test", "let's test!", "number", 1352))) - .put("arrayWithDocs", Jsons.jsonNode(Lists.newArrayList( - Jsons.jsonNode(ImmutableMap.of("title", "One Hundred Years of Solitude", "yearPublished", 1967)), - Jsons.jsonNode(ImmutableMap.of("title", "Chronicle of a Death Foretold", "yearPublished", 1981)), - Jsons.jsonNode(ImmutableMap.of("title", "Love in the Time of Cholera", "yearPublished", 1985))))) - .put("arrayWithStrings", Jsons.jsonNode(Lists.newArrayList("test", "mongo", "types"))) - .build())))); - } - - private void setEmittedAtToNull(final Iterable messages) { - for (final AirbyteMessage actualMessage : messages) { - if (actualMessage.getRecord() != null) { - actualMessage.getRecord().setEmittedAt(null); - } - } - } - -} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceStandaloneAcceptanceTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceStandaloneAcceptanceTest.java deleted file mode 100644 index 8e5bd029bb92..000000000000 --- a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MongoDbSourceStandaloneAcceptanceTest.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.io.airbyte.integration_tests.sources; - -import static io.airbyte.db.mongodb.MongoUtils.MongoInstanceType.STANDALONE; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import com.mongodb.client.MongoCollection; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.mongodb.MongoDatabase; -import io.airbyte.integrations.source.mongodb.MongoDbSource; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.AirbyteCatalog; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import io.airbyte.protocol.models.v0.AirbyteStream; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import java.util.List; -import org.bson.BsonArray; -import org.bson.BsonString; -import org.bson.Document; -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.MongoDBContainer; -import org.testcontainers.utility.DockerImageName; - -public class MongoDbSourceStandaloneAcceptanceTest extends MongoDbSourceAbstractAcceptanceTest { - - private MongoDBContainer mongoDBContainer; - - private static final List SUB_FIELDS = List.of( - Field.of("testObject", JsonSchemaType.OBJECT, List.of( - Field.of("name", JsonSchemaType.STRING), - Field.of("testField1", JsonSchemaType.STRING), - Field.of("testInt", JsonSchemaType.NUMBER), - Field.of("thirdLevelDocument", JsonSchemaType.OBJECT, List.of( - Field.of("data", JsonSchemaType.STRING), - Field.of("intData", JsonSchemaType.NUMBER)))))); - - private static final List FIELDS = List.of( - Field.of("id", JsonSchemaType.STRING), - Field.of("_id", JsonSchemaType.STRING), - Field.of("name", JsonSchemaType.STRING), - Field.of("test_aibyte_transform", JsonSchemaType.STRING), - Field.of("test_array", JsonSchemaType.ARRAY), - Field.of("int_test", JsonSchemaType.NUMBER), - Field.of("double_test", JsonSchemaType.NUMBER), - Field.of("object_test", JsonSchemaType.OBJECT, SUB_FIELDS)); - - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - mongoDBContainer = new MongoDBContainer(DockerImageName.parse("mongo:4.0.10")); - mongoDBContainer.start(); - - final JsonNode instanceConfig = Jsons.jsonNode(ImmutableMap.builder() - .put("instance", STANDALONE.getType()) - .put(JdbcUtils.HOST_KEY, mongoDBContainer.getHost()) - .put(JdbcUtils.PORT_KEY, mongoDBContainer.getFirstMappedPort()) - .put(JdbcUtils.TLS_KEY, false) - .build()); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put("instance_type", instanceConfig) - .put(JdbcUtils.DATABASE_KEY, DATABASE_NAME) - .put("auth_source", "admin") - .build()); - - final var connectionString = String.format("mongodb://%s:%s/", - mongoDBContainer.getHost(), - mongoDBContainer.getFirstMappedPort()); - - database = new MongoDatabase(connectionString, DATABASE_NAME); - - final MongoCollection collection = database.createCollection(COLLECTION_NAME); - final var objectDocument = new Document("testObject", new Document("name", "subName").append("testField1", "testField1").append("testInt", 10) - .append("thirdLevelDocument", new Document("data", "someData").append("intData", 1))); - final var doc1 = new Document("id", "0001").append("name", "Test") - .append("test", 10).append("test_array", new BsonArray(List.of(new BsonString("test"), new BsonString("mongo")))) - .append("double_test", 100.12).append("int_test", 100).append("object_test", objectDocument); - final var doc2 = - new Document("id", "0002").append("name", "Mongo").append("test", "test_value").append("int_test", 201).append("object_test", objectDocument); - final var doc3 = new Document("id", "0003").append("name", "Source").append("test", null) - .append("double_test", 212.11).append("int_test", 302).append("object_test", objectDocument); - - collection.insertMany(List.of(doc1, doc2, doc3)); - } - - @Override - protected void tearDown(final TestDestinationEnv testEnv) throws Exception { - database.close(); - mongoDBContainer.close(); - } - - @Override - protected void verifyCatalog(final AirbyteCatalog catalog) { - final List streams = catalog.getStreams(); - // only one stream is expected; the schema that should be ignored - // must not be included in the retrieved catalog - assertEquals(1, streams.size()); - final AirbyteStream actualStream = streams.get(0); - assertEquals(CatalogHelpers.fieldsToJsonSchema(FIELDS), actualStream.getJsonSchema()); - } - - @Test - public void testCheckIncorrectHost() throws Exception { - final JsonNode instanceConfig = Jsons.jsonNode(ImmutableMap.builder() - .put("instance", STANDALONE.getType()) - .put("host", "localhost2") - .put("port", mongoDBContainer.getFirstMappedPort()) - .put("tls", false) - .build()); - - final JsonNode conf = Jsons.jsonNode(ImmutableMap.builder() - .put("instance_type", instanceConfig) - .put("database", DATABASE_NAME) - .put("auth_source", "admin") - .build()); - final AirbyteConnectionStatus status = new MongoDbSource().check(conf); - assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: -3")); - } - - @Test - public void testCheckIncorrectPort() throws Exception { - final JsonNode instanceConfig = Jsons.jsonNode(ImmutableMap.builder() - .put("instance", STANDALONE.getType()) - .put("host", mongoDBContainer.getHost()) - .put("port", 1234) - .put("tls", false) - .build()); - - final JsonNode conf = Jsons.jsonNode(ImmutableMap.builder() - .put("instance_type", instanceConfig) - .put("database", DATABASE_NAME) - .put("auth_source", "admin") - .build()); - final AirbyteConnectionStatus status = new MongoDbSource().check(conf); - assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: -3")); - } - -} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongoDbSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongoDbSourceAcceptanceTest.java new file mode 100644 index 000000000000..cef1eeb91f10 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test-integration/java/io/airbyte/integrations/source/mongodb/MongoDbSourceAcceptanceTest.java @@ -0,0 +1,659 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb; + +import static io.airbyte.integrations.source.mongodb.MongoConstants.DATABASE_CONFIG_CONFIGURATION_KEY; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Iterables; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.model.Updates; +import io.airbyte.cdk.integrations.debezium.internals.ChangeEventWithMetadata; +import io.airbyte.cdk.integrations.debezium.internals.SnapshotMetadata; +import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbCdcTargetPosition; +import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants; +import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumStateUtil; +import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbResumeTokenHelper; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcState; +import io.airbyte.integrations.source.mongodb.state.InitialSnapshotStatus; +import io.airbyte.integrations.source.mongodb.state.MongoDbStreamState; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteGlobalState; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.AirbyteStreamState; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.ConnectorSpecification; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import io.airbyte.protocol.models.v0.SyncMode; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.apache.commons.lang3.RandomStringUtils; +import org.bson.BsonArray; +import org.bson.BsonDocument; +import org.bson.BsonObjectId; +import org.bson.BsonString; +import org.bson.BsonTimestamp; +import org.bson.BsonValue; +import org.bson.Document; +import org.bson.types.ObjectId; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class MongoDbSourceAcceptanceTest extends SourceAcceptanceTest { + + private static final Path CREDENTIALS_PATH = Path.of("secrets/credentials.json"); + public static final String DOCUMENT_ID_FIELD = "_id"; + private static final String DOUBLE_TEST_FIELD = "double_test"; + private static final String EMPTY_TEST_FIELD = "empty_test"; + private static final String ID_FIELD = "id"; + private static final String INT_TEST_FIELD = "int_test"; + private static final String INVALID_RESUME_TOKEN = "820000000000000000000000296E04"; + private static final String NAME_FIELD = "name"; + private static final String OBJECT_TEST_FIELD = "object_test"; + private static final String TEST_FIELD = "test"; + private static final String TEST_ARRAY_FIELD = "test_array"; + private static final ObjectId OBJECT_ID1 = new ObjectId("64c0029d95ad260d69ef28a0"); + private static final ObjectId OBJECT_ID2 = new ObjectId("64c0029d95ad260d69ef28a1"); + private static final ObjectId OBJECT_ID3 = new ObjectId("64c0029d95ad260d69ef28a2"); + + private JsonNode config; + private String collectionName; + private String databaseName; + private MongoClient mongoClient; + private String otherCollection1Name; + private String otherCollection2Name; + private int recordCount = 0; + + @Override + protected void setupEnvironment(final TestDestinationEnv testEnv) throws Exception { + if (!Files.exists(CREDENTIALS_PATH)) { + throw new IllegalStateException( + "Must provide path to a MongoDB credentials file. By default {module-root}/" + CREDENTIALS_PATH + + ". Override by setting setting path with the CREDENTIALS_PATH constant."); + } + + // Randomly generate the names to avoid collisions with other tests + collectionName = "collection_" + RandomStringUtils.randomAlphabetic(8); + databaseName = "acceptance_test_" + RandomStringUtils.randomAlphabetic(8); + otherCollection1Name = "collection_" + RandomStringUtils.randomAlphabetic(8); + otherCollection2Name = "collection_" + RandomStringUtils.randomAlphabetic(8); + + config = Jsons.deserialize(Files.readString(CREDENTIALS_PATH)); + final ObjectNode databaseConfig = (ObjectNode) config.get(DATABASE_CONFIG_CONFIGURATION_KEY); + databaseConfig.put(MongoConstants.DATABASE_CONFIGURATION_KEY, databaseName); + databaseConfig.put(MongoConstants.IS_TEST_CONFIGURATION_KEY, true); + databaseConfig.put(MongoConstants.CHECKPOINT_INTERVAL_CONFIGURATION_KEY, 1); + ((ObjectNode) config).put(DATABASE_CONFIG_CONFIGURATION_KEY, databaseConfig); + + final MongoDbSourceConfig sourceConfig = new MongoDbSourceConfig(config); + + mongoClient = MongoConnectionUtils.createMongoClient(sourceConfig); + createTestCollections(mongoClient); + insertTestData(mongoClient); + } + + private void createTestCollections(final MongoClient mongoClient) { + mongoClient.getDatabase(databaseName).getCollection(collectionName).drop(); + mongoClient.getDatabase(databaseName).getCollection(otherCollection1Name).drop(); + mongoClient.getDatabase(databaseName).getCollection(otherCollection2Name).drop(); + mongoClient.getDatabase(databaseName).createCollection(collectionName); + mongoClient.getDatabase(databaseName).createCollection(otherCollection1Name); + mongoClient.getDatabase(databaseName).createCollection(otherCollection2Name); + } + + private void insertTestData(final MongoClient mongoClient) { + final MongoCollection collection = mongoClient.getDatabase(databaseName).getCollection(collectionName); + final var objectDocument = + new Document("testObject", new Document(NAME_FIELD, "subName").append("testField1", "testField1").append(INT_TEST_FIELD, 10) + .append("thirdLevelDocument", new Document("data", "someData").append("intData", 1))); + + final var doc1 = new Document(DOCUMENT_ID_FIELD, OBJECT_ID1) + .append(ID_FIELD, "0001").append(NAME_FIELD, "Test") + .append(TEST_FIELD, 10).append(TEST_ARRAY_FIELD, new BsonArray(List.of(new BsonString("test"), new BsonString("mongo")))) + .append(DOUBLE_TEST_FIELD, 100.12).append(INT_TEST_FIELD, 100).append(OBJECT_TEST_FIELD, objectDocument); + + final var doc2 = new Document(DOCUMENT_ID_FIELD, OBJECT_ID2) + .append(ID_FIELD, "0002").append(NAME_FIELD, "Mongo").append(TEST_FIELD, "test_value").append(INT_TEST_FIELD, 201) + .append(OBJECT_TEST_FIELD, objectDocument); + + final var doc3 = new Document(DOCUMENT_ID_FIELD, OBJECT_ID3) + .append(ID_FIELD, "0003").append(NAME_FIELD, "Source").append(TEST_FIELD, null) + .append(DOUBLE_TEST_FIELD, 212.11).append(INT_TEST_FIELD, 302).append(OBJECT_TEST_FIELD, objectDocument); + + final List newDocuments = List.of(doc1, doc2, doc3); + recordCount += newDocuments.size(); + collection.insertMany(newDocuments); + } + + @Override + protected void tearDown(final TestDestinationEnv testEnv) { + mongoClient.getDatabase(databaseName).getCollection(collectionName).drop(); + mongoClient.getDatabase(databaseName).getCollection(otherCollection1Name).drop(); + mongoClient.getDatabase(databaseName).getCollection(otherCollection2Name).drop(); + mongoClient.getDatabase(databaseName).drop(); + mongoClient.close(); + recordCount = 0; + } + + @Override + protected String getImageName() { + return "airbyte/source-mongodb-v2:dev"; + } + + @Override + protected ConnectorSpecification getSpec() throws Exception { + return Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class); + } + + @Override + protected JsonNode getConfig() { + return config; + } + + @Override + protected ConfiguredAirbyteCatalog getConfiguredCatalog() { + final List fields = List.of( + Field.of(DOCUMENT_ID_FIELD, JsonSchemaType.STRING), + Field.of(ID_FIELD, JsonSchemaType.STRING), + Field.of(NAME_FIELD, JsonSchemaType.STRING), + Field.of(TEST_FIELD, JsonSchemaType.STRING), + Field.of(TEST_ARRAY_FIELD, JsonSchemaType.ARRAY), + Field.of(EMPTY_TEST_FIELD, JsonSchemaType.STRING), + Field.of(DOUBLE_TEST_FIELD, JsonSchemaType.NUMBER), + Field.of(INT_TEST_FIELD, JsonSchemaType.NUMBER), + Field.of(OBJECT_TEST_FIELD, JsonSchemaType.OBJECT)); + + return getConfiguredCatalog(fields); + } + + private ConfiguredAirbyteCatalog getConfiguredCatalog(final List enabledFields) { + final AirbyteStream airbyteStream = MongoCatalogHelper.buildAirbyteStream(collectionName, databaseName, enabledFields); + final ConfiguredAirbyteStream configuredIncrementalAirbyteStream = convertToConfiguredAirbyteStream(airbyteStream, SyncMode.INCREMENTAL); + final List streams = new ArrayList<>(); + streams.add(configuredIncrementalAirbyteStream); + return new ConfiguredAirbyteCatalog().withStreams(streams); + } + + @Override + protected JsonNode getState() { + return Jsons.jsonNode(new HashMap<>()); + } + + @Test + public void testIncrementalReadSelectedColumns() throws Exception { + final List selectedColumns = List.of(Field.of(NAME_FIELD, JsonSchemaType.STRING)); + final ConfiguredAirbyteCatalog catalog = getConfiguredCatalog(selectedColumns); + final List allMessages = runRead(catalog); + + final List records = filterRecords(allMessages); + assertFalse(records.isEmpty(), "Expected a incremental sync to produce records"); + verifyFieldNotExist(records, collectionName, DOUBLE_TEST_FIELD); + verifyFieldNotExist(records, collectionName, EMPTY_TEST_FIELD); + verifyFieldNotExist(records, collectionName, ID_FIELD); + verifyFieldNotExist(records, collectionName, INT_TEST_FIELD); + verifyFieldNotExist(records, collectionName, OBJECT_TEST_FIELD); + verifyFieldNotExist(records, collectionName, TEST_FIELD); + verifyFieldNotExist(records, collectionName, TEST_ARRAY_FIELD); + } + + @Test + void testSyncEmptyCollection() throws Exception { + final ConfiguredAirbyteCatalog configuredCatalog = getConfiguredCatalog(); + final AirbyteStream otherAirbyteStream = MongoCatalogHelper.buildAirbyteStream(otherCollection1Name, databaseName, + List.of(Field.of(NAME_FIELD, JsonSchemaType.STRING), Field.of(INT_TEST_FIELD, JsonSchemaType.NUMBER))); + configuredCatalog.withStreams(List.of(convertToConfiguredAirbyteStream(otherAirbyteStream, SyncMode.INCREMENTAL))); + + final List messages = runRead(configuredCatalog); + final List recordMessages = filterRecords(messages); + final List stateMessages = filterStateMessages(messages); + + assertEquals(0, recordMessages.size()); + assertEquals(1, stateMessages.size()); + + final AirbyteStateMessage lastStateMessage = Iterables.getLast(stateMessages); + assertNotNull(lastStateMessage.getGlobal().getSharedState()); + assertFalse(lastStateMessage.getGlobal().getSharedState().isEmpty()); + assertTrue(lastStateMessage.getGlobal().getStreamStates().isEmpty()); + } + + @Test + void testNewStreamAddedToExistingCDCSync() throws Exception { + /* + * Insert the data into the second stream that will be added before the second sync. Do this before + * the first sync to ensure that the resume token stored in the state at the end of the first sync + * accounts for this data. If not, we will get duplicate records from the second sync: one batch + * from the initial snapshot for the second stream and one batch from Debezium via the change event + * stream. This is a known issue that happens when a stream with data inserted/changed AFTER the + * first stream is added to the connection and is currently handled by de-duping during + * normalization, so we will not test that case here. + */ + final int otherCollectionCount = 100; + insertData(databaseName, otherCollection1Name, otherCollectionCount); + + final ConfiguredAirbyteCatalog configuredCatalog = getConfiguredCatalog(); + + // Start a sync with one stream + final List messages = runRead(configuredCatalog); + final List recordMessages = filterRecords(messages); + final List stateMessages = filterStateMessages(messages); + + assertEquals(recordCount, recordMessages.size()); + assertEquals(recordCount + 1, stateMessages.size()); + + final AirbyteStateMessage lastStateMessage = Iterables.getLast(stateMessages); + validateStateMessages(stateMessages); + validateAllStreamsComplete(stateMessages, List.of( + new StreamDescriptor().withName(collectionName).withNamespace(databaseName))); + assertFalse(lastStateMessage.getGlobal().getStreamStates().stream().anyMatch( + createStateStreamFilter(new StreamDescriptor().withName(otherCollection1Name).withNamespace(databaseName)))); + + final List fields = List.of( + Field.of(NAME_FIELD, JsonSchemaType.STRING), + Field.of(INT_TEST_FIELD, JsonSchemaType.NUMBER)); + addStreamToConfiguredCatalog(configuredCatalog, databaseName, otherCollection1Name, fields); + + // Start another sync with a newly added stream + final List messages2 = runRead(configuredCatalog, Jsons.jsonNode(List.of(lastStateMessage))); + final List recordMessages2 = filterRecords(messages2); + final List stateMessages2 = filterStateMessages(messages2); + + assertEquals(otherCollectionCount, recordMessages2.size()); + assertEquals(otherCollectionCount + 1, stateMessages2.size()); + + validateStateMessages(stateMessages2); + validateAllStreamsComplete(stateMessages2, List.of( + new StreamDescriptor().withName(collectionName).withNamespace(databaseName), + new StreamDescriptor().withName(otherCollection1Name).withNamespace(databaseName))); + } + + @Test + void testNoChangeForCDCIncrementalSync() throws Exception { + final ConfiguredAirbyteCatalog configuredCatalog = getConfiguredCatalog(); + + // Start a sync with one stream + final List messages = runRead(configuredCatalog); + final List recordMessages = filterRecords(messages); + final List stateMessages = filterStateMessages(messages); + + assertEquals(recordCount, recordMessages.size()); + assertEquals(recordCount + 1, stateMessages.size()); + + final AirbyteStateMessage lastStateMessage = Iterables.getLast(stateMessages); + validateStateMessages(stateMessages); + validateAllStreamsComplete(stateMessages, List.of( + new StreamDescriptor().withName(collectionName).withNamespace(databaseName))); + + // Start another sync with no changes + final List messages2 = runRead(configuredCatalog, Jsons.jsonNode(List.of(lastStateMessage))); + final List recordMessages2 = filterRecords(messages2); + final List stateMessages2 = filterStateMessages(messages2); + + assertEquals(0, recordMessages2.size()); + assertEquals(1, stateMessages2.size()); + + validateStateMessages(stateMessages2); + validateAllStreamsComplete(stateMessages2, List.of( + new StreamDescriptor().withName(collectionName).withNamespace(databaseName))); + } + + @Test + void testInsertUpdateDeleteIncrementalSync() throws Exception { + final ConfiguredAirbyteCatalog configuredCatalog = getConfiguredCatalog(); + + // Start a sync with one stream + final List messages = runRead(configuredCatalog); + final List recordMessages = filterRecords(messages); + final List stateMessages = filterStateMessages(messages); + + assertEquals(recordCount, recordMessages.size()); + assertEquals(recordCount + 1, stateMessages.size()); + + validateCdcEventRecordData(recordMessages.get(0), new BsonObjectId(OBJECT_ID1), false); + validateCdcEventRecordData(recordMessages.get(1), new BsonObjectId(OBJECT_ID2), false); + validateCdcEventRecordData(recordMessages.get(2), new BsonObjectId(OBJECT_ID3), false); + + final AirbyteStateMessage lastStateMessage = Iterables.getLast(stateMessages); + validateStateMessages(stateMessages); + validateAllStreamsComplete(stateMessages, List.of( + new StreamDescriptor().withName(collectionName).withNamespace(databaseName))); + + final var result = mongoClient.getDatabase(databaseName).getCollection(collectionName).insertOne(createDocument(1)); + final var insertedId = result.getInsertedId(); + + // Start another sync that finds the insert change + final List messages2 = runRead(configuredCatalog, Jsons.jsonNode(List.of(lastStateMessage))); + final List recordMessages2 = filterRecords(messages2); + final List stateMessages2 = filterStateMessages(messages2); + + assertEquals(1, recordMessages2.size()); + assertEquals(1, stateMessages2.size()); + + validateCdcEventRecordData(recordMessages2.get(0), insertedId, false); + + final AirbyteStateMessage lastStateMessage2 = Iterables.getLast(stateMessages2); + validateStateMessages(stateMessages2); + validateAllStreamsComplete(stateMessages2, List.of( + new StreamDescriptor().withName(collectionName).withNamespace(databaseName))); + + final var idFilter = new Document(DOCUMENT_ID_FIELD, insertedId); + mongoClient.getDatabase(databaseName).getCollection(collectionName).updateOne(idFilter, Updates.combine(Updates.set("newField", "new"))); + + // Start another sync that finds the update change + final List messages3 = runRead(configuredCatalog, Jsons.jsonNode(List.of(lastStateMessage2))); + final List recordMessages3 = filterRecords(messages3); + final List stateMessages3 = filterStateMessages(messages3); + + assertEquals(1, recordMessages3.size()); + assertEquals(1, stateMessages3.size()); + + validateCdcEventRecordData(recordMessages3.get(0), insertedId, false); + + final AirbyteStateMessage lastStateMessage3 = Iterables.getLast(stateMessages3); + validateStateMessages(stateMessages3); + validateAllStreamsComplete(stateMessages3, List.of( + new StreamDescriptor().withName(collectionName).withNamespace(databaseName))); + + mongoClient.getDatabase(databaseName).getCollection(collectionName).deleteOne(idFilter); + + // Start another sync that finds the delete change + final List messages4 = runRead(configuredCatalog, Jsons.jsonNode(List.of(lastStateMessage3))); + final List recordMessages4 = filterRecords(messages4); + final List stateMessages4 = filterStateMessages(messages4); + + assertEquals(1, recordMessages4.size()); + assertEquals(1, stateMessages4.size()); + + validateCdcEventRecordData(recordMessages4.get(0), insertedId, true); + + validateStateMessages(stateMessages4); + validateAllStreamsComplete(stateMessages3, List.of( + new StreamDescriptor().withName(collectionName).withNamespace(databaseName))); + + } + + @Test + void testCDCStreamCheckpointingWithMultipleStreams() throws Exception { + final int otherCollection1Count = 100; + insertData(databaseName, otherCollection1Name, otherCollection1Count); + final int otherCollection2Count = 1; + insertData(databaseName, otherCollection2Name, otherCollection2Count); + + final ConfiguredAirbyteCatalog configuredCatalog = getConfiguredCatalog(); + final List streams = configuredCatalog.getStreams(); + final AirbyteStream otherAirbyteStream1 = MongoCatalogHelper.buildAirbyteStream(otherCollection1Name, databaseName, + List.of(Field.of(NAME_FIELD, JsonSchemaType.STRING), Field.of(INT_TEST_FIELD, JsonSchemaType.NUMBER))); + final AirbyteStream otherAirbyteStream2 = MongoCatalogHelper.buildAirbyteStream(otherCollection2Name, databaseName, + List.of(Field.of(NAME_FIELD, JsonSchemaType.STRING), Field.of(INT_TEST_FIELD, JsonSchemaType.NUMBER))); + streams.add(convertToConfiguredAirbyteStream(otherAirbyteStream1, SyncMode.INCREMENTAL)); + streams.add(convertToConfiguredAirbyteStream(otherAirbyteStream2, SyncMode.INCREMENTAL)); + configuredCatalog.withStreams(streams); + + // Start a sync with three streams + final List messages = runRead(configuredCatalog); + final List recordMessages = filterRecords(messages); + final List stateMessages = filterStateMessages(messages); + + assertEquals(recordCount + otherCollection1Count + otherCollection2Count, recordMessages.size()); + assertEquals(recordCount + otherCollection1Count + otherCollection2Count + 1, stateMessages.size()); + + validateStateMessages(stateMessages); + validateAllStreamsComplete(stateMessages, List.of( + new StreamDescriptor().withName(collectionName).withNamespace(databaseName), + new StreamDescriptor().withName(otherCollection1Name).withNamespace(databaseName), + new StreamDescriptor().withName(otherCollection2Name).withNamespace(databaseName))); + + // Start a second sync from somewhere in the middle of stream 2 + final List messages2 = runRead(configuredCatalog, Jsons.jsonNode(List.of(stateMessages.get(recordCount + 50)))); + final List recordMessages2 = filterRecords(messages2); + final List stateMessages2 = filterStateMessages(messages2); + + assertEquals(50, recordMessages2.size()); + assertEquals(51, stateMessages2.size()); + + // get state message where stream 1 has completed, stream 2 is in progress, and stream 3 has not + // started + final AirbyteStateMessage airbyteStateMessage = stateMessages2.get(0); + + final Optional collectionStreamState = getStreamState(airbyteStateMessage, + new StreamDescriptor().withName(collectionName).withNamespace(databaseName)); + assertEquals(InitialSnapshotStatus.COMPLETE, Jsons.object(collectionStreamState.get().getStreamState(), MongoDbStreamState.class).status()); + + final Optional otherCollection1StreamState = getStreamState(airbyteStateMessage, + new StreamDescriptor().withName(otherCollection1Name).withNamespace(databaseName)); + assertTrue(otherCollection1StreamState.isPresent()); + assertEquals(InitialSnapshotStatus.IN_PROGRESS, + Jsons.object(otherCollection1StreamState.get().getStreamState(), MongoDbStreamState.class).status()); + + final Optional otherCollection2StreamState = getStreamState(airbyteStateMessage, + new StreamDescriptor().withName(otherCollection2Name).withNamespace(databaseName)); + assertTrue(otherCollection2StreamState.isEmpty()); + + validateStateMessages(stateMessages2); + validateAllStreamsComplete(stateMessages, List.of( + new StreamDescriptor().withName(collectionName).withNamespace(databaseName), + new StreamDescriptor().withName(otherCollection1Name).withNamespace(databaseName), + new StreamDescriptor().withName(otherCollection2Name).withNamespace(databaseName))); + + // Insert more data for one stream + insertData(databaseName, otherCollection1Name, otherCollection1Count); + + // Start a third sync to test that only the new records are synced via incremental CDC + final List messages3 = runRead(configuredCatalog, Jsons.jsonNode(List.of(Iterables.getLast(stateMessages2)))); + final List recordMessages3 = filterRecords(messages3); + final List stateMessages3 = filterStateMessages(messages3); + + assertEquals(otherCollection1Count, recordMessages3.size()); + assertEquals(0, recordMessages3.stream().map(r -> new StreamDescriptor().withName(r.getStream()).withNamespace(r.getNamespace())) + .filter(createRecordStreamFilter(collectionName, databaseName)).count()); + assertEquals(0, recordMessages3.stream().map(r -> new StreamDescriptor().withName(r.getStream()).withNamespace(r.getNamespace())) + .filter(createRecordStreamFilter(otherCollection2Name, databaseName)).count()); + assertEquals(otherCollection1Count, + recordMessages3.stream().map(r -> new StreamDescriptor().withName(r.getStream()).withNamespace(r.getNamespace())) + .filter(createRecordStreamFilter(otherCollection1Name, databaseName)).count()); + assertEquals(1, stateMessages3.size()); + validateStateMessages(stateMessages3); + validateAllStreamsComplete(stateMessages, List.of( + new StreamDescriptor().withName(collectionName).withNamespace(databaseName), + new StreamDescriptor().withName(otherCollection1Name).withNamespace(databaseName), + new StreamDescriptor().withName(otherCollection2Name).withNamespace(databaseName))); + } + + @Test + void testSyncShouldHandlePurgedLogsGracefully() throws Exception { + final ConfiguredAirbyteCatalog configuredCatalog = getConfiguredCatalog(); + + // Run the sync to establish the next resume token value + final List messages = runRead(configuredCatalog); + final List recordMessages = filterRecords(messages); + final List stateMessages = filterStateMessages(messages); + + assertEquals(recordCount, recordMessages.size()); + assertEquals(recordCount + 1, stateMessages.size()); + + // Modify the state to point to a non-existing resume token value + final AirbyteStateMessage stateMessage = Iterables.getLast(stateMessages); + final String replicaSetName = MongoDbDebeziumStateUtil.getReplicaSetName(mongoClient); + final MongoDbCdcState cdcState = new MongoDbCdcState( + MongoDbDebeziumStateUtil.formatState(databaseName, replicaSetName, INVALID_RESUME_TOKEN)); + stateMessage.getGlobal().setSharedState(Jsons.jsonNode(cdcState)); + final JsonNode state = Jsons.jsonNode(List.of(stateMessage)); + + // Re-run the sync to prove that an initial snapshot is initiated due to invalid resume token + final List messages2 = runRead(configuredCatalog, state); + + final List recordMessages2 = filterRecords(messages2); + final List stateMessages2 = filterStateMessages(messages2); + + assertEquals(recordCount, recordMessages2.size()); + assertEquals(recordCount + 1, stateMessages2.size()); + } + + @Test + void testReachedTargetPosition() { + final long eventTimestamp = Long.MAX_VALUE; + final Integer order = 0; + final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); + final ChangeEventWithMetadata changeEventWithMetadata = mock(ChangeEventWithMetadata.class); + + when(changeEventWithMetadata.isSnapshotEvent()).thenReturn(true); + + assertFalse(targetPosition.reachedTargetPosition(changeEventWithMetadata)); + + when(changeEventWithMetadata.isSnapshotEvent()).thenReturn(false); + when(changeEventWithMetadata.snapshotMetadata()).thenReturn(SnapshotMetadata.LAST); + + assertTrue(targetPosition.reachedTargetPosition(changeEventWithMetadata)); + + when(changeEventWithMetadata.snapshotMetadata()).thenReturn(SnapshotMetadata.FIRST); + when(changeEventWithMetadata.eventValueAsJson()).thenReturn(Jsons.jsonNode( + Map.of(MongoDbDebeziumConstants.ChangeEvent.SOURCE, + Map.of(MongoDbDebeziumConstants.ChangeEvent.SOURCE_TIMESTAMP_MS, eventTimestamp, + MongoDbDebeziumConstants.ChangeEvent.SOURCE_ORDER, order)))); + + assertTrue(targetPosition.reachedTargetPosition(changeEventWithMetadata)); + + assertTrue(targetPosition.reachedTargetPosition(new BsonTimestamp(eventTimestamp))); + assertFalse(targetPosition.reachedTargetPosition(new BsonTimestamp(0L))); + assertFalse(targetPosition.reachedTargetPosition((BsonTimestamp) null)); + } + + @Test + void testIsSameOffset() { + final MongoDbCdcTargetPosition targetPosition = new MongoDbCdcTargetPosition(MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient)); + final BsonDocument resumeToken = MongoDbResumeTokenHelper.getMostRecentResumeToken(mongoClient); + final String resumeTokenString = resumeToken.get("_data").asString().getValue(); + final String replicaSet = MongoDbDebeziumStateUtil.getReplicaSetName(mongoClient); + final Map emptyOffsetA = Map.of(); + final Map emptyOffsetB = Map.of(); + final Map offsetA = Jsons.object(MongoDbDebeziumStateUtil.formatState(databaseName, + replicaSet, resumeTokenString), new TypeReference<>() {}); + final Map offsetB = Jsons.object(MongoDbDebeziumStateUtil.formatState(databaseName, + replicaSet, resumeTokenString), new TypeReference<>() {}); + final Map offsetBDifferent = Jsons.object(MongoDbDebeziumStateUtil.formatState(databaseName, + replicaSet, INVALID_RESUME_TOKEN), new TypeReference<>() {}); + + assertFalse(targetPosition.isSameOffset(null, offsetB)); + assertFalse(targetPosition.isSameOffset(emptyOffsetA, offsetB)); + assertFalse(targetPosition.isSameOffset(offsetA, null)); + assertFalse(targetPosition.isSameOffset(offsetA, emptyOffsetB)); + assertFalse(targetPosition.isSameOffset(offsetA, offsetBDifferent)); + assertTrue(targetPosition.isSameOffset(offsetA, offsetB)); + } + + private ConfiguredAirbyteStream convertToConfiguredAirbyteStream(final AirbyteStream airbyteStream, final SyncMode syncMode) { + return new ConfiguredAirbyteStream() + .withSyncMode(syncMode) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withCursorField(List.of(MongoCatalogHelper.DEFAULT_CURSOR_FIELD)) + .withStream(airbyteStream); + } + + private void verifyFieldNotExist(final List records, final String stream, final String field) { + assertTrue(records.stream() + .filter(r -> r.getStream().equals(stream) && r.getData().get(field) != null) + .collect(Collectors.toList()) + .isEmpty(), "Records contain unselected columns [%s:%s]".formatted(stream, field)); + } + + private List filterStateMessages(final List messages) { + return messages.stream().filter(r -> r.getType() == AirbyteMessage.Type.STATE).map(AirbyteMessage::getState) + .collect(Collectors.toList()); + } + + private void insertData(final String databaseName, final String collectionName, final int numberOfDocuments) { + mongoClient.getDatabase(databaseName).getCollection(collectionName).drop(); + mongoClient.getDatabase(databaseName).createCollection(collectionName); + final MongoCollection collection = mongoClient.getDatabase(databaseName).getCollection(collectionName); + collection + .insertMany(IntStream.range(0, numberOfDocuments).boxed().map(this::createDocument).toList()); + } + + private Document createDocument(final Integer i) { + return new Document(NAME_FIELD, "value" + i).append(INT_TEST_FIELD, i); + } + + private void validateStateMessages(final List stateMessages) { + stateMessages.forEach(stateMessage -> { + assertEquals(AirbyteStateMessage.AirbyteStateType.GLOBAL, stateMessage.getType()); + final AirbyteGlobalState global = stateMessage.getGlobal(); + assertNotNull(global.getSharedState()); + assertFalse(global.getSharedState().isEmpty()); + }); + } + + private void validateAllStreamsComplete(final List stateMessages, final List completedStreams) { + final AirbyteStateMessage lastStateMessage = Iterables.getLast(stateMessages); + assertNotNull(lastStateMessage.getGlobal().getSharedState()); + assertFalse(lastStateMessage.getGlobal().getSharedState().isEmpty()); + completedStreams.forEach(s -> { + assertTrue(lastStateMessage.getGlobal().getStreamStates().stream().anyMatch(createStateStreamFilter(s))); + Assertions.assertEquals(InitialSnapshotStatus.COMPLETE, + Jsons.object(getStreamState(lastStateMessage, s).get().getStreamState(), MongoDbStreamState.class).status()); + }); + } + + private Optional getStreamState(final AirbyteStateMessage stateMessage, final StreamDescriptor streamDescriptor) { + return stateMessage.getGlobal().getStreamStates().stream().filter(createStateStreamFilter(streamDescriptor)).findFirst(); + } + + private Predicate createStateStreamFilter(final StreamDescriptor streamDescriptor) { + return s -> s.getStreamDescriptor().equals(streamDescriptor); + } + + private Predicate createRecordStreamFilter(final String name, final String namespace) { + return s -> s.equals(new StreamDescriptor().withName(name).withNamespace(namespace)); + } + + private void addStreamToConfiguredCatalog(final ConfiguredAirbyteCatalog configuredAirbyteCatalog, + final String databaseName, + final String collectionName, + final List fields) { + final List streams = configuredAirbyteCatalog.getStreams(); + final AirbyteStream otherAirbyteStream = MongoCatalogHelper.buildAirbyteStream(collectionName, databaseName, fields); + streams.add(convertToConfiguredAirbyteStream(otherAirbyteStream, SyncMode.INCREMENTAL)); + configuredAirbyteCatalog.withStreams(streams); + } + + private void validateCdcEventRecordData(final AirbyteRecordMessage airbyteRecordMessage, final BsonValue expectedObjectId, final boolean isDelete) { + final Map data = Jsons.object(airbyteRecordMessage.getData(), new TypeReference<>() {}); + assertEquals(expectedObjectId.asObjectId().getValue().toString(), data.get(DOCUMENT_ID_FIELD)); + assertTrue(data.containsKey(CDC_DELETED_AT)); + assertTrue(data.containsKey(CDC_UPDATED_AT)); + if (isDelete) { + assertNotNull(data.get(CDC_DELETED_AT)); + } else { + assertNull(data.get(CDC_DELETED_AT)); + } + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/debezium/DebeziumMongoDbConnectorTest.kt b/airbyte-integrations/connectors/source-mongodb-v2/src/test/debezium/DebeziumMongoDbConnectorTest.kt new file mode 100644 index 000000000000..f896e5d09d48 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/debezium/DebeziumMongoDbConnectorTest.kt @@ -0,0 +1,252 @@ +package io.airbyte.integrations.source.mongodb + +import com.mongodb.ConnectionString +import com.mongodb.MongoClientSettings +import com.mongodb.ReadPreference +import com.mongodb.client.MongoClients +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings +import io.debezium.connector.mongodb.MongoDbConnector +import io.debezium.connector.mongodb.ResumeTokens +import io.debezium.engine.DebeziumEngine +import org.bson.BsonDocument +import org.bson.BsonTimestamp +import org.slf4j.LoggerFactory +import java.io.IOException +import java.io.ObjectInputStream +import java.nio.ByteBuffer +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import java.util.Properties +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicReference +import java.util.function.Consumer +import java.util.stream.Collectors +import kotlin.collections.Map +import kotlin.collections.component1 +import kotlin.collections.component2 +import kotlinx.cli.ArgParser +import kotlinx.cli.ArgType +import kotlinx.cli.required + + +class DebeziumMongoDbConnectorTest internal constructor(private val connectionString: String, private val databaseName: String, private val collectionName: String, private val username: String, private val password: String) { + + @Throws(InterruptedException::class, IOException::class) + fun startTest() { + val queue: LinkedBlockingQueue> = LinkedBlockingQueue>(10000) + val path = path + LOGGER.info("Using offset storage path '{}'.", path) + testChangeEventStream() + + // will do an initial sync cause offset is null + initialSync(queue, path) + + // will do an incremental processing cause after the initialSync run the offset must be updated + engineWithIncrementalSnapshot(queue, path) + } + + private fun testChangeEventStream() { + val mongoClientSettings = MongoClientSettings.builder() + .applyConnectionString(ConnectionString("mongodb+srv://$username:$password@cluster0.iqgf8.mongodb.net/")) + .readPreference(ReadPreference.secondaryPreferred()) + .build() + MongoClients.create(mongoClientSettings).use { client -> + LOGGER.info("Retrieving change stream...") + val stream = client.watch(BsonDocument::class.java) + LOGGER.info("Retrieving cursor...") + val changeStreamCursor = stream.cursor() + + /* + * Must call tryNext before attempting to get the resume token from the cursor directly. Otherwise, + * both will return null! + */ + val cursorDocument = changeStreamCursor.tryNext() + if (cursorDocument != null) { + LOGGER.info("Resume token from cursor document: {}", cursorDocument.resumeToken) + } else { + LOGGER.info("Cursor document is null.") + } + LOGGER.info("Resume token = {}", changeStreamCursor.resumeToken) + val timestamp: BsonTimestamp = ResumeTokens.getTimestamp(changeStreamCursor.resumeToken) + LOGGER.info("sec {}, ord {}", timestamp.time, timestamp.inc) + } + } + + @Throws(InterruptedException::class, IOException::class) + private fun initialSync(queue: LinkedBlockingQueue>, path: Path) { + val executorService = Executors.newSingleThreadExecutor() + val thrownError = AtomicReference() + val engineLatch = CountDownLatch(1) + val engine: DebeziumEngine> = DebeziumEngine.create(io.debezium.engine.format.Json::class.java) + .using(getDebeziumProperties(path, listOf("$databaseName\\.$collectionName").stream().collect(Collectors.joining(",")))) + .using(io.debezium.engine.spi.OffsetCommitPolicy.AlwaysCommitOffsetPolicy()) + .notifying(Consumer> { e: io.debezium.engine.ChangeEvent -> + // debezium outputs a tombstone event that has a value of null. this is an artifact of how it + // interacts with kafka. we want to ignore it. + // more on the tombstone: + // https://debezium.io/documentation/reference/configuration/event-flattening.html + if (e.value() != null) { + LOGGER.debug("{}", e) + var inserted = false + while (!inserted) { + inserted = queue.offer(e) + } + } + }) + .using { _: Boolean, message: String?, error: Throwable? -> + LOGGER.info("Initial sync Debezium engine shutdown.") + if (error != null) { + LOGGER.error("error occurred: {}", message, error) + } + engineLatch.countDown() + thrownError.set(error) + } + .build() + executorService.execute(engine) + Thread.sleep((45 * 1000).toLong()) + engine.close() + engineLatch.await(5, TimeUnit.MINUTES) + executorService.shutdown() + executorService.awaitTermination(5, TimeUnit.MINUTES) + readOffsetFile(path) + if (thrownError.get() != null) { + throw RuntimeException(thrownError.get()) + } + } + + @Throws(InterruptedException::class, IOException::class) + private fun engineWithIncrementalSnapshot(queue: LinkedBlockingQueue>, path: Path) { + val executorService2 = Executors.newSingleThreadExecutor() + val thrownError2 = AtomicReference() + val engineLatch2 = CountDownLatch(1) + val engine2: DebeziumEngine> = DebeziumEngine.create(io.debezium.engine.format.Json::class.java) + .using(getDebeziumProperties(path, listOf("$databaseName\\.$collectionName").stream().collect(Collectors.joining(",")))) + .using(io.debezium.engine.spi.OffsetCommitPolicy.AlwaysCommitOffsetPolicy()) + .notifying { e: io.debezium.engine.ChangeEvent -> + // debezium outputs a tombstone event that has a value of null. this is an artifact of how it + // interacts with kafka. we want to ignore it. + // more on the tombstone: + // https://debezium.io/documentation/reference/configuration/event-flattening.html + if (e.value() != null) { + LOGGER.info("{}", e) + var inserted = false + while (!inserted) { + inserted = queue.offer(e) + } + } + } + .using(io.debezium.engine.DebeziumEngine.CompletionCallback { success: Boolean, message: String?, error: Throwable? -> + LOGGER.info("Incremental snapshot Debezium engine shutdown.") + if (error != null) { + LOGGER.error("error occurred: {}", message, error) + } + engineLatch2.countDown() + thrownError2.set(error) + }) + .build() + executorService2.execute(engine2) + Thread.sleep((180 * 1000).toLong()) + engine2.close() + engineLatch2.await(5, TimeUnit.MINUTES) + executorService2.shutdown() + executorService2.awaitTermination(5, TimeUnit.MINUTES) + readOffsetFile(path) + if (thrownError2.get() != null) { + throw RuntimeException(thrownError2.get()) + } + } + + protected fun getDebeziumProperties(cdcOffsetFilePath: Path, collectionNames: String): Properties { + val props = Properties() + LOGGER.info("Included collection names regular expression: '{}'.", collectionNames) + props.setProperty("connector.class", MongoDbConnector::class.java.getName()) + props.setProperty("snapshot.mode", "initial") + props.setProperty("name", databaseName.replace("_".toRegex(), "-")) + props.setProperty("mongodb.connection.string", connectionString) + props.setProperty("mongodb.connection.mode", "replica_set") + props.setProperty("mongodb.user", username) + props.setProperty("mongodb.password", password) + props.setProperty("mongodb.authsource", "admin") + props.setProperty("mongodb.ssl.enabled", "true") + props.setProperty("topic.prefix", databaseName) + props.setProperty("capture.mode", "change_streams_update_full") + + // Database/collection selection + props.setProperty("collection.include.list", collectionNames) + props.setProperty("database.include.list", databaseName) + + // Offset storage configuration + props.setProperty("offset.storage", "org.apache.kafka.connect.storage.FileOffsetBackingStore") + props.setProperty("offset.storage.file.filename", cdcOffsetFilePath.toString()) + props.setProperty("offset.flush.interval.ms", "1000") + + // Advanced properties + props.setProperty("max.batch.size", "2048") + props.setProperty("max.queue.size", "8192") + + // https://debezium.io/documentation/reference/configuration/avro.html + props.setProperty("key.converter.schemas.enable", "false") + props.setProperty("value.converter.schemas.enable", "false") + + // By default "decimal.handing.mode=precise" which caused returning this value as a binary. + // The "double" type may cause a loss of precision, so set Debezium's config to store it as a String + // explicitly in its Kafka messages for more details see: + // https://debezium.io/documentation/reference/1.4/connectors/postgresql.html#postgresql-decimal-types + // https://debezium.io/documentation/faq/#how_to_retrieve_decimal_field_from_binary_representation + props.setProperty("decimal.handling.mode", "string") + props.setProperty("errors.log.include.messages", "true") + props.setProperty("errors.log.enable", "true") + props.setProperty("heartbeat.interval.ms", "500") + return props + } + + private val path: Path + get() { + val cdcWorkingDir: Path = try { + Files.createTempDirectory(Path.of("/tmp"), "cdc-state-offset") + } catch (e: IOException) { + throw RuntimeException(e) + } + return cdcWorkingDir.resolve("offset.txt") + } + + @SuppressFBWarnings("OBJECT_DESERIALIZATION") + private fun readOffsetFile(path: Path) { + LOGGER.info("Reading contents of offset file '{}'...", path) + try { + ObjectInputStream(Files.newInputStream(path)).use { ois -> + val raw = ois.readObject() as Map + raw.entries.forEach(Consumer { (key, value): Map.Entry -> LOGGER.info("{}:{}", String(ByteBuffer.wrap(key).array(), StandardCharsets.UTF_8), String(ByteBuffer.wrap(value).array(), StandardCharsets.UTF_8)) }) + } + } catch (e: IOException) { + LOGGER.error("Unable to read offset file '{}'.", path, e) + } catch (e: ClassNotFoundException) { + LOGGER.error("Unable to read offset file '{}'.", path, e) + } + } + + companion object { + private val LOGGER = LoggerFactory.getLogger(DebeziumMongoDbConnectorTest::class.java) + @Throws(IOException::class, InterruptedException::class) + @JvmStatic + fun main(args: Array) { + val parser = ArgParser("Debezium MongoDb Connector Test Harness") + val connectionString by parser.option(ArgType.String, fullName = "connection-string", shortName = "cs", description = "MongoDB Connection String").required() + val databaseName by parser.option(ArgType.String, fullName = "database-name", shortName = "d", description = "Database Name").required() + val collectionName by parser.option(ArgType.String, fullName = "collection-name", shortName = "cn", description = "Collection Name").required() + val username by parser.option(ArgType.String, fullName = "username", shortName = "u", description = "Username").required() + + parser.parse(args) + + println("Enter password: ") + val password = readln() + + val debeziumEngineTest = DebeziumMongoDbConnectorTest(connectionString, databaseName, collectionName, username, password) + debeziumEngineTest.startTest() + } + } +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/generator/MongoDbInsertClient.kt b/airbyte-integrations/connectors/source-mongodb-v2/src/test/generator/MongoDbInsertClient.kt new file mode 100644 index 000000000000..d80a179a5947 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/generator/MongoDbInsertClient.kt @@ -0,0 +1,71 @@ +package io.airbyte.integrations.source.mongodb + +import com.github.javafaker.Faker +import io.airbyte.commons.json.Jsons +import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.cli.ArgParser +import kotlinx.cli.ArgType +import kotlinx.cli.default +import kotlinx.cli.required +import org.bson.BsonTimestamp +import org.bson.Document +import java.lang.System.currentTimeMillis + +object MongoDbInsertClient { + + private const val BATCH_SIZE = 1000 + + private val logger = KotlinLogging.logger {} + + @JvmStatic + fun main(args: Array) { + val parser = ArgParser("MongoDb Insert Client") + val connectionString by parser.option(ArgType.String, fullName = "connection-string", shortName = "cs", description = "MongoDb Connection String").required() + val databaseName by parser.option(ArgType.String, fullName = "database-name", shortName = "d", description = "Database Name").required() + val collectionName by parser.option(ArgType.String, fullName = "collection-name", shortName = "cn", description = "Collection Name").required() + val username by parser.option(ArgType.String, fullName = "username", shortName = "u", description = "Username").required() + val numberOfDocuments by parser.option(ArgType.Int, fullName = "number", shortName = "n", description = "Number of documents to generate").default(10000) + + parser.parse(args) + + println("Enter password: ") + val password = readln() + + var config = mapOf(MongoConstants.DATABASE_CONFIG_CONFIGURATION_KEY to + mapOf( + MongoConstants.DATABASE_CONFIGURATION_KEY to databaseName, + MongoConstants.CONNECTION_STRING_CONFIGURATION_KEY to connectionString, + MongoConstants.AUTH_SOURCE_CONFIGURATION_KEY to "admin", + MongoConstants.USERNAME_CONFIGURATION_KEY to username, + MongoConstants.PASSWORD_CONFIGURATION_KEY to password) + ) + + val faker = Faker(); + + MongoConnectionUtils.createMongoClient(MongoDbSourceConfig(Jsons.deserialize(Jsons.serialize(config)))).use { mongoClient -> + val documents = mutableListOf() + val batches = if (numberOfDocuments > BATCH_SIZE) numberOfDocuments / BATCH_SIZE else 1; + val batchSize = if (numberOfDocuments > BATCH_SIZE) BATCH_SIZE else numberOfDocuments; + logger.info { "Inserting $batches batch(es) of $batchSize document(s) each..." } + for (i in 0..batches) { + logger.info { "Inserting batch ${i}..." } + for (j in 0..batchSize) { + val index = (j+1)+((i+1)*batchSize) + documents += Document().append("name", "Document $index") + .append("title", "${faker.lorem().sentence(10)}") + .append("description", "${faker.lorem().paragraph(25)}") + .append("data", "${faker.lorem().paragraphs(100)}") + .append("paragraph", "${faker.lorem().paragraph(25)}") + .append("doubleField", index.toDouble()) + .append("intField", index) + .append("objectField", mapOf("key" to "value")) + .append("timestamp", BsonTimestamp(currentTimeMillis())) + } + mongoClient.getDatabase(databaseName).getCollection(collectionName).insertMany(documents) + documents.clear() + } + } + + logger.info { "Inserted $numberOfDocuments document(s) to $databaseName.$collectionName" } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/InitialSnapshotHandlerTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/InitialSnapshotHandlerTest.java new file mode 100644 index 000000000000..919903c76575 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/InitialSnapshotHandlerTest.java @@ -0,0 +1,335 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableSet; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoCollection; +import io.airbyte.commons.exceptions.ConfigErrorException; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.integrations.source.mongodb.state.IdType; +import io.airbyte.integrations.source.mongodb.state.MongoDbStateManager; +import io.airbyte.integrations.source.mongodb.state.MongoDbStreamState; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.SyncMode; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import org.bson.Document; +import org.bson.types.ObjectId; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.MongoDBContainer; + +class InitialSnapshotHandlerTest { + + private static final String DB_NAME = "airbyte_test"; + + private static final String CURSOR_FIELD = "_id"; + private static final String NAME_FIELD = "name"; + + private static final String NAMESPACE = "database"; + + private static final String COLLECTION1 = "collection1"; + private static final String COLLECTION2 = "collection2"; + private static final String COLLECTION3 = "collection3"; + + private static final String OBJECT_ID1_STRING = "64c0029d95ad260d69ef28a1"; + private static final ObjectId OBJECT_ID1 = new ObjectId(OBJECT_ID1_STRING); + private static final ObjectId OBJECT_ID2 = new ObjectId("64c0029d95ad260d69ef28a2"); + private static final ObjectId OBJECT_ID3 = new ObjectId("64c0029d95ad260d69ef28a3"); + private static final ObjectId OBJECT_ID4 = new ObjectId("64c0029d95ad260d69ef28a4"); + private static final ObjectId OBJECT_ID5 = new ObjectId("64c0029d95ad260d69ef28a5"); + private static final ObjectId OBJECT_ID6 = new ObjectId("64c0029d95ad260d69ef28a6"); + + private static final String NAME1 = "name1"; + private static final String NAME2 = "name2"; + private static final String NAME3 = "name3"; + private static final String NAME4 = "name4"; + private static final String NAME5 = "name5"; + private static final String NAME6 = "name6"; + + private static final List STREAMS = List.of( + CatalogHelpers.createConfiguredAirbyteStream( + COLLECTION1, + NAMESPACE, + Field.of(CURSOR_FIELD, JsonSchemaType.STRING), + Field.of(NAME_FIELD, JsonSchemaType.STRING)) + .withSyncMode(SyncMode.INCREMENTAL), + CatalogHelpers.createConfiguredAirbyteStream( + COLLECTION2, + NAMESPACE, + Field.of(CURSOR_FIELD, JsonSchemaType.STRING)) + .withSyncMode(SyncMode.INCREMENTAL), + CatalogHelpers.createConfiguredAirbyteStream( + COLLECTION3, + NAMESPACE, + Field.of(CURSOR_FIELD, JsonSchemaType.STRING), + Field.of(NAME_FIELD, JsonSchemaType.STRING)) + .withSyncMode(SyncMode.FULL_REFRESH)); + + private static MongoDBContainer MONGO_DB; + private MongoClient mongoClient; + + @BeforeAll + static void init() { + MONGO_DB = new MongoDBContainer("mongo:6.0.8"); + MONGO_DB.start(); + } + + @BeforeEach + void setup() { + mongoClient = MongoClients.create(MONGO_DB.getConnectionString()); + mongoClient.getDatabase(DB_NAME).drop(); + } + + @AfterEach + void tearDown() { + mongoClient.close(); + } + + @AfterAll + static void cleanup() { + MONGO_DB.stop(); + } + + @Test + void testGetIteratorsEmptyInitialState() { + insertDocuments(COLLECTION1, List.of( + new Document(Map.of( + CURSOR_FIELD, OBJECT_ID1, + NAME_FIELD, NAME1)), + new Document(Map.of( + CURSOR_FIELD, OBJECT_ID2, + NAME_FIELD, NAME2)), + new Document(Map.of( + CURSOR_FIELD, OBJECT_ID3, + NAME_FIELD, NAME3)))); + + insertDocuments(COLLECTION2, List.of( + new Document(Map.of( + CURSOR_FIELD, OBJECT_ID4, + NAME_FIELD, NAME4)), + new Document(Map.of( + CURSOR_FIELD, OBJECT_ID5, + NAME_FIELD, NAME5)))); + + insertDocuments(COLLECTION3, List.of( + new Document(Map.of( + CURSOR_FIELD, OBJECT_ID6, + NAME_FIELD, NAME6)))); + + final InitialSnapshotHandler initialSnapshotHandler = new InitialSnapshotHandler(); + final MongoDbStateManager stateManager = mock(MongoDbStateManager.class); + final List> iterators = + initialSnapshotHandler.getIterators(STREAMS, stateManager, mongoClient.getDatabase(DB_NAME), null, Instant.now(), + MongoConstants.CHECKPOINT_INTERVAL, true); + + assertEquals(iterators.size(), 2, "Only two streams are configured as incremental, full refresh streams should be ignored"); + + final AutoCloseableIterator collection1 = iterators.get(0); + final AutoCloseableIterator collection2 = iterators.get(1); + + // collection1 + final AirbyteMessage collection1StreamMessage1 = collection1.next(); + assertEquals(Type.RECORD, collection1StreamMessage1.getType()); + assertEquals(COLLECTION1, collection1StreamMessage1.getRecord().getStream()); + assertEquals(OBJECT_ID1.toString(), collection1StreamMessage1.getRecord().getData().get(CURSOR_FIELD).asText()); + assertEquals(NAME1, collection1StreamMessage1.getRecord().getData().get(NAME_FIELD).asText()); + assertConfiguredFieldsEqualsRecordDataFields(Set.of(CURSOR_FIELD, NAME_FIELD), collection1StreamMessage1.getRecord().getData()); + + final AirbyteMessage collection1StreamMessage2 = collection1.next(); + assertEquals(Type.RECORD, collection1StreamMessage2.getType()); + assertEquals(COLLECTION1, collection1StreamMessage2.getRecord().getStream()); + assertEquals(OBJECT_ID2.toString(), collection1StreamMessage2.getRecord().getData().get(CURSOR_FIELD).asText()); + assertEquals(NAME2, collection1StreamMessage2.getRecord().getData().get(NAME_FIELD).asText()); + assertConfiguredFieldsEqualsRecordDataFields(Set.of(CURSOR_FIELD, NAME_FIELD), collection1StreamMessage2.getRecord().getData()); + + final AirbyteMessage collection1StreamMessage3 = collection1.next(); + assertEquals(Type.RECORD, collection1StreamMessage3.getType()); + assertEquals(COLLECTION1, collection1StreamMessage3.getRecord().getStream()); + assertEquals(OBJECT_ID3.toString(), collection1StreamMessage3.getRecord().getData().get(CURSOR_FIELD).asText()); + assertEquals(NAME3, collection1StreamMessage3.getRecord().getData().get(NAME_FIELD).asText()); + assertConfiguredFieldsEqualsRecordDataFields(Set.of(CURSOR_FIELD, NAME_FIELD), collection1StreamMessage3.getRecord().getData()); + + final AirbyteMessage collection1SateMessage = collection1.next(); + assertEquals(Type.STATE, collection1SateMessage.getType(), "State message is expected after all records in a stream are emitted"); + + assertFalse(collection1.hasNext()); + + // collection2 + final AirbyteMessage collection2StreamMessage1 = collection2.next(); + assertEquals(Type.RECORD, collection2StreamMessage1.getType()); + assertEquals(COLLECTION2, collection2StreamMessage1.getRecord().getStream()); + assertEquals(OBJECT_ID4.toString(), collection2StreamMessage1.getRecord().getData().get(CURSOR_FIELD).asText()); + assertConfiguredFieldsEqualsRecordDataFields(Set.of(CURSOR_FIELD), collection2StreamMessage1.getRecord().getData()); + + final AirbyteMessage collection2StreamMessage2 = collection2.next(); + assertEquals(Type.RECORD, collection2StreamMessage2.getType()); + assertEquals(COLLECTION2, collection2StreamMessage2.getRecord().getStream()); + assertEquals(OBJECT_ID5.toString(), collection2StreamMessage2.getRecord().getData().get(CURSOR_FIELD).asText()); + assertConfiguredFieldsEqualsRecordDataFields(Set.of(CURSOR_FIELD), collection2StreamMessage1.getRecord().getData()); + + final AirbyteMessage collection2SateMessage = collection2.next(); + assertEquals(Type.STATE, collection2SateMessage.getType(), "State message is expected after all records in a stream are emitted"); + + assertFalse(collection2.hasNext()); + } + + @Test + void testGetIteratorsNonEmptyInitialState() { + insertDocuments(COLLECTION1, List.of( + new Document(Map.of( + CURSOR_FIELD, OBJECT_ID1, + NAME_FIELD, NAME1)), + new Document(Map.of( + CURSOR_FIELD, OBJECT_ID2, + NAME_FIELD, NAME2)))); + + insertDocuments(COLLECTION2, List.of( + new Document(Map.of( + CURSOR_FIELD, OBJECT_ID3, + NAME_FIELD, NAME3)))); + + final InitialSnapshotHandler initialSnapshotHandler = new InitialSnapshotHandler(); + final MongoDbStateManager stateManager = mock(MongoDbStateManager.class); + when(stateManager.getStreamState(COLLECTION1, NAMESPACE)) + .thenReturn(Optional.of(new MongoDbStreamState(OBJECT_ID1_STRING, null, IdType.OBJECT_ID))); + final List> iterators = + initialSnapshotHandler.getIterators(STREAMS, stateManager, mongoClient.getDatabase(DB_NAME), null, Instant.now(), + MongoConstants.CHECKPOINT_INTERVAL, true); + + assertEquals(iterators.size(), 2, "Only two streams are configured as incremental, full refresh streams should be ignored"); + + final AutoCloseableIterator collection1 = iterators.get(0); + final AutoCloseableIterator collection2 = iterators.get(1); + + // collection1, first document should be skipped + final AirbyteMessage collection1StreamMessage1 = collection1.next(); + assertEquals(Type.RECORD, collection1StreamMessage1.getType()); + assertEquals(COLLECTION1, collection1StreamMessage1.getRecord().getStream()); + assertEquals(OBJECT_ID2.toString(), collection1StreamMessage1.getRecord().getData().get(CURSOR_FIELD).asText()); + assertEquals(NAME2, collection1StreamMessage1.getRecord().getData().get(NAME_FIELD).asText()); + assertConfiguredFieldsEqualsRecordDataFields(Set.of(CURSOR_FIELD, NAME_FIELD), collection1StreamMessage1.getRecord().getData()); + + final AirbyteMessage collection1SateMessage = collection1.next(); + assertEquals(Type.STATE, collection1SateMessage.getType(), "State message is expected after all records in a stream are emitted"); + + assertFalse(collection1.hasNext()); + + // collection2, no documents should be skipped + final AirbyteMessage collection2StreamMessage1 = collection2.next(); + assertEquals(Type.RECORD, collection2StreamMessage1.getType()); + assertEquals(COLLECTION2, collection2StreamMessage1.getRecord().getStream()); + assertEquals(OBJECT_ID3.toString(), collection2StreamMessage1.getRecord().getData().get(CURSOR_FIELD).asText()); + assertConfiguredFieldsEqualsRecordDataFields(Set.of(CURSOR_FIELD), collection2StreamMessage1.getRecord().getData()); + + final AirbyteMessage collection2SateMessage = collection2.next(); + assertEquals(Type.STATE, collection2SateMessage.getType(), "State message is expected after all records in a stream are emitted"); + + assertFalse(collection2.hasNext()); + } + + @Test + void testGetIteratorsThrowsExceptionWhenThereAreDifferentIdTypes() { + insertDocuments(COLLECTION1, List.of( + new Document(Map.of( + CURSOR_FIELD, OBJECT_ID1, + NAME_FIELD, NAME1)), + new Document(Map.of( + CURSOR_FIELD, "string-id", + NAME_FIELD, NAME2)))); + + final InitialSnapshotHandler initialSnapshotHandler = new InitialSnapshotHandler(); + final MongoDbStateManager stateManager = mock(MongoDbStateManager.class); + + final var thrown = assertThrows(ConfigErrorException.class, + () -> initialSnapshotHandler.getIterators(STREAMS, stateManager, mongoClient.getDatabase(DB_NAME), null, Instant.now(), + MongoConstants.CHECKPOINT_INTERVAL, true)); + assertTrue(thrown.getMessage().contains("must be consistently typed")); + } + + @Test + void testGetIteratorsThrowsExceptionWhenThereAreUnsupportedIdTypes() { + insertDocuments(COLLECTION1, List.of( + new Document(Map.of( + CURSOR_FIELD, 0.1, + NAME_FIELD, NAME1)))); + + final InitialSnapshotHandler initialSnapshotHandler = new InitialSnapshotHandler(); + final MongoDbStateManager stateManager = mock(MongoDbStateManager.class); + + final var thrown = assertThrows(ConfigErrorException.class, + () -> initialSnapshotHandler.getIterators(STREAMS, stateManager, mongoClient.getDatabase(DB_NAME), null, Instant.now(), + MongoConstants.CHECKPOINT_INTERVAL, true)); + assertTrue(thrown.getMessage().contains("_id fields with the following types are currently supported")); + } + + private void assertConfiguredFieldsEqualsRecordDataFields(final Set configuredStreamFields, final JsonNode recordMessageData) { + final Set recordDataFields = ImmutableSet.copyOf(recordMessageData.fieldNames()); + assertEquals(configuredStreamFields, recordDataFields, + "Fields in record message should be the same as fields in their corresponding stream configuration"); + } + + private void insertDocuments(final String collectionName, final List documents) { + final MongoCollection collection = mongoClient.getDatabase(DB_NAME).getCollection(collectionName); + collection.insertMany(documents); + } + + @Test + void testGetIteratorsWithOneEmptyCollection() { + insertDocuments(COLLECTION1, List.of( + new Document(Map.of( + CURSOR_FIELD, OBJECT_ID1, + NAME_FIELD, NAME1)))); + + final InitialSnapshotHandler initialSnapshotHandler = new InitialSnapshotHandler(); + final MongoDbStateManager stateManager = mock(MongoDbStateManager.class); + final List> iterators = + initialSnapshotHandler.getIterators(STREAMS, stateManager, mongoClient.getDatabase(DB_NAME), null, Instant.now(), + MongoConstants.CHECKPOINT_INTERVAL, true); + + assertEquals(iterators.size(), 2, "Only two streams are configured as incremental, full refresh streams should be ignored"); + + final AutoCloseableIterator collection1 = iterators.get(0); + final AutoCloseableIterator collection2 = iterators.get(1); + + // collection1 + final AirbyteMessage collection1StreamMessage1 = collection1.next(); + assertEquals(Type.RECORD, collection1StreamMessage1.getType()); + assertEquals(COLLECTION1, collection1StreamMessage1.getRecord().getStream()); + assertEquals(OBJECT_ID1.toString(), collection1StreamMessage1.getRecord().getData().get(CURSOR_FIELD).asText()); + assertEquals(NAME1, collection1StreamMessage1.getRecord().getData().get(NAME_FIELD).asText()); + assertConfiguredFieldsEqualsRecordDataFields(Set.of(CURSOR_FIELD, NAME_FIELD), collection1StreamMessage1.getRecord().getData()); + + final AirbyteMessage collection1SateMessage = collection1.next(); + assertEquals(Type.STATE, collection1SateMessage.getType(), "State message is expected after all records in a stream are emitted"); + + assertFalse(collection1.hasNext()); + + // collection2 + assertFalse(collection2.hasNext()); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoCatalogHelperTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoCatalogHelperTest.java new file mode 100644 index 000000000000..f27d382277a6 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoCatalogHelperTest.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb; + +import static io.airbyte.integrations.source.mongodb.MongoCatalogHelper.DEFAULT_CURSOR_FIELD; +import static io.airbyte.integrations.source.mongodb.MongoCatalogHelper.DEFAULT_PRIMARY_KEY; +import static io.airbyte.integrations.source.mongodb.MongoCatalogHelper.SUPPORTED_SYNC_MODES; +import static io.airbyte.integrations.source.mongodb.MongoConstants.ID_FIELD; +import static io.airbyte.integrations.source.mongodb.MongoConstants.SCHEMALESS_MODE_DATA_FIELD; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteStream; +import java.util.List; +import org.junit.jupiter.api.Test; + +class MongoCatalogHelperTest { + + @Test + void testBuildingAirbyteStream() { + final String streamName = "name"; + final String streamNamespace = "namespace"; + final List discoveredFields = List.of(new Field("field1", JsonSchemaType.STRING), + new Field("field2", JsonSchemaType.NUMBER)); + + final AirbyteStream airbyteStream = MongoCatalogHelper.buildAirbyteStream(streamName, streamNamespace, discoveredFields); + + assertNotNull(airbyteStream); + assertEquals(streamNamespace, airbyteStream.getNamespace()); + assertEquals(streamName, airbyteStream.getName()); + assertEquals(List.of(DEFAULT_CURSOR_FIELD), airbyteStream.getDefaultCursorField()); + assertEquals(true, airbyteStream.getSourceDefinedCursor()); + assertEquals(List.of(List.of(DEFAULT_PRIMARY_KEY)), airbyteStream.getSourceDefinedPrimaryKey()); + assertEquals(SUPPORTED_SYNC_MODES, airbyteStream.getSupportedSyncModes()); + assertEquals(5, airbyteStream.getJsonSchema().get("properties").size()); + + discoveredFields.forEach(f -> assertTrue(airbyteStream.getJsonSchema().get("properties").has(f.getName()))); + assertTrue(airbyteStream.getJsonSchema().get("properties").has(DEFAULT_CURSOR_FIELD)); + assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get("type"), + airbyteStream.getJsonSchema().get("properties").get(DEFAULT_CURSOR_FIELD).get("type").asText()); + assertTrue(airbyteStream.getJsonSchema().get("properties").has(DebeziumEventUtils.CDC_DELETED_AT)); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), + airbyteStream.getJsonSchema().get("properties").get(DebeziumEventUtils.CDC_DELETED_AT).get("type").asText()); + assertTrue(airbyteStream.getJsonSchema().get("properties").has(DebeziumEventUtils.CDC_UPDATED_AT)); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), + airbyteStream.getJsonSchema().get("properties").get(DebeziumEventUtils.CDC_UPDATED_AT).get("type").asText()); + + } + + @Test + void testSchemalessModeAirbyteStream() { + final String streamName = "name"; + final String streamNamespace = "namespace"; + final List discoveredFields = List.of(new Field("_id", JsonSchemaType.STRING), new Field("field1", JsonSchemaType.STRING), + new Field("field2", JsonSchemaType.NUMBER)); + + final AirbyteStream airbyteStream = MongoCatalogHelper.buildSchemalessAirbyteStream(streamName, streamNamespace, discoveredFields); + + assertNotNull(airbyteStream); + assertEquals(streamNamespace, airbyteStream.getNamespace()); + assertEquals(streamName, airbyteStream.getName()); + assertEquals(List.of(DEFAULT_CURSOR_FIELD), airbyteStream.getDefaultCursorField()); + assertEquals(true, airbyteStream.getSourceDefinedCursor()); + assertEquals(List.of(List.of(DEFAULT_PRIMARY_KEY)), airbyteStream.getSourceDefinedPrimaryKey()); + assertEquals(SUPPORTED_SYNC_MODES, airbyteStream.getSupportedSyncModes()); + assertEquals(5, airbyteStream.getJsonSchema().get("properties").size()); + + // All discovered fields that are not the _id field should be discarded + assertTrue(airbyteStream.getJsonSchema().get("properties").has(SCHEMALESS_MODE_DATA_FIELD)); + assertEquals(JsonSchemaType.OBJECT.getJsonSchemaTypeMap().get("type"), + airbyteStream.getJsonSchema().get("properties").get(SCHEMALESS_MODE_DATA_FIELD).get("type").asText()); + assertTrue(airbyteStream.getJsonSchema().get("properties").has(ID_FIELD)); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), + airbyteStream.getJsonSchema().get("properties").get(ID_FIELD).get("type").asText()); + assertTrue(airbyteStream.getJsonSchema().get("properties").has(DEFAULT_CURSOR_FIELD)); + assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get("type"), + airbyteStream.getJsonSchema().get("properties").get(DEFAULT_CURSOR_FIELD).get("type").asText()); + assertTrue(airbyteStream.getJsonSchema().get("properties").has(DebeziumEventUtils.CDC_DELETED_AT)); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), + airbyteStream.getJsonSchema().get("properties").get(DebeziumEventUtils.CDC_DELETED_AT).get("type").asText()); + assertTrue(airbyteStream.getJsonSchema().get("properties").has(DebeziumEventUtils.CDC_UPDATED_AT)); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), + airbyteStream.getJsonSchema().get("properties").get(DebeziumEventUtils.CDC_UPDATED_AT).get("type").asText()); + + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoConnectionUtilsTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoConnectionUtilsTest.java new file mode 100644 index 000000000000..08180f774d34 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoConnectionUtilsTest.java @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb; + +import static io.airbyte.integrations.source.mongodb.MongoConstants.CREDENTIALS_PLACEHOLDER; +import static io.airbyte.integrations.source.mongodb.MongoConstants.DATABASE_CONFIG_CONFIGURATION_KEY; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import com.mongodb.ReadPreference; +import com.mongodb.ServerAddress; +import com.mongodb.client.MongoClient; +import com.mongodb.client.internal.MongoClientImpl; +import io.airbyte.commons.json.Jsons; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class MongoConnectionUtilsTest { + + @Test + void testCreateMongoClient() { + final String authSource = "admin"; + final String host = "host"; + final int port = 1234; + final String username = "user"; + final String password = "password"; + final MongoDbSourceConfig config = new MongoDbSourceConfig(Jsons.jsonNode( + Map.of(DATABASE_CONFIG_CONFIGURATION_KEY, + Map.of( + MongoConstants.CONNECTION_STRING_CONFIGURATION_KEY, "mongodb://" + host + ":" + port + "/", + MongoConstants.USERNAME_CONFIGURATION_KEY, username, + MongoConstants.PASSWORD_CONFIGURATION_KEY, password, + MongoConstants.AUTH_SOURCE_CONFIGURATION_KEY, authSource)))); + + final MongoClient mongoClient = MongoConnectionUtils.createMongoClient(config); + + assertNotNull(mongoClient); + assertEquals(List.of(new ServerAddress(host, port)), ((MongoClientImpl) mongoClient).getSettings().getClusterSettings().getHosts()); + assertEquals(ReadPreference.secondaryPreferred(), ((MongoClientImpl) mongoClient).getSettings().getReadPreference()); + assertEquals(false, ((MongoClientImpl) mongoClient).getSettings().getRetryWrites()); + assertEquals(true, ((MongoClientImpl) mongoClient).getSettings().getSslSettings().isEnabled()); + assertEquals(List.of("sync", MongoConstants.DRIVER_NAME), ((MongoClientImpl) mongoClient).getMongoDriverInformation().getDriverNames()); + assertEquals(username, ((MongoClientImpl) mongoClient).getSettings().getCredential().getUserName()); + assertEquals(password, new String(((MongoClientImpl) mongoClient).getSettings().getCredential().getPassword())); + assertEquals(authSource, ((MongoClientImpl) mongoClient).getSettings().getCredential().getSource()); + } + + @Test + void testCreateMongoClientWithQuotesInConnectionString() { + final String authSource = "admin"; + final String host = "host"; + final int port = 1234; + final String username = "user"; + final String password = "password"; + final MongoDbSourceConfig config = new MongoDbSourceConfig(Jsons.jsonNode( + Map.of(DATABASE_CONFIG_CONFIGURATION_KEY, + Map.of( + MongoConstants.CONNECTION_STRING_CONFIGURATION_KEY, "\"mongodb://" + host + ":" + port + "/\"", + MongoConstants.USERNAME_CONFIGURATION_KEY, username, + MongoConstants.PASSWORD_CONFIGURATION_KEY, password, + MongoConstants.AUTH_SOURCE_CONFIGURATION_KEY, authSource)))); + + final MongoClient mongoClient = MongoConnectionUtils.createMongoClient(config); + + assertNotNull(mongoClient); + assertEquals(List.of(new ServerAddress(host, port)), ((MongoClientImpl) mongoClient).getSettings().getClusterSettings().getHosts()); + assertEquals(ReadPreference.secondaryPreferred(), ((MongoClientImpl) mongoClient).getSettings().getReadPreference()); + assertEquals(false, ((MongoClientImpl) mongoClient).getSettings().getRetryWrites()); + assertEquals(true, ((MongoClientImpl) mongoClient).getSettings().getSslSettings().isEnabled()); + assertEquals(List.of("sync", MongoConstants.DRIVER_NAME), ((MongoClientImpl) mongoClient).getMongoDriverInformation().getDriverNames()); + assertEquals(username, ((MongoClientImpl) mongoClient).getSettings().getCredential().getUserName()); + assertEquals(password, new String(((MongoClientImpl) mongoClient).getSettings().getCredential().getPassword())); + assertEquals(authSource, ((MongoClientImpl) mongoClient).getSettings().getCredential().getSource()); + } + + @Test + void testCreateMongoClientWithoutCredentials() { + final String host = "host"; + final int port = 1234; + final MongoDbSourceConfig config = new MongoDbSourceConfig(Jsons.jsonNode( + Map.of(DATABASE_CONFIG_CONFIGURATION_KEY, + Map.of(MongoConstants.CONNECTION_STRING_CONFIGURATION_KEY, "mongodb://" + host + ":" + port + "/")))); + + final MongoClient mongoClient = MongoConnectionUtils.createMongoClient(config); + + assertNotNull(mongoClient); + assertEquals(List.of(new ServerAddress(host, port)), ((MongoClientImpl) mongoClient).getSettings().getClusterSettings().getHosts()); + assertEquals(ReadPreference.secondaryPreferred(), ((MongoClientImpl) mongoClient).getSettings().getReadPreference()); + assertEquals(false, ((MongoClientImpl) mongoClient).getSettings().getRetryWrites()); + assertEquals(true, ((MongoClientImpl) mongoClient).getSettings().getSslSettings().isEnabled()); + assertEquals(List.of("sync", MongoConstants.DRIVER_NAME), ((MongoClientImpl) mongoClient).getMongoDriverInformation().getDriverNames()); + assertNull(((MongoClientImpl) mongoClient).getSettings().getCredential()); + } + + @Test + void testCreateMongoClientWithCredentialPlaceholderInConnectionString() { + final String authSource = "admin"; + final String host = "host"; + final int port = 1234; + final String username = "user"; + final String password = "password"; + final MongoDbSourceConfig config = new MongoDbSourceConfig(Jsons.jsonNode( + Map.of(DATABASE_CONFIG_CONFIGURATION_KEY, + Map.of( + MongoConstants.CONNECTION_STRING_CONFIGURATION_KEY, "mongodb://" + CREDENTIALS_PLACEHOLDER + host + ":" + port + "/", + MongoConstants.USERNAME_CONFIGURATION_KEY, username, + MongoConstants.PASSWORD_CONFIGURATION_KEY, password, + MongoConstants.AUTH_SOURCE_CONFIGURATION_KEY, authSource)))); + + final MongoClient mongoClient = MongoConnectionUtils.createMongoClient(config); + + assertNotNull(mongoClient); + assertEquals(List.of(new ServerAddress(host, port)), ((MongoClientImpl) mongoClient).getSettings().getClusterSettings().getHosts()); + assertEquals(ReadPreference.secondaryPreferred(), ((MongoClientImpl) mongoClient).getSettings().getReadPreference()); + assertEquals(false, ((MongoClientImpl) mongoClient).getSettings().getRetryWrites()); + assertEquals(true, ((MongoClientImpl) mongoClient).getSettings().getSslSettings().isEnabled()); + assertEquals(List.of("sync", MongoConstants.DRIVER_NAME), ((MongoClientImpl) mongoClient).getMongoDriverInformation().getDriverNames()); + assertEquals(username, ((MongoClientImpl) mongoClient).getSettings().getCredential().getUserName()); + assertEquals(password, new String(((MongoClientImpl) mongoClient).getSettings().getCredential().getPassword())); + assertEquals(authSource, ((MongoClientImpl) mongoClient).getSettings().getCredential().getSource()); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbDebugger.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbDebugger.java new file mode 100644 index 000000000000..c492e9064d30 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbDebugger.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ +package io.airbyte.integrations.source.mongodb; + +import io.airbyte.cdk.integrations.debug.DebugUtil; + +public class MongoDbDebugger { + + @SuppressWarnings({"unchecked", "deprecation", "resource"}) + public static void main(final String[] args) throws Exception { + final MongoDbSource mongoDbSource = new MongoDbSource(); + DebugUtil.debug(mongoDbSource); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbSourceConfigTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbSourceConfigTest.java new file mode 100644 index 000000000000..9e8665cf65d7 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbSourceConfigTest.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb; + +import static io.airbyte.integrations.source.mongodb.MongoConstants.AUTH_SOURCE_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.CHECKPOINT_INTERVAL; +import static io.airbyte.integrations.source.mongodb.MongoConstants.CHECKPOINT_INTERVAL_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.DATABASE_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.DATABASE_CONFIG_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.DEFAULT_AUTH_SOURCE; +import static io.airbyte.integrations.source.mongodb.MongoConstants.DEFAULT_DISCOVER_SAMPLE_SIZE; +import static io.airbyte.integrations.source.mongodb.MongoConstants.DISCOVER_SAMPLE_SIZE_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.PASSWORD_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.QUEUE_SIZE_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.SCHEMA_ENFORCED_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.USERNAME_CONFIGURATION_KEY; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import java.util.Map; +import java.util.OptionalInt; +import org.junit.jupiter.api.Test; + +class MongoDbSourceConfigTest { + + @Test + void testCreatingMongoDbSourceConfig() { + final String authSource = "auth"; + final Integer checkpointInterval = 1; + final String database = "database"; + final Integer queueSize = 1; + final String password = "password"; + final Integer sampleSize = 5000; + final String username = "username"; + final boolean isSchemaEnforced = false; + final JsonNode rawConfig = Jsons.jsonNode( + Map.of(DATABASE_CONFIG_CONFIGURATION_KEY, Map.of( + AUTH_SOURCE_CONFIGURATION_KEY, authSource, + CHECKPOINT_INTERVAL_CONFIGURATION_KEY, checkpointInterval, + DATABASE_CONFIGURATION_KEY, database, + DISCOVER_SAMPLE_SIZE_CONFIGURATION_KEY, sampleSize, + PASSWORD_CONFIGURATION_KEY, password, + QUEUE_SIZE_CONFIGURATION_KEY, queueSize, + USERNAME_CONFIGURATION_KEY, username, + SCHEMA_ENFORCED_CONFIGURATION_KEY, isSchemaEnforced))); + final MongoDbSourceConfig sourceConfig = new MongoDbSourceConfig(rawConfig); + assertNotNull(sourceConfig); + assertEquals(authSource, sourceConfig.getAuthSource()); + assertEquals(checkpointInterval, sourceConfig.getCheckpointInterval()); + assertEquals(database, sourceConfig.getDatabaseName()); + assertEquals(password, sourceConfig.getPassword()); + assertEquals(OptionalInt.of(queueSize), sourceConfig.getQueueSize()); + assertEquals(rawConfig.get(DATABASE_CONFIG_CONFIGURATION_KEY), sourceConfig.rawConfig()); + assertEquals(sampleSize, sourceConfig.getSampleSize()); + assertEquals(username, sourceConfig.getUsername()); + assertEquals(isSchemaEnforced, sourceConfig.getEnforceSchema()); + } + + @Test + void testCreatingInvalidMongoDbSourceConfig() { + assertThrows(IllegalArgumentException.class, () -> new MongoDbSourceConfig(Jsons.jsonNode(Map.of()))); + } + + @Test + void testDefaultValues() { + final JsonNode rawConfig = Jsons.jsonNode(Map.of(DATABASE_CONFIG_CONFIGURATION_KEY, Map.of())); + final MongoDbSourceConfig sourceConfig = new MongoDbSourceConfig(rawConfig); + assertNotNull(sourceConfig); + assertEquals(DEFAULT_AUTH_SOURCE, sourceConfig.getAuthSource()); + assertEquals(CHECKPOINT_INTERVAL, sourceConfig.getCheckpointInterval()); + assertEquals(null, sourceConfig.getDatabaseName()); + assertEquals(null, sourceConfig.getPassword()); + assertEquals(OptionalInt.empty(), sourceConfig.getQueueSize()); + assertEquals(rawConfig.get(DATABASE_CONFIG_CONFIGURATION_KEY), sourceConfig.rawConfig()); + assertEquals(DEFAULT_DISCOVER_SAMPLE_SIZE, sourceConfig.getSampleSize()); + assertEquals(null, sourceConfig.getUsername()); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbSourceTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbSourceTest.java new file mode 100644 index 000000000000..49a90c5ec563 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbSourceTest.java @@ -0,0 +1,323 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb; + +import static io.airbyte.integrations.source.mongodb.MongoCatalogHelper.DEFAULT_CURSOR_FIELD; +import static io.airbyte.integrations.source.mongodb.MongoConstants.DATABASE_CONFIG_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.DEFAULT_DISCOVER_SAMPLE_SIZE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.mongodb.MongoCredential; +import com.mongodb.MongoSecurityException; +import com.mongodb.client.AggregateIterable; +import com.mongodb.client.ChangeStreamIterable; +import com.mongodb.client.MongoChangeStreamCursor; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoCursor; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.MongoIterable; +import com.mongodb.connection.ClusterDescription; +import com.mongodb.connection.ClusterType; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcInitializer; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteCatalog; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.bson.BsonDocument; +import org.bson.Document; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class MongoDbSourceTest { + + private static final String DB_NAME = "airbyte_test"; + + private JsonNode airbyteSourceConfig; + private MongoDbSourceConfig sourceConfig; + private MongoClient mongoClient; + private MongoDbCdcInitializer cdcInitializer; + private MongoDbSource source; + + @BeforeEach + void setup() { + airbyteSourceConfig = createConfiguration(Optional.empty(), Optional.empty(), true); + sourceConfig = new MongoDbSourceConfig(airbyteSourceConfig); + mongoClient = mock(MongoClient.class); + cdcInitializer = mock(MongoDbCdcInitializer.class); + source = spy(new MongoDbSource(cdcInitializer)); + final MongoIterable iterable = mock(MongoIterable.class); + + when(iterable.spliterator()).thenReturn(List.of(DB_NAME).spliterator()); + when(mongoClient.listDatabaseNames()).thenReturn(iterable); + doReturn(mongoClient).when(source).createMongoClient(sourceConfig); + } + + @Test + void testCheckOperation() throws IOException { + final ClusterDescription clusterDescription = mock(ClusterDescription.class); + final Document response = Document.parse(MoreResources.readResource("authorized_collections_response.json")); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + + when(clusterDescription.getType()).thenReturn(ClusterType.REPLICA_SET); + when(mongoDatabase.runCommand(any())).thenReturn(response); + when(mongoClient.getDatabase(any())).thenReturn(mongoDatabase); + when(mongoClient.getClusterDescription()).thenReturn(clusterDescription); + + final AirbyteConnectionStatus airbyteConnectionStatus = source.check(airbyteSourceConfig); + assertNotNull(airbyteConnectionStatus); + assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, airbyteConnectionStatus.getStatus()); + } + + @Test + void testCheckOperationMissingDatabase() throws IOException { + final ClusterDescription clusterDescription = mock(ClusterDescription.class); + final Document response = Document.parse(MoreResources.readResource("authorized_collections_response.json")); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + final MongoIterable iterable = mock(MongoIterable.class); + + when(iterable.spliterator()).thenReturn(List.of("other").spliterator()); + when(mongoClient.listDatabaseNames()).thenReturn(iterable); + + when(clusterDescription.getType()).thenReturn(ClusterType.REPLICA_SET); + when(mongoDatabase.runCommand(any())).thenReturn(response); + when(mongoClient.getDatabase(any())).thenReturn(mongoDatabase); + when(mongoClient.getClusterDescription()).thenReturn(clusterDescription); + + final AirbyteConnectionStatus airbyteConnectionStatus = source.check(airbyteSourceConfig); + assertNotNull(airbyteConnectionStatus); + assertEquals(AirbyteConnectionStatus.Status.FAILED, airbyteConnectionStatus.getStatus()); + } + + @Test + void testCheckOperationWithMissingConfiguration() throws IOException { + final ClusterDescription clusterDescription = mock(ClusterDescription.class); + final Document response = Document.parse(MoreResources.readResource("authorized_collections_response.json")); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + + when(clusterDescription.getType()).thenReturn(ClusterType.REPLICA_SET); + when(mongoDatabase.runCommand(any())).thenReturn(response); + when(mongoClient.getDatabase(any())).thenReturn(mongoDatabase); + when(mongoClient.getClusterDescription()).thenReturn(clusterDescription); + + final AirbyteConnectionStatus airbyteConnectionStatus = source.check(Jsons.jsonNode(Map.of())); + assertNotNull(airbyteConnectionStatus); + assertEquals(AirbyteConnectionStatus.Status.FAILED, airbyteConnectionStatus.getStatus()); + assertEquals("Unable to perform connection check operation: Database configuration is missing required 'database_config' property.", + airbyteConnectionStatus.getMessage()); + } + + @Test + void testCheckOperationNoAuthorizedCollections() throws IOException { + final ClusterDescription clusterDescription = mock(ClusterDescription.class); + final Document response = Document.parse(MoreResources.readResource("no_authorized_collections_response.json")); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + + when(clusterDescription.getType()).thenReturn(ClusterType.REPLICA_SET); + when(mongoDatabase.runCommand(any())).thenReturn(response); + when(mongoClient.getDatabase(any())).thenReturn(mongoDatabase); + when(mongoClient.getClusterDescription()).thenReturn(clusterDescription); + + final AirbyteConnectionStatus airbyteConnectionStatus = source.check(airbyteSourceConfig); + assertNotNull(airbyteConnectionStatus); + assertEquals(AirbyteConnectionStatus.Status.FAILED, airbyteConnectionStatus.getStatus()); + assertEquals("Target MongoDB database does not contain any authorized collections.", airbyteConnectionStatus.getMessage()); + } + + @Test + void testCheckOperationInvalidClusterType() throws IOException { + final ClusterDescription clusterDescription = mock(ClusterDescription.class); + final Document response = Document.parse(MoreResources.readResource("authorized_collections_response.json")); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + + when(clusterDescription.getType()).thenReturn(ClusterType.STANDALONE); + when(mongoDatabase.runCommand(any())).thenReturn(response); + when(mongoClient.getDatabase(any())).thenReturn(mongoDatabase); + when(mongoClient.getClusterDescription()).thenReturn(clusterDescription); + + final AirbyteConnectionStatus airbyteConnectionStatus = source.check(airbyteSourceConfig); + assertNotNull(airbyteConnectionStatus); + assertEquals(AirbyteConnectionStatus.Status.FAILED, airbyteConnectionStatus.getStatus()); + assertEquals("Target MongoDB instance is not a replica set cluster.", airbyteConnectionStatus.getMessage()); + } + + @Test + void testCheckOperationAuthenticationFailure() { + final ClusterDescription clusterDescription = mock(ClusterDescription.class); + + when(clusterDescription.getType()).thenReturn(ClusterType.REPLICA_SET); + when(mongoClient.getDatabase(any())).thenThrow(new MongoSecurityException( + MongoCredential.createCredential("username", DB_NAME, "password".toCharArray()), "test")); + when(mongoClient.getClusterDescription()).thenReturn(clusterDescription); + + final AirbyteConnectionStatus airbyteConnectionStatus = source.check(airbyteSourceConfig); + assertNotNull(airbyteConnectionStatus); + assertEquals(AirbyteConnectionStatus.Status.FAILED, airbyteConnectionStatus.getStatus()); + } + + @Test + void testCheckOperationUnexpectedException() { + final String expectedMessage = "This is just a test failure."; + when(mongoClient.getDatabase(any())).thenThrow(new IllegalArgumentException(expectedMessage)); + + final AirbyteConnectionStatus airbyteConnectionStatus = source.check(airbyteSourceConfig); + assertNotNull(airbyteConnectionStatus); + assertEquals(AirbyteConnectionStatus.Status.FAILED, airbyteConnectionStatus.getStatus()); + assertEquals(expectedMessage, airbyteConnectionStatus.getMessage()); + } + + @Test + void testDiscoverOperation() throws IOException { + final AggregateIterable aggregateIterable = mock(AggregateIterable.class); + final List> schemaDiscoveryJsonResponses = + Jsons.deserialize(MoreResources.readResource("schema_discovery_response.json"), new TypeReference<>() {}); + final List schemaDiscoveryResponses = schemaDiscoveryJsonResponses.stream().map(Document::new).toList(); + final Document authorizedCollectionsResponse = Document.parse(MoreResources.readResource("authorized_collections_response.json")); + final MongoCollection mongoCollection = mock(MongoCollection.class); + final MongoCursor cursor = mock(MongoCursor.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + + when(cursor.hasNext()).thenReturn(true, true, false); + when(cursor.next()).thenReturn(schemaDiscoveryResponses.get(0), schemaDiscoveryResponses.get(1)); + when(aggregateIterable.cursor()).thenReturn(cursor); + when(mongoCollection.aggregate(any())).thenReturn(aggregateIterable); + when(mongoDatabase.getCollection(any())).thenReturn(mongoCollection); + when(mongoDatabase.runCommand(any())).thenReturn(authorizedCollectionsResponse); + when(mongoClient.getDatabase(any())).thenReturn(mongoDatabase); + when(aggregateIterable.allowDiskUse(anyBoolean())).thenReturn(aggregateIterable); + + final AirbyteCatalog airbyteCatalog = source.discover(airbyteSourceConfig); + + assertNotNull(airbyteCatalog); + assertEquals(1, airbyteCatalog.getStreams().size()); + + final Optional stream = airbyteCatalog.getStreams().stream().findFirst(); + assertTrue(stream.isPresent()); + assertEquals(DB_NAME, stream.get().getNamespace()); + assertEquals("testCollection", stream.get().getName()); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), + stream.get().getJsonSchema().get("properties").get("_id").get("type").asText()); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), + stream.get().getJsonSchema().get("properties").get("name").get("type").asText()); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), + stream.get().getJsonSchema().get("properties").get("last_updated").get("type").asText()); + assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get("type"), + stream.get().getJsonSchema().get("properties").get("total").get("type").asText()); + assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get("type"), + stream.get().getJsonSchema().get("properties").get("price").get("type").asText()); + assertEquals(JsonSchemaType.ARRAY.getJsonSchemaTypeMap().get("type"), + stream.get().getJsonSchema().get("properties").get("items").get("type").asText()); + assertEquals(JsonSchemaType.OBJECT.getJsonSchemaTypeMap().get("type"), + stream.get().getJsonSchema().get("properties").get("owners").get("type").asText()); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), + stream.get().getJsonSchema().get("properties").get("other").get("type").asText()); + assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get("type"), + stream.get().getJsonSchema().get("properties").get(DEFAULT_CURSOR_FIELD).get("type").asText()); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), + stream.get().getJsonSchema().get("properties").get(DebeziumEventUtils.CDC_DELETED_AT).get("type").asText()); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get("type"), + stream.get().getJsonSchema().get("properties").get(DebeziumEventUtils.CDC_UPDATED_AT).get("type").asText()); + assertEquals(true, stream.get().getSourceDefinedCursor()); + assertEquals(List.of(DEFAULT_CURSOR_FIELD), stream.get().getDefaultCursorField()); + assertEquals(List.of(List.of(MongoCatalogHelper.DEFAULT_PRIMARY_KEY)), stream.get().getSourceDefinedPrimaryKey()); + assertEquals(MongoCatalogHelper.SUPPORTED_SYNC_MODES, stream.get().getSupportedSyncModes()); + } + + @Test + void testDiscoverOperationWithMissingConfiguration() throws IOException { + final AggregateIterable aggregateIterable = mock(AggregateIterable.class); + final List> schemaDiscoveryJsonResponses = + Jsons.deserialize(MoreResources.readResource("schema_discovery_response.json"), new TypeReference<>() {}); + final List schemaDiscoveryResponses = schemaDiscoveryJsonResponses.stream().map(Document::new).toList(); + final Document authorizedCollectionsResponse = Document.parse(MoreResources.readResource("authorized_collections_response.json")); + final MongoCollection mongoCollection = mock(MongoCollection.class); + final MongoCursor cursor = mock(MongoCursor.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + + when(cursor.hasNext()).thenReturn(true, true, false); + when(cursor.next()).thenReturn(schemaDiscoveryResponses.get(0), schemaDiscoveryResponses.get(1)); + when(aggregateIterable.cursor()).thenReturn(cursor); + when(mongoCollection.aggregate(any())).thenReturn(aggregateIterable); + when(mongoDatabase.getCollection(any())).thenReturn(mongoCollection); + when(mongoDatabase.runCommand(any())).thenReturn(authorizedCollectionsResponse); + when(mongoClient.getDatabase(any())).thenReturn(mongoDatabase); + + assertThrows(IllegalArgumentException.class, () -> source.discover(Jsons.jsonNode(Map.of()))); + } + + @Test + void testDiscoverOperationWithUnexpectedFailure() { + final String expectedMessage = "This is just a test failure."; + when(mongoClient.getDatabase(any())).thenThrow(new IllegalArgumentException(expectedMessage)); + + assertThrows(IllegalArgumentException.class, () -> source.discover(airbyteSourceConfig)); + } + + @Test + void testReadClosesMongoClient() { + final MongoClient mongoClient = mock(MongoClient.class); + doReturn(mongoClient).when(source).createMongoClient(sourceConfig); + when(cdcInitializer.createCdcIterators(any(), any(), any(), any(), any(), any())).thenThrow(new RuntimeException()); + assertThrows(RuntimeException.class, () -> source.read(airbyteSourceConfig, null, null)); + verify(mongoClient, times(1)).close(); + } + + @Test + void testReadWithMissingConfiguration() { + final ConfiguredAirbyteCatalog catalog = mock(ConfiguredAirbyteCatalog.class); + final JsonNode state = mock(JsonNode.class); + assertThrows(IllegalArgumentException.class, () -> source.read(Jsons.jsonNode(Map.of()), catalog, state)); + } + + @Test + void testReadKeepsMongoClientOpen() { + final ChangeStreamIterable changeStreamIterable = mock(ChangeStreamIterable.class); + final MongoChangeStreamCursor mongoChangeStreamCursor = mock(MongoChangeStreamCursor.class); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + when(cdcInitializer.createCdcIterators(any(), any(), any(), any(), any(), any())).thenReturn(Collections.emptyList()); + source.read(airbyteSourceConfig, null, null); + verify(mongoClient, never()).close(); + } + + private static JsonNode createConfiguration(final Optional username, final Optional password, final boolean isSchemaEnforced) { + final Map baseConfig = Map.of( + MongoConstants.DATABASE_CONFIGURATION_KEY, DB_NAME, + MongoConstants.CONNECTION_STRING_CONFIGURATION_KEY, "mongodb://localhost:27017/", + MongoConstants.AUTH_SOURCE_CONFIGURATION_KEY, "admin", + MongoConstants.DISCOVER_SAMPLE_SIZE_CONFIGURATION_KEY, DEFAULT_DISCOVER_SAMPLE_SIZE, + MongoConstants.SCHEMA_ENFORCED_CONFIGURATION_KEY, isSchemaEnforced); + + final Map config = new HashMap<>(baseConfig); + username.ifPresent(u -> config.put(MongoConstants.USERNAME_CONFIGURATION_KEY, u)); + password.ifPresent(p -> config.put(MongoConstants.PASSWORD_CONFIGURATION_KEY, p)); + return Jsons.deserialize(Jsons.serialize(Map.of(DATABASE_CONFIG_CONFIGURATION_KEY, config))); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbStateIteratorTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbStateIteratorTest.java new file mode 100644 index 000000000000..7b2a35fabdfa --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoDbStateIteratorTest.java @@ -0,0 +1,367 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb; + +import static java.time.temporal.ChronoUnit.SECONDS; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.mongodb.MongoException; +import com.mongodb.client.MongoCursor; +import io.airbyte.commons.exceptions.ConfigErrorException; +import io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcConnectorMetadataInjector; +import io.airbyte.integrations.source.mongodb.state.IdType; +import io.airbyte.integrations.source.mongodb.state.InitialSnapshotStatus; +import io.airbyte.integrations.source.mongodb.state.MongoDbStateManager; +import io.airbyte.integrations.source.mongodb.state.MongoDbStreamState; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.SyncMode; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import org.bson.Document; +import org.bson.types.ObjectId; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +class MongoDbStateIteratorTest { + + private static final int CHECKPOINT_INTERVAL = 2; + @Mock + private MongoCursor mongoCursor; + private AutoCloseable closeable; + private MongoDbStateManager stateManager; + private MongoDbCdcConnectorMetadataInjector cdcConnectorMetadataInjector; + + @BeforeEach + public void setup() { + closeable = MockitoAnnotations.openMocks(this); + stateManager = MongoDbStateManager.createStateManager(null); + cdcConnectorMetadataInjector = mock(MongoDbCdcConnectorMetadataInjector.class); + } + + @AfterEach + public void teardown() throws Exception { + closeable.close(); + } + + @Test + void happyPath() { + final var docs = docs(); + + when(mongoCursor.hasNext()).thenAnswer(new Answer() { + + private int count = 0; + + @Override + public Boolean answer(final InvocationOnMock invocation) { + count++; + // hasNext will be called for each doc plus for each state message + return count <= (docs.size() + (docs.size() % CHECKPOINT_INTERVAL)); + } + + }); + + when(mongoCursor.next()).thenAnswer(new Answer() { + + private int offset = 0; + + @Override + public Document answer(final InvocationOnMock invocation) { + final var doc = docs.get(offset); + offset++; + return doc; + } + + }); + + final var stream = catalog().getStreams().stream().findFirst().orElseThrow(); + + final var iter = new MongoDbStateIterator(mongoCursor, stateManager, Optional.of(cdcConnectorMetadataInjector), stream, Instant.now(), + CHECKPOINT_INTERVAL, MongoConstants.CHECKPOINT_DURATION, true); + + // with a batch size of 2, the MongoDbStateIterator should return the following after each + // `hasNext`/`next` call: + // true, record Air Force Blue + // true, record Alice Blue + // true, state (with Alice Blue as the state) + // true, record Alizarin Crimson + // true, state (with Alizarin Crimson) + // false + AirbyteMessage message; + assertTrue(iter.hasNext(), "air force blue should be next"); + message = iter.next(); + assertEquals(Type.RECORD, message.getType()); + assertEquals(docs.get(0).get("_id").toString(), message.getRecord().getData().get("_id").asText()); + + assertTrue(iter.hasNext(), "alice blue should be next"); + message = iter.next(); + assertEquals(Type.RECORD, message.getType()); + assertEquals(docs.get(1).get("_id").toString(), message.getRecord().getData().get("_id").asText()); + + assertTrue(iter.hasNext(), "state should be next"); + message = iter.next(); + assertEquals(Type.STATE, message.getType()); + assertEquals( + docs.get(1).get("_id").toString(), + message.getState().getGlobal().getStreamStates().get(0).getStreamState().get("id").asText(), + "state id should match last record id"); + Assertions.assertEquals( + InitialSnapshotStatus.IN_PROGRESS.toString(), + message.getState().getGlobal().getStreamStates().get(0).getStreamState().get("status").asText(), + "state status should be in_progress"); + + assertTrue(iter.hasNext(), "alizarin crimson should be next"); + message = iter.next(); + assertEquals(Type.RECORD, message.getType()); + assertEquals(docs.get(2).get("_id").toString(), message.getRecord().getData().get("_id").asText()); + + assertTrue(iter.hasNext(), "state should be next"); + message = iter.next(); + assertEquals(Type.STATE, message.getType()); + assertEquals( + docs.get(2).get("_id").toString(), + message.getState().getGlobal().getStreamStates().get(0).getStreamState().get("id").asText(), + "state id should match last record id"); + assertEquals( + InitialSnapshotStatus.COMPLETE.toString(), + message.getState().getGlobal().getStreamStates().get(0).getStreamState().get("status").asText(), + "state status should be complete"); + + assertFalse(iter.hasNext(), "should have no more records"); + } + + @Test + void treatHasNextExceptionAsFalse() { + final var docs = docs(); + + // on the second hasNext call, throw an exception + when(mongoCursor.hasNext()) + .thenReturn(true) + .thenThrow(new MongoException("test exception")); + + when(mongoCursor.next()).thenReturn(docs.get(0)); + + final var stream = catalog().getStreams().stream().findFirst().orElseThrow(); + + final var iter = new MongoDbStateIterator(mongoCursor, stateManager, Optional.of(cdcConnectorMetadataInjector), stream, Instant.now(), + CHECKPOINT_INTERVAL, MongoConstants.CHECKPOINT_DURATION, true); + + // with a batch size of 2, the MongoDbStateIterator should return the following after each + // `hasNext`/`next` call: + // true, record Air Force Blue + // true (exception thrown), state (with Air Force Blue as the state) + // false + AirbyteMessage message; + assertTrue(iter.hasNext(), "air force blue should be next"); + message = iter.next(); + assertEquals(Type.RECORD, message.getType()); + assertEquals(docs.get(0).get("_id").toString(), message.getRecord().getData().get("_id").asText()); + + assertTrue(iter.hasNext(), "state should be next"); + message = iter.next(); + assertEquals(Type.STATE, message.getType()); + assertEquals( + docs.get(0).get("_id").toString(), + message.getState().getGlobal().getStreamStates().get(0).getStreamState().get("id").asText(), + "state id should match last record id"); + assertEquals( + InitialSnapshotStatus.IN_PROGRESS.toString(), + message.getState().getGlobal().getStreamStates().get(0).getStreamState().get("status").asText(), + "state status should be in_progress"); + + assertFalse(iter.hasNext(), "should have no more records"); + } + + @Test + void anInvalidIdFieldThrowsAnException() { + final var doc = new Document("_id", 0.1).append("name", "Air Force Blue").append("hex", "#5d8aa8"); + + // on the second hasNext call, throw an exception + when(mongoCursor.hasNext()) + .thenReturn(true, false); + + when(mongoCursor.next()).thenReturn(doc); + + final var stream = catalog().getStreams().stream().findFirst().orElseThrow(); + + final var iter = new MongoDbStateIterator(mongoCursor, stateManager, Optional.of(cdcConnectorMetadataInjector), stream, Instant.now(), + CHECKPOINT_INTERVAL, MongoConstants.CHECKPOINT_DURATION, true); + + assertTrue(iter.hasNext(), "air force blue should be next"); + // first next call should return the document + iter.next(); + assertTrue(iter.hasNext(), "air force blue should be next"); + // second next call should throw an exception + assertThrows(ConfigErrorException.class, iter::next); + } + + @Test + void initialStateIsReturnedIfUnderlyingIteratorIsEmpty() { + // on the second hasNext call, throw an exception + when(mongoCursor.hasNext()).thenReturn(false); + + final var stream = catalog().getStreams().stream().findFirst().orElseThrow(); + final var objectId = "64dfb6a7bb3c3458c30801f4"; + + stateManager.updateStreamState(stream.getStream().getName(), stream.getStream().getNamespace(), + new MongoDbStreamState(objectId, InitialSnapshotStatus.IN_PROGRESS, IdType.OBJECT_ID)); + + final var iter = new MongoDbStateIterator(mongoCursor, stateManager, Optional.of(cdcConnectorMetadataInjector), stream, Instant.now(), + CHECKPOINT_INTERVAL, MongoConstants.CHECKPOINT_DURATION, true); + + // the MongoDbStateIterator should return the following after each + // `hasNext`/`next` call: + // false + // then the generated state message should have the same id as the initial state + assertTrue(iter.hasNext(), "state should be next"); + + final AirbyteMessage message = iter.next(); + assertEquals(Type.STATE, message.getType()); + assertEquals( + objectId, + message.getState().getGlobal().getStreamStates().get(0).getStreamState().get("id").asText(), + "state id should match initial state "); + assertEquals( + InitialSnapshotStatus.COMPLETE.toString(), + message.getState().getGlobal().getStreamStates().get(0).getStreamState().get("status").asText(), + "state status should be in_progress"); + + assertFalse(iter.hasNext(), "should have no more records"); + } + + @Test + void stateEmittedAfterDuration() throws InterruptedException { + // force a 1.5s wait between messages + when(mongoCursor.hasNext()) + .thenReturn(true, true, true, true, false); + + final var docs = docs(); + when(mongoCursor.next()).thenReturn(docs.get(0), docs.get(1)); + + final var stream = catalog().getStreams().stream().findFirst().orElseThrow(); + final var objectId = "64dfb6a7bb3c3458c30801f4"; + + stateManager.updateStreamState(stream.getStream().getName(), stream.getStream().getNamespace(), + new MongoDbStreamState(objectId, InitialSnapshotStatus.IN_PROGRESS, IdType.OBJECT_ID)); + + final var iter = new MongoDbStateIterator(mongoCursor, stateManager, Optional.of(cdcConnectorMetadataInjector), stream, Instant.now(), 1000000, + Duration.of(1, SECONDS), true); + + // with a batch size of 1,000,000 and a 1.5s sleep between hasNext calls, the expected results + // should be + // `hasNext`/`next` call: + // true, record Air Force Blue + // true, state (with Air Force Blue) + // true, record Alice Blue + // true, state (with Alice Blue as the state) + // true, state (final state) + // false + AirbyteMessage message; + assertTrue(iter.hasNext(), "air force blue should be next"); + message = iter.next(); + assertEquals(Type.RECORD, message.getType()); + assertEquals(docs.get(0).get("_id").toString(), message.getRecord().getData().get("_id").asText()); + + Thread.sleep(1500); + + assertTrue(iter.hasNext(), "state should be next"); + message = iter.next(); + assertEquals(Type.STATE, message.getType()); + assertEquals( + docs.get(0).get("_id").toString(), + message.getState().getGlobal().getStreamStates().get(0).getStreamState().get("id").asText(), + "state id should match last record id"); + assertEquals( + InitialSnapshotStatus.IN_PROGRESS.toString(), + message.getState().getGlobal().getStreamStates().get(0).getStreamState().get("status").asText(), + "state status should be in_progress"); + + assertTrue(iter.hasNext(), "alice blue should be next"); + message = iter.next(); + assertEquals(Type.RECORD, message.getType()); + assertEquals(docs.get(1).get("_id").toString(), message.getRecord().getData().get("_id").asText()); + + Thread.sleep(1500); + + assertTrue(iter.hasNext(), "state should be next"); + message = iter.next(); + assertEquals(Type.STATE, message.getType()); + assertEquals( + docs.get(1).get("_id").toString(), + message.getState().getGlobal().getStreamStates().get(0).getStreamState().get("id").asText(), + "state id should match last record id"); + assertEquals( + InitialSnapshotStatus.IN_PROGRESS.toString(), + message.getState().getGlobal().getStreamStates().get(0).getStreamState().get("status").asText(), + "state status should be in_progress"); + + assertTrue(iter.hasNext(), "final state should be next"); + message = iter.next(); + assertEquals(Type.STATE, message.getType()); + assertEquals( + docs.get(1).get("_id").toString(), + message.getState().getGlobal().getStreamStates().get(0).getStreamState().get("id").asText(), + "state id should match last record id"); + assertEquals( + InitialSnapshotStatus.COMPLETE.toString(), + message.getState().getGlobal().getStreamStates().get(0).getStreamState().get("status").asText(), + "state status should be final"); + + assertFalse(iter.hasNext(), "should have no more records"); + } + + @Test + void hasNextNoInitialStateAndNoMoreRecordsInCursor() { + when(mongoCursor.hasNext()).thenReturn(false); + final var stream = catalog().getStreams().stream().findFirst().orElseThrow(); + final var iter = new MongoDbStateIterator(mongoCursor, stateManager, Optional.of(cdcConnectorMetadataInjector), stream, Instant.now(), 1000000, + Duration.of(1, SECONDS), true); + + assertFalse(iter.hasNext()); + } + + private ConfiguredAirbyteCatalog catalog() { + return new ConfiguredAirbyteCatalog().withStreams(List.of( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(List.of("_id")) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withCursorField(List.of("_id")) + .withStream(CatalogHelpers.createAirbyteStream( + "test.unit", + Field.of("_id", JsonSchemaType.STRING), + Field.of("name", JsonSchemaType.STRING), + Field.of("hex", JsonSchemaType.STRING)) + .withSupportedSyncModes(List.of(SyncMode.INCREMENTAL)) + .withDefaultCursorField(List.of("_id"))))); + } + + private List docs() { + return List.of( + new Document("_id", new ObjectId("64c0029d95ad260d69ef28a0")) + .append("name", "Air Force Blue").append("hex", "#5d8aa8"), + new Document("_id", new ObjectId("64c0029d95ad260d69ef28a1")) + .append("name", "Alice Blue").append("hex", "#f0f8ff"), + new Document("_id", new ObjectId("64c0029d95ad260d69ef28a2")) + .append("name", "Alizarin Crimson").append("hex", "#e32636")); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoUtilTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoUtilTest.java new file mode 100644 index 000000000000..0ff81ef13e5d --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/MongoUtilTest.java @@ -0,0 +1,437 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb; + +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.integrations.source.mongodb.MongoCatalogHelper.AIRBYTE_STREAM_PROPERTIES; +import static io.airbyte.integrations.source.mongodb.MongoConstants.DATABASE_CONFIG_CONFIGURATION_KEY; +import static io.airbyte.integrations.source.mongodb.MongoConstants.DEFAULT_DISCOVER_SAMPLE_SIZE; +import static io.airbyte.integrations.source.mongodb.MongoUtil.MAX_QUEUE_SIZE; +import static io.airbyte.integrations.source.mongodb.MongoUtil.MIN_QUEUE_SIZE; +import static io.airbyte.integrations.source.mongodb.MongoUtil.checkSchemaModeMismatch; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.assertj.core.api.AssertionsForClassTypes.catchThrowable; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.mongodb.MongoCommandException; +import com.mongodb.MongoCredential; +import com.mongodb.MongoException; +import com.mongodb.MongoSecurityException; +import com.mongodb.ServerAddress; +import com.mongodb.client.AggregateIterable; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoCursor; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.MongoIterable; +import io.airbyte.commons.exceptions.ConfigErrorException; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcConnectorMetadataInjector; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import org.bson.BsonDocument; +import org.bson.Document; +import org.junit.jupiter.api.Test; + +public class MongoUtilTest { + + private static final String JSON_TYPE_PROPERTY_NAME = "type"; + + @Test + void testCheckDatabaseExists() { + final String databaseName = "test"; + final List databaseNames = List.of("test", "test1", "test2"); + final MongoIterable iterable = mock(MongoIterable.class); + final MongoClient mongoClient = mock(MongoClient.class); + + when(iterable.spliterator()).thenReturn(databaseNames.spliterator()); + when(mongoClient.listDatabaseNames()).thenReturn(iterable); + + assertTrue(MongoUtil.checkDatabaseExists(mongoClient, databaseName)); + assertFalse(MongoUtil.checkDatabaseExists(mongoClient, "other")); + } + + @Test + void testGetAirbyteStreams() throws IOException { + final AggregateIterable aggregateIterable = mock(AggregateIterable.class); + final MongoCursor cursor = mock(MongoCursor.class); + final String databaseName = "database"; + final Document authorizedCollectionsResponse = Document.parse(MoreResources.readResource("authorized_collections_response.json")); + final MongoClient mongoClient = mock(MongoClient.class); + final MongoCollection mongoCollection = mock(MongoCollection.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + final List> schemaDiscoveryJsonResponses = + Jsons.deserialize(MoreResources.readResource("schema_discovery_response.json"), new TypeReference<>() {}); + final List schemaDiscoveryResponses = schemaDiscoveryJsonResponses.stream().map(Document::new).toList(); + + when(cursor.hasNext()).thenReturn(true, true, false); + when(cursor.next()).thenReturn(schemaDiscoveryResponses.get(0), schemaDiscoveryResponses.get(1)); + when(aggregateIterable.cursor()).thenReturn(cursor); + when(mongoCollection.aggregate(any())).thenReturn(aggregateIterable); + when(mongoDatabase.getCollection(any())).thenReturn(mongoCollection); + when(mongoDatabase.runCommand(any())).thenReturn(authorizedCollectionsResponse); + when(aggregateIterable.allowDiskUse(anyBoolean())).thenReturn(aggregateIterable); + when(mongoClient.getDatabase(databaseName)).thenReturn(mongoDatabase); + + final List streams = MongoUtil.getAirbyteStreams(mongoClient, databaseName, DEFAULT_DISCOVER_SAMPLE_SIZE, true); + assertNotNull(streams); + assertEquals(1, streams.size()); + assertEquals(12, streams.get(0).getJsonSchema().get(AIRBYTE_STREAM_PROPERTIES).size()); + } + + @Test + void testGetAirbyteStreamsSchemalessMode() throws IOException { + final AggregateIterable aggregateIterable = mock(AggregateIterable.class); + final MongoCursor cursor = mock(MongoCursor.class); + final String databaseName = "database"; + final Document authorizedCollectionsResponse = Document.parse(MoreResources.readResource("authorized_collections_response.json")); + final MongoClient mongoClient = mock(MongoClient.class); + final MongoCollection mongoCollection = mock(MongoCollection.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + final List> schemaDiscoveryJsonResponses = + Jsons.deserialize(MoreResources.readResource("schema_discovery_response_schemaless.json"), new TypeReference<>() {}); + final List schemaDiscoveryResponses = schemaDiscoveryJsonResponses.stream().map(Document::new).toList(); + + when(cursor.hasNext()).thenReturn(true, true, false); + when(cursor.next()).thenReturn(schemaDiscoveryResponses.get(0)); + when(aggregateIterable.cursor()).thenReturn(cursor); + when(mongoCollection.aggregate(any())).thenReturn(aggregateIterable); + when(mongoDatabase.getCollection(any())).thenReturn(mongoCollection); + when(mongoDatabase.runCommand(any())).thenReturn(authorizedCollectionsResponse); + when(aggregateIterable.allowDiskUse(anyBoolean())).thenReturn(aggregateIterable); + when(mongoClient.getDatabase(databaseName)).thenReturn(mongoDatabase); + + final List streams = MongoUtil.getAirbyteStreams(mongoClient, databaseName, DEFAULT_DISCOVER_SAMPLE_SIZE, false); + assertNotNull(streams); + assertEquals(1, streams.size()); + // In schemaless mode, only the 3 CDC fields + id and data fields should exist. + assertEquals(5, streams.get(0).getJsonSchema().get(AIRBYTE_STREAM_PROPERTIES).size()); + + // Test the schema mismatch logic + final List configuredAirbyteStreams = + streams.stream() + .map(stream -> new ConfiguredAirbyteStream().withStream(stream)) + .collect(Collectors.toList()); + final ConfiguredAirbyteCatalog schemaLessCatalog = + new ConfiguredAirbyteCatalog().withStreams(configuredAirbyteStreams); + Throwable throwable = catchThrowable(() -> checkSchemaModeMismatch(true, true, schemaLessCatalog)); + assertThat(throwable).isInstanceOf(ConfigErrorException.class) + .hasMessageContaining(formatMismatchException(true, false, true)); + throwable = catchThrowable(() -> checkSchemaModeMismatch(false, false, schemaLessCatalog)); + assertThat(throwable).isNull(); + } + + @Test + void testGetAirbyteStreamsEmptyCollection() throws IOException { + final AggregateIterable aggregateIterable = mock(AggregateIterable.class); + final MongoCursor cursor = mock(MongoCursor.class); + final String databaseName = "database"; + final Document authorizedCollectionsResponse = Document.parse(MoreResources.readResource("authorized_collections_response.json")); + final MongoClient mongoClient = mock(MongoClient.class); + final MongoCollection mongoCollection = mock(MongoCollection.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + + when(cursor.hasNext()).thenReturn(false); + when(aggregateIterable.cursor()).thenReturn(cursor); + when(mongoCollection.aggregate(any())).thenReturn(aggregateIterable); + when(mongoDatabase.getCollection(any())).thenReturn(mongoCollection); + when(mongoDatabase.runCommand(any())).thenReturn(authorizedCollectionsResponse); + when(mongoClient.getDatabase(databaseName)).thenReturn(mongoDatabase); + when(aggregateIterable.allowDiskUse(anyBoolean())).thenReturn(aggregateIterable); + + final List streams = MongoUtil.getAirbyteStreams(mongoClient, databaseName, DEFAULT_DISCOVER_SAMPLE_SIZE, true); + assertNotNull(streams); + assertEquals(0, streams.size()); + } + + @Test + void testGetAirbyteStreamsDifferentDataTypes() throws IOException { + final AggregateIterable aggregateIterable = mock(AggregateIterable.class); + final MongoCursor cursor = mock(MongoCursor.class); + final String databaseName = "database"; + final Document authorizedCollectionsResponse = Document.parse(MoreResources.readResource("authorized_collections_response.json")); + final MongoClient mongoClient = mock(MongoClient.class); + final MongoCollection mongoCollection = mock(MongoCollection.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + final List> schemaDiscoveryJsonResponses = + Jsons.deserialize(MoreResources.readResource("schema_discovery_response_different_datatypes.json"), new TypeReference<>() {}); + final List schemaDiscoveryResponses = schemaDiscoveryJsonResponses.stream().map(Document::new).toList(); + + when(cursor.hasNext()).thenReturn(true, true, false); + when(cursor.next()).thenReturn(schemaDiscoveryResponses.get(0), schemaDiscoveryResponses.get(1)); + when(aggregateIterable.cursor()).thenReturn(cursor); + when(mongoCollection.aggregate(any())).thenReturn(aggregateIterable); + when(mongoDatabase.getCollection(any())).thenReturn(mongoCollection); + when(mongoDatabase.runCommand(any())).thenReturn(authorizedCollectionsResponse); + when(mongoClient.getDatabase(databaseName)).thenReturn(mongoDatabase); + when(aggregateIterable.allowDiskUse(anyBoolean())).thenReturn(aggregateIterable); + + final List streams = MongoUtil.getAirbyteStreams(mongoClient, databaseName, DEFAULT_DISCOVER_SAMPLE_SIZE, true); + assertNotNull(streams); + assertEquals(1, streams.size()); + assertEquals(11, streams.get(0).getJsonSchema().get(AIRBYTE_STREAM_PROPERTIES).size()); + assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get(JSON_TYPE_PROPERTY_NAME), + streams.get(0).getJsonSchema().get(AIRBYTE_STREAM_PROPERTIES).get("total").get(JSON_TYPE_PROPERTY_NAME).asText()); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get(JSON_TYPE_PROPERTY_NAME), + streams.get(0).getJsonSchema().get(AIRBYTE_STREAM_PROPERTIES).get(CDC_UPDATED_AT).get(JSON_TYPE_PROPERTY_NAME).asText()); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get(JSON_TYPE_PROPERTY_NAME), + streams.get(0).getJsonSchema().get(AIRBYTE_STREAM_PROPERTIES).get(CDC_DELETED_AT).get(JSON_TYPE_PROPERTY_NAME).asText()); + assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get(JSON_TYPE_PROPERTY_NAME), + streams.get(0).getJsonSchema().get(AIRBYTE_STREAM_PROPERTIES).get(MongoDbCdcConnectorMetadataInjector.CDC_DEFAULT_CURSOR) + .get(JSON_TYPE_PROPERTY_NAME).asText()); + + // Test the schema mismatch logic + final List configuredAirbyteStreams = + streams.stream() + .map(stream -> new ConfiguredAirbyteStream().withStream(stream)) + .collect(Collectors.toList()); + final ConfiguredAirbyteCatalog schemaEnforcedCatalog = + new ConfiguredAirbyteCatalog().withStreams(configuredAirbyteStreams); + Throwable throwable = catchThrowable(() -> checkSchemaModeMismatch(false, false, schemaEnforcedCatalog)); + assertThat(throwable).isInstanceOf(ConfigErrorException.class) + .hasMessageContaining(formatMismatchException(false, true, false)); + throwable = catchThrowable(() -> checkSchemaModeMismatch(true, true, schemaEnforcedCatalog)); + assertThat(throwable).isNull(); + } + + @Test + void testonlyStateMismatchError() throws IOException { + final AggregateIterable aggregateIterable = mock(AggregateIterable.class); + final MongoCursor cursor = mock(MongoCursor.class); + final String databaseName = "database"; + final Document authorizedCollectionsResponse = Document.parse(MoreResources.readResource("authorized_collections_response.json")); + final MongoClient mongoClient = mock(MongoClient.class); + final MongoCollection mongoCollection = mock(MongoCollection.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + final List> schemaDiscoveryJsonResponses = + Jsons.deserialize(MoreResources.readResource("schema_discovery_response_different_datatypes.json"), new TypeReference<>() {}); + final List schemaDiscoveryResponses = schemaDiscoveryJsonResponses.stream().map(Document::new).toList(); + + when(cursor.hasNext()).thenReturn(true, true, false); + when(cursor.next()).thenReturn(schemaDiscoveryResponses.get(0), schemaDiscoveryResponses.get(1)); + when(aggregateIterable.cursor()).thenReturn(cursor); + when(mongoCollection.aggregate(any())).thenReturn(aggregateIterable); + when(mongoDatabase.getCollection(any())).thenReturn(mongoCollection); + when(mongoDatabase.runCommand(any())).thenReturn(authorizedCollectionsResponse); + when(mongoClient.getDatabase(databaseName)).thenReturn(mongoDatabase); + when(aggregateIterable.allowDiskUse(anyBoolean())).thenReturn(aggregateIterable); + + final List streams = MongoUtil.getAirbyteStreams(mongoClient, databaseName, DEFAULT_DISCOVER_SAMPLE_SIZE, true); + assertNotNull(streams); + assertEquals(1, streams.size()); + assertEquals(11, streams.get(0).getJsonSchema().get(AIRBYTE_STREAM_PROPERTIES).size()); + assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get(JSON_TYPE_PROPERTY_NAME), + streams.get(0).getJsonSchema().get(AIRBYTE_STREAM_PROPERTIES).get("total").get(JSON_TYPE_PROPERTY_NAME).asText()); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get(JSON_TYPE_PROPERTY_NAME), + streams.get(0).getJsonSchema().get(AIRBYTE_STREAM_PROPERTIES).get(CDC_UPDATED_AT).get(JSON_TYPE_PROPERTY_NAME).asText()); + assertEquals(JsonSchemaType.STRING.getJsonSchemaTypeMap().get(JSON_TYPE_PROPERTY_NAME), + streams.get(0).getJsonSchema().get(AIRBYTE_STREAM_PROPERTIES).get(CDC_DELETED_AT).get(JSON_TYPE_PROPERTY_NAME).asText()); + assertEquals(JsonSchemaType.NUMBER.getJsonSchemaTypeMap().get(JSON_TYPE_PROPERTY_NAME), + streams.get(0).getJsonSchema().get(AIRBYTE_STREAM_PROPERTIES).get(MongoDbCdcConnectorMetadataInjector.CDC_DEFAULT_CURSOR) + .get(JSON_TYPE_PROPERTY_NAME).asText()); + + // Test the schema mismatch logic + final List configuredAirbyteStreams = + streams.stream() + .map(stream -> new ConfiguredAirbyteStream().withStream(stream)) + .collect(Collectors.toList()); + final ConfiguredAirbyteCatalog schemaEnforcedCatalog = + new ConfiguredAirbyteCatalog().withStreams(configuredAirbyteStreams); + Throwable throwable = catchThrowable(() -> checkSchemaModeMismatch(true, false, schemaEnforcedCatalog)); + assertThat(throwable).isInstanceOf(ConfigErrorException.class) + .hasMessageContaining(formatMismatchException(true, true, false)); + throwable = catchThrowable(() -> checkSchemaModeMismatch(true, true, schemaEnforcedCatalog)); + assertThat(throwable).isNull(); + } + + @Test + void testGetAuthorizedCollections() { + final String databaseName = "test-database"; + final String collectionName = "test-collection"; + final Document result = new Document(Map.of("cursor", Map.of("firstBatch", List.of(Map.of("name", collectionName))))); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + final MongoClient mongoClient = mock(MongoClient.class); + + when(mongoDatabase.runCommand(any())).thenReturn(result); + when(mongoClient.getDatabase(databaseName)).thenReturn(mongoDatabase); + + final Set authorizedCollections = MongoUtil.getAuthorizedCollections(mongoClient, databaseName); + + assertEquals(Set.of(collectionName), authorizedCollections); + } + + @Test + void testGetAuthorizedCollectionsMongoException() { + final String databaseName = "test-database"; + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + final MongoClient mongoClient = mock(MongoClient.class); + + when(mongoDatabase.runCommand(any())).thenThrow(new MongoException("test")); + when(mongoClient.getDatabase(databaseName)).thenReturn(mongoDatabase); + + assertThrows(MongoException.class, () -> MongoUtil.getAuthorizedCollections(mongoClient, databaseName)); + } + + @Test + void testGetAuthorizedCollectionsMongoSecurityException() { + final String databaseName = "test-database"; + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + final MongoClient mongoClient = mock(MongoClient.class); + final MongoCommandException cause = new MongoCommandException(new BsonDocument(), new ServerAddress()); + final MongoSecurityException exception = + new MongoSecurityException(MongoCredential.createCredential("username", databaseName, "password".toCharArray()), "test", cause); + + when(mongoDatabase.runCommand(any())).thenThrow(exception); + when(mongoClient.getDatabase(databaseName)).thenReturn(mongoDatabase); + + assertThrows(MongoSecurityException.class, () -> MongoUtil.getAuthorizedCollections(mongoClient, databaseName)); + } + + @Test + void testGetDebeziumEventQueueSize() { + final int queueSize = 5000; + final MongoDbSourceConfig validQueueSizeConfiguration = new MongoDbSourceConfig( + Jsons.jsonNode(Map.of(DATABASE_CONFIG_CONFIGURATION_KEY, Map.of(MongoConstants.QUEUE_SIZE_CONFIGURATION_KEY, queueSize)))); + final MongoDbSourceConfig tooSmallQueueSizeConfiguration = new MongoDbSourceConfig( + Jsons.jsonNode(Map.of(DATABASE_CONFIG_CONFIGURATION_KEY, Map.of(MongoConstants.QUEUE_SIZE_CONFIGURATION_KEY, Integer.MIN_VALUE)))); + final MongoDbSourceConfig tooLargeQueueSizeConfiguration = new MongoDbSourceConfig( + Jsons.jsonNode(Map.of(DATABASE_CONFIG_CONFIGURATION_KEY, Map.of(MongoConstants.QUEUE_SIZE_CONFIGURATION_KEY, Integer.MAX_VALUE)))); + final MongoDbSourceConfig missingQueueSizeConfiguration = + new MongoDbSourceConfig(Jsons.jsonNode(Map.of(DATABASE_CONFIG_CONFIGURATION_KEY, Map.of()))); + + assertEquals(queueSize, MongoUtil.getDebeziumEventQueueSize(validQueueSizeConfiguration).getAsInt()); + assertEquals(MIN_QUEUE_SIZE, MongoUtil.getDebeziumEventQueueSize(tooSmallQueueSizeConfiguration).getAsInt()); + assertEquals(MAX_QUEUE_SIZE, MongoUtil.getDebeziumEventQueueSize(tooLargeQueueSizeConfiguration).getAsInt()); + assertEquals(MAX_QUEUE_SIZE, MongoUtil.getDebeziumEventQueueSize(missingQueueSizeConfiguration).getAsInt()); + } + + @Test + void testGetCollectionStatistics() throws IOException { + final String collectionName = "test-collection"; + final String databaseName = "test-database"; + final String collStats = MoreResources.readResource("coll_stats_response.json"); + final List> collStatsList = Jsons.deserialize(collStats, new TypeReference<>() {}); + final MongoCursor cursor = mock(MongoCursor.class); + final AggregateIterable aggregateIterable = mock(AggregateIterable.class); + final MongoCollection mongoCollection = mock(MongoCollection.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + final MongoClient mongoClient = mock(MongoClient.class); + + final AirbyteStream stream = new AirbyteStream().withName(collectionName).withNamespace(databaseName); + final ConfiguredAirbyteStream configuredAirbyteStream = new ConfiguredAirbyteStream().withStream(stream); + + when(cursor.hasNext()).thenReturn(true); + when(cursor.next()).thenReturn(new Document(collStatsList.get(0))); + when(aggregateIterable.cursor()).thenReturn(cursor); + when(mongoCollection.aggregate(any())).thenReturn(aggregateIterable); + when(mongoDatabase.getCollection(collectionName)).thenReturn(mongoCollection); + when(mongoClient.getDatabase(databaseName)).thenReturn(mongoDatabase); + when(aggregateIterable.allowDiskUse(anyBoolean())).thenReturn(aggregateIterable); + + final Optional statistics = MongoUtil.getCollectionStatistics(mongoClient, configuredAirbyteStream); + + assertTrue(statistics.isPresent()); + assertEquals(746, statistics.get().count()); + assertEquals(67771, statistics.get().size()); + } + + @Test + void testGetCollectionStatisticsNoResult() { + final String collectionName = "test-collection"; + final String databaseName = "test-database"; + final MongoCursor cursor = mock(MongoCursor.class); + final AggregateIterable aggregateIterable = mock(AggregateIterable.class); + final MongoCollection mongoCollection = mock(MongoCollection.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + final MongoClient mongoClient = mock(MongoClient.class); + + final AirbyteStream stream = new AirbyteStream().withName(collectionName).withNamespace(databaseName); + final ConfiguredAirbyteStream configuredAirbyteStream = new ConfiguredAirbyteStream().withStream(stream); + + when(cursor.hasNext()).thenReturn(false); + when(aggregateIterable.cursor()).thenReturn(cursor); + when(mongoCollection.aggregate(any())).thenReturn(aggregateIterable); + when(mongoDatabase.getCollection(collectionName)).thenReturn(mongoCollection); + when(mongoClient.getDatabase(databaseName)).thenReturn(mongoDatabase); + + final Optional statistics = MongoUtil.getCollectionStatistics(mongoClient, configuredAirbyteStream); + + assertFalse(statistics.isPresent()); + } + + @Test + void testGetCollectionStatisticsEmptyResult() { + final String collectionName = "test-collection"; + final String databaseName = "test-database"; + final List> collStatsList = List.of(Map.of()); + final MongoCursor cursor = mock(MongoCursor.class); + final AggregateIterable aggregateIterable = mock(AggregateIterable.class); + final MongoCollection mongoCollection = mock(MongoCollection.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + final MongoClient mongoClient = mock(MongoClient.class); + + final AirbyteStream stream = new AirbyteStream().withName(collectionName).withNamespace(databaseName); + final ConfiguredAirbyteStream configuredAirbyteStream = new ConfiguredAirbyteStream().withStream(stream); + + when(cursor.hasNext()).thenReturn(true); + when(cursor.next()).thenReturn(new Document(collStatsList.get(0))); + when(aggregateIterable.cursor()).thenReturn(cursor); + when(mongoCollection.aggregate(any())).thenReturn(aggregateIterable); + when(mongoDatabase.getCollection(collectionName)).thenReturn(mongoCollection); + when(mongoClient.getDatabase(databaseName)).thenReturn(mongoDatabase); + + final Optional statistics = MongoUtil.getCollectionStatistics(mongoClient, configuredAirbyteStream); + + assertFalse(statistics.isPresent()); + } + + @Test + void testGetCollectionStatisticsException() { + final String collectionName = "test-collection"; + final String databaseName = "test-database"; + final MongoClient mongoClient = mock(MongoClient.class); + + final AirbyteStream stream = new AirbyteStream().withName(collectionName).withNamespace(databaseName); + final ConfiguredAirbyteStream configuredAirbyteStream = new ConfiguredAirbyteStream().withStream(stream); + + when(mongoClient.getDatabase(databaseName)).thenThrow(new IllegalArgumentException("test")); + + final Optional statistics = MongoUtil.getCollectionStatistics(mongoClient, configuredAirbyteStream); + + assertFalse(statistics.isPresent()); + + } + + private static String formatMismatchException(final boolean isConfigSchemaEnforced, + final boolean isCatalogSchemaEnforcing, + final boolean isStateSchemaEnforced) { + final String remedy = isConfigSchemaEnforced == isCatalogSchemaEnforcing + ? "Please reset your data." + : "Please refresh source schema and reset streams."; + return "Mismatch between schema enforcing mode in sync configuration (%b), catalog (%b) and saved state (%b). " + .formatted(isConfigSchemaEnforced, isCatalogSchemaEnforcing, isStateSchemaEnforced) + + remedy; + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcConnectorMetadataInjectorTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcConnectorMetadataInjectorTest.java new file mode 100644 index 000000000000..78be3791db63 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcConnectorMetadataInjectorTest.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.cdc; + +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants; +import io.airbyte.commons.json.Jsons; +import java.lang.reflect.Field; +import java.time.Instant; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.bson.BsonTimestamp; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class MongoDbCdcConnectorMetadataInjectorTest { + + @BeforeEach + public void reset() throws NoSuchFieldException, IllegalAccessException { + Field instance = MongoDbCdcConnectorMetadataInjector.class.getDeclaredField("mongoDbCdcConnectorMetadataInjector"); + instance.setAccessible(true); + instance.set(null, null); + } + + @Test + void testAddingMetadata() { + final Instant emittedAt = Instant.now(); + final BsonTimestamp expected = new BsonTimestamp( + Long.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis())).intValue(), + 1); + final ObjectNode event = Jsons.emptyObject().withObject(""); + final Map sourceData = Map.of( + MongoDbDebeziumConstants.ChangeEvent.SOURCE_COLLECTION, "test-collection", + MongoDbDebeziumConstants.ChangeEvent.SOURCE_DB, "test-database", + MongoDbDebeziumConstants.ChangeEvent.SOURCE_TIMESTAMP_MS, TimeUnit.SECONDS.toMillis(expected.getTime()), + MongoDbDebeziumConstants.ChangeEvent.SOURCE_ORDER, expected.getInc()); + + final MongoDbCdcConnectorMetadataInjector metadataInjector = MongoDbCdcConnectorMetadataInjector.getInstance(emittedAt); + metadataInjector.addMetaData(event, Jsons.jsonNode(sourceData)); + + assertEquals((emittedAt.getEpochSecond() * 100_000_000) + 1L, event.get(MongoDbCdcConnectorMetadataInjector.CDC_DEFAULT_CURSOR).asLong()); + } + + @Test + void testAddingMetadataToRowsFetchedOutsideDebezium() { + final Instant emittedAt = Instant.now(); + final BsonTimestamp expected = new BsonTimestamp( + Long.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis())).intValue(), + 1); + final String transactionTimestamp = Instant.now().toString(); + final ObjectNode record = Jsons.emptyObject().withObject(""); + + final MongoDbCdcConnectorMetadataInjector metadataInjector = MongoDbCdcConnectorMetadataInjector.getInstance(emittedAt); + metadataInjector.addMetaDataToRowsFetchedOutsideDebezium(record, transactionTimestamp, expected); + + assertEquals(transactionTimestamp, record.get(CDC_UPDATED_AT).asText()); + assertEquals("null", record.get(CDC_DELETED_AT).asText()); + assertEquals((emittedAt.getEpochSecond() * 100_000_000) + 1L, record.get(MongoDbCdcConnectorMetadataInjector.CDC_DEFAULT_CURSOR).asLong()); + } + + @Test + void testGetNamespaceFromSource() { + final Instant emittedAt = Instant.now(); + final String databaseName = "test-database"; + final BsonTimestamp expected = new BsonTimestamp( + Long.valueOf(TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis())).intValue(), + 1); + final Map sourceData = Map.of( + MongoDbDebeziumConstants.ChangeEvent.SOURCE_COLLECTION, "test-collection", + MongoDbDebeziumConstants.ChangeEvent.SOURCE_DB, databaseName, + MongoDbDebeziumConstants.ChangeEvent.SOURCE_TIMESTAMP_MS, TimeUnit.SECONDS.toMillis(expected.getTime()), + MongoDbDebeziumConstants.ChangeEvent.SOURCE_ORDER, expected.getInc()); + + final MongoDbCdcConnectorMetadataInjector metadataInjector = MongoDbCdcConnectorMetadataInjector.getInstance(emittedAt); + + assertEquals(databaseName, metadataInjector.namespace(Jsons.jsonNode(sourceData))); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitialSnapshotUtilsTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitialSnapshotUtilsTest.java new file mode 100644 index 000000000000..e4e9e09a1b5f --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitialSnapshotUtilsTest.java @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.cdc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.mongodb.client.AggregateIterable; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoCursor; +import com.mongodb.client.MongoDatabase; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.integrations.source.mongodb.MongoConstants; +import io.airbyte.integrations.source.mongodb.state.IdType; +import io.airbyte.integrations.source.mongodb.state.InitialSnapshotStatus; +import io.airbyte.integrations.source.mongodb.state.MongoDbStateManager; +import io.airbyte.integrations.source.mongodb.state.MongoDbStreamState; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.SyncMode; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import org.bson.Document; +import org.junit.jupiter.api.Test; + +class MongoDbCdcInitialSnapshotUtilsTest { + + private static final String NAMESPACE = "namespace"; + private static final String COMPLETED_NAME = "completed"; + private static final String IN_PROGRESS_NAME = "in_progress"; + private static final String NEW_NAME = "new"; + + @Test + void testRetrieveInitialSnapshotIterators() throws IOException { + final String collStats = MoreResources.readResource("coll_stats_response.json"); + final List> collStatsList = Jsons.deserialize(collStats, new TypeReference<>() {}); + final MongoDbStateManager stateManager = mock(MongoDbStateManager.class); + final MongoCursor cursor = mock(MongoCursor.class); + final AggregateIterable aggregateIterable = mock(AggregateIterable.class); + final MongoCollection mongoCollection = mock(MongoCollection.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + final MongoClient mongoClient = mock(MongoClient.class); + final ConfiguredAirbyteStream completedStream = createConfiguredAirbyteStream(COMPLETED_NAME, NAMESPACE); + final ConfiguredAirbyteStream inProgressStream = createConfiguredAirbyteStream(IN_PROGRESS_NAME, NAMESPACE); + final ConfiguredAirbyteStream newStream = createConfiguredAirbyteStream(NEW_NAME, NAMESPACE); + final List configuredStreams = List.of(completedStream, inProgressStream, newStream); + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(configuredStreams); + final boolean savedOffsetIsValid = true; + + when(stateManager.getStreamStates()).thenReturn(Map.of( + new AirbyteStreamNameNamespacePair(COMPLETED_NAME, NAMESPACE), new MongoDbStreamState("1", InitialSnapshotStatus.COMPLETE, IdType.OBJECT_ID), + new AirbyteStreamNameNamespacePair(IN_PROGRESS_NAME, NAMESPACE), + new MongoDbStreamState("2", InitialSnapshotStatus.IN_PROGRESS, IdType.OBJECT_ID))); + when(cursor.hasNext()).thenReturn(true); + when(cursor.next()).thenReturn(new Document(collStatsList.get(0))); + when(aggregateIterable.cursor()).thenReturn(cursor); + when(mongoCollection.aggregate(any())).thenReturn(aggregateIterable); + when(mongoDatabase.getCollection(NEW_NAME)).thenReturn(mongoCollection); + when(mongoClient.getDatabase(NAMESPACE)).thenReturn(mongoDatabase); + + final List initialSnapshotStreams = + MongoDbCdcInitialSnapshotUtils.getStreamsForInitialSnapshot(mongoClient, stateManager, catalog, savedOffsetIsValid); + assertEquals(2, initialSnapshotStreams.size()); + assertTrue(initialSnapshotStreams.contains(inProgressStream)); + assertTrue(initialSnapshotStreams.contains(newStream)); + } + + @Test + void testRetrieveInitialSnapshotIteratorsInvalidSavedOffset() { + final MongoDbStateManager stateManager = mock(MongoDbStateManager.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + final MongoClient mongoClient = mock(MongoClient.class); + final ConfiguredAirbyteStream completedStream = createConfiguredAirbyteStream(COMPLETED_NAME, NAMESPACE); + final ConfiguredAirbyteStream inProgressStream = createConfiguredAirbyteStream(IN_PROGRESS_NAME, NAMESPACE); + final ConfiguredAirbyteStream newStream = createConfiguredAirbyteStream(NEW_NAME, NAMESPACE); + final List configuredStreams = List.of(completedStream, inProgressStream, newStream); + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(configuredStreams); + final boolean savedOffsetIsValid = false; + + when(mongoDatabase.runCommand(any())) + .thenReturn(new Document( + Map.of(MongoConstants.COLLECTION_STATISTICS_STORAGE_SIZE_KEY, 1000000L, MongoConstants.COLLECTION_STATISTICS_COUNT_KEY, 10000))); + when(mongoClient.getDatabase(NAMESPACE)).thenReturn(mongoDatabase); + + final List initialSnapshotStreams = + MongoDbCdcInitialSnapshotUtils.getStreamsForInitialSnapshot(mongoClient, stateManager, catalog, savedOffsetIsValid); + + assertEquals(3, initialSnapshotStreams.size()); + assertTrue(initialSnapshotStreams.contains(completedStream)); + assertTrue(initialSnapshotStreams.contains(inProgressStream)); + assertTrue(initialSnapshotStreams.contains(newStream)); + } + + @Test + void testFailureToGenerateEstimateDoesNotImpactSync() { + final MongoDbStateManager stateManager = mock(MongoDbStateManager.class); + final MongoClient mongoClient = mock(MongoClient.class); + final ConfiguredAirbyteStream completedStream = createConfiguredAirbyteStream(COMPLETED_NAME, NAMESPACE); + final ConfiguredAirbyteStream inProgressStream = createConfiguredAirbyteStream(IN_PROGRESS_NAME, NAMESPACE); + final ConfiguredAirbyteStream newStream = createConfiguredAirbyteStream(NEW_NAME, NAMESPACE); + final List configuredStreams = List.of(completedStream, inProgressStream, newStream); + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(configuredStreams); + final boolean savedOffsetIsValid = true; + + when(stateManager.getStreamStates()).thenReturn(Map.of( + new AirbyteStreamNameNamespacePair(COMPLETED_NAME, NAMESPACE), new MongoDbStreamState("1", InitialSnapshotStatus.COMPLETE, IdType.OBJECT_ID), + new AirbyteStreamNameNamespacePair(IN_PROGRESS_NAME, NAMESPACE), + new MongoDbStreamState("2", InitialSnapshotStatus.IN_PROGRESS, IdType.OBJECT_ID))); + when(mongoClient.getDatabase(NAMESPACE)).thenThrow(new IllegalArgumentException("test")); + + final List initialSnapshotStreams = + MongoDbCdcInitialSnapshotUtils.getStreamsForInitialSnapshot(mongoClient, stateManager, catalog, savedOffsetIsValid); + assertEquals(2, initialSnapshotStreams.size()); + assertTrue(initialSnapshotStreams.contains(inProgressStream)); + assertTrue(initialSnapshotStreams.contains(newStream)); + } + + @Test + void testMissingCollectionStatisticsDoNotImpactSync() { + final MongoDbStateManager stateManager = mock(MongoDbStateManager.class); + final MongoClient mongoClient = mock(MongoClient.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + final ConfiguredAirbyteStream completedStream = createConfiguredAirbyteStream(COMPLETED_NAME, NAMESPACE); + final ConfiguredAirbyteStream inProgressStream = createConfiguredAirbyteStream(IN_PROGRESS_NAME, NAMESPACE); + final ConfiguredAirbyteStream newStream = createConfiguredAirbyteStream(NEW_NAME, NAMESPACE); + final List configuredStreams = List.of(completedStream, inProgressStream, newStream); + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(configuredStreams); + final boolean savedOffsetIsValid = true; + + when(stateManager.getStreamStates()).thenReturn(Map.of( + new AirbyteStreamNameNamespacePair(COMPLETED_NAME, NAMESPACE), new MongoDbStreamState("1", InitialSnapshotStatus.COMPLETE, IdType.OBJECT_ID), + new AirbyteStreamNameNamespacePair(IN_PROGRESS_NAME, NAMESPACE), + new MongoDbStreamState("2", InitialSnapshotStatus.IN_PROGRESS, IdType.OBJECT_ID))); + when(mongoClient.getDatabase(NAMESPACE)).thenReturn(mongoDatabase); + + final List initialSnapshotStreams = + MongoDbCdcInitialSnapshotUtils.getStreamsForInitialSnapshot(mongoClient, stateManager, catalog, savedOffsetIsValid); + assertEquals(2, initialSnapshotStreams.size()); + assertTrue(initialSnapshotStreams.contains(inProgressStream)); + assertTrue(initialSnapshotStreams.contains(newStream)); + } + + private AirbyteStream createAirbyteStream(final String name, final String namespace) { + return new AirbyteStream().withName(name).withNamespace(namespace); + } + + private ConfiguredAirbyteStream createConfiguredAirbyteStream(final String name, final String namespace) { + return new ConfiguredAirbyteStream() + .withStream(createAirbyteStream(name, namespace)) + .withSyncMode(SyncMode.INCREMENTAL); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitializerTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitializerTest.java new file mode 100644 index 000000000000..f65d0882adf4 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcInitializerTest.java @@ -0,0 +1,282 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.cdc; + +import static io.airbyte.integrations.source.mongodb.MongoConstants.DATABASE_CONFIG_CONFIGURATION_KEY; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.Mockito.doCallRealMethod; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.JsonNode; +import com.mongodb.MongoCommandException; +import com.mongodb.ServerAddress; +import com.mongodb.client.AggregateIterable; +import com.mongodb.client.ChangeStreamIterable; +import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoChangeStreamCursor; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoCursor; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.changestream.ChangeStreamDocument; +import com.mongodb.connection.ClusterDescription; +import com.mongodb.connection.ClusterType; +import com.mongodb.connection.ServerDescription; +import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants; +import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumStateUtil; +import io.airbyte.commons.exceptions.ConfigErrorException; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.integrations.source.mongodb.MongoDbSourceConfig; +import io.airbyte.integrations.source.mongodb.state.IdType; +import io.airbyte.integrations.source.mongodb.state.InitialSnapshotStatus; +import io.airbyte.integrations.source.mongodb.state.MongoDbStateManager; +import io.airbyte.integrations.source.mongodb.state.MongoDbStreamState; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteCatalog; +import io.airbyte.protocol.models.v0.AirbyteGlobalState; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.AirbyteStreamState; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import io.airbyte.protocol.models.v0.SyncMode; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Consumer; +import org.bson.BsonDocument; +import org.bson.BsonString; +import org.bson.Document; +import org.bson.types.ObjectId; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class MongoDbCdcInitializerTest { + + private static final String COLLECTION = "test-collection"; + private static final String DATABASE = "test-database"; + private static final String ID = "64c0029d95ad260d69ef28a0"; + private static final String REPLICA_SET = "test-replica-set"; + private static final String RESUME_TOKEN1 = "8264BEB9F3000000012B0229296E04"; + private static final String STREAM_NAME = COLLECTION; + private static final String STREAM_NAMESPACE = DATABASE; + + private static final AirbyteCatalog CATALOG = new AirbyteCatalog().withStreams(List.of( + CatalogHelpers.createAirbyteStream( + COLLECTION, + DATABASE, + Field.of("id", JsonSchemaType.INTEGER), + Field.of("string", JsonSchemaType.STRING)) + .withSupportedSyncModes(List.of(SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of("_id"))))); + protected static final ConfiguredAirbyteCatalog CONFIGURED_CATALOG = toConfiguredCatalog(CATALOG); + + final MongoDbSourceConfig CONFIG = new MongoDbSourceConfig(Jsons.jsonNode( + Map.of(DATABASE_CONFIG_CONFIGURATION_KEY, + Map.of( + MongoDbDebeziumConstants.Configuration.CONNECTION_STRING_CONFIGURATION_KEY, "mongodb://host:12345/", + MongoDbDebeziumConstants.Configuration.DATABASE_CONFIGURATION_KEY, DATABASE)))); + + final Instant EMITTED_AT = Instant.now(); + + private MongoDbCdcInitializer cdcInitializer; + private MongoDbDebeziumStateUtil mongoDbDebeziumStateUtil; + private MongoClient mongoClient; + private MongoChangeStreamCursor> mongoChangeStreamCursor; + private AggregateIterable aggregateIterable; + private MongoCursor aggregateCursor; + private MongoCursor findCursor; + private ChangeStreamIterable changeStreamIterable; + private MongoDbCdcConnectorMetadataInjector cdcConnectorMetadataInjector; + + @BeforeEach + void setUp() { + final BsonDocument resumeTokenDocument = new BsonDocument("_data", new BsonString(RESUME_TOKEN1)); + final Document aggregate = Document.parse("{\"_id\": {\"_id\": \"objectId\"}, \"count\": 1}"); + + changeStreamIterable = mock(ChangeStreamIterable.class); + mongoChangeStreamCursor = + mock(MongoChangeStreamCursor.class); + mongoClient = mock(MongoClient.class); + final MongoDatabase mongoDatabase = mock(MongoDatabase.class); + final MongoCollection mongoCollection = mock(MongoCollection.class); + final FindIterable findIterable = mock(FindIterable.class); + findCursor = mock(MongoCursor.class); + final ServerDescription serverDescription = mock(ServerDescription.class); + final ClusterDescription clusterDescription = mock(ClusterDescription.class); + aggregateIterable = mock(AggregateIterable.class); + aggregateCursor = mock(MongoCursor.class); + cdcConnectorMetadataInjector = mock(MongoDbCdcConnectorMetadataInjector.class); + + when(mongoChangeStreamCursor.getResumeToken()).thenReturn(resumeTokenDocument); + when(changeStreamIterable.cursor()).thenReturn(mongoChangeStreamCursor); + when(serverDescription.getSetName()).thenReturn(REPLICA_SET); + when(clusterDescription.getServerDescriptions()).thenReturn(List.of(serverDescription)); + when(clusterDescription.getType()).thenReturn(ClusterType.REPLICA_SET); + when(mongoClient.watch(BsonDocument.class)).thenReturn(changeStreamIterable); + when(mongoClient.getDatabase(DATABASE)).thenReturn(mongoDatabase); + when(mongoClient.getClusterDescription()).thenReturn(clusterDescription); + when(mongoDatabase.getCollection(COLLECTION)).thenReturn(mongoCollection); + when(mongoCollection.aggregate(anyList())).thenReturn(aggregateIterable); + when(aggregateIterable.iterator()).thenReturn(aggregateCursor); + when(aggregateCursor.hasNext()).thenReturn(true, false); + when(aggregateCursor.next()).thenReturn(aggregate); + doCallRealMethod().when(aggregateIterable).forEach(any(Consumer.class)); + when(mongoCollection.find()).thenReturn(findIterable); + when(findIterable.filter(any())).thenReturn(findIterable); + when(findIterable.projection(any())).thenReturn(findIterable); + when(findIterable.sort(any())).thenReturn(findIterable); + when(findIterable.cursor()).thenReturn(findCursor); + when(findCursor.hasNext()).thenReturn(true); + when(findCursor.next()).thenReturn(new Document("_id", new ObjectId(ID))); + when(findIterable.allowDiskUse(anyBoolean())).thenReturn(findIterable); + + mongoDbDebeziumStateUtil = spy(new MongoDbDebeziumStateUtil()); + cdcInitializer = new MongoDbCdcInitializer(mongoDbDebeziumStateUtil); + } + + @Test + void testCreateCdcIteratorsEmptyInitialState() { + final MongoDbStateManager stateManager = MongoDbStateManager.createStateManager(null); + final List> iterators = cdcInitializer + .createCdcIterators(mongoClient, cdcConnectorMetadataInjector, CONFIGURED_CATALOG, stateManager, EMITTED_AT, CONFIG); + assertNotNull(iterators); + assertEquals(2, iterators.size(), "Should always have 2 iterators: 1 for the initial snapshot and 1 for the cdc stream"); + assertTrue(iterators.get(0).hasNext(), + "Initial snapshot iterator should at least have one message if there's no initial snapshot state and collections are not empty"); + } + + @Test + void testCreateCdcIteratorsEmptyInitialStateEmptyCollections() { + when(findCursor.hasNext()).thenReturn(false); + final MongoDbStateManager stateManager = MongoDbStateManager.createStateManager(null); + final List> iterators = cdcInitializer + .createCdcIterators(mongoClient, cdcConnectorMetadataInjector, CONFIGURED_CATALOG, stateManager, EMITTED_AT, CONFIG); + assertNotNull(iterators); + assertEquals(2, iterators.size(), "Should always have 2 iterators: 1 for the initial snapshot and 1 for the cdc stream"); + assertFalse(iterators.get(0).hasNext(), + "Initial snapshot iterator should have no messages if there's no initial snapshot state and collections are empty"); + } + + @Test + void testCreateCdcIteratorsFromInitialStateWithInProgressInitialSnapshot() { + final MongoDbStateManager stateManager = MongoDbStateManager.createStateManager(createInitialDebeziumState(InitialSnapshotStatus.IN_PROGRESS)); + final List> iterators = cdcInitializer + .createCdcIterators(mongoClient, cdcConnectorMetadataInjector, CONFIGURED_CATALOG, stateManager, EMITTED_AT, CONFIG); + assertNotNull(iterators); + assertEquals(2, iterators.size(), "Should always have 2 iterators: 1 for the initial snapshot and 1 for the cdc stream"); + assertTrue(iterators.get(0).hasNext(), + "Initial snapshot iterator should at least have one message if the initial snapshot state is set as in progress"); + } + + @Test + void testCreateCdcIteratorsFromInitialStateWithCompletedInitialSnapshot() { + final MongoDbStateManager stateManager = MongoDbStateManager.createStateManager(createInitialDebeziumState(InitialSnapshotStatus.COMPLETE)); + final List> iterators = cdcInitializer + .createCdcIterators(mongoClient, cdcConnectorMetadataInjector, CONFIGURED_CATALOG, stateManager, EMITTED_AT, CONFIG); + assertNotNull(iterators); + assertEquals(2, iterators.size(), "Should always have 2 iterators: 1 for the initial snapshot and 1 for the cdc stream"); + assertFalse(iterators.get(0).hasNext(), "Initial snapshot iterator should have no messages if its snapshot state is set as complete"); + } + + @Test + void testCreateCdcIteratorsWithCompletedInitialSnapshotSavedOffsetInvalid() { + when(changeStreamIterable.cursor()) + .thenReturn(mongoChangeStreamCursor) + .thenThrow(new MongoCommandException(new BsonDocument(), new ServerAddress())) + .thenReturn(mongoChangeStreamCursor); + final MongoDbStateManager stateManager = MongoDbStateManager.createStateManager(createInitialDebeziumState(InitialSnapshotStatus.COMPLETE)); + final List> iterators = cdcInitializer + .createCdcIterators(mongoClient, cdcConnectorMetadataInjector, CONFIGURED_CATALOG, stateManager, EMITTED_AT, CONFIG); + assertNotNull(iterators); + assertEquals(2, iterators.size(), "Should always have 2 iterators: 1 for the initial snapshot and 1 for the cdc stream"); + assertTrue(iterators.get(0).hasNext(), + "Initial snapshot iterator should at least have one message if its snapshot state is set as complete but needs to start over due to invalid saved offset"); + } + + @Test + void testUnableToExtractOffsetFromStateException() { + final MongoDbStateManager stateManager = MongoDbStateManager.createStateManager(createInitialDebeziumState(InitialSnapshotStatus.COMPLETE)); + doReturn(Optional.empty()).when(mongoDbDebeziumStateUtil).savedOffset(any(), any(), any(), any(), any()); + assertThrows(RuntimeException.class, + () -> cdcInitializer.createCdcIterators(mongoClient, cdcConnectorMetadataInjector, CONFIGURED_CATALOG, stateManager, EMITTED_AT, CONFIG)); + } + + @Test + void testMultipleIdTypesThrowsException() { + final Document aggregate1 = Document.parse("{\"_id\": {\"_id\": \"objectId\"}, \"count\": 1}"); + final Document aggregate2 = Document.parse("{\"_id\": {\"_id\": \"string\"}, \"count\": 1}"); + + when(aggregateCursor.hasNext()).thenReturn(true, true, false); + when(aggregateCursor.next()).thenReturn(aggregate1, aggregate2); + doCallRealMethod().when(aggregateIterable).forEach(any(Consumer.class)); + + final MongoDbStateManager stateManager = MongoDbStateManager.createStateManager(createInitialDebeziumState(InitialSnapshotStatus.IN_PROGRESS)); + + final var thrown = assertThrows(ConfigErrorException.class, () -> cdcInitializer + .createCdcIterators(mongoClient, cdcConnectorMetadataInjector, CONFIGURED_CATALOG, stateManager, EMITTED_AT, CONFIG)); + assertTrue(thrown.getMessage().contains("must be consistently typed")); + } + + @Test + void testUnsupportedIdTypeThrowsException() { + final Document aggregate = Document.parse("{\"_id\": {\"_id\": \"exotic\"}, \"count\": 1}"); + + when(aggregateCursor.hasNext()).thenReturn(true, false); + when(aggregateCursor.next()).thenReturn(aggregate); + doCallRealMethod().when(aggregateIterable).forEach(any(Consumer.class)); + + final MongoDbStateManager stateManager = MongoDbStateManager.createStateManager(null); + + final var thrown = assertThrows(ConfigErrorException.class, () -> cdcInitializer + .createCdcIterators(mongoClient, cdcConnectorMetadataInjector, CONFIGURED_CATALOG, stateManager, EMITTED_AT, CONFIG)); + assertTrue(thrown.getMessage().contains("_id fields with the following types are currently supported")); + } + + private static JsonNode createInitialDebeziumState(final InitialSnapshotStatus initialSnapshotStatus) { + final StreamDescriptor streamDescriptor = new StreamDescriptor().withNamespace(STREAM_NAMESPACE).withName(STREAM_NAME); + final MongoDbCdcState cdcState = new MongoDbCdcState(MongoDbDebeziumStateUtil.formatState(DATABASE, REPLICA_SET, RESUME_TOKEN1)); + final MongoDbStreamState mongoDbStreamState = new MongoDbStreamState(ID, initialSnapshotStatus, IdType.OBJECT_ID); + final JsonNode sharedState = Jsons.jsonNode(cdcState); + final JsonNode streamState = Jsons.jsonNode(mongoDbStreamState); + final AirbyteStreamState airbyteStreamState = new AirbyteStreamState().withStreamDescriptor(streamDescriptor).withStreamState(streamState); + final AirbyteGlobalState airbyteGlobalState = new AirbyteGlobalState().withSharedState(sharedState).withStreamStates(List.of(airbyteStreamState)); + final AirbyteStateMessage airbyteStateMessage = + new AirbyteStateMessage().withType(AirbyteStateMessage.AirbyteStateType.GLOBAL).withGlobal(airbyteGlobalState); + return Jsons.jsonNode(List.of(airbyteStateMessage)); + } + + public static ConfiguredAirbyteCatalog toConfiguredCatalog(final AirbyteCatalog catalog) { + return (new ConfiguredAirbyteCatalog()).withStreams(catalog.getStreams().stream().map(MongoDbCdcInitializerTest::toConfiguredStream).toList()); + } + + public static ConfiguredAirbyteStream toConfiguredStream(final AirbyteStream stream) { + return (new ConfiguredAirbyteStream()) + .withStream(stream) + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(new ArrayList<>()) + .withDestinationSyncMode(DestinationSyncMode.OVERWRITE) + .withPrimaryKey(new ArrayList<>()); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcPropertiesTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcPropertiesTest.java new file mode 100644 index 000000000000..0bb2bfba05f3 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcPropertiesTest.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.cdc; + +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.CAPTURE_MODE_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.CAPTURE_MODE_VALUE; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.CONNECTOR_CLASS_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.CONNECTOR_CLASS_VALUE; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.HEARTBEAT_FREQUENCY_MS; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.HEARTBEAT_INTERVAL_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.SNAPSHOT_MODE_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.SNAPSHOT_MODE_VALUE; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.TOMBSTONE_ON_DELETE_KEY; +import static io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcProperties.TOMBSTONE_ON_DELETE_VALUE; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Properties; +import org.junit.jupiter.api.Test; + +class MongoDbCdcPropertiesTest { + + @Test + void testDebeziumProperties() { + final Properties debeziumProperties = MongoDbCdcProperties.getDebeziumProperties(); + assertEquals(5, debeziumProperties.size()); + assertEquals(CONNECTOR_CLASS_VALUE, debeziumProperties.get(CONNECTOR_CLASS_KEY)); + assertEquals(SNAPSHOT_MODE_VALUE, debeziumProperties.get(SNAPSHOT_MODE_KEY)); + assertEquals(CAPTURE_MODE_VALUE, debeziumProperties.get(CAPTURE_MODE_KEY)); + assertEquals(HEARTBEAT_FREQUENCY_MS, debeziumProperties.get(HEARTBEAT_INTERVAL_KEY)); + assertEquals(TOMBSTONE_ON_DELETE_VALUE, debeziumProperties.get(TOMBSTONE_ON_DELETE_KEY)); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcSavedInfoFetcherTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcSavedInfoFetcherTest.java new file mode 100644 index 000000000000..515903207ce2 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcSavedInfoFetcherTest.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.cdc; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumStateUtil; +import org.junit.jupiter.api.Test; + +class MongoDbCdcSavedInfoFetcherTest { + + private static final String DATABASE = "test-database"; + private static final String REPLICA_SET = "test-replica-set"; + private static final String RESUME_TOKEN = "8264BEB9F3000000012B0229296E04"; + + @Test + void testRetrieveSavedOffsetState() { + final JsonNode offset = MongoDbDebeziumStateUtil.formatState(DATABASE, REPLICA_SET, RESUME_TOKEN); + final MongoDbCdcState offsetState = new MongoDbCdcState(offset); + final MongoDbCdcSavedInfoFetcher cdcSavedInfoFetcher = new MongoDbCdcSavedInfoFetcher(offsetState); + assertEquals(offsetState.state(), cdcSavedInfoFetcher.getSavedOffset()); + } + + @Test + void testRetrieveSchemaHistory() { + final JsonNode offset = MongoDbDebeziumStateUtil.formatState(DATABASE, REPLICA_SET, RESUME_TOKEN); + final MongoDbCdcState offsetState = new MongoDbCdcState(offset); + final MongoDbCdcSavedInfoFetcher cdcSavedInfoFetcher = new MongoDbCdcSavedInfoFetcher(offsetState); + assertThrows(RuntimeException.class, () -> cdcSavedInfoFetcher.getSavedSchemaHistory()); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcStateHandlerTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcStateHandlerTest.java new file mode 100644 index 000000000000..a4a253654cf1 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/cdc/MongoDbCdcStateHandlerTest.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.cdc; + +import static io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType.GLOBAL; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.fasterxml.jackson.core.type.TypeReference; +import io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumStateUtil; +import io.airbyte.integrations.source.mongodb.state.MongoDbStateManager; +import io.airbyte.protocol.models.Jsons; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class MongoDbCdcStateHandlerTest { + + private static final String DATABASE = "test-database"; + private static final String REPLICA_SET = "test-replica-set"; + private static final String RESUME_TOKEN = "8264BEB9F3000000012B0229296E04"; + + private MongoDbCdcStateHandler mongoDbCdcStateHandler; + + @BeforeEach + void setup() { + final MongoDbStateManager mongoDbStateManager = MongoDbStateManager.createStateManager(null); + mongoDbCdcStateHandler = new MongoDbCdcStateHandler(mongoDbStateManager); + } + + @Test + void testSavingState() { + final Map offset = + Jsons.object(MongoDbDebeziumStateUtil.formatState(DATABASE, REPLICA_SET, RESUME_TOKEN), new TypeReference<>() {}); + final AirbyteMessage airbyteMessage = mongoDbCdcStateHandler.saveState(offset, null); + assertNotNull(airbyteMessage); + assertEquals(AirbyteMessage.Type.STATE, airbyteMessage.getType()); + assertNotNull(airbyteMessage.getState()); + assertEquals(GLOBAL, airbyteMessage.getState().getType()); + assertEquals(new MongoDbCdcState(Jsons.jsonNode(offset)), + Jsons.object(airbyteMessage.getState().getGlobal().getSharedState(), MongoDbCdcState.class)); + } + + @Test + void testSaveStateAfterCompletionOfSnapshotOfNewStreams() { + assertThrows(RuntimeException.class, () -> mongoDbCdcStateHandler.saveStateAfterCompletionOfSnapshotOfNewStreams()); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/state/IdTypeTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/state/IdTypeTest.java new file mode 100644 index 000000000000..1f81a16d3bee --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/state/IdTypeTest.java @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.state; + +import static org.junit.jupiter.api.Assertions.*; + +import org.bson.types.ObjectId; +import org.junit.jupiter.api.Test; + +class IdTypeTest { + + @Test + void convert() { + assertEquals(101, IdType.INT.convert("101")); + assertEquals(202L, IdType.LONG.convert("202")); + assertEquals("example", IdType.STRING.convert("example")); + assertEquals(new ObjectId("012301230123012301230123"), IdType.OBJECT_ID.convert("012301230123012301230123")); + } + + @Test + void findByBsonType() { + assertTrue(IdType.findByBsonType("objectId").isPresent(), "objectId not found"); + assertTrue(IdType.findByBsonType("objectid").isPresent(), "should have found nothing as it is case-insensitive"); + assertTrue(IdType.findByBsonType(null).isEmpty(), "passing in a null is fine"); + } + + @Test + void findByJavaType() { + assertTrue(IdType.findByJavaType("objectId").isPresent(), "objectId not found"); + assertTrue(IdType.findByJavaType("objectid").isPresent(), "should have found nothing as it is case-insensitive"); + assertTrue(IdType.findByJavaType("Integer").isPresent(), "Integer not found"); + assertTrue(IdType.findByJavaType(null).isEmpty(), "passing in a null is fine"); + } + + @Test + void supported() { + assertEquals("objectId, string, int, long", IdType.SUPPORTED); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/state/MongoDbStateManagerTest.java b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/state/MongoDbStateManagerTest.java new file mode 100644 index 000000000000..b9d83ace1c47 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/java/io/airbyte/integrations/source/mongodb/state/MongoDbStateManagerTest.java @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mongodb.state; + +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.ChangeEvent.SOURCE_ORDER; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.ChangeEvent.SOURCE_RESUME_TOKEN; +import static io.airbyte.cdk.integrations.debezium.internals.mongodb.MongoDbDebeziumConstants.ChangeEvent.SOURCE_SECONDS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.mongodb.cdc.MongoDbCdcState; +import io.airbyte.protocol.models.v0.AirbyteGlobalState; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStreamState; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class MongoDbStateManagerTest { + + private static final String ID = "64c0029d95ad260d69ef28a0"; + private static final String RESUME_TOKEN = "8264BEB9F3000000012B0229296E04"; + private static final String STREAM_NAME = "test-collection"; + private static final String STREAM_NAMESPACE = "test-database"; + + @Test + void testCreationWithInitialState() { + final StreamDescriptor streamDescriptor = new StreamDescriptor().withNamespace(STREAM_NAMESPACE).withName(STREAM_NAME); + final int seconds = 123456789; + final int order = 1; + final Map offset = Map.of(SOURCE_SECONDS, String.valueOf(seconds), + SOURCE_ORDER, String.valueOf(order), + SOURCE_RESUME_TOKEN, RESUME_TOKEN); + final MongoDbCdcState cdcState = new MongoDbCdcState(Jsons.jsonNode(offset)); + final MongoDbStreamState mongoDbStreamState = new MongoDbStreamState(ID, InitialSnapshotStatus.IN_PROGRESS, IdType.OBJECT_ID); + final JsonNode sharedState = Jsons.jsonNode(cdcState); + final JsonNode streamState = Jsons.jsonNode(mongoDbStreamState); + final AirbyteStreamState airbyteStreamState = new AirbyteStreamState().withStreamDescriptor(streamDescriptor).withStreamState(streamState); + final AirbyteGlobalState airbyteGlobalState = new AirbyteGlobalState().withSharedState(sharedState).withStreamStates(List.of(airbyteStreamState)); + final AirbyteStateMessage airbyteStateMessage = + new AirbyteStateMessage().withType(AirbyteStateMessage.AirbyteStateType.GLOBAL).withGlobal(airbyteGlobalState); + + final MongoDbStateManager stateManager = MongoDbStateManager.createStateManager(Jsons.jsonNode(List.of(airbyteStateMessage))); + assertNotNull(stateManager); + assertNotNull(stateManager.getCdcState()); + Assertions.assertEquals(seconds, stateManager.getCdcState().state().get(SOURCE_SECONDS).asInt()); + Assertions.assertEquals(order, stateManager.getCdcState().state().get(SOURCE_ORDER).asInt()); + Assertions.assertEquals(RESUME_TOKEN, stateManager.getCdcState().state().get(SOURCE_RESUME_TOKEN).asText()); + assertTrue(stateManager.getStreamState(STREAM_NAME, STREAM_NAMESPACE).isPresent()); + assertEquals(ID, stateManager.getStreamState(STREAM_NAME, STREAM_NAMESPACE).get().id()); + } + + @Test + void testCreationWithInitialNullState() { + final MongoDbStateManager stateManager = MongoDbStateManager.createStateManager(null); + assertNotNull(stateManager); + assertNull(stateManager.getCdcState()); + } + + @Test + void testCreationWithInitialEmptyState() { + final MongoDbStateManager stateManager = MongoDbStateManager.createStateManager(Jsons.emptyObject()); + assertNotNull(stateManager); + assertNull(stateManager.getCdcState()); + } + + @Test + void testCreationWithInitialEmptyListState() { + final MongoDbStateManager stateManager = MongoDbStateManager.createStateManager(Jsons.jsonNode(List.of())); + assertNotNull(stateManager); + assertNull(stateManager.getCdcState()); + } + + @Test + void testCreationWithInitialStateTooManyMessages() { + final List stateMessages = List.of(new AirbyteStateMessage(), new AirbyteStateMessage()); + assertThrows(IllegalStateException.class, () -> MongoDbStateManager.createStateManager(Jsons.jsonNode(stateMessages))); + } + + @Test + void testUpdateCdcState() { + final MongoDbStateManager stateManager = MongoDbStateManager.createStateManager(null); + assertNotNull(stateManager); + assertNull(stateManager.getCdcState()); + + final Map offset = Map.of(SOURCE_SECONDS, String.valueOf(123456789), + SOURCE_ORDER, String.valueOf(1), + SOURCE_RESUME_TOKEN, RESUME_TOKEN); + final MongoDbCdcState cdcState = new MongoDbCdcState(Jsons.jsonNode(offset)); + stateManager.updateCdcState(cdcState); + assertNotNull(stateManager.getCdcState()); + Assertions.assertEquals(cdcState, stateManager.getCdcState()); + } + + @Test + void testGeneratingAirbyteStateMessage() { + final StreamDescriptor streamDescriptor = new StreamDescriptor().withNamespace(STREAM_NAMESPACE).withName(STREAM_NAME); + final int seconds = 123456789; + final int order = 1; + final Map offset = Map.of(SOURCE_SECONDS, String.valueOf(seconds), + SOURCE_ORDER, String.valueOf(order), + SOURCE_RESUME_TOKEN, RESUME_TOKEN); + final MongoDbCdcState cdcState = new MongoDbCdcState(Jsons.jsonNode(offset)); + final MongoDbStreamState mongoDbStreamState = new MongoDbStreamState(ID, InitialSnapshotStatus.IN_PROGRESS, IdType.OBJECT_ID); + final JsonNode sharedState = Jsons.jsonNode(cdcState); + final JsonNode streamState = Jsons.jsonNode(mongoDbStreamState); + final AirbyteStreamState airbyteStreamState = new AirbyteStreamState().withStreamDescriptor(streamDescriptor).withStreamState(streamState); + final AirbyteGlobalState airbyteGlobalState = new AirbyteGlobalState().withSharedState(sharedState).withStreamStates(List.of(airbyteStreamState)); + final AirbyteStateMessage airbyteStateMessage = + new AirbyteStateMessage().withType(AirbyteStateMessage.AirbyteStateType.GLOBAL).withGlobal(airbyteGlobalState); + + final MongoDbStateManager stateManager = MongoDbStateManager.createStateManager(Jsons.jsonNode(List.of(airbyteStateMessage))); + final AirbyteStateMessage generated = stateManager.toState(); + + assertNotNull(generated); + assertEquals(airbyteStateMessage, generated); + + final Map offset2 = Map.of(SOURCE_SECONDS, String.valueOf(1112223334), + SOURCE_ORDER, String.valueOf(2), + SOURCE_RESUME_TOKEN, RESUME_TOKEN); + final MongoDbCdcState updatedCdcState = new MongoDbCdcState(Jsons.jsonNode(offset2)); + stateManager.updateCdcState(updatedCdcState); + + final AirbyteStateMessage generated2 = stateManager.toState(); + + assertNotNull(generated2); + assertEquals(updatedCdcState, Jsons.object(generated2.getGlobal().getSharedState(), MongoDbCdcState.class)); + + final MongoDbStreamState updatedStreamState = new MongoDbStreamState("updated", InitialSnapshotStatus.COMPLETE, IdType.OBJECT_ID); + stateManager.updateStreamState(STREAM_NAME, STREAM_NAMESPACE, updatedStreamState); + final AirbyteStateMessage generated3 = stateManager.toState(); + + assertNotNull(generated3); + assertEquals(updatedStreamState.id(), + Jsons.object(generated3.getGlobal().getStreamStates().get(0).getStreamState(), MongoDbStreamState.class).id()); + } + + @Test + void testReset() { + final StreamDescriptor streamDescriptor = new StreamDescriptor().withNamespace(STREAM_NAMESPACE).withName(STREAM_NAME); + final int seconds = 123456789; + final int order = 1; + final Map offset = Map.of(SOURCE_SECONDS, String.valueOf(seconds), + SOURCE_ORDER, String.valueOf(order), + SOURCE_RESUME_TOKEN, RESUME_TOKEN); + final MongoDbCdcState cdcState = new MongoDbCdcState(Jsons.jsonNode(offset)); + final MongoDbStreamState mongoDbStreamState = new MongoDbStreamState(ID, InitialSnapshotStatus.IN_PROGRESS, IdType.OBJECT_ID); + final JsonNode sharedState = Jsons.jsonNode(cdcState); + final JsonNode streamState = Jsons.jsonNode(mongoDbStreamState); + final AirbyteStreamState airbyteStreamState = new AirbyteStreamState().withStreamDescriptor(streamDescriptor).withStreamState(streamState); + final AirbyteGlobalState airbyteGlobalState = new AirbyteGlobalState().withSharedState(sharedState).withStreamStates(List.of(airbyteStreamState)); + final AirbyteStateMessage airbyteStateMessage = + new AirbyteStateMessage().withType(AirbyteStateMessage.AirbyteStateType.GLOBAL).withGlobal(airbyteGlobalState); + + final MongoDbStateManager stateManager = MongoDbStateManager.createStateManager(Jsons.jsonNode(List.of(airbyteStateMessage))); + final MongoDbCdcState newCdcState = new MongoDbCdcState(Jsons.jsonNode(Map.of())); + + stateManager.resetState(newCdcState); + Assertions.assertEquals(newCdcState, stateManager.getCdcState()); + assertEquals(0, stateManager.getStreamStates().size()); + } + +} diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/resources/authorized_collections_response.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/authorized_collections_response.json similarity index 100% rename from airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/resources/authorized_collections_response.json rename to airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/authorized_collections_response.json diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/coll_stats_response.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/coll_stats_response.json new file mode 100644 index 000000000000..02b4b4d2a00d --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/coll_stats_response.json @@ -0,0 +1,530 @@ +[ + { + "ns": "test-database.test-collection", + "host": "host:27017", + "localTime": "2023-08-31T20:02:27.434Z", + "storageStats": { + "size": 67771, + "count": 746, + "avgObjSize": 90, + "storageSize": 90112, + "freeStorageSize": 40960, + "capped": false, + "wiredTiger": { + "metadata": { "formatVersion": 1 }, + "creationString": "access_pattern_hint=none,allocation_size=4KB,app_metadata=(formatVersion=1),assert=(commit_timestamp=none,durable_timestamp=none,read_timestamp=none,write_timestamp=off),block_allocation=best,block_compressor=snappy,cache_resident=false,checksum=on,colgroups=,collator=,columns=,dictionary=0,encryption=(keyid=,name=),exclusive=false,extractor=,format=btree,huffman_key=,huffman_value=,ignore_in_memory_cache_size=false,immutable=false,import=(enabled=false,file_metadata=,repair=false),internal_item_max=0,internal_key_max=0,internal_key_truncate=true,internal_page_max=4KB,key_format=q,key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=32KB,leaf_value_max=64MB,log=(enabled=false),lsm=(auto_throttle=true,bloom=true,bloom_bit_count=16,bloom_config=,bloom_hash_count=8,bloom_oldest=false,chunk_count_limit=0,chunk_max=5GB,chunk_size=10MB,merge_custom=(prefix=,start_generation=0,suffix=),merge_max=15,merge_min=0),memory_page_image_max=0,memory_page_max=10m,os_cache_dirty_max=0,os_cache_max=0,prefix_compression=false,prefix_compression_min=4,readonly=false,source=,split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90,tiered_object=false,tiered_storage=(auth_token=,bucket=,bucket_prefix=,cache_directory=,local_retention=300,name=,object_target_size=0),type=file,value_format=u,verbose=[],write_timestamp_usage=none", + "type": "file", + "uri": "statistics:table:collection-1286--1818790214620652471", + "LSM": { + "bloom filter false positives": 0, + "bloom filter hits": 0, + "bloom filter misses": 0, + "bloom filter pages evicted from cache": 0, + "bloom filter pages read into cache": 0, + "bloom filters in the LSM tree": 0, + "chunks in the LSM tree": 0, + "highest merge generation in the LSM tree": 0, + "queries that could have benefited from a Bloom filter that did not exist": 0, + "sleep for LSM checkpoint throttle": 0, + "sleep for LSM merge throttle": 0, + "total size of bloom filters": 0 + }, + "block-manager": { + "allocations requiring file extension": 0, + "blocks allocated": 0, + "blocks freed": 0, + "checkpoint size": 32768, + "file allocation unit size": 4096, + "file bytes available for reuse": 40960, + "file magic number": 120897, + "file major version number": 1, + "file size in bytes": 90112, + "minor version number": 0 + }, + "btree": { + "btree checkpoint generation": 10440, + "btree clean tree checkpoint expiration time": 9223372036854775807, + "btree compact pages reviewed": 0, + "btree compact pages selected to be rewritten": 0, + "btree compact pages skipped": 0, + "btree skipped by compaction as process would not reduce size": 0, + "column-store fixed-size leaf pages": 0, + "column-store internal pages": 0, + "column-store variable-size RLE encoded values": 0, + "column-store variable-size deleted values": 0, + "column-store variable-size leaf pages": 0, + "fixed-record size": 0, + "maximum internal page key size": 368, + "maximum internal page size": 4096, + "maximum leaf page key size": 2867, + "maximum leaf page size": 32768, + "maximum leaf page value size": 67108864, + "maximum tree depth": 3, + "number of key/value pairs": 0, + "overflow pages": 0, + "pages rewritten by compaction": 0, + "row-store empty values": 0, + "row-store internal pages": 0, + "row-store leaf pages": 0 + }, + "cache": { + "bytes currently in the cache": 238, + "bytes dirty in the cache cumulative": 0, + "bytes read into cache": 715801, + "bytes written from cache": 0, + "checkpoint blocked page eviction": 0, + "checkpoint of history store file blocked non-history store page eviction": 0, + "data source pages selected for eviction unable to be evicted": 0, + "eviction gave up due to detecting an out of order on disk value behind the last update on the chain": 0, + "eviction gave up due to detecting an out of order tombstone ahead of the selected on disk update": 0, + "eviction gave up due to detecting an out of order tombstone ahead of the selected on disk update after validating the update chain": 0, + "eviction gave up due to detecting out of order timestamps on the update chain after the selected on disk update": 0, + "eviction walk passes of a file": 202, + "eviction walk target pages histogram - 0-9": 174, + "eviction walk target pages histogram - 10-31": 28, + "eviction walk target pages histogram - 128 and higher": 0, + "eviction walk target pages histogram - 32-63": 0, + "eviction walk target pages histogram - 64-128": 0, + "eviction walk target pages reduced due to history store cache pressure": 0, + "eviction walks abandoned": 0, + "eviction walks gave up because they restarted their walk twice": 201, + "eviction walks gave up because they saw too many pages and found no candidates": 0, + "eviction walks gave up because they saw too many pages and found too few candidates": 0, + "eviction walks reached end of tree": 403, + "eviction walks restarted": 0, + "eviction walks started from root of tree": 202, + "eviction walks started from saved location in tree": 0, + "hazard pointer blocked page eviction": 0, + "history store table insert calls": 0, + "history store table insert calls that returned restart": 0, + "history store table out-of-order resolved updates that lose their durable timestamp": 0, + "history store table out-of-order updates that were fixed up by reinserting with the fixed timestamp": 0, + "history store table reads": 0, + "history store table reads missed": 0, + "history store table reads requiring squashed modifies": 0, + "history store table truncation by rollback to stable to remove an unstable update": 0, + "history store table truncation by rollback to stable to remove an update": 0, + "history store table truncation to remove an update": 0, + "history store table truncation to remove range of updates due to key being removed from the data page during reconciliation": 0, + "history store table truncation to remove range of updates due to out-of-order timestamp update on data page": 0, + "history store table writes requiring squashed modifies": 0, + "in-memory page passed criteria to be split": 0, + "in-memory page splits": 0, + "internal pages evicted": 0, + "internal pages split during eviction": 0, + "leaf pages split during eviction": 0, + "modified pages evicted": 0, + "overflow pages read into cache": 0, + "page split during eviction deepened the tree": 0, + "page written requiring history store records": 0, + "pages read into cache": 11, + "pages read into cache after truncate": 0, + "pages read into cache after truncate in prepare state": 0, + "pages requested from the cache": 203, + "pages seen by eviction walk": 221, + "pages written from cache": 0, + "pages written requiring in-memory restoration": 0, + "the number of times full update inserted to history store": 0, + "the number of times reverse modify inserted to history store": 0, + "tracked dirty bytes in the cache": 0, + "unmodified pages evicted": 10 + }, + "cache_walk": { + "Average difference between current eviction generation when the page was last considered": 0, + "Average on-disk page image size seen": 0, + "Average time in cache for pages that have been visited by the eviction server": 0, + "Average time in cache for pages that have not been visited by the eviction server": 0, + "Clean pages currently in cache": 0, + "Current eviction generation": 0, + "Dirty pages currently in cache": 0, + "Entries in the root page": 0, + "Internal pages currently in cache": 0, + "Leaf pages currently in cache": 0, + "Maximum difference between current eviction generation when the page was last considered": 0, + "Maximum page size seen": 0, + "Minimum on-disk page image size seen": 0, + "Number of pages never visited by eviction server": 0, + "On-disk page image sizes smaller than a single allocation unit": 0, + "Pages created in memory and never written": 0, + "Pages currently queued for eviction": 0, + "Pages that could not be queued for eviction": 0, + "Refs skipped during cache traversal": 0, + "Size of the root page": 0, + "Total number of pages currently in cache": 0 + }, + "checkpoint-cleanup": { + "pages added for eviction": 0, + "pages removed": 0, + "pages skipped during tree walk": 0, + "pages visited": 0 + }, + "compression": { + "compressed page maximum internal page size prior to compression": 4096, + "compressed page maximum leaf page size prior to compression ": 131072, + "compressed pages read": 10, + "compressed pages written": 0, + "page written failed to compress": 0, + "page written was too small to compress": 0 + }, + "cursor": { + "Total number of entries skipped by cursor next calls": 0, + "Total number of entries skipped by cursor prev calls": 0, + "Total number of entries skipped to position the history store cursor": 0, + "Total number of times a search near has exited due to prefix config": 0, + "bulk loaded cursor insert calls": 0, + "cache cursors reuse count": 202, + "close calls that result in cache": 202, + "create calls": 12, + "cursor next calls that skip due to a globally visible history store tombstone": 0, + "cursor next calls that skip greater than or equal to 100 entries": 0, + "cursor next calls that skip less than 100 entries": 88161, + "cursor prev calls that skip due to a globally visible history store tombstone": 0, + "cursor prev calls that skip greater than or equal to 100 entries": 0, + "cursor prev calls that skip less than 100 entries": 0, + "insert calls": 0, + "insert key and value bytes": 0, + "modify": 0, + "modify key and value bytes affected": 0, + "modify value bytes modified": 0, + "next calls": 88161, + "open cursor count": 0, + "operation restarted": 0, + "prev calls": 0, + "remove calls": 0, + "remove key bytes removed": 0, + "reserve calls": 0, + "reset calls": 480, + "search calls": 746, + "search history store calls": 0, + "search near calls": 75, + "truncate calls": 0, + "update calls": 0, + "update key and value bytes": 0, + "update value size change": 0 + }, + "reconciliation": { + "approximate byte size of timestamps in pages written": 0, + "approximate byte size of transaction IDs in pages written": 0, + "dictionary matches": 0, + "fast-path pages deleted": 0, + "internal page key bytes discarded using suffix compression": 0, + "internal page multi-block writes": 0, + "internal-page overflow keys": 0, + "leaf page key bytes discarded using prefix compression": 0, + "leaf page multi-block writes": 0, + "leaf-page overflow keys": 0, + "maximum blocks required for a page": 0, + "overflow values written": 0, + "page checksum matches": 0, + "page reconciliation calls": 0, + "page reconciliation calls for eviction": 0, + "pages deleted": 0, + "pages written including an aggregated newest start durable timestamp ": 0, + "pages written including an aggregated newest stop durable timestamp ": 0, + "pages written including an aggregated newest stop timestamp ": 0, + "pages written including an aggregated newest stop transaction ID": 0, + "pages written including an aggregated newest transaction ID ": 0, + "pages written including an aggregated oldest start timestamp ": 0, + "pages written including an aggregated prepare": 0, + "pages written including at least one prepare": 0, + "pages written including at least one start durable timestamp": 0, + "pages written including at least one start timestamp": 0, + "pages written including at least one start transaction ID": 0, + "pages written including at least one stop durable timestamp": 0, + "pages written including at least one stop timestamp": 0, + "pages written including at least one stop transaction ID": 0, + "records written including a prepare": 0, + "records written including a start durable timestamp": 0, + "records written including a start timestamp": 0, + "records written including a start transaction ID": 0, + "records written including a stop durable timestamp": 0, + "records written including a stop timestamp": 0, + "records written including a stop transaction ID": 0 + }, + "session": { + "object compaction": 0, + "tiered operations dequeued and processed": 0, + "tiered operations scheduled": 0, + "tiered storage local retention time (secs)": 0 + }, + "transaction": { + "race to read prepared update retry": 0, + "rollback to stable history store records with stop timestamps older than newer records": 0, + "rollback to stable inconsistent checkpoint": 0, + "rollback to stable keys removed": 0, + "rollback to stable keys restored": 0, + "rollback to stable restored tombstones from history store": 0, + "rollback to stable restored updates from history store": 0, + "rollback to stable skipping delete rle": 0, + "rollback to stable skipping stable rle": 0, + "rollback to stable sweeping history store keys": 0, + "rollback to stable updates removed from history store": 0, + "transaction checkpoints due to obsolete pages": 0, + "update conflicts": 0 + } + }, + "nindexes": 1, + "indexDetails": { + "_id_": { + "metadata": { "formatVersion": 8 }, + "creationString": "access_pattern_hint=none,allocation_size=4KB,app_metadata=(formatVersion=8),assert=(commit_timestamp=none,durable_timestamp=none,read_timestamp=none,write_timestamp=off),block_allocation=best,block_compressor=,cache_resident=false,checksum=on,colgroups=,collator=,columns=,dictionary=0,encryption=(keyid=,name=),exclusive=false,extractor=,format=btree,huffman_key=,huffman_value=,ignore_in_memory_cache_size=false,immutable=false,import=(enabled=false,file_metadata=,repair=false),internal_item_max=0,internal_key_max=0,internal_key_truncate=true,internal_page_max=16k,key_format=u,key_gap=10,leaf_item_max=0,leaf_key_max=0,leaf_page_max=16k,leaf_value_max=0,log=(enabled=false),lsm=(auto_throttle=true,bloom=true,bloom_bit_count=16,bloom_config=,bloom_hash_count=8,bloom_oldest=false,chunk_count_limit=0,chunk_max=5GB,chunk_size=10MB,merge_custom=(prefix=,start_generation=0,suffix=),merge_max=15,merge_min=0),memory_page_image_max=0,memory_page_max=5MB,os_cache_dirty_max=0,os_cache_max=0,prefix_compression=true,prefix_compression_min=4,readonly=false,source=,split_deepen_min_child=0,split_deepen_per_child=0,split_pct=90,tiered_object=false,tiered_storage=(auth_token=,bucket=,bucket_prefix=,cache_directory=,local_retention=300,name=,object_target_size=0),type=file,value_format=u,verbose=[],write_timestamp_usage=none", + "type": "file", + "uri": "statistics:table:index-1287--1818790214620652471", + "LSM": { + "bloom filter false positives": 0, + "bloom filter hits": 0, + "bloom filter misses": 0, + "bloom filter pages evicted from cache": 0, + "bloom filter pages read into cache": 0, + "bloom filters in the LSM tree": 0, + "chunks in the LSM tree": 0, + "highest merge generation in the LSM tree": 0, + "queries that could have benefited from a Bloom filter that did not exist": 0, + "sleep for LSM checkpoint throttle": 0, + "sleep for LSM merge throttle": 0, + "total size of bloom filters": 0 + }, + "block-manager": { + "allocations requiring file extension": 0, + "blocks allocated": 0, + "blocks freed": 0, + "checkpoint size": 12288, + "file allocation unit size": 4096, + "file bytes available for reuse": 24576, + "file magic number": 120897, + "file major version number": 1, + "file size in bytes": 53248, + "minor version number": 0 + }, + "btree": { + "btree checkpoint generation": 10440, + "btree clean tree checkpoint expiration time": 9223372036854775807, + "btree compact pages reviewed": 0, + "btree compact pages selected to be rewritten": 0, + "btree compact pages skipped": 0, + "btree skipped by compaction as process would not reduce size": 0, + "column-store fixed-size leaf pages": 0, + "column-store internal pages": 0, + "column-store variable-size RLE encoded values": 0, + "column-store variable-size deleted values": 0, + "column-store variable-size leaf pages": 0, + "fixed-record size": 0, + "maximum internal page key size": 1474, + "maximum internal page size": 16384, + "maximum leaf page key size": 1474, + "maximum leaf page size": 16384, + "maximum leaf page value size": 7372, + "maximum tree depth": 3, + "number of key/value pairs": 0, + "overflow pages": 0, + "pages rewritten by compaction": 0, + "row-store empty values": 0, + "row-store internal pages": 0, + "row-store leaf pages": 0 + }, + "cache": { + "bytes currently in the cache": 238, + "bytes dirty in the cache cumulative": 0, + "bytes read into cache": 11151, + "bytes written from cache": 0, + "checkpoint blocked page eviction": 0, + "checkpoint of history store file blocked non-history store page eviction": 0, + "data source pages selected for eviction unable to be evicted": 0, + "eviction gave up due to detecting an out of order on disk value behind the last update on the chain": 0, + "eviction gave up due to detecting an out of order tombstone ahead of the selected on disk update": 0, + "eviction gave up due to detecting an out of order tombstone ahead of the selected on disk update after validating the update chain": 0, + "eviction gave up due to detecting out of order timestamps on the update chain after the selected on disk update": 0, + "eviction walk passes of a file": 15, + "eviction walk target pages histogram - 0-9": 14, + "eviction walk target pages histogram - 10-31": 1, + "eviction walk target pages histogram - 128 and higher": 0, + "eviction walk target pages histogram - 32-63": 0, + "eviction walk target pages histogram - 64-128": 0, + "eviction walk target pages reduced due to history store cache pressure": 0, + "eviction walks abandoned": 0, + "eviction walks gave up because they restarted their walk twice": 15, + "eviction walks gave up because they saw too many pages and found no candidates": 0, + "eviction walks gave up because they saw too many pages and found too few candidates": 0, + "eviction walks reached end of tree": 30, + "eviction walks restarted": 0, + "eviction walks started from root of tree": 15, + "eviction walks started from saved location in tree": 0, + "hazard pointer blocked page eviction": 0, + "history store table insert calls": 0, + "history store table insert calls that returned restart": 0, + "history store table out-of-order resolved updates that lose their durable timestamp": 0, + "history store table out-of-order updates that were fixed up by reinserting with the fixed timestamp": 0, + "history store table reads": 0, + "history store table reads missed": 0, + "history store table reads requiring squashed modifies": 0, + "history store table truncation by rollback to stable to remove an unstable update": 0, + "history store table truncation by rollback to stable to remove an update": 0, + "history store table truncation to remove an update": 0, + "history store table truncation to remove range of updates due to key being removed from the data page during reconciliation": 0, + "history store table truncation to remove range of updates due to out-of-order timestamp update on data page": 0, + "history store table writes requiring squashed modifies": 0, + "in-memory page passed criteria to be split": 0, + "in-memory page splits": 0, + "internal pages evicted": 0, + "internal pages split during eviction": 0, + "leaf pages split during eviction": 0, + "modified pages evicted": 0, + "overflow pages read into cache": 0, + "page split during eviction deepened the tree": 0, + "page written requiring history store records": 0, + "pages read into cache": 2, + "pages read into cache after truncate": 0, + "pages read into cache after truncate in prepare state": 0, + "pages requested from the cache": 2, + "pages seen by eviction walk": 22, + "pages written from cache": 0, + "pages written requiring in-memory restoration": 0, + "the number of times full update inserted to history store": 0, + "the number of times reverse modify inserted to history store": 0, + "tracked dirty bytes in the cache": 0, + "unmodified pages evicted": 1 + }, + "cache_walk": { + "Average difference between current eviction generation when the page was last considered": 0, + "Average on-disk page image size seen": 0, + "Average time in cache for pages that have been visited by the eviction server": 0, + "Average time in cache for pages that have not been visited by the eviction server": 0, + "Clean pages currently in cache": 0, + "Current eviction generation": 0, + "Dirty pages currently in cache": 0, + "Entries in the root page": 0, + "Internal pages currently in cache": 0, + "Leaf pages currently in cache": 0, + "Maximum difference between current eviction generation when the page was last considered": 0, + "Maximum page size seen": 0, + "Minimum on-disk page image size seen": 0, + "Number of pages never visited by eviction server": 0, + "On-disk page image sizes smaller than a single allocation unit": 0, + "Pages created in memory and never written": 0, + "Pages currently queued for eviction": 0, + "Pages that could not be queued for eviction": 0, + "Refs skipped during cache traversal": 0, + "Size of the root page": 0, + "Total number of pages currently in cache": 0 + }, + "checkpoint-cleanup": { + "pages added for eviction": 0, + "pages removed": 0, + "pages skipped during tree walk": 0, + "pages visited": 0 + }, + "compression": { + "compressed page maximum internal page size prior to compression": 16384, + "compressed page maximum leaf page size prior to compression ": 16384, + "compressed pages read": 0, + "compressed pages written": 0, + "page written failed to compress": 0, + "page written was too small to compress": 0 + }, + "cursor": { + "Total number of entries skipped by cursor next calls": 0, + "Total number of entries skipped by cursor prev calls": 0, + "Total number of entries skipped to position the history store cursor": 0, + "Total number of times a search near has exited due to prefix config": 0, + "bulk loaded cursor insert calls": 0, + "cache cursors reuse count": 2, + "close calls that result in cache": 2, + "create calls": 1, + "cursor next calls that skip due to a globally visible history store tombstone": 0, + "cursor next calls that skip greater than or equal to 100 entries": 0, + "cursor next calls that skip less than 100 entries": 746, + "cursor prev calls that skip due to a globally visible history store tombstone": 0, + "cursor prev calls that skip greater than or equal to 100 entries": 0, + "cursor prev calls that skip less than 100 entries": 0, + "insert calls": 0, + "insert key and value bytes": 0, + "modify": 0, + "modify key and value bytes affected": 0, + "modify value bytes modified": 0, + "next calls": 746, + "open cursor count": 0, + "operation restarted": 0, + "prev calls": 0, + "remove calls": 0, + "remove key bytes removed": 0, + "reserve calls": 0, + "reset calls": 5, + "search calls": 0, + "search history store calls": 0, + "search near calls": 2, + "truncate calls": 0, + "update calls": 0, + "update key and value bytes": 0, + "update value size change": 0 + }, + "reconciliation": { + "approximate byte size of timestamps in pages written": 0, + "approximate byte size of transaction IDs in pages written": 0, + "dictionary matches": 0, + "fast-path pages deleted": 0, + "internal page key bytes discarded using suffix compression": 0, + "internal page multi-block writes": 0, + "internal-page overflow keys": 0, + "leaf page key bytes discarded using prefix compression": 0, + "leaf page multi-block writes": 0, + "leaf-page overflow keys": 0, + "maximum blocks required for a page": 0, + "overflow values written": 0, + "page checksum matches": 0, + "page reconciliation calls": 0, + "page reconciliation calls for eviction": 0, + "pages deleted": 0, + "pages written including an aggregated newest start durable timestamp ": 0, + "pages written including an aggregated newest stop durable timestamp ": 0, + "pages written including an aggregated newest stop timestamp ": 0, + "pages written including an aggregated newest stop transaction ID": 0, + "pages written including an aggregated newest transaction ID ": 0, + "pages written including an aggregated oldest start timestamp ": 0, + "pages written including an aggregated prepare": 0, + "pages written including at least one prepare": 0, + "pages written including at least one start durable timestamp": 0, + "pages written including at least one start timestamp": 0, + "pages written including at least one start transaction ID": 0, + "pages written including at least one stop durable timestamp": 0, + "pages written including at least one stop timestamp": 0, + "pages written including at least one stop transaction ID": 0, + "records written including a prepare": 0, + "records written including a start durable timestamp": 0, + "records written including a start timestamp": 0, + "records written including a start transaction ID": 0, + "records written including a stop durable timestamp": 0, + "records written including a stop timestamp": 0, + "records written including a stop transaction ID": 0 + }, + "session": { + "object compaction": 0, + "tiered operations dequeued and processed": 0, + "tiered operations scheduled": 0, + "tiered storage local retention time (secs)": 0 + }, + "transaction": { + "race to read prepared update retry": 0, + "rollback to stable history store records with stop timestamps older than newer records": 0, + "rollback to stable inconsistent checkpoint": 0, + "rollback to stable keys removed": 0, + "rollback to stable keys restored": 0, + "rollback to stable restored tombstones from history store": 0, + "rollback to stable restored updates from history store": 0, + "rollback to stable skipping delete rle": 0, + "rollback to stable skipping stable rle": 0, + "rollback to stable sweeping history store keys": 0, + "rollback to stable updates removed from history store": 0, + "transaction checkpoints due to obsolete pages": 0, + "update conflicts": 0 + } + } + }, + "indexBuilds": [], + "totalIndexSize": 53248, + "totalSize": 143360, + "indexSizes": { "_id_": 53248 }, + "scaleFactor": 1 + }, + "count": 746 + } +] diff --git a/airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/resources/no_authorized_collections_response.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/no_authorized_collections_response.json similarity index 100% rename from airbyte-integrations/connectors/source-mongodb-internal-poc/src/test/resources/no_authorized_collections_response.json rename to airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/no_authorized_collections_response.json diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/schema_discovery_response.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/schema_discovery_response.json new file mode 100644 index 000000000000..368b1ea42b46 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/schema_discovery_response.json @@ -0,0 +1,26 @@ +[ + { + "_id": { + "_id": "string", + "name": "string", + "last_updated": "date", + "total": "int", + "price": "decimal", + "items": "array", + "owners": "object", + "amount": "null" + } + }, + { + "_id": { + "_id": "string", + "name": "string", + "last_updated": "date", + "total": "int", + "price": "decimal", + "items": "array", + "owners": "object", + "other": "string" + } + } +] diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/schema_discovery_response_different_datatypes.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/schema_discovery_response_different_datatypes.json new file mode 100644 index 000000000000..0f7aef047f40 --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/schema_discovery_response_different_datatypes.json @@ -0,0 +1,25 @@ +[ + { + "_id": { + "_id": "string", + "name": "string", + "last_updated": "date", + "total": "int", + "price": "decimal", + "items": "array", + "owners": "object" + } + }, + { + "_id": { + "_id": "string", + "name": "string", + "last_updated": "date", + "total": "string", + "price": "decimal", + "items": "array", + "owners": "object", + "other": "string" + } + } +] diff --git a/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/schema_discovery_response_schemaless.json b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/schema_discovery_response_schemaless.json new file mode 100644 index 000000000000..5ea895b9847a --- /dev/null +++ b/airbyte-integrations/connectors/source-mongodb-v2/src/test/resources/schema_discovery_response_schemaless.json @@ -0,0 +1,5 @@ +[ + { + "_idType": "object" + } +] diff --git a/airbyte-integrations/connectors/source-mongodb/build.gradle b/airbyte-integrations/connectors/source-mongodb/build.gradle deleted file mode 100644 index e07be5a8efd6..000000000000 --- a/airbyte-integrations/connectors/source-mongodb/build.gradle +++ /dev/null @@ -1,39 +0,0 @@ -plugins { - // Makes building the docker image a dependency of Gradle's "build" command. This way you could run your entire build inside a docker image - // via ./gradlew :airbyte-integrations:connectors:source-mongodb:build - id 'airbyte-docker' - id 'airbyte-standard-source-test-file' -} - -airbyteStandardSourceTestFile { - def dbName = "mongo-airbyte-integration-test" - prehook { - project.exec { - def args = ["docker", "run", "--rm", - "--name", dbName, - "-d", - // assign to a weird port number so we don't conflict with any locally running mongo instances - "-p", "27888:27017", - "-e", "MONGO_INITDB_ROOT_USERNAME=user", - "-e", "MONGO_INITDB_ROOT_PASSWORD=password", - "airbyte/mongodb-integration-test-seed:dev"] - commandLine args - } - } - - posthook { - project.exec { - commandLine "docker", "stop", dbName - } - } - - // All these input paths must live inside this connector's directory (or subdirectories) - configPath = "integration_tests/valid_config.json" - configuredCatalogPath = "integration_tests/configured_catalog.json" - specPath = "lib/spec.json" -} - -dependencies { - implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) -} - diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/.dockerignore b/airbyte-integrations/connectors/source-mssql-strict-encrypt/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile deleted file mode 100644 index b9c0668add68..000000000000 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-mssql-strict-encrypt - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-mssql-strict-encrypt - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=2.0.0 -LABEL io.airbyte.name=airbyte/source-mssql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mssql-strict-encrypt/acceptance-test-config.yml deleted file mode 100644 index ca1b6918bf58..000000000000 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/acceptance-test-config.yml +++ /dev/null @@ -1,6 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-mssql-strict-encrypt:dev -tests: - spec: - - spec_path: "src/test/resources/expected_spec.json" diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-mssql-strict-encrypt/build.gradle deleted file mode 100644 index 75f3de23eb74..000000000000 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/build.gradle +++ /dev/null @@ -1,32 +0,0 @@ -plugins { - id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' -} - -application { - mainClass = 'io.airbyte.integrations.source.mssql.MssqlSourceStrictEncrypt' - applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] -} - -dependencies { - implementation project(':airbyte-db:db-lib') - implementation libs.airbyte.protocol - - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation project(':airbyte-integrations:connectors:source-relational-db') - implementation project(':airbyte-integrations:connectors:source-mssql') - - testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) - - testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation libs.connectors.testcontainers.mssqlserver - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-mssql-strict-encrypt') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) -} - diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/icon.svg b/airbyte-integrations/connectors/source-mssql-strict-encrypt/icon.svg deleted file mode 100644 index edcaeb77c8f2..000000000000 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/icon.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/source-mssql-strict-encrypt/metadata.yaml deleted file mode 100644 index fbdd8bc54b41..000000000000 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/metadata.yaml +++ /dev/null @@ -1,29 +0,0 @@ -data: - allowedHosts: - hosts: - - ${host} - - ${tunnel_method.tunnel_host} - registries: - cloud: - enabled: false # strict encrypt connectors are deployed to Cloud by their non strict encrypt sibling. - oss: - enabled: false # strict encrypt connectors are not used on OSS. - connectorSubtype: database - connectorType: source - definitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 - dockerImageTag: 2.0.0 - dockerRepository: airbyte/source-mssql-strict-encrypt - githubIssueLabel: source-mssql - icon: mssql.svg - license: ELv2 - name: Microsoft SQL Server (MSSQL) - releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/mssql - tags: - - language:java - releases: - breakingChanges: - 2.0.0: - message: "Add default cursor for cdc" - upgradeDeadline: "2023-08-23" -metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/main/java/io.airbyte.integrations.source.mssql/MssqlSourceStrictEncrypt.java b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/main/java/io.airbyte.integrations.source.mssql/MssqlSourceStrictEncrypt.java deleted file mode 100644 index aef8791dcf7f..000000000000 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/main/java/io.airbyte.integrations.source.mssql/MssqlSourceStrictEncrypt.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mssql; - -import com.fasterxml.jackson.databind.node.ArrayNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.base.spec_modification.SpecModifyingSource; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class MssqlSourceStrictEncrypt extends SpecModifyingSource implements Source { - - private static final Logger LOGGER = LoggerFactory.getLogger(MssqlSourceStrictEncrypt.class); - - public MssqlSourceStrictEncrypt() { - super(MssqlSource.sshWrappedSource()); - } - - @Override - public ConnectorSpecification modifySpec(final ConnectorSpecification originalSpec) throws Exception { - final ConnectorSpecification spec = Jsons.clone(originalSpec); - ((ArrayNode) spec.getConnectionSpecification().get("properties").get("ssl_method").get("oneOf")).remove(0); - return spec; - } - - public static void main(final String[] args) throws Exception { - final Source source = new MssqlSourceStrictEncrypt(); - LOGGER.info("starting source: {}", MssqlSourceStrictEncrypt.class); - new IntegrationRunner(source).run(args); - LOGGER.info("completed source: {}", MssqlSourceStrictEncrypt.class); - } - -} diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlStrictEncryptSourceAcceptanceTest.java deleted file mode 100644 index 507ea4d17cd2..000000000000 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlStrictEncryptSourceAcceptanceTest.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mssql; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import java.sql.SQLException; -import java.util.HashMap; -import java.util.Map; -import org.apache.commons.lang3.RandomStringUtils; -import org.jooq.DSLContext; -import org.junit.jupiter.api.AfterAll; -import org.testcontainers.containers.MSSQLServerContainer; -import org.testcontainers.utility.DockerImageName; - -public class MssqlStrictEncryptSourceAcceptanceTest extends SourceAcceptanceTest { - - protected static final String SCHEMA_NAME = "dbo"; - protected static final String STREAM_NAME = "id_and_name"; - protected static MSSQLServerContainer db; - protected JsonNode config; - - @AfterAll - public static void closeContainer() { - if (db != null) { - db.close(); - db.stop(); - } - } - - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws SQLException { - if (db == null) { - db = new MSSQLServerContainer<>(DockerImageName - .parse("airbyte/mssql_ssltest:dev") - .asCompatibleSubstituteFor("mcr.microsoft.com/mssql/server")) - .acceptLicense(); - db.start(); - } - - final JsonNode configWithoutDbName = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) - .build()); - final String dbName = "db_" + RandomStringUtils.randomAlphabetic(10).toLowerCase(); - - try (final DSLContext dslContext = DSLContextFactory.create( - configWithoutDbName.get(JdbcUtils.USERNAME_KEY).asText(), - configWithoutDbName.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.MSSQLSERVER.getDriverClassName(), - String.format("jdbc:sqlserver://%s:%s;encrypt=true;trustServerCertificate=true;", - configWithoutDbName.get(JdbcUtils.HOST_KEY).asText(), - configWithoutDbName.get(JdbcUtils.PORT_KEY).asInt()), - null)) { - final Database database = getDatabase(dslContext); - database.query(ctx -> { - ctx.fetch(String.format("CREATE DATABASE %s;", dbName)); - ctx.fetch(String.format("USE %s;", dbName)); - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200), born DATETIMEOFFSET(7));"); - ctx.fetch( - "INSERT INTO id_and_name (id, name, born) VALUES " + - "(1,'picard', '2124-03-04T01:01:01Z'), " + - "(2, 'crusher', '2124-03-04T01:01:01Z'), " + - "(3, 'vash', '2124-03-04T01:01:01Z');"); - return null; - }); - } - - config = Jsons.clone(configWithoutDbName); - ((ObjectNode) config).put(JdbcUtils.DATABASE_KEY, dbName); - ((ObjectNode) config).put("ssl_method", Jsons.jsonNode(Map.of("ssl_method", "encrypted_trust_server_certificate"))); - } - - private static Database getDatabase(final DSLContext dslContext) { - return new Database(dslContext); - } - - @Override - protected void tearDown(final TestDestinationEnv testEnv) throws Exception {} - - @Override - protected String getImageName() { - return "airbyte/source-mssql-strict-encrypt:dev"; - } - - @Override - protected ConnectorSpecification getSpec() throws Exception { - return SshHelpers.injectSshIntoSpec(Jsons.deserialize(MoreResources.readResource("expected_spec.json"), ConnectorSpecification.class)); - } - - @Override - protected JsonNode getConfig() { - return config; - } - - @Override - protected ConfiguredAirbyteCatalog getConfiguredCatalog() { - return CatalogHelpers.createConfiguredAirbyteCatalog( - STREAM_NAME, - SCHEMA_NAME, - Field.of("id", JsonSchemaType.NUMBER), - Field.of("name", JsonSchemaType.STRING), - Field.of("born", JsonSchemaType.STRING)); - } - - @Override - protected JsonNode getState() { - return Jsons.jsonNode(new HashMap<>()); - } - -} diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mssql/MssqlStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mssql/MssqlStrictEncryptJdbcSourceAcceptanceTest.java deleted file mode 100644 index 4bdbc7c23ea5..000000000000 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mssql/MssqlStrictEncryptJdbcSourceAcceptanceTest.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mssql; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.commons.string.Strings; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import java.sql.JDBCType; -import java.util.function.Function; -import javax.sql.DataSource; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.MSSQLServerContainer; - -public class MssqlStrictEncryptJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { - - private static MSSQLServerContainer dbContainer; - private static DataSource dataSource; - private JsonNode config; - - @BeforeAll - static void init() { - // In mssql, timestamp is generated automatically, so we need to use - // the datetime type instead so that we can set the value manually. - COL_TIMESTAMP_TYPE = "DATETIME"; - - dbContainer = new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2019-latest").acceptLicense(); - dbContainer.start(); - } - - @BeforeEach - public void setup() throws Exception { - final JsonNode configWithoutDbName = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, dbContainer.getHost()) - .put(JdbcUtils.PORT_KEY, dbContainer.getFirstMappedPort()) - .put(JdbcUtils.USERNAME_KEY, dbContainer.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, dbContainer.getPassword()) - .build()); - - dataSource = DataSourceFactory.create( - configWithoutDbName.get(JdbcUtils.USERNAME_KEY).asText(), - configWithoutDbName.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.MSSQLSERVER.getDriverClassName(), - String.format("jdbc:sqlserver://%s:%d", - configWithoutDbName.get(JdbcUtils.HOST_KEY).asText(), - configWithoutDbName.get(JdbcUtils.PORT_KEY).asInt())); - - try { - database = new DefaultJdbcDatabase(dataSource); - - final String dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); - - database.execute(ctx -> ctx.createStatement().execute(String.format("CREATE DATABASE %s;", dbName))); - - config = Jsons.clone(configWithoutDbName); - ((ObjectNode) config).put(JdbcUtils.DATABASE_KEY, dbName); - - super.setup(); - } finally { - DataSourceFactory.close(dataSource); - } - } - - @AfterAll - public static void cleanUp() throws Exception { - dbContainer.close(); - } - - @Override - public boolean supportsSchemas() { - return true; - } - - @Override - public JsonNode getConfig() { - return config; - } - - @Override - public Function getToDatabaseConfigFunction() { - return new MssqlSource()::toDatabaseConfig; - } - - @Override - public String getDriverClass() { - return MssqlSource.DRIVER_CLASS; - } - - @Override - public AbstractJdbcSource getJdbcSource() { - return null; - } - - @Override - public Source getSource() { - return new MssqlSourceStrictEncrypt(); - } - - @Test - void testSpec() throws Exception { - final ConnectorSpecification actual = source.spec(); - final ConnectorSpecification expected = - SshHelpers.injectSshIntoSpec(Jsons.deserialize(MoreResources.readResource("expected_spec.json"), ConnectorSpecification.class)); - - assertEquals(expected, actual); - } - -} diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json deleted file mode 100644 index 55ef10232e9c..000000000000 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json +++ /dev/null @@ -1,158 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/destinations/mssql", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MSSQL Source Spec", - "type": "object", - "required": ["host", "port", "database", "username"], - "properties": { - "host": { - "description": "The hostname of the database.", - "title": "Host", - "type": "string", - "order": 0 - }, - "port": { - "description": "The port of the database.", - "title": "Port", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "examples": ["1433"], - "order": 1 - }, - "database": { - "description": "The name of the database.", - "title": "Database", - "type": "string", - "examples": ["master"], - "order": 2 - }, - "schemas": { - "title": "Schemas", - "description": "The list of schemas to sync from. Defaults to user. Case sensitive.", - "type": "array", - "items": { - "type": "string" - }, - "minItems": 0, - "uniqueItems": true, - "default": ["dbo"], - "order": 3 - }, - "username": { - "description": "The username which is used to access the database.", - "title": "Username", - "type": "string", - "order": 4 - }, - "password": { - "description": "The password associated with the username.", - "title": "Password", - "type": "string", - "airbyte_secret": true, - "order": 5 - }, - "jdbc_url_params": { - "title": "JDBC URL Params", - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", - "type": "string", - "order": 6 - }, - "ssl_method": { - "title": "SSL Method", - "type": "object", - "description": "The encryption method which is used when communicating with the database.", - "order": 7, - "oneOf": [ - { - "title": "Encrypted (trust server certificate)", - "description": "Use the certificate provided by the server without verification. (For testing purposes only!)", - "required": ["ssl_method"], - "properties": { - "ssl_method": { - "type": "string", - "const": "encrypted_trust_server_certificate" - } - } - }, - { - "title": "Encrypted (verify certificate)", - "description": "Verify and use the certificate provided by the server.", - "required": ["ssl_method", "trustStoreName", "trustStorePassword"], - "properties": { - "ssl_method": { - "type": "string", - "const": "encrypted_verify_certificate" - }, - "hostNameInCertificate": { - "title": "Host Name In Certificate", - "type": "string", - "description": "Specifies the host name of the server. The value of this property must match the subject property of the certificate.", - "order": 7 - } - } - } - ] - }, - "replication_method": { - "type": "object", - "title": "Replication Method", - "description": "The replication method used for extracting data from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses {TBC} to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", - "default": "STANDARD", - "order": 8, - "oneOf": [ - { - "title": "Standard", - "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "STANDARD", - "order": 0 - } - } - }, - { - "title": "Logical Replication (CDC)", - "description": "CDC uses {TBC} to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "CDC", - "order": 0 - }, - "data_to_sync": { - "title": "Data to Sync", - "type": "string", - "default": "Existing and New", - "enum": ["Existing and New", "New Changes Only"], - "description": "What data should be synced under the CDC. \"Existing and New\" will read existing data as a snapshot, and sync new changes through CDC. \"New Changes Only\" will skip the initial snapshot, and only sync new changes through CDC.", - "order": 1 - }, - "snapshot_isolation": { - "title": "Initial Snapshot Isolation Level", - "type": "string", - "default": "Snapshot", - "enum": ["Snapshot", "Read Committed"], - "description": "Existing data in the database are synced through an initial snapshot. This parameter controls the isolation level that will be used during the initial snapshotting. If you choose the \"Snapshot\" level, you must enable the snapshot isolation mode on the database.", - "order": 2 - }, - "initial_waiting_seconds": { - "type": "integer", - "title": "Initial Waiting Time in Seconds (Advanced)", - "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", - "default": 300, - "min": 120, - "max": 1200, - "order": 3 - } - } - } - ] - } - } - } -} diff --git a/airbyte-integrations/connectors/source-mssql/.dockerignore b/airbyte-integrations/connectors/source-mssql/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-mssql/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-mssql/Dockerfile b/airbyte-integrations/connectors/source-mssql/Dockerfile deleted file mode 100644 index 790035a77ffb..000000000000 --- a/airbyte-integrations/connectors/source-mssql/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-mssql - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-mssql - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=2.0.0 -LABEL io.airbyte.name=airbyte/source-mssql diff --git a/airbyte-integrations/connectors/source-mssql/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-mssql/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-mssql/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-mssql/build.gradle b/airbyte-integrations/connectors/source-mssql/build.gradle index 050978a5e7ad..b0df535e6141 100644 --- a/airbyte-integrations/connectors/source-mssql/build.gradle +++ b/airbyte-integrations/connectors/source-mssql/build.gradle @@ -1,11 +1,22 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' - id 'airbyte-performance-test-java' - id 'airbyte-connector-acceptance-test' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.10.4' + features = ['db-sources'] + useLocalCdk = false +} + +configurations.all { + resolutionStrategy { + force libs.jooq + } +} + + + application { mainClass = 'io.airbyte.integrations.source.mssql.MssqlSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] @@ -14,29 +25,15 @@ application { dependencies { implementation libs.postgresql - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:bases:debezium') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation project(':airbyte-integrations:connectors:source-relational-db') - implementation libs.debezium.sqlserver + implementation libs.debezium.embedded implementation 'com.microsoft.sqlserver:mssql-jdbc:10.2.1.jre8' implementation 'org.codehaus.plexus:plexus-utils:3.4.2' - testImplementation testFixtures(project(':airbyte-integrations:bases:debezium')) - testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) - testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation libs.connectors.testcontainers.mssqlserver + testImplementation 'org.hamcrest:hamcrest-all:1.3' + testImplementation 'org.awaitility:awaitility:4.2.0' - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - performanceTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-mssql') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - performanceTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + testImplementation libs.testcontainers.mssqlserver + testFixturesImplementation libs.testcontainers.mssqlserver } - diff --git a/airbyte-integrations/connectors/source-mssql/gradle.properties b/airbyte-integrations/connectors/source-mssql/gradle.properties new file mode 100644 index 000000000000..8ef098d20b92 --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/gradle.properties @@ -0,0 +1 @@ +testExecutionConcurrency=-1 \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mssql/metadata.yaml b/airbyte-integrations/connectors/source-mssql/metadata.yaml index 5c3d7d163211..9c04f3be8493 100644 --- a/airbyte-integrations/connectors/source-mssql/metadata.yaml +++ b/airbyte-integrations/connectors/source-mssql/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 - dockerImageTag: 2.0.0 + dockerImageTag: 3.6.0 dockerRepository: airbyte/source-mssql documentationUrl: https://docs.airbyte.com/integrations/sources/mssql githubIssueLabel: source-mssql @@ -18,7 +18,6 @@ data: name: Microsoft SQL Server (MSSQL) registries: cloud: - dockerRepository: airbyte/source-mssql-strict-encrypt enabled: true oss: enabled: true @@ -29,6 +28,9 @@ data: - language:python releases: breakingChanges: + 3.0.0: + message: "Remapped columns of types: date, datetime, datetime2, datetimeoffset, smalldatetime, and time from `String` to their appropriate Airbyte types. Customers whose streams have columns with the affected data types must take action with their connections." + upgradeDeadline: "2023-12-07" 2.0.0: message: "Add default cursor for cdc" upgradeDeadline: "2023-08-23" diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcConnectorMetadataInjector.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcConnectorMetadataInjector.java index be19a7df5d9a..050e4be20d37 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcConnectorMetadataInjector.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcConnectorMetadataInjector.java @@ -10,7 +10,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.integrations.debezium.CdcMetadataInjector; +import io.airbyte.cdk.integrations.debezium.CdcMetadataInjector; import java.time.Instant; import java.util.concurrent.atomic.AtomicLong; @@ -49,6 +49,11 @@ public String namespace(final JsonNode source) { return source.get("schema").asText(); } + @Override + public String name(JsonNode source) { + return source.get("table").asText(); + } + private Long getCdcDefaultCursor() { return this.emittedAtConverted + this.recordCounter.getAndIncrement(); } diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java index bc15f1bcb8c7..840215a69747 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java @@ -6,13 +6,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.debezium.internals.mssql.MSSQLConverter; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.SyncMode; +import java.time.Duration; import java.util.Properties; import java.util.stream.Collectors; import org.codehaus.plexus.util.StringUtils; @@ -27,77 +27,17 @@ public class MssqlCdcHelper { private static final String REPLICATION_FIELD = "replication"; private static final String REPLICATION_TYPE_FIELD = "replication_type"; private static final String METHOD_FIELD = "method"; - private static final String CDC_SNAPSHOT_ISOLATION_FIELD = "snapshot_isolation"; - private static final String CDC_DATA_TO_SYNC_FIELD = "data_to_sync"; + + private static final Duration HEARTBEAT_INTERVAL = Duration.ofSeconds(10L); + + // Test execution latency is lower when heartbeats are more frequent. + private static final Duration HEARTBEAT_INTERVAL_IN_TESTS = Duration.ofSeconds(1L); public enum ReplicationMethod { STANDARD, CDC } - /** - * The default "SNAPSHOT" mode can prevent other (non-Airbyte) transactions from updating table rows - * while we snapshot. References: - * https://docs.microsoft.com/en-us/sql/t-sql/statements/set-transaction-isolation-level-transact-sql?view=sql-server-ver15 - * https://debezium.io/documentation/reference/2.2/connectors/sqlserver.html#sqlserver-property-snapshot-isolation-mode - */ - public enum SnapshotIsolation { - - SNAPSHOT("Snapshot", "snapshot"), - READ_COMMITTED("Read Committed", "read_committed"); - - private final String snapshotIsolationLevel; - private final String debeziumIsolationMode; - - SnapshotIsolation(final String snapshotIsolationLevel, final String debeziumIsolationMode) { - this.snapshotIsolationLevel = snapshotIsolationLevel; - this.debeziumIsolationMode = debeziumIsolationMode; - } - - public String getDebeziumIsolationMode() { - return debeziumIsolationMode; - } - - public static SnapshotIsolation from(final String jsonValue) { - for (final SnapshotIsolation value : values()) { - if (value.snapshotIsolationLevel.equalsIgnoreCase(jsonValue)) { - return value; - } - } - throw new IllegalArgumentException("Unexpected snapshot isolation level: " + jsonValue); - } - - } - - // https://debezium.io/documentation/reference/2.2/connectors/sqlserver.html#sqlserver-property-snapshot-mode - public enum DataToSync { - - EXISTING_AND_NEW("Existing and New", "initial"), - NEW_CHANGES_ONLY("New Changes Only", "schema_only"); - - private final String dataToSyncConfig; - private final String debeziumSnapshotMode; - - DataToSync(final String value, final String debeziumSnapshotMode) { - this.dataToSyncConfig = value; - this.debeziumSnapshotMode = debeziumSnapshotMode; - } - - public String getDebeziumSnapshotMode() { - return debeziumSnapshotMode; - } - - public static DataToSync from(final String value) { - for (final DataToSync s : values()) { - if (s.dataToSyncConfig.equalsIgnoreCase(value)) { - return s; - } - } - throw new IllegalArgumentException("Unexpected data to sync setting: " + value); - } - - } - @VisibleForTesting static boolean isCdc(final JsonNode config) { // new replication method config since version 0.4.0 @@ -117,29 +57,7 @@ static boolean isCdc(final JsonNode config) { return false; } - @VisibleForTesting - static SnapshotIsolation getSnapshotIsolationConfig(final JsonNode config) { - // new replication method config since version 0.4.0 - if (config.hasNonNull(LEGACY_REPLICATION_FIELD) && config.get(LEGACY_REPLICATION_FIELD).isObject()) { - final JsonNode replicationConfig = config.get(LEGACY_REPLICATION_FIELD); - final JsonNode snapshotIsolation = replicationConfig.get(CDC_SNAPSHOT_ISOLATION_FIELD); - return SnapshotIsolation.from(snapshotIsolation.asText()); - } - return SnapshotIsolation.SNAPSHOT; - } - - @VisibleForTesting - static DataToSync getDataToSyncConfig(final JsonNode config) { - // new replication method config since version 0.4.0 - if (config.hasNonNull(LEGACY_REPLICATION_FIELD) && config.get(LEGACY_REPLICATION_FIELD).isObject()) { - final JsonNode replicationConfig = config.get(LEGACY_REPLICATION_FIELD); - final JsonNode dataToSync = replicationConfig.get(CDC_DATA_TO_SYNC_FIELD); - return DataToSync.from(dataToSync.asText()); - } - return DataToSync.EXISTING_AND_NEW; - } - - static Properties getDebeziumProperties(final JdbcDatabase database, final ConfiguredAirbyteCatalog catalog) { + static Properties getDebeziumProperties(final JdbcDatabase database, final ConfiguredAirbyteCatalog catalog, final boolean isSnapshot) { final JsonNode config = database.getSourceConfig(); final JsonNode dbConfig = database.getDatabaseConfig(); @@ -152,14 +70,33 @@ static Properties getDebeziumProperties(final JdbcDatabase database, final Confi props.setProperty("provide.transaction.metadata", "false"); props.setProperty("converters", "mssql_converter"); - props.setProperty("mssql_converter.type", MSSQLConverter.class.getName()); - props.setProperty("snapshot.mode", getDataToSyncConfig(config).getDebeziumSnapshotMode()); - props.setProperty("snapshot.isolation.mode", getSnapshotIsolationConfig(config).getDebeziumIsolationMode()); + props.setProperty("mssql_converter.type", MssqlDebeziumConverter.class.getName()); + + // If new stream(s) are added after a previously successful sync, + // the snapshot.mode needs to be initial_only since we don't want to continue streaming changes + // https://debezium.io/documentation/reference/stable/connectors/sqlserver.html#sqlserver-property-snapshot-mode + if (isSnapshot) { + props.setProperty("snapshot.mode", "initial_only"); + } else { + // If not in snapshot mode, initial will make sure that a snapshot is taken if the transaction log + // is rotated out. This will also end up read streaming changes from the transaction_log. + props.setProperty("snapshot.mode", "initial"); + } + + props.setProperty("snapshot.isolation.mode", "read_committed"); props.setProperty("schema.include.list", getSchema(catalog)); props.setProperty("database.names", config.get(JdbcUtils.DATABASE_KEY).asText()); + final Duration heartbeatInterval = + (database.getSourceConfig().has("is_test") && database.getSourceConfig().get("is_test").asBoolean()) + ? HEARTBEAT_INTERVAL_IN_TESTS + : HEARTBEAT_INTERVAL; + props.setProperty("heartbeat.interval.ms", Long.toString(heartbeatInterval.toMillis())); + // TODO: enable heartbeats in MS SQL Server. + props.setProperty("heartbeat.interval.ms", "0"); + if (config.has("ssl_method")) { final JsonNode sslConfig = config.get("ssl_method"); final String sslMethod = sslConfig.get("ssl_method").asText(); @@ -170,16 +107,17 @@ static Properties getDebeziumProperties(final JdbcDatabase database, final Confi props.setProperty("driver.trustServerCertificate", "true"); } else if ("encrypted_verify_certificate".equals(sslMethod)) { props.setProperty("driver.encrypt", "true"); + props.setProperty("driver.trustServerCertificate", "false"); if (dbConfig.has("trustStore") && !dbConfig.get("trustStore").asText().isEmpty()) { - props.setProperty("database.ssl.truststore", dbConfig.get("trustStore").asText()); + props.setProperty("database.trustStore", dbConfig.get("trustStore").asText()); } if (dbConfig.has("trustStorePassword") && !dbConfig.get("trustStorePassword").asText().isEmpty()) { - props.setProperty("database.ssl.truststore.password", dbConfig.get("trustStorePassword").asText()); + props.setProperty("database.trustStorePassword", dbConfig.get("trustStorePassword").asText()); } if (dbConfig.has("hostNameInCertificate") && !dbConfig.get("hostNameInCertificate").asText().isEmpty()) { - props.setProperty("driver.hostNameInCertificate", dbConfig.get("hostNameInCertificate").asText()); + props.setProperty("database.hostNameInCertificate", dbConfig.get("hostNameInCertificate").asText()); } } } diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcSavedInfoFetcher.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcSavedInfoFetcher.java index ecbff0db084e..921a0178e185 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcSavedInfoFetcher.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcSavedInfoFetcher.java @@ -4,23 +4,28 @@ package io.airbyte.integrations.source.mssql; +import static io.airbyte.integrations.source.mssql.MssqlSource.IS_COMPRESSED; import static io.airbyte.integrations.source.mssql.MssqlSource.MSSQL_CDC_OFFSET; import static io.airbyte.integrations.source.mssql.MssqlSource.MSSQL_DB_HISTORY; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.debezium.CdcSavedInfoFetcher; -import io.airbyte.integrations.source.relationaldb.models.CdcState; +import io.airbyte.cdk.integrations.debezium.CdcSavedInfoFetcher; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteSchemaHistoryStorage.SchemaHistory; +import io.airbyte.cdk.integrations.source.relationaldb.models.CdcState; import java.util.Optional; public class MssqlCdcSavedInfoFetcher implements CdcSavedInfoFetcher { private final JsonNode savedOffset; private final JsonNode savedSchemaHistory; + private final boolean isSavedSchemaHistoryCompressed; protected MssqlCdcSavedInfoFetcher(final CdcState savedState) { final boolean savedStatePresent = savedState != null && savedState.getState() != null; this.savedOffset = savedStatePresent ? savedState.getState().get(MSSQL_CDC_OFFSET) : null; this.savedSchemaHistory = savedStatePresent ? savedState.getState().get(MSSQL_DB_HISTORY) : null; + this.isSavedSchemaHistoryCompressed = + savedStatePresent && savedState.getState().has(IS_COMPRESSED) && savedState.getState().get(IS_COMPRESSED).asBoolean(); } @Override @@ -29,8 +34,8 @@ public JsonNode getSavedOffset() { } @Override - public Optional getSavedSchemaHistory() { - return Optional.ofNullable(savedSchemaHistory); + public SchemaHistory> getSavedSchemaHistory() { + return new SchemaHistory<>(Optional.ofNullable(savedSchemaHistory), isSavedSchemaHistoryCompressed); } } diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcStateHandler.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcStateHandler.java index 370709517021..7b733b3d284a 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcStateHandler.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcStateHandler.java @@ -4,14 +4,16 @@ package io.airbyte.integrations.source.mssql; +import static io.airbyte.integrations.source.mssql.MssqlSource.IS_COMPRESSED; import static io.airbyte.integrations.source.mssql.MssqlSource.MSSQL_CDC_OFFSET; import static io.airbyte.integrations.source.mssql.MssqlSource.MSSQL_DB_HISTORY; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.debezium.CdcStateHandler; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteSchemaHistoryStorage.SchemaHistory; +import io.airbyte.cdk.integrations.source.relationaldb.models.CdcState; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.debezium.CdcStateHandler; -import io.airbyte.integrations.source.relationaldb.models.CdcState; -import io.airbyte.integrations.source.relationaldb.state.StateManager; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.AirbyteStateMessage; @@ -31,10 +33,11 @@ public MssqlCdcStateHandler(final StateManager stateManager) { } @Override - public AirbyteMessage saveState(final Map offset, final String dbHistory) { + public AirbyteMessage saveState(final Map offset, final SchemaHistory dbHistory) { final Map state = new HashMap<>(); state.put(MSSQL_CDC_OFFSET, offset); - state.put(MSSQL_DB_HISTORY, dbHistory); + state.put(MSSQL_DB_HISTORY, dbHistory.schema()); + state.put(IS_COMPRESSED, dbHistory.isCompressed()); final JsonNode asJson = Jsons.jsonNode(state); @@ -52,7 +55,18 @@ public AirbyteMessage saveState(final Map offset, final String d @Override public AirbyteMessage saveStateAfterCompletionOfSnapshotOfNewStreams() { - throw new RuntimeException("Snapshot of individual tables is not implemented in MSSQL"); + LOGGER.info("Snapshot of new tables is complete, saving state"); + /* + * Namespace pair is ignored by global state manager, but is needed for satisfy the API contract. + * Therefore, provide an empty optional. + */ + final AirbyteStateMessage stateMessage = stateManager.emit(Optional.empty()); + return new AirbyteMessage().withType(Type.STATE).withState(stateMessage); + } + + @Override + public boolean compressSchemaHistoryForState() { + return true; } } diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcTargetPosition.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcTargetPosition.java new file mode 100644 index 000000000000..98645cef1d04 --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcTargetPosition.java @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.base.Preconditions; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.debezium.CdcTargetPosition; +import io.airbyte.cdk.integrations.debezium.internals.ChangeEventWithMetadata; +import io.airbyte.cdk.integrations.debezium.internals.SnapshotMetadata; +import io.debezium.connector.sqlserver.Lsn; +import java.io.IOException; +import java.sql.SQLException; +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MssqlCdcTargetPosition implements CdcTargetPosition { + + private static final Logger LOGGER = LoggerFactory.getLogger(MssqlCdcTargetPosition.class); + + public static final Duration MAX_LSN_QUERY_DELAY = Duration.ZERO; + public static final Duration MAX_LSN_QUERY_DELAY_TEST = Duration.ofSeconds(1); + public final Lsn targetLsn; + + public MssqlCdcTargetPosition(final Lsn targetLsn) { + this.targetLsn = targetLsn; + } + + @Override + public boolean reachedTargetPosition(final ChangeEventWithMetadata changeEventWithMetadata) { + if (changeEventWithMetadata.isSnapshotEvent()) { + return false; + } else if (SnapshotMetadata.LAST == changeEventWithMetadata.snapshotMetadata()) { + LOGGER.info("Signalling close because Snapshot is complete"); + return true; + } else { + final Lsn recordLsn = extractLsn(changeEventWithMetadata.eventValueAsJson()); + final boolean isEventLSNAfter = targetLsn.compareTo(recordLsn) <= 0; + if (isEventLSNAfter) { + LOGGER.info("Signalling close because record's LSN : " + recordLsn + " is after target LSN : " + targetLsn); + } + return isEventLSNAfter; + } + } + + @Override + public Lsn extractPositionFromHeartbeatOffset(final Map sourceOffset) { + throw new RuntimeException("Heartbeat is not supported for MSSQL"); + } + + private Lsn extractLsn(final JsonNode valueAsJson) { + return Optional.ofNullable(valueAsJson.get("source")) + .flatMap(source -> Optional.ofNullable(source.get("commit_lsn").asText())) + .map(Lsn::valueOf) + .orElseThrow(() -> new IllegalStateException("Could not find LSN")); + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + final MssqlCdcTargetPosition that = (MssqlCdcTargetPosition) o; + return targetLsn.equals(that.targetLsn); + } + + @Override + public int hashCode() { + return targetLsn.hashCode(); + } + + public static MssqlCdcTargetPosition getTargetPosition(final JdbcDatabase database, final String dbName) { + try { + // We might have to wait a bit before querying the max_lsn to give the CDC capture job + // a chance to catch up. This is important in tests, where reads might occur in quick succession + // which might leave the CT tables (which Debezium consumes) in a stale state. + final JsonNode sourceConfig = database.getSourceConfig(); + final Duration delay = (sourceConfig != null && sourceConfig.has("is_test") && sourceConfig.get("is_test").asBoolean()) + ? MAX_LSN_QUERY_DELAY_TEST + : MAX_LSN_QUERY_DELAY; + final String maxLsnQuery = """ + USE [%s]; + WAITFOR DELAY '%02d:%02d:%02d'; + SELECT sys.fn_cdc_get_max_lsn() AS max_lsn; + """.formatted(dbName, delay.toHours(), delay.toMinutesPart(), delay.toSecondsPart()); + // Query the high-water mark. + final List jsonNodes = database.bufferedResultSetQuery( + connection -> connection.createStatement().executeQuery(maxLsnQuery), + JdbcUtils.getDefaultSourceOperations()::rowToJson); + Preconditions.checkState(jsonNodes.size() == 1); + if (jsonNodes.get(0).get("max_lsn") != null) { + final Lsn maxLsn = Lsn.valueOf(jsonNodes.get(0).get("max_lsn").binaryValue()); + LOGGER.info("identified target lsn: " + maxLsn); + return new MssqlCdcTargetPosition(maxLsn); + } else { + throw new RuntimeException("SQL returned max LSN as null, this might be because the SQL Server Agent is not running. " + + "Please enable the Agent and try again (https://docs.microsoft.com/en-us/sql/ssms/agent/start-stop-or-pause-the-sql-server-agent-service)"); + } + } catch (final SQLException | IOException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlDebeziumConverter.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlDebeziumConverter.java new file mode 100644 index 000000000000..69ffcf2c0499 --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlDebeziumConverter.java @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql; + +import com.microsoft.sqlserver.jdbc.Geography; +import com.microsoft.sqlserver.jdbc.Geometry; +import com.microsoft.sqlserver.jdbc.SQLServerException; +import io.airbyte.cdk.db.DataTypeUtils; +import io.airbyte.cdk.db.jdbc.DateTimeConverter; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumConverterUtils; +import io.debezium.spi.converter.CustomConverter; +import io.debezium.spi.converter.RelationalColumn; +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Objects; +import java.util.Properties; +import java.util.Set; +import microsoft.sql.DateTimeOffset; +import org.apache.commons.codec.binary.Base64; +import org.apache.kafka.connect.data.SchemaBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MssqlDebeziumConverter implements CustomConverter { + + private final Logger LOGGER = LoggerFactory.getLogger(MssqlDebeziumConverter.class); + + private final Set BINARY = Set.of("VARBINARY", "BINARY"); + private final Set DATETIME_TYPES = Set.of("DATETIME", "DATETIME2", "SMALLDATETIME"); + private final String DATE = "DATE"; + private static final String DATETIMEOFFSET = "DATETIMEOFFSET"; + private static final String TIME_TYPE = "TIME"; + private static final String SMALLMONEY_TYPE = "SMALLMONEY"; + private static final String GEOMETRY = "GEOMETRY"; + private static final String GEOGRAPHY = "GEOGRAPHY"; + private static final String DEBEZIUM_DATETIMEOFFSET_FORMAT = "yyyy-MM-dd HH:mm:ss XXX"; + + private static final String DATETIME_FORMAT_MICROSECONDS = "yyyy-MM-dd'T'HH:mm:ss[.][SSSSSS]"; + + @Override + public void configure(Properties props) {} + + @Override + public void converterFor(final RelationalColumn field, + final ConverterRegistration registration) { + if (DATE.equalsIgnoreCase(field.typeName())) { + registerDate(field, registration); + } else if (DATETIME_TYPES.contains(field.typeName().toUpperCase())) { + registerDatetime(field, registration); + } else if (SMALLMONEY_TYPE.equalsIgnoreCase(field.typeName())) { + registerMoney(field, registration); + } else if (BINARY.contains(field.typeName().toUpperCase())) { + registerBinary(field, registration); + } else if (GEOMETRY.equalsIgnoreCase(field.typeName())) { + registerGeometry(field, registration); + } else if (GEOGRAPHY.equalsIgnoreCase(field.typeName())) { + registerGeography(field, registration); + } else if (TIME_TYPE.equalsIgnoreCase(field.typeName())) { + registerTime(field, registration); + } else if (DATETIMEOFFSET.equalsIgnoreCase(field.typeName())) { + registerDateTimeOffSet(field, registration); + } + } + + private void registerGeometry(final RelationalColumn field, + final ConverterRegistration registration) { + registration.register(SchemaBuilder.string(), input -> { + if (Objects.isNull(input)) { + return DebeziumConverterUtils.convertDefaultValue(field); + } + + if (input instanceof byte[]) { + try { + return Geometry.deserialize((byte[]) input).toString(); + } catch (SQLServerException e) { + LOGGER.error(e.getMessage()); + } + } + + LOGGER.warn("Uncovered Geometry class type '{}'. Use default converter", + input.getClass().getName()); + return input.toString(); + }); + } + + private void registerGeography(final RelationalColumn field, + final ConverterRegistration registration) { + registration.register(SchemaBuilder.string(), input -> { + if (Objects.isNull(input)) { + return DebeziumConverterUtils.convertDefaultValue(field); + } + + if (input instanceof byte[]) { + try { + return Geography.deserialize((byte[]) input).toString(); + } catch (SQLServerException e) { + LOGGER.error(e.getMessage()); + } + } + + LOGGER.warn("Uncovered Geography class type '{}'. Use default converter", + input.getClass().getName()); + return input.toString(); + }); + } + + private void registerDate(final RelationalColumn field, + final ConverterRegistration registration) { + registration.register(SchemaBuilder.string(), input -> { + if (Objects.isNull(input)) { + return DebeziumConverterUtils.convertDefaultValue(field); + } + if (field.typeName().equalsIgnoreCase("DATE")) { + return DateTimeConverter.convertToDate(input); + } + return DateTimeConverter.convertToTimestamp(input); + }); + } + + private void registerDatetime(final RelationalColumn field, + final ConverterRegistration registration) { + registration.register(SchemaBuilder.string(), + input -> { + if (Objects.isNull(input)) { + return DebeziumConverterUtils.convertDefaultValue(field); + } + + final LocalDateTime localDateTime = ((Timestamp) input).toLocalDateTime(); + return localDateTime.format(DateTimeFormatter.ofPattern(DATETIME_FORMAT_MICROSECONDS)); + }); + + } + + private void registerDateTimeOffSet(final RelationalColumn field, + final ConverterRegistration registration) { + registration.register(SchemaBuilder.string(), input -> { + if (Objects.isNull(input)) { + return DebeziumConverterUtils.convertDefaultValue(field); + } + + if (input instanceof DateTimeOffset) { + return DataTypeUtils.toISO8601String( + OffsetDateTime.parse(input.toString(), + DateTimeFormatter.ofPattern(DEBEZIUM_DATETIMEOFFSET_FORMAT))); + } + + LOGGER.warn("Uncovered DateTimeOffSet class type '{}'. Use default converter", + input.getClass().getName()); + return input.toString(); + }); + } + + private void registerTime(final RelationalColumn field, + final ConverterRegistration registration) { + registration.register(SchemaBuilder.string(), input -> { + if (Objects.isNull(input)) { + return DebeziumConverterUtils.convertDefaultValue(field); + } + + if (input instanceof Timestamp) { + return DataTypeUtils.toISOTimeString(((Timestamp) input).toLocalDateTime()); + } + + LOGGER.warn("Uncovered time class type '{}'. Use default converter", + input.getClass().getName()); + return input.toString(); + }); + } + + private void registerMoney(final RelationalColumn field, + final ConverterRegistration registration) { + registration.register(SchemaBuilder.float64(), input -> { + if (Objects.isNull(input)) { + return DebeziumConverterUtils.convertDefaultValue(field); + } + + if (input instanceof BigDecimal) { + return ((BigDecimal) input).doubleValue(); + } + + LOGGER.warn("Uncovered money class type '{}'. Use default converter", + input.getClass().getName()); + return input.toString(); + }); + } + + private void registerBinary(final RelationalColumn field, + final ConverterRegistration registration) { + registration.register(SchemaBuilder.string(), input -> { + if (Objects.isNull(input)) { + return DebeziumConverterUtils.convertDefaultValue(field); + } + + if (input instanceof byte[]) { + return Base64.encodeBase64String((byte[]) input); + } + + LOGGER.warn("Uncovered binary class type '{}'. Use default converter", + input.getClass().getName()); + return input.toString(); + }); + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java index 2c72ba028abe..229a5c994080 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java @@ -4,48 +4,58 @@ package io.airbyte.integrations.source.mssql; -import static io.airbyte.integrations.debezium.AirbyteDebeziumHandler.shouldUseCDC; -import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; -import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; -import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifier; -import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifierList; -import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; -import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.getIdentifierWithQuoting; -import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.queryTable; +import static io.airbyte.cdk.integrations.debezium.AirbyteDebeziumHandler.isAnyStreamIncrementalSyncMode; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifier; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifierList; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.getIdentifierWithQuoting; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.queryTable; import static java.util.stream.Collectors.toList; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.microsoft.sqlserver.jdbc.SQLServerResultSetMetaData; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; +import io.airbyte.cdk.db.util.SSLCertificateUtils; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.base.adaptive.AdaptiveSourceRunner; +import io.airbyte.cdk.integrations.base.ssh.SshWrappedSource; +import io.airbyte.cdk.integrations.debezium.AirbyteDebeziumHandler; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager; +import io.airbyte.cdk.integrations.debezium.internals.RecordWaitTimeUtil; +import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.cdk.integrations.source.relationaldb.TableInfo; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; import io.airbyte.commons.functional.CheckedConsumer; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.base.ssh.SshWrappedSource; -import io.airbyte.integrations.debezium.AirbyteDebeziumHandler; -import io.airbyte.integrations.debezium.internals.FirstRecordWaitTimeUtil; -import io.airbyte.integrations.debezium.internals.mssql.MssqlCdcTargetPosition; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.mssql.MssqlCdcHelper.SnapshotIsolation; -import io.airbyte.integrations.source.relationaldb.TableInfo; -import io.airbyte.integrations.source.relationaldb.state.StateManager; +import io.airbyte.commons.util.AutoCloseableIterators; import io.airbyte.protocol.models.CommonField; import io.airbyte.protocol.models.v0.AirbyteCatalog; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.SyncMode; import io.debezium.connector.sqlserver.Lsn; -import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.security.cert.CertificateException; import java.sql.Connection; import java.sql.JDBCType; import java.sql.PreparedStatement; @@ -55,14 +65,13 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.OptionalInt; import java.util.Set; -import java.util.function.Supplier; +import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,24 +86,59 @@ public class MssqlSource extends AbstractJdbcSource implements Source """ SELECT CAST(IIF(EXISTS(SELECT TOP 1 1 FROM "%s"."%s" WHERE "%s" IS NULL), 1, 0) AS BIT) AS %s """; - static final String DRIVER_CLASS = DatabaseDriver.MSSQLSERVER.getDriverClassName(); + public static final String DRIVER_CLASS = DatabaseDriver.MSSQLSERVER.getDriverClassName(); public static final String MSSQL_CDC_OFFSET = "mssql_cdc_offset"; public static final String MSSQL_DB_HISTORY = "mssql_db_history"; + public static final String IS_COMPRESSED = "is_compressed"; public static final String CDC_LSN = "_ab_cdc_lsn"; public static final String CDC_EVENT_SERIAL_NO = "_ab_cdc_event_serial_no"; private static final String HIERARCHYID = "hierarchyid"; private static final int INTERMEDIATE_STATE_EMISSION_FREQUENCY = 10_000; public static final String CDC_DEFAULT_CURSOR = "_ab_cdc_cursor"; + public static final String TUNNEL_METHOD = "tunnel_method"; + public static final String NO_TUNNEL = "NO_TUNNEL"; + public static final String SSL_METHOD = "ssl_method"; + public static final String SSL_METHOD_UNENCRYPTED = "unencrypted"; + + public static final String JDBC_DELIMITER = ";"; private List schemas; - public static Source sshWrappedSource() { - return new SshWrappedSource(new MssqlSource(), JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY); + public static Source sshWrappedSource(final MssqlSource source) { + return new SshWrappedSource(source, JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY); } - MssqlSource() { + public MssqlSource() { super(DRIVER_CLASS, AdaptiveStreamingQueryConfig::new, new MssqlSourceOperations()); } + @Override + protected AirbyteStateType getSupportedStateType(final JsonNode config) { + return MssqlCdcHelper.isCdc(config) ? AirbyteStateType.GLOBAL : AirbyteStateType.STREAM; + } + + @Override + public AirbyteConnectionStatus check(final JsonNode config) throws Exception { + // #15808 Disallow connecting to db with disable, prefer or allow SSL mode when connecting directly + // and not over SSH tunnel + if (cloudDeploymentMode()) { + if (config.has(TUNNEL_METHOD) + && config.get(TUNNEL_METHOD).has(TUNNEL_METHOD) + && config.get(TUNNEL_METHOD).get(TUNNEL_METHOD).asText().equals(NO_TUNNEL)) { + // If no SSH tunnel. + if (config.has(SSL_METHOD) && config.get(SSL_METHOD).has(SSL_METHOD) && + SSL_METHOD_UNENCRYPTED.equalsIgnoreCase(config.get(SSL_METHOD).get(SSL_METHOD).asText())) { + // Fail in case SSL method is unencrypted. + return new AirbyteConnectionStatus() + .withStatus(AirbyteConnectionStatus.Status.FAILED) + .withMessage("Unsecured connection not allowed. " + + "If no SSH Tunnel set up, please use one of the following SSL methods: " + + "encrypted_trust_server_certificate, encrypted_verify_certificate."); + } + } + } + return super.check(config); + } + @Override public AutoCloseableIterator queryTableFullRefresh(final JdbcDatabase database, final List columnNames, @@ -327,7 +371,6 @@ public List> getCheckOperations(final J checkOperations.add(database -> assertCdcEnabledInDb(config, database)); checkOperations.add(database -> assertCdcSchemaQueryable(config, database)); checkOperations.add(database -> assertSqlServerAgentRunning(database)); - checkOperations.add(database -> assertSnapshotIsolationAllowed(config, database)); } return checkOperations; @@ -411,61 +454,64 @@ protected void assertSqlServerAgentRunning(final JdbcDatabase database) throws S } } - protected void assertSnapshotIsolationAllowed(final JsonNode config, final JdbcDatabase database) - throws SQLException { - if (MssqlCdcHelper.getSnapshotIsolationConfig(config) != SnapshotIsolation.SNAPSHOT) { - return; - } - - final List queryResponse = database.queryJsons(connection -> { - final String sql = "SELECT name, snapshot_isolation_state FROM sys.databases WHERE name = ?"; - final PreparedStatement ps = connection.prepareStatement(sql); - ps.setString(1, config.get(JdbcUtils.DATABASE_KEY).asText()); - LOGGER.info(String.format( - "Checking that snapshot isolation is enabled on database '%s' using the query: '%s'", - config.get(JdbcUtils.DATABASE_KEY).asText(), sql)); - return ps; - }, sourceOperations::rowToJson); - - if (queryResponse.size() < 1) { - throw new RuntimeException(String.format( - "Couldn't find '%s' in sys.databases table. Please check the spelling and that the user has relevant permissions (see docs).", - config.get(JdbcUtils.DATABASE_KEY).asText())); - } - if (queryResponse.get(0).get("snapshot_isolation_state").asInt() != 1) { - throw new RuntimeException(String.format( - "Detected that snapshot isolation is not enabled for database '%s'. MSSQL CDC relies on snapshot isolation. " - + "Please check the documentation on how to enable snapshot isolation on MS SQL Server.", - config.get(JdbcUtils.DATABASE_KEY).asText())); - } - } - @Override - public List> getIncrementalIterators( - final JdbcDatabase database, + public List> getIncrementalIterators(final JdbcDatabase database, final ConfiguredAirbyteCatalog catalog, final Map>> tableNameToTable, final StateManager stateManager, final Instant emittedAt) { final JsonNode sourceConfig = database.getSourceConfig(); - if (MssqlCdcHelper.isCdc(sourceConfig) && shouldUseCDC(catalog)) { + if (MssqlCdcHelper.isCdc(sourceConfig) && isAnyStreamIncrementalSyncMode(catalog)) { LOGGER.info("using CDC: {}", true); - final Duration firstRecordWaitTime = FirstRecordWaitTimeUtil.getFirstRecordWaitTime(sourceConfig); - final AirbyteDebeziumHandler handler = - new AirbyteDebeziumHandler<>(sourceConfig, - MssqlCdcTargetPosition.getTargetPosition(database, sourceConfig.get(JdbcUtils.DATABASE_KEY).asText()), true, firstRecordWaitTime, - OptionalInt.empty()); - + final Duration firstRecordWaitTime = RecordWaitTimeUtil.getFirstRecordWaitTime(sourceConfig); + final Duration subsequentRecordWaitTime = RecordWaitTimeUtil.getSubsequentRecordWaitTime(sourceConfig); + final var targetPosition = MssqlCdcTargetPosition.getTargetPosition(database, sourceConfig.get(JdbcUtils.DATABASE_KEY).asText()); + final AirbyteDebeziumHandler handler = new AirbyteDebeziumHandler<>( + sourceConfig, + targetPosition, + true, + firstRecordWaitTime, + subsequentRecordWaitTime, + OptionalInt.empty()); final MssqlCdcConnectorMetadataInjector mssqlCdcConnectorMetadataInjector = MssqlCdcConnectorMetadataInjector.getInstance(emittedAt); - final Supplier> incrementalIteratorSupplier = () -> handler.getIncrementalIterators(catalog, + // Determine if new stream(s) have been added to the catalog after initial sync of existing streams + final List streamsToSnapshot = identifyStreamsToSnapshot(catalog, stateManager); + final ConfiguredAirbyteCatalog streamsToSnapshotCatalog = new ConfiguredAirbyteCatalog().withStreams(streamsToSnapshot); + + final Supplier> incrementalIteratorsSupplier = () -> handler.getIncrementalIterators( + catalog, new MssqlCdcSavedInfoFetcher(stateManager.getCdcStateManager().getCdcState()), new MssqlCdcStateHandler(stateManager), mssqlCdcConnectorMetadataInjector, - MssqlCdcHelper.getDebeziumProperties(database, catalog), - emittedAt, true); + MssqlCdcHelper.getDebeziumProperties(database, catalog, false), + DebeziumPropertiesManager.DebeziumConnectorType.RELATIONALDB, + emittedAt, + true); + + /* + * If the CDC state is null or there is no streams to snapshot, that means no stream has gone + * through the initial sync, so we return the list of incremental iterators + */ + if ((stateManager.getCdcStateManager().getCdcState() == null || + stateManager.getCdcStateManager().getCdcState().getState() == null || + streamsToSnapshot.isEmpty())) { + return List.of(incrementalIteratorsSupplier.get()); + } - return Collections.singletonList(incrementalIteratorSupplier.get()); + // Otherwise, we build the snapshot iterators for the newly added streams(s) + final AutoCloseableIterator snapshotIterators = + handler.getSnapshotIterators(streamsToSnapshotCatalog, + mssqlCdcConnectorMetadataInjector, + MssqlCdcHelper.getDebeziumProperties(database, catalog, true), + new MssqlCdcStateHandler(stateManager), + DebeziumPropertiesManager.DebeziumConnectorType.RELATIONALDB, + emittedAt); + /* + * The incremental iterators needs to be wrapped in a lazy iterator since only 1 Debezium engine for + * the DB can be running at a time + */ + return List.of(snapshotIterators, AutoCloseableIterators.lazyIterator(incrementalIteratorsSupplier, null)); } else { LOGGER.info("using CDC: {}", false); return super.getIncrementalIterators(database, catalog, tableNameToTable, stateManager, emittedAt); @@ -530,30 +576,33 @@ private static AirbyteStream addCdcMetadataColumns(final AirbyteStream stream) { private void readSsl(final JsonNode sslMethod, final List additionalParameters) { final JsonNode config = sslMethod.get("ssl_method"); switch (config.get("ssl_method").asText()) { - case "unencrypted" -> additionalParameters.add("encrypt=false"); + case "unencrypted" -> { + additionalParameters.add("encrypt=false"); + additionalParameters.add("trustServerCertificate=true"); + } case "encrypted_trust_server_certificate" -> { additionalParameters.add("encrypt=true"); additionalParameters.add("trustServerCertificate=true"); } case "encrypted_verify_certificate" -> { additionalParameters.add("encrypt=true"); - - // trust store location code found at https://stackoverflow.com/a/56570588 - final String trustStoreLocation = Optional - .ofNullable(System.getProperty("javax.net.ssl.trustStore")) - .orElseGet(() -> System.getProperty("java.home") + "/lib/security/cacerts"); - final File trustStoreFile = new File(trustStoreLocation); - if (!trustStoreFile.exists()) { - throw new RuntimeException( - "Unable to locate the Java TrustStore: the system property javax.net.ssl.trustStore is undefined or " - + trustStoreLocation + " does not exist."); - } - final String trustStorePassword = System.getProperty("javax.net.ssl.trustStorePassword"); - additionalParameters.add("trustStore=" + trustStoreLocation); - if (trustStorePassword != null && !trustStorePassword.isEmpty()) { + additionalParameters.add("trustServerCertificate=false"); + + if (config.has("certificate")) { + String certificate = config.get("certificate").asText(); + String password = RandomStringUtils.randomAlphanumeric(100); + final URI keyStoreUri; + try { + keyStoreUri = SSLCertificateUtils.keyStoreFromCertificate(certificate, password, null, null); + } catch (IOException | KeyStoreException | NoSuchAlgorithmException | CertificateException e) { + throw new RuntimeException(e); + } additionalParameters - .add("trustStorePassword=" + config.get("trustStorePassword").asText()); + .add("trustStore=" + keyStoreUri.getPath()); + additionalParameters + .add("trustStorePassword=" + password); } + if (config.has("hostNameInCertificate")) { additionalParameters .add("hostNameInCertificate=" + config.get("hostNameInCertificate").asText()); @@ -562,8 +611,21 @@ private void readSsl(final JsonNode sslMethod, final List additionalPara } } + private boolean cloudDeploymentMode() { + return AdaptiveSourceRunner.CLOUD_MODE.equalsIgnoreCase(featureFlags.deploymentMode()); + } + + public Duration getConnectionTimeoutMssql(final Map connectionProperties) { + return getConnectionTimeout(connectionProperties); + } + + @Override + public JdbcDatabase createDatabase(final JsonNode sourceConfig) throws SQLException { + return createDatabase(sourceConfig, JDBC_DELIMITER); + } + public static void main(final String[] args) throws Exception { - final Source source = MssqlSource.sshWrappedSource(); + final Source source = MssqlSource.sshWrappedSource(new MssqlSource()); LOGGER.info("starting source: {}", MssqlSource.class); new IntegrationRunner(source).run(args); LOGGER.info("completed source: {}", MssqlSource.class); diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java index 95728cfc2e7e..4c46601f7fcf 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSourceOperations.java @@ -4,22 +4,31 @@ package io.airbyte.integrations.source.mssql; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_SCHEMA_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_TABLE_NAME; +import static io.airbyte.cdk.db.DataTypeUtils.OFFSETDATETIME_FORMATTER; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_COLUMN_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_SCHEMA_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_TABLE_NAME; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.microsoft.sqlserver.jdbc.Geography; import com.microsoft.sqlserver.jdbc.Geometry; import com.microsoft.sqlserver.jdbc.SQLServerResultSetMetaData; -import io.airbyte.db.jdbc.JdbcSourceOperations; -import java.nio.charset.Charset; +import io.airbyte.cdk.db.jdbc.JdbcSourceOperations; +import io.airbyte.protocol.models.JsonSchemaType; import java.sql.JDBCType; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import microsoft.sql.DateTimeOffset; +import org.apache.commons.codec.binary.Base64; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -88,6 +97,19 @@ public JDBCType getDatabaseFieldType(final JsonNode field) { || typeName.equalsIgnoreCase("hierarchyid")) { return JDBCType.VARCHAR; } + + if (typeName.equalsIgnoreCase("datetime")) { + return JDBCType.TIMESTAMP; + } + + if (typeName.equalsIgnoreCase("datetimeoffset")) { + return JDBCType.TIMESTAMP_WITH_TIMEZONE; + } + + if (typeName.equalsIgnoreCase("real")) { + return JDBCType.REAL; + } + return JDBCType.valueOf(field.get(INTERNAL_COLUMN_TYPE).asInt()); } catch (final IllegalArgumentException ex) { LOGGER.warn(String.format("Could not convert column: %s from table: %s.%s with type: %s. Casting to VARCHAR.", @@ -106,7 +128,7 @@ protected void putBinary(final ObjectNode node, final int index) throws SQLException { final byte[] bytes = resultSet.getBytes(index); - final String value = new String(bytes, Charset.defaultCharset()); + final String value = Base64.encodeBase64String(bytes); node.put(columnName, value); } @@ -126,4 +148,43 @@ protected void putGeography(final ObjectNode node, node.put(columnName, Geography.deserialize(resultSet.getBytes(index)).toString()); } + @Override + protected void putTimestamp(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { + final DateTimeFormatter microsecondsFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss[.][SSSSSS]"); + node.put(columnName, getObject(resultSet, index, LocalDateTime.class).format(microsecondsFormatter)); + } + + @Override + public JsonSchemaType getAirbyteType(final JDBCType jdbcType) { + return switch (jdbcType) { + case TINYINT, SMALLINT, INTEGER, BIGINT -> JsonSchemaType.INTEGER; + case DOUBLE, DECIMAL, FLOAT, NUMERIC, REAL -> JsonSchemaType.NUMBER; + case BOOLEAN, BIT -> JsonSchemaType.BOOLEAN; + case NULL -> JsonSchemaType.NULL; + case BLOB, BINARY, VARBINARY, LONGVARBINARY -> JsonSchemaType.STRING_BASE_64; + case TIME -> JsonSchemaType.STRING_TIME_WITHOUT_TIMEZONE; + case TIMESTAMP_WITH_TIMEZONE -> JsonSchemaType.STRING_TIMESTAMP_WITH_TIMEZONE; + case TIMESTAMP -> JsonSchemaType.STRING_TIMESTAMP_WITHOUT_TIMEZONE; + case DATE -> JsonSchemaType.STRING_DATE; + default -> JsonSchemaType.STRING; + }; + } + + @Override + protected void setTimestampWithTimezone(final PreparedStatement preparedStatement, final int parameterIndex, final String value) + throws SQLException { + try { + final OffsetDateTime offsetDateTime = OffsetDateTime.parse(value, OFFSETDATETIME_FORMATTER); + final Timestamp timestamp = Timestamp.valueOf(offsetDateTime.atZoneSameInstant(offsetDateTime.getOffset()).toLocalDateTime()); + // Final step of conversion from + // OffsetDateTime (a Java construct) object -> Timestamp (a Java construct) -> + // DateTimeOffset (a Microsoft.sql specific construct) + // and provide the offset in minutes to the converter + final DateTimeOffset datetimeoffset = DateTimeOffset.valueOf(timestamp, offsetDateTime.getOffset().getTotalSeconds() / 60); + preparedStatement.setObject(parameterIndex, datetimeoffset); + } catch (final DateTimeParseException e) { + throw new RuntimeException(e); + } + } + } diff --git a/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json index 9be47072c150..005311b9e5a8 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json @@ -4,7 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "MSSQL Source Spec", "type": "object", - "required": ["host", "port", "database", "username"], + "required": ["host", "port", "database", "username", "password"], "properties": { "host": { "description": "The hostname of the database.", @@ -90,7 +90,7 @@ { "title": "Encrypted (verify certificate)", "description": "Verify and use the certificate provided by the server.", - "required": ["ssl_method", "trustStoreName", "trustStorePassword"], + "required": ["ssl_method"], "properties": { "ssl_method": { "type": "string", @@ -100,7 +100,15 @@ "title": "Host Name In Certificate", "type": "string", "description": "Specifies the host name of the server. The value of this property must match the subject property of the certificate.", - "order": 7 + "order": 0 + }, + "certificate": { + "title": "Certificate", + "type": "string", + "description": "certificate of the server, or of the CA that signed the server certificate", + "order": 1, + "airbyte_secret": true, + "multiline": true } } } @@ -108,26 +116,15 @@ }, "replication_method": { "type": "object", - "title": "Replication Method", - "description": "The replication method used for extracting data from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses {TBC} to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", - "default": "STANDARD", + "title": "Update Method", + "description": "Configures how data is extracted from the database.", + "default": "CDC", + "display_type": "radio", "order": 8, "oneOf": [ { - "title": "Standard", - "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "STANDARD", - "order": 0 - } - } - }, - { - "title": "Logical Replication (CDC)", - "description": "CDC uses {TBC} to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", + "title": "Read Changes using Change Data Capture (CDC)", + "description": "Recommended - Incrementally reads new inserts, updates, and deletes using the SQL Server's change data capture feature. This must be enabled on your database.", "required": ["method"], "properties": { "method": { @@ -135,22 +132,6 @@ "const": "CDC", "order": 0 }, - "data_to_sync": { - "title": "Data to Sync", - "type": "string", - "default": "Existing and New", - "enum": ["Existing and New", "New Changes Only"], - "description": "What data should be synced under the CDC. \"Existing and New\" will read existing data as a snapshot, and sync new changes through CDC. \"New Changes Only\" will skip the initial snapshot, and only sync new changes through CDC.", - "order": 1 - }, - "snapshot_isolation": { - "title": "Initial Snapshot Isolation Level", - "type": "string", - "default": "Snapshot", - "enum": ["Snapshot", "Read Committed"], - "description": "Existing data in the database are synced through an initial snapshot. This parameter controls the isolation level that will be used during the initial snapshotting. If you choose the \"Snapshot\" level, you must enable the snapshot isolation mode on the database.", - "order": 2 - }, "initial_waiting_seconds": { "type": "integer", "title": "Initial Waiting Time in Seconds (Advanced)", @@ -161,6 +142,18 @@ "order": 3 } } + }, + { + "title": "Scan Changes with User Defined Cursor", + "description": "Incrementally detects new inserts and updates using the cursor column chosen when configuring a connection (e.g. created_at, updated_at).", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "STANDARD", + "order": 0 + } + } } ] } diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java index 902f9bd39735..32c42ebea52c 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractMssqlSourceDatatypeTest.java @@ -4,18 +4,14 @@ package io.airbyte.integrations.source.mssql; -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; -import io.airbyte.integrations.standardtest.source.TestDataHolder; +import io.airbyte.cdk.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDataHolder; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.JsonSchemaType; -import org.jooq.DSLContext; -import org.testcontainers.containers.MSSQLServerContainer; public abstract class AbstractMssqlSourceDatatypeTest extends AbstractSourceDatabaseTypeTest { - protected static MSSQLServerContainer container; - protected JsonNode config; - protected DSLContext dslContext; + protected MsSQLTestDatabase testdb; @Override protected String getNameSpace() { @@ -28,14 +24,11 @@ protected String getImageName() { } @Override - protected JsonNode getConfig() { - return config; + protected void tearDown(final TestDestinationEnv testEnv) { + testdb.close(); } - protected static final String DB_NAME = "comprehensive"; - - protected static final String CREATE_TABLE_SQL = - "USE " + DB_NAME + "\nCREATE TABLE %1$s(%2$s INTEGER PRIMARY KEY, %3$s %4$s)"; + protected static final String CREATE_TABLE_SQL = "CREATE TABLE %1$s(%2$s INTEGER PRIMARY KEY, %3$s %4$s)"; @Override protected void initTests() { @@ -142,7 +135,7 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("date") - .airbyteType(JsonSchemaType.STRING) + .airbyteType(JsonSchemaType.STRING_DATE) .addInsertValues("'0001-01-01'", "'9999-12-31'", "'1999-01-08'", "null") .addExpectedValues("0001-01-01", "9999-12-31", "1999-01-08", null) .createTablePatternSql(CREATE_TABLE_SQL) @@ -151,7 +144,7 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("smalldatetime") - .airbyteType(JsonSchemaType.STRING) + .airbyteType(JsonSchemaType.STRING_TIMESTAMP_WITHOUT_TIMEZONE) .addInsertValues("'1900-01-01'", "'2079-06-06'", "null") .addExpectedValues("1900-01-01T00:00:00.000000", "2079-06-06T00:00:00.000000", null) .createTablePatternSql(CREATE_TABLE_SQL) @@ -160,27 +153,28 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("datetime") - .airbyteType(JsonSchemaType.STRING) + .airbyteType(JsonSchemaType.STRING_TIMESTAMP_WITHOUT_TIMEZONE) .addInsertValues("'1753-01-01'", "'9999-12-31'", "'9999-12-31T13:00:04'", "'9999-12-31T13:00:04.123'", "null") - .addExpectedValues("1753-01-01T00:00:00.000000", "9999-12-31T00:00:00.000000", "9999-12-31T13:00:04", - "9999-12-31T13:00:04.123", null) + .addExpectedValues("1753-01-01T00:00:00.000000", "9999-12-31T00:00:00.000000", "9999-12-31T13:00:04.000000", + "9999-12-31T13:00:04.123000", null) .createTablePatternSql(CREATE_TABLE_SQL) .build()); addDataTypeTestData( TestDataHolder.builder() .sourceType("datetime2") - .airbyteType(JsonSchemaType.STRING) - .addInsertValues("'0001-01-01'", "'9999-12-31'", "'9999-12-31T13:00:04.123456'", "null") - .addExpectedValues("0001-01-01T00:00:00.000000", "9999-12-31T00:00:00.000000", "9999-12-31T13:00:04.123456", null) + .airbyteType(JsonSchemaType.STRING_TIMESTAMP_WITHOUT_TIMEZONE) + .addInsertValues("'0001-01-01'", "'9999-12-31'", "'9999-12-31T13:00:04.123456'", "null", "'2023-11-08T01:20:11.3733338'") + .addExpectedValues("0001-01-01T00:00:00.000000", "9999-12-31T00:00:00.000000", "9999-12-31T13:00:04.123456", null, + "2023-11-08T01:20:11.373333") .createTablePatternSql(CREATE_TABLE_SQL) .build()); addDataTypeTestData( TestDataHolder.builder() .sourceType("time") - .airbyteType(JsonSchemaType.STRING) + .airbyteType(JsonSchemaType.STRING_TIME_WITHOUT_TIMEZONE) .addInsertValues("null", "'13:00:01'", "'13:00:04Z'", "'13:00:04.123456Z'") .addExpectedValues(null, "13:00:01", "13:00:04", "13:00:04.123456") .createTablePatternSql(CREATE_TABLE_SQL) @@ -189,7 +183,7 @@ protected void initTests() { addDataTypeTestData( TestDataHolder.builder() .sourceType("datetimeoffset") - .airbyteType(JsonSchemaType.STRING) + .airbyteType(JsonSchemaType.STRING_TIMESTAMP_WITH_TIMEZONE) .addInsertValues("'0001-01-10 00:00:00 +01:00'", "'9999-01-10 00:00:00 +01:00'", "null") .addExpectedValues("0001-01-10 00:00:00.0000000 +01:00", "9999-01-10 00:00:00.0000000 +01:00", null) @@ -263,7 +257,7 @@ protected void initTests() { .sourceType("binary") .airbyteType(JsonSchemaType.STRING_BASE_64) .addInsertValues("CAST( 'A' AS BINARY(1))", "null") - .addExpectedValues("A", null) + .addExpectedValues("QQ==", null) .createTablePatternSql(CREATE_TABLE_SQL) .build()); @@ -273,7 +267,7 @@ protected void initTests() { .fullSourceDataType("varbinary(3)") .airbyteType(JsonSchemaType.STRING_BASE_64) .addInsertValues("CAST( 'ABC' AS VARBINARY)", "null") - .addExpectedValues("ABC", null) + .addExpectedValues("QUJD", null) .createTablePatternSql(CREATE_TABLE_SQL) .build()); diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractSshMssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractSshMssqlSourceAcceptanceTest.java index 02a70ecb1540..ee301aacd964 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractSshMssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/AbstractSshMssqlSourceAcceptanceTest.java @@ -5,18 +5,15 @@ package io.airbyte.integrations.source.mssql; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.base.ssh.SshBastionContainer; +import io.airbyte.cdk.integrations.base.ssh.SshHelpers; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.ssh.SshBastionContainer; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.base.ssh.SshTunnel; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.BaseImage; +import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.ContainerModifier; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.CatalogHelpers; @@ -25,91 +22,50 @@ import io.airbyte.protocol.models.v0.ConnectorSpecification; import io.airbyte.protocol.models.v0.DestinationSyncMode; import io.airbyte.protocol.models.v0.SyncMode; +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.HashMap; -import java.util.Objects; -import org.apache.commons.lang3.RandomStringUtils; -import org.jooq.DSLContext; -import org.testcontainers.containers.JdbcDatabaseContainer; -import org.testcontainers.containers.MSSQLServerContainer; -import org.testcontainers.containers.Network; public abstract class AbstractSshMssqlSourceAcceptanceTest extends SourceAcceptanceTest { private static final String STREAM_NAME = "dbo.id_and_name"; private static final String STREAM_NAME2 = "dbo.starships"; - private static final Network network = Network.newNetwork(); - private static JsonNode config; - private String dbName; - private MSSQLServerContainer db; - private final SshBastionContainer bastion = new SshBastionContainer(); public abstract SshTunnel.TunnelMethod getTunnelMethod(); - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - startTestContainers(); - config = bastion.getTunnelConfig(getTunnelMethod(), getMSSQLDbConfigBuilder(db), false); - populateDatabaseTestData(); - } - - public ImmutableMap.Builder getMSSQLDbConfigBuilder(final JdbcDatabaseContainer db) { - dbName = "db_" + RandomStringUtils.randomAlphabetic(10).toLowerCase(); - return ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, Objects.requireNonNull(db.getContainerInfo().getNetworkSettings() - .getNetworks() - .get(((Network.NetworkImpl) network).getName()) - .getIpAddress())) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) - .put(JdbcUtils.PORT_KEY, db.getExposedPorts().get(0)) - .put(JdbcUtils.DATABASE_KEY, dbName); - } - - private static Database getDatabaseFromConfig(final JsonNode config) { - final DSLContext dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.MSSQLSERVER.getDriverClassName(), - String.format("jdbc:sqlserver://%s:%d;", - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asInt()), - null); - return new Database(dslContext); - } - - private void startTestContainers() { - bastion.initAndStartBastion(network); - initAndStartJdbcContainer(); - } + protected MsSQLTestDatabase testdb; + protected SshBastionContainer bastion; - private void initAndStartJdbcContainer() { - db = new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2017-latest") - .withNetwork(network) - .acceptLicense(); - db.start(); + @Override + protected JsonNode getConfig() { + try { + return testdb.integrationTestConfigBuilder() + .with("tunnel_method", bastion.getTunnelMethod(getTunnelMethod(), false)) + .build(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } } - private void populateDatabaseTestData() throws Exception { - SshTunnel.sshWrap( - getConfig(), - JdbcUtils.HOST_LIST_KEY, - JdbcUtils.PORT_LIST_KEY, - mangledConfig -> { - getDatabaseFromConfig(mangledConfig).query(ctx -> { - ctx.fetch(String.format("CREATE DATABASE %s;", dbName)); - ctx.fetch(String.format("ALTER DATABASE %s SET AUTO_CLOSE OFF WITH NO_WAIT;", dbName)); - ctx.fetch(String.format("USE %s;", dbName)); - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200), born DATETIMEOFFSET(7));"); - ctx.fetch( - "INSERT INTO id_and_name (id, name, born) VALUES (1,'picard', '2124-03-04T01:01:01Z'), (2, 'crusher', '2124-03-04T01:01:01Z'), (3, 'vash', '2124-03-04T01:01:01Z');"); - return null; - }); - }); + @Override + protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { + testdb = MsSQLTestDatabase.in(BaseImage.MSSQL_2017, ContainerModifier.NETWORK); + testdb = testdb + .with("ALTER DATABASE %s SET AUTO_CLOSE OFF WITH NO_WAIT;", testdb.getDatabaseName()) + .with("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200), born DATETIMEOFFSET(7));") + .with("INSERT INTO id_and_name (id, name, born) VALUES " + + "(1, 'picard', '2124-03-04T01:01:01Z'), " + + "(2, 'crusher', '2124-03-04T01:01:01Z'), " + + "(3, 'vash', '2124-03-04T01:01:01Z');"); + bastion.initAndStartBastion(testdb.getContainer().getNetwork()); } @Override protected void tearDown(final TestDestinationEnv testEnv) { - bastion.stopAndCloseContainers(db); + bastion.close(); + testdb.close(); } @Override @@ -122,11 +78,6 @@ protected ConnectorSpecification getSpec() throws Exception { return SshHelpers.getSpecAndInjectSsh(); } - @Override - protected JsonNode getConfig() { - return config; - } - @Override protected ConfiguredAirbyteCatalog getConfiguredCatalog() { return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java index 3b69f34de640..a87c3916ac6d 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java @@ -4,20 +4,23 @@ package io.airbyte.integrations.source.mssql; +import static org.junit.jupiter.api.Assertions.assertEquals; + import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.base.ssh.SshHelpers; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.string.Strings; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.BaseImage; +import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.ContainerModifier; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStreamState; import io.airbyte.protocol.models.v0.CatalogHelpers; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; @@ -25,32 +28,17 @@ import io.airbyte.protocol.models.v0.DestinationSyncMode; import io.airbyte.protocol.models.v0.SyncMode; import java.util.List; -import java.util.Map; -import org.jooq.DSLContext; -import org.junit.jupiter.api.AfterAll; -import org.testcontainers.containers.MSSQLServerContainer; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Test; public class CdcMssqlSourceAcceptanceTest extends SourceAcceptanceTest { private static final String SCHEMA_NAME = "dbo"; private static final String STREAM_NAME = "id_and_name"; private static final String STREAM_NAME2 = "starships"; - private static final String TEST_USER_PASSWORD = "testerjester[1]"; private static final String CDC_ROLE_NAME = "cdc_selector"; - public static MSSQLServerContainer container; - private String dbName; - private String testUserName; - private JsonNode config; - private Database database; - private DSLContext dslContext; - - @AfterAll - public static void closeContainer() { - if (container != null) { - container.close(); - container.stop(); - } - } + + private MsSQLTestDatabase testdb; @Override protected String getImageName() { @@ -64,12 +52,19 @@ protected ConnectorSpecification getSpec() throws Exception { @Override protected JsonNode getConfig() { - return config; + return testdb.integrationTestConfigBuilder() + .withCdcReplication() + .withoutSsl() + .build(); } @Override protected ConfiguredAirbyteCatalog getConfiguredCatalog() { - return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( + return new ConfiguredAirbyteCatalog().withStreams(getConfiguredAirbyteStreams()); + } + + protected List getConfiguredAirbyteStreams() { + return Lists.newArrayList( new ConfiguredAirbyteStream() .withSyncMode(SyncMode.INCREMENTAL) .withDestinationSyncMode(DestinationSyncMode.APPEND) @@ -93,7 +88,7 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { .withSourceDefinedCursor(true) .withSourceDefinedPrimaryKey(List.of(List.of("id"))) .withSupportedSyncModes( - Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); + Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)))); } @Override @@ -102,123 +97,85 @@ protected JsonNode getState() { } @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws InterruptedException { - if (container == null) { - container = new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2019-latest").acceptLicense(); - container.addEnv("MSSQL_AGENT_ENABLED", "True"); // need this running for cdc to work - container.start(); - } - - dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); - testUserName = Strings.addRandomSuffix("test", "_", 5).toLowerCase(); - - final JsonNode replicationConfig = Jsons.jsonNode(Map.of( - "method", "CDC", - "data_to_sync", "Existing and New", - "initial_waiting_seconds", 5, - "snapshot_isolation", "Snapshot")); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, dbName) - .put(JdbcUtils.USERNAME_KEY, testUserName) - .put(JdbcUtils.PASSWORD_KEY, TEST_USER_PASSWORD) - .put("replication_method", replicationConfig) - .put("ssl_method", Jsons.jsonNode(Map.of("ssl_method", "unencrypted"))) - .build()); - - dslContext = DSLContextFactory.create(DataSourceFactory.create( - container.getUsername(), - container.getPassword(), - container.getDriverClassName(), - String.format("jdbc:sqlserver://%s:%d;", - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asInt()), - Map.of("encrypt", "false")), null); - database = new Database(dslContext); - - executeQuery("CREATE DATABASE " + dbName + ";"); - executeQuery("ALTER DATABASE " + dbName + "\n\tSET ALLOW_SNAPSHOT_ISOLATION ON"); - executeQuery("USE " + dbName + "\n" + "EXEC sys.sp_cdc_enable_db"); - - setupTestUser(); - revokeAllPermissions(); - createAndPopulateTables(); - grantCorrectPermissions(); + protected void setupEnvironment(final TestDestinationEnv environment) { + testdb = MsSQLTestDatabase.in(BaseImage.MSSQL_2022, ContainerModifier.AGENT); + final var enableCdcSqlFmt = """ + EXEC sys.sp_cdc_enable_table + \t@source_schema = N'%s', + \t@source_name = N'%s', + \t@role_name = N'%s', + \t@supports_net_changes = 0"""; + testdb + .withCdc() + .withWaitUntilAgentRunning() + // create tables + .with("CREATE TABLE %s.%s(id INTEGER PRIMARY KEY, name VARCHAR(200));", SCHEMA_NAME, STREAM_NAME) + .with("CREATE TABLE %s.%s(id INTEGER PRIMARY KEY, name VARCHAR(200));", SCHEMA_NAME, STREAM_NAME2) + // populate tables + .with("INSERT INTO %s.%s (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');", SCHEMA_NAME, STREAM_NAME) + .with("INSERT INTO %s.%s (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');", SCHEMA_NAME, STREAM_NAME2) + // enable cdc on tables for designated role + .with(enableCdcSqlFmt, SCHEMA_NAME, STREAM_NAME, CDC_ROLE_NAME) + .with(enableCdcSqlFmt, SCHEMA_NAME, STREAM_NAME2, CDC_ROLE_NAME) + .withShortenedCapturePollingInterval() + .withWaitUntilMaxLsnAvailable() + // revoke user permissions + .with("REVOKE ALL FROM %s CASCADE;", testdb.getUserName()) + .with("EXEC sp_msforeachtable \"REVOKE ALL ON '?' TO %s;\"", testdb.getUserName()) + // grant user permissions + .with("EXEC sp_addrolemember N'%s', N'%s';", "db_datareader", testdb.getUserName()) + .with("GRANT SELECT ON SCHEMA :: [cdc] TO %s", testdb.getUserName()) + .with("EXEC sp_addrolemember N'%s', N'%s';", CDC_ROLE_NAME, testdb.getUserName()); } - private void setupTestUser() { - executeQuery("USE " + dbName); - executeQuery("CREATE LOGIN " + testUserName + " WITH PASSWORD = '" + TEST_USER_PASSWORD + "';"); - executeQuery("CREATE USER " + testUserName + " FOR LOGIN " + testUserName + ";"); + @Override + protected void tearDown(final TestDestinationEnv testEnv) { + testdb.close(); } - private void revokeAllPermissions() { - executeQuery("REVOKE ALL FROM " + testUserName + " CASCADE;"); - executeQuery("EXEC sp_msforeachtable \"REVOKE ALL ON '?' TO " + testUserName + ";\""); - } + @Test + void testAddNewStreamToExistingSync() throws Exception { + final ConfiguredAirbyteCatalog configuredCatalogWithOneStream = + new ConfiguredAirbyteCatalog().withStreams(List.of(getConfiguredAirbyteStreams().get(0))); - private void createAndPopulateTables() throws InterruptedException { - executeQuery(String.format("CREATE TABLE %s.%s(id INTEGER PRIMARY KEY, name VARCHAR(200));", - SCHEMA_NAME, STREAM_NAME)); - executeQuery(String.format("INSERT INTO %s.%s (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');", - SCHEMA_NAME, STREAM_NAME)); - executeQuery(String.format("CREATE TABLE %s.%s(id INTEGER PRIMARY KEY, name VARCHAR(200));", - SCHEMA_NAME, STREAM_NAME2)); - executeQuery(String.format("INSERT INTO %s.%s (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');", - SCHEMA_NAME, STREAM_NAME2)); - - // sometimes seeing an error that we can't enable cdc on a table while sql server agent is still - // spinning up - // solving with a simple while retry loop - boolean failingToStart = true; - int retryNum = 0; - final int maxRetries = 10; - while (failingToStart) { - try { - // enabling CDC on each table - final String[] tables = {STREAM_NAME, STREAM_NAME2}; - for (final String table : tables) { - executeQuery(String.format( - "EXEC sys.sp_cdc_enable_table\n" - + "\t@source_schema = N'%s',\n" - + "\t@source_name = N'%s', \n" - + "\t@role_name = N'%s',\n" - + "\t@supports_net_changes = 0", - SCHEMA_NAME, table, CDC_ROLE_NAME)); - } - failingToStart = false; - } catch (final Exception e) { - if (retryNum >= maxRetries) { - throw e; - } else { - retryNum++; - Thread.sleep(10000); // 10 seconds - } - } - } - } + // Start a sync with one stream + final List messages = runRead(configuredCatalogWithOneStream); + final List recordMessages = filterRecords(messages); + final List stateMessages = filterStateMessages(messages); + final List streamStates = stateMessages.get(0).getGlobal().getStreamStates(); - private void grantCorrectPermissions() { - executeQuery(String.format("EXEC sp_addrolemember N'%s', N'%s';", "db_datareader", testUserName)); - executeQuery(String.format("USE %s;\n" + "GRANT SELECT ON SCHEMA :: [%s] TO %s", dbName, "cdc", testUserName)); - executeQuery(String.format("EXEC sp_addrolemember N'%s', N'%s';", CDC_ROLE_NAME, testUserName)); - } + assertEquals(3, recordMessages.size()); + assertEquals(1, stateMessages.size()); + assertEquals(1, streamStates.size()); + assertEquals(STREAM_NAME, streamStates.get(0).getStreamDescriptor().getName()); + assertEquals(SCHEMA_NAME, streamStates.get(0).getStreamDescriptor().getNamespace()); + + final AirbyteStateMessage lastStateMessage = Iterables.getLast(stateMessages); + + final ConfiguredAirbyteCatalog configuredCatalogWithTwoStreams = configuredCatalogWithOneStream.withStreams(getConfiguredAirbyteStreams()); + + // Start another sync with a newly added stream + final List messages2 = runRead(configuredCatalogWithTwoStreams, Jsons.jsonNode(List.of(lastStateMessage))); + final List recordMessages2 = filterRecords(messages2); + final List stateMessages2 = filterStateMessages(messages2); + + assertEquals(3, recordMessages2.size()); + assertEquals(2, stateMessages2.size()); - private void executeQuery(final String query) { - try { - database.query( - ctx -> ctx - .execute(query)); - } catch (final Exception e) { - throw new RuntimeException(e); - } + final AirbyteStateMessage lastStateMessage2 = Iterables.getLast(stateMessages2); + final List streamStates2 = lastStateMessage2.getGlobal().getStreamStates(); + + assertEquals(2, streamStates2.size()); + + assertEquals(STREAM_NAME, streamStates2.get(0).getStreamDescriptor().getName()); + assertEquals(SCHEMA_NAME, streamStates2.get(0).getStreamDescriptor().getNamespace()); + assertEquals(STREAM_NAME2, streamStates2.get(1).getStreamDescriptor().getName()); + assertEquals(SCHEMA_NAME, streamStates2.get(1).getStreamDescriptor().getNamespace()); } - @Override - protected void tearDown(final TestDestinationEnv testEnv) { - dslContext.close(); + private List filterStateMessages(final List messages) { + return messages.stream().filter(r -> r.getType() == AirbyteMessage.Type.STATE).map(AirbyteMessage::getState) + .collect(Collectors.toList()); } } diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java index aae9b1b48541..adfa26005af3 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java @@ -5,73 +5,26 @@ package io.airbyte.integrations.source.mssql; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import java.util.Map; -import org.testcontainers.containers.MSSQLServerContainer; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.BaseImage; +import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.ContainerModifier; public class CdcMssqlSourceDatatypeTest extends AbstractMssqlSourceDatatypeTest { @Override - protected void tearDown(final TestDestinationEnv testEnv) { - dslContext.close(); - container.close(); + protected JsonNode getConfig() { + return testdb.integrationTestConfigBuilder() + .withCdcReplication() + .withoutSsl() + .build(); } @Override - protected Database setupDatabase() throws Exception { - container = new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2019-latest") - .acceptLicense(); - container.addEnv("MSSQL_AGENT_ENABLED", "True"); // need this running for cdc to work - container.start(); - - final JsonNode replicationConfig = Jsons.jsonNode(Map.of( - "method", "CDC", - "data_to_sync", "Existing and New", - "initial_waiting_seconds", 5, - "snapshot_isolation", "Snapshot")); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, DB_NAME) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put("replication_method", replicationConfig) - .put("ssl_method", Jsons.jsonNode(Map.of("ssl_method", "unencrypted"))) - .build()); - - dslContext = DSLContextFactory.create(DataSourceFactory.create( - container.getUsername(), - container.getPassword(), - container.getDriverClassName(), - String.format("jdbc:sqlserver://%s:%d;", - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asInt()), - Map.of("encrypt", "false")), null); - final Database database = new Database(dslContext); - - executeQuery("CREATE DATABASE " + DB_NAME + ";"); - executeQuery("ALTER DATABASE " + DB_NAME + "\n\tSET ALLOW_SNAPSHOT_ISOLATION ON"); - executeQuery("USE " + DB_NAME + "\n" + "EXEC sys.sp_cdc_enable_db"); - - return database; - } - - private void executeQuery(final String query) { - try { - final Database database = new Database(dslContext); - database.query( - ctx -> ctx - .execute(query)); - } catch (final Exception e) { - throw new RuntimeException(e); - } + protected Database setupDatabase() { + testdb = MsSQLTestDatabase.in(BaseImage.MSSQL_2022, ContainerModifier.AGENT) + .withCdc(); + return testdb.getDatabase(); } @Override @@ -81,39 +34,44 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc } private void enableCdcOnAllTables() { - executeQuery("USE " + DB_NAME + "\n" - + "DECLARE @TableName VARCHAR(100)\n" - + "DECLARE @TableSchema VARCHAR(100)\n" - + "DECLARE CDC_Cursor CURSOR FOR\n" - + " SELECT * FROM ( \n" - + " SELECT Name,SCHEMA_NAME(schema_id) AS TableSchema\n" - + " FROM sys.objects\n" - + " WHERE type = 'u'\n" - + " AND is_ms_shipped <> 1\n" - + " ) CDC\n" - + "OPEN CDC_Cursor\n" - + "FETCH NEXT FROM CDC_Cursor INTO @TableName,@TableSchema\n" - + "WHILE @@FETCH_STATUS = 0\n" - + " BEGIN\n" - + " DECLARE @SQL NVARCHAR(1000)\n" - + " DECLARE @CDC_Status TINYINT\n" - + " SET @CDC_Status=(SELECT COUNT(*)\n" - + " FROM cdc.change_tables\n" - + " WHERE Source_object_id = OBJECT_ID(@TableSchema+'.'+@TableName))\n" - + " --IF CDC is not enabled on Table, Enable CDC\n" - + " IF @CDC_Status <> 1\n" - + " BEGIN\n" - + " SET @SQL='EXEC sys.sp_cdc_enable_table\n" - + " @source_schema = '''+@TableSchema+''',\n" - + " @source_name = ''' + @TableName\n" - + " + ''',\n" - + " @role_name = null;'\n" - + " EXEC sp_executesql @SQL\n" - + " END\n" - + " FETCH NEXT FROM CDC_Cursor INTO @TableName,@TableSchema\n" - + "END\n" - + "CLOSE CDC_Cursor\n" - + "DEALLOCATE CDC_Cursor"); + testdb.with(""" + DECLARE @TableName VARCHAR(100) + DECLARE @TableSchema VARCHAR(100) + DECLARE CDC_Cursor CURSOR FOR + SELECT * FROM ( + SELECT Name,SCHEMA_NAME(schema_id) AS TableSchema + FROM sys.objects + WHERE type = 'u' + AND is_ms_shipped <> 1 + ) CDC + OPEN CDC_Cursor + FETCH NEXT FROM CDC_Cursor INTO @TableName,@TableSchema + WHILE @@FETCH_STATUS = 0 + BEGIN + DECLARE @SQL NVARCHAR(1000) + DECLARE @CDC_Status TINYINT + SET @CDC_Status=(SELECT COUNT(*) + FROM cdc.change_tables + WHERE Source_object_id = OBJECT_ID(@TableSchema+'.'+@TableName)) + --IF CDC is not enabled on Table, Enable CDC + IF @CDC_Status <> 1 + BEGIN + SET @SQL='EXEC sys.sp_cdc_enable_table + @source_schema = '''+@TableSchema+''', + @source_name = ''' + @TableName + + ''', + @role_name = null;' + EXEC sp_executesql @SQL + END + FETCH NEXT FROM CDC_Cursor INTO @TableName,@TableSchema + END + CLOSE CDC_Cursor + DEALLOCATE CDC_Cursor"""); + } + + @Override + public boolean testCatalog() { + return true; } } diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CloudDeploymentSslEnabledMssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CloudDeploymentSslEnabledMssqlSourceAcceptanceTest.java new file mode 100644 index 000000000000..f2a311d6b455 --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CloudDeploymentSslEnabledMssqlSourceAcceptanceTest.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.commons.features.FeatureFlags; +import io.airbyte.commons.features.FeatureFlagsWrapper; + +public class CloudDeploymentSslEnabledMssqlSourceAcceptanceTest extends MssqlSourceAcceptanceTest { + + @Override + protected void setupEnvironment(final TestDestinationEnv environment) { + final var container = new MsSQLContainerFactory().shared("mcr.microsoft.com/mssql/server:2022-latest"); + testdb = new MsSQLTestDatabase(container); + testdb = testdb + .withConnectionProperty("encrypt", "true") + .withConnectionProperty("trustServerCertificate", "true") + .withConnectionProperty("databaseName", testdb.getDatabaseName()) + .initialized() + .with("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200), born DATETIMEOFFSET(7));") + .with("CREATE TABLE %s.%s(id INTEGER PRIMARY KEY, name VARCHAR(200));", SCHEMA_NAME, STREAM_NAME2) + .with("INSERT INTO id_and_name (id, name, born) VALUES " + + "(1,'picard', '2124-03-04T01:01:01Z'), " + + "(2, 'crusher', '2124-03-04T01:01:01Z'), " + + "(3, 'vash', '2124-03-04T01:01:01Z');") + .with("INSERT INTO %s.%s (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato'), (4, 'Argo');", SCHEMA_NAME, STREAM_NAME2); + } + + @Override + protected FeatureFlags featureFlags() { + return FeatureFlagsWrapper.overridingDeploymentMode(super.featureFlags(), "CLOUD"); + } + + @Override + protected JsonNode getConfig() { + return testdb.integrationTestConfigBuilder() + .withEncrytedTrustServerCertificate() + .build(); + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlRdsSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlRdsSourceAcceptanceTest.java deleted file mode 100644 index bdfb75847daa..000000000000 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlRdsSourceAcceptanceTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mssql; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.commons.io.IOs; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import java.nio.file.Path; -import java.sql.SQLException; -import org.apache.commons.lang3.RandomStringUtils; -import org.jooq.DSLContext; - -public class MssqlRdsSourceAcceptanceTest extends MssqlSourceAcceptanceTest { - - private JsonNode baseConfig; - - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws SQLException { - baseConfig = getStaticConfig(); - final String dbName = "db_" + RandomStringUtils.randomAlphabetic(10).toLowerCase(); - try (final DSLContext dslContext = getDslContext(baseConfig)) { - final Database database = getDatabase(dslContext); - database.query(ctx -> { - ctx.fetch(String.format("CREATE DATABASE %s;", dbName)); - ctx.fetch(String.format("ALTER DATABASE %s SET AUTO_CLOSE OFF WITH NO_WAIT;", dbName)); - ctx.fetch(String.format("USE %s;", dbName)); - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200), born DATETIMEOFFSET(7));"); - ctx.fetch( - "INSERT INTO id_and_name (id, name, born) VALUES (1,'picard', '2124-03-04T01:01:01Z'), (2, 'crusher', '2124-03-04T01:01:01Z'), (3, 'vash', '2124-03-04T01:01:01Z');"); - return null; - }); - } - - config = Jsons.clone(baseConfig); - ((ObjectNode) config).put(JdbcUtils.DATABASE_KEY, dbName); - } - - public JsonNode getStaticConfig() { - return Jsons.deserialize(IOs.readFile(Path.of("secrets/config.json"))); - } - - private DSLContext getDslContext(final JsonNode baseConfig) { - String additionalParameter = ""; - final JsonNode sslMethod = baseConfig.get("ssl_method"); - switch (sslMethod.get("ssl_method").asText()) { - case "unencrypted" -> additionalParameter = "encrypt=false;"; - case "encrypted_trust_server_certificate" -> additionalParameter = "encrypt=true;trustServerCertificate=true;"; - } - return DSLContextFactory.create( - baseConfig.get(JdbcUtils.USERNAME_KEY).asText(), - baseConfig.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.MSSQLSERVER.getDriverClassName(), - String.format("jdbc:sqlserver://%s:%d;%s", - baseConfig.get(JdbcUtils.HOST_KEY).asText(), - baseConfig.get(JdbcUtils.PORT_KEY).asInt(), - additionalParameter), - null); - } - - private Database getDatabase(final DSLContext dslContext) { - return new Database(dslContext); - } - - @Override - protected void tearDown(final TestDestinationEnv testEnv) throws Exception { - final String database = config.get(JdbcUtils.DATABASE_KEY).asText(); - try (final DSLContext dslContext = getDslContext(baseConfig)) { - getDatabase(dslContext).query(ctx -> { - ctx.fetch(String.format("ALTER DATABASE %s SET single_user with rollback immediate;", database)); - ctx.fetch(String.format("DROP DATABASE %s;", database)); - return null; - }); - } - } - -} diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java index 12124e50b416..4bdc5cecf61a 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceAcceptanceTest.java @@ -4,82 +4,58 @@ package io.airbyte.integrations.source.mssql; +import static org.junit.Assert.assertEquals; + import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.base.ssh.SshHelpers; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.string.Strings; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.BaseImage; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStreamState; import io.airbyte.protocol.models.v0.CatalogHelpers; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.ConnectorSpecification; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.SyncMode; import java.sql.SQLException; import java.util.HashMap; -import java.util.Map; -import org.jooq.DSLContext; -import org.junit.jupiter.api.AfterAll; -import org.testcontainers.containers.MSSQLServerContainer; +import java.util.List; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Test; public class MssqlSourceAcceptanceTest extends SourceAcceptanceTest { protected static final String SCHEMA_NAME = "dbo"; protected static final String STREAM_NAME = "id_and_name"; - protected static MSSQLServerContainer db; - protected JsonNode config; - - @AfterAll - public static void closeContainer() { - if (db != null) { - db.close(); - db.stop(); - } - } + protected static final String STREAM_NAME2 = "starships"; + + protected MsSQLTestDatabase testdb; @Override protected void setupEnvironment(final TestDestinationEnv environment) throws SQLException { - if (db == null) { - db = new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2022-RTM-CU2-ubuntu-20.04").acceptLicense(); - db.start(); - } - final var containerAddress = SshHelpers.getOuterContainerAddress(db); - final JsonNode configWithoutDbName = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, containerAddress.left) - .put(JdbcUtils.PORT_KEY, containerAddress.right) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) - .build()); - final String dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); - - try (final DSLContext dslContext = getDslContext(configWithoutDbName)) { - final Database database = getDatabase(dslContext); - database.query(ctx -> { - ctx.fetch(String.format("CREATE DATABASE %s;", dbName)); - ctx.fetch(String.format("USE %s;", dbName)); - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200), born DATETIMEOFFSET(7));"); - ctx.fetch( - "INSERT INTO id_and_name (id, name, born) VALUES " + - "(1,'picard', '2124-03-04T01:01:01Z'), " + - "(2, 'crusher', '2124-03-04T01:01:01Z'), (3, 'vash', '2124-03-04T01:01:01Z');"); - return null; - }); - } - - config = Jsons.clone(configWithoutDbName); - ((ObjectNode) config).put(JdbcUtils.DATABASE_KEY, dbName); - ((ObjectNode) config).put("ssl_method", Jsons.jsonNode(Map.of("ssl_method", "unencrypted"))); + testdb = MsSQLTestDatabase.in(BaseImage.MSSQL_2022) + .with("CREATE TABLE id_and_name (id INTEGER, name VARCHAR(200), born DATETIMEOFFSET(7));") + .with("CREATE TABLE %s.%s(id INTEGER PRIMARY KEY, name VARCHAR(200));", SCHEMA_NAME, STREAM_NAME2) + .with("INSERT INTO id_and_name (id, name, born) VALUES " + + "(1, 'picard', '2124-03-04T01:01:01Z'), " + + "(2, 'crusher', '2124-03-04T01:01:01Z'), " + + "(3, 'vash', '2124-03-04T01:01:01Z');") + .with("INSERT INTO %s.%s (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato'), (4, 'Argo');", SCHEMA_NAME, STREAM_NAME2); } @Override - protected void tearDown(final TestDestinationEnv testEnv) throws Exception {} + protected void tearDown(final TestDestinationEnv testEnv) { + testdb.close(); + } @Override protected String getImageName() { @@ -93,7 +69,9 @@ protected ConnectorSpecification getSpec() throws Exception { @Override protected JsonNode getConfig() { - return config; + return testdb.integrationTestConfigBuilder() + .withoutSsl() + .build(); } @Override @@ -111,19 +89,59 @@ protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); } - private static DSLContext getDslContext(final JsonNode config) { - return DSLContextFactory.create(DataSourceFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.MSSQLSERVER.getDriverClassName(), - String.format("jdbc:sqlserver://%s:%d;", - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asInt()), - Map.of("encrypt", "false")), null); + @Test + protected void testAddNewStreamToExistingSync() throws Exception { + final List configuredAirbyteStreams = + Lists.newArrayList(CatalogHelpers.createConfiguredAirbyteStream(STREAM_NAME, + SCHEMA_NAME, + Field.of("id", JsonSchemaType.NUMBER), + Field.of("name", JsonSchemaType.STRING)) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(List.of("id")), + CatalogHelpers.createConfiguredAirbyteStream(STREAM_NAME2, + SCHEMA_NAME, + Field.of("id", JsonSchemaType.NUMBER), + Field.of("name", JsonSchemaType.STRING)) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(List.of("id"))); + final ConfiguredAirbyteCatalog configuredCatalogWithOneStream = + new ConfiguredAirbyteCatalog().withStreams(List.of(configuredAirbyteStreams.get(0))); + + // Start a sync with one stream + final List messages = runRead(withSourceDefinedCursors(configuredCatalogWithOneStream)); + final List recordMessages = filterRecords(messages); + final List stateMessages = filterStateMessages(messages); + final AirbyteStateMessage lastStateMessage = Iterables.getLast(stateMessages); + final AirbyteStreamState streamState = lastStateMessage.getStream(); + + assertEquals(3, recordMessages.size()); + assertEquals(1, stateMessages.size()); + assertEquals(STREAM_NAME, streamState.getStreamDescriptor().getName()); + assertEquals(SCHEMA_NAME, streamState.getStreamDescriptor().getNamespace()); + + final ConfiguredAirbyteCatalog configuredCatalogWithTwoStreams = + new ConfiguredAirbyteCatalog().withStreams(configuredAirbyteStreams); + + // Start another sync with a newly added stream + final List messages2 = runRead(configuredCatalogWithTwoStreams, Jsons.jsonNode(List.of(lastStateMessage))); + final List recordMessages2 = filterRecords(messages2); + final List stateMessages2 = filterStateMessages(messages2); + + assertEquals(4, recordMessages2.size()); + assertEquals(2, stateMessages2.size()); + + assertEquals(2, stateMessages2.size()); + assertEquals(STREAM_NAME, stateMessages2.get(0).getStream().getStreamDescriptor().getName()); + assertEquals(SCHEMA_NAME, stateMessages2.get(0).getStream().getStreamDescriptor().getNamespace()); + assertEquals(STREAM_NAME2, stateMessages2.get(1).getStream().getStreamDescriptor().getName()); + assertEquals(SCHEMA_NAME, stateMessages2.get(1).getStream().getStreamDescriptor().getNamespace()); } - private static Database getDatabase(final DSLContext dslContext) { - return new Database(dslContext); + private List filterStateMessages(final List messages) { + return messages.stream().filter(r -> r.getType() == AirbyteMessage.Type.STATE).map(AirbyteMessage::getState) + .collect(Collectors.toList()); } } diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceDatatypeTest.java index 707c063145d6..8b11db5c3e77 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceDatatypeTest.java @@ -5,69 +5,22 @@ package io.airbyte.integrations.source.mssql; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import java.util.Map; -import org.jooq.DSLContext; -import org.testcontainers.containers.MSSQLServerContainer; +import io.airbyte.cdk.db.Database; +import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.BaseImage; public class MssqlSourceDatatypeTest extends AbstractMssqlSourceDatatypeTest { @Override - protected Database setupDatabase() throws Exception { - container = new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2019-latest") - .acceptLicense(); - container.start(); - - final JsonNode configWithoutDbName = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .build()); - - dslContext = getDslContext(configWithoutDbName); - final Database database = getDatabase(dslContext); - database.query(ctx -> { - ctx.fetch(String.format("CREATE DATABASE %s;", DB_NAME)); - ctx.fetch(String.format("USE %s;", DB_NAME)); - return null; - }); - - config = Jsons.clone(configWithoutDbName); - ((ObjectNode) config).put(JdbcUtils.DATABASE_KEY, DB_NAME); - ((ObjectNode) config).put("ssl_method", Jsons.jsonNode(Map.of("ssl_method", "unencrypted"))); - - return database; - } - - private static DSLContext getDslContext(final JsonNode config) { - return DSLContextFactory.create(DataSourceFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.MSSQLSERVER.getDriverClassName(), - String.format("jdbc:sqlserver://%s:%d;", - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asInt()), - Map.of("encrypt", "false")), null); - } - - private static Database getDatabase(final DSLContext dslContext) { - return new Database(dslContext); + protected Database setupDatabase() { + testdb = MsSQLTestDatabase.in(BaseImage.MSSQL_2022); + return testdb.getDatabase(); } @Override - protected void tearDown(final TestDestinationEnv testEnv) { - dslContext.close(); - container.stop(); - container.close(); + protected JsonNode getConfig() { + return testdb.integrationTestConfigBuilder() + .withoutSsl() + .build(); } @Override diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceOperationsTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceOperationsTest.java new file mode 100644 index 000000000000..10b3f3d81cfe --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceOperationsTest.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.BaseImage; +import java.sql.Connection; +import java.sql.JDBCType; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class MssqlSourceOperationsTest { + + private final MssqlSourceOperations mssqlSourceOperations = new MssqlSourceOperations(); + + private MsSQLTestDatabase testdb; + + private final String cursorColumn = "cursor_column"; + + @BeforeEach + public void init() { + testdb = MsSQLTestDatabase.in(BaseImage.MSSQL_2022); + } + + @AfterEach + public void tearDown() { + testdb.close(); + } + + @Test + public void setDateTimeOffsetColumnAsCursor() throws SQLException { + final String tableName = "datetimeoffset_table"; + final String createTableQuery = String.format("CREATE TABLE %s(id INTEGER PRIMARY KEY IDENTITY(1,1), %s DATETIMEOFFSET(7));", + tableName, + cursorColumn); + executeQuery(createTableQuery); + final List expectedRecords = new ArrayList<>(); + for (int i = 1; i <= 4; i++) { + final ObjectNode jsonNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); + // Manually generate DATETIMEOFFSET data + final String cursorValue = String.format("'2023-0%s-10 10:00:00.%s00000 +00:00'", i, i * 10); + jsonNode.put("id", i); + // Remove single quotes from string since the date being retrieved will not have quotes + jsonNode.put(cursorColumn, cursorValue.replaceAll("\'", "")); + final String insertQuery = String.format("INSERT INTO %s (%s) VALUES (CAST(%s as DATETIMEOFFSET))", tableName, cursorColumn, cursorValue); + + executeQuery(insertQuery); + expectedRecords.add(jsonNode); + } + final String cursorAnchorValue = "2023-01-01 00:00:00.0000000 +00:00"; + final List actualRecords = new ArrayList<>(); + try (final Connection connection = testdb.getContainer().createConnection("")) { + final PreparedStatement preparedStatement = connection.prepareStatement( + "SELECT * from " + tableName + " WHERE " + cursorColumn + " > ?"); + mssqlSourceOperations.setCursorField(preparedStatement, + 1, + JDBCType.TIMESTAMP_WITH_TIMEZONE, + cursorAnchorValue); + + try (final ResultSet resultSet = preparedStatement.executeQuery()) { + final int columnCount = resultSet.getMetaData().getColumnCount(); + while (resultSet.next()) { + final ObjectNode jsonNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); + for (int i = 1; i <= columnCount; i++) { + mssqlSourceOperations.copyToJsonField(resultSet, i, jsonNode); + } + actualRecords.add(jsonNode); + } + } + } + assertThat(actualRecords, containsInAnyOrder(expectedRecords.toArray())); + } + + protected void executeQuery(final String query) throws SQLException { + try (final Connection connection = testdb.getContainer().createConnection("")) { + connection.createStatement().execute(query); + } + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SshKeyMssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SshKeyMssqlSourceAcceptanceTest.java index 0af4fb3ee085..276bcc7ee804 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SshKeyMssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SshKeyMssqlSourceAcceptanceTest.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.source.mssql; -import io.airbyte.integrations.base.ssh.SshTunnel.TunnelMethod; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel.TunnelMethod; import org.junit.jupiter.api.Disabled; @Disabled diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SshPasswordMssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SshPasswordMssqlSourceAcceptanceTest.java index 01aa8624c466..61b015fc538a 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SshPasswordMssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SshPasswordMssqlSourceAcceptanceTest.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.source.mssql; -import io.airbyte.integrations.base.ssh.SshTunnel.TunnelMethod; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel.TunnelMethod; import org.junit.jupiter.api.Disabled; @Disabled diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SslEnabledMssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SslEnabledMssqlSourceAcceptanceTest.java index eb9723c8626e..ccd887c9a4b9 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SslEnabledMssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/SslEnabledMssqlSourceAcceptanceTest.java @@ -5,83 +5,33 @@ package io.airbyte.integrations.source.mssql; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import java.sql.SQLException; -import java.util.Map; -import org.apache.commons.lang3.RandomStringUtils; -import org.jooq.DSLContext; -import org.junit.jupiter.api.AfterAll; -import org.testcontainers.containers.MSSQLServerContainer; -import org.testcontainers.utility.DockerImageName; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; public class SslEnabledMssqlSourceAcceptanceTest extends MssqlSourceAcceptanceTest { - @AfterAll - public static void closeContainer() { - if (db != null) { - db.close(); - db.stop(); - } - } - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws SQLException { - if (db == null) { - db = new MSSQLServerContainer<>(DockerImageName - .parse("airbyte/mssql_ssltest:dev") - .asCompatibleSubstituteFor("mcr.microsoft.com/mssql/server")) - .acceptLicense(); - db.start(); - } - - final JsonNode configWithoutDbName = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) - .build()); - final String dbName = "db_" + RandomStringUtils.randomAlphabetic(10).toLowerCase(); - - try (final DSLContext dslContext = getDslContext(configWithoutDbName)) { - final Database database = getDatabase(dslContext); - database.query(ctx -> { - ctx.fetch(String.format("CREATE DATABASE %s;", dbName)); - ctx.fetch(String.format("USE %s;", dbName)); - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200), born DATETIMEOFFSET(7));"); - ctx.fetch( - "INSERT INTO id_and_name (id, name, born) VALUES " + - "(1,'picard', '2124-03-04T01:01:01Z'), " + - "(2, 'crusher', '2124-03-04T01:01:01Z'), " + - "(3, 'vash', '2124-03-04T01:01:01Z');"); - return null; - }); - } - - config = Jsons.clone(configWithoutDbName); - ((ObjectNode) config).put(JdbcUtils.DATABASE_KEY, dbName); - ((ObjectNode) config).put("ssl_method", Jsons.jsonNode(Map.of("ssl_method", "encrypted_trust_server_certificate"))); - } - - private static DSLContext getDslContext(final JsonNode baseConfig) { - return DSLContextFactory.create( - baseConfig.get(JdbcUtils.USERNAME_KEY).asText(), - baseConfig.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.MSSQLSERVER.getDriverClassName(), - String.format("jdbc:sqlserver://%s:%d;encrypt=true;trustServerCertificate=true;", - baseConfig.get(JdbcUtils.HOST_KEY).asText(), - baseConfig.get(JdbcUtils.PORT_KEY).asInt()), - null); + protected JsonNode getConfig() { + return testdb.integrationTestConfigBuilder() + .withEncrytedTrustServerCertificate() + .build(); } - private static Database getDatabase(final DSLContext dslContext) { - return new Database(dslContext); + @Override + protected void setupEnvironment(final TestDestinationEnv environment) { + final var container = new MsSQLContainerFactory().shared("mcr.microsoft.com/mssql/server:2022-latest"); + testdb = new MsSQLTestDatabase(container); + testdb = testdb + .withConnectionProperty("encrypt", "true") + .withConnectionProperty("trustServerCertificate", "true") + .withConnectionProperty("databaseName", testdb.getDatabaseName()) + .initialized() + .with("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200), born DATETIMEOFFSET(7));") + .with("CREATE TABLE %s.%s(id INTEGER PRIMARY KEY, name VARCHAR(200));", SCHEMA_NAME, STREAM_NAME2) + .with("INSERT INTO id_and_name (id, name, born) VALUES " + + "(1, 'picard', '2124-03-04T01:01:01Z'), " + + "(2, 'crusher', '2124-03-04T01:01:01Z'), " + + "(3, 'vash', '2124-03-04T01:01:01Z');") + .with("INSERT INTO %s.%s (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato'), (4, 'Argo');", SCHEMA_NAME, STREAM_NAME2);; } } diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/dummy_config.json b/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/dummy_config.json index 560e55333378..1f42c042e746 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/dummy_config.json +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/dummy_config.json @@ -2,5 +2,6 @@ "host": "default", "port": 5555, "database": "default", - "username": "default" + "username": "default", + "password": "default" } diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/expected_spec.json b/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/expected_spec.json index 0b94887ffc1a..c2f000494ee4 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/resources/expected_spec.json @@ -4,7 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "MSSQL Source Spec", "type": "object", - "required": ["host", "port", "database", "username"], + "required": ["host", "port", "database", "username", "password"], "properties": { "host": { "description": "The hostname of the database.", @@ -90,7 +90,7 @@ { "title": "Encrypted (verify certificate)", "description": "Verify and use the certificate provided by the server.", - "required": ["ssl_method", "trustStoreName", "trustStorePassword"], + "required": ["ssl_method"], "properties": { "ssl_method": { "type": "string", @@ -100,7 +100,15 @@ "title": "Host Name In Certificate", "type": "string", "description": "Specifies the host name of the server. The value of this property must match the subject property of the certificate.", - "order": 7 + "order": 0 + }, + "certificate": { + "title": "Certificate", + "type": "string", + "description": "certificate of the server, or of the CA that signed the server certificate", + "order": 1, + "airbyte_secret": true, + "multiline": true } } } @@ -108,26 +116,15 @@ }, "replication_method": { "type": "object", - "title": "Replication Method", - "description": "The replication method used for extracting data from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses {TBC} to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", - "default": "STANDARD", + "title": "Update Method", + "description": "Configures how data is extracted from the database.", + "default": "CDC", + "display_type": "radio", "order": 8, "oneOf": [ { - "title": "Standard", - "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "STANDARD", - "order": 0 - } - } - }, - { - "title": "Logical Replication (CDC)", - "description": "CDC uses {TBC} to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", + "title": "Read Changes using Change Data Capture (CDC)", + "description": "Recommended - Incrementally reads new inserts, updates, and deletes using the SQL Server's change data capture feature. This must be enabled on your database.", "required": ["method"], "properties": { "method": { @@ -135,22 +132,6 @@ "const": "CDC", "order": 0 }, - "data_to_sync": { - "title": "Data to Sync", - "type": "string", - "default": "Existing and New", - "enum": ["Existing and New", "New Changes Only"], - "description": "What data should be synced under the CDC. \"Existing and New\" will read existing data as a snapshot, and sync new changes through CDC. \"New Changes Only\" will skip the initial snapshot, and only sync new changes through CDC.", - "order": 1 - }, - "snapshot_isolation": { - "title": "Initial Snapshot Isolation Level", - "type": "string", - "default": "Snapshot", - "enum": ["Snapshot", "Read Committed"], - "description": "Existing data in the database are synced through an initial snapshot. This parameter controls the isolation level that will be used during the initial snapshotting. If you choose the \"Snapshot\" level, you must enable the snapshot isolation mode on the database.", - "order": 2 - }, "initial_waiting_seconds": { "type": "integer", "title": "Initial Waiting Time in Seconds (Advanced)", @@ -161,6 +142,18 @@ "order": 3 } } + }, + { + "title": "Scan Changes with User Defined Cursor", + "description": "Incrementally detects new inserts and updates using the cursor column chosen when configuring a connection (e.g. created_at, updated_at).", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "STANDARD", + "order": 0 + } + } } ] }, @@ -280,5 +273,7 @@ } } }, + "supportsNormalization": false, + "supportsDBT": false, "supported_destination_sync_modes": [] } diff --git a/airbyte-integrations/connectors/source-mssql/src/test-performance/java/io/airbyte/integrations/source/mssql/FillMsSqlTestDbScriptTest.java b/airbyte-integrations/connectors/source-mssql/src/test-performance/java/io/airbyte/integrations/source/mssql/FillMsSqlTestDbScriptTest.java index ff7816c99554..c309a81c495f 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-performance/java/io/airbyte/integrations/source/mssql/FillMsSqlTestDbScriptTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-performance/java/io/airbyte/integrations/source/mssql/FillMsSqlTestDbScriptTest.java @@ -6,13 +6,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.cdk.integrations.standardtest.source.performancetest.AbstractSourceFillDbWithTestData; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.integrations.standardtest.source.performancetest.AbstractSourceFillDbWithTestData; import java.util.stream.Stream; import org.jooq.DSLContext; import org.junit.jupiter.params.provider.Arguments; diff --git a/airbyte-integrations/connectors/source-mssql/src/test-performance/java/io/airbyte/integrations/source/mssql/MsSqlRdsSourcePerformanceSecretTest.java b/airbyte-integrations/connectors/source-mssql/src/test-performance/java/io/airbyte/integrations/source/mssql/MsSqlRdsSourcePerformanceSecretTest.java deleted file mode 100644 index 510e31fae61b..000000000000 --- a/airbyte-integrations/connectors/source-mssql/src/test-performance/java/io/airbyte/integrations/source/mssql/MsSqlRdsSourcePerformanceSecretTest.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mssql; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.io.IOs; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.performancetest.AbstractSourcePerformanceTest; -import java.nio.file.Path; -import java.util.stream.Stream; -import org.junit.jupiter.params.provider.Arguments; - -public class MsSqlRdsSourcePerformanceSecretTest extends AbstractSourcePerformanceTest { - - private static final String PERFORMANCE_SECRET_CREDS = "secrets/performance-config.json"; - - @Override - protected String getImageName() { - return "airbyte/source-mssql:dev"; - } - - @Override - protected void setupDatabase(final String dbName) { - final JsonNode plainConfig = Jsons.deserialize(IOs.readFile(Path.of(PERFORMANCE_SECRET_CREDS))); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, plainConfig.get(JdbcUtils.HOST_KEY)) - .put(JdbcUtils.PORT_KEY, plainConfig.get(JdbcUtils.PORT_KEY)) - .put(JdbcUtils.DATABASE_KEY, dbName) - .put(JdbcUtils.USERNAME_KEY, plainConfig.get(JdbcUtils.USERNAME_KEY)) - .put(JdbcUtils.PASSWORD_KEY, plainConfig.get(JdbcUtils.PASSWORD_KEY)) - .build()); - } - - /** - * This is a data provider for performance tests, Each argument's group would be ran as a separate - * test. 1st arg - a name of DB that will be used in jdbc connection string. 2nd arg - a schemaName - * that will be used as a NameSpace in Configured Airbyte Catalog. 3rd arg - a number of expected - * records retrieved in each stream. 4th arg - a number of columns in each stream\table that will be - * use for Airbyte Cataloq configuration 5th arg - a number of streams to read in configured airbyte - * Catalog. Each stream\table in DB should be names like "test_0", "test_1",..., test_n. - */ - @Override - protected Stream provideParameters() { - return Stream.of( - Arguments.of("t1000_c240_r200", "dbo", 200, 240, 1000), - Arguments.of("t25_c8_r50k_s10kb", "dbo", 50000, 8, 25), - Arguments.of("t1000_c8_r10k_s500b", "dbo", 10000, 8, 1000)); - } - -} diff --git a/airbyte-integrations/connectors/source-mssql/src/test-performance/java/io/airbyte/integrations/source/mssql/MssqlSourcePerformanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-performance/java/io/airbyte/integrations/source/mssql/MssqlSourcePerformanceTest.java new file mode 100644 index 000000000000..0ba7e248d14b --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/test-performance/java/io/airbyte/integrations/source/mssql/MssqlSourcePerformanceTest.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.standardtest.source.performancetest.AbstractSourcePerformanceTest; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import java.nio.file.Path; +import java.util.stream.Stream; +import org.junit.jupiter.params.provider.Arguments; + +public class MssqlSourcePerformanceTest extends AbstractSourcePerformanceTest { + + private static final String PERFORMANCE_SECRET_CREDS = "secrets/performance-config.json"; + + @Override + protected String getImageName() { + return "airbyte/source-mssql:dev"; + } + + @Override + protected void setupDatabase(final String dbName) { + final JsonNode plainConfig = Jsons.deserialize(IOs.readFile(Path.of(PERFORMANCE_SECRET_CREDS))); + + config = Jsons.jsonNode(ImmutableMap.builder() + .put(JdbcUtils.HOST_KEY, plainConfig.get(JdbcUtils.HOST_KEY)) + .put(JdbcUtils.PORT_KEY, plainConfig.get(JdbcUtils.PORT_KEY)) + .put(JdbcUtils.DATABASE_KEY, dbName) + .put(JdbcUtils.USERNAME_KEY, plainConfig.get(JdbcUtils.USERNAME_KEY)) + .put(JdbcUtils.PASSWORD_KEY, plainConfig.get(JdbcUtils.PASSWORD_KEY)) + .build()); + } + + /** + * This is a data provider for performance tests, Each argument's group would be ran as a separate + * test. 1st arg - a name of DB that will be used in jdbc connection string. 2nd arg - a schemaName + * that will be used as a NameSpace in Configured Airbyte Catalog. 3rd arg - a number of expected + * records retrieved in each stream. 4th arg - a number of columns in each stream\table that will be + * use for Airbyte Cataloq configuration 5th arg - a number of streams to read in configured airbyte + * Catalog. Each stream\table in DB should be names like "test_0", "test_1",..., test_n. + */ + @Override + protected Stream provideParameters() { + return Stream.of( + Arguments.of("t1000_c240_r200", "dbo", 200, 240, 1000), + Arguments.of("t25_c8_r50k_s10kb", "dbo", 50000, 8, 25), + Arguments.of("t1000_c8_r10k_s500b", "dbo", 10000, 8, 1000)); + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceTest.java index e27ab427338a..17a9f42baa75 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceTest.java @@ -4,14 +4,14 @@ package io.airbyte.integrations.source.mssql; -import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; -import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; import static io.airbyte.integrations.source.mssql.MssqlSource.CDC_DEFAULT_CURSOR; import static io.airbyte.integrations.source.mssql.MssqlSource.CDC_EVENT_SERIAL_NO; import static io.airbyte.integrations.source.mssql.MssqlSource.CDC_LSN; -import static io.airbyte.integrations.source.mssql.MssqlSource.DRIVER_CLASS; import static io.airbyte.integrations.source.mssql.MssqlSource.MSSQL_CDC_OFFSET; import static io.airbyte.integrations.source.mssql.MssqlSource.MSSQL_DB_HISTORY; +import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -24,218 +24,169 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.jdbc.StreamingJdbcDatabase; +import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; +import io.airbyte.cdk.integrations.JdbcConnector; +import io.airbyte.cdk.integrations.debezium.CdcSourceTest; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.string.Strings; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.StreamingJdbcDatabase; -import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.debezium.CdcSourceTest; -import io.airbyte.integrations.debezium.internals.mssql.MssqlCdcTargetPosition; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.SyncMode; import io.debezium.connector.sqlserver.Lsn; -import java.sql.SQLException; +import java.time.Duration; import java.util.List; import java.util.Map; import java.util.Optional; import javax.sql.DataSource; -import org.jooq.DSLContext; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; import org.testcontainers.containers.MSSQLServerContainer; +import org.testcontainers.utility.DockerImageName; -public class CdcMssqlSourceTest extends CdcSourceTest { - - private static final String CDC_ROLE_NAME = "cdc_selector"; - private static final String TEST_USER_PASSWORD = "testerjester[1]"; - public static MSSQLServerContainer container; - - private String testUserName; - private String dbName; - private String dbNamewithDot; - private Database database; - private JdbcDatabase testJdbcDatabase; - private MssqlSource source; - private JsonNode config; - private DSLContext dslContext; - private DataSource dataSource; - private DataSource testDataSource; +@TestInstance(Lifecycle.PER_CLASS) +public class CdcMssqlSourceTest extends CdcSourceTest { - @BeforeEach - public void setup() throws SQLException { - init(); - setupTestUser(); - revokeAllPermissions(); - super.setup(); - grantCorrectPermissions(); - } + static private final String CDC_ROLE_NAME = "cdc_selector"; - @BeforeAll - public static void createContainer() { - if (container == null) { - container = new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2022-latest").acceptLicense(); - container.addEnv("MSSQL_AGENT_ENABLED", "True"); // need this running for cdc to work - container.start(); - } - } + static private final String TEST_USER_NAME_PREFIX = "cdc_test_user"; - @AfterAll - public static void closeContainer() { - if (container != null) { - container.close(); - container.stop(); - } + // Deliberately do not share this test container, as we're going to mutate the global SQL Server + // state. + protected final MSSQLServerContainer privateContainer; + + private DataSource testDataSource; + + CdcMssqlSourceTest() { + this.privateContainer = createContainer(); } - private void init() { - dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); - testUserName = Strings.addRandomSuffix("test", "_", 5).toLowerCase(); - dbNamewithDot = Strings.addRandomSuffix("db", ".", 10).toLowerCase(); - source = new MssqlSource(); - - final JsonNode replicationConfig = Jsons.jsonNode(Map.of( - "method", "CDC", - "data_to_sync", "Existing and New", - "initial_waiting_seconds", INITIAL_WAITING_SECONDS, - "snapshot_isolation", "Snapshot")); - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, dbName) - .put(JdbcUtils.SCHEMAS_KEY, List.of(MODELS_SCHEMA, MODELS_SCHEMA + "_random")) - .put(JdbcUtils.USERNAME_KEY, testUserName) - .put(JdbcUtils.PASSWORD_KEY, TEST_USER_PASSWORD) - .put("replication_method", replicationConfig) - .put("ssl_method", Jsons.jsonNode(Map.of("ssl_method", "unencrypted"))) - .build()); - - dataSource = DataSourceFactory.create( - container.getUsername(), - container.getPassword(), - DRIVER_CLASS, - String.format("jdbc:sqlserver://%s:%d", - container.getHost(), - container.getFirstMappedPort()), - Map.of("encrypt", "false")); - - testDataSource = DataSourceFactory.create( - testUserName, - TEST_USER_PASSWORD, - DRIVER_CLASS, - String.format("jdbc:sqlserver://%s:%d", - container.getHost(), - container.getFirstMappedPort()), - Map.of("encrypt", "false")); - - dslContext = DSLContextFactory.create(dataSource, null); - - database = new Database(dslContext); - - testJdbcDatabase = new DefaultJdbcDatabase(testDataSource); - - executeQuery("CREATE DATABASE " + dbName + ";"); - executeQuery("CREATE DATABASE [" + dbNamewithDot + "];"); - switchSnapshotIsolation(true, dbName); + protected MSSQLServerContainer createContainer() { + return new MsSQLContainerFactory() + .createNewContainer(DockerImageName.parse("mcr.microsoft.com/mssql/server:2022-latest")); } - private void switchSnapshotIsolation(final Boolean on, final String db) { - final String onOrOff = on ? "ON" : "OFF"; - executeQuery("ALTER DATABASE " + db + "\n\tSET ALLOW_SNAPSHOT_ISOLATION " + onOrOff); + @BeforeAll + public void beforeAll() { + new MsSQLContainerFactory().withAgent(privateContainer); + privateContainer.start(); } - private void setupTestUser() { - executeQuery("USE " + dbName); - executeQuery("CREATE LOGIN " + testUserName + " WITH PASSWORD = '" + TEST_USER_PASSWORD + "';"); - executeQuery("CREATE USER " + testUserName + " FOR LOGIN " + testUserName + ";"); + @AfterAll + void afterAll() { + privateContainer.close(); } - private void revokeAllPermissions() { - executeQuery("REVOKE ALL FROM " + testUserName + " CASCADE;"); - executeQuery("EXEC sp_msforeachtable \"REVOKE ALL ON '?' TO " + testUserName + ";\""); + protected final String testUserName() { + return testdb.withNamespace(TEST_USER_NAME_PREFIX); } - private void alterPermissionsOnSchema(final Boolean grant, final String schema) { - final String grantOrRemove = grant ? "GRANT" : "REVOKE"; - executeQuery(String.format("USE %s;\n" + "%s SELECT ON SCHEMA :: [%s] TO %s", dbName, grantOrRemove, schema, testUserName)); + @Override + protected MsSQLTestDatabase createTestDatabase() { + final var testdb = new MsSQLTestDatabase(privateContainer); + return testdb + .withConnectionProperty("encrypt", "false") + .withConnectionProperty("databaseName", testdb.getDatabaseName()) + .initialized() + .withWaitUntilAgentRunning() + .withCdc(); } - private void grantCorrectPermissions() { - alterPermissionsOnSchema(true, MODELS_SCHEMA); - alterPermissionsOnSchema(true, MODELS_SCHEMA + "_random"); - alterPermissionsOnSchema(true, "cdc"); - executeQuery(String.format("EXEC sp_addrolemember N'%s', N'%s';", CDC_ROLE_NAME, testUserName)); + @Override + protected MssqlSource source() { + return new MssqlSource(); } @Override - public String createSchemaQuery(final String schemaName) { - return "CREATE SCHEMA " + schemaName; + protected JsonNode config() { + return testdb.configBuilder() + .withHostAndPort() + .withDatabase() + .with(JdbcUtils.USERNAME_KEY, testUserName()) + .with(JdbcUtils.PASSWORD_KEY, testdb.getPassword()) + .withSchemas(modelsSchema(), randomSchema()) + .withCdcReplication() + .withoutSsl() + .build(); } - // TODO : Delete this Override when MSSQL supports individual table snapshot @Override - public void newTableSnapshotTest() { - // Do nothing + @BeforeEach + protected void setup() { + super.setup(); + + // Enables cdc on MODELS_SCHEMA.MODELS_STREAM_NAME, giving CDC_ROLE_NAME select access. + final var enableCdcSqlFmt = """ + EXEC sys.sp_cdc_enable_table + \t@source_schema = N'%s', + \t@source_name = N'%s', + \t@role_name = N'%s', + \t@supports_net_changes = 0"""; + testdb + .with(enableCdcSqlFmt, modelsSchema(), MODELS_STREAM_NAME, CDC_ROLE_NAME) + .with(enableCdcSqlFmt, randomSchema(), RANDOM_TABLE_NAME, CDC_ROLE_NAME) + .withShortenedCapturePollingInterval(); + + // Create a test user to be used by the source, with proper permissions. + testdb + .with("CREATE LOGIN %s WITH PASSWORD = '%s', DEFAULT_DATABASE = %s", testUserName(), testdb.getPassword(), testdb.getDatabaseName()) + .with("CREATE USER %s FOR LOGIN %s WITH DEFAULT_SCHEMA = [dbo]", testUserName(), testUserName()) + .with("REVOKE ALL FROM %s CASCADE;", testUserName()) + .with("EXEC sp_msforeachtable \"REVOKE ALL ON '?' TO %s;\"", testUserName()) + .with("GRANT SELECT ON SCHEMA :: [%s] TO %s", modelsSchema(), testUserName()) + .with("GRANT SELECT ON SCHEMA :: [%s] TO %s", randomSchema(), testUserName()) + .with("GRANT SELECT ON SCHEMA :: [cdc] TO %s", testUserName()) + .with("USE [master]") + .with("GRANT VIEW SERVER STATE TO %s", testUserName()) + .with("USE [%s]", testdb.getDatabaseName()) + .with("EXEC sp_addrolemember N'%s', N'%s';", CDC_ROLE_NAME, testUserName()); + + testDataSource = createTestDataSource(); + } + + protected DataSource createTestDataSource() { + return DataSourceFactory.create( + testUserName(), + testdb.getPassword(), + testdb.getDatabaseDriver().getDriverClassName(), + testdb.getJdbcUrl(), + Map.of("encrypt", "false"), + JdbcConnector.CONNECT_TIMEOUT_DEFAULT); } @Override - protected String randomTableSchema() { - return MODELS_SCHEMA + "_random"; + @AfterEach + protected void tearDown() { + try { + DataSourceFactory.close(testDataSource); + } catch (final Exception e) { + throw new RuntimeException(e); + } + super.tearDown(); + } - private void switchCdcOnDatabase(final Boolean enable, final String db) { - final String storedProc = enable ? "sys.sp_cdc_enable_db" : "sys.sp_cdc_disable_db"; - executeQuery("USE [" + db + "]\n" + "EXEC " + storedProc); + private JdbcDatabase testDatabase() { + return new DefaultJdbcDatabase(testDataSource); } + // TODO : Delete this Override when MSSQL supports individual table snapshot @Override - public void createTable(final String schemaName, final String tableName, final String columnClause) { - switchCdcOnDatabase(true, dbName); - super.createTable(schemaName, tableName, columnClause); - - // sometimes seeing an error that we can't enable cdc on a table while sql server agent is still - // spinning up - // solving with a simple while retry loop - boolean failingToStart = true; - int retryNum = 0; - final int maxRetries = 10; - while (failingToStart) { - try { - executeQuery(String.format( - "EXEC sys.sp_cdc_enable_table\n" - + "\t@source_schema = N'%s',\n" - + "\t@source_name = N'%s', \n" - + "\t@role_name = N'%s',\n" - + "\t@supports_net_changes = 0", - schemaName, tableName, CDC_ROLE_NAME)); // enables cdc on MODELS_SCHEMA.MODELS_STREAM_NAME, giving CDC_ROLE_NAME select access - failingToStart = false; - } catch (final Exception e) { - if (retryNum >= maxRetries) { - throw e; - } else { - retryNum++; - try { - Thread.sleep(10000); // 10 seconds - } catch (final InterruptedException ex) { - throw new RuntimeException(ex); - } - } - } - } + public void newTableSnapshotTest() { + // Do nothing } @Override - public String columnClause(final Map columnsWithDataType, final Optional primaryKey) { + protected String columnClause(final Map columnsWithDataType, final Optional primaryKey) { final StringBuilder columnClause = new StringBuilder(); int i = 0; for (final Map.Entry column : columnsWithDataType.entrySet()) { @@ -254,73 +205,33 @@ public String columnClause(final Map columnsWithDataType, final return columnClause.toString(); } - @AfterEach - public void tearDown() { - try { - dslContext.close(); - DataSourceFactory.close(dataSource); - DataSourceFactory.close(testDataSource); - } catch (final Exception e) { - throw new RuntimeException(e); - } - } - @Test void testAssertCdcEnabledInDb() { // since we enable cdc in setup, assert that we successfully pass this first - assertDoesNotThrow(() -> source.assertCdcEnabledInDb(config, testJdbcDatabase)); + assertDoesNotThrow(() -> source().assertCdcEnabledInDb(config(), testDatabase())); // then disable cdc and assert the check fails - switchCdcOnDatabase(false, dbName); - assertThrows(RuntimeException.class, () -> source.assertCdcEnabledInDb(config, testJdbcDatabase)); + testdb.withoutCdc(); + assertThrows(RuntimeException.class, () -> source().assertCdcEnabledInDb(config(), testDatabase())); } @Test void testAssertCdcSchemaQueryable() { // correct access granted by setup so assert check passes - assertDoesNotThrow(() -> source.assertCdcSchemaQueryable(config, testJdbcDatabase)); + assertDoesNotThrow(() -> source().assertCdcSchemaQueryable(config(), testDatabase())); // now revoke perms and assert that check fails - alterPermissionsOnSchema(false, "cdc"); - assertThrows(com.microsoft.sqlserver.jdbc.SQLServerException.class, () -> source.assertCdcSchemaQueryable(config, testJdbcDatabase)); - } - - private void switchSqlServerAgentAndWait(final Boolean start) throws InterruptedException { - final String startOrStop = start ? "START" : "STOP"; - executeQuery(String.format("EXEC xp_servicecontrol N'%s',N'SQLServerAGENT';", startOrStop)); - Thread.sleep(15 * 1000); // 15 seconds to wait for change of agent state + testdb.with("REVOKE SELECT ON SCHEMA :: [cdc] TO %s", testUserName()); + assertThrows(com.microsoft.sqlserver.jdbc.SQLServerException.class, + () -> source().assertCdcSchemaQueryable(config(), testDatabase())); } @Test - void testAssertSqlServerAgentRunning() throws InterruptedException { - executeQuery(String.format("USE master;\n" + "GRANT VIEW SERVER STATE TO %s", testUserName)); + void testAssertSqlServerAgentRunning() { + testdb.withAgentStopped().withWaitUntilAgentStopped(); // assert expected failure if sql server agent stopped - switchSqlServerAgentAndWait(false); - assertThrows(RuntimeException.class, () -> source.assertSqlServerAgentRunning(testJdbcDatabase)); + assertThrows(RuntimeException.class, () -> source().assertSqlServerAgentRunning(testDatabase())); // assert success if sql server agent running - switchSqlServerAgentAndWait(true); - assertDoesNotThrow(() -> source.assertSqlServerAgentRunning(testJdbcDatabase)); - } - - @Test - void testAssertSnapshotIsolationAllowed() { - // snapshot isolation enabled by setup so assert check passes - assertDoesNotThrow(() -> source.assertSnapshotIsolationAllowed(config, testJdbcDatabase)); - // now disable snapshot isolation and assert that check fails - switchSnapshotIsolation(false, dbName); - assertThrows(RuntimeException.class, () -> source.assertSnapshotIsolationAllowed(config, testJdbcDatabase)); - } - - @Test - void testAssertSnapshotIsolationDisabled() { - final JsonNode replicationConfig = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "CDC") - .put("data_to_sync", "New Changes Only") - // set snapshot_isolation level to "Read Committed" to disable snapshot - .put("snapshot_isolation", "Read Committed") - .build()); - Jsons.replaceNestedValue(config, List.of("replication_method"), replicationConfig); - assertDoesNotThrow(() -> source.assertSnapshotIsolationAllowed(config, testJdbcDatabase)); - switchSnapshotIsolation(false, dbName); - assertDoesNotThrow(() -> source.assertSnapshotIsolationAllowed(config, testJdbcDatabase)); + testdb.withAgentStarted().withWaitUntilAgentRunning(); + assertDoesNotThrow(() -> source().assertSqlServerAgentRunning(testDatabase())); } // Ensure the CDC check operations are included when CDC is enabled @@ -328,47 +239,50 @@ void testAssertSnapshotIsolationDisabled() { @Test void testCdcCheckOperations() throws Exception { // assertCdcEnabledInDb - switchCdcOnDatabase(false, dbName); - AirbyteConnectionStatus status = getSource().check(getConfig()); + testdb.withoutCdc(); + AirbyteConnectionStatus status = source().check(config()); assertEquals(status.getStatus(), AirbyteConnectionStatus.Status.FAILED); - switchCdcOnDatabase(true, dbName); + testdb.withCdc(); // assertCdcSchemaQueryable - alterPermissionsOnSchema(false, "cdc"); - status = getSource().check(getConfig()); + testdb.with("REVOKE SELECT ON SCHEMA :: [cdc] TO %s", testUserName()); + status = source().check(config()); assertEquals(status.getStatus(), AirbyteConnectionStatus.Status.FAILED); - alterPermissionsOnSchema(true, "cdc"); + testdb.with("GRANT SELECT ON SCHEMA :: [cdc] TO %s", testUserName()); + // assertSqlServerAgentRunning - executeQuery(String.format("USE master;\n" + "GRANT VIEW SERVER STATE TO %s", testUserName)); - switchSqlServerAgentAndWait(false); - status = getSource().check(getConfig()); + + testdb.withAgentStopped().withWaitUntilAgentStopped(); + status = source().check(config()); assertEquals(status.getStatus(), AirbyteConnectionStatus.Status.FAILED); - switchSqlServerAgentAndWait(true); - // assertSnapshotIsolationAllowed - switchSnapshotIsolation(false, dbName); - status = getSource().check(getConfig()); + testdb.withAgentStarted().withWaitUntilAgentRunning(); + status = source().check(config()); assertEquals(status.getStatus(), AirbyteConnectionStatus.Status.FAILED); } @Test void testCdcCheckOperationsWithDot() throws Exception { - // assertCdcEnabledInDb and validate escape with special character - switchCdcOnDatabase(true, dbNamewithDot); - final AirbyteConnectionStatus status = getSource().check(getConfig()); + final String dbNameWithDot = testdb.getDatabaseName().replace("_", "."); + testdb.with("CREATE DATABASE [%s];", dbNameWithDot) + .with("USE [%s]", dbNameWithDot) + .with("EXEC sys.sp_cdc_enable_db;"); + final AirbyteConnectionStatus status = source().check(config()); assertEquals(status.getStatus(), AirbyteConnectionStatus.Status.SUCCEEDED); } // todo: check LSN returned is actually the max LSN // todo: check we fail as expected under certain conditions @Test - void testGetTargetPosition() throws InterruptedException { - Thread.sleep(10 * 1000); // Sleeping because sometimes the db is not yet completely ready and the lsn is not found + void testGetTargetPosition() { // check that getTargetPosition returns higher Lsn after inserting new row - final Lsn firstLsn = MssqlCdcTargetPosition.getTargetPosition(testJdbcDatabase, dbName).targetLsn; - executeQuery(String.format("USE %s; INSERT INTO %s.%s (%s, %s, %s) VALUES (%s, %s, '%s');", - dbName, MODELS_SCHEMA, MODELS_STREAM_NAME, COL_ID, COL_MAKE_ID, COL_MODEL, 910019, 1, "another car")); - Thread.sleep(15 * 1000); // 15 seconds to wait for Agent capture job to log cdc change - final Lsn secondLsn = MssqlCdcTargetPosition.getTargetPosition(testJdbcDatabase, dbName).targetLsn; - assertTrue(secondLsn.compareTo(firstLsn) > 0); + testdb.withWaitUntilMaxLsnAvailable(); + final Lsn firstLsn = MssqlCdcTargetPosition.getTargetPosition(testDatabase(), testdb.getDatabaseName()).targetLsn; + testdb.with("INSERT INTO %s.%s (%s, %s, %s) VALUES (%s, %s, '%s');", + modelsSchema(), MODELS_STREAM_NAME, COL_ID, COL_MAKE_ID, COL_MODEL, 910019, 1, "another car"); + // Wait for Agent capture job to log CDC change. + await().atMost(Duration.ofSeconds(45)).until(() -> { + final Lsn secondLsn = MssqlCdcTargetPosition.getTargetPosition(testDatabase(), testdb.getDatabaseName()).targetLsn; + return secondLsn.compareTo(firstLsn) > 0; + }); } @Override @@ -382,24 +296,12 @@ protected void removeCDCColumns(final ObjectNode data) { @Override protected MssqlCdcTargetPosition cdcLatestTargetPosition() { - try { - // Sleeping because sometimes the db is not yet completely ready and the lsn is not found - Thread.sleep(5000); - } catch (final InterruptedException e) { - throw new RuntimeException(e); - } + testdb.withWaitUntilMaxLsnAvailable(); final JdbcDatabase jdbcDatabase = new StreamingJdbcDatabase( - DataSourceFactory.create(config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DRIVER_CLASS, - String.format("jdbc:sqlserver://%s:%s;databaseName=%s;", - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asInt(), - dbName), - Map.of("encrypt", "false")), + testDataSource, new MssqlSourceOperations(), AdaptiveStreamingQueryConfig::new); - return MssqlCdcTargetPosition.getTargetPosition(jdbcDatabase, dbName); + return MssqlCdcTargetPosition.getTargetPosition(jdbcDatabase, testdb.getDatabaseName()); } @Override @@ -451,21 +353,6 @@ protected void addCdcDefaultCursorField(final AirbyteStream stream) { } } - @Override - protected Source getSource() { - return new MssqlSource(); - } - - @Override - protected JsonNode getConfig() { - return config; - } - - @Override - protected Database getDatabase() { - return database; - } - @Override protected void assertExpectedStateMessages(final List stateMessages) { assertEquals(1, stateMessages.size()); diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSslSourceTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSslSourceTest.java new file mode 100644 index 000000000000..fca405bd3948 --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSslSourceTest.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.JdbcConnector; +import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.CertificateKey; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Map; +import javax.sql.DataSource; +import org.testcontainers.containers.MSSQLServerContainer; +import org.testcontainers.utility.DockerImageName; + +public class CdcMssqlSslSourceTest extends CdcMssqlSourceTest { + + CdcMssqlSslSourceTest() { + super(); + } + + protected MSSQLServerContainer createContainer() { + final MsSQLContainerFactory containerFactory = new MsSQLContainerFactory(); + final MSSQLServerContainer container = + containerFactory.createNewContainer(DockerImageName.parse("mcr.microsoft.com/mssql/server:2022-latest")); + containerFactory.withSslCertificates(container); + return container; + } + + @Override + final protected MsSQLTestDatabase createTestDatabase() { + final var testdb = new MsSQLTestDatabase(privateContainer); + return testdb + .withConnectionProperty("encrypt", "true") + .withConnectionProperty("databaseName", testdb.getDatabaseName()) + .withConnectionProperty("trustServerCertificate", "true") + .initialized() + .withCdc() + .withWaitUntilAgentRunning(); + } + + @Override + protected DataSource createTestDataSource() { + return DataSourceFactory.create( + testUserName(), + testdb.getPassword(), + testdb.getDatabaseDriver().getDriverClassName(), + testdb.getJdbcUrl(), + Map.of("encrypt", "true", "databaseName", testdb.getDatabaseName(), "trustServerCertificate", "true"), + JdbcConnector.CONNECT_TIMEOUT_DEFAULT); + } + + @Override + protected JsonNode config() { + final String containerIp; + try { + containerIp = InetAddress.getByName(testdb.getContainer().getHost()) + .getHostAddress(); + } catch (final UnknownHostException e) { + throw new RuntimeException(e); + } + final String certificate = testdb.getCertificate(CertificateKey.SERVER); + return testdb.configBuilder() + .withEncrytedVerifyServerCertificate(certificate, testdb.getContainer().getHost()) + .with(JdbcUtils.HOST_KEY, containerIp) + .with(JdbcUtils.PORT_KEY, testdb.getContainer().getFirstMappedPort()) + .withDatabase() + .with(JdbcUtils.USERNAME_KEY, testUserName()) + .with(JdbcUtils.PASSWORD_KEY, testdb.getPassword()) + .withSchemas(modelsSchema(), randomSchema()) + .withCdcReplication() + .build(); + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcStateCompressionTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcStateCompressionTest.java new file mode 100644 index 000000000000..f1856f79e064 --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcStateCompressionTest.java @@ -0,0 +1,257 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql; + +import static io.airbyte.integrations.source.mssql.MssqlSource.IS_COMPRESSED; +import static io.airbyte.integrations.source.mssql.MssqlSource.MSSQL_CDC_OFFSET; +import static io.airbyte.integrations.source.mssql.MssqlSource.MSSQL_DB_HISTORY; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateGeneratorUtils; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.util.AutoCloseableIterators; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteCatalog; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; +import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.SyncMode; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.MSSQLServerContainer; + +public class CdcStateCompressionTest { + + static private final String CDC_ROLE_NAME = "cdc_selector"; + + static private final String TEST_USER_NAME_PREFIX = "cdc_test_user"; + + static private final String TEST_SCHEMA = "test_schema"; + + static private final int TEST_TABLES = 10; + + static private final int ADDED_COLUMNS = 1000; + + static private final MSSQLServerContainer CONTAINER = new MsSQLContainerFactory().shared( + "mcr.microsoft.com/mssql/server:2022-latest", "withAgent"); + + private MsSQLTestDatabase testdb; + + @BeforeEach + public void setup() { + testdb = new MsSQLTestDatabase(CONTAINER); + testdb = testdb + .withConnectionProperty("encrypt", "false") + .withConnectionProperty("databaseName", testdb.getDatabaseName()) + .initialized() + .withCdc() + .withWaitUntilAgentRunning(); + + // Create a test schema and a bunch of test tables with CDC enabled. + // Insert one row in each table so that they're not empty. + final var enableCdcSqlFmt = """ + EXEC sys.sp_cdc_enable_table + \t@source_schema = N'%s', + \t@source_name = N'test_table_%d', + \t@role_name = N'%s', + \t@supports_net_changes = 0, + \t@capture_instance = N'capture_instance_%d_%d' + """; + testdb.with("CREATE SCHEMA %s;", TEST_SCHEMA); + for (int i = 0; i < TEST_TABLES; i++) { + testdb + .with("CREATE TABLE %s.test_table_%d (id INT IDENTITY(1,1) PRIMARY KEY);", TEST_SCHEMA, i) + .with(enableCdcSqlFmt, TEST_SCHEMA, i, CDC_ROLE_NAME, i, 1) + .withShortenedCapturePollingInterval() + .with("INSERT INTO %s.test_table_%d DEFAULT VALUES", TEST_SCHEMA, i); + } + + // Create a test user to be used by the source, with proper permissions. + testdb + .with("CREATE LOGIN %s WITH PASSWORD = '%s', DEFAULT_DATABASE = %s", testUserName(), testdb.getPassword(), testdb.getDatabaseName()) + .with("CREATE USER %s FOR LOGIN %s WITH DEFAULT_SCHEMA = [dbo]", testUserName(), testUserName()) + .with("REVOKE ALL FROM %s CASCADE;", testUserName()) + .with("EXEC sp_msforeachtable \"REVOKE ALL ON '?' TO %s;\"", testUserName()) + .with("GRANT SELECT ON SCHEMA :: [%s] TO %s", TEST_SCHEMA, testUserName()) + .with("GRANT SELECT ON SCHEMA :: [cdc] TO %s", testUserName()) + .with("USE [master]") + .with("GRANT VIEW SERVER STATE TO %s", testUserName()) + .with("USE [%s]", testdb.getDatabaseName()) + .with("EXEC sp_addrolemember N'%s', N'%s';", CDC_ROLE_NAME, testUserName()); + + // Increase schema history size to trigger state compression. + // We do this by adding lots of columns with long names, + // then migrating to a new CDC capture instance for each table. + // This is admittedly somewhat awkward and perhaps could be improved. + final var disableCdcSqlFmt = """ + EXEC sys.sp_cdc_disable_table + \t@source_schema = N'%s', + \t@source_name = N'test_table_%d', + \t@capture_instance = N'capture_instance_%d_%d' + """; + for (int i = 0; i < TEST_TABLES; i++) { + final var sb = new StringBuilder(); + sb.append("ALTER TABLE ").append(TEST_SCHEMA).append(".test_table_").append(i).append(" ADD"); + for (int j = 0; j < ADDED_COLUMNS; j++) { + sb.append((j > 0) ? ", " : " ") + .append("rather_long_column_name_________________________________________________________________________________________").append(j) + .append(" INT NULL"); + } + testdb + .with(sb.toString()) + .with(enableCdcSqlFmt, TEST_SCHEMA, i, CDC_ROLE_NAME, i, 2) + .with(disableCdcSqlFmt, TEST_SCHEMA, i, i, 1) + .withShortenedCapturePollingInterval(); + } + } + + private AirbyteCatalog getCatalog() { + final var streams = new ArrayList(); + for (int i = 0; i < TEST_TABLES; i++) { + streams.add(CatalogHelpers.createAirbyteStream( + "test_table_%d".formatted(i), + TEST_SCHEMA, + Field.of("id", JsonSchemaType.INTEGER)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of("id")))); + } + return new AirbyteCatalog().withStreams(streams); + } + + private ConfiguredAirbyteCatalog getConfiguredCatalog() { + final var configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog(getCatalog()); + configuredCatalog.getStreams().forEach(s -> s.setSyncMode(SyncMode.INCREMENTAL)); + return configuredCatalog; + } + + private MssqlSource source() { + return new MssqlSource(); + } + + private JsonNode config() { + return testdb.configBuilder() + .withHostAndPort() + .withDatabase() + .with(JdbcUtils.USERNAME_KEY, testUserName()) + .with(JdbcUtils.PASSWORD_KEY, testdb.getPassword()) + .withSchemas(TEST_SCHEMA) + .withoutSsl() + // Configure for CDC replication but with a higher timeout than usual. + // This is because Debezium requires more time than usual to build the initial snapshot. + .with("is_test", true) + .with("replication_method", Map.of( + "method", "CDC", + "initial_waiting_seconds", 60)) + + .build(); + } + + private String testUserName() { + return testdb.withNamespace(TEST_USER_NAME_PREFIX); + } + + /** + * This test is similar in principle to {@link CdcMysqlSourceTest.testCompressedSchemaHistory}. + */ + @Test + public void testCompressedSchemaHistory() throws Exception { + // First sync. + final var firstBatchIterator = source().read(config(), getConfiguredCatalog(), null); + final var dataFromFirstBatch = AutoCloseableIterators.toListAndClose(firstBatchIterator); + final AirbyteStateMessage lastStateMessageFromFirstBatch = + StateGeneratorUtils.convertLegacyStateToGlobalState(Iterables.getLast(extractStateMessages(dataFromFirstBatch))); + assertNotNull(lastStateMessageFromFirstBatch.getGlobal().getSharedState()); + final var lastSharedStateFromFirstBatch = lastStateMessageFromFirstBatch.getGlobal().getSharedState().get("state"); + assertNotNull(lastSharedStateFromFirstBatch); + assertNotNull(lastSharedStateFromFirstBatch.get(MSSQL_DB_HISTORY)); + assertNotNull(lastSharedStateFromFirstBatch.get(MSSQL_CDC_OFFSET)); + assertNotNull(lastSharedStateFromFirstBatch.get(IS_COMPRESSED)); + assertTrue(lastSharedStateFromFirstBatch.get(IS_COMPRESSED).asBoolean()); + final var recordsFromFirstBatch = extractRecordMessages(dataFromFirstBatch); + assertEquals(TEST_TABLES, recordsFromFirstBatch.size()); + for (final var record : recordsFromFirstBatch) { + assertEquals("1", record.getData().get("id").toString()); + } + + // Insert a bunch of records (1 per table, again). + for (int i = 0; i < TEST_TABLES; i++) { + testdb.with("INSERT %s.test_table_%d DEFAULT VALUES;", TEST_SCHEMA, i); + } + + // Second sync. + final var secondBatchStateForRead = Jsons.jsonNode(Collections.singletonList(Iterables.getLast(extractStateMessages(dataFromFirstBatch)))); + final var secondBatchIterator = source().read(config(), getConfiguredCatalog(), secondBatchStateForRead); + final var dataFromSecondBatch = AutoCloseableIterators.toListAndClose(secondBatchIterator); + final AirbyteStateMessage lastStateMessageFromSecondBatch = + StateGeneratorUtils.convertLegacyStateToGlobalState(Iterables.getLast(extractStateMessages(dataFromSecondBatch))); + assertNotNull(lastStateMessageFromSecondBatch.getGlobal().getSharedState()); + final var lastSharedStateFromSecondBatch = lastStateMessageFromSecondBatch.getGlobal().getSharedState().get("state"); + assertNotNull(lastSharedStateFromSecondBatch); + assertNotNull(lastSharedStateFromSecondBatch.get(MSSQL_DB_HISTORY)); + assertEquals(lastSharedStateFromFirstBatch.get(MSSQL_DB_HISTORY), lastSharedStateFromSecondBatch.get(MSSQL_DB_HISTORY)); + assertNotNull(lastSharedStateFromSecondBatch.get(MSSQL_CDC_OFFSET)); + assertNotNull(lastSharedStateFromSecondBatch.get(IS_COMPRESSED)); + assertTrue(lastSharedStateFromSecondBatch.get(IS_COMPRESSED).asBoolean()); + final var recordsFromSecondBatch = extractRecordMessages(dataFromSecondBatch); + assertEquals(TEST_TABLES, recordsFromSecondBatch.size()); + for (final var record : recordsFromSecondBatch) { + assertEquals("2", record.getData().get("id").toString()); + } + } + + @AfterEach + public void tearDown() { + testdb.close(); + } + + private Set extractRecordMessages(final List messages) { + final var recordsPerStream = extractRecordMessagesStreamWise(messages); + return recordsPerStream.values().stream().flatMap(Set::stream).collect(Collectors.toSet()); + } + + private Map> extractRecordMessagesStreamWise(final List messages) { + final var recordsPerStream = messages.stream() + .filter(m -> m.getType() == Type.RECORD) + .map(AirbyteMessage::getRecord) + .collect(Collectors.groupingBy(AirbyteRecordMessage::getStream)); + + final Map> recordsPerStreamWithNoDuplicates = new HashMap<>(); + for (final var entry : recordsPerStream.entrySet()) { + final var set = new HashSet<>(entry.getValue()); + recordsPerStreamWithNoDuplicates.put(entry.getKey(), set); + assertEquals(entry.getValue().size(), set.size(), "duplicate records in sync for " + entry.getKey()); + } + + return recordsPerStreamWithNoDuplicates; + } + + private List extractStateMessages(final List messages) { + return messages.stream() + .filter(r -> r.getType() == Type.STATE) + .map(AirbyteMessage::getState) + .toList(); + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CloudDeploymentMssqlTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CloudDeploymentMssqlTest.java new file mode 100644 index 000000000000..1889315bea6f --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CloudDeploymentMssqlTest.java @@ -0,0 +1,120 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.base.ssh.SshBastionContainer; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import io.airbyte.commons.features.EnvVariableFeatureFlags; +import io.airbyte.commons.features.FeatureFlagsWrapper; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +@Execution(ExecutionMode.CONCURRENT) +public class CloudDeploymentMssqlTest { + + private MsSQLTestDatabase createTestDatabase(String... containerFactoryMethods) { + final var container = new MsSQLContainerFactory().shared( + "mcr.microsoft.com/mssql/server:2022-latest", containerFactoryMethods); + final var testdb = new MsSQLTestDatabase(container); + return testdb + .withConnectionProperty("encrypt", "true") + .withConnectionProperty("trustServerCertificate", "true") + .withConnectionProperty("databaseName", testdb.getDatabaseName()) + .initialized(); + } + + private Source source() { + final var source = new MssqlSource(); + source.setFeatureFlags(FeatureFlagsWrapper.overridingDeploymentMode(new EnvVariableFeatureFlags(), "CLOUD")); + return MssqlSource.sshWrappedSource(source); + } + + @Test + void testStrictSSLUnsecuredNoTunnel() throws Exception { + try (final var testdb = createTestDatabase()) { + final var config = testdb.configBuilder() + .withHostAndPort() + .withDatabase() + .with(JdbcUtils.USERNAME_KEY, testdb.getUserName()) + .with(JdbcUtils.PASSWORD_KEY, "fake") + .withoutSsl() + .with("tunnel_method", ImmutableMap.builder().put("tunnel_method", "NO_TUNNEL").build()) + .build(); + final AirbyteConnectionStatus actual = source().check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, actual.getStatus()); + assertTrue(actual.getMessage().contains("Unsecured connection not allowed"), actual.getMessage()); + } + } + + @Test + void testStrictSSLSecuredNoTunnel() throws Exception { + try (final var testdb = createTestDatabase()) { + final var config = testdb.testConfigBuilder() + .withEncrytedTrustServerCertificate() + .with("tunnel_method", ImmutableMap.builder().put("tunnel_method", "NO_TUNNEL").build()) + .build(); + final AirbyteConnectionStatus actual = source().check(config); + assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, actual.getStatus()); + } + } + + @Test + void testStrictSSLSecuredWithTunnel() throws Exception { + try (final var testdb = createTestDatabase()) { + final var config = testdb.configBuilder() + .withHostAndPort() + .withDatabase() + .with(JdbcUtils.USERNAME_KEY, testdb.getUserName()) + .with(JdbcUtils.PASSWORD_KEY, "fake") + .withEncrytedTrustServerCertificate() + .with("tunnel_method", ImmutableMap.builder().put("tunnel_method", "SSH_KEY_AUTH").build()) + .build(); + final AirbyteConnectionStatus actual = source().check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, actual.getStatus()); + assertTrue(actual.getMessage().contains("Could not connect with provided SSH configuration."), actual.getMessage()); + } + } + + @Test + void testStrictSSLUnsecuredWithTunnel() throws Exception { + try (final var testdb = createTestDatabase()) { + final var config = testdb.configBuilder() + .withHostAndPort() + .withDatabase() + .with(JdbcUtils.USERNAME_KEY, testdb.getUserName()) + .with(JdbcUtils.PASSWORD_KEY, "fake") + .withEncrytedTrustServerCertificate() + .with("tunnel_method", ImmutableMap.builder().put("tunnel_method", "SSH_KEY_AUTH").build()) + .build(); + final AirbyteConnectionStatus actual = source().check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, actual.getStatus()); + assertTrue(actual.getMessage().contains("Could not connect with provided SSH configuration."), actual.getMessage()); + } + } + + @Test + void testCheckWithSslModeDisabled() throws Exception { + try (final var testdb = createTestDatabase("withNetwork")) { + try (final SshBastionContainer bastion = new SshBastionContainer()) { + bastion.initAndStartBastion(testdb.getContainer().getNetwork()); + final var config = testdb.integrationTestConfigBuilder() + .with("tunnel_method", bastion.getTunnelMethod(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, false)) + .withoutSsl() + .build(); + final AirbyteConnectionStatus actual = source().check(config); + assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, actual.getStatus()); + } + } + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlCdcHelperTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlCdcHelperTest.java index a2f29d5064a7..d1ec53fe1915 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlCdcHelperTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlCdcHelperTest.java @@ -4,14 +4,11 @@ package io.airbyte.integrations.source.mssql; -import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.source.mssql.MssqlCdcHelper.DataToSync; -import io.airbyte.integrations.source.mssql.MssqlCdcHelper.SnapshotIsolation; import java.util.Map; import org.junit.jupiter.api.Test; @@ -33,9 +30,7 @@ public void testIsCdc() { final JsonNode newCdc = Jsons.jsonNode(Map.of("replication_method", Jsons.jsonNode(Map.of( - "method", "CDC", - "data_to_sync", "Existing and New", - "snapshot_isolation", "Snapshot")))); + "method", "CDC")))); assertTrue(MssqlCdcHelper.isCdc(newCdc)); // migration from legacy to new config @@ -46,90 +41,10 @@ public void testIsCdc() { final JsonNode mixCdc = Jsons.jsonNode(Map.of( "replication", Jsons.jsonNode(Map.of( - "replication_type", "Standard", - "data_to_sync", "Existing and New", - "snapshot_isolation", "Snapshot")), + "replication_type", "Standard")), "replication_method", Jsons.jsonNode(Map.of( - "method", "CDC", - "data_to_sync", "Existing and New", - "snapshot_isolation", "Snapshot")))); + "method", "CDC")))); assertTrue(MssqlCdcHelper.isCdc(mixCdc)); } - @Test - public void testGetSnapshotIsolation() { - // legacy replication method config before version 0.4.0 - assertEquals(SnapshotIsolation.SNAPSHOT, MssqlCdcHelper.getSnapshotIsolationConfig(LEGACY_CDC_CONFIG)); - - // new replication method config since version 0.4.0 - final JsonNode newCdcNonSnapshot = Jsons.jsonNode(Map.of("replication_method", - Jsons.jsonNode(Map.of( - "method", "CDC", - "data_to_sync", "Existing and New", - "snapshot_isolation", "Read Committed")))); - assertEquals(SnapshotIsolation.READ_COMMITTED, MssqlCdcHelper.getSnapshotIsolationConfig(newCdcNonSnapshot)); - - final JsonNode newCdcSnapshot = Jsons.jsonNode(Map.of("replication_method", - Jsons.jsonNode(Map.of( - "method", "CDC", - "data_to_sync", "Existing and New", - "snapshot_isolation", "Snapshot")))); - assertEquals(SnapshotIsolation.SNAPSHOT, MssqlCdcHelper.getSnapshotIsolationConfig(newCdcSnapshot)); - - // migration from legacy to new config - final JsonNode mixCdcNonSnapshot = Jsons.jsonNode(Map.of( - "replication", "Standard", - "replication_method", Jsons.jsonNode(Map.of( - "method", "CDC", - "data_to_sync", "Existing and New", - "snapshot_isolation", "Read Committed")))); - assertEquals(SnapshotIsolation.READ_COMMITTED, MssqlCdcHelper.getSnapshotIsolationConfig(mixCdcNonSnapshot)); - - final JsonNode mixCdcSnapshot = Jsons.jsonNode(Map.of( - "replication", "Standard", - "replication_method", Jsons.jsonNode(Map.of( - "method", "CDC", - "data_to_sync", "Existing and New", - "snapshot_isolation", "Snapshot")))); - assertEquals(SnapshotIsolation.SNAPSHOT, MssqlCdcHelper.getSnapshotIsolationConfig(mixCdcSnapshot)); - } - - @Test - public void testGetDataToSyncConfig() { - // legacy replication method config before version 0.4.0 - assertEquals(DataToSync.EXISTING_AND_NEW, MssqlCdcHelper.getDataToSyncConfig(LEGACY_CDC_CONFIG)); - - // new replication method config since version 0.4.0 - final JsonNode newCdcExistingAndNew = Jsons.jsonNode(Map.of("replication_method", - Jsons.jsonNode(Map.of( - "method", "CDC", - "data_to_sync", "Existing and New", - "snapshot_isolation", "Read Committed")))); - assertEquals(DataToSync.EXISTING_AND_NEW, MssqlCdcHelper.getDataToSyncConfig(newCdcExistingAndNew)); - - final JsonNode newCdcNewOnly = Jsons.jsonNode(Map.of("replication_method", - Jsons.jsonNode(Map.of( - "method", "CDC", - "data_to_sync", "New Changes Only", - "snapshot_isolation", "Snapshot")))); - assertEquals(DataToSync.NEW_CHANGES_ONLY, MssqlCdcHelper.getDataToSyncConfig(newCdcNewOnly)); - - final JsonNode mixCdcExistingAndNew = Jsons.jsonNode(Map.of( - "replication", "Standard", - "replication_method", Jsons.jsonNode(Map.of( - "method", "CDC", - "data_to_sync", "Existing and New", - "snapshot_isolation", "Read Committed")))); - assertEquals(DataToSync.EXISTING_AND_NEW, MssqlCdcHelper.getDataToSyncConfig(mixCdcExistingAndNew)); - - final JsonNode mixCdcNewOnly = Jsons.jsonNode(Map.of( - "replication", "Standard", - "replication_method", - Jsons.jsonNode(Map.of( - "method", "CDC", - "data_to_sync", "New Changes Only", - "snapshot_isolation", "Snapshot")))); - assertEquals(DataToSync.NEW_CHANGES_ONLY, MssqlCdcHelper.getDataToSyncConfig(mixCdcNewOnly)); - } - } diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlDataSourceFactoryTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlDataSourceFactoryTest.java new file mode 100644 index 000000000000..c653f434a798 --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlDataSourceFactoryTest.java @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import com.zaxxer.hikari.HikariDataSource; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import java.util.Map; +import javax.sql.DataSource; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.MSSQLServerContainer; + +public class MssqlDataSourceFactoryTest { + + @Test + protected void testCreatingDataSourceWithConnectionTimeoutSetBelowDefault() { + final MSSQLServerContainer container = new MsSQLContainerFactory().shared("mcr.microsoft.com/mssql/server:2019-latest"); + final Map connectionProperties = Map.of("loginTimeout", String.valueOf(5)); + final DataSource dataSource = DataSourceFactory.create( + container.getUsername(), + container.getPassword(), + container.getDriverClassName(), + container.getJdbcUrl(), + connectionProperties, + new MssqlSource().getConnectionTimeoutMssql(connectionProperties)); + assertNotNull(dataSource); + assertEquals(HikariDataSource.class, dataSource.getClass()); + assertEquals(5000, ((HikariDataSource) dataSource).getHikariConfigMXBean().getConnectionTimeout()); + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java index d8ba0e6abd85..dc8660c6a36b 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlJdbcSourceAcceptanceTest.java @@ -9,166 +9,148 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.string.Strings; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.BaseImage; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import java.sql.JDBCType; -import java.util.Map; -import javax.sql.DataSource; -import org.junit.jupiter.api.AfterAll; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.SyncMode; +import java.util.Collections; +import java.util.List; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.testcontainers.containers.MSSQLServerContainer; -public class MssqlJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { +public class MssqlJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { protected static final String USERNAME_WITHOUT_PERMISSION = "new_user"; protected static final String PASSWORD_WITHOUT_PERMISSION = "password_3435!"; - private static MSSQLServerContainer dbContainer; - private JsonNode config; - @BeforeAll - static void init() { + static { // In mssql, timestamp is generated automatically, so we need to use // the datetime type instead so that we can set the value manually. - COL_TIMESTAMP_TYPE = "DATETIME"; - - dbContainer = new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2019-latest").acceptLicense(); - dbContainer.start(); + COL_TIMESTAMP_TYPE = "DATETIME2"; } @Override - protected DataSource getDataSource(final JsonNode jdbcConfig) { - final Map connectionProperties = JdbcUtils.parseJdbcParameters(jdbcConfig, JdbcUtils.CONNECTION_PROPERTIES_KEY, - getJdbcParameterDelimiter()); - connectionProperties.put("encrypt", "false"); - return DataSourceFactory.create( - jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText(), - jdbcConfig.has(JdbcUtils.PASSWORD_KEY) ? jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, - getDriverClass(), - jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), - connectionProperties); - } - - @BeforeEach - public void setup() throws Exception { - final JsonNode configWithoutDbName = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, dbContainer.getHost()) - .put(JdbcUtils.PORT_KEY, dbContainer.getFirstMappedPort()) - .put(JdbcUtils.USERNAME_KEY, dbContainer.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, dbContainer.getPassword()) - .build()); - - final DataSource dataSource = DataSourceFactory.create( - configWithoutDbName.get(JdbcUtils.USERNAME_KEY).asText(), - configWithoutDbName.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.MSSQLSERVER.getDriverClassName(), - String.format("jdbc:sqlserver://%s:%d;", - configWithoutDbName.get(JdbcUtils.HOST_KEY).asText(), - configWithoutDbName.get(JdbcUtils.PORT_KEY).asInt()), - Map.of("encrypt", "false")); - - try { - final JdbcDatabase database = new DefaultJdbcDatabase(dataSource); - - final String dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); - - database.execute(ctx -> ctx.createStatement().execute(String.format("CREATE DATABASE %s;", dbName))); - - config = Jsons.clone(configWithoutDbName); - ((ObjectNode) config).put(JdbcUtils.DATABASE_KEY, dbName); - ((ObjectNode) config).put("ssl_method", Jsons.jsonNode(Map.of("ssl_method", "unencrypted"))); - - super.setup(); - } finally { - DataSourceFactory.close(dataSource); - } - } - - @AfterAll - public static void cleanUp() throws Exception { - dbContainer.close(); + protected JsonNode config() { + return testdb.testConfigBuilder() + .withoutSsl() + .build(); } @Override - public boolean supportsSchemas() { - return true; + protected MssqlSource source() { + return new MssqlSource(); } @Override - public JsonNode getConfig() { - return Jsons.clone(config); + protected MsSQLTestDatabase createTestDatabase() { + return MsSQLTestDatabase.in(BaseImage.MSSQL_2022); } @Override - public AbstractJdbcSource getJdbcSource() { - return new MssqlSource(); + public boolean supportsSchemas() { + return true; } @Override - public String getDriverClass() { - return MssqlSource.DRIVER_CLASS; + protected void maybeSetShorterConnectionTimeout(final JsonNode config) { + ((ObjectNode) config).put(JdbcUtils.JDBC_URL_PARAMS_KEY, "loginTimeout=1"); } @Test void testCheckIncorrectPasswordFailure() throws Exception { + final var config = config(); + maybeSetShorterConnectionTimeout(config); ((ObjectNode) config).put(JdbcUtils.PASSWORD_KEY, "fake"); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); Assertions.assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: S0001; Error code: 18456;")); + assertTrue(status.getMessage().contains("State code: S0001; Error code: 18456;"), status.getMessage()); } @Test public void testCheckIncorrectUsernameFailure() throws Exception { + final var config = config(); + maybeSetShorterConnectionTimeout(config); ((ObjectNode) config).put(JdbcUtils.USERNAME_KEY, "fake"); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); Assertions.assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: S0001; Error code: 18456;")); + assertTrue(status.getMessage().contains("State code: S0001; Error code: 18456;"), status.getMessage()); } @Test public void testCheckIncorrectHostFailure() throws Exception { + final var config = config(); + maybeSetShorterConnectionTimeout(config); ((ObjectNode) config).put(JdbcUtils.HOST_KEY, "localhost2"); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); Assertions.assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 08S01;")); + assertTrue(status.getMessage().contains("State code: 08S01;"), status.getMessage()); } @Test public void testCheckIncorrectPortFailure() throws Exception { + final var config = config(); + maybeSetShorterConnectionTimeout(config); ((ObjectNode) config).put(JdbcUtils.PORT_KEY, "0000"); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); Assertions.assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 08S01;")); + assertTrue(status.getMessage().contains("State code: 08S01;"), status.getMessage()); } @Test public void testCheckIncorrectDataBaseFailure() throws Exception { + final var config = config(); + maybeSetShorterConnectionTimeout(config); ((ObjectNode) config).put(JdbcUtils.DATABASE_KEY, "wrongdatabase"); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); Assertions.assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: S0001; Error code: 4060;")); + assertTrue(status.getMessage().contains("State code: S0001; Error code: 4060;"), status.getMessage()); } @Test public void testUserHasNoPermissionToDataBase() throws Exception { - database.execute(ctx -> ctx.createStatement() - .execute(String.format("CREATE LOGIN %s WITH PASSWORD = '%s'; ", USERNAME_WITHOUT_PERMISSION, PASSWORD_WITHOUT_PERMISSION))); + final var config = config(); + maybeSetShorterConnectionTimeout(config); + testdb.with("CREATE LOGIN %s WITH PASSWORD = '%s'; ", USERNAME_WITHOUT_PERMISSION, PASSWORD_WITHOUT_PERMISSION); ((ObjectNode) config).put(JdbcUtils.USERNAME_KEY, USERNAME_WITHOUT_PERMISSION); ((ObjectNode) config).put(JdbcUtils.PASSWORD_KEY, PASSWORD_WITHOUT_PERMISSION); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: S0001; Error code: 4060;")); + assertTrue(status.getMessage().contains("State code: S0001; Error code: 4060;"), status.getMessage()); + } + + @Override + protected AirbyteCatalog getCatalog(final String defaultNamespace) { + return new AirbyteCatalog().withStreams(List.of( + CatalogHelpers.createAirbyteStream( + TABLE_NAME, + defaultNamespace, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID))), + CatalogHelpers.createAirbyteStream( + TABLE_NAME_WITHOUT_PK, + defaultNamespace, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(Collections.emptyList()), + CatalogHelpers.createAirbyteStream( + TABLE_NAME_COMPOSITE_PK, + defaultNamespace, + Field.of(COL_FIRST_NAME, JsonSchemaType.STRING), + Field.of(COL_LAST_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey( + List.of(List.of(COL_FIRST_NAME), List.of(COL_LAST_NAME))))); } } diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSourceTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSourceTest.java index d9154c3a06b9..098ebab0ef41 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSourceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSourceTest.java @@ -9,18 +9,10 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import io.airbyte.commons.exceptions.ConfigErrorException; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.string.Strings; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.BaseImage; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; @@ -29,67 +21,48 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; import io.airbyte.protocol.models.v0.SyncMode; -import java.sql.SQLException; import java.util.Collections; import java.util.List; -import java.util.Map; -import org.jooq.DSLContext; import org.junit.jupiter.api.*; -import org.testcontainers.containers.MSSQLServerContainer; class MssqlSourceTest { - private static final String DB_NAME = "dbo"; private static final String STREAM_NAME = "id_and_name"; private static final AirbyteCatalog CATALOG = new AirbyteCatalog().withStreams(Lists.newArrayList(CatalogHelpers.createAirbyteStream( STREAM_NAME, - DB_NAME, + "dbo", Field.of("id", JsonSchemaType.INTEGER), Field.of("name", JsonSchemaType.STRING), - Field.of("born", JsonSchemaType.STRING)) + Field.of("born", JsonSchemaType.STRING_TIMESTAMP_WITH_TIMEZONE)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) .withSourceDefinedPrimaryKey(List.of(List.of("id"))))); - private JsonNode configWithoutDbName; - private JsonNode config; + private MsSQLTestDatabase testdb; - private static MSSQLServerContainer db; - - @BeforeAll - static void init() { - db = new MSSQLServerContainer<>("mcr.microsoft.com/mssql/server:2019-latest").acceptLicense(); - db.start(); + private MssqlSource source() { + return new MssqlSource(); } // how to interact with the mssql test container manaully. // 1. exec into mssql container (not the test container container) // 2. /opt/mssql-tools/bin/sqlcmd -S localhost -U SA -P "A_Str0ng_Required_Password" @BeforeEach - void setup() throws SQLException { - configWithoutDbName = getConfig(db); - final String dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); - - try (final DSLContext dslContext = getDslContext(configWithoutDbName)) { - final Database database = getDatabase(dslContext); - database.query(ctx -> { - ctx.fetch(String.format("CREATE DATABASE %s;", dbName)); - ctx.fetch(String.format("USE %s;", dbName)); - ctx.fetch("CREATE TABLE id_and_name(id INTEGER NOT NULL, name VARCHAR(200), born DATETIMEOFFSET(7));"); - ctx.fetch( - "INSERT INTO id_and_name (id, name, born) VALUES (1,'picard', '2124-03-04T01:01:01Z'), (2, 'crusher', '2124-03-04T01:01:01Z'), (3, 'vash', '2124-03-04T01:01:01Z');"); - return null; - }); - } - - config = Jsons.clone(configWithoutDbName); - ((ObjectNode) config).put(JdbcUtils.DATABASE_KEY, dbName); - ((ObjectNode) config).put("ssl_method", Jsons.jsonNode(Map.of("ssl_method", "unencrypted"))); + void setup() { + testdb = MsSQLTestDatabase.in(BaseImage.MSSQL_2022) + .with("CREATE TABLE id_and_name(id INTEGER NOT NULL, name VARCHAR(200), born DATETIMEOFFSET(7));") + .with("INSERT INTO id_and_name (id, name, born) VALUES (1,'picard', '2124-03-04T01:01:01Z'), (2, 'crusher', " + + "'2124-03-04T01:01:01Z'), (3, 'vash', '2124-03-04T01:01:01Z');"); + } + + @AfterEach + void cleanUp() { + testdb.close(); } - @AfterAll - static void cleanUp() { - db.stop(); - db.close(); + private JsonNode getConfig() { + return testdb.testConfigBuilder() + .withoutSsl() + .build(); } // if a column in mssql is used as a primary key and in a separate index the discover query returns @@ -97,82 +70,43 @@ static void cleanUp() { // this tests that this de-duplication is successful. @Test void testDiscoverWithPk() throws Exception { - try (final DSLContext dslContext = getDslContext(configWithoutDbName)) { - final Database database = getDatabase(dslContext); - database.query(ctx -> { - ctx.fetch(String.format("USE %s;", config.get(JdbcUtils.DATABASE_KEY))); - ctx.execute("ALTER TABLE id_and_name ADD CONSTRAINT i3pk PRIMARY KEY CLUSTERED (id);"); - ctx.execute("CREATE INDEX i1 ON id_and_name (id);"); - return null; - }); - } - - final AirbyteCatalog actual = new MssqlSource().discover(config); + testdb + .with("ALTER TABLE id_and_name ADD CONSTRAINT i3pk PRIMARY KEY CLUSTERED (id);") + .with("CREATE INDEX i1 ON id_and_name (id);"); + final AirbyteCatalog actual = source().discover(getConfig()); assertEquals(CATALOG, actual); } @Test @Disabled("See https://github.com/airbytehq/airbyte/pull/23908#issuecomment-1463753684, enable once communication is out") public void testTableWithNullCursorValueShouldThrowException() throws Exception { - try (final DSLContext dslContext = getDslContext(configWithoutDbName)) { - final Database database = getDatabase(dslContext); - database.query(ctx -> { - ctx.fetch(String.format("USE %s;", config.get(JdbcUtils.DATABASE_KEY))); - ctx.execute("ALTER TABLE id_and_name ALTER COLUMN id INTEGER NULL"); - ctx.execute("INSERT INTO id_and_name(id) VALUES (7), (8), (NULL)"); - return null; - }); - - ConfiguredAirbyteStream configuredAirbyteStream = new ConfiguredAirbyteStream().withSyncMode( - SyncMode.INCREMENTAL) - .withCursorField(Lists.newArrayList("id")) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withSyncMode(SyncMode.INCREMENTAL) - .withStream(CatalogHelpers.createAirbyteStream( - STREAM_NAME, - DB_NAME, - Field.of("id", JsonSchemaType.INTEGER), - Field.of("name", JsonSchemaType.STRING), - Field.of("born", JsonSchemaType.STRING)) - .withSupportedSyncModes( - Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) - .withSourceDefinedPrimaryKey(List.of(List.of("id")))); - - final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams( - Collections.singletonList(configuredAirbyteStream)); - - final Throwable throwable = catchThrowable(() -> MoreIterators.toSet( - new MssqlSource().read(config, catalog, null))); - assertThat(throwable).isInstanceOf(ConfigErrorException.class) - .hasMessageContaining( - "The following tables have invalid columns selected as cursor, please select a column with a well-defined ordering with no null values as a cursor. {tableName='dbo.id_and_name', cursorColumnName='id', cursorSqlType=INTEGER, cause=Cursor column contains NULL value}"); - } - } - - private JsonNode getConfig(final MSSQLServerContainer db) { - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, db.getHost()) - .put(JdbcUtils.PORT_KEY, db.getFirstMappedPort()) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) - .build()); - } - - private static DSLContext getDslContext(final JsonNode config) { - return DSLContextFactory.create(DataSourceFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.MSSQLSERVER.getDriverClassName(), - String.format("jdbc:sqlserver://%s:%d;", - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asInt()), - Map.of("encrypt", "false")), null); - } - - public static Database getDatabase(final DSLContext dslContext) { - // todo (cgardens) - rework this abstraction so that we do not have to pass a null into the - // constructor. at least explicitly handle it, even if the impl doesn't change. - return new Database(dslContext); + testdb + .with("ALTER TABLE id_and_name ALTER COLUMN id INTEGER NULL") + .with("INSERT INTO id_and_name(id) VALUES (7), (8), (NULL)"); + + ConfiguredAirbyteStream configuredAirbyteStream = new ConfiguredAirbyteStream().withSyncMode( + SyncMode.INCREMENTAL) + .withCursorField(Lists.newArrayList("id")) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withSyncMode(SyncMode.INCREMENTAL) + .withStream(CatalogHelpers.createAirbyteStream( + STREAM_NAME, + testdb.getDatabaseName(), + Field.of("id", JsonSchemaType.INTEGER), + Field.of("name", JsonSchemaType.STRING), + Field.of("born", JsonSchemaType.STRING)) + .withSupportedSyncModes( + Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of("id")))); + + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams( + Collections.singletonList(configuredAirbyteStream)); + + final Throwable throwable = catchThrowable(() -> MoreIterators.toSet( + source().read(getConfig(), catalog, null))); + assertThat(throwable).isInstanceOf(ConfigErrorException.class) + .hasMessageContaining( + "The following tables have invalid columns selected as cursor, please select a column with a well-defined ordering with no null values as a cursor. {tableName='dbo.id_and_name', cursorColumnName='id', cursorSqlType=INTEGER, cause=Cursor column contains NULL value}"); } } diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSslSourceTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSslSourceTest.java new file mode 100644 index 000000000000..3b45cb7e8210 --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlSslSourceTest.java @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql; + +import static org.junit.jupiter.api.Assertions.fail; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.commons.exceptions.ConnectionErrorException; +import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.BaseImage; +import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.CertificateKey; +import io.airbyte.integrations.source.mssql.MsSQLTestDatabase.ContainerModifier; +import io.airbyte.protocol.models.v0.AirbyteCatalog; +import java.net.InetAddress; +import java.util.Map; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MssqlSslSourceTest { + + private MsSQLTestDatabase testDb; + private static final Logger LOGGER = LoggerFactory.getLogger(MssqlSslSourceTest.class); + + @BeforeEach + void setup() { + testDb = MsSQLTestDatabase.in(BaseImage.MSSQL_2022, ContainerModifier.WITH_SSL_CERTIFICATES); + } + + @AfterEach + public void tearDown() { + testDb.close(); + } + + @ParameterizedTest + @EnumSource(CertificateKey.class) + public void testDiscoverWithCertificateTrustHostnameWithValidCertificates(CertificateKey certificateKey) throws Exception { + if (!certificateKey.isValid) { + return; + } + String certificate = testDb.getCertificate(certificateKey); + JsonNode config = testDb.testConfigBuilder() + .withSsl(Map.of("ssl_method", "encrypted_verify_certificate", + "certificate", certificate)) + .build(); + AirbyteCatalog catalog = new MssqlSource().discover(config); + } + + @ParameterizedTest + @EnumSource(CertificateKey.class) + public void testDiscoverWithCertificateTrustHostnameWithInvalidCertificates(CertificateKey certificateKey) throws Exception { + if (certificateKey.isValid) { + return; + } + String certificate = testDb.getCertificate(certificateKey); + JsonNode config = testDb.testConfigBuilder() + .withSsl(Map.of("ssl_method", "encrypted_verify_certificate", + "certificate", certificate)) + .build(); + try { + AirbyteCatalog catalog = new MssqlSource().discover(config); + } catch (ConnectionErrorException e) { + if (!e.getCause().getCause().getMessage().contains("PKIX path validation") && + !e.getCause().getCause().getMessage().contains("PKIX path building failed")) { + throw e; + } + } + } + + @ParameterizedTest + @EnumSource(CertificateKey.class) + public void testDiscoverWithCertificateNoTrustHostnameWrongHostname(CertificateKey certificateKey) throws Throwable { + if (!certificateKey.isValid) { + return; + } + String containerIp = InetAddress.getByName(testDb.getContainer().getHost()).getHostAddress(); + String certificate = testDb.getCertificate(certificateKey); + JsonNode config = testDb.configBuilder() + .withSsl(Map.of("ssl_method", "encrypted_verify_certificate", + "certificate", certificate)) + .with(JdbcUtils.HOST_KEY, containerIp) + .with(JdbcUtils.PORT_KEY, testDb.getContainer().getFirstMappedPort()) + .withCredentials() + .withDatabase() + .build(); + try { + AirbyteCatalog catalog = new MssqlSource().discover(config); + fail("discover should have failed!"); + } catch (ConnectionErrorException e) { + String expectedMessage = + "Failed to validate the server name \"" + containerIp + "\"in a certificate during Secure Sockets Layer (SSL) initialization."; + if (!e.getExceptionMessage().contains(expectedMessage)) { + fail("exception message was " + e.getExceptionMessage() + "\n expected: " + expectedMessage); + } + } + } + + @ParameterizedTest + @EnumSource(CertificateKey.class) + public void testDiscoverWithCertificateNoTrustHostnameAlternateHostname(CertificateKey certificateKey) throws Exception { + final String containerIp = InetAddress.getByName(testDb.getContainer().getHost()).getHostAddress(); + if (certificateKey.isValid) { + String certificate = testDb.getCertificate(certificateKey); + JsonNode config = testDb.configBuilder() + .withSsl(Map.of("ssl_method", "encrypted_verify_certificate", + "certificate", certificate, + "hostNameInCertificate", testDb.getContainer().getHost())) + .with(JdbcUtils.HOST_KEY, containerIp) + .with(JdbcUtils.PORT_KEY, testDb.getContainer().getFirstMappedPort()) + .withCredentials() + .withDatabase() + .build(); + AirbyteCatalog catalog = new MssqlSource().discover(config); + } + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlStressTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlStressTest.java index da497e182a9f..61dee32e78ba 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlStressTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlStressTest.java @@ -7,16 +7,17 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.cdk.integrations.source.jdbc.test.JdbcStressTest; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.test.JdbcStressTest; import java.sql.JDBCType; +import java.time.Duration; import java.util.Map; import java.util.Optional; import javax.sql.DataSource; @@ -29,6 +30,7 @@ @Disabled public class MssqlStressTest extends JdbcStressTest { + private static final Duration CONNECTION_TIMEOUT = Duration.ofSeconds(60); private static MSSQLServerContainer dbContainer; private JsonNode config; @@ -54,7 +56,8 @@ public void setup() throws Exception { String.format("jdbc:sqlserver://%s:%d;", configWithoutDbName.get(JdbcUtils.HOST_KEY).asText(), configWithoutDbName.get(JdbcUtils.PORT_KEY).asInt()), - Map.of("encrypt", "false")); + Map.of("encrypt", "false"), + CONNECTION_TIMEOUT); try { final JdbcDatabase database = new DefaultJdbcDatabase(dataSource); diff --git a/airbyte-integrations/connectors/source-mssql/src/testFixtures/java/io/airbyte/integrations/source/mssql/MsSQLContainerFactory.java b/airbyte-integrations/connectors/source-mssql/src/testFixtures/java/io/airbyte/integrations/source/mssql/MsSQLContainerFactory.java new file mode 100644 index 000000000000..74f6cce2c3f4 --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/testFixtures/java/io/airbyte/integrations/source/mssql/MsSQLContainerFactory.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql; + +import io.airbyte.cdk.testutils.ContainerFactory; +import org.apache.commons.lang3.StringUtils; +import org.testcontainers.containers.MSSQLServerContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.utility.DockerImageName; + +public class MsSQLContainerFactory implements ContainerFactory> { + + @Override + public MSSQLServerContainer createNewContainer(DockerImageName imageName) { + MSSQLServerContainer container = + new MSSQLServerContainer<>(imageName.asCompatibleSubstituteFor("mcr.microsoft.com/mssql/server")).acceptLicense(); + container.addEnv("MSSQL_MEMORY_LIMIT_MB", "384"); + return container; + } + + @Override + public Class getContainerClass() { + return MSSQLServerContainer.class; + } + + /** + * Create a new network and bind it to the container. + */ + public void withNetwork(MSSQLServerContainer container) { + container.withNetwork(Network.newNetwork()); + } + + public void withAgent(MSSQLServerContainer container) { + container.addEnv("MSSQL_AGENT_ENABLED", "True"); + } + + public void withSslCertificates(MSSQLServerContainer container) { + // yes, this is uglier than sin. The reason why I'm doing this is because there's no command to + // reload a SqlServer config. So I need to create all the necessary files before I start the + // SQL server. Hence this horror + String command = StringUtils.replace( + """ + mkdir /tmp/certs/ && + openssl req -nodes -new -x509 -sha256 -keyout /tmp/certs/ca.key -out /tmp/certs/ca.crt -subj "/CN=ca" && + openssl req -nodes -new -x509 -sha256 -keyout /tmp/certs/dummy_ca.key -out /tmp/certs/dummy_ca.crt -subj "/CN=ca" && + openssl req -nodes -new -sha256 -keyout /tmp/certs/server.key -out /tmp/certs/server.csr -subj "/CN={hostName}" && + openssl req -nodes -new -sha256 -keyout /tmp/certs/dummy_server.key -out /tmp/certs/dummy_server.csr -subj "/CN={hostName}" && + + openssl x509 -req -in /tmp/certs/server.csr -CA /tmp/certs/ca.crt -CAkey /tmp/certs/ca.key -out /tmp/certs/server.crt -days 365 -sha256 && + openssl x509 -req -in /tmp/certs/dummy_server.csr -CA /tmp/certs/ca.crt -CAkey /tmp/certs/ca.key -out /tmp/certs/dummy_server.crt -days 365 -sha256 && + openssl x509 -req -in /tmp/certs/server.csr -CA /tmp/certs/dummy_ca.crt -CAkey /tmp/certs/dummy_ca.key -out /tmp/certs/server_dummy_ca.crt -days 365 -sha256 && + chmod 440 /tmp/certs/* && + { + cat > /var/opt/mssql/mssql.conf <<- EOF + [network] + tlscert = /tmp/certs/server.crt + tlskey = /tmp/certs/server.key + tlsprotocols = 1.2 + forceencryption = 1 + EOF + } && /opt/mssql/bin/sqlservr + """, + "{hostName}", container.getHost()); + container.withCommand("bash", "-c", command) + .withUrlParam("trustServerCertificate", "true"); + } + +} diff --git a/airbyte-integrations/connectors/source-mssql/src/testFixtures/java/io/airbyte/integrations/source/mssql/MsSQLTestDatabase.java b/airbyte-integrations/connectors/source-mssql/src/testFixtures/java/io/airbyte/integrations/source/mssql/MsSQLTestDatabase.java new file mode 100644 index 000000000000..8ee98f176cd2 --- /dev/null +++ b/airbyte-integrations/connectors/source-mssql/src/testFixtures/java/io/airbyte/integrations/source/mssql/MsSQLTestDatabase.java @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mssql; + +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.TestDatabase; +import io.debezium.connector.sqlserver.Lsn; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.jooq.SQLDialect; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.MSSQLServerContainer; + +public class MsSQLTestDatabase extends TestDatabase, MsSQLTestDatabase, MsSQLTestDatabase.MsSQLConfigBuilder> { + + static private final Logger LOGGER = LoggerFactory.getLogger(MsSQLTestDatabase.class); + + static public final int MAX_RETRIES = 60; + + public static enum BaseImage { + + MSSQL_2022("mcr.microsoft.com/mssql/server:2022-latest"), + MSSQL_2017("mcr.microsoft.com/mssql/server:2017-latest"), + ; + + private final String reference; + + private BaseImage(final String reference) { + this.reference = reference; + } + + } + + public static enum ContainerModifier { + + NETWORK("withNetwork"), + AGENT("withAgent"), + WITH_SSL_CERTIFICATES("withSslCertificates"); + + private final String methodName; + + private ContainerModifier(final String methodName) { + this.methodName = methodName; + } + + } + + static public MsSQLTestDatabase in(final BaseImage imageName, final ContainerModifier... methods) { + final String[] methodNames = Stream.of(methods).map(im -> im.methodName).toList().toArray(new String[0]); + final var container = new MsSQLContainerFactory().shared(imageName.reference, methodNames); + final var testdb = new MsSQLTestDatabase(container); + return testdb + .withConnectionProperty("encrypt", "false") + .withConnectionProperty("databaseName", testdb.getDatabaseName()) + .initialized(); + } + + public MsSQLTestDatabase(final MSSQLServerContainer container) { + super(container); + } + + public MsSQLTestDatabase withCdc() { + return with("EXEC sys.sp_cdc_enable_db;"); + } + + public MsSQLTestDatabase withoutCdc() { + return with("EXEC sys.sp_cdc_disable_db;"); + } + + public MsSQLTestDatabase withAgentStarted() { + return with("EXEC master.dbo.xp_servicecontrol N'START', N'SQLServerAGENT';"); + } + + public MsSQLTestDatabase withAgentStopped() { + return with("EXEC master.dbo.xp_servicecontrol N'STOP', N'SQLServerAGENT';"); + } + + public MsSQLTestDatabase withWaitUntilAgentRunning() { + waitForAgentState(true); + return self(); + } + + public MsSQLTestDatabase withWaitUntilAgentStopped() { + waitForAgentState(false); + return self(); + } + + public MsSQLTestDatabase withShortenedCapturePollingInterval() { + return with("EXEC sys.sp_cdc_change_job @job_type = 'capture', @pollinginterval = %d;", + MssqlCdcTargetPosition.MAX_LSN_QUERY_DELAY_TEST.toSeconds()); + } + + private void waitForAgentState(final boolean running) { + final String expectedValue = running ? "Running." : "Stopped."; + LOGGER.debug("Waiting for SQLServerAgent state to change to '{}'.", expectedValue); + for (int i = 0; i < MAX_RETRIES; i++) { + try { + final var r = query(ctx -> ctx.fetch("EXEC master.dbo.xp_servicecontrol 'QueryState', N'SQLServerAGENT';").get(0)); + if (expectedValue.equalsIgnoreCase(r.getValue(0).toString())) { + LOGGER.debug("SQLServerAgent state is '{}', as expected.", expectedValue); + return; + } + LOGGER.debug("Retrying, SQLServerAgent state {} does not match expected '{}'.", r, expectedValue); + } catch (final SQLException e) { + LOGGER.debug("Retrying agent state query after catching exception {}.", e.getMessage()); + } + try { + Thread.sleep(1_000); // Wait one second between retries. + } catch (final InterruptedException e) { + throw new RuntimeException(e); + } + } + throw new RuntimeException("Exhausted retry attempts while polling for agent state"); + } + + public MsSQLTestDatabase withWaitUntilMaxLsnAvailable() { + LOGGER.debug("Waiting for max LSN to become available for database {}.", getDatabaseName()); + for (int i = 0; i < MAX_RETRIES; i++) { + try { + final var maxLSN = query(ctx -> ctx.fetch("SELECT sys.fn_cdc_get_max_lsn();").get(0).get(0, byte[].class)); + if (maxLSN != null) { + LOGGER.debug("Max LSN available for database {}: {}", getDatabaseName(), Lsn.valueOf(maxLSN)); + return self(); + } + LOGGER.debug("Retrying, max LSN still not available for database {}.", getDatabaseName()); + } catch (final SQLException e) { + LOGGER.warn("Retrying max LSN query after catching exception {}", e.getMessage()); + } + try { + Thread.sleep(1_000); // Wait one second between retries. + } catch (final InterruptedException e) { + throw new RuntimeException(e); + } + } + throw new RuntimeException("Exhausted retry attempts while polling for max LSN availability"); + } + + @Override + public String getPassword() { + return "S00p3rS33kr3tP4ssw0rd!"; + } + + @Override + public String getJdbcUrl() { + return String.format("jdbc:sqlserver://%s:%d", getContainer().getHost(), getContainer().getFirstMappedPort()); + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.of( + mssqlCmd(Stream.of(String.format("CREATE DATABASE %s", getDatabaseName()))), + mssqlCmd(Stream.of( + String.format("USE %s", getDatabaseName()), + String.format("CREATE LOGIN %s WITH PASSWORD = '%s', DEFAULT_DATABASE = %s", getUserName(), getPassword(), getDatabaseName()), + String.format("ALTER SERVER ROLE [sysadmin] ADD MEMBER %s", getUserName()), + String.format("CREATE USER %s FOR LOGIN %s WITH DEFAULT_SCHEMA = [dbo]", getUserName(), getUserName()), + String.format("ALTER ROLE [db_owner] ADD MEMBER %s", getUserName())))); + } + + /** + * Don't drop anything when closing the test database. Instead, if cleanup is required, call + * {@link #dropDatabaseAndUser()} explicitly. Implicit cleanups may result in deadlocks and so + * aren't really worth it. + */ + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + public void dropDatabaseAndUser() { + execInContainer(mssqlCmd(Stream.of( + String.format("USE master"), + String.format("ALTER DATABASE %s SET single_user WITH ROLLBACK IMMEDIATE", getDatabaseName()), + String.format("DROP DATABASE %s", getDatabaseName())))); + } + + public Stream mssqlCmd(final Stream sql) { + return Stream.of("/opt/mssql-tools/bin/sqlcmd", + "-U", getContainer().getUsername(), + "-P", getContainer().getPassword(), + "-Q", sql.collect(Collectors.joining("; ")), + "-b", "-e"); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return DatabaseDriver.MSSQLSERVER; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.DEFAULT; + } + + public static enum CertificateKey { + + CA(true), + DUMMY_CA(false), + SERVER(true), + DUMMY_SERVER(false), + SERVER_DUMMY_CA(false), + ; + + public final boolean isValid; + + CertificateKey(final boolean isValid) { + this.isValid = isValid; + } + + } + + private Map cachedCerts; + + public synchronized String getCertificate(final CertificateKey certificateKey) { + if (cachedCerts == null) { + final Map cachedCerts = new HashMap<>(); + try { + for (final CertificateKey key : CertificateKey.values()) { + final String command = "cat /tmp/certs/" + key.name().toLowerCase() + ".crt"; + final String certificate = getContainer().execInContainer("bash", "-c", command).getStdout().trim(); + cachedCerts.put(key, certificate); + } + } catch (final IOException e) { + throw new UncheckedIOException(e); + } catch (final InterruptedException e) { + throw new RuntimeException(e); + } + this.cachedCerts = cachedCerts; + } + return cachedCerts.get(certificateKey); + } + + @Override + public MsSQLConfigBuilder configBuilder() { + return new MsSQLConfigBuilder(this); + } + + static public class MsSQLConfigBuilder extends ConfigBuilder { + + protected MsSQLConfigBuilder(final MsSQLTestDatabase testDatabase) { + + super(testDatabase); + with(JdbcUtils.JDBC_URL_PARAMS_KEY, "loginTimeout=2"); + + } + + public MsSQLConfigBuilder withCdcReplication() { + return with("is_test", true) + .with("replication_method", Map.of( + "method", "CDC", + "initial_waiting_seconds", DEFAULT_CDC_REPLICATION_INITIAL_WAIT.getSeconds())); + } + + public MsSQLConfigBuilder withSchemas(final String... schemas) { + return with(JdbcUtils.SCHEMAS_KEY, List.of(schemas)); + } + + @Override + public MsSQLConfigBuilder withoutSsl() { + return withSsl(Map.of("ssl_method", "unencrypted")); + } + + @Deprecated + public MsSQLConfigBuilder withSsl(final Map sslMode) { + return with("ssl_method", sslMode); + } + + public MsSQLConfigBuilder withEncrytedTrustServerCertificate() { + return withSsl(Map.of("ssl_method", "encrypted_trust_server_certificate")); + } + + public MsSQLConfigBuilder withEncrytedVerifyServerCertificate(final String certificate, final String hostnameInCertificate) { + if (hostnameInCertificate != null) { + return withSsl(Map.of("ssl_method", "encrypted_verify_certificate", + "certificate", certificate, + "hostNameInCertificate", hostnameInCertificate)); + } else { + return withSsl(Map.of("ssl_method", "encrypted_verify_certificate", + "certificate", certificate)); + } + } + + } + +} diff --git a/airbyte-integrations/connectors/source-my-hours/Dockerfile b/airbyte-integrations/connectors/source-my-hours/Dockerfile index bb2e89bbd239..8e2bd95d7e2d 100644 --- a/airbyte-integrations/connectors/source-my-hours/Dockerfile +++ b/airbyte-integrations/connectors/source-my-hours/Dockerfile @@ -34,5 +34,5 @@ COPY source_my_hours ./source_my_hours ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/source-my-hours diff --git a/airbyte-integrations/connectors/source-my-hours/README.md b/airbyte-integrations/connectors/source-my-hours/README.md index 0c36c4916434..a63c9d3147c3 100644 --- a/airbyte-integrations/connectors/source-my-hours/README.md +++ b/airbyte-integrations/connectors/source-my-hours/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-my-hours:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/my-hours) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_my_hours/spec.json` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-my-hours:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-my-hours build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-my-hours:airbyteDocker +An image will be built with the tag `airbyte/source-my-hours:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-my-hours:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-my-hours:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-my-hours:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-my-hours:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-my-hours test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-my-hours:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-my-hours:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-my-hours test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/my-hours.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-my-hours/acceptance-test-config.yml b/airbyte-integrations/connectors/source-my-hours/acceptance-test-config.yml index 63087cafebc3..61597984a918 100644 --- a/airbyte-integrations/connectors/source-my-hours/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-my-hours/acceptance-test-config.yml @@ -1,20 +1,28 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-my-hours:dev -tests: +acceptance_tests: spec: - - spec_path: "source_my_hours/spec.json" + tests: + - spec_path: "source_my_hours/spec.json" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + # time_logs stream contains a number of empty fields that are not + # documented in the API. Until we can verify the types on these fields, + # we need to disable this check. + fail_on_extra_columns: false full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-my-hours/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-my-hours/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-my-hours/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-my-hours/build.gradle b/airbyte-integrations/connectors/source-my-hours/build.gradle deleted file mode 100644 index 40b6e37e709f..000000000000 --- a/airbyte-integrations/connectors/source-my-hours/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_my_hours' -} diff --git a/airbyte-integrations/connectors/source-my-hours/metadata.yaml b/airbyte-integrations/connectors/source-my-hours/metadata.yaml index 68328da187e8..ab7bf1e3e5be 100644 --- a/airbyte-integrations/connectors/source-my-hours/metadata.yaml +++ b/airbyte-integrations/connectors/source-my-hours/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: 722ba4bf-06ec-45a4-8dd5-72e4a5cf3903 - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 dockerRepository: airbyte/source-my-hours githubIssueLabel: source-my-hours icon: my-hours.svg diff --git a/airbyte-integrations/connectors/source-my-hours/setup.py b/airbyte-integrations/connectors/source-my-hours/setup.py index cd096054986c..eb4d9a7c9734 100644 --- a/airbyte-integrations/connectors/source-my-hours/setup.py +++ b/airbyte-integrations/connectors/source-my-hours/setup.py @@ -6,14 +6,13 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "requests_mock==1.8.0", "responses~=0.16.0", ] diff --git a/airbyte-integrations/connectors/source-my-hours/source_my_hours/auth.py b/airbyte-integrations/connectors/source-my-hours/source_my_hours/auth.py index 108ca6d0fe91..f16e7222c59e 100644 --- a/airbyte-integrations/connectors/source-my-hours/source_my_hours/auth.py +++ b/airbyte-integrations/connectors/source-my-hours/source_my_hours/auth.py @@ -33,8 +33,8 @@ def retrieve_refresh_token(self, email: str, password: str): json_response = response.json() self.refresh_token = json_response["refreshToken"] - self._access_token = json_response[self.access_token_name] - self._token_expiry_date = t0.add(seconds=json_response[self.expires_in_name]) + self._access_token = json_response[self._access_token_name] + self._token_expiry_date = t0.add(seconds=json_response[self._expires_in_name]) def get_refresh_request_body(self) -> Mapping[str, Any]: payload: MutableMapping[str, Any] = { @@ -46,10 +46,10 @@ def get_refresh_request_body(self) -> Mapping[str, Any]: def refresh_access_token(self) -> Tuple[str, int]: try: - response = requests.request(method="POST", url=self.token_refresh_endpoint, data=self.get_refresh_request_body()) + response = requests.request(method="POST", url=self._token_refresh_endpoint, data=self.get_refresh_request_body()) response.raise_for_status() response_json = response.json() self.refresh_token = response_json["refreshToken"] - return response_json[self.access_token_name], response_json[self.expires_in_name] + return response_json[self._access_token_name], response_json[self._expires_in_name] except Exception as e: raise Exception(f"Error while refreshing access token: {e}") from e diff --git a/airbyte-integrations/connectors/source-my-hours/source_my_hours/schemas/clients.json b/airbyte-integrations/connectors/source-my-hours/source_my_hours/schemas/clients.json index a2beb521d1f4..20153f54d6ec 100644 --- a/airbyte-integrations/connectors/source-my-hours/source_my_hours/schemas/clients.json +++ b/airbyte-integrations/connectors/source-my-hours/source_my_hours/schemas/clients.json @@ -11,6 +11,9 @@ "contactEmail": { "type": ["null", "string"] }, + "customId": { + "type": ["null", "string"] + }, "name": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-my-hours/source_my_hours/schemas/projects.json b/airbyte-integrations/connectors/source-my-hours/source_my_hours/schemas/projects.json index 0604a3476ed0..692e72c151e2 100644 --- a/airbyte-integrations/connectors/source-my-hours/source_my_hours/schemas/projects.json +++ b/airbyte-integrations/connectors/source-my-hours/source_my_hours/schemas/projects.json @@ -20,12 +20,21 @@ "clientName": { "type": ["null", "string"] }, + "clientId": { + "type": ["null", "integer"] + }, + "customId": { + "type": ["null", "string"] + }, "budgetAlertPercent": { "type": ["null", "number"] }, "budgetType": { "type": ["null", "number"] }, + "laborCost": { + "type": ["null", "number"] + }, "totalTimeLogged": { "type": ["null", "number"] }, diff --git a/airbyte-integrations/connectors/source-my-hours/source_my_hours/schemas/time_logs.json b/airbyte-integrations/connectors/source-my-hours/source_my_hours/schemas/time_logs.json index 120a7e88bc94..ffb58a1712df 100644 --- a/airbyte-integrations/connectors/source-my-hours/source_my_hours/schemas/time_logs.json +++ b/airbyte-integrations/connectors/source-my-hours/source_my_hours/schemas/time_logs.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "logId": { "type": ["number"] @@ -135,6 +136,61 @@ }, "roundtype": { "type": ["null", "number"] + }, + "attachments": { + "type": ["null", "array"] + }, + "balance": { + "type": ["null", "number"] + }, + "billableExpense": { + "type": ["null", "number"] + }, + "billableHoursLogBillable": { + "type": ["null", "number"] + }, + "clientCustomId": { + "type": ["null", "string"] + }, + "endTime": { + "type": ["null", "string"] + }, + "invoicedAmount": { + "type": ["null", "number"] + }, + "logDurationBillable": { + "type": ["null", "number"] + }, + "startTime": { + "type": ["null", "string"], + "format": "date-time" + }, + "startEndTime": { + "type": ["null", "string"] + }, + "teams": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "teamsNames": { + "type": ["null", "string"] + }, + "taskListName": { + "type": ["null", "string"] + }, + "taskDueDate": { + "type": ["null", "string"] + }, + "taskStartDate": { + "type": ["null", "string"] + }, + "tagsData": { + "type": ["null", "array"] + }, + "uninvoicedAmount": { + "type": ["null", "number"] } } } diff --git a/airbyte-integrations/connectors/source-my-hours/source_my_hours/schemas/users.json b/airbyte-integrations/connectors/source-my-hours/source_my_hours/schemas/users.json index 6d952c36b151..6c5d22739d6e 100644 --- a/airbyte-integrations/connectors/source-my-hours/source_my_hours/schemas/users.json +++ b/airbyte-integrations/connectors/source-my-hours/source_my_hours/schemas/users.json @@ -34,6 +34,12 @@ }, "isProjectManager": { "type": ["null", "boolean"] + }, + "roleType": { + "type": ["null", "integer"] + }, + "customId": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-my-hours/source_my_hours/spec.json b/airbyte-integrations/connectors/source-my-hours/source_my_hours/spec.json index 6ca8a83e3fea..eef075c9ad3a 100644 --- a/airbyte-integrations/connectors/source-my-hours/source_my_hours/spec.json +++ b/airbyte-integrations/connectors/source-my-hours/source_my_hours/spec.json @@ -5,7 +5,7 @@ "title": "My Hours Spec", "type": "object", "required": ["email", "password", "start_date"], - "additionalProperties": false, + "additionalProperties": true, "properties": { "email": { "title": "Email", diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/.dockerignore b/airbyte-integrations/connectors/source-mysql-strict-encrypt/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile deleted file mode 100644 index d7b6d236fda8..000000000000 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-mysql-strict-encrypt - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-mysql-strict-encrypt - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=3.0.1 - -LABEL io.airbyte.name=airbyte/source-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mysql-strict-encrypt/acceptance-test-config.yml deleted file mode 100644 index c2ef1564874e..000000000000 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/acceptance-test-config.yml +++ /dev/null @@ -1,6 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-mysql-strict-encrypt:dev -tests: - spec: - - spec_path: "src/test/resources/expected_spec.json" diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-mysql-strict-encrypt/build.gradle deleted file mode 100644 index c305939e20bf..000000000000 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/build.gradle +++ /dev/null @@ -1,32 +0,0 @@ -plugins { - id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' -} - -application { - mainClass = 'io.airbyte.integrations.source.mysql_strict_encrypt.MySqlStrictEncryptSource' - applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] -} - -dependencies { - implementation project(':airbyte-db:db-lib') - - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:connectors:source-mysql') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation project(':airbyte-integrations:connectors:source-relational-db') - - testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) - testImplementation project(':airbyte-test-utils') - testImplementation libs.junit.jupiter.system.stubs - - testImplementation libs.connectors.testcontainers.mysql - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) -} - diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/icon.svg b/airbyte-integrations/connectors/source-mysql-strict-encrypt/icon.svg deleted file mode 100644 index 607d361ed765..000000000000 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/icon.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/source-mysql-strict-encrypt/metadata.yaml deleted file mode 100644 index ea210bc1997d..000000000000 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/metadata.yaml +++ /dev/null @@ -1,29 +0,0 @@ -data: - allowedHosts: - hosts: - - ${host} - - ${tunnel_method.tunnel_host} - registries: - cloud: - enabled: false # strict encrypt connectors are deployed to Cloud by their non strict encrypt sibling. - oss: - enabled: false # strict encrypt connectors are not used on OSS. - connectorSubtype: database - connectorType: source - definitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad - dockerImageTag: 3.0.1 - dockerRepository: airbyte/source-mysql-strict-encrypt - githubIssueLabel: source-mysql - icon: mysql.svg - license: ELv2 - name: MySQL - releaseStage: beta - documentationUrl: https://docs.airbyte.com/integrations/sources/mysql - tags: - - language:java - releases: - breakingChanges: - 3.0.0: - message: "Add default cursor for cdc" - upgradeDeadline: "2023-08-17" -metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/main/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSource.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/main/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSource.java deleted file mode 100644 index 704a8806884e..000000000000 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/main/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSource.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mysql_strict_encrypt; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.base.spec_modification.SpecModifyingSource; -import io.airbyte.integrations.source.mysql.MySqlSource; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import java.util.Set; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Secure-only version of MySQL source that can be used in the Airbyte cloud. This connector - * inherently prevent certain insecure connections such as connecting to a database over the public - * internet without encryption. - */ -public class MySqlStrictEncryptSource extends SpecModifyingSource implements Source { - - public static final String TUNNEL_METHOD = "tunnel_method"; - public static final String NO_TUNNEL = "NO_TUNNEL"; - public static final String SSL_MODE = "ssl_mode"; - public static final String MODE = "mode"; - public static final String SSL_MODE_PREFERRED = "preferred"; - public static final String SSL_MODE_REQUIRED = "required"; - - private static final Logger LOGGER = LoggerFactory.getLogger(MySqlStrictEncryptSource.class); - private static final String SSL_MODE_DESCRIPTION = "SSL connection modes. " + - "

    • required - Always connect with SSL. If the MySQL server doesn’t support SSL, the connection will not be established. Certificate Authority (CA) and Hostname are not verified.
    • " - + - "
    • verify-ca - Always connect with SSL. Verifies CA, but allows connection even if Hostname does not match.
    • " + - "
    • Verify Identity - Always connect with SSL. Verify both CA and Hostname.
    • Read more in the docs."; - - MySqlStrictEncryptSource() { - super(MySqlSource.sshWrappedSource()); - } - - @Override - public ConnectorSpecification modifySpec(final ConnectorSpecification originalSpec) { - final ConnectorSpecification spec = Jsons.clone(originalSpec); - ((ObjectNode) spec.getConnectionSpecification().get("properties")).remove(JdbcUtils.SSL_KEY); - ((ObjectNode) spec.getConnectionSpecification().get("properties").get(SSL_MODE)).put("default", SSL_MODE_REQUIRED); - return spec; - } - - @Override - public AirbyteConnectionStatus check(final JsonNode config) throws Exception { - // #15808 Disallow connecting to db with disable, prefer or allow SSL mode when connecting directly - // and not over SSH tunnel - if (config.has(TUNNEL_METHOD) - && config.get(TUNNEL_METHOD).has(TUNNEL_METHOD) - && config.get(TUNNEL_METHOD).get(TUNNEL_METHOD).asText().equals(NO_TUNNEL)) { - // If no SSH tunnel - if (config.has(SSL_MODE) && config.get(SSL_MODE).has(MODE)) { - if (Set.of(SSL_MODE_PREFERRED).contains(config.get(SSL_MODE).get(MODE).asText())) { - // Fail in case SSL mode is preferred - return new AirbyteConnectionStatus() - .withStatus(Status.FAILED) - .withMessage( - "Unsecured connection not allowed. If no SSH Tunnel set up, please use one of the following SSL modes: required, verify-ca, verify-identity"); - } - } - } - return super.check(config); - } - - public static void main(final String[] args) throws Exception { - final Source source = new MySqlStrictEncryptSource(); - LOGGER.info("starting source: {}", MySqlStrictEncryptSource.class); - new IntegrationRunner(source).run(args); - LOGGER.info("completed source: {}", MySqlStrictEncryptSource.class); - } - -} diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/AbstractMySqlSslCertificateStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/AbstractMySqlSslCertificateStrictEncryptSourceAcceptanceTest.java deleted file mode 100644 index 58c71c92b01b..000000000000 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/AbstractMySqlSslCertificateStrictEncryptSourceAcceptanceTest.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mysql_strict_encrypt; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.MySqlUtils; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.testcontainers.containers.MySQLContainer; - -public abstract class AbstractMySqlSslCertificateStrictEncryptSourceAcceptanceTest extends MySqlStrictEncryptSourceAcceptanceTest { - - protected static MySqlUtils.Certificate certs; - protected static final String PASSWORD = "Passw0rd"; - - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - - container = new MySQLContainer<>("mysql:8.0"); - container.start(); - addTestData(container); - certs = MySqlUtils.getCertificate(container, true); - - final var sslMode = getSslConfig(); - final var innerContainerAddress = SshHelpers.getInnerContainerAddress(container); - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "STANDARD") - .build()); - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, innerContainerAddress.left) - .put(JdbcUtils.PORT_KEY, innerContainerAddress.right) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put(JdbcUtils.SSL_KEY, true) - .put(JdbcUtils.SSL_MODE_KEY, sslMode) - .put("replication_method", replicationMethod) - .build()); - } - - public abstract ImmutableMap getSslConfig(); - - private void addTestData(final MySQLContainer container) throws Exception { - final var outerContainerAddress = SshHelpers.getOuterContainerAddress(container); - try (final DSLContext dslContext = DSLContextFactory.create( - container.getUsername(), - container.getPassword(), - DatabaseDriver.MYSQL.getDriverClassName(), - String.format("jdbc:mysql://%s:%s/%s", - outerContainerAddress.left, - outerContainerAddress.right, - container.getDatabaseName()), - SQLDialect.MYSQL)) { - final Database database = new Database(dslContext); - - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch( - "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch( - "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - return null; - }); - } - } - -} diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlSslCaCertificateStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlSslCaCertificateStrictEncryptSourceAcceptanceTest.java deleted file mode 100644 index 0d6440133fee..000000000000 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlSslCaCertificateStrictEncryptSourceAcceptanceTest.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mysql_strict_encrypt; - -import com.google.common.collect.ImmutableMap; -import io.airbyte.db.jdbc.JdbcUtils; - -public class MySqlSslCaCertificateStrictEncryptSourceAcceptanceTest extends AbstractMySqlSslCertificateStrictEncryptSourceAcceptanceTest { - - @Override - public ImmutableMap getSslConfig() { - return ImmutableMap.builder() - .put(JdbcUtils.MODE_KEY, "verify_ca") - .put("ca_certificate", certs.getCaCertificate()) - .put("client_key_password", PASSWORD) - .build(); - } - -} diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlSslFullCertificateStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlSslFullCertificateStrictEncryptSourceAcceptanceTest.java deleted file mode 100644 index 46f43956c34f..000000000000 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlSslFullCertificateStrictEncryptSourceAcceptanceTest.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mysql_strict_encrypt; - -import com.google.common.collect.ImmutableMap; -import io.airbyte.db.jdbc.JdbcUtils; - -public class MySqlSslFullCertificateStrictEncryptSourceAcceptanceTest extends AbstractMySqlSslCertificateStrictEncryptSourceAcceptanceTest { - - @Override - public ImmutableMap getSslConfig() { - return ImmutableMap.builder() - .put(JdbcUtils.MODE_KEY, "verify_ca") - .put("ca_certificate", certs.getCaCertificate()) - .put("client_certificate", certs.getClientCertificate()) - .put("client_key", certs.getClientKey()) - .put("client_key_password", PASSWORD) - .build(); - } - -} diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSourceAcceptanceTest.java deleted file mode 100644 index da195558d32a..000000000000 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSourceAcceptanceTest.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mysql_strict_encrypt; - -import static io.airbyte.integrations.source.mysql.MySqlSource.SSL_PARAMETERS; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.SyncMode; -import java.util.HashMap; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.testcontainers.containers.MySQLContainer; - -public class MySqlStrictEncryptSourceAcceptanceTest extends SourceAcceptanceTest { - - private static final String STREAM_NAME = "id_and_name"; - private static final String STREAM_NAME2 = "public.starships"; - - protected MySQLContainer container; - protected JsonNode config; - - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - container = new MySQLContainer<>("mysql:8.0"); - container.start(); - - var sslMode = ImmutableMap.builder() - .put(JdbcUtils.MODE_KEY, "required") - .build(); - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "STANDARD") - .build()); - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put(JdbcUtils.SSL_MODE_KEY, sslMode) - .put("replication_method", replicationMethod) - .build()); - - try (final DSLContext dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.MYSQL.getDriverClassName(), - String.format("jdbc:mysql://%s:%s/%s?%s", - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asText(), - config.get(JdbcUtils.DATABASE_KEY).asText(), - String.join("&", SSL_PARAMETERS)), - SQLDialect.MYSQL)) { - final Database database = new Database(dslContext); - - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - return null; - }); - } - } - - @Override - protected void tearDown(final TestDestinationEnv testEnv) { - container.close(); - } - - @Override - protected String getImageName() { - return "airbyte/source-mysql-strict-encrypt:dev"; - } - - @Override - protected ConnectorSpecification getSpec() throws Exception { - return SshHelpers.injectSshIntoSpec(Jsons.deserialize(MoreResources.readResource("expected_spec.json"), ConnectorSpecification.class)); - } - - @Override - protected JsonNode getConfig() { - return config; - } - - @Override - protected ConfiguredAirbyteCatalog getConfiguredCatalog() { - return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(Lists.newArrayList("id")) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withStream(CatalogHelpers.createAirbyteStream( - String.format("%s.%s", config.get(JdbcUtils.DATABASE_KEY).asText(), STREAM_NAME), - Field.of("id", JsonSchemaType.NUMBER), - Field.of("name", JsonSchemaType.STRING)) - .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))), - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(Lists.newArrayList("id")) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withStream(CatalogHelpers.createAirbyteStream( - String.format("%s.%s", config.get(JdbcUtils.DATABASE_KEY).asText(), STREAM_NAME2), - Field.of("id", JsonSchemaType.NUMBER), - Field.of("name", JsonSchemaType.STRING)) - .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); - } - - @Override - protected JsonNode getState() { - return Jsons.jsonNode(new HashMap<>()); - } - - @Override - protected boolean supportsPerStream() { - return true; - } - -} diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java deleted file mode 100644 index 82ee0c027787..000000000000 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptJdbcSourceAcceptanceTest.java +++ /dev/null @@ -1,374 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mysql_strict_encrypt; - -import static io.airbyte.integrations.source.mysql.MySqlSource.SSL_PARAMETERS; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.commons.string.Strings; -import io.airbyte.db.Database; -import io.airbyte.db.MySqlUtils; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.base.ssh.SshBastionContainer; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.base.ssh.SshTunnel; -import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; -import io.airbyte.integrations.source.mysql.MySqlSource; -import io.airbyte.integrations.source.relationaldb.models.DbStreamState; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.AirbyteCatalog; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import io.airbyte.protocol.models.v0.SyncMode; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.*; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testcontainers.containers.MySQLContainer; -import org.testcontainers.containers.Network; -import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; -import uk.org.webcompere.systemstubs.jupiter.SystemStub; -import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; - -@ExtendWith(SystemStubsExtension.class) -class MySqlStrictEncryptJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { - - @SystemStub - private EnvironmentVariables environmentVariables; - - protected static final String TEST_USER = "test"; - protected static final String TEST_PASSWORD = "test"; - protected static MySQLContainer container; - private static final SshBastionContainer bastion = new SshBastionContainer(); - private static final Network network = Network.newNetwork(); - - protected Database database; - protected DSLContext dslContext; - - @BeforeAll - static void init() throws SQLException { - container = new MySQLContainer<>("mysql:8.0") - .withUsername(TEST_USER) - .withPassword(TEST_PASSWORD) - .withEnv("MYSQL_ROOT_HOST", "%") - .withEnv("MYSQL_ROOT_PASSWORD", TEST_PASSWORD); - container.start(); - final Connection connection = DriverManager.getConnection(container.getJdbcUrl(), "root", container.getPassword()); - connection.createStatement().execute("GRANT ALL PRIVILEGES ON *.* TO '" + TEST_USER + "'@'%';\n"); - } - - @BeforeEach - public void setup() throws Exception { - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - final var innerContainerAddress = SshHelpers.getInnerContainerAddress(container); - final var outerContainerAddress = SshHelpers.getOuterContainerAddress(container); - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, innerContainerAddress.left) - .put(JdbcUtils.PORT_KEY, innerContainerAddress.right) - .put(JdbcUtils.DATABASE_KEY, Strings.addRandomSuffix("db", "_", 10)) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .build()); - - dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.MYSQL.getDriverClassName(), - String.format("jdbc:mysql://%s:%s?%s", - outerContainerAddress.left, - outerContainerAddress.right, - String.join("&", SSL_PARAMETERS)), - SQLDialect.MYSQL); - database = new Database(dslContext); - - database.query(ctx -> { - ctx.fetch("CREATE DATABASE " + config.get(JdbcUtils.DATABASE_KEY).asText()); - return null; - }); - - super.setup(); - } - - @AfterEach - void tearDownMySql() throws Exception { - dslContext.close(); - super.tearDown(); - } - - @AfterAll - static void cleanUp() { - container.close(); - } - - // MySql does not support schemas in the way most dbs do. Instead we namespace by db name. - @Override - public boolean supportsSchemas() { - return false; - } - - @Override - public MySqlSource getJdbcSource() { - return new MySqlSource(); - } - - @Override - public Source getSource() { - return new MySqlStrictEncryptSource(); - } - - @Override - public String getDriverClass() { - return MySqlSource.DRIVER_CLASS; - } - - @Override - public JsonNode getConfig() { - return Jsons.clone(config); - } - - @Test - void testSpec() throws Exception { - final ConnectorSpecification actual = source.spec(); - final ConnectorSpecification expected = - SshHelpers.injectSshIntoSpec(Jsons.deserialize(MoreResources.readResource("expected_spec.json"), ConnectorSpecification.class)); - assertEquals(expected, actual); - } - - @Override - protected AirbyteCatalog getCatalog(final String defaultNamespace) { - return new AirbyteCatalog().withStreams(List.of( - CatalogHelpers.createAirbyteStream( - TABLE_NAME, - defaultNamespace, - Field.of(COL_ID, JsonSchemaType.INTEGER), - Field.of(COL_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) - .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) - .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID))), - CatalogHelpers.createAirbyteStream( - TABLE_NAME_WITHOUT_PK, - defaultNamespace, - Field.of(COL_ID, JsonSchemaType.INTEGER), - Field.of(COL_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) - .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) - .withSourceDefinedPrimaryKey(Collections.emptyList()), - CatalogHelpers.createAirbyteStream( - TABLE_NAME_COMPOSITE_PK, - defaultNamespace, - Field.of(COL_FIRST_NAME, JsonSchemaType.STRING), - Field.of(COL_LAST_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) - .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) - .withSourceDefinedPrimaryKey( - List.of(List.of(COL_FIRST_NAME), List.of(COL_LAST_NAME))))); - } - - @Override - protected void incrementalDateCheck() throws Exception { - incrementalCursorCheck( - COL_UPDATED_AT, - "2005-10-18", - "2006-10-19", - List.of(getTestMessages().get(1), getTestMessages().get(2))); - } - - @Override - protected List getTestMessages() { - return List.of( - new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_1, - COL_NAME, "picard", - COL_UPDATED_AT, "2004-10-19")))), - new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_2, - COL_NAME, "crusher", - COL_UPDATED_AT, - "2005-10-19")))), - new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_3, - COL_NAME, "vash", - COL_UPDATED_AT, "2006-10-19"))))); - } - - @Override - protected List getExpectedAirbyteMessagesSecondSync(final String namespace) { - final List expectedMessages = new ArrayList<>(); - expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_4, - COL_NAME, "riker", - COL_UPDATED_AT, "2006-10-19"))))); - expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_5, - COL_NAME, "data", - COL_UPDATED_AT, "2006-10-19"))))); - final DbStreamState state = new DbStreamState() - .withStreamName(streamName) - .withStreamNamespace(namespace) - .withCursorField(List.of(COL_ID)) - .withCursor("5") - .withCursorRecordCount(1L); - expectedMessages.addAll(createExpectedTestMessages(List.of(state))); - return expectedMessages; - } - - @Test - void testStrictSSLUnsecuredNoTunnel() throws Exception { - final String PASSWORD = "Passw0rd"; - final var certs = MySqlUtils.getCertificate(container, true); - final var sslMode = ImmutableMap.builder() - .put(JdbcUtils.MODE_KEY, "preferred") - .build(); - - final var tunnelMode = ImmutableMap.builder() - .put("tunnel_method", "NO_TUNNEL") - .build(); - ((ObjectNode) config).put(JdbcUtils.PASSWORD_KEY, "fake") - .put(JdbcUtils.SSL_KEY, true) - .putIfAbsent(JdbcUtils.SSL_MODE_KEY, Jsons.jsonNode(sslMode)); - ((ObjectNode) config).putIfAbsent("tunnel_method", Jsons.jsonNode(tunnelMode)); - - final AirbyteConnectionStatus actual = source.check(config); - assertEquals(Status.FAILED, actual.getStatus()); - assertTrue(actual.getMessage().contains("Unsecured connection not allowed")); - } - - @Test - void testStrictSSLSecuredNoTunnel() throws Exception { - final String PASSWORD = "Passw0rd"; - final var certs = MySqlUtils.getCertificate(container, true); - final var sslMode = ImmutableMap.builder() - .put(JdbcUtils.MODE_KEY, "verify_ca") - .put("ca_certificate", certs.getCaCertificate()) - .put("client_certificate", certs.getClientCertificate()) - .put("client_key", certs.getClientKey()) - .put("client_key_password", PASSWORD) - .build(); - - final var tunnelMode = ImmutableMap.builder() - .put("tunnel_method", "NO_TUNNEL") - .build(); - ((ObjectNode) config).put(JdbcUtils.PASSWORD_KEY, "fake") - .put(JdbcUtils.SSL_KEY, true) - .putIfAbsent(JdbcUtils.SSL_MODE_KEY, Jsons.jsonNode(sslMode)); - ((ObjectNode) config).putIfAbsent("tunnel_method", Jsons.jsonNode(tunnelMode)); - - final AirbyteConnectionStatus actual = source.check(config); - assertEquals(Status.FAILED, actual.getStatus()); - assertFalse(actual.getMessage().contains("Unsecured connection not allowed")); - } - - @Test - void testStrictSSLSecuredWithTunnel() throws Exception { - final String PASSWORD = "Passw0rd"; - final var certs = MySqlUtils.getCertificate(container, true); - final var sslMode = ImmutableMap.builder() - .put(JdbcUtils.MODE_KEY, "verify_ca") - .put("ca_certificate", certs.getCaCertificate()) - .put("client_certificate", certs.getClientCertificate()) - .put("client_key", certs.getClientKey()) - .put("client_key_password", PASSWORD) - .build(); - - final var tunnelMode = ImmutableMap.builder() - .put("tunnel_method", "SSH_KEY_AUTH") - .build(); - ((ObjectNode) config).put(JdbcUtils.PASSWORD_KEY, "fake") - .put(JdbcUtils.SSL_KEY, true) - .putIfAbsent(JdbcUtils.SSL_MODE_KEY, Jsons.jsonNode(sslMode)); - ((ObjectNode) config).putIfAbsent("tunnel_method", Jsons.jsonNode(tunnelMode)); - - final AirbyteConnectionStatus actual = source.check(config); - assertEquals(Status.FAILED, actual.getStatus()); - assertTrue(actual.getMessage().contains("Could not connect with provided SSH configuration.")); - } - - @Test - void testStrictSSLUnsecuredWithTunnel() throws Exception { - final String PASSWORD = "Passw0rd"; - final var certs = MySqlUtils.getCertificate(container, true); - final var sslMode = ImmutableMap.builder() - .put(JdbcUtils.MODE_KEY, "preferred") - .build(); - - final var tunnelMode = ImmutableMap.builder() - .put("tunnel_method", "SSH_KEY_AUTH") - .build(); - ((ObjectNode) config).put(JdbcUtils.PASSWORD_KEY, "fake") - .put(JdbcUtils.SSL_KEY, true) - .putIfAbsent(JdbcUtils.SSL_MODE_KEY, Jsons.jsonNode(sslMode)); - ((ObjectNode) config).putIfAbsent("tunnel_method", Jsons.jsonNode(tunnelMode)); - - final AirbyteConnectionStatus actual = source.check(config); - assertEquals(Status.FAILED, actual.getStatus()); - assertTrue(actual.getMessage().contains("Could not connect with provided SSH configuration.")); - } - - @Test - void testCheckWithSSlModeDisabled() throws Exception { - try (final MySQLContainer db = new MySQLContainer<>("mysql:8.0").withNetwork(network)) { - bastion.initAndStartBastion(network); - db.start(); - final JsonNode configWithSSLModeDisabled = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, Objects.requireNonNull(db.getContainerInfo() - .getNetworkSettings() - .getNetworks() - .entrySet().stream() - .findFirst() - .get().getValue().getIpAddress())) - .put(JdbcUtils.PORT_KEY, db.getExposedPorts().get(0)) - .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.SCHEMAS_KEY, List.of("public")) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) - .put(JdbcUtils.SSL_MODE_KEY, Map.of(JdbcUtils.MODE_KEY, "disable")), false); - - final AirbyteConnectionStatus actual = source.check(configWithSSLModeDisabled); - assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, actual.getStatus()); - } finally { - bastion.stopAndClose(); - } - } - - @Override - protected boolean supportsPerStream() { - return true; - } - -} diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json deleted file mode 100644 index 10e6728338b2..000000000000 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json +++ /dev/null @@ -1,226 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/mysql", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MySql Source Spec", - "type": "object", - "required": ["host", "port", "database", "username", "replication_method"], - "properties": { - "host": { - "description": "The host name of the database.", - "title": "Host", - "type": "string", - "order": 0 - }, - "port": { - "description": "The port to connect to.", - "title": "Port", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 3306, - "examples": ["3306"], - "order": 1 - }, - "database": { - "description": "The database name.", - "title": "Database", - "type": "string", - "order": 2 - }, - "username": { - "description": "The username which is used to access the database.", - "title": "Username", - "type": "string", - "order": 3 - }, - "password": { - "description": "The password associated with the username.", - "title": "Password", - "type": "string", - "airbyte_secret": true, - "order": 4, - "always_show": true - }, - "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", - "title": "JDBC URL Parameters (Advanced)", - "type": "string", - "order": 5 - }, - "ssl_mode": { - "title": "SSL modes", - "description": "SSL connection modes. Read more in the docs.", - "type": "object", - "order": 7, - "default": "required", - "oneOf": [ - { - "title": "preferred", - "description": "Automatically attempt SSL connection. If the MySQL server does not support SSL, continue with a regular connection.", - "required": ["mode"], - "properties": { - "mode": { - "type": "string", - "const": "preferred", - "order": 0 - } - } - }, - { - "title": "required", - "description": "Always connect with SSL. If the MySQL server doesn’t support SSL, the connection will not be established. Certificate Authority (CA) and Hostname are not verified.", - "required": ["mode"], - "properties": { - "mode": { - "type": "string", - "const": "required", - "order": 0 - } - } - }, - { - "title": "Verify CA", - "description": "Always connect with SSL. Verifies CA, but allows connection even if Hostname does not match.", - "required": ["mode", "ca_certificate"], - "properties": { - "mode": { - "type": "string", - "const": "verify_ca", - "order": 0 - }, - "ca_certificate": { - "type": "string", - "title": "CA certificate", - "description": "CA certificate", - "airbyte_secret": true, - "multiline": true, - "order": 1 - }, - "client_certificate": { - "type": "string", - "title": "Client certificate", - "description": "Client certificate (this is not a required field, but if you want to use it, you will need to add the Client key as well)", - "airbyte_secret": true, - "multiline": true, - "order": 2, - "always_show": true - }, - "client_key": { - "type": "string", - "title": "Client key", - "description": "Client key (this is not a required field, but if you want to use it, you will need to add the Client certificate as well)", - "airbyte_secret": true, - "multiline": true, - "order": 3, - "always_show": true - }, - "client_key_password": { - "type": "string", - "title": "Client key password", - "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", - "airbyte_secret": true, - "order": 4 - } - } - }, - { - "title": "Verify Identity", - "description": "Always connect with SSL. Verify both CA and Hostname.", - "required": ["mode", "ca_certificate"], - "properties": { - "mode": { - "type": "string", - "const": "verify_identity", - "order": 0 - }, - "ca_certificate": { - "type": "string", - "title": "CA certificate", - "description": "CA certificate", - "airbyte_secret": true, - "multiline": true, - "order": 1 - }, - "client_certificate": { - "type": "string", - "title": "Client certificate", - "description": "Client certificate (this is not a required field, but if you want to use it, you will need to add the Client key as well)", - "airbyte_secret": true, - "multiline": true, - "order": 2, - "always_show": true - }, - "client_key": { - "type": "string", - "title": "Client key", - "description": "Client key (this is not a required field, but if you want to use it, you will need to add the Client certificate as well)", - "airbyte_secret": true, - "multiline": true, - "order": 3, - "always_show": true - }, - "client_key_password": { - "type": "string", - "title": "Client key password", - "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - }, - "replication_method": { - "type": "object", - "title": "Update Method", - "description": "Configures how data is extracted from the database.", - "order": 8, - "default": "CDC", - "oneOf": [ - { - "title": "Read Changes using Binary Log (CDC)", - "description": "Recommended - Incrementally reads new inserts, updates, and deletes using the MySQL binary log. This must be enabled on your database.", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "CDC", - "order": 0 - }, - "initial_waiting_seconds": { - "type": "integer", - "title": "Initial Waiting Time in Seconds (Advanced)", - "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", - "default": 300, - "min": 120, - "max": 1200, - "order": 1, - "always_show": true - }, - "server_time_zone": { - "type": "string", - "title": "Configured server timezone for the MySQL source (Advanced)", - "description": "Enter the configured MySQL server timezone. This should only be done if the configured timezone in your MySQL instance does not conform to IANNA standard.", - "order": 2, - "always_show": true - } - } - }, - { - "title": "Scan Changes with User Defined Cursor", - "description": "Incrementally detects new inserts and updates using the cursor column chosen when configuring a connection (e.g. created_at, updated_at).", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "STANDARD", - "order": 0 - } - } - } - ] - } - } - } -} diff --git a/airbyte-integrations/connectors/source-mysql/.dockerignore b/airbyte-integrations/connectors/source-mysql/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-mysql/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-mysql/Dockerfile b/airbyte-integrations/connectors/source-mysql/Dockerfile deleted file mode 100644 index 76a9db7cc650..000000000000 --- a/airbyte-integrations/connectors/source-mysql/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-mysql - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-mysql - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=3.0.1 - -LABEL io.airbyte.name=airbyte/source-mysql diff --git a/airbyte-integrations/connectors/source-mysql/README.md b/airbyte-integrations/connectors/source-mysql/README.md index 5c0c64671285..945aff0c9c9e 100644 --- a/airbyte-integrations/connectors/source-mysql/README.md +++ b/airbyte-integrations/connectors/source-mysql/README.md @@ -16,10 +16,11 @@ From the Airbyte repository root, run: #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:source-mysql:airbyteDocker +./gradlew :airbyte-integrations:connectors:source-mysql:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/source-mysql:dev`. the Dockerfile. ## Testing diff --git a/airbyte-integrations/connectors/source-mysql/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mysql/acceptance-test-config.yml index a3dd76400d68..575c91b8bb87 100644 --- a/airbyte-integrations/connectors/source-mysql/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-mysql/acceptance-test-config.yml @@ -3,5 +3,8 @@ connector_image: airbyte/source-mysql:dev tests: spec: - - spec_path: "src/test-integration/resources/expected_spec.json" + - spec_path: "src/test-integration/resources/expected_oss_spec.json" + config_path: "src/test-integration/resources/dummy_config.json" + - deployment_mode: cloud + spec_path: "src/test-integration/resources/expected_cloud_spec.json" config_path: "src/test-integration/resources/dummy_config.json" diff --git a/airbyte-integrations/connectors/source-mysql/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-mysql/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-mysql/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-mysql/build.gradle b/airbyte-integrations/connectors/source-mysql/build.gradle index 8bacfd89490a..77b7c8054490 100644 --- a/airbyte-integrations/connectors/source-mysql/build.gradle +++ b/airbyte-integrations/connectors/source-mysql/build.gradle @@ -2,45 +2,43 @@ import org.jsonschema2pojo.SourceType plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' - id 'airbyte-performance-test-java' - id 'airbyte-connector-acceptance-test' + id 'airbyte-java-connector' id 'org.jsonschema2pojo' version '1.2.1' } +airbyteJavaConnector { + cdkVersionRequired = '0.11.4' + features = ['db-sources'] + useLocalCdk = false +} + +configurations.all { + resolutionStrategy { + force libs.jooq + } +} + + + application { mainClass = 'io.airbyte.integrations.source.mysql.MySqlSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:bases:debezium') - implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:connectors:source-relational-db') + implementation libs.jooq implementation 'mysql:mysql-connector-java:8.0.30' implementation 'org.apache.commons:commons-lang3:3.11' + implementation project(path: ':airbyte-cdk:java:airbyte-cdk:db-sources') - testImplementation testFixtures(project(':airbyte-integrations:bases:debezium')) - testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) testImplementation 'org.apache.commons:commons-lang3:3.11' testImplementation 'org.hamcrest:hamcrest-all:1.3' testImplementation libs.junit.jupiter.system.stubs - testImplementation libs.connectors.testcontainers.mysql - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-mysql') - - performanceTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') + testImplementation libs.testcontainers.mysql + testFixturesImplementation libs.testcontainers.mysql performanceTestJavaImplementation project(':airbyte-integrations:connectors:source-mysql') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - performanceTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } jsonSchema2Pojo { @@ -56,4 +54,3 @@ jsonSchema2Pojo { includeConstructors = false includeSetters = true } - diff --git a/airbyte-integrations/connectors/source-mysql/gradle.properties b/airbyte-integrations/connectors/source-mysql/gradle.properties new file mode 100644 index 000000000000..8ef098d20b92 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/gradle.properties @@ -0,0 +1 @@ +testExecutionConcurrency=-1 \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-mysql/metadata.yaml b/airbyte-integrations/connectors/source-mysql/metadata.yaml index ae7200b1af02..caaf7458309b 100644 --- a/airbyte-integrations/connectors/source-mysql/metadata.yaml +++ b/airbyte-integrations/connectors/source-mysql/metadata.yaml @@ -1,4 +1,7 @@ data: + ab_internal: + ql: 300 + sl: 300 allowedHosts: hosts: - ${host} @@ -6,30 +9,26 @@ data: connectorSubtype: database connectorType: source definitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad - dockerImageTag: 3.0.1 + dockerImageTag: 3.3.2 dockerRepository: airbyte/source-mysql + documentationUrl: https://docs.airbyte.com/integrations/sources/mysql githubIssueLabel: source-mysql icon: mysql.svg license: ELv2 name: MySQL registries: cloud: - dockerRepository: airbyte/source-mysql-strict-encrypt enabled: true oss: enabled: true - releaseStage: beta - documentationUrl: https://docs.airbyte.com/integrations/sources/mysql - tags: - - language:java - - language:python + releaseStage: generally_available releases: breakingChanges: 3.0.0: - message: "Add default cursor for cdc" + message: Add default cursor for cdc upgradeDeadline: "2023-08-17" - ab_internal: - sl: 300 - ql: 300 supportLevel: certified + tags: + - language:java + - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcConnectorMetadataInjector.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcConnectorMetadataInjector.java index c161cfbb8e21..bfe231a96b4f 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcConnectorMetadataInjector.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcConnectorMetadataInjector.java @@ -4,16 +4,16 @@ package io.airbyte.integrations.source.mysql; -import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; -import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; import static io.airbyte.integrations.source.mysql.MySqlSource.CDC_DEFAULT_CURSOR; import static io.airbyte.integrations.source.mysql.MySqlSource.CDC_LOG_FILE; import static io.airbyte.integrations.source.mysql.MySqlSource.CDC_LOG_POS; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.integrations.debezium.CdcMetadataInjector; -import io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes; +import io.airbyte.cdk.integrations.debezium.CdcMetadataInjector; +import io.airbyte.cdk.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes; import java.time.Instant; import java.util.concurrent.atomic.AtomicLong; @@ -61,6 +61,11 @@ public String namespace(final JsonNode source) { return source.get("db").asText(); } + @Override + public String name(JsonNode source) { + return source.get("table").asText(); + } + private Long getCdcDefaultCursor() { return this.emittedAtConverted + this.recordCounter.getAndIncrement(); } diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java index 96e871915da4..15bc34eefdcf 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcProperties.java @@ -4,18 +4,18 @@ package io.airbyte.integrations.source.mysql; -import static io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.CLIENT_KEY_STORE_PASS; -import static io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.CLIENT_KEY_STORE_URL; -import static io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.SSL_MODE; -import static io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.TRUST_KEY_STORE_PASS; -import static io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.TRUST_KEY_STORE_URL; +import static io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils.CLIENT_KEY_STORE_PASS; +import static io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils.CLIENT_KEY_STORE_URL; +import static io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils.SSL_MODE; +import static io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils.TRUST_KEY_STORE_PASS; +import static io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils.TRUST_KEY_STORE_URL; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.debezium.internals.mysql.CustomMySQLTinyIntOneToBooleanConverter; -import io.airbyte.integrations.debezium.internals.mysql.MySQLDateTimeConverter; -import io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.SslMode; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.debezium.internals.mysql.CustomMySQLTinyIntOneToBooleanConverter; +import io.airbyte.cdk.integrations.debezium.internals.mysql.MySQLDateTimeConverter; +import io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils.SslMode; import java.net.URI; import java.nio.file.Path; import java.time.Duration; @@ -26,7 +26,10 @@ public class MySqlCdcProperties { private static final Logger LOGGER = LoggerFactory.getLogger(MySqlCdcProperties.class); - private static final Duration HEARTBEAT_FREQUENCY = Duration.ofSeconds(10); + private static final Duration HEARTBEAT_INTERVAL = Duration.ofSeconds(10L); + + // Test execution latency is lower when heartbeats are more frequent. + private static final Duration HEARTBEAT_INTERVAL_IN_TESTS = Duration.ofSeconds(1L); public static Properties getDebeziumProperties(final JdbcDatabase database) { final JsonNode sourceConfig = database.getSourceConfig(); @@ -61,7 +64,12 @@ private static Properties commonProperties(final JdbcDatabase database) { props.setProperty("converters", "boolean, datetime"); props.setProperty("boolean.type", CustomMySQLTinyIntOneToBooleanConverter.class.getName()); props.setProperty("datetime.type", MySQLDateTimeConverter.class.getName()); - props.setProperty("heartbeat.interval.ms", Long.toString(HEARTBEAT_FREQUENCY.toMillis())); + + final Duration heartbeatInterval = + (database.getSourceConfig().has("is_test") && database.getSourceConfig().get("is_test").asBoolean()) + ? HEARTBEAT_INTERVAL_IN_TESTS + : HEARTBEAT_INTERVAL; + props.setProperty("heartbeat.interval.ms", Long.toString(heartbeatInterval.toMillis())); // For CDC mode, the user cannot provide timezone arguments as JDBC parameters - they are // specifically defined in the replication_method @@ -69,7 +77,12 @@ private static Properties commonProperties(final JdbcDatabase database) { if (sourceConfig.get("replication_method").has("server_time_zone")) { final String serverTimeZone = sourceConfig.get("replication_method").get("server_time_zone").asText(); if (!serverTimeZone.isEmpty()) { - props.setProperty("database.serverTimezone", serverTimeZone); + /** + * Per Debezium docs, + * https://debezium.io/documentation/reference/stable/connectors/mysql.html#mysql-temporal-types + * this property is now connectionTimeZone {@link com.mysql.cj.conf.PropertyKey#connectionTimeZone} + **/ + props.setProperty("database.connectionTimeZone", serverTimeZone); } } @@ -115,12 +128,6 @@ private static Properties commonProperties(final JdbcDatabase database) { return props; } - static Properties getSnapshotProperties(final JdbcDatabase database) { - final Properties props = commonProperties(database); - props.setProperty("snapshot.mode", "initial_only"); - return props; - } - private static int generateServerID() { final int min = 5400; final int max = 6400; diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcSavedInfoFetcher.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcSavedInfoFetcher.java index e99ff2776482..b5d3d3a81643 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcSavedInfoFetcher.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcSavedInfoFetcher.java @@ -4,23 +4,28 @@ package io.airbyte.integrations.source.mysql; -import static io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MYSQL_CDC_OFFSET; -import static io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MYSQL_DB_HISTORY; +import static io.airbyte.cdk.integrations.debezium.internals.mysql.MysqlCdcStateConstants.IS_COMPRESSED; +import static io.airbyte.cdk.integrations.debezium.internals.mysql.MysqlCdcStateConstants.MYSQL_CDC_OFFSET; +import static io.airbyte.cdk.integrations.debezium.internals.mysql.MysqlCdcStateConstants.MYSQL_DB_HISTORY; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.debezium.CdcSavedInfoFetcher; -import io.airbyte.integrations.source.relationaldb.models.CdcState; +import io.airbyte.cdk.integrations.debezium.CdcSavedInfoFetcher; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteSchemaHistoryStorage.SchemaHistory; +import io.airbyte.cdk.integrations.source.relationaldb.models.CdcState; import java.util.Optional; public class MySqlCdcSavedInfoFetcher implements CdcSavedInfoFetcher { private final JsonNode savedOffset; private final JsonNode savedSchemaHistory; + private final boolean isSavedSchemaHistoryCompressed; public MySqlCdcSavedInfoFetcher(final CdcState savedState) { final boolean savedStatePresent = savedState != null && savedState.getState() != null; this.savedOffset = savedStatePresent ? savedState.getState().get(MYSQL_CDC_OFFSET) : null; this.savedSchemaHistory = savedStatePresent ? savedState.getState().get(MYSQL_DB_HISTORY) : null; + this.isSavedSchemaHistoryCompressed = + savedStatePresent && savedState.getState().has(IS_COMPRESSED) && savedState.getState().get(IS_COMPRESSED).asBoolean(); } @Override @@ -29,8 +34,8 @@ public JsonNode getSavedOffset() { } @Override - public Optional getSavedSchemaHistory() { - return Optional.ofNullable(savedSchemaHistory); + public SchemaHistory> getSavedSchemaHistory() { + return new SchemaHistory<>(Optional.ofNullable(savedSchemaHistory), isSavedSchemaHistoryCompressed); } } diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcStateHandler.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcStateHandler.java index 1f3ea8d84c8c..93a489a56b08 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcStateHandler.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcStateHandler.java @@ -4,18 +4,17 @@ package io.airbyte.integrations.source.mysql; -import static io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MYSQL_CDC_OFFSET; -import static io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MYSQL_DB_HISTORY; +import static io.airbyte.cdk.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.serialize; +import static io.airbyte.cdk.integrations.debezium.internals.mysql.MysqlCdcStateConstants.COMPRESSION_ENABLED; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.debezium.CdcStateHandler; -import io.airbyte.integrations.source.relationaldb.models.CdcState; -import io.airbyte.integrations.source.relationaldb.state.StateManager; +import io.airbyte.cdk.integrations.debezium.CdcStateHandler; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteSchemaHistoryStorage.SchemaHistory; +import io.airbyte.cdk.integrations.source.relationaldb.models.CdcState; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.AirbyteStateMessage; -import java.util.HashMap; import java.util.Map; import java.util.Optional; import org.slf4j.Logger; @@ -37,12 +36,8 @@ public boolean isCdcCheckpointEnabled() { } @Override - public AirbyteMessage saveState(final Map offset, final String dbHistory) { - final Map state = new HashMap<>(); - state.put(MYSQL_CDC_OFFSET, offset); - state.put(MYSQL_DB_HISTORY, dbHistory); - - final JsonNode asJson = Jsons.jsonNode(state); + public AirbyteMessage saveState(final Map offset, final SchemaHistory dbHistory) { + final JsonNode asJson = serialize(offset, dbHistory); LOGGER.info("debezium state: {}", asJson); @@ -67,4 +62,9 @@ public AirbyteMessage saveStateAfterCompletionOfSnapshotOfNewStreams() { return new AirbyteMessage().withType(Type.STATE).withState(stateMessage); } + @Override + public boolean compressSchemaHistoryForState() { + return COMPRESSION_ENABLED; + } + } diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlQueryUtils.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlQueryUtils.java index f5c536ad00de..2651e13f4d56 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlQueryUtils.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlQueryUtils.java @@ -4,18 +4,29 @@ package io.airbyte.integrations.source.mysql; -import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.getIdentifierWithQuoting; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Preconditions; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; +import com.google.common.collect.ImmutableList; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.source.relationaldb.CursorInfo; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; +import io.airbyte.integrations.source.mysql.internal.models.CursorBasedStatus; +import io.airbyte.integrations.source.mysql.internal.models.InternalModels.StateType; import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.sql.SQLException; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,26 +36,57 @@ public class MySqlQueryUtils { public record TableSizeInfo(Long tableSize, Long avgRowLength) {} - public static final String TABLE_ESTIMATE_QUERY = - """ - SELECT - (data_length + index_length) as %s, - AVG_ROW_LENGTH as %s - FROM - information_schema.tables - WHERE - table_schema = '%s' AND table_name = '%s'; - """; + public static final String TABLE_ESTIMATE_QUERY = """ + SELECT + (data_length + index_length) as %s, + AVG_ROW_LENGTH as %s + FROM + information_schema.tables + WHERE + table_schema = '%s' AND table_name = '%s'; + """; public static final String MAX_PK_VALUE_QUERY = """ SELECT MAX(%s) as %s FROM %s; """; + public static final String SHOW_TABLE_QUERY = + """ + SHOW TABLE STATUS; + """; + public static final String MAX_CURSOR_VALUE_QUERY = + """ + SELECT %s FROM %s WHERE %s = (SELECT MAX(%s) FROM %s); + """; + public static final String MAX_PK_COL = "max_pk"; public static final String TABLE_SIZE_BYTES_COL = "TotalSizeBytes"; public static final String AVG_ROW_LENGTH = "AVG_ROW_LENGTH"; + // Returns a set of all storage engines used by the configured tables + public static Set getStorageEngines(final JdbcDatabase database, final Set streamNames) { + try { + // Construct the query. + final List jsonNodes = database.bufferedResultSetQuery(conn -> conn.createStatement().executeQuery(SHOW_TABLE_QUERY), + resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet)); + final Set storageEngines = new HashSet<>(); + if (jsonNodes != null) { + jsonNodes.stream().forEach(jsonNode -> { + final String tableName = jsonNode.get("Name").asText(); + final String storageEngine = jsonNode.get("Engine").asText(); + if (streamNames.contains(tableName)) { + storageEngines.add(storageEngine); + } + }); + } + return storageEngines; + } catch (final Exception e) { + LOGGER.info("Storage engines could not be determined"); + return Collections.emptySet(); + } + } + public static String getMaxPkValueForStream(final JdbcDatabase database, final ConfiguredAirbyteStream stream, final String pkFieldName, @@ -54,7 +96,7 @@ public static String getMaxPkValueForStream(final JdbcDatabase database, final String fullTableName = getFullyQualifiedTableNameWithQuoting(namespace, name, quoteString); final String maxPkQuery = String.format(MAX_PK_VALUE_QUERY, - pkFieldName, + getIdentifierWithQuoting(pkFieldName, quoteString), MAX_PK_COL, fullTableName); LOGGER.info("Querying for max pk value: {}", maxPkQuery); @@ -83,31 +125,110 @@ public static Map getTableSizeInf final String fullTableName = getFullyQualifiedTableNameWithQuoting(name, namespace, quoteString); final List tableEstimateResult = getTableEstimate(database, namespace, name); - Preconditions.checkState(tableEstimateResult.size() == 1); - final long tableEstimateBytes = tableEstimateResult.get(0).get(TABLE_SIZE_BYTES_COL).asLong(); - final long avgTableRowSizeBytes = tableEstimateResult.get(0).get(AVG_ROW_LENGTH).asLong(); - LOGGER.info("Stream {} size estimate is {}, average row size estimate is {}", fullTableName, tableEstimateBytes, avgTableRowSizeBytes); - final TableSizeInfo tableSizeInfo = new TableSizeInfo(tableEstimateBytes, avgTableRowSizeBytes); - final AirbyteStreamNameNamespacePair namespacePair = - new AirbyteStreamNameNamespacePair(stream.getStream().getName(), stream.getStream().getNamespace()); - tableSizeInfoMap.put(namespacePair, tableSizeInfo); - } catch (final SQLException e) { + + if (tableEstimateResult != null + && tableEstimateResult.size() == 1 + && tableEstimateResult.get(0).get(TABLE_SIZE_BYTES_COL) != null + && tableEstimateResult.get(0).get(AVG_ROW_LENGTH) != null) { + final long tableEstimateBytes = tableEstimateResult.get(0).get(TABLE_SIZE_BYTES_COL).asLong(); + final long avgTableRowSizeBytes = tableEstimateResult.get(0).get(AVG_ROW_LENGTH).asLong(); + LOGGER.info("Stream {} size estimate is {}, average row size estimate is {}", fullTableName, tableEstimateBytes, avgTableRowSizeBytes); + final TableSizeInfo tableSizeInfo = new TableSizeInfo(tableEstimateBytes, avgTableRowSizeBytes); + final AirbyteStreamNameNamespacePair namespacePair = + new AirbyteStreamNameNamespacePair(stream.getStream().getName(), stream.getStream().getNamespace()); + tableSizeInfoMap.put(namespacePair, tableSizeInfo); + } + } catch (final Exception e) { LOGGER.warn("Error occurred while attempting to estimate sync size", e); } }); return tableSizeInfoMap; } + /** + * Iterates through each stream and find the max cursor value and the record count which has that + * value based on each cursor field provided by the customer per stream This information is saved in + * a Hashmap with the mapping being the AirbyteStreamNameNamespacepair -> CursorBasedStatus + * + * @param database the source db + * @param streams streams to be synced + * @param stateManager stream stateManager + * @return Map of streams to statuses + */ + public static Map getCursorBasedSyncStatusForStreams(final JdbcDatabase database, + final List streams, + final StateManager stateManager, + final String quoteString) { + + final Map cursorBasedStatusMap = new HashMap<>(); + streams.forEach(stream -> { + try { + final String name = stream.getStream().getName(); + final String namespace = stream.getStream().getNamespace(); + final String fullTableName = + getFullyQualifiedTableNameWithQuoting(namespace, name, quoteString); + + final Optional cursorInfoOptional = + stateManager.getCursorInfo(new io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair(name, namespace)); + if (cursorInfoOptional.isEmpty()) { + throw new RuntimeException(String.format("Stream %s was not provided with an appropriate cursor", stream.getStream().getName())); + } + + LOGGER.info("Querying max cursor value for {}.{}", namespace, name); + final String cursorField = cursorInfoOptional.get().getCursorField(); + final String quotedCursorField = getIdentifierWithQuoting(cursorField, quoteString); + final String cursorBasedSyncStatusQuery = String.format(MAX_CURSOR_VALUE_QUERY, + quotedCursorField, + fullTableName, + quotedCursorField, + quotedCursorField, + fullTableName); + final List jsonNodes = database.bufferedResultSetQuery(conn -> conn.prepareStatement(cursorBasedSyncStatusQuery).executeQuery(), + resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet)); + final CursorBasedStatus cursorBasedStatus = new CursorBasedStatus(); + cursorBasedStatus.setStateType(StateType.CURSOR_BASED); + cursorBasedStatus.setVersion(2L); + cursorBasedStatus.setStreamName(name); + cursorBasedStatus.setStreamNamespace(namespace); + cursorBasedStatus.setCursorField(ImmutableList.of(cursorField)); + + if (!jsonNodes.isEmpty()) { + final JsonNode result = jsonNodes.get(0); + cursorBasedStatus.setCursor(result.get(cursorField).asText()); + cursorBasedStatus.setCursorRecordCount((long) jsonNodes.size()); + } + + cursorBasedStatusMap.put(new io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair(name, namespace), cursorBasedStatus); + } catch (final SQLException e) { + throw new RuntimeException(e); + } + }); + + return cursorBasedStatusMap; + } + private static List getTableEstimate(final JdbcDatabase database, final String namespace, final String name) throws SQLException { // Construct the table estimate query. final String tableEstimateQuery = String.format(TABLE_ESTIMATE_QUERY, TABLE_SIZE_BYTES_COL, AVG_ROW_LENGTH, namespace, name); - LOGGER.info("table estimate query: {}", tableEstimateQuery); final List jsonNodes = database.bufferedResultSetQuery(conn -> conn.createStatement().executeQuery(tableEstimateQuery), resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet)); Preconditions.checkState(jsonNodes.size() == 1); return jsonNodes; } + public static void logStreamSyncStatus(final List streams, final String syncType) { + if (streams.isEmpty()) { + LOGGER.info("No Streams will be synced via {}.", syncType); + } else { + LOGGER.info("Streams to be synced via {} : {}", syncType, streams.size()); + LOGGER.info("Streams: {}", prettyPrintConfiguredAirbyteStreamList(streams)); + } + } + + public static String prettyPrintConfiguredAirbyteStreamList(final List streamList) { + return streamList.stream().map(s -> "%s.%s".formatted(s.getStream().getNamespace(), s.getStream().getName())).collect(Collectors.joining(", ")); + } + } diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java index 515e87d848b8..248eed5b06c3 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java @@ -4,13 +4,18 @@ package io.airbyte.integrations.source.mysql; -import static io.airbyte.db.jdbc.JdbcUtils.EQUALS; -import static io.airbyte.integrations.debezium.AirbyteDebeziumHandler.shouldUseCDC; -import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; -import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; -import static io.airbyte.integrations.source.jdbc.JdbcDataSourceUtils.DEFAULT_JDBC_PARAMETERS_DELIMITER; -import static io.airbyte.integrations.source.jdbc.JdbcDataSourceUtils.assertCustomParametersDontOverwriteDefaultParameters; -import static io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.SSL_MODE; +import static io.airbyte.cdk.db.jdbc.JdbcUtils.EQUALS; +import static io.airbyte.cdk.integrations.debezium.AirbyteDebeziumHandler.isAnyStreamIncrementalSyncMode; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.cdk.integrations.source.jdbc.JdbcDataSourceUtils.DEFAULT_JDBC_PARAMETERS_DELIMITER; +import static io.airbyte.cdk.integrations.source.jdbc.JdbcDataSourceUtils.assertCustomParametersDontOverwriteDefaultParameters; +import static io.airbyte.integrations.source.mysql.MySqlQueryUtils.getCursorBasedSyncStatusForStreams; +import static io.airbyte.integrations.source.mysql.MySqlQueryUtils.getTableSizeInfoForStreams; +import static io.airbyte.integrations.source.mysql.MySqlQueryUtils.logStreamSyncStatus; +import static io.airbyte.integrations.source.mysql.initialsync.MySqlInitialReadUtil.convertNameNamespacePairFromV0; +import static io.airbyte.integrations.source.mysql.initialsync.MySqlInitialReadUtil.initPairToPrimaryKeyInfoMap; +import static io.airbyte.integrations.source.mysql.initialsync.MySqlInitialReadUtil.streamsForInitialPrimaryKeyLoad; import static java.util.stream.Collectors.toList; import com.fasterxml.jackson.databind.JsonNode; @@ -20,64 +25,63 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.mysql.cj.MysqlType; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.jdbc.StreamingJdbcDatabase; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.base.adaptive.AdaptiveSourceRunner; +import io.airbyte.cdk.integrations.base.ssh.SshWrappedSource; +import io.airbyte.cdk.integrations.debezium.internals.RecordWaitTimeUtil; +import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.cdk.integrations.source.jdbc.JdbcDataSourceUtils; +import io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils; +import io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils.SslMode; +import io.airbyte.cdk.integrations.source.relationaldb.DbSourceDiscoverUtil; +import io.airbyte.cdk.integrations.source.relationaldb.TableInfo; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateGeneratorUtils; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManagerFactory; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.exceptions.ConfigErrorException; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.functional.CheckedConsumer; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; import io.airbyte.commons.util.AutoCloseableIterator; -import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.StreamingJdbcDatabase; -import io.airbyte.integrations.base.AirbyteTraceMessageUtility; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.base.ssh.SshWrappedSource; -import io.airbyte.integrations.debezium.AirbyteDebeziumHandler; -import io.airbyte.integrations.debezium.internals.FirstRecordWaitTimeUtil; -import io.airbyte.integrations.debezium.internals.mysql.MySqlCdcPosition; -import io.airbyte.integrations.debezium.internals.mysql.MySqlCdcTargetPosition; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.JdbcDataSourceUtils; -import io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils; -import io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.SslMode; +import io.airbyte.integrations.source.mysql.cursor_based.MySqlCursorBasedStateManager; import io.airbyte.integrations.source.mysql.helpers.CdcConfigurationHelper; -import io.airbyte.integrations.source.mysql.initialsync.MySqlFeatureFlags; +import io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadHandler; +import io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadStreamStateManager; import io.airbyte.integrations.source.mysql.initialsync.MySqlInitialReadUtil; -import io.airbyte.integrations.source.relationaldb.DbSourceDiscoverUtil; -import io.airbyte.integrations.source.relationaldb.TableInfo; -import io.airbyte.integrations.source.relationaldb.models.CdcState; -import io.airbyte.integrations.source.relationaldb.state.StateGeneratorUtils; -import io.airbyte.integrations.source.relationaldb.state.StateManager; -import io.airbyte.integrations.source.relationaldb.state.StateManagerFactory; -import io.airbyte.integrations.util.HostPortResolver; +import io.airbyte.integrations.source.mysql.initialsync.MySqlInitialReadUtil.CursorBasedStreams; +import io.airbyte.integrations.source.mysql.initialsync.MySqlInitialReadUtil.InitialLoadStreams; +import io.airbyte.integrations.source.mysql.internal.models.CursorBasedStatus; import io.airbyte.protocol.models.CommonField; import io.airbyte.protocol.models.v0.AirbyteCatalog; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.ConnectorSpecification; import io.airbyte.protocol.models.v0.SyncMode; import java.sql.SQLException; -import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.OptionalInt; import java.util.Set; import java.util.function.Function; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; import javax.sql.DataSource; @@ -86,6 +90,13 @@ public class MySqlSource extends AbstractJdbcSource implements Source { + public static final String TUNNEL_METHOD = "tunnel_method"; + public static final String NO_TUNNEL = "NO_TUNNEL"; + public static final String SSL_MODE = "ssl_mode"; + private static final String MODE = "mode"; + public static final String SSL_MODE_PREFERRED = "preferred"; + public static final String SSL_MODE_REQUIRED = "required"; + private static final Logger LOGGER = LoggerFactory.getLogger(MySqlSource.class); private static final int INTERMEDIATE_STATE_EMISSION_FREQUENCY = 10_000; public static final String NULL_CURSOR_VALUE_WITH_SCHEMA_QUERY = @@ -113,15 +124,52 @@ public class MySqlSource extends AbstractJdbcSource implements Source "useSSL=true", "requireSSL=true"); - private final FeatureFlags featureFlags; + public static Source sshWrappedSource(MySqlSource source) { + return new SshWrappedSource(source, JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY); + } + + private ConnectorSpecification getCloudDeploymentSpec(final ConnectorSpecification originalSpec) { + final ConnectorSpecification spec = Jsons.clone(originalSpec); + // Remove the SSL options + ((ObjectNode) spec.getConnectionSpecification().get("properties")).remove(JdbcUtils.SSL_KEY); + // Set SSL_MODE to required by default + ((ObjectNode) spec.getConnectionSpecification().get("properties").get(SSL_MODE)).put("default", SSL_MODE_REQUIRED); + return spec; + } + + @Override + public ConnectorSpecification spec() throws Exception { + if (cloudDeploymentMode()) { + return getCloudDeploymentSpec(super.spec()); + } + return super.spec(); + } - public static Source sshWrappedSource() { - return new SshWrappedSource(new MySqlSource(), JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY); + @Override + public AirbyteConnectionStatus check(final JsonNode config) throws Exception { + // #15808 Disallow connecting to db with disable, prefer or allow SSL mode when connecting directly + // and not over SSH tunnel + if (cloudDeploymentMode()) { + if (config.has(TUNNEL_METHOD) + && config.get(TUNNEL_METHOD).has(TUNNEL_METHOD) + && config.get(TUNNEL_METHOD).get(TUNNEL_METHOD).asText().equals(NO_TUNNEL)) { + // If no SSH tunnel + if (config.has(SSL_MODE) && config.get(SSL_MODE).has(MODE)) { + if (Set.of(SSL_MODE_PREFERRED).contains(config.get(SSL_MODE).get(MODE).asText())) { + // Fail in case SSL mode is preferred + return new AirbyteConnectionStatus() + .withStatus(Status.FAILED) + .withMessage( + "Unsecured connection not allowed. If no SSH Tunnel set up, please use one of the following SSL modes: required, verify-ca, verify-identity"); + } + } + } + } + return super.check(config); } public MySqlSource() { super(DRIVER_CLASS, MySqlStreamingQueryConfig::new, new MySqlSourceOperations()); - this.featureFlags = new EnvVariableFeatureFlags(); } private static AirbyteStream overrideSyncModes(final AirbyteStream stream) { @@ -179,7 +227,7 @@ public List> getCheckOperations(final J checkOperations.addAll(CdcConfigurationHelper.getCheckOperations()); checkOperations.add(database -> { - FirstRecordWaitTimeUtil.checkFirstRecordWaitTime(config); + RecordWaitTimeUtil.checkFirstRecordWaitTime(config); CdcConfigurationHelper.checkServerTimeZoneConfig(config); }); } @@ -213,7 +261,7 @@ public Collection> readStreams(final JsonN final AirbyteStateType supportedStateType = getSupportedStateType(config); final StateManager stateManager = StateManagerFactory.createStateManager(supportedStateType, - StateGeneratorUtils.deserializeInitialState(state, featureFlags.useStreamCapableState(), supportedStateType), catalog); + StateGeneratorUtils.deserializeInitialState(state, supportedStateType), catalog); final Instant emittedAt = Instant.now(); final JdbcDatabase database = createDatabase(config); @@ -245,6 +293,18 @@ public Collection> readStreams(final JsonN return iteratorList; } + @Override + protected void logPreSyncDebugData(final JdbcDatabase database, final ConfiguredAirbyteCatalog catalog) + throws SQLException { + super.logPreSyncDebugData(database, catalog); + final Set streamNames = new HashSet<>(); + for (final ConfiguredAirbyteStream stream : catalog.getStreams()) { + streamNames.add(stream.getStream().getName()); + } + final Set storageEngines = MySqlQueryUtils.getStorageEngines(database, streamNames); + LOGGER.info(String.format("Detected the following storage engines for MySQL: %s", storageEngines.toString())); + } + @Override public JsonNode toDatabaseConfig(final JsonNode config) { final String encodedDatabaseName = HostPortResolver.encodeValue(config.get(JdbcUtils.DATABASE_KEY).asText()); @@ -258,7 +318,9 @@ public JsonNode toDatabaseConfig(final JsonNode config) { // When using this approach MySql creates a temporary table which may have some effect on db // performance. jdbcUrl.append("?useCursorFetch=true"); - jdbcUrl.append("&zeroDateTimeBehavior=convertToNull"); + // What should happen when the driver encounters DATETIME values that are composed entirely of zeros + // https://dev.mysql.com/doc/connector-j/8.1/en/connector-j-connp-props-datetime-types-processing.html#cj-conn-prop_zeroDateTimeBehavior + jdbcUrl.append("&zeroDateTimeBehavior=CONVERT_TO_NULL"); // ensure the return tinyint(1) is boolean jdbcUrl.append("&tinyInt1isBit=true"); // ensure the return year value is a Date; see the rationale @@ -318,10 +380,6 @@ private static boolean isCdc(final JsonNode config) { @Override protected AirbyteStateType getSupportedStateType(final JsonNode config) { - if (!featureFlags.useStreamCapableState()) { - return AirbyteStateType.LEGACY; - } - return isCdc(config) ? AirbyteStateType.GLOBAL : AirbyteStateType.STREAM; } @@ -331,51 +389,52 @@ public List> getIncrementalIterators(final final Map>> tableNameToTable, final StateManager stateManager, final Instant emittedAt) { - final JsonNode sourceConfig = database.getSourceConfig(); - final MySqlFeatureFlags featureFlags = new MySqlFeatureFlags(sourceConfig); - if (isCdc(sourceConfig) && shouldUseCDC(catalog)) { - if (featureFlags.isCdcInitialSyncViaPkEnabled()) { - LOGGER.info("Using PK + CDC"); - return MySqlInitialReadUtil.getCdcReadIterators(database, catalog, tableNameToTable, stateManager, emittedAt, getQuoteString()); - } - final Duration firstRecordWaitTime = FirstRecordWaitTimeUtil.getFirstRecordWaitTime(sourceConfig); - LOGGER.info("First record waiting time: {} seconds", firstRecordWaitTime.getSeconds()); - final AirbyteDebeziumHandler handler = - new AirbyteDebeziumHandler<>(sourceConfig, MySqlCdcTargetPosition.targetPosition(database), true, firstRecordWaitTime, OptionalInt.empty()); - - final MySqlCdcStateHandler mySqlCdcStateHandler = new MySqlCdcStateHandler(stateManager); - final MySqlCdcConnectorMetadataInjector mySqlCdcConnectorMetadataInjector = MySqlCdcConnectorMetadataInjector.getInstance(emittedAt); - - final List streamsToSnapshot = identifyStreamsToSnapshot(catalog, stateManager); - final Optional cdcState = Optional.ofNullable(stateManager.getCdcStateManager().getCdcState()); - - final Supplier> incrementalIteratorSupplier = () -> handler.getIncrementalIterators(catalog, - new MySqlCdcSavedInfoFetcher(cdcState.orElse(null)), - new MySqlCdcStateHandler(stateManager), - mySqlCdcConnectorMetadataInjector, - MySqlCdcProperties.getDebeziumProperties(database), - emittedAt, - false); - - if (streamsToSnapshot.isEmpty()) { - return Collections.singletonList(incrementalIteratorSupplier.get()); - } - - final AutoCloseableIterator snapshotIterator = handler.getSnapshotIterators( - new ConfiguredAirbyteCatalog().withStreams(streamsToSnapshot), - mySqlCdcConnectorMetadataInjector, - MySqlCdcProperties.getSnapshotProperties(database), - mySqlCdcStateHandler, - emittedAt); - return Collections.singletonList( - AutoCloseableIterators.concatWithEagerClose(AirbyteTraceMessageUtility::emitStreamStatusTrace, snapshotIterator, - AutoCloseableIterators.lazyIterator(incrementalIteratorSupplier, null))); + final JsonNode sourceConfig = database.getSourceConfig(); + if (isCdc(sourceConfig) && isAnyStreamIncrementalSyncMode(catalog)) { + LOGGER.info("Using PK + CDC"); + return MySqlInitialReadUtil.getCdcReadIterators(database, catalog, tableNameToTable, stateManager, emittedAt, getQuoteString()); } else { - LOGGER.info("using CDC: {}", false); - return super.getIncrementalIterators(database, catalog, tableNameToTable, stateManager, - emittedAt); + if (isAnyStreamIncrementalSyncMode(catalog)) { + LOGGER.info("Syncing via Primary Key"); + final MySqlCursorBasedStateManager cursorBasedStateManager = new MySqlCursorBasedStateManager(stateManager.getRawStateMessages(), catalog); + final InitialLoadStreams initialLoadStreams = streamsForInitialPrimaryKeyLoad(cursorBasedStateManager, catalog); + final Map pairToCursorBasedStatus = + getCursorBasedSyncStatusForStreams(database, initialLoadStreams.streamsForInitialLoad(), stateManager, quoteString); + final CursorBasedStreams cursorBasedStreams = + new CursorBasedStreams(MySqlInitialReadUtil.identifyStreamsForCursorBased(catalog, initialLoadStreams.streamsForInitialLoad()), + pairToCursorBasedStatus); + + logStreamSyncStatus(initialLoadStreams.streamsForInitialLoad(), "Primary Key"); + logStreamSyncStatus(cursorBasedStreams.streamsForCursorBased(), "Cursor"); + + final MySqlInitialLoadStreamStateManager mySqlInitialLoadStreamStateManager = + new MySqlInitialLoadStreamStateManager(catalog, initialLoadStreams, + initPairToPrimaryKeyInfoMap(database, initialLoadStreams, tableNameToTable, quoteString)); + final MySqlInitialLoadHandler initialLoadHandler = + new MySqlInitialLoadHandler(sourceConfig, database, new MySqlSourceOperations(), getQuoteString(), mySqlInitialLoadStreamStateManager, + namespacePair -> Jsons.jsonNode(pairToCursorBasedStatus.get(convertNameNamespacePairFromV0(namespacePair))), + getTableSizeInfoForStreams(database, catalog.getStreams(), getQuoteString())); + final List> initialLoadIterator = new ArrayList<>(initialLoadHandler.getIncrementalIterators( + new ConfiguredAirbyteCatalog().withStreams(initialLoadStreams.streamsForInitialLoad()), + tableNameToTable, + emittedAt)); + + // Build Cursor based iterator + final List> cursorBasedIterator = + new ArrayList<>(super.getIncrementalIterators(database, + new ConfiguredAirbyteCatalog().withStreams( + cursorBasedStreams.streamsForCursorBased()), + tableNameToTable, + cursorBasedStateManager, emittedAt)); + + return Stream.of(initialLoadIterator, cursorBasedIterator).flatMap(Collection::stream).collect(Collectors.toList()); + } } + + LOGGER.info("using CDC: {}", false); + return super.getIncrementalIterators(database, catalog, tableNameToTable, stateManager, + emittedAt); } @Override @@ -445,6 +504,10 @@ private String toSslJdbcParam(final SslMode sslMode) { return toSslJdbcParamInternal(sslMode); } + private boolean cloudDeploymentMode() { + return AdaptiveSourceRunner.CLOUD_MODE.equalsIgnoreCase(featureFlags.deploymentMode()); + } + @Override protected int getStateEmissionFrequency() { return INTERMEDIATE_STATE_EMISSION_FREQUENCY; @@ -462,13 +525,15 @@ protected static String toSslJdbcParamInternal(final SslMode sslMode) { public JdbcDatabase createDatabase(final JsonNode sourceConfig) throws SQLException { // return super.createDatabase(sourceConfig, this::getConnectionProperties); final JsonNode jdbcConfig = toDatabaseConfig(sourceConfig); + final Map connectionProperties = this.getConnectionProperties(sourceConfig); // Create the data source final DataSource dataSource = DataSourceFactory.create( jdbcConfig.has(JdbcUtils.USERNAME_KEY) ? jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText() : null, jdbcConfig.has(JdbcUtils.PASSWORD_KEY) ? jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, - driverClass, + driverClassName, jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), - this.getConnectionProperties(sourceConfig)); + connectionProperties, + getConnectionTimeout(connectionProperties, driverClassName)); // Record the data source so that it can be closed. dataSources.add(dataSource); @@ -514,7 +579,7 @@ public static Map parseJdbcParameters(final String jdbcPropertie } public static void main(final String[] args) throws Exception { - final Source source = MySqlSource.sshWrappedSource(); + final Source source = MySqlSource.sshWrappedSource(new MySqlSource()); LOGGER.info("starting source: {}", MySqlSource.class); new IntegrationRunner(source).run(args); LOGGER.info("completed source: {}", MySqlSource.class); diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSourceOperations.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSourceOperations.java index 905cb3681d85..53bdf69e4fe5 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSourceOperations.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSourceOperations.java @@ -30,13 +30,13 @@ import static com.mysql.cj.MysqlType.TINYTEXT; import static com.mysql.cj.MysqlType.VARCHAR; import static com.mysql.cj.MysqlType.YEAR; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_SIZE; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_DECIMAL_DIGITS; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_SCHEMA_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_TABLE_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_COLUMN_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_COLUMN_SIZE; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_DECIMAL_DIGITS; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_SCHEMA_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_TABLE_NAME; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.NullNode; @@ -44,8 +44,8 @@ import com.mysql.cj.MysqlType; import com.mysql.cj.jdbc.result.ResultSetMetaData; import com.mysql.cj.result.Field; -import io.airbyte.db.SourceOperations; -import io.airbyte.db.jdbc.AbstractJdbcCompatibleSourceOperations; +import io.airbyte.cdk.db.SourceOperations; +import io.airbyte.cdk.db.jdbc.AbstractJdbcCompatibleSourceOperations; import io.airbyte.protocol.models.JsonSchemaType; import java.sql.PreparedStatement; import java.sql.ResultSet; diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlStreamingQueryConfig.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlStreamingQueryConfig.java index 33920aabdc7f..029cfc2c3d66 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlStreamingQueryConfig.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlStreamingQueryConfig.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.source.mysql; -import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; +import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; @@ -21,7 +21,6 @@ public MySqlStreamingQueryConfig() { @Override public void initialize(final Connection connection, final Statement preparedStatement) throws SQLException { - connection.setAutoCommit(false); preparedStatement.setFetchSize(Integer.MIN_VALUE); LOGGER.info("Set initial fetch size: {} rows", preparedStatement.getFetchSize()); } diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cursor_based/MySqlCursorBasedStateManager.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cursor_based/MySqlCursorBasedStateManager.java new file mode 100644 index 000000000000..9741c9b2b7ca --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/cursor_based/MySqlCursorBasedStateManager.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql.cursor_based; + +import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.source.relationaldb.CursorInfo; +import io.airbyte.cdk.integrations.source.relationaldb.state.StreamStateManager; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.mysql.internal.models.CursorBasedStatus; +import io.airbyte.integrations.source.mysql.internal.models.InternalModels.StateType; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteStreamState; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MySqlCursorBasedStateManager extends StreamStateManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(MySqlCursorBasedStateManager.class); + + public MySqlCursorBasedStateManager(final List airbyteStateMessages, final ConfiguredAirbyteCatalog catalog) { + super(airbyteStateMessages, catalog); + } + + @Override + public AirbyteStateMessage toState(final Optional pair) { + if (pair.isPresent()) { + final Map pairToCursorInfoMap = getPairToCursorInfoMap(); + final Optional cursorInfo = Optional.ofNullable(pairToCursorInfoMap.get(pair.get())); + + if (cursorInfo.isPresent()) { + LOGGER.debug("Generating state message for {}...", pair); + return new AirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + // Temporarily include legacy state for backwards compatibility with the platform + .withStream(generateStreamState(pair.get(), cursorInfo.get())); + } else { + LOGGER.warn("Cursor information could not be located in state for stream {}. Returning a new, empty state message...", pair); + return new AirbyteStateMessage().withType(AirbyteStateType.STREAM).withStream(new AirbyteStreamState()); + } + } else { + LOGGER.warn("Stream not provided. Returning a new, empty state message..."); + return new AirbyteStateMessage().withType(AirbyteStateType.STREAM).withStream(new AirbyteStreamState()); + } + } + + /** + * Generates the stream state for the given stream and cursor information. + * + * @param airbyteStreamNameNamespacePair The stream. + * @param cursorInfo The current cursor. + * @return The {@link AirbyteStreamState} representing the current state of the stream. + */ + private AirbyteStreamState generateStreamState(final AirbyteStreamNameNamespacePair airbyteStreamNameNamespacePair, + final CursorInfo cursorInfo) { + return new AirbyteStreamState() + .withStreamDescriptor( + new StreamDescriptor().withName(airbyteStreamNameNamespacePair.getName()).withNamespace(airbyteStreamNameNamespacePair.getNamespace())) + .withStreamState(Jsons.jsonNode(generateDbStreamState(airbyteStreamNameNamespacePair, cursorInfo))); + } + + private CursorBasedStatus generateDbStreamState(final AirbyteStreamNameNamespacePair airbyteStreamNameNamespacePair, + final CursorInfo cursorInfo) { + final CursorBasedStatus state = new CursorBasedStatus(); + state.setStateType(StateType.CURSOR_BASED); + state.setVersion(2L); + state.setStreamName(airbyteStreamNameNamespacePair.getName()); + state.setStreamNamespace(airbyteStreamNameNamespacePair.getNamespace()); + state.setCursorField(cursorInfo.getCursorField() == null ? Collections.emptyList() : Lists.newArrayList(cursorInfo.getCursorField())); + state.setCursor(cursorInfo.getCursor()); + if (cursorInfo.getCursorRecordCount() > 0L) { + state.setCursorRecordCount(cursorInfo.getCursorRecordCount()); + } + return state; + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/helpers/CdcConfigurationHelper.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/helpers/CdcConfigurationHelper.java index 170d684d7e57..5e2e93ddc579 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/helpers/CdcConfigurationHelper.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/helpers/CdcConfigurationHelper.java @@ -5,9 +5,9 @@ package io.airbyte.integrations.source.mysql.helpers; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.functional.CheckedConsumer; -import io.airbyte.db.jdbc.JdbcDatabase; import java.sql.SQLException; import java.time.ZoneId; import java.util.List; diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlFeatureFlags.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlFeatureFlags.java deleted file mode 100644 index 752406db8c78..000000000000 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlFeatureFlags.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mysql.initialsync; - -import com.fasterxml.jackson.databind.JsonNode; - -// Feature flags to gate new primary key load features. -public class MySqlFeatureFlags { - - public static final String CDC_VIA_PK = "cdc_via_pk"; - private final JsonNode sourceConfig; - - public MySqlFeatureFlags(final JsonNode sourceConfig) { - this.sourceConfig = sourceConfig; - } - - public boolean isCdcInitialSyncViaPkEnabled() { - return getFlagValue(CDC_VIA_PK); - } - - private boolean getFlagValue(final String flag) { - return sourceConfig.has(flag) && sourceConfig.get(flag).asBoolean(); - } - -} diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadGlobalStateManager.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadGlobalStateManager.java index 0b27f40956f5..e810d860e4c8 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadGlobalStateManager.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadGlobalStateManager.java @@ -5,12 +5,12 @@ package io.airbyte.integrations.source.mysql.initialsync; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.source.relationaldb.models.CdcState; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.mysql.initialsync.MySqlInitialReadUtil.InitialLoadStreams; import io.airbyte.integrations.source.mysql.initialsync.MySqlInitialReadUtil.PrimaryKeyInfo; import io.airbyte.integrations.source.mysql.internal.models.PrimaryKeyLoadStatus; -import io.airbyte.integrations.source.relationaldb.models.CdcState; -import io.airbyte.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.AirbyteGlobalState; import io.airbyte.protocol.models.v0.AirbyteStateMessage; @@ -21,7 +21,6 @@ import io.airbyte.protocol.models.v0.SyncMode; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -40,12 +39,12 @@ public class MySqlInitialLoadGlobalStateManager implements MySqlInitialLoadState // have completed the snapshot. private final Set streamsThatHaveCompletedSnapshot; - MySqlInitialLoadGlobalStateManager(final InitialLoadStreams initialLoadStreams, - final Map pairToPrimaryKeyInfo, - final CdcState cdcState, - final ConfiguredAirbyteCatalog catalog) { + public MySqlInitialLoadGlobalStateManager(final InitialLoadStreams initialLoadStreams, + final Map pairToPrimaryKeyInfo, + final CdcState cdcState, + final ConfiguredAirbyteCatalog catalog) { this.cdcState = cdcState; - this.pairToPrimaryKeyLoadStatus = initPairToPrimaryKeyLoadStatusMap(initialLoadStreams.pairToInitialLoadStatus()); + this.pairToPrimaryKeyLoadStatus = MySqlInitialLoadStateManager.initPairToPrimaryKeyLoadStatusMap(initialLoadStreams.pairToInitialLoadStatus()); this.pairToPrimaryKeyInfo = pairToPrimaryKeyInfo; this.streamsThatHaveCompletedSnapshot = initStreamsCompletedSnapshot(initialLoadStreams, catalog); } @@ -63,16 +62,7 @@ private static Set initStreamsCompletedSnapshot( return streamsThatHaveCompletedSnapshot; } - private static Map initPairToPrimaryKeyLoadStatusMap( - final Map pairToPkStatus) { - final Map map = new HashMap<>(); - pairToPkStatus.forEach((pair, pkStatus) -> { - final AirbyteStreamNameNamespacePair updatedPair = new AirbyteStreamNameNamespacePair(pair.getName(), pair.getNamespace()); - map.put(updatedPair, pkStatus); - }); - return map; - } - + @Override public AirbyteStateMessage createIntermediateStateMessage(final AirbyteStreamNameNamespacePair pair, final PrimaryKeyLoadStatus pkLoadStatus) { final List streamStates = new ArrayList<>(); streamsThatHaveCompletedSnapshot.forEach(stream -> { @@ -95,7 +85,9 @@ public void updatePrimaryKeyLoadState(final AirbyteStreamNameNamespacePair pair, pairToPrimaryKeyLoadStatus.put(pair, pkLoadStatus); } - public AirbyteStateMessage createFinalStateMessage(final AirbyteStreamNameNamespacePair pair, final JsonNode streamStateForIncrementalRun) { + @Override + public AirbyteStateMessage createFinalStateMessage(final AirbyteStreamNameNamespacePair pair, + final JsonNode streamStateForIncrementalRun) { streamsThatHaveCompletedSnapshot.add(pair); final List streamStates = new ArrayList<>(); streamsThatHaveCompletedSnapshot.forEach(stream -> { diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadHandler.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadHandler.java index 28541d2bca59..236c65659f82 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadHandler.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadHandler.java @@ -4,17 +4,24 @@ package io.airbyte.integrations.source.mysql.initialsync; +import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_DURATION_PROPERTY; +import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_RECORDS_PROPERTY; + import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; import com.mysql.cj.MysqlType; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants; +import io.airbyte.cdk.integrations.source.relationaldb.DbSourceDiscoverUtil; +import io.airbyte.cdk.integrations.source.relationaldb.TableInfo; +import io.airbyte.cdk.integrations.source.relationaldb.state.SourceStateIterator; +import io.airbyte.cdk.integrations.source.relationaldb.state.SourceStateIteratorManager; import io.airbyte.commons.stream.AirbyteStreamUtils; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.source.mysql.MySqlQueryUtils.TableSizeInfo; +import io.airbyte.integrations.source.mysql.MySqlSourceOperations; import io.airbyte.integrations.source.mysql.internal.models.PrimaryKeyLoadStatus; -import io.airbyte.integrations.source.relationaldb.DbSourceDiscoverUtil; -import io.airbyte.integrations.source.relationaldb.TableInfo; import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.CommonField; import io.airbyte.protocol.models.v0.AirbyteMessage; @@ -33,6 +40,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -43,7 +51,7 @@ public class MySqlInitialLoadHandler { private static final long RECORD_LOGGING_SAMPLE_RATE = 1_000_000; private final JsonNode config; private final JdbcDatabase database; - private final MySqlInitialLoadSourceOperations sourceOperations; + private final MySqlSourceOperations sourceOperations; private final String quoteString; private final MySqlInitialLoadStateManager initialLoadStateManager; private final Function streamStateForIncrementalRunSupplier; @@ -54,7 +62,7 @@ public class MySqlInitialLoadHandler { public MySqlInitialLoadHandler(final JsonNode config, final JdbcDatabase database, - final MySqlInitialLoadSourceOperations sourceOperations, + final MySqlSourceOperations sourceOperations, final String quoteString, final MySqlInitialLoadStateManager initialLoadStateManager, final Function streamStateForIncrementalRunSupplier, @@ -77,6 +85,7 @@ public List> getIncrementalIterators( final AirbyteStream stream = airbyteStream.getStream(); final String streamName = stream.getName(); final String namespace = stream.getNamespace(); + final List primaryKeys = stream.getSourceDefinedPrimaryKey().stream().flatMap(pk -> Stream.of(pk.get(0))).toList(); final AirbyteStreamNameNamespacePair pair = new AirbyteStreamNameNamespacePair(streamName, namespace); final String fullyQualifiedTableName = DbSourceDiscoverUtil.getFullyQualifiedTableName(namespace, streamName); if (!tableNameToTable.containsKey(fullyQualifiedTableName)) { @@ -92,6 +101,15 @@ public List> getIncrementalIterators( .map(CommonField::getName) .filter(CatalogHelpers.getTopLevelFieldNames(airbyteStream)::contains) .collect(Collectors.toList()); + + // This is to handle the case if the user de-selects the PK column + // Necessary to query the data via pk but won't be added to the final record + primaryKeys.forEach(pk -> { + if (!selectedDatabaseFields.contains(pk)) { + selectedDatabaseFields.add(0, pk); + } + }); + final AutoCloseableIterator queryStream = new MySqlInitialLoadRecordIterator(database, sourceOperations, quoteString, initialLoadStateManager, selectedDatabaseFields, pair, calculateChunkSize(tableSizeInfoMap.get(pair), pair), isCompositePrimaryKey(airbyteStream)); @@ -164,14 +182,17 @@ private AutoCloseableIterator augmentWithState(final AutoCloseab : currentPkLoadStatus.getIncrementalState(); final Duration syncCheckpointDuration = - config.get("sync_checkpoint_seconds") != null ? Duration.ofSeconds(config.get("sync_checkpoint_seconds").asLong()) - : MySqlInitialSyncStateIterator.SYNC_CHECKPOINT_DURATION; - final Long syncCheckpointRecords = config.get("sync_checkpoint_records") != null ? config.get("sync_checkpoint_records").asLong() - : MySqlInitialSyncStateIterator.SYNC_CHECKPOINT_RECORDS; + config.get(SYNC_CHECKPOINT_DURATION_PROPERTY) != null ? Duration.ofSeconds(config.get(SYNC_CHECKPOINT_DURATION_PROPERTY).asLong()) + : DebeziumIteratorConstants.SYNC_CHECKPOINT_DURATION; + final Long syncCheckpointRecords = config.get(SYNC_CHECKPOINT_RECORDS_PROPERTY) != null ? config.get(SYNC_CHECKPOINT_RECORDS_PROPERTY).asLong() + : DebeziumIteratorConstants.SYNC_CHECKPOINT_RECORDS; + + final SourceStateIteratorManager processor = + new MySqlInitialSyncStateIteratorManager(pair, initialLoadStateManager, incrementalState, + syncCheckpointDuration, syncCheckpointRecords); return AutoCloseableIterators.transformIterator( - r -> new MySqlInitialSyncStateIterator(r, pair, initialLoadStateManager, incrementalState, - syncCheckpointDuration, syncCheckpointRecords), + r -> new SourceStateIterator<>(r, processor), recordIterator, pair); } diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadRecordIterator.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadRecordIterator.java index 85069edfcfa5..a0803cea4f06 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadRecordIterator.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadRecordIterator.java @@ -4,18 +4,19 @@ package io.airbyte.integrations.source.mysql.initialsync; -import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifier; -import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifier; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; -import autovalue.shaded.com.google.common.collect.AbstractIterator; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.AbstractIterator; import com.mysql.cj.MysqlType; +import io.airbyte.cdk.db.JdbcCompatibleSourceOperations; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.source.mysql.initialsync.MySqlInitialReadUtil.PrimaryKeyInfo; import io.airbyte.integrations.source.mysql.internal.models.PrimaryKeyLoadStatus; -import io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils; import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; import java.sql.Connection; import java.sql.PreparedStatement; @@ -38,12 +39,13 @@ * from table where pk > pk_max_4 order by pk limit 1,800,000. Final query, since there are zero * records processed here. */ +@SuppressWarnings("try") public class MySqlInitialLoadRecordIterator extends AbstractIterator implements AutoCloseableIterator { private static final Logger LOGGER = LoggerFactory.getLogger(MySqlInitialLoadRecordIterator.class); - private final MySqlInitialLoadSourceOperations sourceOperations; + private final JdbcCompatibleSourceOperations sourceOperations; private final String quoteString; private final MySqlInitialLoadStateManager initialLoadStateManager; @@ -59,7 +61,7 @@ public class MySqlInitialLoadRecordIterator extends AbstractIterator MySqlInitialLoadRecordIterator( final JdbcDatabase database, - final MySqlInitialLoadSourceOperations sourceOperations, + final JdbcCompatibleSourceOperations sourceOperations, final String quoteString, final MySqlInitialLoadStateManager initialLoadStateManager, final List columnNames, diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadSourceOperations.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadSourceOperations.java index 69eef78e7515..3b811d3047cc 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadSourceOperations.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadSourceOperations.java @@ -6,8 +6,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes; import io.airbyte.integrations.source.mysql.MySqlCdcConnectorMetadataInjector; import io.airbyte.integrations.source.mysql.MySqlSourceOperations; import java.sql.ResultSet; diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadStateManager.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadStateManager.java index f65cc7b270aa..be5cec573294 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadStateManager.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadStateManager.java @@ -9,6 +9,8 @@ import io.airbyte.integrations.source.mysql.internal.models.PrimaryKeyLoadStatus; import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import java.util.HashMap; +import java.util.Map; public interface MySqlInitialLoadStateManager { @@ -23,7 +25,8 @@ public interface MySqlInitialLoadStateManager { void updatePrimaryKeyLoadState(final AirbyteStreamNameNamespacePair pair, final PrimaryKeyLoadStatus pkLoadStatus); // Returns the final state message for the initial sync. - AirbyteStateMessage createFinalStateMessage(final AirbyteStreamNameNamespacePair pair, final JsonNode streamStateForIncrementalRun); + AirbyteStateMessage createFinalStateMessage(final AirbyteStreamNameNamespacePair pair, + final JsonNode streamStateForIncrementalRun); // Returns the previous state emitted, represented as a {@link PrimaryKeyLoadStatus} associated with // the stream. @@ -33,4 +36,14 @@ public interface MySqlInitialLoadStateManager { // the column name associated with the stream. PrimaryKeyInfo getPrimaryKeyInfo(final AirbyteStreamNameNamespacePair pair); + static Map initPairToPrimaryKeyLoadStatusMap( + final Map pairToPkStatus) { + final Map map = new HashMap<>(); + pairToPkStatus.forEach((pair, pkStatus) -> { + final AirbyteStreamNameNamespacePair updatedPair = new AirbyteStreamNameNamespacePair(pair.getName(), pair.getNamespace()); + map.put(updatedPair, pkStatus); + }); + return map; + } + } diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadStreamStateManager.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadStreamStateManager.java new file mode 100644 index 000000000000..88859e2ecb84 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialLoadStreamStateManager.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql.initialsync; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.mysql.initialsync.MySqlInitialReadUtil.InitialLoadStreams; +import io.airbyte.integrations.source.mysql.initialsync.MySqlInitialReadUtil.PrimaryKeyInfo; +import io.airbyte.integrations.source.mysql.internal.models.PrimaryKeyLoadStatus; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStreamState; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.StreamDescriptor; +import java.util.Map; +import java.util.Objects; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This state manager extends the StreamStateManager to enable writing the state_type and version + * keys to the stream state when they're going through the iterator Once we have verified that + * expanding StreamStateManager itself to include this functionality, this class will be removed + */ +public class MySqlInitialLoadStreamStateManager implements MySqlInitialLoadStateManager { + + private final Map pairToPrimaryKeyLoadStatus; + + // Map of pair to the primary key info (field name & data type) associated with it. + private final Map pairToPrimaryKeyInfo; + + private static final Logger LOGGER = LoggerFactory.getLogger(MySqlInitialLoadStreamStateManager.class); + + public MySqlInitialLoadStreamStateManager(final ConfiguredAirbyteCatalog catalog, + final InitialLoadStreams initialLoadStreams, + final Map pairToPrimaryKeyInfo) { + this.pairToPrimaryKeyInfo = pairToPrimaryKeyInfo; + this.pairToPrimaryKeyLoadStatus = MySqlInitialLoadStateManager.initPairToPrimaryKeyLoadStatusMap(initialLoadStreams.pairToInitialLoadStatus()); + } + + @Override + public void updatePrimaryKeyLoadState(final io.airbyte.protocol.models.AirbyteStreamNameNamespacePair pair, + final PrimaryKeyLoadStatus pkLoadStatus) { + pairToPrimaryKeyLoadStatus.put(pair, pkLoadStatus); + } + + @Override + public AirbyteStateMessage createFinalStateMessage(final io.airbyte.protocol.models.AirbyteStreamNameNamespacePair pair, + final JsonNode streamStateForIncrementalRun) { + + return new AirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream(getAirbyteStreamState(pair, (streamStateForIncrementalRun))); + } + + @Override + public PrimaryKeyInfo getPrimaryKeyInfo(final io.airbyte.protocol.models.AirbyteStreamNameNamespacePair pair) { + return pairToPrimaryKeyInfo.get(pair); + } + + @Override + public PrimaryKeyLoadStatus getPrimaryKeyLoadStatus(final io.airbyte.protocol.models.AirbyteStreamNameNamespacePair pair) { + return pairToPrimaryKeyLoadStatus.get(pair); + } + + @Override + public AirbyteStateMessage createIntermediateStateMessage(final io.airbyte.protocol.models.AirbyteStreamNameNamespacePair pair, + final PrimaryKeyLoadStatus pkLoadStatus) { + return new AirbyteStateMessage() + .withType(AirbyteStateType.STREAM) + .withStream(getAirbyteStreamState(pair, Jsons.jsonNode(pkLoadStatus))); + } + + private AirbyteStreamState getAirbyteStreamState(final io.airbyte.protocol.models.AirbyteStreamNameNamespacePair pair, final JsonNode stateData) { + LOGGER.info("STATE DATA FOR {}: {}", pair.getNamespace().concat("_").concat(pair.getName()), stateData); + assert Objects.nonNull(pair.getName()); + assert Objects.nonNull(pair.getNamespace()); + + return new AirbyteStreamState() + .withStreamDescriptor( + new StreamDescriptor().withName(pair.getName()).withNamespace(pair.getNamespace())) + .withStreamState(stateData); + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialReadUtil.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialReadUtil.java index de3d7376e300..8ca08abb0ffd 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialReadUtil.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialReadUtil.java @@ -4,41 +4,45 @@ package io.airbyte.integrations.source.mysql.initialsync; -import static io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MYSQL_CDC_OFFSET; +import static io.airbyte.cdk.integrations.debezium.internals.mysql.MysqlCdcStateConstants.MYSQL_CDC_OFFSET; import static io.airbyte.integrations.source.mysql.MySqlQueryUtils.getTableSizeInfoForStreams; +import static io.airbyte.integrations.source.mysql.MySqlQueryUtils.prettyPrintConfiguredAirbyteStreamList; import static io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadGlobalStateManager.STATE_TYPE_KEY; import static io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadStateManager.PRIMARY_KEY_STATE_TYPE; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Sets; import com.mysql.cj.MysqlType; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.cdk.integrations.debezium.AirbyteDebeziumHandler; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager; +import io.airbyte.cdk.integrations.debezium.internals.RecordWaitTimeUtil; +import io.airbyte.cdk.integrations.debezium.internals.mysql.MySqlCdcPosition; +import io.airbyte.cdk.integrations.debezium.internals.mysql.MySqlCdcTargetPosition; +import io.airbyte.cdk.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil; +import io.airbyte.cdk.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes; +import io.airbyte.cdk.integrations.source.relationaldb.CdcStateManager; +import io.airbyte.cdk.integrations.source.relationaldb.DbSourceDiscoverUtil; +import io.airbyte.cdk.integrations.source.relationaldb.TableInfo; +import io.airbyte.cdk.integrations.source.relationaldb.models.CdcState; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.AirbyteTraceMessageUtility; -import io.airbyte.integrations.debezium.AirbyteDebeziumHandler; -import io.airbyte.integrations.debezium.internals.FirstRecordWaitTimeUtil; -import io.airbyte.integrations.debezium.internals.mysql.MySqlCdcPosition; -import io.airbyte.integrations.debezium.internals.mysql.MySqlCdcTargetPosition; -import io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil; -import io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MysqlDebeziumStateAttributes; import io.airbyte.integrations.source.mysql.MySqlCdcConnectorMetadataInjector; import io.airbyte.integrations.source.mysql.MySqlCdcProperties; import io.airbyte.integrations.source.mysql.MySqlCdcSavedInfoFetcher; import io.airbyte.integrations.source.mysql.MySqlCdcStateHandler; import io.airbyte.integrations.source.mysql.MySqlQueryUtils; import io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadSourceOperations.CdcMetadataInjector; +import io.airbyte.integrations.source.mysql.internal.models.CursorBasedStatus; import io.airbyte.integrations.source.mysql.internal.models.PrimaryKeyLoadStatus; -import io.airbyte.integrations.source.relationaldb.CdcStateManager; -import io.airbyte.integrations.source.relationaldb.DbSourceDiscoverUtil; -import io.airbyte.integrations.source.relationaldb.TableInfo; -import io.airbyte.integrations.source.relationaldb.models.CdcState; -import io.airbyte.integrations.source.relationaldb.state.StateManager; import io.airbyte.protocol.models.CommonField; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteStreamState; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.StreamDescriptor; @@ -80,7 +84,8 @@ public static List> getCdcReadIterators(fi final Instant emittedAt, final String quoteString) { final JsonNode sourceConfig = database.getSourceConfig(); - final Duration firstRecordWaitTime = FirstRecordWaitTimeUtil.getFirstRecordWaitTime(sourceConfig); + final Duration firstRecordWaitTime = RecordWaitTimeUtil.getFirstRecordWaitTime(sourceConfig); + final Duration subsequentRecordWaitTime = RecordWaitTimeUtil.getSubsequentRecordWaitTime(sourceConfig); LOGGER.info("First record waiting time: {} seconds", firstRecordWaitTime.getSeconds()); // Determine the streams that need to be loaded via primary key sync. final List> initialLoadIterator = new ArrayList<>(); @@ -104,10 +109,10 @@ public static List> getCdcReadIterators(fi savedOffset.isPresent() && mySqlDebeziumStateUtil.savedOffsetStillPresentOnServer(database, savedOffset.get()); if (!savedOffsetStillPresentOnServer) { - LOGGER.warn("Saved offset no longer present on the server, Airbtye is going to trigger a sync from scratch"); + LOGGER.warn("Saved offset no longer present on the server, Airbyte is going to trigger a sync from scratch"); } - final InitialLoadStreams initialLoadStreams = streamsForInitialPrimaryKeyLoad(stateManager.getCdcStateManager(), catalog, + final InitialLoadStreams initialLoadStreams = cdcStreamsForInitialPrimaryKeyLoad(stateManager.getCdcStateManager(), catalog, savedOffsetStillPresentOnServer); final CdcState stateToBeUsed = (!savedOffsetStillPresentOnServer || (stateManager.getCdcStateManager().getCdcState() == null @@ -134,7 +139,7 @@ public static List> getCdcReadIterators(fi quoteString, initialLoadStateManager, namespacePair -> Jsons.emptyObject(), - getTableSizeInfoForStreams(database, catalog.getStreams(), quoteString)); + getTableSizeInfoForStreams(database, initialLoadStreams.streamsForInitialLoad(), quoteString)); initialLoadIterator.addAll(initialLoadHandler.getIncrementalIterators( new ConfiguredAirbyteCatalog().withStreams(initialLoadStreams.streamsForInitialLoad()), @@ -145,14 +150,20 @@ public static List> getCdcReadIterators(fi } // Build the incremental CDC iterators. - final AirbyteDebeziumHandler handler = - new AirbyteDebeziumHandler<>(sourceConfig, MySqlCdcTargetPosition.targetPosition(database), true, firstRecordWaitTime, OptionalInt.empty()); + final AirbyteDebeziumHandler handler = new AirbyteDebeziumHandler<>( + sourceConfig, + MySqlCdcTargetPosition.targetPosition(database), + true, + firstRecordWaitTime, + subsequentRecordWaitTime, + OptionalInt.empty()); final Supplier> incrementalIteratorSupplier = () -> handler.getIncrementalIterators(catalog, new MySqlCdcSavedInfoFetcher(stateToBeUsed), new MySqlCdcStateHandler(stateManager), metadataInjector, MySqlCdcProperties.getDebeziumProperties(database), + DebeziumPropertiesManager.DebeziumConnectorType.RELATIONALDB, emittedAt, false); @@ -170,12 +181,12 @@ public static List> getCdcReadIterators(fi } /** - * Determines the streams to sync for initial primary key load. These include streams that are (i) - * currently in primary key load (ii) newly added incremental streams. + * CDC specific: Determines the streams to sync for initial primary key load. These include streams + * that are (i) currently in primary key load (ii) newly added incremental streams. */ - public static InitialLoadStreams streamsForInitialPrimaryKeyLoad(final CdcStateManager stateManager, - final ConfiguredAirbyteCatalog fullCatalog, - final boolean savedOffsetStillPresentOnServer) { + public static InitialLoadStreams cdcStreamsForInitialPrimaryKeyLoad(final CdcStateManager stateManager, + final ConfiguredAirbyteCatalog fullCatalog, + final boolean savedOffsetStillPresentOnServer) { if (!savedOffsetStillPresentOnServer) { return new InitialLoadStreams( fullCatalog.getStreams() @@ -222,23 +233,95 @@ public static InitialLoadStreams streamsForInitialPrimaryKeyLoad(final CdcStateM return new InitialLoadStreams(streamsForPkSync, pairToInitialLoadStatus); } - private static List identifyStreamsToSnapshot(final ConfiguredAirbyteCatalog catalog, - final Set alreadySyncedStreams) { + /** + * Determines the streams to sync for initial primary key load. These include streams that are (i) + * currently in primary key load (ii) newly added incremental streams. + */ + public static InitialLoadStreams streamsForInitialPrimaryKeyLoad(final StateManager stateManager, + final ConfiguredAirbyteCatalog fullCatalog) { + + final List rawStateMessages = stateManager.getRawStateMessages(); + final Set streamsStillInPkSync = new HashSet<>(); + final Set alreadySeenStreamPairs = new HashSet<>(); + + // Build a map of stream <-> initial load status for streams that currently have an initial primary + // key load in progress. + final Map pairToInitialLoadStatus = new HashMap<>(); + + if (rawStateMessages != null) { + rawStateMessages.forEach(stateMessage -> { + final AirbyteStreamState stream = stateMessage.getStream(); + final JsonNode streamState = stream.getStreamState(); + final StreamDescriptor streamDescriptor = stateMessage.getStream().getStreamDescriptor(); + if (streamState == null || streamDescriptor == null) { + return; + } + + final AirbyteStreamNameNamespacePair pair = new AirbyteStreamNameNamespacePair(streamDescriptor.getName(), + streamDescriptor.getNamespace()); + + // Build a map of stream <-> initial load status for streams that currently have an initial primary + // key load in progress. + + if (streamState.has(STATE_TYPE_KEY)) { + if (streamState.get(STATE_TYPE_KEY).asText().equalsIgnoreCase(PRIMARY_KEY_STATE_TYPE)) { + final PrimaryKeyLoadStatus primaryKeyLoadStatus = Jsons.object(streamState, PrimaryKeyLoadStatus.class); + pairToInitialLoadStatus.put(pair, primaryKeyLoadStatus); + streamsStillInPkSync.add(pair); + } + } + alreadySeenStreamPairs.add(new AirbyteStreamNameNamespacePair(streamDescriptor.getName(), streamDescriptor.getNamespace())); + }); + } + final List streamsForPkSync = new ArrayList<>(); + fullCatalog.getStreams().stream() + .filter(stream -> streamsStillInPkSync.contains(AirbyteStreamNameNamespacePair.fromAirbyteStream(stream.getStream()))) + .map(Jsons::clone) + .forEach(streamsForPkSync::add); + + final List newlyAddedStreams = identifyStreamsToSnapshot(fullCatalog, + Collections.unmodifiableSet(alreadySeenStreamPairs)); + streamsForPkSync.addAll(newlyAddedStreams); + return new InitialLoadStreams(streamsForPkSync.stream().filter(MySqlInitialReadUtil::streamHasPrimaryKey).collect(Collectors.toList()), + pairToInitialLoadStatus); + } + + private static boolean streamHasPrimaryKey(final ConfiguredAirbyteStream stream) { + return stream.getStream().getSourceDefinedPrimaryKey().size() > 0; + } + + public static List identifyStreamsToSnapshot(final ConfiguredAirbyteCatalog catalog, + final Set alreadySyncedStreams) { final Set allStreams = AirbyteStreamNameNamespacePair.fromConfiguredCatalog(catalog); final Set newlyAddedStreams = new HashSet<>(Sets.difference(allStreams, alreadySyncedStreams)); return catalog.getStreams().stream() .filter(c -> c.getSyncMode() == SyncMode.INCREMENTAL) - .filter(stream -> newlyAddedStreams.contains(AirbyteStreamNameNamespacePair.fromAirbyteStream(stream.getStream()))).map(Jsons::clone) + .filter(stream -> newlyAddedStreams.contains(AirbyteStreamNameNamespacePair.fromAirbyteStream(stream.getStream()))) + .map(Jsons::clone) + .collect(Collectors.toList()); + } + + public static List identifyStreamsForCursorBased(final ConfiguredAirbyteCatalog catalog, + final List streamsForInitialLoad) { + + final Set initialLoadStreamsNamespacePairs = + streamsForInitialLoad.stream().map(stream -> AirbyteStreamNameNamespacePair.fromAirbyteStream(stream.getStream())) + .collect( + Collectors.toSet()); + return catalog.getStreams().stream() + .filter(c -> c.getSyncMode() == SyncMode.INCREMENTAL) + .filter(stream -> !initialLoadStreamsNamespacePairs.contains(AirbyteStreamNameNamespacePair.fromAirbyteStream(stream.getStream()))) + .map(Jsons::clone) .collect(Collectors.toList()); } // Build a map of stream <-> primary key info (primary key field name + datatype) for all streams // currently undergoing initial primary key syncs. - private static Map initPairToPrimaryKeyInfoMap( - final JdbcDatabase database, - final InitialLoadStreams initialLoadStreams, - final Map>> tableNameToTable, - final String quoteString) { + public static Map initPairToPrimaryKeyInfoMap( + final JdbcDatabase database, + final InitialLoadStreams initialLoadStreams, + final Map>> tableNameToTable, + final String quoteString) { final Map pairToPkInfoMap = new HashMap<>(); // For every stream that is in primary initial key sync, we want to maintain information about the // current primary key info associated with the @@ -276,15 +359,20 @@ private static PrimaryKeyInfo getPrimaryKeyInfo(final JdbcDatabase database, return new PrimaryKeyInfo(pkFieldName, pkFieldType, pkMaxValue); } - public static String prettyPrintConfiguredAirbyteStreamList(final List streamList) { - return streamList.stream().map(s -> "%s.%s".formatted(s.getStream().getNamespace(), s.getStream().getName())).collect(Collectors.joining(", ")); - } - public record InitialLoadStreams(List streamsForInitialLoad, Map pairToInitialLoadStatus) { } + public record CursorBasedStreams(List streamsForCursorBased, + Map pairToCursorBasedStatus) { + + } + public record PrimaryKeyInfo(String pkFieldName, MysqlType fieldType, String pkMaxValue) {} + public static AirbyteStreamNameNamespacePair convertNameNamespacePairFromV0(final io.airbyte.protocol.models.AirbyteStreamNameNamespacePair v1NameNamespacePair) { + return new AirbyteStreamNameNamespacePair(v1NameNamespacePair.getName(), v1NameNamespacePair.getNamespace()); + } + } diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialSyncStateIterator.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialSyncStateIterator.java deleted file mode 100644 index 25cc9f72329e..000000000000 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialSyncStateIterator.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mysql.initialsync; - -import static io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadStateManager.MYSQL_STATUS_VERSION; - -import autovalue.shaded.com.google.common.collect.AbstractIterator; -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.source.mysql.internal.models.InternalModels.StateType; -import io.airbyte.integrations.source.mysql.internal.models.PrimaryKeyLoadStatus; -import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteMessage.Type; -import io.airbyte.protocol.models.v0.AirbyteStateMessage; -import java.time.Duration; -import java.time.Instant; -import java.time.OffsetDateTime; -import java.util.Iterator; -import java.util.Objects; -import javax.annotation.CheckForNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class MySqlInitialSyncStateIterator extends AbstractIterator implements Iterator { - - private static final Logger LOGGER = LoggerFactory.getLogger(MySqlInitialSyncStateIterator.class); - public static final Duration SYNC_CHECKPOINT_DURATION = Duration.ofMinutes(15); - public static final Integer SYNC_CHECKPOINT_RECORDS = 100_000; - - private final Iterator messageIterator; - private final AirbyteStreamNameNamespacePair pair; - private boolean hasEmittedFinalState = false; - private PrimaryKeyLoadStatus pkStatus; - private final JsonNode streamStateForIncrementalRun; - private final MySqlInitialLoadStateManager stateManager; - private long recordCount = 0L; - private Instant lastCheckpoint = Instant.now(); - private final Duration syncCheckpointDuration; - private final Long syncCheckpointRecords; - private final String pkFieldName; - - public MySqlInitialSyncStateIterator(final Iterator messageIterator, - final AirbyteStreamNameNamespacePair pair, - final MySqlInitialLoadStateManager stateManager, - final JsonNode streamStateForIncrementalRun, - final Duration checkpointDuration, - final Long checkpointRecords) { - this.messageIterator = messageIterator; - this.pair = pair; - this.stateManager = stateManager; - this.streamStateForIncrementalRun = streamStateForIncrementalRun; - this.syncCheckpointDuration = checkpointDuration; - this.syncCheckpointRecords = checkpointRecords; - this.pkFieldName = stateManager.getPrimaryKeyInfo(pair).pkFieldName(); - this.pkStatus = stateManager.getPrimaryKeyLoadStatus(pair); - } - - @CheckForNull - @Override - protected AirbyteMessage computeNext() { - if (messageIterator.hasNext()) { - if ((recordCount >= syncCheckpointRecords || Duration.between(lastCheckpoint, OffsetDateTime.now()).compareTo(syncCheckpointDuration) > 0) - && Objects.nonNull(pkStatus)) { - LOGGER.info("Emitting initial sync pk state for stream {}, state is {}", pair, pkStatus); - recordCount = 0L; - lastCheckpoint = Instant.now(); - return new AirbyteMessage() - .withType(Type.STATE) - .withState(stateManager.createIntermediateStateMessage(pair, pkStatus)); - } - // Use try-catch to catch Exception that could occur when connection to the database fails - try { - final AirbyteMessage message = messageIterator.next(); - if (Objects.nonNull(message)) { - final String lastPk = message.getRecord().getData().get(pkFieldName).asText(); - pkStatus = new PrimaryKeyLoadStatus() - .withVersion(MYSQL_STATUS_VERSION) - .withStateType(StateType.PRIMARY_KEY) - .withPkName(pkFieldName) - .withPkVal(lastPk) - .withIncrementalState(streamStateForIncrementalRun); - stateManager.updatePrimaryKeyLoadState(pair, pkStatus); - } - recordCount++; - return message; - } catch (final Exception e) { - throw new RuntimeException(e); - } - } else if (!hasEmittedFinalState) { - hasEmittedFinalState = true; - final AirbyteStateMessage finalStateMessage = stateManager.createFinalStateMessage(pair, streamStateForIncrementalRun); - LOGGER.info("Finished initial sync of stream {}, Emitting final state, state is {}", pair, finalStateMessage); - return new AirbyteMessage() - .withType(Type.STATE) - .withState(finalStateMessage); - } else { - return endOfData(); - } - } - -} diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialSyncStateIteratorManager.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialSyncStateIteratorManager.java new file mode 100644 index 000000000000..f7722e1844da --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/initialsync/MySqlInitialSyncStateIteratorManager.java @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql.initialsync; + +import static io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadStateManager.MYSQL_STATUS_VERSION; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.source.relationaldb.state.SourceStateIteratorManager; +import io.airbyte.integrations.source.mysql.internal.models.InternalModels.StateType; +import io.airbyte.integrations.source.mysql.internal.models.PrimaryKeyLoadStatus; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import java.time.Duration; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.util.Objects; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MySqlInitialSyncStateIteratorManager implements SourceStateIteratorManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(MySqlInitialSyncStateIteratorManager.class); + + private final AirbyteStreamNameNamespacePair pair; + private PrimaryKeyLoadStatus pkStatus; + private final JsonNode streamStateForIncrementalRun; + private final MySqlInitialLoadStateManager stateManager; + private final Duration syncCheckpointDuration; + private final Long syncCheckpointRecords; + private final String pkFieldName; + + public MySqlInitialSyncStateIteratorManager( + final AirbyteStreamNameNamespacePair pair, + final MySqlInitialLoadStateManager stateManager, + final JsonNode streamStateForIncrementalRun, + final Duration checkpointDuration, + final Long checkpointRecords) { + this.pair = pair; + this.stateManager = stateManager; + this.streamStateForIncrementalRun = streamStateForIncrementalRun; + this.syncCheckpointDuration = checkpointDuration; + this.syncCheckpointRecords = checkpointRecords; + this.pkFieldName = stateManager.getPrimaryKeyInfo(pair).pkFieldName(); + this.pkStatus = stateManager.getPrimaryKeyLoadStatus(pair); + } + + @Override + public AirbyteStateMessage generateStateMessageAtCheckpoint() { + LOGGER.info("Emitting initial sync pk state for stream {}, state is {}", pair, pkStatus); + return stateManager.createIntermediateStateMessage(pair, pkStatus); + } + + @Override + public AirbyteMessage processRecordMessage(final AirbyteMessage message) { + if (Objects.nonNull(message)) { + final String lastPk = message.getRecord().getData().get(pkFieldName).asText(); + pkStatus = new PrimaryKeyLoadStatus() + .withVersion(MYSQL_STATUS_VERSION) + .withStateType(StateType.PRIMARY_KEY) + .withPkName(pkFieldName) + .withPkVal(lastPk) + .withIncrementalState(streamStateForIncrementalRun); + stateManager.updatePrimaryKeyLoadState(pair, pkStatus); + } + return message; + } + + @Override + public AirbyteStateMessage createFinalStateMessage() { + final AirbyteStateMessage finalStateMessage = stateManager.createFinalStateMessage(pair, streamStateForIncrementalRun); + LOGGER.info("Finished initial sync of stream {}, Emitting final state, state is {}", pair, finalStateMessage); + return finalStateMessage; + } + + @Override + public boolean shouldEmitStateMessage(long recordCount, Instant lastCheckpoint) { + return (recordCount >= syncCheckpointRecords || Duration.between(lastCheckpoint, OffsetDateTime.now()).compareTo(syncCheckpointDuration) > 0) + && Objects.nonNull(pkStatus); + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/main/resources/internal_models/internal_models.yaml b/airbyte-integrations/connectors/source-mysql/src/main/resources/internal_models/internal_models.yaml index 748d2a8f54c1..d7c998e4c714 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/resources/internal_models/internal_models.yaml +++ b/airbyte-integrations/connectors/source-mysql/src/main/resources/internal_models/internal_models.yaml @@ -21,7 +21,7 @@ definitions: type: object extends: type: object - existingJavaType: "io.airbyte.integrations.source.relationaldb.models.DbStreamState" + existingJavaType: "io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState" properties: state_type: "$ref": "#/definitions/StateType" diff --git a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json index 4fada5c46e56..841fa1f3bdba 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json @@ -183,6 +183,7 @@ "description": "Configures how data is extracted from the database.", "order": 8, "default": "CDC", + "display_type": "radio", "oneOf": [ { "title": "Read Changes using Binary Log (CDC)", diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractMySqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractMySqlSourceDatatypeTest.java index 51da2ed05284..689e6a531139 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractMySqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractMySqlSourceDatatypeTest.java @@ -4,11 +4,11 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; -import com.fasterxml.jackson.databind.JsonNode; import com.mysql.cj.MysqlType; -import io.airbyte.db.Database; -import io.airbyte.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; -import io.airbyte.integrations.standardtest.source.TestDataHolder; +import io.airbyte.cdk.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDataHolder; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase; import io.airbyte.protocol.models.JsonSchemaType; import java.io.File; import java.io.IOException; @@ -21,31 +21,26 @@ import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testcontainers.containers.MySQLContainer; public abstract class AbstractMySqlSourceDatatypeTest extends AbstractSourceDatabaseTypeTest { protected static final Logger LOGGER = LoggerFactory.getLogger(AbstractMySqlSourceDatatypeTest.class); - protected MySQLContainer container; - protected JsonNode config; + protected MySQLTestDatabase testdb; @Override - protected JsonNode getConfig() { - return config; + protected String getNameSpace() { + return testdb.getDatabaseName(); } @Override - protected String getImageName() { - return "airbyte/source-mysql:dev"; + protected void tearDown(final TestDestinationEnv testEnv) { + testdb.close(); } @Override - protected abstract Database setupDatabase() throws Exception; - - @Override - protected String getNameSpace() { - return container.getDatabaseName(); + protected String getImageName() { + return "airbyte/source-mysql:dev"; } @Override @@ -232,22 +227,13 @@ protected void initTests() { .addExpectedValues("0.188", null) .build()); - addDataTypeTestData( - TestDataHolder.builder() - .sourceType("decimal") - .airbyteType(JsonSchemaType.NUMBER) - .fullSourceDataType("decimal(19,2)") - .addInsertValues("1700000.01") - .addExpectedValues("1700000.01") - .build()); - addDataTypeTestData( TestDataHolder.builder() .sourceType("decimal") .airbyteType(JsonSchemaType.INTEGER) .fullSourceDataType("decimal(32,0)") - .addInsertValues("1700000.01") - .addExpectedValues("1700000") + .addInsertValues("1700000.01", "123") + .addExpectedValues("1700000", "123") .build()); for (final String type : Set.of("date", "date not null default '0000-00-00'")) { @@ -269,6 +255,14 @@ protected void initTests() { .addExpectedValues((String) null) .build()); + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("date") + .airbyteType(JsonSchemaType.STRING_DATE) + .addInsertValues("0000-00-00") + .addExpectedValues((String) null) + .build()); + for (final String fullSourceType : Set.of("datetime", "datetime not null default now()")) { addDataTypeTestData( TestDataHolder.builder() @@ -451,6 +445,7 @@ protected void initTests() { .addExpectedValues(null, "xs,s", "m,xl") .build()); + addDecimalValuesTest(); } protected void addJsonDataTypeTest() { @@ -492,4 +487,15 @@ private String getFileDataInBase64() { return null; } + protected void addDecimalValuesTest() { + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("decimal") + .airbyteType(JsonSchemaType.NUMBER) + .fullSourceDataType("decimal(19,2)") + .addInsertValues("1700000.01", "'123'") + .addExpectedValues("1700000.01", "123.0") + .build()); + } + } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractMySqlSslCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractMySqlSslCertificateSourceAcceptanceTest.java deleted file mode 100644 index 202512c0f291..000000000000 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractMySqlSslCertificateSourceAcceptanceTest.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.io.airbyte.integration_tests.sources; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.MySqlUtils; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import java.io.IOException; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.testcontainers.containers.MySQLContainer; - -public abstract class AbstractMySqlSslCertificateSourceAcceptanceTest extends MySqlSourceAcceptanceTest { - - protected static MySqlUtils.Certificate certs; - protected static final String PASSWORD = "Passw0rd"; - - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - - container = new MySQLContainer<>("mysql:8.0"); - container.start(); - addTestData(container); - certs = getCertificates(); - - var sslMode = getSslConfig(); - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "STANDARD") - .build()); - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put(JdbcUtils.SSL_KEY, true) - .put(JdbcUtils.SSL_MODE_KEY, sslMode) - .put("replication_method", replicationMethod) - .build()); - } - - public abstract MySqlUtils.Certificate getCertificates() throws IOException, InterruptedException; - - public abstract ImmutableMap getSslConfig(); - - private void addTestData(MySQLContainer container) throws Exception { - try (final DSLContext dslContext = DSLContextFactory.create( - container.getUsername(), - container.getPassword(), - DatabaseDriver.MYSQL.getDriverClassName(), - String.format("jdbc:mysql://%s:%s/%s", - container.getHost(), - container.getFirstMappedPort(), - container.getDatabaseName()), - SQLDialect.MYSQL)) { - final Database database = new Database(dslContext); - - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch( - "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch( - "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - return null; - }); - } - } - -} diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshMySqlSourceAcceptanceTest.java index b885ae872c09..61b79b8d48df 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshMySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshMySqlSourceAcceptanceTest.java @@ -6,12 +6,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.ssh.SshHelpers; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.CatalogHelpers; @@ -28,19 +28,17 @@ public abstract class AbstractSshMySqlSourceAcceptanceTest extends SourceAccepta private static final String STREAM_NAME = "id_and_name"; private static final String STREAM_NAME2 = "starships"; - protected static JsonNode config; + private JsonNode config; public abstract Path getConfigFilePath(); @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { + protected void setupEnvironment(final TestDestinationEnv environment) { config = Jsons.deserialize(IOs.readFile(getConfigFilePath())); } @Override - protected void tearDown(final TestDestinationEnv testEnv) { - - } + protected void tearDown(final TestDestinationEnv testEnv) {} @Override protected String getImageName() { @@ -85,9 +83,4 @@ protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); } - @Override - protected boolean supportsPerStream() { - return true; - } - } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCMySqlDatatypeAccuracyTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCMySqlDatatypeAccuracyTest.java index 5d79b122a472..fd7ea961688a 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCMySqlDatatypeAccuracyTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCMySqlDatatypeAccuracyTest.java @@ -4,97 +4,26 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; -import static io.airbyte.integrations.io.airbyte.integration_tests.sources.utils.TestConstants.INITIAL_CDC_WAITING_SECONDS; - import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.testcontainers.containers.MySQLContainer; +import io.airbyte.cdk.db.Database; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; public class CDCMySqlDatatypeAccuracyTest extends MySqlDatatypeAccuracyTest { - private DSLContext dslContext; - @Override - protected void tearDown(final TestDestinationEnv testEnv) { - dslContext.close(); - super.tearDown(testEnv); + protected JsonNode getConfig() { + return testdb.integrationTestConfigBuilder() + .withoutSsl() + .withCdcReplication() + .with("snapshot_mode", "initial_only") + .build(); } @Override - protected Database setupDatabase() throws Exception { - container = new MySQLContainer<>("mysql:8.0"); - container.start(); - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "CDC") - .put("initial_waiting_seconds", INITIAL_CDC_WAITING_SECONDS) - .build()); - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put("replication_method", replicationMethod) - .put("snapshot_mode", "initial_only") - .put("is_test", true) - .build()); - - dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.MYSQL.getDriverClassName(), - String.format(DatabaseDriver.MYSQL.getUrlFormatString(), - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asInt(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.MYSQL); - final Database database = new Database(dslContext); - - // It disable strict mode in the DB and allows to insert specific values. - // For example, it's possible to insert date with zero values "2021-00-00" - database.query(ctx -> ctx.fetch("SET @@sql_mode=''")); - - revokeAllPermissions(); - grantCorrectPermissions(); - - return database; - } - - private void revokeAllPermissions() { - executeQuery("REVOKE ALL PRIVILEGES, GRANT OPTION FROM " + container.getUsername() + "@'%';"); - } - - private void grantCorrectPermissions() { - executeQuery( - "GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO " - + container.getUsername() + "@'%';"); - } - - private void executeQuery(final String query) { - try (final DSLContext dslContext = DSLContextFactory.create( - "root", - "test", - DatabaseDriver.MYSQL.getDriverClassName(), - String.format(DatabaseDriver.MYSQL.getUrlFormatString(), - container.getHost(), - container.getFirstMappedPort(), - container.getDatabaseName()), - SQLDialect.MYSQL)) { - final Database database = new Database(dslContext); - database.query( - ctx -> ctx - .execute(query)); - } catch (final Exception e) { - throw new RuntimeException(e); - } + protected Database setupDatabase() { + testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8).withoutStrictMode().withCdcPermissions(); + return testdb.getDatabase(); } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcBinlogsMySqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcBinlogsMySqlSourceDatatypeTest.java index 8ef81c8d27b6..82ab112d7e15 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcBinlogsMySqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcBinlogsMySqlSourceDatatypeTest.java @@ -4,36 +4,36 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; -import static io.airbyte.integrations.io.airbyte.integration_tests.sources.utils.TestConstants.INITIAL_CDC_WAITING_SECONDS; - import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.integrations.standardtest.source.TestDataHolder; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.TestDataHolder; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.util.List; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.testcontainers.containers.MySQLContainer; public class CdcBinlogsMySqlSourceDatatypeTest extends AbstractMySqlSourceDatatypeTest { - private DSLContext dslContext; private JsonNode stateAfterFirstSync; @Override - protected void tearDown(final TestDestinationEnv testEnv) { - dslContext.close(); - container.close(); + protected JsonNode getConfig() { + return testdb.integrationTestConfigBuilder() + .withoutSsl() + .withCdcReplication() + .build(); + } + + @Override + protected Database setupDatabase() { + testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8).withoutStrictMode().withCdcPermissions(); + return testdb.getDatabase(); } @Override @@ -46,11 +46,10 @@ protected List runRead(final ConfiguredAirbyteCatalog configured @Override protected void postSetup() throws Exception { - final Database database = setupDatabase(); - initTests(); + final var database = testdb.getDatabase(); for (final TestDataHolder test : testDataHolders) { database.query(ctx -> { - ctx.fetch(test.getCreateSqlQuery()); + ctx.execute("TRUNCATE TABLE " + test.getNameWithTestPrefix() + ";"); return null; }); } @@ -60,14 +59,8 @@ protected void postSetup() throws Exception { catalog.getStreams().add(dummyTableWithData); final List allMessages = super.runRead(catalog); - if (allMessages.size() != 2) { - throw new RuntimeException("First sync should only generate 2 records"); - } final List stateAfterFirstBatch = extractStateMessages(allMessages); - if (stateAfterFirstBatch == null || stateAfterFirstBatch.isEmpty()) { - throw new RuntimeException("stateAfterFirstBatch should not be null or empty"); - } - stateAfterFirstSync = Jsons.jsonNode(stateAfterFirstBatch); + stateAfterFirstSync = Jsons.jsonNode(List.of(Iterables.getLast(stateAfterFirstBatch))); if (stateAfterFirstSync == null) { throw new RuntimeException("stateAfterFirstSync should not be null"); } @@ -79,74 +72,6 @@ protected void postSetup() throws Exception { } } - @Override - protected Database setupDatabase() throws Exception { - container = new MySQLContainer<>("mysql:8.0"); - container.start(); - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "CDC") - .put("initial_waiting_seconds", INITIAL_CDC_WAITING_SECONDS) - .build()); - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put("replication_method", replicationMethod) - .put("is_test", true) - .build()); - - dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.MYSQL.getDriverClassName(), - String.format(DatabaseDriver.MYSQL.getUrlFormatString(), - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asInt(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.MYSQL); - final Database database = new Database(dslContext); - - // It disable strict mode in the DB and allows to insert specific values. - // For example, it's possible to insert date with zero values "2021-00-00" - database.query(ctx -> ctx.fetch("SET @@sql_mode=''")); - - revokeAllPermissions(); - grantCorrectPermissions(); - - return database; - } - - private void revokeAllPermissions() { - executeQuery("REVOKE ALL PRIVILEGES, GRANT OPTION FROM " + container.getUsername() + "@'%';"); - } - - private void grantCorrectPermissions() { - executeQuery( - "GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO " - + container.getUsername() + "@'%';"); - } - - private void executeQuery(final String query) { - try (final DSLContext dslContext = DSLContextFactory.create( - "root", - "test", - DatabaseDriver.MYSQL.getDriverClassName(), - String.format(DatabaseDriver.MYSQL.getUrlFormatString(), - container.getHost(), - container.getFirstMappedPort(), - container.getDatabaseName()), - SQLDialect.MYSQL)) { - final Database database = new Database(dslContext); - database.query( - ctx -> ctx - .execute(query)); - } catch (final Exception e) { - throw new RuntimeException(e); - } - } - @Override public boolean testCatalog() { return true; @@ -175,4 +100,16 @@ protected void addJsonDataTypeTest() { .build()); } + @Override + protected void addDecimalValuesTest() { + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("decimal") + .airbyteType(JsonSchemaType.NUMBER) + .fullSourceDataType("decimal(19,2)") + .addInsertValues("1700000.01", "'123'") + .addExpectedValues("1700000.01", "123.00") + .build()); + } + } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotMySqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotMySqlSourceDatatypeTest.java index 7f996a4ebed6..6b971c86927c 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotMySqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotMySqlSourceDatatypeTest.java @@ -4,97 +4,26 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; -import static io.airbyte.integrations.io.airbyte.integration_tests.sources.utils.TestConstants.INITIAL_CDC_WAITING_SECONDS; - import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.testcontainers.containers.MySQLContainer; +import io.airbyte.cdk.db.Database; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; public class CdcInitialSnapshotMySqlSourceDatatypeTest extends AbstractMySqlSourceDatatypeTest { - private DSLContext dslContext; - @Override - protected void tearDown(final TestDestinationEnv testEnv) { - dslContext.close(); - container.close(); + protected JsonNode getConfig() { + return testdb.integrationTestConfigBuilder() + .withoutSsl() + .withCdcReplication() + .with("snapshot_mode", "initial_only") + .build(); } @Override - protected Database setupDatabase() throws Exception { - container = new MySQLContainer<>("mysql:8.0"); - container.start(); - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "CDC") - .put("initial_waiting_seconds", INITIAL_CDC_WAITING_SECONDS) - .build()); - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put("replication_method", replicationMethod) - .put("snapshot_mode", "initial_only") - .put("is_test", true) - .build()); - - dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.MYSQL.getDriverClassName(), - String.format(DatabaseDriver.MYSQL.getUrlFormatString(), - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asInt(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.MYSQL); - final Database database = new Database(dslContext); - - // It disable strict mode in the DB and allows to insert specific values. - // For example, it's possible to insert date with zero values "2021-00-00" - database.query(ctx -> ctx.fetch("SET @@sql_mode=''")); - - revokeAllPermissions(); - grantCorrectPermissions(); - - return database; - } - - private void revokeAllPermissions() { - executeQuery("REVOKE ALL PRIVILEGES, GRANT OPTION FROM " + container.getUsername() + "@'%';"); - } - - private void grantCorrectPermissions() { - executeQuery( - "GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO " - + container.getUsername() + "@'%';"); - } - - private void executeQuery(final String query) { - try (final DSLContext dslContext = DSLContextFactory.create( - "root", - "test", - DatabaseDriver.MYSQL.getDriverClassName(), - String.format(DatabaseDriver.MYSQL.getUrlFormatString(), - container.getHost(), - container.getFirstMappedPort(), - container.getDatabaseName()), - SQLDialect.MYSQL)) { - final Database database = new Database(dslContext); - database.query( - ctx -> ctx - .execute(query)); - } catch (final Exception e) { - throw new RuntimeException(e); - } + protected Database setupDatabase() { + testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8).withoutStrictMode().withCdcPermissions(); + return testdb.getDatabase(); } @Override diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSourceAcceptanceTest.java index f27506104060..9e12122460b7 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSourceAcceptanceTest.java @@ -4,25 +4,21 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; -import static io.airbyte.integrations.io.airbyte.integration_tests.sources.utils.TestConstants.INITIAL_CDC_WAITING_SECONDS; import static io.airbyte.protocol.models.v0.SyncMode.INCREMENTAL; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; -import io.airbyte.commons.features.EnvVariableFeatureFlags; +import io.airbyte.cdk.integrations.base.ssh.SshHelpers; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.ContainerModifier; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteMessage; @@ -36,25 +32,15 @@ import io.airbyte.protocol.models.v0.SyncMode; import java.util.List; import java.util.stream.Collectors; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; +import org.apache.commons.lang3.ArrayUtils; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testcontainers.containers.MySQLContainer; -import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; -import uk.org.webcompere.systemstubs.jupiter.SystemStub; -import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; -@ExtendWith(SystemStubsExtension.class) public class CdcMySqlSourceAcceptanceTest extends SourceAcceptanceTest { - @SystemStub - private EnvironmentVariables environmentVariables; + protected static final String STREAM_NAME = "id_and_name"; + protected static final String STREAM_NAME2 = "starships"; - private static final String STREAM_NAME = "id_and_name"; - private static final String STREAM_NAME2 = "starships"; - private MySQLContainer container; - private JsonNode config; + protected MySQLTestDatabase testdb; @Override protected String getImageName() { @@ -68,7 +54,10 @@ protected ConnectorSpecification getSpec() throws Exception { @Override protected JsonNode getConfig() { - return config; + return testdb.integrationTestConfigBuilder() + .withCdcReplication() + .withoutSsl() + .build(); } @Override @@ -79,7 +68,7 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { .withDestinationSyncMode(DestinationSyncMode.APPEND) .withStream(CatalogHelpers.createAirbyteStream( String.format("%s", STREAM_NAME), - String.format("%s", config.get(JdbcUtils.DATABASE_KEY).asText()), + testdb.getDatabaseName(), Field.of("id", JsonSchemaType.NUMBER), Field.of("name", JsonSchemaType.STRING)) .withSourceDefinedCursor(true) @@ -91,7 +80,7 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { .withDestinationSyncMode(DestinationSyncMode.APPEND) .withStream(CatalogHelpers.createAirbyteStream( String.format("%s", STREAM_NAME2), - String.format("%s", config.get(JdbcUtils.DATABASE_KEY).asText()), + testdb.getDatabaseName(), Field.of("id", JsonSchemaType.NUMBER), Field.of("name", JsonSchemaType.STRING)) .withSourceDefinedCursor(true) @@ -100,34 +89,6 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { Lists.newArrayList(SyncMode.FULL_REFRESH, INCREMENTAL))))); } - protected ConfiguredAirbyteCatalog getConfiguredCatalogWithPartialColumns() { - return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( - new ConfiguredAirbyteStream() - .withSyncMode(INCREMENTAL) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withStream(CatalogHelpers.createAirbyteStream( - String.format("%s", STREAM_NAME), - String.format("%s", config.get(JdbcUtils.DATABASE_KEY).asText()), - Field.of("id", JsonSchemaType.NUMBER) - /* no name field */) - .withSourceDefinedCursor(true) - .withSourceDefinedPrimaryKey(List.of(List.of("id"))) - .withSupportedSyncModes( - Lists.newArrayList(SyncMode.FULL_REFRESH, INCREMENTAL))), - new ConfiguredAirbyteStream() - .withSyncMode(INCREMENTAL) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withStream(CatalogHelpers.createAirbyteStream( - String.format("%s", STREAM_NAME2), - String.format("%s", config.get(JdbcUtils.DATABASE_KEY).asText()), - /* no id field */ - Field.of("name", JsonSchemaType.STRING)) - .withSourceDefinedCursor(true) - .withSourceDefinedPrimaryKey(List.of(List.of("id"))) - .withSupportedSyncModes( - Lists.newArrayList(SyncMode.FULL_REFRESH, INCREMENTAL))))); - } - @Override protected JsonNode getState() { return null; @@ -135,69 +96,21 @@ protected JsonNode getState() { @Override protected void setupEnvironment(final TestDestinationEnv environment) { - container = new MySQLContainer<>("mysql:8.0"); - container.start(); - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "CDC") - .put("initial_waiting_seconds", INITIAL_CDC_WAITING_SECONDS) - .build()); - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put("replication_method", replicationMethod) - .put("is_test", true) - .build()); - - revokeAllPermissions(); - grantCorrectPermissions(); - createAndPopulateTables(); - } - - private void createAndPopulateTables() { - executeQuery("CREATE TABLE id_and_name(id INTEGER PRIMARY KEY, name VARCHAR(200));"); - executeQuery( - "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - executeQuery("CREATE TABLE starships(id INTEGER PRIMARY KEY, name VARCHAR(200));"); - executeQuery( - "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - } - - private void revokeAllPermissions() { - executeQuery("REVOKE ALL PRIVILEGES, GRANT OPTION FROM " + container.getUsername() + "@'%';"); + testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8, getContainerModifiers()) + .withCdcPermissions() + .with("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));") + .with("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');") + .with("CREATE TABLE starships(id INTEGER, name VARCHAR(200));") + .with("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); } - private void grantCorrectPermissions() { - executeQuery( - "GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO " - + container.getUsername() + "@'%';"); - } - - private void executeQuery(final String query) { - try (final DSLContext dslContext = DSLContextFactory.create( - "root", - "test", - DatabaseDriver.MYSQL.getDriverClassName(), - String.format(DatabaseDriver.MYSQL.getUrlFormatString(), - container.getHost(), - container.getFirstMappedPort(), - container.getDatabaseName()), - SQLDialect.MYSQL)) { - final Database database = new Database(dslContext); - database.query( - ctx -> ctx - .execute(query)); - } catch (final Exception e) { - throw new RuntimeException(e); - } + protected ContainerModifier[] getContainerModifiers() { + return ArrayUtils.toArray(); } @Override protected void tearDown(final TestDestinationEnv testEnv) { - container.close(); + testdb.close(); } @Test @@ -219,19 +132,14 @@ public void testIncrementalSyncShouldNotFailIfBinlogIsDeleted() throws Exception // when we run incremental sync again there should be no new records. Run a sync with the latest // state message and assert no records were emitted. - final JsonNode latestState = Jsons.jsonNode(supportsPerStream() ? stateMessages : List.of(Iterables.getLast(stateMessages))); + final JsonNode latestState = Jsons.jsonNode(List.of(Iterables.getLast(stateMessages))); // RESET MASTER removes all binary log files that are listed in the index file, // leaving only a single, empty binary log file with a numeric suffix of .000001 - executeQuery("RESET MASTER;"); + testdb.with("RESET MASTER;"); assertEquals(6, filterRecords(runRead(configuredCatalog, latestState)).size()); } - @Override - protected boolean supportsPerStream() { - return true; - } - @Test public void testIncrementalReadSelectedColumns() throws Exception { final ConfiguredAirbyteCatalog catalog = getConfiguredCatalogWithPartialColumns(); @@ -240,17 +148,41 @@ public void testIncrementalReadSelectedColumns() throws Exception { final List records = filterRecords(allMessages); assertFalse(records.isEmpty(), "Expected a incremental sync to produce records"); verifyFieldNotExist(records, STREAM_NAME, "name"); - verifyFieldNotExist(records, STREAM_NAME2, "id"); + verifyFieldNotExist(records, STREAM_NAME2, "name"); + } + + private ConfiguredAirbyteCatalog getConfiguredCatalogWithPartialColumns() { + // We cannot strip the primary key field as that is required for a successful CDC sync + return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( + new ConfiguredAirbyteStream() + .withSyncMode(INCREMENTAL) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(CatalogHelpers.createAirbyteStream( + String.format("%s", STREAM_NAME), + testdb.getDatabaseName(), + Field.of("id", JsonSchemaType.NUMBER) + /* no name field */) + .withSourceDefinedCursor(true) + .withSourceDefinedPrimaryKey(List.of(List.of("id"))) + .withSupportedSyncModes( + Lists.newArrayList(SyncMode.FULL_REFRESH, INCREMENTAL))), + new ConfiguredAirbyteStream() + .withSyncMode(INCREMENTAL) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(CatalogHelpers.createAirbyteStream( + String.format("%s", STREAM_NAME2), + testdb.getDatabaseName(), + /* no name field */ + Field.of("id", JsonSchemaType.NUMBER)) + .withSourceDefinedCursor(true) + .withSourceDefinedPrimaryKey(List.of(List.of("id"))) + .withSupportedSyncModes( + Lists.newArrayList(SyncMode.FULL_REFRESH, INCREMENTAL))))); } private void verifyFieldNotExist(final List records, final String stream, final String field) { - assertTrue(records.stream() - .filter(r -> { - return r.getStream().equals(stream) - && r.getData().get(field) != null; - }) - .collect(Collectors.toList()) - .isEmpty(), "Records contain unselected columns [%s:%s]".formatted(stream, field)); + assertTrue(records.stream().noneMatch(r -> r.getStream().equals(stream) && r.getData().get(field) != null), + "Records contain unselected columns [%s:%s]".formatted(stream, field)); } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslCaCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslCaCertificateSourceAcceptanceTest.java index 125fd259b740..98ccf8c7f50f 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslCaCertificateSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslCaCertificateSourceAcceptanceTest.java @@ -4,207 +4,28 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; -import static io.airbyte.integrations.io.airbyte.integration_tests.sources.utils.TestConstants.INITIAL_CDC_WAITING_SECONDS; -import static io.airbyte.protocol.models.v0.SyncMode.INCREMENTAL; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; - import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.MySqlUtils; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import io.airbyte.protocol.models.v0.AirbyteStateMessage; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.SyncMode; -import java.util.List; -import java.util.stream.Collectors; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.MySQLContainer; - -public class CdcMySqlSslCaCertificateSourceAcceptanceTest extends SourceAcceptanceTest { - - private static final String STREAM_NAME = "id_and_name"; - private static final String STREAM_NAME2 = "starships"; - private MySQLContainer container; - private JsonNode config; - private static MySqlUtils.Certificate certs; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.ContainerModifier; +import org.apache.commons.lang3.ArrayUtils; - @Override - protected String getImageName() { - return "airbyte/source-mysql:dev"; - } - - @Override - protected ConnectorSpecification getSpec() throws Exception { - return SshHelpers.getSpecAndInjectSsh(); - } +public class CdcMySqlSslCaCertificateSourceAcceptanceTest extends CdcMySqlSourceAcceptanceTest { @Override protected JsonNode getConfig() { - return config; - } - - @Override - protected ConfiguredAirbyteCatalog getConfiguredCatalog() { - return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( - new ConfiguredAirbyteStream() - .withSyncMode(INCREMENTAL) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withStream(CatalogHelpers.createAirbyteStream( - String.format("%s", STREAM_NAME), - String.format("%s", config.get(JdbcUtils.DATABASE_KEY).asText()), - Field.of("id", JsonSchemaType.NUMBER), - Field.of("name", JsonSchemaType.STRING)) - .withSourceDefinedCursor(true) - .withSourceDefinedPrimaryKey(List.of(List.of("id"))) - .withSupportedSyncModes( - Lists.newArrayList(SyncMode.FULL_REFRESH, INCREMENTAL))), - new ConfiguredAirbyteStream() - .withSyncMode(INCREMENTAL) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withStream(CatalogHelpers.createAirbyteStream( - String.format("%s", STREAM_NAME2), - String.format("%s", config.get(JdbcUtils.DATABASE_KEY).asText()), - Field.of("id", JsonSchemaType.NUMBER), - Field.of("name", JsonSchemaType.STRING)) - .withSourceDefinedCursor(true) - .withSourceDefinedPrimaryKey(List.of(List.of("id"))) - .withSupportedSyncModes( - Lists.newArrayList(SyncMode.FULL_REFRESH, INCREMENTAL))))); - } - - @Override - protected JsonNode getState() { - return null; - } - - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - container = new MySQLContainer<>("mysql:8.0"); - container.start(); - certs = MySqlUtils.getCertificate(container, true); - - final var sslMode = ImmutableMap.builder() - .put(JdbcUtils.MODE_KEY, "verify_ca") - .put("ca_certificate", certs.getCaCertificate()) - .put("client_certificate", certs.getClientCertificate()) - .put("client_key", certs.getClientKey()) - .put("client_key_password", "Passw0rd") + return testdb.integrationTestConfigBuilder() + .withCdcReplication() + .withSsl(ImmutableMap.builder() + .put(JdbcUtils.MODE_KEY, "verify_ca") + .put("ca_certificate", testdb.getCertificates().caCertificate()) + .build()) .build(); - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "CDC") - .put("initial_waiting_seconds", INITIAL_CDC_WAITING_SECONDS) - .build()); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put(JdbcUtils.SSL_KEY, true) - .put(JdbcUtils.SSL_MODE_KEY, sslMode) - .put("replication_method", replicationMethod) - .put("is_test", true) - .build()); - - revokeAllPermissions(); - grantCorrectPermissions(); - createAndPopulateTables(); - } - - private void createAndPopulateTables() { - executeQuery("CREATE TABLE id_and_name(id INTEGER PRIMARY KEY, name VARCHAR(200));"); - executeQuery( - "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - executeQuery("CREATE TABLE starships(id INTEGER PRIMARY KEY, name VARCHAR(200));"); - executeQuery( - "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - } - - private void revokeAllPermissions() { - executeQuery("REVOKE ALL PRIVILEGES, GRANT OPTION FROM " + container.getUsername() + "@'%';"); - } - - private void grantCorrectPermissions() { - executeQuery( - "GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO " - + container.getUsername() + "@'%';"); - } - - private void executeQuery(final String query) { - try (final DSLContext dslContext = DSLContextFactory.create( - "root", - "test", - DatabaseDriver.MYSQL.getDriverClassName(), - String.format(DatabaseDriver.MYSQL.getUrlFormatString(), - container.getHost(), - container.getFirstMappedPort(), - container.getDatabaseName()), - SQLDialect.MYSQL)) { - final Database database = new Database(dslContext); - database.query( - ctx -> ctx - .execute(query)); - } catch (final Exception e) { - throw new RuntimeException(e); - } - } - - @Override - protected void tearDown(final TestDestinationEnv testEnv) { - container.close(); - } - - @Test - public void testIncrementalSyncShouldNotFailIfBinlogIsDeleted() throws Exception { - final ConfiguredAirbyteCatalog configuredCatalog = withSourceDefinedCursors(getConfiguredCatalog()); - // only sync incremental streams - configuredCatalog.setStreams( - configuredCatalog.getStreams().stream().filter(s -> s.getSyncMode() == INCREMENTAL).collect(Collectors.toList())); - - final List airbyteMessages = runRead(configuredCatalog, getState()); - final List recordMessages = filterRecords(airbyteMessages); - final List stateMessages = airbyteMessages - .stream() - .filter(m -> m.getType() == AirbyteMessage.Type.STATE) - .map(AirbyteMessage::getState) - .collect(Collectors.toList()); - assertFalse(recordMessages.isEmpty(), "Expected the first incremental sync to produce records"); - assertFalse(stateMessages.isEmpty(), "Expected incremental sync to produce STATE messages"); - - // when we run incremental sync again there should be no new records. Run a sync with the latest - // state message and assert no records were emitted. - final JsonNode latestState = Jsons.jsonNode(supportsPerStream() ? stateMessages : List.of(Iterables.getLast(stateMessages))); - // RESET MASTER removes all binary log files that are listed in the index file, - // leaving only a single, empty binary log file with a numeric suffix of .000001 - executeQuery("RESET MASTER;"); - - assertEquals(6, filterRecords(runRead(configuredCatalog, latestState)).size()); } @Override - protected boolean supportsPerStream() { - return true; + protected ContainerModifier[] getContainerModifiers() { + return ArrayUtils.toArray(ContainerModifier.ROOT_AND_SERVER_CERTIFICATES); } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslRequiredSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslRequiredSourceAcceptanceTest.java index 8f0137856b2f..f508513b72d8 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslRequiredSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcMySqlSslRequiredSourceAcceptanceTest.java @@ -4,205 +4,32 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; -import static io.airbyte.integrations.io.airbyte.integration_tests.sources.utils.TestConstants.INITIAL_CDC_WAITING_SECONDS; -import static io.airbyte.protocol.models.v0.SyncMode.INCREMENTAL; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; - import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import io.airbyte.protocol.models.v0.AirbyteStateMessage; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.SyncMode; -import java.util.List; -import java.util.stream.Collectors; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.MySQLContainer; - -public class CdcMySqlSslRequiredSourceAcceptanceTest extends SourceAcceptanceTest { - - private static final String STREAM_NAME = "id_and_name"; - private static final String STREAM_NAME2 = "starships"; - private MySQLContainer container; - private JsonNode config; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.ContainerModifier; +import org.apache.commons.lang3.ArrayUtils; - @Override - protected String getImageName() { - return "airbyte/source-mysql:dev"; - } - - @Override - protected ConnectorSpecification getSpec() throws Exception { - return SshHelpers.getSpecAndInjectSsh(); - } +public class CdcMySqlSslRequiredSourceAcceptanceTest extends CdcMySqlSourceAcceptanceTest { @Override protected JsonNode getConfig() { - return config; - } - - @Override - protected ConfiguredAirbyteCatalog getConfiguredCatalog() { - return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( - new ConfiguredAirbyteStream() - .withSyncMode(INCREMENTAL) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withStream(CatalogHelpers.createAirbyteStream( - String.format("%s", STREAM_NAME), - String.format("%s", config.get(JdbcUtils.DATABASE_KEY).asText()), - Field.of("id", JsonSchemaType.NUMBER), - Field.of("name", JsonSchemaType.STRING)) - .withSourceDefinedCursor(true) - .withSourceDefinedPrimaryKey(List.of(List.of("id"))) - .withSupportedSyncModes( - Lists.newArrayList(SyncMode.FULL_REFRESH, INCREMENTAL))), - new ConfiguredAirbyteStream() - .withSyncMode(INCREMENTAL) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withStream(CatalogHelpers.createAirbyteStream( - String.format("%s", STREAM_NAME2), - String.format("%s", config.get(JdbcUtils.DATABASE_KEY).asText()), - Field.of("id", JsonSchemaType.NUMBER), - Field.of("name", JsonSchemaType.STRING)) - .withSourceDefinedCursor(true) - .withSourceDefinedPrimaryKey(List.of(List.of("id"))) - .withSupportedSyncModes( - Lists.newArrayList(SyncMode.FULL_REFRESH, INCREMENTAL))))); - } - - @Override - protected JsonNode getState() { - return null; - } - - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - container = new MySQLContainer<>("mysql:8.0"); - container.start(); - - final var sslMode = ImmutableMap.builder() - .put(JdbcUtils.MODE_KEY, "required") + return testdb.integrationTestConfigBuilder() + .withCdcReplication() + .withSsl(ImmutableMap.builder().put(JdbcUtils.MODE_KEY, "required").build()) .build(); - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "CDC") - .put("initial_waiting_seconds", INITIAL_CDC_WAITING_SECONDS) - .build()); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put(JdbcUtils.SSL_KEY, true) - .put(JdbcUtils.SSL_MODE_KEY, sslMode) - .put("replication_method", replicationMethod) - .put("is_test", true) - .build()); - - revokeAllPermissions(); - grantCorrectPermissions(); - alterUserRequireSsl(); - createAndPopulateTables(); - } - - private void alterUserRequireSsl() { - executeQuery("ALTER USER " + container.getUsername() + " REQUIRE SSL;"); - } - - private void createAndPopulateTables() { - executeQuery("CREATE TABLE id_and_name(id INTEGER PRIMARY KEY, name VARCHAR(200));"); - executeQuery( - "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - executeQuery("CREATE TABLE starships(id INTEGER PRIMARY KEY, name VARCHAR(200));"); - executeQuery( - "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - } - - private void revokeAllPermissions() { - executeQuery("REVOKE ALL PRIVILEGES, GRANT OPTION FROM " + container.getUsername() + "@'%';"); - } - - private void grantCorrectPermissions() { - executeQuery( - "GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO " - + container.getUsername() + "@'%';"); - } - - private void executeQuery(final String query) { - try (final DSLContext dslContext = DSLContextFactory.create( - "root", - "test", - DatabaseDriver.MYSQL.getDriverClassName(), - String.format(DatabaseDriver.MYSQL.getUrlFormatString(), - container.getHost(), - container.getFirstMappedPort(), - container.getDatabaseName()), - SQLDialect.MYSQL)) { - final Database database = new Database(dslContext); - database.query( - ctx -> ctx - .execute(query)); - } catch (final Exception e) { - throw new RuntimeException(e); - } } @Override - protected void tearDown(final TestDestinationEnv testEnv) { - container.close(); - } - - @Test - public void testIncrementalSyncShouldNotFailIfBinlogIsDeleted() throws Exception { - final ConfiguredAirbyteCatalog configuredCatalog = withSourceDefinedCursors(getConfiguredCatalog()); - // only sync incremental streams - configuredCatalog.setStreams( - configuredCatalog.getStreams().stream().filter(s -> s.getSyncMode() == INCREMENTAL).collect(Collectors.toList())); - - final List airbyteMessages = runRead(configuredCatalog, getState()); - final List recordMessages = filterRecords(airbyteMessages); - final List stateMessages = airbyteMessages - .stream() - .filter(m -> m.getType() == AirbyteMessage.Type.STATE) - .map(AirbyteMessage::getState) - .collect(Collectors.toList()); - assertFalse(recordMessages.isEmpty(), "Expected the first incremental sync to produce records"); - assertFalse(stateMessages.isEmpty(), "Expected incremental sync to produce STATE messages"); - - // when we run incremental sync again there should be no new records. Run a sync with the latest - // state message and assert no records were emitted. - final JsonNode latestState = Jsons.jsonNode(supportsPerStream() ? stateMessages : List.of(Iterables.getLast(stateMessages))); - // RESET MASTER removes all binary log files that are listed in the index file, - // leaving only a single, empty binary log file with a numeric suffix of .000001 - executeQuery("RESET MASTER;"); - - assertEquals(6, filterRecords(runRead(configuredCatalog, latestState)).size()); + protected void setupEnvironment(final TestDestinationEnv environment) { + super.setupEnvironment(environment); + testdb.with("ALTER USER %s REQUIRE SSL;", testdb.getUserName()); } @Override - protected boolean supportsPerStream() { - return true; + protected ContainerModifier[] getContainerModifiers() { + return ArrayUtils.toArray(ContainerModifier.ROOT_AND_SERVER_CERTIFICATES, ContainerModifier.CLIENT_CERTITICATE); } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CloudDeploymentMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CloudDeploymentMySqlSourceAcceptanceTest.java new file mode 100644 index 000000000000..8b607cc092f8 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CloudDeploymentMySqlSourceAcceptanceTest.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import io.airbyte.cdk.integrations.base.ssh.SshHelpers; +import io.airbyte.commons.features.FeatureFlags; +import io.airbyte.commons.features.FeatureFlagsWrapper; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.protocol.models.v0.ConnectorSpecification; + +public class CloudDeploymentMySqlSourceAcceptanceTest extends MySqlSslSourceAcceptanceTest { + + @Override + protected FeatureFlags featureFlags() { + return FeatureFlagsWrapper.overridingDeploymentMode(super.featureFlags(), "CLOUD"); + } + + @Override + protected ConnectorSpecification getSpec() throws Exception { + return SshHelpers.injectSshIntoSpec(Jsons.deserialize(MoreResources.readResource("expected_cloud_spec.json"), ConnectorSpecification.class)); + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CloudDeploymentMySqlSslCaCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CloudDeploymentMySqlSslCaCertificateSourceAcceptanceTest.java new file mode 100644 index 000000000000..15f0b5b5a612 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CloudDeploymentMySqlSslCaCertificateSourceAcceptanceTest.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.ssh.SshHelpers; +import io.airbyte.commons.features.FeatureFlags; +import io.airbyte.commons.features.FeatureFlagsWrapper; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.ContainerModifier; +import io.airbyte.protocol.models.v0.ConnectorSpecification; +import org.apache.commons.lang3.ArrayUtils; + +public class CloudDeploymentMySqlSslCaCertificateSourceAcceptanceTest extends MySqlSourceAcceptanceTest { + + private static final String PASSWORD = "Passw0rd"; + + @Override + protected FeatureFlags featureFlags() { + return FeatureFlagsWrapper.overridingDeploymentMode(super.featureFlags(), "CLOUD"); + } + + @Override + protected ContainerModifier[] getContainerModifiers() { + return ArrayUtils.toArray(ContainerModifier.ROOT_AND_SERVER_CERTIFICATES); + } + + @Override + protected JsonNode getConfig() { + return testdb.integrationTestConfigBuilder() + .withStandardReplication() + .withSsl(ImmutableMap.builder() + .put(JdbcUtils.MODE_KEY, "verify_ca") + .put("ca_certificate", testdb.getCaCertificate()) + .put("client_key_password", PASSWORD) + .build()) + .build(); + } + + @Override + protected ConnectorSpecification getSpec() throws Exception { + return SshHelpers.injectSshIntoSpec(Jsons.deserialize(MoreResources.readResource("expected_cloud_spec.json"), ConnectorSpecification.class)); + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CloudDeploymentMySqlSslFullCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CloudDeploymentMySqlSslFullCertificateSourceAcceptanceTest.java new file mode 100644 index 000000000000..298276ee443f --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CloudDeploymentMySqlSslFullCertificateSourceAcceptanceTest.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.ssh.SshHelpers; +import io.airbyte.commons.features.FeatureFlags; +import io.airbyte.commons.features.FeatureFlagsWrapper; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.ContainerModifier; +import io.airbyte.protocol.models.v0.ConnectorSpecification; +import org.apache.commons.lang3.ArrayUtils; + +public class CloudDeploymentMySqlSslFullCertificateSourceAcceptanceTest extends MySqlSourceAcceptanceTest { + + private static final String PASSWORD = "Passw0rd"; + + @Override + protected FeatureFlags featureFlags() { + return FeatureFlagsWrapper.overridingDeploymentMode(super.featureFlags(), "CLOUD"); + } + + @Override + protected ContainerModifier[] getContainerModifiers() { + return ArrayUtils.toArray(ContainerModifier.ROOT_AND_SERVER_CERTIFICATES, ContainerModifier.CLIENT_CERTITICATE); + } + + @Override + protected JsonNode getConfig() { + return testdb.integrationTestConfigBuilder() + .withStandardReplication() + .withSsl(ImmutableMap.builder() + .put(JdbcUtils.MODE_KEY, "verify_ca") + .put("ca_certificate", testdb.getCertificates().caCertificate()) + .put("client_certificate", testdb.getCertificates().clientCertificate()) + .put("client_key", testdb.getCertificates().clientKey()) + .put("client_key_password", PASSWORD) + .build()) + .build(); + } + + @Override + protected ConnectorSpecification getSpec() throws Exception { + return SshHelpers.injectSshIntoSpec(Jsons.deserialize(MoreResources.readResource("expected_cloud_spec.json"), ConnectorSpecification.class)); + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlDatatypeAccuracyTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlDatatypeAccuracyTest.java index 9397bbab9bcd..516d6c20a425 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlDatatypeAccuracyTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlDatatypeAccuracyTest.java @@ -5,28 +5,35 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; import com.mysql.cj.MysqlType; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.TestDataHolder; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.integrations.standardtest.source.TestDataHolder; +import io.airbyte.integrations.source.mysql.MySQLContainerFactory; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase; import io.airbyte.protocol.models.JsonSchemaType; import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.Map.Entry; -import org.jooq.SQLDialect; -import org.testcontainers.containers.MySQLContainer; public class MySqlDatatypeAccuracyTest extends AbstractMySqlSourceDatatypeTest { @Override - protected void tearDown(final TestDestinationEnv testEnv) { - container.close(); + protected JsonNode getConfig() { + return testdb.integrationTestConfigBuilder() + .withoutSsl() + .withStandardReplication() + .build(); + } + + @Override + protected Database setupDatabase() { + final var sharedContainer = new MySQLContainerFactory().shared("mysql:8.0"); + testdb = new MySQLTestDatabase(sharedContainer) + .withConnectionProperty("zeroDateTimeBehavior", "convertToNull") + .initialized() + .withoutStrictMode(); + return testdb.getDatabase(); } private final Map> charsetsCollationsMap = Map.of( @@ -36,41 +43,6 @@ protected void tearDown(final TestDestinationEnv testEnv) { "binary", Arrays.asList("binary"), "CP1250", Arrays.asList("CP1250_general_ci", "cp1250_czech_cs")); - @Override - protected Database setupDatabase() throws Exception { - container = new MySQLContainer<>("mysql:8.0"); - container.start(); - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "STANDARD") - .build()); - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put("replication_method", replicationMethod) - .build()); - - final Database database = new Database( - DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.MYSQL.getDriverClassName(), - String.format(DatabaseDriver.MYSQL.getUrlFormatString(), - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asInt(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.MYSQL, - Map.of("zeroDateTimeBehavior", "convertToNull"))); - - // It disable strict mode in the DB and allows to insert specific values. - // For example, it's possible to insert date with zero values "2021-00-00" - database.query(ctx -> ctx.fetch("SET @@sql_mode=''")); - - return database; - } - @Override public boolean testCatalog() { return true; diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceAcceptanceTest.java index 780e77ad1426..d6a2adffe2c9 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceAcceptanceTest.java @@ -5,17 +5,14 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import io.airbyte.commons.features.EnvVariableFeatureFlags; +import io.airbyte.cdk.integrations.base.ssh.SshHelpers; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.ContainerModifier; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.CatalogHelpers; @@ -25,67 +22,31 @@ import io.airbyte.protocol.models.v0.DestinationSyncMode; import io.airbyte.protocol.models.v0.SyncMode; import java.util.HashMap; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testcontainers.containers.MySQLContainer; -import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; -import uk.org.webcompere.systemstubs.jupiter.SystemStub; -import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; +import org.apache.commons.lang3.ArrayUtils; -@ExtendWith(SystemStubsExtension.class) public class MySqlSourceAcceptanceTest extends SourceAcceptanceTest { - @SystemStub - private EnvironmentVariables environmentVariables; + protected MySQLTestDatabase testdb; private static final String STREAM_NAME = "id_and_name"; private static final String STREAM_NAME2 = "public.starships"; - protected MySQLContainer container; - protected JsonNode config; - @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - container = new MySQLContainer<>("mysql:8.0"); - container.start(); - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "STANDARD") - .build()); - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put("replication_method", replicationMethod) - .build()); - - try (final DSLContext dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.MYSQL.getDriverClassName(), - String.format(DatabaseDriver.MYSQL.getUrlFormatString(), - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asInt(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.MYSQL)) { - final Database database = new Database(dslContext); + testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8, getContainerModifiers()) + .with("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));") + .with("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');") + .with("CREATE TABLE starships(id INTEGER, name VARCHAR(200));") + .with("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); + } - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - return null; - }); - } + protected ContainerModifier[] getContainerModifiers() { + return ArrayUtils.toArray(); } @Override protected void tearDown(final TestDestinationEnv testEnv) { - container.close(); + testdb.close(); } @Override @@ -100,7 +61,10 @@ protected ConnectorSpecification getSpec() throws Exception { @Override protected JsonNode getConfig() { - return config; + return testdb.integrationTestConfigBuilder() + .withStandardReplication() + .withoutSsl() + .build(); } @Override @@ -111,7 +75,7 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { .withCursorField(Lists.newArrayList("id")) .withDestinationSyncMode(DestinationSyncMode.APPEND) .withStream(CatalogHelpers.createAirbyteStream( - String.format("%s.%s", config.get(JdbcUtils.DATABASE_KEY).asText(), STREAM_NAME), + String.format("%s.%s", testdb.getDatabaseName(), STREAM_NAME), Field.of("id", JsonSchemaType.NUMBER), Field.of("name", JsonSchemaType.STRING)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))), @@ -120,7 +84,7 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalog() { .withCursorField(Lists.newArrayList("id")) .withDestinationSyncMode(DestinationSyncMode.APPEND) .withStream(CatalogHelpers.createAirbyteStream( - String.format("%s.%s", config.get(JdbcUtils.DATABASE_KEY).asText(), STREAM_NAME2), + String.format("%s.%s", testdb.getDatabaseName(), STREAM_NAME2), Field.of("id", JsonSchemaType.NUMBER), Field.of("name", JsonSchemaType.STRING)) .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); @@ -131,9 +95,4 @@ protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); } - @Override - protected boolean supportsPerStream() { - return true; - } - } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceDatatypeTest.java index a522fc268a42..cbfa689562dc 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceDatatypeTest.java @@ -5,57 +5,28 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import java.util.Map; -import org.jooq.SQLDialect; -import org.testcontainers.containers.MySQLContainer; +import io.airbyte.cdk.db.Database; +import io.airbyte.integrations.source.mysql.MySQLContainerFactory; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase; public class MySqlSourceDatatypeTest extends AbstractMySqlSourceDatatypeTest { @Override - protected void tearDown(final TestDestinationEnv testEnv) { - container.close(); + protected JsonNode getConfig() { + return testdb.integrationTestConfigBuilder() + .withoutSsl() + .withStandardReplication() + .build(); } @Override - protected Database setupDatabase() throws Exception { - container = new MySQLContainer<>("mysql:8.0"); - container.start(); - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "STANDARD") - .build()); - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put("replication_method", replicationMethod) - .build()); - - final Database database = new Database( - DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.MYSQL.getDriverClassName(), - String.format(DatabaseDriver.MYSQL.getUrlFormatString(), - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asInt(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.MYSQL, - Map.of("zeroDateTimeBehavior", "convertToNull"))); - - // It disable strict mode in the DB and allows to insert specific values. - // For example, it's possible to insert date with zero values "2021-00-00" - database.query(ctx -> ctx.fetch("SET @@sql_mode=''")); - - return database; + protected Database setupDatabase() { + final var sharedContainer = new MySQLContainerFactory().shared("mysql:8.0"); + testdb = new MySQLTestDatabase(sharedContainer) + .withConnectionProperty("zeroDateTimeBehavior", "convertToNull") + .initialized() + .withoutStrictMode(); + return testdb.getDatabase(); } @Override diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSslCaCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSslCaCertificateSourceAcceptanceTest.java index 3084815484f7..71f36aa027f4 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSslCaCertificateSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSslCaCertificateSourceAcceptanceTest.java @@ -4,24 +4,30 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; -import io.airbyte.db.MySqlUtils; -import io.airbyte.db.jdbc.JdbcUtils; -import java.io.IOException; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.ContainerModifier; +import org.apache.commons.lang3.ArrayUtils; -public class MySqlSslCaCertificateSourceAcceptanceTest extends AbstractMySqlSslCertificateSourceAcceptanceTest { +public class MySqlSslCaCertificateSourceAcceptanceTest extends MySqlSourceAcceptanceTest { + + private static final String PASSWORD = "Passw0rd"; @Override - public MySqlUtils.Certificate getCertificates() throws IOException, InterruptedException { - return MySqlUtils.getCertificate(container, false); + protected ContainerModifier[] getContainerModifiers() { + return ArrayUtils.toArray(ContainerModifier.ROOT_AND_SERVER_CERTIFICATES); } @Override - public ImmutableMap getSslConfig() { - return ImmutableMap.builder() - .put(JdbcUtils.MODE_KEY, "verify_ca") - .put("ca_certificate", certs.getCaCertificate()) - .put("client_key_password", PASSWORD) + protected JsonNode getConfig() { + return testdb.integrationTestConfigBuilder() + .withStandardReplication() + .withSsl(ImmutableMap.builder() + .put(JdbcUtils.MODE_KEY, "verify_ca") + .put("ca_certificate", testdb.getCaCertificate()) + .put("client_key_password", PASSWORD) + .build()) .build(); } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSslFullCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSslFullCertificateSourceAcceptanceTest.java index a9fdf7d06141..d9f325d9db31 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSslFullCertificateSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSslFullCertificateSourceAcceptanceTest.java @@ -4,26 +4,32 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; -import io.airbyte.db.MySqlUtils; -import io.airbyte.db.jdbc.JdbcUtils; -import java.io.IOException; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.ContainerModifier; +import org.apache.commons.lang3.ArrayUtils; -public class MySqlSslFullCertificateSourceAcceptanceTest extends AbstractMySqlSslCertificateSourceAcceptanceTest { +public class MySqlSslFullCertificateSourceAcceptanceTest extends MySqlSourceAcceptanceTest { + + private static final String PASSWORD = "Passw0rd"; @Override - public MySqlUtils.Certificate getCertificates() throws IOException, InterruptedException { - return MySqlUtils.getCertificate(container, true); + protected ContainerModifier[] getContainerModifiers() { + return ArrayUtils.toArray(ContainerModifier.ROOT_AND_SERVER_CERTIFICATES, ContainerModifier.CLIENT_CERTITICATE); } @Override - public ImmutableMap getSslConfig() { - return ImmutableMap.builder() - .put(JdbcUtils.MODE_KEY, "verify_ca") - .put("ca_certificate", certs.getCaCertificate()) - .put("client_certificate", certs.getClientCertificate()) - .put("client_key", certs.getClientKey()) - .put("client_key_password", PASSWORD) + protected JsonNode getConfig() { + return testdb.integrationTestConfigBuilder() + .withStandardReplication() + .withSsl(ImmutableMap.builder() + .put(JdbcUtils.MODE_KEY, "verify_ca") + .put("ca_certificate", testdb.getCertificates().caCertificate()) + .put("client_certificate", testdb.getCertificates().clientCertificate()) + .put("client_key", testdb.getCertificates().clientKey()) + .put("client_key_password", PASSWORD) + .build()) .build(); } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSslSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSslSourceAcceptanceTest.java index 17d568981583..5f46e43808e4 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSslSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSslSourceAcceptanceTest.java @@ -6,62 +6,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.testcontainers.containers.MySQLContainer; +import io.airbyte.cdk.db.jdbc.JdbcUtils; public class MySqlSslSourceAcceptanceTest extends MySqlSourceAcceptanceTest { @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - container = new MySQLContainer<>("mysql:8.0"); - container.start(); - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "STANDARD") - .build()); - - var sslMode = ImmutableMap.builder() - .put(JdbcUtils.MODE_KEY, "required") + protected JsonNode getConfig() { + return testdb.integrationTestConfigBuilder() + .withStandardReplication() + .withSsl(ImmutableMap.builder().put(JdbcUtils.MODE_KEY, "required").build()) .build(); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put(JdbcUtils.SSL_KEY, true) - .put(JdbcUtils.SSL_MODE_KEY, sslMode) - .put("replication_method", replicationMethod) - .build()); - - try (final DSLContext dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.MYSQL.getDriverClassName(), - String.format("jdbc:mysql://%s:%s/%s", - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asText(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.MYSQL)) { - final Database database = new Database(dslContext); - - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch( - "INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch( - "INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - return null; - }); - } } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshKeyMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshKeyMySqlSourceAcceptanceTest.java index a65d9bb1744a..7d5f060f34c2 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshKeyMySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshKeyMySqlSourceAcceptanceTest.java @@ -4,26 +4,10 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import java.nio.file.Path; -import org.junit.jupiter.api.extension.ExtendWith; -import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; -import uk.org.webcompere.systemstubs.jupiter.SystemStub; -import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; -@ExtendWith(SystemStubsExtension.class) public class SshKeyMySqlSourceAcceptanceTest extends AbstractSshMySqlSourceAcceptanceTest { - @SystemStub - private EnvironmentVariables environmentVariables; - - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - super.setupEnvironment(environment); - } - @Override public Path getConfigFilePath() { return Path.of("secrets/ssh-key-repl-config.json"); diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordMySqlSourceAcceptanceTest.java index 3ac9007bb643..998e304d7145 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordMySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordMySqlSourceAcceptanceTest.java @@ -7,35 +7,19 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.base.ssh.SshBastionContainer; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; import io.airbyte.commons.exceptions.ConfigErrorException; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.base.ssh.SshBastionContainer; -import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.ContainerModifier; import io.airbyte.integrations.source.mysql.MySqlSource; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import java.nio.file.Path; -import java.util.List; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testcontainers.containers.MySQLContainer; -import org.testcontainers.containers.Network; -import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; -import uk.org.webcompere.systemstubs.jupiter.SystemStub; -import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; -@ExtendWith(SystemStubsExtension.class) public class SshPasswordMySqlSourceAcceptanceTest extends AbstractSshMySqlSourceAcceptanceTest { - @SystemStub - private EnvironmentVariables environmentVariables; - - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - super.setupEnvironment(environment); - } - @Override public Path getConfigFilePath() { return Path.of("secrets/ssh-pwd-repl-config.json"); @@ -43,30 +27,23 @@ public Path getConfigFilePath() { @Test public void sshTimeoutExceptionMarkAsConfigErrorTest() throws Exception { - final SshBastionContainer bastion = new SshBastionContainer(); - final Network network = Network.newNetwork(); - // set up env - final MySQLContainer db = startTestContainers(bastion, network); - config = bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, bastion.getBasicDbConfigBuider(db, List.of("public")), true); - bastion.stopAndClose(); - final Source sshWrappedSource = MySqlSource.sshWrappedSource(); - final Exception exception = assertThrows(ConfigErrorException.class, () -> sshWrappedSource.discover(config)); - - final String expectedMessage = "Timed out while opening a SSH Tunnel. Please double check the given SSH configurations and try again."; - final String actualMessage = exception.getMessage(); - - assertTrue(actualMessage.contains(expectedMessage)); - } - - private MySQLContainer startTestContainers(final SshBastionContainer bastion, final Network network) { - bastion.initAndStartBastion(network); - return initAndStartJdbcContainer(network); - } - - private MySQLContainer initAndStartJdbcContainer(final Network network) { - final MySQLContainer db = new MySQLContainer<>("mysql:8.0").withNetwork(network); - db.start(); - return db; + try (final var testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8, ContainerModifier.NETWORK)) { + final SshBastionContainer bastion = new SshBastionContainer(); + bastion.initAndStartBastion(testdb.getContainer().getNetwork()); + final var config = testdb.integrationTestConfigBuilder() + .withoutSsl() + .with("tunnel_method", bastion.getTunnelMethod(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, true)) + .build(); + bastion.stopAndClose(); + + final Source sshWrappedSource = MySqlSource.sshWrappedSource(new MySqlSource()); + final Exception exception = assertThrows(ConfigErrorException.class, () -> sshWrappedSource.discover(config)); + + final String expectedMessage = + "Timed out while opening a SSH Tunnel. Please double check the given SSH configurations and try again."; + final String actualMessage = exception.getMessage(); + assertTrue(actualMessage.contains(expectedMessage)); + } } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/utils/TestConstants.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/utils/TestConstants.java deleted file mode 100644 index b88948c180fb..000000000000 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/utils/TestConstants.java +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.io.airbyte.integration_tests.sources.utils; - -public class TestConstants { - - public static final int INITIAL_CDC_WAITING_SECONDS = 5; - -} diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/resources/expected_cloud_spec.json b/airbyte-integrations/connectors/source-mysql/src/test-integration/resources/expected_cloud_spec.json new file mode 100644 index 000000000000..50d717a95886 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/resources/expected_cloud_spec.json @@ -0,0 +1,324 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/mysql", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "MySql Source Spec", + "type": "object", + "required": ["host", "port", "database", "username", "replication_method"], + "properties": { + "host": { + "description": "The host name of the database.", + "title": "Host", + "type": "string", + "order": 0 + }, + "port": { + "description": "The port to connect to.", + "title": "Port", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 3306, + "examples": ["3306"], + "order": 1 + }, + "database": { + "description": "The database name.", + "title": "Database", + "type": "string", + "order": 2 + }, + "username": { + "description": "The username which is used to access the database.", + "title": "Username", + "type": "string", + "order": 3 + }, + "password": { + "description": "The password associated with the username.", + "title": "Password", + "type": "string", + "airbyte_secret": true, + "order": 4, + "always_show": true + }, + "jdbc_url_params": { + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", + "title": "JDBC URL Parameters (Advanced)", + "type": "string", + "order": 5 + }, + "ssl_mode": { + "title": "SSL modes", + "description": "SSL connection modes. Read more in the docs.", + "type": "object", + "order": 7, + "oneOf": [ + { + "title": "preferred", + "description": "Automatically attempt SSL connection. If the MySQL server does not support SSL, continue with a regular connection.", + "required": ["mode"], + "properties": { + "mode": { "type": "string", "const": "preferred", "order": 0 } + } + }, + { + "title": "required", + "description": "Always connect with SSL. If the MySQL server doesn’t support SSL, the connection will not be established. Certificate Authority (CA) and Hostname are not verified.", + "required": ["mode"], + "properties": { + "mode": { "type": "string", "const": "required", "order": 0 } + } + }, + { + "title": "Verify CA", + "description": "Always connect with SSL. Verifies CA, but allows connection even if Hostname does not match.", + "required": ["mode", "ca_certificate"], + "properties": { + "mode": { "type": "string", "const": "verify_ca", "order": 0 }, + "ca_certificate": { + "type": "string", + "title": "CA certificate", + "description": "CA certificate", + "airbyte_secret": true, + "multiline": true, + "order": 1 + }, + "client_certificate": { + "type": "string", + "title": "Client certificate", + "description": "Client certificate (this is not a required field, but if you want to use it, you will need to add the Client key as well)", + "airbyte_secret": true, + "multiline": true, + "order": 2, + "always_show": true + }, + "client_key": { + "type": "string", + "title": "Client key", + "description": "Client key (this is not a required field, but if you want to use it, you will need to add the Client certificate as well)", + "airbyte_secret": true, + "multiline": true, + "order": 3, + "always_show": true + }, + "client_key_password": { + "type": "string", + "title": "Client key password", + "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", + "airbyte_secret": true, + "order": 4 + } + } + }, + { + "title": "Verify Identity", + "description": "Always connect with SSL. Verify both CA and Hostname.", + "required": ["mode", "ca_certificate"], + "properties": { + "mode": { + "type": "string", + "const": "verify_identity", + "order": 0 + }, + "ca_certificate": { + "type": "string", + "title": "CA certificate", + "description": "CA certificate", + "airbyte_secret": true, + "multiline": true, + "order": 1 + }, + "client_certificate": { + "type": "string", + "title": "Client certificate", + "description": "Client certificate (this is not a required field, but if you want to use it, you will need to add the Client key as well)", + "airbyte_secret": true, + "multiline": true, + "order": 2, + "always_show": true + }, + "client_key": { + "type": "string", + "title": "Client key", + "description": "Client key (this is not a required field, but if you want to use it, you will need to add the Client certificate as well)", + "airbyte_secret": true, + "multiline": true, + "order": 3, + "always_show": true + }, + "client_key_password": { + "type": "string", + "title": "Client key password", + "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", + "airbyte_secret": true, + "order": 4 + } + } + } + ], + "default": "required" + }, + "replication_method": { + "type": "object", + "title": "Update Method", + "description": "Configures how data is extracted from the database.", + "order": 8, + "default": "CDC", + "display_type": "radio", + "oneOf": [ + { + "title": "Read Changes using Binary Log (CDC)", + "description": "Recommended - Incrementally reads new inserts, updates, and deletes using the MySQL binary log. This must be enabled on your database.", + "required": ["method"], + "properties": { + "method": { "type": "string", "const": "CDC", "order": 0 }, + "initial_waiting_seconds": { + "type": "integer", + "title": "Initial Waiting Time in Seconds (Advanced)", + "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", + "default": 300, + "min": 120, + "max": 1200, + "order": 1, + "always_show": true + }, + "server_time_zone": { + "type": "string", + "title": "Configured server timezone for the MySQL source (Advanced)", + "description": "Enter the configured MySQL server timezone. This should only be done if the configured timezone in your MySQL instance does not conform to IANNA standard.", + "order": 2, + "always_show": true + } + } + }, + { + "title": "Scan Changes with User Defined Cursor", + "description": "Incrementally detects new inserts and updates using the cursor column chosen when configuring a connection (e.g. created_at, updated_at).", + "required": ["method"], + "properties": { + "method": { "type": "string", "const": "STANDARD", "order": 0 } + } + } + ] + }, + "tunnel_method": { + "type": "object", + "title": "SSH Tunnel Method", + "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", + "oneOf": [ + { + "title": "No Tunnel", + "required": ["tunnel_method"], + "properties": { + "tunnel_method": { + "description": "No ssh tunnel needed to connect to database", + "type": "string", + "const": "NO_TUNNEL", + "order": 0 + } + } + }, + { + "title": "SSH Key Authentication", + "required": [ + "tunnel_method", + "tunnel_host", + "tunnel_port", + "tunnel_user", + "ssh_key" + ], + "properties": { + "tunnel_method": { + "description": "Connect through a jump server tunnel host using username and ssh key", + "type": "string", + "const": "SSH_KEY_AUTH", + "order": 0 + }, + "tunnel_host": { + "title": "SSH Tunnel Jump Server Host", + "description": "Hostname of the jump server host that allows inbound ssh tunnel.", + "type": "string", + "order": 1 + }, + "tunnel_port": { + "title": "SSH Connection Port", + "description": "Port on the proxy/jump server that accepts inbound ssh connections.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 22, + "examples": ["22"], + "order": 2 + }, + "tunnel_user": { + "title": "SSH Login Username", + "description": "OS-level username for logging into the jump server host.", + "type": "string", + "order": 3 + }, + "ssh_key": { + "title": "SSH Private Key", + "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", + "type": "string", + "airbyte_secret": true, + "multiline": true, + "order": 4 + } + } + }, + { + "title": "Password Authentication", + "required": [ + "tunnel_method", + "tunnel_host", + "tunnel_port", + "tunnel_user", + "tunnel_user_password" + ], + "properties": { + "tunnel_method": { + "description": "Connect through a jump server tunnel host using username and password authentication", + "type": "string", + "const": "SSH_PASSWORD_AUTH", + "order": 0 + }, + "tunnel_host": { + "title": "SSH Tunnel Jump Server Host", + "description": "Hostname of the jump server host that allows inbound ssh tunnel.", + "type": "string", + "order": 1 + }, + "tunnel_port": { + "title": "SSH Connection Port", + "description": "Port on the proxy/jump server that accepts inbound ssh connections.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 22, + "examples": ["22"], + "order": 2 + }, + "tunnel_user": { + "title": "SSH Login Username", + "description": "OS-level username for logging into the jump server host", + "type": "string", + "order": 3 + }, + "tunnel_user_password": { + "title": "Password", + "description": "OS-level password for logging into the jump server host", + "type": "string", + "airbyte_secret": true, + "order": 4 + } + } + } + ] + } + } + }, + "supportsNormalization": false, + "supportsDBT": false, + "supported_destination_sync_modes": [] +} diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/resources/expected_oss_spec.json b/airbyte-integrations/connectors/source-mysql/src/test-integration/resources/expected_oss_spec.json new file mode 100644 index 000000000000..1a884d8de813 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/resources/expected_oss_spec.json @@ -0,0 +1,348 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/mysql", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "MySql Source Spec", + "type": "object", + "required": ["host", "port", "database", "username", "replication_method"], + "properties": { + "host": { + "description": "The host name of the database.", + "title": "Host", + "type": "string", + "order": 0 + }, + "port": { + "description": "The port to connect to.", + "title": "Port", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 3306, + "examples": ["3306"], + "order": 1 + }, + "database": { + "description": "The database name.", + "title": "Database", + "type": "string", + "order": 2 + }, + "username": { + "description": "The username which is used to access the database.", + "title": "Username", + "type": "string", + "order": 3 + }, + "password": { + "description": "The password associated with the username.", + "title": "Password", + "type": "string", + "airbyte_secret": true, + "order": 4, + "always_show": true + }, + "jdbc_url_params": { + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", + "title": "JDBC URL Parameters (Advanced)", + "type": "string", + "order": 5 + }, + "ssl": { + "title": "SSL Connection", + "description": "Encrypt data using SSL.", + "type": "boolean", + "default": true, + "order": 6 + }, + "ssl_mode": { + "title": "SSL modes", + "description": "SSL connection modes. Read more in the docs.", + "type": "object", + "order": 7, + "oneOf": [ + { + "title": "preferred", + "description": "Automatically attempt SSL connection. If the MySQL server does not support SSL, continue with a regular connection.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "preferred", + "order": 0 + } + } + }, + { + "title": "required", + "description": "Always connect with SSL. If the MySQL server doesn’t support SSL, the connection will not be established. Certificate Authority (CA) and Hostname are not verified.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "required", + "order": 0 + } + } + }, + { + "title": "Verify CA", + "description": "Always connect with SSL. Verifies CA, but allows connection even if Hostname does not match.", + "required": ["mode", "ca_certificate"], + "properties": { + "mode": { + "type": "string", + "const": "verify_ca", + "order": 0 + }, + "ca_certificate": { + "type": "string", + "title": "CA certificate", + "description": "CA certificate", + "airbyte_secret": true, + "multiline": true, + "order": 1 + }, + "client_certificate": { + "type": "string", + "title": "Client certificate", + "description": "Client certificate (this is not a required field, but if you want to use it, you will need to add the Client key as well)", + "airbyte_secret": true, + "multiline": true, + "order": 2, + "always_show": true + }, + "client_key": { + "type": "string", + "title": "Client key", + "description": "Client key (this is not a required field, but if you want to use it, you will need to add the Client certificate as well)", + "airbyte_secret": true, + "multiline": true, + "order": 3, + "always_show": true + }, + "client_key_password": { + "type": "string", + "title": "Client key password", + "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", + "airbyte_secret": true, + "order": 4 + } + } + }, + { + "title": "Verify Identity", + "description": "Always connect with SSL. Verify both CA and Hostname.", + "required": ["mode", "ca_certificate"], + "properties": { + "mode": { + "type": "string", + "const": "verify_identity", + "order": 0 + }, + "ca_certificate": { + "type": "string", + "title": "CA certificate", + "description": "CA certificate", + "airbyte_secret": true, + "multiline": true, + "order": 1 + }, + "client_certificate": { + "type": "string", + "title": "Client certificate", + "description": "Client certificate (this is not a required field, but if you want to use it, you will need to add the Client key as well)", + "airbyte_secret": true, + "multiline": true, + "order": 2, + "always_show": true + }, + "client_key": { + "type": "string", + "title": "Client key", + "description": "Client key (this is not a required field, but if you want to use it, you will need to add the Client certificate as well)", + "airbyte_secret": true, + "multiline": true, + "order": 3, + "always_show": true + }, + "client_key_password": { + "type": "string", + "title": "Client key password", + "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", + "airbyte_secret": true, + "order": 4 + } + } + } + ] + }, + "replication_method": { + "type": "object", + "title": "Update Method", + "description": "Configures how data is extracted from the database.", + "order": 8, + "default": "CDC", + "display_type": "radio", + "oneOf": [ + { + "title": "Read Changes using Binary Log (CDC)", + "description": "Recommended - Incrementally reads new inserts, updates, and deletes using the MySQL binary log. This must be enabled on your database.", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "CDC", + "order": 0 + }, + "initial_waiting_seconds": { + "type": "integer", + "title": "Initial Waiting Time in Seconds (Advanced)", + "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", + "default": 300, + "min": 120, + "max": 1200, + "order": 1, + "always_show": true + }, + "server_time_zone": { + "type": "string", + "title": "Configured server timezone for the MySQL source (Advanced)", + "description": "Enter the configured MySQL server timezone. This should only be done if the configured timezone in your MySQL instance does not conform to IANNA standard.", + "order": 2, + "always_show": true + } + } + }, + { + "title": "Scan Changes with User Defined Cursor", + "description": "Incrementally detects new inserts and updates using the cursor column chosen when configuring a connection (e.g. created_at, updated_at).", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "STANDARD", + "order": 0 + } + } + } + ] + }, + "tunnel_method": { + "type": "object", + "title": "SSH Tunnel Method", + "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", + "oneOf": [ + { + "title": "No Tunnel", + "required": ["tunnel_method"], + "properties": { + "tunnel_method": { + "description": "No ssh tunnel needed to connect to database", + "type": "string", + "const": "NO_TUNNEL", + "order": 0 + } + } + }, + { + "title": "SSH Key Authentication", + "required": [ + "tunnel_method", + "tunnel_host", + "tunnel_port", + "tunnel_user", + "ssh_key" + ], + "properties": { + "tunnel_method": { + "description": "Connect through a jump server tunnel host using username and ssh key", + "type": "string", + "const": "SSH_KEY_AUTH", + "order": 0 + }, + "tunnel_host": { + "title": "SSH Tunnel Jump Server Host", + "description": "Hostname of the jump server host that allows inbound ssh tunnel.", + "type": "string", + "order": 1 + }, + "tunnel_port": { + "title": "SSH Connection Port", + "description": "Port on the proxy/jump server that accepts inbound ssh connections.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 22, + "examples": ["22"], + "order": 2 + }, + "tunnel_user": { + "title": "SSH Login Username", + "description": "OS-level username for logging into the jump server host.", + "type": "string", + "order": 3 + }, + "ssh_key": { + "title": "SSH Private Key", + "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", + "type": "string", + "airbyte_secret": true, + "multiline": true, + "order": 4 + } + } + }, + { + "title": "Password Authentication", + "required": [ + "tunnel_method", + "tunnel_host", + "tunnel_port", + "tunnel_user", + "tunnel_user_password" + ], + "properties": { + "tunnel_method": { + "description": "Connect through a jump server tunnel host using username and password authentication", + "type": "string", + "const": "SSH_PASSWORD_AUTH", + "order": 0 + }, + "tunnel_host": { + "title": "SSH Tunnel Jump Server Host", + "description": "Hostname of the jump server host that allows inbound ssh tunnel.", + "type": "string", + "order": 1 + }, + "tunnel_port": { + "title": "SSH Connection Port", + "description": "Port on the proxy/jump server that accepts inbound ssh connections.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 22, + "examples": ["22"], + "order": 2 + }, + "tunnel_user": { + "title": "SSH Login Username", + "description": "OS-level username for logging into the jump server host", + "type": "string", + "order": 3 + }, + "tunnel_user_password": { + "title": "Password", + "description": "OS-level password for logging into the jump server host", + "type": "string", + "airbyte_secret": true, + "order": 4 + } + } + } + ] + } + } + }, + "supported_destination_sync_modes": [] +} diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/resources/expected_spec.json b/airbyte-integrations/connectors/source-mysql/src/test-integration/resources/expected_spec.json deleted file mode 100644 index 584333ddb9ae..000000000000 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/resources/expected_spec.json +++ /dev/null @@ -1,347 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/mysql", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "MySql Source Spec", - "type": "object", - "required": ["host", "port", "database", "username", "replication_method"], - "properties": { - "host": { - "description": "The host name of the database.", - "title": "Host", - "type": "string", - "order": 0 - }, - "port": { - "description": "The port to connect to.", - "title": "Port", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 3306, - "examples": ["3306"], - "order": 1 - }, - "database": { - "description": "The database name.", - "title": "Database", - "type": "string", - "order": 2 - }, - "username": { - "description": "The username which is used to access the database.", - "title": "Username", - "type": "string", - "order": 3 - }, - "password": { - "description": "The password associated with the username.", - "title": "Password", - "type": "string", - "airbyte_secret": true, - "order": 4, - "always_show": true - }, - "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", - "title": "JDBC URL Parameters (Advanced)", - "type": "string", - "order": 5 - }, - "ssl": { - "title": "SSL Connection", - "description": "Encrypt data using SSL.", - "type": "boolean", - "default": true, - "order": 6 - }, - "ssl_mode": { - "title": "SSL modes", - "description": "SSL connection modes. Read more in the docs.", - "type": "object", - "order": 7, - "oneOf": [ - { - "title": "preferred", - "description": "Automatically attempt SSL connection. If the MySQL server does not support SSL, continue with a regular connection.", - "required": ["mode"], - "properties": { - "mode": { - "type": "string", - "const": "preferred", - "order": 0 - } - } - }, - { - "title": "required", - "description": "Always connect with SSL. If the MySQL server doesn’t support SSL, the connection will not be established. Certificate Authority (CA) and Hostname are not verified.", - "required": ["mode"], - "properties": { - "mode": { - "type": "string", - "const": "required", - "order": 0 - } - } - }, - { - "title": "Verify CA", - "description": "Always connect with SSL. Verifies CA, but allows connection even if Hostname does not match.", - "required": ["mode", "ca_certificate"], - "properties": { - "mode": { - "type": "string", - "const": "verify_ca", - "order": 0 - }, - "ca_certificate": { - "type": "string", - "title": "CA certificate", - "description": "CA certificate", - "airbyte_secret": true, - "multiline": true, - "order": 1 - }, - "client_certificate": { - "type": "string", - "title": "Client certificate", - "description": "Client certificate (this is not a required field, but if you want to use it, you will need to add the Client key as well)", - "airbyte_secret": true, - "multiline": true, - "order": 2, - "always_show": true - }, - "client_key": { - "type": "string", - "title": "Client key", - "description": "Client key (this is not a required field, but if you want to use it, you will need to add the Client certificate as well)", - "airbyte_secret": true, - "multiline": true, - "order": 3, - "always_show": true - }, - "client_key_password": { - "type": "string", - "title": "Client key password", - "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", - "airbyte_secret": true, - "order": 4 - } - } - }, - { - "title": "Verify Identity", - "description": "Always connect with SSL. Verify both CA and Hostname.", - "required": ["mode", "ca_certificate"], - "properties": { - "mode": { - "type": "string", - "const": "verify_identity", - "order": 0 - }, - "ca_certificate": { - "type": "string", - "title": "CA certificate", - "description": "CA certificate", - "airbyte_secret": true, - "multiline": true, - "order": 1 - }, - "client_certificate": { - "type": "string", - "title": "Client certificate", - "description": "Client certificate (this is not a required field, but if you want to use it, you will need to add the Client key as well)", - "airbyte_secret": true, - "multiline": true, - "order": 2, - "always_show": true - }, - "client_key": { - "type": "string", - "title": "Client key", - "description": "Client key (this is not a required field, but if you want to use it, you will need to add the Client certificate as well)", - "airbyte_secret": true, - "multiline": true, - "order": 3, - "always_show": true - }, - "client_key_password": { - "type": "string", - "title": "Client key password", - "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - }, - "replication_method": { - "type": "object", - "title": "Update Method", - "description": "Configures how data is extracted from the database.", - "order": 8, - "default": "CDC", - "oneOf": [ - { - "title": "Read Changes using Binary Log (CDC)", - "description": "Recommended - Incrementally reads new inserts, updates, and deletes using the MySQL binary log. This must be enabled on your database.", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "CDC", - "order": 0 - }, - "initial_waiting_seconds": { - "type": "integer", - "title": "Initial Waiting Time in Seconds (Advanced)", - "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", - "default": 300, - "min": 120, - "max": 1200, - "order": 1, - "always_show": true - }, - "server_time_zone": { - "type": "string", - "title": "Configured server timezone for the MySQL source (Advanced)", - "description": "Enter the configured MySQL server timezone. This should only be done if the configured timezone in your MySQL instance does not conform to IANNA standard.", - "order": 2, - "always_show": true - } - } - }, - { - "title": "Scan Changes with User Defined Cursor", - "description": "Incrementally detects new inserts and updates using the cursor column chosen when configuring a connection (e.g. created_at, updated_at).", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "STANDARD", - "order": 0 - } - } - } - ] - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - } - }, - "supported_destination_sync_modes": [] -} diff --git a/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/FillMySqlTestDbScriptTest.java b/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/FillMySqlTestDbScriptTest.java index 5d2f37b6a746..d9ff7f362056 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/FillMySqlTestDbScriptTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/FillMySqlTestDbScriptTest.java @@ -6,13 +6,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.JdbcConnector; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.cdk.integrations.standardtest.source.performancetest.AbstractSourceFillDbWithTestData; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.integrations.standardtest.source.performancetest.AbstractSourceFillDbWithTestData; import java.util.Map; import java.util.stream.Stream; import org.jooq.SQLDialect; @@ -60,7 +61,8 @@ protected Database setupDatabase(final String dbName) throws Exception { config.get(JdbcUtils.PORT_KEY).asInt(), config.get(JdbcUtils.DATABASE_KEY).asText()), SQLDialect.MYSQL, - Map.of("zeroDateTimeBehavior", "convertToNull"))); + Map.of("zeroDateTimeBehavior", "convertToNull"), + JdbcConnector.CONNECT_TIMEOUT_DEFAULT)); // It disable strict mode in the DB and allows to insert specific values. // For example, it's possible to insert date with zero values "2021-00-00" diff --git a/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/MySqlRdsSourcePerformanceSecretTest.java b/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/MySqlRdsSourcePerformanceSecretTest.java index 18acba05e699..fc9ab1e0e26d 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/MySqlRdsSourcePerformanceSecretTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/MySqlRdsSourcePerformanceSecretTest.java @@ -6,13 +6,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.JdbcConnector; +import io.airbyte.cdk.integrations.standardtest.source.performancetest.AbstractSourcePerformanceTest; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.performancetest.AbstractSourcePerformanceTest; import java.nio.file.Path; import java.util.Map; import java.util.stream.Stream; @@ -51,7 +52,8 @@ protected void setupDatabase(final String dbName) throws Exception { config.get(JdbcUtils.PORT_KEY).asInt(), config.get(JdbcUtils.DATABASE_KEY).asText()), SQLDialect.MYSQL, - Map.of("zeroDateTimeBehavior", "convertToNull"))) { + Map.of("zeroDateTimeBehavior", "convertToNull"), + JdbcConnector.CONNECT_TIMEOUT_DEFAULT)) { final Database database = new Database(dslContext); diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java index 869aa507db16..643acf6d0f72 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CdcMysqlSourceTest.java @@ -4,16 +4,20 @@ package io.airbyte.integrations.source.mysql; -import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; -import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; -import static io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MYSQL_CDC_OFFSET; -import static io.airbyte.integrations.debezium.internals.mysql.MySqlDebeziumStateUtil.MYSQL_DB_HISTORY; +import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_RECORDS_PROPERTY; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.mysql.MysqlCdcStateConstants.IS_COMPRESSED; +import static io.airbyte.cdk.integrations.debezium.internals.mysql.MysqlCdcStateConstants.MYSQL_CDC_OFFSET; +import static io.airbyte.cdk.integrations.debezium.internals.mysql.MysqlCdcStateConstants.MYSQL_DB_HISTORY; import static io.airbyte.integrations.source.mysql.MySqlSource.CDC_DEFAULT_CURSOR; import static io.airbyte.integrations.source.mysql.MySqlSource.CDC_LOG_FILE; import static io.airbyte.integrations.source.mysql.MySqlSource.CDC_LOG_POS; -import static io.airbyte.integrations.source.mysql.MySqlSource.DRIVER_CLASS; +import static io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadStateManager.PRIMARY_KEY_STATE_TYPE; +import static io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadStateManager.STATE_TYPE_KEY; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -22,130 +26,95 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.features.EnvVariableFeatureFlags; +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; +import com.google.common.collect.Streams; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.debezium.CdcSourceTest; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteSchemaHistoryStorage; +import io.airbyte.cdk.integrations.debezium.internals.mysql.MySqlCdcTargetPosition; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.debezium.CdcSourceTest; -import io.airbyte.integrations.debezium.internals.mysql.MySqlCdcTargetPosition; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.ContainerModifier; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; +import io.airbyte.protocol.models.v0.AirbyteGlobalState; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.AirbyteStreamState; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.StreamDescriptor; import io.airbyte.protocol.models.v0.SyncMode; -import java.sql.SQLException; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.Random; import java.util.Set; -import javax.sql.DataSource; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testcontainers.containers.MySQLContainer; -import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; -import uk.org.webcompere.systemstubs.jupiter.SystemStub; -import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; - -@ExtendWith(SystemStubsExtension.class) -public class CdcMysqlSourceTest extends CdcSourceTest { - - @SystemStub - private EnvironmentVariables environmentVariables; - - private static final String DB_NAME = MODELS_SCHEMA; - private MySQLContainer container; - private Database database; - private MySqlSource source; - private JsonNode config; - - @BeforeEach - public void setup() throws SQLException { - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - init(); - revokeAllPermissions(); - grantCorrectPermissions(); - super.setup(); +import org.junit.jupiter.api.Timeout; + +@Order(1) +public class CdcMysqlSourceTest extends CdcSourceTest { + + private static final String INVALID_TIMEZONE_CEST = "CEST"; + + private static final Random RANDOM = new Random(); + + @Override + protected MySQLTestDatabase createTestDatabase() { + return MySQLTestDatabase.in(BaseImage.MYSQL_8, ContainerModifier.INVALID_TIMEZONE_CEST).withCdcPermissions(); } - private void init() { - container = new MySQLContainer<>("mysql:8.0"); - container.start(); - source = new MySqlSource(); - database = new Database(DSLContextFactory.create( - "root", - "test", - DRIVER_CLASS, - String.format("jdbc:mysql://%s:%s", - container.getHost(), - container.getFirstMappedPort()), - SQLDialect.MYSQL)); - - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "CDC") - .put("initial_waiting_seconds", INITIAL_WAITING_SECONDS) - .put("time_zone", "America/Los_Angeles") - .build()); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put("host", container.getHost()) - .put("port", container.getFirstMappedPort()) - .put("database", DB_NAME) - .put("username", container.getUsername()) - .put("password", container.getPassword()) - .put("replication_method", replicationMethod) - .put("sync_checkpoint_records", 1) - .put("is_test", true) - .build()); + @Override + protected MySqlSource source() { + return new MySqlSource(); } - private void revokeAllPermissions() { - executeQuery("REVOKE ALL PRIVILEGES, GRANT OPTION FROM " + container.getUsername() + "@'%';"); + @Override + protected JsonNode config() { + return testdb.testConfigBuilder() + .withCdcReplication() + .with(SYNC_CHECKPOINT_RECORDS_PROPERTY, 1) + .build(); } - private void revokeReplicationClientPermission() { - executeQuery("REVOKE REPLICATION CLIENT ON *.* FROM " + container.getUsername() + "@'%';"); + protected void purgeAllBinaryLogs() { + testdb.with("RESET MASTER;"); } - private void grantCorrectPermissions() { - executeQuery("GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO " + container.getUsername() + "@'%';"); + @Override + protected String createSchemaSqlFmt() { + return "CREATE DATABASE IF NOT EXISTS %s;"; } - protected void purgeAllBinaryLogs() { - executeQuery("RESET MASTER;"); + @Override + protected String modelsSchema() { + return testdb.getDatabaseName(); } - @AfterEach - public void tearDown() { - try { - container.close(); - } catch (final Exception e) { - throw new RuntimeException(e); - } + @Override + protected String randomSchema() { + return testdb.getDatabaseName(); } @Override protected MySqlCdcTargetPosition cdcLatestTargetPosition() { - final DataSource dataSource = DataSourceFactory.create( - "root", - "test", - DRIVER_CLASS, - String.format("jdbc:mysql://%s:%s", - container.getHost(), - container.getFirstMappedPort()), - Collections.emptyMap()); - final JdbcDatabase jdbcDatabase = new DefaultJdbcDatabase(dataSource); - - return MySqlCdcTargetPosition.targetPosition(jdbcDatabase); + return MySqlCdcTargetPosition.targetPosition(new DefaultJdbcDatabase(testdb.getDataSource())); } @Override @@ -206,40 +175,10 @@ protected void addCdcDefaultCursorField(final AirbyteStream stream) { } } - @Override - protected Source getSource() { - return source; - } - - @Override - protected JsonNode getConfig() { - return config; - } - - @Override - protected Database getDatabase() { - return database; - } - - @Override - protected void assertExpectedStateMessages(final List stateMessages) { - assertEquals(1, stateMessages.size()); - assertNotNull(stateMessages.get(0).getData()); - for (final AirbyteStateMessage stateMessage : stateMessages) { - assertNotNull(stateMessage.getData().get("cdc_state").get("state").get(MYSQL_CDC_OFFSET)); - assertNotNull(stateMessage.getData().get("cdc_state").get("state").get(MYSQL_DB_HISTORY)); - } - } - - @Override - protected String randomTableSchema() { - return MODELS_SCHEMA; - } - @Test protected void syncWithReplicationClientPrivilegeRevokedFailsCheck() throws Exception { - revokeReplicationClientPermission(); - final AirbyteConnectionStatus status = getSource().check(getConfig()); + testdb.with("REVOKE REPLICATION CLIENT ON *.* FROM %s@'%%';", testdb.getUserName()); + final AirbyteConnectionStatus status = source().check(config()); final String expectedErrorMessage = "Please grant REPLICATION CLIENT privilege, so that binary log files are available" + " for CDC mode."; assertTrue(status.getStatus().equals(Status.FAILED)); @@ -259,8 +198,8 @@ protected void syncShouldHandlePurgedLogsGracefully() throws Exception { writeModelRecord(record); } - final AutoCloseableIterator firstBatchIterator = getSource() - .read(getConfig(), CONFIGURED_CATALOG, null); + final AutoCloseableIterator firstBatchIterator = source() + .read(config(), getConfiguredCatalog(), null); final List dataFromFirstBatch = AutoCloseableIterators .toListAndClose(firstBatchIterator); final List stateAfterFirstBatch = extractStateMessages(dataFromFirstBatch); @@ -290,8 +229,8 @@ protected void syncShouldHandlePurgedLogsGracefully() throws Exception { purgeAllBinaryLogs(); final JsonNode state = Jsons.jsonNode(Collections.singletonList(stateAfterFirstBatch.get(stateAfterFirstBatch.size() - 1))); - final AutoCloseableIterator secondBatchIterator = getSource() - .read(getConfig(), CONFIGURED_CATALOG, state); + final AutoCloseableIterator secondBatchIterator = source() + .read(config(), getConfiguredCatalog(), state); final List dataFromSecondBatch = AutoCloseableIterators .toListAndClose(secondBatchIterator); @@ -314,10 +253,10 @@ protected void syncShouldHandlePurgedLogsGracefully() throws Exception { @Test protected void verifyCheckpointStatesByRecords() throws Exception { // We require a huge amount of records, otherwise Debezium will notify directly the last offset. - final int recordsToCreate = 20000; + final int recordsToCreate = 20_000; - final AutoCloseableIterator firstBatchIterator = getSource() - .read(getConfig(), CONFIGURED_CATALOG, null); + final AutoCloseableIterator firstBatchIterator = source() + .read(config(), getConfiguredCatalog(), null); final List dataFromFirstBatch = AutoCloseableIterators .toListAndClose(firstBatchIterator); final List stateMessages = extractStateMessages(dataFromFirstBatch); @@ -327,16 +266,14 @@ protected void verifyCheckpointStatesByRecords() throws Exception { assertExpectedStateMessages(stateMessages); for (int recordsCreated = 0; recordsCreated < recordsToCreate; recordsCreated++) { - final JsonNode record = - Jsons.jsonNode(ImmutableMap - .of(COL_ID, 200 + recordsCreated, COL_MAKE_ID, 1, COL_MODEL, - "F-" + recordsCreated)); + final JsonNode record = Jsons.jsonNode(ImmutableMap + .of(COL_ID, 200 + recordsCreated, COL_MAKE_ID, 1, COL_MODEL, "F-" + recordsCreated)); writeModelRecord(record); } final JsonNode stateAfterFirstSync = Jsons.jsonNode(Collections.singletonList(stateMessages.get(stateMessages.size() - 1))); - final AutoCloseableIterator secondBatchIterator = getSource() - .read(getConfig(), CONFIGURED_CATALOG, stateAfterFirstSync); + final AutoCloseableIterator secondBatchIterator = source() + .read(config(), getConfiguredCatalog(), stateAfterFirstSync); final List dataFromSecondBatch = AutoCloseableIterators .toListAndClose(secondBatchIterator); assertEquals(recordsToCreate, extractRecordMessages(dataFromSecondBatch).size()); @@ -345,8 +282,437 @@ protected void verifyCheckpointStatesByRecords() throws Exception { assertEquals(stateMessagesCDC.size(), stateMessagesCDC.stream().distinct().count(), "There are duplicated states."); } - protected void assertStateForSyncShouldHandlePurgedLogsGracefully(final List stateMessages, final int syncNumber) { + @Override + protected void assertExpectedStateMessages(final List stateMessages) { + assertEquals(7, stateMessages.size()); + assertStateTypes(stateMessages, 4); + } + + @Override + protected void assertExpectedStateMessagesWithTotalCount(final List stateMessages, final long totalRecordCount) { + long actualRecordCount = 0L; + for (final AirbyteStateMessage message : stateMessages) { + actualRecordCount += message.getSourceStats().getRecordCount(); + } + assertEquals(actualRecordCount, totalRecordCount); + } + + @Override + protected void assertExpectedStateMessagesFromIncrementalSync(final List stateMessages) { + assertEquals(1, stateMessages.size()); + assertNotNull(stateMessages.get(0).getData()); + for (final AirbyteStateMessage stateMessage : stateMessages) { + assertNotNull(stateMessage.getData().get("cdc_state").get("state").get(MYSQL_CDC_OFFSET)); + assertNotNull(stateMessage.getData().get("cdc_state").get("state").get(MYSQL_DB_HISTORY)); + } + } + + private void assertStateForSyncShouldHandlePurgedLogsGracefully(final List stateMessages, final int syncNumber) { + if (syncNumber == 1) { + assertExpectedStateMessagesForRecordsProducedDuringAndAfterSync(stateMessages); + } else if (syncNumber == 2) { + // Sync number 2 uses the state from sync number 1 but before we trigger the sync 2 we purge the + // binary logs and as a result the validation of + // logs present on the server fails, and we trigger a sync from scratch + assertEquals(47, stateMessages.size()); + assertStateTypes(stateMessages, 44); + } else { + throw new RuntimeException("Unknown sync number"); + } + + } + + @Override + protected void assertExpectedStateMessagesForRecordsProducedDuringAndAfterSync(final List stateAfterFirstBatch) { + assertEquals(27, stateAfterFirstBatch.size()); + assertStateTypes(stateAfterFirstBatch, 24); + } + + @Override + protected void assertExpectedStateMessagesForNoData(final List stateMessages) { + assertEquals(2, stateMessages.size()); + } + + private void assertStateTypes(final List stateMessages, final int indexTillWhichExpectPkState) { + JsonNode sharedState = null; + for (int i = 0; i < stateMessages.size(); i++) { + final AirbyteStateMessage stateMessage = stateMessages.get(i); + assertEquals(AirbyteStateType.GLOBAL, stateMessage.getType()); + final AirbyteGlobalState global = stateMessage.getGlobal(); + assertNotNull(global.getSharedState()); + if (Objects.isNull(sharedState)) { + sharedState = global.getSharedState(); + } else { + assertEquals(sharedState, global.getSharedState()); + } + assertEquals(1, global.getStreamStates().size()); + final AirbyteStreamState streamState = global.getStreamStates().get(0); + if (i <= indexTillWhichExpectPkState) { + assertTrue(streamState.getStreamState().has(STATE_TYPE_KEY)); + assertEquals(PRIMARY_KEY_STATE_TYPE, streamState.getStreamState().get(STATE_TYPE_KEY).asText()); + } else { + assertFalse(streamState.getStreamState().has(STATE_TYPE_KEY)); + } + } + } + + @Override + protected void assertStateMessagesForNewTableSnapshotTest(final List stateMessages, + final AirbyteStateMessage stateMessageEmittedAfterFirstSyncCompletion) { + assertEquals(7, stateMessages.size()); + for (int i = 0; i <= 4; i++) { + final AirbyteStateMessage stateMessage = stateMessages.get(i); + assertEquals(AirbyteStateType.GLOBAL, stateMessage.getType()); + assertEquals(stateMessageEmittedAfterFirstSyncCompletion.getGlobal().getSharedState(), + stateMessage.getGlobal().getSharedState()); + final Set streamsInSnapshotState = stateMessage.getGlobal().getStreamStates() + .stream() + .map(AirbyteStreamState::getStreamDescriptor) + .collect(Collectors.toSet()); + assertEquals(2, streamsInSnapshotState.size()); + assertTrue( + streamsInSnapshotState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME + "_random").withNamespace(randomSchema()))); + assertTrue(streamsInSnapshotState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(testdb.getDatabaseName()))); + + stateMessage.getGlobal().getStreamStates().forEach(s -> { + final JsonNode streamState = s.getStreamState(); + if (s.getStreamDescriptor().equals(new StreamDescriptor().withName(MODELS_STREAM_NAME + "_random").withNamespace(randomSchema()))) { + assertEquals(PRIMARY_KEY_STATE_TYPE, streamState.get(STATE_TYPE_KEY).asText()); + } else if (s.getStreamDescriptor().equals(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(testdb.getDatabaseName()))) { + assertFalse(streamState.has(STATE_TYPE_KEY)); + } else { + throw new RuntimeException("Unknown stream"); + } + }); + } + + final AirbyteStateMessage secondLastSateMessage = stateMessages.get(5); + assertEquals(AirbyteStateType.GLOBAL, secondLastSateMessage.getType()); + assertEquals(stateMessageEmittedAfterFirstSyncCompletion.getGlobal().getSharedState(), + secondLastSateMessage.getGlobal().getSharedState()); + final Set streamsInSnapshotState = secondLastSateMessage.getGlobal().getStreamStates() + .stream() + .map(AirbyteStreamState::getStreamDescriptor) + .collect(Collectors.toSet()); + assertEquals(2, streamsInSnapshotState.size()); + assertTrue( + streamsInSnapshotState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME + "_random").withNamespace(randomSchema()))); + assertTrue(streamsInSnapshotState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(testdb.getDatabaseName()))); + secondLastSateMessage.getGlobal().getStreamStates().forEach(s -> { + final JsonNode streamState = s.getStreamState(); + assertFalse(streamState.has(STATE_TYPE_KEY)); + }); + + final AirbyteStateMessage stateMessageEmittedAfterSecondSyncCompletion = stateMessages.get(6); + assertEquals(AirbyteStateType.GLOBAL, stateMessageEmittedAfterSecondSyncCompletion.getType()); + assertNotEquals(stateMessageEmittedAfterFirstSyncCompletion.getGlobal().getSharedState(), + stateMessageEmittedAfterSecondSyncCompletion.getGlobal().getSharedState()); + final Set streamsInSyncCompletionState = stateMessageEmittedAfterSecondSyncCompletion.getGlobal().getStreamStates() + .stream() + .map(AirbyteStreamState::getStreamDescriptor) + .collect(Collectors.toSet()); + assertEquals(2, streamsInSnapshotState.size()); + assertTrue( + streamsInSyncCompletionState.contains( + new StreamDescriptor().withName(MODELS_STREAM_NAME + "_random").withNamespace(randomSchema()))); + assertTrue(streamsInSyncCompletionState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(testdb.getDatabaseName()))); + assertNotNull(stateMessageEmittedAfterSecondSyncCompletion.getData()); + } + + @Test + @Timeout(value = 60) + public void syncWouldWorkWithDBWithInvalidTimezone() throws Exception { + final String systemTimeZone = "@@system_time_zone"; + final JdbcDatabase jdbcDatabase = source().createDatabase(config()); + final Properties properties = MySqlCdcProperties.getDebeziumProperties(jdbcDatabase); + final String databaseTimezone = jdbcDatabase.unsafeQuery(String.format("SELECT %s;", systemTimeZone)).toList().get(0).get(systemTimeZone) + .asText(); + final String debeziumEngineTimezone = properties.getProperty("database.connectionTimeZone"); + + assertEquals(INVALID_TIMEZONE_CEST, databaseTimezone); + assertEquals("America/Los_Angeles", debeziumEngineTimezone); + + final AutoCloseableIterator read = source() + .read(config(), getConfiguredCatalog(), null); + + final List actualRecords = AutoCloseableIterators.toListAndClose(read); + + final Set recordMessages = extractRecordMessages(actualRecords); + final List stateMessages = extractStateMessages(actualRecords); + + assertExpectedRecords(new HashSet<>(MODEL_RECORDS), recordMessages); assertExpectedStateMessages(stateMessages); + assertExpectedStateMessagesWithTotalCount(stateMessages, 6); + } + + @Test + public void testCompositeIndexInitialLoad() throws Exception { + // Simulate adding a composite index by modifying the catalog. + final ConfiguredAirbyteCatalog configuredCatalog = Jsons.clone(getConfiguredCatalog()); + final List> primaryKeys = configuredCatalog.getStreams().get(0).getStream().getSourceDefinedPrimaryKey(); + primaryKeys.add(List.of("make_id")); + + final AutoCloseableIterator read1 = source() + .read(config(), configuredCatalog, null); + + final List actualRecords1 = AutoCloseableIterators.toListAndClose(read1); + + final Set recordMessages1 = extractRecordMessages(actualRecords1); + final List stateMessages1 = extractStateMessages(actualRecords1); + assertExpectedRecords(new HashSet<>(MODEL_RECORDS), recordMessages1); + assertExpectedStateMessages(stateMessages1); + assertExpectedStateMessagesWithTotalCount(stateMessages1, 6); + + // Re-run the sync with state associated with record w/ id = 15 (second to last record). + // We expect to read 2 records, since in the case of a composite PK we issue a >= query. + // We also expect 3 state records. One associated with the pk state, one to signify end of initial + // load, and + // the last one indicating the cdc position we have synced until. + final JsonNode state = Jsons.jsonNode(Collections.singletonList(stateMessages1.get(4))); + final AutoCloseableIterator read2 = source() + .read(config(), configuredCatalog, state); + + final List actualRecords2 = AutoCloseableIterators.toListAndClose(read2); + final Set recordMessages2 = extractRecordMessages(actualRecords2); + final List stateMessages2 = extractStateMessages(actualRecords2); + + assertExpectedRecords(new HashSet<>(MODEL_RECORDS.subList(4, 6)), recordMessages2); + assertEquals(3, stateMessages2.size()); + assertStateTypes(stateMessages2, 0); + } + + @Test + public void testTwoStreamSync() throws Exception { + // Add another stream models_2 and read that one as well. + final ConfiguredAirbyteCatalog configuredCatalog = Jsons.clone(getConfiguredCatalog()); + + final List MODEL_RECORDS_2 = ImmutableList.of( + Jsons.jsonNode(ImmutableMap.of(COL_ID, 110, COL_MAKE_ID, 1, COL_MODEL, "Fiesta-2")), + Jsons.jsonNode(ImmutableMap.of(COL_ID, 120, COL_MAKE_ID, 1, COL_MODEL, "Focus-2")), + Jsons.jsonNode(ImmutableMap.of(COL_ID, 130, COL_MAKE_ID, 1, COL_MODEL, "Ranger-2")), + Jsons.jsonNode(ImmutableMap.of(COL_ID, 140, COL_MAKE_ID, 2, COL_MODEL, "GLA-2")), + Jsons.jsonNode(ImmutableMap.of(COL_ID, 150, COL_MAKE_ID, 2, COL_MODEL, "A 220-2")), + Jsons.jsonNode(ImmutableMap.of(COL_ID, 160, COL_MAKE_ID, 2, COL_MODEL, "E 350-2"))); + + testdb.with(createTableSqlFmt(), testdb.getDatabaseName(), MODELS_STREAM_NAME + "_2", + columnClause(ImmutableMap.of(COL_ID, "INTEGER", COL_MAKE_ID, "INTEGER", COL_MODEL, "VARCHAR(200)"), Optional.of(COL_ID))); + + for (final JsonNode recordJson : MODEL_RECORDS_2) { + writeRecords(recordJson, testdb.getDatabaseName(), MODELS_STREAM_NAME + "_2", COL_ID, + COL_MAKE_ID, COL_MODEL); + } + + final ConfiguredAirbyteStream airbyteStream = new ConfiguredAirbyteStream() + .withStream(CatalogHelpers.createAirbyteStream( + MODELS_STREAM_NAME + "_2", + testdb.getDatabaseName(), + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_MAKE_ID, JsonSchemaType.INTEGER), + Field.of(COL_MODEL, JsonSchemaType.STRING)) + .withSupportedSyncModes( + Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID)))); + airbyteStream.setSyncMode(SyncMode.INCREMENTAL); + + final List streams = configuredCatalog.getStreams(); + streams.add(airbyteStream); + configuredCatalog.withStreams(streams); + + final AutoCloseableIterator read1 = source() + .read(config(), configuredCatalog, null); + final List actualRecords1 = AutoCloseableIterators.toListAndClose(read1); + + final Set recordMessages1 = extractRecordMessages(actualRecords1); + final List stateMessages1 = extractStateMessages(actualRecords1); + assertEquals(13, stateMessages1.size()); + assertExpectedStateMessagesWithTotalCount(stateMessages1, 12); + + JsonNode sharedState = null; + StreamDescriptor firstStreamInState = null; + for (int i = 0; i < stateMessages1.size(); i++) { + final AirbyteStateMessage stateMessage = stateMessages1.get(i); + assertEquals(AirbyteStateType.GLOBAL, stateMessage.getType()); + final AirbyteGlobalState global = stateMessage.getGlobal(); + assertNotNull(global.getSharedState()); + if (Objects.isNull(sharedState)) { + sharedState = global.getSharedState(); + } else { + assertEquals(sharedState, global.getSharedState()); + } + + if (Objects.isNull(firstStreamInState)) { + assertEquals(1, global.getStreamStates().size()); + firstStreamInState = global.getStreamStates().get(0).getStreamDescriptor(); + } + + if (i <= 4) { + // First 4 state messages are pk state + assertEquals(1, global.getStreamStates().size()); + final AirbyteStreamState streamState = global.getStreamStates().get(0); + assertTrue(streamState.getStreamState().has(STATE_TYPE_KEY)); + assertEquals(PRIMARY_KEY_STATE_TYPE, streamState.getStreamState().get(STATE_TYPE_KEY).asText()); + } else if (i == 5) { + // 5th state message is the final state message emitted for the stream + assertEquals(1, global.getStreamStates().size()); + final AirbyteStreamState streamState = global.getStreamStates().get(0); + assertFalse(streamState.getStreamState().has(STATE_TYPE_KEY)); + } else if (i <= 10) { + // 6th to 10th is the primary_key state message for the 2nd stream but final state message for 1st + // stream + assertEquals(2, global.getStreamStates().size()); + final StreamDescriptor finalFirstStreamInState = firstStreamInState; + global.getStreamStates().forEach(c -> { + if (c.getStreamDescriptor().equals(finalFirstStreamInState)) { + assertFalse(c.getStreamState().has(STATE_TYPE_KEY)); + } else { + assertTrue(c.getStreamState().has(STATE_TYPE_KEY)); + assertEquals(PRIMARY_KEY_STATE_TYPE, c.getStreamState().get(STATE_TYPE_KEY).asText()); + } + }); + } else { + // last 2 state messages don't contain primary_key info cause primary_key sync should be complete + assertEquals(2, global.getStreamStates().size()); + global.getStreamStates().forEach(c -> assertFalse(c.getStreamState().has(STATE_TYPE_KEY))); + } + } + + final Set names = new HashSet<>(STREAM_NAMES); + names.add(MODELS_STREAM_NAME + "_2"); + assertExpectedRecords(Streams.concat(MODEL_RECORDS_2.stream(), MODEL_RECORDS.stream()) + .collect(Collectors.toSet()), + recordMessages1, + names, + names, + testdb.getDatabaseName()); + + assertEquals(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(testdb.getDatabaseName()), firstStreamInState); + + // Triggering a sync with a primary_key state for 1 stream and complete state for other stream + final AutoCloseableIterator read2 = source() + .read(config(), configuredCatalog, Jsons.jsonNode(Collections.singletonList(stateMessages1.get(6)))); + final List actualRecords2 = AutoCloseableIterators.toListAndClose(read2); + + final List stateMessages2 = extractStateMessages(actualRecords2); + + assertEquals(6, stateMessages2.size()); + // State was reset to the 7th; thus 5 remaining records were expected to be reloaded. + assertExpectedStateMessagesWithTotalCount(stateMessages2, 5); + for (int i = 0; i < stateMessages2.size(); i++) { + final AirbyteStateMessage stateMessage = stateMessages2.get(i); + assertEquals(AirbyteStateType.GLOBAL, stateMessage.getType()); + final AirbyteGlobalState global = stateMessage.getGlobal(); + assertNotNull(global.getSharedState()); + assertEquals(2, global.getStreamStates().size()); + + if (i <= 3) { + final StreamDescriptor finalFirstStreamInState = firstStreamInState; + global.getStreamStates().forEach(c -> { + // First 4 state messages are primary_key state for the stream that didn't complete primary_key sync + // the first time + if (c.getStreamDescriptor().equals(finalFirstStreamInState)) { + assertFalse(c.getStreamState().has(STATE_TYPE_KEY)); + } else { + assertTrue(c.getStreamState().has(STATE_TYPE_KEY)); + assertEquals(PRIMARY_KEY_STATE_TYPE, c.getStreamState().get(STATE_TYPE_KEY).asText()); + } + }); + } else { + // last 2 state messages don't contain primary_key info cause primary_key sync should be complete + global.getStreamStates().forEach(c -> assertFalse(c.getStreamState().has(STATE_TYPE_KEY))); + } + } + + final Set recordMessages2 = extractRecordMessages(actualRecords2); + assertEquals(5, recordMessages2.size()); + assertExpectedRecords(new HashSet<>(MODEL_RECORDS_2.subList(1, MODEL_RECORDS_2.size())), + recordMessages2, + names, + names, + testdb.getDatabaseName()); + } + + /** + * This test creates lots of tables increasing the schema history size above the limit of + * {@link AirbyteSchemaHistoryStorage#SIZE_LIMIT_TO_COMPRESS_MB} forcing the + * {@link AirbyteSchemaHistoryStorage#read()} method to compress the schema history blob as part of + * the state message which allows us to test that the next sync is able to work fine when provided + * with a compressed blob in the state. + */ + @Test + public void testCompressedSchemaHistory() throws Exception { + createTablesToIncreaseSchemaHistorySize(); + final AutoCloseableIterator firstBatchIterator = source() + .read(config(), getConfiguredCatalog(), null); + final List dataFromFirstBatch = AutoCloseableIterators + .toListAndClose(firstBatchIterator); + final AirbyteStateMessage lastStateMessageFromFirstBatch = Iterables.getLast(extractStateMessages(dataFromFirstBatch)); + assertNotNull(lastStateMessageFromFirstBatch.getGlobal().getSharedState()); + assertNotNull(lastStateMessageFromFirstBatch.getGlobal().getSharedState().get("state")); + assertNotNull(lastStateMessageFromFirstBatch.getGlobal().getSharedState().get("state").get(IS_COMPRESSED)); + assertNotNull(lastStateMessageFromFirstBatch.getGlobal().getSharedState().get("state").get(MYSQL_DB_HISTORY)); + assertNotNull(lastStateMessageFromFirstBatch.getGlobal().getSharedState().get("state").get(MYSQL_CDC_OFFSET)); + assertTrue(lastStateMessageFromFirstBatch.getGlobal().getSharedState().get("state").get(IS_COMPRESSED).asBoolean()); + + // INSERT records so that events are written to binlog and Debezium tries to parse them + final int recordsToCreate = 20; + // first batch of records. 20 created here and 6 created in setup method. + for (int recordsCreated = 0; recordsCreated < recordsToCreate; recordsCreated++) { + final JsonNode record = + Jsons.jsonNode(ImmutableMap + .of(COL_ID, 100 + recordsCreated, COL_MAKE_ID, 1, COL_MODEL, + "F-" + recordsCreated)); + writeModelRecord(record); + } + + final AutoCloseableIterator secondBatchIterator = source() + .read(config(), getConfiguredCatalog(), Jsons.jsonNode(Collections.singletonList(lastStateMessageFromFirstBatch))); + final List dataFromSecondBatch = AutoCloseableIterators + .toListAndClose(secondBatchIterator); + final AirbyteStateMessage lastStateMessageFromSecondBatch = Iterables.getLast(extractStateMessages(dataFromSecondBatch)); + assertNotNull(lastStateMessageFromSecondBatch.getGlobal().getSharedState()); + assertNotNull(lastStateMessageFromSecondBatch.getGlobal().getSharedState().get("state")); + assertNotNull(lastStateMessageFromSecondBatch.getGlobal().getSharedState().get("state").get(IS_COMPRESSED)); + assertNotNull(lastStateMessageFromSecondBatch.getGlobal().getSharedState().get("state").get(MYSQL_DB_HISTORY)); + assertNotNull(lastStateMessageFromSecondBatch.getGlobal().getSharedState().get("state").get(MYSQL_CDC_OFFSET)); + assertTrue(lastStateMessageFromSecondBatch.getGlobal().getSharedState().get("state").get(IS_COMPRESSED).asBoolean()); + + assertEquals(lastStateMessageFromFirstBatch.getGlobal().getSharedState().get("state").get(MYSQL_DB_HISTORY), + lastStateMessageFromSecondBatch.getGlobal().getSharedState().get("state").get(MYSQL_DB_HISTORY)); + + assertEquals(recordsToCreate, extractRecordMessages(dataFromSecondBatch).size()); + } + + private void createTablesToIncreaseSchemaHistorySize() { + for (int i = 0; i <= 200; i++) { + final String tableName = generateRandomStringOf32Characters(); + final StringBuilder createTableQuery = new StringBuilder("CREATE TABLE " + tableName + "("); + String firstCol = null; + for (int j = 1; j <= 250; j++) { + final String columnName = generateRandomStringOf32Characters(); + if (j == 1) { + firstCol = columnName; + + } + createTableQuery.append(columnName).append(" INTEGER, "); + } + createTableQuery.append("PRIMARY KEY (").append(firstCol).append("));"); + testdb.with(createTableQuery.toString()); + } + } + + private static String generateRandomStringOf32Characters() { + final String characters = "abcdefghijklmnopqrstuvwxyz"; + final int length = 32; + + final StringBuilder randomString = new StringBuilder(length); + + for (int i = 0; i < length; i++) { + final int index = RANDOM.nextInt(characters.length()); + final char randomChar = characters.charAt(index); + randomString.append(randomChar); + } + + return randomString.toString(); } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CloudDeploymentMySqlSslTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CloudDeploymentMySqlSslTest.java new file mode 100644 index 000000000000..92fded997c4d --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/CloudDeploymentMySqlSslTest.java @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.base.ssh.SshBastionContainer; +import io.airbyte.cdk.integrations.base.ssh.SshHelpers; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import io.airbyte.commons.features.EnvVariableFeatureFlags; +import io.airbyte.commons.features.FeatureFlagsWrapper; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import io.airbyte.protocol.models.v0.ConnectorSpecification; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +@Execution(ExecutionMode.CONCURRENT) +public class CloudDeploymentMySqlSslTest { + + private MySQLTestDatabase createTestDatabase(String... containerFactoryMethods) { + final var container = new MySQLContainerFactory().shared("mysql:8.0", containerFactoryMethods); + return new MySQLTestDatabase(container) + .withConnectionProperty("useSSL", "true") + .withConnectionProperty("requireSSL", "true") + .initialized(); + } + + private Source source() { + final var source = new MySqlSource(); + source.setFeatureFlags(FeatureFlagsWrapper.overridingDeploymentMode(new EnvVariableFeatureFlags(), "CLOUD")); + return MySqlSource.sshWrappedSource(source); + } + + @Test + void testSpec() throws Exception { + final ConnectorSpecification actual = source().spec(); + final ConnectorSpecification expected = + SshHelpers.injectSshIntoSpec(Jsons.deserialize(MoreResources.readResource("expected_cloud_spec.json"), ConnectorSpecification.class)); + assertEquals(expected, actual); + } + + @Test + void testStrictSSLUnsecuredNoTunnel() throws Exception { + try (final var testdb = createTestDatabase()) { + final var config = testdb.configBuilder() + .withHostAndPort() + .withDatabase() + .with(JdbcUtils.USERNAME_KEY, testdb.getUserName()) + .with(JdbcUtils.PASSWORD_KEY, "fake") + .with("tunnel_method", ImmutableMap.builder().put("tunnel_method", "NO_TUNNEL").build()) + .withSsl(ImmutableMap.builder() + .put(JdbcUtils.MODE_KEY, "preferred") + .build()) + .build(); + final AirbyteConnectionStatus actual = source().check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, actual.getStatus()); + assertTrue(actual.getMessage().contains("Unsecured connection not allowed"), actual.getMessage()); + } + } + + @Test + void testStrictSSLSecuredNoTunnel() throws Exception { + final String PASSWORD = "Passw0rd"; + try (final var testdb = createTestDatabase("withRootAndServerCertificates", "withClientCertificate")) { + final var config = testdb.testConfigBuilder() + .with("tunnel_method", ImmutableMap.builder().put("tunnel_method", "NO_TUNNEL").build()) + .withSsl(ImmutableMap.builder() + .put(JdbcUtils.MODE_KEY, "verify_ca") + .put("ca_certificate", testdb.getCertificates().caCertificate()) + .put("client_certificate", testdb.getCertificates().clientCertificate()) + .put("client_key", testdb.getCertificates().clientKey()) + .put("client_key_password", PASSWORD) + .build()) + .build(); + final AirbyteConnectionStatus actual = source().check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, actual.getStatus()); + assertTrue(actual.getMessage().contains("Failed to create keystore for Client certificate"), actual.getMessage()); + } + } + + @Test + void testStrictSSLSecuredWithTunnel() throws Exception { + final String PASSWORD = "Passw0rd"; + try (final var testdb = createTestDatabase("withRootAndServerCertificates", "withClientCertificate")) { + final var config = testdb.configBuilder() + .withHostAndPort() + .withDatabase() + .with(JdbcUtils.USERNAME_KEY, testdb.getUserName()) + .with(JdbcUtils.PASSWORD_KEY, "fake") + .withSsl(ImmutableMap.builder() + .put(JdbcUtils.MODE_KEY, "verify_ca") + .put("ca_certificate", testdb.getCertificates().caCertificate()) + .put("client_certificate", testdb.getCertificates().clientCertificate()) + .put("client_key", testdb.getCertificates().clientKey()) + .put("client_key_password", PASSWORD) + .build()) + .with("tunnel_method", ImmutableMap.builder().put("tunnel_method", "SSH_KEY_AUTH").build()) + .build(); + final AirbyteConnectionStatus actual = source().check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, actual.getStatus()); + assertTrue(actual.getMessage().contains("Could not connect with provided SSH configuration."), actual.getMessage()); + } + } + + @Test + void testStrictSSLUnsecuredWithTunnel() throws Exception { + try (final var testdb = createTestDatabase()) { + final var config = testdb.configBuilder() + .withHostAndPort() + .withDatabase() + .with(JdbcUtils.USERNAME_KEY, testdb.getUserName()) + .with(JdbcUtils.PASSWORD_KEY, "fake") + .withSsl(ImmutableMap.builder() + .put(JdbcUtils.MODE_KEY, "preferred") + .build()) + .with("tunnel_method", ImmutableMap.builder().put("tunnel_method", "SSH_KEY_AUTH").build()) + .build(); + final AirbyteConnectionStatus actual = source().check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, actual.getStatus()); + assertTrue(actual.getMessage().contains("Could not connect with provided SSH configuration."), actual.getMessage()); + } + } + + @Test + void testCheckWithSslModeDisabled() throws Exception { + try (final var testdb = createTestDatabase("withNetwork")) { + try (final SshBastionContainer bastion = new SshBastionContainer()) { + bastion.initAndStartBastion(testdb.getContainer().getNetwork()); + final var config = testdb.integrationTestConfigBuilder() + .with("tunnel_method", bastion.getTunnelMethod(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, false)) + .withoutSsl() + .build(); + final AirbyteConnectionStatus actual = source().check(config); + assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, actual.getStatus()); + } + } + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/InitialPkLoadEnabledCdcMysqlSourceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/InitialPkLoadEnabledCdcMysqlSourceTest.java deleted file mode 100644 index 15cd6915e05f..000000000000 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/InitialPkLoadEnabledCdcMysqlSourceTest.java +++ /dev/null @@ -1,361 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.mysql; - -import static io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadStateManager.PRIMARY_KEY_STATE_TYPE; -import static io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadStateManager.STATE_TYPE_KEY; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import com.google.common.collect.Streams; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.util.AutoCloseableIterator; -import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.integrations.source.mysql.initialsync.MySqlFeatureFlags; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.AirbyteGlobalState; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; -import io.airbyte.protocol.models.v0.AirbyteStateMessage; -import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; -import io.airbyte.protocol.models.v0.AirbyteStreamState; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.StreamDescriptor; -import io.airbyte.protocol.models.v0.SyncMode; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import org.junit.jupiter.api.Test; - -public class InitialPkLoadEnabledCdcMysqlSourceTest extends CdcMysqlSourceTest { - - @Override - protected JsonNode getConfig() { - final JsonNode config = super.getConfig(); - ((ObjectNode) config).put(MySqlFeatureFlags.CDC_VIA_PK, true); - return config; - } - - @Override - protected void assertExpectedStateMessages(final List stateMessages) { - assertEquals(7, stateMessages.size()); - assertStateTypes(stateMessages, 4); - } - - @Override - protected void assertExpectedStateMessagesFromIncrementalSync(final List stateMessages) { - super.assertExpectedStateMessages(stateMessages); - } - - @Override - protected void assertStateForSyncShouldHandlePurgedLogsGracefully(final List stateMessages, final int syncNumber) { - if (syncNumber == 1) { - assertExpectedStateMessagesForRecordsProducedDuringAndAfterSync(stateMessages); - } else if (syncNumber == 2) { - // Sync number 2 uses the state from sync number 1 but before we trigger the sync 2 we purge the - // binary logs and as a result the validation of - // logs present on the server fails, and we trigger a sync from scratch - assertEquals(47, stateMessages.size()); - assertStateTypes(stateMessages, 44); - } else { - throw new RuntimeException("Unknown sync number"); - } - - } - - @Override - protected void assertExpectedStateMessagesForRecordsProducedDuringAndAfterSync(final List stateAfterFirstBatch) { - assertEquals(27, stateAfterFirstBatch.size()); - assertStateTypes(stateAfterFirstBatch, 24); - } - - @Override - protected void assertExpectedStateMessagesForNoData(final List stateMessages) { - assertEquals(2, stateMessages.size()); - } - - private void assertStateTypes(final List stateMessages, final int indexTillWhichExpectPkState) { - JsonNode sharedState = null; - for (int i = 0; i < stateMessages.size(); i++) { - final AirbyteStateMessage stateMessage = stateMessages.get(i); - assertEquals(AirbyteStateType.GLOBAL, stateMessage.getType()); - final AirbyteGlobalState global = stateMessage.getGlobal(); - assertNotNull(global.getSharedState()); - if (Objects.isNull(sharedState)) { - sharedState = global.getSharedState(); - } else { - assertEquals(sharedState, global.getSharedState()); - } - assertEquals(1, global.getStreamStates().size()); - final AirbyteStreamState streamState = global.getStreamStates().get(0); - if (i <= indexTillWhichExpectPkState) { - assertTrue(streamState.getStreamState().has(STATE_TYPE_KEY)); - assertEquals(PRIMARY_KEY_STATE_TYPE, streamState.getStreamState().get(STATE_TYPE_KEY).asText()); - } else { - assertFalse(streamState.getStreamState().has(STATE_TYPE_KEY)); - } - } - } - - @Override - protected void assertStateMessagesForNewTableSnapshotTest(final List stateMessages, - final AirbyteStateMessage stateMessageEmittedAfterFirstSyncCompletion) { - assertEquals(7, stateMessages.size()); - for (int i = 0; i <= 4; i++) { - final AirbyteStateMessage stateMessage = stateMessages.get(i); - assertEquals(AirbyteStateMessage.AirbyteStateType.GLOBAL, stateMessage.getType()); - assertEquals(stateMessageEmittedAfterFirstSyncCompletion.getGlobal().getSharedState(), - stateMessage.getGlobal().getSharedState()); - final Set streamsInSnapshotState = stateMessage.getGlobal().getStreamStates() - .stream() - .map(AirbyteStreamState::getStreamDescriptor) - .collect(Collectors.toSet()); - assertEquals(2, streamsInSnapshotState.size()); - assertTrue( - streamsInSnapshotState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME + "_random").withNamespace(randomTableSchema()))); - assertTrue(streamsInSnapshotState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(MODELS_SCHEMA))); - - stateMessage.getGlobal().getStreamStates().forEach(s -> { - final JsonNode streamState = s.getStreamState(); - if (s.getStreamDescriptor().equals(new StreamDescriptor().withName(MODELS_STREAM_NAME + "_random").withNamespace(randomTableSchema()))) { - assertEquals(PRIMARY_KEY_STATE_TYPE, streamState.get(STATE_TYPE_KEY).asText()); - } else if (s.getStreamDescriptor().equals(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(MODELS_SCHEMA))) { - assertFalse(streamState.has(STATE_TYPE_KEY)); - } else { - throw new RuntimeException("Unknown stream"); - } - }); - } - - final AirbyteStateMessage secondLastSateMessage = stateMessages.get(5); - assertEquals(AirbyteStateMessage.AirbyteStateType.GLOBAL, secondLastSateMessage.getType()); - assertEquals(stateMessageEmittedAfterFirstSyncCompletion.getGlobal().getSharedState(), - secondLastSateMessage.getGlobal().getSharedState()); - final Set streamsInSnapshotState = secondLastSateMessage.getGlobal().getStreamStates() - .stream() - .map(AirbyteStreamState::getStreamDescriptor) - .collect(Collectors.toSet()); - assertEquals(2, streamsInSnapshotState.size()); - assertTrue( - streamsInSnapshotState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME + "_random").withNamespace(randomTableSchema()))); - assertTrue(streamsInSnapshotState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(MODELS_SCHEMA))); - secondLastSateMessage.getGlobal().getStreamStates().forEach(s -> { - final JsonNode streamState = s.getStreamState(); - assertFalse(streamState.has(STATE_TYPE_KEY)); - }); - - final AirbyteStateMessage stateMessageEmittedAfterSecondSyncCompletion = stateMessages.get(6); - assertEquals(AirbyteStateMessage.AirbyteStateType.GLOBAL, stateMessageEmittedAfterSecondSyncCompletion.getType()); - assertNotEquals(stateMessageEmittedAfterFirstSyncCompletion.getGlobal().getSharedState(), - stateMessageEmittedAfterSecondSyncCompletion.getGlobal().getSharedState()); - final Set streamsInSyncCompletionState = stateMessageEmittedAfterSecondSyncCompletion.getGlobal().getStreamStates() - .stream() - .map(AirbyteStreamState::getStreamDescriptor) - .collect(Collectors.toSet()); - assertEquals(2, streamsInSnapshotState.size()); - assertTrue( - streamsInSyncCompletionState.contains( - new StreamDescriptor().withName(MODELS_STREAM_NAME + "_random").withNamespace(randomTableSchema()))); - assertTrue(streamsInSyncCompletionState.contains(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(MODELS_SCHEMA))); - assertNotNull(stateMessageEmittedAfterSecondSyncCompletion.getData()); - } - - @Test - public void testCompositeIndexInitialLoad() throws Exception { - // Simulate adding a composite index by modifying the catalog. - final ConfiguredAirbyteCatalog configuredCatalog = Jsons.clone(CONFIGURED_CATALOG); - final List> primaryKeys = configuredCatalog.getStreams().get(0).getStream().getSourceDefinedPrimaryKey(); - primaryKeys.add(List.of("make_id")); - - final AutoCloseableIterator read1 = getSource() - .read(getConfig(), configuredCatalog, null); - - final List actualRecords1 = AutoCloseableIterators.toListAndClose(read1); - - final Set recordMessages1 = extractRecordMessages(actualRecords1); - final List stateMessages1 = extractStateMessages(actualRecords1); - assertExpectedRecords(new HashSet<>(MODEL_RECORDS), recordMessages1); - assertExpectedStateMessages(stateMessages1); - - // Re-run the sync with state associated with record w/ id = 15 (second to last record). - // We expect to read 2 records, since in the case of a composite PK we issue a >= query. - // We also expect 3 state records. One associated with the pk state, one to signify end of initial - // load, and - // the last one indicating the cdc position we have synced until. - final JsonNode state = Jsons.jsonNode(Collections.singletonList(stateMessages1.get(4))); - final AutoCloseableIterator read2 = getSource() - .read(getConfig(), configuredCatalog, state); - - final List actualRecords2 = AutoCloseableIterators.toListAndClose(read2); - final Set recordMessages2 = extractRecordMessages(actualRecords2); - final List stateMessages2 = extractStateMessages(actualRecords2); - - assertExpectedRecords(new HashSet<>(MODEL_RECORDS.subList(4, 6)), recordMessages2); - assertEquals(3, stateMessages2.size()); - assertStateTypes(stateMessages2, 0); - } - - @Test - public void testTwoStreamSync() throws Exception { - // Add another stream models_2 and read that one as well. - final ConfiguredAirbyteCatalog configuredCatalog = Jsons.clone(CONFIGURED_CATALOG); - - final List MODEL_RECORDS_2 = ImmutableList.of( - Jsons.jsonNode(ImmutableMap.of(COL_ID, 110, COL_MAKE_ID, 1, COL_MODEL, "Fiesta-2")), - Jsons.jsonNode(ImmutableMap.of(COL_ID, 120, COL_MAKE_ID, 1, COL_MODEL, "Focus-2")), - Jsons.jsonNode(ImmutableMap.of(COL_ID, 130, COL_MAKE_ID, 1, COL_MODEL, "Ranger-2")), - Jsons.jsonNode(ImmutableMap.of(COL_ID, 140, COL_MAKE_ID, 2, COL_MODEL, "GLA-2")), - Jsons.jsonNode(ImmutableMap.of(COL_ID, 150, COL_MAKE_ID, 2, COL_MODEL, "A 220-2")), - Jsons.jsonNode(ImmutableMap.of(COL_ID, 160, COL_MAKE_ID, 2, COL_MODEL, "E 350-2"))); - - createTable(MODELS_SCHEMA, MODELS_STREAM_NAME + "_2", - columnClause(ImmutableMap.of(COL_ID, "INTEGER", COL_MAKE_ID, "INTEGER", COL_MODEL, "VARCHAR(200)"), Optional.of(COL_ID))); - - for (final JsonNode recordJson : MODEL_RECORDS_2) { - writeRecords(recordJson, MODELS_SCHEMA, MODELS_STREAM_NAME + "_2", COL_ID, - COL_MAKE_ID, COL_MODEL); - } - - final ConfiguredAirbyteStream airbyteStream = new ConfiguredAirbyteStream() - .withStream(CatalogHelpers.createAirbyteStream( - MODELS_STREAM_NAME + "_2", - MODELS_SCHEMA, - Field.of(COL_ID, JsonSchemaType.INTEGER), - Field.of(COL_MAKE_ID, JsonSchemaType.INTEGER), - Field.of(COL_MODEL, JsonSchemaType.STRING)) - .withSupportedSyncModes( - Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) - .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID)))); - airbyteStream.setSyncMode(SyncMode.INCREMENTAL); - - final List streams = configuredCatalog.getStreams(); - streams.add(airbyteStream); - configuredCatalog.withStreams(streams); - - final AutoCloseableIterator read1 = getSource() - .read(getConfig(), configuredCatalog, null); - final List actualRecords1 = AutoCloseableIterators.toListAndClose(read1); - - final Set recordMessages1 = extractRecordMessages(actualRecords1); - final List stateMessages1 = extractStateMessages(actualRecords1); - assertEquals(13, stateMessages1.size()); - JsonNode sharedState = null; - StreamDescriptor firstStreamInState = null; - for (int i = 0; i < stateMessages1.size(); i++) { - final AirbyteStateMessage stateMessage = stateMessages1.get(i); - assertEquals(AirbyteStateType.GLOBAL, stateMessage.getType()); - final AirbyteGlobalState global = stateMessage.getGlobal(); - assertNotNull(global.getSharedState()); - if (Objects.isNull(sharedState)) { - sharedState = global.getSharedState(); - } else { - assertEquals(sharedState, global.getSharedState()); - } - - if (Objects.isNull(firstStreamInState)) { - assertEquals(1, global.getStreamStates().size()); - firstStreamInState = global.getStreamStates().get(0).getStreamDescriptor(); - } - - if (i <= 4) { - // First 4 state messages are pk state - assertEquals(1, global.getStreamStates().size()); - final AirbyteStreamState streamState = global.getStreamStates().get(0); - assertTrue(streamState.getStreamState().has(STATE_TYPE_KEY)); - assertEquals(PRIMARY_KEY_STATE_TYPE, streamState.getStreamState().get(STATE_TYPE_KEY).asText()); - } else if (i == 5) { - // 5th state message is the final state message emitted for the stream - assertEquals(1, global.getStreamStates().size()); - final AirbyteStreamState streamState = global.getStreamStates().get(0); - assertFalse(streamState.getStreamState().has(STATE_TYPE_KEY)); - } else if (i <= 10) { - // 6th to 10th is the primary_key state message for the 2nd stream but final state message for 1st - // stream - assertEquals(2, global.getStreamStates().size()); - final StreamDescriptor finalFirstStreamInState = firstStreamInState; - global.getStreamStates().forEach(c -> { - if (c.getStreamDescriptor().equals(finalFirstStreamInState)) { - assertFalse(c.getStreamState().has(STATE_TYPE_KEY)); - } else { - assertTrue(c.getStreamState().has(STATE_TYPE_KEY)); - assertEquals(PRIMARY_KEY_STATE_TYPE, c.getStreamState().get(STATE_TYPE_KEY).asText()); - } - }); - } else { - // last 2 state messages don't contain primary_key info cause primary_key sync should be complete - assertEquals(2, global.getStreamStates().size()); - global.getStreamStates().forEach(c -> assertFalse(c.getStreamState().has(STATE_TYPE_KEY))); - } - } - - final Set names = new HashSet<>(STREAM_NAMES); - names.add(MODELS_STREAM_NAME + "_2"); - assertExpectedRecords(Streams.concat(MODEL_RECORDS_2.stream(), MODEL_RECORDS.stream()) - .collect(Collectors.toSet()), - recordMessages1, - names, - names, - MODELS_SCHEMA); - - assertEquals(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(MODELS_SCHEMA), firstStreamInState); - - // Triggering a sync with a primary_key state for 1 stream and complete state for other stream - final AutoCloseableIterator read2 = getSource() - .read(getConfig(), configuredCatalog, Jsons.jsonNode(Collections.singletonList(stateMessages1.get(6)))); - final List actualRecords2 = AutoCloseableIterators.toListAndClose(read2); - - final List stateMessages2 = extractStateMessages(actualRecords2); - - assertEquals(6, stateMessages2.size()); - for (int i = 0; i < stateMessages2.size(); i++) { - final AirbyteStateMessage stateMessage = stateMessages2.get(i); - assertEquals(AirbyteStateType.GLOBAL, stateMessage.getType()); - final AirbyteGlobalState global = stateMessage.getGlobal(); - assertNotNull(global.getSharedState()); - assertEquals(2, global.getStreamStates().size()); - - if (i <= 3) { - final StreamDescriptor finalFirstStreamInState = firstStreamInState; - global.getStreamStates().forEach(c -> { - // First 4 state messages are primary_key state for the stream that didn't complete primary_key sync - // the first time - if (c.getStreamDescriptor().equals(finalFirstStreamInState)) { - assertFalse(c.getStreamState().has(STATE_TYPE_KEY)); - } else { - assertTrue(c.getStreamState().has(STATE_TYPE_KEY)); - assertEquals(PRIMARY_KEY_STATE_TYPE, c.getStreamState().get(STATE_TYPE_KEY).asText()); - } - }); - } else { - // last 2 state messages don't contain primary_key info cause primary_key sync should be complete - global.getStreamStates().forEach(c -> assertFalse(c.getStreamState().has(STATE_TYPE_KEY))); - } - } - - final Set recordMessages2 = extractRecordMessages(actualRecords2); - assertEquals(5, recordMessages2.size()); - assertExpectedRecords(new HashSet<>(MODEL_RECORDS_2.subList(1, MODEL_RECORDS_2.size())), - recordMessages2, - names, - names, - MODELS_SCHEMA); - } - -} diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlDebugger.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlDebugger.java new file mode 100644 index 000000000000..67c8f4d68687 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlDebugger.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ +package io.airbyte.integrations.source.mysql; + +import io.airbyte.cdk.integrations.debug.DebugUtil; + +public class MySqlDebugger { + + @SuppressWarnings({"unchecked", "deprecation", "resource"}) + public static void main(final String[] args) throws Exception { + final MySqlSource mysqlSource = new MySqlSource(); + DebugUtil.debug(mysqlSource); + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java index 8d61becf068c..d6597cd2b023 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlJdbcSourceAcceptanceTest.java @@ -4,24 +4,31 @@ package io.airbyte.integrations.source.mysql; +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_RECORDS_PROPERTY; +import static io.airbyte.integrations.source.mysql.initialsync.MySqlInitialLoadStateManager.STATE_TYPE_KEY; +import static java.util.stream.Collectors.toList; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.mysql.cj.MysqlType; -import io.airbyte.commons.features.EnvVariableFeatureFlags; +import com.google.common.collect.Lists; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.commons.string.Strings; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; -import io.airbyte.integrations.source.relationaldb.models.DbStreamState; +import io.airbyte.commons.util.MoreIterators; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; +import io.airbyte.integrations.source.mysql.internal.models.CursorBasedStatus; +import io.airbyte.integrations.source.mysql.internal.models.InternalModels.StateType; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; @@ -29,120 +36,259 @@ import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStateStats; +import io.airbyte.protocol.models.v0.AirbyteStream; +import io.airbyte.protocol.models.v0.AirbyteStreamState; import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.ConnectorSpecification; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.StreamDescriptor; import io.airbyte.protocol.models.v0.SyncMode; -import java.sql.Connection; -import java.sql.DriverManager; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.concurrent.Callable; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testcontainers.containers.MySQLContainer; -import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; -import uk.org.webcompere.systemstubs.jupiter.SystemStub; -import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; -@ExtendWith(SystemStubsExtension.class) -class MySqlJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { - - @SystemStub - private EnvironmentVariables environmentVariables; +@Order(2) +class MySqlJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { protected static final String USERNAME_WITHOUT_PERMISSION = "new_user"; protected static final String PASSWORD_WITHOUT_PERMISSION = "new_password"; - protected static final String TEST_USER = "test"; - protected static final Callable TEST_PASSWORD = () -> "test"; - protected static MySQLContainer container; - - protected Database database; - protected DSLContext dslContext; - - @BeforeAll - static void init() throws Exception { - container = new MySQLContainer<>("mysql:8.0") - .withUsername(TEST_USER) - .withPassword(TEST_PASSWORD.call()) - .withEnv("MYSQL_ROOT_HOST", "%") - .withEnv("MYSQL_ROOT_PASSWORD", TEST_PASSWORD.call()); - container.start(); - final Connection connection = DriverManager.getConnection(container.getJdbcUrl(), "root", TEST_PASSWORD.call()); - connection.createStatement().execute("GRANT ALL PRIVILEGES ON *.* TO '" + TEST_USER + "'@'%';\n"); - } - - @BeforeEach - public void setup() throws Exception { - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, Strings.addRandomSuffix("db", "_", 10)) - .put(JdbcUtils.USERNAME_KEY, TEST_USER) - .put(JdbcUtils.PASSWORD_KEY, TEST_PASSWORD.call()) - .build()); - - dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.MYSQL.getDriverClassName(), - String.format("jdbc:mysql://%s:%s", - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asText()), - SQLDialect.MYSQL); - database = new Database(dslContext); - - database.query(ctx -> { - ctx.fetch("CREATE DATABASE " + config.get(JdbcUtils.DATABASE_KEY).asText()); - return null; - }); - - super.setup(); - } - @AfterEach - void tearDownMySql() throws Exception { - dslContext.close(); - super.tearDown(); + @Override + protected JsonNode config() { + return testdb.testConfigBuilder().build(); } - @AfterAll - static void cleanUp() { - container.close(); + @Override + protected MySqlSource source() { + return new MySqlSource(); } - // MySql does not support schemas in the way most dbs do. Instead we namespace by db name. @Override - public boolean supportsSchemas() { - return false; + protected MySQLTestDatabase createTestDatabase() { + return MySQLTestDatabase.in(BaseImage.MYSQL_8); } @Override - public AbstractJdbcSource getJdbcSource() { - return new MySqlSource(); + protected void maybeSetShorterConnectionTimeout(final JsonNode config) { + ((ObjectNode) config).put(JdbcUtils.JDBC_URL_PARAMS_KEY, "connectTimeout=1000"); } + // MySql does not support schemas in the way most dbs do. Instead we namespace by db name. @Override - public String getDriverClass() { - return MySqlSource.DRIVER_CLASS; + protected boolean supportsSchemas() { + return false; } + @Test @Override - public JsonNode getConfig() { - return Jsons.clone(config); + protected void testReadMultipleTablesIncrementally() throws Exception { + final var config = config(); + ((ObjectNode) config).put(SYNC_CHECKPOINT_RECORDS_PROPERTY, 1); + final String streamOneName = TABLE_NAME + "one"; + // Create a fresh first table + testdb.with("CREATE TABLE %s (\n" + + " id int PRIMARY KEY,\n" + + " name VARCHAR(200) NOT NULL,\n" + + " updated_at VARCHAR(200) NOT NULL\n" + + ");", streamOneName) + .with("INSERT INTO %s(id, name, updated_at) VALUES (1,'picard', '2004-10-19')", + getFullyQualifiedTableName(streamOneName)) + .with("INSERT INTO %s(id, name, updated_at) VALUES (2, 'crusher', '2005-10-19')", + getFullyQualifiedTableName(streamOneName)) + .with("INSERT INTO %s(id, name, updated_at) VALUES (3, 'vash', '2006-10-19')", + getFullyQualifiedTableName(streamOneName)); + + // Create a fresh second table + final String streamTwoName = TABLE_NAME + "two"; + final String streamTwoFullyQualifiedName = getFullyQualifiedTableName(streamTwoName); + // Insert records into second table + testdb.with("CREATE TABLE %s (\n" + + " id int PRIMARY KEY,\n" + + " name VARCHAR(200) NOT NULL,\n" + + " updated_at DATE NOT NULL\n" + + ");", streamTwoName) + .with("INSERT INTO %s(id, name, updated_at) VALUES (40,'Jean Luc','2006-10-19')", + streamTwoFullyQualifiedName) + .with("INSERT INTO %s(id, name, updated_at) VALUES (41, 'Groot', '2006-10-19')", + streamTwoFullyQualifiedName) + .with("INSERT INTO %s(id, name, updated_at) VALUES (42, 'Thanos','2006-10-19')", + streamTwoFullyQualifiedName); + + // Create records list that we expect to see in the state message + final List streamTwoExpectedRecords = Arrays.asList( + createRecord(streamTwoName, getDefaultNamespace(), ImmutableMap.of( + COL_ID, 40, + COL_NAME, "Jean Luc", + COL_UPDATED_AT, "2006-10-19")), + createRecord(streamTwoName, getDefaultNamespace(), ImmutableMap.of( + COL_ID, 41, + COL_NAME, "Groot", + COL_UPDATED_AT, "2006-10-19")), + createRecord(streamTwoName, getDefaultNamespace(), ImmutableMap.of( + COL_ID, 42, + COL_NAME, "Thanos", + COL_UPDATED_AT, "2006-10-19"))); + + // Prep and create a configured catalog to perform sync + final AirbyteStream streamOne = getAirbyteStream(streamOneName, getDefaultNamespace()); + final AirbyteStream streamTwo = getAirbyteStream(streamTwoName, getDefaultNamespace()); + + final ConfiguredAirbyteCatalog configuredCatalog = CatalogHelpers.toDefaultConfiguredCatalog( + new AirbyteCatalog().withStreams(List.of(streamOne, streamTwo))); + configuredCatalog.getStreams().forEach(airbyteStream -> { + airbyteStream.setSyncMode(SyncMode.INCREMENTAL); + airbyteStream.setCursorField(List.of(COL_ID)); + airbyteStream.setDestinationSyncMode(DestinationSyncMode.APPEND); + airbyteStream.withPrimaryKey(List.of(List.of(COL_ID))); + }); + + // Perform initial sync + final List messagesFromFirstSync = MoreIterators + .toList(source().read(config, configuredCatalog, null)); + + final List recordsFromFirstSync = filterRecords(messagesFromFirstSync); + + setEmittedAtToNull(messagesFromFirstSync); + // All records in the 2 configured streams should be present + assertThat(filterRecords(recordsFromFirstSync)).containsExactlyElementsOf( + Stream.concat(getTestMessages(streamOneName).stream().parallel(), + streamTwoExpectedRecords.stream().parallel()).collect(toList())); + + final List actualFirstSyncState = extractStateMessage(messagesFromFirstSync); + // Since we are emitting a state message after each record, we should have 1 state for each record - + // 3 from stream1 and 3 from stream2 + assertEquals(6, actualFirstSyncState.size()); + + // The expected state type should be 2 primaryKey's and the last one being standard + final List expectedStateTypesFromFirstSync = List.of("primary_key", "primary_key", "cursor_based"); + final List stateTypeOfStreamOneStatesFromFirstSync = + extractSpecificFieldFromCombinedMessages(messagesFromFirstSync, streamOneName, STATE_TYPE_KEY); + final List stateTypeOfStreamTwoStatesFromFirstSync = + extractSpecificFieldFromCombinedMessages(messagesFromFirstSync, streamTwoName, STATE_TYPE_KEY); + // It should be the same for stream1 and stream2 + assertEquals(stateTypeOfStreamOneStatesFromFirstSync, expectedStateTypesFromFirstSync); + assertEquals(stateTypeOfStreamTwoStatesFromFirstSync, expectedStateTypesFromFirstSync); + + // Create the expected primaryKeys that we should see + final List expectedPrimaryKeysFromFirstSync = List.of("1", "2"); + final List primaryKeyFromStreamOneStatesFromFirstSync = + extractSpecificFieldFromCombinedMessages(messagesFromFirstSync, streamOneName, "pk_val"); + final List primaryKeyFromStreamTwoStatesFromFirstSync = + extractSpecificFieldFromCombinedMessages(messagesFromFirstSync, streamOneName, "pk_val"); + + // Verifying each element and its index to match. + // Only checking the first 2 elements since we have verified that the last state_type is + // "cursor_based" + assertEquals(primaryKeyFromStreamOneStatesFromFirstSync.get(0), expectedPrimaryKeysFromFirstSync.get(0)); + assertEquals(primaryKeyFromStreamOneStatesFromFirstSync.get(1), expectedPrimaryKeysFromFirstSync.get(1)); + assertEquals(primaryKeyFromStreamTwoStatesFromFirstSync.get(0), expectedPrimaryKeysFromFirstSync.get(0)); + assertEquals(primaryKeyFromStreamTwoStatesFromFirstSync.get(1), expectedPrimaryKeysFromFirstSync.get(1)); + + // Extract only state messages for each stream + final List streamOneStateMessagesFromFirstSync = extractStateMessage(messagesFromFirstSync, streamOneName); + final List streamTwoStateMessagesFromFirstSync = extractStateMessage(messagesFromFirstSync, streamTwoName); + // Extract the incremental states of each stream's first and second state message + final List streamOneIncrementalStatesFromFirstSync = + List.of(streamOneStateMessagesFromFirstSync.get(0).getStream().getStreamState().get("incremental_state"), + streamOneStateMessagesFromFirstSync.get(1).getStream().getStreamState().get("incremental_state")); + final JsonNode streamOneFinalStreamStateFromFirstSync = streamOneStateMessagesFromFirstSync.get(2).getStream().getStreamState(); + + final List streamTwoIncrementalStatesFromFirstSync = + List.of(streamTwoStateMessagesFromFirstSync.get(0).getStream().getStreamState().get("incremental_state"), + streamTwoStateMessagesFromFirstSync.get(1).getStream().getStreamState().get("incremental_state")); + final JsonNode streamTwoFinalStreamStateFromFirstSync = streamTwoStateMessagesFromFirstSync.get(2).getStream().getStreamState(); + + // The incremental_state of each stream's first and second incremental states is expected + // to be identical to the stream_state of the final state message for each stream + assertEquals(streamOneIncrementalStatesFromFirstSync.get(0), streamOneFinalStreamStateFromFirstSync); + assertEquals(streamOneIncrementalStatesFromFirstSync.get(1), streamOneFinalStreamStateFromFirstSync); + assertEquals(streamTwoIncrementalStatesFromFirstSync.get(0), streamTwoFinalStreamStateFromFirstSync); + assertEquals(streamTwoIncrementalStatesFromFirstSync.get(1), streamTwoFinalStreamStateFromFirstSync); + + // Sync should work with a primaryKey state AND a cursor-based state from each stream + // Forcing a sync with + // - stream one state still being the first record read via Primary Key. + // - stream two state being the Primary Key state before the final emitted state before the cursor + // switch + final List messagesFromSecondSyncWithMixedStates = MoreIterators + .toList(source().read(config, configuredCatalog, + Jsons.jsonNode(List.of(streamOneStateMessagesFromFirstSync.get(0), + streamTwoStateMessagesFromFirstSync.get(1))))); + + // Extract only state messages for each stream after second sync + final List streamOneStateMessagesFromSecondSync = + extractStateMessage(messagesFromSecondSyncWithMixedStates, streamOneName); + final List stateTypeOfStreamOneStatesFromSecondSync = + extractSpecificFieldFromCombinedMessages(messagesFromSecondSyncWithMixedStates, streamOneName, STATE_TYPE_KEY); + + final List streamTwoStateMessagesFromSecondSync = + extractStateMessage(messagesFromSecondSyncWithMixedStates, streamTwoName); + final List stateTypeOfStreamTwoStatesFromSecondSync = + extractSpecificFieldFromCombinedMessages(messagesFromSecondSyncWithMixedStates, streamTwoName, STATE_TYPE_KEY); + + // Stream One states after the second sync are expected to have 2 stream states + // - 1 with PrimaryKey state_type and 1 state that is of cursorBased state type + assertEquals(2, streamOneStateMessagesFromSecondSync.size()); + assertEquals(List.of("primary_key", "cursor_based"), stateTypeOfStreamOneStatesFromSecondSync); + + // Stream Two states after the second sync are expected to have 1 stream state + // - The state that is of cursorBased state type + assertEquals(1, streamTwoStateMessagesFromSecondSync.size()); + assertEquals(List.of("cursor_based"), stateTypeOfStreamTwoStatesFromSecondSync); + + // Add some data to each table and perform a third read. + // Expect to see all records be synced via cursorBased method and not primaryKey + testdb.with("INSERT INTO %s(id, name, updated_at) VALUES (4,'Hooper','2006-10-19')", + getFullyQualifiedTableName(streamOneName)) + .with("INSERT INTO %s(id, name, updated_at) VALUES (43, 'Iron Man', '2006-10-19')", + streamTwoFullyQualifiedName); + + final List messagesFromThirdSync = MoreIterators + .toList(source().read(config, configuredCatalog, + Jsons.jsonNode(List.of(streamOneStateMessagesFromSecondSync.get(1), + streamTwoStateMessagesFromSecondSync.get(0))))); + + // Extract only state messages, state type, and cursor for each stream after second sync + final List streamOneStateMessagesFromThirdSync = + extractStateMessage(messagesFromThirdSync, streamOneName); + final List stateTypeOfStreamOneStatesFromThirdSync = + extractSpecificFieldFromCombinedMessages(messagesFromThirdSync, streamOneName, STATE_TYPE_KEY); + final List cursorOfStreamOneStatesFromThirdSync = + extractSpecificFieldFromCombinedMessages(messagesFromThirdSync, streamOneName, "cursor"); + + final List streamTwoStateMessagesFromThirdSync = + extractStateMessage(messagesFromThirdSync, streamTwoName); + final List stateTypeOfStreamTwoStatesFromThirdSync = + extractSpecificFieldFromCombinedMessages(messagesFromThirdSync, streamTwoName, STATE_TYPE_KEY); + final List cursorOfStreamTwoStatesFromThirdSync = + extractSpecificFieldFromCombinedMessages(messagesFromThirdSync, streamTwoName, "cursor"); + + // Both streams should now be synced via standard cursor and have updated max cursor values + // cursor: 4 for stream one + // cursor: 43 for stream two + assertEquals(1, streamOneStateMessagesFromThirdSync.size()); + assertEquals(List.of("cursor_based"), stateTypeOfStreamOneStatesFromThirdSync); + assertEquals(List.of("4"), cursorOfStreamOneStatesFromThirdSync); + + assertEquals(1, streamTwoStateMessagesFromThirdSync.size()); + assertEquals(List.of("cursor_based"), stateTypeOfStreamTwoStatesFromThirdSync); + assertEquals(List.of("43"), cursorOfStreamTwoStatesFromThirdSync); } @Test void testSpec() throws Exception { - final ConnectorSpecification actual = source.spec(); + final ConnectorSpecification actual = source().spec(); final ConnectorSpecification expected = Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class); assertEquals(expected, actual); @@ -158,16 +304,20 @@ void testSpec() throws Exception { */ @Test void testCheckIncorrectPasswordFailure() throws Exception { + final var config = config(); + maybeSetShorterConnectionTimeout(config); ((ObjectNode) config).put(JdbcUtils.PASSWORD_KEY, "fake"); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 08001;")); + assertTrue(status.getMessage().contains("State code: 08001;"), status.getMessage()); } @Test public void testCheckIncorrectUsernameFailure() throws Exception { + final var config = config(); + maybeSetShorterConnectionTimeout(config); ((ObjectNode) config).put(JdbcUtils.USERNAME_KEY, "fake"); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); // do not test for message since there seems to be flakiness where sometimes the test will get the // message with @@ -176,81 +326,93 @@ public void testCheckIncorrectUsernameFailure() throws Exception { @Test public void testCheckIncorrectHostFailure() throws Exception { + final var config = config(); + maybeSetShorterConnectionTimeout(config); ((ObjectNode) config).put(JdbcUtils.HOST_KEY, "localhost2"); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 08S01;")); + assertTrue(status.getMessage().contains("State code: 08S01;"), status.getMessage()); } @Test public void testCheckIncorrectPortFailure() throws Exception { + final var config = config(); + maybeSetShorterConnectionTimeout(config); ((ObjectNode) config).put(JdbcUtils.PORT_KEY, "0000"); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 08S01;")); + assertTrue(status.getMessage().contains("State code: 08S01;"), status.getMessage()); } @Test public void testCheckIncorrectDataBaseFailure() throws Exception { + final var config = config(); + maybeSetShorterConnectionTimeout(config); ((ObjectNode) config).put(JdbcUtils.DATABASE_KEY, "wrongdatabase"); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 42000; Error code: 1049;")); + assertTrue(status.getMessage().contains("State code: 42000; Error code: 1049;"), status.getMessage()); } @Test public void testUserHasNoPermissionToDataBase() throws Exception { - final Connection connection = DriverManager.getConnection(container.getJdbcUrl(), "root", TEST_PASSWORD.call()); - connection.createStatement() - .execute("create user '" + USERNAME_WITHOUT_PERMISSION + "'@'%' IDENTIFIED BY '" + PASSWORD_WITHOUT_PERMISSION + "';\n"); - ((ObjectNode) config).put(JdbcUtils.USERNAME_KEY, USERNAME_WITHOUT_PERMISSION); + final var config = config(); + maybeSetShorterConnectionTimeout(config); + final String usernameWithoutPermission = testdb.withNamespace(USERNAME_WITHOUT_PERMISSION); + testdb.with("CREATE USER '%s'@'%%' IDENTIFIED BY '%s';", usernameWithoutPermission, PASSWORD_WITHOUT_PERMISSION); + ((ObjectNode) config).put(JdbcUtils.USERNAME_KEY, usernameWithoutPermission); ((ObjectNode) config).put(JdbcUtils.PASSWORD_KEY, PASSWORD_WITHOUT_PERMISSION); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 08001;")); + assertTrue(status.getMessage().contains("State code: 08001;"), status.getMessage()); } @Override - protected AirbyteCatalog getCatalog(final String defaultNamespace) { - return new AirbyteCatalog().withStreams(List.of( - CatalogHelpers.createAirbyteStream( - TABLE_NAME, - defaultNamespace, - Field.of(COL_ID, JsonSchemaType.INTEGER), - Field.of(COL_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) - .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) - .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID))), - CatalogHelpers.createAirbyteStream( - TABLE_NAME_WITHOUT_PK, - defaultNamespace, - Field.of(COL_ID, JsonSchemaType.INTEGER), - Field.of(COL_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) - .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) - .withSourceDefinedPrimaryKey(Collections.emptyList()), - CatalogHelpers.createAirbyteStream( - TABLE_NAME_COMPOSITE_PK, - defaultNamespace, - Field.of(COL_FIRST_NAME, JsonSchemaType.STRING), - Field.of(COL_LAST_NAME, JsonSchemaType.STRING), - Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) - .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) - .withSourceDefinedPrimaryKey( - List.of(List.of(COL_FIRST_NAME), List.of(COL_LAST_NAME))))); + protected DbStreamState buildStreamState(final ConfiguredAirbyteStream configuredAirbyteStream, + final String cursorField, + final String cursorValue) { + return new CursorBasedStatus().withStateType(StateType.CURSOR_BASED).withVersion(2L) + .withStreamName(configuredAirbyteStream.getStream().getName()) + .withStreamNamespace(configuredAirbyteStream.getStream().getNamespace()) + .withCursorField(List.of(cursorField)) + .withCursor(cursorValue) + .withCursorRecordCount(1L); } @Override - protected void incrementalDateCheck() throws Exception { - incrementalCursorCheck( - COL_UPDATED_AT, - "2005-10-18", - "2006-10-19", - List.of(getTestMessages().get(1), getTestMessages().get(2))); + protected List getExpectedAirbyteMessagesSecondSync(final String namespace) { + final List expectedMessages = new ArrayList<>(); + expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName()).withNamespace(namespace) + .withData(Jsons.jsonNode(ImmutableMap + .of(COL_ID, ID_VALUE_4, + COL_NAME, "riker", + COL_UPDATED_AT, "2006-10-19"))))); + expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName()).withNamespace(namespace) + .withData(Jsons.jsonNode(ImmutableMap + .of(COL_ID, ID_VALUE_5, + COL_NAME, "data", + COL_UPDATED_AT, "2006-10-19"))))); + final DbStreamState state = new CursorBasedStatus() + .withStateType(StateType.CURSOR_BASED) + .withVersion(2L) + .withStreamName(streamName()) + .withStreamNamespace(namespace) + .withCursorField(ImmutableList.of(COL_ID)) + .withCursor("5") + .withCursorRecordCount(1L); + + expectedMessages.addAll(createExpectedTestMessages(List.of(state), 2L)); + return expectedMessages; } @Override protected List getTestMessages() { + return getTestMessages(streamName()); + } + + protected List getTestMessages(final String streamName) { return List.of( new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) @@ -273,34 +435,81 @@ protected List getTestMessages() { COL_UPDATED_AT, "2006-10-19"))))); } + private AirbyteStream getAirbyteStream(final String tableName, final String namespace) { + return CatalogHelpers.createAirbyteStream( + tableName, + namespace, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID))); + } + @Override - protected List getExpectedAirbyteMessagesSecondSync(final String namespace) { - final List expectedMessages = new ArrayList<>(); - expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_4, - COL_NAME, "riker", - COL_UPDATED_AT, "2006-10-19"))))); - expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_5, - COL_NAME, "data", - COL_UPDATED_AT, "2006-10-19"))))); - final DbStreamState state = new DbStreamState() - .withStreamName(streamName) - .withStreamNamespace(namespace) - .withCursorField(List.of(COL_ID)) - .withCursor("5") - .withCursorRecordCount(1L); - expectedMessages.addAll(createExpectedTestMessages(List.of(state))); - return expectedMessages; + protected AirbyteCatalog getCatalog(final String defaultNamespace) { + return new AirbyteCatalog().withStreams(Lists.newArrayList( + CatalogHelpers.createAirbyteStream( + TABLE_NAME, + defaultNamespace, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID))), + CatalogHelpers.createAirbyteStream( + TABLE_NAME_WITHOUT_PK, + defaultNamespace, + Field.of(COL_ID, JsonSchemaType.INTEGER), + Field.of(COL_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(Collections.emptyList()), + CatalogHelpers.createAirbyteStream( + TABLE_NAME_COMPOSITE_PK, + defaultNamespace, + Field.of(COL_FIRST_NAME, JsonSchemaType.STRING), + Field.of(COL_LAST_NAME, JsonSchemaType.STRING), + Field.of(COL_UPDATED_AT, JsonSchemaType.STRING_DATE)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey( + List.of(List.of(COL_FIRST_NAME), List.of(COL_LAST_NAME))))); } + // Override from parent class as we're no longer including the legacy Data field. @Override - protected boolean supportsPerStream() { - return true; + protected List createExpectedTestMessages(final List states, final long numRecords) { + return states.stream() + .map(s -> new AirbyteMessage().withType(Type.STATE) + .withState( + new AirbyteStateMessage().withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) + .withStreamState(Jsons.jsonNode(s))) + .withSourceStats(new AirbyteStateStats().withRecordCount((double) numRecords)))) + .collect( + Collectors.toList()); + } + + @Override + protected List createState(final List states) { + return states.stream() + .map(s -> new AirbyteStateMessage().withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) + .withStreamState(Jsons.jsonNode(s)))) + .collect( + Collectors.toList()); + } + + @Override + protected JsonNode getStateData(final AirbyteMessage airbyteMessage, final String streamName) { + final JsonNode streamState = airbyteMessage.getState().getStream().getStreamState(); + if (streamState.get("stream_name").asText().equals(streamName)) { + return streamState; + } + + throw new IllegalArgumentException("Stream not found in state message: " + streamName); } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceOperationsTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceOperationsTest.java index 60f6b2c2399e..b989e2163c84 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceOperationsTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceOperationsTest.java @@ -4,17 +4,15 @@ package io.airbyte.integrations.source.mysql; -import static io.airbyte.integrations.source.mysql.MySqlSource.DRIVER_CLASS; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.mysql.cj.MysqlType; +import io.airbyte.cdk.db.jdbc.DateTimeConverter; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.jdbc.DateTimeConverter; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -27,280 +25,107 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; +import java.util.function.Function; +import java.util.function.IntFunction; import org.junit.jupiter.api.Test; -import org.testcontainers.containers.MySQLContainer; public class MySqlSourceOperationsTest { - private final MySqlSourceOperations sqlSourceOperations = new MySqlSourceOperations(); - private MySQLContainer container; - private Database database; - - @BeforeEach - public void init() { - container = new MySQLContainer<>("mysql:8.0"); - container.start(); - database = new Database(DSLContextFactory.create( - "root", - "test", - DRIVER_CLASS, - String.format("jdbc:mysql://%s:%s", - container.getHost(), - container.getFirstMappedPort()), - SQLDialect.MYSQL)); - } - - @AfterEach - public void tearDown() { - try { - container.close(); - } catch (final Exception e) { - throw new RuntimeException(e); - } - } - @Test public void dateColumnAsCursor() throws SQLException { - final String tableName = container.getDatabaseName() + ".table_with_date"; - final String cursorColumn = "cursor_column"; - executeQuery("CREATE TABLE " + tableName + "(id INTEGER PRIMARY KEY, " + cursorColumn + " DATE);"); - - final List expectedRecords = new ArrayList<>(); - for (int i = 1; i <= 4; i++) { - final ObjectNode jsonNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); - jsonNode.put("id", i); - final LocalDate cursorValue = LocalDate.of(2019, 1, i); - jsonNode.put("cursor_column", DateTimeConverter.convertToDate(cursorValue)); - executeQuery("INSERT INTO " + tableName + " VALUES (" + i + ", '" + cursorValue + "');"); - if (i >= 2) { - expectedRecords.add(jsonNode); - } - } - - final List actualRecords = new ArrayList<>(); - try (final Connection connection = container.createConnection("")) { - final PreparedStatement preparedStatement = connection.prepareStatement( - "SELECT * from " + tableName + " WHERE " + cursorColumn + " > ?"); - sqlSourceOperations.setCursorField(preparedStatement, 1, MysqlType.DATE, DateTimeConverter.convertToDate(LocalDate.of(2019, 1, 1))); - - try (final ResultSet resultSet = preparedStatement.executeQuery()) { - while (resultSet.next()) { - final ObjectNode jsonNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); - for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) { - sqlSourceOperations.copyToJsonField(resultSet, i, jsonNode); - } - actualRecords.add(jsonNode); - } - } - } - assertThat(actualRecords, containsInAnyOrder(expectedRecords.toArray())); - - // Test to check backward compatibility for connectors created before PR - // https://github.com/airbytehq/airbyte/pull/15504 - actualRecords.clear(); - try (final Connection connection = container.createConnection("")) { - final PreparedStatement preparedStatement = connection.prepareStatement( - "SELECT * from " + tableName + " WHERE " + cursorColumn + " > ?"); - sqlSourceOperations.setCursorField(preparedStatement, 1, MysqlType.DATE, "2019-01-01T00:00:00Z"); - - try (final ResultSet resultSet = preparedStatement.executeQuery()) { - while (resultSet.next()) { - final ObjectNode jsonNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); - for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) { - sqlSourceOperations.copyToJsonField(resultSet, i, jsonNode); - } - actualRecords.add(jsonNode); - } - } - } - assertThat(actualRecords, containsInAnyOrder(expectedRecords.toArray())); + testImpl( + "DATE", + i -> LocalDate.of(2019, 1, i), + DateTimeConverter::convertToDate, + LocalDate::toString, + MysqlType.DATE, + DateTimeConverter.convertToDate(LocalDate.of(2019, 1, 1)), + "2019-01-01T00:00:00Z"); } @Test public void timeColumnAsCursor() throws SQLException { - final String tableName = container.getDatabaseName() + ".table_with_time"; - final String cursorColumn = "cursor_column"; - executeQuery("CREATE TABLE " + tableName + "(id INTEGER PRIMARY KEY, " + cursorColumn + " TIME);"); - - final List expectedRecords = new ArrayList<>(); - for (int i = 1; i <= 4; i++) { - final ObjectNode jsonNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); - jsonNode.put("id", i); - final LocalTime cursorValue = LocalTime.of(20, i, 0); - jsonNode.put("cursor_column", DateTimeConverter.convertToTime(cursorValue)); - executeQuery("INSERT INTO " + tableName + " VALUES (" + i + ", '" + cursorValue + "');"); - if (i >= 2) { - expectedRecords.add(jsonNode); - } - } - - final List actualRecords = new ArrayList<>(); - try (final Connection connection = container.createConnection("")) { - final PreparedStatement preparedStatement = connection.prepareStatement( - "SELECT * from " + tableName + " WHERE " + cursorColumn + " > ?"); - sqlSourceOperations.setCursorField(preparedStatement, 1, MysqlType.TIME, DateTimeConverter.convertToTime(LocalTime.of(20, 1, 0))); - - try (final ResultSet resultSet = preparedStatement.executeQuery()) { - while (resultSet.next()) { - final ObjectNode jsonNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); - for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) { - sqlSourceOperations.copyToJsonField(resultSet, i, jsonNode); - } - actualRecords.add(jsonNode); - } - } - } - assertThat(actualRecords, containsInAnyOrder(expectedRecords.toArray())); - - // Test to check backward compatibility for connectors created before PR - // https://github.com/airbytehq/airbyte/pull/15504 - actualRecords.clear(); - try (final Connection connection = container.createConnection("")) { - final PreparedStatement preparedStatement = connection.prepareStatement( - "SELECT * from " + tableName + " WHERE " + cursorColumn + " > ?"); - sqlSourceOperations.setCursorField(preparedStatement, 1, MysqlType.TIME, "1970-01-01T20:01:00Z"); - - try (final ResultSet resultSet = preparedStatement.executeQuery()) { - while (resultSet.next()) { - final ObjectNode jsonNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); - for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) { - sqlSourceOperations.copyToJsonField(resultSet, i, jsonNode); - } - actualRecords.add(jsonNode); - } - } - } + testImpl( + "TIME", + i -> LocalTime.of(20, i, 0), + DateTimeConverter::convertToTime, + LocalTime::toString, + MysqlType.TIME, + DateTimeConverter.convertToTime(LocalTime.of(20, 1, 0)), + "1970-01-01T20:01:00Z"); } @Test public void dateTimeColumnAsCursor() throws SQLException { - final String tableName = container.getDatabaseName() + ".table_with_datetime"; - final String cursorColumn = "cursor_column"; - executeQuery("CREATE TABLE " + tableName + "(id INTEGER PRIMARY KEY, " + cursorColumn + " DATETIME);"); - - final List expectedRecords = new ArrayList<>(); - for (int i = 1; i <= 4; i++) { - final ObjectNode jsonNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); - jsonNode.put("id", i); - final LocalDateTime cursorValue = LocalDateTime.of(2019, i, 20, 3, 0, 0); - jsonNode.put("cursor_column", DateTimeConverter.convertToTimestamp(cursorValue)); - executeQuery("INSERT INTO " + tableName + " VALUES (" + i + ", '" + cursorValue + "');"); - if (i >= 2) { - expectedRecords.add(jsonNode); - } - } - - final List actualRecords = new ArrayList<>(); - try (final Connection connection = container.createConnection("")) { - final PreparedStatement preparedStatement = connection.prepareStatement( - "SELECT * from " + tableName + " WHERE " + cursorColumn + " > ?"); - sqlSourceOperations.setCursorField(preparedStatement, 1, MysqlType.DATETIME, - DateTimeConverter.convertToTimestamp(LocalDateTime.of(2019, 1, 20, 3, 0, 0))); - - try (final ResultSet resultSet = preparedStatement.executeQuery()) { - while (resultSet.next()) { - final ObjectNode jsonNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); - for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) { - sqlSourceOperations.copyToJsonField(resultSet, i, jsonNode); - } - actualRecords.add(jsonNode); - } - } - } - assertThat(actualRecords, containsInAnyOrder(expectedRecords.toArray())); - - // Test to check backward compatibility for connectors created before PR - // https://github.com/airbytehq/airbyte/pull/15504 - actualRecords.clear(); - try (final Connection connection = container.createConnection("")) { - final PreparedStatement preparedStatement = connection.prepareStatement( - "SELECT * from " + tableName + " WHERE " + cursorColumn + " > ?"); - sqlSourceOperations.setCursorField(preparedStatement, 1, MysqlType.DATETIME, "2019-01-20T03:00:00.000000"); - - try (final ResultSet resultSet = preparedStatement.executeQuery()) { - while (resultSet.next()) { - final ObjectNode jsonNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); - for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) { - sqlSourceOperations.copyToJsonField(resultSet, i, jsonNode); - } - actualRecords.add(jsonNode); - } - } - } - assertThat(actualRecords, containsInAnyOrder(expectedRecords.toArray())); + testImpl( + "DATETIME", + i -> LocalDateTime.of(2019, i, 20, 3, 0, 0), + DateTimeConverter::convertToTimestamp, + LocalDateTime::toString, + MysqlType.DATETIME, + DateTimeConverter.convertToTimestamp(LocalDateTime.of(2019, 1, 20, 3, 0, 0)), + "2019-01-20T03:00:00.000000"); } @Test public void timestampColumnAsCursor() throws SQLException { - final String tableName = container.getDatabaseName() + ".table_with_timestamp"; - final String cursorColumn = "cursor_column"; - executeQuery("CREATE TABLE " + tableName + "(id INTEGER PRIMARY KEY, " + cursorColumn + " timestamp);"); - - final List expectedRecords = new ArrayList<>(); - for (int i = 1; i <= 4; i++) { - final ObjectNode jsonNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); - jsonNode.put("id", i); - final Instant cursorValue = Instant.ofEpochSecond(1660298508L).plusSeconds(i - 1); - jsonNode.put("cursor_column", DateTimeConverter.convertToTimestampWithTimezone(cursorValue)); - executeQuery("INSERT INTO " + tableName + " VALUES (" + i + ", '" + Timestamp.from(cursorValue) + "');"); - if (i >= 2) { - expectedRecords.add(jsonNode); - } - } - - final List actualRecords = new ArrayList<>(); - try (final Connection connection = container.createConnection("")) { - final PreparedStatement preparedStatement = connection.prepareStatement( - "SELECT * from " + tableName + " WHERE " + cursorColumn + " > ?"); - sqlSourceOperations.setCursorField(preparedStatement, 1, MysqlType.TIMESTAMP, - DateTimeConverter.convertToTimestampWithTimezone(Instant.ofEpochSecond(1660298508L))); + testImpl( + "TIMESTAMP", + i -> Instant.ofEpochSecond(1660298508L).plusSeconds(i - 1), + DateTimeConverter::convertToTimestampWithTimezone, + r -> Timestamp.from(r).toString(), + MysqlType.TIMESTAMP, + DateTimeConverter.convertToTimestampWithTimezone(Instant.ofEpochSecond(1660298508L)), + Instant.ofEpochSecond(1660298508L).toString()); + } - try (final ResultSet resultSet = preparedStatement.executeQuery()) { - while (resultSet.next()) { - final ObjectNode jsonNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); - for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) { - sqlSourceOperations.copyToJsonField(resultSet, i, jsonNode); - } - actualRecords.add(jsonNode); + private void testImpl( + final String sqlType, + IntFunction recordBuilder, + Function airbyteRecordStringifier, + Function sqlRecordStringifier, + MysqlType mysqlType, + String initialCursorFieldValue, + // Test to check backward compatibility for connectors created before PR + // https://github.com/airbytehq/airbyte/pull/15504 + String backwardCompatibleInitialCursorFieldValue) + throws SQLException { + final var sqlSourceOperations = new MySqlSourceOperations(); + final String cursorColumn = "cursor_column"; + try (final var testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8) + .with("CREATE TABLE cursor_table (id INTEGER PRIMARY KEY, %s %s);", cursorColumn, sqlType)) { + + final List expectedRecords = new ArrayList<>(); + for (int i = 1; i <= 4; i++) { + final ObjectNode jsonNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); + jsonNode.put("id", i); + final T cursorValue = recordBuilder.apply(i); + jsonNode.put("cursor_column", airbyteRecordStringifier.apply(cursorValue)); + testdb.with("INSERT INTO cursor_table VALUES (%d, '%s');", i, sqlRecordStringifier.apply(cursorValue)); + if (i >= 2) { + expectedRecords.add(jsonNode); } } - } - - Assertions.assertEquals(3, actualRecords.size()); - // Test to check backward compatibility for connectors created before PR - // https://github.com/airbytehq/airbyte/pull/15504 - actualRecords.clear(); - try (final Connection connection = container.createConnection("")) { - final PreparedStatement preparedStatement = connection.prepareStatement( - "SELECT * from " + tableName + " WHERE " + cursorColumn + " > ?"); - sqlSourceOperations.setCursorField(preparedStatement, 1, MysqlType.TIMESTAMP, Instant.ofEpochSecond(1660298508L).toString()); - - try (final ResultSet resultSet = preparedStatement.executeQuery()) { - while (resultSet.next()) { - final ObjectNode jsonNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); - for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) { - sqlSourceOperations.copyToJsonField(resultSet, i, jsonNode); + try (final Connection connection = testdb.getContainer().createConnection("")) { + final PreparedStatement preparedStatement = connection.prepareStatement( + "SELECT * FROM " + testdb.getDatabaseName() + ".cursor_table WHERE " + cursorColumn + " > ?"); + for (final var initialValue : List.of(initialCursorFieldValue, backwardCompatibleInitialCursorFieldValue)) { + sqlSourceOperations.setCursorField(preparedStatement, 1, mysqlType, initialValue); + final List actualRecords = new ArrayList<>(); + try (final ResultSet resultSet = preparedStatement.executeQuery()) { + while (resultSet.next()) { + final ObjectNode jsonNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); + for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) { + sqlSourceOperations.copyToJsonField(resultSet, i, jsonNode); + } + actualRecords.add(jsonNode); + } } - actualRecords.add(jsonNode); + assertThat(actualRecords, containsInAnyOrder(expectedRecords.toArray())); } } } - Assertions.assertEquals(3, actualRecords.size()); - } - - protected void executeQuery(final String query) { - try { - database.query( - ctx -> ctx - .execute(query)); - } catch (final SQLException e) { - throw new RuntimeException(e); - } } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java index 09bea61fd2ba..ef8bb7646677 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSourceTests.java @@ -13,13 +13,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource.PrimaryKeyAttributesFromDb; import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.string.Strings; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource.PrimaryKeyAttributesFromDb; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.BaseImage; +import io.airbyte.integrations.source.mysql.MySQLTestDatabase.ContainerModifier; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; @@ -28,71 +29,34 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; import io.airbyte.protocol.models.v0.SyncMode; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Properties; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testcontainers.containers.MySQLContainer; -import org.testcontainers.containers.output.Slf4jLogConsumer; public class MySqlSourceTests { - private static final Logger LOGGER = LoggerFactory.getLogger(MySqlSourceTests.class); - - private static final String TEST_USER = "test"; - private static final String TEST_PASSWORD = "test"; + public MySqlSource source() { + return new MySqlSource(); + } @Test public void testSettingTimezones() throws Exception { - // start DB - try (final MySQLContainer container = new MySQLContainer<>("mysql:8.0") - .withUsername(TEST_USER) - .withPassword(TEST_PASSWORD) - .withEnv("MYSQL_ROOT_HOST", "%") - .withEnv("MYSQL_ROOT_PASSWORD", TEST_PASSWORD) - .withEnv("TZ", "Europe/Moscow") - .withLogConsumer(new Slf4jLogConsumer(LOGGER))) { - - container.start(); - - final Properties properties = new Properties(); - properties.putAll(ImmutableMap.of("user", "root", JdbcUtils.PASSWORD_KEY, TEST_PASSWORD, "serverTimezone", "Europe/Moscow")); - DriverManager.getConnection(container.getJdbcUrl(), properties); - final String dbName = Strings.addRandomSuffix("db", "_", 10); - final JsonNode config = getConfig(container, dbName, "serverTimezone=Europe/Moscow"); - - try (final Connection connection = DriverManager.getConnection(container.getJdbcUrl(), properties)) { - connection.createStatement().execute("GRANT ALL PRIVILEGES ON *.* TO '" + TEST_USER + "'@'%';\n"); - connection.createStatement().execute("CREATE DATABASE " + config.get(JdbcUtils.DATABASE_KEY).asText()); - } - final AirbyteConnectionStatus check = new MySqlSource().check(config); - assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, check.getStatus()); + try (final var testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8, ContainerModifier.MOSCOW_TIMEZONE)) { + final var config = testdb.testConfigBuilder() + .with(JdbcUtils.JDBC_URL_PARAMS_KEY, "serverTimezone=Europe/Moscow") + .withoutSsl() + .build(); + final AirbyteConnectionStatus check = source().check(config); + assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, check.getStatus(), check.getMessage()); } } - private static JsonNode getConfig(final MySQLContainer dbContainer, final String dbName, final String jdbcParams) { - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, dbContainer.getHost()) - .put(JdbcUtils.PORT_KEY, dbContainer.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, dbName) - .put(JdbcUtils.USERNAME_KEY, TEST_USER) - .put(JdbcUtils.PASSWORD_KEY, TEST_PASSWORD) - .put(JdbcUtils.JDBC_URL_PARAMS_KEY, jdbcParams) - .build()); - } - @Test void testJdbcUrlWithEscapedDatabaseName() { - final JsonNode jdbcConfig = new MySqlSource().toDatabaseConfig(buildConfigEscapingNeeded()); + final JsonNode jdbcConfig = source().toDatabaseConfig(buildConfigEscapingNeeded()); assertNotNull(jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText()); assertTrue(jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText().startsWith(EXPECTED_JDBC_ESCAPED_URL)); } @@ -109,95 +73,45 @@ private JsonNode buildConfigEscapingNeeded() { @Test @Disabled("See https://github.com/airbytehq/airbyte/pull/23908#issuecomment-1463753684, enable once communication is out") - public void testTableWithNullCursorValueShouldThrowException() throws SQLException { - try (final MySQLContainer db = new MySQLContainer<>("mysql:8.0") - .withUsername(TEST_USER) - .withPassword(TEST_PASSWORD) - .withEnv("MYSQL_ROOT_HOST", "%") - .withEnv("MYSQL_ROOT_PASSWORD", TEST_PASSWORD)) { - db.start(); - final JsonNode config = getConfig(db, "test", ""); - try (Connection connection = DriverManager.getConnection(db.getJdbcUrl(), "root", config.get(JdbcUtils.PASSWORD_KEY).asText())) { - final ConfiguredAirbyteStream table = createTableWithNullValueCursor(connection); - final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(Collections.singletonList(table)); - - final Throwable throwable = catchThrowable(() -> MoreIterators.toSet(new MySqlSource().read(config, catalog, null))); - assertThat(throwable).isInstanceOf(ConfigErrorException.class) - .hasMessageContaining( - "The following tables have invalid columns selected as cursor, please select a column with a well-defined ordering with no null values as a cursor. {tableName='test.null_cursor_table', cursorColumnName='id', cursorSqlType=INT, cause=Cursor column contains NULL value}"); - - } finally { - db.stop(); - } + public void testNullCursorValueShouldThrowException() { + try (final var testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8) + .with("CREATE TABLE null_cursor_table(id INTEGER NULL);") + .with("INSERT INTO null_cursor_table(id) VALUES (1), (2), (NULL);") + .with("CREATE VIEW null_cursor_view(id) AS SELECT null_cursor_table.id FROM null_cursor_table;")) { + final var config = testdb.testConfigBuilder().withoutSsl().build(); + + final var tableStream = new ConfiguredAirbyteStream() + .withCursorField(Lists.newArrayList("id")) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withSyncMode(SyncMode.INCREMENTAL) + .withStream(CatalogHelpers.createAirbyteStream( + "null_cursor_table", + testdb.getDatabaseName(), + Field.of("id", JsonSchemaType.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of("id")))); + final var tableCatalog = new ConfiguredAirbyteCatalog().withStreams(List.of(tableStream)); + final var tableThrowable = catchThrowable(() -> MoreIterators.toSet(source().read(config, tableCatalog, null))); + assertThat(tableThrowable).isInstanceOf(ConfigErrorException.class).hasMessageContaining(NULL_CURSOR_EXCEPTION_MESSAGE_CONTAINS); + + final var viewStream = new ConfiguredAirbyteStream() + .withCursorField(Lists.newArrayList("id")) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withSyncMode(SyncMode.INCREMENTAL) + .withStream(CatalogHelpers.createAirbyteStream( + "null_cursor_view", + testdb.getDatabaseName(), + Field.of("id", JsonSchemaType.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of("id")))); + final var viewCatalog = new ConfiguredAirbyteCatalog().withStreams(List.of(viewStream)); + final var viewThrowable = catchThrowable(() -> MoreIterators.toSet(source().read(config, viewCatalog, null))); + assertThat(viewThrowable).isInstanceOf(ConfigErrorException.class).hasMessageContaining(NULL_CURSOR_EXCEPTION_MESSAGE_CONTAINS); } } - private ConfiguredAirbyteStream createTableWithNullValueCursor(final Connection connection) throws SQLException { - connection.createStatement().execute("GRANT ALL PRIVILEGES ON *.* TO '" + TEST_USER + "'@'%';\n"); - connection.createStatement().execute("CREATE TABLE IF NOT EXISTS test.null_cursor_table(id INTEGER NULL)"); - connection.createStatement().execute("INSERT INTO test.null_cursor_table(id) VALUES (1), (2), (NULL)"); - - return new ConfiguredAirbyteStream().withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(Lists.newArrayList("id")) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withSyncMode(SyncMode.INCREMENTAL) - .withStream(CatalogHelpers.createAirbyteStream( - "null_cursor_table", - "test", - Field.of("id", JsonSchemaType.STRING)) - .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) - .withSourceDefinedPrimaryKey(List.of(List.of("id")))); - - } - - @Test - @Disabled("See https://github.com/airbytehq/airbyte/pull/23908#issuecomment-1463753684, enable once communication is out") - public void viewWithNullValueCursorShouldThrowException() throws SQLException { - try (final MySQLContainer db = new MySQLContainer<>("mysql:8.0") - .withUsername(TEST_USER) - .withPassword(TEST_PASSWORD) - .withEnv("MYSQL_ROOT_HOST", "%") - .withEnv("MYSQL_ROOT_PASSWORD", TEST_PASSWORD)) { - db.start(); - final JsonNode config = getConfig(db, "test", ""); - try (Connection connection = DriverManager.getConnection(db.getJdbcUrl(), "root", config.get(JdbcUtils.PASSWORD_KEY).asText())) { - final ConfiguredAirbyteStream table = createViewWithNullValueCursor(connection); - final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(Collections.singletonList(table)); - - final Throwable throwable = catchThrowable(() -> MoreIterators.toSet(new MySqlSource().read(config, catalog, null))); - assertThat(throwable).isInstanceOf(ConfigErrorException.class) - .hasMessageContaining( - "The following tables have invalid columns selected as cursor, please select a column with a well-defined ordering with no null values as a cursor. {tableName='test.test_view_null_cursor', cursorColumnName='id', cursorSqlType=INT, cause=Cursor column contains NULL value}"); - - } finally { - db.stop(); - } - } - } - - private ConfiguredAirbyteStream createViewWithNullValueCursor(final Connection connection) throws SQLException { - - connection.createStatement().execute("GRANT ALL PRIVILEGES ON *.* TO '" + TEST_USER + "'@'%';\n"); - connection.createStatement().execute("CREATE TABLE IF NOT EXISTS test.test_table_null_cursor(id INTEGER NULL)"); - connection.createStatement().execute(""" - CREATE VIEW test_view_null_cursor(id) as - SELECT test_table_null_cursor.id - FROM test_table_null_cursor - """); - connection.createStatement().execute("INSERT INTO test.test_table_null_cursor(id) VALUES (1), (2), (NULL)"); - - return new ConfiguredAirbyteStream().withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(Lists.newArrayList("id")) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withSyncMode(SyncMode.INCREMENTAL) - .withStream(CatalogHelpers.createAirbyteStream( - "test_view_null_cursor", - "test", - Field.of("id", JsonSchemaType.STRING)) - .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) - .withSourceDefinedPrimaryKey(List.of(List.of("id")))); - - } + static private final String NULL_CURSOR_EXCEPTION_MESSAGE_CONTAINS = "The following tables have invalid columns " + + "selected as cursor, please select a column with a well-defined ordering with no null values as a cursor."; @Test void testParseJdbcParameters() { @@ -210,26 +124,12 @@ void testParseJdbcParameters() { @Test public void testJDBCSessionVariable() throws Exception { - // start DB - try (final MySQLContainer container = new MySQLContainer<>("mysql:8.0") - .withUsername(TEST_USER) - .withPassword(TEST_PASSWORD) - .withEnv("MYSQL_ROOT_HOST", "%") - .withEnv("MYSQL_ROOT_PASSWORD", TEST_PASSWORD) - .withLogConsumer(new Slf4jLogConsumer(LOGGER))) { - - container.start(); - final Properties properties = new Properties(); - properties.putAll(ImmutableMap.of("user", "root", JdbcUtils.PASSWORD_KEY, TEST_PASSWORD)); - DriverManager.getConnection(container.getJdbcUrl(), properties); - final String dbName = Strings.addRandomSuffix("db", "_", 10); - final JsonNode config = getConfig(container, dbName, "sessionVariables=MAX_EXECUTION_TIME=28800000"); - - try (final Connection connection = DriverManager.getConnection(container.getJdbcUrl(), properties)) { - connection.createStatement().execute("GRANT ALL PRIVILEGES ON *.* TO '" + TEST_USER + "'@'%';\n"); - connection.createStatement().execute("CREATE DATABASE " + config.get(JdbcUtils.DATABASE_KEY).asText()); - } - final AirbyteConnectionStatus check = new MySqlSource().check(config); + try (final var testdb = MySQLTestDatabase.in(BaseImage.MYSQL_8)) { + final var config = testdb.testConfigBuilder() + .with(JdbcUtils.JDBC_URL_PARAMS_KEY, "sessionVariables=MAX_EXECUTION_TIME=28800000") + .withoutSsl() + .build(); + final AirbyteConnectionStatus check = source().check(config); assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, check.getStatus()); } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSslJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSslJdbcSourceAcceptanceTest.java index cb6b1ead8267..5d5ac314a928 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSslJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlSslJdbcSourceAcceptanceTest.java @@ -4,49 +4,27 @@ package io.airbyte.integrations.source.mysql; -import static io.airbyte.integrations.source.mysql.MySqlSource.SSL_PARAMETERS; - -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.string.Strings; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.BeforeEach; +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import org.junit.jupiter.api.Order; +@Order(3) class MySqlSslJdbcSourceAcceptanceTest extends MySqlJdbcSourceAcceptanceTest { - @BeforeEach - public void setup() throws Exception { - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, Strings.addRandomSuffix("db", "_", 10)) - .put(JdbcUtils.USERNAME_KEY, TEST_USER) - .put(JdbcUtils.PASSWORD_KEY, TEST_PASSWORD.call()) - .put(JdbcUtils.SSL_KEY, true) - .build()); - - dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.MYSQL.getDriverClassName(), - String.format("jdbc:mysql://%s:%s?%s", - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asText(), - String.join("&", SSL_PARAMETERS)), - SQLDialect.MYSQL); - database = new Database(dslContext); - - database.query(ctx -> { - ctx.fetch("CREATE DATABASE " + config.get(JdbcUtils.DATABASE_KEY).asText()); - ctx.fetch("SHOW STATUS LIKE 'Ssl_cipher'"); - return null; - }); + @Override + protected JsonNode config() { + return testdb.testConfigBuilder() + .with(JdbcUtils.SSL_KEY, true) + .build(); + } - super.setup(); + @Override + protected MySQLTestDatabase createTestDatabase() { + return new MySQLTestDatabase(new MySQLContainerFactory().shared("mysql:8.0")) + .withConnectionProperty("useSSL", "true") + .withConnectionProperty("requireSSL", "true") + .initialized() + .with("SHOW STATUS LIKE 'Ssl_cipher'"); } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java index 5820451ed254..febcd7a5c0c1 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test/java/io/airbyte/integrations/source/mysql/MySqlStressTest.java @@ -7,14 +7,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.mysql.cj.MysqlType; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.cdk.integrations.source.jdbc.test.JdbcStressTest; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.test.JdbcStressTest; import java.sql.Connection; import java.sql.DriverManager; import java.util.Optional; diff --git a/airbyte-integrations/connectors/source-mysql/src/test/resources/expected_cloud_spec.json b/airbyte-integrations/connectors/source-mysql/src/test/resources/expected_cloud_spec.json new file mode 100644 index 000000000000..52441e124b17 --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/test/resources/expected_cloud_spec.json @@ -0,0 +1,227 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/mysql", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "MySql Source Spec", + "type": "object", + "required": ["host", "port", "database", "username", "replication_method"], + "properties": { + "host": { + "description": "The host name of the database.", + "title": "Host", + "type": "string", + "order": 0 + }, + "port": { + "description": "The port to connect to.", + "title": "Port", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 3306, + "examples": ["3306"], + "order": 1 + }, + "database": { + "description": "The database name.", + "title": "Database", + "type": "string", + "order": 2 + }, + "username": { + "description": "The username which is used to access the database.", + "title": "Username", + "type": "string", + "order": 3 + }, + "password": { + "description": "The password associated with the username.", + "title": "Password", + "type": "string", + "airbyte_secret": true, + "order": 4, + "always_show": true + }, + "jdbc_url_params": { + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", + "title": "JDBC URL Parameters (Advanced)", + "type": "string", + "order": 5 + }, + "ssl_mode": { + "title": "SSL modes", + "description": "SSL connection modes. Read more in the docs.", + "type": "object", + "order": 7, + "oneOf": [ + { + "title": "preferred", + "description": "Automatically attempt SSL connection. If the MySQL server does not support SSL, continue with a regular connection.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "preferred", + "order": 0 + } + } + }, + { + "title": "required", + "description": "Always connect with SSL. If the MySQL server doesn’t support SSL, the connection will not be established. Certificate Authority (CA) and Hostname are not verified.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "required", + "order": 0 + } + } + }, + { + "title": "Verify CA", + "description": "Always connect with SSL. Verifies CA, but allows connection even if Hostname does not match.", + "required": ["mode", "ca_certificate"], + "properties": { + "mode": { + "type": "string", + "const": "verify_ca", + "order": 0 + }, + "ca_certificate": { + "type": "string", + "title": "CA certificate", + "description": "CA certificate", + "airbyte_secret": true, + "multiline": true, + "order": 1 + }, + "client_certificate": { + "type": "string", + "title": "Client certificate", + "description": "Client certificate (this is not a required field, but if you want to use it, you will need to add the Client key as well)", + "airbyte_secret": true, + "multiline": true, + "order": 2, + "always_show": true + }, + "client_key": { + "type": "string", + "title": "Client key", + "description": "Client key (this is not a required field, but if you want to use it, you will need to add the Client certificate as well)", + "airbyte_secret": true, + "multiline": true, + "order": 3, + "always_show": true + }, + "client_key_password": { + "type": "string", + "title": "Client key password", + "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", + "airbyte_secret": true, + "order": 4 + } + } + }, + { + "title": "Verify Identity", + "description": "Always connect with SSL. Verify both CA and Hostname.", + "required": ["mode", "ca_certificate"], + "properties": { + "mode": { + "type": "string", + "const": "verify_identity", + "order": 0 + }, + "ca_certificate": { + "type": "string", + "title": "CA certificate", + "description": "CA certificate", + "airbyte_secret": true, + "multiline": true, + "order": 1 + }, + "client_certificate": { + "type": "string", + "title": "Client certificate", + "description": "Client certificate (this is not a required field, but if you want to use it, you will need to add the Client key as well)", + "airbyte_secret": true, + "multiline": true, + "order": 2, + "always_show": true + }, + "client_key": { + "type": "string", + "title": "Client key", + "description": "Client key (this is not a required field, but if you want to use it, you will need to add the Client certificate as well)", + "airbyte_secret": true, + "multiline": true, + "order": 3, + "always_show": true + }, + "client_key_password": { + "type": "string", + "title": "Client key password", + "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", + "airbyte_secret": true, + "order": 4 + } + } + } + ], + "default": "required" + }, + "replication_method": { + "type": "object", + "title": "Update Method", + "description": "Configures how data is extracted from the database.", + "order": 8, + "default": "CDC", + "display_type": "radio", + "oneOf": [ + { + "title": "Read Changes using Binary Log (CDC)", + "description": "Recommended - Incrementally reads new inserts, updates, and deletes using the MySQL binary log. This must be enabled on your database.", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "CDC", + "order": 0 + }, + "initial_waiting_seconds": { + "type": "integer", + "title": "Initial Waiting Time in Seconds (Advanced)", + "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", + "default": 300, + "min": 120, + "max": 1200, + "order": 1, + "always_show": true + }, + "server_time_zone": { + "type": "string", + "title": "Configured server timezone for the MySQL source (Advanced)", + "description": "Enter the configured MySQL server timezone. This should only be done if the configured timezone in your MySQL instance does not conform to IANNA standard.", + "order": 2, + "always_show": true + } + } + }, + { + "title": "Scan Changes with User Defined Cursor", + "description": "Incrementally detects new inserts and updates using the cursor column chosen when configuring a connection (e.g. created_at, updated_at).", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "STANDARD", + "order": 0 + } + } + } + ] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-mysql/src/test/resources/expected_oss_spec.json b/airbyte-integrations/connectors/source-mysql/src/test/resources/expected_oss_spec.json new file mode 100644 index 000000000000..841fa1f3bdba --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/test/resources/expected_oss_spec.json @@ -0,0 +1,233 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/mysql", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "MySql Source Spec", + "type": "object", + "required": ["host", "port", "database", "username", "replication_method"], + "properties": { + "host": { + "description": "The host name of the database.", + "title": "Host", + "type": "string", + "order": 0 + }, + "port": { + "description": "The port to connect to.", + "title": "Port", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 3306, + "examples": ["3306"], + "order": 1 + }, + "database": { + "description": "The database name.", + "title": "Database", + "type": "string", + "order": 2 + }, + "username": { + "description": "The username which is used to access the database.", + "title": "Username", + "type": "string", + "order": 3 + }, + "password": { + "description": "The password associated with the username.", + "title": "Password", + "type": "string", + "airbyte_secret": true, + "order": 4, + "always_show": true + }, + "jdbc_url_params": { + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", + "title": "JDBC URL Parameters (Advanced)", + "type": "string", + "order": 5 + }, + "ssl": { + "title": "SSL Connection", + "description": "Encrypt data using SSL.", + "type": "boolean", + "default": true, + "order": 6 + }, + "ssl_mode": { + "title": "SSL modes", + "description": "SSL connection modes. Read more in the docs.", + "type": "object", + "order": 7, + "oneOf": [ + { + "title": "preferred", + "description": "Automatically attempt SSL connection. If the MySQL server does not support SSL, continue with a regular connection.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "preferred", + "order": 0 + } + } + }, + { + "title": "required", + "description": "Always connect with SSL. If the MySQL server doesn’t support SSL, the connection will not be established. Certificate Authority (CA) and Hostname are not verified.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "required", + "order": 0 + } + } + }, + { + "title": "Verify CA", + "description": "Always connect with SSL. Verifies CA, but allows connection even if Hostname does not match.", + "required": ["mode", "ca_certificate"], + "properties": { + "mode": { + "type": "string", + "const": "verify_ca", + "order": 0 + }, + "ca_certificate": { + "type": "string", + "title": "CA certificate", + "description": "CA certificate", + "airbyte_secret": true, + "multiline": true, + "order": 1 + }, + "client_certificate": { + "type": "string", + "title": "Client certificate", + "description": "Client certificate (this is not a required field, but if you want to use it, you will need to add the Client key as well)", + "airbyte_secret": true, + "multiline": true, + "order": 2, + "always_show": true + }, + "client_key": { + "type": "string", + "title": "Client key", + "description": "Client key (this is not a required field, but if you want to use it, you will need to add the Client certificate as well)", + "airbyte_secret": true, + "multiline": true, + "order": 3, + "always_show": true + }, + "client_key_password": { + "type": "string", + "title": "Client key password", + "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", + "airbyte_secret": true, + "order": 4 + } + } + }, + { + "title": "Verify Identity", + "description": "Always connect with SSL. Verify both CA and Hostname.", + "required": ["mode", "ca_certificate"], + "properties": { + "mode": { + "type": "string", + "const": "verify_identity", + "order": 0 + }, + "ca_certificate": { + "type": "string", + "title": "CA certificate", + "description": "CA certificate", + "airbyte_secret": true, + "multiline": true, + "order": 1 + }, + "client_certificate": { + "type": "string", + "title": "Client certificate", + "description": "Client certificate (this is not a required field, but if you want to use it, you will need to add the Client key as well)", + "airbyte_secret": true, + "multiline": true, + "order": 2, + "always_show": true + }, + "client_key": { + "type": "string", + "title": "Client key", + "description": "Client key (this is not a required field, but if you want to use it, you will need to add the Client certificate as well)", + "airbyte_secret": true, + "multiline": true, + "order": 3, + "always_show": true + }, + "client_key_password": { + "type": "string", + "title": "Client key password", + "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", + "airbyte_secret": true, + "order": 4 + } + } + } + ] + }, + "replication_method": { + "type": "object", + "title": "Update Method", + "description": "Configures how data is extracted from the database.", + "order": 8, + "default": "CDC", + "display_type": "radio", + "oneOf": [ + { + "title": "Read Changes using Binary Log (CDC)", + "description": "Recommended - Incrementally reads new inserts, updates, and deletes using the MySQL binary log. This must be enabled on your database.", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "CDC", + "order": 0 + }, + "initial_waiting_seconds": { + "type": "integer", + "title": "Initial Waiting Time in Seconds (Advanced)", + "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", + "default": 300, + "min": 120, + "max": 1200, + "order": 1, + "always_show": true + }, + "server_time_zone": { + "type": "string", + "title": "Configured server timezone for the MySQL source (Advanced)", + "description": "Enter the configured MySQL server timezone. This should only be done if the configured timezone in your MySQL instance does not conform to IANNA standard.", + "order": 2, + "always_show": true + } + } + }, + { + "title": "Scan Changes with User Defined Cursor", + "description": "Incrementally detects new inserts and updates using the cursor column chosen when configuring a connection (e.g. created_at, updated_at).", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "STANDARD", + "order": 0 + } + } + } + ] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-mysql/src/testFixtures/java/io/airbyte/integrations/source/mysql/MySQLContainerFactory.java b/airbyte-integrations/connectors/source-mysql/src/testFixtures/java/io/airbyte/integrations/source/mysql/MySQLContainerFactory.java new file mode 100644 index 000000000000..74c745cb7f7f --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/testFixtures/java/io/airbyte/integrations/source/mysql/MySQLContainerFactory.java @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql; + +import io.airbyte.cdk.testutils.ContainerFactory; +import java.io.IOException; +import java.io.UncheckedIOException; +import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.utility.DockerImageName; + +public class MySQLContainerFactory implements ContainerFactory> { + + @Override + public MySQLContainer createNewContainer(DockerImageName imageName) { + return new MySQLContainer<>(imageName.asCompatibleSubstituteFor("mysql")); + } + + @Override + public Class getContainerClass() { + return MySQLContainer.class; + } + + /** + * Create a new network and bind it to the container. + */ + public void withNetwork(MySQLContainer container) { + container.withNetwork(Network.newNetwork()); + } + + private static final String INVALID_TIMEZONE_CEST = "CEST"; + + public void withInvalidTimezoneCEST(MySQLContainer container) { + container.withEnv("TZ", INVALID_TIMEZONE_CEST); + } + + public void withMoscowTimezone(MySQLContainer container) { + container.withEnv("TZ", "Europe/Moscow"); + } + + public void withRootAndServerCertificates(MySQLContainer container) { + execInContainer(container, + "sed -i '31 a ssl' /etc/my.cnf", + "sed -i '32 a ssl-ca=/var/lib/mysql/ca.pem' /etc/my.cnf", + "sed -i '33 a ssl-cert=/var/lib/mysql/server-cert.pem' /etc/my.cnf", + "sed -i '34 a ssl-key=/var/lib/mysql/server-key.pem' /etc/my.cnf", + "sed -i '35 a require_secure_transport=ON' /etc/my.cnf"); + } + + public void withClientCertificate(MySQLContainer container) { + execInContainer(container, + "sed -i '39 a [client]' /etc/mysql/my.cnf", + "sed -i '40 a ssl-ca=/var/lib/mysql/ca.pem' /etc/my.cnf", + "sed -i '41 a ssl-cert=/var/lib/mysql/client-cert.pem' /etc/my.cnf", + "sed -i '42 a ssl-key=/var/lib/mysql/client-key.pem' /etc/my.cnf"); + } + + static private void execInContainer(MySQLContainer container, String... commands) { + container.start(); + try { + for (String command : commands) { + container.execInContainer("sh", "-c", command); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/airbyte-integrations/connectors/source-mysql/src/testFixtures/java/io/airbyte/integrations/source/mysql/MySQLTestDatabase.java b/airbyte-integrations/connectors/source-mysql/src/testFixtures/java/io/airbyte/integrations/source/mysql/MySQLTestDatabase.java new file mode 100644 index 000000000000..cd0565ebf25c --- /dev/null +++ b/airbyte-integrations/connectors/source-mysql/src/testFixtures/java/io/airbyte/integrations/source/mysql/MySQLTestDatabase.java @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.mysql; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.testutils.TestDatabase; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.jooq.SQLDialect; +import org.testcontainers.containers.MySQLContainer; + +public class MySQLTestDatabase extends + TestDatabase, MySQLTestDatabase, MySQLTestDatabase.MySQLConfigBuilder> { + + public static enum BaseImage { + + MYSQL_8("mysql:8.0"), + ; + + private final String reference; + + private BaseImage(String reference) { + this.reference = reference; + } + + } + + public static enum ContainerModifier { + + MOSCOW_TIMEZONE("withMoscowTimezone"), + INVALID_TIMEZONE_CEST("withInvalidTimezoneCEST"), + ROOT_AND_SERVER_CERTIFICATES("withRootAndServerCertificates"), + CLIENT_CERTITICATE("withClientCertificate"), + NETWORK("withNetwork"), + ; + + private final String methodName; + + private ContainerModifier(String methodName) { + this.methodName = methodName; + } + + } + + static public MySQLTestDatabase in(BaseImage baseImage, ContainerModifier... methods) { + String[] methodNames = Stream.of(methods).map(im -> im.methodName).toList().toArray(new String[0]); + final var container = new MySQLContainerFactory().shared(baseImage.reference, methodNames); + return new MySQLTestDatabase(container).initialized(); + } + + public MySQLTestDatabase(MySQLContainer container) { + super(container); + } + + public MySQLTestDatabase withCdcPermissions() { + return this + .with("REVOKE ALL PRIVILEGES, GRANT OPTION FROM '%s';", getUserName()) + .with("GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO '%s';", getUserName()); + } + + public MySQLTestDatabase withoutStrictMode() { + // This disables strict mode in the DB and allows to insert specific values. + // For example, it's possible to insert date with zero values "2021-00-00" + return with("SET @@sql_mode=''"); + } + + static private final int MAX_CONNECTIONS = 1000; + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.of(mysqlCmd(Stream.of( + String.format("SET GLOBAL max_connections=%d", MAX_CONNECTIONS), + String.format("CREATE DATABASE %s", getDatabaseName()), + String.format("CREATE USER '%s' IDENTIFIED BY '%s'", getUserName(), getPassword()), + // Grant privileges also to the container's user, which is not root. + String.format("GRANT ALL PRIVILEGES ON *.* TO '%s', '%s' WITH GRANT OPTION", getUserName(), + getContainer().getUsername())))); + } + + @Override + protected Stream inContainerUndoBootstrapCmd() { + return mysqlCmd(Stream.of( + String.format("DROP USER '%s'", getUserName()), + String.format("DROP DATABASE %s", getDatabaseName()))); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return DatabaseDriver.MYSQL; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.MYSQL; + } + + @Override + public MySQLConfigBuilder configBuilder() { + return new MySQLConfigBuilder(this); + } + + public Stream mysqlCmd(Stream sql) { + return Stream.of("bash", "-c", String.format( + "set -o errexit -o pipefail; echo \"%s\" | mysql -v -v -v --user=root --password=test", + sql.collect(Collectors.joining("; ")))); + } + + static public class MySQLConfigBuilder extends ConfigBuilder { + + protected MySQLConfigBuilder(MySQLTestDatabase testDatabase) { + super(testDatabase); + } + + public MySQLConfigBuilder withStandardReplication() { + return with("replication_method", ImmutableMap.builder().put("method", "STANDARD").build()); + } + + public MySQLConfigBuilder withCdcReplication() { + return this + .with("is_test", true) + .with("replication_method", ImmutableMap.builder() + .put("method", "CDC") + .put("initial_waiting_seconds", 5) + .put("server_time_zone", "America/Los_Angeles") + .build()); + } + + } + + private String cachedCaCertificate; + private Certificates cachedCertificates; + + public synchronized String getCaCertificate() { + if (cachedCaCertificate == null) { + cachedCaCertificate = catFileInContainer("/var/lib/mysql/ca.pem"); + } + return cachedCaCertificate; + } + + public synchronized Certificates getCertificates() { + if (cachedCertificates == null) { + cachedCertificates = new Certificates( + catFileInContainer("/var/lib/mysql/ca.pem"), + catFileInContainer("/var/lib/mysql/client-cert.pem"), + catFileInContainer("/var/lib/mysql/client-key.pem")); + } + return cachedCertificates; + } + + public record Certificates(String caCertificate, String clientCertificate, String clientKey) {} + + private String catFileInContainer(String filePath) { + try { + return getContainer().execInContainer("sh", "-c", "cat " + filePath).getStdout().trim(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/airbyte-integrations/connectors/source-n8n/README.md b/airbyte-integrations/connectors/source-n8n/README.md index 1030a8f655ec..414f367c91d5 100644 --- a/airbyte-integrations/connectors/source-n8n/README.md +++ b/airbyte-integrations/connectors/source-n8n/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-n8n:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/n8n) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_n8n/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-n8n:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-n8n build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-n8n:airbyteDocker +An image will be built with the tag `airbyte/source-n8n:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-n8n:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-n8n:dev check --config docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-n8n:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-n8n:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-n8n test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-n8n:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-n8n:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-n8n test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/n8n.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-n8n/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-n8n/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-n8n/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-n8n/build.gradle b/airbyte-integrations/connectors/source-n8n/build.gradle deleted file mode 100644 index 4cb0f6c570e6..000000000000 --- a/airbyte-integrations/connectors/source-n8n/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_n8n' -} diff --git a/airbyte-integrations/connectors/source-nasa/Dockerfile b/airbyte-integrations/connectors/source-nasa/Dockerfile index 5119f9986f53..32d3578ec307 100644 --- a/airbyte-integrations/connectors/source-nasa/Dockerfile +++ b/airbyte-integrations/connectors/source-nasa/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.13-alpine3.15 as base +FROM python:3.9.11-alpine3.15 as base # build and load all requirements FROM base as builder @@ -34,5 +34,5 @@ COPY source_nasa ./source_nasa ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-nasa diff --git a/airbyte-integrations/connectors/source-nasa/README.md b/airbyte-integrations/connectors/source-nasa/README.md index 1eaa431ebb4a..ec9a6ae245ee 100644 --- a/airbyte-integrations/connectors/source-nasa/README.md +++ b/airbyte-integrations/connectors/source-nasa/README.md @@ -1,45 +1,12 @@ # Nasa Source -This is the repository for the Nasa source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/nasa). +This is the repository for the Nasa configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/nasa). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-nasa:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/nasa) +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/nasa) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_nasa/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. @@ -47,28 +14,21 @@ See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source nasa test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-nasa:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-nasa build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-nasa:airbyteDocker +An image will be built with the tag `airbyte/source-nasa:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-nasa:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-nasa:dev check --confi docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-nasa:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-nasa:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-nasa test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-nasa:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-nasa:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-nasa test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/nasa.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-nasa/__init__.py b/airbyte-integrations/connectors/source-nasa/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-nasa/acceptance-test-config.yml b/airbyte-integrations/connectors/source-nasa/acceptance-test-config.yml index b934790d8dba..5be380ce8487 100644 --- a/airbyte-integrations/connectors/source-nasa/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-nasa/acceptance-test-config.yml @@ -1,29 +1,38 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-nasa:dev -tests: +acceptance_tests: spec: - - spec_path: "source_nasa/spec.yaml" + tests: + - spec_path: "source_nasa/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "integration_tests/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] - expect_records: - path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: no - extra_records: yes + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.jsonl" + # extra_fields: no + # exact_order: no + # extra_records: yes incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" + # bypass_reason: "This connector does not implement incremental sync" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-nasa/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-nasa/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-nasa/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-nasa/build.gradle b/airbyte-integrations/connectors/source-nasa/build.gradle deleted file mode 100644 index be5dc20b8d59..000000000000 --- a/airbyte-integrations/connectors/source-nasa/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_nasa' -} diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-nasa/integration_tests/abnormal_state.json index a73baeb404d6..0a4063d4a2dd 100644 --- a/airbyte-integrations/connectors/source-nasa/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/abnormal_state.json @@ -1,5 +1,9 @@ -{ - "nasa_apod": { - "date": "9999-12-31" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "date": "9999-12-31" }, + "stream_descriptor": { "name": "nasa_apod" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-nasa/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-nasa/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/config.json b/airbyte-integrations/connectors/source-nasa/integration_tests/config.json deleted file mode 100644 index 1a58cb54281b..000000000000 --- a/airbyte-integrations/connectors/source-nasa/integration_tests/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "api_key": "DEMO_KEY", - "start_date": "2022-09-10", - "end_date": "2022-09-15" -} diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-nasa/integration_tests/expected_records.jsonl index 93467678bace..3dc5902e73f6 100644 --- a/airbyte-integrations/connectors/source-nasa/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/expected_records.jsonl @@ -1,6 +1 @@ -{"stream": "nasa_apod", "data": {"copyright": "Gerardo Ferrarino", "date": "2022-09-10", "explanation": "This 180 degree panoramic night skyscape captures our Milky Way Galaxy as it arcs above the horizon on a winter's night in August. Near midnight, the galactic center is close to the zenith with the clear waters of Lake Traful, Neuquen, Argentina, South America, planet Earth below. Zodiacal light, dust reflected sunlight along the Solar System's ecliptic plane, is also visible in the region's very dark night sky. The faint band of light reaches up from the distant snowy peaks toward the galaxy's center. Follow the arc of the Milky Way to the left to find the southern hemisphere stellar beacons Alpha and Beta Centauri. Close to the horizon bright star Vega is reflected in the calm mountain lake.", "hdurl": "https://apod.nasa.gov/apod/image/2209/Traful-Lake.jpg", "media_type": "image", "service_version": "v1", "title": "Galaxy by the Lake", "url": "https://apod.nasa.gov/apod/image/2209/Traful-Lake1024.jpg"}, "emitted_at": 1666637798520} -{"stream": "nasa_apod", "data": {"date": "2022-09-11", "explanation": "How does your favorite planet spin? Does it spin rapidly around a nearly vertical axis, or horizontally, or backwards? The featured video animates NASA images of all eight planets in our Solar System to show them spinning side-by-side for an easy comparison. In the time-lapse video, a day on Earth -- one Earth rotation -- takes just a few seconds. Jupiter rotates the fastest, while Venus spins not only the slowest (can you see it?), but backwards. The inner rocky planets across the top underwent dramatic spin-altering collisions during the early days of the Solar System. Why planets spin and tilt as they do remains a topic of research with much insight gained from modern computer modeling and the recent discovery and analysis of hundreds of exoplanets: planets orbiting other stars.", "media_type": "video", "service_version": "v1", "title": "Planets of the Solar System: Tilts and Spins", "url": "https://www.youtube.com/embed/my1euFQHH-o?rel=0"}, "emitted_at": 1666637798523} -{"stream": "nasa_apod", "data": {"copyright": "Daniel \u0160\u010derba", "date": "2022-09-12", "explanation": "What are those red filaments in the sky? They are a rarely seen form of lightning confirmed only about 35 years ago: red sprites. Research has shown that following a powerful positive cloud-to-ground lightning strike, red sprites may start as 100-meter balls of ionized air that shoot down from about 80-km high at 10 percent the speed of light. They are quickly followed by a group of upward streaking ionized balls. The featured image was taken late last month from the Jeseniky Mountains in northern Moravia in the Czech Republic. The distance to the red sprites is about 200 kilometers. Red sprites take only a fraction of a second to occur and are best seen when powerful thunderstorms are visible from the side. APOD in world languages: Arabic, Bulgarian, Catalan, Chinese (Beijing), Chinese (Taiwan), Croatian, Czech, Dutch, Farsi, French, French (Canada), German, Hebrew, Indonesian, Japanese, Korean, Montenegrin, Polish, Russian, Serbian, Slovenian, Spanish, Taiwanese, Turkish, and Ukrainian", "hdurl": "https://apod.nasa.gov/apod/image/2209/sprites_scerba_4240.jpg", "media_type": "image", "service_version": "v1", "title": "Red Sprite Lightning over the Czech Republic", "url": "https://apod.nasa.gov/apod/image/2209/sprites_scerba_960.jpg"}, "emitted_at": 1666637798524} -{"stream": "nasa_apod", "data": {"copyright": "Alan FriedmanAverted Imagination", "date": "2022-09-13", "explanation": "rlier this month, the Sun exhibited one of the longer filaments on record. Visible as the bright curving streak around the image center, the snaking filament's full extent was estimated to be over half of the Sun's radius -- more than 350,000 kilometers long. A filament is composed of hot gas held aloft by the Sun's magnetic field, so that viewed from the side it would appear as a raised prominence. A different, smaller prominence is simultaneously visible at the Sun's edge. The featured image is in false-color and color-inverted to highlight not only the filament but the Sun's carpet chromosphere. The bright dot on the upper right is actually a dark sunspot about the size of the Earth. Solar filaments typically last from hours to days, eventually collapsing to return hot plasma back to the Sun. Sometimes, though, they explode and expel particles into the Solar System, some of which trigger auroras on Earth. The pictured filament appeared in early September and continued to hold steady for about a week.", "hdurl": "https://apod.nasa.gov/apod/image/2209/SnakingFilament_Friedman_960.jpg", "media_type": "image", "service_version": "v1", "title": "A Long Snaking Filament on the Sun", "url": "https://apod.nasa.gov/apod/image/2209/SnakingFilament_Friedman_960.jpg"}, "emitted_at": 1666637798524} -{"stream": "nasa_apod", "data": {"copyright": "Jarmo Ruuth Text: Ata SarajediniFlorida Atlantic U.Astronomy Minute", "date": "2022-09-14", "explanation": "It is one of the largest nebulas on the sky -- why isn't it better known? Roughly the same angular size as the Andromeda Galaxy, the Great Lacerta Nebula can be found toward the constellation of the Lizard (Lacerta). The emission nebula is difficult to see with wide-field binoculars because it is so faint, but also usually difficult to see with a large telescope because it is so great in angle -- spanning about three degrees. The depth, breadth, waves, and beauty of the nebula -- cataloged as Sharpless 126 (Sh2-126) -- can best be seen and appreciated with a long duration camera exposure. The featured image is one such combined exposure -- in this case 10 hours over five different colors and over six nights during this past June and July at the IC Astronomy Observatory in Spain. The hydrogen gas in the Great Lacerta Nebula glows red because it is excited by light from the bright star 10 Lacertae, one of the bright blue stars just above the red-glowing nebula's center. The stars and nebula are about 1,200 light years distant. Harvest Full Moon 2022: Notable Submissions to APOD", "hdurl": "https://apod.nasa.gov/apod/image/2209/GreatLacerta_Ruuth_3719.jpg", "media_type": "image", "service_version": "v1", "title": "Waves of the Great Lacerta Nebula", "url": "https://apod.nasa.gov/apod/image/2209/GreatLacerta_Ruuth_960.jpg"}, "emitted_at": 1666637798524} -{"stream": "nasa_apod", "data": {"copyright": "Dario Giannobile", "date": "2022-09-15", "explanation": "For northern hemisphere dwellers, September's Full Moon was the Harvest Moon. Reflecting warm hues at sunset it rises over the historic town of Castiglione di Sicilia in this telephoto view from September 9. Famed in festival, story, and song Harvest Moon is just the traditional name of the full moon nearest the autumnal equinox. According to lore the name is a fitting one. Despite the diminishing daylight hours as the growing season drew to a close, farmers could harvest crops by the light of a full moon shining on from dusk to dawn. Harvest Full Moon 2022: Notable Submissions to APOD", "hdurl": "https://apod.nasa.gov/apod/image/2209/HarvestMoonCastiglioneSicilyLD.jpg", "media_type": "image", "service_version": "v1", "title": "Harvest Moon over Sicily", "url": "https://apod.nasa.gov/apod/image/2209/HarvestMoonCastiglioneSicily1024.jpg"}, "emitted_at": 1666637798525} +{"stream": "nasa_apod", "data": {"copyright":"\nAbdullahAlharbi\n","date":"2023-10-04","explanation":"Doesthisnebulalookliketheheadofawitch?ThenebulaisknownpopularlyastheWitchHeadNebulabecause,itissaid,thenebula'sshaperesemblesaHalloween-stylecaricatureofawitch'shead.Exactlyhow,though,canbeatopicofimaginativespeculation.WhatisclearisthatIC2118isabout50light-yearsacrossandmadeofgasanddustthatpointsto--becauseithasbeenpartlyerodedby--thenearbystarRigel.OneofthebrighterstarsintheconstellationOrion,Rigelliesbelowthebottomofthefeaturedimage.ThebluecoloroftheWitchHeadNebulaandiscausednotonlybyRigel'sintensebluestarlightbutbecausethedustgrainsscatterbluelightmoreefficientlythanred.ThesamephysicalprocesscausesEarth'sdaytimeskytoappearblue,althoughthescatterersinplanetEarth'satmospherearemoleculesofnitrogenandoxygen.","hdurl":"https://apod.nasa.gov/apod/image/2310/WitchHead_Alharbi_3051.jpg","media_type":"image","service_version":"v1","title":"IC2118:TheWitchHeadNebula","url":"https://apod.nasa.gov/apod/image/2310/WitchHead_Alharbi_1080.jpg"}, "emitted_at": 1666637798524} diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-nasa/integration_tests/sample_config.json index 2f3a9a920871..9938dbd80b45 100644 --- a/airbyte-integrations/connectors/source-nasa/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/sample_config.json @@ -1,5 +1,5 @@ { - "api_key": "DEMO_KEY", - "date": "2022-10-20", - "concept_tags": true + "api_key": "xxxxxxxxxxxxxxxxxxxxxxxx", + "start_date": "2023-10-01", + "end_date": "2023-10-03" } diff --git a/airbyte-integrations/connectors/source-nasa/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-nasa/integration_tests/sample_state.json index 6f6300ecb66b..a2ef1d7bf89d 100644 --- a/airbyte-integrations/connectors/source-nasa/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-nasa/integration_tests/sample_state.json @@ -1,5 +1,13 @@ -{ - "nasa-apod": { - "date": "2022-10-15" +[ + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "nasa_apod" + }, + "stream_state": { + "date": "2023-10-20" + } + } } -} +] diff --git a/airbyte-integrations/connectors/source-nasa/metadata.yaml b/airbyte-integrations/connectors/source-nasa/metadata.yaml index ebc388c7b2e1..1cd017174495 100644 --- a/airbyte-integrations/connectors/source-nasa/metadata.yaml +++ b/airbyte-integrations/connectors/source-nasa/metadata.yaml @@ -2,26 +2,24 @@ data: allowedHosts: hosts: - api.nasa.gov + registries: + oss: + enabled: true + cloud: + enabled: false connectorSubtype: api connectorType: source definitionId: 1a8667d7-7978-43cd-ba4d-d32cbd478971 - dockerImageTag: 0.1.1 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-nasa githubIssueLabel: source-nasa icon: nasa.svg license: MIT - name: NASA - registries: - cloud: - enabled: false - oss: - enabled: true + name: Nasa + releaseDate: 2023-10-09 releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/nasa tags: - - language:python - ab_internal: - sl: 100 - ql: 100 - supportLevel: community + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-nasa/requirements.txt b/airbyte-integrations/connectors/source-nasa/requirements.txt index d6e1198b1ab1..cc57334ef619 100644 --- a/airbyte-integrations/connectors/source-nasa/requirements.txt +++ b/airbyte-integrations/connectors/source-nasa/requirements.txt @@ -1 +1,2 @@ +-e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-nasa/setup.py b/airbyte-integrations/connectors/source-nasa/setup.py index eb9ee522c041..ecd81e7b2fd4 100644 --- a/airbyte-integrations/connectors/source-nasa/setup.py +++ b/airbyte-integrations/connectors/source-nasa/setup.py @@ -5,15 +5,9 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", -] +MAIN_REQUIREMENTS = ["airbyte-cdk"] -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest~=6.1", - "pytest-mock~=3.6.1", -] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( name="source_nasa", diff --git a/airbyte-integrations/connectors/source-nasa/source_nasa/manifest.yaml b/airbyte-integrations/connectors/source-nasa/source_nasa/manifest.yaml new file mode 100644 index 000000000000..95944a141b73 --- /dev/null +++ b/airbyte-integrations/connectors/source-nasa/source_nasa/manifest.yaml @@ -0,0 +1,63 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + + requester: + type: HttpRequester + url_base: "https://api.nasa.gov/" + http_method: "GET" + authenticator: + type: NoAuth + request_parameters: + api_key: "{{ config['api_key'] }}" + start_date: "{{ stream_interval.start_time if stream_interval else config['start_date'] }}" + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + incremental_sync_base: + type: DatetimeBasedCursor + cursor_field: date + datetime_format: "%Y-%m-%d" + cursor_granularity: "PT0.000001S" + lookback_window: "P31D" + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%d" + end_datetime: + datetime: "{{ today_utc() }}" + datetime_format: "%Y-%m-%d" + step: "P1M" + + nasa_apod_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "nasa_apod" + incremental_cursor: "date" + path: "planetary/apod" + incremental_sync: + $ref: "#/definitions/incremental_sync_base" + +streams: + - "#/definitions/nasa_apod_stream" + +check: + type: CheckStream + stream_names: + - "nasa_apod" diff --git a/airbyte-integrations/connectors/source-nasa/source_nasa/schemas/nasa_apod.json b/airbyte-integrations/connectors/source-nasa/source_nasa/schemas/nasa_apod.json index fc6ee91c2b8f..1ab46c0e2282 100644 --- a/airbyte-integrations/connectors/source-nasa/source_nasa/schemas/nasa_apod.json +++ b/airbyte-integrations/connectors/source-nasa/source_nasa/schemas/nasa_apod.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "resource": { "type": ["null", "object"], diff --git a/airbyte-integrations/connectors/source-nasa/source_nasa/source.py b/airbyte-integrations/connectors/source-nasa/source_nasa/source.py index 555527ed3b01..7c47c4d65d79 100644 --- a/airbyte-integrations/connectors/source-nasa/source_nasa/source.py +++ b/airbyte-integrations/connectors/source-nasa/source_nasa/source.py @@ -2,208 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from abc import ABC -from datetime import datetime, time, timedelta -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import IncrementalMixin, Stream -from airbyte_cdk.sources.streams.http import HttpStream +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -date_format = "%Y-%m-%d" +WARNING: Do not modify this file. +""" -class NasaStream(HttpStream, ABC): - - api_key = "api_key" - url_base = "https://api.nasa.gov/" - - def __init__(self, config: Mapping[str, any], **kwargs): - super().__init__() - self.config = config - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return {self.api_key: self.config[self.api_key]} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - r = response.json() - if type(r) is dict: - yield r - else: # We got a list - yield from r - - -class NasaApod(NasaStream, IncrementalMixin): - - cursor_field = "date" - primary_key = "date" - start_date_key = "start_date" - end_date_key = "end_date" - - def __init__(self, config: Mapping[str, any], **kwargs): - super().__init__(config) - self.start_date = ( - datetime.strptime(config.pop(self.start_date_key), date_format) if self.start_date_key in config else datetime.now() - ) - self.end_date = datetime.strptime(config.pop(self.end_date_key), date_format) if self.end_date_key in config else datetime.now() - self.sync_mode = SyncMode.full_refresh - self._cursor_value = self.start_date - - @property - def state(self) -> Mapping[str, Any]: - return {self.cursor_field: self._cursor_value.strftime(date_format)} - - @state.setter - def state(self, value: Mapping[str, Any]): - self._cursor_value = datetime.strptime(value[self.cursor_field], date_format) - - def _chunk_date_range(self, start_date: datetime) -> List[Mapping[str, Any]]: - """ - Returns a list of each day between the start date and end date. - The return value is a list of dicts {'date': date_string}. - """ - dates = [] - while start_date <= self.end_date: - dates.append({self.cursor_field: start_date.strftime(date_format)}) - start_date += timedelta(days=1) - return dates - - def stream_slices( - self, sync_mode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: - if ( - stream_state - and self.cursor_field in stream_state - and datetime.strptime(stream_state[self.cursor_field], date_format) > self.end_date - ): - return [] - if sync_mode == SyncMode.full_refresh: - return [self.start_date] - - start_date = ( - datetime.strptime(stream_state[self.cursor_field], date_format) - if stream_state and self.cursor_field in stream_state - else self.start_date - ) - return self._chunk_date_range(start_date) - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "planetary/apod" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - request_dict = {**self.config, **super().request_params(stream_state, stream_slice, next_page_token)} - if self.sync_mode == SyncMode.full_refresh: - request_dict[self.start_date_key] = self.start_date.strftime(date_format) - request_dict[self.end_date_key] = self.end_date.strftime(date_format) - else: - request_dict[self.primary_key] = stream_slice[self.cursor_field] - return request_dict - - def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]: - self.sync_mode = kwargs.get("sync_mode", SyncMode.full_refresh) - if self._cursor_value and self._cursor_value > self.end_date: - yield [] - else: - for record in super().read_records(*args, **kwargs): - if self._cursor_value: - latest_record_date = datetime.strptime(record[self.cursor_field], date_format) - self._cursor_value = max(self._cursor_value, latest_record_date) - yield record - - -# Source -class SourceNasa(AbstractSource): - - count_key = "count" - start_date_key = "start_date" - end_date_key = "end_date" - min_count_value, max_count_value = 1, 101 - min_date = datetime.strptime("1995-06-16", date_format) - max_date = datetime.combine(datetime.today(), time(0, 0)) + timedelta(days=1) - invalid_conbination_message_template = "Invalid parameter combination. Cannot use {} and {} together." - invalid_parameter_value_template = "Invalid {} value: {}. {}." - invalid_parameter_value_range_template = "The value should be in the range [{},{})" - - def _parse_date(self, date_str: str) -> Union[datetime, str]: - """ - Parses the date string into a datetime object. - - :param date_str: string containing the date according to DATE_FORMAT - :return Union[datetime, str]: str if not correctly formatted or if it does not satify the constraints [self.MIN_DATE, self.MAX_DATE), datetime otherwise. - """ - try: - date = datetime.strptime(date_str, date_format) - if date < self.min_date or date >= self.max_date: - return self.invalid_parameter_value_template.format( - self.date_key, date_str, self.invalid_parameter_value_range_template.format(self.min_date, self.max_date) - ) - else: - return date - except ValueError: - return self.invalid_parameter_value_template.format(self.date_key, date_str, f"It should be formatted as '{date_format}'") - - def check_connection(self, logger, config) -> Tuple[bool, any]: - """ - Verifies that the input configuration supplied by the user can be used to connect to the underlying data source. - - :param config: the user-input config object conforming to the connector's spec.yaml - :param logger: logger object - :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. - """ - if self.start_date_key in config: - start_date = self._parse_date(config[self.start_date_key]) - if type(start_date) is not datetime: - return False, start_date - - if self.count_key in config: - return False, self.invalid_conbination_message_template.format(self.start_date_key, self.count_key) - - if self.end_date_key in config: - end_date = self._parse_date(config[self.end_date_key]) - if type(end_date) is not datetime: - return False, end_date - - if self.count_key in config: - return False, self.invalid_conbination_message_template.format(self.end_date_key, self.count_key) - - if self.start_date_key not in config: - return False, f"Cannot use {self.end_date_key} without specifying {self.start_date_key}." - - if start_date > end_date: - return False, f"Invalid values. start_date ({start_date}) needs to be lower than or equal to end_date ({end_date})." - - if self.count_key in config: - count_value = config[self.count_key] - if count_value < self.min_count_value or count_value >= self.max_count_value: - return False, self.invalid_parameter_value_template.format( - self.count_key, - count_value, - self.invalid_parameter_value_range_template.format(self.min_count_value, self.max_count_value), - ) - - try: - stream = NasaApod(authenticator=None, config=config) - records = stream.read_records(sync_mode=SyncMode.full_refresh) - next(records) - return True, None - except requests.exceptions.RequestException as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - return [NasaApod(authenticator=None, config=config)] +# Declarative Source +class SourceNasa(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-nasa/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-nasa/unit_tests/test_incremental_streams.py deleted file mode 100644 index 93820311d625..000000000000 --- a/airbyte-integrations/connectors/source-nasa/unit_tests/test_incremental_streams.py +++ /dev/null @@ -1,51 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from datetime import datetime, timedelta - -from airbyte_cdk.models import SyncMode -from pytest import fixture -from source_nasa.source import NasaApod - -config = {"api_key": "foobar"} - - -@fixture -def patch_incremental_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(NasaApod, "path", "v0/example_endpoint") - mocker.patch.object(NasaApod, "primary_key", "test_primary_key") - mocker.patch.object(NasaApod, "__abstractmethods__", set()) - - -def test_cursor_field(patch_incremental_base_class): - stream = NasaApod(config=config) - expected_cursor_field = "date" - assert stream.cursor_field == expected_cursor_field - - -def test_stream_slices(patch_incremental_base_class): - stream = NasaApod(config=config) - start_date = datetime.now() - timedelta(days=3) - inputs = {"sync_mode": SyncMode.incremental, "cursor_field": ["date"], "stream_state": {"date": start_date.strftime("%Y-%m-%d")}} - expected_stream_slice = [{"date": (start_date + timedelta(days=x)).strftime("%Y-%m-%d")} for x in range(4)] - assert stream.stream_slices(**inputs) == expected_stream_slice - - -def test_supports_incremental(patch_incremental_base_class, mocker): - mocker.patch.object(NasaApod, "cursor_field", "dummy_field") - stream = NasaApod(config=config) - assert stream.supports_incremental - - -def test_source_defined_cursor(patch_incremental_base_class): - stream = NasaApod(config=config) - assert stream.source_defined_cursor - - -def test_stream_checkpoint_interval(patch_incremental_base_class): - stream = NasaApod(config=config) - expected_checkpoint_interval = None - assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-nasa/unit_tests/test_source.py b/airbyte-integrations/connectors/source-nasa/unit_tests/test_source.py deleted file mode 100644 index 123d1f14b430..000000000000 --- a/airbyte-integrations/connectors/source-nasa/unit_tests/test_source.py +++ /dev/null @@ -1,58 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from datetime import datetime, timedelta -from unittest.mock import MagicMock, patch - -import pytest -from source_nasa.source import NasaApod, SourceNasa - -date_format = "%Y-%m-%d" -min_date = datetime.strptime("1995-06-16", date_format) -tomorrow = SourceNasa().max_date -after_tomorrow_str = (tomorrow + timedelta(days=1)).strftime(date_format) -valid_date_str = (min_date + timedelta(days=10)).strftime(date_format) - - -@pytest.mark.parametrize( - ("config", "expected_return"), - [ - ({"api_key": "foobar"}, (True, None)), - ({"api_key": "foobar", "start_date": valid_date_str}, (True, None)), - ( - {"api_key": "foobar", "start_date": valid_date_str, "count": 5}, - (False, "Invalid parameter combination. Cannot use start_date and count together."), - ), - ( - {"api_key": "foobar", "end_date": valid_date_str, "count": 5}, - (False, "Invalid parameter combination. Cannot use end_date and count together."), - ), - ({"api_key": "foobar", "end_date": valid_date_str}, (False, "Cannot use end_date without specifying start_date.")), - ( - {"api_key": "foobar", "start_date": valid_date_str, "end_date": min_date.strftime(date_format)}, - ( - False, - f"Invalid values. start_date ({datetime.strptime(valid_date_str, date_format)}) needs to be lower than or equal to end_date ({min_date}).", - ), - ), - ({"api_key": "foobar", "start_date": min_date.strftime(date_format), "end_date": valid_date_str}, (True, None)), - ({"api_key": "foobar", "count": 0}, (False, "Invalid count value: 0. The value should be in the range [1,101).")), - ({"api_key": "foobar", "count": 101}, (False, "Invalid count value: 101. The value should be in the range [1,101).")), - ({"api_key": "foobar", "count": 1}, (True, None)), - ], -) -def test_check_connection(mocker, config, expected_return): - with patch.object(NasaApod, "read_records") as mock_http_request: - mock_http_request.return_value = iter([None]) - source = SourceNasa() - logger_mock = MagicMock() - assert source.check_connection(logger_mock, config) == expected_return - - -def test_streams(mocker): - source = SourceNasa() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 1 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-nasa/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-nasa/unit_tests/test_streams.py deleted file mode 100644 index 91e9b2104406..000000000000 --- a/airbyte-integrations/connectors/source-nasa/unit_tests/test_streams.py +++ /dev/null @@ -1,80 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from datetime import datetime -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_nasa.source import NasaApod - -api_key_value = "foobar" -config = {"api_key": api_key_value} - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(NasaApod, "path", "v0/example_endpoint") - mocker.patch.object(NasaApod, "primary_key", "test_primary_key") - mocker.patch.object(NasaApod, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = NasaApod(config={**config, "start_date": "2022-09-10"}) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {"api_key": api_key_value, "start_date": "2022-09-10", "end_date": datetime.now().strftime("%Y-%m-%d")} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = NasaApod(config=config) - inputs = {"response": MagicMock()} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_parse_response(patch_base_class): - stream = NasaApod(config=config) - response_object = [{"foo": "bar", "baz": ["qux"]}] - response_mock = MagicMock() - response_mock.configure_mock(**{"json.return_value": response_object}) - inputs = {"response": response_mock} - assert next(stream.parse_response(**inputs)) == response_object[0] - - -def test_request_headers(patch_base_class): - stream = NasaApod(config=config) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = NasaApod(config=config) - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = NasaApod(config=config) - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = NasaApod(config=config) - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-netsuite/README.md b/airbyte-integrations/connectors/source-netsuite/README.md index 00509cec457f..8b14d70cbf29 100644 --- a/airbyte-integrations/connectors/source-netsuite/README.md +++ b/airbyte-integrations/connectors/source-netsuite/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-netsuite:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/netsuite) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_netsuite_soap/spec.yaml` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-netsuite:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-netsuite build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-netsuite:airbyteDocker +An image will be built with the tag `airbyte/source-netsuite:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-netsuite:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,45 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-netsuite:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-netsuite:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-netsuite:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-netsuite test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-netsuite:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-netsuite:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-netsuite:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -126,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-netsuite test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/netsuite.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-netsuite/acceptance-test-config.yml b/airbyte-integrations/connectors/source-netsuite/acceptance-test-config.yml index 90edf7530a22..55cf23b95144 100644 --- a/airbyte-integrations/connectors/source-netsuite/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-netsuite/acceptance-test-config.yml @@ -11,19 +11,14 @@ tests: status: "failed" discovery: - config_path: "secrets/config.json" - # Discovery stage is dynamic, so timeout iscreased + # Discovery stage is dynamic, so timeout iscreased timeout_seconds: 1200 basic_read: - config_path: "secrets/config.json" # NetSuite has lots of streams available, we test the portion of them only. configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [ - "customrecord01", - "billingaccount", - "charge", - "message", - "salesorder", - ] + empty_streams: + ["customrecord01", "billingaccount", "charge", "message", "salesorder"] timeout_seconds: 3600 full_refresh: - config_path: "secrets/config.json" @@ -34,4 +29,3 @@ tests: configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" timeout_seconds: 7200 - threshold_days: 30 diff --git a/airbyte-integrations/connectors/source-netsuite/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-netsuite/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-netsuite/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-netsuite/build.gradle b/airbyte-integrations/connectors/source-netsuite/build.gradle deleted file mode 100644 index 7639ed1ff369..000000000000 --- a/airbyte-integrations/connectors/source-netsuite/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_netsuite' -} diff --git a/airbyte-integrations/connectors/source-news-api/README.md b/airbyte-integrations/connectors/source-news-api/README.md index 9d32f1b4212a..0408e1fadd77 100644 --- a/airbyte-integrations/connectors/source-news-api/README.md +++ b/airbyte-integrations/connectors/source-news-api/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-news-api:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/news-api) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_news_api/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-news-api:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-news-api build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-news-api:airbyteDocker +An image will be built with the tag `airbyte/source-news-api:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-news-api:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-news-api:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-news-api:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-news-api:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-news-api test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-news-api:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-news-api:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-news-api test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/news-api.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-news-api/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-news-api/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-news-api/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-news-api/build.gradle b/airbyte-integrations/connectors/source-news-api/build.gradle deleted file mode 100644 index 83e27779983d..000000000000 --- a/airbyte-integrations/connectors/source-news-api/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_news_api' -} diff --git a/airbyte-integrations/connectors/source-newsdata/README.md b/airbyte-integrations/connectors/source-newsdata/README.md index 3fed2f2d83e2..c25711a4ce30 100644 --- a/airbyte-integrations/connectors/source-newsdata/README.md +++ b/airbyte-integrations/connectors/source-newsdata/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-newsdata:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/newsdata) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_newsdata/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-newsdata:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-newsdata build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-newsdata:airbyteDocker +An image will be built with the tag `airbyte/source-newsdata:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-newsdata:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-newsdata:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-newsdata:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-newsdata:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-newsdata test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-newsdata:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-newsdata:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-newsdata test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/newsdata.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-newsdata/acceptance-test-config.yml b/airbyte-integrations/connectors/source-newsdata/acceptance-test-config.yml index 911eb6b26ae7..130765ec2edb 100644 --- a/airbyte-integrations/connectors/source-newsdata/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-newsdata/acceptance-test-config.yml @@ -19,7 +19,7 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - incremental: + incremental: bypass_reason: "This connector does not implement incremental sync" full_refresh: tests: diff --git a/airbyte-integrations/connectors/source-newsdata/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-newsdata/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-newsdata/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-newsdata/build.gradle b/airbyte-integrations/connectors/source-newsdata/build.gradle deleted file mode 100644 index 78b0b19c41c0..000000000000 --- a/airbyte-integrations/connectors/source-newsdata/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_newsdata' -} diff --git a/airbyte-integrations/connectors/source-notion/Dockerfile b/airbyte-integrations/connectors/source-notion/Dockerfile deleted file mode 100644 index 3a69375ac3c8..000000000000 --- a/airbyte-integrations/connectors/source-notion/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_notion ./source_notion - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=1.1.1 -LABEL io.airbyte.name=airbyte/source-notion diff --git a/airbyte-integrations/connectors/source-notion/README.md b/airbyte-integrations/connectors/source-notion/README.md index 5f65c1302bfb..2be67fc963b2 100644 --- a/airbyte-integrations/connectors/source-notion/README.md +++ b/airbyte-integrations/connectors/source-notion/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-notion:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/notion) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_notion/spec.json` file. @@ -57,19 +49,70 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-notion:dev + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-notion build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-notion:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. -You can also build the connector image via Gradle: +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-notion:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-notion:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-notion:dev . +# Running the spec command against your patched connector +docker run airbyte/source-notion:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -78,45 +121,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-notion:dev check --con docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-notion:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-notion:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-notion test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-notion:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-notion:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-notion:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -126,8 +140,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -2. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -3. Create a Pull Request. -4. Pat yourself on the back for being an awesome contributor. -5. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-notion test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/notion.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-notion/acceptance-test-config.yml b/airbyte-integrations/connectors/source-notion/acceptance-test-config.yml index 6c943bf0d5d3..ab14b89dc5c8 100644 --- a/airbyte-integrations/connectors/source-notion/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-notion/acceptance-test-config.yml @@ -5,31 +5,35 @@ test_strictness_level: high acceptance_tests: spec: tests: - - spec_path: "source_notion/spec.json" + - spec_path: "source_notion/spec.json" connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "secrets/config_oauth.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "secrets/config_oauth.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: tests: - - config_path: "secrets/config.json" + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + # 2.0.8 introduces a fix to _blocks.properties.table_row.cells, + # which was incorrectly added in 2.0.0 as an object array instead of an array of object arrays. + disable_for_version: 2.0.7 basic_read: tests: - - config_path: "secrets/config.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - fail_on_extra_columns: false + - config_path: "secrets/config.json" + expect_records: + path: "integration_tests/expected_records.jsonl" + fail_on_extra_columns: true incremental: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/incremental_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/incremental_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-notion/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-notion/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-notion/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-notion/build.gradle b/airbyte-integrations/connectors/source-notion/build.gradle deleted file mode 100644 index 04853a9ca945..000000000000 --- a/airbyte-integrations/connectors/source-notion/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_notion' -} diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/catalog.json b/airbyte-integrations/connectors/source-notion/integration_tests/catalog.json index 24d5091f961a..3b76e23aa3c9 100644 --- a/airbyte-integrations/connectors/source-notion/integration_tests/catalog.json +++ b/airbyte-integrations/connectors/source-notion/integration_tests/catalog.json @@ -26,6 +26,13 @@ "source_defined_cursor": true, "default_cursor_field": "last_edited_time", "json_schema": {} + }, + { + "name": "comments", + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": "page_last_edited_time", + "json_schema": {} } ] } diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-notion/integration_tests/configured_catalog.json index 8fccfcd9ab74..d0fa294445a5 100644 --- a/airbyte-integrations/connectors/source-notion/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-notion/integration_tests/configured_catalog.json @@ -52,6 +52,20 @@ "cursor_field": ["last_edited_time"], "sync_mode": "incremental", "destination_sync_mode": "append" + }, + { + "stream": { + "name": "comments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_primary_key": [["id"]], + "default_cursor_field": ["page_last_edited_time"] + }, + "primary_key": [["id"]], + "cursor_field": ["page_last_edited_time"], + "sync_mode": "incremental", + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl index 335e3021715c..27bf2dc60ca8 100644 --- a/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.jsonl @@ -1,10 +1,15 @@ -{"stream":"pages","data":{"object":"page","id":"00074690-3420-4861-84ae-b1c7498b67f1","created_time":"2021-10-19T13:33:00.000Z","last_edited_time":"2021-10-19T13:33:00.000Z","created_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"last_edited_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"cover":null,"icon":null,"parent":{"type":"database_id","database_id":"a1298679-9f79-48a8-a991-834cd72eca17"},"archived":false,"properties":[{"name":"Engineers","value":{"id":"%24v1Q","type":"people","people":[]}},{"name":"Tasks","value":{"id":"6%3Dyp","type":"relation","relation":[],"has_more":false}},{"name":"Type","value":{"id":"9dB%5E","type":"select","select":null}},{"name":"Sprint","value":{"id":"Jz.%40","type":"multi_select","multi_select":[]}},{"name":"Epic","value":{"id":"L%5BK%3C","type":"relation","relation":[],"has_more":false}},{"name":"Timeline","value":{"id":"_G%2Bl","type":"date","date":null}},{"name":"Created","value":{"id":"iwS0","type":"created_time","created_time":"2021-10-19T13:33:00.000Z"}},{"name":"Product Manager","value":{"id":"ma%3AW","type":"people","people":[]}},{"name":"Priority","value":{"id":"%7BMEq","type":"select","select":null}},{"name":"Status","value":{"id":"%7CF4-","type":"select","select":null}},{"name":"Projects","value":{"id":"title","type":"title","title":[{"type":"text","text":{"content":"Project Spec 🗺","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Project Spec 🗺","href":null}]}}],"url":"https://www.notion.so/Project-Spec-000746903420486184aeb1c7498b67f1","public_url":null},"emitted_at":1687166006562} -{"stream":"pages","data":{"object":"page","id":"249f3796-7e81-47b0-9075-00ed2d06439d","created_time":"2021-10-19T13:33:00.000Z","last_edited_time":"2021-10-19T13:33:00.000Z","created_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"last_edited_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"cover":null,"icon":null,"parent":{"type":"database_id","database_id":"a1298679-9f79-48a8-a991-834cd72eca17"},"archived":false,"properties":[{"name":"Engineers","value":{"id":"%24v1Q","type":"people","people":[]}},{"name":"Tasks","value":{"id":"6%3Dyp","type":"relation","relation":[],"has_more":false}},{"name":"Type","value":{"id":"9dB%5E","type":"select","select":{"id":"1497e06a-abf3-4c81-a619-debfa0c70621","name":"Bug 🐞","color":"red"}}},{"name":"Sprint","value":{"id":"Jz.%40","type":"multi_select","multi_select":[{"id":"8d033c95-5515-4662-b8f3-60cb7d86487a","name":"Sprint 21","color":"default"}]}},{"name":"Epic","value":{"id":"L%5BK%3C","type":"relation","relation":[],"has_more":false}},{"name":"Timeline","value":{"id":"_G%2Bl","type":"date","date":null}},{"name":"Created","value":{"id":"iwS0","type":"created_time","created_time":"2021-10-19T13:33:00.000Z"}},{"name":"Product Manager","value":{"id":"ma%3AW","type":"people","people":[]}},{"name":"Priority","value":{"id":"%7BMEq","type":"select","select":{"id":"09fy","name":"P1 🔥","color":"red"}}},{"name":"Status","value":{"id":"%7CF4-","type":"select","select":{"id":"ab7c2b08-ed87-4c04-b30f-fa62440f75d5","name":"In Progress","color":"yellow"}}},{"name":"Projects","value":{"id":"title","type":"title","title":[{"type":"text","text":{"content":"New Emojis Don't Render","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"New Emojis Don't Render","href":null}]}}],"url":"https://www.notion.so/New-Emojis-Don-t-Render-249f37967e8147b0907500ed2d06439d","public_url":null},"emitted_at":1687166006563} -{"stream":"pages","data":{"object":"page","id":"29299296-ef3f-4aff-aef5-02d651a59be3","created_time":"2021-10-19T13:33:00.000Z","last_edited_time":"2021-10-19T13:33:00.000Z","created_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"last_edited_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"cover":null,"icon":{"type":"emoji","emoji":"🔌"},"parent":{"type":"database_id","database_id":"a1298679-9f79-48a8-a991-834cd72eca17"},"archived":false,"properties":[{"name":"Engineers","value":{"id":"%24v1Q","type":"people","people":[]}},{"name":"Tasks","value":{"id":"6%3Dyp","type":"relation","relation":[],"has_more":false}},{"name":"Type","value":{"id":"9dB%5E","type":"select","select":{"id":"ca62f85e-a4ac-474f-b493-82d2df005dff","name":"Epic ⛰️","color":"green"}}},{"name":"Sprint","value":{"id":"Jz.%40","type":"multi_select","multi_select":[]}},{"name":"Epic","value":{"id":"L%5BK%3C","type":"relation","relation":[],"has_more":false}},{"name":"Timeline","value":{"id":"_G%2Bl","type":"date","date":{"start":"2019-10-17","end":"2019-10-05","time_zone":null}}},{"name":"Created","value":{"id":"iwS0","type":"created_time","created_time":"2021-10-19T13:33:00.000Z"}},{"name":"Product Manager","value":{"id":"ma%3AW","type":"people","people":[]}},{"name":"Priority","value":{"id":"%7BMEq","type":"select","select":null}},{"name":"Status","value":{"id":"%7CF4-","type":"select","select":{"id":"ab7c2b08-ed87-4c04-b30f-fa62440f75d5","name":"In Progress","color":"yellow"}}},{"name":"Projects","value":{"id":"title","type":"title","title":[{"type":"text","text":{"content":"Improve Third Party Integrations","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Improve Third Party Integrations","href":null}]}}],"url":"https://www.notion.so/Improve-Third-Party-Integrations-29299296ef3f4affaef502d651a59be3","public_url":null},"emitted_at":1687166006564} -{"stream":"blocks","data":{"object":"block","id":"af0bd3c7-9704-44ab-a0af-1a94462f762e","parent":{"type":"page_id","page_id":"00074690-3420-4861-84ae-b1c7498b67f1"},"created_time":"2021-10-19T13:33:00.000Z","last_edited_time":"2021-10-19T13:33:00.000Z","created_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"last_edited_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"has_children":false,"archived":false,"type":"heading_1","heading_1":{"rich_text":[{"type":"text","text":{"content":"Overview","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Overview","href":null}],"is_toggleable":false,"color":"default"}},"emitted_at":1687166008901} -{"stream":"blocks","data":{"object":"block","id":"391f45c4-6d75-4e9e-90db-7457d640f75e","parent":{"type":"page_id","page_id":"00074690-3420-4861-84ae-b1c7498b67f1"},"created_time":"2021-10-19T13:33:00.000Z","last_edited_time":"2021-10-19T13:33:00.000Z","created_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"last_edited_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"has_children":false,"archived":false,"type":"heading_2","heading_2":{"rich_text":[{"type":"text","text":{"content":"Problem statement","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Problem statement","href":null}],"is_toggleable":false,"color":"default"}},"emitted_at":1687166008902} -{"stream":"blocks","data":{"object":"block","id":"d9698116-183d-48c4-81e2-79a882236000","parent":{"type":"page_id","page_id":"00074690-3420-4861-84ae-b1c7498b67f1"},"created_time":"2021-10-19T13:33:00.000Z","last_edited_time":"2021-10-19T13:33:00.000Z","created_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"last_edited_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"has_children":false,"archived":false,"type":"paragraph","paragraph":{"rich_text":[{"type":"text","text":{"content":"Describe the problem we're trying to solve by doing this work.","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Describe the problem we're trying to solve by doing this work.","href":null}],"color":"gray"}},"emitted_at":1687166008903} -{"stream":"users","data":{"object":"user","id":"5612c094-99ec-4ba3-ac7f-df8d84c8d6be","name":"Sherif Nada","avatar_url":"https://s3-us-west-2.amazonaws.com/public.notion-static.com/305f7efc-2862-4342-ba99-5023f3e34717/6246757.png","type":"person","person":{"email":"sherif@airbyte.io"}},"emitted_at":1687166004972} -{"stream":"users","data":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a","name":"Airyte","avatar_url":null,"type":"person","person":{"email":"integration-test@airbyte.io"}},"emitted_at":1687166004973} -{"stream":"users","data":{"object":"user","id":"c1ff0160-b2af-497a-aab7-8b61e625e4e3","name":"Gil Cho","avatar_url":"https://lh3.googleusercontent.com/a/ALm5wu0ElXfvy3YfVUyRn-aB9EZy5AZ1ougHuNyCGmO2=s100","type":"person","person":{"email":"gil@airbyte.io"}},"emitted_at":1687166004973} -{"stream":"databases","data":{"object":"database","id":"a1298679-9f79-48a8-a991-834cd72eca17","cover":null,"icon":{"type":"emoji","emoji":"🚘"},"created_time":"2021-10-19T13:33:00.000Z","created_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"last_edited_by":{"object":"user","id":"f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"},"last_edited_time":"2023-06-15T09:18:00.000Z","title":[{"type":"text","text":{"content":"Roadmap","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Roadmap","href":null}],"description":[{"type":"text","text":{"content":"Use this template to track all of your project work.\n\n⛰ ","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Use this template to track all of your project work.\n\n⛰ ","href":null},{"type":"text","text":{"content":"Epics","link":null},"annotations":{"bold":true,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Epics","href":null},{"type":"text","text":{"content":" are large overarching initiatives.\n🏃‍♂️ ","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" are large overarching initiatives.\n🏃‍♂️ ","href":null},{"type":"text","text":{"content":"Sprints","link":null},"annotations":{"bold":true,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Sprints","href":null},{"type":"text","text":{"content":" are time-bounded pushes to complete a set of tasks.\n🔨 ","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" are time-bounded pushes to complete a set of tasks.\n🔨 ","href":null},{"type":"text","text":{"content":"Tasks","link":null},"annotations":{"bold":true,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Tasks","href":null},{"type":"text","text":{"content":" are the actions that make up epics.\n🐞 ","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" are the actions that make up epics.\n🐞 ","href":null},{"type":"text","text":{"content":"Bugs","link":null},"annotations":{"bold":true,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"Bugs","href":null},{"type":"text","text":{"content":" are tasks to fix things.\n\n","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" are tasks to fix things.\n\n","href":null},{"type":"text","text":{"content":"↓","link":null},"annotations":{"bold":true,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":"↓","href":null},{"type":"text","text":{"content":" Click ","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" Click ","href":null},{"type":"text","text":{"content":"By Status","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":true,"color":"default"},"plain_text":"By Status","href":null},{"type":"text","text":{"content":" to isolate epics, sprints, tasks or bugs. Sort tasks by status, engineer or product manager. Switch to calendar view to see when work is scheduled to be completed.","link":null},"annotations":{"bold":false,"italic":false,"strikethrough":false,"underline":false,"code":false,"color":"default"},"plain_text":" to isolate epics, sprints, tasks or bugs. Sort tasks by status, engineer or product manager. Switch to calendar view to see when work is scheduled to be completed.","href":null}],"is_inline":false,"properties":[{"name":"Engineers","value":{"id":"%24v1Q","name":"Engineers","type":"people","people":{}}},{"name":"Tasks","value":{"id":"6%3Dyp","name":"Tasks","type":"relation","relation":{"database_id":"a1298679-9f79-48a8-a991-834cd72eca17","type":"dual_property","dual_property":{"synced_property_name":"Epic","synced_property_id":"L%5BK%3C"}}}},{"name":"Type","value":{"id":"9dB%5E","name":"Type","type":"select","select":{"options":[{"id":"ca62f85e-a4ac-474f-b493-82d2df005dff","name":"Epic ⛰️","color":"green"},{"id":"3f806034-9c48-4519-871e-60c9c32d73d8","name":"Task 🔨","color":"yellow"},{"id":"1497e06a-abf3-4c81-a619-debfa0c70621","name":"Bug 🐞","color":"red"}]}}},{"name":"Sprint","value":{"id":"Jz.%40","name":"Sprint","type":"multi_select","multi_select":{"options":[{"id":"8d033c95-5515-4662-b8f3-60cb7d86487a","name":"Sprint 21","color":"default"},{"id":"bf3fcc55-aefc-43a8-82a0-2d4ac1e74d30","name":"Sprint 22","color":"default"},{"id":"7d78a5e4-28ef-4b21-8495-998fa6655014","name":"Sprint 23","color":"default"},{"id":"257e46d2-4a27-4298-b8c7-9b9bfb603bbd","name":"Sprint 20","color":"default"},{"id":"fbdb3f96-7979-4027-a461-aab8abda1ca8","name":"Sprint 24","color":"default"}]}}},{"name":"Epic","value":{"id":"L%5BK%3C","name":"Epic","type":"relation","relation":{"database_id":"a1298679-9f79-48a8-a991-834cd72eca17","type":"dual_property","dual_property":{"synced_property_name":"Tasks","synced_property_id":"6%3Dyp"}}}},{"name":"Timeline","value":{"id":"_G%2Bl","name":"Timeline","type":"date","date":{}}},{"name":"Created","value":{"id":"iwS0","name":"Created","type":"created_time","created_time":{}}},{"name":"Product Manager","value":{"id":"ma%3AW","name":"Product Manager","type":"people","people":{}}},{"name":"Priority","value":{"id":"%7BMEq","name":"Priority","type":"select","select":{"options":[{"id":"09fy","name":"P1 🔥","color":"red"},{"id":"e1b2f058-4989-4dee-a873-4e88f58d4d0a","name":"P2","color":"orange"},{"id":"0bb46e0b-be4f-4b9c-87c2-b990868e9f92","name":"P3","color":"yellow"},{"id":"1a5512c5-39ad-4fb7-959f-68f596849eeb","name":"P4","color":"green"},{"id":"28c7cce9-7163-4407-9569-3f070da82ad1","name":"P5","color":"blue"}]}}},{"name":"Status","value":{"id":"%7CF4-","name":"Status","type":"select","select":{"options":[{"id":"c224a5a5-c284-431e-a65d-90a71712bcac","name":"Not Started","color":"red"},{"id":"ab7c2b08-ed87-4c04-b30f-fa62440f75d5","name":"In Progress","color":"yellow"},{"id":"c410e525-9a47-4ed2-9e72-299abee65575","name":"Complete 🙌","color":"green"}]}}},{"name":"Projects","value":{"id":"title","name":"Projects","type":"title","title":{}}}],"parent":{"type":"workspace","workspace":true},"url":"https://www.notion.so/a12986799f7948a8a991834cd72eca17","public_url":null,"archived":false},"emitted_at":1687166005743} \ No newline at end of file +{"stream": "users", "data": {"object": "user", "id": "5612c094-99ec-4ba3-ac7f-df8d84c8d6be", "name": "Sherif Nada", "avatar_url": "https://s3-us-west-2.amazonaws.com/public.notion-static.com/305f7efc-2862-4342-ba99-5023f3e34717/6246757.png", "type": "person", "person": {"email": "sherif@airbyte.io"}}, "emitted_at": 1697023279924} +{"stream": "users", "data": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}, "emitted_at": 1697023279925} +{"stream": "users", "data": {"object": "user", "id": "c1ff0160-b2af-497a-aab7-8b61e625e4e3", "name": "Gil Cho", "avatar_url": "https://lh3.googleusercontent.com/a/ALm5wu0ElXfvy3YfVUyRn-aB9EZy5AZ1ougHuNyCGmO2=s100", "type": "person", "person": {"email": "gil@airbyte.io"}}, "emitted_at": 1697023279925} +{"stream": "databases", "data": {"object": "database", "id": "b75d2e55-cc80-4afa-a273-c78178ac6b3f", "cover": null, "icon": {"type": "emoji", "emoji": "\ud83d\ude4b"}, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_time": "2021-10-19T13:33:00.000Z", "title": [{"type": "text", "text": {"content": "Engineering Directory ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Engineering Directory ", "href": null}], "description": [{"type": "text", "text": {"content": "Have a question about part of our codebase?\nFind the most knowledgeable person in this directory.\nLearn more about ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Have a question about part of our codebase?\nFind the most knowledgeable person in this directory.\nLearn more about ", "href": null}, {"type": "text", "text": {"content": "Notion databases", "link": {"url": "https://www.notion.so/notion/Database-101-build-and-view-fd8cd2d212f74c50954c11086d85997e"}}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Notion databases", "href": "https://www.notion.so/notion/Database-101-build-and-view-fd8cd2d212f74c50954c11086d85997e"}, {"type": "text", "text": {"content": ".", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": ".", "href": null}], "is_inline": false, "properties": [{"name": "Date Added", "value": {"id": "%2Fkv%22", "name": "Date Added", "type": "created_time", "created_time": {}}}, {"name": "Notes", "value": {"id": "mq%22D", "name": "Notes", "type": "rich_text", "rich_text": {}}}, {"name": "Person", "value": {"id": "uiZ%26", "name": "Person", "type": "people", "people": {}}}, {"name": "Name", "value": {"id": "title", "name": "Name", "type": "title", "title": {}}}], "parent": {"type": "block_id", "block_id": "b81f8caf-3ec4-4455-9a0b-25c2bd3b60cb"}, "url": "https://www.notion.so/b75d2e55cc804afaa273c78178ac6b3f", "public_url": null, "archived": false}, "emitted_at": 1697023281967} +{"stream": "databases", "data": {"object": "database", "id": "fbff7d4e-eca4-4432-91e6-ec64ba4b5a98", "cover": null, "icon": null, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_time": "2021-10-19T13:33:00.000Z", "title": [{"type": "text", "text": {"content": "Questions", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Questions", "href": null}], "description": [], "is_inline": true, "properties": [{"name": "Difficulty", "value": {"id": "'i6%2F", "name": "Difficulty", "type": "select", "select": {"options": [{"id": "f00068b9-7612-45da-91ad-1a7b1d259375", "name": "Easy", "color": "green"}, {"id": "8e244bfe-d4c7-48c5-9088-ffd6926b4ba0", "name": "Medium", "color": "yellow"}, {"id": "9ab57ef4-eab1-4b20-a502-047610b5c97d", "name": "Hard", "color": "red"}]}}}, {"name": "Skills", "value": {"id": "K%3AtR", "name": "Skills", "type": "multi_select", "multi_select": {"options": [{"id": "72f4d134-a773-48c1-ba3d-b529f55c6818", "name": "Front end", "color": "default"}, {"id": "c20f5d57-3e35-4b39-b556-05071203cc1a", "name": "Backend", "color": "default"}, {"id": "31d5735c-d6ba-4bd7-940f-bdcb36091c02", "name": "Architecture", "color": "default"}, {"id": "0398de54-af68-4c3a-9953-3788e8eaadbf", "name": "Algorithms", "color": "default"}, {"id": "df9dff09-7dea-4409-a10f-b5e2b546ad94", "name": "Data Structures", "color": "default"}]}}}, {"name": "Question Name", "value": {"id": "title", "name": "Question Name", "type": "title", "title": {}}}], "parent": {"type": "page_id", "page_id": "4999109d-1b7b-41a2-abb4-84f6b961ee74"}, "url": "https://www.notion.so/fbff7d4eeca4443291e6ec64ba4b5a98", "public_url": null, "archived": false}, "emitted_at": 1697023281968} +{"stream": "databases", "data": {"object": "database", "id": "9b1ce91e-a93a-437c-8c92-81083cd98540", "cover": null, "icon": {"type": "emoji", "emoji": "\u270f\ufe0f"}, "created_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "ec324c09-af75-40f0-b91a-49ded74fdaf5"}, "last_edited_time": "2023-09-13T00:06:00.000Z", "title": [{"type": "text", "text": {"content": "Meeting Notes", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Meeting Notes", "href": null}], "description": [{"type": "text", "text": {"content": "Use this template to capture notes from all meetings in one accessible spot.\nNotes can be tagged by meeting type to make them easy to find. \nSee when each meeting took place and who was there.\n\n", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Use this template to capture notes from all meetings in one accessible spot.\nNotes can be tagged by meeting type to make them easy to find. \nSee when each meeting took place and who was there.\n\n", "href": null}, {"type": "text", "text": {"content": "\u2193", "link": null}, "annotations": {"bold": true, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "\u2193", "href": null}, {"type": "text", "text": {"content": " Click ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " Click ", "href": null}, {"type": "text", "text": {"content": "List View", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": true, "color": "default"}, "plain_text": "List View", "href": null}, {"type": "text", "text": {"content": " to create and see other views, including a board organized by meeting type.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": " to create and see other views, including a board organized by meeting type.", "href": null}], "is_inline": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "name": "Last Edited Time", "type": "last_edited_time", "last_edited_time": {}}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "name": "Created By", "type": "created_by", "created_by": {}}}, {"name": "Created", "value": {"id": "Ird4", "name": "Created", "type": "created_time", "created_time": {}}}, {"name": "Type", "value": {"id": "_%7B%5C7", "name": "Type", "type": "select", "select": {"options": [{"id": "3a8fd64c-899d-4c39-ba97-ac4f565d6e94", "name": "Post-mortem", "color": "red"}, {"id": "28b68013-20d5-4824-b810-45cde8784581", "name": "Standup", "color": "green"}, {"id": "8ee247a9-cb60-430a-9ea6-d5c053253334", "name": "Weekly Sync", "color": "blue"}, {"id": "5fb57c36-999f-49e2-b153-96531d086862", "name": "Sprint Planning", "color": "yellow"}, {"id": "1747fcca-8207-42c8-802f-fd43965c016a", "name": "Ad Hoc", "color": "orange"}]}}}, {"name": "Participants", "value": {"id": "b%3AeA", "name": "Participants", "type": "people", "people": {}}}, {"name": "Name", "value": {"id": "title", "name": "Name", "type": "title", "title": {}}}], "parent": {"type": "workspace", "workspace": true}, "url": "https://www.notion.so/9b1ce91ea93a437c8c9281083cd98540", "public_url": null, "archived": false}, "emitted_at": 1697023281968} +{"stream": "pages", "data": {"object": "page", "id": "39a69b4e-7cc2-4f7a-a656-dd128f3ce855", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "9b1ce91e-a93a-437c-8c92-81083cd98540"}, "archived": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "type": "last_edited_time", "last_edited_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "type": "created_by", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}}}, {"name": "Created", "value": {"id": "Ird4", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Type", "value": {"id": "_%7B%5C7", "type": "select", "select": {"id": "28b68013-20d5-4824-b810-45cde8784581", "name": "Standup", "color": "green"}}}, {"name": "Participants", "value": {"id": "b%3AeA", "type": "people", "people": []}}, {"name": "Name", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Daily Standup", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Daily Standup", "href": null}]}}], "url": "https://www.notion.so/Daily-Standup-39a69b4e7cc24f7aa656dd128f3ce855", "public_url": null}, "emitted_at": 1697023284463} +{"stream": "pages", "data": {"object": "page", "id": "621d3dc4-55fe-46ce-a3ff-83da06e5f9fb", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "9b1ce91e-a93a-437c-8c92-81083cd98540"}, "archived": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "type": "last_edited_time", "last_edited_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "type": "created_by", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}}}, {"name": "Created", "value": {"id": "Ird4", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Type", "value": {"id": "_%7B%5C7", "type": "select", "select": {"id": "5fb57c36-999f-49e2-b153-96531d086862", "name": "Sprint Planning", "color": "yellow"}}}, {"name": "Participants", "value": {"id": "b%3AeA", "type": "people", "people": []}}, {"name": "Name", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Sprint Planning ", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Sprint Planning ", "href": null}]}}], "url": "https://www.notion.so/Sprint-Planning-621d3dc455fe46cea3ff83da06e5f9fb", "public_url": null}, "emitted_at": 1697023284465} +{"stream": "pages", "data": {"object": "page", "id": "6eb2dedc-8b88-486c-8648-d1878bafb106", "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "9b1ce91e-a93a-437c-8c92-81083cd98540"}, "archived": false, "properties": [{"name": "Last Edited Time", "value": {"id": "0AiB", "type": "last_edited_time", "last_edited_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Created By", "value": {"id": "F%5D)%3F", "type": "created_by", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a", "name": "Airyte", "avatar_url": null, "type": "person", "person": {"email": "integration-test@airbyte.io"}}}}, {"name": "Created", "value": {"id": "Ird4", "type": "created_time", "created_time": "2021-10-19T13:33:00.000Z"}}, {"name": "Type", "value": {"id": "_%7B%5C7", "type": "select", "select": {"id": "1747fcca-8207-42c8-802f-fd43965c016a", "name": "Ad Hoc", "color": "orange"}}}, {"name": "Participants", "value": {"id": "b%3AeA", "type": "people", "people": []}}, {"name": "Name", "value": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Ad Hoc Meeting", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Ad Hoc Meeting", "href": null}]}}], "url": "https://www.notion.so/Ad-Hoc-Meeting-6eb2dedc8b88486c8648d1878bafb106", "public_url": null}, "emitted_at": 1697023284465} +{"stream": "blocks", "data": {"object": "block", "id": "b54364a0-ff86-45ba-b78e-a32018446a3f", "parent": {"type": "page_id", "page_id": "39a69b4e-7cc2-4f7a-a656-dd128f3ce855"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "has_children": false, "archived": false, "type": "callout", "callout": {"rich_text": [{"type": "text", "text": {"content": "Change the title to include the date.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Change the title to include the date.", "href": null}], "icon": {"type": "emoji", "emoji": "\ud83d\udca1"}, "color": "gray_background"}}, "emitted_at": 1697023288683} +{"stream": "blocks", "data": {"object": "block", "id": "c6608513-db08-4411-8ec6-e343580cbf84", "parent": {"type": "page_id", "page_id": "39a69b4e-7cc2-4f7a-a656-dd128f3ce855"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "has_children": false, "archived": false, "type": "heading_1", "heading_1": {"rich_text": [{"type": "text", "text": {"content": "What did we do yesterday?", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "What did we do yesterday?", "href": null}], "is_toggleable": false, "color": "default"}}, "emitted_at": 1697023288684} +{"stream": "blocks", "data": {"object": "block", "id": "ffb233aa-59da-4d04-9cc8-e6b767bd1a85", "parent": {"type": "page_id", "page_id": "39a69b4e-7cc2-4f7a-a656-dd128f3ce855"}, "created_time": "2021-10-19T13:33:00.000Z", "last_edited_time": "2021-10-19T13:33:00.000Z", "created_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "last_edited_by": {"object": "user", "id": "f5ac1fcb-a06b-4dcc-80e5-403c40dfb38a"}, "has_children": false, "archived": false, "type": "bulleted_list_item", "bulleted_list_item": {"rich_text": [], "color": "default"}}, "emitted_at": 1697023288684} +{"stream": "comments", "data": {"object": "comment", "id": "e2dd2530-3ef1-4a27-83fb-3f16400b9838", "parent": {"type": "page_id", "page_id": "a55d276e-4bc2-4fcc-9fb3-e60b867c86e7"}, "discussion_id": "15e3cffe-3c9d-4ef2-87b1-86c46f85a205", "created_time": "2023-10-10T13:52:00.000Z", "last_edited_time": "2023-10-10T13:52:00.000Z", "created_by": {"object": "user", "id": "ec324c09-af75-40f0-b91a-49ded74fdaf5"}, "rich_text": [{"type": "text", "text": {"content": "Gathered voices speak,\nIdeas flow, plans take shape,\nWeek's promise whispers.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Gathered voices speak,\nIdeas flow, plans take shape,\nWeek's promise whispers.", "href": null}], "page_last_edited_time": "2021-10-19T13:33:00.000Z"}, "emitted_at": 1697023326959} +{"stream": "comments", "data": {"object": "comment", "id": "e2087302-7eab-4e1d-a95b-472404fb51a2", "parent": {"type": "page_id", "page_id": "249f3796-7e81-47b0-9075-00ed2d06439d"}, "discussion_id": "be372b5d-2610-435e-981a-9be271874b8e", "created_time": "2023-09-12T20:55:00.000Z", "last_edited_time": "2023-09-12T20:55:00.000Z", "created_by": {"object": "user", "id": "ec324c09-af75-40f0-b91a-49ded74fdaf5"}, "rich_text": [{"type": "text", "text": {"content": "Ladybug in flight; Red and black on petal\u2019s edge; Spring\u2019s tiny delight.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Ladybug in flight; Red and black on petal\u2019s edge; Spring\u2019s tiny delight.", "href": null}], "page_last_edited_time": "2021-10-19T13:33:00.000Z"}, "emitted_at": 1697023328874} +{"stream": "comments", "data": {"object": "comment", "id": "3f7ce236-fbc5-4b10-bb75-a0835c00aff3", "parent": {"type": "page_id", "page_id": "29299296-ef3f-4aff-aef5-02d651a59be3"}, "discussion_id": "c30808ec-fabd-4e7b-947f-a36dbec5c1db", "created_time": "2023-09-12T20:56:00.000Z", "last_edited_time": "2023-09-12T20:56:00.000Z", "created_by": {"object": "user", "id": "ec324c09-af75-40f0-b91a-49ded74fdaf5"}, "rich_text": [{"type": "text", "text": {"content": "APIs converge; Fixing gaps in code and docs; Two worlds now as one.", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "APIs converge; Fixing gaps in code and docs; Two worlds now as one.", "href": null}], "page_last_edited_time": "2021-10-19T13:33:00.000Z"}, "emitted_at": 1697023329304} diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-notion/integration_tests/sample_state.json index ef4cc82c0cd0..0c49b5a9f443 100644 --- a/airbyte-integrations/connectors/source-notion/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-notion/integration_tests/sample_state.json @@ -31,5 +31,16 @@ "name": "blocks" } } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { + "last_edited_time": "2021-10-10T04:00:00.000Z" + }, + "stream_descriptor": { + "name": "comments" + } + } } ] diff --git a/airbyte-integrations/connectors/source-notion/metadata.yaml b/airbyte-integrations/connectors/source-notion/metadata.yaml index e744b10f59b3..69f953d8a5e2 100644 --- a/airbyte-integrations/connectors/source-notion/metadata.yaml +++ b/airbyte-integrations/connectors/source-notion/metadata.yaml @@ -1,12 +1,18 @@ data: + ab_internal: + ql: 400 + sl: 200 allowedHosts: hosts: - api.notion.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: api connectorType: source definitionId: 6e00b415-b02e-4160-bf02-58176a0ae687 - dockerImageTag: 1.1.1 + dockerImageTag: 2.0.8 dockerRepository: airbyte/source-notion + documentationUrl: https://docs.airbyte.com/integrations/sources/notion githubIssueLabel: source-notion icon: notion.svg license: MIT @@ -17,11 +23,22 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/notion + releases: + breakingChanges: + 2.0.0: + message: + Version 2.0.0 introduces schema changes to multiple properties shared + by the blocks, databases and pages streams. These changes were introduced + to reflect updates to the Notion API. A full schema refresh and data reset + are required to upgrade to this version. + upgradeDeadline: "2023-11-09" + suggestedStreams: + streams: + - blocks + - databases + - pages + - users + supportLevel: certified tags: - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-notion/setup.py b/airbyte-integrations/connectors/source-notion/setup.py index 102f3ab3947a..80a1ed81533f 100644 --- a/airbyte-integrations/connectors/source-notion/setup.py +++ b/airbyte-integrations/connectors/source-notion/setup.py @@ -7,12 +7,14 @@ MAIN_REQUIREMENTS = [ "airbyte-cdk", + "pendulum==2.1.2", ] TEST_REQUIREMENTS = [ "pytest~=6.1", "pytest-mock~=3.6.1", "requests-mock", + "freezegun", ] setup( diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json index 797a17611b39..0e7131bfde53 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json @@ -13,10 +13,18 @@ "type": "string" }, "created_time": { - "type": "string" + "type": "string", + "format": "date-time" + }, + "created_by": { + "$ref": "user.json" + }, + "last_edited_by": { + "$ref": "user.json" }, "last_edited_time": { - "type": "string" + "type": "string", + "format": "date-time" }, "archived": { "type": "boolean" @@ -26,58 +34,66 @@ }, "type": { "enum": [ - "paragraph", + "bookmark", + "breadcrumb", + "bulleted_list_item", + "callout", + "child_database", + "child_page", + "code", + "column", + "column_list", + "divider", + "embed", + "equation", + "file", "heading_1", "heading_2", "heading_3", - "callout", - "bulleted_list_item", + "image", + "link_preview", + "link_to_page", "numbered_list_item", + "paragraph", + "pdf", + "quote", + "synced_block", + "table", + "table_of_contents", + "table_row", + "template", "to_do", "toggle", - "code", - "child_page", - "child_database", - "embed", - "image", - "video", - "file", - "pdf", - "bookmark", - "equation", - "unsupported" + "unsupported", + "video" ] }, - "paragraph": { "$ref": "text_element.json" }, - "quote": { "$ref": "text_element.json" }, - "bulleted_list_item": { "$ref": "text_element.json" }, - "numbered_list_item": { "$ref": "text_element.json" }, - "toggle": { "$ref": "text_element.json" }, - "heading_1": { "$ref": "heading.json" }, - "heading_2": { "$ref": "heading.json" }, - "heading_3": { "$ref": "heading.json" }, - "callout": { + "bookmark": { "type": "object", "properties": { - "color": { "type": "string" }, - "text": { "type": "array", "items": { "$ref": "rich_text.json" } }, - "icon": { "$ref": "icon.json" }, - "children": { "type": "array", "items": { "type": "object" } } + "url": { "type": "string" }, + "caption": { "type": "array", "items": { "$ref": "rich_text.json" } } } }, - "to_do": { + "breadcrumb": { + "type": "object" + }, + "bulleted_list_item": { "$ref": "text_element.json" }, + "callout": { "type": "object", "properties": { - "text": { "type": "array", "items": { "$ref": "rich_text.json" } }, - "checked": { "type": ["null", "boolean"] }, - "children": { "type": "array", "items": { "type": "object" } } + "color": { "type": "string" }, + "rich_text": { "type": "array", "items": { "$ref": "rich_text.json" } }, + "icon": { "$ref": "icon.json" } } }, + "child_page": { "$ref": "child.json" }, + "child_database": { "$ref": "child.json" }, "code": { "type": "object", "properties": { - "color": { "type": "string" }, - "text": { "type": "array", "items": { "$ref": "rich_text.json" } }, + "caption": { "type": "array", "items": { "$ref": "rich_text.json" } }, + "rich_text": { "type": "array", "items": { "$ref": "rich_text.json" } }, "language": { "enum": [ "abap", @@ -156,30 +172,118 @@ } } }, - "child_page": { "$ref": "child.json" }, - "child_database": { "$ref": "child.json" }, + "column": { + "type": "object" + }, + "column_list": { + "type": "object" + }, + "divider": { + "type": "object" + }, "embed": { "type": "object", "properties": { "url": { "type": "string" } } }, - "image": { "$ref": "file.json" }, - "video": { "$ref": "file.json" }, + "equation": { + "type": "object", + "properties": { + "expression": { "type": "string" } + } + }, "file": { "$ref": "file.json" }, + "heading_1": { "$ref": "heading.json" }, + "heading_2": { "$ref": "heading.json" }, + "heading_3": { "$ref": "heading.json" }, + "image": { "$ref": "file.json" }, + "link_preview": { + "type": "object", + "properties": { + "url": { "type": "string" } + } + }, + "link_to_page": { + "type": "object", + "properties": { + "page_id": { "type": "string" }, + "type": { "type": "string" } + } + }, + "numbered_list_item": { "$ref": "text_element.json" }, + "paragraph": { "$ref": "text_element.json" }, "pdf": { "$ref": "file.json" }, - "bookmark": { + "quote": { "$ref": "text_element.json" }, + "synced_block": { "type": "object", "properties": { - "url": { "type": "string" }, - "caption": { "type": "array", "items": { "$ref": "rich_text.json" } } + "synced_from": { + "type": ["null", "object"], + "properties": { + "type": { + "type": "string", + "enum": ["block_id"] + }, + "block_id": { + "type": "string" + } + } + }, + "children": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true + } + } } }, - "equation": { + "table": { "type": "object", "properties": { - "expression": { "type": "string" } + "table_width": { "type": "integer" }, + "has_column_header": { "type": "boolean" }, + "has_row_header": { "type": "boolean" } + } + }, + "table_of_contents": { + "type": "object", + "properties": { + "color": { "type": "string" } } + }, + "table_row": { + "type": "object", + "properties": { + "cells": { + "type": ["null", "array"], + "items": { + "type": ["null", "array"], + "items": { "$ref": "rich_text.json" } + } + } + } + }, + "template": { + "type": "object", + "properties": { + "rich_text": { "type": "array", "items": { "$ref": "rich_text.json" } } + } + }, + "to_do": { + "type": "object", + "properties": { + "rich_text": { "type": "array", "items": { "$ref": "rich_text.json" } }, + "checked": { "type": ["null", "boolean"] }, + "color": { "type": "string" }, + "children": { "type": "array", "items": { "type": "object" } } + } + }, + "toggle": { "$ref": "text_element.json" }, + "video": { "$ref": "file.json" }, + "unsupported": { + "type": "object" } } } diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/comments.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/comments.json new file mode 100644 index 000000000000..1ab379a06ed9 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/comments.json @@ -0,0 +1,93 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "object": { + "enum": ["comment"] + }, + "id": { + "type": "string" + }, + "parent": { + "type": "object", + "properties": { + "type": { + "enum": ["page_id"] + }, + "page_id": { + "type": "string" + } + } + }, + "discussion_id": { + "type": "string" + }, + "created_time": { + "type": "string", + "format": "date-time" + }, + "last_edited_time": { + "type": "string", + "format": "date-time" + }, + "page_last_edited_time": { + "type": "string", + "format": "date-time" + }, + "created_by": { + "$ref": "user.json" + }, + "rich_text": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "text": { + "type": "object", + "properties": { + "content": { + "type": "string" + }, + "link": { + "type": ["null", "object"] + } + } + }, + "annotations": { + "type": "object", + "properties": { + "bold": { + "type": "boolean" + }, + "italic": { + "type": "boolean" + }, + "strikethrough": { + "type": "boolean" + }, + "underline": { + "type": "boolean" + }, + "code": { + "type": "boolean" + }, + "color": { + "type": "string" + } + } + }, + "plain_text": { + "type": "string" + }, + "href": { + "type": ["null", "string"] + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/databases.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/databases.json index 8b521d46fcb9..55f004c52241 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/databases.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/databases.json @@ -10,10 +10,12 @@ "type": "string" }, "created_time": { - "type": "string" + "type": "string", + "format": "date-time" }, "last_edited_time": { - "type": "string" + "type": "string", + "format": "date-time" }, "title": { "type": "array", @@ -21,6 +23,12 @@ "$ref": "rich_text.json" } }, + "description": { + "type": "array", + "items": { + "$ref": "rich_text.json" + } + }, "last_edited_by": { "$ref": "user.json" }, @@ -45,6 +53,9 @@ "is_inline": { "type": ["null", "boolean"] }, + "public_url": { + "type": ["null", "string"] + }, "properties": { "type": "array", "items": { @@ -57,7 +68,7 @@ "value": { "type": "object", "additionalProperties": true, - "oneOf": [ + "anyOf": [ { "type": "object", "additionalProperties": true, @@ -67,19 +78,26 @@ }, "type": { "enum": [ - "title", - "rich_text", - "date", - "people", - "files", "checkbox", - "url", - "email", - "phone_number", - "created_time", "created_by", + "created_time", + "date", + "email", + "files", + "formula", + "last_edited_by", "last_edited_time", - "last_edited_by" + "multi_select", + "number", + "people", + "phone_number", + "relation", + "rich_text", + "rollup", + "select", + "status", + "title", + "url" ] }, "name": { @@ -87,6 +105,159 @@ } } }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["checkbox"] + }, + "name": { + "type": "string" + }, + "checkbox": { + "type": "object" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["created_by"] + }, + "name": { + "type": "string" + }, + "created_by": { + "type": "object" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["created_time"] + }, + "name": { + "type": "string" + }, + "created_time": { + "type": "object" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["date"] + }, + "name": { + "type": "string" + }, + "date": { + "type": "object" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["email"] + }, + "name": { + "type": "string" + }, + "email": { + "type": "object" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["files"] + }, + "name": { + "type": "string" + }, + "files": { + "type": "object" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["formula"] + }, + "name": { + "type": "string" + }, + "expression": { + "type": "string" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["last_edited_by"] + }, + "name": { + "type": "string" + }, + "last_edited_by": { + "type": "string" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["last_edited_time"] + }, + "name": { + "type": "string" + }, + "last_edited_time": { + "type": "string" + } + } + }, { "type": "object", "properties": { @@ -101,42 +272,46 @@ }, "format": { "enum": [ - "number", - "number_with_commas", - "percent", - "dollar", + "argentine_peso", + "baht", "canadian_dollar", + "chilean_peso", + "colombian_peso", + "danish_krone", + "dirham", + "dollar", "euro", - "pound", - "yen", - "ruble", - "rupee", - "won", - "yuan", - "real", - "lira", - "rupiah", + "forint", "franc", "hong_kong_dollar", - "new_zealand_dollar", + "koruna", "krona", - "norwegian_krone", + "leu", + "lira", "mexican_peso", - "rand", "new_taiwan_dollar", - "danish_krone", - "zloty", - "baht", - "forint", - "koruna", - "shekel", - "chilean_peso", + "new_zealand_dollar", + "norwegian_krone", + "number", + "number_with_commas", + "percent", + "peruvian_sol", "philippine_peso", - "dirham", - "colombian_peso", - "riyal", + "pound", + "rand", + "real", "ringgit", - "leu" + "riyal", + "ruble", + "rupee", + "rupiah", + "shekel", + "singapore_dollar", + "uruguayan_peso", + "won", + "yen", + "yuan", + "zloty" ] } } @@ -168,16 +343,33 @@ "type": "string" }, "type": { - "enum": ["formula"] + "enum": ["people"] }, "name": { "type": "string" }, - "expression": { + "people": { "type": "string" } } }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["phone_number"] + }, + "name": { + "type": "string" + }, + "phone_number": { + "type": "object" + } + } + }, { "type": "object", "properties": { @@ -201,6 +393,23 @@ } } }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["rich_text"] + }, + "name": { + "type": "string" + }, + "rich_text": { + "type": "object" + } + } + }, { "type": "object", "properties": { @@ -227,23 +436,115 @@ }, "function": { "enum": [ - "count_all", - "count_values", - "count_unique_values", - "count_empty", - "count_not_empty", - "percent_empty", - "percent_not_empty", - "sum", "average", + "checked", + "count_per_group", + "count", + "count_values", + "date_range", + "earliest_date", + "empty", + "latest_date", + "max", "median", "min", - "max", + "not_empty", + "percent_checked", + "percent_empty", + "percent_not_empty", + "percent_per_group", + "percent_unchecked", "range", - "show_original" + "unchecked", + "unique", + "show_original", + "show_unique", + "sum" ] } } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["status"] + }, + "name": { + "type": "string" + }, + "status": { + "type": "object", + "properties": { + "options": { + "type": "array", + "items": { + "$ref": "options.json" + } + }, + "groups": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "color": { + "type": "string" + }, + "option_ids": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } + } + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["title"] + }, + "name": { + "type": "string" + }, + "title": { + "type": "object" + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["url"] + }, + "name": { + "type": "string" + }, + "url": { + "type": "object" + } + } } ] } diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json index 14275b932ace..7972b07d6c73 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json @@ -10,14 +10,25 @@ "type": "string" }, "created_time": { - "type": "string" + "type": "string", + "format": "date-time" + }, + "created_by": { + "$ref": "user.json" }, "last_edited_time": { - "type": "string" + "type": "string", + "format": "date-time" + }, + "last_edited_by": { + "$ref": "user.json" }, "archived": { "type": "boolean" }, + "icon": { + "$ref": "icon.json" + }, "cover": { "$ref": "file.json" }, @@ -27,6 +38,9 @@ "url": { "type": "string" }, + "public_url": { + "type": ["null", "string"] + }, "properties": { "type": "array", "items": { @@ -61,7 +75,10 @@ "type": { "enum": ["rich_text"] }, - "rich_text": { "$ref": "rich_text.json" } + "rich_text": { + "type": ["null", "array"], + "items": { "$ref": "rich_text.json" } + } } }, { @@ -213,7 +230,7 @@ "properties": { "id": { "type": "string" }, "type": { "enum": ["phone_number"] }, - "phone_number": { "type": "string" } + "phone_number": { "type": "object" } } }, { @@ -247,6 +264,62 @@ "type": { "enum": ["last_edited_by"] }, "last_edited_by": { "$ref": "user.json" } } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["number"] }, + "number": { + "type": "object", + "properties": { + "format": { "type": "string" } + } + } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["status"] }, + "status": { "$ref": "options.json" } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["unique_id"] }, + "unique_id": { + "type": "object", + "properties": { + "number": { "type": "number" }, + "prefix": { "type": ["null", "string"] } + } + } + } + }, + { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": ["verification"] + }, + "verification": { + "type": "object", + "properties": { + "state": { + "enum": ["verified", "unverified"] + }, + "verified_by": { "$ref": "user.json" }, + "date": { "$ref": "date.json" } + } + } + } } ] } diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/file.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/file.json index 7188775802a7..38dbff8d0903 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/file.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/file.json @@ -29,7 +29,8 @@ "type": "string" }, "expiry_time": { - "type": "string" + "type": "string", + "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/heading.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/heading.json index eb526a17a235..b321541d43f6 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/heading.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/heading.json @@ -4,6 +4,7 @@ "additionalProperties": true, "properties": { "color": { "type": "string" }, - "text": { "type": "array", "items": { "$ref": "rich_text.json" } } + "rich_text": { "type": "array", "items": { "$ref": "rich_text.json" } }, + "is_toggleable": { "type": "boolean" } } } diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/parent.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/parent.json index 5c17153c2a14..db2cf20b9e81 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/parent.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/parent.json @@ -4,7 +4,10 @@ "additionalProperties": true, "properties": { "type": { - "enum": ["database_id", "page_id", "workspace"] + "enum": ["block_id", "database_id", "page_id", "workspace"] + }, + "block_id": { + "type": "string" }, "database_id": { "type": "string" diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json index 781c739a17be..cc003048b875 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json @@ -26,24 +26,21 @@ } } }, - "rich_text": { + "mention": { "type": ["null", "object"], "additionalProperties": true, "properties": { - "content": { + "type": { + "type": ["null", "string"] + } + } + }, + "equation": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "expression": { "type": ["null", "string"] - }, - "link": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "type": { - "enum": ["url"] - }, - "url": { - "type": ["null", "string"] - } - } } } }, diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/text_element.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/text_element.json index ee7ca504414e..65411d825b11 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/text_element.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/text_element.json @@ -4,7 +4,7 @@ "additionalProperties": true, "properties": { "color": { "type": "string" }, - "text": { "type": "array", "items": { "$ref": "rich_text.json" } }, + "rich_text": { "type": "array", "items": { "$ref": "rich_text.json" } }, "children": { "type": "array", "items": { "type": "object" } } } } diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json index 2fb94bfa035e..d893b1147b7e 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json @@ -10,7 +10,7 @@ "type": "string" }, "name": { - "type": "string" + "type": ["null", "string"] }, "avatar_url": { "type": ["null", "string"] @@ -23,7 +23,7 @@ "additionalProperties": true, "properties": { "email": { - "type": "string" + "type": ["null", "string"] } } }, diff --git a/airbyte-integrations/connectors/source-notion/source_notion/source.py b/airbyte-integrations/connectors/source-notion/source_notion/source.py index a0de37c49080..08262edfeee4 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/source.py +++ b/airbyte-integrations/connectors/source-notion/source_notion/source.py @@ -3,51 +3,96 @@ # +import logging +import re from typing import Any, List, Mapping, Tuple +import pendulum import requests -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +from pendulum.parsing.exceptions import ParserError -from .streams import Blocks, Databases, Pages, Users +from .streams import Blocks, Comments, Databases, Pages, Users -class NotionAuthenticator: - def __init__(self, config: Mapping[str, Any]): - self.config = config +class SourceNotion(AbstractSource): + def _get_authenticator(self, config: Mapping[str, Any]) -> TokenAuthenticator: + credentials = config.get("credentials", {}) + auth_type = credentials.get("auth_type") + token = credentials.get("access_token") if auth_type == "OAuth2.0" else credentials.get("token") - def get_access_token(self): - credentials = self.config.get("credentials") - if credentials: - auth_type = credentials.get("auth_type") - if auth_type == "OAuth2.0": - return TokenAuthenticator(credentials.get("access_token")) - return TokenAuthenticator(credentials.get("token")) + if credentials and token: + return TokenAuthenticator(token) - # support the old config - if "access_token" in self.config: - return TokenAuthenticator(self.config.get("access_token")) + # The original implementation did not support OAuth, and therefore had no "credentials" key. + # We can maintain backwards compatibility for OG connections by checking for the deprecated "access_token" key, just in case. + if config.get("access_token"): + return TokenAuthenticator(config["access_token"]) + def _validate_start_date(self, config: Mapping[str, Any]): + start_date = config.get("start_date") -class SourceNotion(AbstractSource): - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: + if start_date: + pattern = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z") + if not pattern.match(start_date): # Compare against the pattern descriptor. + return "Please check the format of the start date against the pattern descriptor." + + try: # Handle invalid dates. + parsed_start_date = pendulum.parse(start_date) + except ParserError: + return "The provided start date is not a valid date. Please check the format and try again." + + if parsed_start_date > pendulum.now("UTC"): # Handle future start date. + return "The start date cannot be greater than the current date." + + return None + + def _extract_error_message(self, response: requests.Response) -> str: + """ + Return a human-readable error message from a Notion API response, for use in connection check. + """ + error_json = response.json() + error_code = error_json.get("code", "unknown_error") + error_message = error_json.get( + "message", "An unspecified error occurred while connecting to Notion. Please check your credentials and try again." + ) + + if error_code == "unauthorized": + return "The provided API access token is invalid. Please double-check that you input the correct token and have granted the necessary permissions to your Notion integration." + if error_code == "restricted_resource": + return "The provided API access token does not have the correct permissions configured. Please double-check that you have granted all the necessary permissions to your Notion integration." + return f"Error: {error_message} (Error code: {error_code})" + + def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, any]: + # First confirm that if start_date is set by user, it is valid. + validation_error = self._validate_start_date(config) + if validation_error: + return False, validation_error try: - authenticator = NotionAuthenticator(config).get_access_token() - stream = Users(authenticator=authenticator, config=config) - records = stream.read_records(sync_mode=SyncMode.full_refresh) - next(records) - return True, None + authenticator = self._get_authenticator(config) + # Notion doesn't have a dedicated ping endpoint, so we can use the users/me endpoint instead. + # Endpoint docs: https://developers.notion.com/reference/get-self + ping_endpoint = "https://api.notion.com/v1/users/me" + notion_version = {"Notion-Version": "2022-06-28"} + response = requests.get(ping_endpoint, auth=authenticator, headers=notion_version) + + if response.status_code == 200: + return True, None + else: + error_message = self._extract_error_message(response) + return False, error_message + except requests.exceptions.RequestException as e: - return False, e + return False, str(e) def streams(self, config: Mapping[str, Any]) -> List[Stream]: - AirbyteLogger().log("INFO", f"Using start_date: {config['start_date']}") - authenticator = NotionAuthenticator(config).get_access_token() + + authenticator = self._get_authenticator(config) args = {"authenticator": authenticator, "config": config} pages = Pages(**args) blocks = Blocks(parent=pages, **args) + comments = Comments(parent=pages, **args) - return [Users(**args), Databases(**args), pages, blocks] + return [Users(**args), Databases(**args), pages, blocks, comments] diff --git a/airbyte-integrations/connectors/source-notion/source_notion/spec.json b/airbyte-integrations/connectors/source-notion/source_notion/spec.json index b237e7691e2b..4b833a567454 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/spec.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/spec.json @@ -4,19 +4,20 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Notion Source Spec", "type": "object", - "required": ["start_date"], + "required": ["credentials"], "properties": { "start_date": { "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00.000Z. Any data before this date will not be replicated.", + "description": "UTC date and time in the format YYYY-MM-DDTHH:MM:SS.000Z. During incremental sync, any data generated before this date will not be replicated. If left blank, the start date will be set to 2 years before the present date.", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z$", + "pattern_descriptor": "YYYY-MM-DDTHH:MM:SS.000Z", "examples": ["2020-11-16T00:00:00.000Z"], "type": "string", "format": "date-time" }, "credentials": { - "title": "Authenticate using", - "description": "Pick an authentication method.", + "title": "Authentication Method", + "description": "Choose either OAuth (recommended for Airbyte Cloud) or Access Token. See our docs for more information.", "type": "object", "order": 1, "oneOf": [ @@ -37,19 +38,19 @@ "client_id": { "title": "Client ID", "type": "string", - "description": "The ClientID of your Notion integration.", + "description": "The Client ID of your Notion integration. See our docs for more information.", "airbyte_secret": true }, "client_secret": { "title": "Client Secret", "type": "string", - "description": "The ClientSecret of your Notion integration.", + "description": "The Client Secret of your Notion integration. See our docs for more information.", "airbyte_secret": true }, "access_token": { "title": "Access Token", "type": "string", - "description": "Access Token is a token you received by complete the OauthWebFlow of Notion.", + "description": "The Access Token received by completing the OAuth flow for your Notion integration. See our docs for more information.", "airbyte_secret": true } } @@ -65,7 +66,7 @@ }, "token": { "title": "Access Token", - "description": "Notion API access token, see the docs for more information on how to obtain this token.", + "description": "The Access Token for your private Notion integration. See the docs for more information on how to obtain this token.", "type": "string", "airbyte_secret": true } diff --git a/airbyte-integrations/connectors/source-notion/source_notion/streams.py b/airbyte-integrations/connectors/source-notion/source_notion/streams.py index 061fa5c3912d..a0546d2116c4 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/streams.py +++ b/airbyte-integrations/connectors/source-notion/source_notion/streams.py @@ -3,14 +3,19 @@ # from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, TypeVar +from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, TypeVar +import pendulum import pydantic import requests +from airbyte_cdk.logger import AirbyteLogger as Logger from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy +from airbyte_cdk.sources import Source +from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream +from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from airbyte_cdk.sources.streams.http.exceptions import UserDefinedBackoffException +from requests import HTTPError from .utils import transform_properties @@ -18,6 +23,20 @@ MAX_BLOCK_DEPTH = 30 +class NotionAvailabilityStrategy(HttpAvailabilityStrategy): + """ + Inherit from HttpAvailabilityStrategy with slight modification to 403 error message. + """ + + def reasons_for_unavailable_status_codes(self, stream: Stream, logger: Logger, source: Source, error: HTTPError) -> Dict[int, str]: + + reasons_for_codes: Dict[int, str] = { + requests.codes.FORBIDDEN: "This is likely due to insufficient permissions for your Notion integration. " + "Please make sure your integration has read access for the resources you are trying to sync" + } + return reasons_for_codes + + class NotionStream(HttpStream, ABC): url_base = "https://api.notion.com/v1/" @@ -30,11 +49,28 @@ class NotionStream(HttpStream, ABC): def __init__(self, config: Mapping[str, Any], **kwargs): super().__init__(**kwargs) - self.start_date = config["start_date"] + self.start_date = config.get("start_date") + + # If start_date is not found in config, set it to 2 years ago and update value in config for use in next stream + if not self.start_date: + self.start_date = pendulum.now().subtract(years=2).in_timezone("UTC").format("YYYY-MM-DDTHH:mm:ss.SSS[Z]") + config["start_date"] = self.start_date + + @property + def availability_strategy(self) -> HttpAvailabilityStrategy: + return NotionAvailabilityStrategy() + + @property + def retry_factor(self) -> int: + return 5 + + @property + def max_retries(self) -> int: + return 7 @property - def availability_strategy(self) -> Optional["AvailabilityStrategy"]: - return None + def max_time(self) -> int: + return 60 * 11 @staticmethod def check_invalid_start_cursor(response: requests.Response): @@ -43,14 +79,40 @@ def check_invalid_start_cursor(response: requests.Response): if message.startswith("The start_cursor provided is invalid: "): return message + @staticmethod + def throttle_request_page_size(current_page_size): + """ + Helper method to halve page_size when encountering a 504 Gateway Timeout error. + """ + throttled_page_size = max(current_page_size // 2, 10) + return throttled_page_size + def backoff_time(self, response: requests.Response) -> Optional[float]: - retry_after = response.headers.get("retry-after") - if retry_after: + """ + Notion's rate limit is approx. 3 requests per second, with larger bursts allowed. + For a 429 response, we can use the retry-header to determine how long to wait before retrying. + For 500-level errors, we use Airbyte CDK's default exponential backoff with a retry_factor of 5. + Docs: https://developers.notion.com/reference/errors#rate-limiting + """ + retry_after = response.headers.get("retry-after", "5") + if response.status_code == 429: return float(retry_after) if self.check_invalid_start_cursor(response): return 10 + return super().backoff_time(response) def should_retry(self, response: requests.Response) -> bool: + # In the case of a 504 Gateway Timeout error, we can lower the page_size when retrying to reduce the load on the server. + if response.status_code == 504: + self.page_size = self.throttle_request_page_size(self.page_size) + self.logger.info(f"Encountered a server timeout. Reducing request page size to {self.page_size} and retrying.") + + # If page_size has been reduced after encountering a 504 Gateway Timeout error, + # we increase it back to the default of 100 once a success response is achieved, for the following API calls. + if response.status_code == 200 and self.page_size != 100: + self.page_size = 100 + self.logger.info(f"Successfully reconnected after a server timeout. Increasing request page size to {self.page_size}.") + return response.status_code == 400 or super().should_retry(response) def request_headers(self, **kwargs) -> Mapping[str, Any]: @@ -70,9 +132,9 @@ def next_page_token( "has_more": true, "results": [ ... ] } - Doc: https://developers.notion.com/reference/pagination + Doc: https://developers.notion.com/reference/intro#pagination """ - next_cursor = response.json()["next_cursor"] + next_cursor = response.json().get("next_cursor") if next_cursor: return {"next_cursor": next_cursor} @@ -158,7 +220,7 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str, state_lmd = stream_state.get(self.cursor_field, "") if isinstance(state_lmd, StateValueWrapper): state_lmd = state_lmd.value - if not stream_state or record_lmd >= state_lmd: + if (not stream_state or record_lmd >= state_lmd) and record_lmd >= self.start_date: yield from transform_properties(record) def get_updated_state( @@ -298,3 +360,64 @@ def should_retry(self, response: requests.Response) -> bool: else: return super().should_retry(response) return super().should_retry(response) + + +class Comments(HttpSubStream, IncrementalNotionStream): + """ + Comments Object Docs: https://developers.notion.com/reference/comment-object + Comments Endpoint Docs: https://developers.notion.com/reference/retrieve-a-comment + """ + + http_method = "GET" + # We can use the "last edited time" of the parent Page as the cursor field, + # since we cannot guarantee the order of comments between pages. + cursor_field = "page_last_edited_time" + + def path(self, **kwargs) -> str: + return "comments" + + def request_params( + self, next_page_token: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, **kwargs + ) -> MutableMapping[str, Any]: + block_id = stream_slice.get("block_id") + params = {"block_id": block_id, "page_size": self.page_size} + + if next_page_token: + params["start_cursor"] = next_page_token["next_cursor"] + + return params + + def parse_response( + self, response: requests.Response, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs + ) -> Iterable[Mapping]: + + # Get the parent's "last edited time" to compare against state + page_last_edited_time = stream_slice.get("page_last_edited_time", "") + records = response.json().get("results", []) + + for record in records: + record["page_last_edited_time"] = page_last_edited_time + state_last_edited_time = stream_state.get(self.cursor_field, "") + + if isinstance(state_last_edited_time, StateValueWrapper): + state_last_edited_time = state_last_edited_time.value + + if not stream_state or page_last_edited_time >= state_last_edited_time: + yield from transform_properties(record) + + def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: + + yield from IncrementalNotionStream.read_records(self, **kwargs) + + def stream_slices( + self, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, **kwargs + ) -> Iterable[Optional[Mapping[str, Any]]]: + + # Gather parent stream records in full + parent_records = self.parent.read_records(sync_mode=SyncMode.full_refresh, cursor_field=self.parent.cursor_field) + + # The parent stream is the Pages stream, but we have to pass its id to the request_params as "block_id" + # because pages are also blocks in the Notion API. + # We also grab the last_edited_time from the parent record to use as the cursor field. + for record in parent_records: + yield {"block_id": record["id"], "page_last_edited_time": record["last_edited_time"]} diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_incremental_streams.py index 90a54129f311..d34afd131fc0 100644 --- a/airbyte-integrations/connectors/source-notion/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_incremental_streams.py @@ -2,11 +2,14 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import re +import time from unittest.mock import MagicMock, patch from airbyte_cdk.models import SyncMode -from pytest import fixture -from source_notion.streams import Blocks, IncrementalNotionStream, Pages +from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException, UserDefinedBackoffException +from pytest import fixture, mark +from source_notion.streams import Blocks, Comments, IncrementalNotionStream, Pages @fixture @@ -37,6 +40,11 @@ def blocks(parent, args): return Blocks(parent=parent, **args) +@fixture +def comments(parent, args): + return Comments(parent=parent, **args) + + def test_cursor_field(stream): expected_cursor_field = "last_edited_time" assert stream.cursor_field == expected_cursor_field @@ -67,7 +75,16 @@ def test_get_updated_state(stream): def test_stream_slices(blocks, requests_mock): stream = blocks - requests_mock.post("https://api.notion.com/v1/search", json={"results": [{"id": "aaa"}, {"id": "bbb"}], "next_cursor": None}) + requests_mock.post( + "https://api.notion.com/v1/search", + json={ + "results": [ + {"id": "aaa", "last_edited_time": "2022-10-10T00:00:00.000Z"}, + {"id": "bbb", "last_edited_time": "2022-10-10T00:00:00.000Z"}, + ], + "next_cursor": None, + }, + ) inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} expected_stream_slice = [{"page_id": "aaa"}, {"page_id": "bbb"}] assert list(stream.stream_slices(**inputs)) == expected_stream_slice @@ -178,10 +195,10 @@ def test_recursive_read(blocks, requests_mock): # |-> record4 root = "aaa" - record1 = {"id": "id1", "type": "heading_1", "has_children": True, "last_edited_time": ""} - record2 = {"id": "id2", "type": "heading_1", "has_children": True, "last_edited_time": ""} - record3 = {"id": "id3", "type": "heading_1", "has_children": False, "last_edited_time": ""} - record4 = {"id": "id4", "type": "heading_1", "has_children": False, "last_edited_time": ""} + record1 = {"id": "id1", "type": "heading_1", "has_children": True, "last_edited_time": "2022-10-10T00:00:00.000Z"} + record2 = {"id": "id2", "type": "heading_1", "has_children": True, "last_edited_time": "2022-10-10T00:00:00.000Z"} + record3 = {"id": "id3", "type": "heading_1", "has_children": False, "last_edited_time": "2022-10-10T00:00:00.000Z"} + record4 = {"id": "id4", "type": "heading_1", "has_children": False, "last_edited_time": "2022-10-10T00:00:00.000Z"} requests_mock.get(f"https://api.notion.com/v1/blocks/{root}/children", json={"results": [record1, record4], "next_cursor": None}) requests_mock.get(f"https://api.notion.com/v1/blocks/{record1['id']}/children", json={"results": [record2], "next_cursor": None}) requests_mock.get(f"https://api.notion.com/v1/blocks/{record2['id']}/children", json={"results": [record3], "next_cursor": None}) @@ -194,12 +211,193 @@ def test_recursive_read(blocks, requests_mock): def test_invalid_start_cursor(parent, requests_mock, caplog): stream = parent error_message = "The start_cursor provided is invalid: wrong_start_cursor" - search_endpoint = requests_mock.post("https://api.notion.com/v1/search", status_code=400, - json={"object": "error", "status": 400, "code": "validation_error", - "message": error_message}) + search_endpoint = requests_mock.post( + "https://api.notion.com/v1/search", + status_code=400, + json={"object": "error", "status": 400, "code": "validation_error", "message": error_message}, + ) inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} with patch.object(stream, "backoff_time", return_value=0.1): list(stream.read_records(**inputs)) - assert search_endpoint.call_count == 6 + assert search_endpoint.call_count == 8 assert f"Skipping stream pages, error message: {error_message}" in caplog.messages + + +@mark.parametrize( + "status_code,error_code,error_message, expected_backoff_time", + [ + (400, "validation_error", "The start_cursor provided is invalid: wrong_start_cursor", [10, 10, 10, 10, 10, 10, 10]), + (429, "rate_limited", "Rate Limited", [5, 5, 5, 5, 5, 5, 5]), # Retry-header is set to 5 seconds for test + (500, "internal_server_error", "Internal server error", [5, 10, 20, 40, 80, 5, 10]), + ], +) +def test_retry_logic(status_code, error_code, error_message, expected_backoff_time, parent, requests_mock, caplog): + stream = parent + + # Set up a generator that alternates between error and success responses, to check the reset of backoff time between failures + mock_responses = ( + [ + { + "status_code": status_code, + "response": {"object": "error", "status": status_code, "code": error_code, "message": error_message}, + } + for _ in range(5) + ] + + [{"status_code": 200, "response": {"object": "list", "results": [], "has_more": True, "next_cursor": "dummy_cursor"}}] + + [ + { + "status_code": status_code, + "response": {"object": "error", "status": status_code, "code": error_code, "message": error_message}, + } + for _ in range(2) + ] + + [{"status_code": 200, "response": {"object": "list", "results": [], "has_more": False, "next_cursor": None}}] + ) + + def response_callback(request, context): + # Get the next response from the mock_responses list + response = mock_responses.pop(0) + context.status_code = response["status_code"] + return response["response"] + + # Mock the time.sleep function to avoid waiting during tests + with patch.object(time, "sleep", return_value=None): + search_endpoint = requests_mock.post( + "https://api.notion.com/v1/search", + json=response_callback, + headers={"retry-after": "5"}, + ) + + inputs = {"sync_mode": SyncMode.full_refresh, "cursor_field": [], "stream_state": {}} + try: + list(stream.read_records(**inputs)) + except (UserDefinedBackoffException, DefaultBackoffException) as e: + return e + + # Check that the endpoint was called the expected number of times + assert search_endpoint.call_count == 9 + + # Additional assertions to check reset of backoff time + # Find the backoff times from the message logs to compare against expected backoff times + log_messages = [record.message for record in caplog.records] + backoff_times = [ + round(float(re.search(r"(\d+(\.\d+)?) seconds", msg).group(1))) + for msg in log_messages + if any(word in msg for word in ["Sleeping", "Waiting"]) + ] + + assert backoff_times == expected_backoff_time, f"Unexpected backoff times: {backoff_times}" + + +# Tests for Comments stream +def test_comments_path(comments): + assert comments.path() == "comments" + + +def test_comments_request_params(comments): + """ + Test that the request_params function returns the correct parameters for the Comments endpoint + """ + params = comments.request_params( + next_page_token=None, stream_slice={"block_id": "block1", "page_last_edited_time": "2021-01-01T00:00:00.000Z"} + ) + + assert params == {"block_id": "block1", "page_size": comments.page_size} + + +def test_comments_stream_slices(comments, requests_mock): + """ + Test that the stream_slices function returns the parent page ids as "block_id" and the last edited time as "page_last_edited_time" + """ + + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": comments.cursor_field, "stream_state": {}} + + requests_mock.post( + "https://api.notion.com/v1/search", + json={ + "results": [ + {"name": "page_1", "id": "id_1", "last_edited_time": "2021-01-01T00:00:00.000Z"}, + {"name": "page_2", "id": "id_2", "last_edited_time": "2021-20-01T00:00:00.000Z"}, + ], + "next_cursor": None, + }, + ) + + expected_stream_slice = [ + {"block_id": "id_1", "page_last_edited_time": "2021-01-01T00:00:00.000Z"}, + {"block_id": "id_2", "page_last_edited_time": "2021-20-01T00:00:00.000Z"}, + ] + + actual_stream_slices_list = list(comments.stream_slices(**inputs)) + assert actual_stream_slices_list == expected_stream_slice + + +@mark.parametrize( + "stream_slice, stream_state, mock_data, expected_records", + [ + # Test that comments with page_last_edited_time >= stream_state are replicated, regardless of each record's LMD + ( + {"block_id": "block_id_1", "page_last_edited_time": "2023-10-10T00:00:00.000Z"}, + {"page_last_edited_time": "2021-10-10T00:00:00.000Z"}, + [ + { + "id": "comment_id_1", + "rich_text": [{"type": "text", "text": {"content": "I am the Alpha comment"}}], + "last_edited_time": "2021-01-01T00:00:00.000Z", + }, + { + "id": "comment_id_2", + "rich_text": [{"type": "text", "text": {"content": "I am the Omega comment"}}], + "last_edited_time": "2022-12-31T00:00:00.000Z", + }, + ], + [ + { + "id": "comment_id_1", + "rich_text": [{"type": "text", "text": {"content": "I am the Alpha comment"}}], + "last_edited_time": "2021-01-01T00:00:00.000Z", + "page_last_edited_time": "2023-10-10T00:00:00.000Z", + }, + { + "id": "comment_id_2", + "rich_text": [{"type": "text", "text": {"content": "I am the Omega comment"}}], + "last_edited_time": "2022-12-31T00:00:00.000Z", + "page_last_edited_time": "2023-10-10T00:00:00.000Z", + }, + ], + ), + # Test that comments with page_last_edited_time < stream_state are not replicated, regardless of each record's LMD + ( + {"block_id": "block_id_2", "page_last_edited_time": "2021-01-01T00:00:00.000Z"}, + {"page_last_edited_time": "2022-20-20T00:00:00.000Z"}, + [ + { + "id": "comment_id_1", + "rich_text": [{"type": "text", "text": {"content": "I will not be replicated"}}], + "last_edited_time": "2021-10-30T00:00:00.000Z", + }, + { + "id": "comment_id_2", + "rich_text": [{"type": "text", "text": {"content": "I will also not be replicated"}}], + "last_edited_time": "2023-01-01T00:00:00.000Z", + }, + ], + [], + ), + ], +) +def test_comments_read_records(comments, requests_mock, stream_slice, stream_state, mock_data, expected_records): + inputs = { + "sync_mode": SyncMode.incremental, + "cursor_field": comments.cursor_field, + "stream_state": stream_state, + "stream_slice": stream_slice, + } + + requests_mock.get( + f"https://api.notion.com/v1/comments?block_id={stream_slice['block_id']}", json={"results": mock_data, "next_cursor": None} + ) + + response = list(comments.read_records(**inputs)) + assert response == expected_records diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_source.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_source.py index f71734a678f3..2831b1b0f8ed 100644 --- a/airbyte-integrations/connectors/source-notion/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_source.py @@ -4,19 +4,83 @@ from unittest.mock import MagicMock +import pytest +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from source_notion.source import SourceNotion +UNAUTHORIZED_ERROR_MESSAGE = "The provided API access token is invalid. Please double-check that you input the correct token and have granted the necessary permissions to your Notion integration." +RESTRICTED_RESOURCE_ERROR_MESSAGE = "The provided API access token does not have the correct permissions configured. Please double-check that you have granted all the necessary permissions to your Notion integration." +GENERIC_ERROR_MESSAGE = "Conflict occurred while saving. Please try again." +DEFAULT_ERROR_MESSAGE = "An unspecified error occurred while connecting to Notion. Please check your credentials and try again." + def test_check_connection(mocker, requests_mock): source = SourceNotion() - logger_mock, config_mock = MagicMock(), MagicMock() - requests_mock.get("https://api.notion.com/v1/users", json={"results": [{"id": "aaa"}], "next_cursor": None}) + logger_mock, config_mock = MagicMock(), {"access_token": "test_token", "start_date": "2021-01-01T00:00:00.000Z"} + requests_mock.get( + "https://api.notion.com/v1/users/me", + json={"results": [{"id": "aaa", "last_edited_time": "2022-01-01T00:00:00.000Z"}], "next_cursor": None}, + ) assert source.check_connection(logger_mock, config_mock) == (True, None) +@pytest.mark.parametrize( + "status_code,json_response,expected_message", + [ + (401, {"code": "unauthorized"}, UNAUTHORIZED_ERROR_MESSAGE), + (403, {"code": "restricted_resource"}, RESTRICTED_RESOURCE_ERROR_MESSAGE), + (409, {"code": "conflict_error", "message": GENERIC_ERROR_MESSAGE}, f"Error: {GENERIC_ERROR_MESSAGE} (Error code: conflict_error)"), + (400, {}, f"Error: {DEFAULT_ERROR_MESSAGE} (Error code: unknown_error)"), + ], +) +def test_check_connection_errors(mocker, requests_mock, status_code, json_response, expected_message): + source = SourceNotion() + logger_mock, config_mock = MagicMock(), {"access_token": "test_token", "start_date": "2021-01-01T00:00:00.000Z"} + requests_mock.get("https://api.notion.com/v1/users/me", status_code=status_code, json=json_response) + result, message = source.check_connection(logger_mock, config_mock) + + assert result is False + assert message == expected_message + + def test_streams(mocker): source = SourceNotion() config_mock = MagicMock() streams = source.streams(config_mock) - expected_streams_number = 4 + expected_streams_number = 5 assert len(streams) == expected_streams_number + + +@pytest.mark.parametrize( + "config, expected_token", + [ + ({"credentials": {"auth_type": "OAuth2.0", "access_token": "oauth_token"}}, "Bearer oauth_token"), + ({"credentials": {"auth_type": "token", "token": "other_token"}}, "Bearer other_token"), + ({}, None), + ], +) +def test_get_authenticator(config, expected_token): + source = SourceNotion() + authenticator = source._get_authenticator(config) # Fixed line + + if expected_token: + assert isinstance(authenticator, TokenAuthenticator) + assert authenticator.token == expected_token # Replace with the actual way to access the token from the authenticator + else: + assert authenticator is None + + +@pytest.mark.parametrize( + "config, expected_return", + [ + ({}, None), + ({"start_date": "2021-01-01T00:00:00.000Z"}, None), + ({"start_date": "2021-99-99T79:89:99.123Z"}, "The provided start date is not a valid date. Please check the format and try again."), + ({"start_date": "2021-01-01T00:00:00.000"}, "Please check the format of the start date against the pattern descriptor."), + ({"start_date": "2025-01-25T00:00:00.000Z"}, "The start date cannot be greater than the current date."), + ], +) +def test_validate_start_date(config, expected_return): + source = SourceNotion() + result = source._validate_start_date(config) + assert result == expected_return diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py index 0a441050829d..ceeb5b1ca5cb 100644 --- a/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py @@ -2,14 +2,16 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging import random from http import HTTPStatus from unittest.mock import MagicMock +import freezegun import pytest import requests from airbyte_cdk.models import SyncMode -from source_notion.streams import Blocks, NotionStream, Users +from source_notion.streams import Blocks, NotionStream, Pages, Users @pytest.fixture @@ -35,6 +37,18 @@ def test_next_page_token(patch_base_class, requests_mock): assert stream.next_page_token(**inputs) == expected_token +@pytest.mark.parametrize( + "response_json, expected_output", + [({"next_cursor": "some_cursor", "has_more": True}, {"next_cursor": "some_cursor"}), ({"has_more": False}, None), ({}, None)], +) +def test_next_page_token_with_no_cursor(patch_base_class, response_json, expected_output): + stream = NotionStream(config=MagicMock()) + mock_response = MagicMock() + mock_response.json.return_value = response_json + result = stream.next_page_token(mock_response) + assert result == expected_output + + def test_parse_response(patch_base_class, requests_mock): stream = NotionStream(config=MagicMock()) requests_mock.get("https://dummy", json={"results": [{"a": 123}, {"b": "xx"}]}) @@ -64,6 +78,8 @@ def test_http_method(patch_base_class): (HTTPStatus.BAD_REQUEST, True), (HTTPStatus.TOO_MANY_REQUESTS, True), (HTTPStatus.INTERNAL_SERVER_ERROR, True), + (HTTPStatus.BAD_GATEWAY, True), + (HTTPStatus.FORBIDDEN, False), ], ) def test_should_retry(patch_base_class, http_status, should_retry): @@ -76,10 +92,10 @@ def test_should_retry(patch_base_class, http_status, should_retry): def test_should_not_retry_with_ai_block(requests_mock): stream = Blocks(parent=None, config=MagicMock()) json_response = { - "object":"error", - "status":400, - "code":"validation_error", - "message":"Block type ai_block is not supported via the API.", + "object": "error", + "status": 400, + "code": "validation_error", + "message": "Block type ai_block is not supported via the API.", } requests_mock.get("https://api.notion.com/v1/blocks/123", json=json_response, status_code=400) test_response = requests.get("https://api.notion.com/v1/blocks/123") @@ -110,10 +126,22 @@ def test_empty_blocks_results(requests_mock): assert list(stream.read_records(sync_mode=SyncMode.incremental, stream_slice=[])) == [] -def test_backoff_time(patch_base_class): - response_mock = MagicMock(headers={"retry-after": "10"}) +@pytest.mark.parametrize( + "status_code,retry_after_header,expected_backoff", + [ + (429, "10", 10.0), # Case for 429 error with retry-after header + (429, None, 5.0), # Case for 429 error without retry-after header, should default to 5.0 + (504, None, None), # Case for 500-level error, should default to None and use CDK exponential backoff + (400, None, 10.0), # Case for specific 400-level error handled by check_invalid_start_cursor + ], +) +def test_backoff_time(status_code, retry_after_header, expected_backoff, patch_base_class): + response_mock = MagicMock(spec=requests.Response) + response_mock.status_code = status_code + response_mock.headers = {"retry-after": retry_after_header} if retry_after_header else {} stream = NotionStream(config=MagicMock()) - assert stream.backoff_time(response_mock) == 10.0 + + assert stream.backoff_time(response_mock) == expected_backoff def test_users_request_params(patch_base_class): @@ -167,3 +195,120 @@ def test_user_stream_handles_pagination_correctly(requests_mock): records = stream.read_records(sync_mode=SyncMode.full_refresh) records_length = sum(1 for _ in records) assert records_length == 220 + + +@pytest.mark.parametrize( + "config, expected_start_date, current_time", + [ + ( + {"authenticator": "secret_token", "start_date": "2021-09-01T00:00:00.000Z"}, + "2021-09-01T00:00:00.000Z", + "2022-09-22T00:00:00.000Z", + ), + ({"authenticator": "super_secret_token", "start_date": None}, "2020-09-22T00:00:00.000Z", "2022-09-22T00:00:00.000Z"), + ({"authenticator": "even_more_secret_token"}, "2021-01-01T12:30:00.000Z", "2023-01-01T12:30:00.000Z"), + ], +) +def test_set_start_date(patch_base_class, config, expected_start_date, current_time): + """ + Test that start_date in config is either: + 1. set to the value provided by the user + 2. defaults to two years from the present date set by the test environment. + """ + with freezegun.freeze_time(current_time): + stream = NotionStream(config=config) + assert stream.start_date == expected_start_date + + +@pytest.mark.parametrize( + "stream,parent,url,status_code,response_content,expected_availability,expected_reason_substring", + [ + ( + Users, + None, + "https://api.notion.com/v1/users", + 403, + b'{"object": "error", "status": 403, "code": "restricted_resource"}', + False, + "This is likely due to insufficient permissions for your Notion integration.", + ), + ( + Blocks, + Pages, + "https://api.notion.com/v1/blocks/123/children", + 403, + b'{"object": "error", "status": 403, "code": "restricted_resource"}', + False, + "This is likely due to insufficient permissions for your Notion integration.", + ), + ( + Users, + None, + "https://api.notion.com/v1/users", + 200, + b'{"object": "list", "results": [{"id": "123", "object": "user", "type": "person"}]}', + True, + None, + ), + ], +) +def test_403_error_handling( + requests_mock, stream, parent, url, status_code, response_content, expected_availability, expected_reason_substring +): + """ + Test that availability strategy flags streams with 403 error as unavailable + and returns custom Notion integration message. + """ + + requests_mock.get(url=url, status_code=status_code, content=response_content) + + if parent: + stream = stream(parent=parent, config=MagicMock()) + stream.parent.stream_slices = MagicMock(return_value=[{"id": "123"}]) + stream.parent.read_records = MagicMock(return_value=[{"id": "123", "object": "page"}]) + else: + stream = stream(config=MagicMock()) + + is_available, reason = stream.check_availability(logger=logging.Logger, source=MagicMock()) + + assert is_available is expected_availability + + if expected_reason_substring: + assert expected_reason_substring in reason + else: + assert reason is None + + +@pytest.mark.parametrize( + "initial_page_size, expected_page_size, mock_response", + [ + (100, 50, {"status_code": 504, "json": {}, "headers": {"retry-after": "1"}}), + (50, 25, {"status_code": 504, "json": {}, "headers": {"retry-after": "1"}}), + (100, 100, {"status_code": 429, "json": {}, "headers": {"retry-after": "1"}}), + (50, 100, {"status_code": 200, "json": {"data": "success"}, "headers": {}}), + ], + ids=[ + "504 error, page_size 100 -> 50", + "504 error, page_size 50 -> 25", + "429 error, page_size 100 -> 100", + "200 success, page_size 50 -> 100", + ], +) +def test_request_throttle(initial_page_size, expected_page_size, mock_response, requests_mock): + """ + Tests that the request page_size is halved when a 504 error is encountered. + Once a 200 success is encountered, the page_size is reset to 100, for use in the next call. + """ + requests_mock.register_uri( + "GET", + "https://api.notion.com/v1/users", + [{"status_code": mock_response["status_code"], "json": mock_response["json"], "headers": mock_response["headers"]}], + ) + + stream = Users(config={"authenticator": "auth"}) + stream.page_size = initial_page_size + response = requests.get("https://api.notion.com/v1/users") + + stream.should_retry(response=response) + + assert stream.page_size == expected_page_size diff --git a/airbyte-integrations/connectors/source-nytimes/README.md b/airbyte-integrations/connectors/source-nytimes/README.md index ba817d58a47d..30cf638f2d8c 100644 --- a/airbyte-integrations/connectors/source-nytimes/README.md +++ b/airbyte-integrations/connectors/source-nytimes/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-nytimes:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/nytimes) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_nytimes/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-nytimes:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-nytimes build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-nytimes:airbyteDocker +An image will be built with the tag `airbyte/source-nytimes:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-nytimes:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-nytimes:dev check --co docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-nytimes:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-nytimes:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-nytimes test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-nytimes:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-nytimes:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-nytimes test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/nytimes.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-nytimes/acceptance-test-config.yml b/airbyte-integrations/connectors/source-nytimes/acceptance-test-config.yml index 1ff862137d56..27d0b6a9fb1e 100644 --- a/airbyte-integrations/connectors/source-nytimes/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-nytimes/acceptance-test-config.yml @@ -19,11 +19,10 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - incremental: + incremental: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - threshold_days: 31 future_state: future_state_path: "integration_tests/abnormal_state.json" full_refresh: diff --git a/airbyte-integrations/connectors/source-nytimes/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-nytimes/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-nytimes/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-nytimes/build.gradle b/airbyte-integrations/connectors/source-nytimes/build.gradle deleted file mode 100644 index bfa7662835c9..000000000000 --- a/airbyte-integrations/connectors/source-nytimes/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_nytimes' -} diff --git a/airbyte-integrations/connectors/source-okta/README.md b/airbyte-integrations/connectors/source-okta/README.md index 547f851b8f51..611b3c1f5f84 100644 --- a/airbyte-integrations/connectors/source-okta/README.md +++ b/airbyte-integrations/connectors/source-okta/README.md @@ -32,16 +32,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle - -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: - -```shell -./gradlew :airbyte-integrations:connectors:source-okta:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/okta) @@ -63,23 +53,20 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build - -First, make sure you build the latest Docker image: -```shell -docker build . -t airbyte/source-okta:dev +#### Build +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-okta build ``` -You can also build the connector image via Gradle: +An image will be built with the tag `airbyte/source-okta:dev`. -```shell -./gradlew :airbyte-integrations:connectors:source-okta:airbyteDocker +**Via `docker build`:** +```bash +docker build -t airbyte/source-okta:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. - #### Run Then run any of the connector commands as follows: @@ -91,62 +78,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-okta:dev discover --co docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-okta:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing - -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: - -```shell -pip install .'[tests]' -``` - -### Unit Tests - -To run unit tests locally, from the connector directory run: - -```shell -python -m pytest unit_tests -``` -### Integration Tests - -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). - -#### Custom Integration tests - -Place custom tests inside the `integration_tests``/` folder, then, from the connector root, run - -``` -python -m pytest integration_tests +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-okta test ``` -#### Acceptance Tests - -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run - -``` -docker build . --no-cache -t airbyte/source-okta:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` - -To run your integration tests with docker - -### Using gradle to run tests - -All commands should be run from airbyte project root. -To run unit tests: - -``` -./gradlew :airbyte-integrations:connectors:source-okta:unitTest -``` - -To run acceptance and custom integration tests: - -``` -./gradlew :airbyte-integrations:connectors:source-okta:integrationTest -``` ## Dependency Management @@ -157,11 +98,12 @@ We split dependencies between two groups, dependencies that are: * required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector - You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-okta test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/okta.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. -1. Make sure your changes are passing unit and integration tests. -2. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -3. Create a Pull Request. -4. Pat yourself on the back for being an awesome contributor. -5. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-okta/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-okta/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-okta/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-okta/build.gradle b/airbyte-integrations/connectors/source-okta/build.gradle deleted file mode 100644 index 87b8c263acfa..000000000000 --- a/airbyte-integrations/connectors/source-okta/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_okta' -} diff --git a/airbyte-integrations/connectors/source-okta/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-okta/unit_tests/test_streams.py index 3b8a09298211..2265d6ebf0e2 100644 --- a/airbyte-integrations/connectors/source-okta/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-okta/unit_tests/test_streams.py @@ -394,6 +394,6 @@ def test_resource_sets_next_page_token(self, requests_mock, patch_base_class, re def test_resource_sets_request_params(self, requests_mock, patch_base_class, resource_set_instance, url_base, api_url, start_date): stream = ResourceSets(url_base=url_base, start_date=start_date) cursor = "iam5cursorFybecursor" - inputs = {"stream_slice": None, "stream_state": {"id": cursor}, "next_page_token": {'after': cursor}} + inputs = {"stream_slice": None, "stream_state": {"id": cursor}, "next_page_token": {"after": cursor}} expected_params = {"limit": 200, "after": "iam5cursorFybecursor"} assert stream.request_params(**inputs) == expected_params diff --git a/airbyte-integrations/connectors/source-omnisend/README.md b/airbyte-integrations/connectors/source-omnisend/README.md index ee2159fb3627..4bfe23ff0473 100644 --- a/airbyte-integrations/connectors/source-omnisend/README.md +++ b/airbyte-integrations/connectors/source-omnisend/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-omnisend:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/omnisend) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_omnisend/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-omnisend:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-omnisend build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-omnisend:airbyteDocker +An image will be built with the tag `airbyte/source-omnisend:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-omnisend:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-omnisend:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-omnisend:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-omnisend:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-omnisend test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-omnisend:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-omnisend:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-omnisend test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/omnisend.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-omnisend/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-omnisend/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-omnisend/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-omnisend/build.gradle b/airbyte-integrations/connectors/source-omnisend/build.gradle deleted file mode 100644 index 1b49d7f33314..000000000000 --- a/airbyte-integrations/connectors/source-omnisend/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_omnisend' -} diff --git a/airbyte-integrations/connectors/source-onesignal/Dockerfile b/airbyte-integrations/connectors/source-onesignal/Dockerfile index 43a620a289db..8b8a74478146 100644 --- a/airbyte-integrations/connectors/source-onesignal/Dockerfile +++ b/airbyte-integrations/connectors/source-onesignal/Dockerfile @@ -34,5 +34,5 @@ COPY source_onesignal ./source_onesignal ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.1 +LABEL io.airbyte.version=1.1.0 LABEL io.airbyte.name=airbyte/source-onesignal diff --git a/airbyte-integrations/connectors/source-onesignal/README.md b/airbyte-integrations/connectors/source-onesignal/README.md index 6e6fc1ed465f..86c23f85b8a4 100644 --- a/airbyte-integrations/connectors/source-onesignal/README.md +++ b/airbyte-integrations/connectors/source-onesignal/README.md @@ -1,74 +1,34 @@ # Onesignal Source -This is the repository for the Onesignal source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/onesignal). +This is the repository for the Onesignal configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/onesignal). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-onesignal:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/onesignal) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_onesignal/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/onesignal) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_onesignal/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source onesignal test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-onesignal:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-onesignal build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-onesignal:airbyteDocker +An image will be built with the tag `airbyte/source-onesignal:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-onesignal:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-onesignal:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-onesignal:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-onesignal:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-onesignal test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-onesignal:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-onesignal:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-onesignal test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/onesignal.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-onesignal/__init__.py b/airbyte-integrations/connectors/source-onesignal/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-onesignal/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-onesignal/acceptance-test-config.yml b/airbyte-integrations/connectors/source-onesignal/acceptance-test-config.yml index 5fcf23851e12..8b4e4c7d5270 100644 --- a/airbyte-integrations/connectors/source-onesignal/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-onesignal/acceptance-test-config.yml @@ -1,29 +1,42 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests connector_image: airbyte/source-onesignal:dev -tests: +acceptance_tests: spec: - - spec_path: "source_onesignal/spec.json" - backward_compatibility_tests_config: - disable_for_version: "0.1.2" + tests: + - spec_path: "source_onesignal/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" - backward_compatibility_tests_config: - disable_for_version: "0.1.2" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: ["devices", "notifications"] -# TODO: enable incremental test after seeding the connector -# incremental: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: notifications + bypass_reason: "Sandbox account cannot seed the stream" + - name: outcomes + bypass_reason: "Sandbox account cannot seed the stream" + - name: devices + bypass_reason: "Sandbox account cannot seed the stream" + expect_records: + path: "integration_tests/expected_records.jsonl" + extra_fields: no + exact_order: no + extra_records: yes + fail_on_extra_columns: false + # TODO: enable incremental test after seeding the connector + # incremental: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-onesignal/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-onesignal/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-onesignal/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-onesignal/bootstrap.md b/airbyte-integrations/connectors/source-onesignal/bootstrap.md deleted file mode 100644 index 64b2f230b651..000000000000 --- a/airbyte-integrations/connectors/source-onesignal/bootstrap.md +++ /dev/null @@ -1,28 +0,0 @@ -# OneSignal - -## Overview - -OneSignal is a customer messaging and engagement platform that allows businesses to create meaningful customer connections. OneSignal REST API allows a developer to retrieve audience and messaging information on the OneSignal platform. - -## Endpoints - -OneSignal API consists of four endpoints which can be extracted data from: - -1. **App**: The collection of audience and messaging channels. -2. **Device**: A customer's device which can send message to, it is associated with app. -3. **Notification**: A messaging activity associated with app. -4. **Outcome**: Aggregated information associated with app, for example, session duration, number of clicks, etc. - -## Quick Notes - -- Each app has its own authentication key to retrieve its devices, notifications and outcomes. The key can be found in the app's endpoint response. - -- Device and notification endpoint has 300 and 50 records limit per request respectively, so the cursor pagination strategy is used for them. - -- Rate limiting follows [https://documentation.onesignal.com/docs/rate-limits](https://documentation.onesignal.com/docs/rate-limits), when a 429 HTTP status code returned. - -- For the outcome endpoint, it needs to specify a comma-separated list of names and the value (sum/count) for the returned outcome data. So this requirement is added to the source spec. - -## API Reference - -The API reference documents: [https://documentation.onesignal.com/reference](https://documentation.onesignal.com/reference) diff --git a/airbyte-integrations/connectors/source-onesignal/build.gradle b/airbyte-integrations/connectors/source-onesignal/build.gradle deleted file mode 100644 index abeb972de4e2..000000000000 --- a/airbyte-integrations/connectors/source-onesignal/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_onesignal' -} diff --git a/airbyte-integrations/connectors/source-onesignal/integration_tests/__init__.py b/airbyte-integrations/connectors/source-onesignal/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-onesignal/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-onesignal/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-onesignal/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-onesignal/integration_tests/abnormal_state.json index b57dff4d6232..52f70ab786ea 100644 --- a/airbyte-integrations/connectors/source-onesignal/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-onesignal/integration_tests/abnormal_state.json @@ -3,7 +3,7 @@ "type": "STREAM", "stream": { "stream_state": { - "8d466489-38af-4067-a6b8-2645ad83f4c2": { "last_active": 2598826148 } + "last_active": "2598826148" }, "stream_descriptor": { "name": "devices" } } @@ -12,7 +12,7 @@ "type": "STREAM", "stream": { "stream_state": { - "8d466489-38af-4067-a6b8-2645ad83f4c2": { "last_active": 2598826148 } + "queued_at": "2598826148" }, "stream_descriptor": { "name": "notifications" } } diff --git a/airbyte-integrations/connectors/source-onesignal/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-onesignal/integration_tests/expected_records.jsonl new file mode 100644 index 000000000000..027b3d58bd06 --- /dev/null +++ b/airbyte-integrations/connectors/source-onesignal/integration_tests/expected_records.jsonl @@ -0,0 +1 @@ +{"stream": "apps", "data": {"id": "8d466489-38af-4067-a6b8-2645ad83f4c2", "name": "Airbyte_test", "gcm_key": null, "fcm_v1_service_account_json": null, "chrome_key": null, "chrome_web_key": null, "chrome_web_origin": null, "chrome_web_gcm_sender_id": null, "chrome_web_default_notification_icon": null, "chrome_web_sub_domain": null, "apns_env": null, "apns_certificates": null, "apns_p8": null, "apns_team_id": null, "apns_key_id": null, "apns_bundle_id": null, "safari_apns_certificate": null, "safari_site_origin": null, "safari_push_id": null, "safari_icon_16_16": "public/safari_packages/8d466489-38af-4067-a6b8-2645ad83f4c2/icons/16x16.png", "safari_icon_32_32": "public/safari_packages/8d466489-38af-4067-a6b8-2645ad83f4c2/icons/16x16@2x.png", "safari_icon_64_64": "public/safari_packages/8d466489-38af-4067-a6b8-2645ad83f4c2/icons/32x32@2x.png", "safari_icon_128_128": "public/safari_packages/8d466489-38af-4067-a6b8-2645ad83f4c2/icons/128x128.png", "safari_icon_256_256": "public/safari_packages/8d466489-38af-4067-a6b8-2645ad83f4c2/icons/128x128@2x.png", "site_name": null, "created_at": "2023-03-13T18:31:21.935Z", "updated_at": "2023-03-13T18:31:22.249Z", "players": 0, "messageable_players": 0, "basic_auth_key": null, "additional_data_is_root_payload": false, "organization_id": null}, "emitted_at": 1693501393086} diff --git a/airbyte-integrations/connectors/source-onesignal/metadata.yaml b/airbyte-integrations/connectors/source-onesignal/metadata.yaml index 26a4f8e5051d..cedb38e32be9 100644 --- a/airbyte-integrations/connectors/source-onesignal/metadata.yaml +++ b/airbyte-integrations/connectors/source-onesignal/metadata.yaml @@ -1,24 +1,25 @@ data: - ab_internal: - ql: 200 - sl: 100 + allowedHosts: + hosts: + - "onesignal.com" + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: bb6afd81-87d5-47e3-97c4-e2c2901b1cf8 - dockerImageTag: 1.0.1 + dockerImageTag: 1.1.0 dockerRepository: airbyte/source-onesignal documentationUrl: https://docs.airbyte.com/integrations/sources/onesignal githubIssueLabel: source-onesignal icon: onesignal.svg license: MIT name: OneSignal - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: 2023-08-31 releaseStage: alpha - supportLevel: community tags: - - language:python + - language:low-code + supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-onesignal/requirements.txt b/airbyte-integrations/connectors/source-onesignal/requirements.txt index d6e1198b1ab1..cc57334ef619 100644 --- a/airbyte-integrations/connectors/source-onesignal/requirements.txt +++ b/airbyte-integrations/connectors/source-onesignal/requirements.txt @@ -1 +1,2 @@ +-e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-onesignal/setup.py b/airbyte-integrations/connectors/source-onesignal/setup.py index f0f2b7829a99..47af71852c6a 100644 --- a/airbyte-integrations/connectors/source-onesignal/setup.py +++ b/airbyte-integrations/connectors/source-onesignal/setup.py @@ -6,14 +6,10 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", + "airbyte-cdk~=0.1", ] -TEST_REQUIREMENTS = [ - "pytest~=6.1", - "pytest-mock~=3.6.1", - "requests-mock", -] +TEST_REQUIREMENTS = ["pytest~=6.2", "pytest-mock~=3.6.1", "connector-acceptance-test"] setup( name="source_onesignal", @@ -22,7 +18,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-onesignal/source_onesignal/__init__.py b/airbyte-integrations/connectors/source-onesignal/source_onesignal/__init__.py index 4a3b71533fd6..ac064edbc409 100644 --- a/airbyte-integrations/connectors/source-onesignal/source_onesignal/__init__.py +++ b/airbyte-integrations/connectors/source-onesignal/source_onesignal/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-onesignal/source_onesignal/manifest.yaml b/airbyte-integrations/connectors/source-onesignal/source_onesignal/manifest.yaml new file mode 100644 index 000000000000..aef1c029398e --- /dev/null +++ b/airbyte-integrations/connectors/source-onesignal/source_onesignal/manifest.yaml @@ -0,0 +1,142 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["{{ parameters.extractorPath }}"] + + user_auth_requester: + type: HttpRequester + url_base: "https://onesignal.com/api/v1/" + http_method: "GET" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['user_auth_key'] }}" + request_parameters: + app_id: "{{ config['applications'][0]['app_id'] }}" + + rest_api_requester: + type: HttpRequester + url_base: "https://onesignal.com/api/v1" + http_method: "GET" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['applications'][0]['app_api_key'] }}" + request_parameters: + app_id: "{{ config['applications'][0]['app_id'] }}" + + retriever_with_pagination: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: "DefaultPaginator" + page_size_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "limit" + pagination_strategy: + type: "OffsetIncrement" + page_size: 5 + page_token_option: + type: "RequestOption" + field_name: "offset" + inject_into: "request_parameter" + requester: + $ref: "#/definitions/rest_api_requester" + + user_auth_base_stream: + type: DeclarativeStream + retriever: + type: SimpleRetriever + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: NoPagination + requester: + $ref: "#/definitions/user_auth_requester" + + rest_api_key_auth_base_stream: + type: DeclarativeStream + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: NoPagination + requester: + $ref: "#/definitions/rest_api_requester" + + apps_stream: + $ref: "#/definitions/user_auth_base_stream" + name: "apps" + primary_key: "id" + $parameters: + path: "apps" + + incremental_sync_base: + type: DatetimeBasedCursor + cursor_field: "{{ parameters.incremental_cursor }}" + datetime_format: "%s" + cursor_granularity: "PT0.000001S" + lookback_window: "P31D" + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + end_datetime: + datetime: "{{ today_utc() }}" + datetime_format: "%Y-%m-%d" + step: "P1M" + + devices_stream: + $ref: "#/definitions/rest_api_key_auth_base_stream" + $parameters: + name: "devices" + primary_key: "id" + extractorPath: "players" + incremental_cursor: "last_active" + path: "players" + incremental_sync: + $ref: "#/definitions/incremental_sync_base" + retriever: + $ref: "#/definitions/retriever_with_pagination" + + notifications_stream: + $ref: "#/definitions/rest_api_key_auth_base_stream" + $parameters: + name: "notifications" + primary_key: "id" + extractorPath: "notifications" + incremental_cursor: "queued_at" + path: "notifications" + incremental_sync: + $ref: "#/definitions/incremental_sync_base" + retriever: + $ref: "#/definitions/retriever_with_pagination" + + outcomes_stream: + $ref: "#/definitions/rest_api_key_auth_base_stream" + name: "outcomes" + primary_key: "id" + $parameters: + extractorPath: "outcomes" + path: "apps/{{ config['applications'][0]['app_id'] }}/outcomes" + +streams: + - "#/definitions/apps_stream" + - "#/definitions/devices_stream" + - "#/definitions/notifications_stream" + - "#/definitions/outcomes_stream" + +check: + type: CheckStream + stream_names: + - "apps" + - "devices" + - "notifications" + - "outcomes" diff --git a/airbyte-integrations/connectors/source-onesignal/source_onesignal/schemas/apps.json b/airbyte-integrations/connectors/source-onesignal/source_onesignal/schemas/apps.json index 8d66d7546299..e69c6d865c2d 100644 --- a/airbyte-integrations/connectors/source-onesignal/source_onesignal/schemas/apps.json +++ b/airbyte-integrations/connectors/source-onesignal/source_onesignal/schemas/apps.json @@ -84,19 +84,19 @@ "type": ["null", "boolean"] }, "apns_key_id": { - "type": ["null", "string"] - }, - "apns_p8": { - "type": ["null", "string"] + "type": ["null", "integer"] }, - "apns_team_id": { + "fcm_v1_service_account_json": { "type": ["null", "string"] }, - "fcm_v1_service_account_json": { + "apns_p8": { "type": ["null", "string"] }, "apns_bundle_id": { - "type": ["null", "string"] + "type": ["null", "integer"] + }, + "apns_team_id": { + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-onesignal/source_onesignal/source.py b/airbyte-integrations/connectors/source-onesignal/source_onesignal/source.py index ef4b9fd280d1..c203c5f38c30 100644 --- a/airbyte-integrations/connectors/source-onesignal/source_onesignal/source.py +++ b/airbyte-integrations/connectors/source-onesignal/source_onesignal/source.py @@ -2,37 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from typing import Any, List, Mapping, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import requests -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +WARNING: Do not modify this file. +""" -from .streams import Apps, Devices, Notifications, Outcomes - -class SourceOnesignal(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - try: - authenticator = TokenAuthenticator(config["user_auth_key"], "Basic") - stream = Apps(authenticator=authenticator, config=config) - records = stream.read_records(sync_mode=SyncMode.full_refresh) - next(records) - return True, None - except requests.exceptions.RequestException as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - AirbyteLogger().log("INFO", f"Using start_date: {config['start_date']}") - - authenticator = TokenAuthenticator(config["user_auth_key"], "Basic") - args = {"authenticator": authenticator, "config": config} - streams = [Apps(**args)] - if config.get("applications"): - streams += Devices(**args), Notifications(**args), Outcomes(**args) - - return streams +# Declarative Source +class SourceOnesignal(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-onesignal/source_onesignal/spec.json b/airbyte-integrations/connectors/source-onesignal/source_onesignal/spec.json deleted file mode 100644 index 1cf4e4f21bac..000000000000 --- a/airbyte-integrations/connectors/source-onesignal/source_onesignal/spec.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/onesignal", - "connectionSpecification": { - "$schema": "https://json-schema.org/draft-07/schema#", - "title": "OneSignal Source Spec", - "type": "object", - "required": [ - "user_auth_key", - "start_date", - "outcome_names", - "applications" - ], - "additionalProperties": true, - "properties": { - "user_auth_key": { - "type": "string", - "title": "User Auth Key", - "description": "OneSignal User Auth Key, see the docs for more information on how to obtain this key.", - "airbyte_secret": true, - "order": 0 - }, - "applications": { - "type": "array", - "title": "Applications", - "description": "Applications keys, see the docs for more information on how to obtain this data", - "items": { - "type": "object", - "properties": { - "app_name": { - "type": "string", - "title": "OneSignal App Name", - "order": 0 - }, - "app_id": { - "type": "string", - "title": "OneSignal App ID", - "order": 1, - "airbyte_secret": true - }, - "app_api_key": { - "type": "string", - "title": "REST API Key", - "order": 2, - "airbyte_secret": true - } - }, - "required": ["app_id", "app_api_key"] - }, - "order": 1 - }, - "start_date": { - "type": "string", - "title": "Start Date", - "description": "The date from which you'd like to replicate data for OneSignal API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", - "examples": ["2020-11-16T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "format": "date-time", - "order": 2 - }, - "outcome_names": { - "type": "string", - "title": "Outcome Names", - "description": "Comma-separated list of names and the value (sum/count) for the returned outcome data. See the docs for more details", - "examples": [ - "os__session_duration.count,os__click.count,CustomOutcomeName.sum" - ], - "order": 3 - } - } - } -} diff --git a/airbyte-integrations/connectors/source-onesignal/source_onesignal/spec.yaml b/airbyte-integrations/connectors/source-onesignal/source_onesignal/spec.yaml new file mode 100644 index 000000000000..889703c95da1 --- /dev/null +++ b/airbyte-integrations/connectors/source-onesignal/source_onesignal/spec.yaml @@ -0,0 +1,69 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/onesignal +connectionSpecification: + $schema: https://json-schema.org/draft-07/schema# + title: OneSignal Source Spec + type: object + required: + - user_auth_key + - start_date + - outcome_names + - applications + additionalProperties: true + properties: + user_auth_key: + type: string + title: User Auth Key + description: + OneSignal User Auth Key, see the docs + for more information on how to obtain this key. + airbyte_secret: true + order: 0 + applications: + type: array + title: Applications + description: + Applications keys, see the docs + for more information on how to obtain this data + items: + type: object + properties: + app_name: + type: string + title: OneSignal App Name + order: 0 + app_id: + type: string + title: OneSignal App ID + order: 1 + airbyte_secret: true + app_api_key: + type: string + title: REST API Key + order: 2 + airbyte_secret: true + required: + - app_id + - app_api_key + order: 1 + start_date: + type: string + title: Start Date + description: + The date from which you'd like to replicate data for OneSignal + API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date + will be replicated. + examples: + - "2020-11-16T00:00:00Z" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + format: date-time + order: 2 + outcome_names: + type: string + title: Outcome Names + description: + Comma-separated list of names and the value (sum/count) for the + returned outcome data. See the docs + for more details + examples: + - os__session_duration.count,os__click.count,CustomOutcomeName.sum + order: 3 diff --git a/airbyte-integrations/connectors/source-onesignal/source_onesignal/streams.py b/airbyte-integrations/connectors/source-onesignal/source_onesignal/streams.py deleted file mode 100644 index 6ae8a7a04f15..000000000000 --- a/airbyte-integrations/connectors/source-onesignal/source_onesignal/streams.py +++ /dev/null @@ -1,209 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import logging -import time -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple - -import pendulum -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import Source -from airbyte_cdk.sources.streams import IncrementalMixin -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator - - -class OnesignalStream(HttpStream, ABC): - url_base = "https://onesignal.com/api/v1/" - - primary_key = "id" - - def __init__(self, config: Mapping[str, Any], **kwargs): - super().__init__(**kwargs) - self.applications = config.get("applications") - - # OneSignal uses epoch timestamp, so we need to convert the start_date - # config to epoch timestamp too. - # start_date example: 2021-01-01T00:00:00Z - self.start_date = pendulum.parse(config["start_date"]).int_timestamp - - def next_page_token( - self, - response: requests.Response, - ) -> Optional[Mapping[str, Any]]: - return None - - def backoff_time(self, response: requests.Response) -> Optional[float]: - """ - OneSignal's API rates limit is 1 request per second with a 10/second burst. - RateLimit* headers will indicate how much quota is left at the time of - the request. For example: - - ratelimit-limit: 10 - ratelimit-remaining: 9 - ratelimit-reset: 1633654403 - - Docs: https://documentation.onesignal.com/docs/rate-limits - """ - reset_time = response.headers.get("ratelimit-reset") - backoff_time = float(reset_time) - time.time() if reset_time else 60 - if backoff_time < 0: - backoff_time = 60 - return backoff_time - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - data = response.json() - yield from data - - def check_availability(self, logger: logging.Logger, source: Optional["Source"] = None) -> Tuple[bool, Optional[str]]: - return True, None - - -class AppSlicesStream(OnesignalStream): - def stream_slices( - self, - sync_mode: SyncMode, - **kwargs, - ) -> Iterable[Optional[Mapping[str, Any]]]: - yield from self.applications - - # default record filter, do nothing - def filter_by_state(self, **kwargs) -> bool: - return True - - def parse_response( - self, response: requests.Response, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any], **kwargs - ) -> Iterable[Mapping]: - data = response.json().get(self.data_field) - for record in data: - if self.filter_by_state(stream_state=stream_state, record=record, stream_slice=stream_slice): - yield record - - def read_records( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Mapping[str, Any]]: - token = stream_slice.get("app_api_key") - self._session.auth = TokenAuthenticator(token, "Basic") - - return super().read_records(sync_mode, cursor_field, stream_slice, stream_state) - - -class IncrementalOnesignalStream(AppSlicesStream, IncrementalMixin, ABC): - _state = {} - cursor_field = "updated_at" - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state, stream_slice, next_page_token) - params["app_id"] = stream_slice["app_id"] - params["limit"] = self.page_size - if next_page_token: - params["offset"] = next_page_token["offset"] - return params - - def next_page_token( - self, - response: requests.Response, - ) -> Optional[Mapping[str, Any]]: - """ - An example of response is: - { - "total_count": 553, - "offset": 0, - "limit": 1, - "notifications": [ ... ] - } - """ - resp = response.json() - total = resp["total_count"] - next_offset = resp["offset"] + resp["limit"] - if next_offset < total: - return {"offset": next_offset} - - def filter_by_state( - self, stream_state: Mapping[str, Any] = None, record: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None - ) -> bool: - app_id = stream_slice.get("app_id") - record_value = record.get(self.cursor_field, 0) - cursor_value = max(stream_state.get(app_id, {}).get(self.cursor_field, 0), record_value) - self.state = {app_id: {self.cursor_field: cursor_value}} - return not stream_state or stream_state.get(app_id, {}).get(self.cursor_field, 0) < record_value - - @property - def state(self) -> MutableMapping[str, Any]: - return self._state - - @state.setter - def state(self, value: MutableMapping[str, Any]): - self._state.update(value) - - -class Apps(OnesignalStream): - """ - Docs: https://documentation.onesignal.com/reference/view-apps-apps - """ - - def path(self, **kwargs) -> str: - return self.name - - -class Devices(IncrementalOnesignalStream): - """ - Docs: https://documentation.onesignal.com/reference/view-devices - """ - - cursor_field = "last_active" - data_field = "players" - page_size = 300 # page size limit set by OneSignal - - def path(self, **kwargs) -> str: - return "players" - - -class Notifications(IncrementalOnesignalStream): - """ - Docs: https://documentation.onesignal.com/reference/view-notifications - """ - - cursor_field = "queued_at" - data_field = "notifications" - page_size = 50 # page size limit set by OneSignal - - def path(self, **kwargs) -> str: - return self.name - - -class Outcomes(AppSlicesStream): - """ - Docs: https://documentation.onesignal.com/reference/view-outcomes - """ - - data_field = "outcomes" - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.outcome_names = kwargs["config"]["outcome_names"] - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return f"apps/{stream_slice['app_id']}/outcomes" - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state, stream_slice, next_page_token) - params["outcome_names"] = self.outcome_names - return params diff --git a/airbyte-integrations/connectors/source-onesignal/unit_tests/__init__.py b/airbyte-integrations/connectors/source-onesignal/unit_tests/__init__.py deleted file mode 100644 index 46b7376756ec..000000000000 --- a/airbyte-integrations/connectors/source-onesignal/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-onesignal/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-onesignal/unit_tests/test_incremental_streams.py deleted file mode 100644 index af71e50e276d..000000000000 --- a/airbyte-integrations/connectors/source-onesignal/unit_tests/test_incremental_streams.py +++ /dev/null @@ -1,112 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import requests -from airbyte_cdk.models import SyncMode -from pytest import fixture -from source_onesignal.streams import Devices, IncrementalOnesignalStream, Notifications - - -@fixture -def patch_incremental_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(IncrementalOnesignalStream, "path", "v0/example_endpoint") - mocker.patch.object(IncrementalOnesignalStream, "primary_key", "test_primary_key") - mocker.patch.object(IncrementalOnesignalStream, "__abstractmethods__", set()) - - -@fixture -def args(): - return {"authenticator": None, - "config": {"user_auth_key": "", - "start_date": "2021-01-01T00:00:00Z", - "outcome_names": "", - "applications": [ - {"app_id": "fake_id", - "app_api_key": "fake_api_key"} - ] - } - } - - -@fixture -def stream(patch_incremental_base_class, args): - return IncrementalOnesignalStream(**args) - - -def test_cursor_field(stream): - expected_cursor_field = "updated_at" - assert stream.cursor_field == expected_cursor_field - - -def test_stream_slices(stream, requests_mock): - expected_stream_slice = [{'app_api_key': 'fake_api_key', 'app_id': 'fake_id'}] - assert list(stream.stream_slices(sync_mode=SyncMode.full_refresh)) == expected_stream_slice - - -def test_supports_incremental(patch_incremental_base_class, mocker, args): - mocker.patch.object(IncrementalOnesignalStream, "cursor_field", "dummy_field") - stream = IncrementalOnesignalStream(**args) - assert stream.supports_incremental - - -def test_source_defined_cursor(stream): - assert stream.source_defined_cursor - - -def test_stream_checkpoint_interval(stream): - expected_checkpoint_interval = None - assert stream.state_checkpoint_interval == expected_checkpoint_interval - - -def test_next_page_token(stream, requests_mock): - requests_mock.get("https://dummy", json={"total_count": 123, "offset": 22, "limit": 33}) - resp = requests.get("https://dummy") - expected_next_page_token = {"offset": 55} - assert stream.next_page_token(resp) == expected_next_page_token - - requests_mock.get("https://dummy", json={"total_count": 123, "offset": 100, "limit": 33}) - resp = requests.get("https://dummy") - expected_next_page_token = None - assert stream.next_page_token(resp) == expected_next_page_token - - -def test_request_params(stream, args): - inputs = {"stream_state": {}, "stream_slice": {"app_id": "abc"}} - inputs2 = {"stream_state": {}, "stream_slice": {"app_id": "abc"}, "next_page_token": {"offset": 42}} - expected_request_params = {"app_id": "abc", "limit": None} - expected_request_params2 = {"app_id": "abc", "limit": None, "offset": 42} - - assert stream.request_params(**inputs) == expected_request_params - assert stream.request_params(**inputs2) == expected_request_params2 - - stream2 = Devices(**args) - expected_request_params["limit"] = 300 - expected_request_params2["limit"] = 300 - assert stream2.request_params(**inputs) == expected_request_params - assert stream2.request_params(**inputs2) == expected_request_params2 - - stream3 = Notifications(**args) - expected_request_params["limit"] = 50 - expected_request_params2["limit"] = 50 - assert stream3.request_params(**inputs) == expected_request_params - assert stream3.request_params(**inputs2) == expected_request_params2 - - -def test_filter_by_state(stream): - inputs = { - "stream_state": {"fake_id": {"updated_at": 100}}, - "record": {"updated_at": 200}, - "stream_slice": {'app_id': "fake_id"} - } - expected_filter_by_state = True - assert stream.filter_by_state(**inputs) == expected_filter_by_state - - inputs = { - "stream_state": {"fake_id": {"updated_at": 200}}, - "record": {"updated_at": 100}, - "stream_slice": {'app_id': "fake_id"} - } - expected_filter_by_state = False - assert stream.filter_by_state(**inputs) == expected_filter_by_state diff --git a/airbyte-integrations/connectors/source-onesignal/unit_tests/test_source.py b/airbyte-integrations/connectors/source-onesignal/unit_tests/test_source.py deleted file mode 100644 index f6a722f59b15..000000000000 --- a/airbyte-integrations/connectors/source-onesignal/unit_tests/test_source.py +++ /dev/null @@ -1,35 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -from pytest import fixture -from source_onesignal.source import SourceOnesignal - - -@fixture -def config(): - return {"config": {"user_auth_key": "", "start_date": "2021-01-01T00:00:00Z", "outcome_names": ""}} - - -def test_check_connection(mocker, requests_mock, config): - source = SourceOnesignal() - logger_mock = MagicMock() - requests_mock.get( - "https://onesignal.com/api/v1/apps", - json=[ - { - "id": "92911750-242d-4260-9e00-9d9034f139ce", - "basic_auth_key": "your key", - } - ], - ) - assert source.check_connection(logger_mock, **config) == (True, None) - - -def test_streams(mocker, config): - source = SourceOnesignal() - streams = source.streams(**config) - expected_streams_number = 1 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-onesignal/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-onesignal/unit_tests/test_streams.py deleted file mode 100644 index 6013acc14f69..000000000000 --- a/airbyte-integrations/connectors/source-onesignal/unit_tests/test_streams.py +++ /dev/null @@ -1,71 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -import requests -from source_onesignal.streams import OnesignalStream - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(OnesignalStream, "path", "v0/example_endpoint") - mocker.patch.object(OnesignalStream, "primary_key", "test_primary_key") - mocker.patch.object(OnesignalStream, "__abstractmethods__", set()) - - -@pytest.fixture -def stream(patch_base_class): - args = {"authenticator": None, "config": {"user_auth_key": "", "start_date": "2021-01-01T00:00:00Z", "outcome_names": ""}} - return OnesignalStream(**args) - - -def test_next_page_token(stream): - inputs = {"response": MagicMock()} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_parse_response(stream, requests_mock): - requests_mock.get("https://dummy", json=[{"id": 123, "basic_auth_key": "xx"}]) - resp = requests.get("https://dummy") - - inputs = {"response": resp, "stream_state": MagicMock()} - expected_parsed_object = {"id": 123, "basic_auth_key": "xx"} - assert next(stream.parse_response(**inputs)) == expected_parsed_object - - -def test_request_headers(stream): - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(stream): - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(stream, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(stream): - response_mock = MagicMock() - expected_backoff_time = 60 - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/Dockerfile b/airbyte-integrations/connectors/source-open-exchange-rates/Dockerfile index c21604d3beef..fe66fbab6753 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/Dockerfile +++ b/airbyte-integrations/connectors/source-open-exchange-rates/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.13-alpine3.15 as base +FROM python:3.9.11-alpine3.15 as base # build and load all requirements FROM base as builder @@ -34,5 +34,5 @@ COPY source_open_exchange_rates ./source_open_exchange_rates ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-open-exchange-rates diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/README.md b/airbyte-integrations/connectors/source-open-exchange-rates/README.md index 964528311341..9b2b5342d226 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/README.md +++ b/airbyte-integrations/connectors/source-open-exchange-rates/README.md @@ -1,45 +1,12 @@ # Open Exchange Rates Source -This is the repository for the Open Exchange Rates source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/open-exchange-rates). +This is the repository for the Open Exchange Rates configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/open-exchange-rates). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-open-exchange-rates:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/open-exchange-rates) +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/open-exchange-rates) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_open_exchange_rates/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. @@ -47,28 +14,21 @@ See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source open-exchange-rates test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-open-exchange-rates:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-open-exchange-rates build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-open-exchange-rates:airbyteDocker +An image will be built with the tag `airbyte/source-open-exchange-rates:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-open-exchange-rates:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-open-exchange-rates:de docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-open-exchange-rates:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-open-exchange-rates:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-open-exchange-rates test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-open-exchange-rates:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-open-exchange-rates:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-open-exchange-rates test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/open-exchange-rates.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/__init__.py b/airbyte-integrations/connectors/source-open-exchange-rates/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-open-exchange-rates/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/acceptance-test-config.yml b/airbyte-integrations/connectors/source-open-exchange-rates/acceptance-test-config.yml index 9135f16903f5..483fe518fa28 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-open-exchange-rates/acceptance-test-config.yml @@ -1,5 +1,7 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests connector_image: airbyte/source-open-exchange-rates:dev - +test_strictness_level: low acceptance_tests: spec: tests: @@ -18,9 +20,19 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] + # expect_records: + # path: "integration_tests/expected_records.jsonl" + # extra_fields: no + # exact_order: no + # extra_records: yes incremental: + # bypass_reason: "This connector does not implement incremental sync" tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" future_state: future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-open-exchange-rates/acceptance-test-docker.sh deleted file mode 100755 index c51577d10690..000000000000 --- a/airbyte-integrations/connectors/source-open-exchange-rates/acceptance-test-docker.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env sh - -# Build latest connector image -docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) - -# Pull latest acctest image -docker pull airbyte/source-acceptance-test:latest - -# Run -docker run --rm -it \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /tmp:/tmp \ - -v $(pwd):/test_input \ - airbyte/source-acceptance-test \ - --acceptance-test-config /test_input - diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/build.gradle b/airbyte-integrations/connectors/source-open-exchange-rates/build.gradle deleted file mode 100644 index 4af9b6e02606..000000000000 --- a/airbyte-integrations/connectors/source-open-exchange-rates/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_open_exchange_rates' -} diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/__init__.py b/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/__init__.py index 1100c1c58cf5..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/abnormal_state.json index 8f84fbf71fca..8d8a421ddc4b 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/abnormal_state.json @@ -1,5 +1,9 @@ -{ - "open_exchange_rates": { - "timestamp": 2052084644 +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "timestamp": 3052084644 }, + "stream_descriptor": { "name": "open_exchange_rates" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/acceptance.py index d49b55882333..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/acceptance.py @@ -10,4 +10,7 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/configured_catalog.json index 5bc2570a65a6..40c44b9de3be 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/configured_catalog.json @@ -3,539 +3,14 @@ { "stream": { "name": "open_exchange_rates", - "json_schema": { - "type": "object", - "properties": { - "base": { - "type": "string" - }, - "timestamp": { - "type": "integer" - }, - "rates": { - "type": "object", - "properties": { - "AED": { - "type": ["null", "number"] - }, - "AFN": { - "type": ["null", "number"] - }, - "ALL": { - "type": ["null", "number"] - }, - "AMD": { - "type": ["null", "number"] - }, - "ANG": { - "type": ["null", "number"] - }, - "AOA": { - "type": ["null", "number"] - }, - "ARS": { - "type": ["null", "number"] - }, - "AUD": { - "type": ["null", "number"] - }, - "AWG": { - "type": ["null", "number"] - }, - "AZN": { - "type": ["null", "number"] - }, - "BAM": { - "type": ["null", "number"] - }, - "BBD": { - "type": ["null", "number"] - }, - "BDT": { - "type": ["null", "number"] - }, - "BGN": { - "type": ["null", "number"] - }, - "BHD": { - "type": ["null", "number"] - }, - "BIF": { - "type": ["null", "number"] - }, - "BMD": { - "type": ["null", "number"] - }, - "BND": { - "type": ["null", "number"] - }, - "BOB": { - "type": ["null", "number"] - }, - "BRL": { - "type": ["null", "number"] - }, - "BSD": { - "type": ["null", "number"] - }, - "BTC": { - "type": ["null", "number"] - }, - "BTN": { - "type": ["null", "number"] - }, - "BWP": { - "type": ["null", "number"] - }, - "BYN": { - "type": ["null", "number"] - }, - "BZD": { - "type": ["null", "number"] - }, - "CAD": { - "type": ["null", "number"] - }, - "CDF": { - "type": ["null", "number"] - }, - "CHF": { - "type": ["null", "number"] - }, - "CLF": { - "type": ["null", "number"] - }, - "CLP": { - "type": ["null", "number"] - }, - "CNH": { - "type": ["null", "number"] - }, - "CNY": { - "type": ["null", "number"] - }, - "COP": { - "type": ["null", "number"] - }, - "CRC": { - "type": ["null", "number"] - }, - "CUC": { - "type": ["null", "number"] - }, - "CUP": { - "type": ["null", "number"] - }, - "CVE": { - "type": ["null", "number"] - }, - "CZK": { - "type": ["null", "number"] - }, - "DJF": { - "type": ["null", "number"] - }, - "DKK": { - "type": ["null", "number"] - }, - "DOP": { - "type": ["null", "number"] - }, - "DZD": { - "type": ["null", "number"] - }, - "EGP": { - "type": ["null", "number"] - }, - "ERN": { - "type": ["null", "number"] - }, - "ETB": { - "type": ["null", "number"] - }, - "EUR": { - "type": ["null", "number"] - }, - "FJD": { - "type": ["null", "number"] - }, - "FKP": { - "type": ["null", "number"] - }, - "GBP": { - "type": ["null", "number"] - }, - "GEL": { - "type": ["null", "number"] - }, - "GGP": { - "type": ["null", "number"] - }, - "GHS": { - "type": ["null", "number"] - }, - "GIP": { - "type": ["null", "number"] - }, - "GMD": { - "type": ["null", "number"] - }, - "GNF": { - "type": ["null", "number"] - }, - "GTQ": { - "type": ["null", "number"] - }, - "GYD": { - "type": ["null", "number"] - }, - "HKD": { - "type": ["null", "number"] - }, - "HNL": { - "type": ["null", "number"] - }, - "HRK": { - "type": ["null", "number"] - }, - "HTG": { - "type": ["null", "number"] - }, - "HUF": { - "type": ["null", "number"] - }, - "IDR": { - "type": ["null", "number"] - }, - "ILS": { - "type": ["null", "number"] - }, - "IMP": { - "type": ["null", "number"] - }, - "INR": { - "type": ["null", "number"] - }, - "IQD": { - "type": ["null", "number"] - }, - "IRR": { - "type": ["null", "number"] - }, - "ISK": { - "type": ["null", "number"] - }, - "JEP": { - "type": ["null", "number"] - }, - "JMD": { - "type": ["null", "number"] - }, - "JOD": { - "type": ["null", "number"] - }, - "JPY": { - "type": ["null", "number"] - }, - "KES": { - "type": ["null", "number"] - }, - "KGS": { - "type": ["null", "number"] - }, - "KHR": { - "type": ["null", "number"] - }, - "KMF": { - "type": ["null", "number"] - }, - "KPW": { - "type": ["null", "number"] - }, - "KRW": { - "type": ["null", "number"] - }, - "KWD": { - "type": ["null", "number"] - }, - "KYD": { - "type": ["null", "number"] - }, - "KZT": { - "type": ["null", "number"] - }, - "LAK": { - "type": ["null", "number"] - }, - "LBP": { - "type": ["null", "number"] - }, - "LKR": { - "type": ["null", "number"] - }, - "LRD": { - "type": ["null", "number"] - }, - "LSL": { - "type": ["null", "number"] - }, - "LYD": { - "type": ["null", "number"] - }, - "MAD": { - "type": ["null", "number"] - }, - "MDL": { - "type": ["null", "number"] - }, - "MGA": { - "type": ["null", "number"] - }, - "MKD": { - "type": ["null", "number"] - }, - "MMK": { - "type": ["null", "number"] - }, - "MNT": { - "type": ["null", "number"] - }, - "MOP": { - "type": ["null", "number"] - }, - "MRO": { - "type": ["null", "number"] - }, - "MRU": { - "type": ["null", "number"] - }, - "MUR": { - "type": ["null", "number"] - }, - "MVR": { - "type": ["null", "number"] - }, - "MWK": { - "type": ["null", "number"] - }, - "MXN": { - "type": ["null", "number"] - }, - "MYR": { - "type": ["null", "number"] - }, - "MZN": { - "type": ["null", "number"] - }, - "NAD": { - "type": ["null", "number"] - }, - "NGN": { - "type": ["null", "number"] - }, - "NIO": { - "type": ["null", "number"] - }, - "NOK": { - "type": ["null", "number"] - }, - "NPR": { - "type": ["null", "number"] - }, - "NZD": { - "type": ["null", "number"] - }, - "OMR": { - "type": ["null", "number"] - }, - "PAB": { - "type": ["null", "number"] - }, - "PEN": { - "type": ["null", "number"] - }, - "PGK": { - "type": ["null", "number"] - }, - "PHP": { - "type": ["null", "number"] - }, - "PKR": { - "type": ["null", "number"] - }, - "PLN": { - "type": ["null", "number"] - }, - "PYG": { - "type": ["null", "number"] - }, - "QAR": { - "type": ["null", "number"] - }, - "RON": { - "type": ["null", "number"] - }, - "RSD": { - "type": ["null", "number"] - }, - "RUB": { - "type": ["null", "number"] - }, - "RWF": { - "type": ["null", "number"] - }, - "SAR": { - "type": ["null", "number"] - }, - "SBD": { - "type": ["null", "number"] - }, - "SCR": { - "type": ["null", "number"] - }, - "SDG": { - "type": ["null", "number"] - }, - "SEK": { - "type": ["null", "number"] - }, - "SGD": { - "type": ["null", "number"] - }, - "SHP": { - "type": ["null", "number"] - }, - "SLL": { - "type": ["null", "number"] - }, - "SOS": { - "type": ["null", "number"] - }, - "SRD": { - "type": ["null", "number"] - }, - "SSP": { - "type": ["null", "number"] - }, - "STD": { - "type": ["null", "number"] - }, - "STN": { - "type": ["null", "number"] - }, - "SVC": { - "type": ["null", "number"] - }, - "SYP": { - "type": ["null", "number"] - }, - "SZL": { - "type": ["null", "number"] - }, - "THB": { - "type": ["null", "number"] - }, - "TJS": { - "type": ["null", "number"] - }, - "TMT": { - "type": ["null", "number"] - }, - "TND": { - "type": ["null", "number"] - }, - "TOP": { - "type": ["null", "number"] - }, - "TRY": { - "type": ["null", "number"] - }, - "TTD": { - "type": ["null", "number"] - }, - "TWD": { - "type": ["null", "number"] - }, - "TZS": { - "type": ["null", "number"] - }, - "UAH": { - "type": ["null", "number"] - }, - "UGX": { - "type": ["null", "number"] - }, - "USD": { - "type": ["null", "number"] - }, - "UYU": { - "type": ["null", "number"] - }, - "UZS": { - "type": ["null", "number"] - }, - "VES": { - "type": ["null", "number"] - }, - "VND": { - "type": ["null", "number"] - }, - "VUV": { - "type": ["null", "number"] - }, - "WST": { - "type": ["null", "number"] - }, - "XAF": { - "type": ["null", "number"] - }, - "XAG": { - "type": ["null", "number"] - }, - "XAU": { - "type": ["null", "number"] - }, - "XCD": { - "type": ["null", "number"] - }, - "XDR": { - "type": ["null", "number"] - }, - "XOF": { - "type": ["null", "number"] - }, - "XPD": { - "type": ["null", "number"] - }, - "XPF": { - "type": ["null", "number"] - }, - "XPT": { - "type": ["null", "number"] - }, - "YER": { - "type": ["null", "number"] - }, - "ZAR": { - "type": ["null", "number"] - }, - "ZMW": { - "type": ["null", "number"] - }, - "ZWL": { - "type": ["null", "number"] - } - } - } - } - }, + "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["timestamp"] + "default_cursor_field": ["timestamp"], + "source_defined_primary_key": [["timestamp"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["timestamp"] + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/expected_records.jsonl new file mode 100644 index 000000000000..d0778e83e52d --- /dev/null +++ b/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/expected_records.jsonl @@ -0,0 +1 @@ +{"stream": "open_exchange_rates", "data": {"disclaimer":"Usage subject to terms: https://openexchangerates.org/terms","license":"https://openexchangerates.org/license","timestamp":1690934399,"base":"USD","rates":{"AED":3.673075,"AFN":86.632624,"ALL":92.945891,"AMD":387.19215,"ANG":1.80297,"AOA":826.5905,"ARS":276.186305,"AUD":1.511123,"AWG":1.8,"AZN":1.7,"BAM":1.781604,"BBD":2,"BDT":108.574165,"BGN":1.77719,"BHD":0.376993,"BIF":2833.020268,"BMD":1,"BND":1.334044,"BOB":6.91298,"BRL":4.7935,"BSD":1,"BTC":3.3707219e-05,"BTN":82.300254,"BWP":13.215643,"BYN":2.525084,"BZD":2.016506,"CAD":1.32715,"CDF":2480.983028,"CHF":0.872626,"CLF":0.030502,"CLP":841.63,"CNH":7.176505,"CNY":7.1775,"COP":3915.417839,"CRC":545.006601,"CUC":1,"CUP":25.75,"CVE":100.606441,"CZK":21.7611,"DJF":178.072377,"DKK":6.7696,"DOP":56.285713,"DZD":135.64355,"EGP":30.9068,"ERN":15,"ETB":54.978478,"EUR":0.908455,"FJD":2.2183,"FKP":0.78154,"GBP":0.78154,"GEL":2.6,"GGP":0.78154,"GHS":11.333518,"GIP":0.78154,"GMD":60.375,"GNF":8625.954835,"GTQ":7.865732,"GYD":209.302328,"HKD":7.794182,"HNL":24.626137,"HRK":6.843146,"HTG":136.558033,"HUF":353.275548,"IDR":15139.102435,"ILS":3.63639,"IMP":0.78154,"INR":82.301851,"IQD":1310.335587,"IRR":42312.5,"ISK":131.09,"JEP":0.78154,"JMD":154.554565,"JOD":0.7083,"JPY":142.963,"KES":142.43631,"KGS":87.7821,"KHR":4131.520737,"KMF":448.30009,"KPW":900,"KRW":1289.399536,"KWD":0.307512,"KYD":0.833675,"KZT":444.670939,"LAK":19243.55048,"LBP":15109.691214,"LKR":319.129179,"LRD":186.499986,"LSL":18.016579,"LYD":4.782894,"MAD":9.857715,"MDL":17.749473,"MGA":4493.058467,"MKD":55.948277,"MMK":2100.856021,"MNT":3450,"MOP":8.030971,"MRU":38.104804,"MUR":45.2,"MVR":15.36,"MWK":1053.305792,"MXN":16.8783,"MYR":4.522,"MZN":63.764993,"NAD":18.28,"NGN":758.52,"NIO":36.568285,"NOK":10.168205,"NPR":131.678783,"NZD":1.62881,"OMR":0.384984,"PAB":1,"PEN":3.606245,"PGK":3.589272,"PHP":54.901495,"PKR":287.461749,"PLN":4.04257,"PYG":7277.293735,"QAR":3.641,"RON":4.4809,"RSD":106.556,"RUB":92.410004,"RWF":1176.872143,"SAR":3.751941,"SBD":8.368787,"SCR":13.151193,"SDG":601.589,"SEK":9.64075,"SGD":1.338502,"SHP":0.78154,"SLL":10260,"SOS":578.5,"SRD":20.9025,"SSP":130.2634,"STD":21050.599925,"SVC":8.747441,"SYP":512.380676,"SZL":18.009,"THB":32.4,"TJS":11.404289,"TMT":3.499986,"TND":2.487649,"TOP":2.272714,"TRY":9.319,"TTD":6.785232,"TWD":27.9,"TZS":2316.200319,"UAH":27.220516,"UGX":3530.977159,"UYU":43.788437,"UZS":10520.400572,"VES":3547812.5,"VND":22724.990112,"VUV":112.573844,"WST":2.515515,"XAF":554.633237,"XAG":0.034503,"XAU":0.000451,"XCD":2.7,"XDR":0.70168,"XOF":554.633237,"XPD":0.000406,"XPF":98.090344,"XPT":0.000824,"YER":250.350154,"ZAR":18.017,"ZMW":15.65,"ZWL":322.355011}}, "emitted_at": 1696204799} diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/sample_state.json index c6e267ec3977..631373b6310e 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-open-exchange-rates/integration_tests/sample_state.json @@ -1,5 +1,9 @@ -{ - "exchange_rates": { - "timestamp": 1673379575 +[ + { + "type": "STREAM", + "stream": { + "stream_state": { "timestamp": 1673379575 }, + "stream_descriptor": { "name": "open_exchange_rates" } + } } -} +] diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/metadata.yaml b/airbyte-integrations/connectors/source-open-exchange-rates/metadata.yaml index adc7e74a580d..7c5f21637fae 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/metadata.yaml +++ b/airbyte-integrations/connectors/source-open-exchange-rates/metadata.yaml @@ -1,24 +1,25 @@ data: + allowedHosts: + hosts: + - openexchangerates.org + registries: + oss: + enabled: true + cloud: + enabled: false connectorSubtype: api connectorType: source definitionId: 77d5ca6b-d345-4dce-ba1e-1935a75778b8 - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-open-exchange-rates githubIssueLabel: source-open-exchange-rates - icon: airbyte.svg + icon: open-exchange-rates.svg license: MIT name: Open Exchange Rates - registries: - cloud: - enabled: false - oss: - enabled: true + releaseDate: 2023-10-02 releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/open-exchange-rates tags: - - language:python - ab_internal: - sl: 100 - ql: 100 - supportLevel: community + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/requirements.txt b/airbyte-integrations/connectors/source-open-exchange-rates/requirements.txt index d6e1198b1ab1..cc57334ef619 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/requirements.txt +++ b/airbyte-integrations/connectors/source-open-exchange-rates/requirements.txt @@ -1 +1,2 @@ +-e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/setup.py b/airbyte-integrations/connectors/source-open-exchange-rates/setup.py index 8e52d07a0702..55fd4589213f 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/setup.py +++ b/airbyte-integrations/connectors/source-open-exchange-rates/setup.py @@ -6,13 +6,12 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", - "pendulum==2.1.2", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", - "pytest~=6.1", + "pytest~=6.2", "pytest-mock~=3.6.1", ] diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/__init__.py b/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/__init__.py index 71a500afcdab..0d1770484ee9 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/__init__.py +++ b/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/manifest.yaml b/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/manifest.yaml new file mode 100644 index 000000000000..5f8f7952682a --- /dev/null +++ b/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/manifest.yaml @@ -0,0 +1,57 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + + requester: + type: HttpRequester + url_base: "https://openexchangerates.org/api/" + http_method: "GET" + authenticator: + type: NoAuth + request_parameters: + app_id: "{{ config['app_id'] }}" + base: "{{ config['base'] }}" + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + incremental_sync_base: + type: DatetimeBasedCursor + cursor_field: "{{ parameters.incremental_cursor }}" + datetime_format: "%s" + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%d" + + open_exchange_rates_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "open_exchange_rates" + incremental_cursor: "timestamp" + path: "historical/{{ format_datetime( config['start_date'] if not stream_state else stream_state['timestamp'], '%Y-%m-%d' ) }}.json" + incremental_sync: + $ref: "#/definitions/incremental_sync_base" + +streams: + - "#/definitions/open_exchange_rates_stream" + +check: + type: CheckStream + stream_names: + - "open_exchange_rates" diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/schemas/open_exchange_rates.json b/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/schemas/open_exchange_rates.json index 92c17f7e56aa..fae8408b06a0 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/schemas/open_exchange_rates.json +++ b/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/schemas/open_exchange_rates.json @@ -1,15 +1,23 @@ { - "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Generated schema for Root", + "type": ["null", "object"], "required": ["base", "rates"], "properties": { "base": { - "type": "string" + "type": ["null", "string"] + }, + "disclaimer": { + "type": ["null", "string"] + }, + "license": { + "type": ["null", "string"] }, "timestamp": { - "type": "integer" + "type": ["null", "integer"] }, "rates": { - "type": "object", + "type": ["null", "object"], "properties": { "AED": { "type": ["null", "number"] diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/source.py b/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/source.py index 2aed45b828bb..b0ea9f7fff6e 100644 --- a/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/source.py +++ b/airbyte-integrations/connectors/source-open-exchange-rates/source_open_exchange_rates/source.py @@ -2,140 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import pendulum -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -from pendulum import DateTime +WARNING: Do not modify this file. +""" -class OpenExchangeRates(HttpStream, ABC): - url_base = "https://openexchangerates.org/api/" - - primary_key = None - cursor_field = "timestamp" - - def __init__(self, base: Optional[str], start_date: str, app_id: str, **kwargs: dict) -> None: - super().__init__(**kwargs) - - self.base = base - self.start_date = pendulum.parse(start_date) - self.app_id = app_id - self._cursor_value = None - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - - params = {} - - if self.base is not None: - params["base"] = self.base - - return params - - @property - def state(self) -> Mapping[str, Any]: - if self._cursor_value: - return {self.cursor_field: self._cursor_value} - else: - return {self.cursor_field: self.start_date.timestamp()} - - @state.setter - def state(self, value: Mapping[str, Any]): - self._cursor_value = value[self.cursor_field] - - def parse_response( - self, - response: requests.Response, - *, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[Mapping]: - response_json = response.json() - - latest_record_timestamp = response_json["timestamp"] - if self._cursor_value and latest_record_timestamp <= self._cursor_value: - return - if self._cursor_value: - self._cursor_value = max(self._cursor_value, latest_record_timestamp) - else: - self._cursor_value = latest_record_timestamp - - yield response_json - - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: - start_date = stream_state[self.cursor_field] if stream_state and self.cursor_field in stream_state else self.start_date - - if isinstance(start_date, int): - start_date = pendulum.from_timestamp(start_date) - - return self._chunk_date_range(start_date) - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return f"historical/{stream_slice['date']}.json" - - def _chunk_date_range(self, start_date: DateTime) -> List[Mapping[str, Any]]: - """ - Returns a list of each day between the start date and now. - The return value is a list of dicts {'date': date_string}. - """ - dates = [] - - while start_date < pendulum.now(): - dates.append({"date": start_date.to_date_string()}) - start_date = start_date.add(days=1) - return dates - - -# Source -class SourceOpenExchangeRates(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - """ - Checks the connection by sending a request to /usage and checks the remaining quota - - :param config: the user-input config object conforming to the connector's spec.yaml - :param logger: logger object - :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. - """ - try: - auth = TokenAuthenticator(token=config["app_id"], auth_method="Token").get_auth_header() - - resp = requests.get(f"{OpenExchangeRates.url_base}usage.json", headers=auth) - status = resp.status_code - - logger.info(f"Ping response code: {status}") - response_dict = resp.json() - - if status == 200: - quota_remaining = response_dict["data"]["usage"]["requests_remaining"] - - if quota_remaining > 0: - return True, None - - return False, "Quota exceeded" - else: - description = response_dict.get("description") - return False, description - except Exception as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - auth = TokenAuthenticator(token=config["app_id"], auth_method="Token") - - return [OpenExchangeRates(base=config["base"], start_date=config["start_date"], app_id=config["app_id"], authenticator=auth)] +# Declarative Source +class SourceOpenExchangeRates(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/unit_tests/__init__.py b/airbyte-integrations/connectors/source-open-exchange-rates/unit_tests/__init__.py deleted file mode 100644 index 1100c1c58cf5..000000000000 --- a/airbyte-integrations/connectors/source-open-exchange-rates/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/unit_tests/conftest.py b/airbyte-integrations/connectors/source-open-exchange-rates/unit_tests/conftest.py deleted file mode 100644 index aa7b8cdb385c..000000000000 --- a/airbyte-integrations/connectors/source-open-exchange-rates/unit_tests/conftest.py +++ /dev/null @@ -1,25 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from pytest import fixture - - -@fixture(name="config") -def config_fixture(requests_mock): - config = {"start_date": "2022-11-13", "base": "USD", "app_id": "KEY"} - - return config - - -@fixture(name="mock_stream") -def mock_stream_fixture(requests_mock): - def _mock_stream(path, response=None, status_code=200): - if response is None: - response = {} - - url = f"https://openexchangerates.org/api/{path}.json" - requests_mock.get(url, json=response, status_code=status_code) - - return _mock_stream diff --git a/airbyte-integrations/connectors/source-open-exchange-rates/unit_tests/test_source.py b/airbyte-integrations/connectors/source-open-exchange-rates/unit_tests/test_source.py deleted file mode 100644 index 3a86398c6c33..000000000000 --- a/airbyte-integrations/connectors/source-open-exchange-rates/unit_tests/test_source.py +++ /dev/null @@ -1,102 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import logging - -from source_open_exchange_rates.source import SourceOpenExchangeRates - -logger = logging.getLogger("airbyte") - - -def test_check_connection(config, mock_stream): - response = { - "status": 200, - "data": { - "app_id": "KEY", - "status": "active", - "plan": { - "name": "Free", - "quota": "1000 requests / month", - "update_frequency": "3600s", - "features": { - "base": False, - "symbols": False, - "experimental": True, - "time-series": False, - "convert": False, - "bid-ask": False, - "ohlc": False, - "spot": False - } - }, - "usage": { - "requests": 27, - "requests_quota": 1000, - "requests_remaining": 973, - "days_elapsed": 1, - "days_remaining": 29, - "daily_average": 27 - } - } - } - - mock_stream(path="usage", response=response) - ok, error_msg = SourceOpenExchangeRates().check_connection(logger, config=config) - logger.info(error_msg) - assert ok - assert error_msg is None - - -def test_check_connection_quota_exceeded_exception(config, mock_stream): - response = { - "status": 200, - "data": { - "app_id": "KEY", - "status": "active", - "plan": { - "name": "Free", - "quota": "1000 requests / month", - "update_frequency": "3600s", - "features": { - "base": False, - "symbols": False, - "experimental": True, - "time-series": False, - "convert": False, - "bid-ask": False, - "ohlc": False, - "spot": False - } - }, - "usage": { - "requests": 1000, - "requests_quota": 1000, - "requests_remaining": 0, - "days_elapsed": 1, - "days_remaining": 29, - "daily_average": 27 - } - } - } - - mock_stream(path="usage", response=response, status_code=200) - ok, error_msg = SourceOpenExchangeRates().check_connection(logger, config=config) - - assert not ok - assert error_msg == "Quota exceeded" - - -def test_check_connection_invalid_appid_exception(config, mock_stream): - response = { - "error": True, - "status": 401, - "message": "invalid_app_id", - "description": "Invalid App ID - please sign up at https://openexchangerates.org/signup, or contact support@openexchangerates.org." - } - - mock_stream(path="usage", response=response, status_code=401) - ok, error_msg = SourceOpenExchangeRates().check_connection(logger, config=config) - - assert not ok - assert error_msg == response['description'] diff --git a/airbyte-integrations/connectors/source-openweather/.dockerignore b/airbyte-integrations/connectors/source-openweather/.dockerignore index 6ed52d920dd8..37cf7cb2b27c 100644 --- a/airbyte-integrations/connectors/source-openweather/.dockerignore +++ b/airbyte-integrations/connectors/source-openweather/.dockerignore @@ -1,6 +1,5 @@ * !Dockerfile -!Dockerfile.test !main.py !source_openweather !setup.py diff --git a/airbyte-integrations/connectors/source-openweather/Dockerfile b/airbyte-integrations/connectors/source-openweather/Dockerfile index 0b5dfbaa095f..6c1b15ff1af7 100644 --- a/airbyte-integrations/connectors/source-openweather/Dockerfile +++ b/airbyte-integrations/connectors/source-openweather/Dockerfile @@ -34,5 +34,5 @@ COPY source_openweather ./source_openweather ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.6 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-openweather diff --git a/airbyte-integrations/connectors/source-openweather/README.md b/airbyte-integrations/connectors/source-openweather/README.md index ef78da15ca26..a995dba6879c 100644 --- a/airbyte-integrations/connectors/source-openweather/README.md +++ b/airbyte-integrations/connectors/source-openweather/README.md @@ -1,74 +1,34 @@ -# Open Weather Source +# Openweather Source -This is the repository for the Open Weather source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/open-weather). +This is the repository for the Openweather configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/openweather). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-openweather:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/open-weather) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_openweather/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/openweather) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_openweather/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source open-weather test creds` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source openweather test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-openweather:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-openweather build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-openweather:airbyteDocker +An image will be built with the tag `airbyte/source-openweather:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-openweather:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-openweather:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-openweather:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-openweather:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-openweather test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-openweather:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-openweather:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-openweather test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/openweather.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-openweather/__init__.py b/airbyte-integrations/connectors/source-openweather/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-openweather/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-openweather/acceptance-test-config.yml b/airbyte-integrations/connectors/source-openweather/acceptance-test-config.yml index 7088a5abbeb1..96f5beef8e92 100644 --- a/airbyte-integrations/connectors/source-openweather/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-openweather/acceptance-test-config.yml @@ -1,18 +1,26 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests connector_image: airbyte/source-openweather:dev -tests: - spec: - - spec_path: "source_openweather/spec.json" +acceptance_tests: connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.6" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" + bypass_reason: "This connector does not implement incremental sync" +# tests: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# future_state: +# future_state_path: "integration_tests/abnormal_state.json" diff --git a/airbyte-integrations/connectors/source-openweather/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-openweather/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-openweather/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-openweather/bootstrap.md b/airbyte-integrations/connectors/source-openweather/bootstrap.md deleted file mode 100644 index 1ded59966a97..000000000000 --- a/airbyte-integrations/connectors/source-openweather/bootstrap.md +++ /dev/null @@ -1,16 +0,0 @@ -# OpenWeather -OpenWeather is an online service offering an API to retrieve historical, current and forecasted weather data over the globe. - -## One Call API -The *One Call API* enable retrieval of multiple weather data for a location in a single call. -I made this stream implementation a priority because it has a free plan that might be valuable for all data teams building models around weather data. -The API returns current weather data along with other time resolutions (minutely, hourly, daily) and weather alerts. - -### Full refresh vs incremental stream implementation -I did not implement a full refresh stream because One Call API calls are not idempotent: two subsequents calls with the same parameters might give different results. Moreover, it has no historical capabilities (there is a specific historical API for that) and only gives current weather conditions and forecasts. It's why I implemented an incremental stream without a feature to request past data. - -### Auth -API calls are authenticated through an API key passed in a query string parameter (`appid`). API keys can be generated from OpenWeather's user account panel. - -### Rate limits -The API does have some rate limiting logic but it's not very transparent to the user. There is no endpoint to check calls consumption. It is stated that the free plan allows 60 calls / minute or 1,000,000 calls/month. If the limit is exceeded the user account (not only the API key) gets blocked for an unknown duration. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-openweather/build.gradle b/airbyte-integrations/connectors/source-openweather/build.gradle deleted file mode 100644 index 4baf3e5740da..000000000000 --- a/airbyte-integrations/connectors/source-openweather/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_openweather' -} diff --git a/airbyte-integrations/connectors/source-openweather/icon.svg b/airbyte-integrations/connectors/source-openweather/icon.svg index f960352d014a..acaf7278b549 100644 --- a/airbyte-integrations/connectors/source-openweather/icon.svg +++ b/airbyte-integrations/connectors/source-openweather/icon.svg @@ -1 +1,40 @@ - \ No newline at end of file + + + + + + + + + + diff --git a/airbyte-integrations/connectors/source-openweather/integration_tests/__init__.py b/airbyte-integrations/connectors/source-openweather/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-openweather/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-openweather/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-openweather/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-openweather/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-openweather/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-openweather/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-openweather/integration_tests/catalog.json b/airbyte-integrations/connectors/source-openweather/integration_tests/catalog.json deleted file mode 100644 index c0e8035aba02..000000000000 --- a/airbyte-integrations/connectors/source-openweather/integration_tests/catalog.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type": "CATALOG", - "catalog": { - "streams": [ - { - "name": "one_call", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - } - ] - } -} diff --git a/airbyte-integrations/connectors/source-openweather/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-openweather/integration_tests/configured_catalog.json index fa34fa13e3c5..432dab8b2e3b 100644 --- a/airbyte-integrations/connectors/source-openweather/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-openweather/integration_tests/configured_catalog.json @@ -2,7 +2,7 @@ "streams": [ { "stream": { - "name": "one_call", + "name": "onecall", "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, diff --git a/airbyte-integrations/connectors/source-openweather/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-openweather/integration_tests/sample_config.json index 36a6498f7ccf..1e750988db76 100644 --- a/airbyte-integrations/connectors/source-openweather/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-openweather/integration_tests/sample_config.json @@ -1,5 +1,8 @@ { - "appid": "my-api-key", - "lat": "-21.24239", - "lon": "55.71004" + "lat": "33.44", + "lon": "-94.04", + "appid": "bd5e378503939ddaee76f12ad7a97608", + "lang": "fr", + "dt": "1693326055", + "units": "standard" } diff --git a/airbyte-integrations/connectors/source-openweather/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-openweather/integration_tests/sample_state.json index 2b7ca2ebdb9e..3587e579822d 100644 --- a/airbyte-integrations/connectors/source-openweather/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-openweather/integration_tests/sample_state.json @@ -1,7 +1,5 @@ { - "one_call": { - "current": { - "dt": 0 - } + "todo-stream-name": { + "todo-field-name": "value" } } diff --git a/airbyte-integrations/connectors/source-openweather/main.py b/airbyte-integrations/connectors/source-openweather/main.py index 975558accee0..398948d54171 100644 --- a/airbyte-integrations/connectors/source-openweather/main.py +++ b/airbyte-integrations/connectors/source-openweather/main.py @@ -6,8 +6,8 @@ import sys from airbyte_cdk.entrypoint import launch -from source_openweather import SourceOpenWeather +from source_openweather import SourceOpenweather if __name__ == "__main__": - source = SourceOpenWeather() + source = SourceOpenweather() launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-openweather/metadata.yaml b/airbyte-integrations/connectors/source-openweather/metadata.yaml index 7276d353f847..b15d5744b4f9 100644 --- a/airbyte-integrations/connectors/source-openweather/metadata.yaml +++ b/airbyte-integrations/connectors/source-openweather/metadata.yaml @@ -1,24 +1,24 @@ data: + allowedHosts: + hosts: + - api.openweathermap.org + registries: + oss: + enabled: true + cloud: + enabled: false connectorSubtype: api connectorType: source - definitionId: d8540a80-6120-485d-b7d6-272bca477d9b - dockerImageTag: 0.1.6 + definitionId: 561d7787-b45e-4f3b-af58-0163c3ba9d5a + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-openweather githubIssueLabel: source-openweather icon: openweather.svg license: MIT - name: OpenWeather - registries: - cloud: - enabled: true - oss: - enabled: true + name: Openweather releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/openweather tags: - - language:python - ab_internal: - sl: 100 - ql: 100 - supportLevel: community + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-openweather/requirements.txt b/airbyte-integrations/connectors/source-openweather/requirements.txt index d6e1198b1ab1..cc57334ef619 100644 --- a/airbyte-integrations/connectors/source-openweather/requirements.txt +++ b/airbyte-integrations/connectors/source-openweather/requirements.txt @@ -1 +1,2 @@ +-e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-openweather/setup.py b/airbyte-integrations/connectors/source-openweather/setup.py index 16f473b254ab..fd32d0604449 100644 --- a/airbyte-integrations/connectors/source-openweather/setup.py +++ b/airbyte-integrations/connectors/source-openweather/setup.py @@ -6,23 +6,24 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", + "airbyte-cdk~=0.1", ] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", - "pytest~=6.1", + "pytest~=6.2", "pytest-mock~=3.6.1", + "connector-acceptance-test", ] setup( name="source_openweather", - description="Source implementation for Open Weather.", + description="Source implementation for Openweather.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-openweather/source_openweather/__init__.py b/airbyte-integrations/connectors/source-openweather/source_openweather/__init__.py index 36eca1bee2ec..8c7afeef13ae 100644 --- a/airbyte-integrations/connectors/source-openweather/source_openweather/__init__.py +++ b/airbyte-integrations/connectors/source-openweather/source_openweather/__init__.py @@ -1,8 +1,8 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from .source import SourceOpenWeather +from .source import SourceOpenweather -__all__ = ["SourceOpenWeather"] +__all__ = ["SourceOpenweather"] diff --git a/airbyte-integrations/connectors/source-openweather/source_openweather/extra_validations.py b/airbyte-integrations/connectors/source-openweather/source_openweather/extra_validations.py deleted file mode 100644 index 6a4c3015dfb2..000000000000 --- a/airbyte-integrations/connectors/source-openweather/source_openweather/extra_validations.py +++ /dev/null @@ -1,33 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from typing import Any, Mapping - - -def check_lat(lat_value) -> float: - try: - lat_value = float(lat_value) - except (ValueError, TypeError): - raise Exception("Wrong value for lat, it must be a decimal number between -90 and 90") - if not -90 <= lat_value <= 90: - raise Exception("Wrong value for lat, it must be between -90 and 90") - return lat_value - - -def check_lon(lon_value) -> float: - try: - lon_value = float(lon_value) - except (ValueError, TypeError): - raise Exception("Wrong value for lon, it must be a decimal number between -180 and 180") - - if not -180 <= lon_value <= 180: - raise Exception("Wrong value for lon, it must be between -180 and 180") - return lon_value - - -def validate(config: Mapping[str, Any]) -> Mapping[str, Any]: - valid_config = {**config} - valid_config["lat"] = check_lat(valid_config["lat"]) - valid_config["lon"] = check_lon(valid_config["lon"]) - return valid_config diff --git a/airbyte-integrations/connectors/source-openweather/source_openweather/manifest.yaml b/airbyte-integrations/connectors/source-openweather/source_openweather/manifest.yaml new file mode 100644 index 000000000000..7fd4496d1852 --- /dev/null +++ b/airbyte-integrations/connectors/source-openweather/source_openweather/manifest.yaml @@ -0,0 +1,189 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + + requester: + type: HttpRequester + url_base: "https://api.openweathermap.org/data/3.0/" + http_method: "GET" + request_parameters: + lat: "{% if -90.00 <= config['lat']|float <= 90.00 %}{{ config['lat'] }}{% else %} WRONG LATITUDE{% endif %}" + lon: "{% if -180.00 <= config['lon']|float <= 180.00 %}{{ config['lon'] }}{% else %}WRONG LONGITUDE{% endif %}" + appid: "{{ config['appid'] }}" + lang: "{{ config.get('lang')}}" + units: "{{ config.get('units')}}" + error_handler: + response_filters: + - http_codes: [500] + action: RETRY + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester" + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + # incremental_sync_base: + # type: DatetimeBasedCursor + # cursor_field: "sync_field" + # datetime_format: "%s" + # cursor_granularity: "PT0.000001S" + # lookback_window: "P5D" + # start_datetime: + # datetime: "{{ config['dt'] }}" + # datetime_format: "%s" + # end_datetime: + # datetime: "{{ format_datetime(now_utc(), '%s') }}" + # datetime_format: "%s" + # step: "P1M" + + onecall_stream: + $ref: "#/definitions/base_stream" + name: "onecall" + retriever: + $ref: "#/definitions/retriever" + $parameters: + path: "onecall" + # the commented section is for incremental sync which is not supported by the API + # path: "{% if config['dt']%}onecall/timemachine{% else %}onecall{% endif %}" + # transformations: + # - type: AddFields + # fields: + # - path: [ "sync_field" ] + # value: "{{ record['current']['dt'] }}" + # incremental_cursor: ["current","dt"] + # incremental_sync: + # $ref: "#/definitions/incremental_sync_base" + +streams: + - "#/definitions/onecall_stream" + +check: + type: CheckStream + stream_names: + - "onecall" + +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/openweather + connection_specification: + title: Openweather Spec + type: object + required: + - lat + - lon + - appid + additionalProperties: true + properties: + lat: + type: string + pattern: "^[-]?\\d{1,2}(\\.\\d+)?$" + description: "Latitude, decimal (-90; 90). If you need the geocoder to automatic convert city names and zip-codes to geo coordinates and the other way around, please use our Geocoding API" + examples: + - "45.7603" + - "-21.249107858038816" + lon: + type: string + pattern: "^[-]?\\d{1,2}(\\.\\d+)?$" + description: "Longitude, decimal (-180; 180). If you need the geocoder to automatic convert city names and zip-codes to geo coordinates and the other way around, please use our Geocoding API" + examples: + - "4.835659" + - "-70.39482074115321" + appid: + type: string + description: "API KEY" + airbyte_secret: true + units: + type: string + description: "Units of measurement. standard, metric and imperial units are available. If you do not use the units parameter, standard units will be applied by default." + enum: + - standard + - metric + - imperial + examples: + - "standard" + - "metric" + - "imperial" + # dt: + # type: string + # description : "Date in UNIX format" + # example: + # - '1693023447' + only_current: + type: boolean + description: "True for particular day" + example: + - "true" + lang: + type: string + description: + You can use lang parameter to get the output in your language. + The contents of the description field will be translated. See here + for the list of supported languages. + enum: + - af + - al + - ar + - az + - bg + - ca + - cz + - da + - de + - el + - en + - eu + - fa + - fi + - fr + - gl + - he + - hi + - hr + - hu + - id + - it + - ja + - kr + - la + - lt + - mk + - "no" + - nl + - pl + - pt + - pt_br + - ro + - ru + - sv + - se + - sk + - sl + - sp + - es + - sr + - th + - tr + - ua + - uk + - vi + - zh_cn + - zh_tw + - zu + examples: + - en + - fr + - pt_br + - uk + - zh_cn + - zh_tw diff --git a/airbyte-integrations/connectors/source-openweather/source_openweather/schemas/one_call.json b/airbyte-integrations/connectors/source-openweather/source_openweather/schemas/one_call.json deleted file mode 100644 index 53acab5894ae..000000000000 --- a/airbyte-integrations/connectors/source-openweather/source_openweather/schemas/one_call.json +++ /dev/null @@ -1,85 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "lat": { - "type": "number" - }, - "lon": { - "type": "number" - }, - "timezone": { - "type": "string" - }, - "timezone_offset": { - "type": "number" - }, - "current": { - "type": "object", - "properties": { - "dt": { - "type": "number" - }, - "sunrise": { - "type": "number" - }, - "sunset": { - "type": "number" - }, - "temp": { - "type": "number" - }, - "feels_like": { - "type": "number" - }, - "pressure": { - "type": "number" - }, - "humidity": { - "type": "number" - }, - "dew_point": { - "type": "number" - }, - "uvi": { - "type": "number" - }, - "clouds": { - "type": "number" - }, - "visibility": { - "type": "number" - }, - "wind_speed": { - "type": "number" - }, - "wind_deg": { - "type": "number" - }, - "weather": { - "type": "array" - }, - "rain": { - "type": "object", - "properties": { - "1h": { - "type": "number" - } - } - } - } - }, - "minutely": { - "type": "array" - }, - "hourly": { - "type": "array" - }, - "daily": { - "type": "array" - }, - "alerts": { - "type": "array" - } - } -} diff --git a/airbyte-integrations/connectors/source-openweather/source_openweather/schemas/onecall.json b/airbyte-integrations/connectors/source-openweather/source_openweather/schemas/onecall.json new file mode 100644 index 000000000000..e06c9fbcadaa --- /dev/null +++ b/airbyte-integrations/connectors/source-openweather/source_openweather/schemas/onecall.json @@ -0,0 +1,88 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "lat": { + "type": "number" + }, + "lon": { + "type": "number" + }, + "timezone": { + "type": "string" + }, + "timezone_offset": { + "type": "number" + }, + "current": { + "type": "object", + "additionalProperties": true, + "properties": { + "dt": { + "type": "number" + }, + "sunrise": { + "type": "number" + }, + "sunset": { + "type": "number" + }, + "temp": { + "type": "number" + }, + "feels_like": { + "type": "number" + }, + "pressure": { + "type": "number" + }, + "humidity": { + "type": "number" + }, + "dew_point": { + "type": "number" + }, + "uvi": { + "type": "number" + }, + "clouds": { + "type": "number" + }, + "visibility": { + "type": "number" + }, + "wind_speed": { + "type": "number" + }, + "wind_deg": { + "type": "number" + }, + "weather": { + "type": "array" + }, + "rain": { + "type": "object", + "additionalProperties": true, + "properties": { + "1h": { + "type": "number" + } + } + } + } + }, + "minutely": { + "type": "array" + }, + "hourly": { + "type": "array" + }, + "daily": { + "type": "array" + }, + "alerts": { + "type": "array" + } + } +} diff --git a/airbyte-integrations/connectors/source-openweather/source_openweather/source.py b/airbyte-integrations/connectors/source-openweather/source_openweather/source.py index f579ee1d6176..45a79e387fc9 100644 --- a/airbyte-integrations/connectors/source-openweather/source_openweather/source.py +++ b/airbyte-integrations/connectors/source-openweather/source_openweather/source.py @@ -2,45 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import Any, List, Mapping, Tuple +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -from . import extra_validations, streams +WARNING: Do not modify this file. +""" -class SourceOpenWeather(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - try: - valid_config = extra_validations.validate(config) - params = { - "appid": valid_config["appid"], - "lat": valid_config["lat"], - "lon": valid_config["lon"], - "lang": valid_config.get("lang"), - "units": valid_config.get("units"), - } - params = {k: v for k, v in params.items() if v is not None} - resp = requests.get(f"{streams.OneCall.url_base}onecall", params=params) - status = resp.status_code - if status == 200: - return True, None - else: - message = resp.json().get("message") - return False, message - except Exception as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - valid_config = extra_validations.validate(config) - return [ - streams.OneCall( - appid=valid_config["appid"], - lat=valid_config["lat"], - lon=valid_config["lon"], - lang=valid_config.get("lang"), - units=valid_config.get("units"), - ) - ] +# Declarative Source +class SourceOpenweather(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-openweather/source_openweather/spec.json b/airbyte-integrations/connectors/source-openweather/source_openweather/spec.json deleted file mode 100644 index 6efb78fef792..000000000000 --- a/airbyte-integrations/connectors/source-openweather/source_openweather/spec.json +++ /dev/null @@ -1,96 +0,0 @@ -{ - "documentationUrl": "https://docsurl.com", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Open Weather Spec", - "type": "object", - "required": ["appid", "lat", "lon"], - "additionalProperties": true, - "properties": { - "lat": { - "title": "Latitude", - "type": "string", - "pattern": "^[-]?\\d{1,2}(\\.\\d+)?$", - "examples": ["45.7603", "-21.249107858038816"], - "description": "Latitude for which you want to get weather condition from. (min -90, max 90)" - }, - "lon": { - "title": "Longitude", - "type": "string", - "pattern": "^[-]?\\d{1,3}(\\.\\d+)?$", - "examples": ["4.835659", "-70.39482074115321"], - "description": "Longitude for which you want to get weather condition from. (min -180, max 180)" - }, - "appid": { - "title": "App ID", - "type": "string", - "description": "Your OpenWeather API Key. See here. The key is case sensitive.", - "airbyte_secret": true - }, - "units": { - "title": "Units", - "type": "string", - "description": "Units of measurement. standard, metric and imperial units are available. If you do not use the units parameter, standard units will be applied by default.", - "enum": ["standard", "metric", "imperial"], - "examples": ["standard", "metric", "imperial"] - }, - "lang": { - "title": "Language", - "type": "string", - "description": "You can use lang parameter to get the output in your language. The contents of the description field will be translated. See here for the list of supported languages.", - "enum": [ - "af", - "al", - "ar", - "az", - "bg", - "ca", - "cz", - "da", - "de", - "el", - "en", - "eu", - "fa", - "fi", - "fr", - "gl", - "he", - "hi", - "hr", - "hu", - "id", - "it", - "ja", - "kr", - "la", - "lt", - "mk", - "no", - "nl", - "pl", - "pt", - "pt_br", - "ro", - "ru", - "sv", - "se", - "sk", - "sl", - "sp", - "es", - "sr", - "th", - "tr", - "ua", - "uk", - "vi", - "zh_cn", - "zh_tw", - "zu" - ], - "examples": ["en", "fr", "pt_br", "uk", "zh_cn", "zh_tw"] - } - } - } -} diff --git a/airbyte-integrations/connectors/source-openweather/source_openweather/streams.py b/airbyte-integrations/connectors/source-openweather/source_openweather/streams.py deleted file mode 100644 index b1f91b4cd752..000000000000 --- a/airbyte-integrations/connectors/source-openweather/source_openweather/streams.py +++ /dev/null @@ -1,52 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from typing import Any, Iterable, Mapping, MutableMapping, Optional - -import requests -from airbyte_cdk.sources.streams.http import HttpStream - - -class OneCall(HttpStream): - - cursor_field = ["current", "dt"] - url_base = "https://api.openweathermap.org/data/3.0/" - primary_key = None - - def __init__(self, appid: str, lat: float, lon: float, lang: str = None, units: str = None): - super().__init__() - self.appid = appid - self.lat = lat - self.lon = lon - self.lang = lang - self.units = units - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "onecall" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - def request_params(self, **kwargs) -> MutableMapping[str, Any]: - params = {"appid": self.appid, "lat": self.lat, "lon": self.lon, "lang": self.lang, "units": self.units} - params = {k: v for k, v in params.items() if v is not None} - return params - - def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: - data = response.json() - if data["current"]["dt"] >= stream_state.get("dt", 0): - return [data] - else: - return [] - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]): - current_stream_state = current_stream_state or {"dt": 0} - current_stream_state["dt"] = max(latest_record["current"]["dt"], current_stream_state["dt"]) - return current_stream_state - - def should_retry(self, response: requests.Response) -> bool: - # Do not retry in case of 429 because the account is blocked for an unknown duration. - return 500 <= response.status_code < 600 diff --git a/airbyte-integrations/connectors/source-openweather/unit_tests/__init__.py b/airbyte-integrations/connectors/source-openweather/unit_tests/__init__.py deleted file mode 100644 index 46b7376756ec..000000000000 --- a/airbyte-integrations/connectors/source-openweather/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-openweather/unit_tests/test_extra_validations.py b/airbyte-integrations/connectors/source-openweather/unit_tests/test_extra_validations.py deleted file mode 100644 index 67bf9a82d01e..000000000000 --- a/airbyte-integrations/connectors/source-openweather/unit_tests/test_extra_validations.py +++ /dev/null @@ -1,64 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pytest -from source_openweather import extra_validations - - -@pytest.mark.parametrize( - "lat_value, error_message", - [ - ("1", None), - (1, None), - ("-12.3", None), - ("-91", "Wrong value for lat, it must be between -90 and 90"), - ("91", "Wrong value for lat, it must be between -90 and 90"), - ("1,2", "Wrong value for lat, it must be a decimal number between -90 and 90"), - ("foo", "Wrong value for lat, it must be a decimal number between -90 and 90"), - (["not_string"], "Wrong value for lat, it must be a decimal number between -90 and 90"), - ], -) -def test_check_lat(lat_value, error_message): - if error_message: - with pytest.raises(Exception, match=error_message): - extra_validations.check_lat(lat_value) - else: - assert extra_validations.check_lat(lat_value) == float(lat_value) - - -@pytest.mark.parametrize( - "lon_value, error_message", - [ - ("1", None), - (1, None), - ("-92.3", None), - ("-191", "Wrong value for lon, it must be between -180 and 180"), - ("191", "Wrong value for lon, it must be between -180 and 180"), - ("1,2", "Wrong value for lon, it must be a decimal number between -180 and 180"), - ("foo", "Wrong value for lon, it must be a decimal number between -180 and 180"), - (["not_string"], "Wrong value for lon, it must be a decimal number between -180 and 180"), - ], -) -def test_check_lon(lon_value, error_message): - if error_message: - with pytest.raises(Exception, match=error_message): - extra_validations.check_lon(lon_value) - else: - assert extra_validations.check_lon(lon_value) == float(lon_value) - - -def test_validate(mocker): - check_lat_mock = mocker.patch("source_openweather.extra_validations.check_lat") - check_lat_mock.return_value = 1.0 - check_lon_mock = mocker.patch("source_openweather.extra_validations.check_lon") - check_lon_mock.return_value = 1.0 - - config_to_validate = {"appid": "foo", "lat": "1", "lon": "1"} - expected_valid_config = {"appid": "foo", "lat": 1.0, "lon": 1.0} - - valid_config = extra_validations.validate(config_to_validate) - assert isinstance(valid_config, dict) - assert valid_config == expected_valid_config - check_lat_mock.assert_called_with("1") - check_lon_mock.assert_called_with("1") diff --git a/airbyte-integrations/connectors/source-openweather/unit_tests/test_source.py b/airbyte-integrations/connectors/source-openweather/unit_tests/test_source.py deleted file mode 100644 index 8bf0e57746cb..000000000000 --- a/airbyte-integrations/connectors/source-openweather/unit_tests/test_source.py +++ /dev/null @@ -1,58 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -import pytest -from source_openweather.source import SourceOpenWeather -from source_openweather.streams import OneCall - - -@pytest.mark.parametrize( - "response_status", - [200, 400], -) -def test_check_connection(mocker, response_status): - validate_mock = mocker.patch("source_openweather.extra_validations.validate") - validate_mock.return_value = {"appid": "test_appid", "lat": 1.0, "lon": 1.0, "lang": None, "units": None} - requests_get_mock = mocker.patch("source_openweather.source.requests.get") - requests_get_mock.return_value.status_code = response_status - logger_mock = MagicMock() - config_mock = MagicMock() - - source = SourceOpenWeather() - if response_status == 200: - assert source.check_connection(logger_mock, config_mock) == (True, None) - else: - assert source.check_connection(logger_mock, config_mock) == (False, requests_get_mock.return_value.json.return_value.get("message")) - validate_mock.assert_called_with(config_mock) - requests_get_mock.assert_called_with( - "https://api.openweathermap.org/data/3.0/onecall", params={"appid": "test_appid", "lat": 1.0, "lon": 1.0} - ) - - -def test_check_connection_validation_error(mocker): - validate_mock = mocker.patch("source_openweather.extra_validations.validate") - error = Exception("expected message") - validate_mock.side_effect = error - logger_mock = MagicMock() - - source = SourceOpenWeather() - assert source.check_connection(logger_mock, {}) == (False, error) - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(OneCall, "__abstractmethods__", set()) - - -def test_streams(patch_base_class, mocker): - config_mock = MagicMock() - validate_mock = mocker.patch("source_openweather.source.extra_validations.validate") - source = SourceOpenWeather() - streams = source.streams(config_mock) - expected_streams_number = 1 - assert len(streams) == expected_streams_number - validate_mock.assert_called_with(config_mock) diff --git a/airbyte-integrations/connectors/source-openweather/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-openweather/unit_tests/test_streams.py deleted file mode 100644 index c9fad79f3acf..000000000000 --- a/airbyte-integrations/connectors/source-openweather/unit_tests/test_streams.py +++ /dev/null @@ -1,68 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_openweather.streams import OneCall - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(OneCall, "__abstractmethods__", set()) - - -@pytest.mark.parametrize( - ("stream", "expected_params"), - [ - (OneCall(appid="test_appid", lat=1.0, lon=1.0), {"appid": "test_appid", "lat": 1.0, "lon": 1.0}), - ( - OneCall(appid="test_appid", lat=1.0, lon=1.0, lang=None, units=None), - {"appid": "test_appid", "lat": 1.0, "lon": 1.0}, - ), - ( - OneCall(appid="test_appid", lat=1.0, lon=1.0, lang="fr", units="metric"), - {"appid": "test_appid", "lat": 1.0, "lon": 1.0, "lang": "fr", "units": "metric"}, - ), - ], -) -def test_request_params(stream, expected_params): - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - assert stream.request_params(**inputs) == expected_params - - -@pytest.mark.parametrize( - ("response_data", "stream_state", "expect_record"), - [ - ({"current": {"dt": 1}}, {}, True), - ({"current": {"dt": 2}}, {"dt": 1}, True), - ({"current": {"dt": 1}}, {"dt": 2}, False), - ], -) -def test_parse_response(patch_base_class, response_data, stream_state, expect_record): - stream = OneCall(appid="test_appid", lat=1.0, lon=1.0) - response_mock = MagicMock() - response_mock.json.return_value = response_data - if expect_record: - assert stream.parse_response(response=response_mock, stream_state=stream_state) == [response_mock.json.return_value] - else: - assert stream.parse_response(response=response_mock, stream_state=stream_state) == [] - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, False), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = OneCall(appid="test_appid", lat=1.0, lon=1.0) - assert stream.should_retry(response_mock) == should_retry diff --git a/airbyte-integrations/connectors/source-opsgenie/Dockerfile b/airbyte-integrations/connectors/source-opsgenie/Dockerfile index e8f4f1c04d6b..b8cc08291b97 100644 --- a/airbyte-integrations/connectors/source-opsgenie/Dockerfile +++ b/airbyte-integrations/connectors/source-opsgenie/Dockerfile @@ -34,5 +34,5 @@ COPY source_opsgenie ./source_opsgenie ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.3.0 LABEL io.airbyte.name=airbyte/source-opsgenie diff --git a/airbyte-integrations/connectors/source-opsgenie/README.md b/airbyte-integrations/connectors/source-opsgenie/README.md index a93a63618b13..d76b42663d15 100644 --- a/airbyte-integrations/connectors/source-opsgenie/README.md +++ b/airbyte-integrations/connectors/source-opsgenie/README.md @@ -1,35 +1,10 @@ # Opsgenie Source -This is the repository for the Opsgenie source connector, written in Python. +This is the repository for the Opsgenie source connector, written in low-code configuration based source connector. For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/opsgenie). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - #### Building via Gradle You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. @@ -57,18 +32,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-opsgenie:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-opsgenie build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-opsgenie:airbyteDocker +An image will be built with the tag `airbyte/source-opsgenie:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-opsgenie:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +54,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-opsgenie:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-opsgenie:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-opsgenie:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-opsgenie test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-opsgenie:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-opsgenie:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,16 +73,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-opsgenie test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/opsgenie.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. -## Code Linting - -Before committing code and raising PRs, make sure you run the linting process from the repo root to make sure that everything's ok: - -```bash -./gradlew :airbyte-integrations:connectors:source-opsgenie:airbytePythonFormat -``` diff --git a/airbyte-integrations/connectors/source-opsgenie/acceptance-test-config.yml b/airbyte-integrations/connectors/source-opsgenie/acceptance-test-config.yml index eedc90b65a7b..bc4e8a53b0c2 100644 --- a/airbyte-integrations/connectors/source-opsgenie/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-opsgenie/acceptance-test-config.yml @@ -14,25 +14,11 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [ - "alerts", - "alert_recipients", - "alert_logs", - "incidents", - "integrations", - "teams", - "user_teams", - "services", - ] - incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - # future_state_path: "integration_tests/abnormal_state.json" - threshold_days: 2 - cursor_paths: - alerts: ["updatedAt"] - alert_recipients: ["updatedAt"] - incidents: ["updatedAt"] + empty_streams: ["incidents", "services"] + # incremental: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-opsgenie/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-opsgenie/acceptance-test-docker.sh deleted file mode 100644 index c51577d10690..000000000000 --- a/airbyte-integrations/connectors/source-opsgenie/acceptance-test-docker.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env sh - -# Build latest connector image -docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) - -# Pull latest acctest image -docker pull airbyte/source-acceptance-test:latest - -# Run -docker run --rm -it \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /tmp:/tmp \ - -v $(pwd):/test_input \ - airbyte/source-acceptance-test \ - --acceptance-test-config /test_input - diff --git a/airbyte-integrations/connectors/source-opsgenie/build.gradle b/airbyte-integrations/connectors/source-opsgenie/build.gradle deleted file mode 100644 index be2691806589..000000000000 --- a/airbyte-integrations/connectors/source-opsgenie/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_opsgenie' -} diff --git a/airbyte-integrations/connectors/source-opsgenie/icon.svg b/airbyte-integrations/connectors/source-opsgenie/icon.svg new file mode 100644 index 000000000000..5e71bf59cdde --- /dev/null +++ b/airbyte-integrations/connectors/source-opsgenie/icon.svg @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-opsgenie/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-opsgenie/integration_tests/acceptance.py index 37dfff891923..5a11cd9ef288 100644 --- a/airbyte-integrations/connectors/source-opsgenie/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-opsgenie/integration_tests/acceptance.py @@ -5,8 +5,6 @@ import pytest -pytest_plugins = ("source_acceptance_test.plugin",) - @pytest.fixture(scope="session", autouse=True) def connector_setup(): diff --git a/airbyte-integrations/connectors/source-opsgenie/metadata.yaml b/airbyte-integrations/connectors/source-opsgenie/metadata.yaml index 44359e08afed..17edbf7b36e5 100644 --- a/airbyte-integrations/connectors/source-opsgenie/metadata.yaml +++ b/airbyte-integrations/connectors/source-opsgenie/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: 06bdb480-2598-40b8-8b0f-fc2e2d2abdda - dockerImageTag: 0.1.0 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-opsgenie githubIssueLabel: source-opsgenie license: MIT @@ -15,7 +15,7 @@ data: releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/opsgenie tags: - - language:python + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-opsgenie/requirements.txt b/airbyte-integrations/connectors/source-opsgenie/requirements.txt index 0411042aa091..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-opsgenie/requirements.txt +++ b/airbyte-integrations/connectors/source-opsgenie/requirements.txt @@ -1,2 +1 @@ --e ../../bases/source-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-opsgenie/setup.py b/airbyte-integrations/connectors/source-opsgenie/setup.py index 2b029a855e75..48fa326b15b9 100644 --- a/airbyte-integrations/connectors/source-opsgenie/setup.py +++ b/airbyte-integrations/connectors/source-opsgenie/setup.py @@ -5,15 +5,12 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", -] +MAIN_REQUIREMENTS = ["airbyte-cdk"] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", - "source-acceptance-test", "responses~=0.19.0", ] diff --git a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/manifest.yaml b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/manifest.yaml new file mode 100644 index 000000000000..a1c6b1e07e72 --- /dev/null +++ b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/manifest.yaml @@ -0,0 +1,232 @@ +version: 0.51.16 +type: DeclarativeSource +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + auth: + type: ApiKeyAuthenticator + api_token: "GenieKey {{ config['api_token'] }}" + inject_into: + type: RequestOption + field_name: Authorization + inject_into: header + on_error: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + backoff_strategies: + - type: WaitUntilTimeFromHeader + header: X-RateLimit-Period-In-Sec + pagination: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("paging", {}).get("next", {}) }}' + stop_condition: '{{ not response.get("paging", {}).get("next", {}) }}' + + requester: + type: HttpRequester + url_base: "https://{{ config['endpoint'] }}" + http_method: GET + request_parameters: {} + request_headers: + Accept: application/json + authenticator: + $ref: "#/definitions/auth" + error_handler: + $ref: "#/definitions/on_error" + request_body_json: {} + + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/pagination" + + base_stream: + schema_loader: + type: JsonFileSchemaLoader + file_path: "./source_opsgenie/schemas/{{ parameters['name'] }}.json" + retriever: + $ref: "#/definitions/retriever" + + users_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "users" + primary_key: "id" + path: "v2/users" + + teams_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "teams" + primary_key: "id" + path: "v2/teams" + + services_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "services" + primary_key: "id" + path: "v1/services" + + integrations_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "integrations" + primary_key: "id" + path: "v2/integrations" + + incidents_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "incidents" + primary_key: "id" + path: "v1/incidents" + + alerts_stream: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/base_stream/retriever" + requester: + $ref: "#/definitions/requester" + request_parameters: + order: "asc" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updatedAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%fZ" + datetime_format: "%s" + start_datetime: + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + $parameters: + name: "alerts" + primary_key: "id" + path: "v2/alerts" + + user_teams_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: user_teams + primary_key: "id" + retriever: + $ref: "#/definitions/base_stream/retriever" + requester: + $ref: "#/definitions/requester" + path: "v2/users/{{ stream_partition['user_id'] }}/teams" + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: user_id + stream: + $ref: "#/definitions/users_stream" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: ["user_id"] + value: "{{ stream_partition['id'] }}" + + alert_recipients_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "alert_recipients" + primary_key: "user_id" + path: "v2/alerts/{{ record }}/recipients" + retriever: + $ref: "#/definitions/base_stream/retriever" + requester: + $ref: "#/definitions/requester" + path: "v2/alerts/{{ stream_partition['alert_id'] }}/recipients" + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: alert_id + stream: + $ref: "#/definitions/alerts_stream" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: ["alert_id"] + value: "{{ stream_partition['id'] }}" + - type: AddFields + fields: + - type: AddedFieldDefinition + path: ["user_id"] + value: "{{ record['user']['id'] }}" + - type: AddFields + fields: + - type: AddedFieldDefinition + path: ["user_username"] + value: "{{ record['user']['username'] }}" + - type: RemoveFields + field_pointers: + - ["user"] + + alert_logs_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "alert_logs" + primary_key: "offset" + path: "v2/alerts/{{ stream_partition.alert_id }}/logs" + retriever: + $ref: "#/definitions/base_stream/retriever" + requester: + $ref: "#/definitions/requester" + request_parameters: + order: asc + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: alert_id + stream: + $ref: "#/definitions/alerts_stream" + transformations: + - type: AddFields + fields: + - type: AddedFieldDefinition + path: ["alert_id"] + value: "{{ stream_partition['id'] }}" + +check: + type: CheckStream + stream_names: + - users + # - teams + # - services + # - incidents + # - integrations + # - alerts + # - user_teams + # - alert_recipients + # - alert_logs + +streams: + - "#/definitions/users_stream" + - "#/definitions/teams_stream" + - "#/definitions/services_stream" + - "#/definitions/integrations_stream" + - "#/definitions/incidents_stream" + - "#/definitions/alerts_stream" + - "#/definitions/user_teams_stream" + - "#/definitions/alert_recipients_stream" + - "#/definitions/alert_logs_stream" diff --git a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/schemas/integrations.json b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/schemas/integrations.json index ec679cbd1599..9a8a388a1d64 100644 --- a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/schemas/integrations.json +++ b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/schemas/integrations.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "string"] @@ -19,6 +20,9 @@ }, "version": { "type": ["null", "string"] + }, + "advanced": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/source.py b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/source.py index 743694d15b54..c7fca3b2212f 100644 --- a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/source.py +++ b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/source.py @@ -2,54 +2,16 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from typing import Any, List, Mapping, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. +WARNING: Do not modify this file. +""" -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from .streams import AlertLogs, AlertRecipients, Alerts, Incidents, Integrations, Services, Teams, Users, UserTeams - - -# Source -class SourceOpsgenie(AbstractSource): - @staticmethod - def get_authenticator(config: Mapping[str, Any]): - return TokenAuthenticator(config["api_token"], auth_method="GenieKey") - - def check_connection(self, logger, config) -> Tuple[bool, any]: - - try: - auth = self.get_authenticator(config) - api_endpoint = f"https://{config['endpoint']}/v2/account" - - response = requests.get( - api_endpoint, - headers=auth.get_auth_header(), - ) - - return response.status_code == requests.codes.ok, None - - except Exception as error: - return False, f"Unable to connect to Opsgenie API with the provided credentials - {repr(error)}" - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - auth = self.get_authenticator(config) - args = {"authenticator": auth, "endpoint": config["endpoint"]} - incremental_args = {**args, "start_date": config.get("start_date", "")} - - users = Users(**args) - alerts = Alerts(**incremental_args) - return [ - alerts, - AlertRecipients(parent_stream=alerts, **args), - AlertLogs(parent_stream=alerts, **args), - Incidents(**incremental_args), - Integrations(**args), - Services(**args), - Teams(**args), - users, - UserTeams(parent_stream=users, **args), - ] +# Declarative Source +class SourceOpsgenie(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/spec.yaml b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/spec.yaml index 82bc07cca77d..b688f8b5b7e3 100644 --- a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/spec.yaml +++ b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/spec.yaml @@ -1,4 +1,4 @@ -documentationUrl: https://docsurl.com +documentationUrl: https://docs.airbyte.com/integrations/sources/opsgenie connectionSpecification: $schema: http://json-schema.org/draft-07/schema# title: Opsgenie Spec diff --git a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/streams.py b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/streams.py deleted file mode 100644 index 6812e8427474..000000000000 --- a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/streams.py +++ /dev/null @@ -1,256 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import time -import urllib.parse as urlparse -from abc import ABC -from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional -from urllib.parse import parse_qs - -import pendulum -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http import HttpStream - - -# Basic full refresh stream -class OpsgenieStream(HttpStream, ABC): - - primary_key = "id" - api_version = "v2" - - flatten_id_keys = [] - flatten_list_keys = [] - - def __init__(self, endpoint: str, **kwargs): - super(OpsgenieStream, self).__init__(**kwargs) - self._endpoint = endpoint - - @property - def url_base(self) -> str: - return f"https://{self._endpoint}/{self.api_version}/" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - params = {} - data = response.json() - if "paging" in data and "next" in data["paging"]: - next_page = data["paging"]["next"] - params = parse_qs(urlparse.urlparse(next_page).query) - - return params - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params: Dict[str, str] = {} - - if next_page_token: - params.update(next_page_token) - - return params - - def request_headers(self, **kwargs) -> Mapping[str, Any]: - return {"Accept": "application/json"} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - response_data = response.json() - data = response_data["data"] - if isinstance(data, list): - for record in data: - yield self.transform(record, **kwargs) - elif isinstance(data, dict): - yield self.transform(data, **kwargs) - else: - Exception(f"Unsupported type of response data for stream {self.name}") - - def transform(self, record: Dict[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs): - for key in self.flatten_id_keys: - self._flatten_id(record, key) - - for key in self.flatten_list_keys: - self._flatten_list(record, key) - - return record - - def _flatten_id(self, record: Dict[str, Any], target: str): - target_value = record.pop(target, None) - record[target + "_id"] = target_value.get("id") if target_value else None - - def _flatten_list(self, record: Dict[str, Any], target: str): - record[target] = [target_data.get("id") for target_data in record.get(target, [])] - - def backoff_time(self, response: requests.Response) -> Optional[float]: - """ - This method is called if we run into the rate limit. - Opsgenie applies rate limits in both requests-per-minute and - requests-per-second buckets. The response will inform which - of these thresholds has been breached by returning a X-RateLimit-Period-In-Sec - header value of 60 and 1 respectively. We take this as the hint for how long - to wait before trying again. - - Rate Limits Docs: https://docs.opsgenie.com/docs/api-rate-limiting - """ - - if "X-RateLimit-Period-In-Sec" in response.headers: - return int(response.headers["X-RateLimit-Period-In-Sec"]) - else: - self.logger.info("X-RateLimit-Period-In-Sec header not found. Using default backoff value") - return 60 - - -class Teams(OpsgenieStream): - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "teams" - - -class Integrations(OpsgenieStream): - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "integrations" - - -class Users(OpsgenieStream): - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "users" - - -class Services(OpsgenieStream): - - api_version = "v1" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "services" - - -class OpsgenieChildStream(OpsgenieStream): - path_list = ["id"] - flatten_parent_id = False - - def __init__(self, parent_stream: OpsgenieStream, **kwargs): - super().__init__(**kwargs) - self.parent_stream = parent_stream - - @property - def path_template(self) -> str: - template = [self.parent_stream.name] + ["{" + path_key + "}" for path_key in self.path_list] - return "/".join(template + [self.name]) - - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - for slice in self.parent_stream.stream_slices(sync_mode=SyncMode.full_refresh): - for record in self.parent_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice): - yield {path_key: record[path_key] for path_key in self.path_list} - - def path(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> str: - return self.path_template.format(**{path_key: stream_slice[path_key] for path_key in self.path_list}) - - def transform(self, record: Dict[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs): - record = super().transform(record, stream_slice, **kwargs) - if self.flatten_parent_id: - record[f"{self.parent_stream.name[:-1]}_id"] = stream_slice["id"] - return record - - -class UserTeams(OpsgenieChildStream): - flatten_parent_id = True - path_template = "users/{id}/teams" - - -# Basic incremental stream -class IncrementalOpsgenieStream(OpsgenieStream, ABC): - def __init__(self, start_date, **kwargs): - super().__init__(**kwargs) - self.start_date = start_date - - state_checkpoint_interval = 100 - cursor_field = "updatedAt" - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - cursor_field = self.cursor_field - latest_record = latest_record.get(self.cursor_field) - - latest_record_date = pendulum.parse(latest_record) - stream_state = current_stream_state.get(cursor_field) - if stream_state: - return {cursor_field: str(max(latest_record_date, pendulum.parse(stream_state)))} - else: - return {cursor_field: str(latest_record_date)} - - def request_params( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state, stream_slice, next_page_token) - start_point = self.start_date - start_date_dt = pendulum.parse(start_point) - - state_value = stream_state.get(self.cursor_field) - if state_value: - state_value_dt = pendulum.parse(state_value) - start_date_dt = max(start_date_dt, state_value_dt) - start_date_dt = min(start_date_dt, pendulum.now()) - - dt_timestamp = int(time.mktime(start_date_dt.timetuple()) * 1000) - - params["query"] = [f"{self.cursor_field}>={dt_timestamp}"] - params["order"] = ["asc"] - - return params - - -class Alerts(IncrementalOpsgenieStream): - def path(self, **kwargs) -> str: - return "alerts" - - -class Incidents(IncrementalOpsgenieStream): - - api_version = "v1" - - def path(self, **kwargs) -> str: - return "incidents" - - -class AlertRecipients(OpsgenieChildStream): - flatten_parent_id = True - path_template = "alerts/{id}/recipients" - flatten_id_keys = ["user"] - primary_key = "user_id" - - -class AlertLogs(OpsgenieChildStream): - primary_key = "offset" - cursor_field = "offset" - path_template = "alerts/{id}/logs" - flatten_parent_id = True - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - alert_id = latest_record.get("alert_id") - latest_cursor_value = latest_record.get(self.cursor_field) - current_state = current_stream_state.get(str(alert_id)) - if current_state: - current_state = current_state.get(self.cursor_field) - current_state_value = current_state or latest_cursor_value - max_value = max(current_state_value, latest_cursor_value) - current_stream_state[str(alert_id)] = {self.cursor_field: str(max_value)} - return current_stream_state - - def request_params( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - stream_state = stream_state or {} - params = super().request_params(stream_state, stream_slice, next_page_token) - - state_alert_value = stream_state.get(str(stream_slice["id"])) - if state_alert_value: - state_value = state_alert_value.get(self.cursor_field) - if state_value: - params[self.cursor_field] = [state_value] - params["order"] = ["asc"] - return params diff --git a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/util.py b/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/util.py deleted file mode 100644 index 32b939c7a213..000000000000 --- a/airbyte-integrations/connectors/source-opsgenie/source_opsgenie/util.py +++ /dev/null @@ -1,12 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from airbyte_cdk.models import SyncMode -from source_opsgenie.streams import OpsgenieStream - - -def read_full_refresh(stream_instance: OpsgenieStream): - slices = stream_instance.stream_slices(sync_mode=SyncMode.full_refresh) - for _slice in slices: - yield from stream_instance.read_records(stream_slice=_slice, sync_mode=SyncMode.full_refresh) diff --git a/airbyte-integrations/connectors/source-opsgenie/unit_tests/__init__.py b/airbyte-integrations/connectors/source-opsgenie/unit_tests/__init__.py deleted file mode 100644 index 1100c1c58cf5..000000000000 --- a/airbyte-integrations/connectors/source-opsgenie/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-opsgenie/unit_tests/test_source.py b/airbyte-integrations/connectors/source-opsgenie/unit_tests/test_source.py deleted file mode 100644 index dfb27745a5c7..000000000000 --- a/airbyte-integrations/connectors/source-opsgenie/unit_tests/test_source.py +++ /dev/null @@ -1,35 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import unittest -from unittest.mock import MagicMock - -import responses -from source_opsgenie.source import SourceOpsgenie - - -class SourceOpsgenieTest(unittest.TestCase): - def test_stream_count(self): - source = SourceOpsgenie() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 9 - self.assertEqual(len(streams), expected_streams_number) - - @responses.activate - def test_check_connection(self): - log_mock, _ = MagicMock(), MagicMock() - source = SourceOpsgenie() - - sample_account = { - "data": {"name": "opsgenie", "userCount": 1450, "plan": {"maxUserCount": 1500, "name": "Enterprise", "isYearly": True}}, - "took": 0.084, - "requestId": "e5122017-f5c5-4681-88ec-84e2898a61ad", - } - - responses.add("GET", "https://api.opsgenie.com/v2/account", json=sample_account) - - (success, err) = source.check_connection(log_mock, {"endpoint": "api.opsgenie.com", "api_token": "123"}) - self.assertTrue(success) - self.assertIsNone(err) diff --git a/airbyte-integrations/connectors/source-opsgenie/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-opsgenie/unit_tests/test_stream.py deleted file mode 100644 index 32003b46a58c..000000000000 --- a/airbyte-integrations/connectors/source-opsgenie/unit_tests/test_stream.py +++ /dev/null @@ -1,508 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import unittest - -import responses -from source_opsgenie.streams import AlertLogs, AlertRecipients, Alerts, Incidents, Integrations, Services, Teams, Users, UserTeams -from source_opsgenie.util import read_full_refresh - - -class TeamsStreamTestCase(unittest.TestCase): - @responses.activate - def test_teams_list(self): - config = {"endpoint": "api.opsgenie.com"} - stream = Teams(**config) - responses.add( - "GET", - "https://api.opsgenie.com/v2/teams", - json={ - "data": [ - {"id": "90098alp9-f0e3-41d3-a060-0ea895027630", "name": "ops_team", "description": ""}, - {"id": "a30alp45-65bf-422f-9d41-67b10a67282a", "name": "TeamName2", "description": "Description"}, - {"id": "c569c016-alp9-4e20-8a28-bd5dc33b798e", "name": "TeamName", "description": ""}, - ], - "took": 1.08, - "requestId": "9cbfalp7-53f5-41ef-a360-be01277a903d", - }, - ) - - records = list(read_full_refresh(stream)) - self.assertEqual(3, len(records)) - - -class UsersStreamTestCase(unittest.TestCase): - @responses.activate - def test_users_list(self): - config = {"endpoint": "api.opsgenie.com"} - stream = Users(**config) - responses.add( - "GET", - "https://api.opsgenie.com/v2/users", - json={ - "totalCount": 8, - "data": [ - { - "blocked": False, - "verified": False, - "id": "b5b92115-bfe7-43eb-8c2a-e467f2e5ddc4", - "username": "john.doe@opsgenie.com", - "fullName": "john doe", - "role": {"id": "Admin", "name": "Admin"}, - "timeZone": "Europe/Kirov", - "locale": "en_US", - "userAddress": {"country": "", "state": "", "city": "", "line": "", "zipCode": ""}, - "createdAt": "2017-05-12T08:34:30.283Z", - }, - { - "blocked": False, - "verified": False, - "id": "e07c63f0-dd8c-4ad4-983e-4ee7dc600463", - "username": "jane.doe@opsgenie.com", - "fullName": "jane doe", - "role": {"id": "Admin", "name": "Admin"}, - "timeZone": "Europe/Moscow", - "locale": "en_GB", - "tags": ["tag1", "tag3"], - "userAddress": { - "country": "US", - "state": "Indiana", - "city": "Terre Haute", - "line": "567 Stratford Park", - "zipCode": "47802", - }, - "details": {"detail1key": ["detail1dvalue1", "detail1value2"], "detail2key": ["detail2value"]}, - "createdAt": "2017-05-12T09:39:14.41Z", - }, - ], - "took": 0.261, - "requestId": "d2c50d0c-1c44-4fa5-99d4-20d1e7ca9938", - }, - ) - - records = list(read_full_refresh(stream)) - self.assertEqual(2, len(records)) - - -class ServicesStreamTestCase(unittest.TestCase): - @responses.activate - def test_services_list(self): - config = {"endpoint": "api.opsgenie.com"} - stream = Services(**config) - responses.add( - "GET", - "https://api.opsgenie.com/v1/services", - json={ - "data": [ - { - "teamId": "2e3c4c13-51e7-4cf6-a353-34c6f75494c7", - "name": "Service API Test Service - Updated", - "description": "Service API Test Service Description [Updated]", - "id": "6aa85159-9e2e-4e54-8088-546f9c15d513", - "tags": [], - "isExternal": False, - }, - { - "teamId": "2e3c4c13-51e7-4cf6-a353-34c6f75494c7", - "name": "Service API Test Service 2 - Updated", - "description": "Service API Test Service 2 Description [Updated]", - "id": "6aa85159-9e2e-4e54-8088-546f9c15d513", - "tags": [], - "isExternal": False, - }, - ], - "requestId": "656cfb15-e19f-11e7-ac88-af7c98633ff2", - }, - ) - - records = list(read_full_refresh(stream)) - self.assertEqual(2, len(records)) - - -class IntegrationsStreamTestCase(unittest.TestCase): - @responses.activate - def test_services_list(self): - config = {"endpoint": "api.opsgenie.com"} - stream = Integrations(**config) - responses.add( - "GET", - "https://api.opsgenie.com/v2/integrations", - json={ - "code": 200, - "data": [ - {"id": "055082dc-9427-48dd-85e0-f93a76e5f4a2", "name": "Signal Sciences", "enabled": True, "type": "SignalSciences"}, - {"id": "073e8e6a-a481-4b9b-8619-5c31d9a6e5da", "name": "Default API", "enabled": True, "type": "API"}, - {"id": "3163a9f9-5950-4e73-b99f-92562956e39c", "name": "Datadog", "enabled": False, "type": "Datadog"}, - {"id": "55e405e3-a130-4c7a-9866-664e498f39a9", "name": "Observium2", "enabled": False, "type": "ObserviumV2"}, - {"id": "72f6f51b-1ea9-4efd-be1b-4f29d1f593c6", "name": "Solarwinds", "enabled": False, "type": "Solarwinds"}, - {"id": "733388de-2ac1-4d70-8a2e-82834cb679d6", "name": "Webhook", "enabled": False, "type": "Webhook"}, - { - "id": "8418d193-2dab-4490-b331-8c02cdd196b7", - "name": "Marid", - "enabled": False, - "type": "Marid", - "teamId": "87311c02-edda-11eb-9a03-0242ac130003", - }, - ], - "took": 0, - "requestId": "9ceeb66b-9890-4687-9dbb-a38abc71eda3", - }, - ) - - records = list(read_full_refresh(stream)) - self.assertEqual(7, len(records)) - - -class UserTeamsStreamTestCase(unittest.TestCase): - @responses.activate - def test_user_teams_list(self): - config = {"endpoint": "api.opsgenie.com"} - users = Users(**config) - stream = UserTeams(parent_stream=users, **config) - responses.add( - "GET", - "https://api.opsgenie.com/v2/users", - json={ - "totalCount": 8, - "data": [ - { - "blocked": False, - "verified": False, - "id": "b5b92115-bfe7-43eb-8c2a-e467f2e5ddc4", - "username": "john.doe@opsgenie.com", - "fullName": "john doe", - "role": {"id": "Admin", "name": "Admin"}, - "timeZone": "Europe/Kirov", - "locale": "en_US", - "userAddress": {"country": "", "state": "", "city": "", "line": "", "zipCode": ""}, - "createdAt": "2017-05-12T08:34:30.283Z", - }, - { - "blocked": False, - "verified": False, - "id": "e07c63f0-dd8c-4ad4-983e-4ee7dc600463", - "username": "jane.doe@opsgenie.com", - "fullName": "jane doe", - "role": {"id": "Admin", "name": "Admin"}, - "timeZone": "Europe/Moscow", - "locale": "en_GB", - "tags": ["tag1", "tag3"], - "userAddress": { - "country": "US", - "state": "Indiana", - "city": "Terre Haute", - "line": "567 Stratford Park", - "zipCode": "47802", - }, - "details": {"detail1key": ["detail1dvalue1", "detail1value2"], "detail2key": ["detail2value"]}, - "createdAt": "2017-05-12T09:39:14.41Z", - }, - ], - "took": 0.261, - "requestId": "d2c50d0c-1c44-4fa5-99d4-20d1e7ca9938", - }, - ) - - responses.add( - "GET", - "https://api.opsgenie.com/v2/users/b5b92115-bfe7-43eb-8c2a-e467f2e5ddc4/teams", - json={ - "data": [{"id": "6fa6848c-8cac-4cea-8a98-ad9ff23d9b16", "name": "TeamName"}], - "took": 0.023, - "requestId": "bc40b7ad-11ee-4dcd-ae5f-6d75dbc16261", - }, - ) - - responses.add( - "GET", - "https://api.opsgenie.com/v2/users/e07c63f0-dd8c-4ad4-983e-4ee7dc600463/teams", - json={ - "data": [ - {"id": "6fa6848c-8cac-4cea-8a98-ad9ff23d9b16", "name": "TeamName"}, - {"id": "bc40b7ad-11ee-4dcd-ae5f-6d75dbc16261", "name": "TeamName"}, - ], - "took": 0.023, - "requestId": "bc40b7ad-11ee-4dcd-ae5f-6d75dbc16261", - }, - ) - - records = list(read_full_refresh(stream)) - self.assertEqual(3, len(records)) - - -class AlertsStreamTestCase(unittest.TestCase): - @responses.activate - def test_alerts_list(self): - config = {"endpoint": "api.opsgenie.com", "start_date": "2022-07-01T00:00:00Z"} - stream = Alerts(**config) - responses.add( - "GET", - "https://api.opsgenie.com/v2/alerts", - json={ - "data": [ - { - "id": "70413a06-38d6-4c85-92b8-5ebc900d42e2", - "tinyId": "1791", - "alias": "event_573", - "message": "Our servers are in danger", - "status": "closed", - "acknowledged": False, - "isSeen": True, - "tags": ["OverwriteQuietHours", "Critical"], - "snoozed": True, - "snoozedUntil": "2017-04-03T20:32:35.143Z", - "count": 79, - "lastOccurredAt": "2017-04-03T20:05:50.894Z", - "createdAt": "2017-03-21T20:32:52.353Z", - "updatedAt": "2017-04-03T20:32:57.301Z", - "source": "Isengard", - "owner": "morpheus@opsgenie.com", - "priority": "P4", - "responders": [ - {"id": "4513b7ea-3b91-438f-b7e4-e3e54af9147c", "type": "team"}, - {"id": "bb4d9938-c3c2-455d-aaab-727aa701c0d8", "type": "user"}, - {"id": "aee8a0de-c80f-4515-a232-501c0bc9d715", "type": "escalation"}, - {"id": "80564037-1984-4f38-b98e-8a1f662df552", "type": "schedule"}, - ], - "integration": {"id": "4513b7ea-3b91-438f-b7e4-e3e54af9147c", "name": "Nebuchadnezzar", "type": "API"}, - "report": { - "ackTime": 15702, - "closeTime": 60503, - "acknowledgedBy": "agent_smith@opsgenie.com", - "closedBy": "neo@opsgenie.com", - }, - }, - { - "id": "70413a06-38d6-4c85-92b8-5ebc900d42e2", - "tinyId": "1791", - "alias": "event_573", - "message": "Sample Message", - "status": "open", - "acknowledged": False, - "isSeen": False, - "tags": ["RandomTag"], - "snoozed": False, - "count": 1, - "lastOccurredAt": "2017-03-21T20:32:52.353Z", - "createdAt": "2017-03-21T20:32:52.353Z", - "updatedAt": "2017-04-03T20:32:57.301Z", - "source": "Zion", - "owner": "", - "priority": "P5", - "responders": [], - "integration": {"id": "4513b7ea-3b91-b7e4-438f-e3e54af9147c", "name": "My_Lovely_Amazon", "type": "CloudWatch"}, - }, - ], - "took": 0.605, - "requestId": "9ae63dd7-ed00-4c81-86f0-c4ffd33142c9", - }, - ) - - records = list(read_full_refresh(stream)) - self.assertEqual(2, len(records)) - - -class IncidentsStreamTestCase(unittest.TestCase): - @responses.activate - def test_alerts_list(self): - config = {"endpoint": "api.opsgenie.com", "start_date": "2022-07-01T00:00:00Z"} - stream = Incidents(**config) - responses.add( - "GET", - "https://api.opsgenie.com/v1/incidents", - json={ - "data": [ - { - "id": "70413a06-38d6-4c85-92b8-5ebc900d42e2", - "tinyId": "1791", - "message": "Our servers are in danger", - "status": "closed", - "tags": ["OverwriteQuietHours", "Critical"], - "createdAt": "2017-03-21T20:32:52.353Z", - "updatedAt": "2017-04-03T20:32:57.301Z", - "priority": "P4", - "responders": [ - {"type": "team", "id": "fc1448b7-46b2-401d-9df8-c02675958e3b"}, - {"type": "team", "id": "fe954a67-813e-4356-87dc-afed1eec6b66"}, - ], - "impactedServices": ["df635094-efd3-48e4-b73a-b8bdfbf1178f", "b6868288-02c7-440b-a693-0a5cf20576f5"], - }, - { - "id": "70413a06-38d6-4c85-92b8-5ebc900d42e2", - "tinyId": "1791", - "message": "Sample Message", - "status": "open", - "tags": ["RandomTag"], - "createdAt": "2017-03-21T20:32:52.353Z", - "updatedAt": "2017-04-03T20:32:57.301Z", - "priority": "P5", - "responders": [{"type": "team", "id": "fc1448b7-46b2-401d-9df8-c02675958e3b"}], - "impactedServices": [], - }, - ], - "took": 0.605, - "requestId": "9ae63dd7-ed00-4c81-86f0-c4ffd33142c9", - }, - ) - - records = list(read_full_refresh(stream)) - self.assertEqual(2, len(records)) - - -class AlertRecipientsStreamTestCase(unittest.TestCase): - @responses.activate - def test_alerts_list(self): - alerts_config = {"endpoint": "api.opsgenie.com", "start_date": "2022-07-01T00:00:00Z"} - config = { - "endpoint": "api.opsgenie.com", - } - stream = AlertRecipients(parent_stream=Alerts(**alerts_config), **config) - responses.add( - "GET", - "https://api.opsgenie.com/v2/alerts", - json={ - "data": [ - { - "id": "70413a06-38d6-4c85-92b8-5ebc900d42e2", - "tinyId": "1791", - "alias": "event_573", - "message": "Our servers are in danger", - "status": "closed", - "acknowledged": False, - "isSeen": True, - "tags": ["OverwriteQuietHours", "Critical"], - "snoozed": True, - "snoozedUntil": "2017-04-03T20:32:35.143Z", - "count": 79, - "lastOccurredAt": "2017-04-03T20:05:50.894Z", - "createdAt": "2017-03-21T20:32:52.353Z", - "updatedAt": "2017-04-03T20:32:57.301Z", - "source": "Isengard", - "owner": "morpheus@opsgenie.com", - "priority": "P4", - "responders": [ - {"id": "4513b7ea-3b91-438f-b7e4-e3e54af9147c", "type": "team"}, - {"id": "bb4d9938-c3c2-455d-aaab-727aa701c0d8", "type": "user"}, - {"id": "aee8a0de-c80f-4515-a232-501c0bc9d715", "type": "escalation"}, - {"id": "80564037-1984-4f38-b98e-8a1f662df552", "type": "schedule"}, - ], - "integration": {"id": "4513b7ea-3b91-438f-b7e4-e3e54af9147c", "name": "Nebuchadnezzar", "type": "API"}, - "report": { - "ackTime": 15702, - "closeTime": 60503, - "acknowledgedBy": "agent_smith@opsgenie.com", - "closedBy": "neo@opsgenie.com", - }, - } - ], - "took": 0.605, - "requestId": "9ae63dd7-ed00-4c81-86f0-c4ffd33142c9", - }, - ) - responses.add( - "GET", - "https://api.opsgenie.com/v2/alerts/70413a06-38d6-4c85-92b8-5ebc900d42e2/recipients", - json={ - "data": [ - { - "user": {"id": "2503a523-8ba5-4158-a4bd-7850074b5cca", "username": "neo@opsgenie.com"}, - "state": "action", - "method": "Acknowledge", - "createdAt": "2017-04-12T12:27:28.52Z", - "updatedAt": "2017-04-12T12:27:52.86Z", - }, - { - "user": {"id": "0966cfd8-fc9a-4f5c-a013-7d1f9318aef8", "username": "trinity@opsgenie.com"}, - "state": "notactive", - "method": "", - "createdAt": "2017-04-12T12:27:28.571Z", - "updatedAt": "2017-04-12T12:27:28.589Z", - }, - ], - "took": 0.605, - "requestId": "9ae63dd7-ed00-4c81-86f0-c4ffd33142c9", - }, - ) - - records = list(read_full_refresh(stream)) - self.assertEqual(2, len(records)) - - -class AlertLogsStreamTestCase(unittest.TestCase): - @responses.activate - def test_alerts_list(self): - alerts_config = {"endpoint": "api.opsgenie.com", "start_date": "2022-07-01T00:00:00Z"} - config = { - "endpoint": "api.opsgenie.com", - } - stream = AlertLogs(parent_stream=Alerts(**alerts_config), **config) - responses.add( - "GET", - "https://api.opsgenie.com/v2/alerts", - json={ - "data": [ - { - "id": "70413a06-38d6-4c85-92b8-5ebc900d42e2", - "tinyId": "1791", - "alias": "event_573", - "message": "Our servers are in danger", - "status": "closed", - "acknowledged": False, - "isSeen": True, - "tags": ["OverwriteQuietHours", "Critical"], - "snoozed": True, - "snoozedUntil": "2017-04-03T20:32:35.143Z", - "count": 79, - "lastOccurredAt": "2017-04-03T20:05:50.894Z", - "createdAt": "2017-03-21T20:32:52.353Z", - "updatedAt": "2017-04-03T20:32:57.301Z", - "source": "Isengard", - "owner": "morpheus@opsgenie.com", - "priority": "P4", - "responders": [ - {"id": "4513b7ea-3b91-438f-b7e4-e3e54af9147c", "type": "team"}, - {"id": "bb4d9938-c3c2-455d-aaab-727aa701c0d8", "type": "user"}, - {"id": "aee8a0de-c80f-4515-a232-501c0bc9d715", "type": "escalation"}, - {"id": "80564037-1984-4f38-b98e-8a1f662df552", "type": "schedule"}, - ], - "integration": {"id": "4513b7ea-3b91-438f-b7e4-e3e54af9147c", "name": "Nebuchadnezzar", "type": "API"}, - "report": { - "ackTime": 15702, - "closeTime": 60503, - "acknowledgedBy": "agent_smith@opsgenie.com", - "closedBy": "neo@opsgenie.com", - }, - } - ], - "took": 0.605, - "requestId": "9ae63dd7-ed00-4c81-86f0-c4ffd33142c9", - }, - ) - responses.add( - "GET", - "https://api.opsgenie.com/v2/alerts/70413a06-38d6-4c85-92b8-5ebc900d42e2/logs", - json={ - "data": [ - { - "log": "Alert acknowledged via web", - "type": "system", - "owner": "neo@opsgenie.com", - "createdAt": "2017-04-12T12:27:52.838Z", - "offset": "1492000072838_1492000072838234593", - }, - { - "log": "Viewed on [web]", - "type": "alertRecipient", - "owner": "trinity@opsgenie.com", - "createdAt": "2017-04-12T12:27:46.379Z", - "offset": "1492000066378_1492000066379000127", - }, - ], - "took": 0.605, - "requestId": "9ae63dd7-ed00-4c81-86f0-c4ffd33142c9", - }, - ) - - records = list(read_full_refresh(stream)) - self.assertEqual(2, len(records)) diff --git a/airbyte-integrations/connectors/source-opsgenie/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-opsgenie/unit_tests/unit_test.py deleted file mode 100644 index 9042065c8925..000000000000 --- a/airbyte-integrations/connectors/source-opsgenie/unit_tests/unit_test.py +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import unittest - -from airbyte_cdk.sources.streams.http.requests_native_auth.token import TokenAuthenticator -from source_opsgenie import SourceOpsgenie - - -class AuthenticatorTestCase(unittest.TestCase): - def test_token(self): - authenticator = SourceOpsgenie.get_authenticator({"api_token": "123"}) - self.assertIsInstance(authenticator, TokenAuthenticator) - self.assertEqual("GenieKey 123", authenticator.token) - self.assertEqual("Authorization", authenticator.auth_header) diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/.dockerignore b/airbyte-integrations/connectors/source-oracle-strict-encrypt/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile deleted file mode 100644 index 9c613f848705..000000000000 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-oracle-strict-encrypt - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-oracle-strict-encrypt -ENV TZ UTC - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.4.0 -LABEL io.airbyte.name=airbyte/source-oracle-strict-encrypt diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/acceptance-test-config.yml b/airbyte-integrations/connectors/source-oracle-strict-encrypt/acceptance-test-config.yml deleted file mode 100644 index bfd298afacb1..000000000000 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/acceptance-test-config.yml +++ /dev/null @@ -1,6 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-oracle-strict-encrypt:dev -tests: - spec: - - spec_path: "src/test/resources/expected_spec.json" diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-oracle-strict-encrypt/build.gradle index af6d4228c3d1..b9ee027a048c 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/build.gradle +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/build.gradle @@ -1,37 +1,39 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.7.7' + features = ['db-sources'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.source.oracle_strict_encrypt.OracleStrictEncryptSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { + implementation project(':airbyte-integrations:connectors:source-oracle') // required so that log4j uses a standard xml parser instead of an oracle one (that gets pulled in by the oracle driver) implementation group: 'xerces', name: 'xercesImpl', version: '2.12.1' - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:connectors:source-oracle') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation project(':airbyte-integrations:connectors:source-relational-db') implementation "com.oracle.database.jdbc:ojdbc8-production:19.7.0.0" - testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) - testImplementation project(':airbyte-test-utils') - testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation libs.connectors.source.testcontainers.oracle.xe + testImplementation libs.testcontainers.oracle.xe - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-oracle-strict-encrypt') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/source-oracle-strict-encrypt/metadata.yaml index 1903adbfff84..657ce07d10c5 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/metadata.yaml +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/metadata.yaml @@ -11,7 +11,7 @@ data: connectorSubtype: database connectorType: source definitionId: b39a7370-74c3-45a6-ac3a-380d48520a83 - dockerImageTag: 0.4.0 + dockerImageTag: 0.5.0 dockerRepository: airbyte/source-oracle-strict-encrypt githubIssueLabel: source-oracle icon: oracle.svg diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/main/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSource.java b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/main/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSource.java index 31549ea770c0..666c7cc5e5cf 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/main/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSource.java +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/main/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSource.java @@ -5,10 +5,10 @@ package io.airbyte.integrations.source.oracle_strict_encrypt; import com.fasterxml.jackson.databind.node.ArrayNode; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.base.spec_modification.SpecModifyingSource; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.base.spec_modification.SpecModifyingSource; import io.airbyte.integrations.source.oracle.OracleSource; import io.airbyte.protocol.models.v0.ConnectorSpecification; import org.slf4j.Logger; diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleSourceNneAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleSourceNneAcceptanceTest.java index 0c068f41cefd..fa2cfdc59fbc 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleSourceNneAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleSourceNneAcceptanceTest.java @@ -10,18 +10,23 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; import java.sql.SQLException; +import java.time.Duration; import java.util.List; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +@Disabled public class OracleSourceNneAcceptanceTest extends OracleStrictEncryptSourceAcceptanceTest { + private static final Duration CONNECTION_TIMEOUT = Duration.ofSeconds(60); + @Test public void testEncryption() throws SQLException { final ObjectNode clone = (ObjectNode) Jsons.clone(getConfig()); @@ -44,7 +49,8 @@ public void testEncryption() throws SQLException { clone.get("sid").asText()), JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED&" + "oracle.net.encryption_types_client=( " - + algorithm + " )"))); + + algorithm + " )"), + CONNECTION_TIMEOUT)); final String networkServiceBanner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; @@ -75,7 +81,8 @@ public void testCheckProtocol() throws SQLException { clone.get(JdbcUtils.PORT_KEY).asInt(), clone.get("sid").asText()), JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" + - "oracle.net.encryption_types_client=( " + algorithm + " )", ";"))); + "oracle.net.encryption_types_client=( " + algorithm + " )", ";"), + CONNECTION_TIMEOUT)); final String networkServiceBanner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual"; final List collect = database.queryJsons(networkServiceBanner); diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptJdbcSourceAcceptanceTest.java index b06adfdd02dd..64c45075348a 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptJdbcSourceAcceptanceTest.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.source.oracle_strict_encrypt; -import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifier; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.enquoteIdentifier; import static org.junit.Assert.assertEquals; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -14,31 +14,30 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.base.ssh.SshHelpers; +import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbState; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; -import io.airbyte.integrations.source.oracle.OracleSource; -import io.airbyte.integrations.source.relationaldb.models.DbState; -import io.airbyte.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteMessage; +import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; +import io.airbyte.protocol.models.v0.AirbyteStreamState; import io.airbyte.protocol.models.v0.CatalogHelpers; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConnectorSpecification; import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.StreamDescriptor; import io.airbyte.protocol.models.v0.SyncMode; import java.math.BigDecimal; import java.sql.Connection; import java.sql.DriverManager; -import java.sql.JDBCType; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -47,18 +46,24 @@ import java.util.List; import java.util.Optional; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -class OracleStrictEncryptJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { +@Disabled +class OracleStrictEncryptJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { private static final Logger LOGGER = LoggerFactory.getLogger(OracleStrictEncryptJdbcSourceAcceptanceTest.class); - private static AirbyteOracleTestContainer ORACLE_DB; + private static final AirbyteOracleTestContainer ORACLE_DB = new AirbyteOracleTestContainer() + .withEnv("NLS_DATE_FORMAT", "YYYY-MM-DD") + .withEnv("RELAX_SECURITY", "1") + .withUsername("TEST_ORA") + .withPassword("oracle") + .usingSid() + .withEnv("RELAX_SECURITY", "1"); @BeforeAll static void init() { @@ -88,62 +93,21 @@ static void init() { CREATE_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s VARCHAR(20))"; INSERT_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES('Hello world :)')"; INSERT_TABLE_NAME_AND_TIMESTAMP_QUERY = "INSERT INTO %s (name, timestamp) VALUES ('%s', TO_TIMESTAMP('%s', 'YYYY-MM-DD HH24:MI:SS'))"; - - ORACLE_DB = new AirbyteOracleTestContainer() - .withUsername("test") - .withPassword("oracle") - .usingSid() - .withEnv("NLS_DATE_FORMAT", "YYYY-MM-DD") - .withEnv("RELAX_SECURITY", "1"); - ORACLE_DB.start(); - } - - @BeforeEach - public void setup() throws Exception { - config = Jsons.jsonNode(ImmutableMap.builder() - .put("host", ORACLE_DB.getHost()) - .put("port", ORACLE_DB.getFirstMappedPort()) - .put("sid", ORACLE_DB.getSid()) - .put("username", ORACLE_DB.getUsername()) - .put("password", ORACLE_DB.getPassword()) - .put("schemas", List.of(SCHEMA_NAME, SCHEMA_NAME2)) - .put("encryption", Jsons.jsonNode(ImmutableMap.builder() - .put("encryption_method", "client_nne") - .put("encryption_algorithm", "3DES168") - .build())) - .build()); - - // Because Oracle doesn't let me create database easily I need to clean up - cleanUpTables(); - - super.setup(); - } - - @AfterEach - public void tearDownOracle() throws Exception { - // ORA-12519 - // https://stackoverflow.com/questions/205160/what-can-cause-intermittent-ora-12519-tns-no-appropriate-handler-found-errors - // sleep for 1000 - executeOracleStatement(String.format("DROP TABLE %s", getFullyQualifiedTableName(TABLE_NAME))); - executeOracleStatement( - String.format("DROP TABLE %s", getFullyQualifiedTableName(TABLE_NAME_WITHOUT_PK))); - executeOracleStatement( - String.format("DROP TABLE %s", getFullyQualifiedTableName(TABLE_NAME_COMPOSITE_PK))); - Thread.sleep(1000); } + @Override protected void incrementalDateCheck() throws Exception { // https://stackoverflow.com/questions/47712930/resultset-meta-data-return-timestamp-instead-of-date-oracle-jdbc // Oracle DATE is a java.sql.Timestamp (java.sql.Types.TIMESTAMP) as far as JDBC (and the SQL // standard) is concerned as it has both a date and time component. incrementalCursorCheck( COL_UPDATED_AT, - "2005-10-18T00:00:00.000000Z", - "2006-10-19T00:00:00.000000Z", + "2005-10-18T00:00:00.000000", + "2006-10-19T00:00:00.000000", Lists.newArrayList(getTestMessages().get(1), getTestMessages().get(2))); } - void cleanUpTables() throws SQLException { + static void cleanUpTables() throws SQLException { final Connection connection = DriverManager.getConnection( ORACLE_DB.getJdbcUrl(), ORACLE_DB.getUsername(), @@ -164,29 +128,25 @@ void cleanUpTables() throws SQLException { } @Override - public boolean supportsSchemas() { - // See https://www.oratable.com/oracle-user-schema-difference/ - return true; + protected OracleStrictEncryptTestDatabase createTestDatabase() { + ORACLE_DB.start(); + return new OracleStrictEncryptTestDatabase(ORACLE_DB, List.of(SCHEMA_NAME, SCHEMA_NAME2)).initialized(); } @Override - public AbstractJdbcSource getJdbcSource() { - return new OracleSource(); + public boolean supportsSchemas() { + // See https://www.oratable.com/oracle-user-schema-difference/ + return true; } @Override - public Source getSource() { + protected OracleStrictEncryptSource source() { return new OracleStrictEncryptSource(); } @Override - public JsonNode getConfig() { - return config; - } - - @Override - public String getDriverClass() { - return OracleSource.DRIVER_CLASS; + public JsonNode config() { + return Jsons.clone(testdb.configBuilder().build()); } @AfterAll @@ -195,7 +155,7 @@ static void cleanUp() { } @Override - public void createSchemas() throws SQLException { + public void createSchemas() { // In Oracle, `CREATE USER` creates a schema. // See https://www.oratable.com/oracle-user-schema-difference/ if (supportsSchemas()) { @@ -208,9 +168,13 @@ public void createSchemas() throws SQLException { } } - @Override - protected String getJdbcParameterDelimiter() { - return ";"; + static void cleanUpTablesAndWait() { + try { + cleanUpTables(); + Thread.sleep(1000); + } catch (final Exception e) { + throw new RuntimeException(e); + } } public void executeOracleStatement(final String query) { @@ -267,12 +231,13 @@ public static boolean ignoreSQLException(final String sqlState) { @Test void testSpec() throws Exception { - final ConnectorSpecification actual = source.spec(); + final ConnectorSpecification actual = source().spec(); final ConnectorSpecification expected = SshHelpers.injectSshIntoSpec(Jsons.deserialize(MoreResources.readResource("expected_spec.json"), ConnectorSpecification.class)); assertEquals(expected, actual); } + @Override protected AirbyteCatalog getCatalog(final String defaultNamespace) { return new AirbyteCatalog().withStreams(List.of( CatalogHelpers.createAirbyteStream( @@ -305,32 +270,33 @@ protected AirbyteCatalog getCatalog(final String defaultNamespace) { @Override protected List getTestMessages() { return Lists.newArrayList( - new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName) + new AirbyteMessage().withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName()) .withNamespace(getDefaultNamespace()) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_1, COL_NAME, "picard", - COL_UPDATED_AT, "2004-10-19T00:00:00.000000Z")))), - new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName) + COL_UPDATED_AT, "2004-10-19T00:00:00.000000")))), + new AirbyteMessage().withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName()) .withNamespace(getDefaultNamespace()) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_2, COL_NAME, "crusher", COL_UPDATED_AT, - "2005-10-19T00:00:00.000000Z")))), - new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName) + "2005-10-19T00:00:00.000000")))), + new AirbyteMessage().withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName()) .withNamespace(getDefaultNamespace()) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_3, COL_NAME, "vash", - COL_UPDATED_AT, "2006-10-19T00:00:00.000000Z"))))); + COL_UPDATED_AT, "2006-10-19T00:00:00.000000"))))); } @Test - void testReadOneTableIncrementallyTwice() throws Exception { + @Override + protected void testReadOneTableIncrementallyTwice() throws Exception { final String namespace = getDefaultNamespace(); final ConfiguredAirbyteCatalog configuredCatalog = getConfiguredCatalogWithOneStream(namespace); configuredCatalog.getStreams().forEach(airbyteStream -> { @@ -340,50 +306,54 @@ void testReadOneTableIncrementallyTwice() throws Exception { }); final DbState state = new DbState() - .withStreams(Lists.newArrayList(new DbStreamState().withStreamName(streamName).withStreamNamespace(namespace))); + .withStreams(Lists.newArrayList(new DbStreamState().withStreamName(streamName()).withStreamNamespace(namespace))); final List actualMessagesFirstSync = MoreIterators - .toList(source.read(config, configuredCatalog, Jsons.jsonNode(state))); + .toList(source().read(config(), configuredCatalog, Jsons.jsonNode(state))); final Optional stateAfterFirstSyncOptional = actualMessagesFirstSync.stream() - .filter(r -> r.getType() == AirbyteMessage.Type.STATE).findFirst(); + .filter(r -> r.getType() == Type.STATE).findFirst(); assertTrue(stateAfterFirstSyncOptional.isPresent()); - database.execute(connection -> { - connection.createStatement().execute( - String.format("INSERT INTO %s(id, name, updated_at) VALUES (4,'riker', '2006-10-19')", - getFullyQualifiedTableName(TABLE_NAME))); - connection.createStatement().execute( - String.format("INSERT INTO %s(id, name, updated_at) VALUES (5, 'data', '2006-10-19')", - getFullyQualifiedTableName(TABLE_NAME))); - }); + testdb.with(String.format("INSERT INTO %s(id, name, updated_at) VALUES (4,'riker', '2006-10-19')", + getFullyQualifiedTableName(TABLE_NAME))); + testdb.with(String.format("INSERT INTO %s(id, name, updated_at) VALUES (5, 'data', '2006-10-19')", + getFullyQualifiedTableName(TABLE_NAME))); final List actualMessagesSecondSync = MoreIterators - .toList(source.read(config, configuredCatalog, + .toList(source().read(config(), configuredCatalog, stateAfterFirstSyncOptional.get().getState().getData())); Assertions.assertEquals(2, - (int) actualMessagesSecondSync.stream().filter(r -> r.getType() == AirbyteMessage.Type.RECORD).count()); + (int) actualMessagesSecondSync.stream().filter(r -> r.getType() == Type.RECORD).count()); final List expectedMessages = new ArrayList<>(); - expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName()).withNamespace(namespace) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_4, COL_NAME, "riker", - COL_UPDATED_AT, "2006-10-19T00:00:00.000000Z"))))); - expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + COL_UPDATED_AT, "2006-10-19T00:00:00.000000"))))); + expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) + .withRecord(new AirbyteRecordMessage().withStream(streamName()).withNamespace(namespace) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_5, COL_NAME, "data", - COL_UPDATED_AT, "2006-10-19T00:00:00.000000Z"))))); + COL_UPDATED_AT, "2006-10-19T00:00:00.000000"))))); expectedMessages.add(new AirbyteMessage() - .withType(AirbyteMessage.Type.STATE) + .withType(Type.STATE) .withState(new AirbyteStateMessage() - .withType(AirbyteStateMessage.AirbyteStateType.LEGACY) + .withType(AirbyteStateMessage.AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withName(streamName()).withNamespace(namespace)) + .withStreamState(Jsons.jsonNode(new DbStreamState() + .withStreamNamespace(namespace) + .withStreamName(streamName()) + .withCursorField(ImmutableList.of(COL_ID)) + .withCursor("5") + .withCursorRecordCount(1L)))) .withData(Jsons.jsonNode(new DbState() .withCdc(false) .withStreams(Lists.newArrayList(new DbStreamState() - .withStreamName(streamName) + .withStreamName(streamName()) .withStreamNamespace(namespace) .withCursorField(ImmutableList.of(COL_ID)) .withCursor("5") @@ -400,8 +370,8 @@ void testReadOneTableIncrementallyTwice() throws Exception { void testIncrementalTimestampCheckCursor() throws Exception { incrementalCursorCheck( COL_UPDATED_AT, - "2005-10-18T00:00:00.000000Z", - "2006-10-19T00:00:00.000000Z", + "2005-10-18T00:00:00.000000", + "2006-10-19T00:00:00.000000", Lists.newArrayList(getTestMessages().get(1), getTestMessages().get(2))); } diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSourceAcceptanceTest.java index d9ecc0deb924..0fc32d0c865a 100644 --- a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptSourceAcceptanceTest.java @@ -7,16 +7,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.ssh.SshHelpers; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.CatalogHelpers; @@ -24,14 +24,18 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.ConnectorSpecification; import io.airbyte.protocol.models.v0.SyncMode; +import java.time.Duration; import java.util.HashMap; import java.util.List; import javax.sql.DataSource; +import org.junit.jupiter.api.Disabled; +@Disabled public class OracleStrictEncryptSourceAcceptanceTest extends SourceAcceptanceTest { private static final String STREAM_NAME = "JDBC_SPACE.ID_AND_NAME"; private static final String STREAM_NAME2 = "JDBC_SPACE.STARSHIPS"; + private static final Duration CONNECTION_TIMEOUT = Duration.ofSeconds(60); protected AirbyteOracleTestContainer container; protected JsonNode config; @@ -66,7 +70,8 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc config.get(JdbcUtils.PORT_KEY).asInt(), config.get("sid").asText()), JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED;" + - "oracle.net.encryption_types_client=( 3DES168 )", ";")); + "oracle.net.encryption_types_client=( 3DES168 )", ";"), + CONNECTION_TIMEOUT); try { final JdbcDatabase database = new DefaultJdbcDatabase(dataSource); diff --git a/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptTestDatabase.java b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptTestDatabase.java new file mode 100644 index 000000000000..662bcd5868fa --- /dev/null +++ b/airbyte-integrations/connectors/source-oracle-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/oracle_strict_encrypt/OracleStrictEncryptTestDatabase.java @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.oracle_strict_encrypt; + +import static io.airbyte.integrations.source.oracle_strict_encrypt.OracleStrictEncryptJdbcSourceAcceptanceTest.cleanUpTablesAndWait; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.TestDatabase; +import io.airbyte.commons.json.Jsons; +import java.util.List; +import java.util.stream.Stream; +import org.jooq.SQLDialect; + +public class OracleStrictEncryptTestDatabase extends + TestDatabase { + + private final AirbyteOracleTestContainer container; + private final List schemaNames; + + protected OracleStrictEncryptTestDatabase(final AirbyteOracleTestContainer container, final List schemaNames) { + super(container); + this.container = container; + this.schemaNames = schemaNames; + } + + @Override + public String getJdbcUrl() { + return container.getJdbcUrl(); + } + + @Override + public String getUserName() { + return container.getUsername(); + } + + @Override + public String getPassword() { + return container.getPassword(); + } + + @Override + public String getDatabaseName() { + return container.getDatabaseName(); + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.empty(); + } + + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return DatabaseDriver.ORACLE; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.DEFAULT; + } + + @Override + public OracleStrictEncryptDbConfigBuilder configBuilder() { + return new OracleStrictEncryptDbConfigBuilder(this) + .with(JdbcUtils.HOST_KEY, container.getHost()) + .with(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) + .with("sid", container.getSid()) + .with(JdbcUtils.USERNAME_KEY, container.getUsername()) + .with(JdbcUtils.PASSWORD_KEY, container.getPassword()) + .with(JdbcUtils.SCHEMAS_KEY, schemaNames) + .with(JdbcUtils.ENCRYPTION_KEY, Jsons.jsonNode(ImmutableMap.builder() + .put("encryption_method", "client_nne") + .put("encryption_algorithm", "3DES168") + .build())); + } + + @Override + public void close() { + cleanUpTablesAndWait(); + } + + static public class OracleStrictEncryptDbConfigBuilder extends ConfigBuilder { + + protected OracleStrictEncryptDbConfigBuilder(final OracleStrictEncryptTestDatabase testdb) { + super(testdb); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-oracle/.dockerignore b/airbyte-integrations/connectors/source-oracle/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-oracle/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-oracle/Dockerfile b/airbyte-integrations/connectors/source-oracle/Dockerfile deleted file mode 100644 index ef85f0ec58d7..000000000000 --- a/airbyte-integrations/connectors/source-oracle/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-oracle -ENV TZ UTC - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar -RUN tar xf ${APPLICATION}.tar --strip-components=1 - -LABEL io.airbyte.version=0.4.0 -LABEL io.airbyte.name=airbyte/source-oracle diff --git a/airbyte-integrations/connectors/source-oracle/README.md b/airbyte-integrations/connectors/source-oracle/README.md index a3a7ebd15aa8..32ce4e9c7987 100644 --- a/airbyte-integrations/connectors/source-oracle/README.md +++ b/airbyte-integrations/connectors/source-oracle/README.md @@ -16,10 +16,11 @@ From the Airbyte repository root, run: #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:source-oracle:airbyteDocker +./gradlew :airbyte-integrations:connectors:source-oracle:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/source-oracle:dev`. the Dockerfile. ## Testing diff --git a/airbyte-integrations/connectors/source-oracle/acceptance-test-config.yml b/airbyte-integrations/connectors/source-oracle/acceptance-test-config.yml deleted file mode 100644 index 1dad4c01150e..000000000000 --- a/airbyte-integrations/connectors/source-oracle/acceptance-test-config.yml +++ /dev/null @@ -1,8 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-oracle:dev -tests: - spec: - - spec_path: "src/test-integration/resources/expected_spec.json" - config_path: "src/test-integration/resources/dummy_config.json" - timeout_seconds: 300 diff --git a/airbyte-integrations/connectors/source-oracle/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-oracle/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-oracle/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-oracle/build.gradle b/airbyte-integrations/connectors/source-oracle/build.gradle index 5ffae2d188c4..dd0bb179e930 100644 --- a/airbyte-integrations/connectors/source-oracle/build.gradle +++ b/airbyte-integrations/connectors/source-oracle/build.gradle @@ -1,10 +1,26 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' - id 'airbyte-connector-acceptance-test' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.7.7' + features = ['db-sources'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileTestJava { + options.compilerArgs.remove("-Werror") + } + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.source.oracle.OracleSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] @@ -15,25 +31,9 @@ dependencies { // required so that log4j uses a standard xml parser instead of an oracle one (that gets pulled in by the oracle driver) implementation group: 'xerces', name: 'xercesImpl', version: '2.12.1' - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation project(':airbyte-integrations:connectors:source-relational-db') implementation "com.oracle.database.jdbc:ojdbc8-production:19.7.0.0" - testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) - testImplementation project(":airbyte-json-validation") - testImplementation project(':airbyte-test-utils') - testImplementation 'org.apache.commons:commons-lang3:3.11' - testImplementation libs.connectors.source.testcontainers.oracle.xe - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-oracle') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + testImplementation libs.testcontainers.oracle.xe } - diff --git a/airbyte-integrations/connectors/source-oracle/metadata.yaml b/airbyte-integrations/connectors/source-oracle/metadata.yaml index c12f2a06ba72..46f53566d421 100644 --- a/airbyte-integrations/connectors/source-oracle/metadata.yaml +++ b/airbyte-integrations/connectors/source-oracle/metadata.yaml @@ -9,7 +9,7 @@ data: connectorSubtype: database connectorType: source definitionId: b39a7370-74c3-45a6-ac3a-380d48520a83 - dockerImageTag: 0.4.0 + dockerImageTag: 0.5.0 dockerRepository: airbyte/source-oracle documentationUrl: https://docs.airbyte.com/integrations/sources/oracle githubIssueLabel: source-oracle @@ -18,7 +18,7 @@ data: name: Oracle DB registries: cloud: - dockerImageTag: 0.3.17 + dockerImageTag: 0.5.0 dockerRepository: airbyte/source-oracle-strict-encrypt enabled: true oss: diff --git a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java index b8531d3a1751..43193f980687 100644 --- a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java +++ b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSource.java @@ -6,16 +6,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.base.ssh.SshWrappedSource; +import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.cdk.integrations.source.relationaldb.TableInfo; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.base.ssh.SshWrappedSource; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.relationaldb.TableInfo; import io.airbyte.protocol.models.CommonField; import java.io.IOException; import java.io.PrintWriter; diff --git a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSourceOperations.java b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSourceOperations.java index e8cc0f8435d8..f306439ea64f 100644 --- a/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSourceOperations.java +++ b/airbyte-integrations/connectors/source-oracle/src/main/java/io/airbyte/integrations/source/oracle/OracleSourceOperations.java @@ -5,8 +5,8 @@ package io.airbyte.integrations.source.oracle; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.db.jdbc.DateTimeConverter; -import io.airbyte.db.jdbc.JdbcSourceOperations; +import io.airbyte.cdk.db.jdbc.DateTimeConverter; +import io.airbyte.cdk.db.jdbc.JdbcSourceOperations; import java.sql.ResultSet; import java.sql.SQLException; import org.slf4j.Logger; diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java index 4db4bcdf1bb5..501bf107aed6 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/AbstractSshOracleSourceAcceptanceTest.java @@ -7,16 +7,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.ssh.SshBastionContainer; +import io.airbyte.cdk.integrations.base.ssh.SshHelpers; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.ssh.SshBastionContainer; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.base.ssh.SshTunnel; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.CatalogHelpers; @@ -28,8 +28,10 @@ import java.util.List; import java.util.Objects; import javax.sql.DataSource; +import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.Network; +@Disabled public abstract class AbstractSshOracleSourceAcceptanceTest extends SourceAcceptanceTest { private static final String STREAM_NAME = "JDBC_SPACE.ID_AND_NAME"; @@ -45,7 +47,7 @@ public abstract class AbstractSshOracleSourceAcceptanceTest extends SourceAccept @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { startTestContainers(); - config = sshBastionContainer.getTunnelConfig(getTunnelMethod(), getBasicOracleDbConfigBuider(db)); + config = sshBastionContainer.getTunnelConfig(getTunnelMethod(), getBasicOracleDbConfigBuider(db), false); populateDatabaseTestData(); } diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java index b6d73e87a600..8045f1c89e89 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleJdbcSourceAcceptanceTest.java @@ -14,15 +14,14 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbState; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; -import io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils; -import io.airbyte.integrations.source.relationaldb.models.DbState; -import io.airbyte.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; @@ -32,15 +31,16 @@ import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.v0.AirbyteStreamState; import io.airbyte.protocol.models.v0.CatalogHelpers; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConnectorSpecification; import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.StreamDescriptor; import io.airbyte.protocol.models.v0.SyncMode; import java.math.BigDecimal; import java.sql.Connection; import java.sql.DriverManager; -import java.sql.JDBCType; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -49,20 +49,26 @@ import java.util.List; import java.util.Optional; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -class OracleJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { +@Disabled +class OracleJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { private static final Logger LOGGER = LoggerFactory.getLogger(OracleJdbcSourceAcceptanceTest.class); protected static final String USERNAME_WITHOUT_PERMISSION = "new_user"; protected static final String PASSWORD_WITHOUT_PERMISSION = "new_password"; - private static AirbyteOracleTestContainer ORACLE_DB; + private static final AirbyteOracleTestContainer ORACLE_DB = new AirbyteOracleTestContainer() + .withEnv("NLS_DATE_FORMAT", "YYYY-MM-DD") + .withEnv("RELAX_SECURITY", "1") + .withUsername("TEST_ORA") + .withPassword("oracle") + .usingSid() + .withEnv("RELAX_SECURITY", "1"); @BeforeAll static void init() { @@ -93,34 +99,59 @@ static void init() { CREATE_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s VARCHAR(20))"; INSERT_TABLE_WITH_NULLABLE_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES('Hello world :)')"; INSERT_TABLE_NAME_AND_TIMESTAMP_QUERY = "INSERT INTO %s (name, timestamp) VALUES ('%s', TO_TIMESTAMP('%s', 'YYYY-MM-DD HH24:MI:SS'))"; + } + + @AfterAll + static void cleanUp() { + ORACLE_DB.close(); + } + + @Override + public boolean supportsSchemas() { + // See https://www.oratable.com/oracle-user-schema-difference/ + return true; + } + + @Override + protected OracleSource source() { + return new OracleSource(); + } - ORACLE_DB = new AirbyteOracleTestContainer() - .withEnv("NLS_DATE_FORMAT", "YYYY-MM-DD") - .withEnv("RELAX_SECURITY", "1") - .withUsername("TEST_ORA") - .withPassword("oracle") - .usingSid() - .withEnv("RELAX_SECURITY", "1"); + @Override + public JsonNode config() { + return Jsons.clone(testdb.configBuilder().build()); + } + + @Override + protected OracleTestDatabase createTestDatabase() { ORACLE_DB.start(); + return new OracleTestDatabase(ORACLE_DB, List.of(SCHEMA_NAME, SCHEMA_NAME2)).initialized(); } - @BeforeEach - public void setup() throws Exception { - config = Jsons.jsonNode(ImmutableMap.builder() - .put("host", ORACLE_DB.getHost()) - .put("port", ORACLE_DB.getFirstMappedPort()) - .put("sid", ORACLE_DB.getSid()) - .put("username", ORACLE_DB.getUsername()) - .put("password", ORACLE_DB.getPassword()) - .put("schemas", List.of(SCHEMA_NAME, SCHEMA_NAME2)) - .build()); - - // Because Oracle doesn't let me create database easily I need to clean up - cleanUpTables(); - - super.setup(); + @Override + public void createSchemas() { + // In Oracle, `CREATE USER` creates a schema. + // See https://www.oratable.com/oracle-user-schema-difference/ + if (supportsSchemas()) { + for (final String schemaName : TEST_SCHEMAS) { + executeOracleStatement( + String.format( + "CREATE USER %s IDENTIFIED BY password DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS", + schemaName)); + } + } } + static void cleanUpTablesAndWait() { + try { + cleanUpTables(); + Thread.sleep(1000); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + @Override protected void incrementalDateCheck() throws Exception { // https://stackoverflow.com/questions/47712930/resultset-meta-data-return-timestamp-instead-of-date-oracle-jdbc // Oracle DATE is a java.sql.Timestamp (java.sql.Types.TIMESTAMP) as far as JDBC (and the SQL @@ -132,6 +163,7 @@ protected void incrementalDateCheck() throws Exception { Lists.newArrayList(getTestMessages().get(1), getTestMessages().get(2))); } + @Override protected AirbyteCatalog getCatalog(final String defaultNamespace) { return new AirbyteCatalog().withStreams(List.of( CatalogHelpers.createAirbyteStream( @@ -161,20 +193,7 @@ protected AirbyteCatalog getCatalog(final String defaultNamespace) { List.of(List.of(COL_FIRST_NAME), List.of(COL_LAST_NAME))))); } - @AfterEach - public void tearDownOracle() throws Exception { - // ORA-12519 - // https://stackoverflow.com/questions/205160/what-can-cause-intermittent-ora-12519-tns-no-appropriate-handler-found-errors - // sleep for 1000 - executeOracleStatement(String.format("DROP TABLE %s", getFullyQualifiedTableName(TABLE_NAME))); - executeOracleStatement( - String.format("DROP TABLE %s", getFullyQualifiedTableName(TABLE_NAME_WITHOUT_PK))); - executeOracleStatement( - String.format("DROP TABLE %s", getFullyQualifiedTableName(TABLE_NAME_COMPOSITE_PK))); - Thread.sleep(1000); - } - - void cleanUpTables() throws SQLException { + static void cleanUpTables() throws SQLException { final Connection connection = DriverManager.getConnection( ORACLE_DB.getJdbcUrl(), ORACLE_DB.getUsername(), @@ -197,14 +216,14 @@ void cleanUpTables() throws SQLException { protected List getTestMessages() { return Lists.newArrayList( new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName) + .withRecord(new AirbyteRecordMessage().withStream(streamName()) .withNamespace(getDefaultNamespace()) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_1, COL_NAME, "picard", COL_UPDATED_AT, "2004-10-19T00:00:00.000000")))), new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName) + .withRecord(new AirbyteRecordMessage().withStream(streamName()) .withNamespace(getDefaultNamespace()) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_2, @@ -212,7 +231,7 @@ protected List getTestMessages() { COL_UPDATED_AT, "2005-10-19T00:00:00.000000")))), new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName) + .withRecord(new AirbyteRecordMessage().withStream(streamName()) .withNamespace(getDefaultNamespace()) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_3, @@ -230,7 +249,8 @@ void testIncrementalTimestampCheckCursor() throws Exception { } @Test - void testReadOneTableIncrementallyTwice() throws Exception { + @Override + protected void testReadOneTableIncrementallyTwice() throws Exception { final String namespace = getDefaultNamespace(); final ConfiguredAirbyteCatalog configuredCatalog = getConfiguredCatalogWithOneStream(namespace); configuredCatalog.getStreams().forEach(airbyteStream -> { @@ -240,38 +260,34 @@ void testReadOneTableIncrementallyTwice() throws Exception { }); final DbState state = new DbState() - .withStreams(Lists.newArrayList(new DbStreamState().withStreamName(streamName).withStreamNamespace(namespace))); + .withStreams(Lists.newArrayList(new DbStreamState().withStreamName(streamName()).withStreamNamespace(namespace))); final List actualMessagesFirstSync = MoreIterators - .toList(source.read(config, configuredCatalog, Jsons.jsonNode(state))); + .toList(source().read(config(), configuredCatalog, Jsons.jsonNode(state))); final Optional stateAfterFirstSyncOptional = actualMessagesFirstSync.stream() .filter(r -> r.getType() == Type.STATE).findFirst(); assertTrue(stateAfterFirstSyncOptional.isPresent()); - database.execute(connection -> { - connection.createStatement().execute( - String.format("INSERT INTO %s(id, name, updated_at) VALUES (4,'riker', '2006-10-19')", - getFullyQualifiedTableName(TABLE_NAME))); - connection.createStatement().execute( - String.format("INSERT INTO %s(id, name, updated_at) VALUES (5, 'data', '2006-10-19')", - getFullyQualifiedTableName(TABLE_NAME))); - }); + testdb.with(String.format("INSERT INTO %s(id, name, updated_at) VALUES (4,'riker', '2006-10-19')", + getFullyQualifiedTableName(TABLE_NAME))); + testdb.with(String.format("INSERT INTO %s(id, name, updated_at) VALUES (5, 'data', '2006-10-19')", + getFullyQualifiedTableName(TABLE_NAME))); final List actualMessagesSecondSync = MoreIterators - .toList(source.read(config, configuredCatalog, + .toList(source().read(config(), configuredCatalog, stateAfterFirstSyncOptional.get().getState().getData())); Assertions.assertEquals(2, (int) actualMessagesSecondSync.stream().filter(r -> r.getType() == Type.RECORD).count()); final List expectedMessages = new ArrayList<>(); expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + .withRecord(new AirbyteRecordMessage().withStream(streamName()).withNamespace(namespace) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_4, COL_NAME, "riker", COL_UPDATED_AT, "2006-10-19T00:00:00.000000"))))); expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + .withRecord(new AirbyteRecordMessage().withStream(streamName()).withNamespace(namespace) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_5, COL_NAME, "data", @@ -279,11 +295,19 @@ void testReadOneTableIncrementallyTwice() throws Exception { expectedMessages.add(new AirbyteMessage() .withType(Type.STATE) .withState(new AirbyteStateMessage() - .withType(AirbyteStateType.LEGACY) + .withType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withName(streamName()).withNamespace(namespace)) + .withStreamState(Jsons.jsonNode(new DbStreamState() + .withStreamNamespace(namespace) + .withStreamName(streamName()) + .withCursorField(ImmutableList.of(COL_ID)) + .withCursor("5") + .withCursorRecordCount(1L)))) .withData(Jsons.jsonNode(new DbState() .withCdc(false) .withStreams(Lists.newArrayList(new DbStreamState() - .withStreamName(streamName) + .withStreamName(streamName()) .withStreamNamespace(namespace) .withCursorField(ImmutableList.of(COL_ID)) .withCursor("5") @@ -296,46 +320,6 @@ void testReadOneTableIncrementallyTwice() throws Exception { assertTrue(actualMessagesSecondSync.containsAll(expectedMessages)); } - @Override - public boolean supportsSchemas() { - // See https://www.oratable.com/oracle-user-schema-difference/ - return true; - } - - @Override - public AbstractJdbcSource getJdbcSource() { - return new OracleSource(); - } - - @Override - public JsonNode getConfig() { - return config; - } - - @Override - public String getDriverClass() { - return OracleSource.DRIVER_CLASS; - } - - @AfterAll - static void cleanUp() { - ORACLE_DB.close(); - } - - @Override - public void createSchemas() throws SQLException { - // In Oracle, `CREATE USER` creates a schema. - // See https://www.oratable.com/oracle-user-schema-difference/ - if (supportsSchemas()) { - for (final String schemaName : TEST_SCHEMAS) { - executeOracleStatement( - String.format( - "CREATE USER %s IDENTIFIED BY password DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS", - schemaName)); - } - } - } - public void executeOracleStatement(final String query) { try ( final Connection connection = DriverManager.getConnection( @@ -392,7 +376,7 @@ public static boolean ignoreSQLException(final String sqlState) { @Test void testSpec() throws Exception { - final ConnectorSpecification actual = source.spec(); + final ConnectorSpecification actual = source().spec(); final ConnectorSpecification expected = Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class); assertEquals(expected, actual); @@ -402,10 +386,11 @@ void testSpec() throws Exception { void testCheckIncorrectPasswordFailure() throws Exception { // by using a fake password oracle can block user account so we will create separate account for // this test + final JsonNode config = config(); executeOracleStatement(String.format("CREATE USER locked_user IDENTIFIED BY password DEFAULT TABLESPACE USERS QUOTA UNLIMITED ON USERS")); ((ObjectNode) config).put(JdbcUtils.USERNAME_KEY, "locked_user"); ((ObjectNode) config).put(JdbcUtils.PASSWORD_KEY, "fake"); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); Assertions.assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); assertEquals("State code: 72000; Error code: 1017; Message: ORA-01017: invalid username/password; logon denied\n", @@ -414,34 +399,38 @@ void testCheckIncorrectPasswordFailure() throws Exception { @Test public void testCheckIncorrectUsernameFailure() throws Exception { + final JsonNode config = config(); ((ObjectNode) config).put(JdbcUtils.USERNAME_KEY, "fake"); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); Assertions.assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); assertTrue(status.getMessage().contains("State code: 72000; Error code: 1017;")); } @Test public void testCheckIncorrectHostFailure() throws Exception { + final JsonNode config = config(); ((ObjectNode) config).put(JdbcUtils.HOST_KEY, "localhost2"); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); Assertions.assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); assertTrue(status.getMessage().contains("State code: 08006; Error code: 17002;")); } @Test public void testCheckIncorrectPortFailure() throws Exception { + final JsonNode config = config(); ((ObjectNode) config).put(JdbcUtils.PORT_KEY, "0000"); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); Assertions.assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); assertTrue(status.getMessage().contains("State code: 08006; Error code: 17002;")); } @Test public void testUserHasNoPermissionToDataBase() throws Exception { + final JsonNode config = config(); executeOracleStatement(String.format("CREATE USER %s IDENTIFIED BY %s", USERNAME_WITHOUT_PERMISSION, PASSWORD_WITHOUT_PERMISSION)); ((ObjectNode) config).put(JdbcUtils.USERNAME_KEY, USERNAME_WITHOUT_PERMISSION); ((ObjectNode) config).put(JdbcUtils.PASSWORD_KEY, PASSWORD_WITHOUT_PERMISSION); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); Assertions.assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); assertTrue(status.getMessage().contains("State code: 72000; Error code: 1045;")); } diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java index 0c276cf766cf..73c9d9081e30 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceAcceptanceTest.java @@ -7,14 +7,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.base.ssh.SshHelpers; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.CatalogHelpers; @@ -25,7 +25,9 @@ import java.util.HashMap; import java.util.List; import javax.sql.DataSource; +import org.junit.jupiter.api.Disabled; +@Disabled public class OracleSourceAcceptanceTest extends SourceAcceptanceTest { private static final String STREAM_NAME = "JDBC_SPACE.ID_AND_NAME"; diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java index 5939a62bf5a9..6154b1ac32b3 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceDatatypeTest.java @@ -6,13 +6,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDataHolder; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; -import io.airbyte.integrations.standardtest.source.TestDataHolder; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.JsonSchemaType; import java.text.DateFormat; import java.text.ParseException; @@ -23,9 +23,11 @@ import java.util.List; import java.util.TimeZone; import org.jooq.DSLContext; +import org.junit.jupiter.api.Disabled; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@Disabled public class OracleSourceDatatypeTest extends AbstractSourceDatabaseTypeTest { private AirbyteOracleTestContainer container; diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java index 83a88c099a45..822ac556650b 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceNneAcceptanceTest.java @@ -10,18 +10,23 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; import java.sql.SQLException; +import java.time.Duration; import java.util.List; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +@Disabled public class OracleSourceNneAcceptanceTest extends OracleSourceAcceptanceTest { + private static final Duration CONNECTION_TIMEOUT = Duration.ofSeconds(60); + @Test public void testEncrytion() throws SQLException { final JsonNode clone = Jsons.clone(getConfig()); @@ -44,7 +49,8 @@ public void testEncrytion() throws SQLException { clone.get("connection_data").get("service_name").asText()), JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED&" + "oracle.net.encryption_types_client=( " - + algorithm + " )"))); + + algorithm + " )"), + CONNECTION_TIMEOUT)); final String networkServiceBanner = "select network_service_banner from v$session_connect_info where sid in (select distinct sid from v$mystat)"; @@ -95,7 +101,8 @@ public void testCheckProtocol() throws SQLException { config.get("connection_data").get("service_name").asText()), JdbcUtils.parseJdbcParameters("oracle.net.encryption_client=REQUIRED&" + "oracle.net.encryption_types_client=( " - + algorithm + " )"))); + + algorithm + " )"), + CONNECTION_TIMEOUT)); final String networkServiceBanner = "SELECT sys_context('USERENV', 'NETWORK_PROTOCOL') as network_protocol FROM dual"; final List collect = database.queryJsons(networkServiceBanner); diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java index 90ac510ea005..fb8de2ff61bc 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleSourceTest.java @@ -10,12 +10,12 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; @@ -35,8 +35,10 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; +@Disabled class OracleSourceTest { private static final String STREAM_NAME = "TEST.ID_AND_NAME"; diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleTestDatabase.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleTestDatabase.java new file mode 100644 index 000000000000..5197bd2baf06 --- /dev/null +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/OracleTestDatabase.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.oracle; + +import static io.airbyte.integrations.source.oracle.OracleJdbcSourceAcceptanceTest.cleanUpTablesAndWait; + +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.TestDatabase; +import java.util.List; +import java.util.stream.Stream; +import org.jooq.SQLDialect; + +public class OracleTestDatabase extends + TestDatabase { + + private final AirbyteOracleTestContainer container; + private final List schemaNames; + + protected OracleTestDatabase(final AirbyteOracleTestContainer container, final List schemaNames) { + super(container); + this.container = container; + this.schemaNames = schemaNames; + } + + @Override + public String getJdbcUrl() { + return container.getJdbcUrl(); + } + + @Override + public String getUserName() { + return container.getUsername(); + } + + @Override + public String getPassword() { + return container.getPassword(); + } + + @Override + public String getDatabaseName() { + return container.getDatabaseName(); + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.empty(); + } + + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return DatabaseDriver.ORACLE; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.DEFAULT; + } + + @Override + public OracleDbConfigBuilder configBuilder() { + return new OracleDbConfigBuilder(this) + .with(JdbcUtils.HOST_KEY, container.getHost()) + .with(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) + .with("sid", container.getSid()) + .with(JdbcUtils.USERNAME_KEY, container.getUsername()) + .with(JdbcUtils.PASSWORD_KEY, container.getPassword()) + .with(JdbcUtils.SCHEMAS_KEY, schemaNames); + } + + @Override + public void close() { + cleanUpTablesAndWait(); + } + + static public class OracleDbConfigBuilder extends TestDatabase.ConfigBuilder { + + protected OracleDbConfigBuilder(final OracleTestDatabase testdb) { + super(testdb); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshKeyOracleSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshKeyOracleSourceAcceptanceTest.java index 0c272506c24b..cc53839eb1d1 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshKeyOracleSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshKeyOracleSourceAcceptanceTest.java @@ -4,8 +4,10 @@ package io.airbyte.integrations.source.oracle; -import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import org.junit.jupiter.api.Disabled; +@Disabled public class SshKeyOracleSourceAcceptanceTest extends AbstractSshOracleSourceAcceptanceTest { @Override diff --git a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshPasswordOracleSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshPasswordOracleSourceAcceptanceTest.java index cf14894a0a3d..44c79902a7b3 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshPasswordOracleSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test-integration/java/io/airbyte/integrations/source/oracle/SshPasswordOracleSourceAcceptanceTest.java @@ -4,8 +4,10 @@ package io.airbyte.integrations.source.oracle; -import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import org.junit.jupiter.api.Disabled; +@Disabled public class SshPasswordOracleSourceAcceptanceTest extends AbstractSshOracleSourceAcceptanceTest { @Override diff --git a/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleStressTest.java b/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleStressTest.java index 9a84d1c2ca11..3a003ae028ec 100644 --- a/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleStressTest.java +++ b/airbyte-integrations/connectors/source-oracle/src/test/java/io/airbyte/integrations/source/oracle/OracleStressTest.java @@ -6,14 +6,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.cdk.integrations.source.jdbc.test.JdbcStressTest; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.test.JdbcStressTest; import java.sql.JDBCType; import java.util.Optional; import java.util.Set; diff --git a/airbyte-integrations/connectors/source-orb/README.md b/airbyte-integrations/connectors/source-orb/README.md index 79f0a0d76bc1..cd3b9c13af7d 100644 --- a/airbyte-integrations/connectors/source-orb/README.md +++ b/airbyte-integrations/connectors/source-orb/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-orb:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/orb) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_orb/spec.json` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-orb:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-orb build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-orb:airbyteDocker +An image will be built with the tag `airbyte/source-orb:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-orb:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-orb:dev check --config docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-orb:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-orb:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-orb test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-orb:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-orb:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-orb test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/orb.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-orb/acceptance-test-config.yml b/airbyte-integrations/connectors/source-orb/acceptance-test-config.yml index 76134772a4dc..b636b4e14d0a 100644 --- a/airbyte-integrations/connectors/source-orb/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-orb/acceptance-test-config.yml @@ -22,14 +22,6 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" - cursor_paths: - # This points to a specific customer's credit ledger entries in the state, - # and this customer is in the integration test account. - credits_ledger_entries: ["hHQF5BT5jtyj9r7V", "created_at"] - - # This points to a specific subscription's usage entries in the state, - # and this subscription is in the integration test account. - subscription_usage: ["FDWRvxuBUiFfZech", "timeframe_start"] full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-orb/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-orb/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-orb/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-orb/build.gradle b/airbyte-integrations/connectors/source-orb/build.gradle deleted file mode 100644 index db679342c380..000000000000 --- a/airbyte-integrations/connectors/source-orb/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_orb' -} diff --git a/airbyte-integrations/connectors/source-orb/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-orb/unit_tests/test_incremental_streams.py index ea59c15494a7..ccf3d2d63f15 100644 --- a/airbyte-integrations/connectors/source-orb/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-orb/unit_tests/test_incremental_streams.py @@ -123,9 +123,10 @@ def test_invoices_request_params(patch_incremental_base_class, mocker, config, c inputs = {"stream_state": current_stream_state, "next_page_token": next_page_token} expected_params = expected_params or {} expected_params["limit"] = OrbStream.page_size - expected_params["status[]"] = ['void', 'paid', 'issued', 'synced'] + expected_params["status[]"] = ["void", "paid", "issued", "synced"] assert stream.request_params(**inputs) == expected_params + # We have specific unit tests for CreditsLedgerEntries incremental stream # because that employs slicing logic @@ -345,6 +346,7 @@ def test_credits_ledger_entries_enriches_with_multiple_entries_per_event(mocker) {"event": {"id": "foo-event-id", "properties": {"ping": "pong"}}, "entry_type": "decrement"}, ] + # We have specific unit tests for SubscriptionUsage incremental stream # because its logic differs from other IncrementalOrbStreams @@ -383,53 +385,85 @@ def test_subscription_usage_get_updated_state(mocker, current_stream_state, late def test_subscription_usage_stream_slices(mocker): mocker.patch.object( - Subscriptions, "read_records", return_value=iter([ - {"id": "1", "plan_id": "2"}, - {"id": "11", "plan_id": "2"}, - {"id": "111", "plan_id": "3"} # should be ignored because plan_id set to 2 - ]) + Subscriptions, + "read_records", + return_value=iter( + [ + {"id": "1", "plan_id": "2"}, + {"id": "11", "plan_id": "2"}, + {"id": "111", "plan_id": "3"}, # should be ignored because plan_id set to 2 + ] + ), ) stream = SubscriptionUsage(plan_id="2", start_date="2022-01-25T12:00:00+00:00", end_date="2022-01-26T12:00:00+00:00") inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} expected_stream_slice = [ {"subscription_id": "1", "timeframe_start": "2022-01-25T12:00:00+00:00", "timeframe_end": "2022-01-26T12:00:00+00:00"}, - {"subscription_id": "11", "timeframe_start": "2022-01-25T12:00:00+00:00", "timeframe_end": "2022-01-26T12:00:00+00:00"} + {"subscription_id": "11", "timeframe_start": "2022-01-25T12:00:00+00:00", "timeframe_end": "2022-01-26T12:00:00+00:00"}, ] assert list(stream.stream_slices(**inputs)) == expected_stream_slice def test_subscription_usage_stream_slices_with_grouping_key(mocker): mocker.patch.object( - Subscriptions, "read_records", return_value=iter([ - {"id": "1", "plan_id": "2"}, - {"id": "11", "plan_id": "2"}, - {"id": "111", "plan_id": "3"} # should be ignored because plan_id set to 2 - ]) + Subscriptions, + "read_records", + return_value=iter( + [ + {"id": "1", "plan_id": "2"}, + {"id": "11", "plan_id": "2"}, + {"id": "111", "plan_id": "3"}, # should be ignored because plan_id set to 2 + ] + ), ) mocker.patch.object( - Plans, "read_records", return_value=iter([ - {"id": "2", "prices": [ - {"billable_metric": {"id": "billableMetricIdA"}}, - {"billable_metric": {"id": "billableMetricIdB"}} - ]}, - {"id": "3", "prices": [ # should be ignored because plan_id is set to 2 - {"billable_metric": {"id": "billableMetricIdC"}} - ]} - ]) + Plans, + "read_records", + return_value=iter( + [ + {"id": "2", "prices": [{"billable_metric": {"id": "billableMetricIdA"}}, {"billable_metric": {"id": "billableMetricIdB"}}]}, + {"id": "3", "prices": [{"billable_metric": {"id": "billableMetricIdC"}}]}, # should be ignored because plan_id is set to 2 + ] + ), ) # when a grouping_key is present, one slice per billable_metric is created because the Orb API # requires one API call per billable metric if the group_by param is in use. - stream = SubscriptionUsage(plan_id="2", subscription_usage_grouping_key="groupKey", start_date="2022-01-25T12:00:00+00:00", end_date="2022-01-26T12:00:00+00:00") + stream = SubscriptionUsage( + plan_id="2", + subscription_usage_grouping_key="groupKey", + start_date="2022-01-25T12:00:00+00:00", + end_date="2022-01-26T12:00:00+00:00", + ) inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} # one slice per billable metric per subscription that matches the input plan expected_stream_slice = [ - {"subscription_id": "1", "billable_metric_id": "billableMetricIdA", "timeframe_start": "2022-01-25T12:00:00+00:00", "timeframe_end": "2022-01-26T12:00:00+00:00"}, - {"subscription_id": "1", "billable_metric_id": "billableMetricIdB", "timeframe_start": "2022-01-25T12:00:00+00:00", "timeframe_end": "2022-01-26T12:00:00+00:00"}, - {"subscription_id": "11", "billable_metric_id": "billableMetricIdA", "timeframe_start": "2022-01-25T12:00:00+00:00", "timeframe_end": "2022-01-26T12:00:00+00:00"}, - {"subscription_id": "11", "billable_metric_id": "billableMetricIdB", "timeframe_start": "2022-01-25T12:00:00+00:00", "timeframe_end": "2022-01-26T12:00:00+00:00"}, + { + "subscription_id": "1", + "billable_metric_id": "billableMetricIdA", + "timeframe_start": "2022-01-25T12:00:00+00:00", + "timeframe_end": "2022-01-26T12:00:00+00:00", + }, + { + "subscription_id": "1", + "billable_metric_id": "billableMetricIdB", + "timeframe_start": "2022-01-25T12:00:00+00:00", + "timeframe_end": "2022-01-26T12:00:00+00:00", + }, + { + "subscription_id": "11", + "billable_metric_id": "billableMetricIdA", + "timeframe_start": "2022-01-25T12:00:00+00:00", + "timeframe_end": "2022-01-26T12:00:00+00:00", + }, + { + "subscription_id": "11", + "billable_metric_id": "billableMetricIdB", + "timeframe_start": "2022-01-25T12:00:00+00:00", + "timeframe_end": "2022-01-26T12:00:00+00:00", + }, ] assert list(stream.stream_slices(**inputs)) == expected_stream_slice @@ -440,44 +474,73 @@ def test_subscription_usage_stream_slices_with_grouping_key(mocker): # Slice matches subscription in state, no grouping ( dict(subscription_id_foo=dict(timeframe_start="2022-01-25T12:00:00+00:00")), - dict(subscription_id="subscription_id_foo", timeframe_start="2022-01-25T12:00:00+00:00", timeframe_end="2022-01-26T12:00:00+00:00"), - None + dict( + subscription_id="subscription_id_foo", + timeframe_start="2022-01-25T12:00:00+00:00", + timeframe_end="2022-01-26T12:00:00+00:00", + ), + None, ), # Slice does not match subscription in state, no grouping ( dict(subscription_id_foo=dict(timeframe_start="2022-01-25T12:00:00+00:00")), - dict(subscription_id="subscription_id_bar", timeframe_start="2022-01-25T12:00:00+00:00", timeframe_end="2022-01-26T12:00:00+00:00"), - None + dict( + subscription_id="subscription_id_bar", + timeframe_start="2022-01-25T12:00:00+00:00", + timeframe_end="2022-01-26T12:00:00+00:00", + ), + None, ), # No existing state, no grouping ( {}, - dict(subscription_id="subscription_id_baz", timeframe_start="2022-01-25T12:00:00+00:00", timeframe_end="2022-01-26T12:00:00+00:00"), - None + dict( + subscription_id="subscription_id_baz", + timeframe_start="2022-01-25T12:00:00+00:00", + timeframe_end="2022-01-26T12:00:00+00:00", + ), + None, ), # Slice matches subscription in state, with grouping ( dict(subscription_id_foo=dict(timeframe_start="2022-01-25T12:00:00+00:00")), - dict(subscription_id="subscription_id_foo", billable_metric_id="billableMetricA", timeframe_start="2022-01-25T12:00:00+00:00", timeframe_end="2022-01-26T12:00:00+00:00"), - "group_key_foo" + dict( + subscription_id="subscription_id_foo", + billable_metric_id="billableMetricA", + timeframe_start="2022-01-25T12:00:00+00:00", + timeframe_end="2022-01-26T12:00:00+00:00", + ), + "group_key_foo", ), # Slice does not match subscription in state, with grouping ( dict(subscription_id_foo=dict(timeframe_start="2022-01-25T12:00:00+00:00")), - dict(subscription_id="subscription_id_bar", billable_metric_id="billableMetricA", timeframe_start="2022-01-25T12:00:00+00:00", timeframe_end="2022-01-26T12:00:00+00:00"), - "group_key_foo" + dict( + subscription_id="subscription_id_bar", + billable_metric_id="billableMetricA", + timeframe_start="2022-01-25T12:00:00+00:00", + timeframe_end="2022-01-26T12:00:00+00:00", + ), + "group_key_foo", ), # No existing state, with grouping ( {}, - dict(subscription_id="subscription_id_baz", billable_metric_id="billableMetricA", timeframe_start="2022-01-25T12:00:00+00:00", timeframe_end="2022-01-26T12:00:00+00:00"), - "group_key_foo" + dict( + subscription_id="subscription_id_baz", + billable_metric_id="billableMetricA", + timeframe_start="2022-01-25T12:00:00+00:00", + timeframe_end="2022-01-26T12:00:00+00:00", + ), + "group_key_foo", ), ], ) def test_subscription_usage_request_params(mocker, current_stream_state, current_stream_slice, grouping_key): if grouping_key: - stream = SubscriptionUsage(start_date="2022-01-25T12:00:00+00:00", end_date="2022-01-26T12:00:00+00:00", subscription_usage_grouping_key=grouping_key) + stream = SubscriptionUsage( + start_date="2022-01-25T12:00:00+00:00", end_date="2022-01-26T12:00:00+00:00", subscription_usage_grouping_key=grouping_key + ) else: stream = SubscriptionUsage(start_date="2022-01-25T12:00:00+00:00", end_date="2022-01-26T12:00:00+00:00") @@ -500,30 +563,13 @@ def test_subscription_usage_yield_transformed_subrecords(mocker): stream = SubscriptionUsage(start_date="2022-01-25T12:00:00+00:00", end_date="2022-01-26T12:00:00+00:00") subscription_usage_response = { - "billable_metric": { - "name": "Metric A", - "id": "billableMetricA" - }, + "billable_metric": {"name": "Metric A", "id": "billableMetricA"}, "usage": [ - { - "quantity": 0, - "timeframe_start": "2022-01-25T12:00:00+00:00", - "timeframe_end": "2022-01-26T12:00:00+00:00" - }, - { - "quantity": 1, - "timeframe_start": "2022-01-25T12:00:00+00:00", - "timeframe_end": "2022-01-26T12:00:00+00:00" - }, - { - "quantity": 2, - "timeframe_start": "2022-01-26T12:00:00+00:00", - "timeframe_end": "2022-01-27T12:00:00+00:00" - } + {"quantity": 0, "timeframe_start": "2022-01-25T12:00:00+00:00", "timeframe_end": "2022-01-26T12:00:00+00:00"}, + {"quantity": 1, "timeframe_start": "2022-01-25T12:00:00+00:00", "timeframe_end": "2022-01-26T12:00:00+00:00"}, + {"quantity": 2, "timeframe_start": "2022-01-26T12:00:00+00:00", "timeframe_end": "2022-01-27T12:00:00+00:00"}, ], - "otherTopLevelField": { - "shouldBeIncluded": "true" - } + "otherTopLevelField": {"shouldBeIncluded": "true"}, } subscription_id = "subscriptionIdA" @@ -536,10 +582,8 @@ def test_subscription_usage_yield_transformed_subrecords(mocker): "timeframe_end": "2022-01-26T12:00:00+00:00", "billable_metric_name": "Metric A", "billable_metric_id": "billableMetricA", - "otherTopLevelField": { - "shouldBeIncluded": "true" - }, - "subscription_id": subscription_id + "otherTopLevelField": {"shouldBeIncluded": "true"}, + "subscription_id": subscription_id, }, { "quantity": 2, @@ -547,11 +591,9 @@ def test_subscription_usage_yield_transformed_subrecords(mocker): "timeframe_end": "2022-01-27T12:00:00+00:00", "billable_metric_name": "Metric A", "billable_metric_id": "billableMetricA", - "otherTopLevelField": { - "shouldBeIncluded": "true" - }, - "subscription_id": subscription_id - } + "otherTopLevelField": {"shouldBeIncluded": "true"}, + "subscription_id": subscription_id, + }, ] actual_output = list(stream.yield_transformed_subrecords(subscription_usage_response, subscription_id)) @@ -560,37 +602,19 @@ def test_subscription_usage_yield_transformed_subrecords(mocker): def test_subscription_usage_yield_transformed_subrecords_with_grouping(mocker): - stream = SubscriptionUsage(start_date="2022-01-25T12:00:00+00:00", end_date="2022-01-26T12:00:00+00:00", subscription_usage_grouping_key="grouping_key") + stream = SubscriptionUsage( + start_date="2022-01-25T12:00:00+00:00", end_date="2022-01-26T12:00:00+00:00", subscription_usage_grouping_key="grouping_key" + ) subscription_usage_response = { - "billable_metric": { - "name": "Metric A", - "id": "billableMetricA" - }, - "metric_group": { - "property_key": "grouping_key", - "property_value": "grouping_value" - }, + "billable_metric": {"name": "Metric A", "id": "billableMetricA"}, + "metric_group": {"property_key": "grouping_key", "property_value": "grouping_value"}, "usage": [ - { - "quantity": 0, - "timeframe_start": "2022-01-25T12:00:00+00:00", - "timeframe_end": "2022-01-26T12:00:00+00:00" - }, - { - "quantity": 1, - "timeframe_start": "2022-01-25T12:00:00+00:00", - "timeframe_end": "2022-01-26T12:00:00+00:00" - }, - { - "quantity": 2, - "timeframe_start": "2022-01-26T12:00:00+00:00", - "timeframe_end": "2022-01-27T12:00:00+00:00" - } + {"quantity": 0, "timeframe_start": "2022-01-25T12:00:00+00:00", "timeframe_end": "2022-01-26T12:00:00+00:00"}, + {"quantity": 1, "timeframe_start": "2022-01-25T12:00:00+00:00", "timeframe_end": "2022-01-26T12:00:00+00:00"}, + {"quantity": 2, "timeframe_start": "2022-01-26T12:00:00+00:00", "timeframe_end": "2022-01-27T12:00:00+00:00"}, ], - "otherTopLevelField": { - "shouldBeIncluded": "true" - } + "otherTopLevelField": {"shouldBeIncluded": "true"}, } subscription_id = "subscriptionIdA" @@ -603,11 +627,9 @@ def test_subscription_usage_yield_transformed_subrecords_with_grouping(mocker): "timeframe_end": "2022-01-26T12:00:00+00:00", "billable_metric_name": "Metric A", "billable_metric_id": "billableMetricA", - "otherTopLevelField": { - "shouldBeIncluded": "true" - }, + "otherTopLevelField": {"shouldBeIncluded": "true"}, "subscription_id": subscription_id, - "grouping_key": "grouping_value" + "grouping_key": "grouping_value", }, { "quantity": 2, @@ -615,12 +637,10 @@ def test_subscription_usage_yield_transformed_subrecords_with_grouping(mocker): "timeframe_end": "2022-01-27T12:00:00+00:00", "billable_metric_name": "Metric A", "billable_metric_id": "billableMetricA", - "otherTopLevelField": { - "shouldBeIncluded": "true" - }, + "otherTopLevelField": {"shouldBeIncluded": "true"}, "subscription_id": subscription_id, - "grouping_key": "grouping_value" - } + "grouping_key": "grouping_value", + }, ] actual_output = list(stream.yield_transformed_subrecords(subscription_usage_response, subscription_id)) diff --git a/airbyte-integrations/connectors/source-orbit/Dockerfile b/airbyte-integrations/connectors/source-orbit/Dockerfile index d20a9c53aa90..a612e49c58ba 100644 --- a/airbyte-integrations/connectors/source-orbit/Dockerfile +++ b/airbyte-integrations/connectors/source-orbit/Dockerfile @@ -34,5 +34,5 @@ COPY source_orbit ./source_orbit ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.3.0 LABEL io.airbyte.name=airbyte/source-orbit diff --git a/airbyte-integrations/connectors/source-orbit/README.md b/airbyte-integrations/connectors/source-orbit/README.md index 7f4df920d80b..8ed271169b83 100644 --- a/airbyte-integrations/connectors/source-orbit/README.md +++ b/airbyte-integrations/connectors/source-orbit/README.md @@ -1,45 +1,12 @@ # Orbit Source -This is the repository for the Orbit source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/orbit). +This is the repository for the Orbit configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/orbit). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-orbit:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/orbit) +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/orbit) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_orbit/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. @@ -47,28 +14,21 @@ See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source orbit test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-orbit:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-orbit build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-orbit:airbyteDocker +An image will be built with the tag `airbyte/source-orbit:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-orbit:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-orbit:dev check --conf docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-orbit:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-orbit:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-orbit test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-orbit:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-orbit:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-orbit test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/orbit.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-orbit/__init__.py b/airbyte-integrations/connectors/source-orbit/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-orbit/acceptance-test-config.yml b/airbyte-integrations/connectors/source-orbit/acceptance-test-config.yml index d49a28f248ac..feeebbfa13ee 100644 --- a/airbyte-integrations/connectors/source-orbit/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-orbit/acceptance-test-config.yml @@ -1,20 +1,26 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-orbit:dev -tests: +test_strictness_level: low +acceptance_tests: spec: - - spec_path: "source_orbit/spec.yaml" + tests: + - spec_path: "source_orbit/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-orbit/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-orbit/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-orbit/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-orbit/build.gradle b/airbyte-integrations/connectors/source-orbit/build.gradle deleted file mode 100644 index b58192d5333e..000000000000 --- a/airbyte-integrations/connectors/source-orbit/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_orbit' -} diff --git a/airbyte-integrations/connectors/source-orbit/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-orbit/integration_tests/invalid_config.json index 7e719f4a39a1..f9ccb6888e3f 100644 --- a/airbyte-integrations/connectors/source-orbit/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-orbit/integration_tests/invalid_config.json @@ -1 +1,5 @@ -{ "api_token": "obw_token", "workspace": "airbyte" } +{ + "api_token": "invalid_api_key", + "workspace": "airbyte", + "start_date": "2022-06-26" +} diff --git a/airbyte-integrations/connectors/source-orbit/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-orbit/integration_tests/sample_config.json new file mode 100644 index 000000000000..c9a6730146d3 --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/integration_tests/sample_config.json @@ -0,0 +1,5 @@ +{ + "api_token": "api_key", + "workspace": "airbyte", + "start_date": "2022-06-26" +} diff --git a/airbyte-integrations/connectors/source-orbit/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-orbit/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-orbit/metadata.yaml b/airbyte-integrations/connectors/source-orbit/metadata.yaml index 60494a01a6e3..5accdc194140 100644 --- a/airbyte-integrations/connectors/source-orbit/metadata.yaml +++ b/airbyte-integrations/connectors/source-orbit/metadata.yaml @@ -1,24 +1,28 @@ data: + allowedHosts: + hosts: + - "*" + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: 95bcc041-1d1a-4c2e-8802-0ca5b1bfa36a - dockerImageTag: 0.1.1 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-orbit githubIssueLabel: source-orbit icon: orbit.svg license: MIT name: Orbit - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: "2022-06-27" releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/orbit tags: - - language:python + - language:low-code ab_internal: sl: 100 ql: 100 - supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-orbit/requirements.txt b/airbyte-integrations/connectors/source-orbit/requirements.txt index d6e1198b1ab1..427ac11784fe 100644 --- a/airbyte-integrations/connectors/source-orbit/requirements.txt +++ b/airbyte-integrations/connectors/source-orbit/requirements.txt @@ -1 +1,2 @@ -e . +-e ../../bases/connector-acceptance-test diff --git a/airbyte-integrations/connectors/source-orbit/setup.py b/airbyte-integrations/connectors/source-orbit/setup.py index c16dff7df89a..ae8d7e9b1c8b 100644 --- a/airbyte-integrations/connectors/source-orbit/setup.py +++ b/airbyte-integrations/connectors/source-orbit/setup.py @@ -5,15 +5,9 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", -] +MAIN_REQUIREMENTS = ["airbyte-cdk"] -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest~=6.1", - "pytest-mock~=3.6.1", -] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( name="source_orbit", diff --git a/airbyte-integrations/connectors/source-orbit/source_orbit/manifest.yaml b/airbyte-integrations/connectors/source-orbit/source_orbit/manifest.yaml new file mode 100644 index 000000000000..be2c6404fcc9 --- /dev/null +++ b/airbyte-integrations/connectors/source-orbit/source_orbit/manifest.yaml @@ -0,0 +1,69 @@ +version: "0.52.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["data"] + requester: + type: HttpRequester + url_base: "https://app.orbit.love/api/v1/" + http_method: "GET" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['api_token'] }}" + request_parameters: + start_date: "{{ config.get('start_date') }}" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + workspace_stream: + $ref: "#/definitions/base_stream" + name: "workspace" + primary_key: "id" + $parameters: + path: "workspaces/{{config['workspace']}}" + + members_stream: + $ref: "#/definitions/base_stream" + name: "members" + primary_key: "id" + $parameters: + path: "{{config['workspace']}}/members" + retriever: + $ref: "#/definitions/retriever" + paginator: + type: "DefaultPaginator" + pagination_strategy: + type: PageIncrement + start_from_page: 1 + page_size: 100 + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + inject_into: request_parameter + field_name: items + type: RequestOption + +streams: + - "#/definitions/workspace_stream" + - "#/definitions/members_stream" + +check: + type: CheckStream + stream_names: + - "workspace" diff --git a/airbyte-integrations/connectors/source-orbit/source_orbit/schemas/members.json b/airbyte-integrations/connectors/source-orbit/source_orbit/schemas/members.json index eac6de53806e..f4e0f9e5f07e 100644 --- a/airbyte-integrations/connectors/source-orbit/source_orbit/schemas/members.json +++ b/airbyte-integrations/connectors/source-orbit/source_orbit/schemas/members.json @@ -1,24 +1,26 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "fake": { - "type": "string" + "type": ["null", "string"] }, "type": { - "type": "string" + "type": ["null", "string"] }, "attributes": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "activities_count": { - "type": "integer" + "type": ["null", "integer"] }, "activities_score": { - "type": "integer" + "type": ["null", "number"] }, "avatar_url": { "type": ["null", "string"] @@ -71,13 +73,13 @@ "tag_list": { "type": ["null", "array"], "items": { - "type": "string" + "type": ["null", "string"] } }, "tags": { "type": ["null", "array"], "items": { - "type": "string" + "type": ["null", "string"] } }, "teammate": { @@ -99,10 +101,10 @@ "type": ["null", "string"] }, "created": { - "type": "boolean" + "type": ["null", "boolean"] }, "id": { - "type": "string" + "type": ["null", "string"] }, "orbit_level": { "type": ["null", "integer"] @@ -140,16 +142,21 @@ "topics": { "type": ["null", "array"], "items": { - "type": "string" + "type": ["null", "string"] } }, "languages": { "type": ["null", "array"], "items": { - "type": "string" + "type": ["null", "string"] } } } + }, + "relationships": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} } } } diff --git a/airbyte-integrations/connectors/source-orbit/source_orbit/schemas/workspace.json b/airbyte-integrations/connectors/source-orbit/source_orbit/schemas/workspace.json index 79a2ed3d7982..1ab324bae28a 100644 --- a/airbyte-integrations/connectors/source-orbit/source_orbit/schemas/workspace.json +++ b/airbyte-integrations/connectors/source-orbit/source_orbit/schemas/workspace.json @@ -1,36 +1,74 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "type": { - "type": "string" + "type": ["null", "string"] + }, + "relationships": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} }, "attributes": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "name": { - "type": "string" + "type": ["null", "string"] }, "slug": { - "type": "string" + "type": ["null", "string"] }, "updated_at": { - "type": "string" + "type": ["null", "string"] }, "created_at": { - "type": "string" + "type": ["null", "string"] }, "members_count": { - "type": "integer" + "type": ["null", "integer"] }, "activities_count": { - "type": "integer" + "type": ["null", "integer"] }, "tags": { - "type": "object" + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} + } + } + }, + "relationships": { + "type": "object", + "properties": { + "last_member": { + "type": ["null", "object"], + "properties": { + "data": { + "type": ["null", "object"] + } + } + }, + "last_activity": { + "type": ["null", "object"], + "properties": { + "data": { + "type": ["null", "object"] + } + } + }, + "repositories": { + "type": ["null", "object"], + "properties": { + "data": { + "type": ["null", "array"] + } + } } } } diff --git a/airbyte-integrations/connectors/source-orbit/source_orbit/source.py b/airbyte-integrations/connectors/source-orbit/source_orbit/source.py index 72d22905c5d2..1fcba693e608 100644 --- a/airbyte-integrations/connectors/source-orbit/source_orbit/source.py +++ b/airbyte-integrations/connectors/source-orbit/source_orbit/source.py @@ -2,36 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from typing import Any, List, Mapping, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +WARNING: Do not modify this file. +""" -from .streams import Members, Workspace - -# Source -class SourceOrbit(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - try: - workspace_stream = Workspace( - authenticator=TokenAuthenticator(token=config["api_token"]), - workspace=config["workspace"], - ) - next(workspace_stream.read_records(sync_mode=SyncMode.full_refresh)) - return True, None - except Exception as e: - return False, f"Please check that your API key and workspace name are entered correctly: {repr(e)}" - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - - stream_kwargs = { - "authenticator": TokenAuthenticator(config["api_token"]), - "workspace": config["workspace"], - "start_date": config["start_date"], - } - - return [Members(**stream_kwargs), Workspace(**stream_kwargs)] +# Declarative Source +class SourceOrbit(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-orbit/source_orbit/spec.yaml b/airbyte-integrations/connectors/source-orbit/source_orbit/spec.yaml index 8277b6d61539..61dbd1b3357d 100644 --- a/airbyte-integrations/connectors/source-orbit/source_orbit/spec.yaml +++ b/airbyte-integrations/connectors/source-orbit/source_orbit/spec.yaml @@ -6,7 +6,7 @@ connectionSpecification: required: - api_token - workspace - additionalProperties: false + additionalProperties: true properties: api_token: type: string diff --git a/airbyte-integrations/connectors/source-orbit/source_orbit/streams.py b/airbyte-integrations/connectors/source-orbit/source_orbit/streams.py deleted file mode 100644 index 6d70301483cf..000000000000 --- a/airbyte-integrations/connectors/source-orbit/source_orbit/streams.py +++ /dev/null @@ -1,96 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import urllib.parse -from abc import ABC -from typing import Any, Iterable, Mapping, MutableMapping, Optional - -import requests -from airbyte_cdk.sources.streams.http import HttpStream - - -class OrbitStream(HttpStream, ABC): - url_base = "https://app.orbit.love/api/v1/" - - def __init__(self, workspace: str, start_date: Optional[str] = None, **kwargs): - super().__init__(**kwargs) - self.workspace = workspace - self.start_date = start_date - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - def parse_response( - self, - response: requests.Response, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[Mapping]: - data = response.json() - records = data["data"] - yield from records - - -class OrbitStreamPaginated(OrbitStream): - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, str]]: - decoded_response = response.json() - links = decoded_response.get("links") - if not links: - return None - - next = links.get("next") - if not next: - return None - - next_url = urllib.parse.urlparse(next) - return {str(k): str(v) for (k, v) in urllib.parse.parse_qsl(next_url.query)} - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - - params = super().request_params(stream_state, stream_slice, next_page_token) - return {**params, **next_page_token} if next_page_token else params - - -class Members(OrbitStreamPaginated): - # Docs: https://docs.orbit.love/reference/members-overview - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return f"{self.workspace}/members" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - - params = super().request_params(stream_state, stream_slice, next_page_token) - params["sort"] = "created_at" - if self.start_date is not None: - params["start_date"] = self.start_date # The start_date parameter is filtering the last_activity_occurred_at field - return params - - -class Workspace(OrbitStream): - # Docs: https://docs.orbit.love/reference/get_workspaces-workspace-slug - # This stream is primarily used for connnection checking. - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return f"workspaces/{self.workspace}" - - def parse_response( - self, - response: requests.Response, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[Mapping]: - data = response.json() - yield data["data"] diff --git a/airbyte-integrations/connectors/source-orbit/unit_tests/test_source.py b/airbyte-integrations/connectors/source-orbit/unit_tests/test_source.py deleted file mode 100644 index a30df4de7351..000000000000 --- a/airbyte-integrations/connectors/source-orbit/unit_tests/test_source.py +++ /dev/null @@ -1,38 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -import pytest -from source_orbit.source import SourceOrbit, Workspace - - -@pytest.mark.parametrize( - "read_records_side_effect, expected_return_value, expected_error_message", - [ - (iter(["foo", "bar"]), True, None), - ( - Exception("connection error"), - False, - "Please check that your API key and workspace name are entered correctly: Exception('connection error')", - ), - ], -) -def test_check_connection(mocker, read_records_side_effect, expected_return_value, expected_error_message): - source = SourceOrbit() - if expected_error_message: - read_records_mock = mocker.Mock(side_effect=read_records_side_effect) - else: - read_records_mock = mocker.Mock(return_value=read_records_side_effect) - mocker.patch.object(Workspace, "read_records", read_records_mock) - logger_mock, config_mock = MagicMock(), MagicMock() - assert source.check_connection(logger_mock, config_mock) == (expected_return_value, expected_error_message) - - -def test_streams(mocker): - source = SourceOrbit() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 2 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-orbit/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-orbit/unit_tests/test_streams.py deleted file mode 100644 index 1840de41cfec..000000000000 --- a/airbyte-integrations/connectors/source-orbit/unit_tests/test_streams.py +++ /dev/null @@ -1,98 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_orbit.streams import Members, OrbitStream, OrbitStreamPaginated - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(OrbitStream, "path", "v0/example_endpoint") - mocker.patch.object(OrbitStream, "primary_key", "test_primary_key") - mocker.patch.object(OrbitStream, "__abstractmethods__", set()) - mocker.patch.object(OrbitStreamPaginated, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = OrbitStream(workspace="workspace") - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = OrbitStream(workspace="workspace") - inputs = {"response": MagicMock()} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_parse_response(patch_base_class, mocker): - stream = OrbitStream(workspace="workspace") - inputs = {"response": mocker.Mock(json=mocker.Mock(return_value={"data": ["foo", "bar"]}))} - gen = stream.parse_response(**inputs) - assert next(gen) == "foo" - assert next(gen) == "bar" - - -def test_request_headers(patch_base_class): - stream = OrbitStream(workspace="workspace") - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = OrbitStream(workspace="workspace") - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = OrbitStream(workspace="workspace") - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = OrbitStream(workspace="workspace") - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time - - -class TestOrbitStreamPaginated: - @pytest.mark.parametrize( - "json_response, expected_token", [({"links": {"next": "http://foo.bar/api?a=b&c=d"}}, {"a": "b", "c": "d"}), ({}, None)] - ) - def test_next_page_token(self, patch_base_class, mocker, json_response, expected_token): - stream = OrbitStreamPaginated(workspace="workspace") - inputs = {"response": mocker.Mock(json=mocker.Mock(return_value=json_response))} - assert stream.next_page_token(**inputs) == expected_token - - -class TestMembers: - @pytest.mark.parametrize("start_date", [None, "2022-06-27"]) - def test_members_request_params(self, patch_base_class, start_date): - stream = Members(workspace="workspace", start_date=start_date) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - if start_date is not None: - expected_params = {"sort": "created_at", "start_date": start_date} - else: - expected_params = {"sort": "created_at"} - assert stream.request_params(**inputs) == expected_params diff --git a/airbyte-integrations/connectors/source-oura/README.md b/airbyte-integrations/connectors/source-oura/README.md index 77794dd6e6b8..33f60603f2e6 100644 --- a/airbyte-integrations/connectors/source-oura/README.md +++ b/airbyte-integrations/connectors/source-oura/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-oura:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/oura) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_oura/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-oura:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-oura build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-oura:airbyteDocker +An image will be built with the tag `airbyte/source-oura:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-oura:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-oura:dev check --confi docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-oura:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-oura:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-oura test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-oura:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-oura:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-oura test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/oura.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-oura/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-oura/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-oura/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-oura/build.gradle b/airbyte-integrations/connectors/source-oura/build.gradle deleted file mode 100644 index facec3b6d3e7..000000000000 --- a/airbyte-integrations/connectors/source-oura/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_oura' -} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/README.md b/airbyte-integrations/connectors/source-outbrain-amplify/README.md index dafe820e594b..8a0bf0a9eb51 100644 --- a/airbyte-integrations/connectors/source-outbrain-amplify/README.md +++ b/airbyte-integrations/connectors/source-outbrain-amplify/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-outbrain-amplify:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/outbrain-amplify) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_outbrain_amplify/spec.yaml` file. @@ -57,24 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-outbrain-amplify:dev -``` -If you want to build the Docker image with the CDK on your local machine (rather than the most recent package published to pypi), from the airbyte base directory run: +#### Build +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** ```bash -CONNECTOR_TAG= CONNECTOR_NAME= sh airbyte-integrations/scripts/build-connector-image-with-local-cdk.sh +airbyte-ci connectors --name=source-outbrain-amplify build ``` +An image will be built with the tag `airbyte/source-outbrain-amplify:dev`. -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-outbrain-amplify:airbyteDocker +**Via `docker build`:** +```bash +docker build -t airbyte/source-outbrain-amplify:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -84,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-outbrain-amplify:dev c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-outbrain-amplify:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-outbrain-amplify:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-outbrain-amplify test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-outbrain-amplify:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-outbrain-amplify:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -131,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-outbrain-amplify test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/outbrain-amplify.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/acceptance-test-config.yml b/airbyte-integrations/connectors/source-outbrain-amplify/acceptance-test-config.yml index 5411d3cede72..77a5173ea17b 100644 --- a/airbyte-integrations/connectors/source-outbrain-amplify/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-outbrain-amplify/acceptance-test-config.yml @@ -52,20 +52,20 @@ acceptance_tests: bypass_reason: "no records" - name: performance_report_marketers_by_interest bypass_reason: "no records" -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.jsonl" -# extra_fields: no -# exact_order: no -# extra_records: yes -# incremental: -# bypass_reason: "This connector does not implement incremental sync" -# TODO uncomment this block this block if your connector implements incremental sync: -# tests: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# future_state: -# future_state_path: "integration_tests/abnormal_state.json" + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.jsonl" + # extra_fields: no + # exact_order: no + # extra_records: yes + # incremental: + # bypass_reason: "This connector does not implement incremental sync" + # TODO uncomment this block this block if your connector implements incremental sync: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-outbrain-amplify/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-outbrain-amplify/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/build.gradle b/airbyte-integrations/connectors/source-outbrain-amplify/build.gradle deleted file mode 100644 index 51126f788062..000000000000 --- a/airbyte-integrations/connectors/source-outbrain-amplify/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_outbrain_amplify' -} diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/metadata.yaml b/airbyte-integrations/connectors/source-outbrain-amplify/metadata.yaml index 0e7bd169237f..502c3d43ef58 100644 --- a/airbyte-integrations/connectors/source-outbrain-amplify/metadata.yaml +++ b/airbyte-integrations/connectors/source-outbrain-amplify/metadata.yaml @@ -14,7 +14,6 @@ data: license: MIT name: Outbrain Amplify releaseStage: alpha - supportUrl: https://docs.airbyte.com/integrations/sources/outbrain-amplify documentationUrl: https://docs.airbyte.com/integrations/sources/outbrain-amplify tags: - language:low-code diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_source.py b/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_source.py index c80a9bc4ef26..d1b77045b244 100644 --- a/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_source.py @@ -9,15 +9,11 @@ def test_check_connection(mocker): config = { - "credentials": - { - "type": "access_token", - "access_token" : "MTY1OTUyO" - }, + "credentials": {"type": "access_token", "access_token": "MTY1OTUyO"}, "report_granularity": "daily", "geo_location_breakdown": "region", - "start_date" : "2022-04-01", - "end_date" : "2022-04-30" + "start_date": "2022-04-01", + "end_date": "2022-04-30", } source = SourceOutbrainAmplify() cond = source.check_connection(True, config)[0] diff --git a/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_streams.py index 1fa51bef5b6f..c683d0d87d45 100644 --- a/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-outbrain-amplify/unit_tests/test_streams.py @@ -32,24 +32,20 @@ def test_next_page_token(patch_base_class): pagination_token = json.dumps(inputs) response = requests.Response() response.status_code = 200 - response.headers['content-type'] = 'application/json' - response._content = pagination_token.encode('utf-8') - expected_token = {'offset': 51} + response.headers["content-type"] = "application/json" + response._content = pagination_token.encode("utf-8") + expected_token = {"offset": 51} assert stream.next_page_token(response) == expected_token def test_parse_response(patch_base_class): stream = OutbrainAmplifyStream() - mock_response = { - "campaigns": [], - "totalCount": 5, - "count": 0 - } + mock_response = {"campaigns": [], "totalCount": 5, "count": 0} mock_response = json.dumps(mock_response) response = requests.Response() response.status_code = 200 - response.headers['Content-Type'] = 'application/json' - response._content = mock_response.encode('utf-8') + response.headers["Content-Type"] = "application/json" + response._content = mock_response.encode("utf-8") result = stream.parse_response(response) expected_result = True assert inspect.isgenerator(result) == expected_result diff --git a/airbyte-integrations/connectors/source-outreach/Dockerfile b/airbyte-integrations/connectors/source-outreach/Dockerfile index 58ceb06732fb..6ec63a2bbd70 100644 --- a/airbyte-integrations/connectors/source-outreach/Dockerfile +++ b/airbyte-integrations/connectors/source-outreach/Dockerfile @@ -34,5 +34,5 @@ COPY source_outreach ./source_outreach ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.4.0 +LABEL io.airbyte.version=0.5.0 LABEL io.airbyte.name=airbyte/source-outreach diff --git a/airbyte-integrations/connectors/source-outreach/README.md b/airbyte-integrations/connectors/source-outreach/README.md index 927f6900a64f..60d09b5cecf0 100644 --- a/airbyte-integrations/connectors/source-outreach/README.md +++ b/airbyte-integrations/connectors/source-outreach/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-outreach:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/outreach) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_outreach/spec.json` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-outreach:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-outreach build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-outreach:airbyteDocker +An image will be built with the tag `airbyte/source-outreach:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-outreach:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-outreach:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-outreach:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-outreach:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-outreach test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-outreach:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-outreach:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-outreach test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/outreach.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-outreach/acceptance-test-config.yml b/airbyte-integrations/connectors/source-outreach/acceptance-test-config.yml index 269933b1f239..ce125a866d2d 100644 --- a/airbyte-integrations/connectors/source-outreach/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-outreach/acceptance-test-config.yml @@ -1,25 +1,41 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-outreach:dev -tests: +acceptance_tests: spec: - - spec_path: "source_outreach/spec.json" + tests: + - spec_path: "source_outreach/spec.json" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" # https://github.com/airbytehq/airbyte/issues/8180 - # basic_read: - # - config_path: "secrets/config.json" - # configured_catalog_path: "integration_tests/configured_catalog.json" - # empty_streams: [] - # incremental: - # - config_path: "secrets/config.json" - # configured_catalog_path: "integration_tests/configured_catalog.json" - # future_state_path: "integration_tests/abnormal_state.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: "sequence_states" + bypass_reason: "Sandbox account cannot seed the stream" + - name: "sequence_steps" + bypass_reason: "Sandbox account cannot seed the stream" + - name: "snippets" + bypass_reason: "Sandbox account cannot seed the stream" + - name: "templates" + bypass_reason: "Sandbox account cannot seed the stream" + incremental: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + skip_comprehensive_incremental_tests: true full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-outreach/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-outreach/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-outreach/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-outreach/build.gradle b/airbyte-integrations/connectors/source-outreach/build.gradle deleted file mode 100644 index 575c2a569c3d..000000000000 --- a/airbyte-integrations/connectors/source-outreach/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_outreach' -} diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-outreach/integration_tests/abnormal_state.json index aecfa1a5a17b..bbc342b704a6 100644 --- a/airbyte-integrations/connectors/source-outreach/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-outreach/integration_tests/abnormal_state.json @@ -32,6 +32,12 @@ "calls": { "updatedAt": "2040-11-16T00:00:00Z" }, + "call_purposes": { + "updatedAt": "2040-11-16T00:00:00Z" + }, + "call_dispositions": { + "updatedAt": "2040-11-16T00:00:00Z" + }, "users": { "updatedAt": "2040-11-16T00:00:00Z" }, diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-outreach/integration_tests/configured_catalog.json index 1139b289868d..687f2734c797 100644 --- a/airbyte-integrations/connectors/source-outreach/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-outreach/integration_tests/configured_catalog.json @@ -1,199 +1,259 @@ { "streams": [ { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { - "name": "prospects", + "default_cursor_field": ["updatedAt"], "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "name": "prospects", + "namespace": null, "source_defined_cursor": true, - "default_cursor_field": ["updatedAt"], - "source_defined_primary_key": [["id"]] + "source_defined_primary_key": [["id"]], + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["updatedAt"] + "sync_mode": "full_refresh" }, { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { - "name": "sequences", + "default_cursor_field": ["updatedAt"], "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "name": "sequences", + "namespace": null, "source_defined_cursor": true, - "default_cursor_field": ["updatedAt"], - "source_defined_primary_key": [["id"]] + "source_defined_primary_key": [["id"]], + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["updatedAt"] + "sync_mode": "full_refresh" }, { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { - "name": "sequence_states", + "default_cursor_field": ["updatedAt"], "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "name": "sequence_states", + "namespace": null, "source_defined_cursor": true, - "default_cursor_field": ["updatedAt"], - "source_defined_primary_key": [["id"]] + "source_defined_primary_key": [["id"]], + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["updatedAt"] + "sync_mode": "full_refresh" }, { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { - "name": "sequence_steps", + "default_cursor_field": ["updatedAt"], "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "name": "sequence_steps", + "namespace": null, "source_defined_cursor": true, - "default_cursor_field": ["updatedAt"], - "source_defined_primary_key": [["id"]] + "source_defined_primary_key": [["id"]], + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["updatedAt"] + "sync_mode": "full_refresh" }, { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { - "name": "accounts", + "default_cursor_field": ["updatedAt"], "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "name": "accounts", + "namespace": null, "source_defined_cursor": true, - "default_cursor_field": ["updatedAt"], - "source_defined_primary_key": [["id"]] + "source_defined_primary_key": [["id"]], + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["updatedAt"] + "sync_mode": "full_refresh" }, { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { - "name": "opportunities", + "default_cursor_field": ["updatedAt"], "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "name": "opportunities", + "namespace": null, "source_defined_cursor": true, - "default_cursor_field": ["updatedAt"], - "source_defined_primary_key": [["id"]] + "source_defined_primary_key": [["id"]], + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["updatedAt"] + "sync_mode": "full_refresh" }, { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { - "name": "personas", + "default_cursor_field": ["updatedAt"], "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "name": "personas", + "namespace": null, "source_defined_cursor": true, - "default_cursor_field": ["updatedAt"], - "source_defined_primary_key": [["id"]] + "source_defined_primary_key": [["id"]], + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["updatedAt"] + "sync_mode": "full_refresh" }, { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { - "name": "mailings", + "default_cursor_field": ["updatedAt"], "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "name": "mailings", + "namespace": null, "source_defined_cursor": true, - "default_cursor_field": ["updatedAt"], - "source_defined_primary_key": [["id"]] + "source_defined_primary_key": [["id"]], + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["updatedAt"] + "sync_mode": "full_refresh" }, { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { - "name": "mailboxes", + "default_cursor_field": ["updatedAt"], "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "name": "mailboxes", + "namespace": null, "source_defined_cursor": true, - "default_cursor_field": ["updatedAt"], - "source_defined_primary_key": [["id"]] + "source_defined_primary_key": [["id"]], + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["updatedAt"] + "sync_mode": "full_refresh" }, { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { - "name": "stages", + "default_cursor_field": ["updatedAt"], "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "name": "stages", + "namespace": null, "source_defined_cursor": true, - "default_cursor_field": ["updatedAt"], - "source_defined_primary_key": [["id"]] + "source_defined_primary_key": [["id"]], + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["updatedAt"] + "sync_mode": "full_refresh" }, { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { - "name": "calls", + "default_cursor_field": ["updatedAt"], "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "name": "calls", + "namespace": null, "source_defined_cursor": true, - "default_cursor_field": ["updatedAt"], - "source_defined_primary_key": [["id"]] + "source_defined_primary_key": [["id"]], + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["updatedAt"] + "sync_mode": "full_refresh" }, { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { - "name": "users", + "default_cursor_field": ["updatedAt"], "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "name": "call_purposes", + "namespace": null, "source_defined_cursor": true, - "default_cursor_field": ["updatedAt"], - "source_defined_primary_key": [["id"]] + "source_defined_primary_key": [["id"]], + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["updatedAt"] + "sync_mode": "full_refresh" }, { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { - "name": "tasks", + "default_cursor_field": ["updatedAt"], "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "name": "call_dispositions", + "namespace": null, "source_defined_cursor": true, - "default_cursor_field": ["updatedAt"], - "source_defined_primary_key": [["id"]] + "source_defined_primary_key": [["id"]], + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["updatedAt"] + "sync_mode": "full_refresh" }, { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { - "name": "templates", + "default_cursor_field": ["updatedAt"], "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "name": "users", + "namespace": null, "source_defined_cursor": true, + "source_defined_primary_key": [["id"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { "default_cursor_field": ["updatedAt"], - "source_defined_primary_key": [["id"]] + "json_schema": {}, + "name": "tasks", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["id"]], + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["updatedAt"] + "sync_mode": "full_refresh" }, { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, "stream": { - "name": "snippets", + "default_cursor_field": ["updatedAt"], "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], + "name": "templates", + "namespace": null, "source_defined_cursor": true, + "source_defined_primary_key": [["id"]], + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "full_refresh" + }, + { + "cursor_field": null, + "destination_sync_mode": "append", + "primary_key": null, + "stream": { "default_cursor_field": ["updatedAt"], - "source_defined_primary_key": [["id"]] + "json_schema": {}, + "name": "snippets", + "namespace": null, + "source_defined_cursor": true, + "source_defined_primary_key": [["id"]], + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["updatedAt"] + "sync_mode": "full_refresh" } ] } diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-outreach/integration_tests/sample_state.json index d11f3710bb3c..0708e17e1142 100644 --- a/airbyte-integrations/connectors/source-outreach/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-outreach/integration_tests/sample_state.json @@ -32,6 +32,12 @@ "calls": { "updatedAt": "2021-06-28T10:10:20Z" }, + "call_purposes": { + "updatedAt": "2021-06-28T10:10:20Z" + }, + "call_dispositions": { + "updatedAt": "2021-06-28T10:10:20Z" + }, "users": { "updatedAt": "2021-06-28T10:10:20Z" }, diff --git a/airbyte-integrations/connectors/source-outreach/metadata.yaml b/airbyte-integrations/connectors/source-outreach/metadata.yaml index 278b8bad0daa..30fa65743964 100644 --- a/airbyte-integrations/connectors/source-outreach/metadata.yaml +++ b/airbyte-integrations/connectors/source-outreach/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 3490c201-5d95-4783-b600-eaf07a4c7787 - dockerImageTag: 0.4.0 + dockerImageTag: 0.5.0 dockerRepository: airbyte/source-outreach documentationUrl: https://docs.airbyte.com/integrations/sources/outreach githubIssueLabel: source-outreach diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/accounts.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/accounts.json index 0518e59d4c10..939890dd88e0 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/accounts.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/accounts.json @@ -6,6 +6,18 @@ "id": { "type": "integer" }, + "assignedTeams": { + "type": ["null", "array"] + }, + "assignedUsers": { + "items": { + "type": ["null", "integer"] + }, + "type": ["null", "array"] + }, + "batches": { + "type": ["null", "array"] + }, "buyerIntentScore": { "type": ["null", "number"] }, @@ -469,6 +481,9 @@ "customId": { "type": ["null", "string"] }, + "defaultPluginMapping": { + "type": ["null", "array"] + }, "description": { "type": ["null", "string"] }, @@ -478,6 +493,9 @@ "externalSource": { "type": ["null", "string"] }, + "favorites": { + "type": ["null", "array"] + }, "followers": { "type": ["null", "integer"] }, diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/call_dispositions.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/call_dispositions.json new file mode 100644 index 000000000000..1b3134149ab0 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/call_dispositions.json @@ -0,0 +1,42 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": ["null", "string"] + }, + "defaultForOutboundVoicemail": { + "type": ["null", "boolean"] + }, + "outcome": { + "type": ["null", "string"] + }, + "order": { + "type": ["null", "integer"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "creator": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "calls": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/call_purposes.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/call_purposes.json new file mode 100644 index 000000000000..7d00cf971497 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/call_purposes.json @@ -0,0 +1,36 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": ["null", "string"] + }, + "order": { + "type": ["null", "integer"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "creator": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "calls": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/calls.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/calls.json index 2a4666dac51b..005292390084 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/calls.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/calls.json @@ -6,6 +6,9 @@ "id": { "type": "integer" }, + "batches": { + "type": ["null", "array"] + }, "externalVendor": { "type": ["null", "string"] }, @@ -18,9 +21,15 @@ "note": { "type": ["null", "string"] }, + "outboundVoicemail": { + "type": ["null", "array"] + }, "outcome": { "type": ["null", "string"] }, + "phone": { + "type": ["null", "array"] + }, "recordingUrl": { "type": ["null", "string"] }, @@ -112,6 +121,9 @@ "type": ["null", "integer"] } }, + "recordingDeletionReason": { + "type": ["null", "string"] + }, "sequence": { "type": ["null", "array"], "items": { diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/mailboxes.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/mailboxes.json index 4285c3ea9271..b75c4773417f 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/mailboxes.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/mailboxes.json @@ -19,15 +19,27 @@ "email": { "type": ["null", "string"] }, + "emailHash": { + "type": ["null", "string"] + }, "emailProvider": { "type": ["null", "string"] }, "emailSignature": { "type": ["null", "string"] }, + "ewsCustomSearchFolder": { + "type": ["null", "boolean"] + }, + "ewsDelegateSync": { + "type": ["null", "boolean"] + }, "ewsEndpoint": { "type": ["null", "string"] }, + "ewsImpersonation": { + "type": ["null", "boolean"] + }, "ewsSslVerifyMode": { "type": ["null", "integer"] }, diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/mailings.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/mailings.json index 29b36cddd5b7..fc0ff4bcad8a 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/mailings.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/mailings.json @@ -6,6 +6,15 @@ "id": { "type": "integer" }, + "attachments": { + "type": ["null", "array"] + }, + "attributableSequenceId": { + "type": ["null", "integer"] + }, + "attributableSequenceName": { + "type": ["null", "string"] + }, "bodyHtml": { "type": ["null", "string"] }, @@ -19,6 +28,9 @@ "clickCount": { "type": ["null", "integer"] }, + "calendar": { + "type": ["null", "array"] + }, "clickedAt": { "type": ["null", "string"], "format": "date-time" @@ -27,6 +39,10 @@ "type": ["null", "string"], "format": "date-time" }, + "desiredAt": { + "type": ["null", "string"], + "format": "date-time" + }, "errorBacktrace": { "type": ["null", "string"] }, @@ -50,6 +66,18 @@ "type": ["null", "string"], "format": "date-time" }, + "meetingDescription": { + "type": ["null", "string"] + }, + "meetingDuration": { + "type": ["null", "integer"] + }, + "meetingLocation": { + "type": ["null", "string"] + }, + "meetingTitle": { + "type": ["null", "string"] + }, "messageId": { "type": ["null", "string"] }, @@ -60,6 +88,9 @@ "type": ["null", "string"], "format": "date-time" }, + "notifyThreadStatus": { + "type": ["null", "string"] + }, "openCount": { "type": ["null", "integer"] }, @@ -67,16 +98,23 @@ "type": ["null", "string"], "format": "date-time" }, + "optimizedScheduledAt": { + "type": ["null", "string"], + "format": "date-time" + }, "overrideSafetySettings": { "type": ["null", "boolean"] }, "references": { - "type": ["null", "string"] + "type": ["null", "array"] }, "repliedAt": { "type": ["null", "string"], "format": "date-time" }, + "replySentiment": { + "type": ["null", "string"] + }, "retryAt": { "type": ["null", "string"], "format": "date-time" @@ -87,6 +125,12 @@ "retryInterval": { "type": ["null", "integer"] }, + "schedule": { + "type": ["null", "array"] + }, + "scheduleId": { + "type": ["null", "integer"] + }, "scheduledAt": { "type": ["null", "string"], "format": "date-time" @@ -126,6 +170,16 @@ "type": ["null", "integer"] } }, + "followUpSequenceId": { + "type": ["null", "integer"] + }, + "followUpSequenceName": { + "type": ["null", "string"] + }, + "followUpSequenceStartingDate": { + "type": ["null", "string"], + "format": "date-time" + }, "mailbox": { "type": ["null", "array"], "items": { diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/opportunities.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/opportunities.json index 997c02324c55..346a64ed1328 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/opportunities.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/opportunities.json @@ -6,6 +6,15 @@ "id": { "type": "integer" }, + "assignedTeams": { + "type": ["null", "array"] + }, + "assignedUsers": { + "items": { + "type": ["null", "integer"] + }, + "type": ["null", "array"] + }, "amount": { "type": ["null", "integer"] }, @@ -470,13 +479,22 @@ "custom99": { "type": ["null", "string"] }, + "defaultPluginMapping": { + "type": ["null", "array"] + }, "description": { "type": ["null", "string"] }, + "externalCreator": { + "type": ["null", "array"] + }, "externalCreatedAt": { "type": ["null", "string"], "format": "date-time" }, + "favorites": { + "type": ["null", "array"] + }, "healthCategory": { "type": ["null", "string"] }, @@ -490,7 +508,7 @@ "type": ["null", "string"] }, "mapNumberOfOverdueTasks": { - "type": ["null", "string"] + "type": ["null", "integer"] }, "mapStatus": { "type": ["null", "string"] @@ -501,9 +519,18 @@ "nextStep": { "type": ["null", "string"] }, + "opportunityHealthFactors": { + "items": { + "type": ["null", "integer"] + }, + "type": ["null", "array"] + }, "opportunityType": { "type": ["null", "string"] }, + "primaryProspect": { + "type": ["null", "array"] + }, "probability": { "type": ["null", "integer"] }, diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/prospects.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/prospects.json index 6a0fe48ab031..0ce82bcfcf85 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/prospects.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/prospects.json @@ -6,6 +6,18 @@ "id": { "type": "integer" }, + "accountName": { + "type": ["null", "string"] + }, + "assignedTeams": { + "type": ["null", "array"] + }, + "assignedUsers": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, "addedAt": { "type": ["null", "string"], "format": "date-time" @@ -35,6 +47,9 @@ "type": ["null", "string"], "format": "date-time" }, + "batches": { + "type": ["null", "array"] + }, "callOptedOut": { "type": ["null", "boolean"] }, @@ -54,6 +69,36 @@ "company": { "type": ["null", "string"] }, + "companyFollowers": { + "type": ["null", "integer"] + }, + "companyFoundedAt": { + "type": ["null", "string"] + }, + "companyIndustry": { + "type": ["null", "string"] + }, + "companyLinkedIn": { + "type": ["null", "string"] + }, + "companyLinkedInEmployees": { + "type": ["null", "string"] + }, + "companyLocality": { + "type": ["null", "string"] + }, + "companyNatural": { + "type": ["null", "string"] + }, + "companySize": { + "type": ["null", "integer"] + }, + "companyStartDate": { + "type": ["null", "string"] + }, + "companyType": { + "type": ["null", "string"] + }, "contactHistogram": { "type": ["null", "array"], "items": { @@ -352,6 +397,9 @@ "custom49": { "type": ["null", "string"] }, + "custom5": { + "type": ["null", "string"] + }, "custom50": { "type": ["null", "string"] }, @@ -518,6 +566,9 @@ "type": ["null", "string"], "format": "date-time" }, + "defaultPluginMapping": { + "type": ["null", "array"] + }, "degree": { "type": ["null", "string"] }, @@ -530,6 +581,54 @@ "type": "string" } }, + "emailContacts": { + "items": { + "additionalProperties": true, + "properties": { + "bounced_at": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "domain_hash": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "email_hash": { + "type": ["null", "string"] + }, + "email_type": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "order": { + "type": ["null", "integer"] + }, + "prospect_id": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "string"] + }, + "status_changed_at": { + "type": ["null", "string"] + }, + "unsubscribed_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "type": ["null", "array"] + }, "emailsOptStatus": { "type": ["null", "string"] }, @@ -637,6 +736,9 @@ "type": "string" } }, + "personaName": { + "type": "null" + }, "personalNote1": { "type": ["null", "string"] }, @@ -664,6 +766,15 @@ "sharingTeamId": { "type": ["null", "string"] }, + "smsOptStatus": { + "type": ["null", "string"] + }, + "smsOptedAt": { + "type": ["null", "string"] + }, + "smsOptedOut": { + "type": ["null", "boolean"] + }, "source": { "type": ["null", "string"] }, @@ -676,6 +787,9 @@ "stackOverflowUrl": { "type": ["null", "string"] }, + "stageName": { + "type": ["null", "string"] + }, "tags": { "type": ["null", "array"], "items": { @@ -702,6 +816,9 @@ "type": ["null", "string"], "format": "date-time" }, + "trashedByAccount": { + "type": ["null", "array"] + }, "twitterUrl": { "type": ["null", "string"] }, @@ -712,6 +829,12 @@ "type": ["null", "string"], "format": "date-time" }, + "updaterId": { + "type": ["null", "integer"] + }, + "updaterType": { + "type": ["null", "string"] + }, "voipPhones": { "type": ["null", "array"], "items": { diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/sequences.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/sequences.json index 910e977a6e67..28d85041c6c1 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/sequences.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/sequences.json @@ -88,6 +88,12 @@ "replyCount": { "type": ["null", "integer"] }, + "schedule": { + "items": { + "type": ["null", "integer"] + }, + "type": ["null", "array"] + }, "scheduleCount": { "type": ["null", "integer"] }, diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/stages.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/stages.json index 291fa583b220..c27dd111d251 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/stages.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/stages.json @@ -10,7 +10,7 @@ "type": ["null", "string"] }, "order": { - "type": ["null", "string"] + "type": ["null", "integer"] }, "color": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/tasks.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/tasks.json index af7241eedc96..950e0ac8ea46 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/tasks.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/tasks.json @@ -27,6 +27,9 @@ "type": ["null", "string"], "format": "date-time" }, + "defaultPluginMapping": { + "type": ["null", "array"] + }, "dueAt": { "type": ["null", "string"], "format": "date-time" @@ -115,6 +118,33 @@ "type": ["null", "integer"] } }, + "prospectAccount": { + "items": { + "type": ["null", "integer"] + }, + "type": ["null", "array"] + }, + "prospectContacts": { + "items": { + "type": ["null", "integer"] + }, + "type": ["null", "array"] + }, + "prospectOwner": { + "items": { + "type": ["null", "integer"] + }, + "type": ["null", "array"] + }, + "prospectPhoneNumbers": { + "type": ["null", "array"] + }, + "prospectStage": { + "items": { + "type": ["null", "integer"] + }, + "type": ["null", "array"] + }, "sequence": { "type": ["null", "array"], "items": { @@ -133,6 +163,18 @@ "type": ["null", "integer"] } }, + "sequenceStateSequenceStep": { + "type": ["null", "array"] + }, + "sequenceStateSequenceStepOverrides": { + "type": ["null", "array"] + }, + "sequenceStateStartingTemplate": { + "type": ["null", "array"] + }, + "sequenceStepOverrideTemplates": { + "type": ["null", "array"] + }, "sequenceStep": { "type": ["null", "array"], "items": { @@ -145,6 +187,9 @@ "type": ["null", "integer"] } }, + "sequenceTemplateTemplate": { + "type": ["null", "array"] + }, "subject": { "type": ["null", "array"], "items": { @@ -157,6 +202,12 @@ "type": ["null", "integer"] } }, + "taskTheme": { + "items": { + "type": ["null", "integer"] + }, + "type": ["null", "array"] + }, "template": { "type": ["null", "array"], "items": { diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/users.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/users.json index d3d1290b9da3..dd338a15e49b 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/users.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/users.json @@ -12,6 +12,15 @@ "activityNotificationsDisabled": { "type": ["null", "boolean"] }, + "activeProspectsCount": { + "type": ["null", "integer"] + }, + "batches": { + "type": ["null", "array"] + }, + "batchesViewId": { + "type": ["null", "integer"] + }, "bounceWarningEmailEnabled": { "type": ["null", "boolean"] }, @@ -21,9 +30,21 @@ "bridgePhoneExtension": { "type": ["null", "string"] }, + "buyerLanguagePreference": { + "type": ["null", "string"] + }, + "calendar": { + "type": ["null", "array"] + }, + "calendarEventsViewId": { + "type": ["null", "integer"] + }, "callsViewId": { "type": ["null", "integer"] }, + "contentCategoryOwnerships": { + "type": ["null", "array"] + }, "controlledTabDefault": { "type": ["null", "string"] }, @@ -53,6 +74,9 @@ "dailyDigestEmailEnabled": { "type": ["null", "boolean"] }, + "defaultScheduleId": { + "type": ["null", "integer"] + }, "defaultRulesetId": { "type": ["null", "integer"] }, @@ -73,6 +97,9 @@ }, "type": ["null", "array"] }, + "dutiesSetAt": { + "type": ["null", "string"] + }, "email": { "type": ["null", "string"] }, @@ -82,9 +109,18 @@ "engagementEmailsEnabled": { "type": ["null", "boolean"] }, + "favorites": { + "type": ["null", "array"] + }, "firstName": { "type": ["null", "string"] }, + "globalId": { + "type": ["null", "string"] + }, + "importsViewId": { + "type": "null" + }, "inboundBridgePhone": { "type": ["null", "string"] }, @@ -106,15 +142,27 @@ "inboundVoicemailPromptType": { "type": ["null", "string"] }, + "jobRole": { + "type": ["null", "array"] + }, "kaiaRecordingsViewId": { "type": ["null", "integer"] }, "keepBridgePhoneConnected": { "type": ["null", "boolean"] }, + "languagePreference": { + "type": ["null", "string"] + }, "lastName": { "type": ["null", "string"] }, + "linkToSequenceStateInTaskFlow": { + "type": ["null", "boolean"] + }, + "liveListenViewId": { + "type": ["null", "integer"] + }, "lastSignInAt": { "type": ["null", "string"], "format": "date-time" @@ -125,9 +173,21 @@ "mailboxErrorEmailEnabled": { "type": ["null", "boolean"] }, + "mailingsDeliveredCount": { + "type": ["null", "integer"] + }, + "mailingsRepliedCount": { + "type": ["null", "integer"] + }, + "mailingsViewId": { + "type": ["null", "integer"] + }, "meetingEngagementNotificationEnabled": { "type": ["null", "boolean"] }, + "meetingTypesViewId": { + "type": ["null", "integer"] + }, "name": { "type": ["null", "string"] }, @@ -155,13 +215,26 @@ "oceWindowMode": { "type": ["null", "boolean"] }, + "onboardedAt": { + "type": ["null", "string"], + "format": "date-time" + }, "opportunitiesViewId": { "type": ["null", "integer"] }, + "orcaStandaloneEnabled": { + "type": ["null", "boolean"] + }, + "outboundVoicemails": { + "type": ["null", "array"] + }, "passwordExpiresAt": { "type": ["null", "string"], "format": "date-time" }, + "phone": { + "type": ["null", "array"] + }, "phoneCountryCode": { "type": ["null", "string"] }, @@ -171,6 +244,15 @@ "phoneType": { "type": ["null", "string"] }, + "phones": { + "type": ["null", "array"] + }, + "phonesViewId": { + "type": ["null", "integer"] + }, + "prospectDetailDefault": { + "type": ["null", "string"] + }, "pluginAlertNotificationEnabled": { "type": ["null", "boolean"] }, @@ -186,9 +268,18 @@ "prospectsViewId": { "type": ["null", "integer"] }, + "reportsSequencePerformanceViewId": { + "type": ["null", "integer"] + }, "reportsTeamPerfViewId": { "type": ["null", "integer"] }, + "reportsTeamPerformanceIntradayViewId": { + "type": ["null", "integer"] + }, + "reportsTeamPerformanceViewId": { + "type": ["null", "integer"] + }, "reportsViewId": { "type": ["null", "integer"] }, @@ -201,15 +292,36 @@ "secondaryTimezone": { "type": ["null", "string"] }, + "sendInviteFallback": { + "type": ["null", "boolean"] + }, "senderNotificationsExcluded": { "type": ["null", "boolean"] }, + "sequenceStatesViewId": { + "type": ["null", "integer"] + }, + "sequencesViewId": { + "type": ["null", "integer"] + }, + "smsPhone": { + "type": ["null", "array"] + }, + "snippetsViewId": { + "type": ["null", "integer"] + }, + "tasksDueCount": { + "type": ["null", "integer"] + }, "tasksViewId": { "type": ["null", "integer"] }, "teamsViewId": { "type": ["null", "integer"] }, + "templatesViewId": { + "type": ["null", "integer"] + }, "tertiaryTimezone": { "type": ["null", "string"] }, @@ -235,6 +347,9 @@ "usersViewId": { "type": ["null", "integer"] }, + "voicemailPrompts": { + "type": ["null", "array"] + }, "voicemailNotificationEnabled": { "type": ["null", "boolean"] }, @@ -289,6 +404,9 @@ "type": ["null", "integer"] } }, + "useSalesNavigatorForLinkedInTasks": { + "type": ["null", "boolean"] + }, "updater": { "type": ["null", "array"], "items": { diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/source.py b/airbyte-integrations/connectors/source-outreach/source_outreach/source.py index 002b27ab7e96..016d92c7f59b 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/source.py +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/source.py @@ -208,6 +208,26 @@ def path(self, **kwargs) -> str: return "calls" +class CallPurposes(IncrementalOutreachStream): + """ + Call Purposes stream. Yields data from the GET /callPurposes endpoint. + See https://developers.outreach.io/api/reference/tag/Call-Purpose/ + """ + + def path(self, **kwargs) -> str: + return "callPurposes" + + +class CallDispositions(IncrementalOutreachStream): + """ + Call Dispositions stream. Yields data from the GET /callDispositions endpoint. + See https://developers.outreach.io/api/reference/tag/Call-Dispositions/ + """ + + def path(self, **kwargs) -> str: + return "callDispositions" + + class Users(IncrementalOutreachStream): """ Users stream. Yields data from the GET /users endpoint. @@ -296,6 +316,8 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Mailboxes(authenticator=auth, **config), Stages(authenticator=auth, **config), Calls(authenticator=auth, **config), + CallPurposes(authenticator=auth, **config), + CallDispositions(authenticator=auth, **config), Users(authenticator=auth, **config), Tasks(authenticator=auth, **config), Templates(authenticator=auth, **config), diff --git a/airbyte-integrations/connectors/source-outreach/unit_tests/test_source.py b/airbyte-integrations/connectors/source-outreach/unit_tests/test_source.py index 5be9a07247fe..f67f33587baa 100644 --- a/airbyte-integrations/connectors/source-outreach/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-outreach/unit_tests/test_source.py @@ -11,5 +11,5 @@ def test_streams(mocker): source = SourceOutreach() config_mock = MagicMock() streams = source.streams(config_mock) - expected_streams_number = 15 + expected_streams_number = 17 assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-pagerduty/.dockerignore b/airbyte-integrations/connectors/source-pagerduty/.dockerignore new file mode 100644 index 000000000000..2d661c3708ef --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_pagerduty +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-pagerduty/Dockerfile b/airbyte-integrations/connectors/source-pagerduty/Dockerfile new file mode 100644 index 000000000000..1471709cc025 --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_pagerduty ./source_pagerduty + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.name=airbyte/source-pagerduty diff --git a/airbyte-integrations/connectors/source-pagerduty/README.md b/airbyte-integrations/connectors/source-pagerduty/README.md new file mode 100644 index 000000000000..aea16529a84b --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/README.md @@ -0,0 +1,67 @@ +# Pagerduty Source + +This is the repository for the Pagerduty configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/pagerduty). + +## Local development + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/pagerduty) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_pagerduty/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source pagerduty test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + + +#### Build +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-pagerduty build +``` + +An image will be built with the tag `airbyte/source-pagerduty:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-pagerduty:dev . +``` + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-pagerduty:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pagerduty:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pagerduty:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-pagerduty:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-pagerduty test +``` + +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-pagerduty test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/pagerduty.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-pagerduty/__init__.py b/airbyte-integrations/connectors/source-pagerduty/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-pagerduty/acceptance-test-config.yml b/airbyte-integrations/connectors/source-pagerduty/acceptance-test-config.yml new file mode 100644 index 000000000000..a2899a14a684 --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/acceptance-test-config.yml @@ -0,0 +1,37 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-pagerduty:dev +test_strictness_level: low +acceptance_tests: + spec: + tests: + - spec_path: "source_pagerduty/spec.yaml" + backward_compatibility_tests_config: + disable_for_version: 0.1.23 + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: 0.1.23 + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + incremental: + bypass_reason: "This connector does not implement incremental sync" + # TODO uncomment this block this block if your connector implements incremental sync: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-pagerduty/icon.svg b/airbyte-integrations/connectors/source-pagerduty/icon.svg new file mode 100644 index 000000000000..98da64d7a2ef --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/icon.svg @@ -0,0 +1,2 @@ + +PagerDuty icon \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-pagerduty/integration_tests/__init__.py b/airbyte-integrations/connectors/source-pagerduty/integration_tests/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-pagerduty/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-pagerduty/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pagerduty/integration_tests/acceptance.py similarity index 100% rename from airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/acceptance.py rename to airbyte-integrations/connectors/source-pagerduty/integration_tests/acceptance.py diff --git a/airbyte-integrations/connectors/source-pagerduty/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-pagerduty/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..990545d018b3 --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/integration_tests/configured_catalog.json @@ -0,0 +1,58 @@ +{ + "streams": [ + { + "stream": { + "name": "incidents", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "incident_logs", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "priorities", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "teams", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "services", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-pagerduty/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-pagerduty/integration_tests/expected_records.jsonl new file mode 100644 index 000000000000..ac09476cd5aa --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/integration_tests/expected_records.jsonl @@ -0,0 +1,6 @@ +{"stream": "incidents", "data": {"incident_number":1,"title":"admin","description":"admin","created_at":"2023-10-07T07:25:26Z","updated_at":"2023-10-07T07:25:26Z","status":"triggered","incident_key":"f311ae09b6744daf8580aa92d2cc1b1f","service":{"id":"P3VUFA7","type":"service_reference","summary":"asd","self":"https://api.pagerduty.com/services/P3VUFA7","html_url":"https://dev-pixil.pagerduty.com/service-directory/P3VUFA7"},"assignments":[{"at":"2023-10-07T07:25:26Z","assignee":{"id":"PLOD9FO","type":"user_reference","summary":"tester tester","self":"https://api.pagerduty.com/users/PLOD9FO","html_url":"https://dev-pixil.pagerduty.com/users/PLOD9FO"}}],"assigned_via":"escalation_policy","last_status_change_at":"2023-10-07T07:25:26Z","resolved_at":null,"first_trigger_log_entry":{"id":"R21I9CLEE8JB0F7ICF7XBE1CMI","type":"trigger_log_entry_reference","summary":"Triggered through the website.","self":"https://api.pagerduty.com/log_entries/R21I9CLEE8JB0F7ICF7XBE1CMI","html_url":"https://dev-pixil.pagerduty.com/incidents/Q2U8NVNBMCM7JW/log_entries/R21I9CLEE8JB0F7ICF7XBE1CMI"},"alert_counts":{"all":0,"triggered":0,"resolved":0},"is_mergeable":true,"escalation_policy":{"id":"PQEF2Q3","type":"escalation_policy_reference","summary":"asd-ep","self":"https://api.pagerduty.com/escalation_policies/PQEF2Q3","html_url":"https://dev-pixil.pagerduty.com/escalation_policies/PQEF2Q3"},"teams":[],"pending_actions":[],"acknowledgements":[],"basic_alert_grouping":null,"alert_grouping":null,"last_status_change_by":{"id":"P3VUFA7","type":"service_reference","summary":"asd","self":"https://api.pagerduty.com/services/P3VUFA7","html_url":"https://dev-pixil.pagerduty.com/service-directory/P3VUFA7"},"priority":{"id":"PXS7CUY","type":"priority","summary":"P1","self":"https://api.pagerduty.com/priorities/PXS7CUY","html_url":null,"account_id":"PLANXGO","color":"a8171c","created_at":"2023-10-07T07:18:42Z","description":"","name":"P1","order":500000000,"schema_version":0,"updated_at":"2023-10-07T07:18:42Z"},"incidents_responders":[],"responder_requests":[],"subscriber_requests":[],"urgency":"high","id":"Q2U8NVNBMCM7JW","type":"incident","summary":"[#1] admin","self":"https://api.pagerduty.com/incidents/Q2U8NVNBMCM7JW","html_url":"https://dev-pixil.pagerduty.com/incidents/Q2U8NVNBMCM7JW"}, "emitted_at": 1693501393086} +{"stream": "incident_logs", "data": {"id":"RNUYTXSTD6TFUACXJH8FS2I4TM","type":"annotate_log_entry","summary":"Noteaddedbytestertester.","self":"https://api.pagerduty.com/log_entries/RNUYTXSTD6TFUACXJH8FS2I4TM","html_url":null,"created_at":"2023-10-07T07:28:46Z","agent":{"id":"PLOD9FO","type":"user_reference","summary":"testertester","self":"https://api.pagerduty.com/users/PLOD9FO","html_url":"https://dev-pixil.pagerduty.com/users/PLOD9FO"},"channel":{"type":"note","summary":"sdgsdf"},"service":{"id":"P3VUFA7","type":"service_reference","summary":"asd","self":"https://api.pagerduty.com/services/P3VUFA7","html_url":"https://dev-pixil.pagerduty.com/service-directory/P3VUFA7"},"incident":{"id":"Q2U8NVNBMCM7JW","type":"incident_reference","summary":"[#1]admin","self":"https://api.pagerduty.com/incidents/Q2U8NVNBMCM7JW","html_url":"https://dev-pixil.pagerduty.com/incidents/Q2U8NVNBMCM7JW"},"teams":[],"contexts":[]}, "emitted_at": 1693501393086} +{"stream": "priorities", "data": {"id":"PXS7CUY","type":"priority","summary":"P1","self":"https://api.pagerduty.com/priorities/PXS7CUY","html_url":null,"account_id":"PLANXGO","color":"a8171c","created_at":"2023-10-07T07:18:42Z","description":"","name":"P1","order":500000000,"schema_version":0,"updated_at":"2023-10-07T07:18:42Z"}, "emitted_at": 1693501393086} +{"stream": "services", "data": {"id":"P3VUFA7","name":"asd","description":"asd","created_at":"2023-10-07T07:25:04Z","updated_at":"2023-10-07T07:25:04Z","status":"warning","teams":[],"alert_creation":"create_alerts_and_incidents","addons":[],"scheduled_actions":[],"support_hours":null,"last_incident_timestamp":"2023-10-07T07:25:26Z","escalation_policy":{"id":"PQEF2Q3","type":"escalation_policy_reference","summary":"asd-ep","self":"https://api.pagerduty.com/escalation_policies/PQEF2Q3","html_url":"https://dev-pixil.pagerduty.com/escalation_policies/PQEF2Q3"},"incident_urgency_rule":{"type":"constant","urgency":"high"},"acknowledgement_timeout":null,"auto_resolve_timeout":null,"alert_grouping":"intelligent","alert_grouping_timeout":null,"alert_grouping_parameters":{"type":"intelligent","config":{"time_window":300,"recommended_time_window":300}},"integrations":[{"id":"PXMYB2G","type":"events_api_v2_inbound_integration_reference","summary":"EventsAPIV2","self":"https://api.pagerduty.com/services/P3VUFA7/integrations/PXMYB2G","html_url":"https://dev-pixil.pagerduty.com/services/P3VUFA7/integrations/PXMYB2G"}],"response_play":null,"type":"service","summary":"asd","self":"https://api.pagerduty.com/services/P3VUFA7","html_url":"https://dev-pixil.pagerduty.com/service-directory/P3VUFA7"}, "emitted_at": 1693501393086} +{"stream": "teams", "data": {"id":"PLYRZEM","name":"asd","description":null,"type":"team","summary":"asd","self":"https://api.pagerduty.com/teams/PLYRZEM","html_url":"https://dev-pixil.pagerduty.com/teams/PLYRZEM","default_role":"manager","parent":null}, "emitted_at": 1693501393086} +{"stream": "users", "data": {"name":"pixil","email":"hesor11831@gekme.com","time_zone":"Etc/UTC","color":"red","avatar_url":"https://secure.gravatar.com/avatar/64b1ea9c6c0ec582d946b5cfecbbfd4f.png?d=mm&r=PG","billed":true,"role":"admin","description":null,"invitation_sent":true,"job_title":null,"teams":[{"id":"PLYRZEM","type":"team_reference","summary":"asd","self":"https://api.pagerduty.com/teams/PLYRZEM","html_url":"https://dev-pixil.pagerduty.com/teams/PLYRZEM"}],"contact_methods":[{"id":"PD9J7I5","type":"email_contact_method_reference","summary":"Default","self":"https://api.pagerduty.com/users/PHCS92L/contact_methods/PD9J7I5","html_url":null}],"notification_rules":[{"id":"POHYPIT","type":"assignment_notification_rule_reference","summary":"0minutes:channelPD9J7I5","self":"https://api.pagerduty.com/users/PHCS92L/notification_rules/POHYPIT","html_url":null},{"id":"PSTJQXG","type":"assignment_notification_rule_reference","summary":"0minutes:channelPD9J7I5","self":"https://api.pagerduty.com/users/PHCS92L/notification_rules/PSTJQXG","html_url":null}],"coordinated_incidents":[{"incident":{"id":"Q2U8NVNBMCM7JW","type":"incident_reference","summary":"[#1]admin","self":"https://api.pagerduty.com/incidents/Q2U8NVNBMCM7JW","html_url":"https://dev-pixil.pagerduty.com/incidents/Q2U8NVNBMCM7JW"},"requester":{"id":"PLOD9FO","type":"user_reference","summary":"testertester","self":"https://api.pagerduty.com/users/PLOD9FO","html_url":"https://dev-pixil.pagerduty.com/users/PLOD9FO"},"message":"Pleasehelpwith\"admin\"","state":"pending","requested_at":"2023-10-07T07:28:35Z"}],"id":"PHCS92L","type":"user","summary":"pixil","self":"https://api.pagerduty.com/users/PHCS92L","html_url":"https://dev-pixil.pagerduty.com/users/PHCS92L"}, "emitted_at": 1693501393086} diff --git a/airbyte-integrations/connectors/source-pagerduty/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-pagerduty/integration_tests/invalid_config.json new file mode 100644 index 000000000000..c1228206caf3 --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "token": "xxxxxxx" +} diff --git a/airbyte-integrations/connectors/source-pagerduty/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-pagerduty/integration_tests/sample_config.json new file mode 100644 index 000000000000..41bdf992eab6 --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "token": "xxxxxxxxxxx" +} diff --git a/airbyte-integrations/connectors/source-pagerduty/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-pagerduty/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-pagerduty/main.py b/airbyte-integrations/connectors/source-pagerduty/main.py new file mode 100644 index 000000000000..61d193268f6b --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pagerduty import SourcePagerduty + +if __name__ == "__main__": + source = SourcePagerduty() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pagerduty/metadata.yaml b/airbyte-integrations/connectors/source-pagerduty/metadata.yaml new file mode 100644 index 000000000000..0b471649a4a4 --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/metadata.yaml @@ -0,0 +1,28 @@ +data: + allowedHosts: + hosts: + - api.pagerduty.com + registries: + oss: + enabled: false + cloud: + enabled: false + connectorSubtype: api + connectorType: source + definitionId: 2544ac39-02be-4bf5-82ad-f52bbb833bf5 + dockerImageTag: 0.2.0 + dockerRepository: airbyte/source-pagerduty + githubIssueLabel: source-pagerduty + icon: pagerduty.svg + license: MIT + name: Pagerduty + releaseDate: 2023-10-10 + releaseStage: alpha + supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/sources/pagerduty + tags: + - language:low-code + ab_internal: + sl: 100 + ql: 100 +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-pagerduty/requirements.txt b/airbyte-integrations/connectors/source-pagerduty/requirements.txt new file mode 100644 index 000000000000..cc57334ef619 --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/connector-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-pagerduty/setup.py b/airbyte-integrations/connectors/source-pagerduty/setup.py new file mode 100644 index 000000000000..aec396ac036d --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/setup.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = ["airbyte-cdk"] + +TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest~=6.2", + "pytest-mock~=3.6.1", +] + +setup( + name="source_pagerduty", + description="Source implementation for Pagerduty.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/__init__.py b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/__init__.py new file mode 100644 index 000000000000..27f6419a3392 --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .source import SourcePagerduty + +__all__ = ["SourcePagerduty"] diff --git a/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/manifest.yaml b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/manifest.yaml new file mode 100644 index 000000000000..39c0f41f49c6 --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/manifest.yaml @@ -0,0 +1,110 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["{{ parameters.extractorPath }}"] + + requester: + type: HttpRequester + url_base: "https://api.pagerduty.com" + http_method: "GET" + authenticator: + type: "ApiKeyAuthenticator" + header: "Authorization" + api_token: "Token token={{ config['token'] }}" + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + incidents_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "incidents" + primary_key: "id" + extractorPath: "incidents" + path: "/incidents" + + incidents_partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/incidents_stream" + parent_key: "id" + partition_field: "incident_id" + + incident_logs_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "incident_logs" + primary_key: "id" + extractorPath: "log_entries" + retriever: + $ref: "#/definitions/retriever" + partition_router: + $ref: "#/definitions/incidents_partition_router" + requester: + $ref: "#/definitions/requester" + path: "/incidents/{{ stream_partition.incident_id }}/log_entries" + + teams_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "teams" + primary_key: "id" + extractorPath: "teams" + path: "/teams" + + services_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "services" + primary_key: "id" + extractorPath: "services" + path: "/services" + + users_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "users" + primary_key: "id" + extractorPath: "users" + path: "/users" + + priorities_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "priorities" + primary_key: "id" + extractorPath: "priorities" + path: "/priorities" + +streams: + - "#/definitions/incidents_stream" + - "#/definitions/incident_logs_stream" + - "#/definitions/teams_stream" + - "#/definitions/services_stream" + - "#/definitions/users_stream" + - "#/definitions/priorities_stream" + +check: + type: CheckStream + stream_names: + - "incidents" + - "incident_logs" + - "teams" + - "services" + - "users" + - "priorities" diff --git a/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/schemas/incident_logs.json b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/schemas/incident_logs.json new file mode 100644 index 000000000000..6cd2ff5583b9 --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/schemas/incident_logs.json @@ -0,0 +1,380 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Incident Logs schema", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "agent": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + } + } + }, + "channel": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "notification": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "address": { + "type": ["null", "string"] + }, + "conferenceAddress": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + } + } + }, + "old_priority": { + "type": ["null", "string"] + }, + "new_priority": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "account_id": { + "type": ["null", "string"] + }, + "color": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "order": { + "type": ["null", "number"] + }, + "schema_version": { + "type": ["null", "number"] + }, + "updated_at": { + "type": ["null", "string"] + } + } + } + } + }, + "service": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + } + } + }, + "incident": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + } + } + }, + "teams": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "contexts": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "message": { + "type": ["null", "string"] + }, + "responder": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + } + } + }, + "channels": { + "type": ["null", "string"] + }, + "responders_list": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + } + } + } + }, + "escalation_policy": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "escalation_rules": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "escalation_delay_in_minutes": { + "type": ["null", "number"] + }, + "targets": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + } + } + } + } + } + } + }, + "services": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + } + } + } + }, + "num_loops": { + "type": ["null", "number"] + }, + "teams": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "description": { + "type": ["null", "string"] + }, + "on_call_handoff_notifications": { + "type": ["null", "string"] + }, + "privilege": { + "type": ["null", "string"] + } + } + }, + "level": { + "type": ["null", "number"] + }, + "action": { + "type": ["null", "string"] + }, + "event_details": { + "type": ["null", "object"], + "properties": { + "description": { + "type": ["null", "string"] + } + } + }, + "user": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + } + } + }, + "assignees": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/schemas/incidents.json b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/schemas/incidents.json new file mode 100644 index 000000000000..26e281c5c413 --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/schemas/incidents.json @@ -0,0 +1,549 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Incidents Schema", + "additionalProperties": true, + "type": ["object", "null"], + "properties": { + "incident_number": { + "type": ["number", "null"] + }, + "title": { + "type": ["string", "null"] + }, + "description": { + "type": ["string", "null"] + }, + "created_at": { + "type": ["string", "null"] + }, + "updated_at": { + "type": ["string", "null"] + }, + "status": { + "type": ["string", "null"] + }, + "incident_key": { + "type": ["string", "null"] + }, + "service": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "summary": { + "type": ["string", "null"] + }, + "self": { + "type": ["string", "null"] + }, + "html_url": { + "type": ["string", "null"] + } + } + }, + "assignments": { + "type": ["array", "null"], + "items": { + "type": ["object", "null"], + "properties": { + "at": { + "type": ["string", "null"] + }, + "assignee": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "summary": { + "type": ["string", "null"] + }, + "self": { + "type": ["string", "null"] + }, + "html_url": { + "type": ["string", "null"] + } + } + } + } + } + }, + "assigned_via": { + "type": ["string", "null"] + }, + "last_status_change_at": { + "type": ["string", "null"] + }, + "resolved_at": { + "type": ["string", "null"] + }, + "first_trigger_log_entry": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "summary": { + "type": ["string", "null"] + }, + "self": { + "type": ["string", "null"] + }, + "html_url": { + "type": ["string", "null"] + } + } + }, + "alert_counts": { + "type": ["object", "null"], + "properties": { + "all": { + "type": ["number", "null"] + }, + "triggered": { + "type": ["number", "null"] + }, + "resolved": { + "type": ["number", "null"] + } + } + }, + "is_mergeable": { + "type": "boolean" + }, + "escalation_policy": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "summary": { + "type": ["string", "null"] + }, + "self": { + "type": ["string", "null"] + }, + "html_url": { + "type": ["string", "null"] + } + } + }, + "teams": { + "type": ["array", "null"], + "items": { + "type": ["string", "null"] + } + }, + "pending_actions": { + "type": ["array", "null"], + "items": { + "type": ["string", "null"] + } + }, + "acknowledgements": { + "type": ["array", "null"], + "items": { + "type": ["object", "null"], + "properties": { + "at": { + "type": ["string", "null"] + }, + "acknowledger": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "summary": { + "type": ["string", "null"] + }, + "self": { + "type": ["string", "null"] + }, + "html_url": { + "type": ["string", "null"] + } + } + } + } + } + }, + "basic_alert_grouping": { + "type": ["string", "null"] + }, + "alert_grouping": { + "type": ["string", "null"] + }, + "last_status_change_by": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "summary": { + "type": ["string", "null"] + }, + "self": { + "type": ["string", "null"] + }, + "html_url": { + "type": ["string", "null"] + } + } + }, + "priority": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "summary": { + "type": ["string", "null"] + }, + "self": { + "type": ["string", "null"] + }, + "html_url": { + "type": ["string", "null"] + }, + "account_id": { + "type": ["string", "null"] + }, + "color": { + "type": ["string", "null"] + }, + "created_at": { + "type": ["string", "null"] + }, + "description": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "order": { + "type": ["number", "null"] + }, + "schema_version": { + "type": ["number", "null"] + }, + "updated_at": { + "type": ["string", "null"] + } + } + }, + "incidents_responders": { + "type": ["array", "null"], + "items": { + "type": ["object", "null"], + "properties": { + "state": { + "type": ["string", "null"] + }, + "user": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "summary": { + "type": ["string", "null"] + }, + "self": { + "type": ["string", "null"] + }, + "html_url": { + "type": ["string", "null"] + }, + "avatar_url": { + "type": ["string", "null"] + }, + "job_title": { + "type": ["string", "null"] + } + } + }, + "incident": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "summary": { + "type": ["string", "null"] + }, + "self": { + "type": ["string", "null"] + }, + "html_url": { + "type": ["string", "null"] + } + } + }, + "updated_at": { + "type": ["string", "null"] + }, + "message": { + "type": ["string", "null"] + }, + "requester": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "summary": { + "type": ["string", "null"] + }, + "self": { + "type": ["string", "null"] + }, + "html_url": { + "type": ["string", "null"] + }, + "avatar_url": { + "type": ["string", "null"] + }, + "job_title": { + "type": ["string", "null"] + } + } + }, + "requested_at": { + "type": ["string", "null"] + }, + "escalation_policy_requests": { + "type": ["array", "null"], + "items": { + "type": ["string", "null"] + } + } + } + } + }, + "responder_requests": { + "type": ["array", "null"], + "items": { + "type": ["object", "null"], + "properties": { + "incident": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "summary": { + "type": ["string", "null"] + }, + "self": { + "type": ["string", "null"] + }, + "html_url": { + "type": ["string", "null"] + } + } + }, + "requester": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "summary": { + "type": ["string", "null"] + }, + "self": { + "type": ["string", "null"] + }, + "html_url": { + "type": ["string", "null"] + } + } + }, + "requested_at": { + "type": ["string", "null"] + }, + "message": { + "type": ["string", "null"] + }, + "responder_request_targets": { + "type": ["array", "null"], + "items": { + "type": ["object", "null"], + "properties": { + "responder_request_target": { + "type": ["object", "null"], + "properties": { + "type": { + "type": ["string", "null"] + }, + "id": { + "type": ["string", "null"] + }, + "summary": { + "type": ["string", "null"] + }, + "incidents_responders": { + "type": ["array", "null"], + "items": { + "type": ["object", "null"], + "properties": { + "state": { + "type": ["string", "null"] + }, + "user": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "summary": { + "type": ["string", "null"] + }, + "self": { + "type": ["string", "null"] + }, + "html_url": { + "type": ["string", "null"] + }, + "avatar_url": { + "type": ["string", "null"] + }, + "job_title": { + "type": ["string", "null"] + } + } + }, + "incident": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "summary": { + "type": ["string", "null"] + }, + "self": { + "type": ["string", "null"] + }, + "html_url": { + "type": ["string", "null"] + } + } + }, + "updated_at": { + "type": ["string", "null"] + }, + "message": { + "type": ["string", "null"] + }, + "requester": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "summary": { + "type": ["string", "null"] + }, + "self": { + "type": ["string", "null"] + }, + "html_url": { + "type": ["string", "null"] + }, + "avatar_url": { + "type": ["string", "null"] + }, + "job_title": { + "type": ["string", "null"] + } + } + }, + "requested_at": { + "type": ["string", "null"] + } + } + } + } + } + } + } + } + } + } + } + }, + "subscriber_requests": { + "type": ["array", "null"], + "items": { + "type": ["string", "null"] + } + }, + "urgency": { + "type": ["string", "null"] + }, + "id": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "summary": { + "type": ["string", "null"] + }, + "self": { + "type": ["string", "null"] + }, + "html_url": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/schemas/priorities.json b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/schemas/priorities.json new file mode 100644 index 000000000000..98267d5eb806 --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/schemas/priorities.json @@ -0,0 +1,47 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Priorities schema", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "account_id": { + "type": ["null", "string"] + }, + "color": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "order": { + "type": ["null", "number"] + }, + "schema_version": { + "type": ["null", "number"] + }, + "updated_at": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/schemas/services.json b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/schemas/services.json new file mode 100644 index 000000000000..c512977ce396 --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/schemas/services.json @@ -0,0 +1,153 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Services schema", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "teams": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "alert_creation": { + "type": ["null", "string"] + }, + "addons": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "scheduled_actions": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "support_hours": { + "type": ["null", "string"] + }, + "last_incident_timestamp": { + "type": ["null", "string"] + }, + "escalation_policy": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + } + } + }, + "incident_urgency_rule": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "urgency": { + "type": ["null", "string"] + } + } + }, + "acknowledgement_timeout": { + "type": ["null", "string"] + }, + "auto_resolve_timeout": { + "type": ["null", "string"] + }, + "alert_grouping": { + "type": ["null", "string"] + }, + "alert_grouping_timeout": { + "type": ["null", "string"] + }, + "alert_grouping_parameters": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "config": { + "type": ["null", "object"], + "properties": { + "time_window": { + "type": ["null", "number"] + }, + "recommended_time_window": { + "type": ["null", "number"] + } + } + } + } + }, + "integrations": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + } + } + } + }, + "response_play": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/schemas/teams.json b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/schemas/teams.json new file mode 100644 index 000000000000..fb3ec34e852b --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/schemas/teams.json @@ -0,0 +1,35 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Teams Schema", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + }, + "default_role": { + "type": ["null", "string"] + }, + "parent": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/schemas/users.json b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/schemas/users.json new file mode 100644 index 000000000000..3e8e0e24c34b --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/schemas/users.json @@ -0,0 +1,179 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Users Schema", + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "time_zone": { + "type": ["null", "string"] + }, + "color": { + "type": ["null", "string"] + }, + "avatar_url": { + "type": ["null", "string"] + }, + "billed": { + "type": "boolean" + }, + "role": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "invitation_sent": { + "type": "boolean" + }, + "job_title": { + "type": ["null", "string"] + }, + "teams": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + } + } + } + }, + "contact_methods": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + } + } + } + }, + "notification_rules": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + } + } + } + }, + "coordinated_incidents": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "incident": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + } + } + }, + "requester": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + } + } + }, + "message": { + "type": ["null", "string"] + }, + "state": { + "type": ["null", "string"] + }, + "requested_at": { + "type": ["null", "string"] + } + } + } + }, + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "summary": { + "type": ["null", "string"] + }, + "self": { + "type": ["null", "string"] + }, + "html_url": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/source.py b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/source.py new file mode 100644 index 000000000000..96ff2d87bbfb --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourcePagerduty(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/spec.yaml b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/spec.yaml new file mode 100644 index 000000000000..b40b90d2779c --- /dev/null +++ b/airbyte-integrations/connectors/source-pagerduty/source_pagerduty/spec.yaml @@ -0,0 +1,77 @@ +documentationUrl: https://docs.faros.ai +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: PagerDuty Spec + type: object + required: + - token + additionalProperties: true + properties: + token: + type: string + title: API key + description: API key for PagerDuty API authentication + airbyte_secret: true + cutoff_days: + type: integer + title: Cutoff Days + default: 90 + description: Fetch pipelines updated in the last number of days + page_size: + type: integer + minimum: 1 + maximum: 25 + default: 25 + title: Page Size + description: page size to use when querying PagerDuty API + incident_log_entries_overview: + type: boolean + title: Incident Log Entries Overview + description: + If true, will return a subset of log entries that show only the + most important changes to the incident. + default: true + default_severity: + type: string + title: Severity category + description: A default severity category if not present + examples: + - Sev1 + - Sev2 + - Sev3 + - Sev4 + - Sev5 + - Custom + pattern: "^(Sev[0-5])?(Custom)?$" + exclude_services: + type: array + items: + type: string + title: Exclude Services + examples: + - service-1 + - service-2 + description: + List of PagerDuty service names to ignore incidents from. If not + set, all incidents will be pulled. + service_details: + type: array + items: + type: string + enum: + - escalation_policies + - teams + - integrations + - auto_pause_notifications_parameters + title: Service Details + description: List of PagerDuty service additional details to include. + max_retries: + type: integer + minimum: 0 + maximum: 8 + default: 5 + title: Max Retries + description: + Maximum number of PagerDuty API request retries to perform upon + connection errors. The source will pause for an exponentially increasing number + of seconds before retrying. diff --git a/airbyte-integrations/connectors/source-pardot/README.md b/airbyte-integrations/connectors/source-pardot/README.md index acf1d421b03d..dcd89f3ddf5e 100644 --- a/airbyte-integrations/connectors/source-pardot/README.md +++ b/airbyte-integrations/connectors/source-pardot/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-pardot:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/pardot) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_pardot/spec.json` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-pardot:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-pardot build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-pardot:airbyteDocker +An image will be built with the tag `airbyte/source-pardot:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-pardot:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pardot:dev check --con docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pardot:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-pardot:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-pardot test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-pardot:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-pardot:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-pardot test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/pardot.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-pardot/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-pardot/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-pardot/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-pardot/build.gradle b/airbyte-integrations/connectors/source-pardot/build.gradle deleted file mode 100644 index 701b7f6d806e..000000000000 --- a/airbyte-integrations/connectors/source-pardot/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_pardot' -} diff --git a/airbyte-integrations/connectors/source-pardot/pardot.md b/airbyte-integrations/connectors/source-pardot/pardot.md index e877c48a103a..bb3e4470173a 100644 --- a/airbyte-integrations/connectors/source-pardot/pardot.md +++ b/airbyte-integrations/connectors/source-pardot/pardot.md @@ -53,4 +53,4 @@ The Pardot connector should not run into Pardot API limitations under normal usa - `client_secret`: The Consumer Secret that can be found when viewing your app in Salesforce - `refresh_token`: Salesforce Refresh Token used for Airbyte to access your Salesforce account. If you don't know what this is, follow [this guide](https://medium.com/@bpmmendis94/obtain-access-refresh-tokens-from-salesforce-rest-api-a324fe4ccd9b) to retrieve it. - `start_date`: UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. Leave blank to skip this filter -- `is_sandbox`: Whether or not the the app is in a Salesforce sandbox. If you do not know what this, assume it is false. +- `is_sandbox`: Whether or not the app is in a Salesforce sandbox. If you do not know what this is, assume it is false. diff --git a/airbyte-integrations/connectors/source-partnerstack/README.md b/airbyte-integrations/connectors/source-partnerstack/README.md index 3521109239b0..e805cbcf54fc 100644 --- a/airbyte-integrations/connectors/source-partnerstack/README.md +++ b/airbyte-integrations/connectors/source-partnerstack/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-partnerstack:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/partnerstack) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_partnerstack/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-partnerstack:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-partnerstack build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-partnerstack:airbyteDocker +An image will be built with the tag `airbyte/source-partnerstack:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-partnerstack:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-partnerstack:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-partnerstack:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-partnerstack:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-partnerstack test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-partnerstack:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-partnerstack:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-partnerstack test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/partnerstack.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-partnerstack/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-partnerstack/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-partnerstack/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-partnerstack/build.gradle b/airbyte-integrations/connectors/source-partnerstack/build.gradle deleted file mode 100644 index 5e63dcfd459f..000000000000 --- a/airbyte-integrations/connectors/source-partnerstack/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_partnerstack' -} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/.dockerignore b/airbyte-integrations/connectors/source-paypal-transaction/.dockerignore index 7d3fd691a105..d4b84ddd812b 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/.dockerignore +++ b/airbyte-integrations/connectors/source-paypal-transaction/.dockerignore @@ -1,6 +1,5 @@ * !Dockerfile -!Dockerfile.test !main.py !source_paypal_transaction !setup.py diff --git a/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile b/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile deleted file mode 100644 index 282bb021a007..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY source_paypal_transaction ./source_paypal_transaction -COPY main.py ./ -COPY setup.py ./ -RUN pip install . - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=2.0.0 -LABEL io.airbyte.name=airbyte/source-paypal-transaction diff --git a/airbyte-integrations/connectors/source-paypal-transaction/README.md b/airbyte-integrations/connectors/source-paypal-transaction/README.md index 8ab3f609ab3a..a451e1a48d23 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/README.md +++ b/airbyte-integrations/connectors/source-paypal-transaction/README.md @@ -1,119 +1,103 @@ # Paypal Transaction Source -This is the repository for the Paypal Transaction source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/paypal-transaction). +This is the repository for the Paypal Transaction configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/paypal-transaction). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/paypal-transaction) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_paypal_transaction/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source paypal-transaction test creds` +and place them into `secrets/config.json`. -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. +### Locally running the connector docker image -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-paypal-transaction:build +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-paypal-transaction build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-paypal-transaction:dev`. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/paypal-transaction) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_paypal_transaction/spec.json` file. -Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source paypal-transaction test creds` -and place them into `secrets/config.json`. +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` +from typing import TYPE_CHECKING -### Locally running the connector docker image +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-paypal-transaction:dev -``` -You can also build the connector image via Gradle: +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-paypal-transaction:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-paypal-transaction:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-paypal-transaction:dev . +# Running the spec command against your patched connector +docker run airbyte/source-paypal-transaction:dev spec +``` #### Run Then run any of the connector commands as follows: ``` docker run --rm airbyte/source-paypal-transaction:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-paypal-transaction:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-paypal-transaction:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-paypal-transaction:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-paypal-transaction:dev check --config /secrets/config_oauth.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-paypal-transaction:dev discover --config /secrets/config_oauth.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-paypal-transaction:dev read --config /secrets/config_oauth.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install '.[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-paypal-transaction test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-paypal-transaction:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-paypal-transaction:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-paypal-transaction:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -123,8 +107,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-paypal-transaction test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/paypal-transaction.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-paypal-transaction/__init__.py b/airbyte-integrations/connectors/source-paypal-transaction/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml index 38e6b7be325e..c4ebc718cf3b 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-config.yml @@ -1,9 +1,11 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests connector_image: airbyte/source-paypal-transaction:dev test_strictness_level: high acceptance_tests: spec: tests: - - spec_path: source_paypal_transaction/spec.json + - spec_path: "source_paypal_transaction/spec.yaml" config_path: secrets/config_oauth.json backward_compatibility_tests_config: disable_for_version: "0.1.13" @@ -21,7 +23,7 @@ acceptance_tests: tests: - config_path: secrets/config_oauth.json backward_compatibility_tests_config: - disable_for_version: "1.0.0" # Balances schema changed + disable_for_version: "2.0.0" # Change in cursor field for transactions stream basic_read: tests: - config_path: secrets/config_oauth.json @@ -49,16 +51,19 @@ acceptance_tests: extra_fields: no exact_order: no extra_records: yes + fail_on_extra_columns: false incremental: tests: - config_path: secrets/config_oauth.json configured_catalog_path: integration_tests/configured_catalog.json future_state: future_state_path: integration_tests/abnormal_state.json - cursor_paths: - transactions: [ "date" ] - balances: [ "date" ] + skip_comprehensive_incremental_tests: true full_refresh: tests: - config_path: secrets/config_oauth.json + ignored_fields: + balances: + - name: last_refresh_time + bypass_reason: "field changes during every read" configured_catalog_path: integration_tests/configured_catalog.json diff --git a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-paypal-transaction/build.gradle b/airbyte-integrations/connectors/source-paypal-transaction/build.gradle deleted file mode 100644 index e67e581b0312..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_paypal_transaction' -} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/__init__.py b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json index 565f41f6fca1..dc44c707ad24 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/abnormal_state.json @@ -3,7 +3,7 @@ "type": "STREAM", "stream": { "stream_state": { - "date": "2023-06-09T00:00:00+00:00" + "as_of_time": "2033-06-09T00:00:00Z" }, "stream_descriptor": { "name": "balances" @@ -14,7 +14,7 @@ "type": "STREAM", "stream": { "stream_state": { - "date": "2023-06-09T00:00:00+00:00" + "transaction_updated_date": "2033-06-09T00:00:00Z" }, "stream_descriptor": { "name": "transactions" diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json index 699f09442f94..e0992dea17b0 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog.json @@ -5,7 +5,7 @@ "name": "transactions", "json_schema": {}, "source_defined_cursor": true, - "default_cursor_field": ["transaction_initiation_date"], + "default_cursor_field": ["transaction_updated_date"], "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_balances.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_balances.json index ca1887a40519..fa4ca9fa21ce 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_balances.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_balances.json @@ -4,6 +4,7 @@ "stream": { "name": "balances", "json_schema": {}, + "default_cursor_field": ["as_of_time"], "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transactions.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transactions.json index 36b9a092df75..ec00ecab10e3 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transactions.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/configured_catalog_transactions.json @@ -5,7 +5,7 @@ "name": "transactions", "json_schema": {}, "source_defined_cursor": true, - "default_cursor_field": ["transaction_initiation_date"], + "default_cursor_field": ["transaction_updated_date"], "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "incremental", diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/expected_records.jsonl index 7841d256e49d..86cbdca7392b 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/expected_records.jsonl @@ -1 +1 @@ -{"stream":"balances","data":{"balances":[{"currency":"USD","primary":true,"total_balance":{"currency_code":"USD","value":"0.00"},"available_balance":{"currency_code":"USD","value":"0.00"},"withheld_balance":{"currency_code":"USD","value":"0.00"}}],"account_id":"QJQSC8WXYCA2L","as_of_time":"2021-07-03T00:00:00+00:00","last_refresh_time":"2023-07-04T07:29:59Z"},"emitted_at":1688463837632} \ No newline at end of file +{"stream": "balances", "data": {"balances": [{"currency": "USD", "primary": true, "total_balance": {"currency_code": "USD", "value": "0.00"}, "available_balance": {"currency_code": "USD", "value": "0.00"}, "withheld_balance": {"currency_code": "USD", "value": "0.00"}}], "account_id": "QJQSC8WXYCA2L", "as_of_time": "2021-07-03T00:00:00Z", "last_refresh_time": "2023-09-18T13:29:59Z"}, "emitted_at": 1695051482452} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/expected_records_sandbox.jsonl b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/expected_records_sandbox.jsonl index 3d60f02eb69d..da4775fecc29 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/expected_records_sandbox.jsonl +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/expected_records_sandbox.jsonl @@ -1,2 +1,4 @@ -{"stream":"transactions","data":{"transaction_info":{"paypal_account_id":"ZE5533HZPGMC6","transaction_id":"23N61105X92314351","transaction_event_code":"T0006","transaction_initiation_date":"2021-07-04T17:13:23+0000","transaction_updated_date":"2021-07-04T17:13:23+0000","transaction_amount":{"currency_code":"USD","value":"30.11"},"fee_amount":{"currency_code":"USD","value":"-1.17"},"insurance_amount":{"currency_code":"USD","value":"0.01"},"shipping_amount":{"currency_code":"USD","value":"1.03"},"shipping_discount_amount":{"currency_code":"USD","value":"1.00"},"transaction_status":"S","transaction_subject":"This is the payment transaction description.","ending_balance":{"currency_code":"USD","value":"202.58"},"available_balance":{"currency_code":"USD","value":"202.58"},"invoice_id":"48787580055","custom_field":"EBAY_EMS_90048630020055","protection_eligibility":"01"},"payer_info":{"account_id":"ZE5533HZPGMC6","email_address":"integration-test-buyer@airbyte.io","address_status":"Y","payer_status":"Y","payer_name":{"given_name":"test","surname":"buyer","alternate_full_name":"test buyer"},"country_code":"US"},"shipping_info":{"name":"Hello World","address":{"line1":"4thFloor","line2":"unit#34","city":"SAn Jose","state":"CA","country_code":"US","postal_code":"95131"}},"cart_info":{"item_details":[{"item_code":"1","item_name":"hat","item_description":"Brown color hat","item_quantity":"5","item_unit_price":{"currency_code":"USD","value":"3.00"},"item_amount":{"currency_code":"USD","value":"15.00"},"tax_amounts":[{"tax_amount":{"currency_code":"USD","value":"0.05"}}],"total_item_amount":{"currency_code":"USD","value":"15.05"},"invoice_number":"48787580055"},{"item_code":"product34","item_name":"handbag","item_description":"Black color hand bag","item_quantity":"1","item_unit_price":{"currency_code":"USD","value":"15.00"},"item_amount":{"currency_code":"USD","value":"15.00"},"tax_amounts":[{"tax_amount":{"currency_code":"USD","value":"0.02"}}],"total_item_amount":{"currency_code":"USD","value":"15.02"},"invoice_number":"48787580055"}]},"store_info":{},"auction_info":{},"incentive_info":{},"transaction_initiation_date":"2021-07-04T17:13:23+0000","transaction_id":"23N61105X92314351"},"emitted_at":1688463620839} -{"stream":"balances","data":{"balances":[{"currency":"USD","primary":true,"total_balance":{"currency_code":"USD","value":"173.64"},"available_balance":{"currency_code":"USD","value":"173.64"},"withheld_balance":{"currency_code":"USD","value":"0.00"}}],"account_id":"MDXWPD67GEP5W","as_of_time":"2021-07-03T00:00:00+00:00","last_refresh_time":"2023-07-04T04:59:59Z"},"emitted_at":1688463625694} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "23N61105X92314351", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-04T17:13:23+0000", "transaction_updated_date": "2021-07-04T17:13:23+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "202.58"}, "available_balance": {"currency_code": "USD", "value": "202.58"}, "invoice_id": "48787580055", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "48787580055"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "48787580055"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-04T17:13:23Z", "transaction_id": "23N61105X92314351"}, "emitted_at": 1694795587519} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "1FN09943JY662130R", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T22:56:54+0000", "transaction_updated_date": "2021-07-05T22:56:54+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "231.52"}, "available_balance": {"currency_code": "USD", "value": "231.52"}, "invoice_id": "65095789448", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "65095789448"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "65095789448"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T22:56:54Z", "transaction_id": "1FN09943JY662130R"}, "emitted_at": 1694795587522} +{"stream": "transactions", "data": {"transaction_info": {"paypal_account_id": "ZE5533HZPGMC6", "transaction_id": "0M443597T0019954R", "transaction_event_code": "T0006", "transaction_initiation_date": "2021-07-05T23:01:13+0000", "transaction_updated_date": "2021-07-05T23:01:13+0000", "transaction_amount": {"currency_code": "USD", "value": "30.11"}, "fee_amount": {"currency_code": "USD", "value": "-1.17"}, "insurance_amount": {"currency_code": "USD", "value": "0.01"}, "shipping_amount": {"currency_code": "USD", "value": "1.03"}, "shipping_discount_amount": {"currency_code": "USD", "value": "1.00"}, "transaction_status": "S", "transaction_subject": "This is the payment transaction description.", "ending_balance": {"currency_code": "USD", "value": "260.46"}, "available_balance": {"currency_code": "USD", "value": "260.46"}, "invoice_id": "41468340464", "custom_field": "EBAY_EMS_90048630020055", "protection_eligibility": "01"}, "payer_info": {"account_id": "ZE5533HZPGMC6", "email_address": "integration-test-buyer@airbyte.io", "address_status": "Y", "payer_status": "Y", "payer_name": {"given_name": "test", "surname": "buyer", "alternate_full_name": "test buyer"}, "country_code": "US"}, "shipping_info": {"name": "Hello World", "address": {"line1": "4thFloor", "line2": "unit#34", "city": "SAn Jose", "state": "CA", "country_code": "US", "postal_code": "95131"}}, "cart_info": {"item_details": [{"item_code": "1", "item_name": "hat", "item_description": "Brown color hat", "item_quantity": "5", "item_unit_price": {"currency_code": "USD", "value": "3.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.05"}}], "total_item_amount": {"currency_code": "USD", "value": "15.05"}, "invoice_number": "41468340464"}, {"item_code": "product34", "item_name": "handbag", "item_description": "Black color hand bag", "item_quantity": "1", "item_unit_price": {"currency_code": "USD", "value": "15.00"}, "item_amount": {"currency_code": "USD", "value": "15.00"}, "tax_amounts": [{"tax_amount": {"currency_code": "USD", "value": "0.02"}}], "total_item_amount": {"currency_code": "USD", "value": "15.02"}, "invoice_number": "41468340464"}]}, "store_info": {}, "auction_info": {}, "incentive_info": {}, "transaction_updated_date": "2021-07-05T23:01:13Z", "transaction_id": "0M443597T0019954R"}, "emitted_at": 1694795587524} +{"stream": "balances", "data": {"balances": [{"currency": "USD", "primary": true, "total_balance": {"currency_code": "USD", "value": "173.64"}, "available_balance": {"currency_code": "USD", "value": "173.64"}, "withheld_balance": {"currency_code": "USD", "value": "0.00"}}], "account_id": "MDXWPD67GEP5W", "as_of_time": "2021-07-03T00:00:00Z", "last_refresh_time": "2023-09-18T08:59:59Z"}, "emitted_at": 1695051579296} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json index 0b2938447d7d..a716a3e4255e 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config.json @@ -1,6 +1,6 @@ { "client_id": "AWAz___", - "secret": "ENC8__", + "client_secret": "ENC8__", "start_date": "2000-06-01T05:00:00+03:00", "is_sandbox": false } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config_oauth.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config_oauth.json index 21019786e6a3..771ae5dbac0a 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config_oauth.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/invalid_config_oauth.json @@ -2,7 +2,7 @@ "credentials": { "auth_type": "oauth2.0", "client_id": "AWA__", - "secret": "ENC__", + "client_secret": "ENC__", "refresh_token": "__" }, "start_date": "2021-07-03T00:00:00+00:00", diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json index 6f5dbf626fac..11d8539eeb3f 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_config.json @@ -1,6 +1,6 @@ { "client_id": "PAYPAL_CLIENT_ID", - "secret": "PAYPAL_SECRET", + "client_secret": "PAYPAL_SECRET", "start_date": "2021-06-01T00:00:00+00:00", "is_sandbox": false } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json index 55e16864a243..830010c0927a 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/sample_state.json @@ -1,8 +1,8 @@ { "transactions": { - "date": "2021-06-04T17:34:43+00:00" + "transaction_updated_date": "2021-06-04T17:34:43+00:00" }, "balances": { - "date": "2021-06-04T17:34:43+00:00" + "as_of_time": "2021-06-04T17:34:43+00:00" } } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json index dc08b6e0fe0e..bbcff50acaf3 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/integration_tests/state.json @@ -1,8 +1,8 @@ { "transactions": { - "date": "2021-06-18T16:24:13+03:00" + "transaction_updated_date": "2021-06-18T16:24:13+03:00" }, "balances": { - "date": "2021-06-18T16:24:13+03:00" + "as_of_time": "2021-06-18T16:24:13+03:00" } } diff --git a/airbyte-integrations/connectors/source-paypal-transaction/main.py b/airbyte-integrations/connectors/source-paypal-transaction/main.py index 51be49033dca..06823a4a71e5 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/main.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_paypal_transaction import SourcePaypalTransaction +from source_paypal_transaction.run import run if __name__ == "__main__": - source = SourcePaypalTransaction() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-paypal-transaction/metadata.yaml b/airbyte-integrations/connectors/source-paypal-transaction/metadata.yaml index 9966b577ccbd..d456b79b905f 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/metadata.yaml +++ b/airbyte-integrations/connectors/source-paypal-transaction/metadata.yaml @@ -1,13 +1,19 @@ data: + ab_internal: + ql: 200 + sl: 200 allowedHosts: hosts: - api-m.paypal.com - api-m.sandbox.paypal.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: api connectorType: source definitionId: d913b0f2-cc51-4e55-a44c-8ba1697b9239 - dockerImageTag: 2.0.0 + dockerImageTag: 2.2.1 dockerRepository: airbyte/source-paypal-transaction + documentationUrl: https://docs.airbyte.com/integrations/sources/paypal-transaction githubIssueLabel: source-paypal-transaction icon: paypal.svg license: MIT @@ -17,12 +23,14 @@ data: enabled: true oss: enabled: true + releaseDate: 2021-06-10 releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/paypal-transaction - tags: - - language:python - ab_internal: - sl: 200 - ql: 400 + releases: + breakingChanges: + 2.1.0: + message: 'Version 2.1.0 changes the format of the state. The format of the cursor changed from "2021-06-18T16:24:13+03:00" to "2021-06-18T16:24:13Z". The state key for the transactions stream changed to "transaction_updated_date" and the key for the balances stream change to "as_of_time". The upgrade is safe, but rolling back is not.' + upgradeDeadline: "2023-09-18" supportLevel: certified + tags: + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-paypal-transaction/setup.py b/airbyte-integrations/connectors/source-paypal-transaction/setup.py index 402f69640e62..a7f633e8ca28 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/setup.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk", + "airbyte-cdk>=0.51.44", ] TEST_REQUIREMENTS = [ @@ -16,13 +16,18 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-paypal-transaction=source_paypal_transaction.run:run", + ], + }, name="source_paypal_transaction", description="Source implementation for Paypal Transaction.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/__init__.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/__init__.py index 6487dc4b3d4f..c8ddefb08bf5 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/__init__.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/__init__.py @@ -1,26 +1,7 @@ -""" -MIT License +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# -Copyright (c) 2021 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" from .source import SourcePaypalTransaction diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/components.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/components.py new file mode 100644 index 000000000000..332549f3b617 --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/components.py @@ -0,0 +1,63 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import base64 +import logging +from dataclasses import dataclass + +import backoff +import requests +from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator +from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException + +logger = logging.getLogger("airbyte") + + +@dataclass +class PayPalOauth2Authenticator(DeclarativeOauth2Authenticator): + """Request example for API token extraction: + For `old_config` scenario: + curl -v POST https://api-m.sandbox.paypal.com/v1/oauth2/token \ + -H "Accept: application/json" \ + -H "Accept-Language: en_US" \ + -u "CLIENT_ID:SECRET" \ + -d "grant_type=client_credentials" + """ + + # config: Mapping[str, Any] + # client_id: Union[InterpolatedString, str] + # client_secret: Union[InterpolatedString, str] + # refresh_request_body: Optional[Mapping[str, Any]] = None + # token_refresh_endpoint: Union[InterpolatedString, str] + # grant_type: Union[InterpolatedString, str] = "refresh_token" + # expires_in_name: Union[InterpolatedString, str] = "expires_in" + # access_token_name: Union[InterpolatedString, str] = "access_token" + # parameters: InitVar[Mapping[str, Any]] + + def get_headers(self): + basic_auth = base64.b64encode(bytes(f"{self.get_client_id()}:{self.get_client_secret()}", "utf-8")).decode("utf-8") + return {"Authorization": f"Basic {basic_auth}"} + + @backoff.on_exception( + backoff.expo, + DefaultBackoffException, + on_backoff=lambda details: logger.info( + f"Caught retryable error after {details['tries']} tries. Waiting {details['wait']} seconds then retrying..." + ), + max_time=300, + ) + def _get_refresh_access_token_response(self): + try: + response = requests.request( + method="POST", url=self.get_token_refresh_endpoint(), data=self.build_refresh_request_body(), headers=self.get_headers() + ) + self._log_response(response) + response.raise_for_status() + return response.json() + except requests.exceptions.RequestException as e: + if e.response.status_code == 429 or e.response.status_code >= 500: + raise DefaultBackoffException(request=e.response.request, response=e.response) + raise + except Exception as e: + raise Exception(f"Error while refreshing access token: {e}") from e diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/manifest.yaml b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/manifest.yaml new file mode 100644 index 000000000000..58f6d026171a --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/manifest.yaml @@ -0,0 +1,212 @@ +version: 0.50.2 +type: DeclarativeSource + +check: + type: CheckStream + stream_names: + - balances + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - "{{ parameters.field_path }}" + + requester: + type: HttpRequester + url_base: 'https://api-m.{{ "sandbox." if config["is_sandbox"] }}paypal.com/v1/reporting/' + path: "{{ parameters.path }}" + http_method: GET + request_headers: + Content-Type: application/json + authenticator: + type: CustomAuthenticator + class_name: source_paypal_transaction.components.PayPalOauth2Authenticator + client_id: "{{ config['client_id'] }}" + client_secret: "{{ config['client_secret'] }}" + refresh_request_body: + Content-Type: application/x-www-form-urlencoded + token_refresh_endpoint: 'https://api-m.{{ "sandbox." if config["is_sandbox"] }}paypal.com/v1/oauth2/token' + grant_type: client_credentials + expires_in_name: expires_in + access_token_name: access_token + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + backoff_strategies: + - type: ConstantBackoffStrategy + backoff_time_in_seconds: 300 + request_body_json: {} + + transactions_stream: + type: DeclarativeStream + primary_key: transaction_id + name: "transactions" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + inject_into: request_parameter + field_name: page_size + type: RequestOption + pagination_strategy: + type: PageIncrement + start_from_page: 1 + page_size: 500 + requester: + $ref: "#/definitions/requester" + request_parameters: + fields: all + transformations: + - type: AddFields + fields: + - path: + - transaction_updated_date + value: >- + {{ format_datetime(record['transaction_info']['transaction_updated_date'], '%Y-%m-%dT%H:%M:%SZ') }} + - type: AddFields + fields: + - path: + - transaction_id + value: "{{ record['transaction_info']['transaction_id'] }}" + value_type: string + incremental_sync: + type: DatetimeBasedCursor + cursor_field: transaction_updated_date + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%SZ" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_datetime: + type: MinMaxDatetime + datetime: >- + {{ max( format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%SZ'), day_delta(-1095, format='%Y-%m-%dT%H:%M:%SZ') ) }} + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: start_date + inject_into: request_parameter + end_time_option: + type: RequestOption + field_name: end_date + inject_into: request_parameter + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + step: "P{{ config.get('time_window', 7) }}D" + cursor_granularity: PT1S + $parameters: + path: "transactions" + field_path: transaction_details + + balances_stream: + type: DeclarativeStream + primary_key: as_of_time + name: "balances" + retriever: + type: SimpleRetriever + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: DefaultPaginator + pagination_strategy: + type: PageIncrement + page_size: 500 + requester: + $ref: "#/definitions/requester" + request_parameters: {} + transformations: + - type: AddFields + fields: + - path: + - as_of_time + value: "{{ format_datetime(record['as_of_time'], '%Y-%m-%dT%H:%M:%SZ') }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: as_of_time + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%SZ" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_datetime: + type: MinMaxDatetime + datetime: >- + {{ max( format_datetime(config['start_date'], '%Y-%m-%dT%H:%M:%SZ'), day_delta(-1095, format='%Y-%m-%dT%H:%M:%SZ') ) }} + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: as_of_time + inject_into: request_parameter + $parameters: + path: "balances" + +streams: + - "#/definitions/transactions_stream" + - "#/definitions/balances_stream" + +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/paypal-transactions + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + type: object + additionalProperties: true + required: + - client_id + - client_secret + - start_date + - is_sandbox + properties: + client_id: + type: string + title: Client ID + description: "The Client ID of your Paypal developer application." + airbyte_secret: true + order: 0 + client_secret: + type: string + title: Client secret + description: "The Client Secret of your Paypal developer application." + airbyte_secret: true + order: 1 + start_date: + title: Start Date + description: >- + Start Date for data extraction in ISO + format. Date must be in range from 3 years till 12 hrs before + present time. + type: string + examples: ["2021-06-11T23:59:59", "2021-06-11T23:59:59+00:00"] + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(|Z|[+-][0-9]{2}:[0-9]{2})$ + format: "date-time" + order: 2 + is_sandbox: + title: "Sandbox" + description: "Determines whether to use the sandbox or production environment." + type: "boolean" + default: false + refresh_token: + type: "string" + title: "Refresh token" + description: "The key to refresh the expired access token." + airbyte_secret: true + time_window: + type: "integer" + title: "Number of days per request" + description: "The number of days per request. Must be a number between 1 and 31." + default: 7 + minimum: 1 + maximum: 31 diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/run.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/run.py new file mode 100644 index 000000000000..1a6d4cc56c0e --- /dev/null +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_paypal_transaction import SourcePaypalTransaction + + +def run(): + source = SourcePaypalTransaction() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json index 5ec4c8f3ac3d..fa69e2db398b 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/balances.json @@ -1,6 +1,7 @@ { "$schema": "https://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "balances": { "type": ["null", "array"], diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json index 82a07b42e9ab..351bf5762b7e 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/schemas/transactions.json @@ -1,9 +1,11 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "transaction_info": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "paypal_reference_id": { "type": ["null", "string"], @@ -40,6 +42,7 @@ }, "transaction_amount": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "currency_code": { "type": "string", @@ -54,6 +57,7 @@ }, "fee_amount": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "currency_code": { "type": "string", @@ -68,6 +72,7 @@ }, "insurance_amount": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "currency_code": { "type": "string", @@ -82,6 +87,7 @@ }, "shipping_amount": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "currency_code": { "type": "string", @@ -96,6 +102,7 @@ }, "shipping_discount_amount": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "currency_code": { "type": "string", @@ -137,8 +144,13 @@ "type": ["null", "string"], "format": "date-time" }, + "transaction_updated_date": { + "type": ["null", "string"], + "format": "date-time" + }, "payer_info": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "account_id": { "type": ["null", "string"], @@ -158,6 +170,7 @@ }, "payer_name": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "given_name": { "type": ["null", "string"], @@ -181,6 +194,7 @@ }, "shipping_info": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "name": { "type": ["null", "string"], @@ -188,6 +202,7 @@ }, "address": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "line1": { "type": ["null", "string"] @@ -213,11 +228,13 @@ }, "cart_info": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "item_details": { "type": "array", "items": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "item_code": { "type": ["null", "string"], @@ -235,6 +252,7 @@ }, "item_unit_price": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "currency_code": { "type": "string", @@ -249,6 +267,7 @@ }, "item_amount": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "currency_code": { "type": "string", @@ -265,9 +284,11 @@ "type": "array", "items": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "tax_amount": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "currency_code": { "type": "string", @@ -285,6 +306,7 @@ }, "total_item_amount": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "currency_code": { "type": "string", @@ -308,6 +330,7 @@ }, "store_info": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "store_id": { "type": ["null", "string"], @@ -321,6 +344,7 @@ }, "auction_info": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "auction_site": { "type": ["null", "string"], @@ -341,11 +365,13 @@ }, "incentive_info": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "incentive_details": { "type": "array", "items": { "type": "object", + "additionalProperties": true, "properties": { "incentive_type": { "type": ["null", "string"], @@ -357,6 +383,7 @@ }, "incentive_amount": { "type": "object", + "additionalProperties": true, "properties": { "currency_code": { "type": "string", diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py index 2fbb106fd3e5..afd56a9278a1 100644 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py +++ b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/source.py @@ -2,545 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import base64 -import json -import logging -import time -from abc import ABC -from datetime import datetime, timedelta -from typing import Any, Callable, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator, Oauth2Authenticator -from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from dateutil.parser import isoparse +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -from .utils import middle_date_slices +WARNING: Do not modify this file. +""" -class PaypalHttpException(Exception): - """HTTPError Exception with detailed info""" - - def __init__(self, error: requests.exceptions.HTTPError): - self.error = error - - def __str__(self): - message = repr(self.error) - - if self.error.response.content: - details = self.error_message() - message = f"{message} Details: {details}" - - return message - - def error_message(self): - content = self.error.response.content.decode() - try: - details = json.loads(content) - except json.decoder.JSONDecodeError: - details = content - - return details - - def __repr__(self): - return self.__str__() - - -def get_endpoint(is_sandbox: bool = False) -> str: - if is_sandbox: - return "https://api-m.sandbox.paypal.com" - - return "https://api-m.paypal.com" - - -class PaypalTransactionStream(HttpStream, ABC): - """Abstract class for Paypal Transaction Stream. - - Important note about 'start_date' params: - 'start_date' is one of required params, it comes from spec configuration or from stream state. - In both cases it must meet the following conditions: - - minimum_allowed_start_date <= start_date <= end_date <= last_refreshed_datetime <= now() - - otherwise API throws an "Data for the given start date is not available" error. - - So the prevent this error 'start_date' will be reset to: - minimum_allowed_start_date - if 'start_date' is too old - min(maximum_allowed_start_date, last_refreshed_datetime) - if 'start_date' is too recent - """ - - page_size = "500" # API limit - - # Date limits are needed to prevent API error: "Data for the given start date is not available" - # API limit: (now() - start_date_min) <= start_date <= end_date <= last_refreshed_datetime <= now - start_date_min: Mapping[str, int] = {"days": 3 * 365} # API limit - 3 years - last_refreshed_datetime: Optional[datetime] = None # extracted from API response. Indicate the most resent possible start_date - stream_slice_period: Mapping[str, int] = {"days": 15} # max period is 31 days (API limit) - - requests_per_minute: int = 30 # API limit is 50 reqs/min from 1 IP to all endpoints, otherwise IP is banned for 5 mins - # if the stream has nested cursor_field, we should trry to unnest it once parsing the recods to avoid normalization conflicts. - unnest_cursor: bool = False - unnest_pk: bool = False - nested_object: str = None - - def __init__( - self, - authenticator: HttpAuthenticator, - start_date: Union[datetime, str], - end_date: Union[datetime, str] = None, - is_sandbox: bool = False, - **kwargs, - ): - now = datetime.now().replace(microsecond=0).astimezone() - - if end_date and isinstance(end_date, str): - end_date = isoparse(end_date) - self.end_date: datetime = end_date if end_date and end_date < now else now - - if start_date and isinstance(start_date, str): - start_date = isoparse(start_date) - - minimum_allowed_start_date = now - timedelta(**self.start_date_min) - if start_date < minimum_allowed_start_date: - self.logger.log( - logging.WARN, - f'Stream {self.name}: start_date "{start_date.isoformat()}" is too old. ' - + f'Reset start_date to the minimum_allowed_start_date "{minimum_allowed_start_date.isoformat()}"', - ) - start_date = minimum_allowed_start_date - - self.maximum_allowed_start_date = min(now, self.end_date) - if start_date > self.maximum_allowed_start_date: - self.logger.log( - logging.WARN, - f'Stream {self.name}: start_date "{start_date.isoformat()}" is too recent. ' - + f'Reset start_date to the maximum_allowed_start_date "{self.maximum_allowed_start_date.isoformat()}"', - ) - start_date = self.maximum_allowed_start_date - - self.start_date = start_date - - self.is_sandbox = is_sandbox - - super().__init__(authenticator=authenticator) - - def validate_input_dates(self): - # Validate input dates - if self.start_date > self.end_date: - raise Exception(f"start_date {self.start_date.isoformat()} is greater than end_date {self.end_date.isoformat()}") - - @property - def url_base(self) -> str: - return f"{get_endpoint(self.is_sandbox)}/v1/reporting/" - - @property - def availability_strategy(self) -> Optional["AvailabilityStrategy"]: - return None - - def request_headers(self, **kwargs) -> Mapping[str, Any]: - return {"Content-Type": "application/json"} - - def backoff_time(self, response: requests.Response) -> Optional[float]: - # API limit is 50 reqs/min from 1 IP to all endpoints, otherwise IP is banned for 5 mins - return 5 * 60.1 - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - json_response = response.json() - - # Save extracted last_refreshed_datetime to use it as maximum allowed start_date - last_refreshed_datetime = json_response.get("last_refreshed_datetime") - self.last_refreshed_datetime = isoparse(last_refreshed_datetime) if last_refreshed_datetime else None - - if self.data_field is not None: - data = json_response.get(self.data_field, []) - else: - data = [json_response] - - for record in data: - # In order to support direct datetime string comparison (which is performed in incremental acceptance tests) - # convert any date format to python iso format string for date based cursors - self.update_field(record, self.cursor_field, lambda date: isoparse(date).isoformat()) - # unnest cursor_field to handle normalization correctly - if self.unnest_cursor: - self.unnest_field(record, self.nested_object, self.cursor_field) - # unnest primary_key to handle normalization correctly - if self.unnest_pk: - self.unnest_field(record, self.nested_object, self.primary_key) - yield record - - # sleep for 1-2 secs to not reach rate limit: 50 requests per minute - time.sleep(60 / self.requests_per_minute) - - @staticmethod - def unnest_field(record: Mapping[str, Any], unnest_from: Dict, cursor_field: str): - """ - Unnest cursor_field to the root level of the record. - """ - if unnest_from in record: - record[cursor_field] = record.get(unnest_from).get(cursor_field) - - @staticmethod - def update_field(record: Mapping[str, Any], field_path: Union[List[str], str], update: Callable[[Any], None]): - if not isinstance(field_path, List): - field_path = [field_path] - - last_field = field_path[-1] - data = PaypalTransactionStream.get_field(record, field_path[:-1]) - if data and last_field in data: - data[last_field] = update(data[last_field]) - - @staticmethod - def get_field(record: Mapping[str, Any], field_path: Union[List[str], str]): - - if not isinstance(field_path, List): - field_path = [field_path] - - data = record - for attr in field_path: - if data and isinstance(data, dict): - data = data.get(attr) - else: - return None - - return data - - @staticmethod - def max_records_in_response_reached(exception: Exception, **kwargs): - message = exception.error_message() - return message.get("name") == "RESULTSET_TOO_LARGE" - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: - # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state - # we then return an updated state object. If this is the first time we run a sync or no state was passed, current_stream_state will be None. - latest_record_date_str: str = self.get_field(latest_record, self.cursor_field) - - if current_stream_state and "date" in current_stream_state and latest_record_date_str: - # isoparse supports different formats, like: - # python iso format: 2021-06-04T00:00:00+03:00 - # format from transactions record: 2021-06-04T00:00:00+0300 - # format from balances record: 2021-06-02T00:00:00Z - latest_record_date = isoparse(latest_record_date_str) - current_parsed_date = isoparse(current_stream_state["date"]) - - return {"date": max(current_parsed_date, latest_record_date).isoformat()} - else: - return {"date": self.start_date.isoformat()} - - def get_last_refreshed_datetime(self, sync_mode): - """Get last_refreshed_datetime attribute from API response by running PaypalTransactionStream().read_records() - with 'empty' stream_slice (range=0) - - last_refreshed_datetime indicates the maximum available start_date for which API has data. - If request start_date > last_refreshed_datetime then API throws an error: - "Data for the given start date is not available" - """ - paypal_stream = self.__class__( - authenticator=self.authenticator, - start_date=self.start_date, - end_date=self.start_date, - is_sandbox=self.is_sandbox, - ) - stream_slice = { - "start_date": self.start_date.isoformat(), - "end_date": self.start_date.isoformat(), - } - list(paypal_stream.read_records(sync_mode=sync_mode, stream_slice=stream_slice)) - return paypal_stream.last_refreshed_datetime - - def stream_slices( - self, sync_mode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, any]]]: - """ - Returns a list of slices for each day (by default) between the start date and end date. - The return value is a list of dicts {'start_date': date_string, 'end_date': date_string}. - """ - period = timedelta(**self.stream_slice_period) - - # get last_refreshed_datetime from API response to use as maximum allowed start_date - self.last_refreshed_datetime = self.get_last_refreshed_datetime(sync_mode) - if self.last_refreshed_datetime: - self.logger.info(f"Maximum allowed start_date is {self.last_refreshed_datetime} based on info from API response") - self.maximum_allowed_start_date = min(self.last_refreshed_datetime, self.maximum_allowed_start_date) - - slice_start_date = self.start_date - - if stream_state: - stream_state_date = isoparse(stream_state.get("date")) - - # slice_start_date should be the most recent date: - slice_start_date = max(slice_start_date, stream_state_date) - - slices = [] - while slice_start_date <= self.maximum_allowed_start_date: - slices.append( - { - "start_date": slice_start_date.isoformat(), - "end_date": min(slice_start_date + period, self.end_date).isoformat(), - } - ) - slice_start_date += period - - return slices - - def _prepared_request( - self, stream_slice: Mapping[str, Any] = None, stream_state: Mapping[str, Any] = None, next_page_token: Optional[dict] = None - ): - request_headers = self.request_headers(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - request = self._create_prepared_request( - path=self.path(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), - headers=dict(request_headers, **self.authenticator.get_auth_header()), - params=self.request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), - json=self.request_body_json(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), - data=self.request_body_data(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), - ) - request_kwargs = self.request_kwargs(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - - return request, request_kwargs - - def read_records( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Mapping[str, Any]]: - stream_state = stream_state or {} - pagination_complete = False - next_page_token = None - while not pagination_complete: - request, request_kwargs = self._prepared_request( - stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token - ) - - try: - response = self._send_request(request, request_kwargs) - except PaypalHttpException as exception: - if self.max_records_in_response_reached(exception) and (date_slices := middle_date_slices(stream_slice)): - for date_slice in date_slices: - yield from self.read_records( - sync_mode, cursor_field=cursor_field, stream_slice=date_slice, stream_state=stream_state - ) - break - raise exception - - yield from self.parse_response(response, stream_state=stream_state, stream_slice=stream_slice) - - next_page_token = self.next_page_token(response) - if not next_page_token: - pagination_complete = True - - # Always return an empty generator just in case no records were ever yielded - yield from [] - - def _send_request(self, request: requests.PreparedRequest, request_kwargs: Mapping[str, Any]) -> requests.Response: - try: - return super()._send_request(request, request_kwargs) - except requests.exceptions.HTTPError as http_error: - raise PaypalHttpException(http_error) - - -class Transactions(PaypalTransactionStream): - """List Paypal Transactions on a specific date range - API Docs: https://developer.paypal.com/docs/integration/direct/transaction-search/#list-transactions - Endpoint: /v1/reporting/transactions - """ - - data_field = "transaction_details" - nested_object = "transaction_info" - - primary_key = "transaction_id" - cursor_field = "transaction_initiation_date" - - unnest_cursor = True - unnest_pk = True - - transformer = TypeTransformer(TransformConfig.CustomSchemaNormalization) - - # TODO handle API error when 1 request returns more than 10000 records. - # https://github.com/airbytehq/airbyte/issues/4404 - records_per_request = 10000 - - def path(self, **kwargs) -> str: - return "transactions" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - decoded_response = response.json() - total_pages = decoded_response.get("total_pages") - page_number = decoded_response.get("page") - if page_number < total_pages: - return {"page": page_number + 1} - else: - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - page_number = 1 - if next_page_token: - page_number = next_page_token.get("page") - - return { - "start_date": stream_slice["start_date"], - "end_date": stream_slice["end_date"], - "fields": "all", - "page_size": self.page_size, - "page": page_number, - } - - @transformer.registerCustomTransform - def transform_function(original_value: Any, field_schema: Dict[str, Any]) -> Any: - if isinstance(original_value, str) and field_schema["type"] == "number": - return float(original_value) - elif isinstance(original_value, str) and field_schema["type"] == "integer": - return int(original_value) - else: - return original_value - - -class Balances(PaypalTransactionStream): - """Get account balance on a specific date - API Docs: https://developer.paypal.com/docs/integration/direct/transaction-search/#check-balancess - """ - - primary_key = "as_of_time" - cursor_field = "as_of_time" - data_field = None - - def path(self, **kwargs) -> str: - return "balances" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return { - "as_of_time": stream_slice["start_date"], - } - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - -class PayPalOauth2Authenticator(Oauth2Authenticator): - """Request example for API token extraction: - For `old_config` scenario: - curl -v POST https://api-m.sandbox.paypal.com/v1/oauth2/token \ - -H "Accept: application/json" \ - -H "Accept-Language: en_US" \ - -u "CLIENT_ID:SECRET" \ - -d "grant_type=client_credentials" - """ - - def __init__(self, config: Dict): - self.old_config: bool = False - # default auth args - self.auth_args: Dict = { - "token_refresh_endpoint": f"{get_endpoint(config['is_sandbox'])}/v1/oauth2/token", - "refresh_token": "", - } - # support old configs - if "client_id" in config and ("client_secret" in config or "secret" in config): - self.old_config = True - client_secret = config.get("client_secret", config.get("secret")) - self.auth_args.update( - **{"client_id": config["client_id"], "client_secret": client_secret, "refresh_token": config.get("refresh_token")} - ) - # new configs - if "credentials" in config: - credentials = config.get("credentials") - auth_type = credentials.get("auth_type") - self.auth_args.update(**{"client_id": credentials["client_id"], "client_secret": credentials["client_secret"]}) - if auth_type == "oauth2.0": - self.auth_args["refresh_token"] = credentials["refresh_token"] - elif auth_type == "private_oauth": - self.old_config = True - - self.config = config - super().__init__(**self.auth_args) - - def get_headers(self): - # support old configs - if self.old_config: - return {"Accept": "application/json", "Accept-Language": "en_US"} - # new configs - basic_auth = base64.b64encode(bytes(f"{self.client_id}:{self.client_secret}", "utf-8")).decode("utf-8") - return {"Authorization": f"Basic {basic_auth}"} - - def get_refresh_request_body(self) -> Mapping[str, Any]: - # support old configs - if self.old_config: - return {"grant_type": "client_credentials"} - # new configs - return {"grant_type": "refresh_token", "refresh_token": self.refresh_token} - - def refresh_access_token(self) -> Tuple[str, int]: - """ - returns a tuple of (access_token, token_lifespan_in_seconds) - """ - request_args = { - "url": self.token_refresh_endpoint, - "data": self.get_refresh_request_body(), - "headers": self.get_headers(), - } - try: - # support old configs - if self.old_config: - request_args["auth"] = (self.client_id, self.client_secret) - response = requests.post(**request_args) - response.raise_for_status() - response_json = response.json() - return response_json["access_token"], response_json["expires_in"] - except Exception as e: - raise Exception(f"Error while refreshing access token: {e}") from e - - -class SourcePaypalTransaction(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - """ - :param config: the user-input config object conforming to the connector's spec.json - :param logger: logger object - :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. - """ - authenticator = PayPalOauth2Authenticator(config) - - # Try to get API TOKEN - token = authenticator.get_access_token() - if not token: - return False, "Unable to fetch Paypal API token due to incorrect client_id or secret" - - # Try to initiate a stream and validate input date params - try: - # validate input date ranges - Transactions(authenticator=authenticator, **config).validate_input_dates() - - # validate if Paypal API is able to extract data for given start_data - start_date = isoparse(config["start_date"]) - end_date = start_date + timedelta(days=1) - stream_slice = { - "start_date": start_date.isoformat(), - "end_date": end_date.isoformat(), - } - records = Transactions(authenticator=authenticator, **config).read_records(sync_mode=None, stream_slice=stream_slice) - # Try to read one value from records iterator - next(records, None) - return True, None - except Exception as e: - if "Data for the given start date is not available" in repr(e): - return False, f"Data for the given start date ({config['start_date']}) is not available, please use more recent start date" - else: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - authenticator = PayPalOauth2Authenticator(config) - - return [ - Transactions(authenticator=authenticator, **config), - Balances(authenticator=authenticator, **config), - ] +# Declarative Source +class SourcePaypalTransaction(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json deleted file mode 100644 index 8dcdd401518f..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/spec.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/paypal-transactions", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Paypal Transaction Search", - "type": "object", - "required": ["client_id", "client_secret", "start_date", "is_sandbox"], - "additionalProperties": true, - "properties": { - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of your Paypal developer application.", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "title": "Client secret", - "description": "The Client Secret of your Paypal developer application.", - "airbyte_secret": true - }, - "refresh_token": { - "type": "string", - "title": "Refresh token", - "description": "The key to refresh the expired access token.", - "airbyte_secret": true - }, - "start_date": { - "type": "string", - "title": "Start Date", - "description": "Start Date for data extraction in ISO format. Date must be in range from 3 years till 12 hrs before present time.", - "examples": ["2021-06-11T23:59:59", "2021-06-11T23:59:59+00:00"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(|Z|[+-][0-9]{2}:[0-9]{2})$", - "format": "date-time" - }, - "is_sandbox": { - "title": "Sandbox", - "description": "Determines whether to use the sandbox or production environment.", - "type": "boolean", - "default": false - } - } - } -} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/utils.py b/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/utils.py deleted file mode 100644 index 732a5119fcb3..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/source_paypal_transaction/utils.py +++ /dev/null @@ -1,30 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from datetime import datetime - - -def to_datetime_str(date: datetime) -> datetime: - """ - Returns the formated datetime string. - :: Output example: '2021-07-15T0:0:0+00:00' FORMAT : "%Y-%m-%dT%H:%M:%S%z" - """ - return datetime.strptime(date, "%Y-%m-%dT%H:%M:%S%z") - - -def middle_date_slices(stream_slice): - """Returns the mid-split datetime slices.""" - start_date, end_date = to_datetime_str(stream_slice["start_date"]), to_datetime_str(stream_slice["end_date"]) - if start_date < end_date: - middle_date = start_date + (end_date - start_date) / 2 - return [ - { - "start_date": start_date.isoformat(), - "end_date": middle_date.isoformat(), - }, - { - "start_date": middle_date.isoformat(), - "end_date": end_date.isoformat(), - }, - ] diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py deleted file mode 100644 index 8e2333572304..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/conftest.py +++ /dev/null @@ -1,102 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import json -import pathlib - -import pytest -from source_paypal_transaction.source import PayPalOauth2Authenticator, SourcePaypalTransaction - - -@pytest.fixture() -def api_endpoint(): - return "https://api-m.paypal.com" - - -@pytest.fixture() -def sandbox_api_endpoint(): - return "https://api-m.sandbox.paypal.com" - - -@pytest.fixture(autouse=True) -def time_sleep_mock(mocker): - time_mock = mocker.patch("time.sleep", lambda x: None) - yield time_mock - - -@pytest.fixture(autouse=True) -def transactions(request): - file = pathlib.Path(request.node.fspath.strpath) - transaction = file.with_name("transaction.json") - with transaction.open() as fp: - return json.load(fp) - - -@pytest.fixture() -def prod_config(): - """ - Credentials for oauth2.0 authorization - """ - return { - "client_id": "some_client_id", - "client_secret": "some_secret", - "start_date": "2021-07-01T00:00:00+00:00", - "end_date": "2021-07-10T00:00:00+00:00", - "is_sandbox": False, - } - - -@pytest.fixture() -def sandbox_config(): - """ - Credentials for oauth2.0 authorization - """ - return { - "client_id": "some_client_id", - "client_secret": "some_secret", - "start_date": "2021-07-01T00:00:00+00:00", - "end_date": "2021-07-10T00:00:00+00:00", - "is_sandbox": True, - } - - -@pytest.fixture() -def new_prod_config(): - """ - Credentials for oauth2.0 authorization - """ - return { - "credentials": { - "auth_type": "oauth2.0", - "client_id": "some_client_id", - "client_secret": "some_client_secret", - "refresh_token": "some_refresh_token", - }, - "start_date": "2021-07-01T00:00:00+00:00", - "end_date": "2021-07-10T00:00:00+00:00", - "is_sandbox": False, - } - - -@pytest.fixture() -def error_while_refreshing_access_token(): - """ - Error raised when using incorrect access token - """ - return "Error while refreshing access token: 'access_token'" - - -@pytest.fixture() -def authenticator_instance(prod_config): - return PayPalOauth2Authenticator(prod_config) - - -@pytest.fixture() -def new_format_authenticator_instance(new_prod_config): - return PayPalOauth2Authenticator(new_prod_config) - - -@pytest.fixture() -def source_instance(): - return SourcePaypalTransaction() diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/test_source.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/test_source.py deleted file mode 100644 index 91c118ecb81a..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/test_source.py +++ /dev/null @@ -1,62 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -from source_paypal_transaction.source import PaypalHttpException, PayPalOauth2Authenticator, get_endpoint - - -class TestAuthentication: - def test_init_token_authentication_init(self, authenticator_instance): - assert isinstance(authenticator_instance, PayPalOauth2Authenticator) - - def test_get_refresh_request_body(self, authenticator_instance): - expected_body = {"grant_type": "client_credentials"} - assert authenticator_instance.get_refresh_request_body() == expected_body - - def test_oauth2_refresh_token_ok(self, requests_mock, authenticator_instance, api_endpoint): - requests_mock.post(f"{api_endpoint}/v1/oauth2/token", json={"access_token": "test_access_token", "expires_in": 12345}) - result = authenticator_instance.refresh_access_token() - assert result == ("test_access_token", 12345) - - def test_oauth2_refresh_token_failed(self, requests_mock, authenticator_instance, api_endpoint, error_while_refreshing_access_token): - requests_mock.post(f"{api_endpoint}/v1/oauth2/token", json={}) - try: - authenticator_instance.refresh_access_token() - except Exception as e: - assert e.args[0] == error_while_refreshing_access_token - - def test_new_oauth2_refresh_token_ok(self, requests_mock, new_format_authenticator_instance, api_endpoint): - requests_mock.post(f"{api_endpoint}/v1/oauth2/token", json={"access_token": "test_access_token", "expires_in": 12345}) - result = new_format_authenticator_instance.refresh_access_token() - assert result == ("test_access_token", 12345) - - def test_streams_count(self, prod_config, source_instance): - assert len(source_instance.streams(prod_config)) == 2 - - def test_check_connection_ok(self, requests_mock, prod_config, api_endpoint, transactions, source_instance): - requests_mock.post(f"{api_endpoint}/v1/oauth2/token", json={"access_token": "test_access_token", "expires_in": 12345}) - url = ( - f"{api_endpoint}/v1/reporting/transactions" - + "?start_date=2021-07-01T00%3A00%3A00%2B00%3A00&end_date=2021-07-02T00%3A00%3A00%2B00%3A00&fields=all&page_size=500&page=1" - ) - requests_mock.get(url, json=transactions) - assert source_instance.check_connection(logger=MagicMock(), config=prod_config) == (True, None) - - def test_check_connection_error(self, requests_mock, prod_config, api_endpoint, source_instance): - requests_mock.post(f"{api_endpoint}/v1/oauth2/token", json={"access_token": "test_access_token", "expires_in": 12345}) - url = ( - f"{api_endpoint}/v1/reporting/transactions" - + "?start_date=2021-07-01T00%3A00%3A00%2B00%3A00&end_date=2021-07-02T00%3A00%3A00%2B00%3A00&fields=all&page_size=500&page=1" - ) - requests_mock.get(url, status_code=400, json={}) - check_result, error = source_instance.check_connection(logger=MagicMock(), config=prod_config) - assert not check_result - assert isinstance(error, PaypalHttpException) - - def test_get_prod_endpoint(self, prod_config, api_endpoint): - assert get_endpoint(prod_config["is_sandbox"]) == api_endpoint - - def test_get_sandbox_endpoint(self, sandbox_config, sandbox_api_endpoint): - assert get_endpoint(sandbox_config["is_sandbox"]) == sandbox_api_endpoint diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/transaction.json b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/transaction.json deleted file mode 100644 index 00621a91e6d1..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/transaction.json +++ /dev/null @@ -1,151 +0,0 @@ -{ - "transaction_details": [ - { - "transaction_info": { - "paypal_account_id": "6STWC2LSUYYYE", - "transaction_id": "5TY05013RG002845M", - "transaction_event_code": "T0006", - "transaction_initiation_date": "2014-07-11T04:03:52+0000", - "transaction_updated_date": "2014-07-11T04:03:52+0000", - "transaction_amount": { - "currency_code": "USD", - "value": "465.00" - }, - "fee_amount": { - "currency_code": "USD", - "value": "-13.79" - }, - "insurance_amount": { - "currency_code": "USD", - "value": "15.00" - }, - "shipping_amount": { - "currency_code": "USD", - "value": "30.00" - }, - "shipping_discount_amount": { - "currency_code": "USD", - "value": "10.00" - }, - "transaction_status": "S", - "transaction_subject": "Bill for your purchase", - "transaction_note": "Check out the latest sales", - "invoice_id": "Invoice-005", - "custom_field": "Thank you for your business", - "protection_eligibility": "01" - }, - "payer_info": { - "account_id": "6STWC2LSUYYYE", - "email_address": "consumer@example.com", - "address_status": "Y", - "payer_status": "Y", - "payer_name": { - "given_name": "test", - "surname": "consumer", - "alternate_full_name": "test consumer" - }, - "country_code": "US" - }, - "shipping_info": { - "name": "Sowmith", - "address": { - "line1": "Eco Space, bellandur", - "line2": "OuterRingRoad", - "city": "Bangalore", - "country_code": "IN", - "postal_code": "560103" - } - }, - "cart_info": { - "item_details": [ - { - "item_code": "ItemCode-1", - "item_name": "Item1 - radio", - "item_description": "Radio", - "item_quantity": "2", - "item_unit_price": { - "currency_code": "USD", - "value": "50.00" - }, - "item_amount": { - "currency_code": "USD", - "value": "100.00" - }, - "tax_amounts": [ - { - "tax_amount": { - "currency_code": "USD", - "value": "20.00" - } - } - ], - "total_item_amount": { - "currency_code": "USD", - "value": "120.00" - }, - "invoice_number": "Invoice-005" - }, - { - "item_code": "ItemCode-2", - "item_name": "Item2 - Headset", - "item_description": "Headset", - "item_quantity": "3", - "item_unit_price": { - "currency_code": "USD", - "value": "100.00" - }, - "item_amount": { - "currency_code": "USD", - "value": "300.00" - }, - "tax_amounts": [ - { - "tax_amount": { - "currency_code": "USD", - "value": "60.00" - } - } - ], - "total_item_amount": { - "currency_code": "USD", - "value": "360.00" - }, - "invoice_number": "Invoice-005" - }, - { - "item_name": "3", - "item_quantity": "1", - "item_unit_price": { - "currency_code": "USD", - "value": "-50.00" - }, - "item_amount": { - "currency_code": "USD", - "value": "-50.00" - }, - "total_item_amount": { - "currency_code": "USD", - "value": "-50.00" - }, - "invoice_number": "Invoice-005" - } - ] - }, - "store_info": {}, - "auction_info": {}, - "incentive_info": {} - } - ], - "account_number": "XZXSPECPDZHZU", - "last_refreshed_datetime": "2017-01-02T06:59:59+0000", - "page": 1, - "total_items": 1, - "total_pages": 1, - "links": [ - { - "href": "https://api-m.sandbox.paypal.com/v1/reporting/transactions?transaction_id=5TY05013RG002845M&fields=all&page_size=100&page=1", - "rel": "self", - "method": "GET" - } - ] -} diff --git a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py deleted file mode 100644 index 9f92bc3d97ca..000000000000 --- a/airbyte-integrations/connectors/source-paypal-transaction/unit_tests/unit_test.py +++ /dev/null @@ -1,352 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from datetime import datetime, timedelta - -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http.auth import NoAuth -from dateutil.parser import isoparse -from pytest import raises -from source_paypal_transaction.source import Balances, PaypalTransactionStream, Transactions - - -def test_minimum_allowed_start_date(): - start_date = now() - timedelta(days=10 * 365) - stream = Transactions(authenticator=NoAuth(), start_date=start_date) - assert stream.start_date != start_date - - -def test_transactions_transform_function(): - start_date = now() - timedelta(days=10 * 365) - stream = Transactions(authenticator=NoAuth(), start_date=start_date) - transformer = stream.transformer - input_data = {"transaction_amount": "123.45", "transaction_id": "111", "transaction_status": "done"} - schema = stream.get_json_schema() - schema["properties"] = { - "transaction_amount": {"type": "number"}, - "transaction_id": {"type": "integer"}, - "transaction_status": {"type": "string"}, - } - transformer.transform(input_data, schema) - expected_data = {"transaction_amount": 123.45, "transaction_id": 111, "transaction_status": "done"} - assert input_data == expected_data - - -def test_get_field(): - record = {"a": {"b": {"c": "d"}}} - # Test expected result - field_path is a list - assert "d" == PaypalTransactionStream.get_field(record, field_path=["a", "b", "c"]) - # Test expected result - field_path is a string - assert {"b": {"c": "d"}} == PaypalTransactionStream.get_field(record, field_path="a") - - # Test failures - not existing field_path - assert None is PaypalTransactionStream.get_field(record, field_path=["a", "b", "x"]) - assert None is PaypalTransactionStream.get_field(record, field_path=["a", "x", "x"]) - assert None is PaypalTransactionStream.get_field(record, field_path=["x", "x", "x"]) - - # Test failures - incorrect record structure - record = {"a": [{"b": {"c": "d"}}]} - assert None is PaypalTransactionStream.get_field(record, field_path=["a", "b", "c"]) - - record = {"a": {"b": "c"}} - assert None is PaypalTransactionStream.get_field(record, field_path=["a", "b", "c"]) - - record = {} - assert None is PaypalTransactionStream.get_field(record, field_path=["a", "b", "c"]) - - -def test_update_field(): - # Test success 1 - record = {"a": {"b": {"c": "d"}}} - PaypalTransactionStream.update_field(record, field_path=["a", "b", "c"], update=lambda x: x.upper()) - assert record == {"a": {"b": {"c": "D"}}} - - # Test success 2 - record = {"a": {"b": {"c": "d"}}} - PaypalTransactionStream.update_field(record, field_path="a", update=lambda x: "updated") - assert record == {"a": "updated"} - - # Test failure - incorrect field_path - record = {"a": {"b": {"c": "d"}}} - PaypalTransactionStream.update_field(record, field_path=["a", "b", "x"], update=lambda x: x.upper()) - assert record == {"a": {"b": {"c": "d"}}} - - # Test failure - incorrect field_path - record = {"a": {"b": {"c": "d"}}} - PaypalTransactionStream.update_field(record, field_path=["a", "x", "x"], update=lambda x: x.upper()) - assert record == {"a": {"b": {"c": "d"}}} - - -def now(): - return datetime.now().replace(microsecond=0).astimezone() - - -def test_transactions_stream_slices(): - start_date_max = {"hours": 0} - - # if start_date > now - **start_date_max then no slices - transactions = Transactions( - authenticator=NoAuth(), - start_date=now() - timedelta(**start_date_max) - timedelta(minutes=2), - ) - transactions.get_last_refreshed_datetime = lambda x: None - stream_slices = transactions.stream_slices(sync_mode="any") - assert 1 == len(stream_slices) - - # start_date <= now - **start_date_max - transactions = Transactions( - authenticator=NoAuth(), - start_date=now() - timedelta(**start_date_max), - ) - transactions.get_last_refreshed_datetime = lambda x: None - stream_slices = transactions.stream_slices(sync_mode="any") - assert 1 == len(stream_slices) - - transactions = Transactions( - authenticator=NoAuth(), - start_date=now() - timedelta(**start_date_max) + timedelta(minutes=2), - ) - transactions.get_last_refreshed_datetime = lambda x: None - stream_slices = transactions.stream_slices(sync_mode="any") - assert 1 == len(stream_slices) - - transactions = Transactions( - authenticator=NoAuth(), - start_date=now() - timedelta(**start_date_max) - timedelta(hours=2), - ) - transactions.get_last_refreshed_datetime = lambda x: None - stream_slices = transactions.stream_slices(sync_mode="any") - assert 1 == len(stream_slices) - - transactions = Transactions( - authenticator=NoAuth(), - start_date=now() - timedelta(**start_date_max) - timedelta(days=1), - ) - transactions.get_last_refreshed_datetime = lambda x: None - transactions.stream_slice_period = {"days": 1} - stream_slices = transactions.stream_slices(sync_mode="any") - assert 2 == len(stream_slices) - - transactions = Transactions( - authenticator=NoAuth(), - start_date=now() - timedelta(**start_date_max) - timedelta(days=1, hours=2), - ) - transactions.get_last_refreshed_datetime = lambda x: None - transactions.stream_slice_period = {"days": 1} - stream_slices = transactions.stream_slices(sync_mode="any") - assert 2 == len(stream_slices) - - transactions = Transactions( - authenticator=NoAuth(), - start_date=now() - timedelta(**start_date_max) - timedelta(days=30, minutes=1), - ) - transactions.get_last_refreshed_datetime = lambda x: None - transactions.stream_slice_period = {"days": 1} - stream_slices = transactions.stream_slices(sync_mode="any") - assert 31 == len(stream_slices) - - # tests with specified end_date - transactions = Transactions( - authenticator=NoAuth(), - start_date=isoparse("2021-06-01T10:00:00+00:00"), - end_date=isoparse("2021-06-04T12:00:00+00:00"), - ) - transactions.get_last_refreshed_datetime = lambda x: None - transactions.stream_slice_period = {"days": 1} - stream_slices = transactions.stream_slices(sync_mode="any") - assert [ - {"start_date": "2021-06-01T10:00:00+00:00", "end_date": "2021-06-02T10:00:00+00:00"}, - {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, - {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-04T10:00:00+00:00"}, - {"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}, - ] == stream_slices - - # tests with specified end_date and stream_state - transactions = Transactions( - authenticator=NoAuth(), - start_date=isoparse("2021-06-01T10:00:00+00:00"), - end_date=isoparse("2021-06-04T12:00:00+00:00"), - ) - transactions.get_last_refreshed_datetime = lambda x: None - transactions.stream_slice_period = {"days": 1} - stream_slices = transactions.stream_slices(sync_mode="any", stream_state={"date": "2021-06-02T10:00:00+00:00"}) - assert [ - {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, - {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-04T10:00:00+00:00"}, - {"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}, - ] == stream_slices - - transactions = Transactions( - authenticator=NoAuth(), - start_date=isoparse("2021-06-01T10:00:00+00:00"), - end_date=isoparse("2021-06-04T12:00:00+00:00"), - ) - transactions.get_last_refreshed_datetime = lambda x: None - stream_slices = transactions.stream_slices(sync_mode="any", stream_state={"date": "2021-06-04T10:00:00+00:00"}) - assert [{"start_date": "2021-06-04T10:00:00+00:00", "end_date": "2021-06-04T12:00:00+00:00"}] == stream_slices - - -def test_balances_stream_slices(): - """Test slices for Balance stream. - Note that is not used by this stream. - """ - now = datetime.now().replace(microsecond=0).astimezone() - - # Test without end_date (it equal by default) - balance = Balances(authenticator=NoAuth(), start_date=now) - balance.get_last_refreshed_datetime = lambda x: None - stream_slices = balance.stream_slices(sync_mode="any") - assert 1 == len(stream_slices) - - balance = Balances(authenticator=NoAuth(), start_date=now - timedelta(minutes=1)) - balance.get_last_refreshed_datetime = lambda x: None - stream_slices = balance.stream_slices(sync_mode="any") - assert 1 == len(stream_slices) - - balance = Balances( - authenticator=NoAuth(), - start_date=now - timedelta(hours=23), - ) - balance.get_last_refreshed_datetime = lambda x: None - stream_slices = balance.stream_slices(sync_mode="any") - assert 1 == len(stream_slices) - - balance = Balances( - authenticator=NoAuth(), - start_date=now - timedelta(days=1), - ) - balance.get_last_refreshed_datetime = lambda x: None - balance.stream_slice_period = {"days": 1} - stream_slices = balance.stream_slices(sync_mode="any") - assert 2 == len(stream_slices) - - balance = Balances( - authenticator=NoAuth(), - start_date=now - timedelta(days=1, minutes=1), - ) - balance.get_last_refreshed_datetime = lambda x: None - balance.stream_slice_period = {"days": 1} - stream_slices = balance.stream_slices(sync_mode="any") - assert 2 == len(stream_slices) - - # test with custom end_date - balance = Balances( - authenticator=NoAuth(), - start_date=isoparse("2021-06-01T10:00:00+00:00"), - end_date=isoparse("2021-06-03T12:00:00+00:00"), - ) - balance.get_last_refreshed_datetime = lambda x: None - balance.stream_slice_period = {"days": 1} - stream_slices = balance.stream_slices(sync_mode="any") - assert [ - {"start_date": "2021-06-01T10:00:00+00:00", "end_date": "2021-06-02T10:00:00+00:00"}, - {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, - {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}, - ] == stream_slices - - # Test with stream state - balance = Balances( - authenticator=NoAuth(), - start_date=isoparse("2021-06-01T10:00:00+00:00"), - end_date=isoparse("2021-06-03T12:00:00+00:00"), - ) - balance.get_last_refreshed_datetime = lambda x: None - balance.stream_slice_period = {"days": 1} - stream_slices = balance.stream_slices(sync_mode="any", stream_state={"date": "2021-06-02T10:00:00+00:00"}) - assert [ - {"start_date": "2021-06-02T10:00:00+00:00", "end_date": "2021-06-03T10:00:00+00:00"}, - {"start_date": "2021-06-03T10:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}, - ] == stream_slices - - balance = Balances( - authenticator=NoAuth(), - start_date=isoparse("2021-06-01T10:00:00+00:00"), - end_date=isoparse("2021-06-03T12:00:00+00:00"), - ) - balance.get_last_refreshed_datetime = lambda x: None - balance.stream_slice_period = {"days": 1} - stream_slices = balance.stream_slices(sync_mode="any", stream_state={"date": "2021-06-03T11:00:00+00:00"}) - assert [{"start_date": "2021-06-03T11:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}] == stream_slices - - balance = Balances( - authenticator=NoAuth(), - start_date=isoparse("2021-06-01T10:00:00+00:00"), - end_date=isoparse("2021-06-03T12:00:00+00:00"), - ) - balance.get_last_refreshed_datetime = lambda x: None - balance.stream_slice_period = {"days": 1} - stream_slices = balance.stream_slices(sync_mode="any", stream_state={"date": "2021-06-03T12:00:00+00:00"}) - assert [{"start_date": "2021-06-03T12:00:00+00:00", "end_date": "2021-06-03T12:00:00+00:00"}] == stream_slices - - -def test_max_records_in_response_reached(transactions, requests_mock): - balance = Transactions( - authenticator=NoAuth(), - start_date=isoparse("2021-07-01T10:00:00+00:00"), - end_date=isoparse("2021-07-29T12:00:00+00:00"), - ) - error_message = { - "name": "RESULTSET_TOO_LARGE", - "message": "Result set size is greater than the maximum limit. Change the filter " "criteria and try again.", - } - url = "https://api-m.paypal.com/v1/reporting/transactions" - - requests_mock.register_uri( - "GET", - url + "?start_date=2021-07-01T12%3A00%3A00%2B00%3A00&end_date=2021-07-29T12%3A00%3A00%2B00%3A00", - json=error_message, - status_code=400, - ) - requests_mock.register_uri( - "GET", url + "?start_date=2021-07-01T12%3A00%3A00%2B00%3A00&end_date=2021-07-15T12%3A00%3A00%2B00%3A00", json=transactions - ) - requests_mock.register_uri( - "GET", url + "?start_date=2021-07-15T12%3A00%3A00%2B00%3A00&end_date=2021-07-29T12%3A00%3A00%2B00%3A00", json=transactions - ) - month_date_slice = {"start_date": "2021-07-01T12:00:00+00:00", "end_date": "2021-07-29T12:00:00+00:00"} - assert len(list(balance.read_records(sync_mode="any", stream_slice=month_date_slice))) == 2 - - requests_mock.register_uri( - "GET", - url + "?start_date=2021-07-01T12%3A00%3A00%2B00%3A00&end_date=2021-07-01T12%3A00%3A00%2B00%3A00", - json=error_message, - status_code=400, - ) - one_day_slice = {"start_date": "2021-07-01T12:00:00+00:00", "end_date": "2021-07-01T12:00:00+00:00"} - with raises(Exception): - assert next(balance.read_records(sync_mode="any", stream_slice=one_day_slice)) - - -def test_unnest_field(): - record = {"transaction_info": {"transaction_id": "123", "transaction_initiation_date": "2014-07-11T04:03:52+0000"}} - # check the cursor is not on the root level - assert Transactions.cursor_field not in record.keys() - - PaypalTransactionStream.unnest_field(record, Transactions.nested_object, Transactions.cursor_field) - # check the cursor now on the root level - assert Transactions.cursor_field in record.keys() - - -def test_get_last_refreshed_datetime(requests_mock, prod_config, api_endpoint): - stream = Balances(authenticator=NoAuth(), **prod_config) - requests_mock.post(f"{api_endpoint}/v1/oauth2/token", json={"access_token": "test_access_token", "expires_in": 12345}) - url = f"{api_endpoint}/v1/reporting/balances" + "?as_of_time=2021-07-01T00%3A00%3A00%2B00%3A00" - requests_mock.get(url, json={}) - assert not stream.get_last_refreshed_datetime(SyncMode.full_refresh) - - -def test_get_updated_state(transactions): - start_date = "2021-06-01T10:00:00+00:00" - stream = Transactions( - authenticator=NoAuth(), - start_date=isoparse(start_date), - end_date=isoparse("2021-06-04T12:00:00+00:00"), - ) - state = stream.get_updated_state(current_stream_state={}, latest_record={}) - assert state == {"date": start_date} - - record = transactions[stream.data_field][0][stream.nested_object] - expected_state = {"date": now().isoformat()} - state = stream.get_updated_state(current_stream_state=expected_state, latest_record=record) - assert state == expected_state diff --git a/airbyte-integrations/connectors/source-paystack/README.md b/airbyte-integrations/connectors/source-paystack/README.md index 0b76ca015f1f..5717838e9160 100644 --- a/airbyte-integrations/connectors/source-paystack/README.md +++ b/airbyte-integrations/connectors/source-paystack/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-paystack:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/paystack) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_paystack/spec.json` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-paystack:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-paystack build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-paystack:airbyteDocker +An image will be built with the tag `airbyte/source-paystack:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-paystack:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-paystack:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-paystack:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-paystack:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-paystack test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-paystack:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-paystack:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-paystack test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/paystack.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-paystack/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-paystack/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-paystack/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-paystack/build.gradle b/airbyte-integrations/connectors/source-paystack/build.gradle deleted file mode 100644 index 481521f515be..000000000000 --- a/airbyte-integrations/connectors/source-paystack/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_paystack' -} diff --git a/airbyte-integrations/connectors/source-pendo/README.md b/airbyte-integrations/connectors/source-pendo/README.md index 58a72550ccf8..a924c66c4f8e 100644 --- a/airbyte-integrations/connectors/source-pendo/README.md +++ b/airbyte-integrations/connectors/source-pendo/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-pendo:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/pendo) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_pendo/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-pendo:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-pendo build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-pendo:airbyteDocker +An image will be built with the tag `airbyte/source-pendo:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-pendo:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pendo:dev check --conf docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pendo:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-pendo:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-pendo test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-pendo:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-pendo:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-pendo test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/pendo.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-pendo/acceptance-test-config.yml b/airbyte-integrations/connectors/source-pendo/acceptance-test-config.yml index 2e21086f17f3..67c768793f8e 100644 --- a/airbyte-integrations/connectors/source-pendo/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-pendo/acceptance-test-config.yml @@ -19,20 +19,20 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.jsonl" -# extra_fields: no -# exact_order: no -# extra_records: yes - incremental: + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.jsonl" + # extra_fields: no + # exact_order: no + # extra_records: yes + incremental: bypass_reason: "This connector does not implement incremental sync" -# TODO uncomment this block this block if your connector implements incremental sync: -# tests: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# future_state: -# future_state_path: "integration_tests/abnormal_state.json" + # TODO uncomment this block this block if your connector implements incremental sync: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-pendo/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-pendo/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-pendo/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-pendo/build.gradle b/airbyte-integrations/connectors/source-pendo/build.gradle deleted file mode 100644 index d4f5bbd08957..000000000000 --- a/airbyte-integrations/connectors/source-pendo/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_pendo' -} diff --git a/airbyte-integrations/connectors/source-persistiq/Dockerfile b/airbyte-integrations/connectors/source-persistiq/Dockerfile index f995b5438c8e..44ea78068a13 100644 --- a/airbyte-integrations/connectors/source-persistiq/Dockerfile +++ b/airbyte-integrations/connectors/source-persistiq/Dockerfile @@ -34,5 +34,5 @@ COPY source_persistiq ./source_persistiq ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-persistiq diff --git a/airbyte-integrations/connectors/source-persistiq/README.md b/airbyte-integrations/connectors/source-persistiq/README.md index e7c3a64b440d..0a4bbfb8c9e5 100644 --- a/airbyte-integrations/connectors/source-persistiq/README.md +++ b/airbyte-integrations/connectors/source-persistiq/README.md @@ -1,75 +1,34 @@ # Persistiq Source -This is the repository for the Persistiq source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/persistiq). +This is the repository for the Persistiq configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/persistiq). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-persistiq:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/persistiq) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_persistiq/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/persistiq) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_persistiq/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. -To obtain credentials, create an account on PersistIq and follow the [documentation](https://apidocs.persistiq.com/#authentication) **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source persistiq test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-persistiq:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-persistiq build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-persistiq:airbyteDocker +An image will be built with the tag `airbyte/source-persistiq:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-persistiq:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -79,44 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-persistiq:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-persistiq:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-persistiq:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-persistiq test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-persistiq:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-persistiq:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -126,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-persistiq test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/persistiq.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-persistiq/__init__.py b/airbyte-integrations/connectors/source-persistiq/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-persistiq/acceptance-test-config.yml b/airbyte-integrations/connectors/source-persistiq/acceptance-test-config.yml index 167e0b33151c..913296e0e8e5 100644 --- a/airbyte-integrations/connectors/source-persistiq/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-persistiq/acceptance-test-config.yml @@ -1,23 +1,39 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-persistiq:dev -tests: +acceptance_tests: spec: - - spec_path: "source_persistiq/spec.json" + tests: + - spec_path: "source_persistiq/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] - expect_records: - path: "integration_tests/expected_campaigns_stream.txt" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.jsonl" + # extra_fields: no + # exact_order: no + # extra_records: yes + incremental: + bypass_reason: "This connector does not implement incremental sync" + # TODO uncomment this block this block if your connector implements incremental sync: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-persistiq/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-persistiq/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-persistiq/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-persistiq/bootstrap.md b/airbyte-integrations/connectors/source-persistiq/bootstrap.md deleted file mode 100644 index c4f0a41ca68a..000000000000 --- a/airbyte-integrations/connectors/source-persistiq/bootstrap.md +++ /dev/null @@ -1,18 +0,0 @@ -# PersistIq -PersistIq is an outbound automation tool designed for small teams to find, reach, and organize customers all in one simple platform. - -## Streams - -This Source is capable of syncing the following streams: -* [Users](https://apidocs.persistiq.com/#users) -* [Leads](https://apidocs.persistiq.com/#leads) -* [Campaigns](https://apidocs.persistiq.com/#campaigns) - -### Incremental streams -Incremental streams were not implemented in the initial version. - -### Next steps -Implement incremental sync and additional streams (`Lead status`, `Lead fields`, `Events`). - -### Rate limits -The API rate limit is at 100 requests/minutes. Read [Rate Limits](https://apidocs.persistiq.com/#error-codes) for more informations. diff --git a/airbyte-integrations/connectors/source-persistiq/build.gradle b/airbyte-integrations/connectors/source-persistiq/build.gradle deleted file mode 100644 index f7d40b3c8513..000000000000 --- a/airbyte-integrations/connectors/source-persistiq/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_persistiq' -} diff --git a/airbyte-integrations/connectors/source-persistiq/integration_tests/__init__.py b/airbyte-integrations/connectors/source-persistiq/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-persistiq/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-persistiq/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-persistiq/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-persistiq/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-persistiq/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-persistiq/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-persistiq/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-persistiq/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-persistiq/integration_tests/catalog.json b/airbyte-integrations/connectors/source-persistiq/integration_tests/catalog.json deleted file mode 100644 index 3a8f681f82c7..000000000000 --- a/airbyte-integrations/connectors/source-persistiq/integration_tests/catalog.json +++ /dev/null @@ -1,195 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "leads", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "bounced": { - "type": ["null", "boolean"] - }, - "owner_id": { - "type": ["null", "string"] - }, - "optedout": { - "type": ["null", "boolean"] - }, - "sent_count": { - "type": ["null", "integer"] - }, - "replied_count": { - "type": ["null", "integer"] - }, - "last_sent_at": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "string"] - }, - "data": { - "company_name": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"], - "format": "email" - }, - "first_name": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "address": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "industry": { - "type": ["null", "string"] - }, - "snippet": { - "type": ["null", "string"] - }, - "snippet1": { - "type": ["null", "string"] - }, - "snippet2": { - "type": ["null", "string"] - }, - "snippet3": { - "type": ["null", "string"] - }, - "snippet4": { - "type": ["null", "string"] - }, - "twitch_name": { - "type": ["null", "string"] - }, - "linkedin": { - "type": ["null", "string"] - }, - "twitter": { - "type": ["null", "string"] - }, - "facebook": { - "type": ["null", "string"] - }, - "salesforce_id": { - "type": ["null", "string"] - } - } - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]], - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "campaigns", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "creator": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"] - } - }, - "stats": { - "prospects_contacted": { - "type": ["null", "integer"] - }, - "prospects_reached": { - "type": ["null", "integer"] - }, - "prospects_opened": { - "type": ["null", "integer"] - }, - "prospects_replied": { - "type": ["null", "integer"] - }, - "prospects_bounced": { - "type": ["null", "integer"] - }, - "prospects_optedout": { - "type": ["null", "integer"] - }, - "total_contacted": { - "type": ["null", "integer"] - } - } - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]], - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "users", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "email": { - "type": ["null", "string"], - "format": "email" - }, - "activated": { - "type": ["null", "boolean"] - }, - "default_mailbox_id": { - "type": ["null", "string"] - }, - "salesforce_id": { - "type": ["null", "string"] - } - } - } - }, - "supported_sync_modes": ["full_refresh"], - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "source_defined_primary_key": [["id"]] - } - ] -} diff --git a/airbyte-integrations/connectors/source-persistiq/integration_tests/expected_campaigns_stream.txt b/airbyte-integrations/connectors/source-persistiq/integration_tests/expected_campaigns_stream.txt deleted file mode 100644 index 47c985a587c9..000000000000 --- a/airbyte-integrations/connectors/source-persistiq/integration_tests/expected_campaigns_stream.txt +++ /dev/null @@ -1,3 +0,0 @@ -{"stream": "campaigns", "data": {"id": "c_3kqMZZ","name": "Schedule Meetings with Reps (sample campaign)","creator": {"id": "u_pdKWk3", "name": "Sherif Nada", "email": "integration-test@airbyte.io"},"stats": {"prospects_contacted": 0, "prospects_reached": 0, "prospects_opened": 0, "prospects_replied": 0, "prospects_bounced": 0, "prospects_optedout": 0, "total_prospects": 3}}, "emitted_at": 1629119628000} -{"stream": "campaigns", "data": {"id": "c_ljDgZB","name": "Schedule Meetings with Managers (sample campaign)","creator": {"id": "u_pdKWk3", "name": "Sherif Nada", "email": "integration-test@airbyte.io"},"stats": {"prospects_contacted": 0, "prospects_reached": 0, "prospects_opened": 0, "prospects_replied": 0, "prospects_bounced": 0, "prospects_optedout": 0, "total_prospects": 3}}, "emitted_at": 1629119628000} -{"stream": "campaigns", "data": {"id": "c_3e01Kb","name": "Schedule Meetings with CEOs (sample campaign)","creator": {"id": "u_pdKWk3", "name": "Sherif Nada", "email": "integration-test@airbyte.io"},"stats": {"prospects_contacted": 0, "prospects_reached": 0, "prospects_opened": 0, "prospects_replied": 0, "prospects_bounced": 0, "prospects_optedout": 0, "total_prospects": 3}}, "emitted_at": 1629119628000} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-persistiq/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-persistiq/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-persistiq/metadata.yaml b/airbyte-integrations/connectors/source-persistiq/metadata.yaml index b518b28eb03c..2d06105f0138 100644 --- a/airbyte-integrations/connectors/source-persistiq/metadata.yaml +++ b/airbyte-integrations/connectors/source-persistiq/metadata.yaml @@ -1,24 +1,25 @@ data: + allowedHosts: + hosts: + - api.persistiq.com + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: 3052c77e-8b91-47e2-97a0-a29a22794b4b - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-persistiq githubIssueLabel: source-persistiq icon: persistiq.svg license: MIT name: PersistIq - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: 2023-10-04 releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/persistiq tags: - - language:python - ab_internal: - sl: 100 - ql: 100 - supportLevel: community + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-persistiq/requirements.txt b/airbyte-integrations/connectors/source-persistiq/requirements.txt index d6e1198b1ab1..cc57334ef619 100644 --- a/airbyte-integrations/connectors/source-persistiq/requirements.txt +++ b/airbyte-integrations/connectors/source-persistiq/requirements.txt @@ -1 +1,2 @@ +-e ../../bases/connector-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-persistiq/setup.py b/airbyte-integrations/connectors/source-persistiq/setup.py index 0c77971c1a87..88f883fbdacf 100644 --- a/airbyte-integrations/connectors/source-persistiq/setup.py +++ b/airbyte-integrations/connectors/source-persistiq/setup.py @@ -6,14 +6,13 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", - "pytest~=6.1", + "pytest~=6.2", "pytest-mock~=3.6.1", - "requests_mock==1.8.0", ] setup( @@ -23,7 +22,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-persistiq/source_persistiq/__init__.py b/airbyte-integrations/connectors/source-persistiq/source_persistiq/__init__.py index a3923d95bbe8..663a63d5f0d9 100644 --- a/airbyte-integrations/connectors/source-persistiq/source_persistiq/__init__.py +++ b/airbyte-integrations/connectors/source-persistiq/source_persistiq/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-persistiq/source_persistiq/manifest.yaml b/airbyte-integrations/connectors/source-persistiq/source_persistiq/manifest.yaml new file mode 100644 index 000000000000..90b0dfe1ed36 --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/source_persistiq/manifest.yaml @@ -0,0 +1,73 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["{{ parameters.extractorPath }}"] + requester: + type: HttpRequester + url_base: "https://api.persistiq.com/v1/" + http_method: "GET" + authenticator: + type: NoAuth + request_headers: + x-api-key: "{{ config['api_key'] }}" + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ last_records['next_page'] }}" + page_token_option: + type: "RequestPath" + field_name: "page" + inject_into: "request_parameter" + requester: + $ref: "#/definitions/requester" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + users_stream: + $ref: "#/definitions/base_stream" + name: "users" + primary_key: "id" + $parameters: + extractorPath: "users" + path: "users" + + leads_stream: + $ref: "#/definitions/base_stream" + name: "leads" + primary_key: "id" + $parameters: + extractorPath: "leads" + path: "leads" + + campaigns_stream: + $ref: "#/definitions/base_stream" + name: "campaigns" + primary_key: "id" + $parameters: + extractorPath: "campaigns" + path: "campaigns" + +streams: + - "#/definitions/users_stream" + - "#/definitions/leads_stream" + - "#/definitions/campaigns_stream" + +check: + type: CheckStream + stream_names: + - "users" + - "leads" + - "campaigns" diff --git a/airbyte-integrations/connectors/source-persistiq/source_persistiq/schemas/leads.json b/airbyte-integrations/connectors/source-persistiq/source_persistiq/schemas/leads.json index cb6ba052a882..b1792076f6ec 100644 --- a/airbyte-integrations/connectors/source-persistiq/source_persistiq/schemas/leads.json +++ b/airbyte-integrations/connectors/source-persistiq/source_persistiq/schemas/leads.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "id": { "type": ["string"] @@ -11,6 +12,9 @@ "owner_id": { "type": ["string"] }, + "creator_id": { + "type": ["null", "string"] + }, "optedout": { "type": ["null", "boolean"] }, diff --git a/airbyte-integrations/connectors/source-persistiq/source_persistiq/source.py b/airbyte-integrations/connectors/source-persistiq/source_persistiq/source.py index 9c0bb9d33849..dfeb8d2f8621 100644 --- a/airbyte-integrations/connectors/source-persistiq/source_persistiq/source.py +++ b/airbyte-integrations/connectors/source-persistiq/source_persistiq/source.py @@ -2,95 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import NoAuth +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -# Basic full refresh stream +WARNING: Do not modify this file. +""" -class PersistiqStream(HttpStream, ABC): - def __init__(self, api_key: str, **kwargs): - super().__init__(**kwargs) - self.api_key = api_key - - url_base = "https://api.persistiq.com/v1/" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - json_response = response.json() - if not json_response.get("has_more", False): - return None - - return {"page": json_response.get("next_page")[-1]} - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return {"page": 1 if not next_page_token else next_page_token["page"]} - - def request_headers(self, **kwargs) -> MutableMapping[str, Any]: - return {"x-api-key": self.api_key} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield response.json() - - -class Users(PersistiqStream): - primary_key = "id" - - def path(self, **kwargs) -> str: - return "users" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - json = response.json() - yield from json["users"] - - -class Leads(PersistiqStream): - primary_key = "id" - - def path(self, **kwargs) -> str: - return "leads" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - json = response.json() - yield from json["leads"] - - -class Campaigns(PersistiqStream): - primary_key = "id" - - def path(self, **kwargs) -> str: - return "campaigns" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - json = response.json() - yield from json["campaigns"] - - -# Source - - -class SourcePersistiq(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - headers = {"x-api-key": config["api_key"]} - url = "https://api.persistiq.com/v1/users" - try: - response = requests.get(url, headers=headers) - response.raise_for_status() - return True, None - except requests.exceptions.RequestException as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - auth = NoAuth() - return [ - Users(authenticator=auth, api_key=config["api_key"]), - Leads(authenticator=auth, api_key=config["api_key"]), - Campaigns(authenticator=auth, api_key=config["api_key"]), - ] +# Declarative Source +class SourcePersistiq(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-persistiq/source_persistiq/spec.json b/airbyte-integrations/connectors/source-persistiq/source_persistiq/spec.json deleted file mode 100644 index 3c5b18ac0ba9..000000000000 --- a/airbyte-integrations/connectors/source-persistiq/source_persistiq/spec.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/persistiq", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Persistiq Spec", - "type": "object", - "required": ["api_key"], - "additionalProperties": false, - "properties": { - "api_key": { - "type": "string", - "description": "PersistIq API Key. See the docs for more information on where to find that key.", - "airbyte_secret": true - } - } - } -} diff --git a/airbyte-integrations/connectors/source-persistiq/source_persistiq/spec.yaml b/airbyte-integrations/connectors/source-persistiq/source_persistiq/spec.yaml new file mode 100644 index 000000000000..b64d9bcbe625 --- /dev/null +++ b/airbyte-integrations/connectors/source-persistiq/source_persistiq/spec.yaml @@ -0,0 +1,15 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/persistiq +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Persistiq Spec + type: object + required: + - api_key + additionalProperties: true + properties: + api_key: + type: string + description: + PersistIq API Key. See the docs + for more information on where to find that key. + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-persistiq/unit_tests/__init__.py b/airbyte-integrations/connectors/source-persistiq/unit_tests/__init__.py deleted file mode 100644 index 46b7376756ec..000000000000 --- a/airbyte-integrations/connectors/source-persistiq/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-persistiq/unit_tests/test_source.py b/airbyte-integrations/connectors/source-persistiq/unit_tests/test_source.py deleted file mode 100644 index c6ccf45c6c07..000000000000 --- a/airbyte-integrations/connectors/source-persistiq/unit_tests/test_source.py +++ /dev/null @@ -1,38 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import requests -from source_persistiq.source import SourcePersistiq - - -def test_check_connection(mocker, requests_mock): - source = SourcePersistiq() - mock_logger = mocker.Mock() - test_config = {"api_key": "mybeautifulkey"} - # success - requests_mock.get( - "https://api.persistiq.com/v1/users", - json={ - "id": "u_3an2Jp", - "name": "Gabriel Rossmann", - "email": "gabriel@punctual.cc", - "activated": "true", - "default_mailbox_id": "mbox_38ymEp", - "salesforce_id": "", - }, - ) - assert source.check_connection(mock_logger, test_config) == (True, None) - - # failure - requests_mock.get("https://api.persistiq.com/v1/users", status_code=500) - connection_success, connection_failure = source.check_connection(mock_logger, test_config) - assert not connection_success - assert isinstance(connection_failure, requests.exceptions.HTTPError) - - -def test_streams(): - source = SourcePersistiq() - config = {"api_key": "my-api-key"} - streams = source.streams(config) - assert len(streams) == 3 diff --git a/airbyte-integrations/connectors/source-persistiq/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-persistiq/unit_tests/test_streams.py deleted file mode 100644 index 04722fe6ad43..000000000000 --- a/airbyte-integrations/connectors/source-persistiq/unit_tests/test_streams.py +++ /dev/null @@ -1,64 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pytest -from source_persistiq.source import PersistiqStream - - -def mocked_requests_get(*args, **kwargs): - class MockResponse: - def __init__(self, json_data, status_code): - self.json_data = json_data - self.status_code = status_code - - def json(self): - return self.json_data - - return MockResponse(json_data=kwargs["json_data"], status_code=kwargs["status_code"]) - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(PersistiqStream, "path", "v0/example_endpoint") - mocker.patch.object(PersistiqStream, "primary_key", "test_primary_key") - mocker.patch.object(PersistiqStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = PersistiqStream(api_key="mybeautifulkey") - inputs = {"next_page_token": {"page": 1}} - expected_params = {"page": 1} - assert stream.request_params(stream_state=None, **inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = PersistiqStream(api_key="mybeautifulkey") - # With next page - response = mocked_requests_get(json_data={"has_more": True, "next_page": "https://api.persistiq.com/v1/users?page=2"}, status_code=200) - expected_token = "2" - assert stream.next_page_token(response=response) == {"page": expected_token} - # Without next page - response = mocked_requests_get(json_data={}, status_code=200) - expected_token = None - assert stream.next_page_token(response=response) == expected_token - - -def test_parse_response(patch_base_class): - stream = PersistiqStream(api_key="mybeautifulkey") - response = mocked_requests_get(json_data={"users": [{"id": 1, "name": "John Doe"}]}, status_code=200) - expected_parsed_object = {"users": [{"id": 1, "name": "John Doe"}]} - assert next(stream.parse_response(response=response)) == expected_parsed_object - - -def test_request_headers(patch_base_class): - stream = PersistiqStream(api_key="mybeautifulkey") - expected_headers = {"x-api-key": "mybeautifulkey"} - assert stream.request_headers() == expected_headers - - -def test_http_method(patch_base_class): - stream = PersistiqStream(api_key="mybeautifulkey") - expected_method = "GET" - assert stream.http_method == expected_method diff --git a/airbyte-integrations/connectors/source-pexels-api/README.md b/airbyte-integrations/connectors/source-pexels-api/README.md index 37f00ad7940b..bb20c4f5d538 100644 --- a/airbyte-integrations/connectors/source-pexels-api/README.md +++ b/airbyte-integrations/connectors/source-pexels-api/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-pexels-api:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/pexels-api) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_pexels_api/spec.yaml` file. @@ -48,18 +40,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-pexels-api:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-pexels-api build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-pexels-api:airbyteDocker +An image will be built with the tag `airbyte/source-pexels-api:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-pexels-api:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -69,25 +62,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pexels-api:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pexels-api:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-pexels-api:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-pexels-api test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-pexels-api:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-pexels-api:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -96,8 +81,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-pexels-api test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/pexels-api.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-pexels-api/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-pexels-api/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-pexels-api/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-pexels-api/build.gradle b/airbyte-integrations/connectors/source-pexels-api/build.gradle deleted file mode 100644 index b03d71ed91d6..000000000000 --- a/airbyte-integrations/connectors/source-pexels-api/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_pexels_api' -} diff --git a/airbyte-integrations/connectors/source-pinterest/Dockerfile b/airbyte-integrations/connectors/source-pinterest/Dockerfile deleted file mode 100644 index a058bae4ec48..000000000000 --- a/airbyte-integrations/connectors/source-pinterest/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_pinterest ./source_pinterest - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.6.0 -LABEL io.airbyte.name=airbyte/source-pinterest diff --git a/airbyte-integrations/connectors/source-pinterest/README.md b/airbyte-integrations/connectors/source-pinterest/README.md index f7b174d7a0d3..e22d33de28b5 100644 --- a/airbyte-integrations/connectors/source-pinterest/README.md +++ b/airbyte-integrations/connectors/source-pinterest/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-pinterest:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/pinterest) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_pinterest/spec.json` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-pinterest:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-pinterest build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-pinterest:airbyteDocker +An image will be built with the tag `airbyte/source-pinterest:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-pinterest:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,45 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pinterest:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pinterest:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-pinterest:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-pinterest test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-pinterest:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-pinterest:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-pinterest:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -126,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-pinterest test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/pinterest.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-pinterest/acceptance-test-config.yml b/airbyte-integrations/connectors/source-pinterest/acceptance-test-config.yml index b9ebb48017b9..4eab013a5fad 100644 --- a/airbyte-integrations/connectors/source-pinterest/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-pinterest/acceptance-test-config.yml @@ -5,13 +5,15 @@ acceptance_tests: tests: - spec_path: source_pinterest/spec.json backward_compatibility_tests_config: - disable_for_version: "0.5.0" # Add Pattern for "start_date" + disable_for_version: "0.7.3" # added custom report + # disable_for_version: "0.7.0" # removed non-working token based auth method + # disable_for_version: "0.5.0" # Add Pattern for "start_date" connection: tests: - config_path: secrets/config.json status: succeed - config_path: integration_tests/invalid_config.json - status: exception + status: failed - config_path: secrets/config_oauth.json status: succeed discovery: @@ -22,26 +24,24 @@ acceptance_tests: tests: - config_path: secrets/config.json empty_streams: - - bypass_reason: The stream could return 0 records, because of low rate-limits - name: ad_account_analytics - - bypass_reason: The stream could return 0 records, because of low rate-limits - name: ad_analytics - - bypass_reason: The stream could return 0 records, because of low rate-limits - name: ad_group_analytics - - bypass_reason: The stream could return 0 records, because of low rate-limits - name: ad_groups - - bypass_reason: The stream could return 0 records, because of low rate-limits - name: ads - - bypass_reason: The stream could return 0 records, because of low rate-limits - name: board_section_pins - - bypass_reason: The stream could return 0 records, because of low rate-limits - name: board_sections - - bypass_reason: The stream could return 0 records, because of low rate-limits - name: campaign_analytics - - bypass_reason: The stream could return 0 records, because of low rate-limits - name: campaigns - - bypass_reason: The stream could return 0 records, because of low rate-limits - name: user_account_analytics + - name: conversion_tags + bypass_reason: Not possible to add data + - name: customer_lists + bypass_reason: Not possible to add data + - name: catalogs + bypass_reason: Not possible to add data + - name: catalogs_feeds + bypass_reason: Not possible to add data + - name: catalogs_product_groups + bypass_reason: Not possible to add data + - name: product_group_report + bypass_reason: Not possible to add data + - name: product_group_targeting_report + bypass_reason: Not possible to add data + - name: product_item_report + bypass_reason: Not possible to add data + - name: keyword_report + bypass_reason: Not possible to add data timeout_seconds: 1200 expect_records: path: "integration_tests/expected_records.jsonl" @@ -49,12 +49,36 @@ acceptance_tests: exact_order: no extra_records: yes fail_on_extra_columns: false + ignored_fields: + board_pins: + - name: "media" + bypass_reason: "urls may change" + board_section_pins: + - name: "media" + bypass_reason: "urls may change" + ads: + - name: "updated_time" + bypass_reason: "can be updated" + ad_groups: + - name: "updated_time" + bypass_reason: "can be updated" + campaigns: + - name: "updated_time" + bypass_reason: "can be updated" + audiences: + - name: "size" + bypass_reason: "can be changed" + - name: "updated_timestamp" + bypass_reason: "can be changed" + - name: "created_timestamp" + bypass_reason: "can be changed" incremental: tests: - config_path: secrets/config.json configured_catalog_path: integration_tests/configured_catalog.json future_state: future_state_path: integration_tests/abnormal_state.json + skip_comprehensive_incremental_tests: true full_refresh: tests: - config_path: secrets/config.json diff --git a/airbyte-integrations/connectors/source-pinterest/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-pinterest/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-pinterest/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-pinterest/build.gradle b/airbyte-integrations/connectors/source-pinterest/build.gradle deleted file mode 100644 index 313d7de77d91..000000000000 --- a/airbyte-integrations/connectors/source-pinterest/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_pinterest' -} diff --git a/airbyte-integrations/connectors/source-pinterest/icon.svg b/airbyte-integrations/connectors/source-pinterest/icon.svg index 2d30595c4025..aa9ac6080415 100644 --- a/airbyte-integrations/connectors/source-pinterest/icon.svg +++ b/airbyte-integrations/connectors/source-pinterest/icon.svg @@ -1 +1 @@ - \ No newline at end of file + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-pinterest/integration_tests/config_custom_report.json b/airbyte-integrations/connectors/source-pinterest/integration_tests/config_custom_report.json new file mode 100644 index 000000000000..c5a3f8de1b40 --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/integration_tests/config_custom_report.json @@ -0,0 +1,26 @@ +{ + "client_id": "1111111", + "client_secret": "XXXX", + "refresh_token": "XXXXX", + "start_date": "2023-01-08", + "custom_reports": [ + { + "name": "vadim_report", + "level": "AD_GROUP", + "granularity": "MONTH", + "click_window_days": 30, + "engagement_window_days": 30, + "view_window_days": 30, + "conversion_report_time": "TIME_OF_CONVERSION", + "attribution_types": ["INDIVIDUAL", "HOUSEHOLD"], + "columns": [ + "ADVERTISER_ID", + "AD_ACCOUNT_ID", + "AD_GROUP_ID", + "CTR", + "IMPRESSION_2" + ], + "start_date": "2023-01-08" + } + ] +} diff --git a/airbyte-integrations/connectors/source-pinterest/integration_tests/configured_catalog_custom_report.json b/airbyte-integrations/connectors/source-pinterest/integration_tests/configured_catalog_custom_report.json new file mode 100644 index 000000000000..18be63947eff --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/integration_tests/configured_catalog_custom_report.json @@ -0,0 +1,15 @@ +{ + "streams": [ + { + "stream": { + "name": "custom_vadim_report", + "json_schema": {}, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": [] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-pinterest/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-pinterest/integration_tests/expected_records.jsonl index 1e2f686ffc27..2d6f69c6364d 100644 --- a/airbyte-integrations/connectors/source-pinterest/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-pinterest/integration_tests/expected_records.jsonl @@ -1,4 +1,24 @@ {"stream": "ad_accounts", "data": {"id": "549761668032", "name": "Airbyte", "owner": {"username": "integrationtest0375", "id": "666744057242074926"}, "country": "US", "currency": "USD", "permissions": ["OWNER"], "created_time": 1603772920, "updated_time": 1623173784}, "emitted_at": 1688461289470} +{"stream": "ad_account_analytics", "data": {"TOTAL_IMPRESSION_FREQUENCY": 1.0, "TOTAL_IMPRESSION_USER": 1.0, "ADVERTISER_ID": "549761668032", "DATE": "2023-10-29", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032"}, "emitted_at": 1699893121669} +{"stream": "ads", "data": {"id": "687218400118", "ad_group_id": "2680068678965", "ad_account_id": "549761668032", "android_deep_link": null, "campaign_id": "626744128956", "carousel_android_deep_links": null, "carousel_destination_urls": null, "carousel_ios_deep_links": null, "click_tracking_url": null, "collection_items_destination_url_template": null, "created_time": 1623245885, "creative_type": "REGULAR", "destination_url": "https://airbyte.io/", "ios_deep_link": null, "is_pin_deleted": false, "is_removable": false, "name": "2021-06-09 | Traffic | Keywords | Data Integration", "pin_id": "666743919837294988", "rejected_reasons": [], "rejection_labels": [], "review_status": "APPROVED", "status": "PAUSED", "summary_status": "PAUSED", "tracking_urls": null, "type": "ad", "updated_time": 1699373013, "view_tracking_url": null, "lead_form_id": null}, "emitted_at": 1699393433303} +{"stream": "ad_analytics", "data": {"PIN_ID": 6.66743919837295e+17, "AD_GROUP_ID": "2680068678993", "AD_GROUP_ENTITY_STATUS": "1", "CAMPAIGN_ENTITY_STATUS": 1.0, "TOTAL_IMPRESSION_FREQUENCY": 1.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "TOTAL_IMPRESSION_USER": 1.0, "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "AD_ID": "687218400210", "ADVERTISER_ID": "549761668032", "PIN_PROMOTION_ID": 687218400210.0, "DATE": "2023-10-29", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness"}, "emitted_at": 1699893196846} +{"stream": "ad_groups", "data": {"id": "2680068678965", "created_time": 1623245885.0, "updated_time": 1699373013.0, "start_time": null, "end_time": null, "bid_in_micro_currency": null, "budget_in_micro_currency": null, "campaign_id": "626744128956", "ad_account_id": "549761668032", "auto_targeting_enabled": true, "type": "adgroup", "budget_type": "CBO_ADGROUP", "billable_event": "CLICKTHROUGH", "status": "ACTIVE", "lifetime_frequency_cap": -1.0, "targeting_spec": {"GENDER": ["female", "male", "unknown"], "APPTYPE": ["web", "web_mobile", "iphone", "ipad", "android_mobile", "android_tablet"], "LOCALE": ["cs", "da", "de", "el", "en", "es", "fi", "fr", "hu", "id", "it", "ja", "ko", "nb", "nl", "pl", "pt", "ro", "ru", "sk", "sv", "tr", "uk", "zh"], "TARGETING_STRATEGY": ["CHOOSE_YOUR_OWN"], "LOCATION": ["US"]}, "name": "2021-06-09 | Traffic | Keywords | Data Integration", "placement_group": "ALL", "pacing_delivery_type": "STANDARD", "tracking_urls": null, "conversion_learning_mode_type": null, "summary_status": "COMPLETED", "feed_profile_id": "0", "placement_traffic_type": null, "optimization_goal_metadata": {}, "bid_strategy_type": "AUTOMATIC_BID"}, "emitted_at": 1699393433712} +{"stream": "ad_group_analytics", "data": {"AD_GROUP_ID": "2680068678993", "AD_GROUP_ENTITY_STATUS": "1", "CAMPAIGN_ENTITY_STATUS": 1.0, "TOTAL_IMPRESSION_FREQUENCY": 1.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "TOTAL_IMPRESSION_USER": 1.0, "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "ADVERTISER_ID": "549761668032", "DATE": "2023-10-29", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness"}, "emitted_at": 1699893280169} {"stream": "boards", "data": {"media": {"pin_thumbnail_urls": [], "image_cover_url": "https://i.pinimg.com/400x300/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}, "owner": {"username": "integrationtest0375"}, "created_at": "2021-06-08T09:37:18", "board_pins_modified_at": "2021-10-25T11:17:56.715000", "id": "666743988523388559", "collaborator_count": 0, "follower_count": 2, "pin_count": 1, "privacy": "PUBLIC", "name": "business", "description": ""}, "emitted_at": 1680356853019} -{"stream":"board_pins","data":{"link":"http://airbyte.io/","dominant_color":"#cacafe","media":{"media_type":"image","images":{"150x150":{"width":150,"height":150,"url":"https://i.pinimg.com/150x150/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"},"400x300":{"width":400,"height":300,"url":"https://i.pinimg.com/400x300/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"},"600x":{"width":600,"height":359,"url":"https://i.pinimg.com/600x/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"},"1200x":{"width":1200,"height":718,"url":"https://i.pinimg.com/1200x/c6/b6/0d/c6b60d6b5f2ec04db7748d35fb1a8004.jpg"}}},"is_standard":true,"creative_type":"REGULAR","is_owner":true,"board_section_id":"5195034916661798218","id":"666743919837294988","description":"Data Integration","has_been_promoted":true,"created_at":"2021-06-08T09:37:30","note":"","product_tags":[],"alt_text":null,"title":"Airbyte","board_owner":{"username":"integrationtest0375"},"parent_pin_id":null,"board_id":"666743988523388559"},"emitted_at":1688054568572} -{"stream": "campaign_analytics_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_DAILY_SPEND_CAP": 750000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 3.0, "TOTAL_IMPRESSION_FREQUENCY": 1.5, "TOTAL_IMPRESSION_USER": 2.0, "DATE": "2023-07-14"}, "emitted_at": 1690299367301} +{"stream": "board_pins", "data": {"description": "Data Integration", "board_owner": {"username": "integrationtest0375"}, "product_tags": [], "has_been_promoted": true,"link":"http://airbyte.io/", "created_at": "2021-06-08T09:37:30", "board_id": "666743988523388559", "note": "", "creative_type": "REGULAR", "parent_pin_id": null, "title": "Airbyte", "alt_text": null, "pin_metrics": null, "dominant_color": "#cacafe", "id": "666743919837294988", "is_owner": true, "board_section_id": "5195034916661798218", "is_standard": true}, "emitted_at": 1698398201666} +{"stream": "board_sections", "data": {"name": "Airbyte_board_section_new", "id": "5195035116725909603"}, "emitted_at": 1699893323493} +{"stream": "board_section_pins","data":{"id":"666743919837294988","dominant_color":"#cacafe","pin_metrics":null,"title":"Airbyte","creative_type":"REGULAR","link":"http://airbyte.io/","board_id":"666743988523388559","created_at":"2021-06-08T09:37:30","is_owner":true,"description":"Data Integration","note":"","alt_text":null,"board_section_id":"5195034916661798218","parent_pin_id":null,"product_tags":[],"board_owner":{"username":"integrationtest0375"},"is_standard":true,"has_been_promoted":true},"emitted_at":1699893364884} +{"stream": "campaigns", "data": {"id": "626744128956", "ad_account_id": "549761668032", "name": "2021-06-09 | Traffic | Keywords | Data Integration", "status": "ACTIVE", "objective_type": "CONSIDERATION", "lifetime_spend_cap": 0, "daily_spend_cap": 3000000, "order_line_id": null, "tracking_urls": null, "created_time": 1623245885, "updated_time": 1691447502, "type": "campaign", "is_flexible_daily_budgets": false, "summary_status": "COMPLETED", "is_campaign_budget_optimization": true, "start_time": 1623196800, "end_time": 1624060800}, "emitted_at": 1699393571700} +{"stream": "campaign_analytics", "data": {"TOTAL_IMPRESSION_FREQUENCY": 1.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "TOTAL_IMPRESSION_USER": 1.0, "CAMPAIGN_ENTITY_STATUS": 1.0, "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "ADVERTISER_ID": 549761668032.0, "DATE": "2023-10-29", "IMPRESSION_2": 1.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness"}, "emitted_at": 1699894065462} +{"stream": "campaign_analytics_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 3.0, "TOTAL_IMPRESSION_FREQUENCY": 1.5, "TOTAL_IMPRESSION_USER": 2.0, "DATE": "2023-07-14"}, "emitted_at": 1690299367301} +{"stream": "campaign_targeting_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "TARGETING_VALUE": "TWOCOLUMN_FEED", "TARGETING_TYPE": "FEED_TYPE", "DATE": "2023-10-29"}, "emitted_at": 1699894287823} +{"stream": "user_account_analytics", "data": {"date": "2023-11-09", "data_status": "READY", "metrics": {"SAVE": 2.0, "OUTBOUND_CLICK_RATE": 0.0043859649122807015, "IMPRESSION": 912.0, "VIDEO_START": 0, "SAVE_RATE": 0.0021929824561403508, "QUARTILE_95_PERCENT_VIEW": 0, "ENGAGEMENT": 22.0, "VIDEO_AVG_WATCH_TIME": 0.0, "ENGAGEMENT_RATE": 0.02412280701754386, "PIN_CLICK": 17, "VIDEO_10S_VIEW": 0, "FULL_SCREEN_PLAY": 0, "CLOSEUP_RATE": 0.017543859649122806, "FULL_SCREEN_PLAYTIME": 0, "VIDEO_V50_WATCH_TIME": 0, "VIDEO_MRC_VIEW": 0, "CLICKTHROUGH": 4.0, "CLICKTHROUGH_RATE": 0.0043859649122807015, "OUTBOUND_CLICK": 4, "CLOSEUP": 16.0, "PIN_CLICK_RATE": 0.01864035087719298}}, "emitted_at": 1699894362486} +{"stream": "keywords", "data": {"archived": false, "id": "2886935172273", "parent_id": "2680068678965", "parent_type": "AD_GROUP", "type": "KEYWORD", "bid": null, "match_type": "BROAD", "value": "data science"}, "emitted_at": 1699393669235} +{"stream": "audiences", "data": {"type": "audience", "id": "2542622254639", "name": "airbyte audience", "ad_account_id": "549761668032", "audience_type": "ENGAGEMENT", "description": "airbyte audience", "status": "TOO_SMALL", "rule": {"engager_type": 1}}, "emitted_at": 1699293090886} +{"stream": "advertiser_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "EENGAGEMENT_RATE": 0.1, "ENGAGEMENT_2": 1.0, "IMPRESSION_2": 10.0, "REPIN_2": 1.0, "TOTAL_ENGAGEMENT": 1.0, "TOTAL_IMPRESSION_FREQUENCY": 5.0, "TOTAL_IMPRESSION_USER": 2.0, "TOTAL_REPIN_RATE": 0.1, "DATE": "2023-02-10"}, "emitted_at": 1699894848024} +{"stream": "advertiser_targeting_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "IMPRESSION_2": 1.0, "TARGETING_VALUE": "Education > Subjects > Science > Applied Science > Technology", "TARGETING_TYPE": "TARGETED_INTEREST", "DATE": "2023-10-29"}, "emitted_at": 1699894982269} +{"stream": "ad_group_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "AD_GROUP_ENTITY_STATUS": "ACTIVE", "AD_GROUP_ID": "2680068678993", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "TOTAL_IMPRESSION_FREQUENCY": 1.0, "TOTAL_IMPRESSION_USER": 1.0, "DATE": "2023-10-29"}, "emitted_at": 1699895043538} +{"stream": "ad_group_targeting_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "AD_GROUP_ENTITY_STATUS": "ACTIVE", "AD_GROUP_ID": "2680068678993", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "TARGETING_VALUE": "TWOCOLUMN_FEED", "TARGETING_TYPE": "FEED_TYPE", "DATE": "2023-10-29"}, "emitted_at": 1699895106949} +{"stream": "pin_promotion_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "AD_GROUP_ENTITY_STATUS": "ACTIVE", "AD_GROUP_ID": "2680068678993", "AD_ID": "687218400210", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "PIN_ID": 6.66743919837295e+17, "PIN_PROMOTION_ID": 687218400210.0, "TOTAL_IMPRESSION_FREQUENCY": 1.0, "TOTAL_IMPRESSION_USER": 1.0, "DATE": "2023-10-29"}, "emitted_at": 1699895200157} +{"stream": "pin_promotion_targeting_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "AD_GROUP_ENTITY_STATUS": "ACTIVE", "AD_GROUP_ID": "2680068678993", "AD_ID": "687218400210", "CAMPAIGN_DAILY_SPEND_CAP": 25000000.0, "CAMPAIGN_ENTITY_STATUS": "ACTIVE", "CAMPAIGN_ID": 626744128982.0, "CAMPAIGN_LIFETIME_SPEND_CAP": 0.0, "CAMPAIGN_NAME": "2021-06-08 09:08 UTC | Brand awareness", "IMPRESSION_2": 1.0, "PIN_ID": 6.66743919837295e+17, "PIN_PROMOTION_ID": 687218400210.0, "TARGETING_VALUE": "Education > Subjects > Science > Applied Science > Technology", "TARGETING_TYPE": "TARGETED_INTEREST", "DATE": "2023-10-29"}, "emitted_at": 1699895289749} +{"stream": "custom_vadim_report", "data": {"ADVERTISER_ID": 549761668032.0, "AD_ACCOUNT_ID": "549761668032", "AD_GROUP_ID": "2680068678993", "IMPRESSION_2": 11.0, "DATE_RANGE": "2023-10-01 - 2023-10-31"}, "emitted_at": 1700158289892} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-pinterest/metadata.yaml b/airbyte-integrations/connectors/source-pinterest/metadata.yaml index cd0e897b6339..e496ea4154c6 100644 --- a/airbyte-integrations/connectors/source-pinterest/metadata.yaml +++ b/airbyte-integrations/connectors/source-pinterest/metadata.yaml @@ -5,8 +5,10 @@ data: connectorSubtype: api connectorType: source definitionId: 5cb7e5fe-38c2-11ec-8d3d-0242ac130003 - dockerImageTag: 0.6.0 + dockerImageTag: 1.1.0 dockerRepository: airbyte/source-pinterest + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c githubIssueLabel: source-pinterest icon: pinterest.svg license: MIT @@ -17,6 +19,22 @@ data: oss: enabled: true releaseStage: generally_available + releases: + breakingChanges: + 1.0.0: + message: "This release updates the date-time fields to use the Airbyte format `timestamp_without_timezone`. This change affects all streams where date-time fields are present, ensuring more accurate and standardized time representations: BoardPins, BoardSectionPins, Boards, Catalogs, and CatalogFeeds. Additionally, the stream names AdvertizerReport and AdvertizerTargetingReport have been renamed to AdvertiserReport and AdvertiserTargetingReport, respectively. Users will need to refresh the source schema and reset affected streams after upgrading." + upgradeDeadline: "2023-12-14" + suggestedStreams: + streams: + - campaign_analytics + - ad_account_analytics + - ad_analytics + - campaigns + - ad_accounts + - ads + - user_account_analytics + - ad_group_analytics + - ad_groups documentationUrl: https://docs.airbyte.com/integrations/sources/pinterest tags: - language:python diff --git a/airbyte-integrations/connectors/source-pinterest/setup.py b/airbyte-integrations/connectors/source-pinterest/setup.py index eac9cebacb4b..5da646d8e719 100644 --- a/airbyte-integrations/connectors/source-pinterest/setup.py +++ b/airbyte-integrations/connectors/source-pinterest/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.2", "pendulum~=2.1.2"] +MAIN_REQUIREMENTS = ["airbyte-cdk", "pendulum~=2.1.2"] TEST_REQUIREMENTS = [ "pytest~=6.1", diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/reports.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/reports.py index fcf5437cedd6..25dfa3ec7d5f 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/reports.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/reports/reports.py @@ -4,12 +4,16 @@ import json from abc import abstractmethod +from functools import lru_cache from typing import Any, Iterable, List, Mapping, MutableMapping, Optional from urllib.parse import urljoin +import airbyte_cdk.sources.utils.casing as casing import backoff import requests from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.core import package_name_from_class +from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader from source_pinterest.streams import PinterestAnalyticsStream from source_pinterest.utils import get_analytics_columns @@ -22,7 +26,7 @@ class PinterestAnalyticsReportStream(PinterestAnalyticsStream): Details - https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report""" http_method = "POST" - report_wait_timeout = 180 + report_wait_timeout = 60 * 10 report_generation_maximum_retries = 5 @property @@ -65,7 +69,7 @@ def request_params( def backoff_max_time(func): def wrapped(self, *args, **kwargs): - return backoff.on_exception(backoff.constant, RetryableException, max_time=self.report_wait_timeout * 60, interval=10)(func)( + return backoff.on_exception(backoff.constant, RetryableException, max_time=self.report_wait_timeout, interval=10)(func)( self, *args, **kwargs ) @@ -73,9 +77,9 @@ def wrapped(self, *args, **kwargs): def backoff_max_tries(func): def wrapped(self, *args, **kwargs): - return backoff.on_exception(backoff.expo, ReportGenerationFailure, max_tries=self.report_generation_maximum_retries)(func)( - self, *args, **kwargs - ) + return backoff.on_exception( + backoff.expo, ReportGenerationFailure, max_tries=self.report_generation_maximum_retries, max_time=self.report_wait_timeout + )(func)(self, *args, **kwargs) return wrapped @@ -164,8 +168,146 @@ def _fetch_report_data(self, url: str) -> dict: """Fetch the report data from the given URL.""" return self._http_get(url) + @lru_cache(maxsize=None) + def get_json_schema(self) -> Mapping[str, Any]: + """ + :return: A dict of the JSON schema representing this stream. + + Schema is the same for all *Report and *TargetingReport streams + """ + + return ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("reports") + + +class PinterestAnalyticsTargetingReportStream(PinterestAnalyticsReportStream): + def request_body_json(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> Optional[Mapping]: + """Return the body of the API request in JSON format.""" + columns = get_analytics_columns().split(",") + # remove keys which are lot suitable for targeting report + for odd_value in ["TOTAL_IMPRESSION_FREQUENCY", "TOTAL_IMPRESSION_USER"]: + if odd_value in columns: + columns.remove(odd_value) + columns = ",".join(columns) + return self._construct_request_body(stream_slice["start_date"], stream_slice["end_date"], self.granularity, columns) + class CampaignAnalyticsReport(PinterestAnalyticsReportStream): @property def level(self): return "CAMPAIGN" + + +class CampaignTargetingReport(PinterestAnalyticsTargetingReportStream): + @property + def level(self): + return "CAMPAIGN_TARGETING" + + +class AdvertiserReport(PinterestAnalyticsReportStream): + @property + def level(self): + return "ADVERTISER" + + +class AdvertiserTargetingReport(PinterestAnalyticsTargetingReportStream): + @property + def level(self): + return "ADVERTISER_TARGETING" + + +class AdGroupReport(PinterestAnalyticsReportStream): + @property + def level(self): + return "AD_GROUP" + + +class AdGroupTargetingReport(PinterestAnalyticsTargetingReportStream): + @property + def level(self): + return "AD_GROUP_TARGETING" + + +class PinPromotionReport(PinterestAnalyticsReportStream): + @property + def level(self): + return "PIN_PROMOTION" + + +class PinPromotionTargetingReport(PinterestAnalyticsTargetingReportStream): + @property + def level(self): + return "PIN_PROMOTION_TARGETING" + + +class ProductGroupReport(PinterestAnalyticsReportStream): + @property + def level(self): + return "PRODUCT_GROUP" + + +class ProductGroupTargetingReport(PinterestAnalyticsTargetingReportStream): + @property + def level(self): + return "PRODUCT_GROUP_TARGETING" + + +class ProductItemReport(PinterestAnalyticsReportStream): + @property + def level(self): + return "PRODUCT_ITEM" + + +class KeywordReport(PinterestAnalyticsTargetingReportStream): + @property + def level(self): + return "KEYWORD" + + +class CustomReport(PinterestAnalyticsTargetingReportStream): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + self._custom_class_name = f"Custom_{self.config['name']}" + self._level = self.config["level"] + self.granularity = self.config["granularity"] + self.click_window_days = self.config["click_window_days"] + self.engagement_window_days = self.config["engagement_window_days"] + self.view_window_days = self.config["view_window_days"] + self.conversion_report_time = self.config["conversion_report_time"] + self.attribution_types = self.config["attribution_types"] + self.columns = self.config["columns"] + + @property + def level(self): + return self._level + + @property + def name(self) -> str: + """We override stream name to let the user change it via configuration.""" + name = self._custom_class_name or self.__class__.__name__ + return casing.camel_to_snake(name) + + def request_body_json(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> Optional[Mapping]: + """Return the body of the API request in JSON format.""" + return { + "start_date": stream_slice["start_date"], + "end_date": stream_slice["end_date"], + "level": self.level, + "granularity": self.granularity, + "click_window_days": self.click_window_days, + "engagement_window_days": self.engagement_window_days, + "view_window_days": self.view_window_days, + "conversion_report_time": self.conversion_report_time, + "attribution_types": self.attribution_types, + "columns": self.columns, + } + + @property + def window_in_days(self): + """Docs: https://developers.pinterest.com/docs/api/v5/#operation/analytics/get_report""" + if self.granularity == "HOUR": + return 2 + elif self.level == "PRODUCT_ITEM": + return 31 + else: + return 185 diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_account_analytics.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_account_analytics.json index 24ddfbf82255..b3a5c8ddfb14 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_account_analytics.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_account_analytics.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "DATE": { diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_accounts.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_accounts.json index a996db16e4ec..5ea3d323f1ee 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_accounts.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_accounts.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "additionalProperties": true, "properties": { diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_analytics.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_analytics.json index 24ddfbf82255..b3a5c8ddfb14 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_analytics.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_analytics.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "DATE": { diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_group_analytics.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_group_analytics.json index 24ddfbf82255..b3a5c8ddfb14 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_group_analytics.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_group_analytics.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "DATE": { diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_groups.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_groups.json index 166bd62c1fb6..c1684f7da3af 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_groups.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_groups.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "name": { @@ -84,6 +84,9 @@ "placement_group": { "type": ["null", "string"] }, + "placement_traffic_type": { + "type": ["null", "string"] + }, "pacing_delivery_type": { "type": ["null", "string"] }, @@ -116,6 +119,67 @@ }, "updated_time": { "type": ["null", "number"] + }, + "optimization_goal_metadata": { + "type": ["null", "object"], + "properties": { + "conversion_tag_v3_goal_metadata": { + "type": ["null", "object"], + "properties": { + "attribution_windows": { + "type": ["null", "object"], + "properties": { + "click_window_days": { + "type": ["null", "integer"] + }, + "engagement_window_days": { + "type": ["null", "integer"] + }, + "view_window_days": { + "type": ["null", "integer"] + } + } + }, + "conversion_event": { + "type": ["null", "string"] + }, + "conversion_tag_id": { + "type": ["null", "string"] + }, + "cpa_goal_value_in_micro_currency": { + "type": ["null", "string"] + }, + "is_roas_optimized": { + "type": ["null", "boolean"] + }, + "learning_mode_type": { + "type": ["null", "string"] + } + } + }, + "frequency_goal_metadata": { + "type": ["null", "object"], + "properties": { + "frequency": { + "type": ["null", "integer"] + }, + "timerange": { + "type": ["null", "string"] + } + } + }, + "scrollup_goal_metadata": { + "type": ["null", "object"], + "properties": { + "scrollup_goal_value_in_micro_currency": { + "type": ["null", "string"] + } + } + } + } + }, + "bid_strategy_type": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ads.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ads.json index 8402db35aaea..d5f238bd9b10 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ads.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ads.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "ad_group_id": { @@ -91,6 +91,9 @@ "view_tracking_url": { "type": ["null", "string"] }, + "lead_form_id": { + "type": ["null", "string"] + }, "ad_account_id": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/audiences.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/audiences.json new file mode 100644 index 000000000000..2ccb1cafad02 --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/audiences.json @@ -0,0 +1,75 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ad_account_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "audience_type": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "rule": { + "type": ["null", "object"], + "properties": { + "country": { + "type": ["null", "string"] + }, + "customer_list_id": { + "type": ["null", "string"] + }, + "engagement_domain": { + "type": ["null", "array"], + "items": {} + }, + "engagement_type": { + "type": ["null", "string"] + }, + "event": { + "type": ["null", "string"] + }, + "percentage": { + "type": ["null", "integer"] + }, + "prefill": { + "type": ["null", "boolean"] + }, + "retention_days": { + "type": ["null", "integer"] + }, + "visitor_source_id": { + "type": ["null", "string"] + }, + "engager_type": { + "type": ["null", "integer"] + }, + "ad_account_id": { + "type": ["null", "string"] + } + } + }, + "size": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "created_timestamp": { + "type": ["null", "integer"] + }, + "updated_timestamp": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_pins.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_pins.json index 5feb51438a6f..43131dbc1818 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_pins.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_pins.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "additionalProperties": true, "properties": { @@ -7,18 +7,12 @@ "type": ["null", "string"] }, "created_at": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, "creative_type": { - "type": ["null", "string"], - "enum": [ - "REGULAR", - "VIDEO", - "CAROUSEL", - "MAX_VIDEO", - "SHOP_THE_PIN", - "IDEA" - ] + "type": ["null", "string"] }, "is_standard": { "type": ["null", "boolean"] @@ -76,6 +70,12 @@ "type": ["null", "string"] } } + }, + "pin_metrics": { + "type": ["null", "object"] + }, + "has_been_promoted": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_section_pins.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_section_pins.json index 8f8761ebb866..4f54f74418ea 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_section_pins.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_section_pins.json @@ -1,12 +1,14 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { "type": ["null", "string"] }, "created_at": { - "type": ["null", "string"] + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, "link": { "type": ["null", "string"] @@ -34,6 +36,9 @@ } } }, + "pin_metrics": { + "type": ["null", "object"] + }, "media": { "type": ["null", "object"], "properties": { diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_sections.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_sections.json index 3dfef4b2a98a..2a07a40e51f1 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_sections.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/board_sections.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "name": { diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/boards.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/boards.json index 2c29d841828f..b945bacb1570 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/boards.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/boards.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "additionalProperties": true, "properties": { @@ -44,11 +44,13 @@ }, "created_at": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, "board_pins_modified_at": { "type": ["null", "string"], - "format": "date-time" + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" } } } diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/campaign_analytics.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/campaign_analytics.json index ce701367ec95..e664e58f6461 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/campaign_analytics.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/campaign_analytics.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "DATE": { diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/campaign_analytics_report.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/campaign_analytics_report.json index 18eec20efafe..c41983853652 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/campaign_analytics_report.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/campaign_analytics_report.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "DATE": { diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/campaigns.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/campaigns.json index 8ab5d4a6abfd..cb91bc3af2d7 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/campaigns.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { @@ -69,6 +69,21 @@ }, "type": { "type": ["null", "string"] + }, + "start_time": { + "type": ["null", "integer"] + }, + "end_time": { + "type": ["null", "integer"] + }, + "summary_status": { + "type": ["null", "string"] + }, + "is_campaign_budget_optimization": { + "type": ["null", "boolean"] + }, + "is_flexible_daily_budgets": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/catalogs.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/catalogs.json new file mode 100644 index 000000000000..a984915e23af --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/catalogs.json @@ -0,0 +1,25 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "id": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "name": { + "type": ["null", "string"] + }, + "catalog_type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/catalogs_feeds.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/catalogs_feeds.json new file mode 100644 index 000000000000..5b7e2acf179b --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/catalogs_feeds.json @@ -0,0 +1,57 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "created_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "id": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "name": { + "type": ["null", "string"] + }, + "format": { + "type": ["null", "string"] + }, + "catalog_type": { + "type": ["null", "string"] + }, + "location": { + "type": ["null", "string"] + }, + "preferred_processing_schedule": { + "type": ["null", "object"], + "properties": { + "time": { + "type": ["null", "string"] + }, + "timezone": { + "type": ["null", "string"] + } + } + }, + "status": { + "type": ["null", "string"] + }, + "default_currency": { + "type": ["null", "string"] + }, + "default_locale": { + "type": ["null", "string"] + }, + "default_country": { + "type": ["null", "string"] + }, + "default_availability": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/catalogs_product_groups.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/catalogs_product_groups.json new file mode 100644 index 000000000000..0d627d1bf1c0 --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/catalogs_product_groups.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "created_at": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + }, + "feed_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "is_featured": { + "type": ["null", "boolean"] + }, + "name": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/conversion_tags.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/conversion_tags.json new file mode 100644 index 000000000000..aa218ebd0bcd --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/conversion_tags.json @@ -0,0 +1,56 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ad_account_id": { + "type": ["null", "string"] + }, + "code_snippet": { + "type": ["null", "string"] + }, + "enhanced_match_status": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "last_fired_time_ms": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "version": { + "type": ["null", "string"] + }, + "configs": { + "type": ["null", "object"], + "properties": { + "aem_enabled": { + "type": ["null", "boolean"] + }, + "md_frequency": { + "type": ["null", "number"] + }, + "aem_fnln_enabled": { + "type": ["null", "boolean"] + }, + "aem_ph_enabled": { + "type": ["null", "boolean"] + }, + "aem_ge_enabled": { + "type": ["null", "boolean"] + }, + "aem_db_enabled": { + "type": ["null", "boolean"] + }, + "aem_loc_enabled": { + "type": ["null", "boolean"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/customer_lists.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/customer_lists.json new file mode 100644 index 000000000000..551d18c2fd8b --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/customer_lists.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ad_account_id": { + "type": ["null", "string"] + }, + "created_time": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "num_batches": { + "type": ["null", "integer"] + }, + "num_removed_user_records": { + "type": ["null", "integer"] + }, + "num_uploaded_user_records": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "updated_time": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/keywords.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/keywords.json new file mode 100644 index 000000000000..8057db2ce275 --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/keywords.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "archived": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "string"] + }, + "parent_id": { + "type": ["null", "string"] + }, + "parent_type": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "bid": { + "type": ["null", "integer"] + }, + "match_type": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/reports.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/reports.json new file mode 100644 index 000000000000..c41983853652 --- /dev/null +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/reports.json @@ -0,0 +1,346 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "DATE": { + "type": ["null", "string"], + "format": "date" + }, + "ADVERTISER_ID": { + "type": ["null", "number"] + }, + "AD_ACCOUNT_ID": { + "type": ["string"] + }, + "AD_ID": { + "type": ["null", "string"] + }, + "AD_GROUP_ENTITY_STATUS": { + "type": ["null", "string"] + }, + "AD_GROUP_ID": { + "type": ["null", "string"] + }, + "CAMPAIGN_DAILY_SPEND_CAP": { + "type": ["null", "number"] + }, + "CAMPAIGN_ENTITY_STATUS": { + "type": ["null", "string"] + }, + "CAMPAIGN_ID": { + "type": ["null", "number"] + }, + "CAMPAIGN_LIFETIME_SPEND_CAP": { + "type": ["null", "number"] + }, + "CAMPAIGN_NAME": { + "type": ["null", "string"] + }, + "CHECKOUT_ROAS": { + "type": ["null", "number"] + }, + "CLICKTHROUGH_1": { + "type": ["null", "number"] + }, + "CLICKTHROUGH_1_GROSS": { + "type": ["null", "number"] + }, + "CLICKTHROUGH_2": { + "type": ["null", "number"] + }, + "CPC_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "CPM_IN_DOLLAR": { + "type": ["null", "number"] + }, + "CPM_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "CTR": { + "type": ["null", "number"] + }, + "CTR_2": { + "type": ["null", "number"] + }, + "ECPCV_IN_DOLLAR": { + "type": ["null", "number"] + }, + "ECPCV_P95_IN_DOLLAR": { + "type": ["null", "number"] + }, + "ECPC_IN_DOLLAR": { + "type": ["null", "number"] + }, + "ECPC_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "ECPE_IN_DOLLAR": { + "type": ["null", "number"] + }, + "ECPM_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "ECPV_IN_DOLLAR": { + "type": ["null", "number"] + }, + "ECTR": { + "type": ["null", "number"] + }, + "EENGAGEMENT_RATE": { + "type": ["null", "number"] + }, + "ENGAGEMENT_1": { + "type": ["null", "number"] + }, + "ENGAGEMENT_2": { + "type": ["null", "number"] + }, + "ENGAGEMENT_RATE": { + "type": ["null", "number"] + }, + "IDEA_PIN_PRODUCT_TAG_VISIT_1": { + "type": ["null", "number"] + }, + "IDEA_PIN_PRODUCT_TAG_VISIT_2": { + "type": ["null", "number"] + }, + "IMPRESSION_1": { + "type": ["null", "number"] + }, + "IMPRESSION_1_GROSS": { + "type": ["null", "number"] + }, + "IMPRESSION_2": { + "type": ["null", "number"] + }, + "INAPP_CHECKOUT_COST_PER_ACTION": { + "type": ["null", "number"] + }, + "OUTBOUND_CLICK_1": { + "type": ["null", "number"] + }, + "OUTBOUND_CLICK_2": { + "type": ["null", "number"] + }, + "PAGE_VISIT_COST_PER_ACTION": { + "type": ["null", "number"] + }, + "PAGE_VISIT_ROAS": { + "type": ["null", "number"] + }, + "PAID_IMPRESSION": { + "type": ["null", "number"] + }, + "PIN_ID": { + "type": ["null", "number"] + }, + "PIN_PROMOTION_ID": { + "type": ["null", "number"] + }, + "REPIN_1": { + "type": ["null", "number"] + }, + "REPIN_2": { + "type": ["null", "number"] + }, + "REPIN_RATE": { + "type": ["null", "number"] + }, + "SPEND_IN_DOLLAR": { + "type": ["null", "number"] + }, + "SPEND_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_CLICKTHROUGH": { + "type": ["null", "number"] + }, + "TOTAL_CLICK_ADD_TO_CART": { + "type": ["null", "number"] + }, + "TOTAL_CLICK_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_CLICK_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_CLICK_LEAD": { + "type": ["null", "number"] + }, + "TOTAL_CLICK_SIGNUP": { + "type": ["null", "number"] + }, + "TOTAL_CLICK_SIGNUP_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_CONVERSIONS": { + "type": ["null", "number"] + }, + "TOTAL_CUSTOM": { + "type": ["null", "number"] + }, + "TOTAL_ENGAGEMENT": { + "type": ["null", "number"] + }, + "TOTAL_ENGAGEMENT_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_ENGAGEMENT_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_ENGAGEMENT_LEAD": { + "type": ["null", "number"] + }, + "TOTAL_ENGAGEMENT_SIGNUP": { + "type": ["null", "number"] + }, + "TOTAL_ENGAGEMENT_SIGNUP_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_IDEA_PIN_PRODUCT_TAG_VISIT": { + "type": ["null", "number"] + }, + "TOTAL_IMPRESSION_FREQUENCY": { + "type": ["null", "number"] + }, + "TOTAL_IMPRESSION_USER": { + "type": ["null", "number"] + }, + "TOTAL_LEAD": { + "type": ["null", "number"] + }, + "TOTAL_OFFLINE_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_PAGE_VISIT": { + "type": ["null", "number"] + }, + "TOTAL_REPIN_RATE": { + "type": ["null", "number"] + }, + "TOTAL_SIGNUP": { + "type": ["null", "number"] + }, + "TOTAL_SIGNUP_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_3SEC_VIEWS": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_AVG_WATCHTIME_IN_SECOND": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_MRC_VIEWS": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_P0_COMBINED": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_P100_COMPLETE": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_P25_COMBINED": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_P50_COMBINED": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_P75_COMBINED": { + "type": ["null", "number"] + }, + "TOTAL_VIDEO_P95_COMBINED": { + "type": ["null", "number"] + }, + "TOTAL_VIEW_ADD_TO_CART": { + "type": ["null", "number"] + }, + "TOTAL_VIEW_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_VIEW_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_VIEW_LEAD": { + "type": ["null", "number"] + }, + "TOTAL_VIEW_SIGNUP": { + "type": ["null", "number"] + }, + "TOTAL_VIEW_SIGNUP_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_WEB_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_WEB_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_WEB_CLICK_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_WEB_CLICK_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_WEB_ENGAGEMENT_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_WEB_ENGAGEMENT_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "TOTAL_WEB_SESSIONS": { + "type": ["null", "number"] + }, + "TOTAL_WEB_VIEW_CHECKOUT": { + "type": ["null", "number"] + }, + "TOTAL_WEB_VIEW_CHECKOUT_VALUE_IN_MICRO_DOLLAR": { + "type": ["null", "number"] + }, + "VIDEO_3SEC_VIEWS_2": { + "type": ["null", "number"] + }, + "VIDEO_LENGTH": { + "type": ["null", "number"] + }, + "VIDEO_MRC_VIEWS_2": { + "type": ["null", "number"] + }, + "VIDEO_P0_COMBINED_2": { + "type": ["null", "number"] + }, + "VIDEO_P100_COMPLETE_2": { + "type": ["null", "number"] + }, + "VIDEO_P25_COMBINED_2": { + "type": ["null", "number"] + }, + "VIDEO_P50_COMBINED_2": { + "type": ["null", "number"] + }, + "VIDEO_P75_COMBINED_2": { + "type": ["null", "number"] + }, + "VIDEO_P95_COMBINED_2": { + "type": ["null", "number"] + }, + "WEB_CHECKOUT_COST_PER_ACTION": { + "type": ["null", "number"] + }, + "WEB_CHECKOUT_ROAS": { + "type": ["null", "number"] + }, + "WEB_SESSIONS_1": { + "type": ["null", "number"] + }, + "WEB_SESSIONS_2": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/user_account_analytics.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/user_account_analytics.json index 0c04e9f61c11..fdb7499ac106 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/user_account_analytics.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/user_account_analytics.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "data_status": { diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py index e110f15f339b..9d36d1451323 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py @@ -3,8 +3,9 @@ # import copy +import logging from base64 import standard_b64encode -from typing import Any, List, Mapping, Tuple +from typing import Any, List, Mapping, Tuple, Type import pendulum import requests @@ -15,6 +16,20 @@ from airbyte_cdk.utils import AirbyteTracedException from source_pinterest.reports import CampaignAnalyticsReport +from .reports.reports import ( + AdGroupReport, + AdGroupTargetingReport, + AdvertiserReport, + AdvertiserTargetingReport, + CampaignTargetingReport, + CustomReport, + KeywordReport, + PinPromotionReport, + PinPromotionTargetingReport, + ProductGroupReport, + ProductGroupTargetingReport, + ProductItemReport, +) from .streams import ( AdAccountAnalytics, AdAccounts, @@ -22,16 +37,25 @@ AdGroupAnalytics, AdGroups, Ads, + Audiences, BoardPins, Boards, BoardSectionPins, BoardSections, CampaignAnalytics, Campaigns, + Catalogs, + CatalogsFeeds, + CatalogsProductGroups, + ConversionTags, + CustomerLists, + Keywords, PinterestStream, UserAccountAnalytics, ) +logger = logging.getLogger("airbyte") + class SourcePinterest(AbstractSource): def _validate_and_transform(self, config: Mapping[str, Any], amount_of_days_allowed_for_lookup: int = 89): @@ -39,21 +63,26 @@ def _validate_and_transform(self, config: Mapping[str, Any], amount_of_days_allo today = pendulum.today() latest_date_allowed_by_api = today.subtract(days=amount_of_days_allowed_for_lookup) - start_date = config["start_date"] - if not start_date: - config["start_date"] = latest_date_allowed_by_api - else: + start_date = config.get("start_date") + + # transform to datetime + if start_date and isinstance(start_date, str): try: - config["start_date"] = pendulum.from_format(config["start_date"], "YYYY-MM-DD") + config["start_date"] = pendulum.from_format(start_date, "YYYY-MM-DD") except ValueError: - message = "Entered `Start Date` does not match format YYYY-MM-DD" + message = f"Entered `Start Date` {start_date} does not match format YYYY-MM-DD" raise AirbyteTracedException( message=message, internal_message=message, failure_type=FailureType.config_error, ) - if (today - config["start_date"]).days > amount_of_days_allowed_for_lookup: - config["start_date"] = latest_date_allowed_by_api + + if not start_date or config["start_date"] < latest_date_allowed_by_api: + logger.info( + f"Current start_date: {start_date} does not meet API report requirements. Resetting start_date to: {latest_date_allowed_by_api}" + ) + config["start_date"] = latest_date_allowed_by_api + return config @staticmethod @@ -76,11 +105,16 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: config = self._validate_and_transform(config) authenticator = self.get_authenticator(config) url = f"{PinterestStream.url_base}user_account" - auth_headers = {"Accept": "application/json", **authenticator.get_auth_header()} try: + auth_headers = {"Accept": "application/json", **authenticator.get_auth_header()} session = requests.get(url, headers=auth_headers) session.raise_for_status() return True, None + except requests.exceptions.HTTPError as e: + if "401 Client Error: Unauthorized for url" in str(e): + return False, f"Try to re-authenticate because current refresh token is not valid. {e}" + else: + return False, e except requests.exceptions.RequestException as e: return False, e @@ -89,19 +123,72 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: report_config = self._validate_and_transform(config, amount_of_days_allowed_for_lookup=913) config = self._validate_and_transform(config) status = ",".join(config.get("status")) if config.get("status") else None + + ad_accounts = AdAccounts(config) + ads = Ads(ad_accounts, config=config, status_filter=status) + ad_groups = AdGroups(ad_accounts, config=config, status_filter=status) + campaigns = Campaigns(ad_accounts, config=config, status_filter=status) + boards = Boards(config) + board_sections = BoardSections(boards, config=config) return [ - AdAccountAnalytics(AdAccounts(config), config=config), - AdAccounts(config), - AdAnalytics(Ads(AdAccounts(config), with_data_slices=False, config=config), config=config), - AdGroupAnalytics(AdGroups(AdAccounts(config), with_data_slices=False, config=config), config=config), - AdGroups(AdAccounts(config), status_filter=status, config=config), - Ads(AdAccounts(config), status_filter=status, config=config), - BoardPins(Boards(config), config=config), - BoardSectionPins(BoardSections(Boards(config), config=config), config=config), - BoardSections(Boards(config), config=config), - Boards(config), - CampaignAnalytics(Campaigns(AdAccounts(config), with_data_slices=False, config=config), config=config), - CampaignAnalyticsReport(AdAccounts(report_config), config=report_config), - Campaigns(AdAccounts(config), status_filter=status, config=config), + ad_accounts, + AdAccountAnalytics(ad_accounts, config=config), + ads, + AdAnalytics(ads, config=config), + ad_groups, + AdGroupAnalytics(ad_groups, config=config), + boards, + BoardPins(boards, config=config), + board_sections, + BoardSectionPins(board_sections, config=config), + campaigns, + CampaignAnalytics(campaigns, config=config), + CampaignAnalyticsReport(ad_accounts, config=report_config), + CampaignTargetingReport(ad_accounts, config=report_config), UserAccountAnalytics(None, config=config), - ] + Keywords(ad_groups, config=config), + Audiences(ad_accounts, config=config), + ConversionTags(ad_accounts, config=config), + CustomerLists(ad_accounts, config=config), + Catalogs(config=config), + CatalogsFeeds(config=config), + CatalogsProductGroups(config=config), + AdvertiserReport(ad_accounts, config=report_config), + AdvertiserTargetingReport(ad_accounts, config=report_config), + AdGroupReport(ad_accounts, config=report_config), + AdGroupTargetingReport(ad_accounts, config=report_config), + PinPromotionReport(ad_accounts, config=report_config), + PinPromotionTargetingReport(ad_accounts, config=report_config), + ProductGroupReport(ad_accounts, config=report_config), + ProductGroupTargetingReport(ad_accounts, config=report_config), + KeywordReport(ad_accounts, config=report_config), + ProductItemReport(ad_accounts, config=report_config), + ] + self.get_custom_report_streams(ad_accounts, config=report_config) + + def get_custom_report_streams(self, parent, config: dict) -> List[Type[Stream]]: + """return custom report streams""" + custom_streams = [] + for report_config in config.get("custom_reports", []): + report_config["authenticator"] = config["authenticator"] + + # https://developers.pinterest.com/docs/api/v5/#operation/analytics/get_report + if report_config.get("granularity") == "HOUR": + # Otherwise: Response Code: 400 {"code":1,"message":"HOURLY request must be less than 3 days"} + amount_of_days_allowed_for_lookup = 2 + elif report_config.get("level") == "PRODUCT_ITEM": + amount_of_days_allowed_for_lookup = 91 + else: + amount_of_days_allowed_for_lookup = 913 + + start_date = report_config.get("start_date") + if not start_date: + report_config["start_date"] = config.get("start_date") + + report_config = self._validate_and_transform(report_config, amount_of_days_allowed_for_lookup) + + stream = CustomReport( + parent=parent, + config=report_config, + ) + custom_streams.append(stream) + return custom_streams diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/spec.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/spec.json index 9b791f42a1b3..eb03421a4585 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/spec.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/spec.json @@ -1,10 +1,9 @@ { "documentationUrl": "https://docs.airbyte.com/integrations/sources/pinterest", "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "title": "Pinterest Spec", "type": "object", - "required": ["start_date"], "additionalProperties": true, "properties": { "start_date": { @@ -18,7 +17,7 @@ }, "status": { "title": "Status", - "description": "Entity statuses based off of campaigns, ad_groups, and ads. If you do not have a status set, it will be ignored completely.", + "description": "For the ads, ad_groups, and campaigns streams, specifying a status will filter out records that do not match the specified ones. If a status is not specified, the source will default to records with a status of either ACTIVE or PAUSED.", "type": ["array", "null"], "items": { "type": "string", @@ -27,58 +26,266 @@ "uniqueItems": true }, "credentials": { - "title": "Authorization Method", + "title": "OAuth2.0", "type": "object", - "oneOf": [ - { - "type": "object", - "title": "OAuth2.0", - "required": ["auth_method", "refresh_token"], - "properties": { - "auth_method": { - "type": "string", - "const": "oauth2.0", - "order": 0 - }, - "client_id": { - "type": "string", - "title": "Client ID", - "description": "The Client ID of your OAuth application", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "The Client Secret of your OAuth application.", - "airbyte_secret": true - }, - "refresh_token": { - "type": "string", - "title": "Refresh Token", - "description": "Refresh Token to obtain new Access Token, when it's expired.", - "airbyte_secret": true - } - } + "required": [ + "auth_method", + "refresh_token", + "client_id", + "client_secret" + ], + "properties": { + "auth_method": { + "type": "string", + "const": "oauth2.0", + "order": 0 }, - { - "type": "object", - "title": "Access Token", - "required": ["auth_method", "access_token"], - "properties": { - "auth_method": { - "type": "string", - "const": "access_token", - "order": 0 - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "The Access Token to make authenticated requests.", - "airbyte_secret": true + "client_id": { + "type": "string", + "title": "Client ID", + "description": "The Client ID of your OAuth application", + "airbyte_secret": true + }, + "client_secret": { + "type": "string", + "title": "Client Secret", + "description": "The Client Secret of your OAuth application.", + "airbyte_secret": true + }, + "refresh_token": { + "type": "string", + "title": "Refresh Token", + "description": "Refresh Token to obtain new Access Token, when it's expired.", + "airbyte_secret": true + } + } + }, + "custom_reports": { + "title": "Custom Reports", + "description": "A list which contains ad statistics entries, each entry must have a name and can contains fields, breakdowns or action_breakdowns. Click on \"add\" to fill this field.", + "type": "array", + "items": { + "title": "ReportConfig", + "description": "Config for custom report", + "type": "object", + "required": ["name", "level", "granularity", "columns"], + "properties": { + "name": { + "title": "Name", + "description": "The name value of report", + "type": "string", + "order": 0 + }, + "level": { + "title": "Level", + "description": "Chosen level for API", + "default": "ADVERTISER", + "enum": [ + "ADVERTISER", + "ADVERTISER_TARGETING", + "CAMPAIGN", + "CAMPAIGN_TARGETING", + "AD_GROUP", + "AD_GROUP_TARGETING", + "PIN_PROMOTION", + "PIN_PROMOTION_TARGETING", + "KEYWORD", + "PRODUCT_GROUP", + "PRODUCT_GROUP_TARGETING", + "PRODUCT_ITEM" + ], + "type": "string", + "order": 1 + }, + "granularity": { + "title": "Granularity", + "description": "Chosen granularity for API", + "default": "TOTAL", + "enum": ["TOTAL", "DAY", "HOUR", "WEEK", "MONTH"], + "type": "string", + "order": 2 + }, + "columns": { + "title": "Columns", + "description": "A list of chosen columns", + "default": [], + "type": "array", + "order": 3, + "items": { + "title": "ValidEnums", + "description": "An enumeration.", + "enum": [ + "ADVERTISER_ID", + "AD_ACCOUNT_ID", + "AD_GROUP_ENTITY_STATUS", + "AD_GROUP_ID", + "AD_ID", + "CAMPAIGN_DAILY_SPEND_CAP", + "CAMPAIGN_ENTITY_STATUS", + "CAMPAIGN_ID", + "CAMPAIGN_LIFETIME_SPEND_CAP", + "CAMPAIGN_NAME", + "CHECKOUT_ROAS", + "CLICKTHROUGH_1", + "CLICKTHROUGH_1_GROSS", + "CLICKTHROUGH_2", + "CPC_IN_MICRO_DOLLAR", + "CPM_IN_DOLLAR", + "CPM_IN_MICRO_DOLLAR", + "CTR", + "CTR_2", + "ECPCV_IN_DOLLAR", + "ECPCV_P95_IN_DOLLAR", + "ECPC_IN_DOLLAR", + "ECPC_IN_MICRO_DOLLAR", + "ECPE_IN_DOLLAR", + "ECPM_IN_MICRO_DOLLAR", + "ECPV_IN_DOLLAR", + "ECTR", + "EENGAGEMENT_RATE", + "ENGAGEMENT_1", + "ENGAGEMENT_2", + "ENGAGEMENT_RATE", + "IDEA_PIN_PRODUCT_TAG_VISIT_1", + "IDEA_PIN_PRODUCT_TAG_VISIT_2", + "IMPRESSION_1", + "IMPRESSION_1_GROSS", + "IMPRESSION_2", + "INAPP_CHECKOUT_COST_PER_ACTION", + "OUTBOUND_CLICK_1", + "OUTBOUND_CLICK_2", + "PAGE_VISIT_COST_PER_ACTION", + "PAGE_VISIT_ROAS", + "PAID_IMPRESSION", + "PIN_ID", + "PIN_PROMOTION_ID", + "REPIN_1", + "REPIN_2", + "REPIN_RATE", + "SPEND_IN_DOLLAR", + "SPEND_IN_MICRO_DOLLAR", + "TOTAL_CHECKOUT", + "TOTAL_CHECKOUT_VALUE_IN_MICRO_DOLLAR", + "TOTAL_CLICKTHROUGH", + "TOTAL_CLICK_ADD_TO_CART", + "TOTAL_CLICK_CHECKOUT", + "TOTAL_CLICK_CHECKOUT_VALUE_IN_MICRO_DOLLAR", + "TOTAL_CLICK_LEAD", + "TOTAL_CLICK_SIGNUP", + "TOTAL_CLICK_SIGNUP_VALUE_IN_MICRO_DOLLAR", + "TOTAL_CONVERSIONS", + "TOTAL_CUSTOM", + "TOTAL_ENGAGEMENT", + "TOTAL_ENGAGEMENT_CHECKOUT", + "TOTAL_ENGAGEMENT_CHECKOUT_VALUE_IN_MICRO_DOLLAR", + "TOTAL_ENGAGEMENT_LEAD", + "TOTAL_ENGAGEMENT_SIGNUP", + "TOTAL_ENGAGEMENT_SIGNUP_VALUE_IN_MICRO_DOLLAR", + "TOTAL_IDEA_PIN_PRODUCT_TAG_VISIT", + "TOTAL_IMPRESSION_FREQUENCY", + "TOTAL_IMPRESSION_USER", + "TOTAL_LEAD", + "TOTAL_OFFLINE_CHECKOUT", + "TOTAL_PAGE_VISIT", + "TOTAL_REPIN_RATE", + "TOTAL_SIGNUP", + "TOTAL_SIGNUP_VALUE_IN_MICRO_DOLLAR", + "TOTAL_VIDEO_3SEC_VIEWS", + "TOTAL_VIDEO_AVG_WATCHTIME_IN_SECOND", + "TOTAL_VIDEO_MRC_VIEWS", + "TOTAL_VIDEO_P0_COMBINED", + "TOTAL_VIDEO_P100_COMPLETE", + "TOTAL_VIDEO_P25_COMBINED", + "TOTAL_VIDEO_P50_COMBINED", + "TOTAL_VIDEO_P75_COMBINED", + "TOTAL_VIDEO_P95_COMBINED", + "TOTAL_VIEW_ADD_TO_CART", + "TOTAL_VIEW_CHECKOUT", + "TOTAL_VIEW_CHECKOUT_VALUE_IN_MICRO_DOLLAR", + "TOTAL_VIEW_LEAD", + "TOTAL_VIEW_SIGNUP", + "TOTAL_VIEW_SIGNUP_VALUE_IN_MICRO_DOLLAR", + "TOTAL_WEB_CHECKOUT", + "TOTAL_WEB_CHECKOUT_VALUE_IN_MICRO_DOLLAR", + "TOTAL_WEB_CLICK_CHECKOUT", + "TOTAL_WEB_CLICK_CHECKOUT_VALUE_IN_MICRO_DOLLAR", + "TOTAL_WEB_ENGAGEMENT_CHECKOUT", + "TOTAL_WEB_ENGAGEMENT_CHECKOUT_VALUE_IN_MICRO_DOLLAR", + "TOTAL_WEB_SESSIONS", + "TOTAL_WEB_VIEW_CHECKOUT", + "TOTAL_WEB_VIEW_CHECKOUT_VALUE_IN_MICRO_DOLLAR", + "VIDEO_3SEC_VIEWS_2", + "VIDEO_LENGTH", + "VIDEO_MRC_VIEWS_2", + "VIDEO_P0_COMBINED_2", + "VIDEO_P100_COMPLETE_2", + "VIDEO_P25_COMBINED_2", + "VIDEO_P50_COMBINED_2", + "VIDEO_P75_COMBINED_2", + "VIDEO_P95_COMBINED_2", + "WEB_CHECKOUT_COST_PER_ACTION", + "WEB_CHECKOUT_ROAS", + "WEB_SESSIONS_1", + "WEB_SESSIONS_2" + ] } + }, + "click_window_days": { + "title": "Click window days", + "description": "Number of days to use as the conversion attribution window for a pin click action.", + "default": 30, + "enum": [0, 1, 7, 14, 30, 60], + "type": "integer", + "order": 4 + }, + "engagement_window_days": { + "title": "Engagement window days", + "description": "Number of days to use as the conversion attribution window for an engagement action.", + "default": [30], + "enum": [0, 1, 7, 14, 30, 60], + "type": "integer", + "order": 5 + }, + "view_window_days": { + "title": "View window days", + "description": "Number of days to use as the conversion attribution window for a view action.", + "default": [30], + "enum": [0, 1, 7, 14, 30, 60], + "type": "integer", + "order": 6 + }, + "conversion_report_time": { + "title": "Conversion report time", + "description": "The date by which the conversion metrics returned from this endpoint will be reported. There are two dates associated with a conversion event: the date that the user interacted with the ad, and the date that the user completed a conversion event..", + "default": "TIME_OF_AD_ACTION", + "enum": ["TIME_OF_AD_ACTION", "TIME_OF_CONVERSION"], + "type": "string", + "order": 7 + }, + "attribution_types": { + "title": "Attribution types", + "description": "List of types of attribution for the conversion report", + "default": ["INDIVIDUAL", "HOUSEHOLD"], + "type": "array", + "items": { + "title": "ValidEnums", + "description": "An enumeration.", + "enum": ["INDIVIDUAL", "HOUSEHOLD"] + }, + "order": 8 + }, + "start_date": { + "type": "string", + "title": "Start Date", + "description": "A date in the format YYYY-MM-DD. If you have not set a date, it would be defaulted to latest allowed date by report api (913 days from today).", + "format": "date", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "pattern_descriptor": "YYYY-MM-DD", + "examples": ["2022-07-28"], + "order": 9 } } - ] + } } } }, diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/streams.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/streams.py index 74add04ca3df..ab36624e6424 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/streams.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/streams.py @@ -2,16 +2,21 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import logging from abc import ABC from datetime import datetime -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional +from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional import pendulum import requests from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import Source +from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream +from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +from requests import HTTPError from .utils import get_analytics_columns, to_datetime_str @@ -20,6 +25,14 @@ MAX_RATE_LIMIT_CODE = 8 +class NonJSONResponse(Exception): + pass + + +class RateLimitExceeded(Exception): + pass + + class PinterestStream(HttpStream, ABC): url_base = "https://api.pinterest.com/v5/" primary_key = "id" @@ -40,10 +53,6 @@ def start_date(self): def window_in_days(self): return 30 # Set window_in_days to 30 days date range - @property - def availability_strategy(self) -> Optional["AvailabilityStrategy"]: - return None - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: next_page = response.json().get("bookmark", {}) if self.data_fields else {} @@ -69,8 +78,13 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str, yield record def should_retry(self, response: requests.Response) -> bool: - if isinstance(response.json(), dict): - self.max_rate_limit_exceeded = response.json().get("code", 0) == MAX_RATE_LIMIT_CODE + try: + resp = response.json() + except requests.exceptions.JSONDecodeError: + raise NonJSONResponse(f"Received unexpected response in non json format: '{response.text}'") + + if isinstance(resp, dict): + self.max_rate_limit_exceeded = resp.get("code", 0) == MAX_RATE_LIMIT_CODE # when max rate limit exceeded, we should skip the stream. if response.status_code == requests.codes.too_many_requests and self.max_rate_limit_exceeded: self.logger.error(f"For stream {self.name} Max Rate Limit exceeded.") @@ -80,7 +94,12 @@ def should_retry(self, response: requests.Response) -> bool: def backoff_time(self, response: requests.Response) -> Optional[float]: if response.status_code == requests.codes.too_many_requests: self.logger.error(f"For stream {self.name} rate limit exceeded.") - return float(response.headers.get("X-RateLimit-Reset", 0)) + sleep_time = float(response.headers.get("X-RateLimit-Reset", 0)) + if sleep_time > 600: + raise RateLimitExceeded( + f"Rate limit exceeded for stream {self.name}. Waiting time is longer than 10 minutes: {sleep_time}s." + ) + return sleep_time class PinterestSubStream(HttpSubStream): @@ -106,6 +125,53 @@ def path(self, **kwargs) -> str: return "boards" +class Catalogs(PinterestStream): + """Docs: https://developers.pinterest.com/docs/api/v5/#operation/catalogs/list""" + + use_cache = True + + def path(self, **kwargs) -> str: + return "catalogs" + + +class CatalogsFeeds(PinterestStream): + """Docs: https://developers.pinterest.com/docs/api/v5/#operation/feeds/list""" + + use_cache = True + + def path(self, **kwargs) -> str: + return "catalogs/feeds" + + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + # Remove sensitive data + for record in super().parse_response(response, stream_state, **kwargs): + record.pop("credentials", None) + yield record + + +class CatalogsProductGroupsAvailabilityStrategy(HttpAvailabilityStrategy): + def reasons_for_unavailable_status_codes( + self, stream: Stream, logger: logging.Logger, source: Optional[Source], error: HTTPError + ) -> Dict[int, str]: + reasons_for_codes: Dict[int, str] = super().reasons_for_unavailable_status_codes(stream, logger, source, error) + reasons_for_codes[409] = "Can't access catalog product groups because there is no existing catalog." + + return reasons_for_codes + + +class CatalogsProductGroups(PinterestStream): + """Docs: https://developers.pinterest.com/docs/api/v5/#operation/catalogs_product_groups/list""" + + use_cache = True + + def path(self, **kwargs) -> str: + return "catalogs/product_groups" + + @property + def availability_strategy(self) -> Optional["AvailabilityStrategy"]: + return CatalogsProductGroupsAvailabilityStrategy() + + class AdAccounts(PinterestStream): use_cache = True @@ -128,6 +194,34 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: return f"boards/{stream_slice['sub_parent']['parent']['id']}/sections/{stream_slice['parent']['id']}/pins" +class Audiences(PinterestSubStream, PinterestStream): + """Docs: https://developers.pinterest.com/docs/api/v5/#operation/audiences/list""" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"ad_accounts/{stream_slice['parent']['id']}/audiences" + + +class Keywords(PinterestSubStream, PinterestStream): + """Docs: https://developers.pinterest.com/docs/api/v5/#operation/keywords/get""" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"ad_accounts/{stream_slice['parent']['ad_account_id']}/keywords?ad_group_id={stream_slice['parent']['id']}" + + +class ConversionTags(PinterestSubStream, PinterestStream): + """Docs: https://developers.pinterest.com/docs/api/v5/#operation/conversion_tags/list""" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"ad_accounts/{stream_slice['parent']['id']}/conversion_tags" + + +class CustomerLists(PinterestSubStream, PinterestStream): + """Docs: https://developers.pinterest.com/docs/api/v5/#tag/customer_lists""" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"ad_accounts/{stream_slice['parent']['id']}/customer_lists" + + class IncrementalPinterestStream(PinterestStream, ABC): def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: default_value = self.start_date.format("YYYY-MM-DD") @@ -283,7 +377,7 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class Campaigns(ServerSideFilterStream): - def __init__(self, parent: HttpStream, with_data_slices: bool = True, status_filter: str = "", **kwargs): + def __init__(self, parent: HttpStream, with_data_slices: bool = False, status_filter: str = "", **kwargs): super().__init__(parent, with_data_slices, **kwargs) self.status_filter = status_filter @@ -300,11 +394,12 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class AdGroups(ServerSideFilterStream): - def __init__(self, parent: HttpStream, with_data_slices: bool = True, status_filter: str = "", **kwargs): + def __init__(self, parent: HttpStream, with_data_slices: bool = False, status_filter: str = "", **kwargs): super().__init__(parent, with_data_slices, **kwargs) self.status_filter = status_filter def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + print(f"=========== stream_slice: {stream_slice} =====================") params = f"?entity_statuses={self.status_filter}" if self.status_filter else "" return f"ad_accounts/{stream_slice['parent']['id']}/ad_groups{params}" @@ -317,7 +412,7 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class Ads(ServerSideFilterStream): - def __init__(self, parent: HttpStream, with_data_slices: bool = True, status_filter: str = "", **kwargs): + def __init__(self, parent: HttpStream, with_data_slices: bool = False, status_filter: str = "", **kwargs): super().__init__(parent, with_data_slices, **kwargs) self.status_filter = status_filter diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/conftest.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/conftest.py index b929d7e18be0..0597c5b7681b 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/conftest.py @@ -72,8 +72,4 @@ def analytics_report_stream(): @fixture def date_range(): - return { - 'start_date': '2023-01-01', - 'end_date': '2023-01-31', - 'parent': {'id': '123'} - } + return {"start_date": "2023-01-01", "end_date": "2023-01-31", "parent": {"id": "123"}} diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py index 75d716c2fc03..9bff0aa37036 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_reports.py @@ -1,22 +1,43 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import copy +import os +from unittest.mock import MagicMock +import pytest import responses +from source_pinterest import SourcePinterest +from source_pinterest.reports import CampaignAnalyticsReport +from source_pinterest.reports.reports import ( + AdGroupReport, + AdGroupTargetingReport, + AdvertiserReport, + AdvertiserTargetingReport, + CampaignTargetingReport, + KeywordReport, + PinPromotionReport, + PinPromotionTargetingReport, + ProductGroupReport, + ProductGroupTargetingReport, + ProductItemReport, +) from source_pinterest.utils import get_analytics_columns +from unit_tests.test_source import setup_responses +os.environ["REQUEST_CACHE_PATH"] = '/tmp' @responses.activate def test_request_body_json(analytics_report_stream, date_range): - granularity = 'DAY' + granularity = "DAY" columns = get_analytics_columns() expected_body = { - 'start_date': date_range['start_date'], - 'end_date': date_range['end_date'], - 'granularity': granularity, - 'columns': columns.split(','), - 'level': analytics_report_stream.level, + "start_date": date_range["start_date"], + "end_date": date_range["end_date"], + "granularity": granularity, + "columns": columns.split(","), + "level": analytics_report_stream.level, } body = analytics_report_stream.request_body_json(date_range) @@ -25,19 +46,12 @@ def test_request_body_json(analytics_report_stream, date_range): @responses.activate def test_read_records(analytics_report_stream, date_range): - report_download_url = 'https://download.report' - report_request_url = 'https://api.pinterest.com/v5/ad_accounts/123/reports' + report_download_url = "https://download.report" + report_request_url = "https://api.pinterest.com/v5/ad_accounts/123/reports" - final_report_status = { - 'report_status': 'FINISHED', - 'url': report_download_url - } + final_report_status = {"report_status": "FINISHED", "url": report_download_url} - initial_response = { - 'report_status': "IN_PROGRESS", - 'token': 'token', - 'message': '' - } + initial_response = {"report_status": "IN_PROGRESS", "token": "token", "message": ""} final_response = {"campaign_id": [{"metric": 1}]} @@ -45,11 +59,11 @@ def test_read_records(analytics_report_stream, date_range): responses.add(responses.GET, report_request_url, json=final_report_status, status=200) responses.add(responses.GET, report_download_url, json=final_response, status=200) - sync_mode = 'full_refresh' - cursor_field = ['last_updated'] + sync_mode = "full_refresh" + cursor_field = ["last_updated"] stream_state = { - 'start_date': '2023-01-01', - 'end_date': '2023-01-31', + "start_date": "2023-01-01", + "end_date": "2023-01-31", } records = analytics_report_stream.read_records(sync_mode, cursor_field, date_range, stream_state) @@ -58,3 +72,54 @@ def test_read_records(analytics_report_stream, date_range): assert next(records) == expected_record assert len(responses.calls) == 3 assert responses.calls[0].request.url == report_request_url + + +@responses.activate +def test_streams(test_config): + setup_responses() + source = SourcePinterest() + streams = source.streams(test_config) + expected_streams_number = 32 + assert len(streams) == expected_streams_number + +@responses.activate +def test_custom_streams(test_config): + config = copy.deepcopy(test_config) + config['custom_reports'] = [{ + "name": "vadim_report", + "level": "AD_GROUP", + "granularity": "MONTH", + "click_window_days": 30, + "engagement_window_days": 30, + "view_window_days": 30, + "conversion_report_time": "TIME_OF_CONVERSION", + "attribution_types": ["INDIVIDUAL", "HOUSEHOLD"], + "columns": ["ADVERTISER_ID", "AD_ACCOUNT_ID", "AD_GROUP_ID", "CTR", "IMPRESSION_2"], + "start_date": "2023-01-08" + }] + setup_responses() + source = SourcePinterest() + streams = source.streams(config) + expected_streams_number = 33 + assert len(streams) == expected_streams_number + +@pytest.mark.parametrize( + "report_name, expected_level", + [ + [CampaignAnalyticsReport, 'CAMPAIGN'], + [CampaignTargetingReport, 'CAMPAIGN_TARGETING'], + [AdvertiserReport, 'ADVERTISER'], + [AdvertiserTargetingReport, 'ADVERTISER_TARGETING'], + [AdGroupReport, 'AD_GROUP'], + [AdGroupTargetingReport, 'AD_GROUP_TARGETING'], + [PinPromotionReport, 'PIN_PROMOTION'], + [PinPromotionTargetingReport, 'PIN_PROMOTION_TARGETING'], + [ProductGroupReport, 'PRODUCT_GROUP'], + [ProductGroupTargetingReport, 'PRODUCT_GROUP_TARGETING'], + [ProductItemReport, 'PRODUCT_ITEM'], + [KeywordReport, 'KEYWORD'] + ], +) +def test_level(test_config, report_name, expected_level): + assert report_name(parent=None, config=MagicMock()).level == expected_level + diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py index 56b16f6cca40..615e0aa9c109 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_source.py @@ -36,16 +36,19 @@ def test_check_wrong_date_connection(wrong_date_config): logger_mock = MagicMock() with pytest.raises(AirbyteTracedException) as e: source.check_connection(logger_mock, wrong_date_config) - assert e.value.message == 'Entered `Start Date` does not match format YYYY-MM-DD' + assert e.value.message == "Entered `Start Date` wrong_date_format does not match format YYYY-MM-DD" @responses.activate -def test_streams(test_config): - setup_responses() +def test_check_connection_expired_token(test_config): + responses.add(responses.POST, "https://api.pinterest.com/v5/oauth/token", status=401) source = SourcePinterest() - streams = source.streams(test_config) - expected_streams_number = 14 - assert len(streams) == expected_streams_number + logger_mock = MagicMock() + assert source.check_connection(logger_mock, test_config) == ( + False, + "Try to re-authenticate because current refresh token is not valid. " + "401 Client Error: Unauthorized for url: https://api.pinterest.com/v5/oauth/token", + ) def test_get_authenticator(test_config): diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py index 8c26fffe401e..bca080a29216 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py @@ -2,6 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import os from http import HTTPStatus from unittest.mock import MagicMock @@ -14,16 +15,27 @@ AdGroupAnalytics, AdGroups, Ads, + Audiences, BoardPins, Boards, BoardSectionPins, BoardSections, CampaignAnalytics, Campaigns, + Catalogs, + CatalogsFeeds, + CatalogsProductGroups, + ConversionTags, + CustomerLists, + Keywords, PinterestStream, PinterestSubStream, + RateLimitExceeded, + UserAccountAnalytics, ) +os.environ["REQUEST_CACHE_PATH"] = "/tmp" + @pytest.fixture def patch_base_class(mocker): @@ -60,6 +72,15 @@ def test_parse_response(patch_base_class, test_response, test_current_stream_sta assert next(stream.parse_response(**inputs)) == expected_parsed_object +def test_parse_response_with_sensitive_data(patch_base_class): + """Test that sensitive data is removed""" + stream = CatalogsFeeds(config=MagicMock()) + response = MagicMock() + response.json.return_value = {"items": [{"id": "CatalogsFeeds1", "credentials": {"password": "bla"}}], "bookmark": "string"} + actual_response = list(stream.parse_response(response=response, stream_state=None)) + assert actual_response == [{"id": "CatalogsFeeds1"}] + + def test_request_headers(patch_base_class): stream = PinterestStream(config=MagicMock()) inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} @@ -112,10 +133,28 @@ def test_should_retry_on_max_rate_limit_error(requests_mock, test_response, stat assert result == expected +def test_non_json_response(requests_mock): + stream = UserAccountAnalytics(parent=None, config=MagicMock()) + url = "https://api.pinterest.com/v5/boards" + requests_mock.get("https://api.pinterest.com/v5/boards", text="some response", status_code=200) + response = requests.get(url) + try: + stream.should_retry(response) + assert False + except Exception as e: + assert "Received unexpected response in non json format" in str(e) + + @pytest.mark.parametrize( "test_response, test_headers, status_code, expected", [ ({"code": 7, "message": "Some other error message"}, {"X-RateLimit-Reset": "2"}, 429, 2.0), + ( + {"code": 7, "message": "Some other error message"}, + {"X-RateLimit-Reset": "2000"}, + 429, + (RateLimitExceeded, "Rate limit exceeded for stream boards. Waiting time is longer than 10 minutes: 2000.0s."), + ), ], ) def test_backoff_on_rate_limit_error(requests_mock, test_response, status_code, test_headers, expected): @@ -129,8 +168,13 @@ def test_backoff_on_rate_limit_error(requests_mock, test_response, status_code, ) response = requests.get(url) - result = stream.backoff_time(response) - assert result == expected + + if isinstance(expected, tuple): + with pytest.raises(expected[0], match=expected[1]): + stream.backoff_time(response) + else: + result = stream.backoff_time(response) + assert result == expected @pytest.mark.parametrize( @@ -164,6 +208,17 @@ def test_backoff_on_rate_limit_error(requests_mock, test_response, status_code, {"sub_parent": {"parent": {"id": "234"}}, "parent": {"id": "123"}}, "ad_accounts/234/ads/analytics", ), + (Catalogs(config=MagicMock()), None, "catalogs"), + (CatalogsFeeds(config=MagicMock()), None, "catalogs/feeds"), + (CatalogsProductGroups(config=MagicMock()), None, "catalogs/product_groups"), + ( + Keywords(parent=None, config=MagicMock()), + {"parent": {"id": "234", "ad_account_id": "AD_ACCOUNT_1"}}, + "ad_accounts/AD_ACCOUNT_1/keywords?ad_group_id=234", + ), + (Audiences(parent=None, config=MagicMock()), {"parent": {"id": "AD_ACCOUNT_1"}}, "ad_accounts/AD_ACCOUNT_1/audiences"), + (ConversionTags(parent=None, config=MagicMock()), {"parent": {"id": "AD_ACCOUNT_1"}}, "ad_accounts/AD_ACCOUNT_1/conversion_tags"), + (CustomerLists(parent=None, config=MagicMock()), {"parent": {"id": "AD_ACCOUNT_1"}}, "ad_accounts/AD_ACCOUNT_1/customer_lists"), ], ) def test_path(patch_base_class, stream_cls, slice, expected): diff --git a/airbyte-integrations/connectors/source-pipedrive/Dockerfile b/airbyte-integrations/connectors/source-pipedrive/Dockerfile index 9ef2f33510dd..8698c9a3fad5 100644 --- a/airbyte-integrations/connectors/source-pipedrive/Dockerfile +++ b/airbyte-integrations/connectors/source-pipedrive/Dockerfile @@ -1,16 +1,38 @@ -FROM python:3.9-slim +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base WORKDIR /airbyte/integration_code -COPY source_pipedrive ./source_pipedrive + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only COPY main.py ./ -COPY setup.py ./ -RUN pip install . +COPY source_pipedrive ./source_pipedrive ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.0.0 +LABEL io.airbyte.version=2.2.2 LABEL io.airbyte.name=airbyte/source-pipedrive diff --git a/airbyte-integrations/connectors/source-pipedrive/README.md b/airbyte-integrations/connectors/source-pipedrive/README.md index 25eaff686dfa..7fbcac238e01 100644 --- a/airbyte-integrations/connectors/source-pipedrive/README.md +++ b/airbyte-integrations/connectors/source-pipedrive/README.md @@ -1,73 +1,34 @@ # Pipedrive Source -This is the repository for the Pipedrive source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/pipedrive). +This is the repository for the Pipedrive configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/pipedrive). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install . -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-pipedrive:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/pipedrive) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_pipedrive/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/pipedrive) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_pipedrive/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source pipedrive test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-pipedrive:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-pipedrive build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-pipedrive:airbyteDocker +An image will be built with the tag `airbyte/source-pipedrive:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-pipedrive:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -77,44 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pipedrive:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pipedrive:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-pipedrive:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-pipedrive test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-pipedrive:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-pipedrive:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -124,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-pipedrive test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/pipedrive.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-pipedrive/__init__.py b/airbyte-integrations/connectors/source-pipedrive/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml b/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml index 928821e53d54..303f046105c3 100644 --- a/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml @@ -1,66 +1,36 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests connector_image: airbyte/source-pipedrive:dev -test_strictness_level: "high" acceptance_tests: spec: tests: - - spec_path: "source_pipedrive/spec.json" + - spec_path: "source_pipedrive/spec.yaml" connection: tests: - config_path: "secrets/config.json" status: "succeed" - - config_path: "secrets/old_config.json" - status: "succeed" - config_path: "integration_tests/invalid_config.json" status: "failed" discovery: tests: - config_path: "secrets/config.json" - backward_compatibility_tests_config: - disable_for_version: 0.1.19 basic_read: tests: - config_path: "secrets/config.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - ignored_fields: - users: - - name: modified - bypass_reason: "constantly increasing date-time field" - - name: last_login - bypass_reason: "constantly increasing date-time field" - deal_fields: - - name: show_in_pipelines - bypass_reason: "Unstable data" - - name: important_flag - bypass_reason: "Unstable data" - - name: pipeline_ids - bypass_reason: "Unstable data" - - name: update_time - bypass_reason: "Unstable data" - - name: last_updated_by_user_id - bypass_reason: "Unstable data" - organization_fields: - - name: update_time - bypass_reason: "Unstable data" - - name: important_flag - bypass_reason: "Unstable data" - - name: last_updated_by_user_id - bypass_reason: "Unstable data" - person_fields: - - name: update_time - bypass_reason: "Unstable data" - - name: important_flag - bypass_reason: "Unstable data" - - name: last_updated_by_user_id - bypass_reason: "Unstable data" - product_fields: - - name: update_time - bypass_reason: "Unstable data" - - name: important_flag - bypass_reason: "Unstable data" - - name: last_updated_by_user_id - bypass_reason: "Unstable data" + configured_catalog_path: "integration_tests/configured_catalog.json" fail_on_extra_columns: false + empty_streams: + - name: files + - name: filters + - name: leads + - name: notes + - name: activities + - name: pipelines + - name: products + - name: stages + - name: deal_products + - name: mail + incremental: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-pipedrive/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-pipedrive/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-pipedrive/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-pipedrive/build.gradle b/airbyte-integrations/connectors/source-pipedrive/build.gradle deleted file mode 100644 index 3622ebe82092..000000000000 --- a/airbyte-integrations/connectors/source-pipedrive/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_pipedrive' -} diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/__init__.py b/airbyte-integrations/connectors/source-pipedrive/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pipedrive/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json index 4ff5d5327d40..02422e10fd1c 100644 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json @@ -8,20 +8,27 @@ "source_defined_cursor": true, "default_cursor_field": ["update_time"] }, - "sync_mode": "incremental", + "sync_mode": "full_refresh", "cursor_field": ["update_time"], "destination_sync_mode": "append" }, { "stream": { - "name": "deal_products", + "name": "deal_fields", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "primary_key": [["id"]] + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "goals", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { @@ -87,7 +94,19 @@ }, { "stream": { - "name": "activity_fields", + "name": "activities", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["update_time"] + }, + "sync_mode": "incremental", + "cursor_field": ["update_time"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "activity_types", "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, @@ -96,52 +115,48 @@ }, { "stream": { - "name": "activity_types", + "name": "activity_fields", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["update_time"], - "source_defined_primary_key": [["id"]] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["update_time"], - "primary_key": [["id"]] + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "activities", + "name": "currencies", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["update_time"] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "incremental", - "cursor_field": ["update_time"], - "destination_sync_mode": "append" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "currencies", + "name": "mail", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "primary_key": [["id"]] + "destination_sync_mode": "overwrite" }, { "stream": { "name": "organizations", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["update_time"] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "incremental", - "cursor_field": ["update_time"], - "destination_sync_mode": "append" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "organization_fields", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { @@ -166,6 +181,15 @@ "cursor_field": ["update_time"], "destination_sync_mode": "append" }, + { + "stream": { + "name": "person_fields", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "pipelines", @@ -178,17 +202,6 @@ "cursor_field": ["update_time"], "destination_sync_mode": "append" }, - { - "stream": { - "name": "product_fields", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "primary_key": [["id"]] - }, { "stream": { "name": "products", @@ -205,74 +218,58 @@ }, { "stream": { - "name": "roles", + "name": "product_fields", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "primary_key": [["id"]] - }, - { - "stream": { - "name": "stages", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["update_time"] - }, - "sync_mode": "incremental", - "cursor_field": ["update_time"], - "destination_sync_mode": "append" + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "users", + "name": "roles", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["modified"] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "incremental", - "cursor_field": ["modified"], - "destination_sync_mode": "append" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "deal_fields", + "name": "stages", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["update_time"] + "default_cursor_field": ["update_time"], + "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", + "destination_sync_mode": "overwrite", "cursor_field": ["update_time"], - "destination_sync_mode": "append" + "primary_key": [["id"]] }, { "stream": { - "name": "organization_fields", + "name": "users", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["update_time"] + "default_cursor_field": ["modified"], + "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", - "cursor_field": ["update_time"], - "destination_sync_mode": "append" + "destination_sync_mode": "overwrite", + "cursor_field": ["modified"], + "primary_key": [["id"]] }, { "stream": { - "name": "person_fields", + "name": "deal_products", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["update_time"] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "incremental", - "cursor_field": ["update_time"], - "destination_sync_mode": "append" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-pipedrive/integration_tests/expected_records.jsonl deleted file mode 100644 index c593d1cc0ba5..000000000000 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/expected_records.jsonl +++ /dev/null @@ -1,62 +0,0 @@ -{"stream": "activities", "data": {"id": 1, "company_id": 7780468, "user_id": 11884360, "done": false, "type": "task", "reference_type": null, "reference_id": null, "conference_meeting_client": null, "conference_meeting_url": null, "due_date": "2021-07-06", "due_time": "", "duration": "", "busy_flag": true, "add_time": "2021-07-06 15:02:04", "marked_as_done_time": "", "last_notification_time": null, "last_notification_user_id": null, "notification_language_id": 1, "subject": "Task1", "public_description": "", "calendar_sync_include_context": null, "location": null, "org_id": 1, "person_id": 1, "deal_id": 10, "lead_id": null, "active_flag": true, "update_time": "2021-07-06 15:02:03", "update_user_id": null, "source_timezone": null, "rec_rule": null, "rec_rule_extension": null, "rec_master_activity_id": null, "conference_meeting_id": null, "original_start_time": null, "private": false, "note": "Note text", "created_by_user_id": 11884360, "location_subpremise": null, "location_street_number": null, "location_route": null, "location_sublocality": null, "location_locality": null, "location_admin_area_level_1": null, "location_admin_area_level_2": null, "location_country": null, "location_postal_code": null, "location_formatted_address": null, "attendees": null, "participants": [{"person_id": 1, "primary_flag": true}], "series": null, "is_recurring": null, "org_name": "Test Organization1", "person_name": "Test Person1", "deal_title": "Test Organization1 deal (copy) (copy) (copy) (copy) (copy) (copy) (copy) (copy) (copy)", "lead_title": null, "owner_name": "Team Airbyte", "person_dropbox_bcc": "airbyte-sandbox@pipedrivemail.com", "deal_dropbox_bcc": "airbyte-sandbox+deal10@pipedrivemail.com", "assigned_to_user_id": 11884360, "type_name": "Task", "lead": null}, "emitted_at": 1690801204738} -{"stream": "activities", "data": {"id": 3, "company_id": 7780468, "user_id": 11884360, "done": true, "type": "deadline", "reference_type": null, "reference_id": null, "conference_meeting_client": null, "conference_meeting_url": null, "due_date": "2021-07-28", "due_time": "15:15", "duration": "00:30", "busy_flag": true, "add_time": "2021-07-06 15:03:31", "marked_as_done_time": "2023-02-22 08:25:45", "last_notification_time": null, "last_notification_user_id": null, "notification_language_id": 1, "subject": "Deadline1", "public_description": "", "calendar_sync_include_context": null, "location": null, "org_id": 1, "person_id": 1, "deal_id": 1, "lead_id": null, "active_flag": true, "update_time": "2023-02-22 08:25:49", "update_user_id": 11884360, "source_timezone": null, "rec_rule": null, "rec_rule_extension": null, "rec_master_activity_id": null, "conference_meeting_id": null, "original_start_time": null, "private": false, "note": "New note text", "created_by_user_id": 11884360, "location_subpremise": null, "location_street_number": null, "location_route": null, "location_sublocality": null, "location_locality": null, "location_admin_area_level_1": null, "location_admin_area_level_2": null, "location_country": null, "location_postal_code": null, "location_formatted_address": null, "attendees": null, "participants": [{"person_id": 1, "primary_flag": true}], "series": null, "is_recurring": null, "org_name": "Test Organization1", "person_name": "Test Person1", "deal_title": "Test Organization1 deal", "lead_title": null, "owner_name": "Team Airbyte", "person_dropbox_bcc": "airbyte-sandbox@pipedrivemail.com", "deal_dropbox_bcc": "airbyte-sandbox+deal1@pipedrivemail.com", "assigned_to_user_id": 11884360, "type_name": "Deadline", "lead": null}, "emitted_at": 1690801204739} -{"stream": "activities", "data": {"id": 7, "company_id": 7780468, "user_id": 11884360, "done": false, "type": "call", "reference_type": null, "reference_id": null, "conference_meeting_client": null, "conference_meeting_url": null, "due_date": "2023-02-22", "due_time": "12:30", "duration": "01:00", "busy_flag": true, "add_time": "2023-02-22 08:52:52", "marked_as_done_time": "", "last_notification_time": null, "last_notification_user_id": null, "notification_language_id": null, "subject": "Call", "public_description": "", "calendar_sync_include_context": null, "location": null, "org_id": 3, "person_id": 7, "deal_id": null, "lead_id": "2ee8eaf0-b28e-11ed-8d6f-01ef91b3ff15", "active_flag": true, "update_time": "2023-02-22 08:52:52", "update_user_id": null, "source_timezone": null, "rec_rule": null, "rec_rule_extension": null, "rec_master_activity_id": null, "conference_meeting_id": null, "original_start_time": null, "private": false, "note": "Test call
      ", "created_by_user_id": 11884360, "location_subpremise": null, "location_street_number": null, "location_route": null, "location_sublocality": null, "location_locality": null, "location_admin_area_level_1": null, "location_admin_area_level_2": null, "location_country": null, "location_postal_code": null, "location_formatted_address": null, "attendees": null, "participants": [{"person_id": 7, "primary_flag": true}, {"person_id": 9, "primary_flag": false}], "series": null, "is_recurring": null, "org_name": "Test Organization 3", "person_name": "User6 Sample", "deal_title": null, "lead_title": "Test Organization 3 lead", "owner_name": "Team Airbyte", "person_dropbox_bcc": "airbyte-sandbox@pipedrivemail.com", "deal_dropbox_bcc": null, "assigned_to_user_id": 11884360, "type_name": "Call", "lead": {"id": "2ee8eaf0-b28e-11ed-8d6f-01ef91b3ff15", "title": "Test Organization 3 lead", "labels": "aecece60-c069-11eb-93bf-b59c4f1731e6", "source": "Manually created", "owner_id": 11884360, "creator_user_id": 11884360, "deal_id": null, "related_person_id": 4, "related_org_id": 2, "person_name": null, "person_phone": null, "person_email": null, "org_name": null, "org_address": null, "active_flag": true, "add_time": "2023-02-22 08:52:06.047", "update_time": "2023-02-22 11:44:31.718", "archive_time": null, "seen": true, "deal_value": 1200, "deal_currency": "USD", "next_activity_id": 7, "next_activity_date": "2023-02-22", "next_activity_time": "12:30:00", "visible_to": 3, "source_reference_id": null, "user": null, "deal_expected_close_date": "2023-05-15", "next_activity_status": null, "next_activity_datetime": null, "befdcfc4f54b8410b8d9105ba6d44658fd5965b9": null, "3ce5b1409718d65a8adc965be1f0da8821d8b9ab": null, "26da5bd3c09d3a700b15c23fb2f33a0b798c9d66": null, "eae9c2a5b618934581aebed0747cc33cd681379e": null, "bed1d9f4cfdaf761fab04b38df20144b0fd156d6": null, "41505adc22569bf93214dd7f7eaa10eaa387947d": null}}, "emitted_at": 1690801204739} -{"stream": "activity_fields", "data": {"id": 1, "key": "id", "name": "ID", "order_nr": 1, "field_type": "int", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": false, "filtering_allowed": true, "sortable_flag": true, "mandatory_flag": true, "active_flag": true, "index_visible_flag": true, "searchable_flag": false}, "emitted_at": 1690801205420} -{"stream": "activity_fields", "data": {"id": 2, "key": "subject", "name": "Subject", "order_nr": 2, "field_type": "varchar", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": true, "add_visible_flag": true, "important_flag": false, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "mandatory_flag": true, "active_flag": true, "index_visible_flag": true, "searchable_flag": false}, "emitted_at": 1690801205420} -{"stream": "activity_fields", "data": {"id": 3, "key": "type", "name": "Type", "order_nr": 3, "field_type": "enum", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": true, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "mandatory_flag": true, "options": [{"id": "call", "label": "Call"}, {"id": "meeting", "label": "Meeting"}, {"id": "task", "label": "Task"}, {"id": "deadline", "label": "Deadline"}, {"id": "email", "label": "Email"}, {"id": "lunch", "label": "Lunch"}, {"id": "test_1", "label": "Test 1"}], "active_flag": true, "index_visible_flag": true, "searchable_flag": false}, "emitted_at": 1690801205421} -{"stream": "activity_types", "data": {"id": 7, "order_nr": 7, "name": "Test 1", "key_string": "test_1", "icon_key": "car", "active_flag": true, "color": null, "is_custom_flag": true, "add_time": "2023-02-22 11:13:54", "update_time": "2023-02-22 11:13:54"}, "emitted_at": 1690801206161} -{"stream": "currencies", "data": {"id": 2, "code": "AFN", "name": "Afghanistan Afghani", "symbol": "AFN", "decimal_points": 2, "active_flag": true, "is_custom_flag": false}, "emitted_at": 1690801206834} -{"stream": "currencies", "data": {"id": 3, "code": "ALL", "name": "Albanian Lek", "symbol": "ALL", "decimal_points": 2, "active_flag": true, "is_custom_flag": false}, "emitted_at": 1690801206834} -{"stream": "currencies", "data": {"id": 41, "code": "DZD", "name": "Algerian Dinar", "symbol": "DZD", "decimal_points": 2, "active_flag": true, "is_custom_flag": false}, "emitted_at": 1690801206835} -{"stream": "deals", "data": {"id": 11, "creator_user_id": 11884360, "user_id": 11884360, "person_id": 10, "org_id": 7, "stage_id": 1, "title": "Test Organization 2 deal", "value": 1500, "currency": "USD", "add_time": "2023-02-22 08:27:53", "update_time": "2023-02-22 08:27:54", "stage_change_time": null, "active": true, "deleted": false, "status": "open", "probability": null, "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "lost_reason": null, "visible_to": "3", "close_time": null, "pipeline_id": 1, "won_time": null, "first_won_time": null, "lost_time": null, "products_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "email_messages_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "participants_count": 1, "expected_close_date": "2023-02-28", "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": null, "stage_order_nr": 0, "person_name": "User9 Sample", "org_name": "Test Organization 7", "next_activity_subject": null, "next_activity_type": null, "next_activity_duration": null, "next_activity_note": null, "formatted_value": "$1,500", "weighted_value": 1500, "formatted_weighted_value": "$1,500", "weighted_value_currency": "USD", "rotten_time": null, "owner_name": "Team Airbyte", "cc_email": "airbyte-sandbox+deal11@pipedrivemail.com", "befdcfc4f54b8410b8d9105ba6d44658fd5965b9": null, "3ce5b1409718d65a8adc965be1f0da8821d8b9ab": null, "26da5bd3c09d3a700b15c23fb2f33a0b798c9d66": null, "eae9c2a5b618934581aebed0747cc33cd681379e": null, "bed1d9f4cfdaf761fab04b38df20144b0fd156d6": null, "41505adc22569bf93214dd7f7eaa10eaa387947d": null, "org_hidden": false, "person_hidden": false}, "emitted_at": 1690801207614} -{"stream": "deals", "data": {"id": 13, "creator_user_id": 11884360, "user_id": 11884360, "person_id": 8, "org_id": 7, "stage_id": 1, "title": "Test Organization 3 deal", "value": 1200, "currency": "USD", "add_time": "2023-02-22 08:53:17", "update_time": "2023-02-22 08:53:18", "stage_change_time": null, "active": true, "deleted": false, "status": "open", "probability": null, "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "lost_reason": null, "visible_to": "3", "close_time": null, "pipeline_id": 1, "won_time": null, "first_won_time": null, "lost_time": null, "products_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "email_messages_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "participants_count": 1, "expected_close_date": "2023-03-31", "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": null, "stage_order_nr": 0, "person_name": "User7 Sample", "org_name": "Test Organization 7", "next_activity_subject": null, "next_activity_type": null, "next_activity_duration": null, "next_activity_note": null, "formatted_value": "$1,200", "weighted_value": 1200, "formatted_weighted_value": "$1,200", "weighted_value_currency": "USD", "rotten_time": null, "owner_name": "Team Airbyte", "cc_email": "airbyte-sandbox+deal13@pipedrivemail.com", "befdcfc4f54b8410b8d9105ba6d44658fd5965b9": null, "3ce5b1409718d65a8adc965be1f0da8821d8b9ab": null, "26da5bd3c09d3a700b15c23fb2f33a0b798c9d66": null, "eae9c2a5b618934581aebed0747cc33cd681379e": null, "bed1d9f4cfdaf761fab04b38df20144b0fd156d6": null, "41505adc22569bf93214dd7f7eaa10eaa387947d": null, "org_hidden": false, "person_hidden": false}, "emitted_at": 1690801207615} -{"stream": "deals", "data": {"id": 14, "creator_user_id": 11884360, "user_id": 11884360, "person_id": 5, "org_id": 1, "stage_id": 1, "title": "Test Organization 3 deal", "value": 1500, "currency": "USD", "add_time": "2023-02-22 08:54:48", "update_time": "2023-02-22 08:54:49", "stage_change_time": null, "active": true, "deleted": false, "status": "open", "probability": null, "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "lost_reason": null, "visible_to": "3", "close_time": null, "pipeline_id": 1, "won_time": null, "first_won_time": null, "lost_time": null, "products_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "email_messages_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "participants_count": 1, "expected_close_date": "2023-04-30", "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": null, "stage_order_nr": 0, "person_name": "User2 Sample", "org_name": "Test Organization1", "next_activity_subject": null, "next_activity_type": null, "next_activity_duration": null, "next_activity_note": null, "formatted_value": "$1,500", "weighted_value": 1500, "formatted_weighted_value": "$1,500", "weighted_value_currency": "USD", "rotten_time": null, "owner_name": "Team Airbyte", "cc_email": "airbyte-sandbox+deal14@pipedrivemail.com", "befdcfc4f54b8410b8d9105ba6d44658fd5965b9": null, "3ce5b1409718d65a8adc965be1f0da8821d8b9ab": null, "26da5bd3c09d3a700b15c23fb2f33a0b798c9d66": null, "eae9c2a5b618934581aebed0747cc33cd681379e": null, "bed1d9f4cfdaf761fab04b38df20144b0fd156d6": null, "41505adc22569bf93214dd7f7eaa10eaa387947d": null, "org_hidden": false, "person_hidden": false}, "emitted_at": 1690801207615} -{"stream": "deal_products", "data": {"id": 11, "deal_id": 17, "product_id": 2, "product_variation_id": null, "name": "Item 1", "order_nr": 1, "item_price": 100, "quantity": 6, "discount_percentage": 0, "duration": 1, "duration_unit": null, "sum_no_discount": 0, "sum": 600, "currency": "USD", "active_flag": true, "enabled_flag": true, "add_time": "2023-02-22 08:59:11", "last_edit": "2023-02-22 08:59:11", "comments": null, "tax": 0, "quantity_formatted": "6", "sum_formatted": "$600", "tax_method": "inclusive", "discount": 0, "discount_type": "percentage", "product": null}, "emitted_at": 1690801209362} -{"stream": "deal_products", "data": {"id": 12, "deal_id": 20, "product_id": 7, "product_variation_id": null, "name": "Item 6", "order_nr": 1, "item_price": 100, "quantity": 30, "discount_percentage": 0, "duration": 1, "duration_unit": null, "sum_no_discount": 0, "sum": 3000, "currency": "USD", "active_flag": true, "enabled_flag": true, "add_time": "2023-02-22 10:24:57", "last_edit": "2023-02-22 10:24:57", "comments": null, "tax": 0, "quantity_formatted": "30", "sum_formatted": "$3,000", "tax_method": "inclusive", "discount": 0, "discount_type": "percentage", "product": null}, "emitted_at": 1690801209809} -{"stream": "deal_products", "data": {"id": 15, "deal_id": 23, "product_id": 11, "product_variation_id": null, "name": "Item 10", "order_nr": 1, "item_price": 20, "quantity": 120, "discount_percentage": 0, "duration": 1, "duration_unit": null, "sum_no_discount": 0, "sum": 2400, "currency": "USD", "active_flag": true, "enabled_flag": true, "add_time": "2023-02-22 10:30:01", "last_edit": "2023-02-22 10:30:01", "comments": null, "tax": 0.01, "quantity_formatted": "120", "sum_formatted": "$2,400", "tax_method": "inclusive", "discount": 0, "discount_type": "percentage", "product": null}, "emitted_at": 1690801210358} -{"stream": "deal_fields", "data": {"id": 12477, "key": "id", "name": "ID", "group_id": null, "order_nr": 0, "field_type": "int", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": false, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "projects_detail_visible_flag": false, "show_in_pipelines": {"show_in_all": true, "pipeline_ids": []}, "mandatory_flag": true}, "emitted_at": 1690801212684} -{"stream": "deal_fields", "data": {"id": 12453, "key": "title", "name": "Title", "group_id": null, "order_nr": 0, "field_type": "varchar", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2023-07-20 09:24:02", "last_updated_by_user_id": 0, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": true, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "projects_detail_visible_flag": false, "show_in_pipelines": {"show_in_all": true, "pipeline_ids": []}, "use_field": "id", "link": "/deal/", "mandatory_flag": true}, "emitted_at": 1690801212685} -{"stream": "deal_fields", "data": {"id": 12454, "key": "creator_user_id", "name": "Creator", "group_id": null, "order_nr": 0, "field_type": "user", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": false, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "projects_detail_visible_flag": false, "show_in_pipelines": {"show_in_all": true, "pipeline_ids": []}, "mandatory_flag": true}, "emitted_at": 1690801212685} -{"stream": "files", "data": {"id": 1, "user_id": 11884360, "log_id": null, "add_time": "2023-02-22 10:52:23", "update_time": "2023-02-22 10:52:23", "file_name": "bd7cc15e-e7a5-4767-b643-63519ac9288b.png", "file_size": 6132, "active_flag": true, "inline_flag": false, "remote_location": "s3", "remote_id": "company/7780468/user/11884360/files/bd7cc15e-e7a5-4767-b643-63519ac9288b.png", "s3_bucket": null, "url": "https://app.pipedrive.com/api/v1/files/1/download", "name": "Airbyte logo 75x75.png", "description": null, "deal_id": null, "lead_id": null, "person_id": null, "org_id": 1, "product_id": null, "activity_id": null, "deal_name": null, "lead_name": null, "person_name": null, "people_name": null, "org_name": "Test Organization1", "product_name": null, "mail_message_id": null, "mail_template_id": null, "cid": null, "file_type": "img"}, "emitted_at": 1690801213534} -{"stream": "filters", "data": {"id": 29, "name": "Deal Update time is exactly or later than last quarter", "active_flag": true, "type": "deals", "temporary_flag": null, "user_id": 11884360, "add_time": "2021-07-07 13:27:24", "update_time": "2021-07-07 13:27:24", "visible_to": "7", "custom_view_id": null}, "emitted_at": 1690801214182} -{"stream": "filters", "data": {"id": 28, "name": "Deal Update time is exactly or later than last quarter", "active_flag": true, "type": "deals", "temporary_flag": null, "user_id": 11884360, "add_time": "2021-07-07 13:27:24", "update_time": "2021-07-07 13:27:24", "visible_to": "7", "custom_view_id": null}, "emitted_at": 1690801214183} -{"stream": "lead_labels", "data": {"id": "aecece60-c069-11eb-93bf-b59c4f1731e6", "name": "Hot", "color": "red", "add_time": "2021-05-29T10:36:10.182Z", "update_time": "2021-05-29T10:36:10.182Z"}, "emitted_at": 1690801214865} -{"stream": "lead_labels", "data": {"id": "aecece61-c069-11eb-93bf-b59c4f1731e6", "name": "Warm", "color": "yellow", "add_time": "2021-05-29T10:36:10.182Z", "update_time": "2021-05-29T10:36:10.182Z"}, "emitted_at": 1690801214866} -{"stream": "lead_labels", "data": {"id": "aecece62-c069-11eb-93bf-b59c4f1731e6", "name": "Cold", "color": "blue", "add_time": "2021-05-29T10:36:10.182Z", "update_time": "2021-05-29T10:36:10.182Z"}, "emitted_at": 1690801214866} -{"stream": "leads", "data": {"id": "7220c7f0-de6b-11eb-a544-5391b24dc0cf", "title": "Airbyte1 lead", "owner_id": 11884360, "creator_id": 11884360, "label_ids": ["aecece60-c069-11eb-93bf-b59c4f1731e6"], "value": {"amount": 10000, "currency": "USD"}, "expected_close_date": "2023-05-16", "person_id": 2, "organization_id": null, "is_archived": false, "source_name": "Manually created", "was_seen": true, "next_activity_id": 24, "add_time": "2021-07-06T15:04:22.255Z", "update_time": "2023-02-22T11:46:49.844Z", "visible_to": "3", "cc_email": "airbyte-sandbox+7780468+leadf6oUYVtZxhWEEyLegJqsRe@pipedrivemail.com"}, "emitted_at": 1690801215787} -{"stream": "leads", "data": {"id": "918d8510-de6b-11eb-8042-15d11dd01d20", "title": "Test Organization1 lead", "owner_id": 11884360, "creator_id": 11884360, "label_ids": ["aecece61-c069-11eb-93bf-b59c4f1731e6"], "value": {"amount": 3333, "currency": "USD"}, "expected_close_date": "2023-04-29", "person_id": 3, "organization_id": 1, "is_archived": false, "source_name": "Manually created", "was_seen": true, "next_activity_id": 25, "add_time": "2021-07-06T15:05:14.977Z", "update_time": "2023-02-22T11:47:18.002Z", "befdcfc4f54b8410b8d9105ba6d44658fd5965b9": 10, "3ce5b1409718d65a8adc965be1f0da8821d8b9ab": "Test", "visible_to": "3", "cc_email": "airbyte-sandbox+7780468+leadiYsMXTdfCQEKU896kY5xDA@pipedrivemail.com"}, "emitted_at": 1690801215788} -{"stream": "leads", "data": {"id": "bcfcf9c0-b28a-11ed-a6b1-df90b19da35a", "title": "Test Organization 2 lead", "owner_id": 11884360, "creator_id": 11884360, "label_ids": ["aecece60-c069-11eb-93bf-b59c4f1731e6"], "value": {"amount": 15000, "currency": "USD"}, "expected_close_date": "2023-04-06", "person_id": 9, "organization_id": 9, "is_archived": false, "source_name": "Manually created", "was_seen": true, "next_activity_id": 26, "add_time": "2023-02-22T08:27:26.428Z", "update_time": "2023-02-22T11:47:36.779Z", "visible_to": "3", "cc_email": "airbyte-sandbox+7780468+leadpkxTNcD2sbWtbcQKvFKfu3@pipedrivemail.com"}, "emitted_at": 1690801215788} -{"stream": "notes", "data": {"id": 1, "user_id": 11884360, "deal_id": null, "person_id": null, "org_id": 1, "lead_id": null, "content": "Test Note 1
      ", "add_time": "2023-02-22 08:24:35", "update_time": "2023-02-22 08:24:35", "active_flag": true, "pinned_to_deal_flag": false, "pinned_to_person_flag": false, "pinned_to_organization_flag": false, "pinned_to_lead_flag": false, "last_update_user_id": null, "organization": {"name": "Test Organization1"}, "person": null, "deal": null, "lead": null, "user": {"email": "integration-test@airbyte.io", "name": "Team Airbyte", "icon_url": null, "is_you": true}}, "emitted_at": 1690801216928} -{"stream": "notes", "data": {"id": 2, "user_id": 11884360, "deal_id": null, "person_id": null, "org_id": 2, "lead_id": null, "content": "Test note 2
      ", "add_time": "2023-02-22 08:31:47", "update_time": "2023-02-22 08:31:47", "active_flag": true, "pinned_to_deal_flag": false, "pinned_to_person_flag": false, "pinned_to_organization_flag": false, "pinned_to_lead_flag": false, "last_update_user_id": null, "organization": {"name": "Test Organization 2"}, "person": null, "deal": null, "lead": null, "user": {"email": "integration-test@airbyte.io", "name": "Team Airbyte", "icon_url": null, "is_you": true}}, "emitted_at": 1690801216929} -{"stream": "notes", "data": {"id": 3, "user_id": 11884360, "deal_id": null, "person_id": null, "org_id": 5, "lead_id": null, "content": "Test Note
      ", "add_time": "2023-02-22 08:59:20", "update_time": "2023-02-22 08:59:20", "active_flag": true, "pinned_to_deal_flag": false, "pinned_to_person_flag": false, "pinned_to_organization_flag": false, "pinned_to_lead_flag": false, "last_update_user_id": null, "organization": {"name": "Test Organization 5"}, "person": null, "deal": null, "lead": null, "user": {"email": "integration-test@airbyte.io", "name": "Team Airbyte", "icon_url": null, "is_you": true}}, "emitted_at": 1690801216929} -{"stream": "organizations", "data": {"id": 12, "company_id": 7780468, "owner_id": 11884360, "name": "Test Organization 7", "open_deals_count": 0, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "email_messages_count": 0, "people_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "category_id": null, "picture_id": null, "country_code": null, "first_char": "t", "update_time": "2023-02-22 08:21:58", "delete_time": null, "add_time": "2023-02-22 08:18:16", "visible_to": "3", "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "label": 5, "address": "DY Patil College, Sant Tukaram Nagar, Pimpri Colony, Pimpri-Chinchwad, Maharashtra, India", "address_subpremise": "", "address_street_number": "", "address_route": "", "address_sublocality": "Pimpri Colony", "address_locality": "Pimpri-Chinchwad", "address_admin_area_level_1": "Maharashtra", "address_admin_area_level_2": "Pune Division", "address_country": "India", "address_postal_code": "411018", "address_formatted_address": "DY Patil College, DR. D Y PATIL MEDICAL COLLEGE, Sant Tukaram Nagar, Pimpri Colony, Pimpri-Chinchwad, Maharashtra 411018, India", "cc_email": "airbyte-sandbox@pipedrivemail.com", "owner_name": "Team Airbyte"}, "emitted_at": 1690801217744} -{"stream": "organizations", "data": {"id": 13, "company_id": 7780468, "owner_id": 11884360, "name": "Test Organization 6", "open_deals_count": 0, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "email_messages_count": 0, "people_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "category_id": null, "picture_id": null, "country_code": null, "first_char": "t", "update_time": "2023-02-22 08:23:17", "delete_time": null, "add_time": "2023-02-22 08:18:33", "visible_to": "3", "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "label": 5, "address": "Anand Vihar Railway Station, Block D, Anand Vihar, Delhi, Uttar Pradesh, India", "address_subpremise": "", "address_street_number": "", "address_route": "", "address_sublocality": "Anand Vihar", "address_locality": "Delhi", "address_admin_area_level_1": "Uttar Pradesh", "address_admin_area_level_2": "Delhi Division", "address_country": "India", "address_postal_code": "261205", "address_formatted_address": "J8X8+F33, Block D, Anand Vihar, Delhi, Uttar Pradesh 261205, India", "cc_email": "airbyte-sandbox@pipedrivemail.com", "owner_name": "Team Airbyte"}, "emitted_at": 1690801217746} -{"stream": "organizations", "data": {"id": 3, "company_id": 7780468, "owner_id": 11884360, "name": "Test Organization 3", "open_deals_count": 1, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "email_messages_count": 0, "people_count": 0, "activities_count": 2, "done_activities_count": 0, "undone_activities_count": 2, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "category_id": null, "picture_id": null, "country_code": null, "first_char": "t", "update_time": "2023-02-22 08:56:24", "delete_time": null, "add_time": "2023-02-22 08:07:28", "visible_to": "3", "next_activity_date": "2023-02-22", "next_activity_time": "12:30:00", "next_activity_id": 7, "last_activity_id": null, "last_activity_date": null, "label": 6, "address": "Strasbourg, France", "address_subpremise": "", "address_street_number": "", "address_route": "", "address_sublocality": "", "address_locality": "Strasbourg", "address_admin_area_level_1": "Grand Est", "address_admin_area_level_2": "Bas-Rhin", "address_country": "France", "address_postal_code": "", "address_formatted_address": "Strasbourg, France", "cc_email": "airbyte-sandbox@pipedrivemail.com", "owner_name": "Team Airbyte"}, "emitted_at": 1690801217746} -{"stream": "organization_fields", "data": {"id": 4012, "key": "id", "name": "ID", "group_id": null, "order_nr": 0, "field_type": "int", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": false, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "mandatory_flag": true}, "emitted_at": 1690801218387} -{"stream": "organization_fields", "data": {"id": 4002, "key": "name", "name": "Name", "group_id": null, "order_nr": 0, "field_type": "varchar", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2023-07-20 09:24:05", "last_updated_by_user_id": 0, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": true, "important_flag": true, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "use_field": "id", "link": "/organization/", "mandatory_flag": true}, "emitted_at": 1690801218387} -{"stream": "organization_fields", "data": {"id": 4003, "key": "label", "name": "Label", "group_id": null, "order_nr": 0, "field_type": "enum", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2023-07-20 09:24:06", "last_updated_by_user_id": 0, "edit_flag": false, "details_visible_flag": true, "add_visible_flag": true, "important_flag": true, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "options": [{"id": 5, "label": "Customer", "color": "green"}, {"id": 6, "label": "Hot lead", "color": "red"}, {"id": 7, "label": "Warm lead", "color": "yellow"}, {"id": 8, "label": "Cold lead", "color": "blue"}], "mandatory_flag": false}, "emitted_at": 1690801218388} -{"stream": "permission_sets", "data": {"id": "79b42bf0-fb6d-11eb-a18f-a7e2db4435cf", "name": "Deals admin", "assignment_count": 1, "app": "sales", "type": "admin"}, "emitted_at": 1690801219068} -{"stream": "permission_sets", "data": {"id": "79b42bf1-fb6d-11eb-a18f-a7e2db4435cf", "name": "Deals regular user", "assignment_count": 4, "app": "sales", "type": "regular"}, "emitted_at": 1690801219069} -{"stream": "permission_sets", "data": {"id": "fa17fff0-db56-11ec-93f1-e9cfc58fcd59", "name": "Global admin", "assignment_count": 1, "app": "global", "type": "admin"}, "emitted_at": 1690801219069} -{"stream": "persons", "data": {"id": 5, "company_id": 7780468, "owner_id": 11884360, "org_id": 1, "name": "User2 Sample", "first_name": "User2", "last_name": "Sample", "open_deals_count": 2, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "participant_open_deals_count": 0, "participant_closed_deals_count": 0, "email_messages_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "phone": [{"label": "work", "value": "+14443332266", "primary": true}], "email": [{"label": "work", "value": "user2.sample.airbyte@gmail.com", "primary": true}], "first_char": "u", "update_time": "2023-02-22 10:44:49", "delete_time": null, "add_time": "2023-02-22 08:13:09", "visible_to": "3", "picture_id": null, "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": 3, "picture_128_url": null, "org_name": "Test Organization1", "aa02d059909fdc632d590bd578d7b3baf4bf9780": null, "owner_name": "Team Airbyte", "cc_email": "airbyte-sandbox@pipedrivemail.com"}, "emitted_at": 1690801219881} -{"stream": "persons", "data": {"id": 6, "company_id": 7780468, "owner_id": 11884360, "org_id": 1, "name": "User3 Sample", "first_name": "User3", "last_name": "Sample", "open_deals_count": 1, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "participant_open_deals_count": 0, "participant_closed_deals_count": 0, "email_messages_count": 0, "activities_count": 0, "done_activities_count": 0, "undone_activities_count": 0, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "phone": [{"label": "work", "value": "+15554442233", "primary": true}], "email": [{"label": "work", "value": "user3.sample.airbyte@outlook.com", "primary": true}], "first_char": "u", "update_time": "2023-02-22 10:45:52", "delete_time": null, "add_time": "2023-02-22 08:13:53", "visible_to": "3", "picture_id": null, "next_activity_date": null, "next_activity_time": null, "next_activity_id": null, "last_activity_id": null, "last_activity_date": null, "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": 2, "picture_128_url": null, "org_name": "Test Organization1", "aa02d059909fdc632d590bd578d7b3baf4bf9780": null, "owner_name": "Team Airbyte", "cc_email": "airbyte-sandbox@pipedrivemail.com"}, "emitted_at": 1690801219883} -{"stream": "persons", "data": {"id": 2, "company_id": 7780468, "owner_id": 11884360, "org_id": 11, "name": "User4 Sample", "first_name": "User4", "last_name": "Sample", "open_deals_count": 2, "related_open_deals_count": 0, "closed_deals_count": 0, "related_closed_deals_count": 0, "participant_open_deals_count": 0, "participant_closed_deals_count": 0, "email_messages_count": 0, "activities_count": 1, "done_activities_count": 0, "undone_activities_count": 1, "files_count": 0, "notes_count": 0, "followers_count": 1, "won_deals_count": 0, "related_won_deals_count": 0, "lost_deals_count": 0, "related_lost_deals_count": 0, "active_flag": true, "phone": [{"label": "work", "value": "+16665552233", "primary": true}], "email": [{"label": "work", "value": "user4.sample.airbyte@outlook.com", "primary": true}], "first_char": "u", "update_time": "2023-02-22 11:46:49", "delete_time": null, "add_time": "2021-07-06 15:04:21", "visible_to": "3", "picture_id": null, "next_activity_date": "2023-03-15", "next_activity_time": "11:30:00", "next_activity_id": 24, "last_activity_id": null, "last_activity_date": null, "last_incoming_mail_time": null, "last_outgoing_mail_time": null, "label": 1, "picture_128_url": null, "org_name": "Test Organization 4", "aa02d059909fdc632d590bd578d7b3baf4bf9780": null, "owner_name": "Team Airbyte", "cc_email": "airbyte-sandbox@pipedrivemail.com"}, "emitted_at": 1690801219883} -{"stream": "person_fields", "data": {"id": 9051, "key": "id", "name": "ID", "group_id": null, "order_nr": 0, "field_type": "int", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": false, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "mandatory_flag": true}, "emitted_at": 1690801220805} -{"stream": "person_fields", "data": {"id": 9039, "key": "name", "name": "Name", "group_id": null, "order_nr": 0, "field_type": "varchar", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2023-07-20 09:24:04", "last_updated_by_user_id": 0, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": true, "important_flag": true, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "use_field": "id", "link": "/person/", "mandatory_flag": true}, "emitted_at": 1690801220806} -{"stream": "person_fields", "data": {"id": 9040, "key": "label", "name": "Label", "group_id": null, "order_nr": 0, "field_type": "enum", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2023-07-20 09:24:04", "last_updated_by_user_id": 0, "edit_flag": false, "details_visible_flag": true, "add_visible_flag": true, "important_flag": true, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "options": [{"id": 1, "label": "Customer", "color": "green"}, {"id": 2, "label": "Hot lead", "color": "red"}, {"id": 3, "label": "Warm lead", "color": "yellow"}, {"id": 4, "label": "Cold lead", "color": "blue"}], "mandatory_flag": false}, "emitted_at": 1690801220806} -{"stream": "pipelines", "data": {"id": 2, "name": "New pipeline", "url_title": "New-pipeline", "order_nr": 2, "active": true, "deal_probability": true, "add_time": "2021-07-06 15:24:11", "update_time": "2021-07-06 15:24:11"}, "emitted_at": 1690801221622} -{"stream": "pipelines", "data": {"id": 3, "name": "New pipeline 2", "url_title": "New-pipeline-2", "order_nr": 3, "active": true, "deal_probability": true, "add_time": "2023-02-22 10:42:31", "update_time": "2023-02-22 10:42:31"}, "emitted_at": 1690801221623} -{"stream": "pipelines", "data": {"id": 4, "name": "New pipeline 3", "url_title": "New-pipeline-3", "order_nr": 4, "active": true, "deal_probability": true, "add_time": "2023-02-22 10:43:12", "update_time": "2023-02-22 10:43:12"}, "emitted_at": 1690801221623} -{"stream": "product_fields", "data": {"id": 23, "key": "id", "name": "ID", "group_id": null, "order_nr": 0, "field_type": "int", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2020-12-10 07:23:48", "last_updated_by_user_id": null, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": false, "important_flag": false, "bulk_edit_allowed": false, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "mandatory_flag": true}, "emitted_at": 1690801222343} -{"stream": "product_fields", "data": {"id": 17, "key": "name", "name": "Name", "group_id": null, "order_nr": 0, "field_type": "varchar", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2023-07-20 09:24:07", "last_updated_by_user_id": 0, "edit_flag": false, "details_visible_flag": false, "add_visible_flag": true, "important_flag": true, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "use_field": "id", "link": "/api/v1/products/", "mandatory_flag": true}, "emitted_at": 1690801222345} -{"stream": "product_fields", "data": {"id": 18, "key": "owner_id", "name": "Owner", "group_id": null, "order_nr": 0, "field_type": "user", "json_column_flag": false, "add_time": "2020-12-10 07:23:48", "update_time": "2023-07-20 09:24:07", "last_updated_by_user_id": 0, "edit_flag": false, "details_visible_flag": true, "add_visible_flag": false, "important_flag": true, "bulk_edit_allowed": true, "filtering_allowed": true, "sortable_flag": true, "searchable_flag": false, "active_flag": true, "use_field": "owner_id", "display_field": "owner_name", "mandatory_flag": true}, "emitted_at": 1690801222345} -{"stream": "products", "data": {"id": 1, "name": "Test Product", "code": "12345", "description": null, "unit": "Kg", "tax": 5, "category": "12", "active_flag": true, "selectable": true, "first_char": "t", "visible_to": "3", "owner_id": 11884360, "files_count": null, "add_time": "2021-11-09 11:32:16", "update_time": "2021-11-09 11:32:31", "prices": [{"id": 1, "product_id": 1, "price": 10, "currency": "USD", "cost": 0, "overhead_cost": null}], "product_variations": [], "owner_name": "Team Airbyte"}, "emitted_at": 1690801223125} -{"stream": "products", "data": {"id": 2, "name": "Item 1", "code": null, "description": null, "unit": null, "tax": 0, "category": null, "active_flag": true, "selectable": true, "first_char": "i", "visible_to": "3", "owner_id": 11884360, "files_count": null, "add_time": "2023-02-22 08:31:13", "update_time": "2023-02-22 08:31:13", "prices": [{"id": 2, "product_id": 2, "price": 0, "currency": "USD", "cost": 0, "overhead_cost": null}], "product_variations": [], "owner_name": "Team Airbyte"}, "emitted_at": 1690801223126} -{"stream": "products", "data": {"id": 3, "name": "Item 2", "code": null, "description": null, "unit": null, "tax": 0, "category": null, "active_flag": true, "selectable": true, "first_char": "i", "visible_to": "3", "owner_id": 11884360, "files_count": null, "add_time": "2023-02-22 08:31:14", "update_time": "2023-02-22 08:31:14", "prices": [{"id": 3, "product_id": 3, "price": 0, "currency": "USD", "cost": 0, "overhead_cost": null}], "product_variations": [], "owner_name": "Team Airbyte"}, "emitted_at": 1690801223126} -{"stream": "roles", "data": {"id": 1, "parent_role_id": null, "name": "(Unassigned users)", "active_flag": true, "assignment_count": "5", "sub_role_count": "0", "level": 1, "description": "This is the default group for managing your visibility settings. New users are added automatically unless you change their group when you invite them."}, "emitted_at": 1690801224387} -{"stream": "stages", "data": {"id": 11, "order_nr": 5, "name": "Negotiations Started", "active_flag": true, "deal_probability": 100, "pipeline_id": 2, "rotten_flag": false, "rotten_days": null, "add_time": "2021-07-06 15:24:11", "update_time": "2023-02-22 10:38:51", "pipeline_name": "New pipeline", "pipeline_deal_probability": true}, "emitted_at": 1690801225072} -{"stream": "stages", "data": {"id": 10, "order_nr": 4, "name": "Proposal Made", "active_flag": true, "deal_probability": 100, "pipeline_id": 2, "rotten_flag": false, "rotten_days": null, "add_time": "2021-07-06 15:24:11", "update_time": "2023-02-22 10:38:51", "pipeline_name": "New pipeline", "pipeline_deal_probability": true}, "emitted_at": 1690801225072} -{"stream": "stages", "data": {"id": 9, "order_nr": 3, "name": "Demo Scheduled", "active_flag": true, "deal_probability": 100, "pipeline_id": 2, "rotten_flag": false, "rotten_days": null, "add_time": "2021-07-06 15:24:11", "update_time": "2023-02-22 10:38:51", "pipeline_name": "New pipeline", "pipeline_deal_probability": true}, "emitted_at": 1690801225072} -{"stream": "users", "data": {"id": 18276123, "name": "User3 Sample", "default_currency": "USD", "timezone_name": "Europe/Kiev", "timezone_offset": "+03:00", "locale": "en_US", "email": "user3.sample.airbyte@outlook.com", "phone": null, "created": "2023-03-22 14:22:43", "modified": "2023-03-22 14:34:12", "lang": 1, "active_flag": true, "is_admin": 0, "last_login": "2023-03-22 14:34:12", "signup_flow_variation": "", "role_id": 1, "has_created_company": false, "icon_url": null, "is_you": false}, "emitted_at": 1690801226031} -{"stream": "users", "data": {"id": 18276134, "name": "User4 Sample", "default_currency": "USD", "timezone_name": "Europe/Kiev", "timezone_offset": "+03:00", "locale": "en_US", "email": "user4.sample.airbyte@outlook.com", "phone": null, "created": "2023-03-22 14:22:44", "modified": "2023-03-22 14:39:38", "lang": 1, "active_flag": true, "is_admin": 0, "last_login": "2023-03-22 14:39:38", "signup_flow_variation": "", "role_id": 1, "has_created_company": false, "icon_url": null, "is_you": false}, "emitted_at": 1690801226032} -{"stream": "users", "data": {"id": 18276145, "name": "User5 Sample", "default_currency": "USD", "timezone_name": "Europe/Kiev", "timezone_offset": "+03:00", "locale": "en_US", "email": "user5.sample.airbyte@outlook.com", "phone": null, "created": "2023-03-22 14:22:44", "modified": "2023-03-22 14:44:35", "lang": 1, "active_flag": true, "is_admin": 0, "last_login": "2023-03-22 14:44:35", "signup_flow_variation": "", "role_id": 1, "has_created_company": false, "icon_url": null, "is_you": false}, "emitted_at": 1690801226032} diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-pipedrive/integration_tests/integration_test.py deleted file mode 100644 index 5544269fad67..000000000000 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/integration_test.py +++ /dev/null @@ -1,7 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -def test_example(): - assert True diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-pipedrive/integration_tests/invalid_config.json index 44d3a2be1b95..7831cc4648f2 100644 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/invalid_config.json @@ -1,7 +1,4 @@ { - "authorization": { - "auth_type": "Token", - "api_token": "wrong-api-token" - }, - "replication_start_date": "2021-01-01T10:10:10Z" + "api_key": "api_key", + "replication_start_date": "2017-01-25 00:00:00Z" } diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-pipedrive/integration_tests/sample_config.json index 5d3a28f22a86..7831cc4648f2 100644 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/sample_config.json @@ -1,4 +1,4 @@ { - "api_token": "", - "replication_start_date": "2021-06-01T10:10:10Z" + "api_key": "api_key", + "replication_start_date": "2017-01-25 00:00:00Z" } diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-pipedrive/integration_tests/sample_state.json index e4a812b80467..d7ca6e9987cf 100644 --- a/airbyte-integrations/connectors/source-pipedrive/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/sample_state.json @@ -1,68 +1,35 @@ -[ - { - "type": "STREAM", - "stream": { - "stream_state": { - "update_time": "2021-06-01 10:10:10" - }, - "stream_descriptor": { - "name": "deals" - } - } +{ + "deals": { + "update_time": "2017-06-01 10:10:10" }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "update_time": "2021-06-01 10:10:10" - }, - "stream_descriptor": { - "name": "activities" - } - } + "files": { + "update_time": "2017-06-01 10:10:10" }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "update_time": "2021-06-01 10:10:10" - }, - "stream_descriptor": { - "name": "persons" - } - } + "filters": { + "update_time": "2017-06-01 10:10:10" }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "update_time": "2021-06-01 10:10:10" - }, - "stream_descriptor": { - "name": "pipelines" - } - } + "notes": { + "update_time": "2017-06-01 10:10:10" }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "update_time": "2021-06-01 10:10:10" - }, - "stream_descriptor": { - "name": "stages" - } - } + "activites": { + "update_time": "2017-06-01 10:10:10" }, - { - "type": "STREAM", - "stream": { - "stream_state": { - "modified": "2021-06-01 10:10:10" - }, - "stream_descriptor": { - "name": "users" - } - } + "oragnizations": { + "update_time": "2017-06-01 10:10:10" + }, + "persons": { + "update_time": "2017-06-01 10:10:10" + }, + "pipelines": { + "update_time": "2017-06-01 10:10:10" + }, + "products": { + "update_time": "2017-06-01 10:10:10" + }, + "stages": { + "update_time": "2017-06-01 10:10:10" + }, + "users": { + "update_time": "2017-06-01 10:10:10" } -] +} diff --git a/airbyte-integrations/connectors/source-pipedrive/main.py b/airbyte-integrations/connectors/source-pipedrive/main.py index fb481bc2e9b2..64fe456c34fd 100644 --- a/airbyte-integrations/connectors/source-pipedrive/main.py +++ b/airbyte-integrations/connectors/source-pipedrive/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_pipedrive import SourcePipedrive +from source_pipedrive.run import run if __name__ == "__main__": - source = SourcePipedrive() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-pipedrive/metadata.yaml b/airbyte-integrations/connectors/source-pipedrive/metadata.yaml index c3c4a3126288..5f0810ee4556 100644 --- a/airbyte-integrations/connectors/source-pipedrive/metadata.yaml +++ b/airbyte-integrations/connectors/source-pipedrive/metadata.yaml @@ -1,27 +1,33 @@ data: + ab_internal: + ql: 300 + sl: 100 allowedHosts: hosts: - - api.pipedrive.com + - api.pipedrive.com # Please change to the hostname of the source. + registries: + cloud: + enabled: true + oss: + enabled: true connectorSubtype: api connectorType: source definitionId: d8286229-c680-4063-8c59-23b9b391c700 - dockerImageTag: 1.0.0 + dockerImageTag: 2.2.2 dockerRepository: airbyte/source-pipedrive + documentationUrl: https://docs.airbyte.com/integrations/sources/pipedrive githubIssueLabel: source-pipedrive icon: pipedrive.svg license: MIT name: Pipedrive - registries: - cloud: - enabled: true - oss: - enabled: true - releaseStage: beta - documentationUrl: https://docs.airbyte.com/integrations/sources/pipedrive + releases: + breakingChanges: + 2.0.0: + upgradeDeadline: 2023-10-04 + message: "This version removes the `pipeline_ids` field from the `deal_fields` stream. Config has changed to only use API key. Please update your config." + releaseDate: 2021-07-19 + releaseStage: alpha + supportLevel: community tags: - - language:python - ab_internal: - sl: 200 - ql: 300 - supportLevel: certified + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-pipedrive/setup.py b/airbyte-integrations/connectors/source-pipedrive/setup.py index 7108dd89b957..5d3c1999b80c 100644 --- a/airbyte-integrations/connectors/source-pipedrive/setup.py +++ b/airbyte-integrations/connectors/source-pipedrive/setup.py @@ -6,21 +6,27 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", - "pendulum~=2.1", - "requests~=2.25", + "airbyte-cdk", ] -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6", "requests_mock~=1.8"] +TEST_REQUIREMENTS = [ + "pytest~=6.2", + "pytest-mock~=3.6.1", +] setup( + entry_points={ + "console_scripts": [ + "source-pipedrive=source_pipedrive.run:run", + ], + }, name="source_pipedrive", description="Source implementation for Pipedrive.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/__init__.py b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/__init__.py index 7e21a5c1b775..a2d5a962fa36 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/__init__.py +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/__init__.py @@ -1,26 +1,7 @@ -""" -MIT License +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" from .source import SourcePipedrive diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/auth.py b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/auth.py deleted file mode 100644 index 97a7a9ae6081..000000000000 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/auth.py +++ /dev/null @@ -1,19 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from requests.auth import AuthBase - - -class QueryStringTokenAuthenticator(AuthBase): - """ - Authenticator that attaches a set of query string parameters (e.g. an API key) to the request. - """ - - def __init__(self, **kwargs): - self.params = kwargs - - def __call__(self, request): - if self.params: - request.prepare_url(request.url, self.params) - return request diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/extractor.py b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/extractor.py new file mode 100644 index 000000000000..961438884da6 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/extractor.py @@ -0,0 +1,55 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from dataclasses import InitVar, dataclass +from typing import Any, List, Mapping, Union + +import requests +from airbyte_cdk.sources.declarative.decoders.decoder import Decoder +from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder +from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor +from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.types import Config + + +@dataclass +class NullCheckedDpathExtractor(RecordExtractor): + """ + Pipedrive requires a custom extractor because the format of its API responses is inconsistent. + + Records are typically found in a nested "data" field, but sometimes the "data" field is null. + This extractor checks for null "data" fields and returns the parent object, which contains the record ID, instead. + + Example faulty records: + ``` + { + "item": "file", + "id": , + "data": null + }, + { + "item": "file", + "id": , + "data": null + } + ``` + """ + + field_path: List[Union[InterpolatedString, str]] + nullable_nested_field: Union[InterpolatedString, str] + config: Config + parameters: InitVar[Mapping[str, Any]] + decoder: Decoder = JsonDecoder(parameters={}) + + def __post_init__(self, parameters: Mapping[str, Any]): + self._dpath_extractor = DpathExtractor( + field_path=self.field_path, + config=self.config, + parameters=parameters, + decoder=self.decoder, + ) + + def extract_records(self, response: requests.Response) -> List[Mapping[str, Any]]: + records = self._dpath_extractor.extract_records(response) + return [record.get(self.nullable_nested_field) or record for record in records] diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/manifest.yaml b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/manifest.yaml new file mode 100644 index 000000000000..8b02277d3d67 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/manifest.yaml @@ -0,0 +1,413 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["data"] + + selector_increment: + type: RecordSelector + extractor: + class_name: source_pipedrive.extractor.NullCheckedDpathExtractor + field_path: ["data", "*"] + nullable_nested_field: data + + selector_users: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["data", "*", "data", "*"] + + requester: + type: HttpRequester + url_base: "https://api.pipedrive.com/v1" + http_method: "GET" + request_parameters: + api_token: "{{ config['api_token'] }}" + limit: "50" + items: "{{ parameters.path_extractor }}" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response['additional_data']['pagination']['next_start'] }}" + page_token_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "start" + requester: + $ref: "#/definitions/requester" + + retriever_increment: + $ref: "#/definitions/retriever" + record_selector: + $ref: "#/definitions/selector_increment" + + incremental_base: + type: DatetimeBasedCursor + cursor_field: "update_time" + datetime_format: "%Y-%m-%d %H:%M:%S" + start_datetime: + datetime: "{{ format_datetime(config['replication_start_date'], '%Y-%m-%d %H:%M:%S') }}" + datetime_format: "%Y-%m-%d %H:%M:%S" + start_time_option: + field_name: "since_timestamp" + inject_into: "request_parameter" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + deals_stream: + $ref: "#/definitions/base_stream" + name: "deals" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever_increment" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "deal" + path: "/recents" + + deal_fields_stream: + $ref: "#/definitions/base_stream" + name: "deal_fields" + $parameters: + path: "/dealFields" + + files_stream: + $ref: "#/definitions/base_stream" + name: "files" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever_increment" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "file" + path: "/recents" + + filters_stream: + $ref: "#/definitions/base_stream" + name: "filters" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever_increment" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "filter" + path: "/recents" + + lead_labels_stream: + # Does not support pagination + $ref: "#/definitions/base_stream" + name: "lead_labels" + retriever: + $ref: "#/definitions/retriever" + paginator: + type: NoPagination + $parameters: + path: "/leadLabels" + + leads_stream: + $ref: "#/definitions/base_stream" + name: "leads" + $parameters: + path: "/leads" + + goals_stream: + $ref: "#/definitions/base_stream" + name: "goals" + primary_key: "id" + $parameters: + path: "/goals/find" + retriever: + $ref: "#/definitions/retriever" + record_selector: + $ref: "#/definitions/selector" + extractor: + type: DpathExtractor + field_path: ["data", "goals"] + paginator: + type: "NoPagination" + + notes_stream: + $ref: "#/definitions/base_stream" + name: "notes" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever_increment" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "note" + path: "/recents" + + activities_stream: + $ref: "#/definitions/base_stream" + name: "activities" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever_increment" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "activity" + path: "/recents" + + activity_types_stream: + # This stream didnt have limit as query_parameter + $ref: "#/definitions/base_stream" + name: "activity_types" + retriever: + $ref: "#/definitions/retriever" + paginator: + type: NoPagination + $parameters: + path: "/activityTypes" + + activity_fields_stream: + $ref: "#/definitions/base_stream" + name: "activity_fields" + $parameters: + path: "/activityFields" + + currencies_stream: + $ref: "#/definitions/base_stream" + name: "currencies" + retriever: + $ref: "#/definitions/retriever" + paginator: + type: NoPagination + $parameters: + path: "/currencies" + + mail_stream: + $ref: "#/definitions/base_stream" + name: mail + primary_key: "id" + retriever: + $ref: "#/definitions/retriever" + partition_router: + - type: SubstreamPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: thread_id + stream: + $ref: "#/definitions/base_stream" + name: mailthreads + primary_key: "id" + retriever: + $ref: "#/definitions/retriever" + partition_router: + - type: ListPartitionRouter + values: + - inbox + - drafts + - sent + - archive + cursor_field: folder + request_option: + inject_into: request_parameter + type: RequestOption + field_name: folder + $parameters: + path: "mailbox/mailThreads" + $parameters: + path: "mailbox/mailThreads/{{ stream_partition.thread_id }}/mailMessages" + + organization_stream: + $ref: "#/definitions/base_stream" + name: "organizations" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever" + $parameters: + path_extractor: "organization" + path: "/organizations" + + organization_fields_stream: + $ref: "#/definitions/base_stream" + name: "organization_fields" + $parameters: + path: "/organizationFields" + + permission_sets_stream: + $ref: "#/definitions/base_stream" + name: "permission_sets" + retriever: + $ref: "#/definitions/retriever" + paginator: + type: NoPagination + $parameters: + path: "/permissionSets" + + persons_stream: + $ref: "#/definitions/base_stream" + name: "persons" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever_increment" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "person" + path: "/recents" + + person_fields_stream: + $ref: "#/definitions/base_stream" + name: "person_fields" + $parameters: + path: "/personFields" + + pipelines_stream: + $ref: "#/definitions/base_stream" + name: "pipelines" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever_increment" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "pipeline" + path: "/recents" + + products_stream: + $ref: "#/definitions/base_stream" + name: "products" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever_increment" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "product" + path: "/recents" + + product_fields_stream: + $ref: "#/definitions/base_stream" + name: "product_fields" + $parameters: + path: "/productFields" + + roles_stream: + $ref: "#/definitions/base_stream" + name: "roles" + $parameters: + path: "/roles" + + stages_stream: + $ref: "#/definitions/base_stream" + name: "stages" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever_increment" + incremental_sync: + $ref: "#/definitions/incremental_base" + $parameters: + path_extractor: "stage" + path: "/recents" + + users_stream: + $ref: "#/definitions/base_stream" + name: "users" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever" + record_selector: + $ref: "#/definitions/selector_users" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: "modified" + datetime_format: "%Y-%m-%d %H:%M:%S" + start_datetime: + datetime: "{{ format_datetime(config['replication_start_date'], '%Y-%m-%d %H:%M:%S') }}" + datetime_format: "%Y-%m-%d %H:%M:%S" + start_time_option: + field_name: "since_timestamp" + inject_into: "request_parameter" + $parameters: + path_extractor: "user" + path: "/recents" + + deal_products_stream: + name: "deal_products" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + path: "deals/{{ stream_slice.parent_id }}/products" + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/deals_stream" + parent_key: "id" + partition_field: "parent_id" + +streams: + - "#/definitions/deals_stream" + - "#/definitions/deal_fields_stream" + - "#/definitions/files_stream" + - "#/definitions/filters_stream" + - "#/definitions/lead_labels_stream" + - "#/definitions/leads_stream" + - "#/definitions/notes_stream" + - "#/definitions/activities_stream" + - "#/definitions/activity_types_stream" + - "#/definitions/activity_fields_stream" + - "#/definitions/currencies_stream" + - "#/definitions/mail_stream" + - "#/definitions/organization_stream" + - "#/definitions/organization_fields_stream" + - "#/definitions/permission_sets_stream" + - "#/definitions/persons_stream" + - "#/definitions/person_fields_stream" + - "#/definitions/pipelines_stream" + - "#/definitions/products_stream" + - "#/definitions/product_fields_stream" + - "#/definitions/roles_stream" + - "#/definitions/stages_stream" + - "#/definitions/users_stream" + - "#/definitions/deal_products_stream" + - "#/definitions/goals_stream" + +check: + type: CheckStream + stream_names: + - "deals" + +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/pipedrive + connection_specification: + title: Pipedrive Spec + type: object + required: + - replication_start_date + - api_token + additionalProperties: true + properties: + api_token: + title: API Token + type: string + description: The Pipedrive API Token. + airbyte_secret: true + replication_start_date: + title: Start Date + description: + UTC date and time in the format 2017-01-25T00:00:00Z. Any data + before this date will not be replicated. When specified and not None, then + stream will behave as incremental + examples: + - "2017-01-25 00:00:00Z" + type: string diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/run.py b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/run.py new file mode 100644 index 000000000000..2ff2b80c12a8 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pipedrive import SourcePipedrive + + +def run(): + source = SourcePipedrive() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/activity_fields.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/activity_fields.json index 6dd7a2100fa9..2901d9c336f4 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/activity_fields.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/activity_fields.json @@ -56,6 +56,18 @@ "sortable_flag": { "type": ["null", "boolean"] }, + "json_column_flag": { + "type": ["null", "boolean"] + }, + "parent_id": { + "type": ["null", "integer"] + }, + "id_suffix": { + "type": ["null", "string"] + }, + "is_subfield": { + "type": ["null", "boolean"] + }, "options": { "type": ["null", "array"], "items": { diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deal_fields.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deal_fields.json index f976f57b22f4..805265c21864 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deal_fields.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deal_fields.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] @@ -14,16 +15,28 @@ "order_nr": { "type": ["null", "integer"] }, + "group_id": { + "type": ["null", "integer"] + }, "field_type": { "type": ["null", "string"] }, + "projects_detail_visible_flag": { + "type": ["null", "boolean"] + }, + "json_column_flag": { + "type": ["null", "boolean"] + }, "add_time": { + "format": "%Y-%m-%d %H:%M:%S", "type": ["null", "string"] }, "update_time": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_without_timezone" + "format": "%Y-%m-%d %H:%M:%S", + "type": ["null", "string"] + }, + "use_field": { + "type": ["null", "string"] }, "last_updated_by_user_id": { "type": ["null", "integer"] @@ -58,25 +71,38 @@ "sortable_flag": { "type": ["null", "boolean"] }, + "parent_id": { + "type": ["null", "integer"] + }, + "id_suffix": { + "type": ["null", "string"] + }, + "link": { + "type": ["null", "string"] + }, + "is_subfield": { + "type": ["null", "boolean"] + }, "options": { "type": ["null", "array"], "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string", "integer"] - }, - "label": { - "type": ["null", "string"] - } + "type": "object", + "additionalProperties": true + } + }, + "show_in_pipelines": { + "type": ["null", "object"], + "items": { + "show_in_all": { + "type": ["null", "boolean"] + }, + "pipeline_ids": { + "type": ["null", "array"] } } }, "mandatory_flag": { - "type": ["null", "string", "boolean", "object"] - }, - "pipeline_ids": { - "type": ["null", "array"] + "type": ["null", "object", "boolean", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deals.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deals.json index 0c559e857070..364d42cfd04e 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deals.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deals.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/goals.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/goals.json new file mode 100644 index 000000000000..28047f879dfb --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/goals.json @@ -0,0 +1,91 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Goals schema", + "additionalProperties": true, + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string", "null"] + }, + "owner_id": { + "type": ["number", "null"] + }, + "title": { + "type": ["string", "null"] + }, + "type": { + "type": ["object", "null"], + "additionalProperties": true, + "properties": { + "name": { + "type": ["string", "null"] + }, + "params": { + "type": ["object", "null"], + "properties": { + "pipeline_id": { + "type": ["array", "null"], + "items": { + "type": ["number", "null"] + } + }, + "activity_type_id": { + "type": ["array", "null"], + "items": { + "type": ["number", "null"] + } + } + } + } + } + }, + "assignee": { + "type": ["object", "null"], + "additionalProperties": true, + "properties": { + "type": { + "type": ["string", "null"] + }, + "id": { + "type": ["number", "null"] + } + } + }, + "interval": { + "type": ["string", "null"] + }, + "duration": { + "type": ["object", "null"], + "additionalProperties": true, + "properties": { + "start": { + "type": ["string", "null"] + }, + "end": { + "type": ["string", "null"] + } + } + }, + "expected_outcome": { + "type": ["object", "null"], + "additionalProperties": true, + "properties": { + "target": { + "type": ["number", "null"] + }, + "tracking_metric": { + "type": ["string", "null"] + } + } + }, + "is_active": { + "type": "boolean" + }, + "report_ids": { + "type": ["array", "null"], + "items": { + "type": ["string", "null"] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/leads.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/leads.json index fb8aab138083..15384b9102e0 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/leads.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/leads.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/mail.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/mail.json new file mode 100644 index 000000000000..cfc2b64f26fd --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/mail.json @@ -0,0 +1,57 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { "type": ["null", "integer"] }, + "from": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "integer"] }, + "email_address": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "linked_person_id": { "type": ["null", "integer"] }, + "linked_person_name": { "type": ["null", "string"] }, + "mail_message_party_id": { "type": ["null", "integer"] } + } + } + }, + "to": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "integer"] }, + "email_address": { "type": ["null", "string"] }, + "name": { "type": ["null", "string"] }, + "linked_person_id": { "type": ["null", "integer"] }, + "linked_person_name": { "type": ["null", "string"] }, + "mail_message_party_id": { "type": ["null", "integer"] } + } + } + }, + "body_url": { "type": ["null", "string"] }, + "account_id": { "type": ["null", "string"] }, + "user_id": { "type": ["null", "integer"] }, + "mail_thread_id": { "type": ["null", "integer"] }, + "subject": { "type": ["null", "string"] }, + "snippet": { "type": ["null", "string"] }, + "mail_link_tracking_enabled_flag": { "type": ["null", "integer"] }, + "read_flag": { "type": ["null", "integer"] }, + "draft_flag": { "type": ["null", "integer"] }, + "synced_flag": { "type": ["null", "integer"] }, + "deleted_flag": { "type": ["null", "integer"] }, + "has_body_flag": { "type": ["null", "integer"] }, + "sent_flag": { "type": ["null", "integer"] }, + "sent_from_pipedrive_flag": { "type": ["null", "integer"] }, + "smart_bcc_flag": { "type": ["null", "integer"] }, + "message_time": { "type": ["null", "string"], "format": "date-time" }, + "add_time": { "type": ["null", "string"], "format": "date-time" }, + "update_time": { "type": ["null", "string"], "format": "date-time" }, + "has_attachments_flag": { "type": ["null", "integer"] }, + "has_inline_attachments_flag": { "type": ["null", "integer"] }, + "has_real_attachments_flag": { "type": ["null", "integer"] } + } +} diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organization_fields.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organization_fields.json index 257c0b7c54c7..ddf70b978863 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organization_fields.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organization_fields.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] @@ -58,6 +59,21 @@ "sortable_flag": { "type": ["null", "boolean"] }, + "parent_id": { + "type": ["null", "integer"] + }, + "id_suffix": { + "type": ["null", "string"] + }, + "is_subfield": { + "type": ["null", "boolean"] + }, + "json_column_flag": { + "type": ["null", "boolean"] + }, + "group_id": { + "type": ["null", "integer"] + }, "options": { "type": ["null", "array"], "items": { @@ -77,6 +93,15 @@ }, "display_name": { "type": ["null", "string"] + }, + "use_field": { + "type": ["null", "string"] + }, + "display_field": { + "type": ["null", "string"] + }, + "link": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organizations.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organizations.json index 5d6642fe0e3f..b5402958aae3 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organizations.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/organizations.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": ["null", "object"], + "additionalProperties": true, "properties": { "id": { "type": ["null", "number"] @@ -132,9 +133,7 @@ "type": ["null", "string"] }, "update_time": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_without_timezone" + "type": ["null", "string"] }, "add_time": { "type": ["null", "string"] @@ -198,6 +197,12 @@ }, "cc_email": { "type": ["null", "string"] + }, + "category_id": { + "type": ["null", "integer"] + }, + "delete_time": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/person_fields.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/person_fields.json index 7926e50d1019..811da2f9dcc9 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/person_fields.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/person_fields.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "http://json-schema.org/schema#", "type": "object", "properties": { "id": { @@ -17,26 +17,21 @@ "field_type": { "type": ["null", "string"] }, + "json_column_flag": { + "type": ["null", "boolean"] + }, "add_time": { "type": ["null", "string"] }, "update_time": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_without_timezone" + "type": ["null", "string"] }, "last_updated_by_user_id": { "type": ["null", "integer"] }, - "active_flag": { - "type": ["null", "boolean"] - }, "edit_flag": { "type": ["null", "boolean"] }, - "index_visible_flag": { - "type": ["null", "boolean"] - }, "details_visible_flag": { "type": ["null", "boolean"] }, @@ -49,31 +44,46 @@ "bulk_edit_allowed": { "type": ["null", "boolean"] }, - "searchable_flag": { - "type": ["null", "boolean"] - }, "filtering_allowed": { "type": ["null", "boolean"] }, "sortable_flag": { "type": ["null", "boolean"] }, + "mandatory_flag": { + "type": ["null", "boolean"] + }, + "link": { + "type": ["null", "string"] + }, + "use_field": { + "type": ["null", "string"] + }, + "display_field": { + "type": ["null", "string"] + }, + "active_flag": { + "type": ["null", "boolean"] + }, + "index_visible_flag": { + "type": ["null", "boolean"] + }, + "searchable_flag": { + "type": ["null", "boolean"] + }, "options": { "type": ["null", "array"], "items": { - "type": ["null", "object"], + "type": "object", "properties": { "id": { - "type": ["null", "integer", "string", "boolean"] + "type": ["null", "boolean", "integer"] }, "label": { "type": ["null", "string"] } } } - }, - "mandatory_flag": { - "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/persons.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/persons.json index 893aad9b7a56..58988de7220f 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/persons.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/persons.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] @@ -196,6 +197,12 @@ }, "cc_email": { "type": ["null", "string"] + }, + "picture_128_url": { + "type": ["null", "string"] + }, + "delete_time": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/product_fields.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/product_fields.json index 811da2f9dcc9..8308c952b82b 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/product_fields.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/product_fields.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { @@ -77,7 +77,7 @@ "type": "object", "properties": { "id": { - "type": ["null", "boolean", "integer"] + "type": ["null", "string", "boolean", "integer"] }, "label": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/products.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/products.json index f59204de4e5f..75c03f810ef4 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/products.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/products.json @@ -1,6 +1,7 @@ { - "$schema": "http://json-schema.org/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/users.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/users.json index 089c0cef654c..46fcaa14aab3 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/users.json +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/users.json @@ -33,9 +33,7 @@ "type": ["null", "string"] }, "modified": { - "type": ["null", "string"], - "format": "date-time", - "airbyte_type": "timestamp_without_timezone" + "type": ["null", "string"] }, "signup_flow_variation": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/source.py b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/source.py index 2c9c76138d31..32dd3a077d5d 100644 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/source.py +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/source.py @@ -2,105 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from typing import Any, List, Mapping, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import pendulum -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator -from source_pipedrive.auth import QueryStringTokenAuthenticator -from source_pipedrive.streams import ( - Activities, - ActivityFields, - ActivityTypes, - Currencies, - DealFields, - DealProducts, - Deals, - Files, - Filters, - LeadLabels, - Leads, - Notes, - OrganizationFields, - Organizations, - PermissionSets, - PersonFields, - Persons, - Pipelines, - ProductFields, - Products, - Roles, - Stages, - Users, -) +WARNING: Do not modify this file. +""" -class SourcePipedrive(AbstractSource): - def _validate_and_transform(self, config: Mapping[str, Any]): - config["replication_start_date"] = pendulum.parse(config["replication_start_date"]) - return config - - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: - config = self._validate_and_transform(config) - try: - stream = Deals(authenticator=self.get_authenticator(config), replication_start_date=config["replication_start_date"]) - records = stream.read_records(sync_mode=SyncMode.full_refresh) - next(records, None) - return True, None - except Exception as error: - return False, f"Unable to connect to Pipedrive API with the provided credentials - {repr(error)}" - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - config = self._validate_and_transform(config) - stream_kwargs = {"authenticator": self.get_authenticator(config)} - incremental_kwargs = {**stream_kwargs, "replication_start_date": config["replication_start_date"]} - deals_stream = Deals(**incremental_kwargs) - streams = [ - Activities(**incremental_kwargs), - ActivityFields(**stream_kwargs), - ActivityTypes(**incremental_kwargs), - Currencies(**stream_kwargs), - deals_stream, - DealProducts(parent=deals_stream, **stream_kwargs), - DealFields(**stream_kwargs), - Files(**incremental_kwargs), - Filters(**incremental_kwargs), - LeadLabels(**stream_kwargs), - Leads(**stream_kwargs), - Notes(**incremental_kwargs), - Organizations(**incremental_kwargs), - OrganizationFields(**stream_kwargs), - PermissionSets(**stream_kwargs), - Persons(**incremental_kwargs), - PersonFields(**stream_kwargs), - Pipelines(**incremental_kwargs), - ProductFields(**stream_kwargs), - Products(**incremental_kwargs), - Roles(**stream_kwargs), - Stages(**incremental_kwargs), - Users(**incremental_kwargs), - ] - return streams - - @staticmethod - def get_authenticator(config: Mapping[str, Any]): - authorization = config.get("authorization") - if authorization: - if authorization["auth_type"] == "Client": - return Oauth2Authenticator( - token_refresh_endpoint="https://oauth.pipedrive.com/oauth/token", - client_id=authorization["client_id"], - client_secret=authorization["client_secret"], - refresh_token=authorization["refresh_token"], - ) - elif authorization["auth_type"] == "Token": - return QueryStringTokenAuthenticator(api_token=authorization["api_token"]) - # backward compatibility - return QueryStringTokenAuthenticator(api_token=config["api_token"]) +# Declarative Source +class SourcePipedrive(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/spec.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/spec.json deleted file mode 100644 index be9a09b98af1..000000000000 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/spec.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/pipedrive", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Pipedrive Spec", - "type": "object", - "required": ["replication_start_date"], - "additionalProperties": true, - "properties": { - "authorization": { - "type": "object", - "title": "API Key Authentication", - "required": ["auth_type", "api_token"], - "properties": { - "auth_type": { - "type": "string", - "const": "Token", - "order": 0 - }, - "api_token": { - "title": "API Token", - "type": "string", - "description": "The Pipedrive API Token.", - "airbyte_secret": true - } - } - }, - "replication_start_date": { - "title": "Start Date", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. When specified and not None, then stream will behave as incremental", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "examples": ["2017-01-25T00:00:00Z"], - "type": "string", - "format": "date-time" - } - } - }, - "supportsIncremental": true -} diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/streams.py b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/streams.py deleted file mode 100755 index bc505e8c2b56..000000000000 --- a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/streams.py +++ /dev/null @@ -1,253 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union - -import pendulum -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http import HttpStream - -PIPEDRIVE_URL_BASE = "https://api.pipedrive.com/v1/" - - -class PipedriveStream(HttpStream, ABC): - url_base = PIPEDRIVE_URL_BASE - primary_key = "id" - data_field = "data" - page_size = 50 - - def __init__(self, replication_start_date=None, **kwargs): - super().__init__(**kwargs) - self._replication_start_date = replication_start_date - - @property - def cursor_field(self) -> Union[str, List[str]]: - if self._replication_start_date: - return "update_time" - return [] - - def path(self, **kwargs) -> str: - if self._replication_start_date: - return "recents" - - class_name = self.__class__.__name__ - return f"{class_name[0].lower()}{class_name[1:]}" - - @property - def path_param(self): - return self.name[:-1] - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - :param response: the most recent response from the API - :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query - the next page in the response. - If there are no more pages in the result, return None. - """ - pagination_data = response.json().get("additional_data", {}).get("pagination", {}) - if pagination_data.get("more_items_in_collection") and pagination_data.get("start") is not None: - start = pagination_data.get("start") + self.page_size - return {"start": start} - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - next_page_token = next_page_token or {} - params = {"limit": self.page_size, **next_page_token} - - replication_start_date = self._replication_start_date - if replication_start_date: - cursor_value = stream_state.get(self.cursor_field) - if cursor_value: - cursor_value = pendulum.parse(cursor_value) - replication_start_date = max(replication_start_date, cursor_value) - - params.update( - { - "items": self.path_param, - "since_timestamp": replication_start_date.strftime("%Y-%m-%d %H:%M:%S"), - } - ) - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - :return an iterable containing each record in the response - """ - records = response.json().get(self.data_field) or [] - for record in records: - record = record.get(self.data_field) or record - if self.primary_key in record and record[self.primary_key] is None: - # Convert "id: null" fields to "id: 0" since id is primary key and SAT checks if it is not null. - record[self.primary_key] = 0 - yield record - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object - and returning an updated state object. - """ - replication_start_date = self._replication_start_date.strftime("%Y-%m-%d %H:%M:%S") - current_stream_state[self.cursor_field] = max( - latest_record.get(self.cursor_field, replication_start_date), - current_stream_state.get(self.cursor_field, replication_start_date), - ) - return current_stream_state - - -class Deals(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/Deals#getDeals, - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - -class DealFields(PipedriveStream): - """https://developers.pipedrive.com/docs/api/v1/DealFields#getDealFields""" - - -class Files(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/Files#getFiles - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - -class Filters(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/Filters#getFilters - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - -class LeadLabels(PipedriveStream): - """https://developers.pipedrive.com/docs/api/v1/LeadLabels#getLeadLabels""" - - -class Leads(PipedriveStream): - """https://developers.pipedrive.com/docs/api/v1/Leads#getLeads""" - - -class Notes(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/Notes#getNotes - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - -class Activities(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/Activities#getActivities, - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - path_param = "activity" - - -class ActivityFields(PipedriveStream): - """https://developers.pipedrive.com/docs/api/v1/ActivityFields#getActivityFields""" - - -class ActivityTypes(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/ActivityTypes#getActivityTypes - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - path_param = "activityType" - - -class Currencies(PipedriveStream): - """https://developers.pipedrive.com/docs/api/v1/Currencies#getCurrencies""" - - -class Organizations(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/Organizations#getOrganizations, - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - -class OrganizationFields(PipedriveStream): - """https://developers.pipedrive.com/docs/api/v1/OrganizationFields#getOrganizationFields""" - - -class PermissionSets(PipedriveStream): - """https://developers.pipedrive.com/docs/api/v1/PermissionSets#getPermissionSets""" - - -class Persons(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/Persons#getPersons, - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - -class PersonFields(PipedriveStream): - """https://developers.pipedrive.com/docs/api/v1/PersonFields#getPersonFields""" - - -class Pipelines(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/Pipelines#getPipelines, - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - -class Products(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/Products#getProducts, - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - -class ProductFields(PipedriveStream): - """https://developers.pipedrive.com/docs/api/v1/ProductFields#getProductFields""" - - -class Roles(PipedriveStream): - """https://developers.pipedrive.com/docs/api/v1/Roles#getRoles""" - - -class Stages(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/Stages#getStages, - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - -class Users(PipedriveStream): - """ - API docs: https://developers.pipedrive.com/docs/api/v1/Users#getUsers, - retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents - """ - - cursor_field = "modified" - page_size = 500 - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - record_gen = super().parse_response(response=response, **kwargs) - for records in record_gen: - yield from records - - -class DealProducts(PipedriveStream): - """https://developers.pipedrive.com/docs/api/v1/Deals#getDealProducts""" - - def __init__(self, parent, **kwargs): - self.parent = parent - super().__init__(**kwargs) - - def path(self, stream_slice, **kwargs) -> str: - return f"deals/{stream_slice['deal_id']}/products" - - def stream_slices(self, sync_mode, cursor_field=None, stream_state=None): - stream_slices = self.parent.stream_slices(sync_mode=SyncMode.full_refresh) - for stream_slice in stream_slices: - records = self.parent.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) - for record in records: - if record["products_count"]: - yield {"deal_id": record["id"]} diff --git a/airbyte-integrations/connectors/source-pipedrive/unit_tests/__init__.py b/airbyte-integrations/connectors/source-pipedrive/unit_tests/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-pipedrive/unit_tests/conftest.py b/airbyte-integrations/connectors/source-pipedrive/unit_tests/conftest.py deleted file mode 100644 index c11da1ff0380..000000000000 --- a/airbyte-integrations/connectors/source-pipedrive/unit_tests/conftest.py +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pendulum -import pytest -from source_pipedrive.source import SourcePipedrive - -replication_start_date = "2017-01-25T00:00:00Z" - - -@pytest.fixture -def config_oauth(): - return { - "authorization": { - "auth_type": "Client", - "client_id": "6779ef20e75817b79602", - "client_secret": "7607999ef26581e81726777b7b79f20e70e75602", - "refresh_token": "refresh_token", - }, - "replication_start_date": replication_start_date, - } - - -@pytest.fixture -def config_token(): - return { - "authorization": { - "auth_type": "Token", - "api_token": "api_token" - }, - "replication_start_date": replication_start_date - } - - -@pytest.fixture -def stream_kwargs(config_token): - return {"authenticator": SourcePipedrive.get_authenticator(config_token)} - - -@pytest.fixture -def incremental_kwargs(config_token): - return { - "authenticator": SourcePipedrive.get_authenticator(config_token), - "replication_start_date": pendulum.parse(config_token["replication_start_date"]) - } diff --git a/airbyte-integrations/connectors/source-pipedrive/unit_tests/test_extractor.py b/airbyte-integrations/connectors/source-pipedrive/unit_tests/test_extractor.py new file mode 100644 index 000000000000..1aef6539ec7e --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/unit_tests/test_extractor.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +import json + +import pytest +import requests +from source_pipedrive.extractor import NullCheckedDpathExtractor + + +@pytest.mark.parametrize( + "response_body, expected_records", + [ + pytest.param( + {"data": [{"id": 1, "data": None}, {"id": 2, "data": None}]}, + [{"id": 1, "data": None}, {"id": 2, "data": None}], + id="test_with_null_nested_field", + ), + pytest.param( + {"data": [{"id": 1, "data": {"id": 1, "user_id": "123"}}, {"id": 2, "data": {"id": 2, "user_id": "123"}}]}, + [{"id": 1, "user_id": "123"}, {"id": 2, "user_id": "123"}], + id="test_with_nested_field", + ), + ], +) +def test_pipedrive_extractor(response_body, expected_records): + extractor = NullCheckedDpathExtractor(field_path=["data", "*"], nullable_nested_field="data", config={}, parameters={}) + response = _create_response(response_body) + records = extractor.extract_records(response) + + assert records == expected_records + + +def _create_response(body): + response = requests.Response() + response._content = json.dumps(body).encode("utf-8") + return response diff --git a/airbyte-integrations/connectors/source-pipedrive/unit_tests/test_source.py b/airbyte-integrations/connectors/source-pipedrive/unit_tests/test_source.py deleted file mode 100644 index f65cf6cfdbb8..000000000000 --- a/airbyte-integrations/connectors/source-pipedrive/unit_tests/test_source.py +++ /dev/null @@ -1,72 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import ( - AirbyteStream, - ConfiguredAirbyteCatalog, - ConfiguredAirbyteStream, - ConnectorSpecification, - DestinationSyncMode, - SyncMode, -) -from source_pipedrive.source import SourcePipedrive - -logger = AirbyteLogger() - -PIPEDRIVE_URL_BASE = "https://api.pipedrive.com/v1/" - - -def test_check_connection(requests_mock, config_token): - body = {"success": "true", "data": [{"id": 1, "update_time": "2020-10-14T11:30:36.551Z"}]} - response = setup_response(200, body) - api_token = config_token["authorization"]["api_token"] - requests_mock.register_uri("GET", PIPEDRIVE_URL_BASE + "recents?limit=50&api_token=" + api_token, response) - - ok, error = SourcePipedrive().check_connection(logger, config_token) - - assert ok - assert not error - - -def test_check_connection_exception(requests_mock, config_token): - response = setup_response(400, {}) - api_token = config_token["authorization"]["api_token"] - requests_mock.register_uri("GET", PIPEDRIVE_URL_BASE + "recents?limit=50&api_token=" + api_token, response) - - ok, error = SourcePipedrive().check_connection(logger, config_token) - - assert not ok - assert error - - -def test_streams(config_token): - streams = SourcePipedrive().streams(config_token) - - assert len(streams) == 23 - - -def setup_response(status, body): - return [ - {"json": body, "status_code": status}, - ] - - -def test_spec(): - spec = SourcePipedrive().spec(logger) - assert isinstance(spec, ConnectorSpecification) - - -def test_read(config_token): - source = SourcePipedrive() - catalog = ConfiguredAirbyteCatalog( - streams=[ - ConfiguredAirbyteStream( - stream=AirbyteStream(name="deals", json_schema={}, supported_sync_modes=["full_refresh", "incremental"]), - sync_mode=SyncMode.full_refresh, - destination_sync_mode=DestinationSyncMode.overwrite, - ) - ] - ) - assert source.read(logger, config_token, catalog) diff --git a/airbyte-integrations/connectors/source-pipedrive/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-pipedrive/unit_tests/test_streams.py deleted file mode 100644 index 53796150dc87..000000000000 --- a/airbyte-integrations/connectors/source-pipedrive/unit_tests/test_streams.py +++ /dev/null @@ -1,112 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pytest -from airbyte_cdk.models import SyncMode -from source_pipedrive.streams import ( - Activities, - ActivityFields, - DealFields, - Deals, - Leads, - OrganizationFields, - Organizations, - PersonFields, - Persons, - PipedriveStream, - Pipelines, - Stages, -) - -PIPEDRIVE_URL_BASE = "https://api.pipedrive.com/v1/" - - -def test_cursor_field_incremental(incremental_kwargs): - stream = PipedriveStream(**incremental_kwargs) - - assert stream.cursor_field == "update_time" - - -def test_cursor_field_refresh(stream_kwargs): - stream = PipedriveStream(**stream_kwargs) - - assert stream.cursor_field == [] - - -def test_path_incremental(incremental_kwargs): - stream = PipedriveStream(**incremental_kwargs) - - assert stream.path() == "recents" - - -def test_path_refresh(stream_kwargs): - stream = PipedriveStream(**stream_kwargs) - - assert stream.path() == "pipedriveStream" - - -@pytest.mark.parametrize( - "stream, endpoint", - [ - (ActivityFields, "activityFields"), - (DealFields, "dealFields"), - (OrganizationFields, "organizationFields"), - (PersonFields, "personFields"), - (Leads, "leads"), - ], -) -def test_streams_full_refresh(stream, endpoint, requests_mock, stream_kwargs): - body = { - "success": "true", - "data": [{"id": 1, "update_time": "2020-10-14T11:30:36.551Z"}, {"id": 2, "update_time": "2020-10-14T11:30:36.551Z"}], - } - - response = setup_response(200, body) - - api_token = stream_kwargs["authenticator"].params["api_token"] - requests_mock.register_uri("GET", PIPEDRIVE_URL_BASE + endpoint + "?limit=50&api_token=" + api_token, response) - - stream = stream(**stream_kwargs) - records = stream.read_records(sync_mode=SyncMode.full_refresh) - - assert records - - -@pytest.mark.parametrize( - "stream", - [ - Activities, - Deals, - Organizations, - Persons, - Pipelines, - Stages, - # Users - ], -) -def test_streams_incremental_sync(stream, requests_mock, incremental_kwargs): - body = { - "success": "true", - "data": [{"id": 1, "update_time": "2020-10-14T11:30:36.551Z"}, {"id": 2, "update_time": "2020-11-14T11:30:36.551Z"}], - } - - response = setup_response(200, body) - - api_token = incremental_kwargs["authenticator"].params["api_token"] - requests_mock.register_uri("GET", PIPEDRIVE_URL_BASE + "recents?limit=50&api_token=" + api_token, response) - - stream = stream(**incremental_kwargs) - records = stream.read_records(sync_mode=SyncMode.incremental) - stream_state = {} - for record in records: - stream_state = stream.get_updated_state(stream_state, latest_record=record) - - assert records - assert stream_state["update_time"] == "2020-11-14T11:30:36.551Z" - - -def setup_response(status, body): - return [ - {"json": body, "status_code": status}, - ] diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/Dockerfile b/airbyte-integrations/connectors/source-pivotal-tracker/Dockerfile index 016438b17b42..751422961308 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/Dockerfile +++ b/airbyte-integrations/connectors/source-pivotal-tracker/Dockerfile @@ -34,5 +34,5 @@ COPY source_pivotal_tracker ./source_pivotal_tracker ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-pivotal-tracker diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/README.md b/airbyte-integrations/connectors/source-pivotal-tracker/README.md index fcce2540445e..bf5fa41cbc36 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/README.md +++ b/airbyte-integrations/connectors/source-pivotal-tracker/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-pivotal-tracker:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/pivotal-tracker) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_pivotal_tracker/spec.json` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-pivotal-tracker:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-pivotal-tracker build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-pivotal-tracker:airbyteDocker +An image will be built with the tag `airbyte/source-pivotal-tracker:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-pivotal-tracker:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pivotal-tracker:dev ch docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pivotal-tracker:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-pivotal-tracker:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-pivotal-tracker test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-pivotal-tracker:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-pivotal-tracker:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-pivotal-tracker test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/pivotal-tracker.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-pivotal-tracker/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-pivotal-tracker/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/build.gradle b/airbyte-integrations/connectors/source-pivotal-tracker/build.gradle deleted file mode 100644 index edb8af069c2f..000000000000 --- a/airbyte-integrations/connectors/source-pivotal-tracker/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_pivotal_tracker' -} diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/metadata.yaml b/airbyte-integrations/connectors/source-pivotal-tracker/metadata.yaml index f3b9c5bb49ef..013eac02808a 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/metadata.yaml +++ b/airbyte-integrations/connectors/source-pivotal-tracker/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: d60f5393-f99e-4310-8d05-b1876820f40e - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 dockerRepository: airbyte/source-pivotal-tracker githubIssueLabel: source-pivotal-tracker icon: pivotal-tracker.svg diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/schemas.zip b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/schemas.zip deleted file mode 100644 index 195c2fae9a82..000000000000 Binary files a/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/schemas.zip and /dev/null differ diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/schemas/activity.json b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/schemas/activity.json index 95a0c31c2fd2..6a9505199d85 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/schemas/activity.json +++ b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/schemas/activity.json @@ -8,6 +8,17 @@ "kind": { "type": ["null", "string"] }, + "project": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} + }, + "secondary_resources": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, "project_id": { "type": ["null", "integer"] }, diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/schemas/epics.json b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/schemas/epics.json index 69ec47fd280b..41f27002ac99 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/schemas/epics.json +++ b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/schemas/epics.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] @@ -8,6 +9,9 @@ "kind": { "type": ["null", "string"] }, + "description": { + "type": ["null", "string"] + }, "created_at": { "type": ["null", "string"], "format": "date-time" diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/schemas/project_memberships.json b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/schemas/project_memberships.json index de2286f8573f..60f98466e9fd 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/schemas/project_memberships.json +++ b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/schemas/project_memberships.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] @@ -8,6 +9,10 @@ "kind": { "type": ["null", "string"] }, + "last_viewed_at": { + "type": ["null", "string"], + "format": "date-time" + }, "created_at": { "type": ["null", "string"], "format": "date-time" diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/schemas/projects.json b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/schemas/projects.json index c837365ea995..2fd8a6fc354f 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/schemas/projects.json +++ b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/schemas/projects.json @@ -8,6 +8,12 @@ "kind": { "type": ["null", "string"] }, + "show_priority_icon": { + "type": ["null", "boolean"] + }, + "show_priority_icon_in_all_panels": { + "type": ["null", "boolean"] + }, "created_at": { "type": ["null", "string"], "format": "date-time" diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/schemas/stories.json b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/schemas/stories.json index 359201c1a98c..97b9dcf38ff7 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/schemas/stories.json +++ b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/schemas/stories.json @@ -8,6 +8,13 @@ "kind": { "type": ["null", "string"] }, + "description": { + "type": ["null", "string"] + }, + "deadline": { + "type": ["null", "string"], + "format": "date-time" + }, "created_at": { "type": ["null", "string"], "format": "date-time" diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/source.py b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/source.py index 116dbb7c2b86..7b590198e46f 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/source.py +++ b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/source.py @@ -140,9 +140,12 @@ def _generate_project_ids(auth: HttpAuthenticator) -> List[str]: return project_ids def check_connection(self, logger, config) -> Tuple[bool, any]: - auth = SourcePivotalTracker._get_authenticator(config) - self._generate_project_ids(auth) - return True, None + try: + auth = SourcePivotalTracker._get_authenticator(config) + self._generate_project_ids(auth) + return True, None + except Exception as e: + return False, e def streams(self, config: Mapping[str, Any]) -> List[Stream]: auth = self._get_authenticator(config) diff --git a/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/spec.json b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/spec.json index 40eaea438109..7eb098c903f0 100644 --- a/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/spec.json +++ b/airbyte-integrations/connectors/source-pivotal-tracker/source_pivotal_tracker/spec.json @@ -5,12 +5,13 @@ "title": "Pivotal Tracker Spec", "type": "object", "required": ["api_token"], - "additionalProperties": false, + "additionalProperties": true, "properties": { "api_token": { "type": "string", "description": "Pivotal Tracker API token", - "examples": ["5c054d0de3440452190fdc5d5a04d871"] + "examples": ["5c054d0de3440452190fdc5d5a04d871"], + "airbyte_secret": true } } } diff --git a/airbyte-integrations/connectors/source-plaid/README.md b/airbyte-integrations/connectors/source-plaid/README.md index f1601229131e..43550443354c 100644 --- a/airbyte-integrations/connectors/source-plaid/README.md +++ b/airbyte-integrations/connectors/source-plaid/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-plaid-new:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/plaid) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_plaid/spec.json` file. @@ -54,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-plaid:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-plaid build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-plaid:airbyteDocker +An image will be built with the tag `airbyte/source-plaid:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-plaid:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,44 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-plaid:dev check --conf docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-plaid:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-plaid:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install '.[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-plaid test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-plaid-new:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-plaid-new:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -122,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-plaid test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/plaid.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-plaid/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-plaid/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-plaid/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-plaid/build.gradle b/airbyte-integrations/connectors/source-plaid/build.gradle deleted file mode 100644 index caa8ccf44904..000000000000 --- a/airbyte-integrations/connectors/source-plaid/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_plaid_singer' -} diff --git a/airbyte-integrations/connectors/source-plausible/README.md b/airbyte-integrations/connectors/source-plausible/README.md index 55a51d5e2645..e3bca2ee96c6 100644 --- a/airbyte-integrations/connectors/source-plausible/README.md +++ b/airbyte-integrations/connectors/source-plausible/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-plausible:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/plausible) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_plausible/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-plausible:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-plausible build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-plausible:airbyteDocker +An image will be built with the tag `airbyte/source-plausible:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-plausible:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-plausible:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-plausible:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-plausible:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-plausible test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-plausible:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-plausible:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. \ No newline at end of file +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-plausible test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/plausible.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-plausible/acceptance-test-config.yml b/airbyte-integrations/connectors/source-plausible/acceptance-test-config.yml index 7baa099ac122..1309f1038604 100644 --- a/airbyte-integrations/connectors/source-plausible/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-plausible/acceptance-test-config.yml @@ -19,9 +19,9 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - incremental: + incremental: bypass_reason: "This connector does not implement incremental sync" full_refresh: tests: - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" \ No newline at end of file + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-plausible/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-plausible/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-plausible/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-plausible/build.gradle b/airbyte-integrations/connectors/source-plausible/build.gradle deleted file mode 100644 index 6103e26cc959..000000000000 --- a/airbyte-integrations/connectors/source-plausible/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_plausible' -} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-pocket/README.md b/airbyte-integrations/connectors/source-pocket/README.md index 8f4cabdb8dec..a01f23cfc0ce 100644 --- a/airbyte-integrations/connectors/source-pocket/README.md +++ b/airbyte-integrations/connectors/source-pocket/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-pocket:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/pocket) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_pocket/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-pocket:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-pocket build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-pocket:airbyteDocker +An image will be built with the tag `airbyte/source-pocket:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-pocket:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pocket:dev check --con docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pocket:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-pocket:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-pocket test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-pocket:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-pocket:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-pocket test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/pocket.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-pocket/acceptance-test-config.yml b/airbyte-integrations/connectors/source-pocket/acceptance-test-config.yml index 6872e3612d7b..7f2b2ac22513 100644 --- a/airbyte-integrations/connectors/source-pocket/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-pocket/acceptance-test-config.yml @@ -19,7 +19,7 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - incremental: + incremental: bypass_reason: "This connector does not implement incremental sync" full_refresh: tests: diff --git a/airbyte-integrations/connectors/source-pocket/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-pocket/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-pocket/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-pocket/build.gradle b/airbyte-integrations/connectors/source-pocket/build.gradle deleted file mode 100644 index 8011b8f52f31..000000000000 --- a/airbyte-integrations/connectors/source-pocket/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_pocket' -} diff --git a/airbyte-integrations/connectors/source-pokeapi/Dockerfile b/airbyte-integrations/connectors/source-pokeapi/Dockerfile index 31f309c2e920..0d27d3737d6f 100644 --- a/airbyte-integrations/connectors/source-pokeapi/Dockerfile +++ b/airbyte-integrations/connectors/source-pokeapi/Dockerfile @@ -1,17 +1,38 @@ -FROM python:3.9-slim +FROM python:3.9.11-alpine3.15 as base -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base WORKDIR /airbyte/integration_code -COPY source_pokeapi ./source_pokeapi + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only COPY main.py ./ -COPY setup.py ./ -RUN pip install . +COPY source_pokeapi ./source_pokeapi ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.5 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-pokeapi diff --git a/airbyte-integrations/connectors/source-pokeapi/README.md b/airbyte-integrations/connectors/source-pokeapi/README.md index 8d422510264a..0cd90facb88b 100644 --- a/airbyte-integrations/connectors/source-pokeapi/README.md +++ b/airbyte-integrations/connectors/source-pokeapi/README.md @@ -1,80 +1,34 @@ # Pokeapi Source -This is the repository for the Pokeapi source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/pokeapi). +This is the repository for the Pokeapi configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/pokeapi). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-pokeapi:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/pokeapi) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_pokeapi/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/pokeapi) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_pokeapi/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. +See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source pokeapi test creds` and place them into `secrets/config.json`. - -### Locally running the connector -``` -python main_dev.py spec -python main_dev.py check --config secrets/config.json -python main_dev.py discover --config secrets/config.json -python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-pokeapi:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-pokeapi build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-pokeapi:airbyteDocker +An image will be built with the tag `airbyte/source-pokeapi:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-pokeapi:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -82,56 +36,32 @@ Then run any of the connector commands as follows: docker run --rm airbyte/source-pokeapi:dev spec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pokeapi:dev check --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pokeapi:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-pokeapi:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-pokeapi:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-pokeapi test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). - -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unittest run: -``` -./gradlew :airbyte-integrations:connectors:source-pokeapi:unitTest -``` -To run acceptance and custom integration tests run: -``` -./gradlew :airbyte-integrations:connectors:source-pokeapi:IntegrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-pokeapi test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/pokeapi.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-pokeapi/__init__.py b/airbyte-integrations/connectors/source-pokeapi/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-pokeapi/acceptance-test-config.yml b/airbyte-integrations/connectors/source-pokeapi/acceptance-test-config.yml index 2f23c6db6c7f..e0d96ad450b3 100644 --- a/airbyte-integrations/connectors/source-pokeapi/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-pokeapi/acceptance-test-config.yml @@ -1,23 +1,29 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests connector_image: airbyte/source-pokeapi:dev -test_strictness_level: high acceptance_tests: spec: - bypass_reason: "The spec is currently invalid: it has additionalProperties set to false" + tests: + - spec_path: "source_pokeapi/spec.yaml" + backward_compatibility_tests_config: + disable_for_version: "0.1.5" connection: tests: - - config_path: "integration_tests/config.json" + - config_path: "secrets/config.json" status: "succeed" discovery: tests: - - config_path: "integration_tests/config.json" + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.5" basic_read: tests: - - config_path: "integration_tests/config.json" - expect_records: - bypass_reason: "We should create an expected_records file" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + incremental: + bypass_reason: "This connector does not implement incremental sync" full_refresh: tests: - - config_path: "integration_tests/config.json" + - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - incremental: - bypass_reason: "This connector does not support incremental syncs." diff --git a/airbyte-integrations/connectors/source-pokeapi/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-pokeapi/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-pokeapi/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-pokeapi/build.gradle b/airbyte-integrations/connectors/source-pokeapi/build.gradle deleted file mode 100644 index e20b1bd1378c..000000000000 --- a/airbyte-integrations/connectors/source-pokeapi/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_pokeapi' -} diff --git a/airbyte-integrations/connectors/source-pokeapi/integration_tests/__init__.py b/airbyte-integrations/connectors/source-pokeapi/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-pokeapi/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-pokeapi/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-pokeapi/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-pokeapi/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-pokeapi/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pokeapi/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-pokeapi/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-pokeapi/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-pokeapi/integration_tests/config.json b/airbyte-integrations/connectors/source-pokeapi/integration_tests/config.json deleted file mode 100644 index 392328e8b027..000000000000 --- a/airbyte-integrations/connectors/source-pokeapi/integration_tests/config.json +++ /dev/null @@ -1 +0,0 @@ -{ "pokemon_name": "ditto" } diff --git a/airbyte-integrations/connectors/source-pokeapi/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-pokeapi/integration_tests/configured_catalog.json index 72ca528d90f7..81e82dbd2fe4 100644 --- a/airbyte-integrations/connectors/source-pokeapi/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-pokeapi/integration_tests/configured_catalog.json @@ -3,15 +3,7 @@ { "stream": { "name": "pokemon", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "pokemon_name": { - "type": "string" - } - } - }, + "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", diff --git a/airbyte-integrations/connectors/source-pokeapi/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-pokeapi/integration_tests/invalid_config.json index 0d853925b60a..af3ddc30a8a4 100644 --- a/airbyte-integrations/connectors/source-pokeapi/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-pokeapi/integration_tests/invalid_config.json @@ -1 +1,3 @@ -{ "name": "datto" } +{ + "name": "datto" +} diff --git a/airbyte-integrations/connectors/source-pokeapi/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-pokeapi/integration_tests/sample_config.json new file mode 100644 index 000000000000..db1339d0787e --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "pokemon_name": "ditto" +} diff --git a/airbyte-integrations/connectors/source-pokeapi/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-pokeapi/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml index dec58781459c..076a75a780a4 100644 --- a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml +++ b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml @@ -1,24 +1,25 @@ data: + allowedHosts: + hosts: + - "*" + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: 6371b14b-bc68-4236-bfbd-468e8df8e968 - dockerImageTag: 0.1.5 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-pokeapi githubIssueLabel: source-pokeapi icon: pokeapi.svg license: MIT name: PokeAPI - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: "2020-05-14" releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/pokeapi tags: - - language:python - ab_internal: - sl: 100 - ql: 100 - supportLevel: community + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-pokeapi/requirements.txt b/airbyte-integrations/connectors/source-pokeapi/requirements.txt index d6e1198b1ab1..cf563bcab685 100644 --- a/airbyte-integrations/connectors/source-pokeapi/requirements.txt +++ b/airbyte-integrations/connectors/source-pokeapi/requirements.txt @@ -1 +1,2 @@ -e . +-e ../../bases/connector-acceptance-test \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-pokeapi/setup.py b/airbyte-integrations/connectors/source-pokeapi/setup.py index e5cdf3627e09..2fa7839b58fc 100644 --- a/airbyte-integrations/connectors/source-pokeapi/setup.py +++ b/airbyte-integrations/connectors/source-pokeapi/setup.py @@ -5,9 +5,15 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1"] +MAIN_REQUIREMENTS = [ + "airbyte-cdk", +] -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1"] +TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest~=6.2", + "pytest-mock~=3.6.1", +] setup( name="source_pokeapi", @@ -16,7 +22,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/__init__.py b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/__init__.py index 091b006498ea..e21b6da3ff71 100644 --- a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/__init__.py +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/__init__.py @@ -1,7 +1,8 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from .source import SourcePokeapi __all__ = ["SourcePokeapi"] diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/manifest.yaml b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/manifest.yaml new file mode 100644 index 000000000000..e556d07b2cb8 --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/manifest.yaml @@ -0,0 +1,40 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + requester: + type: HttpRequester + url_base: "https://pokeapi.co/api/v2/pokemon" + http_method: "GET" + authenticator: + type: NoAuth + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester" + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + pokemon_stream: + $ref: "#/definitions/base_stream" + name: "pokemon" + primary_key: "id" + $parameters: + path: "/{{config['pokemon_name']}}" + +streams: + - "#/definitions/pokemon_stream" + +check: + type: CheckStream + stream_names: + - "pokemon" diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/pokemon_list.py b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/pokemon_list.py deleted file mode 100644 index fa640b8ec06a..000000000000 --- a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/pokemon_list.py +++ /dev/null @@ -1,909 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -""" -pokemon_list.py includes a list of all known pokemon for config validation in source.py. -""" - -POKEMON_LIST = [ - "bulbasaur", - "ivysaur", - "venusaur", - "charmander", - "charmeleon", - "charizard", - "squirtle", - "wartortle", - "blastoise", - "caterpie", - "metapod", - "butterfree", - "weedle", - "kakuna", - "beedrill", - "pidgey", - "pidgeotto", - "pidgeot", - "rattata", - "raticate", - "spearow", - "fearow", - "ekans", - "arbok", - "pikachu", - "raichu", - "sandshrew", - "sandslash", - "nidoranf", - "nidorina", - "nidoqueen", - "nidoranm", - "nidorino", - "nidoking", - "clefairy", - "clefable", - "vulpix", - "ninetales", - "jigglypuff", - "wigglytuff", - "zubat", - "golbat", - "oddish", - "gloom", - "vileplume", - "paras", - "parasect", - "venonat", - "venomoth", - "diglett", - "dugtrio", - "meowth", - "persian", - "psyduck", - "golduck", - "mankey", - "primeape", - "growlithe", - "arcanine", - "poliwag", - "poliwhirl", - "poliwrath", - "abra", - "kadabra", - "alakazam", - "machop", - "machoke", - "machamp", - "bellsprout", - "weepinbell", - "victreebel", - "tentacool", - "tentacruel", - "geodude", - "graveler", - "golem", - "ponyta", - "rapidash", - "slowpoke", - "slowbro", - "magnemite", - "magneton", - "farfetchd", - "doduo", - "dodrio", - "seel", - "dewgong", - "grimer", - "muk", - "shellder", - "cloyster", - "gastly", - "haunter", - "gengar", - "onix", - "drowzee", - "hypno", - "krabby", - "kingler", - "voltorb", - "electrode", - "exeggcute", - "exeggutor", - "cubone", - "marowak", - "hitmonlee", - "hitmonchan", - "lickitung", - "koffing", - "weezing", - "rhyhorn", - "rhydon", - "chansey", - "tangela", - "kangaskhan", - "horsea", - "seadra", - "goldeen", - "seaking", - "staryu", - "starmie", - "mrmime", - "scyther", - "jynx", - "electabuzz", - "magmar", - "pinsir", - "tauros", - "magikarp", - "gyarados", - "lapras", - "ditto", - "eevee", - "vaporeon", - "jolteon", - "flareon", - "porygon", - "omanyte", - "omastar", - "kabuto", - "kabutops", - "aerodactyl", - "snorlax", - "articuno", - "zapdos", - "moltres", - "dratini", - "dragonair", - "dragonite", - "mewtwo", - "mew", - "chikorita", - "bayleef", - "meganium", - "cyndaquil", - "quilava", - "typhlosion", - "totodile", - "croconaw", - "feraligatr", - "sentret", - "furret", - "hoothoot", - "noctowl", - "ledyba", - "ledian", - "spinarak", - "ariados", - "crobat", - "chinchou", - "lanturn", - "pichu", - "cleffa", - "igglybuff", - "togepi", - "togetic", - "natu", - "xatu", - "mareep", - "flaaffy", - "ampharos", - "bellossom", - "marill", - "azumarill", - "sudowoodo", - "politoed", - "hoppip", - "skiploom", - "jumpluff", - "aipom", - "sunkern", - "sunflora", - "yanma", - "wooper", - "quagsire", - "espeon", - "umbreon", - "murkrow", - "slowking", - "misdreavus", - "unown", - "wobbuffet", - "girafarig", - "pineco", - "forretress", - "dunsparce", - "gligar", - "steelix", - "snubbull", - "granbull", - "qwilfish", - "scizor", - "shuckle", - "heracross", - "sneasel", - "teddiursa", - "ursaring", - "slugma", - "magcargo", - "swinub", - "piloswine", - "corsola", - "remoraid", - "octillery", - "delibird", - "mantine", - "skarmory", - "houndour", - "houndoom", - "kingdra", - "phanpy", - "donphan", - "porygon2", - "stantler", - "smeargle", - "tyrogue", - "hitmontop", - "smoochum", - "elekid", - "magby", - "miltank", - "blissey", - "raikou", - "entei", - "suicune", - "larvitar", - "pupitar", - "tyranitar", - "lugia", - "ho-oh", - "celebi", - "treecko", - "grovyle", - "sceptile", - "torchic", - "combusken", - "blaziken", - "mudkip", - "marshtomp", - "swampert", - "poochyena", - "mightyena", - "zigzagoon", - "linoone", - "wurmple", - "silcoon", - "beautifly", - "cascoon", - "dustox", - "lotad", - "lombre", - "ludicolo", - "seedot", - "nuzleaf", - "shiftry", - "taillow", - "swellow", - "wingull", - "pelipper", - "ralts", - "kirlia", - "gardevoir", - "surskit", - "masquerain", - "shroomish", - "breloom", - "slakoth", - "vigoroth", - "slaking", - "nincada", - "ninjask", - "shedinja", - "whismur", - "loudred", - "exploud", - "makuhita", - "hariyama", - "azurill", - "nosepass", - "skitty", - "delcatty", - "sableye", - "mawile", - "aron", - "lairon", - "aggron", - "meditite", - "medicham", - "electrike", - "manectric", - "plusle", - "minun", - "volbeat", - "illumise", - "roselia", - "gulpin", - "swalot", - "carvanha", - "sharpedo", - "wailmer", - "wailord", - "numel", - "camerupt", - "torkoal", - "spoink", - "grumpig", - "spinda", - "trapinch", - "vibrava", - "flygon", - "cacnea", - "cacturne", - "swablu", - "altaria", - "zangoose", - "seviper", - "lunatone", - "solrock", - "barboach", - "whiscash", - "corphish", - "crawdaunt", - "baltoy", - "claydol", - "lileep", - "cradily", - "anorith", - "armaldo", - "feebas", - "milotic", - "castform", - "kecleon", - "shuppet", - "banette", - "duskull", - "dusclops", - "tropius", - "chimecho", - "absol", - "wynaut", - "snorunt", - "glalie", - "spheal", - "sealeo", - "walrein", - "clamperl", - "huntail", - "gorebyss", - "relicanth", - "luvdisc", - "bagon", - "shelgon", - "salamence", - "beldum", - "metang", - "metagross", - "regirock", - "regice", - "registeel", - "latias", - "latios", - "kyogre", - "groudon", - "rayquaza", - "jirachi", - "deoxys", - "turtwig", - "grotle", - "torterra", - "chimchar", - "monferno", - "infernape", - "piplup", - "prinplup", - "empoleon", - "starly", - "staravia", - "staraptor", - "bidoof", - "bibarel", - "kricketot", - "kricketune", - "shinx", - "luxio", - "luxray", - "budew", - "roserade", - "cranidos", - "rampardos", - "shieldon", - "bastiodon", - "burmy", - "wormadam", - "mothim", - "combee", - "vespiquen", - "pachirisu", - "buizel", - "floatzel", - "cherubi", - "cherrim", - "shellos", - "gastrodon", - "ambipom", - "drifloon", - "drifblim", - "buneary", - "lopunny", - "mismagius", - "honchkrow", - "glameow", - "purugly", - "chingling", - "stunky", - "skuntank", - "bronzor", - "bronzong", - "bonsly", - "mimejr", - "happiny", - "chatot", - "spiritomb", - "gible", - "gabite", - "garchomp", - "munchlax", - "riolu", - "lucario", - "hippopotas", - "hippowdon", - "skorupi", - "drapion", - "croagunk", - "toxicroak", - "carnivine", - "finneon", - "lumineon", - "mantyke", - "snover", - "abomasnow", - "weavile", - "magnezone", - "lickilicky", - "rhyperior", - "tangrowth", - "electivire", - "magmortar", - "togekiss", - "yanmega", - "leafeon", - "glaceon", - "gliscor", - "mamoswine", - "porygon-z", - "gallade", - "probopass", - "dusknoir", - "froslass", - "rotom", - "uxie", - "mesprit", - "azelf", - "dialga", - "palkia", - "heatran", - "regigigas", - "giratina", - "cresselia", - "phione", - "manaphy", - "darkrai", - "shaymin", - "arceus", - "victini", - "snivy", - "servine", - "serperior", - "tepig", - "pignite", - "emboar", - "oshawott", - "dewott", - "samurott", - "patrat", - "watchog", - "lillipup", - "herdier", - "stoutland", - "purrloin", - "liepard", - "pansage", - "simisage", - "pansear", - "simisear", - "panpour", - "simipour", - "munna", - "musharna", - "pidove", - "tranquill", - "unfezant", - "blitzle", - "zebstrika", - "roggenrola", - "boldore", - "gigalith", - "woobat", - "swoobat", - "drilbur", - "excadrill", - "audino", - "timburr", - "gurdurr", - "conkeldurr", - "tympole", - "palpitoad", - "seismitoad", - "throh", - "sawk", - "sewaddle", - "swadloon", - "leavanny", - "venipede", - "whirlipede", - "scolipede", - "cottonee", - "whimsicott", - "petilil", - "lilligant", - "basculin", - "sandile", - "krokorok", - "krookodile", - "darumaka", - "darmanitan", - "maractus", - "dwebble", - "crustle", - "scraggy", - "scrafty", - "sigilyph", - "yamask", - "cofagrigus", - "tirtouga", - "carracosta", - "archen", - "archeops", - "trubbish", - "garbodor", - "zorua", - "zoroark", - "minccino", - "cinccino", - "gothita", - "gothorita", - "gothitelle", - "solosis", - "duosion", - "reuniclus", - "ducklett", - "swanna", - "vanillite", - "vanillish", - "vanilluxe", - "deerling", - "sawsbuck", - "emolga", - "karrablast", - "escavalier", - "foongus", - "amoonguss", - "frillish", - "jellicent", - "alomomola", - "joltik", - "galvantula", - "ferroseed", - "ferrothorn", - "klink", - "klang", - "klinklang", - "tynamo", - "eelektrik", - "eelektross", - "elgyem", - "beheeyem", - "litwick", - "lampent", - "chandelure", - "axew", - "fraxure", - "haxorus", - "cubchoo", - "beartic", - "cryogonal", - "shelmet", - "accelgor", - "stunfisk", - "mienfoo", - "mienshao", - "druddigon", - "golett", - "golurk", - "pawniard", - "bisharp", - "bouffalant", - "rufflet", - "braviary", - "vullaby", - "mandibuzz", - "heatmor", - "durant", - "deino", - "zweilous", - "hydreigon", - "larvesta", - "volcarona", - "cobalion", - "terrakion", - "virizion", - "tornadus", - "thundurus", - "reshiram", - "zekrom", - "landorus", - "kyurem", - "keldeo", - "meloetta", - "genesect", - "chespin", - "quilladin", - "chesnaught", - "fennekin", - "braixen", - "delphox", - "froakie", - "frogadier", - "greninja", - "bunnelby", - "diggersby", - "fletchling", - "fletchinder", - "talonflame", - "scatterbug", - "spewpa", - "vivillon", - "litleo", - "pyroar", - "flabebe", - "floette", - "florges", - "skiddo", - "gogoat", - "pancham", - "pangoro", - "furfrou", - "espurr", - "meowstic", - "honedge", - "doublade", - "aegislash", - "spritzee", - "aromatisse", - "swirlix", - "slurpuff", - "inkay", - "malamar", - "binacle", - "barbaracle", - "skrelp", - "dragalge", - "clauncher", - "clawitzer", - "helioptile", - "heliolisk", - "tyrunt", - "tyrantrum", - "amaura", - "aurorus", - "sylveon", - "hawlucha", - "dedenne", - "carbink", - "goomy", - "sliggoo", - "goodra", - "klefki", - "phantump", - "trevenant", - "pumpkaboo", - "gourgeist", - "bergmite", - "avalugg", - "noibat", - "noivern", - "xerneas", - "yveltal", - "zygarde", - "diancie", - "hoopa", - "volcanion", - "rowlet", - "dartrix", - "decidueye", - "litten", - "torracat", - "incineroar", - "popplio", - "brionne", - "primarina", - "pikipek", - "trumbeak", - "toucannon", - "yungoos", - "gumshoos", - "grubbin", - "charjabug", - "vikavolt", - "crabrawler", - "crabominable", - "oricorio", - "cutiefly", - "ribombee", - "rockruff", - "lycanroc", - "wishiwashi", - "mareanie", - "toxapex", - "mudbray", - "mudsdale", - "dewpider", - "araquanid", - "fomantis", - "lurantis", - "morelull", - "shiinotic", - "salandit", - "salazzle", - "stufful", - "bewear", - "bounsweet", - "steenee", - "tsareena", - "comfey", - "oranguru", - "passimian", - "wimpod", - "golisopod", - "sandygast", - "palossand", - "pyukumuku", - "typenull", - "silvally", - "minior", - "komala", - "turtonator", - "togedemaru", - "mimikyu", - "bruxish", - "drampa", - "dhelmise", - "jangmo-o", - "hakamo-o", - "kommo-o", - "tapukoko", - "tapulele", - "tapubulu", - "tapufini", - "cosmog", - "cosmoem", - "solgaleo", - "lunala", - "nihilego", - "buzzwole", - "pheromosa", - "xurkitree", - "celesteela", - "kartana", - "guzzlord", - "necrozma", - "magearna", - "marshadow", - "poipole", - "naganadel", - "stakataka", - "blacephalon", - "zeraora", - "meltan", - "melmetal", - "grookey", - "thwackey", - "rillaboom", - "scorbunny", - "raboot", - "cinderace", - "sobble", - "drizzile", - "inteleon", - "skwovet", - "greedent", - "rookidee", - "corvisquire", - "corviknight", - "blipbug", - "dottler", - "orbeetle", - "nickit", - "thievul", - "gossifleur", - "eldegoss", - "wooloo", - "dubwool", - "chewtle", - "drednaw", - "yamper", - "boltund", - "rolycoly", - "carkol", - "coalossal", - "applin", - "flapple", - "appletun", - "silicobra", - "sandaconda", - "cramorant", - "arrokuda", - "barraskewda", - "toxel", - "toxtricity", - "sizzlipede", - "centiskorch", - "clobbopus", - "grapploct", - "sinistea", - "polteageist", - "hatenna", - "hattrem", - "hatterene", - "impidimp", - "morgrem", - "grimmsnarl", - "obstagoon", - "perrserker", - "cursola", - "sirfetchd", - "mrrime", - "runerigus", - "milcery", - "alcremie", - "falinks", - "pincurchin", - "snom", - "frosmoth", - "stonjourner", - "eiscue", - "indeedee", - "morpeko", - "cufant", - "copperajah", - "dracozolt", - "arctozolt", - "dracovish", - "arctovish", - "duraludon", - "dreepy", - "drakloak", - "dragapult", - "zacian", - "zamazenta", - "eternatus", - "kubfu", - "urshifu", - "zarude", - "regieleki", - "regidrago", - "glastrier", - "spectrier", - "calyrex", -] diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/schemas/TODO.md b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/schemas/TODO.md deleted file mode 100644 index cf1efadb3c9c..000000000000 --- a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/schemas/TODO.md +++ /dev/null @@ -1,25 +0,0 @@ -# TODO: Define your stream schemas -Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). - -The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. - -The schema of a stream is the return value of `Stream.get_json_schema`. - -## Static schemas -By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. - -Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. - -## Dynamic schemas -If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). - -## Dynamically modifying static schemas -Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: -``` -def get_json_schema(self): - schema = super().get_json_schema() - schema['dynamically_determined_property'] = "property" - return schema -``` - -Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/schemas/pokemon.json b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/schemas/pokemon.json index 7c190d9cb6c4..c4cdb193909d 100644 --- a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/schemas/pokemon.json +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/schemas/pokemon.json @@ -14,7 +14,7 @@ "height": { "type": ["null", "integer"] }, - "is_default ": { + "is_default": { "type": ["null", "boolean"] }, "order": { @@ -27,6 +27,7 @@ "type": ["null", "array"], "items": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "is_hidden": { "type": ["null", "boolean"] @@ -36,6 +37,7 @@ }, "ability": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "name": { "type": ["null", "string"] @@ -52,6 +54,7 @@ "type": ["null", "array"], "items": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "name": { "type": ["null", "string"] @@ -66,12 +69,14 @@ "type": ["null", "array"], "items": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "game_index": { "type": ["null", "integer"] }, "version": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "name": { "type": ["null", "string"] @@ -88,9 +93,11 @@ "type": ["null", "array"], "items": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "item": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "name": { "type": ["null", "string"] @@ -104,9 +111,11 @@ "type": ["null", "array"], "items": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "version": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "name": { "type": ["null", "string"] @@ -132,9 +141,11 @@ "type": ["null", "array"], "items": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "move": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "name": { "type": ["null", "string"] @@ -148,9 +159,11 @@ "type": ["null", "array"], "items": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "move_learn_method": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "name": { "type": ["null", "string"] @@ -162,6 +175,7 @@ }, "version_group": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "name": { "type": ["null", "string"] @@ -182,6 +196,7 @@ }, "sprites": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "front_default": { "type": ["null", "string"] @@ -211,6 +226,7 @@ }, "species": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "name": { "type": ["null", "string"] @@ -224,9 +240,11 @@ "type": ["null", "array"], "items": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "stat": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "name": { "type": ["null", "string"] @@ -249,6 +267,7 @@ "type": ["null", "array"], "items": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "slot": { "type": ["null", "integer"] @@ -266,6 +285,51 @@ } } } + }, + "past_types": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "generation": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "name": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + }, + "types": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "slot": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "name": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + } + } + } + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py index 650320579e9c..4e1813d78e7e 100644 --- a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py @@ -2,67 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream +WARNING: Do not modify this file. +""" -from . import pokemon_list - -class PokeapiStream(HttpStream): - url_base = "https://pokeapi.co/api/v2/" - - def __init__(self, pokemon_name: str, **kwargs): - super().__init__(**kwargs) - self.pokemon_name = pokemon_name - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - # The api requires that we include the Pokemon name as a query param so we do that in this method - return {"pokemon_name": self.pokemon_name} - - def parse_response( - self, - response: requests.Response, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[Mapping]: - # The response is a simple JSON whose schema matches our stream's schema exactly, - # so we just return a list containing the response - return [response.json()] - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - # The API does not offer pagination, - # so we return None to indicate there are no more pages in the response - return None - - -class Pokemon(PokeapiStream): - # Set this as a noop. - primary_key = None - - def path(self, **kwargs) -> str: - pokemon_name = self.pokemon_name - return f"pokemon/{pokemon_name}" - - -# Source -class SourcePokeapi(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - input_pokemon = config["pokemon_name"] - if input_pokemon not in pokemon_list.POKEMON_LIST: - return False, f"Input Pokemon {input_pokemon} is invalid. Please check your spelling our input a valid Pokemon." - else: - return True, None - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - return [Pokemon(pokemon_name=config["pokemon_name"])] +# Declarative Source +class SourcePokeapi(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/spec.json b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/spec.json deleted file mode 100644 index 8798a9470575..000000000000 --- a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/spec.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/pokeapi", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Pokeapi Spec", - "type": "object", - "required": ["pokemon_name"], - "additionalProperties": false, - "properties": { - "pokemon_name": { - "type": "string", - "title": "Pokemon Name", - "description": "Pokemon requested from the API.", - "pattern": "^[a-z0-9_\\-]+$", - "examples": ["ditto", "luxray", "snorlax"] - } - } - } -} diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/spec.yaml b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/spec.yaml new file mode 100644 index 000000000000..e08974cd37ec --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/spec.yaml @@ -0,0 +1,916 @@ +documentationUrl: "https://docs.airbyte.com/integrations/sources/pokeapi" +connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Pokeapi Spec" + type: object + required: + - pokemon_name + properties: + pokemon_name: + type: string + title: "Pokemon Name" + description: "Pokemon requested from the API." + pattern: "^[a-z0-9_\\-]+$" + enum: + - bulbasaur + - ivysaur + - venusaur + - charmander + - charmeleon + - charizard + - squirtle + - wartortle + - blastoise + - caterpie + - metapod + - butterfree + - weedle + - kakuna + - beedrill + - pidgey + - pidgeotto + - pidgeot + - rattata + - raticate + - spearow + - fearow + - ekans + - arbok + - pikachu + - raichu + - sandshrew + - sandslash + - nidoranf + - nidorina + - nidoqueen + - nidoranm + - nidorino + - nidoking + - clefairy + - clefable + - vulpix + - ninetales + - jigglypuff + - wigglytuff + - zubat + - golbat + - oddish + - gloom + - vileplume + - paras + - parasect + - venonat + - venomoth + - diglett + - dugtrio + - meowth + - persian + - psyduck + - golduck + - mankey + - primeape + - growlithe + - arcanine + - poliwag + - poliwhirl + - poliwrath + - abra + - kadabra + - alakazam + - machop + - machoke + - machamp + - bellsprout + - weepinbell + - victreebel + - tentacool + - tentacruel + - geodude + - graveler + - golem + - ponyta + - rapidash + - slowpoke + - slowbro + - magnemite + - magneton + - farfetchd + - doduo + - dodrio + - seel + - dewgong + - grimer + - muk + - shellder + - cloyster + - gastly + - haunter + - gengar + - onix + - drowzee + - hypno + - krabby + - kingler + - voltorb + - electrode + - exeggcute + - exeggutor + - cubone + - marowak + - hitmonlee + - hitmonchan + - lickitung + - koffing + - weezing + - rhyhorn + - rhydon + - chansey + - tangela + - kangaskhan + - horsea + - seadra + - goldeen + - seaking + - staryu + - starmie + - mrmime + - scyther + - jynx + - electabuzz + - magmar + - pinsir + - tauros + - magikarp + - gyarados + - lapras + - ditto + - eevee + - vaporeon + - jolteon + - flareon + - porygon + - omanyte + - omastar + - kabuto + - kabutops + - aerodactyl + - snorlax + - articuno + - zapdos + - moltres + - dratini + - dragonair + - dragonite + - mewtwo + - mew + - chikorita + - bayleef + - meganium + - cyndaquil + - quilava + - typhlosion + - totodile + - croconaw + - feraligatr + - sentret + - furret + - hoothoot + - noctowl + - ledyba + - ledian + - spinarak + - ariados + - crobat + - chinchou + - lanturn + - pichu + - cleffa + - igglybuff + - togepi + - togetic + - natu + - xatu + - mareep + - flaaffy + - ampharos + - bellossom + - marill + - azumarill + - sudowoodo + - politoed + - hoppip + - skiploom + - jumpluff + - aipom + - sunkern + - sunflora + - yanma + - wooper + - quagsire + - espeon + - umbreon + - murkrow + - slowking + - misdreavus + - unown + - wobbuffet + - girafarig + - pineco + - forretress + - dunsparce + - gligar + - steelix + - snubbull + - granbull + - qwilfish + - scizor + - shuckle + - heracross + - sneasel + - teddiursa + - ursaring + - slugma + - magcargo + - swinub + - piloswine + - corsola + - remoraid + - octillery + - delibird + - mantine + - skarmory + - houndour + - houndoom + - kingdra + - phanpy + - donphan + - porygon2 + - stantler + - smeargle + - tyrogue + - hitmontop + - smoochum + - elekid + - magby + - miltank + - blissey + - raikou + - entei + - suicune + - larvitar + - pupitar + - tyranitar + - lugia + - ho-oh + - celebi + - treecko + - grovyle + - sceptile + - torchic + - combusken + - blaziken + - mudkip + - marshtomp + - swampert + - poochyena + - mightyena + - zigzagoon + - linoone + - wurmple + - silcoon + - beautifly + - cascoon + - dustox + - lotad + - lombre + - ludicolo + - seedot + - nuzleaf + - shiftry + - taillow + - swellow + - wingull + - pelipper + - ralts + - kirlia + - gardevoir + - surskit + - masquerain + - shroomish + - breloom + - slakoth + - vigoroth + - slaking + - nincada + - ninjask + - shedinja + - whismur + - loudred + - exploud + - makuhita + - hariyama + - azurill + - nosepass + - skitty + - delcatty + - sableye + - mawile + - aron + - lairon + - aggron + - meditite + - medicham + - electrike + - manectric + - plusle + - minun + - volbeat + - illumise + - roselia + - gulpin + - swalot + - carvanha + - sharpedo + - wailmer + - wailord + - numel + - camerupt + - torkoal + - spoink + - grumpig + - spinda + - trapinch + - vibrava + - flygon + - cacnea + - cacturne + - swablu + - altaria + - zangoose + - seviper + - lunatone + - solrock + - barboach + - whiscash + - corphish + - crawdaunt + - baltoy + - claydol + - lileep + - cradily + - anorith + - armaldo + - feebas + - milotic + - castform + - kecleon + - shuppet + - banette + - duskull + - dusclops + - tropius + - chimecho + - absol + - wynaut + - snorunt + - glalie + - spheal + - sealeo + - walrein + - clamperl + - huntail + - gorebyss + - relicanth + - luvdisc + - bagon + - shelgon + - salamence + - beldum + - metang + - metagross + - regirock + - regice + - registeel + - latias + - latios + - kyogre + - groudon + - rayquaza + - jirachi + - deoxys + - turtwig + - grotle + - torterra + - chimchar + - monferno + - infernape + - piplup + - prinplup + - empoleon + - starly + - staravia + - staraptor + - bidoof + - bibarel + - kricketot + - kricketune + - shinx + - luxio + - luxray + - budew + - roserade + - cranidos + - rampardos + - shieldon + - bastiodon + - burmy + - wormadam + - mothim + - combee + - vespiquen + - pachirisu + - buizel + - floatzel + - cherubi + - cherrim + - shellos + - gastrodon + - ambipom + - drifloon + - drifblim + - buneary + - lopunny + - mismagius + - honchkrow + - glameow + - purugly + - chingling + - stunky + - skuntank + - bronzor + - bronzong + - bonsly + - mimejr + - happiny + - chatot + - spiritomb + - gible + - gabite + - garchomp + - munchlax + - riolu + - lucario + - hippopotas + - hippowdon + - skorupi + - drapion + - croagunk + - toxicroak + - carnivine + - finneon + - lumineon + - mantyke + - snover + - abomasnow + - weavile + - magnezone + - lickilicky + - rhyperior + - tangrowth + - electivire + - magmortar + - togekiss + - yanmega + - leafeon + - glaceon + - gliscor + - mamoswine + - porygon-z + - gallade + - probopass + - dusknoir + - froslass + - rotom + - uxie + - mesprit + - azelf + - dialga + - palkia + - heatran + - regigigas + - giratina + - cresselia + - phione + - manaphy + - darkrai + - shaymin + - arceus + - victini + - snivy + - servine + - serperior + - tepig + - pignite + - emboar + - oshawott + - dewott + - samurott + - patrat + - watchog + - lillipup + - herdier + - stoutland + - purrloin + - liepard + - pansage + - simisage + - pansear + - simisear + - panpour + - simipour + - munna + - musharna + - pidove + - tranquill + - unfezant + - blitzle + - zebstrika + - roggenrola + - boldore + - gigalith + - woobat + - swoobat + - drilbur + - excadrill + - audino + - timburr + - gurdurr + - conkeldurr + - tympole + - palpitoad + - seismitoad + - throh + - sawk + - sewaddle + - swadloon + - leavanny + - venipede + - whirlipede + - scolipede + - cottonee + - whimsicott + - petilil + - lilligant + - basculin + - sandile + - krokorok + - krookodile + - darumaka + - darmanitan + - maractus + - dwebble + - crustle + - scraggy + - scrafty + - sigilyph + - yamask + - cofagrigus + - tirtouga + - carracosta + - archen + - archeops + - trubbish + - garbodor + - zorua + - zoroark + - minccino + - cinccino + - gothita + - gothorita + - gothitelle + - solosis + - duosion + - reuniclus + - ducklett + - swanna + - vanillite + - vanillish + - vanilluxe + - deerling + - sawsbuck + - emolga + - karrablast + - escavalier + - foongus + - amoonguss + - frillish + - jellicent + - alomomola + - joltik + - galvantula + - ferroseed + - ferrothorn + - klink + - klang + - klinklang + - tynamo + - eelektrik + - eelektross + - elgyem + - beheeyem + - litwick + - lampent + - chandelure + - axew + - fraxure + - haxorus + - cubchoo + - beartic + - cryogonal + - shelmet + - accelgor + - stunfisk + - mienfoo + - mienshao + - druddigon + - golett + - golurk + - pawniard + - bisharp + - bouffalant + - rufflet + - braviary + - vullaby + - mandibuzz + - heatmor + - durant + - deino + - zweilous + - hydreigon + - larvesta + - volcarona + - cobalion + - terrakion + - virizion + - tornadus + - thundurus + - reshiram + - zekrom + - landorus + - kyurem + - keldeo + - meloetta + - genesect + - chespin + - quilladin + - chesnaught + - fennekin + - braixen + - delphox + - froakie + - frogadier + - greninja + - bunnelby + - diggersby + - fletchling + - fletchinder + - talonflame + - scatterbug + - spewpa + - vivillon + - litleo + - pyroar + - flabebe + - floette + - florges + - skiddo + - gogoat + - pancham + - pangoro + - furfrou + - espurr + - meowstic + - honedge + - doublade + - aegislash + - spritzee + - aromatisse + - swirlix + - slurpuff + - inkay + - malamar + - binacle + - barbaracle + - skrelp + - dragalge + - clauncher + - clawitzer + - helioptile + - heliolisk + - tyrunt + - tyrantrum + - amaura + - aurorus + - sylveon + - hawlucha + - dedenne + - carbink + - goomy + - sliggoo + - goodra + - klefki + - phantump + - trevenant + - pumpkaboo + - gourgeist + - bergmite + - avalugg + - noibat + - noivern + - xerneas + - yveltal + - zygarde + - diancie + - hoopa + - volcanion + - rowlet + - dartrix + - decidueye + - litten + - torracat + - incineroar + - popplio + - brionne + - primarina + - pikipek + - trumbeak + - toucannon + - yungoos + - gumshoos + - grubbin + - charjabug + - vikavolt + - crabrawler + - crabominable + - oricorio + - cutiefly + - ribombee + - rockruff + - lycanroc + - wishiwashi + - mareanie + - toxapex + - mudbray + - mudsdale + - dewpider + - araquanid + - fomantis + - lurantis + - morelull + - shiinotic + - salandit + - salazzle + - stufful + - bewear + - bounsweet + - steenee + - tsareena + - comfey + - oranguru + - passimian + - wimpod + - golisopod + - sandygast + - palossand + - pyukumuku + - typenull + - silvally + - minior + - komala + - turtonator + - togedemaru + - mimikyu + - bruxish + - drampa + - dhelmise + - jangmo-o + - hakamo-o + - kommo-o + - tapukoko + - tapulele + - tapubulu + - tapufini + - cosmog + - cosmoem + - solgaleo + - lunala + - nihilego + - buzzwole + - pheromosa + - xurkitree + - celesteela + - kartana + - guzzlord + - necrozma + - magearna + - marshadow + - poipole + - naganadel + - stakataka + - blacephalon + - zeraora + - meltan + - melmetal + - grookey + - thwackey + - rillaboom + - scorbunny + - raboot + - cinderace + - sobble + - drizzile + - inteleon + - skwovet + - greedent + - rookidee + - corvisquire + - corviknight + - blipbug + - dottler + - orbeetle + - nickit + - thievul + - gossifleur + - eldegoss + - wooloo + - dubwool + - chewtle + - drednaw + - yamper + - boltund + - rolycoly + - carkol + - coalossal + - applin + - flapple + - appletun + - silicobra + - sandaconda + - cramorant + - arrokuda + - barraskewda + - toxel + - toxtricity + - sizzlipede + - centiskorch + - clobbopus + - grapploct + - sinistea + - polteageist + - hatenna + - hattrem + - hatterene + - impidimp + - morgrem + - grimmsnarl + - obstagoon + - perrserker + - cursola + - sirfetchd + - mrrime + - runerigus + - milcery + - alcremie + - falinks + - pincurchin + - snom + - frosmoth + - stonjourner + - eiscue + - indeedee + - morpeko + - cufant + - copperajah + - dracozolt + - arctozolt + - dracovish + - arctovish + - duraludon + - dreepy + - drakloak + - dragapult + - zacian + - zamazenta + - eternatus + - kubfu + - urshifu + - zarude + - regieleki + - regidrago + - glastrier + - spectrier + - calyrex + examples: + - ditto + - luxray + - snorlax diff --git a/airbyte-integrations/connectors/source-pokeapi/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-pokeapi/unit_tests/unit_test.py deleted file mode 100644 index 219ae0142c72..000000000000 --- a/airbyte-integrations/connectors/source-pokeapi/unit_tests/unit_test.py +++ /dev/null @@ -1,7 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -def test_example_method(): - assert True diff --git a/airbyte-integrations/connectors/source-polygon-stock-api/README.md b/airbyte-integrations/connectors/source-polygon-stock-api/README.md index 2e63a49667f3..de012c61b4db 100644 --- a/airbyte-integrations/connectors/source-polygon-stock-api/README.md +++ b/airbyte-integrations/connectors/source-polygon-stock-api/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-polygon-stock-api:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/polygon-stock-api) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_polygon_stock_api/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-polygon-stock-api:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-polygon-stock-api build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-polygon-stock-api:airbyteDocker +An image will be built with the tag `airbyte/source-polygon-stock-api:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-polygon-stock-api:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-polygon-stock-api:dev docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-polygon-stock-api:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-polygon-stock-api:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-polygon-stock-api test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-polygon-stock-api:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-polygon-stock-api:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-polygon-stock-api test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/polygon-stock-api.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-polygon-stock-api/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-polygon-stock-api/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-polygon-stock-api/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-polygon-stock-api/build.gradle b/airbyte-integrations/connectors/source-polygon-stock-api/build.gradle deleted file mode 100644 index 77b2bebfc5be..000000000000 --- a/airbyte-integrations/connectors/source-polygon-stock-api/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_polygon_stock_api' -} diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/.dockerignore b/airbyte-integrations/connectors/source-postgres-strict-encrypt/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile deleted file mode 100644 index 6dbaaab0c232..000000000000 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-postgres-strict-encrypt - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-postgres-strict-encrypt - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=3.1.5 -LABEL io.airbyte.name=airbyte/source-postgres-strict-encrypt diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/acceptance-test-config.yml b/airbyte-integrations/connectors/source-postgres-strict-encrypt/acceptance-test-config.yml deleted file mode 100644 index f60e772766af..000000000000 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/acceptance-test-config.yml +++ /dev/null @@ -1,50 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-postgres-strict-encrypt:dev -custom_environment_variables: - USE_STREAM_CAPABLE_STATE: true -acceptance_tests: - spec: - tests: - - spec_path: "src/test-integration/resources/expected_strict_encrypt_spec.json" - config_path: "secrets/config.json" - backward_compatibility_tests_config: - disable_for_version: "1.0.52" - - spec_path: "src/test-integration/resources/expected_strict_encrypt_spec.json" - config_path: "secrets/config_cdc.json" - backward_compatibility_tests_config: - disable_for_version: "1.0.52" - connection: - tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "secrets/config_cdc.json" - status: "succeed" - discovery: - tests: - - config_path: "secrets/config.json" - - config_path: "secrets/config_cdc.json" - backward_compatibility_tests_config: - disable_for_version: "2.1.1" - basic_read: - tests: - - config_path: "secrets/config.json" - expect_records: - path: "integration_tests/expected_records.txt" - - config_path: "secrets/config_cdc.json" - expect_records: - path: "integration_tests/expected_records.txt" - full_refresh: - tests: - - config_path: "secrets/config.json" - - config_path: "secrets/config_cdc.json" -# incremental: -# tests: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/incremental_configured_catalog.json" -# future_state: -# bypass_reason: "A java.lang.NullPointerException is thrown when a state with an invalid cursor value is passed" -# - config_path: "secrets/config_cdc.json" -# configured_catalog_path: "integration_tests/incremental_configured_catalog.json" -# future_state: -# bypass_reason: "A java.lang.NullPointerException is thrown when a state with an invalid cursor value is passed" diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/build.gradle b/airbyte-integrations/connectors/source-postgres-strict-encrypt/build.gradle deleted file mode 100644 index 484e154ad89f..000000000000 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/build.gradle +++ /dev/null @@ -1,37 +0,0 @@ -plugins { - id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' -} - -application { - mainClass = 'io.airbyte.integrations.source.postgres_strict_encrypt.PostgresSourceStrictEncrypt' - applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] -} - -dependencies { - implementation project(':airbyte-db:db-lib') - - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:connectors:source-postgres') - implementation libs.airbyte.protocol - // todo (cgardens): why are these needed? - implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation project(':airbyte-integrations:connectors:source-relational-db') - - implementation 'org.apache.commons:commons-lang3:3.11' - implementation libs.postgresql - - testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) - testImplementation project(':airbyte-test-utils') - testImplementation libs.junit.jupiter.system.stubs - - testImplementation libs.connectors.testcontainers.jdbc - testImplementation libs.connectors.testcontainers.postgresql - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) -} - diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/icon.svg b/airbyte-integrations/connectors/source-postgres-strict-encrypt/icon.svg deleted file mode 100644 index 0c88b0ec1aad..000000000000 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/icon.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/README.md b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/README.md deleted file mode 100644 index 45e74b238d3c..000000000000 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/README.md +++ /dev/null @@ -1,5 +0,0 @@ -This directory contains files used to run Connector Acceptance Tests. -* `abnormal_state.json` describes a connector state with a non-existing cursor value. -* `expected_records.txt` lists all the records expected as the output of the basic read operation. -* `incremental_configured_catalog.json` is a configured catalog used as an input of the `incremental` test. -* `seed.sql` is the query we manually ran on a test postgres instance to seed it with test data and enable CDC. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/abnormal_state.json deleted file mode 100644 index c3e6b23a2b0d..000000000000 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/abnormal_state.json +++ /dev/null @@ -1,18 +0,0 @@ -[ - { - "type": "STREAM", - "stream": { - "stream_state": { - "stream_name": "id_and_name", - "stream_namespace": "public", - "cursor_field": ["id"], - "cursor": "4", - "cursor_record_count": 1 - }, - "stream_descriptor": { - "name": "id_and_name", - "namespace": "public" - } - } - } -] diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/expected_records.txt deleted file mode 100644 index d886fbe3fe03..000000000000 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/expected_records.txt +++ /dev/null @@ -1,3 +0,0 @@ -{"stream": "id_and_name", "data": {"id": 1, "name": "picard"}, "emitted_at": 999999} -{"stream": "id_and_name", "data": {"id": 2, "name": "crusher"}, "emitted_at": 999999} -{"stream": "id_and_name", "data": {"id": 3, "name": "vash"}, "emitted_at": 999999} diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/incremental_configured_catalog.json b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/incremental_configured_catalog.json deleted file mode 100644 index 648876fb50e1..000000000000 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/incremental_configured_catalog.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "id_and_name", - "json_schema": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "id": { - "type": "number", - "airbyte_type": "integer" - } - } - }, - "supported_sync_modes": ["full_refresh", "incremental"], - "default_cursor_field": [], - "source_defined_primary_key": [], - "namespace": "public" - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["id"], - "user_defined_primary_key": ["id"] - } - ] -} diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed.sql b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed.sql deleted file mode 100644 index 48910082f86f..000000000000 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed.sql +++ /dev/null @@ -1,36 +0,0 @@ -ALTER ROLE postgres WITH REPLICATION; - -CREATE - TABLE - id_and_name( - id INTEGER, - name VARCHAR(200) - ); - -INSERT - INTO - id_and_name( - id, - name - ) - VALUES( - 1, - 'picard' - ), - ( - 2, - 'crusher' - ), - ( - 3, - 'vash' - ); - -SELECT - pg_create_logical_replication_slot( - 'debezium_slot', - 'pgoutput' - ); - -CREATE - PUBLICATION publication FOR ALL TABLES; diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed/basic.sql b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed/basic.sql deleted file mode 100644 index d7cb4fda899e..000000000000 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed/basic.sql +++ /dev/null @@ -1,334 +0,0 @@ -CREATE - SCHEMA POSTGRES_BASIC; - -CREATE - TYPE mood AS ENUM( - 'sad', - 'ok', - 'happy' - ); - -CREATE - TYPE inventory_item AS( - name text, - supplier_id INTEGER, - price NUMERIC - ); -SET -lc_monetary TO 'en_US.utf8'; -SET -TIMEZONE TO 'MST'; - -CREATE - EXTENSION hstore; - -CREATE - TABLE - POSTGRES_BASIC.TEST_DATASET( - id INTEGER PRIMARY KEY, - test_column_1 BIGINT, - test_column_11 CHAR, - test_column_12 CHAR(8), - test_column_13 CHARACTER, - test_column_14 CHARACTER(8), - test_column_15 text, - test_column_16 VARCHAR, - test_column_20 DATE NOT NULL DEFAULT now(), - test_column_21 DATE, - test_column_23 FLOAT, - test_column_24 DOUBLE PRECISION, - test_column_27 INT, - test_column_28 INTEGER, - test_column_3 BIT(1), - test_column_4 BIT(3), - test_column_44 REAL, - test_column_46 SMALLINT, - test_column_51 TIME WITHOUT TIME ZONE, - test_column_52 TIME, - test_column_53 TIME WITHOUT TIME ZONE NOT NULL DEFAULT now(), - test_column_54 TIMESTAMP, - test_column_55 TIMESTAMP WITHOUT TIME ZONE, - test_column_56 TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), - test_column_57 TIMESTAMP, - test_column_58 TIMESTAMP WITHOUT TIME ZONE, - test_column_59 TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT now(), - test_column_60 TIMESTAMP WITH TIME ZONE, - test_column_61 timestamptz, - test_column_7 bool, - test_column_70 TIME WITH TIME ZONE, - test_column_71 timetz, - test_column_8 BOOLEAN - ); - -INSERT - INTO - POSTGRES_BASIC.TEST_DATASET - VALUES( - 1, - - 9223372036854775808, - 'a', - '{asb123}', - 'a', - '{asb123}', - 'a', - 'a', - '1999-01-08', - '1999-01-08', - '123', - '123', - 1001, - 1001, - B'0', - B'101', - 3.4145, - - 32768, - '13:00:01', - '13:00:01', - '13:00:01', - TIMESTAMP '2004-10-19 10:23:00', - TIMESTAMP '2004-10-19 10:23:00', - TIMESTAMP '2004-10-19 10:23:00', - 0, - 0, - 0, - TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', - TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', - TRUE, - '13:00:01', - '13:00:01', - TRUE - ); - -INSERT - INTO - POSTGRES_BASIC.TEST_DATASET - VALUES( - 2, - 9223372036854775807, - '*', - '{asb12}', - '*', - '{asb12}', - 'abc', - 'abc', - '1991-02-10 BC', - '1991-02-10 BC', - '1234567890.1234567', - '1234567890.1234567', - - 2147483648, - - 2147483648, - B'0', - B'101', - 3.4145, - 32767, - '13:00:02+8', - '13:00:02+8', - '13:00:02+8', - TIMESTAMP '2004-10-19 10:23:54.123456', - TIMESTAMP '2004-10-19 10:23:54.123456', - TIMESTAMP '2004-10-19 10:23:54.123456', - 0, - 0, - 0, - TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', - TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', - 'yes', - '13:00:00+8', - '13:00:00+8', - 'yes' - ); - -INSERT - INTO - POSTGRES_BASIC.TEST_DATASET - VALUES( - 3, - 0, - '*', - '{asb12}', - '*', - '{asb12}', - 'Миші йдуть на південь, не питай чому;', - 'Миші йдуть на південь, не питай чому;', - '1991-02-10 BC', - '1991-02-10 BC', - '1234567890.1234567', - '1234567890.1234567', - 2147483647, - 2147483647, - B'0', - B'101', - 3.4145, - 32767, - '13:00:03-8', - '13:00:03-8', - '13:00:03-8', - TIMESTAMP '3004-10-19 10:23:54.123456 BC', - TIMESTAMP '3004-10-19 10:23:54.123456 BC', - TIMESTAMP '3004-10-19 10:23:54.123456 BC', - 0, - 0, - 0, - TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', - TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', - '1', - '13:00:03-8', - '13:00:03-8', - '1' - ); - -INSERT - INTO - POSTGRES_BASIC.TEST_DATASET - VALUES( - 4, - 0, - '*', - '{asb12}', - '*', - '{asb12}', - '櫻花分店', - '櫻花分店', - '1991-02-10 BC', - '1991-02-10 BC', - '1234567890.1234567', - '1234567890.1234567', - 2147483647, - 2147483647, - B'0', - B'101', - 3.4145, - 32767, - '13:00:04Z', - '13:00:04Z', - '13:00:04Z', - TIMESTAMP '0001-01-01 00:00:00.000000', - TIMESTAMP '0001-01-01 00:00:00.000000', - TIMESTAMP '0001-01-01 00:00:00.000000', - 0, - 0, - 0, - TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', - TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', - FALSE, - '13:00:04Z', - '13:00:04Z', - FALSE - ); - -INSERT - INTO - POSTGRES_BASIC.TEST_DATASET - VALUES( - 5, - 0, - '*', - '{asb12}', - '*', - '{asb12}', - '', - '', - '1991-02-10 BC', - '1991-02-10 BC', - '1234567890.1234567', - '1234567890.1234567', - 2147483647, - 2147483647, - B'0', - B'101', - 3.4145, - 32767, - '13:00:05.01234Z+8', - '13:00:05.01234Z+8', - '13:00:05.01234Z+8', - TIMESTAMP '0001-12-31 23:59:59.999999 BC', - TIMESTAMP '0001-12-31 23:59:59.999999 BC', - TIMESTAMP '0001-12-31 23:59:59.999999 BC', - 0, - 0, - 0, - TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', - TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', - 'no', - '13:00:05.012345Z+8', - '13:00:05.012345Z+8', - 'no' - ); - -INSERT - INTO - POSTGRES_BASIC.TEST_DATASET - VALUES( - 6, - 0, - '*', - '{asb12}', - '*', - '{asb12}', - '\xF0\x9F\x9A\x80', - '\xF0\x9F\x9A\x80', - '1991-02-10 BC', - '1991-02-10 BC', - '1234567890.1234567', - '1234567890.1234567', - 2147483647, - 2147483647, - B'0', - B'101', - 3.4145, - 32767, - '13:00:00Z-8', - '13:00:00Z-8', - '13:00:00Z-8', - 'epoch', - 'epoch', - 'epoch', - 0, - 0, - 0, - TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', - TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', - '0', - '13:00:06.00000Z-8', - '13:00:06.00000Z-8', - '0' - ); - -INSERT - INTO - POSTGRES_BASIC.TEST_DATASET - VALUES( - 7, - 0, - '*', - '{asb12}', - '*', - '{asb12}', - '\xF0\x9F\x9A\x80', - '\xF0\x9F\x9A\x80', - '1991-02-10 BC', - '1991-02-10 BC', - '1234567890.1234567', - '1234567890.1234567', - 2147483647, - 2147483647, - B'0', - B'101', - 3.4145, - 32767, - '24:00:00', - '24:00:00', - '24:00:00', - 'epoch', - 'epoch', - 'epoch', - 0, - 0, - 0, - TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', - TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', - '0', - '13:00:06.00000Z-8', - '13:00:06.00000Z-8', - '0' - ); diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed/full.sql b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed/full.sql deleted file mode 100644 index 26edf7749a9f..000000000000 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed/full.sql +++ /dev/null @@ -1,825 +0,0 @@ -CREATE - SCHEMA POSTGRES_FULL; - -CREATE - TYPE mood AS ENUM( - 'sad', - 'ok', - 'happy' - ); - -CREATE - TYPE inventory_item AS( - name text, - supplier_id INTEGER, - price NUMERIC - ); -SET -lc_monetary TO 'en_US.utf8'; -SET -TIMEZONE TO 'MST'; - -CREATE - EXTENSION hstore; - -CREATE - TABLE - POSTGRES_FULL.TEST_DATASET( - id INTEGER PRIMARY KEY, - test_column_1 BIGINT, - test_column_10 bytea, - test_column_11 CHAR, - test_column_12 CHAR(8), - test_column_13 CHARACTER, - test_column_14 CHARACTER(8), - test_column_15 text, - test_column_16 VARCHAR, - test_column_17 CHARACTER VARYING(10), - test_column_18 cidr, - test_column_19 circle, - test_column_2 bigserial, - test_column_20 DATE NOT NULL DEFAULT now(), - test_column_21 DATE, - test_column_22 float8, - test_column_23 FLOAT, - test_column_24 DOUBLE PRECISION, - test_column_25 inet, - test_column_26 int4, - test_column_27 INT, - test_column_28 INTEGER, - test_column_29 INTERVAL, - test_column_3 BIT(1), - test_column_30 json, - test_column_31 jsonb, - test_column_32 line, - test_column_33 lseg, - test_column_34 macaddr, - test_column_35 macaddr8, - test_column_36 money, - test_column_37 DECIMAL, - test_column_38 NUMERIC, - test_column_39 PATH, - test_column_4 BIT(3), - test_column_40 pg_lsn, - test_column_41 point, - test_column_42 polygon, - test_column_43 float4, - test_column_44 REAL, - test_column_45 int2, - test_column_46 SMALLINT, - test_column_47 serial2, - test_column_48 smallserial, - test_column_49 serial4, - test_column_5 BIT VARYING(5), - test_column_51 TIME WITHOUT TIME ZONE, - test_column_52 TIME, - test_column_53 TIME WITHOUT TIME ZONE NOT NULL DEFAULT now(), - test_column_54 TIMESTAMP, - test_column_55 TIMESTAMP WITHOUT TIME ZONE, - test_column_56 TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), - test_column_57 TIMESTAMP, - test_column_58 TIMESTAMP WITHOUT TIME ZONE, - test_column_59 TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT now(), - test_column_6 BIT VARYING(5), - test_column_60 TIMESTAMP WITH TIME ZONE, - test_column_61 timestamptz, - test_column_62 tsquery, - test_column_63 tsvector, - test_column_64 uuid, - test_column_65 xml, - test_column_66 mood, - test_column_67 tsrange, - test_column_68 inventory_item, - test_column_69 hstore, - test_column_7 bool, - test_column_70 TIME WITH TIME ZONE, - test_column_71 timetz, - test_column_72 INT2 [], - test_column_73 INT4 [], - test_column_74 INT8 [], - test_column_75 OID [], - test_column_76 VARCHAR [], - test_column_77 CHAR(1)[], - test_column_78 BPCHAR(2)[], - test_column_79 TEXT [], - test_column_8 BOOLEAN, - test_column_80 NAME [], - test_column_81 NUMERIC [], - test_column_82 DECIMAL [], - test_column_83 FLOAT4 [], - test_column_84 FLOAT8 [], - test_column_85 MONEY [], - test_column_86 BOOL [], - test_column_87 BIT [], - test_column_88 BYTEA [], - test_column_89 DATE [], - test_column_9 box, - test_column_90 TIME(6)[], - test_column_91 TIMETZ [], - test_column_92 TIMESTAMPTZ [], - test_column_93 TIMESTAMP [] - ); - -INSERT - INTO - POSTGRES_FULL.TEST_DATASET - VALUES( - 1, - - 9223372036854775808, - NULL, - 'a', - '{asb123}', - 'a', - '{asb123}', - 'a', - 'a', - '{asb123}', - NULL, - '(5,7),10', - 1, - '1999-01-08', - '1999-01-08', - '123', - '123', - '123', - '198.24.10.0/24', - NULL, - NULL, - NULL, - NULL, - B'0', - NULL, - NULL, - '{4,5,6}', - '((3,7),(15,18))', - NULL, - NULL, - NULL, - '123', - '123', - '((3,7),(15,18))', - B'101', - '7/A25801C8'::pg_lsn, - '(3,7)', - '((3,7),(15,18))', - NULL, - NULL, - NULL, - NULL, - 1, - 1, - 1, - B'101', - '13:00:01', - '13:00:01', - '13:00:01', - TIMESTAMP '2004-10-19 10:23:00', - TIMESTAMP '2004-10-19 10:23:00', - TIMESTAMP '2004-10-19 10:23:00', - 'infinity', - 'infinity', - 'infinity', - B'101', - TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', - TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', - NULL, - to_tsvector('The quick brown fox jumped over the lazy dog.'), - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - XMLPARSE( - DOCUMENT 'Manual...' - ), - 'happy', - '(2010-01-01 14:30, 2010-01-01 15:30)', - ROW( - 'fuzzy dice', - 42, - 1.99 - ), - '"paperback" => "243","publisher" => "postgresqltutorial.com", -"language" => "English","ISBN-13" => "978-1449370000", -"weight" => "11.2 ounces"', - TRUE, - NULL, - NULL, - '{1,2,3}', - '{-2147483648,2147483646}', - '{-9223372036854775808,9223372036854775801}', - '{564182,234181}', - '{lorem ipsum,dolor sit,amet}', - '{l,d,a}', - '{l,d,a}', - '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', - TRUE, - '{object,integer}', - '{131070.23,231072.476596593}', - '{131070.23,231072.476596593}', - '{131070.237689,231072.476596593}', - '{131070.237689,231072.476596593}', - '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', - '{true,yes,1,false,no,0,null}', - '{null,1,0}', - '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', - '{1999-01-08,1991-02-10 BC}', - '((3,7),(15,18))', - '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', - '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', - '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', - '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' - ); - -INSERT - INTO - POSTGRES_FULL.TEST_DATASET - VALUES( - 2, - 9223372036854775807, - decode( - '1234', - 'hex' - ), - '*', - '{asb12}', - '*', - '{asb12}', - 'abc', - 'abc', - '{asb12}', - '192.168.100.128/25', - '(0,0),0', - 9223372036854775807, - '1991-02-10 BC', - '1991-02-10 BC', - '1234567890.1234567', - '1234567890.1234567', - '1234567890.1234567', - '198.24.10.0', - 1001, - 1001, - 1001, - 'P1Y2M3DT4H5M6S', - NULL, - '{"a": 10, "b": 15}', - '[1, 2, 3]'::jsonb, - '{0,1,0}', - '((0,0),(0,0))', - '08:00:2b:01:02:03', - '08:00:2b:01:02:03:04:05', - '999.99', - NULL, - NULL, - '((0,0),(0,0))', - NULL, - '0/0'::pg_lsn, - '(0,0)', - '((0,0),(0,0))', - 3.4145, - 3.4145, - - 32768, - - 32768, - 32767, - 32767, - 2147483647, - NULL, - '13:00:02+8', - '13:00:02+8', - '13:00:02+8', - TIMESTAMP '2004-10-19 10:23:54.123456', - TIMESTAMP '2004-10-19 10:23:54.123456', - TIMESTAMP '2004-10-19 10:23:54.123456', - '-infinity', - '-infinity', - '-infinity', - NULL, - TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', - TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', - 'fat & (rat | cat)'::tsquery, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - 'yes', - '13:00:01', - '13:00:01', - '{4,5,6}', - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - 'yes', - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - '((0,0),(0,0))', - NULL, - NULL, - NULL, - NULL - ); - -INSERT - INTO - POSTGRES_FULL.TEST_DATASET - VALUES( - 3, - 0, - '1234', - NULL, - NULL, - NULL, - NULL, - 'Миші йдуть на південь, не питай чому;', - 'Миші йдуть на південь, не питай чому;', - NULL, - '192.168/24', - '(-10,-4),10', - 0, - NULL, - NULL, - NULL, - NULL, - NULL, - '198.10/8', - - 2147483648, - - 2147483648, - - 2147483648, - '-178000000', - NULL, - NULL, - NULL, - NULL, - NULL, - '08-00-2b-01-02-04', - '08-00-2b-01-02-03-04-06', - '1,001.01', - '1234567890.1234567', - '1234567890.1234567', - NULL, - NULL, - NULL, - '(999999999999999999999999,0)', - '((0,0),(999999999999999999999999,0))', - NULL, - NULL, - 32767, - 32767, - 0, - 0, - 0, - NULL, - '13:00:03-8', - '13:00:03-8', - '13:00:03-8', - TIMESTAMP '3004-10-19 10:23:54.123456 BC', - TIMESTAMP '3004-10-19 10:23:54.123456 BC', - TIMESTAMP '3004-10-19 10:23:54.123456 BC', - NULL, - NULL, - NULL, - NULL, - TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', - TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', - 'fat:ab & cat'::tsquery, - NULL, - NULL, - '', - NULL, - NULL, - NULL, - NULL, - '1', - '13:00:00+8', - '13:00:00+8', - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - '1', - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL - ); - -INSERT - INTO - POSTGRES_FULL.TEST_DATASET - VALUES( - 4, - NULL, - 'abcd', - NULL, - NULL, - NULL, - NULL, - '櫻花分店', - '櫻花分店', - NULL, - '192.168.1', - NULL, - - 9223372036854775808, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - 2147483647, - 2147483647, - 2147483647, - '178000000', - NULL, - NULL, - NULL, - NULL, - NULL, - '08002b:010205', - '08002b:0102030407', - '-1,000', - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - - 32767, - - 32767, - - 2147483647, - NULL, - '13:00:04Z', - '13:00:04Z', - '13:00:04Z', - TIMESTAMP '0001-01-01 00:00:00.000000', - TIMESTAMP '0001-01-01 00:00:00.000000', - TIMESTAMP '0001-01-01 00:00:00.000000', - NULL, - NULL, - NULL, - NULL, - TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', - TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - FALSE, - '13:00:03-8', - '13:00:03-8', - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - FALSE, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL - ); - -INSERT - INTO - POSTGRES_FULL.TEST_DATASET - VALUES( - 5, - NULL, - '\xabcd', - NULL, - NULL, - NULL, - NULL, - '', - '', - NULL, - '128.1', - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - '$999.99', - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - '13:00:05.01234Z+8', - '13:00:05.01234Z+8', - '13:00:05.01234Z+8', - TIMESTAMP '0001-12-31 23:59:59.999999 BC', - TIMESTAMP '0001-12-31 23:59:59.999999 BC', - TIMESTAMP '0001-12-31 23:59:59.999999 BC', - NULL, - NULL, - NULL, - NULL, - TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', - TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - 'no', - '13:00:04Z', - '13:00:04Z', - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - 'no', - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL - ); - -INSERT - INTO - POSTGRES_FULL.TEST_DATASET - VALUES( - 6, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - '2001:4f8:3:ba::/64', - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - '$1001.01', - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - '13:00:00Z-8', - '13:00:00Z-8', - '13:00:00Z-8', - 'epoch', - 'epoch', - 'epoch', - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - '0', - '13:00:05.012345Z+8', - '13:00:05.012345Z+8', - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - '0', - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL - ); - -INSERT - INTO - POSTGRES_FULL.TEST_DATASET - VALUES( - 7, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - '\xF0\x9F\x9A\x80', - '\xF0\x9F\x9A\x80', - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - '-$1,000', - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - '24:00:00', - '24:00:00', - '24:00:00', - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - '13:00:06.00000Z-8', - '13:00:06.00000Z-8', - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL - ); diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed/full_without_nulls.sql b/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed/full_without_nulls.sql deleted file mode 100644 index f101c9b7a3fa..000000000000 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/integration_tests/seed/full_without_nulls.sql +++ /dev/null @@ -1,861 +0,0 @@ -CREATE - SCHEMA POSTGRES_FULL_NN; - -CREATE - TYPE mood AS ENUM( - 'sad', - 'ok', - 'happy' - ); - -CREATE - TYPE inventory_item AS( - name text, - supplier_id INTEGER, - price NUMERIC - ); -SET -lc_monetary TO 'en_US.utf8'; -SET -TIMEZONE TO 'MST'; - -CREATE - EXTENSION hstore; - -CREATE - TABLE - POSTGRES_FULL_NN.TEST_DATASET( - id INTEGER PRIMARY KEY, - test_column_1 BIGINT, - test_column_10 bytea, - test_column_11 CHAR, - test_column_12 CHAR(8), - test_column_13 CHARACTER, - test_column_14 CHARACTER(8), - test_column_15 text, - test_column_16 VARCHAR, - test_column_17 CHARACTER VARYING(10), - test_column_18 cidr, - test_column_19 circle, - test_column_2 bigserial, - test_column_20 DATE NOT NULL DEFAULT now(), - test_column_21 DATE, - test_column_22 float8, - test_column_23 FLOAT, - test_column_24 DOUBLE PRECISION, - test_column_25 inet, - test_column_26 int4, - test_column_27 INT, - test_column_28 INTEGER, - test_column_29 INTERVAL, - test_column_3 BIT(1), - test_column_30 json, - test_column_31 jsonb, - test_column_32 line, - test_column_33 lseg, - test_column_34 macaddr, - test_column_35 macaddr8, - test_column_36 money, - test_column_37 DECIMAL, - test_column_38 NUMERIC, - test_column_39 PATH, - test_column_4 BIT(3), - test_column_40 pg_lsn, - test_column_41 point, - test_column_42 polygon, - test_column_43 float4, - test_column_44 REAL, - test_column_45 int2, - test_column_46 SMALLINT, - test_column_47 serial2, - test_column_48 smallserial, - test_column_49 serial4, - test_column_5 BIT VARYING(5), - test_column_51 TIME WITHOUT TIME ZONE, - test_column_52 TIME, - test_column_53 TIME WITHOUT TIME ZONE NOT NULL DEFAULT now(), - test_column_54 TIMESTAMP, - test_column_55 TIMESTAMP WITHOUT TIME ZONE, - test_column_56 TIMESTAMP WITHOUT TIME ZONE DEFAULT now(), - test_column_57 TIMESTAMP, - test_column_58 TIMESTAMP WITHOUT TIME ZONE, - test_column_59 TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT now(), - test_column_6 BIT VARYING(5), - test_column_60 TIMESTAMP WITH TIME ZONE, - test_column_61 timestamptz, - test_column_62 tsquery, - test_column_63 tsvector, - test_column_64 uuid, - test_column_65 xml, - test_column_66 mood, - test_column_67 tsrange, - test_column_68 inventory_item, - test_column_69 hstore, - test_column_7 bool, - test_column_70 TIME WITH TIME ZONE, - test_column_71 timetz, - test_column_72 INT2 [], - test_column_73 INT4 [], - test_column_74 INT8 [], - test_column_75 OID [], - test_column_76 VARCHAR [], - test_column_77 CHAR(1)[], - test_column_78 BPCHAR(2)[], - test_column_79 TEXT [], - test_column_8 BOOLEAN, - test_column_80 NAME [], - test_column_81 NUMERIC [], - test_column_82 DECIMAL [], - test_column_83 FLOAT4 [], - test_column_84 FLOAT8 [], - test_column_85 MONEY [], - test_column_86 BOOL [], - test_column_87 BIT [], - test_column_88 BYTEA [], - test_column_89 DATE [], - test_column_9 box, - test_column_90 TIME(6)[], - test_column_91 TIMETZ [], - test_column_92 TIMESTAMPTZ [], - test_column_93 TIMESTAMP [] - ); - -INSERT - INTO - POSTGRES_FULL_NN.TEST_DATASET - VALUES( - 1, - - 9223372036854775808, - decode( - '1234', - 'hex' - ), - 'a', - '{asb123}', - 'a', - '{asb123}', - 'a', - 'a', - '{asb123}', - '192.168.100.128/25', - '(5,7),10', - 1, - '1999-01-08', - '1999-01-08', - '123', - '123', - '123', - '198.24.10.0/24', - 1001, - 1001, - 1001, - 'P1Y2M3DT4H5M6S', - B'0', - '{"a": 10, "b": 15}', - '[1, 2, 3]'::jsonb, - '{4,5,6}', - '((3,7),(15,18))', - '08:00:2b:01:02:03', - '08:00:2b:01:02:03:04:05', - '999.99', - '123', - '123', - '((3,7),(15,18))', - B'101', - '7/A25801C8'::pg_lsn, - '(3,7)', - '((3,7),(15,18))', - 3.4145, - 3.4145, - - 32768, - - 32768, - 1, - 1, - 1, - B'101', - '13:00:01', - '13:00:01', - '13:00:01', - TIMESTAMP '2004-10-19 10:23:00', - TIMESTAMP '2004-10-19 10:23:00', - TIMESTAMP '2004-10-19 10:23:00', - 'infinity', - 'infinity', - 'infinity', - B'101', - TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', - TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:00-08', - 'fat & (rat | cat)'::tsquery, - to_tsvector('The quick brown fox jumped over the lazy dog.'), - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - XMLPARSE( - DOCUMENT 'Manual...' - ), - 'happy', - '(2010-01-01 14:30, 2010-01-01 15:30)', - ROW( - 'fuzzy dice', - 42, - 1.99 - ), - '"paperback" => "243","publisher" => "postgresqltutorial.com", -"language" => "English","ISBN-13" => "978-1449370000", -"weight" => "11.2 ounces"', - TRUE, - '13:00:01', - '13:00:01', - '{1,2,3}', - '{-2147483648,2147483646}', - '{-9223372036854775808,9223372036854775801}', - '{564182,234181}', - '{lorem ipsum,dolor sit,amet}', - '{l,d,a}', - '{l,d,a}', - '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', - TRUE, - '{object,integer}', - '{131070.23,231072.476596593}', - '{131070.23,231072.476596593}', - '{131070.237689,231072.476596593}', - '{131070.237689,231072.476596593}', - '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', - '{true,yes,1,false,no,0,null}', - '{null,1,0}', - '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', - '{1999-01-08,1991-02-10 BC}', - '((3,7),(15,18))', - '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', - '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', - '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', - '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' - ); - -INSERT - INTO - POSTGRES_FULL_NN.TEST_DATASET - VALUES( - 2, - 9223372036854775807, - '1234', - '*', - '{asb12}', - '*', - '{asb12}', - 'abc', - 'abc', - '{asb12}', - '192.168/24', - '(0,0),0', - 9223372036854775807, - '1991-02-10 BC', - '1991-02-10 BC', - '1234567890.1234567', - '1234567890.1234567', - '1234567890.1234567', - '198.24.10.0', - - 2147483648, - - 2147483648, - - 2147483648, - '-178000000', - B'0', - '{"a": 10, "b": 15}', - '[1, 2, 3]'::jsonb, - '{0,1,0}', - '((0,0),(0,0))', - '08-00-2b-01-02-04', - '08-00-2b-01-02-03-04-06', - '1,001.01', - '1234567890.1234567', - '1234567890.1234567', - '((0,0),(0,0))', - B'101', - '0/0'::pg_lsn, - '(0,0)', - '((0,0),(0,0))', - 3.4145, - 3.4145, - 32767, - 32767, - 32767, - 32767, - 2147483647, - B'101', - '13:00:02+8', - '13:00:02+8', - '13:00:02+8', - TIMESTAMP '2004-10-19 10:23:54.123456', - TIMESTAMP '2004-10-19 10:23:54.123456', - TIMESTAMP '2004-10-19 10:23:54.123456', - '-infinity', - '-infinity', - '-infinity', - B'101', - TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', - TIMESTAMP WITH TIME ZONE '2004-10-19 10:23:54.123456-08', - 'fat:ab & cat'::tsquery, - to_tsvector('The quick brown fox jumped over the lazy dog.'), - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - '', - 'happy', - '(2010-01-01 14:30, 2010-01-01 15:30)', - ROW( - 'fuzzy dice', - 42, - 1.99 - ), - '"paperback" => "243","publisher" => "postgresqltutorial.com", -"language" => "English","ISBN-13" => "978-1449370000", -"weight" => "11.2 ounces"', - 'yes', - '13:00:00+8', - '13:00:00+8', - '{4,5,6}', - '{-2147483648,2147483646}', - '{-9223372036854775808,9223372036854775801}', - '{564182,234181}', - '{lorem ipsum,dolor sit,amet}', - '{l,d,a}', - '{l,d,a}', - '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', - 'yes', - '{object,integer}', - '{131070.23,231072.476596593}', - '{131070.23,231072.476596593}', - '{131070.237689,231072.476596593}', - '{131070.237689,231072.476596593}', - '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', - '{true,yes,1,false,no,0,null}', - '{null,1,0}', - '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', - '{1999-01-08,1991-02-10 BC}', - '((0,0),(0,0))', - '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', - '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', - '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', - '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' - ); - -INSERT - INTO - POSTGRES_FULL_NN.TEST_DATASET - VALUES( - 3, - 0, - 'abcd', - '*', - '{asb12}', - '*', - '{asb12}', - 'Миші йдуть на південь, не питай чому;', - 'Миші йдуть на південь, не питай чому;', - '{asb12}', - '192.168.1', - '(-10,-4),10', - 0, - '1991-02-10 BC', - '1991-02-10 BC', - '1234567890.1234567', - '1234567890.1234567', - '1234567890.1234567', - '198.10/8', - 2147483647, - 2147483647, - 2147483647, - '178000000', - B'0', - '{"a": 10, "b": 15}', - '[1, 2, 3]'::jsonb, - '{0,1,0}', - '((0,0),(0,0))', - '08002b:010205', - '08002b:0102030407', - '-1,000', - '1234567890.1234567', - '1234567890.1234567', - '((0,0),(0,0))', - B'101', - '0/0'::pg_lsn, - '(999999999999999999999999,0)', - '((0,0),(999999999999999999999999,0))', - 3.4145, - 3.4145, - 32767, - 32767, - 0, - 0, - 0, - B'101', - '13:00:03-8', - '13:00:03-8', - '13:00:03-8', - TIMESTAMP '3004-10-19 10:23:54.123456 BC', - TIMESTAMP '3004-10-19 10:23:54.123456 BC', - TIMESTAMP '3004-10-19 10:23:54.123456 BC', - '-infinity', - '-infinity', - '-infinity', - B'101', - TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', - TIMESTAMP WITH TIME ZONE '3004-10-19 10:23:54.123456-08 BC', - 'fat:ab & cat'::tsquery, - to_tsvector('The quick brown fox jumped over the lazy dog.'), - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - '', - 'happy', - '(2010-01-01 14:30, 2010-01-01 15:30)', - ROW( - 'fuzzy dice', - 42, - 1.99 - ), - '"paperback" => "243","publisher" => "postgresqltutorial.com", -"language" => "English","ISBN-13" => "978-1449370000", -"weight" => "11.2 ounces"', - '1', - '13:00:03-8', - '13:00:03-8', - '{4,5,6}', - '{-2147483648,2147483646}', - '{-9223372036854775808,9223372036854775801}', - '{564182,234181}', - '{lorem ipsum,dolor sit,amet}', - '{l,d,a}', - '{l,d,a}', - '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', - '1', - '{object,integer}', - '{131070.23,231072.476596593}', - '{131070.23,231072.476596593}', - '{131070.237689,231072.476596593}', - '{131070.237689,231072.476596593}', - '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', - '{true,yes,1,false,no,0,null}', - '{null,1,0}', - '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', - '{1999-01-08,1991-02-10 BC}', - '((0,0),(0,0))', - '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', - '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', - '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', - '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' - ); - -INSERT - INTO - POSTGRES_FULL_NN.TEST_DATASET - VALUES( - 4, - 0, - '\xabcd', - '*', - '{asb12}', - '*', - '{asb12}', - '櫻花分店', - '櫻花分店', - '{asb12}', - '128.1', - '(-10,-4),10', - - 9223372036854775808, - '1991-02-10 BC', - '1991-02-10 BC', - '1234567890.1234567', - '1234567890.1234567', - '1234567890.1234567', - '198.10/8', - 2147483647, - 2147483647, - 2147483647, - '178000000', - B'0', - '{"a": 10, "b": 15}', - '[1, 2, 3]'::jsonb, - '{0,1,0}', - '((0,0),(0,0))', - '08002b:010205', - '08002b:0102030407', - '$999.99', - '1234567890.1234567', - '1234567890.1234567', - '((0,0),(0,0))', - B'101', - '0/0'::pg_lsn, - '(999999999999999999999999,0)', - '((0,0),(999999999999999999999999,0))', - 3.4145, - 3.4145, - 32767, - 32767, - - 32767, - - 32767, - - 2147483647, - B'101', - '13:00:04Z', - '13:00:04Z', - '13:00:04Z', - TIMESTAMP '0001-01-01 00:00:00.000000', - TIMESTAMP '0001-01-01 00:00:00.000000', - TIMESTAMP '0001-01-01 00:00:00.000000', - '-infinity', - '-infinity', - '-infinity', - B'101', - TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', - TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC', - 'fat:ab & cat'::tsquery, - to_tsvector('The quick brown fox jumped over the lazy dog.'), - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - '', - 'happy', - '(2010-01-01 14:30, 2010-01-01 15:30)', - ROW( - 'fuzzy dice', - 42, - 1.99 - ), - '"paperback" => "243","publisher" => "postgresqltutorial.com", -"language" => "English","ISBN-13" => "978-1449370000", -"weight" => "11.2 ounces"', - FALSE, - '13:00:04Z', - '13:00:04Z', - '{4,5,6}', - '{-2147483648,2147483646}', - '{-9223372036854775808,9223372036854775801}', - '{564182,234181}', - '{lorem ipsum,dolor sit,amet}', - '{l,d,a}', - '{l,d,a}', - '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', - FALSE, - '{object,integer}', - '{131070.23,231072.476596593}', - '{131070.23,231072.476596593}', - '{131070.237689,231072.476596593}', - '{131070.237689,231072.476596593}', - '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', - '{true,yes,1,false,no,0,null}', - '{null,1,0}', - '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', - '{1999-01-08,1991-02-10 BC}', - '((0,0),(0,0))', - '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', - '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', - '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', - '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' - ); - -INSERT - INTO - POSTGRES_FULL_NN.TEST_DATASET - VALUES( - 5, - 0, - '\xabcd', - '*', - '{asb12}', - '*', - '{asb12}', - '', - '', - '{asb12}', - '2001:4f8:3:ba::/64', - '(-10,-4),10', - - 9223372036854775808, - '1991-02-10 BC', - '1991-02-10 BC', - '1234567890.1234567', - '1234567890.1234567', - '1234567890.1234567', - '198.10/8', - 2147483647, - 2147483647, - 2147483647, - '178000000', - B'0', - '{"a": 10, "b": 15}', - '[1, 2, 3]'::jsonb, - '{0,1,0}', - '((0,0),(0,0))', - '08002b:010205', - '08002b:0102030407', - '$1001.01', - '1234567890.1234567', - '1234567890.1234567', - '((0,0),(0,0))', - B'101', - '0/0'::pg_lsn, - '(999999999999999999999999,0)', - '((0,0),(999999999999999999999999,0))', - 3.4145, - 3.4145, - 32767, - 32767, - - 32767, - - 32767, - - 2147483647, - B'101', - '13:00:05.01234Z+8', - '13:00:05.01234Z+8', - '13:00:05.01234Z+8', - TIMESTAMP '0001-12-31 23:59:59.999999 BC', - TIMESTAMP '0001-12-31 23:59:59.999999 BC', - TIMESTAMP '0001-12-31 23:59:59.999999 BC', - '-infinity', - '-infinity', - '-infinity', - B'101', - TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', - TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', - 'fat:ab & cat'::tsquery, - to_tsvector('The quick brown fox jumped over the lazy dog.'), - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - '', - 'happy', - '(2010-01-01 14:30, 2010-01-01 15:30)', - ROW( - 'fuzzy dice', - 42, - 1.99 - ), - '"paperback" => "243","publisher" => "postgresqltutorial.com", -"language" => "English","ISBN-13" => "978-1449370000", -"weight" => "11.2 ounces"', - 'no', - '13:00:05.012345Z+8', - '13:00:05.012345Z+8', - '{4,5,6}', - '{-2147483648,2147483646}', - '{-9223372036854775808,9223372036854775801}', - '{564182,234181}', - '{lorem ipsum,dolor sit,amet}', - '{l,d,a}', - '{l,d,a}', - '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', - 'no', - '{object,integer}', - '{131070.23,231072.476596593}', - '{131070.23,231072.476596593}', - '{131070.237689,231072.476596593}', - '{131070.237689,231072.476596593}', - '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', - '{true,yes,1,false,no,0,null}', - '{null,1,0}', - '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', - '{1999-01-08,1991-02-10 BC}', - '((0,0),(0,0))', - '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', - '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', - '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', - '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' - ); - -INSERT - INTO - POSTGRES_FULL_NN.TEST_DATASET - VALUES( - 6, - 0, - '\xabcd', - '*', - '{asb12}', - '*', - '{asb12}', - '\xF0\x9F\x9A\x80', - '\xF0\x9F\x9A\x80', - '{asb12}', - '2001:4f8:3:ba::/64', - '(-10,-4),10', - - 9223372036854775808, - '1991-02-10 BC', - '1991-02-10 BC', - '1234567890.1234567', - '1234567890.1234567', - '1234567890.1234567', - '198.10/8', - 2147483647, - 2147483647, - 2147483647, - '178000000', - B'0', - '{"a": 10, "b": 15}', - '[1, 2, 3]'::jsonb, - '{0,1,0}', - '((0,0),(0,0))', - '08002b:010205', - '08002b:0102030407', - '-$1,000', - '1234567890.1234567', - '1234567890.1234567', - '((0,0),(0,0))', - B'101', - '0/0'::pg_lsn, - '(999999999999999999999999,0)', - '((0,0),(999999999999999999999999,0))', - 3.4145, - 3.4145, - 32767, - 32767, - - 32767, - - 32767, - - 2147483647, - B'101', - '13:00:00Z-8', - '13:00:00Z-8', - '13:00:00Z-8', - 'epoch', - 'epoch', - 'epoch', - '-infinity', - '-infinity', - '-infinity', - B'101', - TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', - TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', - 'fat:ab & cat'::tsquery, - to_tsvector('The quick brown fox jumped over the lazy dog.'), - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - '', - 'happy', - '(2010-01-01 14:30, 2010-01-01 15:30)', - ROW( - 'fuzzy dice', - 42, - 1.99 - ), - '"paperback" => "243","publisher" => "postgresqltutorial.com", -"language" => "English","ISBN-13" => "978-1449370000", -"weight" => "11.2 ounces"', - '0', - '13:00:06.00000Z-8', - '13:00:06.00000Z-8', - '{4,5,6}', - '{-2147483648,2147483646}', - '{-9223372036854775808,9223372036854775801}', - '{564182,234181}', - '{lorem ipsum,dolor sit,amet}', - '{l,d,a}', - '{l,d,a}', - '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', - '0', - '{object,integer}', - '{131070.23,231072.476596593}', - '{131070.23,231072.476596593}', - '{131070.237689,231072.476596593}', - '{131070.237689,231072.476596593}', - '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', - '{true,yes,1,false,no,0,null}', - '{null,1,0}', - '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', - '{1999-01-08,1991-02-10 BC}', - '((0,0),(0,0))', - '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', - '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', - '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', - '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' - ); - -INSERT - INTO - POSTGRES_FULL_NN.TEST_DATASET - VALUES( - 7, - 0, - '\xabcd', - '*', - '{asb12}', - '*', - '{asb12}', - '\xF0\x9F\x9A\x80', - '\xF0\x9F\x9A\x80', - '{asb12}', - '2001:4f8:3:ba::/64', - '(-10,-4),10', - - 9223372036854775808, - '1991-02-10 BC', - '1991-02-10 BC', - '1234567890.1234567', - '1234567890.1234567', - '1234567890.1234567', - '198.10/8', - 2147483647, - 2147483647, - 2147483647, - '178000000', - B'0', - '{"a": 10, "b": 15}', - '[1, 2, 3]'::jsonb, - '{0,1,0}', - '((0,0),(0,0))', - '08002b:010205', - '08002b:0102030407', - '-$1,000', - '1234567890.1234567', - '1234567890.1234567', - '((0,0),(0,0))', - B'101', - '0/0'::pg_lsn, - '(999999999999999999999999,0)', - '((0,0),(999999999999999999999999,0))', - 3.4145, - 3.4145, - 32767, - 32767, - - 32767, - - 32767, - - 2147483647, - B'101', - '24:00:00', - '24:00:00', - '24:00:00', - 'epoch', - 'epoch', - 'epoch', - '-infinity', - '-infinity', - '-infinity', - B'101', - TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', - TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC', - 'fat:ab & cat'::tsquery, - to_tsvector('The quick brown fox jumped over the lazy dog.'), - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - '', - 'happy', - '(2010-01-01 14:30, 2010-01-01 15:30)', - ROW( - 'fuzzy dice', - 42, - 1.99 - ), - '"paperback" => "243","publisher" => "postgresqltutorial.com", -"language" => "English","ISBN-13" => "978-1449370000", -"weight" => "11.2 ounces"', - '0', - '13:00:06.00000Z-8', - '13:00:06.00000Z-8', - '{4,5,6}', - '{-2147483648,2147483646}', - '{-9223372036854775808,9223372036854775801}', - '{564182,234181}', - '{lorem ipsum,dolor sit,amet}', - '{l,d,a}', - '{l,d,a}', - '{someeeeee loooooooooong teeeeext,vvvvvvveeeeeeeeeeeruyyyyyyyyy looooooooooooooooong teeeeeeeeeeeeeeext}', - '0', - '{object,integer}', - '{131070.23,231072.476596593}', - '{131070.23,231072.476596593}', - '{131070.237689,231072.476596593}', - '{131070.237689,231072.476596593}', - '{$999.99,$1001.01,45000, $1.001,$800,22222.006, 1001.01}', - '{true,yes,1,false,no,0,null}', - '{null,1,0}', - '{\xA6697E974E6A320F454390BE03F74955E8978F1A6971EA6730542E37B66179BC,\x4B52414B00000000000000000000000000000000000000000000000000000000}', - '{1999-01-08,1991-02-10 BC}', - '((0,0),(0,0))', - '{13:00:01,13:00:02+8,13:00:03-8,13:00:04Z,13:00:05.000000+8,13:00:00Z-8}', - '{null,13:00:01,13:00:00+8,13:00:03-8,13:00:04Z,13:00:05.012345Z+8,13:00:06.00000Z-8,13:00}', - '{null,2004-10-19 10:23:00-08,2004-10-19 10:23:54.123456-08}', - '{null,2004-10-19 10:23:00,2004-10-19 10:23:54.123456,3004-10-19 10:23:54.123456 BC}' - ); diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/metadata.yaml b/airbyte-integrations/connectors/source-postgres-strict-encrypt/metadata.yaml deleted file mode 100644 index a16927b98478..000000000000 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/metadata.yaml +++ /dev/null @@ -1,25 +0,0 @@ -data: - allowedHosts: - hosts: - - ${host} - - ${tunnel_method.tunnel_host} - registries: - cloud: - enabled: false # strict encrypt connectors are deployed to Cloud by their non strict encrypt sibling. - oss: - enabled: false # strict encrypt connectors are not used on OSS. - connectorSubtype: database - connectorType: source - definitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 - maxSecondsBetweenMessages: 7200 - dockerImageTag: 3.1.5 - dockerRepository: airbyte/source-postgres-strict-encrypt - githubIssueLabel: source-postgres - icon: postgresql.svg - license: ELv2 - name: Postgres - releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/postgres - tags: - - language:java -metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/main/java/io/airbyte/integrations/source/postgres_strict_encrypt/PostgresSourceStrictEncrypt.java b/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/main/java/io/airbyte/integrations/source/postgres_strict_encrypt/PostgresSourceStrictEncrypt.java deleted file mode 100644 index d6ef03a25d9e..000000000000 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/main/java/io/airbyte/integrations/source/postgres_strict_encrypt/PostgresSourceStrictEncrypt.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.postgres_strict_encrypt; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.base.spec_modification.SpecModifyingSource; -import io.airbyte.integrations.source.postgres.PostgresSource; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import java.util.Set; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class PostgresSourceStrictEncrypt extends SpecModifyingSource implements Source { - - private static final Logger LOGGER = LoggerFactory.getLogger(PostgresSourceStrictEncrypt.class); - public static final String TUNNEL_METHOD = "tunnel_method"; - public static final String NO_TUNNEL = "NO_TUNNEL"; - public static final String SSL_MODE = "ssl_mode"; - public static final String MODE = "mode"; - public static final String SSL_MODE_ALLOW = "allow"; - public static final String SSL_MODE_PREFER = "prefer"; - public static final String SSL_MODE_DISABLE = "disable"; - public static final String SSL_MODE_REQUIRE = "require"; - - public PostgresSourceStrictEncrypt() { - super(PostgresSource.sshWrappedSource()); - } - - @Override - public ConnectorSpecification modifySpec(final ConnectorSpecification originalSpec) { - final ConnectorSpecification spec = Jsons.clone(originalSpec); - final ObjectNode properties = (ObjectNode) spec.getConnectionSpecification().get("properties"); - ((ObjectNode) properties.get(SSL_MODE)).put("default", SSL_MODE_REQUIRE); - - return spec; - } - - @Override - public AirbyteConnectionStatus check(final JsonNode config) throws Exception { - // #15808 Disallow connecting to db with disable, prefer or allow SSL mode when connecting directly - // and not over SSH tunnel - if (config.has(TUNNEL_METHOD) - && config.get(TUNNEL_METHOD).has(TUNNEL_METHOD) - && config.get(TUNNEL_METHOD).get(TUNNEL_METHOD).asText().equals(NO_TUNNEL)) { - // If no SSH tunnel - if (config.has(SSL_MODE) && config.get(SSL_MODE).has(MODE)) { - if (Set.of(SSL_MODE_DISABLE, SSL_MODE_ALLOW, SSL_MODE_PREFER).contains(config.get(SSL_MODE).get(MODE).asText())) { - // Fail in case SSL mode is disable, allow or prefer - return new AirbyteConnectionStatus() - .withStatus(Status.FAILED) - .withMessage( - "Unsecured connection not allowed. If no SSH Tunnel set up, please use one of the following SSL modes: require, verify-ca, verify-full"); - } - } - } - return super.check(config); - } - - public static void main(final String[] args) throws Exception { - final Source source = new PostgresSourceStrictEncrypt(); - LOGGER.info("starting source: {}", PostgresSourceStrictEncrypt.class); - new IntegrationRunner(source).run(args); - LOGGER.info("completed source: {}", PostgresSourceStrictEncrypt.class); - } - -} diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/postgres_strict_encrypt/PostgresSourceStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/postgres_strict_encrypt/PostgresSourceStrictEncryptAcceptanceTest.java deleted file mode 100644 index d96f3fabe4f3..000000000000 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/postgres_strict_encrypt/PostgresSourceStrictEncryptAcceptanceTest.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.postgres_strict_encrypt; - -import static io.airbyte.db.PostgresUtils.getCertificate; - -import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.db.Database; -import io.airbyte.db.PostgresUtils; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.protocol.models.Field; -import io.airbyte.protocol.models.JsonSchemaType; -import io.airbyte.protocol.models.v0.CatalogHelpers; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.ConnectorSpecification; -import io.airbyte.protocol.models.v0.DestinationSyncMode; -import io.airbyte.protocol.models.v0.SyncMode; -import java.util.HashMap; -import java.util.List; -import java.util.Optional; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.utility.DockerImageName; -import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; -import uk.org.webcompere.systemstubs.jupiter.SystemStub; -import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; - -@ExtendWith(SystemStubsExtension.class) -public class PostgresSourceStrictEncryptAcceptanceTest extends SourceAcceptanceTest { - - private static final String STREAM_NAME = "id_and_name"; - private static final String STREAM_NAME2 = "starships"; - private static final String SCHEMA_NAME = "public"; - @SystemStub - private EnvironmentVariables environmentVariables; - private PostgreSQLContainer container; - private JsonNode config; - - protected static final String PASSWORD = "Passw0rd"; - protected static PostgresUtils.Certificate certs; - - @Override - protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - environmentVariables.set("DEPLOYMENT_MODE", "CLOUD"); - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - container = new PostgreSQLContainer<>(DockerImageName.parse("postgres:bullseye") - .asCompatibleSubstituteFor("postgres")); - container.start(); - certs = getCertificate(container); - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "Standard") - .build()); - final var containerOuterAddress = SshHelpers.getOuterContainerAddress(container); - final var containerInnerAddress = SshHelpers.getInnerContainerAddress(container); - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, containerInnerAddress.left) - .put(JdbcUtils.PORT_KEY, containerInnerAddress.right) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put("replication_method", replicationMethod) - .put("ssl_mode", ImmutableMap.builder() - .put("mode", "verify-ca") - .put("ca_certificate", certs.getCaCertificate()) - .put("client_certificate", certs.getClientCertificate()) - .put("client_key", certs.getClientKey()) - .put("client_key_password", PASSWORD) - .build()) - .build()); - - try (final DSLContext dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), - containerOuterAddress.left, - containerOuterAddress.right, - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.POSTGRES)) { - final Database database = new Database(dslContext); - - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - return null; - }); - } - } - - @Override - protected void tearDown(final TestDestinationEnv testEnv) { - container.close(); - } - - @Override - protected String getImageName() { - return "airbyte/source-postgres-strict-encrypt:dev"; - } - - @Override - protected ConnectorSpecification getSpec() throws Exception { - return SshHelpers.injectSshIntoSpec( - Jsons.deserialize(MoreResources.readResource("expected_strict_encrypt_spec.json"), ConnectorSpecification.class), - Optional.of("security")); - } - - @Override - protected JsonNode getConfig() { - return config; - } - - @Override - protected ConfiguredAirbyteCatalog getConfiguredCatalog() { - return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(Lists.newArrayList("id")) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withStream(CatalogHelpers.createAirbyteStream( - STREAM_NAME, SCHEMA_NAME, - Field.of("id", JsonSchemaType.NUMBER), - Field.of("name", JsonSchemaType.STRING)) - .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) - .withSourceDefinedPrimaryKey(List.of(List.of("id")))), - new ConfiguredAirbyteStream() - .withSyncMode(SyncMode.INCREMENTAL) - .withCursorField(Lists.newArrayList("id")) - .withDestinationSyncMode(DestinationSyncMode.APPEND) - .withStream(CatalogHelpers.createAirbyteStream( - STREAM_NAME2, SCHEMA_NAME, - Field.of("id", JsonSchemaType.NUMBER), - Field.of("name", JsonSchemaType.STRING)) - .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) - .withSourceDefinedPrimaryKey(List.of(List.of("id")))))); - } - - @Override - protected JsonNode getState() { - return Jsons.jsonNode(new HashMap<>()); - } - - @Override - protected boolean supportsPerStream() { - return true; - } - -} diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test-integration/resources/expected_strict_encrypt_spec.json b/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test-integration/resources/expected_strict_encrypt_spec.json deleted file mode 100644 index 44639f1f5636..000000000000 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test-integration/resources/expected_strict_encrypt_spec.json +++ /dev/null @@ -1,454 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/postgres", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Postgres Source Spec", - "type": "object", - "required": ["host", "port", "database", "username"], - "properties": { - "host": { - "title": "Host", - "description": "Hostname of the database.", - "type": "string", - "order": 0, - "group": "db" - }, - "port": { - "title": "Port", - "description": "Port of the database.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 5432, - "examples": ["5432"], - "order": 1, - "group": "db" - }, - "database": { - "title": "Database Name", - "description": "Name of the database.", - "type": "string", - "order": 2, - "group": "db" - }, - "schemas": { - "title": "Schemas", - "description": "The list of schemas (case sensitive) to sync from. Defaults to public.", - "type": "array", - "items": { - "type": "string" - }, - "minItems": 0, - "uniqueItems": true, - "default": ["public"], - "order": 3, - "group": "db" - }, - "username": { - "title": "Username", - "description": "Username to access the database.", - "type": "string", - "order": 4, - "group": "auth" - }, - "password": { - "title": "Password", - "description": "Password associated with the username.", - "type": "string", - "airbyte_secret": true, - "order": 5, - "group": "auth", - "always_show": true - }, - "jdbc_url_params": { - "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", - "title": "JDBC URL Parameters (Advanced)", - "type": "string", - "order": 6, - "group": "advanced", - "pattern_descriptor": "key1=value1&key2=value2" - }, - "ssl_mode": { - "title": "SSL Modes", - "description": "SSL connection modes. \n Read more in the docs.", - "type": "object", - "order": 8, - "group": "security", - "default": "require", - "oneOf": [ - { - "title": "disable", - "additionalProperties": true, - "description": "Disables encryption of communication between Airbyte and source database.", - "required": ["mode"], - "properties": { - "mode": { - "type": "string", - "const": "disable", - "order": 0 - } - } - }, - { - "title": "allow", - "additionalProperties": true, - "description": "Enables encryption only when required by the source database.", - "required": ["mode"], - "properties": { - "mode": { - "type": "string", - "const": "allow", - "order": 0 - } - } - }, - { - "title": "prefer", - "additionalProperties": true, - "description": "Allows unencrypted connection only if the source database does not support encryption.", - "required": ["mode"], - "properties": { - "mode": { - "type": "string", - "const": "prefer", - "order": 0 - } - } - }, - { - "title": "require", - "additionalProperties": true, - "description": "Always require encryption. If the source database server does not support encryption, connection will fail.", - "required": ["mode"], - "properties": { - "mode": { - "type": "string", - "const": "require", - "order": 0 - } - } - }, - { - "title": "verify-ca", - "additionalProperties": true, - "description": "Always require encryption and verifies that the source database server has a valid SSL certificate.", - "required": ["mode", "ca_certificate"], - "properties": { - "mode": { - "type": "string", - "const": "verify-ca", - "order": 0 - }, - "ca_certificate": { - "type": "string", - "title": "CA Certificate", - "description": "CA certificate", - "airbyte_secret": true, - "multiline": true, - "order": 1 - }, - "client_certificate": { - "type": "string", - "title": "Client Certificate", - "description": "Client certificate", - "airbyte_secret": true, - "multiline": true, - "order": 2, - "always_show": true - }, - "client_key": { - "type": "string", - "title": "Client Key", - "description": "Client key", - "airbyte_secret": true, - "multiline": true, - "order": 3, - "always_show": true - }, - "client_key_password": { - "type": "string", - "title": "Client key password", - "description": "Password for keystorage. If you do not add it - the password will be generated automatically.", - "airbyte_secret": true, - "order": 4 - } - } - }, - { - "title": "verify-full", - "additionalProperties": true, - "description": "This is the most secure mode. Always require encryption and verifies the identity of the source database server.", - "required": ["mode", "ca_certificate"], - "properties": { - "mode": { - "type": "string", - "const": "verify-full", - "order": 0 - }, - "ca_certificate": { - "type": "string", - "title": "CA Certificate", - "description": "CA certificate", - "airbyte_secret": true, - "multiline": true, - "order": 1 - }, - "client_certificate": { - "type": "string", - "title": "Client Certificate", - "description": "Client certificate", - "airbyte_secret": true, - "multiline": true, - "order": 2, - "always_show": true - }, - "client_key": { - "type": "string", - "title": "Client Key", - "description": "Client key", - "airbyte_secret": true, - "multiline": true, - "order": 3, - "always_show": true - }, - "client_key_password": { - "type": "string", - "title": "Client key password", - "description": "Password for keystorage. If you do not add it - the password will be generated automatically.", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - }, - "replication_method": { - "type": "object", - "title": "Replication Method", - "description": "Replication method for extracting data from the database.", - "order": 9, - "group": "advanced", - "oneOf": [ - { - "title": "Standard (Xmin)", - "description": "Xmin replication requires no setup on the DB side but will not be able to represent deletions incrementally.", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "Xmin", - "order": 0 - } - } - }, - { - "title": "Logical Replication (CDC)", - "description": "Logical replication uses the Postgres write-ahead log (WAL) to detect inserts, updates, and deletes. This needs to be configured on the source database itself. Only available on Postgres 10 and above. Read the docs.", - "required": ["method", "replication_slot", "publication"], - "additionalProperties": true, - "properties": { - "method": { - "type": "string", - "const": "CDC", - "order": 1 - }, - "plugin": { - "type": "string", - "title": "Plugin", - "description": "A logical decoding plugin installed on the PostgreSQL server.", - "enum": ["pgoutput"], - "default": "pgoutput", - "order": 2 - }, - "replication_slot": { - "type": "string", - "title": "Replication Slot", - "description": "A plugin logical replication slot. Read about replication slots.", - "order": 3 - }, - "publication": { - "type": "string", - "title": "Publication", - "description": "A Postgres publication used for consuming changes. Read about publications and replication identities.", - "order": 4 - }, - "initial_waiting_seconds": { - "type": "integer", - "title": "Initial Waiting Time in Seconds (Advanced)", - "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", - "default": 300, - "order": 5, - "min": 120, - "max": 1200 - }, - "queue_size": { - "type": "integer", - "title": "Size of the queue (Advanced)", - "description": "The size of the internal queue. This may interfere with memory consumption and efficiency of the connector, please be careful.", - "default": 10000, - "order": 6, - "min": 1000, - "max": 10000 - }, - "lsn_commit_behaviour": { - "type": "string", - "title": "LSN commit behaviour", - "description": "Determines when Airbtye should flush the LSN of processed WAL logs in the source database. `After loading Data in the destination` is default. If `While reading Data` is selected, in case of a downstream failure (while loading data into the destination), next sync would result in a full sync.", - "enum": [ - "While reading Data", - "After loading Data in the destination" - ], - "default": "After loading Data in the destination", - "order": 7 - } - } - }, - { - "title": "Standard", - "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "Standard", - "order": 8 - } - } - } - ] - }, - "tunnel_method": { - "type": "object", - "title": "SSH Tunnel Method", - "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", - "group": "security", - "oneOf": [ - { - "title": "No Tunnel", - "required": ["tunnel_method"], - "properties": { - "tunnel_method": { - "description": "No ssh tunnel needed to connect to database", - "type": "string", - "const": "NO_TUNNEL", - "order": 0 - } - } - }, - { - "title": "SSH Key Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "ssh_key" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and ssh key", - "type": "string", - "const": "SSH_KEY_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host.", - "type": "string", - "order": 3 - }, - "ssh_key": { - "title": "SSH Private Key", - "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", - "type": "string", - "airbyte_secret": true, - "multiline": true, - "order": 4 - } - } - }, - { - "title": "Password Authentication", - "required": [ - "tunnel_method", - "tunnel_host", - "tunnel_port", - "tunnel_user", - "tunnel_user_password" - ], - "properties": { - "tunnel_method": { - "description": "Connect through a jump server tunnel host using username and password authentication", - "type": "string", - "const": "SSH_PASSWORD_AUTH", - "order": 0 - }, - "tunnel_host": { - "title": "SSH Tunnel Jump Server Host", - "description": "Hostname of the jump server host that allows inbound ssh tunnel.", - "type": "string", - "order": 1 - }, - "tunnel_port": { - "title": "SSH Connection Port", - "description": "Port on the proxy/jump server that accepts inbound ssh connections.", - "type": "integer", - "minimum": 0, - "maximum": 65536, - "default": 22, - "examples": ["22"], - "order": 2 - }, - "tunnel_user": { - "title": "SSH Login Username", - "description": "OS-level username for logging into the jump server host", - "type": "string", - "order": 3 - }, - "tunnel_user_password": { - "title": "Password", - "description": "OS-level password for logging into the jump server host", - "type": "string", - "airbyte_secret": true, - "order": 4 - } - } - } - ] - } - }, - "groups": [ - { - "id": "db" - }, - { - "id": "auth" - }, - { - "id": "security", - "title": "Security" - }, - { - "id": "advanced", - "title": "Advanced" - } - ] - }, - "supported_destination_sync_modes": [] -} diff --git a/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test/java/io/airbyte/integrations/source/postgres_strict_encrypt/PostgresSourceStrictEncryptTest.java b/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test/java/io/airbyte/integrations/source/postgres_strict_encrypt/PostgresSourceStrictEncryptTest.java deleted file mode 100644 index f7701385fd46..000000000000 --- a/airbyte-integrations/connectors/source-postgres-strict-encrypt/src/test/java/io/airbyte/integrations/source/postgres_strict_encrypt/PostgresSourceStrictEncryptTest.java +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.postgres_strict_encrypt; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.ssh.SshBastionContainer; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.base.ssh.SshTunnel; -import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.Network; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.utility.DockerImageName; - -public class PostgresSourceStrictEncryptTest { - - private final PostgresSourceStrictEncrypt source = new PostgresSourceStrictEncrypt(); - private final PostgreSQLContainer postgreSQLContainerNoSSL = new PostgreSQLContainer<>("postgres:13-alpine"); - private final PostgreSQLContainer postgreSQLContainerWithSSL = - new PostgreSQLContainer<>(DockerImageName.parse("marcosmarxm/postgres-ssl:dev").asCompatibleSubstituteFor("postgres")) - .withCommand("postgres -c ssl=on -c ssl_cert_file=/var/lib/postgresql/server.crt -c ssl_key_file=/var/lib/postgresql/server.key"); - private static final List NON_STRICT_SSL_MODES = List.of("disable", "allow", "prefer"); - private static final String SSL_MODE_REQUIRE = "require"; - - private static final SshBastionContainer bastion = new SshBastionContainer(); - private static final Network network = Network.newNetwork(); - - @Test - void testSSlModesDisableAllowPreferWithTunnelIfServerDoesNotSupportSSL() throws Exception { - - try (final PostgreSQLContainer db = postgreSQLContainerNoSSL.withNetwork(network)) { - bastion.initAndStartBastion(network); - db.start(); - - for (final String sslmode : NON_STRICT_SSL_MODES) { - final AirbyteConnectionStatus connectionStatus = checkWithTunnel(db, sslmode, false); - assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatus.getStatus()); - } - - } finally { - bastion.stopAndClose(); - } - } - - @Test - void testSSlModesDisableAllowPreferWithTunnelIfServerSupportSSL() throws Exception { - try (final PostgreSQLContainer db = postgreSQLContainerWithSSL.withNetwork(network)) { - - bastion.initAndStartBastion(network); - db.start(); - for (final String sslmode : NON_STRICT_SSL_MODES) { - - final AirbyteConnectionStatus connectionStatus = checkWithTunnel(db, sslmode, false); - assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatus.getStatus()); - } - } finally { - bastion.stopAndClose(); - } - } - - @Test - void testSSlModesDisableAllowPreferWithFailedTunnelIfServerSupportSSL() throws Exception { - try (final PostgreSQLContainer db = postgreSQLContainerWithSSL) { - - bastion.initAndStartBastion(network); - db.start(); - for (final String sslmode : NON_STRICT_SSL_MODES) { - - final AirbyteConnectionStatus connectionStatus = checkWithTunnel(db, sslmode, false); - assertEquals(AirbyteConnectionStatus.Status.FAILED, connectionStatus.getStatus()); - assertTrue(connectionStatus.getMessage().contains("Connection is not available")); - - } - } finally { - bastion.stopAndClose(); - } - } - - @Test - void testSSlRequiredWithTunnelIfServerDoesNotSupportSSL() throws Exception { - - try (final PostgreSQLContainer db = postgreSQLContainerNoSSL.withNetwork(network)) { - bastion.initAndStartBastion(network); - db.start(); - final AirbyteConnectionStatus connectionStatus = checkWithTunnel(db, SSL_MODE_REQUIRE, false); - assertEquals(AirbyteConnectionStatus.Status.FAILED, connectionStatus.getStatus()); - assertEquals("State code: 08004; Message: The server does not support SSL.", connectionStatus.getMessage()); - - } finally { - bastion.stopAndClose(); - } - } - - @Test - void testSSlRequiredNoTunnelIfServerSupportSSL() throws Exception { - - try (final PostgreSQLContainer db = postgreSQLContainerWithSSL) { - db.start(); - - final ImmutableMap configBuilderWithSSLMode = getDatabaseConfigBuilderWithSSLMode(db, SSL_MODE_REQUIRE, false).build(); - final JsonNode config = Jsons.jsonNode(configBuilderWithSSLMode); - addNoTunnel((ObjectNode) config); - final AirbyteConnectionStatus connectionStatus = source.check(config); - assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatus.getStatus()); - } - } - - @Test - void testStrictSSLSecuredWithTunnel() throws Exception { - - try (final PostgreSQLContainer db = postgreSQLContainerWithSSL.withNetwork(network)) { - - bastion.initAndStartBastion(network); - db.start(); - - final AirbyteConnectionStatus connectionStatus = checkWithTunnel(db, SSL_MODE_REQUIRE, false); - assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatus.getStatus()); - } finally { - bastion.stopAndClose(); - } - } - - private ImmutableMap.Builder getDatabaseConfigBuilderWithSSLMode(final PostgreSQLContainer db, - final String sslMode, - final boolean innerAddress) { - final var containerAddress = innerAddress ? SshHelpers.getInnerContainerAddress(db) : SshHelpers.getOuterContainerAddress(db); - return ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, Objects.requireNonNull( - containerAddress.left)) - .put(JdbcUtils.PORT_KEY, containerAddress.right) - .put(JdbcUtils.DATABASE_KEY, db.getDatabaseName()) - .put(JdbcUtils.SCHEMAS_KEY, List.of("public")) - .put(JdbcUtils.USERNAME_KEY, db.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, db.getPassword()) - .put(JdbcUtils.SSL_MODE_KEY, Map.of(JdbcUtils.MODE_KEY, sslMode)); - } - - private JsonNode getMockedSSLConfig(final String sslMode) { - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, "test_host") - .put(JdbcUtils.PORT_KEY, 777) - .put(JdbcUtils.DATABASE_KEY, "test_db") - .put(JdbcUtils.USERNAME_KEY, "test_user") - .put(JdbcUtils.PASSWORD_KEY, "test_password") - .put(JdbcUtils.SSL_KEY, true) - .put(JdbcUtils.SSL_MODE_KEY, Map.of(JdbcUtils.MODE_KEY, sslMode)) - .build()); - } - - @Test - void testSslModesUnsecuredNoTunnel() throws Exception { - for (final String sslMode : NON_STRICT_SSL_MODES) { - final JsonNode config = getMockedSSLConfig(sslMode); - addNoTunnel((ObjectNode) config); - - final AirbyteConnectionStatus connectionStatus = source.check(config); - assertEquals(AirbyteConnectionStatus.Status.FAILED, connectionStatus.getStatus()); - assertTrue(connectionStatus.getMessage().contains("Unsecured connection not allowed")); - } - } - - private AirbyteConnectionStatus checkWithTunnel(final PostgreSQLContainer db, final String sslmode, final boolean innerAddress) - throws Exception { - final ImmutableMap.Builder configBuilderWithSSLMode = getDatabaseConfigBuilderWithSSLMode(db, sslmode, true); - final JsonNode configWithSSLModeDisable = - bastion.getTunnelConfig(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, configBuilderWithSSLMode, innerAddress); - return source.check(configWithSSLModeDisable); - } - - private static void addNoTunnel(final ObjectNode config) { - config.putIfAbsent("tunnel_method", Jsons.jsonNode(ImmutableMap.builder() - .put("tunnel_method", "NO_TUNNEL") - .build())); - } - -} diff --git a/airbyte-integrations/connectors/source-postgres/.dockerignore b/airbyte-integrations/connectors/source-postgres/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-postgres/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-postgres/Dockerfile b/airbyte-integrations/connectors/source-postgres/Dockerfile deleted file mode 100644 index 768f54c2bed4..000000000000 --- a/airbyte-integrations/connectors/source-postgres/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-postgres - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-postgres - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=3.1.5 -LABEL io.airbyte.name=airbyte/source-postgres diff --git a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml index 02b32443345c..0cb56b20af67 100644 --- a/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-postgres/acceptance-test-config.yml @@ -1,7 +1,7 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-postgres:dev -# test_strictness_level: high # Uncomment this line to enable high strictness level, requires: https://github.com/airbytehq/airbyte/pull/27872 +test_strictness_level: high custom_environment_variables: USE_STREAM_CAPABLE_STATE: true acceptance_tests: @@ -11,6 +11,11 @@ acceptance_tests: config_path: "secrets/config.json" backward_compatibility_tests_config: disable_for_version: "1.0.52" + - spec_path: "src/test-integration/resources/expected_strict_encrypt_spec.json" + config_path: "secrets/config.json" + deployment_mode: cloud + backward_compatibility_tests_config: + disable_for_version: "1.0.52" - spec_path: "src/test-integration/resources/expected_spec.json" config_path: "secrets/config_cdc.json" backward_compatibility_tests_config: @@ -39,13 +44,13 @@ acceptance_tests: tests: - config_path: "secrets/config.json" - config_path: "secrets/config_cdc.json" -# incremental: -# tests: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/incremental_configured_catalog.json" -# future_state: -# bypass_reason: "A java.lang.NullPointerException is thrown when a state with an invalid cursor value is passed" -# - config_path: "secrets/config_cdc.json" -# configured_catalog_path: "integration_tests/incremental_configured_catalog.json" -# future_state: -# bypass_reason: "A java.lang.NullPointerException is thrown when a state with an invalid cursor value is passed" + incremental: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/incremental_configured_catalog.json" + future_state: + bypass_reason: "A java.lang.NullPointerException is thrown when a state with an invalid cursor value is passed" + - config_path: "secrets/config_cdc.json" + configured_catalog_path: "integration_tests/incremental_configured_catalog.json" + future_state: + bypass_reason: "A java.lang.NullPointerException is thrown when a state with an invalid cursor value is passed" diff --git a/airbyte-integrations/connectors/source-postgres/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-postgres/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-postgres/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-postgres/build.gradle b/airbyte-integrations/connectors/source-postgres/build.gradle index d0997a9ff1be..6fadebd3ed37 100644 --- a/airbyte-integrations/connectors/source-postgres/build.gradle +++ b/airbyte-integrations/connectors/source-postgres/build.gradle @@ -2,47 +2,60 @@ import org.jsonschema2pojo.SourceType plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' - id 'airbyte-performance-test-java' - id 'airbyte-connector-acceptance-test' + id 'airbyte-java-connector' id "org.jsonschema2pojo" version "1.2.1" } +java { + compileJava { + options.compilerArgs += "-Xlint:-try,-rawtypes,-unchecked" + } +} + +airbyteJavaConnector { + cdkVersionRequired = '0.11.5' + features = ['db-sources'] + useLocalCdk = false +} + + application { mainClass = 'io.airbyte.integrations.source.postgres.PostgresSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } +// Add a configuration for our migrations tasks defined below to encapsulate their dependencies +configurations { + migrations.extendsFrom implementation +} + +configurations.all { + resolutionStrategy { + force 'org.jooq:jooq:3.13.4' + } +} + dependencies { - implementation 'io.airbyte:airbyte-cdk:0.0.2' + testImplementation libs.jooq + testImplementation libs.hikaricp - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:bases:debezium') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation project(':airbyte-integrations:connectors:source-relational-db') + migrations libs.testcontainers.postgresql + migrations sourceSets.main.output + + // Lombok + implementation libs.lombok + annotationProcessor libs.lombok implementation 'org.apache.commons:commons-lang3:3.11' implementation libs.postgresql implementation libs.bundles.datadog - testImplementation testFixtures(project(':airbyte-integrations:bases:debezium')) - testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) - testImplementation project(":airbyte-json-validation") - testImplementation project(':airbyte-test-utils') testImplementation 'org.hamcrest:hamcrest-all:1.3' - testImplementation libs.connectors.testcontainers.jdbc - testImplementation libs.connectors.testcontainers.postgresql + testFixturesImplementation libs.testcontainers.jdbc + testFixturesImplementation libs.testcontainers.postgresql + testImplementation libs.testcontainers.jdbc + testImplementation libs.testcontainers.postgresql testImplementation libs.junit.jupiter.system.stubs - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - performanceTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - performanceTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } jsonSchema2Pojo { diff --git a/airbyte-integrations/connectors/source-postgres/gradle.properties b/airbyte-integrations/connectors/source-postgres/gradle.properties new file mode 100644 index 000000000000..8ef098d20b92 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/gradle.properties @@ -0,0 +1 @@ +testExecutionConcurrency=-1 \ No newline at end of file diff --git a/tools/bin/make-big-schema.sh b/airbyte-integrations/connectors/source-postgres/make-big-postgres-schema.sh similarity index 100% rename from tools/bin/make-big-schema.sh rename to airbyte-integrations/connectors/source-postgres/make-big-postgres-schema.sh diff --git a/airbyte-integrations/connectors/source-postgres/metadata.yaml b/airbyte-integrations/connectors/source-postgres/metadata.yaml index dd20d7d2411d..23d5517fe512 100644 --- a/airbyte-integrations/connectors/source-postgres/metadata.yaml +++ b/airbyte-integrations/connectors/source-postgres/metadata.yaml @@ -1,4 +1,7 @@ data: + ab_internal: + ql: 400 + sl: 300 allowedHosts: hosts: - ${host} @@ -6,26 +9,22 @@ data: connectorSubtype: database connectorType: source definitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 - dockerImageTag: 3.1.5 - maxSecondsBetweenMessages: 7200 + dockerImageTag: 3.3.1 dockerRepository: airbyte/source-postgres + documentationUrl: https://docs.airbyte.com/integrations/sources/postgres githubIssueLabel: source-postgres icon: postgresql.svg license: ELv2 + maxSecondsBetweenMessages: 7200 name: Postgres registries: cloud: - dockerRepository: airbyte/source-postgres-strict-encrypt enabled: true oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/postgres + supportLevel: certified tags: - language:java - language:python - ab_internal: - sl: 300 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelper.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelper.java index a65ec1f6f081..898c250d5f65 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelper.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelper.java @@ -4,16 +4,16 @@ package io.airbyte.integrations.source.postgres; -import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_LSN; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_LSN; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.integrations.debezium.internals.DebeziumEventUtils; import io.airbyte.protocol.models.v0.AirbyteStream; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.SyncMode; diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java index 60c3f2615fc5..4cbac03d2115 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresQueryUtils.java @@ -4,23 +4,22 @@ package io.airbyte.integrations.source.postgres; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; import static io.airbyte.integrations.source.postgres.xmin.XminStateManager.XMIN_STATE_VERSION; -import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.source.relationaldb.CursorInfo; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; import io.airbyte.integrations.source.postgres.ctid.CtidUtils.CtidStreams; +import io.airbyte.integrations.source.postgres.ctid.FileNodeHandler; import io.airbyte.integrations.source.postgres.internal.models.CursorBasedStatus; import io.airbyte.integrations.source.postgres.internal.models.InternalModels.StateType; import io.airbyte.integrations.source.postgres.internal.models.XminStatus; -import io.airbyte.integrations.source.relationaldb.CursorInfo; -import io.airbyte.integrations.source.relationaldb.state.StateManager; import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; -import io.airbyte.protocol.models.v0.AirbyteStateMessage; -import io.airbyte.protocol.models.v0.AirbyteStreamState; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import java.sql.SQLException; import java.util.ArrayList; @@ -105,6 +104,15 @@ SELECT pg_relation_filenode('%s') WITH block_sz AS (SELECT current_setting('block_size')::int), rel_sz AS (select pg_relation_size('%s')) SELECT * from block_sz, rel_sz """; + /** + * Query estimates the max tuple in a page. We are estimating in two ways and selecting the greatest + * value. + */ + public static final String CTID_ESTIMATE_MAX_TUPLE = + """ + SELECT COALESCE(MAX((ctid::text::point)[1]::int), 0) AS max_tuple FROM "%s"."%s" + """; + /** * Logs the current xmin status : 1. The number of wraparounds the source DB has undergone. (These * are the epoch bits in the xmin snapshot). 2. The 32-bit xmin value associated with the xmin @@ -138,7 +146,7 @@ public static XminStatus getXminStatus(final JdbcDatabase database) throws SQLEx */ public static Map getCursorBasedSyncStatusForStreams(final JdbcDatabase database, final List streams, - final StateManager stateManager, + final StateManager stateManager, final String quoteString) { final Map cursorBasedStatusMap = new HashMap<>(); @@ -188,28 +196,30 @@ public static Map getCursorBa return cursorBasedStatusMap; } - public static ResultWithFailed> fileNodeForStreams(final JdbcDatabase database, - final List streams, - final String quoteString) { - final Map fileNodes = new HashMap<>(); - final List failedToQuery = new ArrayList<>(); + public static FileNodeHandler fileNodeForStreams(final JdbcDatabase database, + final List streams, + final String quoteString) { + final FileNodeHandler fileNodeHandler = new FileNodeHandler(); streams.forEach(stream -> { try { final AirbyteStreamNameNamespacePair namespacePair = new AirbyteStreamNameNamespacePair(stream.getStream().getName(), stream.getStream().getNamespace()); - final Optional fileNode = fileNodeForStreams(database, namespacePair, quoteString); + final Optional fileNode = fileNodeForIndividualStream(database, namespacePair, quoteString); fileNode.ifPresentOrElse( - l -> fileNodes.put(namespacePair, l), - () -> failedToQuery.add(io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair.fromConfiguredAirbyteSteam(stream))); + l -> fileNodeHandler.updateFileNode(namespacePair, l), + () -> fileNodeHandler + .updateFailedToQuery(io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair.fromConfiguredAirbyteSteam(stream))); } catch (final Exception e) { LOGGER.warn("Failed to fetch relation node for {}.{} .", stream.getStream().getNamespace(), stream.getStream().getName(), e); - failedToQuery.add(io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair.fromConfiguredAirbyteSteam(stream)); + fileNodeHandler.updateFailedToQuery(io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair.fromConfiguredAirbyteSteam(stream)); } }); - return new ResultWithFailed<>(fileNodes, failedToQuery); + return fileNodeHandler; } - public static Optional fileNodeForStreams(final JdbcDatabase database, final AirbyteStreamNameNamespacePair stream, final String quoteString) + public static Optional fileNodeForIndividualStream(final JdbcDatabase database, + final AirbyteStreamNameNamespacePair stream, + final String quoteString) throws SQLException { final String streamName = stream.getName(); final String schemaName = stream.getNamespace(); @@ -243,7 +253,7 @@ public static ResultWithFailed jsonNodes = database.bufferedResultSetQuery( conn -> conn.prepareStatement(CTID_FULL_VACUUM_IN_PROGRESS_QUERY.formatted(fullTableName)).executeQuery(), resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet)); - if (jsonNodes.size() != 0) { + if (!jsonNodes.isEmpty()) { Preconditions.checkState(jsonNodes.size() == 1); LOGGER.warn("Full Vacuum currently in progress for table {} in {} phase, the table will be skipped from syncing data", fullTableName, jsonNodes.get(0).get("phase")); @@ -308,4 +318,38 @@ public static List filterStreamsUnderVacuumForCtidSync( .toList(); } + public static Map getTableMaxTupleForStreams(final JdbcDatabase database, + final List streams, + final String quoteString) { + final Map tableMaxTupleEstimates = new HashMap<>(); + streams.forEach(stream -> { + final AirbyteStreamNameNamespacePair namespacePair = + new AirbyteStreamNameNamespacePair(stream.getStream().getName(), stream.getStream().getNamespace()); + final int maxTuple = getTableMaxTupleForStream(database, namespacePair, quoteString); + tableMaxTupleEstimates.put(namespacePair, maxTuple); + }); + return tableMaxTupleEstimates; + } + + public static int getTableMaxTupleForStream(final JdbcDatabase database, + final AirbyteStreamNameNamespacePair stream, + final String quoteString) { + try { + final String streamName = stream.getName(); + final String schemaName = stream.getNamespace(); + final String fullTableName = + getFullyQualifiedTableNameWithQuoting(schemaName, streamName, quoteString); + LOGGER.debug("running {}", CTID_ESTIMATE_MAX_TUPLE.formatted(schemaName, streamName)); + final List jsonNodes = database.bufferedResultSetQuery( + conn -> conn.prepareStatement(CTID_ESTIMATE_MAX_TUPLE.formatted(schemaName, streamName)).executeQuery(), + resultSet -> JdbcUtils.getDefaultSourceOperations().rowToJson(resultSet)); + Preconditions.checkState(jsonNodes.size() == 1); + final int maxTuple = jsonNodes.get(0).get("max_tuple").asInt(); + LOGGER.info("Stream {} max tuple is {}", fullTableName, maxTuple); + return maxTuple; + } catch (final SQLException e) { + throw new RuntimeException(e); + } + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java index 041a2ff5780c..fd4573d7424b 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java @@ -4,18 +4,20 @@ package io.airbyte.integrations.source.postgres; -import static io.airbyte.db.jdbc.JdbcConstants.JDBC_COLUMN_COLUMN_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.JDBC_INDEX_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.JDBC_INDEX_NON_UNIQUE; -import static io.airbyte.db.jdbc.JdbcUtils.AMPERSAND; -import static io.airbyte.db.jdbc.JdbcUtils.EQUALS; -import static io.airbyte.db.jdbc.JdbcUtils.PLATFORM_DATA_INCREASE_FACTOR; -import static io.airbyte.integrations.debezium.AirbyteDebeziumHandler.shouldUseCDC; -import static io.airbyte.integrations.source.jdbc.JdbcDataSourceUtils.DEFAULT_JDBC_PARAMETERS_DELIMITER; -import static io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.CLIENT_KEY_STORE_PASS; -import static io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.CLIENT_KEY_STORE_URL; -import static io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.PARAM_CA_CERTIFICATE; -import static io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.parseSSLConfig; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.JDBC_COLUMN_COLUMN_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.JDBC_INDEX_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.JDBC_INDEX_NON_UNIQUE; +import static io.airbyte.cdk.db.jdbc.JdbcUtils.AMPERSAND; +import static io.airbyte.cdk.db.jdbc.JdbcUtils.EQUALS; +import static io.airbyte.cdk.db.jdbc.JdbcUtils.PLATFORM_DATA_INCREASE_FACTOR; +import static io.airbyte.cdk.integrations.debezium.AirbyteDebeziumHandler.isAnyStreamIncrementalSyncMode; +import static io.airbyte.cdk.integrations.source.jdbc.JdbcDataSourceUtils.DEFAULT_JDBC_PARAMETERS_DELIMITER; +import static io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils.CLIENT_KEY_STORE_PASS; +import static io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils.CLIENT_KEY_STORE_URL; +import static io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils.PARAM_CA_CERTIFICATE; +import static io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils.parseSSLConfig; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; +import static io.airbyte.cdk.integrations.util.PostgresSslConnectionUtils.PARAM_SSL_MODE; import static io.airbyte.integrations.source.postgres.PostgresQueryUtils.NULL_CURSOR_VALUE_NO_SCHEMA_QUERY; import static io.airbyte.integrations.source.postgres.PostgresQueryUtils.NULL_CURSOR_VALUE_WITH_SCHEMA_QUERY; import static io.airbyte.integrations.source.postgres.PostgresQueryUtils.ROW_COUNT_RESULT_COL; @@ -24,54 +26,56 @@ import static io.airbyte.integrations.source.postgres.PostgresQueryUtils.filterStreamsUnderVacuumForCtidSync; import static io.airbyte.integrations.source.postgres.PostgresQueryUtils.getCursorBasedSyncStatusForStreams; import static io.airbyte.integrations.source.postgres.PostgresQueryUtils.streamsUnderVacuum; -import static io.airbyte.integrations.source.postgres.PostgresUtils.isAnyStreamIncrementalSyncMode; import static io.airbyte.integrations.source.postgres.PostgresUtils.prettyPrintConfiguredAirbyteStreamList; import static io.airbyte.integrations.source.postgres.cdc.PostgresCdcCtidInitializer.cdcCtidIteratorsCombined; import static io.airbyte.integrations.source.postgres.cursor_based.CursorBasedCtidUtils.categoriseStreams; import static io.airbyte.integrations.source.postgres.cursor_based.CursorBasedCtidUtils.reclassifyCategorisedCtidStreams; import static io.airbyte.integrations.source.postgres.xmin.XminCtidUtils.categoriseStreams; import static io.airbyte.integrations.source.postgres.xmin.XminCtidUtils.reclassifyCategorisedCtidStreams; -import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; -import static io.airbyte.integrations.util.PostgresSslConnectionUtils.PARAM_SSL_MODE; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toSet; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import datadog.trace.api.Trace; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.jdbc.StreamingJdbcDatabase; +import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.base.adaptive.AdaptiveSourceRunner; +import io.airbyte.cdk.integrations.base.ssh.SshWrappedSource; +import io.airbyte.cdk.integrations.debezium.internals.postgres.PostgresReplicationConnection; +import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.cdk.integrations.source.jdbc.JdbcDataSourceUtils; +import io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils; +import io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils.SslMode; +import io.airbyte.cdk.integrations.source.jdbc.dto.JdbcPrivilegeDto; +import io.airbyte.cdk.integrations.source.relationaldb.TableInfo; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.exceptions.ConfigErrorException; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.functional.CheckedConsumer; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; import io.airbyte.commons.util.AutoCloseableIterator; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.StreamingJdbcDatabase; -import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; -import io.airbyte.integrations.base.AirbyteTraceMessageUtility; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.base.ssh.SshWrappedSource; -import io.airbyte.integrations.debezium.internals.postgres.PostgresReplicationConnection; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.JdbcDataSourceUtils; -import io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils; -import io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.SslMode; -import io.airbyte.integrations.source.jdbc.dto.JdbcPrivilegeDto; import io.airbyte.integrations.source.postgres.PostgresQueryUtils.ResultWithFailed; import io.airbyte.integrations.source.postgres.PostgresQueryUtils.TableBlockSize; import io.airbyte.integrations.source.postgres.ctid.CtidPerStreamStateManager; import io.airbyte.integrations.source.postgres.ctid.CtidPostgresSourceOperations; import io.airbyte.integrations.source.postgres.ctid.CtidStateManager; +import io.airbyte.integrations.source.postgres.ctid.CtidUtils; import io.airbyte.integrations.source.postgres.ctid.CtidUtils.StreamsCategorised; +import io.airbyte.integrations.source.postgres.ctid.FileNodeHandler; import io.airbyte.integrations.source.postgres.ctid.PostgresCtidHandler; import io.airbyte.integrations.source.postgres.cursor_based.CursorBasedCtidUtils.CursorBasedStreams; import io.airbyte.integrations.source.postgres.cursor_based.PostgresCursorBasedStateManager; @@ -80,9 +84,6 @@ import io.airbyte.integrations.source.postgres.xmin.PostgresXminHandler; import io.airbyte.integrations.source.postgres.xmin.XminCtidUtils.XminStreams; import io.airbyte.integrations.source.postgres.xmin.XminStateManager; -import io.airbyte.integrations.source.relationaldb.TableInfo; -import io.airbyte.integrations.source.relationaldb.state.StateManager; -import io.airbyte.integrations.util.HostPortResolver; import io.airbyte.protocol.models.CommonField; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; @@ -94,6 +95,7 @@ import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.ConnectorSpecification; import java.net.URI; import java.net.URISyntaxException; import java.nio.file.Path; @@ -131,24 +133,41 @@ public class PostgresSource extends AbstractJdbcSource implements public static final String SSL_PASSWORD = "sslpassword"; public static final String MODE = "mode"; + public static final String TUNNEL_METHOD = "tunnel_method"; + public static final String NO_TUNNEL = "NO_TUNNEL"; + public static final String SSL_MODE_ALLOW = "allow"; + public static final String SSL_MODE_PREFER = "prefer"; + public static final String SSL_MODE_DISABLE = "disable"; + public static final String SSL_MODE_REQUIRE = "require"; + private List schemas; private Set publicizedTablesInCdc; - private final FeatureFlags featureFlags; private static final Set INVALID_CDC_SSL_MODES = ImmutableSet.of("allow", "prefer"); private int stateEmissionFrequency; private XminStatus xminStatus; - public static Source sshWrappedSource() { - return new SshWrappedSource(new PostgresSource(), JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY, "security"); + public static Source sshWrappedSource(PostgresSource source) { + return new SshWrappedSource(source, JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY, "security"); } PostgresSource() { super(DRIVER_CLASS, AdaptiveStreamingQueryConfig::new, new PostgresSourceOperations()); - this.featureFlags = new EnvVariableFeatureFlags(); this.stateEmissionFrequency = INTERMEDIATE_STATE_EMISSION_FREQUENCY; } + @Override + public ConnectorSpecification spec() throws Exception { + if (cloudDeploymentMode()) { + final ConnectorSpecification spec = Jsons.clone(super.spec()); + final ObjectNode properties = (ObjectNode) spec.getConnectionSpecification().get("properties"); + ((ObjectNode) properties.get(SSL_MODE)).put("default", SSL_MODE_REQUIRE); + + return spec; + } + return super.spec(); + } + @Override public JsonNode toDatabaseConfig(final JsonNode config) { final List additionalParameters = new ArrayList<>(); @@ -229,7 +248,7 @@ public Set getExcludedInternalNameSpaces() { @Override protected Set getExcludedViews() { - return Set.of("pg_stat_statements", "pg_stat_statements_info"); + return Set.of("pg_stat_statements", "pg_stat_statements_info", "pg_buffercache"); } @Override @@ -296,13 +315,15 @@ public AirbyteCatalog discover(final JsonNode config) throws Exception { @Override public JdbcDatabase createDatabase(final JsonNode sourceConfig) throws SQLException { final JsonNode jdbcConfig = toDatabaseConfig(sourceConfig); + final Map connectionProperties = getConnectionProperties(sourceConfig); // Create the data source final DataSource dataSource = DataSourceFactory.create( jdbcConfig.has(JdbcUtils.USERNAME_KEY) ? jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText() : null, jdbcConfig.has(JdbcUtils.PASSWORD_KEY) ? jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, - driverClass, + driverClassName, jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), - getConnectionProperties(sourceConfig)); + connectionProperties, + getConnectionTimeout(connectionProperties, driverClassName)); // Record the data source so that it can be closed. dataSources.add(dataSource); @@ -455,7 +476,7 @@ public List> getIncrementalIterators(final final StateManager stateManager, final Instant emittedAt) { final JsonNode sourceConfig = database.getSourceConfig(); - if (PostgresUtils.isCdc(sourceConfig) && shouldUseCDC(catalog)) { + if (PostgresUtils.isCdc(sourceConfig) && isAnyStreamIncrementalSyncMode(catalog)) { LOGGER.info("Using ctid + CDC"); return cdcCtidIteratorsCombined(database, catalog, tableNameToTable, stateManager, emittedAt, getQuoteString(), getReplicationSlot(database, sourceConfig).get(0)); @@ -473,26 +494,28 @@ public List> getIncrementalIterators(final List finalListOfStreamsToBeSyncedViaCtid = filterStreamsUnderVacuumForCtidSync(streamsUnderVacuum.result(), streamsCategorised.ctidStreams()); - final ResultWithFailed> fileNodes = + final FileNodeHandler fileNodeHandler = PostgresQueryUtils.fileNodeForStreams(database, finalListOfStreamsToBeSyncedViaCtid, getQuoteString()); // In case we failed to query for fileNode, streams will get reclassified as xmin - if (!fileNodes.failed().isEmpty()) { - reclassifyCategorisedCtidStreams(streamsCategorised, fileNodes.failed()); + if (!fileNodeHandler.getFailedToQuery().isEmpty()) { + reclassifyCategorisedCtidStreams(streamsCategorised, fileNodeHandler.getFailedToQuery()); finalListOfStreamsToBeSyncedViaCtid = filterStreamsUnderVacuumForCtidSync(streamsUnderVacuum.result(), streamsCategorised.ctidStreams()); } - final CtidStateManager ctidStateManager = - new CtidPerStreamStateManager(streamsCategorised.ctidStreams().statesFromCtidSync(), fileNodes.result()); + final CtidStateManager ctidStateManager = new CtidPerStreamStateManager(streamsCategorised.ctidStreams().statesFromCtidSync(), fileNodeHandler); final Map tableBlockSizes = PostgresQueryUtils.getTableBlockSizeForStreams( database, finalListOfStreamsToBeSyncedViaCtid, getQuoteString()); + final Map tablesMaxTuple = + CtidUtils.isTidRangeScanCapableDBServer(database) ? null + : PostgresQueryUtils.getTableMaxTupleForStreams(database, finalListOfStreamsToBeSyncedViaCtid, getQuoteString()); if (!streamsCategorised.ctidStreams().streamsForCtidSync().isEmpty()) { LOGGER.info("Streams to be synced via ctid : {}", finalListOfStreamsToBeSyncedViaCtid.size()); LOGGER.info("Streams: {}", prettyPrintConfiguredAirbyteStreamList(finalListOfStreamsToBeSyncedViaCtid)); @@ -512,16 +535,16 @@ public List> getIncrementalIterators(final final PostgresCtidHandler ctidHandler = new PostgresCtidHandler(sourceConfig, database, new CtidPostgresSourceOperations(Optional.empty()), getQuoteString(), - fileNodes.result(), tableBlockSizes, ctidStateManager, + fileNodeHandler, tableBlockSizes, tablesMaxTuple, ctidStateManager, namespacePair -> Jsons.jsonNode(xminStatus)); - final List> ctidIterators = new ArrayList<>(ctidHandler.getIncrementalIterators( + final List> initialSyncCtidIterators = new ArrayList<>(ctidHandler.getInitialSyncCtidIterator( new ConfiguredAirbyteCatalog().withStreams(finalListOfStreamsToBeSyncedViaCtid), tableNameToTable, emittedAt)); final List> xminIterators = new ArrayList<>(xminHandler.getIncrementalIterators( new ConfiguredAirbyteCatalog().withStreams(streamsCategorised.remainingStreams().streamsForXminSync()), tableNameToTable, emittedAt)); return Stream - .of(ctidIterators, xminIterators) + .of(initialSyncCtidIterators, xminIterators) .flatMap(Collection::stream) .collect(Collectors.toList()); @@ -539,25 +562,30 @@ public List> getIncrementalIterators(final List finalListOfStreamsToBeSyncedViaCtid = filterStreamsUnderVacuumForCtidSync(streamsUnderVacuum.result(), streamsCategorised.ctidStreams()); - final ResultWithFailed> fileNodes = + final FileNodeHandler fileNodeHandler = PostgresQueryUtils.fileNodeForStreams(database, finalListOfStreamsToBeSyncedViaCtid, getQuoteString()); // Streams we failed to query for fileNode - such as in the case of Views are reclassified as // standard - if (!fileNodes.failed().isEmpty()) { - reclassifyCategorisedCtidStreams(streamsCategorised, fileNodes.failed()); + if (!fileNodeHandler.getFailedToQuery().isEmpty()) { + reclassifyCategorisedCtidStreams(streamsCategorised, fileNodeHandler.getFailedToQuery()); finalListOfStreamsToBeSyncedViaCtid = filterStreamsUnderVacuumForCtidSync(streamsUnderVacuum.result(), streamsCategorised.ctidStreams()); } final CtidStateManager ctidStateManager = - new CtidPerStreamStateManager(streamsCategorised.ctidStreams().statesFromCtidSync(), fileNodes.result()); + new CtidPerStreamStateManager(streamsCategorised.ctidStreams().statesFromCtidSync(), fileNodeHandler); final Map tableBlockSizes = PostgresQueryUtils.getTableBlockSizeForStreams( database, finalListOfStreamsToBeSyncedViaCtid, getQuoteString()); + + final Map tablesMaxTuple = + CtidUtils.isTidRangeScanCapableDBServer(database) ? null + : PostgresQueryUtils.getTableMaxTupleForStreams(database, finalListOfStreamsToBeSyncedViaCtid, getQuoteString()); + if (finalListOfStreamsToBeSyncedViaCtid.isEmpty()) { LOGGER.info("No Streams will be synced via ctid."); } else { @@ -580,13 +608,14 @@ public List> getIncrementalIterators(final database, new CtidPostgresSourceOperations(Optional.empty()), getQuoteString(), - fileNodes.result(), + fileNodeHandler, tableBlockSizes, + tablesMaxTuple, ctidStateManager, namespacePair -> Jsons.jsonNode(cursorBasedStatusMap.get(namespacePair))); - final List> ctidIterators = new ArrayList<>( - cursorBasedCtidHandler.getIncrementalIterators(new ConfiguredAirbyteCatalog().withStreams(finalListOfStreamsToBeSyncedViaCtid), + final List> initialSyncCtidIterators = new ArrayList<>( + cursorBasedCtidHandler.getInitialSyncCtidIterator(new ConfiguredAirbyteCatalog().withStreams(finalListOfStreamsToBeSyncedViaCtid), tableNameToTable, emittedAt)); final List> cursorBasedIterators = new ArrayList<>(super.getIncrementalIterators(database, @@ -597,7 +626,7 @@ public List> getIncrementalIterators(final postgresCursorBasedStateManager, emittedAt)); return Stream - .of(ctidIterators, cursorBasedIterators) + .of(initialSyncCtidIterators, cursorBasedIterators) .flatMap(Collection::stream) .collect(Collectors.toList()); } @@ -661,9 +690,6 @@ protected boolean isNotInternalSchema(final JsonNode jsonNode, final Set @Override protected AirbyteStateType getSupportedStateType(final JsonNode config) { - if (!featureFlags.useStreamCapableState()) { - return AirbyteStateType.LEGACY; - } return PostgresUtils.isCdc(config) ? AirbyteStateType.GLOBAL : AirbyteStateType.STREAM; } @@ -678,7 +704,7 @@ protected void setStateEmissionFrequencyForDebug(final int stateEmissionFrequenc } public static void main(final String[] args) throws Exception { - final Source source = PostgresSource.sshWrappedSource(); + final Source source = PostgresSource.sshWrappedSource(new PostgresSource()); LOGGER.info("starting source: {}", PostgresSource.class); new IntegrationRunner(source).run(args); LOGGER.info("completed source: {}", PostgresSource.class); @@ -687,6 +713,25 @@ public static void main(final String[] args) throws Exception { @Override @Trace(operationName = CHECK_TRACE_OPERATION_NAME) public AirbyteConnectionStatus check(final JsonNode config) throws Exception { + // #15808 Disallow connecting to db with disable, prefer or allow SSL mode when connecting directly + // and not over SSH tunnel + if (cloudDeploymentMode()) { + LOGGER.info("Source configured as in Cloud Deployment mode"); + if (config.has(TUNNEL_METHOD) + && config.get(TUNNEL_METHOD).has(TUNNEL_METHOD) + && config.get(TUNNEL_METHOD).get(TUNNEL_METHOD).asText().equals(NO_TUNNEL)) { + // If no SSH tunnel + if (config.has(SSL_MODE) && config.get(SSL_MODE).has(MODE)) { + if (Set.of(SSL_MODE_DISABLE, SSL_MODE_ALLOW, SSL_MODE_PREFER).contains(config.get(SSL_MODE).get(MODE).asText())) { + // Fail in case SSL mode is disable, allow or prefer + return new AirbyteConnectionStatus() + .withStatus(Status.FAILED) + .withMessage( + "Unsecured connection not allowed. If no SSH Tunnel set up, please use one of the following SSL modes: require, verify-ca, verify-full"); + } + } + } + } if (PostgresUtils.isCdc(config)) { if (config.has(SSL_MODE) && config.get(SSL_MODE).has(MODE)) { final String sslModeValue = config.get(SSL_MODE).get(MODE).asText(); @@ -793,4 +838,8 @@ private List getFullTableEstimate(final JdbcDatabase database, return jsonNodes; } + private boolean cloudDeploymentMode() { + return AdaptiveSourceRunner.CLOUD_MODE.equalsIgnoreCase(featureFlags.deploymentMode()); + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java index e6356462fc2f..e0ea271195d7 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSourceOperations.java @@ -4,13 +4,13 @@ package io.airbyte.integrations.source.postgres; -import static io.airbyte.db.DataTypeUtils.TIMESTAMPTZ_FORMATTER; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_DECIMAL_DIGITS; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_SCHEMA_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_TABLE_NAME; +import static io.airbyte.cdk.db.DataTypeUtils.TIMESTAMPTZ_FORMATTER; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_COLUMN_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_DECIMAL_DIGITS; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_SCHEMA_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_TABLE_NAME; import static io.airbyte.integrations.source.postgres.PostgresType.safeGetJdbcType; import com.fasterxml.jackson.core.JsonProcessingException; @@ -21,30 +21,28 @@ import com.fasterxml.jackson.databind.node.NullNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.VisibleForTesting; +import io.airbyte.cdk.db.DataTypeUtils; +import io.airbyte.cdk.db.SourceOperations; +import io.airbyte.cdk.db.jdbc.AbstractJdbcCompatibleSourceOperations; +import io.airbyte.cdk.db.jdbc.DateTimeConverter; import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.DataTypeUtils; -import io.airbyte.db.SourceOperations; -import io.airbyte.db.jdbc.AbstractJdbcCompatibleSourceOperations; -import io.airbyte.db.jdbc.DateTimeConverter; import io.airbyte.protocol.models.JsonSchemaPrimitiveUtil.JsonSchemaPrimitive; import io.airbyte.protocol.models.JsonSchemaType; import java.math.BigDecimal; +import java.sql.Date; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Timestamp; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import java.time.OffsetDateTime; -import java.time.OffsetTime; +import java.time.*; import java.time.format.DateTimeParseException; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import org.postgresql.PGStatement; import org.postgresql.geometric.PGbox; import org.postgresql.geometric.PGcircle; import org.postgresql.geometric.PGline; @@ -67,13 +65,22 @@ public class PostgresSourceOperations extends AbstractJdbcCompatibleSourceOperat private static final Map POSTGRES_TYPE_DICT = new HashMap<>(); private final Map> streamColumnInfo = new HashMap<>(); + private static final String POSITIVE_INFINITY_STRING = "Infinity"; + private static final String NEGATIVE_INFINITY_STRING = "-Infinity"; + private static final Date POSITIVE_INFINITY_DATE = new Date(PGStatement.DATE_POSITIVE_INFINITY); + private static final Date NEGATIVE_INFINITY_DATE = new Date(PGStatement.DATE_NEGATIVE_INFINITY); + private static final Timestamp POSITIVE_INFINITY_TIMESTAMP = new Timestamp(PGStatement.DATE_POSITIVE_INFINITY); + private static final Timestamp NEGATIVE_INFINITY_TIMESTAMP = new Timestamp(PGStatement.DATE_NEGATIVE_INFINITY); + private static final OffsetDateTime POSITIVE_INFINITY_OFFSET_DATE_TIME = OffsetDateTime.MAX; + private static final OffsetDateTime NEGATIVE_INFINITY_OFFSET_DATE_TIME = OffsetDateTime.MIN; + static { Arrays.stream(PostgresType.class.getEnumConstants()).forEach(c -> POSTGRES_TYPE_DICT.put(c.type, c)); } @Override public JsonNode rowToJson(final ResultSet queryContext) throws SQLException { - // the first call communicates with the database. after that the result is cached. + // the first call communicates with the database, after that the result is cached. final ResultSetMetaData metadata = queryContext.getMetaData(); final int columnCount = metadata.getColumnCount(); final ObjectNode jsonNode = (ObjectNode) Jsons.jsonNode(Collections.emptyMap()); @@ -92,6 +99,8 @@ public void setCursorField(final PreparedStatement preparedStatement, final PostgresType cursorFieldType, final String value) throws SQLException { + + LOGGER.warn("SGX setCursorField value=" + value + "cursorFieldType=" + cursorFieldType); switch (cursorFieldType) { case TIMESTAMP -> setTimestamp(preparedStatement, parameterIndex, value); @@ -217,7 +226,13 @@ public void copyToJsonField(final ResultSet resultSet, final int colIndex, final case TIMESTAMP -> putTimestamp(json, columnName, resultSet, colIndex); case BLOB, BINARY, VARBINARY, LONGVARBINARY -> putBinary(json, columnName, resultSet, colIndex); case ARRAY -> putArray(json, columnName, resultSet, colIndex); - default -> json.put(columnName, value); + default -> { + if (columnInfo.columnType.isArrayType()) { + putArray(json, columnName, resultSet, colIndex); + } else { + json.put(columnName, value); + } + } } } } @@ -259,6 +274,10 @@ private void putTimestampArray(final ObjectNode node, final String columnName, f final Timestamp timestamp = arrayResultSet.getTimestamp(2); if (timestamp == null) { arrayNode.add(NullNode.getInstance()); + } else if (POSITIVE_INFINITY_TIMESTAMP.equals(timestamp)) { + arrayNode.add(POSITIVE_INFINITY_STRING); + } else if (NEGATIVE_INFINITY_TIMESTAMP.equals(timestamp)) { + arrayNode.add(NEGATIVE_INFINITY_STRING); } else { arrayNode.add(DateTimeConverter.convertToTimestamp(timestamp)); } @@ -274,6 +293,10 @@ private void putTimestampTzArray(final ObjectNode node, final String columnName, final OffsetDateTime timestamptz = getObject(arrayResultSet, 2, OffsetDateTime.class); if (timestamptz == null) { arrayNode.add(NullNode.getInstance()); + } else if (POSITIVE_INFINITY_OFFSET_DATE_TIME.equals(timestamptz)) { + arrayNode.add(POSITIVE_INFINITY_STRING); + } else if (NEGATIVE_INFINITY_OFFSET_DATE_TIME.equals(timestamptz)) { + arrayNode.add(NEGATIVE_INFINITY_STRING); } else { final LocalDate localDate = timestamptz.toLocalDate(); arrayNode.add(resolveEra(localDate, timestamptz.format(TIMESTAMPTZ_FORMATTER))); @@ -286,9 +309,13 @@ private void putDateArray(final ObjectNode node, final String columnName, final final ArrayNode arrayNode = Jsons.arrayNode(); final ResultSet arrayResultSet = resultSet.getArray(colIndex).getResultSet(); while (arrayResultSet.next()) { - final LocalDate date = getObject(arrayResultSet, 2, LocalDate.class); + final Date date = getObject(arrayResultSet, 2, Date.class); if (date == null) { arrayNode.add(NullNode.getInstance()); + } else if (POSITIVE_INFINITY_DATE.equals(date)) { + arrayNode.add(POSITIVE_INFINITY_STRING); + } else if (NEGATIVE_INFINITY_DATE.equals(date)) { + arrayNode.add(NEGATIVE_INFINITY_STRING); } else { arrayNode.add(DateTimeConverter.convertToDate(date)); } @@ -385,9 +412,57 @@ private void putLongArray(final ObjectNode node, final String columnName, final node.set(columnName, arrayNode); } + @Override + protected void putDate(ObjectNode node, String columnName, ResultSet resultSet, int index) throws SQLException { + Date dateFromResultSet = resultSet.getDate(index); + if (POSITIVE_INFINITY_DATE.equals(dateFromResultSet)) { + node.put(columnName, POSITIVE_INFINITY_STRING); + } else if (NEGATIVE_INFINITY_DATE.equals(dateFromResultSet)) { + node.put(columnName, NEGATIVE_INFINITY_STRING); + } else { + super.putDate(node, columnName, resultSet, index); + } + } + + @Override + protected void putTime(ObjectNode node, String columnName, ResultSet resultSet, int index) throws SQLException { + super.putTime(node, columnName, resultSet, index); + } + @Override protected void putTimestamp(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { - node.put(columnName, DateTimeConverter.convertToTimestamp(resultSet.getTimestamp(index))); + Timestamp timestampFromResultSet = resultSet.getTimestamp(index); + String strValue = resultSet.getString(index); + if (POSITIVE_INFINITY_TIMESTAMP.equals(timestampFromResultSet)) { + node.put(columnName, POSITIVE_INFINITY_STRING); + } else if (NEGATIVE_INFINITY_TIMESTAMP.equals(timestampFromResultSet)) { + node.put(columnName, NEGATIVE_INFINITY_STRING); + } else { + node.put(columnName, DateTimeConverter.convertToTimestamp(timestampFromResultSet)); + } + } + + @Override + protected void putTimeWithTimezone(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { + final OffsetTime timetz = getObject(resultSet, index, OffsetTime.class); + node.put(columnName, DateTimeConverter.convertToTimeWithTimezone(timetz)); + } + + @Override + protected void putTimestampWithTimezone(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) + throws SQLException { + final OffsetDateTime timestampTz = getObject(resultSet, index, OffsetDateTime.class); + final String timestampTzVal; + if (POSITIVE_INFINITY_OFFSET_DATE_TIME.equals(timestampTz)) { + timestampTzVal = POSITIVE_INFINITY_STRING; + } else if (NEGATIVE_INFINITY_OFFSET_DATE_TIME.equals(timestampTz)) { + timestampTzVal = NEGATIVE_INFINITY_STRING; + } else { + final LocalDate localDate = timestampTz.toLocalDate(); + timestampTzVal = resolveEra(localDate, timestampTz.format(TIMESTAMPTZ_FORMATTER)); + } + + node.put(columnName, timestampTzVal); } @Override diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresType.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresType.java index 878f5497dbf1..6afda89271ae 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresType.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresType.java @@ -113,6 +113,15 @@ public Integer getVendorTypeNumber() { return type; } + /** + * Returns true if the PostgresType is an array type, false otherwise. + * + * @return true if the PostgresType is an array type, false otherwise. + */ + public boolean isArrayType() { + return type == Types.ARRAY; + } + /** * Returns the {@code JDBCType} that corresponds to the specified {@code Types} value * diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresUtils.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresUtils.java index e2e8d16b9221..bfd4903cef9f 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresUtils.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresUtils.java @@ -23,9 +23,8 @@ import static io.airbyte.integrations.source.postgres.PostgresType.VARCHAR; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; -import io.airbyte.protocol.models.v0.SyncMode; import java.time.Duration; import java.util.List; import java.util.Optional; @@ -45,11 +44,21 @@ public class PostgresUtils { private static final String PGOUTPUT_PLUGIN = "pgoutput"; public static final Duration MIN_FIRST_RECORD_WAIT_TIME = Duration.ofMinutes(2); - public static final Duration MAX_FIRST_RECORD_WAIT_TIME = Duration.ofMinutes(20); - public static final Duration DEFAULT_FIRST_RECORD_WAIT_TIME = Duration.ofMinutes(5); + public static final Duration MAX_FIRST_RECORD_WAIT_TIME = Duration.ofMinutes(40); + public static final Duration DEFAULT_FIRST_RECORD_WAIT_TIME = Duration.ofMinutes(20); + public static final Duration DEFAULT_SUBSEQUENT_RECORD_WAIT_TIME = Duration.ofMinutes(1); + private static final int MIN_QUEUE_SIZE = 1000; private static final int MAX_QUEUE_SIZE = 10000; + private static final String DROP_AGGREGATE_IF_EXISTS_STATEMENT = "DROP aggregate IF EXISTS EPHEMERAL_HEARTBEAT(float4)"; + private static final String CREATE_AGGREGATE_STATEMENT = "CREATE AGGREGATE EPHEMERAL_HEARTBEAT(float4) (SFUNC = float4pl, STYPE = float4)"; + private static final String DROP_AGGREGATE_STATEMENT = "DROP aggregate EPHEMERAL_HEARTBEAT(float4)"; + private static final List EPHEMERAL_HEARTBEAT_CREATE_STATEMENTS = + List.of(DROP_AGGREGATE_IF_EXISTS_STATEMENT, CREATE_AGGREGATE_STATEMENT, DROP_AGGREGATE_STATEMENT); + + private static final int POSTGRESQL_VERSION_15 = 15; + public static String getPluginValue(final JsonNode field) { return field.has("plugin") ? field.get("plugin").asText() : PGOUTPUT_PLUGIN; } @@ -70,6 +79,10 @@ public static boolean shouldFlushAfterSync(final JsonNode config) { return shouldFlushAfterSync; } + public static boolean isDebugMode(final JsonNode config) { + return config.hasNonNull("debug_mode"); + } + public static Optional getFirstRecordWaitSeconds(final JsonNode config) { final JsonNode replicationMethod = config.get("replication_method"); if (replicationMethod != null && replicationMethod.has("initial_waiting_seconds")) { @@ -91,7 +104,7 @@ private static OptionalInt extractQueueSizeFromConfig(final JsonNode config) { public static int getQueueSize(final JsonNode config) { final OptionalInt sizeFromConfig = extractQueueSizeFromConfig(config); if (sizeFromConfig.isPresent()) { - int size = sizeFromConfig.getAsInt(); + final int size = sizeFromConfig.getAsInt(); if (size < MIN_QUEUE_SIZE) { LOGGER.warn("Queue size is overridden to {} , which is the min allowed for safety.", MIN_QUEUE_SIZE); @@ -159,6 +172,18 @@ public static Duration getFirstRecordWaitTime(final JsonNode config) { return firstRecordWaitTime; } + public static Duration getSubsequentRecordWaitTime(final JsonNode config) { + Duration subsequentRecordWaitTime = DEFAULT_SUBSEQUENT_RECORD_WAIT_TIME; + final boolean isTest = config.has("is_test") && config.get("is_test").asBoolean(); + final Optional firstRecordWaitSeconds = getFirstRecordWaitSeconds(config); + if (isTest && firstRecordWaitSeconds.isPresent()) { + // In tests, reuse the initial_waiting_seconds property to speed things up. + subsequentRecordWaitTime = Duration.ofSeconds(firstRecordWaitSeconds.get()); + } + LOGGER.info("Subsequent record waiting time: {} seconds", subsequentRecordWaitTime.getSeconds()); + return subsequentRecordWaitTime; + } + public static boolean isXmin(final JsonNode config) { final boolean isXmin = config.hasNonNull("replication_method") && config.get("replication_method").get("method").asText().equals("Xmin"); @@ -166,13 +191,19 @@ public static boolean isXmin(final JsonNode config) { return isXmin; } - public static boolean isAnyStreamIncrementalSyncMode(final ConfiguredAirbyteCatalog catalog) { - return catalog.getStreams().stream().map(ConfiguredAirbyteStream::getSyncMode) - .anyMatch(syncMode -> syncMode == SyncMode.INCREMENTAL); - } - public static String prettyPrintConfiguredAirbyteStreamList(final List streamList) { return streamList.stream().map(s -> "%s.%s".formatted(s.getStream().getNamespace(), s.getStream().getName())).collect(Collectors.joining(", ")); } + public static void advanceLsn(final JdbcDatabase database) { + try { + if (database.getMetaData().getDatabaseMajorVersion() < POSTGRESQL_VERSION_15) { + database.executeWithinTransaction(EPHEMERAL_HEARTBEAT_CREATE_STATEMENTS); + LOGGER.info("Succesfully forced LSN advancement by creating & dropping an ephemeral heartbeat aggregate"); + } + } catch (final Exception e) { + LOGGER.info("Failed to force LSN advancement by creating & dropping an ephemeral heartbeat aggregate."); + } + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcConnectorMetadataInjector.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcConnectorMetadataInjector.java index ba64c8a55728..d5ce5cde32e3 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcConnectorMetadataInjector.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcConnectorMetadataInjector.java @@ -4,13 +4,13 @@ package io.airbyte.integrations.source.postgres.cdc; -import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; -import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_LSN; -import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_LSN; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.integrations.debezium.CdcMetadataInjector; +import io.airbyte.cdk.integrations.debezium.CdcMetadataInjector; public class PostgresCdcConnectorMetadataInjector implements CdcMetadataInjector { @@ -32,4 +32,9 @@ public String namespace(final JsonNode source) { return source.get("schema").asText(); } + @Override + public String name(JsonNode source) { + return source.get("table").asText(); + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidInitializer.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidInitializer.java index 9b894ba0b5cb..4077af63a8d9 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidInitializer.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidInitializer.java @@ -5,19 +5,24 @@ package io.airbyte.integrations.source.postgres.cdc; import static io.airbyte.integrations.source.postgres.PostgresQueryUtils.streamsUnderVacuum; +import static io.airbyte.integrations.source.postgres.PostgresUtils.isDebugMode; import static io.airbyte.integrations.source.postgres.PostgresUtils.prettyPrintConfiguredAirbyteStreamList; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.debezium.AirbyteDebeziumHandler; +import io.airbyte.cdk.integrations.debezium.internals.DebeziumPropertiesManager; +import io.airbyte.cdk.integrations.debezium.internals.postgres.PostgresCdcTargetPosition; +import io.airbyte.cdk.integrations.debezium.internals.postgres.PostgresDebeziumStateUtil; +import io.airbyte.cdk.integrations.source.relationaldb.TableInfo; +import io.airbyte.cdk.integrations.source.relationaldb.models.CdcState; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; +import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.debezium.AirbyteDebeziumHandler; -import io.airbyte.integrations.debezium.internals.postgres.PostgresCdcTargetPosition; -import io.airbyte.integrations.debezium.internals.postgres.PostgresDebeziumStateUtil; import io.airbyte.integrations.source.postgres.PostgresQueryUtils; -import io.airbyte.integrations.source.postgres.PostgresQueryUtils.ResultWithFailed; import io.airbyte.integrations.source.postgres.PostgresQueryUtils.TableBlockSize; import io.airbyte.integrations.source.postgres.PostgresType; import io.airbyte.integrations.source.postgres.PostgresUtils; @@ -26,10 +31,9 @@ import io.airbyte.integrations.source.postgres.ctid.CtidPostgresSourceOperations; import io.airbyte.integrations.source.postgres.ctid.CtidPostgresSourceOperations.CdcMetadataInjector; import io.airbyte.integrations.source.postgres.ctid.CtidStateManager; +import io.airbyte.integrations.source.postgres.ctid.CtidUtils; +import io.airbyte.integrations.source.postgres.ctid.FileNodeHandler; import io.airbyte.integrations.source.postgres.ctid.PostgresCtidHandler; -import io.airbyte.integrations.source.relationaldb.TableInfo; -import io.airbyte.integrations.source.relationaldb.models.CdcState; -import io.airbyte.integrations.source.relationaldb.state.StateManager; import io.airbyte.protocol.models.CommonField; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; @@ -66,10 +70,16 @@ public static List> cdcCtidIteratorsCombin try { final JsonNode sourceConfig = database.getSourceConfig(); final Duration firstRecordWaitTime = PostgresUtils.getFirstRecordWaitTime(sourceConfig); + final Duration subsequentRecordWaitTime = PostgresUtils.getSubsequentRecordWaitTime(sourceConfig); final OptionalInt queueSize = OptionalInt.of(PostgresUtils.getQueueSize(sourceConfig)); LOGGER.info("First record waiting time: {} seconds", firstRecordWaitTime.getSeconds()); LOGGER.info("Queue size: {}", queueSize.getAsInt()); + if (isDebugMode(sourceConfig) && !PostgresUtils.shouldFlushAfterSync(sourceConfig)) { + throw new ConfigErrorException("WARNING: The config indicates that we are clearing the WAL while reading data. This will mutate the WAL" + + " associated with the source being debugged and is not advised."); + } + final PostgresDebeziumStateUtil postgresDebeziumStateUtil = new PostgresDebeziumStateUtil(); final JsonNode initialDebeziumState = postgresDebeziumStateUtil.constructInitialDebeziumState(database, @@ -102,7 +112,8 @@ public static List> cdcCtidIteratorsCombin if (!savedOffsetAfterReplicationSlotLSN) { LOGGER.warn("Saved offset is before Replication slot's confirmed_flush_lsn, Airbyte will trigger sync from scratch"); - } else if (PostgresUtils.shouldFlushAfterSync(sourceConfig)) { + } else if (!isDebugMode(sourceConfig) && PostgresUtils.shouldFlushAfterSync(sourceConfig)) { + // We do not want to acknowledge the WAL logs in debug mode. postgresDebeziumStateUtil.commitLSNToPostgresDatabase(database.getDatabaseConfig(), savedOffset, sourceConfig.get("replication_method").get("replication_slot").asText(), @@ -114,7 +125,7 @@ public static List> cdcCtidIteratorsCombin : stateManager.getCdcStateManager().getCdcState(); final CtidStreams ctidStreams = PostgresCdcCtidUtils.streamsToSyncViaCtid(stateManager.getCdcStateManager(), catalog, savedOffsetAfterReplicationSlotLSN); - final List> ctidIterator = new ArrayList<>(); + final List> initialSyncCtidIterators = new ArrayList<>(); final List streamsUnderVacuum = new ArrayList<>(); if (!ctidStreams.streamsForCtidSync().isEmpty()) { streamsUnderVacuum.addAll(streamsUnderVacuum(database, @@ -127,46 +138,59 @@ public static List> cdcCtidIteratorsCombin .toList(); LOGGER.info("Streams to be synced via ctid : {}", finalListOfStreamsToBeSyncedViaCtid.size()); LOGGER.info("Streams: {}", prettyPrintConfiguredAirbyteStreamList(finalListOfStreamsToBeSyncedViaCtid)); - final ResultWithFailed> fileNodes = - PostgresQueryUtils.fileNodeForStreams(database, - finalListOfStreamsToBeSyncedViaCtid, - quoteString); - final CtidStateManager ctidStateManager = new CtidGlobalStateManager(ctidStreams, fileNodes.result(), stateToBeUsed, catalog); + final FileNodeHandler fileNodeHandler = PostgresQueryUtils.fileNodeForStreams(database, + finalListOfStreamsToBeSyncedViaCtid, + quoteString); + final CtidStateManager ctidStateManager = new CtidGlobalStateManager(ctidStreams, fileNodeHandler, stateToBeUsed, catalog); final CtidPostgresSourceOperations ctidPostgresSourceOperations = new CtidPostgresSourceOperations( Optional.of(new CdcMetadataInjector( - emittedAt.toString(), io.airbyte.db.PostgresUtils.getLsn(database).asLong(), new PostgresCdcConnectorMetadataInjector()))); + emittedAt.toString(), io.airbyte.cdk.db.PostgresUtils.getLsn(database).asLong(), new PostgresCdcConnectorMetadataInjector()))); final Map tableBlockSizes = PostgresQueryUtils.getTableBlockSizeForStreams( database, finalListOfStreamsToBeSyncedViaCtid, quoteString); + + final Map tablesMaxTuple = + CtidUtils.isTidRangeScanCapableDBServer(database) ? null + : PostgresQueryUtils.getTableMaxTupleForStreams(database, finalListOfStreamsToBeSyncedViaCtid, quoteString); + final PostgresCtidHandler ctidHandler = new PostgresCtidHandler(sourceConfig, database, ctidPostgresSourceOperations, quoteString, - fileNodes.result(), + fileNodeHandler, tableBlockSizes, + tablesMaxTuple, ctidStateManager, namespacePair -> Jsons.emptyObject()); - ctidIterator.addAll(ctidHandler.getIncrementalIterators( + initialSyncCtidIterators.addAll(ctidHandler.getInitialSyncCtidIterator( new ConfiguredAirbyteCatalog().withStreams(finalListOfStreamsToBeSyncedViaCtid), tableNameToTable, emittedAt)); } else { LOGGER.info("No streams will be synced via ctid"); } + // Gets the target position. + final var targetPosition = PostgresCdcTargetPosition.targetPosition(database); + // Attempt to advance LSN past the target position. For versions of Postgres before PG15, this + // ensures that there is an event that debezium will + // receive that is after the target LSN. + PostgresUtils.advanceLsn(database); final AirbyteDebeziumHandler handler = new AirbyteDebeziumHandler<>(sourceConfig, - PostgresCdcTargetPosition.targetPosition(database), false, firstRecordWaitTime, queueSize); + targetPosition, false, firstRecordWaitTime, subsequentRecordWaitTime, queueSize); final PostgresCdcStateHandler postgresCdcStateHandler = new PostgresCdcStateHandler(stateManager); - final Supplier> incrementalIteratorSupplier = () -> handler.getIncrementalIterators(catalog, + final Supplier> incrementalIteratorSupplier = () -> handler.getIncrementalIterators( + catalog, new PostgresCdcSavedInfoFetcher(stateToBeUsed), postgresCdcStateHandler, new PostgresCdcConnectorMetadataInjector(), PostgresCdcProperties.getDebeziumDefaultProperties(database), + DebeziumPropertiesManager.DebeziumConnectorType.RELATIONALDB, emittedAt, false); - if (ctidIterator.isEmpty()) { + if (initialSyncCtidIterators.isEmpty()) { return Collections.singletonList(incrementalIteratorSupplier.get()); } @@ -176,12 +200,12 @@ public static List> cdcCtidIteratorsCombin // We finish the current CDC once the initial snapshot is complete and the next sync starts // processing the WAL return Stream - .of(ctidIterator, Collections.singletonList(AutoCloseableIterators.lazyIterator(incrementalIteratorSupplier, null))) + .of(initialSyncCtidIterators, Collections.singletonList(AutoCloseableIterators.lazyIterator(incrementalIteratorSupplier, null))) .flatMap(Collection::stream) .collect(Collectors.toList()); } else { LOGGER.warn("Streams are under vacuuming, not going to process WAL"); - return ctidIterator; + return initialSyncCtidIterators; } } catch (final SQLException e) { diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidUtils.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidUtils.java index da07156cde17..0c8ebe4aa354 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidUtils.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcCtidUtils.java @@ -8,9 +8,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Sets; +import io.airbyte.cdk.integrations.source.relationaldb.CdcStateManager; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.postgres.internal.models.CtidStatus; -import io.airbyte.integrations.source.relationaldb.CdcStateManager; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcProperties.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcProperties.java index b1789f792cd6..609bf9def3b9 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcProperties.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcProperties.java @@ -4,16 +4,16 @@ package io.airbyte.integrations.source.postgres.cdc; -import static io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.CLIENT_KEY_STORE_PASS; -import static io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.CLIENT_KEY_STORE_URL; -import static io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.SSL_MODE; -import static io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.TRUST_KEY_STORE_PASS; +import static io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils.CLIENT_KEY_STORE_PASS; +import static io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils.CLIENT_KEY_STORE_URL; +import static io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils.SSL_MODE; +import static io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils.TRUST_KEY_STORE_PASS; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.debezium.internals.postgres.PostgresConverter; -import io.airbyte.integrations.source.jdbc.JdbcSSLConnectionUtils.SslMode; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.debezium.internals.postgres.PostgresConverter; +import io.airbyte.cdk.integrations.source.jdbc.JdbcSSLConnectionUtils.SslMode; import io.airbyte.integrations.source.postgres.PostgresSource; import io.airbyte.integrations.source.postgres.PostgresUtils; import java.net.URI; @@ -25,7 +25,11 @@ public class PostgresCdcProperties { - private static final int HEARTBEAT_FREQUENCY_SEC = 10; + private static final Duration HEARTBEAT_INTERVAL = Duration.ofSeconds(10L); + + // Test execution latency is lower when heartbeats are more frequent. + private static final Duration HEARTBEAT_INTERVAL_IN_TESTS = Duration.ofSeconds(1L); + private static final Logger LOGGER = LoggerFactory.getLogger(PostgresCdcProperties.class); public static Properties getDebeziumDefaultProperties(final JdbcDatabase database) { @@ -58,7 +62,13 @@ private static Properties commonProperties(final JdbcDatabase database) { props.setProperty("converters", "datetime"); props.setProperty("datetime.type", PostgresConverter.class.getName()); props.setProperty("include.unknown.datatypes", "true"); - props.setProperty("heartbeat.interval.ms", Long.toString(Duration.ofSeconds(HEARTBEAT_FREQUENCY_SEC).toMillis())); + + final Duration heartbeatInterval = + (database.getSourceConfig().has("is_test") && database.getSourceConfig().get("is_test").asBoolean()) + ? HEARTBEAT_INTERVAL_IN_TESTS + : HEARTBEAT_INTERVAL; + props.setProperty("heartbeat.interval.ms", Long.toString(heartbeatInterval.toMillis())); + if (PostgresUtils.shouldFlushAfterSync(sourceConfig)) { props.setProperty("flush.lsn.source", "false"); } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcSavedInfoFetcher.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcSavedInfoFetcher.java index 8e5ad5408bd8..8f712e53aef3 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcSavedInfoFetcher.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcSavedInfoFetcher.java @@ -5,8 +5,9 @@ package io.airbyte.integrations.source.postgres.cdc; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.debezium.CdcSavedInfoFetcher; -import io.airbyte.integrations.source.relationaldb.models.CdcState; +import io.airbyte.cdk.integrations.debezium.CdcSavedInfoFetcher; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteSchemaHistoryStorage.SchemaHistory; +import io.airbyte.cdk.integrations.source.relationaldb.models.CdcState; import java.util.Optional; public class PostgresCdcSavedInfoFetcher implements CdcSavedInfoFetcher { @@ -24,8 +25,8 @@ public JsonNode getSavedOffset() { } @Override - public Optional getSavedSchemaHistory() { - return Optional.empty(); + public SchemaHistory> getSavedSchemaHistory() { + throw new RuntimeException("Schema history is not relevant for Postgres"); } } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcStateHandler.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcStateHandler.java index 264db2764da8..1232f744cd12 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcStateHandler.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cdc/PostgresCdcStateHandler.java @@ -5,10 +5,11 @@ package io.airbyte.integrations.source.postgres.cdc; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.debezium.CdcStateHandler; +import io.airbyte.cdk.integrations.debezium.internals.AirbyteSchemaHistoryStorage.SchemaHistory; +import io.airbyte.cdk.integrations.source.relationaldb.models.CdcState; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.debezium.CdcStateHandler; -import io.airbyte.integrations.source.relationaldb.models.CdcState; -import io.airbyte.integrations.source.relationaldb.state.StateManager; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteMessage.Type; import io.airbyte.protocol.models.v0.AirbyteStateMessage; @@ -32,7 +33,7 @@ public boolean isCdcCheckpointEnabled() { } @Override - public AirbyteMessage saveState(final Map offset, final String dbHistory) { + public AirbyteMessage saveState(final Map offset, final SchemaHistory ignored) { final JsonNode asJson = Jsons.jsonNode(offset); LOGGER.info("debezium state: {}", asJson); final CdcState cdcState = new CdcState().withState(asJson); @@ -49,7 +50,7 @@ public AirbyteMessage saveState(final Map offset, final String d * Here we just want to emit the state to update the list of streams in the database to mark the * completion of snapshot of new added streams. The addition of new streams in the state is done * here - * {@link io.airbyte.integrations.source.relationaldb.state.GlobalStateManager#toState(Optional)} + * {@link io.airbyte.cdk.integrations.source.relationaldb.state.GlobalStateManager#toState(Optional)} * which is called inside the {@link StateManager#emit(Optional)} method which is being triggered * below. The toState method adds all the streams present in the catalog in the state. Since there * is no change in the CDC state value, whatever was present in the database will again be stored. diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/Ctid.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/Ctid.java index 11f4ba0d09c6..bd6b4ae9bb4a 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/Ctid.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/Ctid.java @@ -16,6 +16,7 @@ public class Ctid { final Long page; final Long tuple; + public static final Ctid ZERO = Ctid.of(0, 0); public static Ctid of(final long page, final long tuple) { return new Ctid(page, tuple); @@ -71,4 +72,8 @@ public int hashCode() { return Objects.hash(page, tuple); } + public static Ctid inc(final Ctid ctid, final long maxTuple) { + return (ctid.tuple + 1 > maxTuple) ? Ctid.of(ctid.page + 1, 1) : Ctid.of(ctid.page, ctid.tuple + 1); + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidGlobalStateManager.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidGlobalStateManager.java index 9a24253caa30..5def38b9240b 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidGlobalStateManager.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidGlobalStateManager.java @@ -5,11 +5,11 @@ package io.airbyte.integrations.source.postgres.ctid; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.source.relationaldb.models.CdcState; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.postgres.cdc.PostgresCdcCtidUtils.CtidStreams; import io.airbyte.integrations.source.postgres.internal.models.CtidStatus; -import io.airbyte.integrations.source.relationaldb.models.CdcState; -import io.airbyte.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.AirbyteGlobalState; import io.airbyte.protocol.models.v0.AirbyteStateMessage; @@ -37,10 +37,10 @@ public class CtidGlobalStateManager extends CtidStateManager { private final Set streamsThatHaveCompletedSnapshot; public CtidGlobalStateManager(final CtidStreams ctidStreams, - final Map fileNodes, + final FileNodeHandler fileNodeHandler, final CdcState cdcState, final ConfiguredAirbyteCatalog catalog) { - super(filterOutExpiredFileNodes(ctidStreams.pairToCtidStatus(), fileNodes)); + super(filterOutExpiredFileNodes(ctidStreams.pairToCtidStatus(), fileNodeHandler)); this.cdcState = cdcState; this.streamsThatHaveCompletedSnapshot = initStreamsCompletedSnapshot(ctidStreams, catalog); } @@ -60,11 +60,11 @@ private static Set initStreamsCompletedSnapshot( private static Map filterOutExpiredFileNodes( final Map pairToCtidStatus, - final Map fileNodes) { + final FileNodeHandler fileNodeHandler) { final Map filteredMap = new HashMap<>(); pairToCtidStatus.forEach((pair, ctidStatus) -> { final AirbyteStreamNameNamespacePair updatedPair = new AirbyteStreamNameNamespacePair(pair.getName(), pair.getNamespace()); - if (validateRelationFileNode(ctidStatus, updatedPair, fileNodes)) { + if (validateRelationFileNode(ctidStatus, updatedPair, fileNodeHandler)) { filteredMap.put(updatedPair, ctidStatus); } else { LOGGER.warn( @@ -77,6 +77,7 @@ private static Map filterOutExpiredF @Override public AirbyteStateMessage createCtidStateMessage(final AirbyteStreamNameNamespacePair pair, final CtidStatus ctidStatus) { + pairToCtidStatus.put(pair, ctidStatus); final List streamStates = new ArrayList<>(); streamsThatHaveCompletedSnapshot.forEach(stream -> { final DbStreamState state = getFinalState(stream); diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidPerStreamStateManager.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidPerStreamStateManager.java index 425ea6ea4fc9..c3a514c74006 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidPerStreamStateManager.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidPerStreamStateManager.java @@ -29,12 +29,12 @@ public class CtidPerStreamStateManager extends CtidStateManager { .withType(AirbyteStateType.STREAM) .withStream(new AirbyteStreamState()); - public CtidPerStreamStateManager(final List stateMessages, final Map fileNodes) { - super(createPairToCtidStatusMap(stateMessages, fileNodes)); + public CtidPerStreamStateManager(final List stateMessages, final FileNodeHandler fileNodeHandler) { + super(createPairToCtidStatusMap(stateMessages, fileNodeHandler)); } private static Map createPairToCtidStatusMap(final List stateMessages, - final Map fileNodes) { + final FileNodeHandler fileNodeHandler) { final Map localMap = new HashMap<>(); if (stateMessages != null) { for (final AirbyteStateMessage stateMessage : stateMessages) { @@ -50,7 +50,7 @@ private static Map createPairToCtidS } catch (final IllegalArgumentException e) { throw new ConfigErrorException("Invalid per-stream state"); } - if (validateRelationFileNode(ctidStatus, pair, fileNodes)) { + if (validateRelationFileNode(ctidStatus, pair, fileNodeHandler)) { localMap.put(pair, ctidStatus); } else { LOGGER.warn( @@ -65,6 +65,7 @@ private static Map createPairToCtidS @Override public AirbyteStateMessage createCtidStateMessage(final AirbyteStreamNameNamespacePair pair, final CtidStatus ctidStatus) { + pairToCtidStatus.put(pair, ctidStatus); final AirbyteStreamState airbyteStreamState = new AirbyteStreamState() .withStreamDescriptor( diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidStateIterator.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidStateIterator.java index 60c392f19ade..3b9e06e001b6 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidStateIterator.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidStateIterator.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.AbstractIterator; +import io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants; import io.airbyte.integrations.source.postgres.internal.models.CtidStatus; import io.airbyte.integrations.source.postgres.internal.models.InternalModels.StateType; import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; @@ -27,15 +28,15 @@ public class CtidStateIterator extends AbstractIterator implements Iterator { private static final Logger LOGGER = LoggerFactory.getLogger(CtidStateIterator.class); - public static final Duration SYNC_CHECKPOINT_DURATION = Duration.ofMinutes(15); - public static final Integer SYNC_CHECKPOINT_RECORDS = 10_000; + public static final Duration SYNC_CHECKPOINT_DURATION = DebeziumIteratorConstants.SYNC_CHECKPOINT_DURATION; + public static final Integer SYNC_CHECKPOINT_RECORDS = DebeziumIteratorConstants.SYNC_CHECKPOINT_RECORDS; private final Iterator messageIterator; private final AirbyteStreamNameNamespacePair pair; private boolean hasEmittedFinalState; private String lastCtid; private final JsonNode streamStateForIncrementalRun; - private final long relationFileNode; + private final FileNodeHandler fileNodeHandler; private final CtidStateManager stateManager; private long recordCount = 0L; private Instant lastCheckpoint = Instant.now(); @@ -44,14 +45,14 @@ public class CtidStateIterator extends AbstractIterator implemen public CtidStateIterator(final Iterator messageIterator, final AirbyteStreamNameNamespacePair pair, - final long relationFileNode, + final FileNodeHandler fileNodeHandler, final CtidStateManager stateManager, final JsonNode streamStateForIncrementalRun, final Duration checkpointDuration, final Long checkpointRecords) { this.messageIterator = messageIterator; this.pair = pair; - this.relationFileNode = relationFileNode; + this.fileNodeHandler = fileNodeHandler; this.stateManager = stateManager; this.streamStateForIncrementalRun = streamStateForIncrementalRun; this.syncCheckpointDuration = checkpointDuration; @@ -65,12 +66,14 @@ protected AirbyteMessage computeNext() { if ((recordCount >= syncCheckpointRecords || Duration.between(lastCheckpoint, OffsetDateTime.now()).compareTo(syncCheckpointDuration) > 0) && Objects.nonNull(lastCtid) && StringUtils.isNotBlank(lastCtid)) { + final Long fileNode = fileNodeHandler.getFileNode(pair); + assert fileNode != null; final CtidStatus ctidStatus = new CtidStatus() .withVersion(CTID_STATUS_VERSION) .withStateType(StateType.CTID) .withCtid(lastCtid) .withIncrementalState(streamStateForIncrementalRun) - .withRelationFilenode(relationFileNode); + .withRelationFilenode(fileNode); LOGGER.info("Emitting ctid state for stream {}, state is {}", pair, ctidStatus); recordCount = 0L; lastCheckpoint = Instant.now(); diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidStateManager.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidStateManager.java index 86915a90be16..1b58a9ae2852 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidStateManager.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidStateManager.java @@ -15,9 +15,8 @@ public abstract class CtidStateManager { public static final long CTID_STATUS_VERSION = 2; public static final String STATE_TYPE_KEY = "state_type"; - public static final String STATE_VER_KEY = "version"; - private final Map pairToCtidStatus; + protected final Map pairToCtidStatus; protected CtidStateManager(final Map pairToCtidStatus) { this.pairToCtidStatus = pairToCtidStatus; @@ -29,10 +28,10 @@ public CtidStatus getCtidStatus(final AirbyteStreamNameNamespacePair pair) { public static boolean validateRelationFileNode(final CtidStatus ctidstatus, final AirbyteStreamNameNamespacePair pair, - final Map fileNodes) { + final FileNodeHandler fileNodeHandler) { - if (fileNodes.containsKey(pair)) { - final Long fileNode = fileNodes.get(pair); + if (fileNodeHandler.hasFileNode(pair)) { + final Long fileNode = fileNodeHandler.getFileNode(pair); return Objects.equals(ctidstatus.getRelationFilenode(), fileNode); } return true; diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidUtils.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidUtils.java index ce4dc2f4ec64..e98d46025d4f 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidUtils.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/CtidUtils.java @@ -5,6 +5,7 @@ package io.airbyte.integrations.source.postgres.ctid; import com.google.common.collect.Sets; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; import io.airbyte.commons.json.Jsons; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; @@ -15,9 +16,14 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class CtidUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(CtidUtils.class); + public static final int POSTGRESQL_VERSION_TID_RANGE_SCAN_CAPABLE = 14; + public static List identifyNewlyAddedStreams(final ConfiguredAirbyteCatalog fullCatalog, final Set alreadySeenStreams, final SyncMode syncMode) { @@ -53,4 +59,20 @@ public record StreamsCategorised (CtidStreams ctidStreams, } + /** + * Postgres servers version 14 and above are capable of running a tid range scan. Used by ctid + * queries + * + * @param database database + * @return true for Tid scan capable server + */ + public static boolean isTidRangeScanCapableDBServer(final JdbcDatabase database) { + try { + return database.getMetaData().getDatabaseMajorVersion() >= POSTGRESQL_VERSION_TID_RANGE_SCAN_CAPABLE; + } catch (final Exception e) { + LOGGER.warn("Failed to get db server version", e); + } + return true; + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/FileNodeHandler.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/FileNodeHandler.java new file mode 100644 index 000000000000..2976e45277aa --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/FileNodeHandler.java @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres.ctid; + +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class FileNodeHandler { + + private final Map fileNodes; + private final List failedToQuery; + + public FileNodeHandler() { + this.fileNodes = new ConcurrentHashMap<>(); + this.failedToQuery = new ArrayList<>(); + } + + public void updateFileNode(final AirbyteStreamNameNamespacePair namespacePair, final Long fileNode) { + fileNodes.put(namespacePair, fileNode); + } + + public boolean hasFileNode(final AirbyteStreamNameNamespacePair namespacePair) { + return fileNodes.containsKey(namespacePair); + } + + public Long getFileNode(final AirbyteStreamNameNamespacePair namespacePair) { + return fileNodes.get(namespacePair); + } + + public void updateFailedToQuery(final io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair namespacePair) { + failedToQuery.add(namespacePair); + } + + public List getFailedToQuery() { + return Collections.unmodifiableList(failedToQuery); + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIterator.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIterator.java new file mode 100644 index 000000000000..19eb239f8682 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIterator.java @@ -0,0 +1,333 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres.ctid; + +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; +import static io.airbyte.integrations.source.postgres.ctid.InitialSyncCtidIteratorConstants.EIGHT_KB; +import static io.airbyte.integrations.source.postgres.ctid.InitialSyncCtidIteratorConstants.GIGABYTE; +import static io.airbyte.integrations.source.postgres.ctid.InitialSyncCtidIteratorConstants.MAX_ALLOWED_RESYNCS; +import static io.airbyte.integrations.source.postgres.ctid.InitialSyncCtidIteratorConstants.QUERY_TARGET_SIZE_GB; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.AbstractIterator; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils; +import io.airbyte.commons.stream.AirbyteStreamUtils; +import io.airbyte.commons.util.AutoCloseableIterator; +import io.airbyte.commons.util.AutoCloseableIterators; +import io.airbyte.integrations.source.postgres.PostgresQueryUtils; +import io.airbyte.integrations.source.postgres.ctid.CtidPostgresSourceOperations.RowDataWithCtid; +import io.airbyte.integrations.source.postgres.internal.models.CtidStatus; +import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; +import java.util.Queue; +import java.util.stream.Stream; +import javax.annotation.CheckForNull; +import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class is responsible to divide the data of the stream into chunks based on the ctid and + * dynamically create iterator and keep processing them one after another. The class also makes sure + * to check for VACUUM in between processing chunks and if VACUUM happens then re-start syncing the + * data + */ +public class InitialSyncCtidIterator extends AbstractIterator implements AutoCloseableIterator { + + private static final Logger LOGGER = LoggerFactory.getLogger(InitialSyncCtidIterator.class); + public static final int MAX_TUPLES_IN_QUERY = 5_000_000; + private final AirbyteStreamNameNamespacePair airbyteStream; + private final long blockSize; + private final List columnNames; + private final CtidStateManager ctidStateManager; + private final JdbcDatabase database; + private final FileNodeHandler fileNodeHandler; + private final String quoteString; + private final String schemaName; + private final CtidPostgresSourceOperations sourceOperations; + private final Queue> subQueriesPlan; + private final String tableName; + private final long tableSize; + private final int maxTuple; + private final boolean useTestPageSize; + + private AutoCloseableIterator currentIterator; + private Long lastKnownFileNode; + private int numberOfTimesReSynced = 0; + private boolean subQueriesInitialized = false; + private final boolean tidRangeScanCapableDBServer; + + public InitialSyncCtidIterator(final CtidStateManager ctidStateManager, + final JdbcDatabase database, + final CtidPostgresSourceOperations sourceOperations, + final String quoteString, + final List columnNames, + final String schemaName, + final String tableName, + final long tableSize, + final long blockSize, + final int maxTuple, + final FileNodeHandler fileNodeHandler, + final boolean tidRangeScanCapableDBServer, + final boolean useTestPageSize) { + this.airbyteStream = AirbyteStreamUtils.convertFromNameAndNamespace(tableName, schemaName); + this.blockSize = blockSize; + this.maxTuple = maxTuple; + this.columnNames = columnNames; + this.ctidStateManager = ctidStateManager; + this.database = database; + this.fileNodeHandler = fileNodeHandler; + this.quoteString = quoteString; + this.schemaName = schemaName; + this.sourceOperations = sourceOperations; + this.subQueriesPlan = new LinkedList<>(); + this.tableName = tableName; + this.tableSize = tableSize; + this.tidRangeScanCapableDBServer = tidRangeScanCapableDBServer; + this.useTestPageSize = useTestPageSize; + } + + @CheckForNull + @Override + protected RowDataWithCtid computeNext() { + try { + if (!subQueriesInitialized) { + initSubQueries(); + subQueriesInitialized = true; + } + + if (currentIterator == null || !currentIterator.hasNext()) { + do { + final Optional mayBeLatestFileNode = PostgresQueryUtils.fileNodeForIndividualStream(database, airbyteStream, quoteString); + if (mayBeLatestFileNode.isPresent()) { + final Long latestFileNode = mayBeLatestFileNode.get(); + if (lastKnownFileNode != null) { + if (!latestFileNode.equals(lastKnownFileNode)) { + resetSubQueries(latestFileNode); + } else { + LOGGER.info("The latest file node {} for stream {} is equal to the last file node {} known to Airbyte.", + latestFileNode, + airbyteStream, + lastKnownFileNode); + } + } + lastKnownFileNode = latestFileNode; + fileNodeHandler.updateFileNode(airbyteStream, latestFileNode); + } else { + LOGGER.warn("Airbyte could not query the latest file node for stream {}. Continuing sync as usual.", airbyteStream); + } + + if (currentIterator != null) { + currentIterator.close(); + } + + if (subQueriesPlan.isEmpty()) { + return endOfData(); + } + + final Pair p = subQueriesPlan.remove(); + currentIterator = AutoCloseableIterators.fromStream(getStream(p), airbyteStream); + } while (!currentIterator.hasNext()); + } + + return currentIterator.next(); + } catch (final Exception e) { + throw new RuntimeException(e); + } + } + + private Stream getStream(final Pair p) throws SQLException { + return database.unsafeQuery( + connection -> getCtidStatement(connection, p.getLeft(), p.getRight()), + sourceOperations::recordWithCtid); + } + + private void initSubQueries() { + if (useTestPageSize) { + LOGGER.warn("Using test page size"); + } + final CtidStatus currentCtidStatus = ctidStateManager.getCtidStatus(airbyteStream); + subQueriesPlan.clear(); + + subQueriesPlan.addAll(getQueryPlan(currentCtidStatus)); + lastKnownFileNode = currentCtidStatus != null ? currentCtidStatus.getRelationFilenode() : null; + } + + private PreparedStatement getCtidStatement(final Connection connection, + final Ctid lowerBound, + final Ctid upperBound) { + final PreparedStatement ctidStatement = tidRangeScanCapableDBServer ? createCtidQueryStatement(connection, lowerBound, upperBound) + : createCtidLegacyQueryStatement(connection, lowerBound, upperBound); + return ctidStatement; + } + + private List> getQueryPlan(final CtidStatus currentCtidStatus) { + final List> queryPlan = tidRangeScanCapableDBServer + ? ctidQueryPlan((currentCtidStatus == null) ? Ctid.ZERO : Ctid.of(currentCtidStatus.getCtid()), + tableSize, blockSize, QUERY_TARGET_SIZE_GB, useTestPageSize ? EIGHT_KB : GIGABYTE) + : ctidLegacyQueryPlan((currentCtidStatus == null) ? Ctid.ZERO : Ctid.of(currentCtidStatus.getCtid()), + tableSize, blockSize, QUERY_TARGET_SIZE_GB, useTestPageSize ? EIGHT_KB : GIGABYTE, maxTuple); + return queryPlan; + } + + private void resetSubQueries(final Long latestFileNode) { + LOGGER.warn( + "The latest file node {} for stream {} is not equal to the last file node {} known to Airbyte. Airbyte will sync this table from scratch again", + latestFileNode, + airbyteStream, + lastKnownFileNode); + if (numberOfTimesReSynced > MAX_ALLOWED_RESYNCS) { + throw new RuntimeException("Airbyte has tried re-syncing stream " + airbyteStream + " more than " + MAX_ALLOWED_RESYNCS + + " times but VACUUM is still happening in between the sync, Please reach out to the customer to understand their VACUUM frequency."); + } + subQueriesPlan.clear(); + subQueriesPlan.addAll(getQueryPlan(null)); + numberOfTimesReSynced++; + } + + /** + * Builds a plan for subqueries. Each query returning an approximate amount of data. Using + * information about a table size and block (page) size. + * + * @param startCtid starting point + * @param relationSize table size + * @param blockSize page size + * @param chunkSize required amount of data in each partition + * @return a list of ctid that can be used to generate queries. + */ + @VisibleForTesting + static List> ctidQueryPlan(final Ctid startCtid, + final long relationSize, + final long blockSize, + final int chunkSize, + final double dataSize) { + final List> chunks = new ArrayList<>(); + if (blockSize > 0 && chunkSize > 0 && dataSize > 0) { + long lowerBound = startCtid.page; + long upperBound; + final double pages = dataSize / blockSize; + final long eachStep = Math.max((long) pages * chunkSize, 1); + LOGGER.info("Will read {} pages to get {}GB", eachStep, chunkSize); + final long theoreticalLastPage = relationSize / blockSize; + LOGGER.debug("Theoretical last page {}", theoreticalLastPage); + upperBound = lowerBound + eachStep; + + if (upperBound > theoreticalLastPage) { + chunks.add(Pair.of(startCtid, null)); + } else { + chunks.add(Pair.of(Ctid.of(lowerBound, startCtid.tuple), Ctid.of(upperBound, 0))); + while (upperBound < theoreticalLastPage) { + lowerBound = upperBound; + upperBound += eachStep; + chunks.add(Pair.of(Ctid.of(lowerBound, 0), upperBound > theoreticalLastPage ? null : Ctid.of(upperBound, 0))); + } + } + } + // The last pair is (x,y) -> null to indicate an unbounded "WHERE ctid > (x,y)" query. + // The actual last page is approximated. The last subquery will go until the end of table. + return chunks; + } + + static List> ctidLegacyQueryPlan(final Ctid startCtid, + final long relationSize, + final long blockSize, + final int chunkSize, + final double dataSize, + final int tuplesInPage) { + + final List> chunks = new ArrayList<>(); + if (blockSize > 0 && chunkSize > 0 && dataSize > 0 && tuplesInPage > 0) { + // Start reading from one tuple after the last one that was read + final Ctid firstCtid = Ctid.inc(startCtid, tuplesInPage); + long lowerBound = firstCtid.page; + long upperBound; + final double pages = dataSize / blockSize; + // cap each chunk at no more than 5m tuples + final long eachStep = Math.max( + Math.min((long) pages * chunkSize, MAX_TUPLES_IN_QUERY / tuplesInPage), 1); + LOGGER.info("Will read {} pages on each query", eachStep); + final long theoreticalLastPage = relationSize / blockSize; + final long lastPage = (long) ((double) theoreticalLastPage * 1.1); + LOGGER.info("Theoretical last page {}. will read until {}", theoreticalLastPage, lastPage); + upperBound = lowerBound + eachStep; + chunks.add((Pair.of(Ctid.of(lowerBound, firstCtid.tuple), Ctid.of(upperBound, tuplesInPage)))); + while (upperBound < lastPage) { + lowerBound = upperBound + 1; + upperBound += eachStep; + chunks.add(Pair.of(Ctid.of(lowerBound, 1), Ctid.of(upperBound, tuplesInPage))); + } + } + return chunks; + } + + public PreparedStatement createCtidQueryStatement(final Connection connection, + final Ctid lowerBound, + final Ctid upperBound) { + try { + LOGGER.info("Preparing query for table: {}", tableName); + final String fullTableName = getFullyQualifiedTableNameWithQuoting(schemaName, tableName, + quoteString); + final String wrappedColumnNames = RelationalDbQueryUtils.enquoteIdentifierList(columnNames, quoteString); + if (upperBound != null) { + final String sql = "SELECT ctid::text, %s FROM %s WHERE ctid > ?::tid AND ctid <= ?::tid".formatted(wrappedColumnNames, fullTableName); + final PreparedStatement preparedStatement = connection.prepareStatement(sql); + preparedStatement.setObject(1, lowerBound.toString()); + preparedStatement.setObject(2, upperBound.toString()); + LOGGER.info("Executing query for table {}: {} with bindings {} and {}", tableName, sql, lowerBound, upperBound); + return preparedStatement; + } else { + final String sql = "SELECT ctid::text, %s FROM %s WHERE ctid > ?::tid".formatted(wrappedColumnNames, fullTableName); + final PreparedStatement preparedStatement = connection.prepareStatement(sql); + preparedStatement.setObject(1, lowerBound.toString()); + LOGGER.info("Executing query for table {}: {} with binding {}", tableName, sql, lowerBound); + return preparedStatement; + } + } catch (final SQLException e) { + throw new RuntimeException(e); + } + } + + public PreparedStatement createCtidLegacyQueryStatement(final Connection connection, + final Ctid lowerBound, + final Ctid upperBound) { + Preconditions.checkArgument(lowerBound != null, "Lower bound ctid expected"); + Preconditions.checkArgument(upperBound != null, "Upper bound ctid expected"); + try { + LOGGER.info("Preparing query for table: {}", tableName); + final String fullTableName = getFullyQualifiedTableNameWithQuoting(schemaName, tableName, + quoteString); + final String wrappedColumnNames = RelationalDbQueryUtils.enquoteIdentifierList(columnNames, quoteString); + final String sql = + "SELECT ctid::text, %s FROM %s WHERE ctid = ANY (ARRAY (SELECT FORMAT('(%%s,%%s)', page, tuple)::tid tid_addr FROM generate_series(?, ?) as page, generate_series(?,?) as tuple ORDER BY tid_addr))" + .formatted( + wrappedColumnNames, fullTableName); + final PreparedStatement preparedStatement = connection.prepareStatement(sql); + preparedStatement.setLong(1, lowerBound.page); + preparedStatement.setLong(2, upperBound.page); + preparedStatement.setLong(3, lowerBound.tuple); + preparedStatement.setLong(4, upperBound.tuple); + LOGGER.info("Executing query for table {}: {}", tableName, preparedStatement); + return preparedStatement; + } catch (final SQLException e) { + throw new RuntimeException(e); + } + } + + @Override + public void close() throws Exception { + if (currentIterator != null) { + currentIterator.close(); + } + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIteratorConstants.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIteratorConstants.java new file mode 100644 index 000000000000..1cda60d449ba --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIteratorConstants.java @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres.ctid; + +public class InitialSyncCtidIteratorConstants { + + public static final int MAX_ALLOWED_RESYNCS = 5; + public static final int QUERY_TARGET_SIZE_GB = 1; + + private static final double MEGABYTE = Math.pow(1024, 2); + public static final double GIGABYTE = MEGABYTE * 1024; + + /** + * Constants to be used for tests + */ + private static final double ONE_KILOBYTE = 1024; + public static final double EIGHT_KB = ONE_KILOBYTE * 8; + public static final String USE_TEST_CHUNK_SIZE = "use_test_chunk_size"; + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/PostgresCtidHandler.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/PostgresCtidHandler.java index 59e975351247..d8b255ac7185 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/PostgresCtidHandler.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/ctid/PostgresCtidHandler.java @@ -4,21 +4,21 @@ package io.airbyte.integrations.source.postgres.ctid; -import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; +import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_DURATION_PROPERTY; +import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_RECORDS_PROPERTY; +import static io.airbyte.integrations.source.postgres.ctid.InitialSyncCtidIteratorConstants.USE_TEST_CHUNK_SIZE; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.annotations.VisibleForTesting; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.source.relationaldb.DbSourceDiscoverUtil; +import io.airbyte.cdk.integrations.source.relationaldb.TableInfo; import io.airbyte.commons.stream.AirbyteStreamUtils; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.source.postgres.PostgresQueryUtils.TableBlockSize; import io.airbyte.integrations.source.postgres.PostgresType; import io.airbyte.integrations.source.postgres.ctid.CtidPostgresSourceOperations.RowDataWithCtid; import io.airbyte.integrations.source.postgres.internal.models.CtidStatus; -import io.airbyte.integrations.source.relationaldb.DbSourceDiscoverUtil; -import io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils; -import io.airbyte.integrations.source.relationaldb.TableInfo; import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.CommonField; import io.airbyte.protocol.models.v0.AirbyteMessage; @@ -29,18 +29,14 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.SyncMode; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.SQLException; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; -import java.util.stream.Stream; -import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,35 +49,37 @@ public class PostgresCtidHandler { private final CtidPostgresSourceOperations sourceOperations; private final String quoteString; private final CtidStateManager ctidStateManager; - private final Map fileNodes; + private final FileNodeHandler fileNodeHandler; final Map tableBlockSizes; + final Optional> tablesMaxTuple; private final Function streamStateForIncrementalRunSupplier; - private static final int QUERY_TARGET_SIZE_GB = 1; - public static final double MEGABYTE = Math.pow(1024, 2); - public static final double GIGABYTE = MEGABYTE * 1024; + private final boolean tidRangeScanCapableDBServer; public PostgresCtidHandler(final JsonNode config, final JdbcDatabase database, final CtidPostgresSourceOperations sourceOperations, final String quoteString, - final Map fileNodes, + final FileNodeHandler fileNodeHandler, final Map tableBlockSizes, + final Map tablesMaxTuple, final CtidStateManager ctidStateManager, final Function streamStateForIncrementalRunSupplier) { this.config = config; this.database = database; this.sourceOperations = sourceOperations; this.quoteString = quoteString; - this.fileNodes = fileNodes; + this.fileNodeHandler = fileNodeHandler; this.tableBlockSizes = tableBlockSizes; + this.tablesMaxTuple = Optional.ofNullable(tablesMaxTuple); this.ctidStateManager = ctidStateManager; this.streamStateForIncrementalRunSupplier = streamStateForIncrementalRunSupplier; + this.tidRangeScanCapableDBServer = CtidUtils.isTidRangeScanCapableDBServer(database); } - public List> getIncrementalIterators( - final ConfiguredAirbyteCatalog catalog, - final Map>> tableNameToTable, - final Instant emmitedAt) { + public List> getInitialSyncCtidIterator( + final ConfiguredAirbyteCatalog catalog, + final Map>> tableNameToTable, + final Instant emmitedAt) { final List> iteratorList = new ArrayList<>(); for (final ConfiguredAirbyteStream airbyteStream : catalog.getStreams()) { final AirbyteStream stream = airbyteStream.getStream(); @@ -107,7 +105,8 @@ public List> getIncrementalIterators( table.getNameSpace(), table.getName(), tableBlockSizes.get(pair).tableSize(), - tableBlockSizes.get(pair).blockSize()); + tableBlockSizes.get(pair).blockSize(), + tablesMaxTuple.orElseGet(() -> Map.of(pair, -1)).get(pair)); final AutoCloseableIterator recordIterator = getRecordIterator(queryStream, streamName, namespace, emmitedAt.toEpochMilli()); final AutoCloseableIterator recordAndMessageIterator = augmentWithState(recordIterator, pair); @@ -119,107 +118,18 @@ public List> getIncrementalIterators( return iteratorList; } - /** - * Builds a plan for subqueries. Each query returning an approximate amount of data. Using - * information about a table size and block (page) size. - * - * @param startCtid starting point - * @param relationSize table size - * @param blockSize page size - * @param chunkSizeGB required amount of data in each partition - * @return a list of ctid that can be used to generate queries. - */ - @VisibleForTesting - static List> ctidQueryPlan(final Ctid startCtid, final long relationSize, final long blockSize, final int chunkSizeGB) { - final List> chunks = new ArrayList<>(); - long lowerBound = startCtid.page; - long upperBound; - final double oneGigaPages = GIGABYTE / blockSize; - final long eachStep = (long) oneGigaPages * chunkSizeGB; - LOGGER.info("Will read {} pages to get {}GB", eachStep, chunkSizeGB); - final long theoreticalLastPage = relationSize / blockSize; - LOGGER.debug("Theoretical last page {}", theoreticalLastPage); - upperBound = lowerBound + eachStep; - - if (upperBound > theoreticalLastPage) { - chunks.add(Pair.of(startCtid, null)); - } else { - chunks.add(Pair.of(Ctid.of(lowerBound, startCtid.tuple), Ctid.of(upperBound, 0))); - while (upperBound < theoreticalLastPage) { - lowerBound = upperBound; - upperBound += eachStep; - chunks.add(Pair.of(Ctid.of(lowerBound, 0), upperBound > theoreticalLastPage ? null : Ctid.of(upperBound, 0))); - } - } - // The last pair is (x,y) -> null to indicate an unbounded "WHERE ctid > (x,y)" query. - // The actual last page is approximated. The last subquery will go until the end of table. - return chunks; - } - private AutoCloseableIterator queryTableCtid( final List columnNames, final String schemaName, final String tableName, final long tableSize, - final long blockSize) { + final long blockSize, + final int maxTuple) { LOGGER.info("Queueing query for table: {}", tableName); - final AirbyteStreamNameNamespacePair airbyteStream = - AirbyteStreamUtils.convertFromNameAndNamespace(tableName, schemaName); - - final CtidStatus currentCtidStatus = ctidStateManager.getCtidStatus(airbyteStream); - - // Rather than trying to read an entire table with a "WHERE ctid > (0,0)" query, - // We are creating a list of lazy iterators each holding a subquery according to the plan. - // All subqueries are then composed in a single composite iterator. - // Because list consists of lazy iterators, the query is only executing when needed one after the - // other. - final List> subQueriesPlan = - ctidQueryPlan((currentCtidStatus == null) ? Ctid.of(0, 0) : Ctid.of(currentCtidStatus.getCtid()), tableSize, blockSize, QUERY_TARGET_SIZE_GB); - final List> subQueriesIterators = new ArrayList<>(); - subQueriesPlan.forEach(p -> subQueriesIterators.add(AutoCloseableIterators.lazyIterator(() -> { - try { - final Stream stream = database.unsafeQuery( - connection -> createCtidQueryStatement(connection, columnNames, schemaName, tableName, p.getLeft(), p.getRight()), - sourceOperations::recordWithCtid); - - return AutoCloseableIterators.fromStream(stream, airbyteStream); - } catch (final SQLException e) { - throw new RuntimeException(e); - } - }, airbyteStream))); - return AutoCloseableIterators.concatWithEagerClose(subQueriesIterators); - } - - private PreparedStatement createCtidQueryStatement( - final Connection connection, - final List columnNames, - final String schemaName, - final String tableName, - final Ctid lowerBound, - final Ctid upperBound) { - try { - LOGGER.info("Preparing query for table: {}", tableName); - final String fullTableName = getFullyQualifiedTableNameWithQuoting(schemaName, tableName, - quoteString); - final String wrappedColumnNames = RelationalDbQueryUtils.enquoteIdentifierList(columnNames, quoteString); - if (upperBound != null) { - final String sql = "SELECT ctid::text, %s FROM %s WHERE ctid > ?::tid AND ctid <= ?::tid".formatted(wrappedColumnNames, fullTableName); - final PreparedStatement preparedStatement = connection.prepareStatement(sql); - preparedStatement.setObject(1, lowerBound.toString()); - preparedStatement.setObject(2, upperBound.toString()); - LOGGER.info("Executing query for table {}: {}", tableName, preparedStatement); - return preparedStatement; - } else { - final String sql = "SELECT ctid::text, %s FROM %s WHERE ctid > ?::tid".formatted(wrappedColumnNames, fullTableName); - final PreparedStatement preparedStatement = connection.prepareStatement(sql); - preparedStatement.setObject(1, lowerBound.toString()); - LOGGER.info("Executing query for table {}: {}", tableName, preparedStatement); - return preparedStatement; - } - } catch (final SQLException e) { - throw new RuntimeException(e); - } + return new InitialSyncCtidIterator(ctidStateManager, database, sourceOperations, quoteString, columnNames, schemaName, tableName, tableSize, + blockSize, maxTuple, fileNodeHandler, tidRangeScanCapableDBServer, + config.has(USE_TEST_CHUNK_SIZE) && config.get(USE_TEST_CHUNK_SIZE).asBoolean()); } // Transforms the given iterator to create an {@link AirbyteRecordMessage} @@ -261,17 +171,14 @@ private AutoCloseableIterator augmentWithState(final AutoCloseab final JsonNode incrementalState = (currentCtidStatus == null || currentCtidStatus.getIncrementalState() == null) ? streamStateForIncrementalRunSupplier.apply(pair) : currentCtidStatus.getIncrementalState(); - final Long latestFileNode = fileNodes.get(pair); - assert latestFileNode != null; - final Duration syncCheckpointDuration = - config.get("sync_checkpoint_seconds") != null ? Duration.ofSeconds(config.get("sync_checkpoint_seconds").asLong()) + config.get(SYNC_CHECKPOINT_DURATION_PROPERTY) != null ? Duration.ofSeconds(config.get(SYNC_CHECKPOINT_DURATION_PROPERTY).asLong()) : CtidStateIterator.SYNC_CHECKPOINT_DURATION; - final Long syncCheckpointRecords = config.get("sync_checkpoint_records") != null ? config.get("sync_checkpoint_records").asLong() + final Long syncCheckpointRecords = config.get(SYNC_CHECKPOINT_RECORDS_PROPERTY) != null ? config.get(SYNC_CHECKPOINT_RECORDS_PROPERTY).asLong() : CtidStateIterator.SYNC_CHECKPOINT_RECORDS; return AutoCloseableIterators.transformIterator( - r -> new CtidStateIterator(r, pair, latestFileNode, ctidStateManager, incrementalState, + r -> new CtidStateIterator(r, pair, fileNodeHandler, ctidStateManager, incrementalState, syncCheckpointDuration, syncCheckpointRecords), recordIterator, pair); } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cursor_based/CursorBasedCtidUtils.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cursor_based/CursorBasedCtidUtils.java index 2b8b65e5994d..03f74f1558c9 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cursor_based/CursorBasedCtidUtils.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cursor_based/CursorBasedCtidUtils.java @@ -9,10 +9,10 @@ import static io.airbyte.integrations.source.postgres.ctid.CtidUtils.identifyNewlyAddedStreams; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; import io.airbyte.integrations.source.postgres.ctid.CtidUtils.CtidStreams; import io.airbyte.integrations.source.postgres.ctid.CtidUtils.StreamsCategorised; import io.airbyte.integrations.source.postgres.internal.models.InternalModels.StateType; -import io.airbyte.integrations.source.relationaldb.state.StateManager; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; @@ -100,7 +100,7 @@ public record CursorBasedStreams(List streamsForCursorB * @param streamPair stream to reclassify */ public static void reclassifyCategorisedCtidStream(final StreamsCategorised categorisedStreams, - AirbyteStreamNameNamespacePair streamPair) { + final AirbyteStreamNameNamespacePair streamPair) { final Optional foundStream = categorisedStreams .ctidStreams() .streamsForCtidSync().stream().filter(c -> Objects.equals( @@ -128,7 +128,7 @@ public static void reclassifyCategorisedCtidStream(final StreamsCategorised categorisedStreams, - List streamPairs) { + final List streamPairs) { streamPairs.forEach(c -> reclassifyCategorisedCtidStream(categorisedStreams, c)); } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cursor_based/PostgresCursorBasedStateManager.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cursor_based/PostgresCursorBasedStateManager.java index f490d51cce5d..719dc0e4b505 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cursor_based/PostgresCursorBasedStateManager.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/cursor_based/PostgresCursorBasedStateManager.java @@ -5,12 +5,12 @@ package io.airbyte.integrations.source.postgres.cursor_based; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.source.relationaldb.CursorInfo; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbState; +import io.airbyte.cdk.integrations.source.relationaldb.state.StreamStateManager; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.postgres.internal.models.CursorBasedStatus; import io.airbyte.integrations.source.postgres.internal.models.InternalModels.StateType; -import io.airbyte.integrations.source.relationaldb.CursorInfo; -import io.airbyte.integrations.source.relationaldb.models.DbState; -import io.airbyte.integrations.source.relationaldb.state.StreamStateManager; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/xmin/PostgresXminHandler.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/xmin/PostgresXminHandler.java index 5c2a48c5a74f..d58a0d07a2e8 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/xmin/PostgresXminHandler.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/xmin/PostgresXminHandler.java @@ -4,20 +4,20 @@ package io.airbyte.integrations.source.postgres.xmin; -import static io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; +import static io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils.getFullyQualifiedTableNameWithQuoting; import com.fasterxml.jackson.databind.JsonNode; import com.google.common.annotations.VisibleForTesting; +import io.airbyte.cdk.db.JdbcCompatibleSourceOperations; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.source.relationaldb.DbSourceDiscoverUtil; +import io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils; +import io.airbyte.cdk.integrations.source.relationaldb.TableInfo; import io.airbyte.commons.stream.AirbyteStreamUtils; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.db.JdbcCompatibleSourceOperations; -import io.airbyte.db.jdbc.JdbcDatabase; import io.airbyte.integrations.source.postgres.PostgresType; import io.airbyte.integrations.source.postgres.internal.models.XminStatus; -import io.airbyte.integrations.source.relationaldb.DbSourceDiscoverUtil; -import io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils; -import io.airbyte.integrations.source.relationaldb.TableInfo; import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.CommonField; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/xmin/XminCtidUtils.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/xmin/XminCtidUtils.java index 9316376a5bbe..5ed628b9e3d9 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/xmin/XminCtidUtils.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/xmin/XminCtidUtils.java @@ -9,11 +9,11 @@ import static io.airbyte.integrations.source.postgres.xmin.PostgresXminHandler.shouldPerformFullSync; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.postgres.ctid.CtidUtils.CtidStreams; import io.airbyte.integrations.source.postgres.ctid.CtidUtils.StreamsCategorised; import io.airbyte.integrations.source.postgres.internal.models.XminStatus; -import io.airbyte.integrations.source.relationaldb.state.StateManager; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; @@ -106,7 +106,7 @@ public record XminStreams(List streamsForXminSync, } public static void reclassifyCategorisedCtidStream(final StreamsCategorised categorisedStreams, - AirbyteStreamNameNamespacePair streamPair) { + final AirbyteStreamNameNamespacePair streamPair) { final Optional foundStream = categorisedStreams .ctidStreams() .streamsForCtidSync().stream().filter(c -> Objects.equals( @@ -134,7 +134,7 @@ public static void reclassifyCategorisedCtidStream(final StreamsCategorised categorisedStreams, - List streamPairs) { + final List streamPairs) { streamPairs.forEach(c -> reclassifyCategorisedCtidStream(categorisedStreams, c)); } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/xmin/XminStateIterator.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/xmin/XminStateIterator.java index b3c294ed4fce..9b3a31da7067 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/xmin/XminStateIterator.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/xmin/XminStateIterator.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.source.postgres.xmin; -import autovalue.shaded.com.google.common.collect.AbstractIterator; +import com.google.common.collect.AbstractIterator; import io.airbyte.integrations.source.postgres.internal.models.XminStatus; import io.airbyte.protocol.models.AirbyteStreamNameNamespacePair; import io.airbyte.protocol.models.v0.AirbyteMessage; diff --git a/airbyte-integrations/connectors/source-postgres/src/main/resources/internal_models/internal_models.yaml b/airbyte-integrations/connectors/source-postgres/src/main/resources/internal_models/internal_models.yaml index d46c74c24f4d..f3f16fa67669 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/resources/internal_models/internal_models.yaml +++ b/airbyte-integrations/connectors/source-postgres/src/main/resources/internal_models/internal_models.yaml @@ -24,7 +24,7 @@ definitions: type: object extends: type: object - existingJavaType: "io.airbyte.integrations.source.relationaldb.models.DbStreamState" + existingJavaType: "io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState" properties: state_type: "$ref": "#/definitions/StateType" diff --git a/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json b/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json index bf749864f76c..f00b42a5507a 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-postgres/src/main/resources/spec.json @@ -223,26 +223,16 @@ }, "replication_method": { "type": "object", - "title": "Replication Method", - "description": "Replication method for extracting data from the database.", + "title": "Update Method", + "description": "Configures how data is extracted from the database.", "order": 9, "group": "advanced", + "default": "CDC", + "display_type": "radio", "oneOf": [ { - "title": "Standard (Xmin)", - "description": "Xmin replication requires no setup on the DB side but will not be able to represent deletions incrementally.", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "Xmin", - "order": 0 - } - } - }, - { - "title": "Logical Replication (CDC)", - "description": "Logical replication uses the Postgres write-ahead log (WAL) to detect inserts, updates, and deletes. This needs to be configured on the source database itself. Only available on Postgres 10 and above. Read the docs.", + "title": "Read Changes using Write-Ahead Log (CDC)", + "description": "Recommended - Incrementally reads new inserts, updates, and deletes using the Postgres write-ahead log (WAL). This needs to be configured on the source database itself. Recommended for tables of any size.", "required": ["method", "replication_slot", "publication"], "additionalProperties": true, "properties": { @@ -274,11 +264,11 @@ "initial_waiting_seconds": { "type": "integer", "title": "Initial Waiting Time in Seconds (Advanced)", - "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", - "default": 300, + "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 1200 seconds. Valid range: 120 seconds to 2400 seconds. Read about initial waiting time.", + "default": 1200, "order": 5, "min": 120, - "max": 1200 + "max": 2400 }, "queue_size": { "type": "integer", @@ -292,7 +282,7 @@ "lsn_commit_behaviour": { "type": "string", "title": "LSN commit behaviour", - "description": "Determines when Airbtye should flush the LSN of processed WAL logs in the source database. `After loading Data in the destination` is default. If `While reading Data` is selected, in case of a downstream failure (while loading data into the destination), next sync would result in a full sync.", + "description": "Determines when Airbyte should flush the LSN of processed WAL logs in the source database. `After loading Data in the destination` is default. If `While reading Data` is selected, in case of a downstream failure (while loading data into the destination), next sync would result in a full sync.", "enum": [ "While reading Data", "After loading Data in the destination" @@ -303,8 +293,20 @@ } }, { - "title": "Standard", - "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", + "title": "Detect Changes with Xmin System Column", + "description": "Recommended - Incrementally reads new inserts and updates via Postgres Xmin system column. Only recommended for tables up to 500GB.", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "Xmin", + "order": 0 + } + } + }, + { + "title": "Scan Changes with User Defined Cursor", + "description": "Incrementally detects new inserts and updates using the cursor column chosen when configuring a connection (e.g. created_at, updated_at).", "required": ["method"], "properties": { "method": { diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractCdcPostgresSourceSslAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractCdcPostgresSourceSslAcceptanceTest.java index 2a96d65d1548..3d198ee5f4b8 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractCdcPostgresSourceSslAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractCdcPostgresSourceSslAcceptanceTest.java @@ -4,79 +4,39 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; -import static io.airbyte.db.PostgresUtils.getCertificate; - import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.PostgresUtils; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.integrations.util.HostPortResolver; -import java.util.List; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.utility.DockerImageName; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.ContainerModifier; +import java.util.Map; public abstract class AbstractCdcPostgresSourceSslAcceptanceTest extends CdcPostgresSourceAcceptanceTest { protected static final String PASSWORD = "Passw0rd"; - protected static PostgresUtils.Certificate certs; @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - container = new PostgreSQLContainer<>(DockerImageName.parse("postgres:bullseye") - .asCompatibleSubstituteFor("postgres")) - .withCommand("postgres -c wal_level=logical"); - container.start(); - - certs = getCertificate(container); - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "CDC") - .put("replication_slot", SLOT_NAME_BASE) - .put("publication", PUBLICATION) - .put("initial_waiting_seconds", INITIAL_WAITING_SECONDS) - .build()); - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(container)) - .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(container)) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.SCHEMAS_KEY, List.of(NAMESPACE)) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put("replication_method", replicationMethod) - .put(JdbcUtils.SSL_KEY, true) - .put("ssl_mode", getCertificateConfiguration()) - .put("is_test", true) - .build()); - - try (final DSLContext dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), - container.getHost(), - container.getFirstMappedPort(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.POSTGRES)) { - final Database database = new Database(dslContext); + testdb = PostgresTestDatabase.in(getServerImage(), ContainerModifier.WAL_LEVEL_LOGICAL, ContainerModifier.CERT) + .with("CREATE TABLE id_and_name(id INTEGER primary key, name VARCHAR(200));") + .with("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');") + .with("CREATE TABLE starships(id INTEGER primary key, name VARCHAR(200));") + .with("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');") + .withReplicationSlot() + .withPublicationForAllTables(); + } - database.query(ctx -> { - ctx.execute("CREATE TABLE id_and_name(id INTEGER primary key, name VARCHAR(200));"); - ctx.execute("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.execute("CREATE TABLE starships(id INTEGER primary key, name VARCHAR(200));"); - ctx.execute("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - ctx.execute("SELECT pg_create_logical_replication_slot('" + SLOT_NAME_BASE + "', 'pgoutput');"); - ctx.execute("CREATE PUBLICATION " + PUBLICATION + " FOR ALL TABLES;"); - return null; - }); - } + @Override + protected JsonNode getConfig() { + return testdb.integrationTestConfigBuilder() + .withSchemas(NAMESPACE) + .withSsl(getCertificateConfiguration()) + .withCdcReplication() + .build(); } - public abstract ImmutableMap getCertificateConfiguration(); + protected abstract BaseImage getServerImage(); + + public abstract Map getCertificateConfiguration(); } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceAcceptanceTest.java index 5fdb3bed18e8..e69a0cce10a1 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceAcceptanceTest.java @@ -4,8 +4,8 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.base.ssh.SshHelpers; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.protocol.models.v0.ConnectorSpecification; import java.util.Optional; diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java index d7dbe4e02ad8..a87c4395785e 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceDatatypeTest.java @@ -10,20 +10,18 @@ import static io.airbyte.protocol.models.JsonSchemaType.STRING_TIME_WITHOUT_TIMEZONE; import static io.airbyte.protocol.models.JsonSchemaType.STRING_TIME_WITH_TIMEZONE; -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; -import io.airbyte.integrations.standardtest.source.TestDataHolder; +import io.airbyte.cdk.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDataHolder; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase; import io.airbyte.protocol.models.JsonSchemaPrimitiveUtil.JsonSchemaPrimitive; import io.airbyte.protocol.models.JsonSchemaType; import java.util.Set; -import org.jooq.DSLContext; -import org.testcontainers.containers.PostgreSQLContainer; public abstract class AbstractPostgresSourceDatatypeTest extends AbstractSourceDatabaseTypeTest { - protected PostgreSQLContainer container; - protected JsonNode config; - protected DSLContext dslContext; + protected PostgresTestDatabase testdb; + protected static final String SCHEMA_NAME = "test"; @Override @@ -37,8 +35,8 @@ protected String getImageName() { } @Override - protected JsonNode getConfig() { - return config; + protected void tearDown(final TestDestinationEnv testEnv) { + testdb.close(); } @Override @@ -189,8 +187,8 @@ protected void initTests() { .sourceType("date") .fullSourceDataType(type) .airbyteType(JsonSchemaType.STRING_DATE) - .addInsertValues("'1999-01-08'", "'1991-02-10 BC'", "'2022/11/12'", "'1987.12.01'") - .addExpectedValues("1999-01-08", "1991-02-10 BC", "2022-11-12", "1987-12-01") + .addInsertValues("'1999-01-08'", "'1991-02-10 BC'", "'2022/11/12'", "'1987.12.01'", "'-InFinITy'", "'InFinITy'") + .addExpectedValues("1999-01-08", "1991-02-10 BC", "2022-11-12", "1987-12-01", "-Infinity", "Infinity") .build()); } @@ -202,20 +200,6 @@ protected void initTests() { .addExpectedValues((String) null) .build()); - for (final String type : Set.of("double precision", "float", "float8")) { - addDataTypeTestData( - TestDataHolder.builder() - .sourceType(type) - .airbyteType(JsonSchemaType.NUMBER) - .addInsertValues("'123'", "'1234567890.1234567'", "null") - // Postgres source does not support these special values yet - // https://github.com/airbytehq/airbyte/issues/8902 - // "'-Infinity'", "'Infinity'", "'NaN'", "null") - .addExpectedValues("123.0", "1.2345678901234567E9", null) - // "-Infinity", "Infinity", "NaN", null) - .build()); - } - addDataTypeTestData( TestDataHolder.builder() .sourceType("inet") @@ -315,43 +299,38 @@ protected void initTests() { .addExpectedValues("33.345") .build()); - // case of a column type being a NUMERIC data type - // with precision but no decimal + // Verify that large integers are not deserialized into scientific notation addDataTypeTestData( TestDataHolder.builder() .sourceType("numeric") - .fullSourceDataType("NUMERIC(38)") .airbyteType(JsonSchemaType.INTEGER) - .addInsertValues("'33'") - .addExpectedValues("33") + .fullSourceDataType("NUMERIC(38)") + .addInsertValues("'70000'", "'853245'", "'900000000'") + .addExpectedValues("70000", "853245", "900000000") .build()); + // case of a column type being a NUMERIC data type + // with precision but no decimal addDataTypeTestData( TestDataHolder.builder() .sourceType("numeric") - .fullSourceDataType("NUMERIC(28,2)") - .airbyteType(JsonSchemaType.NUMBER) - .addInsertValues( - "'123'", "null", "'14525.22'") - // Postgres source does not support these special values yet - // https://github.com/airbytehq/airbyte/issues/8902 - // "'infinity'", "'-infinity'", "'nan'" - .addExpectedValues("123", null, "14525.22") + .fullSourceDataType("NUMERIC(38,0)") + .airbyteType(JsonSchemaType.INTEGER) + .addInsertValues("'33'", "'123'") + .addExpectedValues("33", "123") .build()); - // Blocked by https://github.com/airbytehq/airbyte/issues/8902 - for (final String type : Set.of("numeric", "decimal")) { + for (final String type : Set.of("double precision", "float", "float8")) { addDataTypeTestData( TestDataHolder.builder() .sourceType(type) - .fullSourceDataType("NUMERIC(20,7)") .airbyteType(JsonSchemaType.NUMBER) - .addInsertValues( - "'123'", "null", "'1234567890.1234567'") + .addInsertValues("'123'", "'1234567890.1234567'", "null") // Postgres source does not support these special values yet // https://github.com/airbytehq/airbyte/issues/8902 - // "'infinity'", "'-infinity'", "'nan'" - .addExpectedValues("123", null, "1.2345678901234567E9") + // "'-Infinity'", "'Infinity'", "'NaN'", "null") + .addExpectedValues("123.0", "1.2345678901234567E9", null) + // "-Infinity", "Infinity", "NaN", null) .build()); } @@ -473,14 +452,16 @@ protected void initTests() { "TIMESTAMP '0001-01-01 00:00:00.000000'", // The last possible timestamp in BCE "TIMESTAMP '0001-12-31 23:59:59.999999 BC'", - "'epoch'") + "'epoch'", + "'-InFinITy'", "'InFinITy'") .addExpectedValues( "2004-10-19T10:23:00.000000", "2004-10-19T10:23:54.123456", "3004-10-19T10:23:54.123456 BC", "0001-01-01T00:00:00.000000", "0001-12-31T23:59:59.999999 BC", - "1970-01-01T00:00:00.000000") + "1970-01-01T00:00:00.000000", + "-Infinity", "Infinity") .build()); } @@ -496,8 +477,6 @@ protected void initTests() { .build()); } - addTimestampWithInfinityValuesTest(); - // timestamp with time zone for (final String fullSourceType : Set.of("timestamptz", "timestamp with time zone")) { addDataTypeTestData( @@ -516,14 +495,14 @@ protected void initTests() { "TIMESTAMP WITH TIME ZONE '0001-12-31 16:00:00.000000-08 BC'", // The last possible timestamp in BCE (15:59-08 == 23:59Z) "TIMESTAMP WITH TIME ZONE '0001-12-31 15:59:59.999999-08 BC'", - "null") + "null", "'-InFinITy'", "'InFinITy'") .addExpectedValues( "2004-10-19T18:23:00.000000Z", "2004-10-19T18:23:54.123456Z", "3004-10-19T18:23:54.123456Z BC", "0001-01-01T00:00:00.000000Z", "0001-12-31T23:59:59.999999Z BC", - null) + null, "-Infinity", "Infinity") .build()); } @@ -592,6 +571,7 @@ protected void initTests() { addTimeWithTimeZoneTest(); addArraysTestData(); addMoneyTest(); + addNumericValuesTest(); } protected void addHstoreTest() { @@ -652,24 +632,6 @@ protected void addTimeWithTimeZoneTest() { } } - protected void addTimestampWithInfinityValuesTest() { - // timestamp without time zone - for (final String fullSourceType : Set.of("timestamp", "timestamp without time zone", "timestamp without time zone not null default now()")) { - addDataTypeTestData( - TestDataHolder.builder() - .sourceType("timestamp") - .fullSourceDataType(fullSourceType) - .airbyteType(JsonSchemaType.STRING_TIMESTAMP_WITHOUT_TIMEZONE) - .addInsertValues( - "'infinity'", - "'-infinity'") - .addExpectedValues( - "+292278994-08-16T23:00:00.000000", - "+292269055-12-02T23:00:00.000000 BC") - .build()); - } - } - private void addArraysTestData() { addDataTypeTestData( TestDataHolder.builder() @@ -847,6 +809,17 @@ private void addArraysTestData() { .addExpectedValues("[131070.237689,231072.476596593]") .build()); + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("jsonb_array") + .fullSourceDataType("JSONB[]") + .airbyteType(JsonSchemaType.builder(JsonSchemaPrimitive.ARRAY) + .withItems(JsonSchemaType.builder(JsonSchemaPrimitive.STRING).build()) + .build()) + .addInsertValues("ARRAY['{\"foo\":\"bar\"}'::JSONB, NULL]") + .addExpectedValues("[\"{\\\"foo\\\": \\\"bar\\\"}\",null]") + .build()); + addDataTypeTestData( TestDataHolder.builder() .sourceType("money_array") @@ -955,4 +928,35 @@ private void addArraysTestData() { .build()); } + protected void addNumericValuesTest() { + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("numeric") + .fullSourceDataType("NUMERIC(28,2)") + .airbyteType(JsonSchemaType.NUMBER) + .addInsertValues( + "'123'", "null", "'14525.22'") + // Postgres source does not support these special values yet + // https://github.com/airbytehq/airbyte/issues/8902 + // "'infinity'", "'-infinity'", "'nan'" + .addExpectedValues("123.0", null, "14525.22") + .build()); + + // Blocked by https://github.com/airbytehq/airbyte/issues/8902 + for (final String type : Set.of("numeric", "decimal")) { + addDataTypeTestData( + TestDataHolder.builder() + .sourceType(type) + .fullSourceDataType("NUMERIC(20,7)") + .airbyteType(JsonSchemaType.NUMBER) + .addInsertValues( + "'123'", "null", "'1234567890.1234567'") + // Postgres source does not support these special values yet + // https://github.com/airbytehq/airbyte/issues/8902 + // "'infinity'", "'-infinity'", "'nan'" + .addExpectedValues("123.0", null, "1.2345678901234567E9") + .build()); + } + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceSSLCertificateAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceSSLCertificateAcceptanceTest.java index 31493c1f1d64..26f6f319f1df 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceSSLCertificateAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractPostgresSourceSSLCertificateAcceptanceTest.java @@ -4,19 +4,13 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; -import static io.airbyte.db.PostgresUtils.getCertificate; - import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import io.airbyte.commons.features.EnvVariableFeatureFlags; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.PostgresUtils; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.integrations.util.HostPortResolver; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.ContainerModifier; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.CatalogHelpers; @@ -26,85 +20,42 @@ import io.airbyte.protocol.models.v0.SyncMode; import java.util.HashMap; import java.util.List; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.utility.DockerImageName; -import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; -import uk.org.webcompere.systemstubs.jupiter.SystemStub; -import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; +import java.util.Map; -@ExtendWith(SystemStubsExtension.class) public abstract class AbstractPostgresSourceSSLCertificateAcceptanceTest extends AbstractPostgresSourceAcceptanceTest { private static final String STREAM_NAME = "id_and_name"; private static final String STREAM_NAME2 = "starships"; private static final String STREAM_NAME_MATERIALIZED_VIEW = "testview"; private static final String SCHEMA_NAME = "public"; - @SystemStub - private EnvironmentVariables environmentVariables; - private PostgreSQLContainer container; - private JsonNode config; protected static final String PASSWORD = "Passw0rd"; - protected static PostgresUtils.Certificate certs; + + protected PostgresTestDatabase testdb; @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - - container = new PostgreSQLContainer<>(DockerImageName.parse("postgres:bullseye") - .asCompatibleSubstituteFor("postgres")); - container.start(); - certs = getCertificate(container); - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "Standard") - .build()); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put("host", HostPortResolver.resolveHost(container)) - .put("port", HostPortResolver.resolvePort(container)) - .put("database", container.getDatabaseName()) - .put("schemas", Jsons.jsonNode(List.of("public"))) - .put("username", "postgres") - .put("password", "postgres") - .put("ssl", true) - .put("replication_method", replicationMethod) - .put("ssl_mode", getCertificateConfiguration()) - .build()); - - try (final DSLContext dslContext = DSLContextFactory.create( - config.get("username").asText(), - config.get("password").asText(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), - container.getHost(), - container.getFirstMappedPort(), - config.get("database").asText()), - SQLDialect.POSTGRES)) { - final Database database = new Database(dslContext); - - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - ctx.fetch("CREATE MATERIALIZED VIEW testview AS select * from id_and_name where id = '2';"); - return null; - }); - } + testdb = PostgresTestDatabase.in(BaseImage.POSTGRES_16, ContainerModifier.CERT) + .with("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));") + .with("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');") + .with("CREATE TABLE starships(id INTEGER, name VARCHAR(200));") + .with("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');") + .with("CREATE MATERIALIZED VIEW testview AS select * from id_and_name where id = '2';"); } - public abstract ImmutableMap getCertificateConfiguration(); + public abstract Map getCertificateConfiguration(); @Override protected void tearDown(final TestDestinationEnv testEnv) { - container.close(); + testdb.close(); } @Override protected JsonNode getConfig() { - return config; + return testdb.integrationTestConfigBuilder() + .withSchemas("public") + .withStandardReplication() + .withSsl(getCertificateConfiguration()) + .build(); } @Override @@ -147,9 +98,4 @@ protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); } - @Override - protected boolean supportsPerStream() { - return true; - } - } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java index 0d735dd45340..db8552a90e09 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java @@ -6,16 +6,18 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; -import io.airbyte.commons.features.EnvVariableFeatureFlags; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.ssh.SshBastionContainer; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.ssh.SshBastionContainer; -import io.airbyte.integrations.base.ssh.SshTunnel; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.ContainerModifier; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.CatalogHelpers; @@ -23,31 +25,27 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; import io.airbyte.protocol.models.v0.SyncMode; +import java.io.IOException; +import java.io.UncheckedIOException; import java.util.HashMap; import java.util.List; import org.jooq.SQLDialect; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testcontainers.containers.Network; -import org.testcontainers.containers.PostgreSQLContainer; -import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; -import uk.org.webcompere.systemstubs.jupiter.SystemStub; -import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; -@ExtendWith(SystemStubsExtension.class) public abstract class AbstractSshPostgresSourceAcceptanceTest extends AbstractPostgresSourceAcceptanceTest { private static final String STREAM_NAME = "id_and_name"; private static final String STREAM_NAME2 = "starships"; private static final String SCHEMA_NAME = "public"; - @SystemStub - private EnvironmentVariables environmentVariables; - private static final Network network = Network.newNetwork(); - private static JsonNode config; + private final SshBastionContainer bastion = new SshBastionContainer(); - private PostgreSQLContainer db; + private PostgresTestDatabase testdb; private void populateDatabaseTestData() throws Exception { - final var outerConfig = bastion.getTunnelConfig(getTunnelMethod(), bastion.getBasicDbConfigBuider(db, List.of("public")), false); + final var outerConfig = testdb.integrationTestConfigBuilder() + .withSchemas("public") + .withoutSsl() + .with("tunnel_method", bastion.getTunnelMethod(getTunnelMethod(), false)) + .build(); SshTunnel.sshWrap( outerConfig, JdbcUtils.HOST_LIST_KEY, @@ -81,31 +79,29 @@ private static Database getDatabaseFromConfig(final JsonNode config) { // requiring data to already be in place. @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - startTestContainers(); - config = bastion.getTunnelConfig(getTunnelMethod(), bastion.getBasicDbConfigBuider(db, List.of("public")), true); + testdb = PostgresTestDatabase.in(BaseImage.POSTGRES_16, ContainerModifier.NETWORK); + bastion.initAndStartBastion(testdb.getContainer().getNetwork()); populateDatabaseTestData(); - - } - - private void startTestContainers() { - bastion.initAndStartBastion(network); - initAndStartJdbcContainer(); - } - - private void initAndStartJdbcContainer() { - db = new PostgreSQLContainer<>("postgres:13-alpine").withNetwork(network); - db.start(); } @Override protected void tearDown(final TestDestinationEnv testEnv) { - bastion.stopAndCloseContainers(db); + bastion.stopAndClose(); } @Override protected JsonNode getConfig() { - return config; + try { + return testdb.integrationTestConfigBuilder() + .withSchemas("public") + .withoutSsl() + .with("tunnel_method", bastion.getTunnelMethod(getTunnelMethod(), true)) + .build(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } } @Override @@ -138,9 +134,4 @@ protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); } - @Override - protected boolean supportsPerStream() { - return true; - } - } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceCaCertificateSslAcceptanceLegacyCtidTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceCaCertificateSslAcceptanceLegacyCtidTest.java new file mode 100644 index 000000000000..612778f616cd --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceCaCertificateSslAcceptanceLegacyCtidTest.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; + +public class CDCPostgresSourceCaCertificateSslAcceptanceLegacyCtidTest extends CDCPostgresSourceCaCertificateSslAcceptanceTest { + + @Override + protected BaseImage getServerImage() { + return BaseImage.POSTGRES_12; + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceCaCertificateSslAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceCaCertificateSslAcceptanceTest.java index 909d62469a1d..5ff4ba1e4e6d 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceCaCertificateSslAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceCaCertificateSslAcceptanceTest.java @@ -5,15 +5,22 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; import com.google.common.collect.ImmutableMap; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; +import java.util.Map; public class CDCPostgresSourceCaCertificateSslAcceptanceTest extends AbstractCdcPostgresSourceSslAcceptanceTest { - public ImmutableMap getCertificateConfiguration() { + public Map getCertificateConfiguration() { return ImmutableMap.builder() .put("mode", "verify-ca") - .put("ca_certificate", certs.getCaCertificate()) + .put("ca_certificate", testdb.getCertificates().caCertificate()) .put("client_key_password", PASSWORD) .build(); } + @Override + protected BaseImage getServerImage() { + return BaseImage.POSTGRES_16; + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceFullCertificateSslAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceFullCertificateSslAcceptanceTest.java index dda783f767ba..65d8f08e68d2 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceFullCertificateSslAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CDCPostgresSourceFullCertificateSslAcceptanceTest.java @@ -5,18 +5,26 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; import com.google.common.collect.ImmutableMap; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; +import java.util.Map; public class CDCPostgresSourceFullCertificateSslAcceptanceTest extends AbstractCdcPostgresSourceSslAcceptanceTest { @Override - public ImmutableMap getCertificateConfiguration() { + public Map getCertificateConfiguration() { + final var certs = testdb.getCertificates(); return ImmutableMap.builder() .put("mode", "verify-ca") - .put("ca_certificate", certs.getCaCertificate()) - .put("client_certificate", certs.getClientCertificate()) - .put("client_key", certs.getClientKey()) + .put("ca_certificate", certs.caCertificate()) + .put("client_certificate", certs.clientCertificate()) + .put("client_key", certs.clientKey()) .put("client_key_password", PASSWORD) .build(); } + @Override + protected BaseImage getServerImage() { + return BaseImage.POSTGRES_16; + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotPostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotPostgresSourceDatatypeTest.java index da0ce367531d..fb063718b8d5 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotPostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcInitialSnapshotPostgresSourceDatatypeTest.java @@ -5,108 +5,41 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.TestDataHolder; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.integrations.util.HostPortResolver; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.integrations.standardtest.source.TestDataHolder; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.ContainerModifier; import io.airbyte.protocol.models.JsonSchemaType; -import java.util.List; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.utility.MountableFile; -import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; -import uk.org.webcompere.systemstubs.jupiter.SystemStub; -import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; -@ExtendWith(SystemStubsExtension.class) public class CdcInitialSnapshotPostgresSourceDatatypeTest extends AbstractPostgresSourceDatatypeTest { private static final String SCHEMA_NAME = "test"; - private static final String SLOT_NAME_BASE = "debezium_slot"; - private static final String PUBLICATION = "publication"; - private static final int INITIAL_WAITING_SECONDS = 30; - - @SystemStub - private EnvironmentVariables environmentVariables; @Override protected Database setupDatabase() throws Exception { - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - container = new PostgreSQLContainer<>("postgres:14-alpine") - .withCopyFileToContainer(MountableFile.forClasspathResource("postgresql.conf"), - "/etc/postgresql/postgresql.conf") - .withCommand("postgres -c config_file=/etc/postgresql/postgresql.conf"); - container.start(); - - /** - * The publication is not being set as part of the config and because of it - * {@link io.airbyte.integrations.source.postgres.PostgresSource#isCdc(JsonNode)} returns false, as - * a result no test in this class runs through the cdc path. - */ - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "CDC") - .put("replication_slot", SLOT_NAME_BASE) - .put("publication", PUBLICATION) - .put("initial_waiting_seconds", INITIAL_WAITING_SECONDS) - .build()); - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(container)) - .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(container)) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.SCHEMAS_KEY, List.of(SCHEMA_NAME)) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put("replication_method", replicationMethod) - .put("is_test", true) - .put(JdbcUtils.SSL_KEY, false) - .build()); - - dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), - container.getHost(), - container.getFirstMappedPort(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.POSTGRES); - final Database database = new Database(dslContext); - - database.query(ctx -> { - ctx.execute( - "SELECT pg_create_logical_replication_slot('" + SLOT_NAME_BASE + "', 'pgoutput');"); - ctx.execute("CREATE PUBLICATION " + PUBLICATION + " FOR ALL TABLES;"); - ctx.execute("CREATE EXTENSION hstore;"); - return null; - }); - - database.query(ctx -> ctx.fetch("CREATE SCHEMA TEST;")); - database.query(ctx -> ctx.fetch("CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy');")); - database.query(ctx -> ctx.fetch("CREATE TYPE inventory_item AS (\n" - + " name text,\n" - + " supplier_id integer,\n" - + " price numeric\n" - + ");")); - - database.query(ctx -> ctx.fetch("SET TIMEZONE TO 'MST'")); - return database; + testdb = PostgresTestDatabase.in(BaseImage.POSTGRES_16, ContainerModifier.CONF) + .with("CREATE EXTENSION hstore;") + .with("CREATE SCHEMA TEST;") + .with("CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy');") + .with("CREATE TYPE inventory_item AS (\n" + + " name text,\n" + + " supplier_id integer,\n" + + " price numeric\n" + + ");") + .with("SET TIMEZONE TO 'MST'") + .withReplicationSlot() + .withPublicationForAllTables(); + return testdb.getDatabase(); } @Override - protected void tearDown(final TestDestinationEnv testEnv) { - dslContext.close(); - container.close(); - } - - public boolean testCatalog() { - return true; + protected JsonNode getConfig() { + return testdb.integrationTestConfigBuilder() + .withSchemas(SCHEMA_NAME) + .withoutSsl() + .withCdcReplication() + .build(); } @Override diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceLegacyCtidTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceLegacyCtidTest.java new file mode 100644 index 000000000000..c821d53716ea --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceLegacyCtidTest.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; + +public class CdcPostgresSourceAcceptanceLegacyCtidTest extends CdcPostgresSourceAcceptanceTest { + + @Override + protected BaseImage getServerImage() { + return BaseImage.POSTGRES_12; + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java index ef78c1188f9e..8e76d19c564f 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcPostgresSourceAcceptanceTest.java @@ -8,16 +8,12 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import io.airbyte.commons.features.EnvVariableFeatureFlags; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.integrations.util.HostPortResolver; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.ContainerModifier; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteMessage; @@ -27,101 +23,46 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; import io.airbyte.protocol.models.v0.SyncMode; +import java.sql.SQLException; import java.util.HashMap; import java.util.List; import java.util.stream.Collectors; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.utility.MountableFile; -import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; -import uk.org.webcompere.systemstubs.jupiter.SystemStub; -import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; // todo (cgardens) - Sanity check that when configured for CDC that postgres performs like any other // incremental source. As we have more sources support CDC we will find a more reusable way of doing // this, but for now this is a solid sanity check. -@ExtendWith(SystemStubsExtension.class) public class CdcPostgresSourceAcceptanceTest extends AbstractPostgresSourceAcceptanceTest { - protected static final String SLOT_NAME_BASE = "debezium_slot"; protected static final String NAMESPACE = "public"; private static final String STREAM_NAME = "id_and_name"; private static final String STREAM_NAME2 = "starships"; - protected static final String PUBLICATION = "publication"; - protected static final int INITIAL_WAITING_SECONDS = 30; - protected PostgreSQLContainer container; - protected JsonNode config; - - @SystemStub - private EnvironmentVariables environmentVariables; - - @BeforeEach - void setup() { - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - } + protected PostgresTestDatabase testdb; @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - container = new PostgreSQLContainer<>("postgres:13-alpine") - .withCopyFileToContainer(MountableFile.forClasspathResource("postgresql.conf"), "/etc/postgresql/postgresql.conf") - .withCommand("postgres -c config_file=/etc/postgresql/postgresql.conf"); - container.start(); - - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "CDC") - .put("replication_slot", SLOT_NAME_BASE) - .put("publication", PUBLICATION) - .put("initial_waiting_seconds", INITIAL_WAITING_SECONDS) - .build()); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(container)) - .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(container)) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.SCHEMAS_KEY, List.of(NAMESPACE)) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put("replication_method", replicationMethod) - .put(JdbcUtils.SSL_KEY, false) - .put("is_test", true) - .build()); - - try (final DSLContext dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), - container.getHost(), - container.getFirstMappedPort(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.POSTGRES)) { - final Database database = new Database(dslContext); - - database.query(ctx -> { - ctx.execute("CREATE TABLE id_and_name(id INTEGER primary key, name VARCHAR(200));"); - ctx.execute("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.execute("CREATE TABLE starships(id INTEGER primary key, name VARCHAR(200));"); - ctx.execute("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - ctx.execute("SELECT pg_create_logical_replication_slot('" + SLOT_NAME_BASE + "', 'pgoutput');"); - ctx.execute("CREATE PUBLICATION " + PUBLICATION + " FOR ALL TABLES;"); - return null; - }); - } + testdb = PostgresTestDatabase.in(getServerImage(), ContainerModifier.CONF) + .with("CREATE TABLE id_and_name(id INTEGER primary key, name VARCHAR(200));") + .with("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');") + .with("CREATE TABLE starships(id INTEGER primary key, name VARCHAR(200));") + .with("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');") + .withReplicationSlot() + .withPublicationForAllTables(); } @Override - protected void tearDown(final TestDestinationEnv testEnv) { - container.close(); + protected void tearDown(final TestDestinationEnv testEnv) throws SQLException { + testdb.close(); } @Override protected JsonNode getConfig() { - return config; + return testdb.integrationTestConfigBuilder() + .withSchemas(NAMESPACE) + .withoutSsl() + .withCdcReplication() + .build(); } @Override @@ -222,4 +163,8 @@ private void verifyFieldNotExist(final List records, final .isEmpty(), "Records contain unselected columns [%s:%s]".formatted(stream, field)); } + protected BaseImage getServerImage() { + return BaseImage.POSTGRES_16; + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java index 4884a4f59a9c..c099d9bce930 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CdcWalLogsPostgresSourceDatatypeTest.java @@ -5,16 +5,12 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.features.EnvVariableFeatureFlags; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.integrations.standardtest.source.TestDataHolder; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.TestDataHolder; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.integrations.util.HostPortResolver; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.ContainerModifier; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; @@ -23,26 +19,12 @@ import java.util.Collections; import java.util.List; import java.util.Set; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.utility.MountableFile; -import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; -import uk.org.webcompere.systemstubs.jupiter.SystemStub; -import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; -@ExtendWith(SystemStubsExtension.class) public class CdcWalLogsPostgresSourceDatatypeTest extends AbstractPostgresSourceDatatypeTest { private static final String SCHEMA_NAME = "test"; - private static final String SLOT_NAME_BASE = "debezium_slot"; - private static final String PUBLICATION = "publication"; - private static final int INITIAL_WAITING_SECONDS = 30; private JsonNode stateAfterFirstSync; - @SystemStub - private EnvironmentVariables environmentVariables; - @Override protected List runRead(final ConfiguredAirbyteCatalog configuredCatalog) throws Exception { if (stateAfterFirstSync == null) { @@ -54,7 +36,6 @@ protected List runRead(final ConfiguredAirbyteCatalog configured @Override protected void postSetup() throws Exception { final Database database = setupDatabase(); - initTests(); for (final TestDataHolder test : testDataHolders) { database.query(ctx -> { ctx.fetch(test.getCreateSqlQuery()); @@ -84,76 +65,29 @@ protected void postSetup() throws Exception { } @Override - protected Database setupDatabase() throws Exception { - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - container = new PostgreSQLContainer<>("postgres:14-alpine") - .withCopyFileToContainer(MountableFile.forClasspathResource("postgresql.conf"), - "/etc/postgresql/postgresql.conf") - .withCommand("postgres -c config_file=/etc/postgresql/postgresql.conf"); - container.start(); - - /** - * The publication is not being set as part of the config and because of it - * {@link io.airbyte.integrations.source.postgres.PostgresSource#isCdc(JsonNode)} returns false, as - * a result no test in this class runs through the cdc path. - */ - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "CDC") - .put("replication_slot", SLOT_NAME_BASE) - .put("publication", PUBLICATION) - .put("initial_waiting_seconds", INITIAL_WAITING_SECONDS) - .build()); - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(container)) - .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(container)) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.SCHEMAS_KEY, List.of(SCHEMA_NAME)) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put("replication_method", replicationMethod) - .put("is_test", true) - .put(JdbcUtils.SSL_KEY, false) - .build()); - - dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), - container.getHost(), - container.getFirstMappedPort(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.POSTGRES); - final Database database = new Database(dslContext); - - database.query(ctx -> { - ctx.execute( - "SELECT pg_create_logical_replication_slot('" + SLOT_NAME_BASE + "', 'pgoutput');"); - ctx.execute("CREATE PUBLICATION " + PUBLICATION + " FOR ALL TABLES;"); - ctx.execute("CREATE EXTENSION hstore;"); - return null; - }); - - database.query(ctx -> ctx.fetch("CREATE SCHEMA TEST;")); - database.query(ctx -> ctx.fetch("CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy');")); - database.query(ctx -> ctx.fetch("CREATE TYPE inventory_item AS (\n" - + " name text,\n" - + " supplier_id integer,\n" - + " price numeric\n" - + ");")); - - database.query(ctx -> ctx.fetch("SET TIMEZONE TO 'MST'")); - return database; + protected Database setupDatabase() { + testdb = PostgresTestDatabase.in(BaseImage.POSTGRES_16, ContainerModifier.CONF) + .with("CREATE EXTENSION hstore;") + .with("CREATE SCHEMA TEST;") + .with("CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy');") + .with("CREATE TYPE inventory_item AS (\n" + + " name text,\n" + + " supplier_id integer,\n" + + " price numeric\n" + + ");") + .with("SET TIMEZONE TO 'MST'") + .withReplicationSlot() + .withPublicationForAllTables(); + return testdb.getDatabase(); } @Override - protected void tearDown(final TestDestinationEnv testEnv) { - dslContext.close(); - container.close(); - } - - public boolean testCatalog() { - return true; + protected JsonNode getConfig() throws Exception { + return testdb.integrationTestConfigBuilder() + .withSchemas(SCHEMA_NAME) + .withoutSsl() + .withCdcReplication() + .build(); } @Override @@ -197,20 +131,33 @@ protected void addTimeWithTimeZoneTest() { } @Override - protected void addTimestampWithInfinityValuesTest() { - // timestamp without time zone - for (final String fullSourceType : Set.of("timestamp", "timestamp without time zone", "timestamp without time zone not null default now()")) { + protected void addNumericValuesTest() { + addDataTypeTestData( + TestDataHolder.builder() + .sourceType("numeric") + .fullSourceDataType("NUMERIC(28,2)") + .airbyteType(JsonSchemaType.NUMBER) + .addInsertValues( + "'123'", "null", "'14525.22'") + // Postgres source does not support these special values yet + // https://github.com/airbytehq/airbyte/issues/8902 + // "'infinity'", "'-infinity'", "'nan'" + .addExpectedValues("123", null, "14525.22") + .build()); + + // Blocked by https://github.com/airbytehq/airbyte/issues/8902 + for (final String type : Set.of("numeric", "decimal")) { addDataTypeTestData( TestDataHolder.builder() - .sourceType("timestamp") - .fullSourceDataType(fullSourceType) - .airbyteType(JsonSchemaType.STRING_TIMESTAMP_WITHOUT_TIMEZONE) + .sourceType(type) + .fullSourceDataType("NUMERIC(20,7)") + .airbyteType(JsonSchemaType.NUMBER) .addInsertValues( - "'infinity'", - "'-infinity'") - .addExpectedValues( - "+294247-01-10T04:00:25.200000", - "+290309-12-21T19:59:27.600000 BC") + "'123'", "null", "'1234567890.1234567'") + // Postgres source does not support these special values yet + // https://github.com/airbytehq/airbyte/issues/8902 + // "'infinity'", "'-infinity'", "'nan'" + .addExpectedValues("123", null, "1.2345678901234567E9") .build()); } } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CloudDeploymentPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CloudDeploymentPostgresSourceAcceptanceTest.java new file mode 100644 index 000000000000..07ca597b64ee --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/CloudDeploymentPostgresSourceAcceptanceTest.java @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.base.adaptive.AdaptiveSourceRunner; +import io.airbyte.cdk.integrations.base.ssh.SshHelpers; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.commons.features.FeatureFlags; +import io.airbyte.commons.features.FeatureFlagsWrapper; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.ContainerModifier; +import io.airbyte.protocol.models.Field; +import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.v0.CatalogHelpers; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.v0.ConnectorSpecification; +import io.airbyte.protocol.models.v0.DestinationSyncMode; +import io.airbyte.protocol.models.v0.SyncMode; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; + +public class CloudDeploymentPostgresSourceAcceptanceTest extends SourceAcceptanceTest { + + private static final String STREAM_NAME = "id_and_name"; + private static final String STREAM_NAME2 = "starships"; + private static final String SCHEMA_NAME = "public"; + + private PostgresTestDatabase testdb; + + protected static final String PASSWORD = "Passw0rd"; + + @Override + protected FeatureFlags featureFlags() { + return FeatureFlagsWrapper.overridingDeploymentMode( + super.featureFlags(), + AdaptiveSourceRunner.CLOUD_MODE); + } + + @Override + protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { + testdb = PostgresTestDatabase.in(BaseImage.POSTGRES_16, ContainerModifier.CERT); + testdb.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); + ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); + ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); + ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); + return null; + }); + } + + @Override + protected void tearDown(final TestDestinationEnv testEnv) { + testdb.close(); + } + + @Override + protected String getImageName() { + return "airbyte/source-postgres:dev"; + } + + @Override + protected ConnectorSpecification getSpec() throws Exception { + return SshHelpers.injectSshIntoSpec( + Jsons.deserialize(MoreResources.readResource("expected_cloud_deployment_spec.json"), ConnectorSpecification.class), + Optional.of("security")); + } + + @Override + protected JsonNode getConfig() { + final var certs = testdb.getCertificates(); + return testdb.integrationTestConfigBuilder() + .withStandardReplication() + .withSsl(ImmutableMap.builder() + .put("mode", "verify-ca") + .put("ca_certificate", certs.caCertificate()) + .put("client_certificate", certs.clientCertificate()) + .put("client_key", certs.clientKey()) + .put("client_key_password", PASSWORD) + .build()) + .build(); + } + + @Override + protected ConfiguredAirbyteCatalog getConfiguredCatalog() { + return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(Lists.newArrayList("id")) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(CatalogHelpers.createAirbyteStream( + STREAM_NAME, SCHEMA_NAME, + Field.of("id", JsonSchemaType.NUMBER), + Field.of("name", JsonSchemaType.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of("id")))), + new ConfiguredAirbyteStream() + .withSyncMode(SyncMode.INCREMENTAL) + .withCursorField(Lists.newArrayList("id")) + .withDestinationSyncMode(DestinationSyncMode.APPEND) + .withStream(CatalogHelpers.createAirbyteStream( + STREAM_NAME2, SCHEMA_NAME, + Field.of("id", JsonSchemaType.NUMBER), + Field.of("name", JsonSchemaType.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of("id")))))); + } + + @Override + protected JsonNode getState() { + return Jsons.jsonNode(new HashMap<>()); + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceLegacyCtidTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceLegacyCtidTest.java new file mode 100644 index 000000000000..ae616e3dcad6 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceLegacyCtidTest.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; + +public class PostgresSourceAcceptanceLegacyCtidTest extends PostgresSourceAcceptanceTest { + + @Override + protected BaseImage getServerImage() { + return BaseImage.POSTGRES_12; + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java index 9a5a349d2573..e657a8886ecb 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java @@ -8,16 +8,13 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import io.airbyte.commons.features.EnvVariableFeatureFlags; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.integrations.util.HostPortResolver; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; @@ -30,87 +27,53 @@ import java.sql.SQLException; import java.util.HashMap; import java.util.List; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testcontainers.containers.PostgreSQLContainer; -import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; -import uk.org.webcompere.systemstubs.jupiter.SystemStub; -import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; -@ExtendWith(SystemStubsExtension.class) public class PostgresSourceAcceptanceTest extends AbstractPostgresSourceAcceptanceTest { private static final String STREAM_NAME = "id_and_name"; private static final String STREAM_NAME2 = "starships"; private static final String STREAM_NAME_MATERIALIZED_VIEW = "testview"; private static final String SCHEMA_NAME = "public"; - @SystemStub - private EnvironmentVariables environmentVariables; public static final String LIMIT_PERMISSION_SCHEMA = "limit_perm_schema"; - public static final String LIMIT_PERMISSION_ROLE = "limit_perm_role"; - public static final String LIMIT_PERMISSION_ROLE_PASSWORD = "test"; + static public final String LIMIT_PERMISSION_ROLE_PASSWORD = "test"; - private PostgreSQLContainer container; + private PostgresTestDatabase testdb; private JsonNode config; - private Database database; - private ConfiguredAirbyteCatalog configCatalog; @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - - container = new PostgreSQLContainer<>("postgres:13-alpine"); - container.start(); - - final String username = container.getUsername(); - final String password = container.getPassword(); - final List schemas = List.of("public"); - config = getConfig(username, password, schemas); - try (final DSLContext dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), - container.getHost(), - container.getFirstMappedPort(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.POSTGRES)) { - database = new Database(dslContext); - - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - ctx.fetch("CREATE MATERIALIZED VIEW testview AS select * from id_and_name where id = '2';"); - return null; - }); - configCatalog = getCommonConfigCatalog(); - } + testdb = PostgresTestDatabase.in(getServerImage()); + config = getConfig(testdb.getUserName(), testdb.getPassword(), "public"); + testdb.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); + ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); + ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); + ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); + ctx.fetch("CREATE MATERIALIZED VIEW testview AS select * from id_and_name where id = '2';"); + return null; + }); } - private JsonNode getConfig(final String username, final String password, final List schemas) { - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "Standard") - .build()); - - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(container)) - .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(container)) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.SCHEMAS_KEY, Jsons.jsonNode(schemas)) - .put(JdbcUtils.USERNAME_KEY, username) - .put(JdbcUtils.PASSWORD_KEY, password) - .put(JdbcUtils.SSL_KEY, false) - .put("replication_method", replicationMethod) - .build()); + private String getLimitPermissionRoleName() { + return testdb.withNamespace("limit_perm_role"); + } + + private JsonNode getConfig(final String username, final String password, String... schemas) { + return testdb.configBuilder() + .withResolvedHostAndPort() + .withDatabase() + .with(JdbcUtils.USERNAME_KEY, username) + .with(JdbcUtils.PASSWORD_KEY, password) + .withSchemas(schemas) + .withoutSsl() + .withStandardReplication() + .build(); } @Override protected void tearDown(final TestDestinationEnv testEnv) { - container.close(); + testdb.close(); } @Override @@ -120,7 +83,7 @@ protected JsonNode getConfig() { @Override protected ConfiguredAirbyteCatalog getConfiguredCatalog() { - return configCatalog; + return getCommonConfigCatalog(); } @Override @@ -128,23 +91,18 @@ protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); } - @Override - protected boolean supportsPerStream() { - return true; - } - @Test public void testFullRefreshWithRevokingSchemaPermissions() throws Exception { - prepareEnvForUserWithoutPermissions(database); + prepareEnvForUserWithoutPermissions(testdb.getDatabase()); - config = getConfig(LIMIT_PERMISSION_ROLE, LIMIT_PERMISSION_ROLE_PASSWORD, List.of(LIMIT_PERMISSION_SCHEMA)); + config = getConfig(getLimitPermissionRoleName(), LIMIT_PERMISSION_ROLE_PASSWORD, LIMIT_PERMISSION_SCHEMA); final ConfiguredAirbyteCatalog configuredCatalog = getLimitPermissionConfiguredCatalog(); final List fullRefreshRecords = filterRecords(runRead(configuredCatalog)); final String assertionMessage = "Expected records after full refresh sync for user with schema permission"; assertFalse(fullRefreshRecords.isEmpty(), assertionMessage); - revokeSchemaPermissions(database); + revokeSchemaPermissions(testdb.getDatabase()); final List lessPermFullRefreshRecords = filterRecords(runRead(configuredCatalog)); final String assertionMessageWithoutPermission = "Expected no records after full refresh sync for user without schema permission"; @@ -154,9 +112,9 @@ public void testFullRefreshWithRevokingSchemaPermissions() throws Exception { @Test public void testDiscoverWithRevokingSchemaPermissions() throws Exception { - prepareEnvForUserWithoutPermissions(database); - revokeSchemaPermissions(database); - config = getConfig(LIMIT_PERMISSION_ROLE, LIMIT_PERMISSION_ROLE_PASSWORD, List.of(LIMIT_PERMISSION_SCHEMA)); + prepareEnvForUserWithoutPermissions(testdb.getDatabase()); + revokeSchemaPermissions(testdb.getDatabase()); + config = getConfig(getLimitPermissionRoleName(), LIMIT_PERMISSION_ROLE_PASSWORD, LIMIT_PERMISSION_SCHEMA); runDiscover(); final AirbyteCatalog lastPersistedCatalogSecond = getLastPersistedCatalog(); @@ -166,20 +124,20 @@ public void testDiscoverWithRevokingSchemaPermissions() throws Exception { private void revokeSchemaPermissions(final Database database) throws SQLException { database.query(ctx -> { - ctx.fetch(String.format("REVOKE USAGE ON schema %s FROM %s;", LIMIT_PERMISSION_SCHEMA, LIMIT_PERMISSION_ROLE)); + ctx.fetch(String.format("REVOKE USAGE ON schema %s FROM %s;", LIMIT_PERMISSION_SCHEMA, getLimitPermissionRoleName())); return null; }); } private void prepareEnvForUserWithoutPermissions(final Database database) throws SQLException { database.query(ctx -> { - ctx.fetch(String.format("CREATE ROLE %s WITH LOGIN PASSWORD '%s';", LIMIT_PERMISSION_ROLE, LIMIT_PERMISSION_ROLE_PASSWORD)); + ctx.fetch(String.format("CREATE ROLE %s WITH LOGIN PASSWORD '%s';", getLimitPermissionRoleName(), LIMIT_PERMISSION_ROLE_PASSWORD)); ctx.fetch(String.format("CREATE SCHEMA %s;", LIMIT_PERMISSION_SCHEMA)); - ctx.fetch(String.format("GRANT CONNECT ON DATABASE test TO %s;", LIMIT_PERMISSION_ROLE)); - ctx.fetch(String.format("GRANT USAGE ON schema %s TO %s;", LIMIT_PERMISSION_SCHEMA, LIMIT_PERMISSION_ROLE)); + ctx.fetch(String.format("GRANT CONNECT ON DATABASE %s TO %s;", testdb.getDatabaseName(), getLimitPermissionRoleName())); + ctx.fetch(String.format("GRANT USAGE ON schema %s TO %s;", LIMIT_PERMISSION_SCHEMA, getLimitPermissionRoleName())); ctx.fetch(String.format("CREATE TABLE %s.id_and_name(id INTEGER, name VARCHAR(200));", LIMIT_PERMISSION_SCHEMA)); ctx.fetch(String.format("INSERT INTO %s.id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');", LIMIT_PERMISSION_SCHEMA)); - ctx.fetch(String.format("GRANT SELECT ON table %s.id_and_name TO %s;", LIMIT_PERMISSION_SCHEMA, LIMIT_PERMISSION_ROLE)); + ctx.fetch(String.format("GRANT SELECT ON table %s.id_and_name TO %s;", LIMIT_PERMISSION_SCHEMA, getLimitPermissionRoleName())); return null; }); } @@ -231,4 +189,8 @@ private ConfiguredAirbyteCatalog getLimitPermissionConfiguredCatalog() { .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))))); } + protected BaseImage getServerImage() { + return BaseImage.POSTGRES_16; + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java index 8ce8007a3f6a..12b50a733c29 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java @@ -5,91 +5,39 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.integrations.util.HostPortResolver; +import io.airbyte.cdk.db.Database; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.ContainerModifier; import java.sql.SQLException; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.utility.MountableFile; -import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; -import uk.org.webcompere.systemstubs.jupiter.SystemStub; -import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; -@ExtendWith(SystemStubsExtension.class) public class PostgresSourceDatatypeTest extends AbstractPostgresSourceDatatypeTest { - @SystemStub - private EnvironmentVariables environmentVariables; - @Override protected Database setupDatabase() throws SQLException { - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - container = new PostgreSQLContainer<>("postgres:14-alpine") - .withCopyFileToContainer(MountableFile.forClasspathResource("postgresql.conf"), - "/etc/postgresql/postgresql.conf") - .withCommand("postgres -c config_file=/etc/postgresql/postgresql.conf"); - container.start(); - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "Standard") - .build()); - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(container)) - .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(container)) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put(JdbcUtils.SSL_KEY, false) - .put("replication_method", replicationMethod) - .build()); - - dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), - container.getHost(), - container.getFirstMappedPort(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.POSTGRES); - final Database database = new Database(dslContext); - - database.query(ctx -> { - ctx.execute(String.format("CREATE SCHEMA %S;", SCHEMA_NAME)); - ctx.execute("CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy');"); - ctx.execute("CREATE TYPE inventory_item AS (name text, supplier_id integer, price numeric);"); - // In one of the test case, we have some money values with currency symbol. Postgres can only - // understand those money values if the symbol corresponds to the monetary locale setting. For - // example, - // if the locale is 'en_GB', '£100' is valid, but '$100' is not. So setting the monetary locate is - // necessary here to make sure the unit test can pass, no matter what the locale the runner VM has. - ctx.execute("SET lc_monetary TO 'en_US.utf8';"); - // Set up a fixed timezone here so that timetz and timestamptz always have the same time zone - // wherever the tests are running on. - ctx.execute("SET TIMEZONE TO 'MST'"); - ctx.execute("CREATE EXTENSION hstore;"); - return null; - }); - - return database; - } - - @Override - protected void tearDown(final TestDestinationEnv testEnv) { - dslContext.close(); - container.close(); + testdb = PostgresTestDatabase.in(BaseImage.POSTGRES_16, ContainerModifier.CONF) + .with("CREATE SCHEMA %S;", SCHEMA_NAME) + .with("CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy');") + .with("CREATE TYPE inventory_item AS (name text, supplier_id integer, price numeric);") + // In one of the test case, we have some money values with currency symbol. Postgres can only + // understand those money values if the symbol corresponds to the monetary locale setting. For + // example, + // if the locale is 'en_GB', '£100' is valid, but '$100' is not. So setting the monetary locate is + // necessary here to make sure the unit test can pass, no matter what the locale the runner VM has. + .with("SET lc_monetary TO 'en_US.utf8';") + // Set up a fixed timezone here so that timetz and timestamptz always have the same time zone + // wherever the tests are running on. + .with("SET TIMEZONE TO 'MST'") + .with("CREATE EXTENSION hstore;"); + return testdb.getDatabase(); } @Override - public boolean testCatalog() { - return true; + protected JsonNode getConfig() throws Exception { + return testdb.integrationTestConfigBuilder() + .withoutSsl() + .withStandardReplication() + .build(); } } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceSSLCaCertificateAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceSSLCaCertificateAcceptanceTest.java index bfecd215194a..eb93444a7201 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceSSLCaCertificateAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceSSLCaCertificateAcceptanceTest.java @@ -5,14 +5,15 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; import com.google.common.collect.ImmutableMap; +import java.util.Map; public class PostgresSourceSSLCaCertificateAcceptanceTest extends AbstractPostgresSourceSSLCertificateAcceptanceTest { @Override - public ImmutableMap getCertificateConfiguration() { + public Map getCertificateConfiguration() { return ImmutableMap.builder() .put("mode", "verify-ca") - .put("ca_certificate", certs.getCaCertificate()) + .put("ca_certificate", testdb.getCertificates().caCertificate()) .put("client_key_password", PASSWORD) .build(); } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceSSLFullCertificateAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceSSLFullCertificateAcceptanceTest.java index bf0282c418ee..dcd4810cd34b 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceSSLFullCertificateAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceSSLFullCertificateAcceptanceTest.java @@ -5,16 +5,17 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; import com.google.common.collect.ImmutableMap; +import java.util.Map; public class PostgresSourceSSLFullCertificateAcceptanceTest extends AbstractPostgresSourceSSLCertificateAcceptanceTest { @Override - public ImmutableMap getCertificateConfiguration() { + public Map getCertificateConfiguration() { return ImmutableMap.builder() .put("mode", "verify-ca") - .put("ca_certificate", certs.getCaCertificate()) - .put("client_certificate", certs.getClientCertificate()) - .put("client_key", certs.getClientKey()) + .put("ca_certificate", testdb.getCertificates().caCertificate()) + .put("client_certificate", testdb.getCertificates().clientCertificate()) + .put("client_key", testdb.getCertificates().clientKey()) .put("client_key_password", PASSWORD) .build(); } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshKeyPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshKeyPostgresSourceAcceptanceTest.java index 0804418808e6..1324156ab255 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshKeyPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshKeyPostgresSourceAcceptanceTest.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; -import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; public class SshKeyPostgresSourceAcceptanceTest extends AbstractSshPostgresSourceAcceptanceTest { diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordPostgresSourceAcceptanceTest.java index f3e5c6041197..4ecbd2611082 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SshPasswordPostgresSourceAcceptanceTest.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; -import io.airbyte.integrations.base.ssh.SshTunnel; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; public class SshPasswordPostgresSourceAcceptanceTest extends AbstractSshPostgresSourceAcceptanceTest { diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/XminPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/XminPostgresSourceAcceptanceTest.java index d45fcc7f2bc1..2986f35a38e9 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/XminPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/XminPostgresSourceAcceptanceTest.java @@ -5,16 +5,11 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import io.airbyte.commons.features.EnvVariableFeatureFlags; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.integrations.util.HostPortResolver; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.CatalogHelpers; @@ -24,104 +19,42 @@ import io.airbyte.protocol.models.v0.SyncMode; import java.util.HashMap; import java.util.List; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testcontainers.containers.PostgreSQLContainer; -import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; -import uk.org.webcompere.systemstubs.jupiter.SystemStub; -import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; -@ExtendWith(SystemStubsExtension.class) public class XminPostgresSourceAcceptanceTest extends AbstractPostgresSourceAcceptanceTest { private static final String STREAM_NAME = "id_and_name"; private static final String STREAM_NAME2 = "starships"; private static final String STREAM_NAME_MATERIALIZED_VIEW = "testview"; private static final String SCHEMA_NAME = "public"; - @SystemStub - private EnvironmentVariables environmentVariables; - private PostgreSQLContainer container; - private JsonNode config; - private Database database; - private ConfiguredAirbyteCatalog configCatalog; + private PostgresTestDatabase testdb; @Override protected JsonNode getConfig() throws Exception { - return config; + return testdb.integrationTestConfigBuilder() + .withSchemas("public") + .withoutSsl() + .withXminReplication() + .build(); } @Override protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - - container = new PostgreSQLContainer<>("postgres:13-alpine"); - container.start(); - final String username = container.getUsername(); - final String password = container.getPassword(); - final List schemas = List.of("public"); - config = getXminConfig(username, password, schemas); - try (final DSLContext dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), - container.getHost(), - container.getFirstMappedPort(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.POSTGRES)) { - database = new Database(dslContext); - - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');"); - ctx.fetch("CREATE TABLE starships(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');"); - ctx.fetch("CREATE MATERIALIZED VIEW testview AS select * from id_and_name where id = '2';"); - return null; - }); - configCatalog = getXminCatalog(); - } - } - - private JsonNode getXminConfig(final String username, final String password, final List schemas) { - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "Xmin") - .build()); - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(container)) - .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(container)) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.SCHEMAS_KEY, Jsons.jsonNode(schemas)) - .put(JdbcUtils.USERNAME_KEY, username) - .put(JdbcUtils.PASSWORD_KEY, password) - .put(JdbcUtils.SSL_KEY, false) - .put("replication_method", replicationMethod) - .build()); + testdb = PostgresTestDatabase.in(BaseImage.POSTGRES_12) + .with("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));") + .with("INSERT INTO id_and_name (id, name) VALUES (1,'picard'), (2, 'crusher'), (3, 'vash');") + .with("CREATE TABLE starships(id INTEGER, name VARCHAR(200));") + .with("INSERT INTO starships (id, name) VALUES (1,'enterprise-d'), (2, 'defiant'), (3, 'yamato');") + .with("CREATE MATERIALIZED VIEW testview AS select * from id_and_name where id = '2';"); } @Override protected void tearDown(final TestDestinationEnv testEnv) throws Exception { - container.close(); + testdb.close(); } @Override protected ConfiguredAirbyteCatalog getConfiguredCatalog() throws Exception { - return configCatalog; - } - - @Override - protected JsonNode getState() throws Exception { - return Jsons.jsonNode(new HashMap<>()); - } - - @Override - protected boolean supportsPerStream() { - return true; - } - - private ConfiguredAirbyteCatalog getXminCatalog() { return new ConfiguredAirbyteCatalog().withStreams(Lists.newArrayList( new ConfiguredAirbyteStream() .withSyncMode(SyncMode.INCREMENTAL) @@ -155,4 +88,9 @@ private ConfiguredAirbyteCatalog getXminCatalog() { .withSourceDefinedPrimaryKey(List.of(List.of("id")))))); } + @Override + protected JsonNode getState() throws Exception { + return Jsons.jsonNode(new HashMap<>()); + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_cloud_deployment_spec.json b/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_cloud_deployment_spec.json new file mode 100644 index 000000000000..375ea5024c99 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_cloud_deployment_spec.json @@ -0,0 +1,456 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/postgres", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Postgres Source Spec", + "type": "object", + "required": ["host", "port", "database", "username"], + "properties": { + "host": { + "title": "Host", + "description": "Hostname of the database.", + "type": "string", + "order": 0, + "group": "db" + }, + "port": { + "title": "Port", + "description": "Port of the database.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 5432, + "examples": ["5432"], + "order": 1, + "group": "db" + }, + "database": { + "title": "Database Name", + "description": "Name of the database.", + "type": "string", + "order": 2, + "group": "db" + }, + "schemas": { + "title": "Schemas", + "description": "The list of schemas (case sensitive) to sync from. Defaults to public.", + "type": "array", + "items": { + "type": "string" + }, + "minItems": 0, + "uniqueItems": true, + "default": ["public"], + "order": 3, + "group": "db" + }, + "username": { + "title": "Username", + "description": "Username to access the database.", + "type": "string", + "order": 4, + "group": "auth" + }, + "password": { + "title": "Password", + "description": "Password associated with the username.", + "type": "string", + "airbyte_secret": true, + "order": 5, + "group": "auth", + "always_show": true + }, + "jdbc_url_params": { + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", + "title": "JDBC URL Parameters (Advanced)", + "type": "string", + "order": 6, + "group": "advanced", + "pattern_descriptor": "key1=value1&key2=value2" + }, + "ssl_mode": { + "title": "SSL Modes", + "description": "SSL connection modes. \n Read more in the docs.", + "type": "object", + "order": 8, + "group": "security", + "default": "require", + "oneOf": [ + { + "title": "disable", + "additionalProperties": true, + "description": "Disables encryption of communication between Airbyte and source database.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "disable", + "order": 0 + } + } + }, + { + "title": "allow", + "additionalProperties": true, + "description": "Enables encryption only when required by the source database.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "allow", + "order": 0 + } + } + }, + { + "title": "prefer", + "additionalProperties": true, + "description": "Allows unencrypted connection only if the source database does not support encryption.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "prefer", + "order": 0 + } + } + }, + { + "title": "require", + "additionalProperties": true, + "description": "Always require encryption. If the source database server does not support encryption, connection will fail.", + "required": ["mode"], + "properties": { + "mode": { + "type": "string", + "const": "require", + "order": 0 + } + } + }, + { + "title": "verify-ca", + "additionalProperties": true, + "description": "Always require encryption and verifies that the source database server has a valid SSL certificate.", + "required": ["mode", "ca_certificate"], + "properties": { + "mode": { + "type": "string", + "const": "verify-ca", + "order": 0 + }, + "ca_certificate": { + "type": "string", + "title": "CA Certificate", + "description": "CA certificate", + "airbyte_secret": true, + "multiline": true, + "order": 1 + }, + "client_certificate": { + "type": "string", + "title": "Client Certificate", + "description": "Client certificate", + "airbyte_secret": true, + "multiline": true, + "order": 2, + "always_show": true + }, + "client_key": { + "type": "string", + "title": "Client Key", + "description": "Client key", + "airbyte_secret": true, + "multiline": true, + "order": 3, + "always_show": true + }, + "client_key_password": { + "type": "string", + "title": "Client key password", + "description": "Password for keystorage. If you do not add it - the password will be generated automatically.", + "airbyte_secret": true, + "order": 4 + } + } + }, + { + "title": "verify-full", + "additionalProperties": true, + "description": "This is the most secure mode. Always require encryption and verifies the identity of the source database server.", + "required": ["mode", "ca_certificate"], + "properties": { + "mode": { + "type": "string", + "const": "verify-full", + "order": 0 + }, + "ca_certificate": { + "type": "string", + "title": "CA Certificate", + "description": "CA certificate", + "airbyte_secret": true, + "multiline": true, + "order": 1 + }, + "client_certificate": { + "type": "string", + "title": "Client Certificate", + "description": "Client certificate", + "airbyte_secret": true, + "multiline": true, + "order": 2, + "always_show": true + }, + "client_key": { + "type": "string", + "title": "Client Key", + "description": "Client key", + "airbyte_secret": true, + "multiline": true, + "order": 3, + "always_show": true + }, + "client_key_password": { + "type": "string", + "title": "Client key password", + "description": "Password for keystorage. If you do not add it - the password will be generated automatically.", + "airbyte_secret": true, + "order": 4 + } + } + } + ] + }, + "replication_method": { + "type": "object", + "title": "Update Method", + "description": "Configures how data is extracted from the database.", + "order": 9, + "group": "advanced", + "default": "CDC", + "display_type": "radio", + "oneOf": [ + { + "title": "Read Changes using Write-Ahead Log (CDC)", + "description": "Recommended - Incrementally reads new inserts, updates, and deletes using the Postgres write-ahead log (WAL). This needs to be configured on the source database itself. Recommended for tables of any size.", + "required": ["method", "replication_slot", "publication"], + "additionalProperties": true, + "properties": { + "method": { + "type": "string", + "const": "CDC", + "order": 1 + }, + "plugin": { + "type": "string", + "title": "Plugin", + "description": "A logical decoding plugin installed on the PostgreSQL server.", + "enum": ["pgoutput"], + "default": "pgoutput", + "order": 2 + }, + "replication_slot": { + "type": "string", + "title": "Replication Slot", + "description": "A plugin logical replication slot. Read about replication slots.", + "order": 3 + }, + "publication": { + "type": "string", + "title": "Publication", + "description": "A Postgres publication used for consuming changes. Read about publications and replication identities.", + "order": 4 + }, + "initial_waiting_seconds": { + "type": "integer", + "title": "Initial Waiting Time in Seconds (Advanced)", + "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 1200 seconds. Valid range: 120 seconds to 2400 seconds. Read about initial waiting time.", + "default": 1200, + "order": 5, + "min": 120, + "max": 2400 + }, + "queue_size": { + "type": "integer", + "title": "Size of the queue (Advanced)", + "description": "The size of the internal queue. This may interfere with memory consumption and efficiency of the connector, please be careful.", + "default": 10000, + "order": 6, + "min": 1000, + "max": 10000 + }, + "lsn_commit_behaviour": { + "type": "string", + "title": "LSN commit behaviour", + "description": "Determines when Airbyte should flush the LSN of processed WAL logs in the source database. `After loading Data in the destination` is default. If `While reading Data` is selected, in case of a downstream failure (while loading data into the destination), next sync would result in a full sync.", + "enum": [ + "While reading Data", + "After loading Data in the destination" + ], + "default": "After loading Data in the destination", + "order": 7 + } + } + }, + { + "title": "Detect Changes with Xmin System Column", + "description": "Recommended - Incrementally reads new inserts and updates via Postgres Xmin system column. Only recommended for tables up to 500GB.", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "Xmin", + "order": 0 + } + } + }, + { + "title": "Scan Changes with User Defined Cursor", + "description": "Incrementally detects new inserts and updates using the cursor column chosen when configuring a connection (e.g. created_at, updated_at).", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "Standard", + "order": 8 + } + } + } + ] + }, + "tunnel_method": { + "type": "object", + "title": "SSH Tunnel Method", + "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use.", + "group": "security", + "oneOf": [ + { + "title": "No Tunnel", + "required": ["tunnel_method"], + "properties": { + "tunnel_method": { + "description": "No ssh tunnel needed to connect to database", + "type": "string", + "const": "NO_TUNNEL", + "order": 0 + } + } + }, + { + "title": "SSH Key Authentication", + "required": [ + "tunnel_method", + "tunnel_host", + "tunnel_port", + "tunnel_user", + "ssh_key" + ], + "properties": { + "tunnel_method": { + "description": "Connect through a jump server tunnel host using username and ssh key", + "type": "string", + "const": "SSH_KEY_AUTH", + "order": 0 + }, + "tunnel_host": { + "title": "SSH Tunnel Jump Server Host", + "description": "Hostname of the jump server host that allows inbound ssh tunnel.", + "type": "string", + "order": 1 + }, + "tunnel_port": { + "title": "SSH Connection Port", + "description": "Port on the proxy/jump server that accepts inbound ssh connections.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 22, + "examples": ["22"], + "order": 2 + }, + "tunnel_user": { + "title": "SSH Login Username", + "description": "OS-level username for logging into the jump server host.", + "type": "string", + "order": 3 + }, + "ssh_key": { + "title": "SSH Private Key", + "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", + "type": "string", + "airbyte_secret": true, + "multiline": true, + "order": 4 + } + } + }, + { + "title": "Password Authentication", + "required": [ + "tunnel_method", + "tunnel_host", + "tunnel_port", + "tunnel_user", + "tunnel_user_password" + ], + "properties": { + "tunnel_method": { + "description": "Connect through a jump server tunnel host using username and password authentication", + "type": "string", + "const": "SSH_PASSWORD_AUTH", + "order": 0 + }, + "tunnel_host": { + "title": "SSH Tunnel Jump Server Host", + "description": "Hostname of the jump server host that allows inbound ssh tunnel.", + "type": "string", + "order": 1 + }, + "tunnel_port": { + "title": "SSH Connection Port", + "description": "Port on the proxy/jump server that accepts inbound ssh connections.", + "type": "integer", + "minimum": 0, + "maximum": 65536, + "default": 22, + "examples": ["22"], + "order": 2 + }, + "tunnel_user": { + "title": "SSH Login Username", + "description": "OS-level username for logging into the jump server host", + "type": "string", + "order": 3 + }, + "tunnel_user_password": { + "title": "Password", + "description": "OS-level password for logging into the jump server host", + "type": "string", + "airbyte_secret": true, + "order": 4 + } + } + } + ] + } + }, + "groups": [ + { + "id": "db" + }, + { + "id": "auth" + }, + { + "id": "security", + "title": "Security" + }, + { + "id": "advanced", + "title": "Advanced" + } + ] + }, + "supported_destination_sync_modes": [] +} diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_spec.json b/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_spec.json index a691f1e7ee09..8b09d54fd1b6 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/resources/expected_spec.json @@ -223,26 +223,16 @@ }, "replication_method": { "type": "object", - "title": "Replication Method", - "description": "Replication method for extracting data from the database.", + "title": "Update Method", + "description": "Configures how data is extracted from the database.", "order": 9, "group": "advanced", + "default": "CDC", + "display_type": "radio", "oneOf": [ { - "title": "Standard (Xmin)", - "description": "Xmin replication requires no setup on the DB side but will not be able to represent deletions incrementally.", - "required": ["method"], - "properties": { - "method": { - "type": "string", - "const": "Xmin", - "order": 0 - } - } - }, - { - "title": "Logical Replication (CDC)", - "description": "Logical replication uses the Postgres write-ahead log (WAL) to detect inserts, updates, and deletes. This needs to be configured on the source database itself. Only available on Postgres 10 and above. Read the docs.", + "title": "Read Changes using Write-Ahead Log (CDC)", + "description": "Recommended - Incrementally reads new inserts, updates, and deletes using the Postgres write-ahead log (WAL). This needs to be configured on the source database itself. Recommended for tables of any size.", "required": ["method", "replication_slot", "publication"], "additionalProperties": true, "properties": { @@ -274,11 +264,11 @@ "initial_waiting_seconds": { "type": "integer", "title": "Initial Waiting Time in Seconds (Advanced)", - "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about initial waiting time.", - "default": 300, + "description": "The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 1200 seconds. Valid range: 120 seconds to 2400 seconds. Read about initial waiting time.", + "default": 1200, "order": 5, "min": 120, - "max": 1200 + "max": 2400 }, "queue_size": { "type": "integer", @@ -292,7 +282,7 @@ "lsn_commit_behaviour": { "type": "string", "title": "LSN commit behaviour", - "description": "Determines when Airbtye should flush the LSN of processed WAL logs in the source database. `After loading Data in the destination` is default. If `While reading Data` is selected, in case of a downstream failure (while loading data into the destination), next sync would result in a full sync.", + "description": "Determines when Airbyte should flush the LSN of processed WAL logs in the source database. `After loading Data in the destination` is default. If `While reading Data` is selected, in case of a downstream failure (while loading data into the destination), next sync would result in a full sync.", "enum": [ "While reading Data", "After loading Data in the destination" @@ -303,8 +293,20 @@ } }, { - "title": "Standard", - "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", + "title": "Detect Changes with Xmin System Column", + "description": "Recommended - Incrementally reads new inserts and updates via Postgres Xmin system column. Only recommended for tables up to 500GB.", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "Xmin", + "order": 0 + } + } + }, + { + "title": "Scan Changes with User Defined Cursor", + "description": "Incrementally detects new inserts and updates using the cursor column chosen when configuring a connection (e.g. created_at, updated_at).", "required": ["method"], "properties": { "method": { diff --git a/airbyte-integrations/connectors/source-postgres/src/test-performance/java/io/airbyte/integrations/source/postgres/FillPostgresTestDbScriptTest.java b/airbyte-integrations/connectors/source-postgres/src/test-performance/java/io/airbyte/integrations/source/postgres/FillPostgresTestDbScriptTest.java index b292501de3cd..3de43a56b014 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-performance/java/io/airbyte/integrations/source/postgres/FillPostgresTestDbScriptTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-performance/java/io/airbyte/integrations/source/postgres/FillPostgresTestDbScriptTest.java @@ -6,13 +6,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.cdk.integrations.standardtest.source.performancetest.AbstractSourceFillDbWithTestData; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.integrations.standardtest.source.performancetest.AbstractSourceFillDbWithTestData; import java.util.stream.Stream; import org.jooq.DSLContext; import org.jooq.SQLDialect; diff --git a/airbyte-integrations/connectors/source-postgres/src/test-performance/java/io/airbyte/integrations/source/postgres/PostgresRdsSourcePerformanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-performance/java/io/airbyte/integrations/source/postgres/PostgresRdsSourcePerformanceTest.java index dfaa21e4e777..e0d551e25c4d 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-performance/java/io/airbyte/integrations/source/postgres/PostgresRdsSourcePerformanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-performance/java/io/airbyte/integrations/source/postgres/PostgresRdsSourcePerformanceTest.java @@ -6,10 +6,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.standardtest.source.performancetest.AbstractSourcePerformanceTest; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.standardtest.source.performancetest.AbstractSourcePerformanceTest; import java.nio.file.Path; import java.util.List; import java.util.stream.Stream; diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceLegacyCtidTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceLegacyCtidTest.java new file mode 100644 index 000000000000..a6d7ecb4d970 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceLegacyCtidTest.java @@ -0,0 +1,16 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres; + +import org.junit.jupiter.api.Order; + +@Order(2) +public class CdcPostgresSourceLegacyCtidTest extends CdcPostgresSourceTest { + + protected static String getServerImageName() { + return "debezium/postgres:13-bullseye"; + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java index 9ab452375bc8..3af3817c650a 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java @@ -4,15 +4,19 @@ package io.airbyte.integrations.source.postgres; -import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; -import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_LSN; -import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; +import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_DURATION_PROPERTY; +import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_RECORDS_PROPERTY; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_DELETED_AT; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_LSN; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_UPDATED_AT; import static io.airbyte.integrations.source.postgres.ctid.CtidStateManager.STATE_TYPE_KEY; +import static io.airbyte.integrations.source.postgres.ctid.InitialSyncCtidIteratorConstants.USE_TEST_CHUNK_SIZE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; @@ -21,25 +25,23 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Streams; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.io.IOs; +import io.airbyte.cdk.db.PgLsn; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.debezium.CdcSourceTest; +import io.airbyte.cdk.integrations.debezium.CdcTargetPosition; +import io.airbyte.cdk.integrations.debezium.internals.postgres.PostgresCdcTargetPosition; +import io.airbyte.cdk.integrations.debezium.internals.postgres.PostgresReplicationConnection; +import io.airbyte.cdk.integrations.util.ConnectorExceptionUtil; +import io.airbyte.commons.exceptions.ConfigErrorException; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.string.Strings; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.db.Database; -import io.airbyte.db.PgLsn; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.debezium.CdcSourceTest; -import io.airbyte.integrations.debezium.internals.postgres.PostgresCdcTargetPosition; -import io.airbyte.integrations.debezium.internals.postgres.PostgresReplicationConnection; -import io.airbyte.integrations.util.ConnectorExceptionUtil; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.ContainerModifier; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; @@ -56,8 +58,6 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.StreamDescriptor; import io.airbyte.protocol.models.v0.SyncMode; -import io.airbyte.test.utils.PostgreSQLContainerHelper; -import java.sql.SQLException; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -66,179 +66,111 @@ import java.util.Set; import java.util.stream.Collectors; import javax.sql.DataSource; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.utility.DockerImageName; -import org.testcontainers.utility.MountableFile; -import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; -import uk.org.webcompere.systemstubs.jupiter.SystemStub; -import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; -@ExtendWith(SystemStubsExtension.class) -public class CdcPostgresSourceTest extends CdcSourceTest { +@Order(1) +public class CdcPostgresSourceTest extends CdcSourceTest { - @SystemStub - private EnvironmentVariables environmentVariables; - - protected static final String SLOT_NAME_BASE = "debezium_slot"; - protected static final String PUBLICATION = "publication"; - protected static final int INITIAL_WAITING_SECONDS = 30; - private PostgreSQLContainer container; - - protected String dbName; - protected Database database; - private DSLContext dslContext; - private PostgresSource source; - private JsonNode config; - private String fullReplicationSlot; - private final String cleanUserName = "airbyte_test"; - private final String cleanUserPassword = "password"; + @Override + protected PostgresTestDatabase createTestDatabase() { + return PostgresTestDatabase.in(getServerImage(), ContainerModifier.CONF).withReplicationSlot(); + } - protected String getPluginName() { - return "pgoutput"; + @Override + protected PostgresSource source() { + return new PostgresSource(); } - @AfterEach - void tearDown() { - dslContext.close(); - container.close(); + @Override + protected JsonNode config() { + return testdb.testConfigBuilder() + .withSchemas(modelsSchema(), modelsSchema() + "_random") + .withoutSsl() + .withCdcReplication("After loading Data in the destination") + .with(SYNC_CHECKPOINT_RECORDS_PROPERTY, 1) + .build(); } + @Override @BeforeEach - protected void setup() throws SQLException { - final DockerImageName myImage = DockerImageName.parse("debezium/postgres:13-alpine").asCompatibleSubstituteFor("postgres"); - container = new PostgreSQLContainer<>(myImage) - .withCopyFileToContainer(MountableFile.forClasspathResource("postgresql.conf"), "/etc/postgresql/postgresql.conf") - .withCommand("postgres -c config_file=/etc/postgresql/postgresql.conf"); - container.start(); - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - source = new PostgresSource(); - dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); - - final String initScriptName = "init_" + dbName.concat(".sql"); - final String tmpFilePath = IOs.writeFileToRandomTmpDir(initScriptName, "CREATE DATABASE " + dbName + ";"); - PostgreSQLContainerHelper.runSqlScript(MountableFile.forHostPath(tmpFilePath), container); - - config = getConfig(dbName, container.getUsername(), container.getPassword()); - fullReplicationSlot = SLOT_NAME_BASE + "_" + dbName; - dslContext = getDslContext(config); - database = getDatabase(dslContext); + protected void setup() { super.setup(); - database.query(ctx -> { - ctx.execute("SELECT pg_create_logical_replication_slot('" + fullReplicationSlot + "', '" + getPluginName() + "');"); - ctx.execute("CREATE PUBLICATION " + PUBLICATION + " FOR ALL TABLES;"); - - return null; - }); - - } - - private JsonNode getConfig(final String dbName, final String userName, final String userPassword) { - final JsonNode replicationMethod = getReplicationMethod(dbName); - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, container.getHost()) - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, dbName) - .put(JdbcUtils.SCHEMAS_KEY, List.of(MODELS_SCHEMA, MODELS_SCHEMA + "_random")) - .put(JdbcUtils.USERNAME_KEY, userName) - .put(JdbcUtils.PASSWORD_KEY, userPassword) - .put(JdbcUtils.SSL_KEY, false) - .put("is_test", true) - .put("replication_method", replicationMethod) - .put("sync_checkpoint_records", 1) - .build()); - } - - private JsonNode getReplicationMethod(final String dbName) { - return Jsons.jsonNode(ImmutableMap.builder() - .put("method", "CDC") - .put("replication_slot", SLOT_NAME_BASE + "_" + dbName) - .put("publication", PUBLICATION) - .put("plugin", getPluginName()) - .put("initial_waiting_seconds", INITIAL_WAITING_SECONDS) - .put("lsn_commit_behaviour", "After loading Data in the destination") - .build()); + testdb.withPublicationForAllTables(); } - private static Database getDatabase(final DSLContext dslContext) { - return new Database(dslContext); - } - - private static DSLContext getDslContext(final JsonNode config) { - return DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asInt(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.POSTGRES); - } - - /** - * Creates a new user without privileges for the access tests - */ - private void createCleanUser() { - executeQuery("CREATE USER " + cleanUserName + " PASSWORD '" + cleanUserPassword + "';"); - } - - /** - * Grants privilege to a user (SUPERUSER, REPLICATION, ...) - */ - private void grantUserPrivilege(final String userName, final String postgresPrivilege) { - executeQuery("ALTER USER " + userName + " " + postgresPrivilege + ";"); + @Test + void testDebugMode() { + final JsonNode invalidDebugConfig = testdb.testConfigBuilder() + .withSchemas(modelsSchema(), modelsSchema() + "_random") + .withoutSsl() + .withCdcReplication("While reading Data") + .with(SYNC_CHECKPOINT_RECORDS_PROPERTY, 1) + .with("debug_mode", true) + .build(); + final ConfiguredAirbyteCatalog configuredCatalog = Jsons.clone(getConfiguredCatalog()); + assertThrows(ConfigErrorException.class, () -> source().read(invalidDebugConfig, configuredCatalog, null)); } @Test void testCheckReplicationAccessSuperUserPrivilege() throws Exception { - createCleanUser(); - final JsonNode test_config = getConfig(dbName, cleanUserName, cleanUserPassword); - grantUserPrivilege(cleanUserName, "SUPERUSER"); - final AirbyteConnectionStatus status = source.check(test_config); + final var cleanUserSuperName = testdb.withNamespace("super_user"); + testdb + .with("CREATE USER %s PASSWORD '%s';", cleanUserSuperName, testdb.getPassword()) + .with("ALTER USER %s SUPERUSER;", cleanUserSuperName) + .onClose("DROP OWNED BY %s;", cleanUserSuperName) + .onClose("DROP USER %s;", cleanUserSuperName); + final JsonNode testConfig = config(); + ((ObjectNode) testConfig).put(JdbcUtils.USERNAME_KEY, cleanUserSuperName); + final AirbyteConnectionStatus status = source().check(testConfig); assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, status.getStatus()); } @Test void testCheckReplicationAccessReplicationPrivilege() throws Exception { - createCleanUser(); - final JsonNode test_config = getConfig(dbName, cleanUserName, cleanUserPassword); - grantUserPrivilege(cleanUserName, "REPLICATION"); - final AirbyteConnectionStatus status = source.check(test_config); + final var cleanUserReplicationName = testdb.withNamespace("replication_user"); + testdb + .with("CREATE USER %s PASSWORD '%s';", cleanUserReplicationName, testdb.getPassword()) + .with("ALTER USER %s REPLICATION;", cleanUserReplicationName) + .onClose("DROP OWNED BY %s;", cleanUserReplicationName) + .onClose("DROP USER %s;", cleanUserReplicationName); + final JsonNode testConfig = config(); + ((ObjectNode) testConfig).put(JdbcUtils.USERNAME_KEY, cleanUserReplicationName); + final AirbyteConnectionStatus status = source().check(testConfig); assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, status.getStatus()); } @Test void testCheckWithoutReplicationPermission() throws Exception { - createCleanUser(); - final JsonNode test_config = getConfig(dbName, cleanUserName, cleanUserPassword); - final AirbyteConnectionStatus status = source.check(test_config); + final var cleanUserVanillaName = testdb.withNamespace("vanilla_user"); + testdb + .with("CREATE USER %s PASSWORD '%s';", cleanUserVanillaName, testdb.getPassword()) + .onClose("DROP OWNED BY %s;", cleanUserVanillaName) + .onClose("DROP USER %s;", cleanUserVanillaName); + final JsonNode testConfig = config(); + ((ObjectNode) testConfig).put(JdbcUtils.USERNAME_KEY, cleanUserVanillaName); + final AirbyteConnectionStatus status = source().check(testConfig); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); assertEquals(String.format(ConnectorExceptionUtil.COMMON_EXCEPTION_MESSAGE_TEMPLATE, - String.format(PostgresReplicationConnection.REPLICATION_PRIVILEGE_ERROR_MESSAGE, test_config.get("username").asText())), + String.format(PostgresReplicationConnection.REPLICATION_PRIVILEGE_ERROR_MESSAGE, testConfig.get("username").asText())), status.getMessage()); } @Test void testCheckWithoutPublication() throws Exception { - database.query(ctx -> ctx.execute("DROP PUBLICATION " + PUBLICATION + ";")); - final AirbyteConnectionStatus status = source.check(getConfig()); + testdb.query(ctx -> ctx.execute("DROP PUBLICATION " + testdb.getPublicationName() + ";")); + final AirbyteConnectionStatus status = source().check(config()); assertEquals(status.getStatus(), AirbyteConnectionStatus.Status.FAILED); + testdb.query(ctx -> ctx.execute("CREATE PUBLICATION " + testdb.getPublicationName() + " FOR ALL TABLES;")); } @Test void testCheckWithoutReplicationSlot() throws Exception { - final String fullReplicationSlot = SLOT_NAME_BASE + "_" + dbName; - database.query(ctx -> ctx.execute("SELECT pg_drop_replication_slot('" + fullReplicationSlot + "');")); - - final AirbyteConnectionStatus status = source.check(getConfig()); + testdb.query(ctx -> ctx.execute("SELECT pg_drop_replication_slot('" + testdb.getReplicationSlotName() + "');")); + final AirbyteConnectionStatus status = source().check(config()); assertEquals(status.getStatus(), AirbyteConnectionStatus.Status.FAILED); + testdb.query(ctx -> ctx.execute("SELECT pg_create_logical_replication_slot('" + testdb.getReplicationSlotName() + "', 'pgoutput');")); } @Override @@ -279,7 +211,7 @@ private void assertStateTypes(final List stateMessages, fin @Override protected void assertStateMessagesForNewTableSnapshotTest(final List stateMessages, final AirbyteStateMessage stateMessageEmittedAfterFirstSyncCompletion) { - assertEquals(7, stateMessages.size()); + assertEquals(7, stateMessages.size(), stateMessages.toString()); for (int i = 0; i <= 4; i++) { final AirbyteStateMessage stateMessage = stateMessages.get(i); assertEquals(AirbyteStateMessage.AirbyteStateType.GLOBAL, stateMessage.getType()); @@ -291,14 +223,14 @@ protected void assertStateMessagesForNewTableSnapshotTest(final List { final JsonNode streamState = s.getStreamState(); - if (s.getStreamDescriptor().equals(new StreamDescriptor().withName(MODELS_STREAM_NAME + "_random").withNamespace(randomTableSchema()))) { + if (s.getStreamDescriptor().equals(new StreamDescriptor().withName(MODELS_STREAM_NAME + "_random").withNamespace(randomSchema()))) { assertEquals("ctid", streamState.get(STATE_TYPE_KEY).asText()); - } else if (s.getStreamDescriptor().equals(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(MODELS_SCHEMA))) { + } else if (s.getStreamDescriptor().equals(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(modelsSchema()))) { assertFalse(streamState.has(STATE_TYPE_KEY)); } else { throw new RuntimeException("Unknown stream"); @@ -316,8 +248,8 @@ protected void assertStateMessagesForNewTableSnapshotTest(final List { final JsonNode streamState = s.getStreamState(); assertFalse(streamState.has(STATE_TYPE_KEY)); @@ -334,14 +266,14 @@ protected void assertStateMessagesForNewTableSnapshotTest(final List MODEL_RECORDS_2 = ImmutableList.of( Jsons.jsonNode(ImmutableMap.of(COL_ID, 110, COL_MAKE_ID, 1, COL_MODEL, "Fiesta-2")), @@ -351,18 +283,18 @@ public void testTwoStreamSync() throws Exception { Jsons.jsonNode(ImmutableMap.of(COL_ID, 150, COL_MAKE_ID, 2, COL_MODEL, "A 220-2")), Jsons.jsonNode(ImmutableMap.of(COL_ID, 160, COL_MAKE_ID, 2, COL_MODEL, "E 350-2"))); - createTable(MODELS_SCHEMA, MODELS_STREAM_NAME + "_2", + testdb.with(createTableSqlFmt(), modelsSchema(), MODELS_STREAM_NAME + "_2", columnClause(ImmutableMap.of(COL_ID, "INTEGER", COL_MAKE_ID, "INTEGER", COL_MODEL, "VARCHAR(200)"), Optional.of(COL_ID))); for (final JsonNode recordJson : MODEL_RECORDS_2) { - writeRecords(recordJson, MODELS_SCHEMA, MODELS_STREAM_NAME + "_2", COL_ID, + writeRecords(recordJson, modelsSchema(), MODELS_STREAM_NAME + "_2", COL_ID, COL_MAKE_ID, COL_MODEL); } final ConfiguredAirbyteStream airbyteStream = new ConfiguredAirbyteStream() .withStream(CatalogHelpers.createAirbyteStream( MODELS_STREAM_NAME + "_2", - MODELS_SCHEMA, + modelsSchema(), Field.of(COL_ID, JsonSchemaType.INTEGER), Field.of(COL_MAKE_ID, JsonSchemaType.INTEGER), Field.of(COL_MODEL, JsonSchemaType.STRING)) @@ -375,8 +307,7 @@ public void testTwoStreamSync() throws Exception { streams.add(airbyteStream); configuredCatalog.withStreams(streams); - final AutoCloseableIterator read1 = getSource() - .read(getConfig(), configuredCatalog, null); + final AutoCloseableIterator read1 = source().read(config(), configuredCatalog, null); final List actualRecords1 = AutoCloseableIterators.toListAndClose(read1); final Set recordMessages1 = extractRecordMessages(actualRecords1); @@ -437,13 +368,13 @@ public void testTwoStreamSync() throws Exception { recordMessages1, names, names, - MODELS_SCHEMA); + modelsSchema()); - assertEquals(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(MODELS_SCHEMA), firstStreamInState); + assertEquals(new StreamDescriptor().withName(MODELS_STREAM_NAME).withNamespace(modelsSchema()), firstStreamInState); // Triggering a sync with a ctid state for 1 stream and complete state for other stream - final AutoCloseableIterator read2 = getSource() - .read(getConfig(), configuredCatalog, Jsons.jsonNode(Collections.singletonList(stateMessages1.get(6)))); + final AutoCloseableIterator read2 = source() + .read(config(), configuredCatalog, Jsons.jsonNode(Collections.singletonList(stateMessages1.get(6)))); final List actualRecords2 = AutoCloseableIterators.toListAndClose(read2); final List stateMessages2 = extractStateMessages(actualRecords2); @@ -480,7 +411,7 @@ public void testTwoStreamSync() throws Exception { recordMessages2, names, names, - MODELS_SCHEMA); + modelsSchema()); } @Override @@ -498,13 +429,13 @@ protected void assertExpectedStateMessagesFromIncrementalSync(final List ctx.execute("DROP PUBLICATION " + PUBLICATION + ";")); - database.query(ctx -> ctx.execute(String.format("CREATE PUBLICATION " + PUBLICATION + " FOR TABLE %s.%s", MODELS_SCHEMA, "models"))); + testdb.query(ctx -> ctx.execute("DROP PUBLICATION " + testdb.getPublicationName() + ";")); + testdb + .query(ctx -> ctx.execute(String.format("CREATE PUBLICATION " + testdb.getPublicationName() + " FOR TABLE %s.%s", modelsSchema(), "models"))); - final AirbyteCatalog catalog = source.discover(getConfig()); + final AirbyteCatalog catalog = source().discover(config()); assertEquals(catalog.getStreams().size(), 2); final AirbyteStream streamInPublication = catalog.getStreams().stream().filter(stream -> stream.getName().equals("models")).findFirst().get(); @@ -612,13 +520,15 @@ void testDiscoverFiltersNonPublication() throws Exception { assertEquals(streamNotInPublication.getSupportedSyncModes(), List.of(SyncMode.FULL_REFRESH)); assertTrue(streamNotInPublication.getSourceDefinedPrimaryKey().isEmpty()); assertFalse(streamNotInPublication.getSourceDefinedCursor()); + testdb.query(ctx -> ctx.execute("DROP PUBLICATION " + testdb.getPublicationName() + ";")); + testdb.query(ctx -> ctx.execute("CREATE PUBLICATION " + testdb.getPublicationName() + " FOR ALL TABLES")); } @Test public void testTableWithTimestampColDefault() throws Exception { createAndPopulateTimestampTable(); final AirbyteCatalog catalog = new AirbyteCatalog().withStreams(List.of( - CatalogHelpers.createAirbyteStream("time_stamp_table", MODELS_SCHEMA, + CatalogHelpers.createAirbyteStream("time_stamp_table", modelsSchema(), Field.of("id", JsonSchemaType.NUMBER), Field.of("name", JsonSchemaType.STRING), Field.of("created_at", JsonSchemaType.STRING_TIMESTAMP_WITH_TIMEZONE)) @@ -629,8 +539,8 @@ public void testTableWithTimestampColDefault() throws Exception { // set all streams to incremental. configuredCatalog.getStreams().forEach(s -> s.setSyncMode(SyncMode.INCREMENTAL)); - final AutoCloseableIterator firstBatchIterator = getSource() - .read(getConfig(), configuredCatalog, null); + final AutoCloseableIterator firstBatchIterator = source() + .read(config(), configuredCatalog, null); final List dataFromFirstBatch = AutoCloseableIterators .toListAndClose(firstBatchIterator); final List stateAfterFirstBatch = extractStateMessages(dataFromFirstBatch); @@ -642,7 +552,7 @@ public void testTableWithTimestampColDefault() throws Exception { } private void createAndPopulateTimestampTable() { - createTable(MODELS_SCHEMA, "time_stamp_table", + testdb.with(createTableSqlFmt(), modelsSchema(), "time_stamp_table", columnClause(ImmutableMap.of("id", "INTEGER", "name", "VARCHAR(200)", "created_at", "TIMESTAMPTZ NOT NULL DEFAULT NOW()"), Optional.of("id"))); final List timestampRecords = ImmutableList.of( @@ -662,10 +572,9 @@ private void createAndPopulateTimestampTable() { .jsonNode(ImmutableMap .of("id", 16000, "name", "blah6"))); for (final JsonNode recordJson : timestampRecords) { - executeQuery( - String.format("INSERT INTO %s.%s (%s, %s) VALUES (%s, '%s');", MODELS_SCHEMA, "time_stamp_table", - "id", "name", - recordJson.get("id").asInt(), recordJson.get("name").asText())); + testdb.with("INSERT INTO %s.%s (%s, %s) VALUES (%s, '%s');", modelsSchema(), "time_stamp_table", + "id", "name", + recordJson.get("id").asInt(), recordJson.get("name").asText()); } } @@ -674,30 +583,25 @@ protected void syncShouldHandlePurgedLogsGracefully() throws Exception { final int recordsToCreate = 20; - final JsonNode config = getConfig(); - final JsonNode replicationMethod = ((ObjectNode) getReplicationMethod(config.get(JdbcUtils.DATABASE_KEY).asText())).put("lsn_commit_behaviour", - "While reading Data"); - ((ObjectNode) config).put("replication_method", replicationMethod); - - final AutoCloseableIterator firstBatchIterator = getSource() - .read(config, CONFIGURED_CATALOG, null); + final JsonNode config = testdb.testConfigBuilder() + .withSchemas(modelsSchema(), modelsSchema() + "_random") + .withoutSsl() + .withCdcReplication() + .with(SYNC_CHECKPOINT_RECORDS_PROPERTY, 1) + .build(); + final AutoCloseableIterator firstBatchIterator = source() + .read(config, getConfiguredCatalog(), null); final List dataFromFirstBatch = AutoCloseableIterators .toListAndClose(firstBatchIterator); final List stateAfterFirstBatch = extractStateMessages(dataFromFirstBatch); assertExpectedStateMessages(stateAfterFirstBatch); // second batch of records again 20 being created - for (int recordsCreated = 0; recordsCreated < recordsToCreate; recordsCreated++) { - final JsonNode record = - Jsons.jsonNode(ImmutableMap - .of(COL_ID, 200 + recordsCreated, COL_MAKE_ID, 1, COL_MODEL, - "F-" + recordsCreated)); - writeModelRecord(record); - } + bulkInsertRecords(recordsToCreate); // Extract the last state message final JsonNode state = Jsons.jsonNode(Collections.singletonList(stateAfterFirstBatch.get(stateAfterFirstBatch.size() - 1))); - final AutoCloseableIterator secondBatchIterator = getSource() - .read(config, CONFIGURED_CATALOG, state); + final AutoCloseableIterator secondBatchIterator = source() + .read(config, getConfiguredCatalog(), state); final List dataFromSecondBatch = AutoCloseableIterators .toListAndClose(secondBatchIterator); final List stateAfterSecondBatch = extractStateMessages(dataFromSecondBatch); @@ -713,8 +617,8 @@ protected void syncShouldHandlePurgedLogsGracefully() throws Exception { // Triggering sync with the first sync's state only which would mimic a scenario that the second // sync failed on destination end, and we didn't save state - final AutoCloseableIterator thirdBatchIterator = getSource() - .read(config, CONFIGURED_CATALOG, state); + final AutoCloseableIterator thirdBatchIterator = source() + .read(config, getConfiguredCatalog(), state); final List dataFromThirdBatch = AutoCloseableIterators .toListAndClose(thirdBatchIterator); @@ -745,52 +649,47 @@ void testReachedTargetPosition() { @Test protected void syncShouldIncrementLSN() throws Exception { final int recordsToCreate = 20; + final var postgresSource = source(); final DataSource dataSource = DataSourceFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), + config().get(JdbcUtils.USERNAME_KEY).asText(), + config().get(JdbcUtils.PASSWORD_KEY).asText(), DatabaseDriver.POSTGRESQL.getDriverClassName(), String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asInt(), - config.get(JdbcUtils.DATABASE_KEY).asText())); + config().get(JdbcUtils.HOST_KEY).asText(), + config().get(JdbcUtils.PORT_KEY).asInt(), + config().get(JdbcUtils.DATABASE_KEY).asText())); final JdbcDatabase defaultJdbcDatabase = new DefaultJdbcDatabase(dataSource); final Long replicationSlotAtTheBeginning = PgLsn.fromPgString( - source.getReplicationSlot(defaultJdbcDatabase, getConfig()).get(0).get("confirmed_flush_lsn").asText()).asLong(); + postgresSource.getReplicationSlot(defaultJdbcDatabase, config()).get(0).get("confirmed_flush_lsn").asText()).asLong(); - final AutoCloseableIterator firstBatchIterator = getSource() - .read(getConfig(), CONFIGURED_CATALOG, null); + final AutoCloseableIterator firstBatchIterator = postgresSource + .read(config(), getConfiguredCatalog(), null); final List dataFromFirstBatch = AutoCloseableIterators .toListAndClose(firstBatchIterator); final List stateAfterFirstBatch = extractStateMessages(dataFromFirstBatch); final Long replicationSlotAfterFirstSync = PgLsn.fromPgString( - source.getReplicationSlot(defaultJdbcDatabase, getConfig()).get(0).get("confirmed_flush_lsn").asText()).asLong(); + postgresSource.getReplicationSlot(defaultJdbcDatabase, config()).get(0).get("confirmed_flush_lsn").asText()).asLong(); // First sync should not make any change to the replication slot status assertLsnPositionForSyncShouldIncrementLSN(replicationSlotAtTheBeginning, replicationSlotAfterFirstSync, 1); // second batch of records again 20 being created - for (int recordsCreated = 0; recordsCreated < recordsToCreate; recordsCreated++) { - final JsonNode record = - Jsons.jsonNode(ImmutableMap - .of(COL_ID, 200 + recordsCreated, COL_MAKE_ID, 1, COL_MODEL, - "F-" + recordsCreated)); - writeModelRecord(record); - } + bulkInsertRecords(recordsToCreate); final JsonNode stateAfterFirstSync = Jsons.jsonNode(Collections.singletonList(stateAfterFirstBatch.get(stateAfterFirstBatch.size() - 1))); - final AutoCloseableIterator secondBatchIterator = getSource() - .read(getConfig(), CONFIGURED_CATALOG, stateAfterFirstSync); + final AutoCloseableIterator secondBatchIterator = postgresSource + .read(config(), getConfiguredCatalog(), stateAfterFirstSync); final List dataFromSecondBatch = AutoCloseableIterators .toListAndClose(secondBatchIterator); final List stateAfterSecondBatch = extractStateMessages(dataFromSecondBatch); assertExpectedStateMessagesFromIncrementalSync(stateAfterSecondBatch); final Long replicationSlotAfterSecondSync = PgLsn.fromPgString( - source.getReplicationSlot(defaultJdbcDatabase, getConfig()).get(0).get("confirmed_flush_lsn").asText()).asLong(); + postgresSource.getReplicationSlot(defaultJdbcDatabase, config()).get(0).get("confirmed_flush_lsn").asText()).asLong(); // Second sync should move the replication slot ahead assertLsnPositionForSyncShouldIncrementLSN(replicationSlotAfterFirstSync, replicationSlotAfterSecondSync, 2); @@ -805,8 +704,8 @@ protected void syncShouldIncrementLSN() throws Exception { // Triggering sync with the first sync's state only which would mimic a scenario that the second // sync failed on destination end, and we didn't save state - final AutoCloseableIterator thirdBatchIterator = getSource() - .read(getConfig(), CONFIGURED_CATALOG, stateAfterFirstSync); + final AutoCloseableIterator thirdBatchIterator = postgresSource + .read(config(), getConfiguredCatalog(), stateAfterFirstSync); final List dataFromThirdBatch = AutoCloseableIterators .toListAndClose(thirdBatchIterator); @@ -816,7 +715,7 @@ protected void syncShouldIncrementLSN() throws Exception { dataFromThirdBatch); final Long replicationSlotAfterThirdSync = PgLsn.fromPgString( - source.getReplicationSlot(defaultJdbcDatabase, getConfig()).get(0).get("confirmed_flush_lsn").asText()).asLong(); + postgresSource.getReplicationSlot(defaultJdbcDatabase, config()).get(0).get("confirmed_flush_lsn").asText()).asLong(); // Since we used the state, no change should happen to the replication slot assertEquals(replicationSlotAfterSecondSync, replicationSlotAfterThirdSync); @@ -830,8 +729,9 @@ protected void syncShouldIncrementLSN() throws Exception { writeModelRecord(record); } - final AutoCloseableIterator fourthBatchIterator = getSource() - .read(getConfig(), CONFIGURED_CATALOG, Jsons.jsonNode(Collections.singletonList(stateAfterThirdBatch.get(stateAfterThirdBatch.size() - 1)))); + final AutoCloseableIterator fourthBatchIterator = postgresSource + .read(config(), getConfiguredCatalog(), + Jsons.jsonNode(Collections.singletonList(stateAfterThirdBatch.get(stateAfterThirdBatch.size() - 1)))); final List dataFromFourthBatch = AutoCloseableIterators .toListAndClose(fourthBatchIterator); @@ -841,7 +741,7 @@ protected void syncShouldIncrementLSN() throws Exception { dataFromFourthBatch); final Long replicationSlotAfterFourthSync = PgLsn.fromPgString( - source.getReplicationSlot(defaultJdbcDatabase, getConfig()).get(0).get("confirmed_flush_lsn").asText()).asLong(); + postgresSource.getReplicationSlot(defaultJdbcDatabase, config()).get(0).get("confirmed_flush_lsn").asText()).asLong(); // Fourth sync should again move the replication slot ahead assertEquals(1, replicationSlotAfterFourthSync.compareTo(replicationSlotAfterThirdSync)); @@ -872,8 +772,8 @@ protected void verifyCheckpointStatesByRecords() throws Exception { // We require a huge amount of records, otherwise Debezium will notify directly the last offset. final int recordsToCreate = 20000; - final AutoCloseableIterator firstBatchIterator = getSource() - .read(getConfig(), CONFIGURED_CATALOG, null); + final AutoCloseableIterator firstBatchIterator = source() + .read(config(), getConfiguredCatalog(), null); final List dataFromFirstBatch = AutoCloseableIterators .toListAndClose(firstBatchIterator); final List stateMessages = extractStateMessages(dataFromFirstBatch); @@ -881,18 +781,11 @@ protected void verifyCheckpointStatesByRecords() throws Exception { // As first `read` operation is from snapshot, it would generate only one state message at the end // of the process. assertExpectedStateMessages(stateMessages); - - for (int recordsCreated = 0; recordsCreated < recordsToCreate; recordsCreated++) { - final JsonNode record = - Jsons.jsonNode(ImmutableMap - .of(COL_ID, 200 + recordsCreated, COL_MAKE_ID, 1, COL_MODEL, - "F-" + recordsCreated)); - writeModelRecord(record); - } + bulkInsertRecords(recordsToCreate); final JsonNode stateAfterFirstSync = Jsons.jsonNode(Collections.singletonList(stateMessages.get(stateMessages.size() - 1))); - final AutoCloseableIterator secondBatchIterator = getSource() - .read(getConfig(), CONFIGURED_CATALOG, stateAfterFirstSync); + final AutoCloseableIterator secondBatchIterator = source() + .read(config(), getConfiguredCatalog(), stateAfterFirstSync); final List dataFromSecondBatch = AutoCloseableIterators .toListAndClose(secondBatchIterator); assertEquals(recordsToCreate, extractRecordMessages(dataFromSecondBatch).size()); @@ -908,14 +801,13 @@ protected void verifyCheckpointStatesByRecords() throws Exception { * * @throws Exception Exception happening in the test. */ - @Disabled("Disabled 'verifyCheckpointStatesBySeconds' test as flaky. https://github.com/airbytehq/airbyte/issues/29411") @Test protected void verifyCheckpointStatesBySeconds() throws Exception { // We require a huge amount of records, otherwise Debezium will notify directly the last offset. - final int recordsToCreate = 20000; + final int recordsToCreate = 40000; - final AutoCloseableIterator firstBatchIterator = getSource() - .read(getConfig(), CONFIGURED_CATALOG, null); + final AutoCloseableIterator firstBatchIterator = source() + .read(config(), getConfiguredCatalog(), null); final List dataFromFirstBatch = AutoCloseableIterators .toListAndClose(firstBatchIterator); final List stateMessages = extractStateMessages(dataFromFirstBatch); @@ -923,21 +815,15 @@ protected void verifyCheckpointStatesBySeconds() throws Exception { // As first `read` operation is from snapshot, it would generate only one state message at the end // of the process. assertExpectedStateMessages(stateMessages); + bulkInsertRecords(recordsToCreate); - for (int recordsCreated = 0; recordsCreated < recordsToCreate; recordsCreated++) { - final JsonNode record = - Jsons.jsonNode(ImmutableMap - .of(COL_ID, 200 + recordsCreated, COL_MAKE_ID, 1, COL_MODEL, - "F-" + recordsCreated)); - writeModelRecord(record); - } - final JsonNode config = getConfig(); - ((ObjectNode) config).put("sync_checkpoint_seconds", 1); - ((ObjectNode) config).put("sync_checkpoint_records", 100_000); + final JsonNode config = config(); + ((ObjectNode) config).put(SYNC_CHECKPOINT_DURATION_PROPERTY, 1); + ((ObjectNode) config).put(SYNC_CHECKPOINT_RECORDS_PROPERTY, 100_000); final JsonNode stateAfterFirstSync = Jsons.jsonNode(Collections.singletonList(stateMessages.get(stateMessages.size() - 1))); - final AutoCloseableIterator secondBatchIterator = getSource() - .read(config, CONFIGURED_CATALOG, stateAfterFirstSync); + final AutoCloseableIterator secondBatchIterator = source() + .read(config, getConfiguredCatalog(), stateAfterFirstSync); final List dataFromSecondBatch = AutoCloseableIterators .toListAndClose(secondBatchIterator); @@ -947,4 +833,76 @@ protected void verifyCheckpointStatesBySeconds() throws Exception { assertEquals(stateMessagesCDC.size(), stateMessagesCDC.stream().distinct().count(), "There are duplicated states."); } + /** + * This test is setup to force + * {@link io.airbyte.integrations.source.postgres.ctid.InitialSyncCtidIterator} create multiple + * pages + */ + @Test + protected void ctidIteratorPageSizeTest() throws Exception { + final int recordsToCreate = 25_000; + final Set expectedIds = new HashSet<>(); + MODEL_RECORDS.forEach(c -> expectedIds.add(c.get(COL_ID).asInt())); + + bulkInsertRecords(recordsToCreate); + for (int recordsCreated = 0; recordsCreated < recordsToCreate; recordsCreated++) { + final int id = 200 + recordsCreated; + expectedIds.add(id); + } + + /** + * Setting the property to make the + * {@link io.airbyte.integrations.source.postgres.ctid.InitialSyncCtidIterator} use smaller page + * size of 8KB instead of default 1GB This allows us to make sure that the iterator logic works with + * multiple pages (sub queries) + */ + final JsonNode config = config(); + ((ObjectNode) config).put(USE_TEST_CHUNK_SIZE, true); + final AutoCloseableIterator firstBatchIterator = source() + .read(config, getConfiguredCatalog(), null); + final List dataFromFirstBatch = AutoCloseableIterators + .toListAndClose(firstBatchIterator); + + final Set airbyteRecordMessages = extractRecordMessages(dataFromFirstBatch); + assertEquals(recordsToCreate + MODEL_RECORDS.size(), airbyteRecordMessages.size()); + + airbyteRecordMessages.forEach(c -> { + assertTrue(expectedIds.contains(c.getData().get(COL_ID).asInt())); + expectedIds.remove(c.getData().get(COL_ID).asInt()); + }); + } + + private void bulkInsertRecords(final int recordsToCreate) { + testdb.with(""" + INSERT INTO %s.%s (%s, %s, %s) + SELECT + 200 + generate_series AS id, + 1 AS make_id, + 'F-' || generate_series AS model + FROM generate_series(0, %d - 1); + """, + modelsSchema(), MODELS_STREAM_NAME, + COL_ID, COL_MAKE_ID, COL_MODEL, + recordsToCreate); + } + + @Override + protected void compareTargetPositionFromTheRecordsWithTargetPostionGeneratedBeforeSync(final CdcTargetPosition targetPosition, + final AirbyteRecordMessage record) { + // The LSN from records should be either equal or grater than the position value before the sync + // started. + // The current Write-Ahead Log (WAL) position can move ahead even without any data modifications + // (INSERT, UPDATE, DELETE) + // The start and end of transactions, even read-only ones, are recorded in the WAL. So, simply + // starting and committing a transaction can cause the WAL location to move forward. + // Periodic checkpoints, which write dirty pages from memory to disk to ensure database consistency, + // generate WAL records. Checkpoints happen even if there are no active data modifications + assert targetPosition instanceof PostgresCdcTargetPosition; + assertTrue(extractPosition(record.getData()).targetLsn.compareTo(((PostgresCdcTargetPosition) targetPosition).targetLsn) >= 0); + } + + protected static BaseImage getServerImage() { + return BaseImage.POSTGRES_16; + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdkImportTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdkImportTest.java deleted file mode 100644 index 2e6d71c5667b..000000000000 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdkImportTest.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.postgres; - -import static org.junit.jupiter.api.Assertions.*; - -import io.airbyte.cdk.CDKConstants; -import org.junit.jupiter.api.Test; - -class CdkImportTest { - - /** - * This test ensures that the CDK is able to be imported and that its version number matches the - * expected pinned version. - * - * This test can be removed once pinned CDK version reaches v0.1, at which point the CDK will be - * used for base-java imports, and this test will no longer be necessary. - */ - @Test - void cdkVersionShouldMatch() { - // Should fail in unit test phase: - assertEquals("0.0.2", CDKConstants.VERSION.replace("-SNAPSHOT", "")); - } - -} diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CloudDeploymentPostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CloudDeploymentPostgresSourceTest.java new file mode 100644 index 000000000000..ab69249121a8 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CloudDeploymentPostgresSourceTest.java @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.base.adaptive.AdaptiveSourceRunner; +import io.airbyte.cdk.integrations.base.ssh.SshBastionContainer; +import io.airbyte.cdk.integrations.base.ssh.SshHelpers; +import io.airbyte.cdk.integrations.base.ssh.SshTunnel; +import io.airbyte.commons.features.EnvVariableFeatureFlags; +import io.airbyte.commons.features.FeatureFlagsWrapper; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.ContainerModifier; +import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.Network; + +public class CloudDeploymentPostgresSourceTest { + + static PostgresTestDatabase DB_NO_SSL_WITH_NETWORK, DB_WITH_SSL, DB_WITH_SSL_WITH_NETWORK; + static SshBastionContainer BASTION_NO_SSL, BASTION_WITH_SSL; + static Network NETWORK_NO_SSL, NETWORK_WITH_SSL; + + @BeforeAll + static void setupContainers() { + DB_NO_SSL_WITH_NETWORK = PostgresTestDatabase.in(BaseImage.POSTGRES_16, ContainerModifier.NETWORK); + NETWORK_NO_SSL = DB_NO_SSL_WITH_NETWORK.getContainer().getNetwork(); + BASTION_NO_SSL = new SshBastionContainer(); + BASTION_NO_SSL.initAndStartBastion(NETWORK_NO_SSL); + + DB_WITH_SSL = PostgresTestDatabase.in(BaseImage.POSTGRES_SSL_DEV, ContainerModifier.SSL); + + DB_WITH_SSL_WITH_NETWORK = PostgresTestDatabase.in(BaseImage.POSTGRES_SSL_DEV, ContainerModifier.SSL, ContainerModifier.NETWORK); + NETWORK_WITH_SSL = DB_WITH_SSL_WITH_NETWORK.getContainer().getNetwork(); + BASTION_WITH_SSL = new SshBastionContainer(); + BASTION_WITH_SSL.initAndStartBastion(NETWORK_WITH_SSL); + } + + @AfterAll + static void tearDownContainers() { + BASTION_NO_SSL.stopAndClose(); + BASTION_WITH_SSL.stopAndClose(); + DB_NO_SSL_WITH_NETWORK.close(); + DB_WITH_SSL_WITH_NETWORK.close(); + DB_WITH_SSL.close(); + } + + private static final List NON_STRICT_SSL_MODES = List.of("disable", "allow", "prefer"); + private static final String SSL_MODE_REQUIRE = "require"; + + private Source source() { + PostgresSource source = new PostgresSource(); + source.setFeatureFlags( + FeatureFlagsWrapper.overridingDeploymentMode( + new EnvVariableFeatureFlags(), AdaptiveSourceRunner.CLOUD_MODE)); + return PostgresSource.sshWrappedSource(source); + } + + @Test + void testSSlModesDisableAllowPreferWithTunnelIfServerDoesNotSupportSSL() throws Exception { + for (final String sslmode : NON_STRICT_SSL_MODES) { + final AirbyteConnectionStatus connectionStatus = checkWithTunnel(DB_NO_SSL_WITH_NETWORK, BASTION_NO_SSL, sslmode); + assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatus.getStatus()); + } + } + + @Test + void testSSlModesDisableAllowPreferWithTunnelIfServerSupportSSL() throws Exception { + for (final String sslmode : NON_STRICT_SSL_MODES) { + final AirbyteConnectionStatus connectionStatus = checkWithTunnel(DB_WITH_SSL_WITH_NETWORK, BASTION_WITH_SSL, sslmode); + assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatus.getStatus()); + } + } + + @Test + void testSSlModesDisableAllowPreferWithFailedTunnelIfServerSupportSSL() throws Exception { + for (final String sslmode : NON_STRICT_SSL_MODES) { + final AirbyteConnectionStatus connectionStatus = checkWithTunnel(DB_WITH_SSL, BASTION_WITH_SSL, sslmode); + assertEquals(AirbyteConnectionStatus.Status.FAILED, connectionStatus.getStatus()); + final String msg = connectionStatus.getMessage(); + assertTrue(msg.matches(".*Connection is not available.*|.*The connection attempt failed.*"), msg); + } + } + + @Test + void testSSlRequiredWithTunnelIfServerDoesNotSupportSSL() throws Exception { + final AirbyteConnectionStatus connectionStatus = checkWithTunnel(DB_NO_SSL_WITH_NETWORK, BASTION_NO_SSL, SSL_MODE_REQUIRE); + assertEquals(AirbyteConnectionStatus.Status.FAILED, connectionStatus.getStatus()); + assertEquals("State code: 08004; Message: The server does not support SSL.", connectionStatus.getMessage()); + } + + @Test + void testSSlRequiredNoTunnelIfServerSupportSSL() throws Exception { + final JsonNode config = configBuilderWithSSLMode(DB_WITH_SSL, SSL_MODE_REQUIRE, false).build(); + addNoTunnel((ObjectNode) config); + final AirbyteConnectionStatus connectionStatus = source().check(config); + assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatus.getStatus()); + } + + @Test + void testStrictSSLSecuredWithTunnel() throws Exception { + final AirbyteConnectionStatus connectionStatus = checkWithTunnel(DB_WITH_SSL_WITH_NETWORK, BASTION_WITH_SSL, SSL_MODE_REQUIRE); + assertEquals(AirbyteConnectionStatus.Status.SUCCEEDED, connectionStatus.getStatus()); + } + + private PostgresTestDatabase.PostgresConfigBuilder configBuilderWithSSLMode( + final PostgresTestDatabase db, + final String sslMode, + final boolean innerAddress) { + final var containerAddress = innerAddress + ? SshHelpers.getInnerContainerAddress(db.getContainer()) + : SshHelpers.getOuterContainerAddress(db.getContainer()); + return db.configBuilder() + .with(JdbcUtils.HOST_KEY, Objects.requireNonNull(containerAddress.left)) + .with(JdbcUtils.PORT_KEY, containerAddress.right) + .withDatabase() + .withSchemas("public") + .withCredentials() + .with(JdbcUtils.SSL_MODE_KEY, Map.of(JdbcUtils.MODE_KEY, sslMode)); + } + + private JsonNode getMockedSSLConfig(final String sslMode) { + return Jsons.jsonNode(ImmutableMap.builder() + .put(JdbcUtils.HOST_KEY, "test_host") + .put(JdbcUtils.PORT_KEY, 777) + .put(JdbcUtils.DATABASE_KEY, "test_db") + .put(JdbcUtils.USERNAME_KEY, "test_user") + .put(JdbcUtils.PASSWORD_KEY, "test_password") + .put(JdbcUtils.SSL_KEY, true) + .put(JdbcUtils.SSL_MODE_KEY, Map.of(JdbcUtils.MODE_KEY, sslMode)) + .build()); + } + + @Test + void testSslModesUnsecuredNoTunnel() throws Exception { + for (final String sslMode : NON_STRICT_SSL_MODES) { + final JsonNode config = getMockedSSLConfig(sslMode); + addNoTunnel((ObjectNode) config); + + final AirbyteConnectionStatus connectionStatus = source().check(config); + assertEquals(AirbyteConnectionStatus.Status.FAILED, connectionStatus.getStatus()); + assertTrue(connectionStatus.getMessage().contains("Unsecured connection not allowed"), connectionStatus.getMessage()); + } + } + + private AirbyteConnectionStatus checkWithTunnel(final PostgresTestDatabase db, SshBastionContainer bastion, final String sslmode) throws Exception { + final var configWithSSLModeDisable = configBuilderWithSSLMode(db, sslmode, true) + .with("tunnel_method", bastion.getTunnelMethod(SshTunnel.TunnelMethod.SSH_PASSWORD_AUTH, false)) + .with(JdbcUtils.JDBC_URL_PARAMS_KEY, "connectTimeout=1") + .build(); + return source().check(configWithSSLModeDisable); + } + + private static void addNoTunnel(final ObjectNode config) { + config.putIfAbsent("tunnel_method", Jsons.jsonNode(ImmutableMap.builder() + .put("tunnel_method", "NO_TUNNEL") + .build())); + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelperTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelperTest.java index 599601b1518b..beb0f207087a 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelperTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresCatalogHelperTest.java @@ -4,7 +4,7 @@ package io.airbyte.integrations.source.postgres; -import static io.airbyte.integrations.debezium.internals.DebeziumEventUtils.CDC_LSN; +import static io.airbyte.cdk.integrations.debezium.internals.DebeziumEventUtils.CDC_LSN; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresCdcGetPublicizedTablesTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresCdcGetPublicizedTablesTest.java index ea12de3c60b6..f40b86c5e0f4 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresCdcGetPublicizedTablesTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresCdcGetPublicizedTablesTest.java @@ -7,31 +7,16 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; -import io.airbyte.commons.io.IOs; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.string.Strings; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.ContainerModifier; import io.airbyte.protocol.models.v0.AirbyteStreamNameNamespacePair; -import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.sql.SQLException; -import java.util.List; import java.util.Set; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.utility.DockerImageName; -import org.testcontainers.utility.MountableFile; /** * This class tests the {@link PostgresCatalogHelper#getPublicizedTables} method. @@ -39,103 +24,49 @@ class PostgresCdcGetPublicizedTablesTest { private static final String SCHEMA_NAME = "public"; - private static final String PUBLICATION = "publication_test_12"; - private static final String REPLICATION_SLOT = "replication_slot_test_12"; - protected static final int INITIAL_WAITING_SECONDS = 30; - private static PostgreSQLContainer container; - private JsonNode config; - - @BeforeAll - static void init() { - final DockerImageName myImage = DockerImageName.parse("debezium/postgres:13-alpine").asCompatibleSubstituteFor("postgres"); - container = new PostgreSQLContainer<>(myImage) - .withCopyFileToContainer(MountableFile.forClasspathResource("postgresql.conf"), "/etc/postgresql/postgresql.conf") - .withCommand("postgres -c config_file=/etc/postgresql/postgresql.conf"); - container.start(); - } - - @AfterAll - static void cleanUp() { - container.close(); - } + private PostgresTestDatabase testdb; @BeforeEach - void setup() throws Exception { - final String dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); - final String initScriptName = "init_" + dbName.concat(".sql"); - final String tmpFilePath = IOs.writeFileToRandomTmpDir(initScriptName, "CREATE DATABASE " + dbName + ";"); - PostgreSQLContainerHelper.runSqlScript(MountableFile.forHostPath(tmpFilePath), container); - - this.config = getConfig(container, dbName); - - try (final DSLContext dslContext = getDslContext(config)) { - final Database database = getDatabase(dslContext); - database.query(ctx -> { - ctx.execute("create table table_1 (id serial primary key, text_column text);"); - ctx.execute("create table table_2 (id serial primary key, text_column text);"); - ctx.execute("create table table_irrelevant (id serial primary key, text_column text);"); - ctx.execute("SELECT pg_create_logical_replication_slot('" + REPLICATION_SLOT + "', 'pgoutput');"); - // create a publication including table_1 and table_2, but not table_irrelevant - ctx.execute("CREATE PUBLICATION " + PUBLICATION + " FOR TABLE table_1, table_2;"); - return null; - }); - } - } - - private JsonNode getConfig(final PostgreSQLContainer psqlDb, final String dbName) { - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, psqlDb.getHost()) - .put(JdbcUtils.PORT_KEY, psqlDb.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, dbName) - .put(JdbcUtils.SCHEMAS_KEY, List.of(SCHEMA_NAME)) - .put(JdbcUtils.USERNAME_KEY, psqlDb.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, psqlDb.getPassword()) - .put(JdbcUtils.SSL_KEY, false) - .put("is_test", true) - .build()); + void setup() { + testdb = PostgresTestDatabase.in(BaseImage.POSTGRES_16, ContainerModifier.CONF) + .with("create table table_1 (id serial primary key, text_column text);") + .with("create table table_2 (id serial primary key, text_column text);") + .with("create table table_irrelevant (id serial primary key, text_column text);") + .withReplicationSlot(); + // create a publication including table_1 and table_2, but not table_irrelevant + testdb = testdb + .with("CREATE PUBLICATION %s FOR TABLE table_1, table_2;", testdb.getPublicationName()) + .onClose("DROP PUBLICATION %s CASCADE", testdb.getPublicationName()); } - private static DSLContext getDslContext(final JsonNode config) { - return DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asInt(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.POSTGRES); + @AfterEach + void tearDown() { + testdb.close(); } - private static Database getDatabase(final DSLContext dslContext) { - return new Database(dslContext); + private JsonNode getConfig() { + return testdb.testConfigBuilder().withSchemas(SCHEMA_NAME).withoutSsl().with("is_test", true).build(); } @Test - public void testGetPublicizedTables() { - try (final DSLContext dslContext = getDslContext(config)) { - final JdbcDatabase database = new DefaultJdbcDatabase(dslContext.diagnosticsDataSource()); - // when source config does not exist - assertEquals(0, PostgresCatalogHelper.getPublicizedTables(database).size()); - - // when config is not cdc - database.setSourceConfig(config); - assertEquals(0, PostgresCatalogHelper.getPublicizedTables(database).size()); - - // when config is cdc - ((ObjectNode) config).set("replication_method", Jsons.jsonNode(ImmutableMap.of( - "replication_slot", REPLICATION_SLOT, - "initial_waiting_seconds", INITIAL_WAITING_SECONDS, - "publication", PUBLICATION))); - database.setSourceConfig(config); - final Set expectedTables = Set.of( - new AirbyteStreamNameNamespacePair("table_1", SCHEMA_NAME), - new AirbyteStreamNameNamespacePair("table_2", SCHEMA_NAME)); - // table_irrelevant is not included because it is not part of the publication - assertEquals(expectedTables, PostgresCatalogHelper.getPublicizedTables(database)); - } catch (final SQLException e) { - throw new RuntimeException(e); - } + public void testGetPublicizedTables() throws SQLException { + final JdbcDatabase database = new DefaultJdbcDatabase(testdb.getDslContext().diagnosticsDataSource()); + // when source config does not exist + assertEquals(0, PostgresCatalogHelper.getPublicizedTables(database).size()); + + // when config is not cdc + database.setSourceConfig(getConfig()); + assertEquals(0, PostgresCatalogHelper.getPublicizedTables(database).size()); + + // when config is cdc + final JsonNode cdcConfig = + testdb.testConfigBuilder().withSchemas(SCHEMA_NAME).withoutSsl().withCdcReplication().build(); + database.setSourceConfig(cdcConfig); + final Set expectedTables = Set.of( + new AirbyteStreamNameNamespacePair("table_1", SCHEMA_NAME), + new AirbyteStreamNameNamespacePair("table_2", SCHEMA_NAME)); + // table_irrelevant is not included because it is not part of the publication + assertEquals(expectedTables, PostgresCatalogHelper.getPublicizedTables(database)); } } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresDebugger.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresDebugger.java new file mode 100644 index 000000000000..0865c043a6cb --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresDebugger.java @@ -0,0 +1,17 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres; + +import io.airbyte.cdk.integrations.debug.DebugUtil; + +public class PostgresDebugger { + + @SuppressWarnings({"unchecked", "deprecation", "resource"}) + public static void main(final String[] args) throws Exception { + final PostgresSource postgresSource = new PostgresSource(); + DebugUtil.debug(postgresSource); + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java index 99e402077d30..705cf416fdc4 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java @@ -4,11 +4,8 @@ package io.airbyte.integrations.source.postgres; +import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_RECORDS_PROPERTY; import static io.airbyte.integrations.source.postgres.ctid.CtidStateManager.STATE_TYPE_KEY; -import static io.airbyte.integrations.source.postgres.utils.PostgresUnitTestsUtil.createRecord; -import static io.airbyte.integrations.source.postgres.utils.PostgresUnitTestsUtil.extractSpecificFieldFromCombinedMessages; -import static io.airbyte.integrations.source.postgres.utils.PostgresUnitTestsUtil.extractStateMessage; -import static io.airbyte.integrations.source.postgres.utils.PostgresUnitTestsUtil.filterRecords; import static io.airbyte.integrations.source.postgres.utils.PostgresUnitTestsUtil.map; import static java.util.stream.Collectors.toList; import static org.assertj.core.api.Assertions.assertThat; @@ -20,21 +17,15 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.io.IOs; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import io.airbyte.cdk.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.commons.string.Strings; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.StreamingJdbcDatabase; -import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; import io.airbyte.integrations.source.postgres.internal.models.CursorBasedStatus; import io.airbyte.integrations.source.postgres.internal.models.InternalModels.StateType; -import io.airbyte.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; @@ -49,51 +40,26 @@ import io.airbyte.protocol.models.v0.ConnectorSpecification; import io.airbyte.protocol.models.v0.DestinationSyncMode; import io.airbyte.protocol.models.v0.SyncMode; -import io.airbyte.test.utils.PostgreSQLContainerHelper; -import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; -import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.utility.MountableFile; -import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; -import uk.org.webcompere.systemstubs.jupiter.SystemStub; -import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; -@ExtendWith(SystemStubsExtension.class) -class PostgresJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { - - @SystemStub - private EnvironmentVariables environmentVariables; +class PostgresJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { private static final String DATABASE = "new_db"; protected static final String USERNAME_WITHOUT_PERMISSION = "new_user"; protected static final String PASSWORD_WITHOUT_PERMISSION = "new_password"; - private static PostgreSQLContainer PSQL_DB; public static String COL_WAKEUP_AT = "wakeup_at"; public static String COL_LAST_VISITED_AT = "last_visited_at"; public static String COL_LAST_COMMENT_AT = "last_comment_at"; - @BeforeAll - static void init() { - PSQL_DB = new PostgreSQLContainer<>("postgres:13-alpine"); - PSQL_DB.start(); - } - - @Override - @BeforeEach - public void setup() throws Exception { - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - final String dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); + static { COLUMN_CLAUSE_WITH_PK = "id INTEGER, name VARCHAR(200) NOT NULL, updated_at DATE NOT NULL, wakeup_at TIMETZ NOT NULL, last_visited_at TIMESTAMPTZ NOT NULL, last_comment_at TIMESTAMP NOT NULL"; COLUMN_CLAUSE_WITHOUT_PK = @@ -101,95 +67,73 @@ public void setup() throws Exception { COLUMN_CLAUSE_WITH_COMPOSITE_PK = "first_name VARCHAR(200) NOT NULL, last_name VARCHAR(200) NOT NULL, updated_at DATE NOT NULL, wakeup_at TIMETZ NOT NULL, last_visited_at TIMESTAMPTZ NOT NULL, last_comment_at TIMESTAMP NOT NULL"; - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, PSQL_DB.getHost()) - .put(JdbcUtils.PORT_KEY, PSQL_DB.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, dbName) - .put(JdbcUtils.SCHEMAS_KEY, List.of(SCHEMA_NAME, SCHEMA_NAME2)) - .put(JdbcUtils.USERNAME_KEY, PSQL_DB.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, PSQL_DB.getPassword()) - .put(JdbcUtils.SSL_KEY, false) - .build()); - - final String initScriptName = "init_" + dbName.concat(".sql"); - final String tmpFilePath = IOs.writeFileToRandomTmpDir(initScriptName, "CREATE DATABASE " + dbName + ";"); - PostgreSQLContainerHelper.runSqlScript(MountableFile.forHostPath(tmpFilePath), PSQL_DB); - - source = getSource(); - final JsonNode jdbcConfig = getToDatabaseConfigFunction().apply(config); - - streamName = TABLE_NAME; - - dataSource = DataSourceFactory.create( - jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText(), - jdbcConfig.has(JdbcUtils.PASSWORD_KEY) ? jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, - getDriverClass(), - jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), - JdbcUtils.parseJdbcParameters(jdbcConfig, JdbcUtils.CONNECTION_PROPERTIES_KEY, getJdbcParameterDelimiter())); - - database = new StreamingJdbcDatabase(dataSource, - JdbcUtils.getDefaultSourceOperations(), - AdaptiveStreamingQueryConfig::new); - - createSchemas(); - - database.execute(connection -> { - - connection.createStatement().execute( - createTableQuery(getFullyQualifiedTableName(TABLE_NAME), COLUMN_CLAUSE_WITH_PK, - primaryKeyClause(Collections.singletonList("id")))); - connection.createStatement().execute( - String.format( - "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (1,'picard', '2004-10-19','10:10:10.123456-05:00','2004-10-19T17:23:54.123456Z','2004-01-01T17:23:54.123456')", - getFullyQualifiedTableName(TABLE_NAME))); - connection.createStatement().execute( - String.format( - "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (2, 'crusher', '2005-10-19','11:11:11.123456-05:00','2005-10-19T17:23:54.123456Z','2005-01-01T17:23:54.123456')", - getFullyQualifiedTableName(TABLE_NAME))); - connection.createStatement().execute( - String.format( - "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (3, 'vash', '2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", - getFullyQualifiedTableName(TABLE_NAME))); - - connection.createStatement().execute( - createTableQuery(getFullyQualifiedTableName(TABLE_NAME_WITHOUT_PK), - COLUMN_CLAUSE_WITHOUT_PK, "")); - connection.createStatement().execute( - String.format( - "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (1,'picard', '2004-10-19','12:12:12.123456-05:00','2004-10-19T17:23:54.123456Z','2004-01-01T17:23:54.123456')", - getFullyQualifiedTableName(TABLE_NAME_WITHOUT_PK))); - connection.createStatement().execute( - String.format( - "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (2, 'crusher', '2005-10-19','11:11:11.123456-05:00','2005-10-19T17:23:54.123456Z','2005-01-01T17:23:54.123456')", - getFullyQualifiedTableName(TABLE_NAME_WITHOUT_PK))); - connection.createStatement().execute( - String.format( - "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (3, 'vash', '2006-10-19','10:10:10.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", - getFullyQualifiedTableName(TABLE_NAME_WITHOUT_PK))); - - connection.createStatement().execute( - createTableQuery(getFullyQualifiedTableName(TABLE_NAME_COMPOSITE_PK), - COLUMN_CLAUSE_WITH_COMPOSITE_PK, - primaryKeyClause(ImmutableList.of("first_name", "last_name")))); - connection.createStatement().execute( - String.format( - "INSERT INTO %s(first_name, last_name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES ('first' ,'picard', '2004-10-19','12:12:12.123456-05:00','2004-10-19T17:23:54.123456Z','2004-01-01T17:23:54.123456')", - getFullyQualifiedTableName(TABLE_NAME_COMPOSITE_PK))); - connection.createStatement().execute( - String.format( - "INSERT INTO %s(first_name, last_name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES ('second', 'crusher', '2005-10-19','11:11:11.123456-05:00','2005-10-19T17:23:54.123456Z','2005-01-01T17:23:54.123456')", - getFullyQualifiedTableName(TABLE_NAME_COMPOSITE_PK))); - connection.createStatement().execute( - String.format( - "INSERT INTO %s(first_name, last_name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES ('third', 'vash', '2006-10-19','10:10:10.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", - getFullyQualifiedTableName(TABLE_NAME_COMPOSITE_PK))); - - }); - CREATE_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s BIT(3) NOT NULL);"; INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES(B'101');"; } + @Override + protected JsonNode config() { + return testdb.testConfigBuilder() + .withSchemas(SCHEMA_NAME, SCHEMA_NAME2) + .withoutSsl() + .build(); + } + + @Override + protected PostgresSource source() { + return new PostgresSource(); + } + + @Override + protected PostgresTestDatabase createTestDatabase() { + return PostgresTestDatabase.in(BaseImage.POSTGRES_16); + } + + @Override + @BeforeEach + public void setup() throws Exception { + testdb = createTestDatabase(); + if (supportsSchemas()) { + createSchemas(); + } + testdb.with(createTableQuery(getFullyQualifiedTableName(TABLE_NAME), COLUMN_CLAUSE_WITH_PK, primaryKeyClause(Collections.singletonList("id")))) + .with( + "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (1,'picard', '2004-10-19','10:10:10.123456-05:00','2004-10-19T17:23:54.123456Z','2004-01-01T17:23:54.123456')", + getFullyQualifiedTableName(TABLE_NAME)) + .with( + "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (2, 'crusher', '2005-10-19','11:11:11.123456-05:00','2005-10-19T17:23:54.123456Z','2005-01-01T17:23:54.123456')", + getFullyQualifiedTableName(TABLE_NAME)) + .with( + "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (3, 'vash', '2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", + getFullyQualifiedTableName(TABLE_NAME)) + .with(createTableQuery(getFullyQualifiedTableName(TABLE_NAME_WITHOUT_PK), COLUMN_CLAUSE_WITHOUT_PK, "")) + .with( + "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (1,'picard', '2004-10-19','12:12:12.123456-05:00','2004-10-19T17:23:54.123456Z','2004-01-01T17:23:54.123456')", + getFullyQualifiedTableName(TABLE_NAME_WITHOUT_PK)) + .with( + "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (2, 'crusher', '2005-10-19','11:11:11.123456-05:00','2005-10-19T17:23:54.123456Z','2005-01-01T17:23:54.123456')", + getFullyQualifiedTableName(TABLE_NAME_WITHOUT_PK)) + .with( + "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (3, 'vash', '2006-10-19','10:10:10.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", + getFullyQualifiedTableName(TABLE_NAME_WITHOUT_PK)) + .with(createTableQuery(getFullyQualifiedTableName(TABLE_NAME_COMPOSITE_PK), COLUMN_CLAUSE_WITH_COMPOSITE_PK, + primaryKeyClause(ImmutableList.of("first_name", "last_name")))) + .with( + "INSERT INTO %s(first_name, last_name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES ('first' ,'picard', '2004-10-19','12:12:12.123456-05:00','2004-10-19T17:23:54.123456Z','2004-01-01T17:23:54.123456')", + getFullyQualifiedTableName(TABLE_NAME_COMPOSITE_PK)) + .with( + "INSERT INTO %s(first_name, last_name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES ('second', 'crusher', '2005-10-19','11:11:11.123456-05:00','2005-10-19T17:23:54.123456Z','2005-01-01T17:23:54.123456')", + getFullyQualifiedTableName(TABLE_NAME_COMPOSITE_PK)) + .with( + "INSERT INTO %s(first_name, last_name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES ('third', 'vash', '2006-10-19','10:10:10.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", + getFullyQualifiedTableName(TABLE_NAME_COMPOSITE_PK)); + } + + @Override + protected void maybeSetShorterConnectionTimeout(final JsonNode config) { + ((ObjectNode) config).put(JdbcUtils.JDBC_URL_PARAMS_KEY, "connectTimeout=1"); + } + @Override protected List getAirbyteMessagesReadOneColumn() { return getTestMessages().stream() @@ -290,29 +234,9 @@ public boolean supportsSchemas() { return true; } - @Override - public AbstractJdbcSource getJdbcSource() { - return new PostgresSource(); - } - - @Override - public JsonNode getConfig() { - return config; - } - - @Override - public String getDriverClass() { - return PostgresSource.DRIVER_CLASS; - } - - @AfterAll - static void cleanUp() { - PSQL_DB.close(); - } - @Test void testSpec() throws Exception { - final ConnectorSpecification actual = source.spec(); + final ConnectorSpecification actual = source().spec(); final ConnectorSpecification expected = Jsons.deserialize(MoreResources.readResource("spec.json"), ConnectorSpecification.class); assertEquals(expected, actual); @@ -320,7 +244,7 @@ void testSpec() throws Exception { @Override protected List getTestMessages() { - return getTestMessages(streamName); + return getTestMessages(streamName()); } protected List getTestMessages(final String streamName) { @@ -355,17 +279,13 @@ protected List getTestMessages(final String streamName) { } @Override - protected void executeStatementReadIncrementallyTwice() throws SQLException { - database.execute(connection -> { - connection.createStatement().execute( - String.format( - "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (4,'riker', '2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", - getFullyQualifiedTableName(TABLE_NAME))); - connection.createStatement().execute( - String.format( - "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (5, 'data', '2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", - getFullyQualifiedTableName(TABLE_NAME))); - }); + protected void executeStatementReadIncrementallyTwice() { + testdb.with( + "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (4,'riker', '2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", + getFullyQualifiedTableName(TABLE_NAME)) + .with( + "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (5, 'data', '2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", + getFullyQualifiedTableName(TABLE_NAME)); } @Override @@ -443,11 +363,6 @@ void incrementalTimestampCheck() throws Exception { getTestMessages().get(2))); } - @Override - protected boolean supportsPerStream() { - return true; - } - /** * Postgres Source Error Codes: *

      @@ -458,105 +373,104 @@ protected boolean supportsPerStream() { */ @Test void testCheckIncorrectPasswordFailure() throws Exception { + final var config = config(); + maybeSetShorterConnectionTimeout(config); ((ObjectNode) config).put(JdbcUtils.PASSWORD_KEY, "fake"); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); assertTrue(status.getMessage().contains("State code: 28P01;")); } @Test public void testCheckIncorrectUsernameFailure() throws Exception { + final var config = config(); + maybeSetShorterConnectionTimeout(config); ((ObjectNode) config).put(JdbcUtils.USERNAME_KEY, "fake"); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); assertTrue(status.getMessage().contains("State code: 28P01;")); } @Test public void testCheckIncorrectHostFailure() throws Exception { + final var config = config(); + maybeSetShorterConnectionTimeout(config); ((ObjectNode) config).put(JdbcUtils.HOST_KEY, "localhost2"); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); assertTrue(status.getMessage().contains("State code: 08001;")); } @Test public void testCheckIncorrectPortFailure() throws Exception { + final var config = config(); + maybeSetShorterConnectionTimeout(config); ((ObjectNode) config).put(JdbcUtils.PORT_KEY, "30000"); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); assertTrue(status.getMessage().contains("State code: 08001;")); } @Test public void testCheckIncorrectDataBaseFailure() throws Exception { + final var config = config(); + maybeSetShorterConnectionTimeout(config); ((ObjectNode) config).put(JdbcUtils.DATABASE_KEY, "wrongdatabase"); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); assertTrue(status.getMessage().contains("State code: 3D000;")); } @Test public void testUserHasNoPermissionToDataBase() throws Exception { - database.execute(connection -> connection.createStatement() - .execute(String.format("create user %s with password '%s';", USERNAME_WITHOUT_PERMISSION, PASSWORD_WITHOUT_PERMISSION))); - database.execute(connection -> connection.createStatement() - .execute(String.format("create database %s;", DATABASE))); - // deny access for database for all users from group public - database.execute(connection -> connection.createStatement() - .execute(String.format("revoke all on database %s from public;", DATABASE))); + final var config = config(); + maybeSetShorterConnectionTimeout(config); + testdb.with("create user %s with password '%s';", USERNAME_WITHOUT_PERMISSION, PASSWORD_WITHOUT_PERMISSION) + .with("create database %s;", DATABASE) + // deny access for database for all users from group public + .with("revoke all on database %s from public;", DATABASE); ((ObjectNode) config).put("username", USERNAME_WITHOUT_PERMISSION); ((ObjectNode) config).put("password", PASSWORD_WITHOUT_PERMISSION); ((ObjectNode) config).put("database", DATABASE); - final AirbyteConnectionStatus status = source.check(config); + final AirbyteConnectionStatus status = source().check(config); Assertions.assertEquals(AirbyteConnectionStatus.Status.FAILED, status.getStatus()); assertTrue(status.getMessage().contains("State code: 42501;")); } @Test - void testReadMultipleTablesIncrementally() throws Exception { - ((ObjectNode) config).put("sync_checkpoint_records", 1); + @Override + protected void testReadMultipleTablesIncrementally() throws Exception { + final var config = config(); + ((ObjectNode) config).put(SYNC_CHECKPOINT_RECORDS_PROPERTY, 1); final String namespace = getDefaultNamespace(); final String streamOneName = TABLE_NAME + "one"; // Create a fresh first table - database.execute(connection -> { - connection.createStatement().execute( - createTableQuery(getFullyQualifiedTableName(streamOneName), COLUMN_CLAUSE_WITH_PK, - primaryKeyClause(Collections.singletonList("id")))); - connection.createStatement().execute( - String.format( - "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (1,'picard', '2004-10-19','10:10:10.123456-05:00','2004-10-19T17:23:54.123456Z','2004-01-01T17:23:54.123456')", - getFullyQualifiedTableName(streamOneName))); - connection.createStatement().execute( - String.format( - "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (2, 'crusher', '2005-10-19','11:11:11.123456-05:00','2005-10-19T17:23:54.123456Z','2005-01-01T17:23:54.123456')", - getFullyQualifiedTableName(streamOneName))); - connection.createStatement().execute( - String.format( - "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (3, 'vash', '2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", - getFullyQualifiedTableName(streamOneName))); - }); + testdb.with(createTableQuery(getFullyQualifiedTableName(streamOneName), COLUMN_CLAUSE_WITH_PK, + primaryKeyClause(Collections.singletonList("id")))) + .with( + "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (1,'picard', '2004-10-19','10:10:10.123456-05:00','2004-10-19T17:23:54.123456Z','2004-01-01T17:23:54.123456')", + getFullyQualifiedTableName(streamOneName)) + .with( + "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (2, 'crusher', '2005-10-19','11:11:11.123456-05:00','2005-10-19T17:23:54.123456Z','2005-01-01T17:23:54.123456')", + getFullyQualifiedTableName(streamOneName)) + .with( + "INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at) VALUES (3, 'vash', '2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", + getFullyQualifiedTableName(streamOneName)); // Create a fresh second table final String streamTwoName = TABLE_NAME + "two"; final String streamTwoFullyQualifiedName = getFullyQualifiedTableName(streamTwoName); // Insert records into second table - database.execute(ctx -> { - ctx.createStatement().execute( - createTableQuery(streamTwoFullyQualifiedName, COLUMN_CLAUSE_WITH_PK, "")); - ctx.createStatement().execute( - String.format("INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at)" - + "VALUES (40,'Jean Luc','2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", - streamTwoFullyQualifiedName)); - ctx.createStatement().execute( - String.format("INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at)" - + "VALUES (41, 'Groot', '2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", - streamTwoFullyQualifiedName)); - ctx.createStatement().execute( - String.format("INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at)" - + "VALUES (42, 'Thanos','2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", - streamTwoFullyQualifiedName)); - }); + testdb.with(createTableQuery(streamTwoFullyQualifiedName, COLUMN_CLAUSE_WITH_PK, "")) + .with("INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at)" + + "VALUES (40,'Jean Luc','2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", + streamTwoFullyQualifiedName) + .with("INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at)" + + "VALUES (41, 'Groot', '2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", + streamTwoFullyQualifiedName) + .with(String.format("INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at)" + + "VALUES (42, 'Thanos','2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", + streamTwoFullyQualifiedName)); // Create records list that we expect to see in the state message final List streamTwoExpectedRecords = Arrays.asList( createRecord(streamTwoName, namespace, map( @@ -596,7 +510,7 @@ void testReadMultipleTablesIncrementally() throws Exception { // Perform initial sync final List messagesFromFirstSync = MoreIterators - .toList(source.read(config, configuredCatalog, null)); + .toList(source().read(config, configuredCatalog, null)); final List recordsFromFirstSync = filterRecords(messagesFromFirstSync); @@ -662,7 +576,7 @@ void testReadMultipleTablesIncrementally() throws Exception { // - stream one state still being the first record read via CTID. // - stream two state being the CTID state before the final emitted state before the cursor switch final List messagesFromSecondSyncWithMixedStates = MoreIterators - .toList(source.read(config, configuredCatalog, + .toList(source().read(config, configuredCatalog, Jsons.jsonNode(List.of(streamOneStateMessagesFromFirstSync.get(0), streamTwoStateMessagesFromFirstSync.get(1))))); @@ -689,20 +603,15 @@ void testReadMultipleTablesIncrementally() throws Exception { // Add some data to each table and perform a third read. // Expect to see all records be synced via cursorBased method and not ctid - - database.execute(ctx -> { - ctx.createStatement().execute( - String.format("INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at)" - + "VALUES (4,'Hooper','2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", - getFullyQualifiedTableName(streamOneName))); - ctx.createStatement().execute( - String.format("INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at)" - + "VALUES (43, 'Iron Man', '2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", - streamTwoFullyQualifiedName)); - }); + testdb.with("INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at)" + + "VALUES (4,'Hooper','2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", + getFullyQualifiedTableName(streamOneName)) + .with("INSERT INTO %s(id, name, updated_at, wakeup_at, last_visited_at, last_comment_at)" + + "VALUES (43, 'Iron Man', '2006-10-19','12:12:12.123456-05:00','2006-10-19T17:23:54.123456Z','2006-01-01T17:23:54.123456')", + streamTwoFullyQualifiedName); final List messagesFromThirdSync = MoreIterators - .toList(source.read(config, configuredCatalog, + .toList(source().read(config, configuredCatalog, Jsons.jsonNode(List.of(streamOneStateMessagesFromSecondSync.get(1), streamTwoStateMessagesFromSecondSync.get(0))))); @@ -749,7 +658,7 @@ protected DbStreamState buildStreamState(final ConfiguredAirbyteStream configure protected List getExpectedAirbyteMessagesSecondSync(final String namespace) { final List expectedMessages = new ArrayList<>(); expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + .withRecord(new AirbyteRecordMessage().withStream(streamName()).withNamespace(namespace) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_4, COL_NAME, "riker", @@ -758,7 +667,7 @@ protected List getExpectedAirbyteMessagesSecondSync(final String COL_LAST_VISITED_AT, "2006-10-19T17:23:54.123456Z", COL_LAST_COMMENT_AT, "2006-01-01T17:23:54.123456"))))); expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) + .withRecord(new AirbyteRecordMessage().withStream(streamName()).withNamespace(namespace) .withData(Jsons.jsonNode(ImmutableMap .of(COL_ID, ID_VALUE_5, COL_NAME, "data", @@ -769,13 +678,13 @@ protected List getExpectedAirbyteMessagesSecondSync(final String final DbStreamState state = new CursorBasedStatus() .withStateType(StateType.CURSOR_BASED) .withVersion(2L) - .withStreamName(streamName) + .withStreamName(streamName()) .withStreamNamespace(namespace) .withCursorField(ImmutableList.of(COL_ID)) .withCursor("5") .withCursorRecordCount(1L); - expectedMessages.addAll(createExpectedTestMessages(List.of(state))); + expectedMessages.addAll(createExpectedTestMessages(List.of(state), 2)); return expectedMessages; } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceOperationsTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceOperationsTest.java index 17d34c976cbb..ebf181a8bcb6 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceOperationsTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceOperationsTest.java @@ -10,14 +10,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.jdbc.DateTimeConverter; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DateTimeConverter; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.util.HostPortResolver; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.ContainerModifier; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -26,71 +22,30 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.utility.MountableFile; class PostgresSourceOperationsTest { private final PostgresSourceOperations postgresSourceOperations = new PostgresSourceOperations(); - private PostgreSQLContainer container; - private Database database; + private PostgresTestDatabase testdb; private final String cursorColumn = "cursor_column"; @BeforeEach - public void init() throws SQLException { - container = new PostgreSQLContainer<>("postgres:14-alpine") - .withCopyFileToContainer(MountableFile.forClasspathResource("postgresql.conf"), - "/etc/postgresql/postgresql.conf") - .withCommand("postgres -c config_file=/etc/postgresql/postgresql.conf"); - container.start(); - final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("method", "Standard") - .build()); - final JsonNode config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, HostPortResolver.resolveHost(container)) - .put(JdbcUtils.PORT_KEY, HostPortResolver.resolvePort(container)) - .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) - .put(JdbcUtils.USERNAME_KEY, container.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put(JdbcUtils.SSL_KEY, false) - .put("replication_method", replicationMethod) - .build()); - - final DSLContext dslContext = DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), - container.getHost(), - container.getFirstMappedPort(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.POSTGRES); - database = new Database(dslContext); - database.query(ctx -> { - ctx.execute(String.format("CREATE SCHEMA %S;", container.getDatabaseName())); - return null; - }); + public void init() { + testdb = PostgresTestDatabase.in(BaseImage.POSTGRES_16, ContainerModifier.CONF); } @AfterEach public void tearDown() { - try { - - container.close(); - } catch (final Exception e) { - throw new RuntimeException(e); - } + testdb.close(); } @Test public void numericColumnAsCursor() throws SQLException { - final String tableName = container.getDatabaseName() + ".numeric_table"; + final String tableName = "numeric_table"; final String createTableQuery = String.format("CREATE TABLE %s(id INTEGER PRIMARY KEY, %s NUMERIC(38, 0));", tableName, cursorColumn); @@ -110,9 +65,9 @@ public void numericColumnAsCursor() throws SQLException { } final List actualRecords = new ArrayList<>(); - try (final Connection connection = container.createConnection("")) { + try (final Connection connection = testdb.getContainer().createConnection("")) { final PreparedStatement preparedStatement = connection.prepareStatement( - "SELECT * from " + tableName + " WHERE " + cursorColumn + " > ?"); + "SELECT * FROM " + tableName + " WHERE " + cursorColumn + " > ?"); postgresSourceOperations.setCursorField(preparedStatement, 1, PostgresType.NUMERIC, @@ -134,7 +89,7 @@ public void numericColumnAsCursor() throws SQLException { @Test public void timeColumnAsCursor() throws SQLException { - final String tableName = container.getDatabaseName() + ".time_table"; + final String tableName = "time_table"; final String createTableQuery = String.format("CREATE TABLE %s(id INTEGER PRIMARY KEY, %s TIME);", tableName, cursorColumn); @@ -150,7 +105,7 @@ public void timeColumnAsCursor() throws SQLException { } final List actualRecords = new ArrayList<>(); - try (final Connection connection = container.createConnection("")) { + try (final Connection connection = testdb.getContainer().createConnection("")) { final PreparedStatement preparedStatement = connection.prepareStatement( "SELECT * from " + tableName + " WHERE " + cursorColumn + " > ?"); postgresSourceOperations.setCursorField(preparedStatement, @@ -182,13 +137,9 @@ public void testParseMoneyValue() { assertEquals("-1000000.001", PostgresSourceOperations.parseMoneyValue("-£1,000,000.001")); } - protected void executeQuery(final String query) { - try { - database.query( - ctx -> ctx - .execute(query)); - } catch (final SQLException e) { - throw new RuntimeException(e); + protected void executeQuery(final String query) throws SQLException { + try (final Connection connection = testdb.getContainer().createConnection("")) { + connection.createStatement().execute(query); } } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceSSLTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceSSLTest.java index 3baa95997ea5..b4bad09de924 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceSSLTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceSSLTest.java @@ -16,14 +16,11 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import io.airbyte.commons.io.IOs; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.string.Strings; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.ContainerModifier; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; @@ -33,22 +30,15 @@ import io.airbyte.protocol.models.v0.CatalogHelpers; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.SyncMode; -import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.math.BigDecimal; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.utility.DockerImageName; -import org.testcontainers.utility.MountableFile; class PostgresSourceSSLTest { @@ -84,89 +74,37 @@ class PostgresSourceSSLTest { createRecord(STREAM_NAME, map("id", new BigDecimal("2.0"), "name", "vegeta", "power", 9000.1), SCHEMA_NAME), createRecord(STREAM_NAME, map("id", null, "name", "piccolo", "power", null), SCHEMA_NAME)); - private static PostgreSQLContainer PSQL_DB; - - private String dbName; - - @BeforeAll - static void init() { - PSQL_DB = new PostgreSQLContainer<>(DockerImageName.parse("marcosmarxm/postgres-ssl:dev").asCompatibleSubstituteFor("postgres")) - .withCommand("postgres -c ssl=on -c ssl_cert_file=/var/lib/postgresql/server.crt -c ssl_key_file=/var/lib/postgresql/server.key"); - PSQL_DB.start(); - } + private PostgresTestDatabase testdb; @BeforeEach void setup() throws Exception { - dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); - - final String initScriptName = "init_" + dbName.concat(".sql"); - final String tmpFilePath = IOs.writeFileToRandomTmpDir(initScriptName, "CREATE DATABASE " + dbName + ";"); - PostgreSQLContainerHelper.runSqlScript(MountableFile.forHostPath(tmpFilePath), PSQL_DB); - - final JsonNode config = getConfig(PSQL_DB, dbName); - try (final DSLContext dslContext = getDslContext(config)) { - final Database database = getDatabase(dslContext); - database.query(ctx -> { - ctx.fetch( - "CREATE TABLE id_and_name(id NUMERIC(20, 10) NOT NULL, name VARCHAR(200) NOT NULL, power double precision NOT NULL, PRIMARY KEY (id));"); - ctx.fetch("CREATE INDEX i1 ON id_and_name (id);"); - ctx.fetch( - "INSERT INTO id_and_name (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); - - ctx.fetch("CREATE TABLE id_and_name2(id NUMERIC(20, 10) NOT NULL, name VARCHAR(200) NOT NULL, power double precision NOT NULL);"); - ctx.fetch( - "INSERT INTO id_and_name2 (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); - - ctx.fetch( - "CREATE TABLE names(first_name VARCHAR(200) NOT NULL, last_name VARCHAR(200) NOT NULL, power double precision NOT NULL, PRIMARY KEY (first_name, last_name));"); - ctx.fetch( + testdb = PostgresTestDatabase.in(BaseImage.POSTGRES_SSL_DEV, ContainerModifier.SSL) + .with("CREATE TABLE id_and_name(id NUMERIC(20, 10) NOT NULL, name VARCHAR(200) NOT NULL, power double precision NOT NULL, PRIMARY KEY (id));") + .with("CREATE INDEX i1 ON id_and_name (id);") + .with("INSERT INTO id_and_name (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');") + .with("CREATE TABLE id_and_name2(id NUMERIC(20, 10) NOT NULL, name VARCHAR(200) NOT NULL, power double precision NOT NULL);") + .with("INSERT INTO id_and_name2 (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');") + .with( + "CREATE TABLE names(first_name VARCHAR(200) NOT NULL, last_name VARCHAR(200) NOT NULL, power double precision NOT NULL, PRIMARY KEY (first_name, last_name));") + .with( "INSERT INTO names (first_name, last_name, power) VALUES ('san', 'goku', 'Infinity'), ('prince', 'vegeta', 9000.1), ('piccolo', 'junior', '-Infinity');"); - return null; - }); - } - } - - private static Database getDatabase(final DSLContext dslContext) { - return new Database(dslContext); - } - - private static DSLContext getDslContext(final JsonNode config) { - return DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asInt(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.POSTGRES); - } - - private JsonNode getConfig(final PostgreSQLContainer psqlDb, final String dbName) { - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, psqlDb.getHost()) - .put(JdbcUtils.PORT_KEY, psqlDb.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, dbName) - .put(JdbcUtils.SCHEMAS_KEY, List.of("public")) - .put(JdbcUtils.USERNAME_KEY, psqlDb.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, psqlDb.getPassword()) - .put(JdbcUtils.SSL_KEY, true) - .put("ssl_mode", ImmutableMap.builder().put("mode", "require").build()) - .build()); } - private JsonNode getConfig(final PostgreSQLContainer psqlDb) { - return getConfig(psqlDb, psqlDb.getDatabaseName()); + @AfterEach + void tearDown() { + testdb.close(); } - @AfterAll - static void cleanUp() { - PSQL_DB.close(); + private JsonNode getConfig() { + return testdb.testConfigBuilder() + .withSchemas("public") + .withSsl(ImmutableMap.builder().put("mode", "require").build()) + .build(); } @Test void testDiscoverWithPk() throws Exception { - final AirbyteCatalog actual = new PostgresSource().discover(getConfig(PSQL_DB, dbName)); + final AirbyteCatalog actual = new PostgresSource().discover(getConfig()); actual.getStreams().forEach(actualStream -> { final Optional expectedStream = CATALOG.getStreams().stream().filter(stream -> stream.getName().equals(actualStream.getName())).findAny(); @@ -181,7 +119,7 @@ void testReadSuccess() throws Exception { CONFIGURED_CATALOG.withStreams(CONFIGURED_CATALOG.getStreams().stream().filter(s -> s.getStream().getName().equals(STREAM_NAME)) .collect(Collectors.toList())); - final Set actualMessages = MoreIterators.toSet(new PostgresSource().read(getConfig(PSQL_DB, dbName), configuredCatalog, null)); + final Set actualMessages = MoreIterators.toSet(new PostgresSource().read(getConfig(), configuredCatalog, null)); setEmittedAtToNull(actualMessages); assertEquals(ASCII_MESSAGES, actualMessages); @@ -189,7 +127,7 @@ void testReadSuccess() throws Exception { @Test void testIsCdc() { - final JsonNode config = getConfig(PSQL_DB, dbName); + final JsonNode config = getConfig(); assertFalse(PostgresUtils.isCdc(config)); diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java index 6ef7d6a0454c..ff5e167fc5ad 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSourceTest.java @@ -18,20 +18,19 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.IncrementalUtils; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.source.relationaldb.CursorInfo; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManager; +import io.airbyte.cdk.integrations.source.relationaldb.state.StateManagerFactory; import io.airbyte.commons.exceptions.ConfigErrorException; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.string.Strings; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.db.Database; -import io.airbyte.db.IncrementalUtils; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.source.relationaldb.CursorInfo; -import io.airbyte.integrations.source.relationaldb.state.StateManager; -import io.airbyte.integrations.source.relationaldb.state.StateManagerFactory; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.ContainerModifier; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaPrimitiveUtil.JsonSchemaPrimitive; import io.airbyte.protocol.models.JsonSchemaType; @@ -47,7 +46,6 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; import io.airbyte.protocol.models.v0.SyncMode; -import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.math.BigDecimal; import java.sql.SQLException; import java.util.ArrayList; @@ -59,22 +57,12 @@ import java.util.stream.Collectors; import org.jooq.DSLContext; import org.jooq.SQLDialect; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.utility.MountableFile; -import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; -import uk.org.webcompere.systemstubs.jupiter.SystemStub; -import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; - -@ExtendWith(SystemStubsExtension.class) -class PostgresSourceTest { - @SystemStub - private EnvironmentVariables environmentVariables; +class PostgresSourceTest { private static final String SCHEMA_NAME = "public"; private static final String STREAM_NAME = "id_and_name"; @@ -143,47 +131,29 @@ class PostgresSourceTest { createRecord(STREAM_NAME_PRIVILEGES_TEST_CASE_VIEW, SCHEMA_NAME, ImmutableMap.of("id", 2, "name", "Jack")), createRecord(STREAM_NAME_PRIVILEGES_TEST_CASE_VIEW, SCHEMA_NAME, ImmutableMap.of("id", 3, "name", "Antuan"))); - private static PostgreSQLContainer PSQL_DB; - - private String dbName; - - @BeforeAll - static void init() { - PSQL_DB = new PostgreSQLContainer<>("postgres:13-alpine"); - PSQL_DB.start(); - } + private PostgresTestDatabase testdb; @BeforeEach - void setup() throws Exception { - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); - - final String initScriptName = "init_" + dbName.concat(".sql"); - final String tmpFilePath = IOs.writeFileToRandomTmpDir(initScriptName, "CREATE DATABASE " + dbName + ";"); - PostgreSQLContainerHelper.runSqlScript(MountableFile.forHostPath(tmpFilePath), PSQL_DB); + void setup() { + testdb = PostgresTestDatabase.in(BaseImage.POSTGRES_16) + .with("CREATE TABLE id_and_name(id NUMERIC(20, 10) NOT NULL, name VARCHAR(200) NOT NULL, power double precision NOT NULL, PRIMARY KEY (id));") + .with("CREATE INDEX i1 ON id_and_name (id);") + .with("INSERT INTO id_and_name (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');") + .with("CREATE TABLE id_and_name2(id NUMERIC(20, 10) NOT NULL, name VARCHAR(200) NOT NULL, power double precision NOT NULL);") + .with("INSERT INTO id_and_name2 (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');") + .with( + "CREATE TABLE names(first_name VARCHAR(200) NOT NULL, last_name VARCHAR(200) NOT NULL, power double precision NOT NULL, PRIMARY KEY (first_name, last_name));") + .with("INSERT INTO names (first_name, last_name, power) VALUES ('san', 'goku', 'Infinity'), ('prince', " + + "'vegeta', 9000.1), ('piccolo', 'junior', '-Infinity');"); + } - final JsonNode config = getConfig(PSQL_DB, dbName); + @AfterEach + void tearDown() { + testdb.close(); + } - try (final DSLContext dslContext = getDslContext(config)) { - final Database database = getDatabase(dslContext); - database.query(ctx -> { - ctx.fetch( - "CREATE TABLE id_and_name(id NUMERIC(20, 10) NOT NULL, name VARCHAR(200) NOT NULL, power double precision NOT NULL, PRIMARY KEY (id));"); - ctx.fetch("CREATE INDEX i1 ON id_and_name (id);"); - ctx.fetch( - "INSERT INTO id_and_name (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); - - ctx.fetch("CREATE TABLE id_and_name2(id NUMERIC(20, 10) NOT NULL, name VARCHAR(200) NOT NULL, power double precision NOT NULL);"); - ctx.fetch( - "INSERT INTO id_and_name2 (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); - - ctx.fetch( - "CREATE TABLE names(first_name VARCHAR(200) NOT NULL, last_name VARCHAR(200) NOT NULL, power double precision NOT NULL, PRIMARY KEY (first_name, last_name));"); - ctx.fetch( - "INSERT INTO names (first_name, last_name, power) VALUES ('san', 'goku', 'Infinity'), ('prince', 'vegeta', 9000.1), ('piccolo', 'junior', '-Infinity');"); - return null; - }); - } + public PostgresSource source() { + return new PostgresSource(); } private static DSLContext getDslContextWithSpecifiedUser(final JsonNode config, final String username, final String password) { @@ -202,34 +172,18 @@ private static Database getDatabase(final DSLContext dslContext) { return new Database(dslContext); } - private static DSLContext getDslContext(final JsonNode config) { - return DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asInt(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.POSTGRES); + private JsonNode getConfig() { + return getConfig(testdb.getUserName(), testdb.getPassword()); } - private JsonNode getConfig(final PostgreSQLContainer psqlDb, final String dbName) { - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, psqlDb.getHost()) - .put(JdbcUtils.PORT_KEY, psqlDb.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, dbName) - .put(JdbcUtils.SCHEMAS_KEY, List.of(SCHEMA_NAME)) - .put(JdbcUtils.USERNAME_KEY, psqlDb.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, psqlDb.getPassword()) - .put(JdbcUtils.SSL_KEY, false) - .build()); + private JsonNode getConfig(final String user, final String password) { + return getConfig(testdb.getDatabaseName(), user, password); } - private JsonNode getConfig(final PostgreSQLContainer psqlDb, final String dbName, final String user, final String password) { + private JsonNode getConfig(final String dbName, final String user, final String password) { return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, psqlDb.getHost()) - .put(JdbcUtils.PORT_KEY, psqlDb.getFirstMappedPort()) + .put(JdbcUtils.HOST_KEY, testdb.getContainer().getHost()) + .put(JdbcUtils.PORT_KEY, testdb.getContainer().getFirstMappedPort()) .put(JdbcUtils.DATABASE_KEY, dbName) .put(JdbcUtils.SCHEMAS_KEY, List.of(SCHEMA_NAME)) .put(JdbcUtils.USERNAME_KEY, user) @@ -238,118 +192,85 @@ private JsonNode getConfig(final PostgreSQLContainer psqlDb, final String dbN .build()); } - private JsonNode getConfig(final PostgreSQLContainer psqlDb, final String user, final String password) { - return getConfig(psqlDb, psqlDb.getDatabaseName(), user, password); - } - - private JsonNode getConfig(final PostgreSQLContainer psqlDb) { - return getConfig(psqlDb, psqlDb.getDatabaseName()); - } - - @AfterAll - static void cleanUp() { - PSQL_DB.close(); - } - @Test public void testCanReadTablesAndColumnsWithDoubleQuotes() throws Exception { - try (final PostgreSQLContainer db = new PostgreSQLContainer<>("postgres:13-alpine")) { - db.start(); - - final AirbyteCatalog airbyteCatalog = new AirbyteCatalog().withStreams(List.of( - CatalogHelpers.createAirbyteStream( - STREAM_NAME_WITH_QUOTES, - SCHEMA_NAME, - Field.of("id", JsonSchemaType.NUMBER), - Field.of("\"test_column\"", JsonSchemaType.STRING)) - .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) - .withSourceDefinedPrimaryKey(List.of(List.of("id"))))); - - final JsonNode config = getConfig(db); - try (final DSLContext dslContext = getDslContext(config)) { - final Database database = getDatabase(dslContext); - - database.query(ctx -> { - ctx.fetch("CREATE TABLE \"\"\"test_dq_table\"\"\"(id INTEGER PRIMARY KEY, \"\"\"test_column\"\"\" varchar);"); - ctx.fetch("INSERT INTO \"\"\"test_dq_table\"\"\" (id, \"\"\"test_column\"\"\") VALUES (1,'test1'), (2, 'test2');"); - return null; - }); - } - final Set actualMessages = - MoreIterators.toSet(new PostgresSource().read(config, CatalogHelpers.toDefaultConfiguredCatalog(airbyteCatalog), null)); - setEmittedAtToNull(actualMessages); - - assertEquals(DOUBLE_QUOTED_MESSAGES, actualMessages); - db.stop(); - } + final AirbyteCatalog airbyteCatalog = new AirbyteCatalog().withStreams(List.of( + CatalogHelpers.createAirbyteStream( + STREAM_NAME_WITH_QUOTES, + SCHEMA_NAME, + Field.of("id", JsonSchemaType.NUMBER), + Field.of("\"test_column\"", JsonSchemaType.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of("id"))))); + testdb.query(ctx -> { + ctx.fetch("CREATE TABLE \"\"\"test_dq_table\"\"\"(id INTEGER PRIMARY KEY, \"\"\"test_column\"\"\" varchar);"); + ctx.fetch("INSERT INTO \"\"\"test_dq_table\"\"\" (id, \"\"\"test_column\"\"\") VALUES (1,'test1'), (2, 'test2');"); + return null; + }); + final Set actualMessages = + MoreIterators.toSet(source().read( + getConfig(), + CatalogHelpers.toDefaultConfiguredCatalog(airbyteCatalog), + null)); + setEmittedAtToNull(actualMessages); + assertEquals(DOUBLE_QUOTED_MESSAGES, actualMessages); + testdb.query(ctx -> ctx.execute("DROP TABLE \"\"\"test_dq_table\"\"\";")); } @Test public void testCanReadUtf8() throws Exception { // force the db server to start with sql_ascii encoding to verify the source can read UTF8 even when // default settings are in another encoding - try (final PostgreSQLContainer db = new PostgreSQLContainer<>("postgres:13-alpine").withCommand("postgres -c client_encoding=sql_ascii")) { - db.start(); - final JsonNode config = getConfig(db); - try (final DSLContext dslContext = getDslContext(config)) { - final Database database = getDatabase(dslContext); - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,E'\\u2013 someutfstring'), (2, E'\\u2215');"); - return null; - }); - } - - final Set actualMessages = MoreIterators.toSet(new PostgresSource().read(config, CONFIGURED_CATALOG, null)); + try (final var asciiTestDB = PostgresTestDatabase.in(BaseImage.POSTGRES_16, ContainerModifier.ASCII) + .with("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));") + .with("INSERT INTO id_and_name (id, name) VALUES (1,E'\\u2013 someutfstring'), (2, E'\\u2215');")) { + final var config = asciiTestDB.testConfigBuilder().withSchemas(SCHEMA_NAME).withoutSsl().build(); + final Set actualMessages = MoreIterators.toSet(source().read(config, CONFIGURED_CATALOG, null)); setEmittedAtToNull(actualMessages); - assertEquals(UTF8_MESSAGES, actualMessages); - db.stop(); } } @Test void testUserDoesntHasPrivilegesToSelectTable() throws Exception { - try (final PostgreSQLContainer db = new PostgreSQLContainer<>("postgres:13-alpine")) { - db.start(); - final JsonNode config = getConfig(db); - try (final DSLContext dslContext = getDslContext(config)) { - final Database database = new Database(dslContext); - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'John'), (2, 'Alfred'), (3, 'Alex');"); - ctx.fetch("CREATE USER test_user_3 password '132';"); - ctx.fetch("GRANT CONNECT ON DATABASE test TO test_user_3;"); - ctx.fetch("REVOKE ALL PRIVILEGES ON TABLE public.id_and_name FROM test_user_3"); - return null; - }); - } - try (final DSLContext dslContext = getDslContextWithSpecifiedUser(config, "test_user_3", "132")) { - final Database database = new Database(dslContext); - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name_3(id INTEGER, name VARCHAR(200));"); - ctx.fetch("CREATE VIEW id_and_name_3_view(id, name) as\n" - + "SELECT id_and_name_3.id,\n" - + " id_and_name_3.name\n" - + "FROM id_and_name_3;\n" - + "ALTER TABLE id_and_name_3_view\n" - + " owner TO test_user_3"); - ctx.fetch("INSERT INTO id_and_name_3 (id, name) VALUES (1,'Zed'), (2, 'Jack'), (3, 'Antuan');"); - return null; - }); - } - final JsonNode anotherUserConfig = getConfig(db, "test_user_3", "132"); - final Set actualMessages = MoreIterators.toSet(new PostgresSource().read(anotherUserConfig, CONFIGURED_CATALOG, null)); - setEmittedAtToNull(actualMessages); - assertEquals(6, actualMessages.size()); - assertEquals(PRIVILEGE_TEST_CASE_EXPECTED_MESSAGES, actualMessages); - db.stop(); + testdb.query(ctx -> { + ctx.execute("DROP TABLE id_and_name CASCADE;"); + ctx.execute("DROP TABLE id_and_name2 CASCADE;"); + ctx.execute("DROP TABLE names CASCADE;"); + ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); + ctx.fetch("INSERT INTO id_and_name (id, name) VALUES (1,'John'), (2, 'Alfred'), (3, 'Alex');"); + ctx.fetch("CREATE USER test_user_3 password '132';"); + ctx.fetch("GRANT CONNECT ON DATABASE " + testdb.getDatabaseName() + " TO test_user_3;"); + ctx.fetch("GRANT ALL ON SCHEMA public TO test_user_3"); + ctx.fetch("REVOKE ALL PRIVILEGES ON TABLE public.id_and_name FROM test_user_3"); + return null; + }); + final JsonNode config = getConfig(); + try (final DSLContext dslContext = getDslContextWithSpecifiedUser(config, "test_user_3", "132")) { + final Database database = new Database(dslContext); + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name_3(id INTEGER, name VARCHAR(200));"); + ctx.fetch("CREATE VIEW id_and_name_3_view(id, name) as\n" + + "SELECT id_and_name_3.id,\n" + + " id_and_name_3.name\n" + + "FROM id_and_name_3;\n" + + "ALTER TABLE id_and_name_3_view\n" + + " owner TO test_user_3"); + ctx.fetch("INSERT INTO id_and_name_3 (id, name) VALUES (1,'Zed'), (2, 'Jack'), (3, 'Antuan');"); + return null; + }); } + final JsonNode anotherUserConfig = getConfig("test_user_3", "132"); + final Set actualMessages = + MoreIterators.toSet(source().read(anotherUserConfig, CONFIGURED_CATALOG, null)); + setEmittedAtToNull(actualMessages); + assertEquals(6, actualMessages.size()); + assertEquals(PRIVILEGE_TEST_CASE_EXPECTED_MESSAGES, actualMessages); } @Test void testDiscoverWithPk() throws Exception { - final AirbyteCatalog actual = new PostgresSource().discover(getConfig(PSQL_DB, dbName)); + final AirbyteCatalog actual = source().discover(getConfig()); actual.getStreams().forEach(actualStream -> { final Optional expectedStream = CATALOG.getStreams().stream().filter(stream -> stream.getName().equals(actualStream.getName())).findAny(); @@ -360,150 +281,150 @@ void testDiscoverWithPk() throws Exception { @Test void testDiscoverRecursiveRolePermissions() throws Exception { - try (final PostgreSQLContainer db = new PostgreSQLContainer<>("postgres:13-alpine")) { - db.start(); - final JsonNode config = getConfig(db); - try (final DSLContext dslContext = getDslContext(config)) { - final Database database = new Database(dslContext); - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name_7(id INTEGER, name VARCHAR(200));"); - ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); + testdb.query(ctx -> { + ctx.execute("DROP TABLE id_and_name CASCADE;"); + ctx.execute("DROP TABLE id_and_name2 CASCADE;"); + ctx.execute("DROP TABLE names CASCADE;"); + ctx.fetch("CREATE TABLE id_and_name_7(id INTEGER, name VARCHAR(200));"); + ctx.fetch("CREATE TABLE id_and_name(id INTEGER, name VARCHAR(200));"); - ctx.fetch("CREATE USER test_user_4 password '132';"); + ctx.fetch("CREATE USER test_user_4 password '132';"); + ctx.fetch("GRANT ALL ON SCHEMA public TO test_user_4"); - ctx.fetch("CREATE ROLE airbyte LOGIN password 'airbyte';"); - ctx.fetch("CREATE ROLE read_only LOGIN password 'read_only';"); - ctx.fetch("CREATE ROLE intermediate LOGIN password 'intermediate';"); + ctx.fetch("CREATE ROLE airbyte LOGIN password 'airbyte';"); + ctx.fetch("CREATE ROLE read_only LOGIN password 'read_only';"); + ctx.fetch("CREATE ROLE intermediate LOGIN password 'intermediate';"); - ctx.fetch("CREATE ROLE access_nothing LOGIN password 'access_nothing';"); + ctx.fetch("CREATE ROLE access_nothing LOGIN password 'access_nothing';"); - ctx.fetch("GRANT intermediate TO airbyte;"); - ctx.fetch("GRANT read_only TO intermediate;"); + ctx.fetch("GRANT intermediate TO airbyte;"); + ctx.fetch("GRANT read_only TO intermediate;"); - ctx.fetch("GRANT SELECT ON id_and_name, id_and_name_7 TO read_only;"); - ctx.fetch("GRANT airbyte TO test_user_4;"); + ctx.fetch("GRANT SELECT ON id_and_name, id_and_name_7 TO read_only;"); + ctx.fetch("GRANT airbyte TO test_user_4;"); - ctx.fetch("CREATE TABLE unseen(id INTEGER, name VARCHAR(200));"); - ctx.fetch("GRANT CONNECT ON DATABASE test TO test_user_4;"); - return null; - }); - } - try (final DSLContext dslContext = getDslContextWithSpecifiedUser(config, "test_user_4", "132")) { - final Database database = new Database(dslContext); - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_name_3(id INTEGER, name VARCHAR(200));"); - return null; - }); - } - AirbyteCatalog actual = new PostgresSource().discover(getConfig(db, "test_user_4", "132")); - Set tableNames = actual.getStreams().stream().map(stream -> stream.getName()).collect(Collectors.toSet()); - assertEquals(Sets.newHashSet("id_and_name", "id_and_name_7", "id_and_name_3"), tableNames); - - actual = new PostgresSource().discover(getConfig(db, "access_nothing", "access_nothing")); - tableNames = actual.getStreams().stream().map(stream -> stream.getName()).collect(Collectors.toSet()); - assertEquals(Sets.newHashSet(), tableNames); - db.stop(); + ctx.fetch("CREATE TABLE unseen(id INTEGER, name VARCHAR(200));"); + ctx.fetch("GRANT CONNECT ON DATABASE " + testdb.getDatabaseName() + " TO test_user_4;"); + return null; + }); + final var config = getConfig(); + + try (final DSLContext dslContext = getDslContextWithSpecifiedUser(config, "test_user_4", "132")) { + final Database database = new Database(dslContext); + database.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_name_3(id INTEGER, name VARCHAR(200));"); + return null; + }); } + AirbyteCatalog actual = source().discover(getConfig("test_user_4", "132")); + Set tableNames = actual.getStreams().stream().map(stream -> stream.getName()).collect(Collectors.toSet()); + assertEquals(Sets.newHashSet("id_and_name", "id_and_name_7", "id_and_name_3"), tableNames); + + actual = source().discover(getConfig("access_nothing", "access_nothing")); + tableNames = actual.getStreams().stream().map(stream -> stream.getName()).collect(Collectors.toSet()); + assertEquals(Sets.newHashSet(), tableNames); } @Test void testDiscoverDifferentGrantAvailability() throws Exception { - try (final PostgreSQLContainer db = new PostgreSQLContainer<>("postgres:13-alpine")) { - db.start(); - final JsonNode config = getConfig(db); - try (final DSLContext dslContext = getDslContext(config)) { - final Database database = new Database(dslContext); - database.query(ctx -> { - ctx.fetch("create table not_granted_table_name_1(column_1 integer);"); - ctx.fetch("create table not_granted_table_name_2(column_1 integer);"); - ctx.fetch("create table not_granted_table_name_3(column_1 integer);"); - ctx.fetch("create table table_granted_by_role(column_1 integer);"); - ctx.fetch("create table test_table_granted_directly(column_1 integer);"); - ctx.fetch("create table table_granted_by_role_with_options(column_1 integer);"); - ctx.fetch("create table test_table_granted_directly_with_options(column_1 integer);"); - - ctx.fetch("create materialized view not_granted_mv_name_1 as SELECT not_granted_table_name_1.column_1 FROM not_granted_table_name_1;"); - ctx.fetch("create materialized view not_granted_mv_name_2 as SELECT not_granted_table_name_2.column_1 FROM not_granted_table_name_2;"); - ctx.fetch("create materialized view not_granted_mv_name_3 as SELECT not_granted_table_name_3.column_1 FROM not_granted_table_name_3;"); - ctx.fetch("create materialized view mv_granted_by_role as SELECT table_granted_by_role.column_1 FROM table_granted_by_role;"); - ctx.fetch( - "create materialized view test_mv_granted_directly as SELECT test_table_granted_directly.column_1 FROM test_table_granted_directly;"); - ctx.fetch( - "create materialized view mv_granted_by_role_with_options as SELECT table_granted_by_role_with_options.column_1 FROM table_granted_by_role_with_options;"); - ctx.fetch( - "create materialized view test_mv_granted_directly_with_options as SELECT test_table_granted_directly_with_options.column_1 FROM test_table_granted_directly_with_options;"); - - ctx.fetch("create view not_granted_view_name_1(column_1) as SELECT not_granted_table_name_1.column_1 FROM not_granted_table_name_1;"); - ctx.fetch("create view not_granted_view_name_2(column_1) as SELECT not_granted_table_name_2.column_1 FROM not_granted_table_name_2;"); - ctx.fetch("create view not_granted_view_name_3(column_1) as SELECT not_granted_table_name_3.column_1 FROM not_granted_table_name_3;"); - ctx.fetch("create view view_granted_by_role(column_1) as SELECT table_granted_by_role.column_1 FROM table_granted_by_role;"); - ctx.fetch( - "create view test_view_granted_directly(column_1) as SELECT test_table_granted_directly.column_1 FROM test_table_granted_directly;"); - ctx.fetch( - "create view view_granted_by_role_with_options(column_1) as SELECT table_granted_by_role_with_options.column_1 FROM table_granted_by_role_with_options;"); - ctx.fetch( - "create view test_view_granted_directly_with_options(column_1) as SELECT test_table_granted_directly_with_options.column_1 FROM test_table_granted_directly_with_options;"); - - ctx.fetch("create role test_role;"); - - ctx.fetch("grant delete on not_granted_table_name_2 to test_role;"); - ctx.fetch("grant delete on not_granted_mv_name_2 to test_role;"); - ctx.fetch("grant delete on not_granted_view_name_2 to test_role;"); - - ctx.fetch("grant select on table_granted_by_role to test_role;"); - ctx.fetch("grant select on mv_granted_by_role to test_role;"); - ctx.fetch("grant select on view_granted_by_role to test_role;"); - - ctx.fetch("grant select on table_granted_by_role_with_options to test_role with grant option;"); - ctx.fetch("grant select on mv_granted_by_role_with_options to test_role with grant option;"); - ctx.fetch("grant select on view_granted_by_role_with_options to test_role with grant option;"); - - ctx.fetch("create user new_test_user;"); - ctx.fetch("ALTER USER new_test_user WITH PASSWORD 'new_pass';"); - ctx.fetch("GRANT CONNECT ON DATABASE test TO new_test_user;"); - - ctx.fetch("grant test_role to new_test_user;"); - - ctx.fetch("grant delete on not_granted_table_name_3 to new_test_user;"); - ctx.fetch("grant delete on not_granted_mv_name_3 to new_test_user;"); - ctx.fetch("grant delete on not_granted_view_name_3 to new_test_user;"); - - ctx.fetch("grant select on test_table_granted_directly to new_test_user;"); - ctx.fetch("grant select on test_mv_granted_directly to new_test_user;"); - ctx.fetch("grant select on test_view_granted_directly to new_test_user;"); - - ctx.fetch("grant select on test_table_granted_directly_with_options to test_role with grant option;"); - ctx.fetch("grant select on test_mv_granted_directly_with_options to test_role with grant option;"); - ctx.fetch("grant select on test_view_granted_directly_with_options to test_role with grant option;"); - return null; - }); - } + final JsonNode config = getConfig(); + testdb.query(ctx -> { + ctx.fetch("create table not_granted_table_name_1(column_1 integer);"); + ctx.fetch("create table not_granted_table_name_2(column_1 integer);"); + ctx.fetch("create table not_granted_table_name_3(column_1 integer);"); + ctx.fetch("create table table_granted_by_role(column_1 integer);"); + ctx.fetch("create table test_table_granted_directly(column_1 integer);"); + ctx.fetch("create table table_granted_by_role_with_options(column_1 integer);"); + ctx.fetch("create table test_table_granted_directly_with_options(column_1 integer);"); + + ctx.fetch( + "create materialized view not_granted_mv_name_1 as SELECT not_granted_table_name_1.column_1 FROM not_granted_table_name_1;"); + ctx.fetch( + "create materialized view not_granted_mv_name_2 as SELECT not_granted_table_name_2.column_1 FROM not_granted_table_name_2;"); + ctx.fetch( + "create materialized view not_granted_mv_name_3 as SELECT not_granted_table_name_3.column_1 FROM not_granted_table_name_3;"); + ctx.fetch( + "create materialized view mv_granted_by_role as SELECT table_granted_by_role.column_1 FROM table_granted_by_role;"); + ctx.fetch( + "create materialized view test_mv_granted_directly as SELECT test_table_granted_directly.column_1 FROM test_table_granted_directly;"); + ctx.fetch( + "create materialized view mv_granted_by_role_with_options as SELECT table_granted_by_role_with_options.column_1 FROM table_granted_by_role_with_options;"); + ctx.fetch( + "create materialized view test_mv_granted_directly_with_options as SELECT test_table_granted_directly_with_options.column_1 FROM test_table_granted_directly_with_options;"); + + ctx.fetch( + "create view not_granted_view_name_1(column_1) as SELECT not_granted_table_name_1.column_1 FROM not_granted_table_name_1;"); + ctx.fetch( + "create view not_granted_view_name_2(column_1) as SELECT not_granted_table_name_2.column_1 FROM not_granted_table_name_2;"); + ctx.fetch( + "create view not_granted_view_name_3(column_1) as SELECT not_granted_table_name_3.column_1 FROM not_granted_table_name_3;"); + ctx.fetch( + "create view view_granted_by_role(column_1) as SELECT table_granted_by_role.column_1 FROM table_granted_by_role;"); + ctx.fetch( + "create view test_view_granted_directly(column_1) as SELECT test_table_granted_directly.column_1 FROM test_table_granted_directly;"); + ctx.fetch( + "create view view_granted_by_role_with_options(column_1) as SELECT table_granted_by_role_with_options.column_1 FROM table_granted_by_role_with_options;"); + ctx.fetch( + "create view test_view_granted_directly_with_options(column_1) as SELECT test_table_granted_directly_with_options.column_1 FROM test_table_granted_directly_with_options;"); + + ctx.fetch("create role test_role;"); + + ctx.fetch("grant delete on not_granted_table_name_2 to test_role;"); + ctx.fetch("grant delete on not_granted_mv_name_2 to test_role;"); + ctx.fetch("grant delete on not_granted_view_name_2 to test_role;"); + + ctx.fetch("grant select on table_granted_by_role to test_role;"); + ctx.fetch("grant select on mv_granted_by_role to test_role;"); + ctx.fetch("grant select on view_granted_by_role to test_role;"); + + ctx.fetch("grant select on table_granted_by_role_with_options to test_role with grant option;"); + ctx.fetch("grant select on mv_granted_by_role_with_options to test_role with grant option;"); + ctx.fetch("grant select on view_granted_by_role_with_options to test_role with grant option;"); + + ctx.fetch("create user new_test_user;"); + ctx.fetch("ALTER USER new_test_user WITH PASSWORD 'new_pass';"); + ctx.fetch("GRANT CONNECT ON DATABASE " + testdb.getDatabaseName() + " TO new_test_user;"); + ctx.fetch("GRANT ALL ON SCHEMA public TO test_user_4"); + + ctx.fetch("grant test_role to new_test_user;"); + + ctx.fetch("grant delete on not_granted_table_name_3 to new_test_user;"); + ctx.fetch("grant delete on not_granted_mv_name_3 to new_test_user;"); + ctx.fetch("grant delete on not_granted_view_name_3 to new_test_user;"); + + ctx.fetch("grant select on test_table_granted_directly to new_test_user;"); + ctx.fetch("grant select on test_mv_granted_directly to new_test_user;"); + ctx.fetch("grant select on test_view_granted_directly to new_test_user;"); + + ctx.fetch("grant select on test_table_granted_directly_with_options to test_role with grant option;"); + ctx.fetch("grant select on test_mv_granted_directly_with_options to test_role with grant option;"); + ctx.fetch("grant select on test_view_granted_directly_with_options to test_role with grant option;"); + return null; + }); - final AirbyteCatalog actual = new PostgresSource().discover(getConfig(db, "new_test_user", "new_pass")); - actual.getStreams().stream().forEach(airbyteStream -> { - assertEquals(2, airbyteStream.getSupportedSyncModes().size()); - assertTrue(airbyteStream.getSupportedSyncModes().contains(SyncMode.FULL_REFRESH)); - assertTrue(airbyteStream.getSupportedSyncModes().contains(SyncMode.INCREMENTAL)); - }); - final Set tableNames = actual.getStreams().stream().map(stream -> stream.getName()).collect(Collectors.toSet()); - final Set expectedVisibleNames = Sets.newHashSet( - "table_granted_by_role", - "table_granted_by_role_with_options", - "test_table_granted_directly", - "test_table_granted_directly_with_options", - "mv_granted_by_role", - "mv_granted_by_role_with_options", - "test_mv_granted_directly", - "test_mv_granted_directly_with_options", - "test_view_granted_directly", - "test_view_granted_directly_with_options", - "view_granted_by_role", - "view_granted_by_role_with_options"); - - assertEquals(tableNames, expectedVisibleNames); - - db.stop(); - } + final AirbyteCatalog actual = source().discover(getConfig("new_test_user", "new_pass")); + actual.getStreams().stream().forEach(airbyteStream -> { + assertEquals(2, airbyteStream.getSupportedSyncModes().size()); + assertTrue(airbyteStream.getSupportedSyncModes().contains(SyncMode.FULL_REFRESH)); + assertTrue(airbyteStream.getSupportedSyncModes().contains(SyncMode.INCREMENTAL)); + }); + final Set tableNames = + actual.getStreams().stream().map(stream -> stream.getName()).collect(Collectors.toSet()); + final Set expectedVisibleNames = Sets.newHashSet( + "table_granted_by_role", + "table_granted_by_role_with_options", + "test_table_granted_directly", + "test_table_granted_directly_with_options", + "mv_granted_by_role", + "mv_granted_by_role_with_options", + "test_mv_granted_directly", + "test_mv_granted_directly_with_options", + "test_view_granted_directly", + "test_view_granted_directly_with_options", + "view_granted_by_role", + "view_granted_by_role_with_options"); + + assertEquals(tableNames, expectedVisibleNames); } @Test @@ -511,7 +432,7 @@ void testReadSuccess() throws Exception { final ConfiguredAirbyteCatalog configuredCatalog = CONFIGURED_CATALOG.withStreams(CONFIGURED_CATALOG.getStreams().stream().filter(s -> s.getStream().getName().equals(STREAM_NAME)).collect( Collectors.toList())); - final Set actualMessages = MoreIterators.toSet(new PostgresSource().read(getConfig(PSQL_DB, dbName), configuredCatalog, null)); + final Set actualMessages = MoreIterators.toSet(source().read(getConfig(), configuredCatalog, null)); setEmittedAtToNull(actualMessages); assertEquals(ASCII_MESSAGES, actualMessages); @@ -519,56 +440,55 @@ void testReadSuccess() throws Exception { @Test void testReadIncrementalSuccess() throws Exception { - final JsonNode config = getConfig(PSQL_DB, dbName); // We want to test ordering, so we can delete the NaN entry and add a 3. - try (final DSLContext dslContext = getDslContext(config)) { - final Database database = getDatabase(dslContext); - database.query(ctx -> { - ctx.fetch("DELETE FROM id_and_name WHERE id = 'NaN';"); - ctx.fetch("INSERT INTO id_and_name (id, name, power) VALUES (3, 'gohan', 222.1);"); - return null; - }); + testdb.query(ctx -> { + ctx.fetch("DELETE FROM id_and_name WHERE id = 'NaN';"); + ctx.fetch("INSERT INTO id_and_name (id, name, power) VALUES (3, 'gohan', 222.1);"); + return null; + }); - final ConfiguredAirbyteCatalog configuredCatalog = - CONFIGURED_INCR_CATALOG - .withStreams(CONFIGURED_INCR_CATALOG.getStreams().stream().filter(s -> s.getStream().getName().equals(STREAM_NAME)).collect( - Collectors.toList())); - final PostgresSource source = new PostgresSource(); - source.setStateEmissionFrequencyForDebug(1); - final List actualMessages = MoreIterators.toList(source.read(getConfig(PSQL_DB, dbName), configuredCatalog, null)); - setEmittedAtToNull(actualMessages); + final ConfiguredAirbyteCatalog configuredCatalog = + CONFIGURED_INCR_CATALOG + .withStreams(CONFIGURED_INCR_CATALOG.getStreams() + .stream() + .filter(s -> s.getStream().getName().equals(STREAM_NAME)) + .toList()); + final PostgresSource source = source(); + source.setStateEmissionFrequencyForDebug(1); + final List actualMessages = MoreIterators.toList(source.read(getConfig(), configuredCatalog, null)); + setEmittedAtToNull(actualMessages); - final List stateAfterFirstBatch = extractStateMessage(actualMessages); + final List stateAfterFirstBatch = extractStateMessage(actualMessages); - setEmittedAtToNull(actualMessages); + setEmittedAtToNull(actualMessages); - final Set expectedOutput = Sets.newHashSet( - createRecord(STREAM_NAME, SCHEMA_NAME, map("id", new BigDecimal("1.0"), "name", "goku", "power", null)), - createRecord(STREAM_NAME, SCHEMA_NAME, map("id", new BigDecimal("2.0"), "name", "vegeta", "power", 9000.1)), - createRecord(STREAM_NAME, SCHEMA_NAME, map("id", new BigDecimal("3.0"), "name", "vegeta", "power", 222.1))); + final Set expectedOutput = Sets.newHashSet( + createRecord(STREAM_NAME, SCHEMA_NAME, map("id", new BigDecimal("1.0"), "name", "goku", "power", null)), + createRecord(STREAM_NAME, SCHEMA_NAME, map("id", new BigDecimal("2.0"), "name", "vegeta", "power", 9000.1)), + createRecord(STREAM_NAME, SCHEMA_NAME, map("id", new BigDecimal("3.0"), "name", "vegeta", "power", 222.1))); - // Assert that the correct number of messages are emitted. - assertEquals(actualMessages.size(), expectedOutput.size() + 1); - assertThat(actualMessages.contains(expectedOutput)); - // Assert that the Postgres source is emitting records & state messages in the correct order. - assertCorrectRecordOrderForIncrementalSync(actualMessages, "id", JsonSchemaPrimitive.NUMBER, configuredCatalog, - new AirbyteStreamNameNamespacePair("id_and_name", "public")); + // Assert that the correct number of messages are emitted. + assertEquals(actualMessages.size(), expectedOutput.size() + 1); + assertThat(actualMessages.contains(expectedOutput)); + // Assert that the Postgres source is emitting records & state messages in the correct order. + assertCorrectRecordOrderForIncrementalSync(actualMessages, "id", JsonSchemaPrimitive.NUMBER, configuredCatalog, + new AirbyteStreamNameNamespacePair("id_and_name", "public")); - final AirbyteStateMessage lastEmittedState = stateAfterFirstBatch.get(stateAfterFirstBatch.size() - 1); - final JsonNode state = Jsons.jsonNode(List.of(lastEmittedState)); + final AirbyteStateMessage lastEmittedState = stateAfterFirstBatch.get(stateAfterFirstBatch.size() - 1); + final JsonNode state = Jsons.jsonNode(List.of(lastEmittedState)); - database.query(ctx -> { - ctx.fetch("INSERT INTO id_and_name (id, name, power) VALUES (5, 'piccolo', 100.0);"); - return null; - }); - // Incremental sync should only read one new message (where id = '5.0') - final Set nextSyncMessages = MoreIterators.toSet(source.read(getConfig(PSQL_DB, dbName), configuredCatalog, state)); - setEmittedAtToNull(nextSyncMessages); + testdb.query(ctx -> { + ctx.fetch("INSERT INTO id_and_name (id, name, power) VALUES (5, 'piccolo', 100.0);"); + return null; + }); + // Incremental sync should only read one new message (where id = '5.0') + final Set nextSyncMessages = + MoreIterators.toSet(source.read(getConfig(), configuredCatalog, state)); + setEmittedAtToNull(nextSyncMessages); - // An extra state message is emitted, in addition to the record messages. - assertEquals(nextSyncMessages.size(), 2); - assertThat(nextSyncMessages.contains(createRecord(STREAM_NAME, SCHEMA_NAME, map("id", "5.0", "name", "piccolo", "power", 100.0)))); - } + // An extra state message is emitted, in addition to the record messages. + assertEquals(nextSyncMessages.size(), 2); + assertThat(nextSyncMessages.contains(createRecord(STREAM_NAME, SCHEMA_NAME, map("id", "5.0", "name", "piccolo", "power", 100.0)))); } /* @@ -607,7 +527,7 @@ private void assertCorrectRecordOrderForIncrementalSync(final List db = new PostgreSQLContainer<>("postgres:13-alpine")) { - db.start(); - final JsonNode config = getConfig(db); - try (final DSLContext dslContext = getDslContext(config)) { - final Database database = new Database(dslContext); - final ConfiguredAirbyteStream tableWithInvalidCursorType = createTableWithInvalidCursorType(database); - final ConfiguredAirbyteCatalog configuredAirbyteCatalog = - new ConfiguredAirbyteCatalog().withStreams(Collections.singletonList(tableWithInvalidCursorType)); - - final Throwable throwable = catchThrowable(() -> MoreIterators.toSet(new PostgresSource().read(config, configuredAirbyteCatalog, null))); - assertThat(throwable).isInstanceOf(ConfigErrorException.class) - .hasMessageContaining( - "The following tables have invalid columns selected as cursor, please select a column with a well-defined ordering with no null values as a cursor. {tableName='public.test_table', cursorColumnName='id', cursorSqlType=OTHER, cause=Unsupported cursor type}"); - } finally { - db.stop(); - } - } + final ConfiguredAirbyteStream tableWithInvalidCursorType = createTableWithInvalidCursorType(testdb.getDatabase()); + final ConfiguredAirbyteCatalog configuredAirbyteCatalog = + new ConfiguredAirbyteCatalog().withStreams(Collections.singletonList(tableWithInvalidCursorType)); + + final Throwable throwable = + catchThrowable(() -> MoreIterators.toSet(source().read(getConfig(), configuredAirbyteCatalog, null))); + assertThat(throwable).isInstanceOf(ConfigErrorException.class) + .hasMessageContaining( + "The following tables have invalid columns selected as cursor, please select a column with a well-defined ordering with no null values as a cursor. {tableName='public.test_table', cursorColumnName='id', cursorSqlType=OTHER, cause=Unsupported cursor type}"); } private ConfiguredAirbyteStream createTableWithInvalidCursorType(final Database database) throws SQLException { @@ -677,7 +589,7 @@ private ConfiguredAirbyteStream createTableWithInvalidCursorType(final Database @Test void testJdbcUrlWithEscapedDatabaseName() { - final JsonNode jdbcConfig = new PostgresSource().toDatabaseConfig(buildConfigEscapingNeeded()); + final JsonNode jdbcConfig = source().toDatabaseConfig(buildConfigEscapingNeeded()); assertEquals(EXPECTED_JDBC_ESCAPED_URL, jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText()); } @@ -694,23 +606,13 @@ private JsonNode buildConfigEscapingNeeded() { @Test public void tableWithNullValueCursorShouldThrowException() throws SQLException { - try (final PostgreSQLContainer db = new PostgreSQLContainer<>("postgres:13-alpine")) { - db.start(); - final JsonNode config = getConfig(db); - try (final DSLContext dslContext = getDslContext(config)) { - final Database database = new Database(dslContext); - final ConfiguredAirbyteStream table = createTableWithNullValueCursor(database); - final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(Collections.singletonList(table)); - - final Throwable throwable = catchThrowable(() -> MoreIterators.toSet(new PostgresSource().read(config, catalog, null))); - assertThat(throwable).isInstanceOf(ConfigErrorException.class) - .hasMessageContaining( - "The following tables have invalid columns selected as cursor, please select a column with a well-defined ordering with no null values as a cursor. {tableName='public.test_table_null_cursor', cursorColumnName='id', cursorSqlType=INTEGER, cause=Cursor column contains NULL value}"); - - } finally { - db.stop(); - } - } + final ConfiguredAirbyteStream table = createTableWithNullValueCursor(testdb.getDatabase()); + final ConfiguredAirbyteCatalog catalog = + new ConfiguredAirbyteCatalog().withStreams(Collections.singletonList(table)); + + final Throwable throwable = catchThrowable(() -> MoreIterators.toSet(source().read(getConfig(), catalog, null))); + assertThat(throwable).isInstanceOf(ConfigErrorException.class).hasMessageContaining( + "The following tables have invalid columns selected as cursor, please select a column with a well-defined ordering with no null values as a cursor. {tableName='public.test_table_null_cursor', cursorColumnName='id', cursorSqlType=INTEGER, cause=Cursor column contains NULL value}"); } private ConfiguredAirbyteStream createTableWithNullValueCursor(final Database database) throws SQLException { @@ -735,23 +637,13 @@ private ConfiguredAirbyteStream createTableWithNullValueCursor(final Database da @Test public void viewWithNullValueCursorShouldThrowException() throws SQLException { - try (final PostgreSQLContainer db = new PostgreSQLContainer<>("postgres:13-alpine")) { - db.start(); - final JsonNode config = getConfig(db); - try (final DSLContext dslContext = getDslContext(config)) { - final Database database = new Database(dslContext); - final ConfiguredAirbyteStream table = createViewWithNullValueCursor(database); - final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(Collections.singletonList(table)); - - final Throwable throwable = catchThrowable(() -> MoreIterators.toSet(new PostgresSource().read(config, catalog, null))); - assertThat(throwable).isInstanceOf(ConfigErrorException.class) - .hasMessageContaining( - "The following tables have invalid columns selected as cursor, please select a column with a well-defined ordering with no null values as a cursor. {tableName='public.test_view_null_cursor', cursorColumnName='id', cursorSqlType=INTEGER, cause=Cursor column contains NULL value}"); - - } finally { - db.stop(); - } - } + final ConfiguredAirbyteStream table = createViewWithNullValueCursor(testdb.getDatabase()); + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog().withStreams(Collections.singletonList(table)); + + final Throwable throwable = catchThrowable(() -> MoreIterators.toSet(source().read(getConfig(), catalog, null))); + assertThat(throwable).isInstanceOf(ConfigErrorException.class) + .hasMessageContaining( + "The following tables have invalid columns selected as cursor, please select a column with a well-defined ordering with no null values as a cursor. {tableName='public.test_view_null_cursor', cursorColumnName='id', cursorSqlType=INTEGER, cause=Cursor column contains NULL value}"); } private ConfiguredAirbyteStream createViewWithNullValueCursor(final Database database) throws SQLException { @@ -804,7 +696,7 @@ private static ConfiguredAirbyteStream toIncrementalConfiguredStream(final Airby @Test void testParseJdbcParameters() { final String jdbcPropertiesString = "foo=bar&options=-c%20search_path=test,public,pg_catalog%20-c%20statement_timeout=90000&baz=quux"; - Map parameters = PostgresSource.parseJdbcParameters(jdbcPropertiesString, "&"); + final Map parameters = PostgresSource.parseJdbcParameters(jdbcPropertiesString, "&"); assertEquals("-c%20search_path=test,public,pg_catalog%20-c%20statement_timeout=90000", parameters.get("options")); assertEquals("bar", parameters.get("foo")); assertEquals("quux", parameters.get("baz")); @@ -812,44 +704,83 @@ void testParseJdbcParameters() { @Test public void testJdbcOptionsParameter() throws Exception { - try (final PostgreSQLContainer db = new PostgreSQLContainer<>("postgres:13-alpine")) { - db.start(); - - // Populate DB. - final JsonNode dbConfig = getConfig(db); - try (final DSLContext dslContext = getDslContext(dbConfig)) { - final Database database = getDatabase(dslContext); - database.query(ctx -> { - ctx.fetch("CREATE TABLE id_and_bytes (id INTEGER, bytes BYTEA);"); - ctx.fetch("INSERT INTO id_and_bytes (id, bytes) VALUES (1, decode('DEADBEEF', 'hex'));"); - return null; - }); + // Populate DB. + final JsonNode dbConfig = getConfig(); + testdb.query(ctx -> { + ctx.fetch("CREATE TABLE id_and_bytes (id INTEGER, bytes BYTEA);"); + ctx.fetch("INSERT INTO id_and_bytes (id, bytes) VALUES (1, decode('DEADBEEF', 'hex'));"); + return null; + }); + + // Read the table contents using the non-default 'escape' format for bytea values. + final JsonNode sourceConfig = Jsons.jsonNode(ImmutableMap.builder() + .putAll(Jsons.flatten(dbConfig)) + .put(JdbcUtils.JDBC_URL_PARAMS_KEY, "options=-c%20statement_timeout=90000%20-c%20bytea_output=escape") + .build()); + final AirbyteStream airbyteStream = CatalogHelpers.createAirbyteStream( + "id_and_bytes", + SCHEMA_NAME, + Field.of("id", JsonSchemaType.NUMBER), + Field.of("bytes", JsonSchemaType.STRING)) + .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSourceDefinedPrimaryKey(List.of(List.of("id"))); + final AirbyteCatalog airbyteCatalog = new AirbyteCatalog().withStreams(List.of(airbyteStream)); + final Set actualMessages = + MoreIterators.toSet(source().read( + sourceConfig, + CatalogHelpers.toDefaultConfiguredCatalog(airbyteCatalog), + null)); + setEmittedAtToNull(actualMessages); + + // Check that the 'options' JDBC URL parameter was parsed correctly + // and that the bytea value is not in the default 'hex' format. + assertEquals(1, actualMessages.size()); + final AirbyteMessage actualMessage = actualMessages.stream().findFirst().get(); + assertTrue(actualMessage.getRecord().getData().has("bytes")); + assertEquals("\\336\\255\\276\\357", actualMessage.getRecord().getData().get("bytes").asText()); + } + + @Test + @DisplayName("Make sure initial incremental load is reading records in a certain order") + void testReadIncrementalRecordOrder() throws Exception { + // We want to test ordering, so we can delete the NaN entry + testdb.query(ctx -> { + ctx.fetch("DELETE FROM id_and_name WHERE id = 'NaN';"); + for (int i = 3; i < 1000; i++) { + ctx.fetch("INSERT INTO id_and_name (id, name, power) VALUES (%d, 'gohan%d', 222.1);".formatted(i, i)); } + return null; + }); - // Read the table contents using the non-default 'escape' format for bytea values. - final JsonNode sourceConfig = Jsons.jsonNode(ImmutableMap.builder() - .putAll(Jsons.flatten(dbConfig)) - .put(JdbcUtils.JDBC_URL_PARAMS_KEY, "options=-c%20statement_timeout=90000%20-c%20bytea_output=escape") - .build()); - final AirbyteStream airbyteStream = CatalogHelpers.createAirbyteStream( - "id_and_bytes", - SCHEMA_NAME, - Field.of("id", JsonSchemaType.NUMBER), - Field.of("bytes", JsonSchemaType.STRING)) - .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) - .withSourceDefinedPrimaryKey(List.of(List.of("id"))); - final AirbyteCatalog airbyteCatalog = new AirbyteCatalog().withStreams(List.of(airbyteStream)); - final Set actualMessages = - MoreIterators.toSet(new PostgresSource().read(sourceConfig, CatalogHelpers.toDefaultConfiguredCatalog(airbyteCatalog), null)); - setEmittedAtToNull(actualMessages); + final ConfiguredAirbyteCatalog configuredCatalog = + CONFIGURED_INCR_CATALOG + .withStreams(CONFIGURED_INCR_CATALOG.getStreams() + .stream() + .filter(s -> s.getStream().getName().equals(STREAM_NAME)) + .toList()); + final PostgresSource source = source(); + source.setStateEmissionFrequencyForDebug(1); + final List actualMessages = MoreIterators.toList(source.read(getConfig(), configuredCatalog, null)); + setEmittedAtToNull(actualMessages); + + // final List stateAfterFirstBatch = extractStateMessage(actualMessages); - // Check that the 'options' JDBC URL parameter was parsed correctly - // and that the bytea value is not in the default 'hex' format. - assertEquals(1, actualMessages.size()); - final AirbyteMessage actualMessage = actualMessages.stream().findFirst().get(); - assertTrue(actualMessage.getRecord().getData().has("bytes")); - assertEquals("\\336\\255\\276\\357", actualMessage.getRecord().getData().get("bytes").asText()); + setEmittedAtToNull(actualMessages); + + final Set expectedOutput = Sets.newHashSet( + createRecord(STREAM_NAME, SCHEMA_NAME, map("id", new BigDecimal("1.0"), "name", "goku", "power", null)), + createRecord(STREAM_NAME, SCHEMA_NAME, map("id", new BigDecimal("2.0"), "name", "vegeta", "power", 9000.1))); + for (int i = 3; i < 1000; i++) { + expectedOutput.add( + createRecord( + STREAM_NAME, + SCHEMA_NAME, + map("id", new BigDecimal("%d.0".formatted(i)), "name", "gohan%d".formatted(i), "power", 222.1))); } + assertThat(actualMessages.contains(expectedOutput)); + // Assert that the Postgres source is emitting records & state messages in the correct order. + assertCorrectRecordOrderForIncrementalSync(actualMessages, "id", JsonSchemaPrimitive.NUMBER, configuredCatalog, + new AirbyteStreamNameNamespacePair("id_and_name", "public")); } } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSpecTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSpecTest.java index 6f2d2553c368..1f63996b4c9e 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSpecTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresSpecTest.java @@ -11,10 +11,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.protocol.models.v0.ConnectorSpecification; import io.airbyte.validation.json.JsonSchemaValidator; import java.io.File; diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresStressTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresStressTest.java index ac60b86506ee..c68be6d1be11 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresStressTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresStressTest.java @@ -6,17 +6,17 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.cdk.integrations.source.jdbc.test.JdbcStressTest; +import io.airbyte.cdk.testutils.PostgreSQLContainerHelper; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.test.JdbcStressTest; -import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.sql.JDBCType; import java.util.Optional; import java.util.Set; @@ -43,7 +43,7 @@ class PostgresStressTest extends JdbcStressTest { @BeforeAll static void init() { - PSQL_DB = new PostgreSQLContainer<>("postgres:13-alpine"); + PSQL_DB = new PostgreSQLContainer<>("postgres:16-bullseye"); PSQL_DB.start(); } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresUtilsTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresUtilsTest.java index 5d80fe4763b7..9d3864ca0bf1 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresUtilsTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresUtilsTest.java @@ -15,8 +15,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.commons.jackson.MoreMappers; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; import java.time.Duration; import java.util.Collections; import java.util.List; @@ -95,4 +96,18 @@ public void shouldFlushAfterSync() { assertFalse(PostgresUtils.shouldFlushAfterSync(config)); } + @Test + void testDebugMode() { + final var config = MoreMappers.initMapper().createObjectNode(); + assertFalse(PostgresUtils.isCdc(config)); + + config.set("replication_method", Jsons.jsonNode(Map.of( + "replication_slot", "slot", + "publication", "ab_pub"))); + assertFalse(PostgresUtils.isDebugMode(config)); + + config.set("debug_mode", Jsons.jsonNode(true)); + assertTrue(PostgresUtils.isDebugMode(config)); + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/XminPostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/XminPostgresSourceTest.java index bba095875aff..7bf7f586918d 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/XminPostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/XminPostgresSourceTest.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.source.postgres; +import static io.airbyte.cdk.integrations.debezium.DebeziumIteratorConstants.SYNC_CHECKPOINT_RECORDS_PROPERTY; import static io.airbyte.integrations.source.postgres.utils.PostgresUnitTestsUtil.createRecord; import static io.airbyte.integrations.source.postgres.utils.PostgresUnitTestsUtil.extractStateMessage; import static io.airbyte.integrations.source.postgres.utils.PostgresUnitTestsUtil.filterRecords; @@ -15,17 +16,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.io.IOs; +import io.airbyte.cdk.integrations.base.Source; import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.string.Strings; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; @@ -38,7 +33,6 @@ import io.airbyte.protocol.models.v0.ConfiguredAirbyteStream; import io.airbyte.protocol.models.v0.DestinationSyncMode; import io.airbyte.protocol.models.v0.SyncMode; -import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; @@ -47,24 +41,12 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.testcontainers.containers.PostgreSQLContainer; -import org.testcontainers.utility.MountableFile; -import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; -import uk.org.webcompere.systemstubs.jupiter.SystemStub; -import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; - -@ExtendWith(SystemStubsExtension.class) + class XminPostgresSourceTest { - @SystemStub - private EnvironmentVariables environmentVariables; private static final String SCHEMA_NAME = "public"; protected static final String STREAM_NAME = "id_and_name"; private static final AirbyteCatalog CATALOG = new AirbyteCatalog().withStreams(List.of( @@ -105,93 +87,48 @@ class XminPostgresSourceTest { protected static final List NEXT_RECORD_MESSAGES = Arrays.asList( createRecord(STREAM_NAME, SCHEMA_NAME, map("id", new BigDecimal("3.0"), "name", "gohan", "power", 222.1))); - protected static PostgreSQLContainer PSQL_DB; - - protected String dbName; + protected PostgresTestDatabase testdb; - @BeforeAll - static void init() { - PSQL_DB = new PostgreSQLContainer<>("postgres:13-alpine"); - PSQL_DB.start(); + protected BaseImage getDatabaseImage() { + return BaseImage.POSTGRES_12; } @BeforeEach - void setup() throws Exception { - dbName = Strings.addRandomSuffix("db", "_", 10).toLowerCase(); - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - - final String initScriptName = "init_" + dbName.concat(".sql"); - final String tmpFilePath = IOs.writeFileToRandomTmpDir(initScriptName, "CREATE DATABASE " + dbName + ";"); - PostgreSQLContainerHelper.runSqlScript(MountableFile.forHostPath(tmpFilePath), PSQL_DB); - - final JsonNode config = getXminConfig(PSQL_DB, dbName); - - try (final DSLContext dslContext = getDslContext(config)) { - final Database database = getDatabase(dslContext); - database.query(ctx -> { - ctx.fetch( - "CREATE TABLE id_and_name(id NUMERIC(20, 10) NOT NULL, name VARCHAR(200) NOT NULL, power double precision NOT NULL, PRIMARY KEY (id));"); - ctx.fetch("CREATE INDEX i1 ON id_and_name (id);"); - ctx.fetch( - "INSERT INTO id_and_name (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); - - ctx.fetch("CREATE TABLE id_and_name2(id NUMERIC(20, 10) NOT NULL, name VARCHAR(200) NOT NULL, power double precision NOT NULL);"); - ctx.fetch( - "INSERT INTO id_and_name2 (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');"); - - ctx.fetch( - "CREATE TABLE names(first_name VARCHAR(200) NOT NULL, last_name VARCHAR(200) NOT NULL, power double precision NOT NULL, PRIMARY KEY (first_name, last_name));"); - ctx.fetch( + protected void setup() { + testdb = PostgresTestDatabase.in(getDatabaseImage()) + .with("CREATE TABLE id_and_name(id NUMERIC(20, 10) NOT NULL, name VARCHAR(200) NOT NULL, power double precision NOT NULL, PRIMARY KEY (id));") + .with("CREATE INDEX i1 ON id_and_name (id);") + .with("INSERT INTO id_and_name (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');") + .with("CREATE TABLE id_and_name2(id NUMERIC(20, 10) NOT NULL, name VARCHAR(200) NOT NULL, power double precision NOT NULL);") + .with("INSERT INTO id_and_name2 (id, name, power) VALUES (1,'goku', 'Infinity'), (2, 'vegeta', 9000.1), ('NaN', 'piccolo', '-Infinity');") + .with( + "CREATE TABLE names(first_name VARCHAR(200) NOT NULL, last_name VARCHAR(200) NOT NULL, power double precision NOT NULL, PRIMARY KEY (first_name, last_name));") + .with( "INSERT INTO names (first_name, last_name, power) VALUES ('san', 'goku', 'Infinity'), ('prince', 'vegeta', 9000.1), ('piccolo', 'junior', '-Infinity');"); - return null; - }); - } - } - - protected static Database getDatabase(final DSLContext dslContext) { - return new Database(dslContext); } - protected static DSLContext getDslContext(final JsonNode config) { - return DSLContextFactory.create( - config.get(JdbcUtils.USERNAME_KEY).asText(), - config.get(JdbcUtils.PASSWORD_KEY).asText(), - DatabaseDriver.POSTGRESQL.getDriverClassName(), - String.format(DatabaseDriver.POSTGRESQL.getUrlFormatString(), - config.get(JdbcUtils.HOST_KEY).asText(), - config.get(JdbcUtils.PORT_KEY).asInt(), - config.get(JdbcUtils.DATABASE_KEY).asText()), - SQLDialect.POSTGRES); + @AfterEach + protected void tearDown() { + testdb.close(); } - protected JsonNode getXminConfig(final PostgreSQLContainer psqlDb, final String dbName) { - return Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, psqlDb.getHost()) - .put(JdbcUtils.PORT_KEY, psqlDb.getFirstMappedPort()) - .put(JdbcUtils.DATABASE_KEY, dbName) - .put(JdbcUtils.SCHEMAS_KEY, List.of(SCHEMA_NAME)) - .put(JdbcUtils.USERNAME_KEY, psqlDb.getUsername()) - .put(JdbcUtils.PASSWORD_KEY, psqlDb.getPassword()) - .put(JdbcUtils.SSL_KEY, false) - .put("replication_method", getReplicationMethod()) - .put("sync_checkpoint_records", 1) - .build()); + protected JsonNode getXminConfig() { + return testdb.testConfigBuilder() + .withSchemas(SCHEMA_NAME) + .withoutSsl() + .withXminReplication() + .with(SYNC_CHECKPOINT_RECORDS_PROPERTY, 1) + .build(); } - private JsonNode getReplicationMethod() { - return Jsons.jsonNode(ImmutableMap.builder() - .put("method", "Xmin") - .build()); - } - - @AfterAll - static void cleanUp() { - PSQL_DB.close(); + protected Source source() { + PostgresSource source = new PostgresSource(); + return PostgresSource.sshWrappedSource(source); } @Test void testDiscover() throws Exception { - final AirbyteCatalog actual = new PostgresSource().discover(getXminConfig(PSQL_DB, dbName)); + final AirbyteCatalog actual = source().discover(getXminConfig()); actual.getStreams().forEach(actualStream -> { final Optional expectedStream = CATALOG.getStreams().stream().filter(stream -> stream.getName().equals(actualStream.getName())).findAny(); @@ -209,7 +146,7 @@ void testReadSuccess() throws Exception { .withStreams(CONFIGURED_XMIN_CATALOG.getStreams().stream().filter(s -> s.getStream().getName().equals(STREAM_NAME)).collect( Collectors.toList())); final List recordsFromFirstSync = - MoreIterators.toList(new PostgresSource().read(getXminConfig(PSQL_DB, dbName), configuredCatalog, null)); + MoreIterators.toList(source().read(getXminConfig(), configuredCatalog, null)); setEmittedAtToNull(recordsFromFirstSync); assertThat(filterRecords(recordsFromFirstSync)).containsExactlyElementsOf(INITIAL_RECORD_MESSAGES); @@ -256,7 +193,7 @@ void testReadSuccess() throws Exception { // Sync should work with a ctid state final List recordsFromSyncRunningWithACtidState = - MoreIterators.toList(new PostgresSource().read(getXminConfig(PSQL_DB, dbName), configuredCatalog, + MoreIterators.toList(source().read(getXminConfig(), configuredCatalog, Jsons.jsonNode(Collections.singletonList(firstStateMessage)))); setEmittedAtToNull(recordsFromSyncRunningWithACtidState); final List expectedDataFromSyncUsingFirstCtidState = new ArrayList<>(2); @@ -280,7 +217,7 @@ void testReadSuccess() throws Exception { // Read with the final xmin state message should return no data final List syncWithXminStateType = - MoreIterators.toList(new PostgresSource().read(getXminConfig(PSQL_DB, dbName), configuredCatalog, + MoreIterators.toList(source().read(getXminConfig(), configuredCatalog, Jsons.jsonNode(Collections.singletonList(thirdStateMessage)))); setEmittedAtToNull(syncWithXminStateType); assertEquals(0, filterRecords(syncWithXminStateType).size()); @@ -294,17 +231,14 @@ void testReadSuccess() throws Exception { // We add some data and perform a third read. We should verify that (i) a delete is not captured and // (ii) the new record that is inserted into the // table is read. - try (final DSLContext dslContext = getDslContext(getXminConfig(PSQL_DB, dbName))) { - final Database database = getDatabase(dslContext); - database.query(ctx -> { - ctx.fetch("DELETE FROM id_and_name WHERE id = 'NaN';"); - ctx.fetch("INSERT INTO id_and_name (id, name, power) VALUES (3, 'gohan', 222.1);"); - return null; - }); - } + testdb.query(ctx -> { + ctx.fetch("DELETE FROM id_and_name WHERE id = 'NaN';"); + ctx.fetch("INSERT INTO id_and_name (id, name, power) VALUES (3, 'gohan', 222.1);"); + return null; + }); final List recordsAfterLastSync = - MoreIterators.toList(new PostgresSource().read(getXminConfig(PSQL_DB, dbName), configuredCatalog, + MoreIterators.toList(source().read(getXminConfig(), configuredCatalog, Jsons.jsonNode(Collections.singletonList(stateAfterXminSync.get(0))))); setEmittedAtToNull(recordsAfterLastSync); assertThat(filterRecords(recordsAfterLastSync)).containsExactlyElementsOf(NEXT_RECORD_MESSAGES); diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/XminPostgresWithOldServerSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/XminPostgresWithOldServerSourceTest.java index 3d3f5d81baca..2027365218fc 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/XminPostgresWithOldServerSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/XminPostgresWithOldServerSourceTest.java @@ -14,24 +14,20 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.MoreIterators; -import io.airbyte.db.Database; +import io.airbyte.integrations.source.postgres.PostgresTestDatabase.BaseImage; import io.airbyte.protocol.models.v0.AirbyteMessage; import io.airbyte.protocol.models.v0.AirbyteStateMessage; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; -import org.jooq.DSLContext; -import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.testcontainers.containers.PostgreSQLContainer; public class XminPostgresWithOldServerSourceTest extends XminPostgresSourceTest { - @BeforeAll - static void init() { - PSQL_DB = new PostgreSQLContainer<>("postgres:9-alpine"); - PSQL_DB.start(); + @Override + protected BaseImage getDatabaseImage() { + return BaseImage.POSTGRES_9; } @Test @@ -44,7 +40,7 @@ void testReadSuccess() throws Exception { .withStreams(CONFIGURED_XMIN_CATALOG.getStreams().stream().filter(s -> s.getStream().getName().equals(STREAM_NAME)).collect( Collectors.toList())); final List recordsFromFirstSync = - MoreIterators.toList(new PostgresSource().read(getXminConfig(PSQL_DB, dbName), configuredCatalog, null)); + MoreIterators.toList(source().read(getXminConfig(), configuredCatalog, null)); setEmittedAtToNull(recordsFromFirstSync); assertThat(filterRecords(recordsFromFirstSync)).containsExactlyElementsOf(INITIAL_RECORD_MESSAGES); @@ -67,7 +63,7 @@ void testReadSuccess() throws Exception { // Read with the final xmin state message should return no data final List syncWithXminStateType = - MoreIterators.toList(new PostgresSource().read(getXminConfig(PSQL_DB, dbName), configuredCatalog, + MoreIterators.toList(source().read(getXminConfig(), configuredCatalog, Jsons.jsonNode(Collections.singletonList(firstSyncStateMessage)))); setEmittedAtToNull(syncWithXminStateType); assertEquals(0, filterRecords(syncWithXminStateType).size()); @@ -81,17 +77,14 @@ void testReadSuccess() throws Exception { // We add some data and perform a third read. We should verify that (i) a delete is not captured and // (ii) the new record that is inserted into the // table is read. - try (final DSLContext dslContext = getDslContext(getXminConfig(PSQL_DB, dbName))) { - final Database database = getDatabase(dslContext); - database.query(ctx -> { - ctx.fetch("DELETE FROM id_and_name WHERE id = 'NaN';"); - ctx.fetch("INSERT INTO id_and_name (id, name, power) VALUES (3, 'gohan', 222.1);"); - return null; - }); - } + testdb.query(ctx -> { + ctx.fetch("DELETE FROM id_and_name WHERE id = 'NaN';"); + ctx.fetch("INSERT INTO id_and_name (id, name, power) VALUES (3, 'gohan', 222.1);"); + return null; + }); final List recordsAfterLastSync = - MoreIterators.toList(new PostgresSource().read(getXminConfig(PSQL_DB, dbName), configuredCatalog, + MoreIterators.toList(source().read(getXminConfig(), configuredCatalog, Jsons.jsonNode(Collections.singletonList(stateAfterXminSync.get(0))))); setEmittedAtToNull(recordsAfterLastSync); assertThat(filterRecords(recordsAfterLastSync)).containsExactlyElementsOf(NEXT_RECORD_MESSAGES); diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIteratorTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIteratorTest.java new file mode 100644 index 000000000000..03b23fdf4b03 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/ctid/InitialSyncCtidIteratorTest.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres.ctid; + +import static io.airbyte.integrations.source.postgres.ctid.InitialSyncCtidIteratorConstants.EIGHT_KB; +import static io.airbyte.integrations.source.postgres.ctid.InitialSyncCtidIteratorConstants.GIGABYTE; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; +import org.apache.commons.lang3.tuple.Pair; +import org.junit.jupiter.api.Test; + +public class InitialSyncCtidIteratorTest { + + @Test + void testCtidQueryBounds() { + var chunks = InitialSyncCtidIterator.ctidQueryPlan(Ctid.ZERO, 380545032192L, 8192L, 50, GIGABYTE); + var expected = List.of( + Pair.of(Ctid.ZERO, Ctid.of(6553600, 0)), + Pair.of(Ctid.of(6553600, 0), Ctid.of(13107200, 0)), + Pair.of(Ctid.of(13107200, 0), Ctid.of(19660800, 0)), + Pair.of(Ctid.of(19660800, 0), Ctid.of(26214400, 0)), + Pair.of(Ctid.of(26214400, 0), Ctid.of(32768000, 0)), + Pair.of(Ctid.of(32768000, 0), Ctid.of(39321600, 0)), + Pair.of(Ctid.of(39321600, 0), Ctid.of(45875200, 0)), + Pair.of(Ctid.of(45875200, 0), null)); + assertEquals(expected, chunks); + + chunks = InitialSyncCtidIterator.ctidQueryPlan(new Ctid("(23000000,123)"), 380545032192L, 8192L, 45, GIGABYTE); + expected = List.of( + Pair.of(Ctid.of("(23000000,123)"), Ctid.of(28898240, 0)), + Pair.of(Ctid.of(28898240, 0), Ctid.of("(34796480,0)")), + Pair.of(Ctid.of("(34796480,0)"), Ctid.of(40694720, 0)), + Pair.of(Ctid.of(40694720, 0), null)); + assertEquals(expected, chunks); + + chunks = InitialSyncCtidIterator.ctidQueryPlan(Ctid.of(0, 0), 380545L, 8192L, 45, GIGABYTE); + expected = List.of(Pair.of(Ctid.ZERO, null)); + assertEquals(expected, chunks); + + chunks = InitialSyncCtidIterator.ctidQueryPlan(Ctid.of(9876, 5432), 380545L, 8192L, 45, GIGABYTE); + expected = List.of( + Pair.of(Ctid.of(9876, 5432), null)); + assertEquals(expected, chunks); + + chunks = InitialSyncCtidIterator.ctidQueryPlan(Ctid.ZERO, 4096L, 8192L, 45, GIGABYTE); + expected = List.of( + Pair.of(Ctid.ZERO, null)); + assertEquals(expected, chunks); + + chunks = InitialSyncCtidIterator.ctidQueryPlan(Ctid.of(8, 1), 819200L, 81920L, 50, EIGHT_KB); + expected = List.of(Pair.of(Ctid.of(8, 1), Ctid.of(9, 0)), Pair.of(Ctid.of(9, 0), Ctid.of(10, 0))); + assertEquals(expected, chunks); + } + + @Test + void testLegacyCtidQueryBounds() { + var chunks = InitialSyncCtidIterator.ctidLegacyQueryPlan(Ctid.of(0, 184), 3805450321L, 8192L, 50, GIGABYTE, 185); + var expected = List.of( + Pair.of(Ctid.of(0, 185), Ctid.of(27027, 185)), + Pair.of(Ctid.of(27028, 1), Ctid.of(54054, 185)), + Pair.of(Ctid.of(54055, 1), Ctid.of(81081, 185)), + Pair.of(Ctid.of(81082, 1), Ctid.of(108108, 185)), + Pair.of(Ctid.of(108109, 1), Ctid.of(135135, 185)), + Pair.of(Ctid.of(135136, 1), Ctid.of(162162, 185)), + Pair.of(Ctid.of(162163, 1), Ctid.of(189189, 185)), + Pair.of(Ctid.of(189190, 1), Ctid.of(216216, 185)), + Pair.of(Ctid.of(216217, 1), Ctid.of(243243, 185)), + Pair.of(Ctid.of(243244, 1), Ctid.of(270270, 185)), + Pair.of(Ctid.of(270271, 1), Ctid.of(297297, 185)), + Pair.of(Ctid.of(297298, 1), Ctid.of(324324, 185)), + Pair.of(Ctid.of(324325, 1), Ctid.of(351351, 185)), + Pair.of(Ctid.of(351352, 1), Ctid.of(378378, 185)), + Pair.of(Ctid.of(378379, 1), Ctid.of(405405, 185)), + Pair.of(Ctid.of(405406, 1), Ctid.of(432432, 185)), + Pair.of(Ctid.of(432433, 1), Ctid.of(459459, 185)), + Pair.of(Ctid.of(459460, 1), Ctid.of(486486, 185)), + Pair.of(Ctid.of(486487, 1), Ctid.of(513513, 185))); + + assertEquals(expected, chunks); + + chunks = InitialSyncCtidIterator.ctidLegacyQueryPlan(new Ctid("(1314328,123)"), 9805450321L, 8192L, 45, GIGABYTE, 10005); + expected = List.of( + Pair.of(Ctid.of("(1314328,124)"), Ctid.of(1314827, 10005)), + Pair.of(Ctid.of("(1314828,1)"), Ctid.of(1315326, 10005)), + Pair.of(Ctid.of("(1315327,1)"), Ctid.of(1315825, 10005)), + Pair.of(Ctid.of("(1315826,1)"), Ctid.of(1316324, 10005)), + Pair.of(Ctid.of("(1316325,1)"), Ctid.of(1316823, 10005))); + assertEquals(expected, chunks); + + chunks = InitialSyncCtidIterator.ctidLegacyQueryPlan(Ctid.of(0, 0), 380545L, 8192L, 45, GIGABYTE, 55); + expected = List.of(Pair.of(Ctid.of(0, 1), Ctid.of(90909, 55))); + assertEquals(expected, chunks); + + chunks = InitialSyncCtidIterator.ctidLegacyQueryPlan(Ctid.of(9876, 5432), 380545L, 8192L, 45, GIGABYTE, 5); + expected = List.of( + Pair.of(Ctid.of(9877, 1), Ctid.of(1009877, 5))); + assertEquals(expected, chunks); + + chunks = InitialSyncCtidIterator.ctidLegacyQueryPlan(Ctid.ZERO, 4096L, 8192L, 45, GIGABYTE, 226); + expected = List.of( + Pair.of(Ctid.of(0, 1), Ctid.of(22123, 226))); + assertEquals(expected, chunks); + + // Simulate an empty table - expected to generate an empty query plan + chunks = InitialSyncCtidIterator.ctidLegacyQueryPlan(Ctid.ZERO, 4096L, 8192L, 45, GIGABYTE, 1); + expected = List.of(Pair.of(Ctid.of(0, 1), Ctid.of(5000000, 1))); + assertEquals(expected, chunks); + + chunks = InitialSyncCtidIterator.ctidLegacyQueryPlan(Ctid.of(8, 1), 819200L, 81920L, 50, EIGHT_KB, 1); + expected = List.of(Pair.of(Ctid.of(9, 1), Ctid.of(10, 1)), Pair.of(Ctid.of(11, 1), Ctid.of(11, 1))); + assertEquals(expected, chunks); + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/ctid/PostgresCtidHandlerTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/ctid/PostgresCtidHandlerTest.java deleted file mode 100644 index caa95f561b70..000000000000 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/ctid/PostgresCtidHandlerTest.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.postgres.ctid; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import java.util.List; -import org.apache.commons.lang3.tuple.Pair; -import org.junit.jupiter.api.Test; - -public class PostgresCtidHandlerTest { - - @Test - void testCtidQueryBounds() { - var chunks = PostgresCtidHandler.ctidQueryPlan(new Ctid(0, 0), 380545032192L, 8192L, 50); - var expected = List.of( - Pair.of(Ctid.of(0, 0), Ctid.of(6553600, 0)), - Pair.of(Ctid.of(6553600, 0), Ctid.of(13107200, 0)), - Pair.of(Ctid.of(13107200, 0), Ctid.of(19660800, 0)), - Pair.of(Ctid.of(19660800, 0), Ctid.of(26214400, 0)), - Pair.of(Ctid.of(26214400, 0), Ctid.of(32768000, 0)), - Pair.of(Ctid.of(32768000, 0), Ctid.of(39321600, 0)), - Pair.of(Ctid.of(39321600, 0), Ctid.of(45875200, 0)), - Pair.of(Ctid.of(45875200, 0), null)); - assertEquals(expected, chunks); - - chunks = PostgresCtidHandler.ctidQueryPlan(new Ctid("(23000000,123)"), 380545032192L, 8192L, 45); - expected = List.of( - Pair.of(Ctid.of("(23000000,123)"), Ctid.of(28898240, 0)), - Pair.of(Ctid.of(28898240, 0), Ctid.of("(34796480,0)")), - Pair.of(Ctid.of("(34796480,0)"), Ctid.of(40694720, 0)), - Pair.of(Ctid.of(40694720, 0), null)); - assertEquals(expected, chunks); - - chunks = PostgresCtidHandler.ctidQueryPlan(Ctid.of(0, 0), 380545L, 8192L, 45); - expected = List.of( - Pair.of(Ctid.of(0, 0), null)); - assertEquals(expected, chunks); - - chunks = PostgresCtidHandler.ctidQueryPlan(Ctid.of(9876, 5432), 380545L, 8192L, 45); - expected = List.of( - Pair.of(Ctid.of(9876, 5432), null)); - assertEquals(expected, chunks); - - chunks = PostgresCtidHandler.ctidQueryPlan(Ctid.of(0, 0), 4096L, 8192L, 45); - expected = List.of( - Pair.of(Ctid.of(0, 0), null)); - assertEquals(expected, chunks); - } - -} diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/cursor_based/CursorBasedCtidUtilsTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/cursor_based/CursorBasedCtidUtilsTest.java index 5bcb8a4b30f6..6bf511681823 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/cursor_based/CursorBasedCtidUtilsTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/cursor_based/CursorBasedCtidUtilsTest.java @@ -14,13 +14,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.source.relationaldb.state.StreamStateManager; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.postgres.ctid.CtidUtils.StreamsCategorised; import io.airbyte.integrations.source.postgres.cursor_based.CursorBasedCtidUtils.CursorBasedStreams; import io.airbyte.integrations.source.postgres.internal.models.CtidStatus; import io.airbyte.integrations.source.postgres.internal.models.CursorBasedStatus; import io.airbyte.integrations.source.postgres.internal.models.InternalModels.StateType; -import io.airbyte.integrations.source.relationaldb.state.StreamStateManager; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteStateMessage; diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/xmin/XminCtidUtilsTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/xmin/XminCtidUtilsTest.java index 8923a7a932f3..01a40e3b2bad 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/xmin/XminCtidUtilsTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/xmin/XminCtidUtilsTest.java @@ -10,13 +10,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.source.relationaldb.state.StreamStateManager; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.postgres.ctid.CtidUtils.StreamsCategorised; import io.airbyte.integrations.source.postgres.internal.models.CtidStatus; import io.airbyte.integrations.source.postgres.internal.models.InternalModels.StateType; import io.airbyte.integrations.source.postgres.internal.models.XminStatus; import io.airbyte.integrations.source.postgres.xmin.XminCtidUtils.XminStreams; -import io.airbyte.integrations.source.relationaldb.state.StreamStateManager; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteStateMessage; diff --git a/airbyte-integrations/connectors/source-postgres/src/testFixtures/java/io/airbyte/integrations/source/postgres/PostgresContainerFactory.java b/airbyte-integrations/connectors/source-postgres/src/testFixtures/java/io/airbyte/integrations/source/postgres/PostgresContainerFactory.java new file mode 100644 index 000000000000..b92c319d9eec --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/testFixtures/java/io/airbyte/integrations/source/postgres/PostgresContainerFactory.java @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres; + +import io.airbyte.cdk.testutils.ContainerFactory; +import java.io.IOException; +import java.io.UncheckedIOException; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.MountableFile; + +public class PostgresContainerFactory implements ContainerFactory> { + + @Override + public PostgreSQLContainer createNewContainer(DockerImageName imageName) { + return new PostgreSQLContainer<>(imageName.asCompatibleSubstituteFor("postgres")); + + } + + @Override + public Class getContainerClass() { + return PostgreSQLContainer.class; + } + + /** + * Apply the postgresql.conf file that we've packaged as a resource. + */ + public void withConf(PostgreSQLContainer container) { + container + .withCopyFileToContainer( + MountableFile.forClasspathResource("postgresql.conf"), + "/etc/postgresql/postgresql.conf") + .withCommand("postgres -c config_file=/etc/postgresql/postgresql.conf"); + } + + /** + * Create a new network and bind it to the container. + */ + public void withNetwork(PostgreSQLContainer container) { + container.withNetwork(Network.newNetwork()); + } + + /** + * Configure postgres with wal_level=logical. + */ + public void withWalLevelLogical(PostgreSQLContainer container) { + container.withCommand("postgres -c wal_level=logical"); + } + + /** + * Generate SSL certificates and tell postgres to enable SSL and use them. + */ + public void withCert(PostgreSQLContainer container) { + container.start(); + String[] commands = { + "psql -U test -c \"CREATE USER postgres WITH PASSWORD 'postgres';\"", + "psql -U test -c \"GRANT CONNECT ON DATABASE \"test\" TO postgres;\"", + "psql -U test -c \"ALTER USER postgres WITH SUPERUSER;\"", + "openssl ecparam -name prime256v1 -genkey -noout -out ca.key", + "openssl req -new -x509 -sha256 -key ca.key -out ca.crt -subj \"/CN=127.0.0.1\"", + "openssl ecparam -name prime256v1 -genkey -noout -out server.key", + "openssl req -new -sha256 -key server.key -out server.csr -subj \"/CN=localhost\"", + "openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 365 -sha256", + "cp server.key /etc/ssl/private/", + "cp server.crt /etc/ssl/private/", + "cp ca.crt /etc/ssl/private/", + "chmod og-rwx /etc/ssl/private/server.* /etc/ssl/private/ca.*", + "chown postgres:postgres /etc/ssl/private/server.crt /etc/ssl/private/server.key /etc/ssl/private/ca.crt", + "echo \"ssl = on\" >> /var/lib/postgresql/data/postgresql.conf", + "echo \"ssl_cert_file = '/etc/ssl/private/server.crt'\" >> /var/lib/postgresql/data/postgresql.conf", + "echo \"ssl_key_file = '/etc/ssl/private/server.key'\" >> /var/lib/postgresql/data/postgresql.conf", + "echo \"ssl_ca_file = '/etc/ssl/private/ca.crt'\" >> /var/lib/postgresql/data/postgresql.conf", + "mkdir root/.postgresql", + "echo \"hostssl all all 127.0.0.1/32 cert clientcert=verify-full\" >> /var/lib/postgresql/data/pg_hba.conf", + "openssl ecparam -name prime256v1 -genkey -noout -out client.key", + "openssl req -new -sha256 -key client.key -out client.csr -subj \"/CN=postgres\"", + "openssl x509 -req -in client.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out client.crt -days 365 -sha256", + "cp client.crt ~/.postgresql/postgresql.crt", + "cp client.key ~/.postgresql/postgresql.key", + "chmod 0600 ~/.postgresql/postgresql.crt ~/.postgresql/postgresql.key", + "cp ca.crt root/.postgresql/ca.crt", + "chown postgres:postgres ~/.postgresql/ca.crt", + "psql -U test -c \"SELECT pg_reload_conf();\"", + }; + for (String cmd : commands) { + try { + container.execInContainer("su", "-c", cmd); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } + + /** + * Tell postgres to enable SSL. + */ + public void withSSL(PostgreSQLContainer container) { + container.withCommand("postgres " + + "-c ssl=on " + + "-c ssl_cert_file=/var/lib/postgresql/server.crt " + + "-c ssl_key_file=/var/lib/postgresql/server.key"); + } + + /** + * Configure postgres with client_encoding=sql_ascii. + */ + public void withASCII(PostgreSQLContainer container) { + container.withCommand("postgres -c client_encoding=sql_ascii"); + } + +} diff --git a/airbyte-integrations/connectors/source-postgres/src/testFixtures/java/io/airbyte/integrations/source/postgres/PostgresTestDatabase.java b/airbyte-integrations/connectors/source-postgres/src/testFixtures/java/io/airbyte/integrations/source/postgres/PostgresTestDatabase.java new file mode 100644 index 000000000000..155b649e96a8 --- /dev/null +++ b/airbyte-integrations/connectors/source-postgres/src/testFixtures/java/io/airbyte/integrations/source/postgres/PostgresTestDatabase.java @@ -0,0 +1,196 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.postgres; + +import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.TestDatabase; +import io.airbyte.commons.json.Jsons; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.List; +import java.util.stream.Stream; +import org.jooq.SQLDialect; +import org.testcontainers.containers.PostgreSQLContainer; + +public class PostgresTestDatabase extends + TestDatabase, PostgresTestDatabase, PostgresTestDatabase.PostgresConfigBuilder> { + + public static enum BaseImage { + + POSTGRES_16("postgres:16-bullseye"), + POSTGRES_12("postgres:12-bullseye"), + POSTGRES_9("postgres:9-alpine"), + POSTGRES_SSL_DEV("marcosmarxm/postgres-ssl:dev"); + + private final String reference; + + private BaseImage(String reference) { + this.reference = reference; + }; + + } + + public static enum ContainerModifier { + + ASCII("withASCII"), + CONF("withConf"), + NETWORK("withNetwork"), + SSL("withSSL"), + WAL_LEVEL_LOGICAL("withWalLevelLogical"), + CERT("withCert"), + ; + + private String methodName; + + private ContainerModifier(String methodName) { + this.methodName = methodName; + } + + } + + static public PostgresTestDatabase in(BaseImage baseImage, ContainerModifier... modifiers) { + String[] methodNames = Stream.of(modifiers).map(im -> im.methodName).toList().toArray(new String[0]); + final var container = new PostgresContainerFactory().shared(baseImage.reference, methodNames); + return new PostgresTestDatabase(container).initialized(); + } + + public PostgresTestDatabase(PostgreSQLContainer container) { + super(container); + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.of(psqlCmd(Stream.of( + String.format("CREATE DATABASE %s", getDatabaseName()), + String.format("CREATE USER %s PASSWORD '%s'", getUserName(), getPassword()), + String.format("GRANT ALL PRIVILEGES ON DATABASE %s TO %s", getDatabaseName(), getUserName()), + String.format("ALTER USER %s WITH SUPERUSER", getUserName())))); + } + + /** + * Close resources held by this instance. This deliberately avoids dropping the database, which is + * really expensive in Postgres. This is because a DROP DATABASE in Postgres triggers a CHECKPOINT. + * Call {@link #dropDatabaseAndUser} to explicitly drop the database and the user. + */ + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + /** + * Drop the database owned by this instance. + */ + public void dropDatabaseAndUser() { + execInContainer(psqlCmd(Stream.of( + String.format("DROP DATABASE %s", getDatabaseName()), + String.format("DROP OWNED BY %s", getUserName()), + String.format("DROP USER %s", getUserName())))); + } + + public Stream psqlCmd(Stream sql) { + return Stream.concat( + Stream.of("psql", + "-d", getContainer().getDatabaseName(), + "-U", getContainer().getUsername(), + "-v", "ON_ERROR_STOP=1", + "-a"), + sql.flatMap(stmt -> Stream.of("-c", stmt))); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return DatabaseDriver.POSTGRESQL; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.POSTGRES; + } + + private Certificates cachedCerts; + + public synchronized Certificates getCertificates() { + if (cachedCerts == null) { + final String caCert, clientKey, clientCert; + try { + caCert = getContainer().execInContainer("su", "-c", "cat ca.crt").getStdout().trim(); + clientKey = getContainer().execInContainer("su", "-c", "cat client.key").getStdout().trim(); + clientCert = getContainer().execInContainer("su", "-c", "cat client.crt").getStdout().trim(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + cachedCerts = new Certificates(caCert, clientCert, clientKey); + } + return cachedCerts; + } + + public record Certificates(String caCertificate, String clientCertificate, String clientKey) {} + + @Override + public PostgresConfigBuilder configBuilder() { + return new PostgresConfigBuilder(this); + } + + public String getReplicationSlotName() { + return withNamespace("debezium_slot"); + } + + public String getPublicationName() { + return withNamespace("publication"); + } + + public PostgresTestDatabase withReplicationSlot() { + return this + .with("SELECT pg_create_logical_replication_slot('%s', 'pgoutput');", getReplicationSlotName()) + .onClose("SELECT pg_drop_replication_slot('%s');", getReplicationSlotName()); + } + + public PostgresTestDatabase withPublicationForAllTables() { + return this + .with("CREATE PUBLICATION %s FOR ALL TABLES;", getPublicationName()) + .onClose("DROP PUBLICATION %s CASCADE;", getPublicationName()); + } + + static public class PostgresConfigBuilder extends ConfigBuilder { + + protected PostgresConfigBuilder(PostgresTestDatabase testdb) { + super(testdb); + } + + public PostgresConfigBuilder withSchemas(String... schemas) { + return with(JdbcUtils.SCHEMAS_KEY, List.of(schemas)); + } + + public PostgresConfigBuilder withStandardReplication() { + return with("replication_method", ImmutableMap.builder().put("method", "Standard").build()); + } + + public PostgresConfigBuilder withCdcReplication() { + return withCdcReplication("While reading Data"); + } + + public PostgresConfigBuilder withCdcReplication(String LsnCommitBehaviour) { + return this + .with("is_test", true) + .with("replication_method", Jsons.jsonNode(ImmutableMap.builder() + .put("method", "CDC") + .put("replication_slot", testDatabase.getReplicationSlotName()) + .put("publication", testDatabase.getPublicationName()) + .put("initial_waiting_seconds", DEFAULT_CDC_REPLICATION_INITIAL_WAIT.getSeconds()) + .put("lsn_commit_behaviour", LsnCommitBehaviour) + .build())); + } + + public PostgresConfigBuilder withXminReplication() { + return this.with("replication_method", Jsons.jsonNode(ImmutableMap.builder().put("method", "Xmin").build())); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-posthog/Dockerfile b/airbyte-integrations/connectors/source-posthog/Dockerfile index 06c2c48b86f2..441ca769608e 100644 --- a/airbyte-integrations/connectors/source-posthog/Dockerfile +++ b/airbyte-integrations/connectors/source-posthog/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.13 +LABEL io.airbyte.version=1.0.0 LABEL io.airbyte.name=airbyte/source-posthog diff --git a/airbyte-integrations/connectors/source-posthog/README.md b/airbyte-integrations/connectors/source-posthog/README.md index 5939e3e3cb17..f4871371a74c 100644 --- a/airbyte-integrations/connectors/source-posthog/README.md +++ b/airbyte-integrations/connectors/source-posthog/README.md @@ -27,14 +27,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-posthog:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/posthog) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_posthog/spec.json` file. @@ -54,18 +46,19 @@ python main.py read --config secrets/config.json --catalog sample_files/configur ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-posthog:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-posthog build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-posthog:airbyteDocker +An image will be built with the tag `airbyte/source-posthog:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-posthog:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,44 +68,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-posthog:dev check --co docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-posthog:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-posthog:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json ``` + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-posthog test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unittest run: -``` -./gradlew :airbyte-integrations:connectors:source-posthog:unitTest -``` -To run acceptance and custom integration tests run: -``` -./gradlew :airbyte-integrations:connectors:source-posthog:IntegrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -122,8 +87,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-posthog test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/posthog.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-posthog/acceptance-test-config.yml b/airbyte-integrations/connectors/source-posthog/acceptance-test-config.yml index 41c805a957a4..e99c5afabb76 100644 --- a/airbyte-integrations/connectors/source-posthog/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-posthog/acceptance-test-config.yml @@ -1,29 +1,25 @@ connector_image: airbyte/source-posthog:dev tests: spec: - - spec_path: "source_posthog/spec.json" + - spec_path: "source_posthog/spec.json" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - fail_on_extra_columns: false + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + fail_on_extra_columns: false incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_events_incremental.json" - future_state_path: "integration_tests/future_state.json" - cursor_paths: - "events": ["2331", "timestamp"] - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_events_incremental.json" - future_state_path: "integration_tests/future_state_old.json" - cursor_paths: - "events": ["2331", "timestamp"] + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_events_incremental.json" + future_state_path: "integration_tests/future_state.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_events_incremental.json" + future_state_path: "integration_tests/future_state_old.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-posthog/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-posthog/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-posthog/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-posthog/build.gradle b/airbyte-integrations/connectors/source-posthog/build.gradle deleted file mode 100644 index 8928eb02e051..000000000000 --- a/airbyte-integrations/connectors/source-posthog/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_posthog' -} diff --git a/airbyte-integrations/connectors/source-posthog/metadata.yaml b/airbyte-integrations/connectors/source-posthog/metadata.yaml index 3d2fcba2975d..792e53553194 100644 --- a/airbyte-integrations/connectors/source-posthog/metadata.yaml +++ b/airbyte-integrations/connectors/source-posthog/metadata.yaml @@ -1,4 +1,7 @@ data: + ab_internal: + ql: 300 + sl: 100 allowedHosts: hosts: - ${base_url} @@ -6,8 +9,9 @@ data: connectorSubtype: api connectorType: source definitionId: af6d50ee-dddf-4126-a8ee-7faee990774f - dockerImageTag: 0.1.13 + dockerImageTag: 1.0.0 dockerRepository: airbyte/source-posthog + documentationUrl: https://docs.airbyte.com/integrations/sources/posthog githubIssueLabel: source-posthog icon: posthog.svg license: MIT @@ -18,12 +22,15 @@ data: oss: enabled: true releaseStage: beta - documentationUrl: https://docs.airbyte.com/integrations/sources/posthog + supportLevel: community + releases: + breakingChanges: + 1.0.0: + message: + The `event` field in the `events` stream has been corrected to the proper data type. + To apply this change, refresh the schema for the `events` stream and reset your data. For more information [visit](https://docs.airbyte.com/integrations/sources/posthog-migrations) + upgradeDeadline: "2024-01-15" tags: - language:low-code - language:python - ab_internal: - sl: 200 - ql: 300 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/manifest.yaml b/airbyte-integrations/connectors/source-posthog/source_posthog/manifest.yaml index 96c9d9ac178f..cedce00b61ca 100644 --- a/airbyte-integrations/connectors/source-posthog/source_posthog/manifest.yaml +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/manifest.yaml @@ -117,8 +117,11 @@ definitions: datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%S%z') }}" datetime_format: "%Y-%m-%dT%H:%M:%S%z" datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%f%z" + - "%Y-%m-%dT%H:%M:%S+00:00" cursor_granularity: "PT0.000001S" - step: "P30D" + step: "P{{ config.get('events_time_step', 30) }}D" cursor_field: timestamp start_time_option: field_name: after diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/events.json b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/events.json index 8e050f455967..cddb4fd6a2ae 100644 --- a/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/events.json +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/schemas/events.json @@ -11,7 +11,7 @@ "type": "object" }, "event": { - "type": ["string", "object"] + "type": "string" }, "timestamp": { "type": "string", diff --git a/airbyte-integrations/connectors/source-posthog/source_posthog/spec.json b/airbyte-integrations/connectors/source-posthog/source_posthog/spec.json index e5da0c79e13b..6da5b45f9060 100644 --- a/airbyte-integrations/connectors/source-posthog/source_posthog/spec.json +++ b/airbyte-integrations/connectors/source-posthog/source_posthog/spec.json @@ -26,6 +26,16 @@ "title": "Base URL", "description": "Base PostHog url. Defaults to PostHog Cloud (https://app.posthog.com).", "examples": ["https://posthog.example.com"] + }, + "events_time_step": { + "type": "integer", + "order": 3, + "default": 30, + "minimum": 1, + "maximum": 91, + "title": "Events stream slice step size (in days)", + "description": "Set lower value in case of failing long running sync of events stream.", + "examples": [30, 10, 5] } } } diff --git a/airbyte-integrations/connectors/source-postmarkapp/README.md b/airbyte-integrations/connectors/source-postmarkapp/README.md index 2a695a6d3075..107e98ce8c1a 100644 --- a/airbyte-integrations/connectors/source-postmarkapp/README.md +++ b/airbyte-integrations/connectors/source-postmarkapp/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-postmarkapp:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/postmarkapp) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_postmarkapp/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-postmarkapp:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-postmarkapp build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-postmarkapp:airbyteDocker +An image will be built with the tag `airbyte/source-postmarkapp:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-postmarkapp:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-postmarkapp:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-postmarkapp:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-postmarkapp:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-postmarkapp test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-postmarkapp:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-postmarkapp:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-postmarkapp test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/postmarkapp.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-postmarkapp/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-postmarkapp/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-postmarkapp/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-postmarkapp/build.gradle b/airbyte-integrations/connectors/source-postmarkapp/build.gradle deleted file mode 100644 index d013a9d71639..000000000000 --- a/airbyte-integrations/connectors/source-postmarkapp/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_postmarkapp' -} diff --git a/airbyte-integrations/connectors/source-prestashop/README.md b/airbyte-integrations/connectors/source-prestashop/README.md index 17d28622803a..992f1d06939d 100644 --- a/airbyte-integrations/connectors/source-prestashop/README.md +++ b/airbyte-integrations/connectors/source-prestashop/README.md @@ -34,16 +34,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle - -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: - -``` -./gradlew :airbyte-integrations:connectors:source-prestashop:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/prestashop) @@ -65,23 +55,20 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build - -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-prestashop:dev +#### Build +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-prestashop build ``` -You can also build the connector image via Gradle: +An image will be built with the tag `airbyte/source-prestashop:dev`. -``` -./gradlew :airbyte-integrations:connectors:source-prestashop:airbyteDocker +**Via `docker build`:** +```bash +docker build -t airbyte/source-prestashop:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. - #### Run Then run any of the connector commands as follows: @@ -93,61 +80,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-prestashop:dev discove docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-prestashop:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing - -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: - -``` -pip install .[tests] -``` - -### Unit Tests - -To run unit tests locally, from the connector directory run: - -``` -python -m pytest unit_tests -``` -### Integration Tests - -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). - -#### Custom Integration tests - -Place custom tests inside `integration_tests/` folder, then, from the connector root, run - -``` -python -m pytest integration_tests +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-prestashop test ``` -#### Acceptance Tests - -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run - -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` - -To run your integration tests with docker - -### Using gradle to run tests - -All commands should be run from airbyte project root. -To run unit tests: - -``` -./gradlew :airbyte-integrations:connectors:source-prestashop:unitTest -``` - -To run acceptance and custom integration tests: - -``` -./gradlew :airbyte-integrations:connectors:source-prestashop:integrationTest -``` ## Dependency Management @@ -158,11 +100,12 @@ We split dependencies between two groups, dependencies that are: - required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector - You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-prestashop test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/prestashop.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-prestashop/acceptance-test-config.yml b/airbyte-integrations/connectors/source-prestashop/acceptance-test-config.yml index eddbf16f18e8..e75738412d47 100644 --- a/airbyte-integrations/connectors/source-prestashop/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-prestashop/acceptance-test-config.yml @@ -4,40 +4,41 @@ connector_image: airbyte/source-prestashop:dev acceptance_tests: spec: tests: - - spec_path: "source_prestashop/spec.yaml" - # unfortunately timeout plugin takes into account setup code as well (docker setup) - timeout_seconds: 300 + - spec_path: + "source_prestashop/spec.yaml" + # unfortunately timeout plugin takes into account setup code as well (docker setup) + timeout_seconds: 300 connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: tests: - - config_path: "secrets/config.json" - backward_compatibility_tests_config: - disable_for_version: "0.3.1" + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.3.1" basic_read: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: - - name: "messages" - bypass_reason: "Can not populate" - expect_records: - path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: no - extra_records: yes - fail_on_extra_columns: false + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: "messages" + bypass_reason: "Can not populate" + expect_records: + path: "integration_tests/expected_records.jsonl" + extra_fields: no + exact_order: no + extra_records: yes + fail_on_extra_columns: false incremental: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-prestashop/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-prestashop/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-prestashop/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-prestashop/build.gradle b/airbyte-integrations/connectors/source-prestashop/build.gradle deleted file mode 100644 index 0c7570c7e0b1..000000000000 --- a/airbyte-integrations/connectors/source-prestashop/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_prestashop' -} diff --git a/airbyte-integrations/connectors/source-prestashop/metadata.yaml b/airbyte-integrations/connectors/source-prestashop/metadata.yaml index a3a5215f6727..184347008685 100644 --- a/airbyte-integrations/connectors/source-prestashop/metadata.yaml +++ b/airbyte-integrations/connectors/source-prestashop/metadata.yaml @@ -1,4 +1,7 @@ data: + ab_internal: + ql: 300 + sl: 100 allowedHosts: hosts: - ${domain} @@ -7,6 +10,7 @@ data: definitionId: d60a46d4-709f-4092-a6b7-2457f7d455f5 dockerImageTag: 1.0.0 dockerRepository: airbyte/source-prestashop + documentationUrl: https://docs.airbyte.com/integrations/sources/prestashop githubIssueLabel: source-prestashop icon: prestashop.svg license: MIT @@ -17,12 +21,8 @@ data: oss: enabled: true releaseStage: beta - documentationUrl: https://docs.airbyte.com/integrations/sources/prestashop + supportLevel: community tags: - language:low-code - language:python - ab_internal: - sl: 200 - ql: 300 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-primetric/README.md b/airbyte-integrations/connectors/source-primetric/README.md index 82805232285e..b5b6bed823e4 100644 --- a/airbyte-integrations/connectors/source-primetric/README.md +++ b/airbyte-integrations/connectors/source-primetric/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-primetric:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/primetric) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_primetric/spec.yaml` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-primetric:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-primetric build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-primetric:airbyteDocker +An image will be built with the tag `airbyte/source-primetric:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-primetric:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-primetric:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-primetric:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-primetric:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-primetric test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-primetric:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-primetric:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-primetric test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/primetric.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-primetric/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-primetric/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-primetric/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-primetric/build.gradle b/airbyte-integrations/connectors/source-primetric/build.gradle deleted file mode 100644 index a91acbf25af7..000000000000 --- a/airbyte-integrations/connectors/source-primetric/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_primetric' -} diff --git a/airbyte-integrations/connectors/source-public-apis/Dockerfile b/airbyte-integrations/connectors/source-public-apis/Dockerfile index 9dc3b03c8017..88ad494451f2 100644 --- a/airbyte-integrations/connectors/source-public-apis/Dockerfile +++ b/airbyte-integrations/connectors/source-public-apis/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.13-alpine3.15 as base +FROM python:3.9.11-alpine3.15 as base # build and load all requirements FROM base as builder @@ -34,5 +34,5 @@ COPY source_public_apis ./source_public_apis ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-public-apis diff --git a/airbyte-integrations/connectors/source-public-apis/README.md b/airbyte-integrations/connectors/source-public-apis/README.md index 7b6d5c541361..19e543a7e8cf 100644 --- a/airbyte-integrations/connectors/source-public-apis/README.md +++ b/airbyte-integrations/connectors/source-public-apis/README.md @@ -1,45 +1,12 @@ # Public Apis Source -This is the repository for the Public Apis source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/public-apis). +This is the repository for the Public Apis configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/public-apis). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-public-apis:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/public-apis) +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/public-apis) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_public_apis/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. @@ -47,28 +14,21 @@ See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source public-apis test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-public-apis:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-public-apis build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-public-apis:airbyteDocker +An image will be built with the tag `airbyte/source-public-apis:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-public-apis:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-public-apis:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-public-apis:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-public-apis:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-public-apis test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-public-apis:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-public-apis:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-public-apis test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/public-apis.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-public-apis/__init__.py b/airbyte-integrations/connectors/source-public-apis/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-public-apis/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-public-apis/acceptance-test-config.yml b/airbyte-integrations/connectors/source-public-apis/acceptance-test-config.yml index 1628949d5750..53347ad0c99d 100644 --- a/airbyte-integrations/connectors/source-public-apis/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-public-apis/acceptance-test-config.yml @@ -1,6 +1,7 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-public-apis:dev +test_strictness_level: low acceptance_tests: spec: tests: @@ -10,7 +11,7 @@ acceptance_tests: - config_path: "secrets/config.json" status: "succeed" - config_path: "integration_tests/invalid_config.json" - status: "failed" + status: "succeed" discovery: tests: - config_path: "secrets/config.json" @@ -18,9 +19,6 @@ acceptance_tests: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] - incremental: - bypass_reason: "This connector does not implement incremental sync" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-public-apis/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-public-apis/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-public-apis/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-public-apis/bootstrap.md b/airbyte-integrations/connectors/source-public-apis/bootstrap.md deleted file mode 100644 index 57d2529e8494..000000000000 --- a/airbyte-integrations/connectors/source-public-apis/bootstrap.md +++ /dev/null @@ -1,19 +0,0 @@ -## Streams - -[Public APIs](https://api.publicapis.org/) is a REST API without authentication. Connector has the following streams, and none of them support incremental refresh. - -* [Services](https://api.publicapis.org#get-entries) -* [Categories](https://api.publicapis.org#get-categories) - - -## Pagination - -[Public APIs](https://api.publicapis.org/) uses NO pagination. - -## Properties - -The connector configuration includes NO properties as this is a publi API without authentication. - -## Authentication - -[Public APIs](https://api.publicapis.org/) is a REST API without authentication. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-public-apis/build.gradle b/airbyte-integrations/connectors/source-public-apis/build.gradle deleted file mode 100644 index 2dc55d841593..000000000000 --- a/airbyte-integrations/connectors/source-public-apis/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_public_apis' -} diff --git a/airbyte-integrations/connectors/source-public-apis/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-public-apis/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-public-apis/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-public-apis/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-public-apis/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-public-apis/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-public-apis/metadata.yaml b/airbyte-integrations/connectors/source-public-apis/metadata.yaml index 8056a2a39b58..c83aba288893 100644 --- a/airbyte-integrations/connectors/source-public-apis/metadata.yaml +++ b/airbyte-integrations/connectors/source-public-apis/metadata.yaml @@ -1,24 +1,27 @@ data: + allowedHosts: + hosts: + - "*" + registries: + cloud: + enabled: false + oss: + enabled: true ab_internal: ql: 200 sl: 100 connectorSubtype: api connectorType: source definitionId: a4617b39-3c14-44cd-a2eb-6e720f269235 - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-public-apis documentationUrl: https://docs.airbyte.com/integrations/sources/public-apis githubIssueLabel: source-public-apis - icon: publicapi.svg + icon: public-apis.svg license: MIT - name: Public APIs - registries: - cloud: - enabled: true - oss: - enabled: true + name: Public Apis releaseStage: alpha supportLevel: community tags: - - language:python + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-public-apis/sample_files/config.json b/airbyte-integrations/connectors/source-public-apis/sample_files/config.json deleted file mode 100644 index 0967ef424bce..000000000000 --- a/airbyte-integrations/connectors/source-public-apis/sample_files/config.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/airbyte-integrations/connectors/source-public-apis/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-public-apis/sample_files/configured_catalog.json deleted file mode 100644 index 414979f92e93..000000000000 --- a/airbyte-integrations/connectors/source-public-apis/sample_files/configured_catalog.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "services", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "categories", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-public-apis/sample_files/invalid_config.json b/airbyte-integrations/connectors/source-public-apis/sample_files/invalid_config.json deleted file mode 100644 index 5a47475c1995..000000000000 --- a/airbyte-integrations/connectors/source-public-apis/sample_files/invalid_config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "invalid": "config_key" -} diff --git a/airbyte-integrations/connectors/source-public-apis/setup.py b/airbyte-integrations/connectors/source-public-apis/setup.py index d4c33672b789..8e3f218e28f7 100644 --- a/airbyte-integrations/connectors/source-public-apis/setup.py +++ b/airbyte-integrations/connectors/source-public-apis/setup.py @@ -6,14 +6,10 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", + "airbyte-cdk", ] -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest~=6.1", - "pytest-mock~=3.6.1", -] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( name="source_public_apis", diff --git a/airbyte-integrations/connectors/source-public-apis/source_public_apis/components.py b/airbyte-integrations/connectors/source-public-apis/source_public_apis/components.py new file mode 100644 index 000000000000..d659c4dfe5b1 --- /dev/null +++ b/airbyte-integrations/connectors/source-public-apis/source_public_apis/components.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, List, Mapping + +import requests +from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor + + +class CustomExtractor(RecordExtractor): + def extract_records(self, response: requests.Response, **kwargs) -> List[Mapping[str, Any]]: + + return [{"name": cat} for cat in response.json()["categories"]] diff --git a/airbyte-integrations/connectors/source-public-apis/source_public_apis/manifest.yaml b/airbyte-integrations/connectors/source-public-apis/source_public_apis/manifest.yaml new file mode 100644 index 000000000000..787e156db0d6 --- /dev/null +++ b/airbyte-integrations/connectors/source-public-apis/source_public_apis/manifest.yaml @@ -0,0 +1,60 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["entries"] + + selector_categories: + type: RecordSelector + extractor: + type: CustomRecordExtractor + class_name: source_public_apis.components.CustomExtractor + + requester: + type: HttpRequester + url_base: "https://api.publicapis.org/" + http_method: "GET" + authenticator: + type: NoAuth + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + categories_stream: + $ref: "#/definitions/base_stream" + retriever: + $ref: "#/definitions/retriever" + record_selector: + $ref: "#/definitions/selector_categories" + $parameters: + name: "categories" + path: "/categories" + + services_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "services" + path: "/entries" + +streams: + - "#/definitions/categories_stream" + - "#/definitions/services_stream" + +check: + type: CheckStream + stream_names: + - "categories" diff --git a/airbyte-integrations/connectors/source-public-apis/source_public_apis/schemas/categories.json b/airbyte-integrations/connectors/source-public-apis/source_public_apis/schemas/categories.json index a90ed1fede99..4855650fdea6 100644 --- a/airbyte-integrations/connectors/source-public-apis/source_public_apis/schemas/categories.json +++ b/airbyte-integrations/connectors/source-public-apis/source_public_apis/schemas/categories.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "name": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-public-apis/source_public_apis/schemas/services.json b/airbyte-integrations/connectors/source-public-apis/source_public_apis/schemas/services.json index 63823694a556..6c3076538df3 100644 --- a/airbyte-integrations/connectors/source-public-apis/source_public_apis/schemas/services.json +++ b/airbyte-integrations/connectors/source-public-apis/source_public_apis/schemas/services.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "API": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-public-apis/source_public_apis/source.py b/airbyte-integrations/connectors/source-public-apis/source_public_apis/source.py index e1aa4a26ec9e..b9925483338d 100644 --- a/airbyte-integrations/connectors/source-public-apis/source_public_apis/source.py +++ b/airbyte-integrations/connectors/source-public-apis/source_public_apis/source.py @@ -2,84 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import NoAuth +WARNING: Do not modify this file. +""" -class PublicApisStream(HttpStream, ABC): - url_base = "https://api.publicapis.org/" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return {} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield {} - - -class Categories(PublicApisStream): - primary_key = "name" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "categories" - - def parse_response( - self, - response: requests.Response, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[Mapping]: - return [{"name": cat} for cat in response.json()["categories"]] - - -class Services(PublicApisStream): - primary_key = "API" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "entries" - - def parse_response( - self, - response: requests.Response, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[Mapping]: - return response.json()["entries"] - - -# Source -class SourcePublicApis(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - """ - :param config: the user-input config object conforming to the connector's spec.yaml - :param logger: logger object - :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. - """ - if len(config) == 0: - return True, None - else: - return False, None - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - auth = NoAuth() - return [Services(authenticator=auth), Categories(authenticator=auth)] +# Declarative Source +class SourcePublicApis(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-public-apis/source_public_apis/spec.yaml b/airbyte-integrations/connectors/source-public-apis/source_public_apis/spec.yaml index dce7f95cc6c1..a995ee2e8f7c 100644 --- a/airbyte-integrations/connectors/source-public-apis/source_public_apis/spec.yaml +++ b/airbyte-integrations/connectors/source-public-apis/source_public_apis/spec.yaml @@ -3,5 +3,5 @@ connectionSpecification: $schema: http://json-schema.org/draft-07/schema# title: Public Apis Spec type: object - required: [] + additionalProperties: true properties: {} diff --git a/airbyte-integrations/connectors/source-public-apis/unit_tests/test_source.py b/airbyte-integrations/connectors/source-public-apis/unit_tests/test_source.py deleted file mode 100644 index 5ede02ae6007..000000000000 --- a/airbyte-integrations/connectors/source-public-apis/unit_tests/test_source.py +++ /dev/null @@ -1,21 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -from source_public_apis.source import SourcePublicApis - - -def test_check_connection(mocker): - source = SourcePublicApis() - logger_mock, config_mock = MagicMock(), MagicMock() - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -def test_streams(mocker): - source = SourcePublicApis() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 2 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-public-apis/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-public-apis/unit_tests/test_streams.py deleted file mode 100644 index e17d2e8df843..000000000000 --- a/airbyte-integrations/connectors/source-public-apis/unit_tests/test_streams.py +++ /dev/null @@ -1,75 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_public_apis.source import PublicApisStream - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(PublicApisStream, "path", "v0/example_endpoint") - mocker.patch.object(PublicApisStream, "primary_key", "test_primary_key") - mocker.patch.object(PublicApisStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = PublicApisStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = PublicApisStream() - inputs = {"response": MagicMock()} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_parse_response(patch_base_class): - stream = PublicApisStream() - inputs = {"response": MagicMock()} - expected_parsed_object = {} - assert next(stream.parse_response(**inputs)) == expected_parsed_object - - -def test_request_headers(patch_base_class): - stream = PublicApisStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = PublicApisStream() - # TODO: replace this with your expected http request method - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = PublicApisStream() - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = PublicApisStream() - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-punk-api/README.md b/airbyte-integrations/connectors/source-punk-api/README.md index 554789bfbcf5..9f142149b0ea 100644 --- a/airbyte-integrations/connectors/source-punk-api/README.md +++ b/airbyte-integrations/connectors/source-punk-api/README.md @@ -28,14 +28,6 @@ Note that while we are installing dependencies from `requirements.txt`, you shou used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-punk-api:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/punk-api) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_punk_api/spec.yaml` file. @@ -47,18 +39,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-punk-api:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-punk-api build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-punk-api:airbyteDocker +An image will be built with the tag `airbyte/source-punk-api:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-punk-api:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -68,25 +61,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-punk-api:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-punk-api:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-punk-api:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-punk-api test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-punk-api:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-punk-api:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -95,8 +80,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-punk-api test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/punk-api.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-punk-api/acceptance-test-config.yml b/airbyte-integrations/connectors/source-punk-api/acceptance-test-config.yml index 63b3771f1518..850dcfb5ca46 100644 --- a/airbyte-integrations/connectors/source-punk-api/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-punk-api/acceptance-test-config.yml @@ -1,4 +1,3 @@ - connector_image: airbyte/source-punk-api:dev acceptance_tests: spec: diff --git a/airbyte-integrations/connectors/source-punk-api/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-punk-api/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-punk-api/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-punk-api/build.gradle b/airbyte-integrations/connectors/source-punk-api/build.gradle deleted file mode 100644 index 80c533c57853..000000000000 --- a/airbyte-integrations/connectors/source-punk-api/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_punk_api' -} diff --git a/airbyte-integrations/connectors/source-pypi/README.md b/airbyte-integrations/connectors/source-pypi/README.md index aaf4457269df..0cb2011e4c2c 100644 --- a/airbyte-integrations/connectors/source-pypi/README.md +++ b/airbyte-integrations/connectors/source-pypi/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-pypi:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/pypi) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_pypi/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-pypi:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-pypi build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-pypi:airbyteDocker +An image will be built with the tag `airbyte/source-pypi:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-pypi:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pypi:dev check --confi docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pypi:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-pypi:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-pypi test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-pypi:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-pypi:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-pypi test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/pypi.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-pypi/acceptance-test-config.yml b/airbyte-integrations/connectors/source-pypi/acceptance-test-config.yml index 2dfd50f20235..62db84d624c0 100644 --- a/airbyte-integrations/connectors/source-pypi/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-pypi/acceptance-test-config.yml @@ -19,7 +19,7 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - incremental: + incremental: bypass_reason: "This connector does not implement incremental sync" full_refresh: tests: diff --git a/airbyte-integrations/connectors/source-pypi/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-pypi/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-pypi/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-pypi/build.gradle b/airbyte-integrations/connectors/source-pypi/build.gradle deleted file mode 100644 index e1fe15dd0e41..000000000000 --- a/airbyte-integrations/connectors/source-pypi/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_pypi' -} diff --git a/airbyte-integrations/connectors/source-python-http-tutorial/README.md b/airbyte-integrations/connectors/source-python-http-tutorial/README.md index 2833a0799ab8..94435e30ab93 100644 --- a/airbyte-integrations/connectors/source-python-http-tutorial/README.md +++ b/airbyte-integrations/connectors/source-python-http-tutorial/README.md @@ -64,17 +64,18 @@ python -m pytest unit_tests ### Locally running the connector docker image #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-python-http-tutorial:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-python-http-tutorial build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-python-http-tutorial:airbyteDocker +An image will be built with the tag `airbyte/source-python-http-tutorial:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-python-http-tutorial:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. + #### Run Then run any of the connector commands as follows: @@ -85,18 +86,25 @@ docker run --rm -v $(pwd)/sample_files:/sample_files airbyte/source-python-http- docker run --rm -v $(pwd)/sample_files:/sample_files -v $(pwd)/sample_files:/sample_files airbyte/source-python-http-tutorial:dev read --config /sample_files/config.json --catalog /sample_files/configured_catalog.json ``` -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-python-http-tutorial:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-python-http-tutorial test +``` + +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. ### Publishing a new version of the connector -You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-python-http-tutorial test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/python-http-tutorial.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-python-http-tutorial/build.gradle b/airbyte-integrations/connectors/source-python-http-tutorial/build.gradle deleted file mode 100644 index 735a02cd833e..000000000000 --- a/airbyte-integrations/connectors/source-python-http-tutorial/build.gradle +++ /dev/null @@ -1,31 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-standard-source-test-file' -} - -airbytePython { - moduleDirectory 'source_python_http_tutorial' -} - -airbyteStandardSourceTestFile { - // For more information on standard source tests, see https://docs.airbyte.com/connector-development/testing-connectors - - // All these input paths must live inside this connector's directory (or subdirectories) - // TODO update the spec JSON file - specPath = "source_python_http_tutorial/spec.json" - - // configPath points to a config file which matches the spec.json supplied above. secrets/ is gitignored by default, so place your config file - // there (in case it contains any credentials) - // TODO update the config file to contain actual credentials - configPath = "sample_files/config.json" - // TODO update the sample configured_catalog JSON for use in testing - // Note: If your source supports incremental syncing, then make sure that the catalog that is returned in the get_catalog method is configured - // for incremental syncing (e.g. include cursor fields, etc). - configuredCatalogPath = "sample_files/configured_catalog.json" -} - - -dependencies { - implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) -} diff --git a/airbyte-integrations/connectors/source-qonto/.dockerignore b/airbyte-integrations/connectors/source-qonto/.dockerignore deleted file mode 100644 index 028a3333acb2..000000000000 --- a/airbyte-integrations/connectors/source-qonto/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -* -!Dockerfile -!main.py -!source_qonto -!setup.py -!secrets diff --git a/airbyte-integrations/connectors/source-qonto/Dockerfile b/airbyte-integrations/connectors/source-qonto/Dockerfile index b753a3e701cb..8b18d125573e 100644 --- a/airbyte-integrations/connectors/source-qonto/Dockerfile +++ b/airbyte-integrations/connectors/source-qonto/Dockerfile @@ -34,5 +34,5 @@ COPY source_qonto ./source_qonto ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-qonto diff --git a/airbyte-integrations/connectors/source-qonto/README.md b/airbyte-integrations/connectors/source-qonto/README.md index c99921193907..10fb3bd5400b 100644 --- a/airbyte-integrations/connectors/source-qonto/README.md +++ b/airbyte-integrations/connectors/source-qonto/README.md @@ -1,14 +1,14 @@ -# Qonto Source +# Metabase Source -This is the repository for the Qonto source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/qonto). +This is the repository for the Metabase source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/metabase). ## Local development ### Prerequisites **To iterate on this connector, make sure to complete this prerequisites section.** -#### Minimum Python version required `= 3.9.0` +#### Minimum Python version required `= 3.7.0` #### Build & Activate Virtual Environment and install dependencies From this connector directory, create a virtual environment: @@ -21,7 +21,6 @@ development environment of choice. To activate it from the terminal, run: ``` source .venv/bin/activate pip install -r requirements.txt -pip install '.[tests]' ``` If you are in an IDE, follow your IDE's instructions to activate the virtualenv. @@ -30,73 +29,30 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-qonto:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/qonto) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_qonto/spec.yaml` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/metabase) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_metabase/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +See `sample_files/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source qonto test creds` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source metabase test creds` and place them into `secrets/config.json`. + ### Locally running the connector ``` python main.py spec python main.py check --config secrets/config.json python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - -### Locally running the connector docker image - -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-qonto:dev -``` - -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-qonto:airbyteDocker +python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. -#### Run -Then run any of the connector commands as follows: -``` -docker run --rm airbyte/source-qonto:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-qonto:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-qonto:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-qonto:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json -``` -## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` ### Unit Tests To run unit tests locally, from the connector directory run: ``` python -m pytest unit_tests ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` #### Acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. @@ -104,29 +60,17 @@ To run your integration tests with acceptance tests, from the connector root, ru ``` python -m pytest integration_tests -p integration_tests.acceptance ``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-qonto:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-qonto:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. -We split dependencies between two groups, dependencies that are: -* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. -* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-qonto test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/qonto.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-qonto/__init__.py b/airbyte-integrations/connectors/source-qonto/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-qonto/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-qonto/acceptance-test-config.yml b/airbyte-integrations/connectors/source-qonto/acceptance-test-config.yml index 5184e369fa39..04b2b86f6277 100644 --- a/airbyte-integrations/connectors/source-qonto/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-qonto/acceptance-test-config.yml @@ -1,20 +1,25 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-qonto:dev -tests: +acceptance_tests: spec: - - spec_path: "source_qonto/spec.yaml" + tests: + - spec_path: "source_qonto/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-qonto/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-qonto/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-qonto/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-qonto/build.gradle b/airbyte-integrations/connectors/source-qonto/build.gradle deleted file mode 100644 index 10cd050c1aa2..000000000000 --- a/airbyte-integrations/connectors/source-qonto/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_qonto' -} diff --git a/airbyte-integrations/connectors/source-qonto/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-qonto/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-qonto/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-qonto/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-qonto/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-qonto/integration_tests/invalid_config.json index a22da8cc220f..a3f53ed110da 100644 --- a/airbyte-integrations/connectors/source-qonto/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-qonto/integration_tests/invalid_config.json @@ -1,7 +1,6 @@ { "endpoint": "fake-endpoint", "iban": "fake_iban", - "organization_slug": "fake_organization", - "secret_key": "fake_secret", + "api_key": "my-orgs:123456", "start_date": "9999-99-99" } diff --git a/airbyte-integrations/connectors/source-qonto/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-qonto/integration_tests/sample_config.json index 52b6ca534b1a..f128c570fbad 100644 --- a/airbyte-integrations/connectors/source-qonto/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-qonto/integration_tests/sample_config.json @@ -1,7 +1,6 @@ { "endpoint": "Test Mocked API Server", "iban": "REPLACEME", - "organization_slug": "REPLACEME", - "secret_key": "REPLACEME", - "start_date": "2022-11-04" + "api_key": "my-orgs:fdsf12345", + "start_date": "2023-11-11" } diff --git a/airbyte-integrations/connectors/source-qonto/metadata.yaml b/airbyte-integrations/connectors/source-qonto/metadata.yaml index ccf5eb801992..05c0393aa6ec 100644 --- a/airbyte-integrations/connectors/source-qonto/metadata.yaml +++ b/airbyte-integrations/connectors/source-qonto/metadata.yaml @@ -1,24 +1,17 @@ data: connectorSubtype: api connectorType: source - definitionId: f7c0b910-5f66-11ed-9b6a-0242ac120002 - dockerImageTag: 0.1.0 + definitionId: ccd3901d-edf3-4e58-900c-942d6990aa59 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-qonto githubIssueLabel: source-qonto icon: qonto.svg license: MIT - name: Qonto - registries: - cloud: - enabled: false - oss: - enabled: true + name: Qonto My + releaseDate: "2023-10-19" releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/qonto tags: - - language:python - ab_internal: - sl: 100 - ql: 100 - supportLevel: community + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-qonto/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-qonto/sample_files/configured_catalog.json deleted file mode 100644 index a4e6293f3212..000000000000 --- a/airbyte-integrations/connectors/source-qonto/sample_files/configured_catalog.json +++ /dev/null @@ -1,157 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "memberships", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "first_name": { - "type": "string" - }, - "last_name": { - "type": "string" - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "labels", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "parent_id": { - "type": "string" - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "transactions", - "json_schema": { - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "transaction_id": { - "type": "string" - }, - "amount": { - "type": "number" - }, - "amount_cents": { - "type": "integer" - }, - "settled_balance": { - "type": "number" - }, - "settled_balance_cents": { - "type": "integer" - }, - "attachment_ids": { - "type": ["array"], - "items": { - "type": ["string"] - } - }, - "local_amount": { - "type": "number" - }, - "local_amount_cents": { - "type": "integer" - }, - "side": { - "type": "string" - }, - "operation_type": { - "type": "string" - }, - "currency": { - "type": "string" - }, - "local_currency": { - "type": "string" - }, - "label": { - "type": "string" - }, - "settled_at": { - "type": "string" - }, - "emitted_at": { - "type": "string" - }, - "updated_at": { - "type": "string" - }, - "status": { - "type": "string" - }, - "note": { - "type": "string" - }, - "reference": { - "type": "string" - }, - "vat_amount": { - "type": "number" - }, - "vat_amount_cents": { - "type": "number" - }, - "vat_rate": { - "type": "number" - }, - "initiator_id": { - "type": "string" - }, - "label_ids": { - "type": ["array"], - "items": { - "type": ["string"] - } - }, - "attachment_lost": { - "type": "boolean" - }, - "attachment_required": { - "type": "boolean" - }, - "card_last_digits": { - "type": "string" - }, - "category": { - "type": "string" - }, - "id": { - "type": "string" - } - } - }, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-qonto/setup.py b/airbyte-integrations/connectors/source-qonto/setup.py index 61647d4573b0..8847f9443761 100644 --- a/airbyte-integrations/connectors/source-qonto/setup.py +++ b/airbyte-integrations/connectors/source-qonto/setup.py @@ -6,20 +6,20 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", - "pytest~=6.1", + "pytest~=6.2", "pytest-mock~=3.6.1", ] setup( name="source_qonto", - description="Source implementation for Qonto API.", - author="Leïla Ballouard", - author_email="leila.ballouard@backmarket.com", + description="Source implementation for Qonto.", + author="Airbyte", + author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, diff --git a/airbyte-integrations/connectors/source-qonto/source_qonto/auth.py b/airbyte-integrations/connectors/source-qonto/source_qonto/auth.py deleted file mode 100644 index 1e049b9e3df9..000000000000 --- a/airbyte-integrations/connectors/source-qonto/source_qonto/auth.py +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator - - -class QontoApiKeyAuthenticator(AbstractHeaderAuthenticator): - """ - QontoApiKeyAuthenticator sets a request header on the HTTP requests sent. - - The header is of the form: - `"Authorization": ""` - - For example, - `QontoApiKeyAuthenticator("my-organization", "3564f")` - will result in the following header set on the HTTP request - `"Authorization": "my-organization:3564f"` - - Attributes: - organization_slug (str): Organization slug to use in the header - secret_key (str): Secret key to use in the header - """ - - def __init__(self, organization_slug: str, secret_key: str): - super().__init__() - self.organization_slug = organization_slug - self.secret_key = secret_key - - @property - def auth_header(self) -> str: - return "Authorization" - - @property - def token(self) -> str: - return f"{self.organization_slug}:{self.secret_key}" diff --git a/airbyte-integrations/connectors/source-qonto/source_qonto/endpoint.py b/airbyte-integrations/connectors/source-qonto/source_qonto/endpoint.py deleted file mode 100644 index 71595f7deacb..000000000000 --- a/airbyte-integrations/connectors/source-qonto/source_qonto/endpoint.py +++ /dev/null @@ -1,17 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from typing import Dict - -QONTO_ENDPOINT_MAP: Dict = { - "Sandbox": "https://thirdparty-sandbox.staging.qonto.co/v2/", - "Production": "https://thirdparty.qonto.com/v2/", - "Test Mocked API Server": "https://stoplight.io/mocks/qonto-next/business-api/8419419/v2/", -} - - -def get_url_base(endpoint: str) -> str: - """Define the URL Base from user's input with respect to the QONTO_ENDPOINT_MAP""" - return QONTO_ENDPOINT_MAP.get(endpoint) diff --git a/airbyte-integrations/connectors/source-qonto/source_qonto/manifest.yaml b/airbyte-integrations/connectors/source-qonto/source_qonto/manifest.yaml new file mode 100644 index 000000000000..3ecb41d2b01d --- /dev/null +++ b/airbyte-integrations/connectors/source-qonto/source_qonto/manifest.yaml @@ -0,0 +1,122 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - "{{ parameters['path'] }}" + requester: + type: HttpRequester + url_base: >- + {{ 'https://thirdparty.qonto.com/v2/' if config['endpoint'] == 'Production' + else ('https://stoplight.io/mocks/qonto-next/business-api/8419419/v2' if config['endpoint'] == 'Test Mocked API Server' + else '') }} + http_method: "GET" + authenticator: + type: ApiKeyAuthenticator + api_token: "{{ config.get('api_key') }}" + inject_into: + type: RequestOption + inject_into: header + field_name: Authorization + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: current_page + pagination_strategy: + type: CursorPagination + cursor_value: '{{ response.get("meta", {}).get("next_page", {}) }}' + stop_condition: '{{ not response.get("meta", {}).get("next_page", {}) }}' + requester: + $ref: "#/definitions/requester" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + primary_key: "id" + + labels_stream: + $ref: "#/definitions/base_stream" + name: "labels" + $parameters: + path: "labels" + + memberships_stream: + $ref: "#/definitions/base_stream" + name: "memberships" + $parameters: + path: "memberships" + + transactions_stream: + $ref: "#/definitions/base_stream" + name: "transactions" + $parameters: + path: "transactions" + request_parameters: + iban: "{{ config['iban'] }}" + settled_at_from: "{{ config['start_date'] }}" + +streams: + - "#/definitions/labels_stream" + - "#/definitions/memberships_stream" + - "#/definitions/transactions_stream" + +check: + type: CheckStream + stream_names: + - labels + - memberships + - transactions + +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/qonto + connection_specification: + title: Qonto Spec + type: object + required: + - endpoint + - iban + additionalProperties: true + properties: + api_key: + title: API key + type: string + description: "Fill it in this format: `:`" + pattern: ^[\w-]+:[\w]+$ + pattern_descriptor: organization_slug:secret_key + examples: + - "my-organization:3564f" + order: 1 + endpoint: + title: Endpoint + type: string + description: Please choose the right endpoint to use in this connection + enum: + #- Sandbox # not yet supported + - Production + - Test Mocked API Server + order: 0 + iban: + title: IBAN + type: string + description: International Bank Account Number linked used with your Qonto Account + pattern: ^[A-Z0-9]*$ + order: 2 + start_date: + title: Start date + type: string + description: Start getting data from that date. + format: date + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ + examples: + - YYYY-MM-DD diff --git a/airbyte-integrations/connectors/source-qonto/source_qonto/schemas/memberships.json b/airbyte-integrations/connectors/source-qonto/source_qonto/schemas/memberships.json index 0be6884ee3dc..6d55d4d08760 100644 --- a/airbyte-integrations/connectors/source-qonto/source_qonto/schemas/memberships.json +++ b/airbyte-integrations/connectors/source-qonto/source_qonto/schemas/memberships.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "string" @@ -10,6 +11,24 @@ }, "last_name": { "type": ["null", "string"] + }, + "birth_country": { + "type": ["null", "string"] + }, + "birthdate": { + "type": ["null", "string"] + }, + "nationality": { + "type": ["null", "string"] + }, + "residence_country": { + "type": ["null", "string"] + }, + "role": { + "type": ["null", "string"] + }, + "ubo": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-qonto/source_qonto/schemas/transactions.json b/airbyte-integrations/connectors/source-qonto/source_qonto/schemas/transactions.json index 60a3a42328fa..44df4b27a870 100644 --- a/airbyte-integrations/connectors/source-qonto/source_qonto/schemas/transactions.json +++ b/airbyte-integrations/connectors/source-qonto/source_qonto/schemas/transactions.json @@ -1,10 +1,19 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "transaction_id": { "type": ["null", "string"] }, + "subject_type": { + "type": ["null", "string"] + }, + "transfer": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} + }, "amount": { "type": ["null", "number"] }, @@ -18,9 +27,9 @@ "type": ["null", "integer"] }, "attachment_ids": { - "type": ["array"], + "type": ["null", "array"], "items": { - "type": ["string"] + "type": ["null", "string"] } }, "local_amount": { @@ -75,9 +84,9 @@ "type": ["null", "string"] }, "label_ids": { - "type": ["array"], + "type": ["null", "array"], "items": { - "type": ["string"] + "type": ["null", "string"] } }, "attachment_lost": { diff --git a/airbyte-integrations/connectors/source-qonto/source_qonto/source.py b/airbyte-integrations/connectors/source-qonto/source_qonto/source.py index 86e70e5fb723..6e2722fa88b3 100644 --- a/airbyte-integrations/connectors/source-qonto/source_qonto/source.py +++ b/airbyte-integrations/connectors/source-qonto/source_qonto/source.py @@ -2,151 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from abc import ABC -from datetime import datetime -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from source_qonto.auth import QontoApiKeyAuthenticator -from source_qonto.endpoint import get_url_base +WARNING: Do not modify this file. +""" -# Basic full refresh stream -class QontoStream(HttpStream, ABC): - """ - This class represents a stream output by the connector. - This is an abstract base class meant to contain all the common functionality at the API level. - - Each stream should extend this class (or another abstract subclass of it) to specify behavior unique to that stream. - """ - - next_page_token_field = "current_page" - primary_key = "id" - - def __init__(self, config: dict, stream_name: str, **kwargs): - auth = QontoApiKeyAuthenticator(organization_slug=config["organization_slug"], secret_key=config["secret_key"]) - super().__init__(authenticator=auth, **kwargs) - self.stream_name = stream_name - self.config = config - - @property - def url_base(self) -> str: - return get_url_base(self.config["endpoint"]) - - def path( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> str: - return self.stream_name - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - Define how a response is parsed. - :return an iterable containing each record in the response - """ - response_json = response.json() - yield from response_json[self.stream_name] - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - Define a pagination strategy. - - :param response: the most recent response from the API - :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. - If there are no more pages in the result, return None. - """ - decoded_response = response.json() - api_metadata = decoded_response.get("meta", None) - if api_metadata is None: - return None - else: - next_page = api_metadata.get("next_page", None) - if next_page is None: - return None - else: - return {"current_page": next_page} - - -class Memberships(QontoStream): - name = "memberships" - - def __init__(self, config, **kwargs): - super().__init__(config, self.name) - - -class Labels(QontoStream): - name = "labels" - - def __init__(self, config, **kwargs): - super().__init__(config, self.name) - - -class Transactions(QontoStream): - name = "transactions" - cursor_date_format = "%Y-%m-%d" - - def __init__(self, config, **kwargs): - super().__init__(config, self.name) - self.start_date = config["start_date"] - self.iban = config["iban"] - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - """ - Define any query parameters to be set. - """ - start_date = datetime.strptime(stream_state.get(self.cursor_field) if stream_state else self.start_date, self.cursor_date_format) - params = {"iban": self.iban, "settled_at_from": start_date.strftime(self.cursor_date_format)} - if next_page_token: - params.update(next_page_token) - return params - - -# Source -class SourceQonto(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - """ - Validate that the user-provided config can be used to connect to the underlying API - - :param config: the user-input config object conforming to the connector's spec.yaml - :param logger: logger object - :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. - """ - try: - headers = {"Authorization": f'{config["organization_slug"]}:{config["secret_key"]}'} - params = {"iban": config["iban"]} - resp = requests.request("GET", url=f"{get_url_base(config['endpoint'])}/transactions", params=params, headers=headers) - status = resp.status_code - logger.info(f"Ping response code: {status}") - if status == 200: - return True, None - if status == 404: - if resp.text == " ": # When Iban is wrong, the request returns only " " as content - message = "Not Found, the specified IBAN might be wrong" - else: - message = resp.json().get("errors")[0].get("detail") - return False, message - if status == 401: - message = "Invalid credentials, the organization slug or secret key might be wrong" - return False, message - return False, message - except Exception as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - Return a the list of streams that will be enabled in the connector - - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - return [Memberships(config), Transactions(config), Labels(config)] +# Declarative Source +class SourceQonto(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-qonto/source_qonto/spec.yaml b/airbyte-integrations/connectors/source-qonto/source_qonto/spec.yaml deleted file mode 100644 index f6d6e654335e..000000000000 --- a/airbyte-integrations/connectors/source-qonto/source_qonto/spec.yaml +++ /dev/null @@ -1,40 +0,0 @@ -documentationUrl: https://docsurl.com -connectionSpecification: - $schema: http://json-schema.org/draft-07/schema# - title: Qonto Spec - type: object - required: - - endpoint - - organization_slug - - secret_key - - iban - properties: - endpoint: - title: Endpoint - type: string - description: Please choose the right endpoint to use in this connection - enum: - #- Sandbox # not yet supported - - Production - - Test Mocked API Server - organization_slug: - title: Organization slug - type: string - description: Organization slug used in Qonto - secret_key: - title: Secret Key - type: string - description: Secret key of the Qonto account - airbyte_secret: true - iban: - title: IBAN - type: string - description: International Bank Account Number linked used with your Qonto Account - pattern: ^[A-Z0-9]*$ - start_date: - title: Start date - type: string - description: Start getting data from that date. - pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}$ - examples: - - YYYY-MM-DD diff --git a/airbyte-integrations/connectors/source-qonto/unit_tests/test_auth.py b/airbyte-integrations/connectors/source-qonto/unit_tests/test_auth.py deleted file mode 100644 index 65d6f513811e..000000000000 --- a/airbyte-integrations/connectors/source-qonto/unit_tests/test_auth.py +++ /dev/null @@ -1,13 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from source_qonto.auth import QontoApiKeyAuthenticator - - -def test_authenticator(): - mocked_config = {"organization_slug": "test_slug", "secret_key": "test_key"} - authenticator = QontoApiKeyAuthenticator(**mocked_config) - expected_authenticator = {"Authorization": "test_slug:test_key"} - assert authenticator.get_auth_header() == expected_authenticator diff --git a/airbyte-integrations/connectors/source-qonto/unit_tests/test_source.py b/airbyte-integrations/connectors/source-qonto/unit_tests/test_source.py deleted file mode 100644 index 9cecd9a16db8..000000000000 --- a/airbyte-integrations/connectors/source-qonto/unit_tests/test_source.py +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock, patch - -import pytest -from source_qonto.source import SourceQonto - - -@pytest.mark.parametrize( - ("http_status", "response_text", "expected_result"), - [ - (HTTPStatus.OK, "", (True, None)), - (HTTPStatus.NOT_FOUND, " ", (False, "Not Found, the specified IBAN might be wrong")), - ( - HTTPStatus.UNAUTHORIZED, - "Invalid credentials", - (False, "Invalid credentials, the organization slug or secret key might be wrong"), - ), - ], -) -def test_check_connection(mocker, http_status, response_text, expected_result): - with patch("requests.request") as mock_request: - mock_request.return_value.status_code = http_status - mock_request.return_value.text = response_text - source = SourceQonto() - logger_mock, config_mock = MagicMock(), MagicMock() - print(http_status) - assert source.check_connection(logger_mock, config_mock) == expected_result - - -def test_streams(mocker): - source = SourceQonto() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 3 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-qonto/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-qonto/unit_tests/test_streams.py deleted file mode 100644 index 9fda40854e11..000000000000 --- a/airbyte-integrations/connectors/source-qonto/unit_tests/test_streams.py +++ /dev/null @@ -1,127 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock, patch - -import pytest -import requests -from source_qonto.source import QontoStream, Transactions - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(QontoStream, "path", "v0/example_endpoint") - mocker.patch.object(QontoStream, "primary_key", "test_primary_key") - - def __mocked_init__(self): - self.stream_name = "test_stream_name" - pass - - mocker.patch.object(QontoStream, "__init__", __mocked_init__) - - -# Base Class -def test_request_params(patch_base_class): - - stream = QontoStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = QontoStream() - simple_page_response_json = { - "transactions": [], - "meta": {"current_page": 1, "next_page": None, "prev_page": None, "total_pages": 3, "total_count": 210, "per_page": 100}, - } - multiple_page_response_json = { - "transactions": [], - "meta": {"current_page": 5, "next_page": 6, "prev_page": 4, "total_pages": 7, "total_count": 210, "per_page": 100}, - } - with patch.object(requests.Response, "json", return_value=simple_page_response_json): - inputs = {"response": requests.Response()} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - with patch.object(requests.Response, "json", return_value=multiple_page_response_json): - inputs = {"response": requests.Response()} - expected_token = {"current_page": 6} - assert stream.next_page_token(**inputs) == expected_token - - -def test_parse_response(patch_base_class): - stream = QontoStream() - mock_response_json = { - "test_stream_name": [ - {"id": "171dba70-c75f-4337-b419-377a59bc9cf3", "name": "Fantastic Marble Wallet", "parent_id": None}, - { - "id": "2487a014-618f-40e3-8a1f-eb76e883efc5", - "name": "Fantastic Bronze Computer", - "parent_id": "171dba70-c75f-4337-b419-377a59bc9cf3", - }, - ], - "meta": {"current_page": 1, "next_page": None, "prev_page": None, "total_pages": 1, "total_count": 2, "per_page": 100}, - } - with patch.object(requests.Response, "json", return_value=mock_response_json): - inputs = {"response": requests.Response()} - expected_parsed_object = {"id": "171dba70-c75f-4337-b419-377a59bc9cf3", "name": "Fantastic Marble Wallet", "parent_id": None} - assert next(stream.parse_response(**inputs)) == expected_parsed_object - - -def test_request_headers(patch_base_class): - stream = QontoStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = QontoStream() - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = QontoStream() - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = QontoStream() - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time - - -# Transactions Class -def test_transactions_request_params(): - mocked_config = { - "organization_slug": "test_slug", - "secret_key": "test_key", - "iban": "FRXXXXXXXXXXXXXXXXXXXXXXXXX", - "start_date": "2022-06-01", - } - stream = Transactions(mocked_config) - - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {"iban": stream.iban, "settled_at_from": stream.start_date} - assert stream.request_params(**inputs) == expected_params - - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": {"current_page": 6}} - expected_params = {"iban": stream.iban, "settled_at_from": stream.start_date, "current_page": 6} - assert stream.request_params(**inputs) == expected_params diff --git a/airbyte-integrations/connectors/source-qualaroo/.dockerignore b/airbyte-integrations/connectors/source-qualaroo/.dockerignore index a224999d042b..72a4047972a5 100644 --- a/airbyte-integrations/connectors/source-qualaroo/.dockerignore +++ b/airbyte-integrations/connectors/source-qualaroo/.dockerignore @@ -1,6 +1,5 @@ * !Dockerfile -!Dockerfile.test !main.py !source_qualaroo !setup.py diff --git a/airbyte-integrations/connectors/source-qualaroo/Dockerfile b/airbyte-integrations/connectors/source-qualaroo/Dockerfile index 94c4c48ff6de..889137e3a673 100644 --- a/airbyte-integrations/connectors/source-qualaroo/Dockerfile +++ b/airbyte-integrations/connectors/source-qualaroo/Dockerfile @@ -6,7 +6,9 @@ WORKDIR /airbyte/integration_code # upgrade pip to the latest version RUN apk --no-cache upgrade \ - && pip install --upgrade pip + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + COPY setup.py ./ # install necessary packages to a temporary folder @@ -18,8 +20,11 @@ WORKDIR /airbyte/integration_code # copy all loaded and built libraries to a pure basic image COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone -# Bash is installed for more convenient debugging. +# bash is installed for more convenient debugging. RUN apk --no-cache add bash # copy payload code only @@ -29,5 +34,5 @@ COPY source_qualaroo ./source_qualaroo ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.version=0.3.0 LABEL io.airbyte.name=airbyte/source-qualaroo diff --git a/airbyte-integrations/connectors/source-qualaroo/README.md b/airbyte-integrations/connectors/source-qualaroo/README.md index 0d7d94544593..2c8fdc2325e2 100644 --- a/airbyte-integrations/connectors/source-qualaroo/README.md +++ b/airbyte-integrations/connectors/source-qualaroo/README.md @@ -1,73 +1,34 @@ # Qualaroo Source -This is the repository for the Qualaroo source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/qualaroo). +This is the repository for the Qualaroo configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/qualaroo). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-qualaroo:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/qualaroo) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_qualaroo/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/qualaroo) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_qualaroo/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source qualaroo test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-qualaroo:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-qualaroo build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-qualaroo:airbyteDocker +An image will be built with the tag `airbyte/source-qualaroo:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-qualaroo:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -77,44 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-qualaroo:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-qualaroo:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-qualaroo:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-qualaroo test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-qualaroo:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-qualaroo:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -124,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-qualaroo test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/qualaroo.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-qualaroo/__init__.py b/airbyte-integrations/connectors/source-qualaroo/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-qualaroo/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-qualaroo/acceptance-test-config.yml b/airbyte-integrations/connectors/source-qualaroo/acceptance-test-config.yml index 3d77add761e5..6241c7d6d3c7 100644 --- a/airbyte-integrations/connectors/source-qualaroo/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-qualaroo/acceptance-test-config.yml @@ -1,20 +1,27 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-qualaroo:dev -tests: +acceptance_tests: spec: - - spec_path: "source_qualaroo/spec.json" + tests: + - spec_path: "source_qualaroo/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + incremental: + bypass_reason: "This connector does not implement incremental sync" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-qualaroo/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-qualaroo/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-qualaroo/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-qualaroo/build.gradle b/airbyte-integrations/connectors/source-qualaroo/build.gradle deleted file mode 100644 index 050d16c27b33..000000000000 --- a/airbyte-integrations/connectors/source-qualaroo/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_qualaroo' -} diff --git a/airbyte-integrations/connectors/source-qualaroo/icon.svg b/airbyte-integrations/connectors/source-qualaroo/icon.svg index bc68344d292e..fa02bf350848 100644 --- a/airbyte-integrations/connectors/source-qualaroo/icon.svg +++ b/airbyte-integrations/connectors/source-qualaroo/icon.svg @@ -1 +1,6 @@ - \ No newline at end of file + + + + + + diff --git a/airbyte-integrations/connectors/source-qualaroo/integration_tests/__init__.py b/airbyte-integrations/connectors/source-qualaroo/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-qualaroo/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-qualaroo/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-qualaroo/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-qualaroo/integration_tests/abnormal_state.json index 4c3b2cd94d48..52b0f2c2118f 100644 --- a/airbyte-integrations/connectors/source-qualaroo/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-qualaroo/integration_tests/abnormal_state.json @@ -1,5 +1,5 @@ { - "actions": { - "date": "3021-08-18T08:35:49.540Z" + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" } } diff --git a/airbyte-integrations/connectors/source-qualaroo/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-qualaroo/integration_tests/acceptance.py index d49b55882333..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-qualaroo/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-qualaroo/integration_tests/acceptance.py @@ -10,4 +10,7 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-qualaroo/integration_tests/catalog.json b/airbyte-integrations/connectors/source-qualaroo/integration_tests/catalog.json deleted file mode 100644 index b5706b1ef05e..000000000000 --- a/airbyte-integrations/connectors/source-qualaroo/integration_tests/catalog.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "surveys", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "responses", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-qualaroo/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-qualaroo/integration_tests/configured_catalog.json index b5706b1ef05e..98d2544aae83 100644 --- a/airbyte-integrations/connectors/source-qualaroo/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-qualaroo/integration_tests/configured_catalog.json @@ -4,8 +4,7 @@ "stream": { "name": "surveys", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -14,8 +13,7 @@ "stream": { "name": "responses", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" diff --git a/airbyte-integrations/connectors/source-qualaroo/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-qualaroo/integration_tests/sample_config.json new file mode 100644 index 000000000000..cf86ff0cf858 --- /dev/null +++ b/airbyte-integrations/connectors/source-qualaroo/integration_tests/sample_config.json @@ -0,0 +1,5 @@ +{ + "token": "sample_token", + "key": "sample_key", + "start_date": "3021-02-11T08:35:49.540Z" +} diff --git a/airbyte-integrations/connectors/source-qualaroo/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-qualaroo/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-qualaroo/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-qualaroo/metadata.yaml b/airbyte-integrations/connectors/source-qualaroo/metadata.yaml index 5e8ab3d7a8dc..e291b82f879a 100644 --- a/airbyte-integrations/connectors/source-qualaroo/metadata.yaml +++ b/airbyte-integrations/connectors/source-qualaroo/metadata.yaml @@ -1,24 +1,25 @@ data: - ab_internal: - ql: 200 - sl: 100 + allowedHosts: + hosts: + - "*" # Please change to the hostname of the source. + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source - definitionId: b08e4776-d1de-4e80-ab5c-1e51dad934a2 - dockerImageTag: 0.2.0 + definitionId: eb655362-28a8-4311-8806-4fcc612734a7 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-qualaroo - documentationUrl: https://docs.airbyte.com/integrations/sources/qualaroo githubIssueLabel: source-qualaroo icon: qualaroo.svg license: MIT name: Qualaroo - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: "2021-08-18" releaseStage: alpha supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/sources/qualaroo tags: - - language:python + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-qualaroo/requirements.txt b/airbyte-integrations/connectors/source-qualaroo/requirements.txt index d6e1198b1ab1..cf563bcab685 100644 --- a/airbyte-integrations/connectors/source-qualaroo/requirements.txt +++ b/airbyte-integrations/connectors/source-qualaroo/requirements.txt @@ -1 +1,2 @@ -e . +-e ../../bases/connector-acceptance-test \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-qualaroo/setup.py b/airbyte-integrations/connectors/source-qualaroo/setup.py index b2929697645c..840126cac8fb 100644 --- a/airbyte-integrations/connectors/source-qualaroo/setup.py +++ b/airbyte-integrations/connectors/source-qualaroo/setup.py @@ -6,23 +6,23 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "requests-mock~=1.9.3", + "pytest~=6.2", "pytest-mock~=3.6.1", - "requests-mock", ] setup( name="source_qualaroo", description="Source implementation for Qualaroo.", - author="Daniel Diamond", - author_email="danieldiamond1@gmail.com", + author="Airbyte", + author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/__init__.py b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/__init__.py index b27f605765f0..99c5fb690f89 100644 --- a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/__init__.py +++ b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/__init__.py @@ -1,25 +1,5 @@ # -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/components.py b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/components.py new file mode 100644 index 000000000000..5e4e619d4d44 --- /dev/null +++ b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/components.py @@ -0,0 +1,34 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from base64 import b64encode +from dataclasses import dataclass +from typing import Any, List, Mapping + +import requests +from airbyte_cdk.sources.declarative.auth.token import BasicHttpAuthenticator +from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor + + +@dataclass +class CustomAuthenticator(BasicHttpAuthenticator): + @property + def token(self): + + key = str(self._username.eval(self.config)).encode("latin1") + token = self._password.eval(self.config).encode("latin1") + encoded_credentials = b64encode(b":".join((key, token))).strip() + token = "Basic " + encoded_credentials.decode("ascii") + return token + + +class CustomExtractor(RecordExtractor): + def extract_records(self, response: requests.Response, **kwargs) -> List[Mapping[str, Any]]: + + extracted = [] + for record in response.json(): + if "answered_questions" in record: + record["answered_questions"] = list(record["answered_questions"].values()) + extracted.append(record) + return extracted diff --git a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/manifest.yaml b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/manifest.yaml new file mode 100644 index 000000000000..863c36e34494 --- /dev/null +++ b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/manifest.yaml @@ -0,0 +1,127 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + record_filter: + condition: "{{ record['id'] in config['survey_ids'] if config['survey_ids'] else true}}" + requester: + type: HttpRequester + url_base: "https://api.qualaroo.com/api/v1/" + http_method: "GET" + authenticator: + class_name: source_qualaroo.components.CustomAuthenticator + username: "{{config['key']}}" + password: "{{config['token']}}" + request_parameters: + limit: "500" + start_date: "{{config['start_date']}}" + error_handler: + response_filters: + - http_codes: [500] + action: FAIL + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "OffsetIncrement" + page_size: 500 + page_token_option: + type: "RequestOption" + field_name: "offset" + inject_into: "request_parameter" + requester: + $ref: "#/definitions/requester" + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + surveys_stream: + $ref: "#/definitions/base_stream" + name: "surveys" + primary_key: "id" + $parameters: + path: "nudges" + + responses_stream: + name: "responses" + primary_key: "id" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + path: "nudges/{{ stream_slice.parent_id }}/responses.json" + record_selector: + type: RecordSelector + extractor: + class_name: source_qualaroo.components.CustomExtractor + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/surveys_stream" + parent_key: "id" + partition_field: "parent_id" + +streams: + - "#/definitions/surveys_stream" + - "#/definitions/responses_stream" + +check: + type: CheckStream + stream_names: + - "responses" + +spec: + type: Spec + documentationUrl: https://docs.airbyte.com/integrations/sources/qualaroo + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + title: Qualaroo Spec + type: object + required: + - token + - key + - start_date + additionalProperties: true + properties: + token: + type: string + title: API token + description: >- + A Qualaroo token. See the docs + for instructions on how to generate it. + airbyte_secret: true + key: + type: string + title: API key + description: >- + A Qualaroo token. See the docs + for instructions on how to generate it. + airbyte_secret: true + start_date: + type: string + title: Start Date + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z$ + description: >- + UTC date and time in the format 2017-01-25T00:00:00Z. Any data before + this date will not be replicated. + examples: + - "2021-03-01T00:00:00.000Z" + survey_ids: + type: array + items: + type: string + pattern: ^[0-9]{1,8}$ + title: Qualaroo survey IDs + description: >- + IDs of the surveys from which you'd like to replicate data. If left + empty, data from all surveys to which you have access will be + replicated. diff --git a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/schemas/responses.json b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/schemas/responses.json index 5dfab07c0dd3..db74aa16cfa5 100644 --- a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/schemas/responses.json +++ b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/schemas/responses.json @@ -15,6 +15,9 @@ "identity": { "type": ["null", "string"] }, + "location": { + "type": ["null", "string"] + }, "page": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/source.py b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/source.py index 1ea0bb6514ac..d9312d834b37 100644 --- a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/source.py +++ b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/source.py @@ -2,80 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from base64 import b64encode -from typing import Any, List, Mapping, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import pendulum -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator +WARNING: Do not modify this file. +""" -from .streams import QualarooStream, Responses, Surveys - -class QualarooAuthenticator(HttpAuthenticator): - """ - Generate auth header for start making requests from API token and API key. - """ - - def __init__( - self, - key: str, - token: str, - auth_header: str = "Authorization", - key_header: str = "oauth_consumer_key", - token_header: str = "oauth_token", - ): - self._key = key - self._token = b64encode(b":".join((key.encode("latin1"), token.encode("latin1")))).strip().decode("ascii") - self.auth_header = auth_header - self.key_header = key_header - self.token_header = token_header - - def get_auth_header(self) -> Mapping[str, Any]: - return {self.auth_header: f"Basic {self._token}"} - - -class SourceQualaroo(AbstractSource): - """ - Source Qualaroo fetch date from web-based, Kanban-style, list-making application. - """ - - @staticmethod - def _get_authenticator(config: dict) -> QualarooAuthenticator: - key, token = config["key"], config["token"] - return QualarooAuthenticator(token=token, key=key) - - def check_connection(self, logger, config) -> Tuple[bool, any]: - """ - Testing connection availability for the connector by granting the credentials. - """ - - try: - url = f"{QualarooStream.url_base}nudges" - - authenticator = self._get_authenticator(config) - - response = requests.get(url, headers=authenticator.get_auth_header()) - - response.raise_for_status() - available_surveys = {row.get("id") for row in response.json()} - for survey_id in config.get("survey_ids", []): - if survey_id not in available_surveys: - return False, f"survey_id {survey_id} not found" - return True, None - except requests.exceptions.RequestException as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - args = {} - # convert start_date to epoch time for qualaroo API - args["start_date"] = pendulum.parse(config["start_date"]).strftime("%s") - args["survey_ids"] = config.get("survey_ids", []) - args["authenticator"] = self._get_authenticator(config) - return [Surveys(**args), Responses(**args)] +# Declarative Source +class SourceQualaroo(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/spec.json b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/spec.json deleted file mode 100644 index 20c61166c40c..000000000000 --- a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/spec.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/qualaroo", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Qualaroo Spec", - "type": "object", - "required": ["token", "key", "start_date"], - "additionalProperties": true, - "properties": { - "token": { - "type": "string", - "title": "API token", - "description": "A Qualaroo token. See the docs for instructions on how to generate it.", - "airbyte_secret": true - }, - "key": { - "type": "string", - "title": "API key", - "description": "A Qualaroo token. See the docs for instructions on how to generate it.", - "airbyte_secret": true - }, - "start_date": { - "type": "string", - "title": "Start Date", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z$", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2021-03-01T00:00:00.000Z"] - }, - "survey_ids": { - "type": "array", - "items": { - "type": "string", - "pattern": "^[0-9]{1,8}$" - }, - "title": "Qualaroo survey IDs", - "description": "IDs of the surveys from which you'd like to replicate data. If left empty, data from all surveys to which you have access will be replicated." - } - } - } -} diff --git a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/streams.py b/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/streams.py deleted file mode 100644 index 6bd5a0cfefb8..000000000000 --- a/airbyte-integrations/connectors/source-qualaroo/source_qualaroo/streams.py +++ /dev/null @@ -1,102 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional - -import pendulum -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http import HttpStream - - -class QualarooStream(HttpStream, ABC): - url_base = "https://api.qualaroo.com/api/v1/" - - # Define primary key as sort key for full_refresh, or very first sync for incremental_refresh - primary_key = "id" - - # Page size - limit = 500 - - extra_params = None - - def __init__(self, start_date: pendulum.datetime, survey_ids: List[str] = [], **kwargs): - super().__init__(**kwargs) - self._start_date = start_date - self._survey_ids = survey_ids - self._offset = 0 - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - resp_json = response.json() - - if len(resp_json) == 500: - self._offset += 500 - return {"offset": self._offset} - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params = {"limit": self.limit, "start_date": self._start_date} - if next_page_token: - params.update(**next_page_token) - if self.extra_params: - params.update(self.extra_params) - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - json_response = response.json() - for record in json_response: - yield record - - -class ChildStreamMixin: - parent_stream_class: Optional[QualarooStream] = None - - def stream_slices(self, sync_mode, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - for item in self.parent_stream_class(config=self.config).read_records(sync_mode=sync_mode): - yield {"id": item["id"]} - - -class Surveys(QualarooStream): - """Return list of all Surveys. - API Docs: https://help.qualaroo.com/hc/en-us/articles/201969438-The-REST-Reporting-API - Endpoint: https://api.qualaroo.com/api/v1/nudges/ - """ - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return "nudges" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - survey_ids = self._survey_ids - result = super().parse_response(response=response, **kwargs) - for record in result: - if not survey_ids or str(record["id"]) in survey_ids: - yield record - - -class Responses(ChildStreamMixin, QualarooStream): - """Return list of all responses of a survey. - API Docs: hhttps://help.qualaroo.com/hc/en-us/articles/201969438-The-REST-Reporting-API - Endpoint: https://api.qualaroo.com/api/v1/nudges//responses.json - """ - - parent_stream_class = Surveys - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - survey_id = stream_slice["survey_id"] - return f"nudges/{survey_id}/responses.json" - - def stream_slices(self, **kwargs): - survey_stream = Surveys(start_date=self._start_date, survey_ids=self._survey_ids, authenticator=self.authenticator) - for survey in survey_stream.read_records(sync_mode=SyncMode.full_refresh): - yield {"survey_id": survey["id"]} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - response_data = response.json() - # de-nest the answered_questions object if exists - for rec in response_data: - if "answered_questions" in rec: - rec["answered_questions"] = list(rec["answered_questions"].values()) - yield from response_data diff --git a/airbyte-integrations/connectors/source-qualaroo/unit_tests/__init__.py b/airbyte-integrations/connectors/source-qualaroo/unit_tests/__init__.py deleted file mode 100644 index 9db886e0930f..000000000000 --- a/airbyte-integrations/connectors/source-qualaroo/unit_tests/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# diff --git a/airbyte-integrations/connectors/source-qualaroo/unit_tests/conftest.py b/airbyte-integrations/connectors/source-qualaroo/unit_tests/conftest.py deleted file mode 100644 index 8d9c18dfc54d..000000000000 --- a/airbyte-integrations/connectors/source-qualaroo/unit_tests/conftest.py +++ /dev/null @@ -1,10 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from pytest import fixture - - -@fixture -def config(): - return {"start_date": "start_date", "authenticator": None} diff --git a/airbyte-integrations/connectors/source-qualaroo/unit_tests/helpers.py b/airbyte-integrations/connectors/source-qualaroo/unit_tests/helpers.py deleted file mode 100644 index b4e062d6522e..000000000000 --- a/airbyte-integrations/connectors/source-qualaroo/unit_tests/helpers.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -NO_SLEEP_HEADERS = { - "x-rate-limit-api-token-max": "1", - "x-rate-limit-api-token-remaining": "1", - "x-rate-limit-api-key-max": "1", - "x-rate-limit-api-key-remaining": "1", -} - - -def read_all_records(stream): - records = [] - slices = stream.stream_slices(sync_mode=None) - for slice in slices: - for record in stream.read_records(sync_mode=None, stream_slice=slice): - records.append(record) - return records diff --git a/airbyte-integrations/connectors/source-qualaroo/unit_tests/test_components.py b/airbyte-integrations/connectors/source-qualaroo/unit_tests/test_components.py new file mode 100644 index 000000000000..f34157a71350 --- /dev/null +++ b/airbyte-integrations/connectors/source-qualaroo/unit_tests/test_components.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json + +import requests +from source_qualaroo.components import CustomAuthenticator, CustomExtractor + + +def test_token_generation(): + + config = {"key": "4524324", "token": "token"} + authenticator = CustomAuthenticator(config=config, username="example@gmail.com", password="api_key", parameters=None) + token = authenticator.token + expected_token = "Basic ZXhhbXBsZUBnbWFpbC5jb206YXBpX2tleQ==" + assert expected_token == token + + +def test_extract_records_with_answered_questions(): + + response_data = [ + {"id": 1, "answered_questions": {"q1": "A1", "q2": "A2"}}, + {"id": 2, "answered_questions": {"q3": "A3"}}, + ] + response = requests.Response() + response._content = json.dumps(response_data).encode("utf-8") + extracted_records = CustomExtractor().extract_records(response) + expected_records = [{"id": 1, "answered_questions": ["A1", "A2"]}, {"id": 2, "answered_questions": ["A3"]}] + assert expected_records == extracted_records diff --git a/airbyte-integrations/connectors/source-qualaroo/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-qualaroo/unit_tests/test_streams.py deleted file mode 100644 index 30e0dad7fdc7..000000000000 --- a/airbyte-integrations/connectors/source-qualaroo/unit_tests/test_streams.py +++ /dev/null @@ -1,102 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from unittest.mock import MagicMock - -from pytest import fixture -from source_qualaroo.source import QualarooStream, Responses, Surveys - -from .helpers import NO_SLEEP_HEADERS, read_all_records - - -@fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(QualarooStream, "path", "v0/example_endpoint") - mocker.patch.object(QualarooStream, "primary_key", "test_primary_key") - mocker.patch.object(QualarooStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class, config): - stream = QualarooStream(**config) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": {"before": "id"}} - expected_params = {"limit": 500, "start_date": "start_date", "before": "id"} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class, config): - stream = QualarooStream(**config) - inputs = {"response": MagicMock()} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_surveys_stream(requests_mock): - mock_surveys_request = requests_mock.get( - "https://api.qualaroo.com/api/v1/nudges?limit=500&start_date=2021-02-11T08%3A35%3A49.540Z", - headers=NO_SLEEP_HEADERS, - json=[{"id": "b11111111111111111111111", "name": "survey_1"}, {"id": "b22222222222222222222222", "name": "survey_2"}], - ) - - args = {"authenticator": None, "start_date": "2021-02-11T08:35:49.540Z", "survey_ids": []} - stream1 = Surveys(**args) - records = read_all_records(stream1) - assert records == [{"id": "b11111111111111111111111", "name": "survey_1"}, {"id": "b22222222222222222222222", "name": "survey_2"}] - - args["survey_ids"] = ["b22222222222222222222222"] - stream2 = Surveys(**args) - records = read_all_records(stream2) - assert records == [{"id": "b22222222222222222222222", "name": "survey_2"}] - - args["survey_ids"] = ["not-found"] - stream3 = Surveys(**args) - records = read_all_records(stream3) - assert records == [] - - assert mock_surveys_request.call_count == 3 - - -def test_responses_stream(requests_mock): - mock_surveys_request = requests_mock.get( - "https://api.qualaroo.com/api/v1/nudges?limit=500&start_date=2021-02-11T08%3A35%3A49.540Z", - headers=NO_SLEEP_HEADERS, - json=[{"id": "b11111111111111111111111", "name": "survey_1"}, {"id": "b22222222222222222222222", "name": "survey_2"}], - ) - - mock_responses_request_1 = requests_mock.get( - "https://api.qualaroo.com/api/v1/nudges/b11111111111111111111111/responses.json", - headers=NO_SLEEP_HEADERS, - json=[{"id": "c11111111111111111111111", "name": "response_1"}, {"id": "c22222222222222222222222", "name": "response_2"}], - ) - - mock_responses_request_2 = requests_mock.get( - "https://api.qualaroo.com/api/v1/nudges/b22222222222222222222222/responses.json", - headers=NO_SLEEP_HEADERS, - json=[{"id": "c33333333333333333333333", "name": "response_3"}, {"id": "c44444444444444444444444", "name": "response_4"}], - ) - - args = {"authenticator": None, "start_date": "2021-02-11T08:35:49.540Z", "survey_ids": []} - stream1 = Responses(**args) - records = read_all_records(stream1) - assert records == [ - {"id": "c11111111111111111111111", "name": "response_1"}, - {"id": "c22222222222222222222222", "name": "response_2"}, - {"id": "c33333333333333333333333", "name": "response_3"}, - {"id": "c44444444444444444444444", "name": "response_4"}, - ] - - args["survey_ids"] = ["b22222222222222222222222"] - stream2 = Responses(**args) - records = read_all_records(stream2) - assert records == [{"id": "c33333333333333333333333", "name": "response_3"}, {"id": "c44444444444444444444444", "name": "response_4"}] - - args["survey_ids"] = ["not-found"] - stream3 = Responses(**args) - records = read_all_records(stream3) - assert records == [] - - assert mock_surveys_request.call_count == 3 - assert mock_responses_request_1.call_count == 1 - assert mock_responses_request_2.call_count == 2 diff --git a/airbyte-integrations/connectors/source-quickbooks/Dockerfile b/airbyte-integrations/connectors/source-quickbooks/Dockerfile index 79811759d651..18808a53082e 100644 --- a/airbyte-integrations/connectors/source-quickbooks/Dockerfile +++ b/airbyte-integrations/connectors/source-quickbooks/Dockerfile @@ -34,5 +34,5 @@ COPY source_quickbooks ./source_quickbooks ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=2.0.4 +LABEL io.airbyte.version=3.0.1 LABEL io.airbyte.name=airbyte/source-quickbooks diff --git a/airbyte-integrations/connectors/source-quickbooks/README.md b/airbyte-integrations/connectors/source-quickbooks/README.md index 919f75785d0a..bf8d8b6eb750 100644 --- a/airbyte-integrations/connectors/source-quickbooks/README.md +++ b/airbyte-integrations/connectors/source-quickbooks/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-quickbooks:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/quickbooks) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_quickbooks/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-quickbooks:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-quickbooks build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-quickbooks:airbyteDocker +An image will be built with the tag `airbyte/source-quickbooks:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-quickbooks:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-quickbooks:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-quickbooks:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-quickbooks:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-quickbooks test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-quickbooks:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-quickbooks:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-quickbooks test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/quickbooks.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-quickbooks/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-quickbooks/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-quickbooks/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-quickbooks/build.gradle b/airbyte-integrations/connectors/source-quickbooks/build.gradle deleted file mode 100644 index 1769a74fe5f8..000000000000 --- a/airbyte-integrations/connectors/source-quickbooks/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_quickbooks' -} diff --git a/airbyte-integrations/connectors/source-quickbooks/metadata.yaml b/airbyte-integrations/connectors/source-quickbooks/metadata.yaml index b25b55b3e4fe..0961738252b3 100644 --- a/airbyte-integrations/connectors/source-quickbooks/metadata.yaml +++ b/airbyte-integrations/connectors/source-quickbooks/metadata.yaml @@ -7,7 +7,7 @@ data: connectorSubtype: api connectorType: source definitionId: cf9c4355-b171-4477-8f2d-6c5cc5fc8b7e - dockerImageTag: 2.0.4 + dockerImageTag: 3.0.1 dockerRepository: airbyte/source-quickbooks githubIssueLabel: source-quickbooks icon: quickbooks.svg @@ -19,6 +19,11 @@ data: oss: enabled: true releaseStage: alpha + releases: + breakingChanges: + 3.0.0: + message: "Some fields in `bills`, `credit_memos`, `items`, `refund_receipts`, and `sales_receipts` streams have been changed from `integer` to `number` to fix normalization. You may need to refresh the connection schema for those streams (skipping the reset), and running a sync. Alternatively, you can just run a reset." + upgradeDeadline: 2023-10-04 documentationUrl: https://docs.airbyte.com/integrations/sources/quickbooks tags: - language:low-code diff --git a/airbyte-integrations/connectors/source-quickbooks/setup.py b/airbyte-integrations/connectors/source-quickbooks/setup.py index 025726239f79..47ca1b128505 100644 --- a/airbyte-integrations/connectors/source-quickbooks/setup.py +++ b/airbyte-integrations/connectors/source-quickbooks/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk>=0.44.0", + "airbyte-cdk>=0.58.8", ] TEST_REQUIREMENTS = [ diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/components.py b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/components.py index d1f3a93fc12e..529738699638 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/components.py +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/components.py @@ -69,7 +69,7 @@ class CustomDatetimeBasedCursor(DatetimeBasedCursor): def close_slice(self, stream_slice: StreamSlice, most_recent_record: typing.Optional[Record]) -> None: super(CustomDatetimeBasedCursor, self).close_slice( stream_slice=stream_slice, - last_record=LastRecordDictProxy(most_recent_record, {self.cursor_field.eval(self.config): "MetaData/LastUpdatedTime"}), + most_recent_record=LastRecordDictProxy(most_recent_record, {self.cursor_field.eval(self.config): "MetaData/LastUpdatedTime"}), ) def _format_datetime(self, dt: datetime.datetime): diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/manifest.yaml b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/manifest.yaml index ac70de7bbc6a..267f04d857d0 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/manifest.yaml +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/manifest.yaml @@ -26,7 +26,6 @@ definitions: client_id: "{{ config['credentials']['client_id'] }}" client_secret: "{{ config['credentials']['client_secret'] }}" refresh_token: "{{ config['credentials']['refresh_token'] }}" - refresh_token_updater: {} retriever: type: SimpleRetriever record_selector: diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/bills.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/bills.json index 8498337dfd9b..166120688ebc 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/bills.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/bills.json @@ -121,7 +121,7 @@ "type": ["null", "string"] }, "UnitPrice": { - "type": ["null", "integer"] + "type": ["null", "number"] } }, "type": ["null", "object"] diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/credit_memos.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/credit_memos.json index 784c98df94c3..91170b2b5fe8 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/credit_memos.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/credit_memos.json @@ -28,7 +28,7 @@ "type": ["null", "string"] }, "Balance": { - "type": ["null", "integer"] + "type": ["null", "number"] }, "CustomerMemo": { "properties": { @@ -81,7 +81,7 @@ "type": ["null", "object"] }, "UnitPrice": { - "type": ["null", "integer"] + "type": ["null", "number"] }, "TaxCodeRef": { "properties": { @@ -224,7 +224,7 @@ "type": ["null", "object"] }, "RemainingCredit": { - "type": ["null", "integer"] + "type": ["null", "number"] }, "ExchangeRate": { "type": ["null", "number"] diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/items.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/items.json index 91b393d430df..de54d96d8e05 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/items.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/items.json @@ -73,7 +73,7 @@ "type": ["null", "integer"] }, "UnitPrice": { - "type": ["null", "integer"] + "type": ["null", "number"] }, "SyncToken": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/refund_receipts.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/refund_receipts.json index afefac60eab4..e7271b4955c4 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/refund_receipts.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/refund_receipts.json @@ -78,7 +78,7 @@ "type": ["null", "object"] }, "UnitPrice": { - "type": ["null", "integer"] + "type": ["null", "number"] } }, "type": ["null", "object"] diff --git a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/sales_receipts.json b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/sales_receipts.json index 291fd278ab75..b9ec030fbd03 100644 --- a/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/sales_receipts.json +++ b/airbyte-integrations/connectors/source-quickbooks/source_quickbooks/schemas/sales_receipts.json @@ -166,7 +166,7 @@ "type": ["null", "number"] }, "UnitPrice": { - "type": ["null", "integer"] + "type": ["null", "number"] }, "TaxCodeRef": { "type": ["null", "object"], diff --git a/airbyte-integrations/connectors/source-quickbooks/unit_tests/test_custom_component.py b/airbyte-integrations/connectors/source-quickbooks/unit_tests/test_custom_component.py index 3519eabaf89c..12e38b4157e1 100644 --- a/airbyte-integrations/connectors/source-quickbooks/unit_tests/test_custom_component.py +++ b/airbyte-integrations/connectors/source-quickbooks/unit_tests/test_custom_component.py @@ -9,13 +9,7 @@ def test_dict_proxy(): - record = { - "Id": "1", - "MetaData": { - "CreateTime": "2023-02-10T14:42:07-08:00", - "LastUpdatedTime": "2023-02-18T13:13:33-08:00" - } - } + record = {"Id": "1", "MetaData": {"CreateTime": "2023-02-10T14:42:07-08:00", "LastUpdatedTime": "2023-02-18T13:13:33-08:00"}} proxy = LastRecordDictProxy(record, {"airbyte_cursor": "MetaData/LastUpdatedTime"}) assert proxy["MetaData/LastUpdatedTime"] == "2023-02-18T13:13:33-08:00" @@ -37,10 +31,7 @@ def test_dict_proxy(): assert "CreateTime" not in record["MetaData"] - assert record == { - "Id": "2", - "MetaData": {"LastUpdatedTime": "0000-00-00T00:00:00+00:00"} - } + assert record == {"Id": "2", "MetaData": {"LastUpdatedTime": "0000-00-00T00:00:00+00:00"}} def test_custom_datetime_based_cursor__close_slice(): @@ -55,22 +46,13 @@ def test_custom_datetime_based_cursor__close_slice(): datetime_format="%Y-%m-%dT%H:%M:%S%z", cursor_granularity="PT0S", config={}, - parameters={} + parameters={}, ) slice_end_time = "2023-03-03T00:00:00+00:00" date_time_based_cursor_component.close_slice( - { - "start_time": "2023-02-01T00:00:00+00:00", - "end_time": slice_end_time - }, - { - "Id": "1", - "MetaData": { - "CreateTime": "2023-02-10T14:42:07-08:00", - "LastUpdatedTime": record_cursor_value - } - } + {"start_time": "2023-02-01T00:00:00+00:00", "end_time": slice_end_time}, + {"Id": "1", "MetaData": {"CreateTime": "2023-02-10T14:42:07-08:00", "LastUpdatedTime": record_cursor_value}}, ) assert date_time_based_cursor_component.get_stream_state() == {cursor_field_name: slice_end_time} @@ -84,7 +66,7 @@ def test_custom_datetime_based_cursor__format_datetime(): datetime_format="%Y-%m-%dT%H:%M:%S%z", cursor_granularity="PT0S", config={}, - parameters={} + parameters={}, ) _format_datetime = getattr(date_time_based_cursor_component, "_format_datetime") @@ -101,7 +83,7 @@ def test_custom_datetime_based_cursor__parse_datetime(): datetime_format="%Y-%m-%dT%H:%M:%S%z", cursor_granularity="PT0S", config={}, - parameters={} + parameters={}, ) datetime_string_original_offset = "2023-02-10T14:42:05-08:00" diff --git a/airbyte-integrations/connectors/source-railz/README.md b/airbyte-integrations/connectors/source-railz/README.md index 298d7f443184..4e8976d2d648 100644 --- a/airbyte-integrations/connectors/source-railz/README.md +++ b/airbyte-integrations/connectors/source-railz/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-railz:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/railz) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_railz/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-railz:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-railz build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-railz:airbyteDocker +An image will be built with the tag `airbyte/source-railz:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-railz:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,27 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-railz:dev check --conf docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-railz:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-railz:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. -If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. - -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-railz test ``` -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-railz:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-railz:integrationTest -``` +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-railz test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/railz.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-railz/acceptance-test-config.yml b/airbyte-integrations/connectors/source-railz/acceptance-test-config.yml index 3fa37ad10467..a2a9431e605e 100644 --- a/airbyte-integrations/connectors/source-railz/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-railz/acceptance-test-config.yml @@ -32,25 +32,6 @@ acceptance_tests: configured_catalog_path: "integration_tests/configured_catalog.json" future_state: future_state_path: "integration_tests/abnormal_state.json" - threshold_days: 30 - cursor_paths: - accounting_transactions: ["b_dynamicsBusinessCentral", "dynamicsBusinessCentral", "postedDate"] - bank_transfers: ["b_xero", "xero", "date"] - bills: ["b_freshbooks", "freshbooks", "postedDate"] - bills_credit_notes: ["b_xero", "xero", "postedDate"] - bills_payments: ["b_dynamicsBusinessCentral", "dynamicsBusinessCentral", "date"] - commerce_disputes: ["b_square", "square", "createdDate"] - commerce_orders: ["b_square", "square", "createdDate"] - commerce_products: ["b_square", "square", "createdDate"] - commerce_transactions: ["b_square", "square", "createdDate"] - deposits: ["b_quickbooks", "quickbooks", "postedDate"] - estimates: ["b_xero", "xero", "postedDate"] - invoices: ["b_sageBusinessCloud", "sageBusinessCloud", "postedDate"] - invoices_credit_notes: ["b_freshbooks", "freshbooks", "postedDate"] - invoices_payments: ["b_freshbooks", "freshbooks", "date"] - journal_entries: ["b_freshbooks", "freshbooks", "postedDate"] - purchase_orders: ["b_oracleNetsuite", "oracleNetsuite", "postedDate"] - refunds: ["b_xero", "xero", "date"] full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-railz/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-railz/acceptance-test-docker.sh deleted file mode 100755 index a8d6ac4bb608..000000000000 --- a/airbyte-integrations/connectors/source-railz/acceptance-test-docker.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env sh - -# Build latest connector image -docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) - -# Pull latest acctest image -docker pull airbyte/connector-acceptance-test:latest - -# Run -docker run --rm -it \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /tmp:/tmp \ - -v $(pwd):/test_input \ - airbyte/connector-acceptance-test \ - --acceptance-test-config /test_input - diff --git a/airbyte-integrations/connectors/source-railz/build.gradle b/airbyte-integrations/connectors/source-railz/build.gradle deleted file mode 100644 index fbfcc81a63a5..000000000000 --- a/airbyte-integrations/connectors/source-railz/build.gradle +++ /dev/null @@ -1,13 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_railz' -} - -dependencies { - implementation files(project(':airbyte-integrations:bases:connector-acceptance-test').airbyteDocker.outputs) -} diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/README.md b/airbyte-integrations/connectors/source-rd-station-marketing/README.md index c734746adca1..3523942fef00 100644 --- a/airbyte-integrations/connectors/source-rd-station-marketing/README.md +++ b/airbyte-integrations/connectors/source-rd-station-marketing/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-rd-station-marketing:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/rd-station-marketing) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_rd_station_marketing/spec.yaml` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/cat ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-rd-station-marketing:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-rd-station-marketing build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-rd-station-marketing:airbyteDocker +An image will be built with the tag `airbyte/source-rd-station-marketing:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-rd-station-marketing:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,45 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-rd-station-marketing:d docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-rd-station-marketing:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-rd-station-marketing:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-rd-station-marketing test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). - -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-rd-station-marketing:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-rd-station-marketing:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -126,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-rd-station-marketing test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/rd-station-marketing.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-rd-station-marketing/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-rd-station-marketing/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-rd-station-marketing/build.gradle b/airbyte-integrations/connectors/source-rd-station-marketing/build.gradle deleted file mode 100644 index 9ef4cd8c1c07..000000000000 --- a/airbyte-integrations/connectors/source-rd-station-marketing/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_rd_station_marketing' -} diff --git a/airbyte-integrations/connectors/source-recharge/Dockerfile b/airbyte-integrations/connectors/source-recharge/Dockerfile deleted file mode 100644 index bbc8964f4f60..000000000000 --- a/airbyte-integrations/connectors/source-recharge/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY source_recharge ./source_recharge -COPY main.py ./ -COPY setup.py ./ -RUN pip install . - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=1.0.0 -LABEL io.airbyte.name=airbyte/source-recharge diff --git a/airbyte-integrations/connectors/source-recharge/README.md b/airbyte-integrations/connectors/source-recharge/README.md index 35e086388522..53853dd9e62c 100644 --- a/airbyte-integrations/connectors/source-recharge/README.md +++ b/airbyte-integrations/connectors/source-recharge/README.md @@ -28,14 +28,6 @@ Note that while we are installing dependencies from `requirements.txt`, you shou used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-recharge:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/recharge) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_recharge/spec.json` file. @@ -55,18 +47,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-recharge:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-recharge build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-recharge:airbyteDocker +An image will be built with the tag `airbyte/source-recharge:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-recharge:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -76,45 +69,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-recharge:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-recharge:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-recharge:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-recharge test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](../../../docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-recharge:dev \ -&& python -m pytest -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-recharge:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-recharge:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -124,8 +88,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-recharge test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/recharge.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml index c7b80b0e67e0..028b6f90e143 100644 --- a/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-recharge/acceptance-test-config.yml @@ -1,56 +1,56 @@ acceptance_tests: basic_read: tests: - - config_path: secrets/config.json - empty_streams: - - name: collections - bypass_reason: "volatile data" - - name: discounts - bypass_reason: "volatile data" - - name: onetimes - bypass_reason: "no data from stream" - - name: orders - bypass_reason: "no data from stream" - - name: subscriptions - bypass_reason: "no data from stream" - ignored_fields: - shop: - - name: shop/updated_at - bypass_reason: "updated after login" - - name: store/updated_at - bypass_reason: "updated after login" - timeout_seconds: 7200 - expect_records: - path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: no - extra_records: yes - fail_on_extra_columns: false + - config_path: secrets/config.json + empty_streams: + - name: collections + bypass_reason: "volatile data" + - name: discounts + bypass_reason: "volatile data" + - name: onetimes + bypass_reason: "no data from stream" + - name: orders + bypass_reason: "no data from stream" + - name: subscriptions + bypass_reason: "no data from stream" + ignored_fields: + shop: + - name: shop/updated_at + bypass_reason: "updated after login" + - name: store/updated_at + bypass_reason: "updated after login" + timeout_seconds: 7200 + expect_records: + path: "integration_tests/expected_records.jsonl" + extra_fields: no + exact_order: no + extra_records: yes + fail_on_extra_columns: false connection: tests: - - config_path: secrets/config.json - status: succeed - - config_path: integration_tests/invalid_config.json - status: failed + - config_path: secrets/config.json + status: succeed + - config_path: integration_tests/invalid_config.json + status: failed discovery: tests: - - backward_compatibility_tests_config: - disable_for_version: 0.2.10 - config_path: secrets/config.json + - backward_compatibility_tests_config: + disable_for_version: 0.2.10 + config_path: secrets/config.json full_refresh: tests: - - config_path: secrets/config.json - configured_catalog_path: integration_tests/configured_catalog.json - timeout_seconds: 3200 + - config_path: secrets/config.json + configured_catalog_path: integration_tests/configured_catalog.json + timeout_seconds: 3200 incremental: tests: - - config_path: secrets/config.json - configured_catalog_path: integration_tests/streams_with_output_records_catalog.json - future_state: - future_state_path: integration_tests/abnormal_state.json - timeout_seconds: 3200 + - config_path: secrets/config.json + configured_catalog_path: integration_tests/streams_with_output_records_catalog.json + future_state: + future_state_path: integration_tests/abnormal_state.json + timeout_seconds: 3200 spec: tests: - - spec_path: source_recharge/spec.json + - spec_path: source_recharge/spec.json connector_image: airbyte/source-recharge:dev test_strictness_level: high diff --git a/airbyte-integrations/connectors/source-recharge/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-recharge/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-recharge/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-recharge/build.gradle b/airbyte-integrations/connectors/source-recharge/build.gradle deleted file mode 100644 index b453adf1f38e..000000000000 --- a/airbyte-integrations/connectors/source-recharge/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_recharge' -} diff --git a/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records.jsonl index 5c192d0d884c..9b1b0db102b2 100644 --- a/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-recharge/integration_tests/expected_records.jsonl @@ -1,8 +1,12 @@ -{"stream": "addresses", "data": {"id": 69105381, "customer_id": 64817252, "payment_method_id": 12482012, "address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "created_at": "2021-05-12T12:04:06+00:00", "discounts": [], "first_name": "Jane", "last_name": "Doe", "order_attributes": [], "order_note": null, "phone": "1234567890", "presentment_currency": "USD", "province": "California", "shipping_lines_conserved": [], "shipping_lines_override": [], "updated_at": "2023-01-16T09:59:09+00:00", "zip": "94118"}, "emitted_at": 1680895024611} -{"stream": "charges", "data": {"id": 386976088, "address_id": 69105381, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Karina", "last_name": "Kuznetsova", "phone": null, "province": "California", "zip": "94118"}, "charge_attempts": 6, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2021-05-12T12:04:07+00:00", "currency": "USD", "customer": {"id": 64817252, "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "hash": "23dee52d73734a81"}, "discounts": [], "error": "None\r\n [May 12, 12:06AM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 13, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 19, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 25, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 31, 4:09PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [Jun 06, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']", "error_type": "CLOSED_MAX_RETRIES_REACHED", "external_order_id": {"ecommerce": null}, "external_transaction_id": {"payment_processor": null}, "external_variant_not_found": null, "has_uncommitted_changes": false, "last_charge_attempt": "2022-06-06T20:10:19+00:00", "line_items": [{"purchase_item_id": 153224593, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684722131115"}, "grams": 0, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T1", "tax_due": "0.00", "tax_lines": [], "taxable": true, "taxable_amount": "24.30", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "S / Black"}], "note": null, "order_attributes": [], "orders_count": 0, "payment_processor": "shopify_payments", "processed_at": null, "retry_date": "2022-06-12T04:00:00+00:00", "scheduled_at": "2022-05-12", "shipping_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Jane", "last_name": "Doe", "phone": "1234567890", "province": "California", "zip": "94118"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "source": "shopify", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "error", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": true, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-01-16T18:08:54+00:00"}, "emitted_at": 1687184458990} -{"stream": "customers", "data": {"id": 64817252, "analytics_data": {"utm_params": []}, "created_at": "2021-05-12T12:04:06+00:00", "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "first_charge_processed_at": "2021-05-12T16:03:59+00:00", "first_name": "Karina", "has_payment_method_in_dunning": false, "has_valid_payment_method": true, "hash": "23dee52d73734a81", "last_name": "Kuznetsova", "phone": null, "subscriptions_active_count": 0, "subscriptions_total_count": 1, "tax_exempt": false, "updated_at": "2023-01-16T18:08:45+00:00"}, "emitted_at": 1687184599794} -{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T07:27:34", "discount_amount": 5.0, "discount_type": "percentage", "handle": "i-make-beats-wool-blend-snapback", "id": 1853639, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_small.jpg"}, "product_id": 6644278001835, "shopify_product_id": 6644278001835, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "I Make Beats Wool Blend Snapback", "updated_at": "2021-05-13T07:27:34"}, "emitted_at": 1680895030371} -{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T08:20:10", "discount_amount": 0.0, "discount_type": "percentage", "handle": "new-mug", "id": 1853655, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_small.jpg"}, "product_id": 6688261701803, "shopify_product_id": 6688261701803, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "NEW!!! MUG", "updated_at": "2021-05-13T08:20:10"}, "emitted_at": 1680895030371} -{"stream": "shop", "data": {"shop": {"allow_customers_to_skip_delivery": 1, "checkout_logo_url": null, "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Wed, 05 Apr 2023 02:44:22 GMT"}, "store": {"checkout_logo_url": null, "checkout_platform": "shopify", "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Wed, 05 Apr 2023 02:44:22 GMT"}}, "emitted_at": 1680895031312} -{"stream": "shop", "data": {"shop": {"allow_customers_to_skip_delivery": 1, "checkout_logo_url": null, "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Wed, 05 Apr 2023 02:44:22 GMT"}, "store": {"checkout_logo_url": null, "checkout_platform": "shopify", "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Wed, 05 Apr 2023 02:44:22 GMT"}}, "emitted_at": 1680895031459} -{"stream": "metafields", "data": {"created_at": "2023-04-10T07:10:45", "description": "customer_phone_number", "id": 3627108, "key": "phone_number", "namespace": "personal_info", "owner_id": "64962974", "owner_resource": "customer", "updated_at": "2023-04-10T07:10:45", "value": "3103103101", "value_type": "integer"}, "emitted_at": 1681125056810} +{"stream": "addresses", "data": {"id": 69105381, "customer_id": 64817252, "payment_method_id": 12482012, "address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "created_at": "2021-05-12T12:04:06+00:00", "discounts": [], "first_name": "Jane", "last_name": "Doe", "order_attributes": [], "order_note": null, "phone": "1234567890", "presentment_currency": "USD", "province": "California", "shipping_lines_conserved": [], "shipping_lines_override": [], "updated_at": "2023-01-16T09:59:09+00:00", "zip": "94118"}, "emitted_at": 1699016394454} +{"stream": "addresses", "data": {"id": 69282975, "customer_id": 64962974, "payment_method_id": 12482030, "address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "created_at": "2021-05-13T09:46:46+00:00", "discounts": [], "first_name": "Kelly", "last_name": "Kozakevich", "order_attributes": [], "order_note": null, "phone": "+16145550188", "presentment_currency": "USD", "province": "Illinois", "shipping_lines_conserved": [], "shipping_lines_override": [], "updated_at": "2023-05-13T04:07:34+00:00", "zip": "60510"}, "emitted_at": 1699016395217} +{"stream": "charges", "data": {"id": 386976088, "address_id": 69105381, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Karina", "last_name": "Kuznetsova", "phone": null, "province": "California", "zip": "94118"}, "charge_attempts": 6, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2021-05-12T12:04:07+00:00", "currency": "USD", "customer": {"id": 64817252, "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "hash": "23dee52d73734a81"}, "discounts": [], "error": "None\r\n [May 12, 12:06AM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 13, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 19, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 25, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [May 31, 4:09PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']\r\n [Jun 06, 4:10PM] ['Inventory unavailable S / Black T1 6642695864491 requested qty. 1, inventory was: -1']", "error_type": "CLOSED_MAX_RETRIES_REACHED", "external_order_id": {"ecommerce": null}, "external_transaction_id": {"payment_processor": null}, "external_variant_not_found": null, "has_uncommitted_changes": false, "last_charge_attempt": "2022-06-06T20:10:19+00:00", "line_items": [{"purchase_item_id": 153224593, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684722131115"}, "grams": 0, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T1", "tax_due": "0.00", "tax_lines": [], "taxable": true, "taxable_amount": "24.30", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "S / Black"}], "note": null, "order_attributes": [], "orders_count": 0, "payment_processor": "shopify_payments", "processed_at": null, "retry_date": "2022-06-12T04:00:00+00:00", "scheduled_at": "2022-05-12", "shipping_address": {"address1": "1 9th Ave", "address2": "1", "city": "San Francisco", "company": null, "country_code": "US", "first_name": "Jane", "last_name": "Doe", "phone": "1234567890", "province": "California", "zip": "94118"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "error", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": true, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-01-16T18:08:54+00:00"}, "emitted_at": 1699016397112} +{"stream": "charges", "data": {"id": 817715206, "address_id": 69282975, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2023-05-13T04:07:34+00:00", "currency": "USD", "customer": {"id": 64962974, "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "hash": "f99bd4a6877257af"}, "discounts": [], "error": null, "error_type": null, "external_order_id": {"ecommerce": null}, "external_transaction_id": {"payment_processor": null}, "has_uncommitted_changes": false, "line_items": [{"purchase_item_id": 153601366, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "grams": 0, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T3", "tax_due": "0.00", "tax_lines": [], "taxable": true, "taxable_amount": "24.30", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "L / City Green"}], "note": null, "order_attributes": [], "orders_count": 0, "payment_processor": "shopify_payments", "processed_at": null, "retry_date": null, "scheduled_at": "2024-05-12", "shipping_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "queued", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": true, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-05-13T04:07:47+00:00"}, "emitted_at": 1699016397881} +{"stream": "charges", "data": {"id": 580825303, "address_id": 69282975, "analytics_data": {"utm_params": []}, "billing_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "client_details": {"browser_ip": null, "user_agent": null}, "created_at": "2022-05-13T04:07:39+00:00", "currency": "USD", "customer": {"id": 64962974, "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "hash": "f99bd4a6877257af"}, "discounts": [], "error": null, "error_type": null, "external_order_id": {"ecommerce": "5006149877931"}, "external_transaction_id": {"payment_processor": "43114102955"}, "has_uncommitted_changes": false, "line_items": [{"purchase_item_id": 153601366, "external_product_id": {"ecommerce": "6642695864491"}, "external_variant_id": {"ecommerce": "39684723835051"}, "grams": null, "handle": null, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "original_price": "24.30", "properties": [], "purchase_item_type": "subscription", "quantity": 1, "sku": "T3", "tax_due": "0.00", "tax_lines": [], "taxable": false, "taxable_amount": "0.00", "title": "Airbit Box Corner Short sleeve t-shirt", "total_price": "24.30", "unit_price": "24.30", "unit_price_includes_tax": false, "variant_title": "L / City Green"}], "note": null, "order_attributes": [], "orders_count": 1, "payment_processor": "shopify_payments", "processed_at": "2023-05-13T04:07:33+00:00", "retry_date": null, "scheduled_at": "2023-05-13", "shipping_address": {"address1": "1921 W Wilson St", "address2": null, "city": "Batavia", "company": null, "country_code": "US", "first_name": "Kelly", "last_name": "Kozakevich", "phone": "+16145550188", "province": "Illinois", "zip": "60510"}, "shipping_lines": [{"code": "Economy", "price": "4.90", "retrieved_at": null, "source": "shopify", "status": "active", "tax_lines": [], "taxable": false, "title": "Economy"}], "status": "success", "subtotal_price": "24.30", "tags": "Subscription, Subscription Recurring Order", "tax_lines": "[]", "taxable": false, "taxes_included": false, "total_discounts": "0.00", "total_duties": "0.00", "total_line_items_price": "24.30", "total_price": "29.20", "total_refunds": "0.00", "total_tax": "0.00", "total_weight_grams": 0, "type": "recurring", "updated_at": "2023-05-13T04:16:51+00:00"}, "emitted_at": 1699016397882} +{"stream": "customers", "data": {"id": 64817252, "analytics_data": {"utm_params": []}, "apply_credit_to_next_recurring_charge": false, "created_at": "2021-05-12T12:04:06+00:00", "email": "nikolaevaka@yahoo.com", "external_customer_id": {"ecommerce": "5212085977259"}, "first_charge_processed_at": "2021-05-12T16:03:59+00:00", "first_name": "Karina", "has_payment_method_in_dunning": false, "has_valid_payment_method": true, "hash": "23dee52d73734a81", "last_name": "Kuznetsova", "phone": null, "subscriptions_active_count": 0, "subscriptions_total_count": 1, "tax_exempt": false, "updated_at": "2023-01-16T18:08:45+00:00"}, "emitted_at": 1699016402746} +{"stream": "customers", "data": {"id": 64962974, "analytics_data": {"utm_params": []}, "apply_credit_to_next_recurring_charge": false, "created_at": "2021-05-13T09:46:44+00:00", "email": "kozakevich_k@example.com", "external_customer_id": {"ecommerce": "5213433266347"}, "first_charge_processed_at": "2021-05-13T13:46:39+00:00", "first_name": "Kelly", "has_payment_method_in_dunning": false, "has_valid_payment_method": true, "hash": "f99bd4a6877257af", "last_name": "Kozakevich", "phone": "+16145550188", "subscriptions_active_count": 1, "subscriptions_total_count": 1, "tax_exempt": false, "updated_at": "2023-05-13T04:16:36+00:00"}, "emitted_at": 1699016403662} +{"stream": "metafields", "data": {"id": 3627108, "owner_id": "64962974", "created_at": "2023-04-10T07:10:45", "description": "customer_phone_number", "key": "phone_number", "namespace": "personal_info", "owner_resource": "customer", "updated_at": "2023-04-10T07:10:45", "value": "3103103101", "value_type": "integer"}, "emitted_at": 1699016408078} +{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T08:08:28", "discount_amount": 5.0, "discount_type": "percentage", "handle": "airbit-box-corner-short-sleeve-t-shirt", "id": 1853649, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/t_neon_green_47f548d4-fda5-4e21-8066-1d4caadbe581_small.jpg"}, "product_id": 6642695864491, "shopify_product_id": 6642695864491, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "Airbit Box Corner Short sleeve t-shirt", "updated_at": "2021-05-13T08:08:28"}, "emitted_at": 1699016425107} +{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T07:27:34", "discount_amount": 5.0, "discount_type": "percentage", "handle": "i-make-beats-wool-blend-snapback", "id": 1853639, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/c_black1_small.jpg"}, "product_id": 6644278001835, "shopify_product_id": 6644278001835, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "I Make Beats Wool Blend Snapback", "updated_at": "2021-05-13T07:27:34"}, "emitted_at": 1699016425108} +{"stream": "products", "data": {"collection_id": null, "created_at": "2021-05-13T08:20:10", "discount_amount": 0.0, "discount_type": "percentage", "handle": "new-mug", "id": 1853655, "images": {"large": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_large.jpg", "medium": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_medium.jpg", "original": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red.jpg", "small": "https://cdn.shopify.com/s/files/1/0565/0628/6251/products/m_black_red_small.jpg"}, "product_id": 6688261701803, "shopify_product_id": 6688261701803, "subscription_defaults": {"apply_cutoff_date_to_checkout": false, "charge_interval_frequency": 30, "cutoff_day_of_month": null, "cutoff_day_of_week": null, "expire_after_specific_number_of_charges": null, "modifiable_properties": [], "number_charges_until_expiration": null, "order_day_of_month": null, "order_day_of_week": null, "order_interval_frequency_options": ["30"], "order_interval_unit": "day", "storefront_purchase_options": "subscription_and_onetime"}, "title": "NEW!!! MUG", "updated_at": "2021-05-13T08:20:10"}, "emitted_at": 1699016425109} +{"stream": "shop", "data": {"shop": {"allow_customers_to_skip_delivery": 1, "checkout_logo_url": null, "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Thu, 13 Jul 2023 15:26:57 GMT"}, "store": {"checkout_logo_url": null, "checkout_platform": "shopify", "created_at": "Wed, 21 Apr 2021 11:44:38 GMT", "currency": "USD", "customer_portal_domain": "", "disabled_currencies_historical": [], "domain": "airbyte.myshopify.com", "email": "integration-test@airbyte.io", "enabled_presentment_currencies": ["USD"], "enabled_presentment_currencies_symbols": [{"currency": "USD", "location": "before", "suffix": " USD", "symbol": "$"}], "external_platform": "shopify", "iana_timezone": "Europe/Zaporozhye", "id": 126593, "my_shopify_domain": "airbyte.myshopify.com", "name": "airbyte", "payment_processor": "shopify_payments", "platform_domain": "airbyte.myshopify.com", "shop_email": "integration-test@airbyte.io", "shop_phone": "1111111111", "subscriptions_enabled": 1, "test_mode": false, "timezone": "(GMT+02:00) Europe/Zaporozhye", "updated_at": "Thu, 13 Jul 2023 15:26:57 GMT"}}, "emitted_at": 1699016427703} diff --git a/airbyte-integrations/connectors/source-recharge/metadata.yaml b/airbyte-integrations/connectors/source-recharge/metadata.yaml index b4a498ec5a26..af32cc639991 100644 --- a/airbyte-integrations/connectors/source-recharge/metadata.yaml +++ b/airbyte-integrations/connectors/source-recharge/metadata.yaml @@ -4,8 +4,10 @@ data: - api.rechargeapps.com connectorSubtype: api connectorType: source + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c definitionId: 45d2e135-2ede-49e1-939f-3e3ec357a65e - dockerImageTag: 1.0.0 + dockerImageTag: 1.1.2 dockerRepository: airbyte/source-recharge githubIssueLabel: source-recharge icon: recharge.svg diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py index c2c9d3e85f8c..126aa6ebfad8 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/api.py +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/api.py @@ -2,7 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from abc import ABC +from abc import ABC, abstractmethod from typing import Any, Iterable, List, Mapping, MutableMapping, Optional import pendulum @@ -11,9 +11,6 @@ from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -API_VERSION = "2021-11" -OLD_API_VERSION = "2021-01" - class RechargeStream(HttpStream, ABC): primary_key = "id" @@ -21,7 +18,7 @@ class RechargeStream(HttpStream, ABC): limit = 250 page_num = 1 - period_in_months = 1 # Slice data request for 1 month + period_in_days = 30 # Slice data request for 1 month raise_on_http_errors = True # registering the default schema transformation @@ -35,34 +32,30 @@ def __init__(self, config, **kwargs): def data_path(self): return self.name + @property + @abstractmethod + def api_version(self) -> str: + pass + def request_headers( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> Mapping[str, Any]: - return {"x-recharge-version": API_VERSION} + return {"x-recharge-version": self.api_version} def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: return self.name + @abstractmethod def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - cursor = response.json().get("next_cursor") - if cursor: - return {"cursor": cursor} + pass + @abstractmethod def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs ) -> MutableMapping[str, Any]: - params = { - "limit": self.limit, - } - - if next_page_token: - params.update(next_page_token) - else: - params.update({"updated_at_min": (stream_state or {}).get("updated_at", self._start_date)}) - - return params + pass def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: response_data = response.json() @@ -92,20 +85,68 @@ def stream_slices( now = pendulum.now() - start_date = pendulum.parse(start_date) + # dates are inclusive, so we add 1 second so that time periods do not overlap + start_date = pendulum.parse(start_date).add(seconds=1) while start_date <= now: - end_date = start_date.add(months=self.period_in_months) + end_date = start_date.add(days=self.period_in_days) yield {"start_date": start_date.strftime("%Y-%m-%d %H:%M:%S"), "end_date": end_date.strftime("%Y-%m-%d %H:%M:%S")} - start_date = end_date + start_date = end_date.add(seconds=1) + + +class RechargeStreamModernAPI(RechargeStream): + api_version = "2021-11" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + cursor = response.json().get("next_cursor") + if cursor: + return {"cursor": cursor} + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs + ) -> MutableMapping[str, Any]: + params = {"limit": self.limit} + + # if a cursor value is passed, only limit can be passed with it! + if next_page_token: + params.update(next_page_token) + else: + params.update( + { + "updated_at_min": (stream_slice or {}).get("start_date", self._start_date), + "updated_at_max": (stream_slice or {}).get("end_date", self._start_date), + } + ) + return params + + +class RechargeStreamDeprecatedAPI(RechargeStream): + api_version = "2021-01" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + stream_data = self.get_stream_data(response.json()) + if len(stream_data) == self.limit: + self.page_num += 1 + return {"page": self.page_num} + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs + ) -> MutableMapping[str, Any]: + params = { + "limit": self.limit, + "updated_at_min": (stream_slice or {}).get("start_date", self._start_date), + "updated_at_max": (stream_slice or {}).get("end_date", self._start_date), + } + + if next_page_token: + params.update(next_page_token) + + return params class IncrementalRechargeStream(RechargeStream, ABC): cursor_field = "updated_at" - - @property - def state_checkpoint_interval(self): - return self.limit + state_checkpoint_interval = 250 def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: latest_benchmark = latest_record[self.cursor_field] @@ -114,37 +155,37 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late return {self.cursor_field: latest_benchmark} -class Addresses(IncrementalRechargeStream): +class Addresses(RechargeStreamModernAPI, IncrementalRechargeStream): """ Addresses Stream: https://developer.rechargepayments.com/v1-shopify?python#list-addresses """ -class Charges(IncrementalRechargeStream): +class Charges(RechargeStreamModernAPI, IncrementalRechargeStream): """ Charges Stream: https://developer.rechargepayments.com/v1-shopify?python#list-charges """ -class Collections(RechargeStream): +class Collections(RechargeStreamModernAPI): """ Collections Stream """ -class Customers(IncrementalRechargeStream): +class Customers(RechargeStreamModernAPI, IncrementalRechargeStream): """ Customers Stream: https://developer.rechargepayments.com/v1-shopify?python#list-customers """ -class Discounts(IncrementalRechargeStream): +class Discounts(RechargeStreamModernAPI, IncrementalRechargeStream): """ Discounts Stream: https://developer.rechargepayments.com/v1-shopify?python#list-discounts """ -class Metafields(RechargeStream): +class Metafields(RechargeStreamModernAPI): """ Metafields Stream: https://developer.rechargepayments.com/v1-shopify?python#list-metafields """ @@ -165,31 +206,27 @@ def stream_slices( yield from [{"owner_resource": owner} for owner in owner_resources] -class Onetimes(IncrementalRechargeStream): +class Onetimes(RechargeStreamModernAPI, IncrementalRechargeStream): """ Onetimes Stream: https://developer.rechargepayments.com/v1-shopify?python#list-onetimes """ -class Orders(IncrementalRechargeStream): +class Orders(RechargeStreamDeprecatedAPI, IncrementalRechargeStream): """ Orders Stream: https://developer.rechargepayments.com/v1-shopify?python#list-orders + Using old API version to avoid schema changes and loosing email, first_name, last_name columns, because in new version it not present """ -class Products(RechargeStream): +class Products(RechargeStreamDeprecatedAPI): """ Products Stream: https://developer.rechargepayments.com/v1-shopify?python#list-products Products endpoint has 422 error with 2021-11 API version """ - def request_headers( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> Mapping[str, Any]: - return {"x-recharge-version": OLD_API_VERSION} - -class Shop(RechargeStream): +class Shop(RechargeStreamDeprecatedAPI): """ Shop Stream: https://developer.rechargepayments.com/v1-shopify?python#shop Shop endpoint is not available in 2021-11 API version @@ -198,13 +235,23 @@ class Shop(RechargeStream): primary_key = ["shop", "store"] data_path = None - def request_headers( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> Mapping[str, Any]: - return {"x-recharge-version": OLD_API_VERSION} + def stream_slices( + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + return [{}] + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs + ) -> MutableMapping[str, Any]: + return {} -class Subscriptions(IncrementalRechargeStream): +class Subscriptions(RechargeStreamModernAPI, IncrementalRechargeStream): """ Subscriptions Stream: https://developer.rechargepayments.com/v1-shopify?python#list-subscriptions """ + + # reduce the slice date range to avoid 504 - Gateway Timeout on the Server side, + # since this stream could contain lots of data, causing the server to timeout. + # related issue: https://github.com/airbytehq/oncall/issues/3424 + period_in_days = 14 diff --git a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/customers.json b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/customers.json index f8fade651879..9771a8c7cdb6 100644 --- a/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/customers.json +++ b/airbyte-integrations/connectors/source-recharge/source_recharge/schemas/customers.json @@ -91,6 +91,9 @@ "updated_at": { "type": ["null", "string"], "format": "date-time" + }, + "apply_credit_to_next_recurring_charge": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py b/airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py index 6e1cc27e492a..19ac0ebf284a 100644 --- a/airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py +++ b/airbyte-integrations/connectors/source-recharge/unit_tests/test_api.py @@ -17,7 +17,8 @@ Onetimes, Orders, Products, - RechargeStream, + RechargeStreamDeprecatedAPI, + RechargeStreamModernAPI, Shop, Subscriptions, ) @@ -34,8 +35,6 @@ def config(): class TestCommon: - main = RechargeStream - @pytest.mark.parametrize( "stream_cls, expected", [ @@ -55,69 +54,6 @@ class TestCommon: def test_primary_key(self, stream_cls, expected): assert expected == stream_cls.primary_key - @pytest.mark.parametrize( - "stream_cls", - [ - (Addresses), - (Charges), - (Collections), - (Customers), - (Discounts), - (Metafields), - (Onetimes), - (Orders), - (Products), - (Shop), - (Subscriptions), - ], - ) - def test_url_base(self, config, stream_cls): - expected = self.main(config, authenticator=None).url_base - result = stream_cls.url_base - assert expected == result - - @pytest.mark.parametrize( - "stream_cls", - [ - (Addresses), - (Charges), - (Collections), - (Customers), - (Discounts), - (Metafields), - (Onetimes), - (Orders), - (Products), - (Shop), - (Subscriptions), - ], - ) - def test_limit(self, config, stream_cls): - expected = self.main(config, authenticator=None).limit - result = stream_cls.limit - assert expected == result - - @pytest.mark.parametrize( - "stream_cls", - [ - (Addresses), - (Charges), - (Collections), - (Customers), - (Discounts), - (Metafields), - (Onetimes), - (Orders), - (Products), - (Shop), - (Subscriptions), - ], - ) - def test_page_num(self, config, stream_cls): - expected = self.main(config, authenticator=None).page_num - result = stream_cls.page_num - assert expected == result - @pytest.mark.parametrize( "stream_cls, stream_type, expected", [ @@ -174,17 +110,20 @@ def test_path(self, config, stream_cls, stream_type, expected): (HTTPStatus.FORBIDDEN, {}, False), ], ) - def test_should_retry(self, config, http_status, headers, should_retry): + @pytest.mark.parametrize("stream_cls", (RechargeStreamDeprecatedAPI, RechargeStreamModernAPI)) + def test_should_retry(self, config, http_status, headers, should_retry, stream_cls): response = requests.Response() response.status_code = http_status response._content = b"" response.headers = headers - stream = RechargeStream(config, authenticator=None) + stream = stream_cls(config, authenticator=None) assert stream.should_retry(response) == should_retry class TestFullRefreshStreams: def generate_records(self, stream_name, count): + if not stream_name: + return {f"record_{1}": f"test_{1}"} result = [] for i in range(0, count): result.append({f"record_{i}": f"test_{i}"}) @@ -193,17 +132,18 @@ def generate_records(self, stream_name, count): @pytest.mark.parametrize( "stream_cls, cursor_response, expected", [ - (Collections, "some next cursor", {"cursor": "some next cursor"}), - (Metafields, "some next cursor", {"cursor": "some next cursor"}), - (Products, "some next cursor", {"cursor": "some next cursor"}), - (Shop, "some next cursor", {"cursor": "some next cursor"}), + (Collections, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), + (Metafields, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), + (Products, {}, {"page": 2}), + (Shop, {}, None), + (Orders, {}, {"page": 2}), ], ) def test_next_page_token(self, config, stream_cls, cursor_response, requests_mock, expected): stream = stream_cls(config, authenticator=None) stream.limit = 2 url = f"{stream.url_base}{stream.path()}" - response = {"next_cursor": cursor_response, stream.name: self.generate_records(stream.name, 2)} + response = {**cursor_response, **self.generate_records(stream.data_path, 2)} requests_mock.get(url, json=response) response = requests.get(url) assert stream.next_page_token(response) == expected @@ -211,10 +151,22 @@ def test_next_page_token(self, config, stream_cls, cursor_response, requests_moc @pytest.mark.parametrize( "stream_cls, next_page_token, stream_state, stream_slice, expected", [ - (Collections, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), + ( + Collections, + None, + {}, + {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, + {"limit": 250, "updated_at_min": "2020-01-01T00:00:00Z", "updated_at_max": "2020-02-01T00:00:00Z"}, + ), (Metafields, {"cursor": "12353"}, {"updated_at": "2030-01-01"}, {}, {"limit": 250, "owner_resource": None, "cursor": "12353"}), - (Products, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), - (Shop, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z",}), + ( + Products, + None, + {}, + {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, + {"limit": 250, "updated_at_min": "2020-01-01T00:00:00Z", "updated_at_max": "2020-02-01T00:00:00Z"}, + ), + (Shop, None, {}, {}, {}), ], ) def test_request_params(self, config, stream_cls, next_page_token, stream_state, stream_slice, expected): @@ -290,20 +242,20 @@ def test_cursor_field(self, config, stream_cls, expected): @pytest.mark.parametrize( "stream_cls, cursor_response, expected", [ - (Addresses, "some next cursor", {"cursor": "some next cursor"}), - (Charges, "some next cursor", {"cursor": "some next cursor"}), - (Customers, "some next cursor", {"cursor": "some next cursor"}), - (Discounts, "some next cursor", {"cursor": "some next cursor"}), - (Onetimes, "some next cursor", {"cursor": "some next cursor"}), - (Orders, "some next cursor", {"cursor": "some next cursor"}), - (Subscriptions, "some next cursor", {"cursor": "some next cursor"}), + (Addresses, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), + (Charges, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), + (Customers, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), + (Discounts, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), + (Onetimes, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), + (Orders, {}, {"page": 2}), + (Subscriptions, {"next_cursor": "some next cursor"}, {"cursor": "some next cursor"}), ], ) def test_next_page_token(self, config, stream_cls, cursor_response, requests_mock, expected): stream = stream_cls(config, authenticator=None) stream.limit = 2 url = f"{stream.url_base}{stream.path()}" - response = {"next_cursor": cursor_response, stream.name: self.generate_records(stream.name, 2)} + response = {**cursor_response, **self.generate_records(stream.data_path, 2)} requests_mock.get(url, json=response) response = requests.get(url) assert stream.next_page_token(response) == expected @@ -311,16 +263,55 @@ def test_next_page_token(self, config, stream_cls, cursor_response, requests_moc @pytest.mark.parametrize( "stream_cls, next_page_token, stream_state, stream_slice, expected", [ - (Addresses, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), - (Charges, {"cursor": "123"}, {"updated_at": "2030-01-01"}, {}, - {"limit": 250, "cursor": "123"}), - (Customers, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), - (Discounts, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), - (Onetimes, {"cursor": "123"}, {"updated_at": "2030-01-01"}, {}, - {"limit": 250, "cursor": "123"}), - (Orders, None, {}, {}, {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), - (Subscriptions, None, {}, {}, - {"limit": 250, "updated_at_min": "2021-08-15T00:00:00Z"}), + ( + Addresses, + None, + {}, + {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, + {"limit": 250, "updated_at_min": "2020-01-01T00:00:00Z", "updated_at_max": "2020-02-01T00:00:00Z"}, + ), + ( + Charges, + {"cursor": "123"}, + {"updated_at": "2030-01-01"}, + {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, + {"limit": 250, "cursor": "123"}, + ), + ( + Customers, + None, + {}, + {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, + {"limit": 250, "updated_at_min": "2020-01-01T00:00:00Z", "updated_at_max": "2020-02-01T00:00:00Z"}, + ), + ( + Discounts, + None, + {}, + {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, + {"limit": 250, "updated_at_min": "2020-01-01T00:00:00Z", "updated_at_max": "2020-02-01T00:00:00Z"}, + ), + ( + Onetimes, + {"cursor": "123"}, + {"updated_at": "2030-01-01"}, + {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, + {"limit": 250, "cursor": "123"}, + ), + ( + Orders, + None, + {}, + {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, + {"limit": 250, "updated_at_min": "2020-01-01T00:00:00Z", "updated_at_max": "2020-02-01T00:00:00Z"}, + ), + ( + Subscriptions, + None, + {}, + {"start_date": "2020-01-01T00:00:00Z", "end_date": "2020-02-01T00:00:00Z"}, + {"limit": 250, "updated_at_min": "2020-01-01T00:00:00Z", "updated_at_max": "2020-02-01T00:00:00Z"}, + ), ], ) def test_request_params(self, config, stream_cls, next_page_token, stream_state, stream_slice, expected): diff --git a/airbyte-integrations/connectors/source-recreation/README.md b/airbyte-integrations/connectors/source-recreation/README.md index c84692eb8a99..5c52deb122ef 100644 --- a/airbyte-integrations/connectors/source-recreation/README.md +++ b/airbyte-integrations/connectors/source-recreation/README.md @@ -6,14 +6,6 @@ For information about how to use this connector within Airbyte, see [the documen The Recreation Information Database (RIDB) provides data resources to citizens, offering a single point of access to information about recreational opportunities nationwide. The RIDB represents an authoritative source of information and services for millions of visitors to federal lands, historic sites, museums, and other attractions/resources. This initiative integrates multiple Federal channels and sources about recreation opportunities into a one-stop, searchable database of recreational areas nationwide. ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-recreation:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/recreation) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_recreation/spec.yaml` file. @@ -25,18 +17,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-recreation:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-recreation build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-recreation:airbyteDocker +An image will be built with the tag `airbyte/source-recreation:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-recreation:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -46,25 +39,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-recreation:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-recreation:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-recreation:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-recreation test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-recreation:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-recreation:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -73,8 +58,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-recreation test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/recreation.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-recreation/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-recreation/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-recreation/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-recreation/build.gradle b/airbyte-integrations/connectors/source-recreation/build.gradle deleted file mode 100644 index 906ed0d42817..000000000000 --- a/airbyte-integrations/connectors/source-recreation/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_recreation' -} diff --git a/airbyte-integrations/connectors/source-recruitee/README.md b/airbyte-integrations/connectors/source-recruitee/README.md index 89d8fb294f94..9bdca2492985 100644 --- a/airbyte-integrations/connectors/source-recruitee/README.md +++ b/airbyte-integrations/connectors/source-recruitee/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-recruitee:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/recruitee) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_recruitee/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-recruitee:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-recruitee build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-recruitee:airbyteDocker +An image will be built with the tag `airbyte/source-recruitee:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-recruitee:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-recruitee:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-recruitee:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-recruitee:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-recruitee test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-recruitee:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-recruitee:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-recruitee test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/recruitee.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-recruitee/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-recruitee/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-recruitee/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-recruitee/build.gradle b/airbyte-integrations/connectors/source-recruitee/build.gradle deleted file mode 100644 index ab150c391b7b..000000000000 --- a/airbyte-integrations/connectors/source-recruitee/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_recruitee' -} diff --git a/airbyte-integrations/connectors/source-recurly/README.md b/airbyte-integrations/connectors/source-recurly/README.md index 936611ed32f9..0b6cedea3f2e 100644 --- a/airbyte-integrations/connectors/source-recurly/README.md +++ b/airbyte-integrations/connectors/source-recurly/README.md @@ -1,15 +1,13 @@ -# Recurly Source +# Amazon Seller-Partner Source -This is the repository for the Recurly source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/recurly). +This is the repository for the Amazon Seller-Partner source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/amazon-seller-partner). ## Local development ### Prerequisites **To iterate on this connector, make sure to complete this prerequisites section.** -#### Minimum Python version required `= 3.7.0` - #### Build & Activate Virtual Environment and install dependencies From this connector directory, create a virtual environment: ``` @@ -29,72 +27,71 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-recurly:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/recurly) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_recurly/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/amazon-seller-partner) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_amazon_seller-partner/integration_tests/spec.json` file. Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. +See `integration_tests/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source recurly test creds` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source amazon-seller-partner test creds` and place them into `secrets/config.json`. - ### Locally running the connector ``` python main.py spec python main.py check --config secrets/config.json python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json ``` ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-recurly:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name source-amazon-seller-partner build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-recurly:airbyteDocker +An image will be built with the tag `airbyte/source-amazon-seller-partner:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-amazon-seller-partner:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-recurly:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-recurly:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-recurly:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-recurly:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm airbyte/source-amazon-seller-partner:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-amazon-seller-partner:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-amazon-seller-partner:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-amazon-seller-partner:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-recurly:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-recurly test +``` + +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-recurly test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/recurly.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-recurly/acceptance-test-config.yml b/airbyte-integrations/connectors/source-recurly/acceptance-test-config.yml index 7359e4c0ad66..863cbc656d27 100644 --- a/airbyte-integrations/connectors/source-recurly/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-recurly/acceptance-test-config.yml @@ -14,8 +14,18 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: ["account_coupon_redemptions", "account_notes", "add_ons", "billing_infos", "credit_payments", - "line_items", "shipping_methods", "unique_coupons", "export_dates"] + empty_streams: + [ + "account_coupon_redemptions", + "account_notes", + "add_ons", + "billing_infos", + "credit_payments", + "line_items", + "shipping_methods", + "unique_coupons", + "export_dates", + ] full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-recurly/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-recurly/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-recurly/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-recurly/build.gradle b/airbyte-integrations/connectors/source-recurly/build.gradle deleted file mode 100644 index f7d5ac4b7194..000000000000 --- a/airbyte-integrations/connectors/source-recurly/build.gradle +++ /dev/null @@ -1,10 +0,0 @@ -plugins { - id 'airbyte-docker' - id 'airbyte-python' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_recurly' -} - diff --git a/airbyte-integrations/connectors/source-redshift/.dockerignore b/airbyte-integrations/connectors/source-redshift/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-redshift/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-redshift/Dockerfile b/airbyte-integrations/connectors/source-redshift/Dockerfile deleted file mode 100644 index 751c8c82da28..000000000000 --- a/airbyte-integrations/connectors/source-redshift/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-redshift - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-redshift - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.4.0 -LABEL io.airbyte.name=airbyte/source-redshift diff --git a/airbyte-integrations/connectors/source-redshift/acceptance-test-config.yml b/airbyte-integrations/connectors/source-redshift/acceptance-test-config.yml deleted file mode 100644 index b69725d9b4b6..000000000000 --- a/airbyte-integrations/connectors/source-redshift/acceptance-test-config.yml +++ /dev/null @@ -1,35 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-redshift:dev -acceptance_tests: - spec: - tests: - - spec_path: "src/test-integration/resources/expected_spec.json" - timeout_seconds: "1200" - config_path: "secrets/config.json" - connection: - tests: - - config_path: "secrets/config.json" - timeout_seconds: "1200" - status: "succeed" - discovery: - tests: - - config_path: "secrets/config.json" - timeout_seconds: "1200" - basic_read: - tests: - - config_path: "secrets/config.json" - timeout_seconds: "1200" - configured_catalog_path: "integration_tests/configured_catalog.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - full_refresh: - tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: "1200" - incremental: - tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_inc.json" - timeout_seconds: "1200" diff --git a/airbyte-integrations/connectors/source-redshift/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-redshift/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-redshift/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-redshift/build.gradle b/airbyte-integrations/connectors/source-redshift/build.gradle index d6722aa55388..9d9fdf35dbec 100644 --- a/airbyte-integrations/connectors/source-redshift/build.gradle +++ b/airbyte-integrations/connectors/source-redshift/build.gradle @@ -1,38 +1,41 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' - id 'airbyte-connector-acceptance-test' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.7.7' + features = ['db-sources'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileTestJava { + options.compilerArgs.remove("-Werror") + } + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.source.redshift.RedshiftSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } -repositories { - maven { url "https://s3.amazonaws.com/redshift-maven-repository/release" } -} - dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation project(':airbyte-integrations:connectors:source-relational-db') implementation 'com.amazon.redshift:redshift-jdbc42:1.2.43.1067' testImplementation 'org.apache.commons:commons-text:1.10.0' testImplementation 'org.apache.commons:commons-lang3:3.11' testImplementation 'org.apache.commons:commons-dbcp2:2.7.0' - testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) - testImplementation project(":airbyte-json-validation") + testImplementation 'org.hamcrest:hamcrest-all:1.3' + integrationTestJavaImplementation libs.testcontainers.jdbc + testImplementation project.project(':airbyte-cdk:java:airbyte-cdk:airbyte-json-validation') - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - integrationTestJavaImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-redshift') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) } - diff --git a/airbyte-integrations/connectors/source-redshift/metadata.yaml b/airbyte-integrations/connectors/source-redshift/metadata.yaml index 373c5157645a..3ef9052048bc 100644 --- a/airbyte-integrations/connectors/source-redshift/metadata.yaml +++ b/airbyte-integrations/connectors/source-redshift/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: source definitionId: e87ffa8e-a3b5-f69c-9076-6011339de1f6 - dockerImageTag: 0.4.0 + dockerImageTag: 0.5.0 dockerRepository: airbyte/source-redshift documentationUrl: https://docs.airbyte.com/integrations/sources/redshift githubIssueLabel: source-redshift diff --git a/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java index 0f3759a21647..d80a2558ef1b 100644 --- a/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java +++ b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSource.java @@ -6,16 +6,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.cdk.integrations.source.jdbc.dto.JdbcPrivilegeDto; +import io.airbyte.cdk.integrations.source.relationaldb.TableInfo; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.dto.JdbcPrivilegeDto; -import io.airbyte.integrations.source.relationaldb.TableInfo; import io.airbyte.protocol.models.CommonField; import java.sql.JDBCType; import java.sql.PreparedStatement; @@ -47,10 +47,7 @@ public JsonNode toDatabaseConfig(final JsonNode redshiftConfig) { final ImmutableMap.Builder builder = ImmutableMap.builder() .put(JdbcUtils.USERNAME_KEY, redshiftConfig.get(JdbcUtils.USERNAME_KEY).asText()) .put(JdbcUtils.PASSWORD_KEY, redshiftConfig.get(JdbcUtils.PASSWORD_KEY).asText()) - .put(JdbcUtils.JDBC_URL_KEY, String.format(DatabaseDriver.REDSHIFT.getUrlFormatString(), - redshiftConfig.get(JdbcUtils.HOST_KEY).asText(), - redshiftConfig.get(JdbcUtils.PORT_KEY).asInt(), - redshiftConfig.get(JdbcUtils.DATABASE_KEY).asText())); + .put(JdbcUtils.JDBC_URL_KEY, getJdbcUrl(redshiftConfig)); if (redshiftConfig.has(JdbcUtils.SCHEMAS_KEY) && redshiftConfig.get(JdbcUtils.SCHEMAS_KEY).isArray()) { schemas = new ArrayList<>(); @@ -75,6 +72,13 @@ public JsonNode toDatabaseConfig(final JsonNode redshiftConfig) { .build()); } + public static String getJdbcUrl(final JsonNode redshiftConfig) { + return String.format(DatabaseDriver.REDSHIFT.getUrlFormatString(), + redshiftConfig.get(JdbcUtils.HOST_KEY).asText(), + redshiftConfig.get(JdbcUtils.PORT_KEY).asInt(), + redshiftConfig.get(JdbcUtils.DATABASE_KEY).asText()); + } + private void addSsl(final List additionalProperties) { additionalProperties.add("ssl=true"); additionalProperties.add("sslfactory=com.amazon.redshift.ssl.NonValidatingFactory"); diff --git a/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSourceOperations.java b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSourceOperations.java index ee16ddccf750..2f3b9f169ee3 100644 --- a/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSourceOperations.java +++ b/airbyte-integrations/connectors/source-redshift/src/main/java/io/airbyte/integrations/source/redshift/RedshiftSourceOperations.java @@ -4,16 +4,17 @@ package io.airbyte.integrations.source.redshift; -import static io.airbyte.db.jdbc.DateTimeConverter.putJavaSQLTime; +import static io.airbyte.cdk.db.jdbc.DateTimeConverter.putJavaSQLTime; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.db.jdbc.DateTimeConverter; -import io.airbyte.db.jdbc.JdbcSourceOperations; +import io.airbyte.cdk.db.jdbc.DateTimeConverter; +import io.airbyte.cdk.db.jdbc.JdbcSourceOperations; import java.sql.Date; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; +import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; import org.slf4j.Logger; @@ -23,6 +24,19 @@ public class RedshiftSourceOperations extends JdbcSourceOperations { private static final Logger LOGGER = LoggerFactory.getLogger(RedshiftSourceOperations.class); + @Override + public void copyToJsonField(final ResultSet resultSet, final int colIndex, final ObjectNode json) throws SQLException { + if ("timestamptz".equalsIgnoreCase(resultSet.getMetaData().getColumnTypeName(colIndex))) { + // Massive hack. Sometimes the JDBCType is TIMESTAMP (i.e. without timezone) + // even though it _should_ be TIMESTAMP_WITH_TIMEZONE. + // Check for this case explicitly. + final String columnName = resultSet.getMetaData().getColumnName(colIndex); + putTimestampWithTimezone(json, columnName, resultSet, colIndex); + } else { + super.copyToJsonField(resultSet, colIndex, json); + } + } + @Override protected void putTime(final ObjectNode node, final String columnName, @@ -44,6 +58,17 @@ protected void setTimestamp(final PreparedStatement preparedStatement, final int preparedStatement.setTimestamp(parameterIndex, Timestamp.valueOf(date)); } + @Override + protected void putTimestampWithTimezone(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) + throws SQLException { + try { + super.putTimestampWithTimezone(node, columnName, resultSet, index); + } catch (final Exception e) { + final Instant instant = resultSet.getTimestamp(index).toInstant(); + node.put(columnName, instant.toString()); + } + } + @Override protected void setDate(final PreparedStatement preparedStatement, final int parameterIndex, final String value) throws SQLException { final LocalDate date = LocalDate.parse(value); diff --git a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftJdbcSourceAcceptanceTest.java index d141c225c2c0..72f64b0069d4 100644 --- a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftJdbcSourceAcceptanceTest.java @@ -5,38 +5,35 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.integrations.source.redshift.RedshiftSource; import java.nio.file.Path; -import java.sql.JDBCType; -import java.sql.SQLException; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; // Run as part of integration tests, instead of unit tests, because there is no test container for // Redshift. -class RedshiftJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { +class RedshiftJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { - private JsonNode config; - - private static JsonNode getStaticConfig() { - return Jsons.deserialize(IOs.readFile(Path.of("secrets/config.json"))); - } + private static JsonNode config; @BeforeAll static void init() { + config = Jsons.deserialize(IOs.readFile(Path.of("secrets/config.json"))); CREATE_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s GEOMETRY)"; INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES(ST_Point(129.77099609375, 62.093299865722656))"; } - @BeforeEach - public void setup() throws Exception { - config = getStaticConfig(); - super.setup(); + @Override + protected RedshiftTestDatabase createTestDatabase() { + final RedshiftTestDatabase testDatabase = new RedshiftTestDatabase(source().toDatabaseConfig(Jsons.clone(config))).initialized(); + try { + for (final String schemaName : TEST_SCHEMAS) { + testDatabase.with(DROP_SCHEMA_QUERY, schemaName); + } + } catch (final Exception ignore) {} + return testDatabase; } @Override @@ -45,23 +42,13 @@ public boolean supportsSchemas() { } @Override - public AbstractJdbcSource getJdbcSource() { + protected RedshiftSource source() { return new RedshiftSource(); } @Override - public JsonNode getConfig() { - return config; - } - - @Override - public String getDriverClass() { - return RedshiftSource.DRIVER_CLASS; - } - - @AfterEach - public void tearDownRedshift() throws SQLException { - super.tearDown(); + protected JsonNode config() { + return Jsons.clone(config); } } diff --git a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java index 2c9be009042f..940d04edbe94 100644 --- a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceAcceptanceTest.java @@ -8,18 +8,18 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.commons.string.Strings; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.source.redshift.RedshiftSource; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; diff --git a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceOperationsTest.java b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceOperationsTest.java new file mode 100644 index 000000000000..1ff7fc23b136 --- /dev/null +++ b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSourceOperationsTest.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.integrations.source.jdbc.JdbcDataSourceUtils; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.redshift.RedshiftSource; +import io.airbyte.integrations.source.redshift.RedshiftSourceOperations; +import java.nio.file.Path; +import java.sql.SQLException; +import java.time.Duration; +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.util.List; +import javax.sql.DataSource; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class RedshiftSourceOperationsTest { + + private static final Duration CONNECTION_TIMEOUT = Duration.ofSeconds(60); + private JdbcDatabase database; + + @BeforeEach + void setup() { + final JsonNode config = Jsons.deserialize(IOs.readFile(Path.of("secrets/config.json"))); + + final DataSource dataSource = DataSourceFactory.create( + config.get("username").asText(), + config.get("password").asText(), + DatabaseDriver.REDSHIFT.getDriverClassName(), + RedshiftSource.getJdbcUrl(config), + JdbcDataSourceUtils.getConnectionProperties(config), + CONNECTION_TIMEOUT); + database = new DefaultJdbcDatabase(dataSource, new RedshiftSourceOperations()); + } + + @Test + void testTimestampWithTimezone() throws SQLException { + // CURRENT_TIMESTAMP is converted to a string by queryJsons. + // CAST(CURRENT_TIMESTAMP AS VARCHAR) does the timestamp -> string conversion on the server side. + // If queryJsons is implemented correctly, both timestamps should be the same. + final List result = database.queryJsons("SELECT CURRENT_TIMESTAMP, CAST(CURRENT_TIMESTAMP AS VARCHAR)"); + + final Instant clientSideParse = Instant.parse(result.get(0).get("timestamptz").asText()); + // Redshift's default timestamp format is "2023-11-17 17:50:36.746606+00", which Instant.parse() + // can't handle. Build a custom datetime formatter. + // (Redshift supports server-side timestamp formatting, but it doesn't provide a way to force + // HH:MM offsets, which are required by Instant.parse) + final Instant serverSideParse = new DateTimeFormatterBuilder() + .append(DateTimeFormatter.ISO_DATE) + .appendLiteral(' ') + .append(DateTimeFormatter.ISO_LOCAL_TIME) + // "X" represents a +/-HH offset + .appendPattern("X") + .toFormatter() + .parse(result.get(0).get("varchar").asText(), Instant::from); + assertEquals(serverSideParse, clientSideParse); + } + +} diff --git a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSslSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSslSourceAcceptanceTest.java index 11f9be18b9e1..787cebc127f3 100644 --- a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSslSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftSslSourceAcceptanceTest.java @@ -5,14 +5,19 @@ package io.airbyte.integrations.io.airbyte.integration_tests.sources; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import java.time.Duration; +import org.junit.jupiter.api.Disabled; +@Disabled public class RedshiftSslSourceAcceptanceTest extends RedshiftSourceAcceptanceTest { + private static final Duration CONNECTION_TIMEOUT = Duration.ofSeconds(60); + @Override protected JdbcDatabase createDatabase(final JsonNode config) { return new DefaultJdbcDatabase( @@ -25,7 +30,8 @@ protected JdbcDatabase createDatabase(final JsonNode config) { config.get(JdbcUtils.PORT_KEY).asInt(), config.get(JdbcUtils.DATABASE_KEY).asText()), JdbcUtils.parseJdbcParameters("ssl=true&" + - "sslfactory=com.amazon.redshift.ssl.NonValidatingFactory"))); + "sslfactory=com.amazon.redshift.ssl.NonValidatingFactory"), + CONNECTION_TIMEOUT)); } } diff --git a/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftTestDatabase.java b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftTestDatabase.java new file mode 100644 index 000000000000..18f7ca2ee73d --- /dev/null +++ b/airbyte-integrations/connectors/source-redshift/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/RedshiftTestDatabase.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import static io.airbyte.cdk.db.factory.DatabaseDriver.REDSHIFT; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.NonContainer; +import io.airbyte.cdk.testutils.TestDatabase; +import java.util.stream.Stream; +import org.jooq.SQLDialect; + +public class RedshiftTestDatabase extends TestDatabase { + + private final String username; + private final String password; + private final String jdbcUrl; + + protected RedshiftTestDatabase(final JsonNode redshiftConfig) { + super(new NonContainer(redshiftConfig.get(JdbcUtils.USERNAME_KEY).asText(), + redshiftConfig.has(JdbcUtils.PASSWORD_KEY) ? redshiftConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, + redshiftConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), REDSHIFT.getDriverClassName(), "")); + this.username = redshiftConfig.get(JdbcUtils.USERNAME_KEY).asText(); + this.password = redshiftConfig.has(JdbcUtils.PASSWORD_KEY) ? redshiftConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null; + this.jdbcUrl = redshiftConfig.get(JdbcUtils.JDBC_URL_KEY).asText(); + } + + @Override + public String getJdbcUrl() { + return jdbcUrl; + } + + @Override + public String getPassword() { + return password; + } + + @Override + public String getUserName() { + return username; + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.empty(); + } + + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return REDSHIFT; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.POSTGRES; + } + + @Override + public void close() {} + + static public class RedshiftConfigBuilder extends TestDatabase.ConfigBuilder { + + protected RedshiftConfigBuilder(RedshiftTestDatabase testdb) { + super(testdb); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-relational-db/.dockerignore b/airbyte-integrations/connectors/source-relational-db/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-relational-db/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-relational-db/Dockerfile b/airbyte-integrations/connectors/source-relational-db/Dockerfile deleted file mode 100644 index 36baea09eafa..000000000000 --- a/airbyte-integrations/connectors/source-relational-db/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-relational-db - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-relational-db - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.3.1 -LABEL io.airbyte.name=airbyte/source-relational-db diff --git a/airbyte-integrations/connectors/source-relational-db/build.gradle b/airbyte-integrations/connectors/source-relational-db/build.gradle deleted file mode 100644 index edfda089d804..000000000000 --- a/airbyte-integrations/connectors/source-relational-db/build.gradle +++ /dev/null @@ -1,41 +0,0 @@ -import org.jsonschema2pojo.SourceType - -plugins { - id "java-library" - id "com.github.eirnym.js2p" version "1.0" -} - -dependencies { - implementation project(':airbyte-commons') - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-json-validation') - implementation project(':airbyte-config-oss:config-models-oss') - - implementation 'org.apache.commons:commons-lang3:3.11' - implementation libs.bundles.datadog - - testImplementation project(':airbyte-test-utils') - - testImplementation libs.postgresql - testImplementation libs.connectors.testcontainers.postgresql - - testImplementation libs.junit.jupiter.system.stubs - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) -} - -jsonSchema2Pojo { - sourceType = SourceType.YAMLSCHEMA - source = files("${sourceSets.main.output.resourcesDir}/db_models") - targetDirectory = new File(project.buildDir, 'generated/src/gen/java/') - removeOldOutput = true - - targetPackage = 'io.airbyte.integrations.source.relationaldb.models' - - useLongIntegers = true - generateBuilders = true - includeConstructors = false - includeSetters = true -} diff --git a/airbyte-integrations/connectors/source-relational-db/readme.md b/airbyte-integrations/connectors/source-relational-db/readme.md deleted file mode 100644 index c84c4df734ab..000000000000 --- a/airbyte-integrations/connectors/source-relational-db/readme.md +++ /dev/null @@ -1,3 +0,0 @@ -# Relational Database Source - -We are not planning to expose this source in the UI yet. It serves as a base upon which we can build all of our other Relational Database-compliant sources. diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/AbstractDbSourceTest.java b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/AbstractDbSourceTest.java deleted file mode 100644 index e3bcf679a831..000000000000 --- a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/AbstractDbSourceTest.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2023 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.relationaldb; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; - -import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.commons.features.EnvVariableFeatureFlags; -import io.airbyte.commons.json.Jsons; -import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.source.relationaldb.state.StateGeneratorUtils; -import io.airbyte.protocol.models.v0.AirbyteStateMessage; -import io.airbyte.protocol.models.v0.AirbyteStateMessage.AirbyteStateType; -import java.io.IOException; -import java.util.List; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import uk.org.webcompere.systemstubs.environment.EnvironmentVariables; -import uk.org.webcompere.systemstubs.jupiter.SystemStub; -import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension; - -/** - * Test suite for the {@link AbstractDbSource} class. - */ -@ExtendWith(SystemStubsExtension.class) -public class AbstractDbSourceTest { - - @SystemStub - private EnvironmentVariables environmentVariables; - - @Test - void testDeserializationOfLegacyState() throws IOException { - final AbstractDbSource dbSource = spy(AbstractDbSource.class); - final JsonNode config = mock(JsonNode.class); - - final String legacyStateJson = MoreResources.readResource("states/legacy.json"); - final JsonNode legacyState = Jsons.deserialize(legacyStateJson); - - final List result = StateGeneratorUtils.deserializeInitialState(legacyState, false, - dbSource.getSupportedStateType(config)); - assertEquals(1, result.size()); - assertEquals(AirbyteStateType.LEGACY, result.get(0).getType()); - } - - @Test - void testDeserializationOfGlobalState() throws IOException { - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - final AbstractDbSource dbSource = spy(AbstractDbSource.class); - final JsonNode config = mock(JsonNode.class); - - final String globalStateJson = MoreResources.readResource("states/global.json"); - final JsonNode globalState = Jsons.deserialize(globalStateJson); - - final List result = - StateGeneratorUtils.deserializeInitialState(globalState, true, dbSource.getSupportedStateType(config)); - assertEquals(1, result.size()); - assertEquals(AirbyteStateType.GLOBAL, result.get(0).getType()); - } - - @Test - void testDeserializationOfStreamState() throws IOException { - environmentVariables.set(EnvVariableFeatureFlags.USE_STREAM_CAPABLE_STATE, "true"); - final AbstractDbSource dbSource = spy(AbstractDbSource.class); - final JsonNode config = mock(JsonNode.class); - - final String streamStateJson = MoreResources.readResource("states/per_stream.json"); - final JsonNode streamState = Jsons.deserialize(streamStateJson); - - final List result = - StateGeneratorUtils.deserializeInitialState(streamState, true, dbSource.getSupportedStateType(config)); - assertEquals(2, result.size()); - assertEquals(AirbyteStateType.STREAM, result.get(0).getType()); - } - - @Test - void testDeserializationOfNullState() throws IOException { - final AbstractDbSource dbSource = spy(AbstractDbSource.class); - final JsonNode config = mock(JsonNode.class); - - final List result = StateGeneratorUtils.deserializeInitialState(null, false, dbSource.getSupportedStateType(config)); - assertEquals(1, result.size()); - assertEquals(dbSource.getSupportedStateType(config), result.get(0).getType()); - } - -} diff --git a/airbyte-integrations/connectors/source-reply-io/README.md b/airbyte-integrations/connectors/source-reply-io/README.md index f8538d3af276..a1cc013d47f9 100644 --- a/airbyte-integrations/connectors/source-reply-io/README.md +++ b/airbyte-integrations/connectors/source-reply-io/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-reply-io:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/reply-io) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_reply_io/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-reply-io:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-reply-io build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-reply-io:airbyteDocker +An image will be built with the tag `airbyte/source-reply-io:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-reply-io:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-reply-io:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-reply-io:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-reply-io:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-reply-io test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-reply-io:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-reply-io:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-reply-io test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/reply-io.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-reply-io/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-reply-io/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-reply-io/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-reply-io/build.gradle b/airbyte-integrations/connectors/source-reply-io/build.gradle deleted file mode 100644 index e1f1f78ef744..000000000000 --- a/airbyte-integrations/connectors/source-reply-io/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_reply_io' -} diff --git a/airbyte-integrations/connectors/source-retently/README.md b/airbyte-integrations/connectors/source-retently/README.md index f9ff6721bd61..462c8f02ec13 100644 --- a/airbyte-integrations/connectors/source-retently/README.md +++ b/airbyte-integrations/connectors/source-retently/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-retently:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/retently) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_retently/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-retently:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-retently build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-retently:airbyteDocker +An image will be built with the tag `airbyte/source-retently:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-retently:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-retently:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-retently:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-retently:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-retently test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-retently:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-retently:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-retently test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/retently.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-retently/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-retently/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-retently/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-retently/build.gradle b/airbyte-integrations/connectors/source-retently/build.gradle deleted file mode 100644 index 3e3ee4a40cf8..000000000000 --- a/airbyte-integrations/connectors/source-retently/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_retently' -} diff --git a/airbyte-integrations/connectors/source-ringcentral/README.md b/airbyte-integrations/connectors/source-ringcentral/README.md index 49742680b28e..e42e5e059dd7 100644 --- a/airbyte-integrations/connectors/source-ringcentral/README.md +++ b/airbyte-integrations/connectors/source-ringcentral/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-ringcentral:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/ringcentral) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_ringcentral/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-ringcentral:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-ringcentral build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-ringcentral:airbyteDocker +An image will be built with the tag `airbyte/source-ringcentral:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-ringcentral:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-ringcentral:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-ringcentral:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-ringcentral:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-ringcentral test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-ringcentral:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-ringcentral:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-ringcentral test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/ringcentral.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-ringcentral/acceptance-test-config.yml b/airbyte-integrations/connectors/source-ringcentral/acceptance-test-config.yml index 229ac91186bd..bc1c7ca356e0 100644 --- a/airbyte-integrations/connectors/source-ringcentral/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-ringcentral/acceptance-test-config.yml @@ -18,7 +18,7 @@ acceptance_tests: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: + empty_streams: - name: "user_business_hours" bypass_reason: "Sandbox Account has no priviledges to seed the stream" - name: "company_business_hours" @@ -37,20 +37,20 @@ acceptance_tests: bypass_reason: "Sandbox Account has no priviledges to seed the stream" - name: "ivr_prompts" bypass_reason: "Sandbox Account has no priviledges to seed the stream" -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.jsonl" -# extra_fields: no -# exact_order: no -# extra_records: yes - incremental: + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.jsonl" + # extra_fields: no + # exact_order: no + # extra_records: yes + incremental: bypass_reason: "This connector does not implement incremental sync" -# TODO uncomment this block this block if your connector implements incremental sync: -# tests: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# future_state: -# future_state_path: "integration_tests/abnormal_state.json" + # TODO uncomment this block this block if your connector implements incremental sync: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-ringcentral/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-ringcentral/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-ringcentral/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-ringcentral/build.gradle b/airbyte-integrations/connectors/source-ringcentral/build.gradle deleted file mode 100644 index 74917dd1d422..000000000000 --- a/airbyte-integrations/connectors/source-ringcentral/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_ringcentral' -} diff --git a/airbyte-integrations/connectors/source-rki-covid/README.md b/airbyte-integrations/connectors/source-rki-covid/README.md index f85765fc16bf..4e23b0ba850d 100644 --- a/airbyte-integrations/connectors/source-rki-covid/README.md +++ b/airbyte-integrations/connectors/source-rki-covid/README.md @@ -50,14 +50,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-rki-covid:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/rki-covid) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_rki_covid/spec.json` file. @@ -77,18 +69,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-rki-covid:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-rki-covid build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-rki-covid:airbyteDocker +An image will be built with the tag `airbyte/source-rki-covid:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-rki-covid:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -98,44 +91,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-rki-covid:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-rki-covid:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-rki-covid:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-rki-covid test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-rki-covid:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-rki-covid:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -145,8 +110,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-rki-covid test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/rki-covid.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-rki-covid/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-rki-covid/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-rki-covid/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-rki-covid/build.gradle b/airbyte-integrations/connectors/source-rki-covid/build.gradle deleted file mode 100644 index fa1ff4ffdb48..000000000000 --- a/airbyte-integrations/connectors/source-rki-covid/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_rki_covid' -} diff --git a/airbyte-integrations/connectors/source-rocket-chat/README.md b/airbyte-integrations/connectors/source-rocket-chat/README.md index 558c975dba1b..ed7f76f3a783 100644 --- a/airbyte-integrations/connectors/source-rocket-chat/README.md +++ b/airbyte-integrations/connectors/source-rocket-chat/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-rocket-chat:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/rocket-chat) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_rocket_chat/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-rocket-chat:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-rocket-chat build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-rocket-chat:airbyteDocker +An image will be built with the tag `airbyte/source-rocket-chat:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-rocket-chat:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-rocket-chat:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-rocket-chat:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-rocket-chat:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-rocket-chat test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-rocket-chat:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-rocket-chat:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-rocket-chat test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/rocket-chat.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-rocket-chat/acceptance-test-config.yml b/airbyte-integrations/connectors/source-rocket-chat/acceptance-test-config.yml index 9f7cda6da974..96d8ee67e153 100644 --- a/airbyte-integrations/connectors/source-rocket-chat/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-rocket-chat/acceptance-test-config.yml @@ -19,7 +19,7 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - incremental: + incremental: bypass_reason: "This connector does not implement incremental sync" full_refresh: tests: diff --git a/airbyte-integrations/connectors/source-rocket-chat/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-rocket-chat/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-rocket-chat/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-rocket-chat/build.gradle b/airbyte-integrations/connectors/source-rocket-chat/build.gradle deleted file mode 100644 index 361e88e9397b..000000000000 --- a/airbyte-integrations/connectors/source-rocket-chat/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_rocket_chat' -} diff --git a/airbyte-integrations/connectors/source-rss/README.md b/airbyte-integrations/connectors/source-rss/README.md index 42c510fe6ff9..85bf860c3633 100644 --- a/airbyte-integrations/connectors/source-rss/README.md +++ b/airbyte-integrations/connectors/source-rss/README.md @@ -1,14 +1,14 @@ -# RSS Source +# Rabbitmq Destination -This is the repository for the RSS source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/rss). +This is the repository for the Rabbitmq destination connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/destinations/rabbitmq). ## Local development ### Prerequisites **To iterate on this connector, make sure to complete this prerequisites section.** -#### Minimum Python version required `= 3.9.0` +#### Minimum Python version required `= 3.7.0` #### Build & Activate Virtual Environment and install dependencies From this connector directory, create a virtual environment: @@ -21,7 +21,6 @@ development environment of choice. To activate it from the terminal, run: ``` source .venv/bin/activate pip install -r requirements.txt -pip install '.[tests]' ``` If you are in an IDE, follow your IDE's instructions to activate the virtualenv. @@ -30,88 +29,57 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/destinations/rabbitmq) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `destination_rabbitmq/spec.json` file. +Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-rss:build -``` - -#### Credentials - -Since this doesn't require auth, the config is just in `integration_tests/sample_config.json` instead of `secrets/config.json`. +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `destination rabbitmq test creds` +and place them into `secrets/config.json`. ### Locally running the connector ``` python main.py spec -python main.py check --config integration_tests/sample_config.json -python main.py discover --config integration_tests/sample_config.json -python main.py read --config integration_tests/sample_config.json --catalog integration_tests/configured_catalog.json +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json ``` ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-rss:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name destination-rabbitmq build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-rss:airbyteDocker +An image will be built with the tag `airbyte/destination-rabbitmq:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/destination-rabbitmq:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-rss:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-rss:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-rss:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-rss:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +docker run --rm airbyte/destination-rabbitmq:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-rabbitmq:dev check --config /secrets/config.json +# messages.jsonl is a file containing line-separated JSON representing AirbyteMessages +cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/destination-rabbitmq:dev write --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-rss test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-rss:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-rss:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-rss:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -121,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-rss test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/rss.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-rss/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-rss/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-rss/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-rss/build.gradle b/airbyte-integrations/connectors/source-rss/build.gradle deleted file mode 100644 index 403d2a6cdb5d..000000000000 --- a/airbyte-integrations/connectors/source-rss/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_rss' -} diff --git a/airbyte-integrations/connectors/source-rss/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-rss/unit_tests/test_streams.py index 371d3a257e3a..2569efcaeb5c 100644 --- a/airbyte-integrations/connectors/source-rss/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-rss/unit_tests/test_streams.py @@ -63,10 +63,10 @@ class SampleResponse: assert next(stream.parse_response(response=SampleResponse(), stream_state={})) == expected_parsed_object # test that the local timezone doesn't impact how this is computed - os.environ['TZ'] = 'Africa/Accra' + os.environ["TZ"] = "Africa/Accra" time.tzset() assert next(stream.parse_response(response=SampleResponse(), stream_state={})) == expected_parsed_object - os.environ['TZ'] = 'Asia/Tokyo' + os.environ["TZ"] = "Asia/Tokyo" time.tzset() assert next(stream.parse_response(response=SampleResponse(), stream_state={})) == expected_parsed_object diff --git a/airbyte-integrations/connectors/source-s3/.coveragerc b/airbyte-integrations/connectors/source-s3/.coveragerc new file mode 100644 index 000000000000..4c1de9ec0853 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/.coveragerc @@ -0,0 +1,7 @@ +[run] +omit = + source_s3/exceptions.py + source_s3/stream.py + source_s3/utils.py + source_s3/source_files_abstract/source.py + source_s3/run.py \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-s3/Dockerfile b/airbyte-integrations/connectors/source-s3/Dockerfile deleted file mode 100644 index fe5dfacb3d59..000000000000 --- a/airbyte-integrations/connectors/source-s3/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM python:3.9-slim as base -FROM base as builder - -RUN apt-get update -WORKDIR /airbyte/integration_code -COPY setup.py ./ -RUN pip install --prefix=/install . - -FROM base -WORKDIR /airbyte/integration_code -COPY --from=builder /install /usr/local - -COPY main.py ./ -COPY source_s3 ./source_s3 - - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=3.1.9 -LABEL io.airbyte.name=airbyte/source-s3 diff --git a/airbyte-integrations/connectors/source-s3/README.md b/airbyte-integrations/connectors/source-s3/README.md index acdd88e53cf9..a057b1b24b62 100644 --- a/airbyte-integrations/connectors/source-s3/README.md +++ b/airbyte-integrations/connectors/source-s3/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-s3:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/s3) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_s3/spec.json` file. @@ -57,19 +49,70 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . --no-cache -t airbyte/source-s3:dev + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-s3 build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-s3:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. -You can also build the connector image via Gradle: +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-s3:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-s3:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-s3:dev . +# Running the spec command against your patched connector +docker run airbyte/source-s3:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -78,45 +121,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-s3:dev check --config docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-s3:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-s3:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install '.[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-s3 test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-s3:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-s3:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-s3:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -126,8 +140,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-s3 test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/s3.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml b/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml index 38a38eac75d6..4d8db46aba24 100644 --- a/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-s3/acceptance-test-config.yml @@ -6,47 +6,52 @@ acceptance_tests: path: integration_tests/expected_records/csv.jsonl exact_order: true timeout_seconds: 1800 - - config_path: secrets/legacy_csv_custom_encoding_config.json + - config_path: secrets/config_iam_role.json + expect_records: + path: integration_tests/expected_records/csv.jsonl + exact_order: true + timeout_seconds: 1800 + - config_path: secrets/v4_csv_custom_encoding_config.json expect_records: path: integration_tests/expected_records/legacy_csv_custom_encoding.jsonl exact_order: true timeout_seconds: 1800 - - config_path: secrets/legacy_csv_custom_format_config.json + - config_path: secrets/v4_csv_custom_format_config.json expect_records: path: integration_tests/expected_records/legacy_csv_custom_format.jsonl exact_order: true timeout_seconds: 1800 - - config_path: secrets/legacy_csv_user_schema_config.json + - config_path: secrets/v4_csv_user_schema_config.json expect_records: path: integration_tests/expected_records/legacy_csv_user_schema.jsonl exact_order: true timeout_seconds: 1800 - - config_path: secrets/legacy_csv_no_header_config.json + - config_path: secrets/v4_csv_no_header_config.json expect_records: path: integration_tests/expected_records/legacy_csv_no_header.jsonl exact_order: true timeout_seconds: 1800 - - config_path: secrets/legacy_csv_skip_rows_config.json + - config_path: secrets/v4_csv_skip_rows_config.json expect_records: path: integration_tests/expected_records/legacy_csv_skip_rows.jsonl exact_order: true timeout_seconds: 1800 - - config_path: secrets/legacy_csv_skip_rows_no_header_config.json + - config_path: secrets/v4_csv_skip_rows_no_header_config.json expect_records: path: integration_tests/expected_records/legacy_csv_skip_rows_no_header.jsonl exact_order: true timeout_seconds: 1800 - - config_path: secrets/legacy_csv_with_nulls_config.json + - config_path: secrets/v4_csv_with_nulls_config.json expect_records: path: integration_tests/expected_records/legacy_csv_with_nulls.jsonl exact_order: true timeout_seconds: 1800 - - config_path: secrets/legacy_csv_with_null_bools_config.json + - config_path: secrets/v4_csv_with_null_bools_config.json expect_records: path: integration_tests/expected_records/legacy_csv_with_null_bools.jsonl exact_order: true timeout_seconds: 1800 - - config_path: secrets/parquet_config.json + - config_path: secrets/v4_parquet_config.json expect_records: path: integration_tests/expected_records/parquet.jsonl exact_order: true @@ -56,131 +61,250 @@ acceptance_tests: path: integration_tests/expected_records/parquet_dataset.jsonl exact_order: true timeout_seconds: 1800 - - config_path: secrets/legacy_parquet_decimal_config.json + - config_path: secrets/v4_parquet_decimal_config.json expect_records: path: integration_tests/expected_records/legacy_parquet_decimal.jsonl timeout_seconds: 1800 - - config_path: secrets/avro_config.json + - config_path: secrets/v4_avro_config.json expect_records: path: integration_tests/expected_records/avro.jsonl exact_order: true timeout_seconds: 1800 - - config_path: secrets/jsonl_config.json + - config_path: secrets/v4_jsonl_config.json expect_records: path: integration_tests/expected_records/jsonl.jsonl exact_order: true timeout_seconds: 1800 - - config_path: secrets/jsonl_newlines_config.json + - config_path: secrets/v4_jsonl_newlines_config.json expect_records: path: integration_tests/expected_records/jsonl_newlines.jsonl exact_order: true timeout_seconds: 1800 + - config_path: secrets/zip_config_csv.json + expect_records: + path: integration_tests/expected_records/zip_csv.jsonl + exact_order: true + timeout_seconds: 1800 + - config_path: secrets/zip_config_csv_custom_encoding.json + expect_records: + path: integration_tests/expected_records/zip_csv_custom_encoding.jsonl + exact_order: true + timeout_seconds: 1800 + - config_path: secrets/zip_config_jsonl.json + expect_records: + path: integration_tests/expected_records/zip_jsonl.jsonl + exact_order: true + timeout_seconds: 1800 + - config_path: secrets/zip_config_avro.json + expect_records: + path: integration_tests/expected_records/zip_avro.jsonl + exact_order: true + timeout_seconds: 1800 + - config_path: secrets/zip_config_parquet.json + expect_records: + path: integration_tests/expected_records/zip_parquet.jsonl + exact_order: true + timeout_seconds: 1800 + - config_path: secrets/unstructured_config.json + expect_records: + path: integration_tests/expected_records/unstructured.jsonl + exact_order: true + timeout_seconds: 1800 connection: tests: - config_path: secrets/config.json status: succeed - - config_path: secrets/legacy_csv_custom_encoding_config.json + - config_path: secrets/config_iam_role.json + status: succeed + - config_path: secrets/v4_csv_custom_encoding_config.json + status: succeed + - config_path: secrets/v4_csv_custom_format_config.json status: succeed - - config_path: secrets/legacy_csv_custom_format_config.json + - config_path: secrets/v4_csv_user_schema_config.json status: succeed - - config_path: secrets/legacy_csv_user_schema_config.json + - config_path: secrets/v4_csv_no_header_config.json status: succeed - - config_path: secrets/legacy_csv_no_header_config.json + - config_path: secrets/v4_csv_skip_rows_config.json status: succeed - - config_path: secrets/legacy_csv_skip_rows_config.json + - config_path: secrets/v4_csv_skip_rows_no_header_config.json status: succeed - - config_path: secrets/legacy_csv_skip_rows_no_header_config.json + - config_path: secrets/v4_csv_with_nulls_config.json status: succeed - - config_path: secrets/legacy_csv_with_nulls_config.json + - config_path: secrets/v4_csv_with_null_bools_config.json status: succeed - - config_path: secrets/legacy_csv_with_null_bools_config.json + - config_path: secrets/v4_parquet_config.json status: succeed - - config_path: secrets/parquet_config.json + - config_path: secrets/v4_avro_config.json status: succeed - - config_path: secrets/avro_config.json + - config_path: secrets/v4_jsonl_config.json status: succeed - - config_path: secrets/jsonl_config.json + - config_path: secrets/v4_jsonl_newlines_config.json status: succeed - - config_path: secrets/jsonl_newlines_config.json + - config_path: secrets/zip_config_csv.json + status: succeed + - config_path: secrets/zip_config_csv_custom_encoding.json + status: succeed + - config_path: secrets/zip_config_jsonl.json + status: succeed + - config_path: secrets/zip_config_avro.json + status: succeed + - config_path: secrets/zip_config_parquet.json status: succeed - config_path: integration_tests/invalid_config.json status: failed discovery: tests: + - config_path: secrets/config_iam_role.json + backward_compatibility_tests_config: + disable_for_version: "4.4.0" # new authentication added - IAM role - config_path: secrets/config.json - - config_path: secrets/legacy_csv_custom_encoding_config.json - - config_path: secrets/legacy_csv_custom_format_config.json - - config_path: secrets/legacy_csv_user_schema_config.json - - config_path: secrets/legacy_csv_no_header_config.json - - config_path: secrets/legacy_csv_skip_rows_config.json - - config_path: secrets/legacy_csv_with_nulls_config.json - - config_path: secrets/parquet_config.json - - config_path: secrets/avro_config.json - - config_path: secrets/jsonl_config.json - - config_path: secrets/jsonl_newlines_config.json + backward_compatibility_tests_config: + disable_for_version: "4.0.3" # removing the `streams.*.file_type` field which was redundant with `streams.*.format` + - config_path: secrets/v4_csv_custom_encoding_config.json + backward_compatibility_tests_config: + disable_for_version: "4.0.3" # removing the `streams.*.file_type` field which was redundant with `streams.*.format` + - config_path: secrets/v4_csv_custom_format_config.json + backward_compatibility_tests_config: + disable_for_version: "4.0.3" # removing the `streams.*.file_type` field which was redundant with `streams.*.format` + - config_path: secrets/v4_csv_user_schema_config.json + backward_compatibility_tests_config: + disable_for_version: "4.0.3" # removing the `streams.*.file_type` field which was redundant with `streams.*.format` + - config_path: secrets/v4_csv_no_header_config.json + backward_compatibility_tests_config: + disable_for_version: "4.0.3" # removing the `streams.*.file_type` field which was redundant with `streams.*.format` + - config_path: secrets/v4_csv_skip_rows_config.json + backward_compatibility_tests_config: + disable_for_version: "4.0.3" # removing the `streams.*.file_type` field which was redundant with `streams.*.format` + - config_path: secrets/v4_csv_with_nulls_config.json + backward_compatibility_tests_config: + disable_for_version: "4.0.3" # removing the `streams.*.file_type` field which was redundant with `streams.*.format` + - config_path: secrets/v4_parquet_config.json + backward_compatibility_tests_config: + disable_for_version: "4.0.3" # removing the `streams.*.file_type` field which was redundant with `streams.*.format` + - config_path: secrets/v4_avro_config.json + backward_compatibility_tests_config: + disable_for_version: "4.0.3" # removing the `streams.*.file_type` field which was redundant with `streams.*.format` + - config_path: secrets/v4_jsonl_config.json + backward_compatibility_tests_config: + disable_for_version: "4.0.3" # removing the `streams.*.file_type` field which was redundant with `streams.*.format` + - config_path: secrets/v4_jsonl_newlines_config.json + backward_compatibility_tests_config: + disable_for_version: "4.0.3" # removing the `streams.*.file_type` field which was redundant with `streams.*.format` + - config_path: secrets/zip_config_csv.json + backward_compatibility_tests_config: + disable_for_version: "4.0.5" # new compression type added - zip + - config_path: secrets/zip_config_csv_custom_encoding.json + backward_compatibility_tests_config: + disable_for_version: "4.0.5" # new compression type added - zip + - config_path: secrets/zip_config_jsonl.json + backward_compatibility_tests_config: + disable_for_version: "4.0.5" # new compression type added - zip + - config_path: secrets/zip_config_avro.json + backward_compatibility_tests_config: + disable_for_version: "4.0.5" # new compression type added - zip + - config_path: secrets/zip_config_parquet.json + backward_compatibility_tests_config: + disable_for_version: "4.0.5" # new compression type added - zip full_refresh: tests: - config_path: secrets/config.json configured_catalog_path: integration_tests/configured_catalogs/csv.json timeout_seconds: 1800 - - config_path: secrets/parquet_config.json + - config_path: secrets/config_iam_role.json + configured_catalog_path: integration_tests/configured_catalogs/csv.json + timeout_seconds: 1800 + - config_path: secrets/v4_parquet_config.json configured_catalog_path: integration_tests/configured_catalogs/parquet.json timeout_seconds: 1800 - - config_path: secrets/avro_config.json + - config_path: secrets/v4_avro_config.json configured_catalog_path: integration_tests/configured_catalogs/avro.json timeout_seconds: 1800 - - config_path: secrets/jsonl_config.json + - config_path: secrets/v4_jsonl_config.json + configured_catalog_path: integration_tests/configured_catalogs/jsonl.json + timeout_seconds: 1800 + - config_path: secrets/v4_jsonl_newlines_config.json configured_catalog_path: integration_tests/configured_catalogs/jsonl.json timeout_seconds: 1800 - - config_path: secrets/jsonl_newlines_config.json + - config_path: secrets/zip_config_csv.json + configured_catalog_path: integration_tests/configured_catalogs/csv.json + timeout_seconds: 1800 + - config_path: secrets/zip_config_csv_custom_encoding.json + configured_catalog_path: integration_tests/configured_catalogs/csv.json + timeout_seconds: 1800 + - config_path: secrets/zip_config_jsonl.json configured_catalog_path: integration_tests/configured_catalogs/jsonl.json timeout_seconds: 1800 + - config_path: secrets/zip_config_avro.json + configured_catalog_path: integration_tests/configured_catalogs/avro.json + timeout_seconds: 1800 + - config_path: secrets/zip_config_parquet.json + configured_catalog_path: integration_tests/configured_catalogs/parquet.json + timeout_seconds: 1800 incremental: tests: - config_path: secrets/config.json configured_catalog_path: integration_tests/configured_catalogs/csv.json - cursor_paths: - test: - - _ab_source_file_last_modified future_state: future_state_path: integration_tests/abnormal_state.json timeout_seconds: 1800 - - config_path: secrets/parquet_config.json + - config_path: secrets/config_iam_role.json + configured_catalog_path: integration_tests/configured_catalogs/csv.json + future_state: + future_state_path: integration_tests/abnormal_state.json + timeout_seconds: 1800 + - config_path: secrets/v4_parquet_config.json configured_catalog_path: integration_tests/configured_catalogs/parquet.json - cursor_paths: - test: - - _ab_source_file_last_modified future_state: future_state_path: integration_tests/abnormal_state.json timeout_seconds: 1800 - - config_path: secrets/avro_config.json + - config_path: secrets/v4_avro_config.json configured_catalog_path: integration_tests/configured_catalogs/avro.json - cursor_paths: - test: - - _ab_source_file_last_modified future_state: future_state_path: integration_tests/abnormal_state.json timeout_seconds: 1800 - - config_path: secrets/jsonl_config.json + - config_path: secrets/v4_jsonl_config.json + configured_catalog_path: integration_tests/configured_catalogs/jsonl.json + future_state: + future_state_path: integration_tests/abnormal_state.json + timeout_seconds: 1800 + - config_path: secrets/v4_jsonl_newlines_config.json configured_catalog_path: integration_tests/configured_catalogs/jsonl.json - cursor_paths: - test: - - _ab_source_file_last_modified future_state: future_state_path: integration_tests/abnormal_state.json timeout_seconds: 1800 - - config_path: secrets/jsonl_newlines_config.json + - config_path: secrets/zip_config_csv.json + configured_catalog_path: integration_tests/configured_catalogs/csv.json + future_state: + future_state_path: integration_tests/abnormal_state.json + timeout_seconds: 1800 + - config_path: secrets/zip_config_csv_custom_encoding.json + configured_catalog_path: integration_tests/configured_catalogs/csv.json + future_state: + future_state_path: integration_tests/abnormal_state.json + timeout_seconds: 1800 + - config_path: secrets/zip_config_jsonl.json configured_catalog_path: integration_tests/configured_catalogs/jsonl.json - cursor_paths: - test: - - _ab_source_file_last_modified + future_state: + future_state_path: integration_tests/abnormal_state.json + timeout_seconds: 1800 + - config_path: secrets/zip_config_avro.json + configured_catalog_path: integration_tests/configured_catalogs/avro.json + future_state: + future_state_path: integration_tests/abnormal_state.json + timeout_seconds: 1800 + - config_path: secrets/zip_config_parquet.json + configured_catalog_path: integration_tests/configured_catalogs/parquet.json future_state: future_state_path: integration_tests/abnormal_state.json timeout_seconds: 1800 spec: tests: - spec_path: integration_tests/spec.json + backward_compatibility_tests_config: + disable_for_version: "4.0.3" # removing the `streams.*.file_type` field which was redundant with `streams.*.format` + - spec_path: integration_tests/cloud_spec.json + deployment_mode: "cloud" connector_image: airbyte/source-s3:dev test_strictness_level: high diff --git a/airbyte-integrations/connectors/source-s3/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-s3/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-s3/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-s3/build.gradle b/airbyte-integrations/connectors/source-s3/build.gradle deleted file mode 100644 index c6b2cab5faac..000000000000 --- a/airbyte-integrations/connectors/source-s3/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_s3' -} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/__init__.py b/airbyte-integrations/connectors/source-s3/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-s3/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-s3/integration_tests/acceptance.py index 6b0c294530cd..706e9eba88be 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-s3/integration_tests/acceptance.py @@ -3,14 +3,46 @@ # +import json +import logging +from pathlib import Path from typing import Iterable import pytest +import yaml pytest_plugins = ("connector_acceptance_test.plugin",) +logger = logging.getLogger("airbyte") @pytest.fixture(scope="session", autouse=True) def connector_setup() -> Iterable[None]: - """This fixture is a placeholder for external resources that acceptance test might require.""" + """This fixture is responsible for configuring AWS credentials that are used for assuming role during the IAM role based authentication.""" + config_file_path = "secrets/config_iam_role.json" + acceptance_test_config_file_path = "acceptance-test-config.yml" + + # Read environment variables from the JSON file + with open(config_file_path, "r") as file: + config = json.load(file) + + # Prepare environment variables to append to the YAML file + env_vars = { + "custom_environment_variables": { + "AWS_ASSUME_ROLE_EXTERNAL_ID": config["acceptance_test_aws_external_id"], + "AWS_ACCESS_KEY_ID": config["acceptance_test_aws_access_key_id"], + "AWS_SECRET_ACCESS_KEY": config["acceptance_test_aws_secret_access_key"], + } + } + + # Append environment variables to the YAML file + yaml_path = Path(acceptance_test_config_file_path) + if yaml_path.is_file(): + with open(acceptance_test_config_file_path, "r") as file: + existing_data = yaml.safe_load(file) or {} + existing_data.update(env_vars) + with open(acceptance_test_config_file_path, "w") as file: + yaml.safe_dump(existing_data, file) + else: + raise Exception(f"{acceptance_test_config_file_path} does not exist.") + yield diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/cloud_spec.json b/airbyte-integrations/connectors/source-s3/integration_tests/cloud_spec.json new file mode 100644 index 000000000000..ed084d3b08d3 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/cloud_spec.json @@ -0,0 +1,658 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/s3", + "connectionSpecification": { + "title": "Config", + "description": "NOTE: When this Spec is changed, legacy_config_transformer.py must also be modified to uptake the changes\nbecause it is responsible for converting legacy S3 v3 configs into v4 configs using the File-Based CDK.", + "type": "object", + "properties": { + "start_date": { + "title": "Start Date", + "description": "UTC date and time in the format 2017-01-25T00:00:00.000000Z. Any file modified before this date will not be replicated.", + "examples": ["2021-01-01T00:00:00.000000Z"], + "format": "date-time", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{6}Z$", + "pattern_descriptor": "YYYY-MM-DDTHH:mm:ss.SSSSSSZ", + "order": 1, + "type": "string" + }, + "streams": { + "title": "The list of streams to sync", + "description": "Each instance of this configuration defines a stream. Use this to define which files belong in the stream, their format, and how they should be parsed and validated. When sending data to warehouse destination such as Snowflake or BigQuery, each stream is a separate table.", + "order": 10, + "type": "array", + "items": { + "title": "FileBasedStreamConfig", + "type": "object", + "properties": { + "name": { + "title": "Name", + "description": "The name of the stream.", + "type": "string" + }, + "globs": { + "title": "Globs", + "default": ["**"], + "order": 1, + "description": "The pattern used to specify which files should be selected from the file system. For more information on glob pattern matching look here.", + "type": "array", + "items": { + "type": "string" + } + }, + "legacy_prefix": { + "title": "Legacy Prefix", + "description": "The path prefix configured in v3 versions of the S3 connector. This option is deprecated in favor of a single glob.", + "airbyte_hidden": true, + "type": "string" + }, + "validation_policy": { + "title": "Validation Policy", + "description": "The name of the validation policy that dictates sync behavior when a record does not adhere to the stream schema.", + "default": "Emit Record", + "enum": ["Emit Record", "Skip Record", "Wait for Discover"] + }, + "input_schema": { + "title": "Input Schema", + "description": "The schema that will be used to validate records extracted from the file. This will override the stream schema that is auto-detected from incoming files.", + "type": "string" + }, + "primary_key": { + "title": "Primary Key", + "description": "The column or columns (for a composite key) that serves as the unique identifier of a record. If empty, the primary key will default to the parser's default primary key.", + "type": "string", + "airbyte_hidden": true + }, + "days_to_sync_if_history_is_full": { + "title": "Days To Sync If History Is Full", + "description": "When the state history of the file store is full, syncs will only read files that were last modified in the provided day range.", + "default": 3, + "type": "integer" + }, + "format": { + "title": "Format", + "description": "The configuration options that are used to alter how to read incoming files that deviate from the standard formatting.", + "type": "object", + "oneOf": [ + { + "title": "Avro Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "avro", + "const": "avro", + "type": "string" + }, + "double_as_string": { + "title": "Convert Double Fields to Strings", + "description": "Whether to convert double fields to strings. This is recommended if you have decimal numbers with a high degree of precision because there can be a loss precision when handling floating point numbers.", + "default": false, + "type": "boolean" + } + }, + "required": ["filetype"] + }, + { + "title": "CSV Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "csv", + "const": "csv", + "type": "string" + }, + "delimiter": { + "title": "Delimiter", + "description": "The character delimiting individual cells in the CSV data. This may only be a 1-character string. For tab-delimited data enter '\\t'.", + "default": ",", + "type": "string" + }, + "quote_char": { + "title": "Quote Character", + "description": "The character used for quoting CSV values. To disallow quoting, make this field blank.", + "default": "\"", + "type": "string" + }, + "escape_char": { + "title": "Escape Character", + "description": "The character used for escaping special characters. To disallow escaping, leave this field blank.", + "type": "string" + }, + "encoding": { + "title": "Encoding", + "description": "The character encoding of the CSV data. Leave blank to default to UTF8. See list of python encodings for allowable options.", + "default": "utf8", + "type": "string" + }, + "double_quote": { + "title": "Double Quote", + "description": "Whether two quotes in a quoted CSV value denote a single quote in the data.", + "default": true, + "type": "boolean" + }, + "null_values": { + "title": "Null Values", + "description": "A set of case-sensitive strings that should be interpreted as null values. For example, if the value 'NA' should be interpreted as null, enter 'NA' in this field.", + "default": [], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "strings_can_be_null": { + "title": "Strings Can Be Null", + "description": "Whether strings can be interpreted as null values. If true, strings that match the null_values set will be interpreted as null. If false, strings that match the null_values set will be interpreted as the string itself.", + "default": true, + "type": "boolean" + }, + "skip_rows_before_header": { + "title": "Skip Rows Before Header", + "description": "The number of rows to skip before the header row. For example, if the header row is on the 3rd row, enter 2 in this field.", + "default": 0, + "type": "integer" + }, + "skip_rows_after_header": { + "title": "Skip Rows After Header", + "description": "The number of rows to skip after the header row.", + "default": 0, + "type": "integer" + }, + "header_definition": { + "title": "CSV Header Definition", + "description": "How headers will be defined. `User Provided` assumes the CSV does not have a header row and uses the headers provided and `Autogenerated` assumes the CSV does not have a header row and the CDK will generate headers using for `f{i}` where `i` is the index starting from 0. Else, the default behavior is to use the header from the CSV file. If a user wants to autogenerate or provide column names for a CSV having headers, they can skip rows.", + "default": { + "header_definition_type": "From CSV" + }, + "oneOf": [ + { + "title": "From CSV", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "From CSV", + "const": "From CSV", + "type": "string" + } + }, + "required": ["header_definition_type"] + }, + { + "title": "Autogenerated", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "Autogenerated", + "const": "Autogenerated", + "type": "string" + } + }, + "required": ["header_definition_type"] + }, + { + "title": "User Provided", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "User Provided", + "const": "User Provided", + "type": "string" + }, + "column_names": { + "title": "Column Names", + "description": "The column names that will be used while emitting the CSV records", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": ["column_names", "header_definition_type"] + } + ], + "type": "object" + }, + "true_values": { + "title": "True Values", + "description": "A set of case-sensitive strings that should be interpreted as true values.", + "default": ["y", "yes", "t", "true", "on", "1"], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "false_values": { + "title": "False Values", + "description": "A set of case-sensitive strings that should be interpreted as false values.", + "default": ["n", "no", "f", "false", "off", "0"], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "inference_type": { + "title": "Inference Type", + "description": "How to infer the types of the columns. If none, inference default to strings.", + "default": "None", + "airbyte_hidden": true, + "enum": ["None", "Primitive Types Only"] + } + }, + "required": ["filetype"] + }, + { + "title": "Jsonl Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "jsonl", + "const": "jsonl", + "type": "string" + } + }, + "required": ["filetype"] + }, + { + "title": "Parquet Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "parquet", + "const": "parquet", + "type": "string" + }, + "decimal_as_float": { + "title": "Convert Decimal Fields to Floats", + "description": "Whether to convert decimal fields to floats. There is a loss of precision when converting decimals to floats, so this is not recommended.", + "default": false, + "type": "boolean" + } + }, + "required": ["filetype"] + }, + { + "title": "Document File Type Format (Experimental)", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "unstructured", + "const": "unstructured", + "type": "string" + }, + "skip_unprocessable_files": { + "type": "boolean", + "default": true, + "title": "Skip Unprocessable Files", + "description": "If true, skip files that cannot be parsed and pass the error message along as the _ab_source_file_parse_error field. If false, fail the sync.", + "always_show": true + }, + "strategy": { + "type": "string", + "always_show": true, + "order": 0, + "default": "auto", + "title": "Parsing Strategy", + "enum": ["auto", "fast", "ocr_only", "hi_res"], + "description": "The strategy used to parse documents. `fast` extracts text directly from the document which doesn't work for all files. `ocr_only` is more reliable, but slower. `hi_res` is the most reliable, but requires an API key and a hosted instance of unstructured and can't be used with local mode. See the unstructured.io documentation for more details: https://unstructured-io.github.io/unstructured/core/partition.html#partition-pdf" + }, + "processing": { + "title": "Processing", + "description": "Processing configuration", + "default": { + "mode": "local" + }, + "type": "object", + "oneOf": [ + { + "title": "Local", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "local", + "const": "local", + "enum": ["local"], + "type": "string" + } + }, + "description": "Process files locally, supporting `fast` and `ocr` modes. This is the default option.", + "required": ["mode"] + } + ] + } + }, + "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file.", + "required": ["filetype"] + } + ] + }, + "schemaless": { + "title": "Schemaless", + "description": "When enabled, syncs will not validate or structure records against the stream's schema.", + "default": false, + "type": "boolean" + } + }, + "required": ["name", "format"] + } + }, + "bucket": { + "title": "Bucket", + "description": "Name of the S3 bucket where the file(s) exist.", + "order": 0, + "type": "string" + }, + "aws_access_key_id": { + "title": "AWS Access Key ID", + "description": "In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper permissions. If accessing publicly available data, this field is not necessary.", + "airbyte_secret": true, + "order": 2, + "type": "string" + }, + "role_arn": { + "title": "AWS Role ARN", + "description": "Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations requested using this profile. Set the External ID to the Airbyte workspace ID, which can be found in the URL of this page.", + "order": 6, + "type": "string" + }, + "aws_secret_access_key": { + "title": "AWS Secret Access Key", + "description": "In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper permissions. If accessing publicly available data, this field is not necessary.", + "airbyte_secret": true, + "order": 3, + "type": "string" + }, + "endpoint": { + "title": "Endpoint", + "description": "Endpoint to an S3 compatible service. Leave empty to use AWS. The custom endpoint must be secure, but the 'https' prefix is not required.", + "default": "", + "examples": ["my-s3-endpoint.com", "https://my-s3-endpoint.com"], + "pattern": "^(?!http://).*$", + "order": 4, + "type": "string" + }, + "dataset": { + "title": "Output Stream Name", + "description": "Deprecated and will be removed soon. Please do not use this field anymore and use streams.name instead. The name of the stream you would like this source to output. Can contain letters, numbers, or underscores.", + "pattern": "^([A-Za-z0-9-_]+)$", + "order": 100, + "type": "string", + "airbyte_hidden": true + }, + "path_pattern": { + "title": "Pattern of files to replicate", + "description": "Deprecated and will be removed soon. Please do not use this field anymore and use streams.globs instead. A regular expression which tells the connector which files to replicate. All files which match this pattern will be replicated. Use | to separate multiple patterns. See this page to understand pattern syntax (GLOBSTAR and SPLIT flags are enabled). Use pattern ** to pick up all files.", + "examples": [ + "**", + "myFolder/myTableFiles/*.csv|myFolder/myOtherTableFiles/*.csv" + ], + "order": 110, + "type": "string", + "airbyte_hidden": true + }, + "format": { + "title": "File Format", + "description": "Deprecated and will be removed soon. Please do not use this field anymore and use streams.format instead. The format of the files you'd like to replicate", + "default": "csv", + "order": 120, + "type": "object", + "oneOf": [ + { + "title": "CSV", + "description": "This connector utilises PyArrow (Apache Arrow) for CSV parsing.", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "csv", + "const": "csv", + "type": "string" + }, + "delimiter": { + "title": "Delimiter", + "description": "The character delimiting individual cells in the CSV data. This may only be a 1-character string. For tab-delimited data enter '\\t'.", + "default": ",", + "minLength": 1, + "order": 0, + "type": "string" + }, + "infer_datatypes": { + "title": "Infer Datatypes", + "description": "Configures whether a schema for the source should be inferred from the current data or not. If set to false and a custom schema is set, then the manually enforced schema is used. If a schema is not manually set, and this is set to false, then all fields will be read as strings", + "default": true, + "order": 1, + "type": "boolean" + }, + "quote_char": { + "title": "Quote Character", + "description": "The character used for quoting CSV values. To disallow quoting, make this field blank.", + "default": "\"", + "order": 2, + "type": "string" + }, + "escape_char": { + "title": "Escape Character", + "description": "The character used for escaping special characters. To disallow escaping, leave this field blank.", + "order": 3, + "type": "string" + }, + "encoding": { + "title": "Encoding", + "description": "The character encoding of the CSV data. Leave blank to default to UTF8. See list of python encodings for allowable options.", + "default": "utf8", + "order": 4, + "type": "string" + }, + "double_quote": { + "title": "Double Quote", + "description": "Whether two quotes in a quoted CSV value denote a single quote in the data.", + "default": true, + "order": 5, + "type": "boolean" + }, + "newlines_in_values": { + "title": "Allow newlines in values", + "description": "Whether newline characters are allowed in CSV values. Turning this on may affect performance. Leave blank to default to False.", + "default": false, + "order": 6, + "type": "boolean" + }, + "additional_reader_options": { + "title": "Additional Reader Options", + "description": "Optionally add a valid JSON string here to provide additional options to the csv reader. Mappings must correspond to options detailed here. 'column_types' is used internally to handle schema so overriding that would likely cause problems.", + "examples": [ + "{\"timestamp_parsers\": [\"%m/%d/%Y %H:%M\", \"%Y/%m/%d %H:%M\"], \"strings_can_be_null\": true, \"null_values\": [\"NA\", \"NULL\"]}" + ], + "order": 7, + "type": "string" + }, + "advanced_options": { + "title": "Advanced Options", + "description": "Optionally add a valid JSON string here to provide additional Pyarrow ReadOptions. Specify 'column_names' here if your CSV doesn't have header, or if you want to use custom column names. 'block_size' and 'encoding' are already used above, specify them again here will override the values above.", + "examples": ["{\"column_names\": [\"column1\", \"column2\"]}"], + "order": 8, + "type": "string" + }, + "block_size": { + "title": "Block Size", + "description": "The chunk size in bytes to process at a time in memory from each file. If your data is particularly wide and failing during schema detection, increasing this should solve it. Beware of raising this too high as you could hit OOM errors.", + "default": 10000, + "minimum": 1, + "maximum": 2147483647, + "order": 9, + "type": "integer" + } + } + }, + { + "title": "Parquet", + "description": "This connector utilises PyArrow (Apache Arrow) for Parquet parsing.", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "parquet", + "const": "parquet", + "type": "string" + }, + "columns": { + "title": "Selected Columns", + "description": "If you only want to sync a subset of the columns from the file(s), add the columns you want here as a comma-delimited list. Leave it empty to sync all columns.", + "order": 0, + "type": "array", + "items": { + "type": "string" + } + }, + "batch_size": { + "title": "Record batch size", + "description": "Maximum number of records per batch read from the input files. Batches may be smaller if there aren’t enough rows in the file. This option can help avoid out-of-memory errors if your data is particularly wide.", + "default": 65536, + "order": 1, + "type": "integer" + }, + "buffer_size": { + "title": "Buffer Size", + "description": "Perform read buffering when deserializing individual column chunks. By default every group column will be loaded fully to memory. This option can help avoid out-of-memory errors if your data is particularly wide.", + "default": 2, + "type": "integer" + } + } + }, + { + "title": "Avro", + "description": "This connector utilises fastavro for Avro parsing.", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "avro", + "const": "avro", + "type": "string" + } + } + }, + { + "title": "Jsonl", + "description": "This connector uses PyArrow for JSON Lines (jsonl) file parsing.", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "jsonl", + "const": "jsonl", + "type": "string" + }, + "newlines_in_values": { + "title": "Allow newlines in values", + "description": "Whether newline characters are allowed in JSON values. Turning this on may affect performance. Leave blank to default to False.", + "default": false, + "order": 0, + "type": "boolean" + }, + "unexpected_field_behavior": { + "title": "Unexpected field behavior", + "description": "How JSON fields outside of explicit_schema (if given) are treated. Check PyArrow documentation for details", + "default": "infer", + "examples": ["ignore", "infer", "error"], + "order": 1, + "enum": ["ignore", "infer", "error"] + }, + "block_size": { + "title": "Block Size", + "description": "The chunk size in bytes to process at a time in memory from each file. If your data is particularly wide and failing during schema detection, increasing this should solve it. Beware of raising this too high as you could hit OOM errors.", + "default": 0, + "order": 2, + "type": "integer" + } + } + } + ], + "airbyte_hidden": true + }, + "schema": { + "title": "Manually enforced data schema", + "description": "Deprecated and will be removed soon. Please do not use this field anymore and use streams.input_schema instead. Optionally provide a schema to enforce, as a valid JSON string. Ensure this is a mapping of { \"column\" : \"type\" }, where types are valid JSON Schema datatypes. Leave as {} to auto-infer the schema.", + "default": "{}", + "examples": [ + "{\"column_1\": \"number\", \"column_2\": \"string\", \"column_3\": \"array\", \"column_4\": \"object\", \"column_5\": \"boolean\"}" + ], + "order": 130, + "type": "string", + "airbyte_hidden": true + }, + "provider": { + "title": "S3: Amazon Web Services", + "type": "object", + "properties": { + "bucket": { + "title": "Bucket", + "description": "Name of the S3 bucket where the file(s) exist.", + "order": 0, + "type": "string" + }, + "aws_access_key_id": { + "title": "AWS Access Key ID", + "description": "In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper permissions. If accessing publicly available data, this field is not necessary.", + "airbyte_secret": true, + "always_show": true, + "order": 1, + "type": "string" + }, + "aws_secret_access_key": { + "title": "AWS Secret Access Key", + "description": "In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper permissions. If accessing publicly available data, this field is not necessary.", + "airbyte_secret": true, + "always_show": true, + "order": 2, + "type": "string" + }, + "role_arn": { + "title": "AWS Role ARN", + "description": "Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations requested using this profile. Set the External ID to the Airbyte workspace ID, which can be found in the URL of this page.", + "always_show": true, + "order": 6, + "type": "string" + }, + "path_prefix": { + "title": "Path Prefix", + "description": "By providing a path-like prefix (e.g. myFolder/thisTable/) under which all the relevant files sit, we can optimize finding these in S3. This is optional but recommended if your bucket contains many folders/files which you don't need to replicate.", + "default": "", + "order": 3, + "type": "string" + }, + "endpoint": { + "title": "Endpoint", + "description": "Endpoint to an S3 compatible service. Leave empty to use AWS.", + "default": "", + "order": 4, + "type": "string" + }, + "start_date": { + "title": "Start Date", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any file modified before this date will not be replicated.", + "examples": ["2021-01-01T00:00:00Z"], + "format": "date-time", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "order": 5, + "type": "string" + } + }, + "required": [], + "order": 111, + "description": "Deprecated and will be removed soon. Please do not use this field anymore and use bucket, aws_access_key_id, aws_secret_access_key and endpoint instead. Use this to load files from S3 or S3-compatible services", + "airbyte_hidden": true + } + }, + "required": ["streams", "bucket"] + } +} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/conftest.py b/airbyte-integrations/connectors/source-s3/integration_tests/conftest.py deleted file mode 100644 index 67636cb06904..000000000000 --- a/airbyte-integrations/connectors/source-s3/integration_tests/conftest.py +++ /dev/null @@ -1,26 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from typing import Any - -from airbyte_cdk import AirbyteLogger - -from .integration_test import TestIncrementalFileStreamS3 - -LOGGER = AirbyteLogger() - - -def pytest_sessionfinish(session: Any, exitstatus: Any) -> None: - """tries to find and remove all temp buckets""" - instance = TestIncrementalFileStreamS3() - instance._s3_connect(instance.credentials) - temp_buckets = [] - for bucket in instance.s3_resource.buckets.all(): - if bucket.name.startswith(instance.temp_bucket_prefix): - temp_buckets.append(bucket.name) - for bucket_name in temp_buckets: - bucket = instance.s3_resource.Bucket(bucket_name) - bucket.objects.all().delete() - bucket.delete() - LOGGER.info(f"S3 Bucket {bucket_name} is now deleted") diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/avro.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/avro.jsonl index f1df4c835fe7..449798e8f2ef 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/avro.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/avro.jsonl @@ -1,10 +1,10 @@ -{"stream": "test", "data": {"id": 0, "fullname_and_valid": {"fullname": "cfjwIzCRTL", "valid": false}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} -{"stream": "test", "data": {"id": 1, "fullname_and_valid": {"fullname": "LYOnPyuTWw", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} -{"stream": "test", "data": {"id": 2, "fullname_and_valid": {"fullname": "hyTFbsxlRB", "valid": false}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} -{"stream": "test", "data": {"id": 3, "fullname_and_valid": {"fullname": "ooEUiFcFqp", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} -{"stream": "test", "data": {"id": 4, "fullname_and_valid": {"fullname": "pveENwAvOg", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} -{"stream": "test", "data": {"id": 5, "fullname_and_valid": {"fullname": "pPhWgQgZFq", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} -{"stream": "test", "data": {"id": 6, "fullname_and_valid": {"fullname": "MRNMXFkXZo", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} -{"stream": "test", "data": {"id": 7, "fullname_and_valid": {"fullname": "MXvEWMgnIr", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} -{"stream": "test", "data": {"id": 8, "fullname_and_valid": {"fullname": "rqmFGqZqdF", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} -{"stream": "test", "data": {"id": 9, "fullname_and_valid": {"fullname": "lmPpQTcPFM", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} +{"stream": "test", "data": {"id": 0, "fullname_and_valid": {"fullname": "cfjwIzCRTL", "valid": false}, "_ab_source_file_last_modified": "2022-05-11T11:54:11.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} +{"stream": "test", "data": {"id": 1, "fullname_and_valid": {"fullname": "LYOnPyuTWw", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} +{"stream": "test", "data": {"id": 2, "fullname_and_valid": {"fullname": "hyTFbsxlRB", "valid": false}, "_ab_source_file_last_modified": "2022-05-11T11:54:11.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} +{"stream": "test", "data": {"id": 3, "fullname_and_valid": {"fullname": "ooEUiFcFqp", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} +{"stream": "test", "data": {"id": 4, "fullname_and_valid": {"fullname": "pveENwAvOg", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} +{"stream": "test", "data": {"id": 5, "fullname_and_valid": {"fullname": "pPhWgQgZFq", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} +{"stream": "test", "data": {"id": 6, "fullname_and_valid": {"fullname": "MRNMXFkXZo", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} +{"stream": "test", "data": {"id": 7, "fullname_and_valid": {"fullname": "MXvEWMgnIr", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} +{"stream": "test", "data": {"id": 8, "fullname_and_valid": {"fullname": "rqmFGqZqdF", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} +{"stream": "test", "data": {"id": 9, "fullname_and_valid": {"fullname": "lmPpQTcPFM", "valid": true}, "_ab_source_file_last_modified": "2022-05-11T11:54:11.000000Z", "_ab_source_file_url": "test_sample.avro"}, "emitted_at": 10000000} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/csv.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/csv.jsonl index f2a8aea4e963..aa19d854f824 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/csv.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/csv.jsonl @@ -1,8 +1,8 @@ -{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 162727468000} -{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false, "_ab_source_file_last_modified": "2021-07-25T15:33:04.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2021-07-25T15:33:04.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2021-07-25T15:33:04.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2021-07-25T15:33:04.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2021-07-25T15:33:04.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2021-07-25T15:33:04.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2021-07-25T15:33:04.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2021-07-25T15:33:04.000000Z", "_ab_source_file_url": "simple_test.csv"}, "emitted_at": 1627227468000} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/custom_server.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/custom_server.jsonl index 14dedea42c97..b41d8841da80 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/custom_server.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/custom_server.jsonl @@ -1,55 +1,55 @@ -{"stream": "test", "data": {"Year": 1960, "Value": 59184116488.9977, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1961, "Value": 49557050182.9631, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1962, "Value": 46685178504.3274, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1963, "Value": 50097303271.0232, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1964, "Value": 59062254890.1871, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1965, "Value": 69709153115.3147, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1966, "Value": 75879434776.1831, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1967, "Value": 72057028559.6741, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1968, "Value": 69993497892.3132, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1969, "Value": 78718820477.9257, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1970, "Value": 91506211306.3745, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1971, "Value": 98562023844.1813, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1972, "Value": 112159813640.376, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1973, "Value": 136769878359.668, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1974, "Value": 142254742077.706, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1975, "Value": 161162492226.686, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1976, "Value": 151627687364.405, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1977, "Value": 172349014326.931, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1978, "Value": 148382111520.192, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1979, "Value": 176856525405.729, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1980, "Value": 189649992463.987, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1981, "Value": 194369049090.197, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1982, "Value": 203549627211.606, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1983, "Value": 228950200773.115, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1984, "Value": 258082147252.256, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1985, "Value": 307479585852.339, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1986, "Value": 298805792971.544, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1987, "Value": 271349773463.863, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1988, "Value": 310722213686.031, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1989, "Value": 345957485871.286, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1990, "Value": 358973230048.399, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1991, "Value": 381454703832.753, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1992, "Value": 424934065934.066, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1993, "Value": 442874596387.119, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1994, "Value": 562261129868.774, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1995, "Value": 732032045217.766, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1996, "Value": 860844098049.121, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1997, "Value": 958159424835.34, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1998, "Value": 1025276902078.73, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 1999, "Value": 1089447108705.89, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2000, "Value": 1205260678391.96, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2001, "Value": 1332234719889.82, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2002, "Value": 1461906487857.92, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2003, "Value": 1649928718134.59, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2004, "Value": 1941745602165.09, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2005, "Value": 2268598904116.28, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2006, "Value": 2729784031906.09, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2007, "Value": 3523094314820.9, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2008, "Value": 4558431073438.2, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2009, "Value": 5059419738267.41, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2010, "Value": 6039658508485.59, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2011, "Value": 7492432097810.11, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2012, "Value": 8461623162714.07, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2013, "Value": 9490602600148.49, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} -{"stream": "test", "data": {"Year": 2014, "Value": 10354831729340.4, "_ab_source_file_last_modified": "2021-09-23T11:48:44Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1960, "Value": 59184116488.9977, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1961, "Value": 49557050182.9631, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1962, "Value": 46685178504.3274, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1963, "Value": 50097303271.0232, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1964, "Value": 59062254890.1871, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1965, "Value": 69709153115.3147, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1966, "Value": 75879434776.1831, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1967, "Value": 72057028559.6741, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1968, "Value": 69993497892.3132, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1969, "Value": 78718820477.9257, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1970, "Value": 91506211306.3745, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1971, "Value": 98562023844.1813, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1972, "Value": 112159813640.376, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1973, "Value": 136769878359.668, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1974, "Value": 142254742077.706, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1975, "Value": 161162492226.686, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1976, "Value": 151627687364.405, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1977, "Value": 172349014326.931, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1978, "Value": 148382111520.192, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1979, "Value": 176856525405.729, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1980, "Value": 189649992463.987, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1981, "Value": 194369049090.197, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1982, "Value": 203549627211.606, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1983, "Value": 228950200773.115, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1984, "Value": 258082147252.256, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1985, "Value": 307479585852.339, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1986, "Value": 298805792971.544, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1987, "Value": 271349773463.863, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1988, "Value": 310722213686.031, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1989, "Value": 345957485871.286, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1990, "Value": 358973230048.399, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1991, "Value": 381454703832.753, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1992, "Value": 424934065934.066, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1993, "Value": 442874596387.119, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1994, "Value": 562261129868.774, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1995, "Value": 732032045217.766, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1996, "Value": 860844098049.121, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1997, "Value": 958159424835.34, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1998, "Value": 1025276902078.73, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 1999, "Value": 1089447108705.89, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2000, "Value": 1205260678391.96, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2001, "Value": 1332234719889.82, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2002, "Value": 1461906487857.92, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2003, "Value": 1649928718134.59, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2004, "Value": 1941745602165.09, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2005, "Value": 2268598904116.28, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2006, "Value": 2729784031906.09, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2007, "Value": 3523094314820.9, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2008, "Value": 4558431073438.2, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2009, "Value": 5059419738267.41, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2010, "Value": 6039658508485.59, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2011, "Value": 7492432097810.11, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2012, "Value": 8461623162714.07, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2013, "Value": 9490602600148.49, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} +{"stream": "test", "data": {"Year": 2014, "Value": 10354831729340.4, "_ab_source_file_last_modified": "2021-09-23T11:48:44.000000Z", "_ab_source_file_url": "china_gdp.csv"}, "emitted_at": 1632398440000} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/jsonl.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/jsonl.jsonl index 019cfd8aff8a..b0aabda52687 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/jsonl.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/jsonl.jsonl @@ -1,2 +1,2 @@ -{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false,"value": 1.2, "event_date": "2022-01-01T00:00:00Z", "_ab_source_file_last_modified": "2022-07-15T08:31:02Z", "_ab_source_file_url": "simple_test.jsonl"}, "emitted_at": 162727468000} -{"stream": "test", "data": {"id": 2, "name": "ABCDEF", "valid": true,"value": 1.0, "event_date": "2023-01-01T00:00:00Z", "_ab_source_file_last_modified": "2022-07-15T08:31:02Z", "_ab_source_file_url": "simple_test.jsonl"}, "emitted_at": 162727468000} \ No newline at end of file +{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false,"value": 1.2, "event_date": "2022-01-01T00:00:00Z", "_ab_source_file_last_modified": "2022-07-15T08:31:02.000000Z", "_ab_source_file_url": "simple_test.jsonl"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"id": 2, "name": "ABCDEF", "valid": true,"value": 1.0, "event_date": "2023-01-01T00:00:00Z", "_ab_source_file_last_modified": "2022-07-15T08:31:02.000000Z", "_ab_source_file_url": "simple_test.jsonl"}, "emitted_at": 162727468000} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/jsonl_newlines.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/jsonl_newlines.jsonl index e3445c34ebb0..fbb95baa91df 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/jsonl_newlines.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/jsonl_newlines.jsonl @@ -1,2 +1,2 @@ -{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false,"value": 1.2, "event_date": "2022-01-01T00:00:00Z", "_ab_source_file_last_modified": "2022-07-15T10:07:00Z", "_ab_source_file_url": "simple_test_newlines.jsonl"}, "emitted_at": 162727468000} -{"stream": "test", "data": {"id": 2, "name": "ABCDEF", "valid": true,"value": 1.0, "event_date": "2023-01-01T00:00:00Z", "_ab_source_file_last_modified": "2022-07-15T10:07:00Z", "_ab_source_file_url": "simple_test_newlines.jsonl"}, "emitted_at": 162727468000} \ No newline at end of file +{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false,"value": 1.2, "event_date": "2022-01-01T00:00:00Z", "_ab_source_file_last_modified": "2022-07-15T10:07:00.000000Z", "_ab_source_file_url": "simple_test_newlines.jsonl"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"id": 2, "name": "ABCDEF", "valid": true,"value": 1.0, "event_date": "2023-01-01T00:00:00Z", "_ab_source_file_last_modified": "2022-07-15T10:07:00.000000Z", "_ab_source_file_url": "simple_test_newlines.jsonl"}, "emitted_at": 162727468000} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_custom_encoding.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_custom_encoding.jsonl index 9dcaefdf36a1..4ae40924ec8f 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_custom_encoding.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_custom_encoding.jsonl @@ -1,8 +1,8 @@ -{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1\u20ac", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 162727468000} -{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1\u20ac", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:46:54.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:46:54.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:46:54.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:46:54.000000Z", "_ab_source_file_url": "csv_tests/csv_encoded_as_cp1252.csv"}, "emitted_at": 1627227468000} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_custom_format.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_custom_format.jsonl index d95d1648f7af..8b159a550df1 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_custom_format.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_custom_format.jsonl @@ -1,8 +1,8 @@ -{"stream": "test", "data": {"id": 1, "name": "PVdhmj|b1", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 162727468000} -{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 1, "name": "PVdhmj|b1", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T17:25:57.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T17:25:57.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T17:25:57.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T17:25:57.000000Z", "_ab_source_file_url": "csv_tests/custom_format.csv"}, "emitted_at": 1627227468000} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_no_header.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_no_header.jsonl index f71905cdbd4a..d0aabb533827 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_no_header.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_no_header.jsonl @@ -1,8 +1,8 @@ -{"stream": "test", "data": {"f0": 1, "f1": "PVdhmjb1", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 162727468000} -{"stream": "test", "data": {"f0": 2, "f1": "j4DyXTS7", "f2": true, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"f0": 3, "f1": "v0w8fTME", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"f0": 4, "f1": "1q6jD8Np", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"f0": 5, "f1": "77h4aiMP", "f2": true, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"f0": 6, "f1": "Le35Wyic", "f2": true, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"f0": 7, "f1": "xZhh1Kyl", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"f0": 8, "f1": "M2t286iJ", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 1, "f1": "PVdhmjb1", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"f0": 2, "f1": "j4DyXTS7", "f2": true, "_ab_source_file_last_modified": "2023-08-03T21:19:26.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 3, "f1": "v0w8fTME", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 4, "f1": "1q6jD8Np", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 5, "f1": "77h4aiMP", "f2": true, "_ab_source_file_last_modified": "2023-08-03T21:19:26.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 6, "f1": "Le35Wyic", "f2": true, "_ab_source_file_last_modified": "2023-08-03T21:19:26.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 7, "f1": "xZhh1Kyl", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 8, "f1": "M2t286iJ", "f2": false, "_ab_source_file_last_modified": "2023-08-03T21:19:26.000000Z", "_ab_source_file_url": "csv_tests/no_header.csv"}, "emitted_at": 1627227468000} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_skip_rows.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_skip_rows.jsonl index cb10832287ad..43c3bf00e506 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_skip_rows.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_skip_rows.jsonl @@ -1,8 +1,8 @@ -{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 162727468000} -{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 1, "name": "PVdhmjb1", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:07:57.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 3, "name": "v0w8fTME", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:07:57.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T20:07:57.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T20:07:57.000000Z", "_ab_source_file_url": "csv_tests/skip_rows.csv"}, "emitted_at": 1627227468000} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_skip_rows_no_header.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_skip_rows_no_header.jsonl index a2ebfa4ff896..baef6e94d681 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_skip_rows_no_header.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_skip_rows_no_header.jsonl @@ -1,8 +1,8 @@ -{"stream": "test", "data": {"f0": 1, "f1": "PVdhmjb1", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 162727468000} -{"stream": "test", "data": {"f0": 2, "f1": "j4DyXTS7", "f2": true, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"f0": 3, "f1": "v0w8fTME", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"f0": 4, "f1": "1q6jD8Np", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"f0": 5, "f1": "77h4aiMP", "f2": true, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"f0": 6, "f1": "Le35Wyic", "f2": true, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"f0": 7, "f1": "xZhh1Kyl", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"f0": 8, "f1": "M2t286iJ", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 1, "f1": "PVdhmjb1", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"f0": 2, "f1": "j4DyXTS7", "f2": true, "_ab_source_file_last_modified": "2023-08-04T01:18:56.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 3, "f1": "v0w8fTME", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 4, "f1": "1q6jD8Np", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 5, "f1": "77h4aiMP", "f2": true, "_ab_source_file_last_modified": "2023-08-04T01:18:56.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 6, "f1": "Le35Wyic", "f2": true, "_ab_source_file_last_modified": "2023-08-04T01:18:56.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 7, "f1": "xZhh1Kyl", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"f0": 8, "f1": "M2t286iJ", "f2": false, "_ab_source_file_last_modified": "2023-08-04T01:18:56.000000Z", "_ab_source_file_url": "csv_tests/skip_rows_no_header.csv"}, "emitted_at": 1627227468000} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_user_schema.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_user_schema.jsonl index 81f1cc5b2d61..17b072cca8ba 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_user_schema.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_user_schema.jsonl @@ -1,8 +1,8 @@ -{"stream": "test", "data": {"id": 1.0, "name": "PVdhmjb1", "valid": false, "valid_string": "False", "array": "[\"a\", \"b\", \"c\"]", "dict": "{\"key\": \"value\"}","_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 162727468000} -{"stream": "test", "data": {"id": 2.0, "name": "j4DyXTS7", "valid": true, "valid_string": "True", "array": "[\"a\", \"b\"]","dict": "{\"key\": \"value_with_comma\\,\"}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 3.0, "name": "v0w8fTME", "valid": false, "valid_string": "False", "array": "[\"a\"]", "dict": "{\"key\": \"value\"}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 4.0, "name": "1q6jD8Np", "valid": false, "valid_string": "False", "array": "[]","dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 5.0, "name": "77h4aiMP", "valid": true, "valid_string": "True", "array": "[\"b\", \"c\"]","dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 6.0, "name": "Le35Wyic", "valid": true, "valid_string": "True", "array": "[\"a\", \"c\"]", "dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 7.0, "name": "xZhh1Kyl", "valid": false, "valid_string": "False", "array": "[\"b\"]","dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 8.0, "name": "M2t286iJ", "valid": false, "valid_string": "False", "array": "[\"c\"]", "dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 1.0, "name": "PVdhmjb1", "valid": false, "valid_string": "False", "array": "[\"a\", \"b\", \"c\"]", "dict": "{\"key\": \"value\"}","_ab_source_file_last_modified": "2023-08-03T22:17:06.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"id": 2.0, "name": "j4DyXTS7", "valid": true, "valid_string": "True", "array": "[\"a\", \"b\"]","dict": "{\"key\": \"value_with_comma\\,\"}", "_ab_source_file_last_modified": "2023-08-03T22:17:06.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 3.0, "name": "v0w8fTME", "valid": false, "valid_string": "False", "array": "[\"a\"]", "dict": "{\"key\": \"value\"}", "_ab_source_file_last_modified": "2023-08-03T22:17:06.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 4.0, "name": "1q6jD8Np", "valid": false, "valid_string": "False", "array": "[]","dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 5.0, "name": "77h4aiMP", "valid": true, "valid_string": "True", "array": "[\"b\", \"c\"]","dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 6.0, "name": "Le35Wyic", "valid": true, "valid_string": "True", "array": "[\"a\", \"c\"]", "dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 7.0, "name": "xZhh1Kyl", "valid": false, "valid_string": "False", "array": "[\"b\"]","dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 8.0, "name": "M2t286iJ", "valid": false, "valid_string": "False", "array": "[\"c\"]", "dict": "{}", "_ab_source_file_last_modified": "2023-08-03T22:17:06.000000Z", "_ab_source_file_url": "csv_tests/user_schema.csv"}, "emitted_at": 1627227468000} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_with_null_bools.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_with_null_bools.jsonl index 18644e9dc903..948f15f6637a 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_with_null_bools.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_with_null_bools.jsonl @@ -1,8 +1,8 @@ -{"stream": "test", "data": {"id": 1, "name": "null", "valid": null, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 162727468000} -{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 3, "name": "NULL", "valid": null, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": null, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 6, "name": "", "valid": true, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-04T01:42:33Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 1, "name": "null", "valid": null, "_ab_source_file_last_modified": "2023-08-04T01:42:33.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-04T01:42:33.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 3, "name": "NULL", "valid": null, "_ab_source_file_last_modified": "2023-08-04T01:42:33.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-04T01:42:33.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": null, "_ab_source_file_last_modified": "2023-08-04T01:42:33.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 6, "name": "", "valid": true, "_ab_source_file_last_modified": "2023-08-04T01:42:33.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-04T01:42:33.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-04T01:42:33.000000Z", "_ab_source_file_url": "csv_tests/csv_with_null_bools.csv"}, "emitted_at": 1627227468000} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_with_nulls.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_with_nulls.jsonl index efb0b16d65bd..946374929179 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_with_nulls.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_csv_with_nulls.jsonl @@ -1,8 +1,8 @@ -{"stream": "test", "data": {"id": 1, "name": null, "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 162727468000} -{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 3, "name": null, "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000} -{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 1, "name": null, "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 162727468000} +{"stream": "test", "data": {"id": 2, "name": "j4DyXTS7", "valid": true, "_ab_source_file_last_modified": "2023-08-03T19:58:00.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 3, "name": null, "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 4, "name": "1q6jD8Np", "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 5, "name": "77h4aiMP", "valid": true, "_ab_source_file_last_modified": "2023-08-03T19:58:00.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 6, "name": "Le35Wyic", "valid": true, "_ab_source_file_last_modified": "2023-08-03T19:58:00.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 7, "name": "xZhh1Kyl", "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000} +{"stream": "test", "data": {"id": 8, "name": "M2t286iJ", "valid": false, "_ab_source_file_last_modified": "2023-08-03T19:58:00.000000Z", "_ab_source_file_url": "csv_tests/csv_with_nulls.csv"}, "emitted_at": 1627227468000} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_parquet_decimal.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_parquet_decimal.jsonl index 388b10b03112..0d8ae08a0c70 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_parquet_decimal.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/legacy_parquet_decimal.jsonl @@ -1,2 +1,2 @@ -{"stream":"test","data":{"id":"row1","value": 12.345,"_ab_source_file_last_modified":"2023-08-06T18:05:00Z","_ab_source_file_url":"parquet_tests/sample_decimal.parquet"},"emitted_at":1683668637642} -{"stream":"test","data":{"id":"row2","value": 67.89,"_ab_source_file_last_modified":"2023-08-06T18:05:00Z","_ab_source_file_url":"parquet_tests/sample_decimal.parquet"},"emitted_at":1683668637643} +{"stream":"test","data":{"id":"row1","value": 12.345,"_ab_source_file_last_modified":"2023-08-06T18:05:00.000000Z","_ab_source_file_url":"parquet_tests/sample_decimal.parquet"},"emitted_at":1683668637642} +{"stream":"test","data":{"id":"row2","value": 67.89,"_ab_source_file_last_modified":"2023-08-06T18:05:00.000000Z","_ab_source_file_url":"parquet_tests/sample_decimal.parquet"},"emitted_at":1683668637643} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/parquet.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/parquet.jsonl index f9d66c2d9918..000eac4ff214 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/parquet.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/parquet.jsonl @@ -1,15 +1,15 @@ -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SCHWARTZ","First_Name":"CHANA","Mid_Init":"H","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1046.25,"Regular_Gross_Paid":47316.74,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":8230.31,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668637642} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"WASHINGTON","First_Name":"DOROTHY","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53373,"Regular_Hours":1825,"Regular_Gross_Paid":47436.44,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":1723.17,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668637643} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SAMUEL","First_Name":"GRACE","Mid_Init":"Y","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":55337,"Regular_Hours":1825,"Regular_Gross_Paid":55185.52,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668639019} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BIEBEL","First_Name":"ANN","Mid_Init":"M","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1825,"Regular_Gross_Paid":76804,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640406} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"CARROLL","First_Name":"FRAN","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1825,"Regular_Gross_Paid":76804,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640407} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BROWNSTEIN","First_Name":"ELFREDA","Mid_Init":"G","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":83504,"Regular_Hours":1825,"Regular_Gross_Paid":83275.15,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640407} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"WARD","First_Name":"RENEE","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53373,"Regular_Hours":1825,"Regular_Gross_Paid":46588.76,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":3409.69,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640408} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SPIVEY","First_Name":"NATASHA","Mid_Init":"L","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53436,"Regular_Hours":1825,"Regular_Gross_Paid":53289.6,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640408} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"DU","First_Name":"MARK","Mid_Init":null,"Agency_Start_Date":"03/24/2014","Work_Location_Borough":null,"Title_Description":"HEARING OFFICER","Base_Salary":36.6,"Regular_Hours":188.75,"Regular_Gross_Paid":5334.45,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Hour","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Hour/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668641811} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"THEIL","First_Name":"JOANNE","Mid_Init":"F","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":80438,"Regular_Hours":1825,"Regular_Gross_Paid":80217.55,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13635.42,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643311} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"DEMAIO","First_Name":"DEIRDRE","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53512,"Regular_Hours":1780,"Regular_Gross_Paid":48727.47,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":3318.35,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643311} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"MCLAURIN TRAPP","First_Name":"CELESTINE","Mid_Init":"T","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":58951,"Regular_Hours":1818,"Regular_Gross_Paid":58563.27,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":8.25,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BUNDRANT","First_Name":"TROY","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":64769,"Regular_Hours":1825,"Regular_Gross_Paid":61817.94,"OT_Hours":62,"Total_OT_Paid":2576.58,"Total_Other_Pay":106.68,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"CHASE JONES","First_Name":"DIANA","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":66000,"Regular_Hours":1825,"Regular_Gross_Paid":65819.25,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"JORDAN","First_Name":"REGINALD","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":75000,"Regular_Hours":1825,"Regular_Gross_Paid":74794.46,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SCHWARTZ","First_Name":"CHANA","Mid_Init":"H","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1046.25,"Regular_Gross_Paid":47316.74,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":8230.31,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668637642} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"WASHINGTON","First_Name":"DOROTHY","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53373,"Regular_Hours":1825,"Regular_Gross_Paid":47436.44,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":1723.17,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668637643} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SAMUEL","First_Name":"GRACE","Mid_Init":"Y","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":55337,"Regular_Hours":1825,"Regular_Gross_Paid":55185.52,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668639019} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BIEBEL","First_Name":"ANN","Mid_Init":"M","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1825,"Regular_Gross_Paid":76804,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640406} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"CARROLL","First_Name":"FRAN","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1825,"Regular_Gross_Paid":76804,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640407} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BROWNSTEIN","First_Name":"ELFREDA","Mid_Init":"G","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":83504,"Regular_Hours":1825,"Regular_Gross_Paid":83275.15,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640407} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"WARD","First_Name":"RENEE","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53373,"Regular_Hours":1825,"Regular_Gross_Paid":46588.76,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":3409.69,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640408} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SPIVEY","First_Name":"NATASHA","Mid_Init":"L","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53436,"Regular_Hours":1825,"Regular_Gross_Paid":53289.6,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640408} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"DU","First_Name":"MARK","Mid_Init":null,"Agency_Start_Date":"03/24/2014","Work_Location_Borough":null,"Title_Description":"HEARING OFFICER","Base_Salary":36.6,"Regular_Hours":188.75,"Regular_Gross_Paid":5334.45,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Hour","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Hour/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668641811} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"THEIL","First_Name":"JOANNE","Mid_Init":"F","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":80438,"Regular_Hours":1825,"Regular_Gross_Paid":80217.55,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13635.42,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643311} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"DEMAIO","First_Name":"DEIRDRE","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53512,"Regular_Hours":1780,"Regular_Gross_Paid":48727.47,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":3318.35,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643311} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"MCLAURIN TRAPP","First_Name":"CELESTINE","Mid_Init":"T","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":58951,"Regular_Hours":1818,"Regular_Gross_Paid":58563.27,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":8.25,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BUNDRANT","First_Name":"TROY","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":64769,"Regular_Hours":1825,"Regular_Gross_Paid":61817.94,"OT_Hours":62,"Total_OT_Paid":2576.58,"Total_Other_Pay":106.68,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"CHASE JONES","First_Name":"DIANA","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":66000,"Regular_Hours":1825,"Regular_Gross_Paid":65819.25,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"JORDAN","First_Name":"REGINALD","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":75000,"Regular_Hours":1825,"Regular_Gross_Paid":74794.46,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/parquet_dataset.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/parquet_dataset.jsonl index f9d66c2d9918..000eac4ff214 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/parquet_dataset.jsonl +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/parquet_dataset.jsonl @@ -1,15 +1,15 @@ -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SCHWARTZ","First_Name":"CHANA","Mid_Init":"H","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1046.25,"Regular_Gross_Paid":47316.74,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":8230.31,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668637642} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"WASHINGTON","First_Name":"DOROTHY","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53373,"Regular_Hours":1825,"Regular_Gross_Paid":47436.44,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":1723.17,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668637643} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SAMUEL","First_Name":"GRACE","Mid_Init":"Y","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":55337,"Regular_Hours":1825,"Regular_Gross_Paid":55185.52,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668639019} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BIEBEL","First_Name":"ANN","Mid_Init":"M","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1825,"Regular_Gross_Paid":76804,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640406} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"CARROLL","First_Name":"FRAN","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1825,"Regular_Gross_Paid":76804,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640407} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BROWNSTEIN","First_Name":"ELFREDA","Mid_Init":"G","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":83504,"Regular_Hours":1825,"Regular_Gross_Paid":83275.15,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640407} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"WARD","First_Name":"RENEE","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53373,"Regular_Hours":1825,"Regular_Gross_Paid":46588.76,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":3409.69,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640408} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SPIVEY","First_Name":"NATASHA","Mid_Init":"L","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53436,"Regular_Hours":1825,"Regular_Gross_Paid":53289.6,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640408} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"DU","First_Name":"MARK","Mid_Init":null,"Agency_Start_Date":"03/24/2014","Work_Location_Borough":null,"Title_Description":"HEARING OFFICER","Base_Salary":36.6,"Regular_Hours":188.75,"Regular_Gross_Paid":5334.45,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Hour","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Hour/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668641811} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"THEIL","First_Name":"JOANNE","Mid_Init":"F","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":80438,"Regular_Hours":1825,"Regular_Gross_Paid":80217.55,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13635.42,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643311} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"DEMAIO","First_Name":"DEIRDRE","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53512,"Regular_Hours":1780,"Regular_Gross_Paid":48727.47,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":3318.35,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643311} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"MCLAURIN TRAPP","First_Name":"CELESTINE","Mid_Init":"T","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":58951,"Regular_Hours":1818,"Regular_Gross_Paid":58563.27,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":8.25,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BUNDRANT","First_Name":"TROY","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":64769,"Regular_Hours":1825,"Regular_Gross_Paid":61817.94,"OT_Hours":62,"Total_OT_Paid":2576.58,"Total_Other_Pay":106.68,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"CHASE JONES","First_Name":"DIANA","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":66000,"Regular_Hours":1825,"Regular_Gross_Paid":65819.25,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312} -{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"JORDAN","First_Name":"REGINALD","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":75000,"Regular_Hours":1825,"Regular_Gross_Paid":74794.46,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SCHWARTZ","First_Name":"CHANA","Mid_Init":"H","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1046.25,"Regular_Gross_Paid":47316.74,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":8230.31,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668637642} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"WASHINGTON","First_Name":"DOROTHY","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53373,"Regular_Hours":1825,"Regular_Gross_Paid":47436.44,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":1723.17,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668637643} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SAMUEL","First_Name":"GRACE","Mid_Init":"Y","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":55337,"Regular_Hours":1825,"Regular_Gross_Paid":55185.52,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ON LEAVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:28.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ON%20LEAVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668639019} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BIEBEL","First_Name":"ANN","Mid_Init":"M","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1825,"Regular_Gross_Paid":76804,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640406} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"CARROLL","First_Name":"FRAN","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":77015,"Regular_Hours":1825,"Regular_Gross_Paid":76804,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640407} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BROWNSTEIN","First_Name":"ELFREDA","Mid_Init":"G","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":83504,"Regular_Hours":1825,"Regular_Gross_Paid":83275.15,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13750.36,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640407} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"WARD","First_Name":"RENEE","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53373,"Regular_Hours":1825,"Regular_Gross_Paid":46588.76,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":3409.69,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640408} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"SPIVEY","First_Name":"NATASHA","Mid_Init":"L","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53436,"Regular_Hours":1825,"Regular_Gross_Paid":53289.6,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668640408} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"DU","First_Name":"MARK","Mid_Init":null,"Agency_Start_Date":"03/24/2014","Work_Location_Borough":null,"Title_Description":"HEARING OFFICER","Base_Salary":36.6,"Regular_Hours":188.75,"Regular_Gross_Paid":5334.45,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2021","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Hour","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2021/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Hour/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668641811} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"THEIL","First_Name":"JOANNE","Mid_Init":"F","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"*ATTORNEY AT LAW","Base_Salary":80438,"Regular_Hours":1825,"Regular_Gross_Paid":80217.55,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":13635.42,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643311} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"DEMAIO","First_Name":"DEIRDRE","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":53512,"Regular_Hours":1780,"Regular_Gross_Paid":48727.47,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":3318.35,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643311} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"MCLAURIN TRAPP","First_Name":"CELESTINE","Mid_Init":"T","Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":58951,"Regular_Hours":1818,"Regular_Gross_Paid":58563.27,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":8.25,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"BUNDRANT","First_Name":"TROY","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":64769,"Regular_Hours":1825,"Regular_Gross_Paid":61817.94,"OT_Hours":62,"Total_OT_Paid":2576.58,"Total_Other_Pay":106.68,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"CHASE JONES","First_Name":"DIANA","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":66000,"Regular_Hours":1825,"Regular_Gross_Paid":65819.25,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312} +{"stream":"test","data":{"Payroll_Number":820,"Last_Name":"JORDAN","First_Name":"REGINALD","Mid_Init":null,"Agency_Start_Date":"07/05/2010","Work_Location_Borough":null,"Title_Description":"ADM MANAGER-NON-MGRL FROM M1/M2","Base_Salary":75000,"Regular_Hours":1825,"Regular_Gross_Paid":74794.46,"OT_Hours":0,"Total_OT_Paid":0,"Total_Other_Pay":0,"Fiscal_Year":"2022","Leave_Status_as_of_June_30":"ACTIVE","Pay_Basis":"per Annum","_ab_source_file_last_modified":"2023-05-09T20:16:29.000000Z","_ab_source_file_url":"test_payroll/Fiscal_Year=2022/Leave_Status_as_of_June_30=ACTIVE/Pay_Basis=per%20Annum/4e0ea65c5a074c0592e43f7b950f3ce8-0.parquet"},"emitted_at":1683668643312} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/unstructured.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/unstructured.jsonl new file mode 100644 index 000000000000..539daeeb36e4 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/unstructured.jsonl @@ -0,0 +1,2 @@ +{"stream": "test", "data": {"document_key": "Testdoc.pdf", "content": "# Heading\n\nThis is the content which is not just a single word", "_ab_source_file_last_modified": "2023-10-20T12:52:38.000000Z", "_ab_source_file_url": "Testdoc.pdf", "_ab_source_file_parse_error": null}, "emitted_at": 162727468000} +{"stream": "test", "data": {"document_key": "Testdoc_OCR.pdf", "content": "This is a test", "_ab_source_file_last_modified": "2023-10-23T10:55:37.000000Z", "_ab_source_file_url": "Testdoc_OCR.pdf", "_ab_source_file_parse_error": null}, "emitted_at": 162727468000} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/zip_avro.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/zip_avro.jsonl new file mode 100644 index 000000000000..113492d6b3d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/zip_avro.jsonl @@ -0,0 +1,20 @@ +{"stream": "test", "data": {"id": 0, "fullname_and_valid": {"fullname": "cfjwIzCRTL", "valid": false}, "_ab_source_file_last_modified": "2023-10-12T14:47:44.000000Z", "_ab_source_file_url": "zip_tests/zip_avro.zip#zip_avro_2.avro"}, "emitted_at": 1697145679798} +{"stream": "test", "data": {"id": 1, "fullname_and_valid": {"fullname": "LYOnPyuTWw", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T14:47:44.000000Z", "_ab_source_file_url": "zip_tests/zip_avro.zip#zip_avro_2.avro"}, "emitted_at": 1697145679800} +{"stream": "test", "data": {"id": 2, "fullname_and_valid": {"fullname": "hyTFbsxlRB", "valid": false}, "_ab_source_file_last_modified": "2023-10-12T14:47:44.000000Z", "_ab_source_file_url": "zip_tests/zip_avro.zip#zip_avro_2.avro"}, "emitted_at": 1697145679800} +{"stream": "test", "data": {"id": 3, "fullname_and_valid": {"fullname": "ooEUiFcFqp", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T14:47:44.000000Z", "_ab_source_file_url": "zip_tests/zip_avro.zip#zip_avro_2.avro"}, "emitted_at": 1697145679800} +{"stream": "test", "data": {"id": 4, "fullname_and_valid": {"fullname": "pveENwAvOg", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T14:47:44.000000Z", "_ab_source_file_url": "zip_tests/zip_avro.zip#zip_avro_2.avro"}, "emitted_at": 1697145679801} +{"stream": "test", "data": {"id": 5, "fullname_and_valid": {"fullname": "pPhWgQgZFq", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T14:47:44.000000Z", "_ab_source_file_url": "zip_tests/zip_avro.zip#zip_avro_2.avro"}, "emitted_at": 1697145679801} +{"stream": "test", "data": {"id": 6, "fullname_and_valid": {"fullname": "MRNMXFkXZo", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T14:47:44.000000Z", "_ab_source_file_url": "zip_tests/zip_avro.zip#zip_avro_2.avro"}, "emitted_at": 1697145679801} +{"stream": "test", "data": {"id": 7, "fullname_and_valid": {"fullname": "MXvEWMgnIr", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T14:47:44.000000Z", "_ab_source_file_url": "zip_tests/zip_avro.zip#zip_avro_2.avro"}, "emitted_at": 1697145679801} +{"stream": "test", "data": {"id": 8, "fullname_and_valid": {"fullname": "rqmFGqZqdF", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T14:47:44.000000Z", "_ab_source_file_url": "zip_tests/zip_avro.zip#zip_avro_2.avro"}, "emitted_at": 1697145679802} +{"stream": "test", "data": {"id": 9, "fullname_and_valid": {"fullname": "lmPpQTcPFM", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T14:47:44.000000Z", "_ab_source_file_url": "zip_tests/zip_avro.zip#zip_avro_2.avro"}, "emitted_at": 1697145679802} +{"stream": "test", "data": {"id": 0, "fullname_and_valid": {"fullname": "NewFullName1", "valid": false}, "_ab_source_file_last_modified": "2023-10-12T20:21:20.000000Z", "_ab_source_file_url": "zip_tests/zip_avro.zip#zip_avro_1.avro"}, "emitted_at": 1697145680288} +{"stream": "test", "data": {"id": 1, "fullname_and_valid": {"fullname": "NewFullName2", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T20:21:20.000000Z", "_ab_source_file_url": "zip_tests/zip_avro.zip#zip_avro_1.avro"}, "emitted_at": 1697145680288} +{"stream": "test", "data": {"id": 2, "fullname_and_valid": {"fullname": "NewFullName3", "valid": false}, "_ab_source_file_last_modified": "2023-10-12T20:21:20.000000Z", "_ab_source_file_url": "zip_tests/zip_avro.zip#zip_avro_1.avro"}, "emitted_at": 1697145680289} +{"stream": "test", "data": {"id": 3, "fullname_and_valid": {"fullname": "NewFullName4", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T20:21:20.000000Z", "_ab_source_file_url": "zip_tests/zip_avro.zip#zip_avro_1.avro"}, "emitted_at": 1697145680289} +{"stream": "test", "data": {"id": 4, "fullname_and_valid": {"fullname": "NewFullName5", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T20:21:20.000000Z", "_ab_source_file_url": "zip_tests/zip_avro.zip#zip_avro_1.avro"}, "emitted_at": 1697145680289} +{"stream": "test", "data": {"id": 5, "fullname_and_valid": {"fullname": "NewFullName6", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T20:21:20.000000Z", "_ab_source_file_url": "zip_tests/zip_avro.zip#zip_avro_1.avro"}, "emitted_at": 1697145680289} +{"stream": "test", "data": {"id": 6, "fullname_and_valid": {"fullname": "NewFullName7", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T20:21:20.000000Z", "_ab_source_file_url": "zip_tests/zip_avro.zip#zip_avro_1.avro"}, "emitted_at": 1697145680289} +{"stream": "test", "data": {"id": 7, "fullname_and_valid": {"fullname": "NewFullName8", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T20:21:20.000000Z", "_ab_source_file_url": "zip_tests/zip_avro.zip#zip_avro_1.avro"}, "emitted_at": 1697145680289} +{"stream": "test", "data": {"id": 8, "fullname_and_valid": {"fullname": "NewFullName9", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T20:21:20.000000Z", "_ab_source_file_url": "zip_tests/zip_avro.zip#zip_avro_1.avro"}, "emitted_at": 1697145680289} +{"stream": "test", "data": {"id": 9, "fullname_and_valid": {"fullname": "NewFullName10", "valid": true}, "_ab_source_file_last_modified": "2023-10-12T20:21:20.000000Z", "_ab_source_file_url": "zip_tests/zip_avro.zip#zip_avro_1.avro"}, "emitted_at": 1697145680289} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/zip_csv.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/zip_csv.jsonl new file mode 100644 index 000000000000..d1b8e1362fa3 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/zip_csv.jsonl @@ -0,0 +1,400 @@ +{"stream": "test", "data": {"id": 0, "name": "ezFAa", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337389} +{"stream": "test", "data": {"id": 1, "name": "KEjdd", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337391} +{"stream": "test", "data": {"id": 2, "name": "pKyJp", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337391} +{"stream": "test", "data": {"id": 3, "name": "MWsBa", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337392} +{"stream": "test", "data": {"id": 4, "name": "EbnKo", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337392} +{"stream": "test", "data": {"id": 5, "name": "FnTdW", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337392} +{"stream": "test", "data": {"id": 6, "name": "iFTcr", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337393} +{"stream": "test", "data": {"id": 7, "name": "rkkpY", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337393} +{"stream": "test", "data": {"id": 8, "name": "jsIUF", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337394} +{"stream": "test", "data": {"id": 9, "name": "IGCfY", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337394} +{"stream": "test", "data": {"id": 10, "name": "VkJsy", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337394} +{"stream": "test", "data": {"id": 11, "name": "geWdF", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337395} +{"stream": "test", "data": {"id": 12, "name": "VMwdf", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337395} +{"stream": "test", "data": {"id": 13, "name": "DOSLy", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337396} +{"stream": "test", "data": {"id": 14, "name": "GUHGw", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337396} +{"stream": "test", "data": {"id": 15, "name": "NJELC", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337396} +{"stream": "test", "data": {"id": 16, "name": "pQcXD", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337397} +{"stream": "test", "data": {"id": 17, "name": "ZumnL", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337397} +{"stream": "test", "data": {"id": 18, "name": "RBZVT", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337398} +{"stream": "test", "data": {"id": 19, "name": "kSdIW", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337398} +{"stream": "test", "data": {"id": 20, "name": "NIEkh", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337398} +{"stream": "test", "data": {"id": 21, "name": "oGdkD", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337399} +{"stream": "test", "data": {"id": 22, "name": "twreK", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337399} +{"stream": "test", "data": {"id": 23, "name": "wFnGG", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337399} +{"stream": "test", "data": {"id": 24, "name": "HRWjO", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337400} +{"stream": "test", "data": {"id": 25, "name": "xxAGQ", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337400} +{"stream": "test", "data": {"id": 26, "name": "HiXkr", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337400} +{"stream": "test", "data": {"id": 27, "name": "WhjaB", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337401} +{"stream": "test", "data": {"id": 28, "name": "XMHZl", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337401} +{"stream": "test", "data": {"id": 29, "name": "DBOrr", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337401} +{"stream": "test", "data": {"id": 30, "name": "mwcUr", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337402} +{"stream": "test", "data": {"id": 31, "name": "ZFZDp", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337402} +{"stream": "test", "data": {"id": 32, "name": "WMeEn", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337403} +{"stream": "test", "data": {"id": 33, "name": "pPPKR", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337403} +{"stream": "test", "data": {"id": 34, "name": "igttH", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337403} +{"stream": "test", "data": {"id": 35, "name": "XtOYX", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337404} +{"stream": "test", "data": {"id": 36, "name": "bbMyb", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337404} +{"stream": "test", "data": {"id": 37, "name": "vQvhY", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337404} +{"stream": "test", "data": {"id": 38, "name": "vvFWw", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337404} +{"stream": "test", "data": {"id": 39, "name": "xZPnu", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337405} +{"stream": "test", "data": {"id": 40, "name": "nTWsz", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337405} +{"stream": "test", "data": {"id": 41, "name": "hruFx", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337405} +{"stream": "test", "data": {"id": 42, "name": "BAYps", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337405} +{"stream": "test", "data": {"id": 43, "name": "YSUbF", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337406} +{"stream": "test", "data": {"id": 44, "name": "ginIc", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337406} +{"stream": "test", "data": {"id": 45, "name": "Sqwbc", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337406} +{"stream": "test", "data": {"id": 46, "name": "TlIVn", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337406} +{"stream": "test", "data": {"id": 47, "name": "PfPKQ", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337407} +{"stream": "test", "data": {"id": 48, "name": "vNekT", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337407} +{"stream": "test", "data": {"id": 49, "name": "Dpnvl", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337407} +{"stream": "test", "data": {"id": 50, "name": "HkxrY", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337407} +{"stream": "test", "data": {"id": 51, "name": "ZUzCq", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337407} +{"stream": "test", "data": {"id": 52, "name": "qVoaC", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337408} +{"stream": "test", "data": {"id": 53, "name": "IaKUC", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337408} +{"stream": "test", "data": {"id": 54, "name": "hOTBB", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337408} +{"stream": "test", "data": {"id": 55, "name": "WijBV", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337408} +{"stream": "test", "data": {"id": 56, "name": "vXTWL", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337408} +{"stream": "test", "data": {"id": 57, "name": "eDGcy", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337409} +{"stream": "test", "data": {"id": 58, "name": "GUAJj", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337409} +{"stream": "test", "data": {"id": 59, "name": "AXSUJ", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337409} +{"stream": "test", "data": {"id": 60, "name": "AxYuv", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337409} +{"stream": "test", "data": {"id": 61, "name": "JvMCP", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337409} +{"stream": "test", "data": {"id": 62, "name": "AzVBn", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337409} +{"stream": "test", "data": {"id": 63, "name": "RRgGz", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337410} +{"stream": "test", "data": {"id": 64, "name": "jLROq", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337410} +{"stream": "test", "data": {"id": 65, "name": "kvzOm", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337410} +{"stream": "test", "data": {"id": 66, "name": "qERMw", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337410} +{"stream": "test", "data": {"id": 67, "name": "cxhzS", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337410} +{"stream": "test", "data": {"id": 68, "name": "sMGCQ", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337410} +{"stream": "test", "data": {"id": 69, "name": "zNEYF", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337410} +{"stream": "test", "data": {"id": 70, "name": "WoXDc", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337411} +{"stream": "test", "data": {"id": 71, "name": "nylfK", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337411} +{"stream": "test", "data": {"id": 72, "name": "iQIlD", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337411} +{"stream": "test", "data": {"id": 73, "name": "fnXPy", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337411} +{"stream": "test", "data": {"id": 74, "name": "Iwlbb", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337411} +{"stream": "test", "data": {"id": 75, "name": "ivixl", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337411} +{"stream": "test", "data": {"id": 76, "name": "aqiOE", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337411} +{"stream": "test", "data": {"id": 77, "name": "IGqnq", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337411} +{"stream": "test", "data": {"id": 78, "name": "Qplhs", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337412} +{"stream": "test", "data": {"id": 79, "name": "SuCoC", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337412} +{"stream": "test", "data": {"id": 80, "name": "QuYXW", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337412} +{"stream": "test", "data": {"id": 81, "name": "SCfiE", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337412} +{"stream": "test", "data": {"id": 82, "name": "dNssE", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337412} +{"stream": "test", "data": {"id": 83, "name": "njKVz", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337412} +{"stream": "test", "data": {"id": 84, "name": "akphf", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337412} +{"stream": "test", "data": {"id": 85, "name": "tHpba", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337412} +{"stream": "test", "data": {"id": 86, "name": "RBqgU", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337412} +{"stream": "test", "data": {"id": 87, "name": "DZdtO", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337413} +{"stream": "test", "data": {"id": 88, "name": "XglaW", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337413} +{"stream": "test", "data": {"id": 89, "name": "zUOOs", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337413} +{"stream": "test", "data": {"id": 90, "name": "jcSbl", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337413} +{"stream": "test", "data": {"id": 91, "name": "UkEkk", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337413} +{"stream": "test", "data": {"id": 92, "name": "VKmlR", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337413} +{"stream": "test", "data": {"id": 93, "name": "SfXTU", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337413} +{"stream": "test", "data": {"id": 94, "name": "xcmCK", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337413} +{"stream": "test", "data": {"id": 95, "name": "tbKIy", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337413} +{"stream": "test", "data": {"id": 96, "name": "NZdRl", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337413} +{"stream": "test", "data": {"id": 97, "name": "sfnoo", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337414} +{"stream": "test", "data": {"id": 98, "name": "nvEDZ", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337414} +{"stream": "test", "data": {"id": 99, "name": "DUVlK", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_bzip2.zip#test_zip.csv"}, "emitted_at": 1697140337414} +{"stream": "test", "data": {"id": 0, "name": "ezFAa", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337992} +{"stream": "test", "data": {"id": 1, "name": "KEjdd", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337993} +{"stream": "test", "data": {"id": 2, "name": "pKyJp", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337994} +{"stream": "test", "data": {"id": 3, "name": "MWsBa", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337994} +{"stream": "test", "data": {"id": 4, "name": "EbnKo", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337994} +{"stream": "test", "data": {"id": 5, "name": "FnTdW", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337994} +{"stream": "test", "data": {"id": 6, "name": "iFTcr", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337994} +{"stream": "test", "data": {"id": 7, "name": "rkkpY", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337995} +{"stream": "test", "data": {"id": 8, "name": "jsIUF", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337995} +{"stream": "test", "data": {"id": 9, "name": "IGCfY", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337995} +{"stream": "test", "data": {"id": 10, "name": "VkJsy", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337995} +{"stream": "test", "data": {"id": 11, "name": "geWdF", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337995} +{"stream": "test", "data": {"id": 12, "name": "VMwdf", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337995} +{"stream": "test", "data": {"id": 13, "name": "DOSLy", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337995} +{"stream": "test", "data": {"id": 14, "name": "GUHGw", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337995} +{"stream": "test", "data": {"id": 15, "name": "NJELC", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337995} +{"stream": "test", "data": {"id": 16, "name": "pQcXD", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337995} +{"stream": "test", "data": {"id": 17, "name": "ZumnL", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337996} +{"stream": "test", "data": {"id": 18, "name": "RBZVT", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337996} +{"stream": "test", "data": {"id": 19, "name": "kSdIW", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337996} +{"stream": "test", "data": {"id": 20, "name": "NIEkh", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337996} +{"stream": "test", "data": {"id": 21, "name": "oGdkD", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337996} +{"stream": "test", "data": {"id": 22, "name": "twreK", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337996} +{"stream": "test", "data": {"id": 23, "name": "wFnGG", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337996} +{"stream": "test", "data": {"id": 24, "name": "HRWjO", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337996} +{"stream": "test", "data": {"id": 25, "name": "xxAGQ", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337996} +{"stream": "test", "data": {"id": 26, "name": "HiXkr", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337997} +{"stream": "test", "data": {"id": 27, "name": "WhjaB", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337997} +{"stream": "test", "data": {"id": 28, "name": "XMHZl", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337997} +{"stream": "test", "data": {"id": 29, "name": "DBOrr", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337997} +{"stream": "test", "data": {"id": 30, "name": "mwcUr", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337997} +{"stream": "test", "data": {"id": 31, "name": "ZFZDp", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337997} +{"stream": "test", "data": {"id": 32, "name": "WMeEn", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337997} +{"stream": "test", "data": {"id": 33, "name": "pPPKR", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337997} +{"stream": "test", "data": {"id": 34, "name": "igttH", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337998} +{"stream": "test", "data": {"id": 35, "name": "XtOYX", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337998} +{"stream": "test", "data": {"id": 36, "name": "bbMyb", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337998} +{"stream": "test", "data": {"id": 37, "name": "vQvhY", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337998} +{"stream": "test", "data": {"id": 38, "name": "vvFWw", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337998} +{"stream": "test", "data": {"id": 39, "name": "xZPnu", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337998} +{"stream": "test", "data": {"id": 40, "name": "nTWsz", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337998} +{"stream": "test", "data": {"id": 41, "name": "hruFx", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337998} +{"stream": "test", "data": {"id": 42, "name": "BAYps", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337999} +{"stream": "test", "data": {"id": 43, "name": "YSUbF", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337999} +{"stream": "test", "data": {"id": 44, "name": "ginIc", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337999} +{"stream": "test", "data": {"id": 45, "name": "Sqwbc", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337999} +{"stream": "test", "data": {"id": 46, "name": "TlIVn", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337999} +{"stream": "test", "data": {"id": 47, "name": "PfPKQ", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337999} +{"stream": "test", "data": {"id": 48, "name": "vNekT", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337999} +{"stream": "test", "data": {"id": 49, "name": "Dpnvl", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337999} +{"stream": "test", "data": {"id": 50, "name": "HkxrY", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337999} +{"stream": "test", "data": {"id": 51, "name": "ZUzCq", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140337999} +{"stream": "test", "data": {"id": 52, "name": "qVoaC", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338000} +{"stream": "test", "data": {"id": 53, "name": "IaKUC", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338000} +{"stream": "test", "data": {"id": 54, "name": "hOTBB", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338000} +{"stream": "test", "data": {"id": 55, "name": "WijBV", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338000} +{"stream": "test", "data": {"id": 56, "name": "vXTWL", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338000} +{"stream": "test", "data": {"id": 57, "name": "eDGcy", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338000} +{"stream": "test", "data": {"id": 58, "name": "GUAJj", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338000} +{"stream": "test", "data": {"id": 59, "name": "AXSUJ", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338000} +{"stream": "test", "data": {"id": 60, "name": "AxYuv", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338000} +{"stream": "test", "data": {"id": 61, "name": "JvMCP", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338000} +{"stream": "test", "data": {"id": 62, "name": "AzVBn", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338001} +{"stream": "test", "data": {"id": 63, "name": "RRgGz", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338001} +{"stream": "test", "data": {"id": 64, "name": "jLROq", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338001} +{"stream": "test", "data": {"id": 65, "name": "kvzOm", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338001} +{"stream": "test", "data": {"id": 66, "name": "qERMw", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338001} +{"stream": "test", "data": {"id": 67, "name": "cxhzS", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338001} +{"stream": "test", "data": {"id": 68, "name": "sMGCQ", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338001} +{"stream": "test", "data": {"id": 69, "name": "zNEYF", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338001} +{"stream": "test", "data": {"id": 70, "name": "WoXDc", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338001} +{"stream": "test", "data": {"id": 71, "name": "nylfK", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338001} +{"stream": "test", "data": {"id": 72, "name": "iQIlD", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338002} +{"stream": "test", "data": {"id": 73, "name": "fnXPy", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338002} +{"stream": "test", "data": {"id": 74, "name": "Iwlbb", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338002} +{"stream": "test", "data": {"id": 75, "name": "ivixl", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338002} +{"stream": "test", "data": {"id": 76, "name": "aqiOE", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338002} +{"stream": "test", "data": {"id": 77, "name": "IGqnq", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338002} +{"stream": "test", "data": {"id": 78, "name": "Qplhs", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338002} +{"stream": "test", "data": {"id": 79, "name": "SuCoC", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338003} +{"stream": "test", "data": {"id": 80, "name": "QuYXW", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338003} +{"stream": "test", "data": {"id": 81, "name": "SCfiE", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338003} +{"stream": "test", "data": {"id": 82, "name": "dNssE", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338003} +{"stream": "test", "data": {"id": 83, "name": "njKVz", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338003} +{"stream": "test", "data": {"id": 84, "name": "akphf", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338003} +{"stream": "test", "data": {"id": 85, "name": "tHpba", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338004} +{"stream": "test", "data": {"id": 86, "name": "RBqgU", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338004} +{"stream": "test", "data": {"id": 87, "name": "DZdtO", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338004} +{"stream": "test", "data": {"id": 88, "name": "XglaW", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338004} +{"stream": "test", "data": {"id": 89, "name": "zUOOs", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338004} +{"stream": "test", "data": {"id": 90, "name": "jcSbl", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338004} +{"stream": "test", "data": {"id": 91, "name": "UkEkk", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338005} +{"stream": "test", "data": {"id": 92, "name": "VKmlR", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338005} +{"stream": "test", "data": {"id": 93, "name": "SfXTU", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338005} +{"stream": "test", "data": {"id": 94, "name": "xcmCK", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338005} +{"stream": "test", "data": {"id": 95, "name": "tbKIy", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338005} +{"stream": "test", "data": {"id": 96, "name": "NZdRl", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338005} +{"stream": "test", "data": {"id": 97, "name": "sfnoo", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338005} +{"stream": "test", "data": {"id": 98, "name": "nvEDZ", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338006} +{"stream": "test", "data": {"id": 99, "name": "DUVlK", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_deflated.zip#test_zip.csv"}, "emitted_at": 1697140338006} +{"stream": "test", "data": {"id": 0, "name": "ezFAa", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338570} +{"stream": "test", "data": {"id": 1, "name": "KEjdd", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338571} +{"stream": "test", "data": {"id": 2, "name": "pKyJp", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338571} +{"stream": "test", "data": {"id": 3, "name": "MWsBa", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338572} +{"stream": "test", "data": {"id": 4, "name": "EbnKo", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338572} +{"stream": "test", "data": {"id": 5, "name": "FnTdW", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338573} +{"stream": "test", "data": {"id": 6, "name": "iFTcr", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338573} +{"stream": "test", "data": {"id": 7, "name": "rkkpY", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338573} +{"stream": "test", "data": {"id": 8, "name": "jsIUF", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338574} +{"stream": "test", "data": {"id": 9, "name": "IGCfY", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338574} +{"stream": "test", "data": {"id": 10, "name": "VkJsy", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338575} +{"stream": "test", "data": {"id": 11, "name": "geWdF", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338575} +{"stream": "test", "data": {"id": 12, "name": "VMwdf", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338575} +{"stream": "test", "data": {"id": 13, "name": "DOSLy", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338576} +{"stream": "test", "data": {"id": 14, "name": "GUHGw", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338576} +{"stream": "test", "data": {"id": 15, "name": "NJELC", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338577} +{"stream": "test", "data": {"id": 16, "name": "pQcXD", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338577} +{"stream": "test", "data": {"id": 17, "name": "ZumnL", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338577} +{"stream": "test", "data": {"id": 18, "name": "RBZVT", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338578} +{"stream": "test", "data": {"id": 19, "name": "kSdIW", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338578} +{"stream": "test", "data": {"id": 20, "name": "NIEkh", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338579} +{"stream": "test", "data": {"id": 21, "name": "oGdkD", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338579} +{"stream": "test", "data": {"id": 22, "name": "twreK", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338580} +{"stream": "test", "data": {"id": 23, "name": "wFnGG", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338580} +{"stream": "test", "data": {"id": 24, "name": "HRWjO", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338580} +{"stream": "test", "data": {"id": 25, "name": "xxAGQ", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338581} +{"stream": "test", "data": {"id": 26, "name": "HiXkr", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338581} +{"stream": "test", "data": {"id": 27, "name": "WhjaB", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338581} +{"stream": "test", "data": {"id": 28, "name": "XMHZl", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338582} +{"stream": "test", "data": {"id": 29, "name": "DBOrr", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338582} +{"stream": "test", "data": {"id": 30, "name": "mwcUr", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338583} +{"stream": "test", "data": {"id": 31, "name": "ZFZDp", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338583} +{"stream": "test", "data": {"id": 32, "name": "WMeEn", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338583} +{"stream": "test", "data": {"id": 33, "name": "pPPKR", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338584} +{"stream": "test", "data": {"id": 34, "name": "igttH", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338584} +{"stream": "test", "data": {"id": 35, "name": "XtOYX", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338584} +{"stream": "test", "data": {"id": 36, "name": "bbMyb", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338584} +{"stream": "test", "data": {"id": 37, "name": "vQvhY", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338585} +{"stream": "test", "data": {"id": 38, "name": "vvFWw", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338585} +{"stream": "test", "data": {"id": 39, "name": "xZPnu", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338585} +{"stream": "test", "data": {"id": 40, "name": "nTWsz", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338586} +{"stream": "test", "data": {"id": 41, "name": "hruFx", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338586} +{"stream": "test", "data": {"id": 42, "name": "BAYps", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338586} +{"stream": "test", "data": {"id": 43, "name": "YSUbF", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338586} +{"stream": "test", "data": {"id": 44, "name": "ginIc", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338587} +{"stream": "test", "data": {"id": 45, "name": "Sqwbc", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338587} +{"stream": "test", "data": {"id": 46, "name": "TlIVn", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338587} +{"stream": "test", "data": {"id": 47, "name": "PfPKQ", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338587} +{"stream": "test", "data": {"id": 48, "name": "vNekT", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338588} +{"stream": "test", "data": {"id": 49, "name": "Dpnvl", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338588} +{"stream": "test", "data": {"id": 50, "name": "HkxrY", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338588} +{"stream": "test", "data": {"id": 51, "name": "ZUzCq", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338588} +{"stream": "test", "data": {"id": 52, "name": "qVoaC", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338588} +{"stream": "test", "data": {"id": 53, "name": "IaKUC", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338589} +{"stream": "test", "data": {"id": 54, "name": "hOTBB", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338589} +{"stream": "test", "data": {"id": 55, "name": "WijBV", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338589} +{"stream": "test", "data": {"id": 56, "name": "vXTWL", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338589} +{"stream": "test", "data": {"id": 57, "name": "eDGcy", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338589} +{"stream": "test", "data": {"id": 58, "name": "GUAJj", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338590} +{"stream": "test", "data": {"id": 59, "name": "AXSUJ", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338590} +{"stream": "test", "data": {"id": 60, "name": "AxYuv", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338590} +{"stream": "test", "data": {"id": 61, "name": "JvMCP", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338590} +{"stream": "test", "data": {"id": 62, "name": "AzVBn", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338590} +{"stream": "test", "data": {"id": 63, "name": "RRgGz", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338590} +{"stream": "test", "data": {"id": 64, "name": "jLROq", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338591} +{"stream": "test", "data": {"id": 65, "name": "kvzOm", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338591} +{"stream": "test", "data": {"id": 66, "name": "qERMw", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338591} +{"stream": "test", "data": {"id": 67, "name": "cxhzS", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338591} +{"stream": "test", "data": {"id": 68, "name": "sMGCQ", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338591} +{"stream": "test", "data": {"id": 69, "name": "zNEYF", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338591} +{"stream": "test", "data": {"id": 70, "name": "WoXDc", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338592} +{"stream": "test", "data": {"id": 71, "name": "nylfK", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338592} +{"stream": "test", "data": {"id": 72, "name": "iQIlD", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338592} +{"stream": "test", "data": {"id": 73, "name": "fnXPy", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338592} +{"stream": "test", "data": {"id": 74, "name": "Iwlbb", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338592} +{"stream": "test", "data": {"id": 75, "name": "ivixl", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338592} +{"stream": "test", "data": {"id": 76, "name": "aqiOE", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338592} +{"stream": "test", "data": {"id": 77, "name": "IGqnq", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338592} +{"stream": "test", "data": {"id": 78, "name": "Qplhs", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338592} +{"stream": "test", "data": {"id": 79, "name": "SuCoC", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338593} +{"stream": "test", "data": {"id": 80, "name": "QuYXW", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338593} +{"stream": "test", "data": {"id": 81, "name": "SCfiE", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338593} +{"stream": "test", "data": {"id": 82, "name": "dNssE", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338593} +{"stream": "test", "data": {"id": 83, "name": "njKVz", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338593} +{"stream": "test", "data": {"id": 84, "name": "akphf", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338593} +{"stream": "test", "data": {"id": 85, "name": "tHpba", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338593} +{"stream": "test", "data": {"id": 86, "name": "RBqgU", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338593} +{"stream": "test", "data": {"id": 87, "name": "DZdtO", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338593} +{"stream": "test", "data": {"id": 88, "name": "XglaW", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338593} +{"stream": "test", "data": {"id": 89, "name": "zUOOs", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338594} +{"stream": "test", "data": {"id": 90, "name": "jcSbl", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338594} +{"stream": "test", "data": {"id": 91, "name": "UkEkk", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338594} +{"stream": "test", "data": {"id": 92, "name": "VKmlR", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338594} +{"stream": "test", "data": {"id": 93, "name": "SfXTU", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338594} +{"stream": "test", "data": {"id": 94, "name": "xcmCK", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338594} +{"stream": "test", "data": {"id": 95, "name": "tbKIy", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338594} +{"stream": "test", "data": {"id": 96, "name": "NZdRl", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338594} +{"stream": "test", "data": {"id": 97, "name": "sfnoo", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338594} +{"stream": "test", "data": {"id": 98, "name": "nvEDZ", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338595} +{"stream": "test", "data": {"id": 99, "name": "DUVlK", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_lzma.zip#test_zip.csv"}, "emitted_at": 1697140338595} +{"stream": "test", "data": {"id": 0, "name": "ezFAa", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339173} +{"stream": "test", "data": {"id": 1, "name": "KEjdd", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339174} +{"stream": "test", "data": {"id": 2, "name": "pKyJp", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339174} +{"stream": "test", "data": {"id": 3, "name": "MWsBa", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339175} +{"stream": "test", "data": {"id": 4, "name": "EbnKo", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339175} +{"stream": "test", "data": {"id": 5, "name": "FnTdW", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339175} +{"stream": "test", "data": {"id": 6, "name": "iFTcr", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339176} +{"stream": "test", "data": {"id": 7, "name": "rkkpY", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339176} +{"stream": "test", "data": {"id": 8, "name": "jsIUF", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339177} +{"stream": "test", "data": {"id": 9, "name": "IGCfY", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339177} +{"stream": "test", "data": {"id": 10, "name": "VkJsy", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339177} +{"stream": "test", "data": {"id": 11, "name": "geWdF", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339178} +{"stream": "test", "data": {"id": 12, "name": "VMwdf", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339178} +{"stream": "test", "data": {"id": 13, "name": "DOSLy", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339179} +{"stream": "test", "data": {"id": 14, "name": "GUHGw", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339179} +{"stream": "test", "data": {"id": 15, "name": "NJELC", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339180} +{"stream": "test", "data": {"id": 16, "name": "pQcXD", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339180} +{"stream": "test", "data": {"id": 17, "name": "ZumnL", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339180} +{"stream": "test", "data": {"id": 18, "name": "RBZVT", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339181} +{"stream": "test", "data": {"id": 19, "name": "kSdIW", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339181} +{"stream": "test", "data": {"id": 20, "name": "NIEkh", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339181} +{"stream": "test", "data": {"id": 21, "name": "oGdkD", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339182} +{"stream": "test", "data": {"id": 22, "name": "twreK", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339182} +{"stream": "test", "data": {"id": 23, "name": "wFnGG", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339183} +{"stream": "test", "data": {"id": 24, "name": "HRWjO", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339183} +{"stream": "test", "data": {"id": 25, "name": "xxAGQ", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339184} +{"stream": "test", "data": {"id": 26, "name": "HiXkr", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339185} +{"stream": "test", "data": {"id": 27, "name": "WhjaB", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339185} +{"stream": "test", "data": {"id": 28, "name": "XMHZl", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339185} +{"stream": "test", "data": {"id": 29, "name": "DBOrr", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339186} +{"stream": "test", "data": {"id": 30, "name": "mwcUr", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339186} +{"stream": "test", "data": {"id": 31, "name": "ZFZDp", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339186} +{"stream": "test", "data": {"id": 32, "name": "WMeEn", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339187} +{"stream": "test", "data": {"id": 33, "name": "pPPKR", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339187} +{"stream": "test", "data": {"id": 34, "name": "igttH", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339188} +{"stream": "test", "data": {"id": 35, "name": "XtOYX", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339188} +{"stream": "test", "data": {"id": 36, "name": "bbMyb", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339188} +{"stream": "test", "data": {"id": 37, "name": "vQvhY", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339189} +{"stream": "test", "data": {"id": 38, "name": "vvFWw", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339189} +{"stream": "test", "data": {"id": 39, "name": "xZPnu", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339189} +{"stream": "test", "data": {"id": 40, "name": "nTWsz", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339189} +{"stream": "test", "data": {"id": 41, "name": "hruFx", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339190} +{"stream": "test", "data": {"id": 42, "name": "BAYps", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339190} +{"stream": "test", "data": {"id": 43, "name": "YSUbF", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339190} +{"stream": "test", "data": {"id": 44, "name": "ginIc", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339191} +{"stream": "test", "data": {"id": 45, "name": "Sqwbc", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339191} +{"stream": "test", "data": {"id": 46, "name": "TlIVn", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339191} +{"stream": "test", "data": {"id": 47, "name": "PfPKQ", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339191} +{"stream": "test", "data": {"id": 48, "name": "vNekT", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339192} +{"stream": "test", "data": {"id": 49, "name": "Dpnvl", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339192} +{"stream": "test", "data": {"id": 50, "name": "HkxrY", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339192} +{"stream": "test", "data": {"id": 51, "name": "ZUzCq", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339192} +{"stream": "test", "data": {"id": 52, "name": "qVoaC", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339192} +{"stream": "test", "data": {"id": 53, "name": "IaKUC", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339193} +{"stream": "test", "data": {"id": 54, "name": "hOTBB", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339193} +{"stream": "test", "data": {"id": 55, "name": "WijBV", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339193} +{"stream": "test", "data": {"id": 56, "name": "vXTWL", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339193} +{"stream": "test", "data": {"id": 57, "name": "eDGcy", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339193} +{"stream": "test", "data": {"id": 58, "name": "GUAJj", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339193} +{"stream": "test", "data": {"id": 59, "name": "AXSUJ", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339194} +{"stream": "test", "data": {"id": 60, "name": "AxYuv", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339194} +{"stream": "test", "data": {"id": 61, "name": "JvMCP", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339194} +{"stream": "test", "data": {"id": 62, "name": "AzVBn", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339194} +{"stream": "test", "data": {"id": 63, "name": "RRgGz", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339194} +{"stream": "test", "data": {"id": 64, "name": "jLROq", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339194} +{"stream": "test", "data": {"id": 65, "name": "kvzOm", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339195} +{"stream": "test", "data": {"id": 66, "name": "qERMw", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339195} +{"stream": "test", "data": {"id": 67, "name": "cxhzS", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339195} +{"stream": "test", "data": {"id": 68, "name": "sMGCQ", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339195} +{"stream": "test", "data": {"id": 69, "name": "zNEYF", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339195} +{"stream": "test", "data": {"id": 70, "name": "WoXDc", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339195} +{"stream": "test", "data": {"id": 71, "name": "nylfK", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339195} +{"stream": "test", "data": {"id": 72, "name": "iQIlD", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339195} +{"stream": "test", "data": {"id": 73, "name": "fnXPy", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339196} +{"stream": "test", "data": {"id": 74, "name": "Iwlbb", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339196} +{"stream": "test", "data": {"id": 75, "name": "ivixl", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339196} +{"stream": "test", "data": {"id": 76, "name": "aqiOE", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339196} +{"stream": "test", "data": {"id": 77, "name": "IGqnq", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339196} +{"stream": "test", "data": {"id": 78, "name": "Qplhs", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339196} +{"stream": "test", "data": {"id": 79, "name": "SuCoC", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339196} +{"stream": "test", "data": {"id": 80, "name": "QuYXW", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339196} +{"stream": "test", "data": {"id": 81, "name": "SCfiE", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339197} +{"stream": "test", "data": {"id": 82, "name": "dNssE", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339197} +{"stream": "test", "data": {"id": 83, "name": "njKVz", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339197} +{"stream": "test", "data": {"id": 84, "name": "akphf", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339197} +{"stream": "test", "data": {"id": 85, "name": "tHpba", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339197} +{"stream": "test", "data": {"id": 86, "name": "RBqgU", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339197} +{"stream": "test", "data": {"id": 87, "name": "DZdtO", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339197} +{"stream": "test", "data": {"id": 88, "name": "XglaW", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339197} +{"stream": "test", "data": {"id": 89, "name": "zUOOs", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339198} +{"stream": "test", "data": {"id": 90, "name": "jcSbl", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339198} +{"stream": "test", "data": {"id": 91, "name": "UkEkk", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339198} +{"stream": "test", "data": {"id": 92, "name": "VKmlR", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339198} +{"stream": "test", "data": {"id": 93, "name": "SfXTU", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339198} +{"stream": "test", "data": {"id": 94, "name": "xcmCK", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339198} +{"stream": "test", "data": {"id": 95, "name": "tbKIy", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339198} +{"stream": "test", "data": {"id": 96, "name": "NZdRl", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339198} +{"stream": "test", "data": {"id": 97, "name": "sfnoo", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339198} +{"stream": "test", "data": {"id": 98, "name": "nvEDZ", "valid": false, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339198} +{"stream": "test", "data": {"id": 99, "name": "DUVlK", "valid": true, "_ab_source_file_last_modified": "2023-10-12T19:41:58.000000Z", "_ab_source_file_url": "zip_tests/csv_stored.zip#test_zip.csv"}, "emitted_at": 1697140339199} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/zip_csv_custom_encoding.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/zip_csv_custom_encoding.jsonl new file mode 100644 index 000000000000..01804a92f5a7 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/zip_csv_custom_encoding.jsonl @@ -0,0 +1,22 @@ +{"stream": "test", "data": {"id": 0, "name": "ezFAa", "valid": true, "_ab_source_file_last_modified": "2023-10-12T23:17:20.000000Z", "_ab_source_file_url": "zip_tests/csv_cp1252.zip#zip_cp1252_1.csv"}, "emitted_at": 1697143721676} +{"stream": "test", "data": {"id": 1, "name": "KEjdd", "valid": true, "_ab_source_file_last_modified": "2023-10-12T23:17:20.000000Z", "_ab_source_file_url": "zip_tests/csv_cp1252.zip#zip_cp1252_1.csv"}, "emitted_at": 1697143721678} +{"stream": "test", "data": {"id": 2, "name": "pKyJp", "valid": true, "_ab_source_file_last_modified": "2023-10-12T23:17:20.000000Z", "_ab_source_file_url": "zip_tests/csv_cp1252.zip#zip_cp1252_1.csv"}, "emitted_at": 1697143721679} +{"stream": "test", "data": {"id": 3, "name": "MWsBa", "valid": false, "_ab_source_file_last_modified": "2023-10-12T23:17:20.000000Z", "_ab_source_file_url": "zip_tests/csv_cp1252.zip#zip_cp1252_1.csv"}, "emitted_at": 1697143721679} +{"stream": "test", "data": {"id": 4, "name": "EbnKo", "valid": true, "_ab_source_file_last_modified": "2023-10-12T23:17:20.000000Z", "_ab_source_file_url": "zip_tests/csv_cp1252.zip#zip_cp1252_1.csv"}, "emitted_at": 1697143721680} +{"stream": "test", "data": {"id": 5, "name": "FnTdW", "valid": false, "_ab_source_file_last_modified": "2023-10-12T23:17:20.000000Z", "_ab_source_file_url": "zip_tests/csv_cp1252.zip#zip_cp1252_1.csv"}, "emitted_at": 1697143721680} +{"stream": "test", "data": {"id": 6, "name": "iFTcr", "valid": true, "_ab_source_file_last_modified": "2023-10-12T23:17:20.000000Z", "_ab_source_file_url": "zip_tests/csv_cp1252.zip#zip_cp1252_1.csv"}, "emitted_at": 1697143721680} +{"stream": "test", "data": {"id": 7, "name": "rkkpY", "valid": true, "_ab_source_file_last_modified": "2023-10-12T23:17:20.000000Z", "_ab_source_file_url": "zip_tests/csv_cp1252.zip#zip_cp1252_1.csv"}, "emitted_at": 1697143721681} +{"stream": "test", "data": {"id": 8, "name": "jsIUF", "valid": true, "_ab_source_file_last_modified": "2023-10-12T23:17:20.000000Z", "_ab_source_file_url": "zip_tests/csv_cp1252.zip#zip_cp1252_1.csv"}, "emitted_at": 1697143721681} +{"stream": "test", "data": {"id": 9, "name": "IGCfY", "valid": false, "_ab_source_file_last_modified": "2023-10-12T23:17:20.000000Z", "_ab_source_file_url": "zip_tests/csv_cp1252.zip#zip_cp1252_1.csv"}, "emitted_at": 1697143721682} +{"stream": "test", "data": {"id": 10, "name": "VkJsy", "valid": true, "_ab_source_file_last_modified": "2023-10-12T23:17:20.000000Z", "_ab_source_file_url": "zip_tests/csv_cp1252.zip#zip_cp1252_1.csv"}, "emitted_at": 1697143721682} +{"stream": "test", "data": {"id": 0, "name": "ezFAa", "valid": true, "_ab_source_file_last_modified": "2023-10-12T23:17:20.000000Z", "_ab_source_file_url": "zip_tests/csv_cp1252.zip#zip_cp1252_2.csv"}, "emitted_at": 1697143722244} +{"stream": "test", "data": {"id": 1, "name": "KEjdd", "valid": true, "_ab_source_file_last_modified": "2023-10-12T23:17:20.000000Z", "_ab_source_file_url": "zip_tests/csv_cp1252.zip#zip_cp1252_2.csv"}, "emitted_at": 1697143722244} +{"stream": "test", "data": {"id": 2, "name": "pKyJp", "valid": true, "_ab_source_file_last_modified": "2023-10-12T23:17:20.000000Z", "_ab_source_file_url": "zip_tests/csv_cp1252.zip#zip_cp1252_2.csv"}, "emitted_at": 1697143722245} +{"stream": "test", "data": {"id": 3, "name": "MWsBa", "valid": false, "_ab_source_file_last_modified": "2023-10-12T23:17:20.000000Z", "_ab_source_file_url": "zip_tests/csv_cp1252.zip#zip_cp1252_2.csv"}, "emitted_at": 1697143722245} +{"stream": "test", "data": {"id": 4, "name": "EbnKo", "valid": true, "_ab_source_file_last_modified": "2023-10-12T23:17:20.000000Z", "_ab_source_file_url": "zip_tests/csv_cp1252.zip#zip_cp1252_2.csv"}, "emitted_at": 1697143722246} +{"stream": "test", "data": {"id": 5, "name": "FnTdW", "valid": false, "_ab_source_file_last_modified": "2023-10-12T23:17:20.000000Z", "_ab_source_file_url": "zip_tests/csv_cp1252.zip#zip_cp1252_2.csv"}, "emitted_at": 1697143722246} +{"stream": "test", "data": {"id": 6, "name": "iFTcr", "valid": true, "_ab_source_file_last_modified": "2023-10-12T23:17:20.000000Z", "_ab_source_file_url": "zip_tests/csv_cp1252.zip#zip_cp1252_2.csv"}, "emitted_at": 1697143722247} +{"stream": "test", "data": {"id": 7, "name": "rkkpY", "valid": true, "_ab_source_file_last_modified": "2023-10-12T23:17:20.000000Z", "_ab_source_file_url": "zip_tests/csv_cp1252.zip#zip_cp1252_2.csv"}, "emitted_at": 1697143722247} +{"stream": "test", "data": {"id": 8, "name": "jsIUF", "valid": true, "_ab_source_file_last_modified": "2023-10-12T23:17:20.000000Z", "_ab_source_file_url": "zip_tests/csv_cp1252.zip#zip_cp1252_2.csv"}, "emitted_at": 1697143722247} +{"stream": "test", "data": {"id": 9, "name": "IGCfY", "valid": false, "_ab_source_file_last_modified": "2023-10-12T23:17:20.000000Z", "_ab_source_file_url": "zip_tests/csv_cp1252.zip#zip_cp1252_2.csv"}, "emitted_at": 1697143722248} +{"stream": "test", "data": {"id": 10, "name": "VkJsy", "valid": true, "_ab_source_file_last_modified": "2023-10-12T23:17:20.000000Z", "_ab_source_file_url": "zip_tests/csv_cp1252.zip#zip_cp1252_2.csv"}, "emitted_at": 1697143722248} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/zip_jsonl.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/zip_jsonl.jsonl new file mode 100644 index 000000000000..12205fabd9a7 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/zip_jsonl.jsonl @@ -0,0 +1,4 @@ +{"stream": "test", "data": {"id123": 1, "name": "PVdhmjb1", "valid": false, "value": 1.2, "event_date": "2022-01-01T00:00:00Z", "_ab_source_file_last_modified": "2023-10-05T17:02:54.000000Z", "_ab_source_file_url": "zip_tests/zip_jsonl.zip#simple_test1.jsonl"}, "emitted_at": 1697145983114} +{"stream": "test", "data": {"id123": 2, "name": "ABCDEF", "valid": true, "value": 1, "event_date": "2023-01-01T00:00:00Z", "_ab_source_file_last_modified": "2023-10-05T17:02:54.000000Z", "_ab_source_file_url": "zip_tests/zip_jsonl.zip#simple_test1.jsonl"}, "emitted_at": 1697145983116} +{"stream": "test", "data": {"id123": 1, "name": "AAAAAAAAAAAAAAAAAAAAAAAAAA", "valid": false, "value": 1.2, "event_date": "2022-01-01T00:00:00Z", "_ab_source_file_last_modified": "2023-10-06T13:59:24.000000Z", "_ab_source_file_url": "zip_tests/zip_jsonl.zip#simple_test2.jsonl"}, "emitted_at": 1697145983881} +{"stream": "test", "data": {"id123": 2, "name": "BBBBBBBBBBBBBBBBBBBBBBB", "valid": true, "value": 1, "event_date": "2023-01-01T00:00:00Z", "_ab_source_file_last_modified": "2023-10-06T13:59:24.000000Z", "_ab_source_file_url": "zip_tests/zip_jsonl.zip#simple_test2.jsonl"}, "emitted_at": 1697145983882} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/zip_parquet.jsonl b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/zip_parquet.jsonl new file mode 100644 index 000000000000..7e75be450b22 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/integration_tests/expected_records/zip_parquet.jsonl @@ -0,0 +1,8 @@ +{"stream": "test", "data": {"number": 1.0, "name": "foo", "flag": true, "delta": -1.0, "_ab_source_file_last_modified": "2023-10-12T20:41:46.000000Z", "_ab_source_file_url": "zip_tests/zip_parquet.zip#zip_parquet_1.parquet"}, "emitted_at": 1697192617416} +{"stream": "test", "data": {"number": 2.0, "name": "-", "flag": false, "delta": 2.5, "_ab_source_file_last_modified": "2023-10-12T20:41:46.000000Z", "_ab_source_file_url": "zip_tests/zip_parquet.zip#zip_parquet_1.parquet"}, "emitted_at": 1697192617418} +{"stream": "test", "data": {"number": 3.0, "name": "bar", "flag": null, "delta": 0.1, "_ab_source_file_last_modified": "2023-10-12T20:41:46.000000Z", "_ab_source_file_url": "zip_tests/zip_parquet.zip#zip_parquet_1.parquet"}, "emitted_at": 1697192617419} +{"stream": "test", "data": {"number": null, "name": "baz", "flag": true, "delta": null, "_ab_source_file_last_modified": "2023-10-12T20:41:46.000000Z", "_ab_source_file_url": "zip_tests/zip_parquet.zip#zip_parquet_1.parquet"}, "emitted_at": 1697192617420} +{"stream": "test", "data": {"number": 4.0, "name": "qux", "flag": false, "delta": 1.2, "_ab_source_file_last_modified": "2023-10-12T20:41:46.000000Z", "_ab_source_file_url": "zip_tests/zip_parquet.zip#zip_parquet_2.parquet"}, "emitted_at": 1697192619208} +{"stream": "test", "data": {"number": 5.0, "name": "quux", "flag": true, "delta": null, "_ab_source_file_last_modified": "2023-10-12T20:41:46.000000Z", "_ab_source_file_url": "zip_tests/zip_parquet.zip#zip_parquet_2.parquet"}, "emitted_at": 1697192619209} +{"stream": "test", "data": {"number": null, "name": "corge", "flag": null, "delta": -0.5, "_ab_source_file_last_modified": "2023-10-12T20:41:46.000000Z", "_ab_source_file_url": "zip_tests/zip_parquet.zip#zip_parquet_2.parquet"}, "emitted_at": 1697192619210} +{"stream": "test", "data": {"number": 6.0, "name": "grault", "flag": false, "delta": 0.0, "_ab_source_file_last_modified": "2023-10-12T20:41:46.000000Z", "_ab_source_file_url": "zip_tests/zip_parquet.zip#zip_parquet_2.parquet"}, "emitted_at": 1697192619210} diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-s3/integration_tests/integration_test.py deleted file mode 100644 index 28a71f1520d5..000000000000 --- a/airbyte-integrations/connectors/source-s3/integration_tests/integration_test.py +++ /dev/null @@ -1,98 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import json -import time -from typing import Iterator, List, Mapping - -import boto3 -from airbyte_cdk import AirbyteLogger -from botocore.errorfactory import ClientError -from source_s3.stream import IncrementalFileStreamS3 - -from .integration_test_abstract import HERE, SAMPLE_DIR, AbstractTestIncrementalFileStream - -LOGGER = AirbyteLogger() - - -class TestIncrementalFileStreamS3(AbstractTestIncrementalFileStream): - region = "eu-west-3" - - @property - def stream_class(self) -> type: - return IncrementalFileStreamS3 - - @property - def credentials(self) -> Mapping: - filename = HERE.parent / "secrets/config.json" - with open(filename) as json_file: - config = json.load(json_file) - return { - "aws_access_key_id": config["provider"]["aws_access_key_id"], - "aws_secret_access_key": config["provider"]["aws_secret_access_key"], - } - - def provider(self, bucket_name: str) -> Mapping: - return {"storage": "S3", "bucket": bucket_name} - - def _s3_connect(self, credentials: Mapping) -> None: - self.s3_client = boto3.client( - "s3", - aws_access_key_id=credentials["aws_access_key_id"], - aws_secret_access_key=credentials["aws_secret_access_key"], - region_name=self.region, - ) - self.s3_resource = boto3.resource( - "s3", aws_access_key_id=credentials["aws_access_key_id"], aws_secret_access_key=credentials["aws_secret_access_key"] - ) - - def cloud_files(self, cloud_bucket_name: str, credentials: Mapping, files_to_upload: List, private: bool = True) -> Iterator[str]: - self._s3_connect(credentials) - - location = {"LocationConstraint": self.region} - bucket_name = cloud_bucket_name - - print("\n") - LOGGER.info(f"Uploading {len(files_to_upload)} file(s) to {'private' if private else 'public'} aws bucket '{bucket_name}'") - try: - self.s3_client.head_bucket(Bucket=bucket_name) - except ClientError: - if private: - self.s3_client.create_bucket(Bucket=bucket_name, CreateBucketConfiguration=location) - else: - self.s3_client.create_bucket(Bucket=bucket_name, CreateBucketConfiguration=location, ObjectOwnership="ObjectWriter") - self.s3_client.delete_public_access_block(Bucket=bucket_name) - self.s3_client.put_bucket_acl(Bucket=bucket_name, ACL="public-read") - - # wait here until the bucket is ready - ready = False - attempts, max_attempts = 0, 30 - while not ready: - time.sleep(1) - try: - self.s3_client.head_bucket(Bucket=bucket_name) - except ClientError: - attempts += 1 - if attempts >= max_attempts: - raise RuntimeError(f"Couldn't get a successful ping on bucket after ~{max_attempts} seconds") - else: - ready = True - LOGGER.info(f"bucket {bucket_name} initialised") - - extra_args = {} - if not private: - extra_args = {"ACL": "public-read"} - for filepath in files_to_upload: - upload_path = str(filepath).replace(str(SAMPLE_DIR), "") - upload_path = upload_path[1:] if upload_path[0] == "/" else upload_path - self.s3_client.upload_file(str(filepath), bucket_name, upload_path, ExtraArgs=extra_args) - yield f"{bucket_name}/{upload_path}" - - def teardown_infra(self, cloud_bucket_name: str, credentials: Mapping) -> None: - self._s3_connect(credentials) - bucket = self.s3_resource.Bucket(cloud_bucket_name) - bucket.objects.all().delete() - bucket.delete() - LOGGER.info(f"S3 Bucket {cloud_bucket_name} is now deleted") diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/integration_test_abstract.py b/airbyte-integrations/connectors/source-s3/integration_tests/integration_test_abstract.py deleted file mode 100644 index 090e12614de5..000000000000 --- a/airbyte-integrations/connectors/source-s3/integration_tests/integration_test_abstract.py +++ /dev/null @@ -1,526 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import time -from abc import ABC, abstractmethod -from pathlib import Path -from typing import Any, Iterator, List, Mapping -from uuid import uuid4 - -import jsonschema -import pytest -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import SyncMode -from source_s3.source_files_abstract.formats.csv_parser import CsvParser -from source_s3.source_files_abstract.stream import FileStream - -HERE = Path(__file__).resolve().parent -SAMPLE_DIR = HERE.joinpath("sample_files/") -LOGGER = AirbyteLogger() -JSONTYPE_TO_PYTHONTYPE = {"string": str, "number": float, "integer": int, "object": dict, "array": list, "boolean": bool, "null": None} - - -class AbstractTestIncrementalFileStream(ABC): - """Prefix this class with Abstract so the tests don't run here but only in the children""" - - temp_bucket_prefix = "airbytetest-" - - @pytest.fixture(scope="session") - def cloud_bucket_prefix(self) -> str: - return self.temp_bucket_prefix - - @pytest.fixture(scope="session") - def format(self) -> Mapping[str, Any]: - return {"filetype": "csv"} - - @pytest.fixture(scope="session") - def airbyte_system_columns(self) -> Mapping[str, str]: - return { - FileStream.ab_last_mod_col: {"type": "string", "format": "date-time"}, - FileStream.ab_file_name_col: {"type": "string"} - } - - @property - @abstractmethod - def stream_class(self) -> type: - """ - :return: provider specific FileStream class (e.g. IncrementalFileStreamS3) - """ - - @property - @abstractmethod - def credentials(self) -> Mapping: - """ - These will be added automatically to the provider property - - :return: mapping of provider specific credentials - """ - - @abstractmethod - def provider(self, bucket_name: str) -> Mapping: - """ - :return: provider specific provider dict as described in spec.json (leave out credentials, they will be added automatically) - """ - - @abstractmethod - def cloud_files(self, cloud_bucket_name: str, credentials: Mapping, files_to_upload: List, private: bool = True) -> Iterator[str]: - """ - See S3 for example what the override of this needs to achieve. - - :param cloud_bucket_name: name of bucket (or equivalent) - :param credentials: mapping of provider specific credentials - :param files_to_upload: list of paths to local files to upload, pass empty list to test zero files case - :param private: whether or not to make the files private and require credentials to read, defaults to True - :yield: url filepath to uploaded file - """ - - @abstractmethod - def teardown_infra(self, cloud_bucket_name: str, credentials: Mapping) -> None: - """ - Provider-specific logic to tidy up any cloud resources. - See S3 for example. - - :param cloud_bucket_name: bucket (or equivalent) name - :param credentials: mapping of provider specific credentials - """ - - def _stream_records_test_logic( - self, - cloud_bucket_name: str, - format: Mapping[str, str], - airbyte_system_columns: Mapping[str, str], - sync_mode: Any, - files: List[str], - path_pattern: str, - private: bool, - num_columns: Any, - num_records: Any, - expected_schema: Mapping[str, Any], - user_schema: Mapping[str, Any], - fails: Any, - state: Any = None, - ) -> Any: - uploaded_files = [fpath for fpath in self.cloud_files(cloud_bucket_name, self.credentials, files, private)] - LOGGER.info(f"file(s) uploaded: {uploaded_files}") - - # emulate state for incremental testing - # since we're not actually saving state out to file here, we pass schema in to our FileStream creation... - # this isn't how it will work in Airbyte but it's a close enough emulation - current_state = state if state is not None else {FileStream.ab_last_mod_col: "1970-01-01T00:00:00Z"} - if (user_schema is None) and ("schema" in current_state.keys()): - user_schema = current_state["schema"] - - full_expected_schema = { - "type": "object", - "properties": {**expected_schema, **airbyte_system_columns}, - } - - str_user_schema = str(user_schema).replace("'", '"') if user_schema is not None else None - total_num_columns = num_columns + len(airbyte_system_columns.keys()) - provider = {**self.provider(cloud_bucket_name), **self.credentials} if private else self.provider(cloud_bucket_name) - - if not fails: - fs = self.stream_class("dataset", provider, format, path_pattern, str_user_schema) - LOGGER.info(f"Testing stream_records() in SyncMode:{sync_mode.value}") - - # check we return correct schema from get_json_schema() - assert fs.get_json_schema() == full_expected_schema - - records = [] - for stream_slice in fs.stream_slices(sync_mode=sync_mode, stream_state=current_state): - if stream_slice is not None: - # we need to do this in order to work out which extra columns (if any) we expect in this stream_slice - expected_columns = [] - for file_dict in stream_slice["files"]: - # TODO: if we ever test other filetypes in these tests this will need fixing - file_reader = CsvParser(format) - storage_file = file_dict["storage_file"] - with storage_file.open(file_reader.is_binary) as f: - expected_columns.extend(list(file_reader.get_inferred_schema(f, storage_file.file_info).keys())) - expected_columns = set(expected_columns) # de-dupe - - for record in fs.read_records(sync_mode, stream_slice=stream_slice): - # check actual record values match expected schema - jsonschema.validate(record, full_expected_schema) - records.append(record) - - assert all([len(r.keys()) == total_num_columns for r in records]) - assert len(records) == num_records - - # returning state by simulating call to get_updated_state() with final record so we can test incremental - return fs.get_updated_state(current_stream_state=current_state, latest_record=records[-1]) - - else: - with pytest.raises(Exception) as e_info: - fs = self.stream_class("dataset", provider, format, path_pattern, str_user_schema) - LOGGER.info(f"Testing EXPECTED FAILURE stream_records() in SyncMode:{sync_mode.value}") - - fs.get_json_schema() - - records = [] - for stream_slice in fs.stream_slices(sync_mode=sync_mode, stream_state=current_state): - for record in fs.read_records(sync_mode, stream_slice=stream_slice): - records.append(record) - - LOGGER.info(f"Failed as expected, error: {e_info}") - - @pytest.mark.parametrize( - # make user_schema None to test auto-inference. Exclude any _airbyte system columns in expected_schema. - "files, path_pattern, private, num_columns, num_records, expected_schema, user_schema, incremental, fails", - [ - # single file tests - ( # public - [SAMPLE_DIR.joinpath("simple_test.csv")], - "**", - False, - 3, - 8, - {"id": "integer", "name": "string", "valid": "boolean"}, - None, - False, - False, - ), - ( # private - [SAMPLE_DIR.joinpath("simple_test.csv")], - "**", - True, - 3, - 8, - {"id": "integer", "name": "string", "valid": "boolean"}, - None, - False, - False, - ), - ( # provided schema exact match to actual schema - [SAMPLE_DIR.joinpath("simple_test.csv")], - "**", - True, - 3, - 8, - {"id": "integer", "name": "string", "valid": "boolean"}, - {"id": "integer", "name": "string", "valid": "boolean"}, - False, - False, - ), - ( # provided schema not matching datatypes, expect successful coercion - [SAMPLE_DIR.joinpath("simple_test.csv")], - "**", - True, - 3, - 8, - {"id": "string", "name": "string", "valid": "string"}, - {"id": "string", "name": "string", "valid": "string"}, - False, - False, - ), - ( # provided incompatible schema, expect fail - [SAMPLE_DIR.joinpath("simple_test.csv")], - "**", - True, - 3, - 8, - {"id": "boolean", "name": "boolean", "valid": "boolean"}, - {"id": "boolean", "name": "boolean", "valid": "boolean"}, - False, - True, - ), - # multiple file tests (all have identical schemas) - ( # public, auto-infer - [ - SAMPLE_DIR.joinpath("simple_test.csv"), - SAMPLE_DIR.joinpath("simple_test_2.csv"), - SAMPLE_DIR.joinpath("simple_test_3.csv"), - ], - "**", - False, - 3, - 17, - {"id": "integer", "name": "string", "valid": "boolean"}, - None, - False, - False, - ), - ( # private, auto-infer - [ - SAMPLE_DIR.joinpath("simple_test.csv"), - SAMPLE_DIR.joinpath("simple_test_2.csv"), - SAMPLE_DIR.joinpath("simple_test_3.csv"), - ], - "**", - True, - 3, - 17, - {"id": "integer", "name": "string", "valid": "boolean"}, - None, - False, - False, - ), - ( # provided schema exact match to actual schema - [ - SAMPLE_DIR.joinpath("simple_test.csv"), - SAMPLE_DIR.joinpath("simple_test_2.csv"), - SAMPLE_DIR.joinpath("simple_test_3.csv"), - ], - "**", - True, - 3, - 17, - {"id": "integer", "name": "string", "valid": "boolean"}, - {"id": "integer", "name": "string", "valid": "boolean"}, - False, - False, - ), - ( # provided schema not matching datatypes, expect successful coercion - [ - SAMPLE_DIR.joinpath("simple_test.csv"), - SAMPLE_DIR.joinpath("simple_test_2.csv"), - SAMPLE_DIR.joinpath("simple_test_3.csv"), - ], - "**", - True, - 3, - 17, - {"id": "string", "name": "string", "valid": "string"}, - {"id": "string", "name": "string", "valid": "string"}, - False, - False, - ), - ( # provided incompatible schema, expect fail - [ - SAMPLE_DIR.joinpath("simple_test.csv"), - SAMPLE_DIR.joinpath("simple_test_2.csv"), - SAMPLE_DIR.joinpath("simple_test_3.csv"), - ], - "**", - True, - 3, - 17, - {"id": "boolean", "name": "boolean", "valid": "boolean"}, - {"id": "boolean", "name": "boolean", "valid": "boolean"}, - False, - True, - ), - ( # provided schema, not containing all columns (extra columns should go into FileStream.ab_additional_col) - [ - SAMPLE_DIR.joinpath("simple_test.csv"), - SAMPLE_DIR.joinpath("multi_file_diffschema_1.csv"), - SAMPLE_DIR.joinpath("multi_file_diffschema_2.csv"), - ], - "**", - True, - 3, - 17, - {"id": "integer", "name": "string", "valid": "boolean"}, - {"id": "integer", "name": "string", "valid": "boolean"}, - False, - False, - ), - # pattern matching tests with additional files present that we don't want to read - ( # at top-level of bucket - [ - SAMPLE_DIR.joinpath("simple_test.csv"), - SAMPLE_DIR.joinpath("simple_test_2.csv"), - SAMPLE_DIR.joinpath("file_to_skip.csv"), - SAMPLE_DIR.joinpath("file_to_skip.txt"), - ], - "simple*", - True, - 3, - 11, - {"id": "integer", "name": "string", "valid": "boolean"}, - None, - False, - False, - ), - ( # at multiple levels of bucket - [ - SAMPLE_DIR.joinpath("simple_test.csv"), - SAMPLE_DIR.joinpath("simple_test_2.csv"), - SAMPLE_DIR.joinpath("file_to_skip.csv"), - SAMPLE_DIR.joinpath("file_to_skip.txt"), - SAMPLE_DIR.joinpath("pattern_match_test/this_folder/simple_test.csv"), - SAMPLE_DIR.joinpath("pattern_match_test/not_this_folder/file_to_skip.csv"), - SAMPLE_DIR.joinpath("pattern_match_test/not_this_folder/file_to_skip.txt"), - ], - "**/simple*", - True, - 3, - 19, - {"id": "integer", "name": "string", "valid": "boolean"}, - None, - False, - False, - ), - # incremental tests (passing num_records/num_columns/fails as lists holding value for each file in order) - ( # auto-infer, all same schema - [ - SAMPLE_DIR.joinpath("simple_test.csv"), - SAMPLE_DIR.joinpath("simple_test_2.csv"), - SAMPLE_DIR.joinpath("simple_test_3.csv"), - ], - "**", - True, - [3, 3, 3], - [8, 3, 6], - {"id": "integer", "name": "string", "valid": "boolean"}, - None, - True, - [False, False, False], - ), - ( # provided schema, all same schema - [ - SAMPLE_DIR.joinpath("simple_test.csv"), - SAMPLE_DIR.joinpath("simple_test_2.csv"), - SAMPLE_DIR.joinpath("simple_test_3.csv"), - ], - "**", - True, - [3, 3, 3], - [8, 3, 6], - {"id": "integer", "name": "string", "valid": "boolean"}, - {"id": "integer", "name": "string", "valid": "boolean"}, - True, - [False, False, False], - ), - ( # auto-infer, (different but merge-able schemas) - [ - SAMPLE_DIR.joinpath("simple_test.csv"), - SAMPLE_DIR.joinpath("multi_file_diffschema_1.csv"), - SAMPLE_DIR.joinpath("multi_file_diffschema_2.csv"), - ], - "**", - True, - [3, 3, 3], - [8, 3, 6], - {"id": "integer", "name": "string", "valid": "boolean"}, - None, - True, - [False, False, False], - ), - ( # same as previous but change order and expect 5 columns instead of 3 in all - [ - SAMPLE_DIR.joinpath("multi_file_diffschema_2.csv"), - SAMPLE_DIR.joinpath("multi_file_diffschema_1.csv"), - SAMPLE_DIR.joinpath("simple_test.csv"), - ], - "**", - True, - [5, 5, 5], - [6, 3, 8], - {"id": "integer", "name": "string", "valid": "boolean", "percentage": "number", "nullable": "string"}, - None, - True, - [False, False, False], - ), - ( # like previous test but with a user_schema limiting columns - [ - SAMPLE_DIR.joinpath("multi_file_diffschema_2.csv"), - SAMPLE_DIR.joinpath("multi_file_diffschema_1.csv"), - SAMPLE_DIR.joinpath("simple_test.csv"), - ], - "**", - True, - [2, 2, 2], - [6, 3, 8], - {"id": "integer", "name": "string"}, - {"id": "integer", "name": "string"}, - True, - [False, False, False], - ), - ( # fail when 2nd file has incompatible schema, auto-infer - [ - SAMPLE_DIR.joinpath("simple_test.csv"), - SAMPLE_DIR.joinpath("incompatible_schema.csv"), - ], - "**", - True, - [3, 3], - [8, 8], - {"id": "integer", "name": "string", "valid": "boolean"}, - None, - True, - [False, True], - ), - ( # fail when 2nd file has incompatible schema, provided schema - [ - SAMPLE_DIR.joinpath("simple_test.csv"), - SAMPLE_DIR.joinpath("incompatible_schema.csv"), - ], - "**", - True, - [3, 3], - [8, 8], - {"id": "integer", "name": "string", "valid": "boolean"}, - {"id": "integer", "name": "string", "valid": "boolean"}, - True, - [False, True], - ), - ], - ) - def test_stream_records( - self, - cloud_bucket_prefix: str, - format: Mapping[str, Any], - airbyte_system_columns: Mapping[str, str], - files: List[str], - path_pattern: str, - private: bool, - num_columns: List[int], - num_records: List[int], - expected_schema: Mapping[str, Any], - user_schema: Mapping[str, Any], - incremental: bool, - fails: List[bool], - ) -> None: - expected_schema = {k: {"type": ["null", v]} for k, v in expected_schema.items()} - try: - if not incremental: # we expect matching behaviour here in either sync_mode - for sync_mode in [ - SyncMode("full_refresh"), - SyncMode("incremental"), - ]: - cloud_bucket_name = f"{cloud_bucket_prefix}{uuid4()}" - self._stream_records_test_logic( - cloud_bucket_name, - format, - airbyte_system_columns, - sync_mode, - files, - path_pattern, - private, - num_columns, - num_records, - expected_schema, - user_schema, - fails, - ) - self.teardown_infra(cloud_bucket_name, self.credentials) - else: - cloud_bucket_name = f"{cloud_bucket_prefix}{uuid4()}" - latest_state = None - for i in range(len(files)): - latest_state = self._stream_records_test_logic( - cloud_bucket_name, - format, - airbyte_system_columns, - SyncMode("incremental"), - [files[i]], - path_pattern, - private, - num_columns[i], - num_records[i], - expected_schema, - user_schema, - fails[i], - state=latest_state, - ) - LOGGER.info(f"incremental state: {latest_state}") - # small delay to ensure next file gets later last_modified timestamp - time.sleep(1) - self.teardown_infra(cloud_bucket_name, self.credentials) - - except Exception as e: - self.teardown_infra(cloud_bucket_name, self.credentials) - raise e diff --git a/airbyte-integrations/connectors/source-s3/integration_tests/spec.json b/airbyte-integrations/connectors/source-s3/integration_tests/spec.json index f0dee45999a1..76a48ffb09a5 100644 --- a/airbyte-integrations/connectors/source-s3/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-s3/integration_tests/spec.json @@ -1,32 +1,408 @@ { "documentationUrl": "https://docs.airbyte.com/integrations/sources/s3", - "changelogUrl": "https://docs.airbyte.com/integrations/sources/s3", "connectionSpecification": { - "title": "S3 Source Spec", + "title": "Config", + "description": "NOTE: When this Spec is changed, legacy_config_transformer.py must also be modified to uptake the changes\nbecause it is responsible for converting legacy S3 v3 configs into v4 configs using the File-Based CDK.", "type": "object", "properties": { + "start_date": { + "title": "Start Date", + "description": "UTC date and time in the format 2017-01-25T00:00:00.000000Z. Any file modified before this date will not be replicated.", + "examples": ["2021-01-01T00:00:00.000000Z"], + "format": "date-time", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{6}Z$", + "pattern_descriptor": "YYYY-MM-DDTHH:mm:ss.SSSSSSZ", + "order": 1, + "type": "string" + }, + "streams": { + "title": "The list of streams to sync", + "description": "Each instance of this configuration defines a stream. Use this to define which files belong in the stream, their format, and how they should be parsed and validated. When sending data to warehouse destination such as Snowflake or BigQuery, each stream is a separate table.", + "order": 10, + "type": "array", + "items": { + "title": "FileBasedStreamConfig", + "type": "object", + "properties": { + "name": { + "title": "Name", + "description": "The name of the stream.", + "type": "string" + }, + "globs": { + "title": "Globs", + "default": ["**"], + "order": 1, + "description": "The pattern used to specify which files should be selected from the file system. For more information on glob pattern matching look here.", + "type": "array", + "items": { + "type": "string" + } + }, + "legacy_prefix": { + "title": "Legacy Prefix", + "description": "The path prefix configured in v3 versions of the S3 connector. This option is deprecated in favor of a single glob.", + "airbyte_hidden": true, + "type": "string" + }, + "validation_policy": { + "title": "Validation Policy", + "description": "The name of the validation policy that dictates sync behavior when a record does not adhere to the stream schema.", + "default": "Emit Record", + "enum": ["Emit Record", "Skip Record", "Wait for Discover"] + }, + "input_schema": { + "title": "Input Schema", + "description": "The schema that will be used to validate records extracted from the file. This will override the stream schema that is auto-detected from incoming files.", + "type": "string" + }, + "primary_key": { + "title": "Primary Key", + "description": "The column or columns (for a composite key) that serves as the unique identifier of a record. If empty, the primary key will default to the parser's default primary key.", + "type": "string", + "airbyte_hidden": true + }, + "days_to_sync_if_history_is_full": { + "title": "Days To Sync If History Is Full", + "description": "When the state history of the file store is full, syncs will only read files that were last modified in the provided day range.", + "default": 3, + "type": "integer" + }, + "format": { + "title": "Format", + "description": "The configuration options that are used to alter how to read incoming files that deviate from the standard formatting.", + "type": "object", + "oneOf": [ + { + "title": "Avro Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "avro", + "const": "avro", + "type": "string" + }, + "double_as_string": { + "title": "Convert Double Fields to Strings", + "description": "Whether to convert double fields to strings. This is recommended if you have decimal numbers with a high degree of precision because there can be a loss precision when handling floating point numbers.", + "default": false, + "type": "boolean" + } + }, + "required": ["filetype"] + }, + { + "title": "CSV Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "csv", + "const": "csv", + "type": "string" + }, + "delimiter": { + "title": "Delimiter", + "description": "The character delimiting individual cells in the CSV data. This may only be a 1-character string. For tab-delimited data enter '\\t'.", + "default": ",", + "type": "string" + }, + "quote_char": { + "title": "Quote Character", + "description": "The character used for quoting CSV values. To disallow quoting, make this field blank.", + "default": "\"", + "type": "string" + }, + "escape_char": { + "title": "Escape Character", + "description": "The character used for escaping special characters. To disallow escaping, leave this field blank.", + "type": "string" + }, + "encoding": { + "title": "Encoding", + "description": "The character encoding of the CSV data. Leave blank to default to UTF8. See list of python encodings for allowable options.", + "default": "utf8", + "type": "string" + }, + "double_quote": { + "title": "Double Quote", + "description": "Whether two quotes in a quoted CSV value denote a single quote in the data.", + "default": true, + "type": "boolean" + }, + "null_values": { + "title": "Null Values", + "description": "A set of case-sensitive strings that should be interpreted as null values. For example, if the value 'NA' should be interpreted as null, enter 'NA' in this field.", + "default": [], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "strings_can_be_null": { + "title": "Strings Can Be Null", + "description": "Whether strings can be interpreted as null values. If true, strings that match the null_values set will be interpreted as null. If false, strings that match the null_values set will be interpreted as the string itself.", + "default": true, + "type": "boolean" + }, + "skip_rows_before_header": { + "title": "Skip Rows Before Header", + "description": "The number of rows to skip before the header row. For example, if the header row is on the 3rd row, enter 2 in this field.", + "default": 0, + "type": "integer" + }, + "skip_rows_after_header": { + "title": "Skip Rows After Header", + "description": "The number of rows to skip after the header row.", + "default": 0, + "type": "integer" + }, + "header_definition": { + "title": "CSV Header Definition", + "description": "How headers will be defined. `User Provided` assumes the CSV does not have a header row and uses the headers provided and `Autogenerated` assumes the CSV does not have a header row and the CDK will generate headers using for `f{i}` where `i` is the index starting from 0. Else, the default behavior is to use the header from the CSV file. If a user wants to autogenerate or provide column names for a CSV having headers, they can skip rows.", + "default": { + "header_definition_type": "From CSV" + }, + "oneOf": [ + { + "title": "From CSV", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "From CSV", + "const": "From CSV", + "type": "string" + } + }, + "required": ["header_definition_type"] + }, + { + "title": "Autogenerated", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "Autogenerated", + "const": "Autogenerated", + "type": "string" + } + }, + "required": ["header_definition_type"] + }, + { + "title": "User Provided", + "type": "object", + "properties": { + "header_definition_type": { + "title": "Header Definition Type", + "default": "User Provided", + "const": "User Provided", + "type": "string" + }, + "column_names": { + "title": "Column Names", + "description": "The column names that will be used while emitting the CSV records", + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": ["column_names", "header_definition_type"] + } + ], + "type": "object" + }, + "true_values": { + "title": "True Values", + "description": "A set of case-sensitive strings that should be interpreted as true values.", + "default": ["y", "yes", "t", "true", "on", "1"], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "false_values": { + "title": "False Values", + "description": "A set of case-sensitive strings that should be interpreted as false values.", + "default": ["n", "no", "f", "false", "off", "0"], + "type": "array", + "items": { + "type": "string" + }, + "uniqueItems": true + }, + "inference_type": { + "title": "Inference Type", + "description": "How to infer the types of the columns. If none, inference default to strings.", + "default": "None", + "airbyte_hidden": true, + "enum": ["None", "Primitive Types Only"] + } + }, + "required": ["filetype"] + }, + { + "title": "Jsonl Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "jsonl", + "const": "jsonl", + "type": "string" + } + }, + "required": ["filetype"] + }, + { + "title": "Parquet Format", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "parquet", + "const": "parquet", + "type": "string" + }, + "decimal_as_float": { + "title": "Convert Decimal Fields to Floats", + "description": "Whether to convert decimal fields to floats. There is a loss of precision when converting decimals to floats, so this is not recommended.", + "default": false, + "type": "boolean" + } + }, + "required": ["filetype"] + }, + { + "title": "Document File Type Format (Experimental)", + "type": "object", + "properties": { + "filetype": { + "title": "Filetype", + "default": "unstructured", + "const": "unstructured", + "type": "string" + }, + "skip_unprocessable_files": { + "type": "boolean", + "default": true, + "title": "Skip Unprocessable Files", + "description": "If true, skip files that cannot be parsed and pass the error message along as the _ab_source_file_parse_error field. If false, fail the sync.", + "always_show": true + }, + "strategy": { + "type": "string", + "always_show": true, + "order": 0, + "default": "auto", + "title": "Parsing Strategy", + "enum": ["auto", "fast", "ocr_only", "hi_res"], + "description": "The strategy used to parse documents. `fast` extracts text directly from the document which doesn't work for all files. `ocr_only` is more reliable, but slower. `hi_res` is the most reliable, but requires an API key and a hosted instance of unstructured and can't be used with local mode. See the unstructured.io documentation for more details: https://unstructured-io.github.io/unstructured/core/partition.html#partition-pdf" + }, + "processing": { + "title": "Processing", + "description": "Processing configuration", + "default": { + "mode": "local" + }, + "type": "object", + "oneOf": [ + { + "title": "Local", + "type": "object", + "properties": { + "mode": { + "title": "Mode", + "default": "local", + "const": "local", + "enum": ["local"], + "type": "string" + } + }, + "description": "Process files locally, supporting `fast` and `ocr` modes. This is the default option.", + "required": ["mode"] + } + ] + } + }, + "description": "Extract text from document formats (.pdf, .docx, .md, .pptx) and emit as one record per file.", + "required": ["filetype"] + } + ] + }, + "schemaless": { + "title": "Schemaless", + "description": "When enabled, syncs will not validate or structure records against the stream's schema.", + "default": false, + "type": "boolean" + } + }, + "required": ["name", "format"] + } + }, + "bucket": { + "title": "Bucket", + "description": "Name of the S3 bucket where the file(s) exist.", + "order": 0, + "type": "string" + }, + "aws_access_key_id": { + "title": "AWS Access Key ID", + "description": "In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper permissions. If accessing publicly available data, this field is not necessary.", + "airbyte_secret": true, + "order": 2, + "type": "string" + }, + "aws_secret_access_key": { + "title": "AWS Secret Access Key", + "description": "In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper permissions. If accessing publicly available data, this field is not necessary.", + "airbyte_secret": true, + "order": 3, + "type": "string" + }, + "role_arn": { + "title": "AWS Role ARN", + "description": "Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations requested using this profile. Set the External ID to the Airbyte workspace ID, which can be found in the URL of this page.", + "order": 6, + "type": "string" + }, + "endpoint": { + "title": "Endpoint", + "description": "Endpoint to an S3 compatible service. Leave empty to use AWS.", + "default": "", + "examples": ["my-s3-endpoint.com", "https://my-s3-endpoint.com"], + "order": 4, + "type": "string" + }, "dataset": { "title": "Output Stream Name", - "description": "The name of the stream you would like this source to output. Can contain letters, numbers, or underscores.", + "description": "Deprecated and will be removed soon. Please do not use this field anymore and use streams.name instead. The name of the stream you would like this source to output. Can contain letters, numbers, or underscores.", "pattern": "^([A-Za-z0-9-_]+)$", - "order": 0, - "type": "string" + "order": 100, + "type": "string", + "airbyte_hidden": true }, "path_pattern": { "title": "Pattern of files to replicate", - "description": "A regular expression which tells the connector which files to replicate. All files which match this pattern will be replicated. Use | to separate multiple patterns. See this page to understand pattern syntax (GLOBSTAR and SPLIT flags are enabled). Use pattern ** to pick up all files.", + "description": "Deprecated and will be removed soon. Please do not use this field anymore and use streams.globs instead. A regular expression which tells the connector which files to replicate. All files which match this pattern will be replicated. Use | to separate multiple patterns. See this page to understand pattern syntax (GLOBSTAR and SPLIT flags are enabled). Use pattern ** to pick up all files.", "examples": [ "**", "myFolder/myTableFiles/*.csv|myFolder/myOtherTableFiles/*.csv" ], - "order": 10, - "type": "string" + "order": 110, + "type": "string", + "airbyte_hidden": true }, "format": { "title": "File Format", - "description": "The format of the files you'd like to replicate", + "description": "Deprecated and will be removed soon. Please do not use this field anymore and use streams.format instead. The format of the files you'd like to replicate", "default": "csv", - "order": 20, + "order": 120, "type": "object", "oneOf": [ { @@ -199,17 +575,19 @@ } } } - ] + ], + "airbyte_hidden": true }, "schema": { "title": "Manually enforced data schema", - "description": "Optionally provide a schema to enforce, as a valid JSON string. Ensure this is a mapping of { \"column\" : \"type\" }, where types are valid JSON Schema datatypes. Leave as {} to auto-infer the schema.", + "description": "Deprecated and will be removed soon. Please do not use this field anymore and use streams.input_schema instead. Optionally provide a schema to enforce, as a valid JSON string. Ensure this is a mapping of { \"column\" : \"type\" }, where types are valid JSON Schema datatypes. Leave as {} to auto-infer the schema.", "default": "{}", "examples": [ "{\"column_1\": \"number\", \"column_2\": \"string\", \"column_3\": \"array\", \"column_4\": \"object\", \"column_5\": \"boolean\"}" ], - "order": 30, - "type": "string" + "order": 130, + "type": "string", + "airbyte_hidden": true }, "provider": { "title": "S3: Amazon Web Services", @@ -225,6 +603,7 @@ "title": "AWS Access Key ID", "description": "In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper permissions. If accessing publicly available data, this field is not necessary.", "airbyte_secret": true, + "always_show": true, "order": 1, "type": "string" }, @@ -232,9 +611,17 @@ "title": "AWS Secret Access Key", "description": "In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper permissions. If accessing publicly available data, this field is not necessary.", "airbyte_secret": true, + "always_show": true, "order": 2, "type": "string" }, + "role_arn": { + "title": "AWS Role ARN", + "description": "Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations requested using this profile. Set the External ID to the Airbyte workspace ID, which can be found in the URL of this page.", + "always_show": true, + "order": 6, + "type": "string" + }, "path_prefix": { "title": "Path Prefix", "description": "By providing a path-like prefix (e.g. myFolder/thisTable/) under which all the relevant files sit, we can optimize finding these in S3. This is optional but recommended if your bucket contains many folders/files which you don't need to replicate.", @@ -259,13 +646,12 @@ "type": "string" } }, - "required": ["bucket"], - "order": 11, - "description": "Use this to load files from S3 or S3-compatible services" + "required": [], + "order": 111, + "description": "Deprecated and will be removed soon. Please do not use this field anymore and use bucket, aws_access_key_id, aws_secret_access_key and endpoint instead. Use this to load files from S3 or S3-compatible services", + "airbyte_hidden": true } }, - "required": ["dataset", "path_pattern", "provider"] - }, - "supportsIncremental": true, - "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"] + "required": ["streams", "bucket"] + } } diff --git a/airbyte-integrations/connectors/source-s3/main.py b/airbyte-integrations/connectors/source-s3/main.py index 753f90101f93..cb0007d5581b 100644 --- a/airbyte-integrations/connectors/source-s3/main.py +++ b/airbyte-integrations/connectors/source-s3/main.py @@ -3,11 +3,7 @@ # -import sys - -from airbyte_cdk.entrypoint import launch -from source_s3 import SourceS3 +from source_s3.run import run if __name__ == "__main__": - source = SourceS3() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-s3/metadata.yaml b/airbyte-integrations/connectors/source-s3/metadata.yaml index d01dba0bc09b..d660125e526b 100644 --- a/airbyte-integrations/connectors/source-s3/metadata.yaml +++ b/airbyte-integrations/connectors/source-s3/metadata.yaml @@ -1,12 +1,18 @@ data: + ab_internal: + ql: 400 + sl: 300 allowedHosts: hosts: - "*.s3.amazonaws.com" + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: file connectorType: source definitionId: 69589781-7828-43c5-9f63-8925b1c1ccc2 - dockerImageTag: 3.1.9 + dockerImageTag: 4.4.0 dockerRepository: airbyte/source-s3 + documentationUrl: https://docs.airbyte.com/integrations/sources/s3 githubIssueLabel: source-s3 icon: s3.svg license: ELv2 @@ -17,11 +23,19 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/s3 + releases: + breakingChanges: + 4.0.0: + message: + UX improvement, multi-stream support and deprecation of some parsing + features + upgradeDeadline: "2023-10-05" + 4.0.4: + message: + Following 4.0.0 config change, we are eliminating the `streams.*.file_type` + field which was redundant with `streams.*.format` + upgradeDeadline: "2023-10-18" + supportLevel: certified tags: - language:python - ab_internal: - sl: 300 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-s3/setup.py b/airbyte-integrations/connectors/source-s3/setup.py index 241605f2aa64..cd2c48e2924d 100644 --- a/airbyte-integrations/connectors/source-s3/setup.py +++ b/airbyte-integrations/connectors/source-s3/setup.py @@ -6,27 +6,15 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk>=0.50.2", - "pyarrow==12.0.1", + "airbyte-cdk[file-based]>=0.57.7", "smart-open[s3]==5.1.0", "wcmatch==8.4", "dill==0.3.4", "pytz", - "fastavro==1.4.11", "python-snappy==0.6.1", ] -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest-mock~=3.6.1", - "pytest~=6.1", - "pandas==2.0.3", - "psutil", - "pytest-order", - "netifaces~=0.11.0", - "docker", - "avro==1.11.0", -] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest-mock~=3.6.1", "pytest~=6.1", "pandas==2.0.3", "docker", "moto"] setup( name="source_s3", @@ -39,4 +27,9 @@ extras_require={ "tests": TEST_REQUIREMENTS, }, + entry_points={ + "console_scripts": [ + "source-s3=source_s3.run:run", + ], + }, ) diff --git a/airbyte-integrations/connectors/source-s3/source_s3/__init__.py b/airbyte-integrations/connectors/source-s3/source_s3/__init__.py index 1743ed6fc867..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/__init__.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/__init__.py @@ -1,27 +1,3 @@ -""" -MIT License - -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" - -from .source import SourceS3 - -__all__ = ["SourceS3"] +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-s3/source_s3/run.py b/airbyte-integrations/connectors/source-s3/source_s3/run.py new file mode 100644 index 000000000000..67379f1ec4ab --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/source_s3/run.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys +import traceback +from datetime import datetime +from typing import List + +from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch +from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type +from source_s3.v4 import Config, Cursor, SourceS3, SourceS3StreamReader + + +def get_source(args: List[str]): + catalog_path = AirbyteEntrypoint.extract_catalog(args) + try: + return SourceS3(SourceS3StreamReader(), Config, catalog_path, cursor_cls=Cursor) + except Exception: + print( + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.ERROR, + emitted_at=int(datetime.now().timestamp() * 1000), + error=AirbyteErrorTraceMessage( + message="Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance.", + stack_trace=traceback.format_exc(), + ), + ), + ).json() + ) + return None + + +def run(): + _args = sys.argv[1:] + source = get_source(_args) + + if source: + launch(source, _args) diff --git a/airbyte-integrations/connectors/source-s3/source_s3/s3_utils.py b/airbyte-integrations/connectors/source-s3/source_s3/s3_utils.py deleted file mode 100644 index 5c49e5cac066..000000000000 --- a/airbyte-integrations/connectors/source-s3/source_s3/s3_utils.py +++ /dev/null @@ -1,56 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import boto3.session -from botocore.client import BaseClient, Config - - -def make_s3_resource(provider: dict, session: boto3.session.Session, config: Config = None) -> object: - """ - Construct boto3 resource with specified config and remote endpoint - :param provider provider configuration from connector configuration. - :param session User session to create client from. - :param config Client config parameter in case of using creds from .aws/config file. - :return Boto3 S3 resource instance. - """ - client_kv_args = _get_s3_client_args(provider, config) - return session.resource("s3", **client_kv_args) - - -def make_s3_client(provider: dict, session: boto3.session.Session = None, config: Config = None) -> BaseClient: - """ - Construct boto3 client with specified config and remote endpoint - :param provider provider configuration from connector configuration. - :param session User session to create client from. Default boto3 sesion in case of session not specified. - :param config Client config parameter in case of using creds from .aws/config file. - :return Boto3 S3 client instance. - """ - client_kv_args = _get_s3_client_args(provider, config) - if session is None: - return boto3.client("s3", **client_kv_args) - else: - return session.client("s3", **client_kv_args) - - -def _get_s3_client_args(provider: dict, config: Config) -> dict: - """ - Returns map of args used for creating s3 boto3 client. - :param provider provider configuration from connector configuration. - :param config Client config parameter in case of using creds from .aws/config file. - :return map of s3 client arguments. - """ - client_kv_args = {"config": config} - endpoint = provider.get("endpoint") - if endpoint: - # endpoint could be None or empty string, set to default Amazon endpoint in - # this case. - client_kv_args["endpoint_url"] = endpoint - client_kv_args["use_ssl"] = provider.get("use_ssl", True) - client_kv_args["verify"] = provider.get("verify_ssl_cert", True) - client_kv_args["config"] = Config(s3={"addressing_style": provider.get("addressing_style", "auto")}) - - return client_kv_args - - -__all__ = ["make_s3_client", "make_s3_resource"] diff --git a/airbyte-integrations/connectors/source-s3/source_s3/s3file.py b/airbyte-integrations/connectors/source-s3/source_s3/s3file.py deleted file mode 100644 index 90e82dd69764..000000000000 --- a/airbyte-integrations/connectors/source-s3/source_s3/s3file.py +++ /dev/null @@ -1,77 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from contextlib import contextmanager -from typing import Any, BinaryIO, Iterator, Mapping, TextIO, Union - -import smart_open -from boto3 import session as boto3session -from botocore import UNSIGNED -from botocore.client import Config as ClientConfig -from botocore.config import Config -from source_s3.s3_utils import make_s3_client, make_s3_resource - -from .source_files_abstract.storagefile import StorageFile - - -class S3File(StorageFile): - def __init__(self, *args: Any, **kwargs: Any): - super().__init__(*args, **kwargs) - self._setup_boto_session() - - def _setup_boto_session(self) -> None: - """ - Making a new Session at file level rather than stream level as boto3 sessions are NOT thread-safe. - Currently grabbing last_modified across multiple files asynchronously and may implement more multi-threading in future. - See https://boto3.amazonaws.com/v1/documentation/api/latest/guide/resources.html (anchor link broken, scroll to bottom) - """ - if self.use_aws_account(self._provider): - self._boto_session = boto3session.Session( - aws_access_key_id=self._provider.get("aws_access_key_id"), - aws_secret_access_key=self._provider.get("aws_secret_access_key"), - ) - self._boto_s3_resource = make_s3_resource(self._provider, session=self._boto_session) - else: - self._boto_session = boto3session.Session() - self._boto_s3_resource = make_s3_resource(self._provider, config=Config(signature_version=UNSIGNED), session=self._boto_session) - - @staticmethod - def use_aws_account(provider: Mapping[str, str]) -> bool: - aws_access_key_id = provider.get("aws_access_key_id") - aws_secret_access_key = provider.get("aws_secret_access_key") - return True if (aws_access_key_id is not None and aws_secret_access_key is not None) else False - - @contextmanager - def open(self, binary: bool) -> Iterator[Union[TextIO, BinaryIO]]: - """ - Utilising smart_open to handle this (https://github.com/RaRe-Technologies/smart_open) - - :param binary: whether or not to open file as binary - :return: file-like object - """ - mode = "rb" if binary else "r" - bucket = self._provider.get("bucket") - if self.use_aws_account(self._provider): - params = {"client": make_s3_client(self._provider, session=self._boto_session)} - else: - config = ClientConfig(signature_version=UNSIGNED) - params = {"client": make_s3_client(self._provider, config=config)} - self.logger.debug(f"try to open {self.file_info}") - # There are rare cases when some keys become unreachable during sync - # and we don't know about it, because catalog has been initially formed only once at the beginning - # This is happen for example if a file was deleted/moved (or anything else) while we proceed with another file - try: - result = smart_open.open(f"s3://{bucket}/{self.url}", transport_params=params, mode=mode) - except OSError as e: - self.logger.warn( - f"We don't have access to {self.url}. " - f"Check whether key {self.url} exists in `{bucket}` bucket and/or has proper ACL permissions" - ) - raise e - # see https://docs.python.org/3/library/contextlib.html#contextlib.contextmanager for why we do this - try: - yield result - finally: - result.close() diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source.py b/airbyte-integrations/connectors/source-s3/source_s3/source.py index 90ed77e775f3..224f7b036e4a 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/source.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/source.py @@ -1,15 +1,11 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - - -from typing import Any, Mapping, Optional +from typing import Optional from pydantic import BaseModel, Field -from .source_files_abstract.source import SourceFilesAbstract from .source_files_abstract.spec import SourceFilesAbstractSpec -from .stream import IncrementalFileStreamS3 class SourceS3Spec(SourceFilesAbstractSpec, BaseModel): @@ -29,6 +25,7 @@ class Config: description="In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper " "permissions. If accessing publicly available data, this field is not necessary.", airbyte_secret=True, + always_show=True, order=1, ) aws_secret_access_key: Optional[str] = Field( @@ -37,8 +34,17 @@ class Config: description="In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper " "permissions. If accessing publicly available data, this field is not necessary.", airbyte_secret=True, + always_show=True, order=2, ) + role_arn: Optional[str] = Field( + title=f"AWS Role ARN", + default=None, + description="Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations " + f"requested using this profile. Set the External ID to the Airbyte workspace ID, which can be found in the URL of this page.", + always_show=True, + order=6, + ) path_prefix: str = Field( default="", description="By providing a path-like prefix (e.g. myFolder/thisTable/) under which all the relevant files sit, " @@ -58,15 +64,3 @@ class Config: ) provider: S3Provider - - -class SourceS3(SourceFilesAbstract): - stream_class = IncrementalFileStreamS3 - spec_class = SourceS3Spec - documentation_url = "https://docs.airbyte.com/integrations/sources/s3" - - def read_config(self, config_path: str) -> Mapping[str, Any]: - config: Mapping[str, Any] = super().read_config(config_path) - if config.get("format", {}).get("delimiter") == r"\t": - config["format"]["delimiter"] = "\t" - return config diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/__init__.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/__init__.py index 9db886e0930f..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/__init__.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/__init__.py @@ -1,23 +1,3 @@ # -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/file_info.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/file_info.py deleted file mode 100644 index cdfaf72779a2..000000000000 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/file_info.py +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from dataclasses import dataclass -from datetime import datetime -from functools import total_ordering - - -@total_ordering -@dataclass -class FileInfo: - """Class for sharing of metadata""" - - key: str - size: int - last_modified: datetime - - @property - def size_in_megabytes(self) -> float: - return self.size / 1024**2 - - def __str__(self) -> str: - return "Key: %s, LastModified: %s, Size: %.4fMb" % (self.key, self.last_modified.isoformat(), self.size_in_megabytes) - - def __repr__(self) -> str: - return self.__str__() - - def __eq__(self, other: object) -> bool: - if isinstance(other, FileInfo): - return self.key == other.key - return self.key == other - - def __lt__(self, other: object) -> bool: - if isinstance(other, FileInfo): - return self.key < other.key - return self.key < str(other) - - def __hash__(self) -> int: - return self.key.__hash__() diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/__init__.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/__init__.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/abstract_file_parser.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/abstract_file_parser.py deleted file mode 100644 index d28c62067cf5..000000000000 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/abstract_file_parser.py +++ /dev/null @@ -1,117 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from abc import ABC, abstractmethod -from typing import Any, BinaryIO, Iterator, Mapping, TextIO, Union - -import pyarrow as pa -from airbyte_cdk.logger import AirbyteLogger -from source_s3.source_files_abstract.file_info import FileInfo - - -class AbstractFileParser(ABC): - logger = AirbyteLogger() - - NON_SCALAR_TYPES = {"struct": "struct", "list": "list"} - TYPE_MAP = { - "boolean": ("bool_", "bool"), - "integer": ("int64", "int8", "int16", "int32", "uint8", "uint16", "uint32", "uint64"), - "number": ("float64", "float16", "float32", "decimal128", "decimal256", "halffloat", "float", "double"), - "string": ("large_string", "string"), - # TODO: support object type rather than coercing to string - "object": ("large_string",), - # TODO: support array type rather than coercing to string - "array": ("large_string",), - "null": ("large_string",), - } - - def __init__(self, format: dict, master_schema: dict = None): - """ - :param format: file format specific mapping as described in spec.json - :param master_schema: superset schema determined from all files, might be unused for some formats, defaults to None - """ - self._format = format - self._master_schema = ( - master_schema - # this may need to be used differently by some formats, pyarrow allows extra columns in csv schema - ) - - @property - @abstractmethod - def is_binary(self) -> bool: - """ - Override this per format so that file-like objects passed in are currently opened as binary or not - """ - - @abstractmethod - def get_inferred_schema(self, file: Union[TextIO, BinaryIO], file_info: FileInfo) -> dict: - """ - Override this with format-specifc logic to infer the schema of file - Note: needs to return inferred schema with JsonSchema datatypes - - :param file: file-like object (opened via StorageFile) - :param file_info: file metadata - :return: mapping of {columns:datatypes} where datatypes are JsonSchema types - """ - - @abstractmethod - def stream_records(self, file: Union[TextIO, BinaryIO], file_info: FileInfo) -> Iterator[Mapping[str, Any]]: - """ - Override this with format-specifc logic to stream each data row from the file as a mapping of {columns:values} - Note: avoid loading the whole file into memory to avoid OOM breakages - - :param file: file-like object (opened via StorageFile) - :param file_info: file metadata - :yield: data record as a mapping of {columns:values} - """ - - @classmethod - def json_type_to_pyarrow_type(cls, typ: str, reverse: bool = False, logger: AirbyteLogger = AirbyteLogger()) -> str: - """ - Converts Json Type to PyArrow types to (or the other way around if reverse=True) - - :param typ: Json type if reverse is False, else PyArrow type - :param reverse: switch to True for PyArrow type -> Json type, defaults to False - :param logger: defaults to AirbyteLogger() - :return: PyArrow type if reverse is False, else Json type - """ - str_typ = str(typ) - # This is a map of airbyte types to pyarrow types. - # The first list element of the pyarrow types should be the one to use where required. - - if not reverse: - for json_type, pyarrow_types in cls.TYPE_MAP.items(): - if str_typ.lower() == json_type: - type_ = next(iter(pyarrow_types)) - if type_ in cls.NON_SCALAR_TYPES: - return cls.NON_SCALAR_TYPES[type_] - # better way might be necessary when we decide to handle more type complexity - return str(getattr(pa, type_).__call__()) - logger.debug(f"JSON type '{str_typ}' is not mapped, falling back to default conversion to large_string") - return str(pa.large_string()) - else: - for json_type, pyarrow_types in cls.TYPE_MAP.items(): - if any(str_typ.startswith(pa_type) for pa_type in pyarrow_types): - return json_type - logger.debug(f"PyArrow type '{str_typ}' is not mapped, falling back to default conversion to string") - return "string" # default type if unspecified in map - - @classmethod - def json_schema_to_pyarrow_schema(cls, schema: Mapping[str, Any], reverse: bool = False) -> Mapping[str, Any]: - """ - Converts a schema with JsonSchema datatypes to one with PyArrow types (or the other way if reverse=True) - This utilises json_type_to_pyarrow_type() to convert each datatype - - :param schema: json/pyarrow schema to convert - :param reverse: switch to True for PyArrow schema -> Json schema, defaults to False - :return: converted schema dict - """ - return {column: cls.json_type_to_pyarrow_type(json_type, reverse=reverse) for column, json_type in schema.items()} - - def _validate_config(self, config: Mapping[str, Any]): - pass - - @classmethod - def set_minimal_block_size(cls, format: Mapping[str, Any]): - pass diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/avro_parser.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/avro_parser.py deleted file mode 100644 index a4aa37708f14..000000000000 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/avro_parser.py +++ /dev/null @@ -1,100 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from typing import Any, BinaryIO, Iterator, Mapping, TextIO, Union - -import fastavro -from fastavro import reader -from source_s3.source_files_abstract.file_info import FileInfo - -from .abstract_file_parser import AbstractFileParser - -# mapping from apache avro docs: https://avro.apache.org/docs/current/spec.html#schema_complex -AVRO_TO_JSON_DATA_TYPE_MAPPING = { - "null": "null", - "boolean": ["boolean", "null"], - "int": ["integer", "null"], - "long": ["integer", "null"], - "float": ["number", "null"], - "double": ["number", "null"], - "bytes": ["string", "null"], - "string": ["string", "null"], - "record": ["object", "null"], - "enum": ["string", "null"], - "array": ["array", "null"], - "map": ["object", "null"], - "fixed": ["string", "null"], -} - - -class AvroParser(AbstractFileParser): - def __init__(self, *args: Any, **kwargs: Any): - super().__init__(*args, **kwargs) - - @property - def is_binary(self) -> bool: - return True - - def avro_type_to_json_type(self, avro_type): - try: - return AVRO_TO_JSON_DATA_TYPE_MAPPING[avro_type] - except KeyError: - raise ValueError(f"Unknown Avro type: {avro_type}") - - def avro_to_jsonschema(self, avro_schema: dict) -> dict: - """Convert data types from avro to json format - :param avro_schema: schema comes with the avro file - :return schema_dict with data types converted from avro to json standards - """ - json_schema = {} - # Process Avro schema fields - for field in avro_schema["fields"]: - field_name = field["name"] - field_type = field["type"] - # Convert Avro types to JSON schema types - if isinstance(field_type, dict) and field_type.get("type") == "array": - field_schema = {"type": ["array", "null"], "items": self.avro_to_jsonschema(field_type.get("items"))} - elif isinstance(field_type, dict): - field_schema = {"type": ["object", "null"], **self.avro_to_jsonschema(field_type)} - elif isinstance(field_type, list) and [x.get("fields") for x in field_type if not isinstance(x, str)]: - # field_type = [x for x in field_type if x != 'null'][0] - field_schema = {"anyOf": [self.avro_to_jsonschema(t) for t in field_type]} - else: - field_type = [x for x in field_type if x != "null"][0] if isinstance(field_type, list) else field_type - field_schema = {"type": self.avro_type_to_json_type(field_type)} - json_schema[field_name] = field_schema - return json_schema - - def _get_avro_schema(self, file: Union[TextIO, BinaryIO]) -> dict: - """Extract schema for records - :param file: file-like object (opened via StorageFile) - :return schema extracted from the avro file - """ - avro_reader = fastavro.reader(file) - schema = avro_reader.writer_schema - if not schema["type"] == "record": - unsupported_type = schema["type"] - raise (f"Only record based avro files are supported. Found {unsupported_type}") - else: - return schema - - def get_inferred_schema(self, file: Union[TextIO, BinaryIO], file_info: FileInfo) -> dict: - """Return schema - :param file: file-like object (opened via StorageFile) - :param file_info: file metadata - :return: mapping of JsonSchema properties {columns:{"type": datatypes}} - """ - avro_schema = self._get_avro_schema(file) - schema_dict = self.avro_to_jsonschema(avro_schema) - return schema_dict - - def stream_records(self, file: Union[TextIO, BinaryIO], file_info: FileInfo) -> Iterator[Mapping[str, Any]]: - """Stream the data using a generator - :param file: file-like object (opened via StorageFile) - :param file_info: file metadata - :yield: data record as a mapping of {columns:values} - """ - avro_reader = reader(file) - for record in avro_reader: - yield record diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/csv_parser.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/csv_parser.py deleted file mode 100644 index d363f0fa8002..000000000000 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/csv_parser.py +++ /dev/null @@ -1,270 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import codecs -import csv -import json -import tempfile -from typing import Any, BinaryIO, Callable, Iterator, Mapping, Optional, TextIO, Tuple, Union - -import pyarrow -import pyarrow as pa -import six # type: ignore[import] -from airbyte_cdk.models import FailureType -from airbyte_cdk.utils.traced_exception import AirbyteTracedException -from pyarrow import csv as pa_csv -from pyarrow.lib import ArrowInvalid -from source_s3.exceptions import S3Exception -from source_s3.source_files_abstract.file_info import FileInfo -from source_s3.utils import get_value_or_json_if_empty_string, run_in_external_process - -from .abstract_file_parser import AbstractFileParser -from .csv_spec import CsvFormat - -MAX_CHUNK_SIZE = 50.0 * 1024**2 # in bytes -TMP_FOLDER = tempfile.mkdtemp() - - -def wrap_exception(exceptions: Tuple[type, ...]): - def wrapper(fn: callable): - def inner(self, file: Union[TextIO, BinaryIO], file_info: FileInfo): - try: - return fn(self, file, file_info) - except exceptions as e: - raise S3Exception(file_info, str(e), str(e), exception=e) - - return inner - - return wrapper - - -class CsvParser(AbstractFileParser): - def __init__(self, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) - self.format_model = None - - @property - def is_binary(self) -> bool: - return True - - @property - def format(self) -> CsvFormat: - if self.format_model is None: - self.format_model = CsvFormat.parse_obj(self._format) - return self.format_model - - @staticmethod - def _validate_field( - format_: Mapping[str, Any], field_name: str, allow_empty: bool = False, disallow_values: Optional[Tuple[Any, ...]] = None - ) -> Optional[str]: - disallow_values = disallow_values or () - field_value = format_.get(field_name) - if not field_value and allow_empty: - return - if field_value and len(field_value) != 1: - return f"{field_name} should contain 1 character only" - if field_value in disallow_values: - return f"{field_name} can not be {field_value}" - - @staticmethod - def _validate_encoding(encoding: str) -> None: - try: - codecs.lookup(encoding) - except LookupError as e: - # UTF8 is the default encoding value, so there is no problem if `encoding` is not set manually - if encoding != "": - raise AirbyteTracedException(str(e), str(e), failure_type=FailureType.config_error) - - @classmethod - def _validate_options(cls, validator: Callable, options_name: str, format_: Mapping[str, Any]) -> Optional[str]: - options = format_.get(options_name, "{}") - try: - options = json.loads(options) - validator(**options) - except json.decoder.JSONDecodeError: - return "Malformed advanced read options!" - except TypeError as e: - return f"One or more read options are invalid: {str(e)}" - - @classmethod - def _validate_read_options(cls, format_: Mapping[str, Any]) -> Optional[str]: - return cls._validate_options(pa.csv.ReadOptions, "advanced_options", format_) - - @classmethod - def _validate_convert_options(cls, format_: Mapping[str, Any]) -> Optional[str]: - return cls._validate_options(pa.csv.ConvertOptions, "additional_reader_options", format_) - - def _validate_config(self, config: Mapping[str, Any]): - format_ = config.get("format", {}) - for error_message in ( - self._validate_field(format_, "delimiter", disallow_values=("\r", "\n")), - self._validate_field(format_, "quote_char"), - self._validate_field(format_, "escape_char", allow_empty=True), - self._validate_read_options(format_), - self._validate_convert_options(format_), - ): - if error_message: - raise AirbyteTracedException(error_message, error_message, failure_type=FailureType.config_error) - - self._validate_encoding(format_.get("encoding", "")) - - def _read_options(self) -> Mapping[str, str]: - """ - https://arrow.apache.org/docs/python/generated/pyarrow.csv.ReadOptions.html - build ReadOptions object like: pa.csv.ReadOptions(**self._read_options()) - """ - advanced_options = get_value_or_json_if_empty_string(self.format.advanced_options) - return { - **{"block_size": self.format.block_size, "encoding": self.format.encoding}, - **json.loads(advanced_options), - } - - def _parse_options(self) -> Mapping[str, str]: - """ - https://arrow.apache.org/docs/python/generated/pyarrow.csv.ParseOptions.html - build ParseOptions object like: pa.csv.ParseOptions(**self._parse_options()) - """ - - return { - "delimiter": self.format.delimiter, - "quote_char": self.format.quote_char, - "double_quote": self.format.double_quote, - "escape_char": self.format.escape_char, - "newlines_in_values": self.format.newlines_in_values, - } - - def _convert_options(self, json_schema: Mapping[str, Any] = None) -> Mapping[str, Any]: - """ - https://arrow.apache.org/docs/python/generated/pyarrow.csv.ConvertOptions.html - build ConvertOptions object like: pa.csv.ConvertOptions(**self._convert_options()) - :param json_schema: if this is passed in, pyarrow will attempt to enforce this schema on read, defaults to None - """ - check_utf8 = self.format.encoding.lower().replace("-", "") == "utf8" - additional_reader_options = get_value_or_json_if_empty_string(self.format.additional_reader_options) - convert_schema = self.json_schema_to_pyarrow_schema(json_schema) if json_schema is not None else None - return { - **{"check_utf8": check_utf8, "column_types": convert_schema}, - **json.loads(additional_reader_options), - } - - @wrap_exception((ValueError,)) - def get_inferred_schema(self, file: Union[TextIO, BinaryIO], file_info: FileInfo) -> Mapping[str, Any]: - """ - https://arrow.apache.org/docs/python/generated/pyarrow.csv.open_csv.html - This now uses multiprocessing in order to timeout the schema inference as it can hang. - Since the hanging code is resistant to signal interrupts, threading/futures doesn't help so needed to multiprocess. - https://issues.apache.org/jira/browse/ARROW-11853?page=com.atlassian.jira.plugin.system.issuetabpanels%3Aall-tabpanel - """ - - def infer_schema_process( - file_sample: str, read_opts: dict, parse_opts: dict, convert_opts: dict - ) -> Tuple[dict, Optional[Exception]]: - """ - we need to reimport here to be functional on Windows systems since it doesn't have fork() - https://docs.python.org/3.7/library/multiprocessing.html#contexts-and-start-methods - This returns a tuple of (schema_dict, None OR Exception). - If return[1] is not None and holds an exception we then raise this in the main process. - This lets us propagate up any errors (that aren't timeouts) and raise correctly. - """ - try: - import tempfile - - import pyarrow as pa - - # writing our file_sample to a temporary file to then read in and schema infer as before - with tempfile.TemporaryFile() as fp: - fp.write(file_sample) # type: ignore[arg-type] - fp.seek(0) - streaming_reader = pa.csv.open_csv( - fp, pa.csv.ReadOptions(**read_opts), pa.csv.ParseOptions(**parse_opts), pa.csv.ConvertOptions(**convert_opts) - ) - schema_dict = {field.name: field.type for field in streaming_reader.schema} - - except Exception as e: - # we pass the traceback up otherwise the main process won't know the exact method+line of error - return (None, e) - else: - return (schema_dict, None) - - # boto3 objects can't be pickled (https://github.com/boto/boto3/issues/678) - # and so we can't multiprocess with the actual fileobject on Windows systems - # we're reading block_size*2 bytes here, which we can then pass in and infer schema from block_size bytes - # the *2 is to give us a buffer as pyarrow figures out where lines actually end so it gets schema correct - schema_dict = self._get_schema_dict(file, infer_schema_process) - return self.json_schema_to_pyarrow_schema(schema_dict, reverse=True) # type: ignore[no-any-return] - - def _get_schema_dict(self, file: Union[TextIO, BinaryIO], infer_schema_process: Callable) -> Mapping[str, Any]: - if not self.format.infer_datatypes: - return self._get_schema_dict_without_inference(file) - self.logger.debug("inferring schema") - file_sample = file.read(self._read_options()["block_size"] * 2) # type: ignore[arg-type] - return run_in_external_process( - fn=infer_schema_process, - timeout=4, - max_timeout=60, - logger=self.logger, - args=[ - file_sample, - self._read_options(), - self._parse_options(), - self._convert_options(), - ], - ) - - # TODO Rename this here and in `_get_schema_dict` - def _get_schema_dict_without_inference(self, file: Union[TextIO, BinaryIO]) -> Mapping[str, Any]: - self.logger.debug("infer_datatypes is False, skipping infer_schema") - delimiter = self.format.delimiter - quote_char = self.format.quote_char - reader = csv.reader([six.ensure_text(file.readline())], delimiter=delimiter, quotechar=quote_char) - field_names = next(reader) - file.seek(0) # the file may be reused later so return the cursor to the very beginning of the file as if nothing happened here - return {field_name.strip(): pyarrow.string() for field_name in field_names} - - @wrap_exception((ValueError,)) - def stream_records(self, file: Union[TextIO, BinaryIO], file_info: FileInfo) -> Iterator[Mapping[str, Any]]: - """ - https://arrow.apache.org/docs/python/generated/pyarrow.csv.open_csv.html - PyArrow returns lists of values for each column so we zip() these up into records which we then yield - """ - # In case master_schema is a user defined schema, it may miss some columns. - # We set their type to `string` as a default type in order to pass a schema with all the file columns to pyarrow - # so that pyarrow wouldn't need to infer data types of missing columns. Type inference may often break syncs: - # it reads a block of data and makes suggestions of its type based on that block. So if the next block contains data - # of different type, things get broken. To fix it you either have to increase block size or pass a predefined schema. - # Even if actual data type is changed because of this hack, it will not break sync because this data is written - # to `_ab_additional_properties` column which is not strictly typed ({'type': 'object'}). That's why this is helpful - # when a schema is defined by user and there's no space to increase a block size. - schema = self._get_schema_dict_without_inference(file) - schema.update(self._master_schema) - - streaming_reader = pa_csv.open_csv( - file, - pa.csv.ReadOptions(**self._read_options()), - pa.csv.ParseOptions(**self._parse_options()), - pa.csv.ConvertOptions(**self._convert_options(schema)), - ) - still_reading = True - while still_reading: - try: - batch = streaming_reader.read_next_batch() - except ArrowInvalid as e: - error_message = "Possibly too small block size used. Please try to increase it" - raise AirbyteTracedException(message=error_message, failure_type=FailureType.config_error) from e - except StopIteration: - still_reading = False - else: - batch_dict = batch.to_pydict() - batch_columns = [col_info.name for col_info in batch.schema] - # this gives us a list of lists where each nested list holds ordered values for a single column - # e.g. [ [1,2,3], ["a", "b", "c"], [True, True, False] ] - columnwise_record_values = [batch_dict[column] for column in batch_columns] - # we zip this to get row-by-row, e.g. [ [1, "a", True], [2, "b", True], [3, "c", False] ] - for record_values in zip(*columnwise_record_values): - # create our record of {col: value, col: value} by dict comprehension, iterating through all cols in batch_columns - yield {batch_columns[i]: record_values[i] for i in range(len(batch_columns))} - - @classmethod - def set_minimal_block_size(cls, format: Mapping[str, Any]): - pass diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/jsonl_parser.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/jsonl_parser.py deleted file mode 100644 index 13558eea9dbc..000000000000 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/jsonl_parser.py +++ /dev/null @@ -1,116 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import logging -from typing import Any, BinaryIO, Iterator, Mapping, TextIO, Union - -import pyarrow as pa -from pyarrow import ArrowNotImplementedError -from pyarrow import json as pa_json -from source_s3.source_files_abstract.file_info import FileInfo - -from .abstract_file_parser import AbstractFileParser -from .jsonl_spec import JsonlFormat - -logger = logging.getLogger("airbyte") - - -class JsonlParser(AbstractFileParser): - TYPE_MAP = { - "boolean": ("bool_", "bool"), - "integer": ("int64", "int8", "int16", "int32", "uint8", "uint16", "uint32", "uint64"), - "number": ("float64", "float16", "float32", "decimal128", "decimal256", "halffloat", "float", "double"), - "string": ("large_string", "string"), - # TODO: support object type rather than coercing to string - "object": ( - "struct", - "large_string", - ), - # TODO: support array type rather than coercing to string - "array": ( - "list", - "large_string", - ), - "null": ("large_string",), - } - - def __init__(self, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) - self.format_model = None - - @property - def is_binary(self) -> bool: - return True - - @property - def format(self) -> JsonlFormat: - if self.format_model is None: - self.format_model = JsonlFormat.parse_obj(self._format) - return self.format_model - - def _read_options(self) -> Mapping[str, str]: - """ - https://arrow.apache.org/docs/python/generated/pyarrow.json.ReadOptions.html - build ReadOptions object like: pa.json.ReadOptions(**self._read_options()) - Disable block size parameter if it set to 0. - """ - return {**{"block_size": self.format.block_size if self.format.block_size else None, "use_threads": True}} - - def _parse_options(self, json_schema: Mapping[str, Any] = None) -> Mapping[str, str]: - """ - https://arrow.apache.org/docs/python/generated/pyarrow.json.ParseOptions.html - build ParseOptions object like: pa.json.ParseOptions(**self._parse_options()) - :param json_schema: if this is passed in, pyarrow will attempt to enforce this schema on read, defaults to None - """ - parse_options = { - "newlines_in_values": self.format.newlines_in_values, - "unexpected_field_behavior": self.format.unexpected_field_behavior, - } - if json_schema: - schema = self.json_schema_to_pyarrow_schema(json_schema) - schema = pa.schema({field: type_ for field, type_ in schema.items() if type_ not in self.NON_SCALAR_TYPES.values()}) - parse_options["explicit_schema"] = schema - return parse_options - - def _read_table(self, file: Union[TextIO, BinaryIO], json_schema: Mapping[str, Any] = None) -> pa.Table: - try: - return pa_json.read_json( - file, pa.json.ReadOptions(**self._read_options()), pa.json.ParseOptions(**self._parse_options(json_schema)) - ) - except ArrowNotImplementedError as e: - message = "Possibly too small block size used. Please try to increase it or set to 0 disable this feature." - logger.warning(message) - raise ValueError(message) from e - - def get_inferred_schema(self, file: Union[TextIO, BinaryIO], file_info: FileInfo) -> Mapping[str, Any]: - """ - https://arrow.apache.org/docs/python/generated/pyarrow.json.read_json.html - Json reader support multi thread hence, donot need to add external process - https://arrow.apache.org/docs/python/generated/pyarrow.json.ReadOptions.html - """ - - def field_type_to_str(type_: Any) -> str: - if isinstance(type_, pa.lib.StructType): - return "struct" - if isinstance(type_, pa.lib.ListType): - return "list" - if isinstance(type_, pa.lib.DataType): - return str(type_) - raise Exception(f"Unknown PyArrow Type: {type_}") - - table = self._read_table(file) - schema_dict = {field.name: field_type_to_str(field.type) for field in table.schema} - return self.json_schema_to_pyarrow_schema(schema_dict, reverse=True) - - def stream_records(self, file: Union[TextIO, BinaryIO], file_info: FileInfo) -> Iterator[Mapping[str, Any]]: - """ - https://arrow.apache.org/docs/python/generated/pyarrow.json.read_json.html - - """ - table = self._read_table(file, self._master_schema) - yield from table.to_pylist() - - @classmethod - def set_minimal_block_size(cls, format: Mapping[str, Any]): - format["block_size"] = 0 diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/parquet_parser.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/parquet_parser.py deleted file mode 100644 index f78cdfaa9848..000000000000 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/parquet_parser.py +++ /dev/null @@ -1,160 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import os -from typing import Any, BinaryIO, Iterator, List, Mapping, TextIO, Tuple, Union -from urllib.parse import unquote - -import pyarrow.parquet as pq -from airbyte_cdk.models import FailureType -from pyarrow.parquet import ParquetFile -from source_s3.exceptions import S3Exception -from source_s3.source_files_abstract.file_info import FileInfo - -from .abstract_file_parser import AbstractFileParser -from .parquet_spec import ParquetFormat - -# All possible parquet data types -PARQUET_TYPES = { - # logical_type: (json_type, parquet_types, convert_function) - # standard types - "string": ("string", ["BYTE_ARRAY"], None), - "boolean": ("boolean", ["BOOLEAN"], None), - "number": ("number", ["DOUBLE", "FLOAT"], None), - "integer": ("integer", ["INT32", "INT64", "INT96"], None), - "decimal": ("number", ["INT32", "INT64", "FIXED_LEN_BYTE_ARRAY"], None), - # supported by PyArrow types - "timestamp": ("string", ["INT32", "INT64", "INT96"], lambda v: v.isoformat()), - "date": ("string", ["INT32", "INT64", "INT96"], lambda v: v.isoformat()), - "time": ("string", ["INT32", "INT64", "INT96"], lambda v: v.isoformat()), -} - - -class ParquetParser(AbstractFileParser): - """Apache Parquet is a free and open-source column-oriented data storage format of the Apache Hadoop ecosystem. - - Docs: https://parquet.apache.org/documentation/latest/ - """ - - is_binary = True - - def __init__(self, *args: Any, **kwargs: Any): - super().__init__(*args, **kwargs) - - # adds default values if necessary attributes are skipped. - for field_name, field in ParquetFormat.__fields__.items(): - if self._format.get(field_name) is not None: - continue - self._format[field_name] = field.default - - def _select_options(self, *names: List[str]) -> dict: - return {name: self._format[name] for name in names} - - def _init_reader(self, file: Union[TextIO, BinaryIO]) -> ParquetFile: - """Generates a new parquet reader - Doc: https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetFile.html - - """ - options = self._select_options("buffer_size") # type: ignore[arg-type] - # Source is a file path and enabling memory_map can improve performance in some environments. - options["memory_map"] = True - return pq.ParquetFile(file, **options) - - @staticmethod - def parse_field_type(needed_logical_type: str, need_physical_type: str = None) -> Tuple[str, str]: - """Pyarrow can parse/support non-JSON types - Docs: https://github.com/apache/arrow/blob/5aa2901beddf6ad7c0a786ead45fdb7843bfcccd/python/pyarrow/_parquet.pxd#L56 - """ - if needed_logical_type not in PARQUET_TYPES: - # by default the pyarrow library marks scalar types as 'none' logical type. - # For these cases we need to look for by a physical type - for logical_type, (json_type, physical_types, _) in PARQUET_TYPES.items(): - if need_physical_type in physical_types: - return json_type, logical_type - else: - json_type, physical_types, _ = PARQUET_TYPES[needed_logical_type] - if need_physical_type and need_physical_type not in physical_types: - raise TypeError(f"incorrect parquet physical type: {need_physical_type}; logical type: {needed_logical_type}") - return json_type, needed_logical_type - - raise TypeError(f"incorrect parquet physical type: {need_physical_type}; logical type: {needed_logical_type}") - - @staticmethod - def convert_field_data(logical_type: str, field_value: Any) -> Any: - """Converts not JSON format to JSON one""" - if field_value is None: - return None - if logical_type in PARQUET_TYPES: - _, _, func = PARQUET_TYPES[logical_type] - return func(field_value) if func else field_value - raise TypeError(f"unsupported field type: {logical_type}, value: {field_value}") - - def get_inferred_schema(self, file: Union[TextIO, BinaryIO], file_info: FileInfo) -> dict: - """ - https://arrow.apache.org/docs/python/parquet.html#finer-grained-reading-and-writing - - A stored schema is a part of metadata and we can extract it without parsing of full file - """ - reader = self._init_reader(file) - schema_dict = { - field.name: self.parse_field_type(field.logical_type.type.lower(), field.physical_type)[0] for field in reader.schema - } | {x: "string" for x in self.get_partition_columns(file_info.key)} - if not schema_dict: - # pyarrow can parse empty parquet files but a connector can't generate dynamic schema - raise S3Exception(file_info, "empty Parquet file", "The .parquet file is empty!", FailureType.config_error) - return schema_dict - - def stream_records(self, file: Union[TextIO, BinaryIO], file_info: FileInfo) -> Iterator[Mapping[str, Any]]: - """ - https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetFile.html - PyArrow reads streaming batches from a Parquet file - """ - - reader = self._init_reader(file) - self.logger.info(f"found {reader.num_row_groups} row groups") - # parsing logical_types with respect to master_schema column names. - logical_types = { - field.name: self.parse_field_type(field.logical_type.type.lower(), field.physical_type)[1] - for field in reader.schema - if field.name in self._master_schema - } - if not reader.schema: - # pyarrow can parse empty parquet files but a connector can't generate dynamic schema - raise S3Exception(file_info, "empty Parquet file", "The .parquet file is empty!", FailureType.config_error) - - args = self._select_options("columns", "batch_size") # type: ignore[arg-type] - self.logger.debug(f"Found the {reader.num_row_groups} Parquet groups") - partition_columns = self.get_partition_columns(file_info.key) - # load batches per page - for num_row_group in range(reader.num_row_groups): - args["row_groups"] = [num_row_group] - for batch in reader.iter_batches(**args): - # this gives us a dist of lists where each nested list holds ordered values for a single column - # {'number': [1.0, 2.0, 3.0], 'name': ['foo', None, 'bar'], 'flag': [True, False, True], 'delta': [-1.0, 2.5, 0.1]} - batch_dict = batch.to_pydict() - # sometimes the batch file has more columns than master_schema declares, like: - # master schema: ['number', 'name', 'flag', 'delta'], - # batch_file_schema: ['number', 'name', 'flag', 'delta', 'EXTRA_COL_NAME']. - # we need to check wether batch_file_schema == master_schema and reject extra columns, otherwise "KeyError" raises. - batch_columns = [column for column in batch_dict.keys() if column in self._master_schema] - columnwise_record_values = [batch_dict[column] for column in batch_columns if column in self._master_schema] - # we zip this to get row-by-row - for record_values in zip(*columnwise_record_values): - yield { - batch_columns[i]: self.convert_field_data(logical_types[batch_columns[i]], record_values[i]) - for i in range(len(batch_columns)) - } | partition_columns - - @staticmethod - def get_partition_columns(file_path: str) -> Mapping[str, Any]: - """ - Parse file path and return dict of partitioned columns names with values, example: - /payroll/Year=2014/Agency_Name=ADMIN/file.parquet -> {"Year": "2014", Agency_Name: "ADMIN"} - """ - partitions_in_path = (unquote(x) for x in file_path.split(os.sep) if "=" in x) - return {x.split("=")[0]: x.split("=")[1] for x in partitions_in_path} - - @classmethod - def set_minimal_block_size(cls, format: Mapping[str, Any]): - format["buffer_size"] = 2 diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/storagefile.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/storagefile.py deleted file mode 100644 index d671ca36c556..000000000000 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/storagefile.py +++ /dev/null @@ -1,60 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from abc import ABC, abstractmethod -from contextlib import contextmanager -from datetime import datetime -from typing import BinaryIO, Iterator, TextIO, Union - -from airbyte_cdk.logger import AirbyteLogger - -from .file_info import FileInfo - - -class StorageFile(ABC): - logger = AirbyteLogger() - - def __init__(self, file_info: FileInfo, provider: dict): - """ - :param url: value yielded by filepath_iterator() in [Incremental]FileStream class. Blob/File path. - :param provider: provider specific mapping as described in spec.json - """ - self.file_info = file_info - self._provider = provider - - @property - def last_modified(self) -> datetime: - """ - Returns last_modified property of the blob/file - """ - return self.file_info.last_modified - - @property - def file_size(self) -> int: - """ - Returns Size property of the blob/file - """ - return self.file_info.size - - @property - def url(self) -> str: - """ - Returns key/name files - This function is needed for backward compatibility - """ - return self.file_info.key - - @contextmanager - @abstractmethod - def open(self, binary: bool) -> Iterator[Union[TextIO, BinaryIO]]: - """ - Override this to implement provider-specific logic. - It should yield exactly one TextIO or BinaryIO, that being the opened file-like object. - Note: This must work as described in https://docs.python.org/3/library/contextlib.html#contextlib.contextmanager. - Using contextmanager eliminates need to write all the boilerplate management code in this class. - See S3File() for example implementation. - - :param binary: whether or not to open file as binary - :return: file-like object - """ diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/stream.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/stream.py deleted file mode 100644 index e8110eb91191..000000000000 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/stream.py +++ /dev/null @@ -1,525 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -import json -from abc import ABC, abstractmethod -from collections import defaultdict -from datetime import datetime, timedelta -from functools import cached_property, lru_cache -from traceback import format_exc -from typing import Any, Dict, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Union - -import pendulum -import pytz -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models.airbyte_protocol import SyncMode -from airbyte_cdk.sources.streams import Stream -from wcmatch.glob import GLOBSTAR, SPLIT, globmatch - -from .file_info import FileInfo -from .formats.abstract_file_parser import AbstractFileParser -from .formats.avro_parser import AvroParser -from .formats.csv_parser import CsvParser -from .formats.jsonl_parser import JsonlParser -from .formats.parquet_parser import ParquetParser -from .storagefile import StorageFile - -JSON_TYPES = ["string", "number", "integer", "object", "array", "boolean", "null"] - -LOGGER = AirbyteLogger() - - -class ConfigurationError(Exception): - """Client mis-configured""" - - -class FileStream(Stream, ABC): - file_formatparser_map = { - "csv": CsvParser, - "parquet": ParquetParser, - "avro": AvroParser, - "jsonl": JsonlParser, - } - # TODO: make these user configurable in spec.json - ab_last_mod_col = "_ab_source_file_last_modified" - ab_file_name_col = "_ab_source_file_url" - airbyte_columns = [ab_last_mod_col, ab_file_name_col] - datetime_format_string = "%Y-%m-%dT%H:%M:%SZ" - # In version 2.0.1 the datetime format has been changed. Since the state may still store values in the old datetime format, - # we need to support both of them for a while - deprecated_datetime_format_string = "%Y-%m-%dT%H:%M:%S%z" - # Handle the datetime format used in V4, in the event that we need to roll back - v4_datetime_format_string = "%Y-%m-%dT%H:%M:%S.%fZ" - - def __init__(self, dataset: str, provider: dict, format: dict, path_pattern: str, schema: str = None): - """ - :param dataset: table name for this stream - :param provider: provider specific mapping as described in spec.json - :param format: file format specific mapping as described in spec.json - :param path_pattern: glob-style pattern for file-matching (https://facelessuser.github.io/wcmatch/glob/) - :param schema: JSON-syntax user provided schema, defaults to None - """ - self.dataset = dataset - self._path_pattern = path_pattern - self._provider = provider - self._format = format - self._user_input_schema: Dict[str, Any] = {} - self.start_date = pendulum.parse(provider.get("start_date")) if provider.get("start_date") else pendulum.from_timestamp(0) - if schema: - self._user_input_schema = self._parse_user_input_schema(schema) - LOGGER.info(f"initialised stream with format: {format}") - - @staticmethod - def _parse_user_input_schema(schema: str) -> Dict[str, Any]: - """ - If the user provided a schema, we run this method to convert to a python dict and verify it - This verifies: - - that the provided string is valid JSON - - that it is a key:value map with no nested values (objects or arrays) - - that all values in the map correspond to a JsonSchema datatype - If this passes, we are confident that the user-provided schema is valid and will work as expected with the rest of the code - - :param schema: JSON-syntax user provided schema - :raises ConfigurationError: if any of the verification steps above fail - :return: the input schema (json string) as a python dict - """ - try: - py_schema: Dict[str, Any] = json.loads(schema) - except json.decoder.JSONDecodeError as err: - error_msg = f"Failed to parse schema {repr(err)}\n{schema}\n{format_exc()}" - raise ConfigurationError(error_msg) from err - # enforce all keys and values are of type string as required (i.e. no nesting) - if not all(isinstance(k, str) and isinstance(v, str) for k, v in py_schema.items()): - raise ConfigurationError("Invalid schema provided, all column names and datatypes must be in string format") - # enforce all values (datatypes) are valid JsonSchema datatypes - if any(datatype not in JSON_TYPES for datatype in py_schema.values()): - raise ConfigurationError(f"Invalid schema provided, datatypes must each be one of {JSON_TYPES}") - - return py_schema - - @classmethod - def with_minimal_block_size(cls, config: MutableMapping[str, Any]): - file_type = config["format"]["filetype"] - file_reader = cls.file_formatparser_map[file_type] - file_reader.set_minimal_block_size(config["format"]) - return cls(**config) - - @property - def name(self) -> str: - return self.dataset - - @property - def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: - return None - - @property - def fileformatparser_class(self) -> type: - """ - :return: reference to the relevant fileformatparser class e.g. CsvParser - """ - filetype = self._format.get("filetype") - file_reader = self.file_formatparser_map.get(filetype) - if not file_reader: - raise RuntimeError( - f"Detected mismatched file format '{filetype}'. Available values: '{list(self.file_formatparser_map.keys())}''." - ) - return file_reader - - @property - @abstractmethod - def storagefile_class(self) -> type: - """ - Override this to point to the relevant provider-specific StorageFile class e.g. S3File - - :return: reference to relevant class - """ - - @abstractmethod - def filepath_iterator(self, stream_state: Mapping[str, Any] = None) -> Iterator[FileInfo]: - """ - Provider-specific method to iterate through bucket/container/etc. and yield each full filepath. - This should supply the 'FileInfo' to use in StorageFile(). This is aggrigate all file properties (last_modified, key, size). - All this meta options are saved during loading of files' list at once. - - :yield: FileInfo object to use in StorageFile() - """ - - def pattern_matched_filepath_iterator(self, file_infos: Iterable[FileInfo]) -> Iterator[FileInfo]: - """ - iterates through iterable file_infos and yields only those file_infos that match user-provided path patterns - - :param file_infos: filepath_iterator(), this is a param rather than method reference in order to unit test this - :yield: FileInfo object to use in StorageFile(), if matching on user-provided path patterns - """ - for file_info in file_infos: - if globmatch(file_info.key, self._path_pattern, flags=GLOBSTAR | SPLIT): - yield file_info - - @lru_cache(maxsize=None) - def get_time_ordered_file_infos(self, stream_state: str = None) -> List[FileInfo]: - """ - Iterates through pattern_matched_filepath_iterator(), acquiring FileInfo objects - with last_modified property of each file to return in time ascending order. - Caches results after first run of method to avoid repeating network calls as this is used more than once - - :return: list in time-ascending order - """ - stream_state = eval(stream_state) if stream_state else None - return sorted( - self.pattern_matched_filepath_iterator(self.filepath_iterator(stream_state=stream_state)), - key=lambda file_info: file_info.last_modified, - ) - - @property - def _raw_schema(self) -> Mapping[str, Any]: - if self._user_input_schema and isinstance(self._user_input_schema, dict): - return self._user_input_schema - return self._auto_inferred_schema - - @property - def _schema(self) -> Mapping[str, Any]: - extra_fields = { - self.ab_last_mod_col: {"type": "string"}, - self.ab_file_name_col: {"type": "string"}, - } - schema = self._raw_schema - return {**schema, **extra_fields} - - def get_json_schema(self) -> Mapping[str, Any]: - # note: making every non-airbyte column nullable for compatibility - properties: Mapping[str, Any] = ( - {column: {"type": ["null", typ]} if column not in self.airbyte_columns else typ for column, typ in self._schema.items()} - if self._format["filetype"] != "avro" - else self._schema - ) - properties[self.ab_last_mod_col]["format"] = "date-time" - return {"type": "object", "properties": properties} - - @cached_property - def _auto_inferred_schema(self) -> Dict[str, Any]: - file_reader = self.fileformatparser_class(self._format) - file_info_iterator = iter(list(self.get_time_ordered_file_infos())) - file_info = next(file_info_iterator, None) - if not file_info: - return {} - storage_file = self.storagefile_class(file_info, self._provider) - with storage_file.open(file_reader.is_binary) as f: - return file_reader.get_inferred_schema(f, file_info) - - def stream_slices( - self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Dict[str, Any]]]: - """ - This builds full-refresh stream_slices regardless of sync_mode param. - For full refresh, 1 file == 1 stream_slice. - The structure of a stream slice is [ {file}, ... ]. - In incremental mode, a stream slice may have more than one file so we mirror that format here. - Incremental stream_slices are implemented in the IncrementalFileStream child class. - """ - - # TODO: this could be optimised via concurrent reads, however we'd lose chronology and need to deal with knock-ons of that - # we could do this concurrently both full and incremental by running batches in parallel - # and then incrementing the cursor per each complete batch - for file_info in self.get_time_ordered_file_infos(): - yield {"files": [{"storage_file": self.storagefile_class(file_info, self._provider)}]} - - def _match_target_schema(self, record: Dict[str, Any], target_columns: List) -> Dict[str, Any]: - """ - This method handles missing or additional fields in each record, according to the provided target_columns. - All missing fields are added, with a value of None (null) - All additional fields are packed into the _ab_additional_properties object column - We start off with a check to see if we're already lined up to target in order to avoid unnecessary iterations (useful if many columns) - - :param record: json-like representation of a data row {column:value} - :param target_columns: list of column names to mutate this record into (obtained via self._schema.keys() as of now) - :return: mutated record with columns lining up to target_columns - """ - compare_columns = [c for c in target_columns if c not in [self.ab_last_mod_col, self.ab_file_name_col]] # missing columns - for c in compare_columns: - if c not in record.keys(): - record[c] = None - for c in record.copy(): - if c not in compare_columns: - del record[c] - return record - - def _add_extra_fields_from_map(self, record: Dict[str, Any], extra_map: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Simple method to take a mapping of columns:values and add them to the provided record - - :param record: json-like representation of a data row {column:value} - :param extra_map: map of additional columns and values to add - :return: mutated record with additional fields - """ - for key, value in extra_map.items(): - record[key] = value - return record - - def _read_from_slice( - self, - file_reader: AbstractFileParser, - stream_slice: Mapping[str, Any], - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Mapping[str, Any]]: - """ - Uses provider-relevant StorageFile to open file and then iterates through stream_records() using format-relevant AbstractFileParser. - Records are mutated on the fly using _match_target_schema() and _add_extra_fields_from_map() to achieve desired final schema. - Since this is called per stream_slice, this method works for both full_refresh and incremental. - """ - for file_item in stream_slice["files"]: - storage_file: StorageFile = file_item["storage_file"] - LOGGER.info(f"Reading from file: {storage_file.file_info}") - try: - with storage_file.open(file_reader.is_binary) as f: - # TODO: make this more efficient than mutating every record one-by-one as they stream - for record in file_reader.stream_records(f, storage_file.file_info): - schema_matched_record = self._match_target_schema(record, list(self._schema.keys())) - complete_record = self._add_extra_fields_from_map( - schema_matched_record, - { - self.ab_last_mod_col: datetime.strftime(storage_file.last_modified, self.datetime_format_string), - self.ab_file_name_col: storage_file.url, - }, - ) - yield complete_record - except OSError: - continue - LOGGER.info("finished reading a stream slice") - - def read_records( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Mapping[str, Any]]: - """ - The heavy lifting sits in _read_from_slice() which is full refresh / incremental agnostic - """ - if stream_slice: - file_reader = self.fileformatparser_class(self._format, self._raw_schema) - yield from self._read_from_slice(file_reader, stream_slice) - - -class IncrementalFileStream(FileStream, ABC): - # TODO: ideally want to checkpoint after every file or stream slice rather than N records - state_checkpoint_interval = None - buffer_days = 3 # keeping track of all files synced in the last N days - sync_all_files_always = False - max_history_size = 1000000000 - - @property - def cursor_field(self) -> str: - """ - :return: The name of the cursor field. - """ - return self.ab_last_mod_col - - @staticmethod - def file_in_history(file_key: str, history: dict) -> bool: - return any(file_key in slot for slot in history.values()) - - def _get_datetime_from_stream_state(self, stream_state: Mapping[str, Any] = None) -> datetime: - """ - Returns the datetime from the stream state. - - If there is no state, defaults to 1970-01-01 in order to pick up all files present. - The datetime object is localized to UTC to match the timezone of the last_modified attribute of objects in S3. - """ - stream_state = self._get_converted_stream_state(stream_state) - if stream_state is not None and self.cursor_field in stream_state.keys(): - try: - state_datetime = datetime.strptime(stream_state[self.cursor_field], self.datetime_format_string) - except ValueError: - state_datetime = datetime.strptime(stream_state[self.cursor_field], self.deprecated_datetime_format_string) - else: - state_datetime = datetime.strptime("1970-01-01T00:00:00Z", self.datetime_format_string) - return state_datetime.astimezone(pytz.utc) - - def get_updated_history(self, current_stream_state, latest_record_datetime, latest_record, current_parsed_datetime, state_date): - """ - History is dict which basically groups files by their modified_at date. - After reading each record we add its file to the history set if it wasn't already there. - Then we drop from the history set any entries whose key is less than now - buffer_days - """ - - history = current_stream_state.get("history", {}) - - file_modification_date = latest_record_datetime.strftime("%Y-%m-%d") - - # add record to history if record modified date in range delta start from state - if latest_record_datetime.date() + timedelta(days=self.buffer_days) >= state_date: - history_item = set(history.setdefault(file_modification_date, set())) - history_item.add(latest_record[self.ab_file_name_col]) - history[file_modification_date] = history_item - - # reset history to new date state - if current_parsed_datetime.date() != state_date: - history = { - date: history[date] - for date in history - if datetime.strptime(date, "%Y-%m-%d").date() + timedelta(days=self.buffer_days) >= state_date - } - - return history - - def size_history_balancer(self, state_dict): - """ - Delete history if state size limit reached - """ - history = state_dict["history"] - - if history.__sizeof__() > self.max_history_size: - self.sync_all_files_always = True - state_dict.pop("history") - - return state_dict - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Inspects the latest record extracted from the data source and the current state object and return an updated state object. - In the case where current_stream_state is null, we default to 1970-01-01 in order to pick up all files present. - We also save the schema into the state here so that we can use it on future incremental batches, allowing for additional/missing columns. - - :param current_stream_state: The stream's current state object - :param latest_record: The latest record extracted from the stream - :return: An updated state object - """ - state_dict: Dict[str, Any] = {} - current_parsed_datetime = self._get_datetime_from_stream_state(current_stream_state) - latest_record_datetime = datetime.strptime( - latest_record.get(self.cursor_field, "1970-01-01T00:00:00Z"), self.datetime_format_string - ) - latest_record_datetime = latest_record_datetime.astimezone(pytz.utc) - state_dict[self.cursor_field] = datetime.strftime(max(current_parsed_datetime, latest_record_datetime), self.datetime_format_string) - - state_date = self._get_datetime_from_stream_state(state_dict).date() - - if not self.sync_all_files_always: - state_dict["history"] = self.get_updated_history( - current_stream_state, latest_record_datetime, latest_record, current_parsed_datetime, state_date - ) - - return self.size_history_balancer(state_dict) - - def stream_slices( - self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Dict[str, Any]]]: - """ - Builds either full_refresh or incremental stream_slices based on sync_mode. - An incremental stream_slice is a group of all files with the exact same last_modified timestamp. - This ensures we only update the cursor state to a given timestamp after ALL files with that timestamp have been successfully read. - - Slight nuance: as we iterate through get_time_ordered_file_infos(), - we yield the stream_slice containing file(s) up to and Excluding the file on the current iteration. - The stream_slice is then cleared (if we yielded it) and this iteration's file appended to the (next) stream_slice - """ - if sync_mode == SyncMode.full_refresh: - yield from super().stream_slices(sync_mode=sync_mode, cursor_field=cursor_field, stream_state=stream_state) - - else: - # logic here is to bundle all files with exact same last modified timestamp together in each slice - prev_file_last_mod: datetime = None # init variable to hold previous iterations last modified - grouped_files_by_time: List[Dict[str, Any]] = [] - for file_info in self.get_time_ordered_file_infos(stream_state=str(stream_state)): - # check if this file belongs in the next slice, if so yield the current slice before this file - if (prev_file_last_mod is not None) and (file_info.last_modified != prev_file_last_mod): - yield {"files": grouped_files_by_time} - grouped_files_by_time.clear() - - # now we either have an empty stream_slice or a stream_slice that this file shares a last modified with, so append it - grouped_files_by_time.append({"storage_file": self.storagefile_class(file_info, self._provider)}) - # update our prev_file_last_mod to the current one for next iteration - prev_file_last_mod = file_info.last_modified - - # now yield the final stream_slice. This is required because our loop only yields the slice previous to its current iteration. - if len(grouped_files_by_time) > 0: - yield {"files": grouped_files_by_time} - else: - # in case we have no files - yield None - - def _is_v4_state_format(self, stream_state: Optional[dict]) -> bool: - """ - Returns True if the stream_state is in the v4 format, otherwise False. - - The stream_state is in the v4 format if the history dictionary is a map - of str to str (instead of str to list) and the cursor value is in the - format `%Y-%m-%dT%H:%M:%S.%fZ` - """ - if not stream_state: - return False - if history := stream_state.get("history"): - item = list(history.items())[0] - if isinstance(item[-1], str): - return True - else: - return False - if cursor := stream_state.get(self.cursor_field): - try: - datetime.strptime(cursor, self.v4_datetime_format_string) - except ValueError: - return False - else: - return True - return False - - def _get_converted_stream_state(self, stream_state: Optional[dict]) -> dict: - """ - Transform the history from the new format to the old. - - This will only be used in the event that we roll back from v4. - - e.g. - { - "stream_name": { - "history": { - "simple_test.csv": "2022-05-26T17:49:11.000000Z", - "simple_test_2.csv": "2022-05-27T01:01:01.000000Z", - "redshift_result.csv": "2022-05-27T04:22:20.000000Z", - ... - }, - "_ab_source_file_last_modified": "2022-05-27T04:22:20.000000Z_redshift_result.csv" - } - } - => - { - "stream_name": { - "history": { - "2022-05-26": ["simple_test.csv.csv"], - "2022-05-27": ["simple_test_2.csv", "redshift_result.csv"], - ... - } - }, - "_ab_source_file_last_modified": "2022-05-26T09:55:16Z" - } - """ - if not self._is_v4_state_format(stream_state): - return stream_state - - converted_history = defaultdict(list) - - for filename, timestamp in stream_state.get("history", {}).items(): - if date_str := self._get_ts_from_millis_ts(timestamp, "%Y-%m-%d"): - converted_history[date_str].append(filename) - - converted_state = {} - if self.cursor_field in stream_state: - timestamp_millis = stream_state[self.cursor_field].split("_")[0] - converted_state[self.cursor_field] = self._get_ts_from_millis_ts(timestamp_millis, self.datetime_format_string) - if "history" in stream_state: - converted_state["history"] = converted_history - - return converted_state - - def _get_ts_from_millis_ts(self, timestamp: Optional[str], output_format: str) -> Optional[str]: - if not timestamp: - return timestamp - try: - timestamp_millis = datetime.strptime(timestamp, self.v4_datetime_format_string) - except ValueError: - self.logger.warning(f"Unable to parse {timestamp} as v4 timestamp.") - return timestamp - return timestamp_millis.strftime(output_format) diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py index 231ff6935f5c..55c3b5708f59 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/config.py @@ -1,10 +1,11 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from typing import Any, Dict, Optional -from typing import Optional - +import dpath.util from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec +from airbyte_cdk.utils import is_cloud_environment from pydantic import AnyUrl, Field, ValidationError, root_validator @@ -26,7 +27,15 @@ def documentation_url(cls) -> AnyUrl: description="In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper " "permissions. If accessing publicly available data, this field is not necessary.", airbyte_secret=True, - order=1, + order=2, + ) + + role_arn: Optional[str] = Field( + title=f"AWS Role ARN", + default=None, + description="Specifies the Amazon Resource Name (ARN) of an IAM role that you want to use to perform operations " + f"requested using this profile. Set the External ID to the Airbyte workspace ID, which can be found in the URL of this page.", + order=6, ) aws_secret_access_key: Optional[str] = Field( @@ -35,11 +44,15 @@ def documentation_url(cls) -> AnyUrl: description="In order to access private Buckets stored on AWS S3, this connector requires credentials with the proper " "permissions. If accessing publicly available data, this field is not necessary.", airbyte_secret=True, - order=2, + order=3, ) endpoint: Optional[str] = Field( - "", title="Endpoint", description="Endpoint to an S3 compatible service. Leave empty to use AWS.", order=4 + default="", + title="Endpoint", + description="Endpoint to an S3 compatible service. Leave empty to use AWS.", + examples=["my-s3-endpoint.com", "https://my-s3-endpoint.com"], + order=4, ) @root_validator @@ -50,4 +63,24 @@ def validate_optional_args(cls, values): raise ValidationError( "`aws_access_key_id` and `aws_secret_access_key` are both required to authenticate with AWS.", model=Config ) + + if is_cloud_environment(): + endpoint = values.get("endpoint") + if endpoint: + if endpoint.startswith("http://"): # ignore-https-check + raise ValidationError("The endpoint must be a secure HTTPS endpoint.", model=Config) + return values + + @classmethod + def schema(cls, *args: Any, **kwargs: Any) -> Dict[str, Any]: + """ + Generates the mapping comprised of the config fields + """ + schema = super().schema(*args, **kwargs) + + # Hide API processing option until https://github.com/airbytehq/airbyte-platform-internal/issues/10354 is fixed + processing_options = dpath.util.get(schema, "properties/streams/items/properties/format/oneOf/4/properties/processing/oneOf") + dpath.util.set(schema, "properties/streams/items/properties/format/oneOf/4/properties/processing/oneOf", processing_options[:1]) + + return schema diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/legacy_config_transformer.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/legacy_config_transformer.py index 6a0897ae6c77..4d04411a6694 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/legacy_config_transformer.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/legacy_config_transformer.py @@ -29,7 +29,6 @@ def convert(cls, legacy_config: SourceS3Spec) -> Mapping[str, Any]: "streams": [ { "name": legacy_config.dataset, - "file_type": legacy_config.format.filetype, "globs": cls._create_globs(legacy_config.path_pattern), "legacy_prefix": legacy_config.provider.path_prefix, "validation_policy": "Emit Record", @@ -117,8 +116,17 @@ def _transform_file_format(cls, format_options: Union[CsvFormat, ParquetFormat, csv_options["skip_rows_before_header"] = skip_rows if skip_rows_after_names := advanced_options.pop("skip_rows_after_names", None): csv_options["skip_rows_after_header"] = skip_rows_after_names - if autogenerate_column_names := advanced_options.pop("autogenerate_column_names", None): - csv_options["autogenerate_column_names"] = autogenerate_column_names + + if column_names := advanced_options.pop("column_names", None): + csv_options["header_definition"] = { + "header_definition_type": "User Provided", + "column_names": column_names, + } + advanced_options.pop("autogenerate_column_names", None) + elif advanced_options.pop("autogenerate_column_names", None): + csv_options["header_definition"] = {"header_definition_type": "Autogenerated"} + else: + csv_options["header_definition"] = {"header_definition_type": "From CSV"} cls._filter_legacy_noops(advanced_options) @@ -151,7 +159,7 @@ def parse_config_options_str(cls, options_field: str, options_value: Optional[st @staticmethod def _filter_legacy_noops(advanced_options: Dict[str, Any]): - ignore_all = ("auto_dict_encode", "timestamp_parsers") + ignore_all = ("auto_dict_encode", "timestamp_parsers", "block_size") ignore_by_value = (("check_utf8", False),) for option in ignore_all: diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/source.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/source.py index 8302673b3ce7..bcb4ccdffef4 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/source.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/source.py @@ -2,12 +2,23 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import Any, Mapping +from typing import Any, Dict, Mapping, Optional +from airbyte_cdk.config_observation import emit_configuration_as_airbyte_control_message +from airbyte_cdk.models import ConnectorSpecification from airbyte_cdk.sources.file_based.file_based_source import FileBasedSource +from airbyte_cdk.utils import is_cloud_environment from source_s3.source import SourceS3Spec from source_s3.v4.legacy_config_transformer import LegacyConfigTransformer +_V3_DEPRECATION_FIELD_MAPPING = { + "dataset": "streams.name", + "format": "streams.format", + "path_pattern": "streams.globs", + "provider": "bucket, aws_access_key_id, aws_secret_access_key and endpoint", + "schema": "streams.input_schema", +} + class SourceS3(FileBasedSource): def read_config(self, config_path: str) -> Mapping[str, Any]: @@ -17,7 +28,86 @@ def read_config(self, config_path: str) -> Mapping[str, Any]: validate the config against the new spec. """ config = super().read_config(config_path) - if not config.get("streams"): + if not self._is_v4_config(config): parsed_legacy_config = SourceS3Spec(**config) - return LegacyConfigTransformer.convert(parsed_legacy_config) + converted_config = LegacyConfigTransformer.convert(parsed_legacy_config) + emit_configuration_as_airbyte_control_message(converted_config) + return converted_config return config + + def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: + s3_spec = SourceS3Spec.schema() + s4_spec = self.spec_class.schema() + + if s3_spec["properties"].keys() & s4_spec["properties"].keys(): + raise ValueError("Overlapping properties between V3 and V4") + + for v3_property_key, v3_property_value in s3_spec["properties"].items(): + s4_spec["properties"][v3_property_key] = v3_property_value + s4_spec["properties"][v3_property_key]["airbyte_hidden"] = True + s4_spec["properties"][v3_property_key]["order"] += 100 + s4_spec["properties"][v3_property_key]["description"] = ( + SourceS3._create_description_with_deprecation_prefix(_V3_DEPRECATION_FIELD_MAPPING.get(v3_property_key, None)) + + s4_spec["properties"][v3_property_key]["description"] + ) + self._clean_required_fields(s4_spec["properties"][v3_property_key]) + + if is_cloud_environment(): + s4_spec["properties"]["endpoint"].update( + { + "description": "Endpoint to an S3 compatible service. Leave empty to use AWS. " + "The custom endpoint must be secure, but the 'https' prefix is not required.", + "pattern": "^(?!http://).*$", # ignore-https-check + } + ) + + return ConnectorSpecification( + documentationUrl=self.spec_class.documentation_url(), + connectionSpecification=s4_spec, + ) + + def _is_v4_config(self, config: Mapping[str, Any]) -> bool: + return "streams" in config + + @staticmethod + def _clean_required_fields(v3_field: Dict[str, Any]) -> None: + """ + Not having V3 fields root level as part of the `required` field is not enough as the platform will create empty objects for those. + For example, filling all non-hidden fields from the form will create a config like: + ``` + { + <...> + "provider": {}, + <...> + } + ``` + + As the field `provider` exists, the JSON validation will be applied and as `provider.bucket` is needed, the validation will fail + with the following error: + ``` + "errors": { + "connectionConfiguration": { + "provider": { + "bucket": { + "message": "form.empty.error", + "type": "required" + } + } + } + } + ``` + + Hence, we need to make any V3 nested fields not required. + """ + if "properties" not in v3_field: + return + + v3_field["required"] = [] + for neste_field in v3_field["properties"]: + SourceS3._clean_required_fields(neste_field) + + @staticmethod + def _create_description_with_deprecation_prefix(new_fields: Optional[str]) -> str: + if new_fields: + return f"Deprecated and will be removed soon. Please do not use this field anymore and use {new_fields} instead. " + return "Deprecated and will be removed soon. Please do not use this field anymore. " diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py index 14618e84cff9..0457dba4ee36 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/stream_reader.py @@ -3,19 +3,27 @@ # import logging -from contextlib import contextmanager +from datetime import datetime from io import IOBase +from os import getenv from typing import Iterable, List, Optional, Set import boto3.session import pytz import smart_open -from airbyte_cdk.sources.file_based.exceptions import ErrorListingFiles, FileBasedSourceError +from airbyte_cdk.models import FailureType +from airbyte_cdk.sources.file_based.exceptions import CustomFileBasedException, ErrorListingFiles, FileBasedSourceError from airbyte_cdk.sources.file_based.file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode from airbyte_cdk.sources.file_based.remote_file import RemoteFile from botocore.client import BaseClient from botocore.client import Config as ClientConfig +from botocore.credentials import RefreshableCredentials +from botocore.exceptions import ClientError +from botocore.session import get_session from source_s3.v4.config import Config +from source_s3.v4.zip_reader import DecompressedStream, RemoteFileInsideArchive, ZipContentReader, ZipFileHandler + +AWS_EXTERNAL_ID = getenv("AWS_ASSUME_ROLE_EXTERNAL_ID") class SourceS3StreamReader(AbstractFileBasedStreamReader): @@ -48,17 +56,67 @@ def s3_client(self) -> BaseClient: # list or read files. raise ValueError("Source config is missing; cannot create the S3 client.") if self._s3_client is None: - if self.config.endpoint: - client_kv_args = _get_s3_compatible_client_args(self.config) - self._s3_client = boto3.client("s3", **client_kv_args) + client_kv_args = _get_s3_compatible_client_args(self.config) if self.config.endpoint else {} + + if self.config.role_arn: + self._s3_client = self._get_iam_s3_client(client_kv_args) else: self._s3_client = boto3.client( "s3", aws_access_key_id=self.config.aws_access_key_id, aws_secret_access_key=self.config.aws_secret_access_key, + **client_kv_args, ) + return self._s3_client + def _get_iam_s3_client(self, client_kv_args: dict) -> BaseClient: + """ + Creates an S3 client using AWS Security Token Service (STS) with assumed role credentials. This method handles + the authentication process by assuming an IAM role, optionally using an external ID for enhanced security. + The obtained credentials are set to auto-refresh upon expiration, ensuring uninterrupted access to the S3 service. + + :param client_kv_args: A dictionary of key-value pairs for the boto3 S3 client constructor. + :return: An instance of a boto3 S3 client with the assumed role credentials. + + The method assumes a role specified in the `self.config.role_arn` and creates a session with the S3 service. + If `AWS_ASSUME_ROLE_EXTERNAL_ID` environment variable is set, it will be used during the role assumption for additional security. + """ + + def refresh(): + client = boto3.client("sts") + if AWS_EXTERNAL_ID: + role = client.assume_role( + RoleArn=self.config.role_arn, + RoleSessionName="airbyte-source-s3", + ExternalId=AWS_EXTERNAL_ID, + ) + else: + role = client.assume_role( + RoleArn=self.config.role_arn, + RoleSessionName="airbyte-source-s3", + ) + + creds = role.get("Credentials", {}) + return { + "access_key": creds["AccessKeyId"], + "secret_key": creds["SecretAccessKey"], + "token": creds["SessionToken"], + "expiry_time": creds["Expiration"].isoformat(), + } + + session_credentials = RefreshableCredentials.create_from_metadata( + metadata=refresh(), + refresh_using=refresh, + method="sts-assume-role", + ) + + session = get_session() + session._credentials = session_credentials + autorefresh_session = boto3.Session(botocore_session=session) + + return autorefresh_session.client("s3", **client_kv_args) + def get_matching_files(self, globs: List[str], prefix: Optional[str], logger: logging.Logger) -> Iterable[RemoteFile]: """ Get all files matching the specified glob patterns. @@ -69,27 +127,31 @@ def get_matching_files(self, globs: List[str], prefix: Optional[str], logger: lo total_n_keys = 0 try: - if prefixes: - for prefix in prefixes: - for remote_file in self._page(s3, globs, self.config.bucket, prefix, seen, logger): - total_n_keys += 1 - yield remote_file - else: - for remote_file in self._page(s3, globs, self.config.bucket, None, seen, logger): + for current_prefix in prefixes if prefixes else [None]: + for remote_file in self._page(s3, globs, self.config.bucket, current_prefix, seen, logger): total_n_keys += 1 yield remote_file logger.info(f"Finished listing objects from S3. Found {total_n_keys} objects total ({len(seen)} unique objects).") + except ClientError as exc: + if exc.response["Error"]["Code"] == "NoSuchBucket": + raise CustomFileBasedException( + f"The bucket {self.config.bucket} does not exist.", failure_type=FailureType.config_error, exception=exc + ) + self._raise_error_listing_files(globs, exc) except Exception as exc: - raise ErrorListingFiles( - FileBasedSourceError.ERROR_LISTING_FILES, - source="s3", - bucket=self.config.bucket, - globs=globs, - endpoint=self.config.endpoint, - ) from exc - - @contextmanager + self._raise_error_listing_files(globs, exc) + + def _raise_error_listing_files(self, globs: List[str], exc: Optional[Exception] = None): + """Helper method to raise the ErrorListingFiles exception.""" + raise ErrorListingFiles( + FileBasedSourceError.ERROR_LISTING_FILES, + source="s3", + bucket=self.config.bucket, + globs=globs, + endpoint=self.config.endpoint, + ) from exc + def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str], logger: logging.Logger) -> IOBase: try: params = {"client": self.s3_client} @@ -98,17 +160,22 @@ def open_file(self, file: RemoteFile, mode: FileReadMode, encoding: Optional[str logger.debug(f"try to open {file.uri}") try: - result = smart_open.open(f"s3://{self.config.bucket}/{file.uri}", transport_params=params, mode=mode.value, encoding=encoding) + if isinstance(file, RemoteFileInsideArchive): + s3_file_object = smart_open.open(f"s3://{self.config.bucket}/{file.uri.split('#')[0]}", transport_params=params, mode="rb") + decompressed_stream = DecompressedStream(s3_file_object, file) + result = ZipContentReader(decompressed_stream, encoding) + else: + result = smart_open.open( + f"s3://{self.config.bucket}/{file.uri}", transport_params=params, mode=mode.value, encoding=encoding + ) except OSError: logger.warning( f"We don't have access to {file.uri}. The file appears to have become unreachable during sync." f"Check whether key {file.uri} exists in `{self.config.bucket}` bucket and/or has proper ACL permissions" ) - # see https://docs.python.org/3/library/contextlib.html#contextlib.contextmanager for why we do this - try: - yield result - finally: - result.close() + + # we can simply return the result here as it is a context manager itself that will release all resources + return result @staticmethod def _is_folder(file) -> bool: @@ -132,10 +199,11 @@ def _page( for file in response["Contents"]: if self._is_folder(file): continue - remote_file = RemoteFile(uri=file["Key"], last_modified=file["LastModified"].astimezone(pytz.utc).replace(tzinfo=None)) - if self.file_matches_globs(remote_file, globs) and remote_file.uri not in seen: - seen.add(remote_file.uri) - yield remote_file + + for remote_file in self._handle_file(file): + if self.file_matches_globs(remote_file, globs) and remote_file.uri not in seen: + seen.add(remote_file.uri) + yield remote_file else: logger.warning(f"Invalid response from S3; missing 'Contents' key. kwargs={kwargs}.") @@ -145,6 +213,31 @@ def _page( logger.info(f"Finished listing objects from S3 for prefix={prefix}. Found {total_n_keys_for_prefix} objects.") break + def _handle_file(self, file): + if file["Key"].endswith(".zip"): + yield from self._handle_zip_file(file) + else: + yield self._handle_regular_file(file) + + def _handle_zip_file(self, file): + zip_handler = ZipFileHandler(self.s3_client, self.config) + zip_members, cd_start = zip_handler.get_zip_files(file["Key"]) + + for zip_member in zip_members: + remote_file = RemoteFileInsideArchive( + uri=file["Key"] + "#" + zip_member.filename, + last_modified=datetime(*zip_member.date_time).astimezone(pytz.utc).replace(tzinfo=None), + start_offset=zip_member.header_offset + cd_start, + compressed_size=zip_member.compress_size, + uncompressed_size=zip_member.file_size, + compression_method=zip_member.compress_type, + ) + yield remote_file + + def _handle_regular_file(self, file): + remote_file = RemoteFile(uri=file["Key"], last_modified=file["LastModified"].astimezone(pytz.utc).replace(tzinfo=None)) + return remote_file + def _get_s3_compatible_client_args(config: Config) -> dict: """ diff --git a/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py b/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py new file mode 100644 index 000000000000..4f475b80c797 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/source_s3/v4/zip_reader.py @@ -0,0 +1,402 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import io +import struct +import zipfile +from typing import IO, List, Optional, Tuple, Union + +from airbyte_cdk.sources.file_based.remote_file import RemoteFile +from botocore.client import BaseClient +from source_s3.v4.config import Config + +# Buffer constants +BUFFER_SIZE_DEFAULT = 1024 * 1024 +MAX_BUFFER_SIZE_DEFAULT: int = 16 * BUFFER_SIZE_DEFAULT + + +class RemoteFileInsideArchive(RemoteFile): + """ + A file inside archive in a file-based stream. + """ + + start_offset: int + compressed_size: int + uncompressed_size: int + compression_method: int + + +class ZipFileHandler: + """ + Handler class for extracting information from ZIP files stored in AWS S3. + """ + + # Class constants for ZIP file signatures + EOCD_SIGNATURE: bytes = b"\x50\x4b\x05\x06" + ZIP64_LOCATOR_SIGNATURE: bytes = b"\x50\x4b\x06\x07" + + # Standard ZIP constants + EOCD_CENTRAL_DIR_START_OFFSET: int = 16 + + # ZIP64 constants + ZIP64_EOCD_OFFSET: int = 8 + ZIP64_EOCD_SIZE: int = 56 + ZIP64_CENTRAL_DIR_START_OFFSET: int = 48 + + def __init__(self, s3_client: BaseClient, config: Config): + """ + Initialize the ZipFileHandler with an S3 client and configuration. + + :param s3_client: The AWS S3 client. + :param config: Configuration containing bucket and other details. + """ + self.s3_client = s3_client + self.config = config + + def _fetch_data_from_s3(self, filename: str, start: int, size: Optional[int] = None) -> bytes: + """ + Fetch a specific range of bytes from a file in S3. + + :param filename: The name of the file in S3. + :param start: The starting byte position. + :param size: The number of bytes to fetch (optional). + :return: The fetched bytes. + """ + end_range = f"{start + size - 1}" if size else "" + range_str = f"bytes={start}-{end_range}" + response = self.s3_client.get_object(Bucket=self.config.bucket, Key=filename, Range=range_str) + return response["Body"].read() + + def _find_signature( + self, + filename: str, + signature: bytes, + initial_buffer_size: int = BUFFER_SIZE_DEFAULT, + max_buffer_size: int = MAX_BUFFER_SIZE_DEFAULT, + ) -> Optional[bytes]: + """ + Search for a specific signature in the file by checking chunks of increasing size. + If the signature is not found within the max_buffer_size, None is returned. + + :param filename: The name of the file in S3. + :param signature: The byte signature to search for. + :param initial_buffer_size: Initial size of the buffer to search in. + :param max_buffer_size: Maximum size of the buffer to search in. + :return: The chunk of data containing the signature or None if not found. + """ + buffer_size = initial_buffer_size + file_size = self.s3_client.head_object(Bucket=self.config.bucket, Key=filename)["ContentLength"] + + while buffer_size <= max_buffer_size: + chunk = self._fetch_data_from_s3(filename, file_size - buffer_size) + index = chunk.rfind(signature) + if index != -1: + return chunk[index:] + buffer_size *= 2 + return None + + def _fetch_zip64_data(self, filename: str) -> bytes: + """ + Fetch the ZIP64 End of Central Directory (EOCD) data from a ZIP file. + + :param filename: The name of the file in S3. + :return: The ZIP64 EOCD data. + """ + chunk = self._find_signature(filename, self.ZIP64_LOCATOR_SIGNATURE) + zip64_eocd_offset = struct.unpack_from(" int: + """ + Determine the starting position of the central directory in the ZIP file. + Adjusts for ZIP64 format if necessary. + + :param filename: The name of the file in S3. + :return: The starting position of the central directory. + """ + eocd_data = self._find_signature(filename, self.EOCD_SIGNATURE) + central_dir_start = struct.unpack_from(" Tuple[List[zipfile.ZipInfo], int]: + """ + Extract metadata about the files inside a ZIP archive stored in S3. + + :param filename: The name of the ZIP file in S3. + :return: A tuple containing a list of ZipInfo objects representing the files inside the ZIP archive + and the starting position of the central directory. + """ + central_dir_start = self._get_central_directory_start(filename) + central_dir_data = self._fetch_data_from_s3(filename, central_dir_start) + + with io.BytesIO(central_dir_data) as bytes_io: + with zipfile.ZipFile(bytes_io, "r") as zf: + return zf.infolist(), central_dir_start + + +class DecompressedStream(io.IOBase): + """ + A custom stream class that handles decompression of data from a given file object. + This class supports seeking, reading, and other basic file operations on compressed data. + """ + + LOCAL_FILE_HEADER_SIZE: int = 30 + NAME_LENGTH_OFFSET: int = 26 + + def __init__(self, file_obj: IO[bytes], file_info: RemoteFileInsideArchive, buffer_size: int = BUFFER_SIZE_DEFAULT): + """ + Initialize a DecompressedStream. + + :param file_obj: Underlying file-like object. + :param file_info: Meta information about the file inside the archive. + :param buffer_size: Size of the buffer for reading data. + """ + self._file = file_obj + self.file_start = self._calculate_actual_start(file_info.start_offset) + self.compressed_size = file_info.compressed_size + self.uncompressed_size = file_info.uncompressed_size + self.compression_method = file_info.compression_method + self._buffer = bytearray() + self.buffer_size = buffer_size + self._reset_decompressor() + self.position = 0 # Current position in uncompressed stream + self._file.seek(self.file_start) + # Mapping between uncompressed and compressed offsets for quick seeking + self.offset_map = {0: self.file_start, self.uncompressed_size: self.file_start + self.compressed_size} + + def _calculate_actual_start(self, file_start: int) -> int: + """ + Determine the actual start position of the file content within the ZIP archive. + + In a ZIP archive, each file entry is preceded by a local file header. This header contains + metadata about the file, including the lengths of the file's name and any extra data. + To accurately locate the start of the actual file content, we need to skip over this header. + + This method calculates the start position by taking into account the length of the file name + and any extra data present in the local file header. + + :param file_start: The starting position of the file entry (including its local file header) + inside the ZIP archive. + :return: The actual starting position of the file content, after skipping the local file header. + """ + self._file.seek(file_start + self.NAME_LENGTH_OFFSET) # Navigate to the position where lengths of name and extra data are stored + name_len, extra_len = struct.unpack(" bytes: + """ + Decompress a chunk of data based on the compression method. + """ + if self.compression_method == zipfile.ZIP_STORED: + return chunk + return self.decompressor.decompress(chunk) + + def read(self, size: int = -1) -> bytes: + """ + Read a specified number of bytes from the stream. + """ + # Size not specified, read till end + if size == -1: + size = self.uncompressed_size - self.position + + # If buffer already has enough data, return it directly + if size <= len(self._buffer): + data = self._buffer[:size] + self._buffer = self._buffer[size:] + self.position += len(data) + return data + + data = self._buffer + self._buffer = bytearray() + while len(data) < size and self._file.tell() - self.file_start < self.compressed_size: + max_read_size = min(self.buffer_size, self.compressed_size + self.file_start - self._file.tell()) + chunk = self._file.read(max_read_size) + + if not chunk: + break + + decompressed_data = self._decompress_chunk(chunk) + + # Buffer excessive data for future reads + if len(data) + len(decompressed_data) > size: + desired_length = size - len(data) + data += decompressed_data[:desired_length] + self._buffer = decompressed_data[desired_length:] + else: + data += decompressed_data + + self.position += len(data) + return data + + def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: + """ + Seek to a specific position in the uncompressed stream. + """ + if whence == io.SEEK_SET: + self._buffer = bytearray() + elif whence == io.SEEK_CUR: + offset = self.position + offset + elif whence == io.SEEK_END: + offset = self.uncompressed_size + offset + + # Ensure the offset is within the file's boundaries + offset = max(0, min(offset, self.uncompressed_size)) + + closest_offset = max(k for k in self.offset_map if k <= offset) + closest_position = self.offset_map[closest_offset] + + self._file.seek(closest_position) + self._reset_decompressor() + self.position = closest_offset + + # Read till desired offset + while self.position < offset: + read_size = min(self.buffer_size, offset - self.position) + self.read(read_size) + + return self.position + + def tell(self) -> int: + """ + Return the current position in the uncompressed stream. + """ + return self.position + + def readable(self) -> bool: + """ + Return if the stream is readable. + """ + return True + + def seekable(self) -> bool: + """ + Return if the stream is seekable. + """ + return True + + def close(self): + """ + Close the stream and underlying file object. + """ + self._file.close() + + +class ZipContentReader: + """ + A custom reader class that provides buffered reading capabilities on a decompressed stream. + Supports reading lines, reading chunks, and iterating over the content. + """ + + def __init__(self, decompressed_stream: DecompressedStream, encoding: Optional[str] = None, buffer_size: int = BUFFER_SIZE_DEFAULT): + """ + Initialize a ZipContentReader. + + :param decompressed_stream: A DecompressedStream object. + :param encoding: Encoding to decode the bytes. If None, bytes are returned. + :param buffer_size: Size of the buffer for reading data. + """ + self.raw = decompressed_stream + self.encoding = encoding + self.buffer_size = buffer_size + self.buffer = bytearray() + self._closed = False + + def __iter__(self): + """ + Make the class iterable. + """ + return self + + def __next__(self) -> Union[str, bytes]: + """ + Iterate over the lines in the reader. + """ + line = self.readline() + if not line: + raise StopIteration + return line + + def readline(self, limit: int = -1) -> Union[str, bytes]: + """ + Read a single line from the stream. + """ + if limit != -1: + raise NotImplementedError("Limits other than -1 not implemented yet") + + line = "" + while True: + char = self.read(1) + if not char: + break + + line += char + if char in ["\n", "\r"]: + # Handling different types of newlines + next_char = self.read(1) + if char == "\r" and next_char == "\n": + line += next_char + else: + self.buffer = next_char.encode(self.encoding) + self.buffer + break + return line + + def read(self, size: int = -1) -> Union[str, bytes]: + """ + Read a specified number of bytes/characters from the reader. + """ + while len(self.buffer) < size: + chunk = self.raw.read(self.buffer_size) + if not chunk: + break + self.buffer += chunk + + data = self.buffer[:size] + self.buffer = self.buffer[size:] + + return data.decode(self.encoding) if self.encoding else bytes(data) + + def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: + """ + Seek to a specific position in the decompressed stream. + """ + self.buffer = bytearray() + return self.raw.seek(offset, whence) + + def close(self): + """ + Close the reader and underlying decompressed stream. + """ + self._closed = True + self.raw.close() + + def tell(self) -> int: + """ + Return the current position in the decompressed stream. + """ + return self.raw.tell() + + @property + def closed(self) -> bool: + """ + Check if the reader is closed. + """ + return self._closed + + def __enter__(self) -> "ZipContentReader": + """Enter the runtime context for the reader.""" + return self + + def __exit__(self, exc_type, exc_value, traceback) -> None: + """Exit the runtime context for the reader and ensure resources are closed.""" + self.close() diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/__init__.py b/airbyte-integrations/connectors/source-s3/unit_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/__init__.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/abstract_test_parser.py b/airbyte-integrations/connectors/source-s3/unit_tests/abstract_test_parser.py deleted file mode 100644 index d7196b40eff4..000000000000 --- a/airbyte-integrations/connectors/source-s3/unit_tests/abstract_test_parser.py +++ /dev/null @@ -1,160 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import linecache -import os -import random -import sys -import tracemalloc -from abc import ABC, abstractmethod -from datetime import datetime, timedelta -from decimal import Decimal -from functools import lru_cache, wraps -from typing import Any, Callable, List, Mapping - -import pytest -from airbyte_cdk import AirbyteLogger -from smart_open import open as smart_open -from source_s3.source_files_abstract.file_info import FileInfo - - -def memory_limit(max_memory_in_megabytes: int, print_limit: int = 20) -> Callable: - """Runs a test function by a separate process with restricted memory""" - - def decorator(func: Callable) -> Callable: - @wraps(func) - def wrapper(*args: List[Any], **kwargs: Any) -> Any: - tracemalloc.start() - result = func(*args, **kwargs) - - # get memory usage immediately after function call, we interested in "first_size" value - first_size, first_peak = tracemalloc.get_traced_memory() - # get snapshot immediately just in case we will use it - snapshot = tracemalloc.take_snapshot() - - # only if we exceeded the quota, build log_messages with traces - first_size_in_megabytes = first_size / 1024**2 - if first_size_in_megabytes > max_memory_in_megabytes: - log_messages: List[str] = [] - top_stats = snapshot.statistics("lineno") - for index, stat in enumerate(top_stats[:print_limit], 1): - frame = stat.traceback[0] - filename = os.sep.join(frame.filename.split(os.sep)[-2:]) - log_messages.append("#%s: %s:%s: %.1f KiB" % (index, filename, frame.lineno, stat.size / 1024)) - line = linecache.getline(frame.filename, frame.lineno).strip() - if line: - log_messages.append(f" {line}") - traceback_log = "\n".join(log_messages) - assert False, f"Overuse of memory, used: {first_size_in_megabytes}Mb, limit: {max_memory_in_megabytes}Mb!!\n{traceback_log}" - - return result - - return wrapper - - return decorator - - -def create_by_local_file(filepath: str) -> FileInfo: - "Generates a FileInfo instance for local files" - if not os.path.exists(filepath): - return FileInfo(key=filepath, size=0, last_modified=datetime.now()) - return FileInfo(key=filepath, size=os.stat(filepath).st_size, last_modified=datetime.fromtimestamp(os.path.getmtime(filepath))) - - -class AbstractTestParser(ABC): - """Prefix this class with Abstract so the tests don't run here but only in the children""" - - logger = AirbyteLogger() - record_types: Mapping[str, Any] = {} - - @classmethod - def _generate_row(cls, types: List[str]) -> List[Any]: - """Generates random values with request types""" - row = [] - for needed_type in types: - for json_type in cls.record_types: - if json_type == needed_type: - row.append(cls._generate_value(needed_type)) - break - return row - - @classmethod - def _generate_value(cls, typ: str) -> Any: - if typ not in ["boolean", "integer"] and cls._generate_value("boolean"): - # return 'None' for +- 33% of all requests - return None - - if typ == "number": - while True: - int_value = cls._generate_value("integer") - if int_value: - break - return float(int_value) + random.random() - elif typ == "integer": - return random.randint(-sys.maxsize - 1, sys.maxsize) - # return random.randint(0, 1000) - elif typ == "boolean": - return random.choice([True, False, None]) - elif typ == "string": - random_length = random.randint(0, 10 * 1024) # max size of bytes is 10k - return os.urandom(random_length) - elif typ == "timestamp": - return datetime.now() + timedelta(seconds=random.randint(0, 7200)) - elif typ == "date": - dt = cls._generate_value("timestamp") - return dt.date() if dt else None - elif typ == "time": - dt = cls._generate_value("timestamp") - return dt.time() if dt else None - elif typ == "decimal": - return Decimal((0, tuple([random.randint(1, 9) for _ in range(10)]), -4)) - raise Exception(f"not supported type: {typ}") - - @classmethod - @lru_cache(maxsize=None) - def cached_cases(cls) -> Mapping[str, Any]: - return cls.cases() - - @classmethod - @abstractmethod - def cases(cls) -> Mapping[str, Any]: - """return a map of test_file dicts in structure: - { - "small_file": {"AbstractFileParser": CsvParser(format, master_schema), "filepath": "...", "num_records": 5, "inferred_schema": {...}, line_checks:{}, fails: []}, - "big_file": {"AbstractFileParser": CsvParser(format, master_schema), "filepath": "...", "num_records": 16, "inferred_schema": {...}, line_checks:{}, fails: []} - ] - note: line_checks index is 1-based to align with row numbers - """ - - def _get_readmode(self, file_info: Mapping[str, Any]) -> str: - return "rb" if file_info["AbstractFileParser"].is_binary else "r" - - @memory_limit(1024) - def test_suite_inferred_schema(self, file_info: Mapping[str, Any]) -> None: - file_info_instance = FileInfo(key=file_info["filepath"], size=os.stat(file_info["filepath"]).st_size, last_modified=datetime.now()) - with smart_open(file_info["filepath"], self._get_readmode(file_info)) as f: - if "test_get_inferred_schema" in file_info["fails"]: - with pytest.raises(Exception) as e_info: - file_info["AbstractFileParser"].get_inferred_schema(f), file_info_instance - self.logger.debug(str(e_info)) - else: - assert file_info["AbstractFileParser"].get_inferred_schema(f, file_info_instance) == file_info["inferred_schema"] - - @memory_limit(1024) - def test_stream_suite_records(self, file_info: Mapping[str, Any]) -> None: - filepath = file_info["filepath"] - file_size = os.stat(filepath).st_size - file_info_instance = FileInfo(key=filepath, size=file_size, last_modified=datetime.now()) - self.logger.info(f"read the file: {filepath}, size: {file_size / (1024 ** 2)}Mb") - with smart_open(filepath, self._get_readmode(file_info)) as f: - if "test_stream_records" in file_info["fails"]: - with pytest.raises(Exception) as e_info: - [print(r) for r in file_info["AbstractFileParser"].stream_records(f, file_info_instance)] - self.logger.debug(str(e_info)) - else: - records = [r for r in file_info["AbstractFileParser"].stream_records(f, file_info_instance)] - - assert len(records) == file_info["num_records"] - for index, expected_record in file_info["line_checks"].items(): - assert records[index - 1] == expected_record diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/conftest.py b/airbyte-integrations/connectors/source-s3/unit_tests/conftest.py deleted file mode 100644 index c5e174f28986..000000000000 --- a/airbyte-integrations/connectors/source-s3/unit_tests/conftest.py +++ /dev/null @@ -1,77 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import json -import os -import shutil -import tempfile -from pathlib import Path -from typing import Any, List, Mapping - -import requests # noqa -from airbyte_cdk import AirbyteLogger -from netifaces import AF_INET, ifaddresses, interfaces -from pytest import fixture -from requests.exceptions import ConnectionError # noqa -from source_s3 import SourceS3 - -logger = AirbyteLogger() - -TMP_FOLDER = os.path.join(tempfile.gettempdir(), "test_generated") - -shutil.rmtree(TMP_FOLDER, ignore_errors=True) -os.makedirs(TMP_FOLDER, exist_ok=True) - - -def pytest_generate_tests(metafunc: Any) -> None: - if "file_info" in metafunc.fixturenames: - cases = metafunc.cls.cached_cases() - metafunc.parametrize("file_info", cases.values(), ids=cases.keys()) - - -def pytest_sessionfinish(session: Any, exitstatus: Any) -> None: - """whole test run finishes.""" - shutil.rmtree(TMP_FOLDER, ignore_errors=True) - - -@fixture(name="config") -def config_fixture(tmp_path): - config_file = tmp_path / "config.json" - with open(config_file, "w") as fp: - json.dump( - { - "dataset": "dummy", - "provider": {"bucket": "test-test", "endpoint": "test", "use_ssl": "test", "verify_ssl_cert": "test"}, - "path_pattern": "", - "format": {"delimiter": "\\t"}, - }, - fp, - ) - source = SourceS3() - config = source.read_config(config_file) - return config - - -def get_local_ip() -> str: - all_interface_ips: List[str] = [] - for iface_name in interfaces(): - all_interface_ips += [i["addr"] for i in ifaddresses(iface_name).setdefault(AF_INET, [{"addr": None}]) if i["addr"]] - logger.info(f"detected interface IPs: {all_interface_ips}") - for ip in sorted(all_interface_ips): - if not ip.startswith("127."): - return ip - - assert False, "not found an non-localhost interface" - - -@fixture(scope="session") -def minio_credentials() -> Mapping[str, Any]: - config_template = Path(__file__).parent / "config_minio.template.json" - assert config_template.is_file() is not None, f"not found {config_template}" - config_file = Path(__file__).parent / "config_minio.json" - config_file.write_text(config_template.read_text().replace("", get_local_ip())) - - with open(str(config_file)) as f: - credentials = json.load(f) - return credentials diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/sample_files/catalog.json b/airbyte-integrations/connectors/source-s3/unit_tests/sample_files/catalog.json new file mode 100644 index 000000000000..ddf6ebecec11 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/unit_tests/sample_files/catalog.json @@ -0,0 +1,35 @@ +{ + "streams": [ + { + "stream": { + "name": "test", + "json_schema": { + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "valid": { + "type": ["null", "boolean"] + }, + "_ab_source_file_last_modified": { + "type": "string", + "format": "date-time" + }, + "_ab_source_file_url": { + "type": "string" + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["_ab_source_file_last_modified"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/sample_files/csv/test_file_invalid_conversation.csv b/airbyte-integrations/connectors/source-s3/unit_tests/sample_files/csv/test_file_invalid_conversation.csv new file mode 100644 index 000000000000..5d01827e600d --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/unit_tests/sample_files/csv/test_file_invalid_conversation.csv @@ -0,0 +1,2 @@ +id,name,value +1,PVdhmjb1,44.2 diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/sample_files/csv/test_file_with_invalid_delimiter.csv b/airbyte-integrations/connectors/source-s3/unit_tests/sample_files/csv/test_file_with_invalid_delimiter.csv new file mode 100644 index 000000000000..fbed090818d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/unit_tests/sample_files/csv/test_file_with_invalid_delimiter.csv @@ -0,0 +1,2 @@ +id,name,text +1,PVdhmjb1,Some text with apples, bananas, pineapples diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/sample_files/v3_config.json b/airbyte-integrations/connectors/source-s3/unit_tests/sample_files/v3_config.json new file mode 100644 index 000000000000..c5648fd856ce --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/unit_tests/sample_files/v3_config.json @@ -0,0 +1,12 @@ +{ + "dataset": "test", + "provider": { + "bucket": "a bucket", + "aws_access_key_id": "a key id", + "aws_secret_access_key": "an access key", + "path_prefix": "" + }, + "format": { "filetype": "csv" }, + "path_pattern": "**", + "schema": "{}" +} diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/sample_files/v4_config.json b/airbyte-integrations/connectors/source-s3/unit_tests/sample_files/v4_config.json new file mode 100644 index 000000000000..bc517bf8099c --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/unit_tests/sample_files/v4_config.json @@ -0,0 +1,11 @@ +{ + "bucket": "a-bucket", + "streams": [ + { + "name": "output-stream-name", + "format": { "filetype": "csv" }, + "file_type": "csv", + "validation_policy": "Emit Record" + } + ] +} diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/test_abstract_file_parser.py b/airbyte-integrations/connectors/source-s3/unit_tests/test_abstract_file_parser.py deleted file mode 100644 index 0a83cc632987..000000000000 --- a/airbyte-integrations/connectors/source-s3/unit_tests/test_abstract_file_parser.py +++ /dev/null @@ -1,113 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from typing import Any, Mapping, Tuple - -import pyarrow as pa -import pytest -from airbyte_cdk import AirbyteLogger -from source_s3.source_files_abstract.formats.abstract_file_parser import AbstractFileParser - -LOGGER = AirbyteLogger() - - -class TestAbstractFileParserStatics: - @pytest.mark.parametrize( # testing all datatypes as laid out here: https://json-schema.org/understanding-json-schema/reference/type.html - "input_json_type, output_pyarrow_type", - [ - ("string", pa.large_string()), - ("number", pa.float64()), - ("integer", pa.int64()), - ("object", pa.large_string()), - ("array", pa.large_string()), - ("boolean", pa.bool_()), - ("null", pa.large_string()), - ], - ) - def test_json_type_to_pyarrow_type(self, input_json_type: str, output_pyarrow_type: Any) -> None: - # Json -> PyArrow direction - LOGGER.info(f"asserting that JSON type '{input_json_type}' converts to PyArrow type '{output_pyarrow_type}'...") - assert AbstractFileParser.json_type_to_pyarrow_type(input_json_type) == output_pyarrow_type - - @pytest.mark.parametrize( # testing all datatypes as laid out here: https://arrow.apache.org/docs/python/api/datatypes.html - "input_pyarrow_types, output_json_type", - [ - ((pa.null(),), "string"), # null type - ((pa.bool_(),), "boolean"), # boolean type - ( - (pa.int8(), pa.int16(), pa.int32(), pa.int64(), pa.uint8(), pa.uint16(), pa.uint32(), pa.uint64()), - "integer", - ), # integer types - ((pa.float16(), pa.float32(), pa.float64(), pa.decimal128(5, 10), pa.decimal256(3, 8)), "number"), # number types - ((pa.time32("s"), pa.time64("ns"), pa.timestamp("ms"), pa.date32(), pa.date64()), "string"), # temporal types - ((pa.binary(), pa.large_binary()), "string"), # binary types - ((pa.string(), pa.utf8(), pa.large_string(), pa.large_utf8()), "string"), # string types - ((pa.list_(pa.string()), pa.large_list(pa.timestamp("us"))), "string"), # array types - ((pa.map_(pa.string(), pa.float32()), pa.dictionary(pa.int16(), pa.list_(pa.string()))), "string"), # object types - ], - ) - def test_json_type_to_pyarrow_type_reverse(self, input_pyarrow_types: Tuple[Any], output_json_type: str) -> None: - # PyArrow -> Json direction (reverse=True) - for typ in input_pyarrow_types: - LOGGER.info(f"asserting that PyArrow type '{typ}' converts to JSON type '{output_json_type}'...") - assert AbstractFileParser.json_type_to_pyarrow_type(typ, reverse=True) == output_json_type - - @pytest.mark.parametrize( # if expecting fail, put pyarrow_schema as None - "json_schema, pyarrow_schema", - [ - ( - {"a": "string", "b": "number", "c": "integer", "d": "object", "e": "array", "f": "boolean", "g": "null"}, - { - "a": pa.large_string(), - "b": pa.float64(), - "c": pa.int64(), - "d": pa.large_string(), - "e": pa.large_string(), - "f": pa.bool_(), - "g": pa.large_string(), - }, - ), - ({"single_column": "object"}, {"single_column": pa.large_string()}), - ({}, {}), - ({"a": "NOT A REAL TYPE", "b": "another fake type"}, {"a": pa.large_string(), "b": pa.large_string()}), - (["string", "object"], None), # bad input type - ], - ) - def test_json_schema_to_pyarrow_schema(self, json_schema: Mapping[str, Any], pyarrow_schema: Mapping[str, Any]) -> None: - # Json -> PyArrow direction - if pyarrow_schema is not None: - assert AbstractFileParser.json_schema_to_pyarrow_schema(json_schema) == pyarrow_schema - else: - with pytest.raises(Exception) as e_info: - AbstractFileParser.json_schema_to_pyarrow_schema(json_schema) - LOGGER.debug(str(e_info)) - - @pytest.mark.parametrize( # if expecting fail, put json_schema as None - "pyarrow_schema, json_schema", - [ - ( - { - "a": pa.utf8(), - "b": pa.float16(), - "c": pa.uint32(), - "d": pa.map_(pa.string(), pa.float32()), - "e": pa.bool_(), - "f": pa.date64(), - }, - {"a": "string", "b": "number", "c": "integer", "d": "string", "e": "boolean", "f": "string"}, - ), - ({"single_column": pa.int32()}, {"single_column": "integer"}), - ({}, {}), - ({"a": "NOT A REAL TYPE", "b": "another fake type"}, {"a": "string", "b": "string"}), - (["string", "object"], None), # bad input type - ], - ) - def test_json_schema_to_pyarrow_schema_reverse(self, pyarrow_schema: Mapping[str, Any], json_schema: Mapping[str, Any]) -> None: - # PyArrow -> Json direction (reverse=True) - if json_schema is not None: - assert AbstractFileParser.json_schema_to_pyarrow_schema(pyarrow_schema, reverse=True) == json_schema - else: - with pytest.raises(Exception) as e_info: - AbstractFileParser.json_schema_to_pyarrow_schema(pyarrow_schema, reverse=True) - LOGGER.debug(str(e_info)) diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/test_avro_parser.py b/airbyte-integrations/connectors/source-s3/unit_tests/test_avro_parser.py deleted file mode 100644 index 789683220020..000000000000 --- a/airbyte-integrations/connectors/source-s3/unit_tests/test_avro_parser.py +++ /dev/null @@ -1,146 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import os -import random -import string -from random import randrange -from typing import Any, Mapping - -from avro import datafile, io, schema -from source_s3.source_files_abstract.formats.avro_parser import AvroParser - -from .abstract_test_parser import AbstractTestParser -from .conftest import TMP_FOLDER - -simple_schema_str = """{ - "type": "record", - "name": "sampleAvro", - "namespace": "AVRO", - "fields": [ - {"name": "name", "type": "string"}, - {"name": "age", "type": ["int", "null"]}, - {"name": "address", "type": ["float", "null"]}, - {"name": "street", "type": "float"}, - {"name": "valid", "type": "boolean"} - ] -}""" - -nested_records_schema_str = """{ - "type": "record", - "name": "sampleAvroNested", - "namespace": "AVRO", - "fields": [ - {"name": "lastname", "type": "string"}, - {"name": "address","type": { - "type" : "record", - "name" : "AddressUSRecord", - "fields" : [ - {"name": "streetaddress", "type": "string"}, - {"name": "city", "type": "string"} - ] - } - } - ] -}""" - -nested_schema_output = {"address": {"type": ["object", "null"], - "city": {"type": ["string", "null"]}, - "streetaddress": {"type": ["string", "null"]}, - }, - "lastname": {"type": ["string", "null"]}} - -master_schema = {"address": {"type": ["number", "null"]}, - "age": {"type": ["integer", "null"]}, - "name": {"type": ["string", "null"]}, - "street": {"type": ["number", "null"]}, - "valid": {"type": ["boolean", "null"]} - } - - -class TestAvroParser(AbstractTestParser): - filetype = "avro" - - @classmethod - def generate_avro_file(cls, out_file, num_rows: int) -> str: - """Creates an avro file and saves to tmp folder to be used by test cases - :param schema_str: valid avro schema as a string - :param out_file: name of file to be created - :param num_rows: number of rows to be generated - :return: string with path to the file created - """ - filename = os.path.join(TMP_FOLDER, out_file + "." + cls.filetype) - parsed_schema = schema.parse(simple_schema_str) - rec_writer = io.DatumWriter(parsed_schema) - file_writer = datafile.DataFileWriter(open(filename, "wb"), rec_writer, parsed_schema) - for _ in range(num_rows): - data = {} - data["name"] = "".join(random.choice(string.ascii_letters) for i in range(10)) - data["age"] = randrange(-100, 100) - data["address"] = random.uniform(1.1, 100.10) - data["street"] = random.uniform(1.1, 100.10) - data["valid"] = random.choice([True, False]) - file_writer.append(data) - file_writer.close() - return filename - - @classmethod - def generate_nested_avro_file(cls, out_file, num_rows: int) -> str: - """Creates an avro file and saves to tmp folder to be used by test cases - :param schema_str: valid avro schema as a string - :param out_file: name of file to be created - :param num_rows: number of rows to be generated - :return: string with path to the file created - """ - filename = os.path.join(TMP_FOLDER, out_file + "." + cls.filetype) - parsed_schema = schema.parse(nested_records_schema_str) - rec_writer = io.DatumWriter(parsed_schema) - file_writer = datafile.DataFileWriter(open(filename, "wb"), rec_writer, parsed_schema) - for _ in range(num_rows): - data = {} - data["lastname"] = "".join(random.choice(string.ascii_letters) for i in range(10)) - data["address"] = { - "streetaddress": "".join(random.choice(string.ascii_letters) for i in range(10)), - "city": "".join(random.choice(string.ascii_letters) for i in range(10)) - } - file_writer.append(data) - file_writer.close() - return filename - - @classmethod - def cases(cls) -> Mapping[str, Any]: - """ - return test cases - """ - cases = {} - # test basic file with data type conversions - cases["simple_test"] = { - "AbstractFileParser": AvroParser(format=cls.filetype), - "filepath": cls.generate_avro_file("test_file", 1000), - "num_records": 1000, - "inferred_schema": master_schema, - "line_checks": {}, - "fails": [], - } - # test file with 0 records. Will pass but not ingest anything - cases["test_zero_rows"] = { - "AbstractFileParser": AvroParser(format=cls.filetype), - "filepath": cls.generate_avro_file("test_file_zero_rows", 0), - "num_records": 0, - "inferred_schema": master_schema, - "line_checks": {}, - "fails": [], - } - - # test for avro schema with nested records. This will pass as all nested records are returned as one string - cases["test_nested_records"] = { - "AbstractFileParser": AvroParser(format=cls.filetype), - "filepath": cls.generate_nested_avro_file("test_nested_records", 10), - "num_records": 10, - "inferred_schema": nested_schema_output, - "line_checks": {}, - "fails": [], - } - - return cases diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/test_csv_parser.py b/airbyte-integrations/connectors/source-s3/unit_tests/test_csv_parser.py deleted file mode 100644 index 8113904a4098..000000000000 --- a/airbyte-integrations/connectors/source-s3/unit_tests/test_csv_parser.py +++ /dev/null @@ -1,430 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import json -import os -import random -import shutil -import string -from contextlib import nullcontext as does_not_raise -from pathlib import Path -from typing import Any, List, Mapping, Tuple -from unittest.mock import Mock - -import pendulum -import pytest -from airbyte_cdk.utils.traced_exception import AirbyteTracedException -from smart_open import open as smart_open -from source_s3.source_files_abstract.file_info import FileInfo -from source_s3.source_files_abstract.formats.csv_parser import CsvParser - -from .abstract_test_parser import AbstractTestParser, memory_limit -from .conftest import TMP_FOLDER - -SAMPLE_DIRECTORY = Path(__file__).resolve().parent.joinpath("sample_files/") - -# All possible CSV data types -CSV_TYPES = { - # logical_type: (json_type, csv_types, convert_function) - # standard types - "string": ("string", ["string"], None), - "boolean": ("boolean", ["boolean"], None), - "number": ("number", ["number"], None), - "integer": ("integer", ["integer"], None), -} - - -def _generate_value(typ: str) -> Any: - if typ == "string": - if AbstractTestParser._generate_value("boolean"): - return None - random_length = random.randint(0, 512) - return "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(random_length)) - return AbstractTestParser._generate_value(typ) - - -def _generate_row(types: List[str]) -> List[Any]: - """Generates random values with request types""" - row = [] - for needed_type in types: - for json_type in CSV_TYPES: - if json_type == needed_type: - value = _generate_value(needed_type) - if value is None: - value = "" - row.append(str(value)) - break - return row - - -def generate_csv_file(filename: str, columns: Mapping[str, str], num_rows: int, delimiter: str) -> str: - """Generates a random CSV data and save it to a tmp file""" - header_line = delimiter.join(columns.keys()) - types = list(columns.values()) if num_rows else [] - with open(filename, "w") as f: - f.write(header_line + "\n") - for _ in range(num_rows): - f.write(delimiter.join(_generate_row(types)) + "\n") - return filename - - -def generate_big_file(filepath: str, size_in_gigabytes: float, columns_number: int, template_file: str = None) -> Tuple[dict, float]: - temp_files = [filepath + ".1", filepath + ".2"] - if template_file: - shutil.copyfile(template_file, filepath) - schema = None - else: - schema = {f"column {i}": random.choice(["integer", "string", "boolean", "number"]) for i in range(columns_number)} - generate_csv_file(filepath, schema, 456, ",") - - skip_headers = False - with open(filepath, "r") as f: - with open(temp_files[0], "w") as tf: - for line in f: - if not skip_headers: - skip_headers = True - continue - tf.write(str(line)) - - with open(filepath, "ab") as f: - while True: - file_size = os.stat(filepath).st_size / (1024**3) - if file_size > size_in_gigabytes: - break - with open(temp_files[0], "rb") as tf: # type: ignore[assignment] - with open(temp_files[1], "wb") as tf2: - buf = tf.read(50 * 1024**2) # by 50Mb - if buf: - f.write(buf) # type: ignore[arg-type] - tf2.write(buf) # type: ignore[arg-type] - temp_files.append(temp_files.pop(0)) - # remove temp files - for temp_file in temp_files: - if os.path.exists(temp_file): - os.remove(temp_file) - return schema, file_size - - -class TestCsvParser(AbstractTestParser): - record_types = CSV_TYPES - filetype = "csv" - - @classmethod - def cases(cls) -> Mapping[str, Any]: - return { - "basic_normal_test": { - "AbstractFileParser": CsvParser( - format={"filetype": "csv"}, - master_schema={ - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - }, - ), - "filepath": os.path.join(SAMPLE_DIRECTORY, "csv/test_file_1.csv"), - "num_records": 8, - "inferred_schema": { - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - }, - "line_checks": {}, - "fails": [], - }, - "custom_csv_parameters": { - # tests custom CSV parameters (odd delimiter, quote_char, escape_char & newlines in values in the file) - "AbstractFileParser": CsvParser( - format={"filetype": "csv", "delimiter": "^", "quote_char": "|", "escape_char": "!", "newlines_in_values": True}, - master_schema={ - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - }, - ), - "filepath": os.path.join(SAMPLE_DIRECTORY, "csv/test_file_2_params.csv"), - "num_records": 8, - "inferred_schema": { - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - }, - "line_checks": {}, - "fails": [], - }, - "encoding_Big5": { - # tests encoding: Big5 - "AbstractFileParser": CsvParser( - format={"filetype": "csv", "encoding": "big5"}, master_schema={"id": "integer", "name": "string", "valid": "boolean"} - ), - "filepath": os.path.join(SAMPLE_DIRECTORY, "csv/test_file_3_enc_Big5.csv"), - "num_records": 8, - "inferred_schema": {"id": "integer", "name": "string", "valid": "boolean"}, - "line_checks": { - 3: { - "id": 3, - "name": "變形金剛,偽裝的機器人", - "valid": False, - } - }, - "fails": [], - }, - "encoding_Arabic_(Windows 1256)": { - # tests encoding: Arabic (Windows 1256) - "AbstractFileParser": CsvParser( - format={"filetype": "csv", "encoding": "windows-1256"}, - master_schema={"id": "integer", "notes": "string", "valid": "boolean"}, - ), - "filepath": os.path.join(SAMPLE_DIRECTORY, "csv/test_file_4_enc_Arabic.csv"), - "num_records": 2, - "inferred_schema": {"id": "integer", "notes": "string", "valid": "boolean"}, - "line_checks": { - 1: { - "id": 1, - "notes": "البايت الجوي هو الأفضل", - "valid": False, - } - }, - "fails": [], - }, - "compression_gzip": { - # tests compression: gzip - "AbstractFileParser": CsvParser( - format={"filetype": "csv"}, - master_schema={ - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - }, - ), - "filepath": os.path.join(SAMPLE_DIRECTORY, "csv/test_file_5.csv.gz"), - "num_records": 8, - "inferred_schema": { - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - }, - "line_checks": { - 7: { - "id": 7, - "name": "xZhh1Kyl", - "valid": False, - "code": 10, - "degrees": -9.2, - "birthday": "2021-07-14", - "last_seen": "2021-07-14 15:30:09.225145", - } - }, - "fails": [], - }, - "compression_bz2": { - # tests compression: bz2 - "AbstractFileParser": CsvParser( - format={"filetype": "csv"}, - master_schema={ - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - }, - ), - "filepath": os.path.join(SAMPLE_DIRECTORY, "csv/test_file_7_bz2.csv.bz2"), - "num_records": 8, - "inferred_schema": { - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - }, - "line_checks": { - 7: { - "id": 7, - "name": "xZhh1Kyl", - "valid": False, - "code": 10, - "degrees": -9.2, - "birthday": "2021-07-14", - "last_seen": "2021-07-14 15:30:09.225145", - } - }, - "fails": [], - }, - "extra_columns_in_master_schema": { - # tests extra columns in master schema - "AbstractFileParser": CsvParser( - format={"filetype": "csv"}, - master_schema={ - "EXTRA_COLUMN_1": "boolean", - "EXTRA_COLUMN_2": "number", - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - }, - ), - "filepath": os.path.join(SAMPLE_DIRECTORY, "csv/test_file_1.csv"), - "num_records": 8, - "inferred_schema": { - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - }, - "line_checks": {}, - "fails": [], - }, - "missing_columns_in_master_schema": { - # tests missing columns in master schema - # TODO: maybe this should fail read_records, but it does pick up all the columns from file despite missing from master schema - "AbstractFileParser": CsvParser(format={"filetype": "csv"}, master_schema={"id": "integer", "name": "string"}), - "filepath": os.path.join(SAMPLE_DIRECTORY, "csv/test_file_1.csv"), - "num_records": 8, - "inferred_schema": { - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - }, - "line_checks": {}, - "fails": [], - }, - "empty_csv_file": { - # tests empty file, SHOULD FAIL INFER & STREAM RECORDS - "AbstractFileParser": CsvParser(format={"filetype": "csv"}, master_schema={}), - "filepath": os.path.join(SAMPLE_DIRECTORY, "csv/test_file_6_empty.csv"), - "num_records": 0, - "inferred_schema": {}, - "line_checks": {}, - "fails": ["test_get_inferred_schema", "test_stream_records"], - }, - "empty_advanced_options": { - "AbstractFileParser": CsvParser( - format={"filetype": "csv", "advanced_options": ""}, - master_schema={ - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - }, - ), - "filepath": os.path.join(SAMPLE_DIRECTORY, "csv/test_file_1.csv"), - "num_records": 8, - "inferred_schema": { - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - }, - "line_checks": {}, - "fails": [], - }, - "no_header_csv_file": { - # no header test - "AbstractFileParser": CsvParser( - format={ - "filetype": "csv", - "advanced_options": json.dumps( - {"column_names": ["id", "name", "valid", "code", "degrees", "birthday", "last_seen"]} - ), - }, - master_schema={}, - ), - "filepath": os.path.join(SAMPLE_DIRECTORY, "csv/test_file_8_no_header.csv"), - "num_records": 8, - "inferred_schema": { - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - }, - "line_checks": {}, - "fails": [], - }, - } - - @memory_limit(20) - @pytest.mark.order(1) - def test_big_file(self) -> None: - """tests a big csv file (>= 1.5G records)""" - filepath = os.path.join(TMP_FOLDER, "big_csv_file." + self.filetype) - schema, file_size = generate_big_file(filepath, 0.1, 123) - expected_count = sum(1 for _ in open(filepath)) - 1 - self.logger.info(f"generated file {filepath} with size {file_size}Gb, lines: {expected_count}") - for _ in range(3): - parser = CsvParser( - format={"filetype": self.filetype, "block_size": 5 * 1024**2}, - master_schema=schema, - ) - expected_file = open(filepath, "r") - # skip the first header line - next(expected_file) - read_count = 0 - with smart_open(filepath, self._get_readmode({"AbstractFileParser": parser})) as f: - for record in parser.stream_records(f, FileInfo(key=filepath, size=file_size, last_modified=pendulum.now())): - record_line = ",".join("" if v is None else str(v) for v in record.values()) - expected_line = next(expected_file).strip("\n") - assert record_line == expected_line - read_count += 1 - assert read_count == expected_count - expected_file.close() - - @pytest.mark.parametrize( - "encoding, expectation", - ( - ("UTF8", does_not_raise()), - ("", does_not_raise()), - ("R2D2", pytest.raises(AirbyteTracedException)), - ) - ) - def test_encoding_validation(self, encoding, expectation) -> None: - parser = CsvParser(format=Mock(), master_schema=Mock()) - with expectation: - parser._validate_encoding(encoding) diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/test_jsonl_parser.py b/airbyte-integrations/connectors/source-s3/unit_tests/test_jsonl_parser.py deleted file mode 100644 index e6d409ce860e..000000000000 --- a/airbyte-integrations/connectors/source-s3/unit_tests/test_jsonl_parser.py +++ /dev/null @@ -1,179 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import os -from pathlib import Path -from typing import Any, Mapping - -from source_s3.source_files_abstract.formats.jsonl_parser import JsonlParser - -from .abstract_test_parser import AbstractTestParser - -SAMPLE_DIRECTORY = Path(__file__).resolve().parent.joinpath("sample_files/") - - -class TestJsonlParser(AbstractTestParser): - @classmethod - def cases(cls) -> Mapping[str, Any]: - return { - "basic_normal_test": { - "AbstractFileParser": JsonlParser(format={"filetype": "jsonl"}), - "filepath": os.path.join(SAMPLE_DIRECTORY, "jsonl/test_file_1.jsonl"), - "num_records": 8, - "inferred_schema": { - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - }, - "line_checks": {}, - "fails": [], - }, - "master_schema_test": { - "AbstractFileParser": JsonlParser( - format={"filetype": "jsonl"}, - master_schema={ - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - }, - ), - "filepath": os.path.join(SAMPLE_DIRECTORY, "jsonl/test_file_1.jsonl"), - "num_records": 8, - "inferred_schema": { - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - }, - "line_checks": {}, - "fails": [], - }, - "encoding_Big5": { - "AbstractFileParser": JsonlParser(format={"filetype": "jsonl"}), - "filepath": os.path.join(SAMPLE_DIRECTORY, "jsonl/test_file_2_enc_Big5.jsonl"), - "num_records": 8, - "inferred_schema": {"id": "integer", "name": "string", "valid": "boolean"}, - "line_checks": {}, - "fails": [], - }, - "encoding_Arabic_(Windows 1256)": { - "AbstractFileParser": JsonlParser(format={"filetype": "jsonl"}), - "filepath": os.path.join(SAMPLE_DIRECTORY, "jsonl/test_file_3_enc_Arabic.jsonl"), - "num_records": 2, - "inferred_schema": {"id": "integer", "notes": "string", "valid": "boolean"}, - "line_checks": {}, - "fails": [], - }, - "compression_gz": { - "AbstractFileParser": JsonlParser( - format={"filetype": "jsonl"}, - master_schema={ - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - }, - ), - "filepath": os.path.join(SAMPLE_DIRECTORY, "jsonl/test_file_4.jsonl.gz"), - "num_records": 8, - "inferred_schema": { - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - }, - "line_checks": { - 7: { - "id": 7, - "name": "xZhh1Kyl", - "valid": False, - "code": 10, - "degrees": -9.2, - "birthday": "2021-07-14", - "last_seen": "2021-07-14 15:30:09.225145", - } - }, - "fails": [], - }, - "extra_columns_in_master_schema": { - # tests extra columns in master schema - "AbstractFileParser": JsonlParser( - format={"filetype": "jsonl"}, - master_schema={ - "EXTRA_COLUMN_1": "boolean", - "EXTRA_COLUMN_2": "number", - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - }, - ), - "filepath": os.path.join(SAMPLE_DIRECTORY, "jsonl/test_file_1.jsonl"), - "num_records": 8, - "inferred_schema": { - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - }, - "line_checks": {}, - "fails": [], - }, - "missing_columns_in_master_schema": { - # tests missing columns in master schema - "AbstractFileParser": JsonlParser(format={"filetype": "jsonl"}, master_schema={"id": "integer", "name": "string"}), - "filepath": os.path.join(SAMPLE_DIRECTORY, "jsonl/test_file_1.jsonl"), - "num_records": 8, - "inferred_schema": { - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - }, - "line_checks": {}, - "fails": [], - }, - "nested_json_test": { - "AbstractFileParser": JsonlParser(format={"filetype": "jsonl"}), - "filepath": os.path.join(SAMPLE_DIRECTORY, "jsonl/test_file_10_nested_structure.jsonl"), - "num_records": 2, - "inferred_schema": {"meta": "object", "payload": "object"}, - "line_checks": {}, - "fails": [], - }, - "array_in_schema_test": { - "AbstractFileParser": JsonlParser(format={"filetype": "jsonl"}), - "filepath": os.path.join(SAMPLE_DIRECTORY, "jsonl/test_file_11_array_in_schema.jsonl"), - "num_records": 3, - "inferred_schema": {"id": "integer", "name": "string", "books": "array"}, - "line_checks": {}, - "fails": [], - }, - } diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/test_parquet_parser.py b/airbyte-integrations/connectors/source-s3/unit_tests/test_parquet_parser.py deleted file mode 100644 index 2ca23020c7f8..000000000000 --- a/airbyte-integrations/connectors/source-s3/unit_tests/test_parquet_parser.py +++ /dev/null @@ -1,253 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import bz2 -import copy -import gzip -import os -import shutil -from pathlib import Path -from typing import Any, List, Mapping - -import pandas as pd -import pyarrow as pa -import pyarrow.parquet as pq -import pytest -from source_s3.source_files_abstract.formats.parquet_parser import PARQUET_TYPES, ParquetParser - -from .abstract_test_parser import AbstractTestParser -from .conftest import TMP_FOLDER - -SAMPLE_DIRECTORY = Path(__file__).resolve().parent.joinpath("sample_files/") - - -def compress(archive_name: str, filename: str) -> str: - compress_filename = f"{filename}.{archive_name}" - with open(filename, "rb") as f_in: - if archive_name == "gz": - with gzip.open(compress_filename, "wb") as f_out: - shutil.copyfileobj(f_in, f_out) - elif archive_name == "bz2": - with bz2.open(compress_filename, "wb") as f_out: # type: ignore[assignment] - shutil.copyfileobj(f_in, f_out) - return compress_filename - - -class TestParquetParser(AbstractTestParser): - filetype = "parquet" - record_types = PARQUET_TYPES - - @classmethod - def generate_parquet_file( - cls, name: str, columns: Mapping[str, str], num_rows: int, custom_rows: Mapping[int, List[str]] = None - ) -> str: - """Generates a random data and save it to a tmp file""" - filename = os.path.join(TMP_FOLDER, name + "." + cls.filetype) - - pq_writer = None - types = list(columns.values()) if num_rows else [] - custom_rows = custom_rows or {} - column_names = list(columns.keys()) - buffer = [] - for i in range(num_rows): - buffer.append(custom_rows.get(i) or cls._generate_row(types)) - if i != (num_rows - 1) and len(buffer) < 100: - continue - data = {col_values[0]: list(col_values[1:]) for col_values in zip(column_names, *buffer)} - buffer = [] - df = pd.DataFrame(data) - table = pa.Table.from_pandas(df) - if not pq_writer: - pq_writer = pq.ParquetWriter(filename, table.schema) - pq_writer.write_table(table, row_group_size=100) - - if not pq_writer: - pq.write_table(pa.Table.from_arrays([]), filename) - return filename - - @classmethod - def cases(cls) -> Mapping[str, Any]: - schema = { - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - "salary": "decimal", - "created_at": "timestamp", - "created_date_at": "date", - "created_time_at": "time", - } - # datetime => string type - - master_schema = {k: ParquetParser.parse_field_type(needed_logical_type=v)[0] for k, v in schema.items()} - cases = {} - # basic 'normal' test - num_records = 10 - params = {"filetype": cls.filetype} - cases["basic_normal_test"] = { - "AbstractFileParser": ParquetParser(format=params, master_schema=master_schema), - "filepath": cls.generate_parquet_file("normal_test", schema, num_records), - "num_records": num_records, - "inferred_schema": master_schema, - "line_checks": {}, - "fails": [], - } - # tests custom Parquet parameters (row_groups, batch_size etc) - params = { - "filetype": cls.filetype, - "buffer_size": 1024, - "columns": ["id", "name", "last_seen"], - } - num_records = 100 - cases["custom_parquet_parameters"] = { - "filepath": cls.generate_parquet_file("normal_params_test", schema, num_records), - "num_records": num_records, - "AbstractFileParser": ParquetParser( - format=params, - master_schema=master_schema, - ), - "inferred_schema": master_schema, - "line_checks": {}, - "fails": [], - } - - # tests a big parquet file (100K records) - params = { - "filetype": cls.filetype, - "batch_size": 200, - "use_threads": False, - } - num_records = 100000 - - cases["big_parquet_file"] = { - "filepath": cls.generate_parquet_file("big_parquet_file", schema, num_records), - "num_records": num_records, - "AbstractFileParser": ParquetParser( - format=params, - master_schema=master_schema, - ), - "inferred_schema": master_schema, - "line_checks": {}, - "fails": [], - } - - # extra columns in master schema - params = {"filetype": cls.filetype} - num_records = 10 - extra_schema = copy.deepcopy(master_schema) - extra_schema.update( - { - "extra_id": "integer", - "extra_name": "string", - } - ) - cases["extra_columns_in_master_schema"] = { - "filepath": cls.generate_parquet_file("normal_test", schema, num_records), - "num_records": num_records, - "AbstractFileParser": ParquetParser( - format=params, - master_schema=extra_schema, - ), - "inferred_schema": master_schema, - "line_checks": {}, - "fails": [], - } - - # tests missing columns in master schema - params = {"filetype": cls.filetype} - num_records = 10 - simplified_schema = copy.deepcopy(master_schema) - simplified_schema.pop("id") - simplified_schema.pop("name") - - cases["tests_missing_columns_in_master_schema"] = { - "filepath": cls.generate_parquet_file("normal_test", schema, num_records), - "num_records": num_records, - "AbstractFileParser": ParquetParser( - format=params, - master_schema=simplified_schema, - ), - "inferred_schema": master_schema, - "line_checks": {}, - "fails": [], - } - - # tests empty file, SHOULD FAIL INFER & STREAM RECORDS - num_records = 0 - cases["empty_file"] = { - "filepath": cls.generate_parquet_file("empty_file", schema, num_records), - "num_records": num_records, - "AbstractFileParser": ParquetParser( - format=params, - master_schema={}, - ), - "inferred_schema": master_schema, - "line_checks": {}, - "fails": ["test_get_inferred_schema", "test_stream_records"], - } - - # check one record - params = {"filetype": cls.filetype} - num_records = 20 - test_record = { - "id": 7, - "name": cls._generate_value("string"), - "valid": False, - "code": 10, - "degrees": -9.2, - "birthday": cls._generate_value("string"), - "last_seen": cls._generate_value("string"), - "salary": cls._generate_value("decimal"), - "created_at": cls._generate_value("timestamp"), - "created_date_at": cls._generate_value("date"), - "created_time_at": cls._generate_value("time"), - } - - expected_record = copy.deepcopy(test_record) - expected_record["salary"] = ParquetParser.convert_field_data("decimal", expected_record["salary"]) - expected_record["created_date_at"] = ParquetParser.convert_field_data("date", expected_record["created_date_at"]) - expected_record["created_time_at"] = ParquetParser.convert_field_data("time", expected_record["created_time_at"]) - expected_record["created_at"] = ParquetParser.convert_field_data("timestamp", expected_record["created_at"]) - - cases["check_one_record"] = { - "filepath": cls.generate_parquet_file("check_one_record", schema, num_records, custom_rows={7: list(test_record.values())}), - "num_records": num_records, - "AbstractFileParser": ParquetParser( - format=params, - master_schema=master_schema, - ), - "inferred_schema": master_schema, - "line_checks": {8: expected_record}, - "fails": [], - } - - # tests compression: gzip - num_records = 10 - for archive_type in ["gz", "bz2"]: - cases[f"compression_{archive_type}"] = { - "filepath": compress( - archive_type, - cls.generate_parquet_file("compression_test", schema, num_records, custom_rows={7: list(test_record.values())}), - ), - "num_records": num_records, - "AbstractFileParser": ParquetParser( - format=params, - master_schema=master_schema, - ), - "inferred_schema": master_schema, - "line_checks": {8: expected_record}, - "fails": [], - } - return cases - - def test_parse_field_type(self): - with pytest.raises(TypeError): - assert ParquetParser.parse_field_type(needed_logical_type="", need_physical_type="") - - def test_convert_field_data(self): - with pytest.raises(TypeError): - ParquetParser.convert_field_data(logical_type="", field_value="") diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/test_s3file.py b/airbyte-integrations/connectors/source-s3/unit_tests/test_s3file.py deleted file mode 100644 index 4f9501c68220..000000000000 --- a/airbyte-integrations/connectors/source-s3/unit_tests/test_s3file.py +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from typing import Mapping -from unittest.mock import MagicMock - -import pytest -import smart_open -from airbyte_cdk import AirbyteLogger -from source_s3.s3file import S3File - -LOGGER = AirbyteLogger() - - -class TestS3File: - @pytest.mark.parametrize( # passing in full provider to emulate real usage (dummy values are unused by func) - "provider, return_true", - [ - ({"storage": "S3", "bucket": "dummy", "aws_access_key_id": "id", "aws_secret_access_key": "key", "path_prefix": "dummy"}, True), - ({"storage": "S3", "bucket": "dummy", "aws_access_key_id": None, "aws_secret_access_key": None, "path_prefix": "dummy"}, False), - ({"storage": "S3", "bucket": "dummy", "path_prefix": "dummy"}, False), - ({"storage": "S3", "bucket": "dummy", "aws_access_key_id": "id", "aws_secret_access_key": None, "path_prefix": "dummy"}, False), - ( - {"storage": "S3", "bucket": "dummy", "aws_access_key_id": None, "aws_secret_access_key": "key", "path_prefix": "dummy"}, - False, - ), - ({"storage": "S3", "bucket": "dummy", "aws_access_key_id": "id", "path_prefix": "dummy"}, False), - ({"storage": "S3", "bucket": "dummy", "aws_secret_access_key": "key", "path_prefix": "dummy"}, False), - ], - ) - def test_use_aws_account(self, provider: Mapping[str, str], return_true: bool) -> None: - assert S3File.use_aws_account(provider) is return_true - - @pytest.mark.parametrize( # passing in full provider to emulate real usage (dummy values are unused by func) - "provider", - [ - ({"storage": "S3", "bucket": "dummy", "aws_access_key_id": "id", "aws_secret_access_key": "key", "path_prefix": "dummy"}), - ({"storage": "S3", "bucket": "dummy", "aws_access_key_id": None, "aws_secret_access_key": None, "path_prefix": "dummy"}), - ], - ) - def test_s3_file_contextmanager(self, provider): - smart_open.open = MagicMock() - with S3File(file_info=MagicMock(), provider=provider).open("rb") as s3_file: - assert s3_file diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/test_source.py b/airbyte-integrations/connectors/source-s3/unit_tests/test_source.py deleted file mode 100644 index b2862a486526..000000000000 --- a/airbyte-integrations/connectors/source-s3/unit_tests/test_source.py +++ /dev/null @@ -1,118 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import json -from unittest.mock import MagicMock, patch - -import pytest -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import ConnectorSpecification -from airbyte_cdk.utils.traced_exception import AirbyteTracedException -from source_s3 import SourceS3 -from source_s3.source_files_abstract.spec import SourceFilesAbstractSpec - -logger = AirbyteLogger() - - -def test_transform_backslash_t_to_tab(tmp_path): - config_file = tmp_path / "config.json" - with open(config_file, "w") as fp: - json.dump({"format": {"delimiter": "\\t"}}, fp) - source = SourceS3() - config = source.read_config(config_file) - assert config["format"]["delimiter"] == "\t" - - -def test_check_connection_empty_config(): - config = {} - ok, error_msg = SourceS3().check_connection(logger, config=config) - - assert not ok - assert error_msg - - -def test_check_connection_exception(config): - ok, error_msg = SourceS3().check_connection(logger, config=config) - - assert not ok - assert error_msg - - -@pytest.mark.parametrize( - "delimiter, quote_char, escape_char, encoding, read_options, convert_options", - [ - ("string", "'", None, "utf8", "{}", "{}"), - ("\n", "'", None, "utf8", "{}", "{}"), - (",", ";,", None, "utf8", "{}", "{}"), - (",", "'", "escape", "utf8", "{}", "{}"), - (",", "'", None, "utf888", "{}", "{}"), - (",", "'", None, "utf8", "{'compression': true}", "{}"), - (",", "'", None, "utf8", "{}", "{'compression: true}"), - ], - ids=[ - "long_delimiter", - "forbidden_delimiter_symbol", - "long_quote_char", - "long_escape_char", - "unknown_encoding", - "invalid read options", - "invalid convert options" - ], -) -def test_check_connection_csv_validation_exception(delimiter, quote_char, escape_char, encoding, read_options, convert_options): - config = { - "dataset": "test", - "provider": { - "storage": "S3", - "bucket": "test-source-s3", - "aws_access_key_id": "key_id", - "aws_secret_access_key": "access_key", - "path_prefix": "" - }, - "path_pattern": "simple_test*.csv", - "schema": "{}", - "format": { - "filetype": "csv", - "delimiter": delimiter, - "quote_char": quote_char, - "escape_char": escape_char, - "encoding": encoding, - "advanced_options": read_options, - "additional_reader_options": convert_options - } - } - ok, error_msg = SourceS3().check_connection(logger, config=config) - - assert not ok - assert error_msg - assert isinstance(error_msg, AirbyteTracedException) - - -def test_check_connection(config): - instance = SourceS3() - with patch.object(instance.stream_class, "filepath_iterator", MagicMock()): - ok, error_msg = instance.check_connection(logger, config=config) - - assert not ok - assert error_msg - - -def test_streams(config): - instance = SourceS3() - assert len(instance.streams(config)) == 1 - - -def test_spec(): - spec = SourceS3().spec() - - assert isinstance(spec, ConnectorSpecification) - - -def test_check_provider_added(): - with pytest.raises(Exception): - SourceFilesAbstractSpec.check_provider_added({"properties": []}) - - -def test_change_format_to_oneOf(): - assert SourceFilesAbstractSpec.change_format_to_oneOf({"properties": {"format": {"oneOf": ""}}}) diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/test_stream.py b/airbyte-integrations/connectors/source-s3/unit_tests/test_stream.py deleted file mode 100644 index 032808b526bb..000000000000 --- a/airbyte-integrations/connectors/source-s3/unit_tests/test_stream.py +++ /dev/null @@ -1,737 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from datetime import datetime, timezone -from typing import Any, Dict, List, Mapping -from unittest.mock import MagicMock, patch - -import pytest -from airbyte_cdk import AirbyteLogger -from airbyte_cdk.models import SyncMode -from source_s3.source_files_abstract.file_info import FileInfo -from source_s3.source_files_abstract.storagefile import StorageFile -from source_s3.source_files_abstract.stream import IncrementalFileStream -from source_s3.stream import IncrementalFileStreamS3 - -from .abstract_test_parser import create_by_local_file, memory_limit - -LOGGER = AirbyteLogger() - - -def mock_big_size_object(): - mock = MagicMock() - mock.__sizeof__.return_value = 1000000001 - mock.items = lambda: [("2023-08-01", ["file1.txt", "file2.txt"])] - return mock - - -class TestIncrementalFileStream: - @pytest.mark.parametrize( # set return_schema to None for an expected fail - "schema_string, return_schema", - [ - ( - '{"id": "integer", "name": "string", "valid": "boolean", "code": "integer", "degrees": "number", "birthday": ' - '"string", "last_seen": "string"}', - { - "id": "integer", - "name": "string", - "valid": "boolean", - "code": "integer", - "degrees": "number", - "birthday": "string", - "last_seen": "string", - }, - ), - ('{"single_column": "boolean"}', {"single_column": "boolean"}), - (r"{}", {}), - ('{this isn\'t right: "integer"}', None), # invalid json - ('[ {"a":"b"} ]', None), # array instead of object - ('{"a": "boolean", "b": {"string": "integer"}}', None), # object as a value - ('{"a": ["boolean", "string"], "b": {"string": "integer"}}', None), # array and object as values - ('{"a": "integer", "b": "NOT A REAL DATATYPE"}', None), # incorrect datatype - ('{"a": "NOT A REAL DATATYPE", "b": "ANOTHER FAKE DATATYPE"}', None), # multiple incorrect datatypes - ], - ) - @memory_limit(512) - def test_parse_user_input_schema(self, schema_string: str, return_schema: str) -> None: - if return_schema is not None: - assert str(IncrementalFileStream._parse_user_input_schema(schema_string)) == str(return_schema) - else: - with pytest.raises(Exception) as e_info: - IncrementalFileStream._parse_user_input_schema(schema_string) - LOGGER.debug(str(e_info)) - - @pytest.mark.parametrize( # set expected_return_record to None for an expected fail - "extra_map, record, expected_return_record", - [ - ( # one extra field - {"friend": "Frodo"}, - {"id": "1", "first_name": "Samwise", "last_name": "Gamgee"}, - {"id": "1", "first_name": "Samwise", "last_name": "Gamgee", "friend": "Frodo"}, - ), - ( # multiple extra fields - {"friend": "Frodo", "enemy": "Gollum", "loves": "PO-TAY-TOES"}, - {"id": "1", "first_name": "Samwise", "last_name": "Gamgee"}, - {"id": "1", "first_name": "Samwise", "last_name": "Gamgee", "friend": "Frodo", "enemy": "Gollum", "loves": "PO-TAY-TOES"}, - ), - ( # empty extra_map - {}, - {"id": "1", "first_name": "Samwise", "last_name": "Gamgee"}, - {"id": "1", "first_name": "Samwise", "last_name": "Gamgee"}, - ), - ], - ids=["one_extra_field", "multiple_extra_fields", "empty_extra_map"], - ) - @patch( - "source_s3.source_files_abstract.stream.IncrementalFileStream.__abstractmethods__", set() - ) # patching abstractmethods to empty set so we can instantiate ABC to test - @memory_limit(512) - def test_add_extra_fields_from_map( - self, extra_map: Mapping[str, Any], record: Dict[str, Any], expected_return_record: Mapping[str, Any] - ) -> None: - fs = IncrementalFileStream(dataset="dummy", provider={}, format={}, path_pattern="") - if expected_return_record is not None: - assert fs._add_extra_fields_from_map(record, extra_map) == expected_return_record - else: - with pytest.raises(Exception) as e_info: - fs._add_extra_fields_from_map(record, extra_map) - LOGGER.debug(str(e_info)) - - @pytest.mark.parametrize( - "patterns, filepaths, expected_filepaths", - [ - ( # 'everything' case - "**", - [ - "file.csv", - "file.parquet", - "folder/file.csv", - "folder/file.parquet", - "folder/nested/file.csv", - "folder/nested/file.parquet", - "a/b/c/d/e/f/file", - ], - [ - "file.csv", - "file.parquet", - "folder/file.csv", - "folder/file.parquet", - "folder/nested/file.csv", - "folder/nested/file.parquet", - "a/b/c/d/e/f/file", - ], - ), - ( # specific filetype only - "**/*.csv", - [ - "file.csv", - "file.parquet", - "folder/file.csv", - "folder/file.parquet", - "folder/nested/file.csv", - "folder/nested/file.parquet", - "a/b/c/d/e/f/file", - ], - ["file.csv", "folder/file.csv", "folder/nested/file.csv"], - ), - ( # specific filetypes only - "**/*.csv|**/*.parquet", - [ - "file.csv", - "file.parquet", - "folder/file.csv", - "folder/file.parquet", - "folder/nested/file.csv", - "folder/nested/file.parquet", - "a/b/c/d/e/f/file", - ], - [ - "file.csv", - "file.parquet", - "folder/file.csv", - "folder/file.parquet", - "folder/nested/file.csv", - "folder/nested/file.parquet", - ], - ), - ( # 'everything' only 1 level deep - "*/*", - [ - "file.csv", - "file.parquet", - "folder/file.csv", - "folder/file.parquet", - "folder/nested/file.csv", - "folder/nested/file.parquet", - "a/b/c/d/e/f/file", - ], - ["folder/file.csv", "folder/file.parquet"], - ), - ( # 'everything' at least 1 level deep - "*/**", - [ - "file.csv", - "file.parquet", - "folder/file.csv", - "folder/file.parquet", - "folder/nested/file.csv", - "folder/nested/file.parquet", - "a/b/c/d/e/f/file", - ], - ["folder/file.csv", "folder/file.parquet", "folder/nested/file.csv", "folder/nested/file.parquet", "a/b/c/d/e/f/file"], - ), - ( # 'everything' at least 3 levels deep - "*/*/*/**", - [ - "file.csv", - "file.parquet", - "folder/file.csv", - "folder/file.parquet", - "folder/nested/file.csv", - "folder/nested/file.parquet", - "a/b/c/d/e/f/file", - ], - ["a/b/c/d/e/f/file"], - ), - ( # specific filetype at least 1 level deep - "*/**/*.csv", - [ - "file.csv", - "file.parquet", - "folder/file.csv", - "folder/file.parquet", - "folder/nested/file.csv", - "folder/nested/file.parquet", - "a/b/c/d/e/f/file", - ], - ["folder/file.csv", "folder/nested/file.csv"], - ), - ( # 'everything' with specific filename (any filetype) - "**/file.*|**/file", - [ - "NOT_THIS_file.csv", - "folder/NOT_THIS_file.csv", - "file.csv", - "file.parquet", - "folder/file.csv", - "folder/file.parquet", - "folder/nested/file.csv", - "folder/nested/file.parquet", - "a/b/c/d/e/f/file", - ], - [ - "file.csv", - "file.parquet", - "folder/file.csv", - "folder/file.parquet", - "folder/nested/file.csv", - "folder/nested/file.parquet", - "a/b/c/d/e/f/file", - ], - ), - ( # specific dir / any dir / specific dir / any file - "folder/*/files/*", - [ - "file.csv", - "folder/file.csv", - "wrongfolder/xyz/files/1", - "a/b/c/d/e/f/file", - "folder/abc/files/1", - "folder/abc/logs/1", - "folder/xyz/files/1", - ], - ["folder/abc/files/1", "folder/xyz/files/1"], - ), - ( # specific file prefix and filetype, anywhere - "**/prefix*.csv", - [ - "file.csv", - "prefix-file.parquet", - "prefix-file.csv", - "folder/file.parquet", - "folder/nested/prefixmylovelyfile.csv", - "folder/nested/prefix-file.parquet", - ], - ["prefix-file.csv", "folder/nested/prefixmylovelyfile.csv"], - ), - ], - ids=[ - "everything case", - "specific filetype only", - "specific filetypes only", - "everything only 1 level deep", - "everything at least 1 level deep", - "everything at least 3 levels deep", - "specific filetype at least 1 level deep", - "everything with specific filename (any filetype)", - "specific dir / any dir / specific dir / any file", - "specific file prefix and filetype, anywhere", - ], - ) - @patch( - "source_s3.source_files_abstract.stream.IncrementalFileStream.__abstractmethods__", set() - ) # patching abstractmethods to empty set so we can instantiate ABC to test - @memory_limit(512) - def test_pattern_matched_filepath_iterator(self, patterns: str, filepaths: List[str], expected_filepaths: List[str]) -> None: - fs = IncrementalFileStream(dataset="dummy", provider={}, format={}, path_pattern=patterns) - file_infos = [create_by_local_file(filepath) for filepath in filepaths] - assert set([p.key for p in fs.pattern_matched_filepath_iterator(file_infos)]) == set(expected_filepaths) - - @pytest.mark.parametrize( - "latest_record, current_stream_state, expected", - [ - ( # overwrite history file - {"id": 1, "_ab_source_file_last_modified": "2022-05-11T11:54:11Z", "_ab_source_file_url": "new_test_file.csv"}, - {"_ab_source_file_last_modified": "2021-07-25T15:33:04Z", "history": {"2021-07-25": {"old_test_file.csv"}}}, - {"2022-05-11": {"new_test_file.csv"}}, - ), - ( # add file to same day - {"id": 1, "_ab_source_file_last_modified": "2022-07-25T11:54:11Z", "_ab_source_file_url": "new_test_file.csv"}, - {"_ab_source_file_last_modified": "2022-07-25T00:00:00Z", "history": {"2022-07-25": {"old_test_file.csv"}}}, - {"2022-07-25": {"new_test_file.csv", "old_test_file.csv"}}, - ), - ( # add new day to history - {"id": 1, "_ab_source_file_last_modified": "2022-07-03T11:54:11Z", "_ab_source_file_url": "new_test_file.csv"}, - {"_ab_source_file_last_modified": "2022-07-01T00:00:00Z", "history": {"2022-07-01": {"old_test_file.csv"}}}, - {"2022-07-01": {"old_test_file.csv"}, "2022-07-03": {"new_test_file.csv"}}, - ), - ( # history size limit reached - {"_ab_source_file_url": "test.csv"}, - {"_ab_source_file_last_modified": "2022-07-01T00:00:00Z", "history": mock_big_size_object()}, - None, - ), - ], - ids=["overwrite_history_file", "add_file_to_same_day ", "add_new_day_to_history", "history_size_limit_reached"], - ) - @patch( - "source_s3.source_files_abstract.stream.IncrementalFileStream.__abstractmethods__", set() - ) # patching abstractmethods to empty set so we can instantiate ABC to test - def test_get_updated_history(self, latest_record, current_stream_state, expected, request) -> None: - fs = IncrementalFileStream(dataset="dummy", provider={}, format={"filetype": "csv"}, path_pattern="**/prefix*.csv") - fs._get_schema_map = MagicMock(return_value={}) - assert fs.get_updated_state(current_stream_state, latest_record).get("history") == expected - - if request.node.callspec.id == "history_size_limit_reached": - assert fs.sync_all_files_always - - @pytest.mark.parametrize( # set expected_return_record to None for an expected fail - "stream_state, expected_error", - [ - (None, False), - ({"_ab_source_file_last_modified": "2021-07-25T15:33:04Z"}, False), - ({"_ab_source_file_last_modified": "2021-07-25T15:33:04Z"}, False), - ({"_ab_source_file_last_modified": "2021-07-25"}, True), - ], - ) - @patch( - "source_s3.source_files_abstract.stream.IncrementalFileStream.__abstractmethods__", set() - ) # patching abstractmethods to empty set so we can instantiate ABC to test - def test_get_datetime_from_stream_state(self, stream_state, expected_error): - if not expected_error: - assert isinstance( - IncrementalFileStream( - dataset="dummy", provider={"bucket": "test-test"}, format={}, path_pattern="**/prefix*.csv" - )._get_datetime_from_stream_state(stream_state=stream_state), - datetime, - ) - else: - with pytest.raises(Exception): - assert isinstance( - IncrementalFileStream( - dataset="dummy", provider={"bucket": "test-test"}, format={}, path_pattern="**/prefix*.csv" - )._get_datetime_from_stream_state(stream_state=stream_state), - datetime, - ) - - def test_read(self): - stream_instance = IncrementalFileStreamS3( - dataset="dummy", provider={"bucket": "test-test"}, format={}, path_pattern="**/prefix*.csv" - ) - stream_instance.filepath_iterator = MagicMock() - - records = [] - slices = stream_instance.stream_slices(sync_mode=SyncMode.full_refresh) - for slice in slices: - records.extend( - list( - stream_instance.read_records( - stream_slice=slice, - sync_mode=SyncMode.full_refresh, - stream_state={"_ab_source_file_last_modified": "1999-01-01T00:00:00Z"}, - ) - ) - ) - - assert not records - - @patch( - "source_s3.source_files_abstract.stream.StorageFile.__abstractmethods__", set() - ) # patching abstractmethods to empty set so we can instantiate ABC to test - def test_storage_file(self): - size = 1 - date = datetime.now() - file_info = FileInfo(key="", size=size, last_modified=date) - assert StorageFile(file_info=file_info, provider={}).last_modified == date - assert StorageFile(file_info=file_info, provider={}).file_size == size - assert file_info.size_in_megabytes == size / 1024**2 - assert file_info.__str__() - assert file_info.__repr__() - assert file_info == file_info - assert not file_info < file_info - - def test_incremental_read(self): - stream_instance = IncrementalFileStreamS3( - dataset="dummy", provider={"bucket": "test-test"}, format={}, path_pattern="**/prefix*.csv" - ) - stream_instance.filepath_iterator = MagicMock() - - records = [] - slices = stream_instance.stream_slices(sync_mode=SyncMode.incremental, stream_state={}) - - for slice in slices: - records.extend(list(stream_instance.read_records(stream_slice=slice, sync_mode=SyncMode.incremental))) - - assert not records - - def test_fileformatparser_map(self): - stream_instance = IncrementalFileStreamS3( - dataset="dummy", provider={"bucket": "test-test"}, format={}, path_pattern="**/prefix*.csv" - ) - assert stream_instance.file_formatparser_map - - @pytest.mark.parametrize( - ("bucket", "path_prefix", "list_v2_objects", "expected_file_info"), - ( - ( # two files in the first response, one in the second - "test_bucket", - "/widescreen", - [ - { - "Contents": [ - {"Key": "Key_A", "Size": 2048, "LastModified": datetime(2020, 2, 20, 20, 0, 2, tzinfo=timezone.utc)}, - {"Key": "Key_B", "Size": 1024, "LastModified": datetime(2020, 2, 20, 20, 22, 2, tzinfo=timezone.utc)}, - ], - "NextContinuationToken": "token", - }, - {"Contents": [{"Key": "Key_C", "Size": 512, "LastModified": datetime(2022, 2, 2, 2, 2, 2, tzinfo=timezone.utc)}]}, - ], - [ - FileInfo(key="Key_A", size=2048, last_modified=datetime(2020, 2, 20, 20, 0, 2, tzinfo=timezone.utc)), - FileInfo(key="Key_B", size=1024, last_modified=datetime(2020, 2, 20, 20, 22, 2, tzinfo=timezone.utc)), - FileInfo(key="Key_C", size=512, last_modified=datetime(2022, 2, 2, 2, 2, 2, tzinfo=timezone.utc)), - ], - ), - ("another_test_bucket", "/fullscreen", [{}], []), # empty response - ( # some keys are not accepted - "almost_real_test_bucket", - "/HD", - [ - { - "Contents": [ - {"Key": "file/path", "Size": 2048, "LastModified": datetime(2020, 2, 20, 20, 0, 2, tzinfo=timezone.utc)}, - {"Key": "file/path/A/", "Size": 1024, "LastModified": datetime(2020, 2, 20, 20, 22, 2, tzinfo=timezone.utc)}, - ], - "NextContinuationToken": "token", - }, - {"Contents": [{"Key": "file/path/B/", "Size": 512, "LastModified": datetime(2022, 2, 2, 2, 2, 2, tzinfo=timezone.utc)}]}, - ], - [ - FileInfo(key="file/path", size=2048, last_modified=datetime(2020, 2, 20, 20, 0, 2, tzinfo=timezone.utc)), - ], - ), - ), - ) - def test_filepath_iterator(self, bucket, path_prefix, list_v2_objects, expected_file_info): - provider = {"aws_access_key_id": "key_id", "aws_secret_access_key": "access_key"} - s3_client_mock = MagicMock(return_value=MagicMock(list_objects_v2=MagicMock(side_effect=list_v2_objects))) - with patch("source_s3.stream.make_s3_client", s3_client_mock): - stream_instance = IncrementalFileStreamS3( - dataset="dummy", - provider={"bucket": bucket, "path_prefix": path_prefix, **provider}, - format={}, - path_pattern="**/prefix*.png", - ) - expected_info = iter(expected_file_info) - - for file_info in stream_instance.filepath_iterator(): - assert file_info == next(expected_info) - - @pytest.mark.parametrize( - ("start_date", "bucket", "path_prefix", "list_v2_objects", "expected_files_count"), - ( - ("2021-01-01T00:00:00Z", - "test_bucket", - "/widescreen", - [ - { - "Contents": [ - {"Key": "Key_A", "Size": 2048, - "LastModified": datetime(2020, 2, 20, 20, 0, 2, tzinfo=timezone.utc)}, - {"Key": "Key_B", "Size": 1024, - "LastModified": datetime(2020, 2, 20, 20, 22, 2, tzinfo=timezone.utc)}, - ], - "NextContinuationToken": "token", - }, - {"Contents": [{"Key": "Key_C", "Size": 512, - "LastModified": datetime(2022, 2, 2, 2, 2, 2, tzinfo=timezone.utc)}]}, - ], - 1, - ), - ("2023-01-01T00:00:00Z", - "almost_real_test_bucket", - "/HD", - [ - { - "Contents": [ - {"Key": "file/path", "Size": 2048, - "LastModified": datetime(2020, 2, 20, 20, 0, 2, tzinfo=timezone.utc)}, - {"Key": "file/path/A/", "Size": 1024, - "LastModified": datetime(2020, 2, 20, 20, 22, 2, tzinfo=timezone.utc)}, - ], - "NextContinuationToken": "token", - }, - {"Contents": [{"Key": "file/path/B/", "Size": 512, - "LastModified": datetime(2022, 2, 2, 2, 2, 2, tzinfo=timezone.utc)}]}, - ], - 0, - ), - ), - ) - def test_filepath_iterator_date_filter(self, start_date, bucket, path_prefix, list_v2_objects, expected_files_count): - provider = {"aws_access_key_id": "key_id", "aws_secret_access_key": "access_key"} - s3_client_mock = MagicMock(return_value=MagicMock(list_objects_v2=MagicMock(side_effect=list_v2_objects))) - with patch("source_s3.stream.make_s3_client", s3_client_mock): - stream_instance = IncrementalFileStreamS3( - dataset="dummy", - provider={"bucket": bucket, "path_prefix": path_prefix, "start_date":start_date, **provider}, - format={}, - path_pattern="**/prefix*.png" - ) - assert len(list(stream_instance.filepath_iterator())) == expected_files_count - - def test_get_schema(self): - stream_instance = IncrementalFileStreamS3( - dataset="dummy", - provider={}, - format={"filetype": "csv"}, - schema="{\"column_A\": \"string\", \"column_B\": \"integer\", \"column_C\": \"boolean\"}", - path_pattern="**/prefix*.csv" - ) - assert stream_instance._schema == { - "_ab_source_file_last_modified": {"type": "string"}, - "_ab_source_file_url": {"type": "string"}, - "column_A": "string", - "column_B": "integer", - "column_C": "boolean", - } - - @pytest.mark.parametrize( - ("file_type", "error_expected"), - ( - ( - "csv", - False, - ), - ("avro", False), - ("parquet", False), - ("png", True), - ), - ) - def test_fileformatparser_class(self, file_type, error_expected): - stream_instance = IncrementalFileStreamS3( - dataset="dummy", provider={}, format={"filetype": file_type}, schema={}, path_pattern="**/prefix*.csv" - ) - if error_expected: - with pytest.raises(RuntimeError): - _ = stream_instance.fileformatparser_class - else: - assert stream_instance.fileformatparser_class - - def test_get_json_schema(self): - stream_instance = IncrementalFileStreamS3( - dataset="dummy", - provider={}, - format={"filetype": "csv"}, - schema="{\"column_A\": \"string\", \"column_B\": \"integer\", \"column_C\": \"boolean\"}", - path_pattern="**/prefix*.csv" - ) - assert stream_instance.get_json_schema() == { - "properties": { - "_ab_source_file_last_modified": {"format": "date-time", "type": "string"}, - "_ab_source_file_url": {"type": "string"}, - "column_A": {"type": ["null", "string"]}, - "column_B": {"type": ["null", "integer"]}, - "column_C": {"type": ["null", "boolean"]}, - }, - "type": "object", - } - - def test_schema_no_files(self, mocker): - stream_instance = IncrementalFileStreamS3( - dataset="dummy", - provider={"bucket": "empty"}, - format={"filetype": "csv"}, - path_pattern="**/prefix*.csv" - ) - mocker.patch.object(stream_instance, "filepath_iterator", MagicMock(return_value=[])) - assert stream_instance.get_json_schema() == { - "properties": { - "_ab_source_file_last_modified": {"format": "date-time", "type": "string"}, - "_ab_source_file_url": {"type": "string"} - }, - "type": "object", - } - - def test_migrate_datetime_format(self): - current_state = {"_ab_source_file_last_modified": "2022-11-09T11:12:00+0000"} - latest_record = {"_ab_source_file_last_modified": "2020-11-09T11:12:00Z"} - stream_instance = IncrementalFileStreamS3( - dataset="dummy", - provider={"bucket": "empty"}, - format={"filetype": "csv"}, - path_pattern="**/prefix*.csv" - ) - assert stream_instance.get_updated_state(current_state, latest_record)["_ab_source_file_last_modified"] == "2022-11-09T11:12:00Z" - - @pytest.mark.parametrize( - "input_state, expected_output", - [ - pytest.param({}, {}, id="empty-input"), - pytest.param( - { - "history": { - "file1.txt": "2023-08-01T00:00:00.000000Z", - }, - "_ab_source_file_last_modified": "2023-08-01T00:00:00.000000Z_file1.txt", - }, - { - "history": { - "2023-08-01": ["file1.txt"], - }, - "_ab_source_file_last_modified": "2023-08-01T00:00:00Z", - }, - id="single-file-single-timestamp", - ), - pytest.param( - { - "history": { - "file1.txt": "2023-08-01T00:00:00.000000Z", - "file2.txt": "2023-08-01T00:00:00.000000Z", - }, - "_ab_source_file_last_modified": "2023-08-01T00:00:00.000000Z_file2.txt", - }, - { - "history": { - "2023-08-01": ["file1.txt", "file2.txt"], - }, - "_ab_source_file_last_modified": "2023-08-01T00:00:00Z", - }, - id="multiple-files-same-timestamp", - ), - pytest.param( - { - "history": { - "file1.txt": "2023-08-01T00:00:00.000000Z", - "file2.txt": "2023-08-02T00:00:00.000000Z", - "file3.txt": "2023-08-02T00:00:00.000000Z", - }, - "_ab_source_file_last_modified": "2023-08-02T00:00:00.000000Z_file3.txt", - }, - { - "history": { - "2023-08-01": ["file1.txt"], - "2023-08-02": ["file2.txt", "file3.txt"], - }, - "_ab_source_file_last_modified": "2023-08-02T00:00:00Z", - }, - id="multiple-files-different-timestamps", - ), - pytest.param( - { - "history": { - "file1.txt": "2023-08-01T10:30:00.000000Z", - "file2.txt": "2023-08-01T15:45:00.000000Z", - "file3.txt": "2023-08-02T02:00:00.000000Z", - }, - "_ab_source_file_last_modified": "2023-08-02T00:00:00.000000Z_file3.txt", - }, - { - "history": { - "2023-08-01": ["file1.txt", "file2.txt"], - "2023-08-02": ["file3.txt"], - }, - "_ab_source_file_last_modified": "2023-08-02T00:00:00Z", - }, - id="handling-different-times", - ), - ], - ) - def test_get_converted_stream_state(self, input_state, expected_output): - assert IncrementalFileStreamS3(dataset="dummy", provider={}, format={}, path_pattern="")._get_converted_stream_state(input_state) == expected_output - - @pytest.mark.parametrize( - "stream_state, expected_output", - [ - pytest.param({}, False, id="empty-stream-state"), - pytest.param( - { - "history": { - "2023-08-01": "file1.txt", - "2023-08-02": "file2.txt", - "2023-08-03": "file3.txt", - }, - "_ab_source_file_last_modified": "2023-08-01T00:00:00.000000Z", - }, - True, - id="v4-format-history-and-cursor", - ), - pytest.param( - { - "history": { - "2023-08-01": ["file1.txt"], - "2023-08-02": ["file2.txt", "file3.txt"], - }, - "_ab_source_file_last_modified": "2023-08-01T00:00:00Z", - }, - False, - id="v3-format-history-and-cursor", - ), - pytest.param( - { - "history": { - "2023-08-01": "file1.txt", - "2023-08-02": "file2.txt", - }, - }, - True, - id="v4-missing-cursor", - ), - pytest.param( - { - "history": { - "2023-08-01": ["file1.txt"], - "2023-08-02": ["file2.txt", "file3.txt"], - }, - }, - False, - id="v3-missing-cursor", - ), - pytest.param( - { - "_ab_source_file_last_modified": "2023-08-01T00:00:00.000000Z", - }, - True, - id="v4-cursor-only", - ), - pytest.param( - { - "_ab_source_file_last_modified": "2023-08-01T00:00:00Z", # Invalid format (missing microseconds) - }, - False, - id="v3-cursor-only", - ), - ], - ) - def test_is_v4_state_format(self, stream_state: Mapping[str, Any], expected_output): - assert IncrementalFileStreamS3(dataset="dummy", provider={}, format={}, path_pattern="")._is_v4_state_format(stream_state) == expected_output diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/__init__.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/v4/__init__.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_config.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_config.py index 76e326fd2a87..66213120a704 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_config.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_config.py @@ -13,16 +13,36 @@ @pytest.mark.parametrize( - "kwargs,expected_error", + "kwargs, is_cloud, expected_error", [ - pytest.param({"bucket": "test", "streams": []}, None, id="required-fields"), - pytest.param({"bucket": "test", "streams": [], "aws_access_key_id": "access_key", "aws_secret_access_key": "secret_access_key"}, None, id="config-created-with-aws-info"), - pytest.param({"bucket": "test", "streams": [], "endpoint": "http://test.com"}, None, id="config-created-with-endpoint"), - pytest.param({"bucket": "test", "streams": [], "aws_access_key_id": "access_key", "aws_secret_access_key": "secret_access_key", "endpoint": "http://test.com"}, None, id="config-created-with-endpoint-and-aws-info"), - pytest.param({"streams": []}, ValidationError, id="missing-bucket"), - ] + pytest.param({"bucket": "test", "streams": []}, False, None, id="required-fields"), + pytest.param( + {"bucket": "test", "streams": [], "aws_access_key_id": "access_key", "aws_secret_access_key": "secret_access_key"}, + True, + None, + id="config-created-with-aws-info", + ), + pytest.param({"bucket": "test", "streams": [], "endpoint": "https://test.com"}, False, None, id="config-created-with-endpoint"), + pytest.param({"bucket": "test", "streams": [], "endpoint": "http://test.com"}, True, ValidationError, id="http-endpoint-error"), + pytest.param({"bucket": "test", "streams": [], "endpoint": "http://test.com"}, False, None, id="http-endpoint-error"), + pytest.param( + { + "bucket": "test", + "streams": [], + "aws_access_key_id": "access_key", + "aws_secret_access_key": "secret_access_key", + "endpoint": "https://test.com", + }, + True, + None, + id="config-created-with-endpoint-and-aws-info", + ), + pytest.param({"streams": []}, False, ValidationError, id="missing-bucket"), + ], ) -def test_config(kwargs, expected_error): +def test_config(mocker, kwargs, is_cloud, expected_error): + mocker.patch("source_s3.v4.config.is_cloud_environment", lambda: is_cloud) + if expected_error: with pytest.raises(expected_error): Config(**kwargs) diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_cursor.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_cursor.py index 7e612ba89e64..f06e8fd9ae7c 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_cursor.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_cursor.py @@ -7,6 +7,7 @@ from unittest.mock import Mock import pytest +from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat from airbyte_cdk.sources.file_based.config.file_based_stream_config import FileBasedStreamConfig from airbyte_cdk.sources.file_based.remote_file import RemoteFile from airbyte_cdk.sources.file_based.stream.cursor.default_file_based_cursor import DefaultFileBasedCursor @@ -144,7 +145,6 @@ def _create_datetime(dt: str) -> datetime: }, id="empty-history-with-cursor", ), - ], ) def test_set_initial_state(input_state: MutableMapping[str, Any], expected_state: MutableMapping[str, Any]) -> None: @@ -416,9 +416,7 @@ def test_list_files_v4_migration(input_state, all_files, expected_files_to_sync, False, id="legacy_state_with_invalid_last_modified_datetime_format_is_not_legacy", ), - pytest.param( - {"_ab_source_file_last_modified": "2023-08-01T00:00:00Z"}, True, id="legacy_state_without_history_is_legacy_state" - ), + pytest.param({"_ab_source_file_last_modified": "2023-08-01T00:00:00Z"}, True, id="legacy_state_without_history_is_legacy_state"), pytest.param({"history": {"2023-08-01": ["file1.txt"]}}, False, id="legacy_state_without_last_modified_cursor_is_not_legacy_state"), pytest.param( { @@ -486,7 +484,7 @@ def test_get_adjusted_date_timestamp(cursor_datetime, file_datetime, expected_ad def _init_cursor_with_state(input_state, max_history_size: Optional[int] = None) -> Cursor: - cursor = Cursor(stream_config=FileBasedStreamConfig(file_type="csv", name="test", validation_policy="Emit Record")) + cursor = Cursor(stream_config=FileBasedStreamConfig(name="test", validation_policy="Emit Record", format=CsvFormat())) cursor.set_initial_state(input_state) if max_history_size is not None: cursor.DEFAULT_MAX_HISTORY_SIZE = max_history_size diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_legacy_config_transformer.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_legacy_config_transformer.py index dfccfe69b1c3..72f0ec5ba346 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_legacy_config_transformer.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_legacy_config_transformer.py @@ -37,7 +37,6 @@ "streams": [ { "name": "test_data", - "file_type": "avro", "globs": ["**/*.avro"], "legacy_prefix": "a_folder/", "validation_policy": "Emit Record", @@ -65,7 +64,6 @@ "streams": [ { "name": "test_data", - "file_type": "avro", "globs": ["**/*.avro"], "legacy_prefix": "", "validation_policy": "Emit Record", @@ -93,17 +91,16 @@ "streams": [ { "name": "test_data", - "file_type": "avro", "globs": ["*.csv", "**/*"], "validation_policy": "Emit Record", "legacy_prefix": "a_prefix/", "format": {"filetype": "avro"}, } - ] - } - , id="test_convert_with_multiple_path_patterns" + ], + }, + id="test_convert_with_multiple_path_patterns", ), - ] + ], ) def test_convert_legacy_config(legacy_config, expected_config): parsed_legacy_config = SourceS3Spec(**legacy_config) @@ -127,8 +124,8 @@ def test_convert_legacy_config(legacy_config, expected_config): "double_quote": False, "newlines_in_values": True, "additional_reader_options": '{"strings_can_be_null": true, "null_values": ["NULL", "NONE"], "true_values": ["yes", "y"], "false_values": ["no", "n"]}', - "advanced_options": '{"skip_rows": 3, "skip_rows_after_names": 5, "autogenerate_column_names": true}', - "blocksize": 20000, + "advanced_options": '{"skip_rows": 3, "skip_rows_after_names": 5, "autogenerate_column_names": true, "block_size": 20000}', + "block_size": 20000, }, { "filetype": "csv", @@ -144,7 +141,7 @@ def test_convert_legacy_config(legacy_config, expected_config): "strings_can_be_null": True, "skip_rows_before_header": 3, "skip_rows_after_header": 5, - "autogenerate_column_names": True, + "header_definition": {"header_definition_type": "Autogenerated"}, }, None, id="test_csv_all_legacy_options_set", @@ -164,11 +161,30 @@ def test_convert_legacy_config(legacy_config, expected_config): "quote_char": "^", "encoding": "utf8", "double_quote": True, - "null_values": ["", "#N/A", "#N/A N/A", "#NA", "-1.#IND", "-1.#QNAN", "-NaN", "-nan", "1.#IND", "1.#QNAN", "N/A", "NA", "NULL", "NaN", "n/a", "nan", "null"], + "null_values": [ + "", + "#N/A", + "#N/A N/A", + "#NA", + "-1.#IND", + "-1.#QNAN", + "-NaN", + "-nan", + "1.#IND", + "1.#QNAN", + "N/A", + "NA", + "NULL", + "NaN", + "n/a", + "nan", + "null", + ], "true_values": ["1", "True", "TRUE", "true"], "false_values": ["0", "False", "FALSE", "false"], "inference_type": "Primitive Types Only", "strings_can_be_null": False, + "header_definition": {"header_definition_type": "From CSV"}, }, None, id="test_csv_only_required_options", @@ -182,11 +198,30 @@ def test_convert_legacy_config(legacy_config, expected_config): "quote_char": '"', "encoding": "utf8", "double_quote": True, - "null_values": ["", "#N/A", "#N/A N/A", "#NA", "-1.#IND", "-1.#QNAN", "-NaN", "-nan", "1.#IND", "1.#QNAN", "N/A", "NA", "NULL", "NaN", "n/a", "nan", "null"], + "null_values": [ + "", + "#N/A", + "#N/A N/A", + "#NA", + "-1.#IND", + "-1.#QNAN", + "-NaN", + "-nan", + "1.#IND", + "1.#QNAN", + "N/A", + "NA", + "NULL", + "NaN", + "n/a", + "nan", + "null", + ], "true_values": ["1", "True", "TRUE", "true"], "false_values": ["0", "False", "FALSE", "false"], "inference_type": "Primitive Types Only", "strings_can_be_null": False, + "header_definition": {"header_definition_type": "From CSV"}, }, None, id="test_csv_empty_format", @@ -218,15 +253,6 @@ def test_convert_legacy_config(legacy_config, expected_config): ValueError, id="test_malformed_advanced_options", ), - pytest.param( - "csv", - { - "advanced_options": '{"column_names": ""}', - }, - None, - ValueError, - id="test_unsupported_advanced_options", - ), pytest.param( "csv", { @@ -261,6 +287,7 @@ def test_convert_legacy_config(legacy_config, expected_config): "false_values": ["0", "False", "FALSE", "false"], "inference_type": "Primitive Types Only", "strings_can_be_null": False, + "header_definition": {"header_definition_type": "From CSV"}, }, None, id="test_unsupported_advanced_options_by_value_succeeds_if_value_matches_ignored_values", @@ -308,10 +335,42 @@ def test_convert_legacy_config(legacy_config, expected_config): "false_values": ["0", "False", "FALSE", "false"], "inference_type": "Primitive Types Only", "strings_can_be_null": False, + "header_definition": {"header_definition_type": "From CSV"}, }, None, id="test_ignored_advanced_options", ), + pytest.param( + "csv", + { + "filetype": "csv", + "delimiter": "&", + "infer_datatypes": False, + "quote_char": "^", + "escape_char": "$", + "encoding": "ansi", + "double_quote": False, + "newlines_in_values": True, + "additional_reader_options": '{"strings_can_be_null": true, "null_values": ["NULL", "NONE"], "true_values": ["yes", "y"], "false_values": ["no", "n"]}', + "advanced_options": '{"autogenerate_column_names": true, "column_names": ["first", "second"]}', + }, + { + "filetype": "csv", + "delimiter": "&", + "quote_char": "^", + "escape_char": "$", + "encoding": "ansi", + "double_quote": False, + "null_values": ["NULL", "NONE"], + "true_values": ["yes", "y"], + "false_values": ["no", "n"], + "inference_type": "None", + "strings_can_be_null": True, + "header_definition": {"header_definition_type": "User Provided", "column_names": ["first", "second"]}, + }, + None, + id="test_csv_user_provided_column_names", + ), pytest.param( "jsonl", { @@ -367,7 +426,6 @@ def test_convert_file_format(file_type, legacy_format_config, expected_format_co "streams": [ { "name": "test_data", - "file_type": file_type, "globs": [f"**/*.{file_type}"], "legacy_prefix": "", "validation_policy": "Emit Record", diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_source.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_source.py new file mode 100644 index 000000000000..f8848368d763 --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_source.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import unittest +from pathlib import Path +from unittest.mock import Mock, patch + +from source_s3.v4 import Config, SourceS3, SourceS3StreamReader + +_V3_FIELDS = ["dataset", "format", "path_pattern", "provider", "schema"] +TEST_FILES_FOLDER = Path(__file__).resolve().parent.parent.joinpath("sample_files") + + +class SourceTest(unittest.TestCase): + def setUp(self) -> None: + self._stream_reader = Mock(spec=SourceS3StreamReader) + self._source = SourceS3(self._stream_reader, Config, str(TEST_FILES_FOLDER.joinpath("catalog.json"))) + + @patch("source_s3.v4.source.emit_configuration_as_airbyte_control_message") + def test_given_config_is_v3_when_read_config_then_emit_new_config(self, emit_config_mock) -> None: + self._source.read_config(str(TEST_FILES_FOLDER.joinpath("v3_config.json"))) + assert emit_config_mock.call_count == 1 + + @patch("source_s3.v4.source.emit_configuration_as_airbyte_control_message") + def test_given_config_is_v4_when_read_config_then_do_not_emit_new_config(self, emit_config_mock) -> None: + self._source.read_config(str(TEST_FILES_FOLDER.joinpath("v4_config.json"))) + assert emit_config_mock.call_count == 0 + + def test_when_spec_then_v3_fields_not_required(self) -> None: + spec = self._source.spec() + assert all(field not in spec.connectionSpecification["required"] for field in _V3_FIELDS) + + def test_when_spec_then_v3_fields_are_hidden(self) -> None: + spec = self._source.spec() + assert all(spec.connectionSpecification["properties"][field]["airbyte_hidden"] for field in _V3_FIELDS) + + def test_when_spec_then_v3_fields_descriptions_are_prefixed_with_deprecation_warning(self) -> None: + spec = self._source.spec() + assert all( + spec.connectionSpecification["properties"][field]["description"].startswith("Deprecated and will be removed soon") + for field in _V3_FIELDS + ) + + def test_when_spec_then_v3_nested_fields_are_not_required(self) -> None: + spec = self._source.spec() + assert not spec.connectionSpecification["properties"]["provider"]["required"] diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py index 9896e298fcf2..b1bede862d22 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_stream_reader.py @@ -16,13 +16,14 @@ from airbyte_cdk.sources.file_based.file_based_stream_reader import FileReadMode from airbyte_cdk.sources.file_based.remote_file import RemoteFile from botocore.stub import Stubber +from moto import mock_sts from pydantic import AnyUrl from source_s3.v4.config import Config from source_s3.v4.stream_reader import SourceS3StreamReader logger = logging.Logger("") -endpoint_values = ["http://fake.com", None] +endpoint_values = ["https://fake.com", None] _get_matching_files_cases = [ pytest.param([], [], False, set(), id="no-files-match-if-no-globs"), pytest.param( @@ -106,11 +107,10 @@ get_matching_files_cases.append(test_case) -@pytest.mark.parametrize( - "globs,mocked_response,multiple_pages,expected_uris,endpoint", - get_matching_files_cases -) -def test_get_matching_files(globs: List[str], mocked_response: List[Dict[str, Any]], multiple_pages: bool, expected_uris: Set[str], endpoint: Optional[str]): +@pytest.mark.parametrize("globs,mocked_response,multiple_pages,expected_uris,endpoint", get_matching_files_cases) +def test_get_matching_files( + globs: List[str], mocked_response: List[Dict[str, Any]], multiple_pages: bool, expected_uris: Set[str], endpoint: Optional[str] +): reader = SourceS3StreamReader() try: aws_access_key_id = aws_secret_access_key = None if endpoint else "test" @@ -133,7 +133,11 @@ def test_get_matching_files(globs: List[str], mocked_response: List[Dict[str, An @patch("boto3.client") def test_given_multiple_pages_when_get_matching_files_then_pass_continuation_token(boto3_client_mock) -> None: boto3_client_mock.return_value.list_objects_v2.side_effect = [ - {"Contents": [{"Key": "1", "LastModified": datetime.now()}, {"Key": "2", "LastModified": datetime.now()}], "KeyCount": 2, "NextContinuationToken": "a key"}, + { + "Contents": [{"Key": "1", "LastModified": datetime.now()}, {"Key": "2", "LastModified": datetime.now()}], + "KeyCount": 2, + "NextContinuationToken": "a key", + }, {"Contents": [{"Key": "1", "LastModified": datetime.now()}, {"Key": "2", "LastModified": datetime.now()}], "KeyCount": 2}, ] reader = SourceS3StreamReader() @@ -192,7 +196,9 @@ def test_open_file_calls_any_open_with_the_right_encoding(smart_open_mock): with reader.open_file(RemoteFile(uri="", last_modified=datetime.now()), FileReadMode.READ, encoding, logger) as fp: fp.read() - smart_open_mock.assert_called_once_with('s3://test/', transport_params={"client": reader.s3_client}, mode=FileReadMode.READ.value, encoding=encoding) + smart_open_mock.assert_called_once_with( + "s3://test/", transport_params={"client": reader.s3_client}, mode=FileReadMode.READ.value, encoding=encoding + ) def test_get_s3_client_without_config_raises_exception(): @@ -233,3 +239,33 @@ def set_stub(reader: SourceS3StreamReader, contents: List[Dict[str, Any]], multi ) s3_stub.activate() return s3_stub + + +@mock_sts +@patch("source_s3.v4.stream_reader.boto3.client") +def test_get_iam_s3_client(boto3_client_mock): + # Mock the STS client assume_role method + boto3_client_mock.return_value.assume_role.return_value = { + "Credentials": { + "AccessKeyId": "assumed_access_key_id", + "SecretAccessKey": "assumed_secret_access_key", + "SessionToken": "assumed_session_token", + "Expiration": datetime.now(), + } + } + + # Instantiate your stream reader and set the config + reader = SourceS3StreamReader() + reader.config = Config( + bucket="test", + role_arn="arn:aws:iam::123456789012:role/my-role", + streams=[], + endpoint=None, + ) + + # Call _get_iam_s3_client + with Stubber(reader.s3_client): + s3_client = reader._get_iam_s3_client({}) + + # Assertions to validate the s3 client + assert s3_client is not None diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_zip_reader.py b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_zip_reader.py new file mode 100644 index 000000000000..c97e468cba6d --- /dev/null +++ b/airbyte-integrations/connectors/source-s3/unit_tests/v4/test_zip_reader.py @@ -0,0 +1,140 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import datetime +import io +import struct +import zipfile +from unittest.mock import MagicMock, patch + +import pytest +from source_s3.v4.zip_reader import DecompressedStream, RemoteFileInsideArchive, ZipContentReader, ZipFileHandler + + +# Mocking the S3 client and config for testing +@pytest.fixture +def mock_s3_client(): + return MagicMock() + + +@pytest.fixture +def mock_config(): + return MagicMock(bucket="test-bucket") + + +@pytest.fixture +def zip_file_handler(mock_s3_client, mock_config): + return ZipFileHandler(mock_s3_client, mock_config) + + +def test_fetch_data_from_s3(zip_file_handler): + zip_file_handler._fetch_data_from_s3("test_file", 0, 10) + zip_file_handler.s3_client.get_object.assert_called_with(Bucket="test-bucket", Key="test_file", Range="bytes=0-9") + + +def test_find_signature(zip_file_handler): + zip_file_handler.s3_client.head_object.return_value = {"ContentLength": 1024} + + # Mocking the _fetch_data_from_s3 method + zip_file_handler._fetch_data_from_s3 = MagicMock(return_value=b"test" + ZipFileHandler.EOCD_SIGNATURE + b"data") + + result = zip_file_handler._find_signature("test_file", ZipFileHandler.EOCD_SIGNATURE) + assert ZipFileHandler.EOCD_SIGNATURE in result + + +def test_get_central_directory_start(zip_file_handler): + zip_file_handler._find_signature = MagicMock(return_value=b"\x00" * 16 + struct.pack(" Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-salesforce:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-salesforce:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-salesforce:dev . +# Running the spec command against your patched connector +docker run airbyte/source-salesforce:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -77,45 +120,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-salesforce:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-salesforce:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-salesforce:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install ".[tests]" -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-salesforce test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-salesforce:dev \ - && python -m pytest -p connector_acceptance_test.plugin -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-salesforce:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-salesforce:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +139,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-salesforce test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/salesforce.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-salesforce/acceptance-test-config.yml b/airbyte-integrations/connectors/source-salesforce/acceptance-test-config.yml index 51dc04d34b46..c3c2133f001c 100644 --- a/airbyte-integrations/connectors/source-salesforce/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-salesforce/acceptance-test-config.yml @@ -5,51 +5,51 @@ test_strictness_level: high acceptance_tests: spec: tests: - - spec_path: "source_salesforce/spec.yaml" + - spec_path: "source_salesforce/spec.yaml" connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "secrets/config_sandbox.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "secrets/config_sandbox.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: tests: - - config_path: "secrets/config.json" + - config_path: "secrets/config.json" basic_read: tests: - - config_path: "secrets/config.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - empty_streams: - - name: "ActiveScratchOrg" - bypass_reason: "impossible to fill the stream with data because it is an organic traffic" - - name: "ActiveScratchOrgFeed" - bypass_reason: "impossible to fill the stream with data because it is an organic traffic" - - name: "ActiveScratchOrgHistory" - bypass_reason: "impossible to fill the stream with data because it is an organic traffic" - - name: "ActiveScratchOrgShare" - bypass_reason: "impossible to fill the stream with data because it is an organic traffic" - - name: "Describe" - bypass_reason: "Data is not permanent" - ignored_fields: - accounts: - - name: LastViewedDate - bypass_reason: The fields is being updated after any manipulations with account - - name: LastReferencedDate - bypass_reason: The fields is being updated after any manipulations with account - fail_on_extra_columns: false - timeout_seconds: 7200 + - config_path: "secrets/config.json" + expect_records: + path: "integration_tests/expected_records.jsonl" + empty_streams: + - name: "ActiveScratchOrg" + bypass_reason: "impossible to fill the stream with data because it is an organic traffic" + - name: "ActiveScratchOrgFeed" + bypass_reason: "impossible to fill the stream with data because it is an organic traffic" + - name: "ActiveScratchOrgHistory" + bypass_reason: "impossible to fill the stream with data because it is an organic traffic" + - name: "ActiveScratchOrgShare" + bypass_reason: "impossible to fill the stream with data because it is an organic traffic" + - name: "Describe" + bypass_reason: "Data is not permanent" + ignored_fields: + accounts: + - name: LastViewedDate + bypass_reason: The fields is being updated after any manipulations with account + - name: LastReferencedDate + bypass_reason: The fields is being updated after any manipulations with account + fail_on_extra_columns: false + timeout_seconds: 7200 incremental: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/incremental_catalog.json" - future_state: - future_state_path: "integration_tests/future_state.json" - timeout_seconds: 7200 + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/incremental_catalog.json" + future_state: + future_state_path: "integration_tests/future_state.json" + timeout_seconds: 7200 full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 3600 + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 3600 diff --git a/airbyte-integrations/connectors/source-salesforce/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-salesforce/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-salesforce/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-salesforce/build.gradle b/airbyte-integrations/connectors/source-salesforce/build.gradle deleted file mode 100644 index b13cd53c8c04..000000000000 --- a/airbyte-integrations/connectors/source-salesforce/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_salesforce' -} diff --git a/airbyte-integrations/connectors/source-salesforce/integration_tests/bulk_error_test.py b/airbyte-integrations/connectors/source-salesforce/integration_tests/bulk_error_test.py index 546b0202bbcb..d51d68d957df 100644 --- a/airbyte-integrations/connectors/source-salesforce/integration_tests/bulk_error_test.py +++ b/airbyte-integrations/connectors/source-salesforce/integration_tests/bulk_error_test.py @@ -10,11 +10,13 @@ import pytest import requests_mock -from airbyte_cdk.models import SyncMode +from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.sources.streams import Stream from source_salesforce.source import SourceSalesforce HERE = Path(__file__).parent +_ANY_CATALOG = ConfiguredAirbyteCatalog.parse_obj({"streams": []}) +_ANY_CONFIG = {} @pytest.fixture(name="input_config") @@ -31,9 +33,9 @@ def parse_input_sandbox_config(): def get_stream(input_config: Mapping[str, Any], stream_name: str) -> Stream: stream_cls = type("a", (object,), {"name": stream_name}) - configured_stream_cls = type("b", (object,), {"stream": stream_cls()}) + configured_stream_cls = type("b", (object,), {"stream": stream_cls(), "sync_mode": "full_refresh"}) catalog_cls = type("c", (object,), {"streams": [configured_stream_cls()]}) - source = SourceSalesforce() + source = SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG) source.catalog = catalog_cls() return source.streams(input_config)[0] @@ -72,10 +74,7 @@ def test_not_queryable_stream(caplog, input_config): ) def test_failed_jobs_with_successful_switching(caplog, input_sandbox_config, stream_name, log_messages): stream = get_stream(input_sandbox_config, stream_name) - stream_slice = { - "start_date": "2023-01-01T00:00:00.000+0000", - "end_date": "2023-02-01T00:00:00.000+0000" - } + stream_slice = {"start_date": "2023-01-01T00:00:00.000+0000", "end_date": "2023-02-01T00:00:00.000+0000"} expected_record_ids = set(record["Id"] for record in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice)) create_query_matcher = re.compile(r"jobs/query$") @@ -92,7 +91,9 @@ def test_failed_jobs_with_successful_switching(caplog, input_sandbox_config, str m.register_uri("GET", job_matcher, json={"state": "Failed", "errorMessage": "unknown error"}) m.register_uri("DELETE", job_matcher, json={}) with caplog.at_level(logging.WARNING): - loaded_record_ids = set(record["Id"] for record in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice)) + loaded_record_ids = set( + record["Id"] for record in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice) + ) caplog_rec_counter = len(caplog.records) - 1 for log_message in log_messages: diff --git a/airbyte-integrations/connectors/source-salesforce/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-salesforce/integration_tests/expected_records.jsonl index dfa6d5b3b5b8..14552380db0e 100644 --- a/airbyte-integrations/connectors/source-salesforce/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-salesforce/integration_tests/expected_records.jsonl @@ -1,30 +1,29 @@ -{"stream": "Account", "data": {"attributes": {"type": "Account", "url": "/services/data/v57.0/sobjects/Account/0014W000027f6V5QAI"}, "Id": "0014W000027f6V5QAI", "IsDeleted": false, "MasterRecordId": null, "Name": "United Oil & Gas, Singapore", "Type": "Customer - Direct", "ParentId": null, "BillingStreet": "9 Tagore Lane\nSingapore, Singapore 787472\nSingapore", "BillingCity": "Singapore", "BillingState": "Singapore", "BillingPostalCode": null, "BillingCountry": null, "BillingLatitude": null, "BillingLongitude": null, "BillingGeocodeAccuracy": null, "BillingAddress": {"city": "Singapore", "country": null, "geocodeAccuracy": null, "latitude": null, "longitude": null, "postalCode": null, "state": "Singapore", "street": "9 Tagore Lane\nSingapore, Singapore 787472\nSingapore"}, "ShippingStreet": "9 Tagore Lane\nSingapore, Singapore 787472\nSingapore", "ShippingCity": null, "ShippingState": null, "ShippingPostalCode": null, "ShippingCountry": null, "ShippingLatitude": null, "ShippingLongitude": null, "ShippingGeocodeAccuracy": null, "ShippingAddress": {"city": null, "country": null, "geocodeAccuracy": null, "latitude": null, "longitude": null, "postalCode": null, "state": null, "street": "9 Tagore Lane\nSingapore, Singapore 787472\nSingapore"}, "Phone": "(650) 450-8810", "Fax": "(650) 450-8820", "AccountNumber": "CD355120-B", "Website": "http://www.uos.com", "PhotoUrl": "/services/images/photo/0014W000027f6V5QAI", "Sic": "4437", "Industry": "Energy", "AnnualRevenue": null, "NumberOfEmployees": 3000, "Ownership": "Public", "TickerSymbol": "UOS", "Description": null, "Rating": null, "Site": null, "OwnerId": "0054W00000BZkk0QAD", "CreatedDate": "2020-10-22T21:03:23.000+0000", "CreatedById": "0054W00000BZkk0QAD", "LastModifiedDate": "2020-10-22T21:03:23.000+0000", "LastModifiedById": "0054W00000BZkk0QAD", "SystemModstamp": "2020-10-22T21:03:23.000+0000", "LastActivityDate": null, "LastViewedDate": null, "LastReferencedDate": null, "Jigsaw": null, "JigsawCompanyId": null, "CleanStatus": "Pending", "AccountSource": null, "DunsNumber": null, "Tradestyle": null, "NaicsCode": null, "NaicsDesc": null, "YearStarted": null, "SicDesc": null, "DandbCompanyId": null, "OperatingHoursId": null, "CustomerPriority__c": "High", "SLA__c": "Platinum", "Active__c": "Yes", "NumberofLocations__c": 6.0, "UpsellOpportunity__c": "Maybe", "SLASerialNumber__c": "2457", "SLAExpirationDate__c": "2021-05-19"}, "emitted_at": 1684265259323} -{"stream": "Account", "data": {"attributes": {"type": "Account", "url": "/services/data/v57.0/sobjects/Account/0014W000027f6V4QAI"}, "Id": "0014W000027f6V4QAI", "IsDeleted": false, "MasterRecordId": null, "Name": "United Oil & Gas, UK", "Type": "Customer - Direct", "ParentId": null, "BillingStreet": "Kings Park, 17th Avenue, Team Valley Trading Estate,\nGateshead, Tyne and Wear NE26 3HS\nUnited Kingdom", "BillingCity": null, "BillingState": "UK", "BillingPostalCode": null, "BillingCountry": null, "BillingLatitude": null, "BillingLongitude": null, "BillingGeocodeAccuracy": null, "BillingAddress": {"city": null, "country": null, "geocodeAccuracy": null, "latitude": null, "longitude": null, "postalCode": null, "state": "UK", "street": "Kings Park, 17th Avenue, Team Valley Trading Estate,\nGateshead, Tyne and Wear NE26 3HS\nUnited Kingdom"}, "ShippingStreet": "Kings Park, 17th Avenue, Team Valley Trading Estate,\nGateshead, Tyne and Wear NE26 3HS\nUnited Kingdom", "ShippingCity": null, "ShippingState": null, "ShippingPostalCode": null, "ShippingCountry": null, "ShippingLatitude": null, "ShippingLongitude": null, "ShippingGeocodeAccuracy": null, "ShippingAddress": {"city": null, "country": null, "geocodeAccuracy": null, "latitude": null, "longitude": null, "postalCode": null, "state": null, "street": "Kings Park, 17th Avenue, Team Valley Trading Estate,\nGateshead, Tyne and Wear NE26 3HS\nUnited Kingdom"}, "Phone": "+44 191 4956203", "Fax": "+44 191 4956620", "AccountNumber": "CD355119-A", "Website": "http://www.uos.com", "PhotoUrl": "/services/images/photo/0014W000027f6V4QAI", "Sic": "4437", "Industry": "Energy", "AnnualRevenue": null, "NumberOfEmployees": 24000, "Ownership": "Public", "TickerSymbol": "UOS", "Description": null, "Rating": null, "Site": null, "OwnerId": "0054W00000BZkk0QAD", "CreatedDate": "2020-10-22T21:03:23.000+0000", "CreatedById": "0054W00000BZkk0QAD", "LastModifiedDate": "2020-10-22T21:03:23.000+0000", "LastModifiedById": "0054W00000BZkk0QAD", "SystemModstamp": "2020-10-22T21:03:23.000+0000", "LastActivityDate": null, "LastViewedDate": null, "LastReferencedDate": null, "Jigsaw": null, "JigsawCompanyId": null, "CleanStatus": "Pending", "AccountSource": null, "DunsNumber": null, "Tradestyle": null, "NaicsCode": null, "NaicsDesc": null, "YearStarted": null, "SicDesc": null, "DandbCompanyId": null, "OperatingHoursId": null, "CustomerPriority__c": "High", "SLA__c": "Platinum", "Active__c": "Yes", "NumberofLocations__c": 34.0, "UpsellOpportunity__c": "No", "SLASerialNumber__c": "3479", "SLAExpirationDate__c": "2020-11-16"}, "emitted_at": 1684265259326} -{"stream": "Account", "data": {"attributes": {"type": "Account", "url": "/services/data/v57.0/sobjects/Account/0014W000027f6V3QAI"}, "Id": "0014W000027f6V3QAI", "IsDeleted": false, "MasterRecordId": null, "Name": "University of Arizona", "Type": "Customer - Direct", "ParentId": null, "BillingStreet": "888 N Euclid \nHallis Center, Room 501\nTucson, AZ 85721\nUnited States", "BillingCity": "Tucson", "BillingState": "AZ", "BillingPostalCode": null, "BillingCountry": null, "BillingLatitude": null, "BillingLongitude": null, "BillingGeocodeAccuracy": null, "BillingAddress": {"city": "Tucson", "country": null, "geocodeAccuracy": null, "latitude": null, "longitude": null, "postalCode": null, "state": "AZ", "street": "888 N Euclid \nHallis Center, Room 501\nTucson, AZ 85721\nUnited States"}, "ShippingStreet": "888 N Euclid \nHallis Center, Room 501\nTucson, AZ 85721\nUnited States", "ShippingCity": null, "ShippingState": null, "ShippingPostalCode": null, "ShippingCountry": null, "ShippingLatitude": null, "ShippingLongitude": null, "ShippingGeocodeAccuracy": null, "ShippingAddress": {"city": null, "country": null, "geocodeAccuracy": null, "latitude": null, "longitude": null, "postalCode": null, "state": null, "street": "888 N Euclid \nHallis Center, Room 501\nTucson, AZ 85721\nUnited States"}, "Phone": "(520) 773-9050", "Fax": "(520) 773-9060", "AccountNumber": "CD736025", "Website": "www.universityofarizona.com", "PhotoUrl": "/services/images/photo/0014W000027f6V3QAI", "Sic": "7321", "Industry": "Education", "AnnualRevenue": null, "NumberOfEmployees": 39000, "Ownership": "Other", "TickerSymbol": null, "Description": "Leading university in AZ offering undergraduate and graduate programs in arts and humanities, pure sciences, engineering, business, and medicine.", "Rating": "Warm", "Site": null, "OwnerId": "0054W00000BZkk0QAD", "CreatedDate": "2020-10-22T21:03:23.000+0000", "CreatedById": "0054W00000BZkk0QAD", "LastModifiedDate": "2020-10-22T21:03:23.000+0000", "LastModifiedById": "0054W00000BZkk0QAD", "SystemModstamp": "2020-10-22T21:03:23.000+0000", "LastActivityDate": null, "LastViewedDate": null, "LastReferencedDate": null, "Jigsaw": null, "JigsawCompanyId": null, "CleanStatus": "Pending", "AccountSource": null, "DunsNumber": null, "Tradestyle": null, "NaicsCode": null, "NaicsDesc": null, "YearStarted": null, "SicDesc": null, "DandbCompanyId": null, "OperatingHoursId": null, "CustomerPriority__c": "Medium", "SLA__c": "Gold", "Active__c": "Yes", "NumberofLocations__c": 3.0, "UpsellOpportunity__c": "Yes", "SLASerialNumber__c": "8350", "SLAExpirationDate__c": "2020-11-16"}, "emitted_at": 1684265259328} -{"stream": "ActiveFeatureLicenseMetric", "data": {"Id": "5H24W000000DA8jSAG", "MetricsDate": "2021-06-06", "FeatureType": "MarketingUser", "SystemModstamp": "2021-06-06T05:04:12.000Z", "AssignedUserCount": 1, "ActiveUserCount": 1, "TotalLicenseCount": 2}, "emitted_at": 1684265266426} -{"stream": "ActiveFeatureLicenseMetric", "data": {"Id": "5H24W000000DA8pSAG", "MetricsDate": "2021-06-06", "FeatureType": "OfflineUser", "SystemModstamp": "2021-06-06T05:04:12.000Z", "AssignedUserCount": 1, "ActiveUserCount": 1, "TotalLicenseCount": 2}, "emitted_at": 1684265266427} -{"stream": "ActiveFeatureLicenseMetric", "data": {"Id": "5H24W000000DA8qSAG", "MetricsDate": "2021-06-06", "FeatureType": "MobileUser", "SystemModstamp": "2021-06-06T05:04:12.000Z", "AssignedUserCount": 1, "ActiveUserCount": 1, "TotalLicenseCount": 3}, "emitted_at": 1684265266428} -{"stream": "ActivePermSetLicenseMetric", "data": {"Id": "5H14W000000I7Y0SAK", "MetricsDate": "2021-06-06", "PermissionSetLicenseId": "0PL4W0000012s3wWAA", "SystemModstamp": "2021-06-06T05:23:44.000Z", "AssignedUserCount": 0, "ActiveUserCount": 0, "DeveloperName": null, "MasterLabel": null, "TotalLicenses": null}, "emitted_at": 1684265287677} -{"stream": "ActivePermSetLicenseMetric", "data": {"Id": "5H14W000000I7Y1SAK", "MetricsDate": "2021-06-06", "PermissionSetLicenseId": "0PL4W0000012s3xWAA", "SystemModstamp": "2021-06-06T05:23:44.000Z", "AssignedUserCount": 0, "ActiveUserCount": 0, "DeveloperName": null, "MasterLabel": null, "TotalLicenses": null}, "emitted_at": 1684265287678} -{"stream": "ActivePermSetLicenseMetric", "data": {"Id": "5H14W000000I7Y2SAK", "MetricsDate": "2021-06-06", "PermissionSetLicenseId": "0PL4W0000012s3yWAA", "SystemModstamp": "2021-06-06T05:23:44.000Z", "AssignedUserCount": 0, "ActiveUserCount": 0, "DeveloperName": null, "MasterLabel": null, "TotalLicenses": null}, "emitted_at": 1684265287679} -{"stream": "ActiveProfileMetric", "data": {"Id": "5H04W000000MrTCSA0", "MetricsDate": "2021-06-06", "UserLicenseId": "1004W000001gXudQAE", "ProfileId": "00e4W000001VsqfQAC", "SystemModstamp": "2021-06-06T06:50:04.000Z", "AssignedUserCount": 0, "ActiveUserCount": 0}, "emitted_at": 1684265307076} -{"stream": "ActiveProfileMetric", "data": {"Id": "5H04W000000MrTDSA0", "MetricsDate": "2021-06-06", "UserLicenseId": "1004W000001gXudQAE", "ProfileId": "00e4W000002LjMOQA0", "SystemModstamp": "2021-06-06T06:50:04.000Z", "AssignedUserCount": 1, "ActiveUserCount": 1}, "emitted_at": 1684265307077} -{"stream": "ActiveProfileMetric", "data": {"Id": "5H04W000000MrTESA0", "MetricsDate": "2021-06-06", "UserLicenseId": "1004W000001gXudQAE", "ProfileId": "00e4W000002LjMtQAK", "SystemModstamp": "2021-06-06T06:50:04.000Z", "AssignedUserCount": 0, "ActiveUserCount": 0}, "emitted_at": 1684265307078} -{"stream": "AppDefinition", "data": {"Id": "000000000000000AAA", "DurableId": "06m4W000001ldIZQAY", "Label": "Sales", "MasterLabel": "salesforce", "NamespacePrefix": "standard", "DeveloperName": "Sales", "LogoUrl": "/img/salesforce-noname-logo-v2.svg", "Description": "The world's most popular sales force automation (SFA) solution", "UiType": "Aloha", "NavType": "Standard", "UtilityBar": null, "HeaderColor": "#0070D2", "IsOverrideOrgTheme": false, "IsSmallFormFactorSupported": false, "IsMediumFormFactorSupported": false, "IsLargeFormFactorSupported": false, "IsNavPersonalizationDisabled": false, "IsNavAutoTempTabsDisabled": false, "IsNavTabPersistenceDisabled": false}, "emitted_at": 1684265329649} -{"stream": "AppDefinition", "data": {"Id": "000000000000000AAA", "DurableId": "06m4W000001ldIdQAI", "Label": "Service", "MasterLabel": "supportforce", "NamespacePrefix": "standard", "DeveloperName": "Service", "LogoUrl": "/img/salesforce-noname-logo-v2.svg", "Description": "Manage customer service with accounts, contacts, cases, and more", "UiType": "Aloha", "NavType": "Standard", "UtilityBar": null, "HeaderColor": "#0070D2", "IsOverrideOrgTheme": false, "IsSmallFormFactorSupported": false, "IsMediumFormFactorSupported": false, "IsLargeFormFactorSupported": true, "IsNavPersonalizationDisabled": false, "IsNavAutoTempTabsDisabled": false, "IsNavTabPersistenceDisabled": false}, "emitted_at": 1684265329650} -{"stream": "AppDefinition", "data": {"Id": "000000000000000AAA", "DurableId": "06m4W000001ldIeQAI", "Label": "Marketing", "MasterLabel": "Marketing", "NamespacePrefix": "standard", "DeveloperName": "Marketing", "LogoUrl": "/img/salesforce-noname-logo-v2.svg", "Description": "Best-in-class on-demand marketing automation", "UiType": "Aloha", "NavType": "Standard", "UtilityBar": null, "HeaderColor": "#0070D2", "IsOverrideOrgTheme": false, "IsSmallFormFactorSupported": false, "IsMediumFormFactorSupported": false, "IsLargeFormFactorSupported": true, "IsNavPersonalizationDisabled": false, "IsNavAutoTempTabsDisabled": false, "IsNavTabPersistenceDisabled": false}, "emitted_at": 1684265329651} -{"stream": "Asset", "data": {"attributes": {"type": "Asset", "url": "/services/data/v57.0/sobjects/Asset/02i4W00000EkJspQAF"}, "Id": "02i4W00000EkJspQAF", "ContactId": null, "AccountId": "0014W00002DkoWNQAZ", "ParentId": null, "RootAssetId": "02i4W00000EkJspQAF", "Product2Id": null, "ProductCode": null, "IsCompetitorProduct": false, "CreatedDate": "2021-01-18T21:44:57.000+0000", "CreatedById": "0054W00000BZkk0QAD", "LastModifiedDate": "2021-01-18T21:44:57.000+0000", "LastModifiedById": "0054W00000BZkk0QAD", "SystemModstamp": "2021-01-18T21:44:57.000+0000", "IsDeleted": false, "Name": "Radish - Black, Winter, Organic", "SerialNumber": null, "InstallDate": null, "PurchaseDate": null, "UsageEndDate": null, "LifecycleStartDate": null, "LifecycleEndDate": null, "Status": null, "Price": null, "Quantity": null, "Description": null, "OwnerId": "0054W00000BZkk0QAD", "AssetProvidedById": null, "AssetServicedById": null, "IsInternal": false, "AssetLevel": 1, "StockKeepingUnit": null, "HasLifecycleManagement": false, "CurrentMrr": null, "CurrentLifecycleEndDate": null, "CurrentQuantity": null, "CurrentAmount": null, "TotalLifecycleAmount": null, "Street": null, "City": null, "State": null, "PostalCode": null, "Country": null, "Latitude": null, "Longitude": null, "GeocodeAccuracy": null, "Address": null, "LastViewedDate": null, "LastReferencedDate": null}, "emitted_at": 1684265330917} -{"stream": "Asset", "data": {"attributes": {"type": "Asset", "url": "/services/data/v57.0/sobjects/Asset/02i4W00000EkJtDQAV"}, "Id": "02i4W00000EkJtDQAV", "ContactId": null, "AccountId": "0014W00002DkoWJQAZ", "ParentId": null, "RootAssetId": "02i4W00000EkJtDQAV", "Product2Id": null, "ProductCode": null, "IsCompetitorProduct": false, "CreatedDate": "2021-01-18T21:44:57.000+0000", "CreatedById": "0054W00000BZkk0QAD", "LastModifiedDate": "2021-01-18T21:44:57.000+0000", "LastModifiedById": "0054W00000BZkk0QAD", "SystemModstamp": "2021-01-18T21:44:57.000+0000", "IsDeleted": false, "Name": "Mushroom - Morels, Dry", "SerialNumber": null, "InstallDate": null, "PurchaseDate": null, "UsageEndDate": null, "LifecycleStartDate": null, "LifecycleEndDate": null, "Status": null, "Price": null, "Quantity": null, "Description": null, "OwnerId": "0054W00000BZkk0QAD", "AssetProvidedById": null, "AssetServicedById": null, "IsInternal": false, "AssetLevel": 1, "StockKeepingUnit": null, "HasLifecycleManagement": false, "CurrentMrr": null, "CurrentLifecycleEndDate": null, "CurrentQuantity": null, "CurrentAmount": null, "TotalLifecycleAmount": null, "Street": null, "City": null, "State": null, "PostalCode": null, "Country": null, "Latitude": null, "Longitude": null, "GeocodeAccuracy": null, "Address": null, "LastViewedDate": null, "LastReferencedDate": null}, "emitted_at": 1684265330919} -{"stream": "Asset", "data": {"attributes": {"type": "Asset", "url": "/services/data/v57.0/sobjects/Asset/02i4W00000EkJsrQAF"}, "Id": "02i4W00000EkJsrQAF", "ContactId": null, "AccountId": "0014W00002DkoW5QAJ", "ParentId": null, "RootAssetId": "02i4W00000EkJsrQAF", "Product2Id": null, "ProductCode": null, "IsCompetitorProduct": false, "CreatedDate": "2021-01-18T21:44:57.000+0000", "CreatedById": "0054W00000BZkk0QAD", "LastModifiedDate": "2021-01-18T21:44:57.000+0000", "LastModifiedById": "0054W00000BZkk0QAD", "SystemModstamp": "2021-01-18T21:44:57.000+0000", "IsDeleted": false, "Name": "Truffle Cups Green", "SerialNumber": null, "InstallDate": null, "PurchaseDate": null, "UsageEndDate": null, "LifecycleStartDate": null, "LifecycleEndDate": null, "Status": null, "Price": null, "Quantity": null, "Description": null, "OwnerId": "0054W00000BZkk0QAD", "AssetProvidedById": null, "AssetServicedById": null, "IsInternal": false, "AssetLevel": 1, "StockKeepingUnit": null, "HasLifecycleManagement": false, "CurrentMrr": null, "CurrentLifecycleEndDate": null, "CurrentQuantity": null, "CurrentAmount": null, "TotalLifecycleAmount": null, "Street": null, "City": null, "State": null, "PostalCode": null, "Country": null, "Latitude": null, "Longitude": null, "GeocodeAccuracy": null, "Address": null, "LastViewedDate": null, "LastReferencedDate": null}, "emitted_at": 1684265330921} -{"stream": "FormulaFunctionAllowedType", "data": {"Id": "000000000000000AAA", "DurableId": "VALIDATION-ABS", "FunctionId": "ABS", "Type": "VALIDATION"}, "emitted_at": 1684265334221} -{"stream": "FormulaFunctionAllowedType", "data": {"Id": "000000000000000AAA", "DurableId": "VALIDATION-ACOS", "FunctionId": "ACOS", "Type": "VALIDATION"}, "emitted_at": 1684265334222} -{"stream": "FormulaFunctionAllowedType", "data": {"Id": "000000000000000AAA", "DurableId": "VALIDATION-ADDMONTHS", "FunctionId": "ADDMONTHS", "Type": "VALIDATION"}, "emitted_at": 1684265334223} -{"stream": "LeadHistory", "data": {"Id": "0174W00010EpxkSQAR", "IsDeleted": false, "LeadId": "00Q4W00001WGXdDUAX", "CreatedById": "0054W00000BZkk0QAD", "CreatedDate": "2021-11-02T00:20:05.000Z", "Field": "Title", "DataType": "Text", "OldValue": "Co-Founder", "NewValue": "History Track"}, "emitted_at": 1684265342972} -{"stream": "LeadHistory", "data": {"Id": "0174W0001FSln5AQQR", "IsDeleted": false, "LeadId": "00Q4W00001WGXdDUAX", "CreatedById": "0054W00000BZkk0QAD", "CreatedDate": "2023-01-17T15:14:50.000Z", "Field": "FirstName", "DataType": "Text", "OldValue": "Jean", "NewValue": "John"}, "emitted_at": 1684265352478} -{"stream": "LeadHistory", "data": {"Id": "0174W0001FSln5BQQR", "IsDeleted": false, "LeadId": "00Q4W00001WGXdDUAX", "CreatedById": "0054W00000BZkk0QAD", "CreatedDate": "2023-01-17T15:14:50.000Z", "Field": "LastName", "DataType": "Text", "OldValue": "Lafleur", "NewValue": "Doe"}, "emitted_at": 1684265352479} -{"stream": "ObjectPermissions", "data": {"Id": "1104W00002Fjqc6QAB", "ParentId": "0PS4W000002mq70WAA", "SobjectType": "AuthorizationFormConsent", "PermissionsCreate": true, "PermissionsRead": true, "PermissionsEdit": true, "PermissionsDelete": true, "PermissionsViewAllRecords": false, "PermissionsModifyAllRecords": false, "CreatedDate": "2020-10-22T21:03:23.000Z", "CreatedById": "0054W00000CVeyaQAD", "LastModifiedDate": "2020-10-22T21:03:23.000Z", "LastModifiedById": "0054W00000CVeyaQAD", "SystemModstamp": "2020-10-22T21:03:23.000Z"}, "emitted_at": 1684265357620} -{"stream": "ObjectPermissions", "data": {"Id": "1104W00002Fjqc7QAB", "ParentId": "0PS4W000002mq70WAA", "SobjectType": "AuthorizationFormDataUse", "PermissionsCreate": true, "PermissionsRead": true, "PermissionsEdit": true, "PermissionsDelete": true, "PermissionsViewAllRecords": false, "PermissionsModifyAllRecords": false, "CreatedDate": "2020-10-22T21:03:23.000Z", "CreatedById": "0054W00000CVeyaQAD", "LastModifiedDate": "2020-10-22T21:03:23.000Z", "LastModifiedById": "0054W00000CVeyaQAD", "SystemModstamp": "2020-10-22T21:03:23.000Z"}, "emitted_at": 1684265357622} -{"stream": "ObjectPermissions", "data": {"Id": "1104W00002Fjqc8QAB", "ParentId": "0PS4W000002mq70WAA", "SobjectType": "AuthorizationFormText", "PermissionsCreate": true, "PermissionsRead": true, "PermissionsEdit": true, "PermissionsDelete": true, "PermissionsViewAllRecords": false, "PermissionsModifyAllRecords": false, "CreatedDate": "2020-10-22T21:03:23.000Z", "CreatedById": "0054W00000CVeyaQAD", "LastModifiedDate": "2020-10-22T21:03:23.000Z", "LastModifiedById": "0054W00000CVeyaQAD", "SystemModstamp": "2020-10-22T21:03:23.000Z"}, "emitted_at": 1684265357622} -{"stream": "PermissionSetTabSetting", "data": {"Id": "01P4W00005O7pKXUAZ", "ParentId": "0PS4W000002mq7WWAQ", "Visibility": "DefaultOn", "Name": "standard-ConsumptionSchedule", "SystemModstamp": "2020-10-22T21:03:23.000Z"}, "emitted_at": 1684265399726} -{"stream": "PermissionSetTabSetting", "data": {"Id": "01P4W00005O7pLCUAZ", "ParentId": "0PS4W000002mq7OWAQ", "Visibility": "DefaultOn", "Name": "standard-AppLauncher", "SystemModstamp": "2020-10-22T21:03:23.000Z"}, "emitted_at": 1684265399727} -{"stream": "PermissionSetTabSetting", "data": {"Id": "01P4W00005O7pLFUAZ", "ParentId": "0PS4W000002mq7PWAQ", "Visibility": "DefaultOn", "Name": "standard-AppLauncher", "SystemModstamp": "2020-10-22T21:03:23.000Z"}, "emitted_at": 1684265399728} +{"stream": "Account", "data": {"attributes": {"type": "Account", "url": "/services/data/v57.0/sobjects/Account/0014W000027f6V3QAI"}, "Id": "0014W000027f6V3QAI", "IsDeleted": false, "MasterRecordId": null, "Name": "University of Arizona", "Type": "Customer - Direct", "ParentId": null, "BillingStreet": "888 N Euclid \nHallis Center, Room 501\nTucson, AZ 85721\nUnited States", "BillingCity": "Tucson", "BillingState": "AZ", "BillingPostalCode": null, "BillingCountry": null, "BillingLatitude": null, "BillingLongitude": null, "BillingGeocodeAccuracy": null, "BillingAddress": {"city": "Tucson", "country": null, "geocodeAccuracy": null, "latitude": null, "longitude": null, "postalCode": null, "state": "AZ", "street": "888 N Euclid \nHallis Center, Room 501\nTucson, AZ 85721\nUnited States"}, "ShippingStreet": "888 N Euclid \nHallis Center, Room 501\nTucson, AZ 85721\nUnited States", "ShippingCity": null, "ShippingState": null, "ShippingPostalCode": null, "ShippingCountry": null, "ShippingLatitude": null, "ShippingLongitude": null, "ShippingGeocodeAccuracy": null, "ShippingAddress": {"city": null, "country": null, "geocodeAccuracy": null, "latitude": null, "longitude": null, "postalCode": null, "state": null, "street": "888 N Euclid \nHallis Center, Room 501\nTucson, AZ 85721\nUnited States"}, "Phone": "(520) 773-9050", "Fax": "(520) 773-9060", "AccountNumber": "CD736025", "Website": "www.universityofarizona.com", "PhotoUrl": "/services/images/photo/0014W000027f6V3QAI", "Sic": "7321", "Industry": "Education", "AnnualRevenue": null, "NumberOfEmployees": 39000, "Ownership": "Other", "TickerSymbol": null, "Description": "Leading university in AZ offering undergraduate and graduate programs in arts and humanities, pure sciences, engineering, business, and medicine.", "Rating": "Warm", "Site": null, "OwnerId": "0054W00000BZkk0QAD", "CreatedDate": "2020-10-22T21:03:23.000+0000", "CreatedById": "0054W00000BZkk0QAD", "LastModifiedDate": "2020-10-22T21:03:23.000+0000", "LastModifiedById": "0054W00000BZkk0QAD", "SystemModstamp": "2020-10-22T21:03:23.000+0000", "LastActivityDate": null, "LastViewedDate": null, "LastReferencedDate": null, "Jigsaw": null, "JigsawCompanyId": null, "CleanStatus": "Pending", "AccountSource": null, "DunsNumber": null, "Tradestyle": null, "NaicsCode": null, "NaicsDesc": null, "YearStarted": null, "SicDesc": null, "DandbCompanyId": null, "OperatingHoursId": null, "CustomerPriority__c": "Medium", "SLA__c": "Gold", "Active__c": "Yes", "NumberofLocations__c": 3.0, "UpsellOpportunity__c": "Yes", "SLASerialNumber__c": "8350", "SLAExpirationDate__c": "2020-11-16"}, "emitted_at": 1697452022709} +{"stream": "Account", "data": {"attributes": {"type": "Account", "url": "/services/data/v57.0/sobjects/Account/0014W000027f6V2QAI"}, "Id": "0014W000027f6V2QAI", "IsDeleted": false, "MasterRecordId": null, "Name": "Express Logistics and Transport", "Type": "Customer - Channel", "ParentId": null, "BillingStreet": "620 SW 5th Avenue Suite 400\nPortland, Oregon 97204\nUnited States", "BillingCity": "Portland", "BillingState": "OR", "BillingPostalCode": null, "BillingCountry": null, "BillingLatitude": null, "BillingLongitude": null, "BillingGeocodeAccuracy": null, "BillingAddress": {"city": "Portland", "country": null, "geocodeAccuracy": null, "latitude": null, "longitude": null, "postalCode": null, "state": "OR", "street": "620 SW 5th Avenue Suite 400\nPortland, Oregon 97204\nUnited States"}, "ShippingStreet": "620 SW 5th Avenue Suite 400\nPortland, Oregon 97204\nUnited States", "ShippingCity": null, "ShippingState": null, "ShippingPostalCode": null, "ShippingCountry": null, "ShippingLatitude": null, "ShippingLongitude": null, "ShippingGeocodeAccuracy": null, "ShippingAddress": {"city": null, "country": null, "geocodeAccuracy": null, "latitude": null, "longitude": null, "postalCode": null, "state": null, "street": "620 SW 5th Avenue Suite 400\nPortland, Oregon 97204\nUnited States"}, "Phone": "(503) 421-7800", "Fax": "(503) 421-7801", "AccountNumber": "CC947211", "Website": "www.expressl&t.net", "PhotoUrl": "/services/images/photo/0014W000027f6V2QAI", "Sic": "8742", "Industry": "Transportation", "AnnualRevenue": 950000000.0, "NumberOfEmployees": 12300, "Ownership": "Public", "TickerSymbol": "EXLT", "Description": "Commerical logistics and transportation company.", "Rating": "Cold", "Site": null, "OwnerId": "0054W00000BZkk0QAD", "CreatedDate": "2020-10-22T21:03:23.000+0000", "CreatedById": "0054W00000BZkk0QAD", "LastModifiedDate": "2020-10-22T21:03:23.000+0000", "LastModifiedById": "0054W00000BZkk0QAD", "SystemModstamp": "2020-10-22T21:03:23.000+0000", "LastActivityDate": null, "LastViewedDate": null, "LastReferencedDate": null, "Jigsaw": null, "JigsawCompanyId": null, "CleanStatus": "Pending", "AccountSource": null, "DunsNumber": null, "Tradestyle": null, "NaicsCode": null, "NaicsDesc": null, "YearStarted": null, "SicDesc": null, "DandbCompanyId": null, "OperatingHoursId": null, "CustomerPriority__c": "Medium", "SLA__c": "Platinum", "Active__c": "Yes", "NumberofLocations__c": 150.0, "UpsellOpportunity__c": "Maybe", "SLASerialNumber__c": "4724", "SLAExpirationDate__c": "2021-05-19"}, "emitted_at": 1697452022711} +{"stream": "Account", "data": {"attributes": {"type": "Account", "url": "/services/data/v57.0/sobjects/Account/0014W000027f6UyQAI"}, "Id": "0014W000027f6UyQAI", "IsDeleted": false, "MasterRecordId": null, "Name": "Pyramid Construction Inc.", "Type": "Customer - Channel", "ParentId": null, "BillingStreet": "2 Place Jussieu", "BillingCity": "Paris", "BillingState": null, "BillingPostalCode": "75251", "BillingCountry": "France", "BillingLatitude": null, "BillingLongitude": null, "BillingGeocodeAccuracy": null, "BillingAddress": {"city": "Paris", "country": "France", "geocodeAccuracy": null, "latitude": null, "longitude": null, "postalCode": "75251", "state": null, "street": "2 Place Jussieu"}, "ShippingStreet": "2 Place Jussieu", "ShippingCity": "Paris", "ShippingState": null, "ShippingPostalCode": "75251", "ShippingCountry": "France", "ShippingLatitude": null, "ShippingLongitude": null, "ShippingGeocodeAccuracy": null, "ShippingAddress": {"city": "Paris", "country": "France", "geocodeAccuracy": null, "latitude": null, "longitude": null, "postalCode": "75251", "state": null, "street": "2 Place Jussieu"}, "Phone": "(014) 427-4427", "Fax": "(014) 427-4428", "AccountNumber": "CC213425", "Website": "www.pyramid.com", "PhotoUrl": "/services/images/photo/0014W000027f6UyQAI", "Sic": "4253", "Industry": "Construction", "AnnualRevenue": 950000000.0, "NumberOfEmployees": 2680, "Ownership": "Public", "TickerSymbol": "PYR", "Description": null, "Rating": null, "Site": null, "OwnerId": "0054W00000BZkk0QAD", "CreatedDate": "2020-10-22T21:03:23.000+0000", "CreatedById": "0054W00000BZkk0QAD", "LastModifiedDate": "2020-10-22T21:03:23.000+0000", "LastModifiedById": "0054W00000BZkk0QAD", "SystemModstamp": "2020-10-22T21:03:23.000+0000", "LastActivityDate": null, "LastViewedDate": null, "LastReferencedDate": null, "Jigsaw": null, "JigsawCompanyId": null, "CleanStatus": "Pending", "AccountSource": null, "DunsNumber": null, "Tradestyle": null, "NaicsCode": null, "NaicsDesc": null, "YearStarted": null, "SicDesc": null, "DandbCompanyId": null, "OperatingHoursId": null, "CustomerPriority__c": null, "SLA__c": "Silver", "Active__c": "Yes", "NumberofLocations__c": 17.0, "UpsellOpportunity__c": "Maybe", "SLASerialNumber__c": "9840", "SLAExpirationDate__c": "2021-05-19"}, "emitted_at": 1697452022713} +{"stream": "ActiveFeatureLicenseMetric", "data": {"Id": "5H24W00000A0HdTSAV", "MetricsDate": "2023-10-22", "FeatureType": "LiveAgentUser", "SystemModstamp": "2023-10-22T06:25:12.000Z", "AssignedUserCount": 0, "ActiveUserCount": 0, "TotalLicenseCount": 2}, "emitted_at": 1698149527840} +{"stream": "ActiveFeatureLicenseMetric", "data": {"Id": "5H24W00000A0HdUSAV", "MetricsDate": "2023-10-22", "FeatureType": "ChatterAnswersUser", "SystemModstamp": "2023-10-22T06:25:12.000Z", "AssignedUserCount": 0, "ActiveUserCount": 0, "TotalLicenseCount": 30}, "emitted_at": 1698149527840} +{"stream": "ActiveFeatureLicenseMetric", "data": {"Id": "5H24W00000A0HdVSAV", "MetricsDate": "2023-10-22", "FeatureType": "WorkDotComUserFeature", "SystemModstamp": "2023-10-22T06:25:12.000Z", "AssignedUserCount": 0, "ActiveUserCount": 0, "TotalLicenseCount": 5}, "emitted_at": 1698149527840} +{"stream": "ActivePermSetLicenseMetric", "data": {"Id": "5H14W00000InhhuSAB", "MetricsDate": "2023-10-22", "PermissionSetLicenseId": "0PL4W0000012s4lWAA", "SystemModstamp": "2023-10-22T05:05:42.000Z", "AssignedUserCount": 0, "ActiveUserCount": 0, "DeveloperName": "SalesforceCPQ_CPQAAPerm", "MasterLabel": "Salesforce CPQ AA License", "TotalLicenses": 2}, "emitted_at": 1698150158139} +{"stream": "ActivePermSetLicenseMetric", "data": {"Id": "5H14W00000InhhvSAB", "MetricsDate": "2023-10-22", "PermissionSetLicenseId": "0PL4W0000012s4mWAA", "SystemModstamp": "2023-10-22T05:05:42.000Z", "AssignedUserCount": 0, "ActiveUserCount": 0, "DeveloperName": "SchedulingLineAmbassadorPsl", "MasterLabel": "Queue Manager", "TotalLicenses": 90}, "emitted_at": 1698150158139} +{"stream": "ActivePermSetLicenseMetric", "data": {"Id": "5H14W00000InhhwSAB", "MetricsDate": "2023-10-22", "PermissionSetLicenseId": "0PL4W0000012s4nWAA", "SystemModstamp": "2023-10-22T05:05:42.000Z", "AssignedUserCount": 0, "ActiveUserCount": 0, "DeveloperName": "SalesforceCPQ_CPQStandardPerm", "MasterLabel": "Salesforce CPQ License", "TotalLicenses": 2}, "emitted_at": 1698150158140} +{"stream": "ActiveProfileMetric", "data": {"Id": "5H04W00000U3Ph4SAF", "MetricsDate": "2023-10-22", "UserLicenseId": "1004W000001gXv2QAE", "ProfileId": "00e4W000002LjMoQAK", "SystemModstamp": "2023-10-22T05:59:12.000Z", "AssignedUserCount": 0, "ActiveUserCount": 0}, "emitted_at": 1698150320258} +{"stream": "ActiveProfileMetric", "data": {"Id": "5H04W00000U3Ph5SAF", "MetricsDate": "2023-10-22", "UserLicenseId": "1004W000001gXv3QAE", "ProfileId": "00e4W000002LjMqQAK", "SystemModstamp": "2023-10-22T05:59:12.000Z", "AssignedUserCount": 0, "ActiveUserCount": 0}, "emitted_at": 1698150320258} +{"stream": "ActiveProfileMetric", "data": {"Id": "5H04W00000U3Ph6SAF", "MetricsDate": "2023-10-22", "UserLicenseId": "1004W000001gXv4QAE", "ProfileId": "00e4W000002LjMrQAK", "SystemModstamp": "2023-10-22T05:59:12.000Z", "AssignedUserCount": 0, "ActiveUserCount": 0}, "emitted_at": 1698150320259} +{"stream": "AppDefinition", "data": {"Id": "000000000000000AAA", "DurableId": "06m4W000001ldIZQAY", "Label": "Sales", "MasterLabel": "salesforce", "NamespacePrefix": "standard", "DeveloperName": "Sales", "LogoUrl": "/img/salesforce-noname-logo-v2.svg", "Description": "The world's most popular sales force automation (SFA) solution", "UiType": "Aloha", "NavType": "Standard", "UtilityBar": null, "HeaderColor": "#0070D2", "IsOverrideOrgTheme": false, "IsSmallFormFactorSupported": false, "IsMediumFormFactorSupported": false, "IsLargeFormFactorSupported": false, "IsNavPersonalizationDisabled": false, "IsNavAutoTempTabsDisabled": false, "IsNavTabPersistenceDisabled": false}, "emitted_at": 1697452785550} +{"stream": "AppDefinition", "data": {"Id": "000000000000000AAA", "DurableId": "06m4W000001ldIdQAI", "Label": "Service", "MasterLabel": "supportforce", "NamespacePrefix": "standard", "DeveloperName": "Service", "LogoUrl": "/img/salesforce-noname-logo-v2.svg", "Description": "Manage customer service with accounts, contacts, cases, and more", "UiType": "Aloha", "NavType": "Standard", "UtilityBar": null, "HeaderColor": "#0070D2", "IsOverrideOrgTheme": false, "IsSmallFormFactorSupported": false, "IsMediumFormFactorSupported": false, "IsLargeFormFactorSupported": true, "IsNavPersonalizationDisabled": false, "IsNavAutoTempTabsDisabled": false, "IsNavTabPersistenceDisabled": false}, "emitted_at": 1697452785551} +{"stream": "AppDefinition", "data": {"Id": "000000000000000AAA", "DurableId": "06m4W000001ldIeQAI", "Label": "Marketing", "MasterLabel": "Marketing", "NamespacePrefix": "standard", "DeveloperName": "Marketing", "LogoUrl": "/img/salesforce-noname-logo-v2.svg", "Description": "Best-in-class on-demand marketing automation", "UiType": "Aloha", "NavType": "Standard", "UtilityBar": null, "HeaderColor": "#0070D2", "IsOverrideOrgTheme": false, "IsSmallFormFactorSupported": false, "IsMediumFormFactorSupported": false, "IsLargeFormFactorSupported": true, "IsNavPersonalizationDisabled": false, "IsNavAutoTempTabsDisabled": false, "IsNavTabPersistenceDisabled": false}, "emitted_at": 1697452785552} +{"stream": "Asset", "data": {"attributes": {"type": "Asset", "url": "/services/data/v57.0/sobjects/Asset/02i4W00000EkJspQAF"}, "Id": "02i4W00000EkJspQAF", "ContactId": null, "AccountId": "0014W00002DkoWNQAZ", "ParentId": null, "RootAssetId": "02i4W00000EkJspQAF", "Product2Id": null, "ProductCode": null, "IsCompetitorProduct": false, "CreatedDate": "2021-01-18T21:44:57.000+0000", "CreatedById": "0054W00000BZkk0QAD", "LastModifiedDate": "2021-01-18T21:44:57.000+0000", "LastModifiedById": "0054W00000BZkk0QAD", "SystemModstamp": "2021-01-18T21:44:57.000+0000", "IsDeleted": false, "Name": "Radish - Black, Winter, Organic", "SerialNumber": null, "InstallDate": null, "PurchaseDate": null, "UsageEndDate": null, "LifecycleStartDate": null, "LifecycleEndDate": null, "Status": null, "Price": null, "Quantity": null, "Description": null, "OwnerId": "0054W00000BZkk0QAD", "AssetProvidedById": null, "AssetServicedById": null, "IsInternal": false, "AssetLevel": 1, "StockKeepingUnit": null, "HasLifecycleManagement": false, "CurrentMrr": null, "CurrentLifecycleEndDate": null, "CurrentQuantity": null, "CurrentAmount": null, "TotalLifecycleAmount": null, "Street": null, "City": null, "State": null, "PostalCode": null, "Country": null, "Latitude": null, "Longitude": null, "GeocodeAccuracy": null, "Address": null, "LastViewedDate": null, "LastReferencedDate": null}, "emitted_at": 1697452787097} +{"stream": "Asset", "data": {"attributes": {"type": "Asset", "url": "/services/data/v57.0/sobjects/Asset/02i4W00000EkJsqQAF"}, "Id": "02i4W00000EkJsqQAF", "ContactId": null, "AccountId": "0014W00002DkoW0QAJ", "ParentId": null, "RootAssetId": "02i4W00000EkJsqQAF", "Product2Id": null, "ProductCode": null, "IsCompetitorProduct": false, "CreatedDate": "2021-01-18T21:44:57.000+0000", "CreatedById": "0054W00000BZkk0QAD", "LastModifiedDate": "2021-01-18T21:44:57.000+0000", "LastModifiedById": "0054W00000BZkk0QAD", "SystemModstamp": "2021-01-18T21:44:57.000+0000", "IsDeleted": false, "Name": "Cheese - Valancey", "SerialNumber": null, "InstallDate": null, "PurchaseDate": null, "UsageEndDate": null, "LifecycleStartDate": null, "LifecycleEndDate": null, "Status": null, "Price": null, "Quantity": null, "Description": null, "OwnerId": "0054W00000BZkk0QAD", "AssetProvidedById": null, "AssetServicedById": null, "IsInternal": false, "AssetLevel": 1, "StockKeepingUnit": null, "HasLifecycleManagement": false, "CurrentMrr": null, "CurrentLifecycleEndDate": null, "CurrentQuantity": null, "CurrentAmount": null, "TotalLifecycleAmount": null, "Street": null, "City": null, "State": null, "PostalCode": null, "Country": null, "Latitude": null, "Longitude": null, "GeocodeAccuracy": null, "Address": null, "LastViewedDate": null, "LastReferencedDate": null}, "emitted_at": 1697452787099} +{"stream": "Asset", "data": {"attributes": {"type": "Asset", "url": "/services/data/v57.0/sobjects/Asset/02i4W00000EkJsrQAF"}, "Id": "02i4W00000EkJsrQAF", "ContactId": null, "AccountId": "0014W00002DkoW5QAJ", "ParentId": null, "RootAssetId": "02i4W00000EkJsrQAF", "Product2Id": null, "ProductCode": null, "IsCompetitorProduct": false, "CreatedDate": "2021-01-18T21:44:57.000+0000", "CreatedById": "0054W00000BZkk0QAD", "LastModifiedDate": "2021-01-18T21:44:57.000+0000", "LastModifiedById": "0054W00000BZkk0QAD", "SystemModstamp": "2021-01-18T21:44:57.000+0000", "IsDeleted": false, "Name": "Truffle Cups Green", "SerialNumber": null, "InstallDate": null, "PurchaseDate": null, "UsageEndDate": null, "LifecycleStartDate": null, "LifecycleEndDate": null, "Status": null, "Price": null, "Quantity": null, "Description": null, "OwnerId": "0054W00000BZkk0QAD", "AssetProvidedById": null, "AssetServicedById": null, "IsInternal": false, "AssetLevel": 1, "StockKeepingUnit": null, "HasLifecycleManagement": false, "CurrentMrr": null, "CurrentLifecycleEndDate": null, "CurrentQuantity": null, "CurrentAmount": null, "TotalLifecycleAmount": null, "Street": null, "City": null, "State": null, "PostalCode": null, "Country": null, "Latitude": null, "Longitude": null, "GeocodeAccuracy": null, "Address": null, "LastViewedDate": null, "LastReferencedDate": null}, "emitted_at": 1697452787100} +{"stream": "FormulaFunctionAllowedType", "data": {"Id": "000000000000000AAA", "DurableId": "VALIDATION-ABS", "FunctionId": "ABS", "Type": "VALIDATION"}, "emitted_at": 1697452795368} +{"stream": "FormulaFunctionAllowedType", "data": {"Id": "000000000000000AAA", "DurableId": "VALIDATION-ACOS", "FunctionId": "ACOS", "Type": "VALIDATION"}, "emitted_at": 1697452795369} +{"stream": "FormulaFunctionAllowedType", "data": {"Id": "000000000000000AAA", "DurableId": "VALIDATION-ADDMONTHS", "FunctionId": "ADDMONTHS", "Type": "VALIDATION"}, "emitted_at": 1697452795369} +{"stream": "LeadHistory", "data": {"Id": "0174W0001FSln5AQQR", "IsDeleted": false, "LeadId": "00Q4W00001WGXdDUAX", "CreatedById": "0054W00000BZkk0QAD", "CreatedDate": "2023-01-17T15:14:50.000Z", "Field": "FirstName", "DataType": "Text", "OldValue": "Jean", "NewValue": "John"}, "emitted_at": 1697452875136} +{"stream": "LeadHistory", "data": {"Id": "0174W0001FSln5BQQR", "IsDeleted": false, "LeadId": "00Q4W00001WGXdDUAX", "CreatedById": "0054W00000BZkk0QAD", "CreatedDate": "2023-01-17T15:14:50.000Z", "Field": "LastName", "DataType": "Text", "OldValue": "Lafleur", "NewValue": "Doe"}, "emitted_at": 1697452875137} +{"stream": "ObjectPermissions", "data": {"Id": "1104W00002AG1dYQAT", "ParentId": "0PS4W000002mq7VWAQ", "SobjectType": "GtwyProvPaymentMethodType", "PermissionsCreate": false, "PermissionsRead": true, "PermissionsEdit": false, "PermissionsDelete": false, "PermissionsViewAllRecords": true, "PermissionsModifyAllRecords": false, "CreatedDate": "2020-10-22T21:04:28.000Z", "CreatedById": "0054W00000BZkk0QAD", "LastModifiedDate": "2020-10-22T21:04:28.000Z", "LastModifiedById": "0054W00000CVeybQAD", "SystemModstamp": "2020-10-22T21:04:28.000Z"}, "emitted_at": 1697452900106} +{"stream": "ObjectPermissions", "data": {"Id": "1104W00002AG21vQAD", "ParentId": "0PS4W000002mq72WAA", "SobjectType": "LocationWaitlistedParty", "PermissionsCreate": true, "PermissionsRead": true, "PermissionsEdit": true, "PermissionsDelete": true, "PermissionsViewAllRecords": true, "PermissionsModifyAllRecords": true, "CreatedDate": "2020-10-22T21:05:07.000Z", "CreatedById": "0054W00000CVeybQAD", "LastModifiedDate": "2020-10-22T21:05:07.000Z", "LastModifiedById": "0054W00000CVeybQAD", "SystemModstamp": "2020-10-22T21:05:07.000Z"}, "emitted_at": 1697452900107} +{"stream": "ObjectPermissions", "data": {"Id": "1104W00002AG229QAD", "ParentId": "0PS4W000002mq72WAA", "SobjectType": "LocationGroup", "PermissionsCreate": true, "PermissionsRead": true, "PermissionsEdit": true, "PermissionsDelete": true, "PermissionsViewAllRecords": true, "PermissionsModifyAllRecords": true, "CreatedDate": "2020-10-22T21:05:07.000Z", "CreatedById": "0054W00000CVeybQAD", "LastModifiedDate": "2020-10-22T21:05:07.000Z", "LastModifiedById": "0054W00000CVeybQAD", "SystemModstamp": "2020-10-22T21:05:07.000Z"}, "emitted_at": 1697452900107} +{"stream": "PermissionSetTabSetting", "data": {"Id": "01P4W00005O7rDUUAZ", "ParentId": "0PS4W000002mq7NWAQ", "Visibility": "DefaultOn", "Name": "standard-Event", "SystemModstamp": "2020-10-22T21:03:23.000Z"}, "emitted_at": 1697453031486} +{"stream": "PermissionSetTabSetting", "data": {"Id": "01P4W00005O7rDaUAJ", "ParentId": "0PS4W000002mq7WWAQ", "Visibility": "DefaultOn", "Name": "standard-Event", "SystemModstamp": "2020-10-22T21:03:23.000Z"}, "emitted_at": 1697453031487} +{"stream": "PermissionSetTabSetting", "data": {"Id": "01P4W00005O7rDoUAJ", "ParentId": "0PS4W000002mq78WAA", "Visibility": "DefaultOn", "Name": "standard-Feed", "SystemModstamp": "2020-10-22T21:03:23.000Z"}, "emitted_at": 1697453031487} diff --git a/airbyte-integrations/connectors/source-salesforce/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-salesforce/integration_tests/integration_test.py index c0daa413049c..0cb90aa8b52a 100644 --- a/airbyte-integrations/connectors/source-salesforce/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/source-salesforce/integration_tests/integration_test.py @@ -86,7 +86,7 @@ def test_update_for_deleted_record(stream): now = pendulum.now(tz="UTC") stream_slice = { "start_date": now.add(days=-1).isoformat(timespec="milliseconds"), - "end_date": now.isoformat(timespec="milliseconds") + "end_date": now.isoformat(timespec="milliseconds"), } notes = set(record["Id"] for record in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice)) try: @@ -110,7 +110,7 @@ def test_update_for_deleted_record(stream): now = pendulum.now(tz="UTC") stream_slice = { "start_date": now.add(days=-1).isoformat(timespec="milliseconds"), - "end_date": now.isoformat(timespec="milliseconds") + "end_date": now.isoformat(timespec="milliseconds"), } for record in stream.read_records(sync_mode=SyncMode.incremental, stream_state=stream_state, stream_slice=stream_slice): if created_note_id == record["Id"]: @@ -147,7 +147,7 @@ def test_deleted_record(stream): now = pendulum.now(tz="UTC") stream_slice = { "start_date": now.add(days=-1).isoformat(timespec="milliseconds"), - "end_date": now.isoformat(timespec="milliseconds") + "end_date": now.isoformat(timespec="milliseconds"), } notes = set(record["Id"] for record in stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice)) try: @@ -173,7 +173,7 @@ def test_deleted_record(stream): now = pendulum.now(tz="UTC") stream_slice = { "start_date": now.add(days=-1).isoformat(timespec="milliseconds"), - "end_date": now.isoformat(timespec="milliseconds") + "end_date": now.isoformat(timespec="milliseconds"), } record = None for record in stream.read_records(sync_mode=SyncMode.incremental, stream_state=stream_state, stream_slice=stream_slice): diff --git a/airbyte-integrations/connectors/source-salesforce/main.py b/airbyte-integrations/connectors/source-salesforce/main.py index 2ce8a19b8a7e..67536217f497 100644 --- a/airbyte-integrations/connectors/source-salesforce/main.py +++ b/airbyte-integrations/connectors/source-salesforce/main.py @@ -3,11 +3,7 @@ # -import sys - -from airbyte_cdk.entrypoint import launch -from source_salesforce import SourceSalesforce +from source_salesforce.run import run if __name__ == "__main__": - source = SourceSalesforce() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-salesforce/metadata.yaml b/airbyte-integrations/connectors/source-salesforce/metadata.yaml index ffb43710b442..498448722b27 100644 --- a/airbyte-integrations/connectors/source-salesforce/metadata.yaml +++ b/airbyte-integrations/connectors/source-salesforce/metadata.yaml @@ -1,12 +1,18 @@ data: + ab_internal: + ql: 400 + sl: 300 allowedHosts: hosts: - "*.salesforce.com" + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: api connectorType: source definitionId: b117307c-14b6-41aa-9422-947e34922962 - dockerImageTag: 2.1.4 + dockerImageTag: 2.2.2 dockerRepository: airbyte/source-salesforce + documentationUrl: https://docs.airbyte.com/integrations/sources/salesforce githubIssueLabel: source-salesforce icon: salesforce.svg license: ELv2 @@ -17,11 +23,7 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/salesforce + supportLevel: certified tags: - language:python - ab_internal: - sl: 300 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-salesforce/setup.py b/airbyte-integrations/connectors/source-salesforce/setup.py index 44c137056bbd..4add132d7cb5 100644 --- a/airbyte-integrations/connectors/source-salesforce/setup.py +++ b/airbyte-integrations/connectors/source-salesforce/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.50", "pandas"] +MAIN_REQUIREMENTS = ["airbyte-cdk~=0.55.2", "pandas"] TEST_REQUIREMENTS = ["freezegun", "pytest~=6.1", "pytest-mock~=3.6", "requests-mock~=1.9.3", "pytest-timeout"] @@ -20,4 +20,9 @@ extras_require={ "tests": TEST_REQUIREMENTS, }, + entry_points={ + "console_scripts": [ + "source-salesforce=source_salesforce.run:run", + ], + }, ) diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py index e88c4db1ff0b..eb0eed9ef70d 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/api.py @@ -51,7 +51,6 @@ "AppTabMember", "CollaborationGroupRecord", "ColorDefinition", - "ContentDocumentLink", "ContentFolderItem", "ContentFolderMember", "DataStatistics", @@ -129,6 +128,19 @@ "UserRecordAccess", ] +PARENT_SALESFORCE_OBJECTS = { + # parent_name - name of parent stream + # field - in each parent record, which is needed for stream slice + # schema_minimal - required for getting proper class name full_refresh/incremental, rest/bulk for parent stream + "ContentDocumentLink": { + "parent_name": "ContentDocument", + "field": "Id", + "schema_minimal": { + "properties": {"Id": {"type": ["string", "null"]}, "SystemModstamp": {"type": ["string", "null"], "format": "date-time"}} + }, + } +} + # The following objects are not supported by the Bulk API. Listed objects are version specific. UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS = [ "AcceptedEventRelation", @@ -184,6 +196,7 @@ UNSUPPORTED_FILTERING_STREAMS = [ "ApiEvent", "BulkApiResultEventStore", + "ContentDocumentLink", "EmbeddedServiceDetail", "EmbeddedServiceLabel", "FormulaFunction", diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/run.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/run.py new file mode 100644 index 000000000000..7fe23dc8958c --- /dev/null +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/run.py @@ -0,0 +1,45 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys +import traceback +from datetime import datetime +from typing import List + +from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch +from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type +from source_salesforce import SourceSalesforce + + +def _get_source(args: List[str]): + catalog_path = AirbyteEntrypoint.extract_catalog(args) + config_path = AirbyteEntrypoint.extract_config(args) + try: + return SourceSalesforce( + SourceSalesforce.read_catalog(catalog_path) if catalog_path else None, + SourceSalesforce.read_config(config_path) if config_path else None, + ) + except Exception as error: + print( + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.ERROR, + emitted_at=int(datetime.now().timestamp() * 1000), + error=AirbyteErrorTraceMessage( + message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}", + stack_trace=traceback.format_exc(), + ), + ), + ).json() + ) + return None + + +def run(): + _args = sys.argv[1:] + source = _get_source(_args) + if source: + launch(source, _args) diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py index 9def53730d79..30eea954dfe0 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/source.py @@ -8,19 +8,34 @@ import requests from airbyte_cdk import AirbyteLogger -from airbyte_cdk.models import AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream -from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.logger import AirbyteLogFormatter +from airbyte_cdk.models import AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, Level, SyncMode +from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource +from airbyte_cdk.sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager +from airbyte_cdk.sources.message import InMemoryMessageRepository from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade +from airbyte_cdk.sources.streams.concurrent.cursor import NoopCursor from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator from airbyte_cdk.sources.utils.schema_helpers import InternalConfig from airbyte_cdk.utils.traced_exception import AirbyteTracedException from dateutil.relativedelta import relativedelta from requests import codes, exceptions # type: ignore[import] -from .api import UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS, UNSUPPORTED_FILTERING_STREAMS, Salesforce -from .streams import BulkIncrementalSalesforceStream, BulkSalesforceStream, Describe, IncrementalRestSalesforceStream, RestSalesforceStream - +from .api import PARENT_SALESFORCE_OBJECTS, UNSUPPORTED_BULK_API_SALESFORCE_OBJECTS, UNSUPPORTED_FILTERING_STREAMS, Salesforce +from .streams import ( + BulkIncrementalSalesforceStream, + BulkSalesforceStream, + BulkSalesforceSubStream, + Describe, + IncrementalRestSalesforceStream, + RestSalesforceStream, + RestSalesforceSubStream, +) + +_DEFAULT_CONCURRENCY = 10 +_MAX_CONCURRENCY = 10 logger = logging.getLogger("airbyte") @@ -28,13 +43,24 @@ class AirbyteStopSync(AirbyteTracedException): pass -class SourceSalesforce(AbstractSource): +class SourceSalesforce(ConcurrentSourceAdapter): DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" START_DATE_OFFSET_IN_YEARS = 2 - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.catalog = None + MAX_WORKERS = 5 + + message_repository = InMemoryMessageRepository(Level(AirbyteLogFormatter.level_mapping[logger.level])) + + def __init__(self, catalog: Optional[ConfiguredAirbyteCatalog], config: Optional[Mapping[str, Any]], **kwargs): + if config: + concurrency_level = min(config.get("num_workers", _DEFAULT_CONCURRENCY), _MAX_CONCURRENCY) + else: + concurrency_level = _DEFAULT_CONCURRENCY + logger.info(f"Using concurrent cdk with concurrency level {concurrency_level}") + concurrent_source = ConcurrentSource.create( + concurrency_level, concurrency_level // 2, logger, self._slice_logger, self.message_repository + ) + super().__init__(concurrent_source) + self.catalog = catalog @staticmethod def _get_sf_object(config: Mapping[str, Any]) -> Salesforce: @@ -61,8 +87,10 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> return True, None @classmethod - def _get_api_type(cls, stream_name: str, properties: Mapping[str, Any], force_use_bulk_api: bool) -> str: + def _get_api_type(cls, stream_name: str, json_schema: Mapping[str, Any], force_use_bulk_api: bool) -> str: + """Get proper API type: rest or bulk""" # Salesforce BULK API currently does not support loading fields with data type base64 and compound data + properties = json_schema.get("properties", {}) properties_not_supported_by_bulk = { key: value for key, value in properties.items() if value.get("format") == "base64" or "object" in value["type"] } @@ -79,6 +107,48 @@ def _get_api_type(cls, stream_name: str, properties: Mapping[str, Any], force_us return "rest" return "bulk" + @classmethod + def _get_stream_type(cls, stream_name: str, api_type: str): + """Get proper stream class: full_refresh, incremental or substream + + SubStreams (like ContentDocumentLink) do not support incremental sync because of query restrictions, look here: + https://developer.salesforce.com/docs/atlas.en-us.object_reference.meta/object_reference/sforce_api_objects_contentdocumentlink.htm + """ + parent_name = PARENT_SALESFORCE_OBJECTS.get(stream_name, {}).get("parent_name") + if api_type == "rest": + full_refresh = RestSalesforceSubStream if parent_name else RestSalesforceStream + incremental = IncrementalRestSalesforceStream + elif api_type == "bulk": + full_refresh = BulkSalesforceSubStream if parent_name else BulkSalesforceStream + incremental = BulkIncrementalSalesforceStream + else: + raise Exception(f"Stream {stream_name} cannot be processed by REST or BULK API.") + return full_refresh, incremental + + @classmethod + def prepare_stream(cls, stream_name: str, json_schema, sobject_options, sf_object, authenticator, config): + """Choose proper stream class: syncMode(full_refresh/incremental), API type(Rest/Bulk), SubStream""" + pk, replication_key = sf_object.get_pk_and_replication_key(json_schema) + stream_kwargs = { + "stream_name": stream_name, + "schema": json_schema, + "pk": pk, + "sobject_options": sobject_options, + "sf_api": sf_object, + "authenticator": authenticator, + "start_date": config.get("start_date"), + } + + api_type = cls._get_api_type(stream_name, json_schema, config.get("force_use_bulk_api", False)) + full_refresh, incremental = cls._get_stream_type(stream_name, api_type) + if replication_key and stream_name not in UNSUPPORTED_FILTERING_STREAMS: + stream_class = incremental + stream_kwargs["replication_key"] = replication_key + else: + stream_class = full_refresh + + return stream_class, stream_kwargs + @classmethod def generate_streams( cls, @@ -86,48 +156,58 @@ def generate_streams( stream_objects: Mapping[str, Any], sf_object: Salesforce, ) -> List[Stream]: - """ "Generates a list of stream by their names. It can be used for different tests too""" + """Generates a list of stream by their names. It can be used for different tests too""" authenticator = TokenAuthenticator(sf_object.access_token) - stream_properties = sf_object.generate_schemas(stream_objects) + schemas = sf_object.generate_schemas(stream_objects) + default_args = [sf_object, authenticator, config] streams = [] for stream_name, sobject_options in stream_objects.items(): - streams_kwargs = {"sobject_options": sobject_options} - selected_properties = stream_properties.get(stream_name, {}).get("properties", {}) - - api_type = cls._get_api_type(stream_name, selected_properties, config.get("force_use_bulk_api", False)) - if api_type == "rest": - full_refresh, incremental = RestSalesforceStream, IncrementalRestSalesforceStream - elif api_type == "bulk": - full_refresh, incremental = BulkSalesforceStream, BulkIncrementalSalesforceStream - else: - raise Exception(f"Stream {stream_name} cannot be processed by REST or BULK API.") - - json_schema = stream_properties.get(stream_name, {}) - pk, replication_key = sf_object.get_pk_and_replication_key(json_schema) - streams_kwargs.update(dict(sf_api=sf_object, pk=pk, stream_name=stream_name, schema=json_schema, authenticator=authenticator)) - if replication_key and stream_name not in UNSUPPORTED_FILTERING_STREAMS: - start_date = config.get( - "start_date", (datetime.now() - relativedelta(years=cls.START_DATE_OFFSET_IN_YEARS)).strftime(cls.DATETIME_FORMAT) - ) - stream = incremental(**streams_kwargs, replication_key=replication_key, start_date=start_date) - else: - stream = full_refresh(**streams_kwargs) + json_schema = schemas.get(stream_name, {}) + + stream_class, kwargs = cls.prepare_stream(stream_name, json_schema, sobject_options, *default_args) + + parent_name = PARENT_SALESFORCE_OBJECTS.get(stream_name, {}).get("parent_name") + if parent_name: + # get minimal schema required for getting proper class name full_refresh/incremental, rest/bulk + parent_schema = PARENT_SALESFORCE_OBJECTS.get(stream_name, {}).get("schema_minimal") + parent_class, parent_kwargs = cls.prepare_stream(parent_name, parent_schema, sobject_options, *default_args) + kwargs["parent"] = parent_class(**parent_kwargs) + + stream = stream_class(**kwargs) + + api_type = cls._get_api_type(stream_name, json_schema, config.get("force_use_bulk_api", False)) if api_type == "rest" and not stream.primary_key and stream.too_many_properties: logger.warning( - f"Can not instantiate stream {stream_name}. " - f"It is not supported by the BULK API and can not be implemented via REST because the number of its properties " - f"exceeds the limit and it lacks a primary key." + f"Can not instantiate stream {stream_name}. It is not supported by the BULK API and can not be " + "implemented via REST because the number of its properties exceeds the limit and it lacks a primary key." ) continue streams.append(stream) return streams def streams(self, config: Mapping[str, Any]) -> List[Stream]: + if not config.get("start_date"): + config["start_date"] = (datetime.now() - relativedelta(years=self.START_DATE_OFFSET_IN_YEARS)).strftime(self.DATETIME_FORMAT) sf = self._get_sf_object(config) stream_objects = sf.get_validated_streams(config=config, catalog=self.catalog) streams = self.generate_streams(config, stream_objects, sf) streams.append(Describe(sf_api=sf, catalog=self.catalog)) - return streams + # TODO: incorporate state & ConcurrentCursor when we support incremental + configured_streams = [] + for stream in streams: + sync_mode = self._get_sync_mode_from_catalog(stream) + if sync_mode == SyncMode.full_refresh: + configured_streams.append(StreamFacade.create_from_stream(stream, self, logger, None, NoopCursor())) + else: + configured_streams.append(stream) + return configured_streams + + def _get_sync_mode_from_catalog(self, stream: Stream) -> Optional[SyncMode]: + if self.catalog: + for catalog_stream in self.catalog.streams: + if stream.name == catalog_stream.stream.name: + return catalog_stream.sync_mode + return None def read( self, diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py index dc7600453649..34c03d1caa94 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py @@ -8,6 +8,7 @@ import os import time import urllib.parse +import uuid from abc import ABC from contextlib import closing from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Type, Union @@ -18,7 +19,7 @@ from airbyte_cdk.models import ConfiguredAirbyteCatalog, FailureType, SyncMode from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.core import Stream, StreamData -from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from airbyte_cdk.utils import AirbyteTracedException from numpy import nan @@ -26,7 +27,7 @@ from requests import codes, exceptions from requests.models import PreparedRequest -from .api import UNSUPPORTED_FILTERING_STREAMS, Salesforce +from .api import PARENT_SALESFORCE_OBJECTS, UNSUPPORTED_FILTERING_STREAMS, Salesforce from .availability_strategy import SalesforceAvailabilityStrategy from .exceptions import SalesforceException, TmpFileIOError from .rate_limiting import default_backoff_handler @@ -44,7 +45,14 @@ class SalesforceStream(HttpStream, ABC): encoding = DEFAULT_ENCODING def __init__( - self, sf_api: Salesforce, pk: str, stream_name: str, sobject_options: Mapping[str, Any] = None, schema: dict = None, **kwargs + self, + sf_api: Salesforce, + pk: str, + stream_name: str, + sobject_options: Mapping[str, Any] = None, + schema: dict = None, + start_date=None, + **kwargs, ): super().__init__(**kwargs) self.sf_api = sf_api @@ -52,6 +60,14 @@ def __init__( self.stream_name = stream_name self.schema: Mapping[str, Any] = schema # type: ignore[assignment] self.sobject_options = sobject_options + self.start_date = self.format_start_date(start_date) + + @staticmethod + def format_start_date(start_date: Optional[str]) -> Optional[str]: + """Transform the format `2021-07-25` into the format `2021-07-25T00:00:00Z`""" + if start_date: + return pendulum.parse(start_date).strftime("%Y-%m-%dT%H:%M:%SZ") # type: ignore[attr-defined,no-any-return] + return None @property def max_properties_length(self) -> int: @@ -140,14 +156,18 @@ def request_params( Salesforce SOQL Query: https://developer.salesforce.com/docs/atlas.en-us.232.0.api_rest.meta/api_rest/dome_queryall.htm """ if next_page_token: - """ - If `next_page_token` is set, subsequent requests use `nextRecordsUrl`, and do not include any parameters. - """ + # If `next_page_token` is set, subsequent requests use `nextRecordsUrl`, and do not include any parameters. return {} property_chunk = property_chunk or {} query = f"SELECT {','.join(property_chunk.keys())} FROM {self.name} " + if self.name in PARENT_SALESFORCE_OBJECTS: + # add where clause: " WHERE ContentDocumentId IN ('06905000000NMXXXXX', ...)" + parent_field = PARENT_SALESFORCE_OBJECTS[self.name]["field"] + parent_ids = [f"'{parent_record[parent_field]}'" for parent_record in stream_slice["parents"]] + query += f" WHERE ContentDocumentId IN ({','.join(parent_ids)})" + if self.primary_key and self.name not in UNSUPPORTED_FILTERING_STREAMS: query += f"ORDER BY {self.primary_key} ASC" @@ -281,6 +301,30 @@ def _fetch_next_page_for_chunk( return request, response +class BatchedSubStream(HttpSubStream): + SLICE_BATCH_SIZE = 200 + + def stream_slices( + self, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + """Instead of yielding one parent record at a time, make stream slice contain a batch of parent records. + + It allows to get records by one requests (instead of only one). + """ + batched_slice = [] + for stream_slice in super().stream_slices(sync_mode, cursor_field, stream_state): + if len(batched_slice) == self.SLICE_BATCH_SIZE: + yield {"parents": batched_slice} + batched_slice = [] + batched_slice.append(stream_slice["parent"]) + if batched_slice: + yield {"parents": batched_slice} + + +class RestSalesforceSubStream(BatchedSubStream, RestSalesforceStream): + pass + + class BulkSalesforceStream(SalesforceStream): DEFAULT_WAIT_TIMEOUT_SECONDS = 86400 # 24-hour bulk job running time MAX_CHECK_INTERVAL_SECONDS = 2.0 @@ -468,7 +512,7 @@ def download_data(self, url: str, chunk_size: int = 1024) -> tuple[str, str, dic Return the tuple containing string with file path of downloaded binary data (Saved temporarily) and file encoding. """ # set filepath for binary data from response - tmp_file = os.path.realpath(os.path.basename(url)) + tmp_file = str(uuid.uuid4()) with closing(self._send_http_request("GET", url, headers={"Accept-Encoding": "gzip"}, stream=True)) as response, open( tmp_file, "wb" ) as data_file: @@ -541,6 +585,12 @@ def request_params( if next_page_token: query += next_page_token["next_token"] + if self.name in PARENT_SALESFORCE_OBJECTS: + # add where clause: " WHERE ContentDocumentId IN ('06905000000NMXXXXX', '06905000000Mxp7XXX', ...)" + parent_field = PARENT_SALESFORCE_OBJECTS[self.name]["field"] + parent_ids = [f"'{parent_record[parent_field]}'" for parent_record in stream_slice["parents"]] + query += f" WHERE ContentDocumentId IN ({','.join(parent_ids)})" + return {"q": query} def read_records( @@ -604,6 +654,10 @@ def get_standard_instance(self) -> SalesforceStream: return new_cls(**stream_kwargs) +class BulkSalesforceSubStream(BatchedSubStream, BulkSalesforceStream): + pass + + @BulkSalesforceStream.transformer.registerCustomTransform def transform_empty_string_to_none(instance: Any, schema: Any): """ @@ -621,17 +675,9 @@ class IncrementalRestSalesforceStream(RestSalesforceStream, ABC): STREAM_SLICE_STEP = 30 _slice = None - def __init__(self, replication_key: str, start_date: Optional[str], **kwargs): + def __init__(self, replication_key: str, **kwargs): super().__init__(**kwargs) self.replication_key = replication_key - self.start_date = self.format_start_date(start_date) - - @staticmethod - def format_start_date(start_date: Optional[str]) -> Optional[str]: - """Transform the format `2021-07-25` into the format `2021-07-25T00:00:00Z`""" - if start_date: - return pendulum.parse(start_date).strftime("%Y-%m-%dT%H:%M:%SZ") # type: ignore[attr-defined,no-any-return] - return None def stream_slices( self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py index 5e3ec0028b8e..8f87e2bd58cd 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/api_test.py @@ -8,6 +8,7 @@ import logging import re from datetime import datetime +from typing import List from unittest.mock import Mock import freezegun @@ -15,6 +16,8 @@ import pytest import requests_mock from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode, Type +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade from airbyte_cdk.utils import AirbyteTracedException from conftest import encoding_symbols_parameters, generate_stream from requests.exceptions import HTTPError @@ -25,23 +28,48 @@ CSV_FIELD_SIZE_LIMIT, BulkIncrementalSalesforceStream, BulkSalesforceStream, + BulkSalesforceSubStream, + Describe, IncrementalRestSalesforceStream, RestSalesforceStream, + SalesforceStream, ) +_ANY_CATALOG = ConfiguredAirbyteCatalog.parse_obj({"streams": []}) +_ANY_CONFIG = {} + @pytest.mark.parametrize( "login_status_code, login_json_resp, expected_error_msg, is_config_error", [ - (400, {"error": "invalid_grant", "error_description": "expired access/refresh token"}, AUTHENTICATION_ERROR_MESSAGE_MAPPING.get("expired access/refresh token"), True), - (400, {"error": "invalid_grant", "error_description": "Authentication failure."}, 'An error occurred: {"error": "invalid_grant", "error_description": "Authentication failure."}', False), - (401, {"error": "Unauthorized", "error_description": "Unautorized"}, 'An error occurred: {"error": "Unauthorized", "error_description": "Unautorized"}', False), - ] + ( + 400, + {"error": "invalid_grant", "error_description": "expired access/refresh token"}, + AUTHENTICATION_ERROR_MESSAGE_MAPPING.get("expired access/refresh token"), + True, + ), + ( + 400, + {"error": "invalid_grant", "error_description": "Authentication failure."}, + 'An error occurred: {"error": "invalid_grant", "error_description": "Authentication failure."}', + False, + ), + ( + 401, + {"error": "Unauthorized", "error_description": "Unautorized"}, + 'An error occurred: {"error": "Unauthorized", "error_description": "Unautorized"}', + False, + ), + ], ) -def test_login_authentication_error_handler(stream_config, requests_mock, login_status_code, login_json_resp, expected_error_msg, is_config_error): - source = SourceSalesforce() +def test_login_authentication_error_handler( + stream_config, requests_mock, login_status_code, login_json_resp, expected_error_msg, is_config_error +): + source = SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG) logger = logging.getLogger("airbyte") - requests_mock.register_uri("POST", "https://login.salesforce.com/services/oauth2/token", json=login_json_resp, status_code=login_status_code) + requests_mock.register_uri( + "POST", "https://login.salesforce.com/services/oauth2/token", json=login_json_resp, status_code=login_status_code + ) if is_config_error: with pytest.raises(AirbyteTracedException) as err: @@ -113,12 +141,15 @@ def test_bulk_sync_pagination(stream_config, stream_api, requests_mock): requests_mock.register_uri("POST", stream.path(), json={"id": job_id}) requests_mock.register_uri("GET", stream.path() + f"/{job_id}", json={"state": "JobComplete"}) resp_text = ["Field1,LastModifiedDate,ID"] + [f"test,2021-11-16,{i}" for i in range(5)] - result_uri = requests_mock.register_uri("GET", stream.path() + f"/{job_id}/results", - [{"text": "\n".join(resp_text), "headers": {"Sforce-Locator": "somelocator_1"}}, - {"text": "\n".join(resp_text), "headers": {"Sforce-Locator": "somelocator_2"}}, - {"text": "\n".join(resp_text), "headers": {"Sforce-Locator": "null"}} - ] - ) + result_uri = requests_mock.register_uri( + "GET", + stream.path() + f"/{job_id}/results", + [ + {"text": "\n".join(resp_text), "headers": {"Sforce-Locator": "somelocator_1"}}, + {"text": "\n".join(resp_text), "headers": {"Sforce-Locator": "somelocator_2"}}, + {"text": "\n".join(resp_text), "headers": {"Sforce-Locator": "null"}}, + ], + ) requests_mock.register_uri("DELETE", stream.path() + f"/{job_id}") stream_slices = next(iter(stream.stream_slices(sync_mode=SyncMode.incremental))) @@ -129,6 +160,9 @@ def test_bulk_sync_pagination(stream_config, stream_api, requests_mock): assert result_uri.request_history[2].query == "locator=somelocator_2" + + + def _prepare_mock(m, stream): job_id = "fake_job_1" m.register_uri("POST", stream.path(), json={"id": job_id}) @@ -201,9 +235,7 @@ def test_bulk_sync_failed_retry(stream_config, stream_api): "start_date_provided,stream_name,expected_start_date", [ (True, "Account", "2010-01-18T21:18:20Z"), - (False, "Account", None), (True, "ActiveFeatureLicenseMetric", "2010-01-18T21:18:20Z"), - (False, "ActiveFeatureLicenseMetric", None), ], ) def test_stream_start_date( @@ -313,7 +345,7 @@ def test_encoding_symbols(stream_config, stream_api, chunk_size, content_type_he def test_check_connection_rate_limit( stream_config, login_status_code, login_json_resp, discovery_status_code, discovery_resp_json, expected_error_msg ): - source = SourceSalesforce() + source = SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG) logger = logging.getLogger("airbyte") with requests_mock.Mocker() as m: @@ -341,7 +373,7 @@ def test_rate_limit_bulk(stream_config, stream_api, bulk_catalog, state): While reading `stream_1` if 403 (Rate Limit) is received, it should finish that stream with success and stop the sync process. Next streams should not be executed. """ - stream_config.update({'start_date': '2021-10-01'}) + stream_config.update({"start_date": "2021-10-01"}) stream_1: BulkIncrementalSalesforceStream = generate_stream("Account", stream_config, stream_api) stream_2: BulkIncrementalSalesforceStream = generate_stream("Asset", stream_config, stream_api) streams = [stream_1, stream_2] @@ -350,7 +382,7 @@ def test_rate_limit_bulk(stream_config, stream_api, bulk_catalog, state): stream_1.page_size = 6 stream_1.state_checkpoint_interval = 5 - source = SourceSalesforce() + source = SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG) source.streams = Mock() source.streams.return_value = streams logger = logging.getLogger("airbyte") @@ -398,7 +430,7 @@ def test_rate_limit_rest(stream_config, stream_api, rest_catalog, state): While reading `stream_1` if 403 (Rate Limit) is received, it should finish that stream with success and stop the sync process. Next streams should not be executed. """ - stream_config.update({'start_date': '2021-11-01'}) + stream_config.update({"start_date": "2021-11-01"}) stream_1: IncrementalRestSalesforceStream = generate_stream("KnowledgeArticle", stream_config, stream_api) stream_2: IncrementalRestSalesforceStream = generate_stream("AcceptedEventRelation", stream_config, stream_api) @@ -406,7 +438,7 @@ def test_rate_limit_rest(stream_config, stream_api, rest_catalog, state): stream_1.state_checkpoint_interval = 3 configure_request_params_mock(stream_1, stream_2) - source = SourceSalesforce() + source = SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG) source.streams = Mock() source.streams.return_value = [stream_1, stream_2] @@ -591,7 +623,7 @@ def test_forwarding_sobject_options(stream_config, stream_names, catalog_stream_ ], }, ) - source = SourceSalesforce() + source = SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG) source.catalog = catalog streams = source.streams(config=stream_config) expected_names = catalog_stream_names if catalog else stream_names @@ -599,10 +631,103 @@ def test_forwarding_sobject_options(stream_config, stream_names, catalog_stream_ for stream in streams: if stream.name != "Describe": - assert stream.sobject_options == {"flag1": True, "queryable": True} + if isinstance(stream, StreamFacade): + assert stream._legacy_stream.sobject_options == {"flag1": True, "queryable": True} + else: + assert stream.sobject_options == {"flag1": True, "queryable": True} return +@pytest.mark.parametrize( + "stream_names,catalog_stream_names,", + ( + ( + ["stream_1", "stream_2", "Describe"], + None, + ), + ( + ["stream_1", "stream_2"], + ["stream_1", "stream_2", "Describe"], + ), + ( + ["stream_1", "stream_2", "stream_3", "Describe"], + ["stream_1", "Describe"], + ), + ), +) +def test_unspecified_and_incremental_streams_are_not_concurrent(stream_config, stream_names, catalog_stream_names) -> None: + for stream in _get_streams(stream_config, stream_names, catalog_stream_names, SyncMode.incremental): + assert isinstance(stream, (SalesforceStream, Describe)) + + +@pytest.mark.parametrize( + "stream_names,catalog_stream_names,", + ( + ( + ["stream_1", "stream_2"], + ["stream_1", "stream_2", "Describe"], + ), + ( + ["stream_1", "stream_2", "stream_3", "Describe"], + ["stream_1", "Describe"], + ), + ), +) +def test_full_refresh_streams_are_concurrent(stream_config, stream_names, catalog_stream_names) -> None: + for stream in _get_streams(stream_config, stream_names, catalog_stream_names, SyncMode.full_refresh): + assert isinstance(stream, StreamFacade) + + +def _get_streams(stream_config, stream_names, catalog_stream_names, sync_type) -> List[Stream]: + sobjects_matcher = re.compile("/sobjects$") + token_matcher = re.compile("/token$") + describe_matcher = re.compile("/describe$") + catalog = None + if catalog_stream_names: + catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=AirbyteStream(name=catalog_stream_name, supported_sync_modes=[sync_type], json_schema={"type": "object"}), + sync_mode=sync_type, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + for catalog_stream_name in catalog_stream_names + ] + ) + with requests_mock.Mocker() as m: + m.register_uri("POST", token_matcher, json={"instance_url": "https://fake-url.com", "access_token": "fake-token"}) + m.register_uri( + "GET", + describe_matcher, + json={ + "fields": [ + { + "name": "field", + "type": "string", + } + ] + }, + ) + m.register_uri( + "GET", + sobjects_matcher, + json={ + "sobjects": [ + { + "name": stream_name, + "flag1": True, + "queryable": True, + } + for stream_name in stream_names + if stream_name != "Describe" + ], + }, + ) + source = SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG) + source.catalog = catalog + return source.streams(config=stream_config) + + def test_csv_field_size_limit(): DEFAULT_CSV_FIELD_SIZE_LIMIT = 1024 * 128 @@ -698,15 +823,22 @@ def test_stream_with_no_records_in_response(stream_config, stream_api_v2_pk_too_ @pytest.mark.parametrize( "status_code,response_json,log_message", [ - (400, [{"errorCode": "INVALIDENTITY", "message": "Account is not supported by the Bulk API"}], "Account is not supported by the Bulk API"), + ( + 400, + [{"errorCode": "INVALIDENTITY", "message": "Account is not supported by the Bulk API"}], + "Account is not supported by the Bulk API", + ), (403, [{"errorCode": "REQUEST_LIMIT_EXCEEDED", "message": "API limit reached"}], "API limit reached"), (400, [{"errorCode": "API_ERROR", "message": "API does not support query"}], "The stream 'Account' is not queryable,"), - (400, [{"errorCode": "LIMIT_EXCEEDED", "message": "Max bulk v2 query jobs (10000) per 24 hrs has been reached (10021)"}], "Your API key for Salesforce has reached its limit for the 24-hour period. We will resume replication once the limit has elapsed.") - ] + ( + 400, + [{"errorCode": "LIMIT_EXCEEDED", "message": "Max bulk v2 query jobs (10000) per 24 hrs has been reached (10021)"}], + "Your API key for Salesforce has reached its limit for the 24-hour period. We will resume replication once the limit has elapsed.", + ), + ], ) def test_bulk_stream_error_in_logs_on_create_job(requests_mock, stream_config, stream_api, status_code, response_json, log_message, caplog): - """ - """ + """ """ stream = generate_stream("Account", stream_config, stream_api) url = f"{stream.sf_api.instance_url}/services/data/{stream.sf_api.version}/jobs/query" requests_mock.register_uri( @@ -726,8 +858,17 @@ def test_bulk_stream_error_in_logs_on_create_job(requests_mock, stream_config, s @pytest.mark.parametrize( "status_code,response_json,error_message", [ - (400, [{"errorCode": "TXN_SECURITY_METERING_ERROR", "message": "We can't complete the action because enabled transaction security policies took too long to complete."}], 'A transient authentication error occurred. To prevent future syncs from failing, assign the "Exempt from Transaction Security" user permission to the authenticated user.'), - ] + ( + 400, + [ + { + "errorCode": "TXN_SECURITY_METERING_ERROR", + "message": "We can't complete the action because enabled transaction security policies took too long to complete.", + } + ], + 'A transient authentication error occurred. To prevent future syncs from failing, assign the "Exempt from Transaction Security" user permission to the authenticated user.', + ), + ], ) def test_bulk_stream_error_on_wait_for_job(requests_mock, stream_config, stream_api, status_code, response_json, error_message): @@ -752,21 +893,22 @@ def test_bulk_stream_slices(stream_config_date_format, stream_api): today = pendulum.today(tz="UTC") start_date = pendulum.parse(stream.start_date, tz="UTC") while start_date < today: - expected_slices.append({ - 'start_date': start_date.isoformat(timespec="milliseconds"), - 'end_date': min(today, start_date.add(days=stream.STREAM_SLICE_STEP)).isoformat(timespec="milliseconds") - }) + expected_slices.append( + { + "start_date": start_date.isoformat(timespec="milliseconds"), + "end_date": min(today, start_date.add(days=stream.STREAM_SLICE_STEP)).isoformat(timespec="milliseconds"), + } + ) start_date = start_date.add(days=stream.STREAM_SLICE_STEP) assert expected_slices == stream_slices - @freezegun.freeze_time("2023-04-01") def test_bulk_stream_request_params_states(stream_config_date_format, stream_api, bulk_catalog, requests_mock): """Check that request params ignore records cursor and use start date from slice ONLY""" stream_config_date_format.update({"start_date": "2023-01-01"}) stream: BulkIncrementalSalesforceStream = generate_stream("Account", stream_config_date_format, stream_api) - source = SourceSalesforce() + source = SourceSalesforce(_ANY_CATALOG, _ANY_CONFIG) source.streams = Mock() source.streams.return_value = [stream] @@ -779,13 +921,15 @@ def test_bulk_stream_request_params_states(stream_config_date_format, stream_api job_id_2 = "fake_job_2" requests_mock.register_uri("GET", stream.path() + f"/{job_id_2}", [{"json": {"state": "JobComplete"}}]) requests_mock.register_uri("DELETE", stream.path() + f"/{job_id_2}") - requests_mock.register_uri("GET", stream.path() + f"/{job_id_2}/results", text="Field1,LastModifiedDate,ID\ntest,2023-04-01,2\ntest,2023-02-20,22") + requests_mock.register_uri( + "GET", stream.path() + f"/{job_id_2}/results", text="Field1,LastModifiedDate,ID\ntest,2023-04-01,2\ntest,2023-02-20,22" + ) requests_mock.register_uri("PATCH", stream.path() + f"/{job_id_2}") job_id_3 = "fake_job_3" - queries_history = requests_mock.register_uri("POST", stream.path(), [{"json": {"id": job_id_1}}, - {"json": {"id": job_id_2}}, - {"json": {"id": job_id_3}}]) + queries_history = requests_mock.register_uri( + "POST", stream.path(), [{"json": {"id": job_id_1}}, {"json": {"id": job_id_2}}, {"json": {"id": job_id_3}}] + ) requests_mock.register_uri("GET", stream.path() + f"/{job_id_3}", [{"json": {"state": "JobComplete"}}]) requests_mock.register_uri("DELETE", stream.path() + f"/{job_id_3}") requests_mock.register_uri("GET", stream.path() + f"/{job_id_3}/results", text="Field1,LastModifiedDate,ID\ntest,2023-04-01,3") @@ -798,11 +942,63 @@ def test_bulk_stream_request_params_states(stream_config_date_format, stream_api actual_state_values = [item.state.data.get("Account").get(stream.cursor_field) for item in result if item.type == Type.STATE] # assert request params - assert "LastModifiedDate >= 2023-01-01T10:10:10.000+00:00 AND LastModifiedDate < 2023-01-31T10:10:10.000+00:00" in queries_history.request_history[0].text - assert "LastModifiedDate >= 2023-01-31T10:10:10.000+00:00 AND LastModifiedDate < 2023-03-02T10:10:10.000+00:00" in queries_history.request_history[1].text - assert "LastModifiedDate >= 2023-03-02T10:10:10.000+00:00 AND LastModifiedDate < 2023-04-01T00:00:00.000+00:00" in queries_history.request_history[2].text + assert ( + "LastModifiedDate >= 2023-01-01T10:10:10.000+00:00 AND LastModifiedDate < 2023-01-31T10:10:10.000+00:00" + in queries_history.request_history[0].text + ) + assert ( + "LastModifiedDate >= 2023-01-31T10:10:10.000+00:00 AND LastModifiedDate < 2023-03-02T10:10:10.000+00:00" + in queries_history.request_history[1].text + ) + assert ( + "LastModifiedDate >= 2023-03-02T10:10:10.000+00:00 AND LastModifiedDate < 2023-04-01T00:00:00.000+00:00" + in queries_history.request_history[2].text + ) # assert states # if connector meets record with cursor `2023-04-01` out of current slice range 2023-01-31 <> 2023-03-02, we ignore all other values and set state to slice end_date expected_state_values = ["2023-01-15T00:00:00+00:00", "2023-03-02T10:10:10+00:00", "2023-04-01T00:00:00+00:00"] assert actual_state_values == expected_state_values + + +def test_request_params_incremental(stream_config_date_format, stream_api): + stream = generate_stream("ContentDocument", stream_config_date_format, stream_api) + params = stream.request_params(stream_state={}, stream_slice={'start_date': '2020', 'end_date': '2021'}) + + assert params == {'q': 'SELECT LastModifiedDate, Id FROM ContentDocument WHERE LastModifiedDate >= 2020 AND LastModifiedDate < 2021'} + + +def test_request_params_substream(stream_config_date_format, stream_api): + stream = generate_stream("ContentDocumentLink", stream_config_date_format, stream_api) + params = stream.request_params(stream_state={}, stream_slice={'parents': [{'Id': 1}, {'Id': 2}]}) + + assert params == {"q": "SELECT LastModifiedDate, Id FROM ContentDocumentLink WHERE ContentDocumentId IN ('1','2')"} + + +@freezegun.freeze_time("2023-03-20") +def test_stream_slices_for_substream(stream_config, stream_api, requests_mock): + """Test BulkSalesforceSubStream for ContentDocumentLink (+ parent ContentDocument) + + ContentDocument return 1 record for each slice request. + Given start/end date leads to 3 date slice for ContentDocument, thus 3 total records + + ContentDocumentLink + It means that ContentDocumentLink should have 2 slices, with 2 and 1 records in each + """ + stream_config['start_date'] = '2023-01-01' + stream: BulkSalesforceSubStream = generate_stream("ContentDocumentLink", stream_config, stream_api) + stream.SLICE_BATCH_SIZE = 2 # each ContentDocumentLink should contain 2 records from parent ContentDocument stream + + job_id = "fake_job" + requests_mock.register_uri("POST", stream.path(), json={"id": job_id}) + requests_mock.register_uri("GET", stream.path() + f"/{job_id}", json={"state": "JobComplete"}) + requests_mock.register_uri("GET", stream.path() + f"/{job_id}/results", [{"text": "Field1,LastModifiedDate,ID\ntest,2021-11-16,123", "headers": {"Sforce-Locator": "null"}}]) + requests_mock.register_uri("DELETE", stream.path() + f"/{job_id}") + + stream_slices = list(stream.stream_slices(sync_mode=SyncMode.full_refresh)) + assert stream_slices == [ + {'parents': [{'Field1': 'test', 'ID': '123', 'LastModifiedDate': '2021-11-16'}, + {'Field1': 'test', 'ID': '123', 'LastModifiedDate': '2021-11-16'}]}, + {'parents': [{'Field1': 'test', 'ID': '123', 'LastModifiedDate': '2021-11-16'}]} + ] + diff --git a/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py b/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py index b128dc65f1fe..eeacdd2235d2 100644 --- a/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-salesforce/unit_tests/conftest.py @@ -107,21 +107,15 @@ def stream_api_pk(stream_config): @pytest.fixture(scope="module") def stream_api_v2_too_many_properties(stream_config): - describe_response_data = { - "fields": [{"name": f"Property{str(i)}", "type": "string"} for i in range(Salesforce.REQUEST_SIZE_LIMITS)] - } + describe_response_data = {"fields": [{"name": f"Property{str(i)}", "type": "string"} for i in range(Salesforce.REQUEST_SIZE_LIMITS)]} describe_response_data["fields"].extend([{"name": "BillingAddress", "type": "address"}]) return _stream_api(stream_config, describe_response_data=describe_response_data) @pytest.fixture(scope="module") def stream_api_v2_pk_too_many_properties(stream_config): - describe_response_data = { - "fields": [{"name": f"Property{str(i)}", "type": "string"} for i in range(Salesforce.REQUEST_SIZE_LIMITS)] - } - describe_response_data["fields"].extend([ - {"name": "BillingAddress", "type": "address"}, {"name": "Id", "type": "string"} - ]) + describe_response_data = {"fields": [{"name": f"Property{str(i)}", "type": "string"} for i in range(Salesforce.REQUEST_SIZE_LIMITS)]} + describe_response_data["fields"].extend([{"name": "BillingAddress", "type": "address"}, {"name": "Id", "type": "string"}]) return _stream_api(stream_config, describe_response_data=describe_response_data) @@ -130,28 +124,36 @@ def generate_stream(stream_name, stream_config, stream_api): def encoding_symbols_parameters(): - return [(x, {"Content-Type": "text/csv; charset=ISO-8859-1"}, b'"\xc4"\n,"4"\n\x00,"\xca \xfc"', [{"Ä": "4"}, {"Ä": "Ê ü"}]) for x in range(1, 11)] + [ - ( - x, - {"Content-Type": "text/csv; charset=utf-8"}, - b'"\xd5\x80"\n "\xd5\xaf","\xd5\xaf"\n\x00,"\xe3\x82\x82 \xe3\x83\xa4 \xe3\x83\xa4 \xf0\x9d\x9c\xb5"', - [{"Հ": "կ"}, {"Հ": "も ヤ ヤ 𝜵"}], - ) - for x in range(1, 11) - ] + [ - ( - x, - {"Content-Type": "text/csv"}, - b'"\xd5\x80"\n "\xd5\xaf","\xd5\xaf"\n\x00,"\xe3\x82\x82 \xe3\x83\xa4 \xe3\x83\xa4 \xf0\x9d\x9c\xb5"', - [{"Հ": "կ"}, {"Հ": "も ヤ ヤ 𝜵"}], - ) - for x in range(1, 11) - ] + [ - ( - x, - {}, - b'"\xd5\x80"\n "\xd5\xaf","\xd5\xaf"\n\x00,"\xe3\x82\x82 \xe3\x83\xa4 \xe3\x83\xa4 \xf0\x9d\x9c\xb5"', - [{"Հ": "կ"}, {"Հ": "も ヤ ヤ 𝜵"}], - ) - for x in range(1, 11) - ] + return ( + [ + (x, {"Content-Type": "text/csv; charset=ISO-8859-1"}, b'"\xc4"\n,"4"\n\x00,"\xca \xfc"', [{"Ä": "4"}, {"Ä": "Ê ü"}]) + for x in range(1, 11) + ] + + [ + ( + x, + {"Content-Type": "text/csv; charset=utf-8"}, + b'"\xd5\x80"\n "\xd5\xaf","\xd5\xaf"\n\x00,"\xe3\x82\x82 \xe3\x83\xa4 \xe3\x83\xa4 \xf0\x9d\x9c\xb5"', + [{"Հ": "կ"}, {"Հ": "も ヤ ヤ 𝜵"}], + ) + for x in range(1, 11) + ] + + [ + ( + x, + {"Content-Type": "text/csv"}, + b'"\xd5\x80"\n "\xd5\xaf","\xd5\xaf"\n\x00,"\xe3\x82\x82 \xe3\x83\xa4 \xe3\x83\xa4 \xf0\x9d\x9c\xb5"', + [{"Հ": "կ"}, {"Հ": "も ヤ ヤ 𝜵"}], + ) + for x in range(1, 11) + ] + + [ + ( + x, + {}, + b'"\xd5\x80"\n "\xd5\xaf","\xd5\xaf"\n\x00,"\xe3\x82\x82 \xe3\x83\xa4 \xe3\x83\xa4 \xf0\x9d\x9c\xb5"', + [{"Հ": "կ"}, {"Հ": "も ヤ ヤ 𝜵"}], + ) + for x in range(1, 11) + ] + ) diff --git a/airbyte-integrations/connectors/source-salesloft/README.md b/airbyte-integrations/connectors/source-salesloft/README.md index 6373a7a16bb7..844a841d9c03 100644 --- a/airbyte-integrations/connectors/source-salesloft/README.md +++ b/airbyte-integrations/connectors/source-salesloft/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-salesloft:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/salesloft) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_salesloft/spec.json` file. @@ -56,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-salesloft:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-salesloft build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-salesloft:airbyteDocker +An image will be built with the tag `airbyte/source-salesloft:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-salesloft:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -77,44 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-salesloft:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-salesloft:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-salesloft:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-salesloft test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-salesloft:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-salesloft:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -124,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-salesloft test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/salesloft.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-salesloft/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-salesloft/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-salesloft/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-salesloft/build.gradle b/airbyte-integrations/connectors/source-salesloft/build.gradle deleted file mode 100644 index 8d049650953a..000000000000 --- a/airbyte-integrations/connectors/source-salesloft/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_salesloft' -} diff --git a/airbyte-integrations/connectors/source-salesloft/metadata.yaml b/airbyte-integrations/connectors/source-salesloft/metadata.yaml index 8f713ca61aa4..7fdf8beebb6b 100644 --- a/airbyte-integrations/connectors/source-salesloft/metadata.yaml +++ b/airbyte-integrations/connectors/source-salesloft/metadata.yaml @@ -1,4 +1,7 @@ data: + ab_internal: + ql: 300 + sl: 100 allowedHosts: hosts: - api.salesloft.com @@ -7,6 +10,7 @@ data: definitionId: 41991d12-d4b5-439e-afd0-260a31d4c53f dockerImageTag: 1.2.0 dockerRepository: airbyte/source-salesloft + documentationUrl: https://docs.airbyte.com/integrations/sources/salesloft githubIssueLabel: source-salesloft icon: salesloft.svg license: MIT @@ -17,11 +21,7 @@ data: oss: enabled: true releaseStage: beta - documentationUrl: https://docs.airbyte.com/integrations/sources/salesloft + supportLevel: community tags: - language:python - ab_internal: - sl: 200 - ql: 300 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-salesloft/unit_tests/conftest.py b/airbyte-integrations/connectors/source-salesloft/unit_tests/conftest.py index 993d94055c18..21102a5f29b3 100644 --- a/airbyte-integrations/connectors/source-salesloft/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-salesloft/unit_tests/conftest.py @@ -14,7 +14,7 @@ def config(): "client_secret": "client_secret", "refresh_token": "refresh_token", "access_token": "access_token", - "token_expiry_date": "2222-02-02T00:00:00Z" + "token_expiry_date": "2222-02-02T00:00:00Z", }, - "start_date": "2020-01-01T00:00:00.000Z" + "start_date": "2020-01-01T00:00:00.000Z", } diff --git a/airbyte-integrations/connectors/source-salesloft/unit_tests/test_source.py b/airbyte-integrations/connectors/source-salesloft/unit_tests/test_source.py index 06efdbbaf2b8..25b78fc52c74 100644 --- a/airbyte-integrations/connectors/source-salesloft/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-salesloft/unit_tests/test_source.py @@ -13,14 +13,7 @@ def test_streams(config): assert len(streams) == expected_streams_number -@pytest.mark.parametrize( - "status_code, check_successful", - ( - (403, False), - (500, False), - (200, True) - ) -) +@pytest.mark.parametrize("status_code, check_successful", ((403, False), (500, False), (200, True))) def test_check_connection(requests_mock, config, status_code, check_successful): requests_mock.post("https://accounts.salesloft.com/oauth/token", json={"access_token": "token", "expires_in": 7200}) requests_mock.get("https://api.salesloft.com/v2/me.json", status_code=status_code) diff --git a/airbyte-integrations/connectors/source-salesloft/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-salesloft/unit_tests/test_streams.py index 857d59bc4d53..18e437a7ecad 100644 --- a/airbyte-integrations/connectors/source-salesloft/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-salesloft/unit_tests/test_streams.py @@ -22,7 +22,7 @@ def test_incremental_request_params(config): "page": 1, "per_page": 100, "created_at[gt]": "2020-01-01T00:00:00.000000Z", - "updated_at[gt]": "2020-01-01T00:00:00.000000Z" + "updated_at[gt]": "2020-01-01T00:00:00.000000Z", } assert stream.request_params(**inputs) == expected_params @@ -51,13 +51,7 @@ def test_get_updated_state(): assert stream.get_updated_state(**inputs) == expected_state -@pytest.mark.parametrize( - "return_value, expected_records", - ( - ({}, []), - ({"data": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]) - ) -) +@pytest.mark.parametrize("return_value, expected_records", (({}, []), ({"data": [{"id": 1}, {"id": 2}]}, [{"id": 1}, {"id": 2}]))) def test_parse_response(config, return_value, expected_records): stream = Users(authenticator=MagicMock(), start_date=config["start_date"]) response = MagicMock(json=MagicMock(return_value=return_value)) diff --git a/airbyte-integrations/connectors/source-sap-fieldglass/README.md b/airbyte-integrations/connectors/source-sap-fieldglass/README.md index 6345f5daff43..c503551e7f98 100644 --- a/airbyte-integrations/connectors/source-sap-fieldglass/README.md +++ b/airbyte-integrations/connectors/source-sap-fieldglass/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-sap-fieldglass:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/sap-fieldglass) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_sap_fieldglass/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-sap-fieldglass:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-sap-fieldglass build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-sap-fieldglass:airbyteDocker +An image will be built with the tag `airbyte/source-sap-fieldglass:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-sap-fieldglass:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-sap-fieldglass:dev che docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-sap-fieldglass:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-sap-fieldglass:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-sap-fieldglass test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-sap-fieldglass:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-sap-fieldglass:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-sap-fieldglass test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/sap-fieldglass.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-sap-fieldglass/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-sap-fieldglass/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-sap-fieldglass/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-sap-fieldglass/build.gradle b/airbyte-integrations/connectors/source-sap-fieldglass/build.gradle deleted file mode 100644 index a3cadd2c7f00..000000000000 --- a/airbyte-integrations/connectors/source-sap-fieldglass/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_sap_fieldglass' -} diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/Dockerfile b/airbyte-integrations/connectors/source-scaffold-java-jdbc/Dockerfile deleted file mode 100644 index a3de3ce4ba33..000000000000 --- a/airbyte-integrations/connectors/source-scaffold-java-jdbc/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-scaffold-java-jdbc - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-scaffold-java-jdbc - -COPY --from=build /airbyte /airbyte - -# Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile. -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-scaffold-java-jdbc diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/README.md b/airbyte-integrations/connectors/source-scaffold-java-jdbc/README.md index a263225fa4d6..31ae071f64b2 100644 --- a/airbyte-integrations/connectors/source-scaffold-java-jdbc/README.md +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/README.md @@ -22,10 +22,10 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: ``` -./gradlew :airbyte-integrations:connectors:source-scaffold-java-jdbc:airbyteDocker +./gradlew :airbyte-integrations:connectors:source-scaffold-java-jdbc:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. + +Once built, the docker image name and tag will be `airbyte/source-scaffold-java-jdbc:dev`. #### Run Then run any of the connector commands as follows: @@ -62,8 +62,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-scaffold-java-jdbc test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/scaffold-java-jdbc.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-scaffold-java-jdbc/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-scaffold-java-jdbc/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/build.gradle b/airbyte-integrations/connectors/source-scaffold-java-jdbc/build.gradle index dbbe4f113e84..94cd790c7271 100644 --- a/airbyte-integrations/connectors/source-scaffold-java-jdbc/build.gradle +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/build.gradle @@ -1,8 +1,12 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' - id 'airbyte-connector-acceptance-test' + id 'airbyte-java-connector' +} + +airbyteJavaConnector { + cdkVersionRequired = '0.7.7' + features = ['db-sources'] + useLocalCdk = false } application { @@ -10,21 +14,13 @@ application { } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation project(':airbyte-integrations:connectors:source-relational-db') //TODO Add jdbc driver import here. Ex: implementation 'com.microsoft.sqlserver:mssql-jdbc:8.4.1.jre14' - testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) - testImplementation 'org.apache.commons:commons-lang3:3.11' + testImplementation libs.testcontainers.jdbc integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-scaffold-java-jdbc') - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + testFixturesImplementation libs.testcontainers.jdbc } diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/metadata.yaml b/airbyte-integrations/connectors/source-scaffold-java-jdbc/metadata.yaml index 65fc8a95d574..83f3cf72db6a 100644 --- a/airbyte-integrations/connectors/source-scaffold-java-jdbc/metadata.yaml +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: database connectorType: source definitionId: FAKE-UUID-0000-0000-000000000000 - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-scaffold-java-jdbc githubIssueLabel: source-scaffold-java-jdbc icon: scaffold-java-jdbc.svg diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSource.java b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSource.java index 1ae9c291284c..c69f17347a74 100644 --- a/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSource.java +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/main/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSource.java @@ -5,11 +5,11 @@ package io.airbyte.integrations.source.scaffold_java_jdbc; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; import java.sql.JDBCType; import java.util.Set; import org.slf4j.Logger; diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSourceAcceptanceTest.java index 109762a47f57..b911468604e9 100644 --- a/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test-integration/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSourceAcceptanceTest.java @@ -5,31 +5,29 @@ package io.airbyte.integrations.source.scaffold_java_jdbc; import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.v0.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.v0.ConnectorSpecification; import java.util.HashMap; +import org.junit.jupiter.api.Disabled; +@Disabled public class ScaffoldJavaJdbcSourceAcceptanceTest extends SourceAcceptanceTest { - private JsonNode config; + private ScaffoldJavaJdbcTestDatabase testdb; @Override protected void setupEnvironment(final TestDestinationEnv testEnv) { - // TODO create new container. Ex: "new OracleContainer("epiclabs/docker-oracle-xe-11g");" - // TODO make container started. Ex: "container.start();" - // TODO init JsonNode config - // TODO crete airbyte Database object "Databases.createJdbcDatabase(...)" - // TODO insert test data to DB. Ex: "database.execute(connection-> ...)" - // TODO close Database. Ex: "database.close();" + // TODO: create new TestDatabase instance and assign `testdb` to it. + // TODO: use it to create and populate test tables in the database. } @Override protected void tearDown(final TestDestinationEnv testEnv) { - // TODO close container that was initialized in setup() method. Ex: "container.close();" + testdb.close(); } @Override @@ -44,7 +42,8 @@ protected ConnectorSpecification getSpec() throws Exception { @Override protected JsonNode getConfig() { - return config; + // TODO: (optional) call more builder methods. + return testdb.integrationTestConfigBuilder().build(); } @Override diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcJdbcSourceAcceptanceTest.java index 8c9d753fe41b..70990256b9b8 100644 --- a/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcJdbcSourceAcceptanceTest.java @@ -5,44 +5,32 @@ package io.airbyte.integrations.source.scaffold_java_jdbc; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; -import java.sql.JDBCType; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; +import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import org.junit.jupiter.api.Disabled; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -class ScaffoldJavaJdbcJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { +@Disabled +class ScaffoldJavaJdbcJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { private static final Logger LOGGER = LoggerFactory.getLogger(ScaffoldJavaJdbcJdbcSourceAcceptanceTest.class); - // TODO declare a test container for DB. EX: org.testcontainers.containers.OracleContainer - - @BeforeAll - static void init() { - // Oracle returns uppercase values - // TODO init test container. Ex: "new OracleContainer("epiclabs/docker-oracle-xe-11g")" - // TODO start container. Ex: "container.start();" - } - - @BeforeEach - public void setup() throws Exception { - // TODO init config. Ex: "config = Jsons.jsonNode(ImmutableMap.builder().put("host", - // host).put("port", port)....build()); - super.setup(); + @Override + protected JsonNode config() { + // TODO: (optional) call more builder methods. + return testdb.testConfigBuilder().build(); } - @AfterEach - public void tearDown() { - // TODO clean used resources + @Override + protected ScaffoldJavaJdbcSource source() { + // TODO: (optional) call `setFeatureFlags` before returning the source to mock setting env vars. + return new ScaffoldJavaJdbcSource(); } @Override - public AbstractJdbcSource getSource() { - return new ScaffoldJavaJdbcSource(); + protected ScaffoldJavaJdbcTestDatabase createTestDatabase() { + // TODO: return a suitable TestDatabase instance. + return new ScaffoldJavaJdbcTestDatabase(null).initialized(); } @Override @@ -51,25 +39,4 @@ public boolean supportsSchemas() { return false; } - @Override - public JsonNode getConfig() { - return config; - } - - @Override - public String getDriverClass() { - return ScaffoldJavaJdbcSource.DRIVER_CLASS; - } - - @Override - public AbstractJdbcSource getJdbcSource() { - // TODO - return null; - } - - @AfterAll - static void cleanUp() { - // TODO close the container. Ex: "container.close();" - } - } diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSourceTests.java b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSourceTests.java index d95d6f4be974..8052105dd71a 100644 --- a/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSourceTests.java +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/test/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcSourceTests.java @@ -5,7 +5,7 @@ package io.airbyte.integrations.source.scaffold_java_jdbc; import com.fasterxml.jackson.databind.JsonNode; -import io.airbyte.db.Database; +import io.airbyte.cdk.db.Database; import org.junit.jupiter.api.Test; public class ScaffoldJavaJdbcSourceTests { diff --git a/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/testFixtures/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcTestDatabase.java b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/testFixtures/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcTestDatabase.java new file mode 100644 index 000000000000..4e0c24508217 --- /dev/null +++ b/airbyte-integrations/connectors/source-scaffold-java-jdbc/src/testFixtures/java/io/airbyte/integrations/source/scaffold_java_jdbc/ScaffoldJavaJdbcTestDatabase.java @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.scaffold_java_jdbc; + +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.testutils.TestDatabase; +import java.util.stream.Stream; +import org.jooq.SQLDialect; +import org.testcontainers.containers.JdbcDatabaseContainer; + +public class ScaffoldJavaJdbcTestDatabase + extends TestDatabase, ScaffoldJavaJdbcTestDatabase, ScaffoldJavaJdbcTestDatabase.ScaffoldJavaJdbcConfigBuilder> { + + public ScaffoldJavaJdbcTestDatabase(JdbcDatabaseContainer container) { + // TODO: (optional) consider also implementing a ContainerFactory to share testcontainer instances. + // Effective use requires parallelizing the tests using JUnit instead of gradle. + // This is best achieved by adding a `gradle.properties` file containing + // `testExecutionConcurrency=-1`. + super(container); + } + + @Override + protected Stream> inContainerBootstrapCmd() { + // TODO: return a stream of streams of command args to be passed to `execInContainer` calls to set + // up the test state. + // This usually involves the execution of CREATE DATABASE and CREATE USER statements as root. + return Stream.empty(); + } + + @Override + protected Stream inContainerUndoBootstrapCmd() { + // TODO: (optional) return a stream of command args to be passed to a `execInContainer` call to + // clean up the test state. + return Stream.empty(); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + // TODO: return a suitable value. + return DatabaseDriver.POSTGRESQL; + } + + @Override + public SQLDialect getSqlDialect() { + // TODO: return a suitable value. + return SQLDialect.DEFAULT; + } + + @Override + public ScaffoldJavaJdbcConfigBuilder configBuilder() { + // TODO: flesh out the ConfigBuilder subclass and return a new instance of it here. + return new ScaffoldJavaJdbcConfigBuilder(this); + } + + public static class ScaffoldJavaJdbcConfigBuilder extends TestDatabase.ConfigBuilder { + + public ScaffoldJavaJdbcConfigBuilder(ScaffoldJavaJdbcTestDatabase testDatabase) { + super(testDatabase); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/.dockerignore b/airbyte-integrations/connectors/source-scaffold-source-http/.dockerignore deleted file mode 100644 index 92e3ef95113a..000000000000 --- a/airbyte-integrations/connectors/source-scaffold-source-http/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -* -!Dockerfile -!main.py -!source_scaffold_source_http -!setup.py -!secrets diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/Dockerfile b/airbyte-integrations/connectors/source-scaffold-source-http/Dockerfile deleted file mode 100644 index 0f081bb95b66..000000000000 --- a/airbyte-integrations/connectors/source-scaffold-source-http/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.13-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_scaffold_source_http ./source_scaffold_source_http - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-scaffold-source-http diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/README.md b/airbyte-integrations/connectors/source-scaffold-source-http/README.md index 608937301fbc..e0724d544bcc 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-http/README.md +++ b/airbyte-integrations/connectors/source-scaffold-source-http/README.md @@ -10,7 +10,7 @@ For information about how to use this connector within Airbyte, see [the documen #### Minimum Python version required `= 3.9.0` -#### Build & Activate Virtual Environment and install dependencies +#### Activate Virtual Environment and install dependencies From this connector directory, create a virtual environment: ``` python -m venv .venv @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-scaffold-source-http:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/scaffold-source-http) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_scaffold_source_http/spec.yaml` file. @@ -57,24 +49,68 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-scaffold-source-http:dev -``` +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: -If you want to build the Docker image with the CDK on your local machine (rather than the most recent package published to pypi), from the airbyte base directory run: ```bash -CONNECTOR_TAG= CONNECTOR_NAME= sh airbyte-integrations/scripts/build-connector-image-with-local-cdk.sh +airbyte-ci connectors --name source-scaffold-source-http build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-scaffold-source-http:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations -You can also build the connector image via Gradle: +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-scaffold-source-http:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-scaffold-source-http:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. + +2. Build your image: +```bash +docker build -t airbyte/source-scaffold-source-http:dev . +# Running the spec command against your patched connector +docker run airbyte/source-scaffold-source-http:dev spec +```` #### Run Then run any of the connector commands as follows: @@ -103,24 +139,13 @@ Place custom tests inside `integration_tests/` folder, then, from the connector ``` python -m pytest integration_tests ``` -#### Acceptance Tests + +### Acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-scaffold-source-http:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-scaffold-source-http:integrationTest +Please run acceptance tests via [airbyte-ci](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#connectors-test-command): +```bash +airbyte-ci connectors --name source-scaffold-source-http test ``` ## Dependency Management @@ -131,8 +156,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-scaffold-source-http test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/scaffold-source-http.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/acceptance-test-config.yml b/airbyte-integrations/connectors/source-scaffold-source-http/acceptance-test-config.yml index 281a803a9111..3d73e0c2b14e 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-http/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-scaffold-source-http/acceptance-test-config.yml @@ -19,20 +19,20 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.jsonl" -# extra_fields: no -# exact_order: no -# extra_records: yes - incremental: + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.jsonl" + # extra_fields: no + # exact_order: no + # extra_records: yes + incremental: bypass_reason: "This connector does not implement incremental sync" -# TODO uncomment this block this block if your connector implements incremental sync: -# tests: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# future_state: -# future_state_path: "integration_tests/abnormal_state.json" + # TODO uncomment this block this block if your connector implements incremental sync: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-scaffold-source-http/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-scaffold-source-http/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/build.gradle b/airbyte-integrations/connectors/source-scaffold-source-http/build.gradle deleted file mode 100644 index dc3098e3b54e..000000000000 --- a/airbyte-integrations/connectors/source-scaffold-source-http/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_scaffold_source_http' -} diff --git a/airbyte-integrations/connectors/source-scaffold-source-http/metadata.yaml b/airbyte-integrations/connectors/source-scaffold-source-http/metadata.yaml index a11162b92d68..3e50afa2130e 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-http/metadata.yaml +++ b/airbyte-integrations/connectors/source-scaffold-source-http/metadata.yaml @@ -7,6 +7,11 @@ data: enabled: false cloud: enabled: false + connectorBuildOptions: + # Please update to the latest version of the connector base image. + # https://hub.docker.com/r/airbyte/python-connector-base + # Please use the full address with sha256 hash to guarantee build reproducibility. + baseImage: docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27 connectorSubtype: api connectorType: source definitionId: FAKE-UUID-0000-0000-000000000000 diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/.dockerignore b/airbyte-integrations/connectors/source-scaffold-source-python/.dockerignore deleted file mode 100644 index 35f32cdb325d..000000000000 --- a/airbyte-integrations/connectors/source-scaffold-source-python/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -* -!Dockerfile -!main.py -!source_scaffold_source_python -!setup.py -!secrets diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/Dockerfile b/airbyte-integrations/connectors/source-scaffold-source-python/Dockerfile deleted file mode 100644 index 30147c935d94..000000000000 --- a/airbyte-integrations/connectors/source-scaffold-source-python/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_scaffold_source_python ./source_scaffold_source_python - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-scaffold-source-python diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/README.md b/airbyte-integrations/connectors/source-scaffold-source-python/README.md index a9f40503d1fe..054cbe3741e9 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-python/README.md +++ b/airbyte-integrations/connectors/source-scaffold-source-python/README.md @@ -10,7 +10,7 @@ For information about how to use this connector within Airbyte, see [the documen #### Minimum Python version required `= 3.9.0` -#### Build & Activate Virtual Environment and install dependencies +#### Activate Virtual Environment and install dependencies From this connector directory, create a virtual environment: ``` python -m venv .venv @@ -21,6 +21,7 @@ development environment of choice. To activate it from the terminal, run: ``` source .venv/bin/activate pip install -r requirements.txt +pip install '.[tests]' ``` If you are in an IDE, follow your IDE's instructions to activate the virtualenv. @@ -29,16 +30,10 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-scaffold-source-python:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/scaffold-source-python) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_scaffold_source_python/spec.yaml` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source scaffold-source-python test creds` @@ -54,23 +49,68 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-scaffold-source-python:dev -``` +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: -If you want to build the Docker image with the CDK on your local machine (rather than the most recent package published to pypi), from the airbyte base directory run: ```bash -CONNECTOR_TAG= CONNECTOR_NAME= sh airbyte-integrations/scripts/build-connector-image-with-local-cdk.sh +airbyte-ci connectors --name source-scaffold-source-python build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-scaffold-source-python:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations -You can also build the connector image via Gradle: +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-scaffold-source-python:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-scaffold-source-python:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. + +2. Build your image: +```bash +docker build -t airbyte/source-scaffold-source-python:dev . +# Running the spec command against your patched connector +docker run airbyte/source-scaffold-source-python:dev spec +```` #### Run Then run any of the connector commands as follows: @@ -81,7 +121,7 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-scaffold-source-python docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-scaffold-source-python:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. First install test dependencies into your virtual environment: ``` pip install .[tests] @@ -99,24 +139,13 @@ Place custom tests inside `integration_tests/` folder, then, from the connector ``` python -m pytest integration_tests ``` -#### Acceptance Tests + +### Acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-scaffold-source-python:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-scaffold-source-python:integrationTest +Please run acceptance tests via [airbyte-ci](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#connectors-test-command): +```bash +airbyte-ci connectors --name source-scaffold-source-python test ``` ## Dependency Management @@ -127,8 +156,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-scaffold-source-python test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/scaffold-source-python.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/acceptance-test-config.yml b/airbyte-integrations/connectors/source-scaffold-source-python/acceptance-test-config.yml index bb370a5d693f..e31b63a31b08 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-python/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-scaffold-source-python/acceptance-test-config.yml @@ -19,20 +19,20 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.jsonl" -# extra_fields: no -# exact_order: no -# extra_records: yes - incremental: + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.jsonl" + # extra_fields: no + # exact_order: no + # extra_records: yes + incremental: bypass_reason: "This connector does not implement incremental sync" -# TODO uncomment this block this block if your connector implements incremental sync: -# tests: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# future_state: -# future_state_path: "integration_tests/abnormal_state.json" + # TODO uncomment this block this block if your connector implements incremental sync: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-scaffold-source-python/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-scaffold-source-python/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/build.gradle b/airbyte-integrations/connectors/source-scaffold-source-python/build.gradle deleted file mode 100644 index afdd6bb8d942..000000000000 --- a/airbyte-integrations/connectors/source-scaffold-source-python/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_scaffold_source_python_singer' -} diff --git a/airbyte-integrations/connectors/source-scaffold-source-python/metadata.yaml b/airbyte-integrations/connectors/source-scaffold-source-python/metadata.yaml index d23f6b533f3a..2a740dccdc56 100644 --- a/airbyte-integrations/connectors/source-scaffold-source-python/metadata.yaml +++ b/airbyte-integrations/connectors/source-scaffold-source-python/metadata.yaml @@ -7,6 +7,11 @@ data: enabled: false cloud: enabled: false + connectorBuildOptions: + # Please update to the latest version of the connector base image. + # https://hub.docker.com/r/airbyte/python-connector-base + # Please use the full address with sha256 hash to guarantee build reproducibility. + baseImage: docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27 connectorSubtype: api connectorType: source definitionId: FAKE-UUID-0000-0000-000000000000 diff --git a/airbyte-integrations/connectors/source-search-metrics/README.md b/airbyte-integrations/connectors/source-search-metrics/README.md index 79b66a3ea19a..e2403678f007 100644 --- a/airbyte-integrations/connectors/source-search-metrics/README.md +++ b/airbyte-integrations/connectors/source-search-metrics/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-search-metrics:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/search-metrics) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_search_metrics/spec.json` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-search-metrics:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-search-metrics build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-search-metrics:airbyteDocker +An image will be built with the tag `airbyte/source-search-metrics:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-search-metrics:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-search-metrics:dev che docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-search-metrics:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-search-metrics:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-search-metrics test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-search-metrics:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-search-metrics:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-search-metrics test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/search-metrics.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-search-metrics/acceptance-test-config.yml b/airbyte-integrations/connectors/source-search-metrics/acceptance-test-config.yml index d8b99cda0027..feccb404059a 100644 --- a/airbyte-integrations/connectors/source-search-metrics/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-search-metrics/acceptance-test-config.yml @@ -14,18 +14,21 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: ["list_market_share_s7", - "list_rankings_domain", - "list_competitors", - "distribution_keywords_s7", - "list_position_spread_historic_s7", - "list_seo_visibility_historic_s7", - "count_domain_keyword"] -# Incremental commented because incremental streams haven't records -# incremental: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# future_state_path: "integration_tests/abnormal_state.json" + empty_streams: + [ + "list_market_share_s7", + "list_rankings_domain", + "list_competitors", + "distribution_keywords_s7", + "list_position_spread_historic_s7", + "list_seo_visibility_historic_s7", + "count_domain_keyword", + ] + # Incremental commented because incremental streams haven't records + # incremental: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-search-metrics/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-search-metrics/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-search-metrics/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-search-metrics/build.gradle b/airbyte-integrations/connectors/source-search-metrics/build.gradle deleted file mode 100644 index 5c9fd46d303d..000000000000 --- a/airbyte-integrations/connectors/source-search-metrics/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_search_metrics' -} diff --git a/airbyte-integrations/connectors/source-secoda/README.md b/airbyte-integrations/connectors/source-secoda/README.md index 6fa8e084a319..3c42e6b401ab 100644 --- a/airbyte-integrations/connectors/source-secoda/README.md +++ b/airbyte-integrations/connectors/source-secoda/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-secoda:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/secoda) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_secoda/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-secoda:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-secoda build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-secoda:airbyteDocker +An image will be built with the tag `airbyte/source-secoda:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-secoda:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-secoda:dev check --con docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-secoda:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-secoda:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-secoda test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-secoda:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-secoda:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-secoda test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/secoda.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-secoda/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-secoda/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-secoda/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-secoda/build.gradle b/airbyte-integrations/connectors/source-secoda/build.gradle deleted file mode 100644 index b4ec4deb89e7..000000000000 --- a/airbyte-integrations/connectors/source-secoda/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_secoda' -} diff --git a/airbyte-integrations/connectors/source-sendgrid/Dockerfile b/airbyte-integrations/connectors/source-sendgrid/Dockerfile deleted file mode 100644 index eb3858abee2d..000000000000 --- a/airbyte-integrations/connectors/source-sendgrid/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY source_sendgrid ./source_sendgrid -COPY main.py ./ -COPY setup.py ./ -RUN pip install . - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.4.0 -LABEL io.airbyte.name=airbyte/source-sendgrid diff --git a/airbyte-integrations/connectors/source-sendgrid/README.md b/airbyte-integrations/connectors/source-sendgrid/README.md index 5818eb529ecb..36f73fd3d575 100644 --- a/airbyte-integrations/connectors/source-sendgrid/README.md +++ b/airbyte-integrations/connectors/source-sendgrid/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-sendgrid:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/sendgrid) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_sendgrid/spec.json` file. @@ -57,19 +49,70 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-sendgrid:dev + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-sendgrid build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-sendgrid:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. -You can also build the connector image via Gradle: +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-sendgrid:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-sendgrid:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-sendgrid:dev . +# Running the spec command against your patched connector +docker run airbyte/source-sendgrid:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -78,45 +121,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-sendgrid:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-sendgrid:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-sendgrid:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install '.[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-sendgrid test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-sendgrid:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-sendgrid:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-sendgrid:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -126,8 +140,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-sendgrid test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/sendgrid.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-sendgrid/acceptance-test-config.yml b/airbyte-integrations/connectors/source-sendgrid/acceptance-test-config.yml index 1f389b938535..7de7e9cc8e79 100644 --- a/airbyte-integrations/connectors/source-sendgrid/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-sendgrid/acceptance-test-config.yml @@ -3,60 +3,53 @@ test_strictness_level: "high" acceptance_tests: spec: tests: - - spec_path: "source_sendgrid/spec.json" - backward_compatibility_tests_config: - disable_for_version: "0.3.0" + - spec_path: "source_sendgrid/spec.json" + backward_compatibility_tests_config: + disable_for_version: "0.3.0" connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "secrets/old_config.json" - status: "succeed" - - config_path: "integration_tests/invalid_time.json" - status: "failed" - - config_path: "integration_tests/invalid_api_key.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "secrets/old_config.json" + status: "succeed" + - config_path: "integration_tests/invalid_time.json" + status: "failed" + - config_path: "integration_tests/invalid_api_key.json" + status: "failed" discovery: tests: - - config_path: "secrets/old_config.json" - backward_compatibility_tests_config: - disable_for_version: "0.3.0" + - config_path: "secrets/old_config.json" + backward_compatibility_tests_config: + disable_for_version: "0.3.0" basic_read: tests: - - config_path: "secrets/config.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: no - extra_records: yes - empty_streams: - - name: spam_reports - bypass_reason: "can not populate" - - name: invalid_emails - bypass_reason: "can not populate" - - name: blocks - bypass_reason: "can not populate" - ignored_fields: - segments: - - name: sample_updated_at - bypass_reason: "depend on current date" - - name: next_sample_update - bypass_reason: "depend on current date" - fail_on_extra_columns: false + - config_path: "secrets/config.json" + expect_records: + path: "integration_tests/expected_records.jsonl" + extra_fields: no + exact_order: no + extra_records: yes + empty_streams: + - name: spam_reports + bypass_reason: "can not populate" + - name: invalid_emails + bypass_reason: "can not populate" + - name: blocks + bypass_reason: "can not populate" + ignored_fields: + segments: + - name: sample_updated_at + bypass_reason: "depend on current date" + - name: next_sample_update + bypass_reason: "depend on current date" + fail_on_extra_columns: false incremental: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/no_spam_reports_configured_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" - cursor_paths: - global_suppressions: ["created"] - blocks: ["created"] - bounces: ["created"] - invalid_emails: ["created"] - # TODO: create spam_reports records - # spam_reports: ["created"] + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/no_spam_reports_configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-sendgrid/build.gradle b/airbyte-integrations/connectors/source-sendgrid/build.gradle deleted file mode 100644 index 0777775e1d34..000000000000 --- a/airbyte-integrations/connectors/source-sendgrid/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_sendgrid' -} diff --git a/airbyte-integrations/connectors/source-sendgrid/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-sendgrid/integration_tests/expected_records.jsonl index 8e972ef31486..188db682c2c6 100644 --- a/airbyte-integrations/connectors/source-sendgrid/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-sendgrid/integration_tests/expected_records.jsonl @@ -128,31 +128,15 @@ {"stream": "templates", "data": {"id": "146adde5-53ca-4ff7-af4e-24b21c6108e3", "name": "Template number 2", "generation": "legacy", "updated_at": "2021-02-03 13:31:18", "versions": []}, "emitted_at": 1631093374000} {"stream": "templates", "data": {"id": "bdc5cb3e-081f-4eeb-9068-c57a99ecd022", "name": "Template number 1", "generation": "legacy", "updated_at": "2021-02-03 13:31:15", "versions": []}, "emitted_at": 1631093374000} {"stream": "templates", "data": {"id": "d94c4158-8729-4dd4-aeca-7e5e903791c4", "name": "Template number 0", "generation": "legacy", "updated_at": "2021-02-03 13:31:09", "versions": []}, "emitted_at": 1631093374000} -{"stream": "global_suppressions", "data": {"created": 1621425285, "email": "vadym.hevlich@zazmic.com"}, "emitted_at": 1678792331625} -{"stream": "global_suppressions", "data": {"created": 1612357462, "email": "test9@example.com"}, "emitted_at": 1678792331625} -{"stream": "global_suppressions", "data": {"created": 1612357462, "email": "test8@example.com"}, "emitted_at": 1678792331625} -{"stream": "global_suppressions", "data": {"created": 1612357462, "email": "test7@example.com"}, "emitted_at": 1678792331625} -{"stream": "global_suppressions", "data": {"created": 1612357462, "email": "test6@example.com"}, "emitted_at": 1678792331625} -{"stream": "global_suppressions", "data": {"created": 1612357462, "email": "test5@example.com"}, "emitted_at": 1678792331625} -{"stream": "global_suppressions", "data": {"created": 1612357462, "email": "test4@example.com"}, "emitted_at": 1678792331626} -{"stream": "global_suppressions", "data": {"created": 1612357462, "email": "test3@example.com"}, "emitted_at": 1678792331626} -{"stream": "global_suppressions", "data": {"created": 1612357462, "email": "test2@example.com"}, "emitted_at": 1678792331626} -{"stream": "global_suppressions", "data": {"created": 1612357462, "email": "test21@example.com"}, "emitted_at": 1678792331626} -{"stream": "global_suppressions", "data": {"created": 1612357462, "email": "test20@example.com"}, "emitted_at": 1678792331626} -{"stream": "global_suppressions", "data": {"created": 1612357462, "email": "test1@example.com"}, "emitted_at": 1678792331626} -{"stream": "global_suppressions", "data": {"created": 1612357462, "email": "test19@example.com"}, "emitted_at": 1678792331626} -{"stream": "global_suppressions", "data": {"created": 1612357462, "email": "test18@example.com"}, "emitted_at": 1678792331627} -{"stream": "global_suppressions", "data": {"created": 1612357462, "email": "test17@example.com"}, "emitted_at": 1678792331627} -{"stream": "global_suppressions", "data": {"created": 1612357462, "email": "test16@example.com"}, "emitted_at": 1678792331627} -{"stream": "global_suppressions", "data": {"created": 1612357462, "email": "test15@example.com"}, "emitted_at": 1678792331627} -{"stream": "global_suppressions", "data": {"created": 1612357462, "email": "test14@example.com"}, "emitted_at": 1678792331628} -{"stream": "global_suppressions", "data": {"created": 1612357462, "email": "test13@example.com"}, "emitted_at": 1678792331628} -{"stream": "global_suppressions", "data": {"created": 1612357462, "email": "test12@example.com"}, "emitted_at": 1678792331628} -{"stream": "global_suppressions", "data": {"created": 1612357462, "email": "test11@example.com"}, "emitted_at": 1678792331628} -{"stream": "global_suppressions", "data": {"created": 1612357462, "email": "test10@example.com"}, "emitted_at": 1678792331628} -{"stream": "global_suppressions", "data": {"created": 1612357462, "email": "test0@example.com"}, "emitted_at": 1678792331628} -{"stream": "global_suppressions", "data": {"created": 1612357235, "email": "test-fake-email-2@testmail.co"}, "emitted_at": 1678792331628} -{"stream": "global_suppressions", "data": {"created": 1612357168, "email": "test-fake-email@testmail.co"}, "emitted_at": 1678792331628} +{"stream": "global_suppressions", "data": { "created": 1612361062, "email": "test2@example.com" }, "emitted_at": 1678792331625} +{"stream": "global_suppressions", "data": { "created": 1612361062, "email": "test5@example.com" }, "emitted_at": 1678792331625} +{"stream": "global_suppressions", "data": { "created": 1612361062, "email": "test3@example.com" }, "emitted_at": 1678792331625} +{"stream": "global_suppressions", "data": { "created": 1612361062, "email": "test19@example.com" }, "emitted_at": 1678792331625} +{"stream": "global_suppressions", "data": { "created": 1612361062, "email": "test11@example.com" }, "emitted_at": 1678792331625} +{"stream": "global_suppressions", "data": { "created": 1612361062, "email": "test13@example.com" }, "emitted_at": 1678792331625} +{"stream": "global_suppressions", "data": { "created": 1612360768, "email": "test-fake-email@testmail.co" }, "emitted_at": 1678792331626} +{"stream": "global_suppressions", "data": { "created": 1612361062, "email": "test8@example.com" }, "emitted_at": 1678792331626} +{"stream": "global_suppressions", "data": { "created": 1612361062, "email": "test1@example.com" }, "emitted_at": 1678792331626} {"stream": "suppression_groups", "data": {"name": "Test Suggestions Group 0", "id": 14760, "description": "Suggestions for testing new stream.", "is_default": false, "unsubscribes": 0}, "emitted_at": 1631093377000} {"stream": "suppression_groups", "data": {"name": "Test Suggestions Group 1", "id": 14761, "description": "Suggestions for testing new stream.", "is_default": false, "unsubscribes": 0}, "emitted_at": 1631093377000} {"stream": "suppression_groups", "data": {"name": "Test Suggestions Group 2", "id": 14762, "description": "Suggestions for testing new stream.", "is_default": false, "unsubscribes": 0}, "emitted_at": 1631093377000} @@ -200,12 +184,12 @@ {"stream": "suppression_group_members", "data": {"email": "test-forsuppressiongroup number8@example.com", "group_id": 14772, "group_name": "Test Suggestions Group 12", "created_at": 1612363238}, "emitted_at": 1631093393000} {"stream": "suppression_group_members", "data": {"email": "test-forsuppressiongroup number9@example.com", "group_id": 14772, "group_name": "Test Suggestions Group 12", "created_at": 1612363238}, "emitted_at": 1631093393000} {"stream": "suppression_group_members", "data": {"email": "avida.d3@gmail.com", "group_id": 14780, "group_name": "Test Suggestions Group 20", "created_at": 1631093329}, "emitted_at": 1631093393000} -{"stream": "bounces", "data": {"created": 1621439283, "email": "vadym.hevlich@zazmic_com", "reason": "Invalid Domain", "status": ""}, "emitted_at": 1678792680684} -{"stream": "bounces", "data": {"created": 1621439221, "email": "vadym.hevlich@zazmicinvalid", "reason": "Invalid Domain", "status": ""}, "emitted_at": 1678792680684} -{"stream": "bounces", "data": {"created": 1621439211, "email": "vadym.hevlich@zazmicio", "reason": "Invalid Domain", "status": ""}, "emitted_at": 1678792680684} -{"stream": "bounces", "data": {"created": 1621437507, "email": "vadym.hevlich@zazmiccom2", "reason": "Invalid Domain", "status": ""}, "emitted_at": 1678792680684} -{"stream": "bounces", "data": {"created": 1621437504, "email": "vadym.hevlich@zazmiccom1", "reason": "Invalid Domain", "status": ""}, "emitted_at": 1678792680685} -{"stream": "bounces", "data": {"created": 1621426437, "email": "vadym.hevlich@zazmiccom", "reason": "Invalid Domain", "status": ""}, "emitted_at": 1678792680685} +{"stream": "bounces", "data": { "created": 1621442821, "email": "vadym.hevlich@zazmicinvalid", "reason": "Invalid Domain", "status": "" }, "emitted_at": 1678792680684} +{"stream": "bounces", "data": { "created": 1621441107, "email": "vadym.hevlich@zazmiccom2", "reason": "Invalid Domain", "status": "" }, "emitted_at": 1678792680684} +{"stream": "bounces", "data": { "created": 1621442883, "email": "vadym.hevlich@zazmic_com", "reason": "Invalid Domain", "status": "" }, "emitted_at": 1678792680684} +{"stream": "bounces", "data": { "created": 1621441104, "email": "vadym.hevlich@zazmiccom1", "reason": "Invalid Domain", "status": "" }, "emitted_at": 1678792680684} +{"stream": "bounces", "data": { "created": 1621442811, "email": "vadym.hevlich@zazmicio", "reason": "Invalid Domain", "status": "" }, "emitted_at": 1678792680685} +{"stream": "bounces", "data": { "created": 1621430037, "email": "vadym.hevlich@zazmiccom", "reason": "Invalid Domain", "status": "" }, "emitted_at": 1678792680685} {"stream": "campaigns", "data": {"created_at": "2021-09-08T09:07:48Z", "id": "3c5a9fa6-1084-11ec-ac32-4228d699bad5", "name": "Untitled Single Send", "status": "triggered", "updated_at": "2021-09-08T09:11:08Z", "is_abtest": false, "channels": ["email"]}, "emitted_at": 1678791750589} {"stream": "campaigns", "data": {"created_at": "2021-09-08T09:04:36Z", "id": "c9f286fb-1083-11ec-ae03-ca0fc7f28419", "name": "Copy of Untitled Single Send", "status": "triggered", "updated_at": "2021-09-08T09:09:08Z", "is_abtest": false, "channels": ["email"]}, "emitted_at": 1678791750589} {"stream": "campaigns", "data": {"created_at": "2021-09-08T08:53:59Z", "id": "4e5be6a3-1082-11ec-8512-9afd40c324e6", "name": "Untitled Single Send", "status": "triggered", "updated_at": "2021-09-08T08:57:08Z", "is_abtest": false, "channels": ["email"]}, "emitted_at": 1678791750590} diff --git a/airbyte-integrations/connectors/source-sendgrid/metadata.yaml b/airbyte-integrations/connectors/source-sendgrid/metadata.yaml index 7aa5500cc96f..fae7388cf808 100644 --- a/airbyte-integrations/connectors/source-sendgrid/metadata.yaml +++ b/airbyte-integrations/connectors/source-sendgrid/metadata.yaml @@ -1,12 +1,18 @@ data: + ab_internal: + ql: 200 + sl: 200 allowedHosts: hosts: - api.sendgrid.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: api connectorType: source definitionId: fbb5fbe2-16ad-4cf4-af7d-ff9d9c316c87 - dockerImageTag: 0.4.0 + dockerImageTag: 0.4.1 dockerRepository: airbyte/source-sendgrid + documentationUrl: https://docs.airbyte.com/integrations/sources/sendgrid githubIssueLabel: source-sendgrid icon: sendgrid.svg license: MIT @@ -17,12 +23,8 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/sendgrid + supportLevel: certified tags: - language:low-code - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py index f26f12261728..14fab1cb6bee 100644 --- a/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py @@ -68,16 +68,16 @@ def test_streams(): assert len(streams) == 15 -@patch.multiple(SendgridStreamOffsetPagination, __abstractmethods__=set()) +@patch.multiple(SendgridStreamOffsetPagination, __abstractmethods__=set(), data_field="result") def test_pagination(mocker): stream = SendgridStreamOffsetPagination() state = {} response = requests.Response() - mocker.patch.object(response, "json", return_value={None: 1}) + mocker.patch.object(response, "json", return_value={"result": range(100)}) mocker.patch.object(response, "request", return_value=MagicMock()) next_page_token = stream.next_page_token(response) request_params = stream.request_params(stream_state=state, next_page_token=next_page_token) - assert request_params == {"limit": 50} + assert request_params == {"limit": 50, "offset": 50} @patch.multiple(SendgridStreamIncrementalMixin, __abstractmethods__=set()) @@ -153,21 +153,21 @@ def test_should_retry_on_permission_error(requests_mock, stream_class, status, e def test_compressed_contact_response(requests_mock): stream = Contacts() - with open(os.path.dirname(__file__)+"/compressed_response", "rb") as compressed_response: + with open(os.path.dirname(__file__) + "/compressed_response", "rb") as compressed_response: url = "https://api.sendgrid.com/v3/marketing/contacts/exports" requests_mock.register_uri("POST", url, [{"json": {"id": "random_id"}, "status_code": 202}]) url = "https://api.sendgrid.com/v3/marketing/contacts/exports/random_id" resp_bodies = [ {"json": {"status": "pending", "id": "random_id", "urls": []}, "status_code": 202}, - {"json": {"status": "ready", "urls": ["https://sample_url/sample_csv.csv.gzip"]}, "status_code": 202} + {"json": {"status": "ready", "urls": ["https://sample_url/sample_csv.csv.gzip"]}, "status_code": 202}, ] requests_mock.register_uri("GET", url, resp_bodies) - requests_mock.register_uri("GET", "https://sample_url/sample_csv.csv.gzip", - [{"body": compressed_response, "status_code": 202}]) + requests_mock.register_uri("GET", "https://sample_url/sample_csv.csv.gzip", [{"body": compressed_response, "status_code": 202}]) recs = list(stream.read_records(sync_mode=SyncMode.full_refresh)) - decompressed_response = pd.read_csv(os.path.dirname(__file__)+"/decompressed_response.csv", dtype=str) - expected_records = [{k.lower(): v for k, v in x.items()} for x in - decompressed_response.replace({nan: None}).to_dict(orient="records")] + decompressed_response = pd.read_csv(os.path.dirname(__file__) + "/decompressed_response.csv", dtype=str) + expected_records = [ + {k.lower(): v for k, v in x.items()} for x in decompressed_response.replace({nan: None}).to_dict(orient="records") + ] assert recs == expected_records @@ -176,8 +176,9 @@ def test_bad_job_response(requests_mock): stream = Contacts() url = "https://api.sendgrid.com/v3/marketing/contacts/exports" - requests_mock.register_uri("POST", url, [{"json": {"errors": [{"field": "field_name","message": "error message"}]}, - "status_code": codes.BAD_REQUEST}]) + requests_mock.register_uri( + "POST", url, [{"json": {"errors": [{"field": "field_name", "message": "error message"}]}, "status_code": codes.BAD_REQUEST}] + ) with pytest.raises(Exception): list(stream.read_records(sync_mode=SyncMode.full_refresh)) diff --git a/airbyte-integrations/connectors/source-sendinblue/Dockerfile b/airbyte-integrations/connectors/source-sendinblue/Dockerfile index 80211cef5582..6e2828317ee3 100644 --- a/airbyte-integrations/connectors/source-sendinblue/Dockerfile +++ b/airbyte-integrations/connectors/source-sendinblue/Dockerfile @@ -34,5 +34,5 @@ COPY source_sendinblue ./source_sendinblue ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-sendinblue diff --git a/airbyte-integrations/connectors/source-sendinblue/README.md b/airbyte-integrations/connectors/source-sendinblue/README.md index e55c4b92276e..36a751299ab5 100644 --- a/airbyte-integrations/connectors/source-sendinblue/README.md +++ b/airbyte-integrations/connectors/source-sendinblue/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-sendinblue:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/sendinblue) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_sendinblue/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-sendinblue:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-sendinblue build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-sendinblue:airbyteDocker +An image will be built with the tag `airbyte/source-sendinblue:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-sendinblue:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-sendinblue:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-sendinblue:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-sendinblue:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-sendinblue test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-sendinblue:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-sendinblue:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-sendinblue test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/sendinblue.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-sendinblue/acceptance-test-config.yml b/airbyte-integrations/connectors/source-sendinblue/acceptance-test-config.yml index 144dbb788a95..106bcca9d06f 100644 --- a/airbyte-integrations/connectors/source-sendinblue/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-sendinblue/acceptance-test-config.yml @@ -26,12 +26,11 @@ acceptance_tests: # exact_order: no # extra_records: yes incremental: - bypass_reason: "This connector does not implement incremental sync" - # TODO uncomment this block this block if your connector implements incremental sync: - # tests: - # - config_path: "secrets/config.json" - # configured_catalog_path: "integration_tests/configured_catalog.json" - # future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-sendinblue/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-sendinblue/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-sendinblue/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-sendinblue/build.gradle b/airbyte-integrations/connectors/source-sendinblue/build.gradle deleted file mode 100644 index fbdafecb5911..000000000000 --- a/airbyte-integrations/connectors/source-sendinblue/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_sendinblue' -} diff --git a/airbyte-integrations/connectors/source-sendinblue/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-sendinblue/integration_tests/abnormal_state.json index 52b0f2c2118f..663a73cd59d6 100644 --- a/airbyte-integrations/connectors/source-sendinblue/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-sendinblue/integration_tests/abnormal_state.json @@ -1,5 +1,13 @@ -{ - "todo-stream-name": { - "todo-field-name": "todo-abnormal-value" +[ + { + "type": "STREAM", + "stream": { + "stream_state": { + "modifiedAt": "2050-10-31T02:00:22.240+01:00" + }, + "stream_descriptor": { + "name": "contacts" + } + } } -} +] diff --git a/airbyte-integrations/connectors/source-sendinblue/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-sendinblue/integration_tests/configured_catalog.json index 8b74ccf7cd93..ec8321689992 100644 --- a/airbyte-integrations/connectors/source-sendinblue/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-sendinblue/integration_tests/configured_catalog.json @@ -22,10 +22,10 @@ "stream": { "name": "contacts", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "sync_mode": "incremental", + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-sendinblue/metadata.yaml b/airbyte-integrations/connectors/source-sendinblue/metadata.yaml index 6d5b26d340ba..4cebe426e4a3 100644 --- a/airbyte-integrations/connectors/source-sendinblue/metadata.yaml +++ b/airbyte-integrations/connectors/source-sendinblue/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 2e88fa20-a2f6-43cc-bba6-98a0a3f244fb - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 dockerRepository: airbyte/source-sendinblue documentationUrl: https://docs.airbyte.com/integrations/sources/sendinblue githubIssueLabel: source-sendinblue diff --git a/airbyte-integrations/connectors/source-sendinblue/setup.py b/airbyte-integrations/connectors/source-sendinblue/setup.py index 03d9985367a3..0b5269c46b84 100644 --- a/airbyte-integrations/connectors/source-sendinblue/setup.py +++ b/airbyte-integrations/connectors/source-sendinblue/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ diff --git a/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/manifest.yaml b/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/manifest.yaml index 94feb419dd32..83667afcd717 100644 --- a/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/manifest.yaml +++ b/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/manifest.yaml @@ -23,6 +23,19 @@ definitions: page_size_option: inject_into: "request_parameter" field_name: "limit" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: modifiedAt + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%f%z" + datetime_format: "%Y-%m-%dT%H:%M:%S.%f%z" + start_datetime: + type: MinMaxDatetime + datetime: "2000-01-01T00:00:00Z" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + inject_into: request_parameter + field_name: modifiedSince retriever: record_selector: $ref: "#/definitions/selector" @@ -53,6 +66,8 @@ definitions: path: "/smtp/templates" contacts_stream: $ref: "#/definitions/base_stream" + incremental_sync: + $ref: "#/definitions/incremental_sync" $parameters: name: contacts primary_key: id diff --git a/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/campaigns.json b/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/campaigns.json index 02ab48afdaf0..b72b892c4360 100644 --- a/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/campaigns.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] @@ -24,7 +25,9 @@ "type": ["null", "string"] }, "sender": { - "type": ["null", "object"] + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} }, "replyTo": { "type": ["null", "string"] @@ -62,6 +65,9 @@ "modifiedAt": { "type": ["null", "string"] }, + "previewText": { + "type": ["null", "string"] + }, "shareLink": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/contacts.json b/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/contacts.json index e23ac3c423a2..d20dfde96bab 100644 --- a/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/contacts.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] @@ -24,7 +25,9 @@ "type": ["null", "string"] }, "attributes": { - "type": ["null", "object"] + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} } } } diff --git a/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/templates.json b/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/templates.json index 8c4fd441f208..e57f22730e2e 100644 --- a/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/templates.json +++ b/airbyte-integrations/connectors/source-sendinblue/source_sendinblue/schemas/templates.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] @@ -18,7 +19,12 @@ "type": ["null", "boolean"] }, "sender": { - "type": ["null", "object"] + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} + }, + "subject": { + "type": ["null", "string"] }, "replyTo": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-senseforce/README.md b/airbyte-integrations/connectors/source-senseforce/README.md index a4ec3316468f..e3ab68570d88 100644 --- a/airbyte-integrations/connectors/source-senseforce/README.md +++ b/airbyte-integrations/connectors/source-senseforce/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-senseforce:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/senseforce) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_senseforce/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-senseforce:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-senseforce build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-senseforce:airbyteDocker +An image will be built with the tag `airbyte/source-senseforce:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-senseforce:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-senseforce:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-senseforce:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-senseforce:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-senseforce test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-senseforce:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-senseforce:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-senseforce test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/senseforce.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-senseforce/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-senseforce/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-senseforce/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-senseforce/build.gradle b/airbyte-integrations/connectors/source-senseforce/build.gradle deleted file mode 100644 index c2f21115da4f..000000000000 --- a/airbyte-integrations/connectors/source-senseforce/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_senseforce' -} diff --git a/airbyte-integrations/connectors/source-sentry/Dockerfile b/airbyte-integrations/connectors/source-sentry/Dockerfile deleted file mode 100644 index 24ceb52fa4bf..000000000000 --- a/airbyte-integrations/connectors/source-sentry/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_sentry ./source_sentry - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.2.4 -LABEL io.airbyte.name=airbyte/source-sentry diff --git a/airbyte-integrations/connectors/source-sentry/README.md b/airbyte-integrations/connectors/source-sentry/README.md index 12625af71610..f592bc7b71a8 100644 --- a/airbyte-integrations/connectors/source-sentry/README.md +++ b/airbyte-integrations/connectors/source-sentry/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-sentry:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/sentry) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_sentry/spec.json` file. @@ -57,19 +49,71 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-sentry:dev + + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name source-sentry build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-sentry:dev`. -You can also build the connector image via Gradle: +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-sentry:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-sentry:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-sentry:dev . +# Running the spec command against your patched connector +docker run airbyte/source-sentry:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -78,45 +122,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-sentry:dev check --con docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-sentry:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-sentry:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-sentry test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-sentry:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-sentry:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-sentry:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -126,8 +141,10 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -2. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -3. Create a Pull Request. -4. Pat yourself on the back for being an awesome contributor. -5. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-sentry test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/sentry.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml b/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml index 91c39b263e9d..c1bbe22949d7 100644 --- a/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-sentry/acceptance-test-config.yml @@ -2,6 +2,11 @@ acceptance_tests: basic_read: tests: - config_path: secrets/config.json + empty_streams: + - name: issues + bypass_reason: "Project sssues are not being returned by the Sentry API." + - name: events + bypass_reason: "No event records exist for the test project." timeout_seconds: 1200 expect_records: path: "integration_tests/expected_records.jsonl" @@ -11,6 +16,8 @@ acceptance_tests: bypass_reason: "Order access return randomly" - name: features bypass_reason: "Order features return randomly" + - name: options + bypass_reason: "Order options return randomly" - name: organization/features bypass_reason: "Order features return randomly" - name: plugins/*/features @@ -22,7 +29,6 @@ acceptance_tests: bypass_reason: "Order features return randomly" - name: organization/features bypass_reason: "Order features return randomly" - fail_on_extra_columns: false connection: tests: - config_path: secrets/config.json @@ -42,9 +48,15 @@ acceptance_tests: incremental: tests: - config_path: secrets/config.json + skip_comprehensive_incremental_tests: true configured_catalog_path: integration_tests/configured_catalog.json future_state: future_state_path: integration_tests/abnormal_state.json + missing_streams: + - name: issues + bypass_reason: "Project issues are not being returned by the Sentry API." + - name: events + bypass_reason: "No event records exist for the test project." spec: tests: - spec_path: source_sentry/spec.json diff --git a/airbyte-integrations/connectors/source-sentry/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-sentry/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-sentry/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-sentry/build.gradle b/airbyte-integrations/connectors/source-sentry/build.gradle deleted file mode 100644 index bdb2ada78084..000000000000 --- a/airbyte-integrations/connectors/source-sentry/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_sentry' -} diff --git a/airbyte-integrations/connectors/source-sentry/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-sentry/integration_tests/expected_records.jsonl index fe181c199a95..6720558b606b 100644 --- a/airbyte-integrations/connectors/source-sentry/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-sentry/integration_tests/expected_records.jsonl @@ -1,6 +1,5 @@ -{"stream": "project_detail", "data": {"id": "5942472", "slug": "airbyte-09", "name": "airbyte-09", "platform": "python", "dateCreated": "2021-09-02T07:42:22.421223Z", "isBookmarked": false, "isMember": true, "features": ["alert-filters", "minidump", "race-free-group-creation", "similarity-indexing", "similarity-view", "releases"], "firstEvent": null, "firstTransactionEvent": false, "access": ["member:read", "org:read", "team:admin", "event:admin", "org:integrations", "team:read", "event:read", "project:write", "project:releases", "alerts:read", "project:read", "event:write", "team:write", "alerts:write", "project:admin"], "hasAccess": true, "hasMinifiedStackTrace": false, "hasMonitors": false, "hasProfiles": false, "hasReplays": false, "hasSessions": false, "isInternal": false, "isPublic": false, "avatar": {"avatarType": "letter_avatar", "avatarUuid": null}, "color": "#803fbf", "status": "active", "team": {"id": "1170523", "slug": "airbyte", "name": "Airbyte"}, "teams": [{"id": "1170523", "slug": "airbyte", "name": "Airbyte"}], "latestRelease": {"version": "checkout-app@3.2"}, "options": {"quotas:spike-protection-disabled": false, "sentry:token": "5006ad000bc111ec95cd8e5fccda0a6a", "sentry:option-epoch": 7, "sentry:csp_ignored_sources_defaults": true, "sentry:csp_ignored_sources": "", "sentry:reprocessing_active": false, "filters:blacklisted_ips": "", "filters:react-hydration-errors": true, "filters:releases": "", "filters:error_messages": "", "feedback:branding": true}, "digestsMinDelay": 300, "digestsMaxDelay": 1800, "subjectPrefix": "", "allowedDomains": ["*"], "resolveAge": 0, "dataScrubber": true, "dataScrubberDefaults": true, "safeFields": [], "recapServerUrl": null, "storeCrashReports": null, "sensitiveFields": [], "subjectTemplate": "$shortID - $title", "securityToken": "5006ad000bc111ec95cd8e5fccda0a6a", "securityTokenHeader": null, "verifySSL": false, "scrubIPAddresses": false, "scrapeJavaScript": true, "groupingConfig": "newstyle:2023-01-11", "groupingEnhancements": "", "groupingEnhancementsBase": null, "secondaryGroupingExpiry": 0, "secondaryGroupingConfig": null, "groupingAutoUpdate": true, "fingerprintingRules": "", "organization": {"id": "985996", "slug": "airbyte-09", "status": {"id": "active", "name": "active"}, "name": "Airbyte", "dateCreated": "2021-09-02T07:41:55.899035Z", "isEarlyAdopter": false, "require2FA": false, "requireEmailVerification": false, "avatar": {"avatarType": "letter_avatar", "avatarUuid": null}, "features": ["anr-rate", "paid-to-free-promotion", "performance-m-n-plus-one-db-queries-visible", "performance-file-io-main-thread-visible", "customer-domains", "performance-landing-page-stats-period", "performance-uncompressed-assets-visible", "profile-image-decode-main-thread-ingest", "integrations-deployment", "performance-consecutive-db-queries-visible", "streamline-targeting-context", "performance-large-http-payload-visible", "assign-to-me", "issue-platform", "performance-render-blocking-asset-span-ingest", "sourcemaps-bundle-indexing", "bundle-plan-checkout", "symbol-sources", "release-health-drop-sessions", "profile-image-decode-main-thread-visible", "performance-db-main-thread-post-process-group", "profile-image-decode-main-thread-post-process-group", "performance-issues-all-events-tab", "performance-consecutive-http-ingest", "performance-n-plus-one-db-queries-post-process-group", "ondemand-budgets", "performance-slow-db-query-ingest", "performance-consecutive-http-post-process-group", "performance-db-main-thread-ingest", "performance-issues-compressed-assets-detector", "crons-timeline-listing-page", "team-project-creation-all", "dynamic-sampling", "promotion-be-adoption-enabled", "onboarding", "performance-onboarding-checklist", "performance-span-histogram-view", "performance-m-n-plus-one-db-queries-ingest", "performance-issues-m-n-plus-one-db-detector", "alert-crash-free-metrics", "performance-large-http-payload-ingest", "invite-members-rate-limits", "integrations-stacktrace-link", "promotion-mobperf-gift50kerr", "performance-render-blocking-asset-span-post-process-group", "transaction-name-normalize", "transaction-metrics-extraction", "business-to-team-promotion", "performance-n-plus-one-api-calls-detector", "dashboards-rh-widget", "transaction-name-mark-scrubbed-as-sanitized", "performance-consecutive-db-queries-ingest", "performance-slow-db-issue", "session-replay", "profile-json-decode-main-thread-visible", "metrics-extraction", "profile-file-io-main-thread-ingest", "performance-m-n-plus-one-db-queries-post-process-group", "performance-uncompressed-assets-post-process-group", "monitors", "onboarding-sdk-selection", "performance-n-plus-one-db-queries-visible", "performance-n-plus-one-api-calls-visible", "performance-n-plus-one-api-calls-post-process-group", "performance-consecutive-db-issue", "performance-mep-bannerless-ui", "performance-consecutive-db-queries-post-process-group", "profile-file-io-main-thread-post-process-group", "mobile-cpu-memory-in-transactions", "performance-slow-db-query-post-process-group", "device-classification", "issue-alert-fallback-targeting", "performance-uncompressed-assets-ingest", "performance-slow-db-query-visible", "profile-file-io-main-thread-visible", "performance-issues-search", "performance-consecutive-http-visible", "promotion-mobperf-discount20", "minute-resolution-sessions", "onboarding-project-deletion-on-back-click", "profiling", "metric-alert-chartcuterie", "mute-metric-alerts", "device-class-synthesis", "performance-new-widget-designs", "getting-started-doc-with-product-selection", "profiling-billing", "profiling-ga", "open-ai-suggestion", "org-subdomains", "performance-view", "session-replay-ui", "performance-n-plus-one-db-queries-ingest", "profile-json-decode-main-thread-post-process-group", "performance-render-blocking-asset-span-visible", "performance-metrics-backed-transaction-summary", "derive-code-mappings", "performance-file-io-main-thread-post-process-group", "performance-db-main-thread-detector", "track-button-click-events", "event-attachments", "performance-file-io-main-thread-ingest", "performance-large-http-payload-post-process-group", "open-membership", "issue-list-better-priority-sort", "shared-issues", "performance-transaction-name-only-search-indexed", "project-stats", "mep-rollout-flag", "ds-sliding-window-org", "am2-billing", "performance-issues-render-blocking-assets-detector", "slack-overage-notifications", "india-promotion", "session-replay-recording-scrubbing", "slack-use-new-lookup", "crons-issue-platform", "performance-file-io-main-thread-detector", "auto-enable-codecov", "discover-events-rate-limit", "performance-db-main-thread-visible", "performance-consecutive-http-detector", "performance-large-http-payload-detector", "profile-json-decode-main-thread-ingest", "advanced-search", "performance-n-plus-one-api-calls-ingest"], "links": {"organizationUrl": "https://airbyte-09.sentry.io", "regionUrl": "https://us.sentry.io"}, "hasAuthProvider": false}, "plugins": [], "platforms": [], "processingIssues": 0, "defaultEnvironment": null, "relayPiiConfig": null, "builtinSymbolSources": ["ios", "microsoft", "android"], "dynamicSamplingBiases": [{"id": "boostEnvironments", "active": true}, {"id": "boostLatestRelease", "active": true}, {"id": "ignoreHealthChecks", "active": true}, {"id": "boostKeyTransactions", "active": true}, {"id": "boostLowVolumeTransactions", "active": true}, {"id": "boostReplayId", "active": true}, {"id": "recalibrationRule", "active": true}], "eventProcessing": {"symbolicationDegraded": false}, "symbolSources": "[]"}, "emitted_at": 1689246410694} -{"stream":"projects","data":{"id":"6712547","slug":"demo-integration","name":"demo-integration","platform":"javascript-react","dateCreated":"2022-09-02T15:01:28.946777Z","isBookmarked":false,"isMember":true,"features":["alert-filters","minidump","race-free-group-creation","similarity-indexing","similarity-view"],"firstEvent":"2022-09-02T15:36:50.870000Z","firstTransactionEvent":false,"access":["org:integrations","team:read","alerts:write","alerts:read","project:read","member:read","project:write","project:admin","event:admin","team:write","event:write","project:releases","team:admin","event:read","org:read"],"hasAccess":true,"hasMinifiedStackTrace":false,"hasMonitors":false,"hasProfiles":false,"hasReplays":false,"hasSessions":false,"isInternal":false,"isPublic":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null},"color":"#bf833f","status":"active","organization":{"id":"985996","slug":"airbyte-09","status":{"id":"active","name":"active"},"name":"Airbyte","dateCreated":"2021-09-02T07:41:55.899035Z","isEarlyAdopter":false,"require2FA":false,"requireEmailVerification":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null},"features":["metrics-extraction","new-spike-protection","promotion-mobperf-discount20","session-replay-recording-scrubbing","performance-m-n-plus-one-db-queries-ingest","performance-consecutive-db-queries-ingest","sql-format","slack-overage-notifications","ondemand-budgets","issue-list-prefetch-issue-on-hover","profiling","performance-render-blocking-asset-span-ingest","getting-started-doc-with-product-selection","performance-consecutive-http-post-process-group","performance-view","integrations-stacktrace-link","dashboards-rh-widget","bundle-plan-checkout","invite-members-rate-limits","advanced-search","performance-db-main-thread-detector","integrations-deployment","profile-json-decode-main-thread-post-process-group","transaction-name-mark-scrubbed-as-sanitized","performance-db-main-thread-ingest","india-promotion","performance-consecutive-db-issue","performance-mep-bannerless-ui","performance-consecutive-http-detector","device-class-synthesis","auto-enable-codecov","performance-n-plus-one-api-calls-post-process-group","performance-render-blocking-asset-span-post-process-group","metric-alert-chartcuterie","session-replay","performance-landing-page-stats-period","transaction-name-normalize","symbol-sources","performance-issues-search","profile-json-decode-main-thread-ingest","sentry-pride-logo-footer","performance-uncompressed-assets-visible","profile-file-io-main-thread-visible","project-stats","performance-n-plus-one-api-calls-visible","performance-file-io-main-thread-ingest","profile-image-decode-main-thread-post-process-group","release-health-drop-sessions","minute-resolution-sessions","onboarding","performance-span-histogram-view","profile-image-decode-main-thread-ingest","profile-json-decode-main-thread-visible","paid-to-free-promotion","performance-file-io-main-thread-post-process-group","source-maps-debug-ids","performance-large-http-payload-post-process-group","performance-m-n-plus-one-db-queries-visible","profile-image-decode-main-thread-visible","ds-sliding-window-org","performance-issues-all-events-tab","performance-consecutive-db-queries-visible","streamline-targeting-context","onboarding-sdk-selection","session-replay-click-search-banner-rollout","performance-onboarding-checklist","discover-events-rate-limit","performance-m-n-plus-one-db-queries-post-process-group","shared-issues","performance-issues-m-n-plus-one-db-detector","profiling-billing","open-ai-suggestion","anr-rate","performance-n-plus-one-db-queries-visible","business-to-team-promotion","monitors","performance-slow-db-issue","team-project-creation-all","session-replay-index-subquery","performance-file-io-main-thread-visible","crons-issue-platform","performance-issues-compressed-assets-detector","performance-db-main-thread-post-process-group","performance-transaction-name-only-search-indexed","alert-crash-free-metrics","performance-slow-db-query-visible","dynamic-sampling-transaction-name-priority","performance-n-plus-one-db-queries-post-process-group","performance-render-blocking-asset-span-visible","onboarding-project-deletion-on-back-click","session-replay-ga","performance-file-io-main-thread-detector","profile-file-io-main-thread-post-process-group","promotion-mobperf-gift50kerr","performance-db-main-thread-visible","performance-n-plus-one-api-calls-ingest","performance-n-plus-one-db-queries-ingest","am2-billing","performance-uncompressed-assets-post-process-group","performance-metrics-backed-transaction-summary","performance-n-plus-one-api-calls-detector","performance-slow-db-query-post-process-group","performance-large-http-payload-ingest","ds-boost-new-projects","customer-domains","mobile-cpu-memory-in-transactions","session-replay-network-details","performance-slow-db-query-ingest","issue-alert-fallback-targeting","performance-consecutive-http-visible","performance-uncompressed-assets-ingest","session-replay-ui","performance-consecutive-db-queries-post-process-group","profiling-ga","open-membership","performance-new-widget-designs","crons-timeline-listing-page","event-attachments","mep-rollout-flag","device-classification","derive-code-mappings","profile-file-io-main-thread-ingest","issue-platform","track-button-click-events","performance-issues-render-blocking-assets-detector","dynamic-sampling","performance-large-http-payload-visible","transaction-metrics-extraction","performance-large-http-payload-detector","performance-consecutive-http-ingest","promotion-be-adoption-enabled","org-subdomains"],"links":{"organizationUrl":"https://airbyte-09.sentry.io","regionUrl":"https://us.sentry.io"},"hasAuthProvider":false}},"emitted_at":1687535328146} -{"stream":"projects","data":{"id":"5942472","slug":"airbyte-09","name":"airbyte-09","platform":"python","dateCreated":"2021-09-02T07:42:22.421223Z","isBookmarked":false,"isMember":true,"features":["alert-filters","minidump","race-free-group-creation","similarity-indexing","similarity-view","releases"],"firstEvent":null,"firstTransactionEvent":false,"access":["org:integrations","team:read","alerts:write","alerts:read","project:read","member:read","project:write","project:admin","event:admin","team:write","event:write","project:releases","team:admin","event:read","org:read"],"hasAccess":true,"hasMinifiedStackTrace":false,"hasMonitors":false,"hasProfiles":false,"hasReplays":false,"hasSessions":false,"isInternal":false,"isPublic":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null},"color":"#803fbf","status":"active","organization":{"id":"985996","slug":"airbyte-09","status":{"id":"active","name":"active"},"name":"Airbyte","dateCreated":"2021-09-02T07:41:55.899035Z","isEarlyAdopter":false,"require2FA":false,"requireEmailVerification":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null},"features":["metrics-extraction","new-spike-protection","promotion-mobperf-discount20","session-replay-recording-scrubbing","performance-m-n-plus-one-db-queries-ingest","performance-consecutive-db-queries-ingest","sql-format","slack-overage-notifications","ondemand-budgets","issue-list-prefetch-issue-on-hover","profiling","performance-render-blocking-asset-span-ingest","getting-started-doc-with-product-selection","performance-consecutive-http-post-process-group","performance-view","integrations-stacktrace-link","dashboards-rh-widget","bundle-plan-checkout","invite-members-rate-limits","advanced-search","performance-db-main-thread-detector","integrations-deployment","profile-json-decode-main-thread-post-process-group","transaction-name-mark-scrubbed-as-sanitized","performance-db-main-thread-ingest","india-promotion","performance-consecutive-db-issue","performance-mep-bannerless-ui","performance-consecutive-http-detector","device-class-synthesis","auto-enable-codecov","performance-n-plus-one-api-calls-post-process-group","performance-render-blocking-asset-span-post-process-group","metric-alert-chartcuterie","session-replay","performance-landing-page-stats-period","transaction-name-normalize","symbol-sources","performance-issues-search","profile-json-decode-main-thread-ingest","sentry-pride-logo-footer","performance-uncompressed-assets-visible","profile-file-io-main-thread-visible","project-stats","performance-n-plus-one-api-calls-visible","performance-file-io-main-thread-ingest","profile-image-decode-main-thread-post-process-group","release-health-drop-sessions","minute-resolution-sessions","onboarding","performance-span-histogram-view","profile-image-decode-main-thread-ingest","profile-json-decode-main-thread-visible","paid-to-free-promotion","performance-file-io-main-thread-post-process-group","source-maps-debug-ids","performance-large-http-payload-post-process-group","performance-m-n-plus-one-db-queries-visible","profile-image-decode-main-thread-visible","ds-sliding-window-org","performance-issues-all-events-tab","performance-consecutive-db-queries-visible","streamline-targeting-context","onboarding-sdk-selection","session-replay-click-search-banner-rollout","performance-onboarding-checklist","discover-events-rate-limit","performance-m-n-plus-one-db-queries-post-process-group","shared-issues","performance-issues-m-n-plus-one-db-detector","profiling-billing","open-ai-suggestion","anr-rate","performance-n-plus-one-db-queries-visible","business-to-team-promotion","monitors","performance-slow-db-issue","team-project-creation-all","session-replay-index-subquery","performance-file-io-main-thread-visible","crons-issue-platform","performance-issues-compressed-assets-detector","performance-db-main-thread-post-process-group","performance-transaction-name-only-search-indexed","alert-crash-free-metrics","performance-slow-db-query-visible","dynamic-sampling-transaction-name-priority","performance-n-plus-one-db-queries-post-process-group","performance-render-blocking-asset-span-visible","onboarding-project-deletion-on-back-click","session-replay-ga","performance-file-io-main-thread-detector","profile-file-io-main-thread-post-process-group","promotion-mobperf-gift50kerr","performance-db-main-thread-visible","performance-n-plus-one-api-calls-ingest","performance-n-plus-one-db-queries-ingest","am2-billing","performance-uncompressed-assets-post-process-group","performance-metrics-backed-transaction-summary","performance-n-plus-one-api-calls-detector","performance-slow-db-query-post-process-group","performance-large-http-payload-ingest","ds-boost-new-projects","customer-domains","mobile-cpu-memory-in-transactions","session-replay-network-details","performance-slow-db-query-ingest","issue-alert-fallback-targeting","performance-consecutive-http-visible","performance-uncompressed-assets-ingest","session-replay-ui","performance-consecutive-db-queries-post-process-group","profiling-ga","open-membership","performance-new-widget-designs","crons-timeline-listing-page","event-attachments","mep-rollout-flag","device-classification","derive-code-mappings","profile-file-io-main-thread-ingest","issue-platform","track-button-click-events","performance-issues-render-blocking-assets-detector","dynamic-sampling","performance-large-http-payload-visible","transaction-metrics-extraction","performance-large-http-payload-detector","performance-consecutive-http-ingest","promotion-be-adoption-enabled","org-subdomains"],"links":{"organizationUrl":"https://airbyte-09.sentry.io","regionUrl":"https://us.sentry.io"},"hasAuthProvider":false}},"emitted_at":1687535328148} +{"stream": "project_detail", "data": {"id": "5942472", "slug": "airbyte-09", "name": "airbyte-09", "platform": "python", "dateCreated": "2021-09-02T07:42:22.421223Z", "isBookmarked": false, "isMember": true, "features": ["alert-filters", "minidump", "race-free-group-creation", "similarity-indexing", "similarity-view", "span-metrics-extraction-resource", "span-metrics-extraction", "releases"], "firstEvent": null, "firstTransactionEvent": false, "access": ["event:admin", "event:write", "project:read", "org:read", "team:read", "org:integrations", "project:write", "alerts:write", "team:write", "event:read", "project:admin", "project:releases", "alerts:read", "member:read", "team:admin"], "hasAccess": true, "hasCustomMetrics": false, "hasMinifiedStackTrace": false, "hasMonitors": false, "hasProfiles": false, "hasReplays": false, "hasFeedbacks": false, "hasNewFeedbacks": false, "hasSessions": false, "isInternal": false, "isPublic": false, "avatar": {"avatarType": "letter_avatar", "avatarUuid": null}, "color": "#803fbf", "status": "active", "team": {"id": "1170523", "slug": "airbyte", "name": "Airbyte"}, "teams": [{"id": "1170523", "slug": "airbyte", "name": "Airbyte"}], "latestRelease": {"version": "checkout-app@3.2"}, "options": {"sentry:csp_ignored_sources_defaults": true, "sentry:csp_ignored_sources": "", "sentry:reprocessing_active": false, "filters:blacklisted_ips": "", "filters:react-hydration-errors": true, "filters:chunk-load-error": true, "filters:releases": "", "filters:error_messages": "", "feedback:branding": true, "quotas:spike-protection-disabled": false}, "digestsMinDelay": 300, "digestsMaxDelay": 1800, "subjectPrefix": "", "allowedDomains": ["*"], "resolveAge": 0, "dataScrubber": true, "dataScrubberDefaults": true, "safeFields": [], "recapServerUrl": null, "storeCrashReports": null, "sensitiveFields": [], "subjectTemplate": "$shortID - $title", "securityToken": "5006ad000bc111ec95cd8e5fccda0a6a", "securityTokenHeader": null, "verifySSL": false, "scrubIPAddresses": false, "scrapeJavaScript": true, "groupingConfig": "newstyle:2023-01-11", "groupingEnhancements": "", "groupingEnhancementsBase": null, "secondaryGroupingExpiry": 0, "secondaryGroupingConfig": null, "groupingAutoUpdate": true, "fingerprintingRules": "", "organization": {"id": "985996", "slug": "airbyte-09", "status": {"id": "active", "name": "active"}, "name": "Airbyte", "dateCreated": "2021-09-02T07:41:55.899035Z", "isEarlyAdopter": false, "require2FA": false, "requireEmailVerification": false, "avatar": {"avatarType": "letter_avatar", "avatarUuid": null, "avatarUrl": null}, "features": ["new-page-filter", "derive-code-mappings", "open-membership", "issue-platform-api-crons-sd", "sourcemaps-upload-release-as-artifact-bundle", "onboarding-sdk-selection", "minute-resolution-sessions", "performance-issues-search", "profile-json-decode-main-thread-ingest", "issue-platform-crons-sd", "profiling-statistical-detectors-breakpoint", "issue-stream-performance", "performance-http-overhead-ingest", "issue-platform", "ondemand-budgets", "session-replay-count-query-optimize", "profile-file-io-main-thread-post-process-group", "session-replay-event-linking", "performance-db-main-thread-detector", "slack-overage-notifications", "suspect-commits-all-frames", "feedback-visible", "feedback-post-process-group", "source-maps-debugger-blue-thunder-edition", "frontend-domainsplit", "device-classification", "crons-disable-new-projects", "transaction-name-mark-scrubbed-as-sanitized", "performance-span-histogram-view", "user-feedback-ingest", "ds-org-recalibration", "device-class-synthesis", "performance-mep-bannerless-ui", "promotion-mobperf-gift50kerr", "performance-transaction-name-only-search-indexed", "profile-image-decode-main-thread-ingest", "monitors", "session-replay-issue-emails", "promotion-be-adoption-enabled", "feedback-ingest", "performance-issues-compressed-assets-detector", "starfish-aggregate-span-waterfall", "performance-large-http-payload-detector", "profiling-global-suspect-functions", "transaction-metrics-extraction", "performance-http-overhead-post-process-group", "am2-billing", "project-stats", "starfish-browser-resource-module-ui", "session-replay", "profiling-ui-frames", "session-replay-recording-scrubbing", "sdk-crash-detection", "performance-consecutive-http-detector", "profiling-billing", "performance-tracing-without-performance", "performance-issues-http-overhead-detector", "session-replay-trial-ended-banner", "profile-image-decode-main-thread-post-process-group", "alerts-migration-enabled", "profiling-battery-usage-chart", "user-feedback-ui", "ds-sliding-window-org", "shared-issues", "issue-stream-performance-cache", "performance-view", "performance-calculate-score-relay", "event-attachments", "session-replay-show-hydration-errors", "performance-landing-page-stats-period", "symbol-sources", "profiling-summary-redesign", "metric-alert-chartcuterie", "discover-events-rate-limit", "starfish-browser-webvitals", "paid-to-free-promotion", "invite-members-rate-limits", "integrations-deployment", "release-health-drop-sessions", "session-replay-ui", "performance-duration-regression-ingest", "mep-rollout-flag", "profiling-view", "metrics-extraction", "profiling-memory-chart", "on-demand-metrics-extraction", "dashboard-widget-indicators", "performance-http-overhead-visible", "integrations-stacktrace-link", "auto-enable-codecov", "noisy-alert-warning", "advanced-search", "profile-json-decode-main-thread-visible", "performance-issues-render-blocking-assets-detector", "session-replay-accessibility-issues", "performance-statistical-detectors-breakpoint", "performance-n-plus-one-api-calls-detector", "customer-domains", "session-replay-a11y-tab", "escalating-issues", "streamline-targeting-context", "dynamic-sampling", "dashboards-rh-widget", "performance-database-view-query-source", "business-to-team-promotion", "profile-frame-drop-experimental-ingest", "mobile-cpu-memory-in-transactions", "starfish-browser-webvitals-pageoverview-v2", "performance-slow-db-issue", "performance-statistical-detectors-ema", "promotion-reserved-txn-discount", "session-replay-slack-new-issue", "profiling-statistical-detectors-ema", "profile-file-io-main-thread-visible", "profile-json-decode-main-thread-post-process-group", "performance-metrics-backed-transaction-summary", "onboarding", "profile-function-regression-exp-ingest", "org-subdomains", "performance-new-widget-designs", "performance-consecutive-db-issue", "performance-file-io-main-thread-detector", "trace-view-load-more", "profile-frame-drop-experimental-post-process-group", "profiling", "escalating-metrics-backend", "alert-crash-free-metrics", "promotion-mobperf-discount20", "dashboards-mep", "issue-alert-fallback-targeting", "transaction-name-normalize", "performance-issues-m-n-plus-one-db-detector", "performance-database-view", "profiling-cpu-chart", "integrations-gh-invite", "profiling-differential-flamegraph", "performance-onboarding-checklist", "profile-file-io-main-thread-ingest", "performance-screens-view", "profile-image-decode-main-thread-visible", "india-promotion", "session-replay-onboarding-cta-button", "performance-issues-all-events-tab"], "links": {"organizationUrl": "https://airbyte-09.sentry.io", "regionUrl": "https://us.sentry.io"}, "hasAuthProvider": false}, "plugins": [], "platforms": [], "processingIssues": 0, "defaultEnvironment": null, "relayPiiConfig": null, "builtinSymbolSources": ["ios", "microsoft", "android"], "dynamicSamplingBiases": [{"id": "boostEnvironments", "active": true}, {"id": "boostLatestRelease", "active": true}, {"id": "ignoreHealthChecks", "active": true}, {"id": "boostKeyTransactions", "active": true}, {"id": "boostLowVolumeTransactions", "active": true}, {"id": "boostReplayId", "active": true}, {"id": "recalibrationRule", "active": true}], "eventProcessing": {"symbolicationDegraded": false}, "symbolSources": "[]"}, "emitted_at": 1704483339623} {"stream": "releases", "data": {"id": 289364918, "version": "checkout-app@3.2", "status": "open", "shortVersion": "checkout-app@3.2", "versionInfo": {"package": "checkout-app", "version": {"raw": "3.2", "major": 3, "minor": 2, "patch": 0, "pre": null, "buildCode": null, "components": 2}, "description": "3.2", "buildHash": null}, "ref": null, "url": null, "dateReleased": null, "dateCreated": "2021-09-02T08:10:12.826000Z", "data": {}, "newGroups": 0, "owner": null, "commitCount": 0, "lastCommit": null, "deployCount": 0, "lastDeploy": null, "authors": [], "projects": [{"id": 5942472, "slug": "airbyte-09", "name": "airbyte-09", "newGroups": 0, "platform": "python", "platforms": [], "hasHealthData": false}], "firstEvent": null, "lastEvent": null, "currentProjectMeta": {}, "userAgent": null}, "emitted_at": 1689246658349} -{"stream": "issues", "data": {"id": "4365423845", "shareId": null, "shortId": "AIRBYTE-09-4", "title": "This is an example Python exception", "culprit": "raven.scripts.runner in main", "permalink": "https://airbyte-09.sentry.io/issues/4365423845/", "logger": null, "level": "error", "status": "unresolved", "statusDetails": {}, "substatus": "ongoing", "isPublic": false, "platform": "python", "project": {"id": "5942472", "name": "airbyte-09", "slug": "airbyte-09", "platform": "python"}, "type": "default", "metadata": {"title": "This is an example Python exception"}, "numComments": 0, "assignedTo": null, "isBookmarked": false, "isSubscribed": false, "subscriptionDetails": null, "hasSeen": true, "annotations": [], "issueType": "error", "issueCategory": "error", "isUnhandled": false, "count": "10", "userCount": 1, "firstSeen": "2023-08-02T23:22:34.982000Z", "lastSeen": "2023-08-02T23:31:20.165000Z"}, "emitted_at": 1691020096265} -{"stream": "events", "data": {"id": "1cce9233aeb04eba8bbdb7bb00f00592", "groupID": "4365423845", "eventID": "1cce9233aeb04eba8bbdb7bb00f00592", "projectID": "5942472", "size": 8151, "entries": [{"data": {"formatted": "This is an example Python exception"}, "type": "message"}, {"data": {"frames": [{"filename": "raven/base.py", "absPath": "/home/ubuntu/.virtualenvs/getsentry/src/raven/raven/base.py", "module": "raven.base", "package": null, "platform": null, "instructionAddr": null, "symbolAddr": null, "function": "build_msg", "rawFunction": null, "symbol": null, "context": [[298, " frames = stack"], [299, ""], [300, " data.update({"], [301, " 'sentry.interfaces.Stacktrace': {"], [302, " 'frames': get_stack_info(frames,"], [303, " transformer=self.transform)"], [304, " },"], [305, " })"], [306, ""], [307, " if 'sentry.interfaces.Stacktrace' in data:"], [308, " if self.include_paths:"]], "lineNo": 303, "colNo": null, "inApp": false, "trust": null, "errors": null, "lock": null, "vars": {"'culprit'": null, "'data'": {"'message'": "u'This is a test message generated using ``raven test``'", "'sentry.interfaces.Message'": {"'message'": "u'This is a test message generated using ``raven test``'", "'params'": []}}, "'date'": "datetime.datetime(2013, 8, 13, 3, 8, 24, 880386)", "'event_id'": "'54a322436e1b47b88e239b78998ae742'", "'event_type'": "'raven.events.Message'", "'extra'": {"'go_deeper'": [["{\"'bar'\":[\"'baz'\"],\"'foo'\":\"'bar'\"}"]], "'loadavg'": [0.37255859375, 0.5341796875, 0.62939453125], "'user'": "'dcramer'"}, "'frames'": "", "'handler'": "", "'k'": "'sentry.interfaces.Message'", "'kwargs'": {"'level'": 20, "'message'": "'This is a test message generated using ``raven test``'"}, "'public_key'": null, "'result'": {"'message'": "u'This is a test message generated using ``raven test``'", "'sentry.interfaces.Message'": {"'message'": "u'This is a test message generated using ``raven test``'", "'params'": []}}, "'self'": "", "'stack'": true, "'tags'": null, "'time_spent'": null, "'v'": {"'message'": "u'This is a test message generated using ``raven test``'", "'params'": []}}}, {"filename": "raven/base.py", "absPath": "/home/ubuntu/.virtualenvs/getsentry/src/raven/raven/base.py", "module": "raven.base", "package": null, "platform": null, "instructionAddr": null, "symbolAddr": null, "function": "capture", "rawFunction": null, "symbol": null, "context": [[454, " if not self.is_enabled():"], [455, " return"], [456, ""], [457, " data = self.build_msg("], [458, " event_type, data, date, time_spent, extra, stack, tags=tags,"], [459, " **kwargs)"], [460, ""], [461, " self.send(**data)"], [462, ""], [463, " return (data.get('event_id'),)"], [464, ""]], "lineNo": 459, "colNo": null, "inApp": false, "trust": null, "errors": null, "lock": null, "vars": {"'data'": null, "'date'": null, "'event_type'": "'raven.events.Message'", "'extra'": {"'go_deeper'": [["{\"'bar'\":[\"'baz'\"],\"'foo'\":\"'bar'\"}"]], "'loadavg'": [0.37255859375, 0.5341796875, 0.62939453125], "'user'": "'dcramer'"}, "'kwargs'": {"'level'": 20, "'message'": "'This is a test message generated using ``raven test``'"}, "'self'": "", "'stack'": true, "'tags'": null, "'time_spent'": null}}, {"filename": "raven/base.py", "absPath": "/home/ubuntu/.virtualenvs/getsentry/src/raven/raven/base.py", "module": "raven.base", "package": null, "platform": null, "instructionAddr": null, "symbolAddr": null, "function": "captureMessage", "rawFunction": null, "symbol": null, "context": [[572, " \"\"\""], [573, " Creates an event from ``message``."], [574, ""], [575, " >>> client.captureMessage('My event just happened!')"], [576, " \"\"\""], [577, " return self.capture('raven.events.Message', message=message, **kwargs)"], [578, ""], [579, " def captureException(self, exc_info=None, **kwargs):"], [580, " \"\"\""], [581, " Creates an event from an exception."], [582, ""]], "lineNo": 577, "colNo": null, "inApp": false, "trust": null, "errors": null, "lock": null, "vars": {"'kwargs'": {"'data'": null, "'extra'": {"'go_deeper'": ["[{\"'bar'\":[\"'baz'\"],\"'foo'\":\"'bar'\"}]"], "'loadavg'": [0.37255859375, 0.5341796875, 0.62939453125], "'user'": "'dcramer'"}, "'level'": 20, "'stack'": true, "'tags'": null}, "'message'": "'This is a test message generated using ``raven test``'", "'self'": ""}}, {"filename": "raven/scripts/runner.py", "absPath": "/home/ubuntu/.virtualenvs/getsentry/src/raven/raven/scripts/runner.py", "module": "raven.scripts.runner", "package": null, "platform": null, "instructionAddr": null, "symbolAddr": null, "function": "send_test_message", "rawFunction": null, "symbol": null, "context": [[72, " level=logging.INFO,"], [73, " stack=True,"], [74, " tags=options.get('tags', {}),"], [75, " extra={"], [76, " 'user': get_uid(),"], [77, " 'loadavg': get_loadavg(),"], [78, " },"], [79, " ))"], [80, ""], [81, " if client.state.did_fail():"], [82, " print('error!')"]], "lineNo": 77, "colNo": null, "inApp": false, "trust": null, "errors": null, "lock": null, "vars": {"'client'": "", "'data'": null, "'k'": "'secret_key'", "'options'": {"'data'": null, "'tags'": null}}}, {"filename": "raven/scripts/runner.py", "absPath": "/home/ubuntu/.virtualenvs/getsentry/src/raven/raven/scripts/runner.py", "module": "raven.scripts.runner", "package": null, "platform": null, "instructionAddr": null, "symbolAddr": null, "function": "main", "rawFunction": null, "symbol": null, "context": [[107, " print(\"Using DSN configuration:\")"], [108, " print(\" \", dsn)"], [109, " print()"], [110, ""], [111, " client = Client(dsn, include_paths=['raven'])"], [112, " send_test_message(client, opts.__dict__)"]], "lineNo": 112, "colNo": null, "inApp": false, "trust": null, "errors": null, "lock": null, "vars": {"'args'": ["'test'", "'https://ebc35f33e151401f9deac549978bda11:f3403f81e12e4c24942d505f086b2cad@sentry.io/1'"], "'client'": "", "'dsn'": "'https://ebc35f33e151401f9deac549978bda11:f3403f81e12e4c24942d505f086b2cad@sentry.io/1'", "'opts'": "", "'parser'": "", "'root'": ""}}], "framesOmitted": null, "registers": null, "hasSystemFrames": false}, "type": "stacktrace"}, {"data": {"apiTarget": null, "method": "GET", "url": "http://example.com/foo", "query": [["foo", "bar"]], "fragment": null, "data": {"hello": "world"}, "headers": [["Content-Type", "application/json"], ["Referer", "http://example.com"], ["User-Agent", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.72 Safari/537.36"]], "cookies": [["foo", "bar"], ["biz", "baz"]], "env": {"ENV": "prod"}, "inferredContentType": "application/json"}, "type": "request"}], "dist": null, "message": "This is an example Python exception", "title": "This is an example Python exception", "location": null, "user": {"id": "1", "email": "sentry@example.com", "username": "sentry", "ip_address": "127.0.0.1", "name": "Sentry", "data": null}, "contexts": {"browser": {"name": "Chrome", "version": "28.0.1500", "type": "browser"}, "client_os": {"name": "Windows", "version": "8", "type": "os"}}, "sdk": null, "context": {"emptyList": [], "emptyMap": {}, "length": 10837790, "results": [1, 2, 3, 4, 5], "session": {"foo": "bar"}, "unauthorized": false, "url": "http://example.org/foo/bar/"}, "packages": {"my.package": "1.0.0"}, "type": "default", "metadata": {"title": "This is an example Python exception"}, "tags": [{"key": "browser", "value": "Chrome 28.0.1500"}, {"key": "browser.name", "value": "Chrome"}, {"key": "client_os", "value": "Windows 8"}, {"key": "client_os.name", "value": "Windows"}, {"key": "environment", "value": "prod"}, {"key": "level", "value": "error"}, {"key": "sample_event", "value": "yes"}, {"key": "server_name", "value": "web01.example.org"}, {"key": "url", "value": "http://example.com/foo"}, {"key": "user", "value": "id:1", "query": "user.id:\"1\""}], "platform": "python", "dateReceived": "2023-08-02T23:31:41.101814Z", "errors": [], "occurrence": null, "_meta": {"entries": {}, "message": null, "user": null, "contexts": null, "sdk": null, "context": null, "packages": null, "tags": {}}, "crashFile": null, "culprit": "raven.scripts.runner in main", "dateCreated": "2023-08-02T23:30:41Z", "fingerprints": ["3a2b45089d0211943e5a6645fb4cea3f"], "groupingConfig": {"id": "newstyle:2023-01-11", "enhancements": "eJybzDRxc15qeXFJZU6qlZGBkbGugaGuoeEEAHJMCAM"}}, "emitted_at": 1691020171602} +{"stream":"projects","data":{"id":"4505884239200256","slug":"flutter_test","name":"flutter_test","platform":"flutter","dateCreated":"2023-09-15T11:26:50.595810Z","isBookmarked":false,"isMember":true,"features":["alert-filters","minidump","race-free-group-creation","similarity-indexing","similarity-view","span-metrics-extraction-resource","span-metrics-extraction"],"firstEvent":null,"firstTransactionEvent":false,"access":["event:write","team:read","project:releases","alerts:read","project:admin","project:read","org:read","org:integrations","team:write","alerts:write","event:admin","member:read","team:admin","project:write","event:read"],"hasAccess":true,"hasCustomMetrics":false,"hasMinifiedStackTrace":false,"hasMonitors":false,"hasProfiles":false,"hasReplays":false,"hasFeedbacks":false,"hasNewFeedbacks":false,"hasSessions":false,"isInternal":false,"isPublic":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null},"color":"#bf603f","status":"active","organization":{"id":"985996","slug":"airbyte-09","status":{"id":"active","name":"active"},"name":"Airbyte","dateCreated":"2021-09-02T07:41:55.899035Z","isEarlyAdopter":false,"require2FA":false,"requireEmailVerification":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null,"avatarUrl":null},"features":["onboarding-sdk-selection","performance-file-io-main-thread-detector","performance-large-http-payload-detector","project-stats","slack-overage-notifications","onboarding","integrations-stacktrace-link","profile-frame-drop-experimental-ingest","ds-org-recalibration","issue-platform-crons-sd","release-health-drop-sessions","performance-new-widget-designs","profile-file-io-main-thread-ingest","promotion-mobperf-gift50kerr","promotion-reserved-txn-discount","issue-platform-api-crons-sd","alert-crash-free-metrics","profile-file-io-main-thread-post-process-group","india-promotion","sourcemaps-upload-release-as-artifact-bundle","profiling-billing","performance-span-histogram-view","performance-tracing-without-performance","performance-http-overhead-ingest","performance-http-overhead-visible","minute-resolution-sessions","issue-stream-performance-cache","performance-db-main-thread-detector","profiling-battery-usage-chart","profiling-memory-chart","issue-alert-fallback-targeting","auto-enable-codecov","am2-billing","transaction-name-mark-scrubbed-as-sanitized","performance-issues-compressed-assets-detector","starfish-browser-webvitals","profile-image-decode-main-thread-visible","metric-alert-chartcuterie","session-replay-event-linking","event-attachments","user-feedback-ui","issue-platform","performance-database-view-query-source","open-membership","performance-transaction-name-only-search-indexed","frontend-domainsplit","profiling-cpu-chart","session-replay-slack-new-issue","escalating-metrics-backend","performance-calculate-score-relay","derive-code-mappings","profiling","performance-issues-m-n-plus-one-db-detector","discover-events-rate-limit","org-subdomains","session-replay-recording-scrubbing","profile-image-decode-main-thread-ingest","streamline-targeting-context","performance-screens-view","starfish-browser-resource-module-ui","starfish-aggregate-span-waterfall","profile-json-decode-main-thread-post-process-group","integrations-deployment","ondemand-budgets","sdk-crash-detection","transaction-metrics-extraction","performance-onboarding-checklist","profiling-statistical-detectors-breakpoint","dynamic-sampling","promotion-be-adoption-enabled","session-replay-count-query-optimize","trace-view-load-more","profiling-global-suspect-functions","source-maps-debugger-blue-thunder-edition","shared-issues","dashboards-mep","profiling-statistical-detectors-ema","dashboards-rh-widget","performance-issues-search","device-classification","feedback-ingest","performance-database-view","user-feedback-ingest","feedback-post-process-group","ds-sliding-window-org","performance-statistical-detectors-ema","session-replay-trial-ended-banner","new-page-filter","paid-to-free-promotion","performance-n-plus-one-api-calls-detector","performance-mep-bannerless-ui","performance-issues-render-blocking-assets-detector","session-replay-a11y-tab","business-to-team-promotion","advanced-search","profile-json-decode-main-thread-visible","profiling-view","suspect-commits-all-frames","alerts-migration-enabled","performance-issues-all-events-tab","performance-statistical-detectors-breakpoint","performance-consecutive-db-issue","profiling-summary-redesign","profile-frame-drop-experimental-post-process-group","performance-duration-regression-ingest","profiling-differential-flamegraph","session-replay-accessibility-issues","performance-slow-db-issue","profile-file-io-main-thread-visible","mobile-cpu-memory-in-transactions","session-replay-show-hydration-errors","performance-view","promotion-mobperf-discount20","session-replay-ui","metrics-extraction","dashboard-widget-indicators","device-class-synthesis","issue-stream-performance","starfish-browser-webvitals-pageoverview-v2","monitors","on-demand-metrics-extraction","performance-metrics-backed-transaction-summary","profile-function-regression-exp-ingest","symbol-sources","performance-landing-page-stats-period","performance-issues-http-overhead-detector","profile-image-decode-main-thread-post-process-group","escalating-issues","session-replay-issue-emails","mep-rollout-flag","customer-domains","transaction-name-normalize","feedback-visible","profile-json-decode-main-thread-ingest","performance-http-overhead-post-process-group","session-replay-onboarding-cta-button","crons-disable-new-projects","performance-consecutive-http-detector","noisy-alert-warning","integrations-gh-invite","profiling-ui-frames","session-replay","invite-members-rate-limits"],"links":{"organizationUrl":"https://airbyte-09.sentry.io","regionUrl":"https://us.sentry.io"},"hasAuthProvider":false}},"emitted_at":1704483641904} +{"stream":"projects","data":{"id":"4505884219408384","slug":"android_test_project","name":"android_test_project","platform":"android","dateCreated":"2023-09-15T11:21:48.131009Z","isBookmarked":false,"isMember":true,"features":["alert-filters","minidump","race-free-group-creation","similarity-indexing","similarity-view","span-metrics-extraction-resource","span-metrics-extraction","releases"],"firstEvent":null,"firstTransactionEvent":false,"access":["event:write","team:read","project:releases","alerts:read","project:admin","project:read","org:read","org:integrations","team:write","alerts:write","event:admin","member:read","team:admin","project:write","event:read"],"hasAccess":true,"hasCustomMetrics":false,"hasMinifiedStackTrace":false,"hasMonitors":false,"hasProfiles":false,"hasReplays":false,"hasFeedbacks":false,"hasNewFeedbacks":false,"hasSessions":false,"isInternal":false,"isPublic":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null},"color":"#3f95bf","status":"active","organization":{"id":"985996","slug":"airbyte-09","status":{"id":"active","name":"active"},"name":"Airbyte","dateCreated":"2021-09-02T07:41:55.899035Z","isEarlyAdopter":false,"require2FA":false,"requireEmailVerification":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null,"avatarUrl":null},"features":["onboarding-sdk-selection","performance-file-io-main-thread-detector","performance-large-http-payload-detector","project-stats","slack-overage-notifications","onboarding","integrations-stacktrace-link","profile-frame-drop-experimental-ingest","ds-org-recalibration","issue-platform-crons-sd","release-health-drop-sessions","performance-new-widget-designs","profile-file-io-main-thread-ingest","promotion-mobperf-gift50kerr","promotion-reserved-txn-discount","issue-platform-api-crons-sd","alert-crash-free-metrics","profile-file-io-main-thread-post-process-group","india-promotion","sourcemaps-upload-release-as-artifact-bundle","profiling-billing","performance-span-histogram-view","performance-tracing-without-performance","performance-http-overhead-ingest","performance-http-overhead-visible","minute-resolution-sessions","issue-stream-performance-cache","performance-db-main-thread-detector","profiling-battery-usage-chart","profiling-memory-chart","issue-alert-fallback-targeting","auto-enable-codecov","am2-billing","transaction-name-mark-scrubbed-as-sanitized","performance-issues-compressed-assets-detector","starfish-browser-webvitals","profile-image-decode-main-thread-visible","metric-alert-chartcuterie","session-replay-event-linking","event-attachments","user-feedback-ui","issue-platform","performance-database-view-query-source","open-membership","performance-transaction-name-only-search-indexed","frontend-domainsplit","profiling-cpu-chart","session-replay-slack-new-issue","escalating-metrics-backend","performance-calculate-score-relay","derive-code-mappings","profiling","performance-issues-m-n-plus-one-db-detector","discover-events-rate-limit","org-subdomains","session-replay-recording-scrubbing","profile-image-decode-main-thread-ingest","streamline-targeting-context","performance-screens-view","starfish-browser-resource-module-ui","starfish-aggregate-span-waterfall","profile-json-decode-main-thread-post-process-group","integrations-deployment","ondemand-budgets","sdk-crash-detection","transaction-metrics-extraction","performance-onboarding-checklist","profiling-statistical-detectors-breakpoint","dynamic-sampling","promotion-be-adoption-enabled","session-replay-count-query-optimize","trace-view-load-more","profiling-global-suspect-functions","source-maps-debugger-blue-thunder-edition","shared-issues","dashboards-mep","profiling-statistical-detectors-ema","dashboards-rh-widget","performance-issues-search","device-classification","feedback-ingest","performance-database-view","user-feedback-ingest","feedback-post-process-group","ds-sliding-window-org","performance-statistical-detectors-ema","session-replay-trial-ended-banner","new-page-filter","paid-to-free-promotion","performance-n-plus-one-api-calls-detector","performance-mep-bannerless-ui","performance-issues-render-blocking-assets-detector","session-replay-a11y-tab","business-to-team-promotion","advanced-search","profile-json-decode-main-thread-visible","profiling-view","suspect-commits-all-frames","alerts-migration-enabled","performance-issues-all-events-tab","performance-statistical-detectors-breakpoint","performance-consecutive-db-issue","profiling-summary-redesign","profile-frame-drop-experimental-post-process-group","performance-duration-regression-ingest","profiling-differential-flamegraph","session-replay-accessibility-issues","performance-slow-db-issue","profile-file-io-main-thread-visible","mobile-cpu-memory-in-transactions","session-replay-show-hydration-errors","performance-view","promotion-mobperf-discount20","session-replay-ui","metrics-extraction","dashboard-widget-indicators","device-class-synthesis","issue-stream-performance","starfish-browser-webvitals-pageoverview-v2","monitors","on-demand-metrics-extraction","performance-metrics-backed-transaction-summary","profile-function-regression-exp-ingest","symbol-sources","performance-landing-page-stats-period","performance-issues-http-overhead-detector","profile-image-decode-main-thread-post-process-group","escalating-issues","session-replay-issue-emails","mep-rollout-flag","customer-domains","transaction-name-normalize","feedback-visible","profile-json-decode-main-thread-ingest","performance-http-overhead-post-process-group","session-replay-onboarding-cta-button","crons-disable-new-projects","performance-consecutive-http-detector","noisy-alert-warning","integrations-gh-invite","profiling-ui-frames","session-replay","invite-members-rate-limits"],"links":{"organizationUrl":"https://airbyte-09.sentry.io","regionUrl":"https://us.sentry.io"},"hasAuthProvider":false}},"emitted_at":1704483641905} +{"stream":"projects","data":{"id":"6712547","slug":"demo-integration","name":"demo-integration","platform":"javascript-react","dateCreated":"2022-09-02T15:01:28.946777Z","isBookmarked":false,"isMember":true,"features":["alert-filters","minidump","race-free-group-creation","similarity-indexing","similarity-view","span-metrics-extraction-resource","span-metrics-extraction"],"firstEvent":"2022-09-02T15:36:50.870000Z","firstTransactionEvent":false,"access":["event:write","team:read","project:releases","alerts:read","project:admin","project:read","org:read","org:integrations","team:write","alerts:write","event:admin","member:read","team:admin","project:write","event:read"],"hasAccess":true,"hasCustomMetrics":false,"hasMinifiedStackTrace":false,"hasMonitors":false,"hasProfiles":false,"hasReplays":false,"hasFeedbacks":false,"hasNewFeedbacks":false,"hasSessions":false,"isInternal":false,"isPublic":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null},"color":"#bf833f","status":"active","organization":{"id":"985996","slug":"airbyte-09","status":{"id":"active","name":"active"},"name":"Airbyte","dateCreated":"2021-09-02T07:41:55.899035Z","isEarlyAdopter":false,"require2FA":false,"requireEmailVerification":false,"avatar":{"avatarType":"letter_avatar","avatarUuid":null,"avatarUrl":null},"features":["onboarding-sdk-selection","performance-file-io-main-thread-detector","performance-large-http-payload-detector","project-stats","slack-overage-notifications","onboarding","integrations-stacktrace-link","profile-frame-drop-experimental-ingest","ds-org-recalibration","issue-platform-crons-sd","release-health-drop-sessions","performance-new-widget-designs","profile-file-io-main-thread-ingest","promotion-mobperf-gift50kerr","promotion-reserved-txn-discount","issue-platform-api-crons-sd","alert-crash-free-metrics","profile-file-io-main-thread-post-process-group","india-promotion","sourcemaps-upload-release-as-artifact-bundle","profiling-billing","performance-span-histogram-view","performance-tracing-without-performance","performance-http-overhead-ingest","performance-http-overhead-visible","minute-resolution-sessions","issue-stream-performance-cache","performance-db-main-thread-detector","profiling-battery-usage-chart","profiling-memory-chart","issue-alert-fallback-targeting","auto-enable-codecov","am2-billing","transaction-name-mark-scrubbed-as-sanitized","performance-issues-compressed-assets-detector","starfish-browser-webvitals","profile-image-decode-main-thread-visible","metric-alert-chartcuterie","session-replay-event-linking","event-attachments","user-feedback-ui","issue-platform","performance-database-view-query-source","open-membership","performance-transaction-name-only-search-indexed","frontend-domainsplit","profiling-cpu-chart","session-replay-slack-new-issue","escalating-metrics-backend","performance-calculate-score-relay","derive-code-mappings","profiling","performance-issues-m-n-plus-one-db-detector","discover-events-rate-limit","org-subdomains","session-replay-recording-scrubbing","profile-image-decode-main-thread-ingest","streamline-targeting-context","performance-screens-view","starfish-browser-resource-module-ui","starfish-aggregate-span-waterfall","profile-json-decode-main-thread-post-process-group","integrations-deployment","ondemand-budgets","sdk-crash-detection","transaction-metrics-extraction","performance-onboarding-checklist","profiling-statistical-detectors-breakpoint","dynamic-sampling","promotion-be-adoption-enabled","session-replay-count-query-optimize","trace-view-load-more","profiling-global-suspect-functions","source-maps-debugger-blue-thunder-edition","shared-issues","dashboards-mep","profiling-statistical-detectors-ema","dashboards-rh-widget","performance-issues-search","device-classification","feedback-ingest","performance-database-view","user-feedback-ingest","feedback-post-process-group","ds-sliding-window-org","performance-statistical-detectors-ema","session-replay-trial-ended-banner","new-page-filter","paid-to-free-promotion","performance-n-plus-one-api-calls-detector","performance-mep-bannerless-ui","performance-issues-render-blocking-assets-detector","session-replay-a11y-tab","business-to-team-promotion","advanced-search","profile-json-decode-main-thread-visible","profiling-view","suspect-commits-all-frames","alerts-migration-enabled","performance-issues-all-events-tab","performance-statistical-detectors-breakpoint","performance-consecutive-db-issue","profiling-summary-redesign","profile-frame-drop-experimental-post-process-group","performance-duration-regression-ingest","profiling-differential-flamegraph","session-replay-accessibility-issues","performance-slow-db-issue","profile-file-io-main-thread-visible","mobile-cpu-memory-in-transactions","session-replay-show-hydration-errors","performance-view","promotion-mobperf-discount20","session-replay-ui","metrics-extraction","dashboard-widget-indicators","device-class-synthesis","issue-stream-performance","starfish-browser-webvitals-pageoverview-v2","monitors","on-demand-metrics-extraction","performance-metrics-backed-transaction-summary","profile-function-regression-exp-ingest","symbol-sources","performance-landing-page-stats-period","performance-issues-http-overhead-detector","profile-image-decode-main-thread-post-process-group","escalating-issues","session-replay-issue-emails","mep-rollout-flag","customer-domains","transaction-name-normalize","feedback-visible","profile-json-decode-main-thread-ingest","performance-http-overhead-post-process-group","session-replay-onboarding-cta-button","crons-disable-new-projects","performance-consecutive-http-detector","noisy-alert-warning","integrations-gh-invite","profiling-ui-frames","session-replay","invite-members-rate-limits"],"links":{"organizationUrl":"https://airbyte-09.sentry.io","regionUrl":"https://us.sentry.io"},"hasAuthProvider":false}},"emitted_at":1704483641905} diff --git a/airbyte-integrations/connectors/source-sentry/metadata.yaml b/airbyte-integrations/connectors/source-sentry/metadata.yaml index 1e4b9e046536..724e98d0823f 100644 --- a/airbyte-integrations/connectors/source-sentry/metadata.yaml +++ b/airbyte-integrations/connectors/source-sentry/metadata.yaml @@ -1,16 +1,22 @@ data: + ab_internal: + ql: 200 + sl: 200 allowedHosts: hosts: - "*" + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: cdaf146a-9b75-49fd-9dd2-9d64a0bb4781 - dockerImageTag: 0.2.4 - maxSecondsBetweenMessages: 64800 + dockerImageTag: 0.4.0 dockerRepository: airbyte/source-sentry + documentationUrl: https://docs.airbyte.com/integrations/sources/sentry githubIssueLabel: source-sentry icon: sentry.svg license: MIT + maxSecondsBetweenMessages: 64800 name: Sentry registries: cloud: @@ -18,12 +24,8 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/sentry + supportLevel: certified tags: - language:low-code - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/events.json b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/events.json index c53ca5f62bf9..70fff3323463 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/events.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/events.json @@ -2,25 +2,26 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { + "type": { "type": ["null", "string"] }, "eventID": { - "type": "string" + "type": ["string", "null"] }, "tags": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "object", + "type": ["object", "null"], "properties": { "value": { - "type": "string" + "type": ["string", "null"] }, "key": { - "type": "string" + "type": ["string", "null"] } } } }, "dateCreated": { - "type": "string", + "type": ["string", "null"], "format": "date-time" }, "user": { @@ -42,32 +43,158 @@ "type": ["null", "object"], "properties": { "isStaff": { - "type": "boolean" + "type": ["boolean", "null"] } } }, "id": { - "type": "string" + "type": ["string", "null"] } } }, "message": { - "type": "string" + "type": ["string", "null"] }, "id": { - "type": "string" + "type": ["string", "null"] }, "platform": { - "type": "string" + "type": ["string", "null"] }, "event.type": { - "type": "string" + "type": ["string", "null"] }, "groupID": { - "type": "string" + "type": ["string", "null"] }, "title": { - "type": "string" - } + "type": ["string", "null"] + }, + "_meta": { + "type": ["null", "object"], + "properties": { + "context": { "type": ["null", "string"] }, + "contexts": { "type": ["null", "string"] }, + "entries": { "type": ["null", "object"] }, + "message": { "type": ["null", "string"] }, + "packages": { "type": ["null", "string"] }, + "sdk": { "type": ["null", "string"] }, + "tags": { "type": ["null", "object"] }, + "user": { "type": ["null", "string"] } + } + }, + "context": { + "type": ["null", "object"], + "properties": { + "emptyList": { + "type": ["null", "array"] + }, + "emptyMap": { + "type": ["null", "object"] + }, + "length": { "type": ["null", "integer"] }, + "results": { + "type": ["null", "array"], + "items": { "type": ["integer", "null"] } + }, + "session": { + "type": ["null", "object"], + "properties": { + "foo": { + "type": ["null", "string"] + } + } + }, + "unauthorized": { "type": ["null", "boolean"] }, + "url": { "type": ["null", "string"] } + } + }, + "contexts": { + "type": ["null", "object"], + "properties": { + "browser": { + "type": ["null", "object"], + "properties": { + "name": { "type": ["null", "string"] }, + "type": { "type": ["null", "string"] }, + "version": { "type": ["null", "string"] } + } + }, + "client_os": { + "type": ["null", "object"], + "properties": { + "name": { "type": ["null", "string"] }, + "type": { "type": ["null", "string"] }, + "version": { "type": ["null", "string"] } + } + } + } + }, + "crashFile": { "type": ["null", "string"] }, + "culprit": { "type": ["null", "string"] }, + "dateReceived": { "type": ["null", "string"], "format": "date-time" }, + "dist": { "type": ["null", "string"] }, + "entries": { + "type": ["null", "array"], + "items": { "type": ["null", "object"] }, + "properties": { + "data": { + "type": ["null", "object"], + "properties": { + "formatted": { "type": ["null", "string"] }, + "frames": { + "type": ["null", "object"], + "properties": { + "absPath": { "type": ["null", "string"] }, + "colNo": { "type": ["null", "string"] }, + "context": { "type": ["null", "array"] }, + "filename": { "type": ["null", "string"] }, + "function": { "type": ["null", "string"] }, + "inApp": { "type": ["null", "boolean"] }, + "lineNo": { "type": ["null", "integer"] }, + "lock": { "type": ["null", "integer"] }, + "module": { "type": ["null", "string"] }, + "package": { "type": ["null", "string"] }, + "platform": { "type": ["null", "string"] }, + "rawFunction": { "type": ["null", "string"] }, + "sourceLink": { "type": ["null", "string"] }, + "symbol": { "type": ["null", "string"] }, + "symbolAddr": { "type": ["null", "string"] }, + "trust": { "type": ["null", "string"] } + } + } + } + }, + "type": { "type": ["null", "string"] } + } + }, + "errors": { + "type": ["null", "array"], + "items": { "type": ["null", "string"] } + }, + "fingerprints": { + "type": ["null", "array"], + "items": { "type": ["null", "string"] } + }, + "groupingConfig": { + "type": ["null", "object"], + "properties": { + "enhancements": { "type": ["null", "string"] }, + "id": { "type": ["null", "string"] } + } + }, + "location": { "type": ["null", "string"] }, + "metadata": { + "type": ["null", "object"], + "properties": { + "in_app_frame_mix": { "type": ["null", "string"] }, + "title": { "type": ["null", "string"] } + } + }, + "occurrence": { "type": ["null", "string"] }, + "packages": { "type": ["null", "object"] }, + "projectID": { "type": ["null", "string"] }, + "sdk": { "type": ["null", "string"] }, + "size": { "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/issues.json b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/issues.json index d4814ea21498..b56907d69728 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/issues.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/issues.json @@ -3,43 +3,43 @@ "type": "object", "properties": { "annotations": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "string" + "type": ["string", "null"] } }, "assignedTo": { "type": ["null", "object"] }, "count": { - "type": "string" + "type": ["string", "null"] }, "culprit": { - "type": "string" + "type": ["string", "null"] }, "firstSeen": { - "type": "string" + "type": ["string", "null"] }, "hasSeen": { - "type": "boolean" + "type": ["boolean", "null"] }, "id": { - "type": "string" + "type": ["string", "null"] }, "isBookmarked": { - "type": "boolean" + "type": ["boolean", "null"] }, "isPublic": { - "type": "boolean" + "type": ["boolean", "null"] }, "isSubscribed": { - "type": "boolean" + "type": ["boolean", "null"] }, "lastSeen": { - "type": "string" + "type": ["string", "null"] }, "level": { - "type": "string" + "type": ["string", "null"] }, "logger": { "type": ["null", "string"] @@ -47,46 +47,46 @@ "metadata": { "anyOf": [ { - "type": "object", + "type": ["object", "null"], "properties": { "title": { - "type": "string" + "type": ["string", "null"] } } }, { - "type": "object", + "type": ["object", "null"], "properties": { "filename": { - "type": "string" + "type": ["string", "null"] }, "type": { - "type": "string" + "type": ["string", "null"] }, "value": { - "type": "string" + "type": ["string", "null"] } } } ] }, "numComments": { - "type": "integer" + "type": ["integer", "null"] }, "permalink": { - "type": "string" + "type": ["string", "null"] }, "project": { - "type": "object", + "type": ["object", "null"], "properties": { "id": { - "type": "string" + "type": ["string", "null"] }, "name": { - "type": "string" + "type": ["string", "null"] }, "slug": { - "type": "string" + "type": ["string", "null"] } } }, @@ -94,40 +94,55 @@ "type": ["null", "string"] }, "shortId": { - "type": "string" + "type": ["string", "null"] }, "stats": { - "type": "object", + "type": ["object", "null"], "properties": { "24h": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "number" + "type": ["number", "null"] } } } } }, "status": { - "type": "string", + "type": ["string", "null"], "enum": ["resolved", "unresolved", "ignored"] }, "statusDetails": { - "type": "object" + "type": ["object", "null"] }, "subscriptionDetails": { "type": ["null", "object"] }, "title": { - "type": "string" + "type": ["string", "null"] }, "type": { - "type": "string" + "type": ["string", "null"] }, "userCount": { - "type": "integer" + "type": ["integer", "null"] + }, + "isUnhandled": { + "type": ["null", "boolean"] + }, + "issueCategory": { + "type": ["null", "string"] + }, + "issueType": { + "type": ["null", "string"] + }, + "platform": { + "type": ["null", "string"] + }, + "substatus": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/project_detail.json b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/project_detail.json index efb12e70ffbc..16e132f4bfff 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/project_detail.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/project_detail.json @@ -3,101 +3,104 @@ "type": "object", "properties": { "allowedDomains": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "string" + "type": ["string", "null"] } }, "avatar": { - "type": "object", + "type": ["object", "null"], "properties": { "avatarType": { - "type": "string" + "type": ["string", "null"] }, "avatarUuid": { "type": ["null", "string"] + }, + "avatarUrl": { + "type": ["null", "string"] } } }, "color": { - "type": "string" + "type": ["string", "null"] }, "dataScrubber": { - "type": "boolean" + "type": ["boolean", "null"] }, "dataScrubberDefaults": { - "type": "boolean" + "type": ["boolean", "null"] }, "dateCreated": { - "type": "string" + "type": ["string", "null"] }, "defaultEnvironment": { "type": ["null", "string"] }, "digestsMaxDelay": { - "type": "integer" + "type": ["integer", "null"] }, "digestsMinDelay": { - "type": "integer" + "type": ["integer", "null"] }, "features": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "string" + "type": ["string", "null"] } }, "firstEvent": { "type": ["null", "string"] }, "hasAccess": { - "type": "boolean" + "type": ["boolean", "null"] }, "id": { - "type": "string" + "type": ["string", "null"] }, "isBookmarked": { - "type": "boolean" + "type": ["boolean", "null"] }, "isInternal": { - "type": "boolean" + "type": ["boolean", "null"] }, "isMember": { - "type": "boolean" + "type": ["boolean", "null"] }, "isPublic": { - "type": "boolean" + "type": ["boolean", "null"] }, "latestRelease": { "type": ["null", "object"], "properties": { "authors": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "object", + "type": ["object", "null"], "properties": { "name": { - "type": "string" + "type": ["string", "null"] }, "email": { - "type": "string" + "type": ["string", "null"] } } } }, "commitCount": { - "type": "integer" + "type": ["integer", "null"] }, "data": { - "type": "object" + "type": ["object", "null"] }, "dateCreated": { - "type": "string" + "type": ["string", "null"] }, "dateReleased": { "type": ["null", "string"] }, "deployCount": { - "type": "integer" + "type": ["integer", "null"] }, "firstEvent": { "type": ["null", "string"] @@ -112,21 +115,21 @@ "type": ["null", "string"] }, "newGroups": { - "type": "integer" + "type": ["integer", "null"] }, "owner": { "type": ["null", "string"] }, "projects": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "object", + "type": ["object", "null"], "properties": { "name": { - "type": "string" + "type": ["string", "null"] }, "slug": { - "type": "string" + "type": ["string", "null"] } } } @@ -135,86 +138,89 @@ "type": ["null", "string"] }, "shortVersion": { - "type": "string" + "type": ["string", "null"] }, "url": { "type": ["null", "string"] }, "version": { - "type": "string" + "type": ["string", "null"] } } }, "name": { - "type": "string" + "type": ["string", "null"] }, "options": { - "type": "object", + "type": ["object", "null"], "properties": { "feedback:branding": { - "type": "boolean" + "type": ["boolean", "null"] }, "filters:blacklisted_ips": { - "type": "string" + "type": ["string", "null"] }, "filters:error_messages": { - "type": "string" + "type": ["string", "null"] }, "filters:releases": { - "type": "string" + "type": ["string", "null"] }, "sentry:csp_ignored_sources": { - "type": "string" + "type": ["string", "null"] }, "sentry:csp_ignored_sources_defaults": { - "type": "boolean" + "type": ["boolean", "null"] }, "sentry:reprocessing_active": { - "type": "boolean" + "type": ["boolean", "null"] } } }, "organization": { - "type": "object", + "type": ["object", "null"], "properties": { "avatar": { - "type": "object", + "type": ["object", "null"], "properties": { "avatarType": { - "type": "string" + "type": ["string", "null"] }, "avatarUuid": { "type": ["null", "string"] + }, + "avatarUrl": { + "type": ["null", "string"] } } }, "dateCreated": { - "type": "string", + "type": ["string", "null"], "format": "date-time" }, "id": { - "type": "string" + "type": ["string", "null"] }, "isEarlyAdopter": { - "type": "boolean" + "type": ["boolean", "null"] }, "name": { - "type": "string" + "type": ["string", "null"] }, "require2FA": { - "type": "boolean" + "type": ["boolean", "null"] }, "slug": { - "type": "string" + "type": ["string", "null"] }, "status": { - "type": "object", + "type": ["object", "null"], "properties": { "id": { - "type": "string" + "type": ["string", "null"] }, "name": { - "type": "string" + "type": ["string", "null"] } } } @@ -224,91 +230,116 @@ "type": ["null", "string"] }, "platforms": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "string" + "type": ["string", "null"] } }, "plugins": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "object", + "type": ["object", "null"], "properties": { + "altIsSentryApp": {}, "assets": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "string" + "type": ["string", "null"] } }, "author": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "name": { - "type": "string" + "type": ["string", "null"] }, "url": { - "type": "string" + "type": ["string", "null"] } } }, "canDisable": { - "type": "boolean" + "type": ["boolean", "null"] }, "contexts": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "string" + "type": ["string", "null"] } }, "description": { - "type": "string" + "type": ["string", "null"] }, + "deprecationDate": {}, "doc": { - "type": "string" + "type": ["string", "null"] }, "enabled": { - "type": "boolean" + "type": ["boolean", "null"] }, + "features": { + "type": ["array", "null"], + "items": { + "type": ["string", "null"] + } + }, + "featureDescriptions": { + "type": ["array", "null"], + "items": { + "type": ["object", "null"], + "additionalProperties": true + } + }, + "firstPartyAlternative": {}, "hasConfiguration": { - "type": "boolean" + "type": ["boolean", "null"] }, "id": { - "type": "string" + "type": ["string", "null"] + }, + "isDeprecated": { + "type": ["boolean", "null"] + }, + "isHidden": { + "type": ["boolean", "null"] }, "isTestable": { - "type": "boolean" + "type": ["boolean", "null"] }, "metadata": { - "type": "object" + "type": ["object", "null"], + "additionalProperties": true }, "name": { - "type": "string" + "type": ["string", "null"] }, "resourceLinks": { "type": ["null", "array"], "items": { - "type": "object", + "type": ["object", "null"], + "additionalProperties": true, "properties": { "title": { - "type": "string" + "type": ["string", "null"] }, "url": { - "type": "string" + "type": ["string", "null"] } } } }, "shortName": { - "type": "string" + "type": ["string", "null"] }, "slug": { - "type": "string" + "type": ["string", "null"] }, "status": { - "type": "string" + "type": ["string", "null"] }, "type": { - "type": "string" + "type": ["string", "null"] }, "version": { "type": ["null", "string"] @@ -317,86 +348,137 @@ } }, "processingIssues": { - "type": "integer" + "type": ["integer", "null"] }, "relayPiiConfig": { "type": ["null", "string"] }, "resolveAge": { - "type": "integer" + "type": ["integer", "null"] }, "safeFields": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "string" + "type": ["string", "null"] } }, "scrapeJavaScript": { - "type": "boolean" + "type": ["boolean", "null"] }, "scrubIPAddresses": { - "type": "boolean" + "type": ["boolean", "null"] }, "securityToken": { - "type": "string" + "type": ["string", "null"] }, "securityTokenHeader": { "type": ["null", "string"] }, "sensitiveFields": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "string" + "type": ["string", "null"] } }, "slug": { - "type": "string" + "type": ["string", "null"] }, "status": { - "type": "string" + "type": ["string", "null"] }, "storeCrashReports": { "type": ["null", "boolean"] }, "subjectPrefix": { - "type": "string" + "type": ["string", "null"] }, "subjectTemplate": { - "type": "string" + "type": ["string", "null"] }, "team": { - "type": "object", + "type": ["object", "null"], "properties": { "id": { - "type": "string" + "type": ["string", "null"] }, "name": { - "type": "string" + "type": ["string", "null"] }, "slug": { - "type": "string" + "type": ["string", "null"] } } }, "teams": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "object", + "type": ["object", "null"], "properties": { "id": { - "type": "string" + "type": ["string", "null"] }, "name": { - "type": "string" + "type": ["string", "null"] }, "slug": { - "type": "string" + "type": ["string", "null"] } } } }, "verifySSL": { - "type": "boolean" - } + "type": ["boolean", "null"] + }, + "access": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "builtinSymbolSources": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "dynamicSamplingBiases": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { "type": ["null", "string"] }, + "active": { "type": ["null", "boolean"] } + } + } + }, + "eventProcessing": { + "type": ["null", "object"], + "properties": { + "symbolicationDegraded": { + "type": ["null", "boolean"] + } + } + }, + "fingerprintingRules": { "type": ["null", "string"] }, + "firstTransactionEvent": { "type": ["null", "boolean"] }, + "groupingAutoUpdate": { "type": ["null", "boolean"] }, + "groupingConfig": { "type": ["null", "string"] }, + "groupingEnhancements": { "type": ["null", "string"] }, + "groupingEnhancementsBase": { "type": ["null", "string"] }, + "hasMinifiedStackTrace": { "type": ["null", "boolean"] }, + "hasMonitors": { "type": ["null", "boolean"] }, + "hasProfiles": { "type": ["null", "boolean"] }, + "hasReplays": { "type": ["null", "boolean"] }, + "hasSessions": { "type": ["null", "boolean"] }, + "hasFeedbacks": { "type": ["null", "boolean"] }, + "hasNewFeedbacks": { "type": ["null", "boolean"] }, + "hasCustomMetrics": { "type": ["null", "boolean"] }, + "recapServerUrl": { "type": ["null", "string"] }, + "secondaryGroupingConfig": { "type": ["null", "string"] }, + "secondaryGroupingExpiry": { "type": ["null", "integer"] }, + "symbolSources": { "type": ["null", "string"] }, + "stats": {}, + "transactionStats": {}, + "sessionStats": {} } } diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/projects.json b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/projects.json index 3656b0b27c2a..6abc90ab49e2 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/projects.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/projects.json @@ -3,104 +3,124 @@ "type": "object", "properties": { "avatar": { - "type": "object", + "type": ["object", "null"], "properties": { "avatarType": { - "type": "string" + "type": ["string", "null"] }, "avatarUuid": { "type": ["null", "string"] + }, + "avatarUrl": { + "type": ["null", "string"] } } }, "color": { - "type": "string" + "type": ["string", "null"] }, "dateCreated": { - "type": "string", + "type": ["string", "null"], "format": "date-time" }, "features": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "string" + "type": ["string", "null"] } }, "firstEvent": { "type": ["null", "string"] }, "hasAccess": { - "type": "boolean" + "type": ["boolean", "null"] }, "id": { - "type": "string" + "type": ["string", "null"] }, "isBookmarked": { - "type": "boolean" + "type": ["boolean", "null"] }, "isInternal": { - "type": "boolean" + "type": ["boolean", "null"] }, "isMember": { - "type": "boolean" + "type": ["boolean", "null"] }, "isPublic": { - "type": "boolean" + "type": ["boolean", "null"] }, "name": { - "type": "string" + "type": ["string", "null"] }, "organization": { - "type": "object", + "type": ["object", "null"], "properties": { "avatar": { - "type": "object", + "type": ["object", "null"], "properties": { "avatarType": { - "type": "string" + "type": ["string", "null"] }, "avatarUuid": { "type": ["null", "string"] + }, + "avatarUrl": { + "type": ["string", "null"] } } }, "dateCreated": { - "type": "string", + "type": ["string", "null"], "format": "date-time" }, "id": { - "type": "string" + "type": ["string", "null"] }, "isEarlyAdopter": { - "type": "boolean" + "type": ["boolean", "null"] + }, + "hasAuthProvider": { + "type": ["boolean", "null"] + }, + "links": { + "type": ["object", "null"], + "properties": { + "organizationUrl": { + "type": ["string", "null"] + }, + "regionUrl": { + "type": ["string", "null"] + } + } }, "name": { - "type": "string" + "type": ["string", "null"] }, "require2FA": { - "type": "boolean" + "type": ["boolean", "null"] }, "slug": { - "type": "string" + "type": ["string", "null"] }, "status": { - "type": "object", + "type": ["object", "null"], "properties": { "id": { - "type": "string" + "type": ["string", "null"] }, "name": { - "type": "string" + "type": ["string", "null"] } } }, "requireEmailVerification": { - "type": "boolean" + "type": ["boolean", "null"] }, "features": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "string" + "type": ["string", "null"] } } } @@ -109,11 +129,26 @@ "type": ["null", "string"] }, "slug": { - "type": "string" + "type": ["string", "null"] }, "status": { - "type": "string", + "type": ["string", "null"], "enum": ["active", "disabled", "pending_deletion", "deletion_in_progress"] - } + }, + "access": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "firstTransactionEvent": { "type": ["null", "boolean"] }, + "hasMonitors": { "type": ["null", "boolean"] }, + "hasProfiles": { "type": ["null", "boolean"] }, + "hasReplays": { "type": ["null", "boolean"] }, + "hasSessions": { "type": ["null", "boolean"] }, + "hasFeedbacks": { "type": ["null", "boolean"] }, + "hasNewFeedbacks": { "type": ["null", "boolean"] }, + "hasCustomMetrics": { "type": ["null", "boolean"] }, + "hasMinifiedStackTrace": { "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/releases.json b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/releases.json index 812fc480bcd2..5ecc0f17f0ef 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/releases.json +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/schemas/releases.json @@ -1,7 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", - "additionalProperties": true, "properties": { "id": { "type": ["null", "integer"] @@ -83,6 +82,31 @@ }, "version": { "type": ["null", "string"] + }, + "currentProjectMeta": { + "type": ["null", "object"] + }, + "status": { "type": ["null", "string"] }, + "userAgent": { "type": ["null", "string"] }, + "versionInfo": { + "type": ["null", "object"], + "properties": { + "buildHash": { "type": ["null", "string"] }, + "description": { "type": ["null", "string"] }, + "package": { "type": ["null", "string"] }, + "version": { + "type": ["null", "object"], + "properties": { + "buildCode": { "type": ["null", "string"] }, + "components": { "type": ["null", "integer"] }, + "major": { "type": ["null", "integer"] }, + "minor": { "type": ["null", "integer"] }, + "patch": { "type": ["null", "integer"] }, + "pre": { "type": ["null", "string"] }, + "raw": { "type": ["null", "string"] } + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py b/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py index 2cd4f4d516cf..1482228b362a 100644 --- a/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py +++ b/airbyte-integrations/connectors/source-sentry/source_sentry/streams.py @@ -118,7 +118,7 @@ def state(self, value: Mapping[str, Any]): class Events(SentryIncremental): """ - Docs: https://docs.sentry.io/api/events/list-a-projects-events/ + Docs: https://docs.sentry.io/api/events/list-a-projects-error-events/ """ primary_key = "id" diff --git a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py index 1ac42a5217e2..87376d158902 100644 --- a/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-sentry/unit_tests/test_streams.py @@ -2,11 +2,12 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from unittest.mock import MagicMock +from unittest.mock import MagicMock, Mock, patch import pendulum as pdm import pytest -from source_sentry.streams import Events, Issues, ProjectDetail, Projects, SentryStreamPagination +import requests +from source_sentry.streams import Events, Issues, ProjectDetail, Projects, SentryIncremental, SentryStreamPagination INIT_ARGS = {"hostname": "sentry.io", "organization": "test-org", "project": "test-project"} @@ -110,6 +111,39 @@ def test_project_detail_request_params(): expected = {} assert stream.request_params(stream_state=None, next_page_token=None) == expected +def test_issues_parse_response(mocker): + with patch('source_sentry.streams.Issues._get_cursor_value') as mock_get_cursor_value: + stream = Issues(**INIT_ARGS) + mock_get_cursor_value.return_value = "time" + state = {} + response = requests.Response() + mocker.patch.object(response, "json", return_value=[{"id": "1"}]) + result = list(stream.parse_response(response, state)) + assert result[0] == {"id": "1"} + +def test_project_detail_parse_response(mocker): + stream = ProjectDetail(organization="test_org", project="test_proj", hostname="sentry.io") + response = requests.Response() + response.json = Mock(return_value={"id": "1"}) + result = list(stream.parse_response(response)) + assert result[0] == {"id": "1"} + +class MockSentryIncremental(SentryIncremental): + def path(): + return '/test/path' + +def test_sentry_incremental_parse_response(mocker): + with patch('source_sentry.streams.SentryIncremental.filter_by_state') as mock_filter_by_state: + stream = MockSentryIncremental(hostname="sentry.io") + mock_filter_by_state.return_value = True + state = None + response = requests.Response() + mocker.patch.object(response, "json", return_value=[{"id": "1"}]) + mock_filter_by_state.return_value = iter(response.json()) + result = list(stream.parse_response(response, state)) + print(result) + assert result[0] == {"id": "1"} + @pytest.mark.parametrize( "state, expected", diff --git a/airbyte-integrations/connectors/source-serpstat/README.md b/airbyte-integrations/connectors/source-serpstat/README.md index c4698cf993b8..74a160fffcf1 100644 --- a/airbyte-integrations/connectors/source-serpstat/README.md +++ b/airbyte-integrations/connectors/source-serpstat/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-serpstat:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/serpstat) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_serpstat/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-serpstat:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-serpstat build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-serpstat:airbyteDocker +An image will be built with the tag `airbyte/source-serpstat:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-serpstat:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-serpstat:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-serpstat:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-serpstat:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-serpstat test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-serpstat:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-serpstat:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-serpstat test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/serpstat.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-serpstat/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-serpstat/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-serpstat/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-serpstat/build.gradle b/airbyte-integrations/connectors/source-serpstat/build.gradle deleted file mode 100644 index 446bea8580d3..000000000000 --- a/airbyte-integrations/connectors/source-serpstat/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_serpstat' -} diff --git a/airbyte-integrations/connectors/source-serpstat/metadata.yaml b/airbyte-integrations/connectors/source-serpstat/metadata.yaml index e764c9c80059..fb9d7de24737 100644 --- a/airbyte-integrations/connectors/source-serpstat/metadata.yaml +++ b/airbyte-integrations/connectors/source-serpstat/metadata.yaml @@ -18,5 +18,5 @@ data: releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/serpstat tags: - - language:lowcode + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-sftp-bulk/README.md b/airbyte-integrations/connectors/source-sftp-bulk/README.md index 9ec848d4dc8a..3aa14ff40808 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/README.md +++ b/airbyte-integrations/connectors/source-sftp-bulk/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-sftp-bulk:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/sftp-bulk) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_sftp_bulk/spec.json` file. @@ -54,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-sftp-bulk:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-sftp-bulk build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-sftp-bulk:airbyteDocker +An image will be built with the tag `airbyte/source-sftp-bulk:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-sftp-bulk:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,44 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-sftp-bulk:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-sftp-bulk:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-sftp-bulk:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-sftp-bulk test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-sftp-bulk:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-sftp-bulk:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -122,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-sftp-bulk test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/sftp-bulk.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-sftp-bulk/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-sftp-bulk/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-sftp-bulk/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-sftp-bulk/build.gradle b/airbyte-integrations/connectors/source-sftp-bulk/build.gradle deleted file mode 100644 index 88b5ebd4a16a..000000000000 --- a/airbyte-integrations/connectors/source-sftp-bulk/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_sftp_bulk' -} diff --git a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/integration_test.py index cdaedea1f455..8901c1dd1661 100644 --- a/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/integration_test.py +++ b/airbyte-integrations/connectors/source-sftp-bulk/integration_tests/integration_test.py @@ -244,8 +244,10 @@ def test_get_files_pattern_csv_new_separator(config: Mapping, configured_catalog def test_get_files_pattern_csv_new_separator_with_config(config: Mapping, configured_catalog: ConfiguredAirbyteCatalog): source = SourceFtp() result_iter = source.read( - logger, {**config, "file_type": "csv", "folder_path": "files/csv", "separator": ";", "file_pattern": "test_2.+"}, - configured_catalog, None + logger, + {**config, "file_type": "csv", "folder_path": "files/csv", "separator": ";", "file_pattern": "test_2.+"}, + configured_catalog, + None, ) result = list(result_iter) assert len(result) == 2 diff --git a/airbyte-integrations/connectors/source-sftp/.dockerignore b/airbyte-integrations/connectors/source-sftp/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-sftp/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-sftp/Dockerfile b/airbyte-integrations/connectors/source-sftp/Dockerfile deleted file mode 100644 index e862291e3814..000000000000 --- a/airbyte-integrations/connectors/source-sftp/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte -ENV APPLICATION source-sftp - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte -ENV APPLICATION source-sftp - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.1.2 -LABEL io.airbyte.name=airbyte/source-sftp diff --git a/airbyte-integrations/connectors/source-sftp/README.md b/airbyte-integrations/connectors/source-sftp/README.md index 60834b02146f..7991b543e3ca 100644 --- a/airbyte-integrations/connectors/source-sftp/README.md +++ b/airbyte-integrations/connectors/source-sftp/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:source-sftp:airbyteDocker +./gradlew :airbyte-integrations:connectors:source-sftp:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/source-sftp:dev`. the Dockerfile. #### Run @@ -61,8 +62,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-sftp test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/sftp.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-sftp/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-sftp/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-sftp/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-sftp/build.gradle b/airbyte-integrations/connectors/source-sftp/build.gradle index dc6c72be8c8d..ba5b5b1a9748 100644 --- a/airbyte-integrations/connectors/source-sftp/build.gradle +++ b/airbyte-integrations/connectors/source-sftp/build.gradle @@ -1,23 +1,30 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' - id 'airbyte-connector-acceptance-test' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.7.7' + features = ['db-sources'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.source.sftp.SftpSource' } dependencies { - implementation project(':airbyte-config-oss:config-models-oss') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:bases:base-java') - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.13.2' implementation 'com.jcraft:jsch:0.1.55' - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-sftp') - testImplementation libs.connectors.testcontainers + testImplementation libs.testcontainers } diff --git a/airbyte-integrations/connectors/source-sftp/metadata.yaml b/airbyte-integrations/connectors/source-sftp/metadata.yaml index dd9961561c93..6cd5d9090420 100644 --- a/airbyte-integrations/connectors/source-sftp/metadata.yaml +++ b/airbyte-integrations/connectors/source-sftp/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: file connectorType: source definitionId: a827c52e-791c-4135-a245-e233c5255199 - dockerImageTag: 0.1.2 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-sftp documentationUrl: https://docs.airbyte.com/integrations/sources/sftp githubIssueLabel: source-sftp diff --git a/airbyte-integrations/connectors/source-sftp/src/main/java/io/airbyte/integrations/source/sftp/SftpSource.java b/airbyte-integrations/connectors/source-sftp/src/main/java/io/airbyte/integrations/source/sftp/SftpSource.java index b64f43f4ba2b..0787005e2194 100644 --- a/airbyte-integrations/connectors/source-sftp/src/main/java/io/airbyte/integrations/source/sftp/SftpSource.java +++ b/airbyte-integrations/connectors/source-sftp/src/main/java/io/airbyte/integrations/source/sftp/SftpSource.java @@ -6,12 +6,13 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.Lists; +import io.airbyte.cdk.integrations.BaseConnector; +import io.airbyte.cdk.integrations.base.AirbyteTraceMessageUtility; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.commons.stream.AirbyteStreamUtils; import io.airbyte.commons.util.AutoCloseableIterator; import io.airbyte.commons.util.AutoCloseableIterators; -import io.airbyte.integrations.BaseConnector; -import io.airbyte.integrations.base.AirbyteTraceMessageUtility; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteMessage; @@ -148,7 +149,8 @@ private AutoCloseableIterator getFileDataIterator(final SftpCommand co } catch (final Exception e) { throw new RuntimeException(e); } - }); + }, + AirbyteStreamUtils.convertFromNameAndNamespace(stream.getName(), stream.getNamespace())); } } diff --git a/airbyte-integrations/connectors/source-sftp/src/test-integration/java/io/airbyte/integrations/source/sftp/SftpSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-sftp/src/test-integration/java/io/airbyte/integrations/source/sftp/SftpSourceAcceptanceTest.java index db7ee79f080b..9bee2007cb56 100644 --- a/airbyte-integrations/connectors/source-sftp/src/test-integration/java/io/airbyte/integrations/source/sftp/SftpSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-sftp/src/test-integration/java/io/airbyte/integrations/source/sftp/SftpSourceAcceptanceTest.java @@ -6,11 +6,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.integrations.util.HostPortResolver; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.CatalogHelpers; diff --git a/airbyte-integrations/connectors/source-shopify/Dockerfile b/airbyte-integrations/connectors/source-shopify/Dockerfile deleted file mode 100644 index 40a304001ec3..000000000000 --- a/airbyte-integrations/connectors/source-shopify/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade && pip install --upgrade pip - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local - -# Bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_shopify ./source_shopify - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.6.2 -LABEL io.airbyte.name=airbyte/source-shopify diff --git a/airbyte-integrations/connectors/source-shopify/README.md b/airbyte-integrations/connectors/source-shopify/README.md index 6b54e00c0703..4a1a550644b9 100644 --- a/airbyte-integrations/connectors/source-shopify/README.md +++ b/airbyte-integrations/connectors/source-shopify/README.md @@ -34,14 +34,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-shopify:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/shopify) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_shopify/spec.json` file. @@ -63,20 +55,70 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-shopify:dev -``` -You can also build the connector image via Gradle: + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-shopify build ``` -./gradlew :airbyte-integrations:connectors:source-shopify:airbyteDocker +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-shopify:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-shopify:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] +``` +Please use this as an example. This is not optimized. + +2. Build your image: +```bash +docker build . --no-cache -t airbyte/source-shopify:dev +# Running the spec command against your patched connector +docker run airbyte/source-shopify:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -86,53 +128,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-shopify:dev discover - docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-shopify:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install '.[tests]' -``` - -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python3 -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python3 -m pytest integration_tests +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-shopify test ``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-shopify:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-shopify:unitTest -``` - -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-shopify:integrationTest -``` - -To build final build the connector: -``` -./gradlew :airbyte-integrations:connectors:source-shopify:build -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -142,8 +147,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-shopify test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/shopify.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml b/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml index 9c1cea89b117..3186b4e26267 100644 --- a/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-shopify/acceptance-test-config.yml @@ -24,10 +24,13 @@ acceptance_tests: discovery: tests: - config_path: "secrets/config.json" + backward_compatibility_tests_config: + # The cursor field for `fulfillments` stream has changed from `id` to `updated_at` + disable_for_version: "1.1.6" basic_read: tests: - config_path: "secrets/config.json" - timeout_seconds: 3600 + timeout_seconds: 4800 expect_records: path: "integration_tests/expected_records.jsonl" empty_streams: @@ -39,15 +42,61 @@ acceptance_tests: bypass_reason: The stream is not available for our sandbox. - name: disputes bypass_reason: The stream requires real purchases to fill in the data. + ignored_fields: + products: + - name: variants/*/updated_at + bypass_reason: Value can change as the account data is not frozen + - name: image/src + bypass_reason: May contain dynamically changed URL params + - name: image/updated_at + bypass_reason: Value can change as the account data is not frozen + - name: images/*/src + bypass_reason: May contain dynamically changed URL params + - name: images/*/updated_at + bypass_reason: Value can change as the account data is not frozen + products_graph_ql: + - name: onlineStorePreviewUrl + bypass_reason: Autogenerated floating URL values + product_variants: + - name: updated_at + bypass_reason: Value can change as the account data is not frozen + product_images: + - name: src + bypass_reason: May contain dynamically changed URL params + - name: updated_at + bypass_reason: Value can change as the account data is not frozen incremental: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - future_state: + future_state: future_state_path: "integration_tests/abnormal_state.json" - timeout_seconds: 3600 + timeout_seconds: 7200 full_refresh: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 3600 + timeout_seconds: 4800 + ignored_fields: + products: + - name: variants/*/updated_at + bypass_reason: Value can change as the account data is not frozen + - name: image/src + bypass_reason: May contain dynamically changed URL params + - name: image/updated_at + bypass_reason: Value can change as the account data is not frozen + - name: images/*/src + bypass_reason: May contain dynamically changed URL params + - name: images/*/updated_at + bypass_reason: Value can change as the account data is not frozen + products_graph_ql: + - name: onlineStorePreviewUrl + bypass_reason: Floating URL values + product_variants: + - name: updated_at + bypass_reason: Value can change as the account data is not frozen + product_images: + - name: src + bypass_reason: May contain dynamically changed URL params + - name: updated_at + bypass_reason: Value can change as the account data is not frozen diff --git a/airbyte-integrations/connectors/source-shopify/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-shopify/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-shopify/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-shopify/build.gradle b/airbyte-integrations/connectors/source-shopify/build.gradle deleted file mode 100644 index b3cf87ff67c0..000000000000 --- a/airbyte-integrations/connectors/source-shopify/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_shopify' -} diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json index e18e70cb07f4..988348821484 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json @@ -3,10 +3,10 @@ "type": "STREAM", "stream": { "stream_state": { - "id": 99999999999999 + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "articles" + "name": "customers" } } }, @@ -14,10 +14,13 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-07-08T05:40:38-07:00" + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "metafield_articles" + "name": "orders" } } }, @@ -25,10 +28,10 @@ "type": "STREAM", "stream": { "stream_state": { - "id": 99999999999999 + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "blogs" + "name": "draft_orders" } } }, @@ -36,10 +39,13 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-07-08T05:40:38-07:00" + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "metafield_blogs" + "name": "products" } } }, @@ -47,10 +53,10 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-07-08T05:40:38-07:00" + "updatedAt": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "customers" + "name": "products_graph_ql" } } }, @@ -58,10 +64,10 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-07-08T05:40:38-07:00" + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "metafield_customers" + "name": "abandoned_checkouts" } } }, @@ -69,10 +75,10 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-07-08T05:40:38-07:00" + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "orders" + "name": "metafields" } } }, @@ -80,10 +86,10 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-07-08T05:40:38-07:00" + "id": 99999999999999 }, "stream_descriptor": { - "name": "draft_orders" + "name": "collects" } } }, @@ -91,10 +97,13 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-07-08T05:40:38-07:00" + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "products" + "name": "custom_collections" } } }, @@ -102,10 +111,16 @@ "type": "STREAM", "stream": { "stream_state": { - "updatedAt": "2025-07-08T05:40:38-07:00" + "orders": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "created_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "products_graph_ql" + "name": "order_refunds" } } }, @@ -113,10 +128,16 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-07-08T05:40:38-07:00" + "orders": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "id": 99999999999999 }, "stream_descriptor": { - "name": "abandoned_checkouts" + "name": "order_risks" } } }, @@ -124,10 +145,16 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-07-08T05:40:38-07:00" + "orders": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "created_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "metafield_draft_orders" + "name": "transactions" } } }, @@ -135,10 +162,10 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-07-08T05:40:38-07:00" + "created_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "metafield_orders" + "name": "transactions_graphql" } } }, @@ -146,10 +173,10 @@ "type": "STREAM", "stream": { "stream_state": { - "id": 99999999999999 + "processed_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "product_images" + "name": "tender_transactions" } } }, @@ -157,10 +184,13 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-07-08T05:40:38-07:00" + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "metafield_product_images" + "name": "pages" } } }, @@ -168,10 +198,13 @@ "type": "STREAM", "stream": { "stream_state": { - "id": 99999999999999 + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "product_variants" + "name": "price_rules" } } }, @@ -179,10 +212,16 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-07-08T05:40:38-07:00" + "price_rules": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "metafield_product_variants" + "name": "discount_codes" } } }, @@ -190,10 +229,16 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-07-08T05:40:38-07:00" + "products": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "metafield_products" + "name": "inventory_items" } } }, @@ -201,10 +246,11 @@ "type": "STREAM", "stream": { "stream_state": { - "id": 99999999999999 + "locations": {}, + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "collects" + "name": "inventory_levels" } } }, @@ -212,10 +258,16 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-07-08T05:40:38-07:00" + "orders": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "id": 99999999999999 }, "stream_descriptor": { - "name": "collections" + "name": "fulfillment_orders" } } }, @@ -223,21 +275,25 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-07-08T05:40:38-07:00" + "orders": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "metafield_collections" + "name": "fulfillments" } } }, { "type": "STREAM", "stream": { - "stream_state": { - "updated_at": "2025-07-08T05:40:38-07:00" - }, + "stream_state": {}, "stream_descriptor": { - "name": "smart_collections" + "name": "balance_transactions" } } }, @@ -245,10 +301,13 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-07-08T05:40:38-07:00" + "id": 99999999999999, + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } }, "stream_descriptor": { - "name": "metafield_smart_collections" + "name": "articles" } } }, @@ -256,10 +315,14 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-07-08T05:40:38-07:00" + "articles": { + "id": 99999999999999, + "deleted": {} + }, + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "custom_collections" + "name": "metafield_articles" } } }, @@ -267,10 +330,13 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-07-08T05:40:38-07:00" + "id": 99999999999999, + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } }, "stream_descriptor": { - "name": "metafield_custom_collections" + "name": "blogs" } } }, @@ -278,13 +344,13 @@ "type": "STREAM", "stream": { "stream_state": { - "created_at": "2025-03-03T03:47:46-08:00", - "orders": { - "updated_at": "2025-03-03T03:47:46-08:00" + "updated_at": "2027-07-11T13:07:45-07:00", + "blogs": { + "id": 99999999999999 } }, "stream_descriptor": { - "name": "order_refunds" + "name": "metafield_blogs" } } }, @@ -292,13 +358,13 @@ "type": "STREAM", "stream": { "stream_state": { - "id": 99999999999999, - "orders": { - "updated_at": "2025-02-22T00:37:28-08:00" - } + "customers": { + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "order_risks" + "name": "metafield_customers" } } }, @@ -306,10 +372,16 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-07-08T05:40:38-07:00" + "orders": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "metafield_locations" + "name": "metafield_orders" } } }, @@ -317,13 +389,13 @@ "type": "STREAM", "stream": { "stream_state": { - "created_at": "2025-03-03T03:47:45-08:00", - "orders": { - "updated_at": "2025-03-03T03:47:46-08:00" - } + "draft_orders": { + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "transactions" + "name": "metafield_draft_orders" } } }, @@ -331,10 +403,16 @@ "type": "STREAM", "stream": { "stream_state": { - "processed_at": "2025-03-03T03:47:45-08:00" + "products": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "tender_transactions" + "name": "metafield_products" } } }, @@ -342,10 +420,16 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-07-08T05:24:10-07:00" + "id": 99999999999999, + "products": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + } }, "stream_descriptor": { - "name": "pages" + "name": "product_images" } } }, @@ -353,10 +437,16 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-07-08T05:24:10-07:00" + "products": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "metafield_pages" + "name": "metafield_product_images" } } }, @@ -364,10 +454,17 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-07-08T05:24:10-07:00" + "id": 99999999999999, + "products": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "price_rules" + "name": "product_variants" } } }, @@ -375,13 +472,16 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-09-10T06:48:10-07:00", - "price_rules": { - "updated_at": "2025-09-10T06:48:10-07:00" - } + "products": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "discount_codes" + "name": "metafield_product_variants" } } }, @@ -389,13 +489,13 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-02-22T00:40:26-08:00", - "products": { - "updated_at": "2025-08-18T02:39:48-07:00" - } + "collects": { + "id": 99999999999999 + }, + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "inventory_items" + "name": "collections" } } }, @@ -403,11 +503,13 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-03-03T03:47:51-08:00", - "locations": {} + "collects": { + "id": 99999999999999 + }, + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "inventory_levels" + "name": "metafield_collections" } } }, @@ -415,13 +517,10 @@ "type": "STREAM", "stream": { "stream_state": { - "id": 99999999999999, - "orders": { - "updated_at": "2025-03-03T03:47:46-08:00" - } + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "fulfillment_orders" + "name": "smart_collections" } } }, @@ -429,13 +528,13 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-02-27T23:49:13-08:00", - "orders": { - "updated_at": "2025-03-03T03:47:46-08:00" - } + "smart_collections": { + "updated_at": "2027-07-11T13:07:45-07:00" + }, + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "fulfillments" + "name": "metafield_smart_collections" } } }, @@ -443,10 +542,16 @@ "type": "STREAM", "stream": { "stream_state": { - "id": 99999999999999 + "pages": { + "updated_at": "2027-07-11T13:07:45-07:00", + "deleted": { + "deleted_at": "2027-07-11T13:07:45-07:00" + } + }, + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "balance_transactions" + "name": "metafield_pages" } } }, @@ -454,7 +559,7 @@ "type": "STREAM", "stream": { "stream_state": { - "updated_at": "2025-07-08T05:24:10-07:00" + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { "name": "metafield_shops" @@ -465,22 +570,27 @@ "type": "STREAM", "stream": { "stream_state": { - "id": 99999999999999, - "customers": { - "updated_at": "2025-02-22T00:37:28-08:00" - } + "locations": {}, + "updated_at": "2027-07-11T13:07:45-07:00" }, "stream_descriptor": { - "name": "customer_address" + "name": "metafield_locations" } } }, { "type": "STREAM", "stream": { - "stream_state": { - "id": 99999999999999 - }, + "stream_state": {}, + "stream_descriptor": { + "name": "disputes" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": {}, "stream_descriptor": { "name": "customer_saved_search" } @@ -490,10 +600,13 @@ "type": "STREAM", "stream": { "stream_state": { - "id": 99999999999999 + "id": 99999999999999, + "customers": { + "updated_at": "2027-07-11T13:07:45-07:00" + } }, "stream_descriptor": { - "name": "disputes" + "name": "customer_address" } } } diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json index 05f4f41ddf4a..ccf6248669bf 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json @@ -336,6 +336,18 @@ "cursor_field": ["created_at"], "destination_sync_mode": "append" }, + { + "stream": { + "name": "transactions_graphql", + "json_schema": {}, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["created_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["created_at"], + "destination_sync_mode": "append" + }, { "stream": { "name": "tender_transactions", diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-shopify/integration_tests/expected_records.jsonl index bc5935226ca0..9ac8feea622b 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/expected_records.jsonl @@ -1,75 +1,95 @@ -{"stream": "articles", "data": {"id": 558137508029, "title": "My new Article title", "created_at": "2022-10-07T16:09:02-07:00", "body_html": "

      I like articles

      \n

      Yea, I like posting them through REST.

      ", "blog_id": 80417685693, "author": "John Smith", "user_id": null, "published_at": "2011-03-24T08:45:47-07:00", "updated_at": "2023-04-14T03:18:26-07:00", "summary_html": null, "template_suffix": null, "handle": "my-new-article-title", "tags": "Has Been Tagged, This Post", "admin_graphql_api_id": "gid://shopify/OnlineStoreArticle/558137508029", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315555203} -{"stream": "articles", "data": {"id": 558627979453, "title": "Test Blog Post", "created_at": "2023-04-14T03:19:02-07:00", "body_html": "Test Blog Post 1", "blog_id": 80417685693, "author": "Airbyte Airbyte", "user_id": "74861019325", "published_at": null, "updated_at": "2023-04-14T03:19:18-07:00", "summary_html": "", "template_suffix": "", "handle": "test-blog-post", "tags": "Has Been Tagged", "admin_graphql_api_id": "gid://shopify/OnlineStoreArticle/558627979453", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315555204} -{"stream": "blogs", "data": {"id": 80417685693, "handle": "news", "title": "News", "updated_at": "2023-04-14T03:20:20-07:00", "commentable": "no", "feedburner": null, "feedburner_location": null, "created_at": "2021-06-22T18:00:25-07:00", "template_suffix": null, "tags": "Has Been Tagged, This Post", "admin_graphql_api_id": "gid://shopify/OnlineStoreBlog/80417685693", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315557505} -{"stream": "collections", "data": {"id": 270889287869, "handle": "frontpage", "title": "Home page", "updated_at": "2023-04-24T11:05:13-07:00", "body_html": "updated_mon_24.04.2023", "published_at": "2021-06-22T18:00:25-07:00", "sort_order": "best-selling", "template_suffix": "", "products_count": 2, "collection_type": "custom", "published_scope": "web", "admin_graphql_api_id": "gid://shopify/Collection/270889287869", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315559587} -{"stream": "collections", "data": {"id": 270889287869, "handle": "frontpage", "title": "Home page", "updated_at": "2023-04-24T11:05:13-07:00", "body_html": "updated_mon_24.04.2023", "published_at": "2021-06-22T18:00:25-07:00", "sort_order": "best-selling", "template_suffix": "", "products_count": 2, "collection_type": "custom", "published_scope": "web", "admin_graphql_api_id": "gid://shopify/Collection/270889287869", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315560044} -{"stream": "collects", "data": {"id": 29229083197629, "collection_id": 270889287869, "product_id": 6796217811133, "created_at": "2021-06-22T18:09:26-07:00", "updated_at": "2021-06-22T18:09:57-07:00", "position": 1, "sort_value": "0000000001", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315561131} -{"stream": "collects", "data": {"id": 29427031703741, "collection_id": 270889287869, "product_id": 6796220989629, "created_at": "2021-07-19T07:01:36-07:00", "updated_at": "2022-03-06T14:12:21-08:00", "position": 2, "sort_value": "0000000002", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315561132} -{"stream": "custom_collections", "data": {"id": 270889287869, "handle": "frontpage", "title": "Home page", "updated_at": "2023-04-24T11:05:13-07:00", "body_html": "updated_mon_24.04.2023", "published_at": "2021-06-22T18:00:25-07:00", "sort_order": "best-selling", "template_suffix": "", "published_scope": "web", "admin_graphql_api_id": "gid://shopify/Collection/270889287869", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315562170} -{"stream": "customers", "data": {"id": 6569096478909, "email": "test@test.com", "accepts_marketing": true, "created_at": "2023-04-13T02:30:04-07:00", "updated_at": "2023-04-24T06:53:48-07:00", "first_name": "New Test", "last_name": "Customer", "orders_count": 0, "state": "disabled", "total_spent": 0.0, "last_order_id": null, "note": "updated_mon_24.04.2023", "verified_email": true, "multipass_identifier": null, "tax_exempt": false, "tags": "", "last_order_name": null, "currency": "USD", "phone": "+380639379992", "addresses": [{"id": 8092523135165, "customer_id": 6569096478909, "first_name": "New Test", "last_name": "Customer", "company": "Test Company", "address1": "My Best Accent", "address2": "", "city": "Fair Lawn", "province": "New Jersey", "country": "United States", "zip": "07410", "phone": "", "name": "New Test Customer", "province_code": "NJ", "country_code": "US", "country_name": "United States", "default": true}], "accepts_marketing_updated_at": "2023-04-13T02:30:04-07:00", "marketing_opt_in_level": "single_opt_in", "tax_exemptions": "[]", "email_marketing_consent": {"state": "subscribed", "opt_in_level": "single_opt_in", "consent_updated_at": "2023-04-13T02:30:04-07:00"}, "sms_marketing_consent": {"state": "not_subscribed", "opt_in_level": "single_opt_in", "consent_updated_at": null, "consent_collected_from": "SHOPIFY"}, "admin_graphql_api_id": "gid://shopify/Customer/6569096478909", "default_address": {"id": 8092523135165, "customer_id": 6569096478909, "first_name": "New Test", "last_name": "Customer", "company": "Test Company", "address1": "My Best Accent", "address2": "", "city": "Fair Lawn", "province": "New Jersey", "country": "United States", "zip": "07410", "phone": "", "name": "New Test Customer", "province_code": "NJ", "country_code": "US", "country_name": "United States", "default": true}, "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315563441} -{"stream": "customers", "data": {"id": 6676027932861, "email": "marcos@airbyte.io", "accepts_marketing": false, "created_at": "2023-07-11T13:07:45-07:00", "updated_at": "2023-07-11T13:07:45-07:00", "first_name": "MArcos", "last_name": "Millnitz", "orders_count": 0, "state": "disabled", "total_spent": 0.0, "last_order_id": null, "note": null, "verified_email": true, "multipass_identifier": null, "tax_exempt": false, "tags": "", "last_order_name": null, "currency": "USD", "phone": null, "addresses": [{"id": 8212915650749, "customer_id": 6676027932861, "first_name": "MArcos", "last_name": "Millnitz", "company": null, "address1": null, "address2": null, "city": null, "province": null, "country": null, "zip": null, "phone": null, "name": "MArcos Millnitz", "province_code": null, "country_code": null, "country_name": null, "default": true}], "accepts_marketing_updated_at": "2023-07-11T13:07:45-07:00", "marketing_opt_in_level": null, "tax_exemptions": "[]", "email_marketing_consent": {"state": "not_subscribed", "opt_in_level": "single_opt_in", "consent_updated_at": null}, "sms_marketing_consent": null, "admin_graphql_api_id": "gid://shopify/Customer/6676027932861", "default_address": {"id": 8212915650749, "customer_id": 6676027932861, "first_name": "MArcos", "last_name": "Millnitz", "company": null, "address1": null, "address2": null, "city": null, "province": null, "country": null, "zip": null, "phone": null, "name": "MArcos Millnitz", "province_code": null, "country_code": null, "country_name": null, "default": true}, "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315563443} -{"stream": "discount_codes", "data": {"id": 11539415990461, "price_rule_id": 945000284349, "code": "updated_mon_24.04.2023", "usage_count": 0, "created_at": "2021-07-07T07:23:11-07:00", "updated_at": "2023-04-24T05:52:22-07:00", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315565794} -{"stream": "draft_orders", "data": {"id": 929019691197, "note": "updated_mon_24.04.2023", "email": null, "taxes_included": true, "currency": "USD", "invoice_sent_at": null, "created_at": "2022-02-22T03:23:19-08:00", "updated_at": "2023-04-24T07:18:06-07:00", "tax_exempt": false, "completed_at": null, "name": "#D21", "status": "open", "line_items": [{"id": 58117295538365, "variant_id": 40090585923773, "product_id": 6796220989629, "title": "4 Ounce Soy Candle", "variant_title": "Metal", "sku": "", "vendor": "Hartmann Group", "quantity": 2, "requires_shipping": true, "taxable": true, "gift_card": false, "fulfillment_service": "manual", "grams": 112, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 6.33}], "applied_discount": null, "name": "4 Ounce Soy Candle - Metal", "properties": [], "custom": false, "price": 19.0, "admin_graphql_api_id": "gid://shopify/DraftOrderLineItem/58117295538365"}, {"id": 58117295571133, "variant_id": null, "product_id": null, "title": "Test Item", "variant_title": null, "sku": null, "vendor": null, "quantity": 1, "requires_shipping": true, "taxable": true, "gift_card": false, "fulfillment_service": "manual", "grams": 1000, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 0.17}], "applied_discount": null, "name": "Test Item", "properties": [], "custom": true, "price": 1.0, "admin_graphql_api_id": "gid://shopify/DraftOrderLineItem/58117295571133"}], "shipping_address": null, "billing_address": null, "invoice_url": "https://airbyte-integration-test.myshopify.com/58033176765/invoices/12893992cc01fc67935ab014fcf9300f", "applied_discount": null, "order_id": null, "shipping_line": {"title": "Test Shipping Fee", "custom": true, "handle": null, "price": 3.0}, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 6.33}, {"rate": 0.2, "title": "PDV", "price": 0.17}], "tags": "", "note_attributes": [], "total_price": "42.00", "subtotal_price": "39.00", "total_tax": "6.50", "payment_terms": null, "admin_graphql_api_id": "gid://shopify/DraftOrder/929019691197", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315566986} -{"stream": "draft_orders", "data": {"id": 988639920317, "note": null, "email": null, "taxes_included": true, "currency": "USD", "invoice_sent_at": null, "created_at": "2023-04-24T11:00:01-07:00", "updated_at": "2023-04-24T11:00:09-07:00", "tax_exempt": false, "completed_at": "2023-04-24T11:00:09-07:00", "name": "#D29", "status": "completed", "line_items": [{"id": 58121808019645, "variant_id": 41561961824445, "product_id": 6796220989629, "title": "4 Ounce Soy Candle", "variant_title": "Test Variant 2", "sku": "", "vendor": "Hartmann Group", "quantity": 1, "requires_shipping": true, "taxable": true, "gift_card": false, "fulfillment_service": "manual", "grams": 112, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 3.17}], "applied_discount": null, "name": "4 Ounce Soy Candle - Test Variant 2", "properties": [], "custom": false, "price": 19.0, "admin_graphql_api_id": "gid://shopify/DraftOrderLineItem/58121808019645"}], "shipping_address": null, "billing_address": null, "invoice_url": "https://airbyte-integration-test.myshopify.com/58033176765/invoices/95271a5eeb083c831f76a98fa3712f89", "applied_discount": null, "order_id": 5033391718589, "shipping_line": null, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 3.17}], "tags": "", "note_attributes": [], "total_price": "19.00", "subtotal_price": "19.00", "total_tax": "3.17", "payment_terms": null, "admin_graphql_api_id": "gid://shopify/DraftOrder/988639920317", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315566988} -{"stream": "draft_orders", "data": {"id": 997801689277, "note": null, "email": null, "taxes_included": true, "currency": "USD", "invoice_sent_at": null, "created_at": "2023-07-11T12:57:53-07:00", "updated_at": "2023-07-11T12:57:55-07:00", "tax_exempt": false, "completed_at": null, "name": "#D30", "status": "open", "line_items": [{"id": 58159126905021, "variant_id": 40090585923773, "product_id": 6796220989629, "title": "4 Ounce Soy Candle", "variant_title": "Metal", "sku": "", "vendor": "Hartmann Group", "quantity": 1, "requires_shipping": true, "taxable": true, "gift_card": false, "fulfillment_service": "manual", "grams": 112, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 3.17}], "applied_discount": null, "name": "4 Ounce Soy Candle - Metal", "properties": [], "custom": false, "price": 19.0, "admin_graphql_api_id": "gid://shopify/DraftOrderLineItem/58159126905021"}], "shipping_address": null, "billing_address": null, "invoice_url": "https://airbyte-integration-test.myshopify.com/58033176765/invoices/a98bc7e113733d6faa36c198cf6c7c1a", "applied_discount": null, "order_id": null, "shipping_line": null, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 3.17}], "tags": "", "note_attributes": [], "total_price": "19.00", "subtotal_price": "19.00", "total_tax": "3.17", "payment_terms": null, "admin_graphql_api_id": "gid://shopify/DraftOrder/997801689277", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315566988} -{"stream": "fulfillment_orders", "data": {"id": 5558588309693, "shop_id": 58033176765, "order_id": 4554821468349, "assigned_location_id": 63590301885, "request_status": "unsubmitted", "status": "closed", "supported_actions": [], "destination": null, "line_items": [{"id": 11564232016061, "shop_id": 58033176765, "fulfillment_order_id": 5558588309693, "quantity": 1, "line_item_id": 11406125564093, "inventory_item_id": 42185212592317, "fulfillable_quantity": 0, "variant_id": 40090597884093}], "fulfill_at": "2022-06-15T05:00:00-07:00", "fulfill_by": null, "international_duties": null, "fulfillment_holds": [], "delivery_method": {"id": 119732437181, "method_type": "none", "min_delivery_date_time": null, "max_delivery_date_time": null}, "assigned_location": {"address1": "Heroiv UPA 72", "address2": null, "city": "Lviv", "country_code": "UA", "location_id": 63590301885, "name": "Heroiv UPA 72", "phone": "", "province": null, "zip": "30100"}, "merchant_requests": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315569283} -{"stream": "fulfillment_orders", "data": {"id": 5138290671805, "shop_id": 58033176765, "order_id": 4147980107965, "assigned_location_id": 63590301885, "request_status": "unsubmitted", "status": "closed", "supported_actions": [], "destination": {"id": 5183946588349, "address1": "San Francisco", "address2": "10", "city": "San Francisco", "company": "Umbrella LLC", "country": "United States", "email": "airbyte@airbyte.com", "first_name": "John", "last_name": "Doe", "phone": "", "province": "California", "zip": "91326"}, "line_items": [{"id": 10713758531773, "shop_id": 58033176765, "fulfillment_order_id": 5138290671805, "quantity": 1, "line_item_id": 10576771317949, "inventory_item_id": 42185195290813, "fulfillable_quantity": 0, "variant_id": 40090580615357}], "fulfill_at": null, "fulfill_by": null, "international_duties": "{'incoterm': None}", "fulfillment_holds": [], "delivery_method": null, "assigned_location": {"address1": "Heroiv UPA 72", "address2": null, "city": "Lviv", "country_code": "UA", "location_id": 63590301885, "name": "Heroiv UPA 72", "phone": "", "province": null, "zip": "30100"}, "merchant_requests": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315569894} -{"stream": "fulfillment_orders", "data": {"id": 4919375659197, "shop_id": 58033176765, "order_id": 3935377129661, "assigned_location_id": 63590301885, "request_status": "unsubmitted", "status": "closed", "supported_actions": [], "destination": null, "line_items": [{"id": 10251692081341, "shop_id": 58033176765, "fulfillment_order_id": 4919375659197, "quantity": 1, "line_item_id": 10130216452285, "inventory_item_id": 42185218719933, "fulfillable_quantity": 1, "variant_id": 40090604011709}], "fulfill_at": null, "fulfill_by": null, "international_duties": null, "fulfillment_holds": [], "delivery_method": null, "assigned_location": {"address1": "Heroiv UPA 72", "address2": null, "city": "Lviv", "country_code": "UA", "location_id": 63590301885, "name": "Heroiv UPA 72", "phone": "", "province": null, "zip": "30100"}, "merchant_requests": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315570459} -{"stream": "fulfillments", "data": {"id": 4075788501181, "order_id": 4554821468349, "status": "success", "created_at": "2022-06-15T05:16:55-07:00", "service": "manual", "updated_at": "2022-06-15T05:16:55-07:00", "tracking_company": null, "shipment_status": null, "location_id": 63590301885, "origin_address": null, "line_items": [{"id": 11406125564093, "variant_id": 40090597884093, "title": "All Black Sneaker Right Foot", "quantity": 1, "sku": "", "variant_title": "ivory", "vendor": "Becker - Moore", "fulfillment_service": "manual", "product_id": 6796226560189, "requires_shipping": false, "taxable": true, "gift_card": false, "name": "All Black Sneaker Right Foot - ivory", "variant_inventory_management": "shopify", "properties": [], "product_exists": true, "fulfillable_quantity": 0, "grams": 0, "price": "59.00", "total_discount": "0.00", "fulfillment_status": "fulfilled", "price_set": {"shop_money": {"amount": "59.00", "currency_code": "USD"}, "presentment_money": {"amount": "59.00", "currency_code": "USD"}}, "total_discount_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "discount_allocations": [{"amount": "1.77", "discount_application_index": 0, "amount_set": {"shop_money": {"amount": "1.77", "currency_code": "USD"}, "presentment_money": {"amount": "1.77", "currency_code": "USD"}}}], "origin_location": {"id": 3007664259261, "country_code": "UA", "province_code": "", "name": "airbyte integration test", "address1": "Heroiv UPA 72", "address2": "", "city": "Lviv", "zip": "30100"}, "admin_graphql_api_id": "gid://shopify/LineItem/11406125564093", "duties": [], "tax_lines": [], "fulfillment_line_item_id": 9633709097149}], "tracking_number": null, "tracking_numbers": [], "tracking_url": null, "tracking_urls": [], "receipt": {}, "name": "#1136.1", "admin_graphql_api_id": "gid://shopify/Fulfillment/4075788501181", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315573677} -{"stream": "fulfillments", "data": {"id": 3693416710333, "order_id": 4147980107965, "status": "success", "created_at": "2021-09-19T09:08:23-07:00", "service": "manual", "updated_at": "2022-02-22T00:35:47-08:00", "tracking_company": "Amazon Logistics US", "shipment_status": null, "location_id": 63590301885, "origin_address": null, "line_items": [{"id": 10576771317949, "variant_id": 40090580615357, "title": "Red & Silver Fishing Lure", "quantity": 1, "sku": "", "variant_title": "Plastic", "vendor": "Harris - Hamill", "fulfillment_service": "manual", "product_id": 6796218302653, "requires_shipping": true, "taxable": true, "gift_card": false, "name": "Red & Silver Fishing Lure - Plastic", "variant_inventory_management": "shopify", "properties": [], "product_exists": true, "fulfillable_quantity": 0, "grams": 285, "price": "27.00", "total_discount": "0.00", "fulfillment_status": "fulfilled", "price_set": {"shop_money": {"amount": "27.00", "currency_code": "USD"}, "presentment_money": {"amount": "27.00", "currency_code": "USD"}}, "total_discount_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "discount_allocations": [], "admin_graphql_api_id": "gid://shopify/LineItem/10576771317949", "duties": [], "tax_lines": [], "fulfillment_line_item_id": 8852381401277}], "tracking_number": "123456", "tracking_numbers": ["123456"], "tracking_url": "https://track.amazon.com/tracking/123456", "tracking_urls": ["https://track.amazon.com/tracking/123456"], "receipt": {}, "name": "#1121.1", "admin_graphql_api_id": "gid://shopify/Fulfillment/3693416710333", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315574294} -{"stream": "fulfillments", "data": {"id": 4465911431357, "order_id": 5033391718589, "status": "success", "created_at": "2023-04-24T11:00:09-07:00", "service": "manual", "updated_at": "2023-04-24T11:00:09-07:00", "tracking_company": null, "shipment_status": null, "location_id": 63590301885, "origin_address": null, "line_items": [{"id": 12247585521853, "variant_id": 41561961824445, "title": "4 Ounce Soy Candle", "quantity": 1, "sku": "", "variant_title": "Test Variant 2", "vendor": "Hartmann Group", "fulfillment_service": "manual", "product_id": 6796220989629, "requires_shipping": true, "taxable": true, "gift_card": false, "name": "4 Ounce Soy Candle - Test Variant 2", "variant_inventory_management": "shopify", "properties": [], "product_exists": true, "fulfillable_quantity": 0, "grams": 112, "price": "19.00", "total_discount": "0.00", "fulfillment_status": "fulfilled", "price_set": {"shop_money": {"amount": "19.00", "currency_code": "USD"}, "presentment_money": {"amount": "19.00", "currency_code": "USD"}}, "total_discount_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "discount_allocations": [], "origin_location": {"id": 3000230707389, "country_code": "UA", "province_code": "", "name": "Heroiv UPA 72", "address1": "Heroiv UPA 72", "address2": "", "city": "Lviv", "zip": "30100"}, "admin_graphql_api_id": "gid://shopify/LineItem/12247585521853", "duties": [], "tax_lines": [{"price": 3.17, "rate": 0.2, "title": "PDV", "price_set": {"shop_money": {"amount": "3.17", "currency_code": "USD"}, "presentment_money": {"amount": "3.17", "currency_code": "USD"}}, "channel_liable": null}], "fulfillment_line_item_id": 10383179514045}], "tracking_number": null, "tracking_numbers": [], "tracking_url": null, "tracking_urls": [], "receipt": {}, "name": "#1145.1", "admin_graphql_api_id": "gid://shopify/Fulfillment/4465911431357", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315575461} -{"stream": "inventory_items", "data": {"id": 42185200631997, "sku": "", "created_at": "2021-06-22T18:09:47-07:00", "updated_at": "2022-02-22T00:40:19-08:00", "requires_shipping": true, "cost": 19.0, "country_code_of_origin": null, "province_code_of_origin": null, "harmonized_system_code": null, "tracked": true, "country_harmonized_system_codes": [], "admin_graphql_api_id": "gid://shopify/InventoryItem/42185200631997", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315577320} -{"stream": "inventory_items", "data": {"id": 43653682495677, "sku": "", "created_at": "2022-03-06T14:09:20-08:00", "updated_at": "2022-03-06T14:09:20-08:00", "requires_shipping": true, "cost": 19.0, "country_code_of_origin": null, "province_code_of_origin": null, "harmonized_system_code": null, "tracked": true, "country_harmonized_system_codes": [], "admin_graphql_api_id": "gid://shopify/InventoryItem/43653682495677", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315577322} -{"stream": "inventory_items", "data": {"id": 43653688524989, "sku": "", "created_at": "2022-03-06T14:12:20-08:00", "updated_at": "2022-03-06T14:12:20-08:00", "requires_shipping": true, "cost": 19.0, "country_code_of_origin": null, "province_code_of_origin": null, "harmonized_system_code": null, "tracked": true, "country_harmonized_system_codes": [], "admin_graphql_api_id": "gid://shopify/InventoryItem/43653688524989", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315577322} -{"stream": "inventory_levels", "data": {"inventory_item_id": 42185194635453, "location_id": 63590301885, "available": 49, "updated_at": "2022-02-27T23:44:30-08:00", "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=42185194635453", "shop_url": "airbyte-integration-test", "id": "63590301885|42185194635453"}, "emitted_at": 1690315579821} -{"stream": "inventory_levels", "data": {"inventory_item_id": 42185194668221, "location_id": 63590301885, "available": 12, "updated_at": "2021-06-22T18:09:27-07:00", "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=42185194668221", "shop_url": "airbyte-integration-test", "id": "63590301885|42185194668221"}, "emitted_at": 1690315579822} -{"stream": "inventory_levels", "data": {"inventory_item_id": 42185194700989, "location_id": 63590301885, "available": 3, "updated_at": "2021-06-22T18:09:27-07:00", "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=42185194700989", "shop_url": "airbyte-integration-test", "id": "63590301885|42185194700989"}, "emitted_at": 1690315579822} -{"stream": "locations", "data": {"id": 63590301885, "name": "Heroiv UPA 72", "address1": "Heroiv UPA 72", "address2": null, "city": "Lviv", "zip": "30100", "province": null, "country": "UA", "phone": "", "created_at": "2021-06-22T18:00:29-07:00", "updated_at": "2023-02-25T16:20:00-08:00", "country_code": "UA", "country_name": "Ukraine", "province_code": null, "legacy": false, "active": true, "admin_graphql_api_id": "gid://shopify/Location/63590301885", "localized_country_name": "Ukraine", "localized_province_name": null, "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315582838} -{"stream": "metafield_articles", "data": {"id": 21519818162365, "namespace": "global", "key": "new", "value": "newvalue", "description": null, "owner_id": 558137508029, "created_at": "2022-10-07T16:09:02-07:00", "updated_at": "2022-10-07T16:09:02-07:00", "owner_resource": "article", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/21519818162365", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315584977} -{"stream": "metafield_articles", "data": {"id": 22365709992125, "namespace": "custom", "key": "test_blog_post_metafield", "value": "Test Article Metafield", "description": null, "owner_id": 558137508029, "created_at": "2023-04-14T03:18:26-07:00", "updated_at": "2023-04-14T03:18:26-07:00", "owner_resource": "article", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365709992125", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315584978} -{"stream": "metafield_articles", "data": {"id": 22365710352573, "namespace": "custom", "key": "test_blog_post_metafield", "value": "Test Blog Post Metafiled", "description": null, "owner_id": 558627979453, "created_at": "2023-04-14T03:19:18-07:00", "updated_at": "2023-04-14T03:19:18-07:00", "owner_resource": "article", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365710352573", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315585483} -{"stream": "metafield_blogs", "data": {"id": 21519428255933, "namespace": "some_fields", "key": "sponsor", "value": "Shopify", "description": null, "owner_id": 80417685693, "created_at": "2022-10-07T06:05:23-07:00", "updated_at": "2022-10-07T06:05:23-07:00", "owner_resource": "blog", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/21519428255933", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315587865} -{"stream": "metafield_blogs", "data": {"id": 22365710745789, "namespace": "custom", "key": "test_blog_metafield", "value": "Test Blog Metafield", "description": null, "owner_id": 80417685693, "created_at": "2023-04-14T03:20:20-07:00", "updated_at": "2023-04-14T03:20:20-07:00", "owner_resource": "blog", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365710745789", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315587866} -{"stream": "metafield_collections", "data": {"id": 21520343367869, "namespace": "my_fields", "key": "discount", "value": "25%", "description": null, "owner_id": 270889287869, "created_at": "2022-10-08T04:44:51-07:00", "updated_at": "2022-10-08T04:44:51-07:00", "owner_resource": "collection", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/21520343367869", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315590398} -{"stream": "metafield_collections", "data": {"id": 22365707174077, "namespace": "custom", "key": "test_collection_metafield", "value": "Test Collection Metafield", "description": null, "owner_id": 270889287869, "created_at": "2023-04-14T03:15:30-07:00", "updated_at": "2023-04-14T03:15:30-07:00", "owner_resource": "collection", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365707174077", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315590400} -{"stream": "metafield_collections", "data": {"id": 21520343367869, "namespace": "my_fields", "key": "discount", "value": "25%", "description": null, "owner_id": 270889287869, "created_at": "2022-10-08T04:44:51-07:00", "updated_at": "2022-10-08T04:44:51-07:00", "owner_resource": "collection", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/21520343367869", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315590879} -{"stream": "metafield_customers", "data": {"id": 22346893361341, "namespace": "custom", "key": "test_definition_list_1", "value": "Teste\n", "description": null, "owner_id": 6569096478909, "created_at": "2023-04-13T04:50:10-07:00", "updated_at": "2023-04-13T04:50:10-07:00", "owner_resource": "customer", "type": "multi_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22346893361341", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315593181} -{"stream": "metafield_customers", "data": {"id": 22346893394109, "namespace": "custom", "key": "test_definition", "value": "Taster", "description": null, "owner_id": 6569096478909, "created_at": "2023-04-13T04:50:10-07:00", "updated_at": "2023-04-13T04:50:10-07:00", "owner_resource": "customer", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22346893394109", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315593182} -{"stream": "metafield_draft_orders", "data": {"id": 22532787175613, "namespace": "new_metafield", "key": "new_metafield", "value": "updated_mon_24.04.2023", "description": null, "owner_id": 929019691197, "created_at": "2023-04-24T07:18:06-07:00", "updated_at": "2023-04-24T07:18:06-07:00", "owner_resource": "draft_order", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22532787175613", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315595976} -{"stream": "metafield_locations", "data": {"id": 21524407255229, "namespace": "inventory", "key": "warehouse_2", "value": "234", "description": null, "owner_id": 63590301885, "created_at": "2022-10-12T02:21:27-07:00", "updated_at": "2022-10-12T02:21:27-07:00", "owner_resource": "location", "type": "number_integer", "admin_graphql_api_id": "gid://shopify/Metafield/21524407255229", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315599010} -{"stream": "metafield_locations", "data": {"id": 21524407681213, "namespace": "inventory", "key": "warehouse_233", "value": "564", "description": null, "owner_id": 63590301885, "created_at": "2022-10-12T02:21:35-07:00", "updated_at": "2022-10-12T02:21:35-07:00", "owner_resource": "location", "type": "number_integer", "admin_graphql_api_id": "gid://shopify/Metafield/21524407681213", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315599012} -{"stream": "metafield_orders", "data": {"id": 22347287855293, "namespace": "my_fields", "key": "purchase_order", "value": "trtrtr", "description": null, "owner_id": 4147980107965, "created_at": "2023-04-13T05:09:08-07:00", "updated_at": "2023-04-13T05:09:08-07:00", "owner_resource": "order", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22347287855293", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315601837} -{"stream": "metafield_orders", "data": {"id": 22365749805245, "namespace": "my_fields", "key": "purchase_order", "value": "Test Draft Order Metafield", "description": null, "owner_id": 3935377129661, "created_at": "2023-04-14T03:52:40-07:00", "updated_at": "2023-04-14T03:52:40-07:00", "owner_resource": "order", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365749805245", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315602352} -{"stream": "metafield_pages", "data": {"id": 22534014828733, "namespace": "new_metafield", "key": "new_metafield", "value": "updated_mon_24.04.2023", "description": null, "owner_id": 83074252989, "created_at": "2023-04-24T11:08:41-07:00", "updated_at": "2023-04-24T11:08:41-07:00", "owner_resource": "page", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22534014828733", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315605006} -{"stream": "metafield_product_images", "data": {"id": 22533588451517, "namespace": "new_metafield", "key": "new_metafield", "value": "updated_mon_24.04.2023", "description": null, "owner_id": 29301297316029, "created_at": "2023-04-24T10:32:19-07:00", "updated_at": "2023-04-24T10:32:19-07:00", "owner_resource": "product_image", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22533588451517", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315606428} -{"stream": "metafield_products", "data": {"id": 22365706944701, "namespace": "custom", "key": "test_product_metafield", "value": "gid://shopify/Product/6796220989629", "description": null, "owner_id": 6796220989629, "created_at": "2023-04-14T03:15:07-07:00", "updated_at": "2023-04-14T03:15:07-07:00", "owner_resource": "product", "type": "product_reference", "admin_graphql_api_id": "gid://shopify/Metafield/22365706944701", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315607933} -{"stream": "metafield_products", "data": {"id": 22365762486461, "namespace": "custom", "key": "product_metafield_test_2", "value": "Test", "description": null, "owner_id": 6796220989629, "created_at": "2023-04-14T03:59:44-07:00", "updated_at": "2023-04-14T03:59:44-07:00", "owner_resource": "product", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365762486461", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315607934} -{"stream": "metafield_product_variants", "data": {"id": 22365715955901, "namespace": "custom", "key": "test_variant_metafield", "value": "Test Varia", "description": null, "owner_id": 41561961824445, "created_at": "2023-04-14T03:24:03-07:00", "updated_at": "2023-04-14T03:24:03-07:00", "owner_resource": "variant", "type": "multi_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365715955901", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315610544} -{"stream": "metafield_shops", "data": {"id": 22534020104381, "namespace": "new_metafield", "key": "new_metafield", "value": "updated_mon_24.04.2023", "description": null, "owner_id": 58033176765, "created_at": "2023-04-24T11:12:38-07:00", "updated_at": "2023-04-24T11:12:38-07:00", "owner_resource": "shop", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22534020104381", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315611668} -{"stream": "metafield_smart_collections", "data": {"id": 21525604106429, "namespace": "my_fields", "key": "discount", "value": "50%", "description": null, "owner_id": 273278566589, "created_at": "2022-10-12T13:36:55-07:00", "updated_at": "2022-10-12T13:36:55-07:00", "owner_resource": "collection", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/21525604106429", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315613924} -{"stream": "metafield_smart_collections", "data": {"id": 22366265573565, "namespace": "my_fields", "key": "new_key", "value": "51%", "description": null, "owner_id": 273278566589, "created_at": "2023-04-14T05:21:58-07:00", "updated_at": "2023-04-14T05:21:58-07:00", "owner_resource": "collection", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22366265573565", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315613926} -{"stream": "order_refunds", "data": {"id": 829538369725, "order_id": 3935377129661, "created_at": "2021-09-21T05:31:59-07:00", "note": "test refund", "user_id": 74861019325, "processed_at": "2021-09-21T05:31:59-07:00", "restock": true, "duties": "[]", "total_duties_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "admin_graphql_api_id": "gid://shopify/Refund/829538369725", "refund_line_items": [{"id": 332807864509, "quantity": 1, "line_item_id": 10130216452285, "location_id": 63590301885, "restock_type": "cancel", "subtotal": 102.0, "total_tax": 17.0, "subtotal_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "total_tax_set": {"shop_money": {"amount": 17.0, "currency_code": "USD"}, "presentment_money": {"amount": 17.0, "currency_code": "USD"}}, "line_item": {"id": 10130216452285, "variant_id": 40090604011709, "title": "8 Ounce Soy Candle", "quantity": 1, "sku": "", "variant_title": "Wooden", "vendor": "Bosco Inc", "fulfillment_service": "manual", "product_id": 6796229509309, "requires_shipping": true, "taxable": true, "gift_card": false, "name": "8 Ounce Soy Candle - Wooden", "variant_inventory_management": "shopify", "properties": [], "product_exists": true, "fulfillable_quantity": 0, "grams": 63, "price": 102.0, "total_discount": 0.0, "fulfillment_status": null, "price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "discount_allocations": [], "duties": [], "admin_graphql_api_id": "gid://shopify/LineItem/10130216452285", "tax_lines": [{"title": "PDV", "price": 17.0, "rate": 0.2, "channel_liable": false, "price_set": {"shop_money": {"amount": 17.0, "currency_code": "USD"}, "presentment_money": {"amount": 17.0, "currency_code": "USD"}}}]}}], "transactions": [{"id": 5189894406333, "order_id": 3935377129661, "kind": "refund", "gateway": "bogus", "status": "success", "message": "Bogus Gateway: Forced success", "created_at": "2021-09-21T05:31:58-07:00", "test": true, "authorization": null, "location_id": null, "user_id": 74861019325, "parent_id": 4933790040253, "processed_at": "2021-09-21T05:31:58-07:00", "device_id": null, "error_code": null, "source_name": "1830279", "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}, "receipt": {"paid_amount": "102.00"}, "amount": "102.00", "currency": "USD", "admin_graphql_api_id": "gid://shopify/OrderTransaction/5189894406333"}], "order_adjustments": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315616894} -{"stream": "order_refunds", "data": {"id": 845032358077, "order_id": 4147980107965, "created_at": "2022-03-07T02:09:04-08:00", "note": null, "user_id": 74861019325, "processed_at": "2022-03-07T02:09:04-08:00", "restock": true, "duties": "[]", "total_duties_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "admin_graphql_api_id": "gid://shopify/Refund/845032358077", "refund_line_items": [{"id": 352716947645, "quantity": 1, "line_item_id": 10576771317949, "location_id": 63590301885, "restock_type": "return", "subtotal": 27.0, "total_tax": 0.0, "subtotal_set": {"shop_money": {"amount": 27.0, "currency_code": "USD"}, "presentment_money": {"amount": 27.0, "currency_code": "USD"}}, "total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "line_item": {"id": 10576771317949, "variant_id": 40090580615357, "title": "Red & Silver Fishing Lure", "quantity": 1, "sku": "", "variant_title": "Plastic", "vendor": "Harris - Hamill", "fulfillment_service": "manual", "product_id": 6796218302653, "requires_shipping": true, "taxable": true, "gift_card": false, "name": "Red & Silver Fishing Lure - Plastic", "variant_inventory_management": "shopify", "properties": [], "product_exists": true, "fulfillable_quantity": 0, "grams": 285, "price": 27.0, "total_discount": 0.0, "fulfillment_status": "fulfilled", "price_set": {"shop_money": {"amount": 27.0, "currency_code": "USD"}, "presentment_money": {"amount": 27.0, "currency_code": "USD"}}, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "discount_allocations": [], "duties": [], "admin_graphql_api_id": "gid://shopify/LineItem/10576771317949", "tax_lines": []}}], "transactions": [], "order_adjustments": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315617504} -{"stream": "order_refunds", "data": {"id": 852809646269, "order_id": 4554821468349, "created_at": "2022-06-15T06:25:43-07:00", "note": null, "user_id": 74861019325, "processed_at": "2022-06-15T06:25:43-07:00", "restock": true, "duties": "[]", "total_duties_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "admin_graphql_api_id": "gid://shopify/Refund/852809646269", "refund_line_items": [{"id": 363131404477, "quantity": 1, "line_item_id": 11406125564093, "location_id": 63590301885, "restock_type": "return", "subtotal": 57.23, "total_tax": 0.0, "subtotal_set": {"shop_money": {"amount": 57.23, "currency_code": "USD"}, "presentment_money": {"amount": 57.23, "currency_code": "USD"}}, "total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "line_item": {"id": 11406125564093, "variant_id": 40090597884093, "title": "All Black Sneaker Right Foot", "quantity": 1, "sku": "", "variant_title": "ivory", "vendor": "Becker - Moore", "fulfillment_service": "manual", "product_id": 6796226560189, "requires_shipping": false, "taxable": true, "gift_card": false, "name": "All Black Sneaker Right Foot - ivory", "variant_inventory_management": "shopify", "properties": [], "product_exists": true, "fulfillable_quantity": 0, "grams": 0, "price": 59.0, "total_discount": 0.0, "fulfillment_status": "fulfilled", "price_set": {"shop_money": {"amount": 59.0, "currency_code": "USD"}, "presentment_money": {"amount": 59.0, "currency_code": "USD"}}, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "discount_allocations": [{"amount": 1.77, "discount_application_index": 0, "amount_set": {"shop_money": {"amount": 1.77, "currency_code": "USD"}, "presentment_money": {"amount": 1.77, "currency_code": "USD"}}}], "duties": [], "admin_graphql_api_id": "gid://shopify/LineItem/11406125564093", "tax_lines": []}}], "transactions": [{"id": 5721170968765, "order_id": 4554821468349, "kind": "refund", "gateway": "bogus", "status": "success", "message": "Bogus Gateway: Forced success", "created_at": "2022-06-15T06:25:42-07:00", "test": true, "authorization": null, "location_id": null, "user_id": null, "parent_id": 5721110872253, "processed_at": "2022-06-15T06:25:42-07:00", "device_id": null, "error_code": null, "source_name": "1830279", "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}, "receipt": {"paid_amount": "57.23"}, "amount": "57.23", "currency": "USD", "admin_graphql_api_id": "gid://shopify/OrderTransaction/5721170968765"}], "order_adjustments": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315618226} -{"stream": "order_risks", "data": {"id": 6446736474301, "order_id": 4147980107965, "checkout_id": null, "source": "External", "score": 1.0, "recommendation": "cancel", "display": true, "cause_cancel": true, "message": "This order came from an anonymous proxy", "merchant_message": "This order came from an anonymous proxy", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315621059} -{"stream": "orders", "data": {"id": 4554821468349, "admin_graphql_api_id": "gid://shopify/Order/4554821468349", "app_id": 580111, "browser_ip": "176.113.167.23", "buyer_accepts_marketing": false, "cancel_reason": null, "cancelled_at": null, "cart_token": null, "checkout_id": 25048437719229, "checkout_token": "cf5d16a0a0688905bd551c6dec591506", "client_details": {"accept_language": "en-US,en;q=0.9,uk;q=0.8", "browser_height": 754, "browser_ip": "176.113.167.23", "browser_width": 1519, "session_hash": null, "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36 Edg/101.0.1210.53"}, "closed_at": "2022-06-15T06:25:43-07:00", "confirmed": true, "contact_email": "integration-test@airbyte.io", "created_at": "2022-06-15T05:16:53-07:00", "currency": "USD", "current_subtotal_price": 0.0, "current_subtotal_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_discounts": 0.0, "current_total_discounts_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_duties_set": null, "current_total_price": 0.0, "current_total_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_tax": 0.0, "current_total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "customer_locale": "en", "device_id": null, "discount_codes": [], "email": "integration-test@airbyte.io", "estimated_taxes": false, "financial_status": "refunded", "fulfillment_status": "fulfilled", "gateway": "bogus", "landing_site": "/wallets/checkouts.json", "landing_site_ref": null, "location_id": null, "merchant_of_record_app_id": null, "name": "#1136", "note": "updated_mon_24.04.2023", "note_attributes": [], "number": 136, "order_number": 1136, "order_status_url": "https://airbyte-integration-test.myshopify.com/58033176765/orders/e4f98630ea44a884e33e700203ce2130/authenticate?key=edf087d6ae55a4541bf1375432f6a4b8", "original_total_duties_set": null, "payment_gateway_names": ["bogus"], "phone": null, "presentment_currency": "USD", "processed_at": "2022-06-15T05:16:53-07:00", "processing_method": "direct", "reference": null, "referring_site": "https://airbyte-integration-test.myshopify.com/products/all-black-sneaker-right-foot", "source_identifier": null, "source_name": "web", "source_url": null, "subtotal_price": 57.23, "subtotal_price_set": {"shop_money": {"amount": 57.23, "currency_code": "USD"}, "presentment_money": {"amount": 57.23, "currency_code": "USD"}}, "tags": "Refund", "tax_lines": [], "taxes_included": true, "test": true, "token": "e4f98630ea44a884e33e700203ce2130", "total_discounts": 1.77, "total_discounts_set": {"shop_money": {"amount": 1.77, "currency_code": "USD"}, "presentment_money": {"amount": 1.77, "currency_code": "USD"}}, "total_line_items_price": 59.0, "total_line_items_price_set": {"shop_money": {"amount": 59.0, "currency_code": "USD"}, "presentment_money": {"amount": 59.0, "currency_code": "USD"}}, "total_outstanding": 0.0, "total_price": 57.23, "total_price_set": {"shop_money": {"amount": 57.23, "currency_code": "USD"}, "presentment_money": {"amount": 57.23, "currency_code": "USD"}}, "total_shipping_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_tax": 0.0, "total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_tip_received": 0.0, "total_weight": 0, "updated_at": "2023-04-24T07:00:37-07:00", "user_id": null, "billing_address": {"first_name": "Iryna", "address1": "2261 Market Street", "phone": null, "city": "San Francisco", "zip": "94114", "province": "California", "country": "United States", "last_name": "Grankova", "address2": "4381", "company": null, "latitude": 37.7647751, "longitude": -122.4320369, "name": "Iryna Grankova", "country_code": "US", "province_code": "CA"}, "customer": {"id": 5362027233469, "email": "integration-test@airbyte.io", "accepts_marketing": false, "created_at": "2021-07-08T05:41:47-07:00", "updated_at": "2022-06-22T03:50:13-07:00", "first_name": "Airbyte", "last_name": "Team", "state": "disabled", "note": null, "verified_email": true, "multipass_identifier": null, "tax_exempt": false, "phone": null, "email_marketing_consent": {"state": "not_subscribed", "opt_in_level": "single_opt_in", "consent_updated_at": null}, "sms_marketing_consent": null, "tags": "", "currency": "USD", "accepts_marketing_updated_at": "2021-07-08T05:41:47-07:00", "marketing_opt_in_level": null, "tax_exemptions": [], "admin_graphql_api_id": "gid://shopify/Customer/5362027233469", "default_address": {"id": 7492260823229, "customer_id": 5362027233469, "first_name": "Airbyte", "last_name": "Team", "company": null, "address1": "2261 Market Street", "address2": "4381", "city": "San Francisco", "province": "California", "country": "United States", "zip": "94114", "phone": null, "name": "Airbyte Team", "province_code": "CA", "country_code": "US", "country_name": "United States", "default": true}}, "discount_applications": [{"target_type": "line_item", "type": "automatic", "value": "3.0", "value_type": "percentage", "allocation_method": "across", "target_selection": "all", "title": "eeeee"}], "fulfillments": [{"id": 4075788501181, "admin_graphql_api_id": "gid://shopify/Fulfillment/4075788501181", "created_at": "2022-06-15T05:16:55-07:00", "location_id": 63590301885, "name": "#1136.1", "order_id": 4554821468349, "origin_address": {}, "receipt": {}, "service": "manual", "shipment_status": null, "status": "success", "tracking_company": null, "tracking_number": null, "tracking_numbers": [], "tracking_url": null, "tracking_urls": [], "updated_at": "2022-06-15T05:16:55-07:00", "line_items": [{"id": 11406125564093, "admin_graphql_api_id": "gid://shopify/LineItem/11406125564093", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 0, "name": "All Black Sneaker Right Foot - ivory", "price": 59.0, "price_set": {"shop_money": {"amount": 59.0, "currency_code": "USD"}, "presentment_money": {"amount": 59.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796226560189, "properties": [], "quantity": 1, "requires_shipping": false, "sku": "", "taxable": true, "title": "All Black Sneaker Right Foot", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 40090597884093, "variant_inventory_management": "shopify", "variant_title": "ivory", "vendor": "Becker - Moore", "tax_lines": [], "duties": [], "discount_allocations": [{"amount": "1.77", "amount_set": {"shop_money": {"amount": "1.77", "currency_code": "USD"}, "presentment_money": {"amount": "1.77", "currency_code": "USD"}}, "discount_application_index": 0}]}]}], "line_items": [{"id": 11406125564093, "admin_graphql_api_id": "gid://shopify/LineItem/11406125564093", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 0, "name": "All Black Sneaker Right Foot - ivory", "price": 59.0, "price_set": {"shop_money": {"amount": 59.0, "currency_code": "USD"}, "presentment_money": {"amount": 59.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796226560189, "properties": [], "quantity": 1, "requires_shipping": false, "sku": "", "taxable": true, "title": "All Black Sneaker Right Foot", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 40090597884093, "variant_inventory_management": "shopify", "variant_title": "ivory", "vendor": "Becker - Moore", "tax_lines": [], "duties": [], "discount_allocations": [{"amount": "1.77", "amount_set": {"shop_money": {"amount": "1.77", "currency_code": "USD"}, "presentment_money": {"amount": "1.77", "currency_code": "USD"}}, "discount_application_index": 0}]}], "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}, "payment_terms": null, "refunds": [{"id": 852809646269, "admin_graphql_api_id": "gid://shopify/Refund/852809646269", "created_at": "2022-06-15T06:25:43-07:00", "note": null, "order_id": 4554821468349, "processed_at": "2022-06-15T06:25:43-07:00", "restock": true, "total_duties_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "user_id": 74861019325, "order_adjustments": [], "transactions": [{"id": 5721170968765, "admin_graphql_api_id": "gid://shopify/OrderTransaction/5721170968765", "amount": "57.23", "authorization": null, "created_at": "2022-06-15T06:25:42-07:00", "currency": "USD", "device_id": null, "error_code": null, "gateway": "bogus", "kind": "refund", "location_id": null, "message": "Bogus Gateway: Forced success", "order_id": 4554821468349, "parent_id": 5721110872253, "processed_at": "2022-06-15T06:25:42-07:00", "receipt": {"paid_amount": "57.23"}, "source_name": "1830279", "status": "success", "test": true, "user_id": null, "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}}], "refund_line_items": [{"id": 363131404477, "line_item_id": 11406125564093, "location_id": 63590301885, "quantity": 1, "restock_type": "return", "subtotal": 57.23, "subtotal_set": {"shop_money": {"amount": "57.23", "currency_code": "USD"}, "presentment_money": {"amount": "57.23", "currency_code": "USD"}}, "total_tax": 0.0, "total_tax_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "line_item": {"id": 11406125564093, "admin_graphql_api_id": "gid://shopify/LineItem/11406125564093", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 0, "name": "All Black Sneaker Right Foot - ivory", "price": "59.00", "price_set": {"shop_money": {"amount": "59.00", "currency_code": "USD"}, "presentment_money": {"amount": "59.00", "currency_code": "USD"}}, "product_exists": true, "product_id": 6796226560189, "properties": [], "quantity": 1, "requires_shipping": false, "sku": "", "taxable": true, "title": "All Black Sneaker Right Foot", "total_discount": "0.00", "total_discount_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "variant_id": 40090597884093, "variant_inventory_management": "shopify", "variant_title": "ivory", "vendor": "Becker - Moore", "tax_lines": [], "duties": [], "discount_allocations": [{"amount": "1.77", "amount_set": {"shop_money": {"amount": "1.77", "currency_code": "USD"}, "presentment_money": {"amount": "1.77", "currency_code": "USD"}}, "discount_application_index": 0}]}}], "duties": []}], "shipping_lines": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315623322} -{"stream": "orders", "data": {"id": 4147980107965, "admin_graphql_api_id": "gid://shopify/Order/4147980107965", "app_id": 5505221, "browser_ip": null, "buyer_accepts_marketing": false, "cancel_reason": null, "cancelled_at": null, "cart_token": null, "checkout_id": null, "checkout_token": null, "closed_at": null, "confirmed": true, "contact_email": "airbyte@airbyte.com", "created_at": "2021-09-19T09:08:23-07:00", "currency": "USD", "current_subtotal_price": 0.0, "current_subtotal_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_discounts": 0.0, "current_total_discounts_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_duties_set": null, "current_total_price": 0.0, "current_total_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_tax": 0.0, "current_total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "customer_locale": null, "device_id": null, "discount_codes": [], "email": "airbyte@airbyte.com", "estimated_taxes": false, "financial_status": "paid", "fulfillment_status": "fulfilled", "gateway": "", "landing_site": null, "landing_site_ref": null, "location_id": null, "merchant_of_record_app_id": null, "name": "#1121", "note": "updated_mon_24.04.2023", "note_attributes": [], "number": 121, "order_number": 1121, "order_status_url": "https://airbyte-integration-test.myshopify.com/58033176765/orders/6adf11e07ccb49b280ea4b9f53d64f12/authenticate?key=4cef2ff10ba4d18f31114df33933f81e", "original_total_duties_set": null, "payment_gateway_names": [], "phone": null, "presentment_currency": "USD", "processed_at": "2021-09-19T09:08:23-07:00", "processing_method": "", "reference": null, "referring_site": null, "source_identifier": null, "source_name": "5505221", "source_url": null, "subtotal_price": 27.0, "subtotal_price_set": {"shop_money": {"amount": 27.0, "currency_code": "USD"}, "presentment_money": {"amount": 27.0, "currency_code": "USD"}}, "tags": "", "tax_lines": [], "taxes_included": false, "test": false, "token": "6adf11e07ccb49b280ea4b9f53d64f12", "total_discounts": 0.0, "total_discounts_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_line_items_price": 27.0, "total_line_items_price_set": {"shop_money": {"amount": 27.0, "currency_code": "USD"}, "presentment_money": {"amount": 27.0, "currency_code": "USD"}}, "total_outstanding": 0.0, "total_price": 27.0, "total_price_set": {"shop_money": {"amount": 27.0, "currency_code": "USD"}, "presentment_money": {"amount": 27.0, "currency_code": "USD"}}, "total_shipping_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_tax": 0.0, "total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_tip_received": 0.0, "total_weight": 0, "updated_at": "2023-04-24T07:03:06-07:00", "user_id": null, "customer": {"id": 5565161144509, "email": "airbyte@airbyte.com", "accepts_marketing": false, "created_at": "2021-09-19T08:31:05-07:00", "updated_at": "2021-09-19T09:08:24-07:00", "first_name": null, "last_name": null, "state": "disabled", "note": null, "verified_email": true, "multipass_identifier": null, "tax_exempt": false, "phone": null, "email_marketing_consent": {"state": "not_subscribed", "opt_in_level": "single_opt_in", "consent_updated_at": null}, "sms_marketing_consent": null, "tags": "", "currency": "USD", "accepts_marketing_updated_at": "2021-09-19T08:31:05-07:00", "marketing_opt_in_level": null, "tax_exemptions": [], "admin_graphql_api_id": "gid://shopify/Customer/5565161144509"}, "discount_applications": [], "fulfillments": [{"id": 3693416710333, "admin_graphql_api_id": "gid://shopify/Fulfillment/3693416710333", "created_at": "2021-09-19T09:08:23-07:00", "location_id": 63590301885, "name": "#1121.1", "order_id": 4147980107965, "origin_address": {}, "receipt": {}, "service": "manual", "shipment_status": null, "status": "success", "tracking_company": "Amazon Logistics US", "tracking_number": "123456", "tracking_numbers": ["123456"], "tracking_url": "https://track.amazon.com/tracking/123456", "tracking_urls": ["https://track.amazon.com/tracking/123456"], "updated_at": "2022-02-22T00:35:47-08:00", "line_items": [{"id": 10576771317949, "admin_graphql_api_id": "gid://shopify/LineItem/10576771317949", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 285, "name": "Red & Silver Fishing Lure - Plastic", "price": 27.0, "price_set": {"shop_money": {"amount": 27.0, "currency_code": "USD"}, "presentment_money": {"amount": 27.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796218302653, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "Red & Silver Fishing Lure", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 40090580615357, "variant_inventory_management": "shopify", "variant_title": "Plastic", "vendor": "Harris - Hamill", "tax_lines": [], "duties": [], "discount_allocations": []}]}], "line_items": [{"id": 10576771317949, "admin_graphql_api_id": "gid://shopify/LineItem/10576771317949", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 285, "name": "Red & Silver Fishing Lure - Plastic", "price": 27.0, "price_set": {"shop_money": {"amount": 27.0, "currency_code": "USD"}, "presentment_money": {"amount": 27.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796218302653, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "Red & Silver Fishing Lure", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 40090580615357, "variant_inventory_management": "shopify", "variant_title": "Plastic", "vendor": "Harris - Hamill", "tax_lines": [], "duties": [], "discount_allocations": []}], "payment_terms": null, "refunds": [{"id": 845032358077, "admin_graphql_api_id": "gid://shopify/Refund/845032358077", "created_at": "2022-03-07T02:09:04-08:00", "note": null, "order_id": 4147980107965, "processed_at": "2022-03-07T02:09:04-08:00", "restock": true, "total_duties_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "user_id": 74861019325, "order_adjustments": [], "transactions": [], "refund_line_items": [{"id": 352716947645, "line_item_id": 10576771317949, "location_id": 63590301885, "quantity": 1, "restock_type": "return", "subtotal": 27.0, "subtotal_set": {"shop_money": {"amount": "27.00", "currency_code": "USD"}, "presentment_money": {"amount": "27.00", "currency_code": "USD"}}, "total_tax": 0.0, "total_tax_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "line_item": {"id": 10576771317949, "admin_graphql_api_id": "gid://shopify/LineItem/10576771317949", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 285, "name": "Red & Silver Fishing Lure - Plastic", "price": "27.00", "price_set": {"shop_money": {"amount": "27.00", "currency_code": "USD"}, "presentment_money": {"amount": "27.00", "currency_code": "USD"}}, "product_exists": true, "product_id": 6796218302653, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "Red & Silver Fishing Lure", "total_discount": "0.00", "total_discount_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "variant_id": 40090580615357, "variant_inventory_management": "shopify", "variant_title": "Plastic", "vendor": "Harris - Hamill", "tax_lines": [], "duties": [], "discount_allocations": []}}], "duties": []}], "shipping_address": {"first_name": "John", "address1": "San Francisco", "phone": "", "city": "San Francisco", "zip": "91326", "province": "California", "country": "United States", "last_name": "Doe", "address2": "10", "company": "Umbrella LLC", "latitude": 34.2894584, "longitude": -118.5622893, "name": "John Doe", "country_code": "US", "province_code": "CA"}, "shipping_lines": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315623326} -{"stream": "orders", "data": {"id": 3935377129661, "admin_graphql_api_id": "gid://shopify/Order/3935377129661", "app_id": 1354745, "browser_ip": "76.14.176.236", "buyer_accepts_marketing": false, "cancel_reason": null, "cancelled_at": null, "cart_token": null, "checkout_id": 21670281707709, "checkout_token": "ea03756d615a5f9e752f3c085e8cf9bd", "client_details": {"accept_language": "en-US,en;q=0.9", "browser_height": null, "browser_ip": "76.14.176.236", "browser_width": null, "session_hash": null, "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36"}, "closed_at": null, "confirmed": true, "contact_email": null, "created_at": "2021-07-02T00:51:50-07:00", "currency": "USD", "current_subtotal_price": 0.0, "current_subtotal_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_discounts": 0.0, "current_total_discounts_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_duties_set": null, "current_total_price": 0.0, "current_total_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_tax": 0.0, "current_total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "customer_locale": null, "device_id": null, "discount_codes": [], "email": "", "estimated_taxes": false, "financial_status": "refunded", "fulfillment_status": null, "gateway": "bogus", "landing_site": null, "landing_site_ref": null, "location_id": 63590301885, "merchant_of_record_app_id": null, "name": "#1001", "note": null, "note_attributes": [], "number": 1, "order_number": 1001, "order_status_url": "https://airbyte-integration-test.myshopify.com/58033176765/orders/16dd6c6e17f562f1f5eee0fefa00b4cb/authenticate?key=931eb302588779d0ab93839d42bf7166", "original_total_duties_set": null, "payment_gateway_names": ["bogus"], "phone": null, "presentment_currency": "USD", "processed_at": "2021-07-02T00:51:49-07:00", "processing_method": "direct", "reference": null, "referring_site": null, "source_identifier": null, "source_name": "shopify_draft_order", "source_url": null, "subtotal_price": 102.0, "subtotal_price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "tags": "teest", "tax_lines": [{"price": 17.0, "rate": 0.2, "title": "PDV", "price_set": {"shop_money": {"amount": 17.0, "currency_code": "USD"}, "presentment_money": {"amount": 17.0, "currency_code": "USD"}}, "channel_liable": false}], "taxes_included": true, "test": true, "token": "16dd6c6e17f562f1f5eee0fefa00b4cb", "total_discounts": 0.0, "total_discounts_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_line_items_price": 102.0, "total_line_items_price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "total_outstanding": 0.0, "total_price": 102.0, "total_price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "total_shipping_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_tax": 17.0, "total_tax_set": {"shop_money": {"amount": 17.0, "currency_code": "USD"}, "presentment_money": {"amount": 17.0, "currency_code": "USD"}}, "total_tip_received": 0.0, "total_weight": 63, "updated_at": "2023-04-24T10:59:00-07:00", "user_id": 74861019325, "customer": {"id": 5349364105405, "email": null, "accepts_marketing": false, "created_at": "2021-07-02T00:51:46-07:00", "updated_at": "2021-07-02T00:51:46-07:00", "first_name": "Bogus", "last_name": "Gateway", "state": "disabled", "note": null, "verified_email": true, "multipass_identifier": null, "tax_exempt": false, "phone": null, "email_marketing_consent": null, "sms_marketing_consent": null, "tags": "", "currency": "USD", "accepts_marketing_updated_at": "2021-07-02T00:51:46-07:00", "marketing_opt_in_level": null, "tax_exemptions": [], "admin_graphql_api_id": "gid://shopify/Customer/5349364105405"}, "discount_applications": [], "fulfillments": [], "line_items": [{"id": 10130216452285, "admin_graphql_api_id": "gid://shopify/LineItem/10130216452285", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": null, "gift_card": false, "grams": 63, "name": "8 Ounce Soy Candle - Wooden", "price": 102.0, "price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796229509309, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "8 Ounce Soy Candle", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 40090604011709, "variant_inventory_management": "shopify", "variant_title": "Wooden", "vendor": "Bosco Inc", "tax_lines": [{"channel_liable": false, "price": 17.0, "price_set": {"shop_money": {"amount": 17.0, "currency_code": "USD"}, "presentment_money": {"amount": 17.0, "currency_code": "USD"}}, "rate": 0.2, "title": "PDV"}], "duties": [], "discount_allocations": []}], "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}, "payment_terms": null, "refunds": [{"id": 829538369725, "admin_graphql_api_id": "gid://shopify/Refund/829538369725", "created_at": "2021-09-21T05:31:59-07:00", "note": "test refund", "order_id": 3935377129661, "processed_at": "2021-09-21T05:31:59-07:00", "restock": true, "total_duties_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "user_id": 74861019325, "order_adjustments": [], "transactions": [{"id": 5189894406333, "admin_graphql_api_id": "gid://shopify/OrderTransaction/5189894406333", "amount": "102.00", "authorization": null, "created_at": "2021-09-21T05:31:58-07:00", "currency": "USD", "device_id": null, "error_code": null, "gateway": "bogus", "kind": "refund", "location_id": null, "message": "Bogus Gateway: Forced success", "order_id": 3935377129661, "parent_id": 4933790040253, "processed_at": "2021-09-21T05:31:58-07:00", "receipt": {"paid_amount": "102.00"}, "source_name": "1830279", "status": "success", "test": true, "user_id": 74861019325, "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}}], "refund_line_items": [{"id": 332807864509, "line_item_id": 10130216452285, "location_id": 63590301885, "quantity": 1, "restock_type": "cancel", "subtotal": 102.0, "subtotal_set": {"shop_money": {"amount": "102.00", "currency_code": "USD"}, "presentment_money": {"amount": "102.00", "currency_code": "USD"}}, "total_tax": 17.0, "total_tax_set": {"shop_money": {"amount": "17.00", "currency_code": "USD"}, "presentment_money": {"amount": "17.00", "currency_code": "USD"}}, "line_item": {"id": 10130216452285, "admin_graphql_api_id": "gid://shopify/LineItem/10130216452285", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": null, "gift_card": false, "grams": 63, "name": "8 Ounce Soy Candle - Wooden", "price": "102.00", "price_set": {"shop_money": {"amount": "102.00", "currency_code": "USD"}, "presentment_money": {"amount": "102.00", "currency_code": "USD"}}, "product_exists": true, "product_id": 6796229509309, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "8 Ounce Soy Candle", "total_discount": "0.00", "total_discount_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "variant_id": 40090604011709, "variant_inventory_management": "shopify", "variant_title": "Wooden", "vendor": "Bosco Inc", "tax_lines": [{"channel_liable": false, "price": "17.00", "price_set": {"shop_money": {"amount": "17.00", "currency_code": "USD"}, "presentment_money": {"amount": "17.00", "currency_code": "USD"}}, "rate": 0.2, "title": "PDV"}], "duties": [], "discount_allocations": []}}], "duties": []}], "shipping_lines": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315623329} -{"stream": "pages", "data": {"id": 83074252989, "title": "Warranty information", "shop_id": 58033176765, "handle": "warranty-information", "body_html": "updated_mon_24.04.2023", "author": "Shopify API", "created_at": "2021-07-08T05:19:00-07:00", "updated_at": "2023-04-24T11:08:41-07:00", "published_at": "2021-07-08T05:19:00-07:00", "template_suffix": null, "admin_graphql_api_id": "gid://shopify/OnlineStorePage/83074252989", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315624471} -{"stream": "price_rules", "data": {"id": 945000284349, "value_type": "percentage", "value": "-3.0", "customer_selection": "all", "target_type": "line_item", "target_selection": "all", "allocation_method": "across", "allocation_limit": null, "once_per_customer": true, "usage_limit": 10, "starts_at": "2021-07-07T07:22:04-07:00", "ends_at": null, "created_at": "2021-07-07T07:23:11-07:00", "updated_at": "2023-04-24T05:52:22-07:00", "entitled_product_ids": [], "entitled_variant_ids": [], "entitled_collection_ids": [], "entitled_country_ids": [], "prerequisite_product_ids": [], "prerequisite_variant_ids": [], "prerequisite_collection_ids": [], "customer_segment_prerequisite_ids": [], "prerequisite_customer_ids": [], "prerequisite_subtotal_range": null, "prerequisite_quantity_range": null, "prerequisite_shipping_price_range": null, "prerequisite_to_entitlement_quantity_ratio": {"prerequisite_quantity": null, "entitled_quantity": null}, "prerequisite_to_entitlement_purchase": {"prerequisite_amount": null}, "title": "1V8Z165KSH5T", "admin_graphql_api_id": "gid://shopify/PriceRule/945000284349", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315625596} -{"stream": "product_images", "data": {"id": 29301297316029, "product_id": 6796220989629, "position": 1, "created_at": "2021-06-22T18:09:47-07:00", "updated_at": "2023-04-24T10:32:19-07:00", "alt": "updated_mon_24.04.2023", "width": 2200, "height": 1467, "src": "https://cdn.shopify.com/s/files/1/0580/3317/6765/products/4-ounce-soy-candle.jpg?v=1682357539", "variant_ids": [], "admin_graphql_api_id": "gid://shopify/ProductImage/29301297316029", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315627231} -{"stream": "products", "data": {"id": 6796220989629, "title": "4 Ounce Soy Candle", "body_html": "updated_mon_24.04.2023", "vendor": "Hartmann Group", "product_type": "Baby", "created_at": "2021-06-22T18:09:47-07:00", "handle": "4-ounce-soy-candle", "updated_at": "2023-04-24T11:05:13-07:00", "published_at": "2021-06-22T18:09:47-07:00", "template_suffix": "", "status": "active", "published_scope": "web", "tags": "developer-tools-generator", "admin_graphql_api_id": "gid://shopify/Product/6796220989629", "variants": [{"id": 40090585923773, "product_id": 6796220989629, "title": "Metal", "price": 19.0, "sku": "", "position": 1, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Metal", "option2": null, "option3": null, "created_at": "2021-06-22T18:09:47-07:00", "updated_at": "2023-04-13T05:00:55-07:00", "taxable": true, "barcode": null, "grams": 112, "image_id": null, "weight": 112.0, "weight_unit": "g", "inventory_item_id": 42185200631997, "inventory_quantity": 15, "old_inventory_quantity": 15, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/40090585923773"}, {"id": 41561955827901, "product_id": 6796220989629, "title": "Test Variant 1", "price": 19.0, "sku": "", "position": 2, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Test Variant 1", "option2": null, "option3": null, "created_at": "2022-03-06T14:09:20-08:00", "updated_at": "2022-03-06T14:12:40-08:00", "taxable": true, "barcode": "", "grams": 112, "image_id": null, "weight": 112.0, "weight_unit": "g", "inventory_item_id": 43653682495677, "inventory_quantity": 2, "old_inventory_quantity": 2, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/41561955827901"}, {"id": 41561961824445, "product_id": 6796220989629, "title": "Test Variant 2", "price": 19.0, "sku": "", "position": 3, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Test Variant 2", "option2": null, "option3": null, "created_at": "2022-03-06T14:12:20-08:00", "updated_at": "2023-04-24T11:00:10-07:00", "taxable": true, "barcode": "", "grams": 112, "image_id": null, "weight": 112.0, "weight_unit": "g", "inventory_item_id": 43653688524989, "inventory_quantity": 0, "old_inventory_quantity": 0, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/41561961824445"}], "options": [{"id": 8720178315453, "product_id": 6796220989629, "name": "Title", "position": 1, "values": ["Metal", "Test Variant 1", "Test Variant 2"]}], "images": [{"id": 29301297316029, "product_id": 6796220989629, "position": 1, "created_at": "2021-06-22T18:09:47-07:00", "updated_at": "2023-04-24T10:32:19-07:00", "alt": "updated_mon_24.04.2023", "width": 2200, "height": 1467, "src": "https://cdn.shopify.com/s/files/1/0580/3317/6765/products/4-ounce-soy-candle.jpg?v=1682357539", "variant_ids": [], "admin_graphql_api_id": "gid://shopify/ProductImage/29301297316029"}], "image": {"id": 29301297316029, "product_id": 6796220989629, "position": 1, "created_at": "2021-06-22T18:09:47-07:00", "updated_at": "2023-04-24T10:32:19-07:00", "alt": "updated_mon_24.04.2023", "width": 2200, "height": 1467, "src": "https://cdn.shopify.com/s/files/1/0580/3317/6765/products/4-ounce-soy-candle.jpg?v=1682357539", "variant_ids": [], "admin_graphql_api_id": "gid://shopify/ProductImage/29301297316029"}, "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315627645} -{"stream": "products_graph_ql", "data": {"id": "gid://shopify/Product/6796220989629", "title": "4 Ounce Soy Candle", "updatedAt": "2023-04-24T18:05:13Z", "createdAt": "2021-06-23T01:09:47Z", "publishedAt": "2021-06-23T01:09:47Z", "status": "ACTIVE", "vendor": "Hartmann Group", "productType": "Baby", "tags": ["developer-tools-generator"], "options": [{"id": "gid://shopify/ProductOption/8720178315453", "name": "Title", "position": 1, "values": ["Metal", "Test Variant 1", "Test Variant 2"]}], "handle": "4-ounce-soy-candle", "description": "updated_mon_24.04.2023", "tracksInventory": true, "totalInventory": 17, "totalVariants": 3, "onlineStoreUrl": null, "onlineStorePreviewUrl": "https://airbyte-integration-test.myshopify.com/products/4-ounce-soy-candle", "descriptionHtml": "updated_mon_24.04.2023", "isGiftCard": false, "legacyResourceId": "6796220989629", "mediaCount": 1, "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315628830} -{"stream": "product_variants", "data": {"id": 40090585923773, "product_id": 6796220989629, "title": "Metal", "price": 19.0, "sku": "", "position": 1, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Metal", "option2": null, "option3": null, "created_at": "2021-06-22T18:09:47-07:00", "updated_at": "2023-04-13T05:00:55-07:00", "taxable": true, "barcode": null, "grams": 112, "image_id": null, "weight": 112.0, "weight_unit": "g", "inventory_item_id": 42185200631997, "inventory_quantity": 15, "old_inventory_quantity": 15, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/40090585923773", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315630410} -{"stream": "product_variants", "data": {"id": 41561955827901, "product_id": 6796220989629, "title": "Test Variant 1", "price": 19.0, "sku": "", "position": 2, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Test Variant 1", "option2": null, "option3": null, "created_at": "2022-03-06T14:09:20-08:00", "updated_at": "2022-03-06T14:12:40-08:00", "taxable": true, "barcode": "", "grams": 112, "image_id": null, "weight": 112.0, "weight_unit": "g", "inventory_item_id": 43653682495677, "inventory_quantity": 2, "old_inventory_quantity": 2, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/41561955827901", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315630418} -{"stream": "product_variants", "data": {"id": 41561961824445, "product_id": 6796220989629, "title": "Test Variant 2", "price": 19.0, "sku": "", "position": 3, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Test Variant 2", "option2": null, "option3": null, "created_at": "2022-03-06T14:12:20-08:00", "updated_at": "2023-04-24T11:00:10-07:00", "taxable": true, "barcode": "", "grams": 112, "image_id": null, "weight": 112.0, "weight_unit": "g", "inventory_item_id": 43653688524989, "inventory_quantity": 0, "old_inventory_quantity": 0, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/41561961824445", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315630419} -{"stream": "shop", "data": {"id": 58033176765, "name": "airbyte integration test", "email": "sherif@airbyte.io", "domain": "airbyte-integration-test.myshopify.com", "province": "California", "country": "US", "address1": "350 29th Avenue", "zip": "94121", "city": "San Francisco", "source": null, "phone": "8023494963", "latitude": 37.7827286, "longitude": -122.4889911, "primary_locale": "en", "address2": "", "created_at": "2021-06-22T18:00:23-07:00", "updated_at": "2023-04-30T09:02:52-07:00", "country_code": "US", "country_name": "United States", "currency": "USD", "customer_email": "sherif@airbyte.io", "timezone": "(GMT-08:00) America/Los_Angeles", "iana_timezone": "America/Los_Angeles", "shop_owner": "Airbyte Airbyte", "money_format": "${{amount}}", "money_with_currency_format": "${{amount}} USD", "weight_unit": "kg", "province_code": "CA", "taxes_included": true, "auto_configure_tax_inclusivity": null, "tax_shipping": null, "county_taxes": true, "plan_display_name": "Developer Preview", "plan_name": "partner_test", "has_discounts": true, "has_gift_cards": false, "myshopify_domain": "airbyte-integration-test.myshopify.com", "google_apps_domain": null, "google_apps_login_enabled": null, "money_in_emails_format": "${{amount}}", "money_with_currency_in_emails_format": "${{amount}} USD", "eligible_for_payments": true, "requires_extra_payments_agreement": false, "password_enabled": true, "has_storefront": true, "finances": true, "primary_location_id": 63590301885, "checkout_api_supported": true, "multi_location_enabled": true, "setup_required": false, "pre_launch_enabled": false, "enabled_presentment_currencies": ["USD"], "transactional_sms_disabled": false, "marketing_sms_consent_enabled_at_checkout": false, "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315631635} -{"stream": "smart_collections", "data": {"id": 273278566589, "handle": "test-collection", "title": "Test Collection", "updated_at": "2023-04-24T10:55:09-07:00", "body_html": "updated_mon_24.04.2023", "published_at": "2021-07-19T07:02:54-07:00", "sort_order": "best-selling", "template_suffix": "", "disjunctive": false, "rules": ["{'column': 'type', 'relation': 'equals', 'condition': 'Beauty'}"], "published_scope": "web", "admin_graphql_api_id": "gid://shopify/Collection/273278566589", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315632721} -{"stream": "tender_transactions", "data": {"id": 4464009117885, "order_id": 5033391718589, "amount": "19.00", "currency": "USD", "user_id": null, "test": false, "processed_at": "2023-04-24T11:00:08-07:00", "remote_reference": null, "payment_details": null, "payment_method": "other", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315633888} -{"stream": "transactions", "data": {"id": 5721110872253, "order_id": 4554821468349, "kind": "sale", "gateway": "bogus", "status": "success", "message": "Bogus Gateway: Forced success", "created_at": "2022-06-15T05:16:52-07:00", "test": true, "authorization": "53433", "location_id": null, "user_id": null, "parent_id": null, "processed_at": "2022-06-15T05:16:52-07:00", "device_id": null, "error_code": null, "source_name": "580111", "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}, "receipt": {"paid_amount": "57.23"}, "amount": 57.23, "currency": "USD", "admin_graphql_api_id": "gid://shopify/OrderTransaction/5721110872253", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315636101} -{"stream": "transactions", "data": {"id": 5721170968765, "order_id": 4554821468349, "kind": "refund", "gateway": "bogus", "status": "success", "message": "Bogus Gateway: Forced success", "created_at": "2022-06-15T06:25:42-07:00", "test": true, "authorization": null, "location_id": null, "user_id": null, "parent_id": 5721110872253, "processed_at": "2022-06-15T06:25:42-07:00", "device_id": null, "error_code": null, "source_name": "1830279", "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}, "receipt": {"paid_amount": "57.23"}, "amount": 57.23, "currency": "USD", "admin_graphql_api_id": "gid://shopify/OrderTransaction/5721170968765", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315636102} -{"stream": "transactions", "data": {"id": 4933790040253, "order_id": 3935377129661, "kind": "sale", "gateway": "bogus", "status": "success", "message": "Bogus Gateway: Forced success", "created_at": "2021-07-02T00:51:49-07:00", "test": true, "authorization": "53433", "location_id": null, "user_id": null, "parent_id": null, "processed_at": "2021-07-02T00:51:49-07:00", "device_id": null, "error_code": null, "source_name": "shopify_draft_order", "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null}, "receipt": {"paid_amount": "102.00"}, "amount": 102.0, "currency": "USD", "admin_graphql_api_id": "gid://shopify/OrderTransaction/4933790040253", "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315637157} -{"stream": "customer_address", "data": {"id": 8092523135165, "customer_id": 6569096478909, "first_name": "New Test", "last_name": "Customer", "company": "Test Company", "address1": "My Best Accent", "address2": "", "city": "Fair Lawn", "province": "New Jersey", "country": "United States", "zip": "07410", "phone": "", "name": "New Test Customer", "province_code": "NJ", "country_code": "US", "country_name": "United States", "default": true, "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315642565} -{"stream": "customer_address", "data": {"id": 8212915650749, "customer_id": 6676027932861, "first_name": "MArcos", "last_name": "Millnitz", "company": null, "address1": null, "address2": null, "city": null, "province": null, "country": null, "zip": null, "phone": null, "name": "MArcos Millnitz", "province_code": null, "country_code": null, "country_name": null, "default": true, "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315643053} -{"stream": "countries", "data": {"id": 417014841533, "name": "Rest of World", "code": "*", "tax_name": "Tax", "tax": 0.0, "provinces": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315644180} -{"stream": "countries", "data": {"id": 417014808765, "name": "Ukraine", "code": "UA", "tax_name": "PDV", "tax": 0.2, "provinces": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1690315644181} +{"stream": "articles", "data": {"id": 558137508029, "title": "My new Article title", "created_at": "2022-10-07T16:09:02-07:00", "body_html": "

      I like articles

      \n

      Yea, I like posting them through REST.

      ", "blog_id": 80417685693, "author": "John Smith", "user_id": null, "published_at": "2011-03-24T08:45:47-07:00", "updated_at": "2023-04-14T03:18:26-07:00", "summary_html": null, "template_suffix": null, "handle": "my-new-article-title", "tags": "Has Been Tagged, This Post", "admin_graphql_api_id": "gid://shopify/OnlineStoreArticle/558137508029", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203649004} +{"stream": "articles", "data": {"id": 558627979453, "title": "Test Blog Post", "created_at": "2023-04-14T03:19:02-07:00", "body_html": "Test Blog Post 1", "blog_id": 80417685693, "author": "Airbyte Airbyte", "user_id": "74861019325", "published_at": null, "updated_at": "2023-04-14T03:19:18-07:00", "summary_html": "", "template_suffix": "", "handle": "test-blog-post", "tags": "Has Been Tagged", "admin_graphql_api_id": "gid://shopify/OnlineStoreArticle/558627979453", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203649005} +{"stream": "articles", "data": {"id": 558999371965, "deleted_at": "2023-09-05T13:50:04-07:00", "updated_at": "2023-09-05T13:50:04-07:00", "deleted_message": "Online Store deleted an article: Test Article 1.", "deleted_description": "Online Store deleted an article: Test Article 1.", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203649584} +{"stream": "blogs", "data": {"id": 80417685693, "handle": "news", "title": "News", "updated_at": "2023-09-05T14:02:00-07:00", "commentable": "no", "feedburner": null, "feedburner_location": null, "created_at": "2021-06-22T18:00:25-07:00", "template_suffix": null, "tags": "Has Been Tagged, This Post", "admin_graphql_api_id": "gid://shopify/OnlineStoreBlog/80417685693", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203651626} +{"stream": "blogs", "data": {"id": 85733114045, "deleted_at": "2023-09-06T03:30:22-07:00", "updated_at": "2023-09-06T03:30:22-07:00", "deleted_message": "Online Store deleted a blog: Test Blog 1.", "deleted_description": "Online Store deleted a blog: Test Blog 1.", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203652217} +{"stream": "collections", "data": {"id": 270889287869, "handle": "frontpage", "title": "Home page", "updated_at": "2023-09-05T14:06:59+00:00", "body_html": "updated_mon_24.04.2023", "published_at": "2021-06-23T01:00:25+00:00", "sort_order": "BEST_SELLING", "template_suffix": "", "products_count": 1, "admin_graphql_api_id": "gid://shopify/Collection/270889287869", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203654253} +{"stream": "collects", "data": {"id": 29427031703741, "collection_id": 270889287869, "product_id": 6796220989629, "created_at": "2021-07-19T07:01:36-07:00", "updated_at": "2022-03-06T14:12:21-08:00", "position": 2, "sort_value": "0000000002", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203655217} +{"stream": "custom_collections", "data": {"id": 270889287869, "handle": "frontpage", "title": "Home page", "updated_at": "2023-09-05T07:06:59-07:00", "body_html": "updated_mon_24.04.2023", "published_at": "2021-06-22T18:00:25-07:00", "sort_order": "best-selling", "template_suffix": "", "published_scope": "web", "admin_graphql_api_id": "gid://shopify/Collection/270889287869", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203656191} +{"stream": "custom_collections", "data": {"id": 294253822141, "deleted_at": "2023-09-06T03:34:39-07:00", "updated_at": "2023-09-06T03:34:39-07:00", "deleted_message": "Airbyte Airbyte deleted a collection.", "deleted_description": "Airbyte Airbyte deleted a collection.", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203656725} +{"stream": "customers", "data": {"id": 6569096478909, "email": "test@test.com", "accepts_marketing": true, "created_at": "2023-04-13T02:30:04-07:00", "updated_at": "2023-04-24T06:53:48-07:00", "first_name": "New Test", "last_name": "Customer", "orders_count": 0, "state": "disabled", "total_spent": 0.0, "last_order_id": null, "note": "updated_mon_24.04.2023", "verified_email": true, "multipass_identifier": null, "tax_exempt": false, "tags": "", "last_order_name": null, "currency": "USD", "phone": "+380639379992", "addresses": [{"id": 8092523135165, "customer_id": 6569096478909, "first_name": "New Test", "last_name": "Customer", "company": "Test Company", "address1": "My Best Accent", "address2": "", "city": "Fair Lawn", "province": "New Jersey", "country": "United States", "zip": "07410", "phone": "", "name": "New Test Customer", "province_code": "NJ", "country_code": "US", "country_name": "United States", "default": true}], "accepts_marketing_updated_at": "2023-04-13T02:30:04-07:00", "marketing_opt_in_level": "single_opt_in", "tax_exemptions": "[]", "email_marketing_consent": {"state": "subscribed", "opt_in_level": "single_opt_in", "consent_updated_at": "2023-04-13T02:30:04-07:00"}, "sms_marketing_consent": {"state": "not_subscribed", "opt_in_level": "single_opt_in", "consent_updated_at": null, "consent_collected_from": "SHOPIFY"}, "admin_graphql_api_id": "gid://shopify/Customer/6569096478909", "default_address": {"id": 8092523135165, "customer_id": 6569096478909, "first_name": "New Test", "last_name": "Customer", "company": "Test Company", "address1": "My Best Accent", "address2": "", "city": "Fair Lawn", "province": "New Jersey", "country": "United States", "zip": "07410", "phone": "", "name": "New Test Customer", "province_code": "NJ", "country_code": "US", "country_name": "United States", "default": true}, "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203657952} +{"stream": "customers", "data": {"id": 6676027932861, "email": "marcos@airbyte.io", "accepts_marketing": false, "created_at": "2023-07-11T13:07:45-07:00", "updated_at": "2023-07-11T13:07:45-07:00", "first_name": "MArcos", "last_name": "Millnitz", "orders_count": 0, "state": "disabled", "total_spent": 0.0, "last_order_id": null, "note": null, "verified_email": true, "multipass_identifier": null, "tax_exempt": false, "tags": "", "last_order_name": null, "currency": "USD", "phone": null, "addresses": [{"id": 8212915650749, "customer_id": 6676027932861, "first_name": "MArcos", "last_name": "Millnitz", "company": null, "address1": null, "address2": null, "city": null, "province": null, "country": null, "zip": null, "phone": null, "name": "MArcos Millnitz", "province_code": null, "country_code": null, "country_name": null, "default": true}], "accepts_marketing_updated_at": "2023-07-11T13:07:45-07:00", "marketing_opt_in_level": null, "tax_exemptions": "[]", "email_marketing_consent": {"state": "not_subscribed", "opt_in_level": "single_opt_in", "consent_updated_at": null}, "sms_marketing_consent": null, "admin_graphql_api_id": "gid://shopify/Customer/6676027932861", "default_address": {"id": 8212915650749, "customer_id": 6676027932861, "first_name": "MArcos", "last_name": "Millnitz", "company": null, "address1": null, "address2": null, "city": null, "province": null, "country": null, "zip": null, "phone": null, "name": "MArcos Millnitz", "province_code": null, "country_code": null, "country_name": null, "default": true}, "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203657954} +{"stream": "discount_codes", "data": {"id": 11539415990461, "price_rule_id": 945000284349, "code": "updated_mon_24.04.2023", "usage_count": 0, "created_at": "2021-07-07T14:23:11+00:00", "updated_at": "2023-04-24T12:52:22+00:00", "summary": "3% off entire order \u2022 One use per customer", "discount_type": "ORDER", "admin_graphql_api_id": "gid://shopify/DiscountRedeemCode/11539415990461", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203660428} +{"stream": "draft_orders", "data": {"id": 874683629757, "note": null, "email": null, "taxes_included": true, "currency": "USD", "invoice_sent_at": null, "created_at": "2021-07-02T00:50:29-07:00", "updated_at": "2023-04-14T05:16:33-07:00", "tax_exempt": false, "completed_at": "2021-07-02T00:51:50-07:00", "name": "#D2", "status": "completed", "line_items": [{"id": 57443281666237, "variant_id": 40090604011709, "product_id": 6796229509309, "title": "8 Ounce Soy Candle", "variant_title": "Wooden", "sku": "", "vendor": "Bosco Inc", "quantity": 1, "requires_shipping": true, "taxable": true, "gift_card": false, "fulfillment_service": "manual", "grams": 63, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 17.0}], "applied_discount": null, "name": "8 Ounce Soy Candle - Wooden", "properties": [], "custom": false, "price": 102.0, "admin_graphql_api_id": "gid://shopify/DraftOrderLineItem/57443281666237"}], "shipping_address": null, "billing_address": null, "invoice_url": "https://airbyte-integration-test.myshopify.com/58033176765/invoices/e155e3254d0c0d64fa90587de417e0f3", "applied_discount": null, "order_id": 3935377129661, "shipping_line": null, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 17.0}], "tags": "", "note_attributes": [], "total_price": "102.00", "subtotal_price": "102.00", "total_tax": "17.00", "payment_terms": null, "admin_graphql_api_id": "gid://shopify/DraftOrder/874683629757", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203663415} +{"stream": "draft_orders", "data": {"id": 929019691197, "note": "updated_mon_24.04.2023", "email": null, "taxes_included": true, "currency": "USD", "invoice_sent_at": null, "created_at": "2022-02-22T03:23:19-08:00", "updated_at": "2023-04-24T07:18:06-07:00", "tax_exempt": false, "completed_at": null, "name": "#D21", "status": "open", "line_items": [{"id": 58117295538365, "variant_id": 40090585923773, "product_id": 6796220989629, "title": "4 Ounce Soy Candle", "variant_title": "Metal", "sku": "", "vendor": "Hartmann Group", "quantity": 2, "requires_shipping": true, "taxable": true, "gift_card": false, "fulfillment_service": "manual", "grams": 112, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 6.33}], "applied_discount": null, "name": "4 Ounce Soy Candle - Metal", "properties": [], "custom": false, "price": 19.0, "admin_graphql_api_id": "gid://shopify/DraftOrderLineItem/58117295538365"}, {"id": 58117295571133, "variant_id": null, "product_id": null, "title": "Test Item", "variant_title": null, "sku": null, "vendor": null, "quantity": 1, "requires_shipping": true, "taxable": true, "gift_card": false, "fulfillment_service": "manual", "grams": 1000, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 0.17}], "applied_discount": null, "name": "Test Item", "properties": [], "custom": true, "price": 1.0, "admin_graphql_api_id": "gid://shopify/DraftOrderLineItem/58117295571133"}], "shipping_address": null, "billing_address": null, "invoice_url": "https://airbyte-integration-test.myshopify.com/58033176765/invoices/12893992cc01fc67935ab014fcf9300f", "applied_discount": null, "order_id": null, "shipping_line": {"title": "Test Shipping Fee", "custom": true, "handle": null, "price": 3.0}, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 6.33}, {"rate": 0.2, "title": "PDV", "price": 0.17}], "tags": "", "note_attributes": [], "total_price": "42.00", "subtotal_price": "39.00", "total_tax": "6.50", "payment_terms": null, "admin_graphql_api_id": "gid://shopify/DraftOrder/929019691197", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203663416} +{"stream": "draft_orders", "data": {"id": 987391033533, "note": null, "email": null, "taxes_included": true, "currency": "USD", "invoice_sent_at": null, "created_at": "2023-04-13T04:56:17-07:00", "updated_at": "2023-04-13T04:56:17-07:00", "tax_exempt": false, "completed_at": null, "name": "#D25", "status": "open", "line_items": [{"id": 58116862083261, "variant_id": 40090585923773, "product_id": 6796220989629, "title": "4 Ounce Soy Candle", "variant_title": "Metal", "sku": "", "vendor": "Hartmann Group", "quantity": 1, "requires_shipping": true, "taxable": true, "gift_card": false, "fulfillment_service": "manual", "grams": 112, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 3.16}], "applied_discount": null, "name": "4 Ounce Soy Candle - Metal", "properties": [], "custom": false, "price": 19.0, "admin_graphql_api_id": "gid://shopify/DraftOrderLineItem/58116862083261"}, {"id": 58116862116029, "variant_id": 41561955827901, "product_id": 6796220989629, "title": "4 Ounce Soy Candle", "variant_title": "Test Variant 1", "sku": "", "vendor": "Hartmann Group", "quantity": 1, "requires_shipping": true, "taxable": true, "gift_card": false, "fulfillment_service": "manual", "grams": 112, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 3.17}], "applied_discount": null, "name": "4 Ounce Soy Candle - Test Variant 1", "properties": [], "custom": false, "price": 19.0, "admin_graphql_api_id": "gid://shopify/DraftOrderLineItem/58116862116029"}, {"id": 58116862148797, "variant_id": 41561961824445, "product_id": 6796220989629, "title": "4 Ounce Soy Candle", "variant_title": "Test Variant 2", "sku": "", "vendor": "Hartmann Group", "quantity": 1, "requires_shipping": true, "taxable": true, "gift_card": false, "fulfillment_service": "manual", "grams": 112, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 3.17}], "applied_discount": null, "name": "4 Ounce Soy Candle - Test Variant 2", "properties": [], "custom": false, "price": 19.0, "admin_graphql_api_id": "gid://shopify/DraftOrderLineItem/58116862148797"}], "shipping_address": null, "billing_address": null, "invoice_url": "https://airbyte-integration-test.myshopify.com/58033176765/invoices/d193a965f7815817a2d37fddb30bfdb2", "applied_discount": null, "order_id": null, "shipping_line": null, "tax_lines": [{"rate": 0.2, "title": "PDV", "price": 3.16}, {"rate": 0.2, "title": "PDV", "price": 3.17}, {"rate": 0.2, "title": "PDV", "price": 3.17}], "tags": "", "note_attributes": [], "total_price": "57.00", "subtotal_price": "57.00", "total_tax": "9.50", "payment_terms": null, "admin_graphql_api_id": "gid://shopify/DraftOrder/987391033533", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203663417} +{"stream": "fulfillment_orders", "data": {"id": 5962451452093, "assigned_location": {"address1": "Heroiv UPA 72", "address2": null, "city": "Lviv", "country_code": "UA", "name": "Heroiv UPA 72", "phone": "", "province": null, "zip": "30100", "location_id": 63590301885}, "destination": null, "delivery_method": {"id": 431686549693, "method_type": "SHIPPING", "min_delivery_date_time": null, "max_delivery_date_time": null}, "fulfill_at": "2023-04-13T12:00:00+00:00", "fulfill_by": null, "international_duties": null, "fulfillment_holds": [], "created_at": "2023-04-13T12:09:45+00:00", "updated_at": "2023-04-13T12:09:46+00:00", "request_status": "UNSUBMITTED", "status": "CLOSED", "supported_actions": [], "shop_id": 58033176765, "order_id": 5010584895677, "assigned_location_id": 63590301885, "line_items": [{"id": 12363725996221, "inventory_item_id": 42185218719933, "shop_id": 58033176765, "fulfillment_order_id": 5962451452093, "quantity": 1, "line_item_id": 12204214845629, "fulfillable_quantity": 0, "variant_id": 40090604011709}], "merchant_requests": [], "admin_graphql_api_id": "gid://shopify/FulfillmentOrder/5962451452093", "shop_url": "airbyte-integration-test"}, "emitted_at": 1702564397004} +{"stream": "fulfillment_orders", "data": {"id": 5962452467901, "assigned_location": {"address1": "Heroiv UPA 72", "address2": null, "city": "Lviv", "country_code": "UA", "name": "Heroiv UPA 72", "phone": "", "province": null, "zip": "30100", "location_id": 63590301885}, "destination": null, "delivery_method": {"id": 431687532733, "method_type": "SHIPPING", "min_delivery_date_time": null, "max_delivery_date_time": null}, "fulfill_at": "2023-04-13T12:00:00+00:00", "fulfill_by": null, "international_duties": null, "fulfillment_holds": [], "created_at": "2023-04-13T12:11:16+00:00", "updated_at": "2023-04-13T12:11:16+00:00", "request_status": "UNSUBMITTED", "status": "CLOSED", "supported_actions": [], "shop_id": 58033176765, "order_id": 5010585911485, "assigned_location_id": 63590301885, "line_items": [{"id": 12363727536317, "inventory_item_id": 43653688524989, "shop_id": 58033176765, "fulfillment_order_id": 5962452467901, "quantity": 1, "line_item_id": 12204216385725, "fulfillable_quantity": 0, "variant_id": 41561961824445}], "merchant_requests": [], "admin_graphql_api_id": "gid://shopify/FulfillmentOrder/5962452467901", "shop_url": "airbyte-integration-test"}, "emitted_at": 1702564397005} +{"stream": "fulfillment_orders", "data": {"id": 5985636450493, "assigned_location": {"address1": "Heroiv UPA 72", "address2": null, "city": "Lviv", "country_code": "UA", "name": "Heroiv UPA 72", "phone": "", "province": null, "zip": "30100", "location_id": 63590301885}, "destination": null, "delivery_method": {"id": 442031046845, "method_type": "SHIPPING", "min_delivery_date_time": null, "max_delivery_date_time": null}, "fulfill_at": "2023-04-24T18:00:00+00:00", "fulfill_by": null, "international_duties": null, "fulfillment_holds": [], "created_at": "2023-04-24T18:00:09+00:00", "updated_at": "2023-04-24T18:00:09+00:00", "request_status": "UNSUBMITTED", "status": "CLOSED", "supported_actions": [], "shop_id": 58033176765, "order_id": 5033391718589, "assigned_location_id": 63590301885, "line_items": [{"id": 12407122067645, "inventory_item_id": 43653688524989, "shop_id": 58033176765, "fulfillment_order_id": 5985636450493, "quantity": 1, "line_item_id": 12247585521853, "fulfillable_quantity": 0, "variant_id": 41561961824445}], "merchant_requests": [], "admin_graphql_api_id": "gid://shopify/FulfillmentOrder/5985636450493", "shop_url": "airbyte-integration-test"}, "emitted_at": 1702564397007} +{"stream": "fulfillments", "data": {"id": 4451164913853, "admin_graphql_api_id": "gid://shopify/Fulfillment/4451164913853", "created_at": "2023-04-13T05:09:45-07:00", "location_id": 63590301885, "name": "#1143.1", "order_id": 5010584895677, "origin_address": {}, "receipt": {}, "service": "manual", "shipment_status": null, "status": "success", "tracking_company": null, "tracking_number": null, "tracking_numbers": [], "tracking_url": null, "tracking_urls": [], "updated_at": "2023-04-13T05:09:45-07:00", "line_items": [{"id": 12204214845629, "admin_graphql_api_id": "gid://shopify/LineItem/12204214845629", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 63, "name": "8 Ounce Soy Candle - Wooden", "price": "102.00", "price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796229509309, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "8 Ounce Soy Candle", "total_discount": "0.00", "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 40090604011709, "variant_inventory_management": "shopify", "variant_title": "Wooden", "vendor": "Bosco Inc", "tax_lines": [{"channel_liable": false, "price": 17.0, "price_set": {"shop_money": {"amount": 17.0, "currency_code": "USD"}, "presentment_money": {"amount": 17.0, "currency_code": "USD"}}, "rate": 0.2, "title": "PDV"}], "duties": [], "discount_allocations": []}], "shop_url": "airbyte-integration-test"}, "emitted_at": 1701365205787} +{"stream": "fulfillments", "data": {"id": 4451169501373, "admin_graphql_api_id": "gid://shopify/Fulfillment/4451169501373", "created_at": "2023-04-13T05:11:16-07:00", "location_id": 63590301885, "name": "#1144.1", "order_id": 5010585911485, "origin_address": {}, "receipt": {}, "service": "manual", "shipment_status": null, "status": "success", "tracking_company": null, "tracking_number": null, "tracking_numbers": [], "tracking_url": null, "tracking_urls": [], "updated_at": "2023-04-13T05:11:16-07:00", "line_items": [{"id": 12204216385725, "admin_graphql_api_id": "gid://shopify/LineItem/12204216385725", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 112, "name": "4 Ounce Soy Candle - Test Variant 2", "price": "19.00", "price_set": {"shop_money": {"amount": 19.0, "currency_code": "USD"}, "presentment_money": {"amount": 19.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796220989629, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "4 Ounce Soy Candle", "total_discount": "0.00", "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 41561961824445, "variant_inventory_management": "shopify", "variant_title": "Test Variant 2", "vendor": "Hartmann Group", "tax_lines": [{"channel_liable": false, "price": 3.17, "price_set": {"shop_money": {"amount": 3.17, "currency_code": "USD"}, "presentment_money": {"amount": 3.17, "currency_code": "USD"}}, "rate": 0.2, "title": "PDV"}], "duties": [], "discount_allocations": []}], "shop_url": "airbyte-integration-test"}, "emitted_at": 1701365205792} +{"stream": "fulfillments", "data": {"id": 4465911431357, "admin_graphql_api_id": "gid://shopify/Fulfillment/4465911431357", "created_at": "2023-04-24T11:00:09-07:00", "location_id": 63590301885, "name": "#1145.1", "order_id": 5033391718589, "origin_address": {}, "receipt": {}, "service": "manual", "shipment_status": null, "status": "success", "tracking_company": null, "tracking_number": null, "tracking_numbers": [], "tracking_url": null, "tracking_urls": [], "updated_at": "2023-04-24T11:00:09-07:00", "line_items": [{"id": 12247585521853, "admin_graphql_api_id": "gid://shopify/LineItem/12247585521853", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 112, "name": "4 Ounce Soy Candle - Test Variant 2", "price": "19.00", "price_set": {"shop_money": {"amount": 19.0, "currency_code": "USD"}, "presentment_money": {"amount": 19.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796220989629, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "4 Ounce Soy Candle", "total_discount": "0.00", "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 41561961824445, "variant_inventory_management": "shopify", "variant_title": "Test Variant 2", "vendor": "Hartmann Group", "tax_lines": [{"channel_liable": false, "price": 3.17, "price_set": {"shop_money": {"amount": 3.17, "currency_code": "USD"}, "presentment_money": {"amount": 3.17, "currency_code": "USD"}}, "rate": 0.2, "title": "PDV"}], "duties": [], "discount_allocations": []}], "shop_url": "airbyte-integration-test"}, "emitted_at": 1701365205799} +{"stream": "inventory_items", "data": {"id": 44871665713341, "country_code_of_origin": null, "harmonized_system_code": null, "province_code_of_origin": null, "updated_at": "2023-04-14T10:29:27+00:00", "created_at": "2023-04-14T10:29:27+00:00", "sku": "", "tracked": true, "requires_shipping": false, "admin_graphql_api_id": "gid://shopify/InventoryItem/44871665713341", "cost": 60.0, "country_harmonized_system_codes": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1702300616024} +{"stream": "inventory_items", "data": {"id": 45419395743933, "country_code_of_origin": "UA", "harmonized_system_code": "330510", "province_code_of_origin": null, "updated_at": "2023-12-11T10:37:41+00:00", "created_at": "2023-12-11T10:37:41+00:00", "sku": "123", "tracked": true, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/InventoryItem/45419395743933", "cost": 29.0, "country_harmonized_system_codes": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1702300616026} +{"stream": "inventory_levels", "data": {"id": "63590301885|43653688524989", "available": 0, "updated_at": "2023-04-24T18:00:10+00:00", "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=43653688524989", "inventory_item_id": 43653688524989, "location_id": 63590301885, "shop_url": "airbyte-integration-test"}, "emitted_at": 1702299709159} +{"stream": "inventory_levels", "data": {"id": "63590301885|44871665713341", "available": 0, "updated_at": "2023-04-14T10:29:27+00:00", "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=44871665713341", "inventory_item_id": 44871665713341, "location_id": 63590301885, "shop_url": "airbyte-integration-test"}, "emitted_at": 1702299709160} +{"stream": "inventory_levels", "data": {"id": "63590301885|45419395743933", "available": 1, "updated_at": "2023-12-11T10:37:41+00:00", "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=45419395743933", "inventory_item_id": 45419395743933, "location_id": 63590301885, "shop_url": "airbyte-integration-test"}, "emitted_at": 1702299709160} +{"stream": "locations", "data": {"id": 63590301885, "name": "Heroiv UPA 72", "address1": "Heroiv UPA 72", "address2": "", "city": "Lviv", "zip": "30100", "province": null, "country": "UA", "phone": "", "created_at": "2021-06-22T18:00:29-07:00", "updated_at": "2023-11-28T07:08:27-08:00", "country_code": "UA", "country_name": "Ukraine", "province_code": null, "legacy": false, "active": true, "admin_graphql_api_id": "gid://shopify/Location/63590301885", "localized_country_name": "Ukraine", "localized_province_name": null, "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203911774} +{"stream": "metafield_articles", "data": {"id": 21519818162365, "namespace": "global", "key": "new", "value": "newvalue", "description": null, "owner_id": 558137508029, "created_at": "2022-10-07T16:09:02-07:00", "updated_at": "2022-10-07T16:09:02-07:00", "owner_resource": "article", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/21519818162365", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203913911} +{"stream": "metafield_articles", "data": {"id": 22365709992125, "namespace": "custom", "key": "test_blog_post_metafield", "value": "Test Article Metafield", "description": null, "owner_id": 558137508029, "created_at": "2023-04-14T03:18:26-07:00", "updated_at": "2023-04-14T03:18:26-07:00", "owner_resource": "article", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365709992125", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203913912} +{"stream": "metafield_articles", "data": {"id": 22365710352573, "namespace": "custom", "key": "test_blog_post_metafield", "value": "Test Blog Post Metafiled", "description": null, "owner_id": 558627979453, "created_at": "2023-04-14T03:19:18-07:00", "updated_at": "2023-04-14T03:19:18-07:00", "owner_resource": "article", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365710352573", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203914366} +{"stream": "metafield_blogs", "data": {"id": 21519428255933, "namespace": "some_fields", "key": "sponsor", "value": "Shopify", "description": null, "owner_id": 80417685693, "created_at": "2022-10-07T06:05:23-07:00", "updated_at": "2022-10-07T06:05:23-07:00", "owner_resource": "blog", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/21519428255933", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203917099} +{"stream": "metafield_blogs", "data": {"id": 22365710745789, "namespace": "custom", "key": "test_blog_metafield", "value": "Test Blog Metafield", "description": null, "owner_id": 80417685693, "created_at": "2023-04-14T03:20:20-07:00", "updated_at": "2023-04-14T03:20:20-07:00", "owner_resource": "blog", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365710745789", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203917100} +{"stream": "metafield_collections", "data": {"id": 22365707174077, "namespace": "custom", "value": "Test Collection Metafield", "key": "test_collection_metafield", "description": null, "created_at": "2023-04-14T10:15:30+00:00", "updated_at": "2023-04-14T10:15:30+00:00", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365707174077", "owner_id": 270889287869, "owner_resource": "collection", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203928649} +{"stream": "metafield_collections", "data": {"id": 22366265573565, "namespace": "my_fields", "value": "51%", "key": "new_key", "description": null, "created_at": "2023-04-14T12:21:58+00:00", "updated_at": "2023-04-14T12:21:58+00:00", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22366265573565", "owner_id": 273278566589, "owner_resource": "collection", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203928650} +{"stream": "metafield_customers", "data": {"id": 22346893361341, "namespace": "custom", "value": "Teste\n", "key": "test_definition_list_1", "description": null, "created_at": "2023-04-13T11:50:10+00:00", "updated_at": "2023-04-13T11:50:10+00:00", "type": "multi_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22346893361341", "owner_id": 6569096478909, "owner_resource": "customer", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203938516} +{"stream": "metafield_customers", "data": {"id": 22346893394109, "namespace": "custom", "value": "Taster", "key": "test_definition", "description": null, "created_at": "2023-04-13T11:50:10+00:00", "updated_at": "2023-04-13T11:50:10+00:00", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22346893394109", "owner_id": 6569096478909, "owner_resource": "customer", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203938517} +{"stream": "metafield_draft_orders", "data": {"id": 22366258528445, "namespace": "discounts", "value": "50%", "key": "hello", "description": null, "created_at": "2023-04-14T12:16:33+00:00", "updated_at": "2023-04-14T12:16:33+00:00", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22366258528445", "owner_id": 874683629757, "owner_resource": "draft_order", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203954874} +{"stream": "metafield_draft_orders", "data": {"id": 22532787175613, "namespace": "new_metafield", "value": "updated_mon_24.04.2023", "key": "new_metafield", "description": null, "created_at": "2023-04-24T14:18:06+00:00", "updated_at": "2023-04-24T14:18:06+00:00", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22532787175613", "owner_id": 929019691197, "owner_resource": "draft_order", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203954875} +{"stream": "metafield_locations", "data": {"id": 26246034161853, "namespace": "custom", "value": "2023-11-28 updated", "key": "test_location_metafield", "description": null, "created_at": "2023-11-28T15:08:26+00:00", "updated_at": "2023-11-28T15:08:26+00:00", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/26246034161853", "owner_id": 63590301885, "owner_resource": "location", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203975501} +{"stream": "metafield_orders", "data": {"id": 22347288150205, "namespace": "my_fields", "value": "asdfasdf", "key": "purchase_order", "description": null, "created_at": "2023-04-13T12:09:50+00:00", "updated_at": "2023-04-13T12:09:50+00:00", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22347288150205", "owner_id": 5010584895677, "owner_resource": "order", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203982564} +{"stream": "metafield_orders", "data": {"id": 22347288740029, "namespace": "my_fields", "value": "asdfasdfasdf", "key": "purchase_order", "description": null, "created_at": "2023-04-13T12:11:20+00:00", "updated_at": "2023-04-13T12:11:20+00:00", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22347288740029", "owner_id": 5010585911485, "owner_resource": "order", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203982565} +{"stream": "metafield_orders", "data": {"id": 22347287855293, "namespace": "my_fields", "value": "trtrtr", "key": "purchase_order", "description": null, "created_at": "2023-04-13T12:09:08+00:00", "updated_at": "2023-04-13T12:09:08+00:00", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22347287855293", "owner_id": 4147980107965, "owner_resource": "order", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203982566} +{"stream": "metafield_pages", "data": {"id": 22365711499453, "namespace": "custom", "key": "test_page_metafield", "value": "Test Page Metafield", "description": null, "owner_id": 93795909821, "created_at": "2023-04-14T03:21:49-07:00", "updated_at": "2023-04-14T03:21:49-07:00", "owner_resource": "page", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365711499453", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203991757} +{"stream": "metafield_pages", "data": {"id": 22534014828733, "namespace": "new_metafield", "key": "new_metafield", "value": "updated_mon_24.04.2023", "description": null, "owner_id": 83074252989, "created_at": "2023-04-24T11:08:41-07:00", "updated_at": "2023-04-24T11:08:41-07:00", "owner_resource": "page", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22534014828733", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203992302} +{"stream": "metafield_product_images", "data": {"id": 22365851517117, "namespace": "my_fields", "value": "natural coton", "key": "liner_material", "description": null, "created_at": "2023-04-14T11:59:27+00:00", "updated_at": "2023-04-14T11:59:27+00:00", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365851517117", "owner_id": 29301295481021, "owner_resource": "product_image", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203999812} +{"stream": "metafield_product_images", "data": {"id": 22533588451517, "namespace": "new_metafield", "value": "updated_mon_24.04.2023", "key": "new_metafield", "description": null, "created_at": "2023-04-24T17:32:19+00:00", "updated_at": "2023-04-24T17:32:19+00:00", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22533588451517", "owner_id": 29301297316029, "owner_resource": "product_image", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701203999814} +{"stream": "metafield_products", "data": {"id": 22365729718461, "namespace": "custom", "value": "Test Product Metafield", "key": "product_metafield_test_2", "description": null, "created_at": "2023-04-14T10:31:19+00:00", "updated_at": "2023-04-14T10:31:19+00:00", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365729718461", "owner_id": 6796226560189, "owner_resource": "product", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204020639} +{"stream": "metafield_products", "data": {"id": 22365729816765, "namespace": "custom", "value": "gid://shopify/Product/6796229574845", "key": "test_product_metafield", "description": null, "created_at": "2023-04-14T10:31:29+00:00", "updated_at": "2023-04-14T10:31:29+00:00", "type": "product_reference", "admin_graphql_api_id": "gid://shopify/Metafield/22365729816765", "owner_id": 6796226560189, "owner_resource": "product", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204020640} +{"stream": "metafield_products", "data": {"id": 22365723689149, "namespace": "custom", "value": "gid://shopify/Product/6796220989629", "key": "test_product_metafield", "description": null, "created_at": "2023-04-14T10:28:30+00:00", "updated_at": "2023-04-14T10:28:30+00:00", "type": "product_reference", "admin_graphql_api_id": "gid://shopify/Metafield/22365723689149", "owner_id": 6796229509309, "owner_resource": "product", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204020641} +{"stream": "metafield_product_variants", "data": {"id": 22365715955901, "namespace": "custom", "value": "Test Varia", "key": "test_variant_metafield", "description": null, "created_at": "2023-04-14T10:24:03+00:00", "updated_at": "2023-04-14T10:24:03+00:00", "type": "multi_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365715955901", "owner_id": 41561961824445, "owner_resource": "product_variant", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204046576} +{"stream": "metafield_product_variants", "data": {"id": 22365724082365, "namespace": "custom", "value": "Test Varia", "key": "test_variant_metafield", "description": null, "created_at": "2023-04-14T10:29:27+00:00", "updated_at": "2023-04-14T10:29:27+00:00", "type": "multi_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22365724082365", "owner_id": 42778150305981, "owner_resource": "product_variant", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204046577} +{"stream": "metafield_shops", "data": {"id": 19716782129341, "namespace": "inventory", "key": "warehouse", "value": "26", "description": null, "owner_id": 58033176765, "created_at": "2021-07-08T03:38:45-07:00", "updated_at": "2023-04-14T05:47:26-07:00", "owner_resource": "shop", "type": "number_integer", "admin_graphql_api_id": "gid://shopify/Metafield/19716782129341", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204048569} +{"stream": "metafield_shops", "data": {"id": 22534020104381, "namespace": "new_metafield", "key": "new_metafield", "value": "updated_mon_24.04.2023", "description": null, "owner_id": 58033176765, "created_at": "2023-04-24T11:12:38-07:00", "updated_at": "2023-04-24T11:12:38-07:00", "owner_resource": "shop", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22534020104381", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204048570} +{"stream": "metafield_smart_collections", "data": {"id": 21525604106429, "namespace": "my_fields", "key": "discount", "value": "50%", "description": null, "owner_id": 273278566589, "created_at": "2022-10-12T13:36:55-07:00", "updated_at": "2022-10-12T13:36:55-07:00", "owner_resource": "collection", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/21525604106429", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204050581} +{"stream": "metafield_smart_collections", "data": {"id": 22366265573565, "namespace": "my_fields", "key": "new_key", "value": "51%", "description": null, "owner_id": 273278566589, "created_at": "2023-04-14T05:21:58-07:00", "updated_at": "2023-04-14T05:21:58-07:00", "owner_resource": "collection", "type": "single_line_text_field", "admin_graphql_api_id": "gid://shopify/Metafield/22366265573565", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204050582} +{"stream": "order_refunds", "data": {"id": 829538369725, "admin_graphql_api_id": "gid://shopify/Refund/829538369725", "created_at": "2021-09-21T05:31:59-07:00", "note": "test refund", "order_id": 3935377129661, "processed_at": "2021-09-21T05:31:59-07:00", "restock": true, "total_duties_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "user_id": 74861019325, "order_adjustments": [], "transactions": [{"id": 5189894406333, "admin_graphql_api_id": "gid://shopify/OrderTransaction/5189894406333", "amount": "102.00", "authorization": null, "created_at": "2021-09-21T05:31:58-07:00", "currency": "USD", "device_id": null, "error_code": null, "gateway": "bogus", "kind": "refund", "location_id": null, "message": "Bogus Gateway: Forced success", "order_id": 3935377129661, "parent_id": 4933790040253, "payment_id": "c21670281707709.2", "processed_at": "2021-09-21T05:31:58-07:00", "receipt": {"paid_amount": "102.00"}, "source_name": "1830279", "status": "success", "test": true, "user_id": 74861019325, "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null, "credit_card_name": "Bogus Gateway", "credit_card_wallet": null, "credit_card_expiration_month": 11, "credit_card_expiration_year": 2023}}], "refund_line_items": [{"id": 332807864509, "line_item_id": 10130216452285, "location_id": 63590301885, "quantity": 1, "restock_type": "cancel", "subtotal": 102.0, "subtotal_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "total_tax": 17.0, "total_tax_set": {"shop_money": {"amount": 17.0, "currency_code": "USD"}, "presentment_money": {"amount": 17.0, "currency_code": "USD"}}, "line_item": {"id": 10130216452285, "admin_graphql_api_id": "gid://shopify/LineItem/10130216452285", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": null, "gift_card": false, "grams": 63, "name": "8 Ounce Soy Candle - Wooden", "price": 102.0, "price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796229509309, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "8 Ounce Soy Candle", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 40090604011709, "variant_inventory_management": "shopify", "variant_title": "Wooden", "vendor": "Bosco Inc", "tax_lines": [{"channel_liable": false, "price": 17.0, "price_set": {"shop_money": {"amount": 17.0, "currency_code": "USD"}, "presentment_money": {"amount": 17.0, "currency_code": "USD"}}, "rate": 0.2, "title": "PDV"}], "duties": [], "discount_allocations": []}}], "duties": "[]", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701367977468} +{"stream": "order_refunds", "data": {"id": 845032358077, "admin_graphql_api_id": "gid://shopify/Refund/845032358077", "created_at": "2022-03-07T02:09:04-08:00", "note": null, "order_id": 4147980107965, "processed_at": "2022-03-07T02:09:04-08:00", "restock": true, "total_duties_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "user_id": 74861019325, "order_adjustments": [], "transactions": [], "refund_line_items": [{"id": 352716947645, "line_item_id": 10576771317949, "location_id": 63590301885, "quantity": 1, "restock_type": "return", "subtotal": 27.0, "subtotal_set": {"shop_money": {"amount": 27.0, "currency_code": "USD"}, "presentment_money": {"amount": 27.0, "currency_code": "USD"}}, "total_tax": 0.0, "total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "line_item": {"id": 10576771317949, "admin_graphql_api_id": "gid://shopify/LineItem/10576771317949", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 285, "name": "Red & Silver Fishing Lure - Plastic", "price": 27.0, "price_set": {"shop_money": {"amount": 27.0, "currency_code": "USD"}, "presentment_money": {"amount": 27.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796218302653, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "Red & Silver Fishing Lure", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 40090580615357, "variant_inventory_management": "shopify", "variant_title": "Plastic", "vendor": "Harris - Hamill", "tax_lines": [], "duties": [], "discount_allocations": []}}], "duties": "[]", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701367977465} +{"stream": "order_refunds", "data": {"id": 852809646269, "admin_graphql_api_id": "gid://shopify/Refund/852809646269", "created_at": "2022-06-15T06:25:43-07:00", "note": null, "order_id": 4554821468349, "processed_at": "2022-06-15T06:25:43-07:00", "restock": true, "total_duties_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "user_id": 74861019325, "order_adjustments": [], "transactions": [{"id": 5721170968765, "admin_graphql_api_id": "gid://shopify/OrderTransaction/5721170968765", "amount": "57.23", "authorization": null, "created_at": "2022-06-15T06:25:42-07:00", "currency": "USD", "device_id": null, "error_code": null, "gateway": "bogus", "kind": "refund", "location_id": null, "message": "Bogus Gateway: Forced success", "order_id": 4554821468349, "parent_id": 5721110872253, "payment_id": "c25048437719229.2", "processed_at": "2022-06-15T06:25:42-07:00", "receipt": {"paid_amount": "57.23"}, "source_name": "1830279", "status": "success", "test": true, "user_id": null, "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null, "credit_card_name": "Bogus Gateway", "credit_card_wallet": null, "credit_card_expiration_month": 2, "credit_card_expiration_year": 2025}}], "refund_line_items": [{"id": 363131404477, "line_item_id": 11406125564093, "location_id": 63590301885, "quantity": 1, "restock_type": "return", "subtotal": 57.23, "subtotal_set": {"shop_money": {"amount": 57.23, "currency_code": "USD"}, "presentment_money": {"amount": 57.23, "currency_code": "USD"}}, "total_tax": 0.0, "total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "line_item": {"id": 11406125564093, "admin_graphql_api_id": "gid://shopify/LineItem/11406125564093", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 0, "name": "All Black Sneaker Right Foot - ivory", "price": 59.0, "price_set": {"shop_money": {"amount": 59.0, "currency_code": "USD"}, "presentment_money": {"amount": 59.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796226560189, "properties": [], "quantity": 1, "requires_shipping": false, "sku": "", "taxable": true, "title": "All Black Sneaker Right Foot", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 40090597884093, "variant_inventory_management": "shopify", "variant_title": "ivory", "vendor": "Becker - Moore", "tax_lines": [], "duties": [], "discount_allocations": [{"amount": 1.77, "amount_set": {"shop_money": {"amount": 1.77, "currency_code": "USD"}, "presentment_money": {"amount": 1.77, "currency_code": "USD"}}, "discount_application_index": 0}]}}], "duties": "[]", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701367977460} +{"stream": "order_risks", "data": {"id": 6446736474301, "order_id": 4147980107965, "checkout_id": null, "source": "External", "score": 1.0, "recommendation": "cancel", "display": true, "cause_cancel": true, "message": "This order came from an anonymous proxy", "merchant_message": "This order came from an anonymous proxy", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204058573} +{"stream": "orders", "data": {"id": 5010584895677, "admin_graphql_api_id": "gid://shopify/Order/5010584895677", "app_id": 1354745, "browser_ip": "109.162.18.117", "buyer_accepts_marketing": false, "cancel_reason": null, "cancelled_at": null, "cart_token": null, "checkout_id": 27351199088829, "checkout_token": "4064bfadc2457c9e15f2c7b4ee7ddb7d", "client_details": {"accept_language": null, "browser_height": null, "browser_ip": "109.162.18.117", "browser_width": null, "session_hash": null, "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36"}, "closed_at": "2023-04-13T05:09:46-07:00", "company": null, "confirmation_number": "ECI3YZGWP", "confirmed": true, "contact_email": null, "created_at": "2023-04-13T05:09:44-07:00", "currency": "USD", "current_subtotal_price": 102.0, "current_subtotal_price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "current_total_additional_fees_set": null, "current_total_discounts": 0.0, "current_total_discounts_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_duties_set": null, "current_total_price": 102.0, "current_total_price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "current_total_tax": 17.0, "current_total_tax_set": {"shop_money": {"amount": 17.0, "currency_code": "USD"}, "presentment_money": {"amount": 17.0, "currency_code": "USD"}}, "customer_locale": "en", "device_id": null, "discount_codes": [], "email": "", "estimated_taxes": false, "financial_status": "paid", "fulfillment_status": "fulfilled", "landing_site": null, "landing_site_ref": null, "location_id": 63590301885, "merchant_of_record_app_id": null, "name": "#1143", "note": null, "note_attributes": [], "number": 143, "order_number": 1143, "order_status_url": "https://airbyte-integration-test.myshopify.com/58033176765/orders/ad85969259bee7d4b380744934e67556/authenticate?key=cdd70808759b04f31c29975ba796fce9", "original_total_additional_fees_set": null, "original_total_duties_set": null, "payment_gateway_names": ["manual"], "phone": null, "po_number": null, "presentment_currency": "USD", "processed_at": "2023-04-13T05:09:44-07:00", "reference": "b9344c8b118753db132edd503dc91515", "referring_site": null, "source_identifier": "b9344c8b118753db132edd503dc91515", "source_name": "shopify_draft_order", "source_url": null, "subtotal_price": 102.0, "subtotal_price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "tags": "", "tax_exempt": false, "tax_lines": [{"price": 17.0, "rate": 0.2, "title": "PDV", "price_set": {"shop_money": {"amount": 17.0, "currency_code": "USD"}, "presentment_money": {"amount": 17.0, "currency_code": "USD"}}, "channel_liable": false}], "taxes_included": true, "test": false, "token": "ad85969259bee7d4b380744934e67556", "total_discounts": 0.0, "total_discounts_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_line_items_price": 102.0, "total_line_items_price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "total_outstanding": 0.0, "total_price": 102.0, "total_price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "total_shipping_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_tax": 17.0, "total_tax_set": {"shop_money": {"amount": 17.0, "currency_code": "USD"}, "presentment_money": {"amount": 17.0, "currency_code": "USD"}}, "total_tip_received": 0.0, "total_weight": 63, "updated_at": "2023-04-13T05:09:50-07:00", "user_id": 74861019325, "billing_address": null, "customer": null, "discount_applications": [], "fulfillments": [{"id": 4451164913853, "admin_graphql_api_id": "gid://shopify/Fulfillment/4451164913853", "created_at": "2023-04-13T05:09:45-07:00", "location_id": 63590301885, "name": "#1143.1", "order_id": 5010584895677, "origin_address": {}, "receipt": {}, "service": "manual", "shipment_status": null, "status": "success", "tracking_company": null, "tracking_number": null, "tracking_numbers": [], "tracking_url": null, "tracking_urls": [], "updated_at": "2023-04-13T05:09:45-07:00", "line_items": [{"id": 12204214845629, "admin_graphql_api_id": "gid://shopify/LineItem/12204214845629", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 63, "name": "8 Ounce Soy Candle - Wooden", "price": 102.0, "price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796229509309, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "8 Ounce Soy Candle", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 40090604011709, "variant_inventory_management": "shopify", "variant_title": "Wooden", "vendor": "Bosco Inc", "tax_lines": [{"channel_liable": false, "price": 17.0, "price_set": {"shop_money": {"amount": 17.0, "currency_code": "USD"}, "presentment_money": {"amount": 17.0, "currency_code": "USD"}}, "rate": 0.2, "title": "PDV"}], "duties": [], "discount_allocations": []}]}], "line_items": [{"id": 12204214845629, "admin_graphql_api_id": "gid://shopify/LineItem/12204214845629", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 63, "name": "8 Ounce Soy Candle - Wooden", "price": 102.0, "price_set": {"shop_money": {"amount": 102.0, "currency_code": "USD"}, "presentment_money": {"amount": 102.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796229509309, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "8 Ounce Soy Candle", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 40090604011709, "variant_inventory_management": "shopify", "variant_title": "Wooden", "vendor": "Bosco Inc", "tax_lines": [{"channel_liable": false, "price": 17.0, "price_set": {"shop_money": {"amount": 17.0, "currency_code": "USD"}, "presentment_money": {"amount": 17.0, "currency_code": "USD"}}, "rate": 0.2, "title": "PDV"}], "duties": [], "discount_allocations": []}], "payment_terms": null, "refunds": [], "shipping_address": null, "shipping_lines": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204061170} +{"stream": "orders", "data": {"id": 5010585911485, "admin_graphql_api_id": "gid://shopify/Order/5010585911485", "app_id": 1354745, "browser_ip": "109.162.18.117", "buyer_accepts_marketing": false, "cancel_reason": null, "cancelled_at": null, "cart_token": null, "checkout_id": 27351203774653, "checkout_token": "117b35b4fd64c8de8d984830b72edfaf", "client_details": {"accept_language": null, "browser_height": null, "browser_ip": "109.162.18.117", "browser_width": null, "session_hash": null, "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36"}, "closed_at": "2023-04-13T05:11:17-07:00", "company": null, "confirmation_number": "CQPWZK5ZU", "confirmed": true, "contact_email": null, "created_at": "2023-04-13T05:11:15-07:00", "currency": "USD", "current_subtotal_price": 19.0, "current_subtotal_price_set": {"shop_money": {"amount": 19.0, "currency_code": "USD"}, "presentment_money": {"amount": 19.0, "currency_code": "USD"}}, "current_total_additional_fees_set": null, "current_total_discounts": 0.0, "current_total_discounts_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_duties_set": null, "current_total_price": 19.0, "current_total_price_set": {"shop_money": {"amount": 19.0, "currency_code": "USD"}, "presentment_money": {"amount": 19.0, "currency_code": "USD"}}, "current_total_tax": 3.17, "current_total_tax_set": {"shop_money": {"amount": 3.17, "currency_code": "USD"}, "presentment_money": {"amount": 3.17, "currency_code": "USD"}}, "customer_locale": "en", "device_id": null, "discount_codes": [], "email": "", "estimated_taxes": false, "financial_status": "paid", "fulfillment_status": "fulfilled", "landing_site": null, "landing_site_ref": null, "location_id": 63590301885, "merchant_of_record_app_id": null, "name": "#1144", "note": null, "note_attributes": [], "number": 144, "order_number": 1144, "order_status_url": "https://airbyte-integration-test.myshopify.com/58033176765/orders/a2d4a8a04fc6ec52a85f1811d269c88f/authenticate?key=a26b27cc1a040df3a1d7c67f6b14df75", "original_total_additional_fees_set": null, "original_total_duties_set": null, "payment_gateway_names": ["manual"], "phone": null, "po_number": null, "presentment_currency": "USD", "processed_at": "2023-04-13T05:11:15-07:00", "reference": "f75c72a120e34e15a4dbc2d32315cc72", "referring_site": null, "source_identifier": "f75c72a120e34e15a4dbc2d32315cc72", "source_name": "shopify_draft_order", "source_url": null, "subtotal_price": 19.0, "subtotal_price_set": {"shop_money": {"amount": 19.0, "currency_code": "USD"}, "presentment_money": {"amount": 19.0, "currency_code": "USD"}}, "tags": "", "tax_exempt": false, "tax_lines": [{"price": 3.17, "rate": 0.2, "title": "PDV", "price_set": {"shop_money": {"amount": 3.17, "currency_code": "USD"}, "presentment_money": {"amount": 3.17, "currency_code": "USD"}}, "channel_liable": false}], "taxes_included": true, "test": false, "token": "a2d4a8a04fc6ec52a85f1811d269c88f", "total_discounts": 0.0, "total_discounts_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_line_items_price": 19.0, "total_line_items_price_set": {"shop_money": {"amount": 19.0, "currency_code": "USD"}, "presentment_money": {"amount": 19.0, "currency_code": "USD"}}, "total_outstanding": 0.0, "total_price": 19.0, "total_price_set": {"shop_money": {"amount": 19.0, "currency_code": "USD"}, "presentment_money": {"amount": 19.0, "currency_code": "USD"}}, "total_shipping_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_tax": 3.17, "total_tax_set": {"shop_money": {"amount": 3.17, "currency_code": "USD"}, "presentment_money": {"amount": 3.17, "currency_code": "USD"}}, "total_tip_received": 0.0, "total_weight": 112, "updated_at": "2023-04-13T05:11:20-07:00", "user_id": 74861019325, "billing_address": null, "customer": null, "discount_applications": [], "fulfillments": [{"id": 4451169501373, "admin_graphql_api_id": "gid://shopify/Fulfillment/4451169501373", "created_at": "2023-04-13T05:11:16-07:00", "location_id": 63590301885, "name": "#1144.1", "order_id": 5010585911485, "origin_address": {}, "receipt": {}, "service": "manual", "shipment_status": null, "status": "success", "tracking_company": null, "tracking_number": null, "tracking_numbers": [], "tracking_url": null, "tracking_urls": [], "updated_at": "2023-04-13T05:11:16-07:00", "line_items": [{"id": 12204216385725, "admin_graphql_api_id": "gid://shopify/LineItem/12204216385725", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 112, "name": "4 Ounce Soy Candle - Test Variant 2", "price": 19.0, "price_set": {"shop_money": {"amount": 19.0, "currency_code": "USD"}, "presentment_money": {"amount": 19.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796220989629, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "4 Ounce Soy Candle", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 41561961824445, "variant_inventory_management": "shopify", "variant_title": "Test Variant 2", "vendor": "Hartmann Group", "tax_lines": [{"channel_liable": false, "price": 3.17, "price_set": {"shop_money": {"amount": 3.17, "currency_code": "USD"}, "presentment_money": {"amount": 3.17, "currency_code": "USD"}}, "rate": 0.2, "title": "PDV"}], "duties": [], "discount_allocations": []}]}], "line_items": [{"id": 12204216385725, "admin_graphql_api_id": "gid://shopify/LineItem/12204216385725", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 112, "name": "4 Ounce Soy Candle - Test Variant 2", "price": 19.0, "price_set": {"shop_money": {"amount": 19.0, "currency_code": "USD"}, "presentment_money": {"amount": 19.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796220989629, "properties": [], "quantity": 1, "requires_shipping": true, "sku": "", "taxable": true, "title": "4 Ounce Soy Candle", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 41561961824445, "variant_inventory_management": "shopify", "variant_title": "Test Variant 2", "vendor": "Hartmann Group", "tax_lines": [{"channel_liable": false, "price": 3.17, "price_set": {"shop_money": {"amount": 3.17, "currency_code": "USD"}, "presentment_money": {"amount": 3.17, "currency_code": "USD"}}, "rate": 0.2, "title": "PDV"}], "duties": [], "discount_allocations": []}], "payment_terms": null, "refunds": [], "shipping_address": null, "shipping_lines": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204061174} +{"stream": "orders", "data": {"id": 4554821468349, "admin_graphql_api_id": "gid://shopify/Order/4554821468349", "app_id": 580111, "browser_ip": "176.113.167.23", "buyer_accepts_marketing": false, "cancel_reason": null, "cancelled_at": null, "cart_token": null, "checkout_id": 25048437719229, "checkout_token": "cf5d16a0a0688905bd551c6dec591506", "client_details": {"accept_language": "en-US,en;q=0.9,uk;q=0.8", "browser_height": 754, "browser_ip": "176.113.167.23", "browser_width": 1519, "session_hash": null, "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36 Edg/101.0.1210.53"}, "closed_at": "2022-06-15T06:25:43-07:00", "company": null, "confirmation_number": null, "confirmed": true, "contact_email": "integration-test@airbyte.io", "created_at": "2022-06-15T05:16:53-07:00", "currency": "USD", "current_subtotal_price": 0.0, "current_subtotal_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_additional_fees_set": null, "current_total_discounts": 0.0, "current_total_discounts_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_duties_set": null, "current_total_price": 0.0, "current_total_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "current_total_tax": 0.0, "current_total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "customer_locale": "en", "device_id": null, "discount_codes": [], "email": "integration-test@airbyte.io", "estimated_taxes": false, "financial_status": "refunded", "fulfillment_status": "fulfilled", "landing_site": "/wallets/checkouts.json", "landing_site_ref": null, "location_id": null, "merchant_of_record_app_id": null, "name": "#1136", "note": "updated_mon_24.04.2023", "note_attributes": [], "number": 136, "order_number": 1136, "order_status_url": "https://airbyte-integration-test.myshopify.com/58033176765/orders/e4f98630ea44a884e33e700203ce2130/authenticate?key=edf087d6ae55a4541bf1375432f6a4b8", "original_total_additional_fees_set": null, "original_total_duties_set": null, "payment_gateway_names": ["bogus"], "phone": null, "po_number": null, "presentment_currency": "USD", "processed_at": "2022-06-15T05:16:53-07:00", "reference": null, "referring_site": "https://airbyte-integration-test.myshopify.com/products/all-black-sneaker-right-foot", "source_identifier": null, "source_name": "web", "source_url": null, "subtotal_price": 57.23, "subtotal_price_set": {"shop_money": {"amount": 57.23, "currency_code": "USD"}, "presentment_money": {"amount": 57.23, "currency_code": "USD"}}, "tags": "Refund", "tax_exempt": false, "tax_lines": [], "taxes_included": true, "test": true, "token": "e4f98630ea44a884e33e700203ce2130", "total_discounts": 1.77, "total_discounts_set": {"shop_money": {"amount": 1.77, "currency_code": "USD"}, "presentment_money": {"amount": 1.77, "currency_code": "USD"}}, "total_line_items_price": 59.0, "total_line_items_price_set": {"shop_money": {"amount": 59.0, "currency_code": "USD"}, "presentment_money": {"amount": 59.0, "currency_code": "USD"}}, "total_outstanding": 0.0, "total_price": 57.23, "total_price_set": {"shop_money": {"amount": 57.23, "currency_code": "USD"}, "presentment_money": {"amount": 57.23, "currency_code": "USD"}}, "total_shipping_price_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_tax": 0.0, "total_tax_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "total_tip_received": 0.0, "total_weight": 0, "updated_at": "2023-04-24T07:00:37-07:00", "user_id": null, "billing_address": {"first_name": "Iryna", "address1": "2261 Market Street", "phone": null, "city": "San Francisco", "zip": "94114", "province": "California", "country": "United States", "last_name": "Grankova", "address2": "4381", "company": null, "latitude": 37.7647751, "longitude": -122.4320369, "name": "Iryna Grankova", "country_code": "US", "province_code": "CA"}, "customer": {"id": 5362027233469, "email": "integration-test@airbyte.io", "accepts_marketing": false, "created_at": "2021-07-08T05:41:47-07:00", "updated_at": "2022-06-22T03:50:13-07:00", "first_name": "Airbyte", "last_name": "Team", "state": "disabled", "note": null, "verified_email": true, "multipass_identifier": null, "tax_exempt": false, "phone": null, "email_marketing_consent": {"state": "not_subscribed", "opt_in_level": "single_opt_in", "consent_updated_at": null}, "sms_marketing_consent": null, "tags": "", "currency": "USD", "accepts_marketing_updated_at": "2021-07-08T05:41:47-07:00", "marketing_opt_in_level": null, "tax_exemptions": [], "admin_graphql_api_id": "gid://shopify/Customer/5362027233469", "default_address": {"id": 7492260823229, "customer_id": 5362027233469, "first_name": "Airbyte", "last_name": "Team", "company": null, "address1": "2261 Market Street", "address2": "4381", "city": "San Francisco", "province": "California", "country": "United States", "zip": "94114", "phone": null, "name": "Airbyte Team", "province_code": "CA", "country_code": "US", "country_name": "United States", "default": true}}, "discount_applications": [{"target_type": "line_item", "type": "automatic", "value": "3.0", "value_type": "percentage", "allocation_method": "across", "target_selection": "all", "title": "eeeee"}], "fulfillments": [{"id": 4075788501181, "admin_graphql_api_id": "gid://shopify/Fulfillment/4075788501181", "created_at": "2022-06-15T05:16:55-07:00", "location_id": 63590301885, "name": "#1136.1", "order_id": 4554821468349, "origin_address": {}, "receipt": {}, "service": "manual", "shipment_status": null, "status": "success", "tracking_company": null, "tracking_number": null, "tracking_numbers": [], "tracking_url": null, "tracking_urls": [], "updated_at": "2022-06-15T05:16:55-07:00", "line_items": [{"id": 11406125564093, "admin_graphql_api_id": "gid://shopify/LineItem/11406125564093", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 0, "name": "All Black Sneaker Right Foot - ivory", "price": 59.0, "price_set": {"shop_money": {"amount": 59.0, "currency_code": "USD"}, "presentment_money": {"amount": 59.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796226560189, "properties": [], "quantity": 1, "requires_shipping": false, "sku": "", "taxable": true, "title": "All Black Sneaker Right Foot", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 40090597884093, "variant_inventory_management": "shopify", "variant_title": "ivory", "vendor": "Becker - Moore", "tax_lines": [], "duties": [], "discount_allocations": [{"amount": "1.77", "amount_set": {"shop_money": {"amount": "1.77", "currency_code": "USD"}, "presentment_money": {"amount": "1.77", "currency_code": "USD"}}, "discount_application_index": 0}]}]}], "line_items": [{"id": 11406125564093, "admin_graphql_api_id": "gid://shopify/LineItem/11406125564093", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 0, "name": "All Black Sneaker Right Foot - ivory", "price": 59.0, "price_set": {"shop_money": {"amount": 59.0, "currency_code": "USD"}, "presentment_money": {"amount": 59.0, "currency_code": "USD"}}, "product_exists": true, "product_id": 6796226560189, "properties": [], "quantity": 1, "requires_shipping": false, "sku": "", "taxable": true, "title": "All Black Sneaker Right Foot", "total_discount": 0.0, "total_discount_set": {"shop_money": {"amount": 0.0, "currency_code": "USD"}, "presentment_money": {"amount": 0.0, "currency_code": "USD"}}, "variant_id": 40090597884093, "variant_inventory_management": "shopify", "variant_title": "ivory", "vendor": "Becker - Moore", "tax_lines": [], "duties": [], "discount_allocations": [{"amount": "1.77", "amount_set": {"shop_money": {"amount": "1.77", "currency_code": "USD"}, "presentment_money": {"amount": "1.77", "currency_code": "USD"}}, "discount_application_index": 0}]}], "payment_terms": null, "refunds": [{"id": 852809646269, "admin_graphql_api_id": "gid://shopify/Refund/852809646269", "created_at": "2022-06-15T06:25:43-07:00", "note": null, "order_id": 4554821468349, "processed_at": "2022-06-15T06:25:43-07:00", "restock": true, "total_duties_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "user_id": 74861019325, "order_adjustments": [], "transactions": [{"id": 5721170968765, "admin_graphql_api_id": "gid://shopify/OrderTransaction/5721170968765", "amount": "57.23", "authorization": null, "created_at": "2022-06-15T06:25:42-07:00", "currency": "USD", "device_id": null, "error_code": null, "gateway": "bogus", "kind": "refund", "location_id": null, "message": "Bogus Gateway: Forced success", "order_id": 4554821468349, "parent_id": 5721110872253, "payment_id": "c25048437719229.2", "processed_at": "2022-06-15T06:25:42-07:00", "receipt": {"paid_amount": "57.23"}, "source_name": "1830279", "status": "success", "test": true, "user_id": null, "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null, "credit_card_name": "Bogus Gateway", "credit_card_wallet": null, "credit_card_expiration_month": 2, "credit_card_expiration_year": 2025}}], "refund_line_items": [{"id": 363131404477, "line_item_id": 11406125564093, "location_id": 63590301885, "quantity": 1, "restock_type": "return", "subtotal": 57.23, "subtotal_set": {"shop_money": {"amount": "57.23", "currency_code": "USD"}, "presentment_money": {"amount": "57.23", "currency_code": "USD"}}, "total_tax": 0.0, "total_tax_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "line_item": {"id": 11406125564093, "admin_graphql_api_id": "gid://shopify/LineItem/11406125564093", "fulfillable_quantity": 0, "fulfillment_service": "manual", "fulfillment_status": "fulfilled", "gift_card": false, "grams": 0, "name": "All Black Sneaker Right Foot - ivory", "price": "59.00", "price_set": {"shop_money": {"amount": "59.00", "currency_code": "USD"}, "presentment_money": {"amount": "59.00", "currency_code": "USD"}}, "product_exists": true, "product_id": 6796226560189, "properties": [], "quantity": 1, "requires_shipping": false, "sku": "", "taxable": true, "title": "All Black Sneaker Right Foot", "total_discount": "0.00", "total_discount_set": {"shop_money": {"amount": "0.00", "currency_code": "USD"}, "presentment_money": {"amount": "0.00", "currency_code": "USD"}}, "variant_id": 40090597884093, "variant_inventory_management": "shopify", "variant_title": "ivory", "vendor": "Becker - Moore", "tax_lines": [], "duties": [], "discount_allocations": [{"amount": "1.77", "amount_set": {"shop_money": {"amount": "1.77", "currency_code": "USD"}, "presentment_money": {"amount": "1.77", "currency_code": "USD"}}, "discount_application_index": 0}]}}], "duties": []}], "shipping_address": null, "shipping_lines": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204061177} +{"stream": "pages", "data": {"id": 93795909821, "title": "Test Page", "shop_id": 58033176765, "handle": "test-page", "body_html": "Test Page 1", "author": null, "created_at": "2023-04-14T03:21:40-07:00", "updated_at": "2023-04-14T03:21:49-07:00", "published_at": "2023-04-14T03:21:40-07:00", "template_suffix": "", "admin_graphql_api_id": "gid://shopify/OnlineStorePage/93795909821", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204062878} +{"stream": "pages", "data": {"id": 83074252989, "title": "Warranty information", "shop_id": 58033176765, "handle": "warranty-information", "body_html": "updated_mon_24.04.2023", "author": "Shopify API", "created_at": "2021-07-08T05:19:00-07:00", "updated_at": "2023-04-24T11:08:41-07:00", "published_at": "2021-07-08T05:19:00-07:00", "template_suffix": null, "admin_graphql_api_id": "gid://shopify/OnlineStorePage/83074252989", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204062879} +{"stream": "pages", "data": {"id": 95926616253, "deleted_at": "2023-09-06T03:37:06-07:00", "updated_at": "2023-09-06T03:37:06-07:00", "deleted_message": "Online Store deleted a page: Test Page for delete.", "deleted_description": "Online Store deleted a page: Test Page for delete.", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204063419} +{"stream": "price_rules", "data": {"id": 1112171741373, "value_type": "fixed_amount", "value": "-10.0", "customer_selection": "all", "target_type": "line_item", "target_selection": "all", "allocation_method": "across", "allocation_limit": null, "once_per_customer": false, "usage_limit": null, "starts_at": "2017-01-19T09:59:10-08:00", "ends_at": null, "created_at": "2022-10-14T10:19:39-07:00", "updated_at": "2023-04-14T05:24:53-07:00", "entitled_product_ids": [], "entitled_variant_ids": [], "entitled_collection_ids": [], "entitled_country_ids": [], "prerequisite_product_ids": [], "prerequisite_variant_ids": [], "prerequisite_collection_ids": [], "customer_segment_prerequisite_ids": [], "prerequisite_customer_ids": [], "prerequisite_subtotal_range": null, "prerequisite_quantity_range": null, "prerequisite_shipping_price_range": null, "prerequisite_to_entitlement_quantity_ratio": {"prerequisite_quantity": null, "entitled_quantity": null}, "prerequisite_to_entitlement_purchase": {"prerequisite_amount": null}, "title": "New Title 2023", "admin_graphql_api_id": "gid://shopify/PriceRule/1112171741373", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204064436} +{"stream": "price_rules", "data": {"id": 945000284349, "value_type": "percentage", "value": "-3.0", "customer_selection": "all", "target_type": "line_item", "target_selection": "all", "allocation_method": "across", "allocation_limit": null, "once_per_customer": true, "usage_limit": 10, "starts_at": "2021-07-07T07:22:04-07:00", "ends_at": null, "created_at": "2021-07-07T07:23:11-07:00", "updated_at": "2023-04-24T05:52:22-07:00", "entitled_product_ids": [], "entitled_variant_ids": [], "entitled_collection_ids": [], "entitled_country_ids": [], "prerequisite_product_ids": [], "prerequisite_variant_ids": [], "prerequisite_collection_ids": [], "customer_segment_prerequisite_ids": [], "prerequisite_customer_ids": [], "prerequisite_subtotal_range": null, "prerequisite_quantity_range": null, "prerequisite_shipping_price_range": null, "prerequisite_to_entitlement_quantity_ratio": {"prerequisite_quantity": null, "entitled_quantity": null}, "prerequisite_to_entitlement_purchase": {"prerequisite_amount": null}, "title": "1V8Z165KSH5T", "admin_graphql_api_id": "gid://shopify/PriceRule/945000284349", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204064438} +{"stream": "price_rules", "data": {"id": 1278552473789, "deleted_at": "2023-09-06T03:48:46-07:00", "updated_at": "2023-09-06T03:48:46-07:00", "deleted_message": "Airbyte Test deleted this discount.", "deleted_description": "Airbyte Test deleted this discount.", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204064975} +{"stream": "product_images", "data": {"id": 29301295481021, "alt": null, "position": 1, "product_id": 6796218138813, "created_at": "2021-06-22T18:09:28-07:00", "updated_at": "2021-06-22T18:09:28-07:00", "admin_graphql_api_id": "gid://shopify/ProductImage/29301295481021", "width": 4393, "height": 2929, "src": "https://cdn.shopify.com/s/files/1/0580/3317/6765/products/tin-of-beard-balm.jpg?v=1624410568", "variant_ids": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204068318} +{"stream": "product_images", "data": {"id": 29301295513789, "alt": null, "position": 1, "product_id": 6796218269885, "created_at": "2021-06-22T18:09:29-07:00", "updated_at": "2021-06-22T18:09:29-07:00", "admin_graphql_api_id": "gid://shopify/ProductImage/29301295513789", "width": 3840, "height": 2560, "src": "https://cdn.shopify.com/s/files/1/0580/3317/6765/products/pair-of-all-black-sneakers.jpg?v=1624410569", "variant_ids": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204068778} +{"stream": "product_images", "data": {"id": 29301295546557, "alt": null, "position": 1, "product_id": 6796218302653, "created_at": "2021-06-22T18:09:29-07:00", "updated_at": "2021-06-22T18:09:29-07:00", "admin_graphql_api_id": "gid://shopify/ProductImage/29301295546557", "width": 3960, "height": 2640, "src": "https://cdn.shopify.com/s/files/1/0580/3317/6765/products/red-silver-fishing-lure.jpg?v=1624410569", "variant_ids": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204069309} +{"stream": "products", "data": {"id": 6796217909437, "title": "Red And Navy Tee Sleeve", "body_html": "Zoom in on the sleeve of a red t-shirt with navy blue trim along the sleeve. Looks like a great tennis outfit.", "vendor": "Little Group", "product_type": "Movies", "created_at": "2021-06-22T18:09:27-07:00", "handle": "red-and-navy-tee-sleeve", "updated_at": "2023-04-20T04:12:25-07:00", "published_at": "2021-06-22T18:09:27-07:00", "template_suffix": null, "published_scope": "web", "tags": "developer-tools-generator", "status": "active", "admin_graphql_api_id": "gid://shopify/Product/6796217909437", "variants": [{"id": 40090579992765, "product_id": 6796217909437, "title": "Plastic", "price": 23.0, "sku": "", "position": 1, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Plastic", "option2": null, "option3": null, "created_at": "2021-06-22T18:09:27-07:00", "updated_at": "2023-10-27T09:55:54-07:00", "taxable": true, "barcode": null, "grams": 39, "image_id": null, "weight": 39.0, "weight_unit": "g", "inventory_item_id": 42185194700989, "inventory_quantity": 3, "old_inventory_quantity": 3, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/40090579992765"}], "options": [{"id": 8720175235261, "product_id": 6796217909437, "name": "Title", "position": 1, "values": ["Plastic"]}], "images": [], "image": null, "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204167548} +{"stream": "products", "data": {"id": 6796217942205, "title": "Grey T-Shirt", "body_html": "A grey t-shirt on a hanger. Simple. Classic. Grey.", "vendor": "Lang - Bogisich", "product_type": "Home", "created_at": "2021-06-22T18:09:27-07:00", "handle": "grey-t-shirt", "updated_at": "2023-04-20T04:12:25-07:00", "published_at": "2021-06-22T18:09:27-07:00", "template_suffix": null, "published_scope": "web", "tags": "developer-tools-generator", "status": "active", "admin_graphql_api_id": "gid://shopify/Product/6796217942205", "variants": [{"id": 40090580025533, "product_id": 6796217942205, "title": "Granite", "price": 70.0, "sku": "", "position": 1, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Granite", "option2": null, "option3": null, "created_at": "2021-06-22T18:09:27-07:00", "updated_at": "2023-10-27T09:55:54-07:00", "taxable": true, "barcode": null, "grams": 0, "image_id": null, "weight": 0.0, "weight_unit": "g", "inventory_item_id": 42185194733757, "inventory_quantity": 38, "old_inventory_quantity": 38, "requires_shipping": false, "admin_graphql_api_id": "gid://shopify/ProductVariant/40090580025533"}], "options": [{"id": 8720175268029, "product_id": 6796217942205, "name": "Title", "position": 1, "values": ["Granite"]}], "images": [], "image": null, "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204167549} +{"stream": "products", "data": {"id": 6796217974973, "title": "Pool Floaty Icecream", "body_html": "Inflatable pink ice cream pool toy.", "vendor": "Fritsch - Ferry", "product_type": "Grocery", "created_at": "2021-06-22T18:09:27-07:00", "handle": "pool-floaty-icecream", "updated_at": "2023-04-20T04:12:25-07:00", "published_at": "2021-06-22T18:09:27-07:00", "template_suffix": null, "published_scope": "web", "tags": "developer-tools-generator", "status": "active", "admin_graphql_api_id": "gid://shopify/Product/6796217974973", "variants": [{"id": 40090580091069, "product_id": 6796217974973, "title": "magenta", "price": 57.0, "sku": "", "position": 1, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "magenta", "option2": null, "option3": null, "created_at": "2021-06-22T18:09:27-07:00", "updated_at": "2023-10-27T09:55:54-07:00", "taxable": true, "barcode": null, "grams": 499, "image_id": null, "weight": 499.0, "weight_unit": "g", "inventory_item_id": 42185194766525, "inventory_quantity": 1, "old_inventory_quantity": 1, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/40090580091069"}], "options": [{"id": 8720175300797, "product_id": 6796217974973, "name": "Title", "position": 1, "values": ["magenta"]}], "images": [], "image": null, "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204167550} +{"stream": "products_graph_ql", "data": {"id": "gid://shopify/Product/6796217843901", "title": "Lace Detail On Womens Top", "updatedAt": "2023-09-05T14:12:05Z", "createdAt": "2021-06-23T01:09:26Z", "publishedAt": null, "status": "ARCHIVED", "vendor": "Hayes, Hettinger and Hauck", "productType": "Beauty", "tags": ["developer-tools-generator"], "options": [{"id": "gid://shopify/ProductOption/8720175169725", "name": "Title", "position": 1, "values": ["Soft"]}], "handle": "lace-detail-on-womens-top", "description": "A close-up side view of a woman's off-white shirt shows the design detail in the lace.", "tracksInventory": true, "totalInventory": 12, "totalVariants": 1, "onlineStoreUrl": null, "onlineStorePreviewUrl": "https://3puoduezinm5n0tq-58033176765.shopifypreview.com/products_preview?preview_key=299ca3a2eff22be63c1ed0b77c22bc57&_bt=eyJfcmFpbHMiOnsibWVzc2FnZSI6IkJBaEpJaXRoYVhKaWVYUmxMV2x1ZEdWbmNtRjBhVzl1TFhSbGMzUXViWGx6YUc5d2FXWjVMbU52YlFZNkJrVlUiLCJleHAiOiIyMDIzLTExLTI4VDIxOjQyOjUwLjc5MloiLCJwdXIiOiJwZXJtYW5lbnRfcGFzc3dvcmRfYnlwYXNzIn19--82f1f08fa7bd0c1e5c17b2c585f1916ac47ba3a6", "descriptionHtml": "A close-up side view of a woman's off-white shirt shows the design detail in the lace.", "isGiftCard": false, "legacyResourceId": "6796217843901", "mediaCount": 0, "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204171312} +{"stream": "products_graph_ql", "data": {"id": "gid://shopify/Product/6796217909437", "title": "Red And Navy Tee Sleeve", "updatedAt": "2023-04-20T11:12:25Z", "createdAt": "2021-06-23T01:09:27Z", "publishedAt": "2021-06-23T01:09:27Z", "status": "ACTIVE", "vendor": "Little Group", "productType": "Movies", "tags": ["developer-tools-generator"], "options": [{"id": "gid://shopify/ProductOption/8720175235261", "name": "Title", "position": 1, "values": ["Plastic"]}], "handle": "red-and-navy-tee-sleeve", "description": "Zoom in on the sleeve of a red t-shirt with navy blue trim along the sleeve. Looks like a great tennis outfit.", "tracksInventory": true, "totalInventory": 3, "totalVariants": 1, "onlineStoreUrl": null, "onlineStorePreviewUrl": "https://airbyte-integration-test.myshopify.com/products/red-and-navy-tee-sleeve", "descriptionHtml": "Zoom in on the sleeve of a red t-shirt with navy blue trim along the sleeve. Looks like a great tennis outfit.", "isGiftCard": false, "legacyResourceId": "6796217909437", "mediaCount": 0, "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204171313} +{"stream": "products_graph_ql", "data": {"id": "gid://shopify/Product/6796217942205", "title": "Grey T-Shirt", "updatedAt": "2023-04-20T11:12:25Z", "createdAt": "2021-06-23T01:09:27Z", "publishedAt": "2021-06-23T01:09:27Z", "status": "ACTIVE", "vendor": "Lang - Bogisich", "productType": "Home", "tags": ["developer-tools-generator"], "options": [{"id": "gid://shopify/ProductOption/8720175268029", "name": "Title", "position": 1, "values": ["Granite"]}], "handle": "grey-t-shirt", "description": "A grey t-shirt on a hanger. Simple. Classic. Grey.", "tracksInventory": true, "totalInventory": 38, "totalVariants": 1, "onlineStoreUrl": null, "onlineStorePreviewUrl": "https://airbyte-integration-test.myshopify.com/products/grey-t-shirt", "descriptionHtml": "A grey t-shirt on a hanger. Simple. Classic. Grey.", "isGiftCard": false, "legacyResourceId": "6796217942205", "mediaCount": 0, "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204171313} +{"stream": "product_variants", "data": {"id": 40090579959997, "product_id": 6796217843901, "title": "Soft", "price": 85.0, "sku": "", "position": 1, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Soft", "option2": null, "option3": null, "created_at": "2021-06-22T18:09:27-07:00", "updated_at": "2023-10-27T09:55:54-07:00", "taxable": true, "barcode": null, "grams": 391, "image_id": null, "weight": 391.0, "weight_unit": "g", "inventory_item_id": 42185194668221, "inventory_quantity": 12, "old_inventory_quantity": 12, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/40090579959997", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204177380} +{"stream": "product_variants", "data": {"id": 40090579992765, "product_id": 6796217909437, "title": "Plastic", "price": 23.0, "sku": "", "position": 1, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Plastic", "option2": null, "option3": null, "created_at": "2021-06-22T18:09:27-07:00", "updated_at": "2023-10-27T09:55:54-07:00", "taxable": true, "barcode": null, "grams": 39, "image_id": null, "weight": 39.0, "weight_unit": "g", "inventory_item_id": 42185194700989, "inventory_quantity": 3, "old_inventory_quantity": 3, "requires_shipping": true, "admin_graphql_api_id": "gid://shopify/ProductVariant/40090579992765", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204177840} +{"stream": "product_variants", "data": {"id": 40090580025533, "product_id": 6796217942205, "title": "Granite", "price": 70.0, "sku": "", "position": 1, "inventory_policy": "deny", "compare_at_price": null, "fulfillment_service": "manual", "inventory_management": "shopify", "option1": "Granite", "option2": null, "option3": null, "created_at": "2021-06-22T18:09:27-07:00", "updated_at": "2023-10-27T09:55:54-07:00", "taxable": true, "barcode": null, "grams": 0, "image_id": null, "weight": 0.0, "weight_unit": "g", "inventory_item_id": 42185194733757, "inventory_quantity": 38, "old_inventory_quantity": 38, "requires_shipping": false, "admin_graphql_api_id": "gid://shopify/ProductVariant/40090580025533", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204178293} +{"stream": "shop", "data": {"id": 58033176765, "name": "airbyte integration test", "email": "sherif@airbyte.io", "domain": "airbyte-integration-test.myshopify.com", "province": "California", "country": "US", "address1": "350 29th Avenue", "zip": "94121", "city": "San Francisco", "source": null, "phone": "8023494963", "latitude": 37.7827286, "longitude": -122.4889911, "primary_locale": "en", "address2": "", "created_at": "2021-06-22T18:00:23-07:00", "updated_at": "2023-04-30T09:02:52-07:00", "country_code": "US", "country_name": "United States", "currency": "USD", "customer_email": "sherif@airbyte.io", "timezone": "(GMT-08:00) America/Los_Angeles", "iana_timezone": "America/Los_Angeles", "shop_owner": "Airbyte Airbyte", "money_format": "${{amount}}", "money_with_currency_format": "${{amount}} USD", "weight_unit": "kg", "province_code": "CA", "taxes_included": true, "auto_configure_tax_inclusivity": null, "tax_shipping": null, "county_taxes": true, "plan_display_name": "Developer Preview", "plan_name": "partner_test", "has_discounts": true, "has_gift_cards": false, "myshopify_domain": "airbyte-integration-test.myshopify.com", "google_apps_domain": null, "google_apps_login_enabled": null, "money_in_emails_format": "${{amount}}", "money_with_currency_in_emails_format": "${{amount}} USD", "eligible_for_payments": true, "requires_extra_payments_agreement": false, "password_enabled": true, "has_storefront": true, "finances": true, "primary_location_id": 63590301885, "checkout_api_supported": true, "multi_location_enabled": true, "setup_required": false, "pre_launch_enabled": false, "enabled_presentment_currencies": ["USD"], "transactional_sms_disabled": false, "marketing_sms_consent_enabled_at_checkout": false, "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204410095} +{"stream": "smart_collections", "data": {"id": 273278566589, "handle": "test-collection", "title": "Test Collection", "updated_at": "2023-09-05T07:12:04-07:00", "body_html": "updated_mon_24.04.2023", "published_at": "2021-07-19T07:02:54-07:00", "sort_order": "best-selling", "template_suffix": "", "disjunctive": false, "rules": ["{'column': 'type', 'relation': 'equals', 'condition': 'Beauty'}"], "published_scope": "web", "admin_graphql_api_id": "gid://shopify/Collection/273278566589", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204411199} +{"stream": "tender_transactions", "data": {"id": 4464009117885, "order_id": 5033391718589, "amount": "19.00", "currency": "USD", "user_id": null, "test": false, "processed_at": "2023-04-24T11:00:08-07:00", "remote_reference": null, "payment_details": null, "payment_method": "other", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204412219} +{"stream": "tender_transactions", "data": {"id": 4448993542333, "order_id": 5010585911485, "amount": "19.00", "currency": "USD", "user_id": null, "test": false, "processed_at": "2023-04-13T05:11:15-07:00", "remote_reference": null, "payment_details": null, "payment_method": "other", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204412220} +{"stream": "tender_transactions", "data": {"id": 4448992690365, "order_id": 5010584895677, "amount": "102.00", "currency": "USD", "user_id": null, "test": false, "processed_at": "2023-04-13T05:09:44-07:00", "remote_reference": null, "payment_details": null, "payment_method": "other", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204412220} +{"stream": "transactions", "data": {"id": 6281692217533, "order_id": 5010584895677, "kind": "sale", "gateway": "manual", "status": "success", "message": "Marked the manual payment as received", "created_at": "2023-04-13T05:09:44-07:00", "test": false, "authorization": null, "location_id": null, "user_id": null, "parent_id": null, "processed_at": "2023-04-13T05:09:44-07:00", "device_id": null, "error_code": null, "source_name": "checkout_one", "receipt": {}, "amount": 102.0, "currency": "USD", "payment_id": "r9BerEaVJ5OzQNmPGZsK2V7zq", "total_unsettled_set": {"presentment_money": {"amount": 0.0, "currency": "USD"}, "shop_money": {"amount": 0.0, "currency": "USD"}}, "admin_graphql_api_id": "gid://shopify/OrderTransaction/6281692217533", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204414331} +{"stream": "transactions", "data": {"id": 6281693561021, "order_id": 5010585911485, "kind": "sale", "gateway": "manual", "status": "success", "message": "Marked the manual payment as received", "created_at": "2023-04-13T05:11:15-07:00", "test": false, "authorization": null, "location_id": null, "user_id": null, "parent_id": null, "processed_at": "2023-04-13T05:11:15-07:00", "device_id": null, "error_code": null, "source_name": "checkout_one", "receipt": {}, "amount": 19.0, "currency": "USD", "payment_id": "rguGpKMnZqzpEzPvDfnSS8x4B", "total_unsettled_set": {"presentment_money": {"amount": 0.0, "currency": "USD"}, "shop_money": {"amount": 0.0, "currency": "USD"}}, "admin_graphql_api_id": "gid://shopify/OrderTransaction/6281693561021", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204414830} +{"stream": "transactions", "data": {"id": 5721110872253, "order_id": 4554821468349, "kind": "sale", "gateway": "bogus", "status": "success", "message": "Bogus Gateway: Forced success", "created_at": "2022-06-15T05:16:52-07:00", "test": true, "authorization": "53433", "location_id": null, "user_id": null, "parent_id": null, "processed_at": "2022-06-15T05:16:52-07:00", "device_id": null, "error_code": null, "source_name": "580111", "payment_details": {"credit_card_bin": "1", "avs_result_code": null, "cvv_result_code": null, "credit_card_number": "\u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 \u2022\u2022\u2022\u2022 1", "credit_card_company": "Bogus", "buyer_action_info": null, "credit_card_name": "Bogus Gateway", "credit_card_wallet": null, "credit_card_expiration_month": 2, "credit_card_expiration_year": 2025}, "receipt": {"paid_amount": "57.23"}, "amount": 57.23, "currency": "USD", "payment_id": "c25048437719229.1", "total_unsettled_set": {"presentment_money": {"amount": 0.0, "currency": "USD"}, "shop_money": {"amount": 0.0, "currency": "USD"}}, "admin_graphql_api_id": "gid://shopify/OrderTransaction/5721110872253", "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204415297} +{"stream": "transactions_graphql", "data": {"id": 6281692217533, "error_code": null, "test": false, "kind": "SALE", "amount": 102.0, "receipt": "{}", "gateway": "manual", "authorization": null, "created_at": "2023-04-13T12:09:44+00:00", "status": "SUCCESS", "processed_at": "2023-04-13T12:09:44+00:00", "total_unsettled_set": {"presentment_money": {"amount": 0.0, "currency": "USD"}, "shop_money": {"amount": 0.0, "currency": "USD"}}, "payment_id": "r9BerEaVJ5OzQNmPGZsK2V7zq", "payment_details": null, "order_id": 5010584895677, "currency": "USD", "admin_graphql_api_id": "gid://shopify/OrderTransaction/6281692217533", "parent_id": null, "shop_url": "airbyte-integration-test"}, "emitted_at": 1702897742227} +{"stream": "transactions_graphql", "data": {"id": 6281693561021, "error_code": null, "test": false, "kind": "SALE", "amount": 19.0, "receipt": "{}", "gateway": "manual", "authorization": null, "created_at": "2023-04-13T12:11:15+00:00", "status": "SUCCESS", "processed_at": "2023-04-13T12:11:15+00:00", "total_unsettled_set": {"presentment_money": {"amount": 0.0, "currency": "USD"}, "shop_money": {"amount": 0.0, "currency": "USD"}}, "payment_id": "rguGpKMnZqzpEzPvDfnSS8x4B", "payment_details": null, "order_id": 5010585911485, "currency": "USD", "admin_graphql_api_id": "gid://shopify/OrderTransaction/6281693561021", "parent_id": null, "shop_url": "airbyte-integration-test"}, "emitted_at": 1702897742229} +{"stream": "transactions_graphql", "data": {"id": 6302086037693, "error_code": null, "test": false, "kind": "SALE", "amount": 19.0, "receipt": "{}", "gateway": "manual", "authorization": null, "created_at": "2023-04-24T18:00:08+00:00", "status": "SUCCESS", "processed_at": "2023-04-24T18:00:08+00:00", "total_unsettled_set": {"presentment_money": {"amount": 0.0, "currency": "USD"}, "shop_money": {"amount": 0.0, "currency": "USD"}}, "payment_id": "ru7Najsh1HavL8RRkZHavCzGe", "payment_details": null, "order_id": 5033391718589, "currency": "USD", "admin_graphql_api_id": "gid://shopify/OrderTransaction/6302086037693", "parent_id": null, "shop_url": "airbyte-integration-test"}, "emitted_at": 1702897742232} +{"stream": "customer_address", "data": {"id": 8092523135165, "customer_id": 6569096478909, "first_name": "New Test", "last_name": "Customer", "company": "Test Company", "address1": "My Best Accent", "address2": "", "city": "Fair Lawn", "province": "New Jersey", "country": "United States", "zip": "07410", "phone": "", "name": "New Test Customer", "province_code": "NJ", "country_code": "US", "country_name": "United States", "default": true, "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204422125} +{"stream": "customer_address", "data": {"id": 8212915650749, "customer_id": 6676027932861, "first_name": "MArcos", "last_name": "Millnitz", "company": null, "address1": null, "address2": null, "city": null, "province": null, "country": null, "zip": null, "phone": null, "name": "MArcos Millnitz", "province_code": null, "country_code": null, "country_name": null, "default": true, "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204422578} +{"stream": "countries", "data": {"id": 417014841533, "name": "Rest of World", "code": "*", "tax_name": "Tax", "tax": 0.0, "provinces": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204423655} +{"stream": "countries", "data": {"id": 417014808765, "name": "Ukraine", "code": "UA", "tax_name": "PDV", "tax": 0.2, "provinces": [], "shop_url": "airbyte-integration-test"}, "emitted_at": 1701204423656} diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/state.json b/airbyte-integrations/connectors/source-shopify/integration_tests/state.json index 652fa38ae934..c29ae675c6c5 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/state.json @@ -1,20 +1,28 @@ { "customers": { - "updated_at": "2022-10-11T08:40:51-07:00" + "updated_at": "2023-07-11T13:07:45-07:00" }, "orders": { - "updated_at": "2022-10-12T11:15:27-07:00" + "deleted": { + "deleted_at": "" + }, + "updated_at": "2023-04-24T11:00:10-07:00" }, "draft_orders": { - "updated_at": "2022-10-08T05:07:29-07:00" + "updated_at": "2023-07-11T12:57:55-07:00" }, "products": { - "updated_at": "2023-03-20T06:08:51-07:00" + "deleted": { + "deleted_at": "2023-09-05T13:32:22-07:00" + }, + "updated_at": "2023-09-05T07:12:05-07:00" }, "products_graph_ql": { - "updatedAt": "2023-03-20T13:08:51Z" + "updatedAt": "2023-09-05T14:12:05Z" + }, + "abandoned_checkouts": { + "updated_at": "2023-07-11T13:07:45-07:00" }, - "abandoned_checkouts": {}, "metafields": { "updated_at": "2022-05-30T23:42:02-07:00" }, @@ -22,162 +30,228 @@ "id": 29427031703741 }, "custom_collections": { - "updated_at": "2023-03-20T06:08:12-07:00" + "deleted": { + "deleted_at": "2023-09-06T03:34:39-07:00" + }, + "updated_at": "2023-09-05T07:06:59-07:00" }, "order_refunds": { "orders": { - "updated_at": "2022-10-12T11:15:27-07:00" + "updated_at": "2023-04-24T11:00:10-07:00", + "deleted": { + "deleted_at": "" + } }, "created_at": "2022-10-10T06:21:53-07:00" }, "order_risks": { "orders": { - "updated_at": "2022-03-07T02:09:04-08:00" + "updated_at": "2023-04-24T07:03:06-07:00", + "deleted": { + "deleted_at": "" + } }, "id": 6446736474301 }, "transactions": { "orders": { - "updated_at": "2022-10-12T11:15:27-07:00" + "updated_at": "2023-04-24T11:00:10-07:00", + "deleted": { + "deleted_at": "" + } }, - "created_at": "2022-10-10T06:21:52-07:00" + "created_at": "2023-04-24T11:00:08-07:00" }, "tender_transactions": { - "processed_at": "2022-10-10T06:21:52-07:00" + "processed_at": "2023-04-24T11:00:08-07:00" }, "pages": { - "updated_at": "2022-10-08T08:07:00-07:00" + "deleted": { + "deleted_at": "2023-09-06T03:37:06-07:00" + }, + "updated_at": "2023-04-24T11:08:41-07:00" }, "price_rules": { - "updated_at": "2022-10-14T10:19:39-07:00" + "deleted": { + "deleted_at": "2023-09-06T03:48:46-07:00" + }, + "updated_at": "2023-04-24T05:52:22-07:00" }, "discount_codes": { "price_rules": { - "updated_at": "2022-10-14T10:10:25-07:00" + "updated_at": "2023-04-24T05:52:22-07:00", + "deleted": { + "deleted_at": "" + } }, - "updated_at": "2022-10-14T10:10:25-07:00" + "updated_at": "2023-04-24T05:52:22-07:00" }, "inventory_items": { "products": { - "updated_at": "2023-03-20T06:08:51-07:00" + "updated_at": "2023-04-20T04:12:51-07:00", + "deleted": { + "deleted_at": "" + } }, - "updated_at": "2023-01-06T10:35:26-08:00" + "updated_at": "2023-04-14T03:29:27-07:00" }, "inventory_levels": { "locations": {}, - "updated_at": "2023-01-06T10:35:27-08:00" + "updated_at": "2023-04-24T11:00:10-07:00" }, "fulfillment_orders": { "orders": { - "updated_at": "2022-10-10T06:05:29-07:00" + "updated_at": "2023-04-24T11:00:10-07:00", + "deleted": { + "deleted_at": "" + } }, - "id": 5567724486845 + "id": 5985636450493 }, "fulfillments": { "orders": { - "updated_at": "2022-10-10T06:05:29-07:00" + "updated_at": "2023-04-24T11:00:10-07:00", + "deleted": { + "deleted_at": "" + } }, - "updated_at": "2022-06-22T03:50:14-07:00" + "updated_at": "2023-04-24T11:00:09-07:00" }, "balance_transactions": {}, "articles": { - "id": 558137508029 + "id": 558999404733, + "deleted": { + "deleted_at": "2023-09-05T14:02:00-07:00" + } }, "metafield_articles": { "articles": { - "id": 558137508029 + "id": 558627979453, + "deleted": {} }, - "updated_at": "2022-10-07T16:09:02-07:00" + "updated_at": "2023-04-14T03:19:18-07:00" }, "blogs": { - "id": 80417685693 + "id": 85733114045, + "deleted": { + "deleted_at": "2023-09-06T03:30:22-07:00" + } }, "metafield_blogs": { + "updated_at": "2022-10-07T06:05:23-07:00", "blogs": { "id": 80417685693 - }, - "updated_at": "2022-10-07T06:05:23-07:00" + } }, "metafield_customers": { "customers": { - "updated_at": "2022-10-11T08:40:51-07:00" + "updated_at": "2023-04-24T06:53:48-07:00" }, - "updated_at": "2022-10-11T08:40:51-07:00" + "updated_at": "2023-04-13T04:50:10-07:00" }, "metafield_orders": { "orders": { - "updated_at": "2022-10-12T11:15:27-07:00" + "updated_at": "2023-04-24T10:59:00-07:00", + "deleted": { + "deleted_at": "" + } }, - "updated_at": "2022-10-12T11:15:27-07:00" + "updated_at": "2023-04-14T03:52:40-07:00" }, "metafield_draft_orders": { "draft_orders": { - "updated_at": "2022-10-08T05:07:29-07:00" + "updated_at": "2023-04-24T07:18:06-07:00" }, - "updated_at": "2022-10-08T05:07:29-07:00" + "updated_at": "2023-04-24T07:18:06-07:00" }, "metafield_products": { "products": { - "updated_at": "2023-03-20T06:08:32-07:00" + "updated_at": "2023-04-20T04:12:59-07:00", + "deleted": { + "deleted_at": "" + } }, - "updated_at": "2022-10-08T08:45:01-07:00" + "updated_at": "2023-04-14T04:04:46-07:00" }, "product_images": { "products": { - "updated_at": "2023-03-20T06:08:51-07:00" + "updated_at": "2023-04-24T11:05:13-07:00", + "deleted": { + "deleted_at": "2023-09-05T13:32:22-07:00" + } }, - "id": 32940458868925 + "updated_at": "2023-04-24T10:27:15-07:00" }, "metafield_product_images": { "products": { - "updated_at": "2023-03-20T06:08:51-07:00" + "updated_at": "", + "deleted": { + "deleted_at": "2023-09-05T13:32:22-07:00" + } }, - "updated_at": "2022-10-13T04:01:00-07:00" + "updated_at": "2023-04-24T10:32:19-07:00" }, "product_variants": { + "id": 42778150305981, "products": { - "updated_at": "2023-03-20T06:08:51-07:00" - }, - "id": 42595864019133 + "updated_at": "", + "deleted": { + "deleted_at": "2023-09-05T13:32:22-07:00" + } + } }, "metafield_product_variants": { "products": { - "updated_at": "2023-03-20T06:08:51-07:00" + "updated_at": "", + "deleted": { + "deleted_at": "2023-09-05T13:32:22-07:00" + } }, - "updated_at": "2022-10-14T10:16:04-07:00" + "updated_at": "2023-04-14T03:29:27-07:00" }, "collections": { - "updated_at": "2023-03-20T06:08:12-07:00", "collects": { "id": 29427031703741 - } + }, + "updated_at": "2023-03-20T06:08:12-07:00" }, "metafield_collections": { - "updated_at": "2022-10-08T04:44:51-07:00", "collects": { "id": 29427031703741 - } + }, + "updated_at": "2022-10-08T04:44:51-07:00" }, "smart_collections": { - "updated_at": "2023-03-20T06:08:12-07:00" + "updated_at": "2023-09-05T07:12:04-07:00" }, "metafield_smart_collections": { "smart_collections": { - "updated_at": "2023-03-20T06:08:12-07:00" + "updated_at": "2023-09-05T07:12:04-07:00" }, - "updated_at": "2022-10-12T13:36:55-07:00" + "updated_at": "2023-04-14T05:21:58-07:00" }, "metafield_pages": { "pages": { - "updated_at": "2022-10-08T08:07:00-07:00" + "updated_at": "2023-04-24T11:08:41-07:00", + "deleted": { + "deleted_at": "" + } }, - "updated_at": "2022-10-08T08:07:00-07:00" + "updated_at": "2023-04-24T11:08:41-07:00" }, "metafield_shops": { - "updated_at": "2022-05-30T23:42:02-07:00" + "updated_at": "2023-04-24T11:12:38-07:00" }, "metafield_locations": { "locations": {}, "updated_at": "2022-10-12T02:21:35-07:00" + }, + "disputes": {}, + "customer_saved_search": {}, + "customer_address": { + "id": 8212915650749, + "customers": { + "updated_at": "2023-07-11T13:07:45-07:00" + } } } diff --git a/airbyte-integrations/connectors/source-shopify/main.py b/airbyte-integrations/connectors/source-shopify/main.py index a45dc5aaf611..aca13eebbb25 100644 --- a/airbyte-integrations/connectors/source-shopify/main.py +++ b/airbyte-integrations/connectors/source-shopify/main.py @@ -3,11 +3,7 @@ # -import sys - -from airbyte_cdk.entrypoint import launch -from source_shopify import SourceShopify +from source_shopify.run import run if __name__ == "__main__": - source = SourceShopify() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-shopify/metadata.yaml b/airbyte-integrations/connectors/source-shopify/metadata.yaml index c711d5f0c45f..0a5306019a51 100644 --- a/airbyte-integrations/connectors/source-shopify/metadata.yaml +++ b/airbyte-integrations/connectors/source-shopify/metadata.yaml @@ -1,11 +1,17 @@ data: ab_internal: - ql: 200 - sl: 100 + ql: 400 + sl: 300 + allowedHosts: + hosts: + - ${shop}.myshopify.com + - shopify.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: 9da77001-af33-4bcd-be46-6252bf9342b9 - dockerImageTag: 0.6.2 + dockerImageTag: 1.2.0 dockerRepository: airbyte/source-shopify documentationUrl: https://docs.airbyte.com/integrations/sources/shopify githubIssueLabel: source-shopify @@ -17,8 +23,51 @@ data: enabled: true oss: enabled: true - releaseStage: alpha - supportLevel: community + releaseStage: generally_available + releases: + breakingChanges: + 1.0.0: + message: + "This upgrade brings changes to certain streams after migration to + Shopify API version `2023-07`, more details in this PR: https://github.com/airbytehq/airbyte/pull/29361." + upgradeDeadline: "2023-09-17" + 1.2.0: + message: "This upgrade brings perfomance impovements and minor stream schema changes. Details are available here: https://github.com/airbytehq/airbyte/pull/32345#issue-1985556333." + upgradeDeadline: "2024-03-01" + suggestedStreams: + streams: + - customers + - order_refunds + - transactions_graphql + - product_variants + - abandoned_checkouts + - discount_codes + - inventory_items + - locations + - shop + - price_rules + - inventory_levels + - custom_collections + - fulfillments + - products_graph_ql + - product_images + - collects + - tender_transactions + - smart_collections + - draft_orders + - metafield_customers + - metafield_products + - metafield_orders + - customer_address + - fulfillment_orders + - metafield_shops + - countries + - metafield_product_variants + - metafield_smart_collections + - metafield_product_images + - metafield_draft_orders + - metafield_locations + supportLevel: certified tags: - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-shopify/setup.py b/airbyte-integrations/connectors/source-shopify/setup.py index d1ec88ea5e7b..4bfe055cc1ee 100644 --- a/airbyte-integrations/connectors/source-shopify/setup.py +++ b/airbyte-integrations/connectors/source-shopify/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk", "sgqlc~=16.0"] +MAIN_REQUIREMENTS = ["airbyte-cdk", "sgqlc~=16.0", "graphql_query"] TEST_REQUIREMENTS = [ "pytest", @@ -24,4 +24,9 @@ extras_require={ "tests": TEST_REQUIREMENTS, }, + entry_points={ + "console_scripts": [ + "source-shopify=source_shopify.run:run", + ], + }, ) diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py b/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py index 30cb4056a659..6d53197ff18a 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/auth.py @@ -8,6 +8,12 @@ from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +class MissingAccessTokenError(Exception): + """ + Raised when the token is `None` instead of the real value + """ + + class NotImplementedAuth(Exception): """Not implemented Auth option error""" @@ -28,13 +34,16 @@ def __init__(self, config: Mapping[str, Any]): self.config = config def get_auth_header(self) -> Mapping[str, Any]: - auth_header: str = "X-Shopify-Access-Token" credentials: Dict = self.config.get("credentials", self.config.get("auth_method")) auth_method: str = credentials.get("auth_method") if auth_method in ["oauth2.0", "access_token"]: - return {auth_header: credentials.get("access_token")} + access_token = credentials.get("access_token") + if access_token: + return {auth_header: access_token} + else: + raise MissingAccessTokenError elif auth_method == "api_password": return {auth_header: credentials.get("api_password")} else: diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/run.py b/airbyte-integrations/connectors/source-shopify/source_shopify/run.py new file mode 100644 index 000000000000..9c13e936ca71 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/run.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch + +from .source import SourceShopify + + +def run(): + source = SourceShopify() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/articles.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/articles.json index 85ec2388ea75..86eaf83caed7 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/articles.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/articles.json @@ -49,6 +49,16 @@ }, "shop_url": { "type": ["null", "string"] + }, + "deleted_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "deleted_message": { + "type": ["null", "string"] + }, + "deleted_description": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/blogs.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/blogs.json index a9ea23eea82e..55ce6bc3c386 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/blogs.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/blogs.json @@ -40,6 +40,16 @@ }, "shop_url": { "type": ["null", "string"] + }, + "deleted_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "deleted_message": { + "type": ["null", "string"] + }, + "deleted_description": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/custom_collections.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/custom_collections.json index 85041b590019..59b2246cc63f 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/custom_collections.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/custom_collections.json @@ -55,6 +55,16 @@ }, "shop_url": { "type": ["null", "string"] + }, + "deleted_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "deleted_message": { + "type": ["null", "string"] + }, + "deleted_description": { + "type": ["null", "string"] } }, "additionalProperties": true, diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/discount_codes.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/discount_codes.json index f3e208e014c5..4b864604d486 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/discount_codes.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/discount_codes.json @@ -22,6 +22,15 @@ "type": ["null", "string"], "format": "date-time" }, + "summary": { + "type": ["null", "string"] + }, + "discount_type": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, "shop_url": { "type": ["null", "string"] } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/draft_orders.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/draft_orders.json index f0bcefdff00e..22b8840bc36c 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/draft_orders.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/draft_orders.json @@ -308,6 +308,9 @@ "payment_terms": { "type": ["null", "string"] }, + "po_number": { + "type": ["null", "string"] + }, "shipping_line": { "properties": { "price": { diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/fulfillment_orders.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/fulfillment_orders.json index 936ed5b4ffbb..a4cf88fd1f73 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/fulfillment_orders.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/fulfillment_orders.json @@ -54,6 +54,14 @@ }, "method_type": { "type": ["null", "string"] + }, + "min_delivery_date_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "max_delivery_date_time": { + "type": ["null", "string"], + "format": "date-time" } } }, @@ -135,7 +143,15 @@ "supported_actions": { "type": ["null", "array"], "items": { - "type": ["null", "string"] + "type": ["null", "object"], + "properties": { + "action": { + "type": ["null", "string"] + }, + "external_url": { + "type": ["null", "string"] + } + } } }, "merchant_requests": { @@ -143,6 +159,9 @@ "items": { "type": ["null", "object"], "properties": { + "id": { + "type": ["null", "integer"] + }, "message": { "type": ["null", "string"] }, @@ -150,7 +169,13 @@ "type": ["null", "string"] }, "request_options": { - "type": ["null", "object"] + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "notify_customer": { + "type": ["null", "boolean"] + } + } } } } @@ -189,6 +214,17 @@ }, "shop_url": { "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "admin_graphql_api_id": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/fulfillments.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/fulfillments.json index d04a0c3686c6..cf42273594c1 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/fulfillments.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/fulfillments.json @@ -115,6 +115,33 @@ "price": { "type": ["null", "string"] }, + "price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "number"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "number"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, "grams": { "type": ["null", "number"] }, @@ -163,6 +190,33 @@ "total_discount": { "type": ["null", "string"] }, + "total_discount_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "number"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "number"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, "fulfillment_status": { "type": ["null", "string"] }, @@ -174,9 +228,39 @@ "items": { "type": ["null", "object"], "properties": { + "channel_liable": { + "type": ["null", "boolean"] + }, "price": { "type": ["null", "number"] }, + "price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "number"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "number"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, "rate": { "type": ["null", "number"] }, @@ -185,6 +269,146 @@ } } } + }, + "duties": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "harmonized_system_code": { + "type": ["null", "string"] + }, + "country_code_of_origin": { + "type": ["null", "string"] + }, + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "tax_lines": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "title": { + "type": ["null", "string"] + }, + "price": { + "type": ["null", "string"] + }, + "rate": { + "type": ["null", "number"] + }, + "price_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "channel_liable": { + "type": ["null", "boolean"] + } + } + } + } + } + } + }, + "discount_allocations": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "discount_application_index": { + "type": ["null", "number"] + }, + "amount_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "string"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "application_type": { + "type": ["null", "string"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json index 3aceaff8f507..662a4be22926 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json @@ -196,15 +196,7 @@ "properties": { "type": ["null", "array"], "items": { - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] + "type": ["null", "string"] } }, "quantity": { @@ -393,6 +385,17 @@ "type": ["null", "object"] } }, + "return": { + "type": ["null", "object"], + "properties": { + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + } + } + }, "transactions": { "type": ["null", "array"], "items": { diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json index 5b5ff73bda95..416b20b7d0f4 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json @@ -60,9 +60,15 @@ "type": ["null", "string"], "format": "date-time" }, + "company": { + "type": ["null", "string"] + }, "confirmed": { "type": ["null", "boolean"] }, + "confirmation_number": { + "type": ["null", "string"] + }, "contact_email": { "type": ["null", "string"] }, @@ -196,6 +202,33 @@ } } }, + "current_total_additional_fees_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "number"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "number"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, "customer_locale": { "type": ["null", "string"] }, @@ -263,9 +296,6 @@ "fulfillment_status": { "type": ["null", "string"] }, - "gateway": { - "type": ["null", "string"] - }, "landing_site": { "type": ["null", "string"] }, @@ -310,6 +340,33 @@ "original_total_duties_set": { "type": ["null", "string"] }, + "original_total_additional_fees_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "number"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "number"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, "payment_gateway_names": { "type": ["null", "array"], "items": { @@ -328,7 +385,7 @@ "processed_at": { "type": ["null", "string"] }, - "processing_method": { + "po_number": { "type": ["null", "string"] }, "reference": { @@ -382,6 +439,9 @@ "tags": { "type": ["null", "string"] }, + "tax_exempt": { + "type": ["null", "boolean"] + }, "tax_lines": { "type": ["null", "array"], "items": { @@ -1672,26 +1732,6 @@ } } }, - "payment_details": { - "type": ["null", "object"], - "properties": { - "credit_card_bin": { - "type": ["null", "string"] - }, - "avs_result_code": { - "type": ["null", "string"] - }, - "cvv_result_code": { - "type": ["null", "string"] - }, - "credit_card_number": { - "type": ["null", "string"] - }, - "credit_card_company": { - "type": ["null", "string"] - } - } - }, "refunds": { "type": ["null", "array"], "items": { @@ -2242,9 +2282,6 @@ "code": { "type": ["null", "string"] }, - "delivery_category": { - "type": ["null", "string"] - }, "discounted_price": { "type": ["null", "number"] }, @@ -2376,6 +2413,16 @@ } } } + }, + "deleted_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "deleted_message": { + "type": ["null", "string"] + }, + "deleted_description": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/pages.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/pages.json index 79a5006372b3..578430a050d2 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/pages.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/pages.json @@ -40,6 +40,16 @@ }, "shop_url": { "type": ["null", "string"] + }, + "deleted_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "deleted_message": { + "type": ["null", "string"] + }, + "deleted_description": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/price_rules.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/price_rules.json index 0bfb04759d5e..cc9bd9e69c1d 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/price_rules.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/price_rules.json @@ -159,6 +159,16 @@ }, "shop_url": { "type": ["null", "string"] + }, + "deleted_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "deleted_message": { + "type": ["null", "string"] + }, + "deleted_description": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/products.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/products.json index 90066b9aefeb..c97a60b74645 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/products.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/products.json @@ -268,6 +268,16 @@ }, "shop_url": { "type": ["null", "string"] + }, + "deleted_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "deleted_message": { + "type": ["null", "string"] + }, + "deleted_description": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/transactions.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/transactions.json index fb91ba9f24de..bbdd8156ca02 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/transactions.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/transactions.json @@ -46,6 +46,36 @@ "status": { "type": ["null", "string"] }, + "total_unsettled_set": { + "type": ["null", "object"], + "properties": { + "shop_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "number"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + }, + "presentment_money": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "number"] + }, + "currency_code": { + "type": ["null", "string"] + } + } + } + } + }, + "payment_id": { + "type": ["null", "string"] + }, "payment_details": { "properties": { "cvv_result_code": { @@ -77,21 +107,29 @@ "type": ["null", "string"] }, "receipt": { - "type": ["null", "object"], - "properties": { - "fee_amount": { - "type": ["null", "number"], - "multipleOf": 1e-10 + "oneOf": [ + { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "fee_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "gross_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + }, + "tax_amount": { + "type": ["null", "number"], + "multipleOf": 1e-10 + } + } }, - "gross_amount": { - "type": ["null", "number"], - "multipleOf": 1e-10 - }, - "tax_amount": { - "type": ["null", "number"], - "multipleOf": 1e-10 + { + "type": ["null", "string"] } - } + ] }, "location_id": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/exceptions.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/exceptions.py new file mode 100644 index 000000000000..51a0a0b004ff --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/exceptions.py @@ -0,0 +1,40 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from airbyte_cdk.utils import AirbyteTracedException +from airbyte_protocol.models import FailureType + + +class ShopifyBulkExceptions: + class BaseBulkException(AirbyteTracedException): + def __init__(self, message: str, **kwargs) -> None: + super().__init__(internal_message=message, failure_type=FailureType.config_error, **kwargs) + + class BulkJobError(BaseBulkException): + """Raised when there are BULK Job Errors in response""" + + class BulkJobUnknownError(BaseBulkException): + """Raised when BULK Job has FAILED with Unknown status""" + + class BulkJobBadResponse(BaseBulkException): + """Raised when the requests.Response object could not be parsed""" + + class BulkJobResultUrlError(BaseBulkException): + """Raised when BULK Job has ACCESS_DENIED status""" + + class BulkRecordProduceError(BaseBulkException): + """Raised when there are error producing records from BULK Job result""" + + class BulkJobFailed(BaseBulkException): + """Raised when BULK Job has FAILED status""" + + class BulkJobTimout(BaseBulkException): + """Raised when BULK Job has TIMEOUT status""" + + class BulkJobAccessDenied(BaseBulkException): + """Raised when BULK Job has ACCESS_DENIED status""" + + class BulkJobConcurrentError(BaseBulkException): + """Raised when BULK Job could not be created, since the 1 Bulk job / shop quota is exceeded.""" diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/job.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/job.py new file mode 100644 index 000000000000..a4ce9b0ce181 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/job.py @@ -0,0 +1,271 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from enum import Enum +from time import sleep, time +from typing import Any, Iterable, List, Mapping, Optional + +import requests +from airbyte_cdk import AirbyteLogger +from requests.exceptions import JSONDecodeError +from source_shopify.utils import ApiTypeEnum +from source_shopify.utils import ShopifyRateLimiter as limiter + +from .exceptions import ShopifyBulkExceptions +from .query import ShopifyBulkTemplates +from .tools import END_OF_FILE, BulkTools + + +class ShopifyBulkStatus(Enum): + CREATED = "CREATED" + COMPLETED = "COMPLETED" + RUNNING = "RUNNING" + FAILED = "FAILED" + TIMEOUT = "TIMEOUT" + ACCESS_DENIED = "ACCESS_DENIED" + + +class ShopifyBulkManager: + def __init__(self, session: requests.Session, base_url: str, logger: AirbyteLogger) -> None: + self.session = session + self.base_url = base_url + self.logger = logger + self.tools: BulkTools = BulkTools() + # current job attributes + self.job_id: str = None + self.job_state: ShopifyBulkStatus[str] = None + + # 5Mb chunk size to save the file + retrieve_chunk_size: int = 1024 * 1024 * 5 + + # time between job status checks + job_check_interval_sec: int = 5 + + # max attempts for job creation + concurrent_max_retry: int = 19 + # attempt counter + concurrent_attempt: int = 0 + # attempt limit indicator + concurrent_max_attempt_reached: bool = False + # sleep time per creation attempt + concurrent_interval_sec = 30 + + @property + def process_job_fn_mapping(self) -> Mapping[str, Any]: + return { + ShopifyBulkStatus.RUNNING.value: self.on_running_job, + ShopifyBulkStatus.TIMEOUT.value: self.on_timeout_job, + ShopifyBulkStatus.FAILED.value: self.on_failed_job, + ShopifyBulkStatus.ACCESS_DENIED.value: self.on_access_denied_job, + } + + @limiter.balance_rate_limit(api_type=ApiTypeEnum.graphql.value) + def check_for_errors(self, response: requests.Response) -> Iterable[Mapping[str, Any]]: + try: + return response.json().get("errors") or response.json().get("data", {}).get("bulkOperationRunQuery", {}).get("userErrors", []) + except (Exception, JSONDecodeError) as e: + raise ShopifyBulkExceptions.BulkJobBadResponse( + f"Couldn't check the `response` for `errors`, response: `{response.text}`. Trace: {repr(e)}." + ) + + def update_status(self, response: Optional[requests.Response] = None) -> None: + if response: + response_data = response.json().get("data", {}).get("node", {}) + self.job_id = response_data.get("id") + self.job_state = response_data.get("status") + self.log_state() + + def reset_state(self) -> None: + # set current job status and id to default value + self.job_state, self.job_id = None, None + + def completed(self) -> bool: + return self.job_state == ShopifyBulkStatus.COMPLETED.value + + def running(self) -> bool: + return self.job_state == ShopifyBulkStatus.RUNNING.value + + def failed(self) -> bool: + return self.job_state == ShopifyBulkStatus.FAILED.value + + def timeout(self) -> bool: + return self.job_state == ShopifyBulkStatus.TIMEOUT.value + + def access_denied(self) -> bool: + return self.job_state == ShopifyBulkStatus.ACCESS_DENIED.value + + def get_status_args(self, bulk_job_id: str) -> Mapping[str, Any]: + return { + "method": "POST", + "url": self.base_url, + "data": ShopifyBulkTemplates.status(bulk_job_id), + "headers": {"Content-Type": "application/graphql"}, + } + + def log_state(self) -> None: + self.logger.info(f"The BULK Job: `{self.job_id}` is {self.job_state}.") + + def on_running_job(self, **kwargs) -> None: + sleep(self.job_check_interval_sec) + + def on_completed_job(self, response: Optional[requests.Response] = None) -> Optional[requests.Response]: + # reset status on COMPLETED job + self.reset_state() + return self.retrieve_result(response) + + def retrieve_result(self, response: Optional[requests.Response] = None) -> Optional[str]: + job_result_url = response.json().get("data", {}).get("node", {}).get("url") if response else None + if job_result_url: + # save to local file using chunks to avoid OOM + filename = self.tools.filename_from_url(job_result_url) + with self.session.get(job_result_url, stream=True) as response: + response.raise_for_status() + with open(filename, "wb") as file: + for chunk in response.iter_content(chunk_size=self.retrieve_chunk_size): + file.write(chunk) + # add `` line to the bottom of the saved data for easy parsing + file.write(END_OF_FILE.encode()) + return filename + + def on_failed_job(self, response: requests.Response) -> None: + raise ShopifyBulkExceptions.BulkJobFailed( + f"The BULK Job: `{self.job_id}` exited with {self.job_state}, details: {response.text}", + ) + + def on_timeout_job(self, **kwargs) -> None: + raise ShopifyBulkExceptions.BulkJobTimout( + f"The BULK Job: `{self.job_id}` exited with {self.job_state}, please reduce the `GraphQL BULK Date Range in Days` in SOURCES > Your Shopify Source > SETTINGS.", + ) + + def on_access_denied_job(self, **kwagrs) -> None: + raise ShopifyBulkExceptions.BulkJobAccessDenied( + f"The BULK Job: `{self.job_id}` exited with {self.job_state}, please check your PERMISSION to fetch the data for this stream.", + ) + + def on_job_with_errors(self, errors: List[Mapping[str, Any]]) -> None: + raise ShopifyBulkExceptions.BulkJobUnknownError(f"Could not validate the status of the BULK Job `{self.job_id}`. Errors: {errors}.") + + def on_successful_job(self, response: requests.Response) -> None: + self.update_status(response) + process_fn = self.process_job_fn_mapping.get(self.job_state) + if process_fn: + process_fn(response=response) + + def check_state(self, bulk_job_id: str, is_running_test: bool = False) -> Optional[str]: + response = None + status_args = self.get_status_args(bulk_job_id) + while not self.completed(): + # re-use of `self._session(*, **)` to make BULK Job status checks + response = self.session.request(**status_args) + errors = self.check_for_errors(response) + if not errors: + self.on_successful_job(response) + if is_running_test: + return None + else: + # execute ERRORS scenario + self.on_job_with_errors(errors) + # return `job_result_url`: str when status is `COMPLETED` + return self.on_completed_job(response) + + def has_running_concurrent_job(self, errors: Optional[Iterable[Mapping[str, Any]]] = None) -> bool: + """ + When concurent BULK Job is already running for the same SHOP we receive: + Error example: + [ + { + 'field': None, + 'message': 'A bulk query operation for this app and shop is already in progress: gid://shopify/BulkOperation/4039184154813.', + } + ] + """ + + concurent_job_pattern = "A bulk query operation for this app and shop is already in progress" + # the errors are handled in `job_check_for_errors` + if errors: + for error in errors: + message = error.get("message", "") + if concurent_job_pattern in message: + return True + # reset the `concurrent_attempt` counter, once there is no concurrent job error + self.concurrent_attempt = 0 + return False + + def has_reached_max_concurrency_attempt(self) -> bool: + return self.concurrent_attempt == self.concurrent_max_retry + + def _job_create(self, request: requests.PreparedRequest) -> requests.Response: + """ + Sends HTTPS request to create Shopify BULK Operatoion. + https://shopify.dev/docs/api/usage/bulk-operations/queries#bulk-query-overview + """ + return self.session.send(request) + + def retry_concurrent_request(self, request: requests.PreparedRequest) -> Optional[requests.Response]: + # increment attempt + self.concurrent_attempt += 1 + # try to execute previous request, it's handy because we can retry / each slice yielded + self.logger.warning( + f"The BULK concurrency limit has reached. Waiting {self.concurrent_interval_sec} sec before retry, atttempt: {self.concurrent_attempt}.", + ) + sleep(self.concurrent_interval_sec) + # retry current `request` + return self.job_healthcheck(self._job_create(request)) + + def job_get_id(self, response: requests.Response) -> Optional[str]: + response_data = response.json() + bulk_response = response_data.get("data", {}).get("bulkOperationRunQuery", {}).get("bulkOperation", {}) + if bulk_response and bulk_response.get("status") == ShopifyBulkStatus.CREATED.value: + job_id = bulk_response.get("id") + self.logger.info(f"The BULK Job: `{job_id}` is {ShopifyBulkStatus.CREATED.value}") + return job_id + else: + return None + + def job_retry_on_concurrency(self, request: requests.PreparedRequest) -> Optional[requests.Response]: + if self.has_reached_max_concurrency_attempt(): + # indicate we're out of attempts to retry with job creation + message = f"The BULK Job couldn't be created at this time, since another job is running." + # log the message + self.logger.error(message) + # raise AibyteTracebackException with `INCOMPLETE` status + raise ShopifyBulkExceptions.BulkJobConcurrentError(message) + else: + return self.retry_concurrent_request(request) + + def job_healthcheck(self, response: requests.Response) -> Optional[requests.Response]: + # errors check + errors = self.check_for_errors(response) + # when the concurrent job takes place, we typically need to wait and retry, but no longer than 10 min. + if not self.has_running_concurrent_job(errors): + return response if not errors else None + else: + # get the latest request to retry + request: requests.PreparedRequest = response.request + return self.job_retry_on_concurrency(request) + + @limiter.balance_rate_limit(api_type=ApiTypeEnum.graphql.value) + def job_check(self, created_job_response: requests.Response, is_running_test: bool = False) -> Optional[str]: + """ + This method checks the status for the BULK Job created, using it's `ID`. + The time spent for the Job execution is tracked to understand the effort. + + :: is_running_test: bool (default = False) - service flag to be able to manage unit_tests for `RUNNING` status. + """ + job_response = self.job_healthcheck(created_job_response) + bulk_job_id: str = self.job_get_id(job_response) + job_started = time() + try: + return self.check_state(bulk_job_id, is_running_test) + except ( + ShopifyBulkExceptions.BulkJobFailed, + ShopifyBulkExceptions.BulkJobTimout, + ShopifyBulkExceptions.BulkJobAccessDenied, + ShopifyBulkExceptions.BulkJobUnknownError, + ) as bulk_job_error: + raise bulk_job_error + finally: + time_elapsed = round((time() - job_started), 3) + self.logger.info(f"The BULK Job: `{bulk_job_id}` time elapsed: {time_elapsed} sec.") diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/query.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/query.py new file mode 100644 index 000000000000..e534c61a13d7 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/query.py @@ -0,0 +1,1382 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from abc import abstractmethod +from enum import Enum +from string import Template +from typing import Any, List, Mapping, MutableMapping, Optional, Union + +from graphql_query import Argument, Field, InlineFragment, Operation, Query + +from .tools import BULK_PARENT_KEY, BulkTools + + +class ShopifyBulkTemplates: + @staticmethod + def status(bulk_job_id: str) -> str: + return Template( + """query { + node(id: "$job_id") { + ... on BulkOperation { + id + status + errorCode + objectCount + fileSize + url + partialDataUrl + } + } + }""" + ).substitute(job_id=bulk_job_id) + + @staticmethod + def prepare(query: str) -> str: + bulk_template = Template( + '''mutation { + bulkOperationRunQuery( + query: """ + $query + """ + ) { + bulkOperation { + id + status + } + userErrors { + field + message + } + } + }''' + ) + return bulk_template.substitute(query=query) + + +class ShopifyBulkQuery: + tools: BulkTools = BulkTools() + + def __init__(self, shop_id: int) -> None: + self.shop_id = shop_id + + def get(self, filter_field: Optional[str] = None, start: Optional[str] = None, end: Optional[str] = None) -> str: + # define filter query string, if passed + filter_query = f"{filter_field}:>='{start}' AND {filter_field}:<='{end}'" if filter_field else None + # building query + self.constructed_query: Query = self.query(filter_query) + # resolving + return self.resolve(self.constructed_query) + + @property + @abstractmethod + def query_name(self) -> str: + """ + Defines the root graph node name to fetch from: https://shopify.dev/docs/api/admin-graphql + """ + + @property + def record_composition(self) -> Optional[Mapping[str, Any]]: + """ + Example: + { + "new_record": "Collection", // the GQL Typename of the parent entity + "record_components": [ + "CollectionPublication" // each `collection` has List `publications` + ], + } + """ + return {} + + @property + def sort_key(self) -> Optional[str]: + """ + The field name by which the records are ASC sorted, if defined. + """ + return None + + @property + def query_nodes(self) -> Optional[Union[List[Field], List[str]]]: + """ + Defines the fields for final graph selection. + https://shopify.dev/docs/api/admin-graphql + """ + return ["__typename", "id"] + + def query(self, filter_query: Optional[str] = None) -> Query: + """ + Overide this method, if you need to customize query build logic. + Output example to BULK query `` with `filter query`: + { + (query: "") { + edges { + node { + id + } + } + } + } + """ + # return the constructed query operation + return self.build(self.query_name, self.query_nodes, filter_query) + + def build( + self, + name: str, + edges: Optional[Union[List[Field], List[InlineFragment], Field, InlineFragment]] = None, + filter_query: Optional[str] = None, + additional_query_args: Optional[Mapping[str, Any]] = None, + ) -> Query: + """ + Defines the root of the graph with edges. + """ + args: List[Argument] = [] + # constructing arguments + if filter_query: + args.append(Argument(name="query", value=f'"{filter_query}"')) + if self.sort_key: + args.append(Argument(name="sortKey", value=self.sort_key)) + if additional_query_args: + for k, v in additional_query_args.items(): + args.append(Argument(name=k, value=v)) + # constructing edges + fields = [ + Field(name="edges", fields=[Field(name="node", fields=edges if edges else ["id"])]), + ] + # return constucted query + return Query(name=name, arguments=args, fields=fields) + + def resolve(self, query: Query) -> str: + """ + Default query resolver from type(Operation) > type(str). + Overide this method to build multiple queries in one, if needed. + """ + # return the constructed query operation + return Operation(type="", queries=[query]).render() + + def record_process_components(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + """ + Defines how to process collected components, default `as is`. + """ + yield record + + +class MetafieldType(Enum): + CUSTOMERS = "customers" + ORDERS = "orders" + DRAFT_ORDERS = "draftOrders" + PRODUCTS = "products" + PRODUCT_IMAGES = ["products", "images"] + PRODUCT_VARIANTS = "productVariants" + COLLECTIONS = "collections" + LOCATIONS = "locations" + + +class Metafield(ShopifyBulkQuery): + """ + Only 2 lvl nesting is available: https://shopify.dev/docs/api/usage/bulk-operations/queries#operation-restrictions + Output example to BULK query `customers.metafields` with `filter query` by `updated_at` sorted `ASC`: + { + ( + query: "updated_at:>='2023-04-13' AND updated_at:<='2023-12-01'" + sortKey: UPDATED_AT + ) { + edges { + node { + __typename + id + metafields { + edges { + node { + __typename + id + namespace + value + key + description + createdAt + updatedAt + type + } + } + } + } + } + } + } + """ + + sort_key = "UPDATED_AT" + record_composition = {"new_record": "Metafield"} + + metafield_fields: List[Field] = [ + "__typename", + "id", + "namespace", + "value", + "key", + "description", + "createdAt", + "updatedAt", + "type", + ] + + @property + def query_name(self) -> str: + if isinstance(self.type.value, list): + return self.type.value[0] + elif isinstance(self.type.value, str): + return self.type.value + + @property + @abstractmethod + def type(self) -> MetafieldType: + """ + Defines the Metafield type to fetch, see `MetafieldType` for more info. + """ + + def get_edge_node(self, name: str, fields: Union[List[str], List[Field], str]) -> Field: + """ + Defines the edge of the graph and it's fields to select for Shopify BULK Operaion. + https://shopify.dev/docs/api/usage/bulk-operations/queries#the-jsonl-data-format + """ + return Field(name=name, fields=[Field(name="edges", fields=[Field(name="node", fields=fields)])]) + + @property + def query_nodes(self) -> List[Field]: + """ + List of available fields: + https://shopify.dev/docs/api/admin-graphql/unstable/objects/Metafield + """ + # define metafield node + metafield_node = self.get_edge_node("metafields", self.metafield_fields) + + if isinstance(self.type.value, list): + return ["__typename", "id", self.get_edge_node(self.type.value[1], ["__typename", "id", metafield_node])] + elif isinstance(self.type.value, str): + return ["__typename", "id", metafield_node] + + def record_process_components(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + # resolve parent id from `str` to `int` + record["owner_id"] = self.tools.resolve_str_id(record.get(BULK_PARENT_KEY)) + # add `owner_resource` field + record["owner_resource"] = self.tools.camel_to_snake(record.get(BULK_PARENT_KEY, "").split("/")[3]) + # remove `__parentId` from record + record.pop(BULK_PARENT_KEY, None) + # convert dates from ISO-8601 to RFC-3339 + record["createdAt"] = self.tools.from_iso8601_to_rfc3339(record, "createdAt") + record["updatedAt"] = self.tools.from_iso8601_to_rfc3339(record, "updatedAt") + record = self.tools.fields_names_to_snake_case(record) + yield record + + +class MetafieldCollection(Metafield): + """ + { + collections(query: "updated_at:>='2023-02-07T00:00:00+00:00' AND updated_at:<='2023-12-04T00:00:00+00:00'", sortKey: UPDATED_AT) { + edges { + node { + id + metafields { + edges { + node { + id + namespace + value + key + description + createdAt + updatedAt + type + } + } + } + } + } + } + } + """ + + type = MetafieldType.COLLECTIONS + + +class MetafieldCustomer(Metafield): + """ + { + customers(query: "updated_at:>='2023-02-07T00:00:00+00:00' AND updated_at:<='2023-12-04T00:00:00+00:00'", sortKey: UPDATED_AT) { + edges { + node { + id + metafields { + edges { + node { + id + namespace + value + key + description + createdAt + updatedAt + type + } + } + } + } + } + } + } + """ + + type = MetafieldType.CUSTOMERS + + +class MetafieldLocation(Metafield): + """ + { + locations { + edges { + node { + id + metafields { + edges { + node { + id + namespace + value + key + description + createdAt + updatedAt + type + } + } + } + } + } + } + } + """ + + sort_key = None + type = MetafieldType.LOCATIONS + + +class MetafieldOrder(Metafield): + """ + { + orders(query: "updated_at:>='2023-02-07T00:00:00+00:00' AND updated_at:<='2023-12-04T00:00:00+00:00'", sortKey: UPDATED_AT) { + edges { + node { + id + metafields { + edges { + node { + id + namespace + value + key + description + createdAt + updatedAt + type + } + } + } + } + } + } + } + """ + + type = MetafieldType.ORDERS + + +class MetafieldDraftOrder(Metafield): + """ + { + draftOrders(query: "updated_at:>='2023-02-07T00:00:00+00:00' AND updated_at:<='2023-12-04T00:00:00+00:00'", sortKey: UPDATED_AT) { + edges { + node { + id + metafields { + edges { + node { + id + namespace + value + key + description + createdAt + updatedAt + type + } + } + } + } + } + } + } + """ + + type = MetafieldType.DRAFT_ORDERS + + +class MetafieldProduct(Metafield): + """ + { + products(query: "updated_at:>='2023-02-07T00:00:00+00:00' AND updated_at:<='2023-12-04T00:00:00+00:00'", sortKey: UPDATED_AT) { + edges { + node { + id + metafields { + edges { + node { + id + namespace + value + key + description + createdAt + updatedAt + type + } + } + } + } + } + } + } + """ + + type = MetafieldType.PRODUCTS + + +class MetafieldProductImage(Metafield): + """ + { + products(query: "updated_at:>='2023-02-07T00:00:00+00:00' AND updated_at:<='2023-12-04T00:00:00+00:00'", sortKey: UPDATED_AT) { + edges { + node { + id + images{ + edges{ + node{ + id + metafields { + edges { + node { + id + namespace + value + key + description + createdAt + updatedAt + type + } + } + } + } + } + } + } + } + } + } + """ + + type = MetafieldType.PRODUCT_IMAGES + + +class MetafieldProductVariant(Metafield): + """ + { + productVariants(query: "updated_at:>='2023-02-07T00:00:00+00:00' AND updated_at:<='2023-12-04T00:00:00+00:00'") { + edges { + node { + id + metafields { + edges { + node { + id + namespace + value + key + description + createdAt + updatedAt + type + } + } + } + } + } + } + } + """ + + sort_key = None + type = MetafieldType.PRODUCT_VARIANTS + + +class DiscountCode(ShopifyBulkQuery): + """ + Output example to BULK query `codeDiscountNodes` with `filter query` by `updated_at` sorted `ASC`: + { + codeDiscountNodes(query: "updated_at:>='2023-12-07T00:00:00Z' AND updated_at:<='2023-12-30T00:00:00Z'", sortKey: UPDATED_AT) { + edges { + node { + __typename + id + codeDiscount { + ... on DiscountCodeApp { + updatedAt + createdAt + discountType: discountClass + codes { + edges { + node { + __typename + usageCount: asyncUsageCount + code + id + } + } + } + } + ... on DiscountCodeBasic { + createdAt + updatedAt + discountType: discountClass + summary + codes { + edges { + node { + __typename + usageCount: asyncUsageCount + code + id + } + } + } + } + ... on DiscountCodeBxgy { + updatedAt + createdAt + discountType: discountClass + summary + codes { + edges { + node { + __typename + usageCount: asyncUsageCount + code + id + } + } + } + } + ... on DiscountCodeFreeShipping { + updatedAt + createdAt + discountType: discountClass + summary + codes { + edges { + node { + __typename + usageCount: asyncUsageCount + code + id + } + } + } + } + } + } + } + } + } + """ + + query_name = "codeDiscountNodes" + sort_key = "UPDATED_AT" + + code_discount_fields: List[Field] = [ + Field(name="discountClass", alias="discountType"), + Field( + name="codes", + fields=[ + Field( + name="edges", + fields=[ + Field( + name="node", + fields=[ + "__typename", + Field(name="asyncUsageCount", alias="usageCount"), + "code", + "id", + ], + ) + ], + ) + ], + ), + ] + + code_discount_fragments: List[InlineFragment] = [ + # the type: DiscountCodeApp has no `"summary"` field available + InlineFragment(type="DiscountCodeApp", fields=["updatedAt", "createdAt", *code_discount_fields]), + InlineFragment(type="DiscountCodeBasic", fields=["updatedAt", "createdAt", "summary", *code_discount_fields]), + InlineFragment(type="DiscountCodeBxgy", fields=["updatedAt", "createdAt", "summary", *code_discount_fields]), + InlineFragment(type="DiscountCodeFreeShipping", fields=["updatedAt", "createdAt", "summary", *code_discount_fields]), + ] + + query_nodes: List[Field] = [ + "__typename", + "id", + Field(name="codeDiscount", fields=code_discount_fragments), + ] + + record_composition = { + "new_record": "DiscountCodeNode", + # each DiscountCodeNode has `DiscountRedeemCode` + "record_components": ["DiscountRedeemCode"], + } + + def record_process_components(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + """ + Defines how to process collected components. + """ + + record_components = record.get("record_components", {}) + if record_components: + discounts = record_components.get("DiscountRedeemCode", []) + if len(discounts) > 0: + for discount in discounts: + # resolve parent id from `str` to `int` + discount["admin_graphql_api_id"] = discount.get("id") + discount["price_rule_id"] = self.tools.resolve_str_id(discount.get(BULK_PARENT_KEY)) + discount["id"] = self.tools.resolve_str_id(discount.get("id")) + code_discount = record.get("codeDiscount", {}) + if code_discount: + discount.update(**code_discount) + discount.pop(BULK_PARENT_KEY, None) + # field names to snake case for discount + discount = self.tools.fields_names_to_snake_case(discount) + # convert dates from ISO-8601 to RFC-3339 + discount["created_at"] = self.tools.from_iso8601_to_rfc3339(discount, "created_at") + discount["updated_at"] = self.tools.from_iso8601_to_rfc3339(discount, "updated_at") + yield discount + + +class Collection(ShopifyBulkQuery): + """ + { + collections(query: "updated_at:>='2023-02-07T00:00:00+00:00' AND updated_at:<='2023-12-04T00:00:00+00:00'", sortKey: UPDATED_AT) { + edges { + node { + __typename + id + handle + title + updatedAt + bodyHtml: descriptionHtml + publications { + edges { + node { + __typename + publishedAt: publishDate + } + } + } + sortOrder + templateSuffix + productsCount + } + } + } + } + """ + + query_name = "collections" + sort_key = "UPDATED_AT" + + publications_fields: List[Field] = [ + Field(name="edges", fields=[Field(name="node", fields=["__typename", Field(name="publishDate", alias="publishedAt")])]) + ] + + query_nodes: List[Field] = [ + "__typename", + "id", + Field(name="handle"), + Field(name="title"), + Field(name="updatedAt"), + Field(name="descriptionHtml", alias="bodyHtml"), + Field(name="publications", fields=publications_fields), + Field(name="sortOrder"), + Field(name="templateSuffix"), + Field(name="productsCount"), + ] + + record_composition = { + "new_record": "Collection", + # each collection has `publications` + "record_components": ["CollectionPublication"], + } + + def record_process_components(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + """ + Defines how to process collected components. + """ + record_components = record.get("record_components", {}) + if record_components: + publications = record_components.get("CollectionPublication", []) + if len(publications) > 0: + record["published_at"] = publications[0].get("publishedAt") + record.pop("record_components") + # convert dates from ISO-8601 to RFC-3339 + record["published_at"] = self.tools.from_iso8601_to_rfc3339(record, "published_at") + record["updatedAt"] = self.tools.from_iso8601_to_rfc3339(record, "updatedAt") + # remove leftovers + record.pop(BULK_PARENT_KEY, None) + yield record + + +class InventoryItem(ShopifyBulkQuery): + """ + { + inventoryItems(query: "updated_at:>='2022-04-13T00:00:00+00:00' AND updated_at:<='2023-02-07T00:00:00+00:00'") { + edges { + node { + __typename + unitCost { + cost: amount + } + countryCodeOfOrigin + countryHarmonizedSystemCodes { + edges { + node { + harmonizedSystemCode + countryCode + } + } + } + harmonizedSystemCode + provinceCodeOfOrigin + updatedAt + createdAt + sku + tracked + requiresShipping + } + } + } + } + """ + + query_name = "inventoryItems" + + country_harmonizedS_system_codes: List[Field] = [ + Field(name="edges", fields=[Field(name="node", fields=["__typename", "harmonizedSystemCode", "countryCode"])]) + ] + + query_nodes: List[Field] = [ + "__typename", + "id", + Field(name="unitCost", fields=[Field(name="amount", alias="cost")]), + Field(name="countryCodeOfOrigin"), + Field(name="countryHarmonizedSystemCodes", fields=country_harmonizedS_system_codes), + Field(name="harmonizedSystemCode"), + Field(name="provinceCodeOfOrigin"), + Field(name="updatedAt"), + Field(name="createdAt"), + Field(name="sku"), + Field(name="tracked"), + Field(name="requiresShipping"), + ] + + record_composition = { + "new_record": "InventoryItem", + } + + def record_process_components(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + """ + Defines how to process collected components. + """ + + # resolve `cost` to root lvl as `number` + unit_cost = record.get("unitCost", {}) + if unit_cost: + record["cost"] = float(unit_cost.get("cost")) + else: + record["cost"] = None + # clean up + record.pop("unitCost", None) + # add empty `country_harmonized_system_codes` array, if missing for record + if "countryHarmonizedSystemCodes" not in record.keys(): + record["country_harmonized_system_codes"] = [] + # convert dates from ISO-8601 to RFC-3339 + record["createdAt"] = self.tools.from_iso8601_to_rfc3339(record, "createdAt") + record["updatedAt"] = self.tools.from_iso8601_to_rfc3339(record, "updatedAt") + record = self.tools.fields_names_to_snake_case(record) + yield record + + +class InventoryLevel(ShopifyBulkQuery): + """ + Output example to BULK query `inventory_levels` from `locations` with `filter query` by `updated_at`: + { + locations(includeLegacy: true, includeInactive: true) { + edges { + node { + __typename + id + inventoryLevels(query: "updated_at:>='2023-04-14T00:00:00+00:00'") { + edges { + node { + __typename + id + available + item { + inventory_item_id: id + } + updatedAt + } + } + } + } + } + } + } + """ + + query_name = "locations" + # in order to return all the locations, additional query args must be provided + # https://shopify.dev/docs/api/admin-graphql/2023-10/queries/locations#query-arguments + locations_query_args = { + "includeLegacy": "true", + "includeInactive": "true", + } + record_composition = { + "new_record": "InventoryLevel", + } + + inventory_levels_fields: List[Field] = [ + "__typename", + "id", + Field(name="available"), + Field(name="item", fields=[Field(name="id", alias="inventory_item_id")]), + Field(name="updatedAt"), + ] + + def query(self, filter_query: Optional[str] = None) -> Query: + # build the nested query first with `filter_query` to have the incremental syncs + inventory_levels: List[Query] = [self.build("inventoryLevels", self.inventory_levels_fields, filter_query)] + # build the main query around previous + # return the constructed query operation + return self.build( + name=self.query_name, + edges=self.query_nodes + inventory_levels, + # passing more query args for `locations` query + additional_query_args=self.locations_query_args, + ) + + def record_process_components(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + """ + Defines how to process collected components. + """ + + # resolve `inventory_item_id` to root lvl + resolve to int + record["inventory_item_id"] = self.tools.resolve_str_id(record.get("item", {}).get("inventory_item_id")) + # add `location_id` from `__parentId` + record["location_id"] = self.tools.resolve_str_id(record[BULK_PARENT_KEY]) + # make composite `id` from `location_id|inventory_item_id` + record["id"] = "|".join((str(record.get("location_id", "")), str(record.get("inventory_item_id", "")))) + # convert dates from ISO-8601 to RFC-3339 + record["updatedAt"] = self.tools.from_iso8601_to_rfc3339(record, "updatedAt") + # remove leftovers + record.pop("item", None) + record.pop(BULK_PARENT_KEY, None) + record = self.tools.fields_names_to_snake_case(record) + yield record + + +class FulfillmentOrder(ShopifyBulkQuery): + """ + Output example to BULK query `fulfillmentOrders` from `orders` with `filter query` by `updated_at`, sorted by `UPDATED_AT`: + { + orders(query: "updated_at:>='2023-04-13T05:00:09Z' and updated_at:<='2023-04-15T05:00:09Z'", sortKey: UPDATED_AT){ + edges { + node { + __typename + id + fulfillmentOrders { + edges { + node { + __typename + id + assignedLocation { + location { + locationId: id + } + address1 + address2 + city + countryCode + name + phone + province + zip + } + destination { + id + address1 + address2 + city + company + countryCode + email + firstName + lastName + phone + province + zip + } + deliveryMethod { + id + methodType + minDeliveryDateTime + maxDeliveryDateTime + } + fulfillAt + fulfillBy + internationalDuties { + incoterm + } + fulfillmentHolds { + reason + reasonNotes + } + lineItems { + edges { + node { + __typename + id + inventoryItemId + lineItem { + lineItemId: id + fulfillableQuantity + quantity: currentQuantity + variant { + variantId: id + } + } + } + } + } + createdAt + updatedAt + requestStatus + status + supportedActions { + action + externalUrl + } + merchantRequests { + edges { + node { + __typename + id + message + kind + requestOptions + } + } + } + } + } + } + } + } + } + } + """ + + query_name = "orders" + sort_key = "UPDATED_AT" + + assigned_location_fields: List[Field] = [ + "address1", + "address2", + "city", + "countryCode", + "name", + "phone", + "province", + "zip", + Field(name="location", fields=[Field(name="id", alias="locationId")]), + ] + + destination_fields: List[Field] = [ + "id", + "address1", + "address2", + "city", + "company", + "countryCode", + "email", + "firstName", + "lastName", + "phone", + "province", + "zip", + ] + + delivery_method_fields: List[Field] = [ + "id", + "methodType", + "minDeliveryDateTime", + "maxDeliveryDateTime", + ] + + line_items_fields: List[Field] = [ + "__typename", + "id", + "inventoryItemId", + Field( + name="lineItem", + fields=[ + Field(name="id", alias="lineItemId"), + "fulfillableQuantity", + Field(name="currentQuantity", alias="quantity"), + Field(name="variant", fields=[Field(name="id", alias="variantId")]), + ], + ), + ] + + merchant_requests_fields: List[Field] = [ + "__typename", + "id", + "message", + "kind", + "requestOptions", + ] + + fulfillment_order_fields: List[Field] = [ + "__typename", + "id", + Field(name="assignedLocation", fields=assigned_location_fields), + Field(name="destination", fields=destination_fields), + Field(name="deliveryMethod", fields=delivery_method_fields), + "fulfillAt", + "fulfillBy", + Field(name="internationalDuties", fields=["incoterm"]), + Field(name="fulfillmentHolds", fields=["reason", "reasonNotes"]), + Field(name="lineItems", fields=[Field(name="edges", fields=[Field(name="node", fields=line_items_fields)])]), + "createdAt", + "updatedAt", + "requestStatus", + "status", + Field(name="supportedActions", fields=["action", "externalUrl"]), + Field(name="merchantRequests", fields=[Field(name="edges", fields=[Field(name="node", fields=merchant_requests_fields)])]), + ] + + query_nodes: List[Field] = [ + "__typename", + "id", + Field(name="fulfillmentOrders", fields=[Field(name="edges", fields=[Field(name="node", fields=fulfillment_order_fields)])]), + ] + + record_composition = { + "new_record": "FulfillmentOrder", + # each FulfillmentOrder has multiple `FulfillmentOrderLineItem` and `FulfillmentOrderMerchantRequest` + "record_components": [ + "FulfillmentOrderLineItem", + "FulfillmentOrderMerchantRequest", + ], + } + + def process_fulfillment_order(self, record: MutableMapping[str, Any], shop_id: int) -> MutableMapping[str, Any]: + # addings + record["shop_id"] = shop_id + record["order_id"] = record.get(BULK_PARENT_KEY) + # unnest nested locationId to the `assignedLocation` + location_id = record.get("assignedLocation", {}).get("location", {}).get("locationId") + record["assignedLocation"]["locationId"] = location_id + record["assigned_location_id"] = location_id + # create nested placeholders for other parts + record["line_items"] = [] + record["merchant_requests"] = [] + # cleaning + record.pop(BULK_PARENT_KEY) + record.get("assignedLocation").pop("location", None) + # resolve ids from `str` to `int` + # location id + location = record.get("assignedLocation", {}) + if location: + location_id = location.get("locationId") + if location_id: + record["assignedLocation"]["locationId"] = self.tools.resolve_str_id(location_id) + # assigned_location_id + record["assigned_location_id"] = self.tools.resolve_str_id(record.get("assigned_location_id")) + # destination id + destination = record.get("destination", {}) + if destination: + destination_id = destination.get("id") + if destination_id: + record["destination"]["id"] = self.tools.resolve_str_id(destination_id) + # delivery method id + delivery_method = record.get("deliveryMethod", {}) + if delivery_method: + delivery_method_id = delivery_method.get("id") + if delivery_method_id: + record["deliveryMethod"]["id"] = self.tools.resolve_str_id(delivery_method_id) + # order id + record["order_id"] = self.tools.resolve_str_id(record.get("order_id")) + # field names to snake for nested objects + # `assignedLocation`(object) field names to snake case + record["assignedLocation"] = self.tools.fields_names_to_snake_case(record.get("assignedLocation")) + # `deliveryMethod`(object) field names to snake case + record["deliveryMethod"] = self.tools.fields_names_to_snake_case(record.get("deliveryMethod")) + # `destination`(object) field names to snake case + record["destination"] = self.tools.fields_names_to_snake_case(record.get("destination")) + # `fulfillmentHolds`(list[object]) field names to snake case + record["fulfillment_holds"] = [self.tools.fields_names_to_snake_case(el) for el in record.get("fulfillment_holds", [])] + # `supportedActions`(list[object]) field names to snake case + record["supported_actions"] = [self.tools.fields_names_to_snake_case(el) for el in record.get("supported_actions", [])] + return record + + def process_line_item(self, record: MutableMapping[str, Any], shop_id: int) -> MutableMapping[str, Any]: + # addings + record["shop_id"] = shop_id + record["fulfillmentOrderId"] = record.get(BULK_PARENT_KEY) + # unnesting nested `lineItem` + line_item = record.get("lineItem", {}) + if line_item: + record["quantity"] = line_item.get("quantity") + record["lineItemId"] = line_item.get("lineItemId") + record["fulfillableQuantity"] = line_item.get("fulfillableQuantity") + variant = line_item.get("variant", {}) + if variant: + record["variantId"] = variant.get("variantId") + # cleaning + record.pop(BULK_PARENT_KEY) + record.pop("lineItem") + # resolve ids from `str` to `int` + record["id"] = self.tools.resolve_str_id(record.get("id")) + # inventoryItemId + record["inventoryItemId"] = self.tools.resolve_str_id(record.get("inventoryItemId")) + # fulfillmentOrderId + record["fulfillmentOrderId"] = self.tools.resolve_str_id(record.get("fulfillmentOrderId")) + # lineItemId + record["lineItemId"] = self.tools.resolve_str_id(record.get("lineItemId")) + # variantId + record["variantId"] = self.tools.resolve_str_id(record.get("variantId")) + # field names to snake case + record = self.tools.fields_names_to_snake_case(record) + return record + + def process_merchant_request(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + # cleaning + record.pop(BULK_PARENT_KEY) + # resolve ids from `str` to `int` + record["id"] = self.tools.resolve_str_id(record.get("id")) + # field names to snake case + record = self.tools.fields_names_to_snake_case(record) + return record + + def record_process_components(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + """ + Defines how to process collected components. + """ + + record = self.process_fulfillment_order(record, self.shop_id) + record_components = record.get("record_components", {}) + if record_components: + line_items = record_components.get("FulfillmentOrderLineItem", []) + if len(line_items) > 0: + for line_item in line_items: + record["line_items"].append(self.process_line_item(line_item, self.shop_id)) + merchant_requests = record_components.get("FulfillmentOrderMerchantRequest", []) + if len(merchant_requests) > 0: + for merchant_request in merchant_requests: + record["merchant_requests"].append(self.process_merchant_request(merchant_request)) + record.pop("record_components") + # convert dates from ISO-8601 to RFC-3339 + record["updatedAt"] = self.tools.from_iso8601_to_rfc3339(record, "updatedAt") + # convert dates from ISO-8601 to RFC-3339 + record["fulfillAt"] = self.tools.from_iso8601_to_rfc3339(record, "fulfillAt") + record["createdAt"] = self.tools.from_iso8601_to_rfc3339(record, "createdAt") + record["updatedAt"] = self.tools.from_iso8601_to_rfc3339(record, "updatedAt") + # delivery method + delivery_method = record.get("deliveryMethod", {}) + if delivery_method: + record["deliveryMethod"]["min_delivery_date_time"] = self.tools.from_iso8601_to_rfc3339( + delivery_method, "min_delivery_date_time" + ) + record["deliveryMethod"]["max_delivery_date_time"] = self.tools.from_iso8601_to_rfc3339( + delivery_method, "max_delivery_date_time" + ) + yield record + + +class Transaction(ShopifyBulkQuery): + """ + Output example to BULK query `transactions` from `orders` with `filter query` by `updated_at` sorted `ASC`: + { + orders(query: "updated_at:>='2021-05-23T00:00:00+00:00' AND updated_at:<'2021-12-22T00:00:00+00:00'", sortKey:UPDATED_AT) { + edges { + node { + __typename + id + currency: currencyCode + transactions { + id + errorCode + parentTransaction { + parentId: id + } + test + kind + amount + receipt: receiptJson + gateway + authorization: authorizationCode + createdAt + status + processedAt + totalUnsettledSet { + presentmentMoney { + amount + currency: currencyCode + } + shopMoney { + amount + currency: currencyCode + } + } + paymentId + paymentDetails { + ... on CardPaymentDetails { + avsResultCode + creditCardBin: bin + creditCardCompany: company + creditCardNumber: number + creditCardName: name + cvvResultCode + creditCardWallet: wallet + creditCardExpirationYear: expirationYear + creditCardExpirationMonth: expirationMonth + } + } + } + } + } + } + } + """ + + query_name = "orders" + sort_key = "UPDATED_AT" + + total_unsettled_set_fields: List[Field] = [ + Field(name="presentmentMoney", fields=["amount", Field(name="currencyCode", alias="currency")]), + Field(name="shopMoney", fields=["amount", Field(name="currencyCode", alias="currency")]), + ] + + payment_details: List[InlineFragment] = [ + InlineFragment( + type="CardPaymentDetails", + fields=[ + "avsResultCode", + "cvvResultCode", + Field(name="bin", alias="creditCardBin"), + Field(name="company", alias="creditCardCompany"), + Field(name="number", alias="creditCardNumber"), + Field(name="name", alias="creditCardName"), + Field(name="wallet", alias="creditCardWallet"), + Field(name="expirationYear", alias="creditCardExpirationYear"), + Field(name="expirationMonth", alias="creditCardExpirationMonth"), + ], + ) + ] + + query_nodes: List[Field] = [ + "__typename", + "id", + Field(name="currencyCode", alias="currency"), + Field( + name="transactions", + fields=[ + "id", + "errorCode", + Field(name="parentTransaction", fields=[Field(name="id", alias="parentId")]), + "test", + "kind", + "amount", + Field(name="receiptJson", alias="receipt"), + "gateway", + Field(name="authorizationCode", alias="authorization"), + "createdAt", + "status", + "processedAt", + Field(name="totalUnsettledSet", fields=total_unsettled_set_fields), + "paymentId", + Field(name="paymentDetails", fields=payment_details), + ], + ), + ] + + record_composition = { + "new_record": "Order", + } + + def process_transaction(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + # save the id before it's resolved + record["admin_graphql_api_id"] = record.get("id") + # unnest nested fields + parent_transaction = record.get("parentTransaction", {}) + if parent_transaction: + record["parent_id"] = parent_transaction.get("parentId") + # str values to float + record["amount"] = float(record.get("amount")) + # convert dates from ISO-8601 to RFC-3339 + record["processedAt"] = self.tools.from_iso8601_to_rfc3339(record, "processedAt") + record["createdAt"] = self.tools.from_iso8601_to_rfc3339(record, "createdAt") + # resolve ids + record["id"] = self.tools.resolve_str_id(record.get("id")) + record["parent_id"] = self.tools.resolve_str_id(record.get("parent_id")) + # remove leftovers + record.pop("parentTransaction", None) + # field names to snake case + total_unsettled_set = record.get("totalUnsettledSet", {}) + if total_unsettled_set: + record["totalUnsettledSet"] = self.tools.fields_names_to_snake_case(total_unsettled_set) + # nested str values to float + record["totalUnsettledSet"]["presentment_money"]["amount"] = float( + total_unsettled_set.get("presentmentMoney", {}).get("amount") + ) + record["totalUnsettledSet"]["shop_money"]["amount"] = float(total_unsettled_set.get("shopMoney", {}).get("amount")) + payment_details = record.get("paymentDetails", {}) + if payment_details: + record["paymentDetails"] = self.tools.fields_names_to_snake_case(payment_details) + # field names to snake case for root level + record = self.tools.fields_names_to_snake_case(record) + return record + + def record_process_components(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + """ + Defines how to process collected components. + """ + + if "transactions" in record.keys(): + transactions = record.get("transactions") + if len(transactions) > 0: + for transaction in transactions: + # populate parent record keys + transaction["order_id"] = record.get("id") + transaction["currency"] = record.get("currency") + yield self.process_transaction(transaction) diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/record.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/record.py new file mode 100644 index 000000000000..3961e60721ca --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/record.py @@ -0,0 +1,143 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from io import TextIOWrapper +from json import loads +from os import remove +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union + +from airbyte_cdk import AirbyteLogger + +from .exceptions import ShopifyBulkExceptions +from .query import ShopifyBulkQuery +from .tools import END_OF_FILE, BulkTools + + +class ShopifyBulkRecord: + def __init__(self, query: ShopifyBulkQuery, logger: AirbyteLogger) -> None: + self.composition = query.record_composition + self.record_process_components = query.record_process_components + self.components: List[str] = self.composition.get("record_components", []) if self.composition else [] + self.buffer: List[MutableMapping[str, Any]] = [] + self.tools: BulkTools = BulkTools() + self.logger = logger + + @staticmethod + def check_type(record: Mapping[str, Any], types: Union[List[str], str]) -> bool: + record_type = record.get("__typename") + if isinstance(types, list): + return any(record_type == t for t in types) + else: + return record_type == types + + def record_new(self, record: MutableMapping[str, Any]) -> None: + record = self.component_prepare(record) + record.pop("__typename") + self.buffer.append(record) + + def record_new_component(self, record: MutableMapping[str, Any]) -> None: + component = record.get("__typename") + record.pop("__typename") + # add component to its placeholder in the components list + self.buffer[-1]["record_components"][component].append(record) + + def component_prepare(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + if self.components: + record["record_components"] = {} + for component in self.components: + record["record_components"][component] = [] + return record + + def buffer_flush(self) -> Iterable[Mapping[str, Any]]: + if len(self.buffer) > 0: + for record in self.buffer: + # resolve id from `str` to `int` + record = self.record_resolve_id(record) + # process record components + yield from self.record_process_components(record) + # clean the buffer + self.buffer.clear() + + def record_compose(self, record: Mapping[str, Any]) -> Optional[Iterable[MutableMapping[str, Any]]]: + """ + Step 1: register the new record by it's `__typename` + Step 2: check for `components` by their `__typename` and add to the placeholder + Step 3: repeat until the ``. + """ + + if self.check_type(record, self.composition.get("new_record")): + # emit from previous iteration, if present + yield from self.buffer_flush() + # register the record + self.record_new(record) + # components check + elif self.check_type(record, self.components): + self.record_new_component(record) + + def process_line(self, jsonl_file: TextIOWrapper) -> Iterable[MutableMapping[str, Any]]: + # process the json lines + for line in jsonl_file: + # we exit from the loop when receive (file ends) + if line == END_OF_FILE: + break + elif line != "": + yield from self.record_compose(loads(line)) + + # emit what's left in the buffer, typically last record + yield from self.buffer_flush() + + def record_resolve_id(self, record: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + """ + The ids are fetched in the format of: " gid://shopify/Order/ " + Input: + { "Id": "gid://shopify/Order/19435458986123"} + We need to extract the actual id from the string instead. + Output: + { "id": 19435458986123, "admin_graphql_api_id": "gid://shopify/Order/19435458986123"} + """ + # save the actual api id to the `admin_graphql_api_id` + # while resolving the `id` in `record_resolve_id`, + # we re-assign the original id like `"gid://shopify/Order/19435458986123"`, + # into `admin_graphql_api_id` have the ability to identify the record oigin correctly in subsequent actions. + id = record.get("id") + if isinstance(id, str): + record["admin_graphql_api_id"] = id + # extracting the int(id) and reassign + record["id"] = self.tools.resolve_str_id(id) + return record + elif isinstance(id, int): + return record + + def produce_records(self, filename: str) -> Iterable[MutableMapping[str, Any]]: + """ + Read the JSONL content saved from `job.job_retrieve_result()` line-by-line to avoid OOM. + The filename example: `bulk-4039263649981.jsonl`, + where `4039263649981` is the `id` of the COMPLETED BULK Jobw with `result_url`. + Note: typically the `filename` is taken from the `result_url` string provided in the response. + """ + + with open(filename, "r") as jsonl_file: + for record in self.process_line(jsonl_file): + yield self.tools.fields_names_to_snake_case(record) + + def read_file(self, filename: str, remove_file: Optional[bool] = True) -> Iterable[Mapping[str, Any]]: + try: + # produce records from saved result + yield from self.produce_records(filename) + except Exception as e: + raise ShopifyBulkExceptions.BulkRecordProduceError( + f"An error occured while producing records from BULK Job result. Trace: {repr(e)}.", + ) + finally: + # removing the tmp file, if requested + if remove_file and filename: + try: + remove(filename) + except Exception as e: + self.logger.info(f"Failed to remove the `tmp job result` file, the file doen't exist. Details: {repr(e)}.") + # we should pass here, if the file wasn't removed , it's either: + # - doesn't exist + # - will be dropped with the container shut down. + pass diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/tools.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/tools.py new file mode 100644 index 000000000000..181d0f111c0f --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/bulk/tools.py @@ -0,0 +1,82 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import os +import re +from typing import Any, Mapping, MutableMapping, Optional, Union +from urllib.parse import parse_qsl, urlparse + +import pendulum as pdm + +from .exceptions import ShopifyBulkExceptions + +# default end line tag +END_OF_FILE: str = "" +BULK_PARENT_KEY: str = "__parentId" + + +class BulkTools: + @staticmethod + def camel_to_snake(camel_case: str) -> str: + snake_case = [] + for char in camel_case: + if char.isupper(): + snake_case.append("_" + char.lower()) + else: + snake_case.append(char) + return "".join(snake_case).lstrip("_") + + @staticmethod + def filename_from_url(job_result_url: str) -> str: + """ + Example of `job_result_url` (str) : + https://storage.googleapis.com/shopify-tiers-assets-prod-us-east1/? + GoogleAccessId=assets-us-prod%40shopify-tiers.iam.gserviceaccount.com& + Expires=1705508208& + Signature=%3D%3D& + response-content-disposition=attachment%3B+filename%3D%22bulk-4147374162109.jsonl%22%3B+filename%2A%3DUTF-8%27%27bulk-4147374162109.jsonl& + response-content-type=application%2Fjsonl + + Output: + (str): bulk-4147374162109.jsonl + """ + # Regular expression pattern to extract the filename + filename_pattern = r'filename\*?=(?:UTF-8\'\')?"([^"]+)"' + parsed_url = dict(parse_qsl(urlparse(job_result_url).query)) + match = re.search(filename_pattern, parsed_url.get("response-content-disposition", "")) + if match: + return match.group(1) + else: + raise ShopifyBulkExceptions.BulkJobResultUrlError( + f"Could not extract the `filename` from `result_url` provided, details: {job_result_url}", + ) + + @staticmethod + def from_iso8601_to_rfc3339(record: Mapping[str, Any], field: str) -> Mapping[str, Any]: + """ + Converts date-time as follows: + Input: "2023-01-01T15:00:00Z" + Output: "2023-01-01T15:00:00+00:00" + If the value of the `field` is `None` we return it `as is`. + """ + # some fields that expected to be resolved as ids, might not be populated for the particular `RECORD`, + # we should return `None` to make the field `null` in the output as the result of the transformation. + target_value = record.get(field) + return pdm.parse(target_value).to_rfc3339_string() if target_value else record.get(field) + + def fields_names_to_snake_case(self, dict_input: Optional[Mapping[str, Any]] = None) -> Optional[MutableMapping[str, Any]]: + # transforming record field names from camel to snake case, leaving the `__parent_id` relation in place + if dict_input: + # the `None` type check is required, to properly handle nested missing entities (return None) + return {self.camel_to_snake(k) if dict_input and k != BULK_PARENT_KEY else k: v for k, v in dict_input.items()} + + @staticmethod + def resolve_str_id(str_input: Optional[str] = None, output_type: Optional[Union[int, str, float]] = int) -> Union[int, str, float]: + # some fields that expected to be resolved as ids, might not be populated for the particular `RECORD`, + # we should return `None` to make the field `null` in the output as the result of the transformation. + if str_input: + return output_type(re.search(r"\d+", str_input).group()) + else: + return None diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/graphql.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/graphql.py similarity index 84% rename from airbyte-integrations/connectors/source-shopify/source_shopify/graphql.py rename to airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/graphql.py index e729c74be0f7..462ad3ea3aa8 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/graphql.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/graphql.py @@ -2,13 +2,14 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from typing import Optional import sgqlc.operation -from . import shopify_schema +from . import schema -_schema = shopify_schema +_schema = schema _schema_root = _schema.shopify_schema @@ -26,10 +27,12 @@ def _camel_to_snake(camel_case: str): def get_query_products(first: int, filter_field: str, filter_value: str, next_page_token: Optional[str]): op = sgqlc.operation.Operation(_schema_root.query_type) snake_case_filter_field = _camel_to_snake(filter_field) - if next_page_token: - products = op.products(first=first, query=f"{snake_case_filter_field}:>'{filter_value}'", after=next_page_token) - else: - products = op.products(first=first, query=f"{snake_case_filter_field}:>'{filter_value}'") + products_args = { + "first": first, + "query": f"{snake_case_filter_field}:>'{filter_value}'" if filter_value else None, + "after": next_page_token, + } + products = op.products(**products_args) products.nodes.id() products.nodes.title() products.nodes.updated_at() diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/schema.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/schema.py new file mode 100644 index 000000000000..d3647a562084 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_graphql/schema.py @@ -0,0 +1,29442 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sgqlc.types +import sgqlc.types.datetime +import sgqlc.types.relay + +shopify_schema = sgqlc.types.Schema() + + +# Unexport Node/PageInfo, let schema re-declare them +shopify_schema -= sgqlc.types.relay.Node +shopify_schema -= sgqlc.types.relay.PageInfo + + +######################################################################## +# Scalars and Enumerations +######################################################################## +class ARN(sgqlc.types.Scalar): + __schema__ = shopify_schema + + +class AbandonmentAbandonmentType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("BROWSE", "CART", "CHECKOUT") + + +class AbandonmentDeliveryState(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("NOT_SENT", "SCHEDULED", "SENT") + + +class AbandonmentEmailState(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("NOT_SENT", "SCHEDULED", "SENT") + + +class AbandonmentEmailStateUpdateUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ABANDONMENT_NOT_FOUND",) + + +class AbandonmentUpdateActivitiesDeliveryStatusesUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ABANDONMENT_NOT_FOUND", "DELIVERY_STATUS_INFO_NOT_FOUND", "MARKETING_ACTIVITY_NOT_FOUND") + + +class AppDeveloperType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("MERCHANT", "PARTNER", "SHOPIFY", "UNKNOWN") + + +class AppInstallationCategory(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CHANNEL", "POS_EMBEDDED") + + +class AppInstallationPrivacy(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("PRIVATE", "PUBLIC") + + +class AppInstallationSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("APP_TITLE", "ID", "INSTALLED_AT", "RELEVANCE") + + +class AppPricingInterval(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ANNUAL", "EVERY_30_DAYS") + + +class AppPublicCategory(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CUSTOM", "OTHER", "PRIVATE", "PUBLIC") + + +class AppPurchaseStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ACTIVE", "DECLINED", "EXPIRED", "PENDING") + + +class AppRevenueAttributionRecordCreateUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("INVALID", "TAKEN") + + +class AppRevenueAttributionRecordDeleteUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("INVALID",) + + +class AppRevenueAttributionRecordSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATED_AT", "ID", "RELEVANCE") + + +class AppRevenueAttributionType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("APPLICATION_PURCHASE", "APPLICATION_SUBSCRIPTION", "APPLICATION_USAGE", "OTHER") + + +class AppSubscriptionReplacementBehavior(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("APPLY_IMMEDIATELY", "APPLY_ON_NEXT_BILLING_CYCLE", "STANDARD") + + +class AppSubscriptionSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATED_AT", "ID", "RELEVANCE") + + +class AppSubscriptionStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ACTIVE", "CANCELLED", "DECLINED", "EXPIRED", "FROZEN", "PENDING") + + +class AppSubscriptionTrialExtendUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("SUBSCRIPTION_NOT_ACTIVE", "SUBSCRIPTION_NOT_FOUND", "TRIAL_NOT_ACTIVE") + + +class AppTransactionSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATED_AT", "ID", "RELEVANCE") + + +class AppUsageRecordSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATED_AT", "ID", "RELEVANCE") + + +class AutomaticDiscountSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATED_AT", "ID", "RELEVANCE") + + +class BadgeType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ATTENTION", "DEFAULT", "INFO", "SUCCESS", "WARNING") + + +class BillingAttemptUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "BLANK", + "CONTRACT_NOT_FOUND", + "CYCLE_INDEX_OUT_OF_RANGE", + "CYCLE_START_DATE_OUT_OF_RANGE", + "INVALID", + "ORIGIN_TIME_BEFORE_CONTRACT_CREATION", + "ORIGIN_TIME_OUT_OF_RANGE", + "UPCOMING_CYCLE_LIMIT_EXCEEDED", + ) + + +Boolean = sgqlc.types.Boolean + + +class BulkMutationErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("INTERNAL_FILE_SERVER_ERROR", "INVALID_MUTATION", "INVALID_STAGED_UPLOAD_FILE", "NO_SUCH_FILE", "OPERATION_IN_PROGRESS") + + +class BulkOperationErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ACCESS_DENIED", "INTERNAL_SERVER_ERROR", "TIMEOUT") + + +class BulkOperationStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CANCELED", "CANCELING", "COMPLETED", "CREATED", "EXPIRED", "FAILED", "RUNNING") + + +class BulkOperationType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("MUTATION", "QUERY") + + +class BulkProductResourceFeedbackCreateUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "BLANK", + "INVALID", + "LESS_THAN_OR_EQUAL_TO", + "MAXIMUM_FEEDBACK_LIMIT_EXCEEDED", + "OUTDATED_FEEDBACK", + "PRESENT", + "PRODUCT_NOT_FOUND", + ) + + +class BusinessCustomerErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "BLANK", + "FAILED_TO_DELETE", + "INTERNAL_ERROR", + "INVALID", + "INVALID_INPUT", + "LIMIT_REACHED", + "NO_INPUT", + "REQUIRED", + "RESOURCE_NOT_FOUND", + "TAKEN", + "TOO_LONG", + "UNEXPECTED_TYPE", + ) + + +class CartTransformCreateUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FUNCTION_ALREADY_REGISTERED", "FUNCTION_DOES_NOT_IMPLEMENT", "FUNCTION_NOT_FOUND", "INPUT_INVALID") + + +class CartTransformDeleteUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("NOT_FOUND", "UNAUTHORIZED_APP_SCOPE") + + +class CatalogSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ID", "RELEVANCE", "TITLE") + + +class CatalogStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ACTIVE", "ARCHIVED", "DRAFT") + + +class CatalogType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("APP", "COMPANY_LOCATION", "MARKET", "NONE") + + +class CatalogUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "APP_CATALOG_PRICE_LIST_ASSIGNMENT", + "BLANK", + "CANNOT_ADD_MORE_THAN_ONE_MARKET", + "CANNOT_CREATE_APP_CATALOG", + "CANNOT_CREATE_MARKET_CATALOG", + "CANNOT_DELETE_APP_CATALOG", + "CANNOT_DELETE_MARKET_CATALOG", + "CANNOT_MODIFY_APP_CATALOG", + "CANNOT_MODIFY_MARKET_CATALOG", + "CATALOG_CONTEXT_DOES_NOT_SUPPORT_QUANTITY_RULES", + "CATALOG_FAILED_TO_SAVE", + "CATALOG_NOT_FOUND", + "COMPANY_LOCATION_CATALOG_STATUS_PLAN", + "COMPANY_LOCATION_NOT_FOUND", + "CONTEXT_ALREADY_ASSIGNED_TO_CATALOG", + "CONTEXT_CATALOG_LIMIT_REACHED", + "CONTEXT_DRIVER_MISMATCH", + "COUNTRY_PRICE_LIST_ASSIGNMENT", + "INVALID", + "INVALID_CATALOG_CONTEXT_TYPE", + "MARKET_AND_PRICE_LIST_CURRENCY_MISMATCH", + "MARKET_CATALOG_STATUS", + "MARKET_NOT_FOUND", + "MARKET_TAKEN", + "MUST_PROVIDE_EXACTLY_ONE_CONTEXT_TYPE", + "PRICE_LIST_FAILED_TO_SAVE", + "PRICE_LIST_LOCKED", + "PRICE_LIST_NOT_ALLOWED_FOR_PRIMARY_MARKET", + "PRICE_LIST_NOT_FOUND", + "PUBLICATION_NOT_FOUND", + "REQUIRES_CONTEXTS_TO_ADD_OR_REMOVE", + "TAKEN", + "TOO_LONG", + "TOO_SHORT", + "UNSUPPORTED_CATALOG_ACTION", + ) + + +class CheckoutProfileSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATED_AT", "EDITED_AT", "ID", "IS_PUBLISHED", "RELEVANCE", "UPDATED_AT") + + +class CodeDiscountSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATED_AT", "ENDS_AT", "ID", "RELEVANCE", "STARTS_AT", "TITLE", "UPDATED_AT") + + +class CollectionAddProductsV2UserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CANT_ADD_TO_SMART_COLLECTION", "COLLECTION_DOES_NOT_EXIST") + + +class CollectionRuleColumn(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "IS_PRICE_REDUCED", + "PRODUCT_METAFIELD_DEFINITION", + "PRODUCT_TAXONOMY_NODE_ID", + "TAG", + "TITLE", + "TYPE", + "VARIANT_COMPARE_AT_PRICE", + "VARIANT_INVENTORY", + "VARIANT_METAFIELD_DEFINITION", + "VARIANT_PRICE", + "VARIANT_TITLE", + "VARIANT_WEIGHT", + "VENDOR", + ) + + +class CollectionRuleRelation(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "CONTAINS", + "ENDS_WITH", + "EQUALS", + "GREATER_THAN", + "IS_NOT_SET", + "IS_SET", + "LESS_THAN", + "NOT_CONTAINS", + "NOT_EQUALS", + "STARTS_WITH", + ) + + +class CollectionSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ID", "RELEVANCE", "TITLE", "UPDATED_AT") + + +class CollectionSortOrder(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ALPHA_ASC", "ALPHA_DESC", "BEST_SELLING", "CREATED", "CREATED_DESC", "MANUAL", "PRICE_ASC", "PRICE_DESC") + + +class CompanyAddressType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("BILLING", "SHIPPING") + + +class CompanyContactRoleAssignmentSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATED_AT", "ID", "LOCATION_NAME", "RELEVANCE", "UPDATED_AT") + + +class CompanyContactRoleSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATED_AT", "ID", "RELEVANCE", "UPDATED_AT") + + +class CompanyContactSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("COMPANY_ID", "CREATED_AT", "EMAIL", "ID", "NAME", "NAME_EMAIL", "RELEVANCE", "TITLE", "UPDATED_AT") + + +class CompanyLocationSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("COMPANY_AND_LOCATION_NAME", "COMPANY_ID", "CREATED_AT", "ID", "NAME", "RELEVANCE", "UPDATED_AT") + + +class CompanySortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATED_AT", "ID", "NAME", "ORDER_COUNT", "RELEVANCE", "SINCE_DATE", "TOTAL_SPENT", "UPDATED_AT") + + +class CountryCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "AC", + "AD", + "AE", + "AF", + "AG", + "AI", + "AL", + "AM", + "AN", + "AO", + "AR", + "AT", + "AU", + "AW", + "AX", + "AZ", + "BA", + "BB", + "BD", + "BE", + "BF", + "BG", + "BH", + "BI", + "BJ", + "BL", + "BM", + "BN", + "BO", + "BQ", + "BR", + "BS", + "BT", + "BV", + "BW", + "BY", + "BZ", + "CA", + "CC", + "CD", + "CF", + "CG", + "CH", + "CI", + "CK", + "CL", + "CM", + "CN", + "CO", + "CR", + "CU", + "CV", + "CW", + "CX", + "CY", + "CZ", + "DE", + "DJ", + "DK", + "DM", + "DO", + "DZ", + "EC", + "EE", + "EG", + "EH", + "ER", + "ES", + "ET", + "FI", + "FJ", + "FK", + "FO", + "FR", + "GA", + "GB", + "GD", + "GE", + "GF", + "GG", + "GH", + "GI", + "GL", + "GM", + "GN", + "GP", + "GQ", + "GR", + "GS", + "GT", + "GW", + "GY", + "HK", + "HM", + "HN", + "HR", + "HT", + "HU", + "ID", + "IE", + "IL", + "IM", + "IN", + "IO", + "IQ", + "IR", + "IS", + "IT", + "JE", + "JM", + "JO", + "JP", + "KE", + "KG", + "KH", + "KI", + "KM", + "KN", + "KP", + "KR", + "KW", + "KY", + "KZ", + "LA", + "LB", + "LC", + "LI", + "LK", + "LR", + "LS", + "LT", + "LU", + "LV", + "LY", + "MA", + "MC", + "MD", + "ME", + "MF", + "MG", + "MK", + "ML", + "MM", + "MN", + "MO", + "MQ", + "MR", + "MS", + "MT", + "MU", + "MV", + "MW", + "MX", + "MY", + "MZ", + "NA", + "NC", + "NE", + "NF", + "NG", + "NI", + "NL", + "NO", + "NP", + "NR", + "NU", + "NZ", + "OM", + "PA", + "PE", + "PF", + "PG", + "PH", + "PK", + "PL", + "PM", + "PN", + "PS", + "PT", + "PY", + "QA", + "RE", + "RO", + "RS", + "RU", + "RW", + "SA", + "SB", + "SC", + "SD", + "SE", + "SG", + "SH", + "SI", + "SJ", + "SK", + "SL", + "SM", + "SN", + "SO", + "SR", + "SS", + "ST", + "SV", + "SX", + "SY", + "SZ", + "TA", + "TC", + "TD", + "TF", + "TG", + "TH", + "TJ", + "TK", + "TL", + "TM", + "TN", + "TO", + "TR", + "TT", + "TV", + "TW", + "TZ", + "UA", + "UG", + "UM", + "US", + "UY", + "UZ", + "VA", + "VC", + "VE", + "VG", + "VN", + "VU", + "WF", + "WS", + "XK", + "YE", + "YT", + "ZA", + "ZM", + "ZW", + "ZZ", + ) + + +class CropRegion(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("BOTTOM", "CENTER", "LEFT", "RIGHT", "TOP") + + +class CurrencyCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "AED", + "AFN", + "ALL", + "AMD", + "ANG", + "AOA", + "ARS", + "AUD", + "AWG", + "AZN", + "BAM", + "BBD", + "BDT", + "BGN", + "BHD", + "BIF", + "BMD", + "BND", + "BOB", + "BRL", + "BSD", + "BTN", + "BWP", + "BYN", + "BZD", + "CAD", + "CDF", + "CHF", + "CLP", + "CNY", + "COP", + "CRC", + "CVE", + "CZK", + "DJF", + "DKK", + "DOP", + "DZD", + "EGP", + "ERN", + "ETB", + "EUR", + "FJD", + "FKP", + "GBP", + "GEL", + "GHS", + "GIP", + "GMD", + "GNF", + "GTQ", + "GYD", + "HKD", + "HNL", + "HRK", + "HTG", + "HUF", + "IDR", + "ILS", + "INR", + "IQD", + "IRR", + "ISK", + "JEP", + "JMD", + "JOD", + "JPY", + "KES", + "KGS", + "KHR", + "KID", + "KMF", + "KRW", + "KWD", + "KYD", + "KZT", + "LAK", + "LBP", + "LKR", + "LRD", + "LSL", + "LTL", + "LVL", + "LYD", + "MAD", + "MDL", + "MGA", + "MKD", + "MMK", + "MNT", + "MOP", + "MRU", + "MUR", + "MVR", + "MWK", + "MXN", + "MYR", + "MZN", + "NAD", + "NGN", + "NIO", + "NOK", + "NPR", + "NZD", + "OMR", + "PAB", + "PEN", + "PGK", + "PHP", + "PKR", + "PLN", + "PYG", + "QAR", + "RON", + "RSD", + "RUB", + "RWF", + "SAR", + "SBD", + "SCR", + "SDG", + "SEK", + "SGD", + "SHP", + "SLL", + "SOS", + "SRD", + "SSP", + "STN", + "SYP", + "SZL", + "THB", + "TJS", + "TMT", + "TND", + "TOP", + "TRY", + "TTD", + "TWD", + "TZS", + "UAH", + "UGX", + "USD", + "UYU", + "UZS", + "VED", + "VES", + "VND", + "VUV", + "WST", + "XAF", + "XCD", + "XOF", + "XPF", + "XXX", + "YER", + "ZAR", + "ZMW", + ) + + +class CustomerConsentCollectedFrom(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("OTHER", "SHOPIFY") + + +class CustomerEmailAddressMarketingState(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("INVALID", "NOT_SUBSCRIBED", "PENDING", "SUBSCRIBED", "UNSUBSCRIBED") + + +class CustomerEmailAddressOpenTrackingLevel(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("OPTED_IN", "OPTED_OUT", "UNKNOWN") + + +class CustomerEmailMarketingConsentUpdateUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("INCLUSION", "INTERNAL_ERROR", "INVALID", "MISSING_ARGUMENT") + + +class CustomerEmailMarketingState(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("INVALID", "NOT_SUBSCRIBED", "PENDING", "REDACTED", "SUBSCRIBED", "UNSUBSCRIBED") + + +class CustomerMarketingOptInLevel(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CONFIRMED_OPT_IN", "SINGLE_OPT_IN", "UNKNOWN") + + +class CustomerMergeErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "CUSTOMER_HAS_GIFT_CARDS", + "INTERNAL_ERROR", + "INVALID_CUSTOMER", + "INVALID_CUSTOMER_ID", + "MISSING_OVERRIDE_ATTRIBUTE", + "OVERRIDE_ATTRIBUTE_INVALID", + ) + + +class CustomerMergeErrorFieldType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "COMPANY_CONTACT", + "CUSTOMER_PAYMENT_METHODS", + "DELETED_AT", + "GIFT_CARDS", + "MERGE_IN_PROGRESS", + "MULTIPASS_IDENTIFIER", + "PENDING_DATA_REQUEST", + "REDACTED_AT", + "SUBSCRIPTIONS", + ) + + +class CustomerMergeRequestStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("COMPLETED", "FAILED", "IN_PROGRESS", "REQUESTED") + + +class CustomerPaymentMethodCreateFromDuplicationDataUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CUSTOMER_DOES_NOT_EXIST", "INVALID_ENCRYPTED_DUPLICATION_DATA", "TOO_MANY_REQUESTS") + + +class CustomerPaymentMethodGetDuplicationDataUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "CUSTOMER_DOES_NOT_EXIST", + "INVALID_INSTRUMENT", + "INVALID_ORGANIZATION_SHOP", + "PAYMENT_METHOD_DOES_NOT_EXIST", + "SAME_SHOP", + "TOO_MANY_REQUESTS", + ) + + +class CustomerPaymentMethodGetUpdateUrlUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CUSTOMER_DOES_NOT_EXIST", "INVALID_INSTRUMENT", "PAYMENT_METHOD_DOES_NOT_EXIST", "TOO_MANY_REQUESTS") + + +class CustomerPaymentMethodRemoteUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "AUTHORIZE_NET_NOT_ENABLED_FOR_SUBSCRIPTIONS", + "BRAINTREE_NOT_ENABLED_FOR_SUBSCRIPTIONS", + "EXACTLY_ONE_REMOTE_REFERENCE_REQUIRED", + "INVALID", + "PRESENT", + "TAKEN", + ) + + +class CustomerPaymentMethodRevocationReason(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "AUTHORIZE_NET_GATEWAY_NOT_ENABLED", + "AUTHORIZE_NET_RETURNED_NO_PAYMENT_METHOD", + "BRAINTREE_API_AUTHENTICATION_ERROR", + "BRAINTREE_GATEWAY_NOT_ENABLED", + "BRAINTREE_PAYMENT_METHOD_NOT_CARD", + "BRAINTREE_RETURNED_NO_PAYMENT_METHOD", + "FAILED_TO_UPDATE_CREDIT_CARD", + "MANUALLY_REVOKED", + "MERGED", + "STRIPE_API_AUTHENTICATION_ERROR", + "STRIPE_API_INVALID_REQUEST_ERROR", + "STRIPE_GATEWAY_NOT_ENABLED", + "STRIPE_PAYMENT_METHOD_NOT_CARD", + "STRIPE_RETURNED_NO_PAYMENT_METHOD", + ) + + +class CustomerPaymentMethodUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("INVALID", "PRESENT", "TAKEN") + + +class CustomerPredictedSpendTier(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("HIGH", "LOW", "MEDIUM") + + +class CustomerProductSubscriberStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ACTIVE", "CANCELLED", "EXPIRED", "FAILED", "NEVER_SUBSCRIBED", "PAUSED") + + +class CustomerSavedSearchSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ID", "NAME", "RELEVANCE") + + +class CustomerSegmentMembersQueryUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("INVALID",) + + +class CustomerSmsMarketingConsentErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("INCLUSION", "INTERNAL_ERROR", "INVALID", "MISSING_ARGUMENT") + + +class CustomerSmsMarketingState(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("NOT_SUBSCRIBED", "PENDING", "REDACTED", "SUBSCRIBED", "UNSUBSCRIBED") + + +class CustomerSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATED_AT", "ID", "LAST_ORDER_DATE", "LOCATION", "NAME", "ORDERS_COUNT", "RELEVANCE", "TOTAL_SPENT", "UPDATED_AT") + + +class CustomerState(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("DECLINED", "DISABLED", "ENABLED", "INVITED") + + +Date = sgqlc.types.datetime.Date + +DateTime = sgqlc.types.datetime.DateTime + + +class DayOfTheWeek(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FRIDAY", "MONDAY", "SATURDAY", "SUNDAY", "THURSDAY", "TUESDAY", "WEDNESDAY") + + +class Decimal(sgqlc.types.Scalar): + __schema__ = shopify_schema + + +class DelegateAccessTokenCreateUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "DELEGATE_ACCESS_TOKEN", + "EMPTY_ACCESS_SCOPE", + "EXPIRES_AFTER_PARENT", + "NEGATIVE_EXPIRES_IN", + "PERSISTENCE_FAILED", + "REFRESH_TOKEN", + "UNKNOWN_SCOPES", + ) + + +class DelegateAccessTokenDestroyUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ACCESS_DENIED", "ACCESS_TOKEN_NOT_FOUND", "CAN_ONLY_DELETE_DELEGATE_TOKENS", "PERSISTENCE_FAILED") + + +class DeletionEventSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATED_AT", "ID", "RELEVANCE") + + +class DeletionEventSubjectType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("COLLECTION", "PRODUCT") + + +class DeliveryConditionField(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("TOTAL_PRICE", "TOTAL_WEIGHT") + + +class DeliveryConditionOperator(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("GREATER_THAN_OR_EQUAL_TO", "LESS_THAN_OR_EQUAL_TO") + + +class DeliveryCustomizationErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "CUSTOM_APP_FUNCTION_NOT_ELIGIBLE", + "DELIVERY_CUSTOMIZATION_FUNCTION_NOT_ELIGIBLE", + "DELIVERY_CUSTOMIZATION_NOT_FOUND", + "FUNCTION_DOES_NOT_IMPLEMENT", + "FUNCTION_ID_CANNOT_BE_CHANGED", + "FUNCTION_NOT_FOUND", + "FUNCTION_PENDING_DELETION", + "INVALID", + "INVALID_METAFIELDS", + "MAXIMUM_ACTIVE_DELIVERY_CUSTOMIZATIONS", + "REQUIRED_INPUT_FIELD", + "UNAUTHORIZED_APP_SCOPE", + ) + + +class DeliveryLegacyModeBlockedReason(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("NO_LOCATIONS_FULFILLING_ONLINE_ORDERS",) + + +class DeliveryLocalPickupTime(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FIVE_OR_MORE_DAYS", "FOUR_HOURS", "ONE_HOUR", "TWENTY_FOUR_HOURS", "TWO_HOURS", "TWO_TO_FOUR_DAYS") + + +class DeliveryLocationLocalPickupSettingsErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ACTIVE_LOCATION_NOT_FOUND", "GENERIC_ERROR") + + +class DeliveryMethodDefinitionType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("MERCHANT", "PARTICIPANT") + + +class DeliveryMethodType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("LOCAL", "NONE", "PICK_UP", "RETAIL", "SHIPPING") + + +class DigitalWallet(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ANDROID_PAY", "APPLE_PAY", "GOOGLE_PAY", "SHOPIFY_PAY") + + +class DiscountApplicationAllocationMethod(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ACROSS", "EACH") + + +class DiscountApplicationLevel(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("LINE", "ORDER") + + +class DiscountApplicationTargetSelection(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ALL", "ENTITLED", "EXPLICIT") + + +class DiscountApplicationTargetType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("LINE_ITEM", "SHIPPING_LINE") + + +class DiscountClass(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ORDER", "PRODUCT", "SHIPPING") + + +class DiscountCodeSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CODE", "CREATED_AT", "ID", "RELEVANCE") + + +class DiscountErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "ACTIVE_PERIOD_OVERLAP", + "BLANK", + "CONFLICT", + "DUPLICATE", + "EQUAL_TO", + "EXCEEDED_MAX", + "GREATER_THAN", + "GREATER_THAN_OR_EQUAL_TO", + "IMPLICIT_DUPLICATE", + "INCLUSION", + "INTERNAL_ERROR", + "INVALID", + "INVALID_COMBINES_WITH_FOR_DISCOUNT_CLASS", + "INVALID_DISCOUNT_CLASS_FOR_PRICE_RULE", + "LESS_THAN", + "LESS_THAN_OR_EQUAL_TO", + "MAX_APP_DISCOUNTS", + "MINIMUM_SUBTOTAL_AND_QUANTITY_RANGE_BOTH_PRESENT", + "MISSING_ARGUMENT", + "PRESENT", + "TAKEN", + "TOO_LONG", + "TOO_MANY_ARGUMENTS", + "TOO_SHORT", + "VALUE_OUTSIDE_RANGE", + ) + + +class DiscountShareableUrlTargetType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("COLLECTION", "HOME", "PRODUCT") + + +class DiscountSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATED_AT", "ENDS_AT", "ID", "RELEVANCE", "STARTS_AT", "TITLE", "UPDATED_AT") + + +class DiscountStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ACTIVE", "EXPIRED", "SCHEDULED") + + +class DiscountTargetType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("LINE_ITEM", "SHIPPING_LINE") + + +class DiscountType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CODE_DISCOUNT", "MANUAL") + + +class DisputeEvidenceUpdateUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "DISPUTE_EVIDENCE_NOT_FOUND", + "EVIDENCE_ALREADY_ACCEPTED", + "EVIDENCE_PAST_DUE_DATE", + "FILES_SIZE_EXCEEDED_LIMIT", + "INVALID", + "TOO_LARGE", + ) + + +class DisputeStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ACCEPTED", "CHARGE_REFUNDED", "LOST", "NEEDS_RESPONSE", "UNDER_REVIEW", "WON") + + +class DisputeType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CHARGEBACK", "INQUIRY") + + +class DraftOrderAppliedDiscountType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FIXED_AMOUNT", "PERCENTAGE") + + +class DraftOrderSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CUSTOMER_NAME", "ID", "NUMBER", "RELEVANCE", "STATUS", "TOTAL_PRICE", "UPDATED_AT") + + +class DraftOrderStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("COMPLETED", "INVOICE_SENT", "OPEN") + + +class ErrorsServerPixelUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ALREADY_EXISTS", "NEEDS_CONFIGURATION_TO_CONNECT", "NOT_FOUND", "PUB_SUB_ERROR") + + +class ErrorsWebPixelUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("BLANK", "INVALID_SETTINGS", "NOT_FOUND", "TAKEN", "UNABLE_TO_DELETE") + + +class EventSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATED_AT", "ID", "RELEVANCE") + + +class FileContentType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FILE", "IMAGE", "VIDEO") + + +class FileCreateInputDuplicateResolutionMode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("APPEND_UUID", "RAISE_ERROR", "REPLACE") + + +class FileErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "DUPLICATE_FILENAME_ERROR", + "EXTERNAL_VIDEO_EMBED_DISABLED", + "EXTERNAL_VIDEO_EMBED_NOT_FOUND_OR_TRANSCODING", + "EXTERNAL_VIDEO_INVALID_ASPECT_RATIO", + "EXTERNAL_VIDEO_NOT_FOUND", + "EXTERNAL_VIDEO_UNLISTED", + "FILE_STORAGE_LIMIT_EXCEEDED", + "GENERIC_FILE_DOWNLOAD_FAILURE", + "GENERIC_FILE_INVALID_SIZE", + "IMAGE_DOWNLOAD_FAILURE", + "IMAGE_PROCESSING_FAILURE", + "INVALID_IMAGE_ASPECT_RATIO", + "INVALID_IMAGE_FILE_SIZE", + "INVALID_IMAGE_RESOLUTION", + "INVALID_SIGNED_URL", + "MEDIA_TIMEOUT_ERROR", + "MODEL3D_GLB_OUTPUT_CREATION_ERROR", + "MODEL3D_GLB_TO_USDZ_CONVERSION_ERROR", + "MODEL3D_PROCESSING_FAILURE", + "MODEL3D_THUMBNAIL_GENERATION_ERROR", + "MODEL3D_THUMBNAIL_REGENERATION_ERROR", + "MODEL3D_VALIDATION_ERROR", + "UNKNOWN", + "UNSUPPORTED_IMAGE_FILE_TYPE", + "VIDEO_INVALID_FILETYPE_ERROR", + "VIDEO_MAX_DURATION_ERROR", + "VIDEO_MAX_HEIGHT_ERROR", + "VIDEO_MAX_WIDTH_ERROR", + "VIDEO_METADATA_READ_ERROR", + "VIDEO_MIN_DURATION_ERROR", + "VIDEO_MIN_HEIGHT_ERROR", + "VIDEO_MIN_WIDTH_ERROR", + "VIDEO_VALIDATION_ERROR", + ) + + +class FileSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATED_AT", "FILENAME", "ID", "ORIGINAL_UPLOAD_SIZE", "RELEVANCE", "UPDATED_AT") + + +class FileStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FAILED", "PROCESSING", "READY", "UPLOADED") + + +class FilesErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "ALT_VALUE_LIMIT_EXCEEDED", + "BLANK_SEARCH", + "FILENAME_ALREADY_EXISTS", + "FILE_DOES_NOT_EXIST", + "FILE_LOCKED", + "INVALID", + "INVALID_DUPLICATE_MODE_FOR_TYPE", + "INVALID_FILENAME", + "INVALID_FILENAME_EXTENSION", + "INVALID_IMAGE_SOURCE_URL", + "INVALID_QUERY", + "MISMATCHED_FILENAME_AND_ORIGINAL_SOURCE", + "MISSING_ARGUMENTS", + "MISSING_FILENAME_FOR_DUPLICATE_MODE_REPLACE", + "NON_IMAGE_MEDIA_PER_SHOP_LIMIT_EXCEEDED", + "NON_READY_STATE", + "TOO_MANY_ARGUMENTS", + "UNACCEPTABLE_ASSET", + "UNACCEPTABLE_TRIAL_ASSET", + "UNACCEPTABLE_UNVERIFIED_TRIAL_ASSET", + "UNSUPPORTED_MEDIA_TYPE_FOR_FILENAME_UPDATE", + ) + + +Float = sgqlc.types.Float + + +class FormattedString(sgqlc.types.Scalar): + __schema__ = shopify_schema + + +class FulfillmentDisplayStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "ATTEMPTED_DELIVERY", + "CANCELED", + "CONFIRMED", + "DELIVERED", + "FAILURE", + "FULFILLED", + "IN_TRANSIT", + "LABEL_PRINTED", + "LABEL_PURCHASED", + "LABEL_VOIDED", + "MARKED_AS_FULFILLED", + "NOT_DELIVERED", + "OUT_FOR_DELIVERY", + "PICKED_UP", + "READY_FOR_PICKUP", + "SUBMITTED", + ) + + +class FulfillmentEventSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("HAPPENED_AT", "ID", "RELEVANCE") + + +class FulfillmentEventStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "ATTEMPTED_DELIVERY", + "CONFIRMED", + "DELIVERED", + "FAILURE", + "IN_TRANSIT", + "LABEL_PRINTED", + "LABEL_PURCHASED", + "OUT_FOR_DELIVERY", + "READY_FOR_PICKUP", + ) + + +class FulfillmentHoldReason(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "AWAITING_PAYMENT", + "HIGH_RISK_OF_FRAUD", + "INCORRECT_ADDRESS", + "INVENTORY_OUT_OF_STOCK", + "ONLINE_STORE_POST_PURCHASE_CROSS_SELL", + "OTHER", + "UNKNOWN_DELIVERY_DATE", + ) + + +class FulfillmentOrderAction(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "CANCEL_FULFILLMENT_ORDER", + "CREATE_FULFILLMENT", + "EXTERNAL", + "HOLD", + "MARK_AS_OPEN", + "MERGE", + "MOVE", + "RELEASE_HOLD", + "REQUEST_CANCELLATION", + "REQUEST_FULFILLMENT", + "SPLIT", + ) + + +class FulfillmentOrderAssignmentStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CANCELLATION_REQUESTED", "FULFILLMENT_ACCEPTED", "FULFILLMENT_REQUESTED") + + +class FulfillmentOrderHoldUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FULFILLMENT_ORDER_NOT_FOUND", "GREATER_THAN_ZERO", "INVALID_LINE_ITEM_QUANTITY", "TAKEN") + + +class FulfillmentOrderLineItemsPreparedForPickupUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FULFILLMENT_ORDER_INVALID", "NO_LINE_ITEMS_TO_PREPARE_FOR_FULFILLMENT_ORDER", "UNABLE_TO_PREPARE_QUANTITY") + + +class FulfillmentOrderMerchantRequestKind(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CANCELLATION_REQUEST", "FULFILLMENT_REQUEST") + + +class FulfillmentOrderMergeUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FULFILLMENT_ORDER_NOT_FOUND", "GREATER_THAN", "INVALID_LINE_ITEM_QUANTITY") + + +class FulfillmentOrderRejectionReason(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("INCORRECT_ADDRESS", "INELIGIBLE_PRODUCT", "INVENTORY_OUT_OF_STOCK", "OTHER", "UNDELIVERABLE_DESTINATION") + + +class FulfillmentOrderReleaseHoldUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FULFILLMENT_ORDER_NOT_FOUND", "GREATER_THAN_ZERO", "INVALID_LINE_ITEM_QUANTITY") + + +class FulfillmentOrderRequestStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "ACCEPTED", + "CANCELLATION_ACCEPTED", + "CANCELLATION_REJECTED", + "CANCELLATION_REQUESTED", + "CLOSED", + "REJECTED", + "SUBMITTED", + "UNSUBMITTED", + ) + + +class FulfillmentOrderRescheduleUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FULFILLMENT_ORDER_NOT_FOUND",) + + +class FulfillmentOrderSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ID", "RELEVANCE") + + +class FulfillmentOrderSplitUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FULFILLMENT_ORDER_NOT_FOUND", "GREATER_THAN", "INVALID_LINE_ITEM_QUANTITY") + + +class FulfillmentOrderStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CANCELLED", "CLOSED", "INCOMPLETE", "IN_PROGRESS", "ON_HOLD", "OPEN", "SCHEDULED") + + +class FulfillmentOrdersReleaseHoldsUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FAILED_TO_CREATE_JOB",) + + +class FulfillmentOrdersSetFulfillmentDeadlineUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FULFILLMENT_ORDERS_NOT_FOUND",) + + +class FulfillmentServiceType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("GIFT_CARD", "MANUAL", "THIRD_PARTY") + + +class FulfillmentStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CANCELLED", "ERROR", "FAILURE", "SUCCESS") + + +class GiftCardErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("GREATER_THAN", "INTERNAL_ERROR", "INVALID", "MISSING_ARGUMENT", "TAKEN", "TOO_LONG", "TOO_SHORT") + + +class GiftCardSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "AMOUNT_SPENT", + "BALANCE", + "CODE", + "CREATED_AT", + "CUSTOMER_NAME", + "DISABLED_AT", + "EXPIRES_ON", + "ID", + "INITIAL_VALUE", + "RELEVANCE", + "UPDATED_AT", + ) + + +class HTML(sgqlc.types.Scalar): + __schema__ = shopify_schema + + +ID = sgqlc.types.ID + + +class ImageContentType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("JPG", "PNG", "WEBP") + + +Int = sgqlc.types.Int + + +class InventoryAdjustQuantitiesUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "ADJUST_QUANTITIES_FAILED", + "INTERNAL_LEDGER_DOCUMENT", + "INVALID_AVAILABLE_DOCUMENT", + "INVALID_INVENTORY_ITEM", + "INVALID_LEDGER_DOCUMENT", + "INVALID_LOCATION", + "INVALID_QUANTITY_DOCUMENT", + "INVALID_QUANTITY_NAME", + "INVALID_QUANTITY_TOO_HIGH", + "INVALID_QUANTITY_TOO_LOW", + "INVALID_REASON", + "INVALID_REFERENCE_DOCUMENT", + "ITEM_NOT_STOCKED_AT_LOCATION", + "MAX_ONE_LEDGER_DOCUMENT", + "NON_MUTABLE_INVENTORY_ITEM", + ) + + +class InventoryBulkToggleActivationUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "CANNOT_DEACTIVATE_FROM_ONLY_LOCATION", + "COMMITTED_INVENTORY_AT_LOCATION", + "FAILED_TO_STOCK_AT_LOCATION", + "FAILED_TO_UNSTOCK_FROM_LOCATION", + "GENERIC_ERROR", + "INCOMING_INVENTORY_AT_LOCATION", + "INVENTORY_ITEM_NOT_FOUND", + "INVENTORY_MANAGED_BY_3RD_PARTY", + "INVENTORY_MANAGED_BY_SHOPIFY", + "LOCATION_NOT_FOUND", + "MISSING_SKU", + "RESERVED_INVENTORY_AT_LOCATION", + ) + + +class InventoryMoveQuantitiesUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "DIFFERENT_LOCATIONS", + "INTERNAL_LEDGER_DOCUMENT", + "INVALID_AVAILABLE_DOCUMENT", + "INVALID_INVENTORY_ITEM", + "INVALID_LEDGER_DOCUMENT", + "INVALID_LOCATION", + "INVALID_QUANTITY_DOCUMENT", + "INVALID_QUANTITY_NAME", + "INVALID_QUANTITY_NEGATIVE", + "INVALID_QUANTITY_TOO_HIGH", + "INVALID_REASON", + "INVALID_REFERENCE_DOCUMENT", + "ITEM_NOT_STOCKED_AT_LOCATION", + "MAXIMUM_LEDGER_DOCUMENT_URIS", + "MOVE_QUANTITIES_FAILED", + "NON_MUTABLE_INVENTORY_ITEM", + "SAME_QUANTITY_NAME", + ) + + +class InventorySetOnHandQuantitiesUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "INVALID_INVENTORY_ITEM", + "INVALID_LOCATION", + "INVALID_QUANTITY_NEGATIVE", + "INVALID_QUANTITY_TOO_HIGH", + "INVALID_REASON", + "INVALID_REFERENCE_DOCUMENT", + "ITEM_NOT_STOCKED_AT_LOCATION", + "NON_MUTABLE_INVENTORY_ITEM", + "SET_ON_HAND_QUANTITIES_FAILED", + ) + + +class JSON(sgqlc.types.Scalar): + __schema__ = shopify_schema + + +class LanguageCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "AF", + "AK", + "AM", + "AR", + "AS", + "AZ", + "BE", + "BG", + "BM", + "BN", + "BO", + "BR", + "BS", + "CA", + "CE", + "CS", + "CU", + "CY", + "DA", + "DE", + "DZ", + "EE", + "EL", + "EN", + "EO", + "ES", + "ET", + "EU", + "FA", + "FF", + "FI", + "FO", + "FR", + "FY", + "GA", + "GD", + "GL", + "GU", + "GV", + "HA", + "HE", + "HI", + "HR", + "HU", + "HY", + "IA", + "ID", + "IG", + "II", + "IS", + "IT", + "JA", + "JV", + "KA", + "KI", + "KK", + "KL", + "KM", + "KN", + "KO", + "KS", + "KU", + "KW", + "KY", + "LB", + "LG", + "LN", + "LO", + "LT", + "LU", + "LV", + "MG", + "MI", + "MK", + "ML", + "MN", + "MR", + "MS", + "MT", + "MY", + "NB", + "ND", + "NE", + "NL", + "NN", + "NO", + "OM", + "OR", + "OS", + "PA", + "PL", + "PS", + "PT", + "PT_BR", + "PT_PT", + "QU", + "RM", + "RN", + "RO", + "RU", + "RW", + "SD", + "SE", + "SG", + "SI", + "SK", + "SL", + "SN", + "SO", + "SQ", + "SR", + "SU", + "SV", + "SW", + "TA", + "TE", + "TG", + "TH", + "TI", + "TK", + "TO", + "TR", + "TT", + "UG", + "UK", + "UR", + "UZ", + "VI", + "VO", + "WO", + "XH", + "YI", + "YO", + "ZH", + "ZH_CN", + "ZH_TW", + "ZU", + ) + + +class LengthUnit(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CENTIMETERS", "FEET", "INCHES", "METERS", "MILLIMETERS", "YARDS") + + +class LocalizableContentType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "FILE_REFERENCE", + "HTML", + "INLINE_RICH_TEXT", + "JSON", + "JSON_STRING", + "LIST_FILE_REFERENCE", + "LIST_MULTI_LINE_TEXT_FIELD", + "LIST_SINGLE_LINE_TEXT_FIELD", + "LIST_URL", + "MULTI_LINE_TEXT_FIELD", + "RICH_TEXT_FIELD", + "SINGLE_LINE_TEXT_FIELD", + "STRING", + "URI", + "URL", + ) + + +class LocalizationExtensionKey(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "SHIPPING_CREDENTIAL_BR", + "SHIPPING_CREDENTIAL_CN", + "SHIPPING_CREDENTIAL_KR", + "TAX_CREDENTIAL_BR", + "TAX_CREDENTIAL_IT", + "TAX_EMAIL_IT", + ) + + +class LocalizationExtensionPurpose(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("SHIPPING", "TAX") + + +class LocationActivateUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("GENERIC_ERROR", "HAS_NON_UNIQUE_NAME", "HAS_ONGOING_RELOCATION", "LOCATION_LIMIT", "LOCATION_NOT_FOUND") + + +class LocationAddUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "APP_NOT_AUTHORIZED", + "BLANK", + "DISALLOWED_OWNER_TYPE", + "GENERIC_ERROR", + "INCLUSION", + "INVALID", + "INVALID_TYPE", + "INVALID_US_ZIPCODE", + "INVALID_VALUE", + "PRESENT", + "TAKEN", + "TOO_LONG", + "TOO_SHORT", + "UNSTRUCTURED_RESERVED_NAMESPACE", + ) + + +class LocationDeactivateUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "CANNOT_DISABLE_ONLINE_ORDER_FULFILLMENT", + "DESTINATION_LOCATION_IS_THE_SAME_LOCATION", + "DESTINATION_LOCATION_NOT_FOUND_OR_INACTIVE", + "FAILED_TO_RELOCATE_ACTIVE_INVENTORIES", + "FAILED_TO_RELOCATE_INCOMING_MOVEMENTS", + "FAILED_TO_RELOCATE_OPEN_PURCHASE_ORDERS", + "FAILED_TO_RELOCATE_OPEN_TRANSFERS", + "HAS_ACTIVE_INVENTORY_ERROR", + "HAS_ACTIVE_RETAIL_SUBSCRIPTIONS", + "HAS_FULFILLMENT_ORDERS_ERROR", + "HAS_INCOMING_MOVEMENTS_ERROR", + "HAS_OPEN_PURCHASE_ORDERS_ERROR", + "HAS_OPEN_TRANSFERS_ERROR", + "LOCATION_NOT_FOUND", + "PERMANENTLY_BLOCKED_FROM_DEACTIVATION_ERROR", + "TEMPORARILY_BLOCKED_FROM_DEACTIVATION_ERROR", + ) + + +class LocationDeleteUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "GENERIC_ERROR", + "LOCATION_HAS_ACTIVE_RETAIL_SUBSCRIPTION", + "LOCATION_HAS_INVENTORY", + "LOCATION_HAS_PENDING_ORDERS", + "LOCATION_IS_ACTIVE", + "LOCATION_NOT_FOUND", + ) + + +class LocationEditUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "APP_NOT_AUTHORIZED", + "BLANK", + "CANNOT_DISABLE_ONLINE_ORDER_FULFILLMENT", + "DISALLOWED_OWNER_TYPE", + "GENERIC_ERROR", + "INCLUSION", + "INVALID", + "INVALID_TYPE", + "INVALID_US_ZIPCODE", + "INVALID_VALUE", + "NOT_FOUND", + "PRESENT", + "TAKEN", + "TOO_LONG", + "TOO_SHORT", + "UNSTRUCTURED_RESERVED_NAMESPACE", + ) + + +class LocationSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ID", "NAME", "RELEVANCE") + + +class MarketCurrencySettingsUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "MANAGED_MARKET", + "MARKET_NOT_FOUND", + "MULTIPLE_CURRENCIES_NOT_SUPPORTED", + "NO_LOCAL_CURRENCIES_ON_SINGLE_COUNTRY_MARKET", + "PRIMARY_MARKET_USES_SHOP_CURRENCY", + "UNSUPPORTED_CURRENCY", + ) + + +class MarketLocalizableResourceType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("METAFIELD",) + + +class MarketUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "BLANK", + "CANNOT_ADD_CUSTOMER_DOMAIN", + "CANNOT_ADD_REGIONS_TO_PRIMARY_MARKET", + "CANNOT_ADD_WEB_PRESENCE_TO_PRIMARY_MARKET", + "CANNOT_DELETE_ONLY_REGION", + "CANNOT_DELETE_PRIMARY_MARKET", + "CANNOT_DELETE_PRIMARY_MARKET_WEB_PRESENCE", + "CANNOT_DISABLE_PRIMARY_MARKET", + "CANNOT_HAVE_SUBFOLDER_AND_DOMAIN", + "CANNOT_SET_DEFAULT_LOCALE_TO_NULL", + "DISABLED_LANGUAGE", + "DOMAIN_NOT_FOUND", + "DUPLICATE_LANGUAGES", + "INVALID", + "MARKET_NOT_FOUND", + "NO_LANGUAGES", + "PRIMARY_MARKET_MUST_USE_PRIMARY_DOMAIN", + "REGION_NOT_FOUND", + "REGION_SPECIFIC_LANGUAGE", + "REQUIRES_DOMAIN_OR_SUBFOLDER", + "REQUIRES_EXACTLY_ONE_OPTION", + "SHOP_REACHED_MARKETS_LIMIT", + "SUBFOLDER_SUFFIX_CANNOT_BE_SCRIPT_CODE", + "SUBFOLDER_SUFFIX_MUST_CONTAIN_ONLY_LETTERS", + "TAKEN", + "TOO_LONG", + "TOO_SHORT", + "UNPUBLISHED_LANGUAGE", + "UNSUPPORTED_COUNTRY_REGION", + "WEB_PRESENCE_NOT_FOUND", + ) + + +class MarketingActivityExtensionAppErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("API_ERROR", "INSTALL_REQUIRED_ERROR", "NOT_ONBOARDED_ERROR", "PLATFORM_ERROR", "VALIDATION_ERROR") + + +class MarketingActivitySortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATED_AT", "ID", "RELEVANCE", "TITLE") + + +class MarketingActivityStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "ACTIVE", + "DELETED", + "DELETED_EXTERNALLY", + "DISCONNECTED", + "DRAFT", + "FAILED", + "INACTIVE", + "PAUSED", + "PENDING", + "SCHEDULED", + "UNDEFINED", + ) + + +class MarketingActivityStatusBadgeType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ATTENTION", "DEFAULT", "INFO", "SUCCESS", "WARNING") + + +class MarketingActivityUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("INVALID", "TAKEN") + + +class MarketingBudgetBudgetType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("DAILY", "LIFETIME") + + +class MarketingChannel(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("DISPLAY", "EMAIL", "REFERRAL", "SEARCH", "SOCIAL") + + +class MarketingEventSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ID", "RELEVANCE", "STARTED_AT") + + +class MarketingTactic(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "ABANDONED_CART", + "AD", + "AFFILIATE", + "DIRECT", + "LINK", + "LOYALTY", + "MESSAGE", + "NEWSLETTER", + "NOTIFICATION", + "POST", + "RETARGETING", + "SEO", + "STOREFRONT_APP", + "TRANSACTIONAL", + ) + + +class MediaContentType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("EXTERNAL_VIDEO", "IMAGE", "MODEL_3D", "VIDEO") + + +class MediaErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "DUPLICATE_FILENAME_ERROR", + "EXTERNAL_VIDEO_EMBED_DISABLED", + "EXTERNAL_VIDEO_EMBED_NOT_FOUND_OR_TRANSCODING", + "EXTERNAL_VIDEO_INVALID_ASPECT_RATIO", + "EXTERNAL_VIDEO_NOT_FOUND", + "EXTERNAL_VIDEO_UNLISTED", + "FILE_STORAGE_LIMIT_EXCEEDED", + "GENERIC_FILE_DOWNLOAD_FAILURE", + "GENERIC_FILE_INVALID_SIZE", + "IMAGE_DOWNLOAD_FAILURE", + "IMAGE_PROCESSING_FAILURE", + "INVALID_IMAGE_ASPECT_RATIO", + "INVALID_IMAGE_FILE_SIZE", + "INVALID_IMAGE_RESOLUTION", + "INVALID_SIGNED_URL", + "MEDIA_TIMEOUT_ERROR", + "MODEL3D_GLB_OUTPUT_CREATION_ERROR", + "MODEL3D_GLB_TO_USDZ_CONVERSION_ERROR", + "MODEL3D_PROCESSING_FAILURE", + "MODEL3D_THUMBNAIL_GENERATION_ERROR", + "MODEL3D_THUMBNAIL_REGENERATION_ERROR", + "MODEL3D_VALIDATION_ERROR", + "UNKNOWN", + "UNSUPPORTED_IMAGE_FILE_TYPE", + "VIDEO_INVALID_FILETYPE_ERROR", + "VIDEO_MAX_DURATION_ERROR", + "VIDEO_MAX_HEIGHT_ERROR", + "VIDEO_MAX_WIDTH_ERROR", + "VIDEO_METADATA_READ_ERROR", + "VIDEO_MIN_DURATION_ERROR", + "VIDEO_MIN_HEIGHT_ERROR", + "VIDEO_MIN_WIDTH_ERROR", + "VIDEO_VALIDATION_ERROR", + ) + + +class MediaHost(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("VIMEO", "YOUTUBE") + + +class MediaPreviewImageStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FAILED", "PROCESSING", "READY", "UPLOADED") + + +class MediaStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FAILED", "PROCESSING", "READY", "UPLOADED") + + +class MediaUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "BLANK", + "INVALID", + "INVALID_MEDIA_TYPE", + "MAXIMUM_VARIANT_MEDIA_PAIRS_EXCEEDED", + "MEDIA_CANNOT_BE_MODIFIED", + "MEDIA_DOES_NOT_EXIST", + "MEDIA_DOES_NOT_EXIST_ON_PRODUCT", + "MEDIA_IS_NOT_ATTACHED_TO_VARIANT", + "MODEL3D_THROTTLE_EXCEEDED", + "MODEL3D_VALIDATION_ERROR", + "NON_READY_MEDIA", + "PRODUCT_DOES_NOT_EXIST", + "PRODUCT_MEDIA_LIMIT_EXCEEDED", + "PRODUCT_VARIANT_ALREADY_HAS_MEDIA", + "PRODUCT_VARIANT_DOES_NOT_EXIST_ON_PRODUCT", + "PRODUCT_VARIANT_SPECIFIED_MULTIPLE_TIMES", + "SHOP_MEDIA_LIMIT_EXCEEDED", + "TOO_MANY_MEDIA_PER_INPUT_PAIR", + "VIDEO_THROTTLE_EXCEEDED", + "VIDEO_VALIDATION_ERROR", + ) + + +class MediaWarningCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("MODEL_LARGE_PHYSICAL_SIZE", "MODEL_SMALL_PHYSICAL_SIZE") + + +class MerchandiseDiscountClass(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ORDER", "PRODUCT") + + +class MetafieldAdminAccess(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("MERCHANT_READ", "MERCHANT_READ_WRITE", "PRIVATE", "PUBLIC_READ") + + +class MetafieldDefinitionCreateUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "DUPLICATE_OPTION", + "INCLUSION", + "INVALID", + "INVALID_CHARACTER", + "INVALID_OPTION", + "LIMIT_EXCEEDED", + "OWNER_TYPE_LIMIT_EXCEEDED_FOR_AUTOMATED_COLLECTIONS", + "PINNED_LIMIT_REACHED", + "PRESENT", + "RESERVED_NAMESPACE_KEY", + "RESOURCE_TYPE_LIMIT_EXCEEDED", + "TAKEN", + "TOO_LONG", + "TOO_SHORT", + "TYPE_NOT_ALLOWED_FOR_CONDITIONS", + "UNSTRUCTURED_ALREADY_EXISTS", + ) + + +class MetafieldDefinitionDeleteUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "DISALLOWED_OWNER_TYPE", + "INTERNAL_ERROR", + "METAFIELD_DEFINITION_IN_USE", + "NOT_FOUND", + "PRESENT", + "REFERENCE_TYPE_DELETION_ERROR", + ) + + +class MetafieldDefinitionPinUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ALREADY_PINNED", "DISALLOWED_OWNER_TYPE", "INTERNAL_ERROR", "NOT_FOUND", "PINNED_LIMIT_REACHED") + + +class MetafieldDefinitionPinnedStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ANY", "PINNED", "UNPINNED") + + +class MetafieldDefinitionSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ID", "NAME", "PINNED_POSITION", "RELEVANCE") + + +class MetafieldDefinitionUnpinUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("DISALLOWED_OWNER_TYPE", "INTERNAL_ERROR", "NOT_FOUND", "NOT_PINNED") + + +class MetafieldDefinitionUpdateUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "INTERNAL_ERROR", + "INVALID_INPUT", + "METAFIELD_DEFINITION_IN_USE", + "NOT_FOUND", + "OWNER_TYPE_LIMIT_EXCEEDED_FOR_AUTOMATED_COLLECTIONS", + "PINNED_LIMIT_REACHED", + "PRESENT", + "TOO_LONG", + "TYPE_NOT_ALLOWED_FOR_CONDITIONS", + ) + + +class MetafieldDefinitionValidationStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ALL_VALID", "IN_PROGRESS", "SOME_INVALID") + + +class MetafieldOwnerType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "API_PERMISSION", + "ARTICLE", + "BLOG", + "COLLECTION", + "COMPANY", + "COMPANY_LOCATION", + "CUSTOMER", + "DELIVERY_CUSTOMIZATION", + "DISCOUNT", + "DRAFTORDER", + "LOCATION", + "MARKET", + "MEDIA_IMAGE", + "ORDER", + "PAGE", + "PAYMENT_CUSTOMIZATION", + "PRODUCT", + "PRODUCTVARIANT", + "SHOP", + ) + + +class MetafieldValidationStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ANY", "INVALID", "VALID") + + +class MetafieldValueType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("BOOLEAN", "INTEGER", "JSON_STRING", "STRING") + + +class MetafieldsSetUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "APP_NOT_AUTHORIZED", + "BLANK", + "INCLUSION", + "INVALID_TYPE", + "INVALID_VALUE", + "LESS_THAN_OR_EQUAL_TO", + "PRESENT", + "TOO_LONG", + "TOO_SHORT", + ) + + +class MetaobjectAdminAccess(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("MERCHANT_READ", "MERCHANT_READ_WRITE", "PRIVATE", "PUBLIC_READ", "PUBLIC_READ_WRITE") + + +class MetaobjectStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ACTIVE", "DRAFT") + + +class MetaobjectStorefrontAccess(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("NONE", "PUBLIC_READ") + + +class MetaobjectUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "BLANK", + "CAPABILITY_NOT_ENABLED", + "DUPLICATE_FIELD_INPUT", + "IMMUTABLE", + "INCLUSION", + "INTERNAL_ERROR", + "INVALID", + "INVALID_OPTION", + "INVALID_TYPE", + "INVALID_VALUE", + "MAX_DEFINITIONS_EXCEEDED", + "MAX_OBJECTS_EXCEEDED", + "NOT_AUTHORIZED", + "OBJECT_FIELD_REQUIRED", + "OBJECT_FIELD_TAKEN", + "PRESENT", + "RECORD_NOT_FOUND", + "RESERVED_NAME", + "TAKEN", + "TOO_LONG", + "TOO_SHORT", + "UNDEFINED_OBJECT_FIELD", + "UNDEFINED_OBJECT_TYPE", + ) + + +class MethodDefinitionSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ID", "RATE_PROVIDER_TYPE", "RELEVANCE") + + +class Money(sgqlc.types.Scalar): + __schema__ = shopify_schema + + +class OrderActionType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ORDER", "ORDER_EDIT", "REFUND", "UNKNOWN") + + +class OrderCancelReason(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CUSTOMER", "DECLINED", "FRAUD", "INVENTORY", "OTHER") + + +class OrderCreateMandatePaymentUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ORDER_MANDATE_PAYMENT_ERROR_CODE",) + + +class OrderDisplayFinancialStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("AUTHORIZED", "EXPIRED", "PAID", "PARTIALLY_PAID", "PARTIALLY_REFUNDED", "PENDING", "REFUNDED", "VOIDED") + + +class OrderDisplayFulfillmentStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "FULFILLED", + "IN_PROGRESS", + "ON_HOLD", + "OPEN", + "PARTIALLY_FULFILLED", + "PENDING_FULFILLMENT", + "RESTOCKED", + "SCHEDULED", + "UNFULFILLED", + ) + + +class OrderInvoiceSendUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ORDER_INVOICE_SEND_UNSUCCESSFUL",) + + +class OrderPaymentStatusResult(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "AUTHORIZED", + "CAPTURED", + "ERROR", + "PROCESSING", + "PURCHASED", + "REDIRECT_REQUIRED", + "REFUNDED", + "RETRYABLE", + "SUCCESS", + "UNKNOWN", + "VOIDED", + ) + + +class OrderReturnStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("INSPECTION_COMPLETE", "IN_PROGRESS", "NO_RETURN", "RETURNED", "RETURN_FAILED", "RETURN_REQUESTED") + + +class OrderRiskLevel(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("HIGH", "LOW", "MEDIUM") + + +class OrderSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "CREATED_AT", + "CUSTOMER_NAME", + "DESTINATION", + "FINANCIAL_STATUS", + "FULFILLMENT_STATUS", + "ID", + "ORDER_NUMBER", + "PO_NUMBER", + "PROCESSED_AT", + "RELEVANCE", + "TOTAL_ITEMS_QUANTITY", + "TOTAL_PRICE", + "UPDATED_AT", + ) + + +class OrderTransactionErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "AMAZON_PAYMENTS_INVALID_PAYMENT_METHOD", + "AMAZON_PAYMENTS_MAX_AMOUNT_CHARGED", + "AMAZON_PAYMENTS_MAX_AMOUNT_REFUNDED", + "AMAZON_PAYMENTS_MAX_AUTHORIZATIONS_CAPTURED", + "AMAZON_PAYMENTS_MAX_REFUNDS_PROCESSED", + "AMAZON_PAYMENTS_ORDER_REFERENCE_CANCELED", + "AMAZON_PAYMENTS_STALE", + "CALL_ISSUER", + "CARD_DECLINED", + "CONFIG_ERROR", + "EXPIRED_CARD", + "GENERIC_ERROR", + "INCORRECT_ADDRESS", + "INCORRECT_CVC", + "INCORRECT_NUMBER", + "INCORRECT_PIN", + "INCORRECT_ZIP", + "INVALID_AMOUNT", + "INVALID_COUNTRY", + "INVALID_CVC", + "INVALID_EXPIRY_DATE", + "INVALID_NUMBER", + "PAYMENT_METHOD_UNAVAILABLE", + "PICK_UP_CARD", + "PROCESSING_ERROR", + "TEST_MODE_LIVE_CARD", + "UNSUPPORTED_FEATURE", + ) + + +class OrderTransactionKind(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("AUTHORIZATION", "CAPTURE", "CHANGE", "EMV_AUTHORIZATION", "REFUND", "SALE", "SUGGESTED_REFUND", "VOID") + + +class OrderTransactionStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("AWAITING_RESPONSE", "ERROR", "FAILURE", "PENDING", "SUCCESS", "UNKNOWN") + + +class ParseErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "BACKFILL_DATE_RANGE_NOT_FOUND", + "COLUMN_NOT_FOUND", + "COMPARE_TO_INCOMPATIBLE_PERIOD", + "COMPARE_TO_INVALID_PERIOD", + "COMPARE_TO_MISSING_PERIOD", + "COMPARISON_WITH_NON_AGGREGATE_FIELDS", + "DATE_INTERVAL_NOT_FOUND", + "DATE_NOT_PARSABLE", + "DATE_TIME_NOT_PARSABLE", + "EXCESS_BACKFILL_DIMENSIONS", + "EXCESS_DIMENSIONS", + "EXCESS_PERIODS", + "EXCESS_PRESENTMENTS", + "FROM_NOT_FOUND", + "FUNCTION_ARGUMENTS_NOT_FOUND", + "FUNCTION_EXCESS_ARGUMENTS", + "FUNCTION_INCOMPATIBLE_TYPES", + "FUNCTION_MODIFIER_NOT_FOUND", + "FUNCTION_NESTED_AGGREGATE", + "FUNCTION_NOT_FOUND", + "INVALID_DATE_RANGE", + "LIMIT_INVALID", + "LIST_MIXED_ARGUMENT_TYPES", + "MIXED_AGGREGATE_AND_NON_AGGREGATE", + "NAMED_DATE_NOT_FOUND", + "OPERATOR_INCOMPATIBLE_TYPES", + "PRESENTMENT_NOT_FOUND", + "REQUIRED_GROUP_BY_NOT_FOUND", + "SEMANTICALLY_INVALID", + "SORT_FIELD_NOT_FOUND", + "SYNTAX_FAILED_PREDICATE", + "SYNTAX_INPUT_MISMATCH", + "SYNTAX_INVALID_TOKEN", + "SYNTAX_MISSING_TOKEN", + "SYNTAX_NOT_RECOGNIZED", + "SYNTAX_NO_VIABLE_ALTERNATIVE", + "SYNTAX_UNWANTED_TOKEN", + "TABLE_NOT_FOUND", + "TIME_FUNCTION_NOT_FOUND", + "UNBACKFILLED_TIME_GROUP_BY_COMPARISON", + "UNKNOWN", + "VALUE_NOT_PARSABLE", + "VISUALIZE_CHART_TYPE_NOT_FOUND", + "VISUALIZE_EXCESS_PROJECTIONS", + "VISUALIZE_GROUP_BY_MIXED_BACKFILL", + "VISUALIZE_GROUP_BY_NOT_FOUND", + "VISUALIZE_INCOMPATIBLE_TYPES", + ) + + +class PaymentCustomizationErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "CUSTOM_APP_FUNCTION_NOT_ELIGIBLE", + "FUNCTION_DOES_NOT_IMPLEMENT", + "FUNCTION_ID_CANNOT_BE_CHANGED", + "FUNCTION_NOT_FOUND", + "FUNCTION_PENDING_DELETION", + "INVALID", + "INVALID_METAFIELDS", + "MAXIMUM_ACTIVE_PAYMENT_CUSTOMIZATIONS", + "PAYMENT_CUSTOMIZATION_FUNCTION_NOT_ELIGIBLE", + "PAYMENT_CUSTOMIZATION_NOT_FOUND", + "REQUIRED_INPUT_FIELD", + ) + + +class PaymentMethods(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "AMERICAN_EXPRESS", + "BITCOIN", + "BOGUS", + "DANKORT", + "DINERS_CLUB", + "DISCOVER", + "DOGECOIN", + "EFTPOS", + "ELO", + "FORBRUGSFORENINGEN", + "INTERAC", + "JCB", + "LITECOIN", + "MAESTRO", + "MASTERCARD", + "PAYPAL", + "UNIONPAY", + "VISA", + ) + + +class PaymentReminderSendUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("PAYMENT_REMINDER_SEND_UNSUCCESSFUL",) + + +class PaymentTermsCreateUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("PAYMENT_TERMS_CREATION_UNSUCCESSFUL",) + + +class PaymentTermsDeleteUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("PAYMENT_TERMS_DELETE_UNSUCCESSFUL",) + + +class PaymentTermsType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FIXED", "FULFILLMENT", "NET", "RECEIPT", "UNKNOWN") + + +class PaymentTermsUpdateUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("PAYMENT_TERMS_UPDATE_UNSUCCESSFUL",) + + +class PaypalExpressSubscriptionsGatewayStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("DISABLED", "ENABLED", "PENDING") + + +class PriceCalculationType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("COMPONENTS_SUM", "FIXED", "NONE") + + +class PriceListAdjustmentType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("PERCENTAGE_DECREASE", "PERCENTAGE_INCREASE") + + +class PriceListCompareAtMode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ADJUSTED", "NULLIFY") + + +class PriceListFixedPricesByProductBulkUpdateUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "DUPLICATE_ID_IN_INPUT", + "ID_MUST_BE_MUTUALLY_EXCLUSIVE", + "NO_UPDATE_OPERATIONS_SPECIFIED", + "PRICES_TO_ADD_CURRENCY_MISMATCH", + "PRICE_LIMIT_EXCEEDED", + "PRICE_LIST_DOES_NOT_EXIST", + "PRODUCT_DOES_NOT_EXIST", + ) + + +class PriceListPriceOriginType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FIXED", "RELATIVE") + + +class PriceListPriceUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("BLANK", "PRICE_LIST_CURRENCY_MISMATCH", "PRICE_LIST_NOT_FOUND", "PRICE_NOT_FIXED", "VARIANT_NOT_FOUND") + + +class PriceListSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ID", "NAME", "RELEVANCE") + + +class PriceListUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "APP_CATALOG_PRICE_LIST_ASSIGNMENT", + "BLANK", + "CATALOG_ASSIGNMENT_NOT_ALLOWED", + "CATALOG_CANNOT_CHANGE_CONTEXT_TYPE", + "CATALOG_CONTEXT_DOES_NOT_SUPPORT_QUANTITY_RULES", + "CATALOG_DOES_NOT_EXIST", + "CATALOG_MARKET_AND_PRICE_LIST_CURRENCY_MISMATCH", + "CATALOG_TAKEN", + "CONTEXT_RULE_COUNTRIES_LIMIT", + "CONTEXT_RULE_COUNTRY_TAKEN", + "CONTEXT_RULE_LIMIT_ONE_OPTION", + "CONTEXT_RULE_MARKET_NOT_FOUND", + "CONTEXT_RULE_MARKET_TAKEN", + "COUNTRY_CURRENCY_MISMATCH", + "COUNTRY_PRICE_LIST_ASSIGNMENT", + "CURRENCY_COUNTRY_MISMATCH", + "CURRENCY_MARKET_MISMATCH", + "CURRENCY_NOT_SUPPORTED", + "GENERIC_ERROR", + "INCLUSION", + "INVALID_ADJUSTMENT_MAX_VALUE", + "INVALID_ADJUSTMENT_MIN_VALUE", + "INVALID_ADJUSTMENT_VALUE", + "MARKET_CURRENCY_MISMATCH", + "PRICE_LIST_LOCKED", + "PRICE_LIST_NOT_ALLOWED_FOR_PRIMARY_MARKET", + "PRICE_LIST_NOT_FOUND", + "TAKEN", + "TOO_LONG", + ) + + +class PriceRuleAllocationMethod(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ACROSS", "EACH") + + +class PriceRuleErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "ALLOCATION_METHOD_MUST_BE_ACROSS_FOR_GIVEN_TARGET_SELECTION", + "APPLIES_ON_NOTHING", + "BLANK", + "BOGO_INVALID_TARGET_SELECTION", + "BOGO_INVALID_TARGET_TYPE", + "BOGO_INVALID_VALUE_TYPE", + "BOTH_CUSTOMER_AND_SAVED_SEARCH_PREREQUISITES_SELECTED", + "BOTH_CUSTOMER_AND_SEGMENT_PREREQUISITES_SELECTED", + "BOTH_SAVED_SEARCH_AND_SEGMENT_PREREQUISITES_SELECTED", + "CANNOT_ENTITLE_COLLECTIONS_WITH_PRODUCTS_OR_VARIANTS", + "CANNOT_PREREQUISITE_COLLECTION_WITH_PRODUCT_OR_VARIANTS", + "CUSTOMER_PREREQUISITES_EXCEEDED_MAX", + "CUSTOMER_PREREQUISITES_INVALID_SELECTION", + "CUSTOMER_PREREQUISITES_MISSING", + "CUSTOMER_PREREQUISITE_DUPLICATE", + "CUSTOMER_SAVED_SEARCH_DUPLICATE", + "CUSTOMER_SAVED_SEARCH_EXCEEDED_MAX", + "CUSTOMER_SAVED_SEARCH_INVALID", + "CUSTOMER_SEGMENT_EXCEEDED_MAX", + "CUSTOMER_SEGMENT_INVALID", + "CUSTOMER_SEGMENT_PREREQUISITE_DUPLICATE", + "DISCOUNT_CODE_DUPLICATE", + "END_DATE_BEFORE_START_DATE", + "EQUAL_TO", + "EXCEEDED_MAX", + "GREATER_THAN", + "GREATER_THAN_OR_EQUAL_TO", + "INTERNAL_ERROR", + "INVALID", + "INVALID_COMBINES_WITH_FOR_DISCOUNT_CLASS", + "INVALID_DISCOUNT_CLASS_FOR_PRICE_RULE", + "INVALID_TARGET_TYPE_PREREQUISITE_SHIPPING_PRICE_RANGE", + "ITEM_ENTITLEMENTS_DUPLICATE_COLLECTION", + "ITEM_ENTITLEMENTS_DUPLICATE_PRODUCT", + "ITEM_ENTITLEMENTS_DUPLICATE_VARIANT", + "ITEM_ENTITLEMENTS_EXCEEDED_MAX_COLLECTION", + "ITEM_ENTITLEMENTS_EXCEEDED_MAX_PRODUCT", + "ITEM_ENTITLEMENTS_EXCEEDED_MAX_VARIANT", + "ITEM_ENTITLEMENTS_INVALID_COLLECTION", + "ITEM_ENTITLEMENTS_INVALID_PRODUCT", + "ITEM_ENTITLEMENTS_INVALID_TARGET_TYPE_OR_SELECTION", + "ITEM_ENTITLEMENTS_INVALID_VARIANT", + "ITEM_ENTITLEMENTS_MISSING", + "ITEM_ENTITLEMENT_INVALID_TYPE", + "ITEM_PREREQUISITES_DUPLICATE_COLLECTION", + "ITEM_PREREQUISITES_DUPLICATE_PRODUCT", + "ITEM_PREREQUISITES_DUPLICATE_VARIANT", + "ITEM_PREREQUISITES_EXCEEDED_MAX", + "ITEM_PREREQUISITES_INVALID_COLLECTION", + "ITEM_PREREQUISITES_INVALID_PRODUCT", + "ITEM_PREREQUISITES_INVALID_TYPE", + "ITEM_PREREQUISITES_INVALID_VARIANT", + "ITEM_PREREQUISITES_MISSING", + "ITEM_PREREQUISITES_MUST_BE_EMPTY", + "LESS_THAN", + "LESS_THAN_OR_EQUAL_TO", + "MISSING_ARGUMENT", + "MULTIPLE_RECURRING_CYCLE_LIMIT_FOR_NON_SUBSCRIPTION_ITEMS", + "PREREQUISITE_SUBTOTAL_AND_QUANTITY_RANGE_BOTH_PRESENT", + "PRICE_RULE_ALLOCATION_LIMIT_IS_ZERO", + "PRICE_RULE_ALLOCATION_LIMIT_ON_NON_BOGO", + "PRICE_RULE_EXCEEDED_MAX_DISCOUNT_CODE", + "PRICE_RULE_PERCENTAGE_VALUE_OUTSIDE_RANGE", + "SHIPPING_ENTITLEMENTS_DUPLICATE_COUNTRY", + "SHIPPING_ENTITLEMENTS_EXCEEDED_MAX", + "SHIPPING_ENTITLEMENTS_INVALID_COUNTRY", + "SHIPPING_ENTITLEMENTS_INVALID_TARGET_TYPE_OR_SELECTION", + "SHIPPING_ENTITLEMENTS_MISSING", + "SHIPPING_ENTITLEMENTS_UNSUPPORTED_DESTINATION_TYPE", + "SHOP_EXCEEDED_MAX_PRICE_RULES", + "TAKEN", + "TOO_LONG", + "TOO_MANY_ARGUMENTS", + "TOO_SHORT", + "VARIANT_ALREADY_ENTITLED_THROUGH_PRODUCT", + ) + + +class PriceRuleFeature(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("BULK", "BUY_ONE_GET_ONE", "BUY_ONE_GET_ONE_WITH_ALLOCATION_LIMIT", "QUANTITY_DISCOUNTS", "SPECIFIC_CUSTOMERS") + + +class PriceRuleShareableUrlTargetType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("COLLECTION", "HOME", "PRODUCT") + + +class PriceRuleSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATED_AT", "ENDS_AT", "ID", "RELEVANCE", "STARTS_AT", "TITLE", "UPDATED_AT") + + +class PriceRuleStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ACTIVE", "EXPIRED", "SCHEDULED") + + +class PriceRuleTarget(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("LINE_ITEM", "SHIPPING_LINE") + + +class PriceRuleTrait(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("BULK", "BUY_ONE_GET_ONE", "BUY_ONE_GET_ONE_WITH_ALLOCATION_LIMIT", "QUANTITY_DISCOUNTS", "SPECIFIC_CUSTOMERS") + + +class PrivateMetafieldValueType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("INTEGER", "JSON_STRING", "STRING") + + +class ProductChangeStatusUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("PRODUCT_NOT_FOUND",) + + +class ProductCollectionSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("BEST_SELLING", "COLLECTION_DEFAULT", "CREATED", "ID", "MANUAL", "PRICE", "RELEVANCE", "TITLE") + + +class ProductDeleteUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("GENERIC_ERROR", "PRODUCT_DOES_NOT_EXIST") + + +class ProductDuplicateUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("BUNDLES_ERROR", "EMPTY_TITLE", "EMPTY_VARIANT", "FAILED_TO_SAVE", "GENERIC_ERROR", "PRODUCT_DOES_NOT_EXIST") + + +class ProductFeedCreateUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("INVALID", "TAKEN") + + +class ProductFeedDeleteUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("INVALID",) + + +class ProductFeedStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ACTIVE", "INACTIVE") + + +class ProductFullSyncUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("INVALID",) + + +class ProductImageSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATED_AT", "ID", "POSITION", "RELEVANCE") + + +class ProductMediaSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ID", "POSITION", "RELEVANCE") + + +class ProductSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATED_AT", "ID", "INVENTORY_TOTAL", "PRODUCT_TYPE", "PUBLISHED_AT", "RELEVANCE", "TITLE", "UPDATED_AT", "VENDOR") + + +class ProductStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ACTIVE", "ARCHIVED", "DRAFT") + + +class ProductVariantInventoryManagement(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FULFILLMENT_SERVICE", "NOT_MANAGED", "SHOPIFY") + + +class ProductVariantInventoryPolicy(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CONTINUE", "DENY") + + +class ProductVariantRelationshipBulkUpdateUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "CIRCULAR_REFERENCE", + "DUPLICATE_PRODUCT_VARIANT_RELATIONSHIP", + "EXCEEDED_PRODUCT_VARIANT_RELATIONSHIP_LIMIT", + "FAILED_TO_CREATE", + "FAILED_TO_REMOVE", + "FAILED_TO_UPDATE", + "FAILED_TO_UPDATE_PARENT_PRODUCT_VARIANT_PRICE", + "INVALID_QUANTITY", + "MUST_SPECIFY_COMPONENTS", + "NESTED_PARENT_PRODUCT_VARIANT", + "PARENT_PRODUCT_VARIANT_CANNOT_BE_GIFT_CARD", + "PARENT_PRODUCT_VARIANT_CANNOT_REQUIRE_SELLING_PLAN", + "PARENT_REQUIRED", + "PRODUCT_EXPANDER_APP_OWNERSHIP_ALREADY_EXISTS", + "PRODUCT_VARIANTS_NOT_COMPONENTS", + "PRODUCT_VARIANTS_NOT_FOUND", + "PRODUCT_VARIANT_RELATIONSHIP_TYPE_CONFLICT", + "UNEXPECTED_ERROR", + "UNSUPPORTED_MULTIPACK_RELATIONSHIP", + "UPDATE_PARENT_VARIANT_PRICE_REQUIRED", + ) + + +class ProductVariantSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "FULL_TITLE", + "ID", + "INVENTORY_LEVELS_AVAILABLE", + "INVENTORY_MANAGEMENT", + "INVENTORY_POLICY", + "INVENTORY_QUANTITY", + "NAME", + "POPULAR", + "POSITION", + "RELEVANCE", + "SKU", + "TITLE", + ) + + +class ProductVariantsBulkCreateUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "GREATER_THAN_OR_EQUAL_TO", + "INVALID", + "MUST_BE_FOR_THIS_PRODUCT", + "NEED_TO_ADD_OPTION_VALUES", + "NEGATIVE_PRICE_VALUE", + "NOT_DEFINED_FOR_SHOP", + "NO_KEY_ON_CREATE", + "OPTION_VALUES_FOR_NUMBER_OF_UNKNOWN_OPTIONS", + "PRODUCT_DOES_NOT_EXIST", + "SUBSCRIPTION_VIOLATION", + "TOO_MANY_INVENTORY_LOCATIONS", + "TRACKED_VARIANT_LOCATION_NOT_FOUND", + "VARIANT_ALREADY_EXISTS", + "VARIANT_ALREADY_EXISTS_CHANGE_OPTION_VALUE", + ) + + +class ProductVariantsBulkDeleteUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("AT_LEAST_ONE_VARIANT_DOES_NOT_BELONG_TO_THE_PRODUCT", "CANNOT_DELETE_LAST_VARIANT", "PRODUCT_DOES_NOT_EXIST") + + +class ProductVariantsBulkReorderUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("DUPLICATED_VARIANT_ID", "INVALID_POSITION", "MISSING_VARIANT", "PRODUCT_DOES_NOT_EXIST") + + +class ProductVariantsBulkUpdateUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "GREATER_THAN_OR_EQUAL_TO", + "NEED_TO_ADD_OPTION_VALUES", + "NEGATIVE_PRICE_VALUE", + "NO_INVENTORY_QUANTITES_DURING_UPDATE", + "NO_INVENTORY_QUANTITIES_ON_VARIANTS_UPDATE", + "OPTION_VALUES_FOR_NUMBER_OF_UNKNOWN_OPTIONS", + "PRODUCT_DOES_NOT_EXIST", + "PRODUCT_VARIANT_DOES_NOT_EXIST", + "PRODUCT_VARIANT_ID_MISSING", + "SUBSCRIPTION_VIOLATION", + "VARIANT_ALREADY_EXISTS", + ) + + +class ProfileItemSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATED_AT", "ID", "INVENTORY_TOTAL", "PRODUCT_TYPE", "PUBLISHED_AT", "RELEVANCE", "TITLE", "UPDATED_AT", "VENDOR") + + +class PubSubWebhookSubscriptionCreateUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("INVALID_PARAMETERS",) + + +class PubSubWebhookSubscriptionUpdateUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("INVALID_PARAMETERS",) + + +class PublicationCreateInputPublicationDefaultState(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ALL_PRODUCTS", "EMPTY") + + +class PublicationUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "BLANK", + "CANNOT_MODIFY_APP_CATALOG", + "CANNOT_MODIFY_APP_CATALOG_PUBLICATION", + "CANNOT_MODIFY_MARKET_CATALOG", + "CANNOT_MODIFY_MARKET_CATALOG_PUBLICATION", + "CATALOG_NOT_FOUND", + "INVALID", + "INVALID_PUBLISHABLE_ID", + "MARKET_NOT_FOUND", + "PRODUCT_TYPE_INCOMPATIBLE_WITH_CATALOG_TYPE", + "PUBLICATION_LOCKED", + "PUBLICATION_NOT_FOUND", + "PUBLICATION_UPDATE_LIMIT_EXCEEDED", + "TAKEN", + "TOO_LONG", + "TOO_SHORT", + "UNSUPPORTED_PUBLICATION_ACTION", + "UNSUPPORTED_PUBLISHABLE_TYPE", + ) + + +class QuantityRuleOriginType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FIXED", "RELATIVE") + + +class QuantityRuleUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "BLANK", + "CATALOG_CONTEXT_DOES_NOT_SUPPORT_QUANTITY_RULES", + "GENERIC_ERROR", + "GREATER_THAN_OR_EQUAL_TO", + "INCREMENT_IS_GREATER_THAN_MINIMUM", + "MAXIMUM_NOT_MULTIPLE_OF_INCREMENT", + "MINIMUM_IS_GREATER_THAN_MAXIMUM", + "MINIMUM_NOT_MULTIPLE_OF_INCREMENT", + "PRICE_LIST_DOES_NOT_EXIST", + "PRODUCT_VARIANT_DOES_NOT_EXIST", + "VARIANT_QUANTITY_RULE_DOES_NOT_EXIST", + ) + + +class RefundDutyRefundType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FULL", "PROPORTIONAL") + + +class RefundLineItemRestockType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CANCEL", "LEGACY_RESTOCK", "NO_RESTOCK", "RETURN") + + +class ResourceAlertIcon(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CHECKMARK_CIRCLE", "INFORMATION_CIRCLE") + + +class ResourceAlertSeverity(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CRITICAL", "DEFAULT", "INFO", "SUCCESS", "WARNING") + + +class ResourceFeedbackState(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ACCEPTED", "REQUIRES_ACTION") + + +class ResourceOperationStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ACTIVE", "COMPLETE", "CREATED") + + +class ReturnDeclineReason(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FINAL_SALE", "OTHER", "RETURN_PERIOD_ENDED") + + +class ReturnErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "ALREADY_EXISTS", + "BLANK", + "CREATION_FAILED", + "EQUAL_TO", + "FEATURE_NOT_ENABLED", + "GREATER_THAN", + "GREATER_THAN_OR_EQUAL_TO", + "INCLUSION", + "INTERNAL_ERROR", + "INVALID", + "INVALID_STATE", + "LESS_THAN", + "LESS_THAN_OR_EQUAL_TO", + "NOTIFICATION_FAILED", + "NOT_A_NUMBER", + "NOT_EDITABLE", + "NOT_FOUND", + "PRESENT", + "TAKEN", + "TOO_BIG", + "TOO_LONG", + "TOO_MANY_ARGUMENTS", + "TOO_SHORT", + "WRONG_LENGTH", + ) + + +class ReturnReason(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "COLOR", + "DEFECTIVE", + "NOT_AS_DESCRIBED", + "OTHER", + "SIZE_TOO_LARGE", + "SIZE_TOO_SMALL", + "STYLE", + "UNKNOWN", + "UNWANTED", + "WRONG_ITEM", + ) + + +class ReturnStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CANCELED", "CLOSED", "DECLINED", "OPEN", "REQUESTED") + + +class ReverseFulfillmentOrderDispositionType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("MISSING", "NOT_RESTOCKED", "PROCESSING_REQUIRED", "RESTOCKED") + + +class ReverseFulfillmentOrderStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CANCELED", "CLOSED", "OPEN") + + +class ReverseFulfillmentOrderThirdPartyConfirmationStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ACCEPTED", "CANCEL_ACCEPTED", "CANCEL_REJECTED", "PENDING_ACCEPTANCE", "PENDING_CANCELATION", "REJECTED") + + +class SaleActionType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ORDER", "RETURN", "UNKNOWN", "UPDATE") + + +class SaleLineType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ADDITIONAL_FEE", "ADJUSTMENT", "DUTY", "GIFT_CARD", "PRODUCT", "SHIPPING", "TIP", "UNKNOWN") + + +class ScriptTagDisplayScope(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ALL", "ONLINE_STORE", "ORDER_STATUS") + + +class SearchResultType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "COLLECTION", + "CUSTOMER", + "DISCOUNT_REDEEM_CODE", + "DRAFT_ORDER", + "FILE", + "ONLINE_STORE_ARTICLE", + "ONLINE_STORE_BLOG", + "ONLINE_STORE_PAGE", + "ORDER", + "PRICE_RULE", + "PRODUCT", + "URL_REDIRECT", + ) + + +class SegmentSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATION_DATE", "ID", "LAST_EDIT_DATE", "RELEVANCE") + + +class SellingPlanAnchorType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("MONTHDAY", "WEEKDAY", "YEARDAY") + + +class SellingPlanCategory(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("OTHER", "PRE_ORDER", "SUBSCRIPTION", "TRY_BEFORE_YOU_BUY") + + +class SellingPlanCheckoutChargeType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("PERCENTAGE", "PRICE") + + +class SellingPlanFixedDeliveryPolicyIntent(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FULFILLMENT_BEGIN",) + + +class SellingPlanFixedDeliveryPolicyPreAnchorBehavior(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ASAP", "NEXT") + + +class SellingPlanFulfillmentTrigger(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ANCHOR", "ASAP", "EXACT_TIME", "UNKNOWN") + + +class SellingPlanGroupSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATED_AT", "ID", "NAME", "RELEVANCE", "UPDATED_AT") + + +class SellingPlanGroupUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "BILLING_AND_DELIVERY_POLICY_TYPES_MUST_BE_THE_SAME", + "BLANK", + "CHECKOUT_CHARGE_VALUE_AND_TYPE_MUST_MATCH", + "EQUAL_TO", + "ERROR_ADDING_RESOURCE_TO_GROUP", + "FULFILLMENT_EXACT_TIME_NOT_ALLOWED", + "FULFILLMENT_EXACT_TIME_REQUIRED", + "GREATER_THAN", + "GREATER_THAN_OR_EQUAL_TO", + "GROUP_COULD_NOT_BE_DELETED", + "GROUP_DOES_NOT_EXIST", + "INCLUSION", + "INVALID", + "LESS_THAN", + "LESS_THAN_OR_EQUAL_TO", + "NOT_A_NUMBER", + "NOT_FOUND", + "ONLY_NEED_ONE_BILLING_POLICY_TYPE", + "ONLY_NEED_ONE_CHECKOUT_CHARGE_VALUE", + "ONLY_NEED_ONE_DELIVERY_POLICY_TYPE", + "ONLY_NEED_ONE_PRICING_POLICY_TYPE", + "ONLY_NEED_ONE_PRICING_POLICY_VALUE", + "ONLY_ONE_OF_FIXED_OR_RECURRING_BILLING", + "ONLY_ONE_OF_FIXED_OR_RECURRING_DELIVERY", + "PLAN_DOES_NOT_EXIST", + "PLAN_ID_MUST_BE_SPECIFIED_TO_UPDATE", + "PRESENT", + "PRICING_POLICY_ADJUSTMENT_VALUE_AND_TYPE_MUST_MATCH", + "PRODUCT_DOES_NOT_EXIST", + "PRODUCT_VARIANT_DOES_NOT_EXIST", + "REMAINING_BALANCE_CHARGE_EXACT_TIME_NOT_ALLOWED", + "REMAINING_BALANCE_CHARGE_EXACT_TIME_REQUIRED", + "REMAINING_BALANCE_CHARGE_TIME_AFTER_CHECKOUT_MUST_BE_GREATER_THAN_ZERO", + "REMAINING_BALANCE_CHARGE_TRIGGER_NO_REMAINING_BALANCE_ON_PARTIAL_PERCENTAGE_CHECKOUT_CHARGE", + "REMAINING_BALANCE_CHARGE_TRIGGER_NO_REMAINING_BALANCE_ON_PRICE_CHECKOUT_CHARGE", + "REMAINING_BALANCE_CHARGE_TRIGGER_ON_FULL_CHECKOUT", + "RESOURCE_LIST_CONTAINS_INVALID_IDS", + "SELLING_PLAN_ANCHORS_NOT_ALLOWED", + "SELLING_PLAN_ANCHORS_REQUIRED", + "SELLING_PLAN_BILLING_AND_DELIVERY_POLICY_ANCHORS_MUST_BE_EQUAL", + "SELLING_PLAN_BILLING_CYCLE_MUST_BE_A_MULTIPLE_OF_DELIVERY_CYCLE", + "SELLING_PLAN_BILLING_POLICY_MISSING", + "SELLING_PLAN_COUNT_LOWER_BOUND", + "SELLING_PLAN_COUNT_UPPER_BOUND", + "SELLING_PLAN_DELIVERY_POLICY_MISSING", + "SELLING_PLAN_DUPLICATE_NAME", + "SELLING_PLAN_DUPLICATE_OPTIONS", + "SELLING_PLAN_FIXED_PRICING_POLICIES_LIMIT", + "SELLING_PLAN_MAX_CYCLES_MUST_BE_GREATER_THAN_MIN_CYCLES", + "SELLING_PLAN_MISSING_OPTION2_LABEL_ON_PARENT_GROUP", + "SELLING_PLAN_MISSING_OPTION3_LABEL_ON_PARENT_GROUP", + "SELLING_PLAN_OPTION2_REQUIRED_AS_DEFINED_ON_PARENT_GROUP", + "SELLING_PLAN_OPTION3_REQUIRED_AS_DEFINED_ON_PARENT_GROUP", + "SELLING_PLAN_PRICING_POLICIES_LIMIT", + "SELLING_PLAN_PRICING_POLICIES_MUST_CONTAIN_A_FIXED_PRICING_POLICY", + "TAKEN", + "TOO_BIG", + "TOO_LONG", + "TOO_SHORT", + "WRONG_LENGTH", + ) + + +class SellingPlanInterval(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("DAY", "MONTH", "WEEK", "YEAR") + + +class SellingPlanPricingPolicyAdjustmentType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FIXED_AMOUNT", "PERCENTAGE", "PRICE") + + +class SellingPlanRecurringDeliveryPolicyIntent(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FULFILLMENT_BEGIN",) + + +class SellingPlanRecurringDeliveryPolicyPreAnchorBehavior(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ASAP", "NEXT") + + +class SellingPlanRemainingBalanceChargeTrigger(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("EXACT_TIME", "NO_REMAINING_BALANCE", "TIME_AFTER_CHECKOUT") + + +class SellingPlanReserve(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ON_FULFILLMENT", "ON_SALE") + + +class ServerPixelStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CONNECTED", "DISCONNECTED_CONFIGURED", "DISCONNECTED_UNCONFIGURED") + + +class ShippingDiscountClass(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("SHIPPING",) + + +class ShippingPackageType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("BOX", "ENVELOPE", "FLAT_RATE", "SOFT_PACK") + + +class ShopBranding(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ROGERS", "SHOPIFY", "SHOPIFY_GOLD", "SHOPIFY_PLUS") + + +class ShopCustomerAccountsSetting(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("DISABLED", "OPTIONAL", "REQUIRED") + + +class ShopPolicyErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("TOO_BIG",) + + +class ShopPolicyType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "CONTACT_INFORMATION", + "LEGAL_NOTICE", + "PRIVACY_POLICY", + "REFUND_POLICY", + "SHIPPING_POLICY", + "SUBSCRIPTION_POLICY", + "TERMS_OF_SALE", + "TERMS_OF_SERVICE", + ) + + +class ShopResourceFeedbackCreateUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("BLANK", "INVALID", "OUTDATED_FEEDBACK", "PRESENT") + + +class ShopTagSort(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ALPHABETICAL", "POPULAR") + + +class ShopifyPaymentsBankAccountStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ERRORED", "NEW", "VALIDATED", "VERIFIED") + + +class ShopifyPaymentsDisputeEvidenceFileType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "CANCELLATION_POLICY_FILE", + "CUSTOMER_COMMUNICATION_FILE", + "REFUND_POLICY_FILE", + "SERVICE_DOCUMENTATION_FILE", + "SHIPPING_DOCUMENTATION_FILE", + "UNCATEGORIZED_FILE", + ) + + +class ShopifyPaymentsDisputeReason(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "BANK_CANNOT_PROCESS", + "CREDIT_NOT_PROCESSED", + "CUSTOMER_INITIATED", + "DEBIT_NOT_AUTHORIZED", + "DUPLICATE", + "FRAUDULENT", + "GENERAL", + "INCORRECT_ACCOUNT_DETAILS", + "INSUFFICIENT_FUNDS", + "PRODUCT_NOT_RECEIVED", + "PRODUCT_UNACCEPTABLE", + "SUBSCRIPTION_CANCELLED", + "UNRECOGNIZED", + ) + + +class ShopifyPaymentsPayoutInterval(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("DAILY", "MANUAL", "MONTHLY", "WEEKLY") + + +class ShopifyPaymentsPayoutStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CANCELED", "FAILED", "IN_TRANSIT", "PAID", "SCHEDULED") + + +class ShopifyPaymentsPayoutTransactionType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("DEPOSIT", "WITHDRAWAL") + + +class ShopifyPaymentsVerificationDocumentType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("DRIVERS_LICENSE", "GOVERNMENT_IDENTIFICATION", "PASSPORT") + + +class ShopifyPaymentsVerificationStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("PENDING", "UNVERIFIED", "VERIFIED") + + +class StaffMemberDefaultImage(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("DEFAULT", "NOT_FOUND", "TRANSPARENT") + + +class StaffMemberPermission(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "APPLICATIONS", + "CHANNELS", + "CUSTOMERS", + "DASHBOARD", + "DOMAINS", + "DRAFT_ORDERS", + "EDIT_ORDERS", + "GIFT_CARDS", + "LINKS", + "LOCATIONS", + "MARKETING", + "MARKETING_SECTION", + "ORDERS", + "OVERVIEWS", + "PAGES", + "PAY_ORDERS_BY_VAULTED_CARD", + "PREFERENCES", + "PRODUCTS", + "REPORTS", + "THEMES", + ) + + +class StagedUploadHttpMethodType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("POST", "PUT") + + +class StagedUploadTargetGenerateUploadResource(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "BULK_MUTATION_VARIABLES", + "COLLECTION_IMAGE", + "FILE", + "IMAGE", + "MODEL_3D", + "PRODUCT_IMAGE", + "RETURN_LABEL", + "SHOP_IMAGE", + "URL_REDIRECT_IMPORT", + "VIDEO", + ) + + +class StandardMetafieldDefinitionEnableUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "INVALID", + "LIMIT_EXCEEDED", + "TAKEN", + "TEMPLATE_NOT_FOUND", + "TYPE_NOT_ALLOWED_FOR_CONDITIONS", + "UNSTRUCTURED_ALREADY_EXISTS", + ) + + +class StorefrontID(sgqlc.types.Scalar): + __schema__ = shopify_schema + + +String = sgqlc.types.String + + +class SubscriptionBillingAttemptErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "AMOUNT_TOO_SMALL", + "AUTHENTICATION_ERROR", + "BUYER_CANCELED_PAYMENT_METHOD", + "CUSTOMER_INVALID", + "CUSTOMER_NOT_FOUND", + "EXPIRED_PAYMENT_METHOD", + "INVALID_CUSTOMER_BILLING_AGREEMENT", + "INVALID_PAYMENT_METHOD", + "INVALID_SHIPPING_ADDRESS", + "INVENTORY_ALLOCATIONS_NOT_FOUND", + "INVOICE_ALREADY_PAID", + "PAYMENT_METHOD_DECLINED", + "PAYMENT_METHOD_INCOMPATIBLE_WITH_GATEWAY_CONFIG", + "PAYMENT_METHOD_NOT_FOUND", + "PAYMENT_PROVIDER_IS_NOT_ENABLED", + "TEST_MODE", + "TRANSIENT_ERROR", + "UNEXPECTED_ERROR", + ) + + +class SubscriptionBillingAttemptsSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATED_AT", "ID", "RELEVANCE") + + +class SubscriptionBillingCycleBillingCycleStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("BILLED", "UNBILLED") + + +class SubscriptionBillingCycleErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "BILLING_DATE_SET_ON_SKIPPED", + "CYCLE_INDEX_OUT_OF_RANGE", + "CYCLE_NOT_FOUND", + "CYCLE_START_DATE_OUT_OF_RANGE", + "EMPTY_BILLING_CYCLE_EDIT_SCHEDULE_INPUT", + "INCOMPLETE_BILLING_ATTEMPTS", + "INVALID", + "INVALID_CYCLE_INDEX", + "INVALID_DATE", + "NO_CYCLE_EDITS", + "OUT_OF_BOUNDS", + "UPCOMING_CYCLE_LIMIT_EXCEEDED", + ) + + +class SubscriptionBillingCycleScheduleEditInputScheduleEditReason(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("BUYER_INITIATED", "DEV_INITIATED", "MERCHANT_INITIATED") + + +class SubscriptionBillingCyclesSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CYCLE_INDEX", "ID", "RELEVANCE") + + +class SubscriptionBillingCyclesTargetSelection(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ALL",) + + +class SubscriptionContractErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("INVALID",) + + +class SubscriptionContractLastPaymentStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("FAILED", "SUCCEEDED") + + +class SubscriptionContractSubscriptionStatus(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ACTIVE", "CANCELLED", "EXPIRED", "FAILED", "PAUSED", "STALE") + + +class SubscriptionDiscountRejectionReason(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "CURRENTLY_INACTIVE", + "CUSTOMER_NOT_ELIGIBLE", + "CUSTOMER_USAGE_LIMIT_REACHED", + "INCOMPATIBLE_PURCHASE_TYPE", + "INTERNAL_ERROR", + "NOT_FOUND", + "NO_ENTITLED_LINE_ITEMS", + "NO_ENTITLED_SHIPPING_LINES", + "PURCHASE_NOT_IN_RANGE", + "QUANTITY_NOT_IN_RANGE", + "USAGE_LIMIT_REACHED", + ) + + +class SubscriptionDraftErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "ALREADY_REMOVED", + "BILLING_CYCLE_ABSENT", + "BILLING_CYCLE_CONTRACT_DRAFT_BILLING_POLICY_INVALID", + "BILLING_CYCLE_CONTRACT_DRAFT_DELIVERY_POLICY_INVALID", + "BILLING_CYCLE_PRESENT", + "BLANK", + "COMMITTED", + "CONCATENATION_BILLING_CYCLE_CONTRACT_DRAFT_REQUIRED", + "CURRENCY_NOT_ENABLED", + "CUSTOMER_DOES_NOT_EXIST", + "CUSTOMER_MISMATCH", + "CYCLE_DISCOUNTS_UNIQUE_AFTER_CYCLE", + "CYCLE_INDEX_OUT_OF_RANGE", + "CYCLE_SELECTOR_VALIDATE_ONE_OF", + "CYCLE_START_DATE_OUT_OF_RANGE", + "DELIVERY_METHOD_REQUIRED", + "DELIVERY_MUST_BE_MULTIPLE_OF_BILLING", + "DUPLICATE_CONCATENATED_CONTRACTS", + "EXCEEDED_MAX_CONCATENATED_CONTRACTS", + "GREATER_THAN", + "GREATER_THAN_OR_EQUAL_TO", + "HAS_FUTURE_EDITS", + "INVALID", + "INVALID_ADJUSTMENT_TYPE", + "INVALID_ADJUSTMENT_VALUE", + "INVALID_BILLING_DATE", + "INVALID_LINES", + "INVALID_NOTE_LENGTH", + "LESS_THAN", + "LESS_THAN_OR_EQUAL_TO", + "MISSING_LOCAL_DELIVERY_OPTIONS", + "NOT_AN_INTEGER", + "NOT_IN_RANGE", + "NO_ENTITLED_LINES", + "PRESENCE", + "SELLING_PLAN_MAX_CYCLES_MUST_BE_GREATER_THAN_MIN_CYCLES", + "STALE_CONTRACT", + "TOO_LONG", + "TOO_SHORT", + "UPCOMING_CYCLE_LIMIT_EXCEEDED", + ) + + +class SuggestedOrderTransactionKind(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("SUGGESTED_REFUND",) + + +class TaxAppConfigureUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("TAX_PARTNER_ALREADY_ACTIVE", "TAX_PARTNER_NOT_FOUND", "TAX_PARTNER_STATE_UPDATE_FAILED") + + +class TaxExemption(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "CA_BC_COMMERCIAL_FISHERY_EXEMPTION", + "CA_BC_CONTRACTOR_EXEMPTION", + "CA_BC_PRODUCTION_AND_MACHINERY_EXEMPTION", + "CA_BC_RESELLER_EXEMPTION", + "CA_BC_SUB_CONTRACTOR_EXEMPTION", + "CA_DIPLOMAT_EXEMPTION", + "CA_MB_COMMERCIAL_FISHERY_EXEMPTION", + "CA_MB_FARMER_EXEMPTION", + "CA_MB_RESELLER_EXEMPTION", + "CA_NS_COMMERCIAL_FISHERY_EXEMPTION", + "CA_NS_FARMER_EXEMPTION", + "CA_ON_PURCHASE_EXEMPTION", + "CA_PE_COMMERCIAL_FISHERY_EXEMPTION", + "CA_SK_COMMERCIAL_FISHERY_EXEMPTION", + "CA_SK_CONTRACTOR_EXEMPTION", + "CA_SK_FARMER_EXEMPTION", + "CA_SK_PRODUCTION_AND_MACHINERY_EXEMPTION", + "CA_SK_RESELLER_EXEMPTION", + "CA_SK_SUB_CONTRACTOR_EXEMPTION", + "CA_STATUS_CARD_EXEMPTION", + "EU_REVERSE_CHARGE_EXEMPTION_RULE", + "US_AK_RESELLER_EXEMPTION", + "US_AL_RESELLER_EXEMPTION", + "US_AR_RESELLER_EXEMPTION", + "US_AZ_RESELLER_EXEMPTION", + "US_CA_RESELLER_EXEMPTION", + "US_CO_RESELLER_EXEMPTION", + "US_CT_RESELLER_EXEMPTION", + "US_DC_RESELLER_EXEMPTION", + "US_DE_RESELLER_EXEMPTION", + "US_FL_RESELLER_EXEMPTION", + "US_GA_RESELLER_EXEMPTION", + "US_HI_RESELLER_EXEMPTION", + "US_IA_RESELLER_EXEMPTION", + "US_ID_RESELLER_EXEMPTION", + "US_IL_RESELLER_EXEMPTION", + "US_IN_RESELLER_EXEMPTION", + "US_KS_RESELLER_EXEMPTION", + "US_KY_RESELLER_EXEMPTION", + "US_LA_RESELLER_EXEMPTION", + "US_MA_RESELLER_EXEMPTION", + "US_MD_RESELLER_EXEMPTION", + "US_ME_RESELLER_EXEMPTION", + "US_MI_RESELLER_EXEMPTION", + "US_MN_RESELLER_EXEMPTION", + "US_MO_RESELLER_EXEMPTION", + "US_MS_RESELLER_EXEMPTION", + "US_MT_RESELLER_EXEMPTION", + "US_NC_RESELLER_EXEMPTION", + "US_ND_RESELLER_EXEMPTION", + "US_NE_RESELLER_EXEMPTION", + "US_NH_RESELLER_EXEMPTION", + "US_NJ_RESELLER_EXEMPTION", + "US_NM_RESELLER_EXEMPTION", + "US_NV_RESELLER_EXEMPTION", + "US_NY_RESELLER_EXEMPTION", + "US_OH_RESELLER_EXEMPTION", + "US_OK_RESELLER_EXEMPTION", + "US_OR_RESELLER_EXEMPTION", + "US_PA_RESELLER_EXEMPTION", + "US_RI_RESELLER_EXEMPTION", + "US_SC_RESELLER_EXEMPTION", + "US_SD_RESELLER_EXEMPTION", + "US_TN_RESELLER_EXEMPTION", + "US_TX_RESELLER_EXEMPTION", + "US_UT_RESELLER_EXEMPTION", + "US_VA_RESELLER_EXEMPTION", + "US_VT_RESELLER_EXEMPTION", + "US_WA_RESELLER_EXEMPTION", + "US_WI_RESELLER_EXEMPTION", + "US_WV_RESELLER_EXEMPTION", + "US_WY_RESELLER_EXEMPTION", + ) + + +class TaxPartnerState(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ACTIVE", "PENDING", "READY") + + +class TranslatableResourceType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "COLLECTION", + "DELIVERY_METHOD_DEFINITION", + "EMAIL_TEMPLATE", + "FILTER", + "LINK", + "METAFIELD", + "METAOBJECT", + "ONLINE_STORE_ARTICLE", + "ONLINE_STORE_BLOG", + "ONLINE_STORE_MENU", + "ONLINE_STORE_PAGE", + "ONLINE_STORE_THEME", + "PACKING_SLIP_TEMPLATE", + "PAYMENT_GATEWAY", + "PRODUCT", + "PRODUCT_OPTION", + "PRODUCT_VARIANT", + "SELLING_PLAN", + "SELLING_PLAN_GROUP", + "SHOP", + "SHOP_POLICY", + ) + + +class TranslationErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "BLANK", + "FAILS_RESOURCE_VALIDATION", + "INVALID", + "INVALID_CODE", + "INVALID_FORMAT", + "INVALID_KEY_FOR_MODEL", + "INVALID_LOCALE_FOR_MARKET", + "INVALID_LOCALE_FOR_SHOP", + "INVALID_MARKET_LOCALIZABLE_CONTENT", + "INVALID_TRANSLATABLE_CONTENT", + "INVALID_VALUE_FOR_HANDLE_TRANSLATION", + "MARKET_CUSTOM_CONTENT_NOT_ALLOWED", + "MARKET_DOES_NOT_EXIST", + "MARKET_LOCALE_CREATION_FAILED", + "RESOURCE_NOT_FOUND", + "RESOURCE_NOT_MARKET_CUSTOMIZABLE", + "RESOURCE_NOT_TRANSLATABLE", + "TOO_MANY_KEYS_FOR_RESOURCE", + ) + + +class URL(sgqlc.types.Scalar): + __schema__ = shopify_schema + + +class UnitSystem(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("IMPERIAL_SYSTEM", "METRIC_SYSTEM") + + +class UnsignedInt64(sgqlc.types.Scalar): + __schema__ = shopify_schema + + +class UrlRedirectBulkDeleteByIdsUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("IDS_EMPTY",) + + +class UrlRedirectBulkDeleteBySavedSearchUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("INVALID_SAVED_SEARCH_QUERY", "SAVED_SEARCH_NOT_FOUND") + + +class UrlRedirectBulkDeleteBySearchUserErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("INVALID_SEARCH_ARGUMENT",) + + +class UrlRedirectErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATE_FAILED", "DELETE_FAILED", "DOES_NOT_EXIST", "UPDATE_FAILED") + + +class UrlRedirectImportErrorCode(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ALREADY_IMPORTED", "IN_PROGRESS", "NOT_FOUND") + + +class UrlRedirectSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("ID", "PATH", "RELEVANCE") + + +class UtcOffset(sgqlc.types.Scalar): + __schema__ = shopify_schema + + +class VisualizationType(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("BAR", "LINE") + + +class WebhookSubscriptionFormat(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("JSON", "XML") + + +class WebhookSubscriptionSortKeys(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("CREATED_AT", "ID", "RELEVANCE") + + +class WebhookSubscriptionTopic(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ( + "APP_PURCHASES_ONE_TIME_UPDATE", + "APP_SUBSCRIPTIONS_APPROACHING_CAPPED_AMOUNT", + "APP_SUBSCRIPTIONS_UPDATE", + "APP_UNINSTALLED", + "ATTRIBUTED_SESSIONS_FIRST", + "ATTRIBUTED_SESSIONS_LAST", + "AUDIT_EVENTS_ADMIN_API_ACTIVITY", + "BULK_OPERATIONS_FINISH", + "CARTS_CREATE", + "CARTS_UPDATE", + "CHANNELS_DELETE", + "CHECKOUTS_CREATE", + "CHECKOUTS_DELETE", + "CHECKOUTS_UPDATE", + "COLLECTIONS_CREATE", + "COLLECTIONS_DELETE", + "COLLECTIONS_UPDATE", + "COLLECTION_LISTINGS_ADD", + "COLLECTION_LISTINGS_REMOVE", + "COLLECTION_LISTINGS_UPDATE", + "COLLECTION_PUBLICATIONS_CREATE", + "COLLECTION_PUBLICATIONS_DELETE", + "COLLECTION_PUBLICATIONS_UPDATE", + "COMPANIES_CREATE", + "COMPANIES_DELETE", + "COMPANIES_UPDATE", + "COMPANY_CONTACTS_CREATE", + "COMPANY_CONTACTS_DELETE", + "COMPANY_CONTACTS_UPDATE", + "COMPANY_CONTACT_ROLES_ASSIGN", + "COMPANY_CONTACT_ROLES_REVOKE", + "COMPANY_LOCATIONS_CREATE", + "COMPANY_LOCATIONS_DELETE", + "COMPANY_LOCATIONS_UPDATE", + "CUSTOMERS_CREATE", + "CUSTOMERS_DELETE", + "CUSTOMERS_DISABLE", + "CUSTOMERS_EMAIL_MARKETING_CONSENT_UPDATE", + "CUSTOMERS_ENABLE", + "CUSTOMERS_MARKETING_CONSENT_UPDATE", + "CUSTOMERS_MERGE", + "CUSTOMERS_UPDATE", + "CUSTOMER_GROUPS_CREATE", + "CUSTOMER_GROUPS_DELETE", + "CUSTOMER_GROUPS_UPDATE", + "CUSTOMER_PAYMENT_METHODS_CREATE", + "CUSTOMER_PAYMENT_METHODS_REVOKE", + "CUSTOMER_PAYMENT_METHODS_UPDATE", + "CUSTOMER_TAGS_ADDED", + "CUSTOMER_TAGS_REMOVED", + "DISPUTES_CREATE", + "DISPUTES_UPDATE", + "DOMAINS_CREATE", + "DOMAINS_DESTROY", + "DOMAINS_UPDATE", + "DRAFT_ORDERS_CREATE", + "DRAFT_ORDERS_DELETE", + "DRAFT_ORDERS_UPDATE", + "FULFILLMENTS_CREATE", + "FULFILLMENTS_UPDATE", + "FULFILLMENT_EVENTS_CREATE", + "FULFILLMENT_EVENTS_DELETE", + "FULFILLMENT_ORDERS_CANCELLATION_REQUEST_ACCEPTED", + "FULFILLMENT_ORDERS_CANCELLATION_REQUEST_REJECTED", + "FULFILLMENT_ORDERS_CANCELLATION_REQUEST_SUBMITTED", + "FULFILLMENT_ORDERS_CANCELLED", + "FULFILLMENT_ORDERS_FULFILLMENT_REQUEST_ACCEPTED", + "FULFILLMENT_ORDERS_FULFILLMENT_REQUEST_REJECTED", + "FULFILLMENT_ORDERS_FULFILLMENT_REQUEST_SUBMITTED", + "FULFILLMENT_ORDERS_FULFILLMENT_SERVICE_FAILED_TO_COMPLETE", + "FULFILLMENT_ORDERS_HOLD_RELEASED", + "FULFILLMENT_ORDERS_LINE_ITEMS_PREPARED_FOR_LOCAL_DELIVERY", + "FULFILLMENT_ORDERS_LINE_ITEMS_PREPARED_FOR_PICKUP", + "FULFILLMENT_ORDERS_MOVED", + "FULFILLMENT_ORDERS_ORDER_ROUTING_COMPLETE", + "FULFILLMENT_ORDERS_PLACED_ON_HOLD", + "FULFILLMENT_ORDERS_RESCHEDULED", + "FULFILLMENT_ORDERS_SCHEDULED_FULFILLMENT_ORDER_READY", + "INVENTORY_ITEMS_CREATE", + "INVENTORY_ITEMS_DELETE", + "INVENTORY_ITEMS_UPDATE", + "INVENTORY_LEVELS_CONNECT", + "INVENTORY_LEVELS_DISCONNECT", + "INVENTORY_LEVELS_UPDATE", + "LOCALES_CREATE", + "LOCALES_UPDATE", + "LOCATIONS_ACTIVATE", + "LOCATIONS_CREATE", + "LOCATIONS_DEACTIVATE", + "LOCATIONS_DELETE", + "LOCATIONS_UPDATE", + "MARKETS_CREATE", + "MARKETS_DELETE", + "MARKETS_UPDATE", + "ORDERS_CANCELLED", + "ORDERS_CREATE", + "ORDERS_DELETE", + "ORDERS_EDITED", + "ORDERS_FULFILLED", + "ORDERS_PAID", + "ORDERS_PARTIALLY_FULFILLED", + "ORDERS_UPDATED", + "ORDER_TRANSACTIONS_CREATE", + "PAYMENT_SCHEDULES_DUE", + "PAYMENT_TERMS_CREATE", + "PAYMENT_TERMS_DELETE", + "PAYMENT_TERMS_UPDATE", + "PRODUCTS_CREATE", + "PRODUCTS_DELETE", + "PRODUCTS_UPDATE", + "PRODUCT_FEEDS_CREATE", + "PRODUCT_FEEDS_FULL_SYNC", + "PRODUCT_FEEDS_INCREMENTAL_SYNC", + "PRODUCT_FEEDS_UPDATE", + "PRODUCT_LISTINGS_ADD", + "PRODUCT_LISTINGS_REMOVE", + "PRODUCT_LISTINGS_UPDATE", + "PRODUCT_PUBLICATIONS_CREATE", + "PRODUCT_PUBLICATIONS_DELETE", + "PRODUCT_PUBLICATIONS_UPDATE", + "PROFILES_CREATE", + "PROFILES_DELETE", + "PROFILES_UPDATE", + "REFUNDS_CREATE", + "RETURNS_APPROVE", + "RETURNS_CANCEL", + "RETURNS_CLOSE", + "RETURNS_DECLINE", + "RETURNS_REOPEN", + "RETURNS_REQUEST", + "REVERSE_DELIVERIES_ATTACH_DELIVERABLE", + "REVERSE_FULFILLMENT_ORDERS_DISPOSE", + "SCHEDULED_PRODUCT_LISTINGS_ADD", + "SCHEDULED_PRODUCT_LISTINGS_REMOVE", + "SCHEDULED_PRODUCT_LISTINGS_UPDATE", + "SEGMENTS_CREATE", + "SEGMENTS_DELETE", + "SEGMENTS_UPDATE", + "SELLING_PLAN_GROUPS_CREATE", + "SELLING_PLAN_GROUPS_DELETE", + "SELLING_PLAN_GROUPS_UPDATE", + "SHIPPING_ADDRESSES_CREATE", + "SHIPPING_ADDRESSES_UPDATE", + "SHOP_UPDATE", + "SUBSCRIPTION_BILLING_ATTEMPTS_CHALLENGED", + "SUBSCRIPTION_BILLING_ATTEMPTS_FAILURE", + "SUBSCRIPTION_BILLING_ATTEMPTS_SUCCESS", + "SUBSCRIPTION_BILLING_CYCLE_EDITS_CREATE", + "SUBSCRIPTION_BILLING_CYCLE_EDITS_DELETE", + "SUBSCRIPTION_BILLING_CYCLE_EDITS_UPDATE", + "SUBSCRIPTION_CONTRACTS_CREATE", + "SUBSCRIPTION_CONTRACTS_UPDATE", + "TAX_PARTNERS_UPDATE", + "TAX_SERVICES_CREATE", + "TAX_SERVICES_UPDATE", + "TENDER_TRANSACTIONS_CREATE", + "THEMES_CREATE", + "THEMES_DELETE", + "THEMES_PUBLISH", + "THEMES_UPDATE", + "VARIANTS_IN_STOCK", + "VARIANTS_OUT_OF_STOCK", + ) + + +class WeightUnit(sgqlc.types.Enum): + __schema__ = shopify_schema + __choices__ = ("GRAMS", "KILOGRAMS", "OUNCES", "POUNDS") + + +######################################################################## +# Input Objects +######################################################################## +class AppPlanInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("app_usage_pricing_details", "app_recurring_pricing_details") + app_usage_pricing_details = sgqlc.types.Field("AppUsagePricingInput", graphql_name="appUsagePricingDetails") + app_recurring_pricing_details = sgqlc.types.Field("AppRecurringPricingInput", graphql_name="appRecurringPricingDetails") + + +class AppRecurringPricingInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("interval", "price", "discount") + interval = sgqlc.types.Field(AppPricingInterval, graphql_name="interval") + price = sgqlc.types.Field(sgqlc.types.non_null("MoneyInput"), graphql_name="price") + discount = sgqlc.types.Field("AppSubscriptionDiscountInput", graphql_name="discount") + + +class AppRevenueAttributionRecordInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("idempotency_key", "captured_at", "amount", "type", "test") + idempotency_key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="idempotencyKey") + captured_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="capturedAt") + amount = sgqlc.types.Field(sgqlc.types.non_null("MoneyInput"), graphql_name="amount") + type = sgqlc.types.Field(sgqlc.types.non_null(AppRevenueAttributionType), graphql_name="type") + test = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="test") + + +class AppSubscriptionDiscountInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("value", "duration_limit_in_intervals") + value = sgqlc.types.Field("AppSubscriptionDiscountValueInput", graphql_name="value") + duration_limit_in_intervals = sgqlc.types.Field(Int, graphql_name="durationLimitInIntervals") + + +class AppSubscriptionDiscountValueInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("percentage", "amount") + percentage = sgqlc.types.Field(Float, graphql_name="percentage") + amount = sgqlc.types.Field(Decimal, graphql_name="amount") + + +class AppSubscriptionLineItemInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("plan",) + plan = sgqlc.types.Field(sgqlc.types.non_null(AppPlanInput), graphql_name="plan") + + +class AppUsagePricingInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("capped_amount", "terms") + capped_amount = sgqlc.types.Field(sgqlc.types.non_null("MoneyInput"), graphql_name="cappedAmount") + terms = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="terms") + + +class AttributeInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("key", "value") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") + + +class BuyerExperienceConfigurationInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("checkout_to_draft", "payment_terms_template_id", "editable_shipping_address") + checkout_to_draft = sgqlc.types.Field(Boolean, graphql_name="checkoutToDraft") + payment_terms_template_id = sgqlc.types.Field(ID, graphql_name="paymentTermsTemplateId") + editable_shipping_address = sgqlc.types.Field(Boolean, graphql_name="editableShippingAddress") + + +class CatalogContextInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("company_location_ids",) + company_location_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="companyLocationIds") + + +class CatalogCreateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("title", "status", "context", "price_list_id", "publication_id") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + status = sgqlc.types.Field(sgqlc.types.non_null(CatalogStatus), graphql_name="status") + context = sgqlc.types.Field(sgqlc.types.non_null(CatalogContextInput), graphql_name="context") + price_list_id = sgqlc.types.Field(ID, graphql_name="priceListId") + publication_id = sgqlc.types.Field(ID, graphql_name="publicationId") + + +class CatalogUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("title", "status", "context", "price_list_id", "publication_id") + title = sgqlc.types.Field(String, graphql_name="title") + status = sgqlc.types.Field(CatalogStatus, graphql_name="status") + context = sgqlc.types.Field(CatalogContextInput, graphql_name="context") + price_list_id = sgqlc.types.Field(ID, graphql_name="priceListId") + publication_id = sgqlc.types.Field(ID, graphql_name="publicationId") + + +class CollectionDeleteInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id",) + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + +class CollectionInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "description_html", + "handle", + "id", + "image", + "products", + "rule_set", + "template_suffix", + "sort_order", + "title", + "metafields", + "seo", + "redirect_new_handle", + ) + description_html = sgqlc.types.Field(String, graphql_name="descriptionHtml") + handle = sgqlc.types.Field(String, graphql_name="handle") + id = sgqlc.types.Field(ID, graphql_name="id") + image = sgqlc.types.Field("ImageInput", graphql_name="image") + products = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="products") + rule_set = sgqlc.types.Field("CollectionRuleSetInput", graphql_name="ruleSet") + template_suffix = sgqlc.types.Field(String, graphql_name="templateSuffix") + sort_order = sgqlc.types.Field(CollectionSortOrder, graphql_name="sortOrder") + title = sgqlc.types.Field(String, graphql_name="title") + metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldInput")), graphql_name="metafields") + seo = sgqlc.types.Field("SEOInput", graphql_name="seo") + redirect_new_handle = sgqlc.types.Field(Boolean, graphql_name="redirectNewHandle") + + +class CollectionPublicationInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("publication_id",) + publication_id = sgqlc.types.Field(ID, graphql_name="publicationId") + + +class CollectionPublishInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "collection_publications") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + collection_publications = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CollectionPublicationInput))), graphql_name="collectionPublications" + ) + + +class CollectionRuleInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("column", "relation", "condition", "condition_object_id") + column = sgqlc.types.Field(sgqlc.types.non_null(CollectionRuleColumn), graphql_name="column") + relation = sgqlc.types.Field(sgqlc.types.non_null(CollectionRuleRelation), graphql_name="relation") + condition = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="condition") + condition_object_id = sgqlc.types.Field(ID, graphql_name="conditionObjectId") + + +class CollectionRuleSetInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("applied_disjunctively", "rules") + applied_disjunctively = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliedDisjunctively") + rules = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(CollectionRuleInput)), graphql_name="rules") + + +class CollectionUnpublishInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "collection_publications") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + collection_publications = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CollectionPublicationInput))), graphql_name="collectionPublications" + ) + + +class CompanyAddressInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("address1", "address2", "city", "zip", "recipient", "phone", "zone_code", "country_code") + address1 = sgqlc.types.Field(String, graphql_name="address1") + address2 = sgqlc.types.Field(String, graphql_name="address2") + city = sgqlc.types.Field(String, graphql_name="city") + zip = sgqlc.types.Field(String, graphql_name="zip") + recipient = sgqlc.types.Field(String, graphql_name="recipient") + phone = sgqlc.types.Field(String, graphql_name="phone") + zone_code = sgqlc.types.Field(String, graphql_name="zoneCode") + country_code = sgqlc.types.Field(CountryCode, graphql_name="countryCode") + + +class CompanyContactInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("first_name", "last_name", "email", "title", "locale", "phone") + first_name = sgqlc.types.Field(String, graphql_name="firstName") + last_name = sgqlc.types.Field(String, graphql_name="lastName") + email = sgqlc.types.Field(String, graphql_name="email") + title = sgqlc.types.Field(String, graphql_name="title") + locale = sgqlc.types.Field(String, graphql_name="locale") + phone = sgqlc.types.Field(String, graphql_name="phone") + + +class CompanyContactRoleAssign(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("company_contact_role_id", "company_location_id") + company_contact_role_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="companyContactRoleId") + company_location_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="companyLocationId") + + +class CompanyCreateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("company", "company_contact", "company_location") + company = sgqlc.types.Field(sgqlc.types.non_null("CompanyInput"), graphql_name="company") + company_contact = sgqlc.types.Field(CompanyContactInput, graphql_name="companyContact") + company_location = sgqlc.types.Field("CompanyLocationInput", graphql_name="companyLocation") + + +class CompanyInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("name", "note", "external_id", "customer_since") + name = sgqlc.types.Field(String, graphql_name="name") + note = sgqlc.types.Field(String, graphql_name="note") + external_id = sgqlc.types.Field(String, graphql_name="externalId") + customer_since = sgqlc.types.Field(DateTime, graphql_name="customerSince") + + +class CompanyLocationInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "name", + "phone", + "locale", + "external_id", + "note", + "buyer_experience_configuration", + "billing_address", + "shipping_address", + "billing_same_as_shipping", + "tax_registration_id", + "tax_exemptions", + ) + name = sgqlc.types.Field(String, graphql_name="name") + phone = sgqlc.types.Field(String, graphql_name="phone") + locale = sgqlc.types.Field(String, graphql_name="locale") + external_id = sgqlc.types.Field(String, graphql_name="externalId") + note = sgqlc.types.Field(String, graphql_name="note") + buyer_experience_configuration = sgqlc.types.Field(BuyerExperienceConfigurationInput, graphql_name="buyerExperienceConfiguration") + billing_address = sgqlc.types.Field(CompanyAddressInput, graphql_name="billingAddress") + shipping_address = sgqlc.types.Field(CompanyAddressInput, graphql_name="shippingAddress") + billing_same_as_shipping = sgqlc.types.Field(Boolean, graphql_name="billingSameAsShipping") + tax_registration_id = sgqlc.types.Field(String, graphql_name="taxRegistrationId") + tax_exemptions = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(TaxExemption)), graphql_name="taxExemptions") + + +class CompanyLocationRoleAssign(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("company_contact_role_id", "company_contact_id") + company_contact_role_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="companyContactRoleId") + company_contact_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="companyContactId") + + +class CompanyLocationUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("name", "phone", "locale", "external_id", "note", "buyer_experience_configuration") + name = sgqlc.types.Field(String, graphql_name="name") + phone = sgqlc.types.Field(String, graphql_name="phone") + locale = sgqlc.types.Field(String, graphql_name="locale") + external_id = sgqlc.types.Field(String, graphql_name="externalId") + note = sgqlc.types.Field(String, graphql_name="note") + buyer_experience_configuration = sgqlc.types.Field(BuyerExperienceConfigurationInput, graphql_name="buyerExperienceConfiguration") + + +class ContextualPricingContext(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("country", "company_location_id") + country = sgqlc.types.Field(CountryCode, graphql_name="country") + company_location_id = sgqlc.types.Field(ID, graphql_name="companyLocationId") + + +class ContextualPublicationContext(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("country", "company_location_id") + country = sgqlc.types.Field(CountryCode, graphql_name="country") + company_location_id = sgqlc.types.Field(ID, graphql_name="companyLocationId") + + +class CountryHarmonizedSystemCodeInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("harmonized_system_code", "country_code") + harmonized_system_code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="harmonizedSystemCode") + country_code = sgqlc.types.Field(sgqlc.types.non_null(CountryCode), graphql_name="countryCode") + + +class CreateMediaInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("original_source", "alt", "media_content_type") + original_source = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="originalSource") + alt = sgqlc.types.Field(String, graphql_name="alt") + media_content_type = sgqlc.types.Field(sgqlc.types.non_null(MediaContentType), graphql_name="mediaContentType") + + +class CustomShippingPackageInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("weight", "dimensions", "default", "name", "type") + weight = sgqlc.types.Field("WeightInput", graphql_name="weight") + dimensions = sgqlc.types.Field("ObjectDimensionsInput", graphql_name="dimensions") + default = sgqlc.types.Field(Boolean, graphql_name="default") + name = sgqlc.types.Field(String, graphql_name="name") + type = sgqlc.types.Field(ShippingPackageType, graphql_name="type") + + +class CustomerDeleteInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id",) + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + +class CustomerEmailMarketingConsentInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("marketing_opt_in_level", "marketing_state", "consent_updated_at") + marketing_opt_in_level = sgqlc.types.Field(CustomerMarketingOptInLevel, graphql_name="marketingOptInLevel") + marketing_state = sgqlc.types.Field(sgqlc.types.non_null(CustomerEmailMarketingState), graphql_name="marketingState") + consent_updated_at = sgqlc.types.Field(DateTime, graphql_name="consentUpdatedAt") + + +class CustomerEmailMarketingConsentUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("customer_id", "email_marketing_consent") + customer_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="customerId") + email_marketing_consent = sgqlc.types.Field( + sgqlc.types.non_null(CustomerEmailMarketingConsentInput), graphql_name="emailMarketingConsent" + ) + + +class CustomerInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "addresses", + "email", + "first_name", + "id", + "last_name", + "locale", + "metafields", + "note", + "phone", + "tags", + "email_marketing_consent", + "sms_marketing_consent", + "tax_exempt", + "tax_exemptions", + ) + addresses = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("MailingAddressInput")), graphql_name="addresses") + email = sgqlc.types.Field(String, graphql_name="email") + first_name = sgqlc.types.Field(String, graphql_name="firstName") + id = sgqlc.types.Field(ID, graphql_name="id") + last_name = sgqlc.types.Field(String, graphql_name="lastName") + locale = sgqlc.types.Field(String, graphql_name="locale") + metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldInput")), graphql_name="metafields") + note = sgqlc.types.Field(String, graphql_name="note") + phone = sgqlc.types.Field(String, graphql_name="phone") + tags = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="tags") + email_marketing_consent = sgqlc.types.Field(CustomerEmailMarketingConsentInput, graphql_name="emailMarketingConsent") + sms_marketing_consent = sgqlc.types.Field("CustomerSmsMarketingConsentInput", graphql_name="smsMarketingConsent") + tax_exempt = sgqlc.types.Field(Boolean, graphql_name="taxExempt") + tax_exemptions = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(TaxExemption)), graphql_name="taxExemptions") + + +class CustomerMergeOverrideFields(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "customer_id_of_first_name_to_keep", + "customer_id_of_last_name_to_keep", + "customer_id_of_email_to_keep", + "customer_id_of_phone_number_to_keep", + "customer_id_of_default_address_to_keep", + "note", + "tags", + ) + customer_id_of_first_name_to_keep = sgqlc.types.Field(ID, graphql_name="customerIdOfFirstNameToKeep") + customer_id_of_last_name_to_keep = sgqlc.types.Field(ID, graphql_name="customerIdOfLastNameToKeep") + customer_id_of_email_to_keep = sgqlc.types.Field(ID, graphql_name="customerIdOfEmailToKeep") + customer_id_of_phone_number_to_keep = sgqlc.types.Field(ID, graphql_name="customerIdOfPhoneNumberToKeep") + customer_id_of_default_address_to_keep = sgqlc.types.Field(ID, graphql_name="customerIdOfDefaultAddressToKeep") + note = sgqlc.types.Field(String, graphql_name="note") + tags = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="tags") + + +class CustomerPaymentMethodRemoteInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("stripe_payment_method", "authorize_net_customer_payment_profile", "braintree_payment_method") + stripe_payment_method = sgqlc.types.Field("RemoteStripePaymentMethodInput", graphql_name="stripePaymentMethod") + authorize_net_customer_payment_profile = sgqlc.types.Field( + "RemoteAuthorizeNetCustomerPaymentProfileInput", graphql_name="authorizeNetCustomerPaymentProfile" + ) + braintree_payment_method = sgqlc.types.Field("RemoteBraintreePaymentMethodInput", graphql_name="braintreePaymentMethod") + + +class CustomerSegmentMembersQueryInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("segment_id", "query", "reverse", "sort_key") + segment_id = sgqlc.types.Field(ID, graphql_name="segmentId") + query = sgqlc.types.Field(String, graphql_name="query") + reverse = sgqlc.types.Field(Boolean, graphql_name="reverse") + sort_key = sgqlc.types.Field(String, graphql_name="sortKey") + + +class CustomerSmsMarketingConsentInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("marketing_opt_in_level", "marketing_state", "consent_updated_at") + marketing_opt_in_level = sgqlc.types.Field(CustomerMarketingOptInLevel, graphql_name="marketingOptInLevel") + marketing_state = sgqlc.types.Field(sgqlc.types.non_null(CustomerSmsMarketingState), graphql_name="marketingState") + consent_updated_at = sgqlc.types.Field(DateTime, graphql_name="consentUpdatedAt") + + +class CustomerSmsMarketingConsentUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("customer_id", "sms_marketing_consent") + customer_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="customerId") + sms_marketing_consent = sgqlc.types.Field(sgqlc.types.non_null(CustomerSmsMarketingConsentInput), graphql_name="smsMarketingConsent") + + +class DelegateAccessTokenInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("delegate_access_scope", "expires_in") + delegate_access_scope = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="delegateAccessScope" + ) + expires_in = sgqlc.types.Field(Int, graphql_name="expiresIn") + + +class DeliveryCountryInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("code", "rest_of_world", "provinces", "include_all_provinces") + code = sgqlc.types.Field(CountryCode, graphql_name="code") + rest_of_world = sgqlc.types.Field(Boolean, graphql_name="restOfWorld") + provinces = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryProvinceInput")), graphql_name="provinces") + include_all_provinces = sgqlc.types.Field(Boolean, graphql_name="includeAllProvinces") + + +class DeliveryCustomizationInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("function_id", "title", "enabled", "metafields") + function_id = sgqlc.types.Field(String, graphql_name="functionId") + title = sgqlc.types.Field(String, graphql_name="title") + enabled = sgqlc.types.Field(Boolean, graphql_name="enabled") + metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldInput")), graphql_name="metafields") + + +class DeliveryLocationGroupZoneInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "name", "countries", "method_definitions_to_create", "method_definitions_to_update") + id = sgqlc.types.Field(ID, graphql_name="id") + name = sgqlc.types.Field(String, graphql_name="name") + countries = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(DeliveryCountryInput)), graphql_name="countries") + method_definitions_to_create = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("DeliveryMethodDefinitionInput")), graphql_name="methodDefinitionsToCreate" + ) + method_definitions_to_update = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("DeliveryMethodDefinitionInput")), graphql_name="methodDefinitionsToUpdate" + ) + + +class DeliveryLocationLocalPickupEnableInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("location_id", "pickup_time", "instructions") + location_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="locationId") + pickup_time = sgqlc.types.Field(sgqlc.types.non_null(DeliveryLocalPickupTime), graphql_name="pickupTime") + instructions = sgqlc.types.Field(String, graphql_name="instructions") + + +class DeliveryMethodDefinitionInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "id", + "name", + "description", + "active", + "rate_definition", + "participant", + "weight_conditions_to_create", + "price_conditions_to_create", + "conditions_to_update", + ) + id = sgqlc.types.Field(ID, graphql_name="id") + name = sgqlc.types.Field(String, graphql_name="name") + description = sgqlc.types.Field(String, graphql_name="description") + active = sgqlc.types.Field(Boolean, graphql_name="active") + rate_definition = sgqlc.types.Field("DeliveryRateDefinitionInput", graphql_name="rateDefinition") + participant = sgqlc.types.Field("DeliveryParticipantInput", graphql_name="participant") + weight_conditions_to_create = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("DeliveryWeightConditionInput")), graphql_name="weightConditionsToCreate" + ) + price_conditions_to_create = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("DeliveryPriceConditionInput")), graphql_name="priceConditionsToCreate" + ) + conditions_to_update = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("DeliveryUpdateConditionInput")), graphql_name="conditionsToUpdate" + ) + + +class DeliveryParticipantInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "carrier_service_id", "fixed_fee", "percentage_of_rate_fee", "participant_services", "adapt_to_new_services") + id = sgqlc.types.Field(ID, graphql_name="id") + carrier_service_id = sgqlc.types.Field(ID, graphql_name="carrierServiceId") + fixed_fee = sgqlc.types.Field("MoneyInput", graphql_name="fixedFee") + percentage_of_rate_fee = sgqlc.types.Field(Float, graphql_name="percentageOfRateFee") + participant_services = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("DeliveryParticipantServiceInput")), graphql_name="participantServices" + ) + adapt_to_new_services = sgqlc.types.Field(Boolean, graphql_name="adaptToNewServices") + + +class DeliveryParticipantServiceInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("name", "active") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + active = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="active") + + +class DeliveryPriceConditionInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("criteria", "operator") + criteria = sgqlc.types.Field("MoneyInput", graphql_name="criteria") + operator = sgqlc.types.Field(DeliveryConditionOperator, graphql_name="operator") + + +class DeliveryProfileInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "name", + "profile_location_groups", + "location_groups_to_create", + "location_groups_to_update", + "location_groups_to_delete", + "variants_to_associate", + "variants_to_dissociate", + "zones_to_delete", + "method_definitions_to_delete", + "conditions_to_delete", + "selling_plan_groups_to_associate", + "selling_plan_groups_to_dissociate", + ) + name = sgqlc.types.Field(String, graphql_name="name") + profile_location_groups = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("DeliveryProfileLocationGroupInput")), graphql_name="profileLocationGroups" + ) + location_groups_to_create = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("DeliveryProfileLocationGroupInput")), graphql_name="locationGroupsToCreate" + ) + location_groups_to_update = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("DeliveryProfileLocationGroupInput")), graphql_name="locationGroupsToUpdate" + ) + location_groups_to_delete = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="locationGroupsToDelete") + variants_to_associate = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="variantsToAssociate") + variants_to_dissociate = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="variantsToDissociate") + zones_to_delete = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="zonesToDelete") + method_definitions_to_delete = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="methodDefinitionsToDelete" + ) + conditions_to_delete = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="conditionsToDelete") + selling_plan_groups_to_associate = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="sellingPlanGroupsToAssociate" + ) + selling_plan_groups_to_dissociate = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="sellingPlanGroupsToDissociate" + ) + + +class DeliveryProfileLocationGroupInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "locations", "locations_to_add", "locations_to_remove", "zones_to_create", "zones_to_update") + id = sgqlc.types.Field(ID, graphql_name="id") + locations = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="locations") + locations_to_add = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="locationsToAdd") + locations_to_remove = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="locationsToRemove") + zones_to_create = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(DeliveryLocationGroupZoneInput)), graphql_name="zonesToCreate" + ) + zones_to_update = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(DeliveryLocationGroupZoneInput)), graphql_name="zonesToUpdate" + ) + + +class DeliveryProvinceInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") + + +class DeliveryRateDefinitionInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "price") + id = sgqlc.types.Field(ID, graphql_name="id") + price = sgqlc.types.Field(sgqlc.types.non_null("MoneyInput"), graphql_name="price") + + +class DeliverySettingInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("legacy_mode_profiles",) + legacy_mode_profiles = sgqlc.types.Field(Boolean, graphql_name="legacyModeProfiles") + + +class DeliveryUpdateConditionInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "criteria", "criteria_unit", "field", "operator") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + criteria = sgqlc.types.Field(Float, graphql_name="criteria") + criteria_unit = sgqlc.types.Field(String, graphql_name="criteriaUnit") + field = sgqlc.types.Field(DeliveryConditionField, graphql_name="field") + operator = sgqlc.types.Field(DeliveryConditionOperator, graphql_name="operator") + + +class DeliveryWeightConditionInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("criteria", "operator") + criteria = sgqlc.types.Field("WeightInput", graphql_name="criteria") + operator = sgqlc.types.Field(DeliveryConditionOperator, graphql_name="operator") + + +class DiscountAmountInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("amount", "applies_on_each_item") + amount = sgqlc.types.Field(Decimal, graphql_name="amount") + applies_on_each_item = sgqlc.types.Field(Boolean, graphql_name="appliesOnEachItem") + + +class DiscountAutomaticAppInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("combines_with", "function_id", "title", "starts_at", "ends_at", "metafields") + combines_with = sgqlc.types.Field("DiscountCombinesWithInput", graphql_name="combinesWith") + function_id = sgqlc.types.Field(String, graphql_name="functionId") + title = sgqlc.types.Field(String, graphql_name="title") + starts_at = sgqlc.types.Field(DateTime, graphql_name="startsAt") + ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") + metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldInput")), graphql_name="metafields") + + +class DiscountAutomaticBasicInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("combines_with", "title", "starts_at", "ends_at", "minimum_requirement", "customer_gets") + combines_with = sgqlc.types.Field("DiscountCombinesWithInput", graphql_name="combinesWith") + title = sgqlc.types.Field(String, graphql_name="title") + starts_at = sgqlc.types.Field(DateTime, graphql_name="startsAt") + ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") + minimum_requirement = sgqlc.types.Field("DiscountMinimumRequirementInput", graphql_name="minimumRequirement") + customer_gets = sgqlc.types.Field("DiscountCustomerGetsInput", graphql_name="customerGets") + + +class DiscountAutomaticBxgyInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("combines_with", "starts_at", "ends_at", "title", "uses_per_order_limit", "customer_buys", "customer_gets") + combines_with = sgqlc.types.Field("DiscountCombinesWithInput", graphql_name="combinesWith") + starts_at = sgqlc.types.Field(DateTime, graphql_name="startsAt") + ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") + title = sgqlc.types.Field(String, graphql_name="title") + uses_per_order_limit = sgqlc.types.Field(UnsignedInt64, graphql_name="usesPerOrderLimit") + customer_buys = sgqlc.types.Field("DiscountCustomerBuysInput", graphql_name="customerBuys") + customer_gets = sgqlc.types.Field("DiscountCustomerGetsInput", graphql_name="customerGets") + + +class DiscountCodeAppInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "combines_with", + "function_id", + "title", + "starts_at", + "ends_at", + "usage_limit", + "applies_once_per_customer", + "customer_selection", + "code", + "metafields", + ) + combines_with = sgqlc.types.Field("DiscountCombinesWithInput", graphql_name="combinesWith") + function_id = sgqlc.types.Field(String, graphql_name="functionId") + title = sgqlc.types.Field(String, graphql_name="title") + starts_at = sgqlc.types.Field(DateTime, graphql_name="startsAt") + ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") + usage_limit = sgqlc.types.Field(Int, graphql_name="usageLimit") + applies_once_per_customer = sgqlc.types.Field(Boolean, graphql_name="appliesOncePerCustomer") + customer_selection = sgqlc.types.Field("DiscountCustomerSelectionInput", graphql_name="customerSelection") + code = sgqlc.types.Field(String, graphql_name="code") + metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldInput")), graphql_name="metafields") + + +class DiscountCodeBasicInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "combines_with", + "title", + "starts_at", + "ends_at", + "usage_limit", + "applies_once_per_customer", + "minimum_requirement", + "customer_gets", + "customer_selection", + "code", + "recurring_cycle_limit", + ) + combines_with = sgqlc.types.Field("DiscountCombinesWithInput", graphql_name="combinesWith") + title = sgqlc.types.Field(String, graphql_name="title") + starts_at = sgqlc.types.Field(DateTime, graphql_name="startsAt") + ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") + usage_limit = sgqlc.types.Field(Int, graphql_name="usageLimit") + applies_once_per_customer = sgqlc.types.Field(Boolean, graphql_name="appliesOncePerCustomer") + minimum_requirement = sgqlc.types.Field("DiscountMinimumRequirementInput", graphql_name="minimumRequirement") + customer_gets = sgqlc.types.Field("DiscountCustomerGetsInput", graphql_name="customerGets") + customer_selection = sgqlc.types.Field("DiscountCustomerSelectionInput", graphql_name="customerSelection") + code = sgqlc.types.Field(String, graphql_name="code") + recurring_cycle_limit = sgqlc.types.Field(Int, graphql_name="recurringCycleLimit") + + +class DiscountCodeBxgyInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "combines_with", + "title", + "starts_at", + "ends_at", + "customer_buys", + "customer_gets", + "customer_selection", + "code", + "usage_limit", + "uses_per_order_limit", + "applies_once_per_customer", + ) + combines_with = sgqlc.types.Field("DiscountCombinesWithInput", graphql_name="combinesWith") + title = sgqlc.types.Field(String, graphql_name="title") + starts_at = sgqlc.types.Field(DateTime, graphql_name="startsAt") + ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") + customer_buys = sgqlc.types.Field("DiscountCustomerBuysInput", graphql_name="customerBuys") + customer_gets = sgqlc.types.Field("DiscountCustomerGetsInput", graphql_name="customerGets") + customer_selection = sgqlc.types.Field("DiscountCustomerSelectionInput", graphql_name="customerSelection") + code = sgqlc.types.Field(String, graphql_name="code") + usage_limit = sgqlc.types.Field(Int, graphql_name="usageLimit") + uses_per_order_limit = sgqlc.types.Field(Int, graphql_name="usesPerOrderLimit") + applies_once_per_customer = sgqlc.types.Field(Boolean, graphql_name="appliesOncePerCustomer") + + +class DiscountCodeFreeShippingInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "combines_with", + "title", + "starts_at", + "ends_at", + "code", + "usage_limit", + "applies_once_per_customer", + "minimum_requirement", + "customer_selection", + "destination", + "maximum_shipping_price", + "recurring_cycle_limit", + "applies_on_one_time_purchase", + "applies_on_subscription", + ) + combines_with = sgqlc.types.Field("DiscountCombinesWithInput", graphql_name="combinesWith") + title = sgqlc.types.Field(String, graphql_name="title") + starts_at = sgqlc.types.Field(DateTime, graphql_name="startsAt") + ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") + code = sgqlc.types.Field(String, graphql_name="code") + usage_limit = sgqlc.types.Field(Int, graphql_name="usageLimit") + applies_once_per_customer = sgqlc.types.Field(Boolean, graphql_name="appliesOncePerCustomer") + minimum_requirement = sgqlc.types.Field("DiscountMinimumRequirementInput", graphql_name="minimumRequirement") + customer_selection = sgqlc.types.Field("DiscountCustomerSelectionInput", graphql_name="customerSelection") + destination = sgqlc.types.Field("DiscountShippingDestinationSelectionInput", graphql_name="destination") + maximum_shipping_price = sgqlc.types.Field(Decimal, graphql_name="maximumShippingPrice") + recurring_cycle_limit = sgqlc.types.Field(Int, graphql_name="recurringCycleLimit") + applies_on_one_time_purchase = sgqlc.types.Field(Boolean, graphql_name="appliesOnOneTimePurchase") + applies_on_subscription = sgqlc.types.Field(Boolean, graphql_name="appliesOnSubscription") + + +class DiscountCollectionsInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("add", "remove") + add = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="add") + remove = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="remove") + + +class DiscountCombinesWithInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("product_discounts", "order_discounts", "shipping_discounts") + product_discounts = sgqlc.types.Field(Boolean, graphql_name="productDiscounts") + order_discounts = sgqlc.types.Field(Boolean, graphql_name="orderDiscounts") + shipping_discounts = sgqlc.types.Field(Boolean, graphql_name="shippingDiscounts") + + +class DiscountCountriesInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("add", "remove", "include_rest_of_world") + add = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(CountryCode)), graphql_name="add") + remove = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(CountryCode)), graphql_name="remove") + include_rest_of_world = sgqlc.types.Field(Boolean, graphql_name="includeRestOfWorld") + + +class DiscountCustomerBuysInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("value", "items") + value = sgqlc.types.Field("DiscountCustomerBuysValueInput", graphql_name="value") + items = sgqlc.types.Field("DiscountItemsInput", graphql_name="items") + + +class DiscountCustomerBuysValueInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("quantity", "amount") + quantity = sgqlc.types.Field(UnsignedInt64, graphql_name="quantity") + amount = sgqlc.types.Field(Decimal, graphql_name="amount") + + +class DiscountCustomerGetsInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("value", "items", "applies_on_one_time_purchase", "applies_on_subscription") + value = sgqlc.types.Field("DiscountCustomerGetsValueInput", graphql_name="value") + items = sgqlc.types.Field("DiscountItemsInput", graphql_name="items") + applies_on_one_time_purchase = sgqlc.types.Field(Boolean, graphql_name="appliesOnOneTimePurchase") + applies_on_subscription = sgqlc.types.Field(Boolean, graphql_name="appliesOnSubscription") + + +class DiscountCustomerGetsValueInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("discount_on_quantity", "percentage", "discount_amount") + discount_on_quantity = sgqlc.types.Field("DiscountOnQuantityInput", graphql_name="discountOnQuantity") + percentage = sgqlc.types.Field(Float, graphql_name="percentage") + discount_amount = sgqlc.types.Field(DiscountAmountInput, graphql_name="discountAmount") + + +class DiscountCustomerSegmentsInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("add", "remove") + add = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="add") + remove = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="remove") + + +class DiscountCustomerSelectionInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("all", "customers", "customer_segments") + all = sgqlc.types.Field(Boolean, graphql_name="all") + customers = sgqlc.types.Field("DiscountCustomersInput", graphql_name="customers") + customer_segments = sgqlc.types.Field(DiscountCustomerSegmentsInput, graphql_name="customerSegments") + + +class DiscountCustomersInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("add", "remove") + add = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="add") + remove = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="remove") + + +class DiscountEffectInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("percentage",) + percentage = sgqlc.types.Field(Float, graphql_name="percentage") + + +class DiscountItemsInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("products", "collections", "all") + products = sgqlc.types.Field("DiscountProductsInput", graphql_name="products") + collections = sgqlc.types.Field(DiscountCollectionsInput, graphql_name="collections") + all = sgqlc.types.Field(Boolean, graphql_name="all") + + +class DiscountMinimumQuantityInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("greater_than_or_equal_to_quantity",) + greater_than_or_equal_to_quantity = sgqlc.types.Field(UnsignedInt64, graphql_name="greaterThanOrEqualToQuantity") + + +class DiscountMinimumRequirementInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("quantity", "subtotal") + quantity = sgqlc.types.Field(DiscountMinimumQuantityInput, graphql_name="quantity") + subtotal = sgqlc.types.Field("DiscountMinimumSubtotalInput", graphql_name="subtotal") + + +class DiscountMinimumSubtotalInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("greater_than_or_equal_to_subtotal",) + greater_than_or_equal_to_subtotal = sgqlc.types.Field(Decimal, graphql_name="greaterThanOrEqualToSubtotal") + + +class DiscountOnQuantityInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("quantity", "effect") + quantity = sgqlc.types.Field(UnsignedInt64, graphql_name="quantity") + effect = sgqlc.types.Field(DiscountEffectInput, graphql_name="effect") + + +class DiscountProductsInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("products_to_add", "products_to_remove", "product_variants_to_add", "product_variants_to_remove") + products_to_add = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="productsToAdd") + products_to_remove = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="productsToRemove") + product_variants_to_add = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="productVariantsToAdd") + product_variants_to_remove = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="productVariantsToRemove") + + +class DiscountRedeemCodeInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") + + +class DiscountShippingDestinationSelectionInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("all", "countries") + all = sgqlc.types.Field(Boolean, graphql_name="all") + countries = sgqlc.types.Field(DiscountCountriesInput, graphql_name="countries") + + +class DraftOrderAppliedDiscountInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("amount", "description", "title", "value", "value_type") + amount = sgqlc.types.Field(Money, graphql_name="amount") + description = sgqlc.types.Field(String, graphql_name="description") + title = sgqlc.types.Field(String, graphql_name="title") + value = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="value") + value_type = sgqlc.types.Field(sgqlc.types.non_null(DraftOrderAppliedDiscountType), graphql_name="valueType") + + +class DraftOrderDeleteInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id",) + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + +class DraftOrderInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "applied_discount", + "billing_address", + "custom_attributes", + "email", + "line_items", + "metafields", + "localization_extensions", + "note", + "shipping_address", + "shipping_line", + "tags", + "tax_exempt", + "use_customer_default_address", + "visible_to_customer", + "reserve_inventory_until", + "presentment_currency_code", + "market_region_country_code", + "phone", + "payment_terms", + "purchasing_entity", + "source_name", + "po_number", + ) + applied_discount = sgqlc.types.Field(DraftOrderAppliedDiscountInput, graphql_name="appliedDiscount") + billing_address = sgqlc.types.Field("MailingAddressInput", graphql_name="billingAddress") + custom_attributes = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(AttributeInput)), graphql_name="customAttributes") + email = sgqlc.types.Field(String, graphql_name="email") + line_items = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("DraftOrderLineItemInput")), graphql_name="lineItems") + metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldInput")), graphql_name="metafields") + localization_extensions = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("LocalizationExtensionInput")), graphql_name="localizationExtensions" + ) + note = sgqlc.types.Field(String, graphql_name="note") + shipping_address = sgqlc.types.Field("MailingAddressInput", graphql_name="shippingAddress") + shipping_line = sgqlc.types.Field("ShippingLineInput", graphql_name="shippingLine") + tags = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="tags") + tax_exempt = sgqlc.types.Field(Boolean, graphql_name="taxExempt") + use_customer_default_address = sgqlc.types.Field(Boolean, graphql_name="useCustomerDefaultAddress") + visible_to_customer = sgqlc.types.Field(Boolean, graphql_name="visibleToCustomer") + reserve_inventory_until = sgqlc.types.Field(DateTime, graphql_name="reserveInventoryUntil") + presentment_currency_code = sgqlc.types.Field(CurrencyCode, graphql_name="presentmentCurrencyCode") + market_region_country_code = sgqlc.types.Field(CountryCode, graphql_name="marketRegionCountryCode") + phone = sgqlc.types.Field(String, graphql_name="phone") + payment_terms = sgqlc.types.Field("PaymentTermsInput", graphql_name="paymentTerms") + purchasing_entity = sgqlc.types.Field("PurchasingEntityInput", graphql_name="purchasingEntity") + source_name = sgqlc.types.Field(String, graphql_name="sourceName") + po_number = sgqlc.types.Field(String, graphql_name="poNumber") + + +class DraftOrderLineItemInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "applied_discount", + "custom_attributes", + "original_unit_price", + "quantity", + "requires_shipping", + "sku", + "taxable", + "title", + "variant_id", + "weight", + ) + applied_discount = sgqlc.types.Field(DraftOrderAppliedDiscountInput, graphql_name="appliedDiscount") + custom_attributes = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(AttributeInput)), graphql_name="customAttributes") + original_unit_price = sgqlc.types.Field(Money, graphql_name="originalUnitPrice") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + requires_shipping = sgqlc.types.Field(Boolean, graphql_name="requiresShipping") + sku = sgqlc.types.Field(String, graphql_name="sku") + taxable = sgqlc.types.Field(Boolean, graphql_name="taxable") + title = sgqlc.types.Field(String, graphql_name="title") + variant_id = sgqlc.types.Field(ID, graphql_name="variantId") + weight = sgqlc.types.Field("WeightInput", graphql_name="weight") + + +class EmailInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("subject", "to", "from_", "body", "bcc", "custom_message") + subject = sgqlc.types.Field(String, graphql_name="subject") + to = sgqlc.types.Field(String, graphql_name="to") + from_ = sgqlc.types.Field(String, graphql_name="from") + body = sgqlc.types.Field(String, graphql_name="body") + bcc = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="bcc") + custom_message = sgqlc.types.Field(String, graphql_name="customMessage") + + +class EventBridgeWebhookSubscriptionInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("arn", "format", "include_fields", "metafield_namespaces") + arn = sgqlc.types.Field(ARN, graphql_name="arn") + format = sgqlc.types.Field(WebhookSubscriptionFormat, graphql_name="format") + include_fields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="includeFields") + metafield_namespaces = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="metafieldNamespaces") + + +class FileCreateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("filename", "original_source", "content_type", "alt", "duplicate_resolution_mode") + filename = sgqlc.types.Field(String, graphql_name="filename") + original_source = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="originalSource") + content_type = sgqlc.types.Field(FileContentType, graphql_name="contentType") + alt = sgqlc.types.Field(String, graphql_name="alt") + duplicate_resolution_mode = sgqlc.types.Field(FileCreateInputDuplicateResolutionMode, graphql_name="duplicateResolutionMode") + + +class FileUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "alt", "original_source", "preview_image_source", "filename") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + alt = sgqlc.types.Field(String, graphql_name="alt") + original_source = sgqlc.types.Field(String, graphql_name="originalSource") + preview_image_source = sgqlc.types.Field(String, graphql_name="previewImageSource") + filename = sgqlc.types.Field(String, graphql_name="filename") + + +class FulfillmentEventInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "address1", + "city", + "country", + "estimated_delivery_at", + "happened_at", + "fulfillment_id", + "latitude", + "longitude", + "message", + "province", + "status", + "zip", + ) + address1 = sgqlc.types.Field(String, graphql_name="address1") + city = sgqlc.types.Field(String, graphql_name="city") + country = sgqlc.types.Field(String, graphql_name="country") + estimated_delivery_at = sgqlc.types.Field(DateTime, graphql_name="estimatedDeliveryAt") + happened_at = sgqlc.types.Field(DateTime, graphql_name="happenedAt") + fulfillment_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="fulfillmentId") + latitude = sgqlc.types.Field(Float, graphql_name="latitude") + longitude = sgqlc.types.Field(Float, graphql_name="longitude") + message = sgqlc.types.Field(String, graphql_name="message") + province = sgqlc.types.Field(String, graphql_name="province") + status = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentEventStatus), graphql_name="status") + zip = sgqlc.types.Field(String, graphql_name="zip") + + +class FulfillmentOrderHoldInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("reason", "reason_notes", "notify_merchant", "external_id", "fulfillment_order_line_items") + reason = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentHoldReason), graphql_name="reason") + reason_notes = sgqlc.types.Field(String, graphql_name="reasonNotes") + notify_merchant = sgqlc.types.Field(Boolean, graphql_name="notifyMerchant") + external_id = sgqlc.types.Field(String, graphql_name="externalId") + fulfillment_order_line_items = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderLineItemInput")), graphql_name="fulfillmentOrderLineItems" + ) + + +class FulfillmentOrderLineItemInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "quantity") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + + +class FulfillmentOrderLineItemsInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_order_id", "fulfillment_order_line_items") + fulfillment_order_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="fulfillmentOrderId") + fulfillment_order_line_items = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(FulfillmentOrderLineItemInput)), graphql_name="fulfillmentOrderLineItems" + ) + + +class FulfillmentOrderLineItemsPreparedForPickupInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("line_items_by_fulfillment_order",) + line_items_by_fulfillment_order = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PreparedFulfillmentOrderLineItemsInput"))), + graphql_name="lineItemsByFulfillmentOrder", + ) + + +class FulfillmentOrderMergeInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("merge_intents",) + merge_intents = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderMergeInputMergeIntent"))), + graphql_name="mergeIntents", + ) + + +class FulfillmentOrderMergeInputMergeIntent(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_order_line_items", "fulfillment_order_id") + fulfillment_order_line_items = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(FulfillmentOrderLineItemInput)), graphql_name="fulfillmentOrderLineItems" + ) + fulfillment_order_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="fulfillmentOrderId") + + +class FulfillmentOrderSplitInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_order_line_items", "fulfillment_order_id") + fulfillment_order_line_items = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FulfillmentOrderLineItemInput))), + graphql_name="fulfillmentOrderLineItems", + ) + fulfillment_order_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="fulfillmentOrderId") + + +class FulfillmentOriginAddressInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("address1", "address2", "city", "zip", "province_code", "country_code") + address1 = sgqlc.types.Field(String, graphql_name="address1") + address2 = sgqlc.types.Field(String, graphql_name="address2") + city = sgqlc.types.Field(String, graphql_name="city") + zip = sgqlc.types.Field(String, graphql_name="zip") + province_code = sgqlc.types.Field(String, graphql_name="provinceCode") + country_code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="countryCode") + + +class FulfillmentTrackingInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("number", "url", "company", "numbers", "urls") + number = sgqlc.types.Field(String, graphql_name="number") + url = sgqlc.types.Field(URL, graphql_name="url") + company = sgqlc.types.Field(String, graphql_name="company") + numbers = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="numbers") + urls = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(URL)), graphql_name="urls") + + +class FulfillmentV2Input(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("tracking_info", "notify_customer", "line_items_by_fulfillment_order", "origin_address") + tracking_info = sgqlc.types.Field(FulfillmentTrackingInput, graphql_name="trackingInfo") + notify_customer = sgqlc.types.Field(Boolean, graphql_name="notifyCustomer") + line_items_by_fulfillment_order = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FulfillmentOrderLineItemsInput))), + graphql_name="lineItemsByFulfillmentOrder", + ) + origin_address = sgqlc.types.Field(FulfillmentOriginAddressInput, graphql_name="originAddress") + + +class GiftCardCreateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("initial_value", "code", "customer_id", "expires_on", "note", "template_suffix") + initial_value = sgqlc.types.Field(sgqlc.types.non_null(Decimal), graphql_name="initialValue") + code = sgqlc.types.Field(String, graphql_name="code") + customer_id = sgqlc.types.Field(ID, graphql_name="customerId") + expires_on = sgqlc.types.Field(Date, graphql_name="expiresOn") + note = sgqlc.types.Field(String, graphql_name="note") + template_suffix = sgqlc.types.Field(String, graphql_name="templateSuffix") + + +class GiftCardUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("note", "expires_on", "customer_id", "template_suffix") + note = sgqlc.types.Field(String, graphql_name="note") + expires_on = sgqlc.types.Field(Date, graphql_name="expiresOn") + customer_id = sgqlc.types.Field(ID, graphql_name="customerId") + template_suffix = sgqlc.types.Field(String, graphql_name="templateSuffix") + + +class ImageInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "alt_text", "src") + id = sgqlc.types.Field(ID, graphql_name="id") + alt_text = sgqlc.types.Field(String, graphql_name="altText") + src = sgqlc.types.Field(String, graphql_name="src") + + +class ImageTransformInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("crop", "max_width", "max_height", "scale", "preferred_content_type") + crop = sgqlc.types.Field(CropRegion, graphql_name="crop") + max_width = sgqlc.types.Field(Int, graphql_name="maxWidth") + max_height = sgqlc.types.Field(Int, graphql_name="maxHeight") + scale = sgqlc.types.Field(Int, graphql_name="scale") + preferred_content_type = sgqlc.types.Field(ImageContentType, graphql_name="preferredContentType") + + +class IncomingRequestLineItemInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_order_line_item_id", "message") + fulfillment_order_line_item_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="fulfillmentOrderLineItemId") + message = sgqlc.types.Field(String, graphql_name="message") + + +class InventoryAdjustItemInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("inventory_item_id", "available_delta") + inventory_item_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="inventoryItemId") + available_delta = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="availableDelta") + + +class InventoryAdjustQuantitiesInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("reason", "name", "reference_document_uri", "changes") + reason = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="reason") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + reference_document_uri = sgqlc.types.Field(String, graphql_name="referenceDocumentUri") + changes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("InventoryChangeInput"))), graphql_name="changes" + ) + + +class InventoryAdjustQuantityInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("inventory_level_id", "available_delta") + inventory_level_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="inventoryLevelId") + available_delta = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="availableDelta") + + +class InventoryBulkToggleActivationInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("location_id", "activate") + location_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="locationId") + activate = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="activate") + + +class InventoryChangeInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("delta", "inventory_item_id", "location_id", "ledger_document_uri") + delta = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="delta") + inventory_item_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="inventoryItemId") + location_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="locationId") + ledger_document_uri = sgqlc.types.Field(String, graphql_name="ledgerDocumentUri") + + +class InventoryItemInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("cost", "tracked") + cost = sgqlc.types.Field(Decimal, graphql_name="cost") + tracked = sgqlc.types.Field(Boolean, graphql_name="tracked") + + +class InventoryItemUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "cost", + "tracked", + "country_code_of_origin", + "province_code_of_origin", + "harmonized_system_code", + "country_harmonized_system_codes", + ) + cost = sgqlc.types.Field(Decimal, graphql_name="cost") + tracked = sgqlc.types.Field(Boolean, graphql_name="tracked") + country_code_of_origin = sgqlc.types.Field(CountryCode, graphql_name="countryCodeOfOrigin") + province_code_of_origin = sgqlc.types.Field(String, graphql_name="provinceCodeOfOrigin") + harmonized_system_code = sgqlc.types.Field(String, graphql_name="harmonizedSystemCode") + country_harmonized_system_codes = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(CountryHarmonizedSystemCodeInput)), graphql_name="countryHarmonizedSystemCodes" + ) + + +class InventoryLevelInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("available_quantity", "location_id") + available_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="availableQuantity") + location_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="locationId") + + +class InventoryMoveQuantitiesInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("reason", "reference_document_uri", "changes") + reason = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="reason") + reference_document_uri = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="referenceDocumentUri") + changes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("InventoryMoveQuantityChange"))), graphql_name="changes" + ) + + +class InventoryMoveQuantityChange(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("inventory_item_id", "quantity", "from_", "to") + inventory_item_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="inventoryItemId") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + from_ = sgqlc.types.Field(sgqlc.types.non_null("InventoryMoveQuantityTerminalInput"), graphql_name="from") + to = sgqlc.types.Field(sgqlc.types.non_null("InventoryMoveQuantityTerminalInput"), graphql_name="to") + + +class InventoryMoveQuantityTerminalInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("location_id", "name", "ledger_document_uri") + location_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="locationId") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + ledger_document_uri = sgqlc.types.Field(String, graphql_name="ledgerDocumentUri") + + +class InventorySetOnHandQuantitiesInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("reason", "reference_document_uri", "set_quantities") + reason = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="reason") + reference_document_uri = sgqlc.types.Field(String, graphql_name="referenceDocumentUri") + set_quantities = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("InventorySetQuantityInput"))), graphql_name="setQuantities" + ) + + +class InventorySetQuantityInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("inventory_item_id", "location_id", "quantity") + inventory_item_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="inventoryItemId") + location_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="locationId") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + + +class LocalizationExtensionInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("key", "value") + key = sgqlc.types.Field(sgqlc.types.non_null(LocalizationExtensionKey), graphql_name="key") + value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") + + +class LocationAddAddressInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("address1", "address2", "city", "phone", "zip", "country_code", "province_code") + address1 = sgqlc.types.Field(String, graphql_name="address1") + address2 = sgqlc.types.Field(String, graphql_name="address2") + city = sgqlc.types.Field(String, graphql_name="city") + phone = sgqlc.types.Field(String, graphql_name="phone") + zip = sgqlc.types.Field(String, graphql_name="zip") + country_code = sgqlc.types.Field(sgqlc.types.non_null(CountryCode), graphql_name="countryCode") + province_code = sgqlc.types.Field(String, graphql_name="provinceCode") + + +class LocationAddInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("name", "address", "fulfills_online_orders", "metafields") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + address = sgqlc.types.Field(sgqlc.types.non_null(LocationAddAddressInput), graphql_name="address") + fulfills_online_orders = sgqlc.types.Field(Boolean, graphql_name="fulfillsOnlineOrders") + metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldInput")), graphql_name="metafields") + + +class LocationEditAddressInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("address1", "address2", "city", "phone", "zip", "country_code", "province_code") + address1 = sgqlc.types.Field(String, graphql_name="address1") + address2 = sgqlc.types.Field(String, graphql_name="address2") + city = sgqlc.types.Field(String, graphql_name="city") + phone = sgqlc.types.Field(String, graphql_name="phone") + zip = sgqlc.types.Field(String, graphql_name="zip") + country_code = sgqlc.types.Field(CountryCode, graphql_name="countryCode") + province_code = sgqlc.types.Field(String, graphql_name="provinceCode") + + +class LocationEditInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("name", "address", "fulfills_online_orders", "metafields") + name = sgqlc.types.Field(String, graphql_name="name") + address = sgqlc.types.Field(LocationEditAddressInput, graphql_name="address") + fulfills_online_orders = sgqlc.types.Field(Boolean, graphql_name="fulfillsOnlineOrders") + metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldInput")), graphql_name="metafields") + + +class MailingAddressInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "address1", + "address2", + "city", + "company", + "country_code", + "first_name", + "last_name", + "phone", + "province_code", + "zip", + ) + address1 = sgqlc.types.Field(String, graphql_name="address1") + address2 = sgqlc.types.Field(String, graphql_name="address2") + city = sgqlc.types.Field(String, graphql_name="city") + company = sgqlc.types.Field(String, graphql_name="company") + country_code = sgqlc.types.Field(CountryCode, graphql_name="countryCode") + first_name = sgqlc.types.Field(String, graphql_name="firstName") + last_name = sgqlc.types.Field(String, graphql_name="lastName") + phone = sgqlc.types.Field(String, graphql_name="phone") + province_code = sgqlc.types.Field(String, graphql_name="provinceCode") + zip = sgqlc.types.Field(String, graphql_name="zip") + + +class MarketCreateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("name", "handle", "enabled", "regions") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + handle = sgqlc.types.Field(String, graphql_name="handle") + enabled = sgqlc.types.Field(Boolean, graphql_name="enabled") + regions = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketRegionCreateInput"))), graphql_name="regions" + ) + + +class MarketCurrencySettingsUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("base_currency", "local_currencies") + base_currency = sgqlc.types.Field(CurrencyCode, graphql_name="baseCurrency") + local_currencies = sgqlc.types.Field(Boolean, graphql_name="localCurrencies") + + +class MarketLocalizationRegisterInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("market_id", "key", "value", "market_localizable_content_digest") + market_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="marketId") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") + market_localizable_content_digest = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="marketLocalizableContentDigest") + + +class MarketRegionCreateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("country_code",) + country_code = sgqlc.types.Field(sgqlc.types.non_null(CountryCode), graphql_name="countryCode") + + +class MarketUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("name", "handle", "enabled") + name = sgqlc.types.Field(String, graphql_name="name") + handle = sgqlc.types.Field(String, graphql_name="handle") + enabled = sgqlc.types.Field(Boolean, graphql_name="enabled") + + +class MarketWebPresenceCreateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("domain_id", "default_locale", "alternate_locales", "subfolder_suffix") + domain_id = sgqlc.types.Field(ID, graphql_name="domainId") + default_locale = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="defaultLocale") + alternate_locales = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="alternateLocales") + subfolder_suffix = sgqlc.types.Field(String, graphql_name="subfolderSuffix") + + +class MarketWebPresenceUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("domain_id", "default_locale", "alternate_locales", "subfolder_suffix") + domain_id = sgqlc.types.Field(ID, graphql_name="domainId") + default_locale = sgqlc.types.Field(String, graphql_name="defaultLocale") + alternate_locales = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="alternateLocales") + subfolder_suffix = sgqlc.types.Field(String, graphql_name="subfolderSuffix") + + +class MarketingActivityBudgetInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("budget_type", "total") + budget_type = sgqlc.types.Field(MarketingBudgetBudgetType, graphql_name="budgetType") + total = sgqlc.types.Field("MoneyInput", graphql_name="total") + + +class MarketingActivityCreateExternalInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "title", + "utm", + "budget", + "ad_spend", + "remote_id", + "remote_url", + "remote_preview_image_url", + "tactic", + "channel", + "referring_domain", + "scheduled_start", + "scheduled_end", + "start", + "end", + ) + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + utm = sgqlc.types.Field(sgqlc.types.non_null("UTMInput"), graphql_name="utm") + budget = sgqlc.types.Field(MarketingActivityBudgetInput, graphql_name="budget") + ad_spend = sgqlc.types.Field("MoneyInput", graphql_name="adSpend") + remote_id = sgqlc.types.Field(String, graphql_name="remoteId") + remote_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="remoteUrl") + remote_preview_image_url = sgqlc.types.Field(URL, graphql_name="remotePreviewImageUrl") + tactic = sgqlc.types.Field(sgqlc.types.non_null(MarketingTactic), graphql_name="tactic") + channel = sgqlc.types.Field(sgqlc.types.non_null(MarketingChannel), graphql_name="channel") + referring_domain = sgqlc.types.Field(String, graphql_name="referringDomain") + scheduled_start = sgqlc.types.Field(DateTime, graphql_name="scheduledStart") + scheduled_end = sgqlc.types.Field(DateTime, graphql_name="scheduledEnd") + start = sgqlc.types.Field(DateTime, graphql_name="start") + end = sgqlc.types.Field(DateTime, graphql_name="end") + + +class MarketingActivityCreateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("marketing_activity_title", "form_data", "marketing_activity_extension_id", "context", "utm", "status", "budget") + marketing_activity_title = sgqlc.types.Field(String, graphql_name="marketingActivityTitle") + form_data = sgqlc.types.Field(String, graphql_name="formData") + marketing_activity_extension_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="marketingActivityExtensionId") + context = sgqlc.types.Field(String, graphql_name="context") + utm = sgqlc.types.Field("UTMInput", graphql_name="utm") + status = sgqlc.types.Field(sgqlc.types.non_null(MarketingActivityStatus), graphql_name="status") + budget = sgqlc.types.Field(MarketingActivityBudgetInput, graphql_name="budget") + + +class MarketingActivityUpdateExternalInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "title", + "utm", + "budget", + "ad_spend", + "remote_url", + "remote_preview_image_url", + "tactic", + "channel", + "referring_domain", + "scheduled_start", + "scheduled_end", + "start", + "end", + ) + title = sgqlc.types.Field(String, graphql_name="title") + utm = sgqlc.types.Field("UTMInput", graphql_name="utm") + budget = sgqlc.types.Field(MarketingActivityBudgetInput, graphql_name="budget") + ad_spend = sgqlc.types.Field("MoneyInput", graphql_name="adSpend") + remote_url = sgqlc.types.Field(URL, graphql_name="remoteUrl") + remote_preview_image_url = sgqlc.types.Field(URL, graphql_name="remotePreviewImageUrl") + tactic = sgqlc.types.Field(MarketingTactic, graphql_name="tactic") + channel = sgqlc.types.Field(MarketingChannel, graphql_name="channel") + referring_domain = sgqlc.types.Field(String, graphql_name="referringDomain") + scheduled_start = sgqlc.types.Field(DateTime, graphql_name="scheduledStart") + scheduled_end = sgqlc.types.Field(DateTime, graphql_name="scheduledEnd") + start = sgqlc.types.Field(DateTime, graphql_name="start") + end = sgqlc.types.Field(DateTime, graphql_name="end") + + +class MarketingActivityUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "id", + "marketing_recommendation_id", + "title", + "budget", + "status", + "target_status", + "form_data", + "utm", + "marketed_resources", + "errors", + ) + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + marketing_recommendation_id = sgqlc.types.Field(ID, graphql_name="marketingRecommendationId") + title = sgqlc.types.Field(String, graphql_name="title") + budget = sgqlc.types.Field(MarketingActivityBudgetInput, graphql_name="budget") + status = sgqlc.types.Field(MarketingActivityStatus, graphql_name="status") + target_status = sgqlc.types.Field(MarketingActivityStatus, graphql_name="targetStatus") + form_data = sgqlc.types.Field(String, graphql_name="formData") + utm = sgqlc.types.Field("UTMInput", graphql_name="utm") + marketed_resources = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="marketedResources") + errors = sgqlc.types.Field(JSON, graphql_name="errors") + + +class MarketingEngagementInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "occurred_on", + "impressions_count", + "views_count", + "clicks_count", + "shares_count", + "favorites_count", + "comments_count", + "unsubscribes_count", + "complaints_count", + "fails_count", + "sends_count", + "unique_views_count", + "unique_clicks_count", + "ad_spend", + "is_cumulative", + "utc_offset", + "fetched_at", + ) + occurred_on = sgqlc.types.Field(sgqlc.types.non_null(Date), graphql_name="occurredOn") + impressions_count = sgqlc.types.Field(Int, graphql_name="impressionsCount") + views_count = sgqlc.types.Field(Int, graphql_name="viewsCount") + clicks_count = sgqlc.types.Field(Int, graphql_name="clicksCount") + shares_count = sgqlc.types.Field(Int, graphql_name="sharesCount") + favorites_count = sgqlc.types.Field(Int, graphql_name="favoritesCount") + comments_count = sgqlc.types.Field(Int, graphql_name="commentsCount") + unsubscribes_count = sgqlc.types.Field(Int, graphql_name="unsubscribesCount") + complaints_count = sgqlc.types.Field(Int, graphql_name="complaintsCount") + fails_count = sgqlc.types.Field(Int, graphql_name="failsCount") + sends_count = sgqlc.types.Field(Int, graphql_name="sendsCount") + unique_views_count = sgqlc.types.Field(Int, graphql_name="uniqueViewsCount") + unique_clicks_count = sgqlc.types.Field(Int, graphql_name="uniqueClicksCount") + ad_spend = sgqlc.types.Field("MoneyInput", graphql_name="adSpend") + is_cumulative = sgqlc.types.Field(Boolean, graphql_name="isCumulative") + utc_offset = sgqlc.types.Field(UtcOffset, graphql_name="utcOffset") + fetched_at = sgqlc.types.Field(DateTime, graphql_name="fetchedAt") + + +class MetafieldAccessInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("admin",) + admin = sgqlc.types.Field(sgqlc.types.non_null(MetafieldAdminAccess), graphql_name="admin") + + +class MetafieldDefinitionInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "namespace", + "key", + "name", + "description", + "owner_type", + "type", + "validations", + "visible_to_storefront_api", + "use_as_collection_condition", + "pin", + "access", + ) + namespace = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="namespace") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + description = sgqlc.types.Field(String, graphql_name="description") + owner_type = sgqlc.types.Field(sgqlc.types.non_null(MetafieldOwnerType), graphql_name="ownerType") + type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="type") + validations = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("MetafieldDefinitionValidationInput")), graphql_name="validations" + ) + visible_to_storefront_api = sgqlc.types.Field(Boolean, graphql_name="visibleToStorefrontApi") + use_as_collection_condition = sgqlc.types.Field(Boolean, graphql_name="useAsCollectionCondition") + pin = sgqlc.types.Field(Boolean, graphql_name="pin") + access = sgqlc.types.Field(MetafieldAccessInput, graphql_name="access") + + +class MetafieldDefinitionUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "namespace", + "key", + "name", + "description", + "owner_type", + "validations", + "pin", + "visible_to_storefront_api", + "use_as_collection_condition", + "access", + ) + namespace = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="namespace") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + name = sgqlc.types.Field(String, graphql_name="name") + description = sgqlc.types.Field(String, graphql_name="description") + owner_type = sgqlc.types.Field(sgqlc.types.non_null(MetafieldOwnerType), graphql_name="ownerType") + validations = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("MetafieldDefinitionValidationInput")), graphql_name="validations" + ) + pin = sgqlc.types.Field(Boolean, graphql_name="pin") + visible_to_storefront_api = sgqlc.types.Field(Boolean, graphql_name="visibleToStorefrontApi") + use_as_collection_condition = sgqlc.types.Field(Boolean, graphql_name="useAsCollectionCondition") + access = sgqlc.types.Field(MetafieldAccessInput, graphql_name="access") + + +class MetafieldDefinitionValidationInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("name", "value") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") + + +class MetafieldDeleteInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id",) + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + +class MetafieldInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "namespace", "key", "value", "type") + id = sgqlc.types.Field(ID, graphql_name="id") + namespace = sgqlc.types.Field(String, graphql_name="namespace") + key = sgqlc.types.Field(String, graphql_name="key") + value = sgqlc.types.Field(String, graphql_name="value") + type = sgqlc.types.Field(String, graphql_name="type") + + +class MetafieldStorefrontVisibilityInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("namespace", "key", "owner_type") + namespace = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="namespace") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + owner_type = sgqlc.types.Field(sgqlc.types.non_null(MetafieldOwnerType), graphql_name="ownerType") + + +class MetafieldsSetInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("owner_id", "namespace", "key", "value", "type") + owner_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="ownerId") + namespace = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="namespace") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") + type = sgqlc.types.Field(String, graphql_name="type") + + +class MetaobjectAccessInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("admin", "storefront") + admin = sgqlc.types.Field(MetaobjectAdminAccess, graphql_name="admin") + storefront = sgqlc.types.Field(MetaobjectStorefrontAccess, graphql_name="storefront") + + +class MetaobjectBulkDeleteWhereCondition(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("type", "ids") + type = sgqlc.types.Field(String, graphql_name="type") + ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="ids") + + +class MetaobjectCapabilityCreateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("publishable", "translatable") + publishable = sgqlc.types.Field("MetaobjectCapabilityPublishableInput", graphql_name="publishable") + translatable = sgqlc.types.Field("MetaobjectCapabilityTranslatableInput", graphql_name="translatable") + + +class MetaobjectCapabilityDataInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("publishable",) + publishable = sgqlc.types.Field("MetaobjectCapabilityDataPublishableInput", graphql_name="publishable") + + +class MetaobjectCapabilityDataPublishableInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("status",) + status = sgqlc.types.Field(sgqlc.types.non_null(MetaobjectStatus), graphql_name="status") + + +class MetaobjectCapabilityPublishableInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("enabled",) + enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="enabled") + + +class MetaobjectCapabilityTranslatableInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("enabled",) + enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="enabled") + + +class MetaobjectCapabilityUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("publishable", "translatable") + publishable = sgqlc.types.Field(MetaobjectCapabilityPublishableInput, graphql_name="publishable") + translatable = sgqlc.types.Field(MetaobjectCapabilityTranslatableInput, graphql_name="translatable") + + +class MetaobjectCreateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("type", "handle", "fields", "capabilities") + type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="type") + handle = sgqlc.types.Field(String, graphql_name="handle") + fields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("MetaobjectFieldInput")), graphql_name="fields") + capabilities = sgqlc.types.Field(MetaobjectCapabilityDataInput, graphql_name="capabilities") + + +class MetaobjectDefinitionCreateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("name", "description", "type", "field_definitions", "access", "display_name_key", "capabilities") + name = sgqlc.types.Field(String, graphql_name="name") + description = sgqlc.types.Field(String, graphql_name="description") + type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="type") + field_definitions = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetaobjectFieldDefinitionCreateInput"))), + graphql_name="fieldDefinitions", + ) + access = sgqlc.types.Field(MetaobjectAccessInput, graphql_name="access") + display_name_key = sgqlc.types.Field(String, graphql_name="displayNameKey") + capabilities = sgqlc.types.Field(MetaobjectCapabilityCreateInput, graphql_name="capabilities") + + +class MetaobjectDefinitionUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("name", "description", "field_definitions", "access", "display_name_key", "reset_field_order", "capabilities") + name = sgqlc.types.Field(String, graphql_name="name") + description = sgqlc.types.Field(String, graphql_name="description") + field_definitions = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("MetaobjectFieldDefinitionOperationInput")), graphql_name="fieldDefinitions" + ) + access = sgqlc.types.Field(MetaobjectAccessInput, graphql_name="access") + display_name_key = sgqlc.types.Field(String, graphql_name="displayNameKey") + reset_field_order = sgqlc.types.Field(Boolean, graphql_name="resetFieldOrder") + capabilities = sgqlc.types.Field(MetaobjectCapabilityUpdateInput, graphql_name="capabilities") + + +class MetaobjectFieldDefinitionCreateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("key", "type", "name", "description", "required", "validations") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="type") + name = sgqlc.types.Field(String, graphql_name="name") + description = sgqlc.types.Field(String, graphql_name="description") + required = sgqlc.types.Field(Boolean, graphql_name="required") + validations = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(MetafieldDefinitionValidationInput)), graphql_name="validations" + ) + + +class MetaobjectFieldDefinitionDeleteInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("key",) + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + + +class MetaobjectFieldDefinitionOperationInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("create", "update", "delete") + create = sgqlc.types.Field(MetaobjectFieldDefinitionCreateInput, graphql_name="create") + update = sgqlc.types.Field("MetaobjectFieldDefinitionUpdateInput", graphql_name="update") + delete = sgqlc.types.Field(MetaobjectFieldDefinitionDeleteInput, graphql_name="delete") + + +class MetaobjectFieldDefinitionUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("key", "name", "description", "required", "validations") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + name = sgqlc.types.Field(String, graphql_name="name") + description = sgqlc.types.Field(String, graphql_name="description") + required = sgqlc.types.Field(Boolean, graphql_name="required") + validations = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(MetafieldDefinitionValidationInput)), graphql_name="validations" + ) + + +class MetaobjectFieldInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("key", "value") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") + + +class MetaobjectHandleInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("type", "handle") + type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="type") + handle = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="handle") + + +class MetaobjectUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("handle", "fields", "capabilities") + handle = sgqlc.types.Field(String, graphql_name="handle") + fields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(MetaobjectFieldInput)), graphql_name="fields") + capabilities = sgqlc.types.Field(MetaobjectCapabilityDataInput, graphql_name="capabilities") + + +class MetaobjectUpsertInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("handle", "fields", "capabilities") + handle = sgqlc.types.Field(String, graphql_name="handle") + fields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(MetaobjectFieldInput)), graphql_name="fields") + capabilities = sgqlc.types.Field(MetaobjectCapabilityDataInput, graphql_name="capabilities") + + +class MoneyInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("amount", "currency_code") + amount = sgqlc.types.Field(sgqlc.types.non_null(Decimal), graphql_name="amount") + currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currencyCode") + + +class MoveInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "new_position") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + new_position = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="newPosition") + + +class ObjectDimensionsInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("length", "width", "height", "unit") + length = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="length") + width = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="width") + height = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="height") + unit = sgqlc.types.Field(sgqlc.types.non_null(LengthUnit), graphql_name="unit") + + +class OrderCaptureInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "parent_transaction_id", "amount", "currency") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + parent_transaction_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="parentTransactionId") + amount = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="amount") + currency = sgqlc.types.Field(CurrencyCode, graphql_name="currency") + + +class OrderCloseInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id",) + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + +class OrderEditAppliedDiscountInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("description", "fixed_value", "percent_value") + description = sgqlc.types.Field(String, graphql_name="description") + fixed_value = sgqlc.types.Field(MoneyInput, graphql_name="fixedValue") + percent_value = sgqlc.types.Field(Float, graphql_name="percentValue") + + +class OrderInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "id", + "email", + "note", + "tags", + "shipping_address", + "custom_attributes", + "metafields", + "localization_extensions", + "po_number", + ) + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + email = sgqlc.types.Field(String, graphql_name="email") + note = sgqlc.types.Field(String, graphql_name="note") + tags = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="tags") + shipping_address = sgqlc.types.Field(MailingAddressInput, graphql_name="shippingAddress") + custom_attributes = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(AttributeInput)), graphql_name="customAttributes") + metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(MetafieldInput)), graphql_name="metafields") + localization_extensions = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(LocalizationExtensionInput)), graphql_name="localizationExtensions" + ) + po_number = sgqlc.types.Field(String, graphql_name="poNumber") + + +class OrderMarkAsPaidInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id",) + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + +class OrderOpenInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id",) + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + +class OrderTransactionInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("amount", "gateway", "kind", "order_id", "parent_id") + amount = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="amount") + gateway = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="gateway") + kind = sgqlc.types.Field(sgqlc.types.non_null(OrderTransactionKind), graphql_name="kind") + order_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="orderId") + parent_id = sgqlc.types.Field(ID, graphql_name="parentId") + + +class PaymentCustomizationInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("function_id", "title", "enabled", "metafields") + function_id = sgqlc.types.Field(String, graphql_name="functionId") + title = sgqlc.types.Field(String, graphql_name="title") + enabled = sgqlc.types.Field(Boolean, graphql_name="enabled") + metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(MetafieldInput)), graphql_name="metafields") + + +class PaymentScheduleInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("issued_at", "due_at") + issued_at = sgqlc.types.Field(DateTime, graphql_name="issuedAt") + due_at = sgqlc.types.Field(DateTime, graphql_name="dueAt") + + +class PaymentTermsCreateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("payment_terms_template_id", "payment_schedules") + payment_terms_template_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="paymentTermsTemplateId") + payment_schedules = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(PaymentScheduleInput)), graphql_name="paymentSchedules") + + +class PaymentTermsDeleteInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("payment_terms_id",) + payment_terms_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="paymentTermsId") + + +class PaymentTermsInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("payment_terms_template_id", "payment_schedules") + payment_terms_template_id = sgqlc.types.Field(ID, graphql_name="paymentTermsTemplateId") + payment_schedules = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(PaymentScheduleInput)), graphql_name="paymentSchedules") + + +class PaymentTermsUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("payment_terms_id", "payment_terms_attributes") + payment_terms_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="paymentTermsId") + payment_terms_attributes = sgqlc.types.Field(sgqlc.types.non_null(PaymentTermsInput), graphql_name="paymentTermsAttributes") + + +class PreparedFulfillmentOrderLineItemsInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_order_id",) + fulfillment_order_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="fulfillmentOrderId") + + +class PriceInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("calculation", "price") + calculation = sgqlc.types.Field(PriceCalculationType, graphql_name="calculation") + price = sgqlc.types.Field(Money, graphql_name="price") + + +class PriceListAdjustmentInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("value", "type") + value = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="value") + type = sgqlc.types.Field(sgqlc.types.non_null(PriceListAdjustmentType), graphql_name="type") + + +class PriceListAdjustmentSettingsInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("compare_at_mode",) + compare_at_mode = sgqlc.types.Field(sgqlc.types.non_null(PriceListCompareAtMode), graphql_name="compareAtMode") + + +class PriceListCreateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("name", "currency", "parent", "catalog_id") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + currency = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currency") + parent = sgqlc.types.Field(sgqlc.types.non_null("PriceListParentCreateInput"), graphql_name="parent") + catalog_id = sgqlc.types.Field(ID, graphql_name="catalogId") + + +class PriceListParentCreateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("adjustment", "settings") + adjustment = sgqlc.types.Field(sgqlc.types.non_null(PriceListAdjustmentInput), graphql_name="adjustment") + settings = sgqlc.types.Field(PriceListAdjustmentSettingsInput, graphql_name="settings") + + +class PriceListParentUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("adjustment", "settings") + adjustment = sgqlc.types.Field(sgqlc.types.non_null(PriceListAdjustmentInput), graphql_name="adjustment") + settings = sgqlc.types.Field(PriceListAdjustmentSettingsInput, graphql_name="settings") + + +class PriceListPriceInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("variant_id", "price", "compare_at_price") + variant_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="variantId") + price = sgqlc.types.Field(sgqlc.types.non_null(MoneyInput), graphql_name="price") + compare_at_price = sgqlc.types.Field(MoneyInput, graphql_name="compareAtPrice") + + +class PriceListProductPriceInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("product_id", "price") + product_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="productId") + price = sgqlc.types.Field(sgqlc.types.non_null(MoneyInput), graphql_name="price") + + +class PriceListUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("name", "currency", "parent", "catalog_id") + name = sgqlc.types.Field(String, graphql_name="name") + currency = sgqlc.types.Field(CurrencyCode, graphql_name="currency") + parent = sgqlc.types.Field(PriceListParentUpdateInput, graphql_name="parent") + catalog_id = sgqlc.types.Field(ID, graphql_name="catalogId") + + +class PriceRuleCustomerSelectionInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("for_all_customers", "segment_ids", "customer_ids_to_add", "customer_ids_to_remove") + for_all_customers = sgqlc.types.Field(Boolean, graphql_name="forAllCustomers") + segment_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="segmentIds") + customer_ids_to_add = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="customerIdsToAdd") + customer_ids_to_remove = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="customerIdsToRemove") + + +class PriceRuleDiscountCodeInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(String, graphql_name="code") + + +class PriceRuleEntitlementToPrerequisiteQuantityRatioInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("entitlement_quantity", "prerequisite_quantity") + entitlement_quantity = sgqlc.types.Field(Int, graphql_name="entitlementQuantity") + prerequisite_quantity = sgqlc.types.Field(Int, graphql_name="prerequisiteQuantity") + + +class PriceRuleInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "combines_with", + "validity_period", + "once_per_customer", + "customer_selection", + "usage_limit", + "title", + "allocation_limit", + "allocation_method", + "value", + "target", + "prerequisite_subtotal_range", + "prerequisite_quantity_range", + "prerequisite_shipping_price_range", + "item_entitlements", + "item_prerequisites", + "shipping_entitlements", + "prerequisite_to_entitlement_quantity_ratio", + ) + combines_with = sgqlc.types.Field(DiscountCombinesWithInput, graphql_name="combinesWith") + validity_period = sgqlc.types.Field("PriceRuleValidityPeriodInput", graphql_name="validityPeriod") + once_per_customer = sgqlc.types.Field(Boolean, graphql_name="oncePerCustomer") + customer_selection = sgqlc.types.Field(PriceRuleCustomerSelectionInput, graphql_name="customerSelection") + usage_limit = sgqlc.types.Field(Int, graphql_name="usageLimit") + title = sgqlc.types.Field(String, graphql_name="title") + allocation_limit = sgqlc.types.Field(Int, graphql_name="allocationLimit") + allocation_method = sgqlc.types.Field(PriceRuleAllocationMethod, graphql_name="allocationMethod") + value = sgqlc.types.Field("PriceRuleValueInput", graphql_name="value") + target = sgqlc.types.Field(PriceRuleTarget, graphql_name="target") + prerequisite_subtotal_range = sgqlc.types.Field("PriceRuleMoneyRangeInput", graphql_name="prerequisiteSubtotalRange") + prerequisite_quantity_range = sgqlc.types.Field("PriceRuleQuantityRangeInput", graphql_name="prerequisiteQuantityRange") + prerequisite_shipping_price_range = sgqlc.types.Field("PriceRuleMoneyRangeInput", graphql_name="prerequisiteShippingPriceRange") + item_entitlements = sgqlc.types.Field("PriceRuleItemEntitlementsInput", graphql_name="itemEntitlements") + item_prerequisites = sgqlc.types.Field("PriceRuleItemPrerequisitesInput", graphql_name="itemPrerequisites") + shipping_entitlements = sgqlc.types.Field("PriceRuleShippingEntitlementsInput", graphql_name="shippingEntitlements") + prerequisite_to_entitlement_quantity_ratio = sgqlc.types.Field( + "PriceRulePrerequisiteToEntitlementQuantityRatioInput", graphql_name="prerequisiteToEntitlementQuantityRatio" + ) + + +class PriceRuleItemEntitlementsInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("target_all_line_items", "product_ids", "product_variant_ids", "collection_ids") + target_all_line_items = sgqlc.types.Field(Boolean, graphql_name="targetAllLineItems") + product_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="productIds") + product_variant_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="productVariantIds") + collection_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="collectionIds") + + +class PriceRuleItemPrerequisitesInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("product_ids", "product_variant_ids", "collection_ids") + product_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="productIds") + product_variant_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="productVariantIds") + collection_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="collectionIds") + + +class PriceRuleMoneyRangeInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("less_than", "less_than_or_equal_to", "greater_than", "greater_than_or_equal_to") + less_than = sgqlc.types.Field(Money, graphql_name="lessThan") + less_than_or_equal_to = sgqlc.types.Field(Money, graphql_name="lessThanOrEqualTo") + greater_than = sgqlc.types.Field(Money, graphql_name="greaterThan") + greater_than_or_equal_to = sgqlc.types.Field(Money, graphql_name="greaterThanOrEqualTo") + + +class PriceRulePrerequisiteToEntitlementQuantityRatioInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("entitlement_quantity", "prerequisite_quantity") + entitlement_quantity = sgqlc.types.Field(Int, graphql_name="entitlementQuantity") + prerequisite_quantity = sgqlc.types.Field(Int, graphql_name="prerequisiteQuantity") + + +class PriceRuleQuantityRangeInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("less_than", "less_than_or_equal_to", "greater_than", "greater_than_or_equal_to") + less_than = sgqlc.types.Field(Int, graphql_name="lessThan") + less_than_or_equal_to = sgqlc.types.Field(Int, graphql_name="lessThanOrEqualTo") + greater_than = sgqlc.types.Field(Int, graphql_name="greaterThan") + greater_than_or_equal_to = sgqlc.types.Field(Int, graphql_name="greaterThanOrEqualTo") + + +class PriceRuleShippingEntitlementsInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("target_all_shipping_lines", "country_codes", "include_rest_of_world") + target_all_shipping_lines = sgqlc.types.Field(Boolean, graphql_name="targetAllShippingLines") + country_codes = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(CountryCode)), graphql_name="countryCodes") + include_rest_of_world = sgqlc.types.Field(Boolean, graphql_name="includeRestOfWorld") + + +class PriceRuleValidityPeriodInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("start", "end") + start = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="start") + end = sgqlc.types.Field(DateTime, graphql_name="end") + + +class PriceRuleValueInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("percentage_value", "fixed_amount_value") + percentage_value = sgqlc.types.Field(Float, graphql_name="percentageValue") + fixed_amount_value = sgqlc.types.Field(Money, graphql_name="fixedAmountValue") + + +class PrivateMetafieldDeleteInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("owner", "namespace", "key") + owner = sgqlc.types.Field(ID, graphql_name="owner") + namespace = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="namespace") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + + +class PrivateMetafieldInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("owner", "namespace", "key", "value_input") + owner = sgqlc.types.Field(ID, graphql_name="owner") + namespace = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="namespace") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + value_input = sgqlc.types.Field(sgqlc.types.non_null("PrivateMetafieldValueInput"), graphql_name="valueInput") + + +class PrivateMetafieldValueInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("value", "value_type") + value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") + value_type = sgqlc.types.Field(sgqlc.types.non_null(PrivateMetafieldValueType), graphql_name="valueType") + + +class ProductAppendImagesInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "images") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + images = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ImageInput))), graphql_name="images") + + +class ProductCategoryInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("product_taxonomy_node_id",) + product_taxonomy_node_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="productTaxonomyNodeId") + + +class ProductDeleteInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id",) + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + +class ProductDuplicateAsyncInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("product_id", "new_title", "new_status", "include_images") + product_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="productId") + new_title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="newTitle") + new_status = sgqlc.types.Field(ProductStatus, graphql_name="newStatus") + include_images = sgqlc.types.Field(Boolean, graphql_name="includeImages") + + +class ProductFeedInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("language", "country") + language = sgqlc.types.Field(sgqlc.types.non_null(LanguageCode), graphql_name="language") + country = sgqlc.types.Field(sgqlc.types.non_null(CountryCode), graphql_name="country") + + +class ProductInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "description_html", + "handle", + "redirect_new_handle", + "seo", + "product_type", + "standardized_product_type", + "product_category", + "custom_product_type", + "tags", + "template_suffix", + "gift_card", + "gift_card_template_suffix", + "title", + "vendor", + "collections_to_join", + "collections_to_leave", + "id", + "metafields", + "options", + "variants", + "status", + "requires_selling_plan", + ) + description_html = sgqlc.types.Field(String, graphql_name="descriptionHtml") + handle = sgqlc.types.Field(String, graphql_name="handle") + redirect_new_handle = sgqlc.types.Field(Boolean, graphql_name="redirectNewHandle") + seo = sgqlc.types.Field("SEOInput", graphql_name="seo") + product_type = sgqlc.types.Field(String, graphql_name="productType") + standardized_product_type = sgqlc.types.Field("StandardizedProductTypeInput", graphql_name="standardizedProductType") + product_category = sgqlc.types.Field(ProductCategoryInput, graphql_name="productCategory") + custom_product_type = sgqlc.types.Field(String, graphql_name="customProductType") + tags = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="tags") + template_suffix = sgqlc.types.Field(String, graphql_name="templateSuffix") + gift_card = sgqlc.types.Field(Boolean, graphql_name="giftCard") + gift_card_template_suffix = sgqlc.types.Field(String, graphql_name="giftCardTemplateSuffix") + title = sgqlc.types.Field(String, graphql_name="title") + vendor = sgqlc.types.Field(String, graphql_name="vendor") + collections_to_join = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="collectionsToJoin") + collections_to_leave = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="collectionsToLeave") + id = sgqlc.types.Field(ID, graphql_name="id") + metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(MetafieldInput)), graphql_name="metafields") + options = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="options") + variants = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariantInput")), graphql_name="variants") + status = sgqlc.types.Field(ProductStatus, graphql_name="status") + requires_selling_plan = sgqlc.types.Field(Boolean, graphql_name="requiresSellingPlan") + + +class ProductPublicationInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("publication_id", "publish_date") + publication_id = sgqlc.types.Field(ID, graphql_name="publicationId") + publish_date = sgqlc.types.Field(DateTime, graphql_name="publishDate") + + +class ProductPublishInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "product_publications") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + product_publications = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProductPublicationInput))), graphql_name="productPublications" + ) + + +class ProductResourceFeedbackInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("product_id", "state", "feedback_generated_at", "product_updated_at", "messages") + product_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="productId") + state = sgqlc.types.Field(sgqlc.types.non_null(ResourceFeedbackState), graphql_name="state") + feedback_generated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="feedbackGeneratedAt") + product_updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="productUpdatedAt") + messages = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="messages") + + +class ProductUnpublishInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "product_publications") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + product_publications = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProductPublicationInput))), graphql_name="productPublications" + ) + + +class ProductVariantAppendMediaInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("variant_id", "media_ids") + variant_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="variantId") + media_ids = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="mediaIds") + + +class ProductVariantDetachMediaInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("variant_id", "media_ids") + variant_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="variantId") + media_ids = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="mediaIds") + + +class ProductVariantGroupRelationshipInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "quantity") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + + +class ProductVariantInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "requires_components", + "barcode", + "compare_at_price", + "harmonized_system_code", + "id", + "media_id", + "media_src", + "inventory_policy", + "inventory_quantities", + "inventory_item", + "metafields", + "options", + "position", + "price", + "product_id", + "requires_shipping", + "sku", + "taxable", + "tax_code", + "weight", + "weight_unit", + ) + requires_components = sgqlc.types.Field(Boolean, graphql_name="requiresComponents") + barcode = sgqlc.types.Field(String, graphql_name="barcode") + compare_at_price = sgqlc.types.Field(Money, graphql_name="compareAtPrice") + harmonized_system_code = sgqlc.types.Field(String, graphql_name="harmonizedSystemCode") + id = sgqlc.types.Field(ID, graphql_name="id") + media_id = sgqlc.types.Field(ID, graphql_name="mediaId") + media_src = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="mediaSrc") + inventory_policy = sgqlc.types.Field(ProductVariantInventoryPolicy, graphql_name="inventoryPolicy") + inventory_quantities = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(InventoryLevelInput)), graphql_name="inventoryQuantities" + ) + inventory_item = sgqlc.types.Field(InventoryItemInput, graphql_name="inventoryItem") + metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(MetafieldInput)), graphql_name="metafields") + options = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="options") + position = sgqlc.types.Field(Int, graphql_name="position") + price = sgqlc.types.Field(Money, graphql_name="price") + product_id = sgqlc.types.Field(ID, graphql_name="productId") + requires_shipping = sgqlc.types.Field(Boolean, graphql_name="requiresShipping") + sku = sgqlc.types.Field(String, graphql_name="sku") + taxable = sgqlc.types.Field(Boolean, graphql_name="taxable") + tax_code = sgqlc.types.Field(String, graphql_name="taxCode") + weight = sgqlc.types.Field(Float, graphql_name="weight") + weight_unit = sgqlc.types.Field(WeightUnit, graphql_name="weightUnit") + + +class ProductVariantPositionInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "position") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + position = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="position") + + +class ProductVariantRelationshipUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "parent_product_variant_id", + "parent_product_id", + "product_variant_relationships_to_create", + "product_variant_relationships_to_update", + "product_variant_relationships_to_remove", + "remove_all_product_variant_relationships", + "price_input", + ) + parent_product_variant_id = sgqlc.types.Field(ID, graphql_name="parentProductVariantId") + parent_product_id = sgqlc.types.Field(ID, graphql_name="parentProductId") + product_variant_relationships_to_create = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(ProductVariantGroupRelationshipInput)), graphql_name="productVariantRelationshipsToCreate" + ) + product_variant_relationships_to_update = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(ProductVariantGroupRelationshipInput)), graphql_name="productVariantRelationshipsToUpdate" + ) + product_variant_relationships_to_remove = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="productVariantRelationshipsToRemove" + ) + remove_all_product_variant_relationships = sgqlc.types.Field(Boolean, graphql_name="removeAllProductVariantRelationships") + price_input = sgqlc.types.Field(PriceInput, graphql_name="priceInput") + + +class ProductVariantsBulkInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "barcode", + "compare_at_price", + "harmonized_system_code", + "id", + "media_src", + "inventory_policy", + "inventory_quantities", + "inventory_item", + "media_id", + "metafields", + "options", + "price", + "requires_shipping", + "sku", + "taxable", + "tax_code", + "weight", + "weight_unit", + ) + barcode = sgqlc.types.Field(String, graphql_name="barcode") + compare_at_price = sgqlc.types.Field(Money, graphql_name="compareAtPrice") + harmonized_system_code = sgqlc.types.Field(String, graphql_name="harmonizedSystemCode") + id = sgqlc.types.Field(ID, graphql_name="id") + media_src = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="mediaSrc") + inventory_policy = sgqlc.types.Field(ProductVariantInventoryPolicy, graphql_name="inventoryPolicy") + inventory_quantities = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(InventoryLevelInput)), graphql_name="inventoryQuantities" + ) + inventory_item = sgqlc.types.Field(InventoryItemInput, graphql_name="inventoryItem") + media_id = sgqlc.types.Field(ID, graphql_name="mediaId") + metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(MetafieldInput)), graphql_name="metafields") + options = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="options") + price = sgqlc.types.Field(Money, graphql_name="price") + requires_shipping = sgqlc.types.Field(Boolean, graphql_name="requiresShipping") + sku = sgqlc.types.Field(String, graphql_name="sku") + taxable = sgqlc.types.Field(Boolean, graphql_name="taxable") + tax_code = sgqlc.types.Field(String, graphql_name="taxCode") + weight = sgqlc.types.Field(Float, graphql_name="weight") + weight_unit = sgqlc.types.Field(WeightUnit, graphql_name="weightUnit") + + +class PubSubWebhookSubscriptionInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("pub_sub_project", "pub_sub_topic", "format", "include_fields", "metafield_namespaces") + pub_sub_project = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="pubSubProject") + pub_sub_topic = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="pubSubTopic") + format = sgqlc.types.Field(WebhookSubscriptionFormat, graphql_name="format") + include_fields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="includeFields") + metafield_namespaces = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="metafieldNamespaces") + + +class PublicationCreateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("catalog_id", "default_state", "auto_publish") + catalog_id = sgqlc.types.Field(ID, graphql_name="catalogId") + default_state = sgqlc.types.Field(PublicationCreateInputPublicationDefaultState, graphql_name="defaultState") + auto_publish = sgqlc.types.Field(Boolean, graphql_name="autoPublish") + + +class PublicationInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("publication_id", "publish_date") + publication_id = sgqlc.types.Field(ID, graphql_name="publicationId") + publish_date = sgqlc.types.Field(DateTime, graphql_name="publishDate") + + +class PublicationUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("publishables_to_add", "publishables_to_remove", "auto_publish") + publishables_to_add = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="publishablesToAdd") + publishables_to_remove = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="publishablesToRemove") + auto_publish = sgqlc.types.Field(Boolean, graphql_name="autoPublish") + + +class PurchasingCompanyInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("company_id", "company_contact_id", "company_location_id") + company_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="companyId") + company_contact_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="companyContactId") + company_location_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="companyLocationId") + + +class PurchasingEntityInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("customer_id", "purchasing_company") + customer_id = sgqlc.types.Field(ID, graphql_name="customerId") + purchasing_company = sgqlc.types.Field(PurchasingCompanyInput, graphql_name="purchasingCompany") + + +class QuantityRuleInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("increment", "maximum", "minimum", "variant_id") + increment = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="increment") + maximum = sgqlc.types.Field(Int, graphql_name="maximum") + minimum = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="minimum") + variant_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="variantId") + + +class RefundDutyInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("duty_id", "refund_type") + duty_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="dutyId") + refund_type = sgqlc.types.Field(RefundDutyRefundType, graphql_name="refundType") + + +class RefundInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("currency", "order_id", "note", "notify", "shipping", "refund_line_items", "refund_duties", "transactions") + currency = sgqlc.types.Field(CurrencyCode, graphql_name="currency") + order_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="orderId") + note = sgqlc.types.Field(String, graphql_name="note") + notify = sgqlc.types.Field(Boolean, graphql_name="notify") + shipping = sgqlc.types.Field("ShippingRefundInput", graphql_name="shipping") + refund_line_items = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("RefundLineItemInput")), graphql_name="refundLineItems") + refund_duties = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(RefundDutyInput)), graphql_name="refundDuties") + transactions = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(OrderTransactionInput)), graphql_name="transactions") + + +class RefundLineItemInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("line_item_id", "quantity", "restock_type", "location_id") + line_item_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="lineItemId") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + restock_type = sgqlc.types.Field(RefundLineItemRestockType, graphql_name="restockType") + location_id = sgqlc.types.Field(ID, graphql_name="locationId") + + +class RefundShippingInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("shipping_refund_amount", "full_refund") + shipping_refund_amount = sgqlc.types.Field(MoneyInput, graphql_name="shippingRefundAmount") + full_refund = sgqlc.types.Field(Boolean, graphql_name="fullRefund") + + +class RemoteAuthorizeNetCustomerPaymentProfileInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("customer_profile_id", "customer_payment_profile_id") + customer_profile_id = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="customerProfileId") + customer_payment_profile_id = sgqlc.types.Field(String, graphql_name="customerPaymentProfileId") + + +class RemoteBraintreePaymentMethodInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("customer_id", "payment_method_token") + customer_id = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="customerId") + payment_method_token = sgqlc.types.Field(String, graphql_name="paymentMethodToken") + + +class RemoteStripePaymentMethodInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("customer_id", "payment_method_id") + customer_id = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="customerId") + payment_method_id = sgqlc.types.Field(String, graphql_name="paymentMethodId") + + +class ResourceFeedbackCreateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("feedback_generated_at", "messages", "state") + feedback_generated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="feedbackGeneratedAt") + messages = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="messages") + state = sgqlc.types.Field(sgqlc.types.non_null(ResourceFeedbackState), graphql_name="state") + + +class ReturnApproveRequestInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id",) + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + +class ReturnDeclineRequestInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "decline_reason") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + decline_reason = sgqlc.types.Field(sgqlc.types.non_null(ReturnDeclineReason), graphql_name="declineReason") + + +class ReturnInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("order_id", "return_line_items", "notify_customer", "requested_at") + order_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="orderId") + return_line_items = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReturnLineItemInput"))), graphql_name="returnLineItems" + ) + notify_customer = sgqlc.types.Field(Boolean, graphql_name="notifyCustomer") + requested_at = sgqlc.types.Field(DateTime, graphql_name="requestedAt") + + +class ReturnLineItemInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_line_item_id", "quantity", "return_reason", "return_reason_note") + fulfillment_line_item_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="fulfillmentLineItemId") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + return_reason = sgqlc.types.Field(sgqlc.types.non_null(ReturnReason), graphql_name="returnReason") + return_reason_note = sgqlc.types.Field(String, graphql_name="returnReasonNote") + + +class ReturnRefundInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("return_id", "return_refund_line_items", "refund_shipping", "refund_duties", "order_transactions", "notify_customer") + return_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="returnId") + return_refund_line_items = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReturnRefundLineItemInput"))), graphql_name="returnRefundLineItems" + ) + refund_shipping = sgqlc.types.Field(RefundShippingInput, graphql_name="refundShipping") + refund_duties = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(RefundDutyInput)), graphql_name="refundDuties") + order_transactions = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("ReturnRefundOrderTransactionInput")), graphql_name="orderTransactions" + ) + notify_customer = sgqlc.types.Field(Boolean, graphql_name="notifyCustomer") + + +class ReturnRefundLineItemInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("return_line_item_id", "quantity") + return_line_item_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="returnLineItemId") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + + +class ReturnRefundOrderTransactionInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("transaction_amount", "parent_id") + transaction_amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyInput), graphql_name="transactionAmount") + parent_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="parentId") + + +class ReturnRequestInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("order_id", "return_line_items") + order_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="orderId") + return_line_items = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReturnRequestLineItemInput"))), graphql_name="returnLineItems" + ) + + +class ReturnRequestLineItemInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_line_item_id", "quantity", "return_reason", "customer_note") + fulfillment_line_item_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="fulfillmentLineItemId") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + return_reason = sgqlc.types.Field(sgqlc.types.non_null(ReturnReason), graphql_name="returnReason") + customer_note = sgqlc.types.Field(String, graphql_name="customerNote") + + +class ReverseDeliveryDisposeInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("reverse_delivery_line_item_id", "quantity", "disposition_type", "location_id") + reverse_delivery_line_item_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="reverseDeliveryLineItemId") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + disposition_type = sgqlc.types.Field(sgqlc.types.non_null(ReverseFulfillmentOrderDispositionType), graphql_name="dispositionType") + location_id = sgqlc.types.Field(ID, graphql_name="locationId") + + +class ReverseDeliveryLabelInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("file_url",) + file_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="fileUrl") + + +class ReverseDeliveryLineItemInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("reverse_fulfillment_order_line_item_id", "quantity") + reverse_fulfillment_order_line_item_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="reverseFulfillmentOrderLineItemId") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + + +class ReverseDeliveryTrackingInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("number", "url") + number = sgqlc.types.Field(String, graphql_name="number") + url = sgqlc.types.Field(URL, graphql_name="url") + + +class ReverseFulfillmentOrderDisposeInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("reverse_fulfillment_order_line_item_id", "quantity", "location_id", "disposition_type") + reverse_fulfillment_order_line_item_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="reverseFulfillmentOrderLineItemId") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + location_id = sgqlc.types.Field(ID, graphql_name="locationId") + disposition_type = sgqlc.types.Field(sgqlc.types.non_null(ReverseFulfillmentOrderDispositionType), graphql_name="dispositionType") + + +class SEOInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("title", "description") + title = sgqlc.types.Field(String, graphql_name="title") + description = sgqlc.types.Field(String, graphql_name="description") + + +class SavedSearchCreateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("resource_type", "name", "query") + resource_type = sgqlc.types.Field(sgqlc.types.non_null(SearchResultType), graphql_name="resourceType") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + query = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="query") + + +class SavedSearchDeleteInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id",) + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + +class SavedSearchUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "name", "query") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + name = sgqlc.types.Field(String, graphql_name="name") + query = sgqlc.types.Field(String, graphql_name="query") + + +class ScriptTagInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("src", "display_scope", "cache") + src = sgqlc.types.Field(URL, graphql_name="src") + display_scope = sgqlc.types.Field(ScriptTagDisplayScope, graphql_name="displayScope") + cache = sgqlc.types.Field(Boolean, graphql_name="cache") + + +class SellingPlanAnchorInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("type", "day", "month", "cutoff_day") + type = sgqlc.types.Field(SellingPlanAnchorType, graphql_name="type") + day = sgqlc.types.Field(Int, graphql_name="day") + month = sgqlc.types.Field(Int, graphql_name="month") + cutoff_day = sgqlc.types.Field(Int, graphql_name="cutoffDay") + + +class SellingPlanBillingPolicyInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("fixed", "recurring") + fixed = sgqlc.types.Field("SellingPlanFixedBillingPolicyInput", graphql_name="fixed") + recurring = sgqlc.types.Field("SellingPlanRecurringBillingPolicyInput", graphql_name="recurring") + + +class SellingPlanCheckoutChargeInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("type", "value") + type = sgqlc.types.Field(SellingPlanCheckoutChargeType, graphql_name="type") + value = sgqlc.types.Field("SellingPlanCheckoutChargeValueInput", graphql_name="value") + + +class SellingPlanCheckoutChargeValueInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("percentage", "fixed_value") + percentage = sgqlc.types.Field(Float, graphql_name="percentage") + fixed_value = sgqlc.types.Field(Decimal, graphql_name="fixedValue") + + +class SellingPlanDeliveryPolicyInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("fixed", "recurring") + fixed = sgqlc.types.Field("SellingPlanFixedDeliveryPolicyInput", graphql_name="fixed") + recurring = sgqlc.types.Field("SellingPlanRecurringDeliveryPolicyInput", graphql_name="recurring") + + +class SellingPlanFixedBillingPolicyInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "remaining_balance_charge_trigger", + "remaining_balance_charge_exact_time", + "remaining_balance_charge_time_after_checkout", + "checkout_charge", + ) + remaining_balance_charge_trigger = sgqlc.types.Field( + SellingPlanRemainingBalanceChargeTrigger, graphql_name="remainingBalanceChargeTrigger" + ) + remaining_balance_charge_exact_time = sgqlc.types.Field(DateTime, graphql_name="remainingBalanceChargeExactTime") + remaining_balance_charge_time_after_checkout = sgqlc.types.Field(String, graphql_name="remainingBalanceChargeTimeAfterCheckout") + checkout_charge = sgqlc.types.Field(SellingPlanCheckoutChargeInput, graphql_name="checkoutCharge") + + +class SellingPlanFixedDeliveryPolicyInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("anchors", "fulfillment_trigger", "fulfillment_exact_time", "cutoff", "intent", "pre_anchor_behavior") + anchors = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(SellingPlanAnchorInput)), graphql_name="anchors") + fulfillment_trigger = sgqlc.types.Field(SellingPlanFulfillmentTrigger, graphql_name="fulfillmentTrigger") + fulfillment_exact_time = sgqlc.types.Field(DateTime, graphql_name="fulfillmentExactTime") + cutoff = sgqlc.types.Field(Int, graphql_name="cutoff") + intent = sgqlc.types.Field(SellingPlanFixedDeliveryPolicyIntent, graphql_name="intent") + pre_anchor_behavior = sgqlc.types.Field(SellingPlanFixedDeliveryPolicyPreAnchorBehavior, graphql_name="preAnchorBehavior") + + +class SellingPlanFixedPricingPolicyInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "adjustment_type", "adjustment_value") + id = sgqlc.types.Field(ID, graphql_name="id") + adjustment_type = sgqlc.types.Field(SellingPlanPricingPolicyAdjustmentType, graphql_name="adjustmentType") + adjustment_value = sgqlc.types.Field("SellingPlanPricingPolicyValueInput", graphql_name="adjustmentValue") + + +class SellingPlanGroupInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "name", + "app_id", + "merchant_code", + "description", + "selling_plans_to_create", + "selling_plans_to_update", + "selling_plans_to_delete", + "options", + "position", + ) + name = sgqlc.types.Field(String, graphql_name="name") + app_id = sgqlc.types.Field(String, graphql_name="appId") + merchant_code = sgqlc.types.Field(String, graphql_name="merchantCode") + description = sgqlc.types.Field(String, graphql_name="description") + selling_plans_to_create = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanInput")), graphql_name="sellingPlansToCreate" + ) + selling_plans_to_update = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanInput")), graphql_name="sellingPlansToUpdate" + ) + selling_plans_to_delete = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="sellingPlansToDelete") + options = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="options") + position = sgqlc.types.Field(Int, graphql_name="position") + + +class SellingPlanGroupResourceInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("product_variant_ids", "product_ids") + product_variant_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="productVariantIds") + product_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="productIds") + + +class SellingPlanInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "id", + "name", + "description", + "billing_policy", + "delivery_policy", + "inventory_policy", + "pricing_policies", + "options", + "position", + "category", + ) + id = sgqlc.types.Field(ID, graphql_name="id") + name = sgqlc.types.Field(String, graphql_name="name") + description = sgqlc.types.Field(String, graphql_name="description") + billing_policy = sgqlc.types.Field(SellingPlanBillingPolicyInput, graphql_name="billingPolicy") + delivery_policy = sgqlc.types.Field(SellingPlanDeliveryPolicyInput, graphql_name="deliveryPolicy") + inventory_policy = sgqlc.types.Field("SellingPlanInventoryPolicyInput", graphql_name="inventoryPolicy") + pricing_policies = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanPricingPolicyInput")), graphql_name="pricingPolicies" + ) + options = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="options") + position = sgqlc.types.Field(Int, graphql_name="position") + category = sgqlc.types.Field(SellingPlanCategory, graphql_name="category") + + +class SellingPlanInventoryPolicyInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("reserve",) + reserve = sgqlc.types.Field(SellingPlanReserve, graphql_name="reserve") + + +class SellingPlanPricingPolicyInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("recurring", "fixed") + recurring = sgqlc.types.Field("SellingPlanRecurringPricingPolicyInput", graphql_name="recurring") + fixed = sgqlc.types.Field(SellingPlanFixedPricingPolicyInput, graphql_name="fixed") + + +class SellingPlanPricingPolicyValueInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("percentage", "fixed_value") + percentage = sgqlc.types.Field(Float, graphql_name="percentage") + fixed_value = sgqlc.types.Field(Decimal, graphql_name="fixedValue") + + +class SellingPlanRecurringBillingPolicyInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("interval", "interval_count", "anchors", "min_cycles", "max_cycles") + interval = sgqlc.types.Field(SellingPlanInterval, graphql_name="interval") + interval_count = sgqlc.types.Field(Int, graphql_name="intervalCount") + anchors = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(SellingPlanAnchorInput)), graphql_name="anchors") + min_cycles = sgqlc.types.Field(Int, graphql_name="minCycles") + max_cycles = sgqlc.types.Field(Int, graphql_name="maxCycles") + + +class SellingPlanRecurringDeliveryPolicyInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("interval", "interval_count", "anchors", "cutoff", "intent", "pre_anchor_behavior") + interval = sgqlc.types.Field(SellingPlanInterval, graphql_name="interval") + interval_count = sgqlc.types.Field(Int, graphql_name="intervalCount") + anchors = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(SellingPlanAnchorInput)), graphql_name="anchors") + cutoff = sgqlc.types.Field(Int, graphql_name="cutoff") + intent = sgqlc.types.Field(SellingPlanRecurringDeliveryPolicyIntent, graphql_name="intent") + pre_anchor_behavior = sgqlc.types.Field(SellingPlanRecurringDeliveryPolicyPreAnchorBehavior, graphql_name="preAnchorBehavior") + + +class SellingPlanRecurringPricingPolicyInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "adjustment_type", "adjustment_value", "after_cycle") + id = sgqlc.types.Field(ID, graphql_name="id") + adjustment_type = sgqlc.types.Field(SellingPlanPricingPolicyAdjustmentType, graphql_name="adjustmentType") + adjustment_value = sgqlc.types.Field(SellingPlanPricingPolicyValueInput, graphql_name="adjustmentValue") + after_cycle = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="afterCycle") + + +class ShippingLineInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("price", "shipping_rate_handle", "title") + price = sgqlc.types.Field(Money, graphql_name="price") + shipping_rate_handle = sgqlc.types.Field(String, graphql_name="shippingRateHandle") + title = sgqlc.types.Field(String, graphql_name="title") + + +class ShippingRefundInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("amount", "full_refund") + amount = sgqlc.types.Field(Money, graphql_name="amount") + full_refund = sgqlc.types.Field(Boolean, graphql_name="fullRefund") + + +class ShopLocaleInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("published", "market_web_presence_ids") + published = sgqlc.types.Field(Boolean, graphql_name="published") + market_web_presence_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="marketWebPresenceIds") + + +class ShopPolicyInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("type", "body") + type = sgqlc.types.Field(sgqlc.types.non_null(ShopPolicyType), graphql_name="type") + body = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="body") + + +class ShopifyPaymentsDisputeEvidenceUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "customer_email_address", + "customer_last_name", + "customer_first_name", + "shipping_address", + "uncategorized_text", + "access_activity_log", + "cancellation_policy_disclosure", + "cancellation_rebuttal", + "refund_policy_disclosure", + "refund_refusal_explanation", + "cancellation_policy_file", + "customer_communication_file", + "refund_policy_file", + "shipping_documentation_file", + "uncategorized_file", + "service_documentation_file", + "submit_evidence", + ) + customer_email_address = sgqlc.types.Field(String, graphql_name="customerEmailAddress") + customer_last_name = sgqlc.types.Field(String, graphql_name="customerLastName") + customer_first_name = sgqlc.types.Field(String, graphql_name="customerFirstName") + shipping_address = sgqlc.types.Field(MailingAddressInput, graphql_name="shippingAddress") + uncategorized_text = sgqlc.types.Field(String, graphql_name="uncategorizedText") + access_activity_log = sgqlc.types.Field(String, graphql_name="accessActivityLog") + cancellation_policy_disclosure = sgqlc.types.Field(String, graphql_name="cancellationPolicyDisclosure") + cancellation_rebuttal = sgqlc.types.Field(String, graphql_name="cancellationRebuttal") + refund_policy_disclosure = sgqlc.types.Field(String, graphql_name="refundPolicyDisclosure") + refund_refusal_explanation = sgqlc.types.Field(String, graphql_name="refundRefusalExplanation") + cancellation_policy_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUploadUpdateInput", graphql_name="cancellationPolicyFile") + customer_communication_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUploadUpdateInput", graphql_name="customerCommunicationFile") + refund_policy_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUploadUpdateInput", graphql_name="refundPolicyFile") + shipping_documentation_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUploadUpdateInput", graphql_name="shippingDocumentationFile") + uncategorized_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUploadUpdateInput", graphql_name="uncategorizedFile") + service_documentation_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUploadUpdateInput", graphql_name="serviceDocumentationFile") + submit_evidence = sgqlc.types.Field(Boolean, graphql_name="submitEvidence") + + +class ShopifyPaymentsDisputeFileUploadUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "destroy") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + destroy = sgqlc.types.Field(Boolean, graphql_name="destroy") + + +class StageImageInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("resource", "filename", "mime_type", "http_method") + resource = sgqlc.types.Field(sgqlc.types.non_null(StagedUploadTargetGenerateUploadResource), graphql_name="resource") + filename = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="filename") + mime_type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="mimeType") + http_method = sgqlc.types.Field(StagedUploadHttpMethodType, graphql_name="httpMethod") + + +class StagedUploadInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("resource", "filename", "mime_type", "http_method", "file_size") + resource = sgqlc.types.Field(sgqlc.types.non_null(StagedUploadTargetGenerateUploadResource), graphql_name="resource") + filename = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="filename") + mime_type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="mimeType") + http_method = sgqlc.types.Field(StagedUploadHttpMethodType, graphql_name="httpMethod") + file_size = sgqlc.types.Field(UnsignedInt64, graphql_name="fileSize") + + +class StagedUploadTargetGenerateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("resource", "filename", "mime_type", "http_method", "file_size") + resource = sgqlc.types.Field(sgqlc.types.non_null(StagedUploadTargetGenerateUploadResource), graphql_name="resource") + filename = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="filename") + mime_type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="mimeType") + http_method = sgqlc.types.Field(StagedUploadHttpMethodType, graphql_name="httpMethod") + file_size = sgqlc.types.Field(UnsignedInt64, graphql_name="fileSize") + + +class StandardizedProductTypeInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("product_taxonomy_node_id",) + product_taxonomy_node_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="productTaxonomyNodeId") + + +class StorefrontAccessTokenDeleteInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id",) + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + +class StorefrontAccessTokenInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("title",) + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + + +class SubscriptionAtomicLineInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("line", "discounts") + line = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionLineInput"), graphql_name="line") + discounts = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionAtomicManualDiscountInput")), graphql_name="discounts" + ) + + +class SubscriptionAtomicManualDiscountInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("title", "value", "recurring_cycle_limit") + title = sgqlc.types.Field(String, graphql_name="title") + value = sgqlc.types.Field("SubscriptionManualDiscountValueInput", graphql_name="value") + recurring_cycle_limit = sgqlc.types.Field(Int, graphql_name="recurringCycleLimit") + + +class SubscriptionBillingAttemptInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("idempotency_key", "origin_time", "billing_cycle_selector") + idempotency_key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="idempotencyKey") + origin_time = sgqlc.types.Field(DateTime, graphql_name="originTime") + billing_cycle_selector = sgqlc.types.Field("SubscriptionBillingCycleSelector", graphql_name="billingCycleSelector") + + +class SubscriptionBillingCycleInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("contract_id", "selector") + contract_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="contractId") + selector = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionBillingCycleSelector"), graphql_name="selector") + + +class SubscriptionBillingCycleScheduleEditInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("skip", "billing_date", "reason") + skip = sgqlc.types.Field(Boolean, graphql_name="skip") + billing_date = sgqlc.types.Field(DateTime, graphql_name="billingDate") + reason = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionBillingCycleScheduleEditInputScheduleEditReason), graphql_name="reason") + + +class SubscriptionBillingCycleSelector(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("index", "date") + index = sgqlc.types.Field(Int, graphql_name="index") + date = sgqlc.types.Field(DateTime, graphql_name="date") + + +class SubscriptionBillingCyclesDateRangeSelector(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("start_date", "end_date") + start_date = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="startDate") + end_date = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="endDate") + + +class SubscriptionBillingCyclesIndexRangeSelector(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("start_index", "end_index") + start_index = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="startIndex") + end_index = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="endIndex") + + +class SubscriptionBillingPolicyInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("interval", "interval_count", "min_cycles", "max_cycles", "anchors") + interval = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanInterval), graphql_name="interval") + interval_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="intervalCount") + min_cycles = sgqlc.types.Field(Int, graphql_name="minCycles") + max_cycles = sgqlc.types.Field(Int, graphql_name="maxCycles") + anchors = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(SellingPlanAnchorInput)), graphql_name="anchors") + + +class SubscriptionContractAtomicCreateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("customer_id", "next_billing_date", "currency_code", "contract", "lines", "discount_codes") + customer_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="customerId") + next_billing_date = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="nextBillingDate") + currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currencyCode") + contract = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionDraftInput"), graphql_name="contract") + lines = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SubscriptionAtomicLineInput))), graphql_name="lines" + ) + discount_codes = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="discountCodes") + + +class SubscriptionContractCreateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("customer_id", "next_billing_date", "currency_code", "contract") + customer_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="customerId") + next_billing_date = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="nextBillingDate") + currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currencyCode") + contract = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionDraftInput"), graphql_name="contract") + + +class SubscriptionContractProductChangeInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("product_variant_id", "current_price") + product_variant_id = sgqlc.types.Field(ID, graphql_name="productVariantId") + current_price = sgqlc.types.Field(Decimal, graphql_name="currentPrice") + + +class SubscriptionDeliveryMethodInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("shipping", "local_delivery", "pickup") + shipping = sgqlc.types.Field("SubscriptionDeliveryMethodShippingInput", graphql_name="shipping") + local_delivery = sgqlc.types.Field("SubscriptionDeliveryMethodLocalDeliveryInput", graphql_name="localDelivery") + pickup = sgqlc.types.Field("SubscriptionDeliveryMethodPickupInput", graphql_name="pickup") + + +class SubscriptionDeliveryMethodLocalDeliveryInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("address", "local_delivery_option") + address = sgqlc.types.Field(MailingAddressInput, graphql_name="address") + local_delivery_option = sgqlc.types.Field("SubscriptionDeliveryMethodLocalDeliveryOptionInput", graphql_name="localDeliveryOption") + + +class SubscriptionDeliveryMethodLocalDeliveryOptionInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("title", "presentment_title", "description", "code", "phone", "instructions") + title = sgqlc.types.Field(String, graphql_name="title") + presentment_title = sgqlc.types.Field(String, graphql_name="presentmentTitle") + description = sgqlc.types.Field(String, graphql_name="description") + code = sgqlc.types.Field(String, graphql_name="code") + phone = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="phone") + instructions = sgqlc.types.Field(String, graphql_name="instructions") + + +class SubscriptionDeliveryMethodPickupInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("pickup_option",) + pickup_option = sgqlc.types.Field("SubscriptionDeliveryMethodPickupOptionInput", graphql_name="pickupOption") + + +class SubscriptionDeliveryMethodPickupOptionInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("title", "presentment_title", "description", "code", "location_id") + title = sgqlc.types.Field(String, graphql_name="title") + presentment_title = sgqlc.types.Field(String, graphql_name="presentmentTitle") + description = sgqlc.types.Field(String, graphql_name="description") + code = sgqlc.types.Field(String, graphql_name="code") + location_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="locationId") + + +class SubscriptionDeliveryMethodShippingInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("address", "shipping_option") + address = sgqlc.types.Field(MailingAddressInput, graphql_name="address") + shipping_option = sgqlc.types.Field("SubscriptionDeliveryMethodShippingOptionInput", graphql_name="shippingOption") + + +class SubscriptionDeliveryMethodShippingOptionInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("title", "presentment_title", "description", "code", "carrier_service_id") + title = sgqlc.types.Field(String, graphql_name="title") + presentment_title = sgqlc.types.Field(String, graphql_name="presentmentTitle") + description = sgqlc.types.Field(String, graphql_name="description") + code = sgqlc.types.Field(String, graphql_name="code") + carrier_service_id = sgqlc.types.Field(ID, graphql_name="carrierServiceId") + + +class SubscriptionDeliveryPolicyInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("interval", "interval_count", "anchors") + interval = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanInterval), graphql_name="interval") + interval_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="intervalCount") + anchors = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(SellingPlanAnchorInput)), graphql_name="anchors") + + +class SubscriptionDraftInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "status", + "payment_method_id", + "next_billing_date", + "billing_policy", + "delivery_policy", + "delivery_price", + "delivery_method", + "note", + "custom_attributes", + ) + status = sgqlc.types.Field(SubscriptionContractSubscriptionStatus, graphql_name="status") + payment_method_id = sgqlc.types.Field(ID, graphql_name="paymentMethodId") + next_billing_date = sgqlc.types.Field(DateTime, graphql_name="nextBillingDate") + billing_policy = sgqlc.types.Field(SubscriptionBillingPolicyInput, graphql_name="billingPolicy") + delivery_policy = sgqlc.types.Field(SubscriptionDeliveryPolicyInput, graphql_name="deliveryPolicy") + delivery_price = sgqlc.types.Field(Decimal, graphql_name="deliveryPrice") + delivery_method = sgqlc.types.Field(SubscriptionDeliveryMethodInput, graphql_name="deliveryMethod") + note = sgqlc.types.Field(String, graphql_name="note") + custom_attributes = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(AttributeInput)), graphql_name="customAttributes") + + +class SubscriptionFreeShippingDiscountInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("title", "recurring_cycle_limit") + title = sgqlc.types.Field(String, graphql_name="title") + recurring_cycle_limit = sgqlc.types.Field(Int, graphql_name="recurringCycleLimit") + + +class SubscriptionLineInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "product_variant_id", + "quantity", + "current_price", + "custom_attributes", + "selling_plan_id", + "selling_plan_name", + "pricing_policy", + ) + product_variant_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="productVariantId") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + current_price = sgqlc.types.Field(sgqlc.types.non_null(Decimal), graphql_name="currentPrice") + custom_attributes = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(AttributeInput)), graphql_name="customAttributes") + selling_plan_id = sgqlc.types.Field(ID, graphql_name="sellingPlanId") + selling_plan_name = sgqlc.types.Field(String, graphql_name="sellingPlanName") + pricing_policy = sgqlc.types.Field("SubscriptionPricingPolicyInput", graphql_name="pricingPolicy") + + +class SubscriptionLineUpdateInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ( + "product_variant_id", + "quantity", + "selling_plan_id", + "selling_plan_name", + "current_price", + "custom_attributes", + "pricing_policy", + ) + product_variant_id = sgqlc.types.Field(ID, graphql_name="productVariantId") + quantity = sgqlc.types.Field(Int, graphql_name="quantity") + selling_plan_id = sgqlc.types.Field(ID, graphql_name="sellingPlanId") + selling_plan_name = sgqlc.types.Field(String, graphql_name="sellingPlanName") + current_price = sgqlc.types.Field(Decimal, graphql_name="currentPrice") + custom_attributes = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(AttributeInput)), graphql_name="customAttributes") + pricing_policy = sgqlc.types.Field("SubscriptionPricingPolicyInput", graphql_name="pricingPolicy") + + +class SubscriptionManualDiscountEntitledLinesInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("all", "lines") + all = sgqlc.types.Field(Boolean, graphql_name="all") + lines = sgqlc.types.Field("SubscriptionManualDiscountLinesInput", graphql_name="lines") + + +class SubscriptionManualDiscountFixedAmountInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("amount", "applies_on_each_item") + amount = sgqlc.types.Field(Float, graphql_name="amount") + applies_on_each_item = sgqlc.types.Field(Boolean, graphql_name="appliesOnEachItem") + + +class SubscriptionManualDiscountInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("title", "value", "recurring_cycle_limit", "entitled_lines") + title = sgqlc.types.Field(String, graphql_name="title") + value = sgqlc.types.Field("SubscriptionManualDiscountValueInput", graphql_name="value") + recurring_cycle_limit = sgqlc.types.Field(Int, graphql_name="recurringCycleLimit") + entitled_lines = sgqlc.types.Field(SubscriptionManualDiscountEntitledLinesInput, graphql_name="entitledLines") + + +class SubscriptionManualDiscountLinesInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("add", "remove") + add = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="add") + remove = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="remove") + + +class SubscriptionManualDiscountValueInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("percentage", "fixed_amount") + percentage = sgqlc.types.Field(Int, graphql_name="percentage") + fixed_amount = sgqlc.types.Field(SubscriptionManualDiscountFixedAmountInput, graphql_name="fixedAmount") + + +class SubscriptionPricingPolicyCycleDiscountsInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("after_cycle", "adjustment_type", "adjustment_value", "computed_price") + after_cycle = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="afterCycle") + adjustment_type = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanPricingPolicyAdjustmentType), graphql_name="adjustmentType") + adjustment_value = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanPricingPolicyValueInput), graphql_name="adjustmentValue") + computed_price = sgqlc.types.Field(sgqlc.types.non_null(Decimal), graphql_name="computedPrice") + + +class SubscriptionPricingPolicyInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("base_price", "cycle_discounts") + base_price = sgqlc.types.Field(sgqlc.types.non_null(Decimal), graphql_name="basePrice") + cycle_discounts = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SubscriptionPricingPolicyCycleDiscountsInput))), + graphql_name="cycleDiscounts", + ) + + +class TranslationInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("locale", "key", "value", "translatable_content_digest", "market_id") + locale = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="locale") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") + translatable_content_digest = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="translatableContentDigest") + market_id = sgqlc.types.Field(ID, graphql_name="marketId") + + +class UTMInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("campaign", "source", "medium") + campaign = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="campaign") + source = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="source") + medium = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="medium") + + +class UpdateMediaInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("id", "preview_image_source", "alt") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + preview_image_source = sgqlc.types.Field(String, graphql_name="previewImageSource") + alt = sgqlc.types.Field(String, graphql_name="alt") + + +class UrlRedirectInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("path", "target") + path = sgqlc.types.Field(String, graphql_name="path") + target = sgqlc.types.Field(String, graphql_name="target") + + +class WebPixelInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("settings",) + settings = sgqlc.types.Field(sgqlc.types.non_null(JSON), graphql_name="settings") + + +class WebhookSubscriptionInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("callback_url", "format", "include_fields", "metafield_namespaces") + callback_url = sgqlc.types.Field(URL, graphql_name="callbackUrl") + format = sgqlc.types.Field(WebhookSubscriptionFormat, graphql_name="format") + include_fields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="includeFields") + metafield_namespaces = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="metafieldNamespaces") + + +class WeightInput(sgqlc.types.Input): + __schema__ = shopify_schema + __field_names__ = ("value", "unit") + value = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="value") + unit = sgqlc.types.Field(sgqlc.types.non_null(WeightUnit), graphql_name="unit") + + +######################################################################## +# Output Objects and Interfaces +######################################################################## +class AppPurchase(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("created_at", "name", "price", "status", "test") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + price = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="price") + status = sgqlc.types.Field(sgqlc.types.non_null(AppPurchaseStatus), graphql_name="status") + test = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="test") + + +class CalculatedDiscountApplication(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("allocation_method", "applied_to", "description", "id", "target_selection", "target_type", "value") + allocation_method = sgqlc.types.Field(sgqlc.types.non_null(DiscountApplicationAllocationMethod), graphql_name="allocationMethod") + applied_to = sgqlc.types.Field(sgqlc.types.non_null(DiscountApplicationLevel), graphql_name="appliedTo") + description = sgqlc.types.Field(String, graphql_name="description") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + target_selection = sgqlc.types.Field(sgqlc.types.non_null(DiscountApplicationTargetSelection), graphql_name="targetSelection") + target_type = sgqlc.types.Field(sgqlc.types.non_null(DiscountApplicationTargetType), graphql_name="targetType") + value = sgqlc.types.Field(sgqlc.types.non_null("PricingValue"), graphql_name="value") + + +class CommentEventSubject(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("has_timeline_comment", "id") + has_timeline_comment = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasTimelineComment") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + +class CustomerMoment(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("occurred_at",) + occurred_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="occurredAt") + + +class DiscountApplication(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("allocation_method", "index", "target_selection", "target_type", "value") + allocation_method = sgqlc.types.Field(sgqlc.types.non_null(DiscountApplicationAllocationMethod), graphql_name="allocationMethod") + index = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="index") + target_selection = sgqlc.types.Field(sgqlc.types.non_null(DiscountApplicationTargetSelection), graphql_name="targetSelection") + target_type = sgqlc.types.Field(sgqlc.types.non_null(DiscountApplicationTargetType), graphql_name="targetType") + value = sgqlc.types.Field(sgqlc.types.non_null("PricingValue"), graphql_name="value") + + +class DisplayableError(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("field", "message") + field = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="field") + message = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="message") + + +class Event(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("app_title", "attribute_to_app", "attribute_to_user", "created_at", "critical_alert", "id", "message") + app_title = sgqlc.types.Field(String, graphql_name="appTitle") + attribute_to_app = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="attributeToApp") + attribute_to_user = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="attributeToUser") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + critical_alert = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="criticalAlert") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + message = sgqlc.types.Field(sgqlc.types.non_null(FormattedString), graphql_name="message") + + +class File(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("alt", "created_at", "file_errors", "file_status", "id", "preview", "updated_at") + alt = sgqlc.types.Field(String, graphql_name="alt") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + file_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FileError"))), graphql_name="fileErrors") + file_status = sgqlc.types.Field(sgqlc.types.non_null(FileStatus), graphql_name="fileStatus") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + preview = sgqlc.types.Field("MediaPreviewImage", graphql_name="preview") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + + +class HasEvents(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("events",) + events = sgqlc.types.Field( + sgqlc.types.non_null("EventConnection"), + graphql_name="events", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(EventSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + + +class HasLocalizationExtensions(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("localization_extensions",) + localization_extensions = sgqlc.types.Field( + sgqlc.types.non_null("LocalizationExtensionConnection"), + graphql_name="localizationExtensions", + args=sgqlc.types.ArgDict( + ( + ( + "country_codes", + sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(CountryCode)), graphql_name="countryCodes", default=None), + ), + ( + "purposes", + sgqlc.types.Arg( + sgqlc.types.list_of(sgqlc.types.non_null(LocalizationExtensionPurpose)), graphql_name="purposes", default=None + ), + ), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + + +class HasMetafieldDefinitions(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("metafield_definitions",) + metafield_definitions = sgqlc.types.Field( + sgqlc.types.non_null("MetafieldDefinitionConnection"), + graphql_name="metafieldDefinitions", + args=sgqlc.types.ArgDict( + ( + ("namespace", sgqlc.types.Arg(String, graphql_name="namespace", default=None)), + ("pinned_status", sgqlc.types.Arg(MetafieldDefinitionPinnedStatus, graphql_name="pinnedStatus", default="ANY")), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(MetafieldDefinitionSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + + +class HasMetafields(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("metafield", "metafields") + metafield = sgqlc.types.Field( + "Metafield", + graphql_name="metafield", + args=sgqlc.types.ArgDict( + ( + ("namespace", sgqlc.types.Arg(String, graphql_name="namespace", default=None)), + ("key", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="key", default=None)), + ) + ), + ) + metafields = sgqlc.types.Field( + sgqlc.types.non_null("MetafieldConnection"), + graphql_name="metafields", + args=sgqlc.types.ArgDict( + ( + ("namespace", sgqlc.types.Arg(String, graphql_name="namespace", default=None)), + ("keys", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="keys", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + + +class HasPublishedTranslations(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("translations",) + translations = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Translation"))), + graphql_name="translations", + args=sgqlc.types.ArgDict( + ( + ("locale", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="locale", default=None)), + ("market_id", sgqlc.types.Arg(ID, graphql_name="marketId", default=None)), + ) + ), + ) + + +class JobResult(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("done", "id") + done = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="done") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + +class LegacyInteroperability(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("legacy_resource_id",) + legacy_resource_id = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="legacyResourceId") + + +class MarketRegion(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("id", "name") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + + +class Media(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("alt", "id", "media_content_type", "media_errors", "media_warnings", "preview", "status") + alt = sgqlc.types.Field(String, graphql_name="alt") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + media_content_type = sgqlc.types.Field(sgqlc.types.non_null(MediaContentType), graphql_name="mediaContentType") + media_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MediaError"))), graphql_name="mediaErrors" + ) + media_warnings = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MediaWarning"))), graphql_name="mediaWarnings" + ) + preview = sgqlc.types.Field("MediaPreviewImage", graphql_name="preview") + status = sgqlc.types.Field(sgqlc.types.non_null(MediaStatus), graphql_name="status") + + +class Navigable(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("default_cursor",) + default_cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="defaultCursor") + + +class Node(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("id",) + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + +class OnlineStorePreviewable(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("online_store_preview_url",) + online_store_preview_url = sgqlc.types.Field(URL, graphql_name="onlineStorePreviewUrl") + + +class Publishable(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ( + "available_publication_count", + "publication_count", + "published_on_current_publication", + "published_on_publication", + "resource_publications", + "resource_publications_v2", + "unpublished_publications", + ) + available_publication_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="availablePublicationCount") + publication_count = sgqlc.types.Field( + sgqlc.types.non_null(Int), + graphql_name="publicationCount", + args=sgqlc.types.ArgDict((("only_published", sgqlc.types.Arg(Boolean, graphql_name="onlyPublished", default=True)),)), + ) + published_on_current_publication = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="publishedOnCurrentPublication") + published_on_publication = sgqlc.types.Field( + sgqlc.types.non_null(Boolean), + graphql_name="publishedOnPublication", + args=sgqlc.types.ArgDict( + (("publication_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="publicationId", default=None)),) + ), + ) + resource_publications = sgqlc.types.Field( + sgqlc.types.non_null("ResourcePublicationConnection"), + graphql_name="resourcePublications", + args=sgqlc.types.ArgDict( + ( + ("only_published", sgqlc.types.Arg(Boolean, graphql_name="onlyPublished", default=True)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + resource_publications_v2 = sgqlc.types.Field( + sgqlc.types.non_null("ResourcePublicationV2Connection"), + graphql_name="resourcePublicationsV2", + args=sgqlc.types.ArgDict( + ( + ("only_published", sgqlc.types.Arg(Boolean, graphql_name="onlyPublished", default=True)), + ("catalog_type", sgqlc.types.Arg(CatalogType, graphql_name="catalogType", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + unpublished_publications = sgqlc.types.Field( + sgqlc.types.non_null("PublicationConnection"), + graphql_name="unpublishedPublications", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + + +class ResourceOperation(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("id", "processed_row_count", "row_count", "status") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + processed_row_count = sgqlc.types.Field(Int, graphql_name="processedRowCount") + row_count = sgqlc.types.Field("RowCount", graphql_name="rowCount") + status = sgqlc.types.Field(sgqlc.types.non_null(ResourceOperationStatus), graphql_name="status") + + +class Sale(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ( + "action_type", + "id", + "line_type", + "quantity", + "taxes", + "total_amount", + "total_discount_amount_after_taxes", + "total_discount_amount_before_taxes", + "total_tax_amount", + ) + action_type = sgqlc.types.Field(sgqlc.types.non_null(SaleActionType), graphql_name="actionType") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + line_type = sgqlc.types.Field(sgqlc.types.non_null(SaleLineType), graphql_name="lineType") + quantity = sgqlc.types.Field(Int, graphql_name="quantity") + taxes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SaleTax"))), graphql_name="taxes") + total_amount = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="totalAmount") + total_discount_amount_after_taxes = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="totalDiscountAmountAfterTaxes") + total_discount_amount_before_taxes = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="totalDiscountAmountBeforeTaxes") + total_tax_amount = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="totalTaxAmount") + + +class SalesAgreement(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("app", "happened_at", "id", "reason", "sales", "user") + app = sgqlc.types.Field("App", graphql_name="app") + happened_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="happenedAt") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + reason = sgqlc.types.Field(sgqlc.types.non_null(OrderActionType), graphql_name="reason") + sales = sgqlc.types.Field( + sgqlc.types.non_null("SaleConnection"), + graphql_name="sales", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + user = sgqlc.types.Field("StaffMember", graphql_name="user") + + +class SegmentFilter(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("localized_name", "multi_value", "query_name") + localized_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="localizedName") + multi_value = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="multiValue") + query_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="queryName") + + +class SellingPlanPricingPolicyBase(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("adjustment_type", "adjustment_value") + adjustment_type = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanPricingPolicyAdjustmentType), graphql_name="adjustmentType") + adjustment_value = sgqlc.types.Field(sgqlc.types.non_null("SellingPlanPricingPolicyAdjustmentValue"), graphql_name="adjustmentValue") + + +class ShopifyPaymentsChargeStatementDescriptor(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("default", "prefix") + default = sgqlc.types.Field(String, graphql_name="default") + prefix = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="prefix") + + +class ShopifyqlResponse(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("parse_errors", "table_data") + parse_errors = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("ParseError")), graphql_name="parseErrors") + table_data = sgqlc.types.Field("TableData", graphql_name="tableData") + + +class SubscriptionContractBase(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ( + "app", + "app_admin_url", + "currency_code", + "custom_attributes", + "customer", + "customer_payment_method", + "delivery_method", + "delivery_price", + "discounts", + "line_count", + "lines", + "note", + "orders", + "updated_at", + ) + app = sgqlc.types.Field("App", graphql_name="app") + app_admin_url = sgqlc.types.Field(URL, graphql_name="appAdminUrl") + currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currencyCode") + custom_attributes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Attribute"))), graphql_name="customAttributes" + ) + customer = sgqlc.types.Field("Customer", graphql_name="customer") + customer_payment_method = sgqlc.types.Field( + "CustomerPaymentMethod", + graphql_name="customerPaymentMethod", + args=sgqlc.types.ArgDict((("show_revoked", sgqlc.types.Arg(Boolean, graphql_name="showRevoked", default=False)),)), + ) + delivery_method = sgqlc.types.Field("SubscriptionDeliveryMethod", graphql_name="deliveryMethod") + delivery_price = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="deliveryPrice") + discounts = sgqlc.types.Field( + sgqlc.types.non_null("SubscriptionManualDiscountConnection"), + graphql_name="discounts", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + line_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="lineCount") + lines = sgqlc.types.Field( + sgqlc.types.non_null("SubscriptionLineConnection"), + graphql_name="lines", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + note = sgqlc.types.Field(String, graphql_name="note") + orders = sgqlc.types.Field( + sgqlc.types.non_null("OrderConnection"), + graphql_name="orders", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + + +class Catalog(sgqlc.types.Interface): + __schema__ = shopify_schema + __field_names__ = ("id", "operations", "price_list", "publication", "status", "title") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + operations = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ResourceOperation))), graphql_name="operations" + ) + price_list = sgqlc.types.Field("PriceList", graphql_name="priceList") + publication = sgqlc.types.Field("Publication", graphql_name="publication") + status = sgqlc.types.Field(sgqlc.types.non_null(CatalogStatus), graphql_name="status") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + + +class AbandonmentEmailStateUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("abandonment", "user_errors") + abandonment = sgqlc.types.Field("Abandonment", graphql_name="abandonment") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AbandonmentEmailStateUpdateUserError"))), graphql_name="userErrors" + ) + + +class AbandonmentUpdateActivitiesDeliveryStatusesPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("abandonment", "user_errors") + abandonment = sgqlc.types.Field("Abandonment", graphql_name="abandonment") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AbandonmentUpdateActivitiesDeliveryStatusesUserError"))), + graphql_name="userErrors", + ) + + +class AccessScope(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("description", "handle") + description = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="description") + handle = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="handle") + + +class AllDiscountItems(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("all_items",) + all_items = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="allItems") + + +class ApiVersion(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("display_name", "handle", "supported") + display_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="displayName") + handle = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="handle") + supported = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="supported") + + +class AppConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("App"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class AppCreditConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppCreditEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppCredit"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class AppCreditEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("AppCredit"), graphql_name="node") + + +class AppDiscountType(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("app", "app_bridge", "app_key", "description", "discount_class", "function_id", "target_type", "title") + app = sgqlc.types.Field(sgqlc.types.non_null("App"), graphql_name="app") + app_bridge = sgqlc.types.Field(sgqlc.types.non_null("FunctionsAppBridge"), graphql_name="appBridge") + app_key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="appKey") + description = sgqlc.types.Field(String, graphql_name="description") + discount_class = sgqlc.types.Field(sgqlc.types.non_null(DiscountClass), graphql_name="discountClass") + function_id = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="functionId") + target_type = sgqlc.types.Field(sgqlc.types.non_null(DiscountApplicationTargetType), graphql_name="targetType") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + + +class AppEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("App"), graphql_name="node") + + +class AppFeedback(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("app", "link", "messages") + app = sgqlc.types.Field(sgqlc.types.non_null("App"), graphql_name="app") + link = sgqlc.types.Field("Link", graphql_name="link") + messages = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="messages") + + +class AppInstallationConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppInstallationEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppInstallation"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class AppInstallationEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("AppInstallation"), graphql_name="node") + + +class AppPlanV2(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("pricing_details",) + pricing_details = sgqlc.types.Field(sgqlc.types.non_null("AppPricingDetails"), graphql_name="pricingDetails") + + +class AppPurchaseOneTimeConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppPurchaseOneTimeEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppPurchaseOneTime"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class AppPurchaseOneTimeCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("app_purchase_one_time", "confirmation_url", "user_errors") + app_purchase_one_time = sgqlc.types.Field("AppPurchaseOneTime", graphql_name="appPurchaseOneTime") + confirmation_url = sgqlc.types.Field(URL, graphql_name="confirmationUrl") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class AppPurchaseOneTimeEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("AppPurchaseOneTime"), graphql_name="node") + + +class AppRecurringPricing(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("discount", "interval", "price") + discount = sgqlc.types.Field("AppSubscriptionDiscount", graphql_name="discount") + interval = sgqlc.types.Field(sgqlc.types.non_null(AppPricingInterval), graphql_name="interval") + price = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="price") + + +class AppRevenueAttributionRecordConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppRevenueAttributionRecordEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppRevenueAttributionRecord"))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class AppRevenueAttributionRecordCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("app_revenue_attribution_record", "user_errors") + app_revenue_attribution_record = sgqlc.types.Field("AppRevenueAttributionRecord", graphql_name="appRevenueAttributionRecord") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppRevenueAttributionRecordCreateUserError"))), + graphql_name="userErrors", + ) + + +class AppRevenueAttributionRecordDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_id", "user_errors") + deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppRevenueAttributionRecordDeleteUserError"))), + graphql_name="userErrors", + ) + + +class AppRevenueAttributionRecordEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("AppRevenueAttributionRecord"), graphql_name="node") + + +class AppSubscriptionCancelPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("app_subscription", "user_errors") + app_subscription = sgqlc.types.Field("AppSubscription", graphql_name="appSubscription") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class AppSubscriptionConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppSubscriptionEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppSubscription"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class AppSubscriptionCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("app_subscription", "confirmation_url", "user_errors") + app_subscription = sgqlc.types.Field("AppSubscription", graphql_name="appSubscription") + confirmation_url = sgqlc.types.Field(URL, graphql_name="confirmationUrl") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class AppSubscriptionDiscount(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("duration_limit_in_intervals", "price_after_discount", "remaining_duration_in_intervals", "value") + duration_limit_in_intervals = sgqlc.types.Field(Int, graphql_name="durationLimitInIntervals") + price_after_discount = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="priceAfterDiscount") + remaining_duration_in_intervals = sgqlc.types.Field(Int, graphql_name="remainingDurationInIntervals") + value = sgqlc.types.Field(sgqlc.types.non_null("AppSubscriptionDiscountValue"), graphql_name="value") + + +class AppSubscriptionDiscountAmount(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("amount",) + amount = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="amount") + + +class AppSubscriptionDiscountPercentage(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("percentage",) + percentage = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="percentage") + + +class AppSubscriptionEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("AppSubscription"), graphql_name="node") + + +class AppSubscriptionLineItem(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("id", "plan", "usage_records") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + plan = sgqlc.types.Field(sgqlc.types.non_null(AppPlanV2), graphql_name="plan") + usage_records = sgqlc.types.Field( + sgqlc.types.non_null("AppUsageRecordConnection"), + graphql_name="usageRecords", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(AppUsageRecordSortKeys, graphql_name="sortKey", default="CREATED_AT")), + ) + ), + ) + + +class AppSubscriptionLineItemUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("app_subscription", "confirmation_url", "user_errors") + app_subscription = sgqlc.types.Field("AppSubscription", graphql_name="appSubscription") + confirmation_url = sgqlc.types.Field(URL, graphql_name="confirmationUrl") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class AppSubscriptionTrialExtendPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("app_subscription", "user_errors") + app_subscription = sgqlc.types.Field("AppSubscription", graphql_name="appSubscription") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppSubscriptionTrialExtendUserError"))), graphql_name="userErrors" + ) + + +class AppUsagePricing(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("balance_used", "capped_amount", "interval", "terms") + balance_used = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="balanceUsed") + capped_amount = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="cappedAmount") + interval = sgqlc.types.Field(sgqlc.types.non_null(AppPricingInterval), graphql_name="interval") + terms = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="terms") + + +class AppUsageRecordConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppUsageRecordEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppUsageRecord"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class AppUsageRecordCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("app_usage_record", "user_errors") + app_usage_record = sgqlc.types.Field("AppUsageRecord", graphql_name="appUsageRecord") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class AppUsageRecordEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("AppUsageRecord"), graphql_name="node") + + +class Attribute(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("key", "value") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + value = sgqlc.types.Field(String, graphql_name="value") + + +class AvailableChannelDefinitionsByChannel(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("channel_definitions", "channel_name") + channel_definitions = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ChannelDefinition"))), graphql_name="channelDefinitions" + ) + channel_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="channelName") + + +class BulkOperationCancelPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("bulk_operation", "user_errors") + bulk_operation = sgqlc.types.Field("BulkOperation", graphql_name="bulkOperation") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class BulkOperationRunMutationPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("bulk_operation", "user_errors") + bulk_operation = sgqlc.types.Field("BulkOperation", graphql_name="bulkOperation") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BulkMutationUserError"))), graphql_name="userErrors" + ) + + +class BulkOperationRunQueryPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("bulk_operation", "user_errors") + bulk_operation = sgqlc.types.Field("BulkOperation", graphql_name="bulkOperation") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class BulkProductResourceFeedbackCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("feedback", "user_errors") + feedback = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("ProductResourceFeedback")), graphql_name="feedback") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BulkProductResourceFeedbackCreateUserError"))), + graphql_name="userErrors", + ) + + +class BundlesFeature(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("eligible_for_bundles", "ineligibility_reason", "sells_bundles") + eligible_for_bundles = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="eligibleForBundles") + ineligibility_reason = sgqlc.types.Field(String, graphql_name="ineligibilityReason") + sells_bundles = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="sellsBundles") + + +class BuyerExperienceConfiguration(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("checkout_to_draft", "editable_shipping_address", "pay_now_only", "payment_terms_template") + checkout_to_draft = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="checkoutToDraft") + editable_shipping_address = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="editableShippingAddress") + pay_now_only = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="payNowOnly") + payment_terms_template = sgqlc.types.Field("PaymentTermsTemplate", graphql_name="paymentTermsTemplate") + + +class CalculatedDiscountAllocation(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("allocated_amount_set", "discount_application") + allocated_amount_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="allocatedAmountSet") + discount_application = sgqlc.types.Field(sgqlc.types.non_null(CalculatedDiscountApplication), graphql_name="discountApplication") + + +class CalculatedDiscountApplicationConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CalculatedDiscountApplicationEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CalculatedDiscountApplication))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class CalculatedDiscountApplicationEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(CalculatedDiscountApplication), graphql_name="node") + + +class CalculatedDraftOrder(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "applied_discount", + "available_shipping_rates", + "billing_address_matches_shipping_address", + "currency_code", + "customer", + "line_items", + "line_items_subtotal_price", + "market_name", + "market_region_country_code", + "phone", + "presentment_currency_code", + "purchasing_entity", + "shipping_line", + "subtotal_price", + "subtotal_price_set", + "tax_lines", + "total_discounts_set", + "total_line_items_price_set", + "total_price", + "total_price_set", + "total_shipping_price", + "total_shipping_price_set", + "total_tax", + "total_tax_set", + ) + applied_discount = sgqlc.types.Field("DraftOrderAppliedDiscount", graphql_name="appliedDiscount") + available_shipping_rates = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShippingRate"))), graphql_name="availableShippingRates" + ) + billing_address_matches_shipping_address = sgqlc.types.Field( + sgqlc.types.non_null(Boolean), graphql_name="billingAddressMatchesShippingAddress" + ) + currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currencyCode") + customer = sgqlc.types.Field("Customer", graphql_name="customer") + line_items = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CalculatedDraftOrderLineItem"))), graphql_name="lineItems" + ) + line_items_subtotal_price = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="lineItemsSubtotalPrice") + market_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="marketName") + market_region_country_code = sgqlc.types.Field(sgqlc.types.non_null(CountryCode), graphql_name="marketRegionCountryCode") + phone = sgqlc.types.Field(String, graphql_name="phone") + presentment_currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="presentmentCurrencyCode") + purchasing_entity = sgqlc.types.Field("PurchasingEntity", graphql_name="purchasingEntity") + shipping_line = sgqlc.types.Field("ShippingLine", graphql_name="shippingLine") + subtotal_price = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="subtotalPrice") + subtotal_price_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="subtotalPriceSet") + tax_lines = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TaxLine"))), graphql_name="taxLines") + total_discounts_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="totalDiscountsSet") + total_line_items_price_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="totalLineItemsPriceSet") + total_price = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="totalPrice") + total_price_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="totalPriceSet") + total_shipping_price = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="totalShippingPrice") + total_shipping_price_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="totalShippingPriceSet") + total_tax = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="totalTax") + total_tax_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="totalTaxSet") + + +class CalculatedDraftOrderLineItem(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "applied_discount", + "custom", + "custom_attributes", + "custom_attributes_v2", + "discounted_total", + "discounted_total_set", + "discounted_unit_price", + "discounted_unit_price_set", + "fulfillment_service", + "image", + "is_gift_card", + "name", + "original_total", + "original_total_set", + "original_unit_price", + "original_unit_price_set", + "product", + "quantity", + "requires_shipping", + "sku", + "taxable", + "title", + "total_discount", + "total_discount_set", + "variant", + "variant_title", + "vendor", + "weight", + ) + applied_discount = sgqlc.types.Field("DraftOrderAppliedDiscount", graphql_name="appliedDiscount") + custom = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="custom") + custom_attributes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Attribute))), graphql_name="customAttributes" + ) + custom_attributes_v2 = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TypedAttribute"))), graphql_name="customAttributesV2" + ) + discounted_total = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="discountedTotal") + discounted_total_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="discountedTotalSet") + discounted_unit_price = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="discountedUnitPrice") + discounted_unit_price_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="discountedUnitPriceSet") + fulfillment_service = sgqlc.types.Field("FulfillmentService", graphql_name="fulfillmentService") + image = sgqlc.types.Field("Image", graphql_name="image") + is_gift_card = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isGiftCard") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + original_total = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="originalTotal") + original_total_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="originalTotalSet") + original_unit_price = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="originalUnitPrice") + original_unit_price_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="originalUnitPriceSet") + product = sgqlc.types.Field("Product", graphql_name="product") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + requires_shipping = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresShipping") + sku = sgqlc.types.Field(String, graphql_name="sku") + taxable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxable") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + total_discount = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="totalDiscount") + total_discount_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="totalDiscountSet") + variant = sgqlc.types.Field("ProductVariant", graphql_name="variant") + variant_title = sgqlc.types.Field(String, graphql_name="variantTitle") + vendor = sgqlc.types.Field(String, graphql_name="vendor") + weight = sgqlc.types.Field("Weight", graphql_name="weight") + + +class CalculatedLineItem(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "calculated_discount_allocations", + "custom_attributes", + "discounted_unit_price_set", + "editable_quantity", + "editable_quantity_before_changes", + "editable_subtotal_set", + "has_staged_line_item_discount", + "id", + "image", + "original_unit_price_set", + "quantity", + "restockable", + "restocking", + "sku", + "staged_changes", + "title", + "uneditable_subtotal_set", + "variant", + "variant_title", + ) + calculated_discount_allocations = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CalculatedDiscountAllocation))), + graphql_name="calculatedDiscountAllocations", + ) + custom_attributes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Attribute))), graphql_name="customAttributes" + ) + discounted_unit_price_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="discountedUnitPriceSet") + editable_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="editableQuantity") + editable_quantity_before_changes = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="editableQuantityBeforeChanges") + editable_subtotal_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="editableSubtotalSet") + has_staged_line_item_discount = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasStagedLineItemDiscount") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + image = sgqlc.types.Field("Image", graphql_name="image") + original_unit_price_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="originalUnitPriceSet") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + restockable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="restockable") + restocking = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="restocking") + sku = sgqlc.types.Field(String, graphql_name="sku") + staged_changes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("OrderStagedChange"))), graphql_name="stagedChanges" + ) + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + uneditable_subtotal_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="uneditableSubtotalSet") + variant = sgqlc.types.Field("ProductVariant", graphql_name="variant") + variant_title = sgqlc.types.Field(String, graphql_name="variantTitle") + + +class CalculatedLineItemConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CalculatedLineItemEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CalculatedLineItem))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class CalculatedLineItemEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(CalculatedLineItem), graphql_name="node") + + +class CardPaymentDetails(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "avs_result_code", + "bin", + "company", + "cvv_result_code", + "expiration_month", + "expiration_year", + "name", + "number", + "wallet", + ) + avs_result_code = sgqlc.types.Field(String, graphql_name="avsResultCode") + bin = sgqlc.types.Field(String, graphql_name="bin") + company = sgqlc.types.Field(String, graphql_name="company") + cvv_result_code = sgqlc.types.Field(String, graphql_name="cvvResultCode") + expiration_month = sgqlc.types.Field(Int, graphql_name="expirationMonth") + expiration_year = sgqlc.types.Field(Int, graphql_name="expirationYear") + name = sgqlc.types.Field(String, graphql_name="name") + number = sgqlc.types.Field(String, graphql_name="number") + wallet = sgqlc.types.Field(DigitalWallet, graphql_name="wallet") + + +class CartTransformConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CartTransformEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CartTransform"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class CartTransformCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cart_transform", "user_errors") + cart_transform = sgqlc.types.Field("CartTransform", graphql_name="cartTransform") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CartTransformCreateUserError"))), graphql_name="userErrors" + ) + + +class CartTransformDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_id", "user_errors") + deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CartTransformDeleteUserError"))), graphql_name="userErrors" + ) + + +class CartTransformEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("CartTransform"), graphql_name="node") + + +class CatalogConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info", "total_count") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CatalogEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Catalog))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + total_count = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="totalCount") + + +class CatalogContextUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("catalog", "user_errors") + catalog = sgqlc.types.Field(Catalog, graphql_name="catalog") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CatalogUserError"))), graphql_name="userErrors" + ) + + +class CatalogCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("catalog", "user_errors") + catalog = sgqlc.types.Field(Catalog, graphql_name="catalog") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CatalogUserError"))), graphql_name="userErrors" + ) + + +class CatalogDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_id", "user_errors") + deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CatalogUserError"))), graphql_name="userErrors" + ) + + +class CatalogEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(Catalog), graphql_name="node") + + +class CatalogUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("catalog", "user_errors") + catalog = sgqlc.types.Field(Catalog, graphql_name="catalog") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CatalogUserError"))), graphql_name="userErrors" + ) + + +class ChannelConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ChannelEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Channel"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class ChannelEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("Channel"), graphql_name="node") + + +class CheckoutProfileConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CheckoutProfileEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CheckoutProfile"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class CheckoutProfileEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("CheckoutProfile"), graphql_name="node") + + +class CollectionAddProductsPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("collection", "user_errors") + collection = sgqlc.types.Field("Collection", graphql_name="collection") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class CollectionAddProductsV2Payload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("job", "user_errors") + job = sgqlc.types.Field("Job", graphql_name="job") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CollectionAddProductsV2UserError"))), graphql_name="userErrors" + ) + + +class CollectionConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CollectionEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Collection"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class CollectionCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("collection", "user_errors") + collection = sgqlc.types.Field("Collection", graphql_name="collection") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class CollectionDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_collection_id", "shop", "user_errors") + deleted_collection_id = sgqlc.types.Field(ID, graphql_name="deletedCollectionId") + shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class CollectionEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("Collection"), graphql_name="node") + + +class CollectionPublication(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("collection", "is_published", "publication", "publish_date") + collection = sgqlc.types.Field(sgqlc.types.non_null("Collection"), graphql_name="collection") + is_published = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isPublished") + publication = sgqlc.types.Field(sgqlc.types.non_null("Publication"), graphql_name="publication") + publish_date = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="publishDate") + + +class CollectionPublicationConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CollectionPublicationEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CollectionPublication))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class CollectionPublicationEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(CollectionPublication), graphql_name="node") + + +class CollectionPublishPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("collection", "collection_publications", "shop", "user_errors") + collection = sgqlc.types.Field("Collection", graphql_name="collection") + collection_publications = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(CollectionPublication)), graphql_name="collectionPublications" + ) + shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class CollectionRemoveProductsPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("job", "user_errors") + job = sgqlc.types.Field("Job", graphql_name="job") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class CollectionReorderProductsPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("job", "user_errors") + job = sgqlc.types.Field("Job", graphql_name="job") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class CollectionRule(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("column", "condition", "condition_object", "relation") + column = sgqlc.types.Field(sgqlc.types.non_null(CollectionRuleColumn), graphql_name="column") + condition = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="condition") + condition_object = sgqlc.types.Field("CollectionRuleConditionObject", graphql_name="conditionObject") + relation = sgqlc.types.Field(sgqlc.types.non_null(CollectionRuleRelation), graphql_name="relation") + + +class CollectionRuleConditions(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("allowed_relations", "default_relation", "rule_object", "rule_type") + allowed_relations = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CollectionRuleRelation))), graphql_name="allowedRelations" + ) + default_relation = sgqlc.types.Field(sgqlc.types.non_null(CollectionRuleRelation), graphql_name="defaultRelation") + rule_object = sgqlc.types.Field("CollectionRuleConditionsRuleObject", graphql_name="ruleObject") + rule_type = sgqlc.types.Field(sgqlc.types.non_null(CollectionRuleColumn), graphql_name="ruleType") + + +class CollectionRuleMetafieldCondition(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("metafield_definition",) + metafield_definition = sgqlc.types.Field(sgqlc.types.non_null("MetafieldDefinition"), graphql_name="metafieldDefinition") + + +class CollectionRuleProductCategoryCondition(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("value",) + value = sgqlc.types.Field(sgqlc.types.non_null("ProductTaxonomyNode"), graphql_name="value") + + +class CollectionRuleSet(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("applied_disjunctively", "rules") + applied_disjunctively = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliedDisjunctively") + rules = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CollectionRule))), graphql_name="rules") + + +class CollectionRuleTextCondition(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("value",) + value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") + + +class CollectionUnpublishPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("collection", "shop", "user_errors") + collection = sgqlc.types.Field("Collection", graphql_name="collection") + shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class CollectionUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("collection", "job", "user_errors") + collection = sgqlc.types.Field("Collection", graphql_name="collection") + job = sgqlc.types.Field("Job", graphql_name="job") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class CommentEventAttachment(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("file_extension", "id", "image", "name", "size", "url") + file_extension = sgqlc.types.Field(String, graphql_name="fileExtension") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + image = sgqlc.types.Field("Image", graphql_name="image") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + size = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="size") + url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") + + +class CompaniesDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_company_ids", "user_errors") + deleted_company_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="deletedCompanyIds") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyAddressDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_address_id", "user_errors") + deleted_address_id = sgqlc.types.Field(ID, graphql_name="deletedAddressId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyAssignCustomerAsContactPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("company_contact", "user_errors") + company_contact = sgqlc.types.Field("CompanyContact", graphql_name="companyContact") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyAssignMainContactPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("company", "user_errors") + company = sgqlc.types.Field("Company", graphql_name="company") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CompanyEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Company"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class CompanyContactAssignRolePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("company_contact_role_assignment", "user_errors") + company_contact_role_assignment = sgqlc.types.Field("CompanyContactRoleAssignment", graphql_name="companyContactRoleAssignment") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyContactAssignRolesPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("role_assignments", "user_errors") + role_assignments = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("CompanyContactRoleAssignment")), graphql_name="roleAssignments" + ) + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyContactConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CompanyContactEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CompanyContact"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class CompanyContactCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("company_contact", "user_errors") + company_contact = sgqlc.types.Field("CompanyContact", graphql_name="companyContact") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyContactDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_company_contact_id", "user_errors") + deleted_company_contact_id = sgqlc.types.Field(ID, graphql_name="deletedCompanyContactId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyContactEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("CompanyContact"), graphql_name="node") + + +class CompanyContactRemoveFromCompanyPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("removed_company_contact_id", "user_errors") + removed_company_contact_id = sgqlc.types.Field(ID, graphql_name="removedCompanyContactId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyContactRevokeRolePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("revoked_company_contact_role_assignment_id", "user_errors") + revoked_company_contact_role_assignment_id = sgqlc.types.Field(ID, graphql_name="revokedCompanyContactRoleAssignmentId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyContactRevokeRolesPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("revoked_role_assignment_ids", "user_errors") + revoked_role_assignment_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="revokedRoleAssignmentIds") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyContactRoleAssignmentConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CompanyContactRoleAssignmentEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CompanyContactRoleAssignment"))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class CompanyContactRoleAssignmentEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("CompanyContactRoleAssignment"), graphql_name="node") + + +class CompanyContactRoleConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CompanyContactRoleEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CompanyContactRole"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class CompanyContactRoleEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("CompanyContactRole"), graphql_name="node") + + +class CompanyContactSendWelcomeEmailPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("company_contact", "user_errors") + company_contact = sgqlc.types.Field("CompanyContact", graphql_name="companyContact") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyContactUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("company_contact", "user_errors") + company_contact = sgqlc.types.Field("CompanyContact", graphql_name="companyContact") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyContactsDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_company_contact_ids", "user_errors") + deleted_company_contact_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="deletedCompanyContactIds") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("company", "user_errors") + company = sgqlc.types.Field("Company", graphql_name="company") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_company_id", "user_errors") + deleted_company_id = sgqlc.types.Field(ID, graphql_name="deletedCompanyId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("Company"), graphql_name="node") + + +class CompanyLocationAssignAddressPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("addresses", "user_errors") + addresses = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("CompanyAddress")), graphql_name="addresses") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyLocationAssignRolesPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("role_assignments", "user_errors") + role_assignments = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("CompanyContactRoleAssignment")), graphql_name="roleAssignments" + ) + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyLocationAssignTaxExemptionsPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("company_location", "user_errors") + company_location = sgqlc.types.Field("CompanyLocation", graphql_name="companyLocation") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyLocationConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CompanyLocationEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CompanyLocation"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class CompanyLocationCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("company_location", "user_errors") + company_location = sgqlc.types.Field("CompanyLocation", graphql_name="companyLocation") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyLocationCreateTaxRegistrationPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("company_location", "user_errors") + company_location = sgqlc.types.Field("CompanyLocation", graphql_name="companyLocation") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyLocationDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_company_location_id", "user_errors") + deleted_company_location_id = sgqlc.types.Field(ID, graphql_name="deletedCompanyLocationId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyLocationEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("CompanyLocation"), graphql_name="node") + + +class CompanyLocationRevokeRolesPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("revoked_role_assignment_ids", "user_errors") + revoked_role_assignment_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="revokedRoleAssignmentIds") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyLocationRevokeTaxExemptionsPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("company_location", "user_errors") + company_location = sgqlc.types.Field("CompanyLocation", graphql_name="companyLocation") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyLocationRevokeTaxRegistrationPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("company_location", "user_errors") + company_location = sgqlc.types.Field("CompanyLocation", graphql_name="companyLocation") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyLocationUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("company_location", "user_errors") + company_location = sgqlc.types.Field("CompanyLocation", graphql_name="companyLocation") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyLocationsDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_company_location_ids", "user_errors") + deleted_company_location_ids = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="deletedCompanyLocationIds" + ) + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyRevokeMainContactPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("company", "user_errors") + company = sgqlc.types.Field("Company", graphql_name="company") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CompanyUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("company", "user_errors") + company = sgqlc.types.Field("Company", graphql_name="company") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), graphql_name="userErrors" + ) + + +class CountriesInShippingZones(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("country_codes", "include_rest_of_world") + country_codes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CountryCode))), graphql_name="countryCodes" + ) + include_rest_of_world = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="includeRestOfWorld") + + +class CountryHarmonizedSystemCode(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("country_code", "harmonized_system_code") + country_code = sgqlc.types.Field(sgqlc.types.non_null(CountryCode), graphql_name="countryCode") + harmonized_system_code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="harmonizedSystemCode") + + +class CountryHarmonizedSystemCodeConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CountryHarmonizedSystemCodeEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CountryHarmonizedSystemCode))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class CountryHarmonizedSystemCodeEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(CountryHarmonizedSystemCode), graphql_name="node") + + +class CurrencyFormats(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("money_format", "money_in_emails_format", "money_with_currency_format", "money_with_currency_in_emails_format") + money_format = sgqlc.types.Field(sgqlc.types.non_null(FormattedString), graphql_name="moneyFormat") + money_in_emails_format = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="moneyInEmailsFormat") + money_with_currency_format = sgqlc.types.Field(sgqlc.types.non_null(FormattedString), graphql_name="moneyWithCurrencyFormat") + money_with_currency_in_emails_format = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="moneyWithCurrencyInEmailsFormat") + + +class CurrencySetting(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("currency_code", "currency_name", "enabled", "rate_updated_at") + currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currencyCode") + currency_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="currencyName") + enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="enabled") + rate_updated_at = sgqlc.types.Field(DateTime, graphql_name="rateUpdatedAt") + + +class CurrencySettingConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CurrencySettingEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CurrencySetting))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class CurrencySettingEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(CurrencySetting), graphql_name="node") + + +class CustomerAddTaxExemptionsPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("customer", "user_errors") + customer = sgqlc.types.Field("Customer", graphql_name="customer") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class CustomerConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Customer"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class CustomerCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("customer", "user_errors") + customer = sgqlc.types.Field("Customer", graphql_name="customer") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class CustomerCreditCard(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "billing_address", + "brand", + "expires_soon", + "expiry_month", + "expiry_year", + "first_digits", + "is_revocable", + "last_digits", + "masked_number", + "name", + "source", + "virtual_last_digits", + ) + billing_address = sgqlc.types.Field("CustomerCreditCardBillingAddress", graphql_name="billingAddress") + brand = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="brand") + expires_soon = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="expiresSoon") + expiry_month = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="expiryMonth") + expiry_year = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="expiryYear") + first_digits = sgqlc.types.Field(String, graphql_name="firstDigits") + is_revocable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isRevocable") + last_digits = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="lastDigits") + masked_number = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="maskedNumber") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + source = sgqlc.types.Field(String, graphql_name="source") + virtual_last_digits = sgqlc.types.Field(String, graphql_name="virtualLastDigits") + + +class CustomerCreditCardBillingAddress(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("address1", "city", "country", "country_code", "first_name", "last_name", "province", "province_code", "zip") + address1 = sgqlc.types.Field(String, graphql_name="address1") + city = sgqlc.types.Field(String, graphql_name="city") + country = sgqlc.types.Field(String, graphql_name="country") + country_code = sgqlc.types.Field(CountryCode, graphql_name="countryCode") + first_name = sgqlc.types.Field(String, graphql_name="firstName") + last_name = sgqlc.types.Field(String, graphql_name="lastName") + province = sgqlc.types.Field(String, graphql_name="province") + province_code = sgqlc.types.Field(String, graphql_name="provinceCode") + zip = sgqlc.types.Field(String, graphql_name="zip") + + +class CustomerDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_customer_id", "shop", "user_errors") + deleted_customer_id = sgqlc.types.Field(ID, graphql_name="deletedCustomerId") + shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class CustomerEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("Customer"), graphql_name="node") + + +class CustomerEmailAddress(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("email_address", "marketing_state", "marketing_unsubscribe_url", "open_tracking_level", "open_tracking_url") + email_address = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="emailAddress") + marketing_state = sgqlc.types.Field(sgqlc.types.non_null(CustomerEmailAddressMarketingState), graphql_name="marketingState") + marketing_unsubscribe_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="marketingUnsubscribeUrl") + open_tracking_level = sgqlc.types.Field(sgqlc.types.non_null(CustomerEmailAddressOpenTrackingLevel), graphql_name="openTrackingLevel") + open_tracking_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="openTrackingUrl") + + +class CustomerEmailMarketingConsentState(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("consent_updated_at", "marketing_opt_in_level", "marketing_state") + consent_updated_at = sgqlc.types.Field(DateTime, graphql_name="consentUpdatedAt") + marketing_opt_in_level = sgqlc.types.Field(CustomerMarketingOptInLevel, graphql_name="marketingOptInLevel") + marketing_state = sgqlc.types.Field(sgqlc.types.non_null(CustomerEmailMarketingState), graphql_name="marketingState") + + +class CustomerEmailMarketingConsentUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("customer", "user_errors") + customer = sgqlc.types.Field("Customer", graphql_name="customer") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerEmailMarketingConsentUpdateUserError"))), + graphql_name="userErrors", + ) + + +class CustomerGenerateAccountActivationUrlPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("account_activation_url", "user_errors") + account_activation_url = sgqlc.types.Field(URL, graphql_name="accountActivationUrl") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class CustomerJourney(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("customer_order_index", "days_to_conversion", "first_visit", "last_visit", "moments") + customer_order_index = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="customerOrderIndex") + days_to_conversion = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="daysToConversion") + first_visit = sgqlc.types.Field(sgqlc.types.non_null("CustomerVisit"), graphql_name="firstVisit") + last_visit = sgqlc.types.Field("CustomerVisit", graphql_name="lastVisit") + moments = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CustomerMoment))), graphql_name="moments") + + +class CustomerJourneySummary(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("customer_order_index", "days_to_conversion", "first_visit", "last_visit", "moments", "moments_count", "ready") + customer_order_index = sgqlc.types.Field(Int, graphql_name="customerOrderIndex") + days_to_conversion = sgqlc.types.Field(Int, graphql_name="daysToConversion") + first_visit = sgqlc.types.Field("CustomerVisit", graphql_name="firstVisit") + last_visit = sgqlc.types.Field("CustomerVisit", graphql_name="lastVisit") + moments = sgqlc.types.Field( + "CustomerMomentConnection", + graphql_name="moments", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + moments_count = sgqlc.types.Field(Int, graphql_name="momentsCount") + ready = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="ready") + + +class CustomerMergeError(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("error_fields", "message") + error_fields = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CustomerMergeErrorFieldType))), graphql_name="errorFields" + ) + message = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="message") + + +class CustomerMergePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("job", "resulting_customer_id", "user_errors") + job = sgqlc.types.Field("Job", graphql_name="job") + resulting_customer_id = sgqlc.types.Field(ID, graphql_name="resultingCustomerId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerMergeUserError"))), graphql_name="userErrors" + ) + + +class CustomerMergePreview(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("alternate_fields", "blocking_fields", "customer_merge_errors", "default_fields", "resulting_customer_id") + alternate_fields = sgqlc.types.Field("CustomerMergePreviewAlternateFields", graphql_name="alternateFields") + blocking_fields = sgqlc.types.Field("CustomerMergePreviewBlockingFields", graphql_name="blockingFields") + customer_merge_errors = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(CustomerMergeError)), graphql_name="customerMergeErrors" + ) + default_fields = sgqlc.types.Field("CustomerMergePreviewDefaultFields", graphql_name="defaultFields") + resulting_customer_id = sgqlc.types.Field(ID, graphql_name="resultingCustomerId") + + +class CustomerMergePreviewAlternateFields(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("default_address", "email", "first_name", "last_name", "phone_number") + default_address = sgqlc.types.Field("MailingAddress", graphql_name="defaultAddress") + email = sgqlc.types.Field(CustomerEmailAddress, graphql_name="email") + first_name = sgqlc.types.Field(String, graphql_name="firstName") + last_name = sgqlc.types.Field(String, graphql_name="lastName") + phone_number = sgqlc.types.Field("CustomerPhoneNumber", graphql_name="phoneNumber") + + +class CustomerMergePreviewBlockingFields(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("note", "tags") + note = sgqlc.types.Field(String, graphql_name="note") + tags = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="tags") + + +class CustomerMergePreviewDefaultFields(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "addresses", + "default_address", + "discount_node_count", + "discount_nodes", + "display_name", + "draft_order_count", + "draft_orders", + "email", + "first_name", + "gift_card_count", + "gift_cards", + "last_name", + "metafield_count", + "note", + "order_count", + "orders", + "phone_number", + "tags", + ) + addresses = sgqlc.types.Field( + sgqlc.types.non_null("MailingAddressConnection"), + graphql_name="addresses", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + default_address = sgqlc.types.Field("MailingAddress", graphql_name="defaultAddress") + discount_node_count = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="discountNodeCount") + discount_nodes = sgqlc.types.Field( + sgqlc.types.non_null("DiscountNodeConnection"), + graphql_name="discountNodes", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(DiscountSortKeys, graphql_name="sortKey", default="CREATED_AT")), + ) + ), + ) + display_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="displayName") + draft_order_count = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="draftOrderCount") + draft_orders = sgqlc.types.Field( + sgqlc.types.non_null("DraftOrderConnection"), + graphql_name="draftOrders", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(DraftOrderSortKeys, graphql_name="sortKey", default="UPDATED_AT")), + ) + ), + ) + email = sgqlc.types.Field(CustomerEmailAddress, graphql_name="email") + first_name = sgqlc.types.Field(String, graphql_name="firstName") + gift_card_count = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="giftCardCount") + gift_cards = sgqlc.types.Field( + sgqlc.types.non_null("GiftCardConnection"), + graphql_name="giftCards", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(GiftCardSortKeys, graphql_name="sortKey", default="CREATED_AT")), + ) + ), + ) + last_name = sgqlc.types.Field(String, graphql_name="lastName") + metafield_count = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="metafieldCount") + note = sgqlc.types.Field(String, graphql_name="note") + order_count = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="orderCount") + orders = sgqlc.types.Field( + sgqlc.types.non_null("OrderConnection"), + graphql_name="orders", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(OrderSortKeys, graphql_name="sortKey", default="PROCESSED_AT")), + ) + ), + ) + phone_number = sgqlc.types.Field("CustomerPhoneNumber", graphql_name="phoneNumber") + tags = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="tags") + + +class CustomerMergeRequest(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("customer_merge_errors", "job_id", "resulting_customer_id", "status") + customer_merge_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CustomerMergeError))), graphql_name="customerMergeErrors" + ) + job_id = sgqlc.types.Field(ID, graphql_name="jobId") + resulting_customer_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="resultingCustomerId") + status = sgqlc.types.Field(sgqlc.types.non_null(CustomerMergeRequestStatus), graphql_name="status") + + +class CustomerMergeable(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("error_fields", "is_mergeable", "merge_in_progress", "reason") + error_fields = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CustomerMergeErrorFieldType))), graphql_name="errorFields" + ) + is_mergeable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isMergeable") + merge_in_progress = sgqlc.types.Field(CustomerMergeRequest, graphql_name="mergeInProgress") + reason = sgqlc.types.Field(String, graphql_name="reason") + + +class CustomerMomentConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerMomentEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CustomerMoment))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class CustomerMomentEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(CustomerMoment), graphql_name="node") + + +class CustomerPaymentInstrumentBillingAddress(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("address1", "city", "country", "country_code", "name", "province", "province_code", "zip") + address1 = sgqlc.types.Field(String, graphql_name="address1") + city = sgqlc.types.Field(String, graphql_name="city") + country = sgqlc.types.Field(String, graphql_name="country") + country_code = sgqlc.types.Field(CountryCode, graphql_name="countryCode") + name = sgqlc.types.Field(String, graphql_name="name") + province = sgqlc.types.Field(String, graphql_name="province") + province_code = sgqlc.types.Field(String, graphql_name="provinceCode") + zip = sgqlc.types.Field(String, graphql_name="zip") + + +class CustomerPaymentMethodConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerPaymentMethodEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerPaymentMethod"))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class CustomerPaymentMethodCreateFromDuplicationDataPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("customer_payment_method", "user_errors") + customer_payment_method = sgqlc.types.Field("CustomerPaymentMethod", graphql_name="customerPaymentMethod") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerPaymentMethodCreateFromDuplicationDataUserError"))), + graphql_name="userErrors", + ) + + +class CustomerPaymentMethodCreditCardCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("customer_payment_method", "user_errors") + customer_payment_method = sgqlc.types.Field("CustomerPaymentMethod", graphql_name="customerPaymentMethod") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class CustomerPaymentMethodCreditCardUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("customer_payment_method", "user_errors") + customer_payment_method = sgqlc.types.Field("CustomerPaymentMethod", graphql_name="customerPaymentMethod") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class CustomerPaymentMethodEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("CustomerPaymentMethod"), graphql_name="node") + + +class CustomerPaymentMethodGetDuplicationDataPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("encrypted_duplication_data", "user_errors") + encrypted_duplication_data = sgqlc.types.Field(String, graphql_name="encryptedDuplicationData") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerPaymentMethodGetDuplicationDataUserError"))), + graphql_name="userErrors", + ) + + +class CustomerPaymentMethodGetUpdateUrlPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("update_payment_method_url", "user_errors") + update_payment_method_url = sgqlc.types.Field(URL, graphql_name="updatePaymentMethodUrl") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerPaymentMethodGetUpdateUrlUserError"))), + graphql_name="userErrors", + ) + + +class CustomerPaymentMethodPaypalBillingAgreementCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("customer_payment_method", "user_errors") + customer_payment_method = sgqlc.types.Field("CustomerPaymentMethod", graphql_name="customerPaymentMethod") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerPaymentMethodUserError"))), graphql_name="userErrors" + ) + + +class CustomerPaymentMethodPaypalBillingAgreementUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("customer_payment_method", "user_errors") + customer_payment_method = sgqlc.types.Field("CustomerPaymentMethod", graphql_name="customerPaymentMethod") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerPaymentMethodUserError"))), graphql_name="userErrors" + ) + + +class CustomerPaymentMethodRemoteCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("customer_payment_method", "user_errors") + customer_payment_method = sgqlc.types.Field("CustomerPaymentMethod", graphql_name="customerPaymentMethod") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerPaymentMethodRemoteUserError"))), graphql_name="userErrors" + ) + + +class CustomerPaymentMethodRemoteCreditCardCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("customer_payment_method", "user_errors") + customer_payment_method = sgqlc.types.Field("CustomerPaymentMethod", graphql_name="customerPaymentMethod") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerPaymentMethodUserError"))), graphql_name="userErrors" + ) + + +class CustomerPaymentMethodRevokePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("revoked_customer_payment_method_id", "user_errors") + revoked_customer_payment_method_id = sgqlc.types.Field(ID, graphql_name="revokedCustomerPaymentMethodId") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class CustomerPaymentMethodSendUpdateEmailPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("customer", "user_errors") + customer = sgqlc.types.Field("Customer", graphql_name="customer") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class CustomerPaypalBillingAgreement(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("billing_address", "inactive", "is_revocable", "paypal_account_email") + billing_address = sgqlc.types.Field(CustomerPaymentInstrumentBillingAddress, graphql_name="billingAddress") + inactive = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="inactive") + is_revocable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isRevocable") + paypal_account_email = sgqlc.types.Field(String, graphql_name="paypalAccountEmail") + + +class CustomerPhoneNumber(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("marketing_state", "phone_number") + marketing_state = sgqlc.types.Field(sgqlc.types.non_null(CustomerSmsMarketingState), graphql_name="marketingState") + phone_number = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="phoneNumber") + + +class CustomerRemoveTaxExemptionsPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("customer", "user_errors") + customer = sgqlc.types.Field("Customer", graphql_name="customer") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class CustomerReplaceTaxExemptionsPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("customer", "user_errors") + customer = sgqlc.types.Field("Customer", graphql_name="customer") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class CustomerSegmentMemberConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "page_info", "statistics", "total_count") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerSegmentMemberEdge"))), graphql_name="edges" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + statistics = sgqlc.types.Field(sgqlc.types.non_null("SegmentStatistics"), graphql_name="statistics") + total_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalCount") + + +class CustomerSegmentMemberEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("CustomerSegmentMember"), graphql_name="node") + + +class CustomerSegmentMembersQueryCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("customer_segment_members_query", "user_errors") + customer_segment_members_query = sgqlc.types.Field("CustomerSegmentMembersQuery", graphql_name="customerSegmentMembersQuery") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerSegmentMembersQueryUserError"))), graphql_name="userErrors" + ) + + +class CustomerShopPayAgreement(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "billing_address", + "expires_soon", + "expiry_month", + "expiry_year", + "inactive", + "is_revocable", + "last_digits", + "masked_number", + "name", + ) + billing_address = sgqlc.types.Field(CustomerCreditCardBillingAddress, graphql_name="billingAddress") + expires_soon = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="expiresSoon") + expiry_month = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="expiryMonth") + expiry_year = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="expiryYear") + inactive = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="inactive") + is_revocable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isRevocable") + last_digits = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="lastDigits") + masked_number = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="maskedNumber") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + + +class CustomerSmsMarketingConsentState(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("consent_collected_from", "consent_updated_at", "marketing_opt_in_level", "marketing_state") + consent_collected_from = sgqlc.types.Field(CustomerConsentCollectedFrom, graphql_name="consentCollectedFrom") + consent_updated_at = sgqlc.types.Field(DateTime, graphql_name="consentUpdatedAt") + marketing_opt_in_level = sgqlc.types.Field(sgqlc.types.non_null(CustomerMarketingOptInLevel), graphql_name="marketingOptInLevel") + marketing_state = sgqlc.types.Field(sgqlc.types.non_null(CustomerSmsMarketingState), graphql_name="marketingState") + + +class CustomerSmsMarketingConsentUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("customer", "user_errors") + customer = sgqlc.types.Field("Customer", graphql_name="customer") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerSmsMarketingConsentError"))), graphql_name="userErrors" + ) + + +class CustomerStatistics(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("predicted_spend_tier",) + predicted_spend_tier = sgqlc.types.Field(CustomerPredictedSpendTier, graphql_name="predictedSpendTier") + + +class CustomerUpdateDefaultAddressPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("customer", "user_errors") + customer = sgqlc.types.Field("Customer", graphql_name="customer") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class CustomerUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("customer", "user_errors") + customer = sgqlc.types.Field("Customer", graphql_name="customer") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class CustomerVisitProductInfo(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("product", "quantity", "variant") + product = sgqlc.types.Field("Product", graphql_name="product") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + variant = sgqlc.types.Field("ProductVariant", graphql_name="variant") + + +class CustomerVisitProductInfoConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerVisitProductInfoEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CustomerVisitProductInfo))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class CustomerVisitProductInfoEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(CustomerVisitProductInfo), graphql_name="node") + + +class DelegateAccessToken(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("access_scopes", "access_token", "created_at") + access_scopes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="accessScopes") + access_token = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="accessToken") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + + +class DelegateAccessTokenCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("delegate_access_token", "shop", "user_errors") + delegate_access_token = sgqlc.types.Field(DelegateAccessToken, graphql_name="delegateAccessToken") + shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DelegateAccessTokenCreateUserError"))), graphql_name="userErrors" + ) + + +class DelegateAccessTokenDestroyPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("shop", "status", "user_errors") + shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") + status = sgqlc.types.Field(Boolean, graphql_name="status") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DelegateAccessTokenDestroyUserError"))), graphql_name="userErrors" + ) + + +class DeletionEvent(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("occurred_at", "subject_id", "subject_type") + occurred_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="occurredAt") + subject_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="subjectId") + subject_type = sgqlc.types.Field(sgqlc.types.non_null(DeletionEventSubjectType), graphql_name="subjectType") + + +class DeletionEventConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeletionEventEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DeletionEvent))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class DeletionEventEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(DeletionEvent), graphql_name="node") + + +class DeliveryAvailableService(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("countries", "name") + countries = sgqlc.types.Field(sgqlc.types.non_null("DeliveryCountryCodesOrRestOfWorld"), graphql_name="countries") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + + +class DeliveryBrandedPromise(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("handle", "name") + handle = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="handle") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + + +class DeliveryCarrierServiceAndLocations(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("carrier_service", "locations") + carrier_service = sgqlc.types.Field(sgqlc.types.non_null("DeliveryCarrierService"), graphql_name="carrierService") + locations = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Location"))), graphql_name="locations") + + +class DeliveryCountryAndZone(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("country", "zone") + country = sgqlc.types.Field(sgqlc.types.non_null("DeliveryCountry"), graphql_name="country") + zone = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="zone") + + +class DeliveryCountryCodeOrRestOfWorld(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("country_code", "rest_of_world") + country_code = sgqlc.types.Field(CountryCode, graphql_name="countryCode") + rest_of_world = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="restOfWorld") + + +class DeliveryCountryCodesOrRestOfWorld(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("country_codes", "rest_of_world") + country_codes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CountryCode))), graphql_name="countryCodes" + ) + rest_of_world = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="restOfWorld") + + +class DeliveryCustomizationActivationPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("ids", "user_errors") + ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="ids") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryCustomizationError"))), graphql_name="userErrors" + ) + + +class DeliveryCustomizationConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryCustomizationEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryCustomization"))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class DeliveryCustomizationCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("delivery_customization", "user_errors") + delivery_customization = sgqlc.types.Field("DeliveryCustomization", graphql_name="deliveryCustomization") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryCustomizationError"))), graphql_name="userErrors" + ) + + +class DeliveryCustomizationDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_id", "user_errors") + deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryCustomizationError"))), graphql_name="userErrors" + ) + + +class DeliveryCustomizationEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("DeliveryCustomization"), graphql_name="node") + + +class DeliveryCustomizationUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("delivery_customization", "user_errors") + delivery_customization = sgqlc.types.Field("DeliveryCustomization", graphql_name="deliveryCustomization") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryCustomizationError"))), graphql_name="userErrors" + ) + + +class DeliveryLegacyModeBlocked(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("blocked", "reasons") + blocked = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="blocked") + reasons = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(DeliveryLegacyModeBlockedReason)), graphql_name="reasons") + + +class DeliveryLocalPickupSettings(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("instructions", "pickup_time") + instructions = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="instructions") + pickup_time = sgqlc.types.Field(sgqlc.types.non_null(DeliveryLocalPickupTime), graphql_name="pickupTime") + + +class DeliveryLocationGroupZone(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("method_definition_counts", "method_definitions", "zone") + method_definition_counts = sgqlc.types.Field( + sgqlc.types.non_null("DeliveryMethodDefinitionCounts"), graphql_name="methodDefinitionCounts" + ) + method_definitions = sgqlc.types.Field( + sgqlc.types.non_null("DeliveryMethodDefinitionConnection"), + graphql_name="methodDefinitions", + args=sgqlc.types.ArgDict( + ( + ("eligible", sgqlc.types.Arg(Boolean, graphql_name="eligible", default=None)), + ("type", sgqlc.types.Arg(DeliveryMethodDefinitionType, graphql_name="type", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(MethodDefinitionSortKeys, graphql_name="sortKey", default="ID")), + ) + ), + ) + zone = sgqlc.types.Field(sgqlc.types.non_null("DeliveryZone"), graphql_name="zone") + + +class DeliveryLocationGroupZoneConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryLocationGroupZoneEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DeliveryLocationGroupZone))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class DeliveryLocationGroupZoneEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(DeliveryLocationGroupZone), graphql_name="node") + + +class DeliveryMethodDefinitionConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryMethodDefinitionEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryMethodDefinition"))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class DeliveryMethodDefinitionCounts(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("participant_definitions_count", "rate_definitions_count") + participant_definitions_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="participantDefinitionsCount") + rate_definitions_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="rateDefinitionsCount") + + +class DeliveryMethodDefinitionEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("DeliveryMethodDefinition"), graphql_name="node") + + +class DeliveryParticipantService(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("active", "name") + active = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="active") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + + +class DeliveryProductVariantsCount(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("capped", "count") + capped = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="capped") + count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="count") + + +class DeliveryProfileConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryProfileEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryProfile"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class DeliveryProfileEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("DeliveryProfile"), graphql_name="node") + + +class DeliveryProfileItemConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryProfileItemEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryProfileItem"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class DeliveryProfileItemEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("DeliveryProfileItem"), graphql_name="node") + + +class DeliveryProfileLocationGroup(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("countries_in_any_zone", "location_group", "location_group_zones") + countries_in_any_zone = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DeliveryCountryAndZone))), graphql_name="countriesInAnyZone" + ) + location_group = sgqlc.types.Field(sgqlc.types.non_null("DeliveryLocationGroup"), graphql_name="locationGroup") + location_group_zones = sgqlc.types.Field( + sgqlc.types.non_null(DeliveryLocationGroupZoneConnection), + graphql_name="locationGroupZones", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + + +class DeliverySetting(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("legacy_mode_blocked", "legacy_mode_profiles") + legacy_mode_blocked = sgqlc.types.Field(sgqlc.types.non_null(DeliveryLegacyModeBlocked), graphql_name="legacyModeBlocked") + legacy_mode_profiles = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="legacyModeProfiles") + + +class DeliverySettingUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("setting", "user_errors") + setting = sgqlc.types.Field(DeliverySetting, graphql_name="setting") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class DeliveryShippingOriginAssignPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("user_errors",) + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class DiscountAllocation(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("allocated_amount_set", "discount_application") + allocated_amount_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="allocatedAmountSet") + discount_application = sgqlc.types.Field(sgqlc.types.non_null(DiscountApplication), graphql_name="discountApplication") + + +class DiscountAmount(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("amount", "applies_on_each_item") + amount = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="amount") + applies_on_each_item = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliesOnEachItem") + + +class DiscountApplicationConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountApplicationEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DiscountApplication))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class DiscountApplicationEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(DiscountApplication), graphql_name="node") + + +class DiscountAutomaticActivatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("automatic_discount_node", "user_errors") + automatic_discount_node = sgqlc.types.Field("DiscountAutomaticNode", graphql_name="automaticDiscountNode") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountAutomaticApp(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "app_discount_type", + "async_usage_count", + "combines_with", + "created_at", + "discount_class", + "discount_id", + "ends_at", + "error_history", + "starts_at", + "status", + "title", + "updated_at", + ) + app_discount_type = sgqlc.types.Field(sgqlc.types.non_null(AppDiscountType), graphql_name="appDiscountType") + async_usage_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="asyncUsageCount") + combines_with = sgqlc.types.Field(sgqlc.types.non_null("DiscountCombinesWith"), graphql_name="combinesWith") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + discount_class = sgqlc.types.Field(sgqlc.types.non_null(DiscountClass), graphql_name="discountClass") + discount_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="discountId") + ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") + error_history = sgqlc.types.Field("FunctionsErrorHistory", graphql_name="errorHistory") + starts_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="startsAt") + status = sgqlc.types.Field(sgqlc.types.non_null(DiscountStatus), graphql_name="status") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + + +class DiscountAutomaticAppCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("automatic_app_discount", "user_errors") + automatic_app_discount = sgqlc.types.Field(DiscountAutomaticApp, graphql_name="automaticAppDiscount") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountAutomaticAppUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("automatic_app_discount", "user_errors") + automatic_app_discount = sgqlc.types.Field(DiscountAutomaticApp, graphql_name="automaticAppDiscount") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountAutomaticBasic(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "async_usage_count", + "combines_with", + "created_at", + "customer_gets", + "discount_class", + "ends_at", + "minimum_requirement", + "short_summary", + "starts_at", + "status", + "summary", + "title", + "updated_at", + ) + async_usage_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="asyncUsageCount") + combines_with = sgqlc.types.Field(sgqlc.types.non_null("DiscountCombinesWith"), graphql_name="combinesWith") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + customer_gets = sgqlc.types.Field(sgqlc.types.non_null("DiscountCustomerGets"), graphql_name="customerGets") + discount_class = sgqlc.types.Field(sgqlc.types.non_null(MerchandiseDiscountClass), graphql_name="discountClass") + ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") + minimum_requirement = sgqlc.types.Field(sgqlc.types.non_null("DiscountMinimumRequirement"), graphql_name="minimumRequirement") + short_summary = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="shortSummary") + starts_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="startsAt") + status = sgqlc.types.Field(sgqlc.types.non_null(DiscountStatus), graphql_name="status") + summary = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="summary") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + + +class DiscountAutomaticBasicCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("automatic_discount_node", "user_errors") + automatic_discount_node = sgqlc.types.Field("DiscountAutomaticNode", graphql_name="automaticDiscountNode") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountAutomaticBasicUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("automatic_discount_node", "user_errors") + automatic_discount_node = sgqlc.types.Field("DiscountAutomaticNode", graphql_name="automaticDiscountNode") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountAutomaticBulkDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("job", "user_errors") + job = sgqlc.types.Field("Job", graphql_name="job") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountAutomaticBxgyCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("automatic_discount_node", "user_errors") + automatic_discount_node = sgqlc.types.Field("DiscountAutomaticNode", graphql_name="automaticDiscountNode") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountAutomaticBxgyUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("automatic_discount_node", "user_errors") + automatic_discount_node = sgqlc.types.Field("DiscountAutomaticNode", graphql_name="automaticDiscountNode") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountAutomaticConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountAutomaticEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountAutomatic"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class DiscountAutomaticDeactivatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("automatic_discount_node", "user_errors") + automatic_discount_node = sgqlc.types.Field("DiscountAutomaticNode", graphql_name="automaticDiscountNode") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountAutomaticDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_automatic_discount_id", "user_errors") + deleted_automatic_discount_id = sgqlc.types.Field(ID, graphql_name="deletedAutomaticDiscountId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountAutomaticEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("DiscountAutomatic"), graphql_name="node") + + +class DiscountAutomaticNodeConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountAutomaticNodeEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountAutomaticNode"))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class DiscountAutomaticNodeEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("DiscountAutomaticNode"), graphql_name="node") + + +class DiscountCodeActivatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("code_discount_node", "user_errors") + code_discount_node = sgqlc.types.Field("DiscountCodeNode", graphql_name="codeDiscountNode") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountCodeApp(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "app_discount_type", + "applies_once_per_customer", + "async_usage_count", + "code_count", + "codes", + "combines_with", + "created_at", + "customer_selection", + "discount_class", + "discount_id", + "ends_at", + "error_history", + "has_timeline_comment", + "recurring_cycle_limit", + "shareable_urls", + "starts_at", + "status", + "title", + "total_sales", + "updated_at", + "usage_limit", + ) + app_discount_type = sgqlc.types.Field(sgqlc.types.non_null(AppDiscountType), graphql_name="appDiscountType") + applies_once_per_customer = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliesOncePerCustomer") + async_usage_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="asyncUsageCount") + code_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="codeCount") + codes = sgqlc.types.Field( + sgqlc.types.non_null("DiscountRedeemCodeConnection"), + graphql_name="codes", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(DiscountCodeSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ) + ), + ) + combines_with = sgqlc.types.Field(sgqlc.types.non_null("DiscountCombinesWith"), graphql_name="combinesWith") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + customer_selection = sgqlc.types.Field(sgqlc.types.non_null("DiscountCustomerSelection"), graphql_name="customerSelection") + discount_class = sgqlc.types.Field(sgqlc.types.non_null(DiscountClass), graphql_name="discountClass") + discount_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="discountId") + ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") + error_history = sgqlc.types.Field("FunctionsErrorHistory", graphql_name="errorHistory") + has_timeline_comment = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasTimelineComment") + recurring_cycle_limit = sgqlc.types.Field(Int, graphql_name="recurringCycleLimit") + shareable_urls = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountShareableUrl"))), graphql_name="shareableUrls" + ) + starts_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="startsAt") + status = sgqlc.types.Field(sgqlc.types.non_null(DiscountStatus), graphql_name="status") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + total_sales = sgqlc.types.Field("MoneyV2", graphql_name="totalSales") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + usage_limit = sgqlc.types.Field(Int, graphql_name="usageLimit") + + +class DiscountCodeAppCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("code_app_discount", "user_errors") + code_app_discount = sgqlc.types.Field(DiscountCodeApp, graphql_name="codeAppDiscount") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountCodeAppUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("code_app_discount", "user_errors") + code_app_discount = sgqlc.types.Field(DiscountCodeApp, graphql_name="codeAppDiscount") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountCodeBasic(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "applies_once_per_customer", + "async_usage_count", + "code_count", + "codes", + "combines_with", + "created_at", + "customer_gets", + "customer_selection", + "discount_class", + "ends_at", + "has_timeline_comment", + "minimum_requirement", + "recurring_cycle_limit", + "shareable_urls", + "short_summary", + "starts_at", + "status", + "summary", + "title", + "total_sales", + "updated_at", + "usage_limit", + ) + applies_once_per_customer = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliesOncePerCustomer") + async_usage_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="asyncUsageCount") + code_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="codeCount") + codes = sgqlc.types.Field( + sgqlc.types.non_null("DiscountRedeemCodeConnection"), + graphql_name="codes", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(DiscountCodeSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ) + ), + ) + combines_with = sgqlc.types.Field(sgqlc.types.non_null("DiscountCombinesWith"), graphql_name="combinesWith") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + customer_gets = sgqlc.types.Field(sgqlc.types.non_null("DiscountCustomerGets"), graphql_name="customerGets") + customer_selection = sgqlc.types.Field(sgqlc.types.non_null("DiscountCustomerSelection"), graphql_name="customerSelection") + discount_class = sgqlc.types.Field(sgqlc.types.non_null(MerchandiseDiscountClass), graphql_name="discountClass") + ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") + has_timeline_comment = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasTimelineComment") + minimum_requirement = sgqlc.types.Field("DiscountMinimumRequirement", graphql_name="minimumRequirement") + recurring_cycle_limit = sgqlc.types.Field(Int, graphql_name="recurringCycleLimit") + shareable_urls = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountShareableUrl"))), graphql_name="shareableUrls" + ) + short_summary = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="shortSummary") + starts_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="startsAt") + status = sgqlc.types.Field(sgqlc.types.non_null(DiscountStatus), graphql_name="status") + summary = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="summary") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + total_sales = sgqlc.types.Field("MoneyV2", graphql_name="totalSales") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + usage_limit = sgqlc.types.Field(Int, graphql_name="usageLimit") + + +class DiscountCodeBasicCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("code_discount_node", "user_errors") + code_discount_node = sgqlc.types.Field("DiscountCodeNode", graphql_name="codeDiscountNode") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountCodeBasicUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("code_discount_node", "user_errors") + code_discount_node = sgqlc.types.Field("DiscountCodeNode", graphql_name="codeDiscountNode") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountCodeBulkActivatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("job", "user_errors") + job = sgqlc.types.Field("Job", graphql_name="job") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountCodeBulkDeactivatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("job", "user_errors") + job = sgqlc.types.Field("Job", graphql_name="job") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountCodeBulkDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("job", "user_errors") + job = sgqlc.types.Field("Job", graphql_name="job") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountCodeBxgy(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "applies_once_per_customer", + "async_usage_count", + "code_count", + "codes", + "combines_with", + "created_at", + "customer_buys", + "customer_gets", + "customer_selection", + "discount_class", + "ends_at", + "has_timeline_comment", + "shareable_urls", + "starts_at", + "status", + "summary", + "title", + "total_sales", + "updated_at", + "usage_limit", + "uses_per_order_limit", + ) + applies_once_per_customer = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliesOncePerCustomer") + async_usage_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="asyncUsageCount") + code_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="codeCount") + codes = sgqlc.types.Field( + sgqlc.types.non_null("DiscountRedeemCodeConnection"), + graphql_name="codes", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(DiscountCodeSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ) + ), + ) + combines_with = sgqlc.types.Field(sgqlc.types.non_null("DiscountCombinesWith"), graphql_name="combinesWith") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + customer_buys = sgqlc.types.Field(sgqlc.types.non_null("DiscountCustomerBuys"), graphql_name="customerBuys") + customer_gets = sgqlc.types.Field(sgqlc.types.non_null("DiscountCustomerGets"), graphql_name="customerGets") + customer_selection = sgqlc.types.Field(sgqlc.types.non_null("DiscountCustomerSelection"), graphql_name="customerSelection") + discount_class = sgqlc.types.Field(sgqlc.types.non_null(MerchandiseDiscountClass), graphql_name="discountClass") + ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") + has_timeline_comment = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasTimelineComment") + shareable_urls = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountShareableUrl"))), graphql_name="shareableUrls" + ) + starts_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="startsAt") + status = sgqlc.types.Field(sgqlc.types.non_null(DiscountStatus), graphql_name="status") + summary = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="summary") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + total_sales = sgqlc.types.Field("MoneyV2", graphql_name="totalSales") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + usage_limit = sgqlc.types.Field(Int, graphql_name="usageLimit") + uses_per_order_limit = sgqlc.types.Field(Int, graphql_name="usesPerOrderLimit") + + +class DiscountCodeBxgyCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("code_discount_node", "user_errors") + code_discount_node = sgqlc.types.Field("DiscountCodeNode", graphql_name="codeDiscountNode") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountCodeBxgyUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("code_discount_node", "user_errors") + code_discount_node = sgqlc.types.Field("DiscountCodeNode", graphql_name="codeDiscountNode") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountCodeDeactivatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("code_discount_node", "user_errors") + code_discount_node = sgqlc.types.Field("DiscountCodeNode", graphql_name="codeDiscountNode") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountCodeDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_code_discount_id", "user_errors") + deleted_code_discount_id = sgqlc.types.Field(ID, graphql_name="deletedCodeDiscountId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountCodeFreeShipping(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "applies_on_one_time_purchase", + "applies_on_subscription", + "applies_once_per_customer", + "async_usage_count", + "code_count", + "codes", + "combines_with", + "created_at", + "customer_selection", + "destination_selection", + "discount_class", + "ends_at", + "has_timeline_comment", + "maximum_shipping_price", + "minimum_requirement", + "recurring_cycle_limit", + "shareable_urls", + "short_summary", + "starts_at", + "status", + "summary", + "title", + "total_sales", + "updated_at", + "usage_limit", + ) + applies_on_one_time_purchase = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliesOnOneTimePurchase") + applies_on_subscription = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliesOnSubscription") + applies_once_per_customer = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliesOncePerCustomer") + async_usage_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="asyncUsageCount") + code_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="codeCount") + codes = sgqlc.types.Field( + sgqlc.types.non_null("DiscountRedeemCodeConnection"), + graphql_name="codes", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(DiscountCodeSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ) + ), + ) + combines_with = sgqlc.types.Field(sgqlc.types.non_null("DiscountCombinesWith"), graphql_name="combinesWith") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + customer_selection = sgqlc.types.Field(sgqlc.types.non_null("DiscountCustomerSelection"), graphql_name="customerSelection") + destination_selection = sgqlc.types.Field( + sgqlc.types.non_null("DiscountShippingDestinationSelection"), graphql_name="destinationSelection" + ) + discount_class = sgqlc.types.Field(sgqlc.types.non_null(ShippingDiscountClass), graphql_name="discountClass") + ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") + has_timeline_comment = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasTimelineComment") + maximum_shipping_price = sgqlc.types.Field("MoneyV2", graphql_name="maximumShippingPrice") + minimum_requirement = sgqlc.types.Field("DiscountMinimumRequirement", graphql_name="minimumRequirement") + recurring_cycle_limit = sgqlc.types.Field(Int, graphql_name="recurringCycleLimit") + shareable_urls = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountShareableUrl"))), graphql_name="shareableUrls" + ) + short_summary = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="shortSummary") + starts_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="startsAt") + status = sgqlc.types.Field(sgqlc.types.non_null(DiscountStatus), graphql_name="status") + summary = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="summary") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + total_sales = sgqlc.types.Field("MoneyV2", graphql_name="totalSales") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + usage_limit = sgqlc.types.Field(Int, graphql_name="usageLimit") + + +class DiscountCodeFreeShippingCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("code_discount_node", "user_errors") + code_discount_node = sgqlc.types.Field("DiscountCodeNode", graphql_name="codeDiscountNode") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountCodeFreeShippingUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("code_discount_node", "user_errors") + code_discount_node = sgqlc.types.Field("DiscountCodeNode", graphql_name="codeDiscountNode") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountCodeNodeConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountCodeNodeEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountCodeNode"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class DiscountCodeNodeEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("DiscountCodeNode"), graphql_name="node") + + +class DiscountCodeRedeemCodeBulkDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("job", "user_errors") + job = sgqlc.types.Field("Job", graphql_name="job") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountCollections(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("collections",) + collections = sgqlc.types.Field( + sgqlc.types.non_null(CollectionConnection), + graphql_name="collections", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + + +class DiscountCombinesWith(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("order_discounts", "product_discounts", "shipping_discounts") + order_discounts = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="orderDiscounts") + product_discounts = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="productDiscounts") + shipping_discounts = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="shippingDiscounts") + + +class DiscountCountries(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("countries", "include_rest_of_world") + countries = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CountryCode))), graphql_name="countries") + include_rest_of_world = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="includeRestOfWorld") + + +class DiscountCountryAll(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("all_countries",) + all_countries = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="allCountries") + + +class DiscountCustomerAll(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("all_customers",) + all_customers = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="allCustomers") + + +class DiscountCustomerBuys(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("items", "value") + items = sgqlc.types.Field(sgqlc.types.non_null("DiscountItems"), graphql_name="items") + value = sgqlc.types.Field(sgqlc.types.non_null("DiscountCustomerBuysValue"), graphql_name="value") + + +class DiscountCustomerGets(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("applies_on_one_time_purchase", "applies_on_subscription", "items", "value") + applies_on_one_time_purchase = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliesOnOneTimePurchase") + applies_on_subscription = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliesOnSubscription") + items = sgqlc.types.Field(sgqlc.types.non_null("DiscountItems"), graphql_name="items") + value = sgqlc.types.Field(sgqlc.types.non_null("DiscountCustomerGetsValue"), graphql_name="value") + + +class DiscountCustomerSegments(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("segments",) + segments = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Segment"))), graphql_name="segments") + + +class DiscountCustomers(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("customers",) + customers = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Customer"))), graphql_name="customers") + + +class DiscountMinimumQuantity(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("greater_than_or_equal_to_quantity",) + greater_than_or_equal_to_quantity = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="greaterThanOrEqualToQuantity") + + +class DiscountMinimumSubtotal(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("greater_than_or_equal_to_subtotal",) + greater_than_or_equal_to_subtotal = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="greaterThanOrEqualToSubtotal") + + +class DiscountNodeConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountNodeEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountNode"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class DiscountNodeEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("DiscountNode"), graphql_name="node") + + +class DiscountOnQuantity(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("effect", "quantity") + effect = sgqlc.types.Field(sgqlc.types.non_null("DiscountEffect"), graphql_name="effect") + quantity = sgqlc.types.Field(sgqlc.types.non_null("DiscountQuantity"), graphql_name="quantity") + + +class DiscountPercentage(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("percentage",) + percentage = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="percentage") + + +class DiscountProducts(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("product_variants", "products") + product_variants = sgqlc.types.Field( + sgqlc.types.non_null("ProductVariantConnection"), + graphql_name="productVariants", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + products = sgqlc.types.Field( + sgqlc.types.non_null("ProductConnection"), + graphql_name="products", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + + +class DiscountPurchaseAmount(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("amount",) + amount = sgqlc.types.Field(sgqlc.types.non_null(Decimal), graphql_name="amount") + + +class DiscountQuantity(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("quantity",) + quantity = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="quantity") + + +class DiscountRedeemCode(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("async_usage_count", "code", "created_by", "id") + async_usage_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="asyncUsageCount") + code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") + created_by = sgqlc.types.Field("App", graphql_name="createdBy") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + +class DiscountRedeemCodeBulkAddPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("bulk_creation", "user_errors") + bulk_creation = sgqlc.types.Field("DiscountRedeemCodeBulkCreation", graphql_name="bulkCreation") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" + ) + + +class DiscountRedeemCodeBulkCreationCode(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("code", "discount_redeem_code", "errors") + code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") + discount_redeem_code = sgqlc.types.Field(DiscountRedeemCode, graphql_name="discountRedeemCode") + errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="errors") + + +class DiscountRedeemCodeBulkCreationCodeConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountRedeemCodeBulkCreationCodeEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DiscountRedeemCodeBulkCreationCode))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class DiscountRedeemCodeBulkCreationCodeEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(DiscountRedeemCodeBulkCreationCode), graphql_name="node") + + +class DiscountRedeemCodeConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountRedeemCodeEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DiscountRedeemCode))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class DiscountRedeemCodeEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(DiscountRedeemCode), graphql_name="node") + + +class DiscountShareableUrl(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("target_item_image", "target_type", "title", "url") + target_item_image = sgqlc.types.Field("Image", graphql_name="targetItemImage") + target_type = sgqlc.types.Field(sgqlc.types.non_null(DiscountShareableUrlTargetType), graphql_name="targetType") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") + + +class DisputeEvidenceUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("dispute_evidence", "user_errors") + dispute_evidence = sgqlc.types.Field("ShopifyPaymentsDisputeEvidence", graphql_name="disputeEvidence") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DisputeEvidenceUpdateUserError"))), graphql_name="userErrors" + ) + + +class DomainLocalization(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("alternate_locales", "country", "default_locale") + alternate_locales = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="alternateLocales" + ) + country = sgqlc.types.Field(String, graphql_name="country") + default_locale = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="defaultLocale") + + +class DraftOrderAppliedDiscount(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("amount_set", "amount_v2", "description", "title", "value", "value_type") + amount_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="amountSet") + amount_v2 = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="amountV2") + description = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="description") + title = sgqlc.types.Field(String, graphql_name="title") + value = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="value") + value_type = sgqlc.types.Field(sgqlc.types.non_null(DraftOrderAppliedDiscountType), graphql_name="valueType") + + +class DraftOrderBulkAddTagsPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("job", "user_errors") + job = sgqlc.types.Field("Job", graphql_name="job") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class DraftOrderBulkDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("job", "user_errors") + job = sgqlc.types.Field("Job", graphql_name="job") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class DraftOrderBulkRemoveTagsPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("job", "user_errors") + job = sgqlc.types.Field("Job", graphql_name="job") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class DraftOrderCalculatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("calculated_draft_order", "user_errors") + calculated_draft_order = sgqlc.types.Field(CalculatedDraftOrder, graphql_name="calculatedDraftOrder") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class DraftOrderCompletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("draft_order", "user_errors") + draft_order = sgqlc.types.Field("DraftOrder", graphql_name="draftOrder") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class DraftOrderConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DraftOrderEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DraftOrder"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class DraftOrderCreateFromOrderPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("draft_order", "user_errors") + draft_order = sgqlc.types.Field("DraftOrder", graphql_name="draftOrder") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class DraftOrderCreateMerchantCheckoutPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("user_errors",) + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class DraftOrderCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("draft_order", "user_errors") + draft_order = sgqlc.types.Field("DraftOrder", graphql_name="draftOrder") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class DraftOrderDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_id", "user_errors") + deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class DraftOrderDuplicatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("draft_order", "user_errors") + draft_order = sgqlc.types.Field("DraftOrder", graphql_name="draftOrder") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class DraftOrderEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("DraftOrder"), graphql_name="node") + + +class DraftOrderInvoicePreviewPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("preview_html", "preview_subject", "user_errors") + preview_html = sgqlc.types.Field(HTML, graphql_name="previewHtml") + preview_subject = sgqlc.types.Field(HTML, graphql_name="previewSubject") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class DraftOrderInvoiceSendPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("draft_order", "user_errors") + draft_order = sgqlc.types.Field("DraftOrder", graphql_name="draftOrder") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class DraftOrderLineItemConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DraftOrderLineItemEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DraftOrderLineItem"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class DraftOrderLineItemEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("DraftOrderLineItem"), graphql_name="node") + + +class DraftOrderUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("draft_order", "user_errors") + draft_order = sgqlc.types.Field("DraftOrder", graphql_name="draftOrder") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class EditableProperty(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("locked", "reason") + locked = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="locked") + reason = sgqlc.types.Field(FormattedString, graphql_name="reason") + + +class ErrorPosition(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("character", "line") + character = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="character") + line = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="line") + + +class EventBridgeServerPixelUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("server_pixel", "user_errors") + server_pixel = sgqlc.types.Field("ServerPixel", graphql_name="serverPixel") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ErrorsServerPixelUserError"))), graphql_name="userErrors" + ) + + +class EventBridgeWebhookSubscriptionCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("user_errors", "webhook_subscription") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + webhook_subscription = sgqlc.types.Field("WebhookSubscription", graphql_name="webhookSubscription") + + +class EventBridgeWebhookSubscriptionUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("user_errors", "webhook_subscription") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + webhook_subscription = sgqlc.types.Field("WebhookSubscription", graphql_name="webhookSubscription") + + +class EventConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("EventEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Event))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class EventEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(Event), graphql_name="node") + + +class ExchangeV2Additions(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("line_items", "subtotal_price_set", "tax_lines", "total_price_set") + line_items = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ExchangeV2LineItem"))), graphql_name="lineItems" + ) + subtotal_price_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="subtotalPriceSet") + tax_lines = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TaxLine"))), graphql_name="taxLines") + total_price_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="totalPriceSet") + + +class ExchangeV2Connection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ExchangeV2Edge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ExchangeV2"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class ExchangeV2Edge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("ExchangeV2"), graphql_name="node") + + +class ExchangeV2LineItem(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "custom_attributes", + "discounted_total_set", + "discounted_unit_price_set", + "fulfillment_service", + "gift_card", + "gift_cards", + "line_item", + "name", + "original_total_set", + "original_unit_price_set", + "quantity", + "requires_shipping", + "sku", + "tax_lines", + "taxable", + "title", + "variant", + "variant_title", + "vendor", + ) + custom_attributes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Attribute))), graphql_name="customAttributes" + ) + discounted_total_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="discountedTotalSet") + discounted_unit_price_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="discountedUnitPriceSet") + fulfillment_service = sgqlc.types.Field("FulfillmentService", graphql_name="fulfillmentService") + gift_card = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="giftCard") + gift_cards = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("GiftCard"))), graphql_name="giftCards") + line_item = sgqlc.types.Field("LineItem", graphql_name="lineItem") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + original_total_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="originalTotalSet") + original_unit_price_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="originalUnitPriceSet") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + requires_shipping = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresShipping") + sku = sgqlc.types.Field(String, graphql_name="sku") + tax_lines = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TaxLine"))), graphql_name="taxLines") + taxable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxable") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + variant = sgqlc.types.Field("ProductVariant", graphql_name="variant") + variant_title = sgqlc.types.Field(String, graphql_name="variantTitle") + vendor = sgqlc.types.Field(String, graphql_name="vendor") + + +class ExchangeV2Returns(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "line_items", + "order_discount_amount_set", + "shipping_refund_amount_set", + "subtotal_price_set", + "tax_lines", + "tip_refund_amount_set", + "total_price_set", + ) + line_items = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ExchangeV2LineItem))), graphql_name="lineItems" + ) + order_discount_amount_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="orderDiscountAmountSet") + shipping_refund_amount_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="shippingRefundAmountSet") + subtotal_price_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="subtotalPriceSet") + tax_lines = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TaxLine"))), graphql_name="taxLines") + tip_refund_amount_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="tipRefundAmountSet") + total_price_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="totalPriceSet") + + +class FailedRequirement(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("action", "message") + action = sgqlc.types.Field("NavigationItem", graphql_name="action") + message = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="message") + + +class FileAcknowledgeUpdateFailedPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("files", "user_errors") + files = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(File)), graphql_name="files") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FilesUserError"))), graphql_name="userErrors" + ) + + +class FileConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FileEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(File))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class FileCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("files", "user_errors") + files = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(File)), graphql_name="files") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FilesUserError"))), graphql_name="userErrors" + ) + + +class FileDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_file_ids", "user_errors") + deleted_file_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="deletedFileIds") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FilesUserError"))), graphql_name="userErrors" + ) + + +class FileEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(File), graphql_name="node") + + +class FileError(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("code", "details", "message") + code = sgqlc.types.Field(sgqlc.types.non_null(FileErrorCode), graphql_name="code") + details = sgqlc.types.Field(String, graphql_name="details") + message = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="message") + + +class FileUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("files", "user_errors") + files = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(File)), graphql_name="files") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FilesUserError"))), graphql_name="userErrors" + ) + + +class FilterOption(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("label", "value") + label = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="label") + value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") + + +class FlowTriggerReceivePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("user_errors",) + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class FulfillmentCancelPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("fulfillment", "user_errors") + fulfillment = sgqlc.types.Field("Fulfillment", graphql_name="fulfillment") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class FulfillmentConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Fulfillment"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class FulfillmentCreateV2Payload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("fulfillment", "user_errors") + fulfillment = sgqlc.types.Field("Fulfillment", graphql_name="fulfillment") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class FulfillmentEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("Fulfillment"), graphql_name="node") + + +class FulfillmentEventConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentEventEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentEvent"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class FulfillmentEventCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_event", "user_errors") + fulfillment_event = sgqlc.types.Field("FulfillmentEvent", graphql_name="fulfillmentEvent") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class FulfillmentEventEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("FulfillmentEvent"), graphql_name="node") + + +class FulfillmentHold(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("held_by", "reason", "reason_notes") + held_by = sgqlc.types.Field(String, graphql_name="heldBy") + reason = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentHoldReason), graphql_name="reason") + reason_notes = sgqlc.types.Field(String, graphql_name="reasonNotes") + + +class FulfillmentLineItemConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentLineItemEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentLineItem"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class FulfillmentLineItemEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("FulfillmentLineItem"), graphql_name="node") + + +class FulfillmentOrderAcceptCancellationRequestPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_order", "user_errors") + fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="fulfillmentOrder") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class FulfillmentOrderAcceptFulfillmentRequestPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_order", "user_errors") + fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="fulfillmentOrder") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class FulfillmentOrderAssignedLocation(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("address1", "address2", "city", "country_code", "location", "name", "phone", "province", "zip") + address1 = sgqlc.types.Field(String, graphql_name="address1") + address2 = sgqlc.types.Field(String, graphql_name="address2") + city = sgqlc.types.Field(String, graphql_name="city") + country_code = sgqlc.types.Field(sgqlc.types.non_null(CountryCode), graphql_name="countryCode") + location = sgqlc.types.Field("Location", graphql_name="location") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + phone = sgqlc.types.Field(String, graphql_name="phone") + province = sgqlc.types.Field(String, graphql_name="province") + zip = sgqlc.types.Field(String, graphql_name="zip") + + +class FulfillmentOrderCancelPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_order", "replacement_fulfillment_order", "user_errors") + fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="fulfillmentOrder") + replacement_fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="replacementFulfillmentOrder") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class FulfillmentOrderClosePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_order", "user_errors") + fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="fulfillmentOrder") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class FulfillmentOrderConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrder"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class FulfillmentOrderEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("FulfillmentOrder"), graphql_name="node") + + +class FulfillmentOrderHoldPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_order", "remaining_fulfillment_order", "user_errors") + fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="fulfillmentOrder") + remaining_fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="remainingFulfillmentOrder") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderHoldUserError"))), graphql_name="userErrors" + ) + + +class FulfillmentOrderInternationalDuties(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("incoterm",) + incoterm = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="incoterm") + + +class FulfillmentOrderLineItemConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderLineItemEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderLineItem"))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class FulfillmentOrderLineItemEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("FulfillmentOrderLineItem"), graphql_name="node") + + +class FulfillmentOrderLineItemWarning(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("description", "title") + description = sgqlc.types.Field(String, graphql_name="description") + title = sgqlc.types.Field(String, graphql_name="title") + + +class FulfillmentOrderLineItemsPreparedForPickupPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("user_errors",) + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderLineItemsPreparedForPickupUserError"))), + graphql_name="userErrors", + ) + + +class FulfillmentOrderLocationForMove(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("location", "message", "movable") + location = sgqlc.types.Field(sgqlc.types.non_null("Location"), graphql_name="location") + message = sgqlc.types.Field(String, graphql_name="message") + movable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="movable") + + +class FulfillmentOrderLocationForMoveConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderLocationForMoveEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FulfillmentOrderLocationForMove))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class FulfillmentOrderLocationForMoveEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentOrderLocationForMove), graphql_name="node") + + +class FulfillmentOrderMerchantRequestConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderMerchantRequestEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderMerchantRequest"))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class FulfillmentOrderMerchantRequestEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("FulfillmentOrderMerchantRequest"), graphql_name="node") + + +class FulfillmentOrderMergePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_order_merges", "user_errors") + fulfillment_order_merges = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderMergeResult")), graphql_name="fulfillmentOrderMerges" + ) + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderMergeUserError"))), graphql_name="userErrors" + ) + + +class FulfillmentOrderMergeResult(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_order",) + fulfillment_order = sgqlc.types.Field(sgqlc.types.non_null("FulfillmentOrder"), graphql_name="fulfillmentOrder") + + +class FulfillmentOrderMovePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("moved_fulfillment_order", "original_fulfillment_order", "remaining_fulfillment_order", "user_errors") + moved_fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="movedFulfillmentOrder") + original_fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="originalFulfillmentOrder") + remaining_fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="remainingFulfillmentOrder") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class FulfillmentOrderOpenPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_order", "user_errors") + fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="fulfillmentOrder") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class FulfillmentOrderRejectCancellationRequestPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_order", "user_errors") + fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="fulfillmentOrder") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class FulfillmentOrderRejectFulfillmentRequestPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_order", "user_errors") + fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="fulfillmentOrder") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class FulfillmentOrderReleaseHoldPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_order", "user_errors") + fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="fulfillmentOrder") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderReleaseHoldUserError"))), graphql_name="userErrors" + ) + + +class FulfillmentOrderReschedulePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_order", "user_errors") + fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="fulfillmentOrder") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderRescheduleUserError"))), graphql_name="userErrors" + ) + + +class FulfillmentOrderSplitPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_order_splits", "user_errors") + fulfillment_order_splits = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderSplitResult")), graphql_name="fulfillmentOrderSplits" + ) + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderSplitUserError"))), graphql_name="userErrors" + ) + + +class FulfillmentOrderSplitResult(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_order", "remaining_fulfillment_order", "replacement_fulfillment_order") + fulfillment_order = sgqlc.types.Field(sgqlc.types.non_null("FulfillmentOrder"), graphql_name="fulfillmentOrder") + remaining_fulfillment_order = sgqlc.types.Field(sgqlc.types.non_null("FulfillmentOrder"), graphql_name="remainingFulfillmentOrder") + replacement_fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="replacementFulfillmentOrder") + + +class FulfillmentOrderSubmitCancellationRequestPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_order", "user_errors") + fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="fulfillmentOrder") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class FulfillmentOrderSubmitFulfillmentRequestPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("original_fulfillment_order", "submitted_fulfillment_order", "unsubmitted_fulfillment_order", "user_errors") + original_fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="originalFulfillmentOrder") + submitted_fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="submittedFulfillmentOrder") + unsubmitted_fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="unsubmittedFulfillmentOrder") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class FulfillmentOrderSupportedAction(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("action", "external_url") + action = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentOrderAction), graphql_name="action") + external_url = sgqlc.types.Field(URL, graphql_name="externalUrl") + + +class FulfillmentOrdersReleaseHoldsPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("job", "user_errors") + job = sgqlc.types.Field("Job", graphql_name="job") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrdersReleaseHoldsUserError"))), graphql_name="userErrors" + ) + + +class FulfillmentOrdersSetFulfillmentDeadlinePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("success", "user_errors") + success = sgqlc.types.Field(Boolean, graphql_name="success") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrdersSetFulfillmentDeadlineUserError"))), + graphql_name="userErrors", + ) + + +class FulfillmentOriginAddress(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("address1", "address2", "city", "country_code", "province_code", "zip") + address1 = sgqlc.types.Field(String, graphql_name="address1") + address2 = sgqlc.types.Field(String, graphql_name="address2") + city = sgqlc.types.Field(String, graphql_name="city") + country_code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="countryCode") + province_code = sgqlc.types.Field(String, graphql_name="provinceCode") + zip = sgqlc.types.Field(String, graphql_name="zip") + + +class FulfillmentService(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "callback_url", + "fulfillment_orders_opt_in", + "handle", + "id", + "inventory_management", + "location", + "permits_sku_sharing", + "product_based", + "service_name", + "type", + ) + callback_url = sgqlc.types.Field(URL, graphql_name="callbackUrl") + fulfillment_orders_opt_in = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="fulfillmentOrdersOptIn") + handle = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="handle") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + inventory_management = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="inventoryManagement") + location = sgqlc.types.Field("Location", graphql_name="location") + permits_sku_sharing = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="permitsSkuSharing") + product_based = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="productBased") + service_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="serviceName") + type = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentServiceType), graphql_name="type") + + +class FulfillmentServiceCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_service", "user_errors") + fulfillment_service = sgqlc.types.Field(FulfillmentService, graphql_name="fulfillmentService") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class FulfillmentServiceDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_id", "user_errors") + deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class FulfillmentServiceUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_service", "user_errors") + fulfillment_service = sgqlc.types.Field(FulfillmentService, graphql_name="fulfillmentService") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class FulfillmentTrackingInfo(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("company", "number", "url") + company = sgqlc.types.Field(String, graphql_name="company") + number = sgqlc.types.Field(String, graphql_name="number") + url = sgqlc.types.Field(URL, graphql_name="url") + + +class FulfillmentTrackingInfoUpdateV2Payload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("fulfillment", "user_errors") + fulfillment = sgqlc.types.Field("Fulfillment", graphql_name="fulfillment") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class FunctionsAppBridge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("create_path", "details_path") + create_path = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="createPath") + details_path = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="detailsPath") + + +class FunctionsErrorHistory(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("errors_first_occurred_at", "first_occurred_at", "has_been_shared_since_last_error", "has_shared_recent_errors") + errors_first_occurred_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="errorsFirstOccurredAt") + first_occurred_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="firstOccurredAt") + has_been_shared_since_last_error = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasBeenSharedSinceLastError") + has_shared_recent_errors = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasSharedRecentErrors") + + +class GiftCardConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("GiftCardEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("GiftCard"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class GiftCardCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("gift_card", "gift_card_code", "user_errors") + gift_card = sgqlc.types.Field("GiftCard", graphql_name="giftCard") + gift_card_code = sgqlc.types.Field(String, graphql_name="giftCardCode") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("GiftCardUserError"))), graphql_name="userErrors" + ) + + +class GiftCardDisablePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("gift_card", "user_errors") + gift_card = sgqlc.types.Field("GiftCard", graphql_name="giftCard") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class GiftCardEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("GiftCard"), graphql_name="node") + + +class GiftCardUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("gift_card", "user_errors") + gift_card = sgqlc.types.Field("GiftCard", graphql_name="giftCard") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ImageConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ImageEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Image"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class ImageEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("Image"), graphql_name="node") + + +class ImageUploadParameter(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("name", "value") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") + + +class InventoryActivatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("inventory_level", "user_errors") + inventory_level = sgqlc.types.Field("InventoryLevel", graphql_name="inventoryLevel") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class InventoryAdjustQuantitiesPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("inventory_adjustment_group", "user_errors") + inventory_adjustment_group = sgqlc.types.Field("InventoryAdjustmentGroup", graphql_name="inventoryAdjustmentGroup") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("InventoryAdjustQuantitiesUserError"))), graphql_name="userErrors" + ) + + +class InventoryAdjustQuantityPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("inventory_level", "user_errors") + inventory_level = sgqlc.types.Field("InventoryLevel", graphql_name="inventoryLevel") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class InventoryBulkAdjustQuantityAtLocationPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("inventory_levels", "user_errors") + inventory_levels = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("InventoryLevel")), graphql_name="inventoryLevels") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class InventoryBulkToggleActivationPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("inventory_item", "inventory_levels", "user_errors") + inventory_item = sgqlc.types.Field("InventoryItem", graphql_name="inventoryItem") + inventory_levels = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("InventoryLevel")), graphql_name="inventoryLevels") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("InventoryBulkToggleActivationUserError"))), graphql_name="userErrors" + ) + + +class InventoryChange(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("delta", "item", "ledger_document_uri", "location", "name", "quantity_after_change") + delta = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="delta") + item = sgqlc.types.Field("InventoryItem", graphql_name="item") + ledger_document_uri = sgqlc.types.Field(String, graphql_name="ledgerDocumentUri") + location = sgqlc.types.Field("Location", graphql_name="location") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + quantity_after_change = sgqlc.types.Field(Int, graphql_name="quantityAfterChange") + + +class InventoryDeactivatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("user_errors",) + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class InventoryItemConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("InventoryItemEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("InventoryItem"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class InventoryItemEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("InventoryItem"), graphql_name="node") + + +class InventoryItemUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("inventory_item", "user_errors") + inventory_item = sgqlc.types.Field("InventoryItem", graphql_name="inventoryItem") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class InventoryLevelConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("InventoryLevelEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("InventoryLevel"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class InventoryLevelEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("InventoryLevel"), graphql_name="node") + + +class InventoryMoveQuantitiesPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("inventory_adjustment_group", "user_errors") + inventory_adjustment_group = sgqlc.types.Field("InventoryAdjustmentGroup", graphql_name="inventoryAdjustmentGroup") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("InventoryMoveQuantitiesUserError"))), graphql_name="userErrors" + ) + + +class InventoryProperties(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("quantity_names",) + quantity_names = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("InventoryQuantityName"))), graphql_name="quantityNames" + ) + + +class InventoryQuantity(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("name", "quantity", "updated_at") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + updated_at = sgqlc.types.Field(DateTime, graphql_name="updatedAt") + + +class InventoryQuantityName(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("belongs_to", "comprises", "display_name", "is_in_use", "name") + belongs_to = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="belongsTo") + comprises = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="comprises") + display_name = sgqlc.types.Field(String, graphql_name="displayName") + is_in_use = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isInUse") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + + +class InventorySetOnHandQuantitiesPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("inventory_adjustment_group", "user_errors") + inventory_adjustment_group = sgqlc.types.Field("InventoryAdjustmentGroup", graphql_name="inventoryAdjustmentGroup") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("InventorySetOnHandQuantitiesUserError"))), graphql_name="userErrors" + ) + + +class Job(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("done", "id", "query") + done = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="done") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + query = sgqlc.types.Field("QueryRoot", graphql_name="query") + + +class LimitedPendingOrderCount(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("at_max", "count") + at_max = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="atMax") + count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="count") + + +class LineItemConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("LineItemEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("LineItem"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class LineItemEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("LineItem"), graphql_name="node") + + +class LineItemGroup(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("id", "quantity", "title") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + + +class LineItemMutableConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("LineItemMutableEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("LineItemMutable"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class LineItemMutableEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("LineItemMutable"), graphql_name="node") + + +class LineItemSellingPlan(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("name", "selling_plan_id") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + selling_plan_id = sgqlc.types.Field(ID, graphql_name="sellingPlanId") + + +class Locale(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("iso_code", "name") + iso_code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="isoCode") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + + +class LocalizationExtension(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("country_code", "key", "purpose", "title", "value") + country_code = sgqlc.types.Field(sgqlc.types.non_null(CountryCode), graphql_name="countryCode") + key = sgqlc.types.Field(sgqlc.types.non_null(LocalizationExtensionKey), graphql_name="key") + purpose = sgqlc.types.Field(sgqlc.types.non_null(LocalizationExtensionPurpose), graphql_name="purpose") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") + + +class LocalizationExtensionConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("LocalizationExtensionEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(LocalizationExtension))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class LocalizationExtensionEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(LocalizationExtension), graphql_name="node") + + +class LocationActivatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("location", "location_activate_user_errors") + location = sgqlc.types.Field("Location", graphql_name="location") + location_activate_user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("LocationActivateUserError"))), + graphql_name="locationActivateUserErrors", + ) + + +class LocationAddPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("location", "user_errors") + location = sgqlc.types.Field("Location", graphql_name="location") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("LocationAddUserError"))), graphql_name="userErrors" + ) + + +class LocationAddress(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "address1", + "address2", + "city", + "country", + "country_code", + "formatted", + "latitude", + "longitude", + "phone", + "province", + "province_code", + "zip", + ) + address1 = sgqlc.types.Field(String, graphql_name="address1") + address2 = sgqlc.types.Field(String, graphql_name="address2") + city = sgqlc.types.Field(String, graphql_name="city") + country = sgqlc.types.Field(String, graphql_name="country") + country_code = sgqlc.types.Field(String, graphql_name="countryCode") + formatted = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="formatted") + latitude = sgqlc.types.Field(Float, graphql_name="latitude") + longitude = sgqlc.types.Field(Float, graphql_name="longitude") + phone = sgqlc.types.Field(String, graphql_name="phone") + province = sgqlc.types.Field(String, graphql_name="province") + province_code = sgqlc.types.Field(String, graphql_name="provinceCode") + zip = sgqlc.types.Field(String, graphql_name="zip") + + +class LocationConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("LocationEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Location"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class LocationDeactivatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("location", "location_deactivate_user_errors") + location = sgqlc.types.Field("Location", graphql_name="location") + location_deactivate_user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("LocationDeactivateUserError"))), + graphql_name="locationDeactivateUserErrors", + ) + + +class LocationDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_location_id", "location_delete_user_errors") + deleted_location_id = sgqlc.types.Field(ID, graphql_name="deletedLocationId") + location_delete_user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("LocationDeleteUserError"))), graphql_name="locationDeleteUserErrors" + ) + + +class LocationEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("Location"), graphql_name="node") + + +class LocationEditPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("location", "user_errors") + location = sgqlc.types.Field("Location", graphql_name="location") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("LocationEditUserError"))), graphql_name="userErrors" + ) + + +class LocationLocalPickupDisablePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("location_id", "user_errors") + location_id = sgqlc.types.Field(ID, graphql_name="locationId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryLocationLocalPickupSettingsError"))), + graphql_name="userErrors", + ) + + +class LocationLocalPickupEnablePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("local_pickup_settings", "user_errors") + local_pickup_settings = sgqlc.types.Field(DeliveryLocalPickupSettings, graphql_name="localPickupSettings") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryLocationLocalPickupSettingsError"))), + graphql_name="userErrors", + ) + + +class LocationSuggestedAddress(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("address1", "address2", "city", "country", "country_code", "formatted", "province", "province_code", "zip") + address1 = sgqlc.types.Field(String, graphql_name="address1") + address2 = sgqlc.types.Field(String, graphql_name="address2") + city = sgqlc.types.Field(String, graphql_name="city") + country = sgqlc.types.Field(String, graphql_name="country") + country_code = sgqlc.types.Field(CountryCode, graphql_name="countryCode") + formatted = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="formatted") + province = sgqlc.types.Field(String, graphql_name="province") + province_code = sgqlc.types.Field(String, graphql_name="provinceCode") + zip = sgqlc.types.Field(String, graphql_name="zip") + + +class MailingAddressConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MailingAddressEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MailingAddress"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class MailingAddressEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("MailingAddress"), graphql_name="node") + + +class MarketCatalogConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketCatalogEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketCatalog"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class MarketCatalogEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("MarketCatalog"), graphql_name="node") + + +class MarketConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Market"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class MarketCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("market", "user_errors") + market = sgqlc.types.Field("Market", graphql_name="market") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketUserError"))), graphql_name="userErrors" + ) + + +class MarketCurrencySettings(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("base_currency", "local_currencies") + base_currency = sgqlc.types.Field(sgqlc.types.non_null(CurrencySetting), graphql_name="baseCurrency") + local_currencies = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="localCurrencies") + + +class MarketCurrencySettingsUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("market", "user_errors") + market = sgqlc.types.Field("Market", graphql_name="market") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketCurrencySettingsUserError"))), graphql_name="userErrors" + ) + + +class MarketDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_id", "user_errors") + deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketUserError"))), graphql_name="userErrors" + ) + + +class MarketEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("Market"), graphql_name="node") + + +class MarketLocalizableContent(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("digest", "key", "value") + digest = sgqlc.types.Field(String, graphql_name="digest") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + value = sgqlc.types.Field(String, graphql_name="value") + + +class MarketLocalizableResource(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("market_localizable_content", "market_localizations", "resource_id") + market_localizable_content = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MarketLocalizableContent))), graphql_name="marketLocalizableContent" + ) + market_localizations = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketLocalization"))), + graphql_name="marketLocalizations", + args=sgqlc.types.ArgDict((("market_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="marketId", default=None)),)), + ) + resource_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="resourceId") + + +class MarketLocalizableResourceConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketLocalizableResourceEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MarketLocalizableResource))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class MarketLocalizableResourceEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(MarketLocalizableResource), graphql_name="node") + + +class MarketLocalization(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("key", "market", "outdated", "updated_at", "value") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + market = sgqlc.types.Field(sgqlc.types.non_null("Market"), graphql_name="market") + outdated = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="outdated") + updated_at = sgqlc.types.Field(DateTime, graphql_name="updatedAt") + value = sgqlc.types.Field(String, graphql_name="value") + + +class MarketLocalizationsRegisterPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("market_localizations", "user_errors") + market_localizations = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(MarketLocalization)), graphql_name="marketLocalizations" + ) + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TranslationUserError"))), graphql_name="userErrors" + ) + + +class MarketLocalizationsRemovePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("market_localizations", "user_errors") + market_localizations = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(MarketLocalization)), graphql_name="marketLocalizations" + ) + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TranslationUserError"))), graphql_name="userErrors" + ) + + +class MarketRegionConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketRegionEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MarketRegion))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class MarketRegionDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_id", "market", "user_errors") + deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") + market = sgqlc.types.Field("Market", graphql_name="market") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketUserError"))), graphql_name="userErrors" + ) + + +class MarketRegionEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(MarketRegion), graphql_name="node") + + +class MarketRegionsCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("market", "user_errors") + market = sgqlc.types.Field("Market", graphql_name="market") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketUserError"))), graphql_name="userErrors" + ) + + +class MarketUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("market", "user_errors") + market = sgqlc.types.Field("Market", graphql_name="market") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketUserError"))), graphql_name="userErrors" + ) + + +class MarketWebPresenceCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("market", "user_errors") + market = sgqlc.types.Field("Market", graphql_name="market") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketUserError"))), graphql_name="userErrors" + ) + + +class MarketWebPresenceDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_id", "market", "user_errors") + deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") + market = sgqlc.types.Field("Market", graphql_name="market") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketUserError"))), graphql_name="userErrors" + ) + + +class MarketWebPresenceRootUrl(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("locale", "url") + locale = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="locale") + url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") + + +class MarketWebPresenceUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("market", "user_errors") + market = sgqlc.types.Field("Market", graphql_name="market") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketUserError"))), graphql_name="userErrors" + ) + + +class MarketingActivityConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketingActivityEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketingActivity"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class MarketingActivityCreateExternalPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("marketing_activity", "user_errors") + marketing_activity = sgqlc.types.Field("MarketingActivity", graphql_name="marketingActivity") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketingActivityUserError"))), graphql_name="userErrors" + ) + + +class MarketingActivityCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("marketing_activity", "redirect_path", "user_errors") + marketing_activity = sgqlc.types.Field("MarketingActivity", graphql_name="marketingActivity") + redirect_path = sgqlc.types.Field(String, graphql_name="redirectPath") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class MarketingActivityEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("MarketingActivity"), graphql_name="node") + + +class MarketingActivityExtensionAppErrors(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("code", "user_errors") + code = sgqlc.types.Field(sgqlc.types.non_null(MarketingActivityExtensionAppErrorCode), graphql_name="code") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class MarketingActivityUpdateExternalPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("marketing_activity", "user_errors") + marketing_activity = sgqlc.types.Field("MarketingActivity", graphql_name="marketingActivity") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketingActivityUserError"))), graphql_name="userErrors" + ) + + +class MarketingActivityUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("marketing_activity", "redirect_path", "user_errors") + marketing_activity = sgqlc.types.Field("MarketingActivity", graphql_name="marketingActivity") + redirect_path = sgqlc.types.Field(String, graphql_name="redirectPath") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class MarketingBudget(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("budget_type", "total") + budget_type = sgqlc.types.Field(sgqlc.types.non_null(MarketingBudgetBudgetType), graphql_name="budgetType") + total = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="total") + + +class MarketingEngagement(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "ad_spend", + "clicks_count", + "comments_count", + "complaints_count", + "fails_count", + "favorites_count", + "fetched_at", + "impressions_count", + "is_cumulative", + "marketing_activity", + "occurred_on", + "sends_count", + "shares_count", + "unique_clicks_count", + "unique_views_count", + "unsubscribes_count", + "utc_offset", + "views_count", + ) + ad_spend = sgqlc.types.Field("MoneyV2", graphql_name="adSpend") + clicks_count = sgqlc.types.Field(Int, graphql_name="clicksCount") + comments_count = sgqlc.types.Field(Int, graphql_name="commentsCount") + complaints_count = sgqlc.types.Field(Int, graphql_name="complaintsCount") + fails_count = sgqlc.types.Field(Int, graphql_name="failsCount") + favorites_count = sgqlc.types.Field(Int, graphql_name="favoritesCount") + fetched_at = sgqlc.types.Field(DateTime, graphql_name="fetchedAt") + impressions_count = sgqlc.types.Field(Int, graphql_name="impressionsCount") + is_cumulative = sgqlc.types.Field(Boolean, graphql_name="isCumulative") + marketing_activity = sgqlc.types.Field(sgqlc.types.non_null("MarketingActivity"), graphql_name="marketingActivity") + occurred_on = sgqlc.types.Field(sgqlc.types.non_null(Date), graphql_name="occurredOn") + sends_count = sgqlc.types.Field(Int, graphql_name="sendsCount") + shares_count = sgqlc.types.Field(Int, graphql_name="sharesCount") + unique_clicks_count = sgqlc.types.Field(Int, graphql_name="uniqueClicksCount") + unique_views_count = sgqlc.types.Field(Int, graphql_name="uniqueViewsCount") + unsubscribes_count = sgqlc.types.Field(Int, graphql_name="unsubscribesCount") + utc_offset = sgqlc.types.Field(UtcOffset, graphql_name="utcOffset") + views_count = sgqlc.types.Field(Int, graphql_name="viewsCount") + + +class MarketingEngagementCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("marketing_engagement", "user_errors") + marketing_engagement = sgqlc.types.Field(MarketingEngagement, graphql_name="marketingEngagement") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class MarketingEventConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketingEventEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketingEvent"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class MarketingEventEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("MarketingEvent"), graphql_name="node") + + +class MediaConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MediaEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Media))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class MediaEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(Media), graphql_name="node") + + +class MediaError(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("code", "details", "message") + code = sgqlc.types.Field(sgqlc.types.non_null(MediaErrorCode), graphql_name="code") + details = sgqlc.types.Field(String, graphql_name="details") + message = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="message") + + +class MediaImageOriginalSource(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("file_size",) + file_size = sgqlc.types.Field(Int, graphql_name="fileSize") + + +class MediaPreviewImage(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("image", "status") + image = sgqlc.types.Field("Image", graphql_name="image") + status = sgqlc.types.Field(sgqlc.types.non_null(MediaPreviewImageStatus), graphql_name="status") + + +class MediaWarning(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("code", "message") + code = sgqlc.types.Field(sgqlc.types.non_null(MediaWarningCode), graphql_name="code") + message = sgqlc.types.Field(String, graphql_name="message") + + +class MerchantApprovalSignals(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("identity_verified", "verified_by_shopify", "verified_by_shopify_tier") + identity_verified = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="identityVerified") + verified_by_shopify = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="verifiedByShopify") + verified_by_shopify_tier = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="verifiedByShopifyTier") + + +class MetafieldAccess(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("admin",) + admin = sgqlc.types.Field(MetafieldAdminAccess, graphql_name="admin") + + +class MetafieldConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Metafield"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class MetafieldDefinitionConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldDefinitionEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldDefinition"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class MetafieldDefinitionCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("created_definition", "user_errors") + created_definition = sgqlc.types.Field("MetafieldDefinition", graphql_name="createdDefinition") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldDefinitionCreateUserError"))), graphql_name="userErrors" + ) + + +class MetafieldDefinitionDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_definition_id", "user_errors") + deleted_definition_id = sgqlc.types.Field(ID, graphql_name="deletedDefinitionId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldDefinitionDeleteUserError"))), graphql_name="userErrors" + ) + + +class MetafieldDefinitionEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("MetafieldDefinition"), graphql_name="node") + + +class MetafieldDefinitionPinPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("pinned_definition", "user_errors") + pinned_definition = sgqlc.types.Field("MetafieldDefinition", graphql_name="pinnedDefinition") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldDefinitionPinUserError"))), graphql_name="userErrors" + ) + + +class MetafieldDefinitionSupportedValidation(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("name", "type") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="type") + + +class MetafieldDefinitionType(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("category", "name", "supported_validations", "supports_definition_migrations") + category = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="category") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + supported_validations = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MetafieldDefinitionSupportedValidation))), + graphql_name="supportedValidations", + ) + supports_definition_migrations = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="supportsDefinitionMigrations") + + +class MetafieldDefinitionUnpinPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("unpinned_definition", "user_errors") + unpinned_definition = sgqlc.types.Field("MetafieldDefinition", graphql_name="unpinnedDefinition") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldDefinitionUnpinUserError"))), graphql_name="userErrors" + ) + + +class MetafieldDefinitionUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("updated_definition", "user_errors", "validation_job") + updated_definition = sgqlc.types.Field("MetafieldDefinition", graphql_name="updatedDefinition") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldDefinitionUpdateUserError"))), graphql_name="userErrors" + ) + validation_job = sgqlc.types.Field(Job, graphql_name="validationJob") + + +class MetafieldDefinitionValidation(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("name", "type", "value") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="type") + value = sgqlc.types.Field(String, graphql_name="value") + + +class MetafieldDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_id", "user_errors") + deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class MetafieldEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("Metafield"), graphql_name="node") + + +class MetafieldReferenceConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldReferenceEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of("MetafieldReference")), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class MetafieldReferenceEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field("MetafieldReference", graphql_name="node") + + +class MetafieldRelation(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("key", "name", "namespace", "referencer", "target") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + namespace = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="namespace") + referencer = sgqlc.types.Field(sgqlc.types.non_null("MetafieldReferencer"), graphql_name="referencer") + target = sgqlc.types.Field(sgqlc.types.non_null("MetafieldReference"), graphql_name="target") + + +class MetafieldRelationConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldRelationEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MetafieldRelation))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class MetafieldRelationEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(MetafieldRelation), graphql_name="node") + + +class MetafieldStorefrontVisibilityConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldStorefrontVisibilityEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldStorefrontVisibility"))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class MetafieldStorefrontVisibilityCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("metafield_storefront_visibility", "user_errors") + metafield_storefront_visibility = sgqlc.types.Field("MetafieldStorefrontVisibility", graphql_name="metafieldStorefrontVisibility") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class MetafieldStorefrontVisibilityDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_metafield_storefront_visibility_id", "user_errors") + deleted_metafield_storefront_visibility_id = sgqlc.types.Field(ID, graphql_name="deletedMetafieldStorefrontVisibilityId") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class MetafieldStorefrontVisibilityEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("MetafieldStorefrontVisibility"), graphql_name="node") + + +class MetafieldsSetPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("metafields", "user_errors") + metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("Metafield")), graphql_name="metafields") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldsSetUserError"))), graphql_name="userErrors" + ) + + +class MetaobjectAccess(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("admin", "storefront") + admin = sgqlc.types.Field(sgqlc.types.non_null(MetaobjectAdminAccess), graphql_name="admin") + storefront = sgqlc.types.Field(sgqlc.types.non_null(MetaobjectStorefrontAccess), graphql_name="storefront") + + +class MetaobjectBulkDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("job", "user_errors") + job = sgqlc.types.Field(Job, graphql_name="job") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetaobjectUserError"))), graphql_name="userErrors" + ) + + +class MetaobjectCapabilities(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("publishable", "translatable") + publishable = sgqlc.types.Field(sgqlc.types.non_null("MetaobjectCapabilitiesPublishable"), graphql_name="publishable") + translatable = sgqlc.types.Field(sgqlc.types.non_null("MetaobjectCapabilitiesTranslatable"), graphql_name="translatable") + + +class MetaobjectCapabilitiesPublishable(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("enabled",) + enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="enabled") + + +class MetaobjectCapabilitiesTranslatable(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("enabled",) + enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="enabled") + + +class MetaobjectCapabilityData(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("publishable",) + publishable = sgqlc.types.Field("MetaobjectCapabilityDataPublishable", graphql_name="publishable") + + +class MetaobjectCapabilityDataPublishable(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("status",) + status = sgqlc.types.Field(sgqlc.types.non_null(MetaobjectStatus), graphql_name="status") + + +class MetaobjectConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetaobjectEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Metaobject"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class MetaobjectCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("metaobject", "user_errors") + metaobject = sgqlc.types.Field("Metaobject", graphql_name="metaobject") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetaobjectUserError"))), graphql_name="userErrors" + ) + + +class MetaobjectDefinitionConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetaobjectDefinitionEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetaobjectDefinition"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class MetaobjectDefinitionCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("metaobject_definition", "user_errors") + metaobject_definition = sgqlc.types.Field("MetaobjectDefinition", graphql_name="metaobjectDefinition") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetaobjectUserError"))), graphql_name="userErrors" + ) + + +class MetaobjectDefinitionDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_id", "user_errors") + deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetaobjectUserError"))), graphql_name="userErrors" + ) + + +class MetaobjectDefinitionEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("MetaobjectDefinition"), graphql_name="node") + + +class MetaobjectDefinitionUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("metaobject_definition", "user_errors") + metaobject_definition = sgqlc.types.Field("MetaobjectDefinition", graphql_name="metaobjectDefinition") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetaobjectUserError"))), graphql_name="userErrors" + ) + + +class MetaobjectDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_id", "user_errors") + deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetaobjectUserError"))), graphql_name="userErrors" + ) + + +class MetaobjectEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("Metaobject"), graphql_name="node") + + +class MetaobjectField(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("definition", "key", "reference", "references", "type", "value") + definition = sgqlc.types.Field(sgqlc.types.non_null("MetaobjectFieldDefinition"), graphql_name="definition") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + reference = sgqlc.types.Field("MetafieldReference", graphql_name="reference") + references = sgqlc.types.Field( + MetafieldReferenceConnection, + graphql_name="references", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ) + ), + ) + type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="type") + value = sgqlc.types.Field(String, graphql_name="value") + + +class MetaobjectFieldDefinition(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("description", "key", "name", "required", "type", "validations") + description = sgqlc.types.Field(String, graphql_name="description") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + required = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="required") + type = sgqlc.types.Field(sgqlc.types.non_null(MetafieldDefinitionType), graphql_name="type") + validations = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MetafieldDefinitionValidation))), graphql_name="validations" + ) + + +class MetaobjectUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("metaobject", "user_errors") + metaobject = sgqlc.types.Field("Metaobject", graphql_name="metaobject") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetaobjectUserError"))), graphql_name="userErrors" + ) + + +class MetaobjectUpsertPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("metaobject", "user_errors") + metaobject = sgqlc.types.Field("Metaobject", graphql_name="metaobject") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetaobjectUserError"))), graphql_name="userErrors" + ) + + +class Model3dBoundingBox(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("size",) + size = sgqlc.types.Field(sgqlc.types.non_null("Vector3"), graphql_name="size") + + +class Model3dSource(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("filesize", "format", "mime_type", "url") + filesize = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="filesize") + format = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="format") + mime_type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="mimeType") + url = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="url") + + +class MoneyBag(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("presentment_money", "shop_money") + presentment_money = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="presentmentMoney") + shop_money = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="shopMoney") + + +class MoneyV2(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("amount", "currency_code") + amount = sgqlc.types.Field(sgqlc.types.non_null(Decimal), graphql_name="amount") + currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currencyCode") + + +class Mutation(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "abandonment_update_activities_delivery_statuses", + "app_purchase_one_time_create", + "app_subscription_cancel", + "app_subscription_create", + "app_subscription_line_item_update", + "app_subscription_trial_extend", + "app_usage_record_create", + "bulk_operation_cancel", + "bulk_operation_run_mutation", + "bulk_operation_run_query", + "bulk_product_resource_feedback_create", + "cart_transform_create", + "cart_transform_delete", + "catalog_context_update", + "catalog_create", + "catalog_delete", + "catalog_update", + "collection_add_products", + "collection_add_products_v2", + "collection_create", + "collection_delete", + "collection_remove_products", + "collection_reorder_products", + "collection_update", + "companies_delete", + "company_address_delete", + "company_assign_customer_as_contact", + "company_assign_main_contact", + "company_contact_assign_role", + "company_contact_assign_roles", + "company_contact_create", + "company_contact_delete", + "company_contact_remove_from_company", + "company_contact_revoke_role", + "company_contact_revoke_roles", + "company_contact_send_welcome_email", + "company_contact_update", + "company_contacts_delete", + "company_create", + "company_delete", + "company_location_assign_address", + "company_location_assign_roles", + "company_location_assign_tax_exemptions", + "company_location_create", + "company_location_create_tax_registration", + "company_location_delete", + "company_location_revoke_roles", + "company_location_revoke_tax_exemptions", + "company_location_revoke_tax_registration", + "company_location_update", + "company_locations_delete", + "company_revoke_main_contact", + "company_update", + "customer_add_tax_exemptions", + "customer_create", + "customer_delete", + "customer_email_marketing_consent_update", + "customer_generate_account_activation_url", + "customer_merge", + "customer_payment_method_create_from_duplication_data", + "customer_payment_method_credit_card_create", + "customer_payment_method_credit_card_update", + "customer_payment_method_get_duplication_data", + "customer_payment_method_get_update_url", + "customer_payment_method_paypal_billing_agreement_create", + "customer_payment_method_paypal_billing_agreement_update", + "customer_payment_method_remote_create", + "customer_payment_method_revoke", + "customer_payment_method_send_update_email", + "customer_remove_tax_exemptions", + "customer_replace_tax_exemptions", + "customer_segment_members_query_create", + "customer_sms_marketing_consent_update", + "customer_update", + "customer_update_default_address", + "delegate_access_token_create", + "delegate_access_token_destroy", + "delivery_customization_activation", + "delivery_customization_create", + "delivery_customization_delete", + "delivery_customization_update", + "delivery_profile_create", + "delivery_profile_remove", + "delivery_profile_update", + "delivery_setting_update", + "delivery_shipping_origin_assign", + "discount_automatic_activate", + "discount_automatic_app_create", + "discount_automatic_app_update", + "discount_automatic_basic_create", + "discount_automatic_basic_update", + "discount_automatic_bulk_delete", + "discount_automatic_bxgy_create", + "discount_automatic_bxgy_update", + "discount_automatic_deactivate", + "discount_automatic_delete", + "discount_code_activate", + "discount_code_app_create", + "discount_code_app_update", + "discount_code_basic_create", + "discount_code_basic_update", + "discount_code_bulk_activate", + "discount_code_bulk_deactivate", + "discount_code_bulk_delete", + "discount_code_bxgy_create", + "discount_code_bxgy_update", + "discount_code_deactivate", + "discount_code_delete", + "discount_code_free_shipping_create", + "discount_code_free_shipping_update", + "discount_code_redeem_code_bulk_delete", + "discount_redeem_code_bulk_add", + "dispute_evidence_update", + "draft_order_bulk_add_tags", + "draft_order_bulk_delete", + "draft_order_bulk_remove_tags", + "draft_order_calculate", + "draft_order_complete", + "draft_order_create", + "draft_order_create_from_order", + "draft_order_create_merchant_checkout", + "draft_order_delete", + "draft_order_duplicate", + "draft_order_invoice_preview", + "draft_order_invoice_send", + "draft_order_update", + "event_bridge_server_pixel_update", + "event_bridge_webhook_subscription_create", + "event_bridge_webhook_subscription_update", + "file_acknowledge_update_failed", + "file_create", + "file_delete", + "file_update", + "flow_trigger_receive", + "fulfillment_cancel", + "fulfillment_create_v2", + "fulfillment_event_create", + "fulfillment_order_accept_cancellation_request", + "fulfillment_order_accept_fulfillment_request", + "fulfillment_order_cancel", + "fulfillment_order_close", + "fulfillment_order_hold", + "fulfillment_order_line_items_prepared_for_pickup", + "fulfillment_order_merge", + "fulfillment_order_move", + "fulfillment_order_open", + "fulfillment_order_reject_cancellation_request", + "fulfillment_order_reject_fulfillment_request", + "fulfillment_order_release_hold", + "fulfillment_order_reschedule", + "fulfillment_order_split", + "fulfillment_order_submit_cancellation_request", + "fulfillment_order_submit_fulfillment_request", + "fulfillment_orders_release_holds", + "fulfillment_orders_set_fulfillment_deadline", + "fulfillment_service_create", + "fulfillment_service_delete", + "fulfillment_service_update", + "fulfillment_tracking_info_update_v2", + "gift_card_create", + "gift_card_disable", + "gift_card_update", + "inventory_activate", + "inventory_adjust_quantities", + "inventory_bulk_toggle_activation", + "inventory_deactivate", + "inventory_item_update", + "inventory_move_quantities", + "inventory_set_on_hand_quantities", + "location_activate", + "location_add", + "location_deactivate", + "location_delete", + "location_edit", + "location_local_pickup_disable", + "location_local_pickup_enable", + "market_create", + "market_currency_settings_update", + "market_delete", + "market_localizations_register", + "market_localizations_remove", + "market_region_delete", + "market_regions_create", + "market_update", + "market_web_presence_create", + "market_web_presence_delete", + "market_web_presence_update", + "marketing_activity_create", + "marketing_activity_create_external", + "marketing_activity_update", + "marketing_activity_update_external", + "marketing_engagement_create", + "metafield_definition_create", + "metafield_definition_delete", + "metafield_definition_pin", + "metafield_definition_unpin", + "metafield_definition_update", + "metafield_delete", + "metafields_set", + "metaobject_bulk_delete", + "metaobject_create", + "metaobject_definition_create", + "metaobject_definition_delete", + "metaobject_definition_update", + "metaobject_delete", + "metaobject_update", + "metaobject_upsert", + "order_capture", + "order_close", + "order_create_mandate_payment", + "order_edit_add_custom_item", + "order_edit_add_line_item_discount", + "order_edit_add_variant", + "order_edit_begin", + "order_edit_commit", + "order_edit_remove_line_item_discount", + "order_edit_set_quantity", + "order_invoice_send", + "order_mark_as_paid", + "order_open", + "order_update", + "payment_customization_activation", + "payment_customization_create", + "payment_customization_delete", + "payment_customization_update", + "payment_reminder_send", + "payment_terms_create", + "payment_terms_delete", + "payment_terms_update", + "price_list_create", + "price_list_delete", + "price_list_fixed_prices_add", + "price_list_fixed_prices_by_product_update", + "price_list_fixed_prices_delete", + "price_list_fixed_prices_update", + "price_list_update", + "product_change_status", + "product_create", + "product_create_media", + "product_delete", + "product_delete_async", + "product_delete_media", + "product_duplicate", + "product_duplicate_async_v2", + "product_feed_create", + "product_feed_delete", + "product_full_sync", + "product_join_selling_plan_groups", + "product_leave_selling_plan_groups", + "product_reorder_media", + "product_update", + "product_update_media", + "product_variant_append_media", + "product_variant_create", + "product_variant_delete", + "product_variant_detach_media", + "product_variant_join_selling_plan_groups", + "product_variant_leave_selling_plan_groups", + "product_variant_relationship_bulk_update", + "product_variant_update", + "product_variants_bulk_create", + "product_variants_bulk_delete", + "product_variants_bulk_reorder", + "product_variants_bulk_update", + "pub_sub_server_pixel_update", + "pub_sub_webhook_subscription_create", + "pub_sub_webhook_subscription_update", + "publication_create", + "publication_delete", + "publication_update", + "publishable_publish", + "publishable_publish_to_current_channel", + "publishable_unpublish", + "publishable_unpublish_to_current_channel", + "quantity_rules_add", + "quantity_rules_delete", + "refund_create", + "return_approve_request", + "return_cancel", + "return_close", + "return_create", + "return_decline_request", + "return_refund", + "return_reopen", + "return_request", + "reverse_delivery_create_with_shipping", + "reverse_delivery_shipping_update", + "reverse_fulfillment_order_dispose", + "saved_search_create", + "saved_search_delete", + "saved_search_update", + "script_tag_create", + "script_tag_delete", + "script_tag_update", + "segment_create", + "segment_delete", + "segment_update", + "selling_plan_group_add_product_variants", + "selling_plan_group_add_products", + "selling_plan_group_create", + "selling_plan_group_delete", + "selling_plan_group_remove_product_variants", + "selling_plan_group_remove_products", + "selling_plan_group_update", + "server_pixel_create", + "server_pixel_delete", + "shipping_package_delete", + "shipping_package_make_default", + "shipping_package_update", + "shop_locale_disable", + "shop_locale_enable", + "shop_locale_update", + "shop_policy_update", + "shop_resource_feedback_create", + "staged_uploads_create", + "standard_metafield_definition_enable", + "standard_metaobject_definition_enable", + "storefront_access_token_create", + "storefront_access_token_delete", + "subscription_billing_attempt_create", + "subscription_billing_cycle_contract_draft_commit", + "subscription_billing_cycle_contract_draft_concatenate", + "subscription_billing_cycle_contract_edit", + "subscription_billing_cycle_edit_delete", + "subscription_billing_cycle_edits_delete", + "subscription_billing_cycle_schedule_edit", + "subscription_contract_atomic_create", + "subscription_contract_create", + "subscription_contract_product_change", + "subscription_contract_set_next_billing_date", + "subscription_contract_update", + "subscription_draft_commit", + "subscription_draft_discount_add", + "subscription_draft_discount_code_apply", + "subscription_draft_discount_remove", + "subscription_draft_discount_update", + "subscription_draft_free_shipping_discount_add", + "subscription_draft_free_shipping_discount_update", + "subscription_draft_line_add", + "subscription_draft_line_remove", + "subscription_draft_line_update", + "subscription_draft_update", + "tags_add", + "tags_remove", + "tax_app_configure", + "translations_register", + "translations_remove", + "url_redirect_bulk_delete_all", + "url_redirect_bulk_delete_by_ids", + "url_redirect_bulk_delete_by_saved_search", + "url_redirect_bulk_delete_by_search", + "url_redirect_create", + "url_redirect_delete", + "url_redirect_import_create", + "url_redirect_import_submit", + "url_redirect_update", + "web_pixel_create", + "web_pixel_delete", + "web_pixel_update", + "webhook_subscription_create", + "webhook_subscription_delete", + "webhook_subscription_update", + ) + abandonment_update_activities_delivery_statuses = sgqlc.types.Field( + AbandonmentUpdateActivitiesDeliveryStatusesPayload, + graphql_name="abandonmentUpdateActivitiesDeliveryStatuses", + args=sgqlc.types.ArgDict( + ( + ("abandonment_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="abandonmentId", default=None)), + ("marketing_activity_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="marketingActivityId", default=None)), + ( + "delivery_status", + sgqlc.types.Arg(sgqlc.types.non_null(AbandonmentDeliveryState), graphql_name="deliveryStatus", default=None), + ), + ("delivered_at", sgqlc.types.Arg(DateTime, graphql_name="deliveredAt", default=None)), + ("delivery_status_change_reason", sgqlc.types.Arg(String, graphql_name="deliveryStatusChangeReason", default=None)), + ) + ), + ) + app_purchase_one_time_create = sgqlc.types.Field( + AppPurchaseOneTimeCreatePayload, + graphql_name="appPurchaseOneTimeCreate", + args=sgqlc.types.ArgDict( + ( + ("name", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="name", default=None)), + ("price", sgqlc.types.Arg(sgqlc.types.non_null(MoneyInput), graphql_name="price", default=None)), + ("return_url", sgqlc.types.Arg(sgqlc.types.non_null(URL), graphql_name="returnUrl", default=None)), + ("test", sgqlc.types.Arg(Boolean, graphql_name="test", default=False)), + ) + ), + ) + app_subscription_cancel = sgqlc.types.Field( + AppSubscriptionCancelPayload, + graphql_name="appSubscriptionCancel", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("prorate", sgqlc.types.Arg(Boolean, graphql_name="prorate", default=False)), + ) + ), + ) + app_subscription_create = sgqlc.types.Field( + AppSubscriptionCreatePayload, + graphql_name="appSubscriptionCreate", + args=sgqlc.types.ArgDict( + ( + ("name", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="name", default=None)), + ( + "line_items", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(AppSubscriptionLineItemInput))), + graphql_name="lineItems", + default=None, + ), + ), + ("test", sgqlc.types.Arg(Boolean, graphql_name="test", default=None)), + ("trial_days", sgqlc.types.Arg(Int, graphql_name="trialDays", default=None)), + ("return_url", sgqlc.types.Arg(sgqlc.types.non_null(URL), graphql_name="returnUrl", default=None)), + ( + "replacement_behavior", + sgqlc.types.Arg(AppSubscriptionReplacementBehavior, graphql_name="replacementBehavior", default="STANDARD"), + ), + ) + ), + ) + app_subscription_line_item_update = sgqlc.types.Field( + AppSubscriptionLineItemUpdatePayload, + graphql_name="appSubscriptionLineItemUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("capped_amount", sgqlc.types.Arg(sgqlc.types.non_null(MoneyInput), graphql_name="cappedAmount", default=None)), + ) + ), + ) + app_subscription_trial_extend = sgqlc.types.Field( + AppSubscriptionTrialExtendPayload, + graphql_name="appSubscriptionTrialExtend", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("days", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="days", default=None)), + ) + ), + ) + app_usage_record_create = sgqlc.types.Field( + AppUsageRecordCreatePayload, + graphql_name="appUsageRecordCreate", + args=sgqlc.types.ArgDict( + ( + ( + "subscription_line_item_id", + sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="subscriptionLineItemId", default=None), + ), + ("price", sgqlc.types.Arg(sgqlc.types.non_null(MoneyInput), graphql_name="price", default=None)), + ("description", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="description", default=None)), + ("idempotency_key", sgqlc.types.Arg(String, graphql_name="idempotencyKey", default=None)), + ) + ), + ) + bulk_operation_cancel = sgqlc.types.Field( + BulkOperationCancelPayload, + graphql_name="bulkOperationCancel", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + bulk_operation_run_mutation = sgqlc.types.Field( + BulkOperationRunMutationPayload, + graphql_name="bulkOperationRunMutation", + args=sgqlc.types.ArgDict( + ( + ("mutation", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="mutation", default=None)), + ("staged_upload_path", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="stagedUploadPath", default=None)), + ("client_identifier", sgqlc.types.Arg(String, graphql_name="clientIdentifier", default=None)), + ) + ), + ) + bulk_operation_run_query = sgqlc.types.Field( + BulkOperationRunQueryPayload, + graphql_name="bulkOperationRunQuery", + args=sgqlc.types.ArgDict((("query", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="query", default=None)),)), + ) + bulk_product_resource_feedback_create = sgqlc.types.Field( + BulkProductResourceFeedbackCreatePayload, + graphql_name="bulkProductResourceFeedbackCreate", + args=sgqlc.types.ArgDict( + ( + ( + "feedback_input", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProductResourceFeedbackInput))), + graphql_name="feedbackInput", + default=None, + ), + ), + ) + ), + ) + cart_transform_create = sgqlc.types.Field( + CartTransformCreatePayload, + graphql_name="cartTransformCreate", + args=sgqlc.types.ArgDict( + (("function_id", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="functionId", default=None)),) + ), + ) + cart_transform_delete = sgqlc.types.Field( + CartTransformDeletePayload, + graphql_name="cartTransformDelete", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + catalog_context_update = sgqlc.types.Field( + CatalogContextUpdatePayload, + graphql_name="catalogContextUpdate", + args=sgqlc.types.ArgDict( + ( + ("catalog_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="catalogId", default=None)), + ("contexts_to_add", sgqlc.types.Arg(CatalogContextInput, graphql_name="contextsToAdd", default=None)), + ("contexts_to_remove", sgqlc.types.Arg(CatalogContextInput, graphql_name="contextsToRemove", default=None)), + ) + ), + ) + catalog_create = sgqlc.types.Field( + CatalogCreatePayload, + graphql_name="catalogCreate", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(CatalogCreateInput), graphql_name="input", default=None)),) + ), + ) + catalog_delete = sgqlc.types.Field( + CatalogDeletePayload, + graphql_name="catalogDelete", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("delete_dependent_resources", sgqlc.types.Arg(Boolean, graphql_name="deleteDependentResources", default=False)), + ) + ), + ) + catalog_update = sgqlc.types.Field( + CatalogUpdatePayload, + graphql_name="catalogUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("input", sgqlc.types.Arg(sgqlc.types.non_null(CatalogUpdateInput), graphql_name="input", default=None)), + ) + ), + ) + collection_add_products = sgqlc.types.Field( + CollectionAddProductsPayload, + graphql_name="collectionAddProducts", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "product_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="productIds", default=None + ), + ), + ) + ), + ) + collection_add_products_v2 = sgqlc.types.Field( + CollectionAddProductsV2Payload, + graphql_name="collectionAddProductsV2", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "product_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="productIds", default=None + ), + ), + ) + ), + ) + collection_create = sgqlc.types.Field( + CollectionCreatePayload, + graphql_name="collectionCreate", + args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(CollectionInput), graphql_name="input", default=None)),)), + ) + collection_delete = sgqlc.types.Field( + CollectionDeletePayload, + graphql_name="collectionDelete", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(CollectionDeleteInput), graphql_name="input", default=None)),) + ), + ) + collection_remove_products = sgqlc.types.Field( + CollectionRemoveProductsPayload, + graphql_name="collectionRemoveProducts", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "product_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="productIds", default=None + ), + ), + ) + ), + ) + collection_reorder_products = sgqlc.types.Field( + CollectionReorderProductsPayload, + graphql_name="collectionReorderProducts", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "moves", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MoveInput))), graphql_name="moves", default=None + ), + ), + ) + ), + ) + collection_update = sgqlc.types.Field( + CollectionUpdatePayload, + graphql_name="collectionUpdate", + args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(CollectionInput), graphql_name="input", default=None)),)), + ) + companies_delete = sgqlc.types.Field( + CompaniesDeletePayload, + graphql_name="companiesDelete", + args=sgqlc.types.ArgDict( + ( + ( + "company_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="companyIds", default=None + ), + ), + ) + ), + ) + company_address_delete = sgqlc.types.Field( + CompanyAddressDeletePayload, + graphql_name="companyAddressDelete", + args=sgqlc.types.ArgDict((("address_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="addressId", default=None)),)), + ) + company_assign_customer_as_contact = sgqlc.types.Field( + CompanyAssignCustomerAsContactPayload, + graphql_name="companyAssignCustomerAsContact", + args=sgqlc.types.ArgDict( + ( + ("company_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyId", default=None)), + ("customer_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerId", default=None)), + ) + ), + ) + company_assign_main_contact = sgqlc.types.Field( + CompanyAssignMainContactPayload, + graphql_name="companyAssignMainContact", + args=sgqlc.types.ArgDict( + ( + ("company_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyId", default=None)), + ("company_contact_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyContactId", default=None)), + ) + ), + ) + company_contact_assign_role = sgqlc.types.Field( + CompanyContactAssignRolePayload, + graphql_name="companyContactAssignRole", + args=sgqlc.types.ArgDict( + ( + ("company_contact_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyContactId", default=None)), + ("company_contact_role_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyContactRoleId", default=None)), + ("company_location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyLocationId", default=None)), + ) + ), + ) + company_contact_assign_roles = sgqlc.types.Field( + CompanyContactAssignRolesPayload, + graphql_name="companyContactAssignRoles", + args=sgqlc.types.ArgDict( + ( + ("company_contact_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyContactId", default=None)), + ( + "roles_to_assign", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CompanyContactRoleAssign))), + graphql_name="rolesToAssign", + default=None, + ), + ), + ) + ), + ) + company_contact_create = sgqlc.types.Field( + CompanyContactCreatePayload, + graphql_name="companyContactCreate", + args=sgqlc.types.ArgDict( + ( + ("company_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyId", default=None)), + ("input", sgqlc.types.Arg(sgqlc.types.non_null(CompanyContactInput), graphql_name="input", default=None)), + ) + ), + ) + company_contact_delete = sgqlc.types.Field( + CompanyContactDeletePayload, + graphql_name="companyContactDelete", + args=sgqlc.types.ArgDict( + (("company_contact_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyContactId", default=None)),) + ), + ) + company_contact_remove_from_company = sgqlc.types.Field( + CompanyContactRemoveFromCompanyPayload, + graphql_name="companyContactRemoveFromCompany", + args=sgqlc.types.ArgDict( + (("company_contact_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyContactId", default=None)),) + ), + ) + company_contact_revoke_role = sgqlc.types.Field( + CompanyContactRevokeRolePayload, + graphql_name="companyContactRevokeRole", + args=sgqlc.types.ArgDict( + ( + ("company_contact_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyContactId", default=None)), + ( + "company_contact_role_assignment_id", + sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyContactRoleAssignmentId", default=None), + ), + ) + ), + ) + company_contact_revoke_roles = sgqlc.types.Field( + CompanyContactRevokeRolesPayload, + graphql_name="companyContactRevokeRoles", + args=sgqlc.types.ArgDict( + ( + ("company_contact_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyContactId", default=None)), + ( + "role_assignment_ids", + sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="roleAssignmentIds", default=None), + ), + ("revoke_all", sgqlc.types.Arg(Boolean, graphql_name="revokeAll", default=False)), + ) + ), + ) + company_contact_send_welcome_email = sgqlc.types.Field( + CompanyContactSendWelcomeEmailPayload, + graphql_name="companyContactSendWelcomeEmail", + args=sgqlc.types.ArgDict( + ( + ("company_contact_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyContactId", default=None)), + ("email", sgqlc.types.Arg(EmailInput, graphql_name="email", default=None)), + ) + ), + ) + company_contact_update = sgqlc.types.Field( + CompanyContactUpdatePayload, + graphql_name="companyContactUpdate", + args=sgqlc.types.ArgDict( + ( + ("company_contact_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyContactId", default=None)), + ("input", sgqlc.types.Arg(sgqlc.types.non_null(CompanyContactInput), graphql_name="input", default=None)), + ) + ), + ) + company_contacts_delete = sgqlc.types.Field( + CompanyContactsDeletePayload, + graphql_name="companyContactsDelete", + args=sgqlc.types.ArgDict( + ( + ( + "company_contact_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="companyContactIds", default=None + ), + ), + ) + ), + ) + company_create = sgqlc.types.Field( + CompanyCreatePayload, + graphql_name="companyCreate", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(CompanyCreateInput), graphql_name="input", default=None)),) + ), + ) + company_delete = sgqlc.types.Field( + CompanyDeletePayload, + graphql_name="companyDelete", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + company_location_assign_address = sgqlc.types.Field( + CompanyLocationAssignAddressPayload, + graphql_name="companyLocationAssignAddress", + args=sgqlc.types.ArgDict( + ( + ("location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="locationId", default=None)), + ("address", sgqlc.types.Arg(sgqlc.types.non_null(CompanyAddressInput), graphql_name="address", default=None)), + ( + "address_types", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CompanyAddressType))), + graphql_name="addressTypes", + default=None, + ), + ), + ) + ), + ) + company_location_assign_roles = sgqlc.types.Field( + CompanyLocationAssignRolesPayload, + graphql_name="companyLocationAssignRoles", + args=sgqlc.types.ArgDict( + ( + ("company_location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyLocationId", default=None)), + ( + "roles_to_assign", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CompanyLocationRoleAssign))), + graphql_name="rolesToAssign", + default=None, + ), + ), + ) + ), + ) + company_location_assign_tax_exemptions = sgqlc.types.Field( + CompanyLocationAssignTaxExemptionsPayload, + graphql_name="companyLocationAssignTaxExemptions", + args=sgqlc.types.ArgDict( + ( + ("company_location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyLocationId", default=None)), + ( + "tax_exemptions", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxExemption))), + graphql_name="taxExemptions", + default=None, + ), + ), + ) + ), + ) + company_location_create = sgqlc.types.Field( + CompanyLocationCreatePayload, + graphql_name="companyLocationCreate", + args=sgqlc.types.ArgDict( + ( + ("company_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyId", default=None)), + ("input", sgqlc.types.Arg(sgqlc.types.non_null(CompanyLocationInput), graphql_name="input", default=None)), + ) + ), + ) + company_location_create_tax_registration = sgqlc.types.Field( + CompanyLocationCreateTaxRegistrationPayload, + graphql_name="companyLocationCreateTaxRegistration", + args=sgqlc.types.ArgDict( + ( + ("location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="locationId", default=None)), + ("tax_id", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="taxId", default=None)), + ) + ), + ) + company_location_delete = sgqlc.types.Field( + CompanyLocationDeletePayload, + graphql_name="companyLocationDelete", + args=sgqlc.types.ArgDict( + (("company_location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyLocationId", default=None)),) + ), + ) + company_location_revoke_roles = sgqlc.types.Field( + CompanyLocationRevokeRolesPayload, + graphql_name="companyLocationRevokeRoles", + args=sgqlc.types.ArgDict( + ( + ("company_location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyLocationId", default=None)), + ( + "roles_to_revoke", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="rolesToRevoke", default=None + ), + ), + ) + ), + ) + company_location_revoke_tax_exemptions = sgqlc.types.Field( + CompanyLocationRevokeTaxExemptionsPayload, + graphql_name="companyLocationRevokeTaxExemptions", + args=sgqlc.types.ArgDict( + ( + ("company_location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyLocationId", default=None)), + ( + "tax_exemptions", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxExemption))), + graphql_name="taxExemptions", + default=None, + ), + ), + ) + ), + ) + company_location_revoke_tax_registration = sgqlc.types.Field( + CompanyLocationRevokeTaxRegistrationPayload, + graphql_name="companyLocationRevokeTaxRegistration", + args=sgqlc.types.ArgDict( + (("company_location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyLocationId", default=None)),) + ), + ) + company_location_update = sgqlc.types.Field( + CompanyLocationUpdatePayload, + graphql_name="companyLocationUpdate", + args=sgqlc.types.ArgDict( + ( + ("company_location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyLocationId", default=None)), + ("input", sgqlc.types.Arg(sgqlc.types.non_null(CompanyLocationUpdateInput), graphql_name="input", default=None)), + ) + ), + ) + company_locations_delete = sgqlc.types.Field( + CompanyLocationsDeletePayload, + graphql_name="companyLocationsDelete", + args=sgqlc.types.ArgDict( + ( + ( + "company_location_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="companyLocationIds", default=None + ), + ), + ) + ), + ) + company_revoke_main_contact = sgqlc.types.Field( + CompanyRevokeMainContactPayload, + graphql_name="companyRevokeMainContact", + args=sgqlc.types.ArgDict((("company_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyId", default=None)),)), + ) + company_update = sgqlc.types.Field( + CompanyUpdatePayload, + graphql_name="companyUpdate", + args=sgqlc.types.ArgDict( + ( + ("company_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyId", default=None)), + ("input", sgqlc.types.Arg(sgqlc.types.non_null(CompanyInput), graphql_name="input", default=None)), + ) + ), + ) + customer_add_tax_exemptions = sgqlc.types.Field( + CustomerAddTaxExemptionsPayload, + graphql_name="customerAddTaxExemptions", + args=sgqlc.types.ArgDict( + ( + ("customer_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerId", default=None)), + ( + "tax_exemptions", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxExemption))), + graphql_name="taxExemptions", + default=None, + ), + ), + ) + ), + ) + customer_create = sgqlc.types.Field( + CustomerCreatePayload, + graphql_name="customerCreate", + args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(CustomerInput), graphql_name="input", default=None)),)), + ) + customer_delete = sgqlc.types.Field( + CustomerDeletePayload, + graphql_name="customerDelete", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(CustomerDeleteInput), graphql_name="input", default=None)),) + ), + ) + customer_email_marketing_consent_update = sgqlc.types.Field( + CustomerEmailMarketingConsentUpdatePayload, + graphql_name="customerEmailMarketingConsentUpdate", + args=sgqlc.types.ArgDict( + ( + ( + "input", + sgqlc.types.Arg(sgqlc.types.non_null(CustomerEmailMarketingConsentUpdateInput), graphql_name="input", default=None), + ), + ) + ), + ) + customer_generate_account_activation_url = sgqlc.types.Field( + CustomerGenerateAccountActivationUrlPayload, + graphql_name="customerGenerateAccountActivationUrl", + args=sgqlc.types.ArgDict((("customer_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerId", default=None)),)), + ) + customer_merge = sgqlc.types.Field( + CustomerMergePayload, + graphql_name="customerMerge", + args=sgqlc.types.ArgDict( + ( + ("customer_one_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerOneId", default=None)), + ("customer_two_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerTwoId", default=None)), + ("override_fields", sgqlc.types.Arg(CustomerMergeOverrideFields, graphql_name="overrideFields", default=None)), + ) + ), + ) + customer_payment_method_create_from_duplication_data = sgqlc.types.Field( + CustomerPaymentMethodCreateFromDuplicationDataPayload, + graphql_name="customerPaymentMethodCreateFromDuplicationData", + args=sgqlc.types.ArgDict( + ( + ("customer_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerId", default=None)), + ( + "billing_address", + sgqlc.types.Arg(sgqlc.types.non_null(MailingAddressInput), graphql_name="billingAddress", default=None), + ), + ( + "encrypted_duplication_data", + sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="encryptedDuplicationData", default=None), + ), + ) + ), + ) + customer_payment_method_credit_card_create = sgqlc.types.Field( + CustomerPaymentMethodCreditCardCreatePayload, + graphql_name="customerPaymentMethodCreditCardCreate", + args=sgqlc.types.ArgDict( + ( + ("customer_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerId", default=None)), + ( + "billing_address", + sgqlc.types.Arg(sgqlc.types.non_null(MailingAddressInput), graphql_name="billingAddress", default=None), + ), + ("session_id", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="sessionId", default=None)), + ) + ), + ) + customer_payment_method_credit_card_update = sgqlc.types.Field( + CustomerPaymentMethodCreditCardUpdatePayload, + graphql_name="customerPaymentMethodCreditCardUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "billing_address", + sgqlc.types.Arg(sgqlc.types.non_null(MailingAddressInput), graphql_name="billingAddress", default=None), + ), + ("session_id", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="sessionId", default=None)), + ) + ), + ) + customer_payment_method_get_duplication_data = sgqlc.types.Field( + CustomerPaymentMethodGetDuplicationDataPayload, + graphql_name="customerPaymentMethodGetDuplicationData", + args=sgqlc.types.ArgDict( + ( + ( + "customer_payment_method_id", + sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerPaymentMethodId", default=None), + ), + ("target_shop_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="targetShopId", default=None)), + ("target_customer_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="targetCustomerId", default=None)), + ) + ), + ) + customer_payment_method_get_update_url = sgqlc.types.Field( + CustomerPaymentMethodGetUpdateUrlPayload, + graphql_name="customerPaymentMethodGetUpdateUrl", + args=sgqlc.types.ArgDict( + ( + ( + "customer_payment_method_id", + sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerPaymentMethodId", default=None), + ), + ) + ), + ) + customer_payment_method_paypal_billing_agreement_create = sgqlc.types.Field( + CustomerPaymentMethodPaypalBillingAgreementCreatePayload, + graphql_name="customerPaymentMethodPaypalBillingAgreementCreate", + args=sgqlc.types.ArgDict( + ( + ("customer_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerId", default=None)), + ("billing_address", sgqlc.types.Arg(MailingAddressInput, graphql_name="billingAddress", default=None)), + ("billing_agreement_id", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="billingAgreementId", default=None)), + ("inactive", sgqlc.types.Arg(Boolean, graphql_name="inactive", default=False)), + ) + ), + ) + customer_payment_method_paypal_billing_agreement_update = sgqlc.types.Field( + CustomerPaymentMethodPaypalBillingAgreementUpdatePayload, + graphql_name="customerPaymentMethodPaypalBillingAgreementUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "billing_address", + sgqlc.types.Arg(sgqlc.types.non_null(MailingAddressInput), graphql_name="billingAddress", default=None), + ), + ) + ), + ) + customer_payment_method_remote_create = sgqlc.types.Field( + CustomerPaymentMethodRemoteCreatePayload, + graphql_name="customerPaymentMethodRemoteCreate", + args=sgqlc.types.ArgDict( + ( + ("customer_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerId", default=None)), + ( + "remote_reference", + sgqlc.types.Arg(sgqlc.types.non_null(CustomerPaymentMethodRemoteInput), graphql_name="remoteReference", default=None), + ), + ) + ), + ) + customer_payment_method_revoke = sgqlc.types.Field( + CustomerPaymentMethodRevokePayload, + graphql_name="customerPaymentMethodRevoke", + args=sgqlc.types.ArgDict( + ( + ( + "customer_payment_method_id", + sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerPaymentMethodId", default=None), + ), + ) + ), + ) + customer_payment_method_send_update_email = sgqlc.types.Field( + CustomerPaymentMethodSendUpdateEmailPayload, + graphql_name="customerPaymentMethodSendUpdateEmail", + args=sgqlc.types.ArgDict( + ( + ( + "customer_payment_method_id", + sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerPaymentMethodId", default=None), + ), + ("email", sgqlc.types.Arg(EmailInput, graphql_name="email", default=None)), + ) + ), + ) + customer_remove_tax_exemptions = sgqlc.types.Field( + CustomerRemoveTaxExemptionsPayload, + graphql_name="customerRemoveTaxExemptions", + args=sgqlc.types.ArgDict( + ( + ("customer_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerId", default=None)), + ( + "tax_exemptions", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxExemption))), + graphql_name="taxExemptions", + default=None, + ), + ), + ) + ), + ) + customer_replace_tax_exemptions = sgqlc.types.Field( + CustomerReplaceTaxExemptionsPayload, + graphql_name="customerReplaceTaxExemptions", + args=sgqlc.types.ArgDict( + ( + ("customer_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerId", default=None)), + ( + "tax_exemptions", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxExemption))), + graphql_name="taxExemptions", + default=None, + ), + ), + ) + ), + ) + customer_segment_members_query_create = sgqlc.types.Field( + CustomerSegmentMembersQueryCreatePayload, + graphql_name="customerSegmentMembersQueryCreate", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(CustomerSegmentMembersQueryInput), graphql_name="input", default=None)),) + ), + ) + customer_sms_marketing_consent_update = sgqlc.types.Field( + CustomerSmsMarketingConsentUpdatePayload, + graphql_name="customerSmsMarketingConsentUpdate", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(CustomerSmsMarketingConsentUpdateInput), graphql_name="input", default=None)),) + ), + ) + customer_update = sgqlc.types.Field( + CustomerUpdatePayload, + graphql_name="customerUpdate", + args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(CustomerInput), graphql_name="input", default=None)),)), + ) + customer_update_default_address = sgqlc.types.Field( + CustomerUpdateDefaultAddressPayload, + graphql_name="customerUpdateDefaultAddress", + args=sgqlc.types.ArgDict( + ( + ("customer_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerId", default=None)), + ("address_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="addressId", default=None)), + ) + ), + ) + delegate_access_token_create = sgqlc.types.Field( + DelegateAccessTokenCreatePayload, + graphql_name="delegateAccessTokenCreate", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(DelegateAccessTokenInput), graphql_name="input", default=None)),) + ), + ) + delegate_access_token_destroy = sgqlc.types.Field( + DelegateAccessTokenDestroyPayload, + graphql_name="delegateAccessTokenDestroy", + args=sgqlc.types.ArgDict( + (("access_token", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="accessToken", default=None)),) + ), + ) + delivery_customization_activation = sgqlc.types.Field( + DeliveryCustomizationActivationPayload, + graphql_name="deliveryCustomizationActivation", + args=sgqlc.types.ArgDict( + ( + ( + "ids", + sgqlc.types.Arg(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="ids", default=None), + ), + ("enabled", sgqlc.types.Arg(sgqlc.types.non_null(Boolean), graphql_name="enabled", default=None)), + ) + ), + ) + delivery_customization_create = sgqlc.types.Field( + DeliveryCustomizationCreatePayload, + graphql_name="deliveryCustomizationCreate", + args=sgqlc.types.ArgDict( + ( + ( + "delivery_customization", + sgqlc.types.Arg(sgqlc.types.non_null(DeliveryCustomizationInput), graphql_name="deliveryCustomization", default=None), + ), + ) + ), + ) + delivery_customization_delete = sgqlc.types.Field( + DeliveryCustomizationDeletePayload, + graphql_name="deliveryCustomizationDelete", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + delivery_customization_update = sgqlc.types.Field( + DeliveryCustomizationUpdatePayload, + graphql_name="deliveryCustomizationUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "delivery_customization", + sgqlc.types.Arg(sgqlc.types.non_null(DeliveryCustomizationInput), graphql_name="deliveryCustomization", default=None), + ), + ) + ), + ) + delivery_profile_create = sgqlc.types.Field( + "deliveryProfileCreatePayload", + graphql_name="deliveryProfileCreate", + args=sgqlc.types.ArgDict( + (("profile", sgqlc.types.Arg(sgqlc.types.non_null(DeliveryProfileInput), graphql_name="profile", default=None)),) + ), + ) + delivery_profile_remove = sgqlc.types.Field( + "deliveryProfileRemovePayload", + graphql_name="deliveryProfileRemove", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + delivery_profile_update = sgqlc.types.Field( + "deliveryProfileUpdatePayload", + graphql_name="deliveryProfileUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("profile", sgqlc.types.Arg(sgqlc.types.non_null(DeliveryProfileInput), graphql_name="profile", default=None)), + ("leave_legacy_mode_profiles", sgqlc.types.Arg(Boolean, graphql_name="leaveLegacyModeProfiles", default=None)), + ) + ), + ) + delivery_setting_update = sgqlc.types.Field( + DeliverySettingUpdatePayload, + graphql_name="deliverySettingUpdate", + args=sgqlc.types.ArgDict( + (("setting", sgqlc.types.Arg(sgqlc.types.non_null(DeliverySettingInput), graphql_name="setting", default=None)),) + ), + ) + delivery_shipping_origin_assign = sgqlc.types.Field( + DeliveryShippingOriginAssignPayload, + graphql_name="deliveryShippingOriginAssign", + args=sgqlc.types.ArgDict((("location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="locationId", default=None)),)), + ) + discount_automatic_activate = sgqlc.types.Field( + DiscountAutomaticActivatePayload, + graphql_name="discountAutomaticActivate", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + discount_automatic_app_create = sgqlc.types.Field( + DiscountAutomaticAppCreatePayload, + graphql_name="discountAutomaticAppCreate", + args=sgqlc.types.ArgDict( + ( + ( + "automatic_app_discount", + sgqlc.types.Arg(sgqlc.types.non_null(DiscountAutomaticAppInput), graphql_name="automaticAppDiscount", default=None), + ), + ) + ), + ) + discount_automatic_app_update = sgqlc.types.Field( + DiscountAutomaticAppUpdatePayload, + graphql_name="discountAutomaticAppUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "automatic_app_discount", + sgqlc.types.Arg(sgqlc.types.non_null(DiscountAutomaticAppInput), graphql_name="automaticAppDiscount", default=None), + ), + ) + ), + ) + discount_automatic_basic_create = sgqlc.types.Field( + DiscountAutomaticBasicCreatePayload, + graphql_name="discountAutomaticBasicCreate", + args=sgqlc.types.ArgDict( + ( + ( + "automatic_basic_discount", + sgqlc.types.Arg(sgqlc.types.non_null(DiscountAutomaticBasicInput), graphql_name="automaticBasicDiscount", default=None), + ), + ) + ), + ) + discount_automatic_basic_update = sgqlc.types.Field( + DiscountAutomaticBasicUpdatePayload, + graphql_name="discountAutomaticBasicUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "automatic_basic_discount", + sgqlc.types.Arg(sgqlc.types.non_null(DiscountAutomaticBasicInput), graphql_name="automaticBasicDiscount", default=None), + ), + ) + ), + ) + discount_automatic_bulk_delete = sgqlc.types.Field( + DiscountAutomaticBulkDeletePayload, + graphql_name="discountAutomaticBulkDelete", + args=sgqlc.types.ArgDict( + ( + ("search", sgqlc.types.Arg(String, graphql_name="search", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ("ids", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="ids", default=None)), + ) + ), + ) + discount_automatic_bxgy_create = sgqlc.types.Field( + DiscountAutomaticBxgyCreatePayload, + graphql_name="discountAutomaticBxgyCreate", + args=sgqlc.types.ArgDict( + ( + ( + "automatic_bxgy_discount", + sgqlc.types.Arg(sgqlc.types.non_null(DiscountAutomaticBxgyInput), graphql_name="automaticBxgyDiscount", default=None), + ), + ) + ), + ) + discount_automatic_bxgy_update = sgqlc.types.Field( + DiscountAutomaticBxgyUpdatePayload, + graphql_name="discountAutomaticBxgyUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "automatic_bxgy_discount", + sgqlc.types.Arg(sgqlc.types.non_null(DiscountAutomaticBxgyInput), graphql_name="automaticBxgyDiscount", default=None), + ), + ) + ), + ) + discount_automatic_deactivate = sgqlc.types.Field( + DiscountAutomaticDeactivatePayload, + graphql_name="discountAutomaticDeactivate", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + discount_automatic_delete = sgqlc.types.Field( + DiscountAutomaticDeletePayload, + graphql_name="discountAutomaticDelete", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + discount_code_activate = sgqlc.types.Field( + DiscountCodeActivatePayload, + graphql_name="discountCodeActivate", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + discount_code_app_create = sgqlc.types.Field( + DiscountCodeAppCreatePayload, + graphql_name="discountCodeAppCreate", + args=sgqlc.types.ArgDict( + ( + ( + "code_app_discount", + sgqlc.types.Arg(sgqlc.types.non_null(DiscountCodeAppInput), graphql_name="codeAppDiscount", default=None), + ), + ) + ), + ) + discount_code_app_update = sgqlc.types.Field( + DiscountCodeAppUpdatePayload, + graphql_name="discountCodeAppUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "code_app_discount", + sgqlc.types.Arg(sgqlc.types.non_null(DiscountCodeAppInput), graphql_name="codeAppDiscount", default=None), + ), + ) + ), + ) + discount_code_basic_create = sgqlc.types.Field( + DiscountCodeBasicCreatePayload, + graphql_name="discountCodeBasicCreate", + args=sgqlc.types.ArgDict( + ( + ( + "basic_code_discount", + sgqlc.types.Arg(sgqlc.types.non_null(DiscountCodeBasicInput), graphql_name="basicCodeDiscount", default=None), + ), + ) + ), + ) + discount_code_basic_update = sgqlc.types.Field( + DiscountCodeBasicUpdatePayload, + graphql_name="discountCodeBasicUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "basic_code_discount", + sgqlc.types.Arg(sgqlc.types.non_null(DiscountCodeBasicInput), graphql_name="basicCodeDiscount", default=None), + ), + ) + ), + ) + discount_code_bulk_activate = sgqlc.types.Field( + DiscountCodeBulkActivatePayload, + graphql_name="discountCodeBulkActivate", + args=sgqlc.types.ArgDict( + ( + ("search", sgqlc.types.Arg(String, graphql_name="search", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ("ids", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="ids", default=None)), + ) + ), + ) + discount_code_bulk_deactivate = sgqlc.types.Field( + DiscountCodeBulkDeactivatePayload, + graphql_name="discountCodeBulkDeactivate", + args=sgqlc.types.ArgDict( + ( + ("search", sgqlc.types.Arg(String, graphql_name="search", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ("ids", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="ids", default=None)), + ) + ), + ) + discount_code_bulk_delete = sgqlc.types.Field( + DiscountCodeBulkDeletePayload, + graphql_name="discountCodeBulkDelete", + args=sgqlc.types.ArgDict( + ( + ("search", sgqlc.types.Arg(String, graphql_name="search", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ("ids", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="ids", default=None)), + ) + ), + ) + discount_code_bxgy_create = sgqlc.types.Field( + DiscountCodeBxgyCreatePayload, + graphql_name="discountCodeBxgyCreate", + args=sgqlc.types.ArgDict( + ( + ( + "bxgy_code_discount", + sgqlc.types.Arg(sgqlc.types.non_null(DiscountCodeBxgyInput), graphql_name="bxgyCodeDiscount", default=None), + ), + ) + ), + ) + discount_code_bxgy_update = sgqlc.types.Field( + DiscountCodeBxgyUpdatePayload, + graphql_name="discountCodeBxgyUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "bxgy_code_discount", + sgqlc.types.Arg(sgqlc.types.non_null(DiscountCodeBxgyInput), graphql_name="bxgyCodeDiscount", default=None), + ), + ) + ), + ) + discount_code_deactivate = sgqlc.types.Field( + DiscountCodeDeactivatePayload, + graphql_name="discountCodeDeactivate", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + discount_code_delete = sgqlc.types.Field( + DiscountCodeDeletePayload, + graphql_name="discountCodeDelete", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + discount_code_free_shipping_create = sgqlc.types.Field( + DiscountCodeFreeShippingCreatePayload, + graphql_name="discountCodeFreeShippingCreate", + args=sgqlc.types.ArgDict( + ( + ( + "free_shipping_code_discount", + sgqlc.types.Arg( + sgqlc.types.non_null(DiscountCodeFreeShippingInput), graphql_name="freeShippingCodeDiscount", default=None + ), + ), + ) + ), + ) + discount_code_free_shipping_update = sgqlc.types.Field( + DiscountCodeFreeShippingUpdatePayload, + graphql_name="discountCodeFreeShippingUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "free_shipping_code_discount", + sgqlc.types.Arg( + sgqlc.types.non_null(DiscountCodeFreeShippingInput), graphql_name="freeShippingCodeDiscount", default=None + ), + ), + ) + ), + ) + discount_code_redeem_code_bulk_delete = sgqlc.types.Field( + DiscountCodeRedeemCodeBulkDeletePayload, + graphql_name="discountCodeRedeemCodeBulkDelete", + args=sgqlc.types.ArgDict( + ( + ("discount_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="discountId", default=None)), + ("search", sgqlc.types.Arg(String, graphql_name="search", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ("ids", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="ids", default=None)), + ) + ), + ) + discount_redeem_code_bulk_add = sgqlc.types.Field( + DiscountRedeemCodeBulkAddPayload, + graphql_name="discountRedeemCodeBulkAdd", + args=sgqlc.types.ArgDict( + ( + ("discount_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="discountId", default=None)), + ( + "codes", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DiscountRedeemCodeInput))), + graphql_name="codes", + default=None, + ), + ), + ) + ), + ) + dispute_evidence_update = sgqlc.types.Field( + DisputeEvidenceUpdatePayload, + graphql_name="disputeEvidenceUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "input", + sgqlc.types.Arg(sgqlc.types.non_null(ShopifyPaymentsDisputeEvidenceUpdateInput), graphql_name="input", default=None), + ), + ) + ), + ) + draft_order_bulk_add_tags = sgqlc.types.Field( + DraftOrderBulkAddTagsPayload, + graphql_name="draftOrderBulkAddTags", + args=sgqlc.types.ArgDict( + ( + ("search", sgqlc.types.Arg(String, graphql_name="search", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ("ids", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="ids", default=None)), + ( + "tags", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="tags", default=None + ), + ), + ) + ), + ) + draft_order_bulk_delete = sgqlc.types.Field( + DraftOrderBulkDeletePayload, + graphql_name="draftOrderBulkDelete", + args=sgqlc.types.ArgDict( + ( + ("search", sgqlc.types.Arg(String, graphql_name="search", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ("ids", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="ids", default=None)), + ) + ), + ) + draft_order_bulk_remove_tags = sgqlc.types.Field( + DraftOrderBulkRemoveTagsPayload, + graphql_name="draftOrderBulkRemoveTags", + args=sgqlc.types.ArgDict( + ( + ("search", sgqlc.types.Arg(String, graphql_name="search", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ("ids", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="ids", default=None)), + ( + "tags", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="tags", default=None + ), + ), + ) + ), + ) + draft_order_calculate = sgqlc.types.Field( + DraftOrderCalculatePayload, + graphql_name="draftOrderCalculate", + args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(DraftOrderInput), graphql_name="input", default=None)),)), + ) + draft_order_complete = sgqlc.types.Field( + DraftOrderCompletePayload, + graphql_name="draftOrderComplete", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("payment_pending", sgqlc.types.Arg(Boolean, graphql_name="paymentPending", default=False)), + ("payment_gateway_id", sgqlc.types.Arg(ID, graphql_name="paymentGatewayId", default=None)), + ("source_name", sgqlc.types.Arg(String, graphql_name="sourceName", default=None)), + ) + ), + ) + draft_order_create = sgqlc.types.Field( + DraftOrderCreatePayload, + graphql_name="draftOrderCreate", + args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(DraftOrderInput), graphql_name="input", default=None)),)), + ) + draft_order_create_from_order = sgqlc.types.Field( + DraftOrderCreateFromOrderPayload, + graphql_name="draftOrderCreateFromOrder", + args=sgqlc.types.ArgDict((("order_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="orderId", default=None)),)), + ) + draft_order_create_merchant_checkout = sgqlc.types.Field( + DraftOrderCreateMerchantCheckoutPayload, + graphql_name="draftOrderCreateMerchantCheckout", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + draft_order_delete = sgqlc.types.Field( + DraftOrderDeletePayload, + graphql_name="draftOrderDelete", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(DraftOrderDeleteInput), graphql_name="input", default=None)),) + ), + ) + draft_order_duplicate = sgqlc.types.Field( + DraftOrderDuplicatePayload, + graphql_name="draftOrderDuplicate", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(ID, graphql_name="id", default=None)),)), + ) + draft_order_invoice_preview = sgqlc.types.Field( + DraftOrderInvoicePreviewPayload, + graphql_name="draftOrderInvoicePreview", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("email", sgqlc.types.Arg(EmailInput, graphql_name="email", default=None)), + ) + ), + ) + draft_order_invoice_send = sgqlc.types.Field( + DraftOrderInvoiceSendPayload, + graphql_name="draftOrderInvoiceSend", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("email", sgqlc.types.Arg(EmailInput, graphql_name="email", default=None)), + ) + ), + ) + draft_order_update = sgqlc.types.Field( + DraftOrderUpdatePayload, + graphql_name="draftOrderUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("input", sgqlc.types.Arg(sgqlc.types.non_null(DraftOrderInput), graphql_name="input", default=None)), + ) + ), + ) + event_bridge_server_pixel_update = sgqlc.types.Field( + EventBridgeServerPixelUpdatePayload, + graphql_name="eventBridgeServerPixelUpdate", + args=sgqlc.types.ArgDict((("arn", sgqlc.types.Arg(sgqlc.types.non_null(ARN), graphql_name="arn", default=None)),)), + ) + event_bridge_webhook_subscription_create = sgqlc.types.Field( + EventBridgeWebhookSubscriptionCreatePayload, + graphql_name="eventBridgeWebhookSubscriptionCreate", + args=sgqlc.types.ArgDict( + ( + ("topic", sgqlc.types.Arg(sgqlc.types.non_null(WebhookSubscriptionTopic), graphql_name="topic", default=None)), + ( + "webhook_subscription", + sgqlc.types.Arg( + sgqlc.types.non_null(EventBridgeWebhookSubscriptionInput), graphql_name="webhookSubscription", default=None + ), + ), + ) + ), + ) + event_bridge_webhook_subscription_update = sgqlc.types.Field( + EventBridgeWebhookSubscriptionUpdatePayload, + graphql_name="eventBridgeWebhookSubscriptionUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "webhook_subscription", + sgqlc.types.Arg( + sgqlc.types.non_null(EventBridgeWebhookSubscriptionInput), graphql_name="webhookSubscription", default=None + ), + ), + ) + ), + ) + file_acknowledge_update_failed = sgqlc.types.Field( + FileAcknowledgeUpdateFailedPayload, + graphql_name="fileAcknowledgeUpdateFailed", + args=sgqlc.types.ArgDict( + ( + ( + "file_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="fileIds", default=None + ), + ), + ) + ), + ) + file_create = sgqlc.types.Field( + FileCreatePayload, + graphql_name="fileCreate", + args=sgqlc.types.ArgDict( + ( + ( + "files", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FileCreateInput))), graphql_name="files", default=None + ), + ), + ) + ), + ) + file_delete = sgqlc.types.Field( + FileDeletePayload, + graphql_name="fileDelete", + args=sgqlc.types.ArgDict( + ( + ( + "file_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="fileIds", default=None + ), + ), + ) + ), + ) + file_update = sgqlc.types.Field( + FileUpdatePayload, + graphql_name="fileUpdate", + args=sgqlc.types.ArgDict( + ( + ( + "files", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FileUpdateInput))), graphql_name="files", default=None + ), + ), + ) + ), + ) + flow_trigger_receive = sgqlc.types.Field( + FlowTriggerReceivePayload, + graphql_name="flowTriggerReceive", + args=sgqlc.types.ArgDict((("body", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="body", default=None)),)), + ) + fulfillment_cancel = sgqlc.types.Field( + FulfillmentCancelPayload, + graphql_name="fulfillmentCancel", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + fulfillment_create_v2 = sgqlc.types.Field( + FulfillmentCreateV2Payload, + graphql_name="fulfillmentCreateV2", + args=sgqlc.types.ArgDict( + ( + ("fulfillment", sgqlc.types.Arg(sgqlc.types.non_null(FulfillmentV2Input), graphql_name="fulfillment", default=None)), + ("message", sgqlc.types.Arg(String, graphql_name="message", default=None)), + ) + ), + ) + fulfillment_event_create = sgqlc.types.Field( + FulfillmentEventCreatePayload, + graphql_name="fulfillmentEventCreate", + args=sgqlc.types.ArgDict( + ( + ( + "fulfillment_event", + sgqlc.types.Arg(sgqlc.types.non_null(FulfillmentEventInput), graphql_name="fulfillmentEvent", default=None), + ), + ) + ), + ) + fulfillment_order_accept_cancellation_request = sgqlc.types.Field( + FulfillmentOrderAcceptCancellationRequestPayload, + graphql_name="fulfillmentOrderAcceptCancellationRequest", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("message", sgqlc.types.Arg(String, graphql_name="message", default=None)), + ) + ), + ) + fulfillment_order_accept_fulfillment_request = sgqlc.types.Field( + FulfillmentOrderAcceptFulfillmentRequestPayload, + graphql_name="fulfillmentOrderAcceptFulfillmentRequest", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("message", sgqlc.types.Arg(String, graphql_name="message", default=None)), + ) + ), + ) + fulfillment_order_cancel = sgqlc.types.Field( + FulfillmentOrderCancelPayload, + graphql_name="fulfillmentOrderCancel", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + fulfillment_order_close = sgqlc.types.Field( + FulfillmentOrderClosePayload, + graphql_name="fulfillmentOrderClose", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("message", sgqlc.types.Arg(String, graphql_name="message", default=None)), + ) + ), + ) + fulfillment_order_hold = sgqlc.types.Field( + FulfillmentOrderHoldPayload, + graphql_name="fulfillmentOrderHold", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "fulfillment_hold", + sgqlc.types.Arg(sgqlc.types.non_null(FulfillmentOrderHoldInput), graphql_name="fulfillmentHold", default=None), + ), + ) + ), + ) + fulfillment_order_line_items_prepared_for_pickup = sgqlc.types.Field( + FulfillmentOrderLineItemsPreparedForPickupPayload, + graphql_name="fulfillmentOrderLineItemsPreparedForPickup", + args=sgqlc.types.ArgDict( + ( + ( + "input", + sgqlc.types.Arg( + sgqlc.types.non_null(FulfillmentOrderLineItemsPreparedForPickupInput), graphql_name="input", default=None + ), + ), + ) + ), + ) + fulfillment_order_merge = sgqlc.types.Field( + FulfillmentOrderMergePayload, + graphql_name="fulfillmentOrderMerge", + args=sgqlc.types.ArgDict( + ( + ( + "fulfillment_order_merge_inputs", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FulfillmentOrderMergeInput))), + graphql_name="fulfillmentOrderMergeInputs", + default=None, + ), + ), + ) + ), + ) + fulfillment_order_move = sgqlc.types.Field( + FulfillmentOrderMovePayload, + graphql_name="fulfillmentOrderMove", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("new_location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="newLocationId", default=None)), + ( + "fulfillment_order_line_items", + sgqlc.types.Arg( + sgqlc.types.list_of(sgqlc.types.non_null(FulfillmentOrderLineItemInput)), + graphql_name="fulfillmentOrderLineItems", + default=None, + ), + ), + ) + ), + ) + fulfillment_order_open = sgqlc.types.Field( + FulfillmentOrderOpenPayload, + graphql_name="fulfillmentOrderOpen", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + fulfillment_order_reject_cancellation_request = sgqlc.types.Field( + FulfillmentOrderRejectCancellationRequestPayload, + graphql_name="fulfillmentOrderRejectCancellationRequest", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("message", sgqlc.types.Arg(String, graphql_name="message", default=None)), + ) + ), + ) + fulfillment_order_reject_fulfillment_request = sgqlc.types.Field( + FulfillmentOrderRejectFulfillmentRequestPayload, + graphql_name="fulfillmentOrderRejectFulfillmentRequest", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("reason", sgqlc.types.Arg(FulfillmentOrderRejectionReason, graphql_name="reason", default=None)), + ("message", sgqlc.types.Arg(String, graphql_name="message", default=None)), + ( + "line_items", + sgqlc.types.Arg( + sgqlc.types.list_of(sgqlc.types.non_null(IncomingRequestLineItemInput)), graphql_name="lineItems", default=None + ), + ), + ) + ), + ) + fulfillment_order_release_hold = sgqlc.types.Field( + FulfillmentOrderReleaseHoldPayload, + graphql_name="fulfillmentOrderReleaseHold", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("external_id", sgqlc.types.Arg(String, graphql_name="externalId", default=None)), + ) + ), + ) + fulfillment_order_reschedule = sgqlc.types.Field( + FulfillmentOrderReschedulePayload, + graphql_name="fulfillmentOrderReschedule", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("fulfill_at", sgqlc.types.Arg(sgqlc.types.non_null(DateTime), graphql_name="fulfillAt", default=None)), + ) + ), + ) + fulfillment_order_split = sgqlc.types.Field( + FulfillmentOrderSplitPayload, + graphql_name="fulfillmentOrderSplit", + args=sgqlc.types.ArgDict( + ( + ( + "fulfillment_order_splits", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FulfillmentOrderSplitInput))), + graphql_name="fulfillmentOrderSplits", + default=None, + ), + ), + ) + ), + ) + fulfillment_order_submit_cancellation_request = sgqlc.types.Field( + FulfillmentOrderSubmitCancellationRequestPayload, + graphql_name="fulfillmentOrderSubmitCancellationRequest", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("message", sgqlc.types.Arg(String, graphql_name="message", default=None)), + ) + ), + ) + fulfillment_order_submit_fulfillment_request = sgqlc.types.Field( + FulfillmentOrderSubmitFulfillmentRequestPayload, + graphql_name="fulfillmentOrderSubmitFulfillmentRequest", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("message", sgqlc.types.Arg(String, graphql_name="message", default=None)), + ("notify_customer", sgqlc.types.Arg(Boolean, graphql_name="notifyCustomer", default=None)), + ( + "fulfillment_order_line_items", + sgqlc.types.Arg( + sgqlc.types.list_of(sgqlc.types.non_null(FulfillmentOrderLineItemInput)), + graphql_name="fulfillmentOrderLineItems", + default=None, + ), + ), + ("shipping_method", sgqlc.types.Arg(String, graphql_name="shippingMethod", default=None)), + ) + ), + ) + fulfillment_orders_release_holds = sgqlc.types.Field( + FulfillmentOrdersReleaseHoldsPayload, + graphql_name="fulfillmentOrdersReleaseHolds", + args=sgqlc.types.ArgDict( + ( + ( + "ids", + sgqlc.types.Arg(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="ids", default=None), + ), + ("external_id", sgqlc.types.Arg(String, graphql_name="externalId", default=None)), + ) + ), + ) + fulfillment_orders_set_fulfillment_deadline = sgqlc.types.Field( + FulfillmentOrdersSetFulfillmentDeadlinePayload, + graphql_name="fulfillmentOrdersSetFulfillmentDeadline", + args=sgqlc.types.ArgDict( + ( + ( + "fulfillment_order_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), + graphql_name="fulfillmentOrderIds", + default=None, + ), + ), + ("fulfillment_deadline", sgqlc.types.Arg(sgqlc.types.non_null(DateTime), graphql_name="fulfillmentDeadline", default=None)), + ) + ), + ) + fulfillment_service_create = sgqlc.types.Field( + FulfillmentServiceCreatePayload, + graphql_name="fulfillmentServiceCreate", + args=sgqlc.types.ArgDict( + ( + ("name", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="name", default=None)), + ("callback_url", sgqlc.types.Arg(sgqlc.types.non_null(URL), graphql_name="callbackUrl", default=None)), + ("tracking_support", sgqlc.types.Arg(Boolean, graphql_name="trackingSupport", default=False)), + ( + "fulfillment_orders_opt_in", + sgqlc.types.Arg(sgqlc.types.non_null(Boolean), graphql_name="fulfillmentOrdersOptIn", default=None), + ), + ("permits_sku_sharing", sgqlc.types.Arg(Boolean, graphql_name="permitsSkuSharing", default=False)), + ("inventory_management", sgqlc.types.Arg(Boolean, graphql_name="inventoryManagement", default=False)), + ) + ), + ) + fulfillment_service_delete = sgqlc.types.Field( + FulfillmentServiceDeletePayload, + graphql_name="fulfillmentServiceDelete", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("destination_location_id", sgqlc.types.Arg(ID, graphql_name="destinationLocationId", default=None)), + ) + ), + ) + fulfillment_service_update = sgqlc.types.Field( + FulfillmentServiceUpdatePayload, + graphql_name="fulfillmentServiceUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("name", sgqlc.types.Arg(String, graphql_name="name", default=None)), + ("callback_url", sgqlc.types.Arg(URL, graphql_name="callbackUrl", default=None)), + ("tracking_support", sgqlc.types.Arg(Boolean, graphql_name="trackingSupport", default=None)), + ("fulfillment_orders_opt_in", sgqlc.types.Arg(Boolean, graphql_name="fulfillmentOrdersOptIn", default=None)), + ("permits_sku_sharing", sgqlc.types.Arg(Boolean, graphql_name="permitsSkuSharing", default=None)), + ) + ), + ) + fulfillment_tracking_info_update_v2 = sgqlc.types.Field( + FulfillmentTrackingInfoUpdateV2Payload, + graphql_name="fulfillmentTrackingInfoUpdateV2", + args=sgqlc.types.ArgDict( + ( + ("fulfillment_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="fulfillmentId", default=None)), + ( + "tracking_info_input", + sgqlc.types.Arg(sgqlc.types.non_null(FulfillmentTrackingInput), graphql_name="trackingInfoInput", default=None), + ), + ("notify_customer", sgqlc.types.Arg(Boolean, graphql_name="notifyCustomer", default=None)), + ) + ), + ) + gift_card_create = sgqlc.types.Field( + GiftCardCreatePayload, + graphql_name="giftCardCreate", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(GiftCardCreateInput), graphql_name="input", default=None)),) + ), + ) + gift_card_disable = sgqlc.types.Field( + GiftCardDisablePayload, + graphql_name="giftCardDisable", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + gift_card_update = sgqlc.types.Field( + GiftCardUpdatePayload, + graphql_name="giftCardUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("input", sgqlc.types.Arg(sgqlc.types.non_null(GiftCardUpdateInput), graphql_name="input", default=None)), + ) + ), + ) + inventory_activate = sgqlc.types.Field( + InventoryActivatePayload, + graphql_name="inventoryActivate", + args=sgqlc.types.ArgDict( + ( + ("inventory_item_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="inventoryItemId", default=None)), + ("location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="locationId", default=None)), + ("available", sgqlc.types.Arg(Int, graphql_name="available", default=None)), + ("on_hand", sgqlc.types.Arg(Int, graphql_name="onHand", default=None)), + ) + ), + ) + inventory_adjust_quantities = sgqlc.types.Field( + InventoryAdjustQuantitiesPayload, + graphql_name="inventoryAdjustQuantities", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(InventoryAdjustQuantitiesInput), graphql_name="input", default=None)),) + ), + ) + inventory_bulk_toggle_activation = sgqlc.types.Field( + InventoryBulkToggleActivationPayload, + graphql_name="inventoryBulkToggleActivation", + args=sgqlc.types.ArgDict( + ( + ("inventory_item_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="inventoryItemId", default=None)), + ( + "inventory_item_updates", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(InventoryBulkToggleActivationInput))), + graphql_name="inventoryItemUpdates", + default=None, + ), + ), + ) + ), + ) + inventory_deactivate = sgqlc.types.Field( + InventoryDeactivatePayload, + graphql_name="inventoryDeactivate", + args=sgqlc.types.ArgDict( + (("inventory_level_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="inventoryLevelId", default=None)),) + ), + ) + inventory_item_update = sgqlc.types.Field( + InventoryItemUpdatePayload, + graphql_name="inventoryItemUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("input", sgqlc.types.Arg(sgqlc.types.non_null(InventoryItemUpdateInput), graphql_name="input", default=None)), + ) + ), + ) + inventory_move_quantities = sgqlc.types.Field( + InventoryMoveQuantitiesPayload, + graphql_name="inventoryMoveQuantities", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(InventoryMoveQuantitiesInput), graphql_name="input", default=None)),) + ), + ) + inventory_set_on_hand_quantities = sgqlc.types.Field( + InventorySetOnHandQuantitiesPayload, + graphql_name="inventorySetOnHandQuantities", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(InventorySetOnHandQuantitiesInput), graphql_name="input", default=None)),) + ), + ) + location_activate = sgqlc.types.Field( + LocationActivatePayload, + graphql_name="locationActivate", + args=sgqlc.types.ArgDict((("location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="locationId", default=None)),)), + ) + location_add = sgqlc.types.Field( + LocationAddPayload, + graphql_name="locationAdd", + args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(LocationAddInput), graphql_name="input", default=None)),)), + ) + location_deactivate = sgqlc.types.Field( + LocationDeactivatePayload, + graphql_name="locationDeactivate", + args=sgqlc.types.ArgDict( + ( + ("location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="locationId", default=None)), + ("destination_location_id", sgqlc.types.Arg(ID, graphql_name="destinationLocationId", default=None)), + ) + ), + ) + location_delete = sgqlc.types.Field( + LocationDeletePayload, + graphql_name="locationDelete", + args=sgqlc.types.ArgDict((("location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="locationId", default=None)),)), + ) + location_edit = sgqlc.types.Field( + LocationEditPayload, + graphql_name="locationEdit", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("input", sgqlc.types.Arg(sgqlc.types.non_null(LocationEditInput), graphql_name="input", default=None)), + ) + ), + ) + location_local_pickup_disable = sgqlc.types.Field( + LocationLocalPickupDisablePayload, + graphql_name="locationLocalPickupDisable", + args=sgqlc.types.ArgDict((("location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="locationId", default=None)),)), + ) + location_local_pickup_enable = sgqlc.types.Field( + LocationLocalPickupEnablePayload, + graphql_name="locationLocalPickupEnable", + args=sgqlc.types.ArgDict( + ( + ( + "local_pickup_settings", + sgqlc.types.Arg( + sgqlc.types.non_null(DeliveryLocationLocalPickupEnableInput), graphql_name="localPickupSettings", default=None + ), + ), + ) + ), + ) + market_create = sgqlc.types.Field( + MarketCreatePayload, + graphql_name="marketCreate", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(MarketCreateInput), graphql_name="input", default=None)),) + ), + ) + market_currency_settings_update = sgqlc.types.Field( + MarketCurrencySettingsUpdatePayload, + graphql_name="marketCurrencySettingsUpdate", + args=sgqlc.types.ArgDict( + ( + ("market_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="marketId", default=None)), + ("input", sgqlc.types.Arg(sgqlc.types.non_null(MarketCurrencySettingsUpdateInput), graphql_name="input", default=None)), + ) + ), + ) + market_delete = sgqlc.types.Field( + MarketDeletePayload, + graphql_name="marketDelete", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + market_localizations_register = sgqlc.types.Field( + MarketLocalizationsRegisterPayload, + graphql_name="marketLocalizationsRegister", + args=sgqlc.types.ArgDict( + ( + ("resource_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="resourceId", default=None)), + ( + "market_localizations", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MarketLocalizationRegisterInput))), + graphql_name="marketLocalizations", + default=None, + ), + ), + ) + ), + ) + market_localizations_remove = sgqlc.types.Field( + MarketLocalizationsRemovePayload, + graphql_name="marketLocalizationsRemove", + args=sgqlc.types.ArgDict( + ( + ("resource_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="resourceId", default=None)), + ( + "market_localization_keys", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), + graphql_name="marketLocalizationKeys", + default=None, + ), + ), + ( + "market_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="marketIds", default=None + ), + ), + ) + ), + ) + market_region_delete = sgqlc.types.Field( + MarketRegionDeletePayload, + graphql_name="marketRegionDelete", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + market_regions_create = sgqlc.types.Field( + MarketRegionsCreatePayload, + graphql_name="marketRegionsCreate", + args=sgqlc.types.ArgDict( + ( + ("market_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="marketId", default=None)), + ( + "regions", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MarketRegionCreateInput))), + graphql_name="regions", + default=None, + ), + ), + ) + ), + ) + market_update = sgqlc.types.Field( + MarketUpdatePayload, + graphql_name="marketUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("input", sgqlc.types.Arg(sgqlc.types.non_null(MarketUpdateInput), graphql_name="input", default=None)), + ) + ), + ) + market_web_presence_create = sgqlc.types.Field( + MarketWebPresenceCreatePayload, + graphql_name="marketWebPresenceCreate", + args=sgqlc.types.ArgDict( + ( + ("market_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="marketId", default=None)), + ( + "web_presence", + sgqlc.types.Arg(sgqlc.types.non_null(MarketWebPresenceCreateInput), graphql_name="webPresence", default=None), + ), + ) + ), + ) + market_web_presence_delete = sgqlc.types.Field( + MarketWebPresenceDeletePayload, + graphql_name="marketWebPresenceDelete", + args=sgqlc.types.ArgDict((("market_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="marketId", default=None)),)), + ) + market_web_presence_update = sgqlc.types.Field( + MarketWebPresenceUpdatePayload, + graphql_name="marketWebPresenceUpdate", + args=sgqlc.types.ArgDict( + ( + ("market_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="marketId", default=None)), + ( + "web_presence", + sgqlc.types.Arg(sgqlc.types.non_null(MarketWebPresenceUpdateInput), graphql_name="webPresence", default=None), + ), + ) + ), + ) + marketing_activity_create = sgqlc.types.Field( + MarketingActivityCreatePayload, + graphql_name="marketingActivityCreate", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(MarketingActivityCreateInput), graphql_name="input", default=None)),) + ), + ) + marketing_activity_create_external = sgqlc.types.Field( + MarketingActivityCreateExternalPayload, + graphql_name="marketingActivityCreateExternal", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(MarketingActivityCreateExternalInput), graphql_name="input", default=None)),) + ), + ) + marketing_activity_update = sgqlc.types.Field( + MarketingActivityUpdatePayload, + graphql_name="marketingActivityUpdate", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(MarketingActivityUpdateInput), graphql_name="input", default=None)),) + ), + ) + marketing_activity_update_external = sgqlc.types.Field( + MarketingActivityUpdateExternalPayload, + graphql_name="marketingActivityUpdateExternal", + args=sgqlc.types.ArgDict( + ( + ("input", sgqlc.types.Arg(sgqlc.types.non_null(MarketingActivityUpdateExternalInput), graphql_name="input", default=None)), + ("marketing_activity_id", sgqlc.types.Arg(ID, graphql_name="marketingActivityId", default=None)), + ("remote_id", sgqlc.types.Arg(String, graphql_name="remoteId", default=None)), + ("utm", sgqlc.types.Arg(UTMInput, graphql_name="utm", default=None)), + ) + ), + ) + marketing_engagement_create = sgqlc.types.Field( + MarketingEngagementCreatePayload, + graphql_name="marketingEngagementCreate", + args=sgqlc.types.ArgDict( + ( + ("marketing_activity_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="marketingActivityId", default=None)), + ( + "marketing_engagement", + sgqlc.types.Arg(sgqlc.types.non_null(MarketingEngagementInput), graphql_name="marketingEngagement", default=None), + ), + ) + ), + ) + metafield_definition_create = sgqlc.types.Field( + MetafieldDefinitionCreatePayload, + graphql_name="metafieldDefinitionCreate", + args=sgqlc.types.ArgDict( + (("definition", sgqlc.types.Arg(sgqlc.types.non_null(MetafieldDefinitionInput), graphql_name="definition", default=None)),) + ), + ) + metafield_definition_delete = sgqlc.types.Field( + MetafieldDefinitionDeletePayload, + graphql_name="metafieldDefinitionDelete", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("delete_all_associated_metafields", sgqlc.types.Arg(Boolean, graphql_name="deleteAllAssociatedMetafields", default=False)), + ) + ), + ) + metafield_definition_pin = sgqlc.types.Field( + MetafieldDefinitionPinPayload, + graphql_name="metafieldDefinitionPin", + args=sgqlc.types.ArgDict( + (("definition_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="definitionId", default=None)),) + ), + ) + metafield_definition_unpin = sgqlc.types.Field( + MetafieldDefinitionUnpinPayload, + graphql_name="metafieldDefinitionUnpin", + args=sgqlc.types.ArgDict( + (("definition_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="definitionId", default=None)),) + ), + ) + metafield_definition_update = sgqlc.types.Field( + MetafieldDefinitionUpdatePayload, + graphql_name="metafieldDefinitionUpdate", + args=sgqlc.types.ArgDict( + ( + ( + "definition", + sgqlc.types.Arg(sgqlc.types.non_null(MetafieldDefinitionUpdateInput), graphql_name="definition", default=None), + ), + ) + ), + ) + metafield_delete = sgqlc.types.Field( + MetafieldDeletePayload, + graphql_name="metafieldDelete", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(MetafieldDeleteInput), graphql_name="input", default=None)),) + ), + ) + metafields_set = sgqlc.types.Field( + MetafieldsSetPayload, + graphql_name="metafieldsSet", + args=sgqlc.types.ArgDict( + ( + ( + "metafields", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MetafieldsSetInput))), + graphql_name="metafields", + default=None, + ), + ), + ) + ), + ) + metaobject_bulk_delete = sgqlc.types.Field( + MetaobjectBulkDeletePayload, + graphql_name="metaobjectBulkDelete", + args=sgqlc.types.ArgDict( + (("where", sgqlc.types.Arg(sgqlc.types.non_null(MetaobjectBulkDeleteWhereCondition), graphql_name="where", default=None)),) + ), + ) + metaobject_create = sgqlc.types.Field( + MetaobjectCreatePayload, + graphql_name="metaobjectCreate", + args=sgqlc.types.ArgDict( + (("metaobject", sgqlc.types.Arg(sgqlc.types.non_null(MetaobjectCreateInput), graphql_name="metaobject", default=None)),) + ), + ) + metaobject_definition_create = sgqlc.types.Field( + MetaobjectDefinitionCreatePayload, + graphql_name="metaobjectDefinitionCreate", + args=sgqlc.types.ArgDict( + ( + ( + "definition", + sgqlc.types.Arg(sgqlc.types.non_null(MetaobjectDefinitionCreateInput), graphql_name="definition", default=None), + ), + ) + ), + ) + metaobject_definition_delete = sgqlc.types.Field( + MetaobjectDefinitionDeletePayload, + graphql_name="metaobjectDefinitionDelete", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + metaobject_definition_update = sgqlc.types.Field( + MetaobjectDefinitionUpdatePayload, + graphql_name="metaobjectDefinitionUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "definition", + sgqlc.types.Arg(sgqlc.types.non_null(MetaobjectDefinitionUpdateInput), graphql_name="definition", default=None), + ), + ) + ), + ) + metaobject_delete = sgqlc.types.Field( + MetaobjectDeletePayload, + graphql_name="metaobjectDelete", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + metaobject_update = sgqlc.types.Field( + MetaobjectUpdatePayload, + graphql_name="metaobjectUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("metaobject", sgqlc.types.Arg(sgqlc.types.non_null(MetaobjectUpdateInput), graphql_name="metaobject", default=None)), + ) + ), + ) + metaobject_upsert = sgqlc.types.Field( + MetaobjectUpsertPayload, + graphql_name="metaobjectUpsert", + args=sgqlc.types.ArgDict( + ( + ("handle", sgqlc.types.Arg(sgqlc.types.non_null(MetaobjectHandleInput), graphql_name="handle", default=None)), + ("metaobject", sgqlc.types.Arg(sgqlc.types.non_null(MetaobjectUpsertInput), graphql_name="metaobject", default=None)), + ) + ), + ) + order_capture = sgqlc.types.Field( + "OrderCapturePayload", + graphql_name="orderCapture", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(OrderCaptureInput), graphql_name="input", default=None)),) + ), + ) + order_close = sgqlc.types.Field( + "OrderClosePayload", + graphql_name="orderClose", + args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(OrderCloseInput), graphql_name="input", default=None)),)), + ) + order_create_mandate_payment = sgqlc.types.Field( + "OrderCreateMandatePaymentPayload", + graphql_name="orderCreateMandatePayment", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("payment_schedule_id", sgqlc.types.Arg(ID, graphql_name="paymentScheduleId", default=None)), + ("idempotency_key", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="idempotencyKey", default=None)), + ("mandate_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="mandateId", default=None)), + ("auto_capture", sgqlc.types.Arg(Boolean, graphql_name="autoCapture", default=True)), + ) + ), + ) + order_edit_add_custom_item = sgqlc.types.Field( + "OrderEditAddCustomItemPayload", + graphql_name="orderEditAddCustomItem", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("title", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="title", default=None)), + ("location_id", sgqlc.types.Arg(ID, graphql_name="locationId", default=None)), + ("price", sgqlc.types.Arg(sgqlc.types.non_null(MoneyInput), graphql_name="price", default=None)), + ("quantity", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="quantity", default=None)), + ("taxable", sgqlc.types.Arg(Boolean, graphql_name="taxable", default=None)), + ("requires_shipping", sgqlc.types.Arg(Boolean, graphql_name="requiresShipping", default=None)), + ) + ), + ) + order_edit_add_line_item_discount = sgqlc.types.Field( + "OrderEditAddLineItemDiscountPayload", + graphql_name="orderEditAddLineItemDiscount", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("line_item_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="lineItemId", default=None)), + ("discount", sgqlc.types.Arg(sgqlc.types.non_null(OrderEditAppliedDiscountInput), graphql_name="discount", default=None)), + ) + ), + ) + order_edit_add_variant = sgqlc.types.Field( + "OrderEditAddVariantPayload", + graphql_name="orderEditAddVariant", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("variant_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="variantId", default=None)), + ("location_id", sgqlc.types.Arg(ID, graphql_name="locationId", default=None)), + ("quantity", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="quantity", default=None)), + ("allow_duplicates", sgqlc.types.Arg(Boolean, graphql_name="allowDuplicates", default=False)), + ) + ), + ) + order_edit_begin = sgqlc.types.Field( + "OrderEditBeginPayload", + graphql_name="orderEditBegin", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + order_edit_commit = sgqlc.types.Field( + "OrderEditCommitPayload", + graphql_name="orderEditCommit", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("notify_customer", sgqlc.types.Arg(Boolean, graphql_name="notifyCustomer", default=None)), + ("staff_note", sgqlc.types.Arg(String, graphql_name="staffNote", default=None)), + ) + ), + ) + order_edit_remove_line_item_discount = sgqlc.types.Field( + "OrderEditRemoveLineItemDiscountPayload", + graphql_name="orderEditRemoveLineItemDiscount", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("discount_application_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="discountApplicationId", default=None)), + ) + ), + ) + order_edit_set_quantity = sgqlc.types.Field( + "OrderEditSetQuantityPayload", + graphql_name="orderEditSetQuantity", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("line_item_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="lineItemId", default=None)), + ("quantity", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="quantity", default=None)), + ("restock", sgqlc.types.Arg(Boolean, graphql_name="restock", default=None)), + ) + ), + ) + order_invoice_send = sgqlc.types.Field( + "OrderInvoiceSendPayload", + graphql_name="orderInvoiceSend", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("email", sgqlc.types.Arg(EmailInput, graphql_name="email", default=None)), + ) + ), + ) + order_mark_as_paid = sgqlc.types.Field( + "OrderMarkAsPaidPayload", + graphql_name="orderMarkAsPaid", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(OrderMarkAsPaidInput), graphql_name="input", default=None)),) + ), + ) + order_open = sgqlc.types.Field( + "OrderOpenPayload", + graphql_name="orderOpen", + args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(OrderOpenInput), graphql_name="input", default=None)),)), + ) + order_update = sgqlc.types.Field( + "OrderUpdatePayload", + graphql_name="orderUpdate", + args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(OrderInput), graphql_name="input", default=None)),)), + ) + payment_customization_activation = sgqlc.types.Field( + "PaymentCustomizationActivationPayload", + graphql_name="paymentCustomizationActivation", + args=sgqlc.types.ArgDict( + ( + ( + "ids", + sgqlc.types.Arg(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="ids", default=None), + ), + ("enabled", sgqlc.types.Arg(sgqlc.types.non_null(Boolean), graphql_name="enabled", default=None)), + ) + ), + ) + payment_customization_create = sgqlc.types.Field( + "PaymentCustomizationCreatePayload", + graphql_name="paymentCustomizationCreate", + args=sgqlc.types.ArgDict( + ( + ( + "payment_customization", + sgqlc.types.Arg(sgqlc.types.non_null(PaymentCustomizationInput), graphql_name="paymentCustomization", default=None), + ), + ) + ), + ) + payment_customization_delete = sgqlc.types.Field( + "PaymentCustomizationDeletePayload", + graphql_name="paymentCustomizationDelete", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + payment_customization_update = sgqlc.types.Field( + "PaymentCustomizationUpdatePayload", + graphql_name="paymentCustomizationUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "payment_customization", + sgqlc.types.Arg(sgqlc.types.non_null(PaymentCustomizationInput), graphql_name="paymentCustomization", default=None), + ), + ) + ), + ) + payment_reminder_send = sgqlc.types.Field( + "PaymentReminderSendPayload", + graphql_name="paymentReminderSend", + args=sgqlc.types.ArgDict( + (("payment_schedule_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="paymentScheduleId", default=None)),) + ), + ) + payment_terms_create = sgqlc.types.Field( + "PaymentTermsCreatePayload", + graphql_name="paymentTermsCreate", + args=sgqlc.types.ArgDict( + ( + ("reference_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="referenceId", default=None)), + ( + "payment_terms_attributes", + sgqlc.types.Arg(sgqlc.types.non_null(PaymentTermsCreateInput), graphql_name="paymentTermsAttributes", default=None), + ), + ) + ), + ) + payment_terms_delete = sgqlc.types.Field( + "PaymentTermsDeletePayload", + graphql_name="paymentTermsDelete", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(PaymentTermsDeleteInput), graphql_name="input", default=None)),) + ), + ) + payment_terms_update = sgqlc.types.Field( + "PaymentTermsUpdatePayload", + graphql_name="paymentTermsUpdate", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(PaymentTermsUpdateInput), graphql_name="input", default=None)),) + ), + ) + price_list_create = sgqlc.types.Field( + "PriceListCreatePayload", + graphql_name="priceListCreate", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(PriceListCreateInput), graphql_name="input", default=None)),) + ), + ) + price_list_delete = sgqlc.types.Field( + "PriceListDeletePayload", + graphql_name="priceListDelete", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + price_list_fixed_prices_add = sgqlc.types.Field( + "PriceListFixedPricesAddPayload", + graphql_name="priceListFixedPricesAdd", + args=sgqlc.types.ArgDict( + ( + ("price_list_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="priceListId", default=None)), + ( + "prices", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(PriceListPriceInput))), + graphql_name="prices", + default=None, + ), + ), + ) + ), + ) + price_list_fixed_prices_by_product_update = sgqlc.types.Field( + "PriceListFixedPricesByProductUpdatePayload", + graphql_name="priceListFixedPricesByProductUpdate", + args=sgqlc.types.ArgDict( + ( + ( + "prices_to_add", + sgqlc.types.Arg( + sgqlc.types.list_of(sgqlc.types.non_null(PriceListProductPriceInput)), graphql_name="pricesToAdd", default=None + ), + ), + ( + "prices_to_delete_by_product_ids", + sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="pricesToDeleteByProductIds", default=None), + ), + ("price_list_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="priceListId", default=None)), + ) + ), + ) + price_list_fixed_prices_delete = sgqlc.types.Field( + "PriceListFixedPricesDeletePayload", + graphql_name="priceListFixedPricesDelete", + args=sgqlc.types.ArgDict( + ( + ("price_list_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="priceListId", default=None)), + ( + "variant_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="variantIds", default=None + ), + ), + ) + ), + ) + price_list_fixed_prices_update = sgqlc.types.Field( + "PriceListFixedPricesUpdatePayload", + graphql_name="priceListFixedPricesUpdate", + args=sgqlc.types.ArgDict( + ( + ("price_list_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="priceListId", default=None)), + ( + "prices_to_add", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(PriceListPriceInput))), + graphql_name="pricesToAdd", + default=None, + ), + ), + ( + "variant_ids_to_delete", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="variantIdsToDelete", default=None + ), + ), + ) + ), + ) + price_list_update = sgqlc.types.Field( + "PriceListUpdatePayload", + graphql_name="priceListUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("input", sgqlc.types.Arg(sgqlc.types.non_null(PriceListUpdateInput), graphql_name="input", default=None)), + ) + ), + ) + product_change_status = sgqlc.types.Field( + "ProductChangeStatusPayload", + graphql_name="productChangeStatus", + args=sgqlc.types.ArgDict( + ( + ("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)), + ("status", sgqlc.types.Arg(sgqlc.types.non_null(ProductStatus), graphql_name="status", default=None)), + ) + ), + ) + product_create = sgqlc.types.Field( + "ProductCreatePayload", + graphql_name="productCreate", + args=sgqlc.types.ArgDict( + ( + ("input", sgqlc.types.Arg(sgqlc.types.non_null(ProductInput), graphql_name="input", default=None)), + ("media", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(CreateMediaInput)), graphql_name="media", default=None)), + ) + ), + ) + product_create_media = sgqlc.types.Field( + "ProductCreateMediaPayload", + graphql_name="productCreateMedia", + args=sgqlc.types.ArgDict( + ( + ("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)), + ( + "media", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CreateMediaInput))), + graphql_name="media", + default=None, + ), + ), + ) + ), + ) + product_delete = sgqlc.types.Field( + "ProductDeletePayload", + graphql_name="productDelete", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(ProductDeleteInput), graphql_name="input", default=None)),) + ), + ) + product_delete_async = sgqlc.types.Field( + "ProductDeleteAsyncPayload", + graphql_name="productDeleteAsync", + args=sgqlc.types.ArgDict((("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)),)), + ) + product_delete_media = sgqlc.types.Field( + "ProductDeleteMediaPayload", + graphql_name="productDeleteMedia", + args=sgqlc.types.ArgDict( + ( + ("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)), + ( + "media_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="mediaIds", default=None + ), + ), + ) + ), + ) + product_duplicate = sgqlc.types.Field( + "ProductDuplicatePayload", + graphql_name="productDuplicate", + args=sgqlc.types.ArgDict( + ( + ("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)), + ("new_title", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="newTitle", default=None)), + ("new_status", sgqlc.types.Arg(ProductStatus, graphql_name="newStatus", default=None)), + ("include_images", sgqlc.types.Arg(Boolean, graphql_name="includeImages", default=False)), + ) + ), + ) + product_duplicate_async_v2 = sgqlc.types.Field( + "ProductDuplicateAsyncV2Payload", + graphql_name="productDuplicateAsyncV2", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(ProductDuplicateAsyncInput), graphql_name="input", default=None)),) + ), + ) + product_feed_create = sgqlc.types.Field( + "ProductFeedCreatePayload", + graphql_name="productFeedCreate", + args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(ProductFeedInput, graphql_name="input", default=None)),)), + ) + product_feed_delete = sgqlc.types.Field( + "ProductFeedDeletePayload", + graphql_name="productFeedDelete", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + product_full_sync = sgqlc.types.Field( + "ProductFullSyncPayload", + graphql_name="productFullSync", + args=sgqlc.types.ArgDict( + ( + ("before_updated_at", sgqlc.types.Arg(DateTime, graphql_name="beforeUpdatedAt", default=None)), + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("updated_at_since", sgqlc.types.Arg(DateTime, graphql_name="updatedAtSince", default=None)), + ) + ), + ) + product_join_selling_plan_groups = sgqlc.types.Field( + "ProductJoinSellingPlanGroupsPayload", + graphql_name="productJoinSellingPlanGroups", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "selling_plan_group_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), + graphql_name="sellingPlanGroupIds", + default=None, + ), + ), + ) + ), + ) + product_leave_selling_plan_groups = sgqlc.types.Field( + "ProductLeaveSellingPlanGroupsPayload", + graphql_name="productLeaveSellingPlanGroups", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "selling_plan_group_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), + graphql_name="sellingPlanGroupIds", + default=None, + ), + ), + ) + ), + ) + product_reorder_media = sgqlc.types.Field( + "ProductReorderMediaPayload", + graphql_name="productReorderMedia", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "moves", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MoveInput))), graphql_name="moves", default=None + ), + ), + ) + ), + ) + product_update = sgqlc.types.Field( + "ProductUpdatePayload", + graphql_name="productUpdate", + args=sgqlc.types.ArgDict( + ( + ("input", sgqlc.types.Arg(sgqlc.types.non_null(ProductInput), graphql_name="input", default=None)), + ("media", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(CreateMediaInput)), graphql_name="media", default=None)), + ) + ), + ) + product_update_media = sgqlc.types.Field( + "ProductUpdateMediaPayload", + graphql_name="productUpdateMedia", + args=sgqlc.types.ArgDict( + ( + ("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)), + ( + "media", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(UpdateMediaInput))), + graphql_name="media", + default=None, + ), + ), + ) + ), + ) + product_variant_append_media = sgqlc.types.Field( + "ProductVariantAppendMediaPayload", + graphql_name="productVariantAppendMedia", + args=sgqlc.types.ArgDict( + ( + ("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)), + ( + "variant_media", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProductVariantAppendMediaInput))), + graphql_name="variantMedia", + default=None, + ), + ), + ) + ), + ) + product_variant_create = sgqlc.types.Field( + "ProductVariantCreatePayload", + graphql_name="productVariantCreate", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(ProductVariantInput), graphql_name="input", default=None)),) + ), + ) + product_variant_delete = sgqlc.types.Field( + "ProductVariantDeletePayload", + graphql_name="productVariantDelete", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + product_variant_detach_media = sgqlc.types.Field( + "ProductVariantDetachMediaPayload", + graphql_name="productVariantDetachMedia", + args=sgqlc.types.ArgDict( + ( + ("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)), + ( + "variant_media", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProductVariantDetachMediaInput))), + graphql_name="variantMedia", + default=None, + ), + ), + ) + ), + ) + product_variant_join_selling_plan_groups = sgqlc.types.Field( + "ProductVariantJoinSellingPlanGroupsPayload", + graphql_name="productVariantJoinSellingPlanGroups", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "selling_plan_group_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), + graphql_name="sellingPlanGroupIds", + default=None, + ), + ), + ) + ), + ) + product_variant_leave_selling_plan_groups = sgqlc.types.Field( + "ProductVariantLeaveSellingPlanGroupsPayload", + graphql_name="productVariantLeaveSellingPlanGroups", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "selling_plan_group_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), + graphql_name="sellingPlanGroupIds", + default=None, + ), + ), + ) + ), + ) + product_variant_relationship_bulk_update = sgqlc.types.Field( + "ProductVariantRelationshipBulkUpdatePayload", + graphql_name="productVariantRelationshipBulkUpdate", + args=sgqlc.types.ArgDict( + ( + ( + "input", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProductVariantRelationshipUpdateInput))), + graphql_name="input", + default=None, + ), + ), + ) + ), + ) + product_variant_update = sgqlc.types.Field( + "ProductVariantUpdatePayload", + graphql_name="productVariantUpdate", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(ProductVariantInput), graphql_name="input", default=None)),) + ), + ) + product_variants_bulk_create = sgqlc.types.Field( + "ProductVariantsBulkCreatePayload", + graphql_name="productVariantsBulkCreate", + args=sgqlc.types.ArgDict( + ( + ( + "variants", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProductVariantsBulkInput))), + graphql_name="variants", + default=None, + ), + ), + ("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)), + ("media", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(CreateMediaInput)), graphql_name="media", default=None)), + ) + ), + ) + product_variants_bulk_delete = sgqlc.types.Field( + "ProductVariantsBulkDeletePayload", + graphql_name="productVariantsBulkDelete", + args=sgqlc.types.ArgDict( + ( + ( + "variants_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="variantsIds", default=None + ), + ), + ("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)), + ) + ), + ) + product_variants_bulk_reorder = sgqlc.types.Field( + "ProductVariantsBulkReorderPayload", + graphql_name="productVariantsBulkReorder", + args=sgqlc.types.ArgDict( + ( + ("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)), + ( + "positions", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProductVariantPositionInput))), + graphql_name="positions", + default=None, + ), + ), + ) + ), + ) + product_variants_bulk_update = sgqlc.types.Field( + "ProductVariantsBulkUpdatePayload", + graphql_name="productVariantsBulkUpdate", + args=sgqlc.types.ArgDict( + ( + ( + "variants", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProductVariantsBulkInput))), + graphql_name="variants", + default=None, + ), + ), + ("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)), + ("media", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(CreateMediaInput)), graphql_name="media", default=None)), + ("allow_partial_updates", sgqlc.types.Arg(Boolean, graphql_name="allowPartialUpdates", default=False)), + ) + ), + ) + pub_sub_server_pixel_update = sgqlc.types.Field( + "PubSubServerPixelUpdatePayload", + graphql_name="pubSubServerPixelUpdate", + args=sgqlc.types.ArgDict( + ( + ("pub_sub_project", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="pubSubProject", default=None)), + ("pub_sub_topic", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="pubSubTopic", default=None)), + ) + ), + ) + pub_sub_webhook_subscription_create = sgqlc.types.Field( + "PubSubWebhookSubscriptionCreatePayload", + graphql_name="pubSubWebhookSubscriptionCreate", + args=sgqlc.types.ArgDict( + ( + ("topic", sgqlc.types.Arg(sgqlc.types.non_null(WebhookSubscriptionTopic), graphql_name="topic", default=None)), + ( + "webhook_subscription", + sgqlc.types.Arg(sgqlc.types.non_null(PubSubWebhookSubscriptionInput), graphql_name="webhookSubscription", default=None), + ), + ) + ), + ) + pub_sub_webhook_subscription_update = sgqlc.types.Field( + "PubSubWebhookSubscriptionUpdatePayload", + graphql_name="pubSubWebhookSubscriptionUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("webhook_subscription", sgqlc.types.Arg(PubSubWebhookSubscriptionInput, graphql_name="webhookSubscription", default=None)), + ) + ), + ) + publication_create = sgqlc.types.Field( + "PublicationCreatePayload", + graphql_name="publicationCreate", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(PublicationCreateInput), graphql_name="input", default=None)),) + ), + ) + publication_delete = sgqlc.types.Field( + "PublicationDeletePayload", + graphql_name="publicationDelete", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + publication_update = sgqlc.types.Field( + "PublicationUpdatePayload", + graphql_name="publicationUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("input", sgqlc.types.Arg(sgqlc.types.non_null(PublicationUpdateInput), graphql_name="input", default=None)), + ) + ), + ) + publishable_publish = sgqlc.types.Field( + "PublishablePublishPayload", + graphql_name="publishablePublish", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "input", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(PublicationInput))), + graphql_name="input", + default=None, + ), + ), + ) + ), + ) + publishable_publish_to_current_channel = sgqlc.types.Field( + "PublishablePublishToCurrentChannelPayload", + graphql_name="publishablePublishToCurrentChannel", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + publishable_unpublish = sgqlc.types.Field( + "PublishableUnpublishPayload", + graphql_name="publishableUnpublish", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "input", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(PublicationInput))), + graphql_name="input", + default=None, + ), + ), + ) + ), + ) + publishable_unpublish_to_current_channel = sgqlc.types.Field( + "PublishableUnpublishToCurrentChannelPayload", + graphql_name="publishableUnpublishToCurrentChannel", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + quantity_rules_add = sgqlc.types.Field( + "QuantityRulesAddPayload", + graphql_name="quantityRulesAdd", + args=sgqlc.types.ArgDict( + ( + ("price_list_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="priceListId", default=None)), + ( + "quantity_rules", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(QuantityRuleInput))), + graphql_name="quantityRules", + default=None, + ), + ), + ) + ), + ) + quantity_rules_delete = sgqlc.types.Field( + "QuantityRulesDeletePayload", + graphql_name="quantityRulesDelete", + args=sgqlc.types.ArgDict( + ( + ("price_list_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="priceListId", default=None)), + ( + "variant_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="variantIds", default=None + ), + ), + ) + ), + ) + refund_create = sgqlc.types.Field( + "RefundCreatePayload", + graphql_name="refundCreate", + args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(RefundInput), graphql_name="input", default=None)),)), + ) + return_approve_request = sgqlc.types.Field( + "ReturnApproveRequestPayload", + graphql_name="returnApproveRequest", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(ReturnApproveRequestInput), graphql_name="input", default=None)),) + ), + ) + return_cancel = sgqlc.types.Field( + "ReturnCancelPayload", + graphql_name="returnCancel", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("notify_customer", sgqlc.types.Arg(Boolean, graphql_name="notifyCustomer", default=False)), + ) + ), + ) + return_close = sgqlc.types.Field( + "ReturnClosePayload", + graphql_name="returnClose", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + return_create = sgqlc.types.Field( + "ReturnCreatePayload", + graphql_name="returnCreate", + args=sgqlc.types.ArgDict( + (("return_input", sgqlc.types.Arg(sgqlc.types.non_null(ReturnInput), graphql_name="returnInput", default=None)),) + ), + ) + return_decline_request = sgqlc.types.Field( + "ReturnDeclineRequestPayload", + graphql_name="returnDeclineRequest", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(ReturnDeclineRequestInput), graphql_name="input", default=None)),) + ), + ) + return_refund = sgqlc.types.Field( + "ReturnRefundPayload", + graphql_name="returnRefund", + args=sgqlc.types.ArgDict( + ( + ( + "return_refund_input", + sgqlc.types.Arg(sgqlc.types.non_null(ReturnRefundInput), graphql_name="returnRefundInput", default=None), + ), + ) + ), + ) + return_reopen = sgqlc.types.Field( + "ReturnReopenPayload", + graphql_name="returnReopen", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + return_request = sgqlc.types.Field( + "ReturnRequestPayload", + graphql_name="returnRequest", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(ReturnRequestInput), graphql_name="input", default=None)),) + ), + ) + reverse_delivery_create_with_shipping = sgqlc.types.Field( + "ReverseDeliveryCreateWithShippingPayload", + graphql_name="reverseDeliveryCreateWithShipping", + args=sgqlc.types.ArgDict( + ( + ( + "reverse_fulfillment_order_id", + sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="reverseFulfillmentOrderId", default=None), + ), + ( + "reverse_delivery_line_items", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ReverseDeliveryLineItemInput))), + graphql_name="reverseDeliveryLineItems", + default=None, + ), + ), + ("tracking_input", sgqlc.types.Arg(ReverseDeliveryTrackingInput, graphql_name="trackingInput", default=None)), + ("label_input", sgqlc.types.Arg(ReverseDeliveryLabelInput, graphql_name="labelInput", default=None)), + ("notify_customer", sgqlc.types.Arg(Boolean, graphql_name="notifyCustomer", default=True)), + ) + ), + ) + reverse_delivery_shipping_update = sgqlc.types.Field( + "ReverseDeliveryShippingUpdatePayload", + graphql_name="reverseDeliveryShippingUpdate", + args=sgqlc.types.ArgDict( + ( + ("reverse_delivery_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="reverseDeliveryId", default=None)), + ("tracking_input", sgqlc.types.Arg(ReverseDeliveryTrackingInput, graphql_name="trackingInput", default=None)), + ("label_input", sgqlc.types.Arg(ReverseDeliveryLabelInput, graphql_name="labelInput", default=None)), + ("notify_customer", sgqlc.types.Arg(Boolean, graphql_name="notifyCustomer", default=True)), + ) + ), + ) + reverse_fulfillment_order_dispose = sgqlc.types.Field( + "ReverseFulfillmentOrderDisposePayload", + graphql_name="reverseFulfillmentOrderDispose", + args=sgqlc.types.ArgDict( + ( + ( + "disposition_inputs", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ReverseFulfillmentOrderDisposeInput))), + graphql_name="dispositionInputs", + default=None, + ), + ), + ) + ), + ) + saved_search_create = sgqlc.types.Field( + "SavedSearchCreatePayload", + graphql_name="savedSearchCreate", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(SavedSearchCreateInput), graphql_name="input", default=None)),) + ), + ) + saved_search_delete = sgqlc.types.Field( + "SavedSearchDeletePayload", + graphql_name="savedSearchDelete", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(SavedSearchDeleteInput), graphql_name="input", default=None)),) + ), + ) + saved_search_update = sgqlc.types.Field( + "SavedSearchUpdatePayload", + graphql_name="savedSearchUpdate", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(SavedSearchUpdateInput), graphql_name="input", default=None)),) + ), + ) + script_tag_create = sgqlc.types.Field( + "ScriptTagCreatePayload", + graphql_name="scriptTagCreate", + args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(ScriptTagInput), graphql_name="input", default=None)),)), + ) + script_tag_delete = sgqlc.types.Field( + "ScriptTagDeletePayload", + graphql_name="scriptTagDelete", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + script_tag_update = sgqlc.types.Field( + "ScriptTagUpdatePayload", + graphql_name="scriptTagUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("input", sgqlc.types.Arg(sgqlc.types.non_null(ScriptTagInput), graphql_name="input", default=None)), + ) + ), + ) + segment_create = sgqlc.types.Field( + "SegmentCreatePayload", + graphql_name="segmentCreate", + args=sgqlc.types.ArgDict( + ( + ("name", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="name", default=None)), + ("query", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="query", default=None)), + ) + ), + ) + segment_delete = sgqlc.types.Field( + "SegmentDeletePayload", + graphql_name="segmentDelete", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + segment_update = sgqlc.types.Field( + "SegmentUpdatePayload", + graphql_name="segmentUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("name", sgqlc.types.Arg(String, graphql_name="name", default=None)), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + selling_plan_group_add_product_variants = sgqlc.types.Field( + "SellingPlanGroupAddProductVariantsPayload", + graphql_name="sellingPlanGroupAddProductVariants", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "product_variant_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="productVariantIds", default=None + ), + ), + ) + ), + ) + selling_plan_group_add_products = sgqlc.types.Field( + "SellingPlanGroupAddProductsPayload", + graphql_name="sellingPlanGroupAddProducts", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "product_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="productIds", default=None + ), + ), + ) + ), + ) + selling_plan_group_create = sgqlc.types.Field( + "SellingPlanGroupCreatePayload", + graphql_name="sellingPlanGroupCreate", + args=sgqlc.types.ArgDict( + ( + ("input", sgqlc.types.Arg(sgqlc.types.non_null(SellingPlanGroupInput), graphql_name="input", default=None)), + ("resources", sgqlc.types.Arg(SellingPlanGroupResourceInput, graphql_name="resources", default=None)), + ) + ), + ) + selling_plan_group_delete = sgqlc.types.Field( + "SellingPlanGroupDeletePayload", + graphql_name="sellingPlanGroupDelete", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + selling_plan_group_remove_product_variants = sgqlc.types.Field( + "SellingPlanGroupRemoveProductVariantsPayload", + graphql_name="sellingPlanGroupRemoveProductVariants", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "product_variant_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="productVariantIds", default=None + ), + ), + ) + ), + ) + selling_plan_group_remove_products = sgqlc.types.Field( + "SellingPlanGroupRemoveProductsPayload", + graphql_name="sellingPlanGroupRemoveProducts", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "product_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="productIds", default=None + ), + ), + ) + ), + ) + selling_plan_group_update = sgqlc.types.Field( + "SellingPlanGroupUpdatePayload", + graphql_name="sellingPlanGroupUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("input", sgqlc.types.Arg(sgqlc.types.non_null(SellingPlanGroupInput), graphql_name="input", default=None)), + ) + ), + ) + server_pixel_create = sgqlc.types.Field("ServerPixelCreatePayload", graphql_name="serverPixelCreate") + server_pixel_delete = sgqlc.types.Field("ServerPixelDeletePayload", graphql_name="serverPixelDelete") + shipping_package_delete = sgqlc.types.Field( + "ShippingPackageDeletePayload", + graphql_name="shippingPackageDelete", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + shipping_package_make_default = sgqlc.types.Field( + "ShippingPackageMakeDefaultPayload", + graphql_name="shippingPackageMakeDefault", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + shipping_package_update = sgqlc.types.Field( + "ShippingPackageUpdatePayload", + graphql_name="shippingPackageUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "shipping_package", + sgqlc.types.Arg(sgqlc.types.non_null(CustomShippingPackageInput), graphql_name="shippingPackage", default=None), + ), + ) + ), + ) + shop_locale_disable = sgqlc.types.Field( + "ShopLocaleDisablePayload", + graphql_name="shopLocaleDisable", + args=sgqlc.types.ArgDict((("locale", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="locale", default=None)),)), + ) + shop_locale_enable = sgqlc.types.Field( + "ShopLocaleEnablePayload", + graphql_name="shopLocaleEnable", + args=sgqlc.types.ArgDict( + ( + ("locale", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="locale", default=None)), + ( + "market_web_presence_ids", + sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="marketWebPresenceIds", default=None), + ), + ) + ), + ) + shop_locale_update = sgqlc.types.Field( + "ShopLocaleUpdatePayload", + graphql_name="shopLocaleUpdate", + args=sgqlc.types.ArgDict( + ( + ("locale", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="locale", default=None)), + ("shop_locale", sgqlc.types.Arg(sgqlc.types.non_null(ShopLocaleInput), graphql_name="shopLocale", default=None)), + ) + ), + ) + shop_policy_update = sgqlc.types.Field( + "ShopPolicyUpdatePayload", + graphql_name="shopPolicyUpdate", + args=sgqlc.types.ArgDict( + (("shop_policy", sgqlc.types.Arg(sgqlc.types.non_null(ShopPolicyInput), graphql_name="shopPolicy", default=None)),) + ), + ) + shop_resource_feedback_create = sgqlc.types.Field( + "ShopResourceFeedbackCreatePayload", + graphql_name="shopResourceFeedbackCreate", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(ResourceFeedbackCreateInput), graphql_name="input", default=None)),) + ), + ) + staged_uploads_create = sgqlc.types.Field( + "StagedUploadsCreatePayload", + graphql_name="stagedUploadsCreate", + args=sgqlc.types.ArgDict( + ( + ( + "input", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(StagedUploadInput))), + graphql_name="input", + default=None, + ), + ), + ) + ), + ) + standard_metafield_definition_enable = sgqlc.types.Field( + "StandardMetafieldDefinitionEnablePayload", + graphql_name="standardMetafieldDefinitionEnable", + args=sgqlc.types.ArgDict( + ( + ("owner_type", sgqlc.types.Arg(sgqlc.types.non_null(MetafieldOwnerType), graphql_name="ownerType", default=None)), + ("id", sgqlc.types.Arg(ID, graphql_name="id", default=None)), + ("namespace", sgqlc.types.Arg(String, graphql_name="namespace", default=None)), + ("key", sgqlc.types.Arg(String, graphql_name="key", default=None)), + ("pin", sgqlc.types.Arg(sgqlc.types.non_null(Boolean), graphql_name="pin", default=False)), + ("visible_to_storefront_api", sgqlc.types.Arg(Boolean, graphql_name="visibleToStorefrontApi", default=None)), + ("use_as_collection_condition", sgqlc.types.Arg(Boolean, graphql_name="useAsCollectionCondition", default=False)), + ) + ), + ) + standard_metaobject_definition_enable = sgqlc.types.Field( + "StandardMetaobjectDefinitionEnablePayload", + graphql_name="standardMetaobjectDefinitionEnable", + args=sgqlc.types.ArgDict((("type", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="type", default=None)),)), + ) + storefront_access_token_create = sgqlc.types.Field( + "StorefrontAccessTokenCreatePayload", + graphql_name="storefrontAccessTokenCreate", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(StorefrontAccessTokenInput), graphql_name="input", default=None)),) + ), + ) + storefront_access_token_delete = sgqlc.types.Field( + "StorefrontAccessTokenDeletePayload", + graphql_name="storefrontAccessTokenDelete", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(StorefrontAccessTokenDeleteInput), graphql_name="input", default=None)),) + ), + ) + subscription_billing_attempt_create = sgqlc.types.Field( + "SubscriptionBillingAttemptCreatePayload", + graphql_name="subscriptionBillingAttemptCreate", + args=sgqlc.types.ArgDict( + ( + ( + "subscription_contract_id", + sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="subscriptionContractId", default=None), + ), + ( + "subscription_billing_attempt_input", + sgqlc.types.Arg( + sgqlc.types.non_null(SubscriptionBillingAttemptInput), graphql_name="subscriptionBillingAttemptInput", default=None + ), + ), + ) + ), + ) + subscription_billing_cycle_contract_draft_commit = sgqlc.types.Field( + "SubscriptionBillingCycleContractDraftCommitPayload", + graphql_name="subscriptionBillingCycleContractDraftCommit", + args=sgqlc.types.ArgDict((("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)),)), + ) + subscription_billing_cycle_contract_draft_concatenate = sgqlc.types.Field( + "SubscriptionBillingCycleContractDraftConcatenatePayload", + graphql_name="subscriptionBillingCycleContractDraftConcatenate", + args=sgqlc.types.ArgDict( + ( + ("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)), + ( + "concatenated_billing_cycle_contracts", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SubscriptionBillingCycleInput))), + graphql_name="concatenatedBillingCycleContracts", + default=None, + ), + ), + ) + ), + ) + subscription_billing_cycle_contract_edit = sgqlc.types.Field( + "SubscriptionBillingCycleContractEditPayload", + graphql_name="subscriptionBillingCycleContractEdit", + args=sgqlc.types.ArgDict( + ( + ( + "billing_cycle_input", + sgqlc.types.Arg(sgqlc.types.non_null(SubscriptionBillingCycleInput), graphql_name="billingCycleInput", default=None), + ), + ) + ), + ) + subscription_billing_cycle_edit_delete = sgqlc.types.Field( + "SubscriptionBillingCycleEditDeletePayload", + graphql_name="subscriptionBillingCycleEditDelete", + args=sgqlc.types.ArgDict( + ( + ( + "billing_cycle_input", + sgqlc.types.Arg(sgqlc.types.non_null(SubscriptionBillingCycleInput), graphql_name="billingCycleInput", default=None), + ), + ) + ), + ) + subscription_billing_cycle_edits_delete = sgqlc.types.Field( + "SubscriptionBillingCycleEditsDeletePayload", + graphql_name="subscriptionBillingCycleEditsDelete", + args=sgqlc.types.ArgDict( + ( + ("contract_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="contractId", default=None)), + ( + "target_selection", + sgqlc.types.Arg( + sgqlc.types.non_null(SubscriptionBillingCyclesTargetSelection), graphql_name="targetSelection", default=None + ), + ), + ) + ), + ) + subscription_billing_cycle_schedule_edit = sgqlc.types.Field( + "SubscriptionBillingCycleScheduleEditPayload", + graphql_name="subscriptionBillingCycleScheduleEdit", + args=sgqlc.types.ArgDict( + ( + ( + "billing_cycle_input", + sgqlc.types.Arg(sgqlc.types.non_null(SubscriptionBillingCycleInput), graphql_name="billingCycleInput", default=None), + ), + ( + "input", + sgqlc.types.Arg(sgqlc.types.non_null(SubscriptionBillingCycleScheduleEditInput), graphql_name="input", default=None), + ), + ) + ), + ) + subscription_contract_atomic_create = sgqlc.types.Field( + "SubscriptionContractAtomicCreatePayload", + graphql_name="subscriptionContractAtomicCreate", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(SubscriptionContractAtomicCreateInput), graphql_name="input", default=None)),) + ), + ) + subscription_contract_create = sgqlc.types.Field( + "SubscriptionContractCreatePayload", + graphql_name="subscriptionContractCreate", + args=sgqlc.types.ArgDict( + (("input", sgqlc.types.Arg(sgqlc.types.non_null(SubscriptionContractCreateInput), graphql_name="input", default=None)),) + ), + ) + subscription_contract_product_change = sgqlc.types.Field( + "SubscriptionContractProductChangePayload", + graphql_name="subscriptionContractProductChange", + args=sgqlc.types.ArgDict( + ( + ( + "subscription_contract_id", + sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="subscriptionContractId", default=None), + ), + ("line_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="lineId", default=None)), + ( + "input", + sgqlc.types.Arg(sgqlc.types.non_null(SubscriptionContractProductChangeInput), graphql_name="input", default=None), + ), + ) + ), + ) + subscription_contract_set_next_billing_date = sgqlc.types.Field( + "SubscriptionContractSetNextBillingDatePayload", + graphql_name="subscriptionContractSetNextBillingDate", + args=sgqlc.types.ArgDict( + ( + ("contract_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="contractId", default=None)), + ("date", sgqlc.types.Arg(sgqlc.types.non_null(DateTime), graphql_name="date", default=None)), + ) + ), + ) + subscription_contract_update = sgqlc.types.Field( + "SubscriptionContractUpdatePayload", + graphql_name="subscriptionContractUpdate", + args=sgqlc.types.ArgDict((("contract_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="contractId", default=None)),)), + ) + subscription_draft_commit = sgqlc.types.Field( + "SubscriptionDraftCommitPayload", + graphql_name="subscriptionDraftCommit", + args=sgqlc.types.ArgDict((("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)),)), + ) + subscription_draft_discount_add = sgqlc.types.Field( + "SubscriptionDraftDiscountAddPayload", + graphql_name="subscriptionDraftDiscountAdd", + args=sgqlc.types.ArgDict( + ( + ("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)), + ("input", sgqlc.types.Arg(sgqlc.types.non_null(SubscriptionManualDiscountInput), graphql_name="input", default=None)), + ) + ), + ) + subscription_draft_discount_code_apply = sgqlc.types.Field( + "SubscriptionDraftDiscountCodeApplyPayload", + graphql_name="subscriptionDraftDiscountCodeApply", + args=sgqlc.types.ArgDict( + ( + ("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)), + ("redeem_code", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="redeemCode", default=None)), + ) + ), + ) + subscription_draft_discount_remove = sgqlc.types.Field( + "SubscriptionDraftDiscountRemovePayload", + graphql_name="subscriptionDraftDiscountRemove", + args=sgqlc.types.ArgDict( + ( + ("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)), + ("discount_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="discountId", default=None)), + ) + ), + ) + subscription_draft_discount_update = sgqlc.types.Field( + "SubscriptionDraftDiscountUpdatePayload", + graphql_name="subscriptionDraftDiscountUpdate", + args=sgqlc.types.ArgDict( + ( + ("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)), + ("discount_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="discountId", default=None)), + ("input", sgqlc.types.Arg(sgqlc.types.non_null(SubscriptionManualDiscountInput), graphql_name="input", default=None)), + ) + ), + ) + subscription_draft_free_shipping_discount_add = sgqlc.types.Field( + "SubscriptionDraftFreeShippingDiscountAddPayload", + graphql_name="subscriptionDraftFreeShippingDiscountAdd", + args=sgqlc.types.ArgDict( + ( + ("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)), + ("input", sgqlc.types.Arg(sgqlc.types.non_null(SubscriptionFreeShippingDiscountInput), graphql_name="input", default=None)), + ) + ), + ) + subscription_draft_free_shipping_discount_update = sgqlc.types.Field( + "SubscriptionDraftFreeShippingDiscountUpdatePayload", + graphql_name="subscriptionDraftFreeShippingDiscountUpdate", + args=sgqlc.types.ArgDict( + ( + ("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)), + ("discount_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="discountId", default=None)), + ("input", sgqlc.types.Arg(sgqlc.types.non_null(SubscriptionFreeShippingDiscountInput), graphql_name="input", default=None)), + ) + ), + ) + subscription_draft_line_add = sgqlc.types.Field( + "SubscriptionDraftLineAddPayload", + graphql_name="subscriptionDraftLineAdd", + args=sgqlc.types.ArgDict( + ( + ("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)), + ("input", sgqlc.types.Arg(sgqlc.types.non_null(SubscriptionLineInput), graphql_name="input", default=None)), + ) + ), + ) + subscription_draft_line_remove = sgqlc.types.Field( + "SubscriptionDraftLineRemovePayload", + graphql_name="subscriptionDraftLineRemove", + args=sgqlc.types.ArgDict( + ( + ("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)), + ("line_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="lineId", default=None)), + ) + ), + ) + subscription_draft_line_update = sgqlc.types.Field( + "SubscriptionDraftLineUpdatePayload", + graphql_name="subscriptionDraftLineUpdate", + args=sgqlc.types.ArgDict( + ( + ("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)), + ("line_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="lineId", default=None)), + ("input", sgqlc.types.Arg(sgqlc.types.non_null(SubscriptionLineUpdateInput), graphql_name="input", default=None)), + ) + ), + ) + subscription_draft_update = sgqlc.types.Field( + "SubscriptionDraftUpdatePayload", + graphql_name="subscriptionDraftUpdate", + args=sgqlc.types.ArgDict( + ( + ("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)), + ("input", sgqlc.types.Arg(sgqlc.types.non_null(SubscriptionDraftInput), graphql_name="input", default=None)), + ) + ), + ) + tags_add = sgqlc.types.Field( + "TagsAddPayload", + graphql_name="tagsAdd", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "tags", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="tags", default=None + ), + ), + ) + ), + ) + tags_remove = sgqlc.types.Field( + "TagsRemovePayload", + graphql_name="tagsRemove", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "tags", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="tags", default=None + ), + ), + ) + ), + ) + tax_app_configure = sgqlc.types.Field( + "TaxAppConfigurePayload", + graphql_name="taxAppConfigure", + args=sgqlc.types.ArgDict((("ready", sgqlc.types.Arg(sgqlc.types.non_null(Boolean), graphql_name="ready", default=None)),)), + ) + translations_register = sgqlc.types.Field( + "TranslationsRegisterPayload", + graphql_name="translationsRegister", + args=sgqlc.types.ArgDict( + ( + ("resource_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="resourceId", default=None)), + ( + "translations", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TranslationInput))), + graphql_name="translations", + default=None, + ), + ), + ) + ), + ) + translations_remove = sgqlc.types.Field( + "TranslationsRemovePayload", + graphql_name="translationsRemove", + args=sgqlc.types.ArgDict( + ( + ("resource_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="resourceId", default=None)), + ( + "translation_keys", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), + graphql_name="translationKeys", + default=None, + ), + ), + ( + "locales", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="locales", default=None + ), + ), + ("market_ids", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="marketIds", default=None)), + ) + ), + ) + url_redirect_bulk_delete_all = sgqlc.types.Field("UrlRedirectBulkDeleteAllPayload", graphql_name="urlRedirectBulkDeleteAll") + url_redirect_bulk_delete_by_ids = sgqlc.types.Field( + "UrlRedirectBulkDeleteByIdsPayload", + graphql_name="urlRedirectBulkDeleteByIds", + args=sgqlc.types.ArgDict( + ( + ( + "ids", + sgqlc.types.Arg(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="ids", default=None), + ), + ) + ), + ) + url_redirect_bulk_delete_by_saved_search = sgqlc.types.Field( + "UrlRedirectBulkDeleteBySavedSearchPayload", + graphql_name="urlRedirectBulkDeleteBySavedSearch", + args=sgqlc.types.ArgDict( + (("saved_search_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="savedSearchId", default=None)),) + ), + ) + url_redirect_bulk_delete_by_search = sgqlc.types.Field( + "UrlRedirectBulkDeleteBySearchPayload", + graphql_name="urlRedirectBulkDeleteBySearch", + args=sgqlc.types.ArgDict((("search", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="search", default=None)),)), + ) + url_redirect_create = sgqlc.types.Field( + "UrlRedirectCreatePayload", + graphql_name="urlRedirectCreate", + args=sgqlc.types.ArgDict( + (("url_redirect", sgqlc.types.Arg(sgqlc.types.non_null(UrlRedirectInput), graphql_name="urlRedirect", default=None)),) + ), + ) + url_redirect_delete = sgqlc.types.Field( + "UrlRedirectDeletePayload", + graphql_name="urlRedirectDelete", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + url_redirect_import_create = sgqlc.types.Field( + "UrlRedirectImportCreatePayload", + graphql_name="urlRedirectImportCreate", + args=sgqlc.types.ArgDict((("url", sgqlc.types.Arg(sgqlc.types.non_null(URL), graphql_name="url", default=None)),)), + ) + url_redirect_import_submit = sgqlc.types.Field( + "UrlRedirectImportSubmitPayload", + graphql_name="urlRedirectImportSubmit", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + url_redirect_update = sgqlc.types.Field( + "UrlRedirectUpdatePayload", + graphql_name="urlRedirectUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("url_redirect", sgqlc.types.Arg(sgqlc.types.non_null(UrlRedirectInput), graphql_name="urlRedirect", default=None)), + ) + ), + ) + web_pixel_create = sgqlc.types.Field( + "WebPixelCreatePayload", + graphql_name="webPixelCreate", + args=sgqlc.types.ArgDict( + (("web_pixel", sgqlc.types.Arg(sgqlc.types.non_null(WebPixelInput), graphql_name="webPixel", default=None)),) + ), + ) + web_pixel_delete = sgqlc.types.Field( + "WebPixelDeletePayload", + graphql_name="webPixelDelete", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + web_pixel_update = sgqlc.types.Field( + "WebPixelUpdatePayload", + graphql_name="webPixelUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("web_pixel", sgqlc.types.Arg(sgqlc.types.non_null(WebPixelInput), graphql_name="webPixel", default=None)), + ) + ), + ) + webhook_subscription_create = sgqlc.types.Field( + "WebhookSubscriptionCreatePayload", + graphql_name="webhookSubscriptionCreate", + args=sgqlc.types.ArgDict( + ( + ("topic", sgqlc.types.Arg(sgqlc.types.non_null(WebhookSubscriptionTopic), graphql_name="topic", default=None)), + ( + "webhook_subscription", + sgqlc.types.Arg(sgqlc.types.non_null(WebhookSubscriptionInput), graphql_name="webhookSubscription", default=None), + ), + ) + ), + ) + webhook_subscription_delete = sgqlc.types.Field( + "WebhookSubscriptionDeletePayload", + graphql_name="webhookSubscriptionDelete", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + webhook_subscription_update = sgqlc.types.Field( + "WebhookSubscriptionUpdatePayload", + graphql_name="webhookSubscriptionUpdate", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ( + "webhook_subscription", + sgqlc.types.Arg(sgqlc.types.non_null(WebhookSubscriptionInput), graphql_name="webhookSubscription", default=None), + ), + ) + ), + ) + + +class MutationsStagedUploadTargetGenerateUploadParameter(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("name", "value") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") + + +class NavigationItem(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("id", "title", "url") + id = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="id") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") + + +class OrderApp(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("icon", "id", "name") + icon = sgqlc.types.Field(sgqlc.types.non_null("Image"), graphql_name="icon") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + + +class OrderCapturePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("transaction", "user_errors") + transaction = sgqlc.types.Field("OrderTransaction", graphql_name="transaction") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class OrderClosePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("order", "user_errors") + order = sgqlc.types.Field("Order", graphql_name="order") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class OrderConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("OrderEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Order"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class OrderCreateMandatePaymentPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("job", "payment_reference_id", "user_errors") + job = sgqlc.types.Field(Job, graphql_name="job") + payment_reference_id = sgqlc.types.Field(String, graphql_name="paymentReferenceId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("OrderCreateMandatePaymentUserError"))), graphql_name="userErrors" + ) + + +class OrderEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("Order"), graphql_name="node") + + +class OrderEditAddCustomItemPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("calculated_line_item", "calculated_order", "user_errors") + calculated_line_item = sgqlc.types.Field(CalculatedLineItem, graphql_name="calculatedLineItem") + calculated_order = sgqlc.types.Field("CalculatedOrder", graphql_name="calculatedOrder") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class OrderEditAddLineItemDiscountPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("added_discount_staged_change", "calculated_line_item", "calculated_order", "user_errors") + added_discount_staged_change = sgqlc.types.Field("OrderStagedChangeAddLineItemDiscount", graphql_name="addedDiscountStagedChange") + calculated_line_item = sgqlc.types.Field(CalculatedLineItem, graphql_name="calculatedLineItem") + calculated_order = sgqlc.types.Field("CalculatedOrder", graphql_name="calculatedOrder") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class OrderEditAddVariantPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("calculated_line_item", "calculated_order", "user_errors") + calculated_line_item = sgqlc.types.Field(CalculatedLineItem, graphql_name="calculatedLineItem") + calculated_order = sgqlc.types.Field("CalculatedOrder", graphql_name="calculatedOrder") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class OrderEditBeginPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("calculated_order", "user_errors") + calculated_order = sgqlc.types.Field("CalculatedOrder", graphql_name="calculatedOrder") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class OrderEditCommitPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("order", "user_errors") + order = sgqlc.types.Field("Order", graphql_name="order") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class OrderEditRemoveLineItemDiscountPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("calculated_line_item", "calculated_order", "user_errors") + calculated_line_item = sgqlc.types.Field(CalculatedLineItem, graphql_name="calculatedLineItem") + calculated_order = sgqlc.types.Field("CalculatedOrder", graphql_name="calculatedOrder") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class OrderEditSetQuantityPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("calculated_line_item", "calculated_order", "user_errors") + calculated_line_item = sgqlc.types.Field(CalculatedLineItem, graphql_name="calculatedLineItem") + calculated_order = sgqlc.types.Field("CalculatedOrder", graphql_name="calculatedOrder") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class OrderInvoiceSendPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("order", "user_errors") + order = sgqlc.types.Field("Order", graphql_name="order") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("OrderInvoiceSendUserError"))), graphql_name="userErrors" + ) + + +class OrderMarkAsPaidPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("order", "user_errors") + order = sgqlc.types.Field("Order", graphql_name="order") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class OrderOpenPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("order", "user_errors") + order = sgqlc.types.Field("Order", graphql_name="order") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class OrderPaymentCollectionDetails(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("additional_payment_collection_url", "vaulted_payment_methods") + additional_payment_collection_url = sgqlc.types.Field(URL, graphql_name="additionalPaymentCollectionUrl") + vaulted_payment_methods = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("PaymentMandate")), graphql_name="vaultedPaymentMethods" + ) + + +class OrderPaymentStatus(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("error_message", "payment_reference_id", "status", "translated_error_message") + error_message = sgqlc.types.Field(String, graphql_name="errorMessage") + payment_reference_id = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="paymentReferenceId") + status = sgqlc.types.Field(sgqlc.types.non_null(OrderPaymentStatusResult), graphql_name="status") + translated_error_message = sgqlc.types.Field(String, graphql_name="translatedErrorMessage") + + +class OrderRisk(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("display", "level", "message") + display = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="display") + level = sgqlc.types.Field(OrderRiskLevel, graphql_name="level") + message = sgqlc.types.Field(String, graphql_name="message") + + +class OrderStagedChangeAddCustomItem(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("original_unit_price", "quantity", "title") + original_unit_price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="originalUnitPrice") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + + +class OrderStagedChangeAddLineItemDiscount(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("description", "id", "value") + description = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="description") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + value = sgqlc.types.Field(sgqlc.types.non_null("PricingValue"), graphql_name="value") + + +class OrderStagedChangeAddShippingLine(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("phone", "presentment_title", "price", "title") + phone = sgqlc.types.Field(String, graphql_name="phone") + presentment_title = sgqlc.types.Field(String, graphql_name="presentmentTitle") + price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="price") + title = sgqlc.types.Field(String, graphql_name="title") + + +class OrderStagedChangeAddVariant(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("quantity", "variant") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + variant = sgqlc.types.Field(sgqlc.types.non_null("ProductVariant"), graphql_name="variant") + + +class OrderStagedChangeConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("OrderStagedChangeEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("OrderStagedChange"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class OrderStagedChangeDecrementItem(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("delta", "line_item", "restock") + delta = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="delta") + line_item = sgqlc.types.Field(sgqlc.types.non_null("LineItem"), graphql_name="lineItem") + restock = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="restock") + + +class OrderStagedChangeEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("OrderStagedChange"), graphql_name="node") + + +class OrderStagedChangeIncrementItem(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("delta", "line_item") + delta = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="delta") + line_item = sgqlc.types.Field(sgqlc.types.non_null("LineItem"), graphql_name="lineItem") + + +class OrderTransactionConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("OrderTransactionEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("OrderTransaction"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") + + +class OrderTransactionEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("OrderTransaction"), graphql_name="node") + + +class OrderUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("order", "user_errors") + order = sgqlc.types.Field("Order", graphql_name="order") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class PageInfo(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("end_cursor", "has_next_page", "has_previous_page", "start_cursor") + end_cursor = sgqlc.types.Field(String, graphql_name="endCursor") + has_next_page = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasNextPage") + has_previous_page = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasPreviousPage") + start_cursor = sgqlc.types.Field(String, graphql_name="startCursor") + + +class ParseError(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("code", "message", "range") + code = sgqlc.types.Field(sgqlc.types.non_null(ParseErrorCode), graphql_name="code") + message = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="message") + range = sgqlc.types.Field("ParseErrorRange", graphql_name="range") + + +class ParseErrorRange(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("end", "start") + end = sgqlc.types.Field(sgqlc.types.non_null(ErrorPosition), graphql_name="end") + start = sgqlc.types.Field(sgqlc.types.non_null(ErrorPosition), graphql_name="start") + + +class PaymentCustomizationActivationPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("ids", "user_errors") + ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="ids") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PaymentCustomizationError"))), graphql_name="userErrors" + ) + + +class PaymentCustomizationConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PaymentCustomizationEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PaymentCustomization"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class PaymentCustomizationCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("payment_customization", "user_errors") + payment_customization = sgqlc.types.Field("PaymentCustomization", graphql_name="paymentCustomization") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PaymentCustomizationError"))), graphql_name="userErrors" + ) + + +class PaymentCustomizationDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_id", "user_errors") + deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PaymentCustomizationError"))), graphql_name="userErrors" + ) + + +class PaymentCustomizationEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("PaymentCustomization"), graphql_name="node") + + +class PaymentCustomizationUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("payment_customization", "user_errors") + payment_customization = sgqlc.types.Field("PaymentCustomization", graphql_name="paymentCustomization") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PaymentCustomizationError"))), graphql_name="userErrors" + ) + + +class PaymentReminderSendPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("success", "user_errors") + success = sgqlc.types.Field(Boolean, graphql_name="success") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PaymentReminderSendUserError"))), graphql_name="userErrors" + ) + + +class PaymentScheduleConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PaymentScheduleEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PaymentSchedule"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class PaymentScheduleEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("PaymentSchedule"), graphql_name="node") + + +class PaymentSettings(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("supported_digital_wallets",) + supported_digital_wallets = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DigitalWallet))), graphql_name="supportedDigitalWallets" + ) + + +class PaymentTermsCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("payment_terms", "user_errors") + payment_terms = sgqlc.types.Field("PaymentTerms", graphql_name="paymentTerms") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PaymentTermsCreateUserError"))), graphql_name="userErrors" + ) + + +class PaymentTermsDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_id", "user_errors") + deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PaymentTermsDeleteUserError"))), graphql_name="userErrors" + ) + + +class PaymentTermsUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("payment_terms", "user_errors") + payment_terms = sgqlc.types.Field("PaymentTerms", graphql_name="paymentTerms") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PaymentTermsUpdateUserError"))), graphql_name="userErrors" + ) + + +class PolarisVizDataPoint(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("key", "value") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + value = sgqlc.types.Field(String, graphql_name="value") + + +class PolarisVizDataSeries(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("data", "is_comparison", "name") + data = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(PolarisVizDataPoint))), graphql_name="data") + is_comparison = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isComparison") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + + +class PriceListAdjustment(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("type", "value") + type = sgqlc.types.Field(sgqlc.types.non_null(PriceListAdjustmentType), graphql_name="type") + value = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="value") + + +class PriceListAdjustmentSettings(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("compare_at_mode",) + compare_at_mode = sgqlc.types.Field(sgqlc.types.non_null(PriceListCompareAtMode), graphql_name="compareAtMode") + + +class PriceListConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceListEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceList"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class PriceListCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("price_list", "user_errors") + price_list = sgqlc.types.Field("PriceList", graphql_name="priceList") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceListUserError"))), graphql_name="userErrors" + ) + + +class PriceListDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_id", "user_errors") + deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceListUserError"))), graphql_name="userErrors" + ) + + +class PriceListEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("PriceList"), graphql_name="node") + + +class PriceListFixedPricesAddPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("prices", "user_errors") + prices = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("PriceListPrice")), graphql_name="prices") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceListPriceUserError"))), graphql_name="userErrors" + ) + + +class PriceListFixedPricesByProductUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("price_list", "prices_to_add_products", "prices_to_delete_products", "user_errors") + price_list = sgqlc.types.Field("PriceList", graphql_name="priceList") + prices_to_add_products = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("Product")), graphql_name="pricesToAddProducts") + prices_to_delete_products = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("Product")), graphql_name="pricesToDeleteProducts" + ) + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceListFixedPricesByProductBulkUpdateUserError"))), + graphql_name="userErrors", + ) + + +class PriceListFixedPricesDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_fixed_price_variant_ids", "user_errors") + deleted_fixed_price_variant_ids = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="deletedFixedPriceVariantIds" + ) + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceListPriceUserError"))), graphql_name="userErrors" + ) + + +class PriceListFixedPricesUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_fixed_price_variant_ids", "price_list", "prices_added", "user_errors") + deleted_fixed_price_variant_ids = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="deletedFixedPriceVariantIds" + ) + price_list = sgqlc.types.Field("PriceList", graphql_name="priceList") + prices_added = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("PriceListPrice")), graphql_name="pricesAdded") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceListPriceUserError"))), graphql_name="userErrors" + ) + + +class PriceListParent(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("adjustment", "settings") + adjustment = sgqlc.types.Field(sgqlc.types.non_null(PriceListAdjustment), graphql_name="adjustment") + settings = sgqlc.types.Field(sgqlc.types.non_null(PriceListAdjustmentSettings), graphql_name="settings") + + +class PriceListPrice(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("compare_at_price", "origin_type", "price", "variant") + compare_at_price = sgqlc.types.Field(MoneyV2, graphql_name="compareAtPrice") + origin_type = sgqlc.types.Field(sgqlc.types.non_null(PriceListPriceOriginType), graphql_name="originType") + price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="price") + variant = sgqlc.types.Field(sgqlc.types.non_null("ProductVariant"), graphql_name="variant") + + +class PriceListPriceConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceListPriceEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(PriceListPrice))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class PriceListPriceEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(PriceListPrice), graphql_name="node") + + +class PriceListUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("price_list", "user_errors") + price_list = sgqlc.types.Field("PriceList", graphql_name="priceList") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceListUserError"))), graphql_name="userErrors" + ) + + +class PriceRuleActivatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("price_rule", "price_rule_user_errors") + price_rule = sgqlc.types.Field("PriceRule", graphql_name="priceRule") + price_rule_user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceRuleUserError"))), graphql_name="priceRuleUserErrors" + ) + + +class PriceRuleConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceRuleEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceRule"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class PriceRuleCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("price_rule", "price_rule_discount_code", "price_rule_user_errors") + price_rule = sgqlc.types.Field("PriceRule", graphql_name="priceRule") + price_rule_discount_code = sgqlc.types.Field("PriceRuleDiscountCode", graphql_name="priceRuleDiscountCode") + price_rule_user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceRuleUserError"))), graphql_name="priceRuleUserErrors" + ) + + +class PriceRuleCustomerSelection(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("customers", "for_all_customers", "segments") + customers = sgqlc.types.Field( + sgqlc.types.non_null(CustomerConnection), + graphql_name="customers", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(CustomerSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ) + ), + ) + for_all_customers = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="forAllCustomers") + segments = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Segment"))), graphql_name="segments") + + +class PriceRuleDeactivatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("price_rule", "price_rule_user_errors") + price_rule = sgqlc.types.Field("PriceRule", graphql_name="priceRule") + price_rule_user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceRuleUserError"))), graphql_name="priceRuleUserErrors" + ) + + +class PriceRuleDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_price_rule_id", "price_rule_user_errors", "shop") + deleted_price_rule_id = sgqlc.types.Field(ID, graphql_name="deletedPriceRuleId") + price_rule_user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceRuleUserError"))), graphql_name="priceRuleUserErrors" + ) + shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") + + +class PriceRuleDiscountCodeConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceRuleDiscountCodeEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceRuleDiscountCode"))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class PriceRuleDiscountCodeCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("price_rule", "price_rule_discount_code", "price_rule_user_errors") + price_rule = sgqlc.types.Field("PriceRule", graphql_name="priceRule") + price_rule_discount_code = sgqlc.types.Field("PriceRuleDiscountCode", graphql_name="priceRuleDiscountCode") + price_rule_user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceRuleUserError"))), graphql_name="priceRuleUserErrors" + ) + + +class PriceRuleDiscountCodeEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("PriceRuleDiscountCode"), graphql_name="node") + + +class PriceRuleDiscountCodeUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("price_rule", "price_rule_discount_code", "price_rule_user_errors") + price_rule = sgqlc.types.Field("PriceRule", graphql_name="priceRule") + price_rule_discount_code = sgqlc.types.Field("PriceRuleDiscountCode", graphql_name="priceRuleDiscountCode") + price_rule_user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceRuleUserError"))), graphql_name="priceRuleUserErrors" + ) + + +class PriceRuleEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("PriceRule"), graphql_name="node") + + +class PriceRuleEntitlementToPrerequisiteQuantityRatio(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("entitlement_quantity", "prerequisite_quantity") + entitlement_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="entitlementQuantity") + prerequisite_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="prerequisiteQuantity") + + +class PriceRuleFixedAmountValue(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("amount",) + amount = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="amount") + + +class PriceRuleItemEntitlements(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("collections", "product_variants", "products", "target_all_line_items") + collections = sgqlc.types.Field( + sgqlc.types.non_null(CollectionConnection), + graphql_name="collections", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + product_variants = sgqlc.types.Field( + sgqlc.types.non_null("ProductVariantConnection"), + graphql_name="productVariants", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + products = sgqlc.types.Field( + sgqlc.types.non_null("ProductConnection"), + graphql_name="products", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + target_all_line_items = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="targetAllLineItems") + + +class PriceRuleLineItemPrerequisites(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("collections", "product_variants", "products") + collections = sgqlc.types.Field( + sgqlc.types.non_null(CollectionConnection), + graphql_name="collections", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + product_variants = sgqlc.types.Field( + sgqlc.types.non_null("ProductVariantConnection"), + graphql_name="productVariants", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + products = sgqlc.types.Field( + sgqlc.types.non_null("ProductConnection"), + graphql_name="products", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + + +class PriceRuleMoneyRange(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("greater_than", "greater_than_or_equal_to", "less_than", "less_than_or_equal_to") + greater_than = sgqlc.types.Field(Money, graphql_name="greaterThan") + greater_than_or_equal_to = sgqlc.types.Field(Money, graphql_name="greaterThanOrEqualTo") + less_than = sgqlc.types.Field(Money, graphql_name="lessThan") + less_than_or_equal_to = sgqlc.types.Field(Money, graphql_name="lessThanOrEqualTo") + + +class PriceRulePercentValue(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("percentage",) + percentage = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="percentage") + + +class PriceRulePrerequisiteToEntitlementQuantityRatio(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("entitlement_quantity", "prerequisite_quantity") + entitlement_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="entitlementQuantity") + prerequisite_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="prerequisiteQuantity") + + +class PriceRuleQuantityRange(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("greater_than", "greater_than_or_equal_to", "less_than", "less_than_or_equal_to") + greater_than = sgqlc.types.Field(Int, graphql_name="greaterThan") + greater_than_or_equal_to = sgqlc.types.Field(Int, graphql_name="greaterThanOrEqualTo") + less_than = sgqlc.types.Field(Int, graphql_name="lessThan") + less_than_or_equal_to = sgqlc.types.Field(Int, graphql_name="lessThanOrEqualTo") + + +class PriceRuleShareableUrl(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("target_item_image", "target_type", "title", "url") + target_item_image = sgqlc.types.Field("Image", graphql_name="targetItemImage") + target_type = sgqlc.types.Field(sgqlc.types.non_null(PriceRuleShareableUrlTargetType), graphql_name="targetType") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") + + +class PriceRuleShippingLineEntitlements(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("country_codes", "include_rest_of_world", "target_all_shipping_lines") + country_codes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CountryCode))), graphql_name="countryCodes" + ) + include_rest_of_world = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="includeRestOfWorld") + target_all_shipping_lines = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="targetAllShippingLines") + + +class PriceRuleUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("price_rule", "price_rule_discount_code", "price_rule_user_errors") + price_rule = sgqlc.types.Field("PriceRule", graphql_name="priceRule") + price_rule_discount_code = sgqlc.types.Field("PriceRuleDiscountCode", graphql_name="priceRuleDiscountCode") + price_rule_user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceRuleUserError"))), graphql_name="priceRuleUserErrors" + ) + + +class PriceRuleValidityPeriod(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("end", "start") + end = sgqlc.types.Field(DateTime, graphql_name="end") + start = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="start") + + +class PricingPercentageValue(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("percentage",) + percentage = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="percentage") + + +class PrivateMetafieldConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PrivateMetafieldEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PrivateMetafield"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class PrivateMetafieldDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_private_metafield_id", "user_errors") + deleted_private_metafield_id = sgqlc.types.Field(ID, graphql_name="deletedPrivateMetafieldId") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class PrivateMetafieldEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("PrivateMetafield"), graphql_name="node") + + +class PrivateMetafieldUpsertPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("private_metafield", "user_errors") + private_metafield = sgqlc.types.Field("PrivateMetafield", graphql_name="privateMetafield") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ProductAppendImagesPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("new_images", "product", "user_errors") + new_images = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("Image")), graphql_name="newImages") + product = sgqlc.types.Field("Product", graphql_name="product") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ProductCategory(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("product_taxonomy_node",) + product_taxonomy_node = sgqlc.types.Field("ProductTaxonomyNode", graphql_name="productTaxonomyNode") + + +class ProductChangeStatusPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("product", "user_errors") + product = sgqlc.types.Field("Product", graphql_name="product") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductChangeStatusUserError"))), graphql_name="userErrors" + ) + + +class ProductConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Product"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class ProductContextualPricing(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("fixed_quantity_rules_count", "max_variant_pricing", "min_variant_pricing", "price_range") + fixed_quantity_rules_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="fixedQuantityRulesCount") + max_variant_pricing = sgqlc.types.Field("ProductVariantContextualPricing", graphql_name="maxVariantPricing") + min_variant_pricing = sgqlc.types.Field("ProductVariantContextualPricing", graphql_name="minVariantPricing") + price_range = sgqlc.types.Field(sgqlc.types.non_null("ProductPriceRangeV2"), graphql_name="priceRange") + + +class ProductCreateMediaPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("media", "media_user_errors", "product") + media = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(Media)), graphql_name="media") + media_user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MediaUserError"))), graphql_name="mediaUserErrors" + ) + product = sgqlc.types.Field("Product", graphql_name="product") + + +class ProductCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("product", "shop", "user_errors") + product = sgqlc.types.Field("Product", graphql_name="product") + shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ProductDeleteAsyncPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("delete_product_id", "job", "user_errors") + delete_product_id = sgqlc.types.Field(ID, graphql_name="deleteProductId") + job = sgqlc.types.Field(Job, graphql_name="job") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductDeleteUserError"))), graphql_name="userErrors" + ) + + +class ProductDeleteImagesPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_image_ids", "product", "user_errors") + deleted_image_ids = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="deletedImageIds" + ) + product = sgqlc.types.Field("Product", graphql_name="product") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ProductDeleteMediaPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_media_ids", "deleted_product_image_ids", "media_user_errors", "product") + deleted_media_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="deletedMediaIds") + deleted_product_image_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="deletedProductImageIds") + media_user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MediaUserError"))), graphql_name="mediaUserErrors" + ) + product = sgqlc.types.Field("Product", graphql_name="product") + + +class ProductDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_product_id", "shop", "user_errors") + deleted_product_id = sgqlc.types.Field(ID, graphql_name="deletedProductId") + shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ProductDuplicateAsyncPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("duplicated_product_id", "job", "user_errors") + duplicated_product_id = sgqlc.types.Field(ID, graphql_name="duplicatedProductId") + job = sgqlc.types.Field(Job, graphql_name="job") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductDuplicateUserError"))), graphql_name="userErrors" + ) + + +class ProductDuplicateAsyncV2Payload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("duplicated_product_id", "product_duplicate_job_id", "user_errors") + duplicated_product_id = sgqlc.types.Field(ID, graphql_name="duplicatedProductId") + product_duplicate_job_id = sgqlc.types.Field(ID, graphql_name="productDuplicateJobId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductDuplicateUserError"))), graphql_name="userErrors" + ) + + +class ProductDuplicateJob(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("done", "id") + done = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="done") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + + +class ProductDuplicatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("image_job", "new_product", "shop", "user_errors") + image_job = sgqlc.types.Field(Job, graphql_name="imageJob") + new_product = sgqlc.types.Field("Product", graphql_name="newProduct") + shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ProductEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("Product"), graphql_name="node") + + +class ProductFeedConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductFeedEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductFeed"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class ProductFeedCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("product_feed", "user_errors") + product_feed = sgqlc.types.Field("ProductFeed", graphql_name="productFeed") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductFeedCreateUserError"))), graphql_name="userErrors" + ) + + +class ProductFeedDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_id", "user_errors") + deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductFeedDeleteUserError"))), graphql_name="userErrors" + ) + + +class ProductFeedEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("ProductFeed"), graphql_name="node") + + +class ProductFullSyncPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("user_errors",) + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductFullSyncUserError"))), graphql_name="userErrors" + ) + + +class ProductImageUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("image", "user_errors") + image = sgqlc.types.Field("Image", graphql_name="image") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ProductJoinSellingPlanGroupsPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("product", "user_errors") + product = sgqlc.types.Field("Product", graphql_name="product") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupUserError"))), graphql_name="userErrors" + ) + + +class ProductLeaveSellingPlanGroupsPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("product", "user_errors") + product = sgqlc.types.Field("Product", graphql_name="product") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupUserError"))), graphql_name="userErrors" + ) + + +class ProductPriceRange(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("max_variant_price", "min_variant_price") + max_variant_price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="maxVariantPrice") + min_variant_price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="minVariantPrice") + + +class ProductPriceRangeV2(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("max_variant_price", "min_variant_price") + max_variant_price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="maxVariantPrice") + min_variant_price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="minVariantPrice") + + +class ProductPublication(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("channel", "is_published", "product", "publish_date") + channel = sgqlc.types.Field(sgqlc.types.non_null("Channel"), graphql_name="channel") + is_published = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isPublished") + product = sgqlc.types.Field(sgqlc.types.non_null("Product"), graphql_name="product") + publish_date = sgqlc.types.Field(DateTime, graphql_name="publishDate") + + +class ProductPublicationConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductPublicationEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProductPublication))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class ProductPublicationEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(ProductPublication), graphql_name="node") + + +class ProductPublishPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("product", "shop", "user_errors") + product = sgqlc.types.Field("Product", graphql_name="product") + shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ProductReorderImagesPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("job", "user_errors") + job = sgqlc.types.Field(Job, graphql_name="job") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ProductReorderMediaPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("job", "media_user_errors") + job = sgqlc.types.Field(Job, graphql_name="job") + media_user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MediaUserError"))), graphql_name="mediaUserErrors" + ) + + +class ProductResourceFeedback(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("feedback_generated_at", "messages", "product_id", "product_updated_at", "state") + feedback_generated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="feedbackGeneratedAt") + messages = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="messages") + product_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="productId") + product_updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="productUpdatedAt") + state = sgqlc.types.Field(sgqlc.types.non_null(ResourceFeedbackState), graphql_name="state") + + +class ProductUnpublishPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("product", "shop", "user_errors") + product = sgqlc.types.Field("Product", graphql_name="product") + shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ProductUpdateMediaPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("media", "media_user_errors", "product") + media = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(Media)), graphql_name="media") + media_user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MediaUserError"))), graphql_name="mediaUserErrors" + ) + product = sgqlc.types.Field("Product", graphql_name="product") + + +class ProductUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("product", "user_errors") + product = sgqlc.types.Field("Product", graphql_name="product") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ProductVariantAppendMediaPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("product", "product_variants", "user_errors") + product = sgqlc.types.Field("Product", graphql_name="product") + product_variants = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariant")), graphql_name="productVariants") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MediaUserError"))), graphql_name="userErrors" + ) + + +class ProductVariantComponentConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariantComponentEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariantComponent"))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class ProductVariantComponentEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("ProductVariantComponent"), graphql_name="node") + + +class ProductVariantConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariantEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariant"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class ProductVariantContextualPricing(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("compare_at_price", "price", "quantity_rule") + compare_at_price = sgqlc.types.Field(MoneyV2, graphql_name="compareAtPrice") + price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="price") + quantity_rule = sgqlc.types.Field(sgqlc.types.non_null("QuantityRule"), graphql_name="quantityRule") + + +class ProductVariantCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("product", "product_variant", "user_errors") + product = sgqlc.types.Field("Product", graphql_name="product") + product_variant = sgqlc.types.Field("ProductVariant", graphql_name="productVariant") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ProductVariantDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_product_variant_id", "product", "user_errors") + deleted_product_variant_id = sgqlc.types.Field(ID, graphql_name="deletedProductVariantId") + product = sgqlc.types.Field("Product", graphql_name="product") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ProductVariantDetachMediaPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("product", "product_variants", "user_errors") + product = sgqlc.types.Field("Product", graphql_name="product") + product_variants = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariant")), graphql_name="productVariants") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MediaUserError"))), graphql_name="userErrors" + ) + + +class ProductVariantEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("ProductVariant"), graphql_name="node") + + +class ProductVariantJoinSellingPlanGroupsPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("product_variant", "user_errors") + product_variant = sgqlc.types.Field("ProductVariant", graphql_name="productVariant") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupUserError"))), graphql_name="userErrors" + ) + + +class ProductVariantLeaveSellingPlanGroupsPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("product_variant", "user_errors") + product_variant = sgqlc.types.Field("ProductVariant", graphql_name="productVariant") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupUserError"))), graphql_name="userErrors" + ) + + +class ProductVariantPricePair(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("compare_at_price", "price") + compare_at_price = sgqlc.types.Field(MoneyV2, graphql_name="compareAtPrice") + price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="price") + + +class ProductVariantPricePairConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariantPricePairEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProductVariantPricePair))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class ProductVariantPricePairEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(ProductVariantPricePair), graphql_name="node") + + +class ProductVariantRelationshipBulkUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("parent_product_variants", "user_errors") + parent_product_variants = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("ProductVariant")), graphql_name="parentProductVariants" + ) + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariantRelationshipBulkUpdateUserError"))), + graphql_name="userErrors", + ) + + +class ProductVariantUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("product", "product_variant", "user_errors") + product = sgqlc.types.Field("Product", graphql_name="product") + product_variant = sgqlc.types.Field("ProductVariant", graphql_name="productVariant") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ProductVariantsBulkCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("product", "product_variants", "user_errors") + product = sgqlc.types.Field("Product", graphql_name="product") + product_variants = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariant")), graphql_name="productVariants") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariantsBulkCreateUserError"))), graphql_name="userErrors" + ) + + +class ProductVariantsBulkDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("product", "user_errors") + product = sgqlc.types.Field("Product", graphql_name="product") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariantsBulkDeleteUserError"))), graphql_name="userErrors" + ) + + +class ProductVariantsBulkReorderPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("product", "user_errors") + product = sgqlc.types.Field("Product", graphql_name="product") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariantsBulkReorderUserError"))), graphql_name="userErrors" + ) + + +class ProductVariantsBulkUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("product", "product_variants", "user_errors") + product = sgqlc.types.Field("Product", graphql_name="product") + product_variants = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariant")), graphql_name="productVariants") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariantsBulkUpdateUserError"))), graphql_name="userErrors" + ) + + +class PubSubServerPixelUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("server_pixel", "user_errors") + server_pixel = sgqlc.types.Field("ServerPixel", graphql_name="serverPixel") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ErrorsServerPixelUserError"))), graphql_name="userErrors" + ) + + +class PubSubWebhookSubscriptionCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("user_errors", "webhook_subscription") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PubSubWebhookSubscriptionCreateUserError"))), + graphql_name="userErrors", + ) + webhook_subscription = sgqlc.types.Field("WebhookSubscription", graphql_name="webhookSubscription") + + +class PubSubWebhookSubscriptionUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("user_errors", "webhook_subscription") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PubSubWebhookSubscriptionUpdateUserError"))), + graphql_name="userErrors", + ) + webhook_subscription = sgqlc.types.Field("WebhookSubscription", graphql_name="webhookSubscription") + + +class PublicationConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PublicationEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Publication"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class PublicationCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("publication", "user_errors") + publication = sgqlc.types.Field("Publication", graphql_name="publication") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PublicationUserError"))), graphql_name="userErrors" + ) + + +class PublicationDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_id", "user_errors") + deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PublicationUserError"))), graphql_name="userErrors" + ) + + +class PublicationEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("Publication"), graphql_name="node") + + +class PublicationUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("publication", "user_errors") + publication = sgqlc.types.Field("Publication", graphql_name="publication") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PublicationUserError"))), graphql_name="userErrors" + ) + + +class PublishablePublishPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("publishable", "shop", "user_errors") + publishable = sgqlc.types.Field(Publishable, graphql_name="publishable") + shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class PublishablePublishToCurrentChannelPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("publishable", "shop", "user_errors") + publishable = sgqlc.types.Field(Publishable, graphql_name="publishable") + shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class PublishableUnpublishPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("publishable", "shop", "user_errors") + publishable = sgqlc.types.Field(Publishable, graphql_name="publishable") + shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class PublishableUnpublishToCurrentChannelPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("publishable", "shop", "user_errors") + publishable = sgqlc.types.Field(Publishable, graphql_name="publishable") + shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class PurchasingCompany(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("company", "contact", "location") + company = sgqlc.types.Field(sgqlc.types.non_null("Company"), graphql_name="company") + contact = sgqlc.types.Field("CompanyContact", graphql_name="contact") + location = sgqlc.types.Field(sgqlc.types.non_null("CompanyLocation"), graphql_name="location") + + +class QuantityRule(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("increment", "is_default", "maximum", "minimum", "origin_type", "product_variant") + increment = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="increment") + is_default = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isDefault") + maximum = sgqlc.types.Field(Int, graphql_name="maximum") + minimum = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="minimum") + origin_type = sgqlc.types.Field(sgqlc.types.non_null(QuantityRuleOriginType), graphql_name="originType") + product_variant = sgqlc.types.Field(sgqlc.types.non_null("ProductVariant"), graphql_name="productVariant") + + +class QuantityRuleConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info", "total_count") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("QuantityRuleEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(QuantityRule))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + total_count = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="totalCount") + + +class QuantityRuleEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(QuantityRule), graphql_name="node") + + +class QuantityRulesAddPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("quantity_rules", "user_errors") + quantity_rules = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(QuantityRule)), graphql_name="quantityRules") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("QuantityRuleUserError"))), graphql_name="userErrors" + ) + + +class QuantityRulesDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_quantity_rules_variant_ids", "user_errors") + deleted_quantity_rules_variant_ids = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="deletedQuantityRulesVariantIds" + ) + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("QuantityRuleUserError"))), graphql_name="userErrors" + ) + + +class QueryRoot(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "abandonment", + "abandonment_by_abandoned_checkout_id", + "app", + "app_by_handle", + "app_by_key", + "app_discount_type", + "app_discount_types", + "app_installation", + "app_installations", + "automatic_discount_node", + "automatic_discount_nodes", + "automatic_discount_saved_searches", + "available_carrier_services", + "available_locales", + "carrier_service", + "cart_transforms", + "catalog", + "catalog_operations", + "catalogs", + "checkout_profile", + "checkout_profiles", + "code_discount_node", + "code_discount_node_by_code", + "code_discount_nodes", + "code_discount_saved_searches", + "collection", + "collection_by_handle", + "collection_rules_conditions", + "collection_saved_searches", + "collections", + "companies", + "company", + "company_contact", + "company_contact_role", + "company_count", + "company_location", + "company_locations", + "current_app_installation", + "current_bulk_operation", + "customer", + "customer_merge_job_status", + "customer_merge_preview", + "customer_payment_method", + "customer_segment_members", + "customer_segment_members_query", + "customer_segment_membership", + "customers", + "deletion_events", + "delivery_customization", + "delivery_customizations", + "delivery_profile", + "delivery_profiles", + "delivery_settings", + "discount_code_count", + "discount_node", + "discount_nodes", + "discount_redeem_code_bulk_creation", + "discount_redeem_code_saved_searches", + "dispute", + "dispute_evidence", + "domain", + "draft_order", + "draft_order_saved_searches", + "draft_order_tag", + "draft_orders", + "file_saved_searches", + "files", + "fulfillment", + "fulfillment_order", + "fulfillment_orders", + "fulfillment_service", + "gift_card", + "gift_cards", + "gift_cards_count", + "inventory_item", + "inventory_items", + "inventory_level", + "inventory_properties", + "job", + "location", + "locations", + "locations_available_for_delivery_profiles_connection", + "manual_holds_fulfillment_orders", + "market", + "market_by_geography", + "market_localizable_resource", + "market_localizable_resources", + "market_localizable_resources_by_ids", + "marketing_activities", + "marketing_activity", + "marketing_event", + "marketing_events", + "markets", + "metafield", + "metafield_definition", + "metafield_definition_types", + "metafield_definitions", + "metaobject", + "metaobject_by_handle", + "metaobject_definition", + "metaobject_definition_by_type", + "metaobject_definitions", + "metaobjects", + "node", + "nodes", + "order", + "order_payment_status", + "order_saved_searches", + "orders", + "payment_customization", + "payment_customizations", + "payment_terms_templates", + "price_list", + "price_lists", + "price_rule_saved_searches", + "primary_market", + "product", + "product_by_handle", + "product_duplicate_job", + "product_feed", + "product_feeds", + "product_resource_feedback", + "product_saved_searches", + "product_variant", + "product_variants", + "products", + "public_api_versions", + "publication", + "publications", + "refund", + "return_", + "returnable_fulfillment", + "returnable_fulfillments", + "reverse_delivery", + "reverse_fulfillment_order", + "script_tag", + "script_tags", + "segment", + "segment_count", + "segment_filter_suggestions", + "segment_filters", + "segment_migrations", + "segment_value_suggestions", + "segments", + "selling_plan_group", + "selling_plan_groups", + "server_pixel", + "shop", + "shop_billing_preferences", + "shop_locales", + "shopify_function", + "shopify_functions", + "shopify_payments_account", + "shopifyql_query", + "staff_member", + "standard_metafield_definition_templates", + "subscription_billing_attempt", + "subscription_billing_attempts", + "subscription_billing_cycle", + "subscription_billing_cycles", + "subscription_contract", + "subscription_contracts", + "subscription_draft", + "tender_transactions", + "translatable_resource", + "translatable_resources", + "translatable_resources_by_ids", + "url_redirect", + "url_redirect_import", + "url_redirect_saved_searches", + "url_redirects", + "web_pixel", + "webhook_subscription", + "webhook_subscriptions", + ) + abandonment = sgqlc.types.Field( + "Abandonment", + graphql_name="abandonment", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + abandonment_by_abandoned_checkout_id = sgqlc.types.Field( + "Abandonment", + graphql_name="abandonmentByAbandonedCheckoutId", + args=sgqlc.types.ArgDict( + (("abandoned_checkout_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="abandonedCheckoutId", default=None)),) + ), + ) + app = sgqlc.types.Field( + "App", graphql_name="app", args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(ID, graphql_name="id", default=None)),)) + ) + app_by_handle = sgqlc.types.Field( + "App", + graphql_name="appByHandle", + args=sgqlc.types.ArgDict((("handle", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="handle", default=None)),)), + ) + app_by_key = sgqlc.types.Field( + "App", + graphql_name="appByKey", + args=sgqlc.types.ArgDict((("api_key", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="apiKey", default=None)),)), + ) + app_discount_type = sgqlc.types.Field( + AppDiscountType, + graphql_name="appDiscountType", + args=sgqlc.types.ArgDict( + (("function_id", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="functionId", default=None)),) + ), + ) + app_discount_types = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(AppDiscountType))), graphql_name="appDiscountTypes" + ) + app_installation = sgqlc.types.Field( + "AppInstallation", + graphql_name="appInstallation", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(ID, graphql_name="id", default=None)),)), + ) + app_installations = sgqlc.types.Field( + sgqlc.types.non_null(AppInstallationConnection), + graphql_name="appInstallations", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(AppInstallationSortKeys, graphql_name="sortKey", default="INSTALLED_AT")), + ("category", sgqlc.types.Arg(AppInstallationCategory, graphql_name="category", default=None)), + ("privacy", sgqlc.types.Arg(AppInstallationPrivacy, graphql_name="privacy", default="PUBLIC")), + ) + ), + ) + automatic_discount_node = sgqlc.types.Field( + "DiscountAutomaticNode", + graphql_name="automaticDiscountNode", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + automatic_discount_nodes = sgqlc.types.Field( + sgqlc.types.non_null(DiscountAutomaticNodeConnection), + graphql_name="automaticDiscountNodes", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(AutomaticDiscountSortKeys, graphql_name="sortKey", default="CREATED_AT")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ) + ), + ) + automatic_discount_saved_searches = sgqlc.types.Field( + sgqlc.types.non_null("SavedSearchConnection"), + graphql_name="automaticDiscountSavedSearches", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + available_carrier_services = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DeliveryCarrierServiceAndLocations))), + graphql_name="availableCarrierServices", + ) + available_locales = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Locale))), graphql_name="availableLocales" + ) + carrier_service = sgqlc.types.Field( + "DeliveryCarrierService", + graphql_name="carrierService", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + cart_transforms = sgqlc.types.Field( + sgqlc.types.non_null(CartTransformConnection), + graphql_name="cartTransforms", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + catalog = sgqlc.types.Field( + Catalog, + graphql_name="catalog", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + catalog_operations = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ResourceOperation))), graphql_name="catalogOperations" + ) + catalogs = sgqlc.types.Field( + sgqlc.types.non_null(CatalogConnection), + graphql_name="catalogs", + args=sgqlc.types.ArgDict( + ( + ("type", sgqlc.types.Arg(CatalogType, graphql_name="type", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(CatalogSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + checkout_profile = sgqlc.types.Field( + "CheckoutProfile", + graphql_name="checkoutProfile", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + checkout_profiles = sgqlc.types.Field( + sgqlc.types.non_null(CheckoutProfileConnection), + graphql_name="checkoutProfiles", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(CheckoutProfileSortKeys, graphql_name="sortKey", default="UPDATED_AT")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + code_discount_node = sgqlc.types.Field( + "DiscountCodeNode", + graphql_name="codeDiscountNode", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + code_discount_node_by_code = sgqlc.types.Field( + "DiscountCodeNode", + graphql_name="codeDiscountNodeByCode", + args=sgqlc.types.ArgDict((("code", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="code", default=None)),)), + ) + code_discount_nodes = sgqlc.types.Field( + sgqlc.types.non_null(DiscountCodeNodeConnection), + graphql_name="codeDiscountNodes", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(CodeDiscountSortKeys, graphql_name="sortKey", default="CREATED_AT")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ) + ), + ) + code_discount_saved_searches = sgqlc.types.Field( + sgqlc.types.non_null("SavedSearchConnection"), + graphql_name="codeDiscountSavedSearches", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + collection = sgqlc.types.Field( + "Collection", + graphql_name="collection", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + collection_by_handle = sgqlc.types.Field( + "Collection", + graphql_name="collectionByHandle", + args=sgqlc.types.ArgDict((("handle", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="handle", default=None)),)), + ) + collection_rules_conditions = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CollectionRuleConditions))), graphql_name="collectionRulesConditions" + ) + collection_saved_searches = sgqlc.types.Field( + sgqlc.types.non_null("SavedSearchConnection"), + graphql_name="collectionSavedSearches", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + collections = sgqlc.types.Field( + sgqlc.types.non_null(CollectionConnection), + graphql_name="collections", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(CollectionSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ) + ), + ) + companies = sgqlc.types.Field( + sgqlc.types.non_null(CompanyConnection), + graphql_name="companies", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(CompanySortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + company = sgqlc.types.Field( + "Company", + graphql_name="company", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + company_contact = sgqlc.types.Field( + "CompanyContact", + graphql_name="companyContact", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + company_contact_role = sgqlc.types.Field( + "CompanyContactRole", + graphql_name="companyContactRole", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + company_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="companyCount") + company_location = sgqlc.types.Field( + "CompanyLocation", + graphql_name="companyLocation", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + company_locations = sgqlc.types.Field( + sgqlc.types.non_null(CompanyLocationConnection), + graphql_name="companyLocations", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(CompanyLocationSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + current_app_installation = sgqlc.types.Field(sgqlc.types.non_null("AppInstallation"), graphql_name="currentAppInstallation") + current_bulk_operation = sgqlc.types.Field( + "BulkOperation", + graphql_name="currentBulkOperation", + args=sgqlc.types.ArgDict((("type", sgqlc.types.Arg(BulkOperationType, graphql_name="type", default="QUERY")),)), + ) + customer = sgqlc.types.Field( + "Customer", + graphql_name="customer", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + customer_merge_job_status = sgqlc.types.Field( + CustomerMergeRequest, + graphql_name="customerMergeJobStatus", + args=sgqlc.types.ArgDict((("job_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="jobId", default=None)),)), + ) + customer_merge_preview = sgqlc.types.Field( + sgqlc.types.non_null(CustomerMergePreview), + graphql_name="customerMergePreview", + args=sgqlc.types.ArgDict( + ( + ("customer_one_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerOneId", default=None)), + ("customer_two_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerTwoId", default=None)), + ("override_fields", sgqlc.types.Arg(CustomerMergeOverrideFields, graphql_name="overrideFields", default=None)), + ) + ), + ) + customer_payment_method = sgqlc.types.Field( + "CustomerPaymentMethod", + graphql_name="customerPaymentMethod", + args=sgqlc.types.ArgDict( + ( + ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), + ("show_revoked", sgqlc.types.Arg(Boolean, graphql_name="showRevoked", default=False)), + ) + ), + ) + customer_segment_members = sgqlc.types.Field( + sgqlc.types.non_null(CustomerSegmentMemberConnection), + graphql_name="customerSegmentMembers", + args=sgqlc.types.ArgDict( + ( + ("segment_id", sgqlc.types.Arg(ID, graphql_name="segmentId", default=None)), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("query_id", sgqlc.types.Arg(ID, graphql_name="queryId", default=None)), + ("timezone", sgqlc.types.Arg(String, graphql_name="timezone", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(String, graphql_name="sortKey", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ) + ), + ) + customer_segment_members_query = sgqlc.types.Field( + "CustomerSegmentMembersQuery", + graphql_name="customerSegmentMembersQuery", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + customer_segment_membership = sgqlc.types.Field( + sgqlc.types.non_null("SegmentMembershipResponse"), + graphql_name="customerSegmentMembership", + args=sgqlc.types.ArgDict( + ( + ( + "segment_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="segmentIds", default=None + ), + ), + ("customer_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerId", default=None)), + ) + ), + ) + customers = sgqlc.types.Field( + sgqlc.types.non_null(CustomerConnection), + graphql_name="customers", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(CustomerSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + deletion_events = sgqlc.types.Field( + sgqlc.types.non_null(DeletionEventConnection), + graphql_name="deletionEvents", + args=sgqlc.types.ArgDict( + ( + ( + "subject_types", + sgqlc.types.Arg( + sgqlc.types.list_of(sgqlc.types.non_null(DeletionEventSubjectType)), graphql_name="subjectTypes", default=None + ), + ), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(DeletionEventSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + delivery_customization = sgqlc.types.Field( + "DeliveryCustomization", + graphql_name="deliveryCustomization", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + delivery_customizations = sgqlc.types.Field( + sgqlc.types.non_null(DeliveryCustomizationConnection), + graphql_name="deliveryCustomizations", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + delivery_profile = sgqlc.types.Field( + "DeliveryProfile", + graphql_name="deliveryProfile", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + delivery_profiles = sgqlc.types.Field( + sgqlc.types.non_null(DeliveryProfileConnection), + graphql_name="deliveryProfiles", + args=sgqlc.types.ArgDict( + ( + ("merchant_owned_only", sgqlc.types.Arg(Boolean, graphql_name="merchantOwnedOnly", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + delivery_settings = sgqlc.types.Field(DeliverySetting, graphql_name="deliverySettings") + discount_code_count = sgqlc.types.Field( + sgqlc.types.non_null(Int), + graphql_name="discountCodeCount", + args=sgqlc.types.ArgDict((("query", sgqlc.types.Arg(String, graphql_name="query", default=None)),)), + ) + discount_node = sgqlc.types.Field( + "DiscountNode", + graphql_name="discountNode", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + discount_nodes = sgqlc.types.Field( + sgqlc.types.non_null(DiscountNodeConnection), + graphql_name="discountNodes", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(DiscountSortKeys, graphql_name="sortKey", default="CREATED_AT")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ) + ), + ) + discount_redeem_code_bulk_creation = sgqlc.types.Field( + "DiscountRedeemCodeBulkCreation", + graphql_name="discountRedeemCodeBulkCreation", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + discount_redeem_code_saved_searches = sgqlc.types.Field( + sgqlc.types.non_null("SavedSearchConnection"), + graphql_name="discountRedeemCodeSavedSearches", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(DiscountCodeSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + dispute = sgqlc.types.Field( + "ShopifyPaymentsDispute", + graphql_name="dispute", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + dispute_evidence = sgqlc.types.Field( + "ShopifyPaymentsDisputeEvidence", + graphql_name="disputeEvidence", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + domain = sgqlc.types.Field( + "Domain", + graphql_name="domain", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + draft_order = sgqlc.types.Field( + "DraftOrder", + graphql_name="draftOrder", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + draft_order_saved_searches = sgqlc.types.Field( + sgqlc.types.non_null("SavedSearchConnection"), + graphql_name="draftOrderSavedSearches", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + draft_order_tag = sgqlc.types.Field( + "DraftOrderTag", + graphql_name="draftOrderTag", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + draft_orders = sgqlc.types.Field( + sgqlc.types.non_null(DraftOrderConnection), + graphql_name="draftOrders", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(DraftOrderSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ) + ), + ) + file_saved_searches = sgqlc.types.Field( + sgqlc.types.non_null("SavedSearchConnection"), + graphql_name="fileSavedSearches", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + files = sgqlc.types.Field( + sgqlc.types.non_null(FileConnection), + graphql_name="files", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(FileSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ) + ), + ) + fulfillment = sgqlc.types.Field( + "Fulfillment", + graphql_name="fulfillment", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + fulfillment_order = sgqlc.types.Field( + "FulfillmentOrder", + graphql_name="fulfillmentOrder", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + fulfillment_orders = sgqlc.types.Field( + sgqlc.types.non_null(FulfillmentOrderConnection), + graphql_name="fulfillmentOrders", + args=sgqlc.types.ArgDict( + ( + ("include_closed", sgqlc.types.Arg(Boolean, graphql_name="includeClosed", default=False)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(FulfillmentOrderSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + fulfillment_service = sgqlc.types.Field( + FulfillmentService, + graphql_name="fulfillmentService", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + gift_card = sgqlc.types.Field( + "GiftCard", + graphql_name="giftCard", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + gift_cards = sgqlc.types.Field( + sgqlc.types.non_null(GiftCardConnection), + graphql_name="giftCards", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(GiftCardSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ) + ), + ) + gift_cards_count = sgqlc.types.Field( + sgqlc.types.non_null(UnsignedInt64), + graphql_name="giftCardsCount", + args=sgqlc.types.ArgDict((("enabled", sgqlc.types.Arg(Boolean, graphql_name="enabled", default=None)),)), + ) + inventory_item = sgqlc.types.Field( + "InventoryItem", + graphql_name="inventoryItem", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + inventory_items = sgqlc.types.Field( + sgqlc.types.non_null(InventoryItemConnection), + graphql_name="inventoryItems", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + inventory_level = sgqlc.types.Field( + "InventoryLevel", + graphql_name="inventoryLevel", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + inventory_properties = sgqlc.types.Field(sgqlc.types.non_null(InventoryProperties), graphql_name="inventoryProperties") + job = sgqlc.types.Field( + Job, + graphql_name="job", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + location = sgqlc.types.Field( + "Location", graphql_name="location", args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(ID, graphql_name="id", default=None)),)) + ) + locations = sgqlc.types.Field( + sgqlc.types.non_null(LocationConnection), + graphql_name="locations", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(LocationSortKeys, graphql_name="sortKey", default="NAME")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("include_legacy", sgqlc.types.Arg(Boolean, graphql_name="includeLegacy", default=False)), + ("include_inactive", sgqlc.types.Arg(Boolean, graphql_name="includeInactive", default=False)), + ) + ), + ) + locations_available_for_delivery_profiles_connection = sgqlc.types.Field( + sgqlc.types.non_null(LocationConnection), + graphql_name="locationsAvailableForDeliveryProfilesConnection", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + manual_holds_fulfillment_orders = sgqlc.types.Field( + sgqlc.types.non_null(FulfillmentOrderConnection), + graphql_name="manualHoldsFulfillmentOrders", + args=sgqlc.types.ArgDict( + ( + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + market = sgqlc.types.Field( + "Market", + graphql_name="market", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + market_by_geography = sgqlc.types.Field( + "Market", + graphql_name="marketByGeography", + args=sgqlc.types.ArgDict( + (("country_code", sgqlc.types.Arg(sgqlc.types.non_null(CountryCode), graphql_name="countryCode", default=None)),) + ), + ) + market_localizable_resource = sgqlc.types.Field( + MarketLocalizableResource, + graphql_name="marketLocalizableResource", + args=sgqlc.types.ArgDict((("resource_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="resourceId", default=None)),)), + ) + market_localizable_resources = sgqlc.types.Field( + sgqlc.types.non_null(MarketLocalizableResourceConnection), + graphql_name="marketLocalizableResources", + args=sgqlc.types.ArgDict( + ( + ( + "resource_type", + sgqlc.types.Arg(sgqlc.types.non_null(MarketLocalizableResourceType), graphql_name="resourceType", default=None), + ), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + market_localizable_resources_by_ids = sgqlc.types.Field( + sgqlc.types.non_null(MarketLocalizableResourceConnection), + graphql_name="marketLocalizableResourcesByIds", + args=sgqlc.types.ArgDict( + ( + ( + "resource_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="resourceIds", default=None + ), + ), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + marketing_activities = sgqlc.types.Field( + sgqlc.types.non_null(MarketingActivityConnection), + graphql_name="marketingActivities", + args=sgqlc.types.ArgDict( + ( + ( + "marketing_activity_ids", + sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="marketingActivityIds", default=()), + ), + ("remote_ids", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="remoteIds", default=())), + ("utm", sgqlc.types.Arg(UTMInput, graphql_name="utm", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(MarketingActivitySortKeys, graphql_name="sortKey", default="CREATED_AT")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ) + ), + ) + marketing_activity = sgqlc.types.Field( + "MarketingActivity", + graphql_name="marketingActivity", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + marketing_event = sgqlc.types.Field( + "MarketingEvent", + graphql_name="marketingEvent", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + marketing_events = sgqlc.types.Field( + sgqlc.types.non_null(MarketingEventConnection), + graphql_name="marketingEvents", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(MarketingEventSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + markets = sgqlc.types.Field( + sgqlc.types.non_null(MarketConnection), + graphql_name="markets", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + metafield = sgqlc.types.Field( + "Metafield", + graphql_name="metafield", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + metafield_definition = sgqlc.types.Field( + "MetafieldDefinition", + graphql_name="metafieldDefinition", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + metafield_definition_types = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MetafieldDefinitionType))), graphql_name="metafieldDefinitionTypes" + ) + metafield_definitions = sgqlc.types.Field( + sgqlc.types.non_null(MetafieldDefinitionConnection), + graphql_name="metafieldDefinitions", + args=sgqlc.types.ArgDict( + ( + ("key", sgqlc.types.Arg(String, graphql_name="key", default=None)), + ("namespace", sgqlc.types.Arg(String, graphql_name="namespace", default=None)), + ("owner_type", sgqlc.types.Arg(sgqlc.types.non_null(MetafieldOwnerType), graphql_name="ownerType", default=None)), + ("pinned_status", sgqlc.types.Arg(MetafieldDefinitionPinnedStatus, graphql_name="pinnedStatus", default="ANY")), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(MetafieldDefinitionSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + metaobject = sgqlc.types.Field( + "Metaobject", + graphql_name="metaobject", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + metaobject_by_handle = sgqlc.types.Field( + "Metaobject", + graphql_name="metaobjectByHandle", + args=sgqlc.types.ArgDict( + (("handle", sgqlc.types.Arg(sgqlc.types.non_null(MetaobjectHandleInput), graphql_name="handle", default=None)),) + ), + ) + metaobject_definition = sgqlc.types.Field( + "MetaobjectDefinition", + graphql_name="metaobjectDefinition", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + metaobject_definition_by_type = sgqlc.types.Field( + "MetaobjectDefinition", + graphql_name="metaobjectDefinitionByType", + args=sgqlc.types.ArgDict((("type", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="type", default=None)),)), + ) + metaobject_definitions = sgqlc.types.Field( + sgqlc.types.non_null(MetaobjectDefinitionConnection), + graphql_name="metaobjectDefinitions", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + metaobjects = sgqlc.types.Field( + sgqlc.types.non_null(MetaobjectConnection), + graphql_name="metaobjects", + args=sgqlc.types.ArgDict( + ( + ("type", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="type", default=None)), + ("sort_key", sgqlc.types.Arg(String, graphql_name="sortKey", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + node = sgqlc.types.Field( + Node, + graphql_name="node", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(Node)), + graphql_name="nodes", + args=sgqlc.types.ArgDict( + ( + ( + "ids", + sgqlc.types.Arg(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="ids", default=None), + ), + ) + ), + ) + order = sgqlc.types.Field( + "Order", + graphql_name="order", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + order_payment_status = sgqlc.types.Field( + OrderPaymentStatus, + graphql_name="orderPaymentStatus", + args=sgqlc.types.ArgDict( + ( + ("payment_reference_id", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="paymentReferenceId", default=None)), + ("order_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="orderId", default=None)), + ) + ), + ) + order_saved_searches = sgqlc.types.Field( + sgqlc.types.non_null("SavedSearchConnection"), + graphql_name="orderSavedSearches", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + orders = sgqlc.types.Field( + sgqlc.types.non_null(OrderConnection), + graphql_name="orders", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(OrderSortKeys, graphql_name="sortKey", default="PROCESSED_AT")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ) + ), + ) + payment_customization = sgqlc.types.Field( + "PaymentCustomization", + graphql_name="paymentCustomization", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + payment_customizations = sgqlc.types.Field( + sgqlc.types.non_null(PaymentCustomizationConnection), + graphql_name="paymentCustomizations", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + payment_terms_templates = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PaymentTermsTemplate"))), + graphql_name="paymentTermsTemplates", + args=sgqlc.types.ArgDict( + (("payment_terms_type", sgqlc.types.Arg(PaymentTermsType, graphql_name="paymentTermsType", default=None)),) + ), + ) + price_list = sgqlc.types.Field( + "PriceList", + graphql_name="priceList", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + price_lists = sgqlc.types.Field( + sgqlc.types.non_null(PriceListConnection), + graphql_name="priceLists", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(PriceListSortKeys, graphql_name="sortKey", default="ID")), + ) + ), + ) + price_rule_saved_searches = sgqlc.types.Field( + sgqlc.types.non_null("SavedSearchConnection"), + graphql_name="priceRuleSavedSearches", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + primary_market = sgqlc.types.Field(sgqlc.types.non_null("Market"), graphql_name="primaryMarket") + product = sgqlc.types.Field( + "Product", + graphql_name="product", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + product_by_handle = sgqlc.types.Field( + "Product", + graphql_name="productByHandle", + args=sgqlc.types.ArgDict((("handle", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="handle", default=None)),)), + ) + product_duplicate_job = sgqlc.types.Field( + sgqlc.types.non_null(ProductDuplicateJob), + graphql_name="productDuplicateJob", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + product_feed = sgqlc.types.Field( + "ProductFeed", + graphql_name="productFeed", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + product_feeds = sgqlc.types.Field( + sgqlc.types.non_null(ProductFeedConnection), + graphql_name="productFeeds", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + product_resource_feedback = sgqlc.types.Field( + ProductResourceFeedback, + graphql_name="productResourceFeedback", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + product_saved_searches = sgqlc.types.Field( + sgqlc.types.non_null("SavedSearchConnection"), + graphql_name="productSavedSearches", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + product_variant = sgqlc.types.Field( + "ProductVariant", + graphql_name="productVariant", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + product_variants = sgqlc.types.Field( + sgqlc.types.non_null(ProductVariantConnection), + graphql_name="productVariants", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(ProductVariantSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ) + ), + ) + products = sgqlc.types.Field( + sgqlc.types.non_null(ProductConnection), + graphql_name="products", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(ProductSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ) + ), + ) + public_api_versions = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ApiVersion))), graphql_name="publicApiVersions" + ) + publication = sgqlc.types.Field( + "Publication", + graphql_name="publication", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + publications = sgqlc.types.Field( + sgqlc.types.non_null(PublicationConnection), + graphql_name="publications", + args=sgqlc.types.ArgDict( + ( + ("catalog_type", sgqlc.types.Arg(CatalogType, graphql_name="catalogType", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + refund = sgqlc.types.Field( + "Refund", + graphql_name="refund", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + return_ = sgqlc.types.Field( + "Return", + graphql_name="return", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + returnable_fulfillment = sgqlc.types.Field( + "ReturnableFulfillment", + graphql_name="returnableFulfillment", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + returnable_fulfillments = sgqlc.types.Field( + sgqlc.types.non_null("ReturnableFulfillmentConnection"), + graphql_name="returnableFulfillments", + args=sgqlc.types.ArgDict( + ( + ("order_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="orderId", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + reverse_delivery = sgqlc.types.Field( + "ReverseDelivery", + graphql_name="reverseDelivery", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + reverse_fulfillment_order = sgqlc.types.Field( + "ReverseFulfillmentOrder", + graphql_name="reverseFulfillmentOrder", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + script_tag = sgqlc.types.Field( + "ScriptTag", + graphql_name="scriptTag", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + script_tags = sgqlc.types.Field( + sgqlc.types.non_null("ScriptTagConnection"), + graphql_name="scriptTags", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("src", sgqlc.types.Arg(URL, graphql_name="src", default=None)), + ) + ), + ) + segment = sgqlc.types.Field( + "Segment", + graphql_name="segment", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + segment_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="segmentCount") + segment_filter_suggestions = sgqlc.types.Field( + sgqlc.types.non_null("SegmentFilterConnection"), + graphql_name="segmentFilterSuggestions", + args=sgqlc.types.ArgDict( + ( + ("search", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="search", default=None)), + ("first", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ) + ), + ) + segment_filters = sgqlc.types.Field( + sgqlc.types.non_null("SegmentFilterConnection"), + graphql_name="segmentFilters", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ) + ), + ) + segment_migrations = sgqlc.types.Field( + sgqlc.types.non_null("SegmentMigrationConnection"), + graphql_name="segmentMigrations", + args=sgqlc.types.ArgDict( + ( + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ) + ), + ) + segment_value_suggestions = sgqlc.types.Field( + sgqlc.types.non_null("SegmentValueConnection"), + graphql_name="segmentValueSuggestions", + args=sgqlc.types.ArgDict( + ( + ("search", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="search", default=None)), + ("filter_query_name", sgqlc.types.Arg(String, graphql_name="filterQueryName", default=None)), + ("function_parameter_query_name", sgqlc.types.Arg(String, graphql_name="functionParameterQueryName", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ) + ), + ) + segments = sgqlc.types.Field( + sgqlc.types.non_null("SegmentConnection"), + graphql_name="segments", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(SegmentSortKeys, graphql_name="sortKey", default="CREATION_DATE")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + selling_plan_group = sgqlc.types.Field( + "SellingPlanGroup", + graphql_name="sellingPlanGroup", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + selling_plan_groups = sgqlc.types.Field( + sgqlc.types.non_null("SellingPlanGroupConnection"), + graphql_name="sellingPlanGroups", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(SellingPlanGroupSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + server_pixel = sgqlc.types.Field("ServerPixel", graphql_name="serverPixel") + shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") + shop_billing_preferences = sgqlc.types.Field(sgqlc.types.non_null("ShopBillingPreferences"), graphql_name="shopBillingPreferences") + shop_locales = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopLocale"))), + graphql_name="shopLocales", + args=sgqlc.types.ArgDict((("published", sgqlc.types.Arg(Boolean, graphql_name="published", default=False)),)), + ) + shopify_function = sgqlc.types.Field( + "ShopifyFunction", + graphql_name="shopifyFunction", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="id", default=None)),)), + ) + shopify_functions = sgqlc.types.Field( + sgqlc.types.non_null("ShopifyFunctionConnection"), + graphql_name="shopifyFunctions", + args=sgqlc.types.ArgDict( + ( + ("api_type", sgqlc.types.Arg(String, graphql_name="apiType", default=None)), + ("use_creation_ui", sgqlc.types.Arg(Boolean, graphql_name="useCreationUi", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + shopify_payments_account = sgqlc.types.Field("ShopifyPaymentsAccount", graphql_name="shopifyPaymentsAccount") + shopifyql_query = sgqlc.types.Field( + ShopifyqlResponse, + graphql_name="shopifyqlQuery", + args=sgqlc.types.ArgDict((("query", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="query", default=None)),)), + ) + staff_member = sgqlc.types.Field( + "StaffMember", graphql_name="staffMember", args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(ID, graphql_name="id", default=None)),)) + ) + standard_metafield_definition_templates = sgqlc.types.Field( + sgqlc.types.non_null("StandardMetafieldDefinitionTemplateConnection"), + graphql_name="standardMetafieldDefinitionTemplates", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + subscription_billing_attempt = sgqlc.types.Field( + "SubscriptionBillingAttempt", + graphql_name="subscriptionBillingAttempt", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + subscription_billing_attempts = sgqlc.types.Field( + sgqlc.types.non_null("SubscriptionBillingAttemptConnection"), + graphql_name="subscriptionBillingAttempts", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(SubscriptionBillingAttemptsSortKeys, graphql_name="sortKey", default="CREATED_AT")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + subscription_billing_cycle = sgqlc.types.Field( + "SubscriptionBillingCycle", + graphql_name="subscriptionBillingCycle", + args=sgqlc.types.ArgDict( + ( + ( + "billing_cycle_input", + sgqlc.types.Arg(sgqlc.types.non_null(SubscriptionBillingCycleInput), graphql_name="billingCycleInput", default=None), + ), + ) + ), + ) + subscription_billing_cycles = sgqlc.types.Field( + sgqlc.types.non_null("SubscriptionBillingCycleConnection"), + graphql_name="subscriptionBillingCycles", + args=sgqlc.types.ArgDict( + ( + ("contract_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="contractId", default=None)), + ( + "billing_cycles_date_range_selector", + sgqlc.types.Arg( + SubscriptionBillingCyclesDateRangeSelector, graphql_name="billingCyclesDateRangeSelector", default=None + ), + ), + ( + "billing_cycles_index_range_selector", + sgqlc.types.Arg( + SubscriptionBillingCyclesIndexRangeSelector, graphql_name="billingCyclesIndexRangeSelector", default=None + ), + ), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(SubscriptionBillingCyclesSortKeys, graphql_name="sortKey", default="CYCLE_INDEX")), + ) + ), + ) + subscription_contract = sgqlc.types.Field( + "SubscriptionContract", + graphql_name="subscriptionContract", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + subscription_contracts = sgqlc.types.Field( + sgqlc.types.non_null("SubscriptionContractConnection"), + graphql_name="subscriptionContracts", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + subscription_draft = sgqlc.types.Field( + "SubscriptionDraft", + graphql_name="subscriptionDraft", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + tender_transactions = sgqlc.types.Field( + sgqlc.types.non_null("TenderTransactionConnection"), + graphql_name="tenderTransactions", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + translatable_resource = sgqlc.types.Field( + "TranslatableResource", + graphql_name="translatableResource", + args=sgqlc.types.ArgDict((("resource_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="resourceId", default=None)),)), + ) + translatable_resources = sgqlc.types.Field( + sgqlc.types.non_null("TranslatableResourceConnection"), + graphql_name="translatableResources", + args=sgqlc.types.ArgDict( + ( + ( + "resource_type", + sgqlc.types.Arg(sgqlc.types.non_null(TranslatableResourceType), graphql_name="resourceType", default=None), + ), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + translatable_resources_by_ids = sgqlc.types.Field( + sgqlc.types.non_null("TranslatableResourceConnection"), + graphql_name="translatableResourcesByIds", + args=sgqlc.types.ArgDict( + ( + ( + "resource_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="resourceIds", default=None + ), + ), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + url_redirect = sgqlc.types.Field( + "UrlRedirect", + graphql_name="urlRedirect", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + url_redirect_import = sgqlc.types.Field( + "UrlRedirectImport", + graphql_name="urlRedirectImport", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + url_redirect_saved_searches = sgqlc.types.Field( + sgqlc.types.non_null("SavedSearchConnection"), + graphql_name="urlRedirectSavedSearches", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + url_redirects = sgqlc.types.Field( + sgqlc.types.non_null("UrlRedirectConnection"), + graphql_name="urlRedirects", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(UrlRedirectSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ) + ), + ) + web_pixel = sgqlc.types.Field( + "WebPixel", graphql_name="webPixel", args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(ID, graphql_name="id", default=None)),)) + ) + webhook_subscription = sgqlc.types.Field( + "WebhookSubscription", + graphql_name="webhookSubscription", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + webhook_subscriptions = sgqlc.types.Field( + sgqlc.types.non_null("WebhookSubscriptionConnection"), + graphql_name="webhookSubscriptions", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(WebhookSubscriptionSortKeys, graphql_name="sortKey", default="CREATED_AT")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("callback_url", sgqlc.types.Arg(URL, graphql_name="callbackUrl", default=None)), + ("format", sgqlc.types.Arg(WebhookSubscriptionFormat, graphql_name="format", default=None)), + ( + "topics", + sgqlc.types.Arg( + sgqlc.types.list_of(sgqlc.types.non_null(WebhookSubscriptionTopic)), graphql_name="topics", default=None + ), + ), + ) + ), + ) + + +class RefundConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("RefundEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Refund"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class RefundCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("order", "refund", "user_errors") + order = sgqlc.types.Field("Order", graphql_name="order") + refund = sgqlc.types.Field("Refund", graphql_name="refund") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class RefundDuty(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("amount_set", "original_duty") + amount_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="amountSet") + original_duty = sgqlc.types.Field("Duty", graphql_name="originalDuty") + + +class RefundEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("Refund"), graphql_name="node") + + +class RefundLineItem(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("line_item", "location", "price_set", "quantity", "restock_type", "restocked", "subtotal_set", "total_tax_set") + line_item = sgqlc.types.Field(sgqlc.types.non_null("LineItem"), graphql_name="lineItem") + location = sgqlc.types.Field("Location", graphql_name="location") + price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="priceSet") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + restock_type = sgqlc.types.Field(sgqlc.types.non_null(RefundLineItemRestockType), graphql_name="restockType") + restocked = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="restocked") + subtotal_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="subtotalSet") + total_tax_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalTaxSet") + + +class RefundLineItemConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("RefundLineItemEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(RefundLineItem))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class RefundLineItemEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(RefundLineItem), graphql_name="node") + + +class ResourceAlert(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("actions", "content", "dismissible_handle", "icon", "severity", "title") + actions = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ResourceAlertAction"))), graphql_name="actions" + ) + content = sgqlc.types.Field(sgqlc.types.non_null(HTML), graphql_name="content") + dismissible_handle = sgqlc.types.Field(String, graphql_name="dismissibleHandle") + icon = sgqlc.types.Field(ResourceAlertIcon, graphql_name="icon") + severity = sgqlc.types.Field(sgqlc.types.non_null(ResourceAlertSeverity), graphql_name="severity") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + + +class ResourceAlertAction(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("primary", "show", "title", "url") + primary = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="primary") + show = sgqlc.types.Field(String, graphql_name="show") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") + + +class ResourceFeedback(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("details", "summary") + details = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(AppFeedback))), graphql_name="details") + summary = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="summary") + + +class ResourceLimit(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("available", "quantity_available", "quantity_limit", "quantity_used") + available = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="available") + quantity_available = sgqlc.types.Field(Int, graphql_name="quantityAvailable") + quantity_limit = sgqlc.types.Field(Int, graphql_name="quantityLimit") + quantity_used = sgqlc.types.Field(Int, graphql_name="quantityUsed") + + +class ResourcePublication(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("is_published", "publication", "publish_date", "publishable") + is_published = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isPublished") + publication = sgqlc.types.Field(sgqlc.types.non_null("Publication"), graphql_name="publication") + publish_date = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="publishDate") + publishable = sgqlc.types.Field(sgqlc.types.non_null(Publishable), graphql_name="publishable") + + +class ResourcePublicationConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ResourcePublicationEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ResourcePublication))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class ResourcePublicationEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(ResourcePublication), graphql_name="node") + + +class ResourcePublicationV2(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("is_published", "publication", "publish_date", "publishable") + is_published = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isPublished") + publication = sgqlc.types.Field(sgqlc.types.non_null("Publication"), graphql_name="publication") + publish_date = sgqlc.types.Field(DateTime, graphql_name="publishDate") + publishable = sgqlc.types.Field(sgqlc.types.non_null(Publishable), graphql_name="publishable") + + +class ResourcePublicationV2Connection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ResourcePublicationV2Edge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ResourcePublicationV2))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class ResourcePublicationV2Edge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(ResourcePublicationV2), graphql_name="node") + + +class ReturnApproveRequestPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("return_", "user_errors") + return_ = sgqlc.types.Field("Return", graphql_name="return") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReturnUserError"))), graphql_name="userErrors" + ) + + +class ReturnCancelPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("return_", "user_errors") + return_ = sgqlc.types.Field("Return", graphql_name="return") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReturnUserError"))), graphql_name="userErrors" + ) + + +class ReturnClosePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("return_", "user_errors") + return_ = sgqlc.types.Field("Return", graphql_name="return") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReturnUserError"))), graphql_name="userErrors" + ) + + +class ReturnConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReturnEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Return"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class ReturnCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("return_", "user_errors") + return_ = sgqlc.types.Field("Return", graphql_name="return") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReturnUserError"))), graphql_name="userErrors" + ) + + +class ReturnDecline(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("note", "reason") + note = sgqlc.types.Field(String, graphql_name="note") + reason = sgqlc.types.Field(sgqlc.types.non_null(ReturnDeclineReason), graphql_name="reason") + + +class ReturnDeclineRequestPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("return_", "user_errors") + return_ = sgqlc.types.Field("Return", graphql_name="return") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReturnUserError"))), graphql_name="userErrors" + ) + + +class ReturnEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("Return"), graphql_name="node") + + +class ReturnLineItemConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReturnLineItemEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReturnLineItem"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class ReturnLineItemEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("ReturnLineItem"), graphql_name="node") + + +class ReturnRefundPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("refund", "user_errors") + refund = sgqlc.types.Field("Refund", graphql_name="refund") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReturnUserError"))), graphql_name="userErrors" + ) + + +class ReturnReopenPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("return_", "user_errors") + return_ = sgqlc.types.Field("Return", graphql_name="return") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReturnUserError"))), graphql_name="userErrors" + ) + + +class ReturnRequestPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("return_", "user_errors") + return_ = sgqlc.types.Field("Return", graphql_name="return") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReturnUserError"))), graphql_name="userErrors" + ) + + +class ReturnableFulfillmentConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReturnableFulfillmentEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReturnableFulfillment"))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class ReturnableFulfillmentEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("ReturnableFulfillment"), graphql_name="node") + + +class ReturnableFulfillmentLineItem(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_line_item", "quantity") + fulfillment_line_item = sgqlc.types.Field(sgqlc.types.non_null("FulfillmentLineItem"), graphql_name="fulfillmentLineItem") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + + +class ReturnableFulfillmentLineItemConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReturnableFulfillmentLineItemEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ReturnableFulfillmentLineItem))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class ReturnableFulfillmentLineItemEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(ReturnableFulfillmentLineItem), graphql_name="node") + + +class ReverseDeliveryConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReverseDeliveryEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReverseDelivery"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class ReverseDeliveryCreateWithShippingPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("reverse_delivery", "user_errors") + reverse_delivery = sgqlc.types.Field("ReverseDelivery", graphql_name="reverseDelivery") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReturnUserError"))), graphql_name="userErrors" + ) + + +class ReverseDeliveryDisposePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("reverse_delivery_line_items", "user_errors") + reverse_delivery_line_items = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("ReverseDeliveryLineItem")), graphql_name="reverseDeliveryLineItems" + ) + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReturnUserError"))), graphql_name="userErrors" + ) + + +class ReverseDeliveryEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("ReverseDelivery"), graphql_name="node") + + +class ReverseDeliveryLabelV2(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("created_at", "public_file_url", "updated_at") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + public_file_url = sgqlc.types.Field(URL, graphql_name="publicFileUrl") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + + +class ReverseDeliveryLineItemConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReverseDeliveryLineItemEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReverseDeliveryLineItem"))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class ReverseDeliveryLineItemEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("ReverseDeliveryLineItem"), graphql_name="node") + + +class ReverseDeliveryShippingDeliverable(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("label", "tracking") + label = sgqlc.types.Field(ReverseDeliveryLabelV2, graphql_name="label") + tracking = sgqlc.types.Field("ReverseDeliveryTrackingV2", graphql_name="tracking") + + +class ReverseDeliveryShippingUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("reverse_delivery", "user_errors") + reverse_delivery = sgqlc.types.Field("ReverseDelivery", graphql_name="reverseDelivery") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReturnUserError"))), graphql_name="userErrors" + ) + + +class ReverseDeliveryTrackingV2(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("carrier_name", "number", "url") + carrier_name = sgqlc.types.Field(String, graphql_name="carrierName") + number = sgqlc.types.Field(String, graphql_name="number") + url = sgqlc.types.Field(URL, graphql_name="url") + + +class ReverseFulfillmentOrderConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReverseFulfillmentOrderEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReverseFulfillmentOrder"))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class ReverseFulfillmentOrderDisposePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("reverse_fulfillment_order_line_items", "user_errors") + reverse_fulfillment_order_line_items = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("ReverseFulfillmentOrderLineItem")), graphql_name="reverseFulfillmentOrderLineItems" + ) + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReturnUserError"))), graphql_name="userErrors" + ) + + +class ReverseFulfillmentOrderEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("ReverseFulfillmentOrder"), graphql_name="node") + + +class ReverseFulfillmentOrderLineItemConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReverseFulfillmentOrderLineItemEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReverseFulfillmentOrderLineItem"))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class ReverseFulfillmentOrderLineItemEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("ReverseFulfillmentOrderLineItem"), graphql_name="node") + + +class ReverseFulfillmentOrderThirdPartyConfirmation(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("status",) + status = sgqlc.types.Field(sgqlc.types.non_null(ReverseFulfillmentOrderThirdPartyConfirmationStatus), graphql_name="status") + + +class RowCount(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("count", "exceeds_max") + count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="count") + exceeds_max = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="exceedsMax") + + +class SEO(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("description", "title") + description = sgqlc.types.Field(String, graphql_name="description") + title = sgqlc.types.Field(String, graphql_name="title") + + +class SaleConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SaleEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Sale))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class SaleEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(Sale), graphql_name="node") + + +class SaleTax(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("amount", "id", "tax_line") + amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="amount") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + tax_line = sgqlc.types.Field(sgqlc.types.non_null("TaxLine"), graphql_name="taxLine") + + +class SalesAgreementConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SalesAgreementEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SalesAgreement))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class SalesAgreementEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(SalesAgreement), graphql_name="node") + + +class SavedSearchConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SavedSearchEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SavedSearch"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class SavedSearchCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("saved_search", "user_errors") + saved_search = sgqlc.types.Field("SavedSearch", graphql_name="savedSearch") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class SavedSearchDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_saved_search_id", "shop", "user_errors") + deleted_saved_search_id = sgqlc.types.Field(ID, graphql_name="deletedSavedSearchId") + shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class SavedSearchEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("SavedSearch"), graphql_name="node") + + +class SavedSearchUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("saved_search", "user_errors") + saved_search = sgqlc.types.Field("SavedSearch", graphql_name="savedSearch") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ScriptTagConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ScriptTagEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ScriptTag"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class ScriptTagCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("script_tag", "user_errors") + script_tag = sgqlc.types.Field("ScriptTag", graphql_name="scriptTag") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ScriptTagDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_script_tag_id", "user_errors") + deleted_script_tag_id = sgqlc.types.Field(ID, graphql_name="deletedScriptTagId") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ScriptTagEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("ScriptTag"), graphql_name="node") + + +class ScriptTagUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("script_tag", "user_errors") + script_tag = sgqlc.types.Field("ScriptTag", graphql_name="scriptTag") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class SearchFilter(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("key", "value") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") + + +class SearchFilterOptions(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("product_availability",) + product_availability = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FilterOption))), graphql_name="productAvailability" + ) + + +class SearchResult(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("description", "image", "reference", "title", "url") + description = sgqlc.types.Field(String, graphql_name="description") + image = sgqlc.types.Field("Image", graphql_name="image") + reference = sgqlc.types.Field(sgqlc.types.non_null(Node), graphql_name="reference") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") + + +class SearchResultConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SearchResultEdge"))), graphql_name="edges") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class SearchResultEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(SearchResult), graphql_name="node") + + +class SegmentAttributeStatistics(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("average", "sum") + average = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="average") + sum = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="sum") + + +class SegmentConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SegmentEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Segment"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class SegmentCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("segment", "user_errors") + segment = sgqlc.types.Field("Segment", graphql_name="segment") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class SegmentDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_segment_id", "user_errors") + deleted_segment_id = sgqlc.types.Field(ID, graphql_name="deletedSegmentId") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class SegmentEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("Segment"), graphql_name="node") + + +class SegmentEventFilterParameter(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("accepts_multiple_values", "localized_description", "localized_name", "optional", "parameter_type", "query_name") + accepts_multiple_values = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="acceptsMultipleValues") + localized_description = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="localizedDescription") + localized_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="localizedName") + optional = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="optional") + parameter_type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="parameterType") + query_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="queryName") + + +class SegmentFilterConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SegmentFilterEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SegmentFilter))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class SegmentFilterEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(SegmentFilter), graphql_name="node") + + +class SegmentMembership(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("is_member", "segment_id") + is_member = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isMember") + segment_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="segmentId") + + +class SegmentMembershipResponse(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("memberships",) + memberships = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SegmentMembership))), graphql_name="memberships" + ) + + +class SegmentMigration(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("id", "saved_search_id", "segment_id") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + saved_search_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="savedSearchId") + segment_id = sgqlc.types.Field(ID, graphql_name="segmentId") + + +class SegmentMigrationConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SegmentMigrationEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SegmentMigration))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class SegmentMigrationEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(SegmentMigration), graphql_name="node") + + +class SegmentStatistics(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("attribute_statistics",) + attribute_statistics = sgqlc.types.Field( + sgqlc.types.non_null(SegmentAttributeStatistics), + graphql_name="attributeStatistics", + args=sgqlc.types.ArgDict( + (("attribute_name", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="attributeName", default=None)),) + ), + ) + + +class SegmentUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("segment", "user_errors") + segment = sgqlc.types.Field("Segment", graphql_name="segment") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class SegmentValue(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("localized_value", "query_name") + localized_value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="localizedValue") + query_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="queryName") + + +class SegmentValueConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SegmentValueEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SegmentValue))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class SegmentValueEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(SegmentValue), graphql_name="node") + + +class SelectedOption(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("name", "value") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") + + +class SellingPlanAnchor(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cutoff_day", "day", "month", "type") + cutoff_day = sgqlc.types.Field(Int, graphql_name="cutoffDay") + day = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="day") + month = sgqlc.types.Field(Int, graphql_name="month") + type = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanAnchorType), graphql_name="type") + + +class SellingPlanCheckoutCharge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("type", "value") + type = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanCheckoutChargeType), graphql_name="type") + value = sgqlc.types.Field(sgqlc.types.non_null("SellingPlanCheckoutChargeValue"), graphql_name="value") + + +class SellingPlanCheckoutChargePercentageValue(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("percentage",) + percentage = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="percentage") + + +class SellingPlanConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlan"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class SellingPlanEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("SellingPlan"), graphql_name="node") + + +class SellingPlanFixedBillingPolicy(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "checkout_charge", + "remaining_balance_charge_exact_time", + "remaining_balance_charge_time_after_checkout", + "remaining_balance_charge_trigger", + ) + checkout_charge = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanCheckoutCharge), graphql_name="checkoutCharge") + remaining_balance_charge_exact_time = sgqlc.types.Field(DateTime, graphql_name="remainingBalanceChargeExactTime") + remaining_balance_charge_time_after_checkout = sgqlc.types.Field(String, graphql_name="remainingBalanceChargeTimeAfterCheckout") + remaining_balance_charge_trigger = sgqlc.types.Field( + sgqlc.types.non_null(SellingPlanRemainingBalanceChargeTrigger), graphql_name="remainingBalanceChargeTrigger" + ) + + +class SellingPlanFixedDeliveryPolicy(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("anchors", "cutoff", "fulfillment_exact_time", "fulfillment_trigger", "intent", "pre_anchor_behavior") + anchors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SellingPlanAnchor))), graphql_name="anchors") + cutoff = sgqlc.types.Field(Int, graphql_name="cutoff") + fulfillment_exact_time = sgqlc.types.Field(DateTime, graphql_name="fulfillmentExactTime") + fulfillment_trigger = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanFulfillmentTrigger), graphql_name="fulfillmentTrigger") + intent = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanFixedDeliveryPolicyIntent), graphql_name="intent") + pre_anchor_behavior = sgqlc.types.Field( + sgqlc.types.non_null(SellingPlanFixedDeliveryPolicyPreAnchorBehavior), graphql_name="preAnchorBehavior" + ) + + +class SellingPlanGroupAddProductVariantsPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("selling_plan_group", "user_errors") + selling_plan_group = sgqlc.types.Field("SellingPlanGroup", graphql_name="sellingPlanGroup") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupUserError"))), graphql_name="userErrors" + ) + + +class SellingPlanGroupAddProductsPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("selling_plan_group", "user_errors") + selling_plan_group = sgqlc.types.Field("SellingPlanGroup", graphql_name="sellingPlanGroup") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupUserError"))), graphql_name="userErrors" + ) + + +class SellingPlanGroupConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroup"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class SellingPlanGroupCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("selling_plan_group", "user_errors") + selling_plan_group = sgqlc.types.Field("SellingPlanGroup", graphql_name="sellingPlanGroup") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupUserError"))), graphql_name="userErrors" + ) + + +class SellingPlanGroupDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_selling_plan_group_id", "user_errors") + deleted_selling_plan_group_id = sgqlc.types.Field(ID, graphql_name="deletedSellingPlanGroupId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupUserError"))), graphql_name="userErrors" + ) + + +class SellingPlanGroupEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("SellingPlanGroup"), graphql_name="node") + + +class SellingPlanGroupRemoveProductVariantsPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("removed_product_variant_ids", "user_errors") + removed_product_variant_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="removedProductVariantIds") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupUserError"))), graphql_name="userErrors" + ) + + +class SellingPlanGroupRemoveProductsPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("removed_product_ids", "user_errors") + removed_product_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="removedProductIds") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupUserError"))), graphql_name="userErrors" + ) + + +class SellingPlanGroupUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_selling_plan_ids", "selling_plan_group", "user_errors") + deleted_selling_plan_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="deletedSellingPlanIds") + selling_plan_group = sgqlc.types.Field("SellingPlanGroup", graphql_name="sellingPlanGroup") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupUserError"))), graphql_name="userErrors" + ) + + +class SellingPlanInventoryPolicy(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("reserve",) + reserve = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanReserve), graphql_name="reserve") + + +class SellingPlanPricingPolicyPercentageValue(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("percentage",) + percentage = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="percentage") + + +class SellingPlanRecurringBillingPolicy(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("anchors", "created_at", "interval", "interval_count", "max_cycles", "min_cycles") + anchors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SellingPlanAnchor))), graphql_name="anchors") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + interval = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanInterval), graphql_name="interval") + interval_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="intervalCount") + max_cycles = sgqlc.types.Field(Int, graphql_name="maxCycles") + min_cycles = sgqlc.types.Field(Int, graphql_name="minCycles") + + +class SellingPlanRecurringDeliveryPolicy(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("anchors", "created_at", "cutoff", "intent", "interval", "interval_count", "pre_anchor_behavior") + anchors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SellingPlanAnchor))), graphql_name="anchors") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + cutoff = sgqlc.types.Field(Int, graphql_name="cutoff") + intent = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanRecurringDeliveryPolicyIntent), graphql_name="intent") + interval = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanInterval), graphql_name="interval") + interval_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="intervalCount") + pre_anchor_behavior = sgqlc.types.Field( + sgqlc.types.non_null(SellingPlanRecurringDeliveryPolicyPreAnchorBehavior), graphql_name="preAnchorBehavior" + ) + + +class ServerPixelCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("server_pixel", "user_errors") + server_pixel = sgqlc.types.Field("ServerPixel", graphql_name="serverPixel") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ErrorsServerPixelUserError"))), graphql_name="userErrors" + ) + + +class ServerPixelDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_server_pixel_id", "user_errors") + deleted_server_pixel_id = sgqlc.types.Field(ID, graphql_name="deletedServerPixelId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ErrorsServerPixelUserError"))), graphql_name="userErrors" + ) + + +class ShippingLine(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "carrier_identifier", + "code", + "custom", + "delivery_category", + "discount_allocations", + "discounted_price_set", + "id", + "original_price_set", + "phone", + "requested_fulfillment_service", + "shipping_rate_handle", + "source", + "tax_lines", + "title", + ) + carrier_identifier = sgqlc.types.Field(String, graphql_name="carrierIdentifier") + code = sgqlc.types.Field(String, graphql_name="code") + custom = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="custom") + delivery_category = sgqlc.types.Field(String, graphql_name="deliveryCategory") + discount_allocations = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DiscountAllocation))), graphql_name="discountAllocations" + ) + discounted_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="discountedPriceSet") + id = sgqlc.types.Field(ID, graphql_name="id") + original_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="originalPriceSet") + phone = sgqlc.types.Field(String, graphql_name="phone") + requested_fulfillment_service = sgqlc.types.Field(FulfillmentService, graphql_name="requestedFulfillmentService") + shipping_rate_handle = sgqlc.types.Field(String, graphql_name="shippingRateHandle") + source = sgqlc.types.Field(String, graphql_name="source") + tax_lines = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TaxLine"))), graphql_name="taxLines") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + + +class ShippingLineConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShippingLineEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ShippingLine))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class ShippingLineEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(ShippingLine), graphql_name="node") + + +class ShippingMethod(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("code", "label") + code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") + label = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="label") + + +class ShippingPackageDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_id", "user_errors") + deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ShippingPackageMakeDefaultPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("user_errors",) + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ShippingPackageUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("user_errors",) + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ShippingRate(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("handle", "price", "title") + handle = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="handle") + price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="price") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + + +class ShippingRefund(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("amount_set", "maximum_refundable_set", "tax_set") + amount_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="amountSet") + maximum_refundable_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="maximumRefundableSet") + tax_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="taxSet") + + +class ShopAlert(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("action", "description") + action = sgqlc.types.Field(sgqlc.types.non_null("ShopAlertAction"), graphql_name="action") + description = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="description") + + +class ShopAlertAction(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("title", "url") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") + + +class ShopBillingPreferences(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("currency",) + currency = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currency") + + +class ShopFeatures(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "avalara_avatax", + "branding", + "bundles", + "captcha", + "captcha_external_domains", + "dynamic_remarketing", + "eligible_for_subscription_migration", + "eligible_for_subscriptions", + "gift_cards", + "harmonized_system_code", + "international_domains", + "international_price_overrides", + "international_price_rules", + "legacy_subscription_gateway_enabled", + "live_view", + "onboarding_visual", + "paypal_express_subscription_gateway_status", + "reports", + "sells_subscriptions", + "show_metrics", + "storefront", + "using_shopify_balance", + ) + avalara_avatax = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="avalaraAvatax") + branding = sgqlc.types.Field(sgqlc.types.non_null(ShopBranding), graphql_name="branding") + bundles = sgqlc.types.Field(sgqlc.types.non_null(BundlesFeature), graphql_name="bundles") + captcha = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="captcha") + captcha_external_domains = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="captchaExternalDomains") + dynamic_remarketing = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="dynamicRemarketing") + eligible_for_subscription_migration = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="eligibleForSubscriptionMigration") + eligible_for_subscriptions = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="eligibleForSubscriptions") + gift_cards = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="giftCards") + harmonized_system_code = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="harmonizedSystemCode") + international_domains = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="internationalDomains") + international_price_overrides = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="internationalPriceOverrides") + international_price_rules = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="internationalPriceRules") + legacy_subscription_gateway_enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="legacySubscriptionGatewayEnabled") + live_view = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="liveView") + onboarding_visual = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="onboardingVisual") + paypal_express_subscription_gateway_status = sgqlc.types.Field( + sgqlc.types.non_null(PaypalExpressSubscriptionsGatewayStatus), graphql_name="paypalExpressSubscriptionGatewayStatus" + ) + reports = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="reports") + sells_subscriptions = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="sellsSubscriptions") + show_metrics = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="showMetrics") + storefront = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="storefront") + using_shopify_balance = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="usingShopifyBalance") + + +class ShopLocale(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("locale", "market_web_presences", "name", "primary", "published") + locale = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="locale") + market_web_presences = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketWebPresence"))), graphql_name="marketWebPresences" + ) + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + primary = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="primary") + published = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="published") + + +class ShopLocaleDisablePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("locale", "user_errors") + locale = sgqlc.types.Field(String, graphql_name="locale") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ShopLocaleEnablePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("shop_locale", "user_errors") + shop_locale = sgqlc.types.Field(ShopLocale, graphql_name="shopLocale") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ShopLocaleUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("shop_locale", "user_errors") + shop_locale = sgqlc.types.Field(ShopLocale, graphql_name="shopLocale") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class ShopPlan(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("display_name", "partner_development", "shopify_plus") + display_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="displayName") + partner_development = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="partnerDevelopment") + shopify_plus = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="shopifyPlus") + + +class ShopPolicyUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("shop_policy", "user_errors") + shop_policy = sgqlc.types.Field("ShopPolicy", graphql_name="shopPolicy") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopPolicyUserError"))), graphql_name="userErrors" + ) + + +class ShopResourceFeedbackCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("feedback", "user_errors") + feedback = sgqlc.types.Field(AppFeedback, graphql_name="feedback") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopResourceFeedbackCreateUserError"))), graphql_name="userErrors" + ) + + +class ShopResourceLimits(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("location_limit", "max_product_options", "max_product_variants", "redirect_limit_reached") + location_limit = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="locationLimit") + max_product_options = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="maxProductOptions") + max_product_variants = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="maxProductVariants") + redirect_limit_reached = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="redirectLimitReached") + + +class ShopifyFunction(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "api_type", + "api_version", + "app", + "app_bridge", + "app_key", + "description", + "id", + "input_query", + "title", + "use_creation_ui", + ) + api_type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="apiType") + api_version = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="apiVersion") + app = sgqlc.types.Field(sgqlc.types.non_null("App"), graphql_name="app") + app_bridge = sgqlc.types.Field(sgqlc.types.non_null(FunctionsAppBridge), graphql_name="appBridge") + app_key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="appKey") + description = sgqlc.types.Field(String, graphql_name="description") + id = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="id") + input_query = sgqlc.types.Field(String, graphql_name="inputQuery") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + use_creation_ui = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="useCreationUi") + + +class ShopifyFunctionConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopifyFunctionEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ShopifyFunction))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class ShopifyFunctionEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(ShopifyFunction), graphql_name="node") + + +class ShopifyPaymentsBankAccountConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopifyPaymentsBankAccountEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopifyPaymentsBankAccount"))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class ShopifyPaymentsBankAccountEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("ShopifyPaymentsBankAccount"), graphql_name="node") + + +class ShopifyPaymentsDisputeConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopifyPaymentsDisputeEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopifyPaymentsDispute"))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class ShopifyPaymentsDisputeEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("ShopifyPaymentsDispute"), graphql_name="node") + + +class ShopifyPaymentsDisputeReasonDetails(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("network_reason_code", "reason") + network_reason_code = sgqlc.types.Field(String, graphql_name="networkReasonCode") + reason = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsDisputeReason), graphql_name="reason") + + +class ShopifyPaymentsExtendedAuthorization(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("extended_authorization_expires_at", "standard_authorization_expires_at") + extended_authorization_expires_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="extendedAuthorizationExpiresAt") + standard_authorization_expires_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="standardAuthorizationExpiresAt") + + +class ShopifyPaymentsFraudSettings(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("decline_charge_on_avs_failure", "decline_charge_on_cvc_failure") + decline_charge_on_avs_failure = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="declineChargeOnAvsFailure") + decline_charge_on_cvc_failure = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="declineChargeOnCvcFailure") + + +class ShopifyPaymentsNotificationSettings(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("payouts",) + payouts = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="payouts") + + +class ShopifyPaymentsPayoutConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopifyPaymentsPayoutEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopifyPaymentsPayout"))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class ShopifyPaymentsPayoutEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("ShopifyPaymentsPayout"), graphql_name="node") + + +class ShopifyPaymentsPayoutSchedule(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("interval", "monthly_anchor", "weekly_anchor") + interval = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsPayoutInterval), graphql_name="interval") + monthly_anchor = sgqlc.types.Field(Int, graphql_name="monthlyAnchor") + weekly_anchor = sgqlc.types.Field(DayOfTheWeek, graphql_name="weeklyAnchor") + + +class ShopifyPaymentsPayoutSummary(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "adjustments_fee", + "adjustments_gross", + "charges_fee", + "charges_gross", + "refunds_fee", + "refunds_fee_gross", + "reserved_funds_fee", + "reserved_funds_gross", + "retried_payouts_fee", + "retried_payouts_gross", + ) + adjustments_fee = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="adjustmentsFee") + adjustments_gross = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="adjustmentsGross") + charges_fee = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="chargesFee") + charges_gross = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="chargesGross") + refunds_fee = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="refundsFee") + refunds_fee_gross = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="refundsFeeGross") + reserved_funds_fee = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="reservedFundsFee") + reserved_funds_gross = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="reservedFundsGross") + retried_payouts_fee = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="retriedPayoutsFee") + retried_payouts_gross = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="retriedPayoutsGross") + + +class ShopifyPaymentsRefundSet(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("acquirer_reference_number",) + acquirer_reference_number = sgqlc.types.Field(String, graphql_name="acquirerReferenceNumber") + + +class ShopifyPaymentsTransactionSet(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("extended_authorization_set", "refund_set") + extended_authorization_set = sgqlc.types.Field(ShopifyPaymentsExtendedAuthorization, graphql_name="extendedAuthorizationSet") + refund_set = sgqlc.types.Field(ShopifyPaymentsRefundSet, graphql_name="refundSet") + + +class ShopifyPaymentsVerificationDocument(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("back_required", "front_required", "type") + back_required = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="backRequired") + front_required = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="frontRequired") + type = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsVerificationDocumentType), graphql_name="type") + + +class ShopifyPaymentsVerificationSubject(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("family_name", "given_name") + family_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="familyName") + given_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="givenName") + + +class StaffMemberConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("StaffMemberEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("StaffMember"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class StaffMemberEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("StaffMember"), graphql_name="node") + + +class StaffMemberPrivateData(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("account_settings_url", "created_at") + account_settings_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="accountSettingsUrl") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + + +class StagedMediaUploadTarget(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("parameters", "resource_url", "url") + parameters = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("StagedUploadParameter"))), graphql_name="parameters" + ) + resource_url = sgqlc.types.Field(URL, graphql_name="resourceUrl") + url = sgqlc.types.Field(URL, graphql_name="url") + + +class StagedUploadParameter(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("name", "value") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") + + +class StagedUploadTarget(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("parameters", "url") + parameters = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ImageUploadParameter))), graphql_name="parameters" + ) + url = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="url") + + +class StagedUploadTargetGeneratePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("parameters", "url", "user_errors") + parameters = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MutationsStagedUploadTargetGenerateUploadParameter))), + graphql_name="parameters", + ) + url = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="url") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class StagedUploadTargetsGeneratePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("urls", "user_errors") + urls = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(StagedUploadTarget)), graphql_name="urls") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class StagedUploadsCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("staged_targets", "user_errors") + staged_targets = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(StagedMediaUploadTarget)), graphql_name="stagedTargets") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class StandardMetafieldDefinitionEnablePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("created_definition", "user_errors") + created_definition = sgqlc.types.Field("MetafieldDefinition", graphql_name="createdDefinition") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("StandardMetafieldDefinitionEnableUserError"))), + graphql_name="userErrors", + ) + + +class StandardMetafieldDefinitionTemplateConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("StandardMetafieldDefinitionTemplateEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("StandardMetafieldDefinitionTemplate"))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class StandardMetafieldDefinitionTemplateEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("StandardMetafieldDefinitionTemplate"), graphql_name="node") + + +class StandardMetaobjectDefinitionEnablePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("metaobject_definition", "user_errors") + metaobject_definition = sgqlc.types.Field("MetaobjectDefinition", graphql_name="metaobjectDefinition") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetaobjectUserError"))), graphql_name="userErrors" + ) + + +class StandardizedProductType(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("product_taxonomy_node",) + product_taxonomy_node = sgqlc.types.Field("ProductTaxonomyNode", graphql_name="productTaxonomyNode") + + +class StorefrontAccessTokenConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("StorefrontAccessTokenEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("StorefrontAccessToken"))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class StorefrontAccessTokenCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("shop", "storefront_access_token", "user_errors") + shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") + storefront_access_token = sgqlc.types.Field("StorefrontAccessToken", graphql_name="storefrontAccessToken") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class StorefrontAccessTokenDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_storefront_access_token_id", "user_errors") + deleted_storefront_access_token_id = sgqlc.types.Field(ID, graphql_name="deletedStorefrontAccessTokenId") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class StorefrontAccessTokenEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("StorefrontAccessToken"), graphql_name="node") + + +class StringConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("StringEdge"))), graphql_name="edges") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class StringEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="node") + + +class SubscriptionAppliedCodeDiscount(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("id", "redeem_code", "rejection_reason") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + redeem_code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="redeemCode") + rejection_reason = sgqlc.types.Field(SubscriptionDiscountRejectionReason, graphql_name="rejectionReason") + + +class SubscriptionBillingAttemptConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionBillingAttemptEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionBillingAttempt"))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class SubscriptionBillingAttemptCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("subscription_billing_attempt", "user_errors") + subscription_billing_attempt = sgqlc.types.Field("SubscriptionBillingAttempt", graphql_name="subscriptionBillingAttempt") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BillingAttemptUserError"))), graphql_name="userErrors" + ) + + +class SubscriptionBillingAttemptEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionBillingAttempt"), graphql_name="node") + + +class SubscriptionBillingCycle(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "billing_attempt_expected_date", + "billing_attempts", + "cycle_end_at", + "cycle_index", + "cycle_start_at", + "edited", + "edited_contract", + "skipped", + "source_contract", + "status", + ) + billing_attempt_expected_date = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="billingAttemptExpectedDate") + billing_attempts = sgqlc.types.Field( + sgqlc.types.non_null(SubscriptionBillingAttemptConnection), + graphql_name="billingAttempts", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + cycle_end_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="cycleEndAt") + cycle_index = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="cycleIndex") + cycle_start_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="cycleStartAt") + edited = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="edited") + edited_contract = sgqlc.types.Field("SubscriptionBillingCycleEditedContract", graphql_name="editedContract") + skipped = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="skipped") + source_contract = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionContract"), graphql_name="sourceContract") + status = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionBillingCycleBillingCycleStatus), graphql_name="status") + + +class SubscriptionBillingCycleConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionBillingCycleEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SubscriptionBillingCycle))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class SubscriptionBillingCycleContractDraftCommitPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("contract", "user_errors") + contract = sgqlc.types.Field("SubscriptionBillingCycleEditedContract", graphql_name="contract") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), graphql_name="userErrors" + ) + + +class SubscriptionBillingCycleContractDraftConcatenatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("draft", "user_errors") + draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), graphql_name="userErrors" + ) + + +class SubscriptionBillingCycleContractEditPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("draft", "user_errors") + draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), graphql_name="userErrors" + ) + + +class SubscriptionBillingCycleEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionBillingCycle), graphql_name="node") + + +class SubscriptionBillingCycleEditDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("billing_cycles", "user_errors") + billing_cycles = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(SubscriptionBillingCycle)), graphql_name="billingCycles") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionBillingCycleUserError"))), graphql_name="userErrors" + ) + + +class SubscriptionBillingCycleEditsDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("billing_cycles", "user_errors") + billing_cycles = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(SubscriptionBillingCycle)), graphql_name="billingCycles") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionBillingCycleUserError"))), graphql_name="userErrors" + ) + + +class SubscriptionBillingCycleScheduleEditPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("billing_cycle", "user_errors") + billing_cycle = sgqlc.types.Field(SubscriptionBillingCycle, graphql_name="billingCycle") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionBillingCycleUserError"))), graphql_name="userErrors" + ) + + +class SubscriptionBillingPolicy(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("anchors", "interval", "interval_count", "max_cycles", "min_cycles") + anchors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SellingPlanAnchor))), graphql_name="anchors") + interval = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanInterval), graphql_name="interval") + interval_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="intervalCount") + max_cycles = sgqlc.types.Field(Int, graphql_name="maxCycles") + min_cycles = sgqlc.types.Field(Int, graphql_name="minCycles") + + +class SubscriptionContractAtomicCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("contract", "user_errors") + contract = sgqlc.types.Field("SubscriptionContract", graphql_name="contract") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), graphql_name="userErrors" + ) + + +class SubscriptionContractConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionContractEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionContract"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class SubscriptionContractCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("draft", "user_errors") + draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), graphql_name="userErrors" + ) + + +class SubscriptionContractEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionContract"), graphql_name="node") + + +class SubscriptionContractProductChangePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("contract", "line_updated", "user_errors") + contract = sgqlc.types.Field("SubscriptionContract", graphql_name="contract") + line_updated = sgqlc.types.Field("SubscriptionLine", graphql_name="lineUpdated") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), graphql_name="userErrors" + ) + + +class SubscriptionContractSetNextBillingDatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("contract", "user_errors") + contract = sgqlc.types.Field("SubscriptionContract", graphql_name="contract") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionContractUserError"))), graphql_name="userErrors" + ) + + +class SubscriptionContractUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("draft", "user_errors") + draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), graphql_name="userErrors" + ) + + +class SubscriptionCyclePriceAdjustment(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("adjustment_type", "adjustment_value", "after_cycle", "computed_price") + adjustment_type = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanPricingPolicyAdjustmentType), graphql_name="adjustmentType") + adjustment_value = sgqlc.types.Field(sgqlc.types.non_null("SellingPlanPricingPolicyAdjustmentValue"), graphql_name="adjustmentValue") + after_cycle = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="afterCycle") + computed_price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="computedPrice") + + +class SubscriptionDeliveryMethodLocalDelivery(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("address", "local_delivery_option") + address = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionMailingAddress"), graphql_name="address") + local_delivery_option = sgqlc.types.Field( + sgqlc.types.non_null("SubscriptionDeliveryMethodLocalDeliveryOption"), graphql_name="localDeliveryOption" + ) + + +class SubscriptionDeliveryMethodLocalDeliveryOption(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("code", "description", "instructions", "phone", "presentment_title", "title") + code = sgqlc.types.Field(String, graphql_name="code") + description = sgqlc.types.Field(String, graphql_name="description") + instructions = sgqlc.types.Field(String, graphql_name="instructions") + phone = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="phone") + presentment_title = sgqlc.types.Field(String, graphql_name="presentmentTitle") + title = sgqlc.types.Field(String, graphql_name="title") + + +class SubscriptionDeliveryMethodPickup(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("pickup_option",) + pickup_option = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionDeliveryMethodPickupOption"), graphql_name="pickupOption") + + +class SubscriptionDeliveryMethodPickupOption(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("code", "description", "location", "presentment_title", "title") + code = sgqlc.types.Field(String, graphql_name="code") + description = sgqlc.types.Field(String, graphql_name="description") + location = sgqlc.types.Field(sgqlc.types.non_null("Location"), graphql_name="location") + presentment_title = sgqlc.types.Field(String, graphql_name="presentmentTitle") + title = sgqlc.types.Field(String, graphql_name="title") + + +class SubscriptionDeliveryMethodShipping(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("address", "shipping_option") + address = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionMailingAddress"), graphql_name="address") + shipping_option = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionDeliveryMethodShippingOption"), graphql_name="shippingOption") + + +class SubscriptionDeliveryMethodShippingOption(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("code", "description", "presentment_title", "title") + code = sgqlc.types.Field(String, graphql_name="code") + description = sgqlc.types.Field(String, graphql_name="description") + presentment_title = sgqlc.types.Field(String, graphql_name="presentmentTitle") + title = sgqlc.types.Field(String, graphql_name="title") + + +class SubscriptionDeliveryOptionResultFailure(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("message",) + message = sgqlc.types.Field(String, graphql_name="message") + + +class SubscriptionDeliveryOptionResultSuccess(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("delivery_options",) + delivery_options = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDeliveryOption"))), graphql_name="deliveryOptions" + ) + + +class SubscriptionDeliveryPolicy(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("anchors", "interval", "interval_count") + anchors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SellingPlanAnchor))), graphql_name="anchors") + interval = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanInterval), graphql_name="interval") + interval_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="intervalCount") + + +class SubscriptionDiscountAllocation(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("amount", "discount") + amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="amount") + discount = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionDiscount"), graphql_name="discount") + + +class SubscriptionDiscountConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDiscountEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDiscount"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class SubscriptionDiscountEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionDiscount"), graphql_name="node") + + +class SubscriptionDiscountEntitledLines(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("all", "lines") + all = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="all") + lines = sgqlc.types.Field( + sgqlc.types.non_null("SubscriptionLineConnection"), + graphql_name="lines", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + + +class SubscriptionDiscountFixedAmountValue(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("amount", "applies_on_each_item") + amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="amount") + applies_on_each_item = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliesOnEachItem") + + +class SubscriptionDiscountPercentageValue(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("percentage",) + percentage = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="percentage") + + +class SubscriptionDraftCommitPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("contract", "user_errors") + contract = sgqlc.types.Field("SubscriptionContract", graphql_name="contract") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), graphql_name="userErrors" + ) + + +class SubscriptionDraftDiscountAddPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("discount_added", "draft", "user_errors") + discount_added = sgqlc.types.Field("SubscriptionManualDiscount", graphql_name="discountAdded") + draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), graphql_name="userErrors" + ) + + +class SubscriptionDraftDiscountCodeApplyPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("applied_discount", "draft", "user_errors") + applied_discount = sgqlc.types.Field(SubscriptionAppliedCodeDiscount, graphql_name="appliedDiscount") + draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), graphql_name="userErrors" + ) + + +class SubscriptionDraftDiscountRemovePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("discount_removed", "draft", "user_errors") + discount_removed = sgqlc.types.Field("SubscriptionDiscount", graphql_name="discountRemoved") + draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), graphql_name="userErrors" + ) + + +class SubscriptionDraftDiscountUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("discount_updated", "draft", "user_errors") + discount_updated = sgqlc.types.Field("SubscriptionManualDiscount", graphql_name="discountUpdated") + draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), graphql_name="userErrors" + ) + + +class SubscriptionDraftFreeShippingDiscountAddPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("discount_added", "draft", "user_errors") + discount_added = sgqlc.types.Field("SubscriptionManualDiscount", graphql_name="discountAdded") + draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), graphql_name="userErrors" + ) + + +class SubscriptionDraftFreeShippingDiscountUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("discount_updated", "draft", "user_errors") + discount_updated = sgqlc.types.Field("SubscriptionManualDiscount", graphql_name="discountUpdated") + draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), graphql_name="userErrors" + ) + + +class SubscriptionDraftLineAddPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("draft", "line_added", "user_errors") + draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") + line_added = sgqlc.types.Field("SubscriptionLine", graphql_name="lineAdded") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), graphql_name="userErrors" + ) + + +class SubscriptionDraftLineRemovePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("discounts_updated", "draft", "line_removed", "user_errors") + discounts_updated = sgqlc.types.Field( + sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionManualDiscount")), graphql_name="discountsUpdated" + ) + draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") + line_removed = sgqlc.types.Field("SubscriptionLine", graphql_name="lineRemoved") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), graphql_name="userErrors" + ) + + +class SubscriptionDraftLineUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("draft", "line_updated", "user_errors") + draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") + line_updated = sgqlc.types.Field("SubscriptionLine", graphql_name="lineUpdated") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), graphql_name="userErrors" + ) + + +class SubscriptionDraftUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("draft", "user_errors") + draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), graphql_name="userErrors" + ) + + +class SubscriptionLine(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "current_price", + "custom_attributes", + "discount_allocations", + "id", + "line_discounted_price", + "pricing_policy", + "product_id", + "quantity", + "requires_shipping", + "selling_plan_id", + "selling_plan_name", + "sku", + "taxable", + "title", + "variant_id", + "variant_image", + "variant_title", + ) + current_price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="currentPrice") + custom_attributes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Attribute))), graphql_name="customAttributes" + ) + discount_allocations = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SubscriptionDiscountAllocation))), graphql_name="discountAllocations" + ) + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + line_discounted_price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="lineDiscountedPrice") + pricing_policy = sgqlc.types.Field("SubscriptionPricingPolicy", graphql_name="pricingPolicy") + product_id = sgqlc.types.Field(ID, graphql_name="productId") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + requires_shipping = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresShipping") + selling_plan_id = sgqlc.types.Field(ID, graphql_name="sellingPlanId") + selling_plan_name = sgqlc.types.Field(String, graphql_name="sellingPlanName") + sku = sgqlc.types.Field(String, graphql_name="sku") + taxable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxable") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + variant_id = sgqlc.types.Field(ID, graphql_name="variantId") + variant_image = sgqlc.types.Field("Image", graphql_name="variantImage") + variant_title = sgqlc.types.Field(String, graphql_name="variantTitle") + + +class SubscriptionLineConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionLineEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SubscriptionLine))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class SubscriptionLineEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionLine), graphql_name="node") + + +class SubscriptionLocalDeliveryOption(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("code", "description", "phone_required", "presentment_title", "price", "title") + code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") + description = sgqlc.types.Field(String, graphql_name="description") + phone_required = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="phoneRequired") + presentment_title = sgqlc.types.Field(String, graphql_name="presentmentTitle") + price = sgqlc.types.Field(MoneyV2, graphql_name="price") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + + +class SubscriptionMailingAddress(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "address1", + "address2", + "city", + "company", + "country", + "country_code", + "first_name", + "last_name", + "name", + "phone", + "province", + "province_code", + "zip", + ) + address1 = sgqlc.types.Field(String, graphql_name="address1") + address2 = sgqlc.types.Field(String, graphql_name="address2") + city = sgqlc.types.Field(String, graphql_name="city") + company = sgqlc.types.Field(String, graphql_name="company") + country = sgqlc.types.Field(String, graphql_name="country") + country_code = sgqlc.types.Field(CountryCode, graphql_name="countryCode") + first_name = sgqlc.types.Field(String, graphql_name="firstName") + last_name = sgqlc.types.Field(String, graphql_name="lastName") + name = sgqlc.types.Field(String, graphql_name="name") + phone = sgqlc.types.Field(String, graphql_name="phone") + province = sgqlc.types.Field(String, graphql_name="province") + province_code = sgqlc.types.Field(String, graphql_name="provinceCode") + zip = sgqlc.types.Field(String, graphql_name="zip") + + +class SubscriptionManualDiscount(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "entitled_lines", + "id", + "recurring_cycle_limit", + "rejection_reason", + "target_type", + "title", + "type", + "usage_count", + "value", + ) + entitled_lines = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionDiscountEntitledLines), graphql_name="entitledLines") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + recurring_cycle_limit = sgqlc.types.Field(Int, graphql_name="recurringCycleLimit") + rejection_reason = sgqlc.types.Field(SubscriptionDiscountRejectionReason, graphql_name="rejectionReason") + target_type = sgqlc.types.Field(sgqlc.types.non_null(DiscountTargetType), graphql_name="targetType") + title = sgqlc.types.Field(String, graphql_name="title") + type = sgqlc.types.Field(sgqlc.types.non_null(DiscountType), graphql_name="type") + usage_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="usageCount") + value = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionDiscountValue"), graphql_name="value") + + +class SubscriptionManualDiscountConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionManualDiscountEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SubscriptionManualDiscount))), graphql_name="nodes" + ) + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class SubscriptionManualDiscountEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionManualDiscount), graphql_name="node") + + +class SubscriptionPickupOption(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("code", "description", "location", "phone_required", "pickup_time", "presentment_title", "price", "title") + code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") + description = sgqlc.types.Field(String, graphql_name="description") + location = sgqlc.types.Field(sgqlc.types.non_null("Location"), graphql_name="location") + phone_required = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="phoneRequired") + pickup_time = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="pickupTime") + presentment_title = sgqlc.types.Field(String, graphql_name="presentmentTitle") + price = sgqlc.types.Field(MoneyV2, graphql_name="price") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + + +class SubscriptionPricingPolicy(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("base_price", "cycle_discounts") + base_price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="basePrice") + cycle_discounts = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SubscriptionCyclePriceAdjustment))), graphql_name="cycleDiscounts" + ) + + +class SubscriptionShippingOption(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("code", "description", "phone_required", "presentment_title", "price", "title") + code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") + description = sgqlc.types.Field(String, graphql_name="description") + phone_required = sgqlc.types.Field(Boolean, graphql_name="phoneRequired") + presentment_title = sgqlc.types.Field(String, graphql_name="presentmentTitle") + price = sgqlc.types.Field(MoneyV2, graphql_name="price") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + + +class SubscriptionShippingOptionResultFailure(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("message",) + message = sgqlc.types.Field(String, graphql_name="message") + + +class SubscriptionShippingOptionResultSuccess(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("shipping_options",) + shipping_options = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SubscriptionShippingOption))), graphql_name="shippingOptions" + ) + + +class SuggestedOrderTransaction(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "account_number", + "amount_set", + "formatted_gateway", + "gateway", + "kind", + "maximum_refundable_set", + "parent_transaction", + ) + account_number = sgqlc.types.Field(String, graphql_name="accountNumber") + amount_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="amountSet") + formatted_gateway = sgqlc.types.Field(String, graphql_name="formattedGateway") + gateway = sgqlc.types.Field(String, graphql_name="gateway") + kind = sgqlc.types.Field(sgqlc.types.non_null(SuggestedOrderTransactionKind), graphql_name="kind") + maximum_refundable_set = sgqlc.types.Field(MoneyBag, graphql_name="maximumRefundableSet") + parent_transaction = sgqlc.types.Field("OrderTransaction", graphql_name="parentTransaction") + + +class SuggestedRefund(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "amount_set", + "discounted_subtotal_set", + "maximum_refundable_set", + "refund_duties", + "refund_line_items", + "shipping", + "subtotal_set", + "suggested_transactions", + "total_cart_discount_amount_set", + "total_duties_set", + "total_tax_set", + ) + amount_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="amountSet") + discounted_subtotal_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="discountedSubtotalSet") + maximum_refundable_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="maximumRefundableSet") + refund_duties = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(RefundDuty))), graphql_name="refundDuties" + ) + refund_line_items = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(RefundLineItem))), graphql_name="refundLineItems" + ) + shipping = sgqlc.types.Field(sgqlc.types.non_null(ShippingRefund), graphql_name="shipping") + subtotal_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="subtotalSet") + suggested_transactions = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SuggestedOrderTransaction))), graphql_name="suggestedTransactions" + ) + total_cart_discount_amount_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalCartDiscountAmountSet") + total_duties_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalDutiesSet") + total_tax_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalTaxSet") + + +class SuggestedReturnRefund(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ( + "amount", + "discounted_subtotal", + "maximum_refundable", + "refund_duties", + "shipping", + "subtotal", + "suggested_transactions", + "total_cart_discount_amount", + "total_duties", + "total_tax", + ) + amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="amount") + discounted_subtotal = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="discountedSubtotal") + maximum_refundable = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="maximumRefundable") + refund_duties = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(RefundDuty))), graphql_name="refundDuties" + ) + shipping = sgqlc.types.Field(sgqlc.types.non_null(ShippingRefund), graphql_name="shipping") + subtotal = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="subtotal") + suggested_transactions = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SuggestedOrderTransaction))), graphql_name="suggestedTransactions" + ) + total_cart_discount_amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalCartDiscountAmount") + total_duties = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalDuties") + total_tax = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalTax") + + +class TableData(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("columns", "row_data", "unformatted_data") + columns = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TableDataColumn"))), graphql_name="columns") + row_data = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))))), + graphql_name="rowData", + ) + unformatted_data = sgqlc.types.Field(sgqlc.types.non_null(JSON), graphql_name="unformattedData") + + +class TableDataColumn(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("compared_to", "data_type", "display_name", "name") + compared_to = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="comparedTo") + data_type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="dataType") + display_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="displayName") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + + +class TagsAddPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("node", "user_errors") + node = sgqlc.types.Field(Node, graphql_name="node") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class TagsRemovePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("node", "user_errors") + node = sgqlc.types.Field(Node, graphql_name="node") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class TaxAppConfiguration(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("state",) + state = sgqlc.types.Field(sgqlc.types.non_null(TaxPartnerState), graphql_name="state") + + +class TaxAppConfigurePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("tax_app_configuration", "user_errors") + tax_app_configuration = sgqlc.types.Field(TaxAppConfiguration, graphql_name="taxAppConfiguration") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TaxAppConfigureUserError"))), graphql_name="userErrors" + ) + + +class TaxLine(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("channel_liable", "price_set", "rate", "rate_percentage", "title") + channel_liable = sgqlc.types.Field(Boolean, graphql_name="channelLiable") + price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="priceSet") + rate = sgqlc.types.Field(Float, graphql_name="rate") + rate_percentage = sgqlc.types.Field(Float, graphql_name="ratePercentage") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + + +class TenderTransactionConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TenderTransactionEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TenderTransaction"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class TenderTransactionCreditCardDetails(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("credit_card_company", "credit_card_number") + credit_card_company = sgqlc.types.Field(String, graphql_name="creditCardCompany") + credit_card_number = sgqlc.types.Field(String, graphql_name="creditCardNumber") + + +class TenderTransactionEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("TenderTransaction"), graphql_name="node") + + +class TranslatableContent(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("digest", "key", "locale", "type", "value") + digest = sgqlc.types.Field(String, graphql_name="digest") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + locale = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="locale") + type = sgqlc.types.Field(sgqlc.types.non_null(LocalizableContentType), graphql_name="type") + value = sgqlc.types.Field(String, graphql_name="value") + + +class TranslatableResource(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("resource_id", "translatable_content", "translations") + resource_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="resourceId") + translatable_content = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TranslatableContent))), graphql_name="translatableContent" + ) + translations = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Translation"))), + graphql_name="translations", + args=sgqlc.types.ArgDict( + ( + ("locale", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="locale", default=None)), + ("outdated", sgqlc.types.Arg(Boolean, graphql_name="outdated", default=None)), + ("market_id", sgqlc.types.Arg(ID, graphql_name="marketId", default=None)), + ) + ), + ) + + +class TranslatableResourceConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TranslatableResourceEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TranslatableResource))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class TranslatableResourceEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null(TranslatableResource), graphql_name="node") + + +class Translation(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("key", "locale", "market", "outdated", "updated_at", "value") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + locale = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="locale") + market = sgqlc.types.Field("Market", graphql_name="market") + outdated = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="outdated") + updated_at = sgqlc.types.Field(DateTime, graphql_name="updatedAt") + value = sgqlc.types.Field(String, graphql_name="value") + + +class TranslationsRegisterPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("translations", "user_errors") + translations = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(Translation)), graphql_name="translations") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TranslationUserError"))), graphql_name="userErrors" + ) + + +class TranslationsRemovePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("translations", "user_errors") + translations = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(Translation)), graphql_name="translations") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TranslationUserError"))), graphql_name="userErrors" + ) + + +class TypedAttribute(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("key", "value") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") + + +class UTMParameters(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("campaign", "content", "medium", "source", "term") + campaign = sgqlc.types.Field(String, graphql_name="campaign") + content = sgqlc.types.Field(String, graphql_name="content") + medium = sgqlc.types.Field(String, graphql_name="medium") + source = sgqlc.types.Field(String, graphql_name="source") + term = sgqlc.types.Field(String, graphql_name="term") + + +class UrlRedirectBulkDeleteAllPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("job", "user_errors") + job = sgqlc.types.Field(Job, graphql_name="job") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class UrlRedirectBulkDeleteByIdsPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("job", "user_errors") + job = sgqlc.types.Field(Job, graphql_name="job") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UrlRedirectBulkDeleteByIdsUserError"))), graphql_name="userErrors" + ) + + +class UrlRedirectBulkDeleteBySavedSearchPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("job", "user_errors") + job = sgqlc.types.Field(Job, graphql_name="job") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UrlRedirectBulkDeleteBySavedSearchUserError"))), + graphql_name="userErrors", + ) + + +class UrlRedirectBulkDeleteBySearchPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("job", "user_errors") + job = sgqlc.types.Field(Job, graphql_name="job") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UrlRedirectBulkDeleteBySearchUserError"))), graphql_name="userErrors" + ) + + +class UrlRedirectConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UrlRedirectEdge"))), graphql_name="edges") + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UrlRedirect"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class UrlRedirectCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("url_redirect", "user_errors") + url_redirect = sgqlc.types.Field("UrlRedirect", graphql_name="urlRedirect") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UrlRedirectUserError"))), graphql_name="userErrors" + ) + + +class UrlRedirectDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_url_redirect_id", "user_errors") + deleted_url_redirect_id = sgqlc.types.Field(ID, graphql_name="deletedUrlRedirectId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UrlRedirectUserError"))), graphql_name="userErrors" + ) + + +class UrlRedirectEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("UrlRedirect"), graphql_name="node") + + +class UrlRedirectImportCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("url_redirect_import", "user_errors") + url_redirect_import = sgqlc.types.Field("UrlRedirectImport", graphql_name="urlRedirectImport") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UrlRedirectImportUserError"))), graphql_name="userErrors" + ) + + +class UrlRedirectImportPreview(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("path", "target") + path = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="path") + target = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="target") + + +class UrlRedirectImportSubmitPayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("job", "user_errors") + job = sgqlc.types.Field(Job, graphql_name="job") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UrlRedirectImportUserError"))), graphql_name="userErrors" + ) + + +class UrlRedirectUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("url_redirect", "user_errors") + url_redirect = sgqlc.types.Field("UrlRedirect", graphql_name="urlRedirect") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UrlRedirectUserError"))), graphql_name="userErrors" + ) + + +class VaultCreditCard(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("billing_address", "brand", "expired", "expiry_month", "expiry_year", "last_digits", "name") + billing_address = sgqlc.types.Field(CustomerCreditCardBillingAddress, graphql_name="billingAddress") + brand = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="brand") + expired = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="expired") + expiry_month = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="expiryMonth") + expiry_year = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="expiryYear") + last_digits = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="lastDigits") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + + +class VaultPaypalBillingAgreement(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("inactive", "name", "paypal_account_email") + inactive = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="inactive") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + paypal_account_email = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="paypalAccountEmail") + + +class Vector3(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("x", "y", "z") + x = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="x") + y = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="y") + z = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="z") + + +class VideoSource(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("file_size", "format", "height", "mime_type", "url", "width") + file_size = sgqlc.types.Field(Int, graphql_name="fileSize") + format = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="format") + height = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="height") + mime_type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="mimeType") + url = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="url") + width = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="width") + + +class WebPixelCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("user_errors", "web_pixel") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ErrorsWebPixelUserError"))), graphql_name="userErrors" + ) + web_pixel = sgqlc.types.Field("WebPixel", graphql_name="webPixel") + + +class WebPixelDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_web_pixel_id", "user_errors") + deleted_web_pixel_id = sgqlc.types.Field(ID, graphql_name="deletedWebPixelId") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ErrorsWebPixelUserError"))), graphql_name="userErrors" + ) + + +class WebPixelUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("user_errors", "web_pixel") + user_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ErrorsWebPixelUserError"))), graphql_name="userErrors" + ) + web_pixel = sgqlc.types.Field("WebPixel", graphql_name="webPixel") + + +class WebhookEventBridgeEndpoint(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("arn",) + arn = sgqlc.types.Field(sgqlc.types.non_null(ARN), graphql_name="arn") + + +class WebhookHttpEndpoint(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("callback_url",) + callback_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="callbackUrl") + + +class WebhookPubSubEndpoint(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("pub_sub_project", "pub_sub_topic") + pub_sub_project = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="pubSubProject") + pub_sub_topic = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="pubSubTopic") + + +class WebhookSubscriptionConnection(sgqlc.types.relay.Connection): + __schema__ = shopify_schema + __field_names__ = ("edges", "nodes", "page_info") + edges = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("WebhookSubscriptionEdge"))), graphql_name="edges" + ) + nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("WebhookSubscription"))), graphql_name="nodes") + page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") + + +class WebhookSubscriptionCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("user_errors", "webhook_subscription") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + webhook_subscription = sgqlc.types.Field("WebhookSubscription", graphql_name="webhookSubscription") + + +class WebhookSubscriptionDeletePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("deleted_webhook_subscription_id", "user_errors") + deleted_webhook_subscription_id = sgqlc.types.Field(ID, graphql_name="deletedWebhookSubscriptionId") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class WebhookSubscriptionEdge(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("cursor", "node") + cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") + node = sgqlc.types.Field(sgqlc.types.non_null("WebhookSubscription"), graphql_name="node") + + +class WebhookSubscriptionUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("user_errors", "webhook_subscription") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + webhook_subscription = sgqlc.types.Field("WebhookSubscription", graphql_name="webhookSubscription") + + +class Weight(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("unit", "value") + unit = sgqlc.types.Field(sgqlc.types.non_null(WeightUnit), graphql_name="unit") + value = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="value") + + +class deliveryProfileCreatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("profile", "user_errors") + profile = sgqlc.types.Field("DeliveryProfile", graphql_name="profile") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class deliveryProfileRemovePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("job", "user_errors") + job = sgqlc.types.Field(Job, graphql_name="job") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class deliveryProfileUpdatePayload(sgqlc.types.Type): + __schema__ = shopify_schema + __field_names__ = ("profile", "user_errors") + profile = sgqlc.types.Field("DeliveryProfile", graphql_name="profile") + user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") + + +class AbandonedCheckout(sgqlc.types.Type, Navigable, Node): + __schema__ = shopify_schema + __field_names__ = ("abandoned_checkout_url", "line_items_quantity", "total_price_set") + abandoned_checkout_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="abandonedCheckoutUrl") + line_items_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="lineItemsQuantity") + total_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalPriceSet") + + +class Abandonment(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "abandoned_checkout_payload", + "abandonment_type", + "app", + "cart_url", + "created_at", + "customer", + "customer_has_no_order_since_abandonment", + "days_since_last_abandonment_email", + "email_sent_at", + "email_state", + "hours_since_last_abandoned_checkout", + "inventory_available", + "is_from_online_store", + "is_from_shop_app", + "is_from_shop_pay", + "is_most_significant_abandonment", + "last_browse_abandonment_date", + "last_cart_abandonment_date", + "last_checkout_abandonment_date", + "most_recent_step", + "products_added_to_cart", + "products_viewed", + "visit_started_at", + ) + abandoned_checkout_payload = sgqlc.types.Field(AbandonedCheckout, graphql_name="abandonedCheckoutPayload") + abandonment_type = sgqlc.types.Field(sgqlc.types.non_null(AbandonmentAbandonmentType), graphql_name="abandonmentType") + app = sgqlc.types.Field(sgqlc.types.non_null("App"), graphql_name="app") + cart_url = sgqlc.types.Field(URL, graphql_name="cartUrl") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + customer = sgqlc.types.Field(sgqlc.types.non_null("Customer"), graphql_name="customer") + customer_has_no_order_since_abandonment = sgqlc.types.Field( + sgqlc.types.non_null(Boolean), graphql_name="customerHasNoOrderSinceAbandonment" + ) + days_since_last_abandonment_email = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="daysSinceLastAbandonmentEmail") + email_sent_at = sgqlc.types.Field(DateTime, graphql_name="emailSentAt") + email_state = sgqlc.types.Field(AbandonmentEmailState, graphql_name="emailState") + hours_since_last_abandoned_checkout = sgqlc.types.Field(Float, graphql_name="hoursSinceLastAbandonedCheckout") + inventory_available = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="inventoryAvailable") + is_from_online_store = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isFromOnlineStore") + is_from_shop_app = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isFromShopApp") + is_from_shop_pay = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isFromShopPay") + is_most_significant_abandonment = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isMostSignificantAbandonment") + last_browse_abandonment_date = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="lastBrowseAbandonmentDate") + last_cart_abandonment_date = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="lastCartAbandonmentDate") + last_checkout_abandonment_date = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="lastCheckoutAbandonmentDate") + most_recent_step = sgqlc.types.Field(sgqlc.types.non_null(AbandonmentAbandonmentType), graphql_name="mostRecentStep") + products_added_to_cart = sgqlc.types.Field( + sgqlc.types.non_null(CustomerVisitProductInfoConnection), + graphql_name="productsAddedToCart", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + products_viewed = sgqlc.types.Field( + sgqlc.types.non_null(CustomerVisitProductInfoConnection), + graphql_name="productsViewed", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + visit_started_at = sgqlc.types.Field(DateTime, graphql_name="visitStartedAt") + + +class AbandonmentEmailStateUpdateUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(AbandonmentEmailStateUpdateUserErrorCode, graphql_name="code") + + +class AbandonmentUpdateActivitiesDeliveryStatusesUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(AbandonmentUpdateActivitiesDeliveryStatusesUserErrorCode, graphql_name="code") + + +class AddAllProductsOperation(sgqlc.types.Type, Node, ResourceOperation): + __schema__ = shopify_schema + __field_names__ = () + + +class AdditionalFee(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("name", "price", "tax_lines") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + price = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="price") + tax_lines = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxLine))), graphql_name="taxLines") + + +class AdditionalFeeSale(sgqlc.types.Type, Sale): + __schema__ = shopify_schema + __field_names__ = ("additional_fee",) + additional_fee = sgqlc.types.Field(sgqlc.types.non_null("SaleAdditionalFee"), graphql_name="additionalFee") + + +class AdjustmentSale(sgqlc.types.Type, Sale): + __schema__ = shopify_schema + __field_names__ = () + + +class App(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "api_key", + "app_store_app_url", + "app_store_developer_url", + "available_access_scopes", + "banner", + "description", + "developer_name", + "developer_type", + "embedded", + "failed_requirements", + "features", + "feedback", + "handle", + "icon", + "install_url", + "installation", + "is_post_purchase_app_in_use", + "previously_installed", + "pricing_details", + "pricing_details_summary", + "privacy_policy_url", + "public_category", + "published", + "requested_access_scopes", + "screenshots", + "shopify_developed", + "title", + "uninstall_message", + "webhook_api_version", + ) + api_key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="apiKey") + app_store_app_url = sgqlc.types.Field(URL, graphql_name="appStoreAppUrl") + app_store_developer_url = sgqlc.types.Field(URL, graphql_name="appStoreDeveloperUrl") + available_access_scopes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(AccessScope))), graphql_name="availableAccessScopes" + ) + banner = sgqlc.types.Field(sgqlc.types.non_null("Image"), graphql_name="banner") + description = sgqlc.types.Field(String, graphql_name="description") + developer_name = sgqlc.types.Field(String, graphql_name="developerName") + developer_type = sgqlc.types.Field(sgqlc.types.non_null(AppDeveloperType), graphql_name="developerType") + embedded = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="embedded") + failed_requirements = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FailedRequirement))), graphql_name="failedRequirements" + ) + features = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="features") + feedback = sgqlc.types.Field(AppFeedback, graphql_name="feedback") + handle = sgqlc.types.Field(String, graphql_name="handle") + icon = sgqlc.types.Field(sgqlc.types.non_null("Image"), graphql_name="icon") + install_url = sgqlc.types.Field(URL, graphql_name="installUrl") + installation = sgqlc.types.Field("AppInstallation", graphql_name="installation") + is_post_purchase_app_in_use = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isPostPurchaseAppInUse") + previously_installed = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="previouslyInstalled") + pricing_details = sgqlc.types.Field(String, graphql_name="pricingDetails") + pricing_details_summary = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="pricingDetailsSummary") + privacy_policy_url = sgqlc.types.Field(URL, graphql_name="privacyPolicyUrl") + public_category = sgqlc.types.Field(sgqlc.types.non_null(AppPublicCategory), graphql_name="publicCategory") + published = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="published") + requested_access_scopes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(AccessScope))), graphql_name="requestedAccessScopes" + ) + screenshots = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Image"))), graphql_name="screenshots") + shopify_developed = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="shopifyDeveloped") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + uninstall_message = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="uninstallMessage") + webhook_api_version = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="webhookApiVersion") + + +class AppCatalog(sgqlc.types.Type, Catalog, Node): + __schema__ = shopify_schema + __field_names__ = ("apps",) + apps = sgqlc.types.Field( + sgqlc.types.non_null(AppConnection), + graphql_name="apps", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + + +class AppCredit(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("amount", "created_at", "description", "test") + amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="amount") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + description = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="description") + test = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="test") + + +class AppInstallation(sgqlc.types.Type, HasMetafields, Node): + __schema__ = shopify_schema + __field_names__ = ( + "access_scopes", + "active_subscriptions", + "all_subscriptions", + "app", + "credits", + "launch_url", + "one_time_purchases", + "publication", + "revenue_attribution_records", + "uninstall_url", + ) + access_scopes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(AccessScope))), graphql_name="accessScopes" + ) + active_subscriptions = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppSubscription"))), graphql_name="activeSubscriptions" + ) + all_subscriptions = sgqlc.types.Field( + sgqlc.types.non_null(AppSubscriptionConnection), + graphql_name="allSubscriptions", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(AppSubscriptionSortKeys, graphql_name="sortKey", default="CREATED_AT")), + ) + ), + ) + app = sgqlc.types.Field(sgqlc.types.non_null(App), graphql_name="app") + credits = sgqlc.types.Field( + sgqlc.types.non_null(AppCreditConnection), + graphql_name="credits", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(AppTransactionSortKeys, graphql_name="sortKey", default="CREATED_AT")), + ) + ), + ) + launch_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="launchUrl") + one_time_purchases = sgqlc.types.Field( + sgqlc.types.non_null(AppPurchaseOneTimeConnection), + graphql_name="oneTimePurchases", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(AppTransactionSortKeys, graphql_name="sortKey", default="CREATED_AT")), + ) + ), + ) + publication = sgqlc.types.Field("Publication", graphql_name="publication") + revenue_attribution_records = sgqlc.types.Field( + sgqlc.types.non_null(AppRevenueAttributionRecordConnection), + graphql_name="revenueAttributionRecords", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(AppRevenueAttributionRecordSortKeys, graphql_name="sortKey", default="CREATED_AT")), + ) + ), + ) + uninstall_url = sgqlc.types.Field(URL, graphql_name="uninstallUrl") + + +class AppPurchaseOneTime(sgqlc.types.Type, AppPurchase, Node): + __schema__ = shopify_schema + __field_names__ = () + + +class AppRevenueAttributionRecord(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("amount", "captured_at", "created_at", "idempotency_key", "test", "type") + amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="amount") + captured_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="capturedAt") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + idempotency_key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="idempotencyKey") + test = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="test") + type = sgqlc.types.Field(sgqlc.types.non_null(AppRevenueAttributionType), graphql_name="type") + + +class AppRevenueAttributionRecordCreateUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(AppRevenueAttributionRecordCreateUserErrorCode, graphql_name="code") + + +class AppRevenueAttributionRecordDeleteUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(AppRevenueAttributionRecordDeleteUserErrorCode, graphql_name="code") + + +class AppSubscription(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("created_at", "current_period_end", "line_items", "name", "return_url", "status", "test", "trial_days") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + current_period_end = sgqlc.types.Field(DateTime, graphql_name="currentPeriodEnd") + line_items = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(AppSubscriptionLineItem))), graphql_name="lineItems" + ) + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + return_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="returnUrl") + status = sgqlc.types.Field(sgqlc.types.non_null(AppSubscriptionStatus), graphql_name="status") + test = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="test") + trial_days = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="trialDays") + + +class AppSubscriptionTrialExtendUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(AppSubscriptionTrialExtendUserErrorCode, graphql_name="code") + + +class AppUsageRecord(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("created_at", "description", "idempotency_key", "price", "subscription_line_item") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + description = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="description") + idempotency_key = sgqlc.types.Field(String, graphql_name="idempotencyKey") + price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="price") + subscription_line_item = sgqlc.types.Field(sgqlc.types.non_null(AppSubscriptionLineItem), graphql_name="subscriptionLineItem") + + +class AutomaticDiscountApplication(sgqlc.types.Type, DiscountApplication): + __schema__ = shopify_schema + __field_names__ = ("title",) + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + + +class BasicEvent(sgqlc.types.Type, Event, Node): + __schema__ = shopify_schema + __field_names__ = () + + +class BillingAttemptUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(BillingAttemptUserErrorCode, graphql_name="code") + + +class BulkMutationUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(BulkMutationErrorCode, graphql_name="code") + + +class BulkOperation(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "completed_at", + "created_at", + "error_code", + "file_size", + "object_count", + "partial_data_url", + "query", + "root_object_count", + "status", + "type", + "url", + ) + completed_at = sgqlc.types.Field(DateTime, graphql_name="completedAt") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + error_code = sgqlc.types.Field(BulkOperationErrorCode, graphql_name="errorCode") + file_size = sgqlc.types.Field(UnsignedInt64, graphql_name="fileSize") + object_count = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="objectCount") + partial_data_url = sgqlc.types.Field(URL, graphql_name="partialDataUrl") + query = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="query") + root_object_count = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="rootObjectCount") + status = sgqlc.types.Field(sgqlc.types.non_null(BulkOperationStatus), graphql_name="status") + type = sgqlc.types.Field(sgqlc.types.non_null(BulkOperationType), graphql_name="type") + url = sgqlc.types.Field(URL, graphql_name="url") + + +class BulkProductResourceFeedbackCreateUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(BulkProductResourceFeedbackCreateUserErrorCode, graphql_name="code") + + +class BusinessCustomerUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(BusinessCustomerErrorCode, graphql_name="code") + + +class CalculatedAutomaticDiscountApplication(sgqlc.types.Type, CalculatedDiscountApplication): + __schema__ = shopify_schema + __field_names__ = () + + +class CalculatedDiscountCodeApplication(sgqlc.types.Type, CalculatedDiscountApplication): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") + + +class CalculatedManualDiscountApplication(sgqlc.types.Type, CalculatedDiscountApplication): + __schema__ = shopify_schema + __field_names__ = () + + +class CalculatedOrder(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "added_discount_applications", + "added_line_items", + "cart_discount_amount_set", + "committed", + "line_items", + "notification_preview_html", + "notification_preview_title", + "original_order", + "staged_changes", + "subtotal_line_items_quantity", + "subtotal_price_set", + "tax_lines", + "total_outstanding_set", + "total_price_set", + ) + added_discount_applications = sgqlc.types.Field( + sgqlc.types.non_null(CalculatedDiscountApplicationConnection), + graphql_name="addedDiscountApplications", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + added_line_items = sgqlc.types.Field( + sgqlc.types.non_null(CalculatedLineItemConnection), + graphql_name="addedLineItems", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + cart_discount_amount_set = sgqlc.types.Field(MoneyBag, graphql_name="cartDiscountAmountSet") + committed = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="committed") + line_items = sgqlc.types.Field( + sgqlc.types.non_null(CalculatedLineItemConnection), + graphql_name="lineItems", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + notification_preview_html = sgqlc.types.Field(HTML, graphql_name="notificationPreviewHtml") + notification_preview_title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="notificationPreviewTitle") + original_order = sgqlc.types.Field(sgqlc.types.non_null("Order"), graphql_name="originalOrder") + staged_changes = sgqlc.types.Field( + sgqlc.types.non_null(OrderStagedChangeConnection), + graphql_name="stagedChanges", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + subtotal_line_items_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="subtotalLineItemsQuantity") + subtotal_price_set = sgqlc.types.Field(MoneyBag, graphql_name="subtotalPriceSet") + tax_lines = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxLine))), graphql_name="taxLines") + total_outstanding_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalOutstandingSet") + total_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalPriceSet") + + +class CalculatedScriptDiscountApplication(sgqlc.types.Type, CalculatedDiscountApplication): + __schema__ = shopify_schema + __field_names__ = () + + +class CartTransform(sgqlc.types.Type, HasMetafields, Node): + __schema__ = shopify_schema + __field_names__ = ("function_id",) + function_id = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="functionId") + + +class CartTransformCreateUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(CartTransformCreateUserErrorCode, graphql_name="code") + + +class CartTransformDeleteUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(CartTransformDeleteUserErrorCode, graphql_name="code") + + +class CatalogCsvOperation(sgqlc.types.Type, Node, ResourceOperation): + __schema__ = shopify_schema + __field_names__ = () + + +class CatalogUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(CatalogUserErrorCode, graphql_name="code") + + +class Channel(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "app", + "collection_publications_v3", + "collections", + "has_collection", + "name", + "product_publications_v3", + "products", + "supports_future_publishing", + ) + app = sgqlc.types.Field(sgqlc.types.non_null(App), graphql_name="app") + collection_publications_v3 = sgqlc.types.Field( + sgqlc.types.non_null(ResourcePublicationConnection), + graphql_name="collectionPublicationsV3", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + collections = sgqlc.types.Field( + sgqlc.types.non_null(CollectionConnection), + graphql_name="collections", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + has_collection = sgqlc.types.Field( + sgqlc.types.non_null(Boolean), + graphql_name="hasCollection", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + product_publications_v3 = sgqlc.types.Field( + sgqlc.types.non_null(ResourcePublicationConnection), + graphql_name="productPublicationsV3", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + products = sgqlc.types.Field( + sgqlc.types.non_null(ProductConnection), + graphql_name="products", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + supports_future_publishing = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="supportsFuturePublishing") + + +class ChannelDefinition(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("channel_name", "handle", "is_marketplace", "sub_channel_name", "svg_icon") + channel_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="channelName") + handle = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="handle") + is_marketplace = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isMarketplace") + sub_channel_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="subChannelName") + svg_icon = sgqlc.types.Field(String, graphql_name="svgIcon") + + +class ChannelInformation(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("app", "channel_definition", "channel_id") + app = sgqlc.types.Field(sgqlc.types.non_null(App), graphql_name="app") + channel_definition = sgqlc.types.Field(ChannelDefinition, graphql_name="channelDefinition") + channel_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="channelId") + + +class CheckoutProfile(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("created_at", "edited_at", "is_published", "name", "updated_at") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + edited_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="editedAt") + is_published = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isPublished") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + + +class Collection(sgqlc.types.Type, HasMetafieldDefinitions, HasMetafields, HasPublishedTranslations, Node, Publishable): + __schema__ = shopify_schema + __field_names__ = ( + "description", + "description_html", + "feedback", + "handle", + "has_product", + "image", + "legacy_resource_id", + "products", + "products_count", + "rule_set", + "seo", + "sort_order", + "template_suffix", + "title", + "updated_at", + ) + description = sgqlc.types.Field( + sgqlc.types.non_null(String), + graphql_name="description", + args=sgqlc.types.ArgDict((("truncate_at", sgqlc.types.Arg(Int, graphql_name="truncateAt", default=None)),)), + ) + description_html = sgqlc.types.Field(sgqlc.types.non_null(HTML), graphql_name="descriptionHtml") + feedback = sgqlc.types.Field(ResourceFeedback, graphql_name="feedback") + handle = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="handle") + has_product = sgqlc.types.Field( + sgqlc.types.non_null(Boolean), + graphql_name="hasProduct", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + image = sgqlc.types.Field("Image", graphql_name="image") + legacy_resource_id = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="legacyResourceId") + products = sgqlc.types.Field( + sgqlc.types.non_null(ProductConnection), + graphql_name="products", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(ProductCollectionSortKeys, graphql_name="sortKey", default="COLLECTION_DEFAULT")), + ) + ), + ) + products_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="productsCount") + rule_set = sgqlc.types.Field(CollectionRuleSet, graphql_name="ruleSet") + seo = sgqlc.types.Field(sgqlc.types.non_null(SEO), graphql_name="seo") + sort_order = sgqlc.types.Field(sgqlc.types.non_null(CollectionSortOrder), graphql_name="sortOrder") + template_suffix = sgqlc.types.Field(String, graphql_name="templateSuffix") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + + +class CollectionAddProductsV2UserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(CollectionAddProductsV2UserErrorCode, graphql_name="code") + + +class CommentEvent(sgqlc.types.Type, Event, Node): + __schema__ = shopify_schema + __field_names__ = ("attachments", "author", "can_delete", "can_edit", "edited", "embed", "raw_message", "subject") + attachments = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CommentEventAttachment))), graphql_name="attachments" + ) + author = sgqlc.types.Field(sgqlc.types.non_null("StaffMember"), graphql_name="author") + can_delete = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="canDelete") + can_edit = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="canEdit") + edited = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="edited") + embed = sgqlc.types.Field("CommentEventEmbed", graphql_name="embed") + raw_message = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="rawMessage") + subject = sgqlc.types.Field(sgqlc.types.non_null(CommentEventSubject), graphql_name="subject") + + +class Company(sgqlc.types.Type, CommentEventSubject, HasEvents, HasMetafieldDefinitions, HasMetafields, Navigable, Node): + __schema__ = shopify_schema + __field_names__ = ( + "contact_count", + "contact_roles", + "contacts", + "created_at", + "customer_since", + "default_role", + "draft_orders", + "external_id", + "lifetime_duration", + "location_count", + "locations", + "main_contact", + "name", + "note", + "order_count", + "orders", + "total_spent", + "updated_at", + ) + contact_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="contactCount") + contact_roles = sgqlc.types.Field( + sgqlc.types.non_null(CompanyContactRoleConnection), + graphql_name="contactRoles", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(CompanyContactRoleSortKeys, graphql_name="sortKey", default="ID")), + ) + ), + ) + contacts = sgqlc.types.Field( + sgqlc.types.non_null(CompanyContactConnection), + graphql_name="contacts", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(CompanyContactSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + customer_since = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="customerSince") + default_role = sgqlc.types.Field("CompanyContactRole", graphql_name="defaultRole") + draft_orders = sgqlc.types.Field( + sgqlc.types.non_null(DraftOrderConnection), + graphql_name="draftOrders", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(DraftOrderSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + external_id = sgqlc.types.Field(String, graphql_name="externalId") + lifetime_duration = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="lifetimeDuration") + location_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="locationCount") + locations = sgqlc.types.Field( + sgqlc.types.non_null(CompanyLocationConnection), + graphql_name="locations", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(CompanyLocationSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + main_contact = sgqlc.types.Field("CompanyContact", graphql_name="mainContact") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + note = sgqlc.types.Field(String, graphql_name="note") + order_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="orderCount") + orders = sgqlc.types.Field( + sgqlc.types.non_null(OrderConnection), + graphql_name="orders", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(OrderSortKeys, graphql_name="sortKey", default="ID")), + ) + ), + ) + total_spent = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="totalSpent") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + + +class CompanyAddress(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "address1", + "address2", + "city", + "company_name", + "country", + "country_code", + "created_at", + "formatted_address", + "formatted_area", + "phone", + "province", + "recipient", + "updated_at", + "zip", + "zone_code", + ) + address1 = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="address1") + address2 = sgqlc.types.Field(String, graphql_name="address2") + city = sgqlc.types.Field(String, graphql_name="city") + company_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="companyName") + country = sgqlc.types.Field(String, graphql_name="country") + country_code = sgqlc.types.Field(sgqlc.types.non_null(CountryCode), graphql_name="countryCode") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + formatted_address = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), + graphql_name="formattedAddress", + args=sgqlc.types.ArgDict( + ( + ("with_name", sgqlc.types.Arg(Boolean, graphql_name="withName", default=False)), + ("with_company_name", sgqlc.types.Arg(Boolean, graphql_name="withCompanyName", default=True)), + ) + ), + ) + formatted_area = sgqlc.types.Field(String, graphql_name="formattedArea") + phone = sgqlc.types.Field(String, graphql_name="phone") + province = sgqlc.types.Field(String, graphql_name="province") + recipient = sgqlc.types.Field(String, graphql_name="recipient") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + zip = sgqlc.types.Field(String, graphql_name="zip") + zone_code = sgqlc.types.Field(String, graphql_name="zoneCode") + + +class CompanyContact(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "company", + "created_at", + "customer", + "draft_orders", + "is_main_contact", + "lifetime_duration", + "locale", + "orders", + "role_assignments", + "title", + "updated_at", + ) + company = sgqlc.types.Field(sgqlc.types.non_null(Company), graphql_name="company") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + customer = sgqlc.types.Field(sgqlc.types.non_null("Customer"), graphql_name="customer") + draft_orders = sgqlc.types.Field( + sgqlc.types.non_null(DraftOrderConnection), + graphql_name="draftOrders", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(DraftOrderSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + is_main_contact = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isMainContact") + lifetime_duration = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="lifetimeDuration") + locale = sgqlc.types.Field(String, graphql_name="locale") + orders = sgqlc.types.Field( + sgqlc.types.non_null(OrderConnection), + graphql_name="orders", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(OrderSortKeys, graphql_name="sortKey", default="ID")), + ) + ), + ) + role_assignments = sgqlc.types.Field( + sgqlc.types.non_null(CompanyContactRoleAssignmentConnection), + graphql_name="roleAssignments", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(CompanyContactRoleAssignmentSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + title = sgqlc.types.Field(String, graphql_name="title") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + + +class CompanyContactRole(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("name", "note") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + note = sgqlc.types.Field(String, graphql_name="note") + + +class CompanyContactRoleAssignment(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("company", "company_contact", "company_location", "created_at", "role", "updated_at") + company = sgqlc.types.Field(sgqlc.types.non_null(Company), graphql_name="company") + company_contact = sgqlc.types.Field(sgqlc.types.non_null(CompanyContact), graphql_name="companyContact") + company_location = sgqlc.types.Field(sgqlc.types.non_null("CompanyLocation"), graphql_name="companyLocation") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + role = sgqlc.types.Field(sgqlc.types.non_null(CompanyContactRole), graphql_name="role") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + + +class CompanyLocation(sgqlc.types.Type, CommentEventSubject, HasEvents, HasMetafieldDefinitions, HasMetafields, Navigable, Node): + __schema__ = shopify_schema + __field_names__ = ( + "billing_address", + "buyer_experience_configuration", + "catalogs", + "company", + "created_at", + "currency", + "draft_orders", + "external_id", + "in_catalog", + "locale", + "market", + "name", + "note", + "order_count", + "orders", + "phone", + "role_assignments", + "shipping_address", + "tax_exemptions", + "tax_registration_id", + "total_spent", + "updated_at", + ) + billing_address = sgqlc.types.Field(CompanyAddress, graphql_name="billingAddress") + buyer_experience_configuration = sgqlc.types.Field(BuyerExperienceConfiguration, graphql_name="buyerExperienceConfiguration") + catalogs = sgqlc.types.Field( + sgqlc.types.non_null(CatalogConnection), + graphql_name="catalogs", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + company = sgqlc.types.Field(sgqlc.types.non_null(Company), graphql_name="company") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + currency = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currency") + draft_orders = sgqlc.types.Field( + sgqlc.types.non_null(DraftOrderConnection), + graphql_name="draftOrders", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(DraftOrderSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + external_id = sgqlc.types.Field(String, graphql_name="externalId") + in_catalog = sgqlc.types.Field( + sgqlc.types.non_null(Boolean), + graphql_name="inCatalog", + args=sgqlc.types.ArgDict((("catalog_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="catalogId", default=None)),)), + ) + locale = sgqlc.types.Field(String, graphql_name="locale") + market = sgqlc.types.Field(sgqlc.types.non_null("Market"), graphql_name="market") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + note = sgqlc.types.Field(String, graphql_name="note") + order_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="orderCount") + orders = sgqlc.types.Field( + sgqlc.types.non_null(OrderConnection), + graphql_name="orders", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(OrderSortKeys, graphql_name="sortKey", default="ID")), + ) + ), + ) + phone = sgqlc.types.Field(String, graphql_name="phone") + role_assignments = sgqlc.types.Field( + sgqlc.types.non_null(CompanyContactRoleAssignmentConnection), + graphql_name="roleAssignments", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(CompanyContactRoleAssignmentSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + shipping_address = sgqlc.types.Field(CompanyAddress, graphql_name="shippingAddress") + tax_exemptions = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxExemption))), graphql_name="taxExemptions" + ) + tax_registration_id = sgqlc.types.Field(String, graphql_name="taxRegistrationId") + total_spent = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="totalSpent") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + + +class CompanyLocationCatalog(sgqlc.types.Type, Catalog, Node): + __schema__ = shopify_schema + __field_names__ = ("company_locations", "company_locations_count") + company_locations = sgqlc.types.Field( + sgqlc.types.non_null(CompanyLocationConnection), + graphql_name="companyLocations", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(CompanyLocationSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + company_locations_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="companyLocationsCount") + + +class Customer(sgqlc.types.Type, CommentEventSubject, HasEvents, HasMetafieldDefinitions, HasMetafields, LegacyInteroperability, Node): + __schema__ = shopify_schema + __field_names__ = ( + "addresses", + "amount_spent", + "can_delete", + "company_contact_profiles", + "created_at", + "default_address", + "display_name", + "email", + "email_marketing_consent", + "first_name", + "image", + "last_name", + "last_order", + "lifetime_duration", + "locale", + "market", + "mergeable", + "multipass_identifier", + "note", + "number_of_orders", + "orders", + "payment_methods", + "phone", + "product_subscriber_status", + "sms_marketing_consent", + "state", + "statistics", + "subscription_contracts", + "tags", + "tax_exempt", + "tax_exemptions", + "unsubscribe_url", + "updated_at", + "valid_email_address", + "verified_email", + ) + addresses = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MailingAddress"))), + graphql_name="addresses", + args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)),)), + ) + amount_spent = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="amountSpent") + can_delete = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="canDelete") + company_contact_profiles = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CompanyContact))), graphql_name="companyContactProfiles" + ) + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + default_address = sgqlc.types.Field("MailingAddress", graphql_name="defaultAddress") + display_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="displayName") + email = sgqlc.types.Field(String, graphql_name="email") + email_marketing_consent = sgqlc.types.Field(CustomerEmailMarketingConsentState, graphql_name="emailMarketingConsent") + first_name = sgqlc.types.Field(String, graphql_name="firstName") + image = sgqlc.types.Field(sgqlc.types.non_null("Image"), graphql_name="image") + last_name = sgqlc.types.Field(String, graphql_name="lastName") + last_order = sgqlc.types.Field("Order", graphql_name="lastOrder") + lifetime_duration = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="lifetimeDuration") + locale = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="locale") + market = sgqlc.types.Field("Market", graphql_name="market") + mergeable = sgqlc.types.Field(sgqlc.types.non_null(CustomerMergeable), graphql_name="mergeable") + multipass_identifier = sgqlc.types.Field(String, graphql_name="multipassIdentifier") + note = sgqlc.types.Field(String, graphql_name="note") + number_of_orders = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="numberOfOrders") + orders = sgqlc.types.Field( + sgqlc.types.non_null(OrderConnection), + graphql_name="orders", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(OrderSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + payment_methods = sgqlc.types.Field( + sgqlc.types.non_null(CustomerPaymentMethodConnection), + graphql_name="paymentMethods", + args=sgqlc.types.ArgDict( + ( + ("show_revoked", sgqlc.types.Arg(Boolean, graphql_name="showRevoked", default=False)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + phone = sgqlc.types.Field(String, graphql_name="phone") + product_subscriber_status = sgqlc.types.Field( + sgqlc.types.non_null(CustomerProductSubscriberStatus), graphql_name="productSubscriberStatus" + ) + sms_marketing_consent = sgqlc.types.Field(CustomerSmsMarketingConsentState, graphql_name="smsMarketingConsent") + state = sgqlc.types.Field(sgqlc.types.non_null(CustomerState), graphql_name="state") + statistics = sgqlc.types.Field(sgqlc.types.non_null(CustomerStatistics), graphql_name="statistics") + subscription_contracts = sgqlc.types.Field( + sgqlc.types.non_null(SubscriptionContractConnection), + graphql_name="subscriptionContracts", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + tags = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="tags") + tax_exempt = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxExempt") + tax_exemptions = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxExemption))), graphql_name="taxExemptions" + ) + unsubscribe_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="unsubscribeUrl") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + valid_email_address = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="validEmailAddress") + verified_email = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="verifiedEmail") + + +class CustomerEmailMarketingConsentUpdateUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(CustomerEmailMarketingConsentUpdateUserErrorCode, graphql_name="code") + + +class CustomerMergeUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(CustomerMergeErrorCode, graphql_name="code") + + +class CustomerPaymentMethod(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("customer", "instrument", "revoked_at", "revoked_reason", "subscription_contracts") + customer = sgqlc.types.Field(Customer, graphql_name="customer") + instrument = sgqlc.types.Field("CustomerPaymentInstrument", graphql_name="instrument") + revoked_at = sgqlc.types.Field(DateTime, graphql_name="revokedAt") + revoked_reason = sgqlc.types.Field(CustomerPaymentMethodRevocationReason, graphql_name="revokedReason") + subscription_contracts = sgqlc.types.Field( + sgqlc.types.non_null(SubscriptionContractConnection), + graphql_name="subscriptionContracts", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + + +class CustomerPaymentMethodCreateFromDuplicationDataUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(CustomerPaymentMethodCreateFromDuplicationDataUserErrorCode, graphql_name="code") + + +class CustomerPaymentMethodGetDuplicationDataUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(CustomerPaymentMethodGetDuplicationDataUserErrorCode, graphql_name="code") + + +class CustomerPaymentMethodGetUpdateUrlUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(CustomerPaymentMethodGetUpdateUrlUserErrorCode, graphql_name="code") + + +class CustomerPaymentMethodRemoteUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(CustomerPaymentMethodRemoteUserErrorCode, graphql_name="code") + + +class CustomerPaymentMethodUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(CustomerPaymentMethodUserErrorCode, graphql_name="code") + + +class CustomerSegmentMember(sgqlc.types.Type, HasMetafields): + __schema__ = shopify_schema + __field_names__ = ( + "amount_spent", + "default_address", + "default_email_address", + "default_phone_number", + "display_name", + "first_name", + "id", + "last_name", + "last_order_id", + "mergeable", + "note", + "number_of_orders", + ) + amount_spent = sgqlc.types.Field(MoneyV2, graphql_name="amountSpent") + default_address = sgqlc.types.Field("MailingAddress", graphql_name="defaultAddress") + default_email_address = sgqlc.types.Field(CustomerEmailAddress, graphql_name="defaultEmailAddress") + default_phone_number = sgqlc.types.Field(CustomerPhoneNumber, graphql_name="defaultPhoneNumber") + display_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="displayName") + first_name = sgqlc.types.Field(String, graphql_name="firstName") + id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") + last_name = sgqlc.types.Field(String, graphql_name="lastName") + last_order_id = sgqlc.types.Field(ID, graphql_name="lastOrderId") + mergeable = sgqlc.types.Field(sgqlc.types.non_null(CustomerMergeable), graphql_name="mergeable") + note = sgqlc.types.Field(String, graphql_name="note") + number_of_orders = sgqlc.types.Field(UnsignedInt64, graphql_name="numberOfOrders") + + +class CustomerSegmentMembersQuery(sgqlc.types.Type, JobResult, Node): + __schema__ = shopify_schema + __field_names__ = ("current_count",) + current_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="currentCount") + + +class CustomerSegmentMembersQueryUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(CustomerSegmentMembersQueryUserErrorCode, graphql_name="code") + + +class CustomerSmsMarketingConsentError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(CustomerSmsMarketingConsentErrorCode, graphql_name="code") + + +class CustomerVisit(sgqlc.types.Type, CustomerMoment, Node): + __schema__ = shopify_schema + __field_names__ = ( + "landing_page", + "landing_page_html", + "marketing_event", + "referral_code", + "referral_info_html", + "referrer_url", + "source", + "source_description", + "source_type", + "utm_parameters", + ) + landing_page = sgqlc.types.Field(URL, graphql_name="landingPage") + landing_page_html = sgqlc.types.Field(HTML, graphql_name="landingPageHtml") + marketing_event = sgqlc.types.Field("MarketingEvent", graphql_name="marketingEvent") + referral_code = sgqlc.types.Field(String, graphql_name="referralCode") + referral_info_html = sgqlc.types.Field(sgqlc.types.non_null(FormattedString), graphql_name="referralInfoHtml") + referrer_url = sgqlc.types.Field(URL, graphql_name="referrerUrl") + source = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="source") + source_description = sgqlc.types.Field(String, graphql_name="sourceDescription") + source_type = sgqlc.types.Field(MarketingTactic, graphql_name="sourceType") + utm_parameters = sgqlc.types.Field(UTMParameters, graphql_name="utmParameters") + + +class DelegateAccessTokenCreateUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(DelegateAccessTokenCreateUserErrorCode, graphql_name="code") + + +class DelegateAccessTokenDestroyUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(DelegateAccessTokenDestroyUserErrorCode, graphql_name="code") + + +class DeliveryCarrierService(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("available_services_for_countries", "formatted_name", "icon", "name") + available_services_for_countries = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DeliveryAvailableService))), + graphql_name="availableServicesForCountries", + args=sgqlc.types.ArgDict( + ( + ("origins", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="origins", default=None)), + ( + "country_codes", + sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(CountryCode)), graphql_name="countryCodes", default=None), + ), + ("rest_of_world", sgqlc.types.Arg(sgqlc.types.non_null(Boolean), graphql_name="restOfWorld", default=None)), + ) + ), + ) + formatted_name = sgqlc.types.Field(String, graphql_name="formattedName") + icon = sgqlc.types.Field(sgqlc.types.non_null("Image"), graphql_name="icon") + name = sgqlc.types.Field(String, graphql_name="name") + + +class DeliveryCondition(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("condition_criteria", "field", "operator") + condition_criteria = sgqlc.types.Field(sgqlc.types.non_null("DeliveryConditionCriteria"), graphql_name="conditionCriteria") + field = sgqlc.types.Field(sgqlc.types.non_null(DeliveryConditionField), graphql_name="field") + operator = sgqlc.types.Field(sgqlc.types.non_null(DeliveryConditionOperator), graphql_name="operator") + + +class DeliveryCountry(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("code", "name", "provinces", "translated_name") + code = sgqlc.types.Field(sgqlc.types.non_null(DeliveryCountryCodeOrRestOfWorld), graphql_name="code") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + provinces = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryProvince"))), graphql_name="provinces" + ) + translated_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="translatedName") + + +class DeliveryCustomization(sgqlc.types.Type, HasMetafieldDefinitions, HasMetafields, Node): + __schema__ = shopify_schema + __field_names__ = ("enabled", "error_history", "function_id", "shopify_function", "title") + enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="enabled") + error_history = sgqlc.types.Field(FunctionsErrorHistory, graphql_name="errorHistory") + function_id = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="functionId") + shopify_function = sgqlc.types.Field(sgqlc.types.non_null(ShopifyFunction), graphql_name="shopifyFunction") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + + +class DeliveryCustomizationError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(DeliveryCustomizationErrorCode, graphql_name="code") + + +class DeliveryLocationGroup(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("locations", "locations_count") + locations = sgqlc.types.Field( + sgqlc.types.non_null(LocationConnection), + graphql_name="locations", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(LocationSortKeys, graphql_name="sortKey", default="NAME")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("include_legacy", sgqlc.types.Arg(Boolean, graphql_name="includeLegacy", default=False)), + ("include_inactive", sgqlc.types.Arg(Boolean, graphql_name="includeInactive", default=False)), + ) + ), + ) + locations_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="locationsCount") + + +class DeliveryLocationLocalPickupSettingsError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(DeliveryLocationLocalPickupSettingsErrorCode, graphql_name="code") + + +class DeliveryMethod(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("branded_promise", "max_delivery_date_time", "method_type", "min_delivery_date_time") + branded_promise = sgqlc.types.Field(DeliveryBrandedPromise, graphql_name="brandedPromise") + max_delivery_date_time = sgqlc.types.Field(DateTime, graphql_name="maxDeliveryDateTime") + method_type = sgqlc.types.Field(sgqlc.types.non_null(DeliveryMethodType), graphql_name="methodType") + min_delivery_date_time = sgqlc.types.Field(DateTime, graphql_name="minDeliveryDateTime") + + +class DeliveryMethodDefinition(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("active", "description", "method_conditions", "name", "rate_provider") + active = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="active") + description = sgqlc.types.Field(String, graphql_name="description") + method_conditions = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DeliveryCondition))), graphql_name="methodConditions" + ) + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + rate_provider = sgqlc.types.Field(sgqlc.types.non_null("DeliveryRateProvider"), graphql_name="rateProvider") + + +class DeliveryParticipant(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("adapt_to_new_services_flag", "carrier_service", "fixed_fee", "participant_services", "percentage_of_rate_fee") + adapt_to_new_services_flag = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="adaptToNewServicesFlag") + carrier_service = sgqlc.types.Field(sgqlc.types.non_null(DeliveryCarrierService), graphql_name="carrierService") + fixed_fee = sgqlc.types.Field(MoneyV2, graphql_name="fixedFee") + participant_services = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DeliveryParticipantService))), graphql_name="participantServices" + ) + percentage_of_rate_fee = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="percentageOfRateFee") + + +class DeliveryProfile(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "active_method_definitions_count", + "default", + "legacy_mode", + "locations_without_rates_count", + "name", + "origin_location_count", + "product_variants_count_v2", + "profile_items", + "profile_location_groups", + "selling_plan_groups", + "unassigned_locations", + "unassigned_locations_paginated", + "zone_country_count", + ) + active_method_definitions_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="activeMethodDefinitionsCount") + default = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="default") + legacy_mode = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="legacyMode") + locations_without_rates_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="locationsWithoutRatesCount") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + origin_location_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="originLocationCount") + product_variants_count_v2 = sgqlc.types.Field(sgqlc.types.non_null(DeliveryProductVariantsCount), graphql_name="productVariantsCountV2") + profile_items = sgqlc.types.Field( + sgqlc.types.non_null(DeliveryProfileItemConnection), + graphql_name="profileItems", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + profile_location_groups = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DeliveryProfileLocationGroup))), + graphql_name="profileLocationGroups", + args=sgqlc.types.ArgDict((("location_group_id", sgqlc.types.Arg(ID, graphql_name="locationGroupId", default=None)),)), + ) + selling_plan_groups = sgqlc.types.Field( + sgqlc.types.non_null(SellingPlanGroupConnection), + graphql_name="sellingPlanGroups", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + unassigned_locations = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Location"))), graphql_name="unassignedLocations" + ) + unassigned_locations_paginated = sgqlc.types.Field( + sgqlc.types.non_null(LocationConnection), + graphql_name="unassignedLocationsPaginated", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + zone_country_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="zoneCountryCount") + + +class DeliveryProfileItem(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("product", "variants") + product = sgqlc.types.Field(sgqlc.types.non_null("Product"), graphql_name="product") + variants = sgqlc.types.Field( + sgqlc.types.non_null(ProductVariantConnection), + graphql_name="variants", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + + +class DeliveryProvince(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("code", "name", "translated_name") + code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + translated_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="translatedName") + + +class DeliveryRateDefinition(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("price",) + price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="price") + + +class DeliveryZone(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("countries", "name") + countries = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DeliveryCountry))), graphql_name="countries" + ) + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + + +class DiscountAutomaticBxgy(sgqlc.types.Type, HasEvents, Node): + __schema__ = shopify_schema + __field_names__ = ( + "async_usage_count", + "combines_with", + "created_at", + "customer_buys", + "customer_gets", + "discount_class", + "ends_at", + "starts_at", + "status", + "summary", + "title", + "updated_at", + "uses_per_order_limit", + ) + async_usage_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="asyncUsageCount") + combines_with = sgqlc.types.Field(sgqlc.types.non_null(DiscountCombinesWith), graphql_name="combinesWith") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + customer_buys = sgqlc.types.Field(sgqlc.types.non_null(DiscountCustomerBuys), graphql_name="customerBuys") + customer_gets = sgqlc.types.Field(sgqlc.types.non_null(DiscountCustomerGets), graphql_name="customerGets") + discount_class = sgqlc.types.Field(sgqlc.types.non_null(MerchandiseDiscountClass), graphql_name="discountClass") + ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") + starts_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="startsAt") + status = sgqlc.types.Field(sgqlc.types.non_null(DiscountStatus), graphql_name="status") + summary = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="summary") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + uses_per_order_limit = sgqlc.types.Field(Int, graphql_name="usesPerOrderLimit") + + +class DiscountAutomaticNode(sgqlc.types.Type, HasEvents, HasMetafieldDefinitions, HasMetafields, Node): + __schema__ = shopify_schema + __field_names__ = ("automatic_discount",) + automatic_discount = sgqlc.types.Field(sgqlc.types.non_null("DiscountAutomatic"), graphql_name="automaticDiscount") + + +class DiscountCodeApplication(sgqlc.types.Type, DiscountApplication): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") + + +class DiscountCodeNode(sgqlc.types.Type, HasEvents, HasMetafieldDefinitions, HasMetafields, Node): + __schema__ = shopify_schema + __field_names__ = ("code_discount",) + code_discount = sgqlc.types.Field(sgqlc.types.non_null("DiscountCode"), graphql_name="codeDiscount") + + +class DiscountNode(sgqlc.types.Type, HasEvents, HasMetafieldDefinitions, HasMetafields, Node): + __schema__ = shopify_schema + __field_names__ = ("discount",) + discount = sgqlc.types.Field(sgqlc.types.non_null("Discount"), graphql_name="discount") + + +class DiscountRedeemCodeBulkCreation(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("codes", "codes_count", "created_at", "discount_code", "done", "failed_count", "imported_count") + codes = sgqlc.types.Field( + sgqlc.types.non_null(DiscountRedeemCodeBulkCreationCodeConnection), + graphql_name="codes", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + codes_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="codesCount") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + discount_code = sgqlc.types.Field(DiscountCodeNode, graphql_name="discountCode") + done = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="done") + failed_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="failedCount") + imported_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="importedCount") + + +class DiscountUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code", "extra_info") + code = sgqlc.types.Field(DiscountErrorCode, graphql_name="code") + extra_info = sgqlc.types.Field(String, graphql_name="extraInfo") + + +class DisputeEvidenceUpdateUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(DisputeEvidenceUpdateUserErrorCode, graphql_name="code") + + +class Domain(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("host", "localization", "market_web_presence", "ssl_enabled", "url") + host = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="host") + localization = sgqlc.types.Field(DomainLocalization, graphql_name="localization") + market_web_presence = sgqlc.types.Field("MarketWebPresence", graphql_name="marketWebPresence") + ssl_enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="sslEnabled") + url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") + + +class DraftOrder( + sgqlc.types.Type, CommentEventSubject, HasEvents, HasLocalizationExtensions, HasMetafields, LegacyInteroperability, Navigable, Node +): + __schema__ = shopify_schema + __field_names__ = ( + "applied_discount", + "billing_address", + "billing_address_matches_shipping_address", + "completed_at", + "created_at", + "currency_code", + "custom_attributes", + "customer", + "email", + "invoice_email_template_subject", + "invoice_sent_at", + "invoice_url", + "line_items", + "line_items_subtotal_price", + "market_name", + "market_region_country_code", + "name", + "note2", + "order", + "payment_terms", + "phone", + "po_number", + "presentment_currency_code", + "purchasing_entity", + "ready", + "reserve_inventory_until", + "shipping_address", + "shipping_line", + "status", + "subtotal_price", + "subtotal_price_set", + "tags", + "tax_exempt", + "tax_lines", + "taxes_included", + "total_discounts_set", + "total_line_items_price_set", + "total_price", + "total_price_set", + "total_shipping_price", + "total_shipping_price_set", + "total_tax", + "total_tax_set", + "total_weight", + "updated_at", + "visible_to_customer", + ) + applied_discount = sgqlc.types.Field(DraftOrderAppliedDiscount, graphql_name="appliedDiscount") + billing_address = sgqlc.types.Field("MailingAddress", graphql_name="billingAddress") + billing_address_matches_shipping_address = sgqlc.types.Field( + sgqlc.types.non_null(Boolean), graphql_name="billingAddressMatchesShippingAddress" + ) + completed_at = sgqlc.types.Field(DateTime, graphql_name="completedAt") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currencyCode") + custom_attributes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Attribute))), graphql_name="customAttributes" + ) + customer = sgqlc.types.Field(Customer, graphql_name="customer") + email = sgqlc.types.Field(String, graphql_name="email") + invoice_email_template_subject = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="invoiceEmailTemplateSubject") + invoice_sent_at = sgqlc.types.Field(DateTime, graphql_name="invoiceSentAt") + invoice_url = sgqlc.types.Field(URL, graphql_name="invoiceUrl") + line_items = sgqlc.types.Field( + sgqlc.types.non_null(DraftOrderLineItemConnection), + graphql_name="lineItems", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + line_items_subtotal_price = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="lineItemsSubtotalPrice") + market_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="marketName") + market_region_country_code = sgqlc.types.Field(sgqlc.types.non_null(CountryCode), graphql_name="marketRegionCountryCode") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + note2 = sgqlc.types.Field(String, graphql_name="note2") + order = sgqlc.types.Field("Order", graphql_name="order") + payment_terms = sgqlc.types.Field("PaymentTerms", graphql_name="paymentTerms") + phone = sgqlc.types.Field(String, graphql_name="phone") + po_number = sgqlc.types.Field(String, graphql_name="poNumber") + presentment_currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="presentmentCurrencyCode") + purchasing_entity = sgqlc.types.Field("PurchasingEntity", graphql_name="purchasingEntity") + ready = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="ready") + reserve_inventory_until = sgqlc.types.Field(DateTime, graphql_name="reserveInventoryUntil") + shipping_address = sgqlc.types.Field("MailingAddress", graphql_name="shippingAddress") + shipping_line = sgqlc.types.Field(ShippingLine, graphql_name="shippingLine") + status = sgqlc.types.Field(sgqlc.types.non_null(DraftOrderStatus), graphql_name="status") + subtotal_price = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="subtotalPrice") + subtotal_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="subtotalPriceSet") + tags = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="tags") + tax_exempt = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxExempt") + tax_lines = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxLine))), graphql_name="taxLines") + taxes_included = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxesIncluded") + total_discounts_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalDiscountsSet") + total_line_items_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalLineItemsPriceSet") + total_price = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="totalPrice") + total_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalPriceSet") + total_shipping_price = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="totalShippingPrice") + total_shipping_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalShippingPriceSet") + total_tax = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="totalTax") + total_tax_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalTaxSet") + total_weight = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="totalWeight") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + visible_to_customer = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="visibleToCustomer") + + +class DraftOrderLineItem(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "applied_discount", + "custom", + "custom_attributes", + "custom_attributes_v2", + "discounted_total", + "discounted_total_set", + "discounted_unit_price", + "discounted_unit_price_set", + "fulfillment_service", + "image", + "is_gift_card", + "name", + "original_total", + "original_total_set", + "original_unit_price", + "original_unit_price_set", + "product", + "quantity", + "requires_shipping", + "sku", + "tax_lines", + "taxable", + "title", + "total_discount", + "total_discount_set", + "variant", + "variant_title", + "vendor", + "weight", + ) + applied_discount = sgqlc.types.Field(DraftOrderAppliedDiscount, graphql_name="appliedDiscount") + custom = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="custom") + custom_attributes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Attribute))), graphql_name="customAttributes" + ) + custom_attributes_v2 = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TypedAttribute))), graphql_name="customAttributesV2" + ) + discounted_total = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="discountedTotal") + discounted_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="discountedTotalSet") + discounted_unit_price = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="discountedUnitPrice") + discounted_unit_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="discountedUnitPriceSet") + fulfillment_service = sgqlc.types.Field(FulfillmentService, graphql_name="fulfillmentService") + image = sgqlc.types.Field("Image", graphql_name="image") + is_gift_card = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isGiftCard") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + original_total = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="originalTotal") + original_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="originalTotalSet") + original_unit_price = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="originalUnitPrice") + original_unit_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="originalUnitPriceSet") + product = sgqlc.types.Field("Product", graphql_name="product") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + requires_shipping = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresShipping") + sku = sgqlc.types.Field(String, graphql_name="sku") + tax_lines = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxLine))), graphql_name="taxLines") + taxable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxable") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + total_discount = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="totalDiscount") + total_discount_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalDiscountSet") + variant = sgqlc.types.Field("ProductVariant", graphql_name="variant") + variant_title = sgqlc.types.Field(String, graphql_name="variantTitle") + vendor = sgqlc.types.Field(String, graphql_name="vendor") + weight = sgqlc.types.Field(Weight, graphql_name="weight") + + +class DraftOrderTag(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("handle", "title") + handle = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="handle") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + + +class Duty(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("country_code_of_origin", "harmonized_system_code", "price", "tax_lines") + country_code_of_origin = sgqlc.types.Field(CountryCode, graphql_name="countryCodeOfOrigin") + harmonized_system_code = sgqlc.types.Field(String, graphql_name="harmonizedSystemCode") + price = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="price") + tax_lines = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxLine))), graphql_name="taxLines") + + +class DutySale(sgqlc.types.Type, Sale): + __schema__ = shopify_schema + __field_names__ = ("duty",) + duty = sgqlc.types.Field(sgqlc.types.non_null(Duty), graphql_name="duty") + + +class ErrorsServerPixelUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(ErrorsServerPixelUserErrorCode, graphql_name="code") + + +class ErrorsWebPixelUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(ErrorsWebPixelUserErrorCode, graphql_name="code") + + +class ExchangeV2(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "additions", + "completed_at", + "created_at", + "location", + "note", + "refunds", + "returns", + "staff_member", + "total_amount_processed_set", + "total_price_set", + "transactions", + ) + additions = sgqlc.types.Field(sgqlc.types.non_null(ExchangeV2Additions), graphql_name="additions") + completed_at = sgqlc.types.Field(DateTime, graphql_name="completedAt") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + location = sgqlc.types.Field("Location", graphql_name="location") + note = sgqlc.types.Field(String, graphql_name="note") + refunds = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Refund"))), graphql_name="refunds") + returns = sgqlc.types.Field(sgqlc.types.non_null(ExchangeV2Returns), graphql_name="returns") + staff_member = sgqlc.types.Field("StaffMember", graphql_name="staffMember") + total_amount_processed_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalAmountProcessedSet") + total_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalPriceSet") + transactions = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("OrderTransaction"))), graphql_name="transactions" + ) + + +class ExternalVideo(sgqlc.types.Type, Media, Node): + __schema__ = shopify_schema + __field_names__ = ("embed_url", "host", "origin_url") + embed_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="embedUrl") + host = sgqlc.types.Field(sgqlc.types.non_null(MediaHost), graphql_name="host") + origin_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="originUrl") + + +class FilesUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(FilesErrorCode, graphql_name="code") + + +class Fulfillment(sgqlc.types.Type, LegacyInteroperability, Node): + __schema__ = shopify_schema + __field_names__ = ( + "created_at", + "delivered_at", + "display_status", + "estimated_delivery_at", + "events", + "fulfillment_line_items", + "fulfillment_orders", + "in_transit_at", + "location", + "name", + "order", + "origin_address", + "requires_shipping", + "service", + "status", + "total_quantity", + "tracking_info", + "updated_at", + ) + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + delivered_at = sgqlc.types.Field(DateTime, graphql_name="deliveredAt") + display_status = sgqlc.types.Field(FulfillmentDisplayStatus, graphql_name="displayStatus") + estimated_delivery_at = sgqlc.types.Field(DateTime, graphql_name="estimatedDeliveryAt") + events = sgqlc.types.Field( + sgqlc.types.non_null(FulfillmentEventConnection), + graphql_name="events", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(FulfillmentEventSortKeys, graphql_name="sortKey", default="HAPPENED_AT")), + ) + ), + ) + fulfillment_line_items = sgqlc.types.Field( + sgqlc.types.non_null(FulfillmentLineItemConnection), + graphql_name="fulfillmentLineItems", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + fulfillment_orders = sgqlc.types.Field( + sgqlc.types.non_null(FulfillmentOrderConnection), + graphql_name="fulfillmentOrders", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + in_transit_at = sgqlc.types.Field(DateTime, graphql_name="inTransitAt") + location = sgqlc.types.Field("Location", graphql_name="location") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + order = sgqlc.types.Field(sgqlc.types.non_null("Order"), graphql_name="order") + origin_address = sgqlc.types.Field(FulfillmentOriginAddress, graphql_name="originAddress") + requires_shipping = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresShipping") + service = sgqlc.types.Field(FulfillmentService, graphql_name="service") + status = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentStatus), graphql_name="status") + total_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalQuantity") + tracking_info = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FulfillmentTrackingInfo))), + graphql_name="trackingInfo", + args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)),)), + ) + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + + +class FulfillmentEvent(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "address1", + "city", + "country", + "estimated_delivery_at", + "happened_at", + "latitude", + "longitude", + "message", + "province", + "status", + "zip", + ) + address1 = sgqlc.types.Field(String, graphql_name="address1") + city = sgqlc.types.Field(String, graphql_name="city") + country = sgqlc.types.Field(String, graphql_name="country") + estimated_delivery_at = sgqlc.types.Field(DateTime, graphql_name="estimatedDeliveryAt") + happened_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="happenedAt") + latitude = sgqlc.types.Field(Float, graphql_name="latitude") + longitude = sgqlc.types.Field(Float, graphql_name="longitude") + message = sgqlc.types.Field(String, graphql_name="message") + province = sgqlc.types.Field(String, graphql_name="province") + status = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentEventStatus), graphql_name="status") + zip = sgqlc.types.Field(String, graphql_name="zip") + + +class FulfillmentLineItem(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("discounted_total_set", "line_item", "original_total_set", "quantity") + discounted_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="discountedTotalSet") + line_item = sgqlc.types.Field(sgqlc.types.non_null("LineItem"), graphql_name="lineItem") + original_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="originalTotalSet") + quantity = sgqlc.types.Field(Int, graphql_name="quantity") + + +class FulfillmentOrder(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "assigned_location", + "created_at", + "delivery_method", + "destination", + "fulfill_at", + "fulfill_by", + "fulfillment_holds", + "fulfillments", + "international_duties", + "line_items", + "locations_for_move", + "merchant_requests", + "order", + "request_status", + "status", + "supported_actions", + "updated_at", + ) + assigned_location = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentOrderAssignedLocation), graphql_name="assignedLocation") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + delivery_method = sgqlc.types.Field(DeliveryMethod, graphql_name="deliveryMethod") + destination = sgqlc.types.Field("FulfillmentOrderDestination", graphql_name="destination") + fulfill_at = sgqlc.types.Field(DateTime, graphql_name="fulfillAt") + fulfill_by = sgqlc.types.Field(DateTime, graphql_name="fulfillBy") + fulfillment_holds = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FulfillmentHold))), graphql_name="fulfillmentHolds" + ) + fulfillments = sgqlc.types.Field( + sgqlc.types.non_null(FulfillmentConnection), + graphql_name="fulfillments", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + international_duties = sgqlc.types.Field(FulfillmentOrderInternationalDuties, graphql_name="internationalDuties") + line_items = sgqlc.types.Field( + sgqlc.types.non_null(FulfillmentOrderLineItemConnection), + graphql_name="lineItems", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + locations_for_move = sgqlc.types.Field( + sgqlc.types.non_null(FulfillmentOrderLocationForMoveConnection), + graphql_name="locationsForMove", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + merchant_requests = sgqlc.types.Field( + sgqlc.types.non_null(FulfillmentOrderMerchantRequestConnection), + graphql_name="merchantRequests", + args=sgqlc.types.ArgDict( + ( + ("kind", sgqlc.types.Arg(FulfillmentOrderMerchantRequestKind, graphql_name="kind", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + order = sgqlc.types.Field(sgqlc.types.non_null("Order"), graphql_name="order") + request_status = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentOrderRequestStatus), graphql_name="requestStatus") + status = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentOrderStatus), graphql_name="status") + supported_actions = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FulfillmentOrderSupportedAction))), graphql_name="supportedActions" + ) + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + + +class FulfillmentOrderDestination(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "address1", + "address2", + "city", + "company", + "country_code", + "email", + "first_name", + "last_name", + "phone", + "province", + "zip", + ) + address1 = sgqlc.types.Field(String, graphql_name="address1") + address2 = sgqlc.types.Field(String, graphql_name="address2") + city = sgqlc.types.Field(String, graphql_name="city") + company = sgqlc.types.Field(String, graphql_name="company") + country_code = sgqlc.types.Field(CountryCode, graphql_name="countryCode") + email = sgqlc.types.Field(String, graphql_name="email") + first_name = sgqlc.types.Field(String, graphql_name="firstName") + last_name = sgqlc.types.Field(String, graphql_name="lastName") + phone = sgqlc.types.Field(String, graphql_name="phone") + province = sgqlc.types.Field(String, graphql_name="province") + zip = sgqlc.types.Field(String, graphql_name="zip") + + +class FulfillmentOrderHoldUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(FulfillmentOrderHoldUserErrorCode, graphql_name="code") + + +class FulfillmentOrderLineItem(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "image", + "inventory_item_id", + "original_unit_price_set", + "product_title", + "remaining_quantity", + "requires_shipping", + "sku", + "total_quantity", + "variant_title", + "vendor", + "warnings", + "weight", + ) + image = sgqlc.types.Field("Image", graphql_name="image") + inventory_item_id = sgqlc.types.Field(ID, graphql_name="inventoryItemId") + original_unit_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="originalUnitPriceSet") + product_title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="productTitle") + remaining_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="remainingQuantity") + requires_shipping = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresShipping") + sku = sgqlc.types.Field(String, graphql_name="sku") + total_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalQuantity") + variant_title = sgqlc.types.Field(String, graphql_name="variantTitle") + vendor = sgqlc.types.Field(String, graphql_name="vendor") + warnings = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FulfillmentOrderLineItemWarning))), graphql_name="warnings" + ) + weight = sgqlc.types.Field(Weight, graphql_name="weight") + + +class FulfillmentOrderLineItemsPreparedForPickupUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(FulfillmentOrderLineItemsPreparedForPickupUserErrorCode, graphql_name="code") + + +class FulfillmentOrderMerchantRequest(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("fulfillment_order", "kind", "message", "request_options", "response_data", "sent_at") + fulfillment_order = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentOrder), graphql_name="fulfillmentOrder") + kind = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentOrderMerchantRequestKind), graphql_name="kind") + message = sgqlc.types.Field(String, graphql_name="message") + request_options = sgqlc.types.Field(JSON, graphql_name="requestOptions") + response_data = sgqlc.types.Field(JSON, graphql_name="responseData") + sent_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="sentAt") + + +class FulfillmentOrderMergeUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(FulfillmentOrderMergeUserErrorCode, graphql_name="code") + + +class FulfillmentOrderReleaseHoldUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(FulfillmentOrderReleaseHoldUserErrorCode, graphql_name="code") + + +class FulfillmentOrderRescheduleUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(FulfillmentOrderRescheduleUserErrorCode, graphql_name="code") + + +class FulfillmentOrderSplitUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(FulfillmentOrderSplitUserErrorCode, graphql_name="code") + + +class FulfillmentOrdersReleaseHoldsUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(FulfillmentOrdersReleaseHoldsUserErrorCode, graphql_name="code") + + +class FulfillmentOrdersSetFulfillmentDeadlineUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(FulfillmentOrdersSetFulfillmentDeadlineUserErrorCode, graphql_name="code") + + +class GenericFile(sgqlc.types.Type, File, Node): + __schema__ = shopify_schema + __field_names__ = ("mime_type", "original_file_size", "url") + mime_type = sgqlc.types.Field(String, graphql_name="mimeType") + original_file_size = sgqlc.types.Field(Int, graphql_name="originalFileSize") + url = sgqlc.types.Field(URL, graphql_name="url") + + +class GiftCard(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "balance", + "created_at", + "customer", + "disabled_at", + "enabled", + "expires_on", + "initial_value", + "last_characters", + "masked_code", + "note", + "order", + ) + balance = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="balance") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + customer = sgqlc.types.Field(Customer, graphql_name="customer") + disabled_at = sgqlc.types.Field(DateTime, graphql_name="disabledAt") + enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="enabled") + expires_on = sgqlc.types.Field(Date, graphql_name="expiresOn") + initial_value = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="initialValue") + last_characters = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="lastCharacters") + masked_code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="maskedCode") + note = sgqlc.types.Field(String, graphql_name="note") + order = sgqlc.types.Field("Order", graphql_name="order") + + +class GiftCardSale(sgqlc.types.Type, Sale): + __schema__ = shopify_schema + __field_names__ = ("line_item",) + line_item = sgqlc.types.Field(sgqlc.types.non_null("LineItem"), graphql_name="lineItem") + + +class GiftCardUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(GiftCardErrorCode, graphql_name="code") + + +class Image(sgqlc.types.Type, HasMetafields): + __schema__ = shopify_schema + __field_names__ = ("alt_text", "height", "id", "url", "width") + alt_text = sgqlc.types.Field(String, graphql_name="altText") + height = sgqlc.types.Field(Int, graphql_name="height") + id = sgqlc.types.Field(ID, graphql_name="id") + url = sgqlc.types.Field( + sgqlc.types.non_null(URL), + graphql_name="url", + args=sgqlc.types.ArgDict((("transform", sgqlc.types.Arg(ImageTransformInput, graphql_name="transform", default=None)),)), + ) + width = sgqlc.types.Field(Int, graphql_name="width") + + +class InventoryAdjustQuantitiesUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(InventoryAdjustQuantitiesUserErrorCode, graphql_name="code") + + +class InventoryAdjustmentGroup(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("app", "changes", "created_at", "reason", "reference_document_uri", "staff_member") + app = sgqlc.types.Field(App, graphql_name="app") + changes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(InventoryChange))), + graphql_name="changes", + args=sgqlc.types.ArgDict( + ( + ( + "inventory_item_ids", + sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="inventoryItemIds", default=None), + ), + ("location_ids", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="locationIds", default=None)), + ( + "quantity_names", + sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="quantityNames", default=None), + ), + ) + ), + ) + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + reason = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="reason") + reference_document_uri = sgqlc.types.Field(String, graphql_name="referenceDocumentUri") + staff_member = sgqlc.types.Field("StaffMember", graphql_name="staffMember") + + +class InventoryBulkToggleActivationUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(InventoryBulkToggleActivationUserErrorCode, graphql_name="code") + + +class InventoryItem(sgqlc.types.Type, LegacyInteroperability, Node): + __schema__ = shopify_schema + __field_names__ = ( + "country_code_of_origin", + "country_harmonized_system_codes", + "created_at", + "duplicate_sku_count", + "harmonized_system_code", + "inventory_history_url", + "inventory_level", + "inventory_levels", + "locations_count", + "province_code_of_origin", + "requires_shipping", + "sku", + "tracked", + "tracked_editable", + "unit_cost", + "updated_at", + "variant", + ) + country_code_of_origin = sgqlc.types.Field(CountryCode, graphql_name="countryCodeOfOrigin") + country_harmonized_system_codes = sgqlc.types.Field( + sgqlc.types.non_null(CountryHarmonizedSystemCodeConnection), + graphql_name="countryHarmonizedSystemCodes", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + duplicate_sku_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="duplicateSkuCount") + harmonized_system_code = sgqlc.types.Field(String, graphql_name="harmonizedSystemCode") + inventory_history_url = sgqlc.types.Field(URL, graphql_name="inventoryHistoryUrl") + inventory_level = sgqlc.types.Field( + "InventoryLevel", + graphql_name="inventoryLevel", + args=sgqlc.types.ArgDict((("location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="locationId", default=None)),)), + ) + inventory_levels = sgqlc.types.Field( + sgqlc.types.non_null(InventoryLevelConnection), + graphql_name="inventoryLevels", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + locations_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="locationsCount") + province_code_of_origin = sgqlc.types.Field(String, graphql_name="provinceCodeOfOrigin") + requires_shipping = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresShipping") + sku = sgqlc.types.Field(String, graphql_name="sku") + tracked = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="tracked") + tracked_editable = sgqlc.types.Field(sgqlc.types.non_null(EditableProperty), graphql_name="trackedEditable") + unit_cost = sgqlc.types.Field(MoneyV2, graphql_name="unitCost") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + variant = sgqlc.types.Field(sgqlc.types.non_null("ProductVariant"), graphql_name="variant") + + +class InventoryLevel(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("can_deactivate", "created_at", "deactivation_alert", "item", "location", "quantities", "updated_at") + can_deactivate = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="canDeactivate") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + deactivation_alert = sgqlc.types.Field(String, graphql_name="deactivationAlert") + item = sgqlc.types.Field(sgqlc.types.non_null(InventoryItem), graphql_name="item") + location = sgqlc.types.Field(sgqlc.types.non_null("Location"), graphql_name="location") + quantities = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(InventoryQuantity))), + graphql_name="quantities", + args=sgqlc.types.ArgDict( + ( + ( + "names", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="names", default=None + ), + ), + ) + ), + ) + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + + +class InventoryMoveQuantitiesUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(InventoryMoveQuantitiesUserErrorCode, graphql_name="code") + + +class InventorySetOnHandQuantitiesUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(InventorySetOnHandQuantitiesUserErrorCode, graphql_name="code") + + +class LineItem(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "contract", + "current_quantity", + "custom_attributes", + "discount_allocations", + "discounted_total_set", + "discounted_unit_price_set", + "duties", + "image", + "line_item_group", + "merchant_editable", + "name", + "non_fulfillable_quantity", + "original_total_set", + "original_unit_price_set", + "product", + "quantity", + "refundable_quantity", + "requires_shipping", + "restockable", + "selling_plan", + "sku", + "staff_member", + "tax_lines", + "taxable", + "title", + "total_discount_set", + "unfulfilled_discounted_total_set", + "unfulfilled_original_total_set", + "unfulfilled_quantity", + "variant", + "variant_title", + "vendor", + ) + contract = sgqlc.types.Field("SubscriptionContract", graphql_name="contract") + current_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="currentQuantity") + custom_attributes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Attribute))), graphql_name="customAttributes" + ) + discount_allocations = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DiscountAllocation))), graphql_name="discountAllocations" + ) + discounted_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="discountedTotalSet") + discounted_unit_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="discountedUnitPriceSet") + duties = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Duty))), graphql_name="duties") + image = sgqlc.types.Field(Image, graphql_name="image") + line_item_group = sgqlc.types.Field(LineItemGroup, graphql_name="lineItemGroup") + merchant_editable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="merchantEditable") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + non_fulfillable_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="nonFulfillableQuantity") + original_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="originalTotalSet") + original_unit_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="originalUnitPriceSet") + product = sgqlc.types.Field("Product", graphql_name="product") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + refundable_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="refundableQuantity") + requires_shipping = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresShipping") + restockable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="restockable") + selling_plan = sgqlc.types.Field(LineItemSellingPlan, graphql_name="sellingPlan") + sku = sgqlc.types.Field(String, graphql_name="sku") + staff_member = sgqlc.types.Field("StaffMember", graphql_name="staffMember") + tax_lines = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxLine))), + graphql_name="taxLines", + args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)),)), + ) + taxable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxable") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + total_discount_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalDiscountSet") + unfulfilled_discounted_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="unfulfilledDiscountedTotalSet") + unfulfilled_original_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="unfulfilledOriginalTotalSet") + unfulfilled_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="unfulfilledQuantity") + variant = sgqlc.types.Field("ProductVariant", graphql_name="variant") + variant_title = sgqlc.types.Field(String, graphql_name="variantTitle") + vendor = sgqlc.types.Field(String, graphql_name="vendor") + + +class LineItemMutable(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "custom_attributes", + "discount_allocations", + "discounted_total_set", + "discounted_unit_price_set", + "fulfillable_quantity", + "fulfillment_service", + "fulfillment_status", + "image", + "merchant_editable", + "name", + "non_fulfillable_quantity", + "original_total_set", + "original_unit_price_set", + "product", + "quantity", + "refundable_quantity", + "requires_shipping", + "restockable", + "sku", + "staff_member", + "tax_lines", + "taxable", + "title", + "total_discount_set", + "unfulfilled_discounted_total_set", + "unfulfilled_original_total_set", + "unfulfilled_quantity", + "variant", + "variant_title", + "vendor", + ) + custom_attributes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Attribute))), graphql_name="customAttributes" + ) + discount_allocations = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DiscountAllocation))), graphql_name="discountAllocations" + ) + discounted_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="discountedTotalSet") + discounted_unit_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="discountedUnitPriceSet") + fulfillable_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="fulfillableQuantity") + fulfillment_service = sgqlc.types.Field(FulfillmentService, graphql_name="fulfillmentService") + fulfillment_status = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="fulfillmentStatus") + image = sgqlc.types.Field(Image, graphql_name="image") + merchant_editable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="merchantEditable") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + non_fulfillable_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="nonFulfillableQuantity") + original_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="originalTotalSet") + original_unit_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="originalUnitPriceSet") + product = sgqlc.types.Field("Product", graphql_name="product") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + refundable_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="refundableQuantity") + requires_shipping = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresShipping") + restockable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="restockable") + sku = sgqlc.types.Field(String, graphql_name="sku") + staff_member = sgqlc.types.Field("StaffMember", graphql_name="staffMember") + tax_lines = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxLine))), + graphql_name="taxLines", + args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)),)), + ) + taxable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxable") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + total_discount_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalDiscountSet") + unfulfilled_discounted_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="unfulfilledDiscountedTotalSet") + unfulfilled_original_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="unfulfilledOriginalTotalSet") + unfulfilled_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="unfulfilledQuantity") + variant = sgqlc.types.Field("ProductVariant", graphql_name="variant") + variant_title = sgqlc.types.Field(String, graphql_name="variantTitle") + vendor = sgqlc.types.Field(String, graphql_name="vendor") + + +class Link(sgqlc.types.Type, HasPublishedTranslations): + __schema__ = shopify_schema + __field_names__ = ("label", "url") + label = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="label") + url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") + + +class Location(sgqlc.types.Type, HasMetafieldDefinitions, HasMetafields, LegacyInteroperability, Node): + __schema__ = shopify_schema + __field_names__ = ( + "activatable", + "address", + "address_verified", + "deactivatable", + "deactivated_at", + "deletable", + "fulfillment_service", + "fulfills_online_orders", + "has_active_inventory", + "has_unfulfilled_orders", + "inventory_level", + "inventory_levels", + "is_active", + "local_pickup_settings_v2", + "name", + "ships_inventory", + "suggested_addresses", + ) + activatable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="activatable") + address = sgqlc.types.Field(sgqlc.types.non_null(LocationAddress), graphql_name="address") + address_verified = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="addressVerified") + deactivatable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="deactivatable") + deactivated_at = sgqlc.types.Field(String, graphql_name="deactivatedAt") + deletable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="deletable") + fulfillment_service = sgqlc.types.Field(FulfillmentService, graphql_name="fulfillmentService") + fulfills_online_orders = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="fulfillsOnlineOrders") + has_active_inventory = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasActiveInventory") + has_unfulfilled_orders = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasUnfulfilledOrders") + inventory_level = sgqlc.types.Field( + InventoryLevel, + graphql_name="inventoryLevel", + args=sgqlc.types.ArgDict( + (("inventory_item_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="inventoryItemId", default=None)),) + ), + ) + inventory_levels = sgqlc.types.Field( + sgqlc.types.non_null(InventoryLevelConnection), + graphql_name="inventoryLevels", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + is_active = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isActive") + local_pickup_settings_v2 = sgqlc.types.Field(DeliveryLocalPickupSettings, graphql_name="localPickupSettingsV2") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + ships_inventory = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="shipsInventory") + suggested_addresses = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(LocationSuggestedAddress))), graphql_name="suggestedAddresses" + ) + + +class LocationActivateUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(LocationActivateUserErrorCode, graphql_name="code") + + +class LocationAddUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(LocationAddUserErrorCode, graphql_name="code") + + +class LocationDeactivateUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(LocationDeactivateUserErrorCode, graphql_name="code") + + +class LocationDeleteUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(LocationDeleteUserErrorCode, graphql_name="code") + + +class LocationEditUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(LocationEditUserErrorCode, graphql_name="code") + + +class MailingAddress(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "address1", + "address2", + "city", + "company", + "coordinates_validated", + "country", + "country_code_v2", + "first_name", + "formatted", + "formatted_area", + "last_name", + "latitude", + "longitude", + "name", + "phone", + "province", + "province_code", + "time_zone", + "zip", + ) + address1 = sgqlc.types.Field(String, graphql_name="address1") + address2 = sgqlc.types.Field(String, graphql_name="address2") + city = sgqlc.types.Field(String, graphql_name="city") + company = sgqlc.types.Field(String, graphql_name="company") + coordinates_validated = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="coordinatesValidated") + country = sgqlc.types.Field(String, graphql_name="country") + country_code_v2 = sgqlc.types.Field(CountryCode, graphql_name="countryCodeV2") + first_name = sgqlc.types.Field(String, graphql_name="firstName") + formatted = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), + graphql_name="formatted", + args=sgqlc.types.ArgDict( + ( + ("with_name", sgqlc.types.Arg(Boolean, graphql_name="withName", default=False)), + ("with_company", sgqlc.types.Arg(Boolean, graphql_name="withCompany", default=True)), + ) + ), + ) + formatted_area = sgqlc.types.Field(String, graphql_name="formattedArea") + last_name = sgqlc.types.Field(String, graphql_name="lastName") + latitude = sgqlc.types.Field(Float, graphql_name="latitude") + longitude = sgqlc.types.Field(Float, graphql_name="longitude") + name = sgqlc.types.Field(String, graphql_name="name") + phone = sgqlc.types.Field(String, graphql_name="phone") + province = sgqlc.types.Field(String, graphql_name="province") + province_code = sgqlc.types.Field(String, graphql_name="provinceCode") + time_zone = sgqlc.types.Field(String, graphql_name="timeZone") + zip = sgqlc.types.Field(String, graphql_name="zip") + + +class ManualDiscountApplication(sgqlc.types.Type, DiscountApplication): + __schema__ = shopify_schema + __field_names__ = ("description", "title") + description = sgqlc.types.Field(String, graphql_name="description") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + + +class Market(sgqlc.types.Type, HasMetafieldDefinitions, HasMetafields, Node): + __schema__ = shopify_schema + __field_names__ = ("catalogs", "currency_settings", "enabled", "handle", "name", "price_list", "primary", "regions", "web_presence") + catalogs = sgqlc.types.Field( + sgqlc.types.non_null(MarketCatalogConnection), + graphql_name="catalogs", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + currency_settings = sgqlc.types.Field(sgqlc.types.non_null(MarketCurrencySettings), graphql_name="currencySettings") + enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="enabled") + handle = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="handle") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + price_list = sgqlc.types.Field("PriceList", graphql_name="priceList") + primary = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="primary") + regions = sgqlc.types.Field( + sgqlc.types.non_null(MarketRegionConnection), + graphql_name="regions", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + web_presence = sgqlc.types.Field("MarketWebPresence", graphql_name="webPresence") + + +class MarketCatalog(sgqlc.types.Type, Catalog, Node): + __schema__ = shopify_schema + __field_names__ = ("markets",) + markets = sgqlc.types.Field( + sgqlc.types.non_null(MarketConnection), + graphql_name="markets", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + + +class MarketCurrencySettingsUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(MarketCurrencySettingsUserErrorCode, graphql_name="code") + + +class MarketRegionCountry(sgqlc.types.Type, MarketRegion, Node): + __schema__ = shopify_schema + __field_names__ = ("code", "currency") + code = sgqlc.types.Field(sgqlc.types.non_null(CountryCode), graphql_name="code") + currency = sgqlc.types.Field(sgqlc.types.non_null(CurrencySetting), graphql_name="currency") + + +class MarketUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(MarketUserErrorCode, graphql_name="code") + + +class MarketWebPresence(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("alternate_locales", "default_locale", "domain", "market", "root_urls", "subfolder_suffix") + alternate_locales = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="alternateLocales" + ) + default_locale = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="defaultLocale") + domain = sgqlc.types.Field(Domain, graphql_name="domain") + market = sgqlc.types.Field(sgqlc.types.non_null(Market), graphql_name="market") + root_urls = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MarketWebPresenceRootUrl))), graphql_name="rootUrls" + ) + subfolder_suffix = sgqlc.types.Field(String, graphql_name="subfolderSuffix") + + +class MarketingActivity(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "activity_list_url", + "ad_spend", + "app", + "app_errors", + "budget", + "created_at", + "form_data", + "in_main_workflow_version", + "marketing_channel", + "marketing_event", + "source_and_medium", + "status", + "status_badge_type_v2", + "status_label", + "status_transitioned_at", + "tactic", + "target_status", + "title", + "updated_at", + "utm_parameters", + ) + activity_list_url = sgqlc.types.Field(URL, graphql_name="activityListUrl") + ad_spend = sgqlc.types.Field(MoneyV2, graphql_name="adSpend") + app = sgqlc.types.Field(sgqlc.types.non_null(App), graphql_name="app") + app_errors = sgqlc.types.Field(MarketingActivityExtensionAppErrors, graphql_name="appErrors") + budget = sgqlc.types.Field(MarketingBudget, graphql_name="budget") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + form_data = sgqlc.types.Field(String, graphql_name="formData") + in_main_workflow_version = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="inMainWorkflowVersion") + marketing_channel = sgqlc.types.Field(sgqlc.types.non_null(MarketingChannel), graphql_name="marketingChannel") + marketing_event = sgqlc.types.Field("MarketingEvent", graphql_name="marketingEvent") + source_and_medium = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="sourceAndMedium") + status = sgqlc.types.Field(sgqlc.types.non_null(MarketingActivityStatus), graphql_name="status") + status_badge_type_v2 = sgqlc.types.Field(BadgeType, graphql_name="statusBadgeTypeV2") + status_label = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="statusLabel") + status_transitioned_at = sgqlc.types.Field(DateTime, graphql_name="statusTransitionedAt") + tactic = sgqlc.types.Field(sgqlc.types.non_null(MarketingTactic), graphql_name="tactic") + target_status = sgqlc.types.Field(MarketingActivityStatus, graphql_name="targetStatus") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + utm_parameters = sgqlc.types.Field(UTMParameters, graphql_name="utmParameters") + + +class MarketingActivityUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(MarketingActivityUserErrorCode, graphql_name="code") + + +class MarketingEvent(sgqlc.types.Type, LegacyInteroperability, Node): + __schema__ = shopify_schema + __field_names__ = ( + "app", + "channel", + "description", + "ended_at", + "manage_url", + "preview_url", + "remote_id", + "scheduled_to_end_at", + "source_and_medium", + "started_at", + "type", + "utm_campaign", + "utm_medium", + "utm_source", + ) + app = sgqlc.types.Field(sgqlc.types.non_null(App), graphql_name="app") + channel = sgqlc.types.Field(MarketingChannel, graphql_name="channel") + description = sgqlc.types.Field(String, graphql_name="description") + ended_at = sgqlc.types.Field(DateTime, graphql_name="endedAt") + manage_url = sgqlc.types.Field(URL, graphql_name="manageUrl") + preview_url = sgqlc.types.Field(URL, graphql_name="previewUrl") + remote_id = sgqlc.types.Field(String, graphql_name="remoteId") + scheduled_to_end_at = sgqlc.types.Field(DateTime, graphql_name="scheduledToEndAt") + source_and_medium = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="sourceAndMedium") + started_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="startedAt") + type = sgqlc.types.Field(sgqlc.types.non_null(MarketingTactic), graphql_name="type") + utm_campaign = sgqlc.types.Field(String, graphql_name="utmCampaign") + utm_medium = sgqlc.types.Field(String, graphql_name="utmMedium") + utm_source = sgqlc.types.Field(String, graphql_name="utmSource") + + +class MediaImage(sgqlc.types.Type, File, HasMetafields, Media, Node): + __schema__ = shopify_schema + __field_names__ = ("image", "mime_type", "original_source") + image = sgqlc.types.Field(Image, graphql_name="image") + mime_type = sgqlc.types.Field(String, graphql_name="mimeType") + original_source = sgqlc.types.Field(MediaImageOriginalSource, graphql_name="originalSource") + + +class MediaUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(MediaUserErrorCode, graphql_name="code") + + +class Metafield(sgqlc.types.Type, LegacyInteroperability, Node): + __schema__ = shopify_schema + __field_names__ = ( + "created_at", + "definition", + "description", + "key", + "namespace", + "owner", + "owner_type", + "reference", + "references", + "type", + "updated_at", + "value", + ) + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + definition = sgqlc.types.Field("MetafieldDefinition", graphql_name="definition") + description = sgqlc.types.Field(String, graphql_name="description") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + namespace = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="namespace") + owner = sgqlc.types.Field(sgqlc.types.non_null(HasMetafields), graphql_name="owner") + owner_type = sgqlc.types.Field(sgqlc.types.non_null(MetafieldOwnerType), graphql_name="ownerType") + reference = sgqlc.types.Field("MetafieldReference", graphql_name="reference") + references = sgqlc.types.Field( + MetafieldReferenceConnection, + graphql_name="references", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ) + ), + ) + type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="type") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") + + +class MetafieldDefinition(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "access", + "description", + "key", + "metafields", + "metafields_count", + "name", + "namespace", + "owner_type", + "pinned_position", + "standard_template", + "type", + "use_as_collection_condition", + "validation_status", + "validations", + "visible_to_storefront_api", + ) + access = sgqlc.types.Field(sgqlc.types.non_null(MetafieldAccess), graphql_name="access") + description = sgqlc.types.Field(String, graphql_name="description") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + metafields = sgqlc.types.Field( + sgqlc.types.non_null(MetafieldConnection), + graphql_name="metafields", + args=sgqlc.types.ArgDict( + ( + ("validation_status", sgqlc.types.Arg(MetafieldValidationStatus, graphql_name="validationStatus", default="ANY")), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + metafields_count = sgqlc.types.Field( + sgqlc.types.non_null(Int), + graphql_name="metafieldsCount", + args=sgqlc.types.ArgDict( + (("validation_status", sgqlc.types.Arg(MetafieldValidationStatus, graphql_name="validationStatus", default=None)),) + ), + ) + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + namespace = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="namespace") + owner_type = sgqlc.types.Field(sgqlc.types.non_null(MetafieldOwnerType), graphql_name="ownerType") + pinned_position = sgqlc.types.Field(Int, graphql_name="pinnedPosition") + standard_template = sgqlc.types.Field("StandardMetafieldDefinitionTemplate", graphql_name="standardTemplate") + type = sgqlc.types.Field(sgqlc.types.non_null(MetafieldDefinitionType), graphql_name="type") + use_as_collection_condition = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="useAsCollectionCondition") + validation_status = sgqlc.types.Field(sgqlc.types.non_null(MetafieldDefinitionValidationStatus), graphql_name="validationStatus") + validations = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MetafieldDefinitionValidation))), graphql_name="validations" + ) + visible_to_storefront_api = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="visibleToStorefrontApi") + + +class MetafieldDefinitionCreateUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(MetafieldDefinitionCreateUserErrorCode, graphql_name="code") + + +class MetafieldDefinitionDeleteUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(MetafieldDefinitionDeleteUserErrorCode, graphql_name="code") + + +class MetafieldDefinitionPinUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(MetafieldDefinitionPinUserErrorCode, graphql_name="code") + + +class MetafieldDefinitionUnpinUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(MetafieldDefinitionUnpinUserErrorCode, graphql_name="code") + + +class MetafieldDefinitionUpdateUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(MetafieldDefinitionUpdateUserErrorCode, graphql_name="code") + + +class MetafieldStorefrontVisibility(sgqlc.types.Type, LegacyInteroperability, Node): + __schema__ = shopify_schema + __field_names__ = ("created_at", "key", "namespace", "owner_type", "updated_at") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + namespace = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="namespace") + owner_type = sgqlc.types.Field(sgqlc.types.non_null(MetafieldOwnerType), graphql_name="ownerType") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + + +class MetafieldsSetUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code", "element_index") + code = sgqlc.types.Field(MetafieldsSetUserErrorCode, graphql_name="code") + element_index = sgqlc.types.Field(Int, graphql_name="elementIndex") + + +class Metaobject(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "capabilities", + "created_by", + "definition", + "display_name", + "field", + "fields", + "handle", + "referenced_by", + "type", + "updated_at", + ) + capabilities = sgqlc.types.Field(sgqlc.types.non_null(MetaobjectCapabilityData), graphql_name="capabilities") + created_by = sgqlc.types.Field(sgqlc.types.non_null(App), graphql_name="createdBy") + definition = sgqlc.types.Field(sgqlc.types.non_null("MetaobjectDefinition"), graphql_name="definition") + display_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="displayName") + field = sgqlc.types.Field( + MetaobjectField, + graphql_name="field", + args=sgqlc.types.ArgDict((("key", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="key", default=None)),)), + ) + fields = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MetaobjectField))), graphql_name="fields") + handle = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="handle") + referenced_by = sgqlc.types.Field( + sgqlc.types.non_null(MetafieldRelationConnection), + graphql_name="referencedBy", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="type") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + + +class MetaobjectDefinition(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "access", + "capabilities", + "description", + "display_name_key", + "field_definitions", + "metaobjects", + "metaobjects_count", + "name", + "type", + ) + access = sgqlc.types.Field(sgqlc.types.non_null(MetaobjectAccess), graphql_name="access") + capabilities = sgqlc.types.Field(sgqlc.types.non_null(MetaobjectCapabilities), graphql_name="capabilities") + description = sgqlc.types.Field(String, graphql_name="description") + display_name_key = sgqlc.types.Field(String, graphql_name="displayNameKey") + field_definitions = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MetaobjectFieldDefinition))), graphql_name="fieldDefinitions" + ) + metaobjects = sgqlc.types.Field( + sgqlc.types.non_null(MetaobjectConnection), + graphql_name="metaobjects", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + metaobjects_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="metaobjectsCount") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="type") + + +class MetaobjectUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code", "element_index", "element_key") + code = sgqlc.types.Field(MetaobjectUserErrorCode, graphql_name="code") + element_index = sgqlc.types.Field(Int, graphql_name="elementIndex") + element_key = sgqlc.types.Field(String, graphql_name="elementKey") + + +class Model3d(sgqlc.types.Type, Media, Node): + __schema__ = shopify_schema + __field_names__ = ("bounding_box", "filename", "original_source", "sources") + bounding_box = sgqlc.types.Field(Model3dBoundingBox, graphql_name="boundingBox") + filename = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="filename") + original_source = sgqlc.types.Field(Model3dSource, graphql_name="originalSource") + sources = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Model3dSource))), graphql_name="sources") + + +class OnlineStoreArticle(sgqlc.types.Type, HasPublishedTranslations, Navigable, Node): + __schema__ = shopify_schema + __field_names__ = () + + +class OnlineStoreBlog(sgqlc.types.Type, HasPublishedTranslations, Node): + __schema__ = shopify_schema + __field_names__ = () + + +class OnlineStorePage(sgqlc.types.Type, HasPublishedTranslations, Navigable, Node): + __schema__ = shopify_schema + __field_names__ = () + + +class Order( + sgqlc.types.Type, + CommentEventSubject, + HasEvents, + HasLocalizationExtensions, + HasMetafieldDefinitions, + HasMetafields, + LegacyInteroperability, + Node, +): + __schema__ = shopify_schema + __field_names__ = ( + "additional_fees", + "agreements", + "alerts", + "app", + "billing_address", + "billing_address_matches_shipping_address", + "can_mark_as_paid", + "can_notify_customer", + "cancel_reason", + "cancelled_at", + "capturable", + "cart_discount_amount_set", + "channel_information", + "client_ip", + "closed", + "closed_at", + "confirmation_number", + "confirmed", + "created_at", + "currency_code", + "current_cart_discount_amount_set", + "current_subtotal_line_items_quantity", + "current_subtotal_price_set", + "current_tax_lines", + "current_total_additional_fees_set", + "current_total_discounts_set", + "current_total_duties_set", + "current_total_price_set", + "current_total_tax_set", + "current_total_weight", + "custom_attributes", + "customer", + "customer_accepts_marketing", + "customer_journey_summary", + "customer_locale", + "discount_applications", + "discount_code", + "discount_codes", + "display_address", + "display_financial_status", + "display_fulfillment_status", + "disputes", + "edited", + "email", + "estimated_taxes", + "exchange_v2s", + "fulfillable", + "fulfillment_orders", + "fulfillments", + "fully_paid", + "line_items", + "merchant_editable", + "merchant_editable_errors", + "merchant_of_record_app", + "name", + "net_payment_set", + "non_fulfillable_line_items", + "note", + "original_total_additional_fees_set", + "original_total_duties_set", + "original_total_price_set", + "payment_collection_details", + "payment_gateway_names", + "payment_terms", + "phone", + "physical_location", + "po_number", + "presentment_currency_code", + "processed_at", + "publication", + "purchasing_entity", + "refund_discrepancy_set", + "refundable", + "refunds", + "registered_source_url", + "requires_shipping", + "restockable", + "return_status", + "returns", + "risk_level", + "risks", + "shipping_address", + "shipping_line", + "shipping_lines", + "source_identifier", + "subtotal_line_items_quantity", + "subtotal_price_set", + "suggested_refund", + "tags", + "tax_exempt", + "tax_lines", + "taxes_included", + "test", + "total_capturable_set", + "total_discounts_set", + "total_outstanding_set", + "total_price_set", + "total_received_set", + "total_refunded_set", + "total_refunded_shipping_set", + "total_shipping_price_set", + "total_tax_set", + "total_tip_received_set", + "total_weight", + "transactions", + "unpaid", + "updated_at", + ) + additional_fees = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(AdditionalFee))), graphql_name="additionalFees" + ) + agreements = sgqlc.types.Field( + sgqlc.types.non_null(SalesAgreementConnection), + graphql_name="agreements", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + alerts = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ResourceAlert))), graphql_name="alerts") + app = sgqlc.types.Field(OrderApp, graphql_name="app") + billing_address = sgqlc.types.Field(MailingAddress, graphql_name="billingAddress") + billing_address_matches_shipping_address = sgqlc.types.Field( + sgqlc.types.non_null(Boolean), graphql_name="billingAddressMatchesShippingAddress" + ) + can_mark_as_paid = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="canMarkAsPaid") + can_notify_customer = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="canNotifyCustomer") + cancel_reason = sgqlc.types.Field(OrderCancelReason, graphql_name="cancelReason") + cancelled_at = sgqlc.types.Field(DateTime, graphql_name="cancelledAt") + capturable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="capturable") + cart_discount_amount_set = sgqlc.types.Field(MoneyBag, graphql_name="cartDiscountAmountSet") + channel_information = sgqlc.types.Field(ChannelInformation, graphql_name="channelInformation") + client_ip = sgqlc.types.Field(String, graphql_name="clientIp") + closed = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="closed") + closed_at = sgqlc.types.Field(DateTime, graphql_name="closedAt") + confirmation_number = sgqlc.types.Field(String, graphql_name="confirmationNumber") + confirmed = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="confirmed") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currencyCode") + current_cart_discount_amount_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="currentCartDiscountAmountSet") + current_subtotal_line_items_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="currentSubtotalLineItemsQuantity") + current_subtotal_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="currentSubtotalPriceSet") + current_tax_lines = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxLine))), graphql_name="currentTaxLines" + ) + current_total_additional_fees_set = sgqlc.types.Field(MoneyBag, graphql_name="currentTotalAdditionalFeesSet") + current_total_discounts_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="currentTotalDiscountsSet") + current_total_duties_set = sgqlc.types.Field(MoneyBag, graphql_name="currentTotalDutiesSet") + current_total_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="currentTotalPriceSet") + current_total_tax_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="currentTotalTaxSet") + current_total_weight = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="currentTotalWeight") + custom_attributes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Attribute))), graphql_name="customAttributes" + ) + customer = sgqlc.types.Field(Customer, graphql_name="customer") + customer_accepts_marketing = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="customerAcceptsMarketing") + customer_journey_summary = sgqlc.types.Field(CustomerJourneySummary, graphql_name="customerJourneySummary") + customer_locale = sgqlc.types.Field(String, graphql_name="customerLocale") + discount_applications = sgqlc.types.Field( + sgqlc.types.non_null(DiscountApplicationConnection), + graphql_name="discountApplications", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + discount_code = sgqlc.types.Field(String, graphql_name="discountCode") + discount_codes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="discountCodes" + ) + display_address = sgqlc.types.Field(MailingAddress, graphql_name="displayAddress") + display_financial_status = sgqlc.types.Field(OrderDisplayFinancialStatus, graphql_name="displayFinancialStatus") + display_fulfillment_status = sgqlc.types.Field( + sgqlc.types.non_null(OrderDisplayFulfillmentStatus), graphql_name="displayFulfillmentStatus" + ) + disputes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("OrderDisputeSummary"))), graphql_name="disputes" + ) + edited = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="edited") + email = sgqlc.types.Field(String, graphql_name="email") + estimated_taxes = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="estimatedTaxes") + exchange_v2s = sgqlc.types.Field( + sgqlc.types.non_null(ExchangeV2Connection), + graphql_name="exchangeV2s", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + fulfillable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="fulfillable") + fulfillment_orders = sgqlc.types.Field( + sgqlc.types.non_null(FulfillmentOrderConnection), + graphql_name="fulfillmentOrders", + args=sgqlc.types.ArgDict( + ( + ("displayable", sgqlc.types.Arg(Boolean, graphql_name="displayable", default=False)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + fulfillments = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Fulfillment))), + graphql_name="fulfillments", + args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)),)), + ) + fully_paid = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="fullyPaid") + line_items = sgqlc.types.Field( + sgqlc.types.non_null(LineItemConnection), + graphql_name="lineItems", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + merchant_editable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="merchantEditable") + merchant_editable_errors = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="merchantEditableErrors" + ) + merchant_of_record_app = sgqlc.types.Field(OrderApp, graphql_name="merchantOfRecordApp") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + net_payment_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="netPaymentSet") + non_fulfillable_line_items = sgqlc.types.Field( + sgqlc.types.non_null(LineItemConnection), + graphql_name="nonFulfillableLineItems", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + note = sgqlc.types.Field(String, graphql_name="note") + original_total_additional_fees_set = sgqlc.types.Field(MoneyBag, graphql_name="originalTotalAdditionalFeesSet") + original_total_duties_set = sgqlc.types.Field(MoneyBag, graphql_name="originalTotalDutiesSet") + original_total_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="originalTotalPriceSet") + payment_collection_details = sgqlc.types.Field( + sgqlc.types.non_null(OrderPaymentCollectionDetails), graphql_name="paymentCollectionDetails" + ) + payment_gateway_names = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="paymentGatewayNames" + ) + payment_terms = sgqlc.types.Field("PaymentTerms", graphql_name="paymentTerms") + phone = sgqlc.types.Field(String, graphql_name="phone") + physical_location = sgqlc.types.Field(Location, graphql_name="physicalLocation") + po_number = sgqlc.types.Field(String, graphql_name="poNumber") + presentment_currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="presentmentCurrencyCode") + processed_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="processedAt") + publication = sgqlc.types.Field("Publication", graphql_name="publication") + purchasing_entity = sgqlc.types.Field("PurchasingEntity", graphql_name="purchasingEntity") + refund_discrepancy_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="refundDiscrepancySet") + refundable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="refundable") + refunds = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Refund"))), + graphql_name="refunds", + args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)),)), + ) + registered_source_url = sgqlc.types.Field(URL, graphql_name="registeredSourceUrl") + requires_shipping = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresShipping") + restockable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="restockable") + return_status = sgqlc.types.Field(sgqlc.types.non_null(OrderReturnStatus), graphql_name="returnStatus") + returns = sgqlc.types.Field( + sgqlc.types.non_null(ReturnConnection), + graphql_name="returns", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + risk_level = sgqlc.types.Field(sgqlc.types.non_null(OrderRiskLevel), graphql_name="riskLevel") + risks = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(OrderRisk))), + graphql_name="risks", + args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)),)), + ) + shipping_address = sgqlc.types.Field(MailingAddress, graphql_name="shippingAddress") + shipping_line = sgqlc.types.Field(ShippingLine, graphql_name="shippingLine") + shipping_lines = sgqlc.types.Field( + sgqlc.types.non_null(ShippingLineConnection), + graphql_name="shippingLines", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + source_identifier = sgqlc.types.Field(String, graphql_name="sourceIdentifier") + subtotal_line_items_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="subtotalLineItemsQuantity") + subtotal_price_set = sgqlc.types.Field(MoneyBag, graphql_name="subtotalPriceSet") + suggested_refund = sgqlc.types.Field( + SuggestedRefund, + graphql_name="suggestedRefund", + args=sgqlc.types.ArgDict( + ( + ("shipping_amount", sgqlc.types.Arg(Money, graphql_name="shippingAmount", default=None)), + ("refund_shipping", sgqlc.types.Arg(Boolean, graphql_name="refundShipping", default=None)), + ( + "refund_line_items", + sgqlc.types.Arg( + sgqlc.types.list_of(sgqlc.types.non_null(RefundLineItemInput)), graphql_name="refundLineItems", default=None + ), + ), + ( + "refund_duties", + sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(RefundDutyInput)), graphql_name="refundDuties", default=None), + ), + ("suggest_full_refund", sgqlc.types.Arg(Boolean, graphql_name="suggestFullRefund", default=False)), + ) + ), + ) + tags = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="tags") + tax_exempt = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxExempt") + tax_lines = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxLine))), graphql_name="taxLines") + taxes_included = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxesIncluded") + test = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="test") + total_capturable_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalCapturableSet") + total_discounts_set = sgqlc.types.Field(MoneyBag, graphql_name="totalDiscountsSet") + total_outstanding_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalOutstandingSet") + total_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalPriceSet") + total_received_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalReceivedSet") + total_refunded_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalRefundedSet") + total_refunded_shipping_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalRefundedShippingSet") + total_shipping_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalShippingPriceSet") + total_tax_set = sgqlc.types.Field(MoneyBag, graphql_name="totalTaxSet") + total_tip_received_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalTipReceivedSet") + total_weight = sgqlc.types.Field(UnsignedInt64, graphql_name="totalWeight") + transactions = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("OrderTransaction"))), + graphql_name="transactions", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("capturable", sgqlc.types.Arg(Boolean, graphql_name="capturable", default=None)), + ("manually_resolvable", sgqlc.types.Arg(Boolean, graphql_name="manuallyResolvable", default=None)), + ) + ), + ) + unpaid = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="unpaid") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + + +class OrderAgreement(sgqlc.types.Type, SalesAgreement): + __schema__ = shopify_schema + __field_names__ = ("order",) + order = sgqlc.types.Field(sgqlc.types.non_null(Order), graphql_name="order") + + +class OrderCreateMandatePaymentUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(OrderCreateMandatePaymentUserErrorCode, graphql_name="code") + + +class OrderDisputeSummary(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("initiated_as", "status") + initiated_as = sgqlc.types.Field(sgqlc.types.non_null(DisputeType), graphql_name="initiatedAs") + status = sgqlc.types.Field(sgqlc.types.non_null(DisputeStatus), graphql_name="status") + + +class OrderEditAgreement(sgqlc.types.Type, SalesAgreement): + __schema__ = shopify_schema + __field_names__ = () + + +class OrderInvoiceSendUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(OrderInvoiceSendUserErrorCode, graphql_name="code") + + +class OrderTransaction(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "account_number", + "amount_set", + "authorization_code", + "authorization_expires_at", + "created_at", + "error_code", + "fees", + "formatted_gateway", + "gateway", + "kind", + "manually_capturable", + "maximum_refundable_v2", + "order", + "parent_transaction", + "payment_details", + "payment_icon", + "payment_id", + "processed_at", + "receipt_json", + "settlement_currency", + "settlement_currency_rate", + "shopify_payments_set", + "status", + "test", + "total_unsettled_set", + "user", + ) + account_number = sgqlc.types.Field(String, graphql_name="accountNumber") + amount_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="amountSet") + authorization_code = sgqlc.types.Field(String, graphql_name="authorizationCode") + authorization_expires_at = sgqlc.types.Field(DateTime, graphql_name="authorizationExpiresAt") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + error_code = sgqlc.types.Field(OrderTransactionErrorCode, graphql_name="errorCode") + fees = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TransactionFee"))), graphql_name="fees") + formatted_gateway = sgqlc.types.Field(String, graphql_name="formattedGateway") + gateway = sgqlc.types.Field(String, graphql_name="gateway") + kind = sgqlc.types.Field(sgqlc.types.non_null(OrderTransactionKind), graphql_name="kind") + manually_capturable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="manuallyCapturable") + maximum_refundable_v2 = sgqlc.types.Field(MoneyV2, graphql_name="maximumRefundableV2") + order = sgqlc.types.Field(Order, graphql_name="order") + parent_transaction = sgqlc.types.Field("OrderTransaction", graphql_name="parentTransaction") + payment_details = sgqlc.types.Field("PaymentDetails", graphql_name="paymentDetails") + payment_icon = sgqlc.types.Field(Image, graphql_name="paymentIcon") + payment_id = sgqlc.types.Field(String, graphql_name="paymentId") + processed_at = sgqlc.types.Field(DateTime, graphql_name="processedAt") + receipt_json = sgqlc.types.Field(JSON, graphql_name="receiptJson") + settlement_currency = sgqlc.types.Field(CurrencyCode, graphql_name="settlementCurrency") + settlement_currency_rate = sgqlc.types.Field(Decimal, graphql_name="settlementCurrencyRate") + shopify_payments_set = sgqlc.types.Field(ShopifyPaymentsTransactionSet, graphql_name="shopifyPaymentsSet") + status = sgqlc.types.Field(sgqlc.types.non_null(OrderTransactionStatus), graphql_name="status") + test = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="test") + total_unsettled_set = sgqlc.types.Field(MoneyBag, graphql_name="totalUnsettledSet") + user = sgqlc.types.Field("StaffMember", graphql_name="user") + + +class PaymentCustomization(sgqlc.types.Type, HasMetafieldDefinitions, HasMetafields, Node): + __schema__ = shopify_schema + __field_names__ = ("enabled", "error_history", "function_id", "shopify_function", "title") + enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="enabled") + error_history = sgqlc.types.Field(FunctionsErrorHistory, graphql_name="errorHistory") + function_id = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="functionId") + shopify_function = sgqlc.types.Field(sgqlc.types.non_null(ShopifyFunction), graphql_name="shopifyFunction") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + + +class PaymentCustomizationError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(PaymentCustomizationErrorCode, graphql_name="code") + + +class PaymentMandate(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("payment_instrument",) + payment_instrument = sgqlc.types.Field(sgqlc.types.non_null("PaymentInstrument"), graphql_name="paymentInstrument") + + +class PaymentReminderSendUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(PaymentReminderSendUserErrorCode, graphql_name="code") + + +class PaymentSchedule(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("amount", "completed_at", "due_at", "issued_at", "payment_terms") + amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="amount") + completed_at = sgqlc.types.Field(DateTime, graphql_name="completedAt") + due_at = sgqlc.types.Field(DateTime, graphql_name="dueAt") + issued_at = sgqlc.types.Field(DateTime, graphql_name="issuedAt") + payment_terms = sgqlc.types.Field(sgqlc.types.non_null("PaymentTerms"), graphql_name="paymentTerms") + + +class PaymentTerms(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "draft_order", + "due_in_days", + "order", + "overdue", + "payment_schedules", + "payment_terms_name", + "payment_terms_type", + "translated_name", + ) + draft_order = sgqlc.types.Field(DraftOrder, graphql_name="draftOrder") + due_in_days = sgqlc.types.Field(Int, graphql_name="dueInDays") + order = sgqlc.types.Field(Order, graphql_name="order") + overdue = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="overdue") + payment_schedules = sgqlc.types.Field( + sgqlc.types.non_null(PaymentScheduleConnection), + graphql_name="paymentSchedules", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + payment_terms_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="paymentTermsName") + payment_terms_type = sgqlc.types.Field(sgqlc.types.non_null(PaymentTermsType), graphql_name="paymentTermsType") + translated_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="translatedName") + + +class PaymentTermsCreateUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(PaymentTermsCreateUserErrorCode, graphql_name="code") + + +class PaymentTermsDeleteUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(PaymentTermsDeleteUserErrorCode, graphql_name="code") + + +class PaymentTermsTemplate(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("description", "due_in_days", "name", "payment_terms_type", "translated_name") + description = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="description") + due_in_days = sgqlc.types.Field(Int, graphql_name="dueInDays") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + payment_terms_type = sgqlc.types.Field(sgqlc.types.non_null(PaymentTermsType), graphql_name="paymentTermsType") + translated_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="translatedName") + + +class PaymentTermsUpdateUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(PaymentTermsUpdateUserErrorCode, graphql_name="code") + + +class PolarisVizResponse(sgqlc.types.Type, ShopifyqlResponse): + __schema__ = shopify_schema + __field_names__ = ("data", "viz_type") + data = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(PolarisVizDataSeries))), graphql_name="data") + viz_type = sgqlc.types.Field(sgqlc.types.non_null(VisualizationType), graphql_name="vizType") + + +class PriceList(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("catalog", "currency", "fixed_prices_count", "name", "parent", "prices", "quantity_rules") + catalog = sgqlc.types.Field(Catalog, graphql_name="catalog") + currency = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currency") + fixed_prices_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="fixedPricesCount") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + parent = sgqlc.types.Field(PriceListParent, graphql_name="parent") + prices = sgqlc.types.Field( + sgqlc.types.non_null(PriceListPriceConnection), + graphql_name="prices", + args=sgqlc.types.ArgDict( + ( + ("origin_type", sgqlc.types.Arg(PriceListPriceOriginType, graphql_name="originType", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + quantity_rules = sgqlc.types.Field( + sgqlc.types.non_null(QuantityRuleConnection), + graphql_name="quantityRules", + args=sgqlc.types.ArgDict( + ( + ("origin_type", sgqlc.types.Arg(QuantityRuleOriginType, graphql_name="originType", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + + +class PriceListFixedPricesByProductBulkUpdateUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(PriceListFixedPricesByProductBulkUpdateUserErrorCode, graphql_name="code") + + +class PriceListPriceUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(PriceListPriceUserErrorCode, graphql_name="code") + + +class PriceListUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(PriceListUserErrorCode, graphql_name="code") + + +class PriceRule(sgqlc.types.Type, CommentEventSubject, HasEvents, LegacyInteroperability, Node): + __schema__ = shopify_schema + __field_names__ = ( + "allocation_limit", + "allocation_method", + "app", + "combines_with", + "created_at", + "customer_selection", + "discount_class", + "discount_codes", + "discount_codes_count", + "ends_at", + "features", + "item_entitlements", + "item_prerequisites", + "once_per_customer", + "prerequisite_quantity_range", + "prerequisite_shipping_price_range", + "prerequisite_subtotal_range", + "prerequisite_to_entitlement_quantity_ratio", + "shareable_urls", + "shipping_entitlements", + "starts_at", + "status", + "summary", + "target", + "title", + "total_sales", + "usage_count", + "usage_limit", + "validity_period", + "value_v2", + ) + allocation_limit = sgqlc.types.Field(Int, graphql_name="allocationLimit") + allocation_method = sgqlc.types.Field(sgqlc.types.non_null(PriceRuleAllocationMethod), graphql_name="allocationMethod") + app = sgqlc.types.Field(App, graphql_name="app") + combines_with = sgqlc.types.Field(sgqlc.types.non_null(DiscountCombinesWith), graphql_name="combinesWith") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + customer_selection = sgqlc.types.Field(sgqlc.types.non_null(PriceRuleCustomerSelection), graphql_name="customerSelection") + discount_class = sgqlc.types.Field(sgqlc.types.non_null(DiscountClass), graphql_name="discountClass") + discount_codes = sgqlc.types.Field( + sgqlc.types.non_null(PriceRuleDiscountCodeConnection), + graphql_name="discountCodes", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(DiscountCodeSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), + ) + ), + ) + discount_codes_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="discountCodesCount") + ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") + features = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(PriceRuleFeature))), graphql_name="features") + item_entitlements = sgqlc.types.Field(sgqlc.types.non_null(PriceRuleItemEntitlements), graphql_name="itemEntitlements") + item_prerequisites = sgqlc.types.Field(sgqlc.types.non_null(PriceRuleLineItemPrerequisites), graphql_name="itemPrerequisites") + once_per_customer = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="oncePerCustomer") + prerequisite_quantity_range = sgqlc.types.Field(PriceRuleQuantityRange, graphql_name="prerequisiteQuantityRange") + prerequisite_shipping_price_range = sgqlc.types.Field(PriceRuleMoneyRange, graphql_name="prerequisiteShippingPriceRange") + prerequisite_subtotal_range = sgqlc.types.Field(PriceRuleMoneyRange, graphql_name="prerequisiteSubtotalRange") + prerequisite_to_entitlement_quantity_ratio = sgqlc.types.Field( + PriceRulePrerequisiteToEntitlementQuantityRatio, graphql_name="prerequisiteToEntitlementQuantityRatio" + ) + shareable_urls = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(PriceRuleShareableUrl))), graphql_name="shareableUrls" + ) + shipping_entitlements = sgqlc.types.Field(sgqlc.types.non_null(PriceRuleShippingLineEntitlements), graphql_name="shippingEntitlements") + starts_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="startsAt") + status = sgqlc.types.Field(sgqlc.types.non_null(PriceRuleStatus), graphql_name="status") + summary = sgqlc.types.Field(String, graphql_name="summary") + target = sgqlc.types.Field(sgqlc.types.non_null(PriceRuleTarget), graphql_name="target") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + total_sales = sgqlc.types.Field(MoneyV2, graphql_name="totalSales") + usage_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="usageCount") + usage_limit = sgqlc.types.Field(Int, graphql_name="usageLimit") + validity_period = sgqlc.types.Field(sgqlc.types.non_null(PriceRuleValidityPeriod), graphql_name="validityPeriod") + value_v2 = sgqlc.types.Field(sgqlc.types.non_null("PricingValue"), graphql_name="valueV2") + + +class PriceRuleDiscountCode(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("app", "code", "usage_count") + app = sgqlc.types.Field(App, graphql_name="app") + code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") + usage_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="usageCount") + + +class PriceRuleUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(PriceRuleErrorCode, graphql_name="code") + + +class PrivateMetafield(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("created_at", "key", "namespace", "updated_at", "value", "value_type") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + namespace = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="namespace") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") + value_type = sgqlc.types.Field(sgqlc.types.non_null(PrivateMetafieldValueType), graphql_name="valueType") + + +class Product( + sgqlc.types.Type, + HasMetafieldDefinitions, + HasMetafields, + HasPublishedTranslations, + LegacyInteroperability, + Navigable, + Node, + OnlineStorePreviewable, + Publishable, +): + __schema__ = shopify_schema + __field_names__ = ( + "collections", + "contextual_pricing", + "created_at", + "description", + "description_html", + "featured_image", + "featured_media", + "feedback", + "gift_card_template_suffix", + "handle", + "has_only_default_variant", + "has_out_of_stock_variants", + "images", + "in_collection", + "is_gift_card", + "media", + "media_count", + "online_store_url", + "options", + "price_range_v2", + "product_category", + "product_type", + "published_at", + "published_in_context", + "requires_selling_plan", + "resource_publication_on_current_publication", + "selling_plan_group_count", + "selling_plan_groups", + "seo", + "status", + "tags", + "template_suffix", + "title", + "total_inventory", + "total_variants", + "tracks_inventory", + "updated_at", + "variants", + "vendor", + ) + collections = sgqlc.types.Field( + sgqlc.types.non_null(CollectionConnection), + graphql_name="collections", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(CollectionSortKeys, graphql_name="sortKey", default="ID")), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + contextual_pricing = sgqlc.types.Field( + sgqlc.types.non_null(ProductContextualPricing), + graphql_name="contextualPricing", + args=sgqlc.types.ArgDict( + (("context", sgqlc.types.Arg(sgqlc.types.non_null(ContextualPricingContext), graphql_name="context", default=None)),) + ), + ) + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + description = sgqlc.types.Field( + sgqlc.types.non_null(String), + graphql_name="description", + args=sgqlc.types.ArgDict((("truncate_at", sgqlc.types.Arg(Int, graphql_name="truncateAt", default=None)),)), + ) + description_html = sgqlc.types.Field(sgqlc.types.non_null(HTML), graphql_name="descriptionHtml") + featured_image = sgqlc.types.Field(Image, graphql_name="featuredImage") + featured_media = sgqlc.types.Field(Media, graphql_name="featuredMedia") + feedback = sgqlc.types.Field(ResourceFeedback, graphql_name="feedback") + gift_card_template_suffix = sgqlc.types.Field(String, graphql_name="giftCardTemplateSuffix") + handle = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="handle") + has_only_default_variant = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasOnlyDefaultVariant") + has_out_of_stock_variants = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasOutOfStockVariants") + images = sgqlc.types.Field( + sgqlc.types.non_null(ImageConnection), + graphql_name="images", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(ProductImageSortKeys, graphql_name="sortKey", default="POSITION")), + ) + ), + ) + in_collection = sgqlc.types.Field( + sgqlc.types.non_null(Boolean), + graphql_name="inCollection", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + is_gift_card = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isGiftCard") + media = sgqlc.types.Field( + sgqlc.types.non_null(MediaConnection), + graphql_name="media", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(ProductMediaSortKeys, graphql_name="sortKey", default="POSITION")), + ) + ), + ) + media_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="mediaCount") + online_store_url = sgqlc.types.Field(URL, graphql_name="onlineStoreUrl") + options = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductOption"))), + graphql_name="options", + args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)),)), + ) + price_range_v2 = sgqlc.types.Field(sgqlc.types.non_null(ProductPriceRangeV2), graphql_name="priceRangeV2") + product_category = sgqlc.types.Field(ProductCategory, graphql_name="productCategory") + product_type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="productType") + published_at = sgqlc.types.Field(DateTime, graphql_name="publishedAt") + published_in_context = sgqlc.types.Field( + sgqlc.types.non_null(Boolean), + graphql_name="publishedInContext", + args=sgqlc.types.ArgDict( + (("context", sgqlc.types.Arg(sgqlc.types.non_null(ContextualPublicationContext), graphql_name="context", default=None)),) + ), + ) + requires_selling_plan = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresSellingPlan") + resource_publication_on_current_publication = sgqlc.types.Field( + ResourcePublicationV2, graphql_name="resourcePublicationOnCurrentPublication" + ) + selling_plan_group_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="sellingPlanGroupCount") + selling_plan_groups = sgqlc.types.Field( + sgqlc.types.non_null(SellingPlanGroupConnection), + graphql_name="sellingPlanGroups", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + seo = sgqlc.types.Field(sgqlc.types.non_null(SEO), graphql_name="seo") + status = sgqlc.types.Field(sgqlc.types.non_null(ProductStatus), graphql_name="status") + tags = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="tags") + template_suffix = sgqlc.types.Field(String, graphql_name="templateSuffix") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + total_inventory = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalInventory") + total_variants = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalVariants") + tracks_inventory = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="tracksInventory") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + variants = sgqlc.types.Field( + sgqlc.types.non_null(ProductVariantConnection), + graphql_name="variants", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(ProductVariantSortKeys, graphql_name="sortKey", default="POSITION")), + ) + ), + ) + vendor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="vendor") + + +class ProductChangeStatusUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(ProductChangeStatusUserErrorCode, graphql_name="code") + + +class ProductDeleteUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(ProductDeleteUserErrorCode, graphql_name="code") + + +class ProductDuplicateUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(ProductDuplicateUserErrorCode, graphql_name="code") + + +class ProductFeed(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("country", "language", "status") + country = sgqlc.types.Field(CountryCode, graphql_name="country") + language = sgqlc.types.Field(LanguageCode, graphql_name="language") + status = sgqlc.types.Field(sgqlc.types.non_null(ProductFeedStatus), graphql_name="status") + + +class ProductFeedCreateUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(ProductFeedCreateUserErrorCode, graphql_name="code") + + +class ProductFeedDeleteUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(ProductFeedDeleteUserErrorCode, graphql_name="code") + + +class ProductFullSyncUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(ProductFullSyncUserErrorCode, graphql_name="code") + + +class ProductOption(sgqlc.types.Type, HasPublishedTranslations, Node): + __schema__ = shopify_schema + __field_names__ = ("name", "position", "values") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + position = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="position") + values = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="values") + + +class ProductSale(sgqlc.types.Type, Sale): + __schema__ = shopify_schema + __field_names__ = ("line_item",) + line_item = sgqlc.types.Field(sgqlc.types.non_null(LineItem), graphql_name="lineItem") + + +class ProductTaxonomyNode(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("full_name", "is_leaf", "is_root", "name") + full_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="fullName") + is_leaf = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isLeaf") + is_root = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isRoot") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + + +class ProductVariant( + sgqlc.types.Type, HasMetafieldDefinitions, HasMetafields, HasPublishedTranslations, LegacyInteroperability, Navigable, Node +): + __schema__ = shopify_schema + __field_names__ = ( + "available_for_sale", + "barcode", + "compare_at_price", + "contextual_pricing", + "created_at", + "delivery_profile", + "display_name", + "fulfillment_service_editable", + "image", + "inventory_item", + "inventory_policy", + "inventory_quantity", + "media", + "position", + "price", + "product", + "product_variant_components", + "requires_components", + "selected_options", + "sellable_online_quantity", + "selling_plan_group_count", + "selling_plan_groups", + "sku", + "tax_code", + "taxable", + "title", + "updated_at", + "weight", + "weight_unit", + ) + available_for_sale = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="availableForSale") + barcode = sgqlc.types.Field(String, graphql_name="barcode") + compare_at_price = sgqlc.types.Field(Money, graphql_name="compareAtPrice") + contextual_pricing = sgqlc.types.Field( + sgqlc.types.non_null(ProductVariantContextualPricing), + graphql_name="contextualPricing", + args=sgqlc.types.ArgDict( + (("context", sgqlc.types.Arg(sgqlc.types.non_null(ContextualPricingContext), graphql_name="context", default=None)),) + ), + ) + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + delivery_profile = sgqlc.types.Field(DeliveryProfile, graphql_name="deliveryProfile") + display_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="displayName") + fulfillment_service_editable = sgqlc.types.Field(sgqlc.types.non_null(EditableProperty), graphql_name="fulfillmentServiceEditable") + image = sgqlc.types.Field(Image, graphql_name="image") + inventory_item = sgqlc.types.Field(sgqlc.types.non_null(InventoryItem), graphql_name="inventoryItem") + inventory_policy = sgqlc.types.Field(sgqlc.types.non_null(ProductVariantInventoryPolicy), graphql_name="inventoryPolicy") + inventory_quantity = sgqlc.types.Field(Int, graphql_name="inventoryQuantity") + media = sgqlc.types.Field( + sgqlc.types.non_null(MediaConnection), + graphql_name="media", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + position = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="position") + price = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="price") + product = sgqlc.types.Field(sgqlc.types.non_null(Product), graphql_name="product") + product_variant_components = sgqlc.types.Field( + sgqlc.types.non_null(ProductVariantComponentConnection), + graphql_name="productVariantComponents", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + requires_components = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresComponents") + selected_options = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SelectedOption))), graphql_name="selectedOptions" + ) + sellable_online_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="sellableOnlineQuantity") + selling_plan_group_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="sellingPlanGroupCount") + selling_plan_groups = sgqlc.types.Field( + sgqlc.types.non_null(SellingPlanGroupConnection), + graphql_name="sellingPlanGroups", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + sku = sgqlc.types.Field(String, graphql_name="sku") + tax_code = sgqlc.types.Field(String, graphql_name="taxCode") + taxable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxable") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + weight = sgqlc.types.Field(Float, graphql_name="weight") + weight_unit = sgqlc.types.Field(sgqlc.types.non_null(WeightUnit), graphql_name="weightUnit") + + +class ProductVariantComponent(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("product_variant", "quantity") + product_variant = sgqlc.types.Field(sgqlc.types.non_null(ProductVariant), graphql_name="productVariant") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + + +class ProductVariantRelationshipBulkUpdateUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(ProductVariantRelationshipBulkUpdateUserErrorCode, graphql_name="code") + + +class ProductVariantsBulkCreateUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(ProductVariantsBulkCreateUserErrorCode, graphql_name="code") + + +class ProductVariantsBulkDeleteUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(ProductVariantsBulkDeleteUserErrorCode, graphql_name="code") + + +class ProductVariantsBulkReorderUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(ProductVariantsBulkReorderUserErrorCode, graphql_name="code") + + +class ProductVariantsBulkUpdateUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(ProductVariantsBulkUpdateUserErrorCode, graphql_name="code") + + +class PubSubWebhookSubscriptionCreateUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(PubSubWebhookSubscriptionCreateUserErrorCode, graphql_name="code") + + +class PubSubWebhookSubscriptionUpdateUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(PubSubWebhookSubscriptionUpdateUserErrorCode, graphql_name="code") + + +class Publication(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "auto_publish", + "catalog", + "collection_publications_v3", + "collections", + "has_collection", + "operation", + "product_publications_v3", + "products", + "supports_future_publishing", + ) + auto_publish = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="autoPublish") + catalog = sgqlc.types.Field(Catalog, graphql_name="catalog") + collection_publications_v3 = sgqlc.types.Field( + sgqlc.types.non_null(ResourcePublicationConnection), + graphql_name="collectionPublicationsV3", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + collections = sgqlc.types.Field( + sgqlc.types.non_null(CollectionConnection), + graphql_name="collections", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + has_collection = sgqlc.types.Field( + sgqlc.types.non_null(Boolean), + graphql_name="hasCollection", + args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), + ) + operation = sgqlc.types.Field("PublicationOperation", graphql_name="operation") + product_publications_v3 = sgqlc.types.Field( + sgqlc.types.non_null(ResourcePublicationConnection), + graphql_name="productPublicationsV3", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + products = sgqlc.types.Field( + sgqlc.types.non_null(ProductConnection), + graphql_name="products", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + supports_future_publishing = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="supportsFuturePublishing") + + +class PublicationResourceOperation(sgqlc.types.Type, Node, ResourceOperation): + __schema__ = shopify_schema + __field_names__ = () + + +class PublicationUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(PublicationUserErrorCode, graphql_name="code") + + +class QuantityRuleUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(QuantityRuleUserErrorCode, graphql_name="code") + + +class Refund(sgqlc.types.Type, LegacyInteroperability, Node): + __schema__ = shopify_schema + __field_names__ = ( + "created_at", + "duties", + "note", + "order", + "refund_line_items", + "return_", + "staff_member", + "total_refunded_set", + "transactions", + "updated_at", + ) + created_at = sgqlc.types.Field(DateTime, graphql_name="createdAt") + duties = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(RefundDuty)), graphql_name="duties") + note = sgqlc.types.Field(String, graphql_name="note") + order = sgqlc.types.Field(sgqlc.types.non_null(Order), graphql_name="order") + refund_line_items = sgqlc.types.Field( + sgqlc.types.non_null(RefundLineItemConnection), + graphql_name="refundLineItems", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + return_ = sgqlc.types.Field("Return", graphql_name="return") + staff_member = sgqlc.types.Field("StaffMember", graphql_name="staffMember") + total_refunded_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalRefundedSet") + transactions = sgqlc.types.Field( + sgqlc.types.non_null(OrderTransactionConnection), + graphql_name="transactions", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + + +class RefundAgreement(sgqlc.types.Type, SalesAgreement): + __schema__ = shopify_schema + __field_names__ = ("refund",) + refund = sgqlc.types.Field(sgqlc.types.non_null(Refund), graphql_name="refund") + + +class Return(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "decline", + "name", + "order", + "refunds", + "return_line_items", + "reverse_fulfillment_orders", + "status", + "suggested_refund", + "total_quantity", + ) + decline = sgqlc.types.Field(ReturnDecline, graphql_name="decline") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + order = sgqlc.types.Field(sgqlc.types.non_null(Order), graphql_name="order") + refunds = sgqlc.types.Field( + sgqlc.types.non_null(RefundConnection), + graphql_name="refunds", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + return_line_items = sgqlc.types.Field( + sgqlc.types.non_null(ReturnLineItemConnection), + graphql_name="returnLineItems", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + reverse_fulfillment_orders = sgqlc.types.Field( + sgqlc.types.non_null(ReverseFulfillmentOrderConnection), + graphql_name="reverseFulfillmentOrders", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + status = sgqlc.types.Field(sgqlc.types.non_null(ReturnStatus), graphql_name="status") + suggested_refund = sgqlc.types.Field( + SuggestedReturnRefund, + graphql_name="suggestedRefund", + args=sgqlc.types.ArgDict( + ( + ( + "return_refund_line_items", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ReturnRefundLineItemInput))), + graphql_name="returnRefundLineItems", + default=None, + ), + ), + ("refund_shipping", sgqlc.types.Arg(RefundShippingInput, graphql_name="refundShipping", default=None)), + ( + "refund_duties", + sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(RefundDutyInput)), graphql_name="refundDuties", default=None), + ), + ) + ), + ) + total_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalQuantity") + + +class ReturnLineItem(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "customer_note", + "fulfillment_line_item", + "quantity", + "refundable_quantity", + "refunded_quantity", + "return_reason", + "return_reason_note", + "total_weight", + "with_code_discounted_total_price_set", + ) + customer_note = sgqlc.types.Field(String, graphql_name="customerNote") + fulfillment_line_item = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentLineItem), graphql_name="fulfillmentLineItem") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + refundable_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="refundableQuantity") + refunded_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="refundedQuantity") + return_reason = sgqlc.types.Field(sgqlc.types.non_null(ReturnReason), graphql_name="returnReason") + return_reason_note = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="returnReasonNote") + total_weight = sgqlc.types.Field(Weight, graphql_name="totalWeight") + with_code_discounted_total_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="withCodeDiscountedTotalPriceSet") + + +class ReturnUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(ReturnErrorCode, graphql_name="code") + + +class ReturnableFulfillment(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("fulfillment", "returnable_fulfillment_line_items") + fulfillment = sgqlc.types.Field(sgqlc.types.non_null(Fulfillment), graphql_name="fulfillment") + returnable_fulfillment_line_items = sgqlc.types.Field( + sgqlc.types.non_null(ReturnableFulfillmentLineItemConnection), + graphql_name="returnableFulfillmentLineItems", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + + +class ReverseDelivery(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("deliverable", "reverse_delivery_line_items", "reverse_fulfillment_order") + deliverable = sgqlc.types.Field("ReverseDeliveryDeliverable", graphql_name="deliverable") + reverse_delivery_line_items = sgqlc.types.Field( + sgqlc.types.non_null(ReverseDeliveryLineItemConnection), + graphql_name="reverseDeliveryLineItems", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + reverse_fulfillment_order = sgqlc.types.Field(sgqlc.types.non_null("ReverseFulfillmentOrder"), graphql_name="reverseFulfillmentOrder") + + +class ReverseDeliveryLineItem(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("dispositions", "quantity", "reverse_fulfillment_order_line_item") + dispositions = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ReverseFulfillmentOrderDisposition"))), graphql_name="dispositions" + ) + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + reverse_fulfillment_order_line_item = sgqlc.types.Field( + sgqlc.types.non_null("ReverseFulfillmentOrderLineItem"), graphql_name="reverseFulfillmentOrderLineItem" + ) + + +class ReverseFulfillmentOrder(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("line_items", "order", "reverse_deliveries", "status", "third_party_confirmation") + line_items = sgqlc.types.Field( + sgqlc.types.non_null(ReverseFulfillmentOrderLineItemConnection), + graphql_name="lineItems", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + order = sgqlc.types.Field(sgqlc.types.non_null(Order), graphql_name="order") + reverse_deliveries = sgqlc.types.Field( + sgqlc.types.non_null(ReverseDeliveryConnection), + graphql_name="reverseDeliveries", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + status = sgqlc.types.Field(sgqlc.types.non_null(ReverseFulfillmentOrderStatus), graphql_name="status") + third_party_confirmation = sgqlc.types.Field(ReverseFulfillmentOrderThirdPartyConfirmation, graphql_name="thirdPartyConfirmation") + + +class ReverseFulfillmentOrderDisposition(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("location", "quantity", "type") + location = sgqlc.types.Field(Location, graphql_name="location") + quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") + type = sgqlc.types.Field(sgqlc.types.non_null(ReverseFulfillmentOrderDispositionType), graphql_name="type") + + +class ReverseFulfillmentOrderLineItem(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("dispositions", "fulfillment_line_item", "total_quantity") + dispositions = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ReverseFulfillmentOrderDisposition))), graphql_name="dispositions" + ) + fulfillment_line_item = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentLineItem), graphql_name="fulfillmentLineItem") + total_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalQuantity") + + +class SaleAdditionalFee(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("name", "price", "tax_lines") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + price = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="price") + tax_lines = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxLine))), graphql_name="taxLines") + + +class SavedSearch(sgqlc.types.Type, LegacyInteroperability, Node): + __schema__ = shopify_schema + __field_names__ = ("filters", "name", "query", "resource_type", "search_terms") + filters = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SearchFilter))), graphql_name="filters") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + query = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="query") + resource_type = sgqlc.types.Field(sgqlc.types.non_null(SearchResultType), graphql_name="resourceType") + search_terms = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="searchTerms") + + +class ScriptDiscountApplication(sgqlc.types.Type, DiscountApplication): + __schema__ = shopify_schema + __field_names__ = ("title",) + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + + +class ScriptTag(sgqlc.types.Type, LegacyInteroperability, Node): + __schema__ = shopify_schema + __field_names__ = ("cache", "created_at", "display_scope", "src", "updated_at") + cache = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="cache") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + display_scope = sgqlc.types.Field(sgqlc.types.non_null(ScriptTagDisplayScope), graphql_name="displayScope") + src = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="src") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + + +class Segment(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("creation_date", "last_edit_date", "name", "query") + creation_date = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="creationDate") + last_edit_date = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="lastEditDate") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + query = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="query") + + +class SegmentAssociationFilter(sgqlc.types.Type, SegmentFilter): + __schema__ = shopify_schema + __field_names__ = () + + +class SegmentBooleanFilter(sgqlc.types.Type, SegmentFilter): + __schema__ = shopify_schema + __field_names__ = () + + +class SegmentDateFilter(sgqlc.types.Type, SegmentFilter): + __schema__ = shopify_schema + __field_names__ = () + + +class SegmentEnumFilter(sgqlc.types.Type, SegmentFilter): + __schema__ = shopify_schema + __field_names__ = () + + +class SegmentEventFilter(sgqlc.types.Type, SegmentFilter): + __schema__ = shopify_schema + __field_names__ = ("parameters", "return_value_type") + parameters = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SegmentEventFilterParameter))), graphql_name="parameters" + ) + return_value_type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="returnValueType") + + +class SegmentFloatFilter(sgqlc.types.Type, SegmentFilter): + __schema__ = shopify_schema + __field_names__ = () + + +class SegmentIntegerFilter(sgqlc.types.Type, SegmentFilter): + __schema__ = shopify_schema + __field_names__ = () + + +class SegmentStringFilter(sgqlc.types.Type, SegmentFilter): + __schema__ = shopify_schema + __field_names__ = () + + +class SellingPlan(sgqlc.types.Type, HasPublishedTranslations, Node): + __schema__ = shopify_schema + __field_names__ = ( + "billing_policy", + "category", + "created_at", + "delivery_policy", + "description", + "inventory_policy", + "name", + "options", + "position", + "pricing_policies", + ) + billing_policy = sgqlc.types.Field(sgqlc.types.non_null("SellingPlanBillingPolicy"), graphql_name="billingPolicy") + category = sgqlc.types.Field(SellingPlanCategory, graphql_name="category") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + delivery_policy = sgqlc.types.Field(sgqlc.types.non_null("SellingPlanDeliveryPolicy"), graphql_name="deliveryPolicy") + description = sgqlc.types.Field(String, graphql_name="description") + inventory_policy = sgqlc.types.Field(SellingPlanInventoryPolicy, graphql_name="inventoryPolicy") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + options = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="options") + position = sgqlc.types.Field(Int, graphql_name="position") + pricing_policies = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanPricingPolicy"))), graphql_name="pricingPolicies" + ) + + +class SellingPlanFixedPricingPolicy(sgqlc.types.Type, SellingPlanPricingPolicyBase): + __schema__ = shopify_schema + __field_names__ = ("created_at",) + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + + +class SellingPlanGroup(sgqlc.types.Type, HasPublishedTranslations, Node): + __schema__ = shopify_schema + __field_names__ = ( + "app_id", + "applies_to_product", + "applies_to_product_variant", + "applies_to_product_variants", + "created_at", + "description", + "merchant_code", + "name", + "options", + "position", + "product_count", + "product_variant_count", + "product_variants", + "products", + "selling_plans", + "summary", + ) + app_id = sgqlc.types.Field(String, graphql_name="appId") + applies_to_product = sgqlc.types.Field( + sgqlc.types.non_null(Boolean), + graphql_name="appliesToProduct", + args=sgqlc.types.ArgDict((("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)),)), + ) + applies_to_product_variant = sgqlc.types.Field( + sgqlc.types.non_null(Boolean), + graphql_name="appliesToProductVariant", + args=sgqlc.types.ArgDict( + (("product_variant_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productVariantId", default=None)),) + ), + ) + applies_to_product_variants = sgqlc.types.Field( + sgqlc.types.non_null(Boolean), + graphql_name="appliesToProductVariants", + args=sgqlc.types.ArgDict((("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)),)), + ) + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + description = sgqlc.types.Field(String, graphql_name="description") + merchant_code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="merchantCode") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + options = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="options") + position = sgqlc.types.Field(Int, graphql_name="position") + product_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="productCount") + product_variant_count = sgqlc.types.Field( + sgqlc.types.non_null(Int), + graphql_name="productVariantCount", + args=sgqlc.types.ArgDict((("product_id", sgqlc.types.Arg(ID, graphql_name="productId", default=None)),)), + ) + product_variants = sgqlc.types.Field( + sgqlc.types.non_null(ProductVariantConnection), + graphql_name="productVariants", + args=sgqlc.types.ArgDict( + ( + ("product_id", sgqlc.types.Arg(ID, graphql_name="productId", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + products = sgqlc.types.Field( + sgqlc.types.non_null(ProductConnection), + graphql_name="products", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + selling_plans = sgqlc.types.Field( + sgqlc.types.non_null(SellingPlanConnection), + graphql_name="sellingPlans", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + summary = sgqlc.types.Field(String, graphql_name="summary") + + +class SellingPlanGroupUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(SellingPlanGroupUserErrorCode, graphql_name="code") + + +class SellingPlanRecurringPricingPolicy(sgqlc.types.Type, SellingPlanPricingPolicyBase): + __schema__ = shopify_schema + __field_names__ = ("after_cycle", "created_at") + after_cycle = sgqlc.types.Field(Int, graphql_name="afterCycle") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + + +class ServerPixel(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("status", "webhook_endpoint_address") + status = sgqlc.types.Field(ServerPixelStatus, graphql_name="status") + webhook_endpoint_address = sgqlc.types.Field(String, graphql_name="webhookEndpointAddress") + + +class ShippingLineSale(sgqlc.types.Type, Sale): + __schema__ = shopify_schema + __field_names__ = ("shipping_line",) + shipping_line = sgqlc.types.Field(ShippingLine, graphql_name="shippingLine") + + +class Shop(sgqlc.types.Type, HasMetafields, HasPublishedTranslations, Node): + __schema__ = shopify_schema + __field_names__ = ( + "alerts", + "all_product_categories", + "assigned_fulfillment_orders", + "available_channel_apps", + "billing_address", + "channel_definitions_for_installed_channels", + "checkout_api_supported", + "contact_email", + "countries_in_shipping_zones", + "currency_code", + "currency_formats", + "currency_settings", + "customer_accounts", + "customer_tags", + "description", + "draft_order_tags", + "email", + "enabled_presentment_currencies", + "features", + "fulfillment_services", + "iana_timezone", + "limited_pending_order_count", + "merchant_approval_signals", + "myshopify_domain", + "name", + "navigation_settings", + "order_number_format_prefix", + "order_number_format_suffix", + "order_tags", + "payment_settings", + "plan", + "primary_domain", + "product_images", + "product_tags", + "product_types", + "product_vendors", + "publication_count", + "resource_limits", + "rich_text_editor_url", + "search", + "search_filters", + "setup_required", + "ships_to_countries", + "shop_policies", + "staff_members", + "storefront_access_tokens", + "tax_shipping", + "taxes_included", + "timezone_abbreviation", + "timezone_offset", + "timezone_offset_minutes", + "transactional_sms_disabled", + "unit_system", + "uploaded_images_by_ids", + "url", + "weight_unit", + ) + alerts = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ShopAlert))), graphql_name="alerts") + all_product_categories = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProductCategory))), graphql_name="allProductCategories" + ) + assigned_fulfillment_orders = sgqlc.types.Field( + sgqlc.types.non_null(FulfillmentOrderConnection), + graphql_name="assignedFulfillmentOrders", + args=sgqlc.types.ArgDict( + ( + ("assignment_status", sgqlc.types.Arg(FulfillmentOrderAssignmentStatus, graphql_name="assignmentStatus", default=None)), + ("location_ids", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="locationIds", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(FulfillmentOrderSortKeys, graphql_name="sortKey", default="ID")), + ) + ), + ) + available_channel_apps = sgqlc.types.Field( + sgqlc.types.non_null(AppConnection), + graphql_name="availableChannelApps", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + billing_address = sgqlc.types.Field(sgqlc.types.non_null("ShopAddress"), graphql_name="billingAddress") + channel_definitions_for_installed_channels = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(AvailableChannelDefinitionsByChannel))), + graphql_name="channelDefinitionsForInstalledChannels", + ) + checkout_api_supported = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="checkoutApiSupported") + contact_email = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="contactEmail") + countries_in_shipping_zones = sgqlc.types.Field(sgqlc.types.non_null(CountriesInShippingZones), graphql_name="countriesInShippingZones") + currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currencyCode") + currency_formats = sgqlc.types.Field(sgqlc.types.non_null(CurrencyFormats), graphql_name="currencyFormats") + currency_settings = sgqlc.types.Field( + sgqlc.types.non_null(CurrencySettingConnection), + graphql_name="currencySettings", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + customer_accounts = sgqlc.types.Field(sgqlc.types.non_null(ShopCustomerAccountsSetting), graphql_name="customerAccounts") + customer_tags = sgqlc.types.Field( + sgqlc.types.non_null(StringConnection), + graphql_name="customerTags", + args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="first", default=None)),)), + ) + description = sgqlc.types.Field(String, graphql_name="description") + draft_order_tags = sgqlc.types.Field( + sgqlc.types.non_null(StringConnection), + graphql_name="draftOrderTags", + args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="first", default=None)),)), + ) + email = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="email") + enabled_presentment_currencies = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CurrencyCode))), graphql_name="enabledPresentmentCurrencies" + ) + features = sgqlc.types.Field(sgqlc.types.non_null(ShopFeatures), graphql_name="features") + fulfillment_services = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FulfillmentService))), graphql_name="fulfillmentServices" + ) + iana_timezone = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="ianaTimezone") + limited_pending_order_count = sgqlc.types.Field(sgqlc.types.non_null(LimitedPendingOrderCount), graphql_name="limitedPendingOrderCount") + merchant_approval_signals = sgqlc.types.Field(MerchantApprovalSignals, graphql_name="merchantApprovalSignals") + myshopify_domain = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="myshopifyDomain") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + navigation_settings = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(NavigationItem))), graphql_name="navigationSettings" + ) + order_number_format_prefix = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="orderNumberFormatPrefix") + order_number_format_suffix = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="orderNumberFormatSuffix") + order_tags = sgqlc.types.Field( + sgqlc.types.non_null(StringConnection), + graphql_name="orderTags", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="first", default=None)), + ("sort", sgqlc.types.Arg(ShopTagSort, graphql_name="sort", default="ALPHABETICAL")), + ) + ), + ) + payment_settings = sgqlc.types.Field(sgqlc.types.non_null(PaymentSettings), graphql_name="paymentSettings") + plan = sgqlc.types.Field(sgqlc.types.non_null(ShopPlan), graphql_name="plan") + primary_domain = sgqlc.types.Field(sgqlc.types.non_null(Domain), graphql_name="primaryDomain") + product_images = sgqlc.types.Field( + sgqlc.types.non_null(ImageConnection), + graphql_name="productImages", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(ProductImageSortKeys, graphql_name="sortKey", default="CREATED_AT")), + ) + ), + ) + product_tags = sgqlc.types.Field( + sgqlc.types.non_null(StringConnection), + graphql_name="productTags", + args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="first", default=None)),)), + ) + product_types = sgqlc.types.Field( + sgqlc.types.non_null(StringConnection), + graphql_name="productTypes", + args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="first", default=None)),)), + ) + product_vendors = sgqlc.types.Field( + sgqlc.types.non_null(StringConnection), + graphql_name="productVendors", + args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="first", default=None)),)), + ) + publication_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="publicationCount") + resource_limits = sgqlc.types.Field(sgqlc.types.non_null(ShopResourceLimits), graphql_name="resourceLimits") + rich_text_editor_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="richTextEditorUrl") + search = sgqlc.types.Field( + sgqlc.types.non_null(SearchResultConnection), + graphql_name="search", + args=sgqlc.types.ArgDict( + ( + ("query", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="query", default=None)), + ("types", sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(SearchResultType)), graphql_name="types", default=None)), + ("first", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ) + ), + ) + search_filters = sgqlc.types.Field(sgqlc.types.non_null(SearchFilterOptions), graphql_name="searchFilters") + setup_required = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="setupRequired") + ships_to_countries = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CountryCode))), graphql_name="shipsToCountries" + ) + shop_policies = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopPolicy"))), graphql_name="shopPolicies" + ) + staff_members = sgqlc.types.Field( + sgqlc.types.non_null(StaffMemberConnection), + graphql_name="staffMembers", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + storefront_access_tokens = sgqlc.types.Field( + sgqlc.types.non_null(StorefrontAccessTokenConnection), + graphql_name="storefrontAccessTokens", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + tax_shipping = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxShipping") + taxes_included = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxesIncluded") + timezone_abbreviation = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="timezoneAbbreviation") + timezone_offset = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="timezoneOffset") + timezone_offset_minutes = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="timezoneOffsetMinutes") + transactional_sms_disabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="transactionalSmsDisabled") + unit_system = sgqlc.types.Field(sgqlc.types.non_null(UnitSystem), graphql_name="unitSystem") + uploaded_images_by_ids = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Image))), + graphql_name="uploadedImagesByIds", + args=sgqlc.types.ArgDict( + ( + ( + "image_ids", + sgqlc.types.Arg( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="imageIds", default=None + ), + ), + ) + ), + ) + url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") + weight_unit = sgqlc.types.Field(sgqlc.types.non_null(WeightUnit), graphql_name="weightUnit") + + +class ShopAddress(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "address1", + "address2", + "city", + "company", + "coordinates_validated", + "country", + "country_code_v2", + "formatted", + "formatted_area", + "latitude", + "longitude", + "phone", + "province", + "province_code", + "zip", + ) + address1 = sgqlc.types.Field(String, graphql_name="address1") + address2 = sgqlc.types.Field(String, graphql_name="address2") + city = sgqlc.types.Field(String, graphql_name="city") + company = sgqlc.types.Field(String, graphql_name="company") + coordinates_validated = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="coordinatesValidated") + country = sgqlc.types.Field(String, graphql_name="country") + country_code_v2 = sgqlc.types.Field(CountryCode, graphql_name="countryCodeV2") + formatted = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), + graphql_name="formatted", + args=sgqlc.types.ArgDict((("with_company", sgqlc.types.Arg(Boolean, graphql_name="withCompany", default=True)),)), + ) + formatted_area = sgqlc.types.Field(String, graphql_name="formattedArea") + latitude = sgqlc.types.Field(Float, graphql_name="latitude") + longitude = sgqlc.types.Field(Float, graphql_name="longitude") + phone = sgqlc.types.Field(String, graphql_name="phone") + province = sgqlc.types.Field(String, graphql_name="province") + province_code = sgqlc.types.Field(String, graphql_name="provinceCode") + zip = sgqlc.types.Field(String, graphql_name="zip") + + +class ShopPolicy(sgqlc.types.Type, HasPublishedTranslations, Node): + __schema__ = shopify_schema + __field_names__ = ("body", "type", "url") + body = sgqlc.types.Field(sgqlc.types.non_null(HTML), graphql_name="body") + type = sgqlc.types.Field(sgqlc.types.non_null(ShopPolicyType), graphql_name="type") + url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") + + +class ShopPolicyUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(ShopPolicyErrorCode, graphql_name="code") + + +class ShopResourceFeedbackCreateUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(ShopResourceFeedbackCreateUserErrorCode, graphql_name="code") + + +class ShopifyPaymentsAccount(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "activated", + "balance", + "bank_accounts", + "charge_statement_descriptors", + "country", + "default_currency", + "disputes", + "fraud_settings", + "notification_settings", + "onboardable", + "payout_schedule", + "payout_statement_descriptor", + "payouts", + "permitted_verification_documents", + "verifications", + ) + activated = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="activated") + balance = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MoneyV2))), graphql_name="balance") + bank_accounts = sgqlc.types.Field( + sgqlc.types.non_null(ShopifyPaymentsBankAccountConnection), + graphql_name="bankAccounts", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + charge_statement_descriptors = sgqlc.types.Field(ShopifyPaymentsChargeStatementDescriptor, graphql_name="chargeStatementDescriptors") + country = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="country") + default_currency = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="defaultCurrency") + disputes = sgqlc.types.Field( + sgqlc.types.non_null(ShopifyPaymentsDisputeConnection), + graphql_name="disputes", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), + ) + ), + ) + fraud_settings = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsFraudSettings), graphql_name="fraudSettings") + notification_settings = sgqlc.types.Field( + sgqlc.types.non_null(ShopifyPaymentsNotificationSettings), graphql_name="notificationSettings" + ) + onboardable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="onboardable") + payout_schedule = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsPayoutSchedule), graphql_name="payoutSchedule") + payout_statement_descriptor = sgqlc.types.Field(String, graphql_name="payoutStatementDescriptor") + payouts = sgqlc.types.Field( + sgqlc.types.non_null(ShopifyPaymentsPayoutConnection), + graphql_name="payouts", + args=sgqlc.types.ArgDict( + ( + ("transaction_type", sgqlc.types.Arg(ShopifyPaymentsPayoutTransactionType, graphql_name="transactionType", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + permitted_verification_documents = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ShopifyPaymentsVerificationDocument))), + graphql_name="permittedVerificationDocuments", + ) + verifications = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopifyPaymentsVerification"))), graphql_name="verifications" + ) + + +class ShopifyPaymentsBankAccount(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "account_number", + "account_number_last_digits", + "bank_name", + "country", + "created_at", + "currency", + "payouts", + "routing_number", + "status", + ) + account_number = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="accountNumber") + account_number_last_digits = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="accountNumberLastDigits") + bank_name = sgqlc.types.Field(String, graphql_name="bankName") + country = sgqlc.types.Field(sgqlc.types.non_null(CountryCode), graphql_name="country") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + currency = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currency") + payouts = sgqlc.types.Field( + sgqlc.types.non_null(ShopifyPaymentsPayoutConnection), + graphql_name="payouts", + args=sgqlc.types.ArgDict( + ( + ("transaction_type", sgqlc.types.Arg(ShopifyPaymentsPayoutTransactionType, graphql_name="transactionType", default=None)), + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + routing_number = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="routingNumber") + status = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsBankAccountStatus), graphql_name="status") + + +class ShopifyPaymentsDefaultChargeStatementDescriptor(sgqlc.types.Type, ShopifyPaymentsChargeStatementDescriptor): + __schema__ = shopify_schema + __field_names__ = () + + +class ShopifyPaymentsDispute(sgqlc.types.Type, LegacyInteroperability, Node): + __schema__ = shopify_schema + __field_names__ = ( + "amount", + "evidence_due_by", + "evidence_sent_on", + "finalized_on", + "initiated_at", + "order", + "reason_details", + "status", + "type", + ) + amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="amount") + evidence_due_by = sgqlc.types.Field(Date, graphql_name="evidenceDueBy") + evidence_sent_on = sgqlc.types.Field(Date, graphql_name="evidenceSentOn") + finalized_on = sgqlc.types.Field(Date, graphql_name="finalizedOn") + initiated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="initiatedAt") + order = sgqlc.types.Field(Order, graphql_name="order") + reason_details = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsDisputeReasonDetails), graphql_name="reasonDetails") + status = sgqlc.types.Field(sgqlc.types.non_null(DisputeStatus), graphql_name="status") + type = sgqlc.types.Field(sgqlc.types.non_null(DisputeType), graphql_name="type") + + +class ShopifyPaymentsDisputeEvidence(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "access_activity_log", + "billing_address", + "cancellation_policy_disclosure", + "cancellation_policy_file", + "cancellation_rebuttal", + "customer_communication_file", + "customer_email_address", + "customer_first_name", + "customer_last_name", + "customer_purchase_ip", + "dispute", + "dispute_file_uploads", + "fulfillments", + "product_description", + "refund_policy_disclosure", + "refund_policy_file", + "refund_refusal_explanation", + "service_documentation_file", + "shipping_address", + "shipping_documentation_file", + "submitted", + "uncategorized_file", + "uncategorized_text", + ) + access_activity_log = sgqlc.types.Field(String, graphql_name="accessActivityLog") + billing_address = sgqlc.types.Field(MailingAddress, graphql_name="billingAddress") + cancellation_policy_disclosure = sgqlc.types.Field(String, graphql_name="cancellationPolicyDisclosure") + cancellation_policy_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUpload", graphql_name="cancellationPolicyFile") + cancellation_rebuttal = sgqlc.types.Field(String, graphql_name="cancellationRebuttal") + customer_communication_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUpload", graphql_name="customerCommunicationFile") + customer_email_address = sgqlc.types.Field(String, graphql_name="customerEmailAddress") + customer_first_name = sgqlc.types.Field(String, graphql_name="customerFirstName") + customer_last_name = sgqlc.types.Field(String, graphql_name="customerLastName") + customer_purchase_ip = sgqlc.types.Field(String, graphql_name="customerPurchaseIp") + dispute = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsDispute), graphql_name="dispute") + dispute_file_uploads = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopifyPaymentsDisputeFileUpload"))), + graphql_name="disputeFileUploads", + ) + fulfillments = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopifyPaymentsDisputeFulfillment"))), graphql_name="fulfillments" + ) + product_description = sgqlc.types.Field(String, graphql_name="productDescription") + refund_policy_disclosure = sgqlc.types.Field(String, graphql_name="refundPolicyDisclosure") + refund_policy_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUpload", graphql_name="refundPolicyFile") + refund_refusal_explanation = sgqlc.types.Field(String, graphql_name="refundRefusalExplanation") + service_documentation_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUpload", graphql_name="serviceDocumentationFile") + shipping_address = sgqlc.types.Field(MailingAddress, graphql_name="shippingAddress") + shipping_documentation_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUpload", graphql_name="shippingDocumentationFile") + submitted = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="submitted") + uncategorized_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUpload", graphql_name="uncategorizedFile") + uncategorized_text = sgqlc.types.Field(String, graphql_name="uncategorizedText") + + +class ShopifyPaymentsDisputeFileUpload(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("dispute_evidence_type", "file_size", "file_type", "original_file_name", "url") + dispute_evidence_type = sgqlc.types.Field(ShopifyPaymentsDisputeEvidenceFileType, graphql_name="disputeEvidenceType") + file_size = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="fileSize") + file_type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="fileType") + original_file_name = sgqlc.types.Field(String, graphql_name="originalFileName") + url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") + + +class ShopifyPaymentsDisputeFulfillment(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("shipping_carrier", "shipping_date", "shipping_tracking_number") + shipping_carrier = sgqlc.types.Field(String, graphql_name="shippingCarrier") + shipping_date = sgqlc.types.Field(Date, graphql_name="shippingDate") + shipping_tracking_number = sgqlc.types.Field(String, graphql_name="shippingTrackingNumber") + + +class ShopifyPaymentsJpChargeStatementDescriptor(sgqlc.types.Type, ShopifyPaymentsChargeStatementDescriptor): + __schema__ = shopify_schema + __field_names__ = ("kana", "kanji") + kana = sgqlc.types.Field(String, graphql_name="kana") + kanji = sgqlc.types.Field(String, graphql_name="kanji") + + +class ShopifyPaymentsPayout(sgqlc.types.Type, LegacyInteroperability, Node): + __schema__ = shopify_schema + __field_names__ = ("bank_account", "issued_at", "net", "status", "summary", "transaction_type") + bank_account = sgqlc.types.Field(ShopifyPaymentsBankAccount, graphql_name="bankAccount") + issued_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="issuedAt") + net = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="net") + status = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsPayoutStatus), graphql_name="status") + summary = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsPayoutSummary), graphql_name="summary") + transaction_type = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsPayoutTransactionType), graphql_name="transactionType") + + +class ShopifyPaymentsVerification(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("status", "subject") + status = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsVerificationStatus), graphql_name="status") + subject = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsVerificationSubject), graphql_name="subject") + + +class StaffMember(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "active", + "avatar", + "email", + "exists", + "first_name", + "initials", + "is_shop_owner", + "last_name", + "locale", + "name", + "phone", + "private_data", + ) + active = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="active") + avatar = sgqlc.types.Field( + sgqlc.types.non_null(Image), + graphql_name="avatar", + args=sgqlc.types.ArgDict((("fallback", sgqlc.types.Arg(StaffMemberDefaultImage, graphql_name="fallback", default="DEFAULT")),)), + ) + email = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="email") + exists = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="exists") + first_name = sgqlc.types.Field(String, graphql_name="firstName") + initials = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="initials") + is_shop_owner = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isShopOwner") + last_name = sgqlc.types.Field(String, graphql_name="lastName") + locale = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="locale") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + phone = sgqlc.types.Field(String, graphql_name="phone") + private_data = sgqlc.types.Field(sgqlc.types.non_null(StaffMemberPrivateData), graphql_name="privateData") + + +class StandardMetafieldDefinitionEnableUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(StandardMetafieldDefinitionEnableUserErrorCode, graphql_name="code") + + +class StandardMetafieldDefinitionTemplate(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("description", "key", "name", "namespace", "owner_types", "type", "validations", "visible_to_storefront_api") + description = sgqlc.types.Field(String, graphql_name="description") + key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") + name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") + namespace = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="namespace") + owner_types = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MetafieldOwnerType))), graphql_name="ownerTypes" + ) + type = sgqlc.types.Field(sgqlc.types.non_null(MetafieldDefinitionType), graphql_name="type") + validations = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MetafieldDefinitionValidation))), graphql_name="validations" + ) + visible_to_storefront_api = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="visibleToStorefrontApi") + + +class StorefrontAccessToken(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("access_scopes", "access_token", "created_at", "title", "updated_at") + access_scopes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(AccessScope))), graphql_name="accessScopes" + ) + access_token = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="accessToken") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + + +class SubscriptionBillingAttempt(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "completed_at", + "created_at", + "error_code", + "error_message", + "idempotency_key", + "next_action_url", + "order", + "origin_time", + "ready", + "subscription_contract", + ) + completed_at = sgqlc.types.Field(DateTime, graphql_name="completedAt") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + error_code = sgqlc.types.Field(SubscriptionBillingAttemptErrorCode, graphql_name="errorCode") + error_message = sgqlc.types.Field(String, graphql_name="errorMessage") + idempotency_key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="idempotencyKey") + next_action_url = sgqlc.types.Field(URL, graphql_name="nextActionUrl") + order = sgqlc.types.Field(Order, graphql_name="order") + origin_time = sgqlc.types.Field(DateTime, graphql_name="originTime") + ready = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="ready") + subscription_contract = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionContract"), graphql_name="subscriptionContract") + + +class SubscriptionBillingCycleEditedContract(sgqlc.types.Type, SubscriptionContractBase): + __schema__ = shopify_schema + __field_names__ = ("billing_cycles", "created_at") + billing_cycles = sgqlc.types.Field( + sgqlc.types.non_null(SubscriptionBillingCycleConnection), + graphql_name="billingCycles", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(SubscriptionBillingCyclesSortKeys, graphql_name="sortKey", default="CYCLE_INDEX")), + ) + ), + ) + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + + +class SubscriptionBillingCycleUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(SubscriptionBillingCycleErrorCode, graphql_name="code") + + +class SubscriptionContract(sgqlc.types.Type, Node, SubscriptionContractBase): + __schema__ = shopify_schema + __field_names__ = ( + "billing_attempts", + "billing_policy", + "created_at", + "delivery_policy", + "last_payment_status", + "next_billing_date", + "origin_order", + "revision_id", + "status", + ) + billing_attempts = sgqlc.types.Field( + sgqlc.types.non_null(SubscriptionBillingAttemptConnection), + graphql_name="billingAttempts", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + billing_policy = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionBillingPolicy), graphql_name="billingPolicy") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + delivery_policy = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionDeliveryPolicy), graphql_name="deliveryPolicy") + last_payment_status = sgqlc.types.Field(SubscriptionContractLastPaymentStatus, graphql_name="lastPaymentStatus") + next_billing_date = sgqlc.types.Field(DateTime, graphql_name="nextBillingDate") + origin_order = sgqlc.types.Field(Order, graphql_name="originOrder") + revision_id = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="revisionId") + status = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionContractSubscriptionStatus), graphql_name="status") + + +class SubscriptionContractUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(SubscriptionContractErrorCode, graphql_name="code") + + +class SubscriptionDraft(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ( + "billing_cycle", + "billing_policy", + "concatenated_billing_cycles", + "currency_code", + "custom_attributes", + "customer", + "customer_payment_method", + "delivery_method", + "delivery_options", + "delivery_policy", + "delivery_price", + "discounts", + "discounts_added", + "discounts_removed", + "discounts_updated", + "lines", + "lines_added", + "lines_removed", + "next_billing_date", + "note", + "original_contract", + "status", + ) + billing_cycle = sgqlc.types.Field(SubscriptionBillingCycle, graphql_name="billingCycle") + billing_policy = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionBillingPolicy), graphql_name="billingPolicy") + concatenated_billing_cycles = sgqlc.types.Field( + sgqlc.types.non_null(SubscriptionBillingCycleConnection), + graphql_name="concatenatedBillingCycles", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ("sort_key", sgqlc.types.Arg(SubscriptionBillingCyclesSortKeys, graphql_name="sortKey", default="CYCLE_INDEX")), + ) + ), + ) + currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currencyCode") + custom_attributes = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Attribute))), graphql_name="customAttributes" + ) + customer = sgqlc.types.Field(sgqlc.types.non_null(Customer), graphql_name="customer") + customer_payment_method = sgqlc.types.Field( + CustomerPaymentMethod, + graphql_name="customerPaymentMethod", + args=sgqlc.types.ArgDict((("show_revoked", sgqlc.types.Arg(Boolean, graphql_name="showRevoked", default=False)),)), + ) + delivery_method = sgqlc.types.Field("SubscriptionDeliveryMethod", graphql_name="deliveryMethod") + delivery_options = sgqlc.types.Field( + "SubscriptionDeliveryOptionResult", + graphql_name="deliveryOptions", + args=sgqlc.types.ArgDict( + (("delivery_address", sgqlc.types.Arg(MailingAddressInput, graphql_name="deliveryAddress", default=None)),) + ), + ) + delivery_policy = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionDeliveryPolicy), graphql_name="deliveryPolicy") + delivery_price = sgqlc.types.Field(MoneyV2, graphql_name="deliveryPrice") + discounts = sgqlc.types.Field( + sgqlc.types.non_null(SubscriptionDiscountConnection), + graphql_name="discounts", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + discounts_added = sgqlc.types.Field( + sgqlc.types.non_null(SubscriptionDiscountConnection), + graphql_name="discountsAdded", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + discounts_removed = sgqlc.types.Field( + sgqlc.types.non_null(SubscriptionDiscountConnection), + graphql_name="discountsRemoved", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + discounts_updated = sgqlc.types.Field( + sgqlc.types.non_null(SubscriptionDiscountConnection), + graphql_name="discountsUpdated", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + lines = sgqlc.types.Field( + sgqlc.types.non_null(SubscriptionLineConnection), + graphql_name="lines", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + lines_added = sgqlc.types.Field( + sgqlc.types.non_null(SubscriptionLineConnection), + graphql_name="linesAdded", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + lines_removed = sgqlc.types.Field( + sgqlc.types.non_null(SubscriptionLineConnection), + graphql_name="linesRemoved", + args=sgqlc.types.ArgDict( + ( + ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), + ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), + ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), + ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), + ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), + ) + ), + ) + next_billing_date = sgqlc.types.Field(DateTime, graphql_name="nextBillingDate") + note = sgqlc.types.Field(String, graphql_name="note") + original_contract = sgqlc.types.Field(SubscriptionContract, graphql_name="originalContract") + status = sgqlc.types.Field(SubscriptionContractSubscriptionStatus, graphql_name="status") + + +class SubscriptionDraftUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(SubscriptionDraftErrorCode, graphql_name="code") + + +class TableResponse(sgqlc.types.Type, ShopifyqlResponse): + __schema__ = shopify_schema + __field_names__ = () + + +class TaxAppConfigureUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(TaxAppConfigureUserErrorCode, graphql_name="code") + + +class TenderTransaction(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("amount", "payment_method", "processed_at", "remote_reference", "test", "transaction_details", "user") + amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="amount") + payment_method = sgqlc.types.Field(String, graphql_name="paymentMethod") + processed_at = sgqlc.types.Field(DateTime, graphql_name="processedAt") + remote_reference = sgqlc.types.Field(String, graphql_name="remoteReference") + test = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="test") + transaction_details = sgqlc.types.Field("TenderTransactionDetails", graphql_name="transactionDetails") + user = sgqlc.types.Field(StaffMember, graphql_name="user") + + +class TipSale(sgqlc.types.Type, Sale): + __schema__ = shopify_schema + __field_names__ = ("line_item",) + line_item = sgqlc.types.Field(sgqlc.types.non_null(LineItem), graphql_name="lineItem") + + +class TransactionFee(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("amount", "flat_fee", "flat_fee_name", "rate", "rate_name", "tax_amount", "type") + amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="amount") + flat_fee = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="flatFee") + flat_fee_name = sgqlc.types.Field(String, graphql_name="flatFeeName") + rate = sgqlc.types.Field(sgqlc.types.non_null(Decimal), graphql_name="rate") + rate_name = sgqlc.types.Field(String, graphql_name="rateName") + tax_amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="taxAmount") + type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="type") + + +class TranslationUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(TranslationErrorCode, graphql_name="code") + + +class UnknownSale(sgqlc.types.Type, Sale): + __schema__ = shopify_schema + __field_names__ = () + + +class UrlRedirect(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("path", "target") + path = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="path") + target = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="target") + + +class UrlRedirectBulkDeleteByIdsUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(UrlRedirectBulkDeleteByIdsUserErrorCode, graphql_name="code") + + +class UrlRedirectBulkDeleteBySavedSearchUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(UrlRedirectBulkDeleteBySavedSearchUserErrorCode, graphql_name="code") + + +class UrlRedirectBulkDeleteBySearchUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(UrlRedirectBulkDeleteBySearchUserErrorCode, graphql_name="code") + + +class UrlRedirectImport(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("count", "created_count", "failed_count", "finished", "finished_at", "preview_redirects", "updated_count") + count = sgqlc.types.Field(Int, graphql_name="count") + created_count = sgqlc.types.Field(Int, graphql_name="createdCount") + failed_count = sgqlc.types.Field(Int, graphql_name="failedCount") + finished = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="finished") + finished_at = sgqlc.types.Field(DateTime, graphql_name="finishedAt") + preview_redirects = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(UrlRedirectImportPreview))), graphql_name="previewRedirects" + ) + updated_count = sgqlc.types.Field(Int, graphql_name="updatedCount") + + +class UrlRedirectImportUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(UrlRedirectImportErrorCode, graphql_name="code") + + +class UrlRedirectUserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = ("code",) + code = sgqlc.types.Field(UrlRedirectErrorCode, graphql_name="code") + + +class UserError(sgqlc.types.Type, DisplayableError): + __schema__ = shopify_schema + __field_names__ = () + + +class Video(sgqlc.types.Type, File, Media, Node): + __schema__ = shopify_schema + __field_names__ = ("duration", "filename", "original_source", "sources") + duration = sgqlc.types.Field(Int, graphql_name="duration") + filename = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="filename") + original_source = sgqlc.types.Field(VideoSource, graphql_name="originalSource") + sources = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(VideoSource))), graphql_name="sources") + + +class WebPixel(sgqlc.types.Type, Node): + __schema__ = shopify_schema + __field_names__ = ("settings",) + settings = sgqlc.types.Field(sgqlc.types.non_null(JSON), graphql_name="settings") + + +class WebhookSubscription(sgqlc.types.Type, LegacyInteroperability, Node): + __schema__ = shopify_schema + __field_names__ = ("api_version", "created_at", "endpoint", "format", "include_fields", "metafield_namespaces", "topic", "updated_at") + api_version = sgqlc.types.Field(sgqlc.types.non_null(ApiVersion), graphql_name="apiVersion") + created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") + endpoint = sgqlc.types.Field(sgqlc.types.non_null("WebhookSubscriptionEndpoint"), graphql_name="endpoint") + format = sgqlc.types.Field(sgqlc.types.non_null(WebhookSubscriptionFormat), graphql_name="format") + include_fields = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="includeFields" + ) + metafield_namespaces = sgqlc.types.Field( + sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="metafieldNamespaces" + ) + topic = sgqlc.types.Field(sgqlc.types.non_null(WebhookSubscriptionTopic), graphql_name="topic") + updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") + + +######################################################################## +# Unions +######################################################################## +class AppPricingDetails(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (AppRecurringPricing, AppUsagePricing) + + +class AppSubscriptionDiscountValue(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (AppSubscriptionDiscountAmount, AppSubscriptionDiscountPercentage) + + +class CollectionRuleConditionObject(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (CollectionRuleMetafieldCondition, CollectionRuleProductCategoryCondition, CollectionRuleTextCondition) + + +class CollectionRuleConditionsRuleObject(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (CollectionRuleMetafieldCondition,) + + +class CommentEventEmbed(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (Customer, DraftOrder, Order, Product, ProductVariant) + + +class CustomerPaymentInstrument(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (CustomerCreditCard, CustomerPaypalBillingAgreement, CustomerShopPayAgreement) + + +class DeliveryConditionCriteria(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (MoneyV2, Weight) + + +class DeliveryRateProvider(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (DeliveryParticipant, DeliveryRateDefinition) + + +class Discount(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = ( + DiscountAutomaticApp, + DiscountAutomaticBasic, + DiscountAutomaticBxgy, + DiscountCodeApp, + DiscountCodeBasic, + DiscountCodeBxgy, + DiscountCodeFreeShipping, + ) + + +class DiscountAutomatic(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (DiscountAutomaticApp, DiscountAutomaticBasic, DiscountAutomaticBxgy) + + +class DiscountCode(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (DiscountCodeApp, DiscountCodeBasic, DiscountCodeBxgy, DiscountCodeFreeShipping) + + +class DiscountCustomerBuysValue(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (DiscountPurchaseAmount, DiscountQuantity) + + +class DiscountCustomerGetsValue(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (DiscountAmount, DiscountOnQuantity, DiscountPercentage) + + +class DiscountCustomerSelection(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (DiscountCustomerAll, DiscountCustomerSegments, DiscountCustomers) + + +class DiscountEffect(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (DiscountPercentage,) + + +class DiscountItems(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (AllDiscountItems, DiscountCollections, DiscountProducts) + + +class DiscountMinimumRequirement(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (DiscountMinimumQuantity, DiscountMinimumSubtotal) + + +class DiscountShippingDestinationSelection(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (DiscountCountries, DiscountCountryAll) + + +class MetafieldReference(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (Collection, GenericFile, MediaImage, Metaobject, OnlineStorePage, Product, ProductVariant, Video) + + +class MetafieldReferencer(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = ( + AppInstallation, + Collection, + Customer, + DeliveryCustomization, + DiscountAutomaticNode, + DiscountCodeNode, + DiscountNode, + DraftOrder, + FulfillmentOrder, + Location, + Market, + Metaobject, + OnlineStoreArticle, + OnlineStoreBlog, + OnlineStorePage, + Order, + PaymentCustomization, + Product, + ProductVariant, + Shop, + ) + + +class OrderStagedChange(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = ( + OrderStagedChangeAddCustomItem, + OrderStagedChangeAddLineItemDiscount, + OrderStagedChangeAddShippingLine, + OrderStagedChangeAddVariant, + OrderStagedChangeDecrementItem, + OrderStagedChangeIncrementItem, + ) + + +class PaymentDetails(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (CardPaymentDetails,) + + +class PaymentInstrument(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (VaultCreditCard, VaultPaypalBillingAgreement) + + +class PriceRuleValue(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (PriceRuleFixedAmountValue, PriceRulePercentValue) + + +class PricingValue(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (MoneyV2, PricingPercentageValue) + + +class PublicationOperation(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (AddAllProductsOperation, CatalogCsvOperation, PublicationResourceOperation) + + +class PurchasingEntity(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (Customer, PurchasingCompany) + + +class ReverseDeliveryDeliverable(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (ReverseDeliveryShippingDeliverable,) + + +class SellingPlanBillingPolicy(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (SellingPlanFixedBillingPolicy, SellingPlanRecurringBillingPolicy) + + +class SellingPlanCheckoutChargeValue(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (MoneyV2, SellingPlanCheckoutChargePercentageValue) + + +class SellingPlanDeliveryPolicy(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (SellingPlanFixedDeliveryPolicy, SellingPlanRecurringDeliveryPolicy) + + +class SellingPlanPricingPolicy(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (SellingPlanFixedPricingPolicy, SellingPlanRecurringPricingPolicy) + + +class SellingPlanPricingPolicyAdjustmentValue(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (MoneyV2, SellingPlanPricingPolicyPercentageValue) + + +class SubscriptionDeliveryMethod(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (SubscriptionDeliveryMethodLocalDelivery, SubscriptionDeliveryMethodPickup, SubscriptionDeliveryMethodShipping) + + +class SubscriptionDeliveryOption(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (SubscriptionLocalDeliveryOption, SubscriptionPickupOption, SubscriptionShippingOption) + + +class SubscriptionDeliveryOptionResult(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (SubscriptionDeliveryOptionResultFailure, SubscriptionDeliveryOptionResultSuccess) + + +class SubscriptionDiscount(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (SubscriptionAppliedCodeDiscount, SubscriptionManualDiscount) + + +class SubscriptionDiscountValue(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (SubscriptionDiscountFixedAmountValue, SubscriptionDiscountPercentageValue) + + +class SubscriptionShippingOptionResult(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (SubscriptionShippingOptionResultFailure, SubscriptionShippingOptionResultSuccess) + + +class TenderTransactionDetails(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (TenderTransactionCreditCardDetails,) + + +class WebhookSubscriptionEndpoint(sgqlc.types.Union): + __schema__ = shopify_schema + __types__ = (WebhookEventBridgeEndpoint, WebhookHttpEndpoint, WebhookPubSubEndpoint) + + +######################################################################## +# Schema Entry Points +######################################################################## +shopify_schema.query_type = QueryRoot +shopify_schema.mutation_type = Mutation +shopify_schema.subscription_type = None diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_schema.py b/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_schema.py deleted file mode 100644 index 81966b99fc95..000000000000 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/shopify_schema.py +++ /dev/null @@ -1,25370 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import sgqlc.types -import sgqlc.types.datetime -import sgqlc.types.relay - -shopify_schema = sgqlc.types.Schema() - - -# Unexport Node/PageInfo, let schema re-declare them -shopify_schema -= sgqlc.types.relay.Node -shopify_schema -= sgqlc.types.relay.PageInfo - - -######################################################################## -# Scalars and Enumerations -######################################################################## -class ARN(sgqlc.types.Scalar): - __schema__ = shopify_schema - - -class AppDeveloperType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("MERCHANT", "PARTNER", "SHOPIFY", "UNKNOWN") - - -class AppInstallationCategory(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CHANNEL", "POS_EMBEDDED") - - -class AppInstallationPrivacy(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("PRIVATE", "PUBLIC") - - -class AppInstallationSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("APP_TITLE", "ID", "INSTALLED_AT", "RELEVANCE") - - -class AppPricingInterval(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ANNUAL", "EVERY_30_DAYS") - - -class AppPublicCategory(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CUSTOM", "OTHER", "PRIVATE", "PUBLIC") - - -class AppPurchaseStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ACTIVE", "DECLINED", "EXPIRED", "PENDING") - - -class AppRevenueAttributionRecordCreateUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("INVALID", "TAKEN") - - -class AppRevenueAttributionRecordDeleteUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("INVALID",) - - -class AppRevenueAttributionRecordSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CREATED_AT", "ID", "RELEVANCE") - - -class AppRevenueAttributionType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("APPLICATION_PURCHASE", "APPLICATION_SUBSCRIPTION", "APPLICATION_USAGE", "OTHER") - - -class AppSubscriptionReplacementBehavior(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("APPLY_IMMEDIATELY", "APPLY_ON_NEXT_BILLING_CYCLE", "STANDARD") - - -class AppSubscriptionSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CREATED_AT", "ID", "RELEVANCE") - - -class AppSubscriptionStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ACTIVE", "CANCELLED", "DECLINED", "EXPIRED", "FROZEN", "PENDING") - - -class AppSubscriptionTrialExtendUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("SUBSCRIPTION_NOT_ACTIVE", "SUBSCRIPTION_NOT_FOUND", "TRIAL_NOT_ACTIVE") - - -class AppTransactionSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CREATED_AT", "ID", "RELEVANCE") - - -class AppUsageRecordSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CREATED_AT", "ID", "RELEVANCE") - - -class AutomaticDiscountSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CREATED_AT", "ID", "RELEVANCE") - - -class BadgeType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ATTENTION", "DEFAULT", "INFO", "SUCCESS", "WARNING") - - -class BillingAttemptUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("BLANK", "CONTRACT_NOT_FOUND", "INVALID") - - -Boolean = sgqlc.types.Boolean - - -class BulkMutationErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "INTERNAL_FILE_SERVER_ERROR", - "INVALID_MUTATION", - "INVALID_STAGED_UPLOAD_FILE", - "NO_SUCH_FILE", - "OPERATION_IN_PROGRESS", - ) - - -class BulkOperationErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ACCESS_DENIED", "INTERNAL_SERVER_ERROR", "TIMEOUT") - - -class BulkOperationStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CANCELED", "CANCELING", "COMPLETED", "CREATED", "EXPIRED", "FAILED", "RUNNING") - - -class BulkOperationType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("MUTATION", "QUERY") - - -class BulkProductResourceFeedbackCreateUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "BLANK", - "INVALID", - "LESS_THAN_OR_EQUAL_TO", - "MAXIMUM_FEEDBACK_LIMIT_EXCEEDED", - "OUTDATED_FEEDBACK", - "PRESENT", - "PRODUCT_NOT_FOUND", - ) - - -class BusinessCustomerErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "BLANK", - "FAILED_TO_DELETE", - "INTERNAL_ERROR", - "INVALID", - "INVALID_INPUT", - "LIMIT_REACHED", - "NO_INPUT", - "REQUIRED", - "RESOURCE_NOT_FOUND", - "TAKEN", - "TOO_LONG", - "UNEXPECTED_TYPE", - ) - - -class CheckoutProfileSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CREATED_AT", "ID", "IS_PUBLISHED", "RELEVANCE", "UPDATED_AT") - - -class CodeDiscountSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CREATED_AT", "ENDS_AT", "ID", "RELEVANCE", "STARTS_AT", "TITLE", "UPDATED_AT") - - -class CollectionAddProductsV2UserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CANT_ADD_TO_SMART_COLLECTION", "COLLECTION_DOES_NOT_EXIST") - - -class CollectionRuleColumn(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "IS_PRICE_REDUCED", - "PRODUCT_TAXONOMY_NODE_ID", - "TAG", - "TITLE", - "TYPE", - "VARIANT_COMPARE_AT_PRICE", - "VARIANT_INVENTORY", - "VARIANT_PRICE", - "VARIANT_TITLE", - "VARIANT_WEIGHT", - "VENDOR", - ) - - -class CollectionRuleRelation(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "CONTAINS", - "ENDS_WITH", - "EQUALS", - "GREATER_THAN", - "IS_NOT_SET", - "IS_SET", - "LESS_THAN", - "NOT_CONTAINS", - "NOT_EQUALS", - "STARTS_WITH", - ) - - -class CollectionSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ID", "RELEVANCE", "TITLE", "UPDATED_AT") - - -class CollectionSortOrder(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "ALPHA_ASC", - "ALPHA_DESC", - "BEST_SELLING", - "CREATED", - "CREATED_DESC", - "MANUAL", - "PRICE_ASC", - "PRICE_DESC", - ) - - -class CompanyAddressType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("BILLING", "SHIPPING") - - -class CompanyContactRoleAssignmentSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CREATED_AT", "ID", "LOCATION_NAME", "RELEVANCE", "UPDATED_AT") - - -class CompanyContactRoleSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CREATED_AT", "ID", "RELEVANCE", "UPDATED_AT") - - -class CompanyContactSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("COMPANY_ID", "CREATED_AT", "EMAIL", "ID", "NAME", "NAME_EMAIL", "RELEVANCE", "TITLE", "UPDATED_AT") - - -class CompanyLocationSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("COMPANY_AND_LOCATION_NAME", "COMPANY_ID", "CREATED_AT", "ID", "NAME", "RELEVANCE", "UPDATED_AT") - - -class CompanySortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CREATED_AT", "ID", "NAME", "ORDER_COUNT", "RELEVANCE", "SINCE_DATE", "TOTAL_SPENT", "UPDATED_AT") - - -class CountryCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "AC", - "AD", - "AE", - "AF", - "AG", - "AI", - "AL", - "AM", - "AN", - "AO", - "AR", - "AT", - "AU", - "AW", - "AX", - "AZ", - "BA", - "BB", - "BD", - "BE", - "BF", - "BG", - "BH", - "BI", - "BJ", - "BL", - "BM", - "BN", - "BO", - "BQ", - "BR", - "BS", - "BT", - "BV", - "BW", - "BY", - "BZ", - "CA", - "CC", - "CD", - "CF", - "CG", - "CH", - "CI", - "CK", - "CL", - "CM", - "CN", - "CO", - "CR", - "CU", - "CV", - "CW", - "CX", - "CY", - "CZ", - "DE", - "DJ", - "DK", - "DM", - "DO", - "DZ", - "EC", - "EE", - "EG", - "EH", - "ER", - "ES", - "ET", - "FI", - "FJ", - "FK", - "FO", - "FR", - "GA", - "GB", - "GD", - "GE", - "GF", - "GG", - "GH", - "GI", - "GL", - "GM", - "GN", - "GP", - "GQ", - "GR", - "GS", - "GT", - "GW", - "GY", - "HK", - "HM", - "HN", - "HR", - "HT", - "HU", - "ID", - "IE", - "IL", - "IM", - "IN", - "IO", - "IQ", - "IR", - "IS", - "IT", - "JE", - "JM", - "JO", - "JP", - "KE", - "KG", - "KH", - "KI", - "KM", - "KN", - "KP", - "KR", - "KW", - "KY", - "KZ", - "LA", - "LB", - "LC", - "LI", - "LK", - "LR", - "LS", - "LT", - "LU", - "LV", - "LY", - "MA", - "MC", - "MD", - "ME", - "MF", - "MG", - "MK", - "ML", - "MM", - "MN", - "MO", - "MQ", - "MR", - "MS", - "MT", - "MU", - "MV", - "MW", - "MX", - "MY", - "MZ", - "NA", - "NC", - "NE", - "NF", - "NG", - "NI", - "NL", - "NO", - "NP", - "NR", - "NU", - "NZ", - "OM", - "PA", - "PE", - "PF", - "PG", - "PH", - "PK", - "PL", - "PM", - "PN", - "PS", - "PT", - "PY", - "QA", - "RE", - "RO", - "RS", - "RU", - "RW", - "SA", - "SB", - "SC", - "SD", - "SE", - "SG", - "SH", - "SI", - "SJ", - "SK", - "SL", - "SM", - "SN", - "SO", - "SR", - "SS", - "ST", - "SV", - "SX", - "SY", - "SZ", - "TA", - "TC", - "TD", - "TF", - "TG", - "TH", - "TJ", - "TK", - "TL", - "TM", - "TN", - "TO", - "TR", - "TT", - "TV", - "TW", - "TZ", - "UA", - "UG", - "UM", - "US", - "UY", - "UZ", - "VA", - "VC", - "VE", - "VG", - "VN", - "VU", - "WF", - "WS", - "XK", - "YE", - "YT", - "ZA", - "ZM", - "ZW", - "ZZ", - ) - - -class CropRegion(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("BOTTOM", "CENTER", "LEFT", "RIGHT", "TOP") - - -class CurrencyCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "AED", - "AFN", - "ALL", - "AMD", - "ANG", - "AOA", - "ARS", - "AUD", - "AWG", - "AZN", - "BAM", - "BBD", - "BDT", - "BGN", - "BHD", - "BIF", - "BMD", - "BND", - "BOB", - "BRL", - "BSD", - "BTN", - "BWP", - "BYN", - "BZD", - "CAD", - "CDF", - "CHF", - "CLP", - "CNY", - "COP", - "CRC", - "CVE", - "CZK", - "DJF", - "DKK", - "DOP", - "DZD", - "EGP", - "ERN", - "ETB", - "EUR", - "FJD", - "FKP", - "GBP", - "GEL", - "GHS", - "GIP", - "GMD", - "GNF", - "GTQ", - "GYD", - "HKD", - "HNL", - "HRK", - "HTG", - "HUF", - "IDR", - "ILS", - "INR", - "IQD", - "IRR", - "ISK", - "JEP", - "JMD", - "JOD", - "JPY", - "KES", - "KGS", - "KHR", - "KID", - "KMF", - "KRW", - "KWD", - "KYD", - "KZT", - "LAK", - "LBP", - "LKR", - "LRD", - "LSL", - "LTL", - "LVL", - "LYD", - "MAD", - "MDL", - "MGA", - "MKD", - "MMK", - "MNT", - "MOP", - "MRU", - "MUR", - "MVR", - "MWK", - "MXN", - "MYR", - "MZN", - "NAD", - "NGN", - "NIO", - "NOK", - "NPR", - "NZD", - "OMR", - "PAB", - "PEN", - "PGK", - "PHP", - "PKR", - "PLN", - "PYG", - "QAR", - "RON", - "RSD", - "RUB", - "RWF", - "SAR", - "SBD", - "SCR", - "SDG", - "SEK", - "SGD", - "SHP", - "SLL", - "SOS", - "SRD", - "SSP", - "STN", - "SYP", - "SZL", - "THB", - "TJS", - "TMT", - "TND", - "TOP", - "TRY", - "TTD", - "TWD", - "TZS", - "UAH", - "UGX", - "USD", - "UYU", - "UZS", - "VED", - "VES", - "VND", - "VUV", - "WST", - "XAF", - "XCD", - "XOF", - "XPF", - "XXX", - "YER", - "ZAR", - "ZMW", - ) - - -class CustomerConsentCollectedFrom(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("OTHER", "SHOPIFY") - - -class CustomerEmailAddressMarketingState(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("INVALID", "NOT_SUBSCRIBED", "PENDING", "SUBSCRIBED", "UNSUBSCRIBED") - - -class CustomerEmailAddressOpenTrackingLevel(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("OPTED_IN", "OPTED_OUT", "UNKNOWN") - - -class CustomerEmailMarketingConsentUpdateUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("INCLUSION", "INTERNAL_ERROR", "INVALID", "MISSING_ARGUMENT") - - -class CustomerEmailMarketingState(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("INVALID", "NOT_SUBSCRIBED", "PENDING", "REDACTED", "SUBSCRIBED", "UNSUBSCRIBED") - - -class CustomerMarketingOptInLevel(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CONFIRMED_OPT_IN", "SINGLE_OPT_IN", "UNKNOWN") - - -class CustomerPaymentMethodGetUpdateUrlUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "CUSTOMER_DOES_NOT_EXIST", - "INVALID_INSTRUMENT", - "PAYMENT_METHOD_DOES_NOT_EXIST", - "TOO_MANY_REQUESTS", - ) - - -class CustomerPaymentMethodRemoteUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "AUTHORIZE_NET_NOT_ENABLED_FOR_SUBSCRIPTIONS", - "BRAINTREE_NOT_ENABLED_FOR_SUBSCRIPTIONS", - "EXACTLY_ONE_REMOTE_REFERENCE_REQUIRED", - "INVALID", - "PRESENT", - "TAKEN", - ) - - -class CustomerPaymentMethodRevocationReason(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "AUTHORIZE_NET_GATEWAY_NOT_ENABLED", - "AUTHORIZE_NET_RETURNED_NO_PAYMENT_METHOD", - "BRAINTREE_API_AUTHENTICATION_ERROR", - "BRAINTREE_GATEWAY_NOT_ENABLED", - "BRAINTREE_PAYMENT_METHOD_NOT_CARD", - "BRAINTREE_RETURNED_NO_PAYMENT_METHOD", - "FAILED_TO_UPDATE_CREDIT_CARD", - "MANUALLY_REVOKED", - "MERGED", - "STRIPE_API_AUTHENTICATION_ERROR", - "STRIPE_API_INVALID_REQUEST_ERROR", - "STRIPE_GATEWAY_NOT_ENABLED", - "STRIPE_PAYMENT_METHOD_NOT_CARD", - "STRIPE_RETURNED_NO_PAYMENT_METHOD", - ) - - -class CustomerPaymentMethodUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("INVALID", "PRESENT", "TAKEN") - - -class CustomerPredictedSpendTier(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("HIGH", "LOW", "MEDIUM") - - -class CustomerProductSubscriberStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ACTIVE", "CANCELLED", "EXPIRED", "FAILED", "NEVER_SUBSCRIBED", "PAUSED") - - -class CustomerSavedSearchSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ID", "NAME", "RELEVANCE") - - -class CustomerSmsMarketingConsentErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("INCLUSION", "INTERNAL_ERROR", "INVALID", "MISSING_ARGUMENT") - - -class CustomerSmsMarketingState(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("NOT_SUBSCRIBED", "PENDING", "REDACTED", "SUBSCRIBED", "UNSUBSCRIBED") - - -class CustomerSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "CREATED_AT", - "ID", - "LAST_ORDER_DATE", - "LOCATION", - "NAME", - "ORDERS_COUNT", - "RELEVANCE", - "TOTAL_SPENT", - "UPDATED_AT", - ) - - -class CustomerState(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("DECLINED", "DISABLED", "ENABLED", "INVITED") - - -Date = sgqlc.types.datetime.Date - -DateTime = sgqlc.types.datetime.DateTime - - -class DayOfTheWeek(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("FRIDAY", "MONDAY", "SATURDAY", "SUNDAY", "THURSDAY", "TUESDAY", "WEDNESDAY") - - -class Decimal(sgqlc.types.Scalar): - __schema__ = shopify_schema - - -class DelegateAccessTokenCreateUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "DELEGATE_ACCESS_TOKEN", - "EMPTY_ACCESS_SCOPE", - "EXPIRES_AFTER_PARENT", - "NEGATIVE_EXPIRES_IN", - "PERSISTENCE_FAILED", - "REFRESH_TOKEN", - "UNKNOWN_SCOPES", - ) - - -class DeletionEventSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CREATED_AT", "ID", "RELEVANCE") - - -class DeletionEventSubjectType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("COLLECTION", "PRODUCT") - - -class DeliveryConditionField(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("TOTAL_PRICE", "TOTAL_WEIGHT") - - -class DeliveryConditionOperator(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("GREATER_THAN_OR_EQUAL_TO", "LESS_THAN_OR_EQUAL_TO") - - -class DeliveryLegacyModeBlockedReason(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("NO_LOCATIONS_FULFILLING_ONLINE_ORDERS",) - - -class DeliveryMethodDefinitionType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("MERCHANT", "PARTICIPANT") - - -class DeliveryMethodType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("LOCAL", "NONE", "PICK_UP", "RETAIL", "SHIPPING") - - -class DigitalWallet(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ANDROID_PAY", "APPLE_PAY", "GOOGLE_PAY", "SHOPIFY_PAY") - - -class DiscountApplicationAllocationMethod(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ACROSS", "EACH") - - -class DiscountApplicationLevel(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("LINE", "ORDER") - - -class DiscountApplicationTargetSelection(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ALL", "ENTITLED", "EXPLICIT") - - -class DiscountApplicationTargetType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("LINE_ITEM", "SHIPPING_LINE") - - -class DiscountClass(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ORDER", "PRODUCT", "SHIPPING") - - -class DiscountCodeSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CODE", "CREATED_AT", "ID", "RELEVANCE") - - -class DiscountErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "ACTIVE_PERIOD_OVERLAP", - "BLANK", - "CONFLICT", - "DUPLICATE", - "EQUAL_TO", - "EXCEEDED_MAX", - "GREATER_THAN", - "GREATER_THAN_OR_EQUAL_TO", - "IMPLICIT_DUPLICATE", - "INCLUSION", - "INTERNAL_ERROR", - "INVALID", - "INVALID_COMBINES_WITH_FOR_DISCOUNT_CLASS", - "INVALID_DISCOUNT_CLASS_FOR_PRICE_RULE", - "LESS_THAN", - "LESS_THAN_OR_EQUAL_TO", - "MAX_APP_DISCOUNTS", - "MINIMUM_SUBTOTAL_AND_QUANTITY_RANGE_BOTH_PRESENT", - "MISSING_ARGUMENT", - "PRESENT", - "TAKEN", - "TOO_LONG", - "TOO_MANY_ARGUMENTS", - "TOO_SHORT", - "VALUE_OUTSIDE_RANGE", - ) - - -class DiscountShareableUrlTargetType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("COLLECTION", "HOME", "PRODUCT") - - -class DiscountSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CREATED_AT", "ENDS_AT", "ID", "RELEVANCE", "STARTS_AT", "TITLE", "UPDATED_AT") - - -class DiscountStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ACTIVE", "EXPIRED", "SCHEDULED") - - -class DiscountTargetType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("LINE_ITEM", "SHIPPING_LINE") - - -class DiscountType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CODE_DISCOUNT", "MANUAL") - - -class DisputeEvidenceUpdateUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "DISPUTE_EVIDENCE_NOT_FOUND", - "EVIDENCE_ALREADY_ACCEPTED", - "EVIDENCE_PAST_DUE_DATE", - "FILES_SIZE_EXCEEDED_LIMIT", - "INVALID", - "TOO_LARGE", - ) - - -class DisputeStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ACCEPTED", "CHARGE_REFUNDED", "LOST", "NEEDS_RESPONSE", "UNDER_REVIEW", "WON") - - -class DisputeType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CHARGEBACK", "INQUIRY") - - -class DraftOrderAppliedDiscountType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("FIXED_AMOUNT", "PERCENTAGE") - - -class DraftOrderSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CUSTOMER_NAME", "ID", "NUMBER", "RELEVANCE", "STATUS", "TOTAL_PRICE", "UPDATED_AT") - - -class DraftOrderStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("COMPLETED", "INVOICE_SENT", "OPEN") - - -class ErrorsWebPixelUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("BLANK", "INVALID_SETTINGS", "NOT_FOUND", "TAKEN", "UNABLE_TO_DELETE") - - -class EventSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CREATED_AT", "ID", "RELEVANCE") - - -class FileContentType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("FILE", "IMAGE", "VIDEO") - - -class FileErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "EXTERNAL_VIDEO_EMBED_DISABLED", - "EXTERNAL_VIDEO_EMBED_NOT_FOUND_OR_TRANSCODING", - "EXTERNAL_VIDEO_INVALID_ASPECT_RATIO", - "EXTERNAL_VIDEO_NOT_FOUND", - "EXTERNAL_VIDEO_UNLISTED", - "FILE_STORAGE_LIMIT_EXCEEDED", - "GENERIC_FILE_DOWNLOAD_FAILURE", - "GENERIC_FILE_INVALID_SIZE", - "IMAGE_DOWNLOAD_FAILURE", - "IMAGE_PROCESSING_FAILURE", - "INVALID_IMAGE_ASPECT_RATIO", - "INVALID_IMAGE_FILE_SIZE", - "INVALID_IMAGE_RESOLUTION", - "INVALID_SIGNED_URL", - "MEDIA_TIMEOUT_ERROR", - "MODEL3D_GLB_OUTPUT_CREATION_ERROR", - "MODEL3D_GLB_TO_USDZ_CONVERSION_ERROR", - "MODEL3D_PROCESSING_FAILURE", - "MODEL3D_THUMBNAIL_GENERATION_ERROR", - "MODEL3D_VALIDATION_ERROR", - "UNKNOWN", - "UNSUPPORTED_IMAGE_FILE_TYPE", - "VIDEO_INVALID_FILETYPE_ERROR", - "VIDEO_MAX_DURATION_ERROR", - "VIDEO_MAX_HEIGHT_ERROR", - "VIDEO_MAX_WIDTH_ERROR", - "VIDEO_METADATA_READ_ERROR", - "VIDEO_MIN_DURATION_ERROR", - "VIDEO_MIN_HEIGHT_ERROR", - "VIDEO_MIN_WIDTH_ERROR", - "VIDEO_VALIDATION_ERROR", - ) - - -class FileSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CREATED_AT", "FILENAME", "ID", "ORIGINAL_UPLOAD_SIZE", "RELEVANCE") - - -class FileStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("FAILED", "PROCESSING", "READY", "UPLOADED") - - -class FilesErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "ALT_VALUE_LIMIT_EXCEEDED", - "BLANK_SEARCH", - "FILE_DOES_NOT_EXIST", - "FILE_LOCKED", - "INVALID", - "INVALID_QUERY", - "MISSING_ARGUMENTS", - "NON_IMAGE_MEDIA_PER_SHOP_LIMIT_EXCEEDED", - "TOO_MANY_ARGUMENTS", - "UNACCEPTABLE_ASSET", - "UNACCEPTABLE_TRIAL_ASSET", - "UNACCEPTABLE_UNVERIFIED_TRIAL_ASSET", - ) - - -Float = sgqlc.types.Float - - -class FormattedString(sgqlc.types.Scalar): - __schema__ = shopify_schema - - -class FulfillmentDisplayStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "ATTEMPTED_DELIVERY", - "CANCELED", - "CONFIRMED", - "DELIVERED", - "FAILURE", - "FULFILLED", - "IN_TRANSIT", - "LABEL_PRINTED", - "LABEL_PURCHASED", - "LABEL_VOIDED", - "MARKED_AS_FULFILLED", - "NOT_DELIVERED", - "OUT_FOR_DELIVERY", - "PICKED_UP", - "READY_FOR_PICKUP", - "SUBMITTED", - ) - - -class FulfillmentEventSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("HAPPENED_AT", "ID", "RELEVANCE") - - -class FulfillmentEventStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "ATTEMPTED_DELIVERY", - "CONFIRMED", - "DELIVERED", - "FAILURE", - "IN_TRANSIT", - "LABEL_PRINTED", - "LABEL_PURCHASED", - "OUT_FOR_DELIVERY", - "READY_FOR_PICKUP", - ) - - -class FulfillmentHoldReason(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "AWAITING_PAYMENT", - "HIGH_RISK_OF_FRAUD", - "INCORRECT_ADDRESS", - "INVENTORY_OUT_OF_STOCK", - "OTHER", - "UNKNOWN_DELIVERY_DATE", - ) - - -class FulfillmentOrderAction(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "CANCEL_FULFILLMENT_ORDER", - "CREATE_FULFILLMENT", - "EXTERNAL", - "HOLD", - "MARK_AS_OPEN", - "MOVE", - "RELEASE_HOLD", - "REQUEST_CANCELLATION", - "REQUEST_FULFILLMENT", - ) - - -class FulfillmentOrderAssignmentStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CANCELLATION_REQUESTED", "FULFILLMENT_ACCEPTED", "FULFILLMENT_REQUESTED") - - -class FulfillmentOrderHoldUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("FULFILLMENT_ORDER_NOT_FOUND", "TAKEN") - - -class FulfillmentOrderMerchantRequestKind(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CANCELLATION_REQUEST", "FULFILLMENT_REQUEST") - - -class FulfillmentOrderRejectionReason(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "INCORRECT_ADDRESS", - "INELIGIBLE_PRODUCT", - "INVENTORY_OUT_OF_STOCK", - "OTHER", - "UNDELIVERABLE_DESTINATION", - ) - - -class FulfillmentOrderReleaseHoldUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("FULFILLMENT_ORDER_NOT_FOUND",) - - -class FulfillmentOrderRequestStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "ACCEPTED", - "CANCELLATION_ACCEPTED", - "CANCELLATION_REJECTED", - "CANCELLATION_REQUESTED", - "CLOSED", - "REJECTED", - "SUBMITTED", - "UNSUBMITTED", - ) - - -class FulfillmentOrderRescheduleUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("FULFILLMENT_ORDER_NOT_FOUND",) - - -class FulfillmentOrderSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ID", "RELEVANCE") - - -class FulfillmentOrderStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CANCELLED", "CLOSED", "INCOMPLETE", "IN_PROGRESS", "ON_HOLD", "OPEN", "SCHEDULED") - - -class FulfillmentOrdersSetFulfillmentDeadlineUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("FULFILLMENT_ORDERS_NOT_FOUND",) - - -class FulfillmentServiceType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("GIFT_CARD", "MANUAL", "THIRD_PARTY") - - -class FulfillmentStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CANCELLED", "ERROR", "FAILURE", "SUCCESS") - - -class GiftCardErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("GREATER_THAN", "INTERNAL_ERROR", "INVALID", "MISSING_ARGUMENT", "TAKEN", "TOO_LONG", "TOO_SHORT") - - -class GiftCardSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "AMOUNT_SPENT", - "BALANCE", - "CODE", - "CREATED_AT", - "CUSTOMER_NAME", - "DISABLED_AT", - "EXPIRES_ON", - "ID", - "INITIAL_VALUE", - "RELEVANCE", - "UPDATED_AT", - ) - - -class HTML(sgqlc.types.Scalar): - __schema__ = shopify_schema - - -ID = sgqlc.types.ID - - -class ImageContentType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("JPG", "PNG", "WEBP") - - -Int = sgqlc.types.Int - - -class InventoryBulkToggleActivationUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("GENERIC_ERROR",) - - -class JSON(sgqlc.types.Scalar): - __schema__ = shopify_schema - - -class LocalizationExtensionKey(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "SHIPPING_CREDENTIAL_BR", - "SHIPPING_CREDENTIAL_CN", - "SHIPPING_CREDENTIAL_KR", - "TAX_CREDENTIAL_BR", - "TAX_CREDENTIAL_IT", - "TAX_EMAIL_IT", - ) - - -class LocalizationExtensionPurpose(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("SHIPPING", "TAX") - - -class LocationActivateUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("GENERIC_ERROR", "HAS_ONGOING_RELOCATION", "LOCATION_LIMIT", "LOCATION_NOT_FOUND") - - -class LocationAddUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("BLANK", "GENERIC_ERROR", "INVALID", "INVALID_US_ZIPCODE", "TAKEN", "TOO_LONG") - - -class LocationDeactivateUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "CANNOT_DISABLE_ONLINE_ORDER_FULFILLMENT", - "DESTINATION_LOCATION_IS_THE_SAME_LOCATION", - "DESTINATION_LOCATION_NOT_FOUND_OR_INACTIVE", - "FAILED_TO_RELOCATE_ACTIVE_INVENTORIES", - "FAILED_TO_RELOCATE_INCOMING_MOVEMENTS", - "FAILED_TO_RELOCATE_OPEN_PURCHASE_ORDERS", - "FAILED_TO_RELOCATE_OPEN_TRANSFERS", - "HAS_ACTIVE_INVENTORY_ERROR", - "HAS_ACTIVE_RETAIL_SUBSCRIPTIONS", - "HAS_FULFILLMENT_ORDERS_ERROR", - "HAS_INCOMING_MOVEMENTS_ERROR", - "HAS_OPEN_PURCHASE_ORDERS_ERROR", - "HAS_OPEN_TRANSFERS_ERROR", - "INVALID", - "LOCATION_NOT_FOUND", - "PERMANENTLY_BLOCKED_FROM_DEACTIVATION_ERROR", - "TEMPORARILY_BLOCKED_FROM_DEACTIVATION_ERROR", - ) - - -class LocationDeleteUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "GENERIC_ERROR", - "LOCATION_HAS_ACTIVE_RETAIL_SUBSCRIPTION", - "LOCATION_HAS_INVENTORY", - "LOCATION_HAS_PENDING_ORDERS", - "LOCATION_IS_ACTIVE", - "LOCATION_NOT_FOUND", - ) - - -class LocationEditUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "BLANK", - "CANNOT_DISABLE_ONLINE_ORDER_FULFILLMENT", - "GENERIC_ERROR", - "INVALID", - "INVALID_US_ZIPCODE", - "NOT_FOUND", - "TOO_LONG", - ) - - -class LocationSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ID", "NAME", "RELEVANCE") - - -class MarketCurrencySettingsUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "MANAGED_MARKET", - "MARKET_NOT_FOUND", - "MULTIPLE_CURRENCIES_NOT_SUPPORTED", - "NO_LOCAL_CURRENCIES_ON_SINGLE_COUNTRY_MARKET", - "PRIMARY_MARKET_USES_SHOP_CURRENCY", - "UNSUPPORTED_CURRENCY", - ) - - -class MarketLocalizableResourceType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("METAFIELD",) - - -class MarketUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "BLANK", - "CANNOT_ADD_CUSTOMER_DOMAIN", - "CANNOT_ADD_REGIONS_TO_PRIMARY_MARKET", - "CANNOT_ADD_WEB_PRESENCE_TO_PRIMARY_MARKET", - "CANNOT_DELETE_ONLY_REGION", - "CANNOT_DELETE_PRIMARY_MARKET", - "CANNOT_DELETE_PRIMARY_MARKET_WEB_PRESENCE", - "CANNOT_DISABLE_PRIMARY_MARKET", - "CANNOT_HAVE_SUBFOLDER_AND_DOMAIN", - "CANNOT_SET_DEFAULT_LOCALE_TO_NULL", - "DISABLED_LANGUAGE", - "DOMAIN_NOT_FOUND", - "DUPLICATE_LANGUAGES", - "INVALID", - "MARKET_NOT_FOUND", - "NO_LANGUAGES", - "PRIMARY_MARKET_MUST_USE_PRIMARY_DOMAIN", - "REGION_NOT_FOUND", - "REGION_SPECIFIC_LANGUAGE", - "REQUIRES_DOMAIN_OR_SUBFOLDER", - "REQUIRES_EXACTLY_ONE_OPTION", - "SHOP_REACHED_MARKETS_LIMIT", - "SUBFOLDER_NOT_ALLOWED_FOR_CCTLD_DOMAINS", - "SUBFOLDER_SUFFIX_MUST_CONTAIN_ONLY_LETTERS", - "TAKEN", - "TOO_LONG", - "TOO_SHORT", - "UNPUBLISHED_LANGUAGE", - "UNSUPPORTED_COUNTRY_REGION", - "WEB_PRESENCE_NOT_FOUND", - ) - - -class MarketingActivityExtensionAppErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("API_ERROR", "INSTALL_REQUIRED_ERROR", "NOT_ONBOARDED_ERROR", "PLATFORM_ERROR", "VALIDATION_ERROR") - - -class MarketingActivitySortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CREATED_AT", "ID", "RELEVANCE", "TITLE") - - -class MarketingActivityStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "ACTIVE", - "DELETED", - "DELETED_EXTERNALLY", - "DISCONNECTED", - "DRAFT", - "FAILED", - "INACTIVE", - "PAUSED", - "PENDING", - "SCHEDULED", - "UNDEFINED", - ) - - -class MarketingActivityStatusBadgeType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ATTENTION", "DEFAULT", "INFO", "SUCCESS", "WARNING") - - -class MarketingActivityUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("INVALID", "TAKEN") - - -class MarketingBudgetBudgetType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("DAILY", "LIFETIME") - - -class MarketingChannel(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("DISPLAY", "EMAIL", "REFERRAL", "SEARCH", "SOCIAL") - - -class MarketingEventSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ID", "RELEVANCE", "STARTED_AT") - - -class MarketingTactic(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "ABANDONED_CART", - "AD", - "AFFILIATE", - "DIRECT", - "LINK", - "LOYALTY", - "MESSAGE", - "NEWSLETTER", - "NOTIFICATION", - "POST", - "RETARGETING", - "SEO", - "STOREFRONT_APP", - "TRANSACTIONAL", - ) - - -class MediaContentType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("EXTERNAL_VIDEO", "IMAGE", "MODEL_3D", "VIDEO") - - -class MediaErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "EXTERNAL_VIDEO_EMBED_DISABLED", - "EXTERNAL_VIDEO_EMBED_NOT_FOUND_OR_TRANSCODING", - "EXTERNAL_VIDEO_INVALID_ASPECT_RATIO", - "EXTERNAL_VIDEO_NOT_FOUND", - "EXTERNAL_VIDEO_UNLISTED", - "FILE_STORAGE_LIMIT_EXCEEDED", - "GENERIC_FILE_DOWNLOAD_FAILURE", - "GENERIC_FILE_INVALID_SIZE", - "IMAGE_DOWNLOAD_FAILURE", - "IMAGE_PROCESSING_FAILURE", - "INVALID_IMAGE_ASPECT_RATIO", - "INVALID_IMAGE_FILE_SIZE", - "INVALID_IMAGE_RESOLUTION", - "INVALID_SIGNED_URL", - "MEDIA_TIMEOUT_ERROR", - "MODEL3D_GLB_OUTPUT_CREATION_ERROR", - "MODEL3D_GLB_TO_USDZ_CONVERSION_ERROR", - "MODEL3D_PROCESSING_FAILURE", - "MODEL3D_THUMBNAIL_GENERATION_ERROR", - "MODEL3D_VALIDATION_ERROR", - "UNKNOWN", - "UNSUPPORTED_IMAGE_FILE_TYPE", - "VIDEO_INVALID_FILETYPE_ERROR", - "VIDEO_MAX_DURATION_ERROR", - "VIDEO_MAX_HEIGHT_ERROR", - "VIDEO_MAX_WIDTH_ERROR", - "VIDEO_METADATA_READ_ERROR", - "VIDEO_MIN_DURATION_ERROR", - "VIDEO_MIN_HEIGHT_ERROR", - "VIDEO_MIN_WIDTH_ERROR", - "VIDEO_VALIDATION_ERROR", - ) - - -class MediaHost(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("VIMEO", "YOUTUBE") - - -class MediaPreviewImageStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("FAILED", "PROCESSING", "READY", "UPLOADED") - - -class MediaStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("FAILED", "PROCESSING", "READY", "UPLOADED") - - -class MediaUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "BLANK", - "INVALID", - "INVALID_MEDIA_TYPE", - "MAXIMUM_VARIANT_MEDIA_PAIRS_EXCEEDED", - "MEDIA_CANNOT_BE_MODIFIED", - "MEDIA_DOES_NOT_EXIST", - "MEDIA_DOES_NOT_EXIST_ON_PRODUCT", - "MEDIA_IS_NOT_ATTACHED_TO_VARIANT", - "MODEL3D_THROTTLE_EXCEEDED", - "MODEL3D_VALIDATION_ERROR", - "NON_READY_MEDIA", - "PRODUCT_DOES_NOT_EXIST", - "PRODUCT_MEDIA_LIMIT_EXCEEDED", - "PRODUCT_VARIANT_ALREADY_HAS_MEDIA", - "PRODUCT_VARIANT_DOES_NOT_EXIST_ON_PRODUCT", - "PRODUCT_VARIANT_SPECIFIED_MULTIPLE_TIMES", - "SHOP_MEDIA_LIMIT_EXCEEDED", - "TOO_MANY_MEDIA_PER_INPUT_PAIR", - "VIDEO_THROTTLE_EXCEEDED", - "VIDEO_VALIDATION_ERROR", - ) - - -class MediaWarningCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("MODEL_LARGE_PHYSICAL_SIZE", "MODEL_SMALL_PHYSICAL_SIZE") - - -class MerchandiseDiscountClass(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ORDER", "PRODUCT") - - -class MetafieldDefinitionCreateUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "DUPLICATE_OPTION", - "INCLUSION", - "INVALID", - "INVALID_CHARACTER", - "INVALID_OPTION", - "LIMIT_EXCEEDED", - "PINNED_LIMIT_REACHED", - "PRESENT", - "RESERVED_NAMESPACE_KEY", - "RESOURCE_TYPE_LIMIT_EXCEEDED", - "TAKEN", - "TOO_LONG", - "TOO_SHORT", - "UNSTRUCTURED_ALREADY_EXISTS", - ) - - -class MetafieldDefinitionDeleteUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("INTERNAL_ERROR", "NOT_FOUND", "PRESENT", "REFERENCE_TYPE_DELETION_ERROR") - - -class MetafieldDefinitionPinUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ALREADY_PINNED", "INTERNAL_ERROR", "NOT_FOUND", "PINNED_LIMIT_REACHED") - - -class MetafieldDefinitionPinnedStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ANY", "PINNED", "UNPINNED") - - -class MetafieldDefinitionSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ID", "NAME", "PINNED_POSITION", "RELEVANCE") - - -class MetafieldDefinitionUnpinUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("INTERNAL_ERROR", "NOT_FOUND", "NOT_PINNED") - - -class MetafieldDefinitionUpdateUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("INTERNAL_ERROR", "INVALID_INPUT", "NOT_FOUND", "PINNED_LIMIT_REACHED", "PRESENT", "TOO_LONG") - - -class MetafieldDefinitionValidationStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ALL_VALID", "IN_PROGRESS", "SOME_INVALID") - - -class MetafieldOwnerType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "API_PERMISSION", - "ARTICLE", - "BLOG", - "COLLECTION", - "CUSTOMER", - "DISCOUNT", - "DRAFTORDER", - "LOCATION", - "ORDER", - "PAGE", - "PRODUCT", - "PRODUCTIMAGE", - "PRODUCTVARIANT", - "SHOP", - ) - - -class MetafieldValidationStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ANY", "INVALID", "VALID") - - -class MetafieldValueType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("BOOLEAN", "INTEGER", "JSON_STRING", "STRING") - - -class MetafieldsSetUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "APP_NOT_AUTHORIZED", - "BLANK", - "INCLUSION", - "INVALID_TYPE", - "INVALID_VALUE", - "LESS_THAN_OR_EQUAL_TO", - "PRESENT", - "TOO_LONG", - "TOO_SHORT", - ) - - -class MethodDefinitionSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ID", "RATE_PROVIDER_TYPE", "RELEVANCE") - - -class Money(sgqlc.types.Scalar): - __schema__ = shopify_schema - - -class OrderActionType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ORDER", "ORDER_EDIT", "REFUND", "UNKNOWN") - - -class OrderCancelReason(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CUSTOMER", "DECLINED", "FRAUD", "INVENTORY", "OTHER") - - -class OrderCreateMandatePaymentUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ORDER_MANDATE_PAYMENT_ERROR_CODE",) - - -class OrderDisplayFinancialStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "AUTHORIZED", - "EXPIRED", - "PAID", - "PARTIALLY_PAID", - "PARTIALLY_REFUNDED", - "PENDING", - "REFUNDED", - "VOIDED", - ) - - -class OrderDisplayFulfillmentStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "FULFILLED", - "IN_PROGRESS", - "ON_HOLD", - "OPEN", - "PARTIALLY_FULFILLED", - "PENDING_FULFILLMENT", - "RESTOCKED", - "SCHEDULED", - "UNFULFILLED", - ) - - -class OrderInvoiceSendUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ORDER_INVOICE_SEND_UNSUCCESSFUL",) - - -class OrderPaymentStatusResult(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "AUTHORIZED", - "CAPTURED", - "ERROR", - "PROCESSING", - "PURCHASED", - "REDIRECT_REQUIRED", - "REFUNDED", - "RETRYABLE", - "SUCCESS", - "UNKNOWN", - "VOIDED", - ) - - -class OrderRiskLevel(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("HIGH", "LOW", "MEDIUM") - - -class OrderSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "CREATED_AT", - "CUSTOMER_NAME", - "FINANCIAL_STATUS", - "FULFILLMENT_STATUS", - "ID", - "ORDER_NUMBER", - "PROCESSED_AT", - "RELEVANCE", - "TOTAL_PRICE", - "UPDATED_AT", - ) - - -class OrderTransactionErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "AMAZON_PAYMENTS_INVALID_PAYMENT_METHOD", - "AMAZON_PAYMENTS_MAX_AMOUNT_CHARGED", - "AMAZON_PAYMENTS_MAX_AMOUNT_REFUNDED", - "AMAZON_PAYMENTS_MAX_AUTHORIZATIONS_CAPTURED", - "AMAZON_PAYMENTS_MAX_REFUNDS_PROCESSED", - "AMAZON_PAYMENTS_ORDER_REFERENCE_CANCELED", - "AMAZON_PAYMENTS_STALE", - "CALL_ISSUER", - "CARD_DECLINED", - "CONFIG_ERROR", - "EXPIRED_CARD", - "GENERIC_ERROR", - "INCORRECT_ADDRESS", - "INCORRECT_CVC", - "INCORRECT_NUMBER", - "INCORRECT_PIN", - "INCORRECT_ZIP", - "INVALID_AMOUNT", - "INVALID_COUNTRY", - "INVALID_CVC", - "INVALID_EXPIRY_DATE", - "INVALID_NUMBER", - "PAYMENT_METHOD_UNAVAILABLE", - "PICK_UP_CARD", - "PROCESSING_ERROR", - "TEST_MODE_LIVE_CARD", - "UNSUPPORTED_FEATURE", - ) - - -class OrderTransactionKind(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "AUTHORIZATION", - "CAPTURE", - "CHANGE", - "EMV_AUTHORIZATION", - "REFUND", - "SALE", - "SUGGESTED_REFUND", - "VOID", - ) - - -class OrderTransactionStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("AWAITING_RESPONSE", "ERROR", "FAILURE", "PENDING", "SUCCESS", "UNKNOWN") - - -class PaymentMethods(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "AMERICAN_EXPRESS", - "BITCOIN", - "BOGUS", - "DANKORT", - "DINERS_CLUB", - "DISCOVER", - "DOGECOIN", - "ELO", - "FORBRUGSFORENINGEN", - "INTERAC", - "JCB", - "LITECOIN", - "MAESTRO", - "MASTERCARD", - "PAYPAL", - "UNIONPAY", - "VISA", - ) - - -class PaymentTermsCreateUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("PAYMENT_TERMS_CREATION_UNSUCCESSFUL",) - - -class PaymentTermsDeleteUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("PAYMENT_TERMS_DELETE_UNSUCCESSFUL",) - - -class PaymentTermsType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("FIXED", "NET", "RECEIPT", "UNKNOWN") - - -class PaymentTermsUpdateUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("PAYMENT_TERMS_UPDATE_UNSUCCESSFUL",) - - -class PaypalExpressSubscriptionsGatewayStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("DISABLED", "ENABLED", "PENDING") - - -class PriceListAdjustmentType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("PERCENTAGE_DECREASE", "PERCENTAGE_INCREASE") - - -class PriceListPriceOriginType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("FIXED", "RELATIVE") - - -class PriceListPriceUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "BLANK", - "PRICE_LIST_CURRENCY_MISMATCH", - "PRICE_LIST_NOT_FOUND", - "PRICE_NOT_FIXED", - "VARIANT_NOT_FOUND", - ) - - -class PriceListSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ID", "NAME", "RELEVANCE") - - -class PriceListUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "CONTEXT_RULE_COUNTRIES_LIMIT", - "CONTEXT_RULE_COUNTRY_TAKEN", - "CONTEXT_RULE_LIMIT_ONE_OPTION", - "CONTEXT_RULE_MARKET_NOT_FOUND", - "CONTEXT_RULE_MARKET_TAKEN", - "COUNTRY_CURRENCY_MISMATCH", - "CURRENCY_COUNTRY_MISMATCH", - "CURRENCY_MARKET_MISMATCH", - "CURRENCY_NOT_SUPPORTED", - "INVALID_ADJUSTMENT_VALUE", - "MARKET_CURRENCY_MISMATCH", - "PRICE_LIST_NOT_ALLOWED_FOR_PRIMARY_MARKET", - "PRICE_LIST_NOT_FOUND", - "TAKEN", - ) - - -class PriceRuleAllocationMethod(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ACROSS", "EACH") - - -class PriceRuleErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "ALLOCATION_METHOD_MUST_BE_ACROSS_FOR_GIVEN_TARGET_SELECTION", - "APPLIES_ON_NOTHING", - "BLANK", - "BOGO_INVALID_TARGET_SELECTION", - "BOGO_INVALID_TARGET_TYPE", - "BOGO_INVALID_VALUE_TYPE", - "BOTH_CUSTOMER_AND_SAVED_SEARCH_PREREQUISITES_SELECTED", - "BOTH_CUSTOMER_AND_SEGMENT_PREREQUISITES_SELECTED", - "BOTH_SAVED_SEARCH_AND_SEGMENT_PREREQUISITES_SELECTED", - "CANNOT_ENTITLE_COLLECTIONS_WITH_PRODUCTS_OR_VARIANTS", - "CANNOT_PREREQUISITE_COLLECTION_WITH_PRODUCT_OR_VARIANTS", - "CUSTOMER_PREREQUISITES_EXCEEDED_MAX", - "CUSTOMER_PREREQUISITES_INVALID_SELECTION", - "CUSTOMER_PREREQUISITES_MISSING", - "CUSTOMER_PREREQUISITE_DUPLICATE", - "CUSTOMER_SAVED_SEARCH_DUPLICATE", - "CUSTOMER_SAVED_SEARCH_EXCEEDED_MAX", - "CUSTOMER_SAVED_SEARCH_INVALID", - "CUSTOMER_SEGMENT_EXCEEDED_MAX", - "CUSTOMER_SEGMENT_INVALID", - "CUSTOMER_SEGMENT_PREREQUISITE_DUPLICATE", - "DISCOUNT_CODE_DUPLICATE", - "END_DATE_BEFORE_START_DATE", - "EQUAL_TO", - "EXCEEDED_MAX", - "GREATER_THAN", - "GREATER_THAN_OR_EQUAL_TO", - "INTERNAL_ERROR", - "INVALID", - "INVALID_COMBINES_WITH_FOR_DISCOUNT_CLASS", - "INVALID_DISCOUNT_CLASS_FOR_PRICE_RULE", - "INVALID_TARGET_TYPE_PREREQUISITE_SHIPPING_PRICE_RANGE", - "ITEM_ENTITLEMENTS_DUPLICATE_COLLECTION", - "ITEM_ENTITLEMENTS_DUPLICATE_PRODUCT", - "ITEM_ENTITLEMENTS_DUPLICATE_VARIANT", - "ITEM_ENTITLEMENTS_EXCEEDED_MAX_COLLECTION", - "ITEM_ENTITLEMENTS_EXCEEDED_MAX_PRODUCT", - "ITEM_ENTITLEMENTS_EXCEEDED_MAX_VARIANT", - "ITEM_ENTITLEMENTS_INVALID_COLLECTION", - "ITEM_ENTITLEMENTS_INVALID_PRODUCT", - "ITEM_ENTITLEMENTS_INVALID_TARGET_TYPE_OR_SELECTION", - "ITEM_ENTITLEMENTS_INVALID_VARIANT", - "ITEM_ENTITLEMENTS_MISSING", - "ITEM_ENTITLEMENT_INVALID_TYPE", - "ITEM_PREREQUISITES_DUPLICATE_COLLECTION", - "ITEM_PREREQUISITES_DUPLICATE_PRODUCT", - "ITEM_PREREQUISITES_DUPLICATE_VARIANT", - "ITEM_PREREQUISITES_EXCEEDED_MAX", - "ITEM_PREREQUISITES_INVALID_COLLECTION", - "ITEM_PREREQUISITES_INVALID_PRODUCT", - "ITEM_PREREQUISITES_INVALID_TYPE", - "ITEM_PREREQUISITES_INVALID_VARIANT", - "ITEM_PREREQUISITES_MISSING", - "ITEM_PREREQUISITES_MUST_BE_EMPTY", - "LESS_THAN", - "LESS_THAN_OR_EQUAL_TO", - "MISSING_ARGUMENT", - "MULTIPLE_RECURRING_CYCLE_LIMIT_FOR_NON_SUBSCRIPTION_ITEMS", - "PREREQUISITE_SUBTOTAL_AND_QUANTITY_RANGE_BOTH_PRESENT", - "PRICE_RULE_ALLOCATION_LIMIT_IS_ZERO", - "PRICE_RULE_ALLOCATION_LIMIT_ON_NON_BOGO", - "PRICE_RULE_EXCEEDED_MAX_DISCOUNT_CODE", - "PRICE_RULE_PERCENTAGE_VALUE_OUTSIDE_RANGE", - "SHIPPING_ENTITLEMENTS_DUPLICATE_COUNTRY", - "SHIPPING_ENTITLEMENTS_EXCEEDED_MAX", - "SHIPPING_ENTITLEMENTS_INVALID_COUNTRY", - "SHIPPING_ENTITLEMENTS_INVALID_TARGET_TYPE_OR_SELECTION", - "SHIPPING_ENTITLEMENTS_MISSING", - "SHIPPING_ENTITLEMENTS_UNSUPPORTED_DESTINATION_TYPE", - "SHOP_EXCEEDED_MAX_PRICE_RULES", - "TAKEN", - "TOO_LONG", - "TOO_MANY_ARGUMENTS", - "TOO_SHORT", - "VARIANT_ALREADY_ENTITLED_THROUGH_PRODUCT", - ) - - -class PriceRuleFeature(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "BULK", - "BUY_ONE_GET_ONE", - "BUY_ONE_GET_ONE_WITH_ALLOCATION_LIMIT", - "QUANTITY_DISCOUNTS", - "SPECIFIC_CUSTOMERS", - ) - - -class PriceRuleShareableUrlTargetType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("COLLECTION", "HOME", "PRODUCT") - - -class PriceRuleSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CREATED_AT", "ENDS_AT", "ID", "RELEVANCE", "STARTS_AT", "TITLE", "UPDATED_AT") - - -class PriceRuleStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ACTIVE", "EXPIRED", "SCHEDULED") - - -class PriceRuleTarget(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("LINE_ITEM", "SHIPPING_LINE") - - -class PriceRuleTrait(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "BULK", - "BUY_ONE_GET_ONE", - "BUY_ONE_GET_ONE_WITH_ALLOCATION_LIMIT", - "QUANTITY_DISCOUNTS", - "SPECIFIC_CUSTOMERS", - ) - - -class PrivateMetafieldValueType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("INTEGER", "JSON_STRING", "STRING") - - -class ProductChangeStatusUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("PRODUCT_NOT_FOUND",) - - -class ProductCollectionSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("BEST_SELLING", "COLLECTION_DEFAULT", "CREATED", "ID", "MANUAL", "PRICE", "RELEVANCE", "TITLE") - - -class ProductImageSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CREATED_AT", "ID", "POSITION", "RELEVANCE") - - -class ProductMediaSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ID", "POSITION", "RELEVANCE") - - -class ProductSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "CREATED_AT", - "ID", - "INVENTORY_TOTAL", - "PRODUCT_TYPE", - "PUBLISHED_AT", - "RELEVANCE", - "TITLE", - "UPDATED_AT", - "VENDOR", - ) - - -class ProductStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ACTIVE", "ARCHIVED", "DRAFT") - - -class ProductVariantInventoryManagement(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("FULFILLMENT_SERVICE", "NOT_MANAGED", "SHOPIFY") - - -class ProductVariantInventoryPolicy(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CONTINUE", "DENY") - - -class ProductVariantSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "FULL_TITLE", - "ID", - "INVENTORY_LEVELS_AVAILABLE", - "INVENTORY_MANAGEMENT", - "INVENTORY_POLICY", - "INVENTORY_QUANTITY", - "NAME", - "POPULAR", - "POSITION", - "RELEVANCE", - "SKU", - "TITLE", - ) - - -class ProductVariantsBulkCreateUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "GREATER_THAN_OR_EQUAL_TO", - "INVALID", - "MUST_BE_FOR_THIS_PRODUCT", - "NEED_TO_ADD_OPTION_VALUES", - "NEGATIVE_PRICE_VALUE", - "NOT_DEFINED_FOR_SHOP", - "NO_KEY_ON_CREATE", - "OPTION_VALUES_FOR_NUMBER_OF_UNKNOWN_OPTIONS", - "PRODUCT_DOES_NOT_EXIST", - "SUBSCRIPTION_VIOLATION", - "TOO_MANY_INVENTORY_LOCATIONS", - "TRACKED_VARIANT_LOCATION_NOT_FOUND", - "VARIANT_ALREADY_EXISTS", - "VARIANT_ALREADY_EXISTS_CHANGE_OPTION_VALUE", - ) - - -class ProductVariantsBulkDeleteUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "AT_LEAST_ONE_VARIANT_DOES_NOT_BELONG_TO_THE_PRODUCT", - "CANNOT_DELETE_LAST_VARIANT", - "PRODUCT_DOES_NOT_EXIST", - ) - - -class ProductVariantsBulkReorderUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("DUPLICATED_VARIANT_ID", "INVALID_POSITION", "MISSING_VARIANT", "PRODUCT_DOES_NOT_EXIST") - - -class ProductVariantsBulkUpdateUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "GREATER_THAN_OR_EQUAL_TO", - "NEED_TO_ADD_OPTION_VALUES", - "NEGATIVE_PRICE_VALUE", - "NO_INVENTORY_QUANTITES_DURING_UPDATE", - "NO_INVENTORY_QUANTITIES_ON_VARIANTS_UPDATE", - "OPTION_VALUES_FOR_NUMBER_OF_UNKNOWN_OPTIONS", - "PRODUCT_DOES_NOT_EXIST", - "PRODUCT_VARIANT_DOES_NOT_EXIST", - "PRODUCT_VARIANT_ID_MISSING", - "SUBSCRIPTION_VIOLATION", - "VARIANT_ALREADY_EXISTS", - ) - - -class ProfileItemSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "CREATED_AT", - "ID", - "INVENTORY_TOTAL", - "PRODUCT_TYPE", - "PUBLISHED_AT", - "RELEVANCE", - "TITLE", - "UPDATED_AT", - "VENDOR", - ) - - -class PubSubWebhookSubscriptionCreateUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("INVALID_PARAMETERS",) - - -class PubSubWebhookSubscriptionUpdateUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("INVALID_PARAMETERS",) - - -class RefundDutyRefundType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("FULL", "PROPORTIONAL") - - -class RefundLineItemRestockType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CANCEL", "LEGACY_RESTOCK", "NO_RESTOCK", "RETURN") - - -class ResourceAlertIcon(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CHECKMARK_CIRCLE", "INFORMATION_CIRCLE") - - -class ResourceAlertSeverity(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CRITICAL", "DEFAULT", "INFO", "SUCCESS", "WARNING") - - -class ResourceFeedbackState(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ACCEPTED", "REQUIRES_ACTION") - - -class SaleActionType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ORDER", "RETURN", "UNKNOWN", "UPDATE") - - -class SaleLineType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ADJUSTMENT", "DUTY", "GIFT_CARD", "PRODUCT", "SHIPPING", "TIP", "UNKNOWN") - - -class ScriptTagDisplayScope(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ALL", "ONLINE_STORE", "ORDER_STATUS") - - -class SearchResultType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "COLLECTION", - "CUSTOMER", - "DISCOUNT_REDEEM_CODE", - "DRAFT_ORDER", - "FILE", - "ONLINE_STORE_ARTICLE", - "ONLINE_STORE_BLOG", - "ONLINE_STORE_PAGE", - "ORDER", - "PRICE_RULE", - "PRODUCT", - "URL_REDIRECT", - ) - - -class SegmentSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CREATION_DATE", "ID", "LAST_EDIT_DATE", "RELEVANCE") - - -class SellingPlanAnchorType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("MONTHDAY", "WEEKDAY", "YEARDAY") - - -class SellingPlanCategory(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("OTHER", "PRE_ORDER", "SUBSCRIPTION", "TRY_BEFORE_YOU_BUY") - - -class SellingPlanCheckoutChargeType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("PERCENTAGE", "PRICE") - - -class SellingPlanFixedDeliveryPolicyIntent(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("FULFILLMENT_BEGIN",) - - -class SellingPlanFixedDeliveryPolicyPreAnchorBehavior(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ASAP", "NEXT") - - -class SellingPlanFulfillmentTrigger(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ANCHOR", "ASAP", "EXACT_TIME", "UNKNOWN") - - -class SellingPlanGroupSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CREATED_AT", "ID", "NAME", "RELEVANCE", "UPDATED_AT") - - -class SellingPlanGroupUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "BILLING_AND_DELIVERY_POLICY_TYPES_MUST_BE_THE_SAME", - "BLANK", - "CHECKOUT_CHARGE_VALUE_AND_TYPE_MUST_MATCH", - "EQUAL_TO", - "ERROR_ADDING_RESOURCE_TO_GROUP", - "FULFILLMENT_EXACT_TIME_NOT_ALLOWED", - "FULFILLMENT_EXACT_TIME_REQUIRED", - "GREATER_THAN", - "GREATER_THAN_OR_EQUAL_TO", - "GROUP_COULD_NOT_BE_DELETED", - "GROUP_DOES_NOT_EXIST", - "INCLUSION", - "INVALID", - "LESS_THAN", - "LESS_THAN_OR_EQUAL_TO", - "NOT_A_NUMBER", - "NOT_FOUND", - "ONLY_NEED_ONE_BILLING_POLICY_TYPE", - "ONLY_NEED_ONE_CHECKOUT_CHARGE_VALUE", - "ONLY_NEED_ONE_DELIVERY_POLICY_TYPE", - "ONLY_NEED_ONE_PRICING_POLICY_TYPE", - "ONLY_NEED_ONE_PRICING_POLICY_VALUE", - "ONLY_ONE_OF_FIXED_OR_RECURRING_BILLING", - "ONLY_ONE_OF_FIXED_OR_RECURRING_DELIVERY", - "PLAN_DOES_NOT_EXIST", - "PLAN_ID_MUST_BE_SPECIFIED_TO_UPDATE", - "PRESENT", - "PRICING_POLICY_ADJUSTMENT_VALUE_AND_TYPE_MUST_MATCH", - "PRODUCT_DOES_NOT_EXIST", - "PRODUCT_VARIANT_DOES_NOT_EXIST", - "REMAINING_BALANCE_CHARGE_EXACT_TIME_NOT_ALLOWED", - "REMAINING_BALANCE_CHARGE_EXACT_TIME_REQUIRED", - "REMAINING_BALANCE_CHARGE_TIME_AFTER_CHECKOUT_MUST_BE_GREATER_THAN_ZERO", - "REMAINING_BALANCE_CHARGE_TRIGGER_NO_REMAINING_BALANCE_ON_PARTIAL_PERCENTAGE_CHECKOUT_CHARGE", - "REMAINING_BALANCE_CHARGE_TRIGGER_NO_REMAINING_BALANCE_ON_PRICE_CHECKOUT_CHARGE", - "REMAINING_BALANCE_CHARGE_TRIGGER_ON_FULL_CHECKOUT", - "RESOURCE_LIST_CONTAINS_INVALID_IDS", - "SELLING_PLAN_ANCHORS_NOT_ALLOWED", - "SELLING_PLAN_ANCHORS_REQUIRED", - "SELLING_PLAN_BILLING_AND_DELIVERY_POLICY_ANCHORS_MUST_BE_EQUAL", - "SELLING_PLAN_BILLING_CYCLE_MUST_BE_A_MULTIPLE_OF_DELIVERY_CYCLE", - "SELLING_PLAN_BILLING_POLICY_MISSING", - "SELLING_PLAN_COUNT_LOWER_BOUND", - "SELLING_PLAN_COUNT_UPPER_BOUND", - "SELLING_PLAN_DELIVERY_POLICY_MISSING", - "SELLING_PLAN_DUPLICATE_NAME", - "SELLING_PLAN_DUPLICATE_OPTIONS", - "SELLING_PLAN_FIXED_PRICING_POLICIES_LIMIT", - "SELLING_PLAN_MAX_CYCLES_MUST_BE_GREATER_THAN_MIN_CYCLES", - "SELLING_PLAN_MISSING_OPTION2_LABEL_ON_PARENT_GROUP", - "SELLING_PLAN_MISSING_OPTION3_LABEL_ON_PARENT_GROUP", - "SELLING_PLAN_OPTION2_REQUIRED_AS_DEFINED_ON_PARENT_GROUP", - "SELLING_PLAN_OPTION3_REQUIRED_AS_DEFINED_ON_PARENT_GROUP", - "SELLING_PLAN_PRICING_POLICIES_LIMIT", - "SELLING_PLAN_PRICING_POLICIES_MUST_CONTAIN_A_FIXED_PRICING_POLICY", - "TAKEN", - "TOO_BIG", - "TOO_LONG", - "TOO_SHORT", - "WRONG_LENGTH", - ) - - -class SellingPlanInterval(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("DAY", "MONTH", "WEEK", "YEAR") - - -class SellingPlanPricingPolicyAdjustmentType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("FIXED_AMOUNT", "PERCENTAGE", "PRICE") - - -class SellingPlanRecurringDeliveryPolicyIntent(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("FULFILLMENT_BEGIN",) - - -class SellingPlanRecurringDeliveryPolicyPreAnchorBehavior(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ASAP", "NEXT") - - -class SellingPlanRemainingBalanceChargeTrigger(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("EXACT_TIME", "NO_REMAINING_BALANCE", "TIME_AFTER_CHECKOUT") - - -class SellingPlanReserve(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ON_FULFILLMENT", "ON_SALE") - - -class ShippingDiscountClass(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("SHIPPING",) - - -class ShopBranding(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ROGERS", "SHOPIFY", "SHOPIFY_GOLD", "SHOPIFY_PLUS") - - -class ShopCustomerAccountsSetting(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("DISABLED", "OPTIONAL", "REQUIRED") - - -class ShopPolicyErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("TOO_BIG",) - - -class ShopPolicyType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "LEGAL_NOTICE", - "PRIVACY_POLICY", - "REFUND_POLICY", - "SHIPPING_POLICY", - "SUBSCRIPTION_POLICY", - "TERMS_OF_SALE", - "TERMS_OF_SERVICE", - ) - - -class ShopTagSort(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ALPHABETICAL", "POPULAR") - - -class ShopifyPaymentsBankAccountStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ERRORED", "NEW", "VALIDATED", "VERIFIED") - - -class ShopifyPaymentsDisputeEvidenceFileType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "CANCELLATION_POLICY_FILE", - "CUSTOMER_COMMUNICATION_FILE", - "REFUND_POLICY_FILE", - "SERVICE_DOCUMENTATION_FILE", - "SHIPPING_DOCUMENTATION_FILE", - "UNCATEGORIZED_FILE", - ) - - -class ShopifyPaymentsDisputeReason(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "BANK_CANNOT_PROCESS", - "CREDIT_NOT_PROCESSED", - "CUSTOMER_INITIATED", - "DEBIT_NOT_AUTHORIZED", - "DUPLICATE", - "FRAUDULENT", - "GENERAL", - "INCORRECT_ACCOUNT_DETAILS", - "INSUFFICIENT_FUNDS", - "PRODUCT_NOT_RECEIVED", - "PRODUCT_UNACCEPTABLE", - "SUBSCRIPTION_CANCELLED", - "UNRECOGNIZED", - ) - - -class ShopifyPaymentsPayoutInterval(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("DAILY", "MANUAL", "MONTHLY", "WEEKLY") - - -class ShopifyPaymentsPayoutStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CANCELED", "FAILED", "IN_TRANSIT", "PAID", "SCHEDULED") - - -class ShopifyPaymentsPayoutTransactionType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("DEPOSIT", "WITHDRAWAL") - - -class ShopifyPaymentsVerificationDocumentType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("DRIVERS_LICENSE", "GOVERNMENT_IDENTIFICATION", "PASSPORT") - - -class ShopifyPaymentsVerificationStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("PENDING", "UNVERIFIED", "VERIFIED") - - -class StaffMemberDefaultImage(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("DEFAULT", "NOT_FOUND", "TRANSPARENT") - - -class StaffMemberPermission(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "APPLICATIONS", - "CHANNELS", - "CUSTOMERS", - "DASHBOARD", - "DOMAINS", - "DRAFT_ORDERS", - "EDIT_ORDERS", - "GIFT_CARDS", - "LINKS", - "LOCATIONS", - "MARKETING", - "MARKETING_SECTION", - "ORDERS", - "OVERVIEWS", - "PAGES", - "PAY_ORDERS_BY_VAULTED_CARD", - "PREFERENCES", - "PRODUCTS", - "REPORTS", - "THEMES", - ) - - -class StagedUploadHttpMethodType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("POST", "PUT") - - -class StagedUploadTargetGenerateUploadResource(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "BULK_MUTATION_VARIABLES", - "COLLECTION_IMAGE", - "FILE", - "IMAGE", - "MODEL_3D", - "PRODUCT_IMAGE", - "SHOP_IMAGE", - "URL_REDIRECT_IMPORT", - "VIDEO", - ) - - -class StandardMetafieldDefinitionEnableUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("INVALID", "LIMIT_EXCEEDED", "TAKEN", "TEMPLATE_NOT_FOUND", "UNSTRUCTURED_ALREADY_EXISTS") - - -class StorefrontID(sgqlc.types.Scalar): - __schema__ = shopify_schema - - -String = sgqlc.types.String - - -class SubscriptionBillingAttemptErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "AMOUNT_TOO_SMALL", - "AUTHENTICATION_ERROR", - "BUYER_CANCELED_PAYMENT_METHOD", - "CUSTOMER_INVALID", - "CUSTOMER_NOT_FOUND", - "EXPIRED_PAYMENT_METHOD", - "INVALID_CUSTOMER_BILLING_AGREEMENT", - "INVALID_PAYMENT_METHOD", - "INVALID_SHIPPING_ADDRESS", - "INVOICE_ALREADY_PAID", - "PAYMENT_METHOD_DECLINED", - "PAYMENT_METHOD_INCOMPATIBLE_WITH_GATEWAY_CONFIG", - "PAYMENT_METHOD_NOT_FOUND", - "PAYMENT_PROVIDER_IS_NOT_ENABLED", - "TEST_MODE", - "UNEXPECTED_ERROR", - ) - - -class SubscriptionBillingCycleBillingCycleStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("BILLED", "UNBILLED") - - -class SubscriptionBillingCycleErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "BILLING_DATE_SET_ON_SKIPPED", - "CYCLE_NOT_FOUND", - "EMPTY_BILLING_CYCLE_EDIT_SCHEDULE_INPUT", - "INVALID", - "INVALID_CYCLE_INDEX", - "INVALID_DATE", - "NO_CYCLE_EDITS", - "OUT_OF_BOUNDS", - ) - - -class SubscriptionBillingCycleScheduleEditInputScheduleEditReason(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("BUYER_INITIATED", "DEV_INITIATED", "MERCHANT_INITIATED") - - -class SubscriptionBillingCyclesSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CYCLE_INDEX", "ID", "RELEVANCE") - - -class SubscriptionBillingCyclesTargetSelection(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ALL",) - - -class SubscriptionContractErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("INVALID",) - - -class SubscriptionContractLastPaymentStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("FAILED", "SUCCEEDED") - - -class SubscriptionContractSubscriptionStatus(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ACTIVE", "CANCELLED", "EXPIRED", "FAILED", "PAUSED") - - -class SubscriptionDiscountRejectionReason(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "CURRENTLY_INACTIVE", - "CUSTOMER_NOT_ELIGIBLE", - "CUSTOMER_USAGE_LIMIT_REACHED", - "INCOMPATIBLE_PURCHASE_TYPE", - "INTERNAL_ERROR", - "NOT_FOUND", - "NO_ENTITLED_LINE_ITEMS", - "NO_ENTITLED_SHIPPING_LINES", - "PURCHASE_NOT_IN_RANGE", - "QUANTITY_NOT_IN_RANGE", - "USAGE_LIMIT_REACHED", - ) - - -class SubscriptionDraftErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "ALREADY_REMOVED", - "BILLING_CYCLE_ABSENT", - "BILLING_CYCLE_CONTRACT_DRAFT_BILLING_POLICY_INVALID", - "BILLING_CYCLE_CONTRACT_DRAFT_DELIVERY_POLICY_INVALID", - "BILLING_CYCLE_PRESENT", - "BLANK", - "COMMITTED", - "CONCATENATION_BILLING_CYCLE_CONTRACT_DRAFT_REQUIRED", - "CURRENCY_NOT_ENABLED", - "CUSTOMER_DOES_NOT_EXIST", - "CUSTOMER_MISMATCH", - "CYCLE_DISCOUNTS_UNIQUE_AFTER_CYCLE", - "CYCLE_INDEX_OUT_OF_RANGE", - "CYCLE_SELECTOR_VALIDATE_ONE_OF", - "CYCLE_START_DATE_OUT_OF_RANGE", - "DELIVERY_METHOD_REQUIRED", - "DELIVERY_MUST_BE_MULTIPLE_OF_BILLING", - "DUPLICATE_CONCATENATED_CONTRACTS", - "EXCEEDED_MAX_CONCATENATED_CONTRACTS", - "GREATER_THAN", - "GREATER_THAN_OR_EQUAL_TO", - "HAS_FUTURE_EDITS", - "INVALID", - "INVALID_ADJUSTMENT_TYPE", - "INVALID_ADJUSTMENT_VALUE", - "INVALID_BILLING_DATE", - "INVALID_LINES", - "INVALID_NOTE_LENGTH", - "LESS_THAN", - "LESS_THAN_OR_EQUAL_TO", - "NOT_AN_INTEGER", - "NOT_IN_RANGE", - "NO_ENTITLED_LINES", - "PRESENCE", - "SELLING_PLAN_MAX_CYCLES_MUST_BE_GREATER_THAN_MIN_CYCLES", - "STALE_CONTRACT", - "TOO_LONG", - "TOO_SHORT", - "UPCOMING_CYCLE_LIMIT_EXCEEDED", - ) - - -class SuggestedOrderTransactionKind(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("SUGGESTED_REFUND",) - - -class TaxExemption(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "CA_BC_COMMERCIAL_FISHERY_EXEMPTION", - "CA_BC_CONTRACTOR_EXEMPTION", - "CA_BC_PRODUCTION_AND_MACHINERY_EXEMPTION", - "CA_BC_RESELLER_EXEMPTION", - "CA_BC_SUB_CONTRACTOR_EXEMPTION", - "CA_DIPLOMAT_EXEMPTION", - "CA_MB_COMMERCIAL_FISHERY_EXEMPTION", - "CA_MB_FARMER_EXEMPTION", - "CA_MB_RESELLER_EXEMPTION", - "CA_NS_COMMERCIAL_FISHERY_EXEMPTION", - "CA_NS_FARMER_EXEMPTION", - "CA_ON_PURCHASE_EXEMPTION", - "CA_PE_COMMERCIAL_FISHERY_EXEMPTION", - "CA_SK_COMMERCIAL_FISHERY_EXEMPTION", - "CA_SK_CONTRACTOR_EXEMPTION", - "CA_SK_FARMER_EXEMPTION", - "CA_SK_PRODUCTION_AND_MACHINERY_EXEMPTION", - "CA_SK_RESELLER_EXEMPTION", - "CA_SK_SUB_CONTRACTOR_EXEMPTION", - "CA_STATUS_CARD_EXEMPTION", - "EU_REVERSE_CHARGE_EXEMPTION_RULE", - "US_AK_RESELLER_EXEMPTION", - "US_AL_RESELLER_EXEMPTION", - "US_AR_RESELLER_EXEMPTION", - "US_AZ_RESELLER_EXEMPTION", - "US_CA_RESELLER_EXEMPTION", - "US_CO_RESELLER_EXEMPTION", - "US_CT_RESELLER_EXEMPTION", - "US_DC_RESELLER_EXEMPTION", - "US_DE_RESELLER_EXEMPTION", - "US_FL_RESELLER_EXEMPTION", - "US_GA_RESELLER_EXEMPTION", - "US_HI_RESELLER_EXEMPTION", - "US_IA_RESELLER_EXEMPTION", - "US_ID_RESELLER_EXEMPTION", - "US_IL_RESELLER_EXEMPTION", - "US_IN_RESELLER_EXEMPTION", - "US_KS_RESELLER_EXEMPTION", - "US_KY_RESELLER_EXEMPTION", - "US_LA_RESELLER_EXEMPTION", - "US_MA_RESELLER_EXEMPTION", - "US_MD_RESELLER_EXEMPTION", - "US_ME_RESELLER_EXEMPTION", - "US_MI_RESELLER_EXEMPTION", - "US_MN_RESELLER_EXEMPTION", - "US_MO_RESELLER_EXEMPTION", - "US_MS_RESELLER_EXEMPTION", - "US_MT_RESELLER_EXEMPTION", - "US_NC_RESELLER_EXEMPTION", - "US_ND_RESELLER_EXEMPTION", - "US_NE_RESELLER_EXEMPTION", - "US_NH_RESELLER_EXEMPTION", - "US_NJ_RESELLER_EXEMPTION", - "US_NM_RESELLER_EXEMPTION", - "US_NV_RESELLER_EXEMPTION", - "US_NY_RESELLER_EXEMPTION", - "US_OH_RESELLER_EXEMPTION", - "US_OK_RESELLER_EXEMPTION", - "US_OR_RESELLER_EXEMPTION", - "US_PA_RESELLER_EXEMPTION", - "US_RI_RESELLER_EXEMPTION", - "US_SC_RESELLER_EXEMPTION", - "US_SD_RESELLER_EXEMPTION", - "US_TN_RESELLER_EXEMPTION", - "US_TX_RESELLER_EXEMPTION", - "US_UT_RESELLER_EXEMPTION", - "US_VA_RESELLER_EXEMPTION", - "US_VT_RESELLER_EXEMPTION", - "US_WA_RESELLER_EXEMPTION", - "US_WI_RESELLER_EXEMPTION", - "US_WV_RESELLER_EXEMPTION", - "US_WY_RESELLER_EXEMPTION", - ) - - -class TranslatableResourceType(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "COLLECTION", - "DELIVERY_METHOD_DEFINITION", - "EMAIL_TEMPLATE", - "LINK", - "METAFIELD", - "ONLINE_STORE_ARTICLE", - "ONLINE_STORE_BLOG", - "ONLINE_STORE_MENU", - "ONLINE_STORE_PAGE", - "ONLINE_STORE_THEME", - "PACKING_SLIP_TEMPLATE", - "PAYMENT_GATEWAY", - "PRODUCT", - "PRODUCT_OPTION", - "PRODUCT_VARIANT", - "SELLING_PLAN", - "SELLING_PLAN_GROUP", - "SHOP", - "SHOP_POLICY", - ) - - -class TranslationErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "BLANK", - "FAILS_RESOURCE_VALIDATION", - "INVALID", - "INVALID_CODE", - "INVALID_FORMAT", - "INVALID_KEY_FOR_MODEL", - "INVALID_LOCALE_FOR_MARKET", - "INVALID_LOCALE_FOR_SHOP", - "INVALID_MARKET_LOCALIZABLE_CONTENT", - "INVALID_TRANSLATABLE_CONTENT", - "MARKET_CUSTOM_CONTENT_NOT_ALLOWED", - "MARKET_DOES_NOT_EXIST", - "MARKET_LOCALE_CREATION_FAILED", - "RESOURCE_NOT_FOUND", - "RESOURCE_NOT_MARKET_CUSTOMIZABLE", - "TOO_MANY_KEYS_FOR_RESOURCE", - ) - - -class URL(sgqlc.types.Scalar): - __schema__ = shopify_schema - - -class UnitSystem(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("IMPERIAL_SYSTEM", "METRIC_SYSTEM") - - -class UnsignedInt64(sgqlc.types.Scalar): - __schema__ = shopify_schema - - -class UrlRedirectBulkDeleteByIdsUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("IDS_EMPTY",) - - -class UrlRedirectBulkDeleteBySavedSearchUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("INVALID_SAVED_SEARCH_QUERY", "SAVED_SEARCH_NOT_FOUND") - - -class UrlRedirectBulkDeleteBySearchUserErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("INVALID_SEARCH_ARGUMENT",) - - -class UrlRedirectErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CREATE_FAILED", "DELETE_FAILED", "DOES_NOT_EXIST", "UPDATE_FAILED") - - -class UrlRedirectImportErrorCode(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ALREADY_IMPORTED", "IN_PROGRESS", "NOT_FOUND") - - -class UrlRedirectSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("ID", "PATH", "RELEVANCE") - - -class UtcOffset(sgqlc.types.Scalar): - __schema__ = shopify_schema - - -class WebhookSubscriptionFormat(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("JSON", "XML") - - -class WebhookSubscriptionSortKeys(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("CREATED_AT", "ID", "RELEVANCE") - - -class WebhookSubscriptionTopic(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ( - "APP_PURCHASES_ONE_TIME_UPDATE", - "APP_SUBSCRIPTIONS_APPROACHING_CAPPED_AMOUNT", - "APP_SUBSCRIPTIONS_UPDATE", - "APP_UNINSTALLED", - "ATTRIBUTED_SESSIONS_FIRST", - "ATTRIBUTED_SESSIONS_LAST", - "BULK_OPERATIONS_FINISH", - "CARTS_CREATE", - "CARTS_UPDATE", - "CHANNELS_DELETE", - "CHECKOUTS_CREATE", - "CHECKOUTS_DELETE", - "CHECKOUTS_UPDATE", - "COLLECTIONS_CREATE", - "COLLECTIONS_DELETE", - "COLLECTIONS_UPDATE", - "COLLECTION_LISTINGS_ADD", - "COLLECTION_LISTINGS_REMOVE", - "COLLECTION_LISTINGS_UPDATE", - "COLLECTION_PUBLICATIONS_CREATE", - "COLLECTION_PUBLICATIONS_DELETE", - "COLLECTION_PUBLICATIONS_UPDATE", - "CUSTOMERS_CREATE", - "CUSTOMERS_DELETE", - "CUSTOMERS_DISABLE", - "CUSTOMERS_ENABLE", - "CUSTOMERS_MARKETING_CONSENT_UPDATE", - "CUSTOMERS_UPDATE", - "CUSTOMER_GROUPS_CREATE", - "CUSTOMER_GROUPS_DELETE", - "CUSTOMER_GROUPS_UPDATE", - "CUSTOMER_PAYMENT_METHODS_CREATE", - "CUSTOMER_PAYMENT_METHODS_REVOKE", - "CUSTOMER_PAYMENT_METHODS_UPDATE", - "DISPUTES_CREATE", - "DISPUTES_UPDATE", - "DOMAINS_CREATE", - "DOMAINS_DESTROY", - "DOMAINS_UPDATE", - "DRAFT_ORDERS_CREATE", - "DRAFT_ORDERS_DELETE", - "DRAFT_ORDERS_UPDATE", - "FULFILLMENTS_CREATE", - "FULFILLMENTS_UPDATE", - "FULFILLMENT_EVENTS_CREATE", - "FULFILLMENT_EVENTS_DELETE", - "INVENTORY_ITEMS_CREATE", - "INVENTORY_ITEMS_DELETE", - "INVENTORY_ITEMS_UPDATE", - "INVENTORY_LEVELS_CONNECT", - "INVENTORY_LEVELS_DISCONNECT", - "INVENTORY_LEVELS_UPDATE", - "LOCALES_CREATE", - "LOCALES_UPDATE", - "LOCATIONS_CREATE", - "LOCATIONS_DELETE", - "LOCATIONS_UPDATE", - "MARKETS_CREATE", - "MARKETS_DELETE", - "MARKETS_UPDATE", - "ORDERS_CANCELLED", - "ORDERS_CREATE", - "ORDERS_DELETE", - "ORDERS_EDITED", - "ORDERS_FULFILLED", - "ORDERS_PAID", - "ORDERS_PARTIALLY_FULFILLED", - "ORDERS_UPDATED", - "ORDER_TRANSACTIONS_CREATE", - "PAYMENT_TERMS_CREATE", - "PAYMENT_TERMS_DELETE", - "PAYMENT_TERMS_UPDATE", - "PRODUCTS_CREATE", - "PRODUCTS_DELETE", - "PRODUCTS_UPDATE", - "PRODUCT_LISTINGS_ADD", - "PRODUCT_LISTINGS_REMOVE", - "PRODUCT_LISTINGS_UPDATE", - "PRODUCT_PUBLICATIONS_CREATE", - "PRODUCT_PUBLICATIONS_DELETE", - "PRODUCT_PUBLICATIONS_UPDATE", - "PROFILES_CREATE", - "PROFILES_DELETE", - "PROFILES_UPDATE", - "REFUNDS_CREATE", - "SCHEDULED_PRODUCT_LISTINGS_ADD", - "SCHEDULED_PRODUCT_LISTINGS_REMOVE", - "SCHEDULED_PRODUCT_LISTINGS_UPDATE", - "SEGMENTS_CREATE", - "SEGMENTS_DELETE", - "SEGMENTS_UPDATE", - "SELLING_PLAN_GROUPS_CREATE", - "SELLING_PLAN_GROUPS_DELETE", - "SELLING_PLAN_GROUPS_UPDATE", - "SHIPPING_ADDRESSES_CREATE", - "SHIPPING_ADDRESSES_UPDATE", - "SHOP_UPDATE", - "SUBSCRIPTION_BILLING_ATTEMPTS_CHALLENGED", - "SUBSCRIPTION_BILLING_ATTEMPTS_FAILURE", - "SUBSCRIPTION_BILLING_ATTEMPTS_SUCCESS", - "SUBSCRIPTION_CONTRACTS_CREATE", - "SUBSCRIPTION_CONTRACTS_UPDATE", - "TAX_SERVICES_CREATE", - "TAX_SERVICES_UPDATE", - "TENDER_TRANSACTIONS_CREATE", - "THEMES_CREATE", - "THEMES_DELETE", - "THEMES_PUBLISH", - "THEMES_UPDATE", - "VARIANTS_IN_STOCK", - "VARIANTS_OUT_OF_STOCK", - ) - - -class WeightUnit(sgqlc.types.Enum): - __schema__ = shopify_schema - __choices__ = ("GRAMS", "KILOGRAMS", "OUNCES", "POUNDS") - - -######################################################################## -# Input Objects -######################################################################## -class AppPlanInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("app_usage_pricing_details", "app_recurring_pricing_details") - app_usage_pricing_details = sgqlc.types.Field("AppUsagePricingInput", graphql_name="appUsagePricingDetails") - app_recurring_pricing_details = sgqlc.types.Field("AppRecurringPricingInput", graphql_name="appRecurringPricingDetails") - - -class AppRecurringPricingInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("interval", "price", "discount") - interval = sgqlc.types.Field(AppPricingInterval, graphql_name="interval") - price = sgqlc.types.Field(sgqlc.types.non_null("MoneyInput"), graphql_name="price") - discount = sgqlc.types.Field("AppSubscriptionDiscountInput", graphql_name="discount") - - -class AppRevenueAttributionRecordInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("idempotency_key", "captured_at", "amount", "type", "test") - idempotency_key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="idempotencyKey") - captured_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="capturedAt") - amount = sgqlc.types.Field(sgqlc.types.non_null("MoneyInput"), graphql_name="amount") - type = sgqlc.types.Field(sgqlc.types.non_null(AppRevenueAttributionType), graphql_name="type") - test = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="test") - - -class AppSubscriptionDiscountInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("value", "duration_limit_in_intervals") - value = sgqlc.types.Field("AppSubscriptionDiscountValueInput", graphql_name="value") - duration_limit_in_intervals = sgqlc.types.Field(Int, graphql_name="durationLimitInIntervals") - - -class AppSubscriptionDiscountValueInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("percentage", "amount") - percentage = sgqlc.types.Field(Float, graphql_name="percentage") - amount = sgqlc.types.Field(Decimal, graphql_name="amount") - - -class AppSubscriptionLineItemInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("plan",) - plan = sgqlc.types.Field(sgqlc.types.non_null(AppPlanInput), graphql_name="plan") - - -class AppUsagePricingInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("capped_amount", "terms") - capped_amount = sgqlc.types.Field(sgqlc.types.non_null("MoneyInput"), graphql_name="cappedAmount") - terms = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="terms") - - -class AttributeInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("key", "value") - key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") - value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") - - -class BuyerExperienceConfigurationInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("checkout_to_draft", "payment_terms_template_id") - checkout_to_draft = sgqlc.types.Field(Boolean, graphql_name="checkoutToDraft") - payment_terms_template_id = sgqlc.types.Field(ID, graphql_name="paymentTermsTemplateId") - - -class CollectionDeleteInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id",) - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - - -class CollectionInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "description_html", - "handle", - "id", - "image", - "products", - "private_metafields", - "rule_set", - "template_suffix", - "sort_order", - "title", - "metafields", - "seo", - "redirect_new_handle", - ) - description_html = sgqlc.types.Field(String, graphql_name="descriptionHtml") - handle = sgqlc.types.Field(String, graphql_name="handle") - id = sgqlc.types.Field(ID, graphql_name="id") - image = sgqlc.types.Field("ImageInput", graphql_name="image") - products = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="products") - private_metafields = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null("PrivateMetafieldInput")), graphql_name="privateMetafields" - ) - rule_set = sgqlc.types.Field("CollectionRuleSetInput", graphql_name="ruleSet") - template_suffix = sgqlc.types.Field(String, graphql_name="templateSuffix") - sort_order = sgqlc.types.Field(CollectionSortOrder, graphql_name="sortOrder") - title = sgqlc.types.Field(String, graphql_name="title") - metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldInput")), graphql_name="metafields") - seo = sgqlc.types.Field("SEOInput", graphql_name="seo") - redirect_new_handle = sgqlc.types.Field(Boolean, graphql_name="redirectNewHandle") - - -class CollectionPublicationInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("publication_id",) - publication_id = sgqlc.types.Field(ID, graphql_name="publicationId") - - -class CollectionPublishInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id", "collection_publications") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - collection_publications = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CollectionPublicationInput))), - graphql_name="collectionPublications", - ) - - -class CollectionRuleInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("column", "relation", "condition") - column = sgqlc.types.Field(sgqlc.types.non_null(CollectionRuleColumn), graphql_name="column") - relation = sgqlc.types.Field(sgqlc.types.non_null(CollectionRuleRelation), graphql_name="relation") - condition = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="condition") - - -class CollectionRuleSetInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("applied_disjunctively", "rules") - applied_disjunctively = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliedDisjunctively") - rules = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(CollectionRuleInput)), graphql_name="rules") - - -class CollectionUnpublishInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id", "collection_publications") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - collection_publications = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CollectionPublicationInput))), - graphql_name="collectionPublications", - ) - - -class CompanyAddressInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("address1", "address2", "city", "zip", "recipient", "phone", "zone_code", "country_code") - address1 = sgqlc.types.Field(String, graphql_name="address1") - address2 = sgqlc.types.Field(String, graphql_name="address2") - city = sgqlc.types.Field(String, graphql_name="city") - zip = sgqlc.types.Field(String, graphql_name="zip") - recipient = sgqlc.types.Field(String, graphql_name="recipient") - phone = sgqlc.types.Field(String, graphql_name="phone") - zone_code = sgqlc.types.Field(String, graphql_name="zoneCode") - country_code = sgqlc.types.Field(CountryCode, graphql_name="countryCode") - - -class CompanyContactInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("first_name", "last_name", "email", "title", "locale", "phone") - first_name = sgqlc.types.Field(String, graphql_name="firstName") - last_name = sgqlc.types.Field(String, graphql_name="lastName") - email = sgqlc.types.Field(String, graphql_name="email") - title = sgqlc.types.Field(String, graphql_name="title") - locale = sgqlc.types.Field(String, graphql_name="locale") - phone = sgqlc.types.Field(String, graphql_name="phone") - - -class CompanyContactRoleAssign(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("company_contact_role_id", "company_location_id") - company_contact_role_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="companyContactRoleId") - company_location_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="companyLocationId") - - -class CompanyCreateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("company", "company_contact", "company_location") - company = sgqlc.types.Field(sgqlc.types.non_null("CompanyInput"), graphql_name="company") - company_contact = sgqlc.types.Field(CompanyContactInput, graphql_name="companyContact") - company_location = sgqlc.types.Field("CompanyLocationInput", graphql_name="companyLocation") - - -class CompanyInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("name", "note", "external_id", "customer_since") - name = sgqlc.types.Field(String, graphql_name="name") - note = sgqlc.types.Field(String, graphql_name="note") - external_id = sgqlc.types.Field(String, graphql_name="externalId") - customer_since = sgqlc.types.Field(DateTime, graphql_name="customerSince") - - -class CompanyLocationInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "name", - "phone", - "locale", - "external_id", - "note", - "buyer_experience_configuration", - "billing_address", - "shipping_address", - "billing_same_as_shipping", - "tax_registration_id", - "tax_exemptions", - ) - name = sgqlc.types.Field(String, graphql_name="name") - phone = sgqlc.types.Field(String, graphql_name="phone") - locale = sgqlc.types.Field(String, graphql_name="locale") - external_id = sgqlc.types.Field(String, graphql_name="externalId") - note = sgqlc.types.Field(String, graphql_name="note") - buyer_experience_configuration = sgqlc.types.Field(BuyerExperienceConfigurationInput, graphql_name="buyerExperienceConfiguration") - billing_address = sgqlc.types.Field(CompanyAddressInput, graphql_name="billingAddress") - shipping_address = sgqlc.types.Field(CompanyAddressInput, graphql_name="shippingAddress") - billing_same_as_shipping = sgqlc.types.Field(Boolean, graphql_name="billingSameAsShipping") - tax_registration_id = sgqlc.types.Field(String, graphql_name="taxRegistrationId") - tax_exemptions = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(TaxExemption)), graphql_name="taxExemptions") - - -class CompanyLocationRoleAssign(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("company_contact_role_id", "company_contact_id") - company_contact_role_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="companyContactRoleId") - company_contact_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="companyContactId") - - -class CompanyLocationUpdateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("name", "phone", "locale", "external_id", "note", "buyer_experience_configuration") - name = sgqlc.types.Field(String, graphql_name="name") - phone = sgqlc.types.Field(String, graphql_name="phone") - locale = sgqlc.types.Field(String, graphql_name="locale") - external_id = sgqlc.types.Field(String, graphql_name="externalId") - note = sgqlc.types.Field(String, graphql_name="note") - buyer_experience_configuration = sgqlc.types.Field(BuyerExperienceConfigurationInput, graphql_name="buyerExperienceConfiguration") - - -class ContextualPricingContext(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("country", "company_location_id") - country = sgqlc.types.Field(CountryCode, graphql_name="country") - company_location_id = sgqlc.types.Field(ID, graphql_name="companyLocationId") - - -class CountryHarmonizedSystemCodeInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("harmonized_system_code", "country_code") - harmonized_system_code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="harmonizedSystemCode") - country_code = sgqlc.types.Field(sgqlc.types.non_null(CountryCode), graphql_name="countryCode") - - -class CreateMediaInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("original_source", "alt", "media_content_type") - original_source = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="originalSource") - alt = sgqlc.types.Field(String, graphql_name="alt") - media_content_type = sgqlc.types.Field(sgqlc.types.non_null(MediaContentType), graphql_name="mediaContentType") - - -class CustomerDeleteInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id",) - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - - -class CustomerEmailMarketingConsentInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("marketing_opt_in_level", "marketing_state", "consent_updated_at") - marketing_opt_in_level = sgqlc.types.Field(CustomerMarketingOptInLevel, graphql_name="marketingOptInLevel") - marketing_state = sgqlc.types.Field(sgqlc.types.non_null(CustomerEmailMarketingState), graphql_name="marketingState") - consent_updated_at = sgqlc.types.Field(DateTime, graphql_name="consentUpdatedAt") - - -class CustomerEmailMarketingConsentUpdateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("customer_id", "email_marketing_consent") - customer_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="customerId") - email_marketing_consent = sgqlc.types.Field( - sgqlc.types.non_null(CustomerEmailMarketingConsentInput), graphql_name="emailMarketingConsent" - ) - - -class CustomerInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "addresses", - "email", - "first_name", - "id", - "last_name", - "locale", - "metafields", - "note", - "phone", - "private_metafields", - "tags", - "email_marketing_consent", - "sms_marketing_consent", - "tax_exempt", - "tax_exemptions", - ) - addresses = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("MailingAddressInput")), graphql_name="addresses") - email = sgqlc.types.Field(String, graphql_name="email") - first_name = sgqlc.types.Field(String, graphql_name="firstName") - id = sgqlc.types.Field(ID, graphql_name="id") - last_name = sgqlc.types.Field(String, graphql_name="lastName") - locale = sgqlc.types.Field(String, graphql_name="locale") - metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldInput")), graphql_name="metafields") - note = sgqlc.types.Field(String, graphql_name="note") - phone = sgqlc.types.Field(String, graphql_name="phone") - private_metafields = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null("PrivateMetafieldInput")), graphql_name="privateMetafields" - ) - tags = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="tags") - email_marketing_consent = sgqlc.types.Field(CustomerEmailMarketingConsentInput, graphql_name="emailMarketingConsent") - sms_marketing_consent = sgqlc.types.Field("CustomerSmsMarketingConsentInput", graphql_name="smsMarketingConsent") - tax_exempt = sgqlc.types.Field(Boolean, graphql_name="taxExempt") - tax_exemptions = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(TaxExemption)), graphql_name="taxExemptions") - - -class CustomerPaymentMethodRemoteInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("stripe_payment_method", "authorize_net_customer_payment_profile", "braintree_payment_method") - stripe_payment_method = sgqlc.types.Field("RemoteStripePaymentMethodInput", graphql_name="stripePaymentMethod") - authorize_net_customer_payment_profile = sgqlc.types.Field( - "RemoteAuthorizeNetCustomerPaymentProfileInput", graphql_name="authorizeNetCustomerPaymentProfile" - ) - braintree_payment_method = sgqlc.types.Field("RemoteBraintreePaymentMethodInput", graphql_name="braintreePaymentMethod") - - -class CustomerSmsMarketingConsentInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("marketing_opt_in_level", "marketing_state", "consent_updated_at") - marketing_opt_in_level = sgqlc.types.Field(CustomerMarketingOptInLevel, graphql_name="marketingOptInLevel") - marketing_state = sgqlc.types.Field(sgqlc.types.non_null(CustomerSmsMarketingState), graphql_name="marketingState") - consent_updated_at = sgqlc.types.Field(DateTime, graphql_name="consentUpdatedAt") - - -class CustomerSmsMarketingConsentUpdateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("customer_id", "sms_marketing_consent") - customer_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="customerId") - sms_marketing_consent = sgqlc.types.Field(sgqlc.types.non_null(CustomerSmsMarketingConsentInput), graphql_name="smsMarketingConsent") - - -class DelegateAccessTokenInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("delegate_access_scope", "expires_in") - delegate_access_scope = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="delegateAccessScope" - ) - expires_in = sgqlc.types.Field(Int, graphql_name="expiresIn") - - -class DeliveryCountryInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("code", "rest_of_world", "provinces", "include_all_provinces") - code = sgqlc.types.Field(CountryCode, graphql_name="code") - rest_of_world = sgqlc.types.Field(Boolean, graphql_name="restOfWorld") - provinces = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryProvinceInput")), graphql_name="provinces") - include_all_provinces = sgqlc.types.Field(Boolean, graphql_name="includeAllProvinces") - - -class DeliveryLocationGroupZoneInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id", "name", "countries", "method_definitions_to_create", "method_definitions_to_update") - id = sgqlc.types.Field(ID, graphql_name="id") - name = sgqlc.types.Field(String, graphql_name="name") - countries = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(DeliveryCountryInput)), graphql_name="countries") - method_definitions_to_create = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null("DeliveryMethodDefinitionInput")), - graphql_name="methodDefinitionsToCreate", - ) - method_definitions_to_update = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null("DeliveryMethodDefinitionInput")), - graphql_name="methodDefinitionsToUpdate", - ) - - -class DeliveryMethodDefinitionInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "id", - "name", - "description", - "active", - "rate_definition", - "participant", - "weight_conditions_to_create", - "price_conditions_to_create", - "conditions_to_update", - ) - id = sgqlc.types.Field(ID, graphql_name="id") - name = sgqlc.types.Field(String, graphql_name="name") - description = sgqlc.types.Field(String, graphql_name="description") - active = sgqlc.types.Field(Boolean, graphql_name="active") - rate_definition = sgqlc.types.Field("DeliveryRateDefinitionInput", graphql_name="rateDefinition") - participant = sgqlc.types.Field("DeliveryParticipantInput", graphql_name="participant") - weight_conditions_to_create = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null("DeliveryWeightConditionInput")), - graphql_name="weightConditionsToCreate", - ) - price_conditions_to_create = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null("DeliveryPriceConditionInput")), graphql_name="priceConditionsToCreate" - ) - conditions_to_update = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null("DeliveryUpdateConditionInput")), graphql_name="conditionsToUpdate" - ) - - -class DeliveryParticipantInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "id", - "carrier_service_id", - "fixed_fee", - "percentage_of_rate_fee", - "participant_services", - "adapt_to_new_services", - ) - id = sgqlc.types.Field(ID, graphql_name="id") - carrier_service_id = sgqlc.types.Field(ID, graphql_name="carrierServiceId") - fixed_fee = sgqlc.types.Field("MoneyInput", graphql_name="fixedFee") - percentage_of_rate_fee = sgqlc.types.Field(Float, graphql_name="percentageOfRateFee") - participant_services = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null("DeliveryParticipantServiceInput")), graphql_name="participantServices" - ) - adapt_to_new_services = sgqlc.types.Field(Boolean, graphql_name="adaptToNewServices") - - -class DeliveryParticipantServiceInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("name", "active") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - active = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="active") - - -class DeliveryPriceConditionInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("criteria", "operator") - criteria = sgqlc.types.Field("MoneyInput", graphql_name="criteria") - operator = sgqlc.types.Field(DeliveryConditionOperator, graphql_name="operator") - - -class DeliveryProfileInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "name", - "profile_location_groups", - "location_groups_to_create", - "location_groups_to_update", - "location_groups_to_delete", - "variants_to_associate", - "variants_to_dissociate", - "zones_to_delete", - "method_definitions_to_delete", - "conditions_to_delete", - "selling_plan_groups_to_associate", - "selling_plan_groups_to_dissociate", - ) - name = sgqlc.types.Field(String, graphql_name="name") - profile_location_groups = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null("DeliveryProfileLocationGroupInput")), - graphql_name="profileLocationGroups", - ) - location_groups_to_create = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null("DeliveryProfileLocationGroupInput")), - graphql_name="locationGroupsToCreate", - ) - location_groups_to_update = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null("DeliveryProfileLocationGroupInput")), - graphql_name="locationGroupsToUpdate", - ) - location_groups_to_delete = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="locationGroupsToDelete") - variants_to_associate = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="variantsToAssociate") - variants_to_dissociate = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="variantsToDissociate") - zones_to_delete = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="zonesToDelete") - method_definitions_to_delete = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="methodDefinitionsToDelete" - ) - conditions_to_delete = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="conditionsToDelete") - selling_plan_groups_to_associate = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="sellingPlanGroupsToAssociate" - ) - selling_plan_groups_to_dissociate = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="sellingPlanGroupsToDissociate" - ) - - -class DeliveryProfileLocationGroupInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id", "locations", "zones_to_create", "zones_to_update") - id = sgqlc.types.Field(ID, graphql_name="id") - locations = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="locations") - zones_to_create = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null(DeliveryLocationGroupZoneInput)), graphql_name="zonesToCreate" - ) - zones_to_update = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null(DeliveryLocationGroupZoneInput)), graphql_name="zonesToUpdate" - ) - - -class DeliveryProvinceInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") - - -class DeliveryRateDefinitionInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id", "price") - id = sgqlc.types.Field(ID, graphql_name="id") - price = sgqlc.types.Field(sgqlc.types.non_null("MoneyInput"), graphql_name="price") - - -class DeliverySettingInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("legacy_mode_profiles",) - legacy_mode_profiles = sgqlc.types.Field(Boolean, graphql_name="legacyModeProfiles") - - -class DeliveryUpdateConditionInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id", "criteria", "criteria_unit", "field", "operator") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - criteria = sgqlc.types.Field(Float, graphql_name="criteria") - criteria_unit = sgqlc.types.Field(String, graphql_name="criteriaUnit") - field = sgqlc.types.Field(DeliveryConditionField, graphql_name="field") - operator = sgqlc.types.Field(DeliveryConditionOperator, graphql_name="operator") - - -class DeliveryWeightConditionInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("criteria", "operator") - criteria = sgqlc.types.Field("WeightInput", graphql_name="criteria") - operator = sgqlc.types.Field(DeliveryConditionOperator, graphql_name="operator") - - -class DiscountAmountInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("amount", "applies_on_each_item") - amount = sgqlc.types.Field(Decimal, graphql_name="amount") - applies_on_each_item = sgqlc.types.Field(Boolean, graphql_name="appliesOnEachItem") - - -class DiscountAutomaticAppInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("combines_with", "function_id", "title", "starts_at", "ends_at", "metafields") - combines_with = sgqlc.types.Field("DiscountCombinesWithInput", graphql_name="combinesWith") - function_id = sgqlc.types.Field(String, graphql_name="functionId") - title = sgqlc.types.Field(String, graphql_name="title") - starts_at = sgqlc.types.Field(DateTime, graphql_name="startsAt") - ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") - metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldInput")), graphql_name="metafields") - - -class DiscountAutomaticBasicInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("combines_with", "title", "starts_at", "ends_at", "minimum_requirement", "customer_gets") - combines_with = sgqlc.types.Field("DiscountCombinesWithInput", graphql_name="combinesWith") - title = sgqlc.types.Field(String, graphql_name="title") - starts_at = sgqlc.types.Field(DateTime, graphql_name="startsAt") - ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") - minimum_requirement = sgqlc.types.Field("DiscountMinimumRequirementInput", graphql_name="minimumRequirement") - customer_gets = sgqlc.types.Field("DiscountCustomerGetsInput", graphql_name="customerGets") - - -class DiscountAutomaticBxgyInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "combines_with", - "starts_at", - "ends_at", - "title", - "uses_per_order_limit", - "customer_buys", - "customer_gets", - ) - combines_with = sgqlc.types.Field("DiscountCombinesWithInput", graphql_name="combinesWith") - starts_at = sgqlc.types.Field(DateTime, graphql_name="startsAt") - ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") - title = sgqlc.types.Field(String, graphql_name="title") - uses_per_order_limit = sgqlc.types.Field(UnsignedInt64, graphql_name="usesPerOrderLimit") - customer_buys = sgqlc.types.Field("DiscountCustomerBuysInput", graphql_name="customerBuys") - customer_gets = sgqlc.types.Field("DiscountCustomerGetsInput", graphql_name="customerGets") - - -class DiscountCodeAppInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "combines_with", - "function_id", - "title", - "starts_at", - "ends_at", - "usage_limit", - "applies_once_per_customer", - "customer_selection", - "code", - "metafields", - ) - combines_with = sgqlc.types.Field("DiscountCombinesWithInput", graphql_name="combinesWith") - function_id = sgqlc.types.Field(String, graphql_name="functionId") - title = sgqlc.types.Field(String, graphql_name="title") - starts_at = sgqlc.types.Field(DateTime, graphql_name="startsAt") - ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") - usage_limit = sgqlc.types.Field(Int, graphql_name="usageLimit") - applies_once_per_customer = sgqlc.types.Field(Boolean, graphql_name="appliesOncePerCustomer") - customer_selection = sgqlc.types.Field("DiscountCustomerSelectionInput", graphql_name="customerSelection") - code = sgqlc.types.Field(String, graphql_name="code") - metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldInput")), graphql_name="metafields") - - -class DiscountCodeBasicInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "combines_with", - "title", - "starts_at", - "ends_at", - "usage_limit", - "applies_once_per_customer", - "minimum_requirement", - "customer_gets", - "customer_selection", - "code", - "recurring_cycle_limit", - ) - combines_with = sgqlc.types.Field("DiscountCombinesWithInput", graphql_name="combinesWith") - title = sgqlc.types.Field(String, graphql_name="title") - starts_at = sgqlc.types.Field(DateTime, graphql_name="startsAt") - ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") - usage_limit = sgqlc.types.Field(Int, graphql_name="usageLimit") - applies_once_per_customer = sgqlc.types.Field(Boolean, graphql_name="appliesOncePerCustomer") - minimum_requirement = sgqlc.types.Field("DiscountMinimumRequirementInput", graphql_name="minimumRequirement") - customer_gets = sgqlc.types.Field("DiscountCustomerGetsInput", graphql_name="customerGets") - customer_selection = sgqlc.types.Field("DiscountCustomerSelectionInput", graphql_name="customerSelection") - code = sgqlc.types.Field(String, graphql_name="code") - recurring_cycle_limit = sgqlc.types.Field(Int, graphql_name="recurringCycleLimit") - - -class DiscountCodeBxgyInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "combines_with", - "title", - "starts_at", - "ends_at", - "customer_buys", - "customer_gets", - "customer_selection", - "code", - "usage_limit", - "uses_per_order_limit", - "applies_once_per_customer", - ) - combines_with = sgqlc.types.Field("DiscountCombinesWithInput", graphql_name="combinesWith") - title = sgqlc.types.Field(String, graphql_name="title") - starts_at = sgqlc.types.Field(DateTime, graphql_name="startsAt") - ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") - customer_buys = sgqlc.types.Field("DiscountCustomerBuysInput", graphql_name="customerBuys") - customer_gets = sgqlc.types.Field("DiscountCustomerGetsInput", graphql_name="customerGets") - customer_selection = sgqlc.types.Field("DiscountCustomerSelectionInput", graphql_name="customerSelection") - code = sgqlc.types.Field(String, graphql_name="code") - usage_limit = sgqlc.types.Field(Int, graphql_name="usageLimit") - uses_per_order_limit = sgqlc.types.Field(Int, graphql_name="usesPerOrderLimit") - applies_once_per_customer = sgqlc.types.Field(Boolean, graphql_name="appliesOncePerCustomer") - - -class DiscountCodeFreeShippingInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "combines_with", - "title", - "starts_at", - "ends_at", - "code", - "usage_limit", - "applies_once_per_customer", - "minimum_requirement", - "customer_selection", - "destination", - "maximum_shipping_price", - "recurring_cycle_limit", - "applies_on_one_time_purchase", - "applies_on_subscription", - ) - combines_with = sgqlc.types.Field("DiscountCombinesWithInput", graphql_name="combinesWith") - title = sgqlc.types.Field(String, graphql_name="title") - starts_at = sgqlc.types.Field(DateTime, graphql_name="startsAt") - ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") - code = sgqlc.types.Field(String, graphql_name="code") - usage_limit = sgqlc.types.Field(Int, graphql_name="usageLimit") - applies_once_per_customer = sgqlc.types.Field(Boolean, graphql_name="appliesOncePerCustomer") - minimum_requirement = sgqlc.types.Field("DiscountMinimumRequirementInput", graphql_name="minimumRequirement") - customer_selection = sgqlc.types.Field("DiscountCustomerSelectionInput", graphql_name="customerSelection") - destination = sgqlc.types.Field("DiscountShippingDestinationSelectionInput", graphql_name="destination") - maximum_shipping_price = sgqlc.types.Field(Decimal, graphql_name="maximumShippingPrice") - recurring_cycle_limit = sgqlc.types.Field(Int, graphql_name="recurringCycleLimit") - applies_on_one_time_purchase = sgqlc.types.Field(Boolean, graphql_name="appliesOnOneTimePurchase") - applies_on_subscription = sgqlc.types.Field(Boolean, graphql_name="appliesOnSubscription") - - -class DiscountCollectionsInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("add", "remove") - add = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="add") - remove = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="remove") - - -class DiscountCombinesWithInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("product_discounts", "order_discounts", "shipping_discounts") - product_discounts = sgqlc.types.Field(Boolean, graphql_name="productDiscounts") - order_discounts = sgqlc.types.Field(Boolean, graphql_name="orderDiscounts") - shipping_discounts = sgqlc.types.Field(Boolean, graphql_name="shippingDiscounts") - - -class DiscountCountriesInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("add", "remove", "include_rest_of_world") - add = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(CountryCode)), graphql_name="add") - remove = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(CountryCode)), graphql_name="remove") - include_rest_of_world = sgqlc.types.Field(Boolean, graphql_name="includeRestOfWorld") - - -class DiscountCustomerBuysInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("value", "items") - value = sgqlc.types.Field("DiscountCustomerBuysValueInput", graphql_name="value") - items = sgqlc.types.Field("DiscountItemsInput", graphql_name="items") - - -class DiscountCustomerBuysValueInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("quantity", "amount") - quantity = sgqlc.types.Field(UnsignedInt64, graphql_name="quantity") - amount = sgqlc.types.Field(Decimal, graphql_name="amount") - - -class DiscountCustomerGetsInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("value", "items", "applies_on_one_time_purchase", "applies_on_subscription") - value = sgqlc.types.Field("DiscountCustomerGetsValueInput", graphql_name="value") - items = sgqlc.types.Field("DiscountItemsInput", graphql_name="items") - applies_on_one_time_purchase = sgqlc.types.Field(Boolean, graphql_name="appliesOnOneTimePurchase") - applies_on_subscription = sgqlc.types.Field(Boolean, graphql_name="appliesOnSubscription") - - -class DiscountCustomerGetsValueInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("discount_on_quantity", "percentage", "discount_amount") - discount_on_quantity = sgqlc.types.Field("DiscountOnQuantityInput", graphql_name="discountOnQuantity") - percentage = sgqlc.types.Field(Float, graphql_name="percentage") - discount_amount = sgqlc.types.Field(DiscountAmountInput, graphql_name="discountAmount") - - -class DiscountCustomerSegmentsInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("add", "remove") - add = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="add") - remove = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="remove") - - -class DiscountCustomerSelectionInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("all", "customers", "customer_segments") - all = sgqlc.types.Field(Boolean, graphql_name="all") - customers = sgqlc.types.Field("DiscountCustomersInput", graphql_name="customers") - customer_segments = sgqlc.types.Field(DiscountCustomerSegmentsInput, graphql_name="customerSegments") - - -class DiscountCustomersInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("add", "remove") - add = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="add") - remove = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="remove") - - -class DiscountEffectInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("percentage",) - percentage = sgqlc.types.Field(Float, graphql_name="percentage") - - -class DiscountItemsInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("products", "collections", "all") - products = sgqlc.types.Field("DiscountProductsInput", graphql_name="products") - collections = sgqlc.types.Field(DiscountCollectionsInput, graphql_name="collections") - all = sgqlc.types.Field(Boolean, graphql_name="all") - - -class DiscountMinimumQuantityInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("greater_than_or_equal_to_quantity",) - greater_than_or_equal_to_quantity = sgqlc.types.Field(UnsignedInt64, graphql_name="greaterThanOrEqualToQuantity") - - -class DiscountMinimumRequirementInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("quantity", "subtotal") - quantity = sgqlc.types.Field(DiscountMinimumQuantityInput, graphql_name="quantity") - subtotal = sgqlc.types.Field("DiscountMinimumSubtotalInput", graphql_name="subtotal") - - -class DiscountMinimumSubtotalInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("greater_than_or_equal_to_subtotal",) - greater_than_or_equal_to_subtotal = sgqlc.types.Field(Decimal, graphql_name="greaterThanOrEqualToSubtotal") - - -class DiscountOnQuantityInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("quantity", "effect") - quantity = sgqlc.types.Field(UnsignedInt64, graphql_name="quantity") - effect = sgqlc.types.Field(DiscountEffectInput, graphql_name="effect") - - -class DiscountProductsInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("products_to_add", "products_to_remove", "product_variants_to_add", "product_variants_to_remove") - products_to_add = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="productsToAdd") - products_to_remove = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="productsToRemove") - product_variants_to_add = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="productVariantsToAdd") - product_variants_to_remove = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="productVariantsToRemove") - - -class DiscountRedeemCodeInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") - - -class DiscountShippingDestinationSelectionInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("all", "countries") - all = sgqlc.types.Field(Boolean, graphql_name="all") - countries = sgqlc.types.Field(DiscountCountriesInput, graphql_name="countries") - - -class DraftOrderAppliedDiscountInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("amount", "description", "title", "value", "value_type") - amount = sgqlc.types.Field(Money, graphql_name="amount") - description = sgqlc.types.Field(String, graphql_name="description") - title = sgqlc.types.Field(String, graphql_name="title") - value = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="value") - value_type = sgqlc.types.Field(sgqlc.types.non_null(DraftOrderAppliedDiscountType), graphql_name="valueType") - - -class DraftOrderDeleteInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id",) - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - - -class DraftOrderInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "applied_discount", - "billing_address", - "custom_attributes", - "email", - "line_items", - "metafields", - "private_metafields", - "localization_extensions", - "note", - "shipping_address", - "shipping_line", - "tags", - "tax_exempt", - "use_customer_default_address", - "visible_to_customer", - "reserve_inventory_until", - "presentment_currency_code", - "market_region_country_code", - "phone", - "payment_terms", - "purchasing_entity", - "source_name", - ) - applied_discount = sgqlc.types.Field(DraftOrderAppliedDiscountInput, graphql_name="appliedDiscount") - billing_address = sgqlc.types.Field("MailingAddressInput", graphql_name="billingAddress") - custom_attributes = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(AttributeInput)), graphql_name="customAttributes") - email = sgqlc.types.Field(String, graphql_name="email") - line_items = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("DraftOrderLineItemInput")), graphql_name="lineItems") - metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldInput")), graphql_name="metafields") - private_metafields = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null("PrivateMetafieldInput")), graphql_name="privateMetafields" - ) - localization_extensions = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null("LocalizationExtensionInput")), graphql_name="localizationExtensions" - ) - note = sgqlc.types.Field(String, graphql_name="note") - shipping_address = sgqlc.types.Field("MailingAddressInput", graphql_name="shippingAddress") - shipping_line = sgqlc.types.Field("ShippingLineInput", graphql_name="shippingLine") - tags = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="tags") - tax_exempt = sgqlc.types.Field(Boolean, graphql_name="taxExempt") - use_customer_default_address = sgqlc.types.Field(Boolean, graphql_name="useCustomerDefaultAddress") - visible_to_customer = sgqlc.types.Field(Boolean, graphql_name="visibleToCustomer") - reserve_inventory_until = sgqlc.types.Field(DateTime, graphql_name="reserveInventoryUntil") - presentment_currency_code = sgqlc.types.Field(CurrencyCode, graphql_name="presentmentCurrencyCode") - market_region_country_code = sgqlc.types.Field(CountryCode, graphql_name="marketRegionCountryCode") - phone = sgqlc.types.Field(String, graphql_name="phone") - payment_terms = sgqlc.types.Field("PaymentTermsInput", graphql_name="paymentTerms") - purchasing_entity = sgqlc.types.Field("PurchasingEntityInput", graphql_name="purchasingEntity") - source_name = sgqlc.types.Field(String, graphql_name="sourceName") - - -class DraftOrderLineItemInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "applied_discount", - "custom_attributes", - "original_unit_price", - "quantity", - "requires_shipping", - "sku", - "taxable", - "title", - "variant_id", - "weight", - ) - applied_discount = sgqlc.types.Field(DraftOrderAppliedDiscountInput, graphql_name="appliedDiscount") - custom_attributes = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(AttributeInput)), graphql_name="customAttributes") - original_unit_price = sgqlc.types.Field(Money, graphql_name="originalUnitPrice") - quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") - requires_shipping = sgqlc.types.Field(Boolean, graphql_name="requiresShipping") - sku = sgqlc.types.Field(String, graphql_name="sku") - taxable = sgqlc.types.Field(Boolean, graphql_name="taxable") - title = sgqlc.types.Field(String, graphql_name="title") - variant_id = sgqlc.types.Field(ID, graphql_name="variantId") - weight = sgqlc.types.Field("WeightInput", graphql_name="weight") - - -class EmailInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("subject", "to", "from_", "body", "bcc", "custom_message") - subject = sgqlc.types.Field(String, graphql_name="subject") - to = sgqlc.types.Field(String, graphql_name="to") - from_ = sgqlc.types.Field(String, graphql_name="from") - body = sgqlc.types.Field(String, graphql_name="body") - bcc = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="bcc") - custom_message = sgqlc.types.Field(String, graphql_name="customMessage") - - -class EventBridgeWebhookSubscriptionInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("arn", "format", "include_fields", "metafield_namespaces") - arn = sgqlc.types.Field(ARN, graphql_name="arn") - format = sgqlc.types.Field(WebhookSubscriptionFormat, graphql_name="format") - include_fields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="includeFields") - metafield_namespaces = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="metafieldNamespaces") - - -class FileCreateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("original_source", "content_type", "alt") - original_source = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="originalSource") - content_type = sgqlc.types.Field(FileContentType, graphql_name="contentType") - alt = sgqlc.types.Field(String, graphql_name="alt") - - -class FileUpdateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id", "alt", "preview_image_source") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - alt = sgqlc.types.Field(String, graphql_name="alt") - preview_image_source = sgqlc.types.Field(String, graphql_name="previewImageSource") - - -class FulfillmentOrderHoldInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("reason", "reason_notes", "notify_merchant") - reason = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentHoldReason), graphql_name="reason") - reason_notes = sgqlc.types.Field(String, graphql_name="reasonNotes") - notify_merchant = sgqlc.types.Field(Boolean, graphql_name="notifyMerchant") - - -class FulfillmentOrderLineItemInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id", "quantity") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") - - -class FulfillmentOrderLineItemsInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("fulfillment_order_id", "fulfillment_order_line_items") - fulfillment_order_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="fulfillmentOrderId") - fulfillment_order_line_items = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null(FulfillmentOrderLineItemInput)), - graphql_name="fulfillmentOrderLineItems", - ) - - -class FulfillmentOriginAddressInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("address1", "address2", "city", "zip", "province_code", "country_code") - address1 = sgqlc.types.Field(String, graphql_name="address1") - address2 = sgqlc.types.Field(String, graphql_name="address2") - city = sgqlc.types.Field(String, graphql_name="city") - zip = sgqlc.types.Field(String, graphql_name="zip") - province_code = sgqlc.types.Field(String, graphql_name="provinceCode") - country_code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="countryCode") - - -class FulfillmentTrackingInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("number", "url", "company", "numbers", "urls") - number = sgqlc.types.Field(String, graphql_name="number") - url = sgqlc.types.Field(URL, graphql_name="url") - company = sgqlc.types.Field(String, graphql_name="company") - numbers = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="numbers") - urls = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(URL)), graphql_name="urls") - - -class FulfillmentV2Input(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("tracking_info", "notify_customer", "line_items_by_fulfillment_order", "origin_address") - tracking_info = sgqlc.types.Field(FulfillmentTrackingInput, graphql_name="trackingInfo") - notify_customer = sgqlc.types.Field(Boolean, graphql_name="notifyCustomer") - line_items_by_fulfillment_order = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FulfillmentOrderLineItemsInput))), - graphql_name="lineItemsByFulfillmentOrder", - ) - origin_address = sgqlc.types.Field(FulfillmentOriginAddressInput, graphql_name="originAddress") - - -class GiftCardCreateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("initial_value", "code", "customer_id", "expires_on", "note", "template_suffix") - initial_value = sgqlc.types.Field(sgqlc.types.non_null(Decimal), graphql_name="initialValue") - code = sgqlc.types.Field(String, graphql_name="code") - customer_id = sgqlc.types.Field(ID, graphql_name="customerId") - expires_on = sgqlc.types.Field(Date, graphql_name="expiresOn") - note = sgqlc.types.Field(String, graphql_name="note") - template_suffix = sgqlc.types.Field(String, graphql_name="templateSuffix") - - -class GiftCardUpdateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("note", "expires_on", "customer_id", "template_suffix") - note = sgqlc.types.Field(String, graphql_name="note") - expires_on = sgqlc.types.Field(Date, graphql_name="expiresOn") - customer_id = sgqlc.types.Field(ID, graphql_name="customerId") - template_suffix = sgqlc.types.Field(String, graphql_name="templateSuffix") - - -class ImageInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id", "alt_text", "src") - id = sgqlc.types.Field(ID, graphql_name="id") - alt_text = sgqlc.types.Field(String, graphql_name="altText") - src = sgqlc.types.Field(String, graphql_name="src") - - -class ImageTransformInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("crop", "max_width", "max_height", "scale", "preferred_content_type") - crop = sgqlc.types.Field(CropRegion, graphql_name="crop") - max_width = sgqlc.types.Field(Int, graphql_name="maxWidth") - max_height = sgqlc.types.Field(Int, graphql_name="maxHeight") - scale = sgqlc.types.Field(Int, graphql_name="scale") - preferred_content_type = sgqlc.types.Field(ImageContentType, graphql_name="preferredContentType") - - -class IncomingRequestLineItemInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("fulfillment_order_line_item_id", "message") - fulfillment_order_line_item_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="fulfillmentOrderLineItemId") - message = sgqlc.types.Field(String, graphql_name="message") - - -class InventoryAdjustItemInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("inventory_item_id", "available_delta") - inventory_item_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="inventoryItemId") - available_delta = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="availableDelta") - - -class InventoryAdjustQuantityInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("inventory_level_id", "available_delta") - inventory_level_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="inventoryLevelId") - available_delta = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="availableDelta") - - -class InventoryBulkToggleActivationInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("location_id", "activate") - location_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="locationId") - activate = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="activate") - - -class InventoryItemInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("cost", "tracked") - cost = sgqlc.types.Field(Decimal, graphql_name="cost") - tracked = sgqlc.types.Field(Boolean, graphql_name="tracked") - - -class InventoryItemUpdateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "cost", - "tracked", - "country_code_of_origin", - "province_code_of_origin", - "harmonized_system_code", - "country_harmonized_system_codes", - ) - cost = sgqlc.types.Field(Decimal, graphql_name="cost") - tracked = sgqlc.types.Field(Boolean, graphql_name="tracked") - country_code_of_origin = sgqlc.types.Field(CountryCode, graphql_name="countryCodeOfOrigin") - province_code_of_origin = sgqlc.types.Field(String, graphql_name="provinceCodeOfOrigin") - harmonized_system_code = sgqlc.types.Field(String, graphql_name="harmonizedSystemCode") - country_harmonized_system_codes = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null(CountryHarmonizedSystemCodeInput)), - graphql_name="countryHarmonizedSystemCodes", - ) - - -class InventoryLevelInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("available_quantity", "location_id") - available_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="availableQuantity") - location_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="locationId") - - -class LocalizationExtensionInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("key", "value") - key = sgqlc.types.Field(sgqlc.types.non_null(LocalizationExtensionKey), graphql_name="key") - value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") - - -class LocationAddAddressInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("address1", "address2", "city", "phone", "zip", "country_code", "province_code") - address1 = sgqlc.types.Field(String, graphql_name="address1") - address2 = sgqlc.types.Field(String, graphql_name="address2") - city = sgqlc.types.Field(String, graphql_name="city") - phone = sgqlc.types.Field(String, graphql_name="phone") - zip = sgqlc.types.Field(String, graphql_name="zip") - country_code = sgqlc.types.Field(sgqlc.types.non_null(CountryCode), graphql_name="countryCode") - province_code = sgqlc.types.Field(String, graphql_name="provinceCode") - - -class LocationAddInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("name", "address", "fulfills_online_orders") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - address = sgqlc.types.Field(sgqlc.types.non_null(LocationAddAddressInput), graphql_name="address") - fulfills_online_orders = sgqlc.types.Field(Boolean, graphql_name="fulfillsOnlineOrders") - - -class LocationEditAddressInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("address1", "address2", "city", "phone", "zip", "country_code", "province_code") - address1 = sgqlc.types.Field(String, graphql_name="address1") - address2 = sgqlc.types.Field(String, graphql_name="address2") - city = sgqlc.types.Field(String, graphql_name="city") - phone = sgqlc.types.Field(String, graphql_name="phone") - zip = sgqlc.types.Field(String, graphql_name="zip") - country_code = sgqlc.types.Field(CountryCode, graphql_name="countryCode") - province_code = sgqlc.types.Field(String, graphql_name="provinceCode") - - -class LocationEditInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("name", "address", "fulfills_online_orders") - name = sgqlc.types.Field(String, graphql_name="name") - address = sgqlc.types.Field(LocationEditAddressInput, graphql_name="address") - fulfills_online_orders = sgqlc.types.Field(Boolean, graphql_name="fulfillsOnlineOrders") - - -class MailingAddressInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "address1", - "address2", - "city", - "company", - "country_code", - "first_name", - "last_name", - "phone", - "province_code", - "zip", - ) - address1 = sgqlc.types.Field(String, graphql_name="address1") - address2 = sgqlc.types.Field(String, graphql_name="address2") - city = sgqlc.types.Field(String, graphql_name="city") - company = sgqlc.types.Field(String, graphql_name="company") - country_code = sgqlc.types.Field(CountryCode, graphql_name="countryCode") - first_name = sgqlc.types.Field(String, graphql_name="firstName") - last_name = sgqlc.types.Field(String, graphql_name="lastName") - phone = sgqlc.types.Field(String, graphql_name="phone") - province_code = sgqlc.types.Field(String, graphql_name="provinceCode") - zip = sgqlc.types.Field(String, graphql_name="zip") - - -class MarketCreateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("name", "enabled", "regions") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - enabled = sgqlc.types.Field(Boolean, graphql_name="enabled") - regions = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketRegionCreateInput"))), - graphql_name="regions", - ) - - -class MarketCurrencySettingsUpdateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("base_currency", "local_currencies") - base_currency = sgqlc.types.Field(CurrencyCode, graphql_name="baseCurrency") - local_currencies = sgqlc.types.Field(Boolean, graphql_name="localCurrencies") - - -class MarketLocalizationRegisterInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("market_id", "key", "value", "market_localizable_content_digest") - market_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="marketId") - key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") - value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") - market_localizable_content_digest = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="marketLocalizableContentDigest") - - -class MarketRegionCreateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("country_code",) - country_code = sgqlc.types.Field(sgqlc.types.non_null(CountryCode), graphql_name="countryCode") - - -class MarketUpdateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("name", "enabled") - name = sgqlc.types.Field(String, graphql_name="name") - enabled = sgqlc.types.Field(Boolean, graphql_name="enabled") - - -class MarketWebPresenceCreateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("domain_id", "default_locale", "alternate_locales", "subfolder_suffix") - domain_id = sgqlc.types.Field(ID, graphql_name="domainId") - default_locale = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="defaultLocale") - alternate_locales = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="alternateLocales") - subfolder_suffix = sgqlc.types.Field(String, graphql_name="subfolderSuffix") - - -class MarketWebPresenceUpdateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("domain_id", "default_locale", "alternate_locales", "subfolder_suffix") - domain_id = sgqlc.types.Field(ID, graphql_name="domainId") - default_locale = sgqlc.types.Field(String, graphql_name="defaultLocale") - alternate_locales = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="alternateLocales") - subfolder_suffix = sgqlc.types.Field(String, graphql_name="subfolderSuffix") - - -class MarketingActivityBudgetInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("budget_type", "total") - budget_type = sgqlc.types.Field(MarketingBudgetBudgetType, graphql_name="budgetType") - total = sgqlc.types.Field("MoneyInput", graphql_name="total") - - -class MarketingActivityCreateExternalInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "title", - "utm", - "budget", - "ad_spend", - "remote_id", - "remote_url", - "remote_preview_image_url", - "tactic", - "channel", - "referring_domain", - "scheduled_start", - "scheduled_end", - "start", - "end", - ) - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - utm = sgqlc.types.Field(sgqlc.types.non_null("UTMInput"), graphql_name="utm") - budget = sgqlc.types.Field(MarketingActivityBudgetInput, graphql_name="budget") - ad_spend = sgqlc.types.Field("MoneyInput", graphql_name="adSpend") - remote_id = sgqlc.types.Field(String, graphql_name="remoteId") - remote_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="remoteUrl") - remote_preview_image_url = sgqlc.types.Field(URL, graphql_name="remotePreviewImageUrl") - tactic = sgqlc.types.Field(sgqlc.types.non_null(MarketingTactic), graphql_name="tactic") - channel = sgqlc.types.Field(sgqlc.types.non_null(MarketingChannel), graphql_name="channel") - referring_domain = sgqlc.types.Field(String, graphql_name="referringDomain") - scheduled_start = sgqlc.types.Field(DateTime, graphql_name="scheduledStart") - scheduled_end = sgqlc.types.Field(DateTime, graphql_name="scheduledEnd") - start = sgqlc.types.Field(DateTime, graphql_name="start") - end = sgqlc.types.Field(DateTime, graphql_name="end") - - -class MarketingActivityCreateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "marketing_activity_title", - "form_data", - "marketing_activity_extension_id", - "context", - "utm", - "status", - "budget", - ) - marketing_activity_title = sgqlc.types.Field(String, graphql_name="marketingActivityTitle") - form_data = sgqlc.types.Field(String, graphql_name="formData") - marketing_activity_extension_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="marketingActivityExtensionId") - context = sgqlc.types.Field(String, graphql_name="context") - utm = sgqlc.types.Field("UTMInput", graphql_name="utm") - status = sgqlc.types.Field(sgqlc.types.non_null(MarketingActivityStatus), graphql_name="status") - budget = sgqlc.types.Field(MarketingActivityBudgetInput, graphql_name="budget") - - -class MarketingActivityUpdateExternalInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "title", - "utm", - "budget", - "ad_spend", - "remote_url", - "remote_preview_image_url", - "tactic", - "channel", - "referring_domain", - "scheduled_start", - "scheduled_end", - "start", - "end", - ) - title = sgqlc.types.Field(String, graphql_name="title") - utm = sgqlc.types.Field("UTMInput", graphql_name="utm") - budget = sgqlc.types.Field(MarketingActivityBudgetInput, graphql_name="budget") - ad_spend = sgqlc.types.Field("MoneyInput", graphql_name="adSpend") - remote_url = sgqlc.types.Field(URL, graphql_name="remoteUrl") - remote_preview_image_url = sgqlc.types.Field(URL, graphql_name="remotePreviewImageUrl") - tactic = sgqlc.types.Field(MarketingTactic, graphql_name="tactic") - channel = sgqlc.types.Field(MarketingChannel, graphql_name="channel") - referring_domain = sgqlc.types.Field(String, graphql_name="referringDomain") - scheduled_start = sgqlc.types.Field(DateTime, graphql_name="scheduledStart") - scheduled_end = sgqlc.types.Field(DateTime, graphql_name="scheduledEnd") - start = sgqlc.types.Field(DateTime, graphql_name="start") - end = sgqlc.types.Field(DateTime, graphql_name="end") - - -class MarketingActivityUpdateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "id", - "marketing_recommendation_id", - "title", - "budget", - "status", - "target_status", - "form_data", - "utm", - "marketed_resources", - "errors", - ) - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - marketing_recommendation_id = sgqlc.types.Field(ID, graphql_name="marketingRecommendationId") - title = sgqlc.types.Field(String, graphql_name="title") - budget = sgqlc.types.Field(MarketingActivityBudgetInput, graphql_name="budget") - status = sgqlc.types.Field(MarketingActivityStatus, graphql_name="status") - target_status = sgqlc.types.Field(MarketingActivityStatus, graphql_name="targetStatus") - form_data = sgqlc.types.Field(String, graphql_name="formData") - utm = sgqlc.types.Field("UTMInput", graphql_name="utm") - marketed_resources = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="marketedResources") - errors = sgqlc.types.Field(JSON, graphql_name="errors") - - -class MarketingEngagementInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "occurred_on", - "impressions_count", - "views_count", - "clicks_count", - "shares_count", - "favorites_count", - "comments_count", - "unsubscribes_count", - "complaints_count", - "fails_count", - "sends_count", - "unique_views_count", - "unique_clicks_count", - "ad_spend", - "is_cumulative", - "utc_offset", - "fetched_at", - ) - occurred_on = sgqlc.types.Field(sgqlc.types.non_null(Date), graphql_name="occurredOn") - impressions_count = sgqlc.types.Field(Int, graphql_name="impressionsCount") - views_count = sgqlc.types.Field(Int, graphql_name="viewsCount") - clicks_count = sgqlc.types.Field(Int, graphql_name="clicksCount") - shares_count = sgqlc.types.Field(Int, graphql_name="sharesCount") - favorites_count = sgqlc.types.Field(Int, graphql_name="favoritesCount") - comments_count = sgqlc.types.Field(Int, graphql_name="commentsCount") - unsubscribes_count = sgqlc.types.Field(Int, graphql_name="unsubscribesCount") - complaints_count = sgqlc.types.Field(Int, graphql_name="complaintsCount") - fails_count = sgqlc.types.Field(Int, graphql_name="failsCount") - sends_count = sgqlc.types.Field(Int, graphql_name="sendsCount") - unique_views_count = sgqlc.types.Field(Int, graphql_name="uniqueViewsCount") - unique_clicks_count = sgqlc.types.Field(Int, graphql_name="uniqueClicksCount") - ad_spend = sgqlc.types.Field("MoneyInput", graphql_name="adSpend") - is_cumulative = sgqlc.types.Field(Boolean, graphql_name="isCumulative") - utc_offset = sgqlc.types.Field(UtcOffset, graphql_name="utcOffset") - fetched_at = sgqlc.types.Field(DateTime, graphql_name="fetchedAt") - - -class MetafieldDefinitionInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "namespace", - "key", - "name", - "description", - "owner_type", - "type", - "validations", - "visible_to_storefront_api", - "pin", - ) - namespace = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="namespace") - key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - description = sgqlc.types.Field(String, graphql_name="description") - owner_type = sgqlc.types.Field(sgqlc.types.non_null(MetafieldOwnerType), graphql_name="ownerType") - type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="type") - validations = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null("MetafieldDefinitionValidationInput")), graphql_name="validations" - ) - visible_to_storefront_api = sgqlc.types.Field(Boolean, graphql_name="visibleToStorefrontApi") - pin = sgqlc.types.Field(Boolean, graphql_name="pin") - - -class MetafieldDefinitionUpdateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("namespace", "key", "name", "description", "owner_type", "pin", "visible_to_storefront_api") - namespace = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="namespace") - key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") - name = sgqlc.types.Field(String, graphql_name="name") - description = sgqlc.types.Field(String, graphql_name="description") - owner_type = sgqlc.types.Field(sgqlc.types.non_null(MetafieldOwnerType), graphql_name="ownerType") - pin = sgqlc.types.Field(Boolean, graphql_name="pin") - visible_to_storefront_api = sgqlc.types.Field(Boolean, graphql_name="visibleToStorefrontApi") - - -class MetafieldDefinitionValidationInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("name", "value") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") - - -class MetafieldDeleteInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id",) - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - - -class MetafieldInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("description", "id", "namespace", "key", "value", "type") - description = sgqlc.types.Field(String, graphql_name="description") - id = sgqlc.types.Field(ID, graphql_name="id") - namespace = sgqlc.types.Field(String, graphql_name="namespace") - key = sgqlc.types.Field(String, graphql_name="key") - value = sgqlc.types.Field(String, graphql_name="value") - type = sgqlc.types.Field(String, graphql_name="type") - - -class MetafieldStorefrontVisibilityInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("namespace", "key", "owner_type") - namespace = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="namespace") - key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") - owner_type = sgqlc.types.Field(sgqlc.types.non_null(MetafieldOwnerType), graphql_name="ownerType") - - -class MetafieldsSetInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("owner_id", "namespace", "key", "value", "type") - owner_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="ownerId") - namespace = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="namespace") - key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") - value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") - type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="type") - - -class MoneyInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("amount", "currency_code") - amount = sgqlc.types.Field(sgqlc.types.non_null(Decimal), graphql_name="amount") - currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currencyCode") - - -class MoveInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id", "new_position") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - new_position = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="newPosition") - - -class OrderCaptureInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id", "parent_transaction_id", "amount", "currency") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - parent_transaction_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="parentTransactionId") - amount = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="amount") - currency = sgqlc.types.Field(CurrencyCode, graphql_name="currency") - - -class OrderCloseInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id",) - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - - -class OrderEditAppliedDiscountInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("description", "fixed_value", "percent_value") - description = sgqlc.types.Field(String, graphql_name="description") - fixed_value = sgqlc.types.Field(MoneyInput, graphql_name="fixedValue") - percent_value = sgqlc.types.Field(Float, graphql_name="percentValue") - - -class OrderInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "id", - "email", - "note", - "tags", - "shipping_address", - "custom_attributes", - "metafields", - "localization_extensions", - ) - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - email = sgqlc.types.Field(String, graphql_name="email") - note = sgqlc.types.Field(String, graphql_name="note") - tags = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="tags") - shipping_address = sgqlc.types.Field(MailingAddressInput, graphql_name="shippingAddress") - custom_attributes = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(AttributeInput)), graphql_name="customAttributes") - metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(MetafieldInput)), graphql_name="metafields") - localization_extensions = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null(LocalizationExtensionInput)), graphql_name="localizationExtensions" - ) - - -class OrderMarkAsPaidInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id",) - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - - -class OrderOpenInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id",) - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - - -class OrderTransactionInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("amount", "gateway", "kind", "order_id", "parent_id") - amount = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="amount") - gateway = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="gateway") - kind = sgqlc.types.Field(sgqlc.types.non_null(OrderTransactionKind), graphql_name="kind") - order_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="orderId") - parent_id = sgqlc.types.Field(ID, graphql_name="parentId") - - -class PaymentScheduleInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("issued_at", "due_at") - issued_at = sgqlc.types.Field(DateTime, graphql_name="issuedAt") - due_at = sgqlc.types.Field(DateTime, graphql_name="dueAt") - - -class PaymentTermsCreateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("payment_terms_template_id", "payment_schedules") - payment_terms_template_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="paymentTermsTemplateId") - payment_schedules = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(PaymentScheduleInput)), graphql_name="paymentSchedules") - - -class PaymentTermsDeleteInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("payment_terms_id",) - payment_terms_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="paymentTermsId") - - -class PaymentTermsInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("payment_terms_template_id", "payment_schedules") - payment_terms_template_id = sgqlc.types.Field(ID, graphql_name="paymentTermsTemplateId") - payment_schedules = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(PaymentScheduleInput)), graphql_name="paymentSchedules") - - -class PaymentTermsUpdateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("payment_terms_id", "payment_terms_attributes") - payment_terms_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="paymentTermsId") - payment_terms_attributes = sgqlc.types.Field(sgqlc.types.non_null(PaymentTermsInput), graphql_name="paymentTermsAttributes") - - -class PriceListAdjustmentInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("value", "type") - value = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="value") - type = sgqlc.types.Field(sgqlc.types.non_null(PriceListAdjustmentType), graphql_name="type") - - -class PriceListContext(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("country",) - country = sgqlc.types.Field(CountryCode, graphql_name="country") - - -class PriceListContextRuleInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("market_id",) - market_id = sgqlc.types.Field(ID, graphql_name="marketId") - - -class PriceListCreateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("name", "currency", "parent", "context_rule") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - currency = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currency") - parent = sgqlc.types.Field(sgqlc.types.non_null("PriceListParentCreateInput"), graphql_name="parent") - context_rule = sgqlc.types.Field(PriceListContextRuleInput, graphql_name="contextRule") - - -class PriceListParentCreateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("adjustment",) - adjustment = sgqlc.types.Field(sgqlc.types.non_null(PriceListAdjustmentInput), graphql_name="adjustment") - - -class PriceListParentUpdateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("adjustment",) - adjustment = sgqlc.types.Field(sgqlc.types.non_null(PriceListAdjustmentInput), graphql_name="adjustment") - - -class PriceListPriceInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("variant_id", "price", "compare_at_price") - variant_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="variantId") - price = sgqlc.types.Field(sgqlc.types.non_null(MoneyInput), graphql_name="price") - compare_at_price = sgqlc.types.Field(MoneyInput, graphql_name="compareAtPrice") - - -class PriceListUpdateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("name", "currency", "context_rule", "parent") - name = sgqlc.types.Field(String, graphql_name="name") - currency = sgqlc.types.Field(CurrencyCode, graphql_name="currency") - context_rule = sgqlc.types.Field(PriceListContextRuleInput, graphql_name="contextRule") - parent = sgqlc.types.Field(PriceListParentUpdateInput, graphql_name="parent") - - -class PriceRuleCustomerSelectionInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("for_all_customers", "segment_ids", "customer_ids_to_add", "customer_ids_to_remove") - for_all_customers = sgqlc.types.Field(Boolean, graphql_name="forAllCustomers") - segment_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="segmentIds") - customer_ids_to_add = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="customerIdsToAdd") - customer_ids_to_remove = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="customerIdsToRemove") - - -class PriceRuleDiscountCodeInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(String, graphql_name="code") - - -class PriceRuleEntitlementToPrerequisiteQuantityRatioInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("entitlement_quantity", "prerequisite_quantity") - entitlement_quantity = sgqlc.types.Field(Int, graphql_name="entitlementQuantity") - prerequisite_quantity = sgqlc.types.Field(Int, graphql_name="prerequisiteQuantity") - - -class PriceRuleInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "combines_with", - "validity_period", - "once_per_customer", - "customer_selection", - "usage_limit", - "title", - "allocation_limit", - "allocation_method", - "value", - "target", - "prerequisite_subtotal_range", - "prerequisite_quantity_range", - "prerequisite_shipping_price_range", - "item_entitlements", - "item_prerequisites", - "shipping_entitlements", - "prerequisite_to_entitlement_quantity_ratio", - ) - combines_with = sgqlc.types.Field(DiscountCombinesWithInput, graphql_name="combinesWith") - validity_period = sgqlc.types.Field("PriceRuleValidityPeriodInput", graphql_name="validityPeriod") - once_per_customer = sgqlc.types.Field(Boolean, graphql_name="oncePerCustomer") - customer_selection = sgqlc.types.Field(PriceRuleCustomerSelectionInput, graphql_name="customerSelection") - usage_limit = sgqlc.types.Field(Int, graphql_name="usageLimit") - title = sgqlc.types.Field(String, graphql_name="title") - allocation_limit = sgqlc.types.Field(Int, graphql_name="allocationLimit") - allocation_method = sgqlc.types.Field(PriceRuleAllocationMethod, graphql_name="allocationMethod") - value = sgqlc.types.Field("PriceRuleValueInput", graphql_name="value") - target = sgqlc.types.Field(PriceRuleTarget, graphql_name="target") - prerequisite_subtotal_range = sgqlc.types.Field("PriceRuleMoneyRangeInput", graphql_name="prerequisiteSubtotalRange") - prerequisite_quantity_range = sgqlc.types.Field("PriceRuleQuantityRangeInput", graphql_name="prerequisiteQuantityRange") - prerequisite_shipping_price_range = sgqlc.types.Field("PriceRuleMoneyRangeInput", graphql_name="prerequisiteShippingPriceRange") - item_entitlements = sgqlc.types.Field("PriceRuleItemEntitlementsInput", graphql_name="itemEntitlements") - item_prerequisites = sgqlc.types.Field("PriceRuleItemPrerequisitesInput", graphql_name="itemPrerequisites") - shipping_entitlements = sgqlc.types.Field("PriceRuleShippingEntitlementsInput", graphql_name="shippingEntitlements") - prerequisite_to_entitlement_quantity_ratio = sgqlc.types.Field( - "PriceRulePrerequisiteToEntitlementQuantityRatioInput", graphql_name="prerequisiteToEntitlementQuantityRatio" - ) - - -class PriceRuleItemEntitlementsInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("target_all_line_items", "product_ids", "product_variant_ids", "collection_ids") - target_all_line_items = sgqlc.types.Field(Boolean, graphql_name="targetAllLineItems") - product_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="productIds") - product_variant_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="productVariantIds") - collection_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="collectionIds") - - -class PriceRuleItemPrerequisitesInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("product_ids", "product_variant_ids", "collection_ids") - product_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="productIds") - product_variant_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="productVariantIds") - collection_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="collectionIds") - - -class PriceRuleMoneyRangeInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("less_than", "less_than_or_equal_to", "greater_than", "greater_than_or_equal_to") - less_than = sgqlc.types.Field(Money, graphql_name="lessThan") - less_than_or_equal_to = sgqlc.types.Field(Money, graphql_name="lessThanOrEqualTo") - greater_than = sgqlc.types.Field(Money, graphql_name="greaterThan") - greater_than_or_equal_to = sgqlc.types.Field(Money, graphql_name="greaterThanOrEqualTo") - - -class PriceRulePrerequisiteToEntitlementQuantityRatioInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("entitlement_quantity", "prerequisite_quantity") - entitlement_quantity = sgqlc.types.Field(Int, graphql_name="entitlementQuantity") - prerequisite_quantity = sgqlc.types.Field(Int, graphql_name="prerequisiteQuantity") - - -class PriceRuleQuantityRangeInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("less_than", "less_than_or_equal_to", "greater_than", "greater_than_or_equal_to") - less_than = sgqlc.types.Field(Int, graphql_name="lessThan") - less_than_or_equal_to = sgqlc.types.Field(Int, graphql_name="lessThanOrEqualTo") - greater_than = sgqlc.types.Field(Int, graphql_name="greaterThan") - greater_than_or_equal_to = sgqlc.types.Field(Int, graphql_name="greaterThanOrEqualTo") - - -class PriceRuleShippingEntitlementsInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("target_all_shipping_lines", "country_codes", "include_rest_of_world") - target_all_shipping_lines = sgqlc.types.Field(Boolean, graphql_name="targetAllShippingLines") - country_codes = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(CountryCode)), graphql_name="countryCodes") - include_rest_of_world = sgqlc.types.Field(Boolean, graphql_name="includeRestOfWorld") - - -class PriceRuleValidityPeriodInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("start", "end") - start = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="start") - end = sgqlc.types.Field(DateTime, graphql_name="end") - - -class PriceRuleValueInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("percentage_value", "fixed_amount_value") - percentage_value = sgqlc.types.Field(Float, graphql_name="percentageValue") - fixed_amount_value = sgqlc.types.Field(Money, graphql_name="fixedAmountValue") - - -class PrivateMetafieldDeleteInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("owner", "namespace", "key") - owner = sgqlc.types.Field(ID, graphql_name="owner") - namespace = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="namespace") - key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") - - -class PrivateMetafieldInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("owner", "namespace", "key", "value_input") - owner = sgqlc.types.Field(ID, graphql_name="owner") - namespace = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="namespace") - key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") - value_input = sgqlc.types.Field(sgqlc.types.non_null("PrivateMetafieldValueInput"), graphql_name="valueInput") - - -class PrivateMetafieldValueInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("value", "value_type") - value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") - value_type = sgqlc.types.Field(sgqlc.types.non_null(PrivateMetafieldValueType), graphql_name="valueType") - - -class ProductAppendImagesInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id", "images") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - images = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ImageInput))), graphql_name="images") - - -class ProductCategoryInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("product_taxonomy_node_id",) - product_taxonomy_node_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="productTaxonomyNodeId") - - -class ProductDeleteInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id",) - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - - -class ProductInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "description_html", - "handle", - "redirect_new_handle", - "seo", - "product_type", - "standardized_product_type", - "product_category", - "custom_product_type", - "tags", - "template_suffix", - "gift_card", - "gift_card_template_suffix", - "title", - "vendor", - "collections_to_join", - "collections_to_leave", - "id", - "images", - "metafields", - "private_metafields", - "options", - "variants", - "status", - "requires_selling_plan", - ) - description_html = sgqlc.types.Field(String, graphql_name="descriptionHtml") - handle = sgqlc.types.Field(String, graphql_name="handle") - redirect_new_handle = sgqlc.types.Field(Boolean, graphql_name="redirectNewHandle") - seo = sgqlc.types.Field("SEOInput", graphql_name="seo") - product_type = sgqlc.types.Field(String, graphql_name="productType") - standardized_product_type = sgqlc.types.Field("StandardizedProductTypeInput", graphql_name="standardizedProductType") - product_category = sgqlc.types.Field(ProductCategoryInput, graphql_name="productCategory") - custom_product_type = sgqlc.types.Field(String, graphql_name="customProductType") - tags = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="tags") - template_suffix = sgqlc.types.Field(String, graphql_name="templateSuffix") - gift_card = sgqlc.types.Field(Boolean, graphql_name="giftCard") - gift_card_template_suffix = sgqlc.types.Field(String, graphql_name="giftCardTemplateSuffix") - title = sgqlc.types.Field(String, graphql_name="title") - vendor = sgqlc.types.Field(String, graphql_name="vendor") - collections_to_join = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="collectionsToJoin") - collections_to_leave = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="collectionsToLeave") - id = sgqlc.types.Field(ID, graphql_name="id") - images = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ImageInput)), graphql_name="images") - metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(MetafieldInput)), graphql_name="metafields") - private_metafields = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null(PrivateMetafieldInput)), graphql_name="privateMetafields" - ) - options = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="options") - variants = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariantInput")), graphql_name="variants") - status = sgqlc.types.Field(ProductStatus, graphql_name="status") - requires_selling_plan = sgqlc.types.Field(Boolean, graphql_name="requiresSellingPlan") - - -class ProductPublicationInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("publication_id", "publish_date") - publication_id = sgqlc.types.Field(ID, graphql_name="publicationId") - publish_date = sgqlc.types.Field(DateTime, graphql_name="publishDate") - - -class ProductPublishInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id", "product_publications") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - product_publications = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProductPublicationInput))), - graphql_name="productPublications", - ) - - -class ProductResourceFeedbackInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("product_id", "state", "feedback_generated_at", "product_updated_at", "messages") - product_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="productId") - state = sgqlc.types.Field(sgqlc.types.non_null(ResourceFeedbackState), graphql_name="state") - feedback_generated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="feedbackGeneratedAt") - product_updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="productUpdatedAt") - messages = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="messages") - - -class ProductUnpublishInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id", "product_publications") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - product_publications = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProductPublicationInput))), - graphql_name="productPublications", - ) - - -class ProductVariantAppendMediaInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("variant_id", "media_ids") - variant_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="variantId") - media_ids = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="mediaIds") - - -class ProductVariantDetachMediaInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("variant_id", "media_ids") - variant_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="variantId") - media_ids = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="mediaIds") - - -class ProductVariantInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "barcode", - "compare_at_price", - "harmonized_system_code", - "id", - "image_id", - "image_src", - "media_src", - "inventory_policy", - "inventory_quantities", - "inventory_item", - "metafields", - "private_metafields", - "options", - "position", - "price", - "product_id", - "requires_shipping", - "sku", - "taxable", - "tax_code", - "weight", - "weight_unit", - ) - barcode = sgqlc.types.Field(String, graphql_name="barcode") - compare_at_price = sgqlc.types.Field(Money, graphql_name="compareAtPrice") - harmonized_system_code = sgqlc.types.Field(String, graphql_name="harmonizedSystemCode") - id = sgqlc.types.Field(ID, graphql_name="id") - image_id = sgqlc.types.Field(ID, graphql_name="imageId") - image_src = sgqlc.types.Field(String, graphql_name="imageSrc") - media_src = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="mediaSrc") - inventory_policy = sgqlc.types.Field(ProductVariantInventoryPolicy, graphql_name="inventoryPolicy") - inventory_quantities = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null(InventoryLevelInput)), graphql_name="inventoryQuantities" - ) - inventory_item = sgqlc.types.Field(InventoryItemInput, graphql_name="inventoryItem") - metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(MetafieldInput)), graphql_name="metafields") - private_metafields = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null(PrivateMetafieldInput)), graphql_name="privateMetafields" - ) - options = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="options") - position = sgqlc.types.Field(Int, graphql_name="position") - price = sgqlc.types.Field(Money, graphql_name="price") - product_id = sgqlc.types.Field(ID, graphql_name="productId") - requires_shipping = sgqlc.types.Field(Boolean, graphql_name="requiresShipping") - sku = sgqlc.types.Field(String, graphql_name="sku") - taxable = sgqlc.types.Field(Boolean, graphql_name="taxable") - tax_code = sgqlc.types.Field(String, graphql_name="taxCode") - weight = sgqlc.types.Field(Float, graphql_name="weight") - weight_unit = sgqlc.types.Field(WeightUnit, graphql_name="weightUnit") - - -class ProductVariantPositionInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id", "position") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - position = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="position") - - -class ProductVariantsBulkInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "barcode", - "compare_at_price", - "harmonized_system_code", - "id", - "image_id", - "image_src", - "media_src", - "inventory_policy", - "inventory_quantities", - "inventory_item", - "metafields", - "private_metafields", - "options", - "price", - "requires_shipping", - "sku", - "taxable", - "tax_code", - "weight", - "weight_unit", - ) - barcode = sgqlc.types.Field(String, graphql_name="barcode") - compare_at_price = sgqlc.types.Field(Money, graphql_name="compareAtPrice") - harmonized_system_code = sgqlc.types.Field(String, graphql_name="harmonizedSystemCode") - id = sgqlc.types.Field(ID, graphql_name="id") - image_id = sgqlc.types.Field(ID, graphql_name="imageId") - image_src = sgqlc.types.Field(String, graphql_name="imageSrc") - media_src = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="mediaSrc") - inventory_policy = sgqlc.types.Field(ProductVariantInventoryPolicy, graphql_name="inventoryPolicy") - inventory_quantities = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null(InventoryLevelInput)), graphql_name="inventoryQuantities" - ) - inventory_item = sgqlc.types.Field(InventoryItemInput, graphql_name="inventoryItem") - metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(MetafieldInput)), graphql_name="metafields") - private_metafields = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null(PrivateMetafieldInput)), graphql_name="privateMetafields" - ) - options = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="options") - price = sgqlc.types.Field(Money, graphql_name="price") - requires_shipping = sgqlc.types.Field(Boolean, graphql_name="requiresShipping") - sku = sgqlc.types.Field(String, graphql_name="sku") - taxable = sgqlc.types.Field(Boolean, graphql_name="taxable") - tax_code = sgqlc.types.Field(String, graphql_name="taxCode") - weight = sgqlc.types.Field(Float, graphql_name="weight") - weight_unit = sgqlc.types.Field(WeightUnit, graphql_name="weightUnit") - - -class PubSubWebhookSubscriptionInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("pub_sub_project", "pub_sub_topic", "format", "include_fields", "metafield_namespaces") - pub_sub_project = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="pubSubProject") - pub_sub_topic = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="pubSubTopic") - format = sgqlc.types.Field(WebhookSubscriptionFormat, graphql_name="format") - include_fields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="includeFields") - metafield_namespaces = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="metafieldNamespaces") - - -class PublicationInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("publication_id", "publish_date") - publication_id = sgqlc.types.Field(ID, graphql_name="publicationId") - publish_date = sgqlc.types.Field(DateTime, graphql_name="publishDate") - - -class PurchasingCompanyInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("company_id", "company_contact_id", "company_location_id") - company_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="companyId") - company_contact_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="companyContactId") - company_location_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="companyLocationId") - - -class PurchasingEntityInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("customer_id", "purchasing_company") - customer_id = sgqlc.types.Field(ID, graphql_name="customerId") - purchasing_company = sgqlc.types.Field(PurchasingCompanyInput, graphql_name="purchasingCompany") - - -class RefundDutyInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("duty_id", "refund_type") - duty_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="dutyId") - refund_type = sgqlc.types.Field(RefundDutyRefundType, graphql_name="refundType") - - -class RefundInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "currency", - "order_id", - "note", - "notify", - "shipping", - "refund_line_items", - "refund_duties", - "transactions", - ) - currency = sgqlc.types.Field(CurrencyCode, graphql_name="currency") - order_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="orderId") - note = sgqlc.types.Field(String, graphql_name="note") - notify = sgqlc.types.Field(Boolean, graphql_name="notify") - shipping = sgqlc.types.Field("ShippingRefundInput", graphql_name="shipping") - refund_line_items = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("RefundLineItemInput")), graphql_name="refundLineItems") - refund_duties = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(RefundDutyInput)), graphql_name="refundDuties") - transactions = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(OrderTransactionInput)), graphql_name="transactions") - - -class RefundLineItemInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("line_item_id", "quantity", "restock_type", "location_id") - line_item_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="lineItemId") - quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") - restock_type = sgqlc.types.Field(RefundLineItemRestockType, graphql_name="restockType") - location_id = sgqlc.types.Field(ID, graphql_name="locationId") - - -class RemoteAuthorizeNetCustomerPaymentProfileInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("customer_profile_id", "customer_payment_profile_id") - customer_profile_id = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="customerProfileId") - customer_payment_profile_id = sgqlc.types.Field(String, graphql_name="customerPaymentProfileId") - - -class RemoteBraintreePaymentMethodInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("customer_id", "payment_method_token") - customer_id = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="customerId") - payment_method_token = sgqlc.types.Field(String, graphql_name="paymentMethodToken") - - -class RemoteStripePaymentMethodInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("customer_id", "payment_method_id") - customer_id = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="customerId") - payment_method_id = sgqlc.types.Field(String, graphql_name="paymentMethodId") - - -class SEOInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("title", "description") - title = sgqlc.types.Field(String, graphql_name="title") - description = sgqlc.types.Field(String, graphql_name="description") - - -class SavedSearchCreateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("resource_type", "name", "query") - resource_type = sgqlc.types.Field(sgqlc.types.non_null(SearchResultType), graphql_name="resourceType") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - query = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="query") - - -class SavedSearchDeleteInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id",) - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - - -class SavedSearchUpdateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id", "name", "query") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - name = sgqlc.types.Field(String, graphql_name="name") - query = sgqlc.types.Field(String, graphql_name="query") - - -class ScriptTagInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("src", "display_scope", "cache") - src = sgqlc.types.Field(URL, graphql_name="src") - display_scope = sgqlc.types.Field(ScriptTagDisplayScope, graphql_name="displayScope") - cache = sgqlc.types.Field(Boolean, graphql_name="cache") - - -class SellingPlanAnchorInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("type", "day", "month", "cutoff_day") - type = sgqlc.types.Field(SellingPlanAnchorType, graphql_name="type") - day = sgqlc.types.Field(Int, graphql_name="day") - month = sgqlc.types.Field(Int, graphql_name="month") - cutoff_day = sgqlc.types.Field(Int, graphql_name="cutoffDay") - - -class SellingPlanBillingPolicyInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("fixed", "recurring") - fixed = sgqlc.types.Field("SellingPlanFixedBillingPolicyInput", graphql_name="fixed") - recurring = sgqlc.types.Field("SellingPlanRecurringBillingPolicyInput", graphql_name="recurring") - - -class SellingPlanCheckoutChargeInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("type", "value") - type = sgqlc.types.Field(SellingPlanCheckoutChargeType, graphql_name="type") - value = sgqlc.types.Field("SellingPlanCheckoutChargeValueInput", graphql_name="value") - - -class SellingPlanCheckoutChargeValueInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("percentage", "fixed_value") - percentage = sgqlc.types.Field(Float, graphql_name="percentage") - fixed_value = sgqlc.types.Field(Decimal, graphql_name="fixedValue") - - -class SellingPlanDeliveryPolicyInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("fixed", "recurring") - fixed = sgqlc.types.Field("SellingPlanFixedDeliveryPolicyInput", graphql_name="fixed") - recurring = sgqlc.types.Field("SellingPlanRecurringDeliveryPolicyInput", graphql_name="recurring") - - -class SellingPlanFixedBillingPolicyInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "remaining_balance_charge_trigger", - "remaining_balance_charge_exact_time", - "remaining_balance_charge_time_after_checkout", - "checkout_charge", - ) - remaining_balance_charge_trigger = sgqlc.types.Field( - SellingPlanRemainingBalanceChargeTrigger, graphql_name="remainingBalanceChargeTrigger" - ) - remaining_balance_charge_exact_time = sgqlc.types.Field(DateTime, graphql_name="remainingBalanceChargeExactTime") - remaining_balance_charge_time_after_checkout = sgqlc.types.Field(String, graphql_name="remainingBalanceChargeTimeAfterCheckout") - checkout_charge = sgqlc.types.Field(SellingPlanCheckoutChargeInput, graphql_name="checkoutCharge") - - -class SellingPlanFixedDeliveryPolicyInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "anchors", - "fulfillment_trigger", - "fulfillment_exact_time", - "cutoff", - "intent", - "pre_anchor_behavior", - ) - anchors = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(SellingPlanAnchorInput)), graphql_name="anchors") - fulfillment_trigger = sgqlc.types.Field(SellingPlanFulfillmentTrigger, graphql_name="fulfillmentTrigger") - fulfillment_exact_time = sgqlc.types.Field(DateTime, graphql_name="fulfillmentExactTime") - cutoff = sgqlc.types.Field(Int, graphql_name="cutoff") - intent = sgqlc.types.Field(SellingPlanFixedDeliveryPolicyIntent, graphql_name="intent") - pre_anchor_behavior = sgqlc.types.Field(SellingPlanFixedDeliveryPolicyPreAnchorBehavior, graphql_name="preAnchorBehavior") - - -class SellingPlanFixedPricingPolicyInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id", "adjustment_type", "adjustment_value") - id = sgqlc.types.Field(ID, graphql_name="id") - adjustment_type = sgqlc.types.Field(SellingPlanPricingPolicyAdjustmentType, graphql_name="adjustmentType") - adjustment_value = sgqlc.types.Field("SellingPlanPricingPolicyValueInput", graphql_name="adjustmentValue") - - -class SellingPlanGroupInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "name", - "app_id", - "merchant_code", - "description", - "selling_plans_to_create", - "selling_plans_to_update", - "selling_plans_to_delete", - "options", - "position", - ) - name = sgqlc.types.Field(String, graphql_name="name") - app_id = sgqlc.types.Field(String, graphql_name="appId") - merchant_code = sgqlc.types.Field(String, graphql_name="merchantCode") - description = sgqlc.types.Field(String, graphql_name="description") - selling_plans_to_create = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanInput")), graphql_name="sellingPlansToCreate" - ) - selling_plans_to_update = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanInput")), graphql_name="sellingPlansToUpdate" - ) - selling_plans_to_delete = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="sellingPlansToDelete") - options = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="options") - position = sgqlc.types.Field(Int, graphql_name="position") - - -class SellingPlanGroupResourceInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("product_variant_ids", "product_ids") - product_variant_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="productVariantIds") - product_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="productIds") - - -class SellingPlanInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "id", - "name", - "description", - "billing_policy", - "delivery_policy", - "inventory_policy", - "pricing_policies", - "options", - "position", - "category", - ) - id = sgqlc.types.Field(ID, graphql_name="id") - name = sgqlc.types.Field(String, graphql_name="name") - description = sgqlc.types.Field(String, graphql_name="description") - billing_policy = sgqlc.types.Field(SellingPlanBillingPolicyInput, graphql_name="billingPolicy") - delivery_policy = sgqlc.types.Field(SellingPlanDeliveryPolicyInput, graphql_name="deliveryPolicy") - inventory_policy = sgqlc.types.Field("SellingPlanInventoryPolicyInput", graphql_name="inventoryPolicy") - pricing_policies = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanPricingPolicyInput")), graphql_name="pricingPolicies" - ) - options = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="options") - position = sgqlc.types.Field(Int, graphql_name="position") - category = sgqlc.types.Field(SellingPlanCategory, graphql_name="category") - - -class SellingPlanInventoryPolicyInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("reserve",) - reserve = sgqlc.types.Field(SellingPlanReserve, graphql_name="reserve") - - -class SellingPlanPricingPolicyInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("recurring", "fixed") - recurring = sgqlc.types.Field("SellingPlanRecurringPricingPolicyInput", graphql_name="recurring") - fixed = sgqlc.types.Field(SellingPlanFixedPricingPolicyInput, graphql_name="fixed") - - -class SellingPlanPricingPolicyValueInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("percentage", "fixed_value") - percentage = sgqlc.types.Field(Float, graphql_name="percentage") - fixed_value = sgqlc.types.Field(Decimal, graphql_name="fixedValue") - - -class SellingPlanRecurringBillingPolicyInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("interval", "interval_count", "anchors", "min_cycles", "max_cycles") - interval = sgqlc.types.Field(SellingPlanInterval, graphql_name="interval") - interval_count = sgqlc.types.Field(Int, graphql_name="intervalCount") - anchors = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(SellingPlanAnchorInput)), graphql_name="anchors") - min_cycles = sgqlc.types.Field(Int, graphql_name="minCycles") - max_cycles = sgqlc.types.Field(Int, graphql_name="maxCycles") - - -class SellingPlanRecurringDeliveryPolicyInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("interval", "interval_count", "anchors", "cutoff", "intent", "pre_anchor_behavior") - interval = sgqlc.types.Field(SellingPlanInterval, graphql_name="interval") - interval_count = sgqlc.types.Field(Int, graphql_name="intervalCount") - anchors = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(SellingPlanAnchorInput)), graphql_name="anchors") - cutoff = sgqlc.types.Field(Int, graphql_name="cutoff") - intent = sgqlc.types.Field(SellingPlanRecurringDeliveryPolicyIntent, graphql_name="intent") - pre_anchor_behavior = sgqlc.types.Field(SellingPlanRecurringDeliveryPolicyPreAnchorBehavior, graphql_name="preAnchorBehavior") - - -class SellingPlanRecurringPricingPolicyInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id", "adjustment_type", "adjustment_value", "after_cycle") - id = sgqlc.types.Field(ID, graphql_name="id") - adjustment_type = sgqlc.types.Field(SellingPlanPricingPolicyAdjustmentType, graphql_name="adjustmentType") - adjustment_value = sgqlc.types.Field(SellingPlanPricingPolicyValueInput, graphql_name="adjustmentValue") - after_cycle = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="afterCycle") - - -class ShippingLineInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("price", "shipping_rate_handle", "title") - price = sgqlc.types.Field(Money, graphql_name="price") - shipping_rate_handle = sgqlc.types.Field(String, graphql_name="shippingRateHandle") - title = sgqlc.types.Field(String, graphql_name="title") - - -class ShippingRefundInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("amount", "full_refund") - amount = sgqlc.types.Field(Money, graphql_name="amount") - full_refund = sgqlc.types.Field(Boolean, graphql_name="fullRefund") - - -class ShopLocaleInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("published", "market_web_presence_ids") - published = sgqlc.types.Field(Boolean, graphql_name="published") - market_web_presence_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="marketWebPresenceIds") - - -class ShopPolicyInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("type", "body") - type = sgqlc.types.Field(sgqlc.types.non_null(ShopPolicyType), graphql_name="type") - body = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="body") - - -class ShopifyPaymentsDisputeEvidenceUpdateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "customer_email_address", - "customer_last_name", - "customer_first_name", - "shipping_address", - "uncategorized_text", - "access_activity_log", - "cancellation_policy_disclosure", - "cancellation_rebuttal", - "refund_policy_disclosure", - "refund_refusal_explanation", - "cancellation_policy_file", - "customer_communication_file", - "refund_policy_file", - "shipping_documentation_file", - "uncategorized_file", - "service_documentation_file", - "submit_evidence", - ) - customer_email_address = sgqlc.types.Field(String, graphql_name="customerEmailAddress") - customer_last_name = sgqlc.types.Field(String, graphql_name="customerLastName") - customer_first_name = sgqlc.types.Field(String, graphql_name="customerFirstName") - shipping_address = sgqlc.types.Field(MailingAddressInput, graphql_name="shippingAddress") - uncategorized_text = sgqlc.types.Field(String, graphql_name="uncategorizedText") - access_activity_log = sgqlc.types.Field(String, graphql_name="accessActivityLog") - cancellation_policy_disclosure = sgqlc.types.Field(String, graphql_name="cancellationPolicyDisclosure") - cancellation_rebuttal = sgqlc.types.Field(String, graphql_name="cancellationRebuttal") - refund_policy_disclosure = sgqlc.types.Field(String, graphql_name="refundPolicyDisclosure") - refund_refusal_explanation = sgqlc.types.Field(String, graphql_name="refundRefusalExplanation") - cancellation_policy_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUploadUpdateInput", graphql_name="cancellationPolicyFile") - customer_communication_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUploadUpdateInput", graphql_name="customerCommunicationFile") - refund_policy_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUploadUpdateInput", graphql_name="refundPolicyFile") - shipping_documentation_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUploadUpdateInput", graphql_name="shippingDocumentationFile") - uncategorized_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUploadUpdateInput", graphql_name="uncategorizedFile") - service_documentation_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUploadUpdateInput", graphql_name="serviceDocumentationFile") - submit_evidence = sgqlc.types.Field(Boolean, graphql_name="submitEvidence") - - -class ShopifyPaymentsDisputeFileUploadUpdateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id", "destroy") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - destroy = sgqlc.types.Field(Boolean, graphql_name="destroy") - - -class StageImageInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("resource", "filename", "mime_type", "http_method") - resource = sgqlc.types.Field(sgqlc.types.non_null(StagedUploadTargetGenerateUploadResource), graphql_name="resource") - filename = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="filename") - mime_type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="mimeType") - http_method = sgqlc.types.Field(StagedUploadHttpMethodType, graphql_name="httpMethod") - - -class StagedUploadInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("resource", "filename", "mime_type", "http_method", "file_size") - resource = sgqlc.types.Field(sgqlc.types.non_null(StagedUploadTargetGenerateUploadResource), graphql_name="resource") - filename = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="filename") - mime_type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="mimeType") - http_method = sgqlc.types.Field(StagedUploadHttpMethodType, graphql_name="httpMethod") - file_size = sgqlc.types.Field(UnsignedInt64, graphql_name="fileSize") - - -class StagedUploadTargetGenerateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("resource", "filename", "mime_type", "http_method", "file_size") - resource = sgqlc.types.Field(sgqlc.types.non_null(StagedUploadTargetGenerateUploadResource), graphql_name="resource") - filename = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="filename") - mime_type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="mimeType") - http_method = sgqlc.types.Field(StagedUploadHttpMethodType, graphql_name="httpMethod") - file_size = sgqlc.types.Field(UnsignedInt64, graphql_name="fileSize") - - -class StandardizedProductTypeInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("product_taxonomy_node_id",) - product_taxonomy_node_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="productTaxonomyNodeId") - - -class StorefrontAccessTokenDeleteInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id",) - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - - -class StorefrontAccessTokenInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("title",) - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - - -class SubscriptionBillingAttemptInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("idempotency_key", "origin_time", "billing_cycle_selector") - idempotency_key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="idempotencyKey") - origin_time = sgqlc.types.Field(DateTime, graphql_name="originTime") - billing_cycle_selector = sgqlc.types.Field("SubscriptionBillingCycleSelector", graphql_name="billingCycleSelector") - - -class SubscriptionBillingCycleInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("contract_id", "selector") - contract_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="contractId") - selector = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionBillingCycleSelector"), graphql_name="selector") - - -class SubscriptionBillingCycleScheduleEditInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("skip", "billing_date", "reason") - skip = sgqlc.types.Field(Boolean, graphql_name="skip") - billing_date = sgqlc.types.Field(DateTime, graphql_name="billingDate") - reason = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionBillingCycleScheduleEditInputScheduleEditReason), graphql_name="reason") - - -class SubscriptionBillingCycleSelector(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("index", "date") - index = sgqlc.types.Field(Int, graphql_name="index") - date = sgqlc.types.Field(DateTime, graphql_name="date") - - -class SubscriptionBillingCyclesDateRangeSelector(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("start_date", "end_date") - start_date = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="startDate") - end_date = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="endDate") - - -class SubscriptionBillingCyclesIndexRangeSelector(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("start_index", "end_index") - start_index = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="startIndex") - end_index = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="endIndex") - - -class SubscriptionBillingPolicyInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("interval", "interval_count", "min_cycles", "max_cycles", "anchors") - interval = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanInterval), graphql_name="interval") - interval_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="intervalCount") - min_cycles = sgqlc.types.Field(Int, graphql_name="minCycles") - max_cycles = sgqlc.types.Field(Int, graphql_name="maxCycles") - anchors = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(SellingPlanAnchorInput)), graphql_name="anchors") - - -class SubscriptionContractCreateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("customer_id", "next_billing_date", "currency_code", "contract") - customer_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="customerId") - next_billing_date = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="nextBillingDate") - currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currencyCode") - contract = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionDraftInput"), graphql_name="contract") - - -class SubscriptionDeliveryMethodInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("shipping", "local_delivery", "pickup") - shipping = sgqlc.types.Field("SubscriptionDeliveryMethodShippingInput", graphql_name="shipping") - local_delivery = sgqlc.types.Field("SubscriptionDeliveryMethodLocalDeliveryInput", graphql_name="localDelivery") - pickup = sgqlc.types.Field("SubscriptionDeliveryMethodPickupInput", graphql_name="pickup") - - -class SubscriptionDeliveryMethodLocalDeliveryInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("address", "local_delivery_option") - address = sgqlc.types.Field(MailingAddressInput, graphql_name="address") - local_delivery_option = sgqlc.types.Field("SubscriptionDeliveryMethodLocalDeliveryOptionInput", graphql_name="localDeliveryOption") - - -class SubscriptionDeliveryMethodLocalDeliveryOptionInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("title", "presentment_title", "description", "code", "phone", "instructions") - title = sgqlc.types.Field(String, graphql_name="title") - presentment_title = sgqlc.types.Field(String, graphql_name="presentmentTitle") - description = sgqlc.types.Field(String, graphql_name="description") - code = sgqlc.types.Field(String, graphql_name="code") - phone = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="phone") - instructions = sgqlc.types.Field(String, graphql_name="instructions") - - -class SubscriptionDeliveryMethodPickupInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("pickup_option",) - pickup_option = sgqlc.types.Field("SubscriptionDeliveryMethodPickupOptionInput", graphql_name="pickupOption") - - -class SubscriptionDeliveryMethodPickupOptionInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("title", "presentment_title", "description", "code", "location_id") - title = sgqlc.types.Field(String, graphql_name="title") - presentment_title = sgqlc.types.Field(String, graphql_name="presentmentTitle") - description = sgqlc.types.Field(String, graphql_name="description") - code = sgqlc.types.Field(String, graphql_name="code") - location_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="locationId") - - -class SubscriptionDeliveryMethodShippingInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("address", "shipping_option") - address = sgqlc.types.Field(MailingAddressInput, graphql_name="address") - shipping_option = sgqlc.types.Field("SubscriptionDeliveryMethodShippingOptionInput", graphql_name="shippingOption") - - -class SubscriptionDeliveryMethodShippingOptionInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("title", "presentment_title", "description", "code", "carrier_service_id") - title = sgqlc.types.Field(String, graphql_name="title") - presentment_title = sgqlc.types.Field(String, graphql_name="presentmentTitle") - description = sgqlc.types.Field(String, graphql_name="description") - code = sgqlc.types.Field(String, graphql_name="code") - carrier_service_id = sgqlc.types.Field(ID, graphql_name="carrierServiceId") - - -class SubscriptionDeliveryPolicyInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("interval", "interval_count", "anchors") - interval = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanInterval), graphql_name="interval") - interval_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="intervalCount") - anchors = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(SellingPlanAnchorInput)), graphql_name="anchors") - - -class SubscriptionDraftInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "status", - "payment_method_id", - "next_billing_date", - "billing_policy", - "delivery_policy", - "delivery_price", - "delivery_method", - "note", - "custom_attributes", - ) - status = sgqlc.types.Field(SubscriptionContractSubscriptionStatus, graphql_name="status") - payment_method_id = sgqlc.types.Field(ID, graphql_name="paymentMethodId") - next_billing_date = sgqlc.types.Field(DateTime, graphql_name="nextBillingDate") - billing_policy = sgqlc.types.Field(SubscriptionBillingPolicyInput, graphql_name="billingPolicy") - delivery_policy = sgqlc.types.Field(SubscriptionDeliveryPolicyInput, graphql_name="deliveryPolicy") - delivery_price = sgqlc.types.Field(Decimal, graphql_name="deliveryPrice") - delivery_method = sgqlc.types.Field(SubscriptionDeliveryMethodInput, graphql_name="deliveryMethod") - note = sgqlc.types.Field(String, graphql_name="note") - custom_attributes = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(AttributeInput)), graphql_name="customAttributes") - - -class SubscriptionFreeShippingDiscountInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("title", "recurring_cycle_limit") - title = sgqlc.types.Field(String, graphql_name="title") - recurring_cycle_limit = sgqlc.types.Field(Int, graphql_name="recurringCycleLimit") - - -class SubscriptionLineInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "product_variant_id", - "quantity", - "current_price", - "custom_attributes", - "selling_plan_id", - "selling_plan_name", - "pricing_policy", - ) - product_variant_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="productVariantId") - quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") - current_price = sgqlc.types.Field(sgqlc.types.non_null(Decimal), graphql_name="currentPrice") - custom_attributes = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(AttributeInput)), graphql_name="customAttributes") - selling_plan_id = sgqlc.types.Field(ID, graphql_name="sellingPlanId") - selling_plan_name = sgqlc.types.Field(String, graphql_name="sellingPlanName") - pricing_policy = sgqlc.types.Field("SubscriptionPricingPolicyInput", graphql_name="pricingPolicy") - - -class SubscriptionLineUpdateInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "product_variant_id", - "quantity", - "selling_plan_id", - "selling_plan_name", - "current_price", - "custom_attributes", - "pricing_policy", - ) - product_variant_id = sgqlc.types.Field(ID, graphql_name="productVariantId") - quantity = sgqlc.types.Field(Int, graphql_name="quantity") - selling_plan_id = sgqlc.types.Field(ID, graphql_name="sellingPlanId") - selling_plan_name = sgqlc.types.Field(String, graphql_name="sellingPlanName") - current_price = sgqlc.types.Field(Decimal, graphql_name="currentPrice") - custom_attributes = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(AttributeInput)), graphql_name="customAttributes") - pricing_policy = sgqlc.types.Field("SubscriptionPricingPolicyInput", graphql_name="pricingPolicy") - - -class SubscriptionManualDiscountEntitledLinesInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("all", "lines") - all = sgqlc.types.Field(Boolean, graphql_name="all") - lines = sgqlc.types.Field("SubscriptionManualDiscountLinesInput", graphql_name="lines") - - -class SubscriptionManualDiscountFixedAmountInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("amount", "applies_on_each_item") - amount = sgqlc.types.Field(Float, graphql_name="amount") - applies_on_each_item = sgqlc.types.Field(Boolean, graphql_name="appliesOnEachItem") - - -class SubscriptionManualDiscountInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("title", "value", "recurring_cycle_limit", "entitled_lines") - title = sgqlc.types.Field(String, graphql_name="title") - value = sgqlc.types.Field("SubscriptionManualDiscountValueInput", graphql_name="value") - recurring_cycle_limit = sgqlc.types.Field(Int, graphql_name="recurringCycleLimit") - entitled_lines = sgqlc.types.Field(SubscriptionManualDiscountEntitledLinesInput, graphql_name="entitledLines") - - -class SubscriptionManualDiscountLinesInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("add", "remove") - add = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="add") - remove = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="remove") - - -class SubscriptionManualDiscountValueInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("percentage", "fixed_amount") - percentage = sgqlc.types.Field(Int, graphql_name="percentage") - fixed_amount = sgqlc.types.Field(SubscriptionManualDiscountFixedAmountInput, graphql_name="fixedAmount") - - -class SubscriptionPricingPolicyCycleDiscountsInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("after_cycle", "adjustment_type", "adjustment_value", "computed_price") - after_cycle = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="afterCycle") - adjustment_type = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanPricingPolicyAdjustmentType), graphql_name="adjustmentType") - adjustment_value = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanPricingPolicyValueInput), graphql_name="adjustmentValue") - computed_price = sgqlc.types.Field(sgqlc.types.non_null(Decimal), graphql_name="computedPrice") - - -class SubscriptionPricingPolicyInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("base_price", "cycle_discounts") - base_price = sgqlc.types.Field(sgqlc.types.non_null(Decimal), graphql_name="basePrice") - cycle_discounts = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SubscriptionPricingPolicyCycleDiscountsInput))), - graphql_name="cycleDiscounts", - ) - - -class TranslationInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("locale", "key", "value", "translatable_content_digest", "market_id") - locale = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="locale") - key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") - value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") - translatable_content_digest = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="translatableContentDigest") - market_id = sgqlc.types.Field(ID, graphql_name="marketId") - - -class UTMInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("campaign", "source", "medium") - campaign = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="campaign") - source = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="source") - medium = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="medium") - - -class UpdateMediaInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("id", "preview_image_source", "alt") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - preview_image_source = sgqlc.types.Field(String, graphql_name="previewImageSource") - alt = sgqlc.types.Field(String, graphql_name="alt") - - -class UrlRedirectInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("path", "target") - path = sgqlc.types.Field(String, graphql_name="path") - target = sgqlc.types.Field(String, graphql_name="target") - - -class WebPixelInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("settings",) - settings = sgqlc.types.Field(sgqlc.types.non_null(JSON), graphql_name="settings") - - -class WebhookSubscriptionInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ( - "callback_url", - "format", - "include_fields", - "metafield_namespaces", - "private_metafield_namespaces", - ) - callback_url = sgqlc.types.Field(URL, graphql_name="callbackUrl") - format = sgqlc.types.Field(WebhookSubscriptionFormat, graphql_name="format") - include_fields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="includeFields") - metafield_namespaces = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="metafieldNamespaces") - private_metafield_namespaces = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="privateMetafieldNamespaces" - ) - - -class WeightInput(sgqlc.types.Input): - __schema__ = shopify_schema - __field_names__ = ("value", "unit") - value = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="value") - unit = sgqlc.types.Field(sgqlc.types.non_null(WeightUnit), graphql_name="unit") - - -######################################################################## -# Output Objects and Interfaces -######################################################################## -class AccessScope(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("description", "handle") - description = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="description") - handle = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="handle") - - -class AllDiscountItems(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("all_items",) - all_items = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="allItems") - - -class ApiVersion(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("display_name", "handle", "supported") - display_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="displayName") - handle = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="handle") - supported = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="supported") - - -class AppConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("App"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class AppCreditConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppCreditEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppCredit"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class AppCreditCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("app_credit", "user_errors") - app_credit = sgqlc.types.Field("AppCredit", graphql_name="appCredit") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class AppCreditEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("AppCredit"), graphql_name="node") - - -class AppDiscountType(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "app", - "app_bridge", - "app_key", - "description", - "discount_class", - "function_id", - "target_type", - "title", - ) - app = sgqlc.types.Field(sgqlc.types.non_null("App"), graphql_name="app") - app_bridge = sgqlc.types.Field(sgqlc.types.non_null("FunctionsAppBridge"), graphql_name="appBridge") - app_key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="appKey") - description = sgqlc.types.Field(String, graphql_name="description") - discount_class = sgqlc.types.Field(sgqlc.types.non_null(DiscountClass), graphql_name="discountClass") - function_id = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="functionId") - target_type = sgqlc.types.Field(sgqlc.types.non_null(DiscountApplicationTargetType), graphql_name="targetType") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - - -class AppEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("App"), graphql_name="node") - - -class AppFeedback(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("app", "link", "messages") - app = sgqlc.types.Field(sgqlc.types.non_null("App"), graphql_name="app") - link = sgqlc.types.Field("Link", graphql_name="link") - messages = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="messages") - - -class AppInstallationConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppInstallationEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppInstallation"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class AppInstallationEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("AppInstallation"), graphql_name="node") - - -class AppPlanV2(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("pricing_details",) - pricing_details = sgqlc.types.Field(sgqlc.types.non_null("AppPricingDetails"), graphql_name="pricingDetails") - - -class AppPurchase(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ("created_at", "name", "price", "status", "test") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - price = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="price") - status = sgqlc.types.Field(sgqlc.types.non_null(AppPurchaseStatus), graphql_name="status") - test = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="test") - - -class AppPurchaseOneTimeConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppPurchaseOneTimeEdge"))), graphql_name="edges" - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppPurchaseOneTime"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class AppPurchaseOneTimeCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("app_purchase_one_time", "confirmation_url", "user_errors") - app_purchase_one_time = sgqlc.types.Field("AppPurchaseOneTime", graphql_name="appPurchaseOneTime") - confirmation_url = sgqlc.types.Field(URL, graphql_name="confirmationUrl") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class AppPurchaseOneTimeEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("AppPurchaseOneTime"), graphql_name="node") - - -class AppRecurringPricing(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("discount", "interval", "price") - discount = sgqlc.types.Field("AppSubscriptionDiscount", graphql_name="discount") - interval = sgqlc.types.Field(sgqlc.types.non_null(AppPricingInterval), graphql_name="interval") - price = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="price") - - -class AppRevenueAttributionRecordConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppRevenueAttributionRecordEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppRevenueAttributionRecord"))), - graphql_name="nodes", - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class AppRevenueAttributionRecordCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("app_revenue_attribution_record", "user_errors") - app_revenue_attribution_record = sgqlc.types.Field("AppRevenueAttributionRecord", graphql_name="appRevenueAttributionRecord") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppRevenueAttributionRecordCreateUserError"))), - graphql_name="userErrors", - ) - - -class AppRevenueAttributionRecordDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_id", "user_errors") - deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppRevenueAttributionRecordDeleteUserError"))), - graphql_name="userErrors", - ) - - -class AppRevenueAttributionRecordEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("AppRevenueAttributionRecord"), graphql_name="node") - - -class AppSubscriptionCancelPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("app_subscription", "user_errors") - app_subscription = sgqlc.types.Field("AppSubscription", graphql_name="appSubscription") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class AppSubscriptionConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppSubscriptionEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppSubscription"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class AppSubscriptionCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("app_subscription", "confirmation_url", "user_errors") - app_subscription = sgqlc.types.Field("AppSubscription", graphql_name="appSubscription") - confirmation_url = sgqlc.types.Field(URL, graphql_name="confirmationUrl") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class AppSubscriptionDiscount(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "duration_limit_in_intervals", - "price_after_discount", - "remaining_duration_in_intervals", - "value", - ) - duration_limit_in_intervals = sgqlc.types.Field(Int, graphql_name="durationLimitInIntervals") - price_after_discount = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="priceAfterDiscount") - remaining_duration_in_intervals = sgqlc.types.Field(Int, graphql_name="remainingDurationInIntervals") - value = sgqlc.types.Field(sgqlc.types.non_null("AppSubscriptionDiscountValue"), graphql_name="value") - - -class AppSubscriptionDiscountAmount(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("amount",) - amount = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="amount") - - -class AppSubscriptionDiscountPercentage(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("percentage",) - percentage = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="percentage") - - -class AppSubscriptionEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("AppSubscription"), graphql_name="node") - - -class AppSubscriptionLineItem(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("id", "plan", "usage_records") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - plan = sgqlc.types.Field(sgqlc.types.non_null(AppPlanV2), graphql_name="plan") - usage_records = sgqlc.types.Field( - sgqlc.types.non_null("AppUsageRecordConnection"), - graphql_name="usageRecords", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(AppUsageRecordSortKeys, graphql_name="sortKey", default="CREATED_AT")), - ) - ), - ) - - -class AppSubscriptionLineItemUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("app_subscription", "confirmation_url", "user_errors") - app_subscription = sgqlc.types.Field("AppSubscription", graphql_name="appSubscription") - confirmation_url = sgqlc.types.Field(URL, graphql_name="confirmationUrl") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class AppSubscriptionTrialExtendPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("app_subscription", "user_errors") - app_subscription = sgqlc.types.Field("AppSubscription", graphql_name="appSubscription") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppSubscriptionTrialExtendUserError"))), - graphql_name="userErrors", - ) - - -class AppUsagePricing(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("balance_used", "capped_amount", "interval", "terms") - balance_used = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="balanceUsed") - capped_amount = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="cappedAmount") - interval = sgqlc.types.Field(sgqlc.types.non_null(AppPricingInterval), graphql_name="interval") - terms = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="terms") - - -class AppUsageRecordConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppUsageRecordEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppUsageRecord"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class AppUsageRecordCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("app_usage_record", "user_errors") - app_usage_record = sgqlc.types.Field("AppUsageRecord", graphql_name="appUsageRecord") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class AppUsageRecordEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("AppUsageRecord"), graphql_name="node") - - -class Attribute(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("key", "value") - key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") - value = sgqlc.types.Field(String, graphql_name="value") - - -class AvailableChannelDefinitionsByChannel(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("channel_definitions", "channel_name") - channel_definitions = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ChannelDefinition"))), - graphql_name="channelDefinitions", - ) - channel_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="channelName") - - -class BulkOperationCancelPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("bulk_operation", "user_errors") - bulk_operation = sgqlc.types.Field("BulkOperation", graphql_name="bulkOperation") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class BulkOperationRunMutationPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("bulk_operation", "user_errors") - bulk_operation = sgqlc.types.Field("BulkOperation", graphql_name="bulkOperation") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BulkMutationUserError"))), - graphql_name="userErrors", - ) - - -class BulkOperationRunQueryPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("bulk_operation", "user_errors") - bulk_operation = sgqlc.types.Field("BulkOperation", graphql_name="bulkOperation") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class BulkProductResourceFeedbackCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("feedback", "user_errors") - feedback = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("ProductResourceFeedback")), graphql_name="feedback") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BulkProductResourceFeedbackCreateUserError"))), - graphql_name="userErrors", - ) - - -class BuyerExperienceConfiguration(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("checkout_to_draft", "pay_now_only", "payment_terms_template") - checkout_to_draft = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="checkoutToDraft") - pay_now_only = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="payNowOnly") - payment_terms_template = sgqlc.types.Field("PaymentTermsTemplate", graphql_name="paymentTermsTemplate") - - -class CalculatedDiscountAllocation(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("allocated_amount_set", "discount_application") - allocated_amount_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="allocatedAmountSet") - discount_application = sgqlc.types.Field(sgqlc.types.non_null("CalculatedDiscountApplication"), graphql_name="discountApplication") - - -class CalculatedDiscountApplication(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ( - "allocation_method", - "applied_to", - "description", - "id", - "target_selection", - "target_type", - "value", - ) - allocation_method = sgqlc.types.Field(sgqlc.types.non_null(DiscountApplicationAllocationMethod), graphql_name="allocationMethod") - applied_to = sgqlc.types.Field(sgqlc.types.non_null(DiscountApplicationLevel), graphql_name="appliedTo") - description = sgqlc.types.Field(String, graphql_name="description") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - target_selection = sgqlc.types.Field(sgqlc.types.non_null(DiscountApplicationTargetSelection), graphql_name="targetSelection") - target_type = sgqlc.types.Field(sgqlc.types.non_null(DiscountApplicationTargetType), graphql_name="targetType") - value = sgqlc.types.Field(sgqlc.types.non_null("PricingValue"), graphql_name="value") - - -class CalculatedDiscountApplicationConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CalculatedDiscountApplicationEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CalculatedDiscountApplication))), - graphql_name="nodes", - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class CalculatedDiscountApplicationEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(CalculatedDiscountApplication), graphql_name="node") - - -class CalculatedDraftOrder(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "applied_discount", - "available_shipping_rates", - "billing_address_matches_shipping_address", - "currency_code", - "customer", - "line_items", - "line_items_subtotal_price", - "market_name", - "market_region_country_code", - "phone", - "presentment_currency_code", - "purchasing_entity", - "shipping_line", - "subtotal_price", - "subtotal_price_set", - "tax_lines", - "total_discounts_set", - "total_line_items_price_set", - "total_price", - "total_price_set", - "total_shipping_price", - "total_shipping_price_set", - "total_tax", - "total_tax_set", - ) - applied_discount = sgqlc.types.Field("DraftOrderAppliedDiscount", graphql_name="appliedDiscount") - available_shipping_rates = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShippingRate"))), - graphql_name="availableShippingRates", - ) - billing_address_matches_shipping_address = sgqlc.types.Field( - sgqlc.types.non_null(Boolean), graphql_name="billingAddressMatchesShippingAddress" - ) - currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currencyCode") - customer = sgqlc.types.Field("Customer", graphql_name="customer") - line_items = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CalculatedDraftOrderLineItem"))), - graphql_name="lineItems", - ) - line_items_subtotal_price = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="lineItemsSubtotalPrice") - market_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="marketName") - market_region_country_code = sgqlc.types.Field(sgqlc.types.non_null(CountryCode), graphql_name="marketRegionCountryCode") - phone = sgqlc.types.Field(String, graphql_name="phone") - presentment_currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="presentmentCurrencyCode") - purchasing_entity = sgqlc.types.Field("PurchasingEntity", graphql_name="purchasingEntity") - shipping_line = sgqlc.types.Field("ShippingLine", graphql_name="shippingLine") - subtotal_price = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="subtotalPrice") - subtotal_price_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="subtotalPriceSet") - tax_lines = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TaxLine"))), graphql_name="taxLines") - total_discounts_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="totalDiscountsSet") - total_line_items_price_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="totalLineItemsPriceSet") - total_price = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="totalPrice") - total_price_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="totalPriceSet") - total_shipping_price = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="totalShippingPrice") - total_shipping_price_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="totalShippingPriceSet") - total_tax = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="totalTax") - total_tax_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="totalTaxSet") - - -class CalculatedDraftOrderLineItem(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "applied_discount", - "custom", - "custom_attributes", - "custom_attributes_v2", - "discounted_total", - "discounted_total_set", - "discounted_unit_price", - "discounted_unit_price_set", - "fulfillment_service", - "image", - "is_gift_card", - "name", - "original_total", - "original_total_set", - "original_unit_price", - "original_unit_price_set", - "product", - "quantity", - "requires_shipping", - "sku", - "taxable", - "title", - "total_discount", - "total_discount_set", - "variant", - "variant_title", - "vendor", - "weight", - ) - applied_discount = sgqlc.types.Field("DraftOrderAppliedDiscount", graphql_name="appliedDiscount") - custom = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="custom") - custom_attributes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Attribute))), graphql_name="customAttributes" - ) - custom_attributes_v2 = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TypedAttribute"))), - graphql_name="customAttributesV2", - ) - discounted_total = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="discountedTotal") - discounted_total_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="discountedTotalSet") - discounted_unit_price = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="discountedUnitPrice") - discounted_unit_price_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="discountedUnitPriceSet") - fulfillment_service = sgqlc.types.Field("FulfillmentService", graphql_name="fulfillmentService") - image = sgqlc.types.Field("Image", graphql_name="image") - is_gift_card = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isGiftCard") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - original_total = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="originalTotal") - original_total_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="originalTotalSet") - original_unit_price = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="originalUnitPrice") - original_unit_price_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="originalUnitPriceSet") - product = sgqlc.types.Field("Product", graphql_name="product") - quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") - requires_shipping = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresShipping") - sku = sgqlc.types.Field(String, graphql_name="sku") - taxable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxable") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - total_discount = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="totalDiscount") - total_discount_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="totalDiscountSet") - variant = sgqlc.types.Field("ProductVariant", graphql_name="variant") - variant_title = sgqlc.types.Field(String, graphql_name="variantTitle") - vendor = sgqlc.types.Field(String, graphql_name="vendor") - weight = sgqlc.types.Field("Weight", graphql_name="weight") - - -class CalculatedLineItem(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "calculated_discount_allocations", - "custom_attributes", - "discounted_unit_price_set", - "editable_quantity", - "editable_quantity_before_changes", - "editable_subtotal_set", - "has_staged_line_item_discount", - "id", - "image", - "original_unit_price_set", - "quantity", - "restockable", - "restocking", - "sku", - "staged_changes", - "title", - "uneditable_subtotal_set", - "variant", - "variant_title", - ) - calculated_discount_allocations = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CalculatedDiscountAllocation))), - graphql_name="calculatedDiscountAllocations", - ) - custom_attributes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Attribute))), graphql_name="customAttributes" - ) - discounted_unit_price_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="discountedUnitPriceSet") - editable_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="editableQuantity") - editable_quantity_before_changes = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="editableQuantityBeforeChanges") - editable_subtotal_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="editableSubtotalSet") - has_staged_line_item_discount = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasStagedLineItemDiscount") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - image = sgqlc.types.Field("Image", graphql_name="image") - original_unit_price_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="originalUnitPriceSet") - quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") - restockable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="restockable") - restocking = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="restocking") - sku = sgqlc.types.Field(String, graphql_name="sku") - staged_changes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("OrderStagedChange"))), - graphql_name="stagedChanges", - ) - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - uneditable_subtotal_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="uneditableSubtotalSet") - variant = sgqlc.types.Field("ProductVariant", graphql_name="variant") - variant_title = sgqlc.types.Field(String, graphql_name="variantTitle") - - -class CalculatedLineItemConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CalculatedLineItemEdge"))), graphql_name="edges" - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CalculatedLineItem))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class CalculatedLineItemEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(CalculatedLineItem), graphql_name="node") - - -class ChannelConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ChannelEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Channel"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class ChannelEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("Channel"), graphql_name="node") - - -class CheckoutProfileConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CheckoutProfileEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CheckoutProfile"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class CheckoutProfileEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("CheckoutProfile"), graphql_name="node") - - -class CollectionAddProductsPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("collection", "user_errors") - collection = sgqlc.types.Field("Collection", graphql_name="collection") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class CollectionAddProductsV2Payload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("job", "user_errors") - job = sgqlc.types.Field("Job", graphql_name="job") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CollectionAddProductsV2UserError"))), - graphql_name="userErrors", - ) - - -class CollectionConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CollectionEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Collection"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class CollectionCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("collection", "user_errors") - collection = sgqlc.types.Field("Collection", graphql_name="collection") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class CollectionDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_collection_id", "shop", "user_errors") - deleted_collection_id = sgqlc.types.Field(ID, graphql_name="deletedCollectionId") - shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class CollectionEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("Collection"), graphql_name="node") - - -class CollectionPublication(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("collection", "is_published", "publication", "publish_date") - collection = sgqlc.types.Field(sgqlc.types.non_null("Collection"), graphql_name="collection") - is_published = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isPublished") - publication = sgqlc.types.Field(sgqlc.types.non_null("Publication"), graphql_name="publication") - publish_date = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="publishDate") - - -class CollectionPublicationConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CollectionPublicationEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CollectionPublication))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class CollectionPublicationEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(CollectionPublication), graphql_name="node") - - -class CollectionPublishPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("collection", "collection_publications", "shop", "user_errors") - collection = sgqlc.types.Field("Collection", graphql_name="collection") - collection_publications = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null(CollectionPublication)), graphql_name="collectionPublications" - ) - shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class CollectionRemoveProductsPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("job", "user_errors") - job = sgqlc.types.Field("Job", graphql_name="job") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class CollectionReorderProductsPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("job", "user_errors") - job = sgqlc.types.Field("Job", graphql_name="job") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class CollectionRule(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("column", "condition", "condition_object", "relation") - column = sgqlc.types.Field(sgqlc.types.non_null(CollectionRuleColumn), graphql_name="column") - condition = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="condition") - condition_object = sgqlc.types.Field("CollectionRuleConditionObject", graphql_name="conditionObject") - relation = sgqlc.types.Field(sgqlc.types.non_null(CollectionRuleRelation), graphql_name="relation") - - -class CollectionRuleConditions(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("allowed_relations", "default_relation", "rule_type") - allowed_relations = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CollectionRuleRelation))), - graphql_name="allowedRelations", - ) - default_relation = sgqlc.types.Field(sgqlc.types.non_null(CollectionRuleRelation), graphql_name="defaultRelation") - rule_type = sgqlc.types.Field(sgqlc.types.non_null(CollectionRuleColumn), graphql_name="ruleType") - - -class CollectionRuleProductCategoryCondition(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("value",) - value = sgqlc.types.Field(sgqlc.types.non_null("ProductTaxonomyNode"), graphql_name="value") - - -class CollectionRuleSet(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("applied_disjunctively", "rules") - applied_disjunctively = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliedDisjunctively") - rules = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CollectionRule))), graphql_name="rules") - - -class CollectionRuleTextCondition(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("value",) - value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") - - -class CollectionUnpublishPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("collection", "shop", "user_errors") - collection = sgqlc.types.Field("Collection", graphql_name="collection") - shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class CollectionUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("collection", "job", "user_errors") - collection = sgqlc.types.Field("Collection", graphql_name="collection") - job = sgqlc.types.Field("Job", graphql_name="job") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class CommentEventAttachment(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("file_extension", "id", "image", "name", "size", "url") - file_extension = sgqlc.types.Field(String, graphql_name="fileExtension") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - image = sgqlc.types.Field("Image", graphql_name="image") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - size = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="size") - url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") - - -class CommentEventSubject(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ("has_timeline_comment", "id") - has_timeline_comment = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasTimelineComment") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - - -class CompaniesDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_company_ids", "user_errors") - deleted_company_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="deletedCompanyIds") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyAddressDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_address_id", "user_errors") - deleted_address_id = sgqlc.types.Field(ID, graphql_name="deletedAddressId") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyAssignCustomerAsContactPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("company_contact", "user_errors") - company_contact = sgqlc.types.Field("CompanyContact", graphql_name="companyContact") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyAssignMainContactPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("company", "user_errors") - company = sgqlc.types.Field("Company", graphql_name="company") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CompanyEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Company"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class CompanyContactAssignRolePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("company_contact_role_assignment", "user_errors") - company_contact_role_assignment = sgqlc.types.Field("CompanyContactRoleAssignment", graphql_name="companyContactRoleAssignment") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyContactAssignRolesPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("role_assignments", "user_errors") - role_assignments = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null("CompanyContactRoleAssignment")), graphql_name="roleAssignments" - ) - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyContactConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CompanyContactEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CompanyContact"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class CompanyContactCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("company_contact", "user_errors") - company_contact = sgqlc.types.Field("CompanyContact", graphql_name="companyContact") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyContactDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_company_contact_id", "user_errors") - deleted_company_contact_id = sgqlc.types.Field(ID, graphql_name="deletedCompanyContactId") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyContactEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("CompanyContact"), graphql_name="node") - - -class CompanyContactRevokeRolePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("revoked_company_contact_role_assignment_id", "user_errors") - revoked_company_contact_role_assignment_id = sgqlc.types.Field(ID, graphql_name="revokedCompanyContactRoleAssignmentId") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyContactRevokeRolesPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("revoked_role_assignment_ids", "user_errors") - revoked_role_assignment_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="revokedRoleAssignmentIds") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyContactRoleAssignmentConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CompanyContactRoleAssignmentEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CompanyContactRoleAssignment"))), - graphql_name="nodes", - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class CompanyContactRoleAssignmentEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("CompanyContactRoleAssignment"), graphql_name="node") - - -class CompanyContactRoleConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CompanyContactRoleEdge"))), graphql_name="edges" - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CompanyContactRole"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class CompanyContactRoleEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("CompanyContactRole"), graphql_name="node") - - -class CompanyContactUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("company_contact", "user_errors") - company_contact = sgqlc.types.Field("CompanyContact", graphql_name="companyContact") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyContactsDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_company_contact_ids", "user_errors") - deleted_company_contact_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="deletedCompanyContactIds") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("company", "user_errors") - company = sgqlc.types.Field("Company", graphql_name="company") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_company_id", "user_errors") - deleted_company_id = sgqlc.types.Field(ID, graphql_name="deletedCompanyId") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("Company"), graphql_name="node") - - -class CompanyLocationAssignAddressPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("addresses", "user_errors") - addresses = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("CompanyAddress")), graphql_name="addresses") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyLocationAssignRolesPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("role_assignments", "user_errors") - role_assignments = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null("CompanyContactRoleAssignment")), graphql_name="roleAssignments" - ) - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyLocationAssignTaxExemptionsPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("company_location", "user_errors") - company_location = sgqlc.types.Field("CompanyLocation", graphql_name="companyLocation") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyLocationConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CompanyLocationEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CompanyLocation"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class CompanyLocationCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("company_location", "user_errors") - company_location = sgqlc.types.Field("CompanyLocation", graphql_name="companyLocation") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyLocationCreateTaxRegistrationPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("company_location", "user_errors") - company_location = sgqlc.types.Field("CompanyLocation", graphql_name="companyLocation") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyLocationDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_company_location_id", "user_errors") - deleted_company_location_id = sgqlc.types.Field(ID, graphql_name="deletedCompanyLocationId") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyLocationEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("CompanyLocation"), graphql_name="node") - - -class CompanyLocationRevokeRolesPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("revoked_role_assignment_ids", "user_errors") - revoked_role_assignment_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="revokedRoleAssignmentIds") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyLocationRevokeTaxExemptionsPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("company_location", "user_errors") - company_location = sgqlc.types.Field("CompanyLocation", graphql_name="companyLocation") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyLocationRevokeTaxRegistrationPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("company_location", "user_errors") - company_location = sgqlc.types.Field("CompanyLocation", graphql_name="companyLocation") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyLocationUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("company_location", "user_errors") - company_location = sgqlc.types.Field("CompanyLocation", graphql_name="companyLocation") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyLocationsDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_company_location_ids", "user_errors") - deleted_company_location_ids = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="deletedCompanyLocationIds" - ) - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyRevokeMainContactPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("company", "user_errors") - company = sgqlc.types.Field("Company", graphql_name="company") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CompanyUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("company", "user_errors") - company = sgqlc.types.Field("Company", graphql_name="company") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BusinessCustomerUserError"))), - graphql_name="userErrors", - ) - - -class CountriesInShippingZones(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("country_codes", "include_rest_of_world") - country_codes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CountryCode))), graphql_name="countryCodes" - ) - include_rest_of_world = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="includeRestOfWorld") - - -class CountryHarmonizedSystemCode(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("country_code", "harmonized_system_code") - country_code = sgqlc.types.Field(sgqlc.types.non_null(CountryCode), graphql_name="countryCode") - harmonized_system_code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="harmonizedSystemCode") - - -class CountryHarmonizedSystemCodeConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CountryHarmonizedSystemCodeEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CountryHarmonizedSystemCode))), - graphql_name="nodes", - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class CountryHarmonizedSystemCodeEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(CountryHarmonizedSystemCode), graphql_name="node") - - -class CurrencyFormats(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "money_format", - "money_in_emails_format", - "money_with_currency_format", - "money_with_currency_in_emails_format", - ) - money_format = sgqlc.types.Field(sgqlc.types.non_null(FormattedString), graphql_name="moneyFormat") - money_in_emails_format = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="moneyInEmailsFormat") - money_with_currency_format = sgqlc.types.Field(sgqlc.types.non_null(FormattedString), graphql_name="moneyWithCurrencyFormat") - money_with_currency_in_emails_format = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="moneyWithCurrencyInEmailsFormat") - - -class CurrencySetting(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("currency_code", "currency_name", "enabled", "rate_updated_at") - currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currencyCode") - currency_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="currencyName") - enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="enabled") - rate_updated_at = sgqlc.types.Field(DateTime, graphql_name="rateUpdatedAt") - - -class CurrencySettingConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CurrencySettingEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CurrencySetting))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class CurrencySettingEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(CurrencySetting), graphql_name="node") - - -class CustomerAddTaxExemptionsPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("customer", "user_errors") - customer = sgqlc.types.Field("Customer", graphql_name="customer") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class CustomerConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Customer"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class CustomerCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("customer", "user_errors") - customer = sgqlc.types.Field("Customer", graphql_name="customer") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class CustomerCreditCard(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "billing_address", - "brand", - "expires_soon", - "expiry_month", - "expiry_year", - "first_digits", - "is_revocable", - "last_digits", - "masked_number", - "name", - "source", - "virtual_last_digits", - ) - billing_address = sgqlc.types.Field("CustomerCreditCardBillingAddress", graphql_name="billingAddress") - brand = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="brand") - expires_soon = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="expiresSoon") - expiry_month = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="expiryMonth") - expiry_year = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="expiryYear") - first_digits = sgqlc.types.Field(String, graphql_name="firstDigits") - is_revocable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isRevocable") - last_digits = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="lastDigits") - masked_number = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="maskedNumber") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - source = sgqlc.types.Field(String, graphql_name="source") - virtual_last_digits = sgqlc.types.Field(String, graphql_name="virtualLastDigits") - - -class CustomerCreditCardBillingAddress(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("address1", "city", "country", "country_code", "province", "province_code", "zip") - address1 = sgqlc.types.Field(String, graphql_name="address1") - city = sgqlc.types.Field(String, graphql_name="city") - country = sgqlc.types.Field(String, graphql_name="country") - country_code = sgqlc.types.Field(CountryCode, graphql_name="countryCode") - province = sgqlc.types.Field(String, graphql_name="province") - province_code = sgqlc.types.Field(String, graphql_name="provinceCode") - zip = sgqlc.types.Field(String, graphql_name="zip") - - -class CustomerDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_customer_id", "shop", "user_errors") - deleted_customer_id = sgqlc.types.Field(ID, graphql_name="deletedCustomerId") - shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class CustomerEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("Customer"), graphql_name="node") - - -class CustomerEmailAddress(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "email_address", - "marketing_state", - "marketing_unsubscribe_url", - "open_tracking_level", - "open_tracking_url", - ) - email_address = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="emailAddress") - marketing_state = sgqlc.types.Field(sgqlc.types.non_null(CustomerEmailAddressMarketingState), graphql_name="marketingState") - marketing_unsubscribe_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="marketingUnsubscribeUrl") - open_tracking_level = sgqlc.types.Field(sgqlc.types.non_null(CustomerEmailAddressOpenTrackingLevel), graphql_name="openTrackingLevel") - open_tracking_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="openTrackingUrl") - - -class CustomerEmailMarketingConsentState(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("consent_updated_at", "marketing_opt_in_level", "marketing_state") - consent_updated_at = sgqlc.types.Field(DateTime, graphql_name="consentUpdatedAt") - marketing_opt_in_level = sgqlc.types.Field(CustomerMarketingOptInLevel, graphql_name="marketingOptInLevel") - marketing_state = sgqlc.types.Field(sgqlc.types.non_null(CustomerEmailMarketingState), graphql_name="marketingState") - - -class CustomerEmailMarketingConsentUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("customer", "user_errors") - customer = sgqlc.types.Field("Customer", graphql_name="customer") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerEmailMarketingConsentUpdateUserError"))), - graphql_name="userErrors", - ) - - -class CustomerGenerateAccountActivationUrlPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("account_activation_url", "user_errors") - account_activation_url = sgqlc.types.Field(URL, graphql_name="accountActivationUrl") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class CustomerJourney(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("customer_order_index", "days_to_conversion", "first_visit", "last_visit", "moments") - customer_order_index = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="customerOrderIndex") - days_to_conversion = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="daysToConversion") - first_visit = sgqlc.types.Field(sgqlc.types.non_null("CustomerVisit"), graphql_name="firstVisit") - last_visit = sgqlc.types.Field("CustomerVisit", graphql_name="lastVisit") - moments = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerMoment"))), graphql_name="moments") - - -class CustomerJourneySummary(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "customer_order_index", - "days_to_conversion", - "first_visit", - "last_visit", - "moments", - "moments_count", - "ready", - ) - customer_order_index = sgqlc.types.Field(Int, graphql_name="customerOrderIndex") - days_to_conversion = sgqlc.types.Field(Int, graphql_name="daysToConversion") - first_visit = sgqlc.types.Field("CustomerVisit", graphql_name="firstVisit") - last_visit = sgqlc.types.Field("CustomerVisit", graphql_name="lastVisit") - moments = sgqlc.types.Field( - "CustomerMomentConnection", - graphql_name="moments", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - moments_count = sgqlc.types.Field(Int, graphql_name="momentsCount") - ready = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="ready") - - -class CustomerMoment(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ("occurred_at",) - occurred_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="occurredAt") - - -class CustomerMomentConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerMomentEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CustomerMoment))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class CustomerMomentEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(CustomerMoment), graphql_name="node") - - -class CustomerPaymentInstrumentBillingAddress(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("address1", "city", "country", "country_code", "name", "province", "province_code", "zip") - address1 = sgqlc.types.Field(String, graphql_name="address1") - city = sgqlc.types.Field(String, graphql_name="city") - country = sgqlc.types.Field(String, graphql_name="country") - country_code = sgqlc.types.Field(CountryCode, graphql_name="countryCode") - name = sgqlc.types.Field(String, graphql_name="name") - province = sgqlc.types.Field(String, graphql_name="province") - province_code = sgqlc.types.Field(String, graphql_name="provinceCode") - zip = sgqlc.types.Field(String, graphql_name="zip") - - -class CustomerPaymentMethodConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerPaymentMethodEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerPaymentMethod"))), graphql_name="nodes" - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class CustomerPaymentMethodCreditCardCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("customer_payment_method", "user_errors") - customer_payment_method = sgqlc.types.Field("CustomerPaymentMethod", graphql_name="customerPaymentMethod") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class CustomerPaymentMethodCreditCardUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("customer_payment_method", "user_errors") - customer_payment_method = sgqlc.types.Field("CustomerPaymentMethod", graphql_name="customerPaymentMethod") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class CustomerPaymentMethodEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("CustomerPaymentMethod"), graphql_name="node") - - -class CustomerPaymentMethodGetUpdateUrlPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("update_payment_method_url", "user_errors") - update_payment_method_url = sgqlc.types.Field(URL, graphql_name="updatePaymentMethodUrl") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerPaymentMethodGetUpdateUrlUserError"))), - graphql_name="userErrors", - ) - - -class CustomerPaymentMethodPaypalBillingAgreementCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("customer_payment_method", "user_errors") - customer_payment_method = sgqlc.types.Field("CustomerPaymentMethod", graphql_name="customerPaymentMethod") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerPaymentMethodUserError"))), - graphql_name="userErrors", - ) - - -class CustomerPaymentMethodPaypalBillingAgreementUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("customer_payment_method", "user_errors") - customer_payment_method = sgqlc.types.Field("CustomerPaymentMethod", graphql_name="customerPaymentMethod") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerPaymentMethodUserError"))), - graphql_name="userErrors", - ) - - -class CustomerPaymentMethodRemoteCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("customer_payment_method", "user_errors") - customer_payment_method = sgqlc.types.Field("CustomerPaymentMethod", graphql_name="customerPaymentMethod") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerPaymentMethodRemoteUserError"))), - graphql_name="userErrors", - ) - - -class CustomerPaymentMethodRemoteCreditCardCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("customer_payment_method", "user_errors") - customer_payment_method = sgqlc.types.Field("CustomerPaymentMethod", graphql_name="customerPaymentMethod") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerPaymentMethodUserError"))), - graphql_name="userErrors", - ) - - -class CustomerPaymentMethodRevokePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("revoked_customer_payment_method_id", "user_errors") - revoked_customer_payment_method_id = sgqlc.types.Field(ID, graphql_name="revokedCustomerPaymentMethodId") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class CustomerPaymentMethodSendUpdateEmailPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("customer", "user_errors") - customer = sgqlc.types.Field("Customer", graphql_name="customer") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class CustomerPaypalBillingAgreement(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("billing_address", "inactive", "is_revocable", "paypal_account_email") - billing_address = sgqlc.types.Field(CustomerPaymentInstrumentBillingAddress, graphql_name="billingAddress") - inactive = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="inactive") - is_revocable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isRevocable") - paypal_account_email = sgqlc.types.Field(String, graphql_name="paypalAccountEmail") - - -class CustomerPhoneNumber(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("marketing_state", "phone_number") - marketing_state = sgqlc.types.Field(sgqlc.types.non_null(CustomerSmsMarketingState), graphql_name="marketingState") - phone_number = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="phoneNumber") - - -class CustomerRemoveTaxExemptionsPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("customer", "user_errors") - customer = sgqlc.types.Field("Customer", graphql_name="customer") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class CustomerReplaceTaxExemptionsPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("customer", "user_errors") - customer = sgqlc.types.Field("Customer", graphql_name="customer") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class CustomerSegmentMember(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "amount_spent", - "default_address", - "default_email_address", - "default_phone_number", - "display_name", - "first_name", - "id", - "last_name", - "last_order_id", - "note", - "number_of_orders", - ) - amount_spent = sgqlc.types.Field("MoneyV2", graphql_name="amountSpent") - default_address = sgqlc.types.Field("MailingAddress", graphql_name="defaultAddress") - default_email_address = sgqlc.types.Field(CustomerEmailAddress, graphql_name="defaultEmailAddress") - default_phone_number = sgqlc.types.Field(CustomerPhoneNumber, graphql_name="defaultPhoneNumber") - display_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="displayName") - first_name = sgqlc.types.Field(String, graphql_name="firstName") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - last_name = sgqlc.types.Field(String, graphql_name="lastName") - last_order_id = sgqlc.types.Field(ID, graphql_name="lastOrderId") - note = sgqlc.types.Field(String, graphql_name="note") - number_of_orders = sgqlc.types.Field(UnsignedInt64, graphql_name="numberOfOrders") - - -class CustomerSegmentMemberConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "page_info", "statistics") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerSegmentMemberEdge"))), - graphql_name="edges", - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - statistics = sgqlc.types.Field(sgqlc.types.non_null("SegmentStatistics"), graphql_name="statistics") - - -class CustomerSegmentMemberEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(CustomerSegmentMember), graphql_name="node") - - -class CustomerShopPayAgreement(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "expires_soon", - "expiry_month", - "expiry_year", - "inactive", - "is_revocable", - "last_digits", - "masked_number", - "name", - ) - expires_soon = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="expiresSoon") - expiry_month = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="expiryMonth") - expiry_year = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="expiryYear") - inactive = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="inactive") - is_revocable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isRevocable") - last_digits = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="lastDigits") - masked_number = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="maskedNumber") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - - -class CustomerSmsMarketingConsentState(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("consent_collected_from", "consent_updated_at", "marketing_opt_in_level", "marketing_state") - consent_collected_from = sgqlc.types.Field(CustomerConsentCollectedFrom, graphql_name="consentCollectedFrom") - consent_updated_at = sgqlc.types.Field(DateTime, graphql_name="consentUpdatedAt") - marketing_opt_in_level = sgqlc.types.Field(sgqlc.types.non_null(CustomerMarketingOptInLevel), graphql_name="marketingOptInLevel") - marketing_state = sgqlc.types.Field(sgqlc.types.non_null(CustomerSmsMarketingState), graphql_name="marketingState") - - -class CustomerSmsMarketingConsentUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("customer", "user_errors") - customer = sgqlc.types.Field("Customer", graphql_name="customer") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("CustomerSmsMarketingConsentError"))), - graphql_name="userErrors", - ) - - -class CustomerStatistics(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("predicted_spend_tier",) - predicted_spend_tier = sgqlc.types.Field(CustomerPredictedSpendTier, graphql_name="predictedSpendTier") - - -class CustomerUpdateDefaultAddressPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("customer", "user_errors") - customer = sgqlc.types.Field("Customer", graphql_name="customer") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class CustomerUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("customer", "user_errors") - customer = sgqlc.types.Field("Customer", graphql_name="customer") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class DelegateAccessToken(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("access_scopes", "access_token", "created_at") - access_scopes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="accessScopes") - access_token = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="accessToken") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - - -class DelegateAccessTokenCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("delegate_access_token", "shop", "user_errors") - delegate_access_token = sgqlc.types.Field(DelegateAccessToken, graphql_name="delegateAccessToken") - shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DelegateAccessTokenCreateUserError"))), - graphql_name="userErrors", - ) - - -class DeletionEvent(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("occurred_at", "subject_id", "subject_type") - occurred_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="occurredAt") - subject_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="subjectId") - subject_type = sgqlc.types.Field(sgqlc.types.non_null(DeletionEventSubjectType), graphql_name="subjectType") - - -class DeletionEventConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeletionEventEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DeletionEvent))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class DeletionEventEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(DeletionEvent), graphql_name="node") - - -class DeliveryAvailableService(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("countries", "name") - countries = sgqlc.types.Field(sgqlc.types.non_null("DeliveryCountryCodesOrRestOfWorld"), graphql_name="countries") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - - -class DeliveryCarrierServiceAndLocations(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("carrier_service", "locations") - carrier_service = sgqlc.types.Field(sgqlc.types.non_null("DeliveryCarrierService"), graphql_name="carrierService") - locations = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Location"))), graphql_name="locations") - - -class DeliveryCountryAndZone(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("country", "zone") - country = sgqlc.types.Field(sgqlc.types.non_null("DeliveryCountry"), graphql_name="country") - zone = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="zone") - - -class DeliveryCountryCodeOrRestOfWorld(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("country_code", "rest_of_world") - country_code = sgqlc.types.Field(CountryCode, graphql_name="countryCode") - rest_of_world = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="restOfWorld") - - -class DeliveryCountryCodesOrRestOfWorld(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("country_codes", "rest_of_world") - country_codes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CountryCode))), graphql_name="countryCodes" - ) - rest_of_world = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="restOfWorld") - - -class DeliveryLegacyModeBlocked(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("blocked", "reasons") - blocked = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="blocked") - reasons = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(DeliveryLegacyModeBlockedReason)), graphql_name="reasons") - - -class DeliveryLocationGroupZone(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("method_definition_counts", "method_definitions", "zone") - method_definition_counts = sgqlc.types.Field( - sgqlc.types.non_null("DeliveryMethodDefinitionCounts"), graphql_name="methodDefinitionCounts" - ) - method_definitions = sgqlc.types.Field( - sgqlc.types.non_null("DeliveryMethodDefinitionConnection"), - graphql_name="methodDefinitions", - args=sgqlc.types.ArgDict( - ( - ("eligible", sgqlc.types.Arg(Boolean, graphql_name="eligible", default=None)), - ("type", sgqlc.types.Arg(DeliveryMethodDefinitionType, graphql_name="type", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(MethodDefinitionSortKeys, graphql_name="sortKey", default="ID")), - ) - ), - ) - zone = sgqlc.types.Field(sgqlc.types.non_null("DeliveryZone"), graphql_name="zone") - - -class DeliveryLocationGroupZoneConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryLocationGroupZoneEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DeliveryLocationGroupZone))), graphql_name="nodes" - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class DeliveryLocationGroupZoneEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(DeliveryLocationGroupZone), graphql_name="node") - - -class DeliveryMethodDefinitionConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryMethodDefinitionEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryMethodDefinition"))), - graphql_name="nodes", - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class DeliveryMethodDefinitionCounts(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("participant_definitions_count", "rate_definitions_count") - participant_definitions_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="participantDefinitionsCount") - rate_definitions_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="rateDefinitionsCount") - - -class DeliveryMethodDefinitionEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("DeliveryMethodDefinition"), graphql_name="node") - - -class DeliveryParticipantService(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("active", "name") - active = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="active") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - - -class DeliveryProductVariantsCount(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("capped", "count") - capped = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="capped") - count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="count") - - -class DeliveryProfileConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryProfileEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryProfile"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class DeliveryProfileEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("DeliveryProfile"), graphql_name="node") - - -class DeliveryProfileItemConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryProfileItemEdge"))), graphql_name="edges" - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryProfileItem"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class DeliveryProfileItemEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("DeliveryProfileItem"), graphql_name="node") - - -class DeliveryProfileLocationGroup(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("countries_in_any_zone", "location_group", "location_group_zones") - countries_in_any_zone = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DeliveryCountryAndZone))), - graphql_name="countriesInAnyZone", - ) - location_group = sgqlc.types.Field(sgqlc.types.non_null("DeliveryLocationGroup"), graphql_name="locationGroup") - location_group_zones = sgqlc.types.Field( - sgqlc.types.non_null(DeliveryLocationGroupZoneConnection), - graphql_name="locationGroupZones", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - - -class DeliverySetting(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("legacy_mode_blocked", "legacy_mode_profiles") - legacy_mode_blocked = sgqlc.types.Field(sgqlc.types.non_null(DeliveryLegacyModeBlocked), graphql_name="legacyModeBlocked") - legacy_mode_profiles = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="legacyModeProfiles") - - -class DeliverySettingUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("setting", "user_errors") - setting = sgqlc.types.Field(DeliverySetting, graphql_name="setting") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class DeliveryShippingOriginAssignPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("user_errors",) - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class DiscountAllocation(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("allocated_amount_set", "discount_application") - allocated_amount_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="allocatedAmountSet") - discount_application = sgqlc.types.Field(sgqlc.types.non_null("DiscountApplication"), graphql_name="discountApplication") - - -class DiscountAmount(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("amount", "applies_on_each_item") - amount = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="amount") - applies_on_each_item = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliesOnEachItem") - - -class DiscountApplication(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ("allocation_method", "index", "target_selection", "target_type", "value") - allocation_method = sgqlc.types.Field(sgqlc.types.non_null(DiscountApplicationAllocationMethod), graphql_name="allocationMethod") - index = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="index") - target_selection = sgqlc.types.Field(sgqlc.types.non_null(DiscountApplicationTargetSelection), graphql_name="targetSelection") - target_type = sgqlc.types.Field(sgqlc.types.non_null(DiscountApplicationTargetType), graphql_name="targetType") - value = sgqlc.types.Field(sgqlc.types.non_null("PricingValue"), graphql_name="value") - - -class DiscountApplicationConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountApplicationEdge"))), graphql_name="edges" - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DiscountApplication))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class DiscountApplicationEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(DiscountApplication), graphql_name="node") - - -class DiscountAutomaticActivatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("automatic_discount_node", "user_errors") - automatic_discount_node = sgqlc.types.Field("DiscountAutomaticNode", graphql_name="automaticDiscountNode") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountAutomaticApp(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "app_discount_type", - "combines_with", - "discount_class", - "discount_id", - "ends_at", - "error_history", - "starts_at", - "status", - "title", - ) - app_discount_type = sgqlc.types.Field(sgqlc.types.non_null(AppDiscountType), graphql_name="appDiscountType") - combines_with = sgqlc.types.Field(sgqlc.types.non_null("DiscountCombinesWith"), graphql_name="combinesWith") - discount_class = sgqlc.types.Field(sgqlc.types.non_null(DiscountClass), graphql_name="discountClass") - discount_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="discountId") - ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") - error_history = sgqlc.types.Field("FunctionsErrorHistory", graphql_name="errorHistory") - starts_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="startsAt") - status = sgqlc.types.Field(sgqlc.types.non_null(DiscountStatus), graphql_name="status") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - - -class DiscountAutomaticAppCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("automatic_app_discount", "user_errors") - automatic_app_discount = sgqlc.types.Field(DiscountAutomaticApp, graphql_name="automaticAppDiscount") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountAutomaticAppUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("automatic_app_discount", "user_errors") - automatic_app_discount = sgqlc.types.Field(DiscountAutomaticApp, graphql_name="automaticAppDiscount") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountAutomaticBasic(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "async_usage_count", - "combines_with", - "created_at", - "customer_gets", - "discount_class", - "ends_at", - "minimum_requirement", - "short_summary", - "starts_at", - "status", - "summary", - "title", - ) - async_usage_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="asyncUsageCount") - combines_with = sgqlc.types.Field(sgqlc.types.non_null("DiscountCombinesWith"), graphql_name="combinesWith") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - customer_gets = sgqlc.types.Field(sgqlc.types.non_null("DiscountCustomerGets"), graphql_name="customerGets") - discount_class = sgqlc.types.Field(sgqlc.types.non_null(MerchandiseDiscountClass), graphql_name="discountClass") - ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") - minimum_requirement = sgqlc.types.Field(sgqlc.types.non_null("DiscountMinimumRequirement"), graphql_name="minimumRequirement") - short_summary = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="shortSummary") - starts_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="startsAt") - status = sgqlc.types.Field(sgqlc.types.non_null(DiscountStatus), graphql_name="status") - summary = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="summary") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - - -class DiscountAutomaticBasicCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("automatic_discount_node", "user_errors") - automatic_discount_node = sgqlc.types.Field("DiscountAutomaticNode", graphql_name="automaticDiscountNode") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountAutomaticBasicUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("automatic_discount_node", "user_errors") - automatic_discount_node = sgqlc.types.Field("DiscountAutomaticNode", graphql_name="automaticDiscountNode") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountAutomaticBulkDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("job", "user_errors") - job = sgqlc.types.Field("Job", graphql_name="job") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountAutomaticBxgyCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("automatic_discount_node", "user_errors") - automatic_discount_node = sgqlc.types.Field("DiscountAutomaticNode", graphql_name="automaticDiscountNode") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountAutomaticBxgyUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("automatic_discount_node", "user_errors") - automatic_discount_node = sgqlc.types.Field("DiscountAutomaticNode", graphql_name="automaticDiscountNode") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountAutomaticConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountAutomaticEdge"))), graphql_name="edges" - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountAutomatic"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class DiscountAutomaticDeactivatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("automatic_discount_node", "user_errors") - automatic_discount_node = sgqlc.types.Field("DiscountAutomaticNode", graphql_name="automaticDiscountNode") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountAutomaticDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_automatic_discount_id", "user_errors") - deleted_automatic_discount_id = sgqlc.types.Field(ID, graphql_name="deletedAutomaticDiscountId") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountAutomaticEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("DiscountAutomatic"), graphql_name="node") - - -class DiscountAutomaticNodeConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountAutomaticNodeEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountAutomaticNode"))), graphql_name="nodes" - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class DiscountAutomaticNodeEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("DiscountAutomaticNode"), graphql_name="node") - - -class DiscountCodeActivatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("code_discount_node", "user_errors") - code_discount_node = sgqlc.types.Field("DiscountCodeNode", graphql_name="codeDiscountNode") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountCodeApp(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "app_discount_type", - "applies_once_per_customer", - "async_usage_count", - "code_count", - "codes", - "combines_with", - "created_at", - "customer_selection", - "discount_class", - "discount_id", - "ends_at", - "error_history", - "has_timeline_comment", - "recurring_cycle_limit", - "shareable_urls", - "starts_at", - "status", - "title", - "total_sales", - "usage_limit", - ) - app_discount_type = sgqlc.types.Field(sgqlc.types.non_null(AppDiscountType), graphql_name="appDiscountType") - applies_once_per_customer = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliesOncePerCustomer") - async_usage_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="asyncUsageCount") - code_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="codeCount") - codes = sgqlc.types.Field( - sgqlc.types.non_null("DiscountRedeemCodeConnection"), - graphql_name="codes", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(DiscountCodeSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ) - ), - ) - combines_with = sgqlc.types.Field(sgqlc.types.non_null("DiscountCombinesWith"), graphql_name="combinesWith") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - customer_selection = sgqlc.types.Field(sgqlc.types.non_null("DiscountCustomerSelection"), graphql_name="customerSelection") - discount_class = sgqlc.types.Field(sgqlc.types.non_null(DiscountClass), graphql_name="discountClass") - discount_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="discountId") - ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") - error_history = sgqlc.types.Field("FunctionsErrorHistory", graphql_name="errorHistory") - has_timeline_comment = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasTimelineComment") - recurring_cycle_limit = sgqlc.types.Field(Int, graphql_name="recurringCycleLimit") - shareable_urls = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountShareableUrl"))), - graphql_name="shareableUrls", - ) - starts_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="startsAt") - status = sgqlc.types.Field(sgqlc.types.non_null(DiscountStatus), graphql_name="status") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - total_sales = sgqlc.types.Field("MoneyV2", graphql_name="totalSales") - usage_limit = sgqlc.types.Field(Int, graphql_name="usageLimit") - - -class DiscountCodeAppCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("code_app_discount", "user_errors") - code_app_discount = sgqlc.types.Field(DiscountCodeApp, graphql_name="codeAppDiscount") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountCodeAppUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("code_app_discount", "user_errors") - code_app_discount = sgqlc.types.Field(DiscountCodeApp, graphql_name="codeAppDiscount") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountCodeBasic(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "applies_once_per_customer", - "async_usage_count", - "code_count", - "codes", - "combines_with", - "created_at", - "customer_gets", - "customer_selection", - "discount_class", - "ends_at", - "has_timeline_comment", - "minimum_requirement", - "recurring_cycle_limit", - "shareable_urls", - "short_summary", - "starts_at", - "status", - "summary", - "title", - "total_sales", - "usage_limit", - ) - applies_once_per_customer = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliesOncePerCustomer") - async_usage_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="asyncUsageCount") - code_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="codeCount") - codes = sgqlc.types.Field( - sgqlc.types.non_null("DiscountRedeemCodeConnection"), - graphql_name="codes", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(DiscountCodeSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ) - ), - ) - combines_with = sgqlc.types.Field(sgqlc.types.non_null("DiscountCombinesWith"), graphql_name="combinesWith") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - customer_gets = sgqlc.types.Field(sgqlc.types.non_null("DiscountCustomerGets"), graphql_name="customerGets") - customer_selection = sgqlc.types.Field(sgqlc.types.non_null("DiscountCustomerSelection"), graphql_name="customerSelection") - discount_class = sgqlc.types.Field(sgqlc.types.non_null(MerchandiseDiscountClass), graphql_name="discountClass") - ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") - has_timeline_comment = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasTimelineComment") - minimum_requirement = sgqlc.types.Field("DiscountMinimumRequirement", graphql_name="minimumRequirement") - recurring_cycle_limit = sgqlc.types.Field(Int, graphql_name="recurringCycleLimit") - shareable_urls = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountShareableUrl"))), - graphql_name="shareableUrls", - ) - short_summary = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="shortSummary") - starts_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="startsAt") - status = sgqlc.types.Field(sgqlc.types.non_null(DiscountStatus), graphql_name="status") - summary = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="summary") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - total_sales = sgqlc.types.Field("MoneyV2", graphql_name="totalSales") - usage_limit = sgqlc.types.Field(Int, graphql_name="usageLimit") - - -class DiscountCodeBasicCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("code_discount_node", "user_errors") - code_discount_node = sgqlc.types.Field("DiscountCodeNode", graphql_name="codeDiscountNode") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountCodeBasicUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("code_discount_node", "user_errors") - code_discount_node = sgqlc.types.Field("DiscountCodeNode", graphql_name="codeDiscountNode") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountCodeBulkActivatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("job", "user_errors") - job = sgqlc.types.Field("Job", graphql_name="job") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountCodeBulkDeactivatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("job", "user_errors") - job = sgqlc.types.Field("Job", graphql_name="job") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountCodeBulkDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("job", "user_errors") - job = sgqlc.types.Field("Job", graphql_name="job") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountCodeBxgy(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "applies_once_per_customer", - "async_usage_count", - "code_count", - "codes", - "combines_with", - "created_at", - "customer_buys", - "customer_gets", - "customer_selection", - "discount_class", - "ends_at", - "has_timeline_comment", - "shareable_urls", - "starts_at", - "status", - "summary", - "title", - "total_sales", - "usage_limit", - "uses_per_order_limit", - ) - applies_once_per_customer = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliesOncePerCustomer") - async_usage_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="asyncUsageCount") - code_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="codeCount") - codes = sgqlc.types.Field( - sgqlc.types.non_null("DiscountRedeemCodeConnection"), - graphql_name="codes", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(DiscountCodeSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ) - ), - ) - combines_with = sgqlc.types.Field(sgqlc.types.non_null("DiscountCombinesWith"), graphql_name="combinesWith") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - customer_buys = sgqlc.types.Field(sgqlc.types.non_null("DiscountCustomerBuys"), graphql_name="customerBuys") - customer_gets = sgqlc.types.Field(sgqlc.types.non_null("DiscountCustomerGets"), graphql_name="customerGets") - customer_selection = sgqlc.types.Field(sgqlc.types.non_null("DiscountCustomerSelection"), graphql_name="customerSelection") - discount_class = sgqlc.types.Field(sgqlc.types.non_null(MerchandiseDiscountClass), graphql_name="discountClass") - ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") - has_timeline_comment = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasTimelineComment") - shareable_urls = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountShareableUrl"))), - graphql_name="shareableUrls", - ) - starts_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="startsAt") - status = sgqlc.types.Field(sgqlc.types.non_null(DiscountStatus), graphql_name="status") - summary = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="summary") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - total_sales = sgqlc.types.Field("MoneyV2", graphql_name="totalSales") - usage_limit = sgqlc.types.Field(Int, graphql_name="usageLimit") - uses_per_order_limit = sgqlc.types.Field(Int, graphql_name="usesPerOrderLimit") - - -class DiscountCodeBxgyCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("code_discount_node", "user_errors") - code_discount_node = sgqlc.types.Field("DiscountCodeNode", graphql_name="codeDiscountNode") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountCodeBxgyUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("code_discount_node", "user_errors") - code_discount_node = sgqlc.types.Field("DiscountCodeNode", graphql_name="codeDiscountNode") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountCodeDeactivatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("code_discount_node", "user_errors") - code_discount_node = sgqlc.types.Field("DiscountCodeNode", graphql_name="codeDiscountNode") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountCodeDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_code_discount_id", "user_errors") - deleted_code_discount_id = sgqlc.types.Field(ID, graphql_name="deletedCodeDiscountId") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountCodeFreeShipping(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "applies_on_one_time_purchase", - "applies_on_subscription", - "applies_once_per_customer", - "async_usage_count", - "code_count", - "codes", - "combines_with", - "created_at", - "customer_selection", - "destination_selection", - "discount_class", - "ends_at", - "has_timeline_comment", - "maximum_shipping_price", - "minimum_requirement", - "recurring_cycle_limit", - "shareable_urls", - "short_summary", - "starts_at", - "status", - "summary", - "title", - "total_sales", - "usage_limit", - ) - applies_on_one_time_purchase = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliesOnOneTimePurchase") - applies_on_subscription = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliesOnSubscription") - applies_once_per_customer = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliesOncePerCustomer") - async_usage_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="asyncUsageCount") - code_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="codeCount") - codes = sgqlc.types.Field( - sgqlc.types.non_null("DiscountRedeemCodeConnection"), - graphql_name="codes", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(DiscountCodeSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ) - ), - ) - combines_with = sgqlc.types.Field(sgqlc.types.non_null("DiscountCombinesWith"), graphql_name="combinesWith") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - customer_selection = sgqlc.types.Field(sgqlc.types.non_null("DiscountCustomerSelection"), graphql_name="customerSelection") - destination_selection = sgqlc.types.Field( - sgqlc.types.non_null("DiscountShippingDestinationSelection"), graphql_name="destinationSelection" - ) - discount_class = sgqlc.types.Field(sgqlc.types.non_null(ShippingDiscountClass), graphql_name="discountClass") - ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") - has_timeline_comment = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasTimelineComment") - maximum_shipping_price = sgqlc.types.Field("MoneyV2", graphql_name="maximumShippingPrice") - minimum_requirement = sgqlc.types.Field("DiscountMinimumRequirement", graphql_name="minimumRequirement") - recurring_cycle_limit = sgqlc.types.Field(Int, graphql_name="recurringCycleLimit") - shareable_urls = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountShareableUrl"))), - graphql_name="shareableUrls", - ) - short_summary = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="shortSummary") - starts_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="startsAt") - status = sgqlc.types.Field(sgqlc.types.non_null(DiscountStatus), graphql_name="status") - summary = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="summary") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - total_sales = sgqlc.types.Field("MoneyV2", graphql_name="totalSales") - usage_limit = sgqlc.types.Field(Int, graphql_name="usageLimit") - - -class DiscountCodeFreeShippingCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("code_discount_node", "user_errors") - code_discount_node = sgqlc.types.Field("DiscountCodeNode", graphql_name="codeDiscountNode") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountCodeFreeShippingUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("code_discount_node", "user_errors") - code_discount_node = sgqlc.types.Field("DiscountCodeNode", graphql_name="codeDiscountNode") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountCodeNodeConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountCodeNodeEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountCodeNode"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class DiscountCodeNodeEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("DiscountCodeNode"), graphql_name="node") - - -class DiscountCodeRedeemCodeBulkDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("job", "user_errors") - job = sgqlc.types.Field("Job", graphql_name="job") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountCollections(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("collections",) - collections = sgqlc.types.Field( - sgqlc.types.non_null(CollectionConnection), - graphql_name="collections", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - - -class DiscountCombinesWith(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("order_discounts", "product_discounts", "shipping_discounts") - order_discounts = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="orderDiscounts") - product_discounts = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="productDiscounts") - shipping_discounts = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="shippingDiscounts") - - -class DiscountCountries(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("countries", "include_rest_of_world") - countries = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CountryCode))), graphql_name="countries") - include_rest_of_world = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="includeRestOfWorld") - - -class DiscountCountryAll(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("all_countries",) - all_countries = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="allCountries") - - -class DiscountCustomerAll(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("all_customers",) - all_customers = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="allCustomers") - - -class DiscountCustomerBuys(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("items", "value") - items = sgqlc.types.Field(sgqlc.types.non_null("DiscountItems"), graphql_name="items") - value = sgqlc.types.Field(sgqlc.types.non_null("DiscountCustomerBuysValue"), graphql_name="value") - - -class DiscountCustomerGets(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("applies_on_one_time_purchase", "applies_on_subscription", "items", "value") - applies_on_one_time_purchase = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliesOnOneTimePurchase") - applies_on_subscription = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliesOnSubscription") - items = sgqlc.types.Field(sgqlc.types.non_null("DiscountItems"), graphql_name="items") - value = sgqlc.types.Field(sgqlc.types.non_null("DiscountCustomerGetsValue"), graphql_name="value") - - -class DiscountCustomerSegments(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("segments",) - segments = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Segment"))), graphql_name="segments") - - -class DiscountCustomers(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("customers",) - customers = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Customer"))), graphql_name="customers") - - -class DiscountMinimumQuantity(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("greater_than_or_equal_to_quantity",) - greater_than_or_equal_to_quantity = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="greaterThanOrEqualToQuantity") - - -class DiscountMinimumSubtotal(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("greater_than_or_equal_to_subtotal",) - greater_than_or_equal_to_subtotal = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="greaterThanOrEqualToSubtotal") - - -class DiscountNodeConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountNodeEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountNode"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class DiscountNodeEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("DiscountNode"), graphql_name="node") - - -class DiscountOnQuantity(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("effect", "quantity") - effect = sgqlc.types.Field(sgqlc.types.non_null("DiscountEffect"), graphql_name="effect") - quantity = sgqlc.types.Field(sgqlc.types.non_null("DiscountQuantity"), graphql_name="quantity") - - -class DiscountPercentage(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("percentage",) - percentage = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="percentage") - - -class DiscountProducts(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("product_variants", "products") - product_variants = sgqlc.types.Field( - sgqlc.types.non_null("ProductVariantConnection"), - graphql_name="productVariants", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - products = sgqlc.types.Field( - sgqlc.types.non_null("ProductConnection"), - graphql_name="products", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - - -class DiscountPurchaseAmount(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("amount",) - amount = sgqlc.types.Field(sgqlc.types.non_null(Decimal), graphql_name="amount") - - -class DiscountQuantity(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("quantity",) - quantity = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="quantity") - - -class DiscountRedeemCode(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("async_usage_count", "code", "created_by", "id") - async_usage_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="asyncUsageCount") - code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") - created_by = sgqlc.types.Field("App", graphql_name="createdBy") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - - -class DiscountRedeemCodeBulkAddPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("bulk_creation", "user_errors") - bulk_creation = sgqlc.types.Field("DiscountRedeemCodeBulkCreation", graphql_name="bulkCreation") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="userErrors" - ) - - -class DiscountRedeemCodeBulkCreationCode(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("code", "discount_redeem_code", "errors") - code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") - discount_redeem_code = sgqlc.types.Field(DiscountRedeemCode, graphql_name="discountRedeemCode") - errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountUserError"))), graphql_name="errors") - - -class DiscountRedeemCodeBulkCreationCodeConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountRedeemCodeBulkCreationCodeEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DiscountRedeemCodeBulkCreationCode))), - graphql_name="nodes", - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class DiscountRedeemCodeBulkCreationCodeEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(DiscountRedeemCodeBulkCreationCode), graphql_name="node") - - -class DiscountRedeemCodeConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DiscountRedeemCodeEdge"))), graphql_name="edges" - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DiscountRedeemCode))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class DiscountRedeemCodeEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(DiscountRedeemCode), graphql_name="node") - - -class DiscountShareableUrl(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("target_item_image", "target_type", "title", "url") - target_item_image = sgqlc.types.Field("Image", graphql_name="targetItemImage") - target_type = sgqlc.types.Field(sgqlc.types.non_null(DiscountShareableUrlTargetType), graphql_name="targetType") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") - - -class DisplayableError(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ("field", "message") - field = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="field") - message = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="message") - - -class DisputeEvidenceUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("dispute_evidence", "user_errors") - dispute_evidence = sgqlc.types.Field("ShopifyPaymentsDisputeEvidence", graphql_name="disputeEvidence") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DisputeEvidenceUpdateUserError"))), - graphql_name="userErrors", - ) - - -class DomainLocalization(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("alternate_locales", "country", "default_locale") - alternate_locales = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="alternateLocales" - ) - country = sgqlc.types.Field(String, graphql_name="country") - default_locale = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="defaultLocale") - - -class DraftOrderAppliedDiscount(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("amount_set", "amount_v2", "description", "title", "value", "value_type") - amount_set = sgqlc.types.Field(sgqlc.types.non_null("MoneyBag"), graphql_name="amountSet") - amount_v2 = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="amountV2") - description = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="description") - title = sgqlc.types.Field(String, graphql_name="title") - value = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="value") - value_type = sgqlc.types.Field(sgqlc.types.non_null(DraftOrderAppliedDiscountType), graphql_name="valueType") - - -class DraftOrderBulkAddTagsPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("job", "user_errors") - job = sgqlc.types.Field("Job", graphql_name="job") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class DraftOrderBulkDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("job", "user_errors") - job = sgqlc.types.Field("Job", graphql_name="job") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class DraftOrderBulkRemoveTagsPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("job", "user_errors") - job = sgqlc.types.Field("Job", graphql_name="job") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class DraftOrderCalculatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("calculated_draft_order", "user_errors") - calculated_draft_order = sgqlc.types.Field(CalculatedDraftOrder, graphql_name="calculatedDraftOrder") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class DraftOrderCompletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("draft_order", "user_errors") - draft_order = sgqlc.types.Field("DraftOrder", graphql_name="draftOrder") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class DraftOrderConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DraftOrderEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DraftOrder"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class DraftOrderCreateFromOrderPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("draft_order", "user_errors") - draft_order = sgqlc.types.Field("DraftOrder", graphql_name="draftOrder") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class DraftOrderCreateMerchantCheckoutPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("user_errors",) - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class DraftOrderCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("draft_order", "user_errors") - draft_order = sgqlc.types.Field("DraftOrder", graphql_name="draftOrder") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class DraftOrderDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_id", "user_errors") - deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class DraftOrderDuplicatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("draft_order", "user_errors") - draft_order = sgqlc.types.Field("DraftOrder", graphql_name="draftOrder") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class DraftOrderEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("DraftOrder"), graphql_name="node") - - -class DraftOrderInvoicePreviewPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("preview_html", "preview_subject", "user_errors") - preview_html = sgqlc.types.Field(HTML, graphql_name="previewHtml") - preview_subject = sgqlc.types.Field(HTML, graphql_name="previewSubject") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class DraftOrderInvoiceSendPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("draft_order", "user_errors") - draft_order = sgqlc.types.Field("DraftOrder", graphql_name="draftOrder") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class DraftOrderLineItemConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DraftOrderLineItemEdge"))), graphql_name="edges" - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DraftOrderLineItem"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class DraftOrderLineItemEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("DraftOrderLineItem"), graphql_name="node") - - -class DraftOrderUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("draft_order", "user_errors") - draft_order = sgqlc.types.Field("DraftOrder", graphql_name="draftOrder") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class EditableProperty(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("locked", "reason") - locked = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="locked") - reason = sgqlc.types.Field(FormattedString, graphql_name="reason") - - -class Event(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ( - "app_title", - "attribute_to_app", - "attribute_to_user", - "created_at", - "critical_alert", - "id", - "message", - ) - app_title = sgqlc.types.Field(String, graphql_name="appTitle") - attribute_to_app = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="attributeToApp") - attribute_to_user = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="attributeToUser") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - critical_alert = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="criticalAlert") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - message = sgqlc.types.Field(sgqlc.types.non_null(FormattedString), graphql_name="message") - - -class EventBridgeWebhookSubscriptionCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("user_errors", "webhook_subscription") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - webhook_subscription = sgqlc.types.Field("WebhookSubscription", graphql_name="webhookSubscription") - - -class EventBridgeWebhookSubscriptionUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("user_errors", "webhook_subscription") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - webhook_subscription = sgqlc.types.Field("WebhookSubscription", graphql_name="webhookSubscription") - - -class EventConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("EventEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Event))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class EventEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(Event), graphql_name="node") - - -class FailedRequirement(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("action", "message") - action = sgqlc.types.Field("NavigationItem", graphql_name="action") - message = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="message") - - -class File(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ("alt", "created_at", "file_errors", "file_status", "preview") - alt = sgqlc.types.Field(String, graphql_name="alt") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - file_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FileError"))), graphql_name="fileErrors") - file_status = sgqlc.types.Field(sgqlc.types.non_null(FileStatus), graphql_name="fileStatus") - preview = sgqlc.types.Field("MediaPreviewImage", graphql_name="preview") - - -class FileConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FileEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(File))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class FileCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("files", "user_errors") - files = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(File)), graphql_name="files") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FilesUserError"))), graphql_name="userErrors" - ) - - -class FileDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_file_ids", "user_errors") - deleted_file_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="deletedFileIds") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FilesUserError"))), graphql_name="userErrors" - ) - - -class FileEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(File), graphql_name="node") - - -class FileError(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("code", "details", "message") - code = sgqlc.types.Field(sgqlc.types.non_null(FileErrorCode), graphql_name="code") - details = sgqlc.types.Field(String, graphql_name="details") - message = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="message") - - -class FileUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("files", "user_errors") - files = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(File)), graphql_name="files") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FilesUserError"))), graphql_name="userErrors" - ) - - -class FilterOption(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("label", "value") - label = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="label") - value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") - - -class FlowTriggerReceivePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("user_errors",) - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class FulfillmentCancelPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("fulfillment", "user_errors") - fulfillment = sgqlc.types.Field("Fulfillment", graphql_name="fulfillment") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class FulfillmentConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Fulfillment"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class FulfillmentCreateV2Payload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("fulfillment", "user_errors") - fulfillment = sgqlc.types.Field("Fulfillment", graphql_name="fulfillment") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class FulfillmentEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("Fulfillment"), graphql_name="node") - - -class FulfillmentEventConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentEventEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentEvent"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class FulfillmentEventEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("FulfillmentEvent"), graphql_name="node") - - -class FulfillmentHold(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("reason", "reason_notes") - reason = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentHoldReason), graphql_name="reason") - reason_notes = sgqlc.types.Field(String, graphql_name="reasonNotes") - - -class FulfillmentLineItemConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentLineItemEdge"))), graphql_name="edges" - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentLineItem"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class FulfillmentLineItemEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("FulfillmentLineItem"), graphql_name="node") - - -class FulfillmentOrderAcceptCancellationRequestPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("fulfillment_order", "user_errors") - fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="fulfillmentOrder") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class FulfillmentOrderAcceptFulfillmentRequestPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("fulfillment_order", "user_errors") - fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="fulfillmentOrder") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class FulfillmentOrderAssignedLocation(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("address1", "address2", "city", "country_code", "location", "name", "phone", "province", "zip") - address1 = sgqlc.types.Field(String, graphql_name="address1") - address2 = sgqlc.types.Field(String, graphql_name="address2") - city = sgqlc.types.Field(String, graphql_name="city") - country_code = sgqlc.types.Field(sgqlc.types.non_null(CountryCode), graphql_name="countryCode") - location = sgqlc.types.Field("Location", graphql_name="location") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - phone = sgqlc.types.Field(String, graphql_name="phone") - province = sgqlc.types.Field(String, graphql_name="province") - zip = sgqlc.types.Field(String, graphql_name="zip") - - -class FulfillmentOrderCancelPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("fulfillment_order", "replacement_fulfillment_order", "user_errors") - fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="fulfillmentOrder") - replacement_fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="replacementFulfillmentOrder") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class FulfillmentOrderClosePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("fulfillment_order", "user_errors") - fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="fulfillmentOrder") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class FulfillmentOrderConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrder"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class FulfillmentOrderEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("FulfillmentOrder"), graphql_name="node") - - -class FulfillmentOrderHoldPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("fulfillment_order", "user_errors") - fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="fulfillmentOrder") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderHoldUserError"))), - graphql_name="userErrors", - ) - - -class FulfillmentOrderInternationalDuties(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("incoterm",) - incoterm = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="incoterm") - - -class FulfillmentOrderLineItemConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderLineItemEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderLineItem"))), - graphql_name="nodes", - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class FulfillmentOrderLineItemEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("FulfillmentOrderLineItem"), graphql_name="node") - - -class FulfillmentOrderLineItemWarning(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("description", "title") - description = sgqlc.types.Field(String, graphql_name="description") - title = sgqlc.types.Field(String, graphql_name="title") - - -class FulfillmentOrderLocationForMove(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("location", "message", "movable") - location = sgqlc.types.Field(sgqlc.types.non_null("Location"), graphql_name="location") - message = sgqlc.types.Field(String, graphql_name="message") - movable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="movable") - - -class FulfillmentOrderLocationForMoveConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderLocationForMoveEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FulfillmentOrderLocationForMove))), - graphql_name="nodes", - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class FulfillmentOrderLocationForMoveEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentOrderLocationForMove), graphql_name="node") - - -class FulfillmentOrderMerchantRequestConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderMerchantRequestEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderMerchantRequest"))), - graphql_name="nodes", - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class FulfillmentOrderMerchantRequestEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("FulfillmentOrderMerchantRequest"), graphql_name="node") - - -class FulfillmentOrderMovePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "moved_fulfillment_order", - "original_fulfillment_order", - "remaining_fulfillment_order", - "user_errors", - ) - moved_fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="movedFulfillmentOrder") - original_fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="originalFulfillmentOrder") - remaining_fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="remainingFulfillmentOrder") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class FulfillmentOrderOpenPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("fulfillment_order", "user_errors") - fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="fulfillmentOrder") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class FulfillmentOrderRejectCancellationRequestPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("fulfillment_order", "user_errors") - fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="fulfillmentOrder") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class FulfillmentOrderRejectFulfillmentRequestPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("fulfillment_order", "user_errors") - fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="fulfillmentOrder") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class FulfillmentOrderReleaseHoldPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("fulfillment_order", "user_errors") - fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="fulfillmentOrder") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderReleaseHoldUserError"))), - graphql_name="userErrors", - ) - - -class FulfillmentOrderReschedulePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("fulfillment_order", "user_errors") - fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="fulfillmentOrder") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrderRescheduleUserError"))), - graphql_name="userErrors", - ) - - -class FulfillmentOrderSubmitCancellationRequestPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("fulfillment_order", "user_errors") - fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="fulfillmentOrder") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class FulfillmentOrderSubmitFulfillmentRequestPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "original_fulfillment_order", - "submitted_fulfillment_order", - "unsubmitted_fulfillment_order", - "user_errors", - ) - original_fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="originalFulfillmentOrder") - submitted_fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="submittedFulfillmentOrder") - unsubmitted_fulfillment_order = sgqlc.types.Field("FulfillmentOrder", graphql_name="unsubmittedFulfillmentOrder") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class FulfillmentOrderSupportedAction(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("action", "external_url") - action = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentOrderAction), graphql_name="action") - external_url = sgqlc.types.Field(URL, graphql_name="externalUrl") - - -class FulfillmentOrdersSetFulfillmentDeadlinePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("success", "user_errors") - success = sgqlc.types.Field(Boolean, graphql_name="success") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("FulfillmentOrdersSetFulfillmentDeadlineUserError"))), - graphql_name="userErrors", - ) - - -class FulfillmentOriginAddress(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("address1", "address2", "city", "country_code", "province_code", "zip") - address1 = sgqlc.types.Field(String, graphql_name="address1") - address2 = sgqlc.types.Field(String, graphql_name="address2") - city = sgqlc.types.Field(String, graphql_name="city") - country_code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="countryCode") - province_code = sgqlc.types.Field(String, graphql_name="provinceCode") - zip = sgqlc.types.Field(String, graphql_name="zip") - - -class FulfillmentService(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "callback_url", - "fulfillment_orders_opt_in", - "handle", - "id", - "inventory_management", - "location", - "permits_sku_sharing", - "product_based", - "service_name", - "shipping_methods", - "type", - ) - callback_url = sgqlc.types.Field(URL, graphql_name="callbackUrl") - fulfillment_orders_opt_in = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="fulfillmentOrdersOptIn") - handle = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="handle") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - inventory_management = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="inventoryManagement") - location = sgqlc.types.Field("Location", graphql_name="location") - permits_sku_sharing = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="permitsSkuSharing") - product_based = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="productBased") - service_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="serviceName") - shipping_methods = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShippingMethod"))), - graphql_name="shippingMethods", - ) - type = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentServiceType), graphql_name="type") - - -class FulfillmentServiceCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("fulfillment_service", "user_errors") - fulfillment_service = sgqlc.types.Field(FulfillmentService, graphql_name="fulfillmentService") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class FulfillmentServiceDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_id", "user_errors") - deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class FulfillmentServiceUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("fulfillment_service", "user_errors") - fulfillment_service = sgqlc.types.Field(FulfillmentService, graphql_name="fulfillmentService") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class FulfillmentTrackingInfo(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("company", "number", "url") - company = sgqlc.types.Field(String, graphql_name="company") - number = sgqlc.types.Field(String, graphql_name="number") - url = sgqlc.types.Field(URL, graphql_name="url") - - -class FulfillmentTrackingInfoUpdateV2Payload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("fulfillment", "user_errors") - fulfillment = sgqlc.types.Field("Fulfillment", graphql_name="fulfillment") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class FunctionsAppBridge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("create_path", "details_path") - create_path = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="createPath") - details_path = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="detailsPath") - - -class FunctionsErrorHistory(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "errors_first_occurred_at", - "first_occurred_at", - "has_been_shared_since_last_error", - "has_shared_recent_errors", - ) - errors_first_occurred_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="errorsFirstOccurredAt") - first_occurred_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="firstOccurredAt") - has_been_shared_since_last_error = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasBeenSharedSinceLastError") - has_shared_recent_errors = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasSharedRecentErrors") - - -class GiftCardConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("GiftCardEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("GiftCard"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class GiftCardCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("gift_card", "gift_card_code", "user_errors") - gift_card = sgqlc.types.Field("GiftCard", graphql_name="giftCard") - gift_card_code = sgqlc.types.Field(String, graphql_name="giftCardCode") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("GiftCardUserError"))), graphql_name="userErrors" - ) - - -class GiftCardDisablePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("gift_card", "user_errors") - gift_card = sgqlc.types.Field("GiftCard", graphql_name="giftCard") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class GiftCardEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("GiftCard"), graphql_name="node") - - -class GiftCardUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("gift_card", "user_errors") - gift_card = sgqlc.types.Field("GiftCard", graphql_name="giftCard") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class HasEvents(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ("events",) - events = sgqlc.types.Field( - sgqlc.types.non_null(EventConnection), - graphql_name="events", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(EventSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - - -class HasLocalizationExtensions(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ("localization_extensions",) - localization_extensions = sgqlc.types.Field( - sgqlc.types.non_null("LocalizationExtensionConnection"), - graphql_name="localizationExtensions", - args=sgqlc.types.ArgDict( - ( - ( - "country_codes", - sgqlc.types.Arg( - sgqlc.types.list_of(sgqlc.types.non_null(CountryCode)), - graphql_name="countryCodes", - default=None, - ), - ), - ( - "purposes", - sgqlc.types.Arg( - sgqlc.types.list_of(sgqlc.types.non_null(LocalizationExtensionPurpose)), - graphql_name="purposes", - default=None, - ), - ), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - - -class HasMetafieldDefinitions(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ("metafield_definitions",) - metafield_definitions = sgqlc.types.Field( - sgqlc.types.non_null("MetafieldDefinitionConnection"), - graphql_name="metafieldDefinitions", - args=sgqlc.types.ArgDict( - ( - ("namespace", sgqlc.types.Arg(String, graphql_name="namespace", default=None)), - ( - "pinned_status", - sgqlc.types.Arg(MetafieldDefinitionPinnedStatus, graphql_name="pinnedStatus", default="ANY"), - ), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(MetafieldDefinitionSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - - -class HasMetafields(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ("metafield", "metafields", "private_metafield", "private_metafields") - metafield = sgqlc.types.Field( - "Metafield", - graphql_name="metafield", - args=sgqlc.types.ArgDict( - ( - ("namespace", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="namespace", default=None)), - ("key", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="key", default=None)), - ) - ), - ) - metafields = sgqlc.types.Field( - sgqlc.types.non_null("MetafieldConnection"), - graphql_name="metafields", - args=sgqlc.types.ArgDict( - ( - ("namespace", sgqlc.types.Arg(String, graphql_name="namespace", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - private_metafield = sgqlc.types.Field( - "PrivateMetafield", - graphql_name="privateMetafield", - args=sgqlc.types.ArgDict( - ( - ("namespace", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="namespace", default=None)), - ("key", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="key", default=None)), - ) - ), - ) - private_metafields = sgqlc.types.Field( - sgqlc.types.non_null("PrivateMetafieldConnection"), - graphql_name="privateMetafields", - args=sgqlc.types.ArgDict( - ( - ("namespace", sgqlc.types.Arg(String, graphql_name="namespace", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - - -class HasPublishedTranslations(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ("translations",) - translations = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PublishedTranslation"))), - graphql_name="translations", - args=sgqlc.types.ArgDict( - ( - ("locale", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="locale", default=None)), - ("market_id", sgqlc.types.Arg(ID, graphql_name="marketId", default=None)), - ) - ), - ) - - -class ImageConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ImageEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Image"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class ImageEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("Image"), graphql_name="node") - - -class ImageUploadParameter(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("name", "value") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") - - -class InventoryActivatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("inventory_level", "user_errors") - inventory_level = sgqlc.types.Field("InventoryLevel", graphql_name="inventoryLevel") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class InventoryAdjustQuantityPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("inventory_level", "user_errors") - inventory_level = sgqlc.types.Field("InventoryLevel", graphql_name="inventoryLevel") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class InventoryBulkAdjustQuantityAtLocationPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("inventory_levels", "user_errors") - inventory_levels = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("InventoryLevel")), graphql_name="inventoryLevels") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class InventoryBulkToggleActivationPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("inventory_item", "inventory_levels", "user_errors") - inventory_item = sgqlc.types.Field("InventoryItem", graphql_name="inventoryItem") - inventory_levels = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("InventoryLevel")), graphql_name="inventoryLevels") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("InventoryBulkToggleActivationUserError"))), - graphql_name="userErrors", - ) - - -class InventoryDeactivatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("user_errors",) - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class InventoryItemConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("InventoryItemEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("InventoryItem"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class InventoryItemEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("InventoryItem"), graphql_name="node") - - -class InventoryItemUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("inventory_item", "user_errors") - inventory_item = sgqlc.types.Field("InventoryItem", graphql_name="inventoryItem") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class InventoryLevelConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("InventoryLevelEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("InventoryLevel"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class InventoryLevelEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("InventoryLevel"), graphql_name="node") - - -class Job(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("done", "id", "query") - done = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="done") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - query = sgqlc.types.Field("QueryRoot", graphql_name="query") - - -class LegacyInteroperability(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ("legacy_resource_id",) - legacy_resource_id = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="legacyResourceId") - - -class LimitedPendingOrderCount(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("at_max", "count") - at_max = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="atMax") - count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="count") - - -class LineItemConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("LineItemEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("LineItem"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class LineItemEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("LineItem"), graphql_name="node") - - -class LineItemMutableConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("LineItemMutableEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("LineItemMutable"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class LineItemMutableEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("LineItemMutable"), graphql_name="node") - - -class LineItemSellingPlan(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("name", "selling_plan_id") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - selling_plan_id = sgqlc.types.Field(ID, graphql_name="sellingPlanId") - - -class Locale(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("iso_code", "name") - iso_code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="isoCode") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - - -class LocalizationExtension(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("country_code", "key", "purpose", "title", "value") - country_code = sgqlc.types.Field(sgqlc.types.non_null(CountryCode), graphql_name="countryCode") - key = sgqlc.types.Field(sgqlc.types.non_null(LocalizationExtensionKey), graphql_name="key") - purpose = sgqlc.types.Field(sgqlc.types.non_null(LocalizationExtensionPurpose), graphql_name="purpose") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") - - -class LocalizationExtensionConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("LocalizationExtensionEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(LocalizationExtension))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class LocalizationExtensionEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(LocalizationExtension), graphql_name="node") - - -class LocationActivatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("location", "location_activate_user_errors") - location = sgqlc.types.Field("Location", graphql_name="location") - location_activate_user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("LocationActivateUserError"))), - graphql_name="locationActivateUserErrors", - ) - - -class LocationAddPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("location", "user_errors") - location = sgqlc.types.Field("Location", graphql_name="location") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("LocationAddUserError"))), - graphql_name="userErrors", - ) - - -class LocationAddress(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "address1", - "address2", - "city", - "country", - "country_code", - "formatted", - "latitude", - "longitude", - "phone", - "province", - "province_code", - "zip", - ) - address1 = sgqlc.types.Field(String, graphql_name="address1") - address2 = sgqlc.types.Field(String, graphql_name="address2") - city = sgqlc.types.Field(String, graphql_name="city") - country = sgqlc.types.Field(String, graphql_name="country") - country_code = sgqlc.types.Field(String, graphql_name="countryCode") - formatted = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="formatted") - latitude = sgqlc.types.Field(Float, graphql_name="latitude") - longitude = sgqlc.types.Field(Float, graphql_name="longitude") - phone = sgqlc.types.Field(String, graphql_name="phone") - province = sgqlc.types.Field(String, graphql_name="province") - province_code = sgqlc.types.Field(String, graphql_name="provinceCode") - zip = sgqlc.types.Field(String, graphql_name="zip") - - -class LocationConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("LocationEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Location"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class LocationDeactivatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("location", "location_deactivate_user_errors") - location = sgqlc.types.Field("Location", graphql_name="location") - location_deactivate_user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("LocationDeactivateUserError"))), - graphql_name="locationDeactivateUserErrors", - ) - - -class LocationDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_location_id", "location_delete_user_errors") - deleted_location_id = sgqlc.types.Field(ID, graphql_name="deletedLocationId") - location_delete_user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("LocationDeleteUserError"))), - graphql_name="locationDeleteUserErrors", - ) - - -class LocationEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("Location"), graphql_name="node") - - -class LocationEditPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("location", "user_errors") - location = sgqlc.types.Field("Location", graphql_name="location") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("LocationEditUserError"))), - graphql_name="userErrors", - ) - - -class LocationSuggestedAddress(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "address1", - "address2", - "city", - "country", - "country_code", - "formatted", - "province", - "province_code", - "zip", - ) - address1 = sgqlc.types.Field(String, graphql_name="address1") - address2 = sgqlc.types.Field(String, graphql_name="address2") - city = sgqlc.types.Field(String, graphql_name="city") - country = sgqlc.types.Field(String, graphql_name="country") - country_code = sgqlc.types.Field(CountryCode, graphql_name="countryCode") - formatted = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="formatted") - province = sgqlc.types.Field(String, graphql_name="province") - province_code = sgqlc.types.Field(String, graphql_name="provinceCode") - zip = sgqlc.types.Field(String, graphql_name="zip") - - -class MarketConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Market"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class MarketCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("market", "user_errors") - market = sgqlc.types.Field("Market", graphql_name="market") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketUserError"))), graphql_name="userErrors" - ) - - -class MarketCurrencySettings(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("base_currency", "local_currencies") - base_currency = sgqlc.types.Field(sgqlc.types.non_null(CurrencySetting), graphql_name="baseCurrency") - local_currencies = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="localCurrencies") - - -class MarketCurrencySettingsUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("market", "user_errors") - market = sgqlc.types.Field("Market", graphql_name="market") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketCurrencySettingsUserError"))), - graphql_name="userErrors", - ) - - -class MarketDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_id", "user_errors") - deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketUserError"))), graphql_name="userErrors" - ) - - -class MarketEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("Market"), graphql_name="node") - - -class MarketLocalizableContent(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("digest", "key", "value") - digest = sgqlc.types.Field(String, graphql_name="digest") - key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") - value = sgqlc.types.Field(String, graphql_name="value") - - -class MarketLocalizableResource(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("market_localizable_content", "market_localizations", "resource_id") - market_localizable_content = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MarketLocalizableContent))), - graphql_name="marketLocalizableContent", - ) - market_localizations = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketLocalization"))), - graphql_name="marketLocalizations", - args=sgqlc.types.ArgDict((("market_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="marketId", default=None)),)), - ) - resource_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="resourceId") - - -class MarketLocalizableResourceConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketLocalizableResourceEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MarketLocalizableResource))), graphql_name="nodes" - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class MarketLocalizableResourceEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(MarketLocalizableResource), graphql_name="node") - - -class MarketLocalization(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("key", "market", "outdated", "value") - key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") - market = sgqlc.types.Field(sgqlc.types.non_null("Market"), graphql_name="market") - outdated = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="outdated") - value = sgqlc.types.Field(String, graphql_name="value") - - -class MarketLocalizationsRegisterPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("market_localizations", "user_errors") - market_localizations = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null(MarketLocalization)), graphql_name="marketLocalizations" - ) - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TranslationUserError"))), - graphql_name="userErrors", - ) - - -class MarketLocalizationsRemovePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("market_localizations", "user_errors") - market_localizations = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null(MarketLocalization)), graphql_name="marketLocalizations" - ) - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TranslationUserError"))), - graphql_name="userErrors", - ) - - -class MarketRegion(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ("id", "name") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - - -class MarketRegionConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketRegionEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MarketRegion))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class MarketRegionDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_id", "market", "user_errors") - deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") - market = sgqlc.types.Field("Market", graphql_name="market") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketUserError"))), graphql_name="userErrors" - ) - - -class MarketRegionEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(MarketRegion), graphql_name="node") - - -class MarketRegionsCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("market", "user_errors") - market = sgqlc.types.Field("Market", graphql_name="market") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketUserError"))), graphql_name="userErrors" - ) - - -class MarketUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("market", "user_errors") - market = sgqlc.types.Field("Market", graphql_name="market") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketUserError"))), graphql_name="userErrors" - ) - - -class MarketWebPresenceCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("market", "user_errors") - market = sgqlc.types.Field("Market", graphql_name="market") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketUserError"))), graphql_name="userErrors" - ) - - -class MarketWebPresenceDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_id", "market", "user_errors") - deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") - market = sgqlc.types.Field("Market", graphql_name="market") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketUserError"))), graphql_name="userErrors" - ) - - -class MarketWebPresenceRootUrl(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("locale", "url") - locale = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="locale") - url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") - - -class MarketWebPresenceUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("market", "user_errors") - market = sgqlc.types.Field("Market", graphql_name="market") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketUserError"))), graphql_name="userErrors" - ) - - -class MarketingActivityConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketingActivityEdge"))), graphql_name="edges" - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketingActivity"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class MarketingActivityCreateExternalPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("marketing_activity", "user_errors") - marketing_activity = sgqlc.types.Field("MarketingActivity", graphql_name="marketingActivity") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketingActivityUserError"))), - graphql_name="userErrors", - ) - - -class MarketingActivityCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("marketing_activity", "redirect_path", "user_errors") - marketing_activity = sgqlc.types.Field("MarketingActivity", graphql_name="marketingActivity") - redirect_path = sgqlc.types.Field(String, graphql_name="redirectPath") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class MarketingActivityEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("MarketingActivity"), graphql_name="node") - - -class MarketingActivityExtensionAppErrors(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("code", "user_errors") - code = sgqlc.types.Field(sgqlc.types.non_null(MarketingActivityExtensionAppErrorCode), graphql_name="code") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class MarketingActivityUpdateExternalPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("marketing_activity", "user_errors") - marketing_activity = sgqlc.types.Field("MarketingActivity", graphql_name="marketingActivity") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketingActivityUserError"))), - graphql_name="userErrors", - ) - - -class MarketingActivityUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("marketing_activity", "redirect_path", "user_errors") - marketing_activity = sgqlc.types.Field("MarketingActivity", graphql_name="marketingActivity") - redirect_path = sgqlc.types.Field(String, graphql_name="redirectPath") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class MarketingBudget(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("budget_type", "total") - budget_type = sgqlc.types.Field(sgqlc.types.non_null(MarketingBudgetBudgetType), graphql_name="budgetType") - total = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="total") - - -class MarketingEngagement(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "ad_spend", - "clicks_count", - "comments_count", - "complaints_count", - "fails_count", - "favorites_count", - "fetched_at", - "impressions_count", - "is_cumulative", - "marketing_activity", - "occurred_on", - "sends_count", - "shares_count", - "unique_clicks_count", - "unique_views_count", - "unsubscribes_count", - "utc_offset", - "views_count", - ) - ad_spend = sgqlc.types.Field("MoneyV2", graphql_name="adSpend") - clicks_count = sgqlc.types.Field(Int, graphql_name="clicksCount") - comments_count = sgqlc.types.Field(Int, graphql_name="commentsCount") - complaints_count = sgqlc.types.Field(Int, graphql_name="complaintsCount") - fails_count = sgqlc.types.Field(Int, graphql_name="failsCount") - favorites_count = sgqlc.types.Field(Int, graphql_name="favoritesCount") - fetched_at = sgqlc.types.Field(DateTime, graphql_name="fetchedAt") - impressions_count = sgqlc.types.Field(Int, graphql_name="impressionsCount") - is_cumulative = sgqlc.types.Field(Boolean, graphql_name="isCumulative") - marketing_activity = sgqlc.types.Field(sgqlc.types.non_null("MarketingActivity"), graphql_name="marketingActivity") - occurred_on = sgqlc.types.Field(sgqlc.types.non_null(Date), graphql_name="occurredOn") - sends_count = sgqlc.types.Field(Int, graphql_name="sendsCount") - shares_count = sgqlc.types.Field(Int, graphql_name="sharesCount") - unique_clicks_count = sgqlc.types.Field(Int, graphql_name="uniqueClicksCount") - unique_views_count = sgqlc.types.Field(Int, graphql_name="uniqueViewsCount") - unsubscribes_count = sgqlc.types.Field(Int, graphql_name="unsubscribesCount") - utc_offset = sgqlc.types.Field(UtcOffset, graphql_name="utcOffset") - views_count = sgqlc.types.Field(Int, graphql_name="viewsCount") - - -class MarketingEngagementCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("marketing_engagement", "user_errors") - marketing_engagement = sgqlc.types.Field(MarketingEngagement, graphql_name="marketingEngagement") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class MarketingEventConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketingEventEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketingEvent"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class MarketingEventEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("MarketingEvent"), graphql_name="node") - - -class Media(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ("alt", "media_content_type", "media_errors", "media_warnings", "preview", "status") - alt = sgqlc.types.Field(String, graphql_name="alt") - media_content_type = sgqlc.types.Field(sgqlc.types.non_null(MediaContentType), graphql_name="mediaContentType") - media_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MediaError"))), graphql_name="mediaErrors" - ) - media_warnings = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MediaWarning"))), graphql_name="mediaWarnings" - ) - preview = sgqlc.types.Field("MediaPreviewImage", graphql_name="preview") - status = sgqlc.types.Field(sgqlc.types.non_null(MediaStatus), graphql_name="status") - - -class MediaConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MediaEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Media))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class MediaEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(Media), graphql_name="node") - - -class MediaError(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("code", "details", "message") - code = sgqlc.types.Field(sgqlc.types.non_null(MediaErrorCode), graphql_name="code") - details = sgqlc.types.Field(String, graphql_name="details") - message = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="message") - - -class MediaImageOriginalSource(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("file_size",) - file_size = sgqlc.types.Field(Int, graphql_name="fileSize") - - -class MediaPreviewImage(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("image", "status") - image = sgqlc.types.Field("Image", graphql_name="image") - status = sgqlc.types.Field(sgqlc.types.non_null(MediaPreviewImageStatus), graphql_name="status") - - -class MediaWarning(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("code", "message") - code = sgqlc.types.Field(sgqlc.types.non_null(MediaWarningCode), graphql_name="code") - message = sgqlc.types.Field(String, graphql_name="message") - - -class MerchantApprovalSignals(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("identity_verified", "verified_by_shopify") - identity_verified = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="identityVerified") - verified_by_shopify = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="verifiedByShopify") - - -class MetafieldConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Metafield"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class MetafieldDefinitionConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldDefinitionEdge"))), graphql_name="edges" - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldDefinition"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class MetafieldDefinitionCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("created_definition", "user_errors") - created_definition = sgqlc.types.Field("MetafieldDefinition", graphql_name="createdDefinition") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldDefinitionCreateUserError"))), - graphql_name="userErrors", - ) - - -class MetafieldDefinitionDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_definition_id", "user_errors") - deleted_definition_id = sgqlc.types.Field(ID, graphql_name="deletedDefinitionId") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldDefinitionDeleteUserError"))), - graphql_name="userErrors", - ) - - -class MetafieldDefinitionEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("MetafieldDefinition"), graphql_name="node") - - -class MetafieldDefinitionPinPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("pinned_definition", "user_errors") - pinned_definition = sgqlc.types.Field("MetafieldDefinition", graphql_name="pinnedDefinition") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldDefinitionPinUserError"))), - graphql_name="userErrors", - ) - - -class MetafieldDefinitionSupportedValidation(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("name", "type") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="type") - - -class MetafieldDefinitionType(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("category", "name", "supported_validations", "supports_definition_migrations") - category = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="category") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - supported_validations = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MetafieldDefinitionSupportedValidation))), - graphql_name="supportedValidations", - ) - supports_definition_migrations = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="supportsDefinitionMigrations") - - -class MetafieldDefinitionUnpinPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("unpinned_definition", "user_errors") - unpinned_definition = sgqlc.types.Field("MetafieldDefinition", graphql_name="unpinnedDefinition") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldDefinitionUnpinUserError"))), - graphql_name="userErrors", - ) - - -class MetafieldDefinitionUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("updated_definition", "user_errors") - updated_definition = sgqlc.types.Field("MetafieldDefinition", graphql_name="updatedDefinition") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldDefinitionUpdateUserError"))), - graphql_name="userErrors", - ) - - -class MetafieldDefinitionValidation(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("name", "type", "value") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="type") - value = sgqlc.types.Field(String, graphql_name="value") - - -class MetafieldDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_id", "user_errors") - deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class MetafieldEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("Metafield"), graphql_name="node") - - -class MetafieldReferenceConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldReferenceEdge"))), graphql_name="edges" - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of("MetafieldReference")), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class MetafieldReferenceEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field("MetafieldReference", graphql_name="node") - - -class MetafieldStorefrontVisibilityConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldStorefrontVisibilityEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldStorefrontVisibility"))), - graphql_name="nodes", - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class MetafieldStorefrontVisibilityCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("metafield_storefront_visibility", "user_errors") - metafield_storefront_visibility = sgqlc.types.Field("MetafieldStorefrontVisibility", graphql_name="metafieldStorefrontVisibility") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class MetafieldStorefrontVisibilityDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_metafield_storefront_visibility_id", "user_errors") - deleted_metafield_storefront_visibility_id = sgqlc.types.Field(ID, graphql_name="deletedMetafieldStorefrontVisibilityId") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class MetafieldStorefrontVisibilityEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("MetafieldStorefrontVisibility"), graphql_name="node") - - -class MetafieldsSetPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("metafields", "user_errors") - metafields = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("Metafield")), graphql_name="metafields") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MetafieldsSetUserError"))), - graphql_name="userErrors", - ) - - -class Model3dBoundingBox(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("size",) - size = sgqlc.types.Field(sgqlc.types.non_null("Vector3"), graphql_name="size") - - -class Model3dSource(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("filesize", "format", "mime_type", "url") - filesize = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="filesize") - format = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="format") - mime_type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="mimeType") - url = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="url") - - -class MoneyBag(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("presentment_money", "shop_money") - presentment_money = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="presentmentMoney") - shop_money = sgqlc.types.Field(sgqlc.types.non_null("MoneyV2"), graphql_name="shopMoney") - - -class MoneyV2(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("amount", "currency_code") - amount = sgqlc.types.Field(sgqlc.types.non_null(Decimal), graphql_name="amount") - currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currencyCode") - - -class Mutation(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "app_credit_create", - "app_purchase_one_time_create", - "app_revenue_attribution_record_create", - "app_revenue_attribution_record_delete", - "app_subscription_cancel", - "app_subscription_create", - "app_subscription_line_item_update", - "app_subscription_trial_extend", - "app_usage_record_create", - "bulk_operation_cancel", - "bulk_operation_run_mutation", - "bulk_operation_run_query", - "bulk_product_resource_feedback_create", - "collection_add_products", - "collection_add_products_v2", - "collection_create", - "collection_delete", - "collection_remove_products", - "collection_reorder_products", - "collection_update", - "companies_delete", - "company_address_delete", - "company_assign_customer_as_contact", - "company_assign_main_contact", - "company_contact_assign_role", - "company_contact_assign_roles", - "company_contact_create", - "company_contact_delete", - "company_contact_revoke_role", - "company_contact_revoke_roles", - "company_contact_update", - "company_contacts_delete", - "company_create", - "company_delete", - "company_location_assign_address", - "company_location_assign_roles", - "company_location_assign_tax_exemptions", - "company_location_create", - "company_location_create_tax_registration", - "company_location_delete", - "company_location_revoke_roles", - "company_location_revoke_tax_exemptions", - "company_location_revoke_tax_registration", - "company_location_update", - "company_locations_delete", - "company_revoke_main_contact", - "company_update", - "customer_add_tax_exemptions", - "customer_create", - "customer_delete", - "customer_email_marketing_consent_update", - "customer_generate_account_activation_url", - "customer_payment_method_credit_card_create", - "customer_payment_method_credit_card_update", - "customer_payment_method_get_update_url", - "customer_payment_method_paypal_billing_agreement_create", - "customer_payment_method_paypal_billing_agreement_update", - "customer_payment_method_remote_create", - "customer_payment_method_revoke", - "customer_payment_method_send_update_email", - "customer_remove_tax_exemptions", - "customer_replace_tax_exemptions", - "customer_sms_marketing_consent_update", - "customer_update", - "customer_update_default_address", - "delegate_access_token_create", - "delivery_profile_create", - "delivery_profile_remove", - "delivery_profile_update", - "delivery_setting_update", - "delivery_shipping_origin_assign", - "discount_automatic_activate", - "discount_automatic_app_create", - "discount_automatic_app_update", - "discount_automatic_basic_create", - "discount_automatic_basic_update", - "discount_automatic_bulk_delete", - "discount_automatic_bxgy_create", - "discount_automatic_bxgy_update", - "discount_automatic_deactivate", - "discount_automatic_delete", - "discount_code_activate", - "discount_code_app_create", - "discount_code_app_update", - "discount_code_basic_create", - "discount_code_basic_update", - "discount_code_bulk_activate", - "discount_code_bulk_deactivate", - "discount_code_bulk_delete", - "discount_code_bxgy_create", - "discount_code_bxgy_update", - "discount_code_deactivate", - "discount_code_delete", - "discount_code_free_shipping_create", - "discount_code_free_shipping_update", - "discount_code_redeem_code_bulk_delete", - "discount_redeem_code_bulk_add", - "dispute_evidence_update", - "draft_order_bulk_add_tags", - "draft_order_bulk_delete", - "draft_order_bulk_remove_tags", - "draft_order_calculate", - "draft_order_complete", - "draft_order_create", - "draft_order_create_from_order", - "draft_order_create_merchant_checkout", - "draft_order_delete", - "draft_order_duplicate", - "draft_order_invoice_preview", - "draft_order_invoice_send", - "draft_order_update", - "event_bridge_webhook_subscription_create", - "event_bridge_webhook_subscription_update", - "file_create", - "file_delete", - "file_update", - "flow_trigger_receive", - "fulfillment_cancel", - "fulfillment_create_v2", - "fulfillment_order_accept_cancellation_request", - "fulfillment_order_accept_fulfillment_request", - "fulfillment_order_cancel", - "fulfillment_order_close", - "fulfillment_order_hold", - "fulfillment_order_move", - "fulfillment_order_open", - "fulfillment_order_reject_cancellation_request", - "fulfillment_order_reject_fulfillment_request", - "fulfillment_order_release_hold", - "fulfillment_order_reschedule", - "fulfillment_order_submit_cancellation_request", - "fulfillment_order_submit_fulfillment_request", - "fulfillment_orders_set_fulfillment_deadline", - "fulfillment_service_create", - "fulfillment_service_delete", - "fulfillment_service_update", - "fulfillment_tracking_info_update_v2", - "gift_card_create", - "gift_card_disable", - "gift_card_update", - "inventory_activate", - "inventory_adjust_quantity", - "inventory_bulk_adjust_quantity_at_location", - "inventory_bulk_toggle_activation", - "inventory_deactivate", - "inventory_item_update", - "location_activate", - "location_add", - "location_deactivate", - "location_delete", - "location_edit", - "market_create", - "market_currency_settings_update", - "market_delete", - "market_localizations_register", - "market_localizations_remove", - "market_region_delete", - "market_regions_create", - "market_update", - "market_web_presence_create", - "market_web_presence_delete", - "market_web_presence_update", - "marketing_activity_create", - "marketing_activity_create_external", - "marketing_activity_update", - "marketing_activity_update_external", - "marketing_engagement_create", - "metafield_definition_create", - "metafield_definition_delete", - "metafield_definition_pin", - "metafield_definition_unpin", - "metafield_definition_update", - "metafield_delete", - "metafield_storefront_visibility_create", - "metafield_storefront_visibility_delete", - "metafields_set", - "order_capture", - "order_close", - "order_create_mandate_payment", - "order_edit_add_custom_item", - "order_edit_add_line_item_discount", - "order_edit_add_variant", - "order_edit_begin", - "order_edit_commit", - "order_edit_remove_line_item_discount", - "order_edit_set_quantity", - "order_invoice_send", - "order_mark_as_paid", - "order_open", - "order_update", - "payment_terms_create", - "payment_terms_delete", - "payment_terms_update", - "price_list_create", - "price_list_delete", - "price_list_fixed_prices_add", - "price_list_fixed_prices_delete", - "price_list_update", - "price_rule_activate", - "price_rule_create", - "price_rule_deactivate", - "price_rule_delete", - "price_rule_discount_code_create", - "price_rule_discount_code_update", - "price_rule_update", - "private_metafield_delete", - "private_metafield_upsert", - "product_append_images", - "product_change_status", - "product_create", - "product_create_media", - "product_delete", - "product_delete_images", - "product_delete_media", - "product_duplicate", - "product_image_update", - "product_join_selling_plan_groups", - "product_leave_selling_plan_groups", - "product_reorder_images", - "product_reorder_media", - "product_update", - "product_update_media", - "product_variant_append_media", - "product_variant_create", - "product_variant_delete", - "product_variant_detach_media", - "product_variant_join_selling_plan_groups", - "product_variant_leave_selling_plan_groups", - "product_variant_update", - "product_variants_bulk_create", - "product_variants_bulk_delete", - "product_variants_bulk_reorder", - "product_variants_bulk_update", - "pub_sub_webhook_subscription_create", - "pub_sub_webhook_subscription_update", - "publishable_publish", - "publishable_publish_to_current_channel", - "publishable_unpublish", - "publishable_unpublish_to_current_channel", - "refund_create", - "saved_search_create", - "saved_search_delete", - "saved_search_update", - "script_tag_create", - "script_tag_delete", - "script_tag_update", - "segment_create", - "segment_delete", - "segment_update", - "selling_plan_group_add_product_variants", - "selling_plan_group_add_products", - "selling_plan_group_create", - "selling_plan_group_delete", - "selling_plan_group_remove_product_variants", - "selling_plan_group_remove_products", - "selling_plan_group_update", - "shipping_package_delete", - "shipping_package_make_default", - "shipping_package_update", - "shop_locale_disable", - "shop_locale_enable", - "shop_locale_update", - "shop_policy_update", - "staged_uploads_create", - "standard_metafield_definition_enable", - "storefront_access_token_create", - "storefront_access_token_delete", - "subscription_billing_attempt_create", - "subscription_billing_cycle_contract_draft_commit", - "subscription_billing_cycle_contract_draft_concatenate", - "subscription_billing_cycle_contract_edit", - "subscription_billing_cycle_edit_delete", - "subscription_billing_cycle_edits_delete", - "subscription_billing_cycle_schedule_edit", - "subscription_contract_create", - "subscription_contract_set_next_billing_date", - "subscription_contract_update", - "subscription_draft_commit", - "subscription_draft_discount_add", - "subscription_draft_discount_code_apply", - "subscription_draft_discount_remove", - "subscription_draft_discount_update", - "subscription_draft_free_shipping_discount_add", - "subscription_draft_free_shipping_discount_update", - "subscription_draft_line_add", - "subscription_draft_line_remove", - "subscription_draft_line_update", - "subscription_draft_update", - "tags_add", - "tags_remove", - "translations_register", - "translations_remove", - "url_redirect_bulk_delete_all", - "url_redirect_bulk_delete_by_ids", - "url_redirect_bulk_delete_by_saved_search", - "url_redirect_bulk_delete_by_search", - "url_redirect_create", - "url_redirect_delete", - "url_redirect_import_create", - "url_redirect_import_submit", - "url_redirect_update", - "web_pixel_create", - "web_pixel_delete", - "web_pixel_update", - "webhook_subscription_create", - "webhook_subscription_delete", - "webhook_subscription_update", - ) - app_credit_create = sgqlc.types.Field( - AppCreditCreatePayload, - graphql_name="appCreditCreate", - args=sgqlc.types.ArgDict( - ( - ( - "description", - sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="description", default=None), - ), - ("amount", sgqlc.types.Arg(sgqlc.types.non_null(MoneyInput), graphql_name="amount", default=None)), - ("test", sgqlc.types.Arg(Boolean, graphql_name="test", default=False)), - ) - ), - ) - app_purchase_one_time_create = sgqlc.types.Field( - AppPurchaseOneTimeCreatePayload, - graphql_name="appPurchaseOneTimeCreate", - args=sgqlc.types.ArgDict( - ( - ("name", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="name", default=None)), - ("price", sgqlc.types.Arg(sgqlc.types.non_null(MoneyInput), graphql_name="price", default=None)), - ("return_url", sgqlc.types.Arg(sgqlc.types.non_null(URL), graphql_name="returnUrl", default=None)), - ("test", sgqlc.types.Arg(Boolean, graphql_name="test", default=False)), - ) - ), - ) - app_revenue_attribution_record_create = sgqlc.types.Field( - AppRevenueAttributionRecordCreatePayload, - graphql_name="appRevenueAttributionRecordCreate", - args=sgqlc.types.ArgDict( - ( - ( - "app_revenue_attribution_record", - sgqlc.types.Arg( - sgqlc.types.non_null(AppRevenueAttributionRecordInput), - graphql_name="appRevenueAttributionRecord", - default=None, - ), - ), - ) - ), - ) - app_revenue_attribution_record_delete = sgqlc.types.Field( - AppRevenueAttributionRecordDeletePayload, - graphql_name="appRevenueAttributionRecordDelete", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - app_subscription_cancel = sgqlc.types.Field( - AppSubscriptionCancelPayload, - graphql_name="appSubscriptionCancel", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("prorate", sgqlc.types.Arg(Boolean, graphql_name="prorate", default=False)), - ) - ), - ) - app_subscription_create = sgqlc.types.Field( - AppSubscriptionCreatePayload, - graphql_name="appSubscriptionCreate", - args=sgqlc.types.ArgDict( - ( - ("name", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="name", default=None)), - ( - "line_items", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(AppSubscriptionLineItemInput))), - graphql_name="lineItems", - default=None, - ), - ), - ("test", sgqlc.types.Arg(Boolean, graphql_name="test", default=None)), - ("trial_days", sgqlc.types.Arg(Int, graphql_name="trialDays", default=None)), - ("return_url", sgqlc.types.Arg(sgqlc.types.non_null(URL), graphql_name="returnUrl", default=None)), - ( - "replacement_behavior", - sgqlc.types.Arg(AppSubscriptionReplacementBehavior, graphql_name="replacementBehavior", default="STANDARD"), - ), - ) - ), - ) - app_subscription_line_item_update = sgqlc.types.Field( - AppSubscriptionLineItemUpdatePayload, - graphql_name="appSubscriptionLineItemUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "capped_amount", - sgqlc.types.Arg(sgqlc.types.non_null(MoneyInput), graphql_name="cappedAmount", default=None), - ), - ) - ), - ) - app_subscription_trial_extend = sgqlc.types.Field( - AppSubscriptionTrialExtendPayload, - graphql_name="appSubscriptionTrialExtend", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("days", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="days", default=None)), - ) - ), - ) - app_usage_record_create = sgqlc.types.Field( - AppUsageRecordCreatePayload, - graphql_name="appUsageRecordCreate", - args=sgqlc.types.ArgDict( - ( - ( - "subscription_line_item_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="subscriptionLineItemId", default=None), - ), - ("price", sgqlc.types.Arg(sgqlc.types.non_null(MoneyInput), graphql_name="price", default=None)), - ( - "description", - sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="description", default=None), - ), - ) - ), - ) - bulk_operation_cancel = sgqlc.types.Field( - BulkOperationCancelPayload, - graphql_name="bulkOperationCancel", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - bulk_operation_run_mutation = sgqlc.types.Field( - BulkOperationRunMutationPayload, - graphql_name="bulkOperationRunMutation", - args=sgqlc.types.ArgDict( - ( - ("mutation", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="mutation", default=None)), - ( - "staged_upload_path", - sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="stagedUploadPath", default=None), - ), - ("client_identifier", sgqlc.types.Arg(String, graphql_name="clientIdentifier", default=None)), - ) - ), - ) - bulk_operation_run_query = sgqlc.types.Field( - BulkOperationRunQueryPayload, - graphql_name="bulkOperationRunQuery", - args=sgqlc.types.ArgDict((("query", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="query", default=None)),)), - ) - bulk_product_resource_feedback_create = sgqlc.types.Field( - BulkProductResourceFeedbackCreatePayload, - graphql_name="bulkProductResourceFeedbackCreate", - args=sgqlc.types.ArgDict( - ( - ( - "feedback_input", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProductResourceFeedbackInput))), - graphql_name="feedbackInput", - default=None, - ), - ), - ) - ), - ) - collection_add_products = sgqlc.types.Field( - CollectionAddProductsPayload, - graphql_name="collectionAddProducts", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "product_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="productIds", - default=None, - ), - ), - ) - ), - ) - collection_add_products_v2 = sgqlc.types.Field( - CollectionAddProductsV2Payload, - graphql_name="collectionAddProductsV2", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "product_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="productIds", - default=None, - ), - ), - ) - ), - ) - collection_create = sgqlc.types.Field( - CollectionCreatePayload, - graphql_name="collectionCreate", - args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(CollectionInput), graphql_name="input", default=None)),)), - ) - collection_delete = sgqlc.types.Field( - CollectionDeletePayload, - graphql_name="collectionDelete", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(CollectionDeleteInput), graphql_name="input", default=None), - ), - ) - ), - ) - collection_remove_products = sgqlc.types.Field( - CollectionRemoveProductsPayload, - graphql_name="collectionRemoveProducts", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "product_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="productIds", - default=None, - ), - ), - ) - ), - ) - collection_reorder_products = sgqlc.types.Field( - CollectionReorderProductsPayload, - graphql_name="collectionReorderProducts", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "moves", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MoveInput))), - graphql_name="moves", - default=None, - ), - ), - ) - ), - ) - collection_update = sgqlc.types.Field( - CollectionUpdatePayload, - graphql_name="collectionUpdate", - args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(CollectionInput), graphql_name="input", default=None)),)), - ) - companies_delete = sgqlc.types.Field( - CompaniesDeletePayload, - graphql_name="companiesDelete", - args=sgqlc.types.ArgDict( - ( - ( - "company_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="companyIds", - default=None, - ), - ), - ) - ), - ) - company_address_delete = sgqlc.types.Field( - CompanyAddressDeletePayload, - graphql_name="companyAddressDelete", - args=sgqlc.types.ArgDict((("address_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="addressId", default=None)),)), - ) - company_assign_customer_as_contact = sgqlc.types.Field( - CompanyAssignCustomerAsContactPayload, - graphql_name="companyAssignCustomerAsContact", - args=sgqlc.types.ArgDict( - ( - ("company_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyId", default=None)), - ("customer_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerId", default=None)), - ) - ), - ) - company_assign_main_contact = sgqlc.types.Field( - CompanyAssignMainContactPayload, - graphql_name="companyAssignMainContact", - args=sgqlc.types.ArgDict( - ( - ("company_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyId", default=None)), - ( - "company_contact_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyContactId", default=None), - ), - ) - ), - ) - company_contact_assign_role = sgqlc.types.Field( - CompanyContactAssignRolePayload, - graphql_name="companyContactAssignRole", - args=sgqlc.types.ArgDict( - ( - ( - "company_contact_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyContactId", default=None), - ), - ( - "company_contact_role_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyContactRoleId", default=None), - ), - ( - "company_location_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyLocationId", default=None), - ), - ) - ), - ) - company_contact_assign_roles = sgqlc.types.Field( - CompanyContactAssignRolesPayload, - graphql_name="companyContactAssignRoles", - args=sgqlc.types.ArgDict( - ( - ( - "company_contact_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyContactId", default=None), - ), - ( - "roles_to_assign", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CompanyContactRoleAssign))), - graphql_name="rolesToAssign", - default=None, - ), - ), - ) - ), - ) - company_contact_create = sgqlc.types.Field( - CompanyContactCreatePayload, - graphql_name="companyContactCreate", - args=sgqlc.types.ArgDict( - ( - ("company_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyId", default=None)), - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(CompanyContactInput), graphql_name="input", default=None), - ), - ) - ), - ) - company_contact_delete = sgqlc.types.Field( - CompanyContactDeletePayload, - graphql_name="companyContactDelete", - args=sgqlc.types.ArgDict( - ( - ( - "company_contact_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyContactId", default=None), - ), - ) - ), - ) - company_contact_revoke_role = sgqlc.types.Field( - CompanyContactRevokeRolePayload, - graphql_name="companyContactRevokeRole", - args=sgqlc.types.ArgDict( - ( - ( - "company_contact_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyContactId", default=None), - ), - ( - "company_contact_role_assignment_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyContactRoleAssignmentId", default=None), - ), - ) - ), - ) - company_contact_revoke_roles = sgqlc.types.Field( - CompanyContactRevokeRolesPayload, - graphql_name="companyContactRevokeRoles", - args=sgqlc.types.ArgDict( - ( - ( - "company_contact_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyContactId", default=None), - ), - ( - "role_assignment_ids", - sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="roleAssignmentIds", default=None), - ), - ("revoke_all", sgqlc.types.Arg(Boolean, graphql_name="revokeAll", default=False)), - ) - ), - ) - company_contact_update = sgqlc.types.Field( - CompanyContactUpdatePayload, - graphql_name="companyContactUpdate", - args=sgqlc.types.ArgDict( - ( - ( - "company_contact_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyContactId", default=None), - ), - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(CompanyContactInput), graphql_name="input", default=None), - ), - ) - ), - ) - company_contacts_delete = sgqlc.types.Field( - CompanyContactsDeletePayload, - graphql_name="companyContactsDelete", - args=sgqlc.types.ArgDict( - ( - ( - "company_contact_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="companyContactIds", - default=None, - ), - ), - ) - ), - ) - company_create = sgqlc.types.Field( - CompanyCreatePayload, - graphql_name="companyCreate", - args=sgqlc.types.ArgDict( - (("input", sgqlc.types.Arg(sgqlc.types.non_null(CompanyCreateInput), graphql_name="input", default=None)),) - ), - ) - company_delete = sgqlc.types.Field( - CompanyDeletePayload, - graphql_name="companyDelete", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - company_location_assign_address = sgqlc.types.Field( - CompanyLocationAssignAddressPayload, - graphql_name="companyLocationAssignAddress", - args=sgqlc.types.ArgDict( - ( - ("location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="locationId", default=None)), - ( - "address", - sgqlc.types.Arg(sgqlc.types.non_null(CompanyAddressInput), graphql_name="address", default=None), - ), - ( - "address_types", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CompanyAddressType))), - graphql_name="addressTypes", - default=None, - ), - ), - ) - ), - ) - company_location_assign_roles = sgqlc.types.Field( - CompanyLocationAssignRolesPayload, - graphql_name="companyLocationAssignRoles", - args=sgqlc.types.ArgDict( - ( - ( - "company_location_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyLocationId", default=None), - ), - ( - "roles_to_assign", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CompanyLocationRoleAssign))), - graphql_name="rolesToAssign", - default=None, - ), - ), - ) - ), - ) - company_location_assign_tax_exemptions = sgqlc.types.Field( - CompanyLocationAssignTaxExemptionsPayload, - graphql_name="companyLocationAssignTaxExemptions", - args=sgqlc.types.ArgDict( - ( - ( - "company_location_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyLocationId", default=None), - ), - ( - "tax_exemptions", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxExemption))), - graphql_name="taxExemptions", - default=None, - ), - ), - ) - ), - ) - company_location_create = sgqlc.types.Field( - CompanyLocationCreatePayload, - graphql_name="companyLocationCreate", - args=sgqlc.types.ArgDict( - ( - ("company_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyId", default=None)), - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(CompanyLocationInput), graphql_name="input", default=None), - ), - ) - ), - ) - company_location_create_tax_registration = sgqlc.types.Field( - CompanyLocationCreateTaxRegistrationPayload, - graphql_name="companyLocationCreateTaxRegistration", - args=sgqlc.types.ArgDict( - ( - ("location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="locationId", default=None)), - ("tax_id", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="taxId", default=None)), - ) - ), - ) - company_location_delete = sgqlc.types.Field( - CompanyLocationDeletePayload, - graphql_name="companyLocationDelete", - args=sgqlc.types.ArgDict( - ( - ( - "company_location_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyLocationId", default=None), - ), - ) - ), - ) - company_location_revoke_roles = sgqlc.types.Field( - CompanyLocationRevokeRolesPayload, - graphql_name="companyLocationRevokeRoles", - args=sgqlc.types.ArgDict( - ( - ( - "company_location_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyLocationId", default=None), - ), - ( - "roles_to_revoke", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="rolesToRevoke", - default=None, - ), - ), - ) - ), - ) - company_location_revoke_tax_exemptions = sgqlc.types.Field( - CompanyLocationRevokeTaxExemptionsPayload, - graphql_name="companyLocationRevokeTaxExemptions", - args=sgqlc.types.ArgDict( - ( - ( - "company_location_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyLocationId", default=None), - ), - ( - "tax_exemptions", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxExemption))), - graphql_name="taxExemptions", - default=None, - ), - ), - ) - ), - ) - company_location_revoke_tax_registration = sgqlc.types.Field( - CompanyLocationRevokeTaxRegistrationPayload, - graphql_name="companyLocationRevokeTaxRegistration", - args=sgqlc.types.ArgDict( - ( - ( - "company_location_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyLocationId", default=None), - ), - ) - ), - ) - company_location_update = sgqlc.types.Field( - CompanyLocationUpdatePayload, - graphql_name="companyLocationUpdate", - args=sgqlc.types.ArgDict( - ( - ( - "company_location_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyLocationId", default=None), - ), - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(CompanyLocationUpdateInput), graphql_name="input", default=None), - ), - ) - ), - ) - company_locations_delete = sgqlc.types.Field( - CompanyLocationsDeletePayload, - graphql_name="companyLocationsDelete", - args=sgqlc.types.ArgDict( - ( - ( - "company_location_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="companyLocationIds", - default=None, - ), - ), - ) - ), - ) - company_revoke_main_contact = sgqlc.types.Field( - CompanyRevokeMainContactPayload, - graphql_name="companyRevokeMainContact", - args=sgqlc.types.ArgDict((("company_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyId", default=None)),)), - ) - company_update = sgqlc.types.Field( - CompanyUpdatePayload, - graphql_name="companyUpdate", - args=sgqlc.types.ArgDict( - ( - ("company_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="companyId", default=None)), - ("input", sgqlc.types.Arg(sgqlc.types.non_null(CompanyInput), graphql_name="input", default=None)), - ) - ), - ) - customer_add_tax_exemptions = sgqlc.types.Field( - CustomerAddTaxExemptionsPayload, - graphql_name="customerAddTaxExemptions", - args=sgqlc.types.ArgDict( - ( - ("customer_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerId", default=None)), - ( - "tax_exemptions", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxExemption))), - graphql_name="taxExemptions", - default=None, - ), - ), - ) - ), - ) - customer_create = sgqlc.types.Field( - CustomerCreatePayload, - graphql_name="customerCreate", - args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(CustomerInput), graphql_name="input", default=None)),)), - ) - customer_delete = sgqlc.types.Field( - CustomerDeletePayload, - graphql_name="customerDelete", - args=sgqlc.types.ArgDict( - (("input", sgqlc.types.Arg(sgqlc.types.non_null(CustomerDeleteInput), graphql_name="input", default=None)),) - ), - ) - customer_email_marketing_consent_update = sgqlc.types.Field( - CustomerEmailMarketingConsentUpdatePayload, - graphql_name="customerEmailMarketingConsentUpdate", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg( - sgqlc.types.non_null(CustomerEmailMarketingConsentUpdateInput), - graphql_name="input", - default=None, - ), - ), - ) - ), - ) - customer_generate_account_activation_url = sgqlc.types.Field( - CustomerGenerateAccountActivationUrlPayload, - graphql_name="customerGenerateAccountActivationUrl", - args=sgqlc.types.ArgDict((("customer_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerId", default=None)),)), - ) - customer_payment_method_credit_card_create = sgqlc.types.Field( - CustomerPaymentMethodCreditCardCreatePayload, - graphql_name="customerPaymentMethodCreditCardCreate", - args=sgqlc.types.ArgDict( - ( - ("customer_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerId", default=None)), - ( - "billing_address", - sgqlc.types.Arg(sgqlc.types.non_null(MailingAddressInput), graphql_name="billingAddress", default=None), - ), - ("session_id", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="sessionId", default=None)), - ) - ), - ) - customer_payment_method_credit_card_update = sgqlc.types.Field( - CustomerPaymentMethodCreditCardUpdatePayload, - graphql_name="customerPaymentMethodCreditCardUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "billing_address", - sgqlc.types.Arg(sgqlc.types.non_null(MailingAddressInput), graphql_name="billingAddress", default=None), - ), - ("session_id", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="sessionId", default=None)), - ) - ), - ) - customer_payment_method_get_update_url = sgqlc.types.Field( - CustomerPaymentMethodGetUpdateUrlPayload, - graphql_name="customerPaymentMethodGetUpdateUrl", - args=sgqlc.types.ArgDict( - ( - ( - "customer_payment_method_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerPaymentMethodId", default=None), - ), - ) - ), - ) - customer_payment_method_paypal_billing_agreement_create = sgqlc.types.Field( - CustomerPaymentMethodPaypalBillingAgreementCreatePayload, - graphql_name="customerPaymentMethodPaypalBillingAgreementCreate", - args=sgqlc.types.ArgDict( - ( - ("customer_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerId", default=None)), - ("billing_address", sgqlc.types.Arg(MailingAddressInput, graphql_name="billingAddress", default=None)), - ( - "billing_agreement_id", - sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="billingAgreementId", default=None), - ), - ("inactive", sgqlc.types.Arg(Boolean, graphql_name="inactive", default=False)), - ) - ), - ) - customer_payment_method_paypal_billing_agreement_update = sgqlc.types.Field( - CustomerPaymentMethodPaypalBillingAgreementUpdatePayload, - graphql_name="customerPaymentMethodPaypalBillingAgreementUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "billing_address", - sgqlc.types.Arg(sgqlc.types.non_null(MailingAddressInput), graphql_name="billingAddress", default=None), - ), - ) - ), - ) - customer_payment_method_remote_create = sgqlc.types.Field( - CustomerPaymentMethodRemoteCreatePayload, - graphql_name="customerPaymentMethodRemoteCreate", - args=sgqlc.types.ArgDict( - ( - ("customer_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerId", default=None)), - ( - "remote_reference", - sgqlc.types.Arg( - sgqlc.types.non_null(CustomerPaymentMethodRemoteInput), - graphql_name="remoteReference", - default=None, - ), - ), - ) - ), - ) - customer_payment_method_revoke = sgqlc.types.Field( - CustomerPaymentMethodRevokePayload, - graphql_name="customerPaymentMethodRevoke", - args=sgqlc.types.ArgDict( - ( - ( - "customer_payment_method_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerPaymentMethodId", default=None), - ), - ) - ), - ) - customer_payment_method_send_update_email = sgqlc.types.Field( - CustomerPaymentMethodSendUpdateEmailPayload, - graphql_name="customerPaymentMethodSendUpdateEmail", - args=sgqlc.types.ArgDict( - ( - ( - "customer_payment_method_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerPaymentMethodId", default=None), - ), - ("email", sgqlc.types.Arg(EmailInput, graphql_name="email", default=None)), - ) - ), - ) - customer_remove_tax_exemptions = sgqlc.types.Field( - CustomerRemoveTaxExemptionsPayload, - graphql_name="customerRemoveTaxExemptions", - args=sgqlc.types.ArgDict( - ( - ("customer_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerId", default=None)), - ( - "tax_exemptions", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxExemption))), - graphql_name="taxExemptions", - default=None, - ), - ), - ) - ), - ) - customer_replace_tax_exemptions = sgqlc.types.Field( - CustomerReplaceTaxExemptionsPayload, - graphql_name="customerReplaceTaxExemptions", - args=sgqlc.types.ArgDict( - ( - ("customer_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerId", default=None)), - ( - "tax_exemptions", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxExemption))), - graphql_name="taxExemptions", - default=None, - ), - ), - ) - ), - ) - customer_sms_marketing_consent_update = sgqlc.types.Field( - CustomerSmsMarketingConsentUpdatePayload, - graphql_name="customerSmsMarketingConsentUpdate", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(CustomerSmsMarketingConsentUpdateInput), graphql_name="input", default=None), - ), - ) - ), - ) - customer_update = sgqlc.types.Field( - CustomerUpdatePayload, - graphql_name="customerUpdate", - args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(CustomerInput), graphql_name="input", default=None)),)), - ) - customer_update_default_address = sgqlc.types.Field( - CustomerUpdateDefaultAddressPayload, - graphql_name="customerUpdateDefaultAddress", - args=sgqlc.types.ArgDict( - ( - ("customer_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerId", default=None)), - ("address_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="addressId", default=None)), - ) - ), - ) - delegate_access_token_create = sgqlc.types.Field( - DelegateAccessTokenCreatePayload, - graphql_name="delegateAccessTokenCreate", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(DelegateAccessTokenInput), graphql_name="input", default=None), - ), - ) - ), - ) - delivery_profile_create = sgqlc.types.Field( - "deliveryProfileCreatePayload", - graphql_name="deliveryProfileCreate", - args=sgqlc.types.ArgDict( - ( - ( - "profile", - sgqlc.types.Arg(sgqlc.types.non_null(DeliveryProfileInput), graphql_name="profile", default=None), - ), - ) - ), - ) - delivery_profile_remove = sgqlc.types.Field( - "deliveryProfileRemovePayload", - graphql_name="deliveryProfileRemove", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - delivery_profile_update = sgqlc.types.Field( - "deliveryProfileUpdatePayload", - graphql_name="deliveryProfileUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "profile", - sgqlc.types.Arg(sgqlc.types.non_null(DeliveryProfileInput), graphql_name="profile", default=None), - ), - ( - "leave_legacy_mode_profiles", - sgqlc.types.Arg(Boolean, graphql_name="leaveLegacyModeProfiles", default=None), - ), - ) - ), - ) - delivery_setting_update = sgqlc.types.Field( - DeliverySettingUpdatePayload, - graphql_name="deliverySettingUpdate", - args=sgqlc.types.ArgDict( - ( - ( - "setting", - sgqlc.types.Arg(sgqlc.types.non_null(DeliverySettingInput), graphql_name="setting", default=None), - ), - ) - ), - ) - delivery_shipping_origin_assign = sgqlc.types.Field( - DeliveryShippingOriginAssignPayload, - graphql_name="deliveryShippingOriginAssign", - args=sgqlc.types.ArgDict((("location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="locationId", default=None)),)), - ) - discount_automatic_activate = sgqlc.types.Field( - DiscountAutomaticActivatePayload, - graphql_name="discountAutomaticActivate", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - discount_automatic_app_create = sgqlc.types.Field( - DiscountAutomaticAppCreatePayload, - graphql_name="discountAutomaticAppCreate", - args=sgqlc.types.ArgDict( - ( - ( - "automatic_app_discount", - sgqlc.types.Arg( - sgqlc.types.non_null(DiscountAutomaticAppInput), - graphql_name="automaticAppDiscount", - default=None, - ), - ), - ) - ), - ) - discount_automatic_app_update = sgqlc.types.Field( - DiscountAutomaticAppUpdatePayload, - graphql_name="discountAutomaticAppUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "automatic_app_discount", - sgqlc.types.Arg( - sgqlc.types.non_null(DiscountAutomaticAppInput), - graphql_name="automaticAppDiscount", - default=None, - ), - ), - ) - ), - ) - discount_automatic_basic_create = sgqlc.types.Field( - DiscountAutomaticBasicCreatePayload, - graphql_name="discountAutomaticBasicCreate", - args=sgqlc.types.ArgDict( - ( - ( - "automatic_basic_discount", - sgqlc.types.Arg( - sgqlc.types.non_null(DiscountAutomaticBasicInput), - graphql_name="automaticBasicDiscount", - default=None, - ), - ), - ) - ), - ) - discount_automatic_basic_update = sgqlc.types.Field( - DiscountAutomaticBasicUpdatePayload, - graphql_name="discountAutomaticBasicUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "automatic_basic_discount", - sgqlc.types.Arg( - sgqlc.types.non_null(DiscountAutomaticBasicInput), - graphql_name="automaticBasicDiscount", - default=None, - ), - ), - ) - ), - ) - discount_automatic_bulk_delete = sgqlc.types.Field( - DiscountAutomaticBulkDeletePayload, - graphql_name="discountAutomaticBulkDelete", - args=sgqlc.types.ArgDict( - ( - ("search", sgqlc.types.Arg(String, graphql_name="search", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ( - "ids", - sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="ids", default=None), - ), - ) - ), - ) - discount_automatic_bxgy_create = sgqlc.types.Field( - DiscountAutomaticBxgyCreatePayload, - graphql_name="discountAutomaticBxgyCreate", - args=sgqlc.types.ArgDict( - ( - ( - "automatic_bxgy_discount", - sgqlc.types.Arg( - sgqlc.types.non_null(DiscountAutomaticBxgyInput), - graphql_name="automaticBxgyDiscount", - default=None, - ), - ), - ) - ), - ) - discount_automatic_bxgy_update = sgqlc.types.Field( - DiscountAutomaticBxgyUpdatePayload, - graphql_name="discountAutomaticBxgyUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "automatic_bxgy_discount", - sgqlc.types.Arg( - sgqlc.types.non_null(DiscountAutomaticBxgyInput), - graphql_name="automaticBxgyDiscount", - default=None, - ), - ), - ) - ), - ) - discount_automatic_deactivate = sgqlc.types.Field( - DiscountAutomaticDeactivatePayload, - graphql_name="discountAutomaticDeactivate", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - discount_automatic_delete = sgqlc.types.Field( - DiscountAutomaticDeletePayload, - graphql_name="discountAutomaticDelete", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - discount_code_activate = sgqlc.types.Field( - DiscountCodeActivatePayload, - graphql_name="discountCodeActivate", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - discount_code_app_create = sgqlc.types.Field( - DiscountCodeAppCreatePayload, - graphql_name="discountCodeAppCreate", - args=sgqlc.types.ArgDict( - ( - ( - "code_app_discount", - sgqlc.types.Arg(sgqlc.types.non_null(DiscountCodeAppInput), graphql_name="codeAppDiscount", default=None), - ), - ) - ), - ) - discount_code_app_update = sgqlc.types.Field( - DiscountCodeAppUpdatePayload, - graphql_name="discountCodeAppUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "code_app_discount", - sgqlc.types.Arg(sgqlc.types.non_null(DiscountCodeAppInput), graphql_name="codeAppDiscount", default=None), - ), - ) - ), - ) - discount_code_basic_create = sgqlc.types.Field( - DiscountCodeBasicCreatePayload, - graphql_name="discountCodeBasicCreate", - args=sgqlc.types.ArgDict( - ( - ( - "basic_code_discount", - sgqlc.types.Arg(sgqlc.types.non_null(DiscountCodeBasicInput), graphql_name="basicCodeDiscount", default=None), - ), - ) - ), - ) - discount_code_basic_update = sgqlc.types.Field( - DiscountCodeBasicUpdatePayload, - graphql_name="discountCodeBasicUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "basic_code_discount", - sgqlc.types.Arg(sgqlc.types.non_null(DiscountCodeBasicInput), graphql_name="basicCodeDiscount", default=None), - ), - ) - ), - ) - discount_code_bulk_activate = sgqlc.types.Field( - DiscountCodeBulkActivatePayload, - graphql_name="discountCodeBulkActivate", - args=sgqlc.types.ArgDict( - ( - ("search", sgqlc.types.Arg(String, graphql_name="search", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ( - "ids", - sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="ids", default=None), - ), - ) - ), - ) - discount_code_bulk_deactivate = sgqlc.types.Field( - DiscountCodeBulkDeactivatePayload, - graphql_name="discountCodeBulkDeactivate", - args=sgqlc.types.ArgDict( - ( - ("search", sgqlc.types.Arg(String, graphql_name="search", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ( - "ids", - sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="ids", default=None), - ), - ) - ), - ) - discount_code_bulk_delete = sgqlc.types.Field( - DiscountCodeBulkDeletePayload, - graphql_name="discountCodeBulkDelete", - args=sgqlc.types.ArgDict( - ( - ("search", sgqlc.types.Arg(String, graphql_name="search", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ( - "ids", - sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="ids", default=None), - ), - ) - ), - ) - discount_code_bxgy_create = sgqlc.types.Field( - DiscountCodeBxgyCreatePayload, - graphql_name="discountCodeBxgyCreate", - args=sgqlc.types.ArgDict( - ( - ( - "bxgy_code_discount", - sgqlc.types.Arg(sgqlc.types.non_null(DiscountCodeBxgyInput), graphql_name="bxgyCodeDiscount", default=None), - ), - ) - ), - ) - discount_code_bxgy_update = sgqlc.types.Field( - DiscountCodeBxgyUpdatePayload, - graphql_name="discountCodeBxgyUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "bxgy_code_discount", - sgqlc.types.Arg(sgqlc.types.non_null(DiscountCodeBxgyInput), graphql_name="bxgyCodeDiscount", default=None), - ), - ) - ), - ) - discount_code_deactivate = sgqlc.types.Field( - DiscountCodeDeactivatePayload, - graphql_name="discountCodeDeactivate", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - discount_code_delete = sgqlc.types.Field( - DiscountCodeDeletePayload, - graphql_name="discountCodeDelete", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - discount_code_free_shipping_create = sgqlc.types.Field( - DiscountCodeFreeShippingCreatePayload, - graphql_name="discountCodeFreeShippingCreate", - args=sgqlc.types.ArgDict( - ( - ( - "free_shipping_code_discount", - sgqlc.types.Arg( - sgqlc.types.non_null(DiscountCodeFreeShippingInput), - graphql_name="freeShippingCodeDiscount", - default=None, - ), - ), - ) - ), - ) - discount_code_free_shipping_update = sgqlc.types.Field( - DiscountCodeFreeShippingUpdatePayload, - graphql_name="discountCodeFreeShippingUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "free_shipping_code_discount", - sgqlc.types.Arg( - sgqlc.types.non_null(DiscountCodeFreeShippingInput), - graphql_name="freeShippingCodeDiscount", - default=None, - ), - ), - ) - ), - ) - discount_code_redeem_code_bulk_delete = sgqlc.types.Field( - DiscountCodeRedeemCodeBulkDeletePayload, - graphql_name="discountCodeRedeemCodeBulkDelete", - args=sgqlc.types.ArgDict( - ( - ("discount_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="discountId", default=None)), - ("search", sgqlc.types.Arg(String, graphql_name="search", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ( - "ids", - sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="ids", default=None), - ), - ) - ), - ) - discount_redeem_code_bulk_add = sgqlc.types.Field( - DiscountRedeemCodeBulkAddPayload, - graphql_name="discountRedeemCodeBulkAdd", - args=sgqlc.types.ArgDict( - ( - ("discount_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="discountId", default=None)), - ( - "codes", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DiscountRedeemCodeInput))), - graphql_name="codes", - default=None, - ), - ), - ) - ), - ) - dispute_evidence_update = sgqlc.types.Field( - DisputeEvidenceUpdatePayload, - graphql_name="disputeEvidenceUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "input", - sgqlc.types.Arg( - sgqlc.types.non_null(ShopifyPaymentsDisputeEvidenceUpdateInput), - graphql_name="input", - default=None, - ), - ), - ) - ), - ) - draft_order_bulk_add_tags = sgqlc.types.Field( - DraftOrderBulkAddTagsPayload, - graphql_name="draftOrderBulkAddTags", - args=sgqlc.types.ArgDict( - ( - ("search", sgqlc.types.Arg(String, graphql_name="search", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ( - "ids", - sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="ids", default=None), - ), - ( - "tags", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), - graphql_name="tags", - default=None, - ), - ), - ) - ), - ) - draft_order_bulk_delete = sgqlc.types.Field( - DraftOrderBulkDeletePayload, - graphql_name="draftOrderBulkDelete", - args=sgqlc.types.ArgDict( - ( - ("search", sgqlc.types.Arg(String, graphql_name="search", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ( - "ids", - sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="ids", default=None), - ), - ) - ), - ) - draft_order_bulk_remove_tags = sgqlc.types.Field( - DraftOrderBulkRemoveTagsPayload, - graphql_name="draftOrderBulkRemoveTags", - args=sgqlc.types.ArgDict( - ( - ("search", sgqlc.types.Arg(String, graphql_name="search", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ( - "ids", - sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="ids", default=None), - ), - ( - "tags", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), - graphql_name="tags", - default=None, - ), - ), - ) - ), - ) - draft_order_calculate = sgqlc.types.Field( - DraftOrderCalculatePayload, - graphql_name="draftOrderCalculate", - args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(DraftOrderInput), graphql_name="input", default=None)),)), - ) - draft_order_complete = sgqlc.types.Field( - DraftOrderCompletePayload, - graphql_name="draftOrderComplete", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("payment_pending", sgqlc.types.Arg(Boolean, graphql_name="paymentPending", default=False)), - ("payment_gateway_id", sgqlc.types.Arg(ID, graphql_name="paymentGatewayId", default=None)), - ("source_name", sgqlc.types.Arg(String, graphql_name="sourceName", default=None)), - ) - ), - ) - draft_order_create = sgqlc.types.Field( - DraftOrderCreatePayload, - graphql_name="draftOrderCreate", - args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(DraftOrderInput), graphql_name="input", default=None)),)), - ) - draft_order_create_from_order = sgqlc.types.Field( - DraftOrderCreateFromOrderPayload, - graphql_name="draftOrderCreateFromOrder", - args=sgqlc.types.ArgDict((("order_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="orderId", default=None)),)), - ) - draft_order_create_merchant_checkout = sgqlc.types.Field( - DraftOrderCreateMerchantCheckoutPayload, - graphql_name="draftOrderCreateMerchantCheckout", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - draft_order_delete = sgqlc.types.Field( - DraftOrderDeletePayload, - graphql_name="draftOrderDelete", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(DraftOrderDeleteInput), graphql_name="input", default=None), - ), - ) - ), - ) - draft_order_duplicate = sgqlc.types.Field( - DraftOrderDuplicatePayload, - graphql_name="draftOrderDuplicate", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(ID, graphql_name="id", default=None)),)), - ) - draft_order_invoice_preview = sgqlc.types.Field( - DraftOrderInvoicePreviewPayload, - graphql_name="draftOrderInvoicePreview", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("email", sgqlc.types.Arg(EmailInput, graphql_name="email", default=None)), - ) - ), - ) - draft_order_invoice_send = sgqlc.types.Field( - DraftOrderInvoiceSendPayload, - graphql_name="draftOrderInvoiceSend", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("email", sgqlc.types.Arg(EmailInput, graphql_name="email", default=None)), - ) - ), - ) - draft_order_update = sgqlc.types.Field( - DraftOrderUpdatePayload, - graphql_name="draftOrderUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("input", sgqlc.types.Arg(sgqlc.types.non_null(DraftOrderInput), graphql_name="input", default=None)), - ) - ), - ) - event_bridge_webhook_subscription_create = sgqlc.types.Field( - EventBridgeWebhookSubscriptionCreatePayload, - graphql_name="eventBridgeWebhookSubscriptionCreate", - args=sgqlc.types.ArgDict( - ( - ( - "topic", - sgqlc.types.Arg(sgqlc.types.non_null(WebhookSubscriptionTopic), graphql_name="topic", default=None), - ), - ( - "webhook_subscription", - sgqlc.types.Arg( - sgqlc.types.non_null(EventBridgeWebhookSubscriptionInput), - graphql_name="webhookSubscription", - default=None, - ), - ), - ) - ), - ) - event_bridge_webhook_subscription_update = sgqlc.types.Field( - EventBridgeWebhookSubscriptionUpdatePayload, - graphql_name="eventBridgeWebhookSubscriptionUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "webhook_subscription", - sgqlc.types.Arg( - sgqlc.types.non_null(EventBridgeWebhookSubscriptionInput), - graphql_name="webhookSubscription", - default=None, - ), - ), - ) - ), - ) - file_create = sgqlc.types.Field( - FileCreatePayload, - graphql_name="fileCreate", - args=sgqlc.types.ArgDict( - ( - ( - "files", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FileCreateInput))), - graphql_name="files", - default=None, - ), - ), - ) - ), - ) - file_delete = sgqlc.types.Field( - FileDeletePayload, - graphql_name="fileDelete", - args=sgqlc.types.ArgDict( - ( - ( - "file_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="fileIds", - default=None, - ), - ), - ) - ), - ) - file_update = sgqlc.types.Field( - FileUpdatePayload, - graphql_name="fileUpdate", - args=sgqlc.types.ArgDict( - ( - ( - "files", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FileUpdateInput))), - graphql_name="files", - default=None, - ), - ), - ) - ), - ) - flow_trigger_receive = sgqlc.types.Field( - FlowTriggerReceivePayload, - graphql_name="flowTriggerReceive", - args=sgqlc.types.ArgDict((("body", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="body", default=None)),)), - ) - fulfillment_cancel = sgqlc.types.Field( - FulfillmentCancelPayload, - graphql_name="fulfillmentCancel", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - fulfillment_create_v2 = sgqlc.types.Field( - FulfillmentCreateV2Payload, - graphql_name="fulfillmentCreateV2", - args=sgqlc.types.ArgDict( - ( - ( - "fulfillment", - sgqlc.types.Arg(sgqlc.types.non_null(FulfillmentV2Input), graphql_name="fulfillment", default=None), - ), - ("message", sgqlc.types.Arg(String, graphql_name="message", default=None)), - ) - ), - ) - fulfillment_order_accept_cancellation_request = sgqlc.types.Field( - FulfillmentOrderAcceptCancellationRequestPayload, - graphql_name="fulfillmentOrderAcceptCancellationRequest", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("message", sgqlc.types.Arg(String, graphql_name="message", default=None)), - ) - ), - ) - fulfillment_order_accept_fulfillment_request = sgqlc.types.Field( - FulfillmentOrderAcceptFulfillmentRequestPayload, - graphql_name="fulfillmentOrderAcceptFulfillmentRequest", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("message", sgqlc.types.Arg(String, graphql_name="message", default=None)), - ) - ), - ) - fulfillment_order_cancel = sgqlc.types.Field( - FulfillmentOrderCancelPayload, - graphql_name="fulfillmentOrderCancel", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - fulfillment_order_close = sgqlc.types.Field( - FulfillmentOrderClosePayload, - graphql_name="fulfillmentOrderClose", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("message", sgqlc.types.Arg(String, graphql_name="message", default=None)), - ) - ), - ) - fulfillment_order_hold = sgqlc.types.Field( - FulfillmentOrderHoldPayload, - graphql_name="fulfillmentOrderHold", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "fulfillment_hold", - sgqlc.types.Arg(sgqlc.types.non_null(FulfillmentOrderHoldInput), graphql_name="fulfillmentHold", default=None), - ), - ) - ), - ) - fulfillment_order_move = sgqlc.types.Field( - FulfillmentOrderMovePayload, - graphql_name="fulfillmentOrderMove", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "new_location_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="newLocationId", default=None), - ), - ) - ), - ) - fulfillment_order_open = sgqlc.types.Field( - FulfillmentOrderOpenPayload, - graphql_name="fulfillmentOrderOpen", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - fulfillment_order_reject_cancellation_request = sgqlc.types.Field( - FulfillmentOrderRejectCancellationRequestPayload, - graphql_name="fulfillmentOrderRejectCancellationRequest", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("message", sgqlc.types.Arg(String, graphql_name="message", default=None)), - ) - ), - ) - fulfillment_order_reject_fulfillment_request = sgqlc.types.Field( - FulfillmentOrderRejectFulfillmentRequestPayload, - graphql_name="fulfillmentOrderRejectFulfillmentRequest", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("reason", sgqlc.types.Arg(FulfillmentOrderRejectionReason, graphql_name="reason", default=None)), - ("message", sgqlc.types.Arg(String, graphql_name="message", default=None)), - ( - "line_items", - sgqlc.types.Arg( - sgqlc.types.list_of(sgqlc.types.non_null(IncomingRequestLineItemInput)), - graphql_name="lineItems", - default=None, - ), - ), - ) - ), - ) - fulfillment_order_release_hold = sgqlc.types.Field( - FulfillmentOrderReleaseHoldPayload, - graphql_name="fulfillmentOrderReleaseHold", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - fulfillment_order_reschedule = sgqlc.types.Field( - FulfillmentOrderReschedulePayload, - graphql_name="fulfillmentOrderReschedule", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("fulfill_at", sgqlc.types.Arg(sgqlc.types.non_null(DateTime), graphql_name="fulfillAt", default=None)), - ) - ), - ) - fulfillment_order_submit_cancellation_request = sgqlc.types.Field( - FulfillmentOrderSubmitCancellationRequestPayload, - graphql_name="fulfillmentOrderSubmitCancellationRequest", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("message", sgqlc.types.Arg(String, graphql_name="message", default=None)), - ) - ), - ) - fulfillment_order_submit_fulfillment_request = sgqlc.types.Field( - FulfillmentOrderSubmitFulfillmentRequestPayload, - graphql_name="fulfillmentOrderSubmitFulfillmentRequest", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("message", sgqlc.types.Arg(String, graphql_name="message", default=None)), - ("notify_customer", sgqlc.types.Arg(Boolean, graphql_name="notifyCustomer", default=None)), - ( - "fulfillment_order_line_items", - sgqlc.types.Arg( - sgqlc.types.list_of(sgqlc.types.non_null(FulfillmentOrderLineItemInput)), - graphql_name="fulfillmentOrderLineItems", - default=None, - ), - ), - ("shipping_method", sgqlc.types.Arg(String, graphql_name="shippingMethod", default=None)), - ) - ), - ) - fulfillment_orders_set_fulfillment_deadline = sgqlc.types.Field( - FulfillmentOrdersSetFulfillmentDeadlinePayload, - graphql_name="fulfillmentOrdersSetFulfillmentDeadline", - args=sgqlc.types.ArgDict( - ( - ( - "fulfillment_order_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="fulfillmentOrderIds", - default=None, - ), - ), - ( - "fulfillment_deadline", - sgqlc.types.Arg(sgqlc.types.non_null(DateTime), graphql_name="fulfillmentDeadline", default=None), - ), - ) - ), - ) - fulfillment_service_create = sgqlc.types.Field( - FulfillmentServiceCreatePayload, - graphql_name="fulfillmentServiceCreate", - args=sgqlc.types.ArgDict( - ( - ("name", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="name", default=None)), - ("callback_url", sgqlc.types.Arg(URL, graphql_name="callbackUrl", default=None)), - ("tracking_support", sgqlc.types.Arg(Boolean, graphql_name="trackingSupport", default=False)), - ( - "fulfillment_orders_opt_in", - sgqlc.types.Arg(Boolean, graphql_name="fulfillmentOrdersOptIn", default=False), - ), - ("permits_sku_sharing", sgqlc.types.Arg(Boolean, graphql_name="permitsSkuSharing", default=False)), - ("inventory_management", sgqlc.types.Arg(Boolean, graphql_name="inventoryManagement", default=False)), - ) - ), - ) - fulfillment_service_delete = sgqlc.types.Field( - FulfillmentServiceDeletePayload, - graphql_name="fulfillmentServiceDelete", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("destination_location_id", sgqlc.types.Arg(ID, graphql_name="destinationLocationId", default=None)), - ) - ), - ) - fulfillment_service_update = sgqlc.types.Field( - FulfillmentServiceUpdatePayload, - graphql_name="fulfillmentServiceUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("name", sgqlc.types.Arg(String, graphql_name="name", default=None)), - ("callback_url", sgqlc.types.Arg(URL, graphql_name="callbackUrl", default=None)), - ("tracking_support", sgqlc.types.Arg(Boolean, graphql_name="trackingSupport", default=None)), - ( - "fulfillment_orders_opt_in", - sgqlc.types.Arg(Boolean, graphql_name="fulfillmentOrdersOptIn", default=None), - ), - ("permits_sku_sharing", sgqlc.types.Arg(Boolean, graphql_name="permitsSkuSharing", default=None)), - ) - ), - ) - fulfillment_tracking_info_update_v2 = sgqlc.types.Field( - FulfillmentTrackingInfoUpdateV2Payload, - graphql_name="fulfillmentTrackingInfoUpdateV2", - args=sgqlc.types.ArgDict( - ( - ( - "fulfillment_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="fulfillmentId", default=None), - ), - ( - "tracking_info_input", - sgqlc.types.Arg(sgqlc.types.non_null(FulfillmentTrackingInput), graphql_name="trackingInfoInput", default=None), - ), - ("notify_customer", sgqlc.types.Arg(Boolean, graphql_name="notifyCustomer", default=None)), - ) - ), - ) - gift_card_create = sgqlc.types.Field( - GiftCardCreatePayload, - graphql_name="giftCardCreate", - args=sgqlc.types.ArgDict( - (("input", sgqlc.types.Arg(sgqlc.types.non_null(GiftCardCreateInput), graphql_name="input", default=None)),) - ), - ) - gift_card_disable = sgqlc.types.Field( - GiftCardDisablePayload, - graphql_name="giftCardDisable", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - gift_card_update = sgqlc.types.Field( - GiftCardUpdatePayload, - graphql_name="giftCardUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(GiftCardUpdateInput), graphql_name="input", default=None), - ), - ) - ), - ) - inventory_activate = sgqlc.types.Field( - InventoryActivatePayload, - graphql_name="inventoryActivate", - args=sgqlc.types.ArgDict( - ( - ( - "inventory_item_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="inventoryItemId", default=None), - ), - ("location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="locationId", default=None)), - ("available", sgqlc.types.Arg(Int, graphql_name="available", default=None)), - ) - ), - ) - inventory_adjust_quantity = sgqlc.types.Field( - InventoryAdjustQuantityPayload, - graphql_name="inventoryAdjustQuantity", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(InventoryAdjustQuantityInput), graphql_name="input", default=None), - ), - ) - ), - ) - inventory_bulk_adjust_quantity_at_location = sgqlc.types.Field( - InventoryBulkAdjustQuantityAtLocationPayload, - graphql_name="inventoryBulkAdjustQuantityAtLocation", - args=sgqlc.types.ArgDict( - ( - ( - "inventory_item_adjustments", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(InventoryAdjustItemInput))), - graphql_name="inventoryItemAdjustments", - default=None, - ), - ), - ("location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="locationId", default=None)), - ) - ), - ) - inventory_bulk_toggle_activation = sgqlc.types.Field( - InventoryBulkToggleActivationPayload, - graphql_name="inventoryBulkToggleActivation", - args=sgqlc.types.ArgDict( - ( - ( - "inventory_item_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="inventoryItemId", default=None), - ), - ( - "inventory_item_updates", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(InventoryBulkToggleActivationInput))), - graphql_name="inventoryItemUpdates", - default=None, - ), - ), - ) - ), - ) - inventory_deactivate = sgqlc.types.Field( - InventoryDeactivatePayload, - graphql_name="inventoryDeactivate", - args=sgqlc.types.ArgDict( - ( - ( - "inventory_level_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="inventoryLevelId", default=None), - ), - ) - ), - ) - inventory_item_update = sgqlc.types.Field( - InventoryItemUpdatePayload, - graphql_name="inventoryItemUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(InventoryItemUpdateInput), graphql_name="input", default=None), - ), - ) - ), - ) - location_activate = sgqlc.types.Field( - LocationActivatePayload, - graphql_name="locationActivate", - args=sgqlc.types.ArgDict((("location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="locationId", default=None)),)), - ) - location_add = sgqlc.types.Field( - LocationAddPayload, - graphql_name="locationAdd", - args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(LocationAddInput), graphql_name="input", default=None)),)), - ) - location_deactivate = sgqlc.types.Field( - LocationDeactivatePayload, - graphql_name="locationDeactivate", - args=sgqlc.types.ArgDict( - ( - ("location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="locationId", default=None)), - ("destination_location_id", sgqlc.types.Arg(ID, graphql_name="destinationLocationId", default=None)), - ) - ), - ) - location_delete = sgqlc.types.Field( - LocationDeletePayload, - graphql_name="locationDelete", - args=sgqlc.types.ArgDict((("location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="locationId", default=None)),)), - ) - location_edit = sgqlc.types.Field( - LocationEditPayload, - graphql_name="locationEdit", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("input", sgqlc.types.Arg(sgqlc.types.non_null(LocationEditInput), graphql_name="input", default=None)), - ) - ), - ) - market_create = sgqlc.types.Field( - MarketCreatePayload, - graphql_name="marketCreate", - args=sgqlc.types.ArgDict( - (("input", sgqlc.types.Arg(sgqlc.types.non_null(MarketCreateInput), graphql_name="input", default=None)),) - ), - ) - market_currency_settings_update = sgqlc.types.Field( - MarketCurrencySettingsUpdatePayload, - graphql_name="marketCurrencySettingsUpdate", - args=sgqlc.types.ArgDict( - ( - ("market_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="marketId", default=None)), - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(MarketCurrencySettingsUpdateInput), graphql_name="input", default=None), - ), - ) - ), - ) - market_delete = sgqlc.types.Field( - MarketDeletePayload, - graphql_name="marketDelete", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - market_localizations_register = sgqlc.types.Field( - MarketLocalizationsRegisterPayload, - graphql_name="marketLocalizationsRegister", - args=sgqlc.types.ArgDict( - ( - ("resource_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="resourceId", default=None)), - ( - "market_localizations", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MarketLocalizationRegisterInput))), - graphql_name="marketLocalizations", - default=None, - ), - ), - ) - ), - ) - market_localizations_remove = sgqlc.types.Field( - MarketLocalizationsRemovePayload, - graphql_name="marketLocalizationsRemove", - args=sgqlc.types.ArgDict( - ( - ("resource_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="resourceId", default=None)), - ( - "market_localization_keys", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), - graphql_name="marketLocalizationKeys", - default=None, - ), - ), - ( - "market_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="marketIds", - default=None, - ), - ), - ) - ), - ) - market_region_delete = sgqlc.types.Field( - MarketRegionDeletePayload, - graphql_name="marketRegionDelete", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - market_regions_create = sgqlc.types.Field( - MarketRegionsCreatePayload, - graphql_name="marketRegionsCreate", - args=sgqlc.types.ArgDict( - ( - ("market_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="marketId", default=None)), - ( - "regions", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MarketRegionCreateInput))), - graphql_name="regions", - default=None, - ), - ), - ) - ), - ) - market_update = sgqlc.types.Field( - MarketUpdatePayload, - graphql_name="marketUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("input", sgqlc.types.Arg(sgqlc.types.non_null(MarketUpdateInput), graphql_name="input", default=None)), - ) - ), - ) - market_web_presence_create = sgqlc.types.Field( - MarketWebPresenceCreatePayload, - graphql_name="marketWebPresenceCreate", - args=sgqlc.types.ArgDict( - ( - ("market_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="marketId", default=None)), - ( - "web_presence", - sgqlc.types.Arg(sgqlc.types.non_null(MarketWebPresenceCreateInput), graphql_name="webPresence", default=None), - ), - ) - ), - ) - market_web_presence_delete = sgqlc.types.Field( - MarketWebPresenceDeletePayload, - graphql_name="marketWebPresenceDelete", - args=sgqlc.types.ArgDict((("market_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="marketId", default=None)),)), - ) - market_web_presence_update = sgqlc.types.Field( - MarketWebPresenceUpdatePayload, - graphql_name="marketWebPresenceUpdate", - args=sgqlc.types.ArgDict( - ( - ("market_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="marketId", default=None)), - ( - "web_presence", - sgqlc.types.Arg(sgqlc.types.non_null(MarketWebPresenceUpdateInput), graphql_name="webPresence", default=None), - ), - ) - ), - ) - marketing_activity_create = sgqlc.types.Field( - MarketingActivityCreatePayload, - graphql_name="marketingActivityCreate", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(MarketingActivityCreateInput), graphql_name="input", default=None), - ), - ) - ), - ) - marketing_activity_create_external = sgqlc.types.Field( - MarketingActivityCreateExternalPayload, - graphql_name="marketingActivityCreateExternal", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(MarketingActivityCreateExternalInput), graphql_name="input", default=None), - ), - ) - ), - ) - marketing_activity_update = sgqlc.types.Field( - MarketingActivityUpdatePayload, - graphql_name="marketingActivityUpdate", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(MarketingActivityUpdateInput), graphql_name="input", default=None), - ), - ) - ), - ) - marketing_activity_update_external = sgqlc.types.Field( - MarketingActivityUpdateExternalPayload, - graphql_name="marketingActivityUpdateExternal", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(MarketingActivityUpdateExternalInput), graphql_name="input", default=None), - ), - ("marketing_activity_id", sgqlc.types.Arg(ID, graphql_name="marketingActivityId", default=None)), - ("remote_id", sgqlc.types.Arg(String, graphql_name="remoteId", default=None)), - ("utm", sgqlc.types.Arg(UTMInput, graphql_name="utm", default=None)), - ) - ), - ) - marketing_engagement_create = sgqlc.types.Field( - MarketingEngagementCreatePayload, - graphql_name="marketingEngagementCreate", - args=sgqlc.types.ArgDict( - ( - ( - "marketing_activity_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="marketingActivityId", default=None), - ), - ( - "marketing_engagement", - sgqlc.types.Arg(sgqlc.types.non_null(MarketingEngagementInput), graphql_name="marketingEngagement", default=None), - ), - ) - ), - ) - metafield_definition_create = sgqlc.types.Field( - MetafieldDefinitionCreatePayload, - graphql_name="metafieldDefinitionCreate", - args=sgqlc.types.ArgDict( - ( - ( - "definition", - sgqlc.types.Arg(sgqlc.types.non_null(MetafieldDefinitionInput), graphql_name="definition", default=None), - ), - ) - ), - ) - metafield_definition_delete = sgqlc.types.Field( - MetafieldDefinitionDeletePayload, - graphql_name="metafieldDefinitionDelete", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "delete_all_associated_metafields", - sgqlc.types.Arg(Boolean, graphql_name="deleteAllAssociatedMetafields", default=False), - ), - ) - ), - ) - metafield_definition_pin = sgqlc.types.Field( - MetafieldDefinitionPinPayload, - graphql_name="metafieldDefinitionPin", - args=sgqlc.types.ArgDict( - (("definition_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="definitionId", default=None)),) - ), - ) - metafield_definition_unpin = sgqlc.types.Field( - MetafieldDefinitionUnpinPayload, - graphql_name="metafieldDefinitionUnpin", - args=sgqlc.types.ArgDict( - (("definition_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="definitionId", default=None)),) - ), - ) - metafield_definition_update = sgqlc.types.Field( - MetafieldDefinitionUpdatePayload, - graphql_name="metafieldDefinitionUpdate", - args=sgqlc.types.ArgDict( - ( - ( - "definition", - sgqlc.types.Arg(sgqlc.types.non_null(MetafieldDefinitionUpdateInput), graphql_name="definition", default=None), - ), - ) - ), - ) - metafield_delete = sgqlc.types.Field( - MetafieldDeletePayload, - graphql_name="metafieldDelete", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(MetafieldDeleteInput), graphql_name="input", default=None), - ), - ) - ), - ) - metafield_storefront_visibility_create = sgqlc.types.Field( - MetafieldStorefrontVisibilityCreatePayload, - graphql_name="metafieldStorefrontVisibilityCreate", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(MetafieldStorefrontVisibilityInput), graphql_name="input", default=None), - ), - ) - ), - ) - metafield_storefront_visibility_delete = sgqlc.types.Field( - MetafieldStorefrontVisibilityDeletePayload, - graphql_name="metafieldStorefrontVisibilityDelete", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - metafields_set = sgqlc.types.Field( - MetafieldsSetPayload, - graphql_name="metafieldsSet", - args=sgqlc.types.ArgDict( - ( - ( - "metafields", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MetafieldsSetInput))), - graphql_name="metafields", - default=None, - ), - ), - ) - ), - ) - order_capture = sgqlc.types.Field( - "OrderCapturePayload", - graphql_name="orderCapture", - args=sgqlc.types.ArgDict( - (("input", sgqlc.types.Arg(sgqlc.types.non_null(OrderCaptureInput), graphql_name="input", default=None)),) - ), - ) - order_close = sgqlc.types.Field( - "OrderClosePayload", - graphql_name="orderClose", - args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(OrderCloseInput), graphql_name="input", default=None)),)), - ) - order_create_mandate_payment = sgqlc.types.Field( - "OrderCreateMandatePaymentPayload", - graphql_name="orderCreateMandatePayment", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("payment_schedule_id", sgqlc.types.Arg(ID, graphql_name="paymentScheduleId", default=None)), - ( - "idempotency_key", - sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="idempotencyKey", default=None), - ), - ("mandate_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="mandateId", default=None)), - ("auto_capture", sgqlc.types.Arg(Boolean, graphql_name="autoCapture", default=True)), - ) - ), - ) - order_edit_add_custom_item = sgqlc.types.Field( - "OrderEditAddCustomItemPayload", - graphql_name="orderEditAddCustomItem", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("title", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="title", default=None)), - ("location_id", sgqlc.types.Arg(ID, graphql_name="locationId", default=None)), - ("price", sgqlc.types.Arg(sgqlc.types.non_null(MoneyInput), graphql_name="price", default=None)), - ("quantity", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="quantity", default=None)), - ("taxable", sgqlc.types.Arg(Boolean, graphql_name="taxable", default=None)), - ("requires_shipping", sgqlc.types.Arg(Boolean, graphql_name="requiresShipping", default=None)), - ) - ), - ) - order_edit_add_line_item_discount = sgqlc.types.Field( - "OrderEditAddLineItemDiscountPayload", - graphql_name="orderEditAddLineItemDiscount", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("line_item_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="lineItemId", default=None)), - ( - "discount", - sgqlc.types.Arg(sgqlc.types.non_null(OrderEditAppliedDiscountInput), graphql_name="discount", default=None), - ), - ) - ), - ) - order_edit_add_variant = sgqlc.types.Field( - "OrderEditAddVariantPayload", - graphql_name="orderEditAddVariant", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("variant_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="variantId", default=None)), - ("location_id", sgqlc.types.Arg(ID, graphql_name="locationId", default=None)), - ("quantity", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="quantity", default=None)), - ("allow_duplicates", sgqlc.types.Arg(Boolean, graphql_name="allowDuplicates", default=False)), - ) - ), - ) - order_edit_begin = sgqlc.types.Field( - "OrderEditBeginPayload", - graphql_name="orderEditBegin", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - order_edit_commit = sgqlc.types.Field( - "OrderEditCommitPayload", - graphql_name="orderEditCommit", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("notify_customer", sgqlc.types.Arg(Boolean, graphql_name="notifyCustomer", default=None)), - ("staff_note", sgqlc.types.Arg(String, graphql_name="staffNote", default=None)), - ) - ), - ) - order_edit_remove_line_item_discount = sgqlc.types.Field( - "OrderEditRemoveLineItemDiscountPayload", - graphql_name="orderEditRemoveLineItemDiscount", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "discount_application_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="discountApplicationId", default=None), - ), - ) - ), - ) - order_edit_set_quantity = sgqlc.types.Field( - "OrderEditSetQuantityPayload", - graphql_name="orderEditSetQuantity", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("line_item_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="lineItemId", default=None)), - ("quantity", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="quantity", default=None)), - ("restock", sgqlc.types.Arg(Boolean, graphql_name="restock", default=None)), - ) - ), - ) - order_invoice_send = sgqlc.types.Field( - "OrderInvoiceSendPayload", - graphql_name="orderInvoiceSend", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("email", sgqlc.types.Arg(EmailInput, graphql_name="email", default=None)), - ) - ), - ) - order_mark_as_paid = sgqlc.types.Field( - "OrderMarkAsPaidPayload", - graphql_name="orderMarkAsPaid", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(OrderMarkAsPaidInput), graphql_name="input", default=None), - ), - ) - ), - ) - order_open = sgqlc.types.Field( - "OrderOpenPayload", - graphql_name="orderOpen", - args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(OrderOpenInput), graphql_name="input", default=None)),)), - ) - order_update = sgqlc.types.Field( - "OrderUpdatePayload", - graphql_name="orderUpdate", - args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(OrderInput), graphql_name="input", default=None)),)), - ) - payment_terms_create = sgqlc.types.Field( - "PaymentTermsCreatePayload", - graphql_name="paymentTermsCreate", - args=sgqlc.types.ArgDict( - ( - ("reference_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="referenceId", default=None)), - ( - "payment_terms_attributes", - sgqlc.types.Arg( - sgqlc.types.non_null(PaymentTermsCreateInput), - graphql_name="paymentTermsAttributes", - default=None, - ), - ), - ) - ), - ) - payment_terms_delete = sgqlc.types.Field( - "PaymentTermsDeletePayload", - graphql_name="paymentTermsDelete", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(PaymentTermsDeleteInput), graphql_name="input", default=None), - ), - ) - ), - ) - payment_terms_update = sgqlc.types.Field( - "PaymentTermsUpdatePayload", - graphql_name="paymentTermsUpdate", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(PaymentTermsUpdateInput), graphql_name="input", default=None), - ), - ) - ), - ) - price_list_create = sgqlc.types.Field( - "PriceListCreatePayload", - graphql_name="priceListCreate", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(PriceListCreateInput), graphql_name="input", default=None), - ), - ) - ), - ) - price_list_delete = sgqlc.types.Field( - "PriceListDeletePayload", - graphql_name="priceListDelete", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - price_list_fixed_prices_add = sgqlc.types.Field( - "PriceListFixedPricesAddPayload", - graphql_name="priceListFixedPricesAdd", - args=sgqlc.types.ArgDict( - ( - ("price_list_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="priceListId", default=None)), - ( - "prices", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(PriceListPriceInput))), - graphql_name="prices", - default=None, - ), - ), - ) - ), - ) - price_list_fixed_prices_delete = sgqlc.types.Field( - "PriceListFixedPricesDeletePayload", - graphql_name="priceListFixedPricesDelete", - args=sgqlc.types.ArgDict( - ( - ("price_list_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="priceListId", default=None)), - ( - "variant_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="variantIds", - default=None, - ), - ), - ) - ), - ) - price_list_update = sgqlc.types.Field( - "PriceListUpdatePayload", - graphql_name="priceListUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(PriceListUpdateInput), graphql_name="input", default=None), - ), - ) - ), - ) - price_rule_activate = sgqlc.types.Field( - "PriceRuleActivatePayload", - graphql_name="priceRuleActivate", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - price_rule_create = sgqlc.types.Field( - "PriceRuleCreatePayload", - graphql_name="priceRuleCreate", - args=sgqlc.types.ArgDict( - ( - ( - "price_rule", - sgqlc.types.Arg(sgqlc.types.non_null(PriceRuleInput), graphql_name="priceRule", default=None), - ), - ( - "price_rule_discount_code", - sgqlc.types.Arg(PriceRuleDiscountCodeInput, graphql_name="priceRuleDiscountCode", default=None), - ), - ) - ), - ) - price_rule_deactivate = sgqlc.types.Field( - "PriceRuleDeactivatePayload", - graphql_name="priceRuleDeactivate", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - price_rule_delete = sgqlc.types.Field( - "PriceRuleDeletePayload", - graphql_name="priceRuleDelete", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - price_rule_discount_code_create = sgqlc.types.Field( - "PriceRuleDiscountCodeCreatePayload", - graphql_name="priceRuleDiscountCodeCreate", - args=sgqlc.types.ArgDict( - ( - ("price_rule_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="priceRuleId", default=None)), - ("code", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="code", default=None)), - ) - ), - ) - price_rule_discount_code_update = sgqlc.types.Field( - "PriceRuleDiscountCodeUpdatePayload", - graphql_name="priceRuleDiscountCodeUpdate", - args=sgqlc.types.ArgDict( - ( - ("price_rule_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="priceRuleId", default=None)), - ("code", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="code", default=None)), - ) - ), - ) - price_rule_update = sgqlc.types.Field( - "PriceRuleUpdatePayload", - graphql_name="priceRuleUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "price_rule", - sgqlc.types.Arg(sgqlc.types.non_null(PriceRuleInput), graphql_name="priceRule", default=None), - ), - ( - "price_rule_discount_code", - sgqlc.types.Arg(PriceRuleDiscountCodeInput, graphql_name="priceRuleDiscountCode", default=None), - ), - ) - ), - ) - private_metafield_delete = sgqlc.types.Field( - "PrivateMetafieldDeletePayload", - graphql_name="privateMetafieldDelete", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(PrivateMetafieldDeleteInput), graphql_name="input", default=None), - ), - ) - ), - ) - private_metafield_upsert = sgqlc.types.Field( - "PrivateMetafieldUpsertPayload", - graphql_name="privateMetafieldUpsert", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(PrivateMetafieldInput), graphql_name="input", default=None), - ), - ) - ), - ) - product_append_images = sgqlc.types.Field( - "ProductAppendImagesPayload", - graphql_name="productAppendImages", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(ProductAppendImagesInput), graphql_name="input", default=None), - ), - ) - ), - ) - product_change_status = sgqlc.types.Field( - "ProductChangeStatusPayload", - graphql_name="productChangeStatus", - args=sgqlc.types.ArgDict( - ( - ("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)), - ("status", sgqlc.types.Arg(sgqlc.types.non_null(ProductStatus), graphql_name="status", default=None)), - ) - ), - ) - product_create = sgqlc.types.Field( - "ProductCreatePayload", - graphql_name="productCreate", - args=sgqlc.types.ArgDict( - ( - ("input", sgqlc.types.Arg(sgqlc.types.non_null(ProductInput), graphql_name="input", default=None)), - ( - "media", - sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(CreateMediaInput)), graphql_name="media", default=None), - ), - ) - ), - ) - product_create_media = sgqlc.types.Field( - "ProductCreateMediaPayload", - graphql_name="productCreateMedia", - args=sgqlc.types.ArgDict( - ( - ("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)), - ( - "media", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CreateMediaInput))), - graphql_name="media", - default=None, - ), - ), - ) - ), - ) - product_delete = sgqlc.types.Field( - "ProductDeletePayload", - graphql_name="productDelete", - args=sgqlc.types.ArgDict( - (("input", sgqlc.types.Arg(sgqlc.types.non_null(ProductDeleteInput), graphql_name="input", default=None)),) - ), - ) - product_delete_images = sgqlc.types.Field( - "ProductDeleteImagesPayload", - graphql_name="productDeleteImages", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "image_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="imageIds", - default=None, - ), - ), - ) - ), - ) - product_delete_media = sgqlc.types.Field( - "ProductDeleteMediaPayload", - graphql_name="productDeleteMedia", - args=sgqlc.types.ArgDict( - ( - ("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)), - ( - "media_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="mediaIds", - default=None, - ), - ), - ) - ), - ) - product_duplicate = sgqlc.types.Field( - "ProductDuplicatePayload", - graphql_name="productDuplicate", - args=sgqlc.types.ArgDict( - ( - ("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)), - ("new_title", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="newTitle", default=None)), - ("new_status", sgqlc.types.Arg(ProductStatus, graphql_name="newStatus", default=None)), - ("include_images", sgqlc.types.Arg(Boolean, graphql_name="includeImages", default=False)), - ) - ), - ) - product_image_update = sgqlc.types.Field( - "ProductImageUpdatePayload", - graphql_name="productImageUpdate", - args=sgqlc.types.ArgDict( - ( - ("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)), - ("image", sgqlc.types.Arg(sgqlc.types.non_null(ImageInput), graphql_name="image", default=None)), - ) - ), - ) - product_join_selling_plan_groups = sgqlc.types.Field( - "ProductJoinSellingPlanGroupsPayload", - graphql_name="productJoinSellingPlanGroups", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "selling_plan_group_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="sellingPlanGroupIds", - default=None, - ), - ), - ) - ), - ) - product_leave_selling_plan_groups = sgqlc.types.Field( - "ProductLeaveSellingPlanGroupsPayload", - graphql_name="productLeaveSellingPlanGroups", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "selling_plan_group_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="sellingPlanGroupIds", - default=None, - ), - ), - ) - ), - ) - product_reorder_images = sgqlc.types.Field( - "ProductReorderImagesPayload", - graphql_name="productReorderImages", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "moves", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MoveInput))), - graphql_name="moves", - default=None, - ), - ), - ) - ), - ) - product_reorder_media = sgqlc.types.Field( - "ProductReorderMediaPayload", - graphql_name="productReorderMedia", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "moves", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MoveInput))), - graphql_name="moves", - default=None, - ), - ), - ) - ), - ) - product_update = sgqlc.types.Field( - "ProductUpdatePayload", - graphql_name="productUpdate", - args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(ProductInput), graphql_name="input", default=None)),)), - ) - product_update_media = sgqlc.types.Field( - "ProductUpdateMediaPayload", - graphql_name="productUpdateMedia", - args=sgqlc.types.ArgDict( - ( - ("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)), - ( - "media", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(UpdateMediaInput))), - graphql_name="media", - default=None, - ), - ), - ) - ), - ) - product_variant_append_media = sgqlc.types.Field( - "ProductVariantAppendMediaPayload", - graphql_name="productVariantAppendMedia", - args=sgqlc.types.ArgDict( - ( - ("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)), - ( - "variant_media", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProductVariantAppendMediaInput))), - graphql_name="variantMedia", - default=None, - ), - ), - ) - ), - ) - product_variant_create = sgqlc.types.Field( - "ProductVariantCreatePayload", - graphql_name="productVariantCreate", - args=sgqlc.types.ArgDict( - (("input", sgqlc.types.Arg(sgqlc.types.non_null(ProductVariantInput), graphql_name="input", default=None)),) - ), - ) - product_variant_delete = sgqlc.types.Field( - "ProductVariantDeletePayload", - graphql_name="productVariantDelete", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - product_variant_detach_media = sgqlc.types.Field( - "ProductVariantDetachMediaPayload", - graphql_name="productVariantDetachMedia", - args=sgqlc.types.ArgDict( - ( - ("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)), - ( - "variant_media", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProductVariantDetachMediaInput))), - graphql_name="variantMedia", - default=None, - ), - ), - ) - ), - ) - product_variant_join_selling_plan_groups = sgqlc.types.Field( - "ProductVariantJoinSellingPlanGroupsPayload", - graphql_name="productVariantJoinSellingPlanGroups", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "selling_plan_group_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="sellingPlanGroupIds", - default=None, - ), - ), - ) - ), - ) - product_variant_leave_selling_plan_groups = sgqlc.types.Field( - "ProductVariantLeaveSellingPlanGroupsPayload", - graphql_name="productVariantLeaveSellingPlanGroups", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "selling_plan_group_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="sellingPlanGroupIds", - default=None, - ), - ), - ) - ), - ) - product_variant_update = sgqlc.types.Field( - "ProductVariantUpdatePayload", - graphql_name="productVariantUpdate", - args=sgqlc.types.ArgDict( - (("input", sgqlc.types.Arg(sgqlc.types.non_null(ProductVariantInput), graphql_name="input", default=None)),) - ), - ) - product_variants_bulk_create = sgqlc.types.Field( - "ProductVariantsBulkCreatePayload", - graphql_name="productVariantsBulkCreate", - args=sgqlc.types.ArgDict( - ( - ( - "variants", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProductVariantsBulkInput))), - graphql_name="variants", - default=None, - ), - ), - ("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)), - ) - ), - ) - product_variants_bulk_delete = sgqlc.types.Field( - "ProductVariantsBulkDeletePayload", - graphql_name="productVariantsBulkDelete", - args=sgqlc.types.ArgDict( - ( - ( - "variants_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="variantsIds", - default=None, - ), - ), - ("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)), - ) - ), - ) - product_variants_bulk_reorder = sgqlc.types.Field( - "ProductVariantsBulkReorderPayload", - graphql_name="productVariantsBulkReorder", - args=sgqlc.types.ArgDict( - ( - ("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)), - ( - "positions", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProductVariantPositionInput))), - graphql_name="positions", - default=None, - ), - ), - ) - ), - ) - product_variants_bulk_update = sgqlc.types.Field( - "ProductVariantsBulkUpdatePayload", - graphql_name="productVariantsBulkUpdate", - args=sgqlc.types.ArgDict( - ( - ( - "variants", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProductVariantsBulkInput))), - graphql_name="variants", - default=None, - ), - ), - ("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)), - ) - ), - ) - pub_sub_webhook_subscription_create = sgqlc.types.Field( - "PubSubWebhookSubscriptionCreatePayload", - graphql_name="pubSubWebhookSubscriptionCreate", - args=sgqlc.types.ArgDict( - ( - ( - "topic", - sgqlc.types.Arg(sgqlc.types.non_null(WebhookSubscriptionTopic), graphql_name="topic", default=None), - ), - ( - "webhook_subscription", - sgqlc.types.Arg( - sgqlc.types.non_null(PubSubWebhookSubscriptionInput), - graphql_name="webhookSubscription", - default=None, - ), - ), - ) - ), - ) - pub_sub_webhook_subscription_update = sgqlc.types.Field( - "PubSubWebhookSubscriptionUpdatePayload", - graphql_name="pubSubWebhookSubscriptionUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "webhook_subscription", - sgqlc.types.Arg(PubSubWebhookSubscriptionInput, graphql_name="webhookSubscription", default=None), - ), - ) - ), - ) - publishable_publish = sgqlc.types.Field( - "PublishablePublishPayload", - graphql_name="publishablePublish", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "input", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(PublicationInput))), - graphql_name="input", - default=None, - ), - ), - ) - ), - ) - publishable_publish_to_current_channel = sgqlc.types.Field( - "PublishablePublishToCurrentChannelPayload", - graphql_name="publishablePublishToCurrentChannel", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - publishable_unpublish = sgqlc.types.Field( - "PublishableUnpublishPayload", - graphql_name="publishableUnpublish", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "input", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(PublicationInput))), - graphql_name="input", - default=None, - ), - ), - ) - ), - ) - publishable_unpublish_to_current_channel = sgqlc.types.Field( - "PublishableUnpublishToCurrentChannelPayload", - graphql_name="publishableUnpublishToCurrentChannel", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - refund_create = sgqlc.types.Field( - "RefundCreatePayload", - graphql_name="refundCreate", - args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(RefundInput), graphql_name="input", default=None)),)), - ) - saved_search_create = sgqlc.types.Field( - "SavedSearchCreatePayload", - graphql_name="savedSearchCreate", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(SavedSearchCreateInput), graphql_name="input", default=None), - ), - ) - ), - ) - saved_search_delete = sgqlc.types.Field( - "SavedSearchDeletePayload", - graphql_name="savedSearchDelete", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(SavedSearchDeleteInput), graphql_name="input", default=None), - ), - ) - ), - ) - saved_search_update = sgqlc.types.Field( - "SavedSearchUpdatePayload", - graphql_name="savedSearchUpdate", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(SavedSearchUpdateInput), graphql_name="input", default=None), - ), - ) - ), - ) - script_tag_create = sgqlc.types.Field( - "ScriptTagCreatePayload", - graphql_name="scriptTagCreate", - args=sgqlc.types.ArgDict((("input", sgqlc.types.Arg(sgqlc.types.non_null(ScriptTagInput), graphql_name="input", default=None)),)), - ) - script_tag_delete = sgqlc.types.Field( - "ScriptTagDeletePayload", - graphql_name="scriptTagDelete", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - script_tag_update = sgqlc.types.Field( - "ScriptTagUpdatePayload", - graphql_name="scriptTagUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("input", sgqlc.types.Arg(sgqlc.types.non_null(ScriptTagInput), graphql_name="input", default=None)), - ) - ), - ) - segment_create = sgqlc.types.Field( - "SegmentCreatePayload", - graphql_name="segmentCreate", - args=sgqlc.types.ArgDict( - ( - ("name", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="name", default=None)), - ("query", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="query", default=None)), - ) - ), - ) - segment_delete = sgqlc.types.Field( - "SegmentDeletePayload", - graphql_name="segmentDelete", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - segment_update = sgqlc.types.Field( - "SegmentUpdatePayload", - graphql_name="segmentUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("name", sgqlc.types.Arg(String, graphql_name="name", default=None)), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - selling_plan_group_add_product_variants = sgqlc.types.Field( - "SellingPlanGroupAddProductVariantsPayload", - graphql_name="sellingPlanGroupAddProductVariants", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "product_variant_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="productVariantIds", - default=None, - ), - ), - ) - ), - ) - selling_plan_group_add_products = sgqlc.types.Field( - "SellingPlanGroupAddProductsPayload", - graphql_name="sellingPlanGroupAddProducts", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "product_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="productIds", - default=None, - ), - ), - ) - ), - ) - selling_plan_group_create = sgqlc.types.Field( - "SellingPlanGroupCreatePayload", - graphql_name="sellingPlanGroupCreate", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(SellingPlanGroupInput), graphql_name="input", default=None), - ), - ("resources", sgqlc.types.Arg(SellingPlanGroupResourceInput, graphql_name="resources", default=None)), - ) - ), - ) - selling_plan_group_delete = sgqlc.types.Field( - "SellingPlanGroupDeletePayload", - graphql_name="sellingPlanGroupDelete", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - selling_plan_group_remove_product_variants = sgqlc.types.Field( - "SellingPlanGroupRemoveProductVariantsPayload", - graphql_name="sellingPlanGroupRemoveProductVariants", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "product_variant_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="productVariantIds", - default=None, - ), - ), - ) - ), - ) - selling_plan_group_remove_products = sgqlc.types.Field( - "SellingPlanGroupRemoveProductsPayload", - graphql_name="sellingPlanGroupRemoveProducts", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "product_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="productIds", - default=None, - ), - ), - ) - ), - ) - selling_plan_group_update = sgqlc.types.Field( - "SellingPlanGroupUpdatePayload", - graphql_name="sellingPlanGroupUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("input", sgqlc.types.Arg(SellingPlanGroupInput, graphql_name="input", default=None)), - ) - ), - ) - shipping_package_delete = sgqlc.types.Field( - "ShippingPackageDeletePayload", - graphql_name="shippingPackageDelete", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - shipping_package_make_default = sgqlc.types.Field( - "ShippingPackageMakeDefaultPayload", - graphql_name="shippingPackageMakeDefault", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - shipping_package_update = sgqlc.types.Field( - "ShippingPackageUpdatePayload", - graphql_name="shippingPackageUpdate", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - shop_locale_disable = sgqlc.types.Field( - "ShopLocaleDisablePayload", - graphql_name="shopLocaleDisable", - args=sgqlc.types.ArgDict((("locale", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="locale", default=None)),)), - ) - shop_locale_enable = sgqlc.types.Field( - "ShopLocaleEnablePayload", - graphql_name="shopLocaleEnable", - args=sgqlc.types.ArgDict( - ( - ("locale", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="locale", default=None)), - ( - "market_web_presence_ids", - sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="marketWebPresenceIds", default=None), - ), - ) - ), - ) - shop_locale_update = sgqlc.types.Field( - "ShopLocaleUpdatePayload", - graphql_name="shopLocaleUpdate", - args=sgqlc.types.ArgDict( - ( - ("locale", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="locale", default=None)), - ( - "shop_locale", - sgqlc.types.Arg(sgqlc.types.non_null(ShopLocaleInput), graphql_name="shopLocale", default=None), - ), - ) - ), - ) - shop_policy_update = sgqlc.types.Field( - "ShopPolicyUpdatePayload", - graphql_name="shopPolicyUpdate", - args=sgqlc.types.ArgDict( - ( - ( - "shop_policy", - sgqlc.types.Arg(sgqlc.types.non_null(ShopPolicyInput), graphql_name="shopPolicy", default=None), - ), - ) - ), - ) - staged_uploads_create = sgqlc.types.Field( - "StagedUploadsCreatePayload", - graphql_name="stagedUploadsCreate", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(StagedUploadInput))), - graphql_name="input", - default=None, - ), - ), - ) - ), - ) - standard_metafield_definition_enable = sgqlc.types.Field( - "StandardMetafieldDefinitionEnablePayload", - graphql_name="standardMetafieldDefinitionEnable", - args=sgqlc.types.ArgDict( - ( - ( - "owner_type", - sgqlc.types.Arg(sgqlc.types.non_null(MetafieldOwnerType), graphql_name="ownerType", default=None), - ), - ("id", sgqlc.types.Arg(ID, graphql_name="id", default=None)), - ("namespace", sgqlc.types.Arg(String, graphql_name="namespace", default=None)), - ("key", sgqlc.types.Arg(String, graphql_name="key", default=None)), - ("pin", sgqlc.types.Arg(sgqlc.types.non_null(Boolean), graphql_name="pin", default=False)), - ( - "visible_to_storefront_api", - sgqlc.types.Arg(Boolean, graphql_name="visibleToStorefrontApi", default=None), - ), - ) - ), - ) - storefront_access_token_create = sgqlc.types.Field( - "StorefrontAccessTokenCreatePayload", - graphql_name="storefrontAccessTokenCreate", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(StorefrontAccessTokenInput), graphql_name="input", default=None), - ), - ) - ), - ) - storefront_access_token_delete = sgqlc.types.Field( - "StorefrontAccessTokenDeletePayload", - graphql_name="storefrontAccessTokenDelete", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(StorefrontAccessTokenDeleteInput), graphql_name="input", default=None), - ), - ) - ), - ) - subscription_billing_attempt_create = sgqlc.types.Field( - "SubscriptionBillingAttemptCreatePayload", - graphql_name="subscriptionBillingAttemptCreate", - args=sgqlc.types.ArgDict( - ( - ( - "subscription_contract_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="subscriptionContractId", default=None), - ), - ( - "subscription_billing_attempt_input", - sgqlc.types.Arg( - sgqlc.types.non_null(SubscriptionBillingAttemptInput), - graphql_name="subscriptionBillingAttemptInput", - default=None, - ), - ), - ) - ), - ) - subscription_billing_cycle_contract_draft_commit = sgqlc.types.Field( - "SubscriptionBillingCycleContractDraftCommitPayload", - graphql_name="subscriptionBillingCycleContractDraftCommit", - args=sgqlc.types.ArgDict((("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)),)), - ) - subscription_billing_cycle_contract_draft_concatenate = sgqlc.types.Field( - "SubscriptionBillingCycleContractDraftConcatenatePayload", - graphql_name="subscriptionBillingCycleContractDraftConcatenate", - args=sgqlc.types.ArgDict( - ( - ("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)), - ( - "concatenated_billing_cycle_contracts", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SubscriptionBillingCycleInput))), - graphql_name="concatenatedBillingCycleContracts", - default=None, - ), - ), - ) - ), - ) - subscription_billing_cycle_contract_edit = sgqlc.types.Field( - "SubscriptionBillingCycleContractEditPayload", - graphql_name="subscriptionBillingCycleContractEdit", - args=sgqlc.types.ArgDict( - ( - ( - "billing_cycle_input", - sgqlc.types.Arg( - sgqlc.types.non_null(SubscriptionBillingCycleInput), - graphql_name="billingCycleInput", - default=None, - ), - ), - ) - ), - ) - subscription_billing_cycle_edit_delete = sgqlc.types.Field( - "SubscriptionBillingCycleEditDeletePayload", - graphql_name="subscriptionBillingCycleEditDelete", - args=sgqlc.types.ArgDict( - ( - ( - "billing_cycle_input", - sgqlc.types.Arg( - sgqlc.types.non_null(SubscriptionBillingCycleInput), - graphql_name="billingCycleInput", - default=None, - ), - ), - ) - ), - ) - subscription_billing_cycle_edits_delete = sgqlc.types.Field( - "SubscriptionBillingCycleEditsDeletePayload", - graphql_name="subscriptionBillingCycleEditsDelete", - args=sgqlc.types.ArgDict( - ( - ("contract_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="contractId", default=None)), - ( - "target_selection", - sgqlc.types.Arg( - sgqlc.types.non_null(SubscriptionBillingCyclesTargetSelection), - graphql_name="targetSelection", - default=None, - ), - ), - ) - ), - ) - subscription_billing_cycle_schedule_edit = sgqlc.types.Field( - "SubscriptionBillingCycleScheduleEditPayload", - graphql_name="subscriptionBillingCycleScheduleEdit", - args=sgqlc.types.ArgDict( - ( - ( - "billing_cycle_input", - sgqlc.types.Arg( - sgqlc.types.non_null(SubscriptionBillingCycleInput), - graphql_name="billingCycleInput", - default=None, - ), - ), - ( - "input", - sgqlc.types.Arg( - sgqlc.types.non_null(SubscriptionBillingCycleScheduleEditInput), - graphql_name="input", - default=None, - ), - ), - ) - ), - ) - subscription_contract_create = sgqlc.types.Field( - "SubscriptionContractCreatePayload", - graphql_name="subscriptionContractCreate", - args=sgqlc.types.ArgDict( - ( - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(SubscriptionContractCreateInput), graphql_name="input", default=None), - ), - ) - ), - ) - subscription_contract_set_next_billing_date = sgqlc.types.Field( - "SubscriptionContractSetNextBillingDatePayload", - graphql_name="subscriptionContractSetNextBillingDate", - args=sgqlc.types.ArgDict( - ( - ("contract_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="contractId", default=None)), - ("date", sgqlc.types.Arg(sgqlc.types.non_null(DateTime), graphql_name="date", default=None)), - ) - ), - ) - subscription_contract_update = sgqlc.types.Field( - "SubscriptionContractUpdatePayload", - graphql_name="subscriptionContractUpdate", - args=sgqlc.types.ArgDict((("contract_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="contractId", default=None)),)), - ) - subscription_draft_commit = sgqlc.types.Field( - "SubscriptionDraftCommitPayload", - graphql_name="subscriptionDraftCommit", - args=sgqlc.types.ArgDict((("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)),)), - ) - subscription_draft_discount_add = sgqlc.types.Field( - "SubscriptionDraftDiscountAddPayload", - graphql_name="subscriptionDraftDiscountAdd", - args=sgqlc.types.ArgDict( - ( - ("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)), - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(SubscriptionManualDiscountInput), graphql_name="input", default=None), - ), - ) - ), - ) - subscription_draft_discount_code_apply = sgqlc.types.Field( - "SubscriptionDraftDiscountCodeApplyPayload", - graphql_name="subscriptionDraftDiscountCodeApply", - args=sgqlc.types.ArgDict( - ( - ("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)), - ("redeem_code", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="redeemCode", default=None)), - ) - ), - ) - subscription_draft_discount_remove = sgqlc.types.Field( - "SubscriptionDraftDiscountRemovePayload", - graphql_name="subscriptionDraftDiscountRemove", - args=sgqlc.types.ArgDict( - ( - ("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)), - ("discount_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="discountId", default=None)), - ) - ), - ) - subscription_draft_discount_update = sgqlc.types.Field( - "SubscriptionDraftDiscountUpdatePayload", - graphql_name="subscriptionDraftDiscountUpdate", - args=sgqlc.types.ArgDict( - ( - ("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)), - ("discount_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="discountId", default=None)), - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(SubscriptionManualDiscountInput), graphql_name="input", default=None), - ), - ) - ), - ) - subscription_draft_free_shipping_discount_add = sgqlc.types.Field( - "SubscriptionDraftFreeShippingDiscountAddPayload", - graphql_name="subscriptionDraftFreeShippingDiscountAdd", - args=sgqlc.types.ArgDict( - ( - ("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)), - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(SubscriptionFreeShippingDiscountInput), graphql_name="input", default=None), - ), - ) - ), - ) - subscription_draft_free_shipping_discount_update = sgqlc.types.Field( - "SubscriptionDraftFreeShippingDiscountUpdatePayload", - graphql_name="subscriptionDraftFreeShippingDiscountUpdate", - args=sgqlc.types.ArgDict( - ( - ("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)), - ("discount_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="discountId", default=None)), - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(SubscriptionFreeShippingDiscountInput), graphql_name="input", default=None), - ), - ) - ), - ) - subscription_draft_line_add = sgqlc.types.Field( - "SubscriptionDraftLineAddPayload", - graphql_name="subscriptionDraftLineAdd", - args=sgqlc.types.ArgDict( - ( - ("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)), - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(SubscriptionLineInput), graphql_name="input", default=None), - ), - ) - ), - ) - subscription_draft_line_remove = sgqlc.types.Field( - "SubscriptionDraftLineRemovePayload", - graphql_name="subscriptionDraftLineRemove", - args=sgqlc.types.ArgDict( - ( - ("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)), - ("line_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="lineId", default=None)), - ) - ), - ) - subscription_draft_line_update = sgqlc.types.Field( - "SubscriptionDraftLineUpdatePayload", - graphql_name="subscriptionDraftLineUpdate", - args=sgqlc.types.ArgDict( - ( - ("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)), - ("line_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="lineId", default=None)), - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(SubscriptionLineUpdateInput), graphql_name="input", default=None), - ), - ) - ), - ) - subscription_draft_update = sgqlc.types.Field( - "SubscriptionDraftUpdatePayload", - graphql_name="subscriptionDraftUpdate", - args=sgqlc.types.ArgDict( - ( - ("draft_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="draftId", default=None)), - ( - "input", - sgqlc.types.Arg(sgqlc.types.non_null(SubscriptionDraftInput), graphql_name="input", default=None), - ), - ) - ), - ) - tags_add = sgqlc.types.Field( - "TagsAddPayload", - graphql_name="tagsAdd", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "tags", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), - graphql_name="tags", - default=None, - ), - ), - ) - ), - ) - tags_remove = sgqlc.types.Field( - "TagsRemovePayload", - graphql_name="tagsRemove", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "tags", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), - graphql_name="tags", - default=None, - ), - ), - ) - ), - ) - translations_register = sgqlc.types.Field( - "TranslationsRegisterPayload", - graphql_name="translationsRegister", - args=sgqlc.types.ArgDict( - ( - ("resource_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="resourceId", default=None)), - ( - "translations", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TranslationInput))), - graphql_name="translations", - default=None, - ), - ), - ) - ), - ) - translations_remove = sgqlc.types.Field( - "TranslationsRemovePayload", - graphql_name="translationsRemove", - args=sgqlc.types.ArgDict( - ( - ("resource_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="resourceId", default=None)), - ( - "translation_keys", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), - graphql_name="translationKeys", - default=None, - ), - ), - ( - "locales", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), - graphql_name="locales", - default=None, - ), - ), - ( - "market_ids", - sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="marketIds", default=None), - ), - ) - ), - ) - url_redirect_bulk_delete_all = sgqlc.types.Field("UrlRedirectBulkDeleteAllPayload", graphql_name="urlRedirectBulkDeleteAll") - url_redirect_bulk_delete_by_ids = sgqlc.types.Field( - "UrlRedirectBulkDeleteByIdsPayload", - graphql_name="urlRedirectBulkDeleteByIds", - args=sgqlc.types.ArgDict( - ( - ( - "ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="ids", - default=None, - ), - ), - ) - ), - ) - url_redirect_bulk_delete_by_saved_search = sgqlc.types.Field( - "UrlRedirectBulkDeleteBySavedSearchPayload", - graphql_name="urlRedirectBulkDeleteBySavedSearch", - args=sgqlc.types.ArgDict( - ( - ( - "saved_search_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="savedSearchId", default=None), - ), - ) - ), - ) - url_redirect_bulk_delete_by_search = sgqlc.types.Field( - "UrlRedirectBulkDeleteBySearchPayload", - graphql_name="urlRedirectBulkDeleteBySearch", - args=sgqlc.types.ArgDict((("search", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="search", default=None)),)), - ) - url_redirect_create = sgqlc.types.Field( - "UrlRedirectCreatePayload", - graphql_name="urlRedirectCreate", - args=sgqlc.types.ArgDict( - ( - ( - "url_redirect", - sgqlc.types.Arg(sgqlc.types.non_null(UrlRedirectInput), graphql_name="urlRedirect", default=None), - ), - ) - ), - ) - url_redirect_delete = sgqlc.types.Field( - "UrlRedirectDeletePayload", - graphql_name="urlRedirectDelete", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - url_redirect_import_create = sgqlc.types.Field( - "UrlRedirectImportCreatePayload", - graphql_name="urlRedirectImportCreate", - args=sgqlc.types.ArgDict((("url", sgqlc.types.Arg(sgqlc.types.non_null(URL), graphql_name="url", default=None)),)), - ) - url_redirect_import_submit = sgqlc.types.Field( - "UrlRedirectImportSubmitPayload", - graphql_name="urlRedirectImportSubmit", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - url_redirect_update = sgqlc.types.Field( - "UrlRedirectUpdatePayload", - graphql_name="urlRedirectUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "url_redirect", - sgqlc.types.Arg(sgqlc.types.non_null(UrlRedirectInput), graphql_name="urlRedirect", default=None), - ), - ) - ), - ) - web_pixel_create = sgqlc.types.Field( - "WebPixelCreatePayload", - graphql_name="webPixelCreate", - args=sgqlc.types.ArgDict( - ( - ( - "web_pixel", - sgqlc.types.Arg(sgqlc.types.non_null(WebPixelInput), graphql_name="webPixel", default=None), - ), - ) - ), - ) - web_pixel_delete = sgqlc.types.Field( - "WebPixelDeletePayload", - graphql_name="webPixelDelete", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - web_pixel_update = sgqlc.types.Field( - "WebPixelUpdatePayload", - graphql_name="webPixelUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "web_pixel", - sgqlc.types.Arg(sgqlc.types.non_null(WebPixelInput), graphql_name="webPixel", default=None), - ), - ) - ), - ) - webhook_subscription_create = sgqlc.types.Field( - "WebhookSubscriptionCreatePayload", - graphql_name="webhookSubscriptionCreate", - args=sgqlc.types.ArgDict( - ( - ( - "topic", - sgqlc.types.Arg(sgqlc.types.non_null(WebhookSubscriptionTopic), graphql_name="topic", default=None), - ), - ( - "webhook_subscription", - sgqlc.types.Arg(sgqlc.types.non_null(WebhookSubscriptionInput), graphql_name="webhookSubscription", default=None), - ), - ) - ), - ) - webhook_subscription_delete = sgqlc.types.Field( - "WebhookSubscriptionDeletePayload", - graphql_name="webhookSubscriptionDelete", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - webhook_subscription_update = sgqlc.types.Field( - "WebhookSubscriptionUpdatePayload", - graphql_name="webhookSubscriptionUpdate", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ( - "webhook_subscription", - sgqlc.types.Arg(sgqlc.types.non_null(WebhookSubscriptionInput), graphql_name="webhookSubscription", default=None), - ), - ) - ), - ) - - -class MutationsStagedUploadTargetGenerateUploadParameter(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("name", "value") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") - - -class Navigable(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ("default_cursor",) - default_cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="defaultCursor") - - -class NavigationItem(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("id", "title", "url") - id = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="id") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") - - -class Node(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ("id",) - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - - -class OnlineStorePreviewable(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ("online_store_preview_url",) - online_store_preview_url = sgqlc.types.Field(URL, graphql_name="onlineStorePreviewUrl") - - -class OrderApp(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("icon", "id", "name") - icon = sgqlc.types.Field(sgqlc.types.non_null("Image"), graphql_name="icon") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - - -class OrderCapturePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("transaction", "user_errors") - transaction = sgqlc.types.Field("OrderTransaction", graphql_name="transaction") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class OrderClosePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("order", "user_errors") - order = sgqlc.types.Field("Order", graphql_name="order") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class OrderConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("OrderEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Order"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class OrderCreateMandatePaymentPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("job", "payment_reference_id", "user_errors") - job = sgqlc.types.Field(Job, graphql_name="job") - payment_reference_id = sgqlc.types.Field(String, graphql_name="paymentReferenceId") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("OrderCreateMandatePaymentUserError"))), - graphql_name="userErrors", - ) - - -class OrderEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("Order"), graphql_name="node") - - -class OrderEditAddCustomItemPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("calculated_line_item", "calculated_order", "user_errors") - calculated_line_item = sgqlc.types.Field(CalculatedLineItem, graphql_name="calculatedLineItem") - calculated_order = sgqlc.types.Field("CalculatedOrder", graphql_name="calculatedOrder") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class OrderEditAddLineItemDiscountPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("added_discount_staged_change", "calculated_line_item", "calculated_order", "user_errors") - added_discount_staged_change = sgqlc.types.Field("OrderStagedChangeAddLineItemDiscount", graphql_name="addedDiscountStagedChange") - calculated_line_item = sgqlc.types.Field(CalculatedLineItem, graphql_name="calculatedLineItem") - calculated_order = sgqlc.types.Field("CalculatedOrder", graphql_name="calculatedOrder") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class OrderEditAddVariantPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("calculated_line_item", "calculated_order", "user_errors") - calculated_line_item = sgqlc.types.Field(CalculatedLineItem, graphql_name="calculatedLineItem") - calculated_order = sgqlc.types.Field("CalculatedOrder", graphql_name="calculatedOrder") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class OrderEditBeginPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("calculated_order", "user_errors") - calculated_order = sgqlc.types.Field("CalculatedOrder", graphql_name="calculatedOrder") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class OrderEditCommitPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("order", "user_errors") - order = sgqlc.types.Field("Order", graphql_name="order") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class OrderEditRemoveLineItemDiscountPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("calculated_line_item", "calculated_order", "user_errors") - calculated_line_item = sgqlc.types.Field(CalculatedLineItem, graphql_name="calculatedLineItem") - calculated_order = sgqlc.types.Field("CalculatedOrder", graphql_name="calculatedOrder") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class OrderEditSetQuantityPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("calculated_line_item", "calculated_order", "user_errors") - calculated_line_item = sgqlc.types.Field(CalculatedLineItem, graphql_name="calculatedLineItem") - calculated_order = sgqlc.types.Field("CalculatedOrder", graphql_name="calculatedOrder") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class OrderInvoiceSendPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("order", "user_errors") - order = sgqlc.types.Field("Order", graphql_name="order") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("OrderInvoiceSendUserError"))), - graphql_name="userErrors", - ) - - -class OrderMarkAsPaidPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("order", "user_errors") - order = sgqlc.types.Field("Order", graphql_name="order") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class OrderOpenPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("order", "user_errors") - order = sgqlc.types.Field("Order", graphql_name="order") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class OrderPaymentCollectionDetails(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("additional_payment_collection_url", "vaulted_payment_methods") - additional_payment_collection_url = sgqlc.types.Field(URL, graphql_name="additionalPaymentCollectionUrl") - vaulted_payment_methods = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null("PaymentMandate")), graphql_name="vaultedPaymentMethods" - ) - - -class OrderPaymentStatus(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("error_message", "payment_reference_id", "status", "translated_error_message") - error_message = sgqlc.types.Field(String, graphql_name="errorMessage") - payment_reference_id = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="paymentReferenceId") - status = sgqlc.types.Field(sgqlc.types.non_null(OrderPaymentStatusResult), graphql_name="status") - translated_error_message = sgqlc.types.Field(String, graphql_name="translatedErrorMessage") - - -class OrderRisk(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("display", "level", "message") - display = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="display") - level = sgqlc.types.Field(OrderRiskLevel, graphql_name="level") - message = sgqlc.types.Field(String, graphql_name="message") - - -class OrderStagedChangeAddCustomItem(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("original_unit_price", "quantity", "title") - original_unit_price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="originalUnitPrice") - quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - - -class OrderStagedChangeAddLineItemDiscount(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("description", "id", "value") - description = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="description") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - value = sgqlc.types.Field(sgqlc.types.non_null("PricingValue"), graphql_name="value") - - -class OrderStagedChangeAddShippingLine(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("phone", "presentment_title", "price", "title") - phone = sgqlc.types.Field(String, graphql_name="phone") - presentment_title = sgqlc.types.Field(String, graphql_name="presentmentTitle") - price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="price") - title = sgqlc.types.Field(String, graphql_name="title") - - -class OrderStagedChangeAddVariant(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("quantity", "variant") - quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") - variant = sgqlc.types.Field(sgqlc.types.non_null("ProductVariant"), graphql_name="variant") - - -class OrderStagedChangeConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("OrderStagedChangeEdge"))), graphql_name="edges" - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("OrderStagedChange"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class OrderStagedChangeDecrementItem(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("delta", "line_item", "restock") - delta = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="delta") - line_item = sgqlc.types.Field(sgqlc.types.non_null("LineItem"), graphql_name="lineItem") - restock = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="restock") - - -class OrderStagedChangeEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("OrderStagedChange"), graphql_name="node") - - -class OrderStagedChangeIncrementItem(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("delta", "line_item") - delta = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="delta") - line_item = sgqlc.types.Field(sgqlc.types.non_null("LineItem"), graphql_name="lineItem") - - -class OrderTransactionConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("OrderTransactionEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("OrderTransaction"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null("PageInfo"), graphql_name="pageInfo") - - -class OrderTransactionEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("OrderTransaction"), graphql_name="node") - - -class OrderUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("order", "user_errors") - order = sgqlc.types.Field("Order", graphql_name="order") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class PageInfo(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("end_cursor", "has_next_page", "has_previous_page", "start_cursor") - end_cursor = sgqlc.types.Field(String, graphql_name="endCursor") - has_next_page = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasNextPage") - has_previous_page = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasPreviousPage") - start_cursor = sgqlc.types.Field(String, graphql_name="startCursor") - - -class PaymentScheduleConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PaymentScheduleEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PaymentSchedule"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class PaymentScheduleEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("PaymentSchedule"), graphql_name="node") - - -class PaymentSettings(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("supported_digital_wallets",) - supported_digital_wallets = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DigitalWallet))), - graphql_name="supportedDigitalWallets", - ) - - -class PaymentTermsCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("payment_terms", "user_errors") - payment_terms = sgqlc.types.Field("PaymentTerms", graphql_name="paymentTerms") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PaymentTermsCreateUserError"))), - graphql_name="userErrors", - ) - - -class PaymentTermsDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_id", "user_errors") - deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PaymentTermsDeleteUserError"))), - graphql_name="userErrors", - ) - - -class PaymentTermsUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("payment_terms", "user_errors") - payment_terms = sgqlc.types.Field("PaymentTerms", graphql_name="paymentTerms") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PaymentTermsUpdateUserError"))), - graphql_name="userErrors", - ) - - -class PriceListAdjustment(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("type", "value") - type = sgqlc.types.Field(sgqlc.types.non_null(PriceListAdjustmentType), graphql_name="type") - value = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="value") - - -class PriceListConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceListEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceList"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class PriceListContextRule(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("countries", "market") - countries = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CountryCode))), graphql_name="countries") - market = sgqlc.types.Field("Market", graphql_name="market") - - -class PriceListCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("price_list", "user_errors") - price_list = sgqlc.types.Field("PriceList", graphql_name="priceList") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceListUserError"))), graphql_name="userErrors" - ) - - -class PriceListDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_id", "user_errors") - deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceListUserError"))), graphql_name="userErrors" - ) - - -class PriceListEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("PriceList"), graphql_name="node") - - -class PriceListFixedPricesAddPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("prices", "user_errors") - prices = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("PriceListPrice")), graphql_name="prices") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceListPriceUserError"))), - graphql_name="userErrors", - ) - - -class PriceListFixedPricesDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_fixed_price_variant_ids", "user_errors") - deleted_fixed_price_variant_ids = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="deletedFixedPriceVariantIds" - ) - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceListPriceUserError"))), - graphql_name="userErrors", - ) - - -class PriceListParent(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("adjustment",) - adjustment = sgqlc.types.Field(sgqlc.types.non_null(PriceListAdjustment), graphql_name="adjustment") - - -class PriceListPrice(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("compare_at_price", "origin_type", "price", "variant") - compare_at_price = sgqlc.types.Field(MoneyV2, graphql_name="compareAtPrice") - origin_type = sgqlc.types.Field(sgqlc.types.non_null(PriceListPriceOriginType), graphql_name="originType") - price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="price") - variant = sgqlc.types.Field(sgqlc.types.non_null("ProductVariant"), graphql_name="variant") - - -class PriceListPriceConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceListPriceEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(PriceListPrice))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class PriceListPriceEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(PriceListPrice), graphql_name="node") - - -class PriceListUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("price_list", "user_errors") - price_list = sgqlc.types.Field("PriceList", graphql_name="priceList") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceListUserError"))), graphql_name="userErrors" - ) - - -class PriceRuleActivatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("price_rule", "price_rule_user_errors") - price_rule = sgqlc.types.Field("PriceRule", graphql_name="priceRule") - price_rule_user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceRuleUserError"))), - graphql_name="priceRuleUserErrors", - ) - - -class PriceRuleConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceRuleEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceRule"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class PriceRuleCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("price_rule", "price_rule_discount_code", "price_rule_user_errors") - price_rule = sgqlc.types.Field("PriceRule", graphql_name="priceRule") - price_rule_discount_code = sgqlc.types.Field("PriceRuleDiscountCode", graphql_name="priceRuleDiscountCode") - price_rule_user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceRuleUserError"))), - graphql_name="priceRuleUserErrors", - ) - - -class PriceRuleCustomerSelection(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("customers", "for_all_customers", "segments") - customers = sgqlc.types.Field( - sgqlc.types.non_null(CustomerConnection), - graphql_name="customers", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(CustomerSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ) - ), - ) - for_all_customers = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="forAllCustomers") - segments = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Segment"))), graphql_name="segments") - - -class PriceRuleDeactivatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("price_rule", "price_rule_user_errors") - price_rule = sgqlc.types.Field("PriceRule", graphql_name="priceRule") - price_rule_user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceRuleUserError"))), - graphql_name="priceRuleUserErrors", - ) - - -class PriceRuleDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_price_rule_id", "price_rule_user_errors", "shop") - deleted_price_rule_id = sgqlc.types.Field(ID, graphql_name="deletedPriceRuleId") - price_rule_user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceRuleUserError"))), - graphql_name="priceRuleUserErrors", - ) - shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") - - -class PriceRuleDiscountCodeConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceRuleDiscountCodeEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceRuleDiscountCode"))), graphql_name="nodes" - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class PriceRuleDiscountCodeCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("price_rule", "price_rule_discount_code", "price_rule_user_errors") - price_rule = sgqlc.types.Field("PriceRule", graphql_name="priceRule") - price_rule_discount_code = sgqlc.types.Field("PriceRuleDiscountCode", graphql_name="priceRuleDiscountCode") - price_rule_user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceRuleUserError"))), - graphql_name="priceRuleUserErrors", - ) - - -class PriceRuleDiscountCodeEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("PriceRuleDiscountCode"), graphql_name="node") - - -class PriceRuleDiscountCodeUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("price_rule", "price_rule_discount_code", "price_rule_user_errors") - price_rule = sgqlc.types.Field("PriceRule", graphql_name="priceRule") - price_rule_discount_code = sgqlc.types.Field("PriceRuleDiscountCode", graphql_name="priceRuleDiscountCode") - price_rule_user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceRuleUserError"))), - graphql_name="priceRuleUserErrors", - ) - - -class PriceRuleEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("PriceRule"), graphql_name="node") - - -class PriceRuleEntitlementToPrerequisiteQuantityRatio(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("entitlement_quantity", "prerequisite_quantity") - entitlement_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="entitlementQuantity") - prerequisite_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="prerequisiteQuantity") - - -class PriceRuleFixedAmountValue(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("amount",) - amount = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="amount") - - -class PriceRuleItemEntitlements(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("collections", "product_variants", "products", "target_all_line_items") - collections = sgqlc.types.Field( - sgqlc.types.non_null(CollectionConnection), - graphql_name="collections", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - product_variants = sgqlc.types.Field( - sgqlc.types.non_null("ProductVariantConnection"), - graphql_name="productVariants", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - products = sgqlc.types.Field( - sgqlc.types.non_null("ProductConnection"), - graphql_name="products", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - target_all_line_items = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="targetAllLineItems") - - -class PriceRuleLineItemPrerequisites(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("collections", "product_variants", "products") - collections = sgqlc.types.Field( - sgqlc.types.non_null(CollectionConnection), - graphql_name="collections", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - product_variants = sgqlc.types.Field( - sgqlc.types.non_null("ProductVariantConnection"), - graphql_name="productVariants", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - products = sgqlc.types.Field( - sgqlc.types.non_null("ProductConnection"), - graphql_name="products", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - - -class PriceRuleMoneyRange(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("greater_than", "greater_than_or_equal_to", "less_than", "less_than_or_equal_to") - greater_than = sgqlc.types.Field(Money, graphql_name="greaterThan") - greater_than_or_equal_to = sgqlc.types.Field(Money, graphql_name="greaterThanOrEqualTo") - less_than = sgqlc.types.Field(Money, graphql_name="lessThan") - less_than_or_equal_to = sgqlc.types.Field(Money, graphql_name="lessThanOrEqualTo") - - -class PriceRulePercentValue(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("percentage",) - percentage = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="percentage") - - -class PriceRulePrerequisiteToEntitlementQuantityRatio(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("entitlement_quantity", "prerequisite_quantity") - entitlement_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="entitlementQuantity") - prerequisite_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="prerequisiteQuantity") - - -class PriceRuleQuantityRange(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("greater_than", "greater_than_or_equal_to", "less_than", "less_than_or_equal_to") - greater_than = sgqlc.types.Field(Int, graphql_name="greaterThan") - greater_than_or_equal_to = sgqlc.types.Field(Int, graphql_name="greaterThanOrEqualTo") - less_than = sgqlc.types.Field(Int, graphql_name="lessThan") - less_than_or_equal_to = sgqlc.types.Field(Int, graphql_name="lessThanOrEqualTo") - - -class PriceRuleShareableUrl(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("target_item_image", "target_type", "title", "url") - target_item_image = sgqlc.types.Field("Image", graphql_name="targetItemImage") - target_type = sgqlc.types.Field(sgqlc.types.non_null(PriceRuleShareableUrlTargetType), graphql_name="targetType") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") - - -class PriceRuleShippingLineEntitlements(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("country_codes", "include_rest_of_world", "target_all_shipping_lines") - country_codes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CountryCode))), graphql_name="countryCodes" - ) - include_rest_of_world = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="includeRestOfWorld") - target_all_shipping_lines = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="targetAllShippingLines") - - -class PriceRuleUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("price_rule", "price_rule_discount_code", "price_rule_user_errors") - price_rule = sgqlc.types.Field("PriceRule", graphql_name="priceRule") - price_rule_discount_code = sgqlc.types.Field("PriceRuleDiscountCode", graphql_name="priceRuleDiscountCode") - price_rule_user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PriceRuleUserError"))), - graphql_name="priceRuleUserErrors", - ) - - -class PriceRuleValidityPeriod(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("end", "start") - end = sgqlc.types.Field(DateTime, graphql_name="end") - start = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="start") - - -class PricingPercentageValue(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("percentage",) - percentage = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="percentage") - - -class PrivateMetafieldConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PrivateMetafieldEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PrivateMetafield"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class PrivateMetafieldDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_private_metafield_id", "user_errors") - deleted_private_metafield_id = sgqlc.types.Field(ID, graphql_name="deletedPrivateMetafieldId") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class PrivateMetafieldEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("PrivateMetafield"), graphql_name="node") - - -class PrivateMetafieldUpsertPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("private_metafield", "user_errors") - private_metafield = sgqlc.types.Field("PrivateMetafield", graphql_name="privateMetafield") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class ProductAppendImagesPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("new_images", "product", "user_errors") - new_images = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("Image")), graphql_name="newImages") - product = sgqlc.types.Field("Product", graphql_name="product") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class ProductCategory(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("product_taxonomy_node",) - product_taxonomy_node = sgqlc.types.Field("ProductTaxonomyNode", graphql_name="productTaxonomyNode") - - -class ProductChangeStatusPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("product", "user_errors") - product = sgqlc.types.Field("Product", graphql_name="product") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductChangeStatusUserError"))), - graphql_name="userErrors", - ) - - -class ProductConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Product"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class ProductContextualPricing(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("max_variant_pricing", "min_variant_pricing", "price_range") - max_variant_pricing = sgqlc.types.Field("ProductVariantContextualPricing", graphql_name="maxVariantPricing") - min_variant_pricing = sgqlc.types.Field("ProductVariantContextualPricing", graphql_name="minVariantPricing") - price_range = sgqlc.types.Field(sgqlc.types.non_null("ProductPriceRangeV2"), graphql_name="priceRange") - - -class ProductCreateMediaPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("media", "media_user_errors", "product") - media = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(Media)), graphql_name="media") - media_user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MediaUserError"))), - graphql_name="mediaUserErrors", - ) - product = sgqlc.types.Field("Product", graphql_name="product") - - -class ProductCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("product", "shop", "user_errors") - product = sgqlc.types.Field("Product", graphql_name="product") - shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class ProductDeleteImagesPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_image_ids", "product", "user_errors") - deleted_image_ids = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), graphql_name="deletedImageIds" - ) - product = sgqlc.types.Field("Product", graphql_name="product") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class ProductDeleteMediaPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_media_ids", "deleted_product_image_ids", "media_user_errors", "product") - deleted_media_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="deletedMediaIds") - deleted_product_image_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="deletedProductImageIds") - media_user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MediaUserError"))), - graphql_name="mediaUserErrors", - ) - product = sgqlc.types.Field("Product", graphql_name="product") - - -class ProductDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_product_id", "shop", "user_errors") - deleted_product_id = sgqlc.types.Field(ID, graphql_name="deletedProductId") - shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class ProductDuplicatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("image_job", "new_product", "shop", "user_errors") - image_job = sgqlc.types.Field(Job, graphql_name="imageJob") - new_product = sgqlc.types.Field("Product", graphql_name="newProduct") - shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class ProductEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("Product"), graphql_name="node") - - -class ProductImageUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("image", "user_errors") - image = sgqlc.types.Field("Image", graphql_name="image") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class ProductJoinSellingPlanGroupsPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("product", "user_errors") - product = sgqlc.types.Field("Product", graphql_name="product") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupUserError"))), - graphql_name="userErrors", - ) - - -class ProductLeaveSellingPlanGroupsPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("product", "user_errors") - product = sgqlc.types.Field("Product", graphql_name="product") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupUserError"))), - graphql_name="userErrors", - ) - - -class ProductPriceRange(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("max_variant_price", "min_variant_price") - max_variant_price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="maxVariantPrice") - min_variant_price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="minVariantPrice") - - -class ProductPriceRangeV2(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("max_variant_price", "min_variant_price") - max_variant_price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="maxVariantPrice") - min_variant_price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="minVariantPrice") - - -class ProductPublication(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("channel", "is_published", "product", "publish_date") - channel = sgqlc.types.Field(sgqlc.types.non_null("Channel"), graphql_name="channel") - is_published = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isPublished") - product = sgqlc.types.Field(sgqlc.types.non_null("Product"), graphql_name="product") - publish_date = sgqlc.types.Field(DateTime, graphql_name="publishDate") - - -class ProductPublicationConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductPublicationEdge"))), graphql_name="edges" - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProductPublication))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class ProductPublicationEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(ProductPublication), graphql_name="node") - - -class ProductPublishPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("product", "shop", "user_errors") - product = sgqlc.types.Field("Product", graphql_name="product") - shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class ProductReorderImagesPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("job", "user_errors") - job = sgqlc.types.Field(Job, graphql_name="job") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class ProductReorderMediaPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("job", "media_user_errors") - job = sgqlc.types.Field(Job, graphql_name="job") - media_user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MediaUserError"))), - graphql_name="mediaUserErrors", - ) - - -class ProductResourceFeedback(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("feedback_generated_at", "messages", "product_id", "product_updated_at", "state") - feedback_generated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="feedbackGeneratedAt") - messages = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="messages") - product_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="productId") - product_updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="productUpdatedAt") - state = sgqlc.types.Field(sgqlc.types.non_null(ResourceFeedbackState), graphql_name="state") - - -class ProductUnpublishPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("product", "shop", "user_errors") - product = sgqlc.types.Field("Product", graphql_name="product") - shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class ProductUpdateMediaPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("media", "media_user_errors", "product") - media = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(Media)), graphql_name="media") - media_user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MediaUserError"))), - graphql_name="mediaUserErrors", - ) - product = sgqlc.types.Field("Product", graphql_name="product") - - -class ProductUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("product", "user_errors") - product = sgqlc.types.Field("Product", graphql_name="product") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class ProductVariantAppendMediaPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("product", "product_variants", "user_errors") - product = sgqlc.types.Field("Product", graphql_name="product") - product_variants = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariant")), graphql_name="productVariants") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MediaUserError"))), graphql_name="userErrors" - ) - - -class ProductVariantConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariantEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariant"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class ProductVariantContextualPricing(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("compare_at_price", "price") - compare_at_price = sgqlc.types.Field(MoneyV2, graphql_name="compareAtPrice") - price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="price") - - -class ProductVariantCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("product", "product_variant", "user_errors") - product = sgqlc.types.Field("Product", graphql_name="product") - product_variant = sgqlc.types.Field("ProductVariant", graphql_name="productVariant") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class ProductVariantDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_product_variant_id", "product", "user_errors") - deleted_product_variant_id = sgqlc.types.Field(ID, graphql_name="deletedProductVariantId") - product = sgqlc.types.Field("Product", graphql_name="product") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class ProductVariantDetachMediaPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("product", "product_variants", "user_errors") - product = sgqlc.types.Field("Product", graphql_name="product") - product_variants = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariant")), graphql_name="productVariants") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MediaUserError"))), graphql_name="userErrors" - ) - - -class ProductVariantEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("ProductVariant"), graphql_name="node") - - -class ProductVariantJoinSellingPlanGroupsPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("product_variant", "user_errors") - product_variant = sgqlc.types.Field("ProductVariant", graphql_name="productVariant") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupUserError"))), - graphql_name="userErrors", - ) - - -class ProductVariantLeaveSellingPlanGroupsPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("product_variant", "user_errors") - product_variant = sgqlc.types.Field("ProductVariant", graphql_name="productVariant") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupUserError"))), - graphql_name="userErrors", - ) - - -class ProductVariantPricePair(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("compare_at_price", "price") - compare_at_price = sgqlc.types.Field(MoneyV2, graphql_name="compareAtPrice") - price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="price") - - -class ProductVariantPricePairConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariantPricePairEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProductVariantPricePair))), graphql_name="nodes" - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class ProductVariantPricePairEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(ProductVariantPricePair), graphql_name="node") - - -class ProductVariantUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("product", "product_variant", "user_errors") - product = sgqlc.types.Field("Product", graphql_name="product") - product_variant = sgqlc.types.Field("ProductVariant", graphql_name="productVariant") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class ProductVariantsBulkCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("product", "product_variants", "user_errors") - product = sgqlc.types.Field("Product", graphql_name="product") - product_variants = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariant")), graphql_name="productVariants") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariantsBulkCreateUserError"))), - graphql_name="userErrors", - ) - - -class ProductVariantsBulkDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("product", "user_errors") - product = sgqlc.types.Field("Product", graphql_name="product") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariantsBulkDeleteUserError"))), - graphql_name="userErrors", - ) - - -class ProductVariantsBulkReorderPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("product", "user_errors") - product = sgqlc.types.Field("Product", graphql_name="product") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariantsBulkReorderUserError"))), - graphql_name="userErrors", - ) - - -class ProductVariantsBulkUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("product", "product_variants", "user_errors") - product = sgqlc.types.Field("Product", graphql_name="product") - product_variants = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariant")), graphql_name="productVariants") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductVariantsBulkUpdateUserError"))), - graphql_name="userErrors", - ) - - -class PubSubWebhookSubscriptionCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("user_errors", "webhook_subscription") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PubSubWebhookSubscriptionCreateUserError"))), - graphql_name="userErrors", - ) - webhook_subscription = sgqlc.types.Field("WebhookSubscription", graphql_name="webhookSubscription") - - -class PubSubWebhookSubscriptionUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("user_errors", "webhook_subscription") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PubSubWebhookSubscriptionUpdateUserError"))), - graphql_name="userErrors", - ) - webhook_subscription = sgqlc.types.Field("WebhookSubscription", graphql_name="webhookSubscription") - - -class PublicationConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PublicationEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Publication"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class PublicationEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("Publication"), graphql_name="node") - - -class Publishable(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ( - "available_publication_count", - "publication_count", - "published_on_current_publication", - "published_on_publication", - "resource_publications", - "resource_publications_v2", - "unpublished_publications", - ) - available_publication_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="availablePublicationCount") - publication_count = sgqlc.types.Field( - sgqlc.types.non_null(Int), - graphql_name="publicationCount", - args=sgqlc.types.ArgDict((("only_published", sgqlc.types.Arg(Boolean, graphql_name="onlyPublished", default=True)),)), - ) - published_on_current_publication = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="publishedOnCurrentPublication") - published_on_publication = sgqlc.types.Field( - sgqlc.types.non_null(Boolean), - graphql_name="publishedOnPublication", - args=sgqlc.types.ArgDict( - (("publication_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="publicationId", default=None)),) - ), - ) - resource_publications = sgqlc.types.Field( - sgqlc.types.non_null("ResourcePublicationConnection"), - graphql_name="resourcePublications", - args=sgqlc.types.ArgDict( - ( - ("only_published", sgqlc.types.Arg(Boolean, graphql_name="onlyPublished", default=True)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - resource_publications_v2 = sgqlc.types.Field( - sgqlc.types.non_null("ResourcePublicationV2Connection"), - graphql_name="resourcePublicationsV2", - args=sgqlc.types.ArgDict( - ( - ("only_published", sgqlc.types.Arg(Boolean, graphql_name="onlyPublished", default=True)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - unpublished_publications = sgqlc.types.Field( - sgqlc.types.non_null(PublicationConnection), - graphql_name="unpublishedPublications", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - - -class PublishablePublishPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("publishable", "shop", "user_errors") - publishable = sgqlc.types.Field(Publishable, graphql_name="publishable") - shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class PublishablePublishToCurrentChannelPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("publishable", "shop", "user_errors") - publishable = sgqlc.types.Field(Publishable, graphql_name="publishable") - shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class PublishableUnpublishPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("publishable", "shop", "user_errors") - publishable = sgqlc.types.Field(Publishable, graphql_name="publishable") - shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class PublishableUnpublishToCurrentChannelPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("publishable", "shop", "user_errors") - publishable = sgqlc.types.Field(Publishable, graphql_name="publishable") - shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class PublishedTranslation(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("key", "locale", "market_id", "value") - key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") - locale = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="locale") - market_id = sgqlc.types.Field(ID, graphql_name="marketId") - value = sgqlc.types.Field(String, graphql_name="value") - - -class PurchasingCompany(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("company", "contact", "location") - company = sgqlc.types.Field(sgqlc.types.non_null("Company"), graphql_name="company") - contact = sgqlc.types.Field("CompanyContact", graphql_name="contact") - location = sgqlc.types.Field(sgqlc.types.non_null("CompanyLocation"), graphql_name="location") - - -class QueryRoot(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "app", - "app_by_handle", - "app_by_key", - "app_discount_type", - "app_discount_types", - "app_installation", - "app_installations", - "automatic_discount_node", - "automatic_discount_nodes", - "automatic_discount_saved_searches", - "available_carrier_services", - "available_locales", - "carrier_service", - "checkout_profile", - "checkout_profiles", - "code_discount_node", - "code_discount_node_by_code", - "code_discount_nodes", - "code_discount_saved_searches", - "collection", - "collection_by_handle", - "collection_rules_conditions", - "collection_saved_searches", - "collections", - "companies", - "company", - "company_contact", - "company_contact_role", - "company_count", - "company_location", - "company_locations", - "current_app_installation", - "current_bulk_operation", - "customer", - "customer_payment_method", - "customer_segment_members", - "customer_segment_membership", - "customers", - "deletion_events", - "delivery_profile", - "delivery_profiles", - "delivery_settings", - "discount_code_count", - "discount_node", - "discount_nodes", - "discount_redeem_code_bulk_creation", - "discount_redeem_code_saved_searches", - "dispute", - "dispute_evidence", - "domain", - "draft_order", - "draft_order_saved_searches", - "draft_order_tag", - "draft_orders", - "file_saved_searches", - "files", - "fulfillment", - "fulfillment_order", - "fulfillment_service", - "gift_card", - "gift_cards", - "gift_cards_count", - "inventory_item", - "inventory_items", - "inventory_level", - "job", - "location", - "locations", - "locations_available_for_delivery_profiles_connection", - "manual_holds_fulfillment_orders", - "market", - "market_by_geography", - "market_localizable_resource", - "market_localizable_resources", - "market_localizable_resources_by_ids", - "marketing_activities", - "marketing_activity", - "marketing_event", - "marketing_events", - "markets", - "metafield", - "metafield_definition", - "metafield_definition_types", - "metafield_definitions", - "metafield_storefront_visibilities", - "metafield_storefront_visibility", - "node", - "nodes", - "order", - "order_payment_status", - "order_saved_searches", - "orders", - "payment_terms_templates", - "price_list", - "price_lists", - "price_rule", - "price_rule_saved_searches", - "price_rules", - "primary_market", - "private_metafield", - "private_metafields", - "product", - "product_by_handle", - "product_resource_feedback", - "product_saved_searches", - "product_variant", - "product_variants", - "products", - "public_api_versions", - "publication", - "publications", - "refund", - "script_tag", - "script_tags", - "segment", - "segment_count", - "segment_filter_suggestions", - "segment_filters", - "segment_migrations", - "segment_value_suggestions", - "segments", - "selling_plan_group", - "selling_plan_groups", - "shop", - "shop_locales", - "shopify_payments_account", - "staff_member", - "standard_metafield_definition_templates", - "subscription_billing_attempt", - "subscription_billing_cycle", - "subscription_billing_cycles", - "subscription_contract", - "subscription_contracts", - "subscription_draft", - "tender_transactions", - "translatable_resource", - "translatable_resources", - "translatable_resources_by_ids", - "url_redirect", - "url_redirect_import", - "url_redirect_saved_searches", - "url_redirects", - "web_pixel", - "webhook_subscription", - "webhook_subscriptions", - ) - app = sgqlc.types.Field( - "App", - graphql_name="app", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(ID, graphql_name="id", default=None)),)), - ) - app_by_handle = sgqlc.types.Field( - "App", - graphql_name="appByHandle", - args=sgqlc.types.ArgDict((("handle", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="handle", default=None)),)), - ) - app_by_key = sgqlc.types.Field( - "App", - graphql_name="appByKey", - args=sgqlc.types.ArgDict((("api_key", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="apiKey", default=None)),)), - ) - app_discount_type = sgqlc.types.Field( - AppDiscountType, - graphql_name="appDiscountType", - args=sgqlc.types.ArgDict( - (("function_id", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="functionId", default=None)),) - ), - ) - app_discount_types = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(AppDiscountType))), - graphql_name="appDiscountTypes", - ) - app_installation = sgqlc.types.Field( - "AppInstallation", - graphql_name="appInstallation", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(ID, graphql_name="id", default=None)),)), - ) - app_installations = sgqlc.types.Field( - sgqlc.types.non_null(AppInstallationConnection), - graphql_name="appInstallations", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(AppInstallationSortKeys, graphql_name="sortKey", default="INSTALLED_AT")), - ("category", sgqlc.types.Arg(AppInstallationCategory, graphql_name="category", default=None)), - ("privacy", sgqlc.types.Arg(AppInstallationPrivacy, graphql_name="privacy", default="PUBLIC")), - ) - ), - ) - automatic_discount_node = sgqlc.types.Field( - "DiscountAutomaticNode", - graphql_name="automaticDiscountNode", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - automatic_discount_nodes = sgqlc.types.Field( - sgqlc.types.non_null(DiscountAutomaticNodeConnection), - graphql_name="automaticDiscountNodes", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(AutomaticDiscountSortKeys, graphql_name="sortKey", default="CREATED_AT")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ) - ), - ) - automatic_discount_saved_searches = sgqlc.types.Field( - sgqlc.types.non_null("SavedSearchConnection"), - graphql_name="automaticDiscountSavedSearches", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - available_carrier_services = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DeliveryCarrierServiceAndLocations))), - graphql_name="availableCarrierServices", - ) - available_locales = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Locale))), graphql_name="availableLocales" - ) - carrier_service = sgqlc.types.Field( - "DeliveryCarrierService", - graphql_name="carrierService", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - checkout_profile = sgqlc.types.Field( - "CheckoutProfile", - graphql_name="checkoutProfile", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - checkout_profiles = sgqlc.types.Field( - sgqlc.types.non_null(CheckoutProfileConnection), - graphql_name="checkoutProfiles", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(CheckoutProfileSortKeys, graphql_name="sortKey", default="UPDATED_AT")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - code_discount_node = sgqlc.types.Field( - "DiscountCodeNode", - graphql_name="codeDiscountNode", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - code_discount_node_by_code = sgqlc.types.Field( - "DiscountCodeNode", - graphql_name="codeDiscountNodeByCode", - args=sgqlc.types.ArgDict((("code", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="code", default=None)),)), - ) - code_discount_nodes = sgqlc.types.Field( - sgqlc.types.non_null(DiscountCodeNodeConnection), - graphql_name="codeDiscountNodes", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(CodeDiscountSortKeys, graphql_name="sortKey", default="CREATED_AT")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ) - ), - ) - code_discount_saved_searches = sgqlc.types.Field( - sgqlc.types.non_null("SavedSearchConnection"), - graphql_name="codeDiscountSavedSearches", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - collection = sgqlc.types.Field( - "Collection", - graphql_name="collection", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - collection_by_handle = sgqlc.types.Field( - "Collection", - graphql_name="collectionByHandle", - args=sgqlc.types.ArgDict((("handle", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="handle", default=None)),)), - ) - collection_rules_conditions = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CollectionRuleConditions))), - graphql_name="collectionRulesConditions", - ) - collection_saved_searches = sgqlc.types.Field( - sgqlc.types.non_null("SavedSearchConnection"), - graphql_name="collectionSavedSearches", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - collections = sgqlc.types.Field( - sgqlc.types.non_null(CollectionConnection), - graphql_name="collections", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(CollectionSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ) - ), - ) - companies = sgqlc.types.Field( - sgqlc.types.non_null(CompanyConnection), - graphql_name="companies", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(CompanySortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - company = sgqlc.types.Field( - "Company", - graphql_name="company", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - company_contact = sgqlc.types.Field( - "CompanyContact", - graphql_name="companyContact", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - company_contact_role = sgqlc.types.Field( - "CompanyContactRole", - graphql_name="companyContactRole", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - company_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="companyCount") - company_location = sgqlc.types.Field( - "CompanyLocation", - graphql_name="companyLocation", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - company_locations = sgqlc.types.Field( - sgqlc.types.non_null(CompanyLocationConnection), - graphql_name="companyLocations", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(CompanyLocationSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - current_app_installation = sgqlc.types.Field(sgqlc.types.non_null("AppInstallation"), graphql_name="currentAppInstallation") - current_bulk_operation = sgqlc.types.Field( - "BulkOperation", - graphql_name="currentBulkOperation", - args=sgqlc.types.ArgDict((("type", sgqlc.types.Arg(BulkOperationType, graphql_name="type", default="QUERY")),)), - ) - customer = sgqlc.types.Field( - "Customer", - graphql_name="customer", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - customer_payment_method = sgqlc.types.Field( - "CustomerPaymentMethod", - graphql_name="customerPaymentMethod", - args=sgqlc.types.ArgDict( - ( - ("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)), - ("show_revoked", sgqlc.types.Arg(Boolean, graphql_name="showRevoked", default=False)), - ) - ), - ) - customer_segment_members = sgqlc.types.Field( - sgqlc.types.non_null(CustomerSegmentMemberConnection), - graphql_name="customerSegmentMembers", - args=sgqlc.types.ArgDict( - ( - ("segment_id", sgqlc.types.Arg(ID, graphql_name="segmentId", default=None)), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("timezone", sgqlc.types.Arg(String, graphql_name="timezone", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(String, graphql_name="sortKey", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ) - ), - ) - customer_segment_membership = sgqlc.types.Field( - sgqlc.types.non_null("SegmentMembershipResponse"), - graphql_name="customerSegmentMembership", - args=sgqlc.types.ArgDict( - ( - ( - "segment_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="segmentIds", - default=None, - ), - ), - ("customer_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="customerId", default=None)), - ) - ), - ) - customers = sgqlc.types.Field( - sgqlc.types.non_null(CustomerConnection), - graphql_name="customers", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(CustomerSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - deletion_events = sgqlc.types.Field( - sgqlc.types.non_null(DeletionEventConnection), - graphql_name="deletionEvents", - args=sgqlc.types.ArgDict( - ( - ( - "subject_types", - sgqlc.types.Arg( - sgqlc.types.list_of(sgqlc.types.non_null(DeletionEventSubjectType)), - graphql_name="subjectTypes", - default=None, - ), - ), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(DeletionEventSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - delivery_profile = sgqlc.types.Field( - "DeliveryProfile", - graphql_name="deliveryProfile", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - delivery_profiles = sgqlc.types.Field( - sgqlc.types.non_null(DeliveryProfileConnection), - graphql_name="deliveryProfiles", - args=sgqlc.types.ArgDict( - ( - ("merchant_owned_only", sgqlc.types.Arg(Boolean, graphql_name="merchantOwnedOnly", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - delivery_settings = sgqlc.types.Field(DeliverySetting, graphql_name="deliverySettings") - discount_code_count = sgqlc.types.Field( - sgqlc.types.non_null(Int), - graphql_name="discountCodeCount", - args=sgqlc.types.ArgDict((("query", sgqlc.types.Arg(String, graphql_name="query", default=None)),)), - ) - discount_node = sgqlc.types.Field( - "DiscountNode", - graphql_name="discountNode", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - discount_nodes = sgqlc.types.Field( - sgqlc.types.non_null(DiscountNodeConnection), - graphql_name="discountNodes", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(DiscountSortKeys, graphql_name="sortKey", default="CREATED_AT")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ) - ), - ) - discount_redeem_code_bulk_creation = sgqlc.types.Field( - "DiscountRedeemCodeBulkCreation", - graphql_name="discountRedeemCodeBulkCreation", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - discount_redeem_code_saved_searches = sgqlc.types.Field( - sgqlc.types.non_null("SavedSearchConnection"), - graphql_name="discountRedeemCodeSavedSearches", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(DiscountCodeSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - dispute = sgqlc.types.Field( - "ShopifyPaymentsDispute", - graphql_name="dispute", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - dispute_evidence = sgqlc.types.Field( - "ShopifyPaymentsDisputeEvidence", - graphql_name="disputeEvidence", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - domain = sgqlc.types.Field( - "Domain", - graphql_name="domain", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - draft_order = sgqlc.types.Field( - "DraftOrder", - graphql_name="draftOrder", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - draft_order_saved_searches = sgqlc.types.Field( - sgqlc.types.non_null("SavedSearchConnection"), - graphql_name="draftOrderSavedSearches", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - draft_order_tag = sgqlc.types.Field( - "DraftOrderTag", - graphql_name="draftOrderTag", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - draft_orders = sgqlc.types.Field( - sgqlc.types.non_null(DraftOrderConnection), - graphql_name="draftOrders", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(DraftOrderSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ) - ), - ) - file_saved_searches = sgqlc.types.Field( - sgqlc.types.non_null("SavedSearchConnection"), - graphql_name="fileSavedSearches", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - files = sgqlc.types.Field( - sgqlc.types.non_null(FileConnection), - graphql_name="files", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(FileSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ) - ), - ) - fulfillment = sgqlc.types.Field( - "Fulfillment", - graphql_name="fulfillment", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - fulfillment_order = sgqlc.types.Field( - "FulfillmentOrder", - graphql_name="fulfillmentOrder", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - fulfillment_service = sgqlc.types.Field( - FulfillmentService, - graphql_name="fulfillmentService", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - gift_card = sgqlc.types.Field( - "GiftCard", - graphql_name="giftCard", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - gift_cards = sgqlc.types.Field( - sgqlc.types.non_null(GiftCardConnection), - graphql_name="giftCards", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(GiftCardSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ) - ), - ) - gift_cards_count = sgqlc.types.Field( - sgqlc.types.non_null(UnsignedInt64), - graphql_name="giftCardsCount", - args=sgqlc.types.ArgDict((("enabled", sgqlc.types.Arg(Boolean, graphql_name="enabled", default=None)),)), - ) - inventory_item = sgqlc.types.Field( - "InventoryItem", - graphql_name="inventoryItem", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - inventory_items = sgqlc.types.Field( - sgqlc.types.non_null(InventoryItemConnection), - graphql_name="inventoryItems", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - inventory_level = sgqlc.types.Field( - "InventoryLevel", - graphql_name="inventoryLevel", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - job = sgqlc.types.Field( - Job, - graphql_name="job", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - location = sgqlc.types.Field( - "Location", - graphql_name="location", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(ID, graphql_name="id", default=None)),)), - ) - locations = sgqlc.types.Field( - sgqlc.types.non_null(LocationConnection), - graphql_name="locations", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(LocationSortKeys, graphql_name="sortKey", default="NAME")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("include_legacy", sgqlc.types.Arg(Boolean, graphql_name="includeLegacy", default=False)), - ("include_inactive", sgqlc.types.Arg(Boolean, graphql_name="includeInactive", default=False)), - ) - ), - ) - locations_available_for_delivery_profiles_connection = sgqlc.types.Field( - sgqlc.types.non_null(LocationConnection), - graphql_name="locationsAvailableForDeliveryProfilesConnection", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - manual_holds_fulfillment_orders = sgqlc.types.Field( - sgqlc.types.non_null(FulfillmentOrderConnection), - graphql_name="manualHoldsFulfillmentOrders", - args=sgqlc.types.ArgDict( - ( - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - market = sgqlc.types.Field( - "Market", - graphql_name="market", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - market_by_geography = sgqlc.types.Field( - "Market", - graphql_name="marketByGeography", - args=sgqlc.types.ArgDict( - ( - ( - "country_code", - sgqlc.types.Arg(sgqlc.types.non_null(CountryCode), graphql_name="countryCode", default=None), - ), - ) - ), - ) - market_localizable_resource = sgqlc.types.Field( - MarketLocalizableResource, - graphql_name="marketLocalizableResource", - args=sgqlc.types.ArgDict((("resource_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="resourceId", default=None)),)), - ) - market_localizable_resources = sgqlc.types.Field( - sgqlc.types.non_null(MarketLocalizableResourceConnection), - graphql_name="marketLocalizableResources", - args=sgqlc.types.ArgDict( - ( - ( - "resource_type", - sgqlc.types.Arg(sgqlc.types.non_null(MarketLocalizableResourceType), graphql_name="resourceType", default=None), - ), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - market_localizable_resources_by_ids = sgqlc.types.Field( - sgqlc.types.non_null(MarketLocalizableResourceConnection), - graphql_name="marketLocalizableResourcesByIds", - args=sgqlc.types.ArgDict( - ( - ( - "resource_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="resourceIds", - default=None, - ), - ), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - marketing_activities = sgqlc.types.Field( - sgqlc.types.non_null(MarketingActivityConnection), - graphql_name="marketingActivities", - args=sgqlc.types.ArgDict( - ( - ( - "marketing_activity_ids", - sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="marketingActivityIds", default=()), - ), - ( - "remote_ids", - sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="remoteIds", default=()), - ), - ("utm", sgqlc.types.Arg(UTMInput, graphql_name="utm", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(MarketingActivitySortKeys, graphql_name="sortKey", default="CREATED_AT")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ) - ), - ) - marketing_activity = sgqlc.types.Field( - "MarketingActivity", - graphql_name="marketingActivity", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - marketing_event = sgqlc.types.Field( - "MarketingEvent", - graphql_name="marketingEvent", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - marketing_events = sgqlc.types.Field( - sgqlc.types.non_null(MarketingEventConnection), - graphql_name="marketingEvents", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(MarketingEventSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - markets = sgqlc.types.Field( - sgqlc.types.non_null(MarketConnection), - graphql_name="markets", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - metafield = sgqlc.types.Field( - "Metafield", - graphql_name="metafield", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - metafield_definition = sgqlc.types.Field( - "MetafieldDefinition", - graphql_name="metafieldDefinition", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - metafield_definition_types = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MetafieldDefinitionType))), - graphql_name="metafieldDefinitionTypes", - ) - metafield_definitions = sgqlc.types.Field( - sgqlc.types.non_null(MetafieldDefinitionConnection), - graphql_name="metafieldDefinitions", - args=sgqlc.types.ArgDict( - ( - ("key", sgqlc.types.Arg(String, graphql_name="key", default=None)), - ("namespace", sgqlc.types.Arg(String, graphql_name="namespace", default=None)), - ( - "owner_type", - sgqlc.types.Arg(sgqlc.types.non_null(MetafieldOwnerType), graphql_name="ownerType", default=None), - ), - ( - "pinned_status", - sgqlc.types.Arg(MetafieldDefinitionPinnedStatus, graphql_name="pinnedStatus", default="ANY"), - ), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(MetafieldDefinitionSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - metafield_storefront_visibilities = sgqlc.types.Field( - sgqlc.types.non_null(MetafieldStorefrontVisibilityConnection), - graphql_name="metafieldStorefrontVisibilities", - args=sgqlc.types.ArgDict( - ( - ("namespace", sgqlc.types.Arg(String, graphql_name="namespace", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - metafield_storefront_visibility = sgqlc.types.Field( - "MetafieldStorefrontVisibility", - graphql_name="metafieldStorefrontVisibility", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - node = sgqlc.types.Field( - Node, - graphql_name="node", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(Node)), - graphql_name="nodes", - args=sgqlc.types.ArgDict( - ( - ( - "ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="ids", - default=None, - ), - ), - ) - ), - ) - order = sgqlc.types.Field( - "Order", - graphql_name="order", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - order_payment_status = sgqlc.types.Field( - OrderPaymentStatus, - graphql_name="orderPaymentStatus", - args=sgqlc.types.ArgDict( - ( - ( - "payment_reference_id", - sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="paymentReferenceId", default=None), - ), - ("order_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="orderId", default=None)), - ) - ), - ) - order_saved_searches = sgqlc.types.Field( - sgqlc.types.non_null("SavedSearchConnection"), - graphql_name="orderSavedSearches", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - orders = sgqlc.types.Field( - sgqlc.types.non_null(OrderConnection), - graphql_name="orders", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(OrderSortKeys, graphql_name="sortKey", default="PROCESSED_AT")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ) - ), - ) - payment_terms_templates = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("PaymentTermsTemplate"))), - graphql_name="paymentTermsTemplates", - args=sgqlc.types.ArgDict( - (("payment_terms_type", sgqlc.types.Arg(PaymentTermsType, graphql_name="paymentTermsType", default=None)),) - ), - ) - price_list = sgqlc.types.Field( - "PriceList", - graphql_name="priceList", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - price_lists = sgqlc.types.Field( - sgqlc.types.non_null(PriceListConnection), - graphql_name="priceLists", - args=sgqlc.types.ArgDict( - ( - ("match_rule", sgqlc.types.Arg(PriceListContext, graphql_name="matchRule", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(PriceListSortKeys, graphql_name="sortKey", default="ID")), - ) - ), - ) - price_rule = sgqlc.types.Field( - "PriceRule", - graphql_name="priceRule", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - price_rule_saved_searches = sgqlc.types.Field( - sgqlc.types.non_null("SavedSearchConnection"), - graphql_name="priceRuleSavedSearches", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - price_rules = sgqlc.types.Field( - sgqlc.types.non_null(PriceRuleConnection), - graphql_name="priceRules", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(PriceRuleSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ) - ), - ) - primary_market = sgqlc.types.Field(sgqlc.types.non_null("Market"), graphql_name="primaryMarket") - private_metafield = sgqlc.types.Field( - "PrivateMetafield", - graphql_name="privateMetafield", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - private_metafields = sgqlc.types.Field( - sgqlc.types.non_null(PrivateMetafieldConnection), - graphql_name="privateMetafields", - args=sgqlc.types.ArgDict( - ( - ("namespace", sgqlc.types.Arg(String, graphql_name="namespace", default=None)), - ("owner", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="owner", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - product = sgqlc.types.Field( - "Product", - graphql_name="product", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - product_by_handle = sgqlc.types.Field( - "Product", - graphql_name="productByHandle", - args=sgqlc.types.ArgDict((("handle", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="handle", default=None)),)), - ) - product_resource_feedback = sgqlc.types.Field( - ProductResourceFeedback, - graphql_name="productResourceFeedback", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - product_saved_searches = sgqlc.types.Field( - sgqlc.types.non_null("SavedSearchConnection"), - graphql_name="productSavedSearches", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - product_variant = sgqlc.types.Field( - "ProductVariant", - graphql_name="productVariant", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - product_variants = sgqlc.types.Field( - sgqlc.types.non_null(ProductVariantConnection), - graphql_name="productVariants", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(ProductVariantSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ) - ), - ) - products = sgqlc.types.Field( - sgqlc.types.non_null(ProductConnection), - graphql_name="products", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(ProductSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ) - ), - ) - public_api_versions = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ApiVersion))), graphql_name="publicApiVersions" - ) - publication = sgqlc.types.Field( - "Publication", - graphql_name="publication", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - publications = sgqlc.types.Field( - sgqlc.types.non_null(PublicationConnection), - graphql_name="publications", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - refund = sgqlc.types.Field( - "Refund", - graphql_name="refund", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - script_tag = sgqlc.types.Field( - "ScriptTag", - graphql_name="scriptTag", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - script_tags = sgqlc.types.Field( - sgqlc.types.non_null("ScriptTagConnection"), - graphql_name="scriptTags", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("src", sgqlc.types.Arg(URL, graphql_name="src", default=None)), - ) - ), - ) - segment = sgqlc.types.Field( - "Segment", - graphql_name="segment", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - segment_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="segmentCount") - segment_filter_suggestions = sgqlc.types.Field( - sgqlc.types.non_null("SegmentFilterConnection"), - graphql_name="segmentFilterSuggestions", - args=sgqlc.types.ArgDict( - ( - ("search", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="search", default=None)), - ("first", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ) - ), - ) - segment_filters = sgqlc.types.Field( - sgqlc.types.non_null("SegmentFilterConnection"), - graphql_name="segmentFilters", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ) - ), - ) - segment_migrations = sgqlc.types.Field( - sgqlc.types.non_null("SegmentMigrationConnection"), - graphql_name="segmentMigrations", - args=sgqlc.types.ArgDict( - ( - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ) - ), - ) - segment_value_suggestions = sgqlc.types.Field( - sgqlc.types.non_null("SegmentValueConnection"), - graphql_name="segmentValueSuggestions", - args=sgqlc.types.ArgDict( - ( - ("search", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="search", default=None)), - ("filter_query_name", sgqlc.types.Arg(String, graphql_name="filterQueryName", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ) - ), - ) - segments = sgqlc.types.Field( - sgqlc.types.non_null("SegmentConnection"), - graphql_name="segments", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(SegmentSortKeys, graphql_name="sortKey", default="CREATION_DATE")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - selling_plan_group = sgqlc.types.Field( - "SellingPlanGroup", - graphql_name="sellingPlanGroup", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - selling_plan_groups = sgqlc.types.Field( - sgqlc.types.non_null("SellingPlanGroupConnection"), - graphql_name="sellingPlanGroups", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(SellingPlanGroupSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") - shop_locales = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopLocale"))), - graphql_name="shopLocales", - args=sgqlc.types.ArgDict((("published", sgqlc.types.Arg(Boolean, graphql_name="published", default=False)),)), - ) - shopify_payments_account = sgqlc.types.Field("ShopifyPaymentsAccount", graphql_name="shopifyPaymentsAccount") - staff_member = sgqlc.types.Field( - "StaffMember", - graphql_name="staffMember", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(ID, graphql_name="id", default=None)),)), - ) - standard_metafield_definition_templates = sgqlc.types.Field( - sgqlc.types.non_null("StandardMetafieldDefinitionTemplateConnection"), - graphql_name="standardMetafieldDefinitionTemplates", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - subscription_billing_attempt = sgqlc.types.Field( - "SubscriptionBillingAttempt", - graphql_name="subscriptionBillingAttempt", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - subscription_billing_cycle = sgqlc.types.Field( - "SubscriptionBillingCycle", - graphql_name="subscriptionBillingCycle", - args=sgqlc.types.ArgDict( - ( - ( - "billing_cycle_input", - sgqlc.types.Arg( - sgqlc.types.non_null(SubscriptionBillingCycleInput), - graphql_name="billingCycleInput", - default=None, - ), - ), - ) - ), - ) - subscription_billing_cycles = sgqlc.types.Field( - sgqlc.types.non_null("SubscriptionBillingCycleConnection"), - graphql_name="subscriptionBillingCycles", - args=sgqlc.types.ArgDict( - ( - ("contract_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="contractId", default=None)), - ( - "billing_cycles_date_range_selector", - sgqlc.types.Arg( - SubscriptionBillingCyclesDateRangeSelector, - graphql_name="billingCyclesDateRangeSelector", - default=None, - ), - ), - ( - "billing_cycles_index_range_selector", - sgqlc.types.Arg( - SubscriptionBillingCyclesIndexRangeSelector, - graphql_name="billingCyclesIndexRangeSelector", - default=None, - ), - ), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ( - "sort_key", - sgqlc.types.Arg(SubscriptionBillingCyclesSortKeys, graphql_name="sortKey", default="CYCLE_INDEX"), - ), - ) - ), - ) - subscription_contract = sgqlc.types.Field( - "SubscriptionContract", - graphql_name="subscriptionContract", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - subscription_contracts = sgqlc.types.Field( - sgqlc.types.non_null("SubscriptionContractConnection"), - graphql_name="subscriptionContracts", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - subscription_draft = sgqlc.types.Field( - "SubscriptionDraft", - graphql_name="subscriptionDraft", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - tender_transactions = sgqlc.types.Field( - sgqlc.types.non_null("TenderTransactionConnection"), - graphql_name="tenderTransactions", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - translatable_resource = sgqlc.types.Field( - "TranslatableResource", - graphql_name="translatableResource", - args=sgqlc.types.ArgDict((("resource_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="resourceId", default=None)),)), - ) - translatable_resources = sgqlc.types.Field( - sgqlc.types.non_null("TranslatableResourceConnection"), - graphql_name="translatableResources", - args=sgqlc.types.ArgDict( - ( - ( - "resource_type", - sgqlc.types.Arg(sgqlc.types.non_null(TranslatableResourceType), graphql_name="resourceType", default=None), - ), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - translatable_resources_by_ids = sgqlc.types.Field( - sgqlc.types.non_null("TranslatableResourceConnection"), - graphql_name="translatableResourcesByIds", - args=sgqlc.types.ArgDict( - ( - ( - "resource_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="resourceIds", - default=None, - ), - ), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - url_redirect = sgqlc.types.Field( - "UrlRedirect", - graphql_name="urlRedirect", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - url_redirect_import = sgqlc.types.Field( - "UrlRedirectImport", - graphql_name="urlRedirectImport", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - url_redirect_saved_searches = sgqlc.types.Field( - sgqlc.types.non_null("SavedSearchConnection"), - graphql_name="urlRedirectSavedSearches", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - url_redirects = sgqlc.types.Field( - sgqlc.types.non_null("UrlRedirectConnection"), - graphql_name="urlRedirects", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(UrlRedirectSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ) - ), - ) - web_pixel = sgqlc.types.Field( - "WebPixel", - graphql_name="webPixel", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - webhook_subscription = sgqlc.types.Field( - "WebhookSubscription", - graphql_name="webhookSubscription", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - webhook_subscriptions = sgqlc.types.Field( - sgqlc.types.non_null("WebhookSubscriptionConnection"), - graphql_name="webhookSubscriptions", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ( - "sort_key", - sgqlc.types.Arg(WebhookSubscriptionSortKeys, graphql_name="sortKey", default="CREATED_AT"), - ), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("callback_url", sgqlc.types.Arg(URL, graphql_name="callbackUrl", default=None)), - ("format", sgqlc.types.Arg(WebhookSubscriptionFormat, graphql_name="format", default=None)), - ( - "topics", - sgqlc.types.Arg( - sgqlc.types.list_of(sgqlc.types.non_null(WebhookSubscriptionTopic)), - graphql_name="topics", - default=None, - ), - ), - ) - ), - ) - - -class RefundCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("order", "refund", "user_errors") - order = sgqlc.types.Field("Order", graphql_name="order") - refund = sgqlc.types.Field("Refund", graphql_name="refund") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class RefundDuty(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("amount_set", "original_duty") - amount_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="amountSet") - original_duty = sgqlc.types.Field("Duty", graphql_name="originalDuty") - - -class RefundLineItem(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "line_item", - "location", - "price_set", - "quantity", - "restock_type", - "restocked", - "subtotal_set", - "total_tax_set", - ) - line_item = sgqlc.types.Field(sgqlc.types.non_null("LineItem"), graphql_name="lineItem") - location = sgqlc.types.Field("Location", graphql_name="location") - price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="priceSet") - quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") - restock_type = sgqlc.types.Field(sgqlc.types.non_null(RefundLineItemRestockType), graphql_name="restockType") - restocked = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="restocked") - subtotal_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="subtotalSet") - total_tax_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalTaxSet") - - -class RefundLineItemConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("RefundLineItemEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(RefundLineItem))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class RefundLineItemEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(RefundLineItem), graphql_name="node") - - -class ResourceAlert(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("actions", "content", "dismissible_handle", "icon", "severity", "title") - actions = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ResourceAlertAction"))), graphql_name="actions" - ) - content = sgqlc.types.Field(sgqlc.types.non_null(HTML), graphql_name="content") - dismissible_handle = sgqlc.types.Field(String, graphql_name="dismissibleHandle") - icon = sgqlc.types.Field(ResourceAlertIcon, graphql_name="icon") - severity = sgqlc.types.Field(sgqlc.types.non_null(ResourceAlertSeverity), graphql_name="severity") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - - -class ResourceAlertAction(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("primary", "show", "title", "url") - primary = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="primary") - show = sgqlc.types.Field(String, graphql_name="show") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") - - -class ResourceFeedback(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("details", "summary") - details = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(AppFeedback))), graphql_name="details") - summary = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="summary") - - -class ResourceLimit(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("available", "quantity_available", "quantity_limit", "quantity_used") - available = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="available") - quantity_available = sgqlc.types.Field(Int, graphql_name="quantityAvailable") - quantity_limit = sgqlc.types.Field(Int, graphql_name="quantityLimit") - quantity_used = sgqlc.types.Field(Int, graphql_name="quantityUsed") - - -class ResourcePublication(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("is_published", "publication", "publish_date", "publishable") - is_published = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isPublished") - publication = sgqlc.types.Field(sgqlc.types.non_null("Publication"), graphql_name="publication") - publish_date = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="publishDate") - publishable = sgqlc.types.Field(sgqlc.types.non_null(Publishable), graphql_name="publishable") - - -class ResourcePublicationConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ResourcePublicationEdge"))), graphql_name="edges" - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ResourcePublication))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class ResourcePublicationEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(ResourcePublication), graphql_name="node") - - -class ResourcePublicationV2(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("is_published", "publication", "publish_date", "publishable") - is_published = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isPublished") - publication = sgqlc.types.Field(sgqlc.types.non_null("Publication"), graphql_name="publication") - publish_date = sgqlc.types.Field(DateTime, graphql_name="publishDate") - publishable = sgqlc.types.Field(sgqlc.types.non_null(Publishable), graphql_name="publishable") - - -class ResourcePublicationV2Connection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ResourcePublicationV2Edge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ResourcePublicationV2))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class ResourcePublicationV2Edge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(ResourcePublicationV2), graphql_name="node") - - -class SEO(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("description", "title") - description = sgqlc.types.Field(String, graphql_name="description") - title = sgqlc.types.Field(String, graphql_name="title") - - -class Sale(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ( - "action_type", - "id", - "line_type", - "quantity", - "taxes", - "total_amount", - "total_discount_amount_after_taxes", - "total_discount_amount_before_taxes", - "total_tax_amount", - ) - action_type = sgqlc.types.Field(sgqlc.types.non_null(SaleActionType), graphql_name="actionType") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - line_type = sgqlc.types.Field(sgqlc.types.non_null(SaleLineType), graphql_name="lineType") - quantity = sgqlc.types.Field(Int, graphql_name="quantity") - taxes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SaleTax"))), graphql_name="taxes") - total_amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalAmount") - total_discount_amount_after_taxes = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalDiscountAmountAfterTaxes") - total_discount_amount_before_taxes = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalDiscountAmountBeforeTaxes") - total_tax_amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalTaxAmount") - - -class SaleConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SaleEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Sale))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class SaleEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(Sale), graphql_name="node") - - -class SaleTax(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("amount", "id", "tax_line") - amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="amount") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - tax_line = sgqlc.types.Field(sgqlc.types.non_null("TaxLine"), graphql_name="taxLine") - - -class SalesAgreement(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ("app", "happened_at", "id", "reason", "sales", "user") - app = sgqlc.types.Field("App", graphql_name="app") - happened_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="happenedAt") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - reason = sgqlc.types.Field(sgqlc.types.non_null(OrderActionType), graphql_name="reason") - sales = sgqlc.types.Field( - sgqlc.types.non_null(SaleConnection), - graphql_name="sales", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - user = sgqlc.types.Field("StaffMember", graphql_name="user") - - -class SalesAgreementConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SalesAgreementEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SalesAgreement))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class SalesAgreementEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(SalesAgreement), graphql_name="node") - - -class SavedSearchConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SavedSearchEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SavedSearch"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class SavedSearchCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("saved_search", "user_errors") - saved_search = sgqlc.types.Field("SavedSearch", graphql_name="savedSearch") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class SavedSearchDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_saved_search_id", "shop", "user_errors") - deleted_saved_search_id = sgqlc.types.Field(ID, graphql_name="deletedSavedSearchId") - shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class SavedSearchEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("SavedSearch"), graphql_name="node") - - -class SavedSearchUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("saved_search", "user_errors") - saved_search = sgqlc.types.Field("SavedSearch", graphql_name="savedSearch") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class ScriptTagConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ScriptTagEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ScriptTag"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class ScriptTagCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("script_tag", "user_errors") - script_tag = sgqlc.types.Field("ScriptTag", graphql_name="scriptTag") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class ScriptTagDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_script_tag_id", "user_errors") - deleted_script_tag_id = sgqlc.types.Field(ID, graphql_name="deletedScriptTagId") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class ScriptTagEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("ScriptTag"), graphql_name="node") - - -class ScriptTagUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("script_tag", "user_errors") - script_tag = sgqlc.types.Field("ScriptTag", graphql_name="scriptTag") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class SearchFilter(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("key", "value") - key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") - value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") - - -class SearchFilterOptions(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("product_availability",) - product_availability = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FilterOption))), - graphql_name="productAvailability", - ) - - -class SearchResult(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("description", "image", "reference", "title", "url") - description = sgqlc.types.Field(String, graphql_name="description") - image = sgqlc.types.Field("Image", graphql_name="image") - reference = sgqlc.types.Field(sgqlc.types.non_null(Node), graphql_name="reference") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") - - -class SearchResultConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SearchResultEdge"))), graphql_name="edges") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class SearchResultEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(SearchResult), graphql_name="node") - - -class SegmentAssociationFilterValue(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("localized_name", "query_name") - localized_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="localizedName") - query_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="queryName") - - -class SegmentAssociationFilterValueConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SegmentAssociationFilterValueEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SegmentAssociationFilterValue))), - graphql_name="nodes", - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class SegmentAssociationFilterValueEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(SegmentAssociationFilterValue), graphql_name="node") - - -class SegmentAttributeStatistics(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("average", "sum") - average = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="average") - sum = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="sum") - - -class SegmentConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SegmentEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Segment"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class SegmentCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("segment", "user_errors") - segment = sgqlc.types.Field("Segment", graphql_name="segment") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class SegmentDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_segment_id", "user_errors") - deleted_segment_id = sgqlc.types.Field(ID, graphql_name="deletedSegmentId") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class SegmentEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("Segment"), graphql_name="node") - - -class SegmentEnumFilterValue(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("localized_name", "query_name") - localized_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="localizedName") - query_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="queryName") - - -class SegmentEnumFilterValueConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SegmentEnumFilterValueEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SegmentEnumFilterValue))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class SegmentEnumFilterValueEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(SegmentEnumFilterValue), graphql_name="node") - - -class SegmentEventFilterParameter(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("localized_description", "localized_name", "optional", "parameter_type", "query_name") - localized_description = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="localizedDescription") - localized_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="localizedName") - optional = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="optional") - parameter_type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="parameterType") - query_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="queryName") - - -class SegmentEventFilterValue(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("localized_name", "query_name") - localized_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="localizedName") - query_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="queryName") - - -class SegmentEventFilterValueConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SegmentEventFilterValueEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SegmentEventFilterValue))), graphql_name="nodes" - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class SegmentEventFilterValueEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(SegmentEventFilterValue), graphql_name="node") - - -class SegmentFilter(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ("localized_name", "multi_value", "query_name") - localized_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="localizedName") - multi_value = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="multiValue") - query_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="queryName") - - -class SegmentFilterConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SegmentFilterEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SegmentFilter))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class SegmentFilterEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(SegmentFilter), graphql_name="node") - - -class SegmentMembership(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("is_member", "segment_id") - is_member = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isMember") - segment_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="segmentId") - - -class SegmentMembershipResponse(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("memberships",) - memberships = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SegmentMembership))), graphql_name="memberships" - ) - - -class SegmentMigration(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("id", "saved_search_id", "segment_id") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - saved_search_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="savedSearchId") - segment_id = sgqlc.types.Field(ID, graphql_name="segmentId") - - -class SegmentMigrationConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SegmentMigrationEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SegmentMigration))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class SegmentMigrationEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(SegmentMigration), graphql_name="node") - - -class SegmentStatistics(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("attribute_statistics", "total_count") - attribute_statistics = sgqlc.types.Field( - sgqlc.types.non_null(SegmentAttributeStatistics), - graphql_name="attributeStatistics", - args=sgqlc.types.ArgDict( - ( - ( - "attribute_name", - sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="attributeName", default=None), - ), - ) - ), - ) - total_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalCount") - - -class SegmentStringFilterValue(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("localized_name", "query_name") - localized_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="localizedName") - query_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="queryName") - - -class SegmentStringFilterValueConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SegmentStringFilterValueEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SegmentStringFilterValue))), graphql_name="nodes" - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class SegmentStringFilterValueEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(SegmentStringFilterValue), graphql_name="node") - - -class SegmentUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("segment", "user_errors") - segment = sgqlc.types.Field("Segment", graphql_name="segment") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class SegmentValue(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("localized_value", "query_name") - localized_value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="localizedValue") - query_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="queryName") - - -class SegmentValueConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SegmentValueEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SegmentValue))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class SegmentValueEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(SegmentValue), graphql_name="node") - - -class SelectedOption(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("name", "value") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") - - -class SellingPlanAnchor(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cutoff_day", "day", "month", "type") - cutoff_day = sgqlc.types.Field(Int, graphql_name="cutoffDay") - day = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="day") - month = sgqlc.types.Field(Int, graphql_name="month") - type = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanAnchorType), graphql_name="type") - - -class SellingPlanCheckoutCharge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("type", "value") - type = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanCheckoutChargeType), graphql_name="type") - value = sgqlc.types.Field(sgqlc.types.non_null("SellingPlanCheckoutChargeValue"), graphql_name="value") - - -class SellingPlanCheckoutChargePercentageValue(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("percentage",) - percentage = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="percentage") - - -class SellingPlanConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlan"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class SellingPlanEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("SellingPlan"), graphql_name="node") - - -class SellingPlanFixedBillingPolicy(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "checkout_charge", - "remaining_balance_charge_exact_time", - "remaining_balance_charge_time_after_checkout", - "remaining_balance_charge_trigger", - ) - checkout_charge = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanCheckoutCharge), graphql_name="checkoutCharge") - remaining_balance_charge_exact_time = sgqlc.types.Field(DateTime, graphql_name="remainingBalanceChargeExactTime") - remaining_balance_charge_time_after_checkout = sgqlc.types.Field(String, graphql_name="remainingBalanceChargeTimeAfterCheckout") - remaining_balance_charge_trigger = sgqlc.types.Field( - sgqlc.types.non_null(SellingPlanRemainingBalanceChargeTrigger), graphql_name="remainingBalanceChargeTrigger" - ) - - -class SellingPlanFixedDeliveryPolicy(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "anchors", - "cutoff", - "fulfillment_exact_time", - "fulfillment_trigger", - "intent", - "pre_anchor_behavior", - ) - anchors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SellingPlanAnchor))), graphql_name="anchors") - cutoff = sgqlc.types.Field(Int, graphql_name="cutoff") - fulfillment_exact_time = sgqlc.types.Field(DateTime, graphql_name="fulfillmentExactTime") - fulfillment_trigger = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanFulfillmentTrigger), graphql_name="fulfillmentTrigger") - intent = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanFixedDeliveryPolicyIntent), graphql_name="intent") - pre_anchor_behavior = sgqlc.types.Field( - sgqlc.types.non_null(SellingPlanFixedDeliveryPolicyPreAnchorBehavior), graphql_name="preAnchorBehavior" - ) - - -class SellingPlanGroupAddProductVariantsPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("selling_plan_group", "user_errors") - selling_plan_group = sgqlc.types.Field("SellingPlanGroup", graphql_name="sellingPlanGroup") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupUserError"))), - graphql_name="userErrors", - ) - - -class SellingPlanGroupAddProductsPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("selling_plan_group", "user_errors") - selling_plan_group = sgqlc.types.Field("SellingPlanGroup", graphql_name="sellingPlanGroup") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupUserError"))), - graphql_name="userErrors", - ) - - -class SellingPlanGroupConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroup"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class SellingPlanGroupCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("selling_plan_group", "user_errors") - selling_plan_group = sgqlc.types.Field("SellingPlanGroup", graphql_name="sellingPlanGroup") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupUserError"))), - graphql_name="userErrors", - ) - - -class SellingPlanGroupDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_selling_plan_group_id", "user_errors") - deleted_selling_plan_group_id = sgqlc.types.Field(ID, graphql_name="deletedSellingPlanGroupId") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupUserError"))), - graphql_name="userErrors", - ) - - -class SellingPlanGroupEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("SellingPlanGroup"), graphql_name="node") - - -class SellingPlanGroupRemoveProductVariantsPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("removed_product_variant_ids", "user_errors") - removed_product_variant_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="removedProductVariantIds") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupUserError"))), - graphql_name="userErrors", - ) - - -class SellingPlanGroupRemoveProductsPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("removed_product_ids", "user_errors") - removed_product_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="removedProductIds") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupUserError"))), - graphql_name="userErrors", - ) - - -class SellingPlanGroupUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_selling_plan_ids", "selling_plan_group", "user_errors") - deleted_selling_plan_ids = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="deletedSellingPlanIds") - selling_plan_group = sgqlc.types.Field("SellingPlanGroup", graphql_name="sellingPlanGroup") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanGroupUserError"))), - graphql_name="userErrors", - ) - - -class SellingPlanInventoryPolicy(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("reserve",) - reserve = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanReserve), graphql_name="reserve") - - -class SellingPlanPricingPolicyBase(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ("adjustment_type", "adjustment_value") - adjustment_type = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanPricingPolicyAdjustmentType), graphql_name="adjustmentType") - adjustment_value = sgqlc.types.Field(sgqlc.types.non_null("SellingPlanPricingPolicyAdjustmentValue"), graphql_name="adjustmentValue") - - -class SellingPlanPricingPolicyPercentageValue(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("percentage",) - percentage = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="percentage") - - -class SellingPlanRecurringBillingPolicy(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("anchors", "created_at", "interval", "interval_count", "max_cycles", "min_cycles") - anchors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SellingPlanAnchor))), graphql_name="anchors") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - interval = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanInterval), graphql_name="interval") - interval_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="intervalCount") - max_cycles = sgqlc.types.Field(Int, graphql_name="maxCycles") - min_cycles = sgqlc.types.Field(Int, graphql_name="minCycles") - - -class SellingPlanRecurringDeliveryPolicy(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("anchors", "created_at", "cutoff", "intent", "interval", "interval_count", "pre_anchor_behavior") - anchors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SellingPlanAnchor))), graphql_name="anchors") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - cutoff = sgqlc.types.Field(Int, graphql_name="cutoff") - intent = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanRecurringDeliveryPolicyIntent), graphql_name="intent") - interval = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanInterval), graphql_name="interval") - interval_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="intervalCount") - pre_anchor_behavior = sgqlc.types.Field( - sgqlc.types.non_null(SellingPlanRecurringDeliveryPolicyPreAnchorBehavior), graphql_name="preAnchorBehavior" - ) - - -class ShippingLine(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "carrier_identifier", - "code", - "custom", - "delivery_category", - "discount_allocations", - "discounted_price_set", - "id", - "original_price_set", - "phone", - "requested_fulfillment_service", - "shipping_rate_handle", - "source", - "tax_lines", - "title", - ) - carrier_identifier = sgqlc.types.Field(String, graphql_name="carrierIdentifier") - code = sgqlc.types.Field(String, graphql_name="code") - custom = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="custom") - delivery_category = sgqlc.types.Field(String, graphql_name="deliveryCategory") - discount_allocations = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DiscountAllocation))), - graphql_name="discountAllocations", - ) - discounted_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="discountedPriceSet") - id = sgqlc.types.Field(ID, graphql_name="id") - original_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="originalPriceSet") - phone = sgqlc.types.Field(String, graphql_name="phone") - requested_fulfillment_service = sgqlc.types.Field(FulfillmentService, graphql_name="requestedFulfillmentService") - shipping_rate_handle = sgqlc.types.Field(String, graphql_name="shippingRateHandle") - source = sgqlc.types.Field(String, graphql_name="source") - tax_lines = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TaxLine"))), graphql_name="taxLines") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - - -class ShippingLineConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShippingLineEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ShippingLine))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class ShippingLineEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(ShippingLine), graphql_name="node") - - -class ShippingMethod(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("code", "label") - code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") - label = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="label") - - -class ShippingPackageDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_id", "user_errors") - deleted_id = sgqlc.types.Field(ID, graphql_name="deletedId") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class ShippingPackageMakeDefaultPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("user_errors",) - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class ShippingPackageUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("user_errors",) - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class ShippingRate(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("handle", "price", "title") - handle = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="handle") - price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="price") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - - -class ShippingRefund(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("amount_set", "maximum_refundable_set", "tax_set") - amount_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="amountSet") - maximum_refundable_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="maximumRefundableSet") - tax_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="taxSet") - - -class ShopAlert(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("action", "description") - action = sgqlc.types.Field(sgqlc.types.non_null("ShopAlertAction"), graphql_name="action") - description = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="description") - - -class ShopAlertAction(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("title", "url") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") - - -class ShopFeatures(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "avalara_avatax", - "branding", - "captcha", - "captcha_external_domains", - "dynamic_remarketing", - "eligible_for_subscription_migration", - "eligible_for_subscriptions", - "gift_cards", - "harmonized_system_code", - "international_domains", - "international_price_overrides", - "international_price_rules", - "legacy_subscription_gateway_enabled", - "live_view", - "onboarding_visual", - "paypal_express_subscription_gateway_status", - "reports", - "sells_subscriptions", - "show_metrics", - "storefront", - "using_shopify_balance", - ) - avalara_avatax = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="avalaraAvatax") - branding = sgqlc.types.Field(sgqlc.types.non_null(ShopBranding), graphql_name="branding") - captcha = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="captcha") - captcha_external_domains = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="captchaExternalDomains") - dynamic_remarketing = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="dynamicRemarketing") - eligible_for_subscription_migration = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="eligibleForSubscriptionMigration") - eligible_for_subscriptions = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="eligibleForSubscriptions") - gift_cards = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="giftCards") - harmonized_system_code = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="harmonizedSystemCode") - international_domains = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="internationalDomains") - international_price_overrides = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="internationalPriceOverrides") - international_price_rules = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="internationalPriceRules") - legacy_subscription_gateway_enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="legacySubscriptionGatewayEnabled") - live_view = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="liveView") - onboarding_visual = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="onboardingVisual") - paypal_express_subscription_gateway_status = sgqlc.types.Field( - sgqlc.types.non_null(PaypalExpressSubscriptionsGatewayStatus), - graphql_name="paypalExpressSubscriptionGatewayStatus", - ) - reports = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="reports") - sells_subscriptions = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="sellsSubscriptions") - show_metrics = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="showMetrics") - storefront = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="storefront") - using_shopify_balance = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="usingShopifyBalance") - - -class ShopLocale(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("locale", "market_web_presences", "name", "primary", "published") - locale = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="locale") - market_web_presences = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MarketWebPresence"))), - graphql_name="marketWebPresences", - ) - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - primary = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="primary") - published = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="published") - - -class ShopLocaleDisablePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("locale", "user_errors") - locale = sgqlc.types.Field(String, graphql_name="locale") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class ShopLocaleEnablePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("shop_locale", "user_errors") - shop_locale = sgqlc.types.Field(ShopLocale, graphql_name="shopLocale") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class ShopLocaleUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("shop_locale", "user_errors") - shop_locale = sgqlc.types.Field(ShopLocale, graphql_name="shopLocale") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class ShopPlan(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("display_name", "partner_development", "shopify_plus") - display_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="displayName") - partner_development = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="partnerDevelopment") - shopify_plus = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="shopifyPlus") - - -class ShopPolicyUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("shop_policy", "user_errors") - shop_policy = sgqlc.types.Field("ShopPolicy", graphql_name="shopPolicy") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopPolicyUserError"))), - graphql_name="userErrors", - ) - - -class ShopResourceLimits(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "location_limit", - "max_product_options", - "max_product_variants", - "redirect_limit_reached", - "sku_resource_limits", - ) - location_limit = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="locationLimit") - max_product_options = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="maxProductOptions") - max_product_variants = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="maxProductVariants") - redirect_limit_reached = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="redirectLimitReached") - sku_resource_limits = sgqlc.types.Field(sgqlc.types.non_null(ResourceLimit), graphql_name="skuResourceLimits") - - -class ShopifyPaymentsBankAccountConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopifyPaymentsBankAccountEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopifyPaymentsBankAccount"))), - graphql_name="nodes", - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class ShopifyPaymentsBankAccountEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("ShopifyPaymentsBankAccount"), graphql_name="node") - - -class ShopifyPaymentsChargeStatementDescriptor(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ("default", "prefix") - default = sgqlc.types.Field(String, graphql_name="default") - prefix = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="prefix") - - -class ShopifyPaymentsDisputeConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopifyPaymentsDisputeEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopifyPaymentsDispute"))), graphql_name="nodes" - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class ShopifyPaymentsDisputeEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("ShopifyPaymentsDispute"), graphql_name="node") - - -class ShopifyPaymentsDisputeReasonDetails(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("network_reason_code", "reason") - network_reason_code = sgqlc.types.Field(String, graphql_name="networkReasonCode") - reason = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsDisputeReason), graphql_name="reason") - - -class ShopifyPaymentsExtendedAuthorization(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("extended_authorization_expires_at", "standard_authorization_expires_at") - extended_authorization_expires_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="extendedAuthorizationExpiresAt") - standard_authorization_expires_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="standardAuthorizationExpiresAt") - - -class ShopifyPaymentsFraudSettings(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("decline_charge_on_avs_failure", "decline_charge_on_cvc_failure") - decline_charge_on_avs_failure = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="declineChargeOnAvsFailure") - decline_charge_on_cvc_failure = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="declineChargeOnCvcFailure") - - -class ShopifyPaymentsNotificationSettings(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("payouts",) - payouts = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="payouts") - - -class ShopifyPaymentsPayoutConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopifyPaymentsPayoutEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopifyPaymentsPayout"))), graphql_name="nodes" - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class ShopifyPaymentsPayoutEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("ShopifyPaymentsPayout"), graphql_name="node") - - -class ShopifyPaymentsPayoutSchedule(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("interval", "monthly_anchor", "weekly_anchor") - interval = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsPayoutInterval), graphql_name="interval") - monthly_anchor = sgqlc.types.Field(Int, graphql_name="monthlyAnchor") - weekly_anchor = sgqlc.types.Field(DayOfTheWeek, graphql_name="weeklyAnchor") - - -class ShopifyPaymentsPayoutSummary(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "adjustments_fee", - "adjustments_gross", - "charges_fee", - "charges_gross", - "refunds_fee", - "refunds_fee_gross", - "reserved_funds_fee", - "reserved_funds_gross", - "retried_payouts_fee", - "retried_payouts_gross", - ) - adjustments_fee = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="adjustmentsFee") - adjustments_gross = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="adjustmentsGross") - charges_fee = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="chargesFee") - charges_gross = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="chargesGross") - refunds_fee = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="refundsFee") - refunds_fee_gross = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="refundsFeeGross") - reserved_funds_fee = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="reservedFundsFee") - reserved_funds_gross = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="reservedFundsGross") - retried_payouts_fee = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="retriedPayoutsFee") - retried_payouts_gross = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="retriedPayoutsGross") - - -class ShopifyPaymentsRefundSet(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("acquirer_reference_number",) - acquirer_reference_number = sgqlc.types.Field(String, graphql_name="acquirerReferenceNumber") - - -class ShopifyPaymentsTransactionSet(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("extended_authorization_set", "refund_set") - extended_authorization_set = sgqlc.types.Field(ShopifyPaymentsExtendedAuthorization, graphql_name="extendedAuthorizationSet") - refund_set = sgqlc.types.Field(ShopifyPaymentsRefundSet, graphql_name="refundSet") - - -class ShopifyPaymentsVerificationDocument(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("back_required", "front_required", "type") - back_required = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="backRequired") - front_required = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="frontRequired") - type = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsVerificationDocumentType), graphql_name="type") - - -class ShopifyPaymentsVerificationSubject(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("family_name", "given_name") - family_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="familyName") - given_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="givenName") - - -class StaffMemberConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("StaffMemberEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("StaffMember"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class StaffMemberEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("StaffMember"), graphql_name="node") - - -class StaffMemberPrivateData(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("account_settings_url", "created_at") - account_settings_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="accountSettingsUrl") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - - -class StagedMediaUploadTarget(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("parameters", "resource_url", "url") - parameters = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("StagedUploadParameter"))), - graphql_name="parameters", - ) - resource_url = sgqlc.types.Field(URL, graphql_name="resourceUrl") - url = sgqlc.types.Field(URL, graphql_name="url") - - -class StagedUploadParameter(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("name", "value") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") - - -class StagedUploadTarget(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("parameters", "url") - parameters = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ImageUploadParameter))), graphql_name="parameters" - ) - url = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="url") - - -class StagedUploadTargetGeneratePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("parameters", "url", "user_errors") - parameters = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MutationsStagedUploadTargetGenerateUploadParameter))), - graphql_name="parameters", - ) - url = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="url") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class StagedUploadTargetsGeneratePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("urls", "user_errors") - urls = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(StagedUploadTarget)), graphql_name="urls") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class StagedUploadsCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("staged_targets", "user_errors") - staged_targets = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(StagedMediaUploadTarget)), graphql_name="stagedTargets") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class StandardMetafieldDefinitionEnablePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("created_definition", "user_errors") - created_definition = sgqlc.types.Field("MetafieldDefinition", graphql_name="createdDefinition") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("StandardMetafieldDefinitionEnableUserError"))), - graphql_name="userErrors", - ) - - -class StandardMetafieldDefinitionTemplateConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("StandardMetafieldDefinitionTemplateEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("StandardMetafieldDefinitionTemplate"))), - graphql_name="nodes", - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class StandardMetafieldDefinitionTemplateEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("StandardMetafieldDefinitionTemplate"), graphql_name="node") - - -class StandardizedProductType(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("product_taxonomy_node",) - product_taxonomy_node = sgqlc.types.Field("ProductTaxonomyNode", graphql_name="productTaxonomyNode") - - -class StorefrontAccessTokenConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("StorefrontAccessTokenEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("StorefrontAccessToken"))), graphql_name="nodes" - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class StorefrontAccessTokenCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("shop", "storefront_access_token", "user_errors") - shop = sgqlc.types.Field(sgqlc.types.non_null("Shop"), graphql_name="shop") - storefront_access_token = sgqlc.types.Field("StorefrontAccessToken", graphql_name="storefrontAccessToken") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class StorefrontAccessTokenDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_storefront_access_token_id", "user_errors") - deleted_storefront_access_token_id = sgqlc.types.Field(ID, graphql_name="deletedStorefrontAccessTokenId") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class StorefrontAccessTokenEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("StorefrontAccessToken"), graphql_name="node") - - -class StringConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("StringEdge"))), graphql_name="edges") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class StringEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="node") - - -class SubscriptionAppliedCodeDiscount(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("id", "redeem_code", "rejection_reason") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - redeem_code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="redeemCode") - rejection_reason = sgqlc.types.Field(SubscriptionDiscountRejectionReason, graphql_name="rejectionReason") - - -class SubscriptionBillingAttemptConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionBillingAttemptEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionBillingAttempt"))), - graphql_name="nodes", - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class SubscriptionBillingAttemptCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("subscription_billing_attempt", "user_errors") - subscription_billing_attempt = sgqlc.types.Field("SubscriptionBillingAttempt", graphql_name="subscriptionBillingAttempt") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("BillingAttemptUserError"))), - graphql_name="userErrors", - ) - - -class SubscriptionBillingAttemptEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionBillingAttempt"), graphql_name="node") - - -class SubscriptionBillingCycle(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "billing_attempt_expected_date", - "billing_attempts", - "cycle_end_at", - "cycle_index", - "cycle_start_at", - "edited", - "edited_contract", - "skipped", - "source_contract", - "status", - ) - billing_attempt_expected_date = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="billingAttemptExpectedDate") - billing_attempts = sgqlc.types.Field( - sgqlc.types.non_null(SubscriptionBillingAttemptConnection), - graphql_name="billingAttempts", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - cycle_end_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="cycleEndAt") - cycle_index = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="cycleIndex") - cycle_start_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="cycleStartAt") - edited = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="edited") - edited_contract = sgqlc.types.Field("SubscriptionBillingCycleEditedContract", graphql_name="editedContract") - skipped = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="skipped") - source_contract = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionContract"), graphql_name="sourceContract") - status = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionBillingCycleBillingCycleStatus), graphql_name="status") - - -class SubscriptionBillingCycleConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionBillingCycleEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SubscriptionBillingCycle))), graphql_name="nodes" - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class SubscriptionBillingCycleContractDraftCommitPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("contract", "user_errors") - contract = sgqlc.types.Field("SubscriptionBillingCycleEditedContract", graphql_name="contract") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), - graphql_name="userErrors", - ) - - -class SubscriptionBillingCycleContractDraftConcatenatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("draft", "user_errors") - draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), - graphql_name="userErrors", - ) - - -class SubscriptionBillingCycleContractEditPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("draft", "user_errors") - draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), - graphql_name="userErrors", - ) - - -class SubscriptionBillingCycleEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionBillingCycle), graphql_name="node") - - -class SubscriptionBillingCycleEditDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("billing_cycles", "user_errors") - billing_cycles = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(SubscriptionBillingCycle)), graphql_name="billingCycles") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionBillingCycleUserError"))), - graphql_name="userErrors", - ) - - -class SubscriptionBillingCycleEditsDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("billing_cycles", "user_errors") - billing_cycles = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(SubscriptionBillingCycle)), graphql_name="billingCycles") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionBillingCycleUserError"))), - graphql_name="userErrors", - ) - - -class SubscriptionBillingCycleScheduleEditPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("billing_cycle", "user_errors") - billing_cycle = sgqlc.types.Field(SubscriptionBillingCycle, graphql_name="billingCycle") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionBillingCycleUserError"))), - graphql_name="userErrors", - ) - - -class SubscriptionBillingPolicy(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("anchors", "interval", "interval_count", "max_cycles", "min_cycles") - anchors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SellingPlanAnchor))), graphql_name="anchors") - interval = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanInterval), graphql_name="interval") - interval_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="intervalCount") - max_cycles = sgqlc.types.Field(Int, graphql_name="maxCycles") - min_cycles = sgqlc.types.Field(Int, graphql_name="minCycles") - - -class SubscriptionContractBase(sgqlc.types.Interface): - __schema__ = shopify_schema - __field_names__ = ( - "app", - "app_admin_url", - "currency_code", - "custom_attributes", - "customer", - "customer_payment_method", - "delivery_method", - "delivery_price", - "discounts", - "line_count", - "lines", - "note", - "orders", - "updated_at", - ) - app = sgqlc.types.Field("App", graphql_name="app") - app_admin_url = sgqlc.types.Field(URL, graphql_name="appAdminUrl") - currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currencyCode") - custom_attributes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Attribute))), graphql_name="customAttributes" - ) - customer = sgqlc.types.Field("Customer", graphql_name="customer") - customer_payment_method = sgqlc.types.Field( - "CustomerPaymentMethod", - graphql_name="customerPaymentMethod", - args=sgqlc.types.ArgDict((("show_revoked", sgqlc.types.Arg(Boolean, graphql_name="showRevoked", default=False)),)), - ) - delivery_method = sgqlc.types.Field("SubscriptionDeliveryMethod", graphql_name="deliveryMethod") - delivery_price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="deliveryPrice") - discounts = sgqlc.types.Field( - sgqlc.types.non_null("SubscriptionManualDiscountConnection"), - graphql_name="discounts", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - line_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="lineCount") - lines = sgqlc.types.Field( - sgqlc.types.non_null("SubscriptionLineConnection"), - graphql_name="lines", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - note = sgqlc.types.Field(String, graphql_name="note") - orders = sgqlc.types.Field( - sgqlc.types.non_null(OrderConnection), - graphql_name="orders", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - - -class SubscriptionContractConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionContractEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionContract"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class SubscriptionContractCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("draft", "user_errors") - draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), - graphql_name="userErrors", - ) - - -class SubscriptionContractEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionContract"), graphql_name="node") - - -class SubscriptionContractSetNextBillingDatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("contract", "user_errors") - contract = sgqlc.types.Field("SubscriptionContract", graphql_name="contract") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionContractUserError"))), - graphql_name="userErrors", - ) - - -class SubscriptionContractUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("draft", "user_errors") - draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), - graphql_name="userErrors", - ) - - -class SubscriptionCyclePriceAdjustment(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("adjustment_type", "adjustment_value", "after_cycle", "computed_price") - adjustment_type = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanPricingPolicyAdjustmentType), graphql_name="adjustmentType") - adjustment_value = sgqlc.types.Field(sgqlc.types.non_null("SellingPlanPricingPolicyAdjustmentValue"), graphql_name="adjustmentValue") - after_cycle = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="afterCycle") - computed_price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="computedPrice") - - -class SubscriptionDeliveryMethodLocalDelivery(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("address", "local_delivery_option") - address = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionMailingAddress"), graphql_name="address") - local_delivery_option = sgqlc.types.Field( - sgqlc.types.non_null("SubscriptionDeliveryMethodLocalDeliveryOption"), graphql_name="localDeliveryOption" - ) - - -class SubscriptionDeliveryMethodLocalDeliveryOption(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("code", "description", "instructions", "phone", "presentment_title", "title") - code = sgqlc.types.Field(String, graphql_name="code") - description = sgqlc.types.Field(String, graphql_name="description") - instructions = sgqlc.types.Field(String, graphql_name="instructions") - phone = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="phone") - presentment_title = sgqlc.types.Field(String, graphql_name="presentmentTitle") - title = sgqlc.types.Field(String, graphql_name="title") - - -class SubscriptionDeliveryMethodPickup(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("pickup_option",) - pickup_option = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionDeliveryMethodPickupOption"), graphql_name="pickupOption") - - -class SubscriptionDeliveryMethodPickupOption(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("code", "description", "location", "presentment_title", "title") - code = sgqlc.types.Field(String, graphql_name="code") - description = sgqlc.types.Field(String, graphql_name="description") - location = sgqlc.types.Field(sgqlc.types.non_null("Location"), graphql_name="location") - presentment_title = sgqlc.types.Field(String, graphql_name="presentmentTitle") - title = sgqlc.types.Field(String, graphql_name="title") - - -class SubscriptionDeliveryMethodShipping(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("address", "shipping_option") - address = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionMailingAddress"), graphql_name="address") - shipping_option = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionDeliveryMethodShippingOption"), graphql_name="shippingOption") - - -class SubscriptionDeliveryMethodShippingOption(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("carrier_service", "code", "description", "presentment_title", "title") - carrier_service = sgqlc.types.Field("DeliveryCarrierService", graphql_name="carrierService") - code = sgqlc.types.Field(String, graphql_name="code") - description = sgqlc.types.Field(String, graphql_name="description") - presentment_title = sgqlc.types.Field(String, graphql_name="presentmentTitle") - title = sgqlc.types.Field(String, graphql_name="title") - - -class SubscriptionDeliveryOptionResultFailure(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("message",) - message = sgqlc.types.Field(String, graphql_name="message") - - -class SubscriptionDeliveryOptionResultSuccess(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("delivery_options",) - delivery_options = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDeliveryOption"))), - graphql_name="deliveryOptions", - ) - - -class SubscriptionDeliveryPolicy(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("anchors", "interval", "interval_count") - anchors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SellingPlanAnchor))), graphql_name="anchors") - interval = sgqlc.types.Field(sgqlc.types.non_null(SellingPlanInterval), graphql_name="interval") - interval_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="intervalCount") - - -class SubscriptionDiscountAllocation(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("amount", "discount") - amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="amount") - discount = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionDiscount"), graphql_name="discount") - - -class SubscriptionDiscountConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDiscountEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDiscount"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class SubscriptionDiscountEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionDiscount"), graphql_name="node") - - -class SubscriptionDiscountEntitledLines(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("all", "lines") - all = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="all") - lines = sgqlc.types.Field( - sgqlc.types.non_null("SubscriptionLineConnection"), - graphql_name="lines", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - - -class SubscriptionDiscountFixedAmountValue(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("amount", "applies_on_each_item") - amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="amount") - applies_on_each_item = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="appliesOnEachItem") - - -class SubscriptionDiscountPercentageValue(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("percentage",) - percentage = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="percentage") - - -class SubscriptionDraftCommitPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("contract", "user_errors") - contract = sgqlc.types.Field("SubscriptionContract", graphql_name="contract") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), - graphql_name="userErrors", - ) - - -class SubscriptionDraftDiscountAddPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("discount_added", "draft", "user_errors") - discount_added = sgqlc.types.Field("SubscriptionManualDiscount", graphql_name="discountAdded") - draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), - graphql_name="userErrors", - ) - - -class SubscriptionDraftDiscountCodeApplyPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("applied_discount", "draft", "user_errors") - applied_discount = sgqlc.types.Field(SubscriptionAppliedCodeDiscount, graphql_name="appliedDiscount") - draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), - graphql_name="userErrors", - ) - - -class SubscriptionDraftDiscountRemovePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("discount_removed", "draft", "user_errors") - discount_removed = sgqlc.types.Field("SubscriptionDiscount", graphql_name="discountRemoved") - draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), - graphql_name="userErrors", - ) - - -class SubscriptionDraftDiscountUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("discount_updated", "draft", "user_errors") - discount_updated = sgqlc.types.Field("SubscriptionManualDiscount", graphql_name="discountUpdated") - draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), - graphql_name="userErrors", - ) - - -class SubscriptionDraftFreeShippingDiscountAddPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("discount_added", "draft", "user_errors") - discount_added = sgqlc.types.Field("SubscriptionManualDiscount", graphql_name="discountAdded") - draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), - graphql_name="userErrors", - ) - - -class SubscriptionDraftFreeShippingDiscountUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("discount_updated", "draft", "user_errors") - discount_updated = sgqlc.types.Field("SubscriptionManualDiscount", graphql_name="discountUpdated") - draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), - graphql_name="userErrors", - ) - - -class SubscriptionDraftLineAddPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("draft", "line_added", "user_errors") - draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") - line_added = sgqlc.types.Field("SubscriptionLine", graphql_name="lineAdded") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), - graphql_name="userErrors", - ) - - -class SubscriptionDraftLineRemovePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("discounts_updated", "draft", "line_removed", "user_errors") - discounts_updated = sgqlc.types.Field( - sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionManualDiscount")), graphql_name="discountsUpdated" - ) - draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") - line_removed = sgqlc.types.Field("SubscriptionLine", graphql_name="lineRemoved") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), - graphql_name="userErrors", - ) - - -class SubscriptionDraftLineUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("draft", "line_updated", "user_errors") - draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") - line_updated = sgqlc.types.Field("SubscriptionLine", graphql_name="lineUpdated") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), - graphql_name="userErrors", - ) - - -class SubscriptionDraftUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("draft", "user_errors") - draft = sgqlc.types.Field("SubscriptionDraft", graphql_name="draft") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionDraftUserError"))), - graphql_name="userErrors", - ) - - -class SubscriptionLine(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "current_price", - "custom_attributes", - "discount_allocations", - "id", - "line_discounted_price", - "pricing_policy", - "product_id", - "quantity", - "requires_shipping", - "selling_plan_id", - "selling_plan_name", - "sku", - "taxable", - "title", - "variant_id", - "variant_image", - "variant_title", - ) - current_price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="currentPrice") - custom_attributes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Attribute))), graphql_name="customAttributes" - ) - discount_allocations = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SubscriptionDiscountAllocation))), - graphql_name="discountAllocations", - ) - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - line_discounted_price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="lineDiscountedPrice") - pricing_policy = sgqlc.types.Field("SubscriptionPricingPolicy", graphql_name="pricingPolicy") - product_id = sgqlc.types.Field(ID, graphql_name="productId") - quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") - requires_shipping = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresShipping") - selling_plan_id = sgqlc.types.Field(ID, graphql_name="sellingPlanId") - selling_plan_name = sgqlc.types.Field(String, graphql_name="sellingPlanName") - sku = sgqlc.types.Field(String, graphql_name="sku") - taxable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxable") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - variant_id = sgqlc.types.Field(ID, graphql_name="variantId") - variant_image = sgqlc.types.Field("Image", graphql_name="variantImage") - variant_title = sgqlc.types.Field(String, graphql_name="variantTitle") - - -class SubscriptionLineConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionLineEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SubscriptionLine))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class SubscriptionLineEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionLine), graphql_name="node") - - -class SubscriptionLocalDeliveryOption(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("code", "description", "phone_required", "presentment_title", "price", "title") - code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") - description = sgqlc.types.Field(String, graphql_name="description") - phone_required = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="phoneRequired") - presentment_title = sgqlc.types.Field(String, graphql_name="presentmentTitle") - price = sgqlc.types.Field(MoneyV2, graphql_name="price") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - - -class SubscriptionMailingAddress(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "address1", - "address2", - "city", - "company", - "country", - "country_code", - "first_name", - "last_name", - "name", - "phone", - "province", - "province_code", - "zip", - ) - address1 = sgqlc.types.Field(String, graphql_name="address1") - address2 = sgqlc.types.Field(String, graphql_name="address2") - city = sgqlc.types.Field(String, graphql_name="city") - company = sgqlc.types.Field(String, graphql_name="company") - country = sgqlc.types.Field(String, graphql_name="country") - country_code = sgqlc.types.Field(CountryCode, graphql_name="countryCode") - first_name = sgqlc.types.Field(String, graphql_name="firstName") - last_name = sgqlc.types.Field(String, graphql_name="lastName") - name = sgqlc.types.Field(String, graphql_name="name") - phone = sgqlc.types.Field(String, graphql_name="phone") - province = sgqlc.types.Field(String, graphql_name="province") - province_code = sgqlc.types.Field(String, graphql_name="provinceCode") - zip = sgqlc.types.Field(String, graphql_name="zip") - - -class SubscriptionManualDiscount(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "entitled_lines", - "id", - "recurring_cycle_limit", - "rejection_reason", - "target_type", - "title", - "type", - "usage_count", - "value", - ) - entitled_lines = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionDiscountEntitledLines), graphql_name="entitledLines") - id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="id") - recurring_cycle_limit = sgqlc.types.Field(Int, graphql_name="recurringCycleLimit") - rejection_reason = sgqlc.types.Field(SubscriptionDiscountRejectionReason, graphql_name="rejectionReason") - target_type = sgqlc.types.Field(sgqlc.types.non_null(DiscountTargetType), graphql_name="targetType") - title = sgqlc.types.Field(String, graphql_name="title") - type = sgqlc.types.Field(sgqlc.types.non_null(DiscountType), graphql_name="type") - usage_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="usageCount") - value = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionDiscountValue"), graphql_name="value") - - -class SubscriptionManualDiscountConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SubscriptionManualDiscountEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SubscriptionManualDiscount))), - graphql_name="nodes", - ) - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class SubscriptionManualDiscountEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionManualDiscount), graphql_name="node") - - -class SubscriptionPickupOption(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "code", - "description", - "location", - "phone_required", - "pickup_time", - "presentment_title", - "price", - "title", - ) - code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") - description = sgqlc.types.Field(String, graphql_name="description") - location = sgqlc.types.Field(sgqlc.types.non_null("Location"), graphql_name="location") - phone_required = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="phoneRequired") - pickup_time = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="pickupTime") - presentment_title = sgqlc.types.Field(String, graphql_name="presentmentTitle") - price = sgqlc.types.Field(MoneyV2, graphql_name="price") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - - -class SubscriptionPricingPolicy(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("base_price", "cycle_discounts") - base_price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="basePrice") - cycle_discounts = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SubscriptionCyclePriceAdjustment))), - graphql_name="cycleDiscounts", - ) - - -class SubscriptionShippingOption(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "carrier_service", - "code", - "description", - "phone_required", - "presentment_title", - "price", - "title", - ) - carrier_service = sgqlc.types.Field("DeliveryCarrierService", graphql_name="carrierService") - code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") - description = sgqlc.types.Field(String, graphql_name="description") - phone_required = sgqlc.types.Field(Boolean, graphql_name="phoneRequired") - presentment_title = sgqlc.types.Field(String, graphql_name="presentmentTitle") - price = sgqlc.types.Field(MoneyV2, graphql_name="price") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - - -class SubscriptionShippingOptionResultFailure(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("message",) - message = sgqlc.types.Field(String, graphql_name="message") - - -class SubscriptionShippingOptionResultSuccess(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("shipping_options",) - shipping_options = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SubscriptionShippingOption))), - graphql_name="shippingOptions", - ) - - -class SuggestedOrderTransaction(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "account_number", - "amount_set", - "formatted_gateway", - "gateway", - "kind", - "maximum_refundable_set", - "parent_transaction", - ) - account_number = sgqlc.types.Field(String, graphql_name="accountNumber") - amount_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="amountSet") - formatted_gateway = sgqlc.types.Field(String, graphql_name="formattedGateway") - gateway = sgqlc.types.Field(String, graphql_name="gateway") - kind = sgqlc.types.Field(sgqlc.types.non_null(SuggestedOrderTransactionKind), graphql_name="kind") - maximum_refundable_set = sgqlc.types.Field(MoneyBag, graphql_name="maximumRefundableSet") - parent_transaction = sgqlc.types.Field("OrderTransaction", graphql_name="parentTransaction") - - -class SuggestedRefund(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ( - "amount_set", - "discounted_subtotal_set", - "maximum_refundable_set", - "refund_duties", - "refund_line_items", - "shipping", - "subtotal_set", - "suggested_transactions", - "total_cart_discount_amount_set", - "total_duties_set", - "total_tax_set", - ) - amount_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="amountSet") - discounted_subtotal_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="discountedSubtotalSet") - maximum_refundable_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="maximumRefundableSet") - refund_duties = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(RefundDuty))), graphql_name="refundDuties" - ) - refund_line_items = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(RefundLineItem))), graphql_name="refundLineItems" - ) - shipping = sgqlc.types.Field(sgqlc.types.non_null(ShippingRefund), graphql_name="shipping") - subtotal_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="subtotalSet") - suggested_transactions = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SuggestedOrderTransaction))), - graphql_name="suggestedTransactions", - ) - total_cart_discount_amount_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalCartDiscountAmountSet") - total_duties_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalDutiesSet") - total_tax_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalTaxSet") - - -class TagsAddPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("node", "user_errors") - node = sgqlc.types.Field(Node, graphql_name="node") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class TagsRemovePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("node", "user_errors") - node = sgqlc.types.Field(Node, graphql_name="node") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class TaxLine(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("channel_liable", "price_set", "rate", "rate_percentage", "title") - channel_liable = sgqlc.types.Field(Boolean, graphql_name="channelLiable") - price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="priceSet") - rate = sgqlc.types.Field(Float, graphql_name="rate") - rate_percentage = sgqlc.types.Field(Float, graphql_name="ratePercentage") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - - -class TenderTransactionConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TenderTransactionEdge"))), graphql_name="edges" - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TenderTransaction"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class TenderTransactionCreditCardDetails(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("credit_card_company", "credit_card_number") - credit_card_company = sgqlc.types.Field(String, graphql_name="creditCardCompany") - credit_card_number = sgqlc.types.Field(String, graphql_name="creditCardNumber") - - -class TenderTransactionEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("TenderTransaction"), graphql_name="node") - - -class TranslatableContent(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("digest", "key", "locale", "value") - digest = sgqlc.types.Field(String, graphql_name="digest") - key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") - locale = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="locale") - value = sgqlc.types.Field(String, graphql_name="value") - - -class TranslatableResource(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("resource_id", "translatable_content", "translations") - resource_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="resourceId") - translatable_content = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TranslatableContent))), - graphql_name="translatableContent", - ) - translations = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Translation"))), - graphql_name="translations", - args=sgqlc.types.ArgDict( - ( - ("locale", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="locale", default=None)), - ("outdated", sgqlc.types.Arg(Boolean, graphql_name="outdated", default=None)), - ("market_id", sgqlc.types.Arg(ID, graphql_name="marketId", default=None)), - ) - ), - ) - - -class TranslatableResourceConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TranslatableResourceEdge"))), - graphql_name="edges", - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TranslatableResource))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class TranslatableResourceEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null(TranslatableResource), graphql_name="node") - - -class Translation(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("key", "locale", "market", "outdated", "value") - key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") - locale = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="locale") - market = sgqlc.types.Field("Market", graphql_name="market") - outdated = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="outdated") - value = sgqlc.types.Field(String, graphql_name="value") - - -class TranslationsRegisterPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("translations", "user_errors") - translations = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(Translation)), graphql_name="translations") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TranslationUserError"))), - graphql_name="userErrors", - ) - - -class TranslationsRemovePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("translations", "user_errors") - translations = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(Translation)), graphql_name="translations") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TranslationUserError"))), - graphql_name="userErrors", - ) - - -class TypedAttribute(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("key", "value") - key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") - value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") - - -class UTMParameters(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("campaign", "content", "medium", "source", "term") - campaign = sgqlc.types.Field(String, graphql_name="campaign") - content = sgqlc.types.Field(String, graphql_name="content") - medium = sgqlc.types.Field(String, graphql_name="medium") - source = sgqlc.types.Field(String, graphql_name="source") - term = sgqlc.types.Field(String, graphql_name="term") - - -class UrlRedirectBulkDeleteAllPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("job", "user_errors") - job = sgqlc.types.Field(Job, graphql_name="job") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class UrlRedirectBulkDeleteByIdsPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("job", "user_errors") - job = sgqlc.types.Field(Job, graphql_name="job") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UrlRedirectBulkDeleteByIdsUserError"))), - graphql_name="userErrors", - ) - - -class UrlRedirectBulkDeleteBySavedSearchPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("job", "user_errors") - job = sgqlc.types.Field(Job, graphql_name="job") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UrlRedirectBulkDeleteBySavedSearchUserError"))), - graphql_name="userErrors", - ) - - -class UrlRedirectBulkDeleteBySearchPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("job", "user_errors") - job = sgqlc.types.Field(Job, graphql_name="job") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UrlRedirectBulkDeleteBySearchUserError"))), - graphql_name="userErrors", - ) - - -class UrlRedirectConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UrlRedirectEdge"))), graphql_name="edges") - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UrlRedirect"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class UrlRedirectCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("url_redirect", "user_errors") - url_redirect = sgqlc.types.Field("UrlRedirect", graphql_name="urlRedirect") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UrlRedirectUserError"))), - graphql_name="userErrors", - ) - - -class UrlRedirectDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_url_redirect_id", "user_errors") - deleted_url_redirect_id = sgqlc.types.Field(ID, graphql_name="deletedUrlRedirectId") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UrlRedirectUserError"))), - graphql_name="userErrors", - ) - - -class UrlRedirectEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("UrlRedirect"), graphql_name="node") - - -class UrlRedirectImportCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("url_redirect_import", "user_errors") - url_redirect_import = sgqlc.types.Field("UrlRedirectImport", graphql_name="urlRedirectImport") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UrlRedirectImportUserError"))), - graphql_name="userErrors", - ) - - -class UrlRedirectImportPreview(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("path", "target") - path = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="path") - target = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="target") - - -class UrlRedirectImportSubmitPayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("job", "user_errors") - job = sgqlc.types.Field(Job, graphql_name="job") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UrlRedirectImportUserError"))), - graphql_name="userErrors", - ) - - -class UrlRedirectUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("url_redirect", "user_errors") - url_redirect = sgqlc.types.Field("UrlRedirect", graphql_name="urlRedirect") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UrlRedirectUserError"))), - graphql_name="userErrors", - ) - - -class VaultCreditCard(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("billing_address", "brand", "expired", "expiry_month", "expiry_year", "last_digits", "name") - billing_address = sgqlc.types.Field(CustomerCreditCardBillingAddress, graphql_name="billingAddress") - brand = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="brand") - expired = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="expired") - expiry_month = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="expiryMonth") - expiry_year = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="expiryYear") - last_digits = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="lastDigits") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - - -class VaultPaypalBillingAgreement(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("inactive", "name", "paypal_account_email") - inactive = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="inactive") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - paypal_account_email = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="paypalAccountEmail") - - -class Vector3(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("x", "y", "z") - x = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="x") - y = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="y") - z = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="z") - - -class VideoSource(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("file_size", "format", "height", "mime_type", "url", "width") - file_size = sgqlc.types.Field(Int, graphql_name="fileSize") - format = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="format") - height = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="height") - mime_type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="mimeType") - url = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="url") - width = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="width") - - -class WebPixelCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("user_errors", "web_pixel") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ErrorsWebPixelUserError"))), - graphql_name="userErrors", - ) - web_pixel = sgqlc.types.Field("WebPixel", graphql_name="webPixel") - - -class WebPixelDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_web_pixel_id", "user_errors") - deleted_web_pixel_id = sgqlc.types.Field(ID, graphql_name="deletedWebPixelId") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ErrorsWebPixelUserError"))), - graphql_name="userErrors", - ) - - -class WebPixelUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("user_errors", "web_pixel") - user_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ErrorsWebPixelUserError"))), - graphql_name="userErrors", - ) - web_pixel = sgqlc.types.Field("WebPixel", graphql_name="webPixel") - - -class WebhookEventBridgeEndpoint(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("arn",) - arn = sgqlc.types.Field(sgqlc.types.non_null(ARN), graphql_name="arn") - - -class WebhookHttpEndpoint(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("callback_url",) - callback_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="callbackUrl") - - -class WebhookPubSubEndpoint(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("pub_sub_project", "pub_sub_topic") - pub_sub_project = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="pubSubProject") - pub_sub_topic = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="pubSubTopic") - - -class WebhookSubscriptionConnection(sgqlc.types.relay.Connection): - __schema__ = shopify_schema - __field_names__ = ("edges", "nodes", "page_info") - edges = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("WebhookSubscriptionEdge"))), graphql_name="edges" - ) - nodes = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("WebhookSubscription"))), graphql_name="nodes") - page_info = sgqlc.types.Field(sgqlc.types.non_null(PageInfo), graphql_name="pageInfo") - - -class WebhookSubscriptionCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("user_errors", "webhook_subscription") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - webhook_subscription = sgqlc.types.Field("WebhookSubscription", graphql_name="webhookSubscription") - - -class WebhookSubscriptionDeletePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("deleted_webhook_subscription_id", "user_errors") - deleted_webhook_subscription_id = sgqlc.types.Field(ID, graphql_name="deletedWebhookSubscriptionId") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class WebhookSubscriptionEdge(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("cursor", "node") - cursor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="cursor") - node = sgqlc.types.Field(sgqlc.types.non_null("WebhookSubscription"), graphql_name="node") - - -class WebhookSubscriptionUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("user_errors", "webhook_subscription") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - webhook_subscription = sgqlc.types.Field("WebhookSubscription", graphql_name="webhookSubscription") - - -class Weight(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("unit", "value") - unit = sgqlc.types.Field(sgqlc.types.non_null(WeightUnit), graphql_name="unit") - value = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="value") - - -class deliveryProfileCreatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("profile", "user_errors") - profile = sgqlc.types.Field("DeliveryProfile", graphql_name="profile") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class deliveryProfileRemovePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("job", "user_errors") - job = sgqlc.types.Field(Job, graphql_name="job") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class deliveryProfileUpdatePayload(sgqlc.types.Type): - __schema__ = shopify_schema - __field_names__ = ("profile", "user_errors") - profile = sgqlc.types.Field("DeliveryProfile", graphql_name="profile") - user_errors = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("UserError"))), graphql_name="userErrors") - - -class AdjustmentSale(sgqlc.types.Type, Sale): - __schema__ = shopify_schema - __field_names__ = () - - -class App(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "api_key", - "app_store_app_url", - "app_store_developer_url", - "available_access_scopes", - "banner", - "description", - "developer_name", - "developer_type", - "embedded", - "failed_requirements", - "features", - "feedback", - "handle", - "icon", - "install_url", - "installation", - "is_post_purchase_app_in_use", - "previously_installed", - "pricing_details", - "pricing_details_summary", - "privacy_policy_url", - "public_category", - "published", - "requested_access_scopes", - "screenshots", - "shopify_developed", - "title", - "uninstall_message", - "webhook_api_version", - ) - api_key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="apiKey") - app_store_app_url = sgqlc.types.Field(URL, graphql_name="appStoreAppUrl") - app_store_developer_url = sgqlc.types.Field(URL, graphql_name="appStoreDeveloperUrl") - available_access_scopes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(AccessScope))), - graphql_name="availableAccessScopes", - ) - banner = sgqlc.types.Field(sgqlc.types.non_null("Image"), graphql_name="banner") - description = sgqlc.types.Field(String, graphql_name="description") - developer_name = sgqlc.types.Field(String, graphql_name="developerName") - developer_type = sgqlc.types.Field(sgqlc.types.non_null(AppDeveloperType), graphql_name="developerType") - embedded = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="embedded") - failed_requirements = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FailedRequirement))), - graphql_name="failedRequirements", - ) - features = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="features") - feedback = sgqlc.types.Field(AppFeedback, graphql_name="feedback") - handle = sgqlc.types.Field(String, graphql_name="handle") - icon = sgqlc.types.Field(sgqlc.types.non_null("Image"), graphql_name="icon") - install_url = sgqlc.types.Field(URL, graphql_name="installUrl") - installation = sgqlc.types.Field("AppInstallation", graphql_name="installation") - is_post_purchase_app_in_use = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isPostPurchaseAppInUse") - previously_installed = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="previouslyInstalled") - pricing_details = sgqlc.types.Field(String, graphql_name="pricingDetails") - pricing_details_summary = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="pricingDetailsSummary") - privacy_policy_url = sgqlc.types.Field(URL, graphql_name="privacyPolicyUrl") - public_category = sgqlc.types.Field(sgqlc.types.non_null(AppPublicCategory), graphql_name="publicCategory") - published = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="published") - requested_access_scopes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(AccessScope))), - graphql_name="requestedAccessScopes", - ) - screenshots = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Image"))), graphql_name="screenshots") - shopify_developed = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="shopifyDeveloped") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - uninstall_message = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="uninstallMessage") - webhook_api_version = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="webhookApiVersion") - - -class AppCredit(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("amount", "created_at", "description", "test") - amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="amount") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - description = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="description") - test = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="test") - - -class AppInstallation(sgqlc.types.Type, HasMetafields, Node): - __schema__ = shopify_schema - __field_names__ = ( - "access_scopes", - "active_subscriptions", - "all_subscriptions", - "app", - "credits", - "launch_url", - "one_time_purchases", - "publication", - "revenue_attribution_records", - "uninstall_url", - ) - access_scopes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(AccessScope))), graphql_name="accessScopes" - ) - active_subscriptions = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("AppSubscription"))), - graphql_name="activeSubscriptions", - ) - all_subscriptions = sgqlc.types.Field( - sgqlc.types.non_null(AppSubscriptionConnection), - graphql_name="allSubscriptions", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(AppSubscriptionSortKeys, graphql_name="sortKey", default="CREATED_AT")), - ) - ), - ) - app = sgqlc.types.Field(sgqlc.types.non_null(App), graphql_name="app") - credits = sgqlc.types.Field( - sgqlc.types.non_null(AppCreditConnection), - graphql_name="credits", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(AppTransactionSortKeys, graphql_name="sortKey", default="CREATED_AT")), - ) - ), - ) - launch_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="launchUrl") - one_time_purchases = sgqlc.types.Field( - sgqlc.types.non_null(AppPurchaseOneTimeConnection), - graphql_name="oneTimePurchases", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(AppTransactionSortKeys, graphql_name="sortKey", default="CREATED_AT")), - ) - ), - ) - publication = sgqlc.types.Field("Publication", graphql_name="publication") - revenue_attribution_records = sgqlc.types.Field( - sgqlc.types.non_null(AppRevenueAttributionRecordConnection), - graphql_name="revenueAttributionRecords", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ( - "sort_key", - sgqlc.types.Arg(AppRevenueAttributionRecordSortKeys, graphql_name="sortKey", default="CREATED_AT"), - ), - ) - ), - ) - uninstall_url = sgqlc.types.Field(URL, graphql_name="uninstallUrl") - - -class AppPurchaseOneTime(sgqlc.types.Type, AppPurchase, Node): - __schema__ = shopify_schema - __field_names__ = () - - -class AppRevenueAttributionRecord(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("amount", "captured_at", "created_at", "idempotency_key", "test", "type") - amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="amount") - captured_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="capturedAt") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - idempotency_key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="idempotencyKey") - test = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="test") - type = sgqlc.types.Field(sgqlc.types.non_null(AppRevenueAttributionType), graphql_name="type") - - -class AppRevenueAttributionRecordCreateUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(AppRevenueAttributionRecordCreateUserErrorCode, graphql_name="code") - - -class AppRevenueAttributionRecordDeleteUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(AppRevenueAttributionRecordDeleteUserErrorCode, graphql_name="code") - - -class AppSubscription(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "created_at", - "current_period_end", - "line_items", - "name", - "return_url", - "status", - "test", - "trial_days", - ) - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - current_period_end = sgqlc.types.Field(DateTime, graphql_name="currentPeriodEnd") - line_items = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(AppSubscriptionLineItem))), - graphql_name="lineItems", - ) - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - return_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="returnUrl") - status = sgqlc.types.Field(sgqlc.types.non_null(AppSubscriptionStatus), graphql_name="status") - test = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="test") - trial_days = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="trialDays") - - -class AppSubscriptionTrialExtendUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(AppSubscriptionTrialExtendUserErrorCode, graphql_name="code") - - -class AppUsageRecord(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("created_at", "description", "price", "subscription_line_item") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - description = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="description") - price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="price") - subscription_line_item = sgqlc.types.Field(sgqlc.types.non_null(AppSubscriptionLineItem), graphql_name="subscriptionLineItem") - - -class AutomaticDiscountApplication(sgqlc.types.Type, DiscountApplication): - __schema__ = shopify_schema - __field_names__ = ("title",) - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - - -class BasicEvent(sgqlc.types.Type, Event, Node): - __schema__ = shopify_schema - __field_names__ = () - - -class BillingAttemptUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(BillingAttemptUserErrorCode, graphql_name="code") - - -class BulkMutationUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(BulkMutationErrorCode, graphql_name="code") - - -class BulkOperation(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "completed_at", - "created_at", - "error_code", - "file_size", - "object_count", - "partial_data_url", - "query", - "root_object_count", - "status", - "type", - "url", - ) - completed_at = sgqlc.types.Field(DateTime, graphql_name="completedAt") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - error_code = sgqlc.types.Field(BulkOperationErrorCode, graphql_name="errorCode") - file_size = sgqlc.types.Field(UnsignedInt64, graphql_name="fileSize") - object_count = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="objectCount") - partial_data_url = sgqlc.types.Field(URL, graphql_name="partialDataUrl") - query = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="query") - root_object_count = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="rootObjectCount") - status = sgqlc.types.Field(sgqlc.types.non_null(BulkOperationStatus), graphql_name="status") - type = sgqlc.types.Field(sgqlc.types.non_null(BulkOperationType), graphql_name="type") - url = sgqlc.types.Field(URL, graphql_name="url") - - -class BulkProductResourceFeedbackCreateUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(BulkProductResourceFeedbackCreateUserErrorCode, graphql_name="code") - - -class BusinessCustomerUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(BusinessCustomerErrorCode, graphql_name="code") - - -class CalculatedAutomaticDiscountApplication(sgqlc.types.Type, CalculatedDiscountApplication): - __schema__ = shopify_schema - __field_names__ = () - - -class CalculatedDiscountCodeApplication(sgqlc.types.Type, CalculatedDiscountApplication): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") - - -class CalculatedManualDiscountApplication(sgqlc.types.Type, CalculatedDiscountApplication): - __schema__ = shopify_schema - __field_names__ = () - - -class CalculatedOrder(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "added_discount_applications", - "added_line_items", - "cart_discount_amount_set", - "committed", - "line_items", - "notification_preview_html", - "notification_preview_title", - "original_order", - "staged_changes", - "subtotal_line_items_quantity", - "subtotal_price_set", - "tax_lines", - "total_outstanding_set", - "total_price_set", - ) - added_discount_applications = sgqlc.types.Field( - sgqlc.types.non_null(CalculatedDiscountApplicationConnection), - graphql_name="addedDiscountApplications", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - added_line_items = sgqlc.types.Field( - sgqlc.types.non_null(CalculatedLineItemConnection), - graphql_name="addedLineItems", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - cart_discount_amount_set = sgqlc.types.Field(MoneyBag, graphql_name="cartDiscountAmountSet") - committed = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="committed") - line_items = sgqlc.types.Field( - sgqlc.types.non_null(CalculatedLineItemConnection), - graphql_name="lineItems", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - notification_preview_html = sgqlc.types.Field(HTML, graphql_name="notificationPreviewHtml") - notification_preview_title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="notificationPreviewTitle") - original_order = sgqlc.types.Field(sgqlc.types.non_null("Order"), graphql_name="originalOrder") - staged_changes = sgqlc.types.Field( - sgqlc.types.non_null(OrderStagedChangeConnection), - graphql_name="stagedChanges", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - subtotal_line_items_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="subtotalLineItemsQuantity") - subtotal_price_set = sgqlc.types.Field(MoneyBag, graphql_name="subtotalPriceSet") - tax_lines = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxLine))), graphql_name="taxLines") - total_outstanding_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalOutstandingSet") - total_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalPriceSet") - - -class CalculatedScriptDiscountApplication(sgqlc.types.Type, CalculatedDiscountApplication): - __schema__ = shopify_schema - __field_names__ = () - - -class Channel(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "app", - "collection_publications_v3", - "collections", - "has_collection", - "name", - "product_publications_v3", - "products", - "supports_future_publishing", - ) - app = sgqlc.types.Field(sgqlc.types.non_null(App), graphql_name="app") - collection_publications_v3 = sgqlc.types.Field( - sgqlc.types.non_null(ResourcePublicationConnection), - graphql_name="collectionPublicationsV3", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - collections = sgqlc.types.Field( - sgqlc.types.non_null(CollectionConnection), - graphql_name="collections", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - has_collection = sgqlc.types.Field( - sgqlc.types.non_null(Boolean), - graphql_name="hasCollection", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - product_publications_v3 = sgqlc.types.Field( - sgqlc.types.non_null(ResourcePublicationConnection), - graphql_name="productPublicationsV3", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - products = sgqlc.types.Field( - sgqlc.types.non_null(ProductConnection), - graphql_name="products", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - supports_future_publishing = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="supportsFuturePublishing") - - -class ChannelDefinition(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("channel_name", "handle", "sub_channel_name", "svg_icon") - channel_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="channelName") - handle = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="handle") - sub_channel_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="subChannelName") - svg_icon = sgqlc.types.Field(String, graphql_name="svgIcon") - - -class ChannelInformation(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("app", "channel_definition", "channel_id") - app = sgqlc.types.Field(sgqlc.types.non_null(App), graphql_name="app") - channel_definition = sgqlc.types.Field(ChannelDefinition, graphql_name="channelDefinition") - channel_id = sgqlc.types.Field(sgqlc.types.non_null(ID), graphql_name="channelId") - - -class CheckoutProfile(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("created_at", "is_published", "name", "updated_at") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - is_published = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isPublished") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - - -class Collection(sgqlc.types.Type, HasMetafieldDefinitions, HasMetafields, HasPublishedTranslations, Node, Publishable): - __schema__ = shopify_schema - __field_names__ = ( - "description", - "description_html", - "feedback", - "handle", - "has_product", - "image", - "legacy_resource_id", - "products", - "products_count", - "rule_set", - "seo", - "sort_order", - "template_suffix", - "title", - "updated_at", - ) - description = sgqlc.types.Field( - sgqlc.types.non_null(String), - graphql_name="description", - args=sgqlc.types.ArgDict((("truncate_at", sgqlc.types.Arg(Int, graphql_name="truncateAt", default=None)),)), - ) - description_html = sgqlc.types.Field(sgqlc.types.non_null(HTML), graphql_name="descriptionHtml") - feedback = sgqlc.types.Field(ResourceFeedback, graphql_name="feedback") - handle = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="handle") - has_product = sgqlc.types.Field( - sgqlc.types.non_null(Boolean), - graphql_name="hasProduct", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - image = sgqlc.types.Field("Image", graphql_name="image") - legacy_resource_id = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="legacyResourceId") - products = sgqlc.types.Field( - sgqlc.types.non_null(ProductConnection), - graphql_name="products", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ( - "sort_key", - sgqlc.types.Arg(ProductCollectionSortKeys, graphql_name="sortKey", default="COLLECTION_DEFAULT"), - ), - ) - ), - ) - products_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="productsCount") - rule_set = sgqlc.types.Field(CollectionRuleSet, graphql_name="ruleSet") - seo = sgqlc.types.Field(sgqlc.types.non_null(SEO), graphql_name="seo") - sort_order = sgqlc.types.Field(sgqlc.types.non_null(CollectionSortOrder), graphql_name="sortOrder") - template_suffix = sgqlc.types.Field(String, graphql_name="templateSuffix") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - - -class CollectionAddProductsV2UserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(CollectionAddProductsV2UserErrorCode, graphql_name="code") - - -class CommentEvent(sgqlc.types.Type, Event, Node): - __schema__ = shopify_schema - __field_names__ = ("attachments", "author", "can_delete", "can_edit", "edited", "embed", "raw_message", "subject") - attachments = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CommentEventAttachment))), - graphql_name="attachments", - ) - author = sgqlc.types.Field(sgqlc.types.non_null("StaffMember"), graphql_name="author") - can_delete = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="canDelete") - can_edit = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="canEdit") - edited = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="edited") - embed = sgqlc.types.Field("CommentEventEmbed", graphql_name="embed") - raw_message = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="rawMessage") - subject = sgqlc.types.Field(sgqlc.types.non_null(CommentEventSubject), graphql_name="subject") - - -class Company(sgqlc.types.Type, CommentEventSubject, HasEvents, Navigable, Node): - __schema__ = shopify_schema - __field_names__ = ( - "contact_count", - "contact_roles", - "contacts", - "created_at", - "customer_since", - "default_role", - "draft_orders", - "external_id", - "lifetime_duration", - "location_count", - "locations", - "main_contact", - "name", - "note", - "order_count", - "orders", - "total_spent", - "updated_at", - ) - contact_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="contactCount") - contact_roles = sgqlc.types.Field( - sgqlc.types.non_null(CompanyContactRoleConnection), - graphql_name="contactRoles", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(CompanyContactRoleSortKeys, graphql_name="sortKey", default="ID")), - ) - ), - ) - contacts = sgqlc.types.Field( - sgqlc.types.non_null(CompanyContactConnection), - graphql_name="contacts", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(CompanyContactSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - customer_since = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="customerSince") - default_role = sgqlc.types.Field("CompanyContactRole", graphql_name="defaultRole") - draft_orders = sgqlc.types.Field( - sgqlc.types.non_null(DraftOrderConnection), - graphql_name="draftOrders", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(DraftOrderSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - external_id = sgqlc.types.Field(String, graphql_name="externalId") - lifetime_duration = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="lifetimeDuration") - location_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="locationCount") - locations = sgqlc.types.Field( - sgqlc.types.non_null(CompanyLocationConnection), - graphql_name="locations", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(CompanyLocationSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - main_contact = sgqlc.types.Field("CompanyContact", graphql_name="mainContact") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - note = sgqlc.types.Field(String, graphql_name="note") - order_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="orderCount") - orders = sgqlc.types.Field( - sgqlc.types.non_null(OrderConnection), - graphql_name="orders", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(OrderSortKeys, graphql_name="sortKey", default="ID")), - ) - ), - ) - total_spent = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="totalSpent") - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - - -class CompanyAddress(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "address1", - "address2", - "city", - "company_name", - "country", - "country_code", - "created_at", - "formatted_address", - "formatted_area", - "phone", - "province", - "recipient", - "updated_at", - "zip", - "zone_code", - ) - address1 = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="address1") - address2 = sgqlc.types.Field(String, graphql_name="address2") - city = sgqlc.types.Field(String, graphql_name="city") - company_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="companyName") - country = sgqlc.types.Field(String, graphql_name="country") - country_code = sgqlc.types.Field(sgqlc.types.non_null(CountryCode), graphql_name="countryCode") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - formatted_address = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), - graphql_name="formattedAddress", - args=sgqlc.types.ArgDict( - ( - ("with_name", sgqlc.types.Arg(Boolean, graphql_name="withName", default=False)), - ("with_company_name", sgqlc.types.Arg(Boolean, graphql_name="withCompanyName", default=True)), - ) - ), - ) - formatted_area = sgqlc.types.Field(String, graphql_name="formattedArea") - phone = sgqlc.types.Field(String, graphql_name="phone") - province = sgqlc.types.Field(String, graphql_name="province") - recipient = sgqlc.types.Field(String, graphql_name="recipient") - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - zip = sgqlc.types.Field(String, graphql_name="zip") - zone_code = sgqlc.types.Field(String, graphql_name="zoneCode") - - -class CompanyContact(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "company", - "created_at", - "customer", - "draft_orders", - "is_main_contact", - "lifetime_duration", - "locale", - "orders", - "role_assignments", - "title", - "updated_at", - ) - company = sgqlc.types.Field(sgqlc.types.non_null(Company), graphql_name="company") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - customer = sgqlc.types.Field(sgqlc.types.non_null("Customer"), graphql_name="customer") - draft_orders = sgqlc.types.Field( - sgqlc.types.non_null(DraftOrderConnection), - graphql_name="draftOrders", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(DraftOrderSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - is_main_contact = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isMainContact") - lifetime_duration = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="lifetimeDuration") - locale = sgqlc.types.Field(String, graphql_name="locale") - orders = sgqlc.types.Field( - sgqlc.types.non_null(OrderConnection), - graphql_name="orders", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(OrderSortKeys, graphql_name="sortKey", default="ID")), - ) - ), - ) - role_assignments = sgqlc.types.Field( - sgqlc.types.non_null(CompanyContactRoleAssignmentConnection), - graphql_name="roleAssignments", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ( - "sort_key", - sgqlc.types.Arg(CompanyContactRoleAssignmentSortKeys, graphql_name="sortKey", default="ID"), - ), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - title = sgqlc.types.Field(String, graphql_name="title") - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - - -class CompanyContactRole(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("name", "note") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - note = sgqlc.types.Field(String, graphql_name="note") - - -class CompanyContactRoleAssignment(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("company", "company_contact", "company_location", "created_at", "role", "updated_at") - company = sgqlc.types.Field(sgqlc.types.non_null(Company), graphql_name="company") - company_contact = sgqlc.types.Field(sgqlc.types.non_null(CompanyContact), graphql_name="companyContact") - company_location = sgqlc.types.Field(sgqlc.types.non_null("CompanyLocation"), graphql_name="companyLocation") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - role = sgqlc.types.Field(sgqlc.types.non_null(CompanyContactRole), graphql_name="role") - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - - -class CompanyLocation(sgqlc.types.Type, CommentEventSubject, HasEvents, Navigable, Node): - __schema__ = shopify_schema - __field_names__ = ( - "billing_address", - "buyer_experience_configuration", - "company", - "created_at", - "currency", - "draft_orders", - "external_id", - "locale", - "market", - "name", - "note", - "order_count", - "orders", - "phone", - "role_assignments", - "shipping_address", - "tax_exemptions", - "tax_registration_id", - "total_spent", - "updated_at", - ) - billing_address = sgqlc.types.Field(CompanyAddress, graphql_name="billingAddress") - buyer_experience_configuration = sgqlc.types.Field(BuyerExperienceConfiguration, graphql_name="buyerExperienceConfiguration") - company = sgqlc.types.Field(sgqlc.types.non_null(Company), graphql_name="company") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - currency = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currency") - draft_orders = sgqlc.types.Field( - sgqlc.types.non_null(DraftOrderConnection), - graphql_name="draftOrders", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(DraftOrderSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - external_id = sgqlc.types.Field(String, graphql_name="externalId") - locale = sgqlc.types.Field(String, graphql_name="locale") - market = sgqlc.types.Field(sgqlc.types.non_null("Market"), graphql_name="market") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - note = sgqlc.types.Field(String, graphql_name="note") - order_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="orderCount") - orders = sgqlc.types.Field( - sgqlc.types.non_null(OrderConnection), - graphql_name="orders", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(OrderSortKeys, graphql_name="sortKey", default="ID")), - ) - ), - ) - phone = sgqlc.types.Field(String, graphql_name="phone") - role_assignments = sgqlc.types.Field( - sgqlc.types.non_null(CompanyContactRoleAssignmentConnection), - graphql_name="roleAssignments", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ( - "sort_key", - sgqlc.types.Arg(CompanyContactRoleAssignmentSortKeys, graphql_name="sortKey", default="ID"), - ), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - shipping_address = sgqlc.types.Field(CompanyAddress, graphql_name="shippingAddress") - tax_exemptions = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxExemption))), graphql_name="taxExemptions" - ) - tax_registration_id = sgqlc.types.Field(String, graphql_name="taxRegistrationId") - total_spent = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="totalSpent") - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - - -class Customer( - sgqlc.types.Type, - CommentEventSubject, - HasEvents, - HasMetafieldDefinitions, - HasMetafields, - LegacyInteroperability, - Node, -): - __schema__ = shopify_schema - __field_names__ = ( - "addresses", - "amount_spent", - "average_order_amount_v2", - "can_delete", - "company_contact_profiles", - "created_at", - "default_address", - "display_name", - "email", - "email_marketing_consent", - "first_name", - "image", - "last_name", - "last_order", - "lifetime_duration", - "locale", - "market", - "multipass_identifier", - "note", - "number_of_orders", - "orders", - "payment_methods", - "phone", - "product_subscriber_status", - "sms_marketing_consent", - "state", - "statistics", - "subscription_contracts", - "tags", - "tax_exempt", - "tax_exemptions", - "unsubscribe_url", - "updated_at", - "valid_email_address", - "verified_email", - ) - addresses = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("MailingAddress"))), - graphql_name="addresses", - args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)),)), - ) - amount_spent = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="amountSpent") - average_order_amount_v2 = sgqlc.types.Field(MoneyV2, graphql_name="averageOrderAmountV2") - can_delete = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="canDelete") - company_contact_profiles = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CompanyContact))), - graphql_name="companyContactProfiles", - ) - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - default_address = sgqlc.types.Field("MailingAddress", graphql_name="defaultAddress") - display_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="displayName") - email = sgqlc.types.Field(String, graphql_name="email") - email_marketing_consent = sgqlc.types.Field(CustomerEmailMarketingConsentState, graphql_name="emailMarketingConsent") - first_name = sgqlc.types.Field(String, graphql_name="firstName") - image = sgqlc.types.Field(sgqlc.types.non_null("Image"), graphql_name="image") - last_name = sgqlc.types.Field(String, graphql_name="lastName") - last_order = sgqlc.types.Field("Order", graphql_name="lastOrder") - lifetime_duration = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="lifetimeDuration") - locale = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="locale") - market = sgqlc.types.Field("Market", graphql_name="market") - multipass_identifier = sgqlc.types.Field(String, graphql_name="multipassIdentifier") - note = sgqlc.types.Field(String, graphql_name="note") - number_of_orders = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="numberOfOrders") - orders = sgqlc.types.Field( - sgqlc.types.non_null(OrderConnection), - graphql_name="orders", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(OrderSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - payment_methods = sgqlc.types.Field( - sgqlc.types.non_null(CustomerPaymentMethodConnection), - graphql_name="paymentMethods", - args=sgqlc.types.ArgDict( - ( - ("show_revoked", sgqlc.types.Arg(Boolean, graphql_name="showRevoked", default=False)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - phone = sgqlc.types.Field(String, graphql_name="phone") - product_subscriber_status = sgqlc.types.Field( - sgqlc.types.non_null(CustomerProductSubscriberStatus), graphql_name="productSubscriberStatus" - ) - sms_marketing_consent = sgqlc.types.Field(CustomerSmsMarketingConsentState, graphql_name="smsMarketingConsent") - state = sgqlc.types.Field(sgqlc.types.non_null(CustomerState), graphql_name="state") - statistics = sgqlc.types.Field(sgqlc.types.non_null(CustomerStatistics), graphql_name="statistics") - subscription_contracts = sgqlc.types.Field( - sgqlc.types.non_null(SubscriptionContractConnection), - graphql_name="subscriptionContracts", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - tags = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="tags") - tax_exempt = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxExempt") - tax_exemptions = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxExemption))), graphql_name="taxExemptions" - ) - unsubscribe_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="unsubscribeUrl") - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - valid_email_address = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="validEmailAddress") - verified_email = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="verifiedEmail") - - -class CustomerEmailMarketingConsentUpdateUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(CustomerEmailMarketingConsentUpdateUserErrorCode, graphql_name="code") - - -class CustomerPaymentMethod(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("customer", "instrument", "revoked_at", "revoked_reason", "subscription_contracts") - customer = sgqlc.types.Field(Customer, graphql_name="customer") - instrument = sgqlc.types.Field("CustomerPaymentInstrument", graphql_name="instrument") - revoked_at = sgqlc.types.Field(DateTime, graphql_name="revokedAt") - revoked_reason = sgqlc.types.Field(CustomerPaymentMethodRevocationReason, graphql_name="revokedReason") - subscription_contracts = sgqlc.types.Field( - sgqlc.types.non_null(SubscriptionContractConnection), - graphql_name="subscriptionContracts", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - - -class CustomerPaymentMethodGetUpdateUrlUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(CustomerPaymentMethodGetUpdateUrlUserErrorCode, graphql_name="code") - - -class CustomerPaymentMethodRemoteUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(CustomerPaymentMethodRemoteUserErrorCode, graphql_name="code") - - -class CustomerPaymentMethodUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(CustomerPaymentMethodUserErrorCode, graphql_name="code") - - -class CustomerSmsMarketingConsentError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(CustomerSmsMarketingConsentErrorCode, graphql_name="code") - - -class CustomerVisit(sgqlc.types.Type, CustomerMoment, Node): - __schema__ = shopify_schema - __field_names__ = ( - "landing_page", - "landing_page_html", - "marketing_event", - "referral_code", - "referral_info_html", - "referrer_url", - "source", - "source_description", - "source_type", - "utm_parameters", - ) - landing_page = sgqlc.types.Field(URL, graphql_name="landingPage") - landing_page_html = sgqlc.types.Field(HTML, graphql_name="landingPageHtml") - marketing_event = sgqlc.types.Field("MarketingEvent", graphql_name="marketingEvent") - referral_code = sgqlc.types.Field(String, graphql_name="referralCode") - referral_info_html = sgqlc.types.Field(sgqlc.types.non_null(FormattedString), graphql_name="referralInfoHtml") - referrer_url = sgqlc.types.Field(URL, graphql_name="referrerUrl") - source = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="source") - source_description = sgqlc.types.Field(String, graphql_name="sourceDescription") - source_type = sgqlc.types.Field(MarketingTactic, graphql_name="sourceType") - utm_parameters = sgqlc.types.Field(UTMParameters, graphql_name="utmParameters") - - -class DelegateAccessTokenCreateUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(DelegateAccessTokenCreateUserErrorCode, graphql_name="code") - - -class DeliveryCarrierService(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("available_services_for_countries", "formatted_name", "icon", "name") - available_services_for_countries = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DeliveryAvailableService))), - graphql_name="availableServicesForCountries", - args=sgqlc.types.ArgDict( - ( - ( - "origins", - sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="origins", default=None), - ), - ( - "country_codes", - sgqlc.types.Arg( - sgqlc.types.list_of(sgqlc.types.non_null(CountryCode)), - graphql_name="countryCodes", - default=None, - ), - ), - ( - "rest_of_world", - sgqlc.types.Arg(sgqlc.types.non_null(Boolean), graphql_name="restOfWorld", default=None), - ), - ) - ), - ) - formatted_name = sgqlc.types.Field(String, graphql_name="formattedName") - icon = sgqlc.types.Field(sgqlc.types.non_null("Image"), graphql_name="icon") - name = sgqlc.types.Field(String, graphql_name="name") - - -class DeliveryCondition(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("condition_criteria", "field", "operator") - condition_criteria = sgqlc.types.Field(sgqlc.types.non_null("DeliveryConditionCriteria"), graphql_name="conditionCriteria") - field = sgqlc.types.Field(sgqlc.types.non_null(DeliveryConditionField), graphql_name="field") - operator = sgqlc.types.Field(sgqlc.types.non_null(DeliveryConditionOperator), graphql_name="operator") - - -class DeliveryCountry(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("code", "name", "provinces", "translated_name") - code = sgqlc.types.Field(sgqlc.types.non_null(DeliveryCountryCodeOrRestOfWorld), graphql_name="code") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - provinces = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("DeliveryProvince"))), graphql_name="provinces" - ) - translated_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="translatedName") - - -class DeliveryLocationGroup(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("locations",) - locations = sgqlc.types.Field( - sgqlc.types.non_null(LocationConnection), - graphql_name="locations", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(LocationSortKeys, graphql_name="sortKey", default="NAME")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("include_legacy", sgqlc.types.Arg(Boolean, graphql_name="includeLegacy", default=False)), - ("include_inactive", sgqlc.types.Arg(Boolean, graphql_name="includeInactive", default=False)), - ) - ), - ) - - -class DeliveryMethod(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("max_delivery_date_time", "method_type", "min_delivery_date_time") - max_delivery_date_time = sgqlc.types.Field(DateTime, graphql_name="maxDeliveryDateTime") - method_type = sgqlc.types.Field(sgqlc.types.non_null(DeliveryMethodType), graphql_name="methodType") - min_delivery_date_time = sgqlc.types.Field(DateTime, graphql_name="minDeliveryDateTime") - - -class DeliveryMethodDefinition(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("active", "description", "method_conditions", "name", "rate_provider") - active = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="active") - description = sgqlc.types.Field(String, graphql_name="description") - method_conditions = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DeliveryCondition))), - graphql_name="methodConditions", - ) - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - rate_provider = sgqlc.types.Field(sgqlc.types.non_null("DeliveryRateProvider"), graphql_name="rateProvider") - - -class DeliveryParticipant(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "adapt_to_new_services_flag", - "carrier_service", - "fixed_fee", - "participant_services", - "percentage_of_rate_fee", - ) - adapt_to_new_services_flag = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="adaptToNewServicesFlag") - carrier_service = sgqlc.types.Field(sgqlc.types.non_null(DeliveryCarrierService), graphql_name="carrierService") - fixed_fee = sgqlc.types.Field(MoneyV2, graphql_name="fixedFee") - participant_services = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DeliveryParticipantService))), - graphql_name="participantServices", - ) - percentage_of_rate_fee = sgqlc.types.Field(sgqlc.types.non_null(Float), graphql_name="percentageOfRateFee") - - -class DeliveryProfile(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "active_method_definitions_count", - "default", - "legacy_mode", - "locations_without_rates_count", - "name", - "origin_location_count", - "product_variants_count_v2", - "profile_items", - "profile_location_groups", - "selling_plan_groups", - "unassigned_locations", - "zone_country_count", - ) - active_method_definitions_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="activeMethodDefinitionsCount") - default = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="default") - legacy_mode = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="legacyMode") - locations_without_rates_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="locationsWithoutRatesCount") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - origin_location_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="originLocationCount") - product_variants_count_v2 = sgqlc.types.Field(sgqlc.types.non_null(DeliveryProductVariantsCount), graphql_name="productVariantsCountV2") - profile_items = sgqlc.types.Field( - sgqlc.types.non_null(DeliveryProfileItemConnection), - graphql_name="profileItems", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - profile_location_groups = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DeliveryProfileLocationGroup))), - graphql_name="profileLocationGroups", - ) - selling_plan_groups = sgqlc.types.Field( - sgqlc.types.non_null(SellingPlanGroupConnection), - graphql_name="sellingPlanGroups", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - unassigned_locations = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Location"))), graphql_name="unassignedLocations" - ) - zone_country_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="zoneCountryCount") - - -class DeliveryProfileItem(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("product", "variants") - product = sgqlc.types.Field(sgqlc.types.non_null("Product"), graphql_name="product") - variants = sgqlc.types.Field( - sgqlc.types.non_null(ProductVariantConnection), - graphql_name="variants", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - - -class DeliveryProvince(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("code", "name", "translated_name") - code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - translated_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="translatedName") - - -class DeliveryRateDefinition(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("price",) - price = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="price") - - -class DeliveryZone(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("countries", "name") - countries = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DeliveryCountry))), graphql_name="countries" - ) - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - - -class DiscountAutomaticBxgy(sgqlc.types.Type, HasEvents, Node): - __schema__ = shopify_schema - __field_names__ = ( - "async_usage_count", - "combines_with", - "created_at", - "customer_buys", - "customer_gets", - "discount_class", - "ends_at", - "starts_at", - "status", - "summary", - "title", - "uses_per_order_limit", - ) - async_usage_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="asyncUsageCount") - combines_with = sgqlc.types.Field(sgqlc.types.non_null(DiscountCombinesWith), graphql_name="combinesWith") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - customer_buys = sgqlc.types.Field(sgqlc.types.non_null(DiscountCustomerBuys), graphql_name="customerBuys") - customer_gets = sgqlc.types.Field(sgqlc.types.non_null(DiscountCustomerGets), graphql_name="customerGets") - discount_class = sgqlc.types.Field(sgqlc.types.non_null(MerchandiseDiscountClass), graphql_name="discountClass") - ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") - starts_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="startsAt") - status = sgqlc.types.Field(sgqlc.types.non_null(DiscountStatus), graphql_name="status") - summary = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="summary") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - uses_per_order_limit = sgqlc.types.Field(Int, graphql_name="usesPerOrderLimit") - - -class DiscountAutomaticNode(sgqlc.types.Type, HasEvents, HasMetafieldDefinitions, HasMetafields, Node): - __schema__ = shopify_schema - __field_names__ = ("automatic_discount",) - automatic_discount = sgqlc.types.Field(sgqlc.types.non_null("DiscountAutomatic"), graphql_name="automaticDiscount") - - -class DiscountCodeApplication(sgqlc.types.Type, DiscountApplication): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") - - -class DiscountCodeNode(sgqlc.types.Type, HasEvents, HasMetafieldDefinitions, HasMetafields, Node): - __schema__ = shopify_schema - __field_names__ = ("code_discount",) - code_discount = sgqlc.types.Field(sgqlc.types.non_null("DiscountCode"), graphql_name="codeDiscount") - - -class DiscountNode(sgqlc.types.Type, HasEvents, HasMetafieldDefinitions, HasMetafields, Node): - __schema__ = shopify_schema - __field_names__ = ("discount",) - discount = sgqlc.types.Field(sgqlc.types.non_null("Discount"), graphql_name="discount") - - -class DiscountRedeemCodeBulkCreation(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("codes", "codes_count", "created_at", "discount_code", "done", "failed_count", "imported_count") - codes = sgqlc.types.Field( - sgqlc.types.non_null(DiscountRedeemCodeBulkCreationCodeConnection), - graphql_name="codes", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - codes_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="codesCount") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - discount_code = sgqlc.types.Field(DiscountCodeNode, graphql_name="discountCode") - done = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="done") - failed_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="failedCount") - imported_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="importedCount") - - -class DiscountUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code", "extra_info") - code = sgqlc.types.Field(DiscountErrorCode, graphql_name="code") - extra_info = sgqlc.types.Field(String, graphql_name="extraInfo") - - -class DisputeEvidenceUpdateUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(DisputeEvidenceUpdateUserErrorCode, graphql_name="code") - - -class Domain(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("host", "localization", "market_web_presence", "ssl_enabled", "url") - host = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="host") - localization = sgqlc.types.Field(DomainLocalization, graphql_name="localization") - market_web_presence = sgqlc.types.Field("MarketWebPresence", graphql_name="marketWebPresence") - ssl_enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="sslEnabled") - url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") - - -class DraftOrder( - sgqlc.types.Type, - CommentEventSubject, - HasEvents, - HasLocalizationExtensions, - HasMetafields, - LegacyInteroperability, - Navigable, - Node, -): - __schema__ = shopify_schema - __field_names__ = ( - "applied_discount", - "billing_address", - "billing_address_matches_shipping_address", - "completed_at", - "created_at", - "currency_code", - "custom_attributes", - "customer", - "email", - "invoice_email_template_subject", - "invoice_sent_at", - "invoice_url", - "line_items", - "line_items_subtotal_price", - "market_name", - "market_region_country_code", - "name", - "note2", - "order", - "payment_terms", - "phone", - "presentment_currency_code", - "purchasing_entity", - "ready", - "reserve_inventory_until", - "shipping_address", - "shipping_line", - "status", - "subtotal_price", - "subtotal_price_set", - "tags", - "tax_exempt", - "tax_lines", - "taxes_included", - "total_discounts_set", - "total_line_items_price_set", - "total_price", - "total_price_set", - "total_shipping_price", - "total_shipping_price_set", - "total_tax", - "total_tax_set", - "total_weight", - "updated_at", - "visible_to_customer", - ) - applied_discount = sgqlc.types.Field(DraftOrderAppliedDiscount, graphql_name="appliedDiscount") - billing_address = sgqlc.types.Field("MailingAddress", graphql_name="billingAddress") - billing_address_matches_shipping_address = sgqlc.types.Field( - sgqlc.types.non_null(Boolean), graphql_name="billingAddressMatchesShippingAddress" - ) - completed_at = sgqlc.types.Field(DateTime, graphql_name="completedAt") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currencyCode") - custom_attributes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Attribute))), graphql_name="customAttributes" - ) - customer = sgqlc.types.Field(Customer, graphql_name="customer") - email = sgqlc.types.Field(String, graphql_name="email") - invoice_email_template_subject = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="invoiceEmailTemplateSubject") - invoice_sent_at = sgqlc.types.Field(DateTime, graphql_name="invoiceSentAt") - invoice_url = sgqlc.types.Field(URL, graphql_name="invoiceUrl") - line_items = sgqlc.types.Field( - sgqlc.types.non_null(DraftOrderLineItemConnection), - graphql_name="lineItems", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - line_items_subtotal_price = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="lineItemsSubtotalPrice") - market_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="marketName") - market_region_country_code = sgqlc.types.Field(sgqlc.types.non_null(CountryCode), graphql_name="marketRegionCountryCode") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - note2 = sgqlc.types.Field(String, graphql_name="note2") - order = sgqlc.types.Field("Order", graphql_name="order") - payment_terms = sgqlc.types.Field("PaymentTerms", graphql_name="paymentTerms") - phone = sgqlc.types.Field(String, graphql_name="phone") - presentment_currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="presentmentCurrencyCode") - purchasing_entity = sgqlc.types.Field("PurchasingEntity", graphql_name="purchasingEntity") - ready = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="ready") - reserve_inventory_until = sgqlc.types.Field(DateTime, graphql_name="reserveInventoryUntil") - shipping_address = sgqlc.types.Field("MailingAddress", graphql_name="shippingAddress") - shipping_line = sgqlc.types.Field(ShippingLine, graphql_name="shippingLine") - status = sgqlc.types.Field(sgqlc.types.non_null(DraftOrderStatus), graphql_name="status") - subtotal_price = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="subtotalPrice") - subtotal_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="subtotalPriceSet") - tags = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="tags") - tax_exempt = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxExempt") - tax_lines = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxLine))), graphql_name="taxLines") - taxes_included = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxesIncluded") - total_discounts_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalDiscountsSet") - total_line_items_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalLineItemsPriceSet") - total_price = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="totalPrice") - total_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalPriceSet") - total_shipping_price = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="totalShippingPrice") - total_shipping_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalShippingPriceSet") - total_tax = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="totalTax") - total_tax_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalTaxSet") - total_weight = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="totalWeight") - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - visible_to_customer = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="visibleToCustomer") - - -class DraftOrderLineItem(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "applied_discount", - "custom", - "custom_attributes", - "custom_attributes_v2", - "discounted_total", - "discounted_total_set", - "discounted_unit_price", - "discounted_unit_price_set", - "fulfillment_service", - "image", - "is_gift_card", - "name", - "original_total", - "original_total_set", - "original_unit_price", - "original_unit_price_set", - "product", - "quantity", - "requires_shipping", - "sku", - "tax_lines", - "taxable", - "title", - "total_discount", - "total_discount_set", - "variant", - "variant_title", - "vendor", - "weight", - ) - applied_discount = sgqlc.types.Field(DraftOrderAppliedDiscount, graphql_name="appliedDiscount") - custom = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="custom") - custom_attributes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Attribute))), graphql_name="customAttributes" - ) - custom_attributes_v2 = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TypedAttribute))), - graphql_name="customAttributesV2", - ) - discounted_total = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="discountedTotal") - discounted_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="discountedTotalSet") - discounted_unit_price = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="discountedUnitPrice") - discounted_unit_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="discountedUnitPriceSet") - fulfillment_service = sgqlc.types.Field(FulfillmentService, graphql_name="fulfillmentService") - image = sgqlc.types.Field("Image", graphql_name="image") - is_gift_card = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isGiftCard") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - original_total = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="originalTotal") - original_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="originalTotalSet") - original_unit_price = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="originalUnitPrice") - original_unit_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="originalUnitPriceSet") - product = sgqlc.types.Field("Product", graphql_name="product") - quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") - requires_shipping = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresShipping") - sku = sgqlc.types.Field(String, graphql_name="sku") - tax_lines = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxLine))), graphql_name="taxLines") - taxable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxable") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - total_discount = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="totalDiscount") - total_discount_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalDiscountSet") - variant = sgqlc.types.Field("ProductVariant", graphql_name="variant") - variant_title = sgqlc.types.Field(String, graphql_name="variantTitle") - vendor = sgqlc.types.Field(String, graphql_name="vendor") - weight = sgqlc.types.Field(Weight, graphql_name="weight") - - -class DraftOrderTag(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("handle", "title") - handle = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="handle") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - - -class Duty(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("country_code_of_origin", "harmonized_system_code", "price", "tax_lines") - country_code_of_origin = sgqlc.types.Field(CountryCode, graphql_name="countryCodeOfOrigin") - harmonized_system_code = sgqlc.types.Field(String, graphql_name="harmonizedSystemCode") - price = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="price") - tax_lines = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxLine))), graphql_name="taxLines") - - -class DutySale(sgqlc.types.Type, Sale): - __schema__ = shopify_schema - __field_names__ = ("duty",) - duty = sgqlc.types.Field(sgqlc.types.non_null(Duty), graphql_name="duty") - - -class ErrorsWebPixelUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(ErrorsWebPixelUserErrorCode, graphql_name="code") - - -class ExternalVideo(sgqlc.types.Type, Media, Node): - __schema__ = shopify_schema - __field_names__ = ("embed_url", "host", "origin_url") - embed_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="embedUrl") - host = sgqlc.types.Field(sgqlc.types.non_null(MediaHost), graphql_name="host") - origin_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="originUrl") - - -class FilesUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(FilesErrorCode, graphql_name="code") - - -class Fulfillment(sgqlc.types.Type, LegacyInteroperability, Node): - __schema__ = shopify_schema - __field_names__ = ( - "created_at", - "delivered_at", - "display_status", - "estimated_delivery_at", - "events", - "fulfillment_line_items", - "fulfillment_orders", - "in_transit_at", - "location", - "name", - "order", - "origin_address", - "requires_shipping", - "service", - "status", - "total_quantity", - "tracking_info", - "updated_at", - ) - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - delivered_at = sgqlc.types.Field(DateTime, graphql_name="deliveredAt") - display_status = sgqlc.types.Field(FulfillmentDisplayStatus, graphql_name="displayStatus") - estimated_delivery_at = sgqlc.types.Field(DateTime, graphql_name="estimatedDeliveryAt") - events = sgqlc.types.Field( - sgqlc.types.non_null(FulfillmentEventConnection), - graphql_name="events", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(FulfillmentEventSortKeys, graphql_name="sortKey", default="HAPPENED_AT")), - ) - ), - ) - fulfillment_line_items = sgqlc.types.Field( - sgqlc.types.non_null(FulfillmentLineItemConnection), - graphql_name="fulfillmentLineItems", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - fulfillment_orders = sgqlc.types.Field( - sgqlc.types.non_null(FulfillmentOrderConnection), - graphql_name="fulfillmentOrders", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - in_transit_at = sgqlc.types.Field(DateTime, graphql_name="inTransitAt") - location = sgqlc.types.Field("Location", graphql_name="location") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - order = sgqlc.types.Field(sgqlc.types.non_null("Order"), graphql_name="order") - origin_address = sgqlc.types.Field(FulfillmentOriginAddress, graphql_name="originAddress") - requires_shipping = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresShipping") - service = sgqlc.types.Field(FulfillmentService, graphql_name="service") - status = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentStatus), graphql_name="status") - total_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalQuantity") - tracking_info = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FulfillmentTrackingInfo))), - graphql_name="trackingInfo", - args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)),)), - ) - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - - -class FulfillmentEvent(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("happened_at", "status") - happened_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="happenedAt") - status = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentEventStatus), graphql_name="status") - - -class FulfillmentLineItem(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("discounted_total_set", "line_item", "original_total_set", "quantity") - discounted_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="discountedTotalSet") - line_item = sgqlc.types.Field(sgqlc.types.non_null("LineItem"), graphql_name="lineItem") - original_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="originalTotalSet") - quantity = sgqlc.types.Field(Int, graphql_name="quantity") - - -class FulfillmentOrder(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "assigned_location", - "delivery_method", - "destination", - "fulfill_at", - "fulfill_by", - "fulfillment_holds", - "fulfillments", - "international_duties", - "line_items", - "locations_for_move", - "merchant_requests", - "order", - "request_status", - "status", - "supported_actions", - ) - assigned_location = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentOrderAssignedLocation), graphql_name="assignedLocation") - delivery_method = sgqlc.types.Field(DeliveryMethod, graphql_name="deliveryMethod") - destination = sgqlc.types.Field("FulfillmentOrderDestination", graphql_name="destination") - fulfill_at = sgqlc.types.Field(DateTime, graphql_name="fulfillAt") - fulfill_by = sgqlc.types.Field(DateTime, graphql_name="fulfillBy") - fulfillment_holds = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FulfillmentHold))), - graphql_name="fulfillmentHolds", - ) - fulfillments = sgqlc.types.Field( - sgqlc.types.non_null(FulfillmentConnection), - graphql_name="fulfillments", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - international_duties = sgqlc.types.Field(FulfillmentOrderInternationalDuties, graphql_name="internationalDuties") - line_items = sgqlc.types.Field( - sgqlc.types.non_null(FulfillmentOrderLineItemConnection), - graphql_name="lineItems", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - locations_for_move = sgqlc.types.Field( - sgqlc.types.non_null(FulfillmentOrderLocationForMoveConnection), - graphql_name="locationsForMove", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - merchant_requests = sgqlc.types.Field( - sgqlc.types.non_null(FulfillmentOrderMerchantRequestConnection), - graphql_name="merchantRequests", - args=sgqlc.types.ArgDict( - ( - ("kind", sgqlc.types.Arg(FulfillmentOrderMerchantRequestKind, graphql_name="kind", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - order = sgqlc.types.Field(sgqlc.types.non_null("Order"), graphql_name="order") - request_status = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentOrderRequestStatus), graphql_name="requestStatus") - status = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentOrderStatus), graphql_name="status") - supported_actions = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FulfillmentOrderSupportedAction))), - graphql_name="supportedActions", - ) - - -class FulfillmentOrderDestination(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "address1", - "address2", - "city", - "company", - "country_code", - "email", - "first_name", - "last_name", - "phone", - "province", - "zip", - ) - address1 = sgqlc.types.Field(String, graphql_name="address1") - address2 = sgqlc.types.Field(String, graphql_name="address2") - city = sgqlc.types.Field(String, graphql_name="city") - company = sgqlc.types.Field(String, graphql_name="company") - country_code = sgqlc.types.Field(CountryCode, graphql_name="countryCode") - email = sgqlc.types.Field(String, graphql_name="email") - first_name = sgqlc.types.Field(String, graphql_name="firstName") - last_name = sgqlc.types.Field(String, graphql_name="lastName") - phone = sgqlc.types.Field(String, graphql_name="phone") - province = sgqlc.types.Field(String, graphql_name="province") - zip = sgqlc.types.Field(String, graphql_name="zip") - - -class FulfillmentOrderHoldUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(FulfillmentOrderHoldUserErrorCode, graphql_name="code") - - -class FulfillmentOrderLineItem(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("line_item", "remaining_quantity", "total_quantity", "warnings") - line_item = sgqlc.types.Field(sgqlc.types.non_null("LineItem"), graphql_name="lineItem") - remaining_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="remainingQuantity") - total_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalQuantity") - warnings = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FulfillmentOrderLineItemWarning))), - graphql_name="warnings", - ) - - -class FulfillmentOrderMerchantRequest(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("kind", "message", "request_options", "response_data", "sent_at") - kind = sgqlc.types.Field(sgqlc.types.non_null(FulfillmentOrderMerchantRequestKind), graphql_name="kind") - message = sgqlc.types.Field(String, graphql_name="message") - request_options = sgqlc.types.Field(JSON, graphql_name="requestOptions") - response_data = sgqlc.types.Field(JSON, graphql_name="responseData") - sent_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="sentAt") - - -class FulfillmentOrderReleaseHoldUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(FulfillmentOrderReleaseHoldUserErrorCode, graphql_name="code") - - -class FulfillmentOrderRescheduleUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(FulfillmentOrderRescheduleUserErrorCode, graphql_name="code") - - -class FulfillmentOrdersSetFulfillmentDeadlineUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(FulfillmentOrdersSetFulfillmentDeadlineUserErrorCode, graphql_name="code") - - -class GenericFile(sgqlc.types.Type, File, Node): - __schema__ = shopify_schema - __field_names__ = ("mime_type", "original_file_size", "url") - mime_type = sgqlc.types.Field(String, graphql_name="mimeType") - original_file_size = sgqlc.types.Field(Int, graphql_name="originalFileSize") - url = sgqlc.types.Field(URL, graphql_name="url") - - -class GiftCard(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "balance", - "created_at", - "customer", - "disabled_at", - "enabled", - "expires_on", - "initial_value", - "last_characters", - "masked_code", - "note", - "order", - ) - balance = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="balance") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - customer = sgqlc.types.Field(Customer, graphql_name="customer") - disabled_at = sgqlc.types.Field(DateTime, graphql_name="disabledAt") - enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="enabled") - expires_on = sgqlc.types.Field(Date, graphql_name="expiresOn") - initial_value = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="initialValue") - last_characters = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="lastCharacters") - masked_code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="maskedCode") - note = sgqlc.types.Field(String, graphql_name="note") - order = sgqlc.types.Field("Order", graphql_name="order") - - -class GiftCardSale(sgqlc.types.Type, Sale): - __schema__ = shopify_schema - __field_names__ = ("line_item",) - line_item = sgqlc.types.Field(sgqlc.types.non_null("LineItem"), graphql_name="lineItem") - - -class GiftCardUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(GiftCardErrorCode, graphql_name="code") - - -class Image(sgqlc.types.Type, HasMetafields): - __schema__ = shopify_schema - __field_names__ = ("alt_text", "height", "id", "url", "width") - alt_text = sgqlc.types.Field(String, graphql_name="altText") - height = sgqlc.types.Field(Int, graphql_name="height") - id = sgqlc.types.Field(ID, graphql_name="id") - url = sgqlc.types.Field( - sgqlc.types.non_null(URL), - graphql_name="url", - args=sgqlc.types.ArgDict((("transform", sgqlc.types.Arg(ImageTransformInput, graphql_name="transform", default=None)),)), - ) - width = sgqlc.types.Field(Int, graphql_name="width") - - -class InventoryBulkToggleActivationUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(InventoryBulkToggleActivationUserErrorCode, graphql_name="code") - - -class InventoryItem(sgqlc.types.Type, LegacyInteroperability, Node): - __schema__ = shopify_schema - __field_names__ = ( - "country_code_of_origin", - "country_harmonized_system_codes", - "created_at", - "duplicate_sku_count", - "harmonized_system_code", - "inventory_history_url", - "inventory_level", - "inventory_levels", - "locations_count", - "province_code_of_origin", - "requires_shipping", - "sku", - "tracked", - "tracked_editable", - "unit_cost", - "updated_at", - "variant", - ) - country_code_of_origin = sgqlc.types.Field(CountryCode, graphql_name="countryCodeOfOrigin") - country_harmonized_system_codes = sgqlc.types.Field( - sgqlc.types.non_null(CountryHarmonizedSystemCodeConnection), - graphql_name="countryHarmonizedSystemCodes", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - duplicate_sku_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="duplicateSkuCount") - harmonized_system_code = sgqlc.types.Field(String, graphql_name="harmonizedSystemCode") - inventory_history_url = sgqlc.types.Field(URL, graphql_name="inventoryHistoryUrl") - inventory_level = sgqlc.types.Field( - "InventoryLevel", - graphql_name="inventoryLevel", - args=sgqlc.types.ArgDict((("location_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="locationId", default=None)),)), - ) - inventory_levels = sgqlc.types.Field( - sgqlc.types.non_null(InventoryLevelConnection), - graphql_name="inventoryLevels", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - locations_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="locationsCount") - province_code_of_origin = sgqlc.types.Field(String, graphql_name="provinceCodeOfOrigin") - requires_shipping = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresShipping") - sku = sgqlc.types.Field(String, graphql_name="sku") - tracked = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="tracked") - tracked_editable = sgqlc.types.Field(sgqlc.types.non_null(EditableProperty), graphql_name="trackedEditable") - unit_cost = sgqlc.types.Field(MoneyV2, graphql_name="unitCost") - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - variant = sgqlc.types.Field(sgqlc.types.non_null("ProductVariant"), graphql_name="variant") - - -class InventoryLevel(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "available", - "can_deactivate", - "created_at", - "deactivation_alert", - "deactivation_alert_html", - "incoming", - "item", - "location", - "updated_at", - ) - available = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="available") - can_deactivate = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="canDeactivate") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - deactivation_alert = sgqlc.types.Field(String, graphql_name="deactivationAlert") - deactivation_alert_html = sgqlc.types.Field(FormattedString, graphql_name="deactivationAlertHtml") - incoming = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="incoming") - item = sgqlc.types.Field(sgqlc.types.non_null(InventoryItem), graphql_name="item") - location = sgqlc.types.Field(sgqlc.types.non_null("Location"), graphql_name="location") - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - - -class LineItem(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "contract", - "current_quantity", - "custom_attributes", - "discount_allocations", - "discounted_total_set", - "discounted_unit_price_set", - "duties", - "image", - "merchant_editable", - "name", - "non_fulfillable_quantity", - "original_total_set", - "original_unit_price_set", - "product", - "quantity", - "refundable_quantity", - "requires_shipping", - "restockable", - "selling_plan", - "sku", - "staff_member", - "tax_lines", - "taxable", - "title", - "total_discount_set", - "unfulfilled_discounted_total_set", - "unfulfilled_original_total_set", - "unfulfilled_quantity", - "variant", - "variant_title", - "vendor", - ) - contract = sgqlc.types.Field("SubscriptionContract", graphql_name="contract") - current_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="currentQuantity") - custom_attributes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Attribute))), graphql_name="customAttributes" - ) - discount_allocations = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DiscountAllocation))), - graphql_name="discountAllocations", - ) - discounted_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="discountedTotalSet") - discounted_unit_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="discountedUnitPriceSet") - duties = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Duty))), graphql_name="duties") - image = sgqlc.types.Field(Image, graphql_name="image") - merchant_editable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="merchantEditable") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - non_fulfillable_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="nonFulfillableQuantity") - original_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="originalTotalSet") - original_unit_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="originalUnitPriceSet") - product = sgqlc.types.Field("Product", graphql_name="product") - quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") - refundable_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="refundableQuantity") - requires_shipping = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresShipping") - restockable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="restockable") - selling_plan = sgqlc.types.Field(LineItemSellingPlan, graphql_name="sellingPlan") - sku = sgqlc.types.Field(String, graphql_name="sku") - staff_member = sgqlc.types.Field("StaffMember", graphql_name="staffMember") - tax_lines = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxLine))), - graphql_name="taxLines", - args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)),)), - ) - taxable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxable") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - total_discount_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalDiscountSet") - unfulfilled_discounted_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="unfulfilledDiscountedTotalSet") - unfulfilled_original_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="unfulfilledOriginalTotalSet") - unfulfilled_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="unfulfilledQuantity") - variant = sgqlc.types.Field("ProductVariant", graphql_name="variant") - variant_title = sgqlc.types.Field(String, graphql_name="variantTitle") - vendor = sgqlc.types.Field(String, graphql_name="vendor") - - -class LineItemMutable(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "custom_attributes", - "discount_allocations", - "discounted_total_set", - "discounted_unit_price_set", - "fulfillable_quantity", - "fulfillment_service", - "fulfillment_status", - "image", - "merchant_editable", - "name", - "non_fulfillable_quantity", - "original_total_set", - "original_unit_price_set", - "product", - "quantity", - "refundable_quantity", - "requires_shipping", - "restockable", - "sku", - "staff_member", - "tax_lines", - "taxable", - "title", - "total_discount_set", - "unfulfilled_discounted_total_set", - "unfulfilled_original_total_set", - "unfulfilled_quantity", - "variant", - "variant_title", - "vendor", - ) - custom_attributes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Attribute))), graphql_name="customAttributes" - ) - discount_allocations = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(DiscountAllocation))), - graphql_name="discountAllocations", - ) - discounted_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="discountedTotalSet") - discounted_unit_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="discountedUnitPriceSet") - fulfillable_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="fulfillableQuantity") - fulfillment_service = sgqlc.types.Field(FulfillmentService, graphql_name="fulfillmentService") - fulfillment_status = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="fulfillmentStatus") - image = sgqlc.types.Field(Image, graphql_name="image") - merchant_editable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="merchantEditable") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - non_fulfillable_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="nonFulfillableQuantity") - original_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="originalTotalSet") - original_unit_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="originalUnitPriceSet") - product = sgqlc.types.Field("Product", graphql_name="product") - quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="quantity") - refundable_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="refundableQuantity") - requires_shipping = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresShipping") - restockable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="restockable") - sku = sgqlc.types.Field(String, graphql_name="sku") - staff_member = sgqlc.types.Field("StaffMember", graphql_name="staffMember") - tax_lines = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxLine))), - graphql_name="taxLines", - args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)),)), - ) - taxable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxable") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - total_discount_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalDiscountSet") - unfulfilled_discounted_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="unfulfilledDiscountedTotalSet") - unfulfilled_original_total_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="unfulfilledOriginalTotalSet") - unfulfilled_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="unfulfilledQuantity") - variant = sgqlc.types.Field("ProductVariant", graphql_name="variant") - variant_title = sgqlc.types.Field(String, graphql_name="variantTitle") - vendor = sgqlc.types.Field(String, graphql_name="vendor") - - -class Link(sgqlc.types.Type, HasPublishedTranslations): - __schema__ = shopify_schema - __field_names__ = ("label", "url") - label = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="label") - url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") - - -class Location(sgqlc.types.Type, HasMetafieldDefinitions, HasMetafields, LegacyInteroperability, Node): - __schema__ = shopify_schema - __field_names__ = ( - "activatable", - "address", - "address_verified", - "deactivatable", - "deactivated_at", - "deletable", - "fulfillment_service", - "fulfills_online_orders", - "has_active_inventory", - "has_unfulfilled_orders", - "inventory_level", - "inventory_levels", - "is_active", - "name", - "ships_inventory", - "suggested_addresses", - ) - activatable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="activatable") - address = sgqlc.types.Field(sgqlc.types.non_null(LocationAddress), graphql_name="address") - address_verified = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="addressVerified") - deactivatable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="deactivatable") - deactivated_at = sgqlc.types.Field(String, graphql_name="deactivatedAt") - deletable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="deletable") - fulfillment_service = sgqlc.types.Field(FulfillmentService, graphql_name="fulfillmentService") - fulfills_online_orders = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="fulfillsOnlineOrders") - has_active_inventory = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasActiveInventory") - has_unfulfilled_orders = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasUnfulfilledOrders") - inventory_level = sgqlc.types.Field( - InventoryLevel, - graphql_name="inventoryLevel", - args=sgqlc.types.ArgDict( - ( - ( - "inventory_item_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="inventoryItemId", default=None), - ), - ) - ), - ) - inventory_levels = sgqlc.types.Field( - sgqlc.types.non_null(InventoryLevelConnection), - graphql_name="inventoryLevels", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - is_active = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isActive") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - ships_inventory = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="shipsInventory") - suggested_addresses = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(LocationSuggestedAddress))), - graphql_name="suggestedAddresses", - ) - - -class LocationActivateUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(LocationActivateUserErrorCode, graphql_name="code") - - -class LocationAddUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(LocationAddUserErrorCode, graphql_name="code") - - -class LocationDeactivateUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(LocationDeactivateUserErrorCode, graphql_name="code") - - -class LocationDeleteUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(LocationDeleteUserErrorCode, graphql_name="code") - - -class LocationEditUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(LocationEditUserErrorCode, graphql_name="code") - - -class MailingAddress(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "address1", - "address2", - "city", - "company", - "coordinates_validated", - "country", - "country_code_v2", - "first_name", - "formatted", - "formatted_area", - "last_name", - "latitude", - "longitude", - "name", - "phone", - "province", - "province_code", - "zip", - ) - address1 = sgqlc.types.Field(String, graphql_name="address1") - address2 = sgqlc.types.Field(String, graphql_name="address2") - city = sgqlc.types.Field(String, graphql_name="city") - company = sgqlc.types.Field(String, graphql_name="company") - coordinates_validated = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="coordinatesValidated") - country = sgqlc.types.Field(String, graphql_name="country") - country_code_v2 = sgqlc.types.Field(CountryCode, graphql_name="countryCodeV2") - first_name = sgqlc.types.Field(String, graphql_name="firstName") - formatted = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), - graphql_name="formatted", - args=sgqlc.types.ArgDict( - ( - ("with_name", sgqlc.types.Arg(Boolean, graphql_name="withName", default=False)), - ("with_company", sgqlc.types.Arg(Boolean, graphql_name="withCompany", default=True)), - ) - ), - ) - formatted_area = sgqlc.types.Field(String, graphql_name="formattedArea") - last_name = sgqlc.types.Field(String, graphql_name="lastName") - latitude = sgqlc.types.Field(Float, graphql_name="latitude") - longitude = sgqlc.types.Field(Float, graphql_name="longitude") - name = sgqlc.types.Field(String, graphql_name="name") - phone = sgqlc.types.Field(String, graphql_name="phone") - province = sgqlc.types.Field(String, graphql_name="province") - province_code = sgqlc.types.Field(String, graphql_name="provinceCode") - zip = sgqlc.types.Field(String, graphql_name="zip") - - -class ManualDiscountApplication(sgqlc.types.Type, DiscountApplication): - __schema__ = shopify_schema - __field_names__ = ("description", "title") - description = sgqlc.types.Field(String, graphql_name="description") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - - -class Market(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("currency_settings", "enabled", "name", "price_list", "primary", "regions", "web_presence") - currency_settings = sgqlc.types.Field(sgqlc.types.non_null(MarketCurrencySettings), graphql_name="currencySettings") - enabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="enabled") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - price_list = sgqlc.types.Field("PriceList", graphql_name="priceList") - primary = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="primary") - regions = sgqlc.types.Field( - sgqlc.types.non_null(MarketRegionConnection), - graphql_name="regions", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - web_presence = sgqlc.types.Field("MarketWebPresence", graphql_name="webPresence") - - -class MarketCurrencySettingsUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(MarketCurrencySettingsUserErrorCode, graphql_name="code") - - -class MarketRegionCountry(sgqlc.types.Type, MarketRegion, Node): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(sgqlc.types.non_null(CountryCode), graphql_name="code") - - -class MarketUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(MarketUserErrorCode, graphql_name="code") - - -class MarketWebPresence(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("alternate_locales", "default_locale", "domain", "market", "root_urls", "subfolder_suffix") - alternate_locales = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="alternateLocales" - ) - default_locale = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="defaultLocale") - domain = sgqlc.types.Field(Domain, graphql_name="domain") - market = sgqlc.types.Field(sgqlc.types.non_null(Market), graphql_name="market") - root_urls = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MarketWebPresenceRootUrl))), - graphql_name="rootUrls", - ) - subfolder_suffix = sgqlc.types.Field(String, graphql_name="subfolderSuffix") - - -class MarketingActivity(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "activity_list_url", - "ad_spend", - "app", - "app_errors", - "budget", - "created_at", - "form_data", - "in_main_workflow_version", - "marketing_channel", - "marketing_event", - "source_and_medium", - "status", - "status_badge_type_v2", - "status_label", - "status_transitioned_at", - "tactic", - "target_status", - "title", - "updated_at", - "utm_parameters", - ) - activity_list_url = sgqlc.types.Field(URL, graphql_name="activityListUrl") - ad_spend = sgqlc.types.Field(MoneyV2, graphql_name="adSpend") - app = sgqlc.types.Field(sgqlc.types.non_null(App), graphql_name="app") - app_errors = sgqlc.types.Field(MarketingActivityExtensionAppErrors, graphql_name="appErrors") - budget = sgqlc.types.Field(MarketingBudget, graphql_name="budget") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - form_data = sgqlc.types.Field(String, graphql_name="formData") - in_main_workflow_version = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="inMainWorkflowVersion") - marketing_channel = sgqlc.types.Field(sgqlc.types.non_null(MarketingChannel), graphql_name="marketingChannel") - marketing_event = sgqlc.types.Field("MarketingEvent", graphql_name="marketingEvent") - source_and_medium = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="sourceAndMedium") - status = sgqlc.types.Field(sgqlc.types.non_null(MarketingActivityStatus), graphql_name="status") - status_badge_type_v2 = sgqlc.types.Field(BadgeType, graphql_name="statusBadgeTypeV2") - status_label = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="statusLabel") - status_transitioned_at = sgqlc.types.Field(DateTime, graphql_name="statusTransitionedAt") - tactic = sgqlc.types.Field(sgqlc.types.non_null(MarketingTactic), graphql_name="tactic") - target_status = sgqlc.types.Field(MarketingActivityStatus, graphql_name="targetStatus") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - utm_parameters = sgqlc.types.Field(UTMParameters, graphql_name="utmParameters") - - -class MarketingActivityUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(MarketingActivityUserErrorCode, graphql_name="code") - - -class MarketingEvent(sgqlc.types.Type, LegacyInteroperability, Node): - __schema__ = shopify_schema - __field_names__ = ( - "app", - "channel", - "description", - "ended_at", - "manage_url", - "preview_url", - "remote_id", - "scheduled_to_end_at", - "source_and_medium", - "started_at", - "type", - "utm_campaign", - "utm_medium", - "utm_source", - ) - app = sgqlc.types.Field(sgqlc.types.non_null(App), graphql_name="app") - channel = sgqlc.types.Field(MarketingChannel, graphql_name="channel") - description = sgqlc.types.Field(String, graphql_name="description") - ended_at = sgqlc.types.Field(DateTime, graphql_name="endedAt") - manage_url = sgqlc.types.Field(URL, graphql_name="manageUrl") - preview_url = sgqlc.types.Field(URL, graphql_name="previewUrl") - remote_id = sgqlc.types.Field(String, graphql_name="remoteId") - scheduled_to_end_at = sgqlc.types.Field(DateTime, graphql_name="scheduledToEndAt") - source_and_medium = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="sourceAndMedium") - started_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="startedAt") - type = sgqlc.types.Field(sgqlc.types.non_null(MarketingTactic), graphql_name="type") - utm_campaign = sgqlc.types.Field(String, graphql_name="utmCampaign") - utm_medium = sgqlc.types.Field(String, graphql_name="utmMedium") - utm_source = sgqlc.types.Field(String, graphql_name="utmSource") - - -class MediaImage(sgqlc.types.Type, File, Media, Node): - __schema__ = shopify_schema - __field_names__ = ("image", "mime_type", "original_source") - image = sgqlc.types.Field(Image, graphql_name="image") - mime_type = sgqlc.types.Field(String, graphql_name="mimeType") - original_source = sgqlc.types.Field(MediaImageOriginalSource, graphql_name="originalSource") - - -class MediaUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(MediaUserErrorCode, graphql_name="code") - - -class Metafield(sgqlc.types.Type, LegacyInteroperability, Node): - __schema__ = shopify_schema - __field_names__ = ( - "created_at", - "definition", - "description", - "key", - "namespace", - "owner", - "owner_type", - "reference", - "references", - "type", - "updated_at", - "value", - ) - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - definition = sgqlc.types.Field("MetafieldDefinition", graphql_name="definition") - description = sgqlc.types.Field(String, graphql_name="description") - key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") - namespace = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="namespace") - owner = sgqlc.types.Field(sgqlc.types.non_null(HasMetafields), graphql_name="owner") - owner_type = sgqlc.types.Field(sgqlc.types.non_null(MetafieldOwnerType), graphql_name="ownerType") - reference = sgqlc.types.Field("MetafieldReference", graphql_name="reference") - references = sgqlc.types.Field( - MetafieldReferenceConnection, - graphql_name="references", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ) - ), - ) - type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="type") - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") - - -class MetafieldDefinition(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "description", - "key", - "metafields", - "metafields_count", - "name", - "namespace", - "owner_type", - "pinned_position", - "standard_template", - "type", - "validation_status", - "validations", - "visible_to_storefront_api", - ) - description = sgqlc.types.Field(String, graphql_name="description") - key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") - metafields = sgqlc.types.Field( - sgqlc.types.non_null(MetafieldConnection), - graphql_name="metafields", - args=sgqlc.types.ArgDict( - ( - ( - "validation_status", - sgqlc.types.Arg(MetafieldValidationStatus, graphql_name="validationStatus", default="ANY"), - ), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - metafields_count = sgqlc.types.Field( - sgqlc.types.non_null(Int), - graphql_name="metafieldsCount", - args=sgqlc.types.ArgDict( - ( - ( - "validation_status", - sgqlc.types.Arg(MetafieldValidationStatus, graphql_name="validationStatus", default=None), - ), - ) - ), - ) - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - namespace = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="namespace") - owner_type = sgqlc.types.Field(sgqlc.types.non_null(MetafieldOwnerType), graphql_name="ownerType") - pinned_position = sgqlc.types.Field(Int, graphql_name="pinnedPosition") - standard_template = sgqlc.types.Field("StandardMetafieldDefinitionTemplate", graphql_name="standardTemplate") - type = sgqlc.types.Field(sgqlc.types.non_null(MetafieldDefinitionType), graphql_name="type") - validation_status = sgqlc.types.Field(sgqlc.types.non_null(MetafieldDefinitionValidationStatus), graphql_name="validationStatus") - validations = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MetafieldDefinitionValidation))), - graphql_name="validations", - ) - visible_to_storefront_api = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="visibleToStorefrontApi") - - -class MetafieldDefinitionCreateUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(MetafieldDefinitionCreateUserErrorCode, graphql_name="code") - - -class MetafieldDefinitionDeleteUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(MetafieldDefinitionDeleteUserErrorCode, graphql_name="code") - - -class MetafieldDefinitionPinUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(MetafieldDefinitionPinUserErrorCode, graphql_name="code") - - -class MetafieldDefinitionUnpinUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(MetafieldDefinitionUnpinUserErrorCode, graphql_name="code") - - -class MetafieldDefinitionUpdateUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(MetafieldDefinitionUpdateUserErrorCode, graphql_name="code") - - -class MetafieldStorefrontVisibility(sgqlc.types.Type, LegacyInteroperability, Node): - __schema__ = shopify_schema - __field_names__ = ("created_at", "key", "namespace", "owner_type", "updated_at") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") - namespace = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="namespace") - owner_type = sgqlc.types.Field(sgqlc.types.non_null(MetafieldOwnerType), graphql_name="ownerType") - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - - -class MetafieldsSetUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code", "element_index") - code = sgqlc.types.Field(MetafieldsSetUserErrorCode, graphql_name="code") - element_index = sgqlc.types.Field(Int, graphql_name="elementIndex") - - -class Model3d(sgqlc.types.Type, Media, Node): - __schema__ = shopify_schema - __field_names__ = ("bounding_box", "filename", "original_source", "sources") - bounding_box = sgqlc.types.Field(Model3dBoundingBox, graphql_name="boundingBox") - filename = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="filename") - original_source = sgqlc.types.Field(Model3dSource, graphql_name="originalSource") - sources = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Model3dSource))), graphql_name="sources") - - -class OnlineStoreArticle(sgqlc.types.Type, HasPublishedTranslations, Navigable, Node): - __schema__ = shopify_schema - __field_names__ = () - - -class OnlineStoreBlog(sgqlc.types.Type, HasPublishedTranslations, Node): - __schema__ = shopify_schema - __field_names__ = () - - -class OnlineStorePage(sgqlc.types.Type, HasPublishedTranslations, Navigable, Node): - __schema__ = shopify_schema - __field_names__ = () - - -class Order( - sgqlc.types.Type, - CommentEventSubject, - HasEvents, - HasLocalizationExtensions, - HasMetafieldDefinitions, - HasMetafields, - LegacyInteroperability, - Node, -): - __schema__ = shopify_schema - __field_names__ = ( - "agreements", - "alerts", - "app", - "billing_address", - "billing_address_matches_shipping_address", - "can_mark_as_paid", - "can_notify_customer", - "cancel_reason", - "cancelled_at", - "capturable", - "cart_discount_amount_set", - "channel_information", - "client_ip", - "closed", - "closed_at", - "confirmed", - "created_at", - "currency_code", - "current_cart_discount_amount_set", - "current_subtotal_line_items_quantity", - "current_subtotal_price_set", - "current_tax_lines", - "current_total_discounts_set", - "current_total_duties_set", - "current_total_price_set", - "current_total_tax_set", - "current_total_weight", - "custom_attributes", - "customer", - "customer_accepts_marketing", - "customer_journey_summary", - "customer_locale", - "discount_applications", - "discount_code", - "discount_codes", - "display_address", - "display_financial_status", - "display_fulfillment_status", - "disputes", - "edited", - "email", - "estimated_taxes", - "fulfillable", - "fulfillment_orders", - "fulfillments", - "fully_paid", - "line_items", - "merchant_editable", - "merchant_editable_errors", - "merchant_of_record_app", - "name", - "net_payment_set", - "non_fulfillable_line_items", - "note", - "original_total_duties_set", - "original_total_price_set", - "payment_collection_details", - "payment_gateway_names", - "payment_terms", - "phone", - "physical_location", - "presentment_currency_code", - "processed_at", - "publication", - "purchasing_entity", - "refund_discrepancy_set", - "refundable", - "refunds", - "registered_source_url", - "requires_shipping", - "restockable", - "risk_level", - "risks", - "shipping_address", - "shipping_line", - "shipping_lines", - "source_identifier", - "subtotal_line_items_quantity", - "subtotal_price_set", - "suggested_refund", - "tags", - "tax_lines", - "taxes_included", - "test", - "total_capturable_set", - "total_discounts_set", - "total_outstanding_set", - "total_price_set", - "total_received_set", - "total_refunded_set", - "total_refunded_shipping_set", - "total_shipping_price_set", - "total_tax_set", - "total_tip_received_set", - "total_weight", - "transactions", - "unpaid", - "updated_at", - ) - agreements = sgqlc.types.Field( - sgqlc.types.non_null(SalesAgreementConnection), - graphql_name="agreements", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - alerts = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ResourceAlert))), graphql_name="alerts") - app = sgqlc.types.Field(OrderApp, graphql_name="app") - billing_address = sgqlc.types.Field(MailingAddress, graphql_name="billingAddress") - billing_address_matches_shipping_address = sgqlc.types.Field( - sgqlc.types.non_null(Boolean), graphql_name="billingAddressMatchesShippingAddress" - ) - can_mark_as_paid = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="canMarkAsPaid") - can_notify_customer = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="canNotifyCustomer") - cancel_reason = sgqlc.types.Field(OrderCancelReason, graphql_name="cancelReason") - cancelled_at = sgqlc.types.Field(DateTime, graphql_name="cancelledAt") - capturable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="capturable") - cart_discount_amount_set = sgqlc.types.Field(MoneyBag, graphql_name="cartDiscountAmountSet") - channel_information = sgqlc.types.Field(ChannelInformation, graphql_name="channelInformation") - client_ip = sgqlc.types.Field(String, graphql_name="clientIp") - closed = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="closed") - closed_at = sgqlc.types.Field(DateTime, graphql_name="closedAt") - confirmed = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="confirmed") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currencyCode") - current_cart_discount_amount_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="currentCartDiscountAmountSet") - current_subtotal_line_items_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="currentSubtotalLineItemsQuantity") - current_subtotal_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="currentSubtotalPriceSet") - current_tax_lines = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxLine))), graphql_name="currentTaxLines" - ) - current_total_discounts_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="currentTotalDiscountsSet") - current_total_duties_set = sgqlc.types.Field(MoneyBag, graphql_name="currentTotalDutiesSet") - current_total_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="currentTotalPriceSet") - current_total_tax_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="currentTotalTaxSet") - current_total_weight = sgqlc.types.Field(sgqlc.types.non_null(UnsignedInt64), graphql_name="currentTotalWeight") - custom_attributes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Attribute))), graphql_name="customAttributes" - ) - customer = sgqlc.types.Field(Customer, graphql_name="customer") - customer_accepts_marketing = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="customerAcceptsMarketing") - customer_journey_summary = sgqlc.types.Field(CustomerJourneySummary, graphql_name="customerJourneySummary") - customer_locale = sgqlc.types.Field(String, graphql_name="customerLocale") - discount_applications = sgqlc.types.Field( - sgqlc.types.non_null(DiscountApplicationConnection), - graphql_name="discountApplications", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - discount_code = sgqlc.types.Field(String, graphql_name="discountCode") - discount_codes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="discountCodes" - ) - display_address = sgqlc.types.Field(MailingAddress, graphql_name="displayAddress") - display_financial_status = sgqlc.types.Field(OrderDisplayFinancialStatus, graphql_name="displayFinancialStatus") - display_fulfillment_status = sgqlc.types.Field( - sgqlc.types.non_null(OrderDisplayFulfillmentStatus), graphql_name="displayFulfillmentStatus" - ) - disputes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("OrderDisputeSummary"))), graphql_name="disputes" - ) - edited = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="edited") - email = sgqlc.types.Field(String, graphql_name="email") - estimated_taxes = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="estimatedTaxes") - fulfillable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="fulfillable") - fulfillment_orders = sgqlc.types.Field( - sgqlc.types.non_null(FulfillmentOrderConnection), - graphql_name="fulfillmentOrders", - args=sgqlc.types.ArgDict( - ( - ("displayable", sgqlc.types.Arg(Boolean, graphql_name="displayable", default=False)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - fulfillments = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Fulfillment))), - graphql_name="fulfillments", - args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)),)), - ) - fully_paid = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="fullyPaid") - line_items = sgqlc.types.Field( - sgqlc.types.non_null(LineItemConnection), - graphql_name="lineItems", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - merchant_editable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="merchantEditable") - merchant_editable_errors = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="merchantEditableErrors" - ) - merchant_of_record_app = sgqlc.types.Field(OrderApp, graphql_name="merchantOfRecordApp") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - net_payment_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="netPaymentSet") - non_fulfillable_line_items = sgqlc.types.Field( - sgqlc.types.non_null(LineItemConnection), - graphql_name="nonFulfillableLineItems", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - note = sgqlc.types.Field(String, graphql_name="note") - original_total_duties_set = sgqlc.types.Field(MoneyBag, graphql_name="originalTotalDutiesSet") - original_total_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="originalTotalPriceSet") - payment_collection_details = sgqlc.types.Field( - sgqlc.types.non_null(OrderPaymentCollectionDetails), graphql_name="paymentCollectionDetails" - ) - payment_gateway_names = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="paymentGatewayNames" - ) - payment_terms = sgqlc.types.Field("PaymentTerms", graphql_name="paymentTerms") - phone = sgqlc.types.Field(String, graphql_name="phone") - physical_location = sgqlc.types.Field(Location, graphql_name="physicalLocation") - presentment_currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="presentmentCurrencyCode") - processed_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="processedAt") - publication = sgqlc.types.Field("Publication", graphql_name="publication") - purchasing_entity = sgqlc.types.Field("PurchasingEntity", graphql_name="purchasingEntity") - refund_discrepancy_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="refundDiscrepancySet") - refundable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="refundable") - refunds = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("Refund"))), - graphql_name="refunds", - args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)),)), - ) - registered_source_url = sgqlc.types.Field(URL, graphql_name="registeredSourceUrl") - requires_shipping = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresShipping") - restockable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="restockable") - risk_level = sgqlc.types.Field(sgqlc.types.non_null(OrderRiskLevel), graphql_name="riskLevel") - risks = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(OrderRisk))), - graphql_name="risks", - args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)),)), - ) - shipping_address = sgqlc.types.Field(MailingAddress, graphql_name="shippingAddress") - shipping_line = sgqlc.types.Field(ShippingLine, graphql_name="shippingLine") - shipping_lines = sgqlc.types.Field( - sgqlc.types.non_null(ShippingLineConnection), - graphql_name="shippingLines", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - source_identifier = sgqlc.types.Field(String, graphql_name="sourceIdentifier") - subtotal_line_items_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="subtotalLineItemsQuantity") - subtotal_price_set = sgqlc.types.Field(MoneyBag, graphql_name="subtotalPriceSet") - suggested_refund = sgqlc.types.Field( - SuggestedRefund, - graphql_name="suggestedRefund", - args=sgqlc.types.ArgDict( - ( - ("shipping_amount", sgqlc.types.Arg(Money, graphql_name="shippingAmount", default=None)), - ("refund_shipping", sgqlc.types.Arg(Boolean, graphql_name="refundShipping", default=None)), - ( - "refund_line_items", - sgqlc.types.Arg( - sgqlc.types.list_of(sgqlc.types.non_null(RefundLineItemInput)), - graphql_name="refundLineItems", - default=None, - ), - ), - ( - "refund_duties", - sgqlc.types.Arg( - sgqlc.types.list_of(sgqlc.types.non_null(RefundDutyInput)), - graphql_name="refundDuties", - default=None, - ), - ), - ("suggest_full_refund", sgqlc.types.Arg(Boolean, graphql_name="suggestFullRefund", default=False)), - ) - ), - ) - tags = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="tags") - tax_lines = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(TaxLine))), graphql_name="taxLines") - taxes_included = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxesIncluded") - test = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="test") - total_capturable_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalCapturableSet") - total_discounts_set = sgqlc.types.Field(MoneyBag, graphql_name="totalDiscountsSet") - total_outstanding_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalOutstandingSet") - total_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalPriceSet") - total_received_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalReceivedSet") - total_refunded_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalRefundedSet") - total_refunded_shipping_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalRefundedShippingSet") - total_shipping_price_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalShippingPriceSet") - total_tax_set = sgqlc.types.Field(MoneyBag, graphql_name="totalTaxSet") - total_tip_received_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalTipReceivedSet") - total_weight = sgqlc.types.Field(UnsignedInt64, graphql_name="totalWeight") - transactions = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("OrderTransaction"))), - graphql_name="transactions", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("capturable", sgqlc.types.Arg(Boolean, graphql_name="capturable", default=None)), - ("manually_resolvable", sgqlc.types.Arg(Boolean, graphql_name="manuallyResolvable", default=None)), - ) - ), - ) - unpaid = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="unpaid") - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - - -class OrderAgreement(sgqlc.types.Type, SalesAgreement): - __schema__ = shopify_schema - __field_names__ = ("order",) - order = sgqlc.types.Field(sgqlc.types.non_null(Order), graphql_name="order") - - -class OrderCreateMandatePaymentUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(OrderCreateMandatePaymentUserErrorCode, graphql_name="code") - - -class OrderDisputeSummary(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("initiated_as", "status") - initiated_as = sgqlc.types.Field(sgqlc.types.non_null(DisputeType), graphql_name="initiatedAs") - status = sgqlc.types.Field(sgqlc.types.non_null(DisputeStatus), graphql_name="status") - - -class OrderEditAgreement(sgqlc.types.Type, SalesAgreement): - __schema__ = shopify_schema - __field_names__ = () - - -class OrderInvoiceSendUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(OrderInvoiceSendUserErrorCode, graphql_name="code") - - -class OrderTransaction(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "account_number", - "amount_set", - "authorization_code", - "authorization_expires_at", - "created_at", - "error_code", - "fees", - "formatted_gateway", - "gateway", - "kind", - "manually_capturable", - "maximum_refundable_v2", - "order", - "parent_transaction", - "payment_icon", - "processed_at", - "receipt_json", - "settlement_currency", - "settlement_currency_rate", - "shopify_payments_set", - "status", - "test", - "total_unsettled_set", - "user", - ) - account_number = sgqlc.types.Field(String, graphql_name="accountNumber") - amount_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="amountSet") - authorization_code = sgqlc.types.Field(String, graphql_name="authorizationCode") - authorization_expires_at = sgqlc.types.Field(DateTime, graphql_name="authorizationExpiresAt") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - error_code = sgqlc.types.Field(OrderTransactionErrorCode, graphql_name="errorCode") - fees = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("TransactionFee"))), graphql_name="fees") - formatted_gateway = sgqlc.types.Field(String, graphql_name="formattedGateway") - gateway = sgqlc.types.Field(String, graphql_name="gateway") - kind = sgqlc.types.Field(sgqlc.types.non_null(OrderTransactionKind), graphql_name="kind") - manually_capturable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="manuallyCapturable") - maximum_refundable_v2 = sgqlc.types.Field(MoneyV2, graphql_name="maximumRefundableV2") - order = sgqlc.types.Field(Order, graphql_name="order") - parent_transaction = sgqlc.types.Field("OrderTransaction", graphql_name="parentTransaction") - payment_icon = sgqlc.types.Field(Image, graphql_name="paymentIcon") - processed_at = sgqlc.types.Field(DateTime, graphql_name="processedAt") - receipt_json = sgqlc.types.Field(JSON, graphql_name="receiptJson") - settlement_currency = sgqlc.types.Field(CurrencyCode, graphql_name="settlementCurrency") - settlement_currency_rate = sgqlc.types.Field(Decimal, graphql_name="settlementCurrencyRate") - shopify_payments_set = sgqlc.types.Field(ShopifyPaymentsTransactionSet, graphql_name="shopifyPaymentsSet") - status = sgqlc.types.Field(sgqlc.types.non_null(OrderTransactionStatus), graphql_name="status") - test = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="test") - total_unsettled_set = sgqlc.types.Field(MoneyBag, graphql_name="totalUnsettledSet") - user = sgqlc.types.Field("StaffMember", graphql_name="user") - - -class PaymentMandate(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("payment_instrument",) - payment_instrument = sgqlc.types.Field(sgqlc.types.non_null("PaymentInstrument"), graphql_name="paymentInstrument") - - -class PaymentSchedule(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("amount", "completed_at", "due_at", "issued_at") - amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="amount") - completed_at = sgqlc.types.Field(DateTime, graphql_name="completedAt") - due_at = sgqlc.types.Field(DateTime, graphql_name="dueAt") - issued_at = sgqlc.types.Field(DateTime, graphql_name="issuedAt") - - -class PaymentTerms(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "due_in_days", - "overdue", - "payment_schedules", - "payment_terms_name", - "payment_terms_type", - "translated_name", - ) - due_in_days = sgqlc.types.Field(Int, graphql_name="dueInDays") - overdue = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="overdue") - payment_schedules = sgqlc.types.Field( - sgqlc.types.non_null(PaymentScheduleConnection), - graphql_name="paymentSchedules", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - payment_terms_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="paymentTermsName") - payment_terms_type = sgqlc.types.Field(sgqlc.types.non_null(PaymentTermsType), graphql_name="paymentTermsType") - translated_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="translatedName") - - -class PaymentTermsCreateUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(PaymentTermsCreateUserErrorCode, graphql_name="code") - - -class PaymentTermsDeleteUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(PaymentTermsDeleteUserErrorCode, graphql_name="code") - - -class PaymentTermsTemplate(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("description", "due_in_days", "name", "payment_terms_type", "translated_name") - description = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="description") - due_in_days = sgqlc.types.Field(Int, graphql_name="dueInDays") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - payment_terms_type = sgqlc.types.Field(sgqlc.types.non_null(PaymentTermsType), graphql_name="paymentTermsType") - translated_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="translatedName") - - -class PaymentTermsUpdateUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(PaymentTermsUpdateUserErrorCode, graphql_name="code") - - -class PriceList(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("context_rule", "currency", "name", "parent", "prices") - context_rule = sgqlc.types.Field(PriceListContextRule, graphql_name="contextRule") - currency = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currency") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - parent = sgqlc.types.Field(PriceListParent, graphql_name="parent") - prices = sgqlc.types.Field( - sgqlc.types.non_null(PriceListPriceConnection), - graphql_name="prices", - args=sgqlc.types.ArgDict( - ( - ("origin_type", sgqlc.types.Arg(PriceListPriceOriginType, graphql_name="originType", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - - -class PriceListPriceUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(PriceListPriceUserErrorCode, graphql_name="code") - - -class PriceListUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(PriceListUserErrorCode, graphql_name="code") - - -class PriceRule(sgqlc.types.Type, CommentEventSubject, HasEvents, LegacyInteroperability, Node): - __schema__ = shopify_schema - __field_names__ = ( - "allocation_limit", - "allocation_method", - "app", - "combines_with", - "created_at", - "customer_selection", - "discount_class", - "discount_codes", - "discount_codes_count", - "ends_at", - "features", - "item_entitlements", - "item_prerequisites", - "once_per_customer", - "prerequisite_quantity_range", - "prerequisite_shipping_price_range", - "prerequisite_subtotal_range", - "prerequisite_to_entitlement_quantity_ratio", - "shareable_urls", - "shipping_entitlements", - "starts_at", - "status", - "summary", - "target", - "title", - "total_sales", - "usage_count", - "usage_limit", - "validity_period", - "value_v2", - ) - allocation_limit = sgqlc.types.Field(Int, graphql_name="allocationLimit") - allocation_method = sgqlc.types.Field(sgqlc.types.non_null(PriceRuleAllocationMethod), graphql_name="allocationMethod") - app = sgqlc.types.Field(App, graphql_name="app") - combines_with = sgqlc.types.Field(sgqlc.types.non_null(DiscountCombinesWith), graphql_name="combinesWith") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - customer_selection = sgqlc.types.Field(sgqlc.types.non_null(PriceRuleCustomerSelection), graphql_name="customerSelection") - discount_class = sgqlc.types.Field(sgqlc.types.non_null(DiscountClass), graphql_name="discountClass") - discount_codes = sgqlc.types.Field( - sgqlc.types.non_null(PriceRuleDiscountCodeConnection), - graphql_name="discountCodes", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(DiscountCodeSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ("saved_search_id", sgqlc.types.Arg(ID, graphql_name="savedSearchId", default=None)), - ) - ), - ) - discount_codes_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="discountCodesCount") - ends_at = sgqlc.types.Field(DateTime, graphql_name="endsAt") - features = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(PriceRuleFeature))), graphql_name="features") - item_entitlements = sgqlc.types.Field(sgqlc.types.non_null(PriceRuleItemEntitlements), graphql_name="itemEntitlements") - item_prerequisites = sgqlc.types.Field(sgqlc.types.non_null(PriceRuleLineItemPrerequisites), graphql_name="itemPrerequisites") - once_per_customer = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="oncePerCustomer") - prerequisite_quantity_range = sgqlc.types.Field(PriceRuleQuantityRange, graphql_name="prerequisiteQuantityRange") - prerequisite_shipping_price_range = sgqlc.types.Field(PriceRuleMoneyRange, graphql_name="prerequisiteShippingPriceRange") - prerequisite_subtotal_range = sgqlc.types.Field(PriceRuleMoneyRange, graphql_name="prerequisiteSubtotalRange") - prerequisite_to_entitlement_quantity_ratio = sgqlc.types.Field( - PriceRulePrerequisiteToEntitlementQuantityRatio, graphql_name="prerequisiteToEntitlementQuantityRatio" - ) - shareable_urls = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(PriceRuleShareableUrl))), - graphql_name="shareableUrls", - ) - shipping_entitlements = sgqlc.types.Field(sgqlc.types.non_null(PriceRuleShippingLineEntitlements), graphql_name="shippingEntitlements") - starts_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="startsAt") - status = sgqlc.types.Field(sgqlc.types.non_null(PriceRuleStatus), graphql_name="status") - summary = sgqlc.types.Field(String, graphql_name="summary") - target = sgqlc.types.Field(sgqlc.types.non_null(PriceRuleTarget), graphql_name="target") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - total_sales = sgqlc.types.Field(MoneyV2, graphql_name="totalSales") - usage_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="usageCount") - usage_limit = sgqlc.types.Field(Int, graphql_name="usageLimit") - validity_period = sgqlc.types.Field(sgqlc.types.non_null(PriceRuleValidityPeriod), graphql_name="validityPeriod") - value_v2 = sgqlc.types.Field(sgqlc.types.non_null("PricingValue"), graphql_name="valueV2") - - -class PriceRuleDiscountCode(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("app", "code", "usage_count") - app = sgqlc.types.Field(App, graphql_name="app") - code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="code") - usage_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="usageCount") - - -class PriceRuleUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(PriceRuleErrorCode, graphql_name="code") - - -class PrivateMetafield(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("created_at", "key", "namespace", "updated_at", "value", "value_type") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") - namespace = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="namespace") - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - value = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="value") - value_type = sgqlc.types.Field(sgqlc.types.non_null(PrivateMetafieldValueType), graphql_name="valueType") - - -class Product( - sgqlc.types.Type, - HasMetafieldDefinitions, - HasMetafields, - HasPublishedTranslations, - LegacyInteroperability, - Navigable, - Node, - OnlineStorePreviewable, - Publishable, -): - __schema__ = shopify_schema - __field_names__ = ( - "collections", - "contextual_pricing", - "created_at", - "description", - "description_html", - "featured_image", - "featured_media", - "feedback", - "gift_card_template_suffix", - "handle", - "has_only_default_variant", - "has_out_of_stock_variants", - "images", - "in_collection", - "is_gift_card", - "media", - "media_count", - "online_store_url", - "options", - "price_range_v2", - "product_category", - "product_type", - "published_at", - "requires_selling_plan", - "resource_publication_on_current_publication", - "selling_plan_group_count", - "selling_plan_groups", - "seo", - "status", - "tags", - "template_suffix", - "title", - "total_inventory", - "total_variants", - "tracks_inventory", - "updated_at", - "variants", - "vendor", - ) - collections = sgqlc.types.Field( - sgqlc.types.non_null(CollectionConnection), - graphql_name="collections", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(CollectionSortKeys, graphql_name="sortKey", default="ID")), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - contextual_pricing = sgqlc.types.Field( - sgqlc.types.non_null(ProductContextualPricing), - graphql_name="contextualPricing", - args=sgqlc.types.ArgDict( - ( - ( - "context", - sgqlc.types.Arg(sgqlc.types.non_null(ContextualPricingContext), graphql_name="context", default=None), - ), - ) - ), - ) - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - description = sgqlc.types.Field( - sgqlc.types.non_null(String), - graphql_name="description", - args=sgqlc.types.ArgDict((("truncate_at", sgqlc.types.Arg(Int, graphql_name="truncateAt", default=None)),)), - ) - description_html = sgqlc.types.Field(sgqlc.types.non_null(HTML), graphql_name="descriptionHtml") - featured_image = sgqlc.types.Field(Image, graphql_name="featuredImage") - featured_media = sgqlc.types.Field(Media, graphql_name="featuredMedia") - feedback = sgqlc.types.Field(ResourceFeedback, graphql_name="feedback") - gift_card_template_suffix = sgqlc.types.Field(String, graphql_name="giftCardTemplateSuffix") - handle = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="handle") - has_only_default_variant = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasOnlyDefaultVariant") - has_out_of_stock_variants = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="hasOutOfStockVariants") - images = sgqlc.types.Field( - sgqlc.types.non_null(ImageConnection), - graphql_name="images", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(ProductImageSortKeys, graphql_name="sortKey", default="POSITION")), - ) - ), - ) - in_collection = sgqlc.types.Field( - sgqlc.types.non_null(Boolean), - graphql_name="inCollection", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - is_gift_card = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isGiftCard") - media = sgqlc.types.Field( - sgqlc.types.non_null(MediaConnection), - graphql_name="media", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(ProductMediaSortKeys, graphql_name="sortKey", default="POSITION")), - ) - ), - ) - media_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="mediaCount") - online_store_url = sgqlc.types.Field(URL, graphql_name="onlineStoreUrl") - options = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ProductOption"))), - graphql_name="options", - args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)),)), - ) - price_range_v2 = sgqlc.types.Field(sgqlc.types.non_null(ProductPriceRangeV2), graphql_name="priceRangeV2") - product_category = sgqlc.types.Field(ProductCategory, graphql_name="productCategory") - product_type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="productType") - published_at = sgqlc.types.Field(DateTime, graphql_name="publishedAt") - requires_selling_plan = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="requiresSellingPlan") - resource_publication_on_current_publication = sgqlc.types.Field( - ResourcePublicationV2, graphql_name="resourcePublicationOnCurrentPublication" - ) - selling_plan_group_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="sellingPlanGroupCount") - selling_plan_groups = sgqlc.types.Field( - sgqlc.types.non_null(SellingPlanGroupConnection), - graphql_name="sellingPlanGroups", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - seo = sgqlc.types.Field(sgqlc.types.non_null(SEO), graphql_name="seo") - status = sgqlc.types.Field(sgqlc.types.non_null(ProductStatus), graphql_name="status") - tags = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="tags") - template_suffix = sgqlc.types.Field(String, graphql_name="templateSuffix") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - total_inventory = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalInventory") - total_variants = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="totalVariants") - tracks_inventory = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="tracksInventory") - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - variants = sgqlc.types.Field( - sgqlc.types.non_null(ProductVariantConnection), - graphql_name="variants", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(ProductVariantSortKeys, graphql_name="sortKey", default="POSITION")), - ) - ), - ) - vendor = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="vendor") - - -class ProductChangeStatusUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(ProductChangeStatusUserErrorCode, graphql_name="code") - - -class ProductOption(sgqlc.types.Type, HasPublishedTranslations, Node): - __schema__ = shopify_schema - __field_names__ = ("name", "position", "values") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - position = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="position") - values = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="values") - - -class ProductSale(sgqlc.types.Type, Sale): - __schema__ = shopify_schema - __field_names__ = ("line_item",) - line_item = sgqlc.types.Field(sgqlc.types.non_null(LineItem), graphql_name="lineItem") - - -class ProductTaxonomyNode(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("full_name", "is_leaf", "is_root", "name") - full_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="fullName") - is_leaf = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isLeaf") - is_root = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isRoot") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - - -class ProductVariant( - sgqlc.types.Type, - HasMetafieldDefinitions, - HasMetafields, - HasPublishedTranslations, - LegacyInteroperability, - Navigable, - Node, -): - __schema__ = shopify_schema - __field_names__ = ( - "available_for_sale", - "barcode", - "compare_at_price", - "contextual_pricing", - "created_at", - "delivery_profile", - "display_name", - "fulfillment_service_editable", - "image", - "inventory_item", - "inventory_policy", - "inventory_quantity", - "media", - "position", - "price", - "product", - "selected_options", - "sellable_online_quantity", - "selling_plan_group_count", - "selling_plan_groups", - "sku", - "tax_code", - "taxable", - "title", - "updated_at", - "weight", - "weight_unit", - ) - available_for_sale = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="availableForSale") - barcode = sgqlc.types.Field(String, graphql_name="barcode") - compare_at_price = sgqlc.types.Field(Money, graphql_name="compareAtPrice") - contextual_pricing = sgqlc.types.Field( - sgqlc.types.non_null(ProductVariantContextualPricing), - graphql_name="contextualPricing", - args=sgqlc.types.ArgDict( - ( - ( - "context", - sgqlc.types.Arg(sgqlc.types.non_null(ContextualPricingContext), graphql_name="context", default=None), - ), - ) - ), - ) - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - delivery_profile = sgqlc.types.Field(DeliveryProfile, graphql_name="deliveryProfile") - display_name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="displayName") - fulfillment_service_editable = sgqlc.types.Field(sgqlc.types.non_null(EditableProperty), graphql_name="fulfillmentServiceEditable") - image = sgqlc.types.Field(Image, graphql_name="image") - inventory_item = sgqlc.types.Field(sgqlc.types.non_null(InventoryItem), graphql_name="inventoryItem") - inventory_policy = sgqlc.types.Field(sgqlc.types.non_null(ProductVariantInventoryPolicy), graphql_name="inventoryPolicy") - inventory_quantity = sgqlc.types.Field(Int, graphql_name="inventoryQuantity") - media = sgqlc.types.Field( - sgqlc.types.non_null(MediaConnection), - graphql_name="media", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - position = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="position") - price = sgqlc.types.Field(sgqlc.types.non_null(Money), graphql_name="price") - product = sgqlc.types.Field(sgqlc.types.non_null(Product), graphql_name="product") - selected_options = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SelectedOption))), graphql_name="selectedOptions" - ) - sellable_online_quantity = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="sellableOnlineQuantity") - selling_plan_group_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="sellingPlanGroupCount") - selling_plan_groups = sgqlc.types.Field( - sgqlc.types.non_null(SellingPlanGroupConnection), - graphql_name="sellingPlanGroups", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - sku = sgqlc.types.Field(String, graphql_name="sku") - tax_code = sgqlc.types.Field(String, graphql_name="taxCode") - taxable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxable") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - weight = sgqlc.types.Field(Float, graphql_name="weight") - weight_unit = sgqlc.types.Field(sgqlc.types.non_null(WeightUnit), graphql_name="weightUnit") - - -class ProductVariantsBulkCreateUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(ProductVariantsBulkCreateUserErrorCode, graphql_name="code") - - -class ProductVariantsBulkDeleteUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(ProductVariantsBulkDeleteUserErrorCode, graphql_name="code") - - -class ProductVariantsBulkReorderUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(ProductVariantsBulkReorderUserErrorCode, graphql_name="code") - - -class ProductVariantsBulkUpdateUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(ProductVariantsBulkUpdateUserErrorCode, graphql_name="code") - - -class PubSubWebhookSubscriptionCreateUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(PubSubWebhookSubscriptionCreateUserErrorCode, graphql_name="code") - - -class PubSubWebhookSubscriptionUpdateUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(PubSubWebhookSubscriptionUpdateUserErrorCode, graphql_name="code") - - -class Publication(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "app", - "collection_publications_v3", - "collections", - "has_collection", - "name", - "product_publications_v3", - "products", - "supports_future_publishing", - ) - app = sgqlc.types.Field(sgqlc.types.non_null(App), graphql_name="app") - collection_publications_v3 = sgqlc.types.Field( - sgqlc.types.non_null(ResourcePublicationConnection), - graphql_name="collectionPublicationsV3", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - collections = sgqlc.types.Field( - sgqlc.types.non_null(CollectionConnection), - graphql_name="collections", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - has_collection = sgqlc.types.Field( - sgqlc.types.non_null(Boolean), - graphql_name="hasCollection", - args=sgqlc.types.ArgDict((("id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="id", default=None)),)), - ) - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - product_publications_v3 = sgqlc.types.Field( - sgqlc.types.non_null(ResourcePublicationConnection), - graphql_name="productPublicationsV3", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - products = sgqlc.types.Field( - sgqlc.types.non_null(ProductConnection), - graphql_name="products", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - supports_future_publishing = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="supportsFuturePublishing") - - -class Refund(sgqlc.types.Type, LegacyInteroperability, Node): - __schema__ = shopify_schema - __field_names__ = ( - "created_at", - "duties", - "note", - "order", - "refund_line_items", - "staff_member", - "total_refunded_set", - "transactions", - "updated_at", - ) - created_at = sgqlc.types.Field(DateTime, graphql_name="createdAt") - duties = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(RefundDuty)), graphql_name="duties") - note = sgqlc.types.Field(String, graphql_name="note") - order = sgqlc.types.Field(sgqlc.types.non_null(Order), graphql_name="order") - refund_line_items = sgqlc.types.Field( - sgqlc.types.non_null(RefundLineItemConnection), - graphql_name="refundLineItems", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - staff_member = sgqlc.types.Field("StaffMember", graphql_name="staffMember") - total_refunded_set = sgqlc.types.Field(sgqlc.types.non_null(MoneyBag), graphql_name="totalRefundedSet") - transactions = sgqlc.types.Field( - sgqlc.types.non_null(OrderTransactionConnection), - graphql_name="transactions", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - - -class RefundAgreement(sgqlc.types.Type, SalesAgreement): - __schema__ = shopify_schema - __field_names__ = ("refund",) - refund = sgqlc.types.Field(sgqlc.types.non_null(Refund), graphql_name="refund") - - -class SavedSearch(sgqlc.types.Type, LegacyInteroperability, Node): - __schema__ = shopify_schema - __field_names__ = ("filters", "name", "query", "resource_type", "search_terms") - filters = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SearchFilter))), graphql_name="filters") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - query = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="query") - resource_type = sgqlc.types.Field(sgqlc.types.non_null(SearchResultType), graphql_name="resourceType") - search_terms = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="searchTerms") - - -class ScriptDiscountApplication(sgqlc.types.Type, DiscountApplication): - __schema__ = shopify_schema - __field_names__ = ("title",) - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - - -class ScriptTag(sgqlc.types.Type, LegacyInteroperability, Node): - __schema__ = shopify_schema - __field_names__ = ("cache", "created_at", "display_scope", "src", "updated_at") - cache = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="cache") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - display_scope = sgqlc.types.Field(sgqlc.types.non_null(ScriptTagDisplayScope), graphql_name="displayScope") - src = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="src") - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - - -class Segment(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("creation_date", "last_edit_date", "name", "query") - creation_date = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="creationDate") - last_edit_date = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="lastEditDate") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - query = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="query") - - -class SegmentAssociationFilter(sgqlc.types.Type, SegmentFilter): - __schema__ = shopify_schema - __field_names__ = ("values",) - values = sgqlc.types.Field( - sgqlc.types.non_null(SegmentAssociationFilterValueConnection), - graphql_name="values", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ) - ), - ) - - -class SegmentBooleanFilter(sgqlc.types.Type, SegmentFilter): - __schema__ = shopify_schema - __field_names__ = () - - -class SegmentDateFilter(sgqlc.types.Type, SegmentFilter): - __schema__ = shopify_schema - __field_names__ = () - - -class SegmentEnumFilter(sgqlc.types.Type, SegmentFilter): - __schema__ = shopify_schema - __field_names__ = ("values",) - values = sgqlc.types.Field( - sgqlc.types.non_null(SegmentEnumFilterValueConnection), - graphql_name="values", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ) - ), - ) - - -class SegmentEventFilter(sgqlc.types.Type, SegmentFilter): - __schema__ = shopify_schema - __field_names__ = ("parameters", "return_value_type", "values") - parameters = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(SegmentEventFilterParameter))), - graphql_name="parameters", - ) - return_value_type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="returnValueType") - values = sgqlc.types.Field( - sgqlc.types.non_null(SegmentEventFilterValueConnection), - graphql_name="values", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ) - ), - ) - - -class SegmentFloatFilter(sgqlc.types.Type, SegmentFilter): - __schema__ = shopify_schema - __field_names__ = () - - -class SegmentIntegerFilter(sgqlc.types.Type, SegmentFilter): - __schema__ = shopify_schema - __field_names__ = () - - -class SegmentStringFilter(sgqlc.types.Type, SegmentFilter): - __schema__ = shopify_schema - __field_names__ = ("values",) - values = sgqlc.types.Field( - sgqlc.types.non_null(SegmentStringFilterValueConnection), - graphql_name="values", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ) - ), - ) - - -class SellingPlan(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "billing_policy", - "category", - "created_at", - "delivery_policy", - "description", - "inventory_policy", - "name", - "options", - "position", - "pricing_policies", - ) - billing_policy = sgqlc.types.Field(sgqlc.types.non_null("SellingPlanBillingPolicy"), graphql_name="billingPolicy") - category = sgqlc.types.Field(SellingPlanCategory, graphql_name="category") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - delivery_policy = sgqlc.types.Field(sgqlc.types.non_null("SellingPlanDeliveryPolicy"), graphql_name="deliveryPolicy") - description = sgqlc.types.Field(String, graphql_name="description") - inventory_policy = sgqlc.types.Field(SellingPlanInventoryPolicy, graphql_name="inventoryPolicy") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - options = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="options") - position = sgqlc.types.Field(Int, graphql_name="position") - pricing_policies = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("SellingPlanPricingPolicy"))), - graphql_name="pricingPolicies", - ) - - -class SellingPlanFixedPricingPolicy(sgqlc.types.Type, SellingPlanPricingPolicyBase): - __schema__ = shopify_schema - __field_names__ = ("created_at",) - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - - -class SellingPlanGroup(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "app_id", - "applies_to_product", - "applies_to_product_variant", - "applies_to_product_variants", - "created_at", - "description", - "merchant_code", - "name", - "options", - "position", - "product_count", - "product_variant_count", - "product_variants", - "products", - "selling_plans", - "summary", - ) - app_id = sgqlc.types.Field(String, graphql_name="appId") - applies_to_product = sgqlc.types.Field( - sgqlc.types.non_null(Boolean), - graphql_name="appliesToProduct", - args=sgqlc.types.ArgDict((("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)),)), - ) - applies_to_product_variant = sgqlc.types.Field( - sgqlc.types.non_null(Boolean), - graphql_name="appliesToProductVariant", - args=sgqlc.types.ArgDict( - ( - ( - "product_variant_id", - sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productVariantId", default=None), - ), - ) - ), - ) - applies_to_product_variants = sgqlc.types.Field( - sgqlc.types.non_null(Boolean), - graphql_name="appliesToProductVariants", - args=sgqlc.types.ArgDict((("product_id", sgqlc.types.Arg(sgqlc.types.non_null(ID), graphql_name="productId", default=None)),)), - ) - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - description = sgqlc.types.Field(String, graphql_name="description") - merchant_code = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="merchantCode") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - options = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="options") - position = sgqlc.types.Field(Int, graphql_name="position") - product_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="productCount") - product_variant_count = sgqlc.types.Field( - sgqlc.types.non_null(Int), - graphql_name="productVariantCount", - args=sgqlc.types.ArgDict((("product_id", sgqlc.types.Arg(ID, graphql_name="productId", default=None)),)), - ) - product_variants = sgqlc.types.Field( - sgqlc.types.non_null(ProductVariantConnection), - graphql_name="productVariants", - args=sgqlc.types.ArgDict( - ( - ("product_id", sgqlc.types.Arg(ID, graphql_name="productId", default=None)), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - products = sgqlc.types.Field( - sgqlc.types.non_null(ProductConnection), - graphql_name="products", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - selling_plans = sgqlc.types.Field( - sgqlc.types.non_null(SellingPlanConnection), - graphql_name="sellingPlans", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - summary = sgqlc.types.Field(String, graphql_name="summary") - - -class SellingPlanGroupUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(SellingPlanGroupUserErrorCode, graphql_name="code") - - -class SellingPlanRecurringPricingPolicy(sgqlc.types.Type, SellingPlanPricingPolicyBase): - __schema__ = shopify_schema - __field_names__ = ("after_cycle", "created_at") - after_cycle = sgqlc.types.Field(Int, graphql_name="afterCycle") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - - -class ShippingLineSale(sgqlc.types.Type, Sale): - __schema__ = shopify_schema - __field_names__ = ("shipping_line",) - shipping_line = sgqlc.types.Field(ShippingLine, graphql_name="shippingLine") - - -class Shop(sgqlc.types.Type, HasMetafields, HasPublishedTranslations, Node): - __schema__ = shopify_schema - __field_names__ = ( - "alerts", - "all_product_categories", - "assigned_fulfillment_orders", - "available_channel_apps", - "billing_address", - "channel_definitions_for_installed_channels", - "checkout_api_supported", - "contact_email", - "countries_in_shipping_zones", - "currency_code", - "currency_formats", - "currency_settings", - "customer_accounts", - "customer_tags", - "description", - "draft_order_tags", - "email", - "enabled_presentment_currencies", - "features", - "fulfillment_services", - "iana_timezone", - "limited_pending_order_count", - "merchant_approval_signals", - "myshopify_domain", - "name", - "navigation_settings", - "order_number_format_prefix", - "order_number_format_suffix", - "order_tags", - "payment_settings", - "plan", - "primary_domain", - "product_images", - "product_tags", - "product_types", - "product_vendors", - "publication_count", - "resource_limits", - "rich_text_editor_url", - "search", - "search_filters", - "setup_required", - "ships_to_countries", - "shop_policies", - "staff_members", - "storefront_access_tokens", - "tax_shipping", - "taxes_included", - "timezone_abbreviation", - "timezone_offset", - "timezone_offset_minutes", - "transactional_sms_disabled", - "unit_system", - "uploaded_images_by_ids", - "url", - "weight_unit", - ) - alerts = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ShopAlert))), graphql_name="alerts") - all_product_categories = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ProductCategory))), - graphql_name="allProductCategories", - ) - assigned_fulfillment_orders = sgqlc.types.Field( - sgqlc.types.non_null(FulfillmentOrderConnection), - graphql_name="assignedFulfillmentOrders", - args=sgqlc.types.ArgDict( - ( - ( - "assignment_status", - sgqlc.types.Arg(FulfillmentOrderAssignmentStatus, graphql_name="assignmentStatus", default=None), - ), - ( - "location_ids", - sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(ID)), graphql_name="locationIds", default=None), - ), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(FulfillmentOrderSortKeys, graphql_name="sortKey", default="ID")), - ) - ), - ) - available_channel_apps = sgqlc.types.Field( - sgqlc.types.non_null(AppConnection), - graphql_name="availableChannelApps", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - billing_address = sgqlc.types.Field(sgqlc.types.non_null(MailingAddress), graphql_name="billingAddress") - channel_definitions_for_installed_channels = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(AvailableChannelDefinitionsByChannel))), - graphql_name="channelDefinitionsForInstalledChannels", - ) - checkout_api_supported = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="checkoutApiSupported") - contact_email = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="contactEmail") - countries_in_shipping_zones = sgqlc.types.Field(sgqlc.types.non_null(CountriesInShippingZones), graphql_name="countriesInShippingZones") - currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currencyCode") - currency_formats = sgqlc.types.Field(sgqlc.types.non_null(CurrencyFormats), graphql_name="currencyFormats") - currency_settings = sgqlc.types.Field( - sgqlc.types.non_null(CurrencySettingConnection), - graphql_name="currencySettings", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - customer_accounts = sgqlc.types.Field(sgqlc.types.non_null(ShopCustomerAccountsSetting), graphql_name="customerAccounts") - customer_tags = sgqlc.types.Field( - sgqlc.types.non_null(StringConnection), - graphql_name="customerTags", - args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="first", default=None)),)), - ) - description = sgqlc.types.Field(String, graphql_name="description") - draft_order_tags = sgqlc.types.Field( - sgqlc.types.non_null(StringConnection), - graphql_name="draftOrderTags", - args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="first", default=None)),)), - ) - email = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="email") - enabled_presentment_currencies = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CurrencyCode))), - graphql_name="enabledPresentmentCurrencies", - ) - features = sgqlc.types.Field(sgqlc.types.non_null(ShopFeatures), graphql_name="features") - fulfillment_services = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(FulfillmentService))), - graphql_name="fulfillmentServices", - ) - iana_timezone = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="ianaTimezone") - limited_pending_order_count = sgqlc.types.Field(sgqlc.types.non_null(LimitedPendingOrderCount), graphql_name="limitedPendingOrderCount") - merchant_approval_signals = sgqlc.types.Field(MerchantApprovalSignals, graphql_name="merchantApprovalSignals") - myshopify_domain = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="myshopifyDomain") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - navigation_settings = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(NavigationItem))), - graphql_name="navigationSettings", - ) - order_number_format_prefix = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="orderNumberFormatPrefix") - order_number_format_suffix = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="orderNumberFormatSuffix") - order_tags = sgqlc.types.Field( - sgqlc.types.non_null(StringConnection), - graphql_name="orderTags", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="first", default=None)), - ("sort", sgqlc.types.Arg(ShopTagSort, graphql_name="sort", default="ALPHABETICAL")), - ) - ), - ) - payment_settings = sgqlc.types.Field(sgqlc.types.non_null(PaymentSettings), graphql_name="paymentSettings") - plan = sgqlc.types.Field(sgqlc.types.non_null(ShopPlan), graphql_name="plan") - primary_domain = sgqlc.types.Field(sgqlc.types.non_null(Domain), graphql_name="primaryDomain") - product_images = sgqlc.types.Field( - sgqlc.types.non_null(ImageConnection), - graphql_name="productImages", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("sort_key", sgqlc.types.Arg(ProductImageSortKeys, graphql_name="sortKey", default="CREATED_AT")), - ) - ), - ) - product_tags = sgqlc.types.Field( - sgqlc.types.non_null(StringConnection), - graphql_name="productTags", - args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="first", default=None)),)), - ) - product_types = sgqlc.types.Field( - sgqlc.types.non_null(StringConnection), - graphql_name="productTypes", - args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="first", default=None)),)), - ) - product_vendors = sgqlc.types.Field( - sgqlc.types.non_null(StringConnection), - graphql_name="productVendors", - args=sgqlc.types.ArgDict((("first", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="first", default=None)),)), - ) - publication_count = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="publicationCount") - resource_limits = sgqlc.types.Field(sgqlc.types.non_null(ShopResourceLimits), graphql_name="resourceLimits") - rich_text_editor_url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="richTextEditorUrl") - search = sgqlc.types.Field( - sgqlc.types.non_null(SearchResultConnection), - graphql_name="search", - args=sgqlc.types.ArgDict( - ( - ("query", sgqlc.types.Arg(sgqlc.types.non_null(String), graphql_name="query", default=None)), - ( - "types", - sgqlc.types.Arg(sgqlc.types.list_of(sgqlc.types.non_null(SearchResultType)), graphql_name="types", default=None), - ), - ("first", sgqlc.types.Arg(sgqlc.types.non_null(Int), graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ) - ), - ) - search_filters = sgqlc.types.Field(sgqlc.types.non_null(SearchFilterOptions), graphql_name="searchFilters") - setup_required = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="setupRequired") - ships_to_countries = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(CountryCode))), graphql_name="shipsToCountries" - ) - shop_policies = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopPolicy"))), graphql_name="shopPolicies" - ) - staff_members = sgqlc.types.Field( - sgqlc.types.non_null(StaffMemberConnection), - graphql_name="staffMembers", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - storefront_access_tokens = sgqlc.types.Field( - sgqlc.types.non_null(StorefrontAccessTokenConnection), - graphql_name="storefrontAccessTokens", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - tax_shipping = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxShipping") - taxes_included = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="taxesIncluded") - timezone_abbreviation = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="timezoneAbbreviation") - timezone_offset = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="timezoneOffset") - timezone_offset_minutes = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="timezoneOffsetMinutes") - transactional_sms_disabled = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="transactionalSmsDisabled") - unit_system = sgqlc.types.Field(sgqlc.types.non_null(UnitSystem), graphql_name="unitSystem") - uploaded_images_by_ids = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Image))), - graphql_name="uploadedImagesByIds", - args=sgqlc.types.ArgDict( - ( - ( - "image_ids", - sgqlc.types.Arg( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ID))), - graphql_name="imageIds", - default=None, - ), - ), - ) - ), - ) - url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") - weight_unit = sgqlc.types.Field(sgqlc.types.non_null(WeightUnit), graphql_name="weightUnit") - - -class ShopPolicy(sgqlc.types.Type, HasPublishedTranslations, Node): - __schema__ = shopify_schema - __field_names__ = ("body", "type", "url") - body = sgqlc.types.Field(sgqlc.types.non_null(HTML), graphql_name="body") - type = sgqlc.types.Field(sgqlc.types.non_null(ShopPolicyType), graphql_name="type") - url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") - - -class ShopPolicyUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(ShopPolicyErrorCode, graphql_name="code") - - -class ShopifyPaymentsAccount(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "activated", - "balance", - "bank_accounts", - "charge_statement_descriptors", - "country", - "default_currency", - "disputes", - "fraud_settings", - "notification_settings", - "onboardable", - "payout_schedule", - "payout_statement_descriptor", - "payouts", - "permitted_verification_documents", - "verifications", - ) - activated = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="activated") - balance = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MoneyV2))), graphql_name="balance") - bank_accounts = sgqlc.types.Field( - sgqlc.types.non_null(ShopifyPaymentsBankAccountConnection), - graphql_name="bankAccounts", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - charge_statement_descriptors = sgqlc.types.Field(ShopifyPaymentsChargeStatementDescriptor, graphql_name="chargeStatementDescriptors") - country = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="country") - default_currency = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="defaultCurrency") - disputes = sgqlc.types.Field( - sgqlc.types.non_null(ShopifyPaymentsDisputeConnection), - graphql_name="disputes", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ("query", sgqlc.types.Arg(String, graphql_name="query", default=None)), - ) - ), - ) - fraud_settings = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsFraudSettings), graphql_name="fraudSettings") - notification_settings = sgqlc.types.Field( - sgqlc.types.non_null(ShopifyPaymentsNotificationSettings), graphql_name="notificationSettings" - ) - onboardable = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="onboardable") - payout_schedule = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsPayoutSchedule), graphql_name="payoutSchedule") - payout_statement_descriptor = sgqlc.types.Field(String, graphql_name="payoutStatementDescriptor") - payouts = sgqlc.types.Field( - sgqlc.types.non_null(ShopifyPaymentsPayoutConnection), - graphql_name="payouts", - args=sgqlc.types.ArgDict( - ( - ( - "transaction_type", - sgqlc.types.Arg(ShopifyPaymentsPayoutTransactionType, graphql_name="transactionType", default=None), - ), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - permitted_verification_documents = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(ShopifyPaymentsVerificationDocument))), - graphql_name="permittedVerificationDocuments", - ) - verifications = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopifyPaymentsVerification"))), - graphql_name="verifications", - ) - - -class ShopifyPaymentsBankAccount(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "account_number", - "account_number_last_digits", - "bank_name", - "country", - "created_at", - "currency", - "payouts", - "routing_number", - "status", - ) - account_number = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="accountNumber") - account_number_last_digits = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="accountNumberLastDigits") - bank_name = sgqlc.types.Field(String, graphql_name="bankName") - country = sgqlc.types.Field(sgqlc.types.non_null(CountryCode), graphql_name="country") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - currency = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currency") - payouts = sgqlc.types.Field( - sgqlc.types.non_null(ShopifyPaymentsPayoutConnection), - graphql_name="payouts", - args=sgqlc.types.ArgDict( - ( - ( - "transaction_type", - sgqlc.types.Arg(ShopifyPaymentsPayoutTransactionType, graphql_name="transactionType", default=None), - ), - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - routing_number = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="routingNumber") - status = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsBankAccountStatus), graphql_name="status") - - -class ShopifyPaymentsDefaultChargeStatementDescriptor(sgqlc.types.Type, ShopifyPaymentsChargeStatementDescriptor): - __schema__ = shopify_schema - __field_names__ = () - - -class ShopifyPaymentsDispute(sgqlc.types.Type, LegacyInteroperability, Node): - __schema__ = shopify_schema - __field_names__ = ( - "amount", - "evidence_due_by", - "evidence_sent_on", - "finalized_on", - "initiated_at", - "order", - "reason_details", - "status", - "type", - ) - amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="amount") - evidence_due_by = sgqlc.types.Field(Date, graphql_name="evidenceDueBy") - evidence_sent_on = sgqlc.types.Field(Date, graphql_name="evidenceSentOn") - finalized_on = sgqlc.types.Field(Date, graphql_name="finalizedOn") - initiated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="initiatedAt") - order = sgqlc.types.Field(Order, graphql_name="order") - reason_details = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsDisputeReasonDetails), graphql_name="reasonDetails") - status = sgqlc.types.Field(sgqlc.types.non_null(DisputeStatus), graphql_name="status") - type = sgqlc.types.Field(sgqlc.types.non_null(DisputeType), graphql_name="type") - - -class ShopifyPaymentsDisputeEvidence(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "access_activity_log", - "billing_address", - "cancellation_policy_disclosure", - "cancellation_policy_file", - "cancellation_rebuttal", - "customer_communication_file", - "customer_email_address", - "customer_first_name", - "customer_last_name", - "customer_purchase_ip", - "dispute", - "dispute_file_uploads", - "fulfillments", - "product_description", - "refund_policy_disclosure", - "refund_policy_file", - "refund_refusal_explanation", - "service_documentation_file", - "shipping_address", - "shipping_documentation_file", - "submitted", - "uncategorized_file", - "uncategorized_text", - ) - access_activity_log = sgqlc.types.Field(String, graphql_name="accessActivityLog") - billing_address = sgqlc.types.Field(MailingAddress, graphql_name="billingAddress") - cancellation_policy_disclosure = sgqlc.types.Field(String, graphql_name="cancellationPolicyDisclosure") - cancellation_policy_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUpload", graphql_name="cancellationPolicyFile") - cancellation_rebuttal = sgqlc.types.Field(String, graphql_name="cancellationRebuttal") - customer_communication_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUpload", graphql_name="customerCommunicationFile") - customer_email_address = sgqlc.types.Field(String, graphql_name="customerEmailAddress") - customer_first_name = sgqlc.types.Field(String, graphql_name="customerFirstName") - customer_last_name = sgqlc.types.Field(String, graphql_name="customerLastName") - customer_purchase_ip = sgqlc.types.Field(String, graphql_name="customerPurchaseIp") - dispute = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsDispute), graphql_name="dispute") - dispute_file_uploads = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopifyPaymentsDisputeFileUpload"))), - graphql_name="disputeFileUploads", - ) - fulfillments = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null("ShopifyPaymentsDisputeFulfillment"))), - graphql_name="fulfillments", - ) - product_description = sgqlc.types.Field(String, graphql_name="productDescription") - refund_policy_disclosure = sgqlc.types.Field(String, graphql_name="refundPolicyDisclosure") - refund_policy_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUpload", graphql_name="refundPolicyFile") - refund_refusal_explanation = sgqlc.types.Field(String, graphql_name="refundRefusalExplanation") - service_documentation_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUpload", graphql_name="serviceDocumentationFile") - shipping_address = sgqlc.types.Field(MailingAddress, graphql_name="shippingAddress") - shipping_documentation_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUpload", graphql_name="shippingDocumentationFile") - submitted = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="submitted") - uncategorized_file = sgqlc.types.Field("ShopifyPaymentsDisputeFileUpload", graphql_name="uncategorizedFile") - uncategorized_text = sgqlc.types.Field(String, graphql_name="uncategorizedText") - - -class ShopifyPaymentsDisputeFileUpload(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("dispute_evidence_type", "file_size", "file_type", "original_file_name", "url") - dispute_evidence_type = sgqlc.types.Field(ShopifyPaymentsDisputeEvidenceFileType, graphql_name="disputeEvidenceType") - file_size = sgqlc.types.Field(sgqlc.types.non_null(Int), graphql_name="fileSize") - file_type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="fileType") - original_file_name = sgqlc.types.Field(String, graphql_name="originalFileName") - url = sgqlc.types.Field(sgqlc.types.non_null(URL), graphql_name="url") - - -class ShopifyPaymentsDisputeFulfillment(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("shipping_carrier", "shipping_date", "shipping_tracking_number") - shipping_carrier = sgqlc.types.Field(String, graphql_name="shippingCarrier") - shipping_date = sgqlc.types.Field(Date, graphql_name="shippingDate") - shipping_tracking_number = sgqlc.types.Field(String, graphql_name="shippingTrackingNumber") - - -class ShopifyPaymentsJpChargeStatementDescriptor(sgqlc.types.Type, ShopifyPaymentsChargeStatementDescriptor): - __schema__ = shopify_schema - __field_names__ = ("kana", "kanji") - kana = sgqlc.types.Field(String, graphql_name="kana") - kanji = sgqlc.types.Field(String, graphql_name="kanji") - - -class ShopifyPaymentsPayout(sgqlc.types.Type, LegacyInteroperability, Node): - __schema__ = shopify_schema - __field_names__ = ("bank_account", "issued_at", "net", "status", "summary", "transaction_type") - bank_account = sgqlc.types.Field(ShopifyPaymentsBankAccount, graphql_name="bankAccount") - issued_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="issuedAt") - net = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="net") - status = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsPayoutStatus), graphql_name="status") - summary = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsPayoutSummary), graphql_name="summary") - transaction_type = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsPayoutTransactionType), graphql_name="transactionType") - - -class ShopifyPaymentsVerification(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("status", "subject") - status = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsVerificationStatus), graphql_name="status") - subject = sgqlc.types.Field(sgqlc.types.non_null(ShopifyPaymentsVerificationSubject), graphql_name="subject") - - -class StaffMember(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "active", - "avatar", - "email", - "exists", - "first_name", - "initials", - "is_shop_owner", - "last_name", - "locale", - "name", - "phone", - "private_data", - ) - active = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="active") - avatar = sgqlc.types.Field( - sgqlc.types.non_null(Image), - graphql_name="avatar", - args=sgqlc.types.ArgDict((("fallback", sgqlc.types.Arg(StaffMemberDefaultImage, graphql_name="fallback", default="DEFAULT")),)), - ) - email = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="email") - exists = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="exists") - first_name = sgqlc.types.Field(String, graphql_name="firstName") - initials = sgqlc.types.Field(sgqlc.types.list_of(sgqlc.types.non_null(String)), graphql_name="initials") - is_shop_owner = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="isShopOwner") - last_name = sgqlc.types.Field(String, graphql_name="lastName") - locale = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="locale") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - phone = sgqlc.types.Field(String, graphql_name="phone") - private_data = sgqlc.types.Field(sgqlc.types.non_null(StaffMemberPrivateData), graphql_name="privateData") - - -class StandardMetafieldDefinitionEnableUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(StandardMetafieldDefinitionEnableUserErrorCode, graphql_name="code") - - -class StandardMetafieldDefinitionTemplate(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "description", - "key", - "name", - "namespace", - "owner_types", - "type", - "validations", - "visible_to_storefront_api", - ) - description = sgqlc.types.Field(String, graphql_name="description") - key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="key") - name = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="name") - namespace = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="namespace") - owner_types = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MetafieldOwnerType))), graphql_name="ownerTypes" - ) - type = sgqlc.types.Field(sgqlc.types.non_null(MetafieldDefinitionType), graphql_name="type") - validations = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(MetafieldDefinitionValidation))), - graphql_name="validations", - ) - visible_to_storefront_api = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="visibleToStorefrontApi") - - -class StorefrontAccessToken(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("access_scopes", "access_token", "created_at", "title", "updated_at") - access_scopes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(AccessScope))), graphql_name="accessScopes" - ) - access_token = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="accessToken") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - title = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="title") - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - - -class SubscriptionBillingAttempt(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "completed_at", - "created_at", - "error_code", - "error_message", - "idempotency_key", - "next_action_url", - "order", - "origin_time", - "ready", - "subscription_contract", - ) - completed_at = sgqlc.types.Field(DateTime, graphql_name="completedAt") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - error_code = sgqlc.types.Field(SubscriptionBillingAttemptErrorCode, graphql_name="errorCode") - error_message = sgqlc.types.Field(String, graphql_name="errorMessage") - idempotency_key = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="idempotencyKey") - next_action_url = sgqlc.types.Field(URL, graphql_name="nextActionUrl") - order = sgqlc.types.Field(Order, graphql_name="order") - origin_time = sgqlc.types.Field(DateTime, graphql_name="originTime") - ready = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="ready") - subscription_contract = sgqlc.types.Field(sgqlc.types.non_null("SubscriptionContract"), graphql_name="subscriptionContract") - - -class SubscriptionBillingCycleEditedContract(sgqlc.types.Type, SubscriptionContractBase): - __schema__ = shopify_schema - __field_names__ = ("billing_cycles", "created_at") - billing_cycles = sgqlc.types.Field( - sgqlc.types.non_null(SubscriptionBillingCycleConnection), - graphql_name="billingCycles", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ( - "sort_key", - sgqlc.types.Arg(SubscriptionBillingCyclesSortKeys, graphql_name="sortKey", default="CYCLE_INDEX"), - ), - ) - ), - ) - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - - -class SubscriptionBillingCycleUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(SubscriptionBillingCycleErrorCode, graphql_name="code") - - -class SubscriptionContract(sgqlc.types.Type, Node, SubscriptionContractBase): - __schema__ = shopify_schema - __field_names__ = ( - "billing_attempts", - "billing_policy", - "created_at", - "delivery_policy", - "last_payment_status", - "next_billing_date", - "origin_order", - "status", - ) - billing_attempts = sgqlc.types.Field( - sgqlc.types.non_null(SubscriptionBillingAttemptConnection), - graphql_name="billingAttempts", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - billing_policy = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionBillingPolicy), graphql_name="billingPolicy") - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - delivery_policy = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionDeliveryPolicy), graphql_name="deliveryPolicy") - last_payment_status = sgqlc.types.Field(SubscriptionContractLastPaymentStatus, graphql_name="lastPaymentStatus") - next_billing_date = sgqlc.types.Field(DateTime, graphql_name="nextBillingDate") - origin_order = sgqlc.types.Field(Order, graphql_name="originOrder") - status = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionContractSubscriptionStatus), graphql_name="status") - - -class SubscriptionContractUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(SubscriptionContractErrorCode, graphql_name="code") - - -class SubscriptionDraft(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "billing_cycle", - "billing_policy", - "concatenated_billing_cycles", - "currency_code", - "custom_attributes", - "customer", - "customer_payment_method", - "delivery_method", - "delivery_options", - "delivery_policy", - "delivery_price", - "discounts", - "discounts_added", - "discounts_removed", - "discounts_updated", - "lines", - "lines_added", - "lines_removed", - "next_billing_date", - "note", - "original_contract", - "status", - ) - billing_cycle = sgqlc.types.Field(SubscriptionBillingCycle, graphql_name="billingCycle") - billing_policy = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionBillingPolicy), graphql_name="billingPolicy") - concatenated_billing_cycles = sgqlc.types.Field( - sgqlc.types.non_null(SubscriptionBillingCycleConnection), - graphql_name="concatenatedBillingCycles", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ( - "sort_key", - sgqlc.types.Arg(SubscriptionBillingCyclesSortKeys, graphql_name="sortKey", default="CYCLE_INDEX"), - ), - ) - ), - ) - currency_code = sgqlc.types.Field(sgqlc.types.non_null(CurrencyCode), graphql_name="currencyCode") - custom_attributes = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(Attribute))), graphql_name="customAttributes" - ) - customer = sgqlc.types.Field(sgqlc.types.non_null(Customer), graphql_name="customer") - customer_payment_method = sgqlc.types.Field( - CustomerPaymentMethod, - graphql_name="customerPaymentMethod", - args=sgqlc.types.ArgDict((("show_revoked", sgqlc.types.Arg(Boolean, graphql_name="showRevoked", default=False)),)), - ) - delivery_method = sgqlc.types.Field("SubscriptionDeliveryMethod", graphql_name="deliveryMethod") - delivery_options = sgqlc.types.Field( - "SubscriptionDeliveryOptionResult", - graphql_name="deliveryOptions", - args=sgqlc.types.ArgDict( - (("delivery_address", sgqlc.types.Arg(MailingAddressInput, graphql_name="deliveryAddress", default=None)),) - ), - ) - delivery_policy = sgqlc.types.Field(sgqlc.types.non_null(SubscriptionDeliveryPolicy), graphql_name="deliveryPolicy") - delivery_price = sgqlc.types.Field(MoneyV2, graphql_name="deliveryPrice") - discounts = sgqlc.types.Field( - sgqlc.types.non_null(SubscriptionDiscountConnection), - graphql_name="discounts", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - discounts_added = sgqlc.types.Field( - sgqlc.types.non_null(SubscriptionDiscountConnection), - graphql_name="discountsAdded", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - discounts_removed = sgqlc.types.Field( - sgqlc.types.non_null(SubscriptionDiscountConnection), - graphql_name="discountsRemoved", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - discounts_updated = sgqlc.types.Field( - sgqlc.types.non_null(SubscriptionDiscountConnection), - graphql_name="discountsUpdated", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - lines = sgqlc.types.Field( - sgqlc.types.non_null(SubscriptionLineConnection), - graphql_name="lines", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - lines_added = sgqlc.types.Field( - sgqlc.types.non_null(SubscriptionLineConnection), - graphql_name="linesAdded", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - lines_removed = sgqlc.types.Field( - sgqlc.types.non_null(SubscriptionLineConnection), - graphql_name="linesRemoved", - args=sgqlc.types.ArgDict( - ( - ("first", sgqlc.types.Arg(Int, graphql_name="first", default=None)), - ("after", sgqlc.types.Arg(String, graphql_name="after", default=None)), - ("last", sgqlc.types.Arg(Int, graphql_name="last", default=None)), - ("before", sgqlc.types.Arg(String, graphql_name="before", default=None)), - ("reverse", sgqlc.types.Arg(Boolean, graphql_name="reverse", default=False)), - ) - ), - ) - next_billing_date = sgqlc.types.Field(DateTime, graphql_name="nextBillingDate") - note = sgqlc.types.Field(String, graphql_name="note") - original_contract = sgqlc.types.Field(SubscriptionContract, graphql_name="originalContract") - status = sgqlc.types.Field(SubscriptionContractSubscriptionStatus, graphql_name="status") - - -class SubscriptionDraftUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(SubscriptionDraftErrorCode, graphql_name="code") - - -class TenderTransaction(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "amount", - "payment_method", - "processed_at", - "remote_reference", - "test", - "transaction_details", - "user", - ) - amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="amount") - payment_method = sgqlc.types.Field(String, graphql_name="paymentMethod") - processed_at = sgqlc.types.Field(DateTime, graphql_name="processedAt") - remote_reference = sgqlc.types.Field(String, graphql_name="remoteReference") - test = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="test") - transaction_details = sgqlc.types.Field("TenderTransactionDetails", graphql_name="transactionDetails") - user = sgqlc.types.Field(StaffMember, graphql_name="user") - - -class TipSale(sgqlc.types.Type, Sale): - __schema__ = shopify_schema - __field_names__ = ("line_item",) - line_item = sgqlc.types.Field(sgqlc.types.non_null(LineItem), graphql_name="lineItem") - - -class TransactionFee(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("amount", "flat_fee", "flat_fee_name", "rate", "rate_name", "tax_amount", "type") - amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="amount") - flat_fee = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="flatFee") - flat_fee_name = sgqlc.types.Field(String, graphql_name="flatFeeName") - rate = sgqlc.types.Field(sgqlc.types.non_null(Decimal), graphql_name="rate") - rate_name = sgqlc.types.Field(String, graphql_name="rateName") - tax_amount = sgqlc.types.Field(sgqlc.types.non_null(MoneyV2), graphql_name="taxAmount") - type = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="type") - - -class TranslationUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(TranslationErrorCode, graphql_name="code") - - -class UrlRedirect(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("path", "target") - path = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="path") - target = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="target") - - -class UrlRedirectBulkDeleteByIdsUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(UrlRedirectBulkDeleteByIdsUserErrorCode, graphql_name="code") - - -class UrlRedirectBulkDeleteBySavedSearchUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(UrlRedirectBulkDeleteBySavedSearchUserErrorCode, graphql_name="code") - - -class UrlRedirectBulkDeleteBySearchUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(UrlRedirectBulkDeleteBySearchUserErrorCode, graphql_name="code") - - -class UrlRedirectImport(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ( - "count", - "created_count", - "failed_count", - "finished", - "finished_at", - "preview_redirects", - "updated_count", - ) - count = sgqlc.types.Field(Int, graphql_name="count") - created_count = sgqlc.types.Field(Int, graphql_name="createdCount") - failed_count = sgqlc.types.Field(Int, graphql_name="failedCount") - finished = sgqlc.types.Field(sgqlc.types.non_null(Boolean), graphql_name="finished") - finished_at = sgqlc.types.Field(DateTime, graphql_name="finishedAt") - preview_redirects = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(UrlRedirectImportPreview))), - graphql_name="previewRedirects", - ) - updated_count = sgqlc.types.Field(Int, graphql_name="updatedCount") - - -class UrlRedirectImportUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(UrlRedirectImportErrorCode, graphql_name="code") - - -class UrlRedirectUserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = ("code",) - code = sgqlc.types.Field(UrlRedirectErrorCode, graphql_name="code") - - -class UserError(sgqlc.types.Type, DisplayableError): - __schema__ = shopify_schema - __field_names__ = () - - -class Video(sgqlc.types.Type, File, Media, Node): - __schema__ = shopify_schema - __field_names__ = ("duration", "filename", "original_source", "sources") - duration = sgqlc.types.Field(Int, graphql_name="duration") - filename = sgqlc.types.Field(sgqlc.types.non_null(String), graphql_name="filename") - original_source = sgqlc.types.Field(VideoSource, graphql_name="originalSource") - sources = sgqlc.types.Field(sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(VideoSource))), graphql_name="sources") - - -class WebPixel(sgqlc.types.Type, Node): - __schema__ = shopify_schema - __field_names__ = ("settings",) - settings = sgqlc.types.Field(sgqlc.types.non_null(JSON), graphql_name="settings") - - -class WebhookSubscription(sgqlc.types.Type, LegacyInteroperability, Node): - __schema__ = shopify_schema - __field_names__ = ( - "created_at", - "endpoint", - "format", - "include_fields", - "metafield_namespaces", - "private_metafield_namespaces", - "topic", - "updated_at", - ) - created_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="createdAt") - endpoint = sgqlc.types.Field(sgqlc.types.non_null("WebhookSubscriptionEndpoint"), graphql_name="endpoint") - format = sgqlc.types.Field(sgqlc.types.non_null(WebhookSubscriptionFormat), graphql_name="format") - include_fields = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="includeFields" - ) - metafield_namespaces = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), graphql_name="metafieldNamespaces" - ) - private_metafield_namespaces = sgqlc.types.Field( - sgqlc.types.non_null(sgqlc.types.list_of(sgqlc.types.non_null(String))), - graphql_name="privateMetafieldNamespaces", - ) - topic = sgqlc.types.Field(sgqlc.types.non_null(WebhookSubscriptionTopic), graphql_name="topic") - updated_at = sgqlc.types.Field(sgqlc.types.non_null(DateTime), graphql_name="updatedAt") - - -######################################################################## -# Unions -######################################################################## -class AppPricingDetails(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (AppRecurringPricing, AppUsagePricing) - - -class AppSubscriptionDiscountValue(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (AppSubscriptionDiscountAmount, AppSubscriptionDiscountPercentage) - - -class CollectionRuleConditionObject(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (CollectionRuleProductCategoryCondition, CollectionRuleTextCondition) - - -class CommentEventEmbed(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (Customer, DraftOrder, Order, Product, ProductVariant) - - -class CustomerPaymentInstrument(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (CustomerCreditCard, CustomerPaypalBillingAgreement, CustomerShopPayAgreement) - - -class DeliveryConditionCriteria(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (MoneyV2, Weight) - - -class DeliveryRateProvider(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (DeliveryParticipant, DeliveryRateDefinition) - - -class Discount(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = ( - DiscountAutomaticApp, - DiscountAutomaticBasic, - DiscountAutomaticBxgy, - DiscountCodeApp, - DiscountCodeBasic, - DiscountCodeBxgy, - DiscountCodeFreeShipping, - ) - - -class DiscountAutomatic(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (DiscountAutomaticApp, DiscountAutomaticBasic, DiscountAutomaticBxgy) - - -class DiscountCode(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (DiscountCodeApp, DiscountCodeBasic, DiscountCodeBxgy, DiscountCodeFreeShipping) - - -class DiscountCustomerBuysValue(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (DiscountPurchaseAmount, DiscountQuantity) - - -class DiscountCustomerGetsValue(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (DiscountAmount, DiscountOnQuantity, DiscountPercentage) - - -class DiscountCustomerSelection(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (DiscountCustomerAll, DiscountCustomerSegments, DiscountCustomers) - - -class DiscountEffect(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (DiscountPercentage,) - - -class DiscountItems(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (AllDiscountItems, DiscountCollections, DiscountProducts) - - -class DiscountMinimumRequirement(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (DiscountMinimumQuantity, DiscountMinimumSubtotal) - - -class DiscountShippingDestinationSelection(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (DiscountCountries, DiscountCountryAll) - - -class MetafieldReference(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (Collection, GenericFile, MediaImage, OnlineStorePage, Product, ProductVariant, Video) - - -class OrderStagedChange(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = ( - OrderStagedChangeAddCustomItem, - OrderStagedChangeAddLineItemDiscount, - OrderStagedChangeAddShippingLine, - OrderStagedChangeAddVariant, - OrderStagedChangeDecrementItem, - OrderStagedChangeIncrementItem, - ) - - -class PaymentInstrument(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (VaultCreditCard, VaultPaypalBillingAgreement) - - -class PriceRuleValue(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (PriceRuleFixedAmountValue, PriceRulePercentValue) - - -class PricingValue(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (MoneyV2, PricingPercentageValue) - - -class PurchasingEntity(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (Customer, PurchasingCompany) - - -class SellingPlanBillingPolicy(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (SellingPlanFixedBillingPolicy, SellingPlanRecurringBillingPolicy) - - -class SellingPlanCheckoutChargeValue(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (MoneyV2, SellingPlanCheckoutChargePercentageValue) - - -class SellingPlanDeliveryPolicy(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (SellingPlanFixedDeliveryPolicy, SellingPlanRecurringDeliveryPolicy) - - -class SellingPlanPricingPolicy(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (SellingPlanFixedPricingPolicy, SellingPlanRecurringPricingPolicy) - - -class SellingPlanPricingPolicyAdjustmentValue(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (MoneyV2, SellingPlanPricingPolicyPercentageValue) - - -class SubscriptionDeliveryMethod(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = ( - SubscriptionDeliveryMethodLocalDelivery, - SubscriptionDeliveryMethodPickup, - SubscriptionDeliveryMethodShipping, - ) - - -class SubscriptionDeliveryOption(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (SubscriptionLocalDeliveryOption, SubscriptionPickupOption, SubscriptionShippingOption) - - -class SubscriptionDeliveryOptionResult(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (SubscriptionDeliveryOptionResultFailure, SubscriptionDeliveryOptionResultSuccess) - - -class SubscriptionDiscount(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (SubscriptionAppliedCodeDiscount, SubscriptionManualDiscount) - - -class SubscriptionDiscountValue(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (SubscriptionDiscountFixedAmountValue, SubscriptionDiscountPercentageValue) - - -class SubscriptionShippingOptionResult(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (SubscriptionShippingOptionResultFailure, SubscriptionShippingOptionResultSuccess) - - -class TenderTransactionDetails(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (TenderTransactionCreditCardDetails,) - - -class WebhookSubscriptionEndpoint(sgqlc.types.Union): - __schema__ = shopify_schema - __types__ = (WebhookEventBridgeEndpoint, WebhookHttpEndpoint, WebhookPubSubEndpoint) - - -######################################################################## -# Schema Entry Points -######################################################################## -shopify_schema.query_type = QueryRoot -shopify_schema.mutation_type = Mutation -shopify_schema.subscription_type = None diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py index 88f9b2563b2e..62533bcef885 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py @@ -3,798 +3,69 @@ # -import logging -from abc import ABC, abstractmethod -from functools import cached_property -from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union -from urllib.parse import parse_qsl, urlparse +from typing import Any, List, Mapping, Tuple import requests from airbyte_cdk import AirbyteLogger +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream from requests.exceptions import ConnectionError, InvalidURL, JSONDecodeError, RequestException, SSLError -from .auth import ShopifyAuthenticator -from .graphql import get_query_products -from .transform import DataTypeEnforcer -from .utils import SCOPES_MAPPING, ApiTypeEnum -from .utils import EagerlyCachedStreamState as stream_state_cache -from .utils import ShopifyAccessScopesError, ShopifyBadJsonError, ShopifyConnectionError, ShopifyNonRetryableErrors -from .utils import ShopifyRateLimiter as limiter -from .utils import ShopifyWrongShopNameError - - -class ShopifyStream(HttpStream, ABC): - # Latest Stable Release - api_version = "2022-10" - # Page size - limit = 250 - # Define primary key as sort key for full_refresh, or very first sync for incremental_refresh - primary_key = "id" - order_field = "updated_at" - filter_field = "updated_at_min" - - # define default logger - logger = logging.getLogger("airbyte") - - raise_on_http_errors = True - max_retries = 5 - - def __init__(self, config: Dict): - super().__init__(authenticator=config["authenticator"]) - self._transformer = DataTypeEnforcer(self.get_json_schema()) - self.config = config - - @property - def url_base(self) -> str: - return f"https://{self.config['shop']}.myshopify.com/admin/api/{self.api_version}/" - - @property - def default_filter_field_value(self) -> Union[int, str]: - # certain streams are using `since_id` field as `filter_field`, which requires to use `int` type, - # but many other use `str` values for this, we determine what to use based on `filter_field` value - # by default, we use the user defined `Start Date` as initial value, or 0 for `id`-dependent streams. - return 0 if self.filter_field == "since_id" else self.config["start_date"] - - @staticmethod - def next_page_token(response: requests.Response) -> Optional[Mapping[str, Any]]: - next_page = response.links.get("next", None) - if next_page: - return dict(parse_qsl(urlparse(next_page.get("url")).query)) - else: - return None - - def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: - params = {"limit": self.limit} - if next_page_token: - params.update(**next_page_token) - else: - params["order"] = f"{self.order_field} asc" - params[self.filter_field] = self.default_filter_field_value - return params - - @limiter.balance_rate_limit() - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - if response.status_code is requests.codes.OK: - try: - json_response = response.json() - records = json_response.get(self.data_field, []) if self.data_field is not None else json_response - yield from self.produce_records(records) - except RequestException as e: - self.logger.warning(f"Unexpected error in `parse_ersponse`: {e}, the actual response data: {response.text}") - yield {} - - def produce_records(self, records: Union[Iterable[Mapping[str, Any]], Mapping[str, Any]] = None) -> Iterable[Mapping[str, Any]]: - # transform method was implemented according to issue 4841 - # Shopify API returns price fields as a string and it should be converted to number - # this solution designed to convert string into number, but in future can be modified for general purpose - if isinstance(records, dict): - # for cases when we have a single record as dict - # add shop_url to the record to make querying easy - records["shop_url"] = self.config["shop"] - yield self._transformer.transform(records) - else: - # for other cases - for record in records: - # add shop_url to the record to make querying easy - record["shop_url"] = self.config["shop"] - yield self._transformer.transform(record) - - def should_retry(self, response: requests.Response) -> bool: - known_errors = ShopifyNonRetryableErrors(self.name) - status = response.status_code - if status in known_errors.keys(): - setattr(self, "raise_on_http_errors", False) - self.logger.warning(known_errors.get(status)) - return False - else: - return super().should_retry(response) - - @property - @abstractmethod - def data_field(self) -> str: - """The name of the field in the response which contains the data""" - - -class IncrementalShopifyStream(ShopifyStream, ABC): - - # Setting the check point interval to the limit of the records output - @property - def state_checkpoint_interval(self) -> int: - return super().limit - - # Setting the default cursor field for all streams - cursor_field = "updated_at" - - @property - def default_state_comparison_value(self) -> Union[int, str]: - # certain streams are using `id` field as `cursor_field`, which requires to use `int` type, - # but many other use `str` values for this, we determine what to use based on `cursor_field` value - return 0 if self.cursor_field == "id" else "" - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - last_record_value = latest_record.get(self.cursor_field) or self.default_state_comparison_value - current_state_value = current_stream_state.get(self.cursor_field) or self.default_state_comparison_value - return {self.cursor_field: max(last_record_value, current_state_value)} - - @stream_state_cache.cache_stream_state - def request_params(self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs): - params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) - # If there is a next page token then we should only send pagination-related parameters. - if not next_page_token: - params["order"] = f"{self.order_field} asc" - if stream_state: - params[self.filter_field] = stream_state.get(self.cursor_field) - return params - - # Parse the `stream_slice` with respect to `stream_state` for `Incremental refresh` - # cases where we slice the stream, the endpoints for those classes don't accept any other filtering, - # but they provide us with the updated_at field in most cases, so we used that as incremental filtering during the order slicing. - def filter_records_newer_than_state(self, stream_state: Mapping[str, Any] = None, records_slice: Iterable[Mapping] = None) -> Iterable: - # Getting records >= state - if stream_state: - state_value = stream_state.get(self.cursor_field) - for record in records_slice: - if self.cursor_field in record: - record_value = record.get(self.cursor_field, self.default_state_comparison_value) - if record_value: - if record_value >= state_value: - yield record - else: - # old entities could have cursor field in place, but set to null - self.logger.warning( - f"Stream `{self.name}`, Record ID: `{record.get(self.primary_key)}` cursor value is: {record_value}, record is emitted without state comparison" - ) - yield record - else: - # old entities could miss the cursor field - self.logger.warning( - f"Stream `{self.name}`, Record ID: `{record.get(self.primary_key)}` missing cursor field: {self.cursor_field}, record is emitted without state comparison" - ) - yield record - else: - yield from records_slice - - -class ShopifySubstream(IncrementalShopifyStream): - """ - ShopifySubstream - provides slicing functionality for streams using parts of data from parent stream. - For example: - - `Refunds Orders` is the entity of `Orders`, - - `OrdersRisks` is the entity of `Orders`, - - `DiscountCodes` is the entity of `PriceRules`, etc. - - :: @ parent_stream - defines the parent stream object to read from - :: @ slice_key - defines the name of the property in stream slices dict. - :: @ nested_record - the name of the field inside of parent stream record. Default is `id`. - :: @ nested_record_field_name - the name of the field inside of nested_record. - :: @ nested_substream - the name of the nested entity inside of parent stream, helps to reduce the number of - API Calls, if present, see `OrderRefunds` stream for more. - """ - - parent_stream_class: object = None - slice_key: str = None - nested_record: str = "id" - nested_record_field_name: str = None - nested_substream = None - nested_substream_list_field_id = None - - @cached_property - def parent_stream(self) -> object: - """ - Returns the instance of parent stream, if the substream has a `parent_stream_class` dependency. - """ - return self.parent_stream_class(self.config) if self.parent_stream_class else None - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """UPDATING THE STATE OBJECT: - Stream: Transactions - Parent Stream: Orders - Returns: - { - {...}, - "transactions": { - "created_at": "2022-03-03T03:47:45-08:00", - "orders": { - "updated_at": "2022-03-03T03:47:46-08:00" - } - }, - {...}, - } - """ - updated_state = super().get_updated_state(current_stream_state, latest_record) - # add parent_stream_state to `updated_state` - updated_state[self.parent_stream.name] = stream_state_cache.cached_state.get(self.parent_stream.name) - return updated_state - - def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: - params = {"limit": self.limit} - if next_page_token: - params.update(**next_page_token) - return params - - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: - """ - Reading the parent stream for slices with structure: - EXAMPLE: for given nested_record as `id` of Orders, - - Outputs: - [ - {slice_key: 123}, - {slice_key: 456}, - {...}, - {slice_key: 999 - ] - """ - sorted_substream_slices = [] - - # reading parent nested stream_state from child stream state - parent_stream_state = stream_state.get(self.parent_stream.name) if stream_state else {} - - # reading the parent stream - for record in self.parent_stream.read_records(stream_state=parent_stream_state, **kwargs): - # updating the `stream_state` with the state of it's parent stream - # to have the child stream sync independently from the parent stream - stream_state_cache.cached_state[self.parent_stream.name] = self.parent_stream.get_updated_state({}, record) - # to limit the number of API Calls and reduce the time of data fetch, - # we can pull the ready data for child_substream, if nested data is present, - # and corresponds to the data of child_substream we need. - if self.nested_substream and self.nested_substream_list_field_id: - if record.get(self.nested_substream): - sorted_substream_slices.extend( - [ - { - self.slice_key: sub_record[self.nested_substream_list_field_id], - self.cursor_field: record[self.nested_substream][0].get( - self.cursor_field, self.default_state_comparison_value - ), - } - for sub_record in record[self.nested_record] - ] - ) - elif self.nested_substream: - if record.get(self.nested_substream): - sorted_substream_slices.append( - { - self.slice_key: record[self.nested_record], - self.cursor_field: record[self.nested_substream][0].get(self.cursor_field, self.default_state_comparison_value), - } - ) - else: - yield {self.slice_key: record[self.nested_record]} - - # output slice from sorted list to avoid filtering older records - if self.nested_substream: - if len(sorted_substream_slices) > 0: - # sort by cursor_field - sorted_substream_slices.sort(key=lambda x: x.get(self.cursor_field)) - for sorted_slice in sorted_substream_slices: - yield {self.slice_key: sorted_slice[self.slice_key]} - - def read_records( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Optional[Mapping[str, Any]] = None, - **kwargs, - ) -> Iterable[Mapping[str, Any]]: - """Reading child streams records for each `id`""" - - slice_data = stream_slice.get(self.slice_key) - # sometimes the stream_slice.get(self.slice_key) has the list of records, - # to avoid data exposition inside the logs, we should get the data we need correctly out of stream_slice. - if isinstance(slice_data, list) and self.nested_record_field_name is not None and len(slice_data) > 0: - slice_data = slice_data[0].get(self.nested_record_field_name) - - self.logger.info(f"Reading {self.name} for {self.slice_key}: {slice_data}") - records = super().read_records(stream_slice=stream_slice, **kwargs) - yield from self.filter_records_newer_than_state(stream_state=stream_state, records_slice=records) - - -class MetafieldShopifySubstream(ShopifySubstream): - slice_key = "id" - data_field = "metafields" - - parent_stream_class: object = None - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - object_id = stream_slice[self.slice_key] - return f"{self.parent_stream_class.data_field}/{object_id}/{self.data_field}.json" - - -class Articles(IncrementalShopifyStream): - data_field = "articles" - cursor_field = "id" - order_field = "id" - filter_field = "since_id" - - def path(self, **kwargs) -> str: - return f"{self.data_field}.json" - - -class MetafieldArticles(MetafieldShopifySubstream): - parent_stream_class: object = Articles - - -class Blogs(IncrementalShopifyStream): - cursor_field = "id" - order_field = "id" - data_field = "blogs" - filter_field = "since_id" - - def path(self, **kwargs) -> str: - return f"{self.data_field}.json" - - -class MetafieldBlogs(MetafieldShopifySubstream): - parent_stream_class: object = Blogs - - -class Customers(IncrementalShopifyStream): - data_field = "customers" - - def path(self, **kwargs) -> str: - return f"{self.data_field}.json" - - -class MetafieldCustomers(MetafieldShopifySubstream): - parent_stream_class: object = Customers - - -class Orders(IncrementalShopifyStream): - data_field = "orders" - - def path(self, **kwargs) -> str: - return f"{self.data_field}.json" - - def request_params( - self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) - if not next_page_token: - params["status"] = "any" - return params - - -class Disputes(IncrementalShopifyStream): - data_field = "disputes" - filter_field = "since_id" - cursor_field = "id" - order_field = "id" - - def path(self, **kwargs) -> str: - return f"shopify_payments/{self.data_field}.json" - - -class MetafieldOrders(MetafieldShopifySubstream): - parent_stream_class: object = Orders - - -class DraftOrders(IncrementalShopifyStream): - data_field = "draft_orders" - - def path(self, **kwargs) -> str: - return f"{self.data_field}.json" - - -class MetafieldDraftOrders(MetafieldShopifySubstream): - parent_stream_class: object = DraftOrders - - -class Products(IncrementalShopifyStream): - use_cache = True - data_field = "products" - - def path(self, **kwargs) -> str: - return f"{self.data_field}.json" - - -class ProductsGraphQl(IncrementalShopifyStream): - filter_field = "updatedAt" - cursor_field = "updatedAt" - data_field = "graphql" - http_method = "POST" - - def path(self, **kwargs) -> str: - return f"{self.data_field}.json" - - def request_params( - self, - stream_state: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - **kwargs, - ) -> MutableMapping[str, Any]: - return {} - - def request_body_json( - self, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None, - ) -> Optional[Mapping]: - state_value = stream_state.get(self.filter_field) - if state_value: - filter_value = state_value - else: - filter_value = self.default_filter_field_value - query = get_query_products( - first=self.limit, filter_field=self.filter_field, filter_value=filter_value, next_page_token=next_page_token - ) - return {"query": query} - - @staticmethod - def next_page_token(response: requests.Response) -> Optional[Mapping[str, Any]]: - page_info = response.json()["data"]["products"]["pageInfo"] - has_next_page = page_info["hasNextPage"] - if has_next_page: - return page_info["endCursor"] - else: - return None - - @limiter.balance_rate_limit(api_type=ApiTypeEnum.graphql.value) - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - if response.status_code is requests.codes.OK: - try: - json_response = response.json()["data"]["products"]["nodes"] - yield from self.produce_records(json_response) - except RequestException as e: - self.logger.warning(f"Unexpected error in `parse_ersponse`: {e}, the actual response data: {response.text}") - yield {} - - -class MetafieldProducts(MetafieldShopifySubstream): - parent_stream_class: object = Products - - -class ProductImages(ShopifySubstream): - parent_stream_class: object = Products - cursor_field = "id" - slice_key = "product_id" - data_field = "images" - nested_substream = "images" - filter_field = "since_id" - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - product_id = stream_slice[self.slice_key] - return f"products/{product_id}/{self.data_field}.json" - - -class MetafieldProductImages(MetafieldShopifySubstream): - parent_stream_class: object = Products - nested_record = "images" - slice_key = "images" - nested_substream = "images" - nested_substream_list_field_id = "id" - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - image_id = stream_slice[self.slice_key] - return f"product_images/{image_id}/{self.data_field}.json" - - -class ProductVariants(ShopifySubstream): - parent_stream_class: object = Products - cursor_field = "id" - slice_key = "product_id" - data_field = "variants" - nested_substream = "variants" - filter_field = "since_id" - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - product_id = stream_slice[self.slice_key] - return f"products/{product_id}/{self.data_field}.json" - - -class MetafieldProductVariants(MetafieldShopifySubstream): - parent_stream_class: object = Products - nested_record = "variants" - slice_key = "variants" - nested_substream = "variants" - nested_substream_list_field_id = "id" - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - variant_id = stream_slice[self.slice_key] - return f"variants/{variant_id}/{self.data_field}.json" - - -class AbandonedCheckouts(IncrementalShopifyStream): - data_field = "checkouts" - - def path(self, **kwargs) -> str: - return f"{self.data_field}.json" - - def request_params( - self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) - # If there is a next page token then we should only send pagination-related parameters. - if not next_page_token: - params["status"] = "any" - return params - - -class CustomCollections(IncrementalShopifyStream): - data_field = "custom_collections" - - def path(self, **kwargs) -> str: - return f"{self.data_field}.json" - - -class SmartCollections(IncrementalShopifyStream): - data_field = "smart_collections" - - def path(self, **kwargs) -> str: - return f"{self.data_field}.json" - - -class MetafieldSmartCollections(MetafieldShopifySubstream): - parent_stream_class: object = SmartCollections - - -class Collects(IncrementalShopifyStream): - """ - Collects stream does not support Incremental Refresh based on datetime fields, only `since_id` is supported: - https://shopify.dev/docs/admin-api/rest/reference/products/collect - - The Collect stream is the link between Products and Collections, if the Collection is created for Products, - the `collect` record is created, it's reasonable to Full Refresh all collects. As for Incremental refresh - - we would use the since_id specificaly for this stream. - """ - - data_field = "collects" - cursor_field = "id" - order_field = "id" - filter_field = "since_id" - - def path(self, **kwargs) -> str: - return f"{self.data_field}.json" - - -class Collections(ShopifySubstream): - parent_stream_class: object = Collects - nested_record = "collection_id" - slice_key = "collection_id" - data_field = "collection" - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - collection_id = stream_slice[self.slice_key] - return f"collections/{collection_id}.json" - - -class MetafieldCollections(MetafieldShopifySubstream): - parent_stream_class: object = Collects - slice_key = "collection_id" - nested_record = "collection_id" - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - object_id = stream_slice[self.slice_key] - return f"collections/{object_id}/{self.data_field}.json" - - -class BalanceTransactions(IncrementalShopifyStream): - - """ - PaymentsTransactions stream does not support Incremental Refresh based on datetime fields, only `since_id` is supported: - https://shopify.dev/api/admin-rest/2021-07/resources/transactions - """ - - data_field = "transactions" - cursor_field = "id" - order_field = "id" - filter_field = "since_id" - - def path(self, **kwargs) -> str: - return f"shopify_payments/balance/{self.data_field}.json" - - -class OrderRefunds(ShopifySubstream): - parent_stream_class: object = Orders - slice_key = "order_id" - data_field = "refunds" - cursor_field = "created_at" - # we pull out the records that we already know has the refunds data from Orders object - nested_substream = "refunds" - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - order_id = stream_slice["order_id"] - return f"orders/{order_id}/{self.data_field}.json" - - -class OrderRisks(ShopifySubstream): - parent_stream_class: object = Orders - slice_key = "order_id" - data_field = "risks" - cursor_field = "id" - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - order_id = stream_slice["order_id"] - return f"orders/{order_id}/{self.data_field}.json" - - -class Transactions(ShopifySubstream): - parent_stream_class: object = Orders - slice_key = "order_id" - data_field = "transactions" - cursor_field = "created_at" - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - order_id = stream_slice["order_id"] - return f"orders/{order_id}/{self.data_field}.json" - - -class TenderTransactions(IncrementalShopifyStream): - data_field = "tender_transactions" - cursor_field = "processed_at" - filter_field = "processed_at_min" - - def path(self, **kwargs) -> str: - return f"{self.data_field}.json" - - -class Pages(IncrementalShopifyStream): - data_field = "pages" - - def path(self, **kwargs) -> str: - return f"{self.data_field}.json" - - -class MetafieldPages(MetafieldShopifySubstream): - parent_stream_class: object = Pages - - -class PriceRules(IncrementalShopifyStream): - data_field = "price_rules" - - def path(self, **kwargs) -> str: - return f"{self.data_field}.json" - - -class DiscountCodes(ShopifySubstream): - parent_stream_class: object = PriceRules - slice_key = "price_rule_id" - data_field = "discount_codes" - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - price_rule_id = stream_slice["price_rule_id"] - return f"price_rules/{price_rule_id}/{self.data_field}.json" - - -class Locations(ShopifyStream): - """ - The location API does not support any form of filtering. - https://shopify.dev/api/admin-rest/2021-07/resources/location - - Therefore, only FULL_REFRESH mode is supported. - """ - - data_field = "locations" - - def path(self, **kwargs): - return f"{self.data_field}.json" - - -class MetafieldLocations(MetafieldShopifySubstream): - parent_stream_class: object = Locations - - -class InventoryLevels(ShopifySubstream): - parent_stream_class: object = Locations - slice_key = "location_id" - data_field = "inventory_levels" - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - location_id = stream_slice["location_id"] - return f"locations/{location_id}/{self.data_field}.json" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - records_stream = super().parse_response(response, **kwargs) - - def generate_key(record): - record.update({"id": "|".join((str(record.get("location_id", "")), str(record.get("inventory_item_id", ""))))}) - return record - - # associate the surrogate key - yield from map(generate_key, records_stream) - - -class InventoryItems(ShopifySubstream): - parent_stream_class: object = Products - slice_key = "id" - nested_record = "variants" - nested_record_field_name = "inventory_item_id" - data_field = "inventory_items" - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - ids = ",".join(str(x[self.nested_record_field_name]) for x in stream_slice[self.slice_key]) - return f"inventory_items.json?ids={ids}" - - -class FulfillmentOrders(ShopifySubstream): - parent_stream_class: object = Orders - slice_key = "order_id" - data_field = "fulfillment_orders" - cursor_field = "id" - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - order_id = stream_slice[self.slice_key] - return f"orders/{order_id}/{self.data_field}.json" - - -class Fulfillments(ShopifySubstream): - parent_stream_class: object = Orders - slice_key = "order_id" - data_field = "fulfillments" - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - order_id = stream_slice[self.slice_key] - return f"orders/{order_id}/{self.data_field}.json" - - -class Shop(ShopifyStream): - data_field = "shop" - - def path(self, **kwargs) -> str: - return f"{self.data_field}.json" - - -class MetafieldShops(IncrementalShopifyStream): - data_field = "metafields" - - def path(self, **kwargs) -> str: - return f"{self.data_field}.json" - - -class CustomerSavedSearch(IncrementalShopifyStream): - api_version = "2022-01" - cursor_field = "id" - order_field = "id" - data_field = "customer_saved_searches" - filter_field = "since_id" - - def path(self, **kwargs) -> str: - return f"{self.data_field}.json" - - -class CustomerAddress(ShopifySubstream): - parent_stream_class: object = Customers - slice_key = "id" - data_field = "addresses" - cursor_field = "id" - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - customer_id = stream_slice[self.slice_key] - return f"customers/{customer_id}/{self.data_field}.json" - - -class Countries(ShopifyStream): - data_field = "countries" - - def path(self, **kwargs) -> str: - return f"{self.data_field}.json" +from .auth import MissingAccessTokenError, ShopifyAuthenticator +from .streams.streams import ( + AbandonedCheckouts, + Articles, + BalanceTransactions, + Blogs, + Collections, + Collects, + Countries, + CustomCollections, + CustomerAddress, + Customers, + CustomerSavedSearch, + DiscountCodes, + Disputes, + DraftOrders, + FulfillmentOrders, + Fulfillments, + InventoryItems, + InventoryLevels, + Locations, + MetafieldArticles, + MetafieldBlogs, + MetafieldCollections, + MetafieldCustomers, + MetafieldDraftOrders, + MetafieldLocations, + MetafieldOrders, + MetafieldPages, + MetafieldProductImages, + MetafieldProducts, + MetafieldProductVariants, + MetafieldShops, + MetafieldSmartCollections, + OrderRefunds, + OrderRisks, + Orders, + Pages, + PriceRules, + ProductImages, + Products, + ProductsGraphQl, + ProductVariants, + Shop, + SmartCollections, + TenderTransactions, + Transactions, + TransactionsGraphql, +) +from .utils import SCOPES_MAPPING, ShopifyAccessScopesError, ShopifyBadJsonError, ShopifyConnectionError, ShopifyWrongShopNameError class ConnectionCheckTest: - def __init__(self, config: Mapping[str, Any]): + def __init__(self, config: Mapping[str, Any]) -> None: self.config = config # use `Shop` as a test stream for connection check self.test_stream = Shop(self.config) @@ -808,6 +79,7 @@ def describe_error(self, pattern: str, shop_name: str = None, details: Any = Non "connection_error": f"Connection could not be established using `Shopify Store`: {shop_name}. Make sure it's valid and try again.", "request_exception": f"Request was not successfull, check your `input configuation` and try again. Details: {details}", "index_error": f"Failed to access the Shopify store `{shop_name}`. Verify the entered Shopify store or API Key in `input configuration`.", + "missing_token_error": "Authentication was unsuccessful. Please verify your authentication credentials or login is correct.", # add the other patterns and description, if needed... } return connection_check_errors_map.get(pattern) @@ -818,7 +90,7 @@ def test_connection(self) -> tuple[bool, str]: return False, "The `Shopify Store` name is missing. Make sure it's entered and valid." try: - response = list(self.test_stream.read_records(sync_mode=None)) + response = list(self.test_stream.read_records(sync_mode=SyncMode.full_refresh)) # check for the shop_id is present in the response shop_id = response[0].get("id") if shop_id is not None: @@ -831,9 +103,27 @@ def test_connection(self) -> tuple[bool, str]: return False, self.describe_error("request_exception", details=req_error) except IndexError: return False, self.describe_error("index_error", shop_name, response) + except MissingAccessTokenError: + return False, self.describe_error("missing_token_error") + + def get_shop_id(self) -> str: + """ + We need to have the `shop_id` value available to have it passed elsewhere and fill-in the missing data. + By the time this method is tiggered, we are sure we've passed the `Connection Checks` and have the `shop_id` value. + """ + response = list(self.test_stream.read_records(sync_mode=SyncMode.full_refresh)) + shop_id = response[0].get("id") + if shop_id: + return shop_id + else: + raise Exception(f"Couldn't get `shop_id`. Actual `response`: {response}.") class SourceShopify(AbstractSource): + @property + def continue_sync_on_stream_failure(self) -> bool: + return True + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: """ Testing connection availability for the connector. @@ -849,13 +139,12 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ config["shop"] = self.get_shop_name(config) config["authenticator"] = ShopifyAuthenticator(config) + # add `shop_id` int value + config["shop_id"] = ConnectionCheckTest(config).get_shop_id() user_scopes = self.get_user_scopes(config) always_permitted_streams = ["MetafieldShops", "Shop", "Countries"] permitted_streams = [ - stream - for user_scope in user_scopes - if user_scope["handle"] in SCOPES_MAPPING - for stream in SCOPES_MAPPING.get(user_scope["handle"]) + stream for stream, stream_scopes in SCOPES_MAPPING.items() if all(scope in user_scopes for scope in stream_scopes) ] + always_permitted_streams # before adding stream to stream_instances list, please add it to SCOPES_MAPPING @@ -902,6 +191,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: SmartCollections(config), TenderTransactions(config), Transactions(config), + TransactionsGraphql(config), CustomerSavedSearch(config), CustomerAddress(config), Countries(config), @@ -910,14 +200,13 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: return [stream_instance for stream_instance in stream_instances if self.format_name(stream_instance.name) in permitted_streams] @staticmethod - def get_user_scopes(config): + def get_user_scopes(config) -> list[Any]: session = requests.Session() url = f"https://{config['shop']}.myshopify.com/admin/oauth/access_scopes.json" headers = config["authenticator"].get_auth_header() - try: response = session.get(url, headers=headers).json() - access_scopes = response.get("access_scopes") + access_scopes = [scope.get("handle") for scope in response.get("access_scopes")] except InvalidURL: raise ShopifyWrongShopNameError(url) except JSONDecodeError as json_error: @@ -931,11 +220,11 @@ def get_user_scopes(config): raise ShopifyAccessScopesError(response) @staticmethod - def get_shop_name(config): + def get_shop_name(config) -> str: split_pattern = ".myshopify.com" shop_name = config.get("shop") return shop_name.split(split_pattern)[0] if split_pattern in shop_name else shop_name @staticmethod - def format_name(name): + def format_name(name) -> str: return "".join(x.capitalize() for x in name.split("_")) diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/spec.json b/airbyte-integrations/connectors/source-shopify/source_shopify/spec.json index c73f7884f6e7..6d44fec5d047 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/spec.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/spec.json @@ -1,17 +1,17 @@ { "documentationUrl": "https://docs.airbyte.com/integrations/sources/shopify", "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "title": "Shopify Source CDK Specifications", "type": "object", - "required": ["shop", "start_date"], + "required": ["shop"], "additionalProperties": true, "properties": { "shop": { "type": "string", "title": "Shopify Store", "description": "The name of your Shopify store found in the URL. For example, if your URL was https://NAME.myshopify.com, then the name would be 'NAME' or 'NAME.myshopify.com'.", - "pattern": "^(?!https://)(?!http://).*", + "pattern": "^(?!https://)(?!https://).*", "examples": ["my-store", "my-store.myshopify.com"], "order": 1 }, @@ -21,26 +21,6 @@ "type": "object", "order": 2, "oneOf": [ - { - "title": "API Password", - "description": "API Password Auth", - "type": "object", - "required": ["auth_method", "api_password"], - "properties": { - "auth_method": { - "type": "string", - "const": "api_password", - "order": 0 - }, - "api_password": { - "type": "string", - "title": "API Password", - "description": "The API Password for your private application in the `Shopify` store.", - "airbyte_secret": true, - "order": 1 - } - } - }, { "type": "object", "title": "OAuth2.0", @@ -74,6 +54,26 @@ "order": 3 } } + }, + { + "title": "API Password", + "description": "API Password Auth", + "type": "object", + "required": ["auth_method", "api_password"], + "properties": { + "auth_method": { + "type": "string", + "const": "api_password", + "order": 0 + }, + "api_password": { + "type": "string", + "title": "API Password", + "description": "The API Password for your private application in the `Shopify` store.", + "airbyte_secret": true, + "order": 1 + } + } } ] }, @@ -81,10 +81,16 @@ "type": "string", "title": "Replication Start Date", "description": "The date you would like to replicate data from. Format: YYYY-MM-DD. Any data before this date will not be replicated.", - "examples": ["2021-01-01"], + "default": "2020-01-01", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", "format": "date", "order": 3 + }, + "bulk_window_in_days": { + "type": "integer", + "title": "GraphQL BULK Date Range in Days", + "description": "Defines what would be a date range per single BULK Job", + "default": 30 } } }, diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/streams/base_streams.py b/airbyte-integrations/connectors/source-shopify/source_shopify/streams/base_streams.py new file mode 100644 index 000000000000..a31db051db3c --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/streams/base_streams.py @@ -0,0 +1,672 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import logging +from abc import ABC, abstractmethod +from functools import cached_property +from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional, Union +from urllib.parse import parse_qsl, urlparse + +import pendulum as pdm +import requests +from airbyte_cdk.sources.streams.http import HttpStream +from requests.exceptions import RequestException +from source_shopify.shopify_graphql.bulk.job import ShopifyBulkManager +from source_shopify.shopify_graphql.bulk.query import ShopifyBulkQuery, ShopifyBulkTemplates +from source_shopify.shopify_graphql.bulk.record import ShopifyBulkRecord +from source_shopify.transform import DataTypeEnforcer +from source_shopify.utils import EagerlyCachedStreamState as stream_state_cache +from source_shopify.utils import ShopifyNonRetryableErrors +from source_shopify.utils import ShopifyRateLimiter as limiter + + +class ShopifyStream(HttpStream, ABC): + # define default logger + logger = logging.getLogger("airbyte") + + # Latest Stable Release + api_version = "2023-07" + # Page size + limit = 250 + + primary_key = "id" + order_field = "updated_at" + filter_field = "updated_at_min" + + raise_on_http_errors = True + max_retries = 5 + + def __init__(self, config: Dict) -> None: + super().__init__(authenticator=config["authenticator"]) + self._transformer = DataTypeEnforcer(self.get_json_schema()) + self.config = config + + @property + @abstractmethod + def data_field(self) -> str: + """The name of the field in the response which contains the data""" + + @property + def url_base(self) -> str: + return f"https://{self.config['shop']}.myshopify.com/admin/api/{self.api_version}/" + + @property + def default_filter_field_value(self) -> Union[int, str]: + # certain streams are using `since_id` field as `filter_field`, which requires to use `int` type, + # but many other use `str` values for this, we determine what to use based on `filter_field` value + # by default, we use the user defined `Start Date` as initial value, or 0 for `id`-dependent streams. + return 0 if self.filter_field == "since_id" else (self.config.get("start_date") or "") + + def path(self, **kwargs) -> str: + return f"{self.data_field}.json" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + next_page = response.links.get("next", None) + if next_page: + return dict(parse_qsl(urlparse(next_page.get("url")).query)) + else: + return None + + def request_params(self, next_page_token: Optional[Mapping[str, Any]] = None, **kwargs) -> MutableMapping[str, Any]: + params = {"limit": self.limit} + if next_page_token: + params.update(**next_page_token) + else: + params["order"] = f"{self.order_field} asc" + params[self.filter_field] = self.default_filter_field_value + return params + + @limiter.balance_rate_limit() + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + if response.status_code is requests.codes.OK: + try: + json_response = response.json() + records = json_response.get(self.data_field, []) if self.data_field is not None else json_response + yield from self.produce_records(records) + except RequestException as e: + self.logger.warning(f"Unexpected error in `parse_ersponse`: {e}, the actual response data: {response.text}") + yield {} + + def produce_records(self, records: Optional[Union[Iterable[Mapping[str, Any]], Mapping[str, Any]]] = None) -> Mapping[str, Any]: + # transform method was implemented according to issue 4841 + # Shopify API returns price fields as a string and it should be converted to number + # this solution designed to convert string into number, but in future can be modified for general purpose + if isinstance(records, dict): + # for cases when we have a single record as dict + # add shop_url to the record to make querying easy + records["shop_url"] = self.config["shop"] + yield self._transformer.transform(records) + else: + # for other cases + for record in records: + # add shop_url to the record to make querying easy + record["shop_url"] = self.config["shop"] + yield self._transformer.transform(record) + + def should_retry(self, response: requests.Response) -> bool: + known_errors = ShopifyNonRetryableErrors(self.name) + status = response.status_code + if status in known_errors.keys(): + setattr(self, "raise_on_http_errors", False) + self.logger.warning(known_errors.get(status)) + return False + else: + return super().should_retry(response) + + +class ShopifyDeletedEventsStream(ShopifyStream): + data_field = "events" + primary_key = "id" + cursor_field = "deleted_at" + + def __init__(self, config: Dict, deleted_events_api_name: str) -> None: + self.deleted_events_api_name = deleted_events_api_name + super().__init__(config) + + @property + def availability_strategy(self) -> None: + """ + No need to apply the `availability strategy` for this service stream. + """ + return None + + def get_json_schema(self) -> None: + """ + No need to apply the `schema` for this service stream. + Return `{}` to satisfy the `self._transformer.transform(record)` logic. + """ + return {} + + def produce_deleted_records_from_events(self, delete_events: Iterable[Mapping[str, Any]] = []) -> Mapping[str, Any]: + for event in delete_events: + yield { + "id": event["subject_id"], + self.cursor_field: event["created_at"], + "updated_at": event["created_at"], + "deleted_message": event["message"], + "deleted_description": event["description"], + "shop_url": event["shop_url"], + } + + def read_records(self, stream_state: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + delete_events = super().read_records(stream_state=stream_state, **kwargs) + yield from self.produce_deleted_records_from_events(delete_events) + + def request_params( + self, + stream_state: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + **kwargs, + ) -> MutableMapping[str, Any]: + params: Mapping[str, Any] = {} + + if next_page_token: + # `filter` and `verb` cannot be passed, when `page_info` is present. + # See https://shopify.dev/api/usage/pagination-rest + params.update(**next_page_token) + else: + params.update(**{"filter": self.deleted_events_api_name, "verb": "destroy"}) + if stream_state: + state = stream_state.get("deleted", {}).get(self.cursor_field) + if state: + params["created_at_min"] = state + return params + + +class IncrementalShopifyStream(ShopifyStream, ABC): + # Setting the check point interval to the limit of the records output + @property + def state_checkpoint_interval(self) -> int: + return super().limit + + # Setting the default cursor field for all streams + cursor_field = "updated_at" + deleted_cursor_field = "deleted_at" + + @property + def default_state_comparison_value(self) -> Union[int, str]: + # certain streams are using `id` field as `cursor_field`, which requires to use `int` type, + # but many other use `str` values for this, we determine what to use based on `cursor_field` value + return 0 if self.cursor_field == "id" else "" + + def get_updated_state( + self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any] + ) -> MutableMapping[str, Any]: + last_record_value = latest_record.get(self.cursor_field) or self.default_state_comparison_value + current_state_value = current_stream_state.get(self.cursor_field) or self.default_state_comparison_value + return {self.cursor_field: max(last_record_value, current_state_value)} + + @stream_state_cache.cache_stream_state + def request_params( + self, stream_state: Optional[Mapping[str, Any]] = None, next_page_token: Optional[Mapping[str, Any]] = None, **kwargs + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) + # If there is a next page token then we should only send pagination-related parameters. + if not next_page_token: + params["order"] = f"{self.order_field} asc" + if stream_state: + params[self.filter_field] = stream_state.get(self.cursor_field) + return params + + # Parse the `stream_slice` with respect to `stream_state` for `Incremental refresh` + # cases where we slice the stream, the endpoints for those classes don't accept any other filtering, + # but they provide us with the updated_at field in most cases, so we used that as incremental filtering during the order slicing. + def filter_records_newer_than_state( + self, stream_state: Optional[Mapping[str, Any]] = None, records_slice: Optional[Iterable[Mapping]] = None + ) -> Iterable: + # Getting records >= state + if stream_state: + state_value = stream_state.get(self.cursor_field) + for record in records_slice: + if self.cursor_field in record: + record_value = record.get(self.cursor_field, self.default_state_comparison_value) + if record_value: + if record_value >= state_value: + yield record + else: + # old entities could have cursor field in place, but set to null + self.logger.warning( + f"Stream `{self.name}`, Record ID: `{record.get(self.primary_key)}` cursor value is: {record_value}, record is emitted without state comparison" + ) + yield record + else: + # old entities could miss the cursor field + self.logger.warning( + f"Stream `{self.name}`, Record ID: `{record.get(self.primary_key)}` missing cursor field: {self.cursor_field}, record is emitted without state comparison" + ) + yield record + else: + yield from records_slice + + +class IncrementalShopifySubstream(IncrementalShopifyStream): + """ + IncrementalShopifySubstream - provides slicing functionality for streams using parts of data from parent stream. + For example: + - `Refunds Orders` is the entity of `Orders`, + - `OrdersRisks` is the entity of `Orders`, + - `DiscountCodes` is the entity of `PriceRules`, etc. + + :: @ parent_stream - defines the parent stream object to read from + :: @ slice_key - defines the name of the property in stream slices dict. + :: @ nested_record - the name of the field inside of parent stream record. Default is `id`. + :: @ nested_record_field_name - the name of the field inside of nested_record. + :: @ nested_substream - the name of the nested entity inside of parent stream, helps to reduce the number of + API Calls, if present, see `OrderRefunds` stream for more. + """ + + parent_stream_class: Union[ShopifyStream, IncrementalShopifyStream] = None + slice_key: str = None + nested_record: str = "id" + nested_record_field_name: str = None + nested_substream = None + nested_substream_list_field_id = None + + @cached_property + def parent_stream(self) -> Union[ShopifyStream, IncrementalShopifyStream]: + """ + Returns the instance of parent stream, if the substream has a `parent_stream_class` dependency. + """ + return self.parent_stream_class(self.config) if self.parent_stream_class else None + + def get_updated_state( + self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any] + ) -> MutableMapping[str, Any]: + """UPDATING THE STATE OBJECT: + Stream: Transactions + Parent Stream: Orders + Returns: + { + {...}, + "transactions": { + "created_at": "2022-03-03T03:47:45-08:00", + "orders": { + "updated_at": "2022-03-03T03:47:46-08:00" + } + }, + {...}, + } + """ + updated_state = super().get_updated_state(current_stream_state, latest_record) + # add parent_stream_state to `updated_state` + updated_state[self.parent_stream.name] = stream_state_cache.cached_state.get(self.parent_stream.name) + return updated_state + + def request_params(self, next_page_token: Optional[Mapping[str, Any]] = None, **kwargs) -> MutableMapping[str, Any]: + params = {"limit": self.limit} + if next_page_token: + params.update(**next_page_token) + return params + + def stream_slices(self, stream_state: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + """ + Reading the parent stream for slices with structure: + EXAMPLE: for given nested_record as `id` of Orders, + + Outputs: + [ + {slice_key: 123}, + {slice_key: 456}, + {...}, + {slice_key: 999 + ] + """ + sorted_substream_slices = [] + + # reading parent nested stream_state from child stream state + parent_stream_state = stream_state.get(self.parent_stream.name) if stream_state else {} + + # reading the parent stream + for record in self.parent_stream.read_records(stream_state=parent_stream_state, **kwargs): + # updating the `stream_state` with the state of it's parent stream + # to have the child stream sync independently from the parent stream + stream_state_cache.cached_state[self.parent_stream.name] = self.parent_stream.get_updated_state({}, record) + # to limit the number of API Calls and reduce the time of data fetch, + # we can pull the ready data for child_substream, if nested data is present, + # and corresponds to the data of child_substream we need. + if self.nested_substream and self.nested_substream_list_field_id: + if record.get(self.nested_substream): + sorted_substream_slices.extend( + [ + { + self.slice_key: sub_record[self.nested_substream_list_field_id], + self.cursor_field: record[self.nested_substream][0].get( + self.cursor_field, self.default_state_comparison_value + ), + } + for sub_record in record[self.nested_record] + ] + ) + elif self.nested_substream: + if record.get(self.nested_substream): + sorted_substream_slices.append( + { + self.slice_key: record[self.nested_record], + self.cursor_field: record[self.nested_substream][0].get(self.cursor_field, self.default_state_comparison_value), + } + ) + else: + # avoid checking `deleted` records for substreams, a.k.a `Metafields` streams, + # since `deleted` records are not available, thus we avoid HTTP-400 errors. + if self.deleted_cursor_field not in record: + yield {self.slice_key: record[self.nested_record]} + + # output slice from sorted list to avoid filtering older records + if self.nested_substream: + if len(sorted_substream_slices) > 0: + # sort by cursor_field + sorted_substream_slices.sort(key=lambda x: x.get(self.cursor_field)) + for sorted_slice in sorted_substream_slices: + yield {self.slice_key: sorted_slice[self.slice_key]} + + # the stream_state caching is required to avoid the STATE collisions for Substreams + @stream_state_cache.cache_stream_state + def read_records( + self, + stream_state: Optional[Mapping[str, Any]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + **kwargs, + ) -> Iterable[Mapping[str, Any]]: + """Reading child streams records for each `id`""" + + slice_data = stream_slice.get(self.slice_key) + # sometimes the stream_slice.get(self.slice_key) has the list of records, + # to avoid data exposition inside the logs, we should get the data we need correctly out of stream_slice. + if isinstance(slice_data, list) and self.nested_record_field_name is not None and len(slice_data) > 0: + slice_data = slice_data[0].get(self.nested_record_field_name) + + # reading substream records + self.logger.info(f"Reading {self.name} for {self.slice_key}: {slice_data}") + records = super().read_records(stream_slice=stream_slice, **kwargs) + # get the cached substream state, to avoid state collisions for Incremental Syncs + cached_substream_state = stream_state_cache.cached_state.get(self.name, {}) + # filtering the portion of already emmited substream records using cached state value, + # since the actual `metafields` endpoint doesn't support the server-side filtering using query params + # thus to avoid the duplicates - we filter the records following the cached state, + # which is freezed every time the sync starts using the actual STATE provided, + # while the active STATE is updated during the sync and saved as expected, in the end. + yield from self.filter_records_newer_than_state(stream_state=cached_substream_state, records_slice=records) + + +class MetafieldShopifySubstream(IncrementalShopifySubstream): + slice_key = "id" + data_field = "metafields" + + parent_stream_class: Union[ShopifyStream, IncrementalShopifyStream] = None + + def path(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> str: + object_id = stream_slice[self.slice_key] + return f"{self.parent_stream_class.data_field}/{object_id}/{self.data_field}.json" + + +class IncrementalShopifyNestedStream(IncrementalShopifyStream): + """ + IncrementalShopifyNestedStream - provides slicing functionality for streams using parts of data from parent stream. + + For example: + - `refunds` is the entity of `order.refunds`, + - `fulfillments` is the entity of the `order.fulfillments` which is nested sub-entity + + :: @ parent_stream - defines the parent stream object to read from + :: @ mutation_map - defines how the nested record should be populated with additional values, + available from parent record. + Example: + >> mutation_map = {"parent_id": "id"}, + where `parent_id` is the new field that created for each subrecord available. + and `id` is the parent_key named `id`, we take the value from. + + :: @ nested_entity - the name of the nested entity inside of parent stream, helps to reduce the number of + API Calls, if present, see `OrderRefunds` or `Fulfillments` streams for more info. + """ + + data_field = None + parent_stream_class: Union[ShopifyStream, IncrementalShopifyStream] = None + mutation_map: Mapping[str, Any] = None + nested_entity = None + + @cached_property + def parent_stream(self) -> object: + """ + Returns the instance of parent stream, if the substream has a `parent_stream_class` dependency. + """ + return self.parent_stream_class(self.config) if self.parent_stream_class else None + + def path(self, **kwargs) -> str: + """ + NOT USED FOR THIS TYPE OF STREAMS. + """ + return "" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + NOT USED FOR THIS TYPE OF STREAMS. + """ + return None + + def request_params(self, **kwargs) -> MutableMapping[str, Any]: + """ + NOT USED FOR THIS TYPE OF STREAMS. + """ + return {} + + def get_updated_state( + self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any] + ) -> MutableMapping[str, Any]: + """UPDATING THE STATE OBJECT: + Stream: Transactions + Parent Stream: Orders + Returns: + { + {...}, + "transactions": { + "created_at": "2022-03-03T03:47:45-08:00", + "orders": { + "updated_at": "2022-03-03T03:47:46-08:00" + } + }, + {...}, + } + """ + updated_state = super().get_updated_state(current_stream_state, latest_record) + # add parent_stream_state to `updated_state` + updated_state[self.parent_stream.name] = stream_state_cache.cached_state.get(self.parent_stream.name) + return updated_state + + def add_parent_id(self, record: Optional[Mapping[str, Any]] = None) -> Mapping[str, Any]: + """ + Adds new field to the record with name `key` based on the `value` key from record. + """ + if self.mutation_map and record: + for subrecord in record.get(self.nested_entity, []): + for k, v in self.mutation_map.items(): + subrecord[k] = record.get(v) + else: + return record + + # the stream_state caching is required to avoid the STATE collisions for Substreams + @stream_state_cache.cache_stream_state + def stream_slices(self, stream_state: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + parent_stream_state = stream_state.get(self.parent_stream.name) if stream_state else {} + for record in self.parent_stream.read_records(stream_state=parent_stream_state, **kwargs): + # updating the `stream_state` with the state of it's parent stream + # to have the child stream sync independently from the parent stream + stream_state_cache.cached_state[self.parent_stream.name] = self.parent_stream.get_updated_state({}, record) + # to limit the number of API Calls and reduce the time of data fetch, + # we can pull the ready data for child_substream, if nested data is present, + # and corresponds to the data of child_substream we need. + if self.nested_entity in record.keys(): + # add parent_id key, value from mutation_map, if passed. + self.add_parent_id(record) + # yield nested sub-rcords + yield from [{self.nested_entity: sub_record} for sub_record in record.get(self.nested_entity, [])] + + def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + # get the cached substream state, to avoid state collisions for Incremental Syncs + cached_state = stream_state_cache.cached_state.get(self.name, {}) + # emitting nested parent entity + yield from self.filter_records_newer_than_state(cached_state, self.produce_records(stream_slice.get(self.nested_entity, []))) + + +class IncrementalShopifyStreamWithDeletedEvents(IncrementalShopifyStream): + @property + @abstractmethod + def deleted_events_api_name(self) -> str: + """ + The string value of the Shopify Events Object to pull: + + articles -> Article + blogs -> Blog + custom_collections -> Collection + orders -> Order + pages -> Page + price_rules -> PriceRule + products -> Product + + """ + + @property + def deleted_events(self) -> ShopifyDeletedEventsStream: + """ + The Events stream instance to fetch the `destroyed` records for specified `deleted_events_api_name`, like: `Product`. + See more in `ShopifyDeletedEventsStream` class. + """ + return ShopifyDeletedEventsStream(self.config, self.deleted_events_api_name) + + @property + def default_deleted_state_comparison_value(self) -> Union[int, str]: + """ + Set the default STATE comparison value for cases when the deleted record doesn't have it's value. + We expect the `deleted_at` cursor field for destroyed records would be always type of String. + """ + return "" + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + We extend the stream state with `deleted` property to store the `destroyed` records STATE separetely from the Stream State. + """ + state = super().get_updated_state(current_stream_state, latest_record) + # add `deleted` property to each stream supports `deleted events`, + # to provide the `Incremental` sync mode, for the `Incremental Delete` records. + last_deleted_record_value = latest_record.get(self.deleted_cursor_field) or self.default_deleted_state_comparison_value + current_deleted_state_value = current_stream_state.get(self.deleted_cursor_field) or self.default_deleted_state_comparison_value + state["deleted"] = {self.deleted_cursor_field: max(last_deleted_record_value, current_deleted_state_value)} + return state + + def read_records( + self, + stream_state: Optional[Mapping[str, Any]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + **kwargs, + ) -> Iterable[Mapping[str, Any]]: + """Override to fetch deleted records for supported streams""" + # main records stream + yield from super().read_records(stream_state=stream_state, stream_slice=stream_slice, **kwargs) + # fetch deleted events after the Stream data is pulled + yield from self.deleted_events.read_records(stream_state=stream_state, **kwargs) + + +class IncrementalShopifyGraphQlBulkStream(IncrementalShopifyStream): + filter_field = "updated_at" + cursor_field = "updated_at" + data_field = "graphql" + http_method = "POST" + + def __init__(self, config: Dict) -> None: + super().__init__(config) + # init BULK Query instance, pass `shop_id` from config + self.query = self.bulk_query(shop_id=config.get("shop_id")) + # define BULK Manager instance + self.job_manager: ShopifyBulkManager = ShopifyBulkManager( + session=self._session, + base_url=f"{self.url_base}/{self.path()}", + logger=self.logger, + ) + # define Record Producer instance + self.record_producer: ShopifyBulkRecord = ShopifyBulkRecord(self.query, self.logger) + + @property + def slice_interval_in_days(self) -> int: + """ + Defines date range per single BULK Job. + """ + return self.config.get("bulk_window_in_days", 30) + + @property + @abstractmethod + def bulk_query(self) -> ShopifyBulkQuery: + """ + This method property should be defined in the stream class instance, + and should be instantiated from the `ShopifyBulkQuery` class. + """ + + def add_shop_url_field(self, records: Iterable[MutableMapping[str, Any]] = []) -> Iterable[MutableMapping[str, Any]]: + # ! Mandatory, add shop_url to the record to make querying easy + # more info: https://github.com/airbytehq/airbyte/issues/25110 + for record in records: + if record: + record["shop_url"] = self.config["shop"] + yield record + + # CDK OVERIDES + @property + def availability_strategy(self) -> None: + """NOT USED FOR BULK OPERATIONS TO SAVE THE RATE LIMITS AND TIME FOR THE SYNC.""" + return None + + def request_params(self, **kwargs) -> MutableMapping[str, Any]: + """ + NOT USED FOR SHOPIFY BULK OPERARTIONS. + https://shopify.dev/docs/api/usage/bulk-operations/queries#write-a-bulk-operation + """ + return {} + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + NOT USED FOR SHOPIFY BULK OPERATIONS. + https://shopify.dev/docs/api/usage/bulk-operations/queries#write-a-bulk-operation + """ + return None + + def request_body_json(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Mapping[str, Any]: + """ + Override for _send_request CDK method to send HTTP request to Shopify BULK Operatoions. + https://shopify.dev/docs/api/usage/bulk-operations/queries#bulk-query-overview + """ + return {"query": ShopifyBulkTemplates.prepare(stream_slice.get("query"))} + + @stream_state_cache.cache_stream_state + def stream_slices(self, stream_state: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + if self.filter_field: + start = pdm.parse(stream_state.get(self.cursor_field) if stream_state else self.config.get("start_date")) + end = pdm.now() + while start < end: + slice_end = start.add(days=self.slice_interval_in_days) + # check end period is less than now() or now() is applied otherwise. + slice_end = slice_end if slice_end < end else end + # making pre-defined sliced query to pass it directly + prepared_query = self.query.get(self.filter_field, start.to_rfc3339_string(), slice_end.to_rfc3339_string()) + self.logger.info(f"Stream: `{self.name}` requesting BULK Job for period: {start} -- {slice_end}.") + yield {"query": prepared_query} + start = slice_end + else: + # for the streams that don't support filtering + yield {"query": self.query.get()} + + def process_bulk_results(self, response: requests.Response, stream_state: Optional[Mapping[str, Any]] = None) -> Mapping[str, Any]: + # get results fetched from COMPLETED BULK Job or `None` + filename = self.job_manager.job_check(response) + # the `filename` could be `None`, meaning there are no data available for the slice period. + if filename: + # add `shop_url` field to each record produced + records = self.add_shop_url_field( + # produce records from saved bulk job result + self.record_producer.read_file(filename) + ) + yield from self.filter_records_newer_than_state(stream_state, records) + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + # get the cached substream state, to avoid state collisions for Incremental Syncs + stream_state = stream_state_cache.cached_state.get(self.name, {self.cursor_field: self.config["start_date"]}) + yield from self.process_bulk_results(response, stream_state) diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/streams/streams.py b/airbyte-integrations/connectors/source-shopify/source_shopify/streams/streams.py new file mode 100644 index 000000000000..61d5663d89e7 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/streams/streams.py @@ -0,0 +1,386 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from typing import Any, Iterable, Mapping, MutableMapping, Optional + +import requests +from airbyte_cdk.sources.streams.core import package_name_from_class +from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader +from requests.exceptions import RequestException +from source_shopify.shopify_graphql.bulk.query import ( + Collection, + DiscountCode, + FulfillmentOrder, + InventoryItem, + InventoryLevel, + MetafieldCollection, + MetafieldCustomer, + MetafieldDraftOrder, + MetafieldLocation, + MetafieldOrder, + MetafieldProduct, + MetafieldProductImage, + MetafieldProductVariant, + Transaction, +) +from source_shopify.shopify_graphql.graphql import get_query_products +from source_shopify.utils import ApiTypeEnum +from source_shopify.utils import ShopifyRateLimiter as limiter + +from .base_streams import ( + IncrementalShopifyGraphQlBulkStream, + IncrementalShopifyNestedStream, + IncrementalShopifyStream, + IncrementalShopifyStreamWithDeletedEvents, + IncrementalShopifySubstream, + MetafieldShopifySubstream, + ShopifyStream, +) + + +class Articles(IncrementalShopifyStreamWithDeletedEvents): + data_field = "articles" + cursor_field = "id" + order_field = "id" + filter_field = "since_id" + deleted_events_api_name = "Article" + + +class MetafieldArticles(MetafieldShopifySubstream): + parent_stream_class = Articles + + +class Blogs(IncrementalShopifyStreamWithDeletedEvents): + cursor_field = "id" + order_field = "id" + data_field = "blogs" + filter_field = "since_id" + deleted_events_api_name = "Blog" + + +class MetafieldBlogs(MetafieldShopifySubstream): + parent_stream_class = Blogs + + +class Customers(IncrementalShopifyStream): + data_field = "customers" + + +class MetafieldCustomers(IncrementalShopifyGraphQlBulkStream): + bulk_query: MetafieldCustomer = MetafieldCustomer + + +class Orders(IncrementalShopifyStreamWithDeletedEvents): + data_field = "orders" + deleted_events_api_name = "Order" + + def request_params( + self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) + if not next_page_token: + params["status"] = "any" + return params + + +class Disputes(IncrementalShopifyStream): + data_field = "disputes" + filter_field = "since_id" + cursor_field = "id" + order_field = "id" + + def path(self, **kwargs) -> str: + return f"shopify_payments/{self.data_field}.json" + + +class MetafieldOrders(IncrementalShopifyGraphQlBulkStream): + bulk_query: MetafieldOrder = MetafieldOrder + + +class DraftOrders(IncrementalShopifyStream): + data_field = "draft_orders" + + +class MetafieldDraftOrders(IncrementalShopifyGraphQlBulkStream): + bulk_query: MetafieldDraftOrder = MetafieldDraftOrder + + +class Products(IncrementalShopifyStreamWithDeletedEvents): + use_cache = True + data_field = "products" + deleted_events_api_name = "Product" + + +class ProductsGraphQl(IncrementalShopifyStream): + filter_field = "updatedAt" + cursor_field = "updatedAt" + data_field = "graphql" + http_method = "POST" + + def request_params( + self, + stream_state: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + **kwargs, + ) -> MutableMapping[str, Any]: + return {} + + def request_body_json( + self, + stream_state: Mapping[str, Any], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Optional[Mapping]: + state_value = stream_state.get(self.filter_field) + if state_value: + filter_value = state_value + else: + filter_value = self.default_filter_field_value + query = get_query_products( + first=self.limit, filter_field=self.filter_field, filter_value=filter_value, next_page_token=next_page_token + ) + return {"query": query} + + @staticmethod + def next_page_token(response: requests.Response) -> Optional[Mapping[str, Any]]: + page_info = response.json()["data"]["products"]["pageInfo"] + has_next_page = page_info["hasNextPage"] + if has_next_page: + return page_info["endCursor"] + else: + return None + + @limiter.balance_rate_limit(api_type=ApiTypeEnum.graphql.value) + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + if response.status_code is requests.codes.OK: + try: + json_response = response.json()["data"]["products"]["nodes"] + yield from self.produce_records(json_response) + except RequestException as e: + self.logger.warning(f"Unexpected error in `parse_ersponse`: {e}, the actual response data: {response.text}") + + +class MetafieldProducts(IncrementalShopifyGraphQlBulkStream): + bulk_query: MetafieldProduct = MetafieldProduct + + +class ProductImages(IncrementalShopifyNestedStream): + parent_stream_class = Products + nested_entity = "images" + # add `product_id` to each nested subrecord + mutation_map = {"product_id": "id"} + + +class MetafieldProductImages(IncrementalShopifyGraphQlBulkStream): + bulk_query: MetafieldProductImage = MetafieldProductImage + + +class ProductVariants(IncrementalShopifyNestedStream): + parent_stream_class = Products + nested_entity = "variants" + # add `product_id` to each nested subrecord + mutation_map = {"product_id": "id"} + + +class MetafieldProductVariants(IncrementalShopifyGraphQlBulkStream): + bulk_query: MetafieldProductVariant = MetafieldProductVariant + + +class AbandonedCheckouts(IncrementalShopifyStream): + data_field = "checkouts" + + def request_params( + self, stream_state: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, **kwargs + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state=stream_state, next_page_token=next_page_token, **kwargs) + # If there is a next page token then we should only send pagination-related parameters. + if not next_page_token: + params["status"] = "any" + return params + + +class CustomCollections(IncrementalShopifyStreamWithDeletedEvents): + data_field = "custom_collections" + deleted_events_api_name = "Collection" + + +class SmartCollections(IncrementalShopifyStream): + data_field = "smart_collections" + + +class MetafieldSmartCollections(MetafieldShopifySubstream): + parent_stream_class = SmartCollections + + +class Collects(IncrementalShopifyStream): + """ + Collects stream does not support Incremental Refresh based on datetime fields, only `since_id` is supported: + https://shopify.dev/docs/admin-api/rest/reference/products/collect + + The Collect stream is the link between Products and Collections, if the Collection is created for Products, + the `collect` record is created, it's reasonable to Full Refresh all collects. As for Incremental refresh - + we would use the since_id specificaly for this stream. + """ + + data_field = "collects" + cursor_field = "id" + order_field = "id" + filter_field = "since_id" + + +class Collections(IncrementalShopifyGraphQlBulkStream): + bulk_query: Collection = Collection + + +class MetafieldCollections(IncrementalShopifyGraphQlBulkStream): + bulk_query: MetafieldCollection = MetafieldCollection + + +class BalanceTransactions(IncrementalShopifyStream): + + """ + PaymentsTransactions stream does not support Incremental Refresh based on datetime fields, only `since_id` is supported: + https://shopify.dev/api/admin-rest/2021-07/resources/transactions + """ + + data_field = "transactions" + cursor_field = "id" + order_field = "id" + filter_field = "since_id" + + def path(self, **kwargs) -> str: + return f"shopify_payments/balance/{self.data_field}.json" + + +class OrderRefunds(IncrementalShopifyNestedStream): + parent_stream_class = Orders + # override default cursor field + cursor_field = "created_at" + nested_entity = "refunds" + + +class OrderRisks(IncrementalShopifySubstream): + parent_stream_class = Orders + slice_key = "order_id" + data_field = "risks" + cursor_field = "id" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + order_id = stream_slice["order_id"] + return f"orders/{order_id}/{self.data_field}.json" + + +class Transactions(IncrementalShopifySubstream): + parent_stream_class = Orders + slice_key = "order_id" + data_field = "transactions" + cursor_field = "created_at" + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + order_id = stream_slice["order_id"] + return f"orders/{order_id}/{self.data_field}.json" + + +class TransactionsGraphql(IncrementalShopifyGraphQlBulkStream): + bulk_query: Transaction = Transaction + cursor_field = "created_at" + + def get_json_schema(self) -> Mapping[str, Any]: + """ + This stream has the same schema as `Transactions` stream, except of: + - fields: [ `device_id, source_name, user_id, location_id` ] + + Specifically: + - `user_id` field requires `Shopify Plus` / be authorised via `Financialy Embedded App`. + - additional `read_users` scope is required https://shopify.dev/docs/api/usage/access-scopes#authenticated-access-scopes + """ + return ResourceSchemaLoader(package_name_from_class(Transactions)).get_schema("transactions") + + +class TenderTransactions(IncrementalShopifyStream): + data_field = "tender_transactions" + cursor_field = "processed_at" + filter_field = "processed_at_min" + + +class Pages(IncrementalShopifyStreamWithDeletedEvents): + data_field = "pages" + deleted_events_api_name = "Page" + + +class MetafieldPages(MetafieldShopifySubstream): + parent_stream_class = Pages + + +class PriceRules(IncrementalShopifyStreamWithDeletedEvents): + data_field = "price_rules" + deleted_events_api_name = "PriceRule" + + +class DiscountCodes(IncrementalShopifyGraphQlBulkStream): + bulk_query: DiscountCode = DiscountCode + + +class Locations(ShopifyStream): + """ + The location API does not support any form of filtering. + https://shopify.dev/api/admin-rest/2021-07/resources/location + + Therefore, only FULL_REFRESH mode is supported. + """ + + data_field = "locations" + + +class MetafieldLocations(IncrementalShopifyGraphQlBulkStream): + bulk_query: MetafieldLocation = MetafieldLocation + filter_field = None + + +class InventoryLevels(IncrementalShopifyGraphQlBulkStream): + bulk_query: InventoryLevel = InventoryLevel + + +class InventoryItems(IncrementalShopifyGraphQlBulkStream): + bulk_query: InventoryItem = InventoryItem + + +class FulfillmentOrders(IncrementalShopifyGraphQlBulkStream): + bulk_query: FulfillmentOrder = FulfillmentOrder + + +class Fulfillments(IncrementalShopifyNestedStream): + parent_stream_class = Orders + nested_entity = "fulfillments" + + +class Shop(ShopifyStream): + data_field = "shop" + + +class MetafieldShops(IncrementalShopifyStream): + data_field = "metafields" + + +class CustomerSavedSearch(IncrementalShopifyStream): + api_version = "2022-01" + cursor_field = "id" + order_field = "id" + data_field = "customer_saved_searches" + filter_field = "since_id" + + +class CustomerAddress(IncrementalShopifyNestedStream): + """ + https://shopify.dev/docs/api/admin-rest/2023-10/resources/customer#resource-object + """ + + parent_stream_class = Customers + cursor_field = "id" + nested_entity = "addresses" + + +class Countries(ShopifyStream): + data_field = "countries" diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/transform.py b/airbyte-integrations/connectors/source-shopify/source_shopify/transform.py index 87b1e82b3107..496e527b09e4 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/transform.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/transform.py @@ -41,6 +41,9 @@ def _get_json_types(value_type: Any) -> List[str]: type(None): [ "null", ], + # overflow, when we need to read nested entity from the parent record, + # that has been already transformed. + Decimal: ["number"], } return json_types.get(value_type) diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py b/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py index 0ef38c922dcf..6e3a65e3a5fe 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/utils.py @@ -6,46 +6,69 @@ import enum from functools import wraps from time import sleep -from typing import Any, Dict, List, Mapping, Optional +from typing import Any, Callable, Dict, List, Mapping, Optional import requests +from airbyte_cdk.utils import AirbyteTracedException +from airbyte_protocol.models import FailureType SCOPES_MAPPING = { - "read_customers": ["Customers", "MetafieldCustomers", "CustomerSavedSearch", "CustomerAddress"], - "read_orders": [ - "Orders", - "AbandonedCheckouts", - "TenderTransactions", - "Transactions", - "Fulfillments", - "OrderRefunds", - "OrderRisks", - "MetafieldOrders", - ], - "read_draft_orders": ["DraftOrders", "MetafieldDraftOrders"], - "read_products": [ - "Products", - "ProductsGraphQl", - "MetafieldProducts", - "ProductImages", - "MetafieldProductImages", - "MetafieldProductVariants", - "CustomCollections", - "Collects", - "Collections", - "ProductVariants", - "MetafieldCollections", - "SmartCollections", - "MetafieldSmartCollections", - ], - "read_content": ["Pages", "MetafieldPages"], - "read_price_rules": ["PriceRules"], - "read_discounts": ["DiscountCodes"], - "read_locations": ["Locations", "MetafieldLocations"], - "read_inventory": ["InventoryItems", "InventoryLevels"], - "read_merchant_managed_fulfillment_orders": ["FulfillmentOrders"], - "read_shopify_payments_payouts": ["BalanceTransactions", "Disputes"], - "read_online_store_pages": ["Articles", "MetafieldArticles", "Blogs", "MetafieldBlogs"], + # SCOPE: read_customers + "Customers": ["read_customers"], + "MetafieldCustomers": ["read_customers"], + "CustomerSavedSearch": ["read_customers"], + "CustomerAddress": ["read_customers"], + # SCOPE: read_orders + "Orders": ["read_orders"], + "AbandonedCheckouts": ["read_orders"], + "TenderTransactions": ["read_orders"], + "Transactions": ["read_orders"], + "TransactionsGraphql": ["read_orders"], + "Fulfillments": ["read_orders"], + "OrderRefunds": ["read_orders"], + "OrderRisks": ["read_orders"], + "MetafieldOrders": ["read_orders"], + # SCOPE: read_draft_orders + "DraftOrders": ["read_draft_orders"], + "MetafieldDraftOrders": ["read_draft_orders"], + # SCOPE: read_products + "Products": ["read_products"], + "ProductsGraphQl": ["read_products"], + "MetafieldProducts": ["read_products"], + "ProductImages": ["read_products"], + "MetafieldProductImages": ["read_products"], + "MetafieldProductVariants": ["read_products"], + "CustomCollections": ["read_products"], + "Collects": ["read_products"], + "ProductVariants": ["read_products"], + "MetafieldCollections": ["read_products"], + "SmartCollections": ["read_products"], + "MetafieldSmartCollections": ["read_products"], + # SCOPE: read_products, read_publications + "Collections": ["read_products", "read_publications"], + # SCOPE: read_content + "Pages": ["read_content"], + "MetafieldPages": ["read_content"], + # SCOPE: read_price_rules + "PriceRules": ["read_price_rules"], + # SCOPE: read_discounts + "DiscountCodes": ["read_discounts"], + # SCOPE: read_locations + "Locations": ["read_locations"], + "MetafieldLocations": ["read_locations"], + # SCOPE: read_inventory + "InventoryItems": ["read_inventory"], + "InventoryLevels": ["read_inventory"], + # SCOPE: read_merchant_managed_fulfillment_orders + "FulfillmentOrders": ["read_merchant_managed_fulfillment_orders"], + # SCOPE: read_shopify_payments_payouts + "BalanceTransactions": ["read_shopify_payments_payouts"], + "Disputes": ["read_shopify_payments_payouts"], + # SCOPE: read_online_store_pages + "Articles": ["read_online_store_pages"], + "MetafieldArticles": ["read_online_store_pages"], + "Blogs": ["read_online_store_pages"], + "MetafieldBlogs": ["read_online_store_pages"], } @@ -63,38 +86,38 @@ def __new__(self, stream: str) -> Mapping[str, Any]: } -class ShopifyAccessScopesError(Exception): +class ShopifyAccessScopesError(AirbyteTracedException): """Raises the error if authenticated user doesn't have access to verify the grantted scopes.""" help_url = "https://shopify.dev/docs/api/usage/access-scopes#authenticated-access-scopes" - def __init__(self, response): - super().__init__( - f"Reason: Scopes are not available, make sure you're using the correct `Shopify Store` name. Actual response: {response}. More info about: {self.help_url}" - ) + def __init__(self, response, **kwargs) -> None: + self.message = f"Reason: Scopes are not available, make sure you're using the correct `Shopify Store` name. Actual response: {response}. More info about: {self.help_url}" + super().__init__(internal_message=self.message, failure_type=FailureType.config_error, **kwargs) -class ShopifyBadJsonError(ShopifyAccessScopesError): +class ShopifyBadJsonError(AirbyteTracedException): """Raises the error when Shopify replies with broken json for `access_scopes` request""" - def __init__(self, message): - super().__init__(f"Reason: Bad JSON Response from the Shopify server. Details: {message}.") + def __init__(self, message, **kwargs) -> None: + self.message = f"Reason: Bad JSON Response from the Shopify server. Details: {message}." + super().__init__(internal_message=self.message, failure_type=FailureType.config_error, **kwargs) -class ShopifyConnectionError(ShopifyAccessScopesError): - """Raises the error when Shopify replies with broken connection error for `access_scopes` request""" +class ShopifyConnectionError(AirbyteTracedException): + """Raises the error when Shopify resources couldn't be accessed because of the ConnectionError occured (100-x)""" - def __init__(self, details): - super().__init__(f"Invalid `Shopify Store` name used or `host` couldn't be verified by Shopify. Details: {details}") + def __init__(self, details, **kwargs) -> None: + self.message = f"Invalid `Shopify Store` name used or `host` couldn't be verified by Shopify. Details: {details}" + super().__init__(internal_message=self.message, failure_type=FailureType.config_error, **kwargs) -class ShopifyWrongShopNameError(Exception): +class ShopifyWrongShopNameError(AirbyteTracedException): """Raises the error when `Shopify Store` name is incorrect or couldn't be verified by the Shopify""" - def __init__(self, url): - super().__init__( - f"Reason: The `Shopify Store` name is invalid or missing for `input configuration`, make sure it's valid. Details: {url}" - ) + def __init__(self, url, **kwargs) -> None: + self.message = f"The `Shopify Store` name is invalid or missing for `input configuration`, make sure it's valid. Details: {url}" + super().__init__(internal_message=self.message, failure_type=FailureType.config_error, **kwargs) class UnrecognisedApiType(Exception): @@ -150,7 +173,7 @@ def _convert_load_to_time(load: Optional[float], threshold: float) -> float: return wait_time @staticmethod - def get_rest_api_wait_time(*args, threshold: float = 0.9, rate_limit_header: str = "X-Shopify-Shop-Api-Call-Limit"): + def get_rest_api_wait_time(*args, threshold: float = 0.9, rate_limit_header: str = "X-Shopify-Shop-Api-Call-Limit") -> float: """ To avoid reaching Shopify REST API Rate Limits, use the "X-Shopify-Shop-Api-Call-Limit" header value, to determine the current rate limits and load and handle wait_time based on load %. @@ -181,7 +204,7 @@ def get_rest_api_wait_time(*args, threshold: float = 0.9, rate_limit_header: str return wait_time @staticmethod - def get_graphql_api_wait_time(*args, threshold: float = 0.9): + def get_graphql_api_wait_time(*args, threshold: float = 0.9) -> float: """ To avoid reaching Shopify Graphql API Rate Limits, use the extensions dict in the response. @@ -229,7 +252,7 @@ def get_graphql_api_wait_time(*args, threshold: float = 0.9): return wait_time @staticmethod - def wait_time(wait_time: float): + def wait_time(wait_time: float) -> None: return sleep(wait_time) @staticmethod @@ -237,15 +260,15 @@ def balance_rate_limit( threshold: float = 0.9, rate_limit_header: str = "X-Shopify-Shop-Api-Call-Limit", api_type: ApiTypeEnum = ApiTypeEnum.rest.value, - ): + ) -> Callable[..., Any]: """ The decorator function. Adjust `threshold`, `rate_limit_header` and `api_type` if needed. """ - def decorator(func): + def decorator(func) -> Callable[..., Any]: @wraps(func) - def wrapper_balance_rate_limit(*args, **kwargs): + def wrapper_balance_rate_limit(*args, **kwargs) -> Any: if api_type == ApiTypeEnum.rest.value: ShopifyRateLimiter.wait_time( ShopifyRateLimiter.get_rest_api_wait_time(*args, threshold=threshold, rate_limit_header=rate_limit_header) @@ -285,7 +308,7 @@ def stream_state_to_tmp(*args, state_object: Dict = cached_state, **kwargs) -> D # Map the input *args, the sequece should be always keeped up to the input function # change the mapping if needed stream: object = args[0] # the self instance of the stream - current_stream_state: Dict = kwargs["stream_state"] or {} + current_stream_state: Dict = kwargs.get("stream_state") or {} # get the current tmp_state_value tmp_stream_state_value = state_object.get(stream.name, {}).get(stream.cursor_field, "") # Save the curent stream value for current sync, if present. @@ -299,9 +322,9 @@ def stream_state_to_tmp(*args, state_object: Dict = cached_state, **kwargs) -> D return state_object - def cache_stream_state(func): + def cache_stream_state(func) -> Callable[..., Any]: @wraps(func) - def decorator(*args, **kwargs): + def decorator(*args, **kwargs) -> Any: EagerlyCachedStreamState.stream_state_to_tmp(*args, **kwargs) return func(*args, **kwargs) diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py b/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py index e0c062eeff74..a36049224a01 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/conftest.py @@ -2,12 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import os +from json import dumps +from typing import Any import pytest import requests from airbyte_cdk import AirbyteLogger from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, DestinationSyncMode, SyncMode +os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" + @pytest.fixture def logger(): @@ -19,6 +24,16 @@ def basic_config(): return {"shop": "test_shop", "credentials": {"auth_method": "api_password", "api_password": "api_password"}} +@pytest.fixture +def auth_config(): + return { + "shop": "test_shop", + "start_date": "2023-01-01", + "credentials": {"auth_method": "api_password", "api_password": "api_password"}, + "authenticator": None, + } + + @pytest.fixture def catalog_with_streams(): def _catalog_with_streams(names): @@ -43,3 +58,748 @@ def response_with_bad_json(): response.status_code = 200 response._content = bad_json_str.encode("utf-8") return response + + +@pytest.fixture +def bulk_error() -> dict[str, Any]: + return { + "data": { + "bulkOperationRunQuery": { + "bulkOperation": None, + "userErrors": [ + { + "field": "some_field", + "message": "something wrong with the requested field.", + }, + ], + } + }, + "extensions": { + "cost": { + "requestedQueryCost": 10, + "actualQueryCost": 10, + "throttleStatus": { + "maximumAvailable": 1000.0, + "currentlyAvailable": 990, + "restoreRate": 50.0, + }, + } + }, + } + + +@pytest.fixture +def bulk_unknown_error() -> dict[str, Any]: + return { + "errors": [ + { + "message": "something wrong with the job", + }, + ], + "extensions": { + "cost": { + "requestedQueryCost": 10, + "actualQueryCost": 10, + "throttleStatus": { + "maximumAvailable": 1000.0, + "currentlyAvailable": 990, + "restoreRate": 50.0, + }, + } + }, + } + + +@pytest.fixture +def bulk_no_errors() -> dict[str, Any]: + return {} + + +@pytest.fixture +def bulk_error_with_concurrent_job(): + return { + "data": { + "bulkOperationRunQuery": { + "bulkOperation": None, + "userErrors": [ + { + "field": None, + "message": "", + }, + { + "field": None, + "message": "A bulk query operation for this app and shop is already in progress: gid://shopify/BulkOperation/4046676525245.", + }, + ], + } + }, + "extensions": { + "cost": { + "requestedQueryCost": 10, + "actualQueryCost": 10, + "throttleStatus": { + "maximumAvailable": 1000.0, + "currentlyAvailable": 990, + "restoreRate": 50.0, + }, + } + }, + } + + +@pytest.fixture +def bulk_successful_response(): + return { + "data": { + "bulkOperationRunQuery": { + "bulkOperation": { + "id": "gid://shopify/BulkOperation/4046733967549", + "status": "CREATED", + }, + "userErrors": [], + } + }, + "extensions": { + "cost": { + "requestedQueryCost": 10, + "actualQueryCost": 10, + "throttleStatus": { + "maximumAvailable": 1000.0, + "currentlyAvailable": 990, + "restoreRate": 50.0, + }, + } + }, + } + + +@pytest.fixture +def bulk_successful_response_with_errors(): + return { + "data": { + "node": { + "id": "gid://shopify/BulkOperation/4046733967549", + "status": "RUNNING", + }, + "bulkOperationRunQuery": { + "userErrors": [ + { + "message": "something wrong with the job", + }, + ], + }, + }, + "extensions": { + "cost": { + "requestedQueryCost": 10, + "actualQueryCost": 10, + "throttleStatus": { + "maximumAvailable": 1000.0, + "currentlyAvailable": 990, + "restoreRate": 50.0, + }, + } + }, + } + + +@pytest.fixture +def bulk_successful_response_with_no_id(): + return { + "data": { + "bulkOperationRunQuery": { + "bulkOperation": { + "status": "RUNNING", + }, + "userErrors": [], + } + }, + "extensions": { + "cost": { + "requestedQueryCost": 10, + "actualQueryCost": 10, + "throttleStatus": { + "maximumAvailable": 1000.0, + "currentlyAvailable": 990, + "restoreRate": 50.0, + }, + } + }, + } + + +@pytest.fixture +def bulk_successful_completed_response(): + return { + "data": { + "bulkOperationRunQuery": { + "bulkOperation": { + "id": "gid://shopify/BulkOperation/4046733967549", + "status": "CREATED", + "url": '"https://some_url/response-content-disposition=attachment;+filename="bulk-123456789.jsonl";+filename*=UTF-8' + 'bulk-4047416819901.jsonl&response-content-type=application/jsonl"', + }, + "userErrors": [], + } + }, + "extensions": { + "cost": { + "requestedQueryCost": 10, + "actualQueryCost": 10, + "throttleStatus": { + "maximumAvailable": 1000.0, + "currentlyAvailable": 990, + "restoreRate": 50.0, + }, + } + }, + } + + +@pytest.fixture +def bulk_job_created_response(): + return { + "data": { + "bulkOperationRunQuery": { + "bulkOperation": { + "id": "gid://shopify/BulkOperation/4046733967549", + "status": "CREATED", + }, + "userErrors": [], + } + }, + "extensions": { + "cost": { + "requestedQueryCost": 10, + "actualQueryCost": 10, + "throttleStatus": { + "maximumAvailable": 1000.0, + "currentlyAvailable": 990, + "restoreRate": 50.0, + }, + } + }, + } + + +@pytest.fixture +def bulk_job_completed_response(): + return { + "data": { + "node": { + "id": "gid://shopify/BulkOperation/4047052112061", + "status": "COMPLETED", + "errorCode": None, + "objectCount": "0", + "fileSize": None, + "url": 'https://some_url?response-content-disposition=attachment;+filename="bulk-123456789.jsonl";+filename*=UTF-8' + "bulk-123456789.jsonl&response-content-type=application/jsonl", + "partialDataUrl": None, + } + }, + "extensions": { + "cost": { + "requestedQueryCost": 1, + "actualQueryCost": 1, + "throttleStatus": { + "maximumAvailable": 1000.0, + "currentlyAvailable": 999, + "restoreRate": 50.0, + }, + } + }, + } + + +@pytest.fixture +def bulk_job_failed_response(): + return { + "data": { + "node": { + "id": "gid://shopify/BulkOperation/4047052112061", + "status": "FAILED", + "errorCode": None, + "objectCount": "0", + "fileSize": None, + "url": None, + "partialDataUrl": None, + } + }, + "extensions": { + "cost": { + "requestedQueryCost": 1, + "actualQueryCost": 1, + "throttleStatus": { + "maximumAvailable": 1000.0, + "currentlyAvailable": 999, + "restoreRate": 50.0, + }, + } + }, + } + + +@pytest.fixture +def bulk_job_timeout_response(): + return { + "data": { + "node": { + "id": "gid://shopify/BulkOperation/4047052112061", + "status": "TIMEOUT", + "errorCode": None, + "objectCount": "0", + "fileSize": None, + "url": None, + "partialDataUrl": None, + } + }, + "extensions": { + "cost": { + "requestedQueryCost": 1, + "actualQueryCost": 1, + "throttleStatus": { + "maximumAvailable": 1000.0, + "currentlyAvailable": 999, + "restoreRate": 50.0, + }, + } + }, + } + + +@pytest.fixture +def bulk_job_running_response(): + return { + "data": { + "node": { + "id": "gid://shopify/BulkOperation/4047052112061", + "status": "RUNNING", + "errorCode": None, + "objectCount": "0", + "fileSize": None, + "url": None, + "partialDataUrl": None, + } + }, + "extensions": { + "cost": { + "requestedQueryCost": 1, + "actualQueryCost": 1, + "throttleStatus": { + "maximumAvailable": 1000.0, + "currentlyAvailable": 999, + "restoreRate": 50.0, + }, + } + }, + } + + +@pytest.fixture +def bulk_job_running_response_without_id(): + return { + "data": { + "node": { + # "id": "gid://shopify/BulkOperation/4047052112061", + "status": "RUNNING", + "errorCode": None, + "objectCount": "0", + "fileSize": None, + "url": None, + "partialDataUrl": None, + } + }, + "extensions": { + "cost": { + "requestedQueryCost": 1, + "actualQueryCost": 1, + "throttleStatus": { + "maximumAvailable": 1000.0, + "currentlyAvailable": 999, + "restoreRate": 50.0, + }, + } + }, + } + + +@pytest.fixture +def bulk_job_access_denied_response(): + return { + "data": { + "node": { + "id": "gid://shopify/BulkOperation/4047052112061", + "status": "ACCESS_DENIED", + "errorCode": None, + "objectCount": "0", + "fileSize": None, + "url": None, + "partialDataUrl": None, + } + }, + "extensions": { + "cost": { + "requestedQueryCost": 1, + "actualQueryCost": 1, + "throttleStatus": { + "maximumAvailable": 1000.0, + "currentlyAvailable": 999, + "restoreRate": 50.0, + }, + } + }, + } + + +@pytest.fixture +def bulk_job_unknown_status_response(): + return { + "data": { + "node": { + "id": "gid://shopify/BulkOperation/4047052112061", + "status": None, + "errorCode": None, + "objectCount": "0", + "fileSize": None, + "url": None, + "partialDataUrl": None, + } + }, + "extensions": { + "cost": { + "requestedQueryCost": 1, + "actualQueryCost": 1, + "throttleStatus": { + "maximumAvailable": 1000.0, + "currentlyAvailable": 999, + "restoreRate": 50.0, + }, + } + }, + } + + +@pytest.fixture +def metafield_jsonl_content_example(): + return ( + dumps( + { + "__typename": "Metafield", + "id": "gid://shopify/Metafield/123", + "__parentId": "gid://shopify/Order/1234567", + "createdAt": "2023-01-01T01:01:01Z", + "updatedAt": "2023-01-01T01:01:01Z", + } + ) + + "\n" + ) + + +@pytest.fixture +def filfillment_order_jsonl_content_example(): + return """{"__typename":"Order","id":"gid:\/\/shopify\/Order\/1"} +{"__typename":"FulfillmentOrder","id":"gid:\/\/shopify\/FulfillmentOrder\/2","assignedLocation":{"address1":"Test","address2":null,"city":"Test","countryCode":"Test","name":"Test","phone":"","province":null,"zip":"00000","location":{"locationId":"gid:\/\/shopify\/Location\/123"}},"destination":{"id":"gid:\/\/shopify\/Destination\/777"},"deliveryMethod":{"id":"gid:\/\/shopify\/DeliveryMethod\/123","methodType":"SHIPPING","minDeliveryDateTime":"2023-04-13T12:00:00Z","maxDeliveryDateTime":"2023-04-13T12:00:00Z"},"fulfillAt":"2023-04-13T12:00:00Z","fulfillBy":null,"internationalDuties":null,"fulfillmentHolds":[{}],"createdAt":"2023-04-13T12:09:45Z","updatedAt":"2023-04-13T12:09:46Z","requestStatus":"UNSUBMITTED","status":"CLOSED","supportedActions":[{}],"__parentId":"gid:\/\/shopify\/Order\/1"} +{"__typename":"FulfillmentOrderLineItem","id":"gid:\/\/shopify\/FulfillmentOrderLineItem\/3","inventoryItemId":"gid:\/\/shopify\/InventoryItem\/33","lineItem":{"lineItemId":"gid:\/\/shopify\/LineItem\/31","fulfillableQuantity":0,"quantity":1,"variant":{"variantId":"gid:\/\/shopify\/ProductVariant\/333"}},"__parentId":"gid:\/\/shopify\/FulfillmentOrder\/2"} +{"__typename":"FulfillmentOrderMerchantRequest","id":"gid:\/\/shopify\/FulfillmentOrderMerchantRequest\/333","message":null,"kind":"FULFILLMENT_REQUEST","requestOptions":{"notify_customer":true},"__parentId":"gid:\/\/shopify\/FulfillmentOrder\/2"}\n""" + + +@pytest.fixture +def inventory_items_jsonl_content_example(): + return """{"__typename":"InventoryItem","id":"gid:\/\/shopify\/InventoryItem\/44871665713341","unitCost":null,"countryCodeOfOrigin":null,"harmonizedSystemCode":null,"provinceCodeOfOrigin":null,"updatedAt":"2023-04-14T10:29:27Z","createdAt":"2023-04-14T10:29:27Z","sku":"","tracked":true,"requiresShipping":false} +{"__typename":"InventoryItem","id":"gid:\/\/shopify\/InventoryItem\/45419395743933","unitCost":{"cost":"29.0"},"countryCodeOfOrigin":"UA","harmonizedSystemCode":"330510","provinceCodeOfOrigin":null,"updatedAt":"2023-12-11T10:37:41Z","createdAt":"2023-12-11T10:37:41Z","sku":"123","tracked":true,"requiresShipping":true}\n""" + + +@pytest.fixture +def inventory_levels_jsonl_content_example(): + return """{"__typename":"Location","id":"gid:\/\/shopify\/Location\/63590301885"} +{"__typename":"InventoryLevel","id":"gid:\/\/shopify\/InventoryLevel\/97912455357?inventory_item_id=42185200631997","available":15,"item":{"inventory_item_id":"gid:\/\/shopify\/InventoryItem\/42185200631997"},"updatedAt":"2023-04-13T12:00:55Z","__parentId":"gid:\/\/shopify\/Location\/63590301885"} +{"__typename":"InventoryLevel","id":"gid:\/\/shopify\/InventoryLevel\/97912455357?inventory_item_id=42185218719933","available":8,"item":{"inventory_item_id":"gid:\/\/shopify\/InventoryItem\/42185218719933"},"updatedAt":"2023-04-13T12:09:45Z","__parentId":"gid:\/\/shopify\/Location\/63590301885"}\n""" + + +@pytest.fixture +def discount_codes_jsonl_content_example(): + return """{"__typename":"DiscountCodeNode","id":"gid:\/\/shopify\/DiscountCodeNode\/945205379261","codeDiscount":{"updatedAt":"2023-12-07T11:40:44Z","createdAt":"2021-07-08T12:40:37Z","summary":"Free shipping on all products • Minimum purchase of $1.00 • For all countries","discountType":"SHIPPING"}} +{"__typename":"DiscountRedeemCode","usageCount":0,"code":"TEST","id":"gid:\/\/shopify\/DiscountRedeemCode\/11545139282109","__parentId":"gid:\/\/shopify\/DiscountCodeNode\/945205379261"} +{"__typename":"DiscountRedeemCode","usageCount":0,"code":"TEST2","id":"gid:\/\/shopify\/DiscountRedeemCode\/13175793582269","__parentId":"gid:\/\/shopify\/DiscountCodeNode\/945205379261"}\n""" + + +@pytest.fixture +def collections_jsonl_content_example(): + return """{"__typename":"Collection","id":"gid:\/\/shopify\/Collection\/270889287869","handle":"frontpage","title":"Home page","updatedAt":"2023-09-05T14:06:59Z","bodyHtml":"updated_mon_24.04.2023","sortOrder":"BEST_SELLING","templateSuffix":"","productsCount":1} +{"__typename":"CollectionPublication","publishedAt":"2021-06-23T01:00:25Z","__parentId":"gid:\/\/shopify\/Collection\/270889287869"} +{"__typename":"CollectionPublication","publishedAt":"2021-08-18T09:39:34Z","__parentId":"gid:\/\/shopify\/Collection\/270889287869"} +{"__typename":"CollectionPublication","publishedAt":"2023-04-20T11:12:24Z","__parentId":"gid:\/\/shopify\/Collection\/270889287869"} +{"__typename":"Collection","id":"gid:\/\/shopify\/Collection\/273278566589","handle":"test-collection","title":"Test Collection","updatedAt":"2023-09-05T14:12:04Z","bodyHtml":"updated_mon_24.04.2023","sortOrder":"BEST_SELLING","templateSuffix":"","productsCount":26} +{"__typename":"CollectionPublication","publishedAt":"2021-07-19T14:02:54Z","__parentId":"gid:\/\/shopify\/Collection\/273278566589"} +{"__typename":"CollectionPublication","publishedAt":"2021-08-18T09:39:34Z","__parentId":"gid:\/\/shopify\/Collection\/273278566589"} +{"__typename":"CollectionPublication","publishedAt":"2023-04-20T11:12:24Z","__parentId":"gid:\/\/shopify\/Collection\/273278566589"}\n""" + + +@pytest.fixture +def transactions_jsonl_content_example(): + return ( + dumps( + { + "__typename": "Order", + "id": "gid://shopify/Order/1", + "currency": "USD", + "transactions": [ + { + "id": "gid://shopify/OrderTransaction/1", + "errorCode": None, + "parentTransaction": {"parentId": "gid://shopify/ParentOrderTransaction/0"}, + "test": True, + "kind": "SALE", + "amount": "102.00", + "receipt": '{"paid_amount":"102.00"}', + "gateway": "test", + "authorization": "1234", + "createdAt": "2030-07-02T07:51:49Z", + "status": "SUCCESS", + "processedAt": "2030-07-02T07:51:49Z", + "totalUnsettledSet": { + "presentmentMoney": {"amount": "0.0", "currency": "USD"}, + "shopMoney": {"amount": "0.0", "currency": "USD"}, + }, + "paymentId": "some_payment_id.1", + "paymentDetails": { + "avsResultCode": None, + "cvvResultCode": None, + "creditCardBin": "1", + "creditCardCompany": "Test", + "creditCardNumber": "•••• •••• •••• 1", + "creditCardName": "Test Gateway", + "creditCardWallet": None, + "creditCardExpirationYear": 2023, + "creditCardExpirationMonth": 11, + }, + } + ], + } + ) + + "\n" + ) + + +@pytest.fixture +def metafield_parse_response_expected_result(): + return { + "id": 123, + "admin_graphql_api_id": "gid://shopify/Metafield/123", + "owner_id": 1234567, + "owner_resource": "order", + "shop_url": "test_shop", + "created_at": "2023-01-01T01:01:01+00:00", + "updated_at": "2023-01-01T01:01:01+00:00", + } + + +@pytest.fixture +def fulfillment_orders_response_expected_result(): + return { + "id": 2, + "assigned_location": { + "address1": "Test", + "address2": None, + "city": "Test", + "country_code": "Test", + "name": "Test", + "phone": "", + "province": None, + "zip": "00000", + "location_id": 123, + }, + "destination": { + "id": 777, + }, + "delivery_method": { + "id": 123, + "method_type": "SHIPPING", + "min_delivery_date_time": "2023-04-13T12:00:00+00:00", + "max_delivery_date_time": "2023-04-13T12:00:00+00:00", + }, + "fulfill_at": "2023-04-13T12:00:00+00:00", + "fulfill_by": None, + "international_duties": None, + "fulfillment_holds": [], + "created_at": "2023-04-13T12:09:45+00:00", + "updated_at": "2023-04-13T12:09:46+00:00", + "request_status": "UNSUBMITTED", + "status": "CLOSED", + "supported_actions": [], + "shop_id": None, + "order_id": 1, + "assigned_location_id": 123, + "line_items": [ + { + "id": 3, + "inventory_item_id": 33, + "shop_id": None, + "fulfillment_order_id": 2, + "quantity": 1, + "line_item_id": 31, + "fulfillable_quantity": 0, + "variant_id": 333, + }, + ], + "merchant_requests": [{"id": 333, "message": None, "kind": "FULFILLMENT_REQUEST", "request_options": {"notify_customer": True}}], + "admin_graphql_api_id": "gid://shopify/FulfillmentOrder/2", + "shop_url": "test_shop", + } + + +@pytest.fixture +def inventory_items_response_expected_result(): + return [ + { + "id": 44871665713341, + "country_code_of_origin": None, + "harmonized_system_code": None, + "province_code_of_origin": None, + "updated_at": "2023-04-14T10:29:27+00:00", + "created_at": "2023-04-14T10:29:27+00:00", + "sku": "", + "tracked": True, + "requires_shipping": False, + "admin_graphql_api_id": "gid://shopify/InventoryItem/44871665713341", + "cost": None, + "country_harmonized_system_codes": [], + "shop_url": "test_shop", + }, + { + "id": 45419395743933, + "country_code_of_origin": "UA", + "harmonized_system_code": "330510", + "province_code_of_origin": None, + "updated_at": "2023-12-11T10:37:41+00:00", + "created_at": "2023-12-11T10:37:41+00:00", + "sku": "123", + "tracked": True, + "requires_shipping": True, + "admin_graphql_api_id": "gid://shopify/InventoryItem/45419395743933", + "cost": 29.0, + "country_harmonized_system_codes": [], + "shop_url": "test_shop", + }, + ] + + +@pytest.fixture +def inventory_levels_response_expected_result(): + return [ + { + "id": "63590301885|42185200631997", + "available": 15, + "updated_at": "2023-04-13T12:00:55+00:00", + "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=42185200631997", + "inventory_item_id": 42185200631997, + "location_id": 63590301885, + "shop_url": "test_shop", + }, + { + "id": "63590301885|42185218719933", + "available": 8, + "updated_at": "2023-04-13T12:09:45+00:00", + "admin_graphql_api_id": "gid://shopify/InventoryLevel/97912455357?inventory_item_id=42185218719933", + "inventory_item_id": 42185218719933, + "location_id": 63590301885, + "shop_url": "test_shop", + }, + ] + + +@pytest.fixture +def discount_codes_response_expected_result(): + return [ + { + "usage_count": 0, + "code": "TEST", + "id": 11545139282109, + "admin_graphql_api_id": "gid://shopify/DiscountRedeemCode/11545139282109", + "price_rule_id": 945205379261, + "updated_at": "2023-12-07T11:40:44+00:00", + "created_at": "2021-07-08T12:40:37+00:00", + "summary": "Free shipping on all products • Minimum purchase of $1.00 • For all countries", + "discount_type": "SHIPPING", + "shop_url": "test_shop", + }, + { + "usage_count": 0, + "code": "TEST2", + "id": 13175793582269, + "admin_graphql_api_id": "gid://shopify/DiscountRedeemCode/13175793582269", + "price_rule_id": 945205379261, + "updated_at": "2023-12-07T11:40:44+00:00", + "created_at": "2021-07-08T12:40:37+00:00", + "summary": "Free shipping on all products • Minimum purchase of $1.00 • For all countries", + "discount_type": "SHIPPING", + "shop_url": "test_shop", + }, + ] + + +@pytest.fixture +def collections_response_expected_result(): + return [ + { + "id": 270889287869, + "handle": "frontpage", + "title": "Home page", + "updated_at": "2023-09-05T14:06:59+00:00", + "body_html": "updated_mon_24.04.2023", + "sort_order": "BEST_SELLING", + "template_suffix": "", + "products_count": 1, + "admin_graphql_api_id": "gid://shopify/Collection/270889287869", + "published_at": "2021-06-23T01:00:25+00:00", + "shop_url": "test_shop", + }, + { + "id": 273278566589, + "handle": "test-collection", + "title": "Test Collection", + "updated_at": "2023-09-05T14:12:04+00:00", + "body_html": "updated_mon_24.04.2023", + "sort_order": "BEST_SELLING", + "template_suffix": "", + "products_count": 26, + "admin_graphql_api_id": "gid://shopify/Collection/273278566589", + "published_at": "2021-07-19T14:02:54+00:00", + "shop_url": "test_shop", + }, + ] + + +@pytest.fixture +def transactions_response_expected_result(): + return { + "id": 1, + "error_code": None, + "test": True, + "kind": "SALE", + "amount": 102.0, + "receipt": '{"paid_amount":"102.00"}', + "gateway": "test", + "authorization": "1234", + "created_at": "2030-07-02T07:51:49+00:00", + "status": "SUCCESS", + "processed_at": "2030-07-02T07:51:49+00:00", + "total_unsettled_set": {"presentment_money": {"amount": 0.0, "currency": "USD"}, "shop_money": {"amount": 0.0, "currency": "USD"}}, + "payment_id": "some_payment_id.1", + "payment_details": { + "avs_result_code": None, + "cvv_result_code": None, + "credit_card_bin": "1", + "credit_card_company": "Test", + "credit_card_number": "•••• •••• •••• 1", + "credit_card_name": "Test Gateway", + "credit_card_wallet": None, + "credit_card_expiration_year": 2023, + "credit_card_expiration_month": 11, + }, + "order_id": 1, + "currency": "USD", + "admin_graphql_api_id": "gid://shopify/OrderTransaction/1", + "parent_id": 0, + "shop_url": "test_shop", + } diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_job.py b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_job.py new file mode 100644 index 000000000000..368936de39a3 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_job.py @@ -0,0 +1,271 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest +import requests +from source_shopify.shopify_graphql.bulk.exceptions import ShopifyBulkExceptions +from source_shopify.shopify_graphql.bulk.job import ShopifyBulkStatus +from source_shopify.streams.streams import ( + Collections, + DiscountCodes, + FulfillmentOrders, + InventoryItems, + InventoryLevels, + MetafieldOrders, + TransactionsGraphql, +) + + +@pytest.mark.parametrize( + "bulk_job_response, expected_len", + [ + ("bulk_error", 1), + ("bulk_unknown_error", 1), + ("bulk_no_errors", 0), + ], +) +def test_check_for_errors(request, requests_mock, bulk_job_response, expected_len, auth_config) -> None: + stream = MetafieldOrders(auth_config) + requests_mock.get(stream.job_manager.base_url, json=request.getfixturevalue(bulk_job_response)) + test_response = requests.get(stream.job_manager.base_url) + test_errors = stream.job_manager.check_for_errors(test_response) + assert len(test_errors) == expected_len + + +def test_get_errors_from_response_invalid_response(auth_config) -> None: + expected = "Couldn't check the `response` for `errors`" + stream = MetafieldOrders(auth_config) + response = requests.Response() + response.status_code = 404 + response.url = "https://example.com/invalid" + with pytest.raises(ShopifyBulkExceptions.BulkJobBadResponse) as error: + stream.job_manager.check_for_errors(response) + assert expected in repr(error.value) + + +@pytest.mark.parametrize( + "bulk_job_response, expected", + [ + ("bulk_error_with_concurrent_job", True), + ("bulk_successful_response", False), + ("bulk_error", False), + ], +) +def test_has_running_concurrent_job(request, requests_mock, bulk_job_response, auth_config, expected) -> None: + stream = MetafieldOrders(auth_config) + requests_mock.get(stream.job_manager.base_url, json=request.getfixturevalue(bulk_job_response)) + test_response = requests.get(stream.job_manager.base_url) + test_errors = stream.job_manager.check_for_errors(test_response) + assert stream.job_manager.has_running_concurrent_job(test_errors) == expected + + +@pytest.mark.parametrize( + "bulk_job_response, expected", + [ + ("bulk_successful_response", "gid://shopify/BulkOperation/4046733967549"), + ("bulk_error", None), + ("bulk_successful_response_with_no_id", None), + ], +) +def test_job_get_id(request, requests_mock, bulk_job_response, auth_config, expected) -> None: + stream = MetafieldOrders(auth_config) + requests_mock.get(stream.job_manager.base_url, json=request.getfixturevalue(bulk_job_response)) + test_response = requests.get(stream.job_manager.base_url) + assert stream.job_manager.job_get_id(test_response) == expected + + +@pytest.mark.parametrize( + "bulk_job_response, error_type, expected", + [ + ("bulk_successful_response", None, "gid://shopify/BulkOperation/4046733967549"), + ("bulk_error_with_concurrent_job", None, None), + ], +) +def test_job_create(request, requests_mock, bulk_job_response, auth_config, error_type, expected) -> None: + stream = MetafieldOrders(auth_config) + # patching concurent settings + stream.job_manager.concurrent_max_retry = 1 # 1 attempt max + stream.job_manager.concurrent_interval_sec = 1 # 1 sec + + requests_mock.get(stream.job_manager.base_url, json=request.getfixturevalue(bulk_job_response)) + result = stream.job_manager._job_create(requests.get(stream.job_manager.base_url).request) + assert stream.job_manager.job_get_id(result) == expected + +def test_job_state_completed(auth_config) -> None: + stream = MetafieldOrders(auth_config) + stream.job_manager.job_state = ShopifyBulkStatus.COMPLETED.value + assert stream.job_manager.completed() == True + +def test_job_state_running(auth_config) -> None: + stream = MetafieldOrders(auth_config) + stream.job_manager.job_state = ShopifyBulkStatus.RUNNING.value + assert stream.job_manager.running() == True + +def test_job_state_failed(auth_config) -> None: + stream = MetafieldOrders(auth_config) + stream.job_manager.job_state = ShopifyBulkStatus.FAILED.value + assert stream.job_manager.failed() == True + +def test_job_state_timeout(auth_config) -> None: + stream = MetafieldOrders(auth_config) + stream.job_manager.job_state = ShopifyBulkStatus.TIMEOUT.value + assert stream.job_manager.timeout() == True + +def test_job_state_access_denied(auth_config) -> None: + stream = MetafieldOrders(auth_config) + stream.job_manager.job_state = ShopifyBulkStatus.ACCESS_DENIED.value + assert stream.job_manager.access_denied() == True + +@pytest.mark.parametrize( + "bulk_job_response, concurrent_max_retry, error_type, expected", + [ + # method should return this response fixture, once retried. + ("bulk_successful_completed_response", 2, None, "gid://shopify/BulkOperation/4046733967549"), + # method should raise AirbyteTracebackException, because the concurrent BULK Job is in progress + ( + "bulk_error_with_concurrent_job", + 1, + ShopifyBulkExceptions.BulkJobConcurrentError, + "The BULK Job couldn't be created at this time, since another job is running", + ), + ], + ids=[ + "regular concurrent request", + "max atttempt reached", + ] +) +def test_job_retry_on_concurrency(request, requests_mock, bulk_job_response, concurrent_max_retry, error_type, auth_config, expected) -> None: + stream = MetafieldOrders(auth_config) + # patching concurent settings + stream.job_manager.concurrent_max_retry = concurrent_max_retry + stream.job_manager.concurrent_interval_sec = 1 + requests_mock.get(stream.job_manager.base_url, json=request.getfixturevalue(bulk_job_response)) + if error_type: + with pytest.raises(error_type) as error: + stream.job_manager.job_retry_on_concurrency(requests.get(stream.job_manager.base_url).request) + assert expected in repr(error.value) + else: + result = stream.job_manager.job_retry_on_concurrency(requests.get(stream.job_manager.base_url).request) + assert stream.job_manager.job_get_id(result) == expected + + + +@pytest.mark.parametrize( + "job_response, error_type, patch_healthcheck, is_test, expected", + [ + ( + "bulk_job_completed_response", + None, + False, + False, + "bulk-123456789.jsonl", + ), + ("bulk_job_failed_response", ShopifyBulkExceptions.BulkJobFailed, False, False, "exited with FAILED"), + ("bulk_job_timeout_response", ShopifyBulkExceptions.BulkJobTimout, False, False, "exited with TIMEOUT"), + ("bulk_job_access_denied_response", ShopifyBulkExceptions.BulkJobAccessDenied, False, False, "exited with ACCESS_DENIED"), + ("bulk_successful_response_with_errors", ShopifyBulkExceptions.BulkJobUnknownError, True, False, "Could not validate the status of the BULK Job"), + # is_test should be set to `True` to exit from the while loop in `job_check_status()` + ("bulk_job_running_response", None, False, True, None), + ("bulk_job_running_response_without_id", None, False, True, None), + ], + ids=[ + "completed", + "failed", + "timeout", + "access_denied", + "success with errors (edge)", + "running", + "running_no_id (edge)", + ], +) +def test_job_check(mocker, request, requests_mock, job_response, auth_config, error_type, patch_healthcheck, is_test, expected) -> None: + stream = MetafieldOrders(auth_config) + # modify the sleep time for the test + stream.job_manager.concurrent_max_retry = 1 + stream.job_manager.concurrent_interval_sec = 1 + stream.job_manager.job_check_interval_sec = 1 + is_test = is_test if is_test else False + # get job_id from FIXTURE + job_id = request.getfixturevalue(job_response).get("data", {}).get("node", {}).get("id") + # patching the method to get the right ID checks + if job_id: + mocker.patch("source_shopify.shopify_graphql.bulk.job.ShopifyBulkManager.job_get_id", value=job_id) + if patch_healthcheck: + mocker.patch("source_shopify.shopify_graphql.bulk.job.ShopifyBulkManager.job_healthcheck", value=job_response) + # mocking the response for STATUS CHECKS + requests_mock.post(stream.job_manager.base_url, json=request.getfixturevalue(job_response)) + test_job_status_response = requests.post(stream.job_manager.base_url) + job_result_url = test_job_status_response.json().get("data", {}).get("node", {}).get("url") + if error_type: + with pytest.raises(error_type) as error: + stream.job_manager.job_check(test_job_status_response, is_test) + assert expected in repr(error.value) + else: + if job_result_url: + # mocking the nested request call to retrieve the data from result URL + requests_mock.get(job_result_url, json=request.getfixturevalue(job_response)) + result = stream.job_manager.job_check(test_job_status_response, is_test) + assert expected == result + + +def test_job_read_file_invalid_filename(mocker, auth_config) -> None: + stream = MetafieldOrders(auth_config) + expected = "An error occured while producing records from BULK Job result" + # patching the method to get the filename + # mocker.patch("source_shopify.shopify_graphql.bulk.job.ShopifyBulkManager.retrieve_result", value="test.jsonl") + mocker.patch("source_shopify.shopify_graphql.bulk.record.ShopifyBulkRecord.produce_records", side_effect=Exception) + with pytest.raises(ShopifyBulkExceptions.BulkRecordProduceError) as error: + list(stream.record_producer.read_file("test.jsonl")) + # print(repr(error.value)) + assert expected in repr(error.value) + + +@pytest.mark.parametrize( + "stream, json_content_example, expected", + [ + (MetafieldOrders, "metafield_jsonl_content_example", "metafield_parse_response_expected_result"), + (FulfillmentOrders, "filfillment_order_jsonl_content_example", "fulfillment_orders_response_expected_result"), + (DiscountCodes, "discount_codes_jsonl_content_example", "discount_codes_response_expected_result"), + (Collections, "collections_jsonl_content_example", "collections_response_expected_result"), + (TransactionsGraphql, "transactions_jsonl_content_example", "transactions_response_expected_result"), + (InventoryItems, "inventory_items_jsonl_content_example", "inventory_items_response_expected_result"), + (InventoryLevels, "inventory_levels_jsonl_content_example", "inventory_levels_response_expected_result"), + ], + ids=[ + "MetafieldOrders", + "FulfillmentOrders", + "DiscountCodes", + "Collections", + "TransactionsGraphql", + "InventoryItems", + "InventoryLevels", + ], +) +def test_bulk_stream_parse_response( + # mocker, + request, + requests_mock, + bulk_job_completed_response, + stream, + json_content_example, + expected, + auth_config, +) -> None: + stream = stream(auth_config) + # get the mocked job_result_url + test_result_url = bulk_job_completed_response.get("data").get("node").get("url") + # mocking the result url with jsonl content + requests_mock.post(stream.job_manager.base_url, json=bulk_job_completed_response) + # getting mock response + test_bulk_response: requests.Response = requests.post(stream.job_manager.base_url) + # mocking nested api call to get data from result url + requests_mock.get(test_result_url, text=request.getfixturevalue(json_content_example)) + # parsing result from completed job + test_records = list(stream.parse_response(test_bulk_response)) + expected_result = request.getfixturevalue(expected) + if isinstance(expected_result, dict): + assert test_records == [expected_result] + elif isinstance(expected_result, list): + assert test_records == expected_result diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py new file mode 100644 index 000000000000..9b9bb8da0873 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_query.py @@ -0,0 +1,209 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest +from graphql_query import Argument, Field, Operation, Query +from source_shopify.shopify_graphql.bulk.query import ( + InventoryLevel, + MetafieldCustomer, + MetafieldProductImage, + ShopifyBulkQuery, + ShopifyBulkTemplates, +) + + +def test_query_status() -> None: + expected = """query { + node(id: "gid://shopify/BulkOperation/4047052112061") { + ... on BulkOperation { + id + status + errorCode + objectCount + fileSize + url + partialDataUrl + } + } + }""" + + input_job_id = "gid://shopify/BulkOperation/4047052112061" + template = ShopifyBulkTemplates.status(input_job_id) + assert repr(template) == repr(expected) + + +def test_bulk_query_prepare() -> None: + expected = '''mutation { + bulkOperationRunQuery( + query: """ + {some_query} + """ + ) { + bulkOperation { + id + status + } + userErrors { + field + message + } + } + }''' + + input_query_from_slice = "{some_query}" + template = ShopifyBulkTemplates.prepare(input_query_from_slice) + assert repr(template) == repr(expected) + + +@pytest.mark.parametrize( + "query_name, fields, filter_field, start, end, expected", + [ + ( + "test_root", + ["test_field1", "test_field2"], + "updated_at", + "2023-01-01", + "2023-01-02", + Query( + name='test_root', + arguments=[ + Argument(name="query", value=f"\"updated_at:>'2023-01-01' AND updated_at:<='2023-01-02'\""), + ], + fields=[Field(name='edges', fields=[Field(name='node', fields=["test_field1", "test_field2"])])] + ) + ) + ], + ids=["simple query with filter and sort"] +) +def test_base_build_query(query_name, fields, filter_field, start, end, expected) -> None: + """ + Expected result rendered: + ''' + { + test_root(query: "updated_at:>'2023-01-01' AND updated_at:<='2023-01-02'") { + edges { + node { + id + test_field1 + test_field2 + } + } + } + ''' + """ + + + builder = ShopifyBulkQuery(shop_id=0) + filter_query = f"{filter_field}:>'{start}' AND {filter_field}:<='{end}'" + built_query = builder.build(query_name, fields, filter_query) + assert expected.render() == built_query.render() + + +@pytest.mark.parametrize( + "query_class, filter_field, start, end, expected", + [ + ( + MetafieldCustomer, + "updated_at", + "2023-01-01", + "2023-01-02", + Operation( + type="", + queries=[ + Query( + name='customers', + arguments=[ + Argument(name="query", value=f"\"updated_at:>='2023-01-01' AND updated_at:<='2023-01-02'\""), + Argument(name="sortKey", value="UPDATED_AT"), + ], + fields=[Field(name='edges', fields=[Field(name='node', fields=['__typename', 'id', Field(name="metafields", fields=[Field(name="edges", fields=[Field(name="node", fields=["__typename", "id", "namespace", "value", "key", "description", "createdAt", "updatedAt", "type"])])])])])] + ) + ] + ), + ), + ( + MetafieldProductImage, + "updated_at", + "2023-01-01", + "2023-01-02", + Operation( + type="", + queries=[ + Query( + name='products', + arguments=[ + Argument(name="query", value=f"\"updated_at:>='2023-01-01' AND updated_at:<='2023-01-02'\""), + Argument(name="sortKey", value="UPDATED_AT"), + ], + fields=[Field(name='edges', fields=[Field(name='node', fields=['__typename','id',Field(name="images", fields=[Field(name="edges", fields=[Field(name="node", fields=["__typename", "id", Field(name="metafields", fields=[Field(name="edges", fields=[Field(name="node", fields=["__typename", "id", "namespace", "value", "key", "description", "createdAt", "updatedAt", "type"])])])])])])])])] + ) + ] + ), + ), + ( + InventoryLevel, + "updated_at", + "2023-01-01", + "2023-01-02", + Operation( + type="", + queries=[ + Query( + name='locations', + arguments=[ + Argument(name="includeLegacy", value="true"), + Argument(name="includeInactive", value="true"), + ], + fields=[ + Field( + name='edges', + fields=[ + Field( + name='node', + fields=[ + '__typename', + 'id', + Query( + name="inventoryLevels", + arguments=[ + Argument(name="query", value=f"\"updated_at:>='2023-01-01' AND updated_at:<='2023-01-02'\""), + ], + fields=[ + Field( + name="edges", + fields=[ + Field( + name="node", + fields=[ + "__typename", + "id", + Field(name="available"), + Field(name="item", fields=[Field(name="id", alias="inventory_item_id")]), + Field(name="updatedAt") + ] + ) + ] + ) + ] + ) + ] + ) + ] + ) + ] + ) + ] + ), + ), + ], + ids=[ + "MetafieldCustomers query with 1 query_path(str)", + "MetafieldProductImages query with composite quey_path(List[2])", + "InventoryLevel query", + ] +) +def test_bulk_query(query_class, filter_field, start, end, expected) -> None: + stream = query_class(shop_id=0) + assert stream.get(filter_field, start, end) == expected.render() \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_record.py b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_record.py new file mode 100644 index 000000000000..ee0be4211c26 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_record.py @@ -0,0 +1,198 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + + +import pytest +from source_shopify.shopify_graphql.bulk.query import ShopifyBulkQuery +from source_shopify.shopify_graphql.bulk.record import ShopifyBulkRecord + + +@pytest.mark.parametrize( + "record, expected", + [ + ( + {"id": "gid://shopify/Order/19435458986123"}, + {"id": 19435458986123, "admin_graphql_api_id": "gid://shopify/Order/19435458986123"}, + ), + ({"id": 123}, {"id": 123}), + ], +) +def test_record_resolve_id(record, expected, logger) -> None: + bulk_query = ShopifyBulkQuery(shop_id=0) + assert ShopifyBulkRecord(bulk_query, logger).record_resolve_id(record) == expected + + +@pytest.mark.parametrize( + "record, types, expected", + [ + ({"__typename": "Order", "id": "gid://shopify/Order/19435458986123"}, ["Test", "Order"], True), + ({"__typename": "Test", "id": "gid://shopify/Order/19435458986123"}, "Other", False), + ({}, "Other", False), + ], +) +def test_check_type(record, types, expected, logger) -> None: + query = ShopifyBulkQuery(shop_id=0) + assert ShopifyBulkRecord(query, logger).check_type(record, types) == expected + + +@pytest.mark.parametrize( + "record, expected", + [ + ( + { + "id": "gid://shopify/Metafield/123", + "__parentId": "gid://shopify/Order/102030", + }, + { + "id": 123, + "admin_graphql_api_id": "gid://shopify/Metafield/123", + "__parentId": "gid://shopify/Order/102030", + }, + ) + ], +) +def test_record_resolver(record, expected, logger) -> None: + query = ShopifyBulkQuery(shop_id=0) + record_instance = ShopifyBulkRecord(query, logger) + assert record_instance.record_resolve_id(record) == expected + + +@pytest.mark.parametrize( + "record, expected", + [ + ( + {"id": "gid://shopify/Order/1234567890", "__typename": "Order"}, + {"id": "gid://shopify/Order/1234567890"}, + ), + ], +) +def test_record_new(record, expected, logger) -> None: + query = ShopifyBulkQuery(shop_id=0) + record_instance = ShopifyBulkRecord(query, logger) + record_instance.record_new(record) + assert record_instance.buffer == [expected] + + +@pytest.mark.parametrize( + "records_from_jsonl, record_components, expected", + [ + ( + [ + {"__typename": "NewRecord", "id": "gid://shopify/NewRecord/1234567890", "name": "new_record"}, + {"__typename": "RecordComponent", "id": "gid://shopify/RecordComponent/1234567890", "name": "new_component"}, + ], + {"new_record": "NewRecord", "record_components": ["RecordComponent"]}, + [ + { + "id": "gid://shopify/NewRecord/1234567890", + "name": "new_record", + "record_components": { + "RecordComponent": [ + { + "id": "gid://shopify/RecordComponent/1234567890", + "name": "new_component", + }, + ] + }, + } + ], + ), + ], + ids=["add_component"], +) +def test_record_new_component(records_from_jsonl, record_components, expected, logger) -> None: + query = ShopifyBulkQuery(shop_id=0) + record_instance = ShopifyBulkRecord(query, logger) + record_instance.components = record_components.get("record_components") + # register new record first + record_instance.record_new(records_from_jsonl[0]) + assert len(record_instance.buffer) > 0 + # check the components placeholder was created for new record registered + assert "record_components" in record_instance.buffer[-1].keys() + # register record component + record_instance.record_new_component(records_from_jsonl[1]) + # check the component was proccessed + assert len(record_instance.buffer[-1]["record_components"]["RecordComponent"]) > 0 + # general check + assert record_instance.buffer == expected + + +@pytest.mark.parametrize( + "buffered_record, expected", + [ + ( + { + "id": "gid://shopify/NewRecord/1234567890", + "name": "new_record", + "record_components": { + "RecordComponent": [ + { + "id": "gid://shopify/RecordComponent/1234567890", + "name": "new_component", + } + ] + }, + }, + [ + { + "id": 1234567890, + "name": "new_record", + "record_components": { + "RecordComponent": [ + { + "id": "gid://shopify/RecordComponent/1234567890", + "name": "new_component", + }, + ] + }, + "admin_graphql_api_id": "gid://shopify/NewRecord/1234567890", + } + ], + ), + ], +) +def test_buffer_flush(buffered_record, expected, logger) -> None: + query = ShopifyBulkQuery(shop_id=0) + record_instance = ShopifyBulkRecord(query, logger) + # populate the buffer with record + record_instance.buffer.append(buffered_record) + assert list(record_instance.buffer_flush()) == expected + + +@pytest.mark.parametrize( + "records_from_jsonl, record_composition, expected", + [ + ( + [ + {"__typename": "NewRecord", "id": "gid://shopify/NewRecord/1234567890", "name": "new_record"}, + {"__typename": "RecordComponent", "id": "gid://shopify/RecordComponent/1234567890", "name": "new_component"}, + ], + {"new_record": "NewRecord", "record_components": ["RecordComponent"]}, + [ + { + "id": "gid://shopify/NewRecord/1234567890", + "name": "new_record", + "record_components": { + "RecordComponent": [ + { + "id": "gid://shopify/RecordComponent/1234567890", + "name": "new_component", + }, + ] + }, + } + ], + ), + ], + ids=["test_compose"], +) +def test_record_compose(records_from_jsonl, record_composition, expected, logger) -> None: + query = ShopifyBulkQuery(shop_id=0) + # query.record_composition = record_composition + record_instance = ShopifyBulkRecord(query, logger) + record_instance.composition = record_composition + record_instance.components = record_composition.get("record_components") + # process read jsonl records + for record in records_from_jsonl: + list(record_instance.record_compose(record)) + + assert record_instance.buffer == expected diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_tools.py b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_tools.py new file mode 100644 index 000000000000..ba7e0474f494 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/graphql_bulk/test_tools.py @@ -0,0 +1,54 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + + +import pytest +from source_shopify.shopify_graphql.bulk.exceptions import ShopifyBulkExceptions +from source_shopify.shopify_graphql.bulk.tools import BulkTools + + +def test_camel_to_snake() -> None: + assert BulkTools.camel_to_snake("camelCase") == "camel_case" + assert BulkTools.camel_to_snake("snake_case") == "snake_case" + assert BulkTools.camel_to_snake("PascalCase") == "pascal_case" + + +@pytest.mark.parametrize( + "job_result_url, error_type, expected", + [ + ( + "https://storage.googleapis.com/shopify-tiers-assets-prod-us-east1/?GoogleAccessId=assets-us-prod%40shopify-tiers.iam.gserviceaccount.com&Expires=1705508208&Signature=%3D%3D&response-content-disposition=attachment%3B+filename%3D%22bulk-4147374162109.jsonl%22%3B+filename%2A%3DUTF-8%27%27bulk-4147374162109.jsonl&response-content-type=application%2Fjsonl", + None, + "bulk-4147374162109.jsonl", + ), + ( + "https://storage.googleapis.com/shopify-tiers-assets-prod-us-east1/?GoogleAccessId=assets-us-prod%40shopify-tiers.iam.gserviceaccount.com&Expires=1705508208", + ShopifyBulkExceptions.BulkJobResultUrlError, + "Could not extract the `filename` from `result_url` provided", + ), + ], + ids=["success", "error"], +) +def test_filename_from_url(job_result_url, error_type, expected) -> None: + if error_type: + with pytest.raises(error_type) as error: + BulkTools.filename_from_url(job_result_url) + assert expected in repr(error.value) + else: + assert BulkTools.filename_from_url(job_result_url) == expected + + +def test_from_iso8601_to_rfc3339() -> None: + record = {"date": "2023-01-01T15:00:00Z"} + assert BulkTools.from_iso8601_to_rfc3339(record, "date") == "2023-01-01T15:00:00+00:00" + + +def test_fields_names_to_snake_case() -> None: + dict_input = {"camelCase": "value", "snake_case": "value", "__parentId": "value"} + expected_output = {"camel_case": "value", "snake_case": "value", "__parentId": "value"} + assert BulkTools().fields_names_to_snake_case(dict_input) == expected_output + + +def test_resolve_str_id() -> None: + assert BulkTools.resolve_str_id("123") == 123 + assert BulkTools.resolve_str_id("456", str) == "456" + assert BulkTools.resolve_str_id(None) is None diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_auth.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_auth.py index 4577987825cf..ee1ae56a74a1 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_auth.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_auth.py @@ -5,6 +5,7 @@ import pytest from source_shopify.auth import NotImplementedAuth, ShopifyAuthenticator +from source_shopify.source import ConnectionCheckTest TEST_ACCESS_TOKEN = "test_access_token" TEST_API_PASSWORD = "test_api_password" @@ -25,6 +26,11 @@ def config_not_implemented_auth_method(): return {"credentials": {"auth_method": "not_implemented_auth_method"}} +@pytest.fixture +def config_missing_access_token(): + return {"shop": "SHOP_NAME", "credentials": {"auth_method": "oauth2.0", "access_token": None}} + + @pytest.fixture def expected_auth_header_access_token(): return {"X-Shopify-Access-Token": TEST_ACCESS_TOKEN} @@ -47,6 +53,14 @@ def test_shopify_authenticator_api_password(config_api_password, expected_auth_h def test_raises_notimplemented_auth(config_not_implemented_auth_method): authenticator = ShopifyAuthenticator(config=(config_not_implemented_auth_method)) - with pytest.raises(NotImplementedAuth) as not_implemented_exc: - print(not_implemented_exc) + with pytest.raises(NotImplementedAuth): authenticator.get_auth_header() + + +def test_raises_missing_access_token(config_missing_access_token): + config_missing_access_token["authenticator"] = ShopifyAuthenticator(config=(config_missing_access_token)) + failed_check = ConnectionCheckTest(config_missing_access_token).test_connection() + assert failed_check == ( + False, + "Authentication was unsuccessful. Please verify your authentication credentials or login is correct.", + ) diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_cached_stream_state.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_cached_stream_state.py index 031e7871e8ef..80cc6115c449 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_cached_stream_state.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_cached_stream_state.py @@ -4,7 +4,7 @@ import pytest -from source_shopify.source import OrderRefunds, Orders +from source_shopify.streams.streams import OrderRefunds, Orders from source_shopify.utils import EagerlyCachedStreamState as stream_state_cache # Define the Stream instances for the tests diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_control_rate_limit.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_control_rate_limit.py index 2451e5f48745..8a79dd8ae35c 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_control_rate_limit.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_control_rate_limit.py @@ -39,9 +39,7 @@ def test_rest_api_with_unknown_load(requests_mock): requests_mock.get("https://test.myshopify.com/", headers=test_response_header) test_response = requests.get("https://test.myshopify.com/") - actual_sleep_time = limiter.get_rest_api_wait_time( - test_response, threshold=TEST_THRESHOLD, rate_limit_header=TEST_RATE_LIMIT_HEADER - ) + actual_sleep_time = limiter.get_rest_api_wait_time(test_response, threshold=TEST_THRESHOLD, rate_limit_header=TEST_RATE_LIMIT_HEADER) assert limiter.on_unknown_load == actual_sleep_time @@ -55,9 +53,7 @@ def test_rest_api_with_low_load(requests_mock): requests_mock.get("https://test.myshopify.com/", headers=test_response_header) test_response = requests.get("https://test.myshopify.com/") - actual_sleep_time = limiter.get_rest_api_wait_time( - test_response, threshold=TEST_THRESHOLD, rate_limit_header=TEST_RATE_LIMIT_HEADER - ) + actual_sleep_time = limiter.get_rest_api_wait_time(test_response, threshold=TEST_THRESHOLD, rate_limit_header=TEST_RATE_LIMIT_HEADER) assert limiter.on_low_load == actual_sleep_time @@ -71,9 +67,7 @@ def test_rest_api_with_mid_load(requests_mock): requests_mock.get("https://test.myshopify.com/", headers=test_response_header) test_response = requests.get("https://test.myshopify.com/") - actual_sleep_time = limiter.get_rest_api_wait_time( - test_response, threshold=TEST_THRESHOLD, rate_limit_header=TEST_RATE_LIMIT_HEADER - ) + actual_sleep_time = limiter.get_rest_api_wait_time(test_response, threshold=TEST_THRESHOLD, rate_limit_header=TEST_RATE_LIMIT_HEADER) assert limiter.on_mid_load == actual_sleep_time @@ -87,9 +81,7 @@ def test_rest_api_with_high_load(requests_mock): requests_mock.get("https://test.myshopify.com/", headers=test_response_header) test_response = requests.get("https://test.myshopify.com/") - actual_sleep_time = limiter.get_rest_api_wait_time( - test_response, threshold=TEST_THRESHOLD, rate_limit_header=TEST_RATE_LIMIT_HEADER - ) + actual_sleep_time = limiter.get_rest_api_wait_time(test_response, threshold=TEST_THRESHOLD, rate_limit_header=TEST_RATE_LIMIT_HEADER) assert limiter.on_high_load == actual_sleep_time diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_deleted_events_stream.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_deleted_events_stream.py new file mode 100644 index 000000000000..126d28b7e66d --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_deleted_events_stream.py @@ -0,0 +1,245 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest +from source_shopify.auth import ShopifyAuthenticator +from source_shopify.streams.base_streams import ShopifyDeletedEventsStream +from source_shopify.streams.streams import Products + + +@pytest.fixture +def config(basic_config): + basic_config["start_date"] = "2020-11-01" + basic_config["authenticator"] = ShopifyAuthenticator(basic_config) + return basic_config + + +@pytest.mark.parametrize( + "stream,expected_main_path,expected_events_path", + [ + (Products, "products.json", "events.json"), + ], +) +def test_path(stream, expected_main_path, expected_events_path, config) -> None: + stream = stream(config) + main_path = stream.path() + events_path = stream.deleted_events.path() + assert main_path == expected_main_path + assert events_path == expected_events_path + + +@pytest.mark.parametrize( + "stream,expected_events_schema", + [ + (Products, {}), + ], +) +def test_get_json_schema(stream, expected_events_schema, config) -> None: + stream = stream(config) + schema = stream.deleted_events.get_json_schema() + # no schema is expected + assert schema == expected_events_schema + + +@pytest.mark.parametrize( + "stream,expected_data_field,expected_pk,expected_cursor_field", + [ + (Products, "events", "id", "deleted_at"), + ], +) +def test_has_correct_instance_vars(stream, expected_data_field, expected_pk, expected_cursor_field, config) -> None: + stream = stream(config) + assert stream.deleted_events.data_field == expected_data_field + assert stream.deleted_events.primary_key == expected_pk + assert stream.deleted_events.cursor_field == expected_cursor_field + + +@pytest.mark.parametrize( + "stream,expected", + [ + (Products, None), + ], +) +def test_has_no_availability_strategy(stream, expected, config) -> None: + stream = stream(config) + # no availability_strategy is expected + assert stream.deleted_events.availability_strategy is expected + + +@pytest.mark.parametrize( + "stream,deleted_records_json,expected", + [ + ( + Products, + [ + { + "id": 123, + "subject_id": 234, + "created_at": "2023-09-05T14:02:00-07:00", + "subject_type": "Product", + "verb": "destroy", + "arguments": [], + "message": "Test Message", + "author": "Online Store", + "description": "Test Description", + "shop_url": "airbyte-integration-test", + }, + ], + [ + { + "id": 123, + "subject_id": 234, + "created_at": "2023-09-05T14:02:00-07:00", + "subject_type": "Product", + "verb": "destroy", + "arguments": [], + "message": "Test Message", + "author": "Online Store", + "description": "Test Description", + "shop_url": "airbyte-integration-test", + }, + ], + ), + ], +) +def test_read_deleted_records(stream, requests_mock, deleted_records_json, expected, config, mocker) -> None: + stream = stream(config) + deleted_records_url = stream.url_base + stream.deleted_events.path() + requests_mock.get(deleted_records_url, json=deleted_records_json) + mocker.patch("source_shopify.streams.base_streams.IncrementalShopifyStreamWithDeletedEvents.read_records", return_value=deleted_records_json) + assert list(stream.read_records(sync_mode=None)) == expected + + +@pytest.mark.parametrize( + "stream,input,expected", + [ + ( + Products, + [ + { + "id": 123, + "subject_id": 234, + "created_at": "2023-09-05T14:02:00-07:00", + "subject_type": "Product", + "verb": "destroy", + "arguments": [], + "message": "Test Message", + "author": "Online Store", + "description": "Test Description", + "shop_url": "airbyte-integration-test", + } + ], + [ + { + "id": 234, + "deleted_at": "2023-09-05T14:02:00-07:00", + "updated_at": "2023-09-05T14:02:00-07:00", + "deleted_message": "Test Message", + "deleted_description": "Test Description", + "shop_url": "airbyte-integration-test", + } + ], + ), + ], +) +def test_produce_deleted_records_from_events(stream, input, expected, config) -> None: + stream = stream(config) + result = stream.deleted_events.produce_deleted_records_from_events(input) + assert list(result) == expected + + +@pytest.mark.parametrize( + "stream, stream_state, next_page_token, expected_stream_params, expected_deleted_params", + [ + # params with NO STATE + ( + Products, + {}, + None, + {"limit": 250, "order": "updated_at asc", "updated_at_min": "2020-11-01"}, + {"filter": "Product", "verb": "destroy"}, + ), + # params with STATE + ( + Products, + {"updated_at": "2028-01-01", "deleted": {"deleted_at": "2029-01-01"}}, + None, + {"limit": 250, "order": "updated_at asc", "updated_at_min": "2028-01-01"}, + {"created_at_min": "2029-01-01", "filter": "Product", "verb": "destroy"}, + ), + # params with NO STATE but with NEXT_PAGE_TOKEN + ( + Products, + {}, + {"page_info": "next_page_token"}, + {"limit": 250, "page_info": "next_page_token"}, + {"page_info": "next_page_token"}, + ), + ], +) +def test_request_params(config, stream, stream_state, next_page_token, expected_stream_params, expected_deleted_params) -> None: + stream = stream(config) + assert stream.request_params(stream_state=stream_state, next_page_token=next_page_token) == expected_stream_params + assert stream.deleted_events.request_params(stream_state=stream_state, next_page_token=next_page_token) == expected_deleted_params + + +@pytest.mark.parametrize( + "stream,expected", + [ + (Products, ShopifyDeletedEventsStream), + ], +) +def test_deleted_events_instance(stream, config, expected) -> None: + stream = stream(config) + assert isinstance(stream.deleted_events, expected) + + +@pytest.mark.parametrize( + "stream,expected", + [ + (Products, ""), + ], +) +def test_default_deleted_state_comparison_value(stream, config, expected) -> None: + stream = stream(config) + assert stream.default_deleted_state_comparison_value == expected + + +@pytest.mark.parametrize( + "stream, last_record, current_state, expected", + [ + # NO INITIAL STATE + ( + Products, + {"id": 1, "updated_at": "2021-01-01"}, + {}, + {"updated_at": "2021-01-01", "deleted": {"deleted_at": ""}}, + ), + # with INITIAL STATE + ( + Products, + {"id": 1, "updated_at": "2022-01-01"}, + {"updated_at": "2021-01-01", "deleted": {"deleted_at": ""}}, + {"updated_at": "2022-01-01", "deleted": {"deleted_at": ""}}, + ), + # with NO Last Record value and NO current state value + ( + Products, + {}, + {}, + {"updated_at": "", "deleted": {"deleted_at": ""}}, + ), + # with NO Last Record value but with Current state value + ( + Products, + {}, + {"updated_at": "2030-01-01", "deleted": {"deleted_at": ""}}, + {"updated_at": "2030-01-01", "deleted": {"deleted_at": ""}}, + ), + ], +) +def test_get_updated_state(config, stream, last_record, current_state, expected) -> None: + stream = stream(config) + assert stream.get_updated_state(current_state, last_record) == expected diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_graphql_products.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_graphql_products.py new file mode 100644 index 000000000000..a6c99f9c3c44 --- /dev/null +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_graphql_products.py @@ -0,0 +1,16 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import pytest +from source_shopify.shopify_graphql.graphql import get_query_products + + +@pytest.mark.parametrize( + "page_size, filter_value, next_page_token, expected_query", + [ + (100, None, None, 'query {\n products(first: 100, query: null, after: null) {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}'), + (200, "2027-07-11T13:07:45-07:00", None, 'query {\n products(first: 200, query: "updated_at:>\'2027-07-11T13:07:45-07:00\'", after: null) {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}'), + (250, "2027-07-11T13:07:45-07:00", "end_cursor_value", 'query {\n products(first: 250, query: "updated_at:>\'2027-07-11T13:07:45-07:00\'", after: "end_cursor_value") {\n nodes {\n id\n title\n updatedAt\n createdAt\n publishedAt\n status\n vendor\n productType\n tags\n options {\n id\n name\n position\n values\n }\n handle\n description\n tracksInventory\n totalInventory\n totalVariants\n onlineStoreUrl\n onlineStorePreviewUrl\n descriptionHtml\n isGiftCard\n legacyResourceId\n mediaCount\n }\n pageInfo {\n hasNextPage\n endCursor\n }\n }\n}'), + ], +) +def test_get_query_products(page_size, filter_value, next_page_token, expected_query): + assert get_query_products(page_size, 'updatedAt', filter_value, next_page_token) == expected_query diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py index 41ee39eff733..76441b213ff9 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/test_source.py @@ -7,7 +7,8 @@ import pytest from source_shopify.auth import ShopifyAuthenticator -from source_shopify.source import ( +from source_shopify.source import SourceShopify +from source_shopify.streams.streams import ( AbandonedCheckouts, Articles, Blogs, @@ -41,14 +42,13 @@ Products, ProductVariants, Shop, - SourceShopify, TenderTransactions, Transactions, ) @pytest.fixture -def config(basic_config): +def config(basic_config) -> dict: basic_config["start_date"] = "2020-11-01" basic_config["authenticator"] = ShopifyAuthenticator(basic_config) return basic_config @@ -61,18 +61,22 @@ def config(basic_config): (Blogs, None, "blogs.json"), (MetafieldBlogs, {"id": 123}, "blogs/123/metafields.json"), (MetafieldArticles, {"id": 123}, "articles/123/metafields.json"), - (MetafieldCustomers, {"id": 123}, "customers/123/metafields.json"), - (MetafieldOrders, {"id": 123}, "orders/123/metafields.json"), - (MetafieldDraftOrders, {"id": 123}, "draft_orders/123/metafields.json"), - (MetafieldProducts, {"id": 123}, "products/123/metafields.json"), - (MetafieldProductVariants, {"variants": 123}, "variants/123/metafields.json"), + # GraphQL Bulk Streams + (MetafieldCustomers, None, "graphql.json"), + (MetafieldOrders, None, "graphql.json"), + (MetafieldDraftOrders, None, "graphql.json"), + (MetafieldProducts, None, "graphql.json"), + (MetafieldProductVariants, None, "graphql.json"), + (MetafieldLocations, None, "graphql.json"), + (MetafieldCollections, None, "graphql.json"), + # (MetafieldSmartCollections, {"id": 123}, "smart_collections/123/metafields.json"), - (MetafieldCollections, {"collection_id": 123}, "collections/123/metafields.json"), (MetafieldPages, {"id": 123}, "pages/123/metafields.json"), - (MetafieldLocations, {"id": 123}, "locations/123/metafields.json"), (MetafieldShops, None, "metafields.json"), - (ProductImages, {"product_id": 123}, "products/123/images.json"), - (ProductVariants, {"product_id": 123}, "products/123/variants.json"), + # Nested Substreams + (ProductImages, None, ""), + (ProductVariants, None, ""), + # (Customers, None, "customers.json"), (Orders, None, "orders.json"), (DraftOrders, None, "draft_orders.json"), @@ -87,7 +91,7 @@ def config(basic_config): (CustomCollections, None, "custom_collections.json"), ], ) -def test_customers_path(stream, stream_slice, expected_path, config): +def test_path(stream, stream_slice, expected_path, config) -> None: stream = stream(config) if stream_slice: result = stream.path(stream_slice) @@ -99,39 +103,47 @@ def test_customers_path(stream, stream_slice, expected_path, config): @pytest.mark.parametrize( "stream,stream_slice,expected_path", [ - (OrderRefunds, {"order_id": 12345}, "orders/12345/refunds.json"), (OrderRisks, {"order_id": 12345}, "orders/12345/risks.json"), (Transactions, {"order_id": 12345}, "orders/12345/transactions.json"), - (DiscountCodes, {"price_rule_id": 12345}, "price_rules/12345/discount_codes.json"), - (InventoryLevels, {"location_id": 12345}, "locations/12345/inventory_levels.json"), - (FulfillmentOrders, {"order_id": 12345}, "orders/12345/fulfillment_orders.json"), - (Fulfillments, {"order_id": 12345}, "orders/12345/fulfillments.json"), + # Nested Substreams + (OrderRefunds, None, ""), + (Fulfillments, None, ""), + # GQL BULK stream + (DiscountCodes, None, "graphql.json"), + (FulfillmentOrders, None, "graphql.json"), + (InventoryLevels, None, "graphql.json"), ], ) -def test_customers_path_with_stream_slice_param(stream, stream_slice, expected_path, config): +def test_path_with_stream_slice_param(stream, stream_slice, expected_path, config) -> None: stream = stream(config) - assert stream.path(stream_slice) == expected_path + if stream_slice: + result = stream.path(stream_slice) + else: + result = stream.path() + assert result == expected_path -def test_check_connection(config, mocker): - mocker.patch("source_shopify.source.Shop.read_records", return_value=[{"id": 1}]) +def test_check_connection(config, mocker) -> None: + mocker.patch("source_shopify.streams.streams.Shop.read_records", return_value=[{"id": 1}]) source = SourceShopify() logger_mock = MagicMock() assert source.check_connection(logger_mock, config) == (True, None) -def test_read_records(config, mocker): +def test_read_records(config, mocker) -> None: records = [{"created_at": "2022-10-10T06:21:53-07:00", "orders": {"updated_at": "2022-10-10T06:21:53-07:00"}}] stream_slice = records[0] stream = OrderRefunds(config) - mocker.patch("source_shopify.source.IncrementalShopifyStream.read_records", return_value=records) - assert next(stream.read_records(stream_slice=stream_slice)) == records[0] + mocker.patch("source_shopify.streams.base_streams.IncrementalShopifyNestedStream.read_records", return_value=records) + assert stream.read_records(stream_slice=stream_slice)[0] == records[0] @pytest.mark.parametrize( "stream, expected", [ - (OrderRefunds, {"limit": 250}), + # Nested Substream + (OrderRefunds, {}), + # (Orders, {"limit": 250, "status": "any", "order": "updated_at asc", "updated_at_min": "2020-11-01"}), ( AbandonedCheckouts, @@ -139,7 +151,7 @@ def test_read_records(config, mocker): ), ], ) -def test_request_params(config, stream, expected): +def test_request_params(config, stream, expected) -> None: assert stream(config).request_params() == expected @@ -147,17 +159,17 @@ def test_request_params(config, stream, expected): "last_record, current_state, expected", [ # no init state - ({"created_at": "2022-10-10T06:21:53-07:00"}, {}, {'created_at': '2022-10-10T06:21:53-07:00', 'orders': None}), + ({"created_at": "2022-10-10T06:21:53-07:00"}, {}, {"created_at": "2022-10-10T06:21:53-07:00", "orders": None}), # state is empty str - ({"created_at": "2022-10-10T06:21:53-07:00"}, {"created_at": ""}, {'created_at': '2022-10-10T06:21:53-07:00', 'orders': None}), + ({"created_at": "2022-10-10T06:21:53-07:00"}, {"created_at": ""}, {"created_at": "2022-10-10T06:21:53-07:00", "orders": None}), # state is None - ({"created_at": "2022-10-10T06:21:53-07:00"}, {"created_at": None}, {'created_at': '2022-10-10T06:21:53-07:00', 'orders': None}), + ({"created_at": "2022-10-10T06:21:53-07:00"}, {"created_at": None}, {"created_at": "2022-10-10T06:21:53-07:00", "orders": None}), # last rec cursor is None - ({"created_at": None}, {"created_at": None}, {'created_at': '', 'orders': None}), + ({"created_at": None}, {"created_at": None}, {"created_at": "", "orders": None}), # last rec cursor is empty str - ({"created_at": ""}, {"created_at": "null"}, {'created_at': 'null', 'orders': None}), + ({"created_at": ""}, {"created_at": "null"}, {"created_at": "null", "orders": None}), # no values at all - ({}, {}, {'created_at': '', 'orders': None}) + ({}, {}, {"created_at": "", "orders": None}), ], ids=[ "no init state", @@ -166,14 +178,14 @@ def test_request_params(config, stream, expected): "last rec cursor is None", "last rec cursor is empty str", "no values at all", - ] + ], ) -def test_get_updated_state(config, last_record, current_state, expected): +def test_get_updated_state(config, last_record, current_state, expected) -> None: stream = OrderRefunds(config) assert stream.get_updated_state(current_state, last_record) == expected -def test_parse_response_with_bad_json(config, response_with_bad_json): +def test_parse_response_with_bad_json(config, response_with_bad_json) -> None: stream = Customers(config) assert list(stream.parse_response(response_with_bad_json)) == [{}] @@ -184,9 +196,9 @@ def test_parse_response_with_bad_json(config, response_with_bad_json): ("test-store", "test-store"), ("test-store.myshopify.com", "test-store"), ], - ids=["old style", "oauth style"] + ids=["old style", "oauth style"], ) -def test_get_shop_name(config, shop, expected): +def test_get_shop_name(config, shop, expected) -> None: source = SourceShopify() config["shop"] = shop actual = source.get_shop_name(config) diff --git a/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py index a1fea8e4538a..f04cb7c12a8b 100644 --- a/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-shopify/unit_tests/unit_test.py @@ -3,15 +3,19 @@ # +from unittest.mock import patch + import pytest import requests -from source_shopify.source import BalanceTransactions, DiscountCodes, FulfillmentOrders, PriceRules, ShopifyStream, SourceShopify +from source_shopify.source import ConnectionCheckTest, SourceShopify +from source_shopify.streams.streams import BalanceTransactions, DiscountCodes, FulfillmentOrders, PriceRules -def test_get_next_page_token(requests_mock): +def test_get_next_page_token(requests_mock, auth_config): """ Test shows that next_page parameters are parsed correctly from the response object and could be passed for next request API call, """ + stream = PriceRules(auth_config) response_header_links = { "Date": "Thu, 32 Jun 2099 24:24:24 GMT", "Content-Type": "application/json; charset=utf-8", @@ -25,17 +29,17 @@ def test_get_next_page_token(requests_mock): requests_mock.get("https://test.myshopify.com/", headers=response_header_links) response = requests.get("https://test.myshopify.com/") - test = ShopifyStream.next_page_token(response) + test = stream.next_page_token(response=response) assert test == expected_output_token def test_privileges_validation(requests_mock, basic_config): + requests_mock.get( "https://test_shop.myshopify.com/admin/oauth/access_scopes.json", json={"access_scopes": [{"handle": "read_orders"}]}, ) - source = SourceShopify() - + expected = [ "abandoned_checkouts", "fulfillments", @@ -47,10 +51,14 @@ def test_privileges_validation(requests_mock, basic_config): "shop", "tender_transactions", "transactions", + "transactions_graphql", "countries", ] - - assert [stream.name for stream in source.streams(basic_config)] == expected + # mock the get_shop_id method + with patch.object(ConnectionCheckTest, "get_shop_id", return_value=123) as mock: + source = SourceShopify() + streams = source.streams(basic_config) + assert [stream.name for stream in streams] == expected @pytest.mark.parametrize( diff --git a/airbyte-integrations/connectors/source-shortio/README.md b/airbyte-integrations/connectors/source-shortio/README.md index 3e8a6bdb3870..ec115936fa33 100644 --- a/airbyte-integrations/connectors/source-shortio/README.md +++ b/airbyte-integrations/connectors/source-shortio/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-shortio:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/shortio) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_shortio/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-shortio:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-shortio build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-shortio:airbyteDocker +An image will be built with the tag `airbyte/source-shortio:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-shortio:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-shortio:dev check --co docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-shortio:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-shortio:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-shortio test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-shortio:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-shortio:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-shortio test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/shortio.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-shortio/acceptance-test-config.yml b/airbyte-integrations/connectors/source-shortio/acceptance-test-config.yml index 3314e7f6cb11..c08152adef62 100644 --- a/airbyte-integrations/connectors/source-shortio/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-shortio/acceptance-test-config.yml @@ -26,7 +26,7 @@ acceptance_tests: extra_fields: no exact_order: no extra_records: yes - incremental: + incremental: # bypass_reason: "This connector does not implement incremental sync" tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-shortio/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-shortio/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-shortio/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-shortio/build.gradle b/airbyte-integrations/connectors/source-shortio/build.gradle deleted file mode 100644 index 5fc3faab7871..000000000000 --- a/airbyte-integrations/connectors/source-shortio/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_shortio' -} diff --git a/airbyte-integrations/connectors/source-shortio/metadata.yaml b/airbyte-integrations/connectors/source-shortio/metadata.yaml index ad64fec0533c..6eced88a6f02 100644 --- a/airbyte-integrations/connectors/source-shortio/metadata.yaml +++ b/airbyte-integrations/connectors/source-shortio/metadata.yaml @@ -23,7 +23,7 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/shortio tags: - language:python - - language:lowcode + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-slack/Dockerfile b/airbyte-integrations/connectors/source-slack/Dockerfile deleted file mode 100644 index 73a5c97a1814..000000000000 --- a/airbyte-integrations/connectors/source-slack/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -ENV CODE_PATH="source_slack" -ENV WORKDIR=/airbyte/integration_code - -WORKDIR $WORKDIR - -COPY setup.py ./ -RUN pip install . - -COPY $CODE_PATH ./$CODE_PATH -COPY main.py ./ - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.2.0 -LABEL io.airbyte.name=airbyte/source-slack diff --git a/airbyte-integrations/connectors/source-slack/README.md b/airbyte-integrations/connectors/source-slack/README.md index b53c174c59a6..1b407f1aed59 100644 --- a/airbyte-integrations/connectors/source-slack/README.md +++ b/airbyte-integrations/connectors/source-slack/README.md @@ -1,100 +1,67 @@ -# Slack Source +# Orbit Source -This is the repository for the Slack source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/slack). +This is the repository for the Orbit configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/orbit). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-slack:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/slack) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_slack/spec.json` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/orbit) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_orbit/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source slack test creds` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source orbit test creds` and place them into `secrets/config.json`. - -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-slack:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name source-orbit build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-slack:airbyteDocker +An image will be built with the tag `airbyte/source-orbit:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-orbit:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-slack:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-slack:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-slack:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-slack:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm airbyte/source-orbit:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-orbit:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-orbit:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-orbit:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-slack:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-slack test +``` + +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-slack test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/slack.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml b/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml index c59968bdac20..2fd1ba9af04b 100644 --- a/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-slack/acceptance-test-config.yml @@ -3,38 +3,41 @@ test_strictness_level: high acceptance_tests: spec: tests: - - spec_path: "source_slack/spec.json" - backward_compatibility_tests_config: - # edited `min`/`max` > `minimum`/`maximum` for `lookback_window` field - disable_for_version: "0.1.26" + - spec_path: "source_slack/spec.json" + backward_compatibility_tests_config: + # edited `min`/`max` > `minimum`/`maximum` for `lookback_window` field + disable_for_version: "0.1.26" connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "secrets/config_oauth.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" - - config_path: "integration_tests/invalid_oauth_config.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "secrets/config_oauth.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + - config_path: "integration_tests/invalid_oauth_config.json" + status: "failed" discovery: tests: - - config_path: "secrets/config.json" + - config_path: "secrets/config.json" basic_read: tests: - - config_path: "secrets/config.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - timeout_seconds: 4800 - fail_on_extra_columns: false + - config_path: "secrets/config.json" + expect_records: + path: "integration_tests/expected_records.jsonl" + timeout_seconds: 4800 full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/full_refresh_catalog.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/full_refresh_catalog.json" + timeout_seconds: 4800 incremental: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/incremental_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" - timeout_seconds: 4800 + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/incremental_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + timeout_seconds: 4800 + # When running multiple syncs in a row, we may get the same record set because of a lookback window. + # This may fail the test but this is expected behavior of the connector. + skip_comprehensive_incremental_tests: true diff --git a/airbyte-integrations/connectors/source-slack/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-slack/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-slack/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-slack/build.gradle b/airbyte-integrations/connectors/source-slack/build.gradle deleted file mode 100644 index 3327ad910fe7..000000000000 --- a/airbyte-integrations/connectors/source-slack/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_slack' -} diff --git a/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.jsonl index ab42da44c205..9ddbf42168e5 100644 --- a/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-slack/integration_tests/expected_records.jsonl @@ -1,6 +1,6 @@ -{"stream": "channels", "data": {"id": "C04KX3KEZ54", "name": "general", "is_channel": true, "is_group": false, "is_im": false, "is_mpim": false, "is_private": false, "created": 1674485468, "is_archived": false, "is_general": true, "unlinked": 0, "name_normalized": "general", "is_shared": false, "is_org_shared": false, "is_pending_ext_shared": false, "pending_shared": [], "context_team_id": "T04KX3KDDU6", "updated": 1681216123063, "parent_conversation": null, "creator": "U04L65GPMKN", "is_ext_shared": false, "shared_team_ids": ["T04KX3KDDU6"], "pending_connected_team_ids": [], "is_member": true, "topic": {"value": "", "creator": "", "last_set": 0}, "purpose": {"value": "This is the one channel that will always include everyone. It\u2019s a great spot for announcements and team-wide conversations.", "creator": "U04L65GPMKN", "last_set": 1674485468}, "previous_names": [], "num_members": 3}, "emitted_at": 1683105171043} -{"stream": "channels", "data": {"id": "C04L3M4PTJ6", "name": "random", "is_channel": true, "is_group": false, "is_im": false, "is_mpim": false, "is_private": false, "created": 1674485468, "is_archived": false, "is_general": false, "unlinked": 0, "name_normalized": "random", "is_shared": false, "is_org_shared": false, "is_pending_ext_shared": false, "pending_shared": [], "context_team_id": "T04KX3KDDU6", "updated": 1681216123075, "parent_conversation": null, "creator": "U04L65GPMKN", "is_ext_shared": false, "shared_team_ids": ["T04KX3KDDU6"], "pending_connected_team_ids": [], "is_member": true, "topic": {"value": "", "creator": "", "last_set": 0}, "purpose": {"value": "This channel is for... well, everything else. It\u2019s a place for team jokes, spur-of-the-moment ideas, and funny GIFs. Go wild!", "creator": "U04L65GPMKN", "last_set": 1674485468}, "previous_names": [], "num_members": 3}, "emitted_at": 1683105171043} -{"stream": "channels", "data": {"id": "C04LTCM2Y56", "name": "integrationtest", "is_channel": true, "is_group": false, "is_im": false, "is_mpim": false, "is_private": false, "created": 1674485589, "is_archived": false, "is_general": false, "unlinked": 0, "name_normalized": "integrationtest", "is_shared": false, "is_org_shared": false, "is_pending_ext_shared": false, "pending_shared": [], "context_team_id": "T04KX3KDDU6", "updated": 1681216123086, "parent_conversation": null, "creator": "U04L65GPMKN", "is_ext_shared": false, "shared_team_ids": ["T04KX3KDDU6"], "pending_connected_team_ids": [], "is_member": true, "topic": {"value": "", "creator": "", "last_set": 0}, "purpose": {"value": "This channel is for everything #integrationtest. Hold meetings, share docs, and make decisions together with your team.", "creator": "U04L65GPMKN", "last_set": 1674485589}, "previous_names": [], "num_members": 3}, "emitted_at": 1683105171044} +{"stream": "channels", "data": {"id": "C04KX3KEZ54", "name": "general", "is_channel": true, "is_group": false, "is_im": false, "is_mpim": false, "is_private": false, "created": 1674485468, "is_archived": false, "is_general": true, "unlinked": 0, "name_normalized": "general", "is_shared": false, "is_org_shared": false, "is_pending_ext_shared": false, "pending_shared": [], "context_team_id": "T04KX3KDDU6", "updated": 1681216123063, "parent_conversation": null, "creator": "U04L65GPMKN", "is_ext_shared": false, "shared_team_ids": ["T04KX3KDDU6"], "pending_connected_team_ids": [], "is_member": true, "topic": {"value": "", "creator": "", "last_set": 0}, "purpose": {"value": "This is the one channel that will always include everyone. It\u2019s a great spot for announcements and team-wide conversations.", "creator": "U04L65GPMKN", "last_set": 1674485468}, "previous_names": [], "num_members": 3}, "emitted_at": 1695111176963} +{"stream": "channels", "data": {"id": "C04L3M4PTJ6", "name": "random", "is_channel": true, "is_group": false, "is_im": false, "is_mpim": false, "is_private": false, "created": 1674485468, "is_archived": false, "is_general": false, "unlinked": 0, "name_normalized": "random", "is_shared": false, "is_org_shared": false, "is_pending_ext_shared": false, "pending_shared": [], "context_team_id": "T04KX3KDDU6", "updated": 1681216123075, "parent_conversation": null, "creator": "U04L65GPMKN", "is_ext_shared": false, "shared_team_ids": ["T04KX3KDDU6"], "pending_connected_team_ids": [], "is_member": true, "topic": {"value": "", "creator": "", "last_set": 0}, "purpose": {"value": "This channel is for... well, everything else. It\u2019s a place for team jokes, spur-of-the-moment ideas, and funny GIFs. Go wild!", "creator": "U04L65GPMKN", "last_set": 1674485468}, "previous_names": [], "num_members": 3}, "emitted_at": 1695111176963} +{"stream": "channels", "data": {"id": "C04LTCM2Y56", "name": "integrationtest", "is_channel": true, "is_group": false, "is_im": false, "is_mpim": false, "is_private": false, "created": 1674485589, "is_archived": false, "is_general": false, "unlinked": 0, "name_normalized": "integrationtest", "is_shared": false, "is_org_shared": false, "is_pending_ext_shared": false, "pending_shared": [], "context_team_id": "T04KX3KDDU6", "updated": 1681216123086, "parent_conversation": null, "creator": "U04L65GPMKN", "is_ext_shared": false, "shared_team_ids": ["T04KX3KDDU6"], "pending_connected_team_ids": [], "is_member": true, "topic": {"value": "", "creator": "", "last_set": 0}, "purpose": {"value": "This channel is for everything #integrationtest. Hold meetings, share docs, and make decisions together with your team.", "creator": "U04L65GPMKN", "last_set": 1674485589}, "previous_names": [], "num_members": 3}, "emitted_at": 1695111176963} {"stream": "channel_members", "data": {"member_id": "U04L65GPMKN", "channel_id": "C04KX3KEZ54"}, "emitted_at": 1683105171299} {"stream": "channel_members", "data": {"member_id": "U04LY6NARHU", "channel_id": "C04KX3KEZ54"}, "emitted_at": 1683105171299} {"stream": "channel_members", "data": {"member_id": "U04M23SBJGM", "channel_id": "C04KX3KEZ54"}, "emitted_at": 1683105171299} @@ -10,21 +10,20 @@ {"stream": "channel_members", "data": {"member_id": "U04L65GPMKN", "channel_id": "C04LTCM2Y56"}, "emitted_at": 1683105171629} {"stream": "channel_members", "data": {"member_id": "U04LY6NARHU", "channel_id": "C04LTCM2Y56"}, "emitted_at": 1683105171630} {"stream": "channel_members", "data": {"member_id": "U04M23SBJGM", "channel_id": "C04LTCM2Y56"}, "emitted_at": 1683105171630} -{"stream": "channel_messages", "data": {"client_msg_id": "3ae60d35-58b8-441c-923a-75de35a4ed8a", "type": "message", "text": "Test Thread 2", "user": "U04L65GPMKN", "ts": "1683104542.931169", "blocks": [{"type": "rich_text", "block_id": "WLB", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 2"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104568.059569", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104568.059569", "channel_id": "C04KX3KEZ54", "float_ts": 1683104542.931169}, "emitted_at": 1683105172463} -{"stream": "channel_messages", "data": {"client_msg_id": "e27672c0-451e-42a6-8eff-a14d2db8ac1e", "type": "message", "text": "Test Thread 1", "user": "U04L65GPMKN", "ts": "1683104499.808709", "blocks": [{"type": "rich_text", "block_id": "0j7", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 1"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104499.808709", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104528.084359", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104528.084359", "channel_id": "C04LTCM2Y56", "float_ts": 1683104499.808709}, "emitted_at": 1683105173575} -{"stream": "threads", "data": {"client_msg_id": "3ae60d35-58b8-441c-923a-75de35a4ed8a", "type": "message", "text": "Test Thread 2", "user": "U04L65GPMKN", "ts": "1683104542.931169", "blocks": [{"type": "rich_text", "block_id": "WLB", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 2"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104568.059569", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104568.059569", "channel_id": "C04KX3KEZ54", "float_ts": 1683104542.931169}, "emitted_at": 1683105174282} -{"stream": "threads", "data": {"client_msg_id": "3e96d351-270c-493f-a1a0-fdc3c4c0e11f", "type": "message", "text": "<@U04M23SBJGM> test test test", "user": "U04L65GPMKN", "ts": "1683104559.922849", "blocks": [{"type": "rich_text", "block_id": "tX6vr", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04M23SBJGM"}, {"type": "text", "text": " test test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "parent_user_id": "U04L65GPMKN", "channel_id": "C04KX3KEZ54", "float_ts": 1683104559.922849}, "emitted_at": 1683105174283} -{"stream": "threads", "data": {"client_msg_id": "08023e44-9d18-41ed-81dd-5f04ed699656", "type": "message", "text": "<@U04LY6NARHU> test test", "user": "U04L65GPMKN", "ts": "1683104568.059569", "blocks": [{"type": "rich_text", "block_id": "IyUF", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04LY6NARHU"}, {"type": "text", "text": " test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "parent_user_id": "U04L65GPMKN", "channel_id": "C04KX3KEZ54", "float_ts": 1683104568.059569}, "emitted_at": 1683105174284} -{"stream": "threads", "data": {"client_msg_id": "e27672c0-451e-42a6-8eff-a14d2db8ac1e", "type": "message", "text": "Test Thread 1", "user": "U04L65GPMKN", "ts": "1683104499.808709", "blocks": [{"type": "rich_text", "block_id": "0j7", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 1"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104499.808709", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104528.084359", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104528.084359", "channel_id": "C04LTCM2Y56", "float_ts": 1683104499.808709}, "emitted_at": 1683105175190} -{"stream": "threads", "data": {"client_msg_id": "e1e2d142-a0dd-4587-86e3-2dcb439ead82", "type": "message", "text": "<@U04LY6NARHU> Test test", "user": "U04L65GPMKN", "ts": "1683104515.919709", "blocks": [{"type": "rich_text", "block_id": "xVnQ", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04LY6NARHU"}, {"type": "text", "text": " Test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104499.808709", "parent_user_id": "U04L65GPMKN", "channel_id": "C04LTCM2Y56", "float_ts": 1683104515.919709}, "emitted_at": 1683105175191} -{"stream": "threads", "data": {"client_msg_id": "ffccbb24-8dd6-476d-87bf-65e5fa033cb9", "type": "message", "text": "<@U04M23SBJGM> test test test", "user": "U04L65GPMKN", "ts": "1683104528.084359", "blocks": [{"type": "rich_text", "block_id": "Lvl", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04M23SBJGM"}, {"type": "text", "text": " test test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104499.808709", "parent_user_id": "U04L65GPMKN", "channel_id": "C04LTCM2Y56", "float_ts": 1683104528.084359}, "emitted_at": 1683105175191} -{"stream": "users", "data": {"id": "USLACKBOT", "team_id": "T04KX3KDDU6", "name": "slackbot", "deleted": false, "color": "757575", "real_name": "Slackbot", "tz": "America/Los_Angeles", "tz_label": "Pacific Daylight Time", "tz_offset": -25200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Slackbot", "real_name_normalized": "Slackbot", "display_name": "Slackbot", "display_name_normalized": "Slackbot", "fields": {}, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "sv41d8cd98f0", "always_active": true, "first_name": "slackbot", "last_name": "", "image_24": "https://a.slack-edge.com/80588/img/slackbot_24.png", "image_32": "https://a.slack-edge.com/80588/img/slackbot_32.png", "image_48": "https://a.slack-edge.com/80588/img/slackbot_48.png", "image_72": "https://a.slack-edge.com/80588/img/slackbot_72.png", "image_192": "https://a.slack-edge.com/80588/marketing/img/avatars/slackbot/avatar-slackbot.png", "image_512": "https://a.slack-edge.com/80588/img/slackbot_512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 0, "is_email_confirmed": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1683105175456} -{"stream": "users", "data": {"id": "U04KUMXNYMV", "team_id": "T04KX3KDDU6", "name": "deactivateduser693438", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-24.png", "image_32": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-32.png", "image_48": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-48.png", "image_72": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-72.png", "image_192": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-192.png", "image_512": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090804, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1683105175456} -{"stream": "users", "data": {"id": "U04L2KY5CES", "team_id": "T04KX3KDDU6", "name": "deactivateduser686066", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-24.png", "image_32": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-32.png", "image_48": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-48.png", "image_72": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-72.png", "image_192": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-192.png", "image_512": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090785, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1683105175457} -{"stream": "users", "data": {"id": "U04L2LC770E", "team_id": "T04KX3KDDU6", "name": "deactivateduser521176", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-24.png", "image_32": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-32.png", "image_48": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-48.png", "image_72": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-72.png", "image_192": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-192.png", "image_512": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090821, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1683105175457} -{"stream": "users", "data": {"id": "U04L65GPMKN", "team_id": "T04KX3KDDU6", "name": "integration-test", "deleted": false, "color": "9f69e7", "real_name": "integration-test", "tz": "Asia/Jerusalem", "tz_label": "Israel Daylight Time", "tz_offset": 10800, "profile": {"title": "", "phone": "", "skype": "", "real_name": "integration-test", "real_name_normalized": "integration-test", "display_name": "", "display_name_normalized": "", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g0a7841feac7", "first_name": "integration-test", "last_name": "", "image_24": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0013-24.png", "image_32": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0013-32.png", "image_48": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0013-48.png", "image_72": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0013-72.png", "image_192": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0013-192.png", "image_512": "https://secure.gravatar.com/avatar/0a7841feac7218131ce7b427283c24ef.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0013-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_admin": true, "is_owner": true, "is_primary_owner": true, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1674485468, "is_email_confirmed": true, "has_2fa": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1683105175457} -{"stream": "users", "data": {"id": "U04L69BPZFX", "team_id": "T04KX3KDDU6", "name": "deactivateduser839125", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-24.png", "image_32": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-32.png", "image_48": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-48.png", "image_72": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-72.png", "image_192": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-192.png", "image_512": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1681811889, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1683105175458} -{"stream": "users", "data": {"id": "U04L94Y2JPM", "team_id": "T04KX3KDDU6", "name": "deactivateduser962255", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-24.png", "image_32": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-32.png", "image_48": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-48.png", "image_72": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-72.png", "image_192": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-192.png", "image_512": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090815, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1683105175460} -{"stream": "users", "data": {"id": "U04LMS8F7JM", "team_id": "T04KX3KDDU6", "name": "deactivateduser421996", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-24.png", "image_32": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-32.png", "image_48": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-48.png", "image_72": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-72.png", "image_192": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-192.png", "image_512": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1681811683, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1683105175461} -{"stream": "users", "data": {"id": "U04LY6NARHU", "team_id": "T04KX3KDDU6", "name": "user1.sample", "deleted": false, "color": "684b6c", "real_name": "User1 Sample", "tz": "Europe/Helsinki", "tz_label": "Eastern European Summer Time", "tz_offset": 10800, "profile": {"title": "", "phone": "", "skype": "", "real_name": "User1 Sample", "real_name_normalized": "User1 Sample", "display_name": "User1 Sample", "display_name_normalized": "User1 Sample", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g76d12585ef1", "first_name": "User1", "last_name": "Sample", "image_24": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-24.png", "image_32": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-32.png", "image_48": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-48.png", "image_72": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-72.png", "image_192": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-192.png", "image_512": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1675090572, "is_email_confirmed": true, "has_2fa": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1683105175461} -{"stream": "users", "data": {"id": "U04M23SBJGM", "team_id": "T04KX3KDDU6", "name": "user2.sample.airbyte", "deleted": false, "color": "5b89d5", "real_name": "User2 Sample", "tz": "Europe/Helsinki", "tz_label": "Eastern European Summer Time", "tz_offset": 10800, "profile": {"title": "", "phone": "", "skype": "", "real_name": "User2 Sample", "real_name_normalized": "User2 Sample", "display_name": "User2 Sample", "display_name_normalized": "User2 Sample", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "gce662542f72", "first_name": "User2", "last_name": "Sample", "image_24": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-24.png", "image_32": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-32.png", "image_48": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-48.png", "image_72": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-72.png", "image_192": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-192.png", "image_512": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1675092508, "is_email_confirmed": true, "has_2fa": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1683105175461} +{"stream": "channel_messages", "data": {"client_msg_id": "3ae60d35-58b8-441c-923a-75de35a4ed8a", "type": "message", "text": "Test Thread 2", "user": "U04L65GPMKN", "ts": "1683104542.931169", "blocks": [{"type": "rich_text", "block_id": "WLB", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 2"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104568.059569", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104568.059569", "channel_id": "C04KX3KEZ54", "float_ts": 1683104542.931169}, "emitted_at": 1695111199104} +{"stream": "channel_messages", "data": {"client_msg_id": "e27672c0-451e-42a6-8eff-a14d2db8ac1e", "type": "message", "text": "Test Thread 1", "user": "U04L65GPMKN", "ts": "1683104499.808709", "blocks": [{"type": "rich_text", "block_id": "0j7", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 1"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104499.808709", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104528.084359", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104528.084359", "channel_id": "C04LTCM2Y56", "float_ts": 1683104499.808709}, "emitted_at": 1695111707952} +{"stream": "threads", "data": {"client_msg_id": "3ae60d35-58b8-441c-923a-75de35a4ed8a", "type": "message", "text": "Test Thread 2", "user": "U04L65GPMKN", "ts": "1683104542.931169", "blocks": [{"type": "rich_text", "block_id": "WLB", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 2"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104568.059569", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104568.059569", "channel_id": "C04KX3KEZ54", "float_ts": 1683104542.931169}, "emitted_at": 1695110320605} +{"stream": "threads", "data": {"client_msg_id": "3e96d351-270c-493f-a1a0-fdc3c4c0e11f", "type": "message", "text": "<@U04M23SBJGM> test test test", "user": "U04L65GPMKN", "ts": "1683104559.922849", "blocks": [{"type": "rich_text", "block_id": "tX6vr", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04M23SBJGM"}, {"type": "text", "text": " test test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "parent_user_id": "U04L65GPMKN", "channel_id": "C04KX3KEZ54", "float_ts": 1683104559.922849}, "emitted_at": 1695110320606} +{"stream": "threads", "data": {"client_msg_id": "08023e44-9d18-41ed-81dd-5f04ed699656", "type": "message", "text": "<@U04LY6NARHU> test test", "user": "U04L65GPMKN", "ts": "1683104568.059569", "blocks": [{"type": "rich_text", "block_id": "IyUF", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04LY6NARHU"}, {"type": "text", "text": " test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104542.931169", "parent_user_id": "U04L65GPMKN", "channel_id": "C04KX3KEZ54", "float_ts": 1683104568.059569}, "emitted_at": 1695110320606} +{"stream": "threads", "data": {"client_msg_id": "e27672c0-451e-42a6-8eff-a14d2db8ac1e", "type": "message", "text": "Test Thread 1", "user": "U04L65GPMKN", "ts": "1683104499.808709", "blocks": [{"type": "rich_text", "block_id": "0j7", "elements": [{"type": "rich_text_section", "elements": [{"type": "text", "text": "Test Thread 1"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104499.808709", "reply_count": 2, "reply_users_count": 1, "latest_reply": "1683104528.084359", "reply_users": ["U04L65GPMKN"], "is_locked": false, "subscribed": true, "last_read": "1683104528.084359", "channel_id": "C04LTCM2Y56", "float_ts": 1683104499.808709}, "emitted_at": 1695112005658} +{"stream": "threads", "data": {"client_msg_id": "e1e2d142-a0dd-4587-86e3-2dcb439ead82", "type": "message", "text": "<@U04LY6NARHU> Test test", "user": "U04L65GPMKN", "ts": "1683104515.919709", "blocks": [{"type": "rich_text", "block_id": "xVnQ", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04LY6NARHU"}, {"type": "text", "text": " Test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104499.808709", "parent_user_id": "U04L65GPMKN", "channel_id": "C04LTCM2Y56", "float_ts": 1683104515.919709}, "emitted_at": 1695112005659} +{"stream": "threads", "data": {"client_msg_id": "ffccbb24-8dd6-476d-87bf-65e5fa033cb9", "type": "message", "text": "<@U04M23SBJGM> test test test", "user": "U04L65GPMKN", "ts": "1683104528.084359", "blocks": [{"type": "rich_text", "block_id": "Lvl", "elements": [{"type": "rich_text_section", "elements": [{"type": "user", "user_id": "U04M23SBJGM"}, {"type": "text", "text": " test test test"}]}]}], "team": "T04KX3KDDU6", "thread_ts": "1683104499.808709", "parent_user_id": "U04L65GPMKN", "channel_id": "C04LTCM2Y56", "float_ts": 1683104528.084359}, "emitted_at": 1695112005659} +{"stream": "users", "data": {"id": "USLACKBOT", "team_id": "T04KX3KDDU6", "name": "slackbot", "deleted": false, "color": "757575", "real_name": "Slackbot", "tz": "America/Los_Angeles", "tz_label": "Pacific Standard Time", "tz_offset": -28800, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Slackbot", "real_name_normalized": "Slackbot", "display_name": "Slackbot", "display_name_normalized": "Slackbot", "fields": {}, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "sv41d8cd98f0", "always_active": true, "first_name": "slackbot", "last_name": "", "image_24": "https://a.slack-edge.com/80588/img/slackbot_24.png", "image_32": "https://a.slack-edge.com/80588/img/slackbot_32.png", "image_48": "https://a.slack-edge.com/80588/img/slackbot_48.png", "image_72": "https://a.slack-edge.com/80588/img/slackbot_72.png", "image_192": "https://a.slack-edge.com/80588/marketing/img/avatars/slackbot/avatar-slackbot.png", "image_512": "https://a.slack-edge.com/80588/img/slackbot_512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 0, "is_email_confirmed": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1699645354906} +{"stream": "users", "data": {"id": "U04KUMXNYMV", "team_id": "T04KX3KDDU6", "name": "deactivateduser693438", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-24.png", "image_32": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-32.png", "image_48": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-48.png", "image_72": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-72.png", "image_192": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-192.png", "image_512": "https://secure.gravatar.com/avatar/d5320ceddda202563fd9e6222c07c00a.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0011-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090804, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354909} +{"stream": "users", "data": {"id": "U04L2KY5CES", "team_id": "T04KX3KDDU6", "name": "deactivateduser686066", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-24.png", "image_32": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-32.png", "image_48": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-48.png", "image_72": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-72.png", "image_192": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-192.png", "image_512": "https://secure.gravatar.com/avatar/cacb225265b3b19c4e72029a62cf1ef1.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0009-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090785, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354909} +{"stream": "users", "data": {"id": "U04L2LC770E", "team_id": "T04KX3KDDU6", "name": "deactivateduser521176", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-24.png", "image_32": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-32.png", "image_48": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-48.png", "image_72": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-72.png", "image_192": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-192.png", "image_512": "https://secure.gravatar.com/avatar/4f9ad3a69a21af3357625e466658e9ee.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090821, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354910} +{"stream": "users", "data": {"id": "U04L69BPZFX", "team_id": "T04KX3KDDU6", "name": "deactivateduser839125", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-24.png", "image_32": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-32.png", "image_48": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-48.png", "image_72": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-72.png", "image_192": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-192.png", "image_512": "https://secure.gravatar.com/avatar/95f67810af139bb2658d257c02efed94.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0006-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1681811889, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354910} +{"stream": "users", "data": {"id": "U04L94Y2JPM", "team_id": "T04KX3KDDU6", "name": "deactivateduser962255", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-24.png", "image_32": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-32.png", "image_48": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-48.png", "image_72": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-72.png", "image_192": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-192.png", "image_512": "https://secure.gravatar.com/avatar/e440ef9f864bc712f65ce09fb95b97ca.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0025-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1675090815, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354911} +{"stream": "users", "data": {"id": "U04LMS8F7JM", "team_id": "T04KX3KDDU6", "name": "deactivateduser421996", "deleted": true, "profile": {"title": "", "phone": "", "skype": "", "real_name": "Deactivated User", "real_name_normalized": "Deactivated User", "display_name": "deactivateduser", "display_name_normalized": "deactivateduser", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g849cc56ed76", "huddle_state": "default_unset", "first_name": "Deactivated", "last_name": "User", "image_24": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-24.png", "image_32": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-32.png", "image_48": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-48.png", "image_72": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-72.png", "image_192": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-192.png", "image_512": "https://secure.gravatar.com/avatar/931c3e24cbc7cbea399764403f3ef9bb.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0016-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_bot": false, "is_app_user": false, "updated": 1681811683, "is_forgotten": true, "is_invited_user": true}, "emitted_at": 1699645354911} +{"stream": "users", "data": {"id": "U04LY6NARHU", "team_id": "T04KX3KDDU6", "name": "user1.sample", "deleted": false, "color": "684b6c", "real_name": "User1 Sample", "tz": "Europe/Helsinki", "tz_label": "Eastern European Time", "tz_offset": 7200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "User1 Sample", "real_name_normalized": "User1 Sample", "display_name": "User1 Sample", "display_name_normalized": "User1 Sample", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "g76d12585ef1", "first_name": "User1", "last_name": "Sample", "image_24": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-24.png", "image_32": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-32.png", "image_48": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-48.png", "image_72": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-72.png", "image_192": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-192.png", "image_512": "https://secure.gravatar.com/avatar/76d12585ef1a889b0624c7fdaa20b4e3.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0026-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1675090572, "is_email_confirmed": true, "has_2fa": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1699645354911} +{"stream": "users", "data": {"id": "U04M23SBJGM", "team_id": "T04KX3KDDU6", "name": "user2.sample.airbyte", "deleted": false, "color": "5b89d5", "real_name": "User2 Sample", "tz": "Europe/Helsinki", "tz_label": "Eastern European Time", "tz_offset": 7200, "profile": {"title": "", "phone": "", "skype": "", "real_name": "User2 Sample", "real_name_normalized": "User2 Sample", "display_name": "User2 Sample", "display_name_normalized": "User2 Sample", "fields": null, "status_text": "", "status_emoji": "", "status_emoji_display_info": [], "status_expiration": 0, "avatar_hash": "gce662542f72", "first_name": "User2", "last_name": "Sample", "image_24": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=24&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-24.png", "image_32": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=32&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-32.png", "image_48": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=48&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-48.png", "image_72": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=72&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-72.png", "image_192": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=192&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-192.png", "image_512": "https://secure.gravatar.com/avatar/ce662542f721de62628c4e9c83b8904f.jpg?s=512&d=https%3A%2F%2Fa.slack-edge.com%2Fdf10d%2Fimg%2Favatars%2Fava_0012-512.png", "status_text_canonical": "", "team": "T04KX3KDDU6"}, "is_admin": false, "is_owner": false, "is_primary_owner": false, "is_restricted": false, "is_ultra_restricted": false, "is_bot": false, "is_app_user": false, "updated": 1675092508, "is_email_confirmed": true, "has_2fa": false, "who_can_share_contact_card": "EVERYONE"}, "emitted_at": 1699645354912} diff --git a/airbyte-integrations/connectors/source-slack/main.py b/airbyte-integrations/connectors/source-slack/main.py index 735ad5e72296..b2ff9c851163 100644 --- a/airbyte-integrations/connectors/source-slack/main.py +++ b/airbyte-integrations/connectors/source-slack/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_slack import SourceSlack +from source_slack.run import run if __name__ == "__main__": - source = SourceSlack() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-slack/metadata.yaml b/airbyte-integrations/connectors/source-slack/metadata.yaml index 3195f1807a01..85b8fbb337be 100644 --- a/airbyte-integrations/connectors/source-slack/metadata.yaml +++ b/airbyte-integrations/connectors/source-slack/metadata.yaml @@ -1,12 +1,18 @@ data: + ab_internal: + ql: 400 + sl: 200 allowedHosts: hosts: - slack.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: api connectorType: source definitionId: c2281cee-86f9-4a86-bb48-d23286b4c7bd - dockerImageTag: 0.2.0 + dockerImageTag: 0.3.7 dockerRepository: airbyte/source-slack + documentationUrl: https://docs.airbyte.com/integrations/sources/slack githubIssueLabel: source-slack icon: slack.svg license: MIT @@ -17,11 +23,14 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/slack + suggestedStreams: + streams: + - users + - channels + - channel_messages + - channel_members + - threads + supportLevel: certified tags: - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-slack/setup.py b/airbyte-integrations/connectors/source-slack/setup.py index 66fc79eeca33..f1040f3acca2 100644 --- a/airbyte-integrations/connectors/source-slack/setup.py +++ b/airbyte-integrations/connectors/source-slack/setup.py @@ -12,6 +12,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-slack=source_slack.run:run", + ], + }, name="source_slack", description="Source implementation for Slack.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-slack/source_slack/run.py b/airbyte-integrations/connectors/source-slack/source_slack/run.py new file mode 100644 index 000000000000..14caa9ab08e1 --- /dev/null +++ b/airbyte-integrations/connectors/source-slack/source_slack/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_slack import SourceSlack + + +def run(): + source = SourceSlack() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json b/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json index 86974bc8be5f..38e48b0dd61c 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json +++ b/airbyte-integrations/connectors/source-slack/source_slack/schemas/channel_messages.json @@ -187,6 +187,9 @@ "float_ts": { "type": ["null", "number"] }, + "is_locked": { + "type": ["null", "boolean"] + }, "type": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-slack/source_slack/schemas/channels.json b/airbyte-integrations/connectors/source-slack/source_slack/schemas/channels.json index 0c0aee8d5b28..f3401747fe71 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/schemas/channels.json +++ b/airbyte-integrations/connectors/source-slack/source_slack/schemas/channels.json @@ -17,6 +17,9 @@ "is_im": { "type": "boolean" }, + "context_team_id": { + "type": "string" + }, "created": { "type": "integer" }, @@ -32,6 +35,9 @@ "unlinked": { "type": "integer" }, + "updated": { + "type": "integer" + }, "name_normalized": { "type": "string" }, @@ -96,6 +102,12 @@ } } }, + "shared_team_ids": { + "type": "array", + "items": { + "type": ["null", "string"] + } + }, "previous_names": { "type": "array", "items": { @@ -105,6 +117,15 @@ "num_members": { "type": "integer" }, + "parent_conversation": { + "type": ["null", "string"] + }, + "pending_connected_team_ids": { + "type": "array", + "items": { + "type": ["null", "string"] + } + }, "locale": { "type": "string" } diff --git a/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json b/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json index f573873fb526..dee131fed53f 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json +++ b/airbyte-integrations/connectors/source-slack/source_slack/schemas/threads.json @@ -5,6 +5,9 @@ "channel_id": { "type": ["null", "string"] }, + "bot_id": { + "type": ["null", "string"] + }, "client_msg_id": { "type": ["null", "string"] }, @@ -23,6 +26,18 @@ "float_ts": { "type": ["null", "number"] }, + "subtype": { + "type": ["null", "string"] + }, + "is_locked": { + "type": ["null", "boolean"] + }, + "last_read": { + "type": ["null", "string"] + }, + "parent_user_id": { + "type": ["null", "string"] + }, "team": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-slack/source_slack/schemas/users.json b/airbyte-integrations/connectors/source-slack/source_slack/schemas/users.json index d4f40c1bc696..d13dce63ee13 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/schemas/users.json +++ b/airbyte-integrations/connectors/source-slack/source_slack/schemas/users.json @@ -120,9 +120,21 @@ "is_bot": { "type": "boolean" }, + "is_forgotten": { + "type": "boolean" + }, + "is_invited_user": { + "type": "boolean" + }, + "is_email_confirmed": { + "type": "boolean" + }, "updated": { "type": "integer" }, + "who_can_share_contact_card": { + "type": "string" + }, "is_app_user": { "type": "boolean" }, diff --git a/airbyte-integrations/connectors/source-slack/source_slack/source.py b/airbyte-integrations/connectors/source-slack/source_slack/source.py index c66eba276105..d59a46f5ae58 100644 --- a/airbyte-integrations/connectors/source-slack/source_slack/source.py +++ b/airbyte-integrations/connectors/source-slack/source_slack/source.py @@ -12,7 +12,6 @@ from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from pendulum import DateTime, Period @@ -28,10 +27,6 @@ def max_retries(self) -> int: # Slack's rate limiting can be unpredictable so we increase the max number of retries by a lot before failing return 20 - @property - def availability_strategy(self) -> Optional["AvailabilityStrategy"]: - return None - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: """Slack uses a cursor-based pagination strategy. Extract the cursor from the response if it exists and return it in a format @@ -214,8 +209,9 @@ def read_records( stream_state: Mapping[str, Any] = None, ) -> Iterable[Mapping[str, Any]]: if not stream_slice: + # return an empty iterator # this is done to emit at least one state message when no slices are generated - return [] + return iter([]) return super().read_records(sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state) @@ -300,18 +296,6 @@ def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Ite # yield an empty slice to checkpoint state later yield {} - def read_records(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: - """ - Filtering already read records for incremental sync. Copied state value to X after the last sync - to really 100% make sure no one can edit the state during the run. - """ - - initial_state = copy.deepcopy(stream_state) or {} - - for record in super().read_records(stream_state=stream_state, **kwargs): - if record.get(self.cursor_field, 0) >= initial_state.get(self.cursor_field, 0): - yield record - class JoinChannelsStream(HttpStream): """ @@ -380,7 +364,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: end_date = config.get("end_date") end_date = end_date and pendulum.parse(end_date) threads_lookback_window = pendulum.Duration(days=config["lookback_window"]) - channel_filter = config["channel_filter"] + channel_filter = config.get("channel_filter", []) channels = Channels(authenticator=authenticator, channel_filter=channel_filter) streams = [ diff --git a/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py b/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py index 6957dea2c22a..2fa9d3d332fe 100644 --- a/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-slack/unit_tests/conftest.py @@ -3,10 +3,13 @@ # import copy +import os from typing import MutableMapping import pytest +os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" + @pytest.fixture(autouse=True) def conversations_list(requests_mock): diff --git a/airbyte-integrations/connectors/source-smaily/README.md b/airbyte-integrations/connectors/source-smaily/README.md index 0760b9e830fc..8e0e61140e44 100644 --- a/airbyte-integrations/connectors/source-smaily/README.md +++ b/airbyte-integrations/connectors/source-smaily/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-smaily:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/smaily) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_smaily/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-smaily:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-smaily build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-smaily:airbyteDocker +An image will be built with the tag `airbyte/source-smaily:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-smaily:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-smaily:dev check --con docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-smaily:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-smaily:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-smaily test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-smaily:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-smaily:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-smaily test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/smaily.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-smaily/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-smaily/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-smaily/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-smaily/build.gradle b/airbyte-integrations/connectors/source-smaily/build.gradle deleted file mode 100644 index 92ecef1df11e..000000000000 --- a/airbyte-integrations/connectors/source-smaily/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_smaily' -} diff --git a/airbyte-integrations/connectors/source-smartengage/README.md b/airbyte-integrations/connectors/source-smartengage/README.md index 58667e02d6d3..3bc76848fb22 100644 --- a/airbyte-integrations/connectors/source-smartengage/README.md +++ b/airbyte-integrations/connectors/source-smartengage/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-smartengage:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/smartengage) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_smartengage/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-smartengage:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-smartengage build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-smartengage:airbyteDocker +An image will be built with the tag `airbyte/source-smartengage:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-smartengage:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-smartengage:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-smartengage:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-smartengage:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-smartengage test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-smartengage:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-smartengage:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-smartengage test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/smartengage.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-smartengage/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-smartengage/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-smartengage/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-smartengage/build.gradle b/airbyte-integrations/connectors/source-smartengage/build.gradle deleted file mode 100644 index 3c6f65f1bd51..000000000000 --- a/airbyte-integrations/connectors/source-smartengage/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_smartengage' -} diff --git a/airbyte-integrations/connectors/source-smartsheets/Dockerfile b/airbyte-integrations/connectors/source-smartsheets/Dockerfile index 023118cbedd7..00a47d39b74f 100644 --- a/airbyte-integrations/connectors/source-smartsheets/Dockerfile +++ b/airbyte-integrations/connectors/source-smartsheets/Dockerfile @@ -14,5 +14,5 @@ COPY $CODE_PATH ./$CODE_PATH ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.1.1 +LABEL io.airbyte.version=1.1.2 LABEL io.airbyte.name=airbyte/source-smartsheets diff --git a/airbyte-integrations/connectors/source-smartsheets/README.md b/airbyte-integrations/connectors/source-smartsheets/README.md index a1625cbc5d7f..3938470a843b 100644 --- a/airbyte-integrations/connectors/source-smartsheets/README.md +++ b/airbyte-integrations/connectors/source-smartsheets/README.md @@ -1,139 +1,67 @@ -# Smartsheets Source +# Customer Io Source -This is the repository for the Smartsheets source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/smartsheets). - -### Author -This connector was created by Nate Nowack, github.com/zzstoatzz. - -## How Airbyte connects to your Smartsheets -This version of the Airbyte-Smartsheet source connector utilizes API keys to access your source sheet(s). You can generate an API key for your account from a session of your Smartsheet webapp by clicking: - - -- Account (top-right icon) -- Apps & Integrations -- API Access -- Generate new access token - - -Airbyte will ask for this token when you configure the source connector. You will also need the ID of the Smartsheet, which you can copy from your Smartsheet app session by going to: -- File -- Properties - -Optionally you can include a list of metadata fields to be ingested by the connector, these fields are optional however the config file does require the metadata_fields be present, see sample_config.json under integrations tests for an example. - -| Supported Metadata Fields | -|------| -|sheetcreatedAt| -|sheetid| -|sheetmodifiedAt| -|sheetname| -|sheetpermalink| -|sheetversion| -|sheetaccess_level| -|row_id| -|row_access_level| -|row_created_at| -|row_created_by| -|row_expanded| -|row_modified_by| -|row_parent_id| -|row_permalink| -|row_number| -|row_version| +This is the repository for the Customer Io configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/customer-io). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-smartsheets:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/smartsheets) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_smartsheets/spec.json` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/customer-io) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_customer_io/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source smartsheets test creds` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source customer-io test creds` and place them into `secrets/config.json`. - -### Locally running the connector -``` -python main_dev.py spec -python main_dev.py check --config secrets/config.json -python main_dev.py discover --config secrets/config.json -python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-smartsheets:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name source-customer-io build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-smartsheets:airbyteDocker +An image will be built with the tag `airbyte/source-customer-io:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-customer-io:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-smartsheets:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-smartsheets:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-smartsheets:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-smartsheets:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm airbyte/source-customer-io:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-customer-io:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-customer-io:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-customer-io:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-smartsheets test ``` -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-smartsheets:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-smartsheets test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/smartsheets.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-smartsheets/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-smartsheets/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-smartsheets/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-smartsheets/build.gradle b/airbyte-integrations/connectors/source-smartsheets/build.gradle deleted file mode 100644 index 7e2e135ccc74..000000000000 --- a/airbyte-integrations/connectors/source-smartsheets/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_smartsheets' -} diff --git a/airbyte-integrations/connectors/source-smartsheets/main.py b/airbyte-integrations/connectors/source-smartsheets/main.py index 3603f2be666a..62f5650b92ea 100644 --- a/airbyte-integrations/connectors/source-smartsheets/main.py +++ b/airbyte-integrations/connectors/source-smartsheets/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_smartsheets import SourceSmartsheets +from source_smartsheets.run import run if __name__ == "__main__": - source = SourceSmartsheets() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-smartsheets/metadata.yaml b/airbyte-integrations/connectors/source-smartsheets/metadata.yaml index c88443efa7e2..cf3f522c9bf4 100644 --- a/airbyte-integrations/connectors/source-smartsheets/metadata.yaml +++ b/airbyte-integrations/connectors/source-smartsheets/metadata.yaml @@ -1,4 +1,7 @@ data: + ab_internal: + ql: 300 + sl: 100 allowedHosts: hosts: - app.smartsheet.com @@ -6,8 +9,9 @@ data: connectorSubtype: api connectorType: source definitionId: 374ebc65-6636-4ea0-925c-7d35999a8ffc - dockerImageTag: 1.1.1 + dockerImageTag: 1.1.2 dockerRepository: airbyte/source-smartsheets + documentationUrl: https://docs.airbyte.com/integrations/sources/smartsheets githubIssueLabel: source-smartsheets icon: smartsheet.svg license: MIT @@ -18,11 +22,7 @@ data: oss: enabled: true releaseStage: beta - documentationUrl: https://docs.airbyte.com/integrations/sources/smartsheets + supportLevel: community tags: - language:python - ab_internal: - sl: 200 - ql: 300 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-smartsheets/setup.py b/airbyte-integrations/connectors/source-smartsheets/setup.py index f30812c4b62f..661a68ca12be 100644 --- a/airbyte-integrations/connectors/source-smartsheets/setup.py +++ b/airbyte-integrations/connectors/source-smartsheets/setup.py @@ -9,6 +9,11 @@ TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-smartsheets=source_smartsheets.run:run", + ], + }, name="source_smartsheets", description="Source implementation for Smartsheets.", author="Nate Nowack", diff --git a/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/run.py b/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/run.py new file mode 100644 index 000000000000..6195e74166ce --- /dev/null +++ b/airbyte-integrations/connectors/source-smartsheets/source_smartsheets/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_smartsheets import SourceSmartsheets + + +def run(): + source = SourceSmartsheets() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-smartsheets/unit_tests/test_sheets.py b/airbyte-integrations/connectors/source-smartsheets/unit_tests/test_sheets.py index 2fe754607890..326e6ef71eed 100644 --- a/airbyte-integrations/connectors/source-smartsheets/unit_tests/test_sheets.py +++ b/airbyte-integrations/connectors/source-smartsheets/unit_tests/test_sheets.py @@ -17,20 +17,11 @@ def test_fetch_sheet(config, get_sheet_mocker): mock, resp = get_sheet_mocker(sheet) sheet._fetch_sheet() - mock.assert_called_once_with( - spreadsheet_id, - rows_modified_since=None, - page_size=1, - include=["rowPermalink", "writerInfo"] - ) + mock.assert_called_once_with(spreadsheet_id, rows_modified_since=None, page_size=1, include=["rowPermalink", "writerInfo"]) assert sheet.data == resp sheet._fetch_sheet(from_dt="2022-03-04T00:00:00Z") - mock.assert_called_with( - spreadsheet_id, - rows_modified_since="2022-03-04T00:00:00Z", - include=["rowPermalink", "writerInfo"] - ) + mock.assert_called_with(spreadsheet_id, rows_modified_since="2022-03-04T00:00:00Z", include=["rowPermalink", "writerInfo"]) assert sheet.data == resp @@ -124,5 +115,7 @@ def test_different_cell_order_produces_same_result(get_sheet_mocker, config, row get_sheet_mocker(sheet, data=Mock(return_value=sheet_mock)) records = sheet.read_records(from_dt="2020-01-01T00:00:00Z") - expected_records = [] if not row else [{"id": "11", "first_name": "Leonardo", "last_name": "Dicaprio", "modifiedAt": ANY, "row_id": ANY}] + expected_records = ( + [] if not row else [{"id": "11", "first_name": "Leonardo", "last_name": "Dicaprio", "modifiedAt": ANY, "row_id": ANY}] + ) assert list(records) == expected_records diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/README.md b/airbyte-integrations/connectors/source-snapchat-marketing/README.md index d03ca7655ba2..e9459deb5411 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/README.md +++ b/airbyte-integrations/connectors/source-snapchat-marketing/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-snapchat-marketing:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/snapchat-marketing) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_snapchat_marketing/spec.json` file. @@ -56,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-snapchat-marketing:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-snapchat-marketing build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-snapchat-marketing:airbyteDocker +An image will be built with the tag `airbyte/source-snapchat-marketing:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-snapchat-marketing:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -77,44 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-snapchat-marketing:dev docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-snapchat-marketing:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-snapchat-marketing:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .'[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-snapchat-marketing test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-snapchat-marketing:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-snapchat-marketing:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -124,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-snapchat-marketing test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/snapchat-marketing.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-snapchat-marketing/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-snapchat-marketing/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/build.gradle b/airbyte-integrations/connectors/source-snapchat-marketing/build.gradle deleted file mode 100644 index 48139190ea95..000000000000 --- a/airbyte-integrations/connectors/source-snapchat-marketing/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_snapchat_marketing' -} diff --git a/airbyte-integrations/connectors/source-snapchat-marketing/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-snapchat-marketing/unit_tests/unit_test.py index d35c1dbe601c..04ac41e34515 100644 --- a/airbyte-integrations/connectors/source-snapchat-marketing/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-snapchat-marketing/unit_tests/unit_test.py @@ -390,8 +390,9 @@ def test_retry_get_access_token(requests_mock): def test_should_retry_403_error(requests_mock): - requests_mock.register_uri("GET", "https://adsapi.snapchat.com/v1/me/organizations", - [{"status_code": 403, "json": {"organizations": []}}]) + requests_mock.register_uri( + "GET", "https://adsapi.snapchat.com/v1/me/organizations", [{"status_code": 403, "json": {"organizations": []}}] + ) stream = Organizations(**config_mock) records = list(stream.read_records(sync_mode=SyncMode.full_refresh)) diff --git a/airbyte-integrations/connectors/source-snowflake/.dockerignore b/airbyte-integrations/connectors/source-snowflake/.dockerignore deleted file mode 100644 index 65c7d0ad3e73..000000000000 --- a/airbyte-integrations/connectors/source-snowflake/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!Dockerfile -!build diff --git a/airbyte-integrations/connectors/source-snowflake/Dockerfile b/airbyte-integrations/connectors/source-snowflake/Dockerfile deleted file mode 100644 index 4aec528145cd..000000000000 --- a/airbyte-integrations/connectors/source-snowflake/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-snowflake - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-snowflake - -COPY --from=build /airbyte /airbyte - -LABEL io.airbyte.version=0.2.0 -LABEL io.airbyte.name=airbyte/source-snowflake diff --git a/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml b/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml index 6268f7193dba..ce05a63e5413 100644 --- a/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-snowflake/acceptance-test-config.yml @@ -10,9 +10,9 @@ acceptance_tests: tests: - config_path: "secrets/config.json" status: "succeed" - discovery: - tests: - - config_path: "secrets/config.json" + # discovery: + # tests: + # - config_path: "secrets/config.json" basic_read: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-snowflake/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-snowflake/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-snowflake/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-snowflake/build.gradle b/airbyte-integrations/connectors/source-snowflake/build.gradle index 3b8184d8b668..93784d0202dc 100644 --- a/airbyte-integrations/connectors/source-snowflake/build.gradle +++ b/airbyte-integrations/connectors/source-snowflake/build.gradle @@ -1,31 +1,41 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' - id 'airbyte-connector-acceptance-test' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.7.7' + features = ['db-sources'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.source.snowflake.SnowflakeSourceRunner' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } +configurations { + all { + resolutionStrategy { + force 'org.jooq:jooq:3.13.4' + } + } +} + dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation project(':airbyte-integrations:connectors:source-relational-db') - implementation libs.airbyte.protocol - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - implementation group: 'net.snowflake', name: 'snowflake-jdbc', version: '3.13.22' + implementation group: 'net.snowflake', name: 'snowflake-jdbc', version: '3.14.1' implementation 'com.zaxxer:HikariCP:5.0.1' - testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) - testImplementation project(':airbyte-test-utils') - - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-snowflake') - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + testImplementation 'org.hamcrest:hamcrest-all:1.3' + integrationTestJavaImplementation libs.testcontainers.jdbc integrationTestJavaImplementation 'org.apache.commons:commons-lang3:3.11' } - diff --git a/airbyte-integrations/connectors/source-snowflake/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-snowflake/integration_tests/expected_records.jsonl index 1e489309c367..efab00f9bec2 100644 --- a/airbyte-integrations/connectors/source-snowflake/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-snowflake/integration_tests/expected_records.jsonl @@ -1,7 +1,7 @@ -{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 1, "TEST_COLUMN_1" : 99999999999999999999999999999999999999, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : -9007199254740990.0, "TEST_COLUMN_12" : 1e-307, "TEST_COLUMN_14" : "\u0442\u0435\u0441\u0442", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-!-", "TEST_COLUMN_17" : "a", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : true, "TEST_COLUMN_2" : 9223372036854775807, "TEST_COLUMN_20" : "0001-01-01", "TEST_COLUMN_21" : "0001-01-01T00:00:00.000000", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123", "TEST_COLUMN_24" : "2018-03-22T07:00:00.123000Z", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123", "TEST_COLUMN_26" : "2018-03-22T07:00:00.123000Z", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : 99999999999999999999999999999999999999, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n -122.35,\n 37.55\n ],\n \"type\": \"Point\"\n}", "TEST_COLUMN_4" : 99999999999999999999999999999999999999, "TEST_COLUMN_5" : 9223372036854775807, "TEST_COLUMN_6" : 9223372036854775807, "TEST_COLUMN_7" : 9223372036854775807, "TEST_COLUMN_8" : 9223372036854775807, "TEST_COLUMN_9" : 9223372036854775807 }, "emitted_at" : 1670334357227} -{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 2, "TEST_COLUMN_1" : -99999999999999999999999999999999999999, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : 9007199254740990.0, "TEST_COLUMN_12" : 1e+308, "TEST_COLUMN_14" : "\u26a1 test \ufffd\ufffd", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-%-", "TEST_COLUMN_17" : "\u30b9", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : true, "TEST_COLUMN_2" : -9223372036854775808, "TEST_COLUMN_20" : "9999-12-31", "TEST_COLUMN_21" : "9999-12-31T23:59:59", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_24" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_26" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : -99999999999999999999999999999999999999, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n [\n -124.2,\n 42\n ],\n [\n -120.01,\n 41.99\n ]\n ],\n \"type\": \"LineString\"\n}", "TEST_COLUMN_4" : -99999999999999999999999999999999999999, "TEST_COLUMN_5" : -9223372036854775808, "TEST_COLUMN_6" : -9223372036854775808, "TEST_COLUMN_7" : -9223372036854775808, "TEST_COLUMN_8" : -9223372036854775808, "TEST_COLUMN_9" : -9223372036854775808 }, "emitted_at" : 1670334357227} -{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 3, "TEST_COLUMN_1" : 9223372036854775807, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : 9007199254740990.0, "TEST_COLUMN_12" : 1e+308, "TEST_COLUMN_14" : "!\"#$%&'()*+,-./:;<=>?@[]^_`{|}~", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-%-", "TEST_COLUMN_17" : "!", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : false, "TEST_COLUMN_2" : -9223372036854775808, "TEST_COLUMN_20" : "9999-12-31", "TEST_COLUMN_21" : "9999-12-31T23:59:59.123456", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_24" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_26" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : 9223372036854775807, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n [\n -124.2,\n 42\n ],\n [\n -120.01,\n 41.99\n ]\n ],\n \"type\": \"LineString\"\n}", "TEST_COLUMN_4" : -99999999999999999999999999999999999999, "TEST_COLUMN_5" : -9223372036854775808, "TEST_COLUMN_6" : -9223372036854775808, "TEST_COLUMN_7" : -9223372036854775808, "TEST_COLUMN_8" : -9223372036854775808, "TEST_COLUMN_9" : -9223372036854775808 }, "emitted_at" : 1670334357227} -{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 4, "TEST_COLUMN_1" : -9223372036854775808, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : 9007199254740990.0, "TEST_COLUMN_12" : 1e+308, "TEST_COLUMN_14" : "!\"#$%&'()*+,-./:;<=>?@[]^_`{|}~", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-%-", "TEST_COLUMN_17" : "\u0457", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : false, "TEST_COLUMN_2" : -9223372036854775808, "TEST_COLUMN_20" : "9999-12-31", "TEST_COLUMN_21" : "9999-12-31T23:59:59.123456", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_24" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_26" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : -9223372036854775808, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n [\n -124.2,\n 42\n ],\n [\n -120.01,\n 41.99\n ]\n ],\n \"type\": \"LineString\"\n}", "TEST_COLUMN_4" : -99999999999999999999999999999999999999, "TEST_COLUMN_5" : -9223372036854775808, "TEST_COLUMN_6" : -9223372036854775808, "TEST_COLUMN_7" : -9223372036854775808, "TEST_COLUMN_8" : -9223372036854775808, "TEST_COLUMN_9" : -9223372036854775808 }, "emitted_at" : 1670334357227} -{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 5, "TEST_COLUMN_1" : -9223372036854775808, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : 9007199254740990.0, "TEST_COLUMN_12" : 1e+308, "TEST_COLUMN_14" : "!\"#$%&'()*+,-./:;<=>?@[]^_`{|}~", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-%-", "TEST_COLUMN_17" : "\u0457", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : true, "TEST_COLUMN_2" : -9223372036854775808, "TEST_COLUMN_20" : "9999-12-31", "TEST_COLUMN_21" : "9999-12-31T23:59:59.123456", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_24" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_26" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : -9223372036854775808, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n [\n -124.2,\n 42\n ],\n [\n -120.01,\n 41.99\n ]\n ],\n \"type\": \"LineString\"\n}", "TEST_COLUMN_4" : -99999999999999999999999999999999999999, "TEST_COLUMN_5" : -9223372036854775808, "TEST_COLUMN_6" : -9223372036854775808, "TEST_COLUMN_7" : -9223372036854775808, "TEST_COLUMN_8" : -9223372036854775808, "TEST_COLUMN_9" : -9223372036854775808 }, "emitted_at" : 1670334357227} -{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 6, "TEST_COLUMN_1" : -9223372036854775808, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : 9007199254740990.0, "TEST_COLUMN_12" : 1e+308, "TEST_COLUMN_14" : "!\"#$%&'()*+,-./:;<=>?@[]^_`{|}~", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-%-", "TEST_COLUMN_17" : "\u0457", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : false, "TEST_COLUMN_2" : -9223372036854775808, "TEST_COLUMN_20" : "9999-12-31", "TEST_COLUMN_21" : "9999-12-31T23:59:59.123456", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_24" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_26" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : -9223372036854775808, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n [\n -124.2,\n 42\n ],\n [\n -120.01,\n 41.99\n ]\n ],\n \"type\": \"LineString\"\n}", "TEST_COLUMN_4" : -99999999999999999999999999999999999999, "TEST_COLUMN_5" : -9223372036854775808, "TEST_COLUMN_6" : -9223372036854775808, "TEST_COLUMN_7" : -9223372036854775808, "TEST_COLUMN_8" : -9223372036854775808, "TEST_COLUMN_9" : -9223372036854775808 }, "emitted_at" : 1670334357227} -{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 7, "TEST_COLUMN_1" : -9223372036854775808, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : 9007199254740990.0, "TEST_COLUMN_12" : 1e+308, "TEST_COLUMN_14" : "!\"#$%&'()*+,-./:;<=>?@[]^_`{|}~", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-%-", "TEST_COLUMN_17" : "\u0457", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : false, "TEST_COLUMN_2" : -9223372036854775808, "TEST_COLUMN_20" : "9999-12-31", "TEST_COLUMN_21" : "9999-12-31T23:59:59.123456", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_24" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_26" : "2018-03-22T07:00:00.123456Z", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : -9223372036854775808, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n [\n -124.2,\n 42\n ],\n [\n -120.01,\n 41.99\n ]\n ],\n \"type\": \"LineString\"\n}", "TEST_COLUMN_4" : -99999999999999999999999999999999999999, "TEST_COLUMN_5" : -9223372036854775808, "TEST_COLUMN_6" : -9223372036854775808, "TEST_COLUMN_7" : -9223372036854775808, "TEST_COLUMN_8" : -9223372036854775808, "TEST_COLUMN_9" : -9223372036854775808 }, "emitted_at" : 1670334357227} +{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 1, "TEST_COLUMN_1" : 99999999999999999999999999999999999999, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : -9007199254740990.0, "TEST_COLUMN_12" : 1e-307, "TEST_COLUMN_14" : "\u0442\u0435\u0441\u0442", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-!-", "TEST_COLUMN_17" : "a", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : true, "TEST_COLUMN_2" : 9223372036854775807, "TEST_COLUMN_20" : "0001-01-01", "TEST_COLUMN_21" : "0001-01-01T00:00:00.000000", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123", "TEST_COLUMN_24" : "2018-03-22T00:00:00.123000-07:00", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123", "TEST_COLUMN_26" : "2018-03-22T12:00:00.123000+05:00", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : 99999999999999999999999999999999999999, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n -122.35,\n 37.55\n ],\n \"type\": \"Point\"\n}", "TEST_COLUMN_4" : 99999999999999999999999999999999999999, "TEST_COLUMN_5" : 9223372036854775807, "TEST_COLUMN_6" : 9223372036854775807, "TEST_COLUMN_7" : 9223372036854775807, "TEST_COLUMN_8" : 9223372036854775807, "TEST_COLUMN_9" : 9223372036854775807 }, "emitted_at" : 1670334357227} +{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 2, "TEST_COLUMN_1" : -99999999999999999999999999999999999999, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : 9007199254740990.0, "TEST_COLUMN_12" : 1e+308, "TEST_COLUMN_14" : "\u26a1 test \ufffd\ufffd", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-%-", "TEST_COLUMN_17" : "\u30b9", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : true, "TEST_COLUMN_2" : -9223372036854775808, "TEST_COLUMN_20" : "9999-12-31", "TEST_COLUMN_21" : "9999-12-31T23:59:59", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_24" : "2018-03-22T00:00:00.123000-07:00", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_26" : "2018-03-22T12:00:00.123000+05:00", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : -99999999999999999999999999999999999999, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n [\n -124.2,\n 42\n ],\n [\n -120.01,\n 41.99\n ]\n ],\n \"type\": \"LineString\"\n}", "TEST_COLUMN_4" : -99999999999999999999999999999999999999, "TEST_COLUMN_5" : -9223372036854775808, "TEST_COLUMN_6" : -9223372036854775808, "TEST_COLUMN_7" : -9223372036854775808, "TEST_COLUMN_8" : -9223372036854775808, "TEST_COLUMN_9" : -9223372036854775808 }, "emitted_at" : 1670334357227} +{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 3, "TEST_COLUMN_1" : 9223372036854775807, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : 9007199254740990.0, "TEST_COLUMN_12" : 1e+308, "TEST_COLUMN_14" : "!\"#$%&'()*+,-./:;<=>?@[]^_`{|}~", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-%-", "TEST_COLUMN_17" : "!", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : false, "TEST_COLUMN_2" : -9223372036854775808, "TEST_COLUMN_20" : "9999-12-31", "TEST_COLUMN_21" : "9999-12-31T23:59:59.123456", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_24" : "2018-03-22T00:00:00.123000-07:00", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_26" : "2018-03-22T12:00:00.123000+05:00", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : 9223372036854775807, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n [\n -124.2,\n 42\n ],\n [\n -120.01,\n 41.99\n ]\n ],\n \"type\": \"LineString\"\n}", "TEST_COLUMN_4" : -99999999999999999999999999999999999999, "TEST_COLUMN_5" : -9223372036854775808, "TEST_COLUMN_6" : -9223372036854775808, "TEST_COLUMN_7" : -9223372036854775808, "TEST_COLUMN_8" : -9223372036854775808, "TEST_COLUMN_9" : -9223372036854775808 }, "emitted_at" : 1670334357227} +{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 4, "TEST_COLUMN_1" : -9223372036854775808, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : 9007199254740990.0, "TEST_COLUMN_12" : 1e+308, "TEST_COLUMN_14" : "!\"#$%&'()*+,-./:;<=>?@[]^_`{|}~", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-%-", "TEST_COLUMN_17" : "\u0457", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : false, "TEST_COLUMN_2" : -9223372036854775808, "TEST_COLUMN_20" : "9999-12-31", "TEST_COLUMN_21" : "9999-12-31T23:59:59.123456", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_24" : "2018-03-22T00:00:00.123000-07:00", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_26" : "2018-03-22T12:00:00.123000+05:00", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : -9223372036854775808, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n [\n -124.2,\n 42\n ],\n [\n -120.01,\n 41.99\n ]\n ],\n \"type\": \"LineString\"\n}", "TEST_COLUMN_4" : -99999999999999999999999999999999999999, "TEST_COLUMN_5" : -9223372036854775808, "TEST_COLUMN_6" : -9223372036854775808, "TEST_COLUMN_7" : -9223372036854775808, "TEST_COLUMN_8" : -9223372036854775808, "TEST_COLUMN_9" : -9223372036854775808 }, "emitted_at" : 1670334357227} +{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 5, "TEST_COLUMN_1" : -9223372036854775808, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : 9007199254740990.0, "TEST_COLUMN_12" : 1e+308, "TEST_COLUMN_14" : "!\"#$%&'()*+,-./:;<=>?@[]^_`{|}~", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-%-", "TEST_COLUMN_17" : "\u0457", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : true, "TEST_COLUMN_2" : -9223372036854775808, "TEST_COLUMN_20" : "9999-12-31", "TEST_COLUMN_21" : "9999-12-31T23:59:59.123456", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_24" : "2018-03-22T00:00:00.123000-07:00", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_26" : "2018-03-22T12:00:00.123000+05:00", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : -9223372036854775808, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n [\n -124.2,\n 42\n ],\n [\n -120.01,\n 41.99\n ]\n ],\n \"type\": \"LineString\"\n}", "TEST_COLUMN_4" : -99999999999999999999999999999999999999, "TEST_COLUMN_5" : -9223372036854775808, "TEST_COLUMN_6" : -9223372036854775808, "TEST_COLUMN_7" : -9223372036854775808, "TEST_COLUMN_8" : -9223372036854775808, "TEST_COLUMN_9" : -9223372036854775808 }, "emitted_at" : 1670334357227} +{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 6, "TEST_COLUMN_1" : -9223372036854775808, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : 9007199254740990.0, "TEST_COLUMN_12" : 1e+308, "TEST_COLUMN_14" : "!\"#$%&'()*+,-./:;<=>?@[]^_`{|}~", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-%-", "TEST_COLUMN_17" : "\u0457", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : false, "TEST_COLUMN_2" : -9223372036854775808, "TEST_COLUMN_20" : "9999-12-31", "TEST_COLUMN_21" : "9999-12-31T23:59:59.123456", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_24" : "2018-03-22T00:00:00.123000-07:00", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_26" : "2018-03-22T12:00:00.123000+05:00", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : -9223372036854775808, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n [\n -124.2,\n 42\n ],\n [\n -120.01,\n 41.99\n ]\n ],\n \"type\": \"LineString\"\n}", "TEST_COLUMN_4" : -99999999999999999999999999999999999999, "TEST_COLUMN_5" : -9223372036854775808, "TEST_COLUMN_6" : -9223372036854775808, "TEST_COLUMN_7" : -9223372036854775808, "TEST_COLUMN_8" : -9223372036854775808, "TEST_COLUMN_9" : -9223372036854775808 }, "emitted_at" : 1670334357227} +{ "stream" : "SAT_BASIC_DATASET", "data" : { "ID" : 7, "TEST_COLUMN_1" : -9223372036854775808, "TEST_COLUMN_10" : 10.12345, "TEST_COLUMN_11" : 9007199254740990.0, "TEST_COLUMN_12" : 1e+308, "TEST_COLUMN_14" : "!\"#$%&'()*+,-./:;<=>?@[]^_`{|}~", "TEST_COLUMN_15" : "\u30c6\u30b9\u30c8", "TEST_COLUMN_16" : "-%-", "TEST_COLUMN_17" : "\u0457", "TEST_COLUMN_18" : "SEVMUA==", "TEST_COLUMN_19" : false, "TEST_COLUMN_2" : -9223372036854775808, "TEST_COLUMN_20" : "9999-12-31", "TEST_COLUMN_21" : "9999-12-31T23:59:59.123456", "TEST_COLUMN_23" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_24" : "2018-03-22T00:00:00.123000-07:00", "TEST_COLUMN_25" : "2018-03-22T12:00:00.123456", "TEST_COLUMN_26" : "2018-03-22T12:00:00.123000+05:00", "TEST_COLUMN_27" : "{\n \"key1\": \"value1\",\n \"key2\": \"value2\"\n}", "TEST_COLUMN_28" : "[\n 1,\n 2,\n 3\n]", "TEST_COLUMN_29" : "{\n \"outer_key1\": {\n \"inner_key1A\": \"1a\",\n \"inner_key1B\": \"1b\"\n },\n \"outer_key2\": {\n \"inner_key2\": 2\n }\n}", "TEST_COLUMN_3" : -9223372036854775808, "TEST_COLUMN_30" : "{\n \"coordinates\": [\n [\n -124.2,\n 42\n ],\n [\n -120.01,\n 41.99\n ]\n ],\n \"type\": \"LineString\"\n}", "TEST_COLUMN_4" : -99999999999999999999999999999999999999, "TEST_COLUMN_5" : -9223372036854775808, "TEST_COLUMN_6" : -9223372036854775808, "TEST_COLUMN_7" : -9223372036854775808, "TEST_COLUMN_8" : -9223372036854775808, "TEST_COLUMN_9" : -9223372036854775808 }, "emitted_at" : 1670334357227} diff --git a/airbyte-integrations/connectors/source-snowflake/metadata.yaml b/airbyte-integrations/connectors/source-snowflake/metadata.yaml index deb9bc02d5a1..9b9bc3661ee2 100644 --- a/airbyte-integrations/connectors/source-snowflake/metadata.yaml +++ b/airbyte-integrations/connectors/source-snowflake/metadata.yaml @@ -8,7 +8,7 @@ data: connectorSubtype: database connectorType: source definitionId: e2d65910-8c8b-40a1-ae7d-ee2416b2bfa2 - dockerImageTag: 0.2.0 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-snowflake documentationUrl: https://docs.airbyte.com/integrations/sources/snowflake githubIssueLabel: source-snowflake diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeDataSourceUtils.java b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeDataSourceUtils.java index bc2bc0990eef..6c0177a049b8 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeDataSourceUtils.java +++ b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeDataSourceUtils.java @@ -9,8 +9,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; import java.io.IOException; import java.net.URI; import java.net.URLEncoder; diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java index b5187e545072..c3c0214b1e1d 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java +++ b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSource.java @@ -10,14 +10,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.jdbc.StreamingJdbcDatabase; +import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.StreamingJdbcDatabase; -import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import java.io.IOException; import java.sql.JDBCType; import java.sql.SQLException; diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java index 15e4f98e9e06..7713d781fbe3 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java +++ b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceOperations.java @@ -4,17 +4,18 @@ package io.airbyte.integrations.source.snowflake; -import static io.airbyte.db.jdbc.DateTimeConverter.putJavaSQLTime; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_SCHEMA_NAME; -import static io.airbyte.db.jdbc.JdbcConstants.INTERNAL_TABLE_NAME; +import static io.airbyte.cdk.db.jdbc.DateTimeConverter.putJavaSQLTime; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_COLUMN_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_COLUMN_TYPE_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_SCHEMA_NAME; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.INTERNAL_TABLE_NAME; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.db.jdbc.DateTimeConverter; -import io.airbyte.db.jdbc.JdbcSourceOperations; +import io.airbyte.cdk.db.DataTypeUtils; +import io.airbyte.cdk.db.jdbc.DateTimeConverter; +import io.airbyte.cdk.db.jdbc.JdbcSourceOperations; import io.airbyte.protocol.models.JsonSchemaType; import java.math.BigDecimal; import java.sql.Date; @@ -24,6 +25,9 @@ import java.sql.SQLException; import java.sql.Timestamp; import java.time.LocalDate; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -121,11 +125,22 @@ protected void setDate(final PreparedStatement preparedStatement, final int para preparedStatement.setDate(parameterIndex, Date.valueOf(date)); } + private static final DateTimeFormatter SNOWFLAKE_TIMESTAMPTZ_FORMATTER = new DateTimeFormatterBuilder() + .parseCaseInsensitive() + .append(DateTimeFormatter.ISO_LOCAL_DATE) + .appendLiteral(' ') + .append(DateTimeFormatter.ISO_LOCAL_TIME) + .optionalStart() + .appendLiteral(' ') + .append(DateTimeFormatter.ofPattern("XX")) + .toFormatter(); + @Override protected void putTimestampWithTimezone(final ObjectNode node, final String columnName, final ResultSet resultSet, final int index) throws SQLException { - final Timestamp timestamp = resultSet.getTimestamp(index); - node.put(columnName, DateTimeConverter.convertToTimestampWithTimezone(timestamp)); + final String timestampAsString = resultSet.getString(index); + OffsetDateTime timestampWithOffset = OffsetDateTime.parse(timestampAsString, SNOWFLAKE_TIMESTAMPTZ_FORMATTER); + node.put(columnName, timestampWithOffset.format(DataTypeUtils.TIMESTAMPTZ_FORMATTER)); } @Override diff --git a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceRunner.java b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceRunner.java index a033224f60e0..6b5223435337 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceRunner.java +++ b/airbyte-integrations/connectors/source-snowflake/src/main/java/io.airbyte.integrations.source.snowflake/SnowflakeSourceRunner.java @@ -8,7 +8,7 @@ import static io.airbyte.integrations.source.snowflake.SnowflakeDataSourceUtils.AIRBYTE_OSS; import static io.airbyte.integrations.source.snowflake.SnowflakeSource.SCHEDULED_EXECUTOR_SERVICE; -import io.airbyte.integrations.base.adaptive.AdaptiveSourceRunner; +import io.airbyte.cdk.integrations.base.adaptive.AdaptiveSourceRunner; public class SnowflakeSourceRunner { diff --git a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeJdbcSourceAcceptanceTest.java index ebde43d6b6ab..012c71bb208b 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeJdbcSourceAcceptanceTest.java @@ -13,39 +13,29 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import io.airbyte.cdk.integrations.source.relationaldb.RelationalDbQueryUtils; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.string.Strings; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; -import io.airbyte.integrations.source.relationaldb.RelationalDbQueryUtils; -import io.airbyte.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.integrations.source.snowflake.SnowflakeSource; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.AirbyteCatalog; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus.Status; -import io.airbyte.protocol.models.v0.AirbyteMessage; -import io.airbyte.protocol.models.v0.AirbyteRecordMessage; import io.airbyte.protocol.models.v0.CatalogHelpers; import io.airbyte.protocol.models.v0.SyncMode; import java.math.BigDecimal; import java.nio.file.Path; -import java.sql.JDBCType; -import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -class SnowflakeJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { +class SnowflakeJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { private static JsonNode snConfig; @@ -78,67 +68,73 @@ static void init() { INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES(true)"; } - @BeforeEach - public void setup() throws Exception { - super.setup(); - } - - @AfterEach - public void clean() throws Exception { - super.tearDown(); - DataSourceFactory.close(dataSource); - } - @Override public boolean supportsSchemas() { return true; } @Override - public JsonNode getConfig() { + protected JsonNode config() { return Jsons.clone(snConfig); } @Override - public String getDriverClass() { - return SnowflakeSource.DRIVER_CLASS; + protected SnowflakeTestDatabase createTestDatabase() { + final SnowflakeTestDatabase snowflakeTestDatabase = new SnowflakeTestDatabase(source().toDatabaseConfig(Jsons.clone(snConfig))); + for (final String schemaName : TEST_SCHEMAS) { + snowflakeTestDatabase.onClose(DROP_SCHEMA_QUERY, schemaName); + } + return snowflakeTestDatabase.initialized(); } @Override - public AbstractJdbcSource getJdbcSource() { + protected SnowflakeSource source() { return new SnowflakeSource(AIRBYTE_OSS); } @Test - void testCheckFailure() throws Exception { + @Override + protected void testCheckFailure() throws Exception { + final JsonNode config = config(); ((ObjectNode) config).with("credentials").put(JdbcUtils.PASSWORD_KEY, "fake"); - final AirbyteConnectionStatus status = source.check(config); - assertEquals(Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 08001; Error code: 390100;")); + try (SnowflakeSource source = source()) { + final AirbyteConnectionStatus status = source.check(config); + assertEquals(Status.FAILED, status.getStatus()); + assertTrue(status.getMessage().contains("State code: 08001; Error code: 390100;")); + } } @Test public void testCheckIncorrectUsernameFailure() throws Exception { + final JsonNode config = config(); ((ObjectNode) config).with("credentials").put(JdbcUtils.USERNAME_KEY, "fake"); - final AirbyteConnectionStatus status = source.check(config); - assertEquals(Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 08001; Error code: 390100;")); + try (SnowflakeSource source = source()) { + final AirbyteConnectionStatus status = source.check(config); + assertEquals(Status.FAILED, status.getStatus()); + assertTrue(status.getMessage().contains("State code: 08001; Error code: 390100;")); + } } @Test public void testCheckEmptyUsernameFailure() throws Exception { + final JsonNode config = config(); ((ObjectNode) config).with("credentials").put(JdbcUtils.USERNAME_KEY, ""); - final AirbyteConnectionStatus status = source.check(config); - assertEquals(Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("State code: 28000; Error code: 200011;")); + try (SnowflakeSource source = source()) { + final AirbyteConnectionStatus status = source.check(config); + assertEquals(Status.FAILED, status.getStatus()); + assertTrue(status.getMessage().contains("State code: 28000; Error code: 200011;")); + } } @Test public void testCheckIncorrectHostFailure() throws Exception { + final JsonNode config = config(); ((ObjectNode) config).put(JdbcUtils.HOST_KEY, "localhost2"); - final AirbyteConnectionStatus status = source.check(config); - assertEquals(Status.FAILED, status.getStatus()); - assertTrue(status.getMessage().contains("Could not connect with provided configuration")); + try (SnowflakeSource source = source()) { + final AirbyteConnectionStatus status = source.check(config); + assertEquals(Status.FAILED, status.getStatus()); + assertTrue(status.getMessage().contains("Could not connect with provided configuration")); + } } @Override @@ -171,30 +167,6 @@ protected AirbyteCatalog getCatalog(final String defaultNamespace) { List.of(List.of(COL_FIRST_NAME), List.of(COL_LAST_NAME))))); } - @Override - protected List getTestMessages() { - return List.of( - new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_1, - COL_NAME, "picard", - COL_UPDATED_AT, "2004-10-19")))), - new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_2, - COL_NAME, "crusher", - COL_UPDATED_AT, - "2005-10-19")))), - new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_3, - COL_NAME, "vash", - COL_UPDATED_AT, "2006-10-19"))))); - } - @Override protected void incrementalDateCheck() throws Exception { super.incrementalCursorCheck(COL_UPDATED_AT, @@ -204,73 +176,33 @@ protected void incrementalDateCheck() throws Exception { getTestMessages().get(2))); } - @Override - protected List getExpectedAirbyteMessagesSecondSync(final String namespace) { - final List expectedMessages = new ArrayList<>(); - expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_4, - COL_NAME, "riker", - COL_UPDATED_AT, "2006-10-19"))))); - expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) - .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) - .withData(Jsons.jsonNode(Map - .of(COL_ID, ID_VALUE_5, - COL_NAME, "data", - COL_UPDATED_AT, "2006-10-19"))))); - final DbStreamState state = new DbStreamState() - .withStreamName(streamName) - .withStreamNamespace(namespace) - .withCursorField(List.of(COL_ID)) - .withCursor("5") - .withCursorRecordCount(1L); - expectedMessages.addAll(createExpectedTestMessages(List.of(state))); - return expectedMessages; - } - /* Test that schema config key is making discover pull tables of this schema only */ @Test void testDiscoverSchemaConfig() throws Exception { - // add table and data to a separate schema. - database.execute(connection -> { - connection.createStatement().execute( - String.format("CREATE TABLE %s(id VARCHAR(200) NOT NULL, name VARCHAR(200) NOT NULL)", - RelationalDbQueryUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))); - connection.createStatement() - .execute(String.format("INSERT INTO %s(id, name) VALUES ('1','picard')", - RelationalDbQueryUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))); - connection.createStatement() - .execute(String.format("INSERT INTO %s(id, name) VALUES ('2', 'crusher')", - RelationalDbQueryUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))); - connection.createStatement() - .execute(String.format("INSERT INTO %s(id, name) VALUES ('3', 'vash')", - RelationalDbQueryUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))); - connection.createStatement().execute( - String.format("CREATE TABLE %s(id VARCHAR(200) NOT NULL, name VARCHAR(200) NOT NULL)", - RelationalDbQueryUtils.getFullyQualifiedTableName(SCHEMA_NAME, Strings.addRandomSuffix(TABLE_NAME, "_", 4)))); - }); + // add table to a separate schema. + testdb.with(String.format("CREATE TABLE %s(id VARCHAR(200) NOT NULL, name VARCHAR(200) NOT NULL)", + RelationalDbQueryUtils.getFullyQualifiedTableName(SCHEMA_NAME2, TABLE_NAME))) + .with(String.format("CREATE TABLE %s(id VARCHAR(200) NOT NULL, name VARCHAR(200) NOT NULL)", + RelationalDbQueryUtils.getFullyQualifiedTableName(SCHEMA_NAME, Strings.addRandomSuffix(TABLE_NAME, "_", 4)))); + final JsonNode config = config(); JsonNode confWithSchema = ((ObjectNode) config).put("schema", SCHEMA_NAME); - AirbyteCatalog actual = source.discover(confWithSchema); - - assertFalse(actual.getStreams().isEmpty()); + try (SnowflakeSource source = source()) { + AirbyteCatalog actual = source.discover(confWithSchema); - var streams = actual.getStreams().stream() - .filter(s -> !s.getNamespace().equals(SCHEMA_NAME)) - .collect(Collectors.toList()); + assertFalse(actual.getStreams().isEmpty()); - assertTrue(streams.isEmpty()); + var streams = actual.getStreams().stream().filter(s -> !s.getNamespace().equals(SCHEMA_NAME)).collect(Collectors.toList()); - confWithSchema = ((ObjectNode) config).put("schema", SCHEMA_NAME2); - actual = source.discover(confWithSchema); - assertFalse(actual.getStreams().isEmpty()); + assertTrue(streams.isEmpty()); - streams = actual.getStreams().stream() - .filter(s -> !s.getNamespace().equals(SCHEMA_NAME2)) - .collect(Collectors.toList()); + confWithSchema = ((ObjectNode) config).put("schema", SCHEMA_NAME2); + actual = source.discover(confWithSchema); + assertFalse(actual.getStreams().isEmpty()); - assertTrue(streams.isEmpty()); + streams = actual.getStreams().stream().filter(s -> !s.getNamespace().equals(SCHEMA_NAME2)).collect(Collectors.toList()); + assertTrue(streams.isEmpty()); + } } } diff --git a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAcceptanceTest.java index cc21c788fff1..dfbbbf21cfb6 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAcceptanceTest.java @@ -9,17 +9,17 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.DefaultJdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.DefaultJdbcDatabase; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.source.snowflake.SnowflakeSource; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.CatalogHelpers; @@ -29,6 +29,7 @@ import io.airbyte.protocol.models.v0.DestinationSyncMode; import io.airbyte.protocol.models.v0.SyncMode; import java.nio.file.Path; +import java.time.Duration; import java.util.HashMap; import java.util.Map; import javax.sql.DataSource; @@ -41,6 +42,7 @@ public class SnowflakeSourceAcceptanceTest extends SourceAcceptanceTest { + RandomStringUtils.randomAlphanumeric(4).toUpperCase(); private static final String STREAM_NAME1 = "ID_AND_NAME1"; private static final String STREAM_NAME2 = "ID_AND_NAME2"; + private static final Duration CONNECTION_TIMEOUT = Duration.ofSeconds(60); // config which refers to the schema that the test is being run in. protected JsonNode config; @@ -142,7 +144,8 @@ protected DataSource createDataSource() { String.format(DatabaseDriver.SNOWFLAKE.getUrlFormatString(), config.get(JdbcUtils.HOST_KEY).asText()), Map.of("role", config.get("role").asText(), "warehouse", config.get("warehouse").asText(), - JdbcUtils.DATABASE_KEY, config.get(JdbcUtils.DATABASE_KEY).asText())); + JdbcUtils.DATABASE_KEY, config.get(JdbcUtils.DATABASE_KEY).asText()), + CONNECTION_TIMEOUT); } @Test diff --git a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAuthAcceptanceTest.java b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAuthAcceptanceTest.java index acd668ccbabf..8b3408cf22a9 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAuthAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceAuthAcceptanceTest.java @@ -7,9 +7,9 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.zaxxer.hikari.HikariDataSource; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.source.snowflake.SnowflakeDataSourceUtils; import java.io.IOException; import java.nio.file.Path; diff --git a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java index e2521c7290ed..ba4b784f2f46 100644 --- a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeSourceDatatypeTest.java @@ -6,18 +6,19 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDataHolder; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.source.snowflake.SnowflakeSource; -import io.airbyte.integrations.standardtest.source.AbstractSourceDatabaseTypeTest; -import io.airbyte.integrations.standardtest.source.TestDataHolder; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.JsonSchemaType; import java.nio.file.Path; +import java.time.Duration; import java.util.Map; import org.apache.commons.lang3.RandomStringUtils; import org.jooq.DSLContext; @@ -28,6 +29,7 @@ public class SnowflakeSourceDatatypeTest extends AbstractSourceDatabaseTypeTest private static final String SCHEMA_NAME = "SOURCE_DATA_TYPE_TEST_" + RandomStringUtils.randomAlphanumeric(4).toUpperCase(); private static final String INSERT_SEMI_STRUCTURED_SQL = "INSERT INTO %1$s (ID, TEST_COLUMN) SELECT %2$s, %3$s"; + private static final Duration CONNECTION_TIMEOUT = Duration.ofSeconds(60); private JsonNode config; private Database database; @@ -57,7 +59,8 @@ protected Database setupDatabase() throws Exception { Map.of( "role", config.get("role").asText(), "warehouse", config.get("warehouse").asText(), - JdbcUtils.DATABASE_KEY, config.get(JdbcUtils.DATABASE_KEY).asText())); + JdbcUtils.DATABASE_KEY, config.get(JdbcUtils.DATABASE_KEY).asText()), + CONNECTION_TIMEOUT); database = getDatabase(); @@ -285,30 +288,38 @@ protected void initTests() { TestDataHolder.builder() .sourceType("TIMESTAMP") .airbyteType(JsonSchemaType.STRING_TIMESTAMP_WITHOUT_TIMEZONE) - .addInsertValues("null", "'2018-03-22 12:00:00.123'", "'2018-03-22 12:00:00.123456'") - .addExpectedValues(null, "2018-03-22T12:00:00.123", "2018-03-22T12:00:00.123456") - .build()); + .addInsertValues("null", "'2018-03-26 12:00:00.123'", "'2018-03-26 12:00:00.123456'") + .addExpectedValues(null, "2018-03-26T12:00:00.123", "2018-03-26T12:00:00.123456") + .build());// This is very brittle. A change of parameters on the customer's account could change the values + // returned by snowflake addDataTypeTestData( TestDataHolder.builder() .sourceType("TIMESTAMP_LTZ") .airbyteType(JsonSchemaType.STRING_TIMESTAMP_WITH_TIMEZONE) - .addInsertValues("null", "'2018-03-22 12:00:00.123 +05:00'", "'2018-03-22 12:00:00.123456 +05:00'") - .addExpectedValues(null, "2018-03-22T07:00:00.123000Z", "2018-03-22T07:00:00.123456Z") - .build()); + .addInsertValues("null", "'2018-03-25 12:00:00.123 +05:00'", "'2018-03-25 12:00:00.123456 +05:00'") + .addExpectedValues(null, "2018-03-25T00:00:00.123000-07:00", "2018-03-25T00:00:00.123000-07:00") + // We moved from +5 to -7 timezone, so 12:00 becomes 00:00. + // Snowflake default timestamp precision is TIME(3), so we lose anything past ms + .build());// This is extremely brittle. A change of parameters on the customer's account, + // or a change of timezone where this code is executed (!!) could change the values returned by + // snowflake addDataTypeTestData( TestDataHolder.builder() .sourceType("TIMESTAMP_NTZ") .airbyteType(JsonSchemaType.STRING_TIMESTAMP_WITHOUT_TIMEZONE) - .addInsertValues("null", "'2018-03-22 12:00:00.123 +05:00'", "'2018-03-22 12:00:00.123456 +05:00'") - .addExpectedValues(null, "2018-03-22T12:00:00.123", "2018-03-22T12:00:00.123456") - .build()); + .addInsertValues("null", "'2018-03-24 12:00:00.123 +05:00'", "'2018-03-24 12:00:00.123456 +05:00'") + .addExpectedValues(null, "2018-03-24T12:00:00.123", "2018-03-24T12:00:00.123456") + .build()); // This is very brittle. A change of parameters on the customer's account could change the values + // returned by snowflake addDataTypeTestData( TestDataHolder.builder() .sourceType("TIMESTAMP_TZ") .airbyteType(JsonSchemaType.STRING_TIMESTAMP_WITH_TIMEZONE) - .addInsertValues("null", "'2018-03-22 12:00:00.123 +05:00'", "'2018-03-22 12:00:00.123456 +05:00'") - .addExpectedValues(null, "2018-03-22T07:00:00.123000Z", "2018-03-22T07:00:00.123456Z") - .build()); + .addInsertValues("null", "'2018-03-23 12:00:00.123 +05:00'", "'2018-03-23 12:00:00.123456 +05:00'") + .addExpectedValues(null, "2018-03-23T12:00:00.123000+05:00", "2018-03-23T12:00:00.123000+05:00") + // Snowflake default timestamp-to-string conversion is TIME(3), so we lose anything past ms + .build());// This is very brittle. A change of parameters on the customer's account could change the values + // returned by snowflake // Semi-structured Data Types addDataTypeTestData( diff --git a/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeTestDatabase.java b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeTestDatabase.java new file mode 100644 index 000000000000..a4e194c8b099 --- /dev/null +++ b/airbyte-integrations/connectors/source-snowflake/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/SnowflakeTestDatabase.java @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.io.airbyte.integration_tests.sources; + +import static io.airbyte.cdk.db.factory.DatabaseDriver.SNOWFLAKE; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.NonContainer; +import io.airbyte.cdk.testutils.TestDatabase; +import java.util.stream.Stream; +import org.jooq.SQLDialect; + +public class SnowflakeTestDatabase extends TestDatabase { + + private final String username; + private final String password; + private final String jdbcUrl; + + protected SnowflakeTestDatabase(final JsonNode snowflakeConfig) { + super(new NonContainer(snowflakeConfig.get(JdbcUtils.USERNAME_KEY).asText(), + snowflakeConfig.has(JdbcUtils.PASSWORD_KEY) ? snowflakeConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, + snowflakeConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), SNOWFLAKE.getDriverClassName(), "")); + this.username = snowflakeConfig.get(JdbcUtils.USERNAME_KEY).asText(); + this.password = snowflakeConfig.has(JdbcUtils.PASSWORD_KEY) ? snowflakeConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null; + this.jdbcUrl = snowflakeConfig.get(JdbcUtils.JDBC_URL_KEY).asText(); + } + + @Override + public String getJdbcUrl() { + return jdbcUrl; + } + + @Override + public String getPassword() { + return password; + } + + @Override + public String getUserName() { + return username; + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.empty(); + } + + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return SNOWFLAKE; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.DEFAULT; + } + + static public class SnowflakeConfigBuilder extends TestDatabase.ConfigBuilder { + + protected SnowflakeConfigBuilder(SnowflakeTestDatabase testdb) { + super(testdb); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-sonar-cloud/README.md b/airbyte-integrations/connectors/source-sonar-cloud/README.md index 8cbed405be80..b71a61e01dea 100644 --- a/airbyte-integrations/connectors/source-sonar-cloud/README.md +++ b/airbyte-integrations/connectors/source-sonar-cloud/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-sonar-cloud:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/sonar-cloud) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_sonar_cloud/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-sonar-cloud:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-sonar-cloud build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-sonar-cloud:airbyteDocker +An image will be built with the tag `airbyte/source-sonar-cloud:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-sonar-cloud:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-sonar-cloud:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-sonar-cloud:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-sonar-cloud:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-sonar-cloud test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-sonar-cloud:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-sonar-cloud:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-sonar-cloud test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/sonar-cloud.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-sonar-cloud/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-sonar-cloud/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-sonar-cloud/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-sonar-cloud/build.gradle b/airbyte-integrations/connectors/source-sonar-cloud/build.gradle deleted file mode 100644 index 74e2c688e3eb..000000000000 --- a/airbyte-integrations/connectors/source-sonar-cloud/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_sonar_cloud' -} diff --git a/airbyte-integrations/connectors/source-spacex-api/Dockerfile b/airbyte-integrations/connectors/source-spacex-api/Dockerfile index 15e9c34652c0..a2114a5f5b63 100644 --- a/airbyte-integrations/connectors/source-spacex-api/Dockerfile +++ b/airbyte-integrations/connectors/source-spacex-api/Dockerfile @@ -34,5 +34,5 @@ COPY source_spacex_api ./source_spacex_api ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-spacex-api diff --git a/airbyte-integrations/connectors/source-spacex-api/README.md b/airbyte-integrations/connectors/source-spacex-api/README.md index bac4ec13c35a..913ec4c891c9 100644 --- a/airbyte-integrations/connectors/source-spacex-api/README.md +++ b/airbyte-integrations/connectors/source-spacex-api/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-spacex-api:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/spacex-api) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_spacex_api/spec.yaml` file. @@ -48,18 +40,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-spacex-api:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-spacex-api build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-spacex-api:airbyteDocker +An image will be built with the tag `airbyte/source-spacex-api:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-spacex-api:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -69,25 +62,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-spacex-api:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-spacex-api:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-spacex-api:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-spacex-api test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-spacex-api:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-spacex-api:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -96,8 +81,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-spacex-api test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/spacex-api.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-spacex-api/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-spacex-api/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-spacex-api/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-spacex-api/build.gradle b/airbyte-integrations/connectors/source-spacex-api/build.gradle deleted file mode 100644 index f6146e282406..000000000000 --- a/airbyte-integrations/connectors/source-spacex-api/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_spacex_api' -} diff --git a/airbyte-integrations/connectors/source-spacex-api/metadata.yaml b/airbyte-integrations/connectors/source-spacex-api/metadata.yaml index 41f21b162e59..1da6b6776c15 100644 --- a/airbyte-integrations/connectors/source-spacex-api/metadata.yaml +++ b/airbyte-integrations/connectors/source-spacex-api/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: 62235e65-af7a-4138-9130-0bda954eb6a8 - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 dockerRepository: airbyte/source-spacex-api githubIssueLabel: source-spacex-api icon: spacex.svg diff --git a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/manifest.yaml b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/manifest.yaml index b00591f11b26..18e963abcef4 100644 --- a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/manifest.yaml +++ b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/manifest.yaml @@ -1,147 +1,1715 @@ -version: "0.29.0" - -definitions: - selector: - extractor: - field_path: [] - - requester: - url_base: "https://api.spacexdata.com/v4/" - http_method: "GET" - - retriever: - record_selector: - $ref: "#/definitions/selector" - paginator: - type: NoPagination - requester: - $ref: "#/definitions/requester" - - base_stream: - schema_loader: - type: JsonFileSchemaLoader - file_path: "./source_spacex_api/schemas/{{ parameters['name'] }}.json" - retriever: - $ref: "#/definitions/retriever" - - launches_stream: - $ref: "#/definitions/base_stream" - $parameters: - name: "launches" - primary_key: "id" - path: "/launches/{{config['options'] or config['id'] or latest}}" - - capsules_stream: - $ref: "#/definitions/base_stream" - $parameters: - name: "capsules" - primary_key: "id" - path: "/capsules" - - company_stream: - $ref: "#/definitions/base_stream" - $parameters: - name: "company" - primary_key: "id" - path: "/company" - - crew_stream: - $ref: "#/definitions/base_stream" - $parameters: - name: "crew" - primary_key: "id" - path: "/crew" - - cores_stream: - $ref: "#/definitions/base_stream" - $parameters: - name: "cores" - primary_key: "id" - path: "/cores" - - dragons_stream: - $ref: "#/definitions/base_stream" - $parameters: - name: "dragons" - primary_key: "id" - path: "/dragons" - - landpads_stream: - $ref: "#/definitions/base_stream" - $parameters: - name: "landpads" - primary_key: "id" - path: "/landpads" - - payloads_stream: - $ref: "#/definitions/base_stream" - $parameters: - name: "payloads" - primary_key: "id" - path: "/payloads" - - history_stream: - $ref: "#/definitions/base_stream" - $parameters: - name: "history" - primary_key: "id" - path: "/history" - - rockets_stream: - $ref: "#/definitions/base_stream" - $parameters: - name: "rockets" - primary_key: "id" - path: "/rockets" - - roadster_stream: - $ref: "#/definitions/base_stream" - $parameters: - name: "roadster" - primary_key: "id" - path: "/roadster" - - ships_stream: - $ref: "#/definitions/base_stream" - $parameters: - name: "ships" - primary_key: "id" - path: "/ships" - - starlink_stream: - $ref: "#/definitions/base_stream" - $parameters: - name: "starlink" - primary_key: "id" - path: "/starlink" - -streams: - - "#/definitions/launches_stream" - - "#/definitions/capsules_stream" - - "#/definitions/company_stream" - - "#/definitions/crew_stream" - - "#/definitions/cores_stream" - - "#/definitions/dragons_stream" - - "#/definitions/landpads_stream" - - "#/definitions/payloads_stream" - - "#/definitions/history_stream" - - "#/definitions/rockets_stream" - - "#/definitions/roadster_stream" - - "#/definitions/ships_stream" - - "#/definitions/starlink_stream" - +version: 0.51.41 +type: DeclarativeSource check: + type: CheckStream stream_names: - - "launches" - - "capsules" - - "company" - - "crew" - - "cores" - - "dragons" - - "landpads" - - "payloads" - - "history" - - "rockets" - - "roadster" - - "ships" - - "starlink" + - launches + - capsules + - company + - crew + - cores + - dragons + - landpads + - payloads + - history + - rockets + - roadster + - ships + - starlink +streams: + - type: DeclarativeStream + name: launches + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + auto_update: + type: boolean + capsules: + items: + type: + - "null" + - string + type: array + cores: + items: + properties: + core: + type: + - "null" + - string + flight: + type: number + gridfins: + type: boolean + landing_attempt: + type: boolean + landing_success: + type: boolean + landing_type: + type: + - "null" + - string + landpad: + type: + - "null" + - string + legs: + type: boolean + reused: + type: boolean + type: object + type: array + crew: + items: + type: + - "null" + - string + type: array + date_local: + type: + - "null" + - string + date_precision: + type: + - "null" + - string + date_unix: + type: number + date_utc: + type: + - "null" + - string + static_fire_date_unix: + type: + - "null" + - number + static_fire_date_utc: + type: + - "null" + - string + window: + type: + - "null" + - number + details: + type: string + fairings: + properties: + recovered: + type: boolean + recovery_attempt: + type: boolean + reused: + type: boolean + ships: + items: + type: + - "null" + - string + type: array + type: object + failures: + type: array + flight_number: + type: number + id: + type: + - "null" + - string + launch_library_id: + type: + - "null" + - string + launchpad: + type: + - "null" + - string + links: + properties: + flickr: + properties: + original: + items: + type: + - "null" + - string + type: array + small: + type: array + type: object + patch: + properties: + large: + type: + - "null" + - string + small: + type: + - "null" + - string + type: object + reddit: + properties: + launch: + type: + - "null" + - string + type: object + article: + type: + - "null" + - string + webcast: + type: + - "null" + - string + wikipedia: + type: + - "null" + - string + youtube_id: + type: + - "null" + - string + type: object + name: + type: + - "null" + - string + net: + type: boolean + payloads: + items: + type: + - "null" + - string + type: array + rocket: + type: + - "null" + - string + ships: + type: array + success: + type: boolean + tbd: + type: boolean + upcoming: + type: boolean + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.spacexdata.com/v4/ + path: /launches/{{config['options'] or config['id'] or 'latest'}} + http_method: GET + request_parameters: {} + request_headers: {} + authenticator: + type: NoAuth + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: NoPagination + - type: DeclarativeStream + name: capsules + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + id: + type: + - "null" + - string + land_landings: + type: number + last_update: + type: + - "null" + - string + launches: + items: + type: + - "null" + - string + type: array + reuse_count: + type: number + serial: + type: + - "null" + - string + status: + type: + - "null" + - string + type: + type: + - "null" + - string + water_landings: + type: number + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.spacexdata.com/v4/ + path: /capsules + http_method: GET + request_parameters: {} + request_headers: {} + authenticator: + type: NoAuth + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: NoPagination + - type: DeclarativeStream + name: company + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + ceo: + type: + - "null" + - string + coo: + type: + - "null" + - string + cto: + type: + - "null" + - string + cto_propulsion: + type: + - "null" + - string + employees: + type: number + founded: + type: number + founder: + type: + - "null" + - string + headquarters: + properties: + address: + type: + - "null" + - string + city: + type: + - "null" + - string + state: + type: + - "null" + - string + type: object + id: + type: + - "null" + - string + launch_sites: + type: number + links: + properties: + elon_twitter: + type: + - "null" + - string + flickr: + type: + - "null" + - string + twitter: + type: + - "null" + - string + website: + type: + - "null" + - string + type: object + name: + type: + - "null" + - string + summary: + type: + - "null" + - string + test_sites: + type: number + valuation: + type: number + vehicles: + type: number + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.spacexdata.com/v4/ + path: /company + http_method: GET + request_parameters: {} + request_headers: {} + authenticator: + type: NoAuth + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: NoPagination + - type: DeclarativeStream + name: crew + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + agency: + type: + - "null" + - string + id: + type: + - "null" + - string + image: + type: + - "null" + - string + launches: + items: + type: + - "null" + - string + type: array + name: + type: + - "null" + - string + status: + type: + - "null" + - string + wikipedia: + type: + - "null" + - string + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.spacexdata.com/v4/ + path: /crew + http_method: GET + request_parameters: {} + request_headers: {} + authenticator: + type: NoAuth + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: NoPagination + - type: DeclarativeStream + name: cores + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + asds_attempts: + type: number + asds_landings: + type: number + block: + type: + - "null" + - number + id: + type: + - "null" + - string + last_update: + type: + - "null" + - string + launches: + items: + type: + - "null" + - string + type: array + reuse_count: + type: number + rtls_attempts: + type: number + rtls_landings: + type: number + serial: + type: + - "null" + - string + status: + type: + - "null" + - string + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.spacexdata.com/v4/ + path: /cores + http_method: GET + request_parameters: {} + request_headers: {} + authenticator: + type: NoAuth + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: NoPagination + - type: DeclarativeStream + name: dragons + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + active: + type: boolean + crew_capacity: + type: number + description: + type: + - "null" + - string + diameter: + properties: + feet: + type: number + meters: + type: number + type: object + dry_mass_kg: + type: number + dry_mass_lb: + type: number + first_flight: + type: + - "null" + - string + flickr_images: + items: + type: + - "null" + - string + type: array + heat_shield: + properties: + dev_partner: + type: + - "null" + - string + material: + type: + - "null" + - string + size_meters: + type: number + temp_degrees: + type: number + type: object + height_w_trunk: + properties: + feet: + type: number + meters: + type: number + type: object + id: + type: + - "null" + - string + launch_payload_mass: + properties: + kg: + type: number + lb: + type: number + type: object + launch_payload_vol: + properties: + cubic_feet: + type: number + cubic_meters: + type: number + type: object + name: + type: + - "null" + - string + orbit_duration_yr: + type: number + pressurized_capsule: + properties: + payload_volume: + properties: + cubic_feet: + type: number + cubic_meters: + type: number + type: object + type: object + return_payload_mass: + properties: + kg: + type: number + lb: + type: number + type: object + return_payload_vol: + properties: + cubic_feet: + type: number + cubic_meters: + type: number + type: object + sidewall_angle_deg: + type: number + thrusters: + items: + properties: + amount: + type: number + fuel_1: + type: + - "null" + - string + fuel_2: + type: + - "null" + - string + isp: + type: number + pods: + type: number + thrust: + properties: + kN: + type: number + lbf: + type: number + type: object + type: + type: + - "null" + - string + type: object + type: array + trunk: + properties: + cargo: + properties: + solar_array: + type: number + unpressurized_cargo: + type: boolean + type: object + trunk_volume: + properties: + cubic_feet: + type: number + cubic_meters: + type: number + type: object + type: object + type: + type: + - "null" + - string + wikipedia: + type: + - "null" + - string + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.spacexdata.com/v4/ + path: /dragons + http_method: GET + request_parameters: {} + request_headers: {} + authenticator: + type: NoAuth + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: NoPagination + - type: DeclarativeStream + name: landpads + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + details: + type: + - "null" + - string + full_name: + type: + - "null" + - string + id: + type: + - "null" + - string + images: + properties: + large: + items: + type: + - "null" + - string + type: array + type: object + landing_attempts: + type: number + landing_successes: + type: number + latitude: + type: number + launches: + items: + type: + - "null" + - string + type: array + locality: + type: + - "null" + - string + longitude: + type: number + name: + type: + - "null" + - string + region: + type: + - "null" + - string + status: + type: + - "null" + - string + type: + type: + - "null" + - string + wikipedia: + type: + - "null" + - string + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.spacexdata.com/v4/ + path: /landpads + http_method: GET + request_parameters: {} + request_headers: {} + authenticator: + type: NoAuth + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: NoPagination + - type: DeclarativeStream + name: payloads + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + apoapsis_km: + type: + - "null" + - number + arg_of_pericenter: + type: + - "null" + - number + customers: + items: + type: + - "null" + - string + type: array + dragon: + properties: + capsule: + type: + - "null" + - string + flight_time_sec: + type: + - "null" + - number + land_landing: + type: + - boolean + - "null" + manifest: + type: + - "null" + - string + mass_returned_kg: + type: + - "null" + - number + mass_returned_lbs: + type: + - "null" + - number + water_landing: + type: + - boolean + - "null" + type: object + eccentricity: + type: + - "null" + - number + epoch: + type: + - "null" + - string + id: + type: + - "null" + - string + inclination_deg: + type: + - "null" + - number + launch: + type: + - "null" + - string + lifespan_years: + type: + - "null" + - number + longitude: + type: + - "null" + - number + manufacturers: + items: + type: + - "null" + - string + type: array + mass_kg: + type: + - "null" + - number + mass_lbs: + type: + - "null" + - number + mean_anomaly: + type: + - "null" + - number + mean_motion: + type: + - "null" + - number + name: + type: + - "null" + - string + nationalities: + items: + type: + - "null" + - string + type: array + norad_ids: + items: + type: number + type: array + orbit: + type: + - "null" + - string + periapsis_km: + type: + - "null" + - number + period_min: + type: + - "null" + - number + raan: + type: + - "null" + - number + reference_system: + type: + - "null" + - string + regime: + type: + - "null" + - string + reused: + type: boolean + semi_major_axis_km: + type: + - "null" + - number + type: + type: + - "null" + - string + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.spacexdata.com/v4/ + path: /payloads + http_method: GET + request_parameters: {} + request_headers: {} + authenticator: + type: NoAuth + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: NoPagination + - type: DeclarativeStream + name: history + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + details: + type: + - "null" + - string + event_date_unix: + type: number + event_date_utc: + type: + - "null" + - string + id: + type: + - "null" + - string + links: + properties: + article: + type: + - "null" + - string + type: object + title: + type: + - "null" + - string + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.spacexdata.com/v4/ + path: /history + http_method: GET + request_parameters: {} + request_headers: {} + authenticator: + type: NoAuth + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: NoPagination + - type: DeclarativeStream + name: rockets + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + active: + type: boolean + boosters: + type: number + company: + type: + - "null" + - string + cost_per_launch: + type: number + country: + type: + - "null" + - string + description: + type: + - "null" + - string + diameter: + properties: + feet: + type: number + meters: + type: number + type: object + engines: + properties: + engine_loss_max: + type: + - "null" + - number + isp: + properties: + sea_level: + type: number + vacuum: + type: number + type: object + layout: + type: + - "null" + - string + number: + type: number + propellant_1: + type: + - "null" + - string + propellant_2: + type: + - "null" + - string + thrust_sea_level: + properties: + kN: + type: number + lbf: + type: number + type: object + thrust_to_weight: + type: number + thrust_vacuum: + properties: + kN: + type: number + lbf: + type: number + type: object + type: + type: + - "null" + - string + version: + type: + - "null" + - string + type: object + first_flight: + type: + - "null" + - string + first_stage: + properties: + burn_time_sec: + type: + - "null" + - number + engines: + type: number + fuel_amount_tons: + type: number + reusable: + type: boolean + thrust_sea_level: + properties: + kN: + type: number + lbf: + type: number + type: object + thrust_vacuum: + properties: + kN: + type: number + lbf: + type: number + type: object + type: object + flickr_images: + items: + type: + - "null" + - string + type: array + height: + properties: + feet: + type: number + meters: + type: number + type: object + id: + type: + - "null" + - string + landing_legs: + properties: + material: + type: + - "null" + - string + number: + type: number + type: object + mass: + properties: + kg: + type: number + lb: + type: number + type: object + name: + type: + - "null" + - string + payload_weights: + items: + properties: + id: + type: + - "null" + - string + kg: + type: number + lb: + type: number + name: + type: + - "null" + - string + type: object + type: array + second_stage: + properties: + burn_time_sec: + type: + - "null" + - number + engines: + type: number + fuel_amount_tons: + type: number + payloads: + properties: + composite_fairing: + properties: + diameter: + properties: + feet: + type: + - "null" + - number + meters: + type: + - "null" + - number + type: object + height: + properties: + feet: + type: + - "null" + - number + meters: + type: + - "null" + - number + type: object + type: object + option_1: + type: + - "null" + - string + type: object + reusable: + type: boolean + thrust: + properties: + kN: + type: number + lbf: + type: number + type: object + type: object + stages: + type: number + success_rate_pct: + type: number + type: + type: + - "null" + - string + wikipedia: + type: + - "null" + - string + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.spacexdata.com/v4/ + path: /rockets + http_method: GET + request_parameters: {} + request_headers: {} + authenticator: + type: NoAuth + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: NoPagination + - type: DeclarativeStream + name: roadster + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + apoapsis_au: + type: number + details: + type: + - "null" + - string + earth_distance_km: + type: number + earth_distance_mi: + type: number + eccentricity: + type: number + epoch_jd: + type: number + flickr_images: + items: + type: + - "null" + - string + type: array + id: + type: + - "null" + - string + inclination: + type: number + launch_date_unix: + type: number + launch_date_utc: + type: + - "null" + - string + launch_mass_kg: + type: number + launch_mass_lbs: + type: number + longitude: + type: number + mars_distance_km: + type: number + mars_distance_mi: + type: number + name: + type: + - "null" + - string + norad_id: + type: number + orbit_type: + type: + - "null" + - string + periapsis_arg: + type: number + periapsis_au: + type: number + period_days: + type: number + semi_major_axis_au: + type: number + speed_kph: + type: number + speed_mph: + type: number + video: + type: + - "null" + - string + wikipedia: + type: + - "null" + - string + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.spacexdata.com/v4/ + path: /roadster + http_method: GET + request_parameters: {} + request_headers: {} + authenticator: + type: NoAuth + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: NoPagination + - type: DeclarativeStream + name: ships + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + abs: + type: + - "null" + - number + active: + type: boolean + class: + type: + - "null" + - number + home_port: + type: + - "null" + - string + id: + type: + - "null" + - string + image: + type: + - "null" + - string + imo: + type: + - "null" + - number + latitude: + type: + - "null" + - number + course_deg: + type: + - "null" + - number + last_ais_update: + type: + - "null" + - string + speed_kn: + type: + - "null" + - number + launches: + items: + type: + - "null" + - string + type: array + legacy_id: + type: + - "null" + - string + link: + type: + - "null" + - string + longitude: + type: + - "null" + - number + mass_kg: + type: + - "null" + - number + mass_lbs: + type: + - "null" + - number + mmsi: + type: + - "null" + - number + model: + type: + - "null" + - string + name: + type: + - "null" + - string + roles: + items: + type: + - "null" + - string + type: array + status: + type: + - "null" + - string + type: + type: + - "null" + - string + year_built: + type: + - "null" + - number + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.spacexdata.com/v4/ + path: /ships + http_method: GET + request_parameters: {} + request_headers: {} + authenticator: + type: NoAuth + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: NoPagination + - type: DeclarativeStream + name: starlink + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + $schema: http://json-schema.org/schema# + properties: + height_km: + type: + - "null" + - number + id: + type: + - "null" + - string + latitude: + type: + - "null" + - number + launch: + type: + - "null" + - string + longitude: + type: + - "null" + - number + spaceTrack: + properties: + APOAPSIS: + type: number + ARG_OF_PERICENTER: + type: number + BSTAR: + type: number + CCSDS_OMM_VERS: + type: + - "null" + - string + CENTER_NAME: + type: + - "null" + - string + CLASSIFICATION_TYPE: + type: + - "null" + - string + COMMENT: + type: + - "null" + - string + COUNTRY_CODE: + type: + - "null" + - string + CREATION_DATE: + type: + - "null" + - string + DECAYED: + type: number + DECAY_DATE: + type: + - "null" + - string + ECCENTRICITY: + type: number + ELEMENT_SET_NO: + type: number + EPHEMERIS_TYPE: + type: number + EPOCH: + type: + - "null" + - string + FILE: + type: number + GP_ID: + type: number + INCLINATION: + type: number + LAUNCH_DATE: + type: + - "null" + - string + MEAN_ANOMALY: + type: number + MEAN_ELEMENT_THEORY: + type: + - "null" + - string + MEAN_MOTION: + type: number + MEAN_MOTION_DDOT: + type: number + MEAN_MOTION_DOT: + type: number + NORAD_CAT_ID: + type: number + OBJECT_ID: + type: + - "null" + - string + OBJECT_NAME: + type: + - "null" + - string + OBJECT_TYPE: + type: + - "null" + - string + ORIGINATOR: + type: + - "null" + - string + PERIAPSIS: + type: number + PERIOD: + type: number + RA_OF_ASC_NODE: + type: number + RCS_SIZE: + type: + - "null" + - string + REF_FRAME: + type: + - "null" + - string + REV_AT_EPOCH: + type: number + SEMIMAJOR_AXIS: + type: number + SITE: + type: + - "null" + - string + TIME_SYSTEM: + type: + - "null" + - string + TLE_LINE0: + type: + - "null" + - string + TLE_LINE1: + type: + - "null" + - string + TLE_LINE2: + type: + - "null" + - string + type: object + velocity_kms: + type: + - "null" + - number + version: + type: + - "null" + - string + type: object + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.spacexdata.com/v4/ + path: /starlink + http_method: GET + request_parameters: {} + request_headers: {} + authenticator: + type: NoAuth + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: NoPagination +spec: + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + type: object + required: [] + properties: + id: + title: Unique ID for specific source target + type: string + desciption: Optional, For a specific ID + order: 0 + options: + title: Configuration options for endpoints + type: string + desciption: >- + Optional, Possible values for an endpoint. Example values for + launches-latest, upcoming, past + order: 1 + additionalProperties: true + type: Spec +metadata: + autoImportSchema: + launches: false + capsules: false + company: false + crew: false + cores: false + dragons: false + landpads: false + payloads: false + history: false + rockets: false + roadster: false + ships: false + starlink: false diff --git a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/capsules.json b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/capsules.json deleted file mode 100644 index 27d3f20054cf..000000000000 --- a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/capsules.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "capsules": { - "title": "capsules", - "type": "object", - "properties": { - "serial": { "type": "string" }, - "status": { - "type": "string", - "enum": ["unknown", "active", "retired", "destroyed"] - }, - "type": { - "type": "string", - "enum": ["Dragon 1.0", "Dragon 1.1", "Dragon 2.0"] - }, - "dragon": { - "type": "string", - "x-ref": "Dragon", - "description": "Refers to Dragon" - }, - "reuse_count": { "type": "number", "default": 0 }, - "water_landings": { "type": "number", "default": 0 }, - "land_landings": { "type": "number", "default": 0 }, - "last_update": { "type": ["string", "null"], "default": null }, - "launches": { - "type": "array", - "items": { - "type": "string", - "x-ref": "Launch", - "description": "Refers to Launch" - } - } - }, - "required": ["serial", "status", "type"] - }, - "id": { "type": "string", "pattern": "^[0-9a-fA-F]{24}$" } - } -} diff --git a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/company.json b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/company.json deleted file mode 100644 index ec948f3229dd..000000000000 --- a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/company.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "company": { - "title": "company", - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "founder": { - "type": "string" - }, - "founded": { - "type": "number" - }, - "employees": { - "type": "number" - }, - "vehicles": { - "type": "number" - }, - "launch_sites": { - "type": "number" - }, - "test_sites": { - "type": "number" - }, - "ceo": { - "type": "string" - }, - "cto": { - "type": "string" - }, - "coo": { - "type": "string" - }, - "cto_propulsion": { - "type": "string" - }, - "valuation": { - "type": "number" - }, - "headquarters": { - "title": "headquarters", - "type": "object", - "properties": { - "address": { - "type": "string" - }, - "city": { - "type": "string" - }, - "state": { - "type": "string" - } - } - }, - "links": { - "title": "links", - "type": "object", - "properties": { - "website": { - "type": "string" - }, - "flickr": { - "type": "string" - }, - "twitter": { - "type": "string" - }, - "elon_twitter": { - "type": "string" - } - } - }, - "summary": { - "type": "string" - } - } - }, - "id": { - "type": "string", - "pattern": "^[0-9a-fA-F]{24}$" - } - } -} diff --git a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/cores.json b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/cores.json deleted file mode 100644 index 95c5e1cf0246..000000000000 --- a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/cores.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "cores": { - "title": "cores", - "type": "object", - "properties": { - "serial": { - "type": "string" - }, - "block": { - "type": ["number", "null"], - "default": null - }, - "status": { - "type": "string", - "enum": [ - "active", - "inactive", - "unknown", - "expended", - "lost", - "retired" - ] - }, - "reuse_count": { - "type": "number", - "default": 0 - }, - "rtls_attempts": { - "type": "number", - "default": 0 - }, - "rtls_landings": { - "type": "number", - "default": 0 - }, - "asds_attempts": { - "type": "number", - "default": 0 - }, - "asds_landings": { - "type": "number", - "default": 0 - }, - "last_update": { - "type": ["string", "null"], - "default": null - }, - "launches": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": ["serial", "status"] - }, - "id": { - "type": "string", - "pattern": "^[0-9a-fA-F]{24}$" - } - } -} diff --git a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/crew.json b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/crew.json deleted file mode 100644 index effdd0a1d06c..000000000000 --- a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/crew.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "crew": { - "title": "crew", - "type": "object", - "properties": { - "name": { - "type": ["string", "null"], - "default": null - }, - "status": { - "type": "string", - "enum": ["active", "inactive", "retired", "unknown"] - }, - "agency": { - "type": ["string", "null"], - "default": null - }, - "image": { - "type": ["string", "null"], - "default": null - }, - "wikipedia": { - "type": ["string", "null"], - "default": null - }, - "launches": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": ["status"] - }, - "id": { - "type": "string", - "pattern": "^[0-9a-fA-F]{24}$" - } - } -} diff --git a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/dragons.json b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/dragons.json deleted file mode 100644 index 052c552cff8a..000000000000 --- a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/dragons.json +++ /dev/null @@ -1,211 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "dragons": { - "title": "dragons", - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "type": { - "type": "string" - }, - "active": { - "type": "boolean" - }, - "crew_capacity": { - "type": "number" - }, - "sidewall_angle_deg": { - "type": "number" - }, - "orbit_duration_yr": { - "type": "number" - }, - "dry_mass_kg": { - "type": "number" - }, - "dry_mass_lb": { - "type": "number" - }, - "first_flight": { - "type": ["string", "null"], - "default": null - }, - "heat_shield": { - "title": "heat_shield", - "type": "object", - "properties": { - "material": { - "type": "string" - }, - "size_meters": { - "type": "number" - }, - "temp_degrees": { - "type": "number" - }, - "dev_partner": { - "type": "string" - } - }, - "required": ["material", "size_meters"] - }, - "thrusters": { - "type": "array", - "items": { - "type": "string" - } - }, - "launch_payload_mass": { - "title": "launch_payload_mass", - "type": "object", - "properties": { - "kg": { - "type": "number" - }, - "lb": { - "type": "number" - } - } - }, - "launch_payload_vol": { - "title": "launch_payload_vol", - "type": "object", - "properties": { - "cubic_meters": { - "type": "number" - }, - "cubic_feet": { - "type": "number" - } - } - }, - "return_payload_mass": { - "title": "return_payload_mass", - "type": "object", - "properties": { - "kg": { - "type": "number" - }, - "lb": { - "type": "number" - } - } - }, - "return_payload_vol": { - "title": "return_payload_vol", - "type": "object", - "properties": { - "cubic_meters": { - "type": "number" - }, - "cubic_feet": { - "type": "number" - } - } - }, - "pressurized_capsule": { - "title": "pressurized_capsule", - "type": "object", - "properties": { - "payload_volume": { - "title": "payload_volume", - "type": "object", - "properties": { - "cubic_meters": { - "type": "number" - }, - "cubic_feet": { - "type": "number" - } - } - } - } - }, - "trunk": { - "title": "trunk", - "type": "object", - "properties": { - "trunk_volume": { - "title": "trunk_volume", - "type": "object", - "properties": { - "cubic_meters": { - "type": "number" - }, - "cubic_feet": { - "type": "number" - } - } - }, - "cargo": { - "title": "cargo", - "type": "object", - "properties": { - "solar_array": { - "type": "number" - }, - "unpressurized_cargo": { - "type": "boolean" - } - } - } - } - }, - "height_w_trunk": { - "title": "height_w_trunk", - "type": "object", - "properties": { - "meters": { - "type": "number" - }, - "feet": { - "type": "number" - } - } - }, - "diameter": { - "title": "diameter", - "type": "object", - "properties": { - "meters": { - "type": "number" - }, - "feet": { - "type": "number" - } - } - }, - "flickr_images": { - "type": "array", - "items": { - "type": "string" - } - }, - "wikipedia": { - "type": "string" - }, - "description": { - "type": "string" - } - }, - "required": [ - "name", - "type", - "active", - "crew_capacity", - "sidewall_angle_deg", - "orbit_duration_yr", - "dry_mass_kg", - "dry_mass_lb" - ] - }, - "id": { - "type": "string", - "pattern": "^[0-9a-fA-F]{24}$" - } - } -} diff --git a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/history.json b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/history.json deleted file mode 100644 index 9653d18b2c8d..000000000000 --- a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/history.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "history": { - "title": "history", - "type": "object", - "properties": { - "title": { - "type": ["string", "null"], - "default": null - }, - "event_date_utc": { - "type": ["string", "null"], - "default": null - }, - "event_date_unix": { - "type": ["number", "null"], - "default": null - }, - "details": { - "type": ["string", "null"], - "default": null - }, - "links": { - "title": "links", - "type": "object", - "properties": { - "article": { - "type": ["string", "null"], - "default": null - } - } - } - } - }, - "id": { - "type": "string", - "pattern": "^[0-9a-fA-F]{24}$" - } - } -} diff --git a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/landpads.json b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/landpads.json deleted file mode 100644 index a990d9d006d6..000000000000 --- a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/landpads.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "landpads": { - "title": "landpads", - "type": "object", - "properties": { - "name": { "type": ["string", "null"], "default": null }, - "full_name": { "type": ["string", "null"], "default": null }, - "status": { - "type": "string", - "enum": [ - "active", - "inactive", - "unknown", - "retired", - "lost", - "under construction" - ] - }, - "type": { "type": ["string", "null"], "default": null }, - "locality": { "type": ["string", "null"], "default": null }, - "region": { "type": ["string", "null"], "default": null }, - "latitude": { "type": ["number", "null"], "default": null }, - "longitude": { "type": ["number", "null"], "default": null }, - "landing_attempts": { "type": "number", "default": 0 }, - "landing_successes": { "type": "number", "default": 0 }, - "wikipedia": { "type": ["string", "null"], "default": null }, - "details": { "type": ["string", "null"], "default": null }, - "launches": { "type": "array", "items": { "type": "string" } }, - "images": { - "title": "images", - "type": "object", - "properties": { - "large": { "type": "array", "items": { "type": "string" } } - } - } - }, - "required": ["status"] - }, - "id": { "type": "string", "pattern": "^[0-9a-fA-F]{24}$" } - } -} diff --git a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/launches.json b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/launches.json deleted file mode 100644 index 767e5902b1df..000000000000 --- a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/launches.json +++ /dev/null @@ -1,287 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "flight_number": { - "type": "number" - }, - "name": { - "type": "string" - }, - "date_utc": { - "type": "string" - }, - "date_unix": { - "type": "number" - }, - "date_local": { - "type": "string" - }, - "date_precision": { - "type": "string", - "enum": ["half", "quarter", "year", "month", "day", "hour"] - }, - "static_fire_date_utc": { - "type": ["string", "null"], - "default": null - }, - "static_fire_date_unix": { - "type": ["number", "null"], - "default": null - }, - "tbd": { - "type": "boolean", - "default": false - }, - "net": { - "type": "boolean", - "default": false - }, - "window": { - "type": ["number", "null"], - "default": null - }, - "rocket": { - "type": ["string", "null"], - "default": null - }, - "success": { - "type": ["boolean", "null"], - "default": null - }, - "failures": { - "type": "array", - "items": { - "title": "itemOf_failures", - "type": "object", - "properties": { - "time": { - "type": ["null", "number"] - }, - "altitude": { - "type": ["null", "number"] - }, - "reason": { - "type": ["null", "string"] - } - } - } - }, - "upcoming": { - "type": "boolean" - }, - "details": { - "type": ["string", "null"], - "default": null - }, - "fairings": { - "title": "fairings", - "type": ["object", "null"], - "properties": { - "reused": { - "type": ["boolean", "null"], - "default": null - }, - "recovery_attempt": { - "type": ["boolean", "null"], - "default": null - }, - "recovered": { - "type": ["boolean", "null"], - "default": null - }, - "ships": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "crew": { - "type": ["array", "string"], - "items": { - "title": "itemOf_crew", - "type": ["object", "string"], - "properties": { - "crew": { - "type": ["string", "null"], - "default": null - }, - "role": { - "type": ["string", "null"], - "default": null - } - } - } - }, - "ships": { - "type": "array", - "items": { - "type": "string" - } - }, - "capsules": { - "type": "array", - "items": { - "type": "string" - } - }, - "payloads": { - "type": "array", - "items": { - "type": "string" - } - }, - "launchpad": { - "type": ["string", "null"], - "default": null - }, - "cores": { - "type": "array", - "items": { - "title": "itemOf_cores", - "type": "object", - "properties": { - "core": { - "type": ["string", "null"], - "default": null - }, - "flight": { - "type": ["number", "null"], - "default": null - }, - "gridfins": { - "type": ["boolean", "null"], - "default": null - }, - "legs": { - "type": ["boolean", "null"], - "default": null - }, - "reused": { - "type": ["boolean", "null"], - "default": null - }, - "landing_attempt": { - "type": ["boolean", "null"], - "default": null - }, - "landing_success": { - "type": ["boolean", "null"], - "default": null - }, - "landing_type": { - "type": ["string", "null"], - "default": null - }, - "landpad": { - "type": ["string", "null"], - "default": null - } - } - } - }, - "links": { - "title": "links", - "type": "object", - "properties": { - "patch": { - "title": "patch", - "type": ["object", "null"], - "properties": { - "small": { - "type": ["string", "null"], - "default": null - }, - "large": { - "type": ["string", "null"], - "default": null - } - } - }, - "reddit": { - "title": "reddit", - "type": ["object", "null"], - "properties": { - "campaign": { - "type": ["string", "null"], - "default": null - }, - "launch": { - "type": ["string", "null"], - "default": null - }, - "media": { - "type": ["string", "null"], - "default": null - }, - "recovery": { - "type": ["string", "null"], - "default": null - } - } - }, - "flickr": { - "title": "flickr", - "type": "object", - "properties": { - "small": { - "type": "array", - "items": { - "type": "string" - } - }, - "original": { - "type": "array", - "items": { - "type": "string" - } - } - } - }, - "presskit": { - "type": ["string", "null"], - "default": null - }, - "webcast": { - "type": ["string", "null"], - "default": null - }, - "youtube_id": { - "type": ["string", "null"], - "default": null - }, - "article": { - "type": ["string", "null"], - "default": null - }, - "wikipedia": { - "type": ["string", "null"], - "default": null - } - } - }, - "auto_update": { - "type": "boolean", - "default": true - }, - "launch_library_id": { - "type": ["string", "null"], - "default": null - }, - "id": { - "type": "string", - "pattern": "^[0-9a-fA-F]{24}$" - } - }, - "required": [ - "flight_number", - "name", - "date_utc", - "date_unix", - "date_local", - "date_precision", - "upcoming" - ] -} diff --git a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/payloads.json b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/payloads.json deleted file mode 100644 index 4fccb0f859e9..000000000000 --- a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/payloads.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "payload": { - "title": "payload", - "type": "object", - "properties": { - "name": { - "type": ["string", "null"], - "default": null - }, - "type": { - "type": ["string", "null"], - "default": null - }, - "reused": { - "type": "boolean", - "default": false - }, - "launch": { - "type": ["string", "null"], - "default": null - }, - "customers": { - "type": "array", - "items": { - "type": "string" - } - }, - "norad_ids": { - "type": "array", - "items": { - "type": "number" - } - }, - "nationalities": { - "type": "array", - "items": { - "type": "string" - } - }, - "manufacturers": { - "type": "array", - "items": { - "type": "string" - } - }, - "mass_kg": { - "type": ["number", "null"], - "default": null - }, - "mass_lbs": { - "type": ["number", "null"], - "default": null - }, - "orbit": { - "type": ["string", "null"], - "default": null - }, - "reference_system": { - "type": ["string", "null"], - "default": null - }, - "regime": { - "type": ["string", "null"], - "default": null - }, - "longitude": { - "type": ["number", "null"], - "default": null - }, - "semi_major_axis_km": { - "type": ["number", "null"], - "default": null - }, - "eccentricity": { - "type": ["number", "null"], - "default": null - }, - "periapsis_km": { - "type": ["number", "null"], - "default": null - }, - "apoapsis_km": { - "type": ["number", "null"], - "default": null - }, - "inclination_deg": { - "type": ["number", "null"], - "default": null - }, - "period_min": { - "type": ["number", "null"], - "default": null - }, - "lifespan_years": { - "type": ["number", "null"], - "default": null - }, - "epoch": { - "type": ["string", "null"], - "default": null - }, - "mean_motion": { - "type": ["number", "null"], - "default": null - }, - "raan": { - "type": ["number", "null"], - "default": null - }, - "arg_of_pericenter": { - "type": ["number", "null"], - "default": null - }, - "mean_anomaly": { - "type": ["number", "null"], - "default": null - }, - "dragon": { - "title": "dragon", - "type": "object", - "properties": { - "capsule": { - "type": ["string", "null"], - "default": null - }, - "mass_returned_kg": { - "type": ["number", "null"], - "default": null - }, - "mass_returned_lbs": { - "type": ["number", "null"], - "default": null - }, - "flight_time_sec": { - "type": ["number", "null"], - "default": null - }, - "manifest": { - "type": ["string", "null"], - "default": null - }, - "water_landing": { - "type": ["boolean", "null"], - "default": null - }, - "land_landing": { - "type": ["boolean", "null"], - "default": null - } - } - } - } - }, - "id": { - "type": "string", - "pattern": "^[0-9a-fA-F]{24}$" - } - } -} diff --git a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/roadster.json b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/roadster.json deleted file mode 100644 index 223dd6b8e1e8..000000000000 --- a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/roadster.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "roadster": { - "title": "roadster", - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "launch_date_utc": { - "type": "string" - }, - "launch_date_unix": { - "type": "number" - }, - "launch_mass_kg": { - "type": "number" - }, - "launch_mass_lbs": { - "type": "number" - }, - "norad_id": { - "type": "number" - }, - "epoch_jd": { - "type": "number" - }, - "orbit_type": { - "type": "string" - }, - "apoapsis_au": { - "type": "number" - }, - "periapsis_au": { - "type": "number" - }, - "semi_major_axis_au": { - "type": "number" - }, - "eccentricity": { - "type": "number" - }, - "inclination": { - "type": "number" - }, - "longitude": { - "type": "number" - }, - "periapsis_arg": { - "type": "number" - }, - "period_days": { - "type": "number" - }, - "speed_kph": { - "type": "number" - }, - "speed_mph": { - "type": "number" - }, - "earth_distance_km": { - "type": "number" - }, - "earth_distance_mi": { - "type": "number" - }, - "mars_distance_km": { - "type": "number" - }, - "mars_distance_mi": { - "type": "number" - }, - "flickr_images": { - "type": "array", - "items": { - "type": "string" - } - }, - "wikipedia": { - "type": "string" - }, - "video": { - "type": "string" - }, - "details": { - "type": "string" - } - } - }, - "id": { - "type": "string", - "pattern": "^[0-9a-fA-F]{24}$" - } - } -} diff --git a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/rockets.json b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/rockets.json deleted file mode 100644 index 93d9dd73b38b..000000000000 --- a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/rockets.json +++ /dev/null @@ -1,292 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "rockets": { - "title": "rockets", - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "type": { - "type": "string" - }, - "active": { - "type": "boolean" - }, - "stages": { - "type": "number" - }, - "boosters": { - "type": "number" - }, - "cost_per_launch": { - "type": "number" - }, - "success_rate_pct": { - "type": "number" - }, - "first_flight": { - "type": "string" - }, - "country": { - "type": "string" - }, - "company": { - "type": "string" - }, - "height": { - "title": "height", - "type": "object", - "properties": { - "meters": { - "type": "number" - }, - "feet": { - "type": "number" - } - } - }, - "diameter": { - "title": "diameter", - "type": "object", - "properties": { - "meters": { - "type": "number" - }, - "feet": { - "type": "number" - } - } - }, - "mass": { - "title": "mass", - "type": "object", - "properties": { - "kg": { - "type": "number" - }, - "lb": { - "type": "number" - } - } - }, - "payload_weights": { - "type": "array", - "items": { - "type": "number" - } - }, - "first_stage": { - "title": "first_stage", - "type": "object", - "properties": { - "reusable": { - "type": "boolean" - }, - "engines": { - "type": "number" - }, - "fuel_amount_tons": { - "type": "number" - }, - "burn_time_sec": { - "type": "number" - }, - "thrust_sea_level": { - "title": "thrust_sea_level", - "type": "object", - "properties": { - "kN": { - "type": "number" - }, - "lbf": { - "type": "number" - } - } - }, - "thrust_vacuum": { - "title": "thrust_vacuum", - "type": "object", - "properties": { - "kN": { - "type": "number" - }, - "lbf": { - "type": "number" - } - } - } - } - }, - "second_stage": { - "title": "second_stage", - "type": "object", - "properties": { - "reusable": { - "type": "boolean" - }, - "engines": { - "type": "number" - }, - "fuel_amount_tons": { - "type": "number" - }, - "burn_time_sec": { - "type": "number" - }, - "thrust": { - "title": "thrust", - "type": "object", - "properties": { - "kN": { - "type": "number" - }, - "lbf": { - "type": "number" - } - } - }, - "payloads": { - "title": "payloads", - "type": "object", - "properties": { - "option_1": { - "type": "string" - }, - "composite_fairing": { - "title": "composite_fairing", - "type": "object", - "properties": { - "height": { - "title": "height", - "type": "object", - "properties": { - "meters": { - "type": "number" - }, - "feet": { - "type": "number" - } - } - }, - "diameter": { - "title": "diameter", - "type": "object", - "properties": { - "meters": { - "type": "number" - }, - "feet": { - "type": "number" - } - } - } - } - } - } - } - } - }, - "engines": { - "title": "engines", - "type": "object", - "properties": { - "number": { - "type": "number" - }, - "type": { - "type": "string" - }, - "version": { - "type": "string" - }, - "layout": { - "type": "string" - }, - "isp": { - "title": "isp", - "type": "object", - "properties": { - "sea_level": { - "type": "number" - }, - "vacuum": { - "type": "number" - } - } - }, - "engine_loss_max": { - "type": "number" - }, - "propellant_1": { - "type": "string" - }, - "propellant_2": { - "type": "string" - }, - "thrust_sea_level": { - "title": "thrust_sea_level", - "type": "object", - "properties": { - "kN": { - "type": "number" - }, - "lbf": { - "type": "number" - } - } - }, - "thrust_vacuum": { - "title": "thrust_vacuum", - "type": "object", - "properties": { - "kN": { - "type": "number" - }, - "lbf": { - "type": "number" - } - } - }, - "thrust_to_weight": { - "type": "number" - } - } - }, - "landing_legs": { - "title": "landing_legs", - "type": "object", - "properties": { - "number": { - "type": "number" - }, - "material": { - "type": "array", - "items": { - "type": "number" - } - } - } - }, - "flickr_images": { - "type": "array", - "items": { - "type": "string" - } - }, - "wikipedia": { - "type": "string" - }, - "description": { - "type": "string" - } - } - }, - "id": { - "type": "string", - "pattern": "^[0-9a-fA-F]{24}$" - } - } -} diff --git a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/ships.json b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/ships.json deleted file mode 100644 index 6a4f25d198da..000000000000 --- a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/ships.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "ships": { - "title": "ships", - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "legacy_id": { - "type": ["string", "null"], - "default": null - }, - "model": { - "type": ["string", "null"], - "default": null - }, - "type": { - "type": ["string", "null"], - "default": null - }, - "roles": { - "type": "array", - "items": { - "type": "string" - } - }, - "active": { - "type": "boolean" - }, - "imo": { - "type": ["number", "null"], - "default": null - }, - "mmsi": { - "type": ["number", "null"], - "default": null - }, - "abs": { - "type": ["number", "null"], - "default": null - }, - "class": { - "type": ["number", "null"], - "default": null - }, - "mass_kg": { - "type": ["number", "null"], - "default": null - }, - "mass_lbs": { - "type": ["number", "null"], - "default": null - }, - "year_built": { - "type": ["number", "null"], - "default": null - }, - "home_port": { - "type": ["string", "null"], - "default": null - }, - "status": { - "type": ["string", "null"], - "default": null - }, - "speed_kn": { - "type": ["number", "null"], - "default": null - }, - "course_deg": { - "type": ["number", "null"], - "default": null - }, - "latitude": { - "type": ["number", "null"], - "default": null - }, - "longitude": { - "type": ["number", "null"], - "default": null - }, - "last_ais_update": { - "type": ["string", "null"], - "default": null - }, - "link": { - "type": ["string", "null"], - "default": null - }, - "image": { - "type": ["string", "null"], - "default": null - }, - "launches": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": ["name", "active"] - }, - "id": { - "type": "string", - "pattern": "^[0-9a-fA-F]{24}$" - } - } -} diff --git a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/starlink.json b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/starlink.json deleted file mode 100644 index 48f0c5b1d1ed..000000000000 --- a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/schemas/starlink.json +++ /dev/null @@ -1,210 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "starlink": { - "title": "starlink", - "type": "object", - "properties": { - "version": { - "type": ["string", "null"], - "default": null - }, - "launch": { - "type": ["string", "null"], - "default": null - }, - "longitude": { - "type": ["number", "null"], - "default": null - }, - "latitude": { - "type": ["number", "null"], - "default": null - }, - "height_km": { - "type": ["number", "null"], - "default": null - }, - "velocity_kms": { - "type": ["number", "null"], - "default": null - }, - "spaceTrack": { - "title": "spaceTrack", - "type": "object", - "properties": { - "CCSDS_OMM_VERS": { - "type": ["string", "null"], - "default": null - }, - "COMMENT": { - "type": ["string", "null"], - "default": null - }, - "CREATION_DATE": { - "type": ["string", "null"], - "default": null - }, - "ORIGINATOR": { - "type": ["string", "null"], - "default": null - }, - "OBJECT_NAME": { - "type": ["string", "null"], - "default": null - }, - "OBJECT_ID": { - "type": ["string", "null"], - "default": null - }, - "CENTER_NAME": { - "type": ["string", "null"], - "default": null - }, - "REF_FRAME": { - "type": ["string", "null"], - "default": null - }, - "TIME_SYSTEM": { - "type": ["string", "null"], - "default": null - }, - "MEAN_ELEMENT_THEORY": { - "type": ["string", "null"], - "default": null - }, - "EPOCH": { - "type": ["string", "null"], - "default": null - }, - "MEAN_MOTION": { - "type": ["number", "null"], - "default": null - }, - "ECCENTRICITY": { - "type": ["number", "null"], - "default": null - }, - "INCLINATION": { - "type": ["number", "null"], - "default": null - }, - "RA_OF_ASC_NODE": { - "type": ["number", "null"], - "default": null - }, - "ARG_OF_PERICENTER": { - "type": ["number", "null"], - "default": null - }, - "MEAN_ANOMALY": { - "type": ["number", "null"], - "default": null - }, - "EPHEMERIS_TYPE": { - "type": ["number", "null"], - "default": null - }, - "CLASSIFICATION_TYPE": { - "type": ["string", "null"], - "default": null - }, - "NORAD_CAT_ID": { - "type": ["number", "null"], - "default": null - }, - "ELEMENT_SET_NO": { - "type": ["number", "null"], - "default": null - }, - "REV_AT_EPOCH": { - "type": ["number", "null"], - "default": null - }, - "BSTAR": { - "type": ["number", "null"], - "default": null - }, - "MEAN_MOTION_DOT": { - "type": ["number", "null"], - "default": null - }, - "MEAN_MOTION_DDOT": { - "type": ["number", "null"], - "default": null - }, - "SEMIMAJOR_AXIS": { - "type": ["number", "null"], - "default": null - }, - "PERIOD": { - "type": ["number", "null"], - "default": null - }, - "APOAPSIS": { - "type": ["number", "null"], - "default": null - }, - "PERIAPSIS": { - "type": ["number", "null"], - "default": null - }, - "OBJECT_TYPE": { - "type": ["string", "null"], - "default": null - }, - "RCS_SIZE": { - "type": ["string", "null"], - "default": null - }, - "COUNTRY_CODE": { - "type": ["string", "null"], - "default": null - }, - "LAUNCH_DATE": { - "type": ["string", "null"], - "default": null - }, - "SITE": { - "type": ["string", "null"], - "default": null - }, - "DECAY_DATE": { - "type": ["string", "null"], - "default": null - }, - "DECAYED": { - "type": ["number", "null"], - "default": null - }, - "FILE": { - "type": ["number", "null"], - "default": null - }, - "GP_ID": { - "type": ["number", "null"], - "default": null - }, - "TLE_LINE0": { - "type": ["string", "null"], - "default": null - }, - "TLE_LINE1": { - "type": ["string", "null"], - "default": null - }, - "TLE_LINE2": { - "type": ["string", "null"], - "default": null - } - } - } - } - }, - "id": { - "type": "string", - "pattern": "^[0-9a-fA-F]{24}$" - } - } -} diff --git a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/spec.yaml b/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/spec.yaml deleted file mode 100644 index 2344eb7c202b..000000000000 --- a/airbyte-integrations/connectors/source-spacex-api/source_spacex_api/spec.yaml +++ /dev/null @@ -1,15 +0,0 @@ -documentationUrl: https://docs.airbyte.com/integrations/sources/spacex-api -connectionSpecification: - $schema: http://json-schema.org/draft-07/schema# - title: Spacex Api Spec - type: object - additionalProperties: true - properties: - id: - title: Unique ID for specific source target - type: string - desciption: Optional, For a specific ID - options: - title: Configuration options for endpoints - type: string - desciption: Optional, Possible values for an endpoint. Example values for launches-latest, upcoming, past diff --git a/airbyte-integrations/connectors/source-square/Dockerfile b/airbyte-integrations/connectors/source-square/Dockerfile index c18311a47f48..5b9127dfed34 100644 --- a/airbyte-integrations/connectors/source-square/Dockerfile +++ b/airbyte-integrations/connectors/source-square/Dockerfile @@ -34,5 +34,5 @@ COPY source_square ./source_square ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=1.1.2 +LABEL io.airbyte.version=1.6.1 LABEL io.airbyte.name=airbyte/source-square diff --git a/airbyte-integrations/connectors/source-square/README.md b/airbyte-integrations/connectors/source-square/README.md index ebace58ab427..956131909881 100644 --- a/airbyte-integrations/connectors/source-square/README.md +++ b/airbyte-integrations/connectors/source-square/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-square:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/square) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_square/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-square:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-square build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-square:airbyteDocker +An image will be built with the tag `airbyte/source-square:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-square:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-square:dev check --con docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-square:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-square:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-square test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-square:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-square:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -2. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -3. Create a Pull Request. -4. Pat yourself on the back for being an awesome contributor. -5. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-square test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/square.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-square/acceptance-test-config.yml b/airbyte-integrations/connectors/source-square/acceptance-test-config.yml index e4593c92cfcf..9ea5ef9f749e 100644 --- a/airbyte-integrations/connectors/source-square/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-square/acceptance-test-config.yml @@ -3,58 +3,54 @@ test_strictness_level: high acceptance_tests: spec: tests: - - spec_path: "source_square/spec.yaml" - backward_compatibility_tests_config: - disable_for_version: "1.0.0" + - spec_path: "source_square/spec.yaml" + backward_compatibility_tests_config: + disable_for_version: "1.0.0" connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "secrets/config_oauth.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "secrets/config_oauth.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: tests: - - config_path: "secrets/config.json" - backward_compatibility_tests_config: - disable_for_version: "1.0.0" - - config_path: "secrets/config_oauth.json" - backward_compatibility_tests_config: - disable_for_version: "1.0.0" + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "1.0.0" + - config_path: "secrets/config_oauth.json" + backward_compatibility_tests_config: + disable_for_version: "1.0.0" basic_read: tests: - - config_path: "secrets/config_oauth.json" - empty_streams: - - name: shifts - bypass_reason: "Not able to fill stream" - - name: orders - bypass_reason: "Not able to fill stream" - - name: payments - bypass_reason: "Not able to fill stream" - - name: refunds - bypass_reason: "Not able to fill stream" - expect_records: - path: "integration_tests/expected_records_oauth.jsonl" - ignored_fields: - items: - - name: version - bypass_reason: "Floating data" - - name: updated_at - bypass_reason: "Floating data" - categories: - - name: version - bypass_reason: "Floating data" - - name: updated_at - bypass_reason: "Floating data" - fail_on_extra_columns: false + - config_path: "secrets/config.json" + empty_streams: + - name: cash_drawers + bypass_reason: "Not able to fill stream" + - name: bank_accounts + bypass_reason: "Not able to fill stream" + expect_records: + path: "integration_tests/expected_records.jsonl" + ignored_fields: + items: + - name: version + bypass_reason: "Floating data" + - name: updated_at + bypass_reason: "Floating data" + categories: + - name: version + bypass_reason: "Floating data" + - name: updated_at + bypass_reason: "Floating data" + fail_on_extra_columns: false incremental: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-square/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-square/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-square/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-square/build.gradle b/airbyte-integrations/connectors/source-square/build.gradle deleted file mode 100644 index 026d076902d9..000000000000 --- a/airbyte-integrations/connectors/source-square/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_square' -} diff --git a/airbyte-integrations/connectors/source-square/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-square/integration_tests/configured_catalog.json index be5690c59532..83ace2c03220 100644 --- a/airbyte-integrations/connectors/source-square/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-square/integration_tests/configured_catalog.json @@ -1,5 +1,17 @@ { "streams": [ + { + "stream": { + "name": "loyalty", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "full_refresh", + "cursor_field": ["id"], + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "items", @@ -24,6 +36,18 @@ "cursor_field": ["updated_at"], "destination_sync_mode": "append" }, + { + "stream": { + "name": "inventory", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["catalog_object_id"] + }, + "sync_mode": "full_refresh", + "cursor_field": ["catalog_object_id"], + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "discounts", @@ -155,6 +179,30 @@ "sync_mode": "incremental", "cursor_field": ["updated_at"], "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "cash_drawers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "full_refresh", + "cursor_field": ["id"], + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "bank_accounts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["id"] + }, + "sync_mode": "full_refresh", + "cursor_field": ["id"], + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-square/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-square/integration_tests/expected_records.jsonl index a2c1000cccc3..0c23b34e2e92 100644 --- a/airbyte-integrations/connectors/source-square/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-square/integration_tests/expected_records.jsonl @@ -1,29 +1,38 @@ -{"stream":"items","data":{"type":"ITEM","id":"K7CL577FVTGEGWEHZUU3NA6P","updated_at":"2021-06-10T22:17:32.995Z","created_at":"2021-06-10T09:58:41.311Z","version":1623363452995,"is_deleted":false,"custom_attribute_values":{"Square:eca67bfb-68a4-4218-950a-09eec906143d":{"name":"With coffee cup","string_value":"Yes","custom_attribute_definition_id":"VR43EHV5M3Z4P4CWA7K3ZXPA","type":"STRING","key":"Square:eca67bfb-68a4-4218-950a-09eec906143d"}},"present_at_all_locations":true,"item_data":{"name":"Coffee","description":"Some coffee drink","visibility":"PRIVATE","category_id":"WBVNPPUWI2YCVI2XJZNHPSQC","modifier_list_info":[{"modifier_list_id":"ZYESF2MGAMVORYB66VVXFW6V","visibility":"PUBLIC","min_selected_modifiers":-1,"max_selected_modifiers":-1,"enabled":true}],"variations":[{"type":"ITEM_VARIATION","id":"DT52FVGPUEJ7KL5WYPIK5TTP","updated_at":"2021-06-10T09:58:41.311Z","created_at":"2021-06-10T09:58:41.311Z","version":1623319121311,"is_deleted":false,"present_at_all_locations":true,"item_variation_data":{"item_id":"K7CL577FVTGEGWEHZUU3NA6P","name":"Black","sku":"1","ordinal":0,"pricing_type":"FIXED_PRICING","price_money":{"amount":900,"currency":"USD"},"location_overrides":[{"location_id":"LH2XR7AMG39HX","track_inventory":true}],"item_option_values":[{"item_option_id":"QTE3WP7JI64XLUD7AX5ER2ZI","item_option_value_id":"KRNNA4P57TDNVBJLWIBW5D47"}],"sellable":true,"stockable":true}},{"type":"ITEM_VARIATION","id":"SZTS6NG7OGC25KGTRXJEUAKK","updated_at":"2021-06-10T09:58:41.311Z","created_at":"2021-06-10T09:58:41.311Z","version":1623319121311,"is_deleted":false,"present_at_all_locations":true,"item_variation_data":{"item_id":"K7CL577FVTGEGWEHZUU3NA6P","name":"White","sku":"2","ordinal":1,"pricing_type":"FIXED_PRICING","price_money":{"amount":1000,"currency":"USD"},"location_overrides":[{"location_id":"LH2XR7AMG39HX","track_inventory":true}],"item_option_values":[{"item_option_id":"QTE3WP7JI64XLUD7AX5ER2ZI","item_option_value_id":"LIB3NSHGZUXFM3NFDHTUM4CJ"}],"sellable":true,"stockable":true}}],"product_type":"REGULAR","skip_modifier_screen":false,"item_options":[{"item_option_id":"QTE3WP7JI64XLUD7AX5ER2ZI"}],"ecom_visibility":"UNINDEXED"}},"emitted_at":1668290115725} -{"stream":"items","data":{"type":"ITEM","id":"SY3I7GJTCYIJTOD6PKLO6VKI","updated_at":"2021-06-10T21:07:19.929Z","created_at":"2021-06-10T21:07:19.929Z","version":1623359239929,"is_deleted":false,"present_at_all_locations":true,"item_data":{"name":"Tea","description":"Just the tea example","visibility":"PRIVATE","category_id":"WBVNPPUWI2YCVI2XJZNHPSQC","variations":[{"type":"ITEM_VARIATION","id":"PGOQKJWR6ALTCFPVGV54LHA6","updated_at":"2021-06-10T21:07:19.929Z","created_at":"2021-06-10T21:07:19.929Z","version":1623359239929,"is_deleted":false,"present_at_all_locations":true,"item_variation_data":{"item_id":"SY3I7GJTCYIJTOD6PKLO6VKI","name":"Regular","sku":"3","ordinal":1,"pricing_type":"FIXED_PRICING","price_money":{"amount":500,"currency":"USD"},"location_overrides":[{"location_id":"LH2XR7AMG39HX","track_inventory":true}],"sellable":true,"stockable":true}}],"product_type":"REGULAR","skip_modifier_screen":false,"ecom_visibility":"UNINDEXED"}},"emitted_at":1668290115725} -{"stream":"items","data":{"type":"ITEM","id":"UCYFEEPFPQFTWAGMH6T56L4U","updated_at":"2021-06-14T11:24:52.498Z","created_at":"2021-06-10T21:16:07.366Z","version":1623669892498,"is_deleted":false,"present_at_all_locations":true,"image_id":"7JPVLNMPVEBNXPQT5JXYCNF2","item_data":{"name":"Beer","description":"Unfiltered","visibility":"PRIVATE","category_id":"WBVNPPUWI2YCVI2XJZNHPSQC","variations":[{"type":"ITEM_VARIATION","id":"AVZZR4PLYHND3GQU5KD25GYD","updated_at":"2021-06-10T21:16:07.366Z","created_at":"2021-06-10T21:16:07.366Z","version":1623359767366,"is_deleted":false,"present_at_all_locations":true,"item_variation_data":{"item_id":"UCYFEEPFPQFTWAGMH6T56L4U","name":"Light","sku":"4","ordinal":1,"pricing_type":"FIXED_PRICING","price_money":{"amount":1100,"currency":"USD"},"location_overrides":[{"location_id":"LH2XR7AMG39HX","track_inventory":true}],"sellable":true,"stockable":true}},{"type":"ITEM_VARIATION","id":"QR3GL6QNSGG7TPGX3W6F72BK","updated_at":"2021-06-10T21:46:15.762Z","created_at":"2021-06-10T21:46:15.762Z","version":1623361575762,"is_deleted":false,"present_at_all_locations":false,"present_at_location_ids":["LH2XR7AMG39HX"],"item_variation_data":{"item_id":"UCYFEEPFPQFTWAGMH6T56L4U","name":"Unfiltered","sku":"6","ordinal":1,"pricing_type":"FIXED_PRICING","price_money":{"amount":1300,"currency":"USD"},"location_overrides":[{"location_id":"LH2XR7AMG39HX","track_inventory":true}],"sellable":true,"stockable":true}},{"type":"ITEM_VARIATION","id":"DJQ5F7NPJZCO7CMCPAU4GMTN","updated_at":"2021-06-10T21:16:07.366Z","created_at":"2021-06-10T21:16:07.366Z","version":1623359767366,"is_deleted":false,"present_at_all_locations":true,"item_variation_data":{"item_id":"UCYFEEPFPQFTWAGMH6T56L4U","name":"Dark","sku":"5","ordinal":2,"pricing_type":"FIXED_PRICING","price_money":{"amount":1200,"currency":"USD"},"location_overrides":[{"location_id":"LH2XR7AMG39HX","track_inventory":true}],"sellable":true,"stockable":true}}],"product_type":"REGULAR","skip_modifier_screen":false,"ecom_visibility":"UNINDEXED"}},"emitted_at":1668290115726} -{"stream":"categories","data":{"type":"CATEGORY","id":"WBVNPPUWI2YCVI2XJZNHPSQC","updated_at":"2022-10-19T19:33:30.646Z","created_at":"2021-06-10T09:58:41.311Z","version":1666208010646,"is_deleted":false,"present_at_all_locations":true,"category_data":{"name":"Drinks","is_top_level":true}},"emitted_at":1668290117366} -{"stream":"categories","data":{"type":"CATEGORY","id":"FIMYVNYAQ3JS337TP6YBQBBQ","updated_at":"2022-10-19T19:33:30.646Z","created_at":"2021-06-10T21:56:26.794Z","version":1666208010646,"is_deleted":false,"present_at_all_locations":true,"category_data":{"name":"Sign","is_top_level":true}},"emitted_at":1668290117366} -{"stream":"categories","data":{"type":"CATEGORY","id":"NC7RMZ5L7KR262JLJVJTWBDY","updated_at":"2022-10-19T19:33:30.646Z","created_at":"2021-06-10T21:56:26.794Z","version":1666208010646,"is_deleted":false,"present_at_all_locations":true,"category_data":{"name":"Quality","is_top_level":true}},"emitted_at":1668290117366} -{"stream":"discounts","data":{"type":"DISCOUNT","id":"Q7AQZ6WPGAWPFAIYBTYT7XXP","updated_at":"2021-06-14T13:47:48.539Z","created_at":"2021-06-10T22:15:06.693Z","version":1623678468539,"is_deleted":false,"present_at_all_locations":false,"present_at_location_ids":["L9A5Y0JR014G1"],"discount_data":{"name":"discount_20_p","discount_type":"FIXED_PERCENTAGE","percentage":"20.0","application_method":"MANUALLY_APPLIED","modify_tax_basis":"MODIFY_TAX_BASIS"}},"emitted_at":1668290117986} -{"stream":"discounts","data":{"type":"DISCOUNT","id":"HKYNSSNT2XWGYGPQNPVWFEAG","updated_at":"2021-06-14T13:48:20.638Z","created_at":"2021-06-10T22:15:45.239Z","version":1623678500638,"is_deleted":false,"present_at_all_locations":false,"present_at_location_ids":["LH2XR7AMG39HX"],"discount_data":{"name":"discount_5_p","discount_type":"FIXED_PERCENTAGE","percentage":"5.0","application_method":"MANUALLY_APPLIED","modify_tax_basis":"MODIFY_TAX_BASIS"}},"emitted_at":1668290117987} -{"stream":"discounts","data":{"type":"DISCOUNT","id":"TN6YCTI5DDYJTDQUW3VQ733L","updated_at":"2021-06-10T22:16:29.308Z","created_at":"2021-06-10T22:16:29.308Z","version":1623363389308,"is_deleted":false,"present_at_all_locations":true,"discount_data":{"name":"discount_1_usd","discount_type":"FIXED_AMOUNT","amount_money":{"amount":100,"currency":"USD"},"application_method":"MANUALLY_APPLIED","modify_tax_basis":"MODIFY_TAX_BASIS"}},"emitted_at":1668290117988} -{"stream":"taxes","data":{"type":"TAX","id":"CWU3GDBRZJ4TPNCVTX7AL6Q7","updated_at":"2021-06-14T13:12:47.779Z","created_at":"2021-06-10T22:13:33.029Z","version":1623676367779,"is_deleted":false,"present_at_all_locations":true,"tax_data":{"name":"20_p","calculation_phase":"TAX_SUBTOTAL_PHASE","inclusion_type":"ADDITIVE","percentage":"20.0","applies_to_custom_amounts":false,"enabled":true,"tax_type_id":"us_sales_tax","tax_type_name":"Sales Tax"}},"emitted_at":1668290118598} -{"stream":"taxes","data":{"type":"TAX","id":"C3EB6HITDFUUSQJIHM7KGFRU","updated_at":"2021-06-15T13:17:49.723Z","created_at":"2021-06-10T22:13:47.037Z","version":1623763069723,"is_deleted":false,"present_at_all_locations":true,"absent_at_location_ids":["LH2XR7AMG39HX"],"tax_data":{"name":"5_p","calculation_phase":"TAX_SUBTOTAL_PHASE","inclusion_type":"INCLUSIVE","percentage":"5.0","applies_to_custom_amounts":true,"enabled":true,"tax_type_id":"us_sales_tax","tax_type_name":"Sales Tax"}},"emitted_at":1668290118598} -{"stream":"taxes","data":{"type":"TAX","id":"5X7QCTRTQ7MEUFFWF2ESR3IA","updated_at":"2021-06-15T13:18:45.628Z","created_at":"2021-06-15T13:18:45.628Z","version":1623763125628,"is_deleted":false,"present_at_all_locations":true,"absent_at_location_ids":["L9A5Y0JR014G1"],"tax_data":{"name":"15_p","calculation_phase":"TAX_SUBTOTAL_PHASE","inclusion_type":"ADDITIVE","percentage":"15.0","applies_to_custom_amounts":true,"enabled":true,"tax_type_id":"us_sales_tax","tax_type_name":"Sales Tax"}},"emitted_at":1668290118598} -{"stream":"locations","data":{"id":"L9A5Y0JR014G1","name":"Coffe_shop","address":{"address_line_1":"1600 Pennsylvania Ave NW","locality":"Washington","administrative_district_level_1":"DC","postal_code":"20500","country":"US"},"timezone":"UTC","capabilities":["CREDIT_CARD_PROCESSING","AUTOMATIC_TRANSFERS"],"status":"ACTIVE","created_at":"2021-06-14T13:40:57.441Z","merchant_id":"ML7SCCR7EMAK4","country":"US","language_code":"en-US","currency":"USD","phone_number":"+1 800-444-4444","business_name":"Second_Test_Location","type":"PHYSICAL","website_url":"example.com","business_hours":{},"business_email":"some_email@coffee.com","description":"a brief bio","twitter_username":"test","instagram_username":"test","facebook_url":"facebook.com/example","mcc":"7299"},"emitted_at":1668290119400} -{"stream":"locations","data":{"id":"LH2XR7AMG39HX","name":"Default Test Account","address":{"address_line_1":"1600 Pennsylvania Ave NW","locality":"Washington","administrative_district_level_1":"DC","postal_code":"20500","country":"US"},"timezone":"UTC","capabilities":["CREDIT_CARD_PROCESSING","AUTOMATIC_TRANSFERS"],"status":"ACTIVE","created_at":"2021-04-30T05:16:05.977Z","merchant_id":"ML7SCCR7EMAK4","country":"US","language_code":"en-US","currency":"USD","business_name":"Default Test Account","type":"PHYSICAL","business_hours":{},"mcc":"7299"},"emitted_at":1668290119400} -{"stream":"team_members","data":{"id":"TMA-T96eUCnR9DkX","is_owner":true,"status":"ACTIVE","given_name":"Sandbox","family_name":"Seller","email_address":"sandbox-merchant+ryeggsjovidbpszhnwkskzvma10qzjcb@squareup.com","created_at":"2021-04-30T05:16:05Z","updated_at":"2021-04-30T05:16:05Z","assigned_locations":{"assignment_type":"ALL_CURRENT_AND_FUTURE_LOCATIONS"}},"emitted_at":1668290120212} -{"stream":"team_members","data":{"id":"TMcnrxWIJPlmU4c5","reference_id":"2","is_owner":false,"status":"ACTIVE","given_name":"Team","family_name":"Member_2","email_address":"team_member_2@airbyte.com","phone_number":"+19008080808","created_at":"2021-06-18T13:17:37Z","updated_at":"2021-06-18T13:17:37Z","assigned_locations":{"assignment_type":"EXPLICIT_LOCATIONS"}},"emitted_at":1668290120213} -{"stream":"team_members","data":{"id":"TMx95KdTStPnIxgp","reference_id":"1","is_owner":false,"status":"ACTIVE","given_name":"Team","family_name":"Member_1","email_address":"team_member_1@airbyte.com","phone_number":"+18008080808","created_at":"2021-06-18T13:15:49Z","updated_at":"2021-06-18T13:17:06Z","assigned_locations":{"assignment_type":"EXPLICIT_LOCATIONS"}},"emitted_at":1668290120213} -{"stream":"team_member_wages","data":{"id":"XGC1R9wiiymBJ4M1K8puuJGZ","team_member_id":"TMA-T96eUCnR9DkX","title":"Owner"},"emitted_at":1668290121140} -{"stream":"team_member_wages","data":{"id":"hFDaXrhWZ1BhnZLbrJTqqCfm","team_member_id":"TMcnrxWIJPlmU4c5","title":"Barista","hourly_rate":{"amount":2000,"currency":"USD"}},"emitted_at":1668290121141} -{"stream":"team_member_wages","data":{"id":"pC3birEsVhGyF58XjPvQ6BhD","team_member_id":"TMx95KdTStPnIxgp","title":"Cashier","hourly_rate":{"amount":2404,"currency":"USD"}},"emitted_at":1668290121141} -{"stream":"refunds","data":{"id":"NWO7kC96bJDUNKLovcUnapxGeOWZY_0um3GHK0AHt273xEe6I3u1y96Lnm018b0WAtyyOYRrP","status":"COMPLETED","amount_money":{"amount":1485,"currency":"USD"},"payment_id":"NWO7kC96bJDUNKLovcUnapxGeOWZY","order_id":"NpZRjYMGKOKeTe0BTp7N5r8kM0LZY","created_at":"2021-06-18T13:37:34.471Z","updated_at":"2021-06-18T13:37:37.319Z","processing_fee":[{"effective_at":"2021-06-18T15:31:43.000Z","type":"INITIAL","amount_money":{"amount":-51,"currency":"USD"}}],"location_id":"LH2XR7AMG39HX","reason":"Broken item","destination_type":"CARD"},"emitted_at":1668290122154} -{"stream":"refunds","data":{"id":"NWO7kC96bJDUNKLovcUnapxGeOWZY_BH7uyAXe6SqRc99uEExljMwERWZPci10Og6zIyfynAB","status":"COMPLETED","amount_money":{"amount":500,"currency":"USD"},"payment_id":"NWO7kC96bJDUNKLovcUnapxGeOWZY","order_id":"BFkIiV4W7baTDDx2CGGhdEzvTOCZY","created_at":"2021-06-22T19:35:20.612Z","updated_at":"2021-06-22T19:35:23.683Z","processing_fee":[{"effective_at":"2021-06-18T15:31:43.000Z","type":"INITIAL","amount_money":{"amount":-18,"currency":"USD"}}],"location_id":"LH2XR7AMG39HX","reason":"Overpayment","destination_type":"CARD"},"emitted_at":1668290122155} -{"stream":"customers","data":{"id":"WYP9CC9M156J71DMQF41Q8VMWW","created_at":"2021-06-18T14:02:43.476Z","updated_at":"2021-06-18T14:03:25Z","cards":[{"id":"ccof:k0ZuyEJ7sQGFsgfA4GB","card_brand":"VISA","last_4":"1111","exp_month":6,"exp_year":2026,"cardholder_name":"Test Customer","billing_address":{"postal_code":"12345"}}],"given_name":"Test","family_name":"Customer_1","email_address":"test_customer_1@airbyte.io","address":{"address_line_1":"street","address_line_2":"apt","locality":"city","administrative_district_level_1":"AL","postal_code":"35242"},"phone_number":"+18009090909","reference_id":"12345","company_name":"Company","preferences":{"email_unsubscribed":false},"creation_source":"DIRECTORY","birthday":"1990-08-09T00:00:00-00:00","segment_ids":["ML7SCCR7EMAK4.REACHABLE","ML7SCCR7EMAK4.CARDS_ON_FILE"],"version":1},"emitted_at":1668290124016} -{"stream":"modifier_list","data":{"type":"MODIFIER_LIST","id":"ZYESF2MGAMVORYB66VVXFW6V","updated_at":"2021-06-10T22:17:15.317Z","created_at":"2021-06-10T22:17:15.317Z","version":1623363435317,"is_deleted":false,"present_at_all_locations":true,"modifier_list_data":{"name":"With_accessory","selection_type":"MULTIPLE","modifiers":[{"type":"MODIFIER","id":"EW5DQKRKJH5LF2O5OM3TLT32","updated_at":"2021-06-10T22:17:15.317Z","created_at":"2021-06-10T22:17:15.317Z","version":1623363435317,"is_deleted":false,"present_at_all_locations":true,"modifier_data":{"name":"1_accessory","price_money":{"amount":100,"currency":"USD"},"on_by_default":false,"ordinal":1,"modifier_list_id":"ZYESF2MGAMVORYB66VVXFW6V"}},{"type":"MODIFIER","id":"APBZ4WD5P3FPUYSLG4U7MEJF","updated_at":"2021-06-10T22:17:15.317Z","created_at":"2021-06-10T22:17:15.317Z","version":1623363435317,"is_deleted":false,"present_at_all_locations":true,"modifier_data":{"name":"2_accessory","price_money":{"amount":200,"currency":"USD"},"on_by_default":false,"ordinal":2,"modifier_list_id":"ZYESF2MGAMVORYB66VVXFW6V"}}]}},"emitted_at":1668290124798} -{"stream":"modifier_list","data":{"type":"MODIFIER_LIST","id":"MKW7LLF4IRUX773KBHH4XQZA","updated_at":"2021-06-14T13:10:54.797Z","created_at":"2021-06-14T13:10:54.797Z","version":1623676254797,"is_deleted":false,"present_at_all_locations":true,"modifier_list_data":{"name":"With_something_else","selection_type":"MULTIPLE","modifiers":[{"type":"MODIFIER","id":"IA66H4C4C6JNXMHXQI3LDWFP","updated_at":"2021-06-14T13:10:54.797Z","created_at":"2021-06-14T13:10:54.797Z","version":1623676254797,"is_deleted":false,"present_at_all_locations":true,"modifier_data":{"name":"something_else","price_money":{"amount":1000,"currency":"USD"},"on_by_default":false,"ordinal":1,"modifier_list_id":"MKW7LLF4IRUX773KBHH4XQZA"}},{"type":"MODIFIER","id":"CS5VQADEB4GZXEL3TWHQDRER","updated_at":"2021-06-14T13:10:54.797Z","created_at":"2021-06-14T13:10:54.797Z","version":1623676254797,"is_deleted":false,"present_at_all_locations":true,"modifier_data":{"name":"something_else_2","price_money":{"amount":1500,"currency":"USD"},"on_by_default":false,"ordinal":2,"modifier_list_id":"MKW7LLF4IRUX773KBHH4XQZA"}}]}},"emitted_at":1668290124799} -{"stream":"shifts","data":{"id":"M60G9R7E1H52J","employee_id":"TMA-T96eUCnR9DkX","location_id":"L9A5Y0JR014G1","timezone":"UTC","start_at":"2021-06-17T08:00:00Z","end_at":"2021-06-17T20:00:00Z","wage":{"title":"Owner","hourly_rate":{"amount":4050,"currency":"USD"}},"breaks":[{"id":"ZXR4CMNAEGXW6","start_at":"2021-06-17T10:00:00Z","end_at":"2021-06-17T11:00:00Z","break_type_id":"HDY9769K81MN7","name":"Lunch Break","expected_duration":"PT1H","is_paid":true},{"id":"2N4RYD910S698","start_at":"2021-06-17T17:00:00Z","end_at":"2021-06-17T17:30:00Z","break_type_id":"NEHDKJ0V03XP2","name":"Tea Break","expected_duration":"PT30M","is_paid":true}],"status":"CLOSED","version":1,"created_at":"2021-06-18T20:46:59Z","updated_at":"2021-06-18T20:46:59Z","team_member_id":"TMA-T96eUCnR9DkX"},"emitted_at":1668290125739} -{"stream":"shifts","data":{"id":"WET1AZXN164BB","employee_id":"TMA-T96eUCnR9DkX","location_id":"L9A5Y0JR014G1","timezone":"UTC","start_at":"2019-01-25T08:11:00Z","end_at":"2019-01-25T18:11:00Z","wage":{"hourly_rate":{"amount":1100,"currency":"USD"}},"breaks":[{"id":"Q00NYDFJ36K9Y","start_at":"2019-01-25T11:11:00Z","end_at":"2019-01-25T11:41:00Z","break_type_id":"NEHDKJ0V03XP2","name":"Tea Break","expected_duration":"PT30M","is_paid":true}],"status":"CLOSED","version":1,"created_at":"2021-06-18T20:37:39Z","updated_at":"2021-06-18T20:37:39Z","team_member_id":"TMA-T96eUCnR9DkX"},"emitted_at":1668290125741} -{"stream":"orders","data":{"id":"xHRYELdzIG3Fb9gJPuPk4shKafUZY","location_id":"LH2XR7AMG39HX","line_items":[{"uid":"h5XEkuLUYQ1LS7B8o42dVD","catalog_object_id":"PJF2F6CAXLYBCTSKHAY2HXXW","catalog_version":1624046773184,"quantity":"2","name":"Health drop","variation_name":"Congress","base_price_money":{"amount":10900,"currency":"USD"},"gross_sales_money":{"amount":21800,"currency":"USD"},"total_tax_money":{"amount":0,"currency":"USD"},"total_discount_money":{"amount":0,"currency":"USD"},"total_money":{"amount":21800,"currency":"USD"},"variation_total_price_money":{"amount":21800,"currency":"USD"},"item_type":"ITEM"}],"fulfillments":[{"uid":"8ghcTCAgVT6kqWbrCdQxVB","type":"PICKUP","state":"COMPLETED","pickup_details":{"expires_at":"2021-07-01T20:21:54.859Z","pickup_at":"2021-07-01T19:21:54.859Z","note":"Pour over coffee","placed_at":"2021-06-29T21:32:32.387Z","accepted_at":"2021-06-29T21:47:07.804Z","ready_at":"2021-06-29T21:47:14.292Z","schedule_type":"SCHEDULED","recipient":{"display_name":"Jaiden Urie"},"auto_complete_duration":"P0DT1H0S","picked_up_at":"2021-06-29T21:47:16.440Z"}}],"created_at":"2021-06-29T21:09:26.402Z","updated_at":"2021-06-29T21:47:16.441Z","state":"COMPLETED","version":8,"total_tax_money":{"amount":0,"currency":"USD"},"total_discount_money":{"amount":0,"currency":"USD"},"total_tip_money":{"amount":0,"currency":"USD"},"total_money":{"amount":21800,"currency":"USD"},"closed_at":"2021-06-29T21:47:16.441Z","tenders":[{"id":"RK7310wbjzTWtIxoW1btCPNqYT6YY","location_id":"LH2XR7AMG39HX","transaction_id":"xHRYELdzIG3Fb9gJPuPk4shKafUZY","created_at":"2021-06-29T21:32:32Z","amount_money":{"amount":21800,"currency":"USD"},"type":"CARD","card_details":{"status":"CAPTURED","card":{"card_brand":"VISA","last_4":"5858","fingerprint":"sq-1-ebU3Ci-dcOxf-pVya9fDChHVLpXmNo73UaTsGbKLjBVtdqie8txHwuAY1SxA2F3c0g"},"entry_method":"KEYED"},"payment_id":"RK7310wbjzTWtIxoW1btCPNqYT6YY"}],"total_service_charge_money":{"amount":0,"currency":"USD"},"net_amounts":{"total_money":{"amount":21800,"currency":"USD"},"tax_money":{"amount":0,"currency":"USD"},"discount_money":{"amount":0,"currency":"USD"},"tip_money":{"amount":0,"currency":"USD"},"service_charge_money":{"amount":0,"currency":"USD"}},"source":{"name":"Sandbox for sq0idp-7KVC6qHcSDMXsm40SAA9TA"}},"emitted_at":1668290127537} -{"stream":"orders","data":{"id":"rOIV8oJ0c4jHUym6dQUscxJSUb4F","location_id":"L9A5Y0JR014G1","line_items":[{"uid":"EZHPqvnWeiTrOw8ONrDBKB","catalog_object_id":"DT52FVGPUEJ7KL5WYPIK5TTP","catalog_version":1624046773184,"quantity":"2","name":"Coffee","variation_name":"Black","base_price_money":{"amount":900,"currency":"USD"},"modifiers":[{"uid":"x5bl7VpXSZNqByeAvcvZyD","base_price_money":{"amount":100,"currency":"USD"},"total_price_money":{"amount":200,"currency":"USD"},"name":"1_accessory","catalog_object_id":"EW5DQKRKJH5LF2O5OM3TLT32","catalog_version":1624046773184}],"gross_sales_money":{"amount":2000,"currency":"USD"},"total_tax_money":{"amount":0,"currency":"USD"},"total_discount_money":{"amount":0,"currency":"USD"},"total_money":{"amount":2000,"currency":"USD"},"variation_total_price_money":{"amount":1800,"currency":"USD"},"item_type":"ITEM"}],"fulfillments":[{"uid":"yNY39gzb50jUC1oZ9HxsFD","type":"PICKUP","state":"COMPLETED","pickup_details":{"expires_at":"2021-07-01T20:21:54.859Z","pickup_at":"2021-07-01T19:21:54.859Z","note":"Pour over coffee","placed_at":"2021-06-29T21:36:04.916Z","accepted_at":"2021-06-29T21:47:18.618Z","ready_at":"2021-06-29T21:47:18.618Z","schedule_type":"SCHEDULED","recipient":{"display_name":"Jaiden Urie"},"auto_complete_duration":"P0DT1H0S","picked_up_at":"2021-06-29T21:47:18.618Z"}}],"created_at":"2021-06-29T21:06:22.910Z","updated_at":"2021-06-29T21:47:18.620Z","state":"COMPLETED","version":5,"total_tax_money":{"amount":0,"currency":"USD"},"total_discount_money":{"amount":0,"currency":"USD"},"total_tip_money":{"amount":0,"currency":"USD"},"total_money":{"amount":2000,"currency":"USD"},"closed_at":"2021-06-29T21:47:18.620Z","tenders":[{"id":"HrQAhauwBhT6fdNRfguwoPyCPTdZY","location_id":"L9A5Y0JR014G1","transaction_id":"rOIV8oJ0c4jHUym6dQUscxJSUb4F","created_at":"2021-06-29T21:36:04Z","amount_money":{"amount":2000,"currency":"USD"},"type":"CASH","cash_details":{"buyer_tendered_money":{"amount":3000,"currency":"USD"},"change_back_money":{"amount":1000,"currency":"USD"}},"payment_id":"HrQAhauwBhT6fdNRfguwoPyCPTdZY"}],"total_service_charge_money":{"amount":0,"currency":"USD"},"net_amounts":{"total_money":{"amount":2000,"currency":"USD"},"tax_money":{"amount":0,"currency":"USD"},"discount_money":{"amount":0,"currency":"USD"},"tip_money":{"amount":0,"currency":"USD"},"service_charge_money":{"amount":0,"currency":"USD"}},"source":{"name":"Sandbox for sq0idp-7KVC6qHcSDMXsm40SAA9TA"}},"emitted_at":1668290127538} +{"stream": "team_member_wages", "data": {"id": "XGC1R9wiiymBJ4M1K8puuJGZ", "team_member_id": "TMA-T96eUCnR9DkX", "title": "Owner"}, "emitted_at": 1697217912786} +{"stream": "team_member_wages", "data": {"id": "hFDaXrhWZ1BhnZLbrJTqqCfm", "team_member_id": "TMcnrxWIJPlmU4c5", "title": "Barista", "hourly_rate": {"amount": 2000, "currency": "USD"}}, "emitted_at": 1697217912790} +{"stream": "team_member_wages", "data": {"id": "pC3birEsVhGyF58XjPvQ6BhD", "team_member_id": "TMx95KdTStPnIxgp", "title": "Cashier", "hourly_rate": {"amount": 2404, "currency": "USD"}}, "emitted_at": 1697217912793} +{"stream": "refunds", "data": {"id": "NWO7kC96bJDUNKLovcUnapxGeOWZY_0um3GHK0AHt273xEe6I3u1y96Lnm018b0WAtyyOYRrP", "status": "COMPLETED", "amount_money": {"amount": 1485, "currency": "USD"}, "payment_id": "NWO7kC96bJDUNKLovcUnapxGeOWZY", "order_id": "NpZRjYMGKOKeTe0BTp7N5r8kM0LZY", "created_at": "2021-06-18T13:37:34.471Z", "updated_at": "2021-06-18T13:37:37.319Z", "processing_fee": [{"effective_at": "2021-06-18T15:31:43.000Z", "type": "INITIAL", "amount_money": {"amount": -51, "currency": "USD"}}], "location_id": "LH2XR7AMG39HX", "reason": "Broken item", "destination_type": "CARD"}, "emitted_at": 1697217913234} +{"stream": "refunds", "data": {"id": "NWO7kC96bJDUNKLovcUnapxGeOWZY_BH7uyAXe6SqRc99uEExljMwERWZPci10Og6zIyfynAB", "status": "COMPLETED", "amount_money": {"amount": 500, "currency": "USD"}, "payment_id": "NWO7kC96bJDUNKLovcUnapxGeOWZY", "order_id": "BFkIiV4W7baTDDx2CGGhdEzvTOCZY", "created_at": "2021-06-22T19:35:20.612Z", "updated_at": "2021-06-22T19:35:23.683Z", "processing_fee": [{"effective_at": "2021-06-18T15:31:43.000Z", "type": "INITIAL", "amount_money": {"amount": -18, "currency": "USD"}}], "location_id": "LH2XR7AMG39HX", "reason": "Overpayment", "destination_type": "CARD"}, "emitted_at": 1697217913243} +{"stream": "customers", "data": {"id": "WYP9CC9M156J71DMQF41Q8VMWW", "created_at": "2021-06-18T14:02:43.476Z", "updated_at": "2021-06-18T14:03:25Z", "cards": [{"id": "ccof:k0ZuyEJ7sQGFsgfA4GB", "card_brand": "VISA", "last_4": "1111", "exp_month": 6, "exp_year": 2026, "cardholder_name": "Test Customer", "billing_address": {"postal_code": "12345"}}], "given_name": "Test", "family_name": "Customer_1", "email_address": "test_customer_1@airbyte.io", "address": {"address_line_1": "street", "address_line_2": "apt", "locality": "city", "administrative_district_level_1": "AL", "postal_code": "35242"}, "phone_number": "+18009090909", "reference_id": "12345", "company_name": "Company", "preferences": {"email_unsubscribed": false}, "creation_source": "DIRECTORY", "birthday": "1990-08-09T00:00:00-00:00", "segment_ids": ["ML7SCCR7EMAK4.CARDS_ON_FILE", "ML7SCCR7EMAK4.REACHABLE"], "version": 1}, "emitted_at": 1697217931447} +{"stream": "taxes", "data": {"type": "TAX", "id": "CWU3GDBRZJ4TPNCVTX7AL6Q7", "updated_at": "2021-06-14T13:12:47.779Z", "created_at": "2021-06-10T22:13:33.029Z", "version": 1623676367779, "is_deleted": false, "present_at_all_locations": true, "tax_data": {"name": "20_p", "calculation_phase": "TAX_SUBTOTAL_PHASE", "inclusion_type": "ADDITIVE", "percentage": "20.0", "applies_to_custom_amounts": false, "enabled": true, "tax_type_id": "us_sales_tax", "tax_type_name": "Sales Tax"}}, "emitted_at": 1697217904696} +{"stream": "taxes", "data": {"type": "TAX", "id": "C3EB6HITDFUUSQJIHM7KGFRU", "updated_at": "2021-06-15T13:17:49.723Z", "created_at": "2021-06-10T22:13:47.037Z", "version": 1623763069723, "is_deleted": false, "present_at_all_locations": true, "absent_at_location_ids": ["LH2XR7AMG39HX"], "tax_data": {"name": "5_p", "calculation_phase": "TAX_SUBTOTAL_PHASE", "inclusion_type": "INCLUSIVE", "percentage": "5.0", "applies_to_custom_amounts": true, "enabled": true, "tax_type_id": "us_sales_tax", "tax_type_name": "Sales Tax"}}, "emitted_at": 1697217904702} +{"stream": "taxes", "data": {"type": "TAX", "id": "5X7QCTRTQ7MEUFFWF2ESR3IA", "updated_at": "2021-06-15T13:18:45.628Z", "created_at": "2021-06-15T13:18:45.628Z", "version": 1623763125628, "is_deleted": false, "present_at_all_locations": true, "absent_at_location_ids": ["L9A5Y0JR014G1"], "tax_data": {"name": "15_p", "calculation_phase": "TAX_SUBTOTAL_PHASE", "inclusion_type": "ADDITIVE", "percentage": "15.0", "applies_to_custom_amounts": true, "enabled": true, "tax_type_id": "us_sales_tax", "tax_type_name": "Sales Tax"}}, "emitted_at": 1697217904706} +{"stream": "payments", "data": {"id": "9m4YvEyzLRUvwUeBf2DNtVOh6cIZY", "created_at": "2021-06-08T20:21:39.212Z", "updated_at": "2021-06-08T20:21:41.165Z", "amount_money": {"amount": 100, "currency": "USD"}, "status": "COMPLETED", "delay_duration": "PT168H", "source_type": "CARD", "card_details": {"status": "CAPTURED", "card": {"card_brand": "MASTERCARD", "last_4": "9029", "exp_month": 6, "exp_year": 2023, "fingerprint": "sq-1-MTQOLCjEOIzHvJvKX4yxf6qBvj6DAFuB8wlWoKW4NI1BAFV5cdlJmge8ehPFGUSeuw", "card_type": "CREDIT", "prepaid_type": "NOT_PREPAID", "bin": "540988"}, "entry_method": "KEYED", "cvv_status": "CVV_ACCEPTED", "avs_status": "AVS_ACCEPTED", "statement_description": "SQ *DEFAULT TEST ACCOUNT", "card_payment_timeline": {"authorized_at": "2021-06-08T20:21:39.320Z", "captured_at": "2021-06-08T20:21:39.395Z"}}, "location_id": "LH2XR7AMG39HX", "order_id": "jqYrf6arFpUo7zElfWu9GRF5lAWZY", "risk_evaluation": {"created_at": "2021-06-08T20:21:39.321Z", "risk_level": "NORMAL"}, "processing_fee": [{"effective_at": "2021-06-08T22:21:41.000Z", "type": "INITIAL", "amount_money": {"amount": 33, "currency": "USD"}}], "total_money": {"amount": 100, "currency": "USD"}, "approved_money": {"amount": 100, "currency": "USD"}, "receipt_number": "9m4Y", "receipt_url": "https://squareupsandbox.com/receipt/preview/9m4YvEyzLRUvwUeBf2DNtVOh6cIZY", "delay_action": "CANCEL", "delayed_until": "2021-06-15T20:21:39.212Z", "application_details": {"square_product": "ECOMMERCE_API", "application_id": "sandbox-sq0idb-Nd7U5HfhPMxxK3f1Me-yKw"}, "version_token": "d4BjlOwbOUGifHe9BMhuSCRTDGvKA1MYm3aaTzOjbCT6o"}, "emitted_at": 1697217922482} +{"stream": "payments", "data": {"id": "rLBl9k8kKVV8uXNymUEct6S2ebIZY", "created_at": "2021-06-18T13:30:27.850Z", "updated_at": "2021-06-18T13:30:28.721Z", "amount_money": {"amount": 2056, "currency": "USD"}, "status": "COMPLETED", "delay_duration": "PT168H", "source_type": "CARD", "card_details": {"status": "CAPTURED", "card": {"card_brand": "VISA", "last_4": "1111", "exp_month": 6, "exp_year": 2026, "fingerprint": "sq-1-mqW9yIk2eKV4LdXhGzf-FYu1knqb1IT7lXybOaFbMwIH2-9d1qdVOGNUMA8TDALoqg", "card_type": "CREDIT", "bin": "411111"}, "entry_method": "KEYED", "cvv_status": "CVV_ACCEPTED", "avs_status": "AVS_ACCEPTED", "statement_description": "SQ *DEFAULT TEST ACCOUNT", "card_payment_timeline": {"authorized_at": "2021-06-18T13:30:27.959Z", "captured_at": "2021-06-18T13:30:28.030Z"}}, "location_id": "LH2XR7AMG39HX", "order_id": "hD1xqUBBHQ3ejMBQiSSmncrYg7OZY", "processing_fee": [{"effective_at": "2021-06-18T15:30:28.000Z", "type": "INITIAL", "amount_money": {"amount": 87, "currency": "USD"}}], "note": "20$ money payment", "total_money": {"amount": 2056, "currency": "USD"}, "approved_money": {"amount": 2056, "currency": "USD"}, "employee_id": "TMA-T96eUCnR9DkX", "receipt_number": "rLBl", "receipt_url": "https://squareupsandbox.com/receipt/preview/rLBl9k8kKVV8uXNymUEct6S2ebIZY", "delay_action": "CANCEL", "delayed_until": "2021-06-25T13:30:27.850Z", "team_member_id": "TMA-T96eUCnR9DkX", "application_details": {"square_product": "VIRTUAL_TERMINAL", "application_id": "sandbox-sq0idb-BbZvlaIkgSYnVUI4rpSedg"}, "version_token": "KcdXvgWNGUYOUmWdF4K0Cmi5bhfxSLMIc12PwLSAG9e6o"}, "emitted_at": 1697217922488} +{"stream": "payments", "data": {"id": "NWO7kC96bJDUNKLovcUnapxGeOWZY", "created_at": "2021-06-18T13:31:43.040Z", "updated_at": "2021-06-22T19:35:23.683Z", "amount_money": {"amount": 11385, "currency": "USD"}, "refunded_money": {"amount": 1985, "currency": "USD"}, "status": "COMPLETED", "delay_duration": "PT168H", "source_type": "CARD", "card_details": {"status": "CAPTURED", "card": {"card_brand": "VISA", "last_4": "1111", "exp_month": 6, "exp_year": 2026, "fingerprint": "sq-1-mqW9yIk2eKV4LdXhGzf-FYu1knqb1IT7lXybOaFbMwIH2-9d1qdVOGNUMA8TDALoqg", "card_type": "CREDIT", "bin": "411111"}, "entry_method": "KEYED", "cvv_status": "CVV_ACCEPTED", "avs_status": "AVS_ACCEPTED", "statement_description": "SQ *DEFAULT TEST ACCOUNT", "card_payment_timeline": {"authorized_at": "2021-06-18T13:31:43.148Z", "captured_at": "2021-06-18T13:31:43.258Z"}}, "location_id": "LH2XR7AMG39HX", "order_id": "BxCc4Y2KBt10BUWQheazcgRUR7bZY", "refund_ids": ["NWO7kC96bJDUNKLovcUnapxGeOWZY_0um3GHK0AHt273xEe6I3u1y96Lnm018b0WAtyyOYRrP", "NWO7kC96bJDUNKLovcUnapxGeOWZY_BH7uyAXe6SqRc99uEExljMwERWZPci10Og6zIyfynAB"], "processing_fee": [{"effective_at": "2021-06-18T15:31:43.000Z", "type": "INITIAL", "amount_money": {"amount": 413, "currency": "USD"}}], "note": "113,85$ payment", "total_money": {"amount": 11385, "currency": "USD"}, "approved_money": {"amount": 11385, "currency": "USD"}, "employee_id": "TMA-T96eUCnR9DkX", "receipt_number": "NWO7", "receipt_url": "https://squareupsandbox.com/receipt/preview/NWO7kC96bJDUNKLovcUnapxGeOWZY", "delay_action": "CANCEL", "delayed_until": "2021-06-25T13:31:43.040Z", "team_member_id": "TMA-T96eUCnR9DkX", "application_details": {"square_product": "VIRTUAL_TERMINAL", "application_id": "sandbox-sq0idb-BbZvlaIkgSYnVUI4rpSedg"}, "version_token": "JuGNurRABx1mNvkMlYLtu8LI05JpiYOupyZwZtoQELk6o"}, "emitted_at": 1697217922492} +{"stream": "categories", "data": {"type": "CATEGORY", "id": "WBVNPPUWI2YCVI2XJZNHPSQC", "updated_at": "2022-10-19T19:33:30.646Z", "created_at": "2021-06-10T09:58:41.311Z", "version": 1666208010646, "is_deleted": false, "present_at_all_locations": true, "category_data": {"name": "Drinks", "is_top_level": true}}, "emitted_at": 1697217887093} +{"stream": "categories", "data": {"type": "CATEGORY", "id": "FIMYVNYAQ3JS337TP6YBQBBQ", "updated_at": "2022-10-19T19:33:30.646Z", "created_at": "2021-06-10T21:56:26.794Z", "version": 1666208010646, "is_deleted": false, "present_at_all_locations": true, "category_data": {"name": "Sign", "is_top_level": true}}, "emitted_at": 1697217887096} +{"stream": "categories", "data": {"type": "CATEGORY", "id": "NC7RMZ5L7KR262JLJVJTWBDY", "updated_at": "2022-10-19T19:33:30.646Z", "created_at": "2021-06-10T21:56:26.794Z", "version": 1666208010646, "is_deleted": false, "present_at_all_locations": true, "category_data": {"name": "Quality", "is_top_level": true}}, "emitted_at": 1697217887099} +{"stream": "team_members", "data": {"id": "TMA-T96eUCnR9DkX", "is_owner": true, "status": "ACTIVE", "given_name": "Sandbox", "family_name": "Seller", "email_address": "sandbox-merchant+ryeggsjovidbpszhnwkskzvma10qzjcb@squareup.com", "created_at": "2021-04-30T05:16:05Z", "updated_at": "2023-07-07T17:07:41Z", "assigned_locations": {"assignment_type": "ALL_CURRENT_AND_FUTURE_LOCATIONS"}}, "emitted_at": 1697217912327} +{"stream": "team_members", "data": {"id": "TMcnrxWIJPlmU4c5", "reference_id": "2", "is_owner": false, "status": "ACTIVE", "given_name": "Team", "family_name": "Member_2", "email_address": "team_member_2@airbyte.com", "phone_number": "+19008080808", "created_at": "2021-06-18T13:17:37Z", "updated_at": "2021-06-18T13:17:37Z", "assigned_locations": {"assignment_type": "EXPLICIT_LOCATIONS"}}, "emitted_at": 1697217912332} +{"stream": "team_members", "data": {"id": "TMx95KdTStPnIxgp", "reference_id": "1", "is_owner": false, "status": "ACTIVE", "given_name": "Team", "family_name": "Member_1", "email_address": "team_member_1@airbyte.com", "phone_number": "+18008080808", "created_at": "2021-06-18T13:15:49Z", "updated_at": "2021-06-18T13:17:06Z", "assigned_locations": {"assignment_type": "EXPLICIT_LOCATIONS"}}, "emitted_at": 1697217912335} +{"stream": "shifts", "data": {"id": "M60G9R7E1H52J", "employee_id": "TMA-T96eUCnR9DkX", "location_id": "L9A5Y0JR014G1", "timezone": "UTC", "start_at": "2021-06-17T08:00:00Z", "end_at": "2021-06-17T20:00:00Z", "wage": {"title": "Owner", "hourly_rate": {"amount": 4050, "currency": "USD"}}, "breaks": [{"id": "ZXR4CMNAEGXW6", "start_at": "2021-06-17T10:00:00Z", "end_at": "2021-06-17T11:00:00Z", "break_type_id": "HDY9769K81MN7", "name": "Lunch Break", "expected_duration": "PT1H", "is_paid": true}, {"id": "2N4RYD910S698", "start_at": "2021-06-17T17:00:00Z", "end_at": "2021-06-17T17:30:00Z", "break_type_id": "NEHDKJ0V03XP2", "name": "Tea Break", "expected_duration": "PT30M", "is_paid": true}], "status": "CLOSED", "version": 1, "created_at": "2021-06-18T20:46:59Z", "updated_at": "2021-06-18T20:46:59Z", "team_member_id": "TMA-T96eUCnR9DkX"}, "emitted_at": 1697217938884} +{"stream": "shifts", "data": {"id": "WET1AZXN164BB", "employee_id": "TMA-T96eUCnR9DkX", "location_id": "L9A5Y0JR014G1", "timezone": "UTC", "start_at": "2019-01-25T08:11:00Z", "end_at": "2019-01-25T18:11:00Z", "wage": {"hourly_rate": {"amount": 1100, "currency": "USD"}}, "breaks": [{"id": "Q00NYDFJ36K9Y", "start_at": "2019-01-25T11:11:00Z", "end_at": "2019-01-25T11:41:00Z", "break_type_id": "NEHDKJ0V03XP2", "name": "Tea Break", "expected_duration": "PT30M", "is_paid": true}], "status": "CLOSED", "version": 1, "created_at": "2021-06-18T20:37:39Z", "updated_at": "2021-06-18T20:37:39Z", "team_member_id": "TMA-T96eUCnR9DkX"}, "emitted_at": 1697217938888} +{"stream": "locations", "data": {"id": "L9A5Y0JR014G1", "name": "Coffe_shop", "address": {"address_line_1": "1600 Pennsylvania Ave NW", "locality": "Washington", "administrative_district_level_1": "DC", "postal_code": "20500", "country": "US"}, "timezone": "UTC", "capabilities": ["CREDIT_CARD_PROCESSING", "AUTOMATIC_TRANSFERS"], "status": "ACTIVE", "created_at": "2021-06-14T13:40:57.441Z", "merchant_id": "ML7SCCR7EMAK4", "country": "US", "language_code": "en-US", "currency": "USD", "phone_number": "+1 800-444-4444", "business_name": "Second_Test_Location", "type": "PHYSICAL", "website_url": "example.com", "business_hours": {}, "business_email": "some_email@coffee.com", "description": "a brief bio", "twitter_username": "test", "instagram_username": "test", "facebook_url": "facebook.com/example", "mcc": "7299"}, "emitted_at": 1697217911705} +{"stream": "locations", "data": {"id": "LH2XR7AMG39HX", "name": "Default Test Account", "address": {"address_line_1": "1600 Pennsylvania Ave NW", "locality": "Washington", "administrative_district_level_1": "DC", "postal_code": "20500", "country": "US"}, "timezone": "UTC", "capabilities": ["CREDIT_CARD_PROCESSING", "AUTOMATIC_TRANSFERS"], "status": "ACTIVE", "created_at": "2021-04-30T05:16:05.977Z", "merchant_id": "ML7SCCR7EMAK4", "country": "US", "language_code": "en-US", "currency": "USD", "business_name": "Default Test Account", "type": "PHYSICAL", "business_hours": {}, "mcc": "7299"}, "emitted_at": 1697217911713} +{"stream": "locations", "data": {"id": "LGTMGD6Y4MH5R", "name": "test", "address": {"address_line_1": "1600 Pennsylvania Ave NW", "locality": "Washington", "administrative_district_level_1": "DC", "postal_code": "20500", "country": "US"}, "timezone": "UTC", "capabilities": ["CREDIT_CARD_PROCESSING", "AUTOMATIC_TRANSFERS"], "status": "ACTIVE", "created_at": "2023-01-04T15:20:43.345Z", "merchant_id": "ML7SCCR7EMAK4", "country": "US", "language_code": "en-US", "currency": "USD", "business_name": "Default Test Account", "type": "PHYSICAL", "business_hours": {}, "business_email": "test@test.com", "mcc": "7299"}, "emitted_at": 1697217911717} +{"stream": "items", "data": {"type": "ITEM", "id": "K7CL577FVTGEGWEHZUU3NA6P", "updated_at": "2023-02-28T17:03:10.233Z", "created_at": "2021-06-10T09:58:41.311Z", "version": 1677603790233, "is_deleted": false, "custom_attribute_values": {"Square:eca67bfb-68a4-4218-950a-09eec906143d": {"name": "With coffee cup", "string_value": "Yes", "custom_attribute_definition_id": "VR43EHV5M3Z4P4CWA7K3ZXPA", "type": "STRING", "key": "Square:eca67bfb-68a4-4218-950a-09eec906143d"}}, "present_at_all_locations": true, "item_data": {"name": "Coffee", "description": "Some coffee drink", "is_taxable": true, "visibility": "PRIVATE", "category_id": "WBVNPPUWI2YCVI2XJZNHPSQC", "modifier_list_info": [{"modifier_list_id": "ZYESF2MGAMVORYB66VVXFW6V", "visibility": "PUBLIC", "min_selected_modifiers": -1, "max_selected_modifiers": -1, "enabled": true}], "variations": [{"type": "ITEM_VARIATION", "id": "DT52FVGPUEJ7KL5WYPIK5TTP", "updated_at": "2021-06-10T09:58:41.311Z", "created_at": "2021-06-10T09:58:41.311Z", "version": 1623319121311, "is_deleted": false, "present_at_all_locations": true, "item_variation_data": {"item_id": "K7CL577FVTGEGWEHZUU3NA6P", "name": "Black", "sku": "1", "ordinal": 0, "pricing_type": "FIXED_PRICING", "price_money": {"amount": 900, "currency": "USD"}, "location_overrides": [{"location_id": "LH2XR7AMG39HX", "track_inventory": true}], "item_option_values": [{"item_option_id": "QTE3WP7JI64XLUD7AX5ER2ZI", "item_option_value_id": "KRNNA4P57TDNVBJLWIBW5D47"}], "sellable": true, "stockable": true}}, {"type": "ITEM_VARIATION", "id": "SZTS6NG7OGC25KGTRXJEUAKK", "updated_at": "2021-06-10T09:58:41.311Z", "created_at": "2021-06-10T09:58:41.311Z", "version": 1623319121311, "is_deleted": false, "present_at_all_locations": true, "item_variation_data": {"item_id": "K7CL577FVTGEGWEHZUU3NA6P", "name": "White", "sku": "2", "ordinal": 1, "pricing_type": "FIXED_PRICING", "price_money": {"amount": 1000, "currency": "USD"}, "location_overrides": [{"location_id": "LH2XR7AMG39HX", "track_inventory": true}], "item_option_values": [{"item_option_id": "QTE3WP7JI64XLUD7AX5ER2ZI", "item_option_value_id": "LIB3NSHGZUXFM3NFDHTUM4CJ"}], "sellable": true, "stockable": true}}], "product_type": "REGULAR", "skip_modifier_screen": false, "item_options": [{"item_option_id": "QTE3WP7JI64XLUD7AX5ER2ZI"}], "ecom_visibility": "UNINDEXED"}}, "emitted_at": 1697217792587} +{"stream": "items", "data": {"type": "ITEM", "id": "SY3I7GJTCYIJTOD6PKLO6VKI", "updated_at": "2023-02-28T17:03:10.233Z", "created_at": "2021-06-10T21:07:19.929Z", "version": 1677603790233, "is_deleted": false, "present_at_all_locations": true, "item_data": {"name": "Tea", "description": "Just the tea example", "is_taxable": true, "visibility": "PRIVATE", "category_id": "WBVNPPUWI2YCVI2XJZNHPSQC", "variations": [{"type": "ITEM_VARIATION", "id": "PGOQKJWR6ALTCFPVGV54LHA6", "updated_at": "2021-06-10T21:07:19.929Z", "created_at": "2021-06-10T21:07:19.929Z", "version": 1623359239929, "is_deleted": false, "present_at_all_locations": true, "item_variation_data": {"item_id": "SY3I7GJTCYIJTOD6PKLO6VKI", "name": "Regular", "sku": "3", "ordinal": 1, "pricing_type": "FIXED_PRICING", "price_money": {"amount": 500, "currency": "USD"}, "location_overrides": [{"location_id": "LH2XR7AMG39HX", "track_inventory": true}], "sellable": true, "stockable": true}}], "product_type": "REGULAR", "skip_modifier_screen": false, "ecom_visibility": "UNINDEXED"}}, "emitted_at": 1697217792588} +{"stream": "items", "data": {"type": "ITEM", "id": "UCYFEEPFPQFTWAGMH6T56L4U", "updated_at": "2023-02-28T17:03:10.233Z", "created_at": "2021-06-10T21:16:07.366Z", "version": 1677603790233, "is_deleted": false, "present_at_all_locations": true, "image_id": "7JPVLNMPVEBNXPQT5JXYCNF2", "item_data": {"name": "Beer", "description": "Unfiltered", "is_taxable": true, "visibility": "PRIVATE", "category_id": "WBVNPPUWI2YCVI2XJZNHPSQC", "variations": [{"type": "ITEM_VARIATION", "id": "AVZZR4PLYHND3GQU5KD25GYD", "updated_at": "2021-06-10T21:16:07.366Z", "created_at": "2021-06-10T21:16:07.366Z", "version": 1623359767366, "is_deleted": false, "present_at_all_locations": true, "item_variation_data": {"item_id": "UCYFEEPFPQFTWAGMH6T56L4U", "name": "Light", "sku": "4", "ordinal": 1, "pricing_type": "FIXED_PRICING", "price_money": {"amount": 1100, "currency": "USD"}, "location_overrides": [{"location_id": "LH2XR7AMG39HX", "track_inventory": true}], "sellable": true, "stockable": true}}, {"type": "ITEM_VARIATION", "id": "QR3GL6QNSGG7TPGX3W6F72BK", "updated_at": "2021-06-10T21:46:15.762Z", "created_at": "2021-06-10T21:46:15.762Z", "version": 1623361575762, "is_deleted": false, "present_at_all_locations": false, "present_at_location_ids": ["LH2XR7AMG39HX"], "item_variation_data": {"item_id": "UCYFEEPFPQFTWAGMH6T56L4U", "name": "Unfiltered", "sku": "6", "ordinal": 1, "pricing_type": "FIXED_PRICING", "price_money": {"amount": 1300, "currency": "USD"}, "location_overrides": [{"location_id": "LH2XR7AMG39HX", "track_inventory": true}], "sellable": true, "stockable": true}}, {"type": "ITEM_VARIATION", "id": "DJQ5F7NPJZCO7CMCPAU4GMTN", "updated_at": "2021-06-10T21:16:07.366Z", "created_at": "2021-06-10T21:16:07.366Z", "version": 1623359767366, "is_deleted": false, "present_at_all_locations": true, "item_variation_data": {"item_id": "UCYFEEPFPQFTWAGMH6T56L4U", "name": "Dark", "sku": "5", "ordinal": 2, "pricing_type": "FIXED_PRICING", "price_money": {"amount": 1200, "currency": "USD"}, "location_overrides": [{"location_id": "LH2XR7AMG39HX", "track_inventory": true}], "sellable": true, "stockable": true}}], "product_type": "REGULAR", "skip_modifier_screen": false, "ecom_visibility": "UNINDEXED"}}, "emitted_at": 1697217792590} +{"stream": "discounts", "data": {"type": "DISCOUNT", "id": "Q7AQZ6WPGAWPFAIYBTYT7XXP", "updated_at": "2021-06-14T13:47:48.539Z", "created_at": "2021-06-10T22:15:06.693Z", "version": 1623678468539, "is_deleted": false, "present_at_all_locations": false, "present_at_location_ids": ["L9A5Y0JR014G1"], "discount_data": {"name": "discount_20_p", "discount_type": "FIXED_PERCENTAGE", "percentage": "20.0", "application_method": "MANUALLY_APPLIED", "modify_tax_basis": "MODIFY_TAX_BASIS"}}, "emitted_at": 1697217897637} +{"stream": "discounts", "data": {"type": "DISCOUNT", "id": "HKYNSSNT2XWGYGPQNPVWFEAG", "updated_at": "2021-06-14T13:48:20.638Z", "created_at": "2021-06-10T22:15:45.239Z", "version": 1623678500638, "is_deleted": false, "present_at_all_locations": false, "present_at_location_ids": ["LH2XR7AMG39HX"], "discount_data": {"name": "discount_5_p", "discount_type": "FIXED_PERCENTAGE", "percentage": "5.0", "application_method": "MANUALLY_APPLIED", "modify_tax_basis": "MODIFY_TAX_BASIS"}}, "emitted_at": 1697217897646} +{"stream": "discounts", "data": {"type": "DISCOUNT", "id": "TN6YCTI5DDYJTDQUW3VQ733L", "updated_at": "2021-06-10T22:16:29.308Z", "created_at": "2021-06-10T22:16:29.308Z", "version": 1623363389308, "is_deleted": false, "present_at_all_locations": true, "discount_data": {"name": "discount_1_usd", "discount_type": "FIXED_AMOUNT", "amount_money": {"amount": 100, "currency": "USD"}, "application_method": "MANUALLY_APPLIED", "modify_tax_basis": "MODIFY_TAX_BASIS"}}, "emitted_at": 1697217897651} +{"stream": "modifier_list", "data": {"type": "MODIFIER_LIST", "id": "ZYESF2MGAMVORYB66VVXFW6V", "updated_at": "2021-06-10T22:17:15.317Z", "created_at": "2021-06-10T22:17:15.317Z", "version": 1623363435317, "is_deleted": false, "present_at_all_locations": true, "modifier_list_data": {"name": "With_accessory", "selection_type": "MULTIPLE", "modifiers": [{"type": "MODIFIER", "id": "EW5DQKRKJH5LF2O5OM3TLT32", "updated_at": "2021-06-10T22:17:15.317Z", "created_at": "2021-06-10T22:17:15.317Z", "version": 1623363435317, "is_deleted": false, "present_at_all_locations": true, "modifier_data": {"name": "1_accessory", "price_money": {"amount": 100, "currency": "USD"}, "on_by_default": false, "ordinal": 1, "modifier_list_id": "ZYESF2MGAMVORYB66VVXFW6V"}}, {"type": "MODIFIER", "id": "APBZ4WD5P3FPUYSLG4U7MEJF", "updated_at": "2021-06-10T22:17:15.317Z", "created_at": "2021-06-10T22:17:15.317Z", "version": 1623363435317, "is_deleted": false, "present_at_all_locations": true, "modifier_data": {"name": "2_accessory", "price_money": {"amount": 200, "currency": "USD"}, "on_by_default": false, "ordinal": 2, "modifier_list_id": "ZYESF2MGAMVORYB66VVXFW6V"}}]}}, "emitted_at": 1697217931779} +{"stream": "modifier_list", "data": {"type": "MODIFIER_LIST", "id": "MKW7LLF4IRUX773KBHH4XQZA", "updated_at": "2021-06-14T13:10:54.797Z", "created_at": "2021-06-14T13:10:54.797Z", "version": 1623676254797, "is_deleted": false, "present_at_all_locations": true, "modifier_list_data": {"name": "With_something_else", "selection_type": "MULTIPLE", "modifiers": [{"type": "MODIFIER", "id": "IA66H4C4C6JNXMHXQI3LDWFP", "updated_at": "2021-06-14T13:10:54.797Z", "created_at": "2021-06-14T13:10:54.797Z", "version": 1623676254797, "is_deleted": false, "present_at_all_locations": true, "modifier_data": {"name": "something_else", "price_money": {"amount": 1000, "currency": "USD"}, "on_by_default": false, "ordinal": 1, "modifier_list_id": "MKW7LLF4IRUX773KBHH4XQZA"}}, {"type": "MODIFIER", "id": "CS5VQADEB4GZXEL3TWHQDRER", "updated_at": "2021-06-14T13:10:54.797Z", "created_at": "2021-06-14T13:10:54.797Z", "version": 1623676254797, "is_deleted": false, "present_at_all_locations": true, "modifier_data": {"name": "something_else_2", "price_money": {"amount": 1500, "currency": "USD"}, "on_by_default": false, "ordinal": 2, "modifier_list_id": "MKW7LLF4IRUX773KBHH4XQZA"}}]}}, "emitted_at": 1697217931785} +{"stream": "inventory", "data": {"catalog_object_id": "ARZ6U6FLKCLA6EOIKBYK3DZ7", "catalog_object_type": "ITEM_VARIATION", "state": "IN_STOCK", "location_id": "LH2XR7AMG39HX", "quantity": "124", "calculated_at": "2023-01-06T21:08:23.095Z"}, "emitted_at": 1697217897015} +{"stream": "inventory", "data": {"catalog_object_id": "ARZ6U6FLKCLA6EOIKBYK3DZ7", "catalog_object_type": "ITEM_VARIATION", "state": "IN_STOCK", "location_id": "L9A5Y0JR014G1", "quantity": "124", "calculated_at": "2023-01-06T21:06:55.36Z"}, "emitted_at": 1697217897017} +{"stream": "inventory", "data": {"catalog_object_id": "YUDRKASZGJ3AFJGOQUMPP3EJ", "catalog_object_type": "ITEM_VARIATION", "state": "IN_STOCK", "location_id": "LH2XR7AMG39HX", "quantity": "500", "calculated_at": "2023-01-06T20:00:26.338Z"}, "emitted_at": 1697217897019} +{"stream": "orders", "data": {"id": "jqYrf6arFpUo7zElfWu9GRF5lAWZY", "location_id": "LH2XR7AMG39HX", "line_items": [{"uid": "JYEv3BLPY5FmSaVXDdbESD", "quantity": "1", "base_price_money": {"amount": 100, "currency": "USD"}, "gross_sales_money": {"amount": 100, "currency": "USD"}, "total_tax_money": {"amount": 0, "currency": "USD"}, "total_discount_money": {"amount": 0, "currency": "USD"}, "total_money": {"amount": 100, "currency": "USD"}, "variation_total_price_money": {"amount": 100, "currency": "USD"}, "item_type": "CUSTOM_AMOUNT"}], "created_at": "2021-06-08T20:21:39.163Z", "updated_at": "2021-06-08T20:21:41.000Z", "state": "COMPLETED", "version": 4, "total_tax_money": {"amount": 0, "currency": "USD"}, "total_discount_money": {"amount": 0, "currency": "USD"}, "total_tip_money": {"amount": 0, "currency": "USD"}, "total_money": {"amount": 100, "currency": "USD"}, "closed_at": "2021-06-08T20:21:39.406Z", "tenders": [{"id": "9m4YvEyzLRUvwUeBf2DNtVOh6cIZY", "location_id": "LH2XR7AMG39HX", "transaction_id": "jqYrf6arFpUo7zElfWu9GRF5lAWZY", "created_at": "2021-06-08T20:21:39Z", "amount_money": {"amount": 100, "currency": "USD"}, "type": "CARD", "card_details": {"status": "CAPTURED", "card": {"card_brand": "MASTERCARD", "last_4": "9029", "fingerprint": "sq-1-MTQOLCjEOIzHvJvKX4yxf6qBvj6DAFuB8wlWoKW4NI1BAFV5cdlJmge8ehPFGUSeuw"}, "entry_method": "KEYED"}, "payment_id": "9m4YvEyzLRUvwUeBf2DNtVOh6cIZY"}], "total_service_charge_money": {"amount": 0, "currency": "USD"}, "net_amounts": {"total_money": {"amount": 100, "currency": "USD"}, "tax_money": {"amount": 0, "currency": "USD"}, "discount_money": {"amount": 0, "currency": "USD"}, "tip_money": {"amount": 0, "currency": "USD"}, "service_charge_money": {"amount": 0, "currency": "USD"}}, "source": {"name": "Sandbox for sq0idp-7KVC6qHcSDMXsm40SAA9TA"}}, "emitted_at": 1697217939855} +{"stream": "orders", "data": {"id": "hD1xqUBBHQ3ejMBQiSSmncrYg7OZY", "location_id": "LH2XR7AMG39HX", "line_items": [{"uid": "v6KbyuUoPvjZ6hHLVLpvi", "quantity": "1", "base_price_money": {"amount": 2056, "currency": "USD"}, "note": "20$ money payment", "gross_sales_money": {"amount": 2056, "currency": "USD"}, "total_tax_money": {"amount": 0, "currency": "USD"}, "total_discount_money": {"amount": 0, "currency": "USD"}, "total_money": {"amount": 2056, "currency": "USD"}, "variation_total_price_money": {"amount": 2056, "currency": "USD"}, "item_type": "CUSTOM_AMOUNT"}], "created_at": "2021-06-18T13:30:27.796Z", "updated_at": "2021-06-18T13:30:30.000Z", "state": "COMPLETED", "version": 4, "total_tax_money": {"amount": 0, "currency": "USD"}, "total_discount_money": {"amount": 0, "currency": "USD"}, "total_tip_money": {"amount": 0, "currency": "USD"}, "total_money": {"amount": 2056, "currency": "USD"}, "closed_at": "2021-06-18T13:30:28.042Z", "tenders": [{"id": "rLBl9k8kKVV8uXNymUEct6S2ebIZY", "location_id": "LH2XR7AMG39HX", "transaction_id": "hD1xqUBBHQ3ejMBQiSSmncrYg7OZY", "created_at": "2021-06-18T13:30:27Z", "note": "20$ money payment", "amount_money": {"amount": 2056, "currency": "USD"}, "type": "CARD", "card_details": {"status": "CAPTURED", "card": {"card_brand": "VISA", "last_4": "1111", "fingerprint": "sq-1-mqW9yIk2eKV4LdXhGzf-FYu1knqb1IT7lXybOaFbMwIH2-9d1qdVOGNUMA8TDALoqg"}, "entry_method": "KEYED"}, "payment_id": "rLBl9k8kKVV8uXNymUEct6S2ebIZY"}], "total_service_charge_money": {"amount": 0, "currency": "USD"}, "net_amounts": {"total_money": {"amount": 2056, "currency": "USD"}, "tax_money": {"amount": 0, "currency": "USD"}, "discount_money": {"amount": 0, "currency": "USD"}, "tip_money": {"amount": 0, "currency": "USD"}, "service_charge_money": {"amount": 0, "currency": "USD"}}, "source": {"name": "Sandbox for sq0idp-4Uw2-7Sy15Umdnct7FTeuQ"}}, "emitted_at": 1697217939866} +{"stream": "orders", "data": {"id": "NpZRjYMGKOKeTe0BTp7N5r8kM0LZY", "location_id": "LH2XR7AMG39HX", "created_at": "2021-06-18T13:37:33.422Z", "updated_at": "2021-06-18T13:37:37.000Z", "state": "COMPLETED", "version": 4, "closed_at": "2021-06-18T13:37:34.544Z", "returns": [{"uid": "utVo8VtdQxlKZdVh1n6jaD", "source_order_id": "BxCc4Y2KBt10BUWQheazcgRUR7bZY", "return_line_items": [{"uid": "Ck0UkAjcVHe6guD1HNWub", "quantity": "1", "item_type": "CUSTOM_AMOUNT", "base_price_money": {"amount": 1485, "currency": "USD"}, "variation_total_price_money": {"amount": 1485, "currency": "USD"}, "gross_return_money": {"amount": 1485, "currency": "USD"}, "total_tax_money": {"amount": 0, "currency": "USD"}, "total_discount_money": {"amount": 0, "currency": "USD"}, "total_money": {"amount": 1485, "currency": "USD"}}]}], "return_amounts": {"total_money": {"amount": 1485, "currency": "USD"}, "tax_money": {"amount": 0, "currency": "USD"}, "discount_money": {"amount": 0, "currency": "USD"}, "tip_money": {"amount": 0, "currency": "USD"}, "service_charge_money": {"amount": 0, "currency": "USD"}}, "refunds": [{"id": "0um3GHK0AHt273xEe6I3u1y96Lnm018b0WAtyyOYRrP", "location_id": "LH2XR7AMG39HX", "transaction_id": "BxCc4Y2KBt10BUWQheazcgRUR7bZY", "tender_id": "NWO7kC96bJDUNKLovcUnapxGeOWZY", "created_at": "2021-06-18T13:37:33Z", "reason": "Broken item", "amount_money": {"amount": 1485, "currency": "USD"}, "status": "APPROVED"}], "source": {}}, "emitted_at": 1697217939875} +{"stream": "loyalty", "data": {"id": "ce21ddea-e73c-4d32-aa51-b50312ac7422", "program_id": "b7517cdb-8ab4-4d22-8b44-38bb2405087d", "balance": 0, "lifetime_points": 0, "customer_id": "WYP9CC9M156J71DMQF41Q8VMWW", "enrolled_at": "2023-10-16T17:31:24Z", "created_at": "2023-10-16T17:33:20Z", "updated_at": "2023-10-16T17:33:20Z", "mapping": {"id": "9909bd55-6e5a-4b43-b47d-8ac845ab3795", "created_at": "2023-10-16T17:33:20Z", "phone_number": "+15035551234"}}, "emitted_at": 1697478038492} diff --git a/airbyte-integrations/connectors/source-square/metadata.yaml b/airbyte-integrations/connectors/source-square/metadata.yaml index 23c1c36c7654..71b100e1f7c1 100644 --- a/airbyte-integrations/connectors/source-square/metadata.yaml +++ b/airbyte-integrations/connectors/source-square/metadata.yaml @@ -1,4 +1,7 @@ data: + ab_internal: + ql: 300 + sl: 100 allowedHosts: hosts: - connect.squareupsandbox.com @@ -6,8 +9,9 @@ data: connectorSubtype: api connectorType: source definitionId: 77225a51-cd15-4a13-af02-65816bd0ecf4 - dockerImageTag: 1.1.2 + dockerImageTag: 1.6.1 dockerRepository: airbyte/source-square + documentationUrl: https://docs.airbyte.com/integrations/sources/square githubIssueLabel: source-square icon: square.svg license: MIT @@ -18,12 +22,8 @@ data: oss: enabled: true releaseStage: beta - documentationUrl: https://docs.airbyte.com/integrations/sources/square + supportLevel: community tags: - language:low-code - language:python - ab_internal: - sl: 200 - ql: 300 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-square/setup.py b/airbyte-integrations/connectors/source-square/setup.py index 051433b5675c..fe3012decd5d 100644 --- a/airbyte-integrations/connectors/source-square/setup.py +++ b/airbyte-integrations/connectors/source-square/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk>=0.44.2", + "airbyte-cdk>=0.51.31", ] TEST_REQUIREMENTS = [ diff --git a/airbyte-integrations/connectors/source-square/source_square/manifest.yaml b/airbyte-integrations/connectors/source-square/source_square/manifest.yaml index 255f2a403888..c3680d620ece 100644 --- a/airbyte-integrations/connectors/source-square/source_square/manifest.yaml +++ b/airbyte-integrations/connectors/source-square/source_square/manifest.yaml @@ -74,6 +74,9 @@ definitions: start_time_option: field_name: begin_time inject_into: body_json + end_time_option: + field_name: end_time + inject_into: body_json retriever: $ref: "#/definitions/retriever" requester: @@ -115,6 +118,9 @@ definitions: start_time_option: field_name: begin_time inject_into: body_json + end_time_option: + field_name: end_time + inject_into: body_json retriever: $ref: "#/definitions/retriever" record_selector: @@ -184,6 +190,24 @@ definitions: primary_key: "id" path: "/team-members/search" + inventory_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "inventory" + primary_key: "catalog_object_id" + path: "/inventory/counts/batch-retrieve" + retriever: + $ref: "#/definitions/base_stream_page_json_limit/retriever" + requester: + $ref: "#/definitions/base_stream_page_json_limit/retriever/requester" + http_method: "POST" + request_body_json: + limit: "{{ 500 }}" + record_selector: + $ref: "#/definitions/selector" + extractor: + field_path: ["counts"] + team_member_wages_stream: $ref: "#/definitions/base_stream" $parameters: @@ -207,6 +231,32 @@ definitions: primary_key: "id" path: "/locations" + locations_partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/locations_stream" + parent_key: "id" + partition_field: "location_id" + + cash_drawers_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "cash_drawers" + primary_key: "id" + path: "/cash-drawers/shifts" + retriever: + $ref: "#/definitions/retriever" + partition_router: + $ref: "#/definitions/locations_partition_router" + requester: + $ref: "#/definitions/requester" + request_parameters: + location_id: "{{ stream_partition.location_id }}" + record_selector: + $ref: "#/definitions/selector" + extractor: + field_path: ["cash_drawer_shifts"] + categories_stream: $ref: "#/definitions/base_catalog_objects_stream" $parameters: @@ -260,6 +310,24 @@ definitions: retriever: $ref: "#/definitions/base_incremental_stream/retriever" + loyalty_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "loyalty" + primary_key: "id" + path: "/loyalty/accounts/search" + retriever: + $ref: "#/definitions/base_stream_page_json_limit/retriever" + requester: + $ref: "#/definitions/base_stream_page_json_limit/retriever/requester" + http_method: "POST" + request_body_json: + limit: "{{ 200 }}" + record_selector: + $ref: "#/definitions/selector" + extractor: + field_path: ["loyalty_accounts"] + orders_stream: $ref: "#/definitions/base_stream_page_json_limit" $parameters: @@ -289,6 +357,13 @@ definitions: request_body_json: limit: "{{ 500 }}" + bank_accounts_stream: + $ref: "#/definitions/base_stream" + $parameters: + name: "bank_accounts" + primary_key: "id" + path: "/bank-accounts" + streams: - "#/definitions/customers_stream" - "#/definitions/locations_stream" @@ -301,8 +376,12 @@ streams: - "#/definitions/taxes_stream" - "#/definitions/modifier_list_stream" - "#/definitions/refunds_stream" + - "#/definitions/inventory_stream" - "#/definitions/payments_stream" - "#/definitions/orders_stream" + - "#/definitions/bank_accounts_stream" + - "#/definitions/cash_drawers_stream" + - "#/definitions/loyalty_stream" check: stream_names: diff --git a/airbyte-integrations/connectors/source-square/source_square/schemas/bank_accounts.json b/airbyte-integrations/connectors/source-square/source_square/schemas/bank_accounts.json new file mode 100644 index 000000000000..a8e1d1c8006e --- /dev/null +++ b/airbyte-integrations/connectors/source-square/source_square/schemas/bank_accounts.json @@ -0,0 +1,46 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "string"] + }, + "account_number_suffix": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "account_type": { + "type": ["null", "string"] + }, + "holder_name": { + "type": ["null", "string"] + }, + "primary_bank_identification_number": { + "type": ["null", "string"] + }, + "location_id": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "creditable": { + "type": ["null", "boolean"] + }, + "debitable": { + "type": ["null", "boolean"] + }, + "version": { + "type": ["null", "integer"] + }, + "bank_name": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-square/source_square/schemas/cash_drawers.json b/airbyte-integrations/connectors/source-square/source_square/schemas/cash_drawers.json new file mode 100644 index 000000000000..2881cd872572 --- /dev/null +++ b/airbyte-integrations/connectors/source-square/source_square/schemas/cash_drawers.json @@ -0,0 +1,59 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Cash Drawer Shifts Schema", + "additionalProperties": true, + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string", "null"] + }, + "state": { + "type": ["string", "null"] + }, + "opened_at": { + "type": ["string", "null"] + }, + "ended_at": { + "type": ["string", "null"] + }, + "closed_at": { + "type": ["string", "null"] + }, + "description": { + "type": ["string", "null"] + }, + "opened_cash_money": { + "type": ["object", "null"], + "properties": { + "amount": { + "type": ["number", "null"] + }, + "currency": { + "type": ["string", "null"] + } + } + }, + "expected_cash_money": { + "type": ["object", "null"], + "properties": { + "amount": { + "type": ["number", "null"] + }, + "currency": { + "type": ["string", "null"] + } + } + }, + "closed_cash_money": { + "type": ["object", "null"], + "properties": { + "amount": { + "type": ["number", "null"] + }, + "currency": { + "type": ["string", "null"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-square/source_square/schemas/inventory.json b/airbyte-integrations/connectors/source-square/source_square/schemas/inventory.json new file mode 100644 index 000000000000..275a6eace961 --- /dev/null +++ b/airbyte-integrations/connectors/source-square/source_square/schemas/inventory.json @@ -0,0 +1,26 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Inventory Schema", + "additionalProperties": true, + "type": ["object", "null"], + "properties": { + "catalog_object_id": { + "type": ["string", "null"] + }, + "catalog_object_type": { + "type": ["string", "null"] + }, + "state": { + "type": ["string", "null"] + }, + "location_id": { + "type": ["string", "null"] + }, + "quantity": { + "type": ["string", "null"] + }, + "calculated_at": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-square/source_square/schemas/loyalty.json b/airbyte-integrations/connectors/source-square/source_square/schemas/loyalty.json new file mode 100644 index 000000000000..d2c748842236 --- /dev/null +++ b/airbyte-integrations/connectors/source-square/source_square/schemas/loyalty.json @@ -0,0 +1,43 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Loyalty schema", + "type": ["object", "null"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["string", "null"] + }, + "mapping": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string", "null"] + }, + "phone_number": { + "type": ["string", "null"] + }, + "created_at": { + "type": ["string", "null"] + } + } + }, + "program_id": { + "type": ["string", "null"] + }, + "balance": { + "type": ["number", "null"] + }, + "lifetime_points": { + "type": ["number", "null"] + }, + "customer_id": { + "type": ["string", "null"] + }, + "created_at": { + "type": ["string", "null"] + }, + "updated_at": { + "type": ["string", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-square/source_square/schemas/orders.json b/airbyte-integrations/connectors/source-square/source_square/schemas/orders.json index 952d03495d6f..2cb7b1c6d895 100644 --- a/airbyte-integrations/connectors/source-square/source_square/schemas/orders.json +++ b/airbyte-integrations/connectors/source-square/source_square/schemas/orders.json @@ -7,6 +7,9 @@ "location_id": { "type": ["null", "string"] }, + "customer_id": { + "type": ["null", "string"] + }, "line_items": { "type": ["null", "array"], "items": { diff --git a/airbyte-integrations/connectors/source-square/source_square/schemas/payments.json b/airbyte-integrations/connectors/source-square/source_square/schemas/payments.json index bb414b66076d..ea3686e855e1 100644 --- a/airbyte-integrations/connectors/source-square/source_square/schemas/payments.json +++ b/airbyte-integrations/connectors/source-square/source_square/schemas/payments.json @@ -21,6 +21,9 @@ } } }, + "customer_id": { + "type": ["null", "string"] + }, "status": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-square/unit_tests/test_component.py b/airbyte-integrations/connectors/source-square/unit_tests/test_component.py index d991026313d5..c749f1e10e9f 100644 --- a/airbyte-integrations/connectors/source-square/unit_tests/test_component.py +++ b/airbyte-integrations/connectors/source-square/unit_tests/test_component.py @@ -57,6 +57,7 @@ def test_refresh_access_token(req_mock): client_id="client_id", refresh_token="refresh_token", token_expiry_date_format="YYYY-MM-DDTHH:mm:ss[Z]", + token_expiry_is_time_of_expiration=True, config=config, parameters=parameters, ) diff --git a/airbyte-integrations/connectors/source-statuspage/README.md b/airbyte-integrations/connectors/source-statuspage/README.md index 53b49bb1a1ac..7a25aad43412 100644 --- a/airbyte-integrations/connectors/source-statuspage/README.md +++ b/airbyte-integrations/connectors/source-statuspage/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-statuspage:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/statuspage) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_statuspage/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-statuspage:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-statuspage build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-statuspage:airbyteDocker +An image will be built with the tag `airbyte/source-statuspage:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-statuspage:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-statuspage:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-statuspage:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-statuspage:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-statuspage test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-statuspage:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-statuspage:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-statuspage test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/statuspage.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-statuspage/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-statuspage/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-statuspage/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-statuspage/build.gradle b/airbyte-integrations/connectors/source-statuspage/build.gradle deleted file mode 100644 index 8ffd90d6c0a1..000000000000 --- a/airbyte-integrations/connectors/source-statuspage/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_statuspage' -} diff --git a/airbyte-integrations/connectors/source-stock-ticker-api-tutorial/build.gradle b/airbyte-integrations/connectors/source-stock-ticker-api-tutorial/build.gradle deleted file mode 100644 index 27dd58f4d228..000000000000 --- a/airbyte-integrations/connectors/source-stock-ticker-api-tutorial/build.gradle +++ /dev/null @@ -1,17 +0,0 @@ -plugins { - // Makes building the docker image a dependency of Gradle's "build" command. This way you could run your entire build inside a docker image - // via ./gradlew :airbyte-integrations:connectors:source-stock-ticker-api:build - id 'airbyte-docker' - id 'airbyte-standard-source-test-file' -} - -airbyteStandardSourceTestFile { - // All these input paths must live inside this connector's directory (or subdirectories) - configPath = "secrets/valid_config.json" - configuredCatalogPath = "fullrefresh_configured_catalog.json" - specPath = "spec.json" -} - -dependencies { - implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) -} diff --git a/airbyte-integrations/connectors/source-strava/.dockerignore b/airbyte-integrations/connectors/source-strava/.dockerignore index a36e07dfe555..c934d6fb0e6e 100644 --- a/airbyte-integrations/connectors/source-strava/.dockerignore +++ b/airbyte-integrations/connectors/source-strava/.dockerignore @@ -1,6 +1,5 @@ * !Dockerfile -!Dockerfile.test !main.py !source_strava !setup.py diff --git a/airbyte-integrations/connectors/source-strava/Dockerfile b/airbyte-integrations/connectors/source-strava/Dockerfile index 9bedb7afde30..e46b3c863344 100644 --- a/airbyte-integrations/connectors/source-strava/Dockerfile +++ b/airbyte-integrations/connectors/source-strava/Dockerfile @@ -34,5 +34,5 @@ COPY source_strava ./source_strava ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-strava diff --git a/airbyte-integrations/connectors/source-strava/README.md b/airbyte-integrations/connectors/source-strava/README.md index 12ceab1d0a90..5e5bee698092 100644 --- a/airbyte-integrations/connectors/source-strava/README.md +++ b/airbyte-integrations/connectors/source-strava/README.md @@ -1,74 +1,34 @@ # Strava Source -This is the repository for the Strava source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/strava). +This is the repository for the Strava configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/strava). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-strava:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/strava) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_strava/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/strava) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_strava/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/invalid_config.json` for a sample config file with fake tokens. +See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source strava test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-strava:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-strava build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-strava:airbyteDocker +An image will be built with the tag `airbyte/source-strava:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-strava:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-strava:dev check --con docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-strava:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-strava:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-strava test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-strava:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-strava:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-strava test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/strava.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-strava/__init__.py b/airbyte-integrations/connectors/source-strava/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-strava/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-strava/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-strava/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-strava/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-strava/build.gradle b/airbyte-integrations/connectors/source-strava/build.gradle deleted file mode 100644 index b4e1fb7be55f..000000000000 --- a/airbyte-integrations/connectors/source-strava/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_strava' -} diff --git a/airbyte-integrations/connectors/source-strava/integration_tests/__init__.py b/airbyte-integrations/connectors/source-strava/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-strava/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-strava/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-strava/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-strava/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-strava/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-strava/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-strava/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-strava/integration_tests/expected_records.jsonl index bdb4c114a195..e296f65990a4 100644 --- a/airbyte-integrations/connectors/source-strava/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-strava/integration_tests/expected_records.jsonl @@ -1,4 +1,3 @@ -{"stream": "athlete_stats", "data": {"biggest_ride_distance": 37825.1, "biggest_climb_elevation_gain": 102.30000000000001, "recent_ride_totals": {"count": 0, "distance": 0.0, "moving_time": 0, "elapsed_time": 0, "elevation_gain": 0.0, "achievement_count": 0}, "all_ride_totals": {"count": 46, "distance": 794446, "moving_time": 178718, "elapsed_time": 267074, "elevation_gain": 8366}, "recent_run_totals": {"count": 0, "distance": 0.0, "moving_time": 0, "elapsed_time": 0, "elevation_gain": 0.0, "achievement_count": 0}, "all_run_totals": {"count": 124, "distance": 841523, "moving_time": 302192, "elapsed_time": 342187, "elevation_gain": 9425}, "recent_swim_totals": {"count": 0, "distance": 0.0, "moving_time": 0, "elapsed_time": 0, "elevation_gain": 0.0, "achievement_count": 0}, "all_swim_totals": {"count": 0, "distance": 0, "moving_time": 0, "elapsed_time": 0, "elevation_gain": 0}, "ytd_ride_totals": {"count": 0, "distance": 0, "moving_time": 0, "elapsed_time": 0, "elevation_gain": 0}, "ytd_run_totals": {"count": 6, "distance": 29359, "moving_time": 13577, "elapsed_time": 14526, "elevation_gain": 379}, "ytd_swim_totals": {"count": 0, "distance": 0, "moving_time": 0, "elapsed_time": 0, "elevation_gain": 0}}, "emitted_at": 1681112894112} -{"stream": "activities", "data": {"resource_state": 2, "athlete": {"id": 17831421, "resource_state": 1}, "name": "Afternoon Run", "distance": 5180.3, "moving_time": 1558, "elapsed_time": 1570, "total_elevation_gain": 41.7, "type": "Run", "sport_type": "Run", "workout_type": 0, "id": 732659975, "start_date": "2016-10-03T00:23:29Z", "start_date_local": "2016-10-02T17:23:29Z", "timezone": "(GMT-08:00) America/Los_Angeles", "utc_offset": -25200.0, "location_city": null, "location_state": null, "location_country": null, "achievement_count": 3, "kudos_count": 2, "comment_count": 1, "athlete_count": 1, "photo_count": 0, "map": {"id": "a732659975", "summary_polyline": "ijdfFnfhiVu@uCmaAnEwAlByAkDiLnBvI~EjDyB`BhBjw@oEnBvRpOA|ApWqAfJvAeH}AiZiAyKeCQaAgG", "resource_state": 2}, "trainer": false, "commute": false, "manual": false, "private": false, "visibility": "everyone", "flagged": false, "gear_id": null, "start_latlng": [37.874456, -122.271915], "end_latlng": [37.874412, -122.272033], "average_speed": 3.325, "max_speed": 5.3, "has_heartrate": false, "heartrate_opt_out": false, "display_hide_heartrate_option": false, "elev_high": 81.9, "elev_low": 37.5, "upload_id": 807763220, "upload_id_str": "807763220", "external_id": "51fbc33d23a743089b1e732118e37bc5", "from_accepted_tag": false, "pr_count": 1, "total_photo_count": 0, "has_kudoed": false}, "emitted_at": 1684220833770} -{"stream": "activities", "data": {"resource_state": 2, "athlete": {"id": 17831421, "resource_state": 1}, "name": "New shorts, who dis", "distance": 8374.9, "moving_time": 2605, "elapsed_time": 2659, "total_elevation_gain": 75.0, "type": "Run", "sport_type": "Run", "workout_type": 0, "id": 735803811, "start_date": "2016-10-06T01:24:28Z", "start_date_local": "2016-10-05T18:24:28Z", "timezone": "(GMT-08:00) America/Los_Angeles", "utc_offset": -25200.0, "location_city": null, "location_state": null, "location_country": null, "achievement_count": 3, "kudos_count": 1, "comment_count": 1, "athlete_count": 1, "photo_count": 0, "map": {"id": "a735803811", "summary_polyline": "kjdfFlfhiVc@eD}aAvEsAxByAoDaLlBdIhF`IiGpi@sDeAeZxY_DAuKbIqB@{F~B_F~HaI|Hw@fAnKoJ`AeApCJ`S}SbBXjVbGr{@uA`JtA}G_E_o@wD_B", "resource_state": 2}, "trainer": false, "commute": false, "manual": false, "private": false, "visibility": "everyone", "flagged": false, "gear_id": null, "start_latlng": [37.874465, -122.271906], "end_latlng": [37.874449, -122.271743], "average_speed": 3.215, "max_speed": 5.4, "has_heartrate": false, "heartrate_opt_out": false, "display_hide_heartrate_option": false, "elev_high": 84.3, "elev_low": 37.5, "upload_id": 811413504, "upload_id_str": "811413504", "external_id": "f77a35ec330412183402e6fac2b5be20", "from_accepted_tag": false, "pr_count": 0, "total_photo_count": 0, "has_kudoed": false}, "emitted_at": 1684220833771} -{"stream": "activities", "data": {"resource_state": 2, "athlete": {"id": 17831421, "resource_state": 1}, "name": "Afternoon Run", "distance": 5457.4, "moving_time": 1598, "elapsed_time": 1659, "total_elevation_gain": 12.8, "type": "Run", "sport_type": "Run", "workout_type": 0, "id": 744774691, "start_date": "2016-10-15T00:51:50Z", "start_date_local": "2016-10-14T17:51:50Z", "timezone": "(GMT-08:00) America/Los_Angeles", "utc_offset": -25200.0, "location_city": null, "location_state": null, "location_country": null, "achievement_count": 5, "kudos_count": 0, "comment_count": 0, "athlete_count": 1, "photo_count": 0, "map": {"id": "a744774691", "summary_polyline": "{z_oEngunUx@pJfeCV}}@mBudAPy@q@GwFiG_@tDV", "resource_state": 2}, "trainer": false, "commute": false, "manual": false, "private": false, "visibility": "everyone", "flagged": false, "gear_id": null, "start_latlng": [34.083181, -117.914954], "end_latlng": [34.083389, -117.914936], "average_speed": 3.415, "max_speed": 4.7, "has_heartrate": false, "heartrate_opt_out": false, "display_hide_heartrate_option": false, "elev_high": 140.8, "elev_low": 127.8, "upload_id": 821881164, "upload_id_str": "821881164", "external_id": "4f6161a407a088bc4db49f5959e9673a", "from_accepted_tag": false, "pr_count": 2, "total_photo_count": 0, "has_kudoed": false}, "emitted_at": 1684220833771} +{"stream": "athlete_stats", "data": {"biggest_ride_distance": null, "biggest_climb_elevation_gain": null, "recent_ride_totals": {"count": 1, "distance": 0.0, "moving_time": 2300, "elapsed_time": 2300, "elevation_gain": 0.0, "achievement_count": 0}, "all_ride_totals": {"count": 1, "distance": 0, "moving_time": 2300, "elapsed_time": 2300, "elevation_gain": 0}, "recent_run_totals": {"count": 0, "distance": 0.0, "moving_time": 0, "elapsed_time": 0, "elevation_gain": 0.0, "achievement_count": 0}, "all_run_totals": {"count": 1, "distance": 0, "moving_time": 3600, "elapsed_time": 3600, "elevation_gain": 0}, "recent_swim_totals": {"count": 0, "distance": 0.0, "moving_time": 0, "elapsed_time": 0, "elevation_gain": 0.0, "achievement_count": 0}, "all_swim_totals": {"count": 0, "distance": 0, "moving_time": 0, "elapsed_time": 0, "elevation_gain": 0}, "ytd_ride_totals": {"count": 1, "distance": 0, "moving_time": 2300, "elapsed_time": 2300, "elevation_gain": 0}, "ytd_run_totals": {"count": 1, "distance": 0, "moving_time": 3600, "elapsed_time": 3600, "elevation_gain": 0}, "ytd_swim_totals": {"count": 0, "distance": 0, "moving_time": 0, "elapsed_time": 0, "elevation_gain": 0}}, "emitted_at": 1698179192900} +{"stream": "activities", "data": {"resource_state": 2, "athlete": {"id": 95370757, "resource_state": 1}, "name": "My example activity", "distance": 0.0, "moving_time": 3600, "elapsed_time": 3600, "total_elevation_gain": 0, "type": "Run", "sport_type": "Run", "workout_type": null, "id": 10097971825, "start_date": "2023-01-01T06:00:00Z", "start_date_local": "2023-01-01T00:00:00Z", "timezone": "(GMT-06:00) America/Chicago", "utc_offset": -21600.0, "location_city": null, "location_state": null, "location_country": "United States", "achievement_count": 0, "kudos_count": 0, "comment_count": 0, "athlete_count": 1, "photo_count": 0, "map": {"id": "a10097971825", "summary_polyline": "", "resource_state": 2}, "trainer": false, "commute": false, "manual": true, "private": false, "visibility": "everyone", "flagged": false, "gear_id": null, "start_latlng": [], "end_latlng": [], "average_speed": 0.0, "max_speed": 0, "has_heartrate": false, "heartrate_opt_out": false, "display_hide_heartrate_option": false, "upload_id": null, "external_id": null, "from_accepted_tag": false, "pr_count": 0, "total_photo_count": 0, "has_kudoed": false}, "emitted_at": 1698179195177} +{"stream": "activities", "data": {"resource_state": 2, "athlete": {"id": 95370757, "resource_state": 1}, "name": "Ride example", "distance": 0.0, "moving_time": 2300, "elapsed_time": 2300, "total_elevation_gain": 0, "type": "Ride", "sport_type": "Ride", "workout_type": null, "id": 10097974186, "start_date": "2023-10-01T05:00:00Z", "start_date_local": "2023-10-01T00:00:00Z", "timezone": "(GMT-06:00) America/Chicago", "utc_offset": -18000.0, "location_city": null, "location_state": null, "location_country": "United States", "achievement_count": 0, "kudos_count": 0, "comment_count": 0, "athlete_count": 1, "photo_count": 0, "map": {"id": "a10097974186", "summary_polyline": "", "resource_state": 2}, "trainer": false, "commute": false, "manual": true, "private": false, "visibility": "everyone", "flagged": false, "gear_id": null, "start_latlng": [], "end_latlng": [], "average_speed": 0.0, "max_speed": 0, "has_heartrate": false, "heartrate_opt_out": false, "display_hide_heartrate_option": false, "upload_id": null, "external_id": null, "from_accepted_tag": false, "pr_count": 0, "total_photo_count": 0, "has_kudoed": false}, "emitted_at": 1698179195185} diff --git a/airbyte-integrations/connectors/source-strava/metadata.yaml b/airbyte-integrations/connectors/source-strava/metadata.yaml index 99c0c5bd2ef0..0b8ffd4826b4 100644 --- a/airbyte-integrations/connectors/source-strava/metadata.yaml +++ b/airbyte-integrations/connectors/source-strava/metadata.yaml @@ -2,26 +2,27 @@ data: allowedHosts: hosts: - strava.com + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: 7a4327c4-315a-11ec-8d3d-0242ac130003 - dockerImageTag: 0.1.4 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-strava githubIssueLabel: source-strava icon: strava.svg license: MIT name: Strava - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: 2021-10-18 releaseStage: beta + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/strava tags: - - language:python + - language:low-code ab_internal: sl: 100 ql: 300 - supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-strava/setup.py b/airbyte-integrations/connectors/source-strava/setup.py index 1afea1f455f7..8dda89709cd4 100644 --- a/airbyte-integrations/connectors/source-strava/setup.py +++ b/airbyte-integrations/connectors/source-strava/setup.py @@ -5,13 +5,11 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = [ - "airbyte-cdk", -] +MAIN_REQUIREMENTS = ["airbyte-cdk"] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", - "pytest~=6.1", + "pytest~=6.2", "pytest-mock~=3.6.1", ] @@ -22,7 +20,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-strava/source_strava/__init__.py b/airbyte-integrations/connectors/source-strava/source_strava/__init__.py index 06b41c9c0e64..3a22985bf30c 100644 --- a/airbyte-integrations/connectors/source-strava/source_strava/__init__.py +++ b/airbyte-integrations/connectors/source-strava/source_strava/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-strava/source_strava/manifest.yaml b/airbyte-integrations/connectors/source-strava/source_strava/manifest.yaml new file mode 100644 index 000000000000..41c9f03b3e7a --- /dev/null +++ b/airbyte-integrations/connectors/source-strava/source_strava/manifest.yaml @@ -0,0 +1,189 @@ +version: "0.29.0" + +type: DeclarativeSource +check: + type: CheckStream + stream_names: + - athlete_stats +streams: + - type: DeclarativeStream + name: athlete_stats + primary_key: [] + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://www.strava.com/api/v3/ + path: athletes/{{ config.athlete_id }}/stats + http_method: GET + authenticator: + type: OAuthAuthenticator + client_id: "{{ config['client_id'] }}" + client_secret: "{{ config['client_secret'] }}" + refresh_token: "{{ config['refresh_token'] }}" + token_refresh_endpoint: https://www.strava.com/oauth/token + expires_in_name: expires_at + scopes: + - read_all + - activity:read_all + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: NoPagination + - type: DeclarativeStream + name: activities + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://www.strava.com/api/v3/ + path: athlete/activities + http_method: GET + request_parameters: {} + request_headers: {} + authenticator: + type: OAuthAuthenticator + client_id: "{{ config['client_id'] }}" + client_secret: "{{ config['client_secret'] }}" + refresh_token: "{{ config['refresh_token'] }}" + token_refresh_endpoint: https://www.strava.com/oauth/token + expires_in_name: expires_at + scopes: + - read_all + - activity:read_all + request_body_json: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + inject_into: request_parameter + field_name: per_page + type: RequestOption + pagination_strategy: + type: PageIncrement + start_from_page: 1 + page_size: 30 + incremental_sync: + type: DatetimeBasedCursor + cursor_field: start_date + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%SZ" + datetime_format: "%s" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%S" + start_time_option: + inject_into: request_parameter + field_name: after + type: RequestOption +spec: + documentation_url: https://docs.airbyte.com/integrations/sources/strava + connection_specification: + "$schema": https://json-schema.org/draft-07/schema# + title: Strava Spec + type: object + required: + - client_id + - client_secret + - refresh_token + - athlete_id + - start_date + additionalProperties: true + properties: + auth_type: + type: string + const: Client + enum: + - Client + default: Client + client_id: + type: string + description: The Client ID of your Strava developer application. + title: Client ID + pattern: "^[0-9_\\-]+$" + examples: + - "12345" + order: 0 + client_secret: + type: string + description: The Client Secret of your Strava developer application. + title: Client Secret + pattern: "^[0-9a-fA-F]+$" + examples: + - fc6243f283e51f6ca989aab298b17da125496f50 + airbyte_secret: true + order: 1 + refresh_token: + type: string + description: "The Refresh Token with the activity: read_all permissions." + title: Refresh Token + pattern: "^[0-9a-fA-F]+$" + examples: + - fc6243f283e51f6ca989aab298b17da125496f50 + airbyte_secret: true + order: 2 + athlete_id: + type: integer + description: The Athlete ID of your Strava developer application. + title: Athlete ID + pattern: "^[0-9_\\-]+$" + examples: + - "17831421" + order: 3 + start_date: + type: string + description: UTC date and time. Any data before this date will not be replicated. + title: Start Date + examples: + - "2021-03-01T00:00:00Z" + format: date-time + order: 4 + advanced_auth: + auth_flow_type: oauth2.0 + predicate_key: + - auth_type + predicate_value: Client + oauth_config_specification: + complete_oauth_output_specification: + type: object + additionalProperties: true + properties: + refresh_token: + type: string + path_in_connector_config: + - refresh_token + complete_oauth_server_input_specification: + type: object + additionalProperties: true + properties: + client_id: + type: string + client_secret: + type: string + complete_oauth_server_output_specification: + type: object + additionalProperties: false + properties: + client_id: + type: string + path_in_connector_config: + - client_id + client_secret: + type: string + path_in_connector_config: + - client_secret diff --git a/airbyte-integrations/connectors/source-strava/source_strava/schemas/athlete_stats.json b/airbyte-integrations/connectors/source-strava/source_strava/schemas/athlete_stats.json index 9abb1bdb3dd2..1eca03cafddf 100644 --- a/airbyte-integrations/connectors/source-strava/source_strava/schemas/athlete_stats.json +++ b/airbyte-integrations/connectors/source-strava/source_strava/schemas/athlete_stats.json @@ -1,227 +1,227 @@ { "$schema": "https://json-schema.org/draft-07/schema#", - "type": "object", + "type": ["null", "object"], "additionalProperties": true, "properties": { "biggest_ride_distance": { - "type": "number" + "type": ["null", "number"] }, "biggest_climb_elevation_gain": { - "type": "number" + "type": ["null", "number"] }, "recent_ride_totals": { - "type": "object", + "type": ["null", "object"], "additionalProperties": true, "properties": { "count": { - "type": "integer" + "type": ["null", "integer"] }, "distance": { - "type": "number" + "type": ["null", "number"] }, "moving_time": { - "type": "integer" + "type": ["null", "integer"] }, "elapsed_time": { - "type": "integer" + "type": ["null", "integer"] }, "elevation_gain": { - "type": "number" + "type": ["null", "number"] }, "achievement_count": { - "type": "integer" + "type": ["null", "integer"] } } }, "recent_run_totals": { - "type": "object", + "type": ["null", "object"], "additionalProperties": true, "properties": { "count": { - "type": "integer" + "type": ["null", "integer"] }, "distance": { - "type": "number" + "type": ["null", "number"] }, "moving_time": { - "type": "integer" + "type": ["null", "integer"] }, "elapsed_time": { - "type": "integer" + "type": ["null", "integer"] }, "elevation_gain": { - "type": "number" + "type": ["null", "number"] }, "achievement_count": { - "type": "integer" + "type": ["null", "integer"] } } }, "recent_swim_totals": { - "type": "object", + "type": ["null", "object"], "additionalProperties": true, "properties": { "count": { - "type": "integer" + "type": ["null", "integer"] }, "distance": { - "type": "number" + "type": ["null", "number"] }, "moving_time": { - "type": "integer" + "type": ["null", "integer"] }, "elapsed_time": { - "type": "integer" + "type": ["null", "integer"] }, "elevation_gain": { - "type": "number" + "type": ["null", "number"] }, "achievement_count": { - "type": "integer" + "type": ["null", "integer"] } } }, "ytd_ride_totals": { - "type": "object", + "type": ["null", "object"], "additionalProperties": true, "properties": { "count": { - "type": "integer" + "type": ["null", "integer"] }, "distance": { - "type": "number" + "type": ["null", "number"] }, "moving_time": { - "type": "integer" + "type": ["null", "integer"] }, "elapsed_time": { - "type": "integer" + "type": ["null", "integer"] }, "elevation_gain": { - "type": "number" + "type": ["null", "number"] }, "achievement_count": { - "type": "integer" + "type": ["null", "integer"] } } }, "ytd_run_totals": { - "type": "object", + "type": ["null", "object"], "additionalProperties": true, "properties": { "count": { - "type": "integer" + "type": ["null", "integer"] }, "distance": { - "type": "number" + "type": ["null", "number"] }, "moving_time": { - "type": "integer" + "type": ["null", "integer"] }, "elapsed_time": { - "type": "integer" + "type": ["null", "integer"] }, "elevation_gain": { - "type": "number" + "type": ["null", "number"] }, "achievement_count": { - "type": "integer" + "type": ["null", "integer"] } } }, "ytd_swim_totals": { - "type": "object", + "type": ["null", "object"], "additionalProperties": true, "properties": { "count": { - "type": "integer" + "type": ["null", "integer"] }, "distance": { - "type": "number" + "type": ["null", "number"] }, "moving_time": { - "type": "integer" + "type": ["null", "integer"] }, "elapsed_time": { - "type": "integer" + "type": ["null", "integer"] }, "elevation_gain": { - "type": "number" + "type": ["null", "number"] }, "achievement_count": { - "type": "integer" + "type": ["null", "integer"] } } }, "all_ride_totals": { - "type": "object", + "type": ["null", "object"], "additionalProperties": true, "properties": { "count": { - "type": "integer" + "type": ["null", "integer"] }, "distance": { - "type": "number" + "type": ["null", "number"] }, "moving_time": { - "type": "integer" + "type": ["null", "integer"] }, "elapsed_time": { - "type": "integer" + "type": ["null", "integer"] }, "elevation_gain": { - "type": "number" + "type": ["null", "number"] }, "achievement_count": { - "type": "integer" + "type": ["null", "integer"] } } }, "all_run_totals": { - "type": "object", + "type": ["null", "object"], "additionalProperties": true, "properties": { "count": { - "type": "integer" + "type": ["null", "integer"] }, "distance": { - "type": "number" + "type": ["null", "number"] }, "moving_time": { - "type": "integer" + "type": ["null", "integer"] }, "elapsed_time": { - "type": "integer" + "type": ["null", "integer"] }, "elevation_gain": { - "type": "number" + "type": ["null", "number"] }, "achievement_count": { - "type": "integer" + "type": ["null", "integer"] } } }, "all_swim_totals": { - "type": "object", + "type": ["null", "object"], "additionalProperties": true, "properties": { "count": { - "type": "integer" + "type": ["null", "integer"] }, "distance": { - "type": "number" + "type": ["null", "number"] }, "moving_time": { - "type": "integer" + "type": ["null", "integer"] }, "elapsed_time": { - "type": "integer" + "type": ["null", "integer"] }, "elevation_gain": { - "type": "number" + "type": ["null", "number"] }, "achievement_count": { - "type": "integer" + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-strava/source_strava/source.py b/airbyte-integrations/connectors/source-strava/source_strava/source.py index d22efc1fcf5e..9a13e3ee3cb4 100644 --- a/airbyte-integrations/connectors/source-strava/source_strava/source.py +++ b/airbyte-integrations/connectors/source-strava/source_strava/source.py @@ -2,44 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import Any, List, Mapping, Tuple +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator -from source_strava.streams import Activities, AthleteStats +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. +WARNING: Do not modify this file. +""" -# Source -class SourceStrava(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - """ - :param config: the user-input config object conforming to the connector's spec.json - :param logger: logger object - :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. - """ - try: - auth = self.get_oauth(config) - _ = auth.get_auth_header() - return True, None - except Exception as e: - return False, repr(e) - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - auth = self.get_oauth(config) - return [ - AthleteStats(authenticator=auth, athlete_id=config["athlete_id"]), - Activities(authenticator=auth, after=config["start_date"]), - ] - - def get_oauth(self, config): - return Oauth2Authenticator( - token_refresh_endpoint="https://www.strava.com/oauth/token", - client_id=config["client_id"], - client_secret=config["client_secret"], - refresh_token=config["refresh_token"], - scopes=["read_all", "activity:read_all"], - ) +# Declarative Source +class SourceStrava(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-strava/source_strava/spec.json b/airbyte-integrations/connectors/source-strava/source_strava/spec.json deleted file mode 100644 index cbe4dc14645f..000000000000 --- a/airbyte-integrations/connectors/source-strava/source_strava/spec.json +++ /dev/null @@ -1,109 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/strava", - "connectionSpecification": { - "$schema": "https://json-schema.org/draft-07/schema#", - "title": "Strava Spec", - "type": "object", - "required": [ - "client_id", - "client_secret", - "refresh_token", - "athlete_id", - "start_date" - ], - "additionalProperties": true, - "properties": { - "auth_type": { - "type": "string", - "const": "Client", - "enum": ["Client"], - "default": "Client" - }, - "client_id": { - "type": "string", - "description": "The Client ID of your Strava developer application.", - "title": "Client ID", - "pattern": "^[0-9_\\-]+$", - "examples": ["12345"], - "order": 0 - }, - "client_secret": { - "type": "string", - "description": "The Client Secret of your Strava developer application.", - "title": "Client Secret", - "pattern": "^[0-9a-fA-F]+$", - "examples": ["fc6243f283e51f6ca989aab298b17da125496f50"], - "airbyte_secret": true, - "order": 1 - }, - "refresh_token": { - "type": "string", - "description": "The Refresh Token with the activity: read_all permissions.", - "title": "Refresh Token", - "pattern": "^[0-9a-fA-F]+$", - "examples": ["fc6243f283e51f6ca989aab298b17da125496f50"], - "airbyte_secret": true, - "order": 2 - }, - "athlete_id": { - "type": "integer", - "description": "The Athlete ID of your Strava developer application.", - "title": "Athlete ID", - "pattern": "^[0-9_\\-]+$", - "examples": ["17831421"], - "order": 3 - }, - "start_date": { - "type": "string", - "description": "UTC date and time. Any data before this date will not be replicated.", - "title": "Start Date", - "examples": ["2021-03-01T00:00:00Z"], - "format": "date-time", - "order": 4 - } - } - }, - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["auth_type"], - "predicate_value": "Client", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": true, - "properties": { - "refresh_token": { - "type": "string", - "path_in_connector_config": ["refresh_token"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": true, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["client_secret"] - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-strava/source_strava/streams.py b/airbyte-integrations/connectors/source-strava/source_strava/streams.py deleted file mode 100644 index c2066d079f65..000000000000 --- a/airbyte-integrations/connectors/source-strava/source_strava/streams.py +++ /dev/null @@ -1,91 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from abc import ABC -from typing import Any, Iterable, Mapping, MutableMapping, Optional - -import requests -from airbyte_cdk.sources.streams.http import HttpStream -from dateutil import parser - - -# Basic full refresh stream -class StravaStream(HttpStream, ABC): - url_base = "https://www.strava.com/api/v3/" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield response.json() - - -# Basic incremental stream -class IncrementalStravaStream(StravaStream, ABC): - def __init__(self, after, **kwargs): - super().__init__(**kwargs) - self.after = parser.parse(after).timestamp() - - per_page = 30 # default strava value - curr_page = 1 - - @property - def cursor_field(self) -> str: - return "start_date" - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - return {self.cursor_field: max(latest_record.get(self.cursor_field, ""), current_stream_state.get(self.cursor_field, ""))} - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - if len(response.json()) != 0: - self.curr_page += 1 - return {"page": self.curr_page} - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return {"per_page": self.per_page, "page": self.curr_page, "after": self.after} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield from response.json() - - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - if stream_state: - self.after = parser.parse(stream_state.get(self.cursor_field)).timestamp() - return [{"after": self.after}] - - -class AthleteStats(StravaStream): - """ - Returns the activity stats of an athlete. Only includes data from activities set to Everyone visibilty. - API Docs: https://developers.strava.com/docs/reference/#api-Athletes-getStats - Endpoint: https://www.strava.com/api/v3//stats - """ - - primary_key = "" - - def __init__(self, athlete_id: int, **kwargs): - super().__init__(**kwargs) - self.athlete_id = athlete_id - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - athlete_id = self.athlete_id - return f"athletes/{athlete_id}/stats" - - -class Activities(IncrementalStravaStream): - """ - Returns the activities of an athlete. - API Docs: https://developers.strava.com/docs/reference/#api-Activities-getLoggedInAthleteActivities - Endpoint: https://www.strava.com/api/v3/athlete/activities - """ - - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "athlete/activities" diff --git a/airbyte-integrations/connectors/source-strava/unit_tests/__init__.py b/airbyte-integrations/connectors/source-strava/unit_tests/__init__.py deleted file mode 100644 index 46b7376756ec..000000000000 --- a/airbyte-integrations/connectors/source-strava/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-strava/unit_tests/conftest.py b/airbyte-integrations/connectors/source-strava/unit_tests/conftest.py deleted file mode 100644 index a201da1bf22d..000000000000 --- a/airbyte-integrations/connectors/source-strava/unit_tests/conftest.py +++ /dev/null @@ -1,21 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from pytest import fixture - - -@fixture -def start_date(): - return "2016-01-01T00:00:00" - - -@fixture -def config(): - return { - "client_id": "12345", - "client_secret": "0000000000000000000000000000000000000000", - "refresh_token": "0000000000000000000000000000000000000000", - "athlete_id": 12345678, - "start_date": "2016-01-01 00:00:00" - } diff --git a/airbyte-integrations/connectors/source-strava/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-strava/unit_tests/test_incremental_streams.py deleted file mode 100644 index 1c25496b12f7..000000000000 --- a/airbyte-integrations/connectors/source-strava/unit_tests/test_incremental_streams.py +++ /dev/null @@ -1,60 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from airbyte_cdk.models import SyncMode -from pytest import fixture -from source_strava.streams import IncrementalStravaStream - - -@fixture -def patch_incremental_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(IncrementalStravaStream, "path", "v0/example_endpoint") - mocker.patch.object(IncrementalStravaStream, "primary_key", "test_primary_key") - mocker.patch.object(IncrementalStravaStream, "__abstractmethods__", set()) - - -def test_cursor_field(patch_incremental_base_class, start_date): - stream = IncrementalStravaStream(start_date) - expected_cursor_field = "start_date" - assert stream.cursor_field == expected_cursor_field - - -def test_get_updated_state(patch_incremental_base_class, start_date): - stream = IncrementalStravaStream(start_date) - expected_cursor_field = "start_date" - inputs = { - "current_stream_state": {expected_cursor_field: "2015-01-01T00:00:00Z"}, - "latest_record": {expected_cursor_field: "2016-01-01T00:00:00Z"}, - } - expected_state = {"start_date": "2016-01-01T00:00:00Z"} - assert stream.get_updated_state(**inputs) == expected_state - - -def test_stream_slices(patch_incremental_base_class, start_date): - stream = IncrementalStravaStream(start_date) - cursor_field = "start_date" - epoch_string = "1970-01-01T00:00:00Z" - epoch_timestamp = 0 - inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [cursor_field], "stream_state": {cursor_field: epoch_string}} - expected_stream_slice = [{"after": epoch_timestamp}] - assert stream.stream_slices(**inputs) == expected_stream_slice - - -def test_supports_incremental(patch_incremental_base_class, mocker, start_date): - mocker.patch.object(IncrementalStravaStream, "cursor_field", "dummy_field") - stream = IncrementalStravaStream(start_date) - assert stream.supports_incremental - - -def test_source_defined_cursor(patch_incremental_base_class, start_date): - stream = IncrementalStravaStream(start_date) - assert stream.source_defined_cursor - - -def test_stream_checkpoint_interval(patch_incremental_base_class, start_date): - stream = IncrementalStravaStream(start_date) - expected_checkpoint_interval = None - assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-strava/unit_tests/test_source.py b/airbyte-integrations/connectors/source-strava/unit_tests/test_source.py deleted file mode 100644 index fe449907de99..000000000000 --- a/airbyte-integrations/connectors/source-strava/unit_tests/test_source.py +++ /dev/null @@ -1,22 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from source_strava.source import SourceStrava - - -def test_source_streams(config): - streams = SourceStrava().streams(config=config) - assert len(streams) == 2 - - -def test_source_check_connection_success(config, requests_mock): - requests_mock.post("https://www.strava.com/oauth/token", json={"access_token": "my_access_token", "expires_in": 64000}) - results = SourceStrava().check_connection(logger=None, config=config) - assert results == (True, None) - - -def test_source_check_connection_failed(config, requests_mock): - requests_mock.post("https://www.strava.com/oauth/token", status_code=401) - results = SourceStrava().check_connection(logger=None, config=config) - assert results == (False, "HTTPError('401 Client Error: None for url: https://www.strava.com/oauth/token')") diff --git a/airbyte-integrations/connectors/source-strava/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-strava/unit_tests/test_streams.py deleted file mode 100644 index e4571f7e91f5..000000000000 --- a/airbyte-integrations/connectors/source-strava/unit_tests/test_streams.py +++ /dev/null @@ -1,75 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_strava.streams import Activities, AthleteStats, StravaStream - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(StravaStream, "path", "v0/example_endpoint") - mocker.patch.object(StravaStream, "primary_key", "test_primary_key") - mocker.patch.object(StravaStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = StravaStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = StravaStream() - inputs = {"response": MagicMock()} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_request_headers(patch_base_class): - stream = StravaStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = StravaStream() - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = StravaStream() - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = StravaStream() - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time - - -def test_path_activities(config): - assert Activities(authenticator=None, after=config["start_date"]).path() == 'athlete/activities' - - -def test_path_athlete_stats(config): - assert AthleteStats(authenticator=None, athlete_id=config["athlete_id"]).path() == 'athletes/12345678/stats' diff --git a/airbyte-integrations/connectors/source-stripe/Dockerfile b/airbyte-integrations/connectors/source-stripe/Dockerfile deleted file mode 100644 index 7038660a56b2..000000000000 --- a/airbyte-integrations/connectors/source-stripe/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY setup.py ./ -RUN pip install . -COPY source_stripe ./source_stripe -COPY main.py ./ - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - - -LABEL io.airbyte.version=3.17.4 -LABEL io.airbyte.name=airbyte/source-stripe diff --git a/airbyte-integrations/connectors/source-stripe/README.md b/airbyte-integrations/connectors/source-stripe/README.md index d8f47a922e1d..a941634a17cc 100644 --- a/airbyte-integrations/connectors/source-stripe/README.md +++ b/airbyte-integrations/connectors/source-stripe/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-stripe:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/stripe) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_stripe/spec.yaml` file. @@ -70,61 +62,16 @@ docker build . --no-cache -t airbyte/source-stripe:dev \ && python -m pytest integration_tests -p integration_tests.acceptance ``` -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-stripe:unitTest -``` - -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-stripe:integrationTest -``` - -#### Build -To run your integration tests with docker localy - -First, make sure you build the latest Docker image: -``` -docker build --no-cache . -t airbyte/source-stripe:dev -``` - -You can also build the connector image via Gradle: -``` -./gradlew clean :airbyte-integrations:connectors:source-stripe:airbyteDocker -``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. - -#### Run -Then run any of the connector commands as follows: -``` -docker run --rm airbyte/source-stripe:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-stripe:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-stripe:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-stripe:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json -``` - -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-stripe:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -2. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -3. Create a Pull Request -4. Pat yourself on the back for being an awesome contributor -5. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master - - -### additional connector/streams properties of note +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-stripe test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/stripe.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. -Some stripe streams are mutable, meaning that after an incremental update, new data items could appear *before* -the latest update date. To work around that, define the lookback_window_days to define a window in days to fetch results -before the latest state date, in order to capture "delayed" data items. diff --git a/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml b/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml index 1a76cc309b6f..c3002b6d31f3 100644 --- a/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-stripe/acceptance-test-config.yml @@ -14,7 +14,7 @@ acceptance_tests: tests: - config_path: "secrets/config.json" backward_compatibility_tests_config: - disable_for_version: "3.17.0" # invoices schema fix + disable_for_version: 4.4.2 basic_read: tests: - config_path: "secrets/config.json" @@ -23,44 +23,39 @@ acceptance_tests: - name: "application_fees" bypass_reason: "This stream can't be seeded in our sandbox account" - name: "application_fees_refunds" - bypass_reason: "this stream can't be seeded in our sandbox account" + bypass_reason: "This stream can't be seeded in our sandbox account" - name: "authorizations" bypass_reason: "This stream can't be seeded in our sandbox account" - name: "bank_accounts" - bypass_reason: "this stream can't be seeded in our sandbox account" + bypass_reason: "This stream can't be seeded in our sandbox account" - name: "cards" - bypass_reason: "this stream can't be seeded in our sandbox account" + bypass_reason: "This stream can't be seeded in our sandbox account" - name: "early_fraud_warnings" bypass_reason: "This stream can't be seeded in our sandbox account" - name: "external_account_bank_accounts" bypass_reason: "This stream can't be seeded in our sandbox account" - name: "external_account_cards" bypass_reason: "This stream can't be seeded in our sandbox account" - - name: "checkout_sessions" - bypass_reason: "This stream can't be seeded in our sandbox account" - - name: "checkout_sessions_line_items" - bypass_reason: "This stream can't be seeded in our sandbox account" - name: "payment_methods" - bypass_reason: "this stream can't be seeded in our sandbox account" + bypass_reason: "This stream can't be seeded in our sandbox account" - name: "persons" - bypass_reason: "this stream can't be seeded in our sandbox account" + bypass_reason: "This stream can't be seeded in our sandbox account" - name: "reviews" - bypass_reason: "this stream can't be seeded in our sandbox account" - - name: "transfers" bypass_reason: "This stream can't be seeded in our sandbox account" - name: "transactions" bypass_reason: "This stream can't be seeded in our sandbox account" + - name: "events" + bypass_reason: "Data expires every 30 days." expect_records: path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: no - extra_records: yes ignored_fields: invoices: - name: invoice_pdf bypass_reason: "URL changes upon each request for privacy/security" - name: hosted_invoice_url bypass_reason: "URL changes upon each request for privacy/security" + - name: lines/data/*/margins + bypass_reason: "API randomly returns this field" charges: - name: receipt_url bypass_reason: "URL changes upon each request for privacy/security" @@ -86,6 +81,19 @@ acceptance_tests: usage_records: - name: id bypass_reason: "id field is randomly generated" + invoice_line_items: + - name: margins + bypass_reason: "API randomly returns this field" + subscriptions: + - name: current_period_start + bypass_reason: "Frequently changing data" + - name: current_period_end + bypass_reason: "Frequently changing data" + - name: latest_invoice + bypass_reason: "Frequently changing data" + customers: + - name: next_invoice_sequence + bypass_reason: "Frequently changing data" incremental: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-stripe/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-stripe/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-stripe/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-stripe/build.gradle b/airbyte-integrations/connectors/source-stripe/build.gradle deleted file mode 100644 index 9155c2e56cc9..000000000000 --- a/airbyte-integrations/connectors/source-stripe/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_stripe' -} diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-stripe/integration_tests/abnormal_state.json index fa0c3e0dcf85..97d865ec3c49 100644 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-stripe/integration_tests/abnormal_state.json @@ -6,13 +6,6 @@ "stream_descriptor": { "name": "application_fees" } } }, - { - "type": "STREAM", - "stream": { - "stream_state": { "date": 10000000000 }, - "stream_descriptor": { "name": "application_fees_refunds" } - } - }, { "type": "STREAM", "stream": { @@ -69,13 +62,6 @@ "stream_descriptor": { "name": "plans" } } }, - { - "type": "STREAM", - "stream": { - "stream_state": { "created": 10000000000 }, - "stream_descriptor": { "name": "persons" } - } - }, { "type": "STREAM", "stream": { @@ -177,14 +163,14 @@ { "type": "STREAM", "stream": { - "stream_state": { "expires_at": 10000000000 }, + "stream_state": { "updated": 10000000000 }, "stream_descriptor": { "name": "checkout_sessions" } } }, { "type": "STREAM", "stream": { - "stream_state": { "checkout_session_expires_at": 10000000000 }, + "stream_state": { "checkout_session_updated": 10000000000 }, "stream_descriptor": { "name": "checkout_sessions_line_items" } } }, @@ -236,5 +222,61 @@ "stream_state": { "created": 10000000000 }, "stream_descriptor": { "name": "transactions" } } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated": 10000000000 }, + "stream_descriptor": { "name": "application_fees_refunds" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated": 10000000000 }, + "stream_descriptor": { "name": "bank_accounts" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated": 10000000000 }, + "stream_descriptor": { "name": "credit_notes" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated": 10000000000 }, + "stream_descriptor": { "name": "early_fraud_warnings" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated": 10000000000 }, + "stream_descriptor": { "name": "external_account_bank_accounts" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated": 10000000000 }, + "stream_descriptor": { "name": "external_account_cards" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated": 10000000000 }, + "stream_descriptor": { "name": "payment_methods" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated": 10000000000 }, + "stream_descriptor": { "name": "persons" } + } } ] diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-stripe/integration_tests/configured_catalog.json index 31f8336f0ba2..281642987467 100644 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-stripe/integration_tests/configured_catalog.json @@ -4,35 +4,40 @@ "stream": { "name": "accounts", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] }, + "primary_key": [["id"]], "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, { "stream": { - "name": "application_fees_refunds", + "name": "application_fees", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, + "primary_key": [["id"]], + "cursor_field": ["updated"], "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "primary_key": [["id"]] + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "application_fees", + "name": "application_fees_refunds", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created"], + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { @@ -40,13 +45,13 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created"], + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { @@ -57,21 +62,38 @@ "default_cursor_field": ["created"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", + "primary_key": [["id"]], "cursor_field": ["created"], - "primary_key": [["id"]] + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { "name": "bank_accounts", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated"], + "source_defined_primary_key": [["id"]] + }, + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "cardholders", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, + "primary_key": [["id"]], + "cursor_field": ["updated"], "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "primary_key": [["id"]] + "destination_sync_mode": "overwrite" }, { "stream": { @@ -79,228 +101,287 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created"], + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "cardholders", + "name": "charges", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created"], + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "external_account_bank_accounts", + "name": "checkout_sessions", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, + "primary_key": [["id"]], + "cursor_field": ["updated"], "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "primary_key": [["id"]] + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "invoices", + "name": "checkout_sessions_line_items", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created"], + "default_cursor_field": ["checkout_session_updated"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] + "primary_key": [["id"]], + "cursor_field": ["checkout_session_updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "payment_intents", + "name": "coupons", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created"], + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "payouts", + "name": "credit_notes", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created"], + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "plans", + "name": "customer_balance_transactions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "primary_key": [["id"]], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "customers", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created"], + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "prices", + "name": "disputes", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created"], + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "products", + "name": "early_fraud_warnings", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created"], + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "charges", + "name": "events", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["created"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", + "primary_key": [["id"]], "cursor_field": ["created"], - "primary_key": [["id"]] + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "checkout_sessions", + "name": "external_account_bank_accounts", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["expires_at"], + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, + "primary_key": [["id"]], + "cursor_field": ["updated"], "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "cursor_field": ["expires_at"], - "primary_key": [["id"]] + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "checkout_sessions_line_items", + "name": "external_account_cards", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["checkout_session_expires_at"], + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, + "primary_key": [["id"]], + "cursor_field": ["updated"], "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "cursor_field": ["checkout_session_expires_at"], - "primary_key": [["id"]] + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "coupons", + "name": "file_links", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["created"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", + "primary_key": [["id"]], "cursor_field": ["created"], - "primary_key": [["id"]] + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "shipping_rates", + "name": "files", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["created"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", + "primary_key": [["id"]], "cursor_field": ["created"], - "primary_key": [["id"]] + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "subscription_items", + "name": "invoice_items", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, + "primary_key": [["id"]], + "cursor_field": ["updated"], "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "primary_key": [["id"]] + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "customer_balance_transactions", + "name": "invoice_line_items", "json_schema": {}, "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, + "primary_key": [["id"]], "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "primary_key": [["id"]] + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "files", + "name": "invoices", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created"], + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "payment_intents", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated"], + "source_defined_primary_key": [["id"]] + }, + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "payment_methods", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated"], + "source_defined_primary_key": [["id"]] + }, + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "payouts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated"], + "source_defined_primary_key": [["id"]] + }, + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { @@ -308,13 +389,41 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created"], + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "plans", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated"], + "source_defined_primary_key": [["id"]] + }, + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "prices", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated"], + "source_defined_primary_key": [["id"]] + }, + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { @@ -322,62 +431,162 @@ "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created"], + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "file_links", + "name": "promotion_codes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated"], + "source_defined_primary_key": [["id"]] + }, + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "refunds", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["created"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", + "primary_key": [["id"]], "cursor_field": ["created"], - "primary_key": [["id"]] + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "top_ups", + "name": "reviews", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated"], + "source_defined_primary_key": [["id"]] + }, + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "setup_attempts", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["created"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", + "primary_key": [["id"]], "cursor_field": ["created"], - "primary_key": [["id"]] + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "setup_attempts", + "name": "setup_intents", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated"], + "source_defined_primary_key": [["id"]] + }, + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "shipping_rates", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["created"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", + "primary_key": [["id"]], "cursor_field": ["created"], - "primary_key": [["id"]] + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "usage_records", + "name": "subscription_items", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "primary_key": [["id"]], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "subscriptions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated"], + "source_defined_primary_key": [["id"]] + }, + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "subscription_schedule", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated"], + "source_defined_primary_key": [["id"]] + }, + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "top_ups", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated"], + "source_defined_primary_key": [["id"]] }, + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "transactions", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated"], + "source_defined_primary_key": [["id"]] + }, + "primary_key": [["id"]], + "cursor_field": ["updated"], "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" }, @@ -388,27 +597,27 @@ "supported_sync_modes": ["full_refresh"], "source_defined_primary_key": [["id"]] }, + "primary_key": [["id"]], "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "primary_key": [["id"]] + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "transactions", + "name": "transfers", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, - "default_cursor_field": ["created"], + "default_cursor_field": ["updated"], "source_defined_primary_key": [["id"]] }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] + "primary_key": [["id"]], + "cursor_field": ["updated"], + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { "stream": { - "name": "credit_notes", + "name": "usage_records", "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/connected_account_configured_catalog.json b/airbyte-integrations/connectors/source-stripe/integration_tests/connected_account_configured_catalog.json deleted file mode 100644 index 29700a83523b..000000000000 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/connected_account_configured_catalog.json +++ /dev/null @@ -1,97 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "subscriptions", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] - }, - { - "stream": { - "name": "shipping_rates", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] - }, - { - "stream": { - "name": "application_fees", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] - }, - { - "stream": { - "name": "invoices", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] - }, - { - "stream": { - "name": "early_fraud_warnings", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "payment_intents", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "append", - "cursor_field": ["created"], - "primary_key": [["id"]] - }, - { - "stream": { - "name": "subscription_schedule", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] - } - ] -} diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl index 8ca6426f4d31..d3b48ade375a 100644 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-stripe/integration_tests/expected_records.jsonl @@ -1,58 +1,72 @@ -{"stream": "accounts", "data": {"id": "acct_1Jx8unEYmRTj5on1", "object": "account", "business_profile": {"mcc": null, "name": "Airbyte", "support_address": null, "support_email": null, "support_phone": null, "support_url": null, "url": null}, "capabilities": {}, "charges_enabled": false, "controller": {"type": "account"}, "country": "US", "default_currency": "usd", "details_submitted": false, "email": null, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "metadata": {}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": [], "pending_verification": []}, "settings": {"bacs_debit_payments": {}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "Etc/UTC"}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "sepa_debit_payments": {}}, "type": "standard"}, "emitted_at": 1689684208922} -{"stream": "accounts", "data": {"id": "acct_1HRPLyCpK2Z3jTFF", "object": "account", "capabilities": {"acss_debit_payments": "inactive", "afterpay_clearpay_payments": "inactive", "bancontact_payments": "inactive", "card_payments": "inactive", "eps_payments": "inactive", "giropay_payments": "inactive", "ideal_payments": "inactive", "p24_payments": "inactive", "sepa_debit_payments": "inactive", "sofort_payments": "inactive", "transfers": "inactive"}, "charges_enabled": false, "country": "US", "default_currency": "usd", "details_submitted": false, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.mcc", "business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "individual.dob.day", "individual.dob.month", "individual.dob.year", "individual.email", "individual.first_name", "individual.last_name", "individual.phone", "individual.ssn_last_4", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.mcc", "business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "individual.dob.day", "individual.dob.month", "individual.dob.year", "individual.email", "individual.first_name", "individual.last_name", "individual.phone", "individual.ssn_last_4", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": ["business_profile.mcc", "business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "individual.dob.day", "individual.dob.month", "individual.dob.year", "individual.email", "individual.first_name", "individual.last_name", "individual.phone", "individual.ssn_last_4", "tos_acceptance.date", "tos_acceptance.ip"], "pending_verification": []}, "settings": {"bacs_debit_payments": {}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "America/Los_Angeles"}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "sepa_debit_payments": {}}, "type": "standard"}, "emitted_at": 1689684208922} -{"stream": "accounts", "data": {"id": "acct_1MwD6tIyVv44cUB4", "object": "account", "business_profile": {"mcc": null, "name": null, "product_description": null, "support_address": null, "support_email": null, "support_phone": null, "support_url": null, "url": null}, "business_type": null, "capabilities": {"card_payments": "inactive", "transfers": "inactive"}, "charges_enabled": false, "country": "US", "created": 1681342196, "default_currency": "usd", "details_submitted": false, "email": "jenny.rosen@example.com", "external_accounts": {"object": "list", "data": [], "has_more": false, "total_count": 0, "url": "/v1/accounts/acct_1MwD6tIyVv44cUB4/external_accounts"}, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "metadata": {}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "pending_verification": []}, "settings": {"bacs_debit_payments": {}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"decline_on": {"avs_failure": false, "cvc_failure": false}, "statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "Etc/UTC"}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "payouts": {"debit_negative_balances": false, "schedule": {"delay_days": 2, "interval": "daily"}, "statement_descriptor": null}, "sepa_debit_payments": {}}, "tos_acceptance": {"date": null, "ip": null, "user_agent": null}, "type": "custom"}, "emitted_at": 1689684209297} -{"stream": "balance_transactions", "data": {"id": "txn_1KVQhfEcXtiJtvvhF7ox3YEm", "object": "balance_transaction", "amount": -9164, "available_on": 1645488000, "created": 1645406919, "currency": "usd", "description": "STRIPE PAYOUT", "exchange_rate": null, "fee": 0, "fee_details": [], "net": -9164, "reporting_category": "payout", "source": "po_1KVQhfEcXtiJtvvhZlUkl08U", "status": "available", "type": "payout"}, "emitted_at": 1689684215036} -{"stream": "balance_transactions", "data": {"id": "txn_3K9FSOEcXtiJtvvh0KoS5mx7", "object": "balance_transaction", "amount": 5300, "available_on": 1640649600, "created": 1640120473, "currency": "usd", "description": null, "exchange_rate": null, "fee": 184, "fee_details": [{"amount": 184, "application": null, "currency": "usd", "description": "Stripe processing fees", "type": "stripe_fee"}], "net": 5116, "reporting_category": "charge", "source": "ch_3K9FSOEcXtiJtvvh0zxb7clc", "status": "available", "type": "charge"}, "emitted_at": 1689684215036} -{"stream": "balance_transactions", "data": {"id": "txn_3K9F5DEcXtiJtvvh1qsqmHcH", "object": "balance_transaction", "amount": 4200, "available_on": 1640649600, "created": 1640119035, "currency": "usd", "description": "edgao test", "exchange_rate": null, "fee": 152, "fee_details": [{"amount": 152, "application": null, "currency": "usd", "description": "Stripe processing fees", "type": "stripe_fee"}], "net": 4048, "reporting_category": "charge", "source": "ch_3K9F5DEcXtiJtvvh1w2MaTpj", "status": "available", "type": "charge"}, "emitted_at": 1689684215037} -{"stream": "cardholders", "data": {"id": "ich_1KUKBeEcXtiJtvvhCEFgko6h", "object": "issuing.cardholder", "billing": {"address": {"city": "San Francisco", "country": "US", "line1": "1234 Main Street", "line2": null, "postal_code": "94111", "state": "CA"}}, "company": null, "created": 1645143542, "email": "jenny.rosen@example.com", "individual": null, "livemode": false, "metadata": {}, "name": "Jenny Rosen", "phone_number": "+18888675309", "preferred_locales": [], "requirements": {"disabled_reason": null, "past_due": []}, "spending_controls": {"allowed_categories": [], "blocked_categories": [], "spending_limits": [], "spending_limits_currency": null}, "status": "active", "type": "individual"}, "emitted_at": 1689684219593} -{"stream": "charges", "data": {"id": "ch_3K9FSOEcXtiJtvvh0zxb7clc", "object": "charge", "amount": 5300, "amount_captured": 5300, "amount_refunded": 0, "amount_updates": [], "application": null, "application_fee": null, "application_fee_amount": null, "balance_transaction": "txn_3K9FSOEcXtiJtvvh0KoS5mx7", "billing_details": {"address": {"city": null, "country": null, "line1": null, "line2": null, "postal_code": null, "state": null}, "email": null, "name": null, "phone": null}, "calculated_statement_descriptor": "AIRBYTE.IO", "captured": true, "created": 1640120473, "currency": "usd", "customer": null, "description": null, "destination": null, "dispute": null, "disputed": false, "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "fraud_details": {}, "invoice": null, "livemode": false, "metadata": {}, "on_behalf_of": null, "order": null, "outcome": {"network_status": "approved_by_network", "reason": null, "risk_level": "normal", "risk_score": 48, "seller_message": "Payment complete.", "type": "authorized"}, "paid": true, "payment_intent": "pi_3K9FSOEcXtiJtvvh0AEIFllC", "payment_method": "src_1K9FSOEcXtiJtvvhHGu1qtOx", "payment_method_details": {"card": {"brand": "visa", "checks": {"address_line1_check": null, "address_postal_code_check": null, "cvc_check": "pass"}, "country": "US", "exp_month": 12, "exp_year": 2034, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "installments": null, "last4": "4242", "mandate": null, "network": "visa", "network_token": {"used": false}, "three_d_secure": null, "wallet": null}, "type": "card"}, "receipt_email": null, "receipt_number": "1509-9197", "receipt_url": "https://pay.stripe.com/receipts/payment/CAcaFwoVYWNjdF8xSndub2lFY1h0aUp0dnZoKJ6C36UGMgbHNxWFIZs6LBZJopnCEqXAr1eGbMC_s5Z-CjimuY7h3BPBSgA3kwq3H409GQNJ-c_Rq1jL", "refunded": false, "refunds": {"object": "list", "data": [], "has_more": false, "total_count": 0.0, "url": "/v1/charges/ch_3K9FSOEcXtiJtvvh0zxb7clc/refunds"}, "review": null, "shipping": null, "source": {"id": "src_1K9FSOEcXtiJtvvhHGu1qtOx", "object": "source", "amount": null, "card": {"address_line1_check": null, "address_zip_check": null, "brand": "Visa", "country": "US", "cvc_check": "pass", "dynamic_last4": null, "exp_month": 12, "exp_year": 2034, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "last4": "4242", "name": null, "three_d_secure": "optional", "tokenization_method": null}, "client_secret": "src_client_secret_3WszbFGtWT8vmMjqnNztOwhU", "created": 1640120473, "currency": null, "flow": "none", "livemode": false, "metadata": {}, "owner": {"address": null, "email": null, "name": null, "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "consumed", "type": "card", "usage": "reusable"}, "source_transfer": null, "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "succeeded", "transfer_data": null, "transfer_group": null}, "emitted_at": 1689764126216} -{"stream": "charges", "data": {"id": "ch_3K9F5DEcXtiJtvvh1w2MaTpj", "object": "charge", "amount": 4200, "amount_captured": 4200, "amount_refunded": 0, "amount_updates": [], "application": null, "application_fee": null, "application_fee_amount": null, "balance_transaction": "txn_3K9F5DEcXtiJtvvh1qsqmHcH", "billing_details": {"address": {"city": null, "country": null, "line1": null, "line2": null, "postal_code": null, "state": null}, "email": null, "name": null, "phone": null}, "calculated_statement_descriptor": "AIRBYTE.IO", "captured": true, "created": 1640119035, "currency": "usd", "customer": null, "description": "edgao test", "destination": null, "dispute": null, "disputed": false, "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "fraud_details": {}, "invoice": null, "livemode": false, "metadata": {}, "on_behalf_of": null, "order": null, "outcome": {"network_status": "approved_by_network", "reason": null, "risk_level": "normal", "risk_score": 63, "seller_message": "Payment complete.", "type": "authorized"}, "paid": true, "payment_intent": "pi_3K9F5DEcXtiJtvvh16scJMp6", "payment_method": "src_1K9F5CEcXtiJtvvhrsZdur8Y", "payment_method_details": {"card": {"brand": "visa", "checks": {"address_line1_check": null, "address_postal_code_check": null, "cvc_check": "pass"}, "country": "US", "exp_month": 9, "exp_year": 2028, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "installments": null, "last4": "4242", "mandate": null, "network": "visa", "network_token": {"used": false}, "three_d_secure": null, "wallet": null}, "type": "card"}, "receipt_email": null, "receipt_number": "1549-5630", "receipt_url": "https://pay.stripe.com/receipts/payment/CAcaFwoVYWNjdF8xSndub2lFY1h0aUp0dnZoKJ6C36UGMgap0zETEvk6LBZ9WiYPxmb7YdOMYue__fsqDfzaiB9KIoJo1the8s2BD-W_hVVO6OkQI4Mu", "refunded": false, "refunds": {"object": "list", "data": [], "has_more": false, "total_count": 0.0, "url": "/v1/charges/ch_3K9F5DEcXtiJtvvh1w2MaTpj/refunds"}, "review": null, "shipping": null, "source": {"id": "src_1K9F5CEcXtiJtvvhrsZdur8Y", "object": "source", "amount": null, "card": {"address_line1_check": null, "address_zip_check": null, "brand": "Visa", "country": "US", "cvc_check": "pass", "dynamic_last4": null, "exp_month": 9, "exp_year": 2028, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "last4": "4242", "name": null, "three_d_secure": "optional", "tokenization_method": null}, "client_secret": "src_client_secret_QyH8xuqSyiZh8oxzzIszqQ92", "created": 1640119035, "currency": null, "flow": "none", "livemode": false, "metadata": {}, "owner": {"address": null, "email": null, "name": null, "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "consumed", "type": "card", "usage": "reusable"}, "source_transfer": null, "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "succeeded", "transfer_data": null, "transfer_group": null}, "emitted_at": 1689764126217} -{"stream": "charges", "data": {"id": "ch_3K9F4mEcXtiJtvvh1kUzxjwN", "object": "charge", "amount": 4200, "amount_captured": 0, "amount_refunded": 0, "amount_updates": [], "application": null, "application_fee": null, "application_fee_amount": null, "balance_transaction": null, "billing_details": {"address": {"city": null, "country": null, "line1": null, "line2": null, "postal_code": null, "state": null}, "email": null, "name": null, "phone": null}, "calculated_statement_descriptor": "AIRBYTE.IO", "captured": false, "created": 1640119009, "currency": "usd", "customer": null, "description": "edgao test", "destination": null, "dispute": null, "disputed": false, "failure_balance_transaction": null, "failure_code": "card_declined", "failure_message": "Your card was declined. Your request was in test mode, but used a non test (live) card. For a list of valid test cards, visit: https://stripe.com/docs/testing.", "fraud_details": {}, "invoice": null, "livemode": false, "metadata": {}, "on_behalf_of": null, "order": null, "outcome": {"network_status": "not_sent_to_network", "reason": "test_mode_live_card", "risk_level": "normal", "risk_score": 6, "seller_message": "This charge request was in test mode, but did not use a Stripe test card number. For the list of these numbers, see stripe.com/docs/testing", "type": "invalid"}, "paid": false, "payment_intent": "pi_3K9F4mEcXtiJtvvh18NKhEuo", "payment_method": "src_1K9F4hEcXtiJtvvhrUEwvCyi", "payment_method_details": {"card": {"brand": "visa", "checks": {"address_line1_check": null, "address_postal_code_check": null, "cvc_check": null}, "country": "US", "exp_month": 9, "exp_year": 2028, "fingerprint": "Re3p4j8issXA77iI", "funding": "credit", "installments": null, "last4": "8097", "mandate": null, "network": "visa", "network_token": {"used": false}, "three_d_secure": null, "wallet": null}, "type": "card"}, "receipt_email": null, "receipt_number": null, "receipt_url": null, "refunded": false, "refunds": {"object": "list", "data": [], "has_more": false, "total_count": 0.0, "url": "/v1/charges/ch_3K9F4mEcXtiJtvvh1kUzxjwN/refunds"}, "review": null, "shipping": null, "source": {"id": "src_1K9F4hEcXtiJtvvhrUEwvCyi", "object": "source", "amount": null, "card": {"address_line1_check": null, "address_zip_check": null, "brand": "Visa", "country": "US", "cvc_check": null, "dynamic_last4": null, "exp_month": 9, "exp_year": 2028, "fingerprint": "Re3p4j8issXA77iI", "funding": "credit", "last4": "8097", "name": null, "three_d_secure": "optional", "tokenization_method": null}, "client_secret": "src_client_secret_b3v8YqNMLGykB120fqv2Tjhq", "created": 1640119009, "currency": null, "flow": "none", "livemode": false, "metadata": {}, "owner": {"address": null, "email": null, "name": null, "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "consumed", "type": "card", "usage": "reusable"}, "source_transfer": null, "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "failed", "transfer_data": null, "transfer_group": null}, "emitted_at": 1689764126219} -{"stream": "coupons", "data": {"id": "Coupon000001", "object": "coupon", "amount_off": 500, "created": 1675345584, "currency": "usd", "duration": "once", "duration_in_months": null, "livemode": false, "max_redemptions": null, "metadata": {}, "name": "Test Coupon 1", "percent_off": null, "redeem_by": null, "times_redeemed": 1, "valid": true}, "emitted_at": 1689684226504} -{"stream": "coupons", "data": {"id": "4SUEGKZg", "object": "coupon", "amount_off": 200, "created": 1674209030, "currency": "usd", "duration": "repeating", "duration_in_months": 3, "livemode": false, "max_redemptions": null, "metadata": {}, "name": "\u0406\u0435\u043a\u0448\u0437\u0443", "percent_off": null, "redeem_by": null, "times_redeemed": 0, "valid": true}, "emitted_at": 1689684226504} -{"stream": "coupons", "data": {"id": "iJ6qlwM5", "object": "coupon", "amount_off": null, "created": 1674208993, "currency": null, "duration": "forever", "duration_in_months": null, "livemode": false, "max_redemptions": null, "metadata": {}, "name": "\u0415\u0443\u0456\u0435", "percent_off": 10.0, "redeem_by": null, "times_redeemed": 3, "valid": true}, "emitted_at": 1689684226505} -{"stream": "customer_balance_transactions", "data": {"id": "cbtxn_1MX2zPEcXtiJtvvhr4L2D3Q1", "object": "customer_balance_transaction", "amount": -50000.0, "created": 1675345091, "credit_note": null, "currency": "usd", "customer": "cus_NGoTFiJFVbSsvZ", "description": null, "ending_balance": 0.0, "invoice": "in_1MX2yFEcXtiJtvvhMXhUCgKx", "livemode": false, "metadata": {}, "type": "applied_to_invoice"}, "emitted_at": 1689684227663} -{"stream": "customer_balance_transactions", "data": {"id": "cbtxn_1MWIPLEcXtiJtvvhLnQYjVCj", "object": "customer_balance_transaction", "amount": 50000.0, "created": 1675166031, "credit_note": null, "currency": "usd", "customer": "cus_NGoTFiJFVbSsvZ", "description": "Test credit balance", "ending_balance": 50000.0, "invoice": null, "livemode": false, "metadata": {}, "type": "adjustment"}, "emitted_at": 1689684227664} -{"stream": "customers", "data": {"id": "cus_LIiHR6omh14Xdg", "object": "customer", "address": {"city": "san francisco", "country": "US", "line1": "san francisco", "line2": "", "postal_code": "", "state": "CA"}, "balance": 0, "created": 1646998902, "currency": "usd", "default_source": "card_1MSHU1EcXtiJtvvhytSN6V54", "delinquent": false, "description": "test", "discount": null, "email": "test@airbyte_integration_test.com", "invoice_prefix": "09A6A98F", "invoice_settings": {"custom_fields": null, "default_payment_method": null, "footer": null, "rendering_options": null}, "livemode": false, "metadata": {}, "name": "Test", "next_invoice_sequence": 1, "phone": null, "preferred_locales": [], "shipping": {"address": {"city": "", "country": "US", "line1": "", "line2": "", "postal_code": "", "state": ""}, "name": "", "phone": ""}, "tax_exempt": "none", "test_clock": null}, "emitted_at": 1689691086995} -{"stream": "disputes", "data": {"id": "dp_1MSI78EcXtiJtvvhxC77m2kh", "object": "dispute", "amount": 700, "balance_transaction": "txn_1MSI78EcXtiJtvvhAGjxP1UM", "balance_transactions": [{"id": "txn_1MSI78EcXtiJtvvhAGjxP1UM", "object": "balance_transaction", "amount": -700, "available_on": 1674518400, "created": 1674211590, "currency": "usd", "description": "Chargeback withdrawal for ch_3MSI77EcXtiJtvvh1GzoukUC", "exchange_rate": null, "fee": 1500, "fee_details": [{"amount": 1500, "application": null, "currency": "usd", "description": "Dispute fee", "type": "stripe_fee"}], "net": -2200, "reporting_category": "dispute", "source": "dp_1MSI78EcXtiJtvvhxC77m2kh", "status": "available", "type": "adjustment"}], "charge": "ch_3MSI77EcXtiJtvvh1GzoukUC", "created": 1674211590, "currency": "usd", "evidence": {"access_activity_log": null, "billing_address": "12345", "cancellation_policy": null, "cancellation_policy_disclosure": null, "cancellation_rebuttal": null, "customer_communication": null, "customer_email_address": null, "customer_name": null, "customer_purchase_ip": null, "customer_signature": null, "duplicate_charge_documentation": null, "duplicate_charge_explanation": null, "duplicate_charge_id": null, "product_description": null, "receipt": null, "refund_policy": null, "refund_policy_disclosure": null, "refund_refusal_explanation": null, "service_date": null, "service_documentation": null, "shipping_address": null, "shipping_carrier": null, "shipping_date": null, "shipping_documentation": null, "shipping_tracking_number": null, "uncategorized_file": null, "uncategorized_text": null}, "evidence_details": {"due_by": 1675036799.0, "has_evidence": false, "past_due": false, "submission_count": 0}, "is_charge_refundable": false, "livemode": false, "metadata": {}, "payment_intent": "pi_3MSI77EcXtiJtvvh1glmQd8s", "reason": "fraudulent", "status": "lost"}, "emitted_at": 1689691088952} -{"stream": "events", "data": {"id": "evt_1NSrYfEcXtiJtvvhoo4M4yDP", "object": "event", "api_version": "2020-08-27", "created": 1689124173, "data": {"object": {"object": "balance", "available": [{"amount": 513474, "currency": "usd", "source_types": {"card": 513474}}], "connect_reserved": [{"amount": 0, "currency": "usd"}], "issuing": {"available": [{"amount": 150000, "currency": "usd"}]}, "livemode": false, "pending": [{"amount": 0, "currency": "usd", "source_types": {"card": 0}}]}}, "livemode": false, "pending_webhooks": 0, "request": {"id": null, "idempotency_key": null}, "type": "balance.available"}, "emitted_at": 1689691091429} -{"stream": "invoice_items", "data": {"id": "ii_1K9GKLEcXtiJtvvhmr2AYOAx", "object": "invoiceitem", "amount": 8400, "currency": "usd", "customer": "cus_Kou8knsO3qQOwU", "date": 1640123817, "description": "a box of parsnips", "discountable": true, "discounts": [], "invoice": "in_1K9GK0EcXtiJtvvhSo2LvGqT", "livemode": false, "metadata": {}, "period": {"end": 1640123817, "start": 1640123817}, "plan": null, "price": {"id": "price_1K9GKLEcXtiJtvvhXbrg33lq", "object": "price", "active": false, "billing_scheme": "per_unit", "created": 1640123817, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_Kou8cQxtIpF1p7", "recurring": null, "tax_behavior": "unspecified", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 8400, "unit_amount_decimal": "8400"}, "proration": false, "quantity": 1, "subscription": null, "tax_rates": [], "test_clock": null, "unit_amount": 8400, "unit_amount_decimal": "8400"}, "emitted_at": 1689691092541} -{"stream": "invoice_items", "data": {"id": "ii_1MX384EcXtiJtvvhguyn3iYb", "object": "invoiceitem", "amount": 6000, "currency": "usd", "customer": "cus_NGoTFiJFVbSsvZ", "date": 1675345628, "description": "Test Product 1", "discountable": true, "discounts": ["di_1MX384EcXtiJtvvhkOrY57Ep"], "invoice": "in_1MX37hEcXtiJtvvhRSl1KbQm", "livemode": false, "metadata": {}, "period": {"end": 1675345628, "start": 1675345628}, "plan": null, "price": {"id": "price_1MX364EcXtiJtvvhE3WgTl4O", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1675345504, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NHcKselSHfKdfc", "recurring": null, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 2000, "unit_amount_decimal": "2000"}, "proration": false, "quantity": 3, "subscription": null, "tax_rates": [], "test_clock": null, "unit_amount": 2000, "unit_amount_decimal": "2000"}, "emitted_at": 1689691092833} -{"stream": "invoice_items", "data": {"id": "ii_1MX2yfEcXtiJtvvhfhyOG7SP", "object": "invoiceitem", "amount": 25200, "currency": "usd", "customer": "cus_NGoTFiJFVbSsvZ", "date": 1675345045, "description": "edgao-test-product", "discountable": true, "discounts": ["di_1MX2ysEcXtiJtvvh8ORqRVKm"], "invoice": "in_1MX2yFEcXtiJtvvhMXhUCgKx", "livemode": false, "metadata": {}, "period": {"end": 1675345045, "start": 1675345045}, "plan": null, "price": {"id": "price_1K9GbqEcXtiJtvvhJ3lZe4i5", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1640124902, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_KouQ5ez86yREmB", "recurring": null, "tax_behavior": "inclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 12600, "unit_amount_decimal": "12600"}, "proration": false, "quantity": 2, "subscription": null, "tax_rates": [], "test_clock": null, "unit_amount": 12600, "unit_amount_decimal": "12600"}, "emitted_at": 1689691092834} -{"stream": "invoice_line_items", "data": {"id": "il_1K9GKLEcXtiJtvvhhHaYMebN", "object": "line_item", "amount": 8400, "amount_excluding_tax": 8400, "currency": "usd", "description": "a box of parsnips", "discount_amounts": [], "discountable": true, "discounts": [], "invoice_item": "ii_1K9GKLEcXtiJtvvhmr2AYOAx", "livemode": false, "metadata": {}, "period": {"end": 1640123817, "start": 1640123817}, "plan": null, "price": {"id": "price_1K9GKLEcXtiJtvvhXbrg33lq", "object": "price", "active": false, "billing_scheme": "per_unit", "created": 1640123817, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_Kou8cQxtIpF1p7", "recurring": null, "tax_behavior": "unspecified", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 8400, "unit_amount_decimal": "8400"}, "proration": false, "proration_details": {"credited_items": null}, "quantity": 1, "subscription": null, "tax_amounts": [], "tax_rates": [], "type": "invoiceitem", "unit_amount_excluding_tax": "8400", "invoice_id": "in_1K9GK0EcXtiJtvvhSo2LvGqT"}, "emitted_at": 1689691095089} -{"stream": "invoices", "data": {"id": "in_1K9GK0EcXtiJtvvhSo2LvGqT", "object": "invoice", "account_country": "US", "account_name": "Airbyte, Inc.", "account_tax_ids": null, "amount_due": 0, "amount_paid": 0, "amount_remaining": 0, "amount_shipping": 0, "application": null, "application_fee_amount": null, "attempt_count": 0, "attempted": true, "auto_advance": false, "automatic_tax": {"enabled": false, "status": null}, "billing_reason": "manual", "charge": null, "collection_method": "send_invoice", "created": 1640123796, "currency": "usd", "custom_fields": null, "customer": "cus_Kou8knsO3qQOwU", "customer_address": null, "customer_email": "edward.gao+stripe-test-customer-1@airbyte.io", "customer_name": "edgao-test-customer-1", "customer_phone": null, "customer_shipping": null, "customer_tax_exempt": "none", "customer_tax_ids": [], "default_payment_method": null, "default_source": null, "default_tax_rates": [], "description": null, "discount": null, "discounts": [], "due_date": 1688750070.0, "effective_at": 1686158070, "ending_balance": 0, "footer": null, "from_invoice": null, "hosted_invoice_url": "https://invoice.stripe.com/i/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9Lb3U4bk9YR0lWV3BhN2EzMXZNUFJSaEdXUUVNR1J0LDgwODE4MzQ00200vG3gv95N?s=ap", "invoice_pdf": "https://pay.stripe.com/invoice/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9Lb3U4bk9YR0lWV3BhN2EzMXZNUFJSaEdXUUVNR1J0LDgwODE4MzQ00200vG3gv95N/pdf?s=ap", "last_finalization_error": null, "latest_revision": null, "lines": {"object": "list", "data": [{"id": "il_1K9GKLEcXtiJtvvhhHaYMebN", "object": "line_item", "amount": 8400, "amount_excluding_tax": 8400, "currency": "usd", "description": "a box of parsnips", "discount_amounts": [], "discountable": true, "discounts": [], "invoice_item": "ii_1K9GKLEcXtiJtvvhmr2AYOAx", "livemode": false, "metadata": {}, "period": {"end": 1640123817, "start": 1640123817}, "plan": null, "price": {"id": "price_1K9GKLEcXtiJtvvhXbrg33lq", "object": "price", "active": false, "billing_scheme": "per_unit", "created": 1640123817, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_Kou8cQxtIpF1p7", "recurring": null, "tax_behavior": "unspecified", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 8400, "unit_amount_decimal": "8400"}, "proration": false, "proration_details": {"credited_items": null}, "quantity": 1, "subscription": null, "tax_amounts": [], "tax_rates": [], "type": "invoiceitem", "unit_amount_excluding_tax": "8400"}], "has_more": false, "total_count": 1, "url": "/v1/invoices/in_1K9GK0EcXtiJtvvhSo2LvGqT/lines"}, "livemode": false, "metadata": {}, "next_payment_attempt": null, "number": "CA35DF83-0001", "on_behalf_of": null, "paid": true, "paid_out_of_band": false, "payment_intent": null, "payment_settings": {"default_mandate": null, "payment_method_options": null, "payment_method_types": null}, "period_end": 1640123795.0, "period_start": 1640123795.0, "post_payment_credit_notes_amount": 0, "pre_payment_credit_notes_amount": 8400, "quote": null, "receipt_number": null, "rendering_options": null, "shipping_cost": null, "shipping_details": null, "starting_balance": 0, "statement_descriptor": null, "status": "paid", "status_transitions": {"finalized_at": 1686158070, "marked_uncollectible_at": null, "paid_at": 1686158100, "voided_at": null}, "subscription": null, "subtotal": 8400, "subtotal_excluding_tax": 8400, "tax": null, "test_clock": null, "total": 8400, "total_discount_amounts": [], "total_excluding_tax": 8400, "total_tax_amounts": [], "transfer_data": null, "webhooks_delivered_at": 1640123796.0}, "emitted_at": 1690277544798} -{"stream": "payment_intents", "data": {"id": "pi_3K9FSOEcXtiJtvvh0AEIFllC", "object": "payment_intent", "amount": 5300, "amount_capturable": 0, "amount_details": {"tip": {}}, "amount_received": 5300, "application": null, "application_fee_amount": null, "automatic_payment_methods": null, "canceled_at": null, "cancellation_reason": null, "capture_method": "automatic", "client_secret": "pi_3K9FSOEcXtiJtvvh0AEIFllC_secret_uPUtIaSltgtW0qK7mLD0uF2Mr", "confirmation_method": "automatic", "created": 1640120472, "currency": "usd", "customer": null, "description": null, "invoice": null, "last_payment_error": null, "latest_charge": "ch_3K9FSOEcXtiJtvvh0zxb7clc", "livemode": false, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": null, "payment_method_options": {"card": {"installments": null, "mandate_options": null, "network": null, "request_three_d_secure": "automatic"}}, "payment_method_types": ["card"], "processing": null, "receipt_email": null, "review": null, "setup_future_usage": null, "shipping": null, "source": "src_1K9FSOEcXtiJtvvhHGu1qtOx", "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "succeeded", "transfer_data": null, "transfer_group": null}, "emitted_at": 1689691099579} -{"stream": "payment_intents", "data": {"id": "pi_3K9F5DEcXtiJtvvh16scJMp6", "object": "payment_intent", "amount": 4200, "amount_capturable": 0, "amount_details": {"tip": {}}, "amount_received": 4200, "application": null, "application_fee_amount": null, "automatic_payment_methods": null, "canceled_at": null, "cancellation_reason": null, "capture_method": "automatic", "client_secret": "pi_3K9F5DEcXtiJtvvh16scJMp6_secret_YwhzCTpXtfcKYeklXnPnysRRi", "confirmation_method": "automatic", "created": 1640119035, "currency": "usd", "customer": null, "description": "edgao test", "invoice": null, "last_payment_error": null, "latest_charge": "ch_3K9F5DEcXtiJtvvh1w2MaTpj", "livemode": false, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": null, "payment_method_options": {"card": {"installments": null, "mandate_options": null, "network": null, "request_three_d_secure": "automatic"}}, "payment_method_types": ["card"], "processing": null, "receipt_email": null, "review": null, "setup_future_usage": null, "shipping": null, "source": "src_1K9F5CEcXtiJtvvhrsZdur8Y", "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "succeeded", "transfer_data": null, "transfer_group": null}, "emitted_at": 1689691099579} -{"stream": "payment_intents", "data": {"id": "pi_3K9F4mEcXtiJtvvh18NKhEuo", "object": "payment_intent", "amount": 4200, "amount_capturable": 0, "amount_details": {"tip": {}}, "amount_received": 0, "application": null, "application_fee_amount": null, "automatic_payment_methods": null, "canceled_at": null, "cancellation_reason": null, "capture_method": "automatic", "client_secret": "pi_3K9F4mEcXtiJtvvh18NKhEuo_secret_pfUt7CTkPjVdJacycm0bMpdLt", "confirmation_method": "automatic", "created": 1640119008, "currency": "usd", "customer": null, "description": "edgao test", "invoice": null, "last_payment_error": {"charge": "ch_3K9F4mEcXtiJtvvh1kUzxjwN", "code": "card_declined", "decline_code": "test_mode_live_card", "doc_url": "https://stripe.com/docs/error-codes/card-declined", "message": "Your card was declined. Your request was in test mode, but used a non test (live) card. For a list of valid test cards, visit: https://stripe.com/docs/testing.", "source": {"id": "src_1K9F4hEcXtiJtvvhrUEwvCyi", "object": "source", "amount": null, "card": {"address_line1_check": null, "address_zip_check": null, "brand": "Visa", "country": "US", "cvc_check": "unchecked", "dynamic_last4": null, "exp_month": 9, "exp_year": 2028, "fingerprint": "Re3p4j8issXA77iI", "funding": "credit", "last4": "8097", "name": null, "three_d_secure": "optional", "tokenization_method": null}, "client_secret": "src_client_secret_b3v8YqNMLGykB120fqv2Tjhq", "created": 1640119003, "currency": null, "flow": "none", "livemode": false, "metadata": {}, "owner": {"address": null, "email": null, "name": null, "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "consumed", "type": "card", "usage": "reusable"}, "type": "card_error"}, "latest_charge": "ch_3K9F4mEcXtiJtvvh1kUzxjwN", "livemode": false, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": null, "payment_method_options": {"card": {"installments": null, "mandate_options": null, "network": null, "request_three_d_secure": "automatic"}}, "payment_method_types": ["card"], "processing": null, "receipt_email": null, "review": null, "setup_future_usage": null, "shipping": null, "source": null, "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "requires_payment_method", "transfer_data": null, "transfer_group": null}, "emitted_at": 1689691099580} -{"stream": "payouts", "data": {"id": "po_1KVQhfEcXtiJtvvhZlUkl08U", "object": "payout", "amount": 9164, "arrival_date": 1645488000, "automatic": true, "balance_transaction": "txn_1KVQhfEcXtiJtvvhF7ox3YEm", "created": 1645406919, "currency": "usd", "description": "STRIPE PAYOUT", "destination": "ba_1KUL7UEcXtiJtvvhAEUlStmv", "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "livemode": false, "metadata": {}, "method": "standard", "original_payout": null, "reconciliation_status": "completed", "reversed_by": null, "source_balance": null, "source_type": "card", "statement_descriptor": null, "status": "paid", "type": "bank_account"}, "emitted_at": 1689691101795} -{"stream": "payouts", "data": {"id": "po_1MXKoPEcXtiJtvvhwmqjvKoO", "object": "payout", "amount": 665880, "arrival_date": 1675382400, "automatic": false, "balance_transaction": "txn_1MXKoQEcXtiJtvvh65SHFZS6", "created": 1675413601, "currency": "usd", "description": "", "destination": "ba_1MSI1fEcXtiJtvvhPlqZqPlw", "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "livemode": false, "metadata": {}, "method": "standard", "original_payout": null, "reconciliation_status": "not_applicable", "reversed_by": null, "source_balance": null, "source_type": "card", "statement_descriptor": "airbyte.io", "status": "paid", "type": "bank_account"}, "emitted_at": 1689691102064} -{"stream": "payouts", "data": {"id": "po_1MWHzjEcXtiJtvvhIdUQHhLq", "object": "payout", "amount": 154900, "arrival_date": 1675123200, "automatic": false, "balance_transaction": "txn_1MWHzjEcXtiJtvvhtydemd2Y", "created": 1675164443, "currency": "usd", "description": "Test", "destination": "ba_1MSI1fEcXtiJtvvhPlqZqPlw", "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "livemode": false, "metadata": {}, "method": "standard", "original_payout": null, "reconciliation_status": "not_applicable", "reversed_by": null, "source_balance": "issuing", "source_type": "issuing", "statement_descriptor": "airbyte.io", "status": "paid", "type": "bank_account"}, "emitted_at": 1689691102066} -{"stream": "plans", "data": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "emitted_at": 1689691103696} -{"stream": "prices", "data": {"id": "price_1K9GbqEcXtiJtvvhJ3lZe4i5", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1640124902, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_KouQ5ez86yREmB", "recurring": null, "tax_behavior": "inclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 12600.0, "unit_amount_decimal": "12600"}, "emitted_at": 1690480900454} -{"stream": "prices", "data": {"id": "price_1MX364EcXtiJtvvh6jKcimNL", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1675345504, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NHcKselSHfKdfc", "recurring": null, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 1700.0, "unit_amount_decimal": "1700"}, "emitted_at": 1690480900634} -{"stream": "prices", "data": {"id": "price_1MX364EcXtiJtvvhE3WgTl4O", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1675345504, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NHcKselSHfKdfc", "recurring": null, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 2000.0, "unit_amount_decimal": "2000"}, "emitted_at": 1690480900634} -{"stream": "products", "data": {"id": "prod_KouQ5ez86yREmB", "object": "product", "active": true, "attributes": [], "created": 1640124902, "default_price": "price_1K9GbqEcXtiJtvvhJ3lZe4i5", "description": null, "images": [], "livemode": false, "metadata": {}, "name": "edgao-test-product", "package_dimensions": null, "shippable": null, "statement_descriptor": null, "tax_code": "txcd_10000000", "type": "service", "unit_label": null, "updated": 1675345058, "url": null}, "emitted_at": 1689684235151} -{"stream": "products", "data": {"id": "prod_NHcKselSHfKdfc", "object": "product", "active": true, "attributes": [], "created": 1675345504, "default_price": "price_1MX364EcXtiJtvvhE3WgTl4O", "description": "Test Product 1 description", "images": ["https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfdjBOT09UaHRiNVl2WmJ6clNYRUlmcFFD00cCBRNHnV"], "livemode": false, "metadata": {}, "name": "Test Product 1", "package_dimensions": null, "shippable": null, "statement_descriptor": null, "tax_code": "txcd_10301000", "type": "service", "unit_label": null, "updated": 1675345505, "url": null}, "emitted_at": 1689684235408} -{"stream": "products", "data": {"id": "prod_NCgx1XP2IFQyKF", "object": "product", "active": true, "attributes": [], "created": 1674209524, "default_price": null, "description": null, "images": [], "livemode": false, "metadata": {}, "name": "tu", "package_dimensions": null, "shippable": null, "statement_descriptor": null, "tax_code": "txcd_10000000", "type": "service", "unit_label": null, "updated": 1674209524, "url": null}, "emitted_at": 1689684235411} -{"stream": "promotion_codes", "data": {"id": "promo_1MVtmyEcXtiJtvvhkV5jPFPU", "object": "promotion_code", "active": true, "code": "g20", "coupon": {"id": "iJ6qlwM5", "object": "coupon", "amount_off": null, "created": 1674208993, "currency": null, "duration": "forever", "duration_in_months": null, "livemode": false, "max_redemptions": null, "metadata": {}, "name": "\u0415\u0443\u0456\u0435", "percent_off": 10.0, "redeem_by": null, "times_redeemed": 3, "valid": true}, "created": 1675071396, "customer": null, "expires_at": null, "livemode": false, "max_redemptions": null, "metadata": {}, "restrictions": {"first_time_transaction": false, "minimum_amount": null, "minimum_amount_currency": null}, "times_redeemed": 0}, "emitted_at": 1689691105334} -{"stream": "promotion_codes", "data": {"id": "promo_1MVtmkEcXtiJtvvht0RA3MKg", "object": "promotion_code", "active": true, "code": "FRIENDS20", "coupon": {"id": "iJ6qlwM5", "object": "coupon", "amount_off": null, "created": 1674208993, "currency": null, "duration": "forever", "duration_in_months": null, "livemode": false, "max_redemptions": null, "metadata": {}, "name": "\u0415\u0443\u0456\u0435", "percent_off": 10.0, "redeem_by": null, "times_redeemed": 3, "valid": true}, "created": 1675071382, "customer": null, "expires_at": null, "livemode": false, "max_redemptions": null, "metadata": {}, "restrictions": {"first_time_transaction": true, "minimum_amount": 10000, "minimum_amount_currency": "usd"}, "times_redeemed": 0}, "emitted_at": 1689691105335} -{"stream": "refunds", "data": {"id": "re_3MVuZyEcXtiJtvvh0A6rSbeJ", "object": "refund", "amount": 200000, "balance_transaction": "txn_3MVuZyEcXtiJtvvh0v0QyAMx", "charge": "ch_3MVuZyEcXtiJtvvh0tiVC7DI", "created": 1675074488, "currency": "usd", "metadata": {}, "payment_intent": "pi_3MVuZyEcXtiJtvvh07Ehi4cx", "reason": "fraudulent", "receipt_number": "3278-5368", "source_transfer_reversal": null, "status": "succeeded", "transfer_reversal": null}, "emitted_at": 1689691106971} -{"stream": "subscription_items", "data": {"id": "si_O2toUlN7ELjLcM", "object": "subscription_item", "billing_thresholds": null, "created": 1686250591, "metadata": {}, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "price": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "recurring": {"aggregate_usage": null, "interval": "month", "interval_count": 1, "trial_period_days": null, "usage_type": "licensed"}, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "recurring", "unit_amount": 600, "unit_amount_decimal": "600"}, "quantity": 1, "subscription": "sub_1NGo0YEcXtiJtvvh9rKhuT2H", "tax_rates": []}, "emitted_at": 1689691109493} -{"stream": "subscriptions", "data": {"id": "sub_1NGo0YEcXtiJtvvh9rKhuT2H", "object": "subscription", "application": null, "application_fee_percent": null, "automatic_tax": {"enabled": true}, "billing_cycle_anchor": 1686250590.0, "billing_thresholds": null, "cancel_at": null, "cancel_at_period_end": false, "canceled_at": null, "cancellation_details": {"comment": null, "feedback": null, "reason": null}, "collection_method": "charge_automatically", "created": 1686250590, "currency": "usd", "current_period_end": 1691520990.0, "current_period_start": 1688842590, "customer": "cus_O2topVBsfeTMXg", "days_until_due": null, "default_payment_method": "pm_1NGo0XEcXtiJtvvhZosRvz8G", "default_source": null, "default_tax_rates": [], "description": null, "discount": null, "ended_at": null, "items": {"object": "list", "data": [{"id": "si_O2toUlN7ELjLcM", "object": "subscription_item", "billing_thresholds": null, "created": 1686250591, "metadata": {}, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "price": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "recurring": {"aggregate_usage": null, "interval": "month", "interval_count": 1, "trial_period_days": null, "usage_type": "licensed"}, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "recurring", "unit_amount": 600, "unit_amount_decimal": "600"}, "quantity": 1, "subscription": "sub_1NGo0YEcXtiJtvvh9rKhuT2H", "tax_rates": []}], "has_more": false, "total_count": 1.0, "url": "/v1/subscription_items?subscription=sub_1NGo0YEcXtiJtvvh9rKhuT2H"}, "latest_invoice": "in_1NRgM1EcXtiJtvvhQT20qPcf", "livemode": false, "metadata": {}, "next_pending_invoice_item_invoice": null, "on_behalf_of": null, "pause_collection": null, "payment_settings": {"payment_method_options": null, "payment_method_types": null, "save_default_payment_method": "off"}, "pending_invoice_item_interval": null, "pending_setup_intent": null, "pending_update": null, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "quantity": 1, "schedule": null, "start_date": 1686250590, "status": "active", "test_clock": null, "transfer_data": null, "trial_end": null, "trial_settings": {"end_behavior": {"missing_payment_method": "create_invoice"}}, "trial_start": null}, "emitted_at": 1689691110777} -{"stream": "subscription_schedule", "data": {"id": "sub_sched_1NGRenEcXtiJtvvhmM4eGaJN", "object": "subscription_schedule", "application": null, "canceled_at": null, "completed_at": null, "created": 1686164673, "current_phase": {"end_date": 1717787073, "start_date": 1686164673}, "customer": "cus_NGoTFiJFVbSsvZ", "default_settings": {"application_fee_percent": null, "automatic_tax": {"enabled": false}, "billing_cycle_anchor": "automatic", "billing_thresholds": null, "collection_method": "charge_automatically", "default_payment_method": null, "default_source": null, "description": null, "invoice_settings": null, "on_behalf_of": null, "transfer_data": null}, "end_behavior": "cancel", "livemode": false, "metadata": {}, "phases": [{"add_invoice_items": [], "application_fee_percent": null, "automatic_tax": {"enabled": true}, "billing_cycle_anchor": null, "billing_thresholds": null, "collection_method": null, "coupon": null, "currency": "usd", "default_payment_method": null, "default_tax_rates": [], "description": null, "end_date": 1717787073, "invoice_settings": "{'days_until_due': None}", "items": [{"billing_thresholds": null, "metadata": {}, "plan": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "price": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "quantity": 1, "tax_rates": []}], "metadata": {}, "on_behalf_of": null, "proration_behavior": "create_prorations", "start_date": 1686164673, "transfer_data": null, "trial_end": null}, {"add_invoice_items": [], "application_fee_percent": null, "automatic_tax": {"enabled": true}, "billing_cycle_anchor": null, "billing_thresholds": null, "collection_method": null, "coupon": null, "currency": "usd", "default_payment_method": null, "default_tax_rates": [], "description": null, "end_date": 1749323073, "invoice_settings": "{'days_until_due': None}", "items": [{"billing_thresholds": null, "metadata": {}, "plan": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "price": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "quantity": 1, "tax_rates": []}], "metadata": {}, "on_behalf_of": null, "proration_behavior": "create_prorations", "start_date": 1717787073, "transfer_data": null, "trial_end": null}], "released_at": null, "released_subscription": null, "renewal_interval": null, "status": "active", "subscription": "sub_1NGReoEcXtiJtvvhq6goDeqt", "test_clock": null}, "emitted_at": 1689691112514} -{"stream": "setup_intents", "data": {"id": "seti_1KnfIjEcXtiJtvvhPw5znVKY", "object": "setup_intent", "application": null, "automatic_payment_methods": null, "cancellation_reason": null, "client_secret": "seti_1KnfIjEcXtiJtvvhPw5znVKY_secret_LUebPsqMz6AF4ivxIg4LMaAT0OdZF5L", "created": 1649752937, "customer": null, "description": null, "flow_directions": null, "last_setup_error": null, "latest_attempt": "setatt_1KnfIjEcXtiJtvvhqDfSlpM4", "livemode": false, "mandate": null, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": "pm_1KnfIj2eZvKYlo2CAlv2Vhqc", "payment_method_options": {"acss_debit": {"currency": "cad", "mandate_options": {"interval_description": "First day of every month", "payment_schedule": "interval", "transaction_type": "personal"}, "verification_method": "automatic"}}, "payment_method_types": ["acss_debit"], "single_use_mandate": null, "status": "succeeded", "usage": "off_session"}, "emitted_at": 1689691114375} -{"stream": "setup_intents", "data": {"id": "seti_1KnfIcEcXtiJtvvh61qlCaDf", "object": "setup_intent", "application": null, "automatic_payment_methods": null, "cancellation_reason": null, "client_secret": "seti_1KnfIcEcXtiJtvvh61qlCaDf_secret_LUebcbyw8V1e8Pxk3aAjzDXMOXdFMCe", "created": 1649752930, "customer": null, "description": null, "flow_directions": null, "last_setup_error": null, "latest_attempt": "setatt_1KnfIdEcXtiJtvvhpDrYVlRP", "livemode": false, "mandate": null, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": "pm_1KnfIc2eZvKYlo2Civ7snSPy", "payment_method_options": {"acss_debit": {"currency": "cad", "mandate_options": {"interval_description": "First day of every month", "payment_schedule": "interval", "transaction_type": "personal"}, "verification_method": "automatic"}}, "payment_method_types": ["acss_debit"], "single_use_mandate": null, "status": "succeeded", "usage": "off_session"}, "emitted_at": 1689691114377} -{"stream": "setup_intents", "data": {"id": "seti_1KnfIVEcXtiJtvvhWiIbMkpH", "object": "setup_intent", "application": null, "automatic_payment_methods": null, "cancellation_reason": null, "client_secret": "seti_1KnfIVEcXtiJtvvhWiIbMkpH_secret_LUebIUsiFnm75EzDUzf2RLhJ9WQ92Dp", "created": 1649752923, "customer": null, "description": null, "flow_directions": null, "last_setup_error": null, "latest_attempt": "setatt_1KnfIVEcXtiJtvvhqouWGuhD", "livemode": false, "mandate": null, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": "pm_1KnfIV2eZvKYlo2CaOLGBF00", "payment_method_options": {"acss_debit": {"currency": "cad", "mandate_options": {"interval_description": "First day of every month", "payment_schedule": "interval", "transaction_type": "personal"}, "verification_method": "automatic"}}, "payment_method_types": ["acss_debit"], "single_use_mandate": null, "status": "succeeded", "usage": "off_session"}, "emitted_at": 1689691114378} -{"stream": "shipping_rates", "data": {"id": "shr_1NXgplEcXtiJtvvhA1ntV782", "object": "shipping_rate", "active": true, "created": 1690274589, "delivery_estimate": "{'maximum': {'unit': 'business_day', 'value': 14}, 'minimum': {'unit': 'business_day', 'value': 10}}", "display_name": "Test Ground Shipping", "fixed_amount": {"amount": 999, "currency": "usd"}, "livemode": false, "metadata": {}, "tax_behavior": "inclusive", "tax_code": "txcd_92010001", "type": "fixed_amount"}, "emitted_at": 1690274706704} -{"stream": "credit_notes", "data": {"id": "cn_1NGPwmEcXtiJtvvhNXwHpgJF", "object": "credit_note", "amount": 8400, "amount_shipping": 0, "created": 1686158100, "currency": "usd", "customer": "cus_Kou8knsO3qQOwU", "customer_balance_transaction": null, "discount_amount": "0", "discount_amounts": [], "effective_at": 1686158100, "invoice": "in_1K9GK0EcXtiJtvvhSo2LvGqT", "lines": {"object": "list", "data": [{"id": "cnli_1NGPwmEcXtiJtvvhcL7yEIBJ", "object": "credit_note_line_item", "amount": 8400, "amount_excluding_tax": 8400, "description": "a box of parsnips", "discount_amount": 0, "discount_amounts": [], "invoice_line_item": "il_1K9GKLEcXtiJtvvhhHaYMebN", "livemode": false, "quantity": 1, "tax_amounts": [], "tax_rates": [], "type": "invoice_line_item", "unit_amount": 8400, "unit_amount_decimal": 8400.0, "unit_amount_excluding_tax": 8400.0}], "has_more": false, "url": "/v1/credit_notes/cn_1NGPwmEcXtiJtvvhNXwHpgJF/lines"}, "livemode": false, "memo": null, "metadata": {}, "number": "CA35DF83-0001-CN-01", "out_of_band_amount": null, "pdf": "https://pay.stripe.com/credit_notes/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9PMlV3dFlJelh4NHM1R0VIWnhMR3RjWUtlejFlRWtILDgwODIyOTk50200TyP0Z9BQ/pdf?s=ap", "reason": null, "refund": null, "shipping_cost": null, "status": "issued", "subtotal": 8400, "subtotal_excluding_tax": 8400, "tax_amounts": [], "total": 8400, "total_excluding_tax": 8400, "type": "pre_payment", "voided_at": null}, "emitted_at": 1690282199229} -{"stream": "top_ups", "data": {"id": "tu_1MXKmvEcXtiJtvvhAbDiH3sm", "object": "topup", "amount": 100000, "balance_transaction": null, "created": 1675413509, "currency": "usd", "description": "Test test", "destination_balance": null, "expected_availability_date": 1675413509, "failure_code": "R03", "failure_message": "no_account", "livemode": false, "metadata": {}, "source": {"id": "src_1MXKmvEcXtiJtvvhV0fY3ZBF", "object": "source", "ach_debit": {"bank_name": "STRIPE TEST BANK", "country": "US", "fingerprint": "KUgiD3MWWaMMufNe", "last4": "1116", "routing_number": "110000000", "type": "individual"}, "amount": null, "client_secret": "src_client_secret_qghd0H1WkqAuSDuJBrYBSfD6", "code_verification": {"attempts_remaining": 3, "status": "succeeded"}, "created": 1675413509, "currency": "usd", "flow": "code_verification", "livemode": false, "metadata": {}, "owner": {"address": {"city": null, "country": "US", "line1": null, "line2": null, "postal_code": null, "state": null}, "email": null, "name": "Jenny Rosen", "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "chargeable", "type": "ach_debit", "usage": "reusable"}, "statement_descriptor": "Test", "status": "failed", "transfer_group": null}, "emitted_at": 1689684238843} -{"stream": "files", "data": {"id": "file_1Jx65GEcXtiJtvvhxZSXTW0X", "object": "file", "created": 1637224506, "expires_at": null, "filename": "1200x1200 logo.png", "links": {"object": "list", "data": [{"id": "link_1Jx65KEcXtiJtvvhtzDKT46T", "object": "file_link", "created": 1637224510, "expired": 0, "expires_at": null, "file": "file_1Jx65GEcXtiJtvvhxZSXTW0X", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfS0VrR1lkejVjaTFlNmpuMUNHYmFkYmVT00UG0pCDcI"}, {"id": "link_1Jx65HEcXtiJtvvhJxpyHQyb", "object": "file_link", "created": 1637224507, "expired": 0, "expires_at": null, "file": "file_1Jx65GEcXtiJtvvhxZSXTW0X", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfdTlISjJ4UzM3QUlVa3M4cDdGRm9HWEUz00YHXUch3u"}], "has_more": false, "url": "/v1/file_links?file=file_1Jx65GEcXtiJtvvhxZSXTW0X"}, "purpose": "business_logo", "size": 188116, "title": null, "type": "png", "url": "https://files.stripe.com/v1/files/file_1Jx65GEcXtiJtvvhxZSXTW0X/contents"}, "emitted_at": 1689684229233} -{"stream": "file_links", "data": {"id": "link_1KnfIiEcXtiJtvvhCNceSyei", "object": "file_link", "created": 1649752936, "expired": false, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfY1FvanBFTmt0dUdrRWJXTHBpUlVYVUtu007305bsv3"}, "emitted_at": 1689684236617} -{"stream": "setup_attempts", "data": {"id": "setatt_1KnfIjEcXtiJtvvhqDfSlpM4", "object": "setup_attempt", "application": null, "created": 1649752937, "customer": null, "flow_directions": null, "livemode": false, "on_behalf_of": null, "payment_method": "pm_1KnfIj2eZvKYlo2CAlv2Vhqc", "payment_method_details": {"acss_debit": {}, "type": "acss_debit"}, "setup_error": null, "setup_intent": "seti_1KnfIjEcXtiJtvvhPw5znVKY", "status": "succeeded", "usage": "off_session"}, "emitted_at": 1689684241319} -{"stream": "setup_attempts", "data": {"id": "setatt_1KnfIdEcXtiJtvvhpDrYVlRP", "object": "setup_attempt", "application": null, "created": 1649752931, "customer": null, "flow_directions": null, "livemode": false, "on_behalf_of": null, "payment_method": "pm_1KnfIc2eZvKYlo2Civ7snSPy", "payment_method_details": {"acss_debit": {}, "type": "acss_debit"}, "setup_error": null, "setup_intent": "seti_1KnfIcEcXtiJtvvh61qlCaDf", "status": "succeeded", "usage": "off_session"}, "emitted_at": 1689684242319} -{"stream": "setup_attempts", "data": {"id": "setatt_1KnfIVEcXtiJtvvhqouWGuhD", "object": "setup_attempt", "application": null, "created": 1649752923, "customer": null, "flow_directions": null, "livemode": false, "on_behalf_of": null, "payment_method": "pm_1KnfIV2eZvKYlo2CaOLGBF00", "payment_method_details": {"acss_debit": {}, "type": "acss_debit"}, "setup_error": null, "setup_intent": "seti_1KnfIVEcXtiJtvvhWiIbMkpH", "status": "succeeded", "usage": "off_session"}, "emitted_at": 1689684243299} -{"stream": "usage_records", "data": {"id": "sis_1NVDGQEcXtiJtvvhMFxjewz0", "object": "usage_record_summary", "invoice": null, "livemode": false, "period": {"end": null, "start": null}, "subscription_item": "si_O2toUlN7ELjLcM", "total_usage": 1}, "emitted_at": 1689684267054} -{"stream": "usage_records", "data": {"id": "sis_1NVDGREcXtiJtvvhoIIYCISg", "object": "usage_record_summary", "invoice": null, "livemode": false, "period": {"end": null, "start": null}, "subscription_item": "si_O2WiEt5xsnTylQ", "total_usage": 1}, "emitted_at": 1689684267303} -{"stream": "usage_records", "data": {"id": "sis_1NVDGREcXtiJtvvhlkyGWg6j", "object": "usage_record_summary", "invoice": null, "livemode": false, "period": {"end": null, "start": null}, "subscription_item": "si_O2WdoFASN6MHzF", "total_usage": 1}, "emitted_at": 1689684267550} -{"stream": "usage_records", "data": {"id": "sis_1NVDGREcXtiJtvvhx53SMf1G", "object": "usage_record_summary", "invoice": null, "livemode": false, "period": {"end": null, "start": null}, "subscription_item": "si_O2WZCuG0IxqQHl", "total_usage": 1}, "emitted_at": 1689684267794} -{"stream": "transfer_reversals", "data": {"id": "trr_1NGolCEcXtiJtvvhOYPck3CP", "object": "transfer_reversal", "amount": 100, "balance_transaction": "txn_1NGolCEcXtiJtvvhZRy4Kd5S", "created": 1686253482, "currency": "usd", "destination_payment_refund": "pyr_1NGolBEYmRTj5on1STal3rmp", "metadata": {}, "source_refund": null, "transfer": "tr_1NGoaCEcXtiJtvvhjmHtOGOm"}, "emitted_at": 1689684270142} +{"stream": "checkout_sessions_line_items", "data": {"checkout_session_id": "cs_test_a1uSLwxkrTLjGhRXgzJweMwh09uvSZcWIkGLcIqDXzYADowSPwkAmJUrAN", "checkout_session_expires_at": 1697713523, "checkout_session_created": 1697627124, "checkout_session_updated": 1697627124, "id": "li_1O2XZ1EcXtiJtvvh26q22omU", "object": "item", "amount_discount": 0, "amount_subtotal": 3400, "amount_tax": 0, "amount_total": 3400, "currency": "usd", "description": "Test Product 1", "discounts": [], "price": {"id": "price_1MX364EcXtiJtvvh6jKcimNL", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1675345504, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NHcKselSHfKdfc", "recurring": null, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 1700, "unit_amount_decimal": "1700"}, "quantity": 2, "taxes": []}, "emitted_at": 1699376426293} +{"stream": "customer_balance_transactions", "data": {"id": "cbtxn_1MX2zPEcXtiJtvvhr4L2D3Q1", "object": "customer_balance_transaction", "amount": -50000.0, "created": 1675345091, "credit_note": null, "currency": "usd", "customer": "cus_NGoTFiJFVbSsvZ", "description": null, "ending_balance": 0.0, "invoice": "in_1MX2yFEcXtiJtvvhMXhUCgKx", "livemode": false, "metadata": {}, "type": "applied_to_invoice"}, "emitted_at": 1697627222916} +{"stream": "customer_balance_transactions", "data": {"id": "cbtxn_1MWIPLEcXtiJtvvhLnQYjVCj", "object": "customer_balance_transaction", "amount": 50000.0, "created": 1675166031, "credit_note": null, "currency": "usd", "customer": "cus_NGoTFiJFVbSsvZ", "description": "Test credit balance", "ending_balance": 50000.0, "invoice": null, "livemode": false, "metadata": {}, "type": "adjustment"}, "emitted_at": 1697627222918} +{"stream": "setup_attempts", "data": {"id": "setatt_1KnfIjEcXtiJtvvhqDfSlpM4", "object": "setup_attempt", "application": null, "created": 1649752937, "customer": null, "flow_directions": null, "livemode": false, "on_behalf_of": null, "payment_method": "pm_1KnfIj2eZvKYlo2CAlv2Vhqc", "payment_method_details": {"acss_debit": {}, "type": "acss_debit"}, "setup_error": null, "setup_intent": "seti_1KnfIjEcXtiJtvvhPw5znVKY", "status": "succeeded", "usage": "off_session"}, "emitted_at": 1697627241471} +{"stream": "setup_attempts", "data": {"id": "setatt_1KnfIdEcXtiJtvvhpDrYVlRP", "object": "setup_attempt", "application": null, "created": 1649752931, "customer": null, "flow_directions": null, "livemode": false, "on_behalf_of": null, "payment_method": "pm_1KnfIc2eZvKYlo2Civ7snSPy", "payment_method_details": {"acss_debit": {}, "type": "acss_debit"}, "setup_error": null, "setup_intent": "seti_1KnfIcEcXtiJtvvh61qlCaDf", "status": "succeeded", "usage": "off_session"}, "emitted_at": 1697627242509} +{"stream": "setup_attempts", "data": {"id": "setatt_1KnfIVEcXtiJtvvhqouWGuhD", "object": "setup_attempt", "application": null, "created": 1649752923, "customer": null, "flow_directions": null, "livemode": false, "on_behalf_of": null, "payment_method": "pm_1KnfIV2eZvKYlo2CaOLGBF00", "payment_method_details": {"acss_debit": {}, "type": "acss_debit"}, "setup_error": null, "setup_intent": "seti_1KnfIVEcXtiJtvvhWiIbMkpH", "status": "succeeded", "usage": "off_session"}, "emitted_at": 1697627243547} +{"stream": "accounts", "data": {"id": "acct_1NGp6SD04fX0Aizk", "object": "account", "capabilities": {"acss_debit_payments": "active", "affirm_payments": "active", "afterpay_clearpay_payments": "active", "bancontact_payments": "active", "card_payments": "active", "cartes_bancaires_payments": "pending", "cashapp_payments": "active", "eps_payments": "active", "giropay_payments": "active", "ideal_payments": "active", "klarna_payments": "active", "link_payments": "active", "p24_payments": "active", "sepa_debit_payments": "active", "sofort_payments": "active", "transfers": "active", "us_bank_account_ach_payments": "active"}, "charges_enabled": true, "country": "US", "default_currency": "usd", "details_submitted": true, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "payouts_enabled": true, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "settings": {"bacs_debit_payments": {"display_name": null, "service_user_number": null}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"statement_descriptor_prefix": "AIRBYTE", "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": "Airbyte", "timezone": "Asia/Tbilisi"}, "payments": {"statement_descriptor": "WWW.AIRBYTE.COM", "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "sepa_debit_payments": {}}, "type": "standard"}, "emitted_at": 1697627267880} +{"stream": "accounts", "data": {"id": "acct_1MwD6tIyVv44cUB4", "object": "account", "business_profile": {"mcc": null, "name": null, "product_description": null, "support_address": null, "support_email": null, "support_phone": null, "support_url": null, "url": null}, "business_type": null, "capabilities": {"card_payments": "inactive", "transfers": "inactive"}, "charges_enabled": false, "country": "US", "created": 1681342196, "default_currency": "usd", "details_submitted": false, "email": "jenny.rosen@example.com", "external_accounts": {"object": "list", "data": [], "has_more": false, "total_count": 0, "url": "/v1/accounts/acct_1MwD6tIyVv44cUB4/external_accounts"}, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "metadata": {}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": ["business_profile.mcc", "business_profile.url", "business_type", "external_account", "representative.first_name", "representative.last_name", "tos_acceptance.date", "tos_acceptance.ip"], "pending_verification": []}, "settings": {"bacs_debit_payments": {"display_name": null, "service_user_number": null}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"decline_on": {"avs_failure": false, "cvc_failure": false}, "statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "Etc/UTC"}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "payouts": {"debit_negative_balances": false, "schedule": {"delay_days": 2, "interval": "daily"}, "statement_descriptor": null}, "sepa_debit_payments": {}}, "tos_acceptance": {"date": null, "ip": null, "user_agent": null}, "type": "custom"}, "emitted_at": 1697627267882} +{"stream": "accounts", "data": {"id": "acct_1Jx8unEYmRTj5on1", "object": "account", "business_profile": {"mcc": null, "name": "Airbyte", "support_address": null, "support_email": null, "support_phone": null, "support_url": null, "url": null}, "capabilities": {}, "charges_enabled": false, "controller": {"type": "account"}, "country": "US", "default_currency": "usd", "details_submitted": false, "email": null, "future_requirements": {"alternatives": [], "current_deadline": null, "currently_due": [], "disabled_reason": null, "errors": [], "eventually_due": [], "past_due": [], "pending_verification": []}, "metadata": {}, "payouts_enabled": false, "requirements": {"alternatives": [], "current_deadline": null, "currently_due": ["business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "tos_acceptance.date", "tos_acceptance.ip"], "disabled_reason": "requirements.past_due", "errors": [], "eventually_due": ["business_profile.product_description", "business_profile.support_phone", "business_profile.url", "external_account", "tos_acceptance.date", "tos_acceptance.ip"], "past_due": [], "pending_verification": []}, "settings": {"bacs_debit_payments": {"display_name": null, "service_user_number": null}, "branding": {"icon": null, "logo": null, "primary_color": null, "secondary_color": null}, "card_issuing": {"tos_acceptance": {"date": null, "ip": null}}, "card_payments": {"statement_descriptor_prefix": null, "statement_descriptor_prefix_kana": null, "statement_descriptor_prefix_kanji": null}, "dashboard": {"display_name": null, "timezone": "Etc/UTC"}, "payments": {"statement_descriptor": null, "statement_descriptor_kana": null, "statement_descriptor_kanji": null}, "sepa_debit_payments": {}}, "type": "standard"}, "emitted_at": 1697627267884} +{"stream": "shipping_rates", "data": {"id": "shr_1NXgplEcXtiJtvvhA1ntV782", "object": "shipping_rate", "active": true, "created": 1690274589, "delivery_estimate": "{'maximum': {'unit': 'business_day', 'value': 14}, 'minimum': {'unit': 'business_day', 'value': 10}}", "display_name": "Test Ground Shipping", "fixed_amount": {"amount": 999, "currency": "usd"}, "livemode": false, "metadata": {}, "tax_behavior": "inclusive", "tax_code": "txcd_92010001", "type": "fixed_amount"}, "emitted_at": 1697627269309} +{"stream": "balance_transactions", "data": {"id": "txn_1KVQhfEcXtiJtvvhF7ox3YEm", "object": "balance_transaction", "amount": -9164, "available_on": 1645488000, "created": 1645406919, "currency": "usd", "description": "STRIPE PAYOUT", "exchange_rate": null, "fee": 0, "fee_details": [], "net": -9164, "reporting_category": "payout", "source": "po_1KVQhfEcXtiJtvvhZlUkl08U", "status": "available", "type": "payout"}, "emitted_at": 1697627270253} +{"stream": "balance_transactions", "data": {"id": "txn_3K9FSOEcXtiJtvvh0KoS5mx7", "object": "balance_transaction", "amount": 5300, "available_on": 1640649600, "created": 1640120473, "currency": "usd", "description": null, "exchange_rate": null, "fee": 184, "fee_details": [{"amount": 184, "application": null, "currency": "usd", "description": "Stripe processing fees", "type": "stripe_fee"}], "net": 5116, "reporting_category": "charge", "source": "ch_3K9FSOEcXtiJtvvh0zxb7clc", "status": "available", "type": "charge"}, "emitted_at": 1697627270254} +{"stream": "balance_transactions", "data": {"id": "txn_3K9F5DEcXtiJtvvh1qsqmHcH", "object": "balance_transaction", "amount": 4200, "available_on": 1640649600, "created": 1640119035, "currency": "usd", "description": "edgao test", "exchange_rate": null, "fee": 152, "fee_details": [{"amount": 152, "application": null, "currency": "usd", "description": "Stripe processing fees", "type": "stripe_fee"}], "net": 4048, "reporting_category": "charge", "source": "ch_3K9F5DEcXtiJtvvh1w2MaTpj", "status": "available", "type": "charge"}, "emitted_at": 1697627270255} +{"stream": "files", "data": {"id": "file_1Jx65GEcXtiJtvvhxZSXTW0X", "object": "file", "created": 1637224506, "expires_at": null, "filename": "1200x1200 logo.png", "links": {"object": "list", "data": [{"id": "link_1O0NBJEcXtiJtvvhv4Ul0hJ0", "object": "file_link", "created": 1697110557, "expired": 0, "expires_at": null, "file": "file_1Jx65GEcXtiJtvvhxZSXTW0X", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfRXpIVTlXU0JrQ2NlZFpLYTZJWHZuSFp400oVZPV9jH"}, {"id": "link_1O0NB1EcXtiJtvvhr93peBrc", "object": "file_link", "created": 1697110539, "expired": 0, "expires_at": null, "file": "file_1Jx65GEcXtiJtvvhxZSXTW0X", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfZm1pVk40RjVLVWNXd1hteXU0OHdGVUJD00yITeDHKr"}, {"id": "link_1O0LqGEcXtiJtvvhi1y00iXv", "object": "file_link", "created": 1697105408, "expired": 0, "expires_at": null, "file": "file_1Jx65GEcXtiJtvvhxZSXTW0X", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfM2lPVDFoRG1iZVNvNjNMVDdSVWVpU2gx00HCuZLBj2"}, {"id": "link_1Nmze7EcXtiJtvvhulvEwb5e", "object": "file_link", "created": 1693921823, "expired": 0, "expires_at": null, "file": "file_1Jx65GEcXtiJtvvhxZSXTW0X", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfN2pUcjAydzJtaDlGamh2OW1hUHRaOUUz00r6eeU87s"}, {"id": "link_1NkT9LEcXtiJtvvhjNhf6FnP", "object": "file_link", "created": 1693320251, "expired": 0, "expires_at": null, "file": "file_1Jx65GEcXtiJtvvhxZSXTW0X", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfRnBRVEJyZVNxVzRiVzdqcXZ1TkV4b0NZ00ZTSZxlGC"}, {"id": "link_1Jx65KEcXtiJtvvhtzDKT46T", "object": "file_link", "created": 1637224510, "expired": 0, "expires_at": null, "file": "file_1Jx65GEcXtiJtvvhxZSXTW0X", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfS0VrR1lkejVjaTFlNmpuMUNHYmFkYmVT00UG0pCDcI"}, {"id": "link_1Jx65HEcXtiJtvvhJxpyHQyb", "object": "file_link", "created": 1637224507, "expired": 0, "expires_at": null, "file": "file_1Jx65GEcXtiJtvvhxZSXTW0X", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfdTlISjJ4UzM3QUlVa3M4cDdGRm9HWEUz00YHXUch3u"}], "has_more": false, "url": "/v1/file_links?file=file_1Jx65GEcXtiJtvvhxZSXTW0X"}, "purpose": "business_logo", "size": 188116, "title": null, "type": "png", "url": "https://files.stripe.com/v1/files/file_1Jx65GEcXtiJtvvhxZSXTW0X/contents"}, "emitted_at": 1697627272312} +{"stream": "files", "data": {"id": "file_1Jx631EcXtiJtvvh9J1J59wL", "object": "file", "created": 1637224367, "expires_at": null, "filename": "1200x1200 logo.png", "links": {"object": "list", "data": [{"id": "link_1O0ipbEcXtiJtvvhMiXEug9P", "object": "file_link", "created": 1697193779, "expired": 0, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfT3BOYjRNbjk3N2FwaU02ZFlvRHlGR0Ju00ThNZnIQo"}, {"id": "link_1O0iodEcXtiJtvvhpRS0S9Bs", "object": "file_link", "created": 1697193719, "expired": 0, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfaGRwMTM4dElkaUZUSldvMVRJTjgyVHpK00vHpI07Tw"}, {"id": "link_1O0imYEcXtiJtvvhCP6HgZRW", "object": "file_link", "created": 1697193590, "expired": 0, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfaTU1bXVHRkJSQ2QxRGtkNmhaczFiRUk100jMApxFMO"}, {"id": "link_1O0ikjEcXtiJtvvhtSSgkYbl", "object": "file_link", "created": 1697193477, "expired": 0, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfRjZzTmE4RlAzTWN5d1ZPZVEzc0E3VEM200As4eLMkd"}, {"id": "link_1O0iefEcXtiJtvvhGTHAOpjl", "object": "file_link", "created": 1697193101, "expired": 0, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfU1F3ekdNRmx2NXN1YkhYQ0hPU0JpVzV6003Ir1ultx"}, {"id": "link_1O0iYDEcXtiJtvvhecp92xaX", "object": "file_link", "created": 1697192701, "expired": 0, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfQ3A5MXFZM3pDTUJsWVEyVzVYaEtkQ0k500Y5IH1OzK"}, {"id": "link_1O0iY8EcXtiJtvvhXo5URgJu", "object": "file_link", "created": 1697192696, "expired": 0, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfT3NsRGh1RXUyampaMVliNlFHdVNOZXZp00YyOeS579"}, {"id": "link_1O0iXyEcXtiJtvvhHv8T9K3D", "object": "file_link", "created": 1697192686, "expired": 0, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfSnJJU3dTSVJIcW8yS2lsamZJVXN2Sjh500so1uzdO7"}, {"id": "link_1O0iXgEcXtiJtvvhvPo3Q3Ma", "object": "file_link", "created": 1697192668, "expired": 0, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfS05nR1ZLNkJUejdDNkRadG1vWGFNTm5C00OVD5VBbc"}, {"id": "link_1NlT2qEcXtiJtvvhjgJaaRT9", "object": "file_link", "created": 1693558176, "expired": 0, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfSlhJOHZLQklSZEptZHNIaGF2MkdJUjVo00vt3TBEuO"}], "has_more": true, "url": "/v1/file_links?file=file_1Jx631EcXtiJtvvh9J1J59wL"}, "purpose": "business_logo", "size": 188116, "title": null, "type": "png", "url": "https://files.stripe.com/v1/files/file_1Jx631EcXtiJtvvh9J1J59wL/contents"}, "emitted_at": 1697627272315} +{"stream": "files", "data": {"id": "file_1MlOuuEcXtiJtvvhe4fgQC6E", "object": "file", "created": 1678765972, "expires_at": 1678848772, "filename": "Invoice-C09C1837-0001.pdf", "links": {"object": "list", "data": [], "has_more": false, "url": "/v1/file_links?file=file_1MlOuuEcXtiJtvvhe4fgQC6E"}, "purpose": "invoice_statement", "size": 24234, "title": null, "type": "pdf", "url": null}, "emitted_at": 1697627272604} +{"stream": "file_links", "data": {"id": "link_1KnfIiEcXtiJtvvhCNceSyei", "object": "file_link", "created": 1649752936, "expired": false, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfY1FvanBFTmt0dUdrRWJXTHBpUlVYVUtu007305bsv3"}, "emitted_at": 1697627273833} +{"stream": "file_links", "data": {"id": "link_1KnfIbEcXtiJtvvhyBLUqkSt", "object": "file_link", "created": 1649752929, "expired": false, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfaXh1blBqMmY0MzI3SHZWbUZIeFVGU3Nl0022JjupYq"}, "emitted_at": 1697627273834} +{"stream": "file_links", "data": {"id": "link_1KnfIUEcXtiJtvvh0ktKHfWz", "object": "file_link", "created": 1649752922, "expired": false, "expires_at": null, "file": "file_1Jx631EcXtiJtvvh9J1J59wL", "livemode": false, "metadata": {}, "url": "https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfNzhlbE9MUGNYbkJzMkRLSWdEcnhvY3FH00DK5jBVaH"}, "emitted_at": 1697627273835} +{"stream": "checkout_sessions", "data": {"id": "cs_test_a1uSLwxkrTLjGhRXgzJweMwh09uvSZcWIkGLcIqDXzYADowSPwkAmJUrAN", "object": "checkout.session", "after_expiration": null, "allow_promotion_codes": null, "amount_subtotal": 3400, "amount_total": 3400, "automatic_tax": {"enabled": false, "status": null}, "billing_address_collection": null, "cancel_url": null, "client_reference_id": null, "client_secret": null, "consent": null, "consent_collection": null, "created": 1697627124, "currency": "usd", "currency_conversion": null, "custom_fields": [], "custom_text": {"after_submit": null,"shipping_address": null, "submit": null, "terms_of_service_acceptance": null}, "customer": null, "customer_creation": "always", "customer_details": null, "customer_email": null, "expires_at": 1697713523, "invoice": null, "invoice_creation": {"enabled": false, "invoice_data": {"account_tax_ids": null, "custom_fields": null, "description": null, "footer": null, "metadata": {}, "rendering_options": null}}, "livemode": false, "locale": null, "metadata": {}, "mode": "payment", "payment_intent": "pi_3O2XZ1EcXtiJtvvh0zWGn33E", "payment_link": null, "payment_method_collection": "always", "payment_method_configuration_details": {"id": "pmc_1MC0oMEcXtiJtvvhmhbSUwTJ", "parent": null}, "payment_method_options": {"us_bank_account": {"financial_connections": {"permissions": ["payment_method"], "prefetch": []}, "verification_method": "automatic"}, "wechat_pay": {"app_id": null, "client": "web"}}, "payment_method_types": ["card", "alipay", "klarna", "link", "us_bank_account", "wechat_pay", "cashapp"], "payment_status": "unpaid", "phone_number_collection": {"enabled": false}, "recovered_from": null, "setup_intent": null, "shipping_address_collection": null, "shipping_cost": null, "shipping_details": null, "shipping_options": [], "status": "expired", "submit_type": null, "subscription": null, "success_url": "https://example.com/success", "total_details": {"amount_discount": 0, "amount_shipping": 0, "amount_tax": 0}, "ui_mode": "hosted", "url": null, "updated": 1697627124}, "emitted_at": 1697627275062} +{"stream": "credit_notes", "data": {"id": "cn_1NGPwmEcXtiJtvvhNXwHpgJF", "object": "credit_note", "amount": 8400, "amount_shipping": 0, "created": 1686158100, "currency": "usd", "customer": "cus_Kou8knsO3qQOwU", "customer_balance_transaction": null, "discount_amount": "0", "discount_amounts": [], "effective_at": 1686158100, "invoice": "in_1K9GK0EcXtiJtvvhSo2LvGqT", "lines": {"object": "list", "data": [{"id": "cnli_1NGPwmEcXtiJtvvhcL7yEIBJ", "object": "credit_note_line_item", "amount": 8400, "amount_excluding_tax": 8400, "description": "a box of parsnips", "discount_amount": 0, "discount_amounts": [], "invoice_line_item": "il_1K9GKLEcXtiJtvvhhHaYMebN", "livemode": false, "quantity": 1, "tax_amounts": [], "tax_rates": [], "type": "invoice_line_item", "unit_amount": 8400, "unit_amount_decimal": 8400.0, "unit_amount_excluding_tax": 8400.0}], "has_more": false, "url": "/v1/credit_notes/cn_1NGPwmEcXtiJtvvhNXwHpgJF/lines"}, "livemode": false, "memo": null, "metadata": {}, "number": "CA35DF83-0001-CN-01", "out_of_band_amount": null, "pdf": "https://pay.stripe.com/credit_notes/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9PMlV3dFlJelh4NHM1R0VIWnhMR3RjWUtlejFlRWtILDg4MTY4MDc20200Sa50llWu/pdf?s=ap", "reason": null, "refund": null, "shipping_cost": null, "status": "issued", "subtotal": 8400, "subtotal_excluding_tax": 8400, "tax_amounts": [], "total": 8400, "total_excluding_tax": 8400, "type": "pre_payment", "voided_at": null, "updated": 1686158100}, "emitted_at": 1697627276386} +{"stream": "customers", "data": {"id": "cus_LIiHR6omh14Xdg", "object": "customer", "address": {"city": "san francisco", "country": "US", "line1": "san francisco", "line2": "", "postal_code": "", "state": "CA"}, "balance": 0, "created": 1646998902, "currency": "usd", "default_source": "card_1MSHU1EcXtiJtvvhytSN6V54", "delinquent": false, "description": "test", "discount": null, "email": "test@airbyte_integration_test.com", "invoice_prefix": "09A6A98F", "invoice_settings": {"custom_fields": null, "default_payment_method": null, "footer": null, "rendering_options": null}, "livemode": false, "metadata": {}, "name": "Test", "next_invoice_sequence": 1, "phone": null, "preferred_locales": [], "shipping": {"address": {"city": "", "country": "US", "line1": "", "line2": "", "postal_code": "", "state": ""}, "name": "", "phone": ""}, "tax_exempt": "none", "test_clock": null, "updated": 1646998902}, "emitted_at": 1697627278433} +{"stream": "customers", "data": {"id": "cus_Kou8knsO3qQOwU", "object": "customer", "address": null, "balance": 0, "created": 1640123795, "currency": "usd", "default_source": "src_1MSID8EcXtiJtvvhxIT9lXRy", "delinquent": false, "description": null, "discount": null, "email": "edward.gao+stripe-test-customer-1@airbyte.io", "invoice_prefix": "CA35DF83", "invoice_settings": {"custom_fields": null, "default_payment_method": null, "footer": null, "rendering_options": null}, "livemode": false, "metadata": {}, "name": "edgao-test-customer-1", "next_invoice_sequence": 2, "phone": null, "preferred_locales": [], "shipping": null, "tax_exempt": "none", "test_clock": null, "updated": 1640123795}, "emitted_at": 1697627278435} +{"stream": "customers", "data": {"id": "cus_NGoTFiJFVbSsvZ", "object": "customer", "address": {"city": "", "country": "US", "line1": "Street 2, 34567", "line2": "", "postal_code": "94114", "state": "CA"}, "balance": 0, "created": 1675160053, "currency": "usd", "default_source": "src_1MWGs8EcXtiJtvvh4nYdQvEr", "delinquent": false, "description": "Test Customer 2 description", "discount": null, "email": "user1.sample@zohomail.eu", "invoice_prefix": "C09C1837", "invoice_settings": {"custom_fields": null, "default_payment_method": null, "footer": null, "rendering_options": null}, "livemode": false, "metadata": {}, "name": "Test Customer 2", "next_invoice_sequence": 15, "phone": null, "preferred_locales": ["en-US"], "shipping": {"address": {"city": "", "country": "US", "line1": "Street 2, 34567", "line2": "", "postal_code": "94114", "state": "CA"}, "name": "Test Customer 2", "phone": ""}, "tax_exempt": "none", "test_clock": null, "updated": 1675160053}, "emitted_at": 1697627278439} +{"stream": "cardholders", "data": {"id": "ich_1KUKBeEcXtiJtvvhCEFgko6h", "object": "issuing.cardholder", "billing": {"address": {"city": "San Francisco", "country": "US", "line1": "1234 Main Street", "line2": null, "postal_code": "94111", "state": "CA"}}, "company": null, "created": 1645143542, "email": "jenny.rosen@example.com", "individual": null, "livemode": false, "metadata": {}, "name": "Jenny Rosen", "phone_number": "+18888675309", "preferred_locales": [], "requirements": {"disabled_reason": null, "past_due": []}, "spending_controls": {"allowed_categories": [], "blocked_categories": [], "spending_limits": [], "spending_limits_currency": null}, "status": "active", "type": "individual", "updated": 1645143542}, "emitted_at": 1697627292209} +{"stream": "charges", "data": {"id": "ch_3K9FSOEcXtiJtvvh0zxb7clc", "object": "charge", "amount": 5300, "amount_captured": 5300, "amount_refunded": 0, "amount_updates": [], "application": null, "application_fee": null, "application_fee_amount": null, "balance_transaction": "txn_3K9FSOEcXtiJtvvh0KoS5mx7", "billing_details": {"address": {"city": null, "country": null, "line1": null, "line2": null, "postal_code": null, "state": null}, "email": null, "name": null, "phone": null}, "calculated_statement_descriptor": "AIRBYTE.IO", "captured": true, "created": 1640120473, "currency": "usd", "customer": null, "description": null, "destination": null, "dispute": null, "disputed": false, "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "fraud_details": {}, "invoice": null, "livemode": false, "metadata": {}, "on_behalf_of": null, "order": null, "outcome": {"network_status": "approved_by_network", "reason": null, "risk_level": "normal", "risk_score": 48, "seller_message": "Payment complete.", "type": "authorized"}, "paid": true, "payment_intent": "pi_3K9FSOEcXtiJtvvh0AEIFllC", "payment_method": "src_1K9FSOEcXtiJtvvhHGu1qtOx", "payment_method_details": {"card": {"amount_authorized": 5300, "brand": "visa", "checks": {"address_line1_check": null, "address_postal_code_check": null, "cvc_check": "pass"}, "country": "US", "exp_month": 12, "exp_year": 2034, "extended_authorization": {"status": "disabled"}, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "incremental_authorization": {"status": "unavailable"}, "installments": null, "last4": "4242", "mandate": null, "multicapture": {"status": "unavailable"}, "network": "visa", "network_token": {"used": false}, "overcapture": {"maximum_amount_capturable": 5300, "status": "unavailable"}, "three_d_secure": null, "wallet": null}, "type": "card"}, "receipt_email": null, "receipt_number": "1509-9197", "receipt_url": "https://pay.stripe.com/receipts/payment/CAcaFwoVYWNjdF8xSndub2lFY1h0aUp0dnZoKJ35vqkGMgYYlboX7Hs6LBbBoR6yFToo5WeMCCwbkvCz7nl3E1KToovFFZKMJYnrpAHBlWJrVMJK6BWm", "refunded": false, "refunds": {"object": "list", "data": [], "has_more": false, "total_count": 0.0, "url": "/v1/charges/ch_3K9FSOEcXtiJtvvh0zxb7clc/refunds"}, "review": null, "shipping": null, "source": {"id": "src_1K9FSOEcXtiJtvvhHGu1qtOx", "object": "source", "amount": null, "card": {"address_line1_check": null, "address_zip_check": null, "brand": "Visa", "country": "US", "cvc_check": "pass", "dynamic_last4": null, "exp_month": 12, "exp_year": 2034, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "last4": "4242", "name": null, "three_d_secure": "optional", "tokenization_method": null}, "client_secret": "src_client_secret_3WszbFGtWT8vmMjqnNztOwhU", "created": 1640120473, "currency": null, "flow": "none", "livemode": false, "metadata": {}, "owner": {"address": null, "email": null, "name": null, "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "consumed", "type": "card", "usage": "reusable"}, "source_transfer": null, "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "succeeded", "transfer_data": null, "transfer_group": null, "updated": 1640120473}, "emitted_at": 1697627293840} +{"stream": "charges", "data": {"id": "ch_3K9F5DEcXtiJtvvh1w2MaTpj", "object": "charge", "amount": 4200, "amount_captured": 4200, "amount_refunded": 0, "amount_updates": [], "application": null, "application_fee": null, "application_fee_amount": null, "balance_transaction": "txn_3K9F5DEcXtiJtvvh1qsqmHcH", "billing_details": {"address": {"city": null, "country": null, "line1": null, "line2": null, "postal_code": null, "state": null}, "email": null, "name": null, "phone": null}, "calculated_statement_descriptor": "AIRBYTE.IO", "captured": true, "created": 1640119035, "currency": "usd", "customer": null, "description": "edgao test", "destination": null, "dispute": null, "disputed": false, "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "fraud_details": {}, "invoice": null, "livemode": false, "metadata": {}, "on_behalf_of": null, "order": null, "outcome": {"network_status": "approved_by_network", "reason": null, "risk_level": "normal", "risk_score": 63, "seller_message": "Payment complete.", "type": "authorized"}, "paid": true, "payment_intent": "pi_3K9F5DEcXtiJtvvh16scJMp6", "payment_method": "src_1K9F5CEcXtiJtvvhrsZdur8Y", "payment_method_details": {"card": {"amount_authorized": 4200, "brand": "visa", "checks": {"address_line1_check": null, "address_postal_code_check": null, "cvc_check": "pass"}, "country": "US", "exp_month": 9, "exp_year": 2028, "extended_authorization": {"status": "disabled"}, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "incremental_authorization": {"status": "unavailable"}, "installments": null, "last4": "4242", "mandate": null, "multicapture": {"status": "unavailable"}, "network": "visa", "network_token": {"used": false}, "overcapture": {"maximum_amount_capturable": 4200, "status": "unavailable"}, "three_d_secure": null, "wallet": null}, "type": "card"}, "receipt_email": null, "receipt_number": "1549-5630", "receipt_url": "https://pay.stripe.com/receipts/payment/CAcaFwoVYWNjdF8xSndub2lFY1h0aUp0dnZoKJ35vqkGMgbg2Y1Ao1M6LBYViHyCHYtYZtCIzc8I1Pm_oXAcXtgPDTNCfzyB3XOfFO4N-RK2w9sLuPjq", "refunded": false, "refunds": {"object": "list", "data": [], "has_more": false, "total_count": 0.0, "url": "/v1/charges/ch_3K9F5DEcXtiJtvvh1w2MaTpj/refunds"}, "review": null, "shipping": null, "source": {"id": "src_1K9F5CEcXtiJtvvhrsZdur8Y", "object": "source", "amount": null, "card": {"address_line1_check": null, "address_zip_check": null, "brand": "Visa", "country": "US", "cvc_check": "pass", "dynamic_last4": null, "exp_month": 9, "exp_year": 2028, "fingerprint": "X7e9fFB0r8MMcdo6", "funding": "credit", "last4": "4242", "name": null, "three_d_secure": "optional", "tokenization_method": null}, "client_secret": "src_client_secret_QyH8xuqSyiZh8oxzzIszqQ92", "created": 1640119035, "currency": null, "flow": "none", "livemode": false, "metadata": {}, "owner": {"address": null, "email": null, "name": null, "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "consumed", "type": "card", "usage": "reusable"}, "source_transfer": null, "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "succeeded", "transfer_data": null, "transfer_group": null, "updated": 1640119035}, "emitted_at": 1697627293843} +{"stream": "charges", "data": {"id": "ch_3K9F4mEcXtiJtvvh1kUzxjwN", "object": "charge", "amount": 4200, "amount_captured": 0, "amount_refunded": 0, "amount_updates": [], "application": null, "application_fee": null, "application_fee_amount": null, "balance_transaction": null, "billing_details": {"address": {"city": null, "country": null, "line1": null, "line2": null, "postal_code": null, "state": null}, "email": null, "name": null, "phone": null}, "calculated_statement_descriptor": "AIRBYTE.IO", "captured": false, "created": 1640119009, "currency": "usd", "customer": null, "description": "edgao test", "destination": null, "dispute": null, "disputed": false, "failure_balance_transaction": null, "failure_code": "card_declined", "failure_message": "Your card was declined. Your request was in test mode, but used a non test (live) card. For a list of valid test cards, visit: https://stripe.com/docs/testing.", "fraud_details": {}, "invoice": null, "livemode": false, "metadata": {}, "on_behalf_of": null, "order": null, "outcome": {"network_status": "not_sent_to_network", "reason": "test_mode_live_card", "risk_level": "normal", "risk_score": 6, "seller_message": "This charge request was in test mode, but did not use a Stripe test card number. For the list of these numbers, see stripe.com/docs/testing", "type": "invalid"}, "paid": false, "payment_intent": "pi_3K9F4mEcXtiJtvvh18NKhEuo", "payment_method": "src_1K9F4hEcXtiJtvvhrUEwvCyi", "payment_method_details": {"card": {"amount_authorized": null, "brand": "visa", "checks": {"address_line1_check": null, "address_postal_code_check": null, "cvc_check": null}, "country": "US", "exp_month": 9, "exp_year": 2028, "extended_authorization": {"status": "disabled"}, "fingerprint": "Re3p4j8issXA77iI", "funding": "credit", "incremental_authorization": {"status": "unavailable"}, "installments": null, "last4": "8097", "mandate": null, "multicapture": {"status": "unavailable"}, "network": "visa", "network_token": {"used": false}, "overcapture": {"maximum_amount_capturable": 4200, "status": "unavailable"}, "three_d_secure": null, "wallet": null}, "type": "card"}, "receipt_email": null, "receipt_number": null, "receipt_url": null, "refunded": false, "refunds": {"object": "list", "data": [], "has_more": false, "total_count": 0.0, "url": "/v1/charges/ch_3K9F4mEcXtiJtvvh1kUzxjwN/refunds"}, "review": null, "shipping": null, "source": {"id": "src_1K9F4hEcXtiJtvvhrUEwvCyi", "object": "source", "amount": null, "card": {"address_line1_check": null, "address_zip_check": null, "brand": "Visa", "country": "US", "cvc_check": null, "dynamic_last4": null, "exp_month": 9, "exp_year": 2028, "fingerprint": "Re3p4j8issXA77iI", "funding": "credit", "last4": "8097", "name": null, "three_d_secure": "optional", "tokenization_method": null}, "client_secret": "src_client_secret_b3v8YqNMLGykB120fqv2Tjhq", "created": 1640119009, "currency": null, "flow": "none", "livemode": false, "metadata": {}, "owner": {"address": null, "email": null, "name": null, "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "consumed", "type": "card", "usage": "reusable"}, "source_transfer": null, "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "failed", "transfer_data": null, "transfer_group": null, "updated": 1640119009}, "emitted_at": 1697627293846} +{"stream": "coupons", "data": {"id": "Coupon000001", "object": "coupon", "amount_off": 500, "created": 1675345584, "currency": "usd", "duration": "once", "duration_in_months": null, "livemode": false, "max_redemptions": null, "metadata": {}, "name": "Test Coupon 1", "percent_off": null, "redeem_by": null, "times_redeemed": 1, "valid": true, "updated": 1675345584}, "emitted_at": 1697627296924} +{"stream": "coupons", "data": {"id": "4SUEGKZg", "object": "coupon", "amount_off": 200, "created": 1674209030, "currency": "usd", "duration": "repeating", "duration_in_months": 3, "livemode": false, "max_redemptions": null, "metadata": {}, "name": "\u0406\u0435\u043a\u0448\u0437\u0443", "percent_off": null, "redeem_by": null, "times_redeemed": 0, "valid": true, "updated": 1674209030}, "emitted_at": 1697627296927} +{"stream": "coupons", "data": {"id": "iJ6qlwM5", "object": "coupon", "amount_off": null, "created": 1674208993, "currency": null, "duration": "forever", "duration_in_months": null, "livemode": false, "max_redemptions": null, "metadata": {}, "name": "\u0415\u0443\u0456\u0435", "percent_off": 10.0, "redeem_by": null, "times_redeemed": 3, "valid": true, "updated": 1674208993}, "emitted_at": 1697627296929} +{"stream": "disputes", "data": {"id": "dp_1MSI78EcXtiJtvvhxC77m2kh", "object": "dispute", "amount": 700, "balance_transaction": "txn_1MSI78EcXtiJtvvhAGjxP1UM", "balance_transactions": [{"id": "txn_1MSI78EcXtiJtvvhAGjxP1UM", "object": "balance_transaction", "amount": -700, "available_on": 1674518400, "created": 1674211590, "currency": "usd", "description": "Chargeback withdrawal for ch_3MSI77EcXtiJtvvh1GzoukUC", "exchange_rate": null, "fee": 1500, "fee_details": [{"amount": 1500, "application": null, "currency": "usd", "description": "Dispute fee", "type": "stripe_fee"}], "net": -2200, "reporting_category": "dispute", "source": "dp_1MSI78EcXtiJtvvhxC77m2kh", "status": "available", "type": "adjustment"}], "charge": "ch_3MSI77EcXtiJtvvh1GzoukUC", "created": 1674211590, "currency": "usd", "evidence": {"access_activity_log": null, "billing_address": "12345", "cancellation_policy": null, "cancellation_policy_disclosure": null, "cancellation_rebuttal": null, "customer_communication": null, "customer_email_address": null, "customer_name": null, "customer_purchase_ip": null, "customer_signature": null, "duplicate_charge_documentation": null, "duplicate_charge_explanation": null, "duplicate_charge_id": null, "product_description": null, "receipt": null, "refund_policy": null, "refund_policy_disclosure": null, "refund_refusal_explanation": null, "service_date": null, "service_documentation": null, "shipping_address": null, "shipping_carrier": null, "shipping_date": null, "shipping_documentation": null, "shipping_tracking_number": null, "uncategorized_file": null, "uncategorized_text": null}, "evidence_details": {"due_by": 1675036799.0, "has_evidence": false, "past_due": false, "submission_count": 0}, "is_charge_refundable": false, "livemode": false, "metadata": {}, "payment_intent": "pi_3MSI77EcXtiJtvvh1glmQd8s", "payment_method_details": {"card": {"brand": "visa", "network_reason_code": "83"}, "type": "card"}, "reason": "fraudulent", "status": "lost", "updated": 1674211590}, "emitted_at": 1697627298351} +{"stream":"invoices","data":{"id":"in_1K9GK0EcXtiJtvvhSo2LvGqT","object":"invoice","account_country":"US","account_name":"Airbyte, Inc.","account_tax_ids":null,"amount_due":0,"amount_paid":0,"amount_remaining":0,"amount_shipping":0,"application":null,"application_fee_amount":null,"attempt_count":0,"attempted":true,"auto_advance":false,"automatic_tax":{"enabled":false,"liability":null,"status":null},"billing_reason":"manual","charge":null,"collection_method":"send_invoice","created":1640123796,"currency":"usd","custom_fields":null,"customer":"cus_Kou8knsO3qQOwU","customer_address":null,"customer_email":"edward.gao+stripe-test-customer-1@airbyte.io","customer_name":"edgao-test-customer-1","customer_phone":null,"customer_shipping":null,"customer_tax_exempt":"none","customer_tax_ids":[],"default_payment_method":null,"default_source":null,"default_tax_rates":[],"description":null,"discount":null,"discounts":[],"due_date":1688750070,"effective_at":1686158070,"ending_balance":0,"footer":null,"from_invoice":null,"hosted_invoice_url":"https://invoice.stripe.com/i/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9Lb3U4bk9YR0lWV3BhN2EzMXZNUFJSaEdXUUVNR1J0LDk2MTc3MTMw0200ygIKWRVO?s=ap","invoice_pdf":"https://pay.stripe.com/invoice/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9Lb3U4bk9YR0lWV3BhN2EzMXZNUFJSaEdXUUVNR1J0LDk2MTc3MTMw0200ygIKWRVO/pdf?s=ap","issuer":{"type":"self"},"last_finalization_error":null,"latest_revision":null,"lines":{"object":"list","data":[{"id":"il_1K9GKLEcXtiJtvvhhHaYMebN","object":"line_item","amount":8400,"amount_excluding_tax":8400,"currency":"usd","description":"a box of parsnips","discount_amounts":[],"discountable":true,"discounts":[],"invoice_item":"ii_1K9GKLEcXtiJtvvhmr2AYOAx","livemode":false,"metadata":{},"period":{"end":1640123817,"start":1640123817},"plan":null,"price":{"id":"price_1K9GKLEcXtiJtvvhXbrg33lq","object":"price","active":false,"billing_scheme":"per_unit","created":1640123817,"currency":"usd","custom_unit_amount":null,"livemode":false,"lookup_key":null,"metadata":{},"nickname":null,"product":"prod_Kou8cQxtIpF1p7","recurring":null,"tax_behavior":"unspecified","tiers_mode":null,"transform_quantity":null,"type":"one_time","unit_amount":8400,"unit_amount_decimal":"8400"},"proration":false,"proration_details":{"credited_items":null},"quantity":1,"subscription":null,"tax_amounts":[],"tax_rates":[],"type":"invoiceitem","unit_amount_excluding_tax":"8400"}],"has_more":false,"total_count":1,"url":"/v1/invoices/in_1K9GK0EcXtiJtvvhSo2LvGqT/lines"},"livemode":false,"metadata":{},"next_payment_attempt":null,"number":"CA35DF83-0001","on_behalf_of":null,"paid":true,"paid_out_of_band":false,"payment_intent":null,"payment_settings":{"default_mandate":null,"payment_method_options":null,"payment_method_types":null},"period_end":1640123795,"period_start":1640123795,"post_payment_credit_notes_amount":0,"pre_payment_credit_notes_amount":8400,"quote":null,"receipt_number":null,"rendering":null,"rendering_options":null,"shipping_cost":null,"shipping_details":null,"starting_balance":0,"statement_descriptor":null,"status":"paid","status_transitions":{"finalized_at":1686158070,"marked_uncollectible_at":null,"paid_at":1686158100,"voided_at":null},"subscription":null,"subscription_details":{"metadata":null},"subtotal":8400,"subtotal_excluding_tax":8400,"tax":null,"test_clock":null,"total":8400,"total_discount_amounts":[],"total_excluding_tax":8400,"total_tax_amounts":[],"transfer_data":null,"webhooks_delivered_at":1640123796,"updated":1640123796},"emitted_at":1705636368179} +{"stream":"invoices","data":{"id":"in_1MX37hEcXtiJtvvhRSl1KbQm","object":"invoice","account_country":"US","account_name":"Airbyte, Inc.","account_tax_ids":null,"amount_due":5500,"amount_paid":5500,"amount_remaining":0,"amount_shipping":0,"application":null,"application_fee_amount":null,"attempt_count":1,"attempted":true,"auto_advance":false,"automatic_tax":{"enabled":true,"liability":{"type":"self"},"status":"complete"},"billing_reason":"manual","charge":"ch_3MX38QEcXtiJtvvh1y8YAJpg","collection_method":"send_invoice","created":1675345605,"currency":"usd","custom_fields":null,"customer":"cus_NGoTFiJFVbSsvZ","customer_address":{"city":"","country":"US","line1":"Street 2, 34567","line2":"","postal_code":"94114","state":"CA"},"customer_email":"user1.sample@zohomail.eu","customer_name":"Test Customer 2","customer_phone":null,"customer_shipping":{"address":{"city":"","country":"US","line1":"Street 2, 34567","line2":"","postal_code":"94114","state":"CA"},"name":"Test Customer 2","phone":""},"customer_tax_exempt":"none","customer_tax_ids":[],"default_payment_method":null,"default_source":null,"default_tax_rates":[],"description":"Thanks for your business!","discount":null,"discounts":[],"due_date":1677937605,"effective_at":1675345650,"ending_balance":0,"footer":"Test Invoice","from_invoice":null,"hosted_invoice_url":"https://invoice.stripe.com/i/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9OSGNNamM2RTF0TmlLcGFRRVFKeHpPdTgzOFFOVzNDLDk2MTc3MTY20200GeEOAZdh?s=ap","invoice_pdf":"https://pay.stripe.com/invoice/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9OSGNNamM2RTF0TmlLcGFRRVFKeHpPdTgzOFFOVzNDLDk2MTc3MTY20200GeEOAZdh/pdf?s=ap","issuer":{"type":"self"},"last_finalization_error":null,"latest_revision":null,"lines":{"object":"list","data":[{"id":"il_1MX384EcXtiJtvvh3j2K123f","object":"line_item","amount":6000,"amount_excluding_tax":6000,"currency":"usd","description":"Test Product 1","discount_amounts":[{"amount":500,"discount":"di_1MX384EcXtiJtvvhkOrY57Ep"}],"discountable":true,"discounts":["di_1MX384EcXtiJtvvhkOrY57Ep"],"invoice_item":"ii_1MX384EcXtiJtvvhguyn3iYb","livemode":false,"metadata":{},"period":{"end":1675345628,"start":1675345628},"plan":null,"price":{"id":"price_1MX364EcXtiJtvvhE3WgTl4O","object":"price","active":true,"billing_scheme":"per_unit","created":1675345504,"currency":"usd","custom_unit_amount":null,"livemode":false,"lookup_key":null,"metadata":{},"nickname":null,"product":"prod_NHcKselSHfKdfc","recurring":null,"tax_behavior":"exclusive","tiers_mode":null,"transform_quantity":null,"type":"one_time","unit_amount":2000,"unit_amount_decimal":"2000"},"proration":false,"proration_details":{"credited_items":null},"quantity":3,"subscription":null,"tax_amounts":[{"amount":0,"inclusive":false,"tax_rate":"txr_1MX384EcXtiJtvvhAhVE20Ii","taxability_reason":"not_collecting","taxable_amount":0}],"tax_rates":[],"type":"invoiceitem","unit_amount_excluding_tax":"2000"}],"has_more":false,"total_count":1,"url":"/v1/invoices/in_1MX37hEcXtiJtvvhRSl1KbQm/lines"},"livemode":false,"metadata":{},"next_payment_attempt":null,"number":"C09C1837-0002","on_behalf_of":null,"paid":true,"paid_out_of_band":false,"payment_intent":"pi_3MX38QEcXtiJtvvh10zsQJTC","payment_settings":{"default_mandate":null,"payment_method_options":null,"payment_method_types":null},"period_end":1675345605,"period_start":1675345605,"post_payment_credit_notes_amount":0,"pre_payment_credit_notes_amount":0,"quote":null,"receipt_number":null,"rendering":null,"rendering_options":null,"shipping_cost":null,"shipping_details":null,"starting_balance":0,"statement_descriptor":null,"status":"paid","status_transitions":{"finalized_at":1675345650,"marked_uncollectible_at":null,"paid_at":1675345673,"voided_at":null},"subscription":null,"subscription_details":{"metadata":null},"subtotal":5500,"subtotal_excluding_tax":5500,"tax":0,"test_clock":null,"total":5500,"total_discount_amounts":[{"amount":500,"discount":"di_1MX384EcXtiJtvvhkOrY57Ep"}],"total_excluding_tax":5500,"total_tax_amounts":[{"amount":0,"inclusive":false,"tax_rate":"txr_1MX384EcXtiJtvvhAhVE20Ii","taxability_reason":"not_collecting","taxable_amount":0}],"transfer_data":null,"webhooks_delivered_at":1675345605,"updated":1675345605},"emitted_at":1705636368184} +{"stream":"invoices","data":{"id":"in_1MX2yFEcXtiJtvvhMXhUCgKx","object":"invoice","account_country":"US","account_name":"Airbyte, Inc.","account_tax_ids":null,"amount_due":72680,"amount_paid":72680,"amount_remaining":0,"amount_shipping":0,"application":null,"application_fee_amount":null,"attempt_count":1,"attempted":true,"auto_advance":false,"automatic_tax":{"enabled":true,"liability":{"type":"self"},"status":"complete"},"billing_reason":"manual","charge":"ch_3MX2zPEcXtiJtvvh1BUGw8EC","collection_method":"send_invoice","created":1675345019,"currency":"usd","custom_fields":null,"customer":"cus_NGoTFiJFVbSsvZ","customer_address":{"city":"","country":"US","line1":"Street 2, 34567","line2":"","postal_code":"94114","state":"CA"},"customer_email":"user1.sample@zohomail.eu","customer_name":"Test Customer 2","customer_phone":null,"customer_shipping":{"address":{"city":"","country":"US","line1":"Street 2, 34567","line2":"","postal_code":"94114","state":"CA"},"name":"Test Customer 2","phone":""},"customer_tax_exempt":"none","customer_tax_ids":[],"default_payment_method":null,"default_source":null,"default_tax_rates":[],"description":"Thanks for your business!","discount":null,"discounts":[],"due_date":1677937018,"effective_at":1675345090,"ending_balance":0,"footer":"Test Invoice","from_invoice":null,"hosted_invoice_url":"https://invoice.stripe.com/i/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9OSGNDT3BXU2sxR0NJUDNaTTZnbXFINW10NHNiaWhDLDk2MTc3MTY20200j28bcP5i?s=ap","invoice_pdf":"https://pay.stripe.com/invoice/acct_1JwnoiEcXtiJtvvh/test_YWNjdF8xSndub2lFY1h0aUp0dnZoLF9OSGNDT3BXU2sxR0NJUDNaTTZnbXFINW10NHNiaWhDLDk2MTc3MTY20200j28bcP5i/pdf?s=ap","issuer":{"type":"self"},"last_finalization_error":null,"latest_revision":null,"lines":{"object":"list","data":[{"id":"il_1MX2yfEcXtiJtvvhiunY2j1x","object":"line_item","amount":25200,"amount_excluding_tax":25200,"currency":"usd","description":"edgao-test-product","discount_amounts":[{"amount":2520,"discount":"di_1MX2ysEcXtiJtvvh8ORqRVKm"}],"discountable":true,"discounts":["di_1MX2ysEcXtiJtvvh8ORqRVKm"],"invoice_item":"ii_1MX2yfEcXtiJtvvhfhyOG7SP","livemode":false,"metadata":{},"period":{"end":1675345045,"start":1675345045},"plan":null,"price":{"id":"price_1K9GbqEcXtiJtvvhJ3lZe4i5","object":"price","active":true,"billing_scheme":"per_unit","created":1640124902,"currency":"usd","custom_unit_amount":null,"livemode":false,"lookup_key":null,"metadata":{},"nickname":null,"product":"prod_KouQ5ez86yREmB","recurring":null,"tax_behavior":"inclusive","tiers_mode":null,"transform_quantity":null,"type":"one_time","unit_amount":12600,"unit_amount_decimal":"12600"},"proration":false,"proration_details":{"credited_items":null},"quantity":2,"subscription":null,"tax_amounts":[{"amount":0,"inclusive":true,"tax_rate":"txr_1MX2yfEcXtiJtvvhVcMEMTRj","taxability_reason":"not_collecting","taxable_amount":0}],"tax_rates":[],"type":"invoiceitem","unit_amount_excluding_tax":"12600"}],"has_more":false,"total_count":1,"url":"/v1/invoices/in_1MX2yFEcXtiJtvvhMXhUCgKx/lines"},"livemode":false,"metadata":{},"next_payment_attempt":null,"number":"C09C1837-0001","on_behalf_of":null,"paid":true,"paid_out_of_band":false,"payment_intent":"pi_3MX2zPEcXtiJtvvh12VBcp6m","payment_settings":{"default_mandate":null,"payment_method_options":null,"payment_method_types":null},"period_end":1675345018,"period_start":1675345018,"post_payment_credit_notes_amount":0,"pre_payment_credit_notes_amount":0,"quote":null,"receipt_number":null,"rendering":null,"rendering_options":null,"shipping_cost":null,"shipping_details":null,"starting_balance":50000,"statement_descriptor":null,"status":"paid","status_transitions":{"finalized_at":1675345090,"marked_uncollectible_at":null,"paid_at":1675345122,"voided_at":null},"subscription":null,"subscription_details":{"metadata":null},"subtotal":22680,"subtotal_excluding_tax":22680,"tax":0,"test_clock":null,"total":22680,"total_discount_amounts":[{"amount":2520,"discount":"di_1MX2ysEcXtiJtvvh8ORqRVKm"}],"total_excluding_tax":22680,"total_tax_amounts":[{"amount":0,"inclusive":true,"tax_rate":"txr_1MX2yfEcXtiJtvvhVcMEMTRj","taxability_reason":"not_collecting","taxable_amount":0}],"transfer_data":null,"webhooks_delivered_at":1675345019,"updated":1675345019},"emitted_at":1705636368184} +{"stream": "invoice_items", "data": {"id": "ii_1K9GKLEcXtiJtvvhmr2AYOAx", "object": "invoiceitem", "amount": 8400, "currency": "usd", "customer": "cus_Kou8knsO3qQOwU", "date": 1640123817, "description": "a box of parsnips", "discountable": true, "discounts": [], "invoice": "in_1K9GK0EcXtiJtvvhSo2LvGqT", "livemode": false, "metadata": {}, "period": {"end": 1640123817, "start": 1640123817}, "plan": null, "price": {"id": "price_1K9GKLEcXtiJtvvhXbrg33lq", "object": "price", "active": false, "billing_scheme": "per_unit", "created": 1640123817, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_Kou8cQxtIpF1p7", "recurring": null, "tax_behavior": "unspecified", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 8400, "unit_amount_decimal": "8400"}, "proration": false, "quantity": 1, "subscription": null, "tax_rates": [], "test_clock": null, "unit_amount": 8400, "unit_amount_decimal": "8400", "updated": 1640123817}, "emitted_at": 1697627302138} +{"stream": "invoice_items", "data": {"id": "ii_1MX384EcXtiJtvvhguyn3iYb", "object": "invoiceitem", "amount": 6000, "currency": "usd", "customer": "cus_NGoTFiJFVbSsvZ", "date": 1675345628, "description": "Test Product 1", "discountable": true, "discounts": ["di_1MX384EcXtiJtvvhkOrY57Ep"], "invoice": "in_1MX37hEcXtiJtvvhRSl1KbQm", "livemode": false, "metadata": {}, "period": {"end": 1675345628, "start": 1675345628}, "plan": null, "price": {"id": "price_1MX364EcXtiJtvvhE3WgTl4O", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1675345504, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NHcKselSHfKdfc", "recurring": null, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 2000, "unit_amount_decimal": "2000"}, "proration": false, "quantity": 3, "subscription": null, "tax_rates": [], "test_clock": null, "unit_amount": 2000, "unit_amount_decimal": "2000", "updated": 1675345628}, "emitted_at": 1697627302391} +{"stream": "invoice_items", "data": {"id": "ii_1MX2yfEcXtiJtvvhfhyOG7SP", "object": "invoiceitem", "amount": 25200, "currency": "usd", "customer": "cus_NGoTFiJFVbSsvZ", "date": 1675345045, "description": "edgao-test-product", "discountable": true, "discounts": ["di_1MX2ysEcXtiJtvvh8ORqRVKm"], "invoice": "in_1MX2yFEcXtiJtvvhMXhUCgKx", "livemode": false, "metadata": {}, "period": {"end": 1675345045, "start": 1675345045}, "plan": null, "price": {"id": "price_1K9GbqEcXtiJtvvhJ3lZe4i5", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1640124902, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_KouQ5ez86yREmB", "recurring": null, "tax_behavior": "inclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 12600, "unit_amount_decimal": "12600"}, "proration": false, "quantity": 2, "subscription": null, "tax_rates": [], "test_clock": null, "unit_amount": 12600, "unit_amount_decimal": "12600", "updated": 1675345045}, "emitted_at": 1697627302392} +{"stream": "payouts", "data": {"id": "po_1KVQhfEcXtiJtvvhZlUkl08U", "object": "payout", "amount": 9164, "arrival_date": 1645488000, "automatic": true, "balance_transaction": "txn_1KVQhfEcXtiJtvvhF7ox3YEm", "created": 1645406919, "currency": "usd", "description": "STRIPE PAYOUT", "destination": "ba_1KUL7UEcXtiJtvvhAEUlStmv", "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "livemode": false, "metadata": {}, "method": "standard", "original_payout": null, "reconciliation_status": "completed", "reversed_by": null, "source_balance": null, "source_type": "card", "statement_descriptor": null, "status": "paid", "type": "bank_account", "updated": 1645406919}, "emitted_at": 1697627303542} +{"stream": "payouts", "data": {"id": "po_1MXKoPEcXtiJtvvhwmqjvKoO", "object": "payout", "amount": 665880, "arrival_date": 1675382400, "automatic": false, "balance_transaction": "txn_1MXKoQEcXtiJtvvh65SHFZS6", "created": 1675413601, "currency": "usd", "description": "", "destination": "ba_1MSI1fEcXtiJtvvhPlqZqPlw", "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "livemode": false, "metadata": {}, "method": "standard", "original_payout": null, "reconciliation_status": "not_applicable", "reversed_by": null, "source_balance": null, "source_type": "card", "statement_descriptor": "airbyte.io", "status": "paid", "type": "bank_account", "updated": 1675413601}, "emitted_at": 1697627303817} +{"stream": "payouts", "data": {"id": "po_1MWHzjEcXtiJtvvhIdUQHhLq", "object": "payout", "amount": 154900, "arrival_date": 1675123200, "automatic": false, "balance_transaction": "txn_1MWHzjEcXtiJtvvhtydemd2Y", "created": 1675164443, "currency": "usd", "description": "Test", "destination": "ba_1MSI1fEcXtiJtvvhPlqZqPlw", "failure_balance_transaction": null, "failure_code": null, "failure_message": null, "livemode": false, "metadata": {}, "method": "standard", "original_payout": null, "reconciliation_status": "not_applicable", "reversed_by": null, "source_balance": "issuing", "source_type": "issuing", "statement_descriptor": "airbyte.io", "status": "paid", "type": "bank_account", "updated": 1675164443}, "emitted_at": 1697627303818} +{"stream": "plans", "data": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed", "updated": 1674209524}, "emitted_at": 1697627305150} +{"stream": "prices", "data": {"id": "price_1K9GbqEcXtiJtvvhJ3lZe4i5", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1640124902, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_KouQ5ez86yREmB", "recurring": null, "tax_behavior": "inclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 12600.0, "unit_amount_decimal": "12600", "updated": 1640124902}, "emitted_at": 1697627306255} +{"stream": "prices", "data": {"id": "price_1MX364EcXtiJtvvh6jKcimNL", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1675345504, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NHcKselSHfKdfc", "recurring": null, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 1700.0, "unit_amount_decimal": "1700", "updated": 1675345504}, "emitted_at": 1697627306506} +{"stream": "prices", "data": {"id": "price_1MX364EcXtiJtvvhE3WgTl4O", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1675345504, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NHcKselSHfKdfc", "recurring": null, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 2000.0, "unit_amount_decimal": "2000", "updated": 1675345504}, "emitted_at": 1697627306508} +{"stream": "products", "data": {"id": "prod_KouQ5ez86yREmB", "object": "product", "active": true, "attributes": [], "created": 1640124902, "default_price": "price_1K9GbqEcXtiJtvvhJ3lZe4i5", "description": null, "features": [], "images": [], "livemode": false, "metadata": {}, "name": "edgao-test-product", "package_dimensions": null, "shippable": null, "statement_descriptor": null, "tax_code": "txcd_10000000", "type": "service", "unit_label": null, "updated": 1696839715, "url": null}, "emitted_at": 1697627307635} +{"stream": "products", "data": {"id": "prod_NHcKselSHfKdfc", "object": "product", "active": true, "attributes": [], "created": 1675345504, "default_price": "price_1MX364EcXtiJtvvhE3WgTl4O", "description": "Test Product 1 description", "features": [], "images": ["https://files.stripe.com/links/MDB8YWNjdF8xSndub2lFY1h0aUp0dnZofGZsX3Rlc3RfdjBOT09UaHRiNVl2WmJ6clNYRUlmcFFD00cCBRNHnV"], "livemode": false, "metadata": {}, "name": "Test Product 1", "package_dimensions": null, "shippable": null, "statement_descriptor": null, "tax_code": "txcd_10301000", "type": "service", "unit_label": null, "updated": 1696839789, "url": null}, "emitted_at": 1697627307877} +{"stream": "products", "data": {"id": "prod_NCgx1XP2IFQyKF", "object": "product", "active": true, "attributes": [], "created": 1674209524, "default_price": null, "description": null, "features": [], "images": [], "livemode": false, "metadata": {}, "name": "tu", "package_dimensions": null, "shippable": null, "statement_descriptor": null, "tax_code": "txcd_10000000", "type": "service", "unit_label": null, "updated": 1696839225, "url": null}, "emitted_at": 1697627307879} +{"stream": "subscriptions", "data": {"id": "sub_1O2Dg0EcXtiJtvvhz7Q4zS0n", "object": "subscription", "application": null, "application_fee_percent": null, "automatic_tax": {"enabled": true, "liability": {"type": "self"}}, "billing_cycle_anchor": 1697550676.0, "billing_cycle_anchor_config": null, "billing_thresholds": null, "cancel_at": null, "cancel_at_period_end": false, "canceled_at": 1697550676.0, "cancellation_details": {"comment": null, "feedback": null, "reason": "cancellation_requested"}, "collection_method": "charge_automatically", "created": 1697550676, "currency": "usd", "current_period_end": 1705499476.0, "current_period_start": 1702821076, "customer": "cus_NGoTFiJFVbSsvZ", "days_until_due": null, "default_payment_method": null, "default_source": null, "default_tax_rates": [], "description": null, "discount": null, "ended_at": 1705329724.0, "invoice_settings": {"issuer": {"type": "self"}}, "items": {"object": "list", "data": [{"id": "si_OptSP2o3XZUBpx", "object": "subscription_item", "billing_thresholds": null, "created": 1697550677, "metadata": {}, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "price": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "recurring": {"aggregate_usage": null, "interval": "month", "interval_count": 1, "trial_period_days": null, "usage_type": "licensed"}, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "recurring", "unit_amount": 600, "unit_amount_decimal": "600"}, "quantity": 1, "subscription": "sub_1O2Dg0EcXtiJtvvhz7Q4zS0n", "tax_rates": []}], "has_more": false, "total_count": 1.0, "url": "/v1/subscription_items?subscription=sub_1O2Dg0EcXtiJtvvhz7Q4zS0n"}, "latest_invoice": "in_1OOKkUEcXtiJtvvheUUavyuB", "livemode": false, "metadata": {}, "next_pending_invoice_item_invoice": null, "on_behalf_of": null, "pause_collection": null, "payment_settings": {"payment_method_options": null, "payment_method_types": null, "save_default_payment_method": null}, "pending_invoice_item_interval": null, "pending_setup_intent": null, "pending_update": null, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "quantity": 1, "schedule": "sub_sched_1O2Dg0EcXtiJtvvh7GtbtIhP", "start_date": 1697550676, "status": "canceled", "test_clock": null, "transfer_data": null, "trial_end": null, "trial_settings": {"end_behavior": {"missing_payment_method": "create_invoice"}}, "trial_start": null, "updated": 1697550676}, "emitted_at": 1705636378387} +{"stream":"subscription_schedule","data":{"id":"sub_sched_1O2Dg0EcXtiJtvvh7GtbtIhP","object":"subscription_schedule","application":null,"canceled_at":"1705329724","completed_at":null,"created":1697550676,"current_phase":null,"customer":"cus_NGoTFiJFVbSsvZ","default_settings":{"application_fee_percent":null,"automatic_tax":{"enabled":false},"billing_cycle_anchor":"automatic","billing_thresholds":null,"collection_method":"charge_automatically","default_payment_method":null,"default_source":null,"description":"Test Test","invoice_settings":"{'days_until_due': None}","on_behalf_of":null,"transfer_data":null},"end_behavior":"cancel","livemode":false,"metadata":{},"phases":[{"add_invoice_items":[],"application_fee_percent":null,"automatic_tax":{"enabled":true},"billing_cycle_anchor":null,"billing_thresholds":null,"collection_method":"charge_automatically","coupon":null,"currency":"usd","default_payment_method":null,"default_tax_rates":[],"description":"Test Test","end_date":1705499476,"invoice_settings":"{'days_until_due': None}","items":[{"billing_thresholds":null,"metadata":{},"plan":"price_1MSHZoEcXtiJtvvh6O8TYD8T","price":"price_1MSHZoEcXtiJtvvh6O8TYD8T","quantity":1,"tax_rates":[]}],"metadata":{},"on_behalf_of":null,"proration_behavior":"create_prorations","start_date":1697550676,"transfer_data":null,"trial_end":null}],"released_at":null,"released_subscription":null,"renewal_interval":null,"status":"canceled","subscription":"sub_1O2Dg0EcXtiJtvvhz7Q4zS0n","test_clock":null,"updated":1697550676},"emitted_at":1705636378620} +{"stream": "transfers", "data": {"id": "tr_1NH18zEcXtiJtvvhnd827cNO", "object": "transfer", "amount": 10000, "amount_reversed": 0, "balance_transaction": "txn_1NH190EcXtiJtvvhBO3PeR7p", "created": 1686301085, "currency": "usd", "description": null, "destination": "acct_1Jx8unEYmRTj5on1", "destination_payment": "py_1NH18zEYmRTj5on1GkCCsqLK", "livemode": false, "metadata": {}, "reversals": {"object": "list", "data": [], "has_more": false, "total_count": 0.0, "url": "/v1/transfers/tr_1NH18zEcXtiJtvvhnd827cNO/reversals"}, "reversed": false, "source_transaction": null, "source_type": "card", "transfer_group": null, "updated": 1686301085}, "emitted_at": 1697627313262} +{"stream": "transfers", "data": {"id": "tr_1NGoaCEcXtiJtvvhjmHtOGOm", "object": "transfer", "amount": 100, "amount_reversed": 100, "balance_transaction": "txn_1NGoaDEcXtiJtvvhsZrNMsdJ", "created": 1686252800, "currency": "usd", "description": null, "destination": "acct_1Jx8unEYmRTj5on1", "destination_payment": "py_1NGoaCEYmRTj5on1LAlAIG3a", "livemode": false, "metadata": {}, "reversals": {"object": "list", "data": [{"id": "trr_1NGolCEcXtiJtvvhOYPck3CP", "object": "transfer_reversal", "amount": 100, "balance_transaction": "txn_1NGolCEcXtiJtvvhZRy4Kd5S", "created": 1686253482, "currency": "usd", "destination_payment_refund": "pyr_1NGolBEYmRTj5on1STal3rmp", "metadata": {}, "source_refund": null, "transfer": "tr_1NGoaCEcXtiJtvvhjmHtOGOm"}], "has_more": false, "total_count": 1.0, "url": "/v1/transfers/tr_1NGoaCEcXtiJtvvhjmHtOGOm/reversals"}, "reversed": true, "source_transaction": null, "source_type": "card", "transfer_group": "ORDER10", "updated": 1686252800}, "emitted_at": 1697627313264} +{"stream": "refunds", "data": {"id": "re_3MVuZyEcXtiJtvvh0A6rSbeJ", "object": "refund", "amount": 200000, "balance_transaction": "txn_3MVuZyEcXtiJtvvh0v0QyAMx", "charge": "ch_3MVuZyEcXtiJtvvh0tiVC7DI", "created": 1675074488, "currency": "usd", "destination_details": {"card": {"reference": "5871771120000631", "reference_status": "available", "reference_type": "acquirer_reference_number", "type": "refund"}, "type": "card"}, "metadata": {}, "payment_intent": "pi_3MVuZyEcXtiJtvvh07Ehi4cx", "reason": "fraudulent", "receipt_number": "3278-5368", "source_transfer_reversal": null, "status": "succeeded", "transfer_reversal": null}, "emitted_at": 1701882752716} +{"stream": "refunds", "data": {"id": "re_3NcwAGEcXtiJtvvh1UT4PBe6", "object": "refund", "amount": 600, "balance_transaction": "txn_3NcwAGEcXtiJtvvh1AcNi3Ma", "charge": "ch_3NcwAGEcXtiJtvvh1m0SSmfQ", "created": 1692782173, "currency": "usd", "destination_details": {"card": {"reference": "1726973908953368", "reference_status": "available", "reference_type": "acquirer_reference_number", "type": "refund"}, "type": "card"}, "metadata": {}, "payment_intent": "pi_3NcwAGEcXtiJtvvh1olHTPmH", "reason": null, "receipt_number": null, "source_transfer_reversal": null, "status": "succeeded", "transfer_reversal": null}, "emitted_at": 1701882752742} +{"stream": "refunds", "data": {"id": "re_3MngeoEcXtiJtvvh0c4KeMOd", "object": "refund", "amount": 540, "balance_transaction": "txn_3MngeoEcXtiJtvvh0Cz3qwU2", "charge": "ch_3MngeoEcXtiJtvvh0SBFQWe2", "created": 1683889626, "currency": "usd", "destination_details": {"card": {"reference": "9953330538701266", "reference_status": "available", "reference_type": "acquirer_reference_number", "type": "refund"}, "type": "card"}, "metadata": {}, "payment_intent": "pi_3MngeoEcXtiJtvvh0B7Tcbr4", "reason": "requested_by_customer", "receipt_number": null, "source_transfer_reversal": null, "status": "succeeded", "transfer_reversal": null}, "emitted_at": 1701882752743} +{"stream": "payment_intents", "data": {"id": "pi_3K9FSOEcXtiJtvvh0AEIFllC", "object": "payment_intent", "amount": 5300, "amount_capturable": 0, "amount_details": {"tip": {}}, "amount_received": 5300, "application": null, "application_fee_amount": null, "automatic_payment_methods": null, "canceled_at": null, "cancellation_reason": null, "capture_method": "automatic", "client_secret": "pi_3K9FSOEcXtiJtvvh0AEIFllC_secret_uPUtIaSltgtW0qK7mLD0uF2Mr", "confirmation_method": "automatic", "created": 1640120472, "currency": "usd", "customer": null, "description": null, "invoice": null, "last_payment_error": null, "latest_charge": "ch_3K9FSOEcXtiJtvvh0zxb7clc", "livemode": false, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": null, "payment_method_configuration_details": null, "payment_method_options": {"card": {"installments": null, "mandate_options": null, "network": null, "request_three_d_secure": "automatic"}}, "payment_method_types": ["card"], "processing": null, "receipt_email": null, "review": null, "setup_future_usage": null, "shipping": null, "source": "src_1K9FSOEcXtiJtvvhHGu1qtOx", "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "succeeded", "transfer_data": null, "transfer_group": null, "updated": 1640120472}, "emitted_at": 1697627315508} +{"stream": "payment_intents", "data": {"id": "pi_3K9F5DEcXtiJtvvh16scJMp6", "object": "payment_intent", "amount": 4200, "amount_capturable": 0, "amount_details": {"tip": {}}, "amount_received": 4200, "application": null, "application_fee_amount": null, "automatic_payment_methods": null, "canceled_at": null, "cancellation_reason": null, "capture_method": "automatic", "client_secret": "pi_3K9F5DEcXtiJtvvh16scJMp6_secret_YwhzCTpXtfcKYeklXnPnysRRi", "confirmation_method": "automatic", "created": 1640119035, "currency": "usd", "customer": null, "description": "edgao test", "invoice": null, "last_payment_error": null, "latest_charge": "ch_3K9F5DEcXtiJtvvh1w2MaTpj", "livemode": false, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": null, "payment_method_configuration_details": null, "payment_method_options": {"card": {"installments": null, "mandate_options": null, "network": null, "request_three_d_secure": "automatic"}}, "payment_method_types": ["card"], "processing": null, "receipt_email": null, "review": null, "setup_future_usage": null, "shipping": null, "source": "src_1K9F5CEcXtiJtvvhrsZdur8Y", "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "succeeded", "transfer_data": null, "transfer_group": null, "updated": 1640119035}, "emitted_at": 1697627315511} +{"stream": "payment_intents", "data": {"id": "pi_3K9F4mEcXtiJtvvh18NKhEuo", "object": "payment_intent", "amount": 4200, "amount_capturable": 0, "amount_details": {"tip": {}}, "amount_received": 0, "application": null, "application_fee_amount": null, "automatic_payment_methods": null, "canceled_at": null, "cancellation_reason": null, "capture_method": "automatic", "client_secret": "pi_3K9F4mEcXtiJtvvh18NKhEuo_secret_pfUt7CTkPjVdJacycm0bMpdLt", "confirmation_method": "automatic", "created": 1640119008, "currency": "usd", "customer": null, "description": "edgao test", "invoice": null, "last_payment_error": {"charge": "ch_3K9F4mEcXtiJtvvh1kUzxjwN", "code": "card_declined", "decline_code": "test_mode_live_card", "doc_url": "https://stripe.com/docs/error-codes/card-declined", "message": "Your card was declined. Your request was in test mode, but used a non test (live) card. For a list of valid test cards, visit: https://stripe.com/docs/testing.", "source": {"id": "src_1K9F4hEcXtiJtvvhrUEwvCyi", "object": "source", "amount": null, "card": {"address_line1_check": null, "address_zip_check": null, "brand": "Visa", "country": "US", "cvc_check": "unchecked", "dynamic_last4": null, "exp_month": 9, "exp_year": 2028, "fingerprint": "Re3p4j8issXA77iI", "funding": "credit", "last4": "8097", "name": null, "three_d_secure": "optional", "tokenization_method": null}, "client_secret": "src_client_secret_b3v8YqNMLGykB120fqv2Tjhq", "created": 1640119003, "currency": null, "flow": "none", "livemode": false, "metadata": {}, "owner": {"address": null, "email": null, "name": null, "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "consumed", "type": "card", "usage": "reusable"}, "type": "card_error"}, "latest_charge": "ch_3K9F4mEcXtiJtvvh1kUzxjwN", "livemode": false, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": null, "payment_method_configuration_details": null, "payment_method_options": {"card": {"installments": null, "mandate_options": null, "network": null, "request_three_d_secure": "automatic"}}, "payment_method_types": ["card"], "processing": null, "receipt_email": null, "review": null, "setup_future_usage": null, "shipping": null, "source": null, "statement_descriptor": "airbyte.io", "statement_descriptor_suffix": null, "status": "requires_payment_method", "transfer_data": null, "transfer_group": null, "updated": 1640119008}, "emitted_at": 1697627315513} +{"stream": "promotion_codes", "data": {"id": "promo_1MVtmyEcXtiJtvvhkV5jPFPU", "object": "promotion_code", "active": true, "code": "g20", "coupon": {"id": "iJ6qlwM5", "object": "coupon", "amount_off": null, "created": 1674208993, "currency": null, "duration": "forever", "duration_in_months": null, "livemode": false, "max_redemptions": null, "metadata": {}, "name": "\u0415\u0443\u0456\u0435", "percent_off": 10.0, "redeem_by": null, "times_redeemed": 3, "valid": true}, "created": 1675071396, "customer": null, "expires_at": null, "livemode": false, "max_redemptions": null, "metadata": {}, "restrictions": {"first_time_transaction": false, "minimum_amount": null, "minimum_amount_currency": null}, "times_redeemed": 0, "updated": 1675071396}, "emitted_at": 1697627317910} +{"stream": "promotion_codes", "data": {"id": "promo_1MVtmkEcXtiJtvvht0RA3MKg", "object": "promotion_code", "active": true, "code": "FRIENDS20", "coupon": {"id": "iJ6qlwM5", "object": "coupon", "amount_off": null, "created": 1674208993, "currency": null, "duration": "forever", "duration_in_months": null, "livemode": false, "max_redemptions": null, "metadata": {}, "name": "\u0415\u0443\u0456\u0435", "percent_off": 10.0, "redeem_by": null, "times_redeemed": 3, "valid": true}, "created": 1675071382, "customer": null, "expires_at": null, "livemode": false, "max_redemptions": null, "metadata": {}, "restrictions": {"first_time_transaction": true, "minimum_amount": 10000, "minimum_amount_currency": "usd"}, "times_redeemed": 0, "updated": 1675071382}, "emitted_at": 1697627317911} +{"stream": "setup_intents", "data": {"id": "seti_1KnfIjEcXtiJtvvhPw5znVKY", "object": "setup_intent", "application": null, "automatic_payment_methods": null, "cancellation_reason": null, "client_secret": "seti_1KnfIjEcXtiJtvvhPw5znVKY_secret_LUebPsqMz6AF4ivxIg4LMaAT0OdZF5L", "created": 1649752937, "customer": null, "description": null, "flow_directions": null, "last_setup_error": null, "latest_attempt": "setatt_1KnfIjEcXtiJtvvhqDfSlpM4", "livemode": false, "mandate": null, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": "pm_1KnfIj2eZvKYlo2CAlv2Vhqc", "payment_method_configuration_details": null, "payment_method_options": {"acss_debit": {"currency": "cad", "mandate_options": {"interval_description": "First day of every month", "payment_schedule": "interval", "transaction_type": "personal"}, "verification_method": "automatic"}}, "payment_method_types": ["acss_debit"], "single_use_mandate": null, "status": "succeeded", "usage": "off_session", "updated": 1649752937}, "emitted_at": 1697627319186} +{"stream": "setup_intents", "data": {"id": "seti_1KnfIcEcXtiJtvvh61qlCaDf", "object": "setup_intent", "application": null, "automatic_payment_methods": null, "cancellation_reason": null, "client_secret": "seti_1KnfIcEcXtiJtvvh61qlCaDf_secret_LUebcbyw8V1e8Pxk3aAjzDXMOXdFMCe", "created": 1649752930, "customer": null, "description": null, "flow_directions": null, "last_setup_error": null, "latest_attempt": "setatt_1KnfIdEcXtiJtvvhpDrYVlRP", "livemode": false, "mandate": null, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": "pm_1KnfIc2eZvKYlo2Civ7snSPy", "payment_method_configuration_details": null, "payment_method_options": {"acss_debit": {"currency": "cad", "mandate_options": {"interval_description": "First day of every month", "payment_schedule": "interval", "transaction_type": "personal"}, "verification_method": "automatic"}}, "payment_method_types": ["acss_debit"], "single_use_mandate": null, "status": "succeeded", "usage": "off_session", "updated": 1649752930}, "emitted_at": 1697627319189} +{"stream": "setup_intents", "data": {"id": "seti_1KnfIVEcXtiJtvvhWiIbMkpH", "object": "setup_intent", "application": null, "automatic_payment_methods": null, "cancellation_reason": null, "client_secret": "seti_1KnfIVEcXtiJtvvhWiIbMkpH_secret_LUebIUsiFnm75EzDUzf2RLhJ9WQ92Dp", "created": 1649752923, "customer": null, "description": null, "flow_directions": null, "last_setup_error": null, "latest_attempt": "setatt_1KnfIVEcXtiJtvvhqouWGuhD", "livemode": false, "mandate": null, "metadata": {}, "next_action": null, "on_behalf_of": null, "payment_method": "pm_1KnfIV2eZvKYlo2CaOLGBF00", "payment_method_configuration_details": null, "payment_method_options": {"acss_debit": {"currency": "cad", "mandate_options": {"interval_description": "First day of every month", "payment_schedule": "interval", "transaction_type": "personal"}, "verification_method": "automatic"}}, "payment_method_types": ["acss_debit"], "single_use_mandate": null, "status": "succeeded", "usage": "off_session", "updated": 1649752923}, "emitted_at": 1697627319190} +{"stream": "top_ups", "data": {"id": "tu_1MXKmvEcXtiJtvvhAbDiH3sm", "object": "topup", "amount": 100000, "balance_transaction": null, "created": 1675413509, "currency": "usd", "description": "Test test", "destination_balance": null, "expected_availability_date": 1675413509, "failure_code": "R03", "failure_message": "no_account", "livemode": false, "metadata": {}, "source": {"id": "src_1MXKmvEcXtiJtvvhV0fY3ZBF", "object": "source", "ach_debit": {"bank_name": "STRIPE TEST BANK", "country": "US", "fingerprint": "KUgiD3MWWaMMufNe", "last4": "1116", "routing_number": "110000000", "type": "individual"}, "amount": null, "client_secret": "src_client_secret_qghd0H1WkqAuSDuJBrYBSfD6", "code_verification": {"attempts_remaining": 3, "status": "succeeded"}, "created": 1675413509, "currency": "usd", "flow": "code_verification", "livemode": false, "metadata": {}, "owner": {"address": {"city": null, "country": "US", "line1": null, "line2": null, "postal_code": null, "state": null}, "email": null, "name": "Jenny Rosen", "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "chargeable", "type": "ach_debit", "usage": "reusable"}, "statement_descriptor": "Test", "status": "failed", "transfer_group": null, "updated": 1675413509}, "emitted_at": 1697627323940} +{"stream": "top_ups", "data": {"id": "tu_1MWIGdEcXtiJtvvhEd8RlSF4", "object": "topup", "amount": 300000, "balance_transaction": "txn_1MWIGdEcXtiJtvvhzFY7mzQ1", "created": 1675165491, "currency": "usd", "description": "airbyte.io", "destination_balance": null, "expected_availability_date": 1675165491, "failure_code": null, "failure_message": null, "livemode": false, "metadata": {}, "source": {"id": "src_1MSHpIEcXtiJtvvhTLi0M106", "object": "source", "ach_credit_transfer": {"account_number": "test_b6fc39627c05", "bank_name": "TEST BANK", "fingerprint": "HvbzMJ6aNgEC2mdU", "purpose": "topups", "refund_account_holder_name": null, "refund_account_holder_type": null, "refund_routing_number": null, "routing_number": "110000000", "swift_code": "TSTEZ122"}, "amount": null, "client_secret": "src_client_secret_rgsgOHKzvVdUKHOGVd2KfGJe", "created": 1674210484, "currency": "usd", "flow": "receiver", "livemode": false, "metadata": {}, "owner": {"address": null, "email": "amount_300000@example.com", "name": null, "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "receiver": {"address": "110000000-test_b6fc39627c05", "amount_charged": 307200, "amount_received": 307200, "amount_returned": 0, "refund_attributes_method": "none", "refund_attributes_status": "missing"}, "statement_descriptor": null, "status": "pending", "type": "ach_credit_transfer", "usage": "reusable"}, "statement_descriptor": null, "status": "succeeded", "transfer_group": null, "updated": 1675165491}, "emitted_at": 1697627323942} +{"stream": "top_ups", "data": {"id": "tu_1MWH6aEcXtiJtvvhLGJLzCLX", "object": "topup", "amount": 150900, "balance_transaction": "txn_1MWH6aEcXtiJtvvh4IIJFtL3", "created": 1675161024, "currency": "usd", "description": "Test funds", "destination_balance": "issuing", "expected_availability_date": 1675161024, "failure_code": null, "failure_message": null, "livemode": false, "metadata": {}, "source": {"id": "src_1MWH6aEcXtiJtvvhsZE5wEoz", "object": "source", "ach_debit": {"bank_name": "STRIPE TEST BANK", "country": "US", "fingerprint": "EYRDOxbGrk2CWvEr", "last4": "6789", "routing_number": "110000000", "type": "individual"}, "amount": null, "client_secret": "src_client_secret_5g6GJ5hB2E8xi1EK0pGrdchm", "code_verification": {"attempts_remaining": 3, "status": "succeeded"}, "created": 1675161024, "currency": "usd", "flow": "code_verification", "livemode": false, "metadata": {}, "owner": {"address": {"city": null, "country": "US", "line1": null, "line2": null, "postal_code": null, "state": null}, "email": null, "name": "Jenny Rosen", "phone": null, "verified_address": null, "verified_email": null, "verified_name": null, "verified_phone": null}, "statement_descriptor": null, "status": "chargeable", "type": "ach_debit", "usage": "reusable"}, "statement_descriptor": "Add test funds", "status": "succeeded", "transfer_group": null, "updated": 1675161024}, "emitted_at": 1697627323943} +{"stream": "invoice_line_items", "data": {"id": "il_1K9GKLEcXtiJtvvhhHaYMebN", "object": "line_item", "amount": 8400, "amount_excluding_tax": 8400, "currency": "usd", "description": "a box of parsnips", "discount_amounts": [], "discountable": true, "discounts": [], "invoice_item": "ii_1K9GKLEcXtiJtvvhmr2AYOAx", "livemode": false, "metadata": {}, "period": {"end": 1640123817, "start": 1640123817}, "plan": null, "price": {"id": "price_1K9GKLEcXtiJtvvhXbrg33lq", "object": "price", "active": false, "billing_scheme": "per_unit", "created": 1640123817, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_Kou8cQxtIpF1p7", "recurring": null, "tax_behavior": "unspecified", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 8400, "unit_amount_decimal": "8400"}, "proration": false, "proration_details": {"credited_items": null}, "quantity": 1, "subscription": null, "tax_amounts": [], "tax_rates": [], "type": "invoiceitem", "unit_amount_excluding_tax": "8400", "invoice_id": "in_1K9GK0EcXtiJtvvhSo2LvGqT"}, "emitted_at": 1697627336438} +{"stream": "invoice_line_items", "data": {"id": "il_1MX384EcXtiJtvvh3j2K123f", "object": "line_item", "amount": 6000, "amount_excluding_tax": 6000, "currency": "usd", "description": "Test Product 1", "discount_amounts": [{"amount": 500, "discount": "di_1MX384EcXtiJtvvhkOrY57Ep"}], "discountable": true, "discounts": ["di_1MX384EcXtiJtvvhkOrY57Ep"], "invoice_item": "ii_1MX384EcXtiJtvvhguyn3iYb", "livemode": false, "metadata": {}, "period": {"end": 1675345628, "start": 1675345628}, "plan": null, "price": {"id": "price_1MX364EcXtiJtvvhE3WgTl4O", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1675345504, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NHcKselSHfKdfc", "recurring": null, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 2000, "unit_amount_decimal": "2000"}, "proration": false, "proration_details": {"credited_items": null}, "quantity": 3, "subscription": null, "tax_amounts": [{"amount": 0, "inclusive": false, "tax_rate": "txr_1MX384EcXtiJtvvhAhVE20Ii", "taxability_reason": "not_collecting", "taxable_amount": 0}], "tax_rates": [], "type": "invoiceitem", "unit_amount_excluding_tax": "2000", "invoice_id": "in_1MX37hEcXtiJtvvhRSl1KbQm"}, "emitted_at": 1697627336446} +{"stream": "invoice_line_items", "data": {"id": "il_1MX2yfEcXtiJtvvhiunY2j1x", "object": "line_item", "amount": 25200, "amount_excluding_tax": 25200, "currency": "usd", "description": "edgao-test-product", "discount_amounts": [{"amount": 2520, "discount": "di_1MX2ysEcXtiJtvvh8ORqRVKm"}], "discountable": true, "discounts": ["di_1MX2ysEcXtiJtvvh8ORqRVKm"], "invoice_item": "ii_1MX2yfEcXtiJtvvhfhyOG7SP", "livemode": false, "metadata": {}, "period": {"end": 1675345045, "start": 1675345045}, "plan": null, "price": {"id": "price_1K9GbqEcXtiJtvvhJ3lZe4i5", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1640124902, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_KouQ5ez86yREmB", "recurring": null, "tax_behavior": "inclusive", "tiers_mode": null, "transform_quantity": null, "type": "one_time", "unit_amount": 12600, "unit_amount_decimal": "12600"}, "proration": false, "proration_details": {"credited_items": null}, "quantity": 2, "subscription": null, "tax_amounts": [{"amount": 0, "inclusive": true, "tax_rate": "txr_1MX2yfEcXtiJtvvhVcMEMTRj", "taxability_reason": "not_collecting", "taxable_amount": 0}], "tax_rates": [], "type": "invoiceitem", "unit_amount_excluding_tax": "12600", "invoice_id": "in_1MX2yFEcXtiJtvvhMXhUCgKx"}, "emitted_at": 1697627336449} +{"stream": "subscription_items", "data": {"id": "si_OptSP2o3XZUBpx", "object": "subscription_item", "billing_thresholds": null, "created": 1697550677, "metadata": {}, "plan": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "plan", "active": true, "aggregate_usage": null, "amount": 600, "amount_decimal": "600", "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "interval": "month", "interval_count": 1, "livemode": false, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "tiers_mode": null, "transform_usage": null, "trial_period_days": null, "usage_type": "licensed"}, "price": {"id": "price_1MSHZoEcXtiJtvvh6O8TYD8T", "object": "price", "active": true, "billing_scheme": "per_unit", "created": 1674209524, "currency": "usd", "custom_unit_amount": null, "livemode": false, "lookup_key": null, "metadata": {}, "nickname": null, "product": "prod_NCgx1XP2IFQyKF", "recurring": {"aggregate_usage": null, "interval": "month", "interval_count": 1, "trial_period_days": null, "usage_type": "licensed"}, "tax_behavior": "exclusive", "tiers_mode": null, "transform_quantity": null, "type": "recurring", "unit_amount": 600, "unit_amount_decimal": "600"}, "quantity": 1, "subscription": "sub_1O2Dg0EcXtiJtvvhz7Q4zS0n", "tax_rates": []}, "emitted_at": 1697627337431} +{"stream": "transfer_reversals", "data": {"id": "trr_1NGolCEcXtiJtvvhOYPck3CP", "object": "transfer_reversal", "amount": 100, "balance_transaction": "txn_1NGolCEcXtiJtvvhZRy4Kd5S", "created": 1686253482, "currency": "usd", "destination_payment_refund": "pyr_1NGolBEYmRTj5on1STal3rmp", "metadata": {}, "source_refund": null, "transfer": "tr_1NGoaCEcXtiJtvvhjmHtOGOm"}, "emitted_at": 1697627338960} +{"stream": "usage_records", "data": {"id": "sis_1OUqWiEcXtiJtvvh3WGqc4Vk", "object": "usage_record_summary", "invoice": null, "livemode": false, "period": {"end": null, "start": 1702821076}, "subscription_item": "si_OptSP2o3XZUBpx", "total_usage": 1}, "emitted_at": 1700233660884} diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/full_refresh_configured_catalog.json b/airbyte-integrations/connectors/source-stripe/integration_tests/full_refresh_configured_catalog.json deleted file mode 100644 index d20b624cce1c..000000000000 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/full_refresh_configured_catalog.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "bank_accounts", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "external_account_bank_accounts", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "primary_key": [["id"]] - }, - { - "stream": { - "name": "customer_balance_transactions", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "primary_key": [["id"]] - }, - { - "stream": { - "name": "checkout_sessions", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "primary_key": [["id"]] - }, - { - "stream": { - "name": "checkout_sessions_line_items", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["checkout_session_expires_at"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "cursor_field": ["checkout_session_expires_at"], - "primary_key": [["id"]] - }, - { - "stream": { - "name": "accounts", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] - } - ] -} diff --git a/airbyte-integrations/connectors/source-stripe/integration_tests/non_invoice_line_items_catalog.json b/airbyte-integrations/connectors/source-stripe/integration_tests/non_invoice_line_items_catalog.json deleted file mode 100644 index f9e9239038e4..000000000000 --- a/airbyte-integrations/connectors/source-stripe/integration_tests/non_invoice_line_items_catalog.json +++ /dev/null @@ -1,172 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "customers", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] - }, - { - "stream": { - "name": "disputes", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] - }, - { - "stream": { - "name": "balance_transactions", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] - }, - { - "stream": { - "name": "charges", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] - }, - { - "stream": { - "name": "coupons", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] - }, - { - "stream": { - "name": "invoice_items", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["date"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["date"], - "primary_key": [["id"]] - }, - { - "stream": { - "name": "payouts", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] - }, - { - "stream": { - "name": "plans", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] - }, - { - "stream": { - "name": "promotion_codes", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] - }, - { - "stream": { - "name": "refunds", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] - }, - { - "stream": { - "name": "transfers", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] - }, - { - "stream": { - "name": "events", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["created"], - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "incremental", - "destination_sync_mode": "overwrite", - "cursor_field": ["created"], - "primary_key": [["id"]] - } - ] -} diff --git a/airbyte-integrations/connectors/source-stripe/main.py b/airbyte-integrations/connectors/source-stripe/main.py index 645bca7dd53f..971f33a69dd1 100644 --- a/airbyte-integrations/connectors/source-stripe/main.py +++ b/airbyte-integrations/connectors/source-stripe/main.py @@ -3,11 +3,7 @@ # -import sys - -from airbyte_cdk.entrypoint import launch -from source_stripe import SourceStripe +from source_stripe.run import run if __name__ == "__main__": - source = SourceStripe() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-stripe/metadata.yaml b/airbyte-integrations/connectors/source-stripe/metadata.yaml index e46be4d24e8f..cfedccfdf019 100644 --- a/airbyte-integrations/connectors/source-stripe/metadata.yaml +++ b/airbyte-integrations/connectors/source-stripe/metadata.yaml @@ -1,12 +1,18 @@ data: + ab_internal: + ql: 400 + sl: 300 allowedHosts: hosts: - api.stripe.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: api connectorType: source definitionId: e094cb9a-26de-4645-8761-65c0c425d1de - dockerImageTag: 3.17.4 + dockerImageTag: 5.2.0 dockerRepository: airbyte/source-stripe + documentationUrl: https://docs.airbyte.com/integrations/sources/stripe githubIssueLabel: source-stripe icon: stripe.svg license: ELv2 @@ -17,11 +23,29 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/stripe + releases: + breakingChanges: + 4.0.0: + message: + Version 4.0.0 changes the cursors in most of the Stripe streams that + support incremental sync mode. This is done to not only sync the data that + was created since previous sync, but also the data that was modified. A + schema refresh of all effected streams is required to use the new cursor + format. + upgradeDeadline: "2023-09-14" + 5.0.0: + message: + Version 5.0.0 introduces fixes for the `CheckoutSessions`, `CheckoutSessionsLineItems` and `Refunds` streams. The cursor field is changed for the `CheckoutSessionsLineItems` and `Refunds` streams. This will prevent data loss during incremental syncs. + Also, the `Invoices`, `Subscriptions` and `SubscriptionSchedule` stream schemas have been updated. + upgradeDeadline: "2023-12-11" + suggestedStreams: + streams: + - customers + - invoices + - charges + - subscriptions + - refunds + supportLevel: certified tags: - language:python - ab_internal: - sl: 300 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-stripe/setup.py b/airbyte-integrations/connectors/source-stripe/setup.py index 2ec69eca75f0..b059da570f32 100644 --- a/airbyte-integrations/connectors/source-stripe/setup.py +++ b/airbyte-integrations/connectors/source-stripe/setup.py @@ -5,14 +5,10 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk", "stripe==2.56.0", "pendulum==2.1.2"] +MAIN_REQUIREMENTS = ["airbyte-cdk==0.58.9", "stripe==2.56.0", "pendulum==2.1.2"] -TEST_REQUIREMENTS = [ - "pytest-mock~=3.6.1", - "pytest~=6.1", - "requests-mock", - "requests_mock~=1.8", -] +# we set `requests-mock~=1.11.0` to ensure concurrency is supported +TEST_REQUIREMENTS = ["pytest-mock~=3.6.1", "pytest~=6.1", "requests-mock~=1.11.0", "requests_mock~=1.8", "freezegun==1.2.2"] setup( name="source_stripe", @@ -25,4 +21,9 @@ extras_require={ "tests": TEST_REQUIREMENTS, }, + entry_points={ + "console_scripts": [ + "source-stripe=source_stripe.run:run", + ], + }, ) diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/availability_strategy.py b/airbyte-integrations/connectors/source-stripe/source_stripe/availability_strategy.py index 3d8caf860ee5..6226ffc12ea9 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/availability_strategy.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/availability_strategy.py @@ -3,13 +3,16 @@ # import logging -from typing import Optional, Tuple +from typing import Any, Mapping, Optional, Tuple +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import Source from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from requests import HTTPError +from .stream_helpers import get_first_record_for_slice, get_first_stream_slice + STRIPE_ERROR_CODES = { "more_permissions_required": "This is most likely due to insufficient permissions on the credentials in use. " "Try to grant required permissions/scopes or re-authenticate", @@ -20,6 +23,60 @@ class StripeAvailabilityStrategy(HttpAvailabilityStrategy): + def _check_availability_for_sync_mode( + self, + stream: Stream, + sync_mode: SyncMode, + logger: logging.Logger, + source: Optional["Source"], + stream_state: Optional[Mapping[str, Any]], + ) -> Tuple[bool, Optional[str]]: + try: + # Some streams need a stream slice to read records (e.g. if they have a SubstreamPartitionRouter) + # Streams that don't need a stream slice will return `None` as their first stream slice. + stream_slice = get_first_stream_slice(stream, sync_mode, stream_state) + except StopIteration: + # If stream_slices has no `next()` item (Note - this is different from stream_slices returning [None]!) + # This can happen when a substream's `stream_slices` method does a `for record in parent_records: yield ` + # without accounting for the case in which the parent stream is empty. + reason = f"Cannot attempt to connect to stream {stream.name} - no stream slices were found, likely because the parent stream is empty." + return False, reason + except HTTPError as error: + is_available, reason = self.handle_http_error(stream, logger, source, error) + if not is_available: + reason = f"Unable to get slices for {stream.name} stream, because of error in parent stream. {reason}" + return is_available, reason + + try: + get_first_record_for_slice(stream, sync_mode, stream_slice, stream_state) + return True, None + except StopIteration: + logger.info(f"Successfully connected to stream {stream.name}, but got 0 records.") + return True, None + except HTTPError as error: + is_available, reason = self.handle_http_error(stream, logger, source, error) + if not is_available: + reason = f"Unable to read {stream.name} stream. {reason}" + return is_available, reason + + def check_availability(self, stream: Stream, logger: logging.Logger, source: Optional["Source"]) -> Tuple[bool, Optional[str]]: + """ + Check stream availability by attempting to read the first record of the + stream. + + :param stream: stream + :param logger: source logger + :param source: (optional) source + :return: A tuple of (boolean, str). If boolean is true, then the stream + is available, and no str is required. Otherwise, the stream is unavailable + for some reason and the str should describe what went wrong and how to + resolve the unavailability, if possible. + """ + is_available, reason = self._check_availability_for_sync_mode(stream, SyncMode.full_refresh, logger, source, None) + if not is_available or not stream.supports_incremental: + return is_available, reason + return self._check_availability_for_sync_mode(stream, SyncMode.incremental, logger, source, {stream.cursor_field: 0}) + def handle_http_error( self, stream: Stream, logger: logging.Logger, source: Optional["Source"], error: HTTPError ) -> Tuple[bool, Optional[str]]: @@ -39,7 +96,7 @@ def handle_http_error( return False, reason -class StripeSubStreamAvailabilityStrategy(HttpAvailabilityStrategy): +class StripeSubStreamAvailabilityStrategy(StripeAvailabilityStrategy): def check_availability(self, stream: Stream, logger: logging.Logger, source: Optional[Source]) -> Tuple[bool, Optional[str]]: """Traverse through all the parents of a given stream and run availability strategy on each of them""" try: @@ -47,7 +104,7 @@ def check_availability(self, stream: Stream, logger: logging.Logger, source: Opt except AttributeError: return super().check_availability(stream, logger, source) if parent_stream: - parent_stream_instance = getattr(current_stream, "get_parent_stream_instance")() + parent_stream_instance = getattr(current_stream, "parent") # Accessing the `availability_strategy` property will instantiate AvailabilityStrategy under the hood availability_strategy = parent_stream_instance.availability_strategy if availability_strategy: diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/run.py b/airbyte-integrations/connectors/source-stripe/source_stripe/run.py new file mode 100644 index 000000000000..b72025a2c726 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/run.py @@ -0,0 +1,45 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys +import traceback +from datetime import datetime +from typing import List + +from airbyte_cdk.entrypoint import AirbyteEntrypoint, launch +from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteMessage, AirbyteTraceMessage, TraceType, Type +from source_stripe import SourceStripe + + +def _get_source(args: List[str]): + catalog_path = AirbyteEntrypoint.extract_catalog(args) + config_path = AirbyteEntrypoint.extract_config(args) + try: + return SourceStripe( + SourceStripe.read_catalog(catalog_path) if catalog_path else None, + SourceStripe.read_config(config_path) if config_path else None, + ) + except Exception as error: + print( + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.ERROR, + emitted_at=int(datetime.now().timestamp() * 1000), + error=AirbyteErrorTraceMessage( + message=f"Error starting the sync. This could be due to an invalid configuration or catalog. Please contact Support for assistance. Error: {error}", + stack_trace=traceback.format_exc(), + ), + ), + ).json() + ) + return None + + +def run(): + _args = sys.argv[1:] + source = _get_source(_args) + if source: + launch(source, _args) diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/application_fees.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/application_fees.json index cd520cca84e5..16e07b9dcb04 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/application_fees.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/application_fees.json @@ -31,6 +31,9 @@ "created": { "type": ["null", "number"] }, + "updated": { + "type": ["null", "integer"] + }, "currency": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/application_fees_refunds.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/application_fees_refunds.json index 1ad153698443..7f462a752dd7 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/application_fees_refunds.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/application_fees_refunds.json @@ -19,6 +19,9 @@ "created": { "type": ["null", "number"] }, + "updated": { + "type": ["null", "integer"] + }, "currency": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/authorizations.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/authorizations.json index 861a99722d0a..e6e562ce3510 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/authorizations.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/authorizations.json @@ -36,6 +36,9 @@ "created": { "type": ["null", "integer"] }, + "updated": { + "type": ["null", "integer"] + }, "currency": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/bank_accounts.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/bank_accounts.json index 8f44bb792566..90361867fd2d 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/bank_accounts.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/bank_accounts.json @@ -13,6 +13,9 @@ "account_holder_type": { "type": ["null", "string"] }, + "account_type": { + "type": ["null", "string"] + }, "bank_name": { "type": ["null", "string"] }, @@ -40,6 +43,12 @@ }, "status": { "type": ["null", "string"] + }, + "updated": { + "type": ["null", "integer"] + }, + "is_deleted": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/charges.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/charges.json index 0a6bb196269d..b275b90898eb 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/charges.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/charges.json @@ -348,6 +348,9 @@ "created": { "type": ["null", "integer"] }, + "updated": { + "type": ["null", "integer"] + }, "order": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions.json index 0d5ee3731c4c..b331e12c6427 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions.json @@ -213,6 +213,250 @@ } } }, - "url": { "type": ["null", "string"] } + "url": { "type": ["null", "string"] }, + "updated": { "type": ["null", "integer"] }, + "created": { "type": ["null", "integer"] }, + "currency_conversion": { + "type": ["null", "object"], + "properties": { + "amount_subtotal": { + "type": ["null", "integer"] + }, + "amount_total": { + "type": ["null", "integer"] + }, + "fix_rate": { + "type": ["null", "string"] + }, + "source_currency": { + "type": ["null", "string"] + } + } + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "dropdown": { + "type": ["null", "object"], + "properties": { + "options": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"] + } + }, + "value": { + "type": ["null", "string"] + } + } + }, + "key": { + "type": ["null", "string"] + }, + "label": { + "type": ["null", "object"], + "properties": { + "custom": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + }, + "numeric": { + "type": ["null", "object"], + "properties": { + "maximum_length": { + "type": ["null", "integer"] + }, + "minimum_length": { + "type": ["null", "integer"] + }, + "value": { + "type": ["null", "string"] + } + } + }, + "optional": { + "type": ["null", "boolean"] + }, + "text": { + "type": ["null", "object"], + "properties": { + "maximum_length": { + "type": ["null", "integer"] + }, + "minimum_length": { + "type": ["null", "integer"] + }, + "value": { + "type": ["null", "string"] + } + } + }, + "type": { + "type": ["null", "string"] + } + } + } + }, + "custom_text": { + "type": ["null", "object"], + "properties": { + "shipping_address": { + "type": ["null", "object"], + "properties": { + "message": { + "type": ["null", "string"] + } + } + }, + "submit": { + "type": ["null", "string"], + "properties": { + "message": { + "type": ["null", "string"] + } + } + }, + "terms_of_service": { + "type": ["null", "object"], + "properties": { + "message": { + "type": ["null", "string"] + } + } + } + } + }, + "customer_creation": { "type": ["null", "string"] }, + "invoice": { "type": ["null", "string"] }, + "invoice_creation": { + "type": ["null", "object"], + "properties": { + "enabled": { + "type": ["null", "boolean"] + }, + "invoice_data": { + "type": ["null", "object"], + "properties": { + "account_tax_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "custom_fields": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + }, + "description": { + "type": ["null", "string"] + }, + "footer": { + "type": ["null", "string"] + }, + "metadata": { + "type": ["null", "object"] + }, + "rendering_options": { + "type": ["null", "object"], + "properties": { + "amount_tax_display": { + "type": ["null", "string"] + } + } + } + } + } + } + }, + "payment_link": { "type": ["null", "string"] }, + "payment_method_collection": { "type": ["null", "string"] }, + "shipping_cost": { + "type": ["null", "object"], + "properties": { + "amount_total": { + "type": ["null", "integer"] + }, + "amount_subtotal": { + "type": ["null", "integer"] + }, + "amount_tax": { + "type": ["null", "integer"] + }, + "shipping_rate": { + "type": ["null", "string"] + }, + "taxes": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "rate": { + "$ref": "tax_rate.json" + }, + "taxability_reason": { + "type": ["null", "string"] + }, + "taxable_amount": { + "type": ["null", "integer"] + } + } + } + } + } + }, + "shipping_details": { + "type": ["null", "object"], + "properties": { + "address": { + "$ref": "address.json" + }, + "name": { + "type": ["null", "string"] + } + } + }, + "shipping_options": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "shipping_amount": { + "type": ["null", "integer"] + }, + "shipping_rate": { + "type": ["null", "string"] + } + } + } + }, + "status": { "type": ["null", "string"] }, + "payment_method_configuration_details": { + "$ref": "payment_method_configuration_details.json" + }, + "client_secret": { + "type": ["null", "string"] + }, + "ui_mode": { + "type": ["null", "string"] + } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions_line_items.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions_line_items.json index 053ce4f6d6cc..b00f6569d12e 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions_line_items.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/checkout_sessions_line_items.json @@ -5,8 +5,12 @@ "id": { "type": ["null", "string"] }, "checkout_session_id": { "type": ["null", "string"] }, "checkout_session_expires_at": { "type": ["null", "integer"] }, + "checkout_session_created": { "type": ["null", "integer"] }, + "checkout_session_updated": { "type": ["null", "integer"] }, "object": { "type": ["null", "string"] }, "amount_subtotal": { "type": ["null", "integer"] }, + "amount_tax": { "type": ["null", "integer"] }, + "amount_discount": { "type": ["null", "integer"] }, "amount_total": { "type": ["null", "integer"] }, "currency": { "type": ["null", "string"] }, "description": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/coupons.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/coupons.json index 44b5c61a8527..4f5a22146fe8 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/coupons.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/coupons.json @@ -47,8 +47,14 @@ "created": { "type": ["null", "integer"] }, + "updated": { + "type": ["null", "integer"] + }, "percent_off": { "type": ["null", "number"] + }, + "is_deleted": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/credit_notes.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/credit_notes.json index b5f962fc3101..bfcc21ceddc5 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/credit_notes.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/credit_notes.json @@ -18,6 +18,9 @@ "created": { "type": ["null", "integer"] }, + "updated": { + "type": ["null", "integer"] + }, "currency": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/disputes.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/disputes.json index b627b72dd7e0..d0983dcaacba 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/disputes.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/disputes.json @@ -27,6 +27,9 @@ "created": { "type": ["null", "integer"] }, + "updated": { + "type": ["null", "integer"] + }, "currency": { "type": ["null", "string"] }, @@ -154,6 +157,25 @@ }, "balance_transaction": { "type": ["null", "string"] + }, + "payment_method_details": { + "type": ["null", "object"], + "properties": { + "card": { + "type": ["null", "object"], + "properties": { + "brand": { + "type": ["null", "string"] + }, + "network_reason_code": { + "type": ["null", "string"] + } + } + }, + "type": { + "type": ["null", "string"] + } + } } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/early_fraud_warnings.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/early_fraud_warnings.json index 63b52a80af0d..0b2890c57a62 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/early_fraud_warnings.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/early_fraud_warnings.json @@ -19,6 +19,9 @@ "created": { "type": ["null", "number"] }, + "updated": { + "type": ["null", "integer"] + }, "fraud_type": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/external_account_bank_accounts.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/external_account_bank_accounts.json index 330c37645db0..872b617e0ecb 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/external_account_bank_accounts.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/external_account_bank_accounts.json @@ -43,6 +43,12 @@ }, "account": { "type": ["string", "null"] + }, + "updated": { + "type": ["null", "integer"] + }, + "is_deleted": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/external_account_cards.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/external_account_cards.json index 00c1203e15e7..ff161461449e 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/external_account_cards.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/external_account_cards.json @@ -73,6 +73,12 @@ }, "account": { "type": ["string", "null"] + }, + "updated": { + "type": ["null", "integer"] + }, + "is_deleted": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_items.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_items.json index 0c2f07854952..04142b340a3e 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_items.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_items.json @@ -93,8 +93,7 @@ "properties": {} }, "updated": { - "type": ["null", "string"], - "format": "date-time" + "type": ["null", "string"] } } }, @@ -121,6 +120,9 @@ "date": { "type": ["null", "integer"] }, + "updated": { + "type": ["null", "integer"] + }, "object": { "type": ["null", "string"] }, @@ -168,6 +170,9 @@ }, "unit_amount_decimal": { "type": ["null", "string"] + }, + "is_deleted": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_line_items.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_line_items.json index 3ade45cbe340..f59bb1a3a04b 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_line_items.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoice_line_items.json @@ -38,6 +38,12 @@ "livemode": { "type": ["null", "boolean"] }, + "margins": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, "proration": { "type": ["null", "boolean"] }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json index 8189cefc7df7..c21e5c93fd3e 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/invoices.json @@ -4,6 +4,9 @@ "created": { "type": ["null", "integer"] }, + "updated": { + "type": ["null", "integer"] + }, "next_payment_attempt": { "type": ["null", "number"] }, @@ -206,11 +209,7 @@ "type": ["null", "integer"] }, "lines": { - "type": ["null", "array", "object"], - "items": { - "type": ["null", "string"] - }, - "properties": {} + "type": ["null", "object"] }, "forgiven": { "type": ["null", "boolean"] @@ -236,6 +235,14 @@ "subscription": { "type": ["null", "string"] }, + "subscription_details": { + "type": ["null", "object"], + "properties": { + "metadata": { + "type": ["null", "object"] + } + } + }, "status": { "type": ["null", "string"] }, @@ -492,10 +499,7 @@ "type": ["null", "integer"] }, "default_tax_rates": { - "type": ["null", "array"], - "items": { - "$ref": "tax_rates.json" - } + "$ref": "tax_rates.json" }, "total_excluding_tax": { "type": ["null", "integer"] @@ -526,6 +530,14 @@ } } }, + "issuer": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + } + } + }, "latest_revision": { "type": ["null", "string"] }, @@ -586,6 +598,25 @@ }, "account_country": { "type": ["null", "string"] + }, + "is_deleted": { + "type": ["null", "boolean"] + }, + "rendering": { + "type": ["object", "null"], + "properties": { + "amount_tax_display": { + "type": ["null", "string"] + }, + "pdf": { + "type": ["null", "object"], + "properties": { + "page_size": { + "type": ["null", "string"] + } + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/payouts.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/payouts.json index a057a81ec4fb..1afc989ceda0 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/payouts.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/payouts.json @@ -22,6 +22,9 @@ "created": { "type": ["null", "integer"] }, + "updated": { + "type": ["null", "integer"] + }, "amount_reversed": { "type": ["null", "integer"] }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/persons.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/persons.json index a409b35c1551..8a4cfb640ee5 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/persons.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/persons.json @@ -75,6 +75,9 @@ "created": { "type": ["null", "integer"] }, + "updated": { + "type": ["null", "integer"] + }, "dob": { "type": ["null", "object"], "properties": { @@ -417,6 +420,9 @@ "type": ["null", "string"] } } + }, + "is_deleted": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/plans.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/plans.json index 2b46c1435c85..2cdf3d8f2340 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/plans.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/plans.json @@ -36,6 +36,9 @@ "created": { "type": ["null", "integer"] }, + "updated": { + "type": ["null", "integer"] + }, "statement_description": { "type": ["null", "string"] }, @@ -90,6 +93,9 @@ }, "amount_decimal": { "type": ["null", "string"] + }, + "is_deleted": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/prices.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/prices.json index 699186837509..4ec44e6f6d9f 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/prices.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/prices.json @@ -19,6 +19,9 @@ "created": { "type": ["null", "integer"] }, + "updated": { + "type": ["null", "integer"] + }, "currency": { "type": ["null", "string"] }, @@ -82,6 +85,9 @@ }, "unit_amount_decimal": { "type": ["null", "string"] + }, + "is_deleted": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/products.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/products.json index b7416f78a356..e7db7e052c60 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/products.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/products.json @@ -87,6 +87,20 @@ }, "tax_code": { "type": ["null", "string"] + }, + "features": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + } + } + } + }, + "is_deleted": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/promotion_codes.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/promotion_codes.json index c602c4398f13..0d487173b4a4 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/promotion_codes.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/promotion_codes.json @@ -43,6 +43,7 @@ "object": { "type": ["null", "string"] }, "active": { "type": ["null", "boolean"] }, "created": { "type": ["null", "integer"] }, + "updated": { "type": ["null", "integer"] }, "customer": { "type": ["null", "string"] }, "expires_at": { "type": ["null", "integer"] }, "livemode": { "type": ["null", "boolean"] }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/refunds.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/refunds.json index 89800f4f9318..1e44cce8b5ca 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/refunds.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/refunds.json @@ -19,6 +19,9 @@ "created": { "type": ["null", "integer"] }, + "updated": { + "type": ["null", "integer"] + }, "currency": { "type": ["null", "string"] }, @@ -43,6 +46,31 @@ }, "transfer_reversal": { "type": ["null", "string"] + }, + "destination_details": { + "type": ["null", "object"], + "properties": { + "type": { + "type": ["null", "string"] + }, + "card": { + "type": ["null", "object"], + "properties": { + "reference": { + "type": ["null", "string"] + }, + "reference_status": { + "type": ["null", "string"] + }, + "reference_type": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/reviews.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/reviews.json index aef12a767d8a..2f00ebc7a6d8 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/reviews.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/reviews.json @@ -15,6 +15,9 @@ "created": { "type": ["null", "integer"] }, + "updated": { + "type": ["null", "integer"] + }, "id": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/setup_attempts.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/setup_attempts.json index 5b6d7122987d..84ac87118e66 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/setup_attempts.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/setup_attempts.json @@ -186,7 +186,268 @@ "$ref": "payment_intent.json" }, "payment_method": { - "$ref": "payment_method.json" + "type": ["null", "object"], + "properties": { + "afterpay_clearpay": { + "additionalProperties": true, + "type": ["null", "object"] + }, + "alipay": { + "additionalProperties": true, + "type": ["null", "object"] + }, + "au_becs_debit": { + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "bsb_number": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + } + } + }, + "bacs_debit": { + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "fingerprint": { + "type": ["null", "string"] + }, + "last4": { + "type": ["null", "string"] + }, + "sort_code": { + "type": ["null", "string"] + } + } + }, + "bancontact": { + "additionalProperties": true, + "type": ["null", "object"] + }, + "billing_details": { + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "address": { + "$ref": "address.json" + }, + "email": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + } + } + }, + "card": { + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "brand": { + "type": ["null", "string"] + }, + "checks": { + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "address_line1_check": { + "type": ["null", "string"] + }, + "address_postal_code_check": { + "type": ["null", "string"] + }, + "cvc_check": { + "type": ["null", "string"] + } + } + }, + "country": { + "type": ["null", "string"] + }, + "exp_month": { + "type": ["null", "integer"] + }, + "exp_year": { + "type": ["null", "integer"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "funding": { + "type": ["null", "string"] + }, + "generated_from": { + "additionalProperties": true, + "type": ["null", "object"] + }, + "last4": { + "type": ["null", "string"] + }, + "networks": { + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "available": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "preferred": { + "type": ["null", "string"] + } + } + }, + "three_d_secure_usage": { + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "supported": { + "type": ["null", "boolean"] + } + } + }, + "wallet": { + "additionalProperties": true, + "type": ["null", "object"] + } + } + }, + "card_present": { + "additionalProperties": true, + "type": ["null", "object"] + }, + "created": { + "type": ["null", "integer"] + }, + "updated": { + "type": ["null", "integer"] + }, + "customer": { + "type": ["null", "string"] + }, + "eps": { + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "bank": { + "type": ["null", "string"] + } + } + }, + "fpx": { + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "bank": { + "type": ["null", "string"] + } + } + }, + "giropay": { + "additionalProperties": true, + "type": ["null", "object"] + }, + "grabpay": { + "additionalProperties": true, + "type": ["null", "object"] + }, + "id": { + "type": ["null", "string"] + }, + "ideal": { + "additionalProperties": true, + "type": ["null", "object"], + "properties": { + "bank": { + "type": ["null", "string"] + }, + "bic": { + "type": ["null", "string"] + } + } + }, + "interac_present": { + "additionalProperties": true, + "type": ["null", "object"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "metadata": { + "additionalProperties": true, + "type": ["null", "object"] + }, + "object": { + "type": ["null", "string"] + }, + "oxxo": { + "additionalProperties": true, + "type": ["null", "object"] + }, + "p24": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "bank": { + "type": ["null", "string"] + } + } + }, + "sepa_debit": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "bank_code": { + "type": ["null", "string"] + }, + "branch_code": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "fingerprint": { + "type": ["null", "string"] + }, + "generated_from": { + "type": ["null", "object"], + "properties": { + "charge": { + "type": ["null", "string"] + }, + "setup_attempt": { + "type": ["null", "string"] + } + } + }, + "last4": { + "type": ["null", "string"] + } + } + }, + "sofort": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "country": { + "type": ["null", "string"] + } + } + }, + "type": { + "type": ["null", "string"] + } + } }, "payment_method_type": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/card.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/card.json index 532e23a24519..8b4e9239adc3 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/card.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/card.json @@ -13,6 +13,9 @@ "created": { "type": ["null", "integer"] }, + "updated": { + "type": ["null", "integer"] + }, "currency": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/cardholder.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/cardholder.json index 8c48e4e47803..1ff421aba58c 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/cardholder.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/cardholder.json @@ -20,6 +20,9 @@ "created": { "type": ["null", "integer"] }, + "updated": { + "type": ["null", "integer"] + }, "email": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/customer.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/customer.json index 9d7850a5d562..075a50a3369c 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/customer.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/customer.json @@ -833,11 +833,17 @@ "created": { "type": ["null", "integer"] }, + "updated": { + "type": ["null", "integer"] + }, "tax_info": { "type": ["null", "string"] }, "test_clock": { "type": ["null", "string"] + }, + "is_deleted": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/payment_intent.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/payment_intent.json index ba4e006c1ee8..176df702afe7 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/payment_intent.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/payment_intent.json @@ -62,6 +62,9 @@ "created": { "type": ["null", "integer"] }, + "updated": { + "type": ["null", "integer"] + }, "currency": { "type": ["null", "string"] }, @@ -887,6 +890,9 @@ "type": ["null", "boolean"] } } + }, + "payment_method_configuration_details": { + "$ref": "payment_method_configuration_details.json" } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/payment_method.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/payment_method.json index e181f7ea0e9b..29d7eef47d6e 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/payment_method.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/payment_method.json @@ -142,6 +142,9 @@ "created": { "type": ["null", "integer"] }, + "updated": { + "type": ["null", "integer"] + }, "customer": { "$ref": "customer.json" }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/payment_method_configuration_details.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/payment_method_configuration_details.json new file mode 100644 index 000000000000..96d8e75c41bd --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/payment_method_configuration_details.json @@ -0,0 +1,11 @@ +{ + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "parent": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/setup_intent.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/setup_intent.json index 878fae7e2f0f..e92c4fc79224 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/setup_intent.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/setup_intent.json @@ -17,6 +17,9 @@ "created": { "type": ["integer"] }, + "updated": { + "type": ["null", "integer"] + }, "customer": { "type": ["null", "string"] }, @@ -86,6 +89,9 @@ "type": ["null", "boolean"] } } + }, + "payment_method_configuration_details": { + "$ref": "payment_method_configuration_details.json" } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/tax_rate.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/tax_rate.json new file mode 100644 index 000000000000..5468844c8226 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/tax_rate.json @@ -0,0 +1,50 @@ +{ + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "object": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + }, + "country": { + "type": ["null", "string"] + }, + "created": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + }, + "display_name": { + "type": ["null", "string"] + }, + "effective_percentage": { + "type": ["null", "number"] + }, + "inclusive": { + "type": ["null", "boolean"] + }, + "jurisdiction": { + "type": ["null", "string"] + }, + "livemode": { + "type": ["null", "boolean"] + }, + "metadata": { + "type": ["null", "object"] + }, + "percentage": { + "type": ["null", "number"] + }, + "state": { + "type": ["null", "string"] + }, + "tax_type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/tax_rates.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/tax_rates.json index ee5b078c5d73..705a7c1a9ee7 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/tax_rates.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/shared/tax_rates.json @@ -1,53 +1,6 @@ { "type": ["null", "array"], "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "object": { - "type": ["null", "string"] - }, - "active": { - "type": ["null", "boolean"] - }, - "country": { - "type": ["null", "string"] - }, - "created": { - "type": ["null", "integer"] - }, - "description": { - "type": ["null", "string"] - }, - "display_name": { - "type": ["null", "string"] - }, - "effective_percentage": { - "type": ["null", "number"] - }, - "inclusive": { - "type": ["null", "boolean"] - }, - "jurisdiction": { - "type": ["null", "string"] - }, - "livemode": { - "type": ["null", "boolean"] - }, - "metadata": { - "type": ["null", "object"] - }, - "percentage": { - "type": ["null", "number"] - }, - "state": { - "type": ["null", "string"] - }, - "tax_type": { - "type": ["null", "string"] - } - } + "$ref": "tax_rate.json" } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_items.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_items.json index c6d36caab561..40186badb810 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_items.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_items.json @@ -6,12 +6,10 @@ "properties": {} }, "canceled_at": { - "type": ["null", "string"], - "format": "date-time" + "type": ["null", "string"] }, "current_period_end": { - "type": ["null", "string"], - "format": "date-time" + "type": ["null", "string"] }, "plan": { "type": ["null", "object", "string"], diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_schedule.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_schedule.json index e8729bfefab4..cc14a57138fd 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_schedule.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscription_schedule.json @@ -21,6 +21,9 @@ "created": { "type": ["null", "integer"] }, + "updated": { + "type": ["null", "integer"] + }, "current_phase": { "type": ["null", "object"], "additionalProperties": true, @@ -104,54 +107,7 @@ "type": ["null", "string"] }, "tax_rates": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "additionalProperties": true, - "properties": { - "id": { - "type": ["null", "string"] - }, - "object": { - "type": ["null", "string"] - }, - "active": { - "type": ["null", "boolean"] - }, - "country": { - "type": ["null", "string"] - }, - "created": { - "type": ["null", "string"], - "format": "date-time" - }, - "display_name": { - "type": ["null", "string"] - }, - "inclusive": { - "type": ["null", "boolean"] - }, - "jurisdiction": { - "type": ["null", "string"] - }, - "livemode": { - "type": ["null", "boolean"] - }, - "metadata": { - "type": ["null", "object"], - "additionalProperties": true - }, - "percentage": { - "type": ["null", "number"] - }, - "state": { - "type": ["null", "string"] - }, - "tax_type": { - "type": ["null", "string"] - } - } - } + "$ref": "tax_rates.json" } } } @@ -178,11 +134,7 @@ "type": ["null", "string"] }, "default_tax_rates": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "additionalProperties": true - } + "$ref": "tax_rates.json" }, "description": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json index 405647bd85cc..89f180cd4532 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/subscriptions.json @@ -49,6 +49,12 @@ "billing_cycle_anchor": { "type": ["null", "number"] }, + "billing_cycle_anchor_config": { + "type": ["null", "object"] + }, + "invoice_settings": { + "type": ["null", "object"] + }, "cancel_at_period_end": { "type": ["null", "boolean"] }, @@ -233,6 +239,9 @@ "created": { "type": ["null", "integer"] }, + "updated": { + "type": ["null", "integer"] + }, "ended_at": { "type": ["null", "number"] }, @@ -320,10 +329,7 @@ } }, "default_tax_rates": { - "type": ["null", "array"], - "items": { - "$ref": "tax_rates.json" - } + "$ref": "tax_rates.json" }, "pause_collection": { "type": ["null", "object"], @@ -442,6 +448,9 @@ "type": ["null", "boolean"] } } + }, + "is_deleted": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/top_ups.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/top_ups.json index ec3424a67763..8d25d5cb7a4b 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/top_ups.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/top_ups.json @@ -31,6 +31,9 @@ "created": { "type": ["null", "integer"] }, + "updated": { + "type": ["null", "integer"] + }, "destination_balance": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/transactions.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/transactions.json index fed1ef8855e9..74af29cec809 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/transactions.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/transactions.json @@ -29,6 +29,9 @@ "created": { "type": ["null", "integer"] }, + "updated": { + "type": ["null", "integer"] + }, "currency": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/transfers.json b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/transfers.json index 45ace8c56e4f..70248975b80e 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/transfers.json +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/schemas/transfers.json @@ -42,6 +42,9 @@ "created": { "type": ["null", "integer"] }, + "updated": { + "type": ["null", "integer"] + }, "amount_reversed": { "type": ["null", "integer"] }, @@ -86,6 +89,9 @@ }, "description": { "type": ["null", "string"] + }, + "destination_payment": { + "type": ["null", "string"] } }, "type": ["null", "object"] diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py index a2f2b3303a12..ebb5dd7a1a00 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/source.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/source.py @@ -2,129 +2,534 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -from typing import Any, List, Mapping, Tuple +import logging +import os +from datetime import timedelta +from typing import Any, List, Mapping, MutableMapping, Optional, Tuple import pendulum import stripe from airbyte_cdk import AirbyteLogger -from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.entrypoint import logger as entrypoint_logger +from airbyte_cdk.models import ConfiguredAirbyteCatalog, FailureType +from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource +from airbyte_cdk.sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter +from airbyte_cdk.sources.message.repository import InMemoryMessageRepository from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.call_rate import AbstractAPIBudget, HttpAPIBudget, HttpRequestMatcher, MovingWindowCallRatePolicy, Rate +from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade +from airbyte_cdk.sources.streams.concurrent.cursor import NoopCursor from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from airbyte_cdk.utils.traced_exception import AirbyteTracedException +from airbyte_protocol.models import SyncMode from source_stripe.streams import ( - Accounts, - ApplicationFees, - ApplicationFeesRefunds, - Authorizations, - BalanceTransactions, - BankAccounts, - Cardholders, - Cards, - Charges, - CheckoutSessions, - CheckoutSessionsLineItems, - Coupons, - CreditNotes, + CreatedCursorIncrementalStripeStream, CustomerBalanceTransactions, - Customers, - Disputes, - EarlyFraudWarnings, Events, - ExternalAccountBankAccounts, - ExternalAccountCards, - FileLinks, - Files, - InvoiceItems, - InvoiceLineItems, - Invoices, - PaymentIntents, - PaymentMethods, - Payouts, + IncrementalStripeStream, + ParentIncrementalStipeSubStream, Persons, - Plans, - Prices, - Products, - PromotionCodes, - Refunds, - Reviews, SetupAttempts, - SetupIntents, - ShippingRates, - SubscriptionItems, - Subscriptions, - SubscriptionSchedule, - TopUps, - Transactions, - TransferReversals, - Transfers, - UsageRecords, + StripeLazySubStream, + StripeStream, + StripeSubStream, + UpdatedCursorIncrementalStripeLazySubStream, + UpdatedCursorIncrementalStripeStream, ) +logger = logging.getLogger("airbyte") + +_MAX_CONCURRENCY = 20 +_DEFAULT_CONCURRENCY = 10 +_CACHE_DISABLED = os.environ.get("CACHE_DISABLED") +USE_CACHE = not _CACHE_DISABLED +STRIPE_TEST_ACCOUNT_PREFIX = "sk_test_" + + +class SourceStripe(ConcurrentSourceAdapter): + + message_repository = InMemoryMessageRepository(entrypoint_logger.level) + + def __init__(self, catalog: Optional[ConfiguredAirbyteCatalog], config: Optional[Mapping[str, Any]], **kwargs): + if config: + concurrency_level = min(config.get("num_workers", _DEFAULT_CONCURRENCY), _MAX_CONCURRENCY) + else: + concurrency_level = _DEFAULT_CONCURRENCY + logger.info(f"Using concurrent cdk with concurrency level {concurrency_level}") + concurrent_source = ConcurrentSource.create( + concurrency_level, concurrency_level // 2, logger, self._slice_logger, self.message_repository + ) + super().__init__(concurrent_source) + if catalog: + self._streams_configured_as_full_refresh = { + configured_stream.stream.name + for configured_stream in catalog.streams + if configured_stream.sync_mode == SyncMode.full_refresh + } + else: + # things will NOT be executed concurrently + self._streams_configured_as_full_refresh = set() + + @staticmethod + def validate_and_fill_with_defaults(config: MutableMapping) -> MutableMapping: + start_date, lookback_window_days, slice_range = ( + config.get("start_date"), + config.get("lookback_window_days"), + config.get("slice_range"), + ) + if lookback_window_days is None: + config["lookback_window_days"] = 0 + elif not isinstance(lookback_window_days, int) or lookback_window_days < 0: + message = f"Invalid lookback window {lookback_window_days}. Please use only positive integer values or 0." + raise AirbyteTracedException( + message=message, + internal_message=message, + failure_type=FailureType.config_error, + ) + if start_date: + # verifies the start_date is parseable + SourceStripe._start_date_to_timestamp(start_date) + if slice_range is None: + config["slice_range"] = 365 + elif not isinstance(slice_range, int) or slice_range < 1: + message = f"Invalid slice range value {slice_range}. Please use positive integer values only." + raise AirbyteTracedException( + message=message, + internal_message=message, + failure_type=FailureType.config_error, + ) + return config -class SourceStripe(AbstractSource): def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: + self.validate_and_fill_with_defaults(config) + stripe.api_key = config["client_secret"] try: - stripe.api_key = config["client_secret"] stripe.Account.retrieve(config["account_id"]) - return True, None - except Exception as e: - return False, e + except (stripe.error.AuthenticationError, stripe.error.PermissionError) as e: + return False, str(e) + return True, None + + @staticmethod + def customers(**args): + # The Customers stream is instantiated in a dedicated method to allow parametrization and avoid duplicated code. + # It can be used with and without expanded items (as an independent stream or as a parent stream for other streams). + return IncrementalStripeStream( + name="customers", + path="customers", + use_cache=USE_CACHE, + event_types=["customer.created", "customer.updated", "customer.deleted"], + **args, + ) + + @staticmethod + def is_test_account(config: Mapping[str, Any]) -> bool: + """Check if configuration uses Stripe test account (https://stripe.com/docs/keys#obtain-api-keys) + + :param config: + :return: True if configured to use a test account, False - otherwise + """ + + return str(config["client_secret"]).startswith(STRIPE_TEST_ACCOUNT_PREFIX) + + def get_api_call_budget(self, config: Mapping[str, Any]) -> AbstractAPIBudget: + """Get API call budget which connector is allowed to use. + + :param config: + :return: + """ + + max_call_rate = 25 if self.is_test_account(config) else 100 + if config.get("call_rate_limit"): + call_limit = config["call_rate_limit"] + if call_limit > max_call_rate: + logger.warning( + "call_rate_limit is larger than maximum allowed %s, fallback to default %s.", + max_call_rate, + max_call_rate, + ) + call_limit = max_call_rate + else: + call_limit = max_call_rate + + policies = [ + MovingWindowCallRatePolicy( + rates=[Rate(limit=20, interval=timedelta(seconds=1))], + matchers=[ + HttpRequestMatcher(url="https://api.stripe.com/v1/files"), + HttpRequestMatcher(url="https://api.stripe.com/v1/file_links"), + ], + ), + MovingWindowCallRatePolicy( + rates=[Rate(limit=call_limit, interval=timedelta(seconds=1))], + matchers=[], + ), + ] + + return HttpAPIBudget(policies=policies) def streams(self, config: Mapping[str, Any]) -> List[Stream]: + config = self.validate_and_fill_with_defaults(config) authenticator = TokenAuthenticator(config["client_secret"]) - start_date = pendulum.parse(config["start_date"]).int_timestamp + + if "start_date" in config: + start_timestamp = self._start_date_to_timestamp(config["start_date"]) + else: + start_timestamp = pendulum.datetime(2017, 1, 25).int_timestamp args = { "authenticator": authenticator, "account_id": config["account_id"], - "start_date": start_date, - "slice_range": config.get("slice_range"), + "start_date": start_timestamp, + "slice_range": config["slice_range"], + "api_budget": self.get_api_call_budget(config), } - incremental_args = {**args, "lookback_window_days": config.get("lookback_window_days")} - return [ - Accounts(**args), - ApplicationFees(**incremental_args), - ApplicationFeesRefunds(**args), - Authorizations(**incremental_args), - BalanceTransactions(**incremental_args), - BankAccounts(**args), - Cardholders(**incremental_args), - Cards(**incremental_args), - Charges(**incremental_args), - CheckoutSessions(**args), - CheckoutSessionsLineItems(**args), - Coupons(**incremental_args), - CreditNotes(**args), + incremental_args = {**args, "lookback_window_days": config["lookback_window_days"]} + subscriptions = IncrementalStripeStream( + name="subscriptions", + path="subscriptions", + use_cache=USE_CACHE, + extra_request_params={"status": "all"}, + event_types=[ + "customer.subscription.created", + "customer.subscription.paused", + "customer.subscription.pending_update_applied", + "customer.subscription.pending_update_expired", + "customer.subscription.resumed", + "customer.subscription.trial_will_end", + "customer.subscription.updated", + "customer.subscription.deleted", + ], + **args, + ) + subscription_items = StripeLazySubStream( + name="subscription_items", + path="subscription_items", + extra_request_params=lambda self, stream_slice, *args, **kwargs: {"subscription": stream_slice["parent"]["id"]}, + parent=subscriptions, + use_cache=USE_CACHE, + sub_items_attr="items", + **args, + ) + transfers = IncrementalStripeStream( + name="transfers", + path="transfers", + use_cache=USE_CACHE, + event_types=["transfer.created", "transfer.reversed", "transfer.updated"], + **args, + ) + application_fees = IncrementalStripeStream( + name="application_fees", + path="application_fees", + use_cache=USE_CACHE, + event_types=["application_fee.created", "application_fee.refunded"], + **args, + ) + invoices = IncrementalStripeStream( + name="invoices", + path="invoices", + use_cache=USE_CACHE, + event_types=[ + "invoice.created", + "invoice.finalization_failed", + "invoice.finalized", + "invoice.marked_uncollectible", + "invoice.paid", + "invoice.payment_action_required", + "invoice.payment_failed", + "invoice.payment_succeeded", + "invoice.sent", + "invoice.updated", + "invoice.voided", + "invoice.deleted", + ], + **args, + ) + checkout_sessions = UpdatedCursorIncrementalStripeStream( + name="checkout_sessions", + path="checkout/sessions", + use_cache=USE_CACHE, + legacy_cursor_field="created", + event_types=[ + "checkout.session.async_payment_failed", + "checkout.session.async_payment_succeeded", + "checkout.session.completed", + "checkout.session.expired", + ], + **args, + ) + + streams = [ + checkout_sessions, CustomerBalanceTransactions(**args), - Customers(**incremental_args), - Disputes(**incremental_args), - EarlyFraudWarnings(**args), Events(**incremental_args), - ExternalAccountBankAccounts(**args), - ExternalAccountCards(**args), - FileLinks(**incremental_args), - Files(**incremental_args), - InvoiceItems(**incremental_args), - InvoiceLineItems(**args), - Invoices(**incremental_args), - PaymentIntents(**incremental_args), - PaymentMethods(**args), - Payouts(**incremental_args), - Persons(**incremental_args), - Plans(**incremental_args), - Prices(**incremental_args), - Products(**incremental_args), - PromotionCodes(**incremental_args), - Refunds(**incremental_args), - Reviews(**incremental_args), + UpdatedCursorIncrementalStripeStream( + name="external_account_cards", + path=lambda self, *args, **kwargs: f"accounts/{self.account_id}/external_accounts", + event_types=["account.external_account.created", "account.external_account.updated", "account.external_account.deleted"], + legacy_cursor_field=None, + extra_request_params={"object": "card"}, + response_filter=lambda record: record["object"] == "card", + **args, + ), + UpdatedCursorIncrementalStripeStream( + name="external_account_bank_accounts", + path=lambda self, *args, **kwargs: f"accounts/{self.account_id}/external_accounts", + event_types=["account.external_account.created", "account.external_account.updated", "account.external_account.deleted"], + legacy_cursor_field=None, + extra_request_params={"object": "bank_account"}, + response_filter=lambda record: record["object"] == "bank_account", + **args, + ), + Persons(**args), SetupAttempts(**incremental_args), - SetupIntents(**incremental_args), - ShippingRates(**incremental_args), - SubscriptionItems(**args), - Subscriptions(**incremental_args), - SubscriptionSchedule(**incremental_args), - TopUps(**incremental_args), - Transactions(**incremental_args), - TransferReversals(**args), - Transfers(**incremental_args), - UsageRecords(**args), + StripeStream(name="accounts", path="accounts", use_cache=USE_CACHE, **args), + CreatedCursorIncrementalStripeStream(name="shipping_rates", path="shipping_rates", **incremental_args), + CreatedCursorIncrementalStripeStream(name="balance_transactions", path="balance_transactions", **incremental_args), + CreatedCursorIncrementalStripeStream(name="files", path="files", **incremental_args), + CreatedCursorIncrementalStripeStream(name="file_links", path="file_links", **incremental_args), + # The Refunds stream does not utilize the Events API as it created issues with data loss during the incremental syncs. + # Therefore, we're using the regular API with the `created` cursor field. A bug has been filed with Stripe. + # See more at https://github.com/airbytehq/oncall/issues/3090, https://github.com/airbytehq/oncall/issues/3428 + CreatedCursorIncrementalStripeStream(name="refunds", path="refunds", **incremental_args), + UpdatedCursorIncrementalStripeStream( + name="payment_methods", + path="payment_methods", + event_types=[ + "payment_method.attached", + "payment_method.automatically_updated", + "payment_method.detached", + "payment_method.updated", + ], + **args, + ), + UpdatedCursorIncrementalStripeStream( + name="credit_notes", + path="credit_notes", + event_types=["credit_note.created", "credit_note.updated", "credit_note.voided"], + **args, + ), + UpdatedCursorIncrementalStripeStream( + name="early_fraud_warnings", + path="radar/early_fraud_warnings", + event_types=["radar.early_fraud_warning.created", "radar.early_fraud_warning.updated"], + **args, + ), + IncrementalStripeStream( + name="authorizations", + path="issuing/authorizations", + event_types=["issuing_authorization.created", "issuing_authorization.request", "issuing_authorization.updated"], + **args, + ), + self.customers(**args), + IncrementalStripeStream( + name="cardholders", + path="issuing/cardholders", + event_types=["issuing_cardholder.created", "issuing_cardholder.updated"], + **args, + ), + IncrementalStripeStream( + name="charges", + path="charges", + expand_items=["data.refunds"], + event_types=[ + "charge.captured", + "charge.expired", + "charge.failed", + "charge.pending", + "charge.refunded", + "charge.succeeded", + "charge.updated", + ], + **args, + ), + IncrementalStripeStream( + name="coupons", path="coupons", event_types=["coupon.created", "coupon.updated", "coupon.deleted"], **args + ), + IncrementalStripeStream( + name="disputes", + path="disputes", + event_types=[ + "charge.dispute.closed", + "charge.dispute.created", + "charge.dispute.funds_reinstated", + "charge.dispute.funds_withdrawn", + "charge.dispute.updated", + ], + **args, + ), + application_fees, + invoices, + IncrementalStripeStream( + name="invoice_items", + path="invoiceitems", + legacy_cursor_field="date", + event_types=["invoiceitem.created", "invoiceitem.updated", "invoiceitem.deleted"], + **args, + ), + IncrementalStripeStream( + name="payouts", + path="payouts", + event_types=[ + "payout.canceled", + "payout.created", + "payout.failed", + "payout.paid", + "payout.reconciliation_completed", + "payout.updated", + ], + **args, + ), + IncrementalStripeStream( + name="plans", + path="plans", + expand_items=["data.tiers"], + event_types=["plan.created", "plan.updated", "plan.deleted"], + **args, + ), + IncrementalStripeStream(name="prices", path="prices", event_types=["price.created", "price.updated", "price.deleted"], **args), + IncrementalStripeStream( + name="products", path="products", event_types=["product.created", "product.updated", "product.deleted"], **args + ), + IncrementalStripeStream(name="reviews", path="reviews", event_types=["review.closed", "review.opened"], **args), + subscriptions, + IncrementalStripeStream( + name="subscription_schedule", + path="subscription_schedules", + event_types=[ + "subscription_schedule.aborted", + "subscription_schedule.canceled", + "subscription_schedule.completed", + "subscription_schedule.created", + "subscription_schedule.expiring", + "subscription_schedule.released", + "subscription_schedule.updated", + ], + **args, + ), + transfers, + IncrementalStripeStream( + name="payment_intents", + path="payment_intents", + event_types=[ + "payment_intent.amount_capturable_updated", + "payment_intent.canceled", + "payment_intent.created", + "payment_intent.partially_funded", + "payment_intent.payment_failed", + "payment_intent.processing", + "payment_intent.requires_action", + "payment_intent.succeeded", + ], + **args, + ), + IncrementalStripeStream( + name="promotion_codes", + path="promotion_codes", + event_types=["promotion_code.created", "promotion_code.updated"], + **args, + ), + IncrementalStripeStream( + name="setup_intents", + path="setup_intents", + event_types=[ + "setup_intent.canceled", + "setup_intent.created", + "setup_intent.requires_action", + "setup_intent.setup_failed", + "setup_intent.succeeded", + ], + **args, + ), + IncrementalStripeStream( + name="cards", path="issuing/cards", event_types=["issuing_card.created", "issuing_card.updated"], **args + ), + IncrementalStripeStream( + name="transactions", + path="issuing/transactions", + event_types=["issuing_transaction.created", "issuing_transaction.updated"], + **args, + ), + IncrementalStripeStream( + name="top_ups", + path="topups", + event_types=["topup.canceled", "topup.created", "topup.failed", "topup.reversed", "topup.succeeded"], + **args, + ), + UpdatedCursorIncrementalStripeLazySubStream( + name="application_fees_refunds", + path=lambda self, stream_slice, *args, **kwargs: f"application_fees/{stream_slice['parent']['id']}/refunds", + parent=application_fees, + event_types=["application_fee.refund.updated"], + sub_items_attr="refunds", + **args, + ), + UpdatedCursorIncrementalStripeLazySubStream( + name="bank_accounts", + path=lambda self, stream_slice, *args, **kwargs: f"customers/{stream_slice['parent']['id']}/bank_accounts", + parent=self.customers(expand_items=["data.sources"], **args), + event_types=["customer.source.created", "customer.source.expiring", "customer.source.updated", "customer.source.deleted"], + legacy_cursor_field=None, + sub_items_attr="sources", + response_filter=lambda record: record["object"] == "bank_account", + **args, + ), + ParentIncrementalStipeSubStream( + name="checkout_sessions_line_items", + path=lambda self, stream_slice, *args, **kwargs: f"checkout/sessions/{stream_slice['parent']['id']}/line_items", + parent=checkout_sessions, + expand_items=["data.discounts", "data.taxes"], + cursor_field="checkout_session_updated", + slice_data_retriever=lambda record, stream_slice: { + "checkout_session_id": stream_slice["parent"]["id"], + "checkout_session_expires_at": stream_slice["parent"]["expires_at"], + "checkout_session_created": stream_slice["parent"]["created"], + "checkout_session_updated": stream_slice["parent"]["updated"], + **record, + }, + **args, + ), + StripeLazySubStream( + name="invoice_line_items", + path=lambda self, stream_slice, *args, **kwargs: f"invoices/{stream_slice['parent']['id']}/lines", + parent=invoices, + sub_items_attr="lines", + slice_data_retriever=lambda record, stream_slice: {"invoice_id": stream_slice["parent"]["id"], **record}, + **args, + ), + subscription_items, + StripeSubStream( + name="transfer_reversals", + path=lambda self, stream_slice, *args, **kwargs: f"transfers/{stream_slice['parent']['id']}/reversals", + parent=transfers, + **args, + ), + StripeSubStream( + name="usage_records", + path=lambda self, stream_slice, *args, **kwargs: f"subscription_items/{stream_slice['parent']['id']}/usage_record_summaries", + parent=subscription_items, + primary_key=None, + **args, + ), ] + + return [ + StreamFacade.create_from_stream(stream, self, entrypoint_logger, self._create_empty_state(), NoopCursor()) + if stream.name in self._streams_configured_as_full_refresh + else stream + for stream in streams + ] + + def _create_empty_state(self) -> MutableMapping[str, Any]: + # The state is known to be empty because concurrent CDK is currently only used for full refresh + return {} + + @staticmethod + def _start_date_to_timestamp(start_date: str) -> int: + try: + return pendulum.parse(start_date).int_timestamp + except pendulum.parsing.exceptions.ParserError as e: + message = f"Invalid start date {start_date}. Please use YYYY-MM-DDTHH:MM:SSZ format." + raise AirbyteTracedException( + message=message, + internal_message=message, + failure_type=FailureType.config_error, + ) from e diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml b/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml index 872aa665492f..719177412a96 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/spec.yaml @@ -6,7 +6,6 @@ connectionSpecification: required: - client_secret - account_id - - start_date properties: account_id: type: string @@ -30,6 +29,7 @@ connectionSpecification: description: >- UTC date and time in the format 2017-01-25T00:00:00Z. Only data generated after this date will be replicated. + default: "2017-01-25T00:00:00Z" examples: - "2017-01-25T00:00:00Z" format: date-time @@ -42,7 +42,8 @@ connectionSpecification: description: >- When set, the connector will always re-export data from the past N days, where N is the value set here. This is useful if your data is frequently updated - after creation. More info here order: 3 slice_range: @@ -56,3 +57,22 @@ connectionSpecification: the less requests will be made and faster the sync will be. On the other hand, the more seldom the state is persisted. order: 4 + num_workers: + type: integer + title: Number of concurrent workers + minimum: 1 + maximum: 20 + default: 10 + examples: [1, 2, 3] + description: >- + The number of worker thread to use for the sync. + The performance upper boundary depends on call_rate_limit setting and type of account. + order: 5 + call_rate_limit: + type: integer + title: Max number of API calls per second + examples: [25, 100] + description: >- + The number of API calls per second that you allow connector to make. This value can not be bigger than real + API call rate limit (https://stripe.com/docs/rate-limits). If not specified the default maximum is 25 and 100 + calls per second for test and production tokens respectively. diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/stream_helpers.py b/airbyte-integrations/connectors/source-stripe/source_stripe/stream_helpers.py new file mode 100644 index 000000000000..dad073ae485b --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/stream_helpers.py @@ -0,0 +1,41 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Mapping, Optional + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.core import Stream, StreamData + + +def get_first_stream_slice(stream, sync_mode, stream_state) -> Optional[Mapping[str, Any]]: + """ + Gets the first stream_slice from a given stream's stream_slices. + :param stream: stream + :param sync_mode: sync_mode + :param stream_state: stream_state + :raises StopIteration: if there is no first slice to return (the stream_slices generator is empty) + :return: first stream slice from 'stream_slices' generator (`None` is a valid stream slice) + """ + # We wrap the return output of stream_slices() because some implementations return types that are iterable, + # but not iterators such as lists or tuples + slices = iter(stream.stream_slices(sync_mode=sync_mode, cursor_field=stream.cursor_field, stream_state=stream_state)) + return next(slices) + + +def get_first_record_for_slice( + stream: Stream, sync_mode: SyncMode, stream_slice: Optional[Mapping[str, Any]], stream_state: Optional[Mapping[str, Any]] +) -> StreamData: + """ + Gets the first record for a stream_slice of a stream. + :param stream: stream + :param sync_mode: sync_mode + :param stream_slice: stream_slice + :param stream_state: stream_state + :raises StopIteration: if there is no first record to return (the read_records generator is empty) + :return: StreamData containing the first record in the slice + """ + # We wrap the return output of read_records() because some implementations return types that are iterable, + # but not iterators such as lists or tuples + records_for_slice = iter(stream.read_records(sync_mode=sync_mode, stream_slice=stream_slice, stream_state=stream_state)) + return next(records_for_slice) diff --git a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py index 224a33a84ee1..d8958d9453b7 100644 --- a/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py +++ b/airbyte-integrations/connectors/source-stripe/source_stripe/streams.py @@ -2,25 +2,93 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import copy import math +import os from abc import ABC, abstractmethod from itertools import chain -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Type +from typing import Any, Callable, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union import pendulum import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy +from airbyte_cdk.sources.streams.core import StreamData from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream +from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from source_stripe.availability_strategy import StripeAvailabilityStrategy, StripeSubStreamAvailabilityStrategy STRIPE_API_VERSION = "2022-11-15" +CACHE_DISABLED = os.environ.get("CACHE_DISABLED") +IS_TESTING = os.environ.get("DEPLOYMENT_MODE") == "testing" +USE_CACHE = not CACHE_DISABLED + + +class IRecordExtractor(ABC): + @abstractmethod + def extract_records(self, records: Iterable[MutableMapping], stream_slice: Optional[Mapping[str, Any]] = None) -> Iterable[Mapping]: + pass + + +class DefaultRecordExtractor(IRecordExtractor): + def __init__(self, response_filter: Optional[Callable] = None, slice_data_retriever: Optional[Callable] = None): + self._response_filter = response_filter or (lambda record: record) + self._slice_data_retriever = slice_data_retriever or (lambda record, *_: record) + + def extract_records( + self, records: Iterable[MutableMapping], stream_slice: Optional[Mapping[str, Any]] = None + ) -> Iterable[MutableMapping]: + yield from filter(self._response_filter, map(lambda x: self._slice_data_retriever(x, stream_slice), records)) + + +class EventRecordExtractor(DefaultRecordExtractor): + def __init__(self, cursor_field: str, response_filter: Optional[Callable] = None, slice_data_retriever: Optional[Callable] = None): + super().__init__(response_filter, slice_data_retriever) + self.cursor_field = cursor_field + + def extract_records( + self, records: Iterable[MutableMapping], stream_slice: Optional[Mapping[str, Any]] = None + ) -> Iterable[MutableMapping]: + for record in records: + item = record["data"]["object"] + item[self.cursor_field] = record["created"] + if record["type"].endswith(".deleted"): + item["is_deleted"] = True + if self._response_filter(item): + yield self._slice_data_retriever(item, stream_slice) + + +class UpdatedCursorIncrementalRecordExtractor(DefaultRecordExtractor): + def __init__( + self, + cursor_field: str, + legacy_cursor_field: Optional[str], + response_filter: Optional[Callable] = None, + slice_data_retriever: Optional[Callable] = None, + ): + super().__init__(response_filter, slice_data_retriever) + self.cursor_field = cursor_field + self.legacy_cursor_field = legacy_cursor_field + + def extract_records( + self, records: Iterable[MutableMapping], stream_slice: Optional[Mapping[str, Any]] = None + ) -> Iterable[MutableMapping]: + records = super().extract_records(records, stream_slice) + for record in records: + if self.cursor_field in record: + yield record + continue # Skip the rest of the loop iteration + + # fetch legacy_cursor_field from record; default to current timestamp for initial syncs without an any cursor. + current_cursor_value = record.get(self.legacy_cursor_field, pendulum.now().int_timestamp) + + # yield the record with the added cursor_field + yield record | {self.cursor_field: current_cursor_value} class StripeStream(HttpStream, ABC): url_base = "https://api.stripe.com/v1/" - primary_key = "id" DEFAULT_SLICE_RANGE = 365 transformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) @@ -28,11 +96,66 @@ class StripeStream(HttpStream, ABC): def availability_strategy(self) -> Optional[AvailabilityStrategy]: return StripeAvailabilityStrategy() - def __init__(self, start_date: int, account_id: str, slice_range: int = DEFAULT_SLICE_RANGE, **kwargs): - super().__init__(**kwargs) + @property + def primary_key(self) -> Optional[Union[str, List[str], List[List[str]]]]: + return self._primary_key + + @property + def name(self) -> str: + if self._name: + return self._name + return super().name + + def path(self, *args, **kwargs) -> str: + if self._path: + return self._path if isinstance(self._path, str) else self._path(self, *args, **kwargs) + return super().path(*args, **kwargs) + + @property + def use_cache(self) -> bool: + return self._use_cache + + @property + def expand_items(self) -> Optional[List[str]]: + return self._expand_items + + def extra_request_params(self, *args, **kwargs) -> Mapping[str, Any]: + if callable(self._extra_request_params): + return self._extra_request_params(self, *args, **kwargs) + return self._extra_request_params or {} + + @property + def record_extractor(self) -> IRecordExtractor: + return self._record_extractor + + def __init__( + self, + start_date: int, + account_id: str, + *args, + slice_range: int = DEFAULT_SLICE_RANGE, + record_extractor: Optional[IRecordExtractor] = None, + name: Optional[str] = None, + path: Optional[Union[Callable, str]] = None, + use_cache: bool = False, + expand_items: Optional[List[str]] = None, + extra_request_params: Optional[Union[Mapping[str, Any], Callable]] = None, + response_filter: Optional[Callable] = None, + slice_data_retriever: Optional[Callable] = None, + primary_key: Optional[str] = "id", + **kwargs, + ): self.account_id = account_id self.start_date = start_date self.slice_range = slice_range or self.DEFAULT_SLICE_RANGE + self._record_extractor = record_extractor or DefaultRecordExtractor(response_filter, slice_data_retriever) + self._name = name + self._path = path + self._use_cache = use_cache + self._expand_items = expand_items + self._extra_request_params = extra_request_params + self._primary_key = primary_key + super().__init__(*args, **kwargs) def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: decoded_response = response.json() @@ -47,36 +170,87 @@ def request_params( next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: # Stripe default pagination is 10, max is 100 - params = {"limit": 100} + params = { + "limit": 100, + **self.extra_request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token), + } + if self.expand_items: + params["expand[]"] = self.expand_items # Handle pagination by inserting the next page's token in the request parameters if next_page_token: params.update(next_page_token) return params + def parse_response( + self, + response: requests.Response, + *, + stream_state: Mapping[str, Any], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> Iterable[Mapping[str, Any]]: + yield from self.record_extractor.extract_records(response.json().get("data", []), stream_slice) + def request_headers(self, **kwargs) -> Mapping[str, Any]: headers = {"Stripe-Version": STRIPE_API_VERSION} if self.account_id: headers["Stripe-Account"] = self.account_id return headers - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - response_json = response.json() - yield from response_json.get("data", []) # Stripe puts records in a container array "data" + def retry_factor(self) -> float: + """ + Override for testing purposes + """ + return 0 if IS_TESTING else super(StripeStream, self).retry_factor + + +class IStreamSelector(ABC): + @abstractmethod + def get_parent_stream(self, stream_state: Mapping[str, Any]) -> StripeStream: + pass + + +class CreatedCursorIncrementalStripeStream(StripeStream): + # Stripe returns most recently created objects first, so we don't want to persist state until the entire stream has been read + state_checkpoint_interval = math.inf + + @property + def cursor_field(self) -> str: + return self._cursor_field + + def __init__( + self, + *args, + lookback_window_days: int = 0, + start_date_max_days_from_now: Optional[int] = None, + cursor_field: str = "created", + **kwargs, + ): + super().__init__(*args, **kwargs) + self.lookback_window_days = lookback_window_days + self.start_date_max_days_from_now = start_date_max_days_from_now + self._cursor_field = cursor_field + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object + and returning an updated state object. + """ + state_cursor_value = current_stream_state.get(self.cursor_field, 0) + latest_record_value = latest_record.get(self.cursor_field) + if state_cursor_value: + return {self.cursor_field: max(latest_record_value, state_cursor_value)} + return {self.cursor_field: latest_record_value} -class BasePaginationStripeStream(StripeStream, ABC): def request_params( self, - stream_state: Mapping[str, Any], + stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state, stream_slice, next_page_token) - for key in ("created[gte]", "created[lte]"): - if key in stream_slice: - params[key] = stream_slice[key] - return params + params = super(CreatedCursorIncrementalStripeStream, self).request_params(stream_state, stream_slice, next_page_token) + return {"created[gte]": stream_slice["created[gte]"], "created[lte]": stream_slice["created[lte]"], **params} def chunk_dates(self, start_date_ts: int) -> Iterable[Tuple[int, int]]: now = pendulum.now().int_timestamp @@ -88,155 +262,268 @@ def chunk_dates(self, start_date_ts: int) -> Iterable[Tuple[int, int]]: after_ts = before_ts + 1 def stream_slices( - self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: - for start, end in self.chunk_dates(self.start_date): - yield {"created[gte]": start, "created[lte]": end} - - def read_records( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Mapping[str, Any]]: - if stream_slice is None: - return [] - - yield from super().read_records(sync_mode, cursor_field, stream_slice, stream_state) - - -class IncrementalStripeStream(BasePaginationStripeStream, ABC): - # Stripe returns most recently created objects first, so we don't want to persist state until the entire stream has been read - state_checkpoint_interval = math.inf - - def __init__(self, lookback_window_days: int = 0, **kwargs): - super().__init__(**kwargs) - self.lookback_window_days = lookback_window_days - - @property - @abstractmethod - def cursor_field(self) -> str: - """ - Defining a cursor field indicates that a stream is incremental, so any incremental stream must extend this class - and define a cursor field. - """ - pass - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object - and returning an updated state object. - """ - return {self.cursor_field: max(latest_record.get(self.cursor_field), current_stream_state.get(self.cursor_field, 0))} - - def stream_slices( - self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: + stream_state = stream_state or {} start_ts = self.get_start_timestamp(stream_state) if start_ts >= pendulum.now().int_timestamp: - # if the state is in the future - this will produce a state message but not make an API request - yield None - else: - for start, end in self.chunk_dates(start_ts): - yield {"created[gte]": start, "created[lte]": end} + return [] + for start, end in self.chunk_dates(start_ts): + yield {"created[gte]": start, "created[lte]": end} def get_start_timestamp(self, stream_state) -> int: start_point = self.start_date - if stream_state and self.cursor_field in stream_state: - start_point = max(start_point, stream_state[self.cursor_field]) + # we use +1 second because date range is inclusive + start_point = max(start_point, stream_state.get(self.cursor_field, 0) + 1) if start_point and self.lookback_window_days: self.logger.info(f"Applying lookback window of {self.lookback_window_days} days to stream {self.name}") start_point = int(pendulum.from_timestamp(start_point).subtract(days=abs(self.lookback_window_days)).timestamp()) + if self.start_date_max_days_from_now: + allowed_start_date = pendulum.now().subtract(days=self.start_date_max_days_from_now).int_timestamp + if start_point < allowed_start_date: + self.logger.info( + f"Applying the restriction of maximum {self.start_date_max_days_from_now} days lookback to stream {self.name}" + ) + start_point = allowed_start_date return start_point -class Authorizations(IncrementalStripeStream): +class Events(CreatedCursorIncrementalStripeStream): """ - API docs: https://stripe.com/docs/api/issuing/authorizations/list + API docs: https://stripe.com/docs/api/events/list """ - cursor_field = "created" + def __init__(self, *args, event_types: Optional[Iterable[str]] = None, **kwargs): + super().__init__(*args, **kwargs) + self.event_types = event_types + + def request_params( + self, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + if self.event_types: + params["types[]"] = self.event_types + return params - def path(self, **kwargs) -> str: - return "issuing/authorizations" + def path(self, **kwargs): + return "events" -class Customers(IncrementalStripeStream): +class UpdatedCursorIncrementalStripeStream(StripeStream): """ - API docs: https://stripe.com/docs/api/customers/list + `CreatedCursorIncrementalStripeStream` does not provide a way to read updated data since given date because the API does not allow to do this. + It only returns newly created entities since given date. So to have all the updated data as well we need to make use of the Events API, + which allows to retrieve updated data since given date for a number of predefined events which are associated with the corresponding + entities. """ - cursor_field = "created" - use_cache = True + @property + def cursor_field(self): + return self._cursor_field - def path(self, **kwargs) -> str: - return "customers" + @property + def legacy_cursor_field(self): + return self._legacy_cursor_field + @property + def event_types(self) -> Iterable[str]: + """A list of event types that are associated with entity.""" + return self._event_types -class BalanceTransactions(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/balance_transactions/list - """ + def __init__( + self, + *args, + cursor_field: str = "updated", + legacy_cursor_field: Optional[str] = "created", + event_types: Optional[List[str]] = None, + record_extractor: Optional[IRecordExtractor] = None, + response_filter: Optional[Callable] = None, + **kwargs, + ): + self._event_types = event_types + self._cursor_field = cursor_field + self._legacy_cursor_field = legacy_cursor_field + record_extractor = record_extractor or UpdatedCursorIncrementalRecordExtractor( + self.cursor_field, self.legacy_cursor_field, response_filter + ) + super().__init__(*args, record_extractor=record_extractor, **kwargs) + # `lookback_window_days` is hardcoded as it does not make any sense to re-export events, + # as each event holds the latest value of a record. + # `start_date_max_days_from_now` represents the events API limitation. + self.events_stream = Events( + authenticator=self.authenticator, + lookback_window_days=0, + start_date_max_days_from_now=30, + account_id=self.account_id, + start_date=self.start_date, + slice_range=self.slice_range, + event_types=self.event_types, + cursor_field=self.cursor_field, + record_extractor=EventRecordExtractor(cursor_field=self.cursor_field, response_filter=response_filter), + ) - cursor_field = "created" - name = "balance_transactions" + def update_cursor_field(self, stream_state: MutableMapping[str, Any]) -> MutableMapping[str, Any]: + if not self.legacy_cursor_field: + # Streams that used to support only full_refresh mode. + # Now they support event-based incremental syncs but have a cursor field only in that mode. + return stream_state + # support for both legacy and new cursor fields + current_stream_state_value = stream_state.get(self.cursor_field, stream_state.get(self.legacy_cursor_field, 0)) + return {self.cursor_field: current_stream_state_value} - def path(self, **kwargs) -> str: - return "balance_transactions" + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + latest_record_value = latest_record.get(self.cursor_field) + current_stream_state = self.update_cursor_field(current_stream_state) + current_state_value = current_stream_state.get(self.cursor_field) + if current_state_value: + return {self.cursor_field: max(latest_record_value, current_state_value)} + return {self.cursor_field: latest_record_value} + def stream_slices( + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + # When reading from a stream, a `read_records` is called once per slice. + # We yield a single slice here because we don't want to make duplicate calls for event based incremental syncs. + yield {} + + def read_event_increments( + self, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None + ) -> Iterable[StreamData]: + stream_state = self.update_cursor_field(stream_state or {}) + for event_slice in self.events_stream.stream_slices( + sync_mode=SyncMode.incremental, cursor_field=cursor_field, stream_state=stream_state + ): + yield from self.events_stream.read_records( + SyncMode.incremental, cursor_field=cursor_field, stream_slice=event_slice, stream_state=stream_state + ) -class Cardholders(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/issuing/cardholders/list - """ + def read_records( + self, + sync_mode: SyncMode, + cursor_field: Optional[List[str]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + stream_state: Optional[Mapping[str, Any]] = None, + ) -> Iterable[StreamData]: + if not stream_state: + # both full refresh and initial incremental sync should use usual endpoints + yield from super().read_records(sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state) + return + yield from self.read_event_increments(cursor_field=cursor_field, stream_state=stream_state) - cursor_field = "created" - def path(self, **kwargs) -> str: - return "issuing/cardholders" +class IncrementalStripeStreamSelector(IStreamSelector): + def __init__( + self, + created_cursor_incremental_stream: CreatedCursorIncrementalStripeStream, + updated_cursor_incremental_stream: UpdatedCursorIncrementalStripeStream, + ): + self._created_cursor_stream = created_cursor_incremental_stream + self._updated_cursor_stream = updated_cursor_incremental_stream + def get_parent_stream(self, stream_state: Mapping[str, Any]) -> StripeStream: + return self._updated_cursor_stream if stream_state else self._created_cursor_stream -class Charges(IncrementalStripeStream): + +class IncrementalStripeStream(StripeStream): """ - API docs: https://stripe.com/docs/api/charges/list + This class combines both normal incremental sync and event based sync. It uses common endpoints for sliced data syncs in + the full refresh sync mode and initial incremental sync. For incremental syncs with a state, event based sync comes into action. """ - cursor_field = "created" + def __init__( + self, + *args, + cursor_field: str = "updated", + legacy_cursor_field: Optional[str] = "created", + event_types: Optional[List[str]] = None, + **kwargs, + ): + super().__init__(*args, **kwargs) + self._cursor_field = cursor_field + created_cursor_stream = CreatedCursorIncrementalStripeStream( + *args, + cursor_field=cursor_field, + # `lookback_window_days` set to 0 because this particular instance is in charge of full_refresh/initial incremental syncs only + lookback_window_days=0, + record_extractor=UpdatedCursorIncrementalRecordExtractor(cursor_field, legacy_cursor_field), + **kwargs, + ) + updated_cursor_stream = UpdatedCursorIncrementalStripeStream( + *args, + cursor_field=cursor_field, + legacy_cursor_field=legacy_cursor_field, + event_types=event_types, + **kwargs, + ) + self._parent_stream = None + self.stream_selector = IncrementalStripeStreamSelector(created_cursor_stream, updated_cursor_stream) - def path(self, **kwargs) -> str: - return "charges" + @property + def parent_stream(self): + return self._parent_stream - def request_params( + @parent_stream.setter + def parent_stream(self, stream): + self._parent_stream = stream + + @property + def cursor_field(self) -> Union[str, List[str]]: + return self._cursor_field + + def stream_slices( + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + self.parent_stream = self.stream_selector.get_parent_stream(stream_state) + yield from self.parent_stream.stream_slices(sync_mode, cursor_field, stream_state) + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + return self.parent_stream.get_updated_state(current_stream_state, latest_record) + + def read_records( self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - params["expand[]"] = ["data.refunds"] - return params + sync_mode: SyncMode, + cursor_field: Optional[List[str]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + stream_state: Optional[Mapping[str, Any]] = None, + ) -> Iterable[StreamData]: + yield from self.parent_stream.read_records(sync_mode, cursor_field, stream_slice, stream_state) -class CustomerBalanceTransactions(BasePaginationStripeStream): +class CustomerBalanceTransactions(StripeStream): """ API docs: https://stripe.com/docs/api/customer_balance_transactions/list """ - name = "customer_balance_transactions" + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.parent = IncrementalStripeStream( + name="customers", + path="customers", + use_cache=USE_CACHE, + event_types=["customer.created", "customer.updated", "customer.deleted"], + authenticator=self.authenticator, + account_id=self.account_id, + start_date=self.start_date, + ) def path(self, stream_slice: Mapping[str, Any] = None, **kwargs): return f"customers/{stream_slice['id']}/balance_transactions" + @property + def availability_strategy(self) -> Optional[AvailabilityStrategy]: + return StripeSubStreamAvailabilityStrategy() + def stream_slices( - self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream = Customers(authenticator=self.authenticator, account_id=self.account_id, start_date=self.start_date) - slices = parent_stream.stream_slices(sync_mode=SyncMode.full_refresh) + slices = self.parent.stream_slices(sync_mode=SyncMode.full_refresh) for _slice in slices: - for customer in parent_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=_slice): + for customer in self.parent.read_records(sync_mode=SyncMode.full_refresh, stream_slice=_slice): # we use `get` here because some attributes may not be returned by some API versions if customer.get("next_invoice_sequence") == 1 and customer.get("balance") == 0: # We're making this check in order to speed up a sync. if a customer's balance is 0 and there are no @@ -245,49 +532,98 @@ def stream_slices( yield customer -class Coupons(IncrementalStripeStream): +class SetupAttempts(CreatedCursorIncrementalStripeStream, HttpSubStream): """ - API docs: https://stripe.com/docs/api/coupons/list + Docs: https://stripe.com/docs/api/setup_attempts/list """ - cursor_field = "created" - - def path(self, **kwargs): - return "coupons" + def __init__(self, **kwargs): + # SetupAttempts needs lookback_window, but it's parent class does not + parent_kwargs = copy.copy(kwargs) + parent_kwargs.pop("lookback_window_days") + parent = IncrementalStripeStream( + name="setup_intents", + path="setup_intents", + event_types=[ + "setup_intent.canceled", + "setup_intent.created", + "setup_intent.requires_action", + "setup_intent.setup_failed", + "setup_intent.succeeded", + ], + **parent_kwargs, + ) + super().__init__(parent=parent, **kwargs) + def path(self, **kwargs) -> str: + return "setup_attempts" -class Disputes(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/disputes/list - """ + @property + def availability_strategy(self) -> Optional[AvailabilityStrategy]: + # we use the default http availability strategy here because parent stream may lack data in the incremental stream mode + # and this stream would be marked inaccessible which is not actually true + return HttpAvailabilityStrategy() - cursor_field = "created" + def stream_slices( + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + # this is a unique combination of CreatedCursorIncrementalStripeStream and HttpSubStream, + # so we need to have all the parent IDs multiplied by all the date slices + incremental_slices = list( + CreatedCursorIncrementalStripeStream.stream_slices( + self, sync_mode=sync_mode, cursor_field=cursor_field, stream_state=stream_state + ) + ) + if incremental_slices: + parent_records = HttpSubStream.stream_slices(self, sync_mode=sync_mode, cursor_field=cursor_field, stream_state=stream_state) + yield from (slice | rec for rec in parent_records for slice in incremental_slices) + else: + yield from [] - def path(self, **kwargs): - return "disputes" + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + setup_intent_id = stream_slice.get("parent", {}).get("id") + params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + params.update(setup_intent=setup_intent_id) + return params -class EarlyFraudWarnings(StripeStream): +class Persons(UpdatedCursorIncrementalStripeStream, HttpSubStream): """ - API docs: https://stripe.com/docs/api/radar/early_fraud_warnings/list + API docs: https://stripe.com/docs/api/persons/list """ - def path(self, **kwargs): - return "radar/early_fraud_warnings" + event_types = ["person.created", "person.updated", "person.deleted"] + def __init__(self, *args, **kwargs): + parent = StripeStream(*args, name="accounts", path="accounts", use_cache=USE_CACHE, **kwargs) + super().__init__(*args, parent=parent, **kwargs) -class Events(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/events/list - """ + @property + def availability_strategy(self) -> Optional[AvailabilityStrategy]: + return StripeSubStreamAvailabilityStrategy() - cursor_field = "created" + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs): + return f"accounts/{stream_slice['parent']['id']}/persons" + + def stream_slices( + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + parent = HttpSubStream if not stream_state else UpdatedCursorIncrementalStripeStream + yield from parent.stream_slices(self, sync_mode, cursor_field=cursor_field, stream_state=stream_state) - def path(self, **kwargs): - return "events" + +class StripeSubStream(StripeStream, HttpSubStream): + @property + def availability_strategy(self) -> Optional[AvailabilityStrategy]: + return StripeSubStreamAvailabilityStrategy() -class StripeSubStream(BasePaginationStripeStream, ABC): +class StripeLazySubStream(StripeStream, HttpSubStream): """ Research shows that records related to SubStream can be extracted from Parent streams which already contain 1st page of needed items. Thus, it significantly decreases a number of requests needed to get @@ -329,30 +665,22 @@ class StripeSubStream(BasePaginationStripeStream, ABC): } """ - filter: Optional[Mapping[str, Any]] = None - add_parent_id: bool = False - - @property - @abstractmethod - def parent(self) -> Type[StripeStream]: - """ - :return: parent stream which contains needed records in - """ - - @property - @abstractmethod - def parent_id(self) -> str: - """ - :return: string with attribute name - """ - @property - @abstractmethod def sub_items_attr(self) -> str: """ :return: string if single primary key, list of strings if composite primary key, list of list of strings if composite primary key consisting of nested fields. If the stream has no primary keys, return None. """ + return self._sub_items_attr + + def __init__( + self, + *args, + sub_items_attr: Optional[str] = None, + **kwargs, + ): + super().__init__(*args, **kwargs) + self._sub_items_attr = sub_items_attr @property def availability_strategy(self) -> Optional[AvailabilityStrategy]: @@ -367,634 +695,155 @@ def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs): return params - def get_parent_stream_instance(self): - return self.parent(authenticator=self.authenticator, account_id=self.account_id, start_date=self.start_date) - - def stream_slices( - self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream = self.get_parent_stream_instance() - slices = parent_stream.stream_slices(sync_mode=SyncMode.full_refresh) - for _slice in slices: - yield from parent_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=_slice) - def read_records(self, sync_mode: SyncMode, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: - parent_record = stream_slice - items_obj = parent_record.get(self.sub_items_attr, {}) + items_obj = stream_slice["parent"].get(self.sub_items_attr, {}) if not items_obj: return - items = items_obj.get("data", []) - if self.filter: - items = [i for i in items if i.get(self.filter["attr"]) == self.filter["value"]] - - # get next pages items_next_pages = [] + items = list(self.record_extractor.extract_records(items_obj.get("data", []), stream_slice)) if items_obj.get("has_more") and items: - stream_slice = {self.parent_id: parent_record["id"], "starting_after": items[-1]["id"]} + stream_slice = {"starting_after": items[-1]["id"], **stream_slice} items_next_pages = super().read_records(sync_mode=SyncMode.full_refresh, stream_slice=stream_slice, **kwargs) + yield from chain(items, items_next_pages) - for item in chain(items, items_next_pages): - if self.add_parent_id: - # add reference to parent object when item doesn't have it already - item[self.parent_id] = parent_record["id"] - yield item - - -class ApplicationFees(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/application_fees - """ - cursor_field = "created" +class IncrementalStripeLazySubStreamSelector(IStreamSelector): + def __init__(self, updated_cursor_incremental_stream: UpdatedCursorIncrementalStripeStream, lazy_sub_stream: StripeLazySubStream): + self._updated_incremental_stream = updated_cursor_incremental_stream + self._lazy_sub_stream = lazy_sub_stream - def path(self, **kwargs): - return "application_fees" + def get_parent_stream(self, stream_state: Mapping[str, Any]) -> StripeStream: + return self._updated_incremental_stream if stream_state else self._lazy_sub_stream -class ApplicationFeesRefunds(StripeSubStream): +class UpdatedCursorIncrementalStripeLazySubStream(StripeStream, ABC): """ - API docs: https://stripe.com/docs/api/fee_refunds/list + This stream uses StripeLazySubStream under the hood to run full refresh or initial incremental syncs. + In case of subsequent incremental syncs, it uses the UpdatedCursorIncrementalStripeStream class. """ - name = "application_fees_refunds" + def __init__( + self, + parent: StripeStream, + *args, + cursor_field: str = "updated", + legacy_cursor_field: Optional[str] = "created", + event_types: Optional[List[str]] = None, + sub_items_attr: Optional[str] = None, + response_filter: Optional[Callable] = None, + **kwargs, + ): + super().__init__(*args, **kwargs) + self._cursor_field = cursor_field + self.updated_cursor_incremental_stream = UpdatedCursorIncrementalStripeStream( + *args, + cursor_field=cursor_field, + legacy_cursor_field=legacy_cursor_field, + event_types=event_types, + response_filter=response_filter, + **kwargs, + ) + self.lazy_substream = StripeLazySubStream( + *args, + parent=parent, + sub_items_attr=sub_items_attr, + record_extractor=UpdatedCursorIncrementalRecordExtractor( + cursor_field=cursor_field, legacy_cursor_field=legacy_cursor_field, response_filter=response_filter + ), + **kwargs, + ) + self._parent_stream = None + self.stream_selector = IncrementalStripeLazySubStreamSelector(self.updated_cursor_incremental_stream, self.lazy_substream) - parent = ApplicationFees - parent_id: str = "refund_id" - sub_items_attr = "refunds" - add_parent_id = True + @property + def cursor_field(self) -> Union[str, List[str]]: + return self._cursor_field - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs): - return f"application_fees/{stream_slice[self.parent_id]}/refunds" + @property + def parent_stream(self): + return self._parent_stream + @parent_stream.setter + def parent_stream(self, stream): + self._parent_stream = stream -class Invoices(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/invoices/list - """ + def stream_slices( + self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + ) -> Iterable[Optional[Mapping[str, Any]]]: + self.parent_stream = self.stream_selector.get_parent_stream(stream_state) + yield from self.parent_stream.stream_slices(sync_mode, cursor_field=cursor_field, stream_state=stream_state) - cursor_field = "created" + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + # important note: do not call self.parent_stream here as one of the parents does not have the needed method implemented + return self.updated_cursor_incremental_stream.get_updated_state(current_stream_state, latest_record) - def path(self, **kwargs): - return "invoices" + def read_records( + self, + sync_mode: SyncMode, + cursor_field: Optional[List[str]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + stream_state: Optional[Mapping[str, Any]] = None, + ) -> Iterable[StreamData]: + yield from self.parent_stream.read_records( + sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) -class InvoiceLineItems(StripeSubStream): +class ParentIncrementalStipeSubStream(StripeSubStream): """ - API docs: https://stripe.com/docs/api/invoices/invoice_lines + This stream differs from others in that it runs parent stream in exactly same sync mode it is run itself to generate stream slices. + It also uses regular /v1 API endpoints to sync data no matter what the sync mode is. This means that the event-based API can only + be utilized by the parent stream. """ - name = "invoice_line_items" - - parent = Invoices - parent_id: str = "invoice_id" - sub_items_attr = "lines" - add_parent_id = True - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs): - return f"invoices/{stream_slice[self.parent_id]}/lines" - - -class InvoiceItems(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/invoiceitems/list - """ - - cursor_field = "date" - name = "invoice_items" - - def path(self, **kwargs): - return "invoiceitems" - - -class Payouts(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/payouts/list - """ - - cursor_field = "created" - - def path(self, **kwargs): - return "payouts" - - -class Plans(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/plans/list - """ - - cursor_field = "created" - - def path(self, **kwargs): - return "plans" - - def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs): - params = super().request_params(stream_slice=stream_slice, **kwargs) - params["expand[]"] = ["data.tiers"] - return params - - -class Prices(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/prices/list - """ - - cursor_field = "created" - - def path(self, **kwargs): - return "prices" - - -class Products(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/products/list - """ - - cursor_field = "created" - - def path(self, **kwargs): - return "products" - - -class ShippingRates(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/shipping_rates/list - """ - - cursor_field = "created" - - def path(self, **kwargs): - return "shipping_rates" - - -class Reviews(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/radar/reviews/list - """ - - cursor_field = "created" - - def path(self, **kwargs): - return "reviews" - - -class Subscriptions(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/subscriptions/list - """ - - use_cache = True - cursor_field = "created" - status = "all" - - def path(self, **kwargs): - return "subscriptions" - - def request_params(self, stream_state=None, **kwargs): - stream_state = stream_state or {} - params = super().request_params(stream_state=stream_state, **kwargs) - params["status"] = self.status - return params - - -class SubscriptionItems(StripeSubStream): - """ - API docs: https://stripe.com/docs/api/subscription_items/list - """ - - use_cache = True - - name = "subscription_items" - - parent: StripeStream = Subscriptions - parent_id: str = "subscription_id" - sub_items_attr: str = "items" - - def path(self, **kwargs): - return "subscription_items" - - def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs): - params = super().request_params(stream_slice=stream_slice, **kwargs) - params["subscription"] = stream_slice[self.parent_id] - return params - - -class SubscriptionSchedule(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/subscription_schedules - """ - - cursor_field = "created" - - def path(self, **kwargs): - return "subscription_schedules" - - -class Transfers(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/transfers/list - """ - - use_cache = True - cursor_field = "created" - - def path(self, **kwargs): - return "transfers" - - -class Refunds(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/refunds/list - """ - - cursor_field = "created" - - def path(self, **kwargs): - return "refunds" - - -class PaymentIntents(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/payment_intents/list - """ - - cursor_field = "created" - - def path(self, **kwargs): - return "payment_intents" - - -class PaymentMethods(StripeStream): - """ - API docs: https://stripe.com/docs/api/payment_methods/list - """ - - def path(self, **kwargs): - return "payment_methods" - - -class BankAccounts(StripeSubStream): - """ - API docs: https://stripe.com/docs/api/customer_bank_accounts/list - """ - - name = "bank_accounts" - - parent = Customers - parent_id = "customer_id" - sub_items_attr = "sources" - filter = {"attr": "object", "value": "bank_account"} - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs): - return f"customers/{stream_slice[self.parent_id]}/sources" - - def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: - params = super().request_params(stream_slice=stream_slice, **kwargs) - params["object"] = "bank_account" - return params - - -class CheckoutSessions(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/checkout/sessions/list - """ - - name = "checkout_sessions" - - cursor_field = "expires_at" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - # https://stripe.com/docs/api/checkout/sessions/create#create_checkout_session-expires_at - # 'expires_at' - can be anywhere from 1 to 24 hours after Checkout Session creation. - # thus we should always add 1 day to lookback window to avoid possible checkout_sessions losses - self.lookback_window_days = self.lookback_window_days + 1 - - def stream_slices( - self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: - yield from [{}] - - def path(self, **kwargs): - return "checkout/sessions" - - def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping]: - since_date = self.get_start_timestamp(stream_state) - for item in super().parse_response(response, **kwargs): - # Filter out too old items - expires_at = item.get(self.cursor_field) - if expires_at and expires_at > since_date: - yield item - - -class CheckoutSessionsLineItems(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/checkout/sessions/line_items - """ - - name = "checkout_sessions_line_items" - - cursor_field = "checkout_session_expires_at" + @property + def cursor_field(self) -> str: + return self._cursor_field - def __init__(self, *args, **kwargs): + def __init__(self, cursor_field: str, *args, **kwargs): + self._cursor_field = cursor_field super().__init__(*args, **kwargs) - # https://stripe.com/docs/api/checkout/sessions/create#create_checkout_session-expires_at - # 'expires_at' - can be anywhere from 1 to 24 hours after Checkout Session creation. - # thus we should always add 1 day to lookback window to avoid possible checkout_sessions losses - self.lookback_window_days = self.lookback_window_days + 1 - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs): - return f"checkout/sessions/{stream_slice['checkout_session_id']}/line_items" - def stream_slices( - self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None + self, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None ) -> Iterable[Optional[Mapping[str, Any]]]: - checkout_session_state = None + stream_state = stream_state or {} if stream_state: - checkout_session_state = {"expires_at": stream_state["checkout_session_expires_at"]} - checkout_session_stream = CheckoutSessions(authenticator=self.authenticator, account_id=self.account_id, start_date=self.start_date) - for checkout_session in checkout_session_stream.read_records( - sync_mode=SyncMode.full_refresh, stream_state=checkout_session_state, stream_slice={} - ): - yield { - "checkout_session_id": checkout_session["id"], - "expires_at": checkout_session["expires_at"], - } + # state is shared between self and parent, but cursor fields are different + stream_state = {self.parent.cursor_field: stream_state.get(self.cursor_field, 0)} + parent_stream_slices = self.parent.stream_slices(sync_mode=sync_mode, cursor_field=cursor_field, stream_state=stream_state) + for stream_slice in parent_stream_slices: + parent_records = self.parent.read_records( + sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + for record in parent_records: + yield {"parent": record} - def request_params(self, stream_slice: Mapping[str, Any] = None, **kwargs): - params = super().request_params(stream_slice=stream_slice, **kwargs) - params["expand[]"] = ["data.discounts", "data.taxes"] - return params + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + return {self.cursor_field: max(current_stream_state.get(self.cursor_field, 0), latest_record[self.cursor_field])} @property - def raise_on_http_errors(self): + def raise_on_http_errors(self) -> bool: return False - def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping]: + def parse_response(self, response: requests.Response, *args, **kwargs) -> Iterable[Mapping[str, Any]]: + if response.status_code == 200: + return super().parse_response(response, *args, **kwargs) if response.status_code == 404: - self.logger.warning(response.json()) - return + # When running incremental sync with state, the returned parent object very likely will not contain sub-items + # as the events API does not support expandable items. Parent class will try getting sub-items from this object, + # then from its own API. In case there are no sub-items at all for this entity, API will raise 404 error. + self.logger.warning( + f"Data was not found for URL: {response.request.url}. " + "If this is a path for getting child attributes like /v1/checkout/sessions//line_items when running " + "the incremental sync, you may safely ignore this warning." + ) + return [] response.raise_for_status() - response_json = response.json() - data = response_json.get("data", []) - if data and stream_slice: - self.logger.info(f"stream_slice: {stream_slice}") - cs_id = stream_slice.get("checkout_session_id", None) - cs_expires_at = stream_slice.get("expires_at", None) - for e in data: - e["checkout_session_id"] = cs_id - e["checkout_session_expires_at"] = cs_expires_at - yield from data - - -class PromotionCodes(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/promotion_codes/list - """ - - cursor_field = "created" - - def path(self, **kwargs): - return "promotion_codes" - - -class ExternalAccount(BasePaginationStripeStream, ABC): - """ - Bank Accounts and Cards are separate streams because they have different schemas - """ - - object = "" - - def path(self, **kwargs): - return f"accounts/{self.account_id}/external_accounts" - - def stream_slices( - self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: - yield from [{}] - - def request_params(self, **kwargs): - params = super().request_params(**kwargs) - return {**params, **{"object": self.object}} - - -class ExternalAccountBankAccounts(ExternalAccount): - """ - https://stripe.com/docs/api/external_account_bank_accounts/list - """ - - object = "bank_account" - - -class ExternalAccountCards(ExternalAccount): - """ - https://stripe.com/docs/api/external_account_cards/list - """ - - object = "card" - - -class SetupIntents(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/setup_intents/list - """ - - cursor_field = "created" - - def path(self, **kwargs): - return "setup_intents" - - -class Accounts(BasePaginationStripeStream): - """ - Docs: https://stripe.com/docs/api/accounts/list - Even the endpoint allow to filter based on created the data usually don't have this field. - """ - - def path(self, **kwargs): - return "accounts" - - -class Persons(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/persons/list - """ - - name = "persons" - cursor_field = "created" - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs): - return f"accounts/{stream_slice['id']}/persons" - - def stream_slices( - self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: - parent_stream = Accounts(authenticator=self.authenticator, account_id=self.account_id, start_date=self.start_date) - slices = parent_stream.stream_slices(sync_mode=SyncMode.full_refresh) - for _slice in slices: - for account in parent_stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=_slice): - # we use `get` here because some attributes may not be returned by some API versions - yield account - - -class CreditNotes(StripeStream): - """ - API docs: https://stripe.com/docs/api/credit_notes/list - """ - - name = "credit_notes" - - def path(self, **kwargs) -> str: - return "credit_notes" - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return next_page_token or {} - - def stream_slices( - self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: - yield from [{}] - - -class Cards(IncrementalStripeStream): - """ - Docs: https://stripe.com/docs/api/issuing/cards/list - """ - - cursor_field = "created" - - def path(self, **kwargs): - return "issuing/cards" - - -class TopUps(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/topups/list - """ - - name = "top_ups" - cursor_field = "created" - - def path(self, **kwargs) -> str: - return "topups" - - -class Files(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/files/list - """ - - name = "files" - cursor_field = "created" - - def path(self, **kwargs) -> str: - return "files" - - -class FileLinks(IncrementalStripeStream): - """ - API docs: https://stripe.com/docs/api/file_links/list - """ - - name = "file_links" - cursor_field = "created" - - def path(self, **kwargs) -> str: - return "file_links" - - -class SetupAttempts(IncrementalStripeStream, HttpSubStream): - """ - Docs: https://stripe.com/docs/api/setup_attempts/list - """ - - cursor_field = "created" - - def __init__(self, **kwargs): - parent = SetupIntents(**kwargs) - super().__init__(parent=parent, **kwargs) - - def path(self, **kwargs) -> str: - return "setup_attempts" - - def stream_slices( - self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None - ) -> Iterable[Optional[Mapping[str, Any]]]: - - incremental_slices = list( - IncrementalStripeStream.stream_slices(self, sync_mode=sync_mode, cursor_field=cursor_field, stream_state=stream_state) - ) - if incremental_slices: - parent_records = HttpSubStream.stream_slices(self, sync_mode=sync_mode, cursor_field=cursor_field, stream_state=stream_state) - yield from (slice | rec for rec in parent_records for slice in incremental_slices) - else: - yield None - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - setup_intent_id = stream_slice.get("parent", {}).get("id") - params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - params.update(setup_intent=setup_intent_id) - return params - - -class UsageRecords(StripeStream, HttpSubStream): - """ - Docs: https://stripe.com/docs/api/usage_records/subscription_item_summary_list - """ - - primary_key = None - - def __init__(self, **kwargs): - parent = SubscriptionItems(**kwargs) - super().__init__(parent=parent, **kwargs) - - def path( - self, - *, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> str: - subscription_item_id = stream_slice.get("parent", {}).get("id") - return f"subscription_items/{subscription_item_id}/usage_record_summaries" - - -class TransferReversals(StripeStream, HttpSubStream): - """ - Docs: https://stripe.com/docs/api/transfer_reversals/list - """ - - def __init__(self, **kwargs): - parent = Transfers(**kwargs) - super().__init__(parent=parent, **kwargs) - - def path( - self, - *, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> str: - transfer_id = stream_slice.get("parent", {}).get("id") - return f"transfers/{transfer_id}/reversals" - - -class Transactions(IncrementalStripeStream): - """ - Docs: https://stripe.com/docs/api/issuing/transactions/list - """ - - cursor_field = "created" - - def path(self, **kwargs) -> str: - return "issuing/transactions" + @property + def availability_strategy(self) -> Optional[AvailabilityStrategy]: + # we use the default http availability strategy here because parent stream may lack data in the incremental stream mode + # and this stream would be marked inaccessible which is not actually true + return HttpAvailabilityStrategy() diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py b/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py index f72068c051d9..8e81ce968306 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/conftest.py @@ -2,38 +2,18 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import os + import pytest from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator - -@pytest.fixture(autouse=True) -def disable_cache(mocker): - mocker.patch( - "source_stripe.streams.Customers.use_cache", - new_callable=mocker.PropertyMock, - return_value=False - ) - mocker.patch( - "source_stripe.streams.Transfers.use_cache", - new_callable=mocker.PropertyMock, - return_value=False - ) - mocker.patch( - "source_stripe.streams.Subscriptions.use_cache", - new_callable=mocker.PropertyMock, - return_value=False - ) - mocker.patch( - "source_stripe.streams.SubscriptionItems.use_cache", - new_callable=mocker.PropertyMock, - return_value=False - ) +os.environ["CACHE_DISABLED"] = "true" +os.environ["DEPLOYMENT_MODE"] = "testing" @pytest.fixture(name="config") def config_fixture(): - config = {"client_secret": "sk_test(live)_", - "account_id": "", "start_date": "2020-05-01T00:00:00Z"} + config = {"client_secret": "sk_test(live)_", "account_id": "", "start_date": "2020-05-01T00:00:00Z"} return config @@ -47,3 +27,23 @@ def stream_args_fixture(): "slice_range": 365, } return args + + +@pytest.fixture(name="incremental_stream_args") +def incremental_args_fixture(stream_args): + return {"lookback_window_days": 14, **stream_args} + + +@pytest.fixture() +def stream_by_name(config): + # use local import in favour of global because we need to make imports after setting the env variables + from source_stripe.source import SourceStripe + + def mocker(stream_name, source_config=config): + source = SourceStripe(None, source_config) + streams = source.streams(source_config) + for stream in streams: + if stream.name == stream_name: + return stream + + return mocker diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/__init__.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/config.py new file mode 100644 index 000000000000..d048407320d1 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/config.py @@ -0,0 +1,36 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime +from typing import Any, Dict + + +class ConfigBuilder: + def __init__(self) -> None: + self._config: Dict[str, Any] = { + "client_secret": "ConfigBuilder default client secret", + "account_id": "ConfigBuilder default account id", + "start_date": "2020-05-01T00:00:00Z" + } + + def with_account_id(self, account_id: str) -> "ConfigBuilder": + self._config["account_id"] = account_id + return self + + def with_client_secret(self, client_secret: str) -> "ConfigBuilder": + self._config["client_secret"] = client_secret + return self + + def with_start_date(self, start_datetime: datetime) -> "ConfigBuilder": + self._config["start_date"] = start_datetime.isoformat()[:-13]+"Z" + return self + + def with_lookback_window_in_days(self, number_of_days: int) -> "ConfigBuilder": + self._config["lookback_window_days"] = number_of_days + return self + + def with_slice_range_in_days(self, number_of_days: int) -> "ConfigBuilder": + self._config["slice_range"] = number_of_days + return self + + def build(self) -> Dict[str, Any]: + return self._config diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/pagination.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/pagination.py new file mode 100644 index 000000000000..acfe9a613271 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/pagination.py @@ -0,0 +1,11 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from typing import Any, Dict + +from airbyte_cdk.test.mock_http.response_builder import PaginationStrategy + + +class StripePaginationStrategy(PaginationStrategy): + @staticmethod + def update(response: Dict[str, Any]) -> None: + response["has_more"] = True diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/request_builder.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/request_builder.py new file mode 100644 index 000000000000..7a2c8219c5d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/request_builder.py @@ -0,0 +1,143 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime +from typing import List, Optional + +from airbyte_cdk.test.mock_http import HttpRequest +from airbyte_cdk.test.mock_http.request import ANY_QUERY_PARAMS + + +class StripeRequestBuilder: + + @classmethod + def accounts_endpoint(cls, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls("accounts", account_id, client_secret) + + @classmethod + def application_fees_endpoint(cls, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls("application_fees", account_id, client_secret) + + @classmethod + def application_fees_refunds_endpoint(cls, application_fee_id: str, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls(f"application_fees/{application_fee_id}/refunds", account_id, client_secret) + + @classmethod + def customers_endpoint(cls, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls("customers", account_id, client_secret) + + @classmethod + def customers_bank_accounts_endpoint(cls, customer_id: str, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls(f"customers/{customer_id}/bank_accounts", account_id, client_secret) + + @classmethod + def events_endpoint(cls, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls("events", account_id, client_secret) + + @classmethod + def external_accounts_endpoint(cls, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls(f"accounts/{account_id}/external_accounts", account_id, client_secret) + + @classmethod + def issuing_authorizations_endpoint(cls, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls("issuing/authorizations", account_id, client_secret) + + @classmethod + def issuing_cards_endpoint(cls, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls("issuing/cards", account_id, client_secret) + + @classmethod + def issuing_transactions_endpoint(cls, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls("issuing/transactions", account_id, client_secret) + + @classmethod + def payment_methods_endpoint(cls, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls("payment_methods", account_id, client_secret) + + @classmethod + def persons_endpoint(cls, parent_account_id: str, account_id: str, client_secret: str, ) -> "StripeRequestBuilder": + return cls(f"accounts/{parent_account_id}/persons", account_id, client_secret) + + @classmethod + def radar_early_fraud_warnings_endpoint(cls, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls("radar/early_fraud_warnings", account_id, client_secret) + + @classmethod + def reviews_endpoint(cls, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls("reviews", account_id, client_secret) + + @classmethod + def _for_endpoint(cls, endpoint: str, account_id: str, client_secret: str) -> "StripeRequestBuilder": + return cls(endpoint, account_id, client_secret) + + def __init__(self, resource: str, account_id: str, client_secret: str) -> None: + self._resource = resource + self._account_id = account_id + self._client_secret = client_secret + self._any_query_params = False + self._created_gte: Optional[datetime] = None + self._created_lte: Optional[datetime] = None + self._limit: Optional[int] = None + self._object: Optional[str] = None + self._starting_after_id: Optional[str] = None + self._types: List[str] = [] + self._expands: List[str] = [] + + def with_created_gte(self, created_gte: datetime) -> "StripeRequestBuilder": + self._created_gte = created_gte + return self + + def with_created_lte(self, created_lte: datetime) -> "StripeRequestBuilder": + self._created_lte = created_lte + return self + + def with_limit(self, limit: int) -> "StripeRequestBuilder": + self._limit = limit + return self + + def with_object(self, object_name: str) -> "StripeRequestBuilder": + self._object = object_name + return self + + def with_starting_after(self, starting_after_id: str) -> "StripeRequestBuilder": + self._starting_after_id = starting_after_id + return self + + def with_any_query_params(self) -> "StripeRequestBuilder": + self._any_query_params = True + return self + + def with_types(self, types: List[str]) -> "StripeRequestBuilder": + self._types = types + return self + + def with_expands(self, expands: List[str]) -> "StripeRequestBuilder": + self._expands = expands + return self + + def build(self) -> HttpRequest: + query_params = {} + if self._created_gte: + query_params["created[gte]"] = str(int(self._created_gte.timestamp())) + if self._created_lte: + query_params["created[lte]"] = str(int(self._created_lte.timestamp())) + if self._limit: + query_params["limit"] = str(self._limit) + if self._starting_after_id: + query_params["starting_after"] = self._starting_after_id + if self._types: + query_params["types[]"] = self._types + if self._object: + query_params["object"] = self._object + if self._expands: + query_params["expand[]"] = self._expands + + if self._any_query_params: + if query_params: + raise ValueError(f"Both `any_query_params` and {list(query_params.keys())} were configured. Provide only one of none but not both.") + query_params = ANY_QUERY_PARAMS + + return HttpRequest( + url=f"https://api.stripe.com/v1/{self._resource}", + query_params=query_params, + headers={"Stripe-Account": self._account_id, "Authorization": f"Bearer {self._client_secret}"}, + ) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/response_builder.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/response_builder.py new file mode 100644 index 000000000000..7495bffeb4a9 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/response_builder.py @@ -0,0 +1,10 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json + +from airbyte_cdk.test.mock_http import HttpResponse +from airbyte_cdk.test.mock_http.response_builder import find_template + + +def a_response_with_status(status_code: int) -> HttpResponse: + return HttpResponse(json.dumps(find_template(str(status_code), __file__)), status_code) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees.py new file mode 100644 index 000000000000..9619010e4c24 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees.py @@ -0,0 +1,376 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_EVENT_TYPES = ["application_fee.created", "application_fee.refunded"] + +_DATA_FIELD = NestedPath(["data", "object"]) +_STREAM_NAME = "application_fees" +_ENDPOINT_TEMPLATE_NAME = "application_fees" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _application_fees_request() -> StripeRequestBuilder: + return StripeRequestBuilder.application_fees_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=75)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any]) -> SourceStripe: + return SourceStripe(catalog, config) + + +def _an_event() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _events_response() -> HttpResponseBuilder: + return create_response_builder( + find_template("events", __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _an_application_fee() -> RecordBuilder: + return create_record_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _application_fees_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _given_application_fees_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.application_fees_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _application_fees_response().build() + ) + + +def _given_events_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().build() + ) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config), config, catalog, state, expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response().with_record(_an_application_fee()).with_record(_an_application_fee()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response().with_pagination().with_record(_an_application_fee().with_id("last_record_id_from_first_page")).build(), + ) + http_mocker.get( + _application_fees_request().with_starting_after("last_record_id_from_first_page").with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response().with_record(_an_application_fee()).with_record(_an_application_fee()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_no_state_when_read_then_return_ignore_lookback(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response().with_record(_an_application_fee()).build(), + ) + + self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + # request matched http_mocker + + @HttpMocker() + def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response().with_record(_an_application_fee()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + assert output.records[0].record.data["updated"] == output.records[0].record.data["created"] + + @HttpMocker() + def test_given_slice_range_when_read_then_perform_multiple_requests(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=30) + slice_range = timedelta(days=20) + slice_datetime = start_date + slice_range + + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(), + _application_fees_response().build(), + ) + http_mocker.get( + _application_fees_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response().build(), + ) + + self._read(_config().with_start_date(start_date).with_slice_range_in_days(slice_range.days)) + + # request matched http_mocker + + @HttpMocker() + def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _application_fees_request().with_any_query_params().build(), + a_response_with_status(400), + ) + output = self._read(_config()) + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + @HttpMocker() + def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _application_fees_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _application_fees_response().with_record(_an_application_fee()).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_any_query_params().build(), + [a_response_with_status(500), _application_fees_response().with_record(_an_application_fee()).build()], + ) + output = self._read(_config()) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _application_fees_request().with_any_query_params().build(), + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None: + # see https://github.com/airbytehq/airbyte/issues/33499 + events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build() + http_mocker.get( + events_requests, + _events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now + ) + http_mocker.get( + _application_fees_request().with_any_query_params().build(), + _application_fees_response().build(), + ) + + self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1)) + + http_mocker.assert_number_of_calls(events_requests, 30) + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_state_when_read_then_use_application_fees_endpoint(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + cursor_value = int(_A_START_DATE.timestamp()) + 1 + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response().with_record(_an_application_fee().with_cursor(cursor_value)).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_when_read_then_query_events_using_types_and_state_value_plus_1(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + cursor_value = int(state_datetime.timestamp()) + 1 + + _given_application_fees_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_cursor(cursor_value).with_field(_DATA_FIELD, _an_application_fee().build()) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_application_fees_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_pagination().with_record( + _an_event().with_id("last_record_id_from_first_page").with_field(_DATA_FIELD, _an_application_fee().build()) + ).build(), + ) + http_mocker.get( + _events_request().with_starting_after("last_record_id_from_first_page").with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + self._an_application_fee_event() + ).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_state_and_small_slice_range_when_read_then_perform_multiple_queries(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + slice_range = timedelta(days=3) + slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range + + _given_application_fees_availability_check(http_mocker) + _given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_application_fee_event()).build(), + ) + http_mocker.get( + _events_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_application_fee_event()).with_record(self._an_application_fee_event()).build(), + ) + + output = self._read( + _config().with_start_date(_NOW - timedelta(days=30)).with_slice_range_in_days(slice_range.days), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None: + # this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the + # application fees endpoint + _given_application_fees_availability_check(http_mocker) + start_date = _NOW - timedelta(days=40) + state_value = _NOW - timedelta(days=39) + events_lower_boundary = _NOW - timedelta(days=30) + http_mocker.get( + _events_request().with_created_gte(events_lower_boundary).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_application_fee_event()).build(), + ) + + self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_value.timestamp())}).build(), + ) + + # request matched http_mocker + + def _an_application_fee_event(self) -> RecordBuilder: + return _an_event().with_field(_DATA_FIELD, _an_application_fee().build()) + + def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees_refunds.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees_refunds.py new file mode 100644 index 000000000000..f01c6da9689e --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_application_fees_refunds.py @@ -0,0 +1,518 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + + +import json +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_EVENT_TYPES = ["application_fee.refund.updated"] + +_DATA_FIELD = NestedPath(["data", "object"]) +_REFUNDS_FIELD = FieldPath("refunds") +_STREAM_NAME = "application_fees_refunds" +_APPLICATION_FEES_TEMPLATE_NAME = "application_fees" +_REFUNDS_TEMPLATE_NAME = "application_fees_refunds" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _application_fees_request() -> StripeRequestBuilder: + return StripeRequestBuilder.application_fees_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _application_fees_refunds_request(application_fee_id: str) -> StripeRequestBuilder: + return StripeRequestBuilder.application_fees_refunds_endpoint(application_fee_id, _ACCOUNT_ID, _CLIENT_SECRET) + + +def _events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=75)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any]) -> SourceStripe: + return SourceStripe(catalog, config) + + +def _an_event() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _events_response() -> HttpResponseBuilder: + return create_response_builder( + find_template("events", __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _an_application_fee() -> RecordBuilder: + return create_record_builder( + find_template(_APPLICATION_FEES_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _application_fees_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_APPLICATION_FEES_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _a_refund() -> RecordBuilder: + return create_record_builder( + find_template(_REFUNDS_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _refunds_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_REFUNDS_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _given_application_fees_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.application_fees_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _application_fees_response().with_record(_an_application_fee()).build() # there needs to be a record in the parent stream for the child to be available + ) + + +def _given_events_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().build() + ) + + +def _as_dict(response_builder: HttpResponseBuilder) -> Dict[str, Any]: + return json.loads(response_builder.build().body) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config), config, catalog, state, expecting_exception) + + +def _assert_not_available(output: EntrypointOutput) -> None: + # right now, no stream statuses means stream unavailable + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response() + .with_record( + _an_application_fee() + .with_field( + _REFUNDS_FIELD, + _as_dict( + _refunds_response() + .with_record(_a_refund()) + .with_record(_a_refund()) + ) + ) + ) + .with_record( + _an_application_fee() + .with_field(_REFUNDS_FIELD, _as_dict(_refunds_response().with_record(_a_refund()))) + ).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_multiple_refunds_pages_when_read_then_query_pagination_on_child(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response() + .with_record( + _an_application_fee() + .with_id("parent_id") + .with_field( + _REFUNDS_FIELD, + _as_dict( + _refunds_response() + .with_pagination() + .with_record(_a_refund().with_id("latest_refund_id")) + ) + ) + ).build(), + ) + http_mocker.get( + # we do not use slice boundaries here because: + # * there should be no duplicates parents (application fees) returned by the stripe API as it is using cursor pagination + # * it is implicitly lower bounder by the parent creation + # * the upper boundary is not configurable and is always + _application_fees_refunds_request("parent_id").with_limit(100).with_starting_after("latest_refund_id").build(), + _refunds_response().with_record(_a_refund()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_multiple_application_fees_pages_when_read_then_query_pagination_on_parent(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response() + .with_pagination() + .with_record( + _an_application_fee() + .with_id("parent_id") + .with_field( + _REFUNDS_FIELD, + _as_dict( + _refunds_response() + .with_record(_a_refund()) + ) + ) + ).build(), + ) + http_mocker.get( + _application_fees_request().with_starting_after("parent_id").with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response() + .with_record( + _an_application_fee() + .with_field( + _REFUNDS_FIELD, + _as_dict( + _refunds_response() + .with_record(_a_refund()) + ) + ) + ).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_parent_stream_without_refund_when_read_then_stream_is_unavailable(self, http_mocker: HttpMocker) -> None: + # events stream is not validated as application fees is validated first + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response().build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + _assert_not_available(output) + + @HttpMocker() + def test_given_slice_range_when_read_then_perform_multiple_requests(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=30) + slice_range = timedelta(days=20) + slice_datetime = start_date + slice_range + + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(), + _application_fees_response().with_record( + _an_application_fee() + .with_field(_REFUNDS_FIELD, _as_dict(_refunds_response().with_record(_a_refund()))) + ).build(), + ) + http_mocker.get( + _application_fees_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response().with_record( + _an_application_fee() + .with_field(_REFUNDS_FIELD, _as_dict(_refunds_response().with_record(_a_refund()))) + ).build(), + ) + + output = self._read(_config().with_start_date(start_date).with_slice_range_in_days(slice_range.days)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_slice_range_and_refunds_pagination_when_read_then_do_not_slice_child(self, http_mocker: HttpMocker) -> None: + """ + This means that if the user attempt to configure the slice range, it will only apply on the parent stream + """ + start_date = _NOW - timedelta(days=30) + slice_range = timedelta(days=20) + slice_datetime = start_date + slice_range + + _given_events_availability_check(http_mocker) + http_mocker.get( + StripeRequestBuilder.application_fees_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _application_fees_response().build() + ) # catching subsequent slicing request that we don't really care for this test + http_mocker.get( + _application_fees_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(), + _application_fees_response().with_record( + _an_application_fee() + .with_id("parent_id") + .with_field( + _REFUNDS_FIELD, + _as_dict( + _refunds_response() + .with_pagination() + .with_record(_a_refund().with_id("latest_refund_id")) + ) + ) + ).build(), + ) + http_mocker.get( + # slice range is not applied here + _application_fees_refunds_request("parent_id").with_limit(100).with_starting_after("latest_refund_id").build(), + _refunds_response().with_record(_a_refund()).build(), + ) + + self._read(_config().with_start_date(start_date).with_slice_range_in_days(slice_range.days)) + + # request matched http_mocker + + @HttpMocker() + def test_given_no_state_when_read_then_return_ignore_lookback(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response().with_record(_an_application_fee()).build(), + ) + + self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + # request matched http_mocker + + @HttpMocker() + def test_given_one_page_when_read_then_cursor_field_is_set(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response() + .with_record( + _an_application_fee() + .with_field( + _REFUNDS_FIELD, + _as_dict( + _refunds_response() + .with_record(_a_refund()) + ) + ) + ).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert output.records[0].record.data["updated"] == output.records[0].record.data["created"] + + @HttpMocker() + def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _application_fees_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _application_fees_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _application_fees_response().with_record(_an_application_fee().with_field( + _REFUNDS_FIELD, + _as_dict( + _refunds_response() + .with_record(_a_refund()) + ) + )).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + request = _application_fees_request().with_any_query_params().build() + http_mocker.get( + request, + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_state_when_read_then_use_application_fees_endpoint(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + cursor_value = int(_A_START_DATE.timestamp()) + 1 + http_mocker.get( + _application_fees_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _application_fees_response().with_record( + _an_application_fee() + .with_field(_REFUNDS_FIELD, _as_dict(_refunds_response().with_record(_a_refund().with_cursor(cursor_value)))) + ).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_when_read_then_query_events_using_types_and_state_value_plus_1(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + cursor_value = int(state_datetime.timestamp()) + 1 + + _given_application_fees_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_cursor(cursor_value).with_field(_DATA_FIELD, _a_refund().build()) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_application_fees_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_pagination().with_record( + _an_event().with_id("last_record_id_from_first_page").with_field(_DATA_FIELD, _a_refund().build()) + ).build(), + ) + http_mocker.get( + _events_request().with_starting_after("last_record_id_from_first_page").with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_refund_event()).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_state_and_small_slice_range_when_read_then_perform_multiple_queries(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + slice_range = timedelta(days=3) + slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range + + _given_application_fees_availability_check(http_mocker) + _given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_refund_event()).build(), + ) + http_mocker.get( + _events_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_refund_event()).with_record(self._a_refund_event()).build(), + ) + + output = self._read( + _config().with_start_date(_NOW - timedelta(days=30)).with_slice_range_in_days(slice_range.days), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None: + # this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the + # application fees endpoint + _given_application_fees_availability_check(http_mocker) + start_date = _NOW - timedelta(days=40) + state_value = _NOW - timedelta(days=39) + events_lower_boundary = _NOW - timedelta(days=30) + http_mocker.get( + _events_request().with_created_gte(events_lower_boundary).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_refund_event()).build(), + ) + + self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_value.timestamp())}).build(), + ) + + # request matched http_mocker + + def _a_refund_event(self) -> RecordBuilder: + return _an_event().with_field(_DATA_FIELD, _a_refund().build()) + + def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_authorizations.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_authorizations.py new file mode 100644 index 000000000000..ab140559840f --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_authorizations.py @@ -0,0 +1,374 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_EVENT_TYPES = ["issuing_authorization.created", "issuing_authorization.request", "issuing_authorization.updated"] + +_DATA_FIELD = NestedPath(["data", "object"]) +_STREAM_NAME = "authorizations" +_ENDPOINT_TEMPLATE_NAME = "issuing_authorizations" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _authorizations_request() -> StripeRequestBuilder: + return StripeRequestBuilder.issuing_authorizations_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=75)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any]) -> SourceStripe: + return SourceStripe(catalog, config) + + +def _an_event() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _events_response() -> HttpResponseBuilder: + return create_response_builder( + find_template("events", __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _an_authorization() -> RecordBuilder: + return create_record_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _authorizations_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _given_authorizations_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.issuing_authorizations_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _authorizations_response().build() + ) + + +def _given_events_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().build() + ) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config), config, catalog, state, expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _authorizations_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _authorizations_response().with_record(_an_authorization()).with_record(_an_authorization()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _authorizations_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _authorizations_response().with_pagination().with_record(_an_authorization().with_id("last_record_id_from_first_page")).build(), + ) + http_mocker.get( + _authorizations_request().with_starting_after("last_record_id_from_first_page").with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _authorizations_response().with_record(_an_authorization()).with_record(_an_authorization()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_no_state_when_read_then_return_ignore_lookback(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _authorizations_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _authorizations_response().with_record(_an_authorization()).build(), + ) + + self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + # request matched http_mocker + + @HttpMocker() + def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _authorizations_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _authorizations_response().with_record(_an_authorization()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + assert output.records[0].record.data["updated"] == output.records[0].record.data["created"] + + @HttpMocker() + def test_given_slice_range_when_read_then_perform_multiple_requests(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=30) + slice_range = timedelta(days=20) + slice_datetime = start_date + slice_range + + _given_events_availability_check(http_mocker) + http_mocker.get( + _authorizations_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(), + _authorizations_response().build(), + ) + http_mocker.get( + _authorizations_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).build(), + _authorizations_response().build(), + ) + + self._read(_config().with_start_date(start_date).with_slice_range_in_days(slice_range.days)) + + # request matched http_mocker + + @HttpMocker() + def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _authorizations_request().with_any_query_params().build(), + a_response_with_status(400), + ) + output = self._read(_config()) + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + @HttpMocker() + def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _authorizations_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _authorizations_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _authorizations_response().with_record(_an_authorization()).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _authorizations_request().with_any_query_params().build(), + [a_response_with_status(500), _authorizations_response().with_record(_an_authorization()).build()], + ) + output = self._read(_config()) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _authorizations_request().with_any_query_params().build(), + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None: + # see https://github.com/airbytehq/airbyte/issues/33499 + events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build() + http_mocker.get( + events_requests, + _events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now + ) + http_mocker.get( + _authorizations_request().with_any_query_params().build(), + _authorizations_response().build(), + ) + + self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1)) + + http_mocker.assert_number_of_calls(events_requests, 30) + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_state_when_read_then_use_authorizations_endpoint(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + cursor_value = int(_A_START_DATE.timestamp()) + 1 + http_mocker.get( + _authorizations_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _authorizations_response().with_record(_an_authorization().with_cursor(cursor_value)).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_when_read_then_query_events_using_types_and_state_value_plus_1(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + cursor_value = int(state_datetime.timestamp()) + 1 + + _given_authorizations_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_cursor(cursor_value).with_field(_DATA_FIELD, _an_authorization().build()) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_authorizations_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_pagination().with_record( + _an_event().with_id("last_record_id_from_first_page").with_field(_DATA_FIELD, _an_authorization().build()) + ).build(), + ) + http_mocker.get( + _events_request().with_starting_after("last_record_id_from_first_page").with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_authorization_event()).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_state_and_small_slice_range_when_read_then_perform_multiple_queries(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + slice_range = timedelta(days=3) + slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range + + _given_authorizations_availability_check(http_mocker) + _given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_authorization_event()).build(), + ) + http_mocker.get( + _events_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_authorization_event()).with_record(self._an_authorization_event()).build(), + ) + + output = self._read( + _config().with_start_date(_NOW - timedelta(days=30)).with_slice_range_in_days(slice_range.days), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None: + # this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the + # authorizations endpoint + _given_authorizations_availability_check(http_mocker) + start_date = _NOW - timedelta(days=40) + state_value = _NOW - timedelta(days=39) + events_lower_boundary = _NOW - timedelta(days=30) + http_mocker.get( + _events_request().with_created_gte(events_lower_boundary).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_authorization_event()).build(), + ) + + self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_value.timestamp())}).build(), + ) + + # request matched http_mocker + + def _an_authorization_event(self) -> RecordBuilder: + return _an_event().with_field(_DATA_FIELD, _an_authorization().build()) + + def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_bank_accounts.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_bank_accounts.py new file mode 100644 index 000000000000..c0995a6c11b3 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_bank_accounts.py @@ -0,0 +1,563 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + + +import json +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_EVENT_TYPES = ["customer.source.created", "customer.source.expiring", "customer.source.updated", "customer.source.deleted"] + +_DATA_FIELD = NestedPath(["data", "object"]) +_SOURCES_FIELD = FieldPath("sources") +_STREAM_NAME = "bank_accounts" +_CUSTOMERS_TEMPLATE_NAME = "customers_expand_data_source" +_BANK_ACCOUNTS_TEMPLATE_NAME = "bank_accounts" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +# FIXME expand[] is not documented anymore in stripe API doc (see https://github.com/airbytehq/airbyte/issues/33714) +_EXPANDS = ["data.sources"] +_OBJECT = "bank_account" +_NOT_A_BANK_ACCOUNT = RecordBuilder({"object": "NOT a bank account"}, None, None) +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _customers_request() -> StripeRequestBuilder: + return StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _customers_bank_accounts_request(customer_id: str) -> StripeRequestBuilder: + return StripeRequestBuilder.customers_bank_accounts_endpoint(customer_id, _ACCOUNT_ID, _CLIENT_SECRET) + + +def _events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=75)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any]) -> SourceStripe: + return SourceStripe(catalog, config) + + +def _an_event() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _events_response() -> HttpResponseBuilder: + return create_response_builder( + find_template("events", __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _a_customer() -> RecordBuilder: + return create_record_builder( + find_template(_CUSTOMERS_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _customers_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_CUSTOMERS_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _a_bank_account() -> RecordBuilder: + return create_record_builder( + find_template(_BANK_ACCOUNTS_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + ) + + +def _bank_accounts_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_BANK_ACCOUNTS_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _given_customers_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _customers_response().with_record(_a_customer()).build() # there needs to be a record in the parent stream for the child to be available + ) + + +def _given_events_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().build() + ) + + +def _as_dict(response_builder: HttpResponseBuilder) -> Dict[str, Any]: + return json.loads(response_builder.build().body) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config), config, catalog, state, expecting_exception) + + +def _assert_not_available(output: EntrypointOutput) -> None: + # right now, no stream statuses means stream unavailable + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _customers_response() + .with_record( + _a_customer() + .with_field( + _SOURCES_FIELD, + _as_dict( + _bank_accounts_response() + .with_record(_a_bank_account()) + .with_record(_a_bank_account()) + ) + ) + ) + .with_record( + _a_customer() + .with_field(_SOURCES_FIELD, _as_dict(_bank_accounts_response().with_record(_a_bank_account()))) + ).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_source_is_not_bank_account_when_read_then_filter_record(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _customers_response() + .with_record( + _a_customer() + .with_field( + _SOURCES_FIELD, + _as_dict( + _bank_accounts_response() + .with_record(_NOT_A_BANK_ACCOUNT) + ) + ) + ).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 0 + + @HttpMocker() + def test_given_multiple_bank_accounts_pages_when_read_then_query_pagination_on_child(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _customers_response() + .with_record( + _a_customer() + .with_id("parent_id") + .with_field( + _SOURCES_FIELD, + _as_dict( + _bank_accounts_response() + .with_pagination() + .with_record(_a_bank_account().with_id("latest_bank_account_id")) + ) + ) + ).build(), + ) + http_mocker.get( + # we do not use slice boundaries here because: + # * there should be no duplicates parents (application fees) returned by the stripe API as it is using cursor pagination + # * it is implicitly lower bounder by the parent creation + # * the upper boundary is not configurable and is always + _customers_bank_accounts_request("parent_id").with_limit(100).with_starting_after("latest_bank_account_id").build(), + _bank_accounts_response().with_record(_a_bank_account()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_multiple_customers_pages_when_read_then_query_pagination_on_parent(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _customers_response() + .with_pagination() + .with_record( + _a_customer() + .with_id("parent_id") + .with_field( + _SOURCES_FIELD, + _as_dict( + _bank_accounts_response() + .with_record(_a_bank_account()) + ) + ) + ).build(), + ) + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_starting_after("parent_id").with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _customers_response() + .with_record( + _a_customer() + .with_field( + _SOURCES_FIELD, + _as_dict( + _bank_accounts_response() + .with_record(_a_bank_account()) + ) + ) + ).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_parent_stream_without_bank_accounts_when_read_then_stream_is_unavailable(self, http_mocker: HttpMocker) -> None: + # events stream is not validated as application fees is validated first + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _customers_response().build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + _assert_not_available(output) + + @HttpMocker() + def test_given_slice_range_when_read_then_perform_multiple_requests(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=30) + slice_range = timedelta(days=20) + slice_datetime = start_date + slice_range + + _given_events_availability_check(http_mocker) + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(), + _customers_response().with_record( + _a_customer() + .with_field(_SOURCES_FIELD, _as_dict(_bank_accounts_response().with_record(_a_bank_account()))) + ).build(), + ) + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).build(), + _customers_response().with_record( + _a_customer() + .with_field(_SOURCES_FIELD, _as_dict(_bank_accounts_response().with_record(_a_bank_account()))) + ).build(), + ) + + output = self._read(_config().with_start_date(start_date).with_slice_range_in_days(slice_range.days)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_slice_range_and_bank_accounts_pagination_when_read_then_do_not_slice_child(self, http_mocker: HttpMocker) -> None: + """ + This means that if the user attempt to configure the slice range, it will only apply on the parent stream + """ + start_date = _NOW - timedelta(days=30) + slice_range = timedelta(days=20) + slice_datetime = start_date + slice_range + + _given_events_availability_check(http_mocker) + http_mocker.get( + StripeRequestBuilder.customers_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _customers_response().build() + ) # catching subsequent slicing request that we don't really care for this test + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(), + _customers_response().with_record( + _a_customer() + .with_id("parent_id") + .with_field( + _SOURCES_FIELD, + _as_dict( + _bank_accounts_response() + .with_pagination() + .with_record(_a_bank_account().with_id("latest_bank_account_id")) + ) + ) + ).build(), + ) + http_mocker.get( + # slice range is not applied here + _customers_bank_accounts_request("parent_id").with_limit(100).with_starting_after("latest_bank_account_id").build(), + _bank_accounts_response().with_record(_a_bank_account()).build(), + ) + + self._read(_config().with_start_date(start_date).with_slice_range_in_days(slice_range.days)) + + # request matched http_mocker + + @HttpMocker() + def test_given_no_state_when_read_then_return_ignore_lookback(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _customers_response().with_record(_a_customer()).build(), + ) + + self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + # request matched http_mocker + + @HttpMocker() + def test_given_one_page_when_read_then_cursor_field_is_set(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _customers_response() + .with_record( + _a_customer() + .with_field( + _SOURCES_FIELD, + _as_dict( + _bank_accounts_response() + .with_record(_a_bank_account()) + ) + ) + ).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert output.records[0].record.data["updated"] == int(_NOW.timestamp()) + + @HttpMocker() + def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _customers_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _customers_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _customers_response().with_record(_a_customer().with_field( + _SOURCES_FIELD, + _as_dict( + _bank_accounts_response() + .with_record(_a_bank_account()) + ) + )).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + request = _customers_request().with_any_query_params().build() + http_mocker.get( + request, + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_state_and_successful_sync_when_read_then_set_state_to_now(self, http_mocker: HttpMocker) -> None: + # If stripe takes some time to ingest the data, we should recommend to use a lookback window when syncing the bank_accounts stream + # to make sure that we don't lose data between the first and the second sync + _given_events_availability_check(http_mocker) + http_mocker.get( + _customers_request().with_expands(_EXPANDS).with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _customers_response().with_record( + _a_customer() + .with_field(_SOURCES_FIELD, _as_dict(_bank_accounts_response().with_record(_a_bank_account()))) + ).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": int(_NOW.timestamp())}} + + @HttpMocker() + def test_given_state_when_read_then_query_events_using_types_and_state_value_plus_1(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + cursor_value = int(state_datetime.timestamp()) + 1 + + _given_customers_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_cursor(cursor_value).with_field(_DATA_FIELD, _a_bank_account().build()) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_customers_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_pagination().with_record( + _an_event().with_id("last_record_id_from_first_page").with_field(_DATA_FIELD, _a_bank_account().build()) + ).build(), + ) + http_mocker.get( + _events_request().with_starting_after("last_record_id_from_first_page").with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_bank_account_event()).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_state_and_small_slice_range_when_read_then_perform_multiple_queries(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + slice_range = timedelta(days=3) + slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range + + _given_customers_availability_check(http_mocker) + _given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_bank_account_event()).build(), + ) + http_mocker.get( + _events_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_bank_account_event()).with_record(self._a_bank_account_event()).build(), + ) + + output = self._read( + _config().with_start_date(_NOW - timedelta(days=30)).with_slice_range_in_days(slice_range.days), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None: + # this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the + # customer endpoint + _given_customers_availability_check(http_mocker) + start_date = _NOW - timedelta(days=40) + state_value = _NOW - timedelta(days=39) + events_lower_boundary = _NOW - timedelta(days=30) + http_mocker.get( + _events_request().with_created_gte(events_lower_boundary).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_bank_account_event()).build(), + ) + + self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_value.timestamp())}).build(), + ) + + # request matched http_mocker + + @HttpMocker() + def test_given_source_is_not_bank_account_when_read_then_filter_record(self, http_mocker: HttpMocker) -> None: + _given_customers_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_field(_DATA_FIELD, _NOT_A_BANK_ACCOUNT.build()) + ).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 0 + + def _a_bank_account_event(self) -> RecordBuilder: + return _an_event().with_field(_DATA_FIELD, _a_bank_account().build()) + + def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_cards.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_cards.py new file mode 100644 index 000000000000..573ba8824a9e --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_cards.py @@ -0,0 +1,374 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_EVENT_TYPES = ["issuing_card.created", "issuing_card.updated"] + +_DATA_FIELD = NestedPath(["data", "object"]) +_STREAM_NAME = "cards" +_ENDPOINT_TEMPLATE_NAME = "issuing_cards" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _cards_request() -> StripeRequestBuilder: + return StripeRequestBuilder.issuing_cards_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=75)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any]) -> SourceStripe: + return SourceStripe(catalog, config) + + +def _an_event() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _events_response() -> HttpResponseBuilder: + return create_response_builder( + find_template("events", __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _a_card() -> RecordBuilder: + return create_record_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _cards_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _given_cards_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.issuing_cards_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _cards_response().build() + ) + + +def _given_events_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().build() + ) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config), config, catalog, state, expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _cards_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _cards_response().with_record(_a_card()).with_record(_a_card()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _cards_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _cards_response().with_pagination().with_record(_a_card().with_id("last_record_id_from_first_page")).build(), + ) + http_mocker.get( + _cards_request().with_starting_after("last_record_id_from_first_page").with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _cards_response().with_record(_a_card()).with_record(_a_card()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_no_state_when_read_then_return_ignore_lookback(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _cards_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _cards_response().with_record(_a_card()).build(), + ) + + self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + # request matched http_mocker + + @HttpMocker() + def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _cards_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _cards_response().with_record(_a_card()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + assert output.records[0].record.data["updated"] == output.records[0].record.data["created"] + + @HttpMocker() + def test_given_slice_range_when_read_then_perform_multiple_requests(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=30) + slice_range = timedelta(days=20) + slice_datetime = start_date + slice_range + + _given_events_availability_check(http_mocker) + http_mocker.get( + _cards_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(), + _cards_response().build(), + ) + http_mocker.get( + _cards_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).build(), + _cards_response().build(), + ) + + self._read(_config().with_start_date(start_date).with_slice_range_in_days(slice_range.days)) + + # request matched http_mocker + + @HttpMocker() + def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _cards_request().with_any_query_params().build(), + a_response_with_status(400), + ) + output = self._read(_config()) + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + @HttpMocker() + def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _cards_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _cards_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _cards_response().with_record(_a_card()).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _cards_request().with_any_query_params().build(), + [a_response_with_status(500), _cards_response().with_record(_a_card()).build()], + ) + output = self._read(_config()) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _cards_request().with_any_query_params().build(), + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None: + # see https://github.com/airbytehq/airbyte/issues/33499 + events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build() + http_mocker.get( + events_requests, + _events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now + ) + http_mocker.get( + _cards_request().with_any_query_params().build(), + _cards_response().build(), + ) + + self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1)) + + http_mocker.assert_number_of_calls(events_requests, 30) + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_state_when_read_then_use_cards_endpoint(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + cursor_value = int(_A_START_DATE.timestamp()) + 1 + http_mocker.get( + _cards_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _cards_response().with_record(_a_card().with_cursor(cursor_value)).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_when_read_then_query_events_using_types_and_state_value_plus_1(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + cursor_value = int(state_datetime.timestamp()) + 1 + + _given_cards_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_cursor(cursor_value).with_field(_DATA_FIELD, _a_card().build()) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_cards_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_pagination().with_record( + _an_event().with_id("last_record_id_from_first_page").with_field(_DATA_FIELD, _a_card().build()) + ).build(), + ) + http_mocker.get( + _events_request().with_starting_after("last_record_id_from_first_page").with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_card_event()).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_state_and_small_slice_range_when_read_then_perform_multiple_queries(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + slice_range = timedelta(days=3) + slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range + + _given_cards_availability_check(http_mocker) + _given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_card_event()).build(), + ) + http_mocker.get( + _events_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_card_event()).with_record(self._a_card_event()).build(), + ) + + output = self._read( + _config().with_start_date(_NOW - timedelta(days=30)).with_slice_range_in_days(slice_range.days), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None: + # this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the + # cards endpoint + _given_cards_availability_check(http_mocker) + start_date = _NOW - timedelta(days=40) + state_value = _NOW - timedelta(days=39) + events_lower_boundary = _NOW - timedelta(days=30) + http_mocker.get( + _events_request().with_created_gte(events_lower_boundary).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_card_event()).build(), + ) + + self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_value.timestamp())}).build(), + ) + + # request matched http_mocker + + def _a_card_event(self) -> RecordBuilder: + return _an_event().with_field(_DATA_FIELD, _a_card().build()) + + def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_early_fraud_warnings.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_early_fraud_warnings.py new file mode 100644 index 000000000000..f4c2165c582a --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_early_fraud_warnings.py @@ -0,0 +1,342 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_EVENT_TYPES = ["radar.early_fraud_warning.created", "radar.early_fraud_warning.updated"] + +_DATA_FIELD = NestedPath(["data", "object"]) +_STREAM_NAME = "early_fraud_warnings" +_ENDPOINT_TEMPLATE_NAME = "radar_early_fraud_warnings" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _early_fraud_warnings_request() -> StripeRequestBuilder: + return StripeRequestBuilder.radar_early_fraud_warnings_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=75)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any]) -> SourceStripe: + return SourceStripe(catalog, config) + + +def _an_event() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _events_response() -> HttpResponseBuilder: + return create_response_builder( + find_template("events", __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _an_early_fraud_warning() -> RecordBuilder: + return create_record_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _early_fraud_warnings_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _given_early_fraud_warnings_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.radar_early_fraud_warnings_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _early_fraud_warnings_response().build() + ) + + +def _given_events_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().build() + ) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config), config, catalog, state, expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _early_fraud_warnings_request().with_limit(100).build(), + _early_fraud_warnings_response().with_record(_an_early_fraud_warning()).with_record(_an_early_fraud_warning()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _early_fraud_warnings_request().with_limit(100).build(), + _early_fraud_warnings_response().with_pagination().with_record(_an_early_fraud_warning().with_id("last_record_id_from_first_page")).build(), + ) + http_mocker.get( + _early_fraud_warnings_request().with_starting_after("last_record_id_from_first_page").with_limit(100).build(), + _early_fraud_warnings_response().with_record(_an_early_fraud_warning()).with_record(_an_early_fraud_warning()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 3 + + @HttpMocker() + def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _early_fraud_warnings_request().with_limit(100).build(), + _early_fraud_warnings_response().with_record(_an_early_fraud_warning()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + assert output.records[0].record.data["updated"] == output.records[0].record.data["created"] + + @HttpMocker() + def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _early_fraud_warnings_request().with_any_query_params().build(), + a_response_with_status(400), + ) + output = self._read(_config()) + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + @HttpMocker() + def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _early_fraud_warnings_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _early_fraud_warnings_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _early_fraud_warnings_response().with_record(_an_early_fraud_warning()).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _early_fraud_warnings_request().with_any_query_params().build(), + [a_response_with_status(500), _early_fraud_warnings_response().with_record(_an_early_fraud_warning()).build()], + ) + output = self._read(_config()) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _early_fraud_warnings_request().with_any_query_params().build(), + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None: + # see https://github.com/airbytehq/airbyte/issues/33499 + events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build() + http_mocker.get( + events_requests, + _events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now + ) + http_mocker.get( + _early_fraud_warnings_request().with_any_query_params().build(), + _early_fraud_warnings_response().build(), + ) + + self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1)) + + http_mocker.assert_number_of_calls(events_requests, 30) + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_state_when_read_then_use_early_fraud_warnings_endpoint(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + cursor_value = int(_A_START_DATE.timestamp()) + 1 + http_mocker.get( + _early_fraud_warnings_request().with_limit(100).build(), + _early_fraud_warnings_response().with_record(_an_early_fraud_warning().with_cursor(cursor_value)).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_when_read_then_query_events_using_types_and_state_value_plus_1(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + cursor_value = int(state_datetime.timestamp()) + 1 + + _given_early_fraud_warnings_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_cursor(cursor_value).with_field(_DATA_FIELD, _an_early_fraud_warning().build()) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_early_fraud_warnings_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_pagination().with_record( + _an_event().with_id("last_record_id_from_first_page").with_field(_DATA_FIELD, _an_early_fraud_warning().build()) + ).build(), + ) + http_mocker.get( + _events_request().with_starting_after("last_record_id_from_first_page").with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_early_fraud_warning_event()).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_state_and_small_slice_range_when_read_then_perform_multiple_queries(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + slice_range = timedelta(days=3) + slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range + + _given_early_fraud_warnings_availability_check(http_mocker) + _given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_early_fraud_warning_event()).build(), + ) + http_mocker.get( + _events_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_early_fraud_warning_event()).with_record(self._an_early_fraud_warning_event()).build(), + ) + + output = self._read( + _config().with_start_date(_NOW - timedelta(days=30)).with_slice_range_in_days(slice_range.days), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None: + # this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the + # radar/early_fraud_warnings endpoint + _given_early_fraud_warnings_availability_check(http_mocker) + start_date = _NOW - timedelta(days=40) + state_value = _NOW - timedelta(days=39) + events_lower_boundary = _NOW - timedelta(days=30) + http_mocker.get( + _events_request().with_created_gte(events_lower_boundary).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_early_fraud_warning_event()).build(), + ) + + self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_value.timestamp())}).build(), + ) + + # request matched http_mocker + + def _an_early_fraud_warning_event(self) -> RecordBuilder: + return _an_event().with_field(_DATA_FIELD, _an_early_fraud_warning().build()) + + def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_events.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_events.py new file mode 100644 index 000000000000..c93bc691a96e --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_events.py @@ -0,0 +1,272 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_STREAM_NAME = "events" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) +_SECOND_REQUEST = timedelta(seconds=1) +_THIRD_REQUEST = timedelta(seconds=2) + + +def _a_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=73)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any]) -> SourceStripe: + return SourceStripe(catalog, config) + + +def _a_record() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _a_response() -> HttpResponseBuilder: + return create_response_builder(find_template("events", __file__), FieldPath("data"), pagination_strategy=StripePaginationStrategy()) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config), config, catalog, state, expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _a_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _a_response().with_record(_a_record()).with_record(_a_record()).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 2 + + @HttpMocker() + def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _a_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _a_response().with_pagination().with_record(_a_record().with_id("last_record_id_from_first_page")).build(), + ) + http_mocker.get( + _a_request().with_starting_after("last_record_id_from_first_page").with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _a_response().with_record(_a_record()).with_record(_a_record()).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 3 + + @HttpMocker() + def test_given_start_date_before_30_days_stripe_limit_and_slice_range_when_read_then_perform_request_before_30_days(self, http_mocker: HttpMocker) -> None: + """ + This case is special because the source queries for a time range that is before 30 days. That being said as of 2023-12-13, the API + mentions that "We only guarantee access to events through the Retrieve Event API for 30 days." (see + https://stripe.com/docs/api/events) + """ + start_date = _NOW - timedelta(days=61) + slice_range = timedelta(days=30) + slice_datetime = start_date + slice_range + http_mocker.get( # this first request has both gte and lte before 30 days even though we know there should not be records returned + _a_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(), + _a_response().build(), + ) + http_mocker.get( + _a_request().with_created_gte(slice_datetime + _SECOND_REQUEST).with_created_lte(slice_datetime + slice_range + _SECOND_REQUEST).with_limit(100).build(), + _a_response().build(), + ) + http_mocker.get( + _a_request().with_created_gte(slice_datetime + slice_range + _THIRD_REQUEST).with_created_lte(_NOW).with_limit(100).build(), + _a_response().build(), + ) + + self._read(_config().with_start_date(start_date).with_slice_range_in_days(slice_range.days)) + + # request matched http_mocker + + @HttpMocker() + def test_given_lookback_window_when_read_then_request_before_start_date(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=30) + lookback_window = timedelta(days=10) + http_mocker.get( + _a_request().with_created_gte(start_date - lookback_window).with_created_lte(_NOW).with_limit(100).build(), + _a_response().build(), + ) + + self._read(_config().with_start_date(start_date).with_lookback_window_in_days(lookback_window.days)) + + # request matched http_mocker + + @HttpMocker() + def test_given_slice_range_when_read_then_perform_multiple_requests(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=30) + slice_range = timedelta(days=20) + slice_datetime = start_date + slice_range + http_mocker.get( + _a_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(), + _a_response().build(), + ) + http_mocker.get( + _a_request().with_created_gte(slice_datetime + _SECOND_REQUEST).with_created_lte(_NOW).with_limit(100).build(), + _a_response().build(), + ) + + self._read(_config().with_start_date(start_date).with_slice_range_in_days(slice_range.days)) + + @HttpMocker() + def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _a_request().with_any_query_params().build(), + a_response_with_status(400), + ) + output = self._read(_config()) + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + @HttpMocker() + def test_given_http_status_401_when_read_then_stream_is_incomplete(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _a_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config().with_start_date(_A_START_DATE), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _a_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _a_response().with_record(_a_record()).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _a_request().with_any_query_params().build(), + [a_response_with_status(500), _a_response().with_record(_a_record()).build()], + ) + output = self._read(_config()) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _a_request().with_any_query_params().build(), + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_when_read_then_validate_availability_for_full_refresh_and_incremental(self, http_mocker: HttpMocker) -> None: + request = _a_request().with_any_query_params().build() + http_mocker.get( + request, + _a_response().build(), + ) + self._read(_config().with_start_date(_A_START_DATE)) + http_mocker.assert_number_of_calls(request, 3) # one call for full_refresh availability, one call for incremental availability and one call for the actual read + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_initial_state_when_read_then_return_state_based_on_cursor_field(self, http_mocker: HttpMocker) -> None: + cursor_value = int(_A_START_DATE.timestamp()) + 1 + http_mocker.get( + _a_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _a_response().with_record(_a_record().with_cursor(cursor_value)).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + assert output.most_recent_state == {"events": {"created": cursor_value}} + + @HttpMocker() + def test_given_state_when_read_then_use_state_for_query_params(self, http_mocker: HttpMocker) -> None: + state_value = _A_START_DATE + timedelta(seconds=1) + availability_check_requests = _a_request().with_any_query_params().build() + http_mocker.get( + availability_check_requests, + _a_response().with_record(_a_record()).build(), + ) + http_mocker.get( + _a_request().with_created_gte(state_value + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).build(), + _a_response().with_record(_a_record()).build(), + ) + + self._read( + _config().with_start_date(_A_START_DATE), + StateBuilder().with_stream_state("events", {"created": int(state_value.timestamp())}).build() + ) + + # request matched http_mocker + + @HttpMocker() + def test_given_state_more_recent_than_cursor_when_read_then_return_state_based_on_cursor_field(self, http_mocker: HttpMocker) -> None: + cursor_value = int(_A_START_DATE.timestamp()) + 1 + more_recent_than_record_cursor = int(_NOW.timestamp()) - 1 + http_mocker.get( + _a_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _a_response().with_record(_a_record().with_cursor(cursor_value)).build(), + ) + + output = self._read( + _config().with_start_date(_A_START_DATE), + StateBuilder().with_stream_state("events", {"created": more_recent_than_record_cursor}).build() + ) + + assert output.most_recent_state == {"events": {"created": more_recent_than_record_cursor}} + + def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_bank_accounts.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_bank_accounts.py new file mode 100644 index 000000000000..f1c5804173c1 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_bank_accounts.py @@ -0,0 +1,361 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_EVENT_TYPES = ["account.external_account.created", "account.external_account.updated", "account.external_account.deleted"] + +_DATA_FIELD = NestedPath(["data", "object"]) +_OBJECT = "bank_account" +_STREAM_NAME = "external_account_bank_accounts" +_ENDPOINT_TEMPLATE_NAME = "external_bank_accounts" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _external_accounts_request() -> StripeRequestBuilder: + return StripeRequestBuilder.external_accounts_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=75)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any]) -> SourceStripe: + return SourceStripe(catalog, config) + + +def _an_event() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _events_response() -> HttpResponseBuilder: + return create_response_builder( + find_template("events", __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _an_external_bank_account() -> RecordBuilder: + return create_record_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + ) + + +def _external_bank_accounts_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _given_external_accounts_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.external_accounts_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _external_bank_accounts_response().build() + ) + + +def _given_events_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().build() + ) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config), config, catalog, state, expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_object(_OBJECT).with_limit(100).build(), + _external_bank_accounts_response().with_record(_an_external_bank_account()).with_record(_an_external_bank_account()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_object(_OBJECT).with_limit(100).build(), + _external_bank_accounts_response().with_pagination().with_record(_an_external_bank_account().with_id("last_record_id_from_first_page")).build(), + ) + http_mocker.get( + _external_accounts_request().with_starting_after("last_record_id_from_first_page").with_object(_OBJECT).with_limit(100).build(), + _external_bank_accounts_response().with_record(_an_external_bank_account()).with_record(_an_external_bank_account()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 3 + + @HttpMocker() + def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_object(_OBJECT).with_limit(100).build(), + _external_bank_accounts_response().with_record(_an_external_bank_account()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + assert output.records[0].record.data["updated"] == int(_NOW.timestamp()) + + @HttpMocker() + def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + a_response_with_status(400), + ) + output = self._read(_config()) + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + @HttpMocker() + def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _external_bank_accounts_response().with_record(_an_external_bank_account()).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + [a_response_with_status(500), _external_bank_accounts_response().with_record(_an_external_bank_account()).build()], + ) + output = self._read(_config()) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None: + # see https://github.com/airbytehq/airbyte/issues/33499 + events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build() + http_mocker.get( + events_requests, + _events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now + ) + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + _external_bank_accounts_response().build(), + ) + + self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1)) + + http_mocker.assert_number_of_calls(events_requests, 30) + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_state_when_read_then_use_external_accounts_endpoint(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_object(_OBJECT).with_limit(100).build(), + _external_bank_accounts_response().with_record(_an_external_bank_account()).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + assert output.most_recent_state == {_STREAM_NAME: {"updated": int(_NOW.timestamp())}} + + @HttpMocker() + def test_given_state_when_read_then_query_events_using_types_and_state_value_plus_1(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + cursor_value = int(state_datetime.timestamp()) + 1 + + _given_external_accounts_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_cursor(cursor_value).with_field(_DATA_FIELD, _an_external_bank_account().build()) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_object_is_not_back_account_when_read_then_filter_out(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + + _given_external_accounts_availability_check(http_mocker) + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().with_record( + _an_event().with_field(_DATA_FIELD, {"object": "not a bank account"}) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 0 + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_external_accounts_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_pagination().with_record( + _an_event().with_id("last_record_id_from_first_page").with_field(_DATA_FIELD, _an_external_bank_account().build()) + ).build(), + ) + http_mocker.get( + _events_request().with_starting_after("last_record_id_from_first_page").with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_external_account_event()).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_state_and_small_slice_range_when_read_then_perform_multiple_queries(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + slice_range = timedelta(days=3) + slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range + + _given_external_accounts_availability_check(http_mocker) + _given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_external_account_event()).build(), + ) + http_mocker.get( + _events_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_external_account_event()).with_record(self._an_external_account_event()).build(), + ) + + output = self._read( + _config().with_start_date(_NOW - timedelta(days=30)).with_slice_range_in_days(slice_range.days), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None: + # this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the + # external_accounts endpoint + _given_external_accounts_availability_check(http_mocker) + start_date = _NOW - timedelta(days=40) + state_value = _NOW - timedelta(days=39) + events_lower_boundary = _NOW - timedelta(days=30) + http_mocker.get( + _events_request().with_created_gte(events_lower_boundary).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_external_account_event()).build(), + ) + + self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_value.timestamp())}).build(), + ) + + # request matched http_mocker + + def _an_external_account_event(self) -> RecordBuilder: + return _an_event().with_field(_DATA_FIELD, _an_external_bank_account().build()) + + def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_cards.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_cards.py new file mode 100644 index 000000000000..705faf8530a0 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_external_account_cards.py @@ -0,0 +1,366 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_EVENT_TYPES = ["account.external_account.created", "account.external_account.updated", "account.external_account.deleted"] + +_DATA_FIELD = NestedPath(["data", "object"]) +_OBJECT = "card" +_STREAM_NAME = "external_account_cards" +_ENDPOINT_TEMPLATE_NAME = "external_account_cards" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _external_accounts_request() -> StripeRequestBuilder: + return StripeRequestBuilder.external_accounts_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=75)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any]) -> SourceStripe: + return SourceStripe(catalog, config) + + +def _an_event() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _events_response() -> HttpResponseBuilder: + return create_response_builder( + find_template("events", __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _an_external_account_card() -> RecordBuilder: + return create_record_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + ) + + +def _external_accounts_card_response() -> HttpResponseBuilder: + """ + WARNING: this response will not fully match the template as external accounts card are queried by ID and the field "url" is not updated + to match that (it is currently hardcoded to "/v1/accounts/acct_1032D82eZvKYlo2C/external_accounts"). As this has no impact on the + tests, we will leave it as is for now. + """ + return create_response_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _given_external_accounts_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.external_accounts_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _external_accounts_card_response().build() + ) + + +def _given_events_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().build() + ) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config), config, catalog, state, expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_object(_OBJECT).with_limit(100).build(), + _external_accounts_card_response().with_record(_an_external_account_card()).with_record(_an_external_account_card()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_object(_OBJECT).with_limit(100).build(), + _external_accounts_card_response().with_pagination().with_record(_an_external_account_card().with_id("last_record_id_from_first_page")).build(), + ) + http_mocker.get( + _external_accounts_request().with_starting_after("last_record_id_from_first_page").with_object(_OBJECT).with_limit(100).build(), + _external_accounts_card_response().with_record(_an_external_account_card()).with_record(_an_external_account_card()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 3 + + @HttpMocker() + def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_object(_OBJECT).with_limit(100).build(), + _external_accounts_card_response().with_record(_an_external_account_card()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + assert output.records[0].record.data["updated"] == int(_NOW.timestamp()) + + @HttpMocker() + def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + a_response_with_status(400), + ) + output = self._read(_config()) + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + @HttpMocker() + def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _external_accounts_card_response().with_record(_an_external_account_card()).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + [a_response_with_status(500), _external_accounts_card_response().with_record(_an_external_account_card()).build()], + ) + output = self._read(_config()) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None: + # see https://github.com/airbytehq/airbyte/issues/33499 + events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build() + http_mocker.get( + events_requests, + _events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now + ) + http_mocker.get( + _external_accounts_request().with_any_query_params().build(), + _external_accounts_card_response().build(), + ) + + self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1)) + + http_mocker.assert_number_of_calls(events_requests, 30) + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_state_when_read_then_use_external_accounts_endpoint(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _external_accounts_request().with_object(_OBJECT).with_limit(100).build(), + _external_accounts_card_response().with_record(_an_external_account_card()).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + assert output.most_recent_state == {_STREAM_NAME: {"updated": int(_NOW.timestamp())}} + + @HttpMocker() + def test_given_state_when_read_then_query_events_using_types_and_state_value_plus_1(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + cursor_value = int(state_datetime.timestamp()) + 1 + + _given_external_accounts_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_cursor(cursor_value).with_field(_DATA_FIELD, _an_external_account_card().build()) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_object_is_not_back_account_when_read_then_filter_out(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + + _given_external_accounts_availability_check(http_mocker) + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().with_record( + _an_event().with_field(_DATA_FIELD, {"object": "not a card"}) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 0 + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_external_accounts_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_pagination().with_record( + _an_event().with_id("last_record_id_from_first_page").with_field(_DATA_FIELD, _an_external_account_card().build()) + ).build(), + ) + http_mocker.get( + _events_request().with_starting_after("last_record_id_from_first_page").with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_external_account_event()).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_state_and_small_slice_range_when_read_then_perform_multiple_queries(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + slice_range = timedelta(days=3) + slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range + + _given_external_accounts_availability_check(http_mocker) + _given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_external_account_event()).build(), + ) + http_mocker.get( + _events_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_external_account_event()).with_record(self._an_external_account_event()).build(), + ) + + output = self._read( + _config().with_start_date(_NOW - timedelta(days=30)).with_slice_range_in_days(slice_range.days), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None: + # this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the + # external_accounts endpoint + _given_external_accounts_availability_check(http_mocker) + start_date = _NOW - timedelta(days=40) + state_value = _NOW - timedelta(days=39) + events_lower_boundary = _NOW - timedelta(days=30) + http_mocker.get( + _events_request().with_created_gte(events_lower_boundary).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._an_external_account_event()).build(), + ) + + self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_value.timestamp())}).build(), + ) + + # request matched http_mocker + + def _an_external_account_event(self) -> RecordBuilder: + return _an_event().with_field(_DATA_FIELD, _an_external_account_card().build()) + + def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payment_methods.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payment_methods.py new file mode 100644 index 000000000000..d8e9f1450c66 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_payment_methods.py @@ -0,0 +1,347 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_EVENT_TYPES = [ + "payment_method.attached", + "payment_method.automatically_updated", + "payment_method.detached", + "payment_method.updated", +] + +_DATA_FIELD = NestedPath(["data", "object"]) +_STREAM_NAME = "payment_methods" +_ENDPOINT_TEMPLATE_NAME = "payment_methods" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _payment_methods_request() -> StripeRequestBuilder: + return StripeRequestBuilder.payment_methods_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=75)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any]) -> SourceStripe: + return SourceStripe(catalog, config) + + +def _an_event() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _events_response() -> HttpResponseBuilder: + return create_response_builder( + find_template("events", __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _a_payment_method() -> RecordBuilder: + return create_record_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _payment_methods_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _given_payment_methods_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.payment_methods_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _payment_methods_response().build() + ) + + +def _given_events_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().build() + ) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config), config, catalog, state, expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _payment_methods_request().with_limit(100).build(), + _payment_methods_response().with_record(_a_payment_method()).with_record(_a_payment_method()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _payment_methods_request().with_limit(100).build(), + _payment_methods_response().with_pagination().with_record(_a_payment_method().with_id("last_record_id_from_first_page")).build(), + ) + http_mocker.get( + _payment_methods_request().with_starting_after("last_record_id_from_first_page").with_limit(100).build(), + _payment_methods_response().with_record(_a_payment_method()).with_record(_a_payment_method()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 3 + + @HttpMocker() + def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _payment_methods_request().with_limit(100).build(), + _payment_methods_response().with_record(_a_payment_method()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + assert output.records[0].record.data["updated"] == output.records[0].record.data["created"] + + @HttpMocker() + def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _payment_methods_request().with_any_query_params().build(), + a_response_with_status(400), + ) + output = self._read(_config()) + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + @HttpMocker() + def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _payment_methods_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _payment_methods_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _payment_methods_response().with_record(_a_payment_method()).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _payment_methods_request().with_any_query_params().build(), + [a_response_with_status(500), _payment_methods_response().with_record(_a_payment_method()).build()], + ) + output = self._read(_config()) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _payment_methods_request().with_any_query_params().build(), + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None: + # see https://github.com/airbytehq/airbyte/issues/33499 + events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build() + http_mocker.get( + events_requests, + _events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now + ) + http_mocker.get( + _payment_methods_request().with_any_query_params().build(), + _payment_methods_response().build(), + ) + + self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1)) + + http_mocker.assert_number_of_calls(events_requests, 30) + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_state_when_read_then_use_payment_methods_endpoint(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + cursor_value = int(_A_START_DATE.timestamp()) + 1 + http_mocker.get( + _payment_methods_request().with_limit(100).build(), + _payment_methods_response().with_record(_a_payment_method().with_cursor(cursor_value)).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_when_read_then_query_events_using_types_and_state_value_plus_1(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + cursor_value = int(state_datetime.timestamp()) + 1 + + _given_payment_methods_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_cursor(cursor_value).with_field(_DATA_FIELD, _a_payment_method().build()) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_payment_methods_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_pagination().with_record( + _an_event().with_id("last_record_id_from_first_page").with_field(_DATA_FIELD, _a_payment_method().build()) + ).build(), + ) + http_mocker.get( + _events_request().with_starting_after("last_record_id_from_first_page").with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_payment_method_event()).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_state_and_small_slice_range_when_read_then_perform_multiple_queries(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + slice_range = timedelta(days=3) + slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range + + _given_payment_methods_availability_check(http_mocker) + _given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_payment_method_event()).build(), + ) + http_mocker.get( + _events_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_payment_method_event()).with_record(self._a_payment_method_event()).build(), + ) + + output = self._read( + _config().with_start_date(_NOW - timedelta(days=30)).with_slice_range_in_days(slice_range.days), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None: + # this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the + # payment_methods endpoint + _given_payment_methods_availability_check(http_mocker) + start_date = _NOW - timedelta(days=40) + state_value = _NOW - timedelta(days=39) + events_lower_boundary = _NOW - timedelta(days=30) + http_mocker.get( + _events_request().with_created_gte(events_lower_boundary).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_payment_method_event()).build(), + ) + + self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_value.timestamp())}).build(), + ) + + # request matched http_mocker + + def _a_payment_method_event(self) -> RecordBuilder: + return _an_event().with_field(_DATA_FIELD, _a_payment_method().build()) + + def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_persons.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_persons.py new file mode 100644 index 000000000000..db000211be08 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_persons.py @@ -0,0 +1,636 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime, timedelta, timezone +from typing import List +from unittest import TestCase + +import freezegun +from airbyte_cdk.models import FailureType, SyncMode +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import read +from airbyte_cdk.test.mock_http import HttpMocker +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import AirbyteStreamStatus, Level +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_STREAM_NAME = "persons" +_ACCOUNT_ID = "acct_1G9HZLIEn49ers" +_CLIENT_SECRET = "ConfigBuilder default client secret" +_NOW = datetime.now(timezone.utc) +_CONFIG = { + "client_secret": _CLIENT_SECRET, + "account_id": _ACCOUNT_ID, +} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _create_config() -> ConfigBuilder: + return ConfigBuilder().with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _create_catalog(sync_mode: SyncMode = SyncMode.full_refresh): + return CatalogBuilder().with_stream(name="persons", sync_mode=sync_mode).build() + + +def _create_accounts_request() -> StripeRequestBuilder: + return StripeRequestBuilder.accounts_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _create_persons_request(parent_account_id: str = _ACCOUNT_ID) -> StripeRequestBuilder: + return StripeRequestBuilder.persons_endpoint(parent_account_id, _ACCOUNT_ID, _CLIENT_SECRET) + + +def _create_events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _create_response() -> HttpResponseBuilder: + return create_response_builder( + response_template=find_template("accounts", __file__), + records_path=FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _create_record(resource: str) -> RecordBuilder: + return create_record_builder( + find_template(resource, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created") + ) + + +def _create_persons_event_record(event_type: str) -> RecordBuilder: + event_record = create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + person_record = create_record_builder( + find_template("persons", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created") + ) + + return event_record.with_field(NestedPath(["data", "object"]), person_record.build()).with_field(NestedPath(["type"]), event_type) + + +def emits_successful_sync_status_messages(status_messages: List[AirbyteStreamStatus]) -> bool: + return (len(status_messages) == 3 and status_messages[0] == AirbyteStreamStatus.STARTED + and status_messages[1] == AirbyteStreamStatus.RUNNING and status_messages[2] == AirbyteStreamStatus.COMPLETE) + + +@freezegun.freeze_time(_NOW.isoformat()) +class PersonsTest(TestCase): + @HttpMocker() + def test_full_refresh(self, http_mocker): + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + ) + + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types(["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog()) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses(_STREAM_NAME)) + assert len(actual_messages.records) == 2 + + @HttpMocker() + def test_parent_pagination(self, http_mocker): + # First parent stream accounts first page request + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts").with_id("last_page_record_id")).with_pagination().build(), + ) + + # Second parent stream accounts second page request + http_mocker.get( + _create_accounts_request().with_limit(100).with_starting_after("last_page_record_id").build(), + _create_response().with_record(record=_create_record("accounts").with_id("last_page_record_id")).build(), + ) + + # Persons stream first page request + http_mocker.get( + _create_persons_request(parent_account_id="last_page_record_id").with_limit(100).build(), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + ) + + # The persons stream makes a final call to events endpoint + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog()) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses(_STREAM_NAME)) + assert len(actual_messages.records) == 4 + + @HttpMocker() + def test_substream_pagination(self, http_mocker): + # First parent stream accounts first page request + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + # Persons stream first page request + http_mocker.get( + _create_persons_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons").with_id("last_page_record_id")).with_pagination().build(), + ) + + # Persons stream second page request + http_mocker.get( + _create_persons_request().with_limit(100).with_starting_after("last_page_record_id").build(), + _create_response().with_record(record=_create_record("persons")).with_record( + record=_create_record("persons")).build(), + ) + + # The persons stream makes a final call to events endpoint + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog()) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses(_STREAM_NAME)) + assert len(actual_messages.records) == 4 + + @HttpMocker() + def test_accounts_400_error(self, http_mocker: HttpMocker): + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + a_response_with_status(400), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog()) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + error_log_messages = [message for message in actual_messages.logs if message.log.level == Level.ERROR] + + # For Stripe, streams that get back a 400 or 403 response code are skipped over silently without throwing an error as part of + # this connector's availability strategy + assert len(actual_messages.get_stream_statuses(_STREAM_NAME)) == 0 + assert len(error_log_messages) > 0 + + @HttpMocker() + def test_persons_400_error(self, http_mocker: HttpMocker): + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + # Persons stream first page request + http_mocker.get( + _create_persons_request().with_limit(100).build(), + a_response_with_status(400), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog()) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + error_log_messages = [message for message in actual_messages.logs if message.log.level == Level.ERROR] + + # For Stripe, streams that get back a 400 or 403 response code are skipped over silently without throwing an error as part of + # this connector's availability strategy. They are however reported in the log messages + assert len(actual_messages.get_stream_statuses(_STREAM_NAME)) == 0 + assert len(error_log_messages) > 0 + + @HttpMocker() + def test_accounts_401_error(self, http_mocker: HttpMocker): + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + a_response_with_status(401), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog()) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog(), expecting_exception=True) + + assert actual_messages.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_persons_401_error(self, http_mocker: HttpMocker): + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + # Persons stream first page request + http_mocker.get( + _create_persons_request().with_limit(100).build(), + a_response_with_status(401), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog()) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog(), expecting_exception=True) + + assert actual_messages.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_persons_403_error(self, http_mocker: HttpMocker): + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + # Persons stream first page request + http_mocker.get( + _create_persons_request().with_limit(100).build(), + a_response_with_status(403), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog()) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog(), expecting_exception=True) + error_log_messages = [message for message in actual_messages.logs if message.log.level == Level.ERROR] + + # For Stripe, streams that get back a 400 or 403 response code are skipped over silently without throwing an error as part of + # this connector's availability strategy + assert len(actual_messages.get_stream_statuses(_STREAM_NAME)) == 0 + assert len(error_log_messages) > 0 + + @HttpMocker() + def test_incremental_with_recent_state(self, http_mocker: HttpMocker): + state_datetime = _NOW - timedelta(days=5) + cursor_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + ) + + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types(["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_persons_event_record(event_type="person.created")).with_record(record=_create_persons_event_record(event_type="person.created")).build(), + ) + + http_mocker.get( + _create_events_request().with_created_gte(cursor_datetime).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_persons_event_record(event_type="person.created")).build(), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(sync_mode=SyncMode.incremental)) + actual_messages = read( + source, + config=_CONFIG, + catalog=_create_catalog(sync_mode=SyncMode.incremental), + state=StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses(_STREAM_NAME)) + assert actual_messages.most_recent_state == {"persons": {"updated": int(state_datetime.timestamp())}} + assert len(actual_messages.records) == 1 + + @HttpMocker() + def test_incremental_with_deleted_event(self, http_mocker: HttpMocker): + state_datetime = _NOW - timedelta(days=5) + cursor_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + ) + + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types(["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_persons_event_record(event_type="person.created")).with_record(record=_create_persons_event_record(event_type="person.deleted")).build(), + ) + + http_mocker.get( + _create_events_request().with_created_gte(cursor_datetime).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_persons_event_record(event_type="person.deleted")).build(), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(sync_mode=SyncMode.incremental)) + actual_messages = read( + source, + config=_CONFIG, + catalog=_create_catalog(sync_mode=SyncMode.incremental), + state=StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses(_STREAM_NAME)) + assert actual_messages.most_recent_state == {"persons": {"updated": int(state_datetime.timestamp())}} + assert len(actual_messages.records) == 1 + assert actual_messages.records[0].record.data.get("is_deleted") + + @HttpMocker() + def test_incremental_with_newer_start_date(self, http_mocker): + start_datetime = _NOW - timedelta(days=7) + state_datetime = _NOW - timedelta(days=15) + config = _create_config().with_start_date(start_datetime).build() + + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + ) + + http_mocker.get( + _create_events_request().with_created_gte(start_datetime).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_persons_event_record(event_type="person.created")).build(), + ) + + source = SourceStripe(config=config, catalog=_create_catalog(sync_mode=SyncMode.incremental)) + actual_messages = read( + source, + config=config, + catalog=_create_catalog(sync_mode=SyncMode.incremental), + state=StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses(_STREAM_NAME)) + assert actual_messages.most_recent_state == {"persons": {"updated": int(state_datetime.timestamp())}} + assert len(actual_messages.records) == 1 + + @HttpMocker() + def test_rate_limited_parent_stream_accounts(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + [ + a_response_with_status(429), + _create_response().with_record(record=_create_record("accounts")).build(), + ], + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + ) + + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog()) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses(_STREAM_NAME)) + assert len(actual_messages.records) == 2 + + @HttpMocker() + def test_rate_limited_substream_persons(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + [ + a_response_with_status(429), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + ] + ) + + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog()) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses(_STREAM_NAME)) + assert len(actual_messages.records) == 2 + + @HttpMocker() + def test_rate_limited_incremental_events(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + cursor_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + ) + + # Mock when check_availability is run on the persons incremental stream + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_persons_event_record(event_type="person.created")).with_record( + record=_create_persons_event_record(event_type="person.created")).build(), + ) + + http_mocker.get( + _create_events_request().with_created_gte(cursor_datetime).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + [ + a_response_with_status(429), + _create_response().with_record(record=_create_persons_event_record(event_type="person.created")).build(), + ] + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(sync_mode=SyncMode.incremental)) + actual_messages = read( + source, + config=_CONFIG, + catalog=_create_catalog(sync_mode=SyncMode.incremental), + state=StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses(_STREAM_NAME)) + assert actual_messages.most_recent_state == {"persons": {"updated": int(state_datetime.timestamp())}} + assert len(actual_messages.records) == 1 + + @HttpMocker() + def test_rate_limit_max_attempts_exceeded(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + [ + # Used to pass the initial check_availability before starting the sync + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + a_response_with_status(429), # Returns 429 on all subsequent requests to test the maximum number of retries + ] + ) + + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog()) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert len(actual_messages.errors) == 1 + + @HttpMocker() + def test_incremental_rate_limit_max_attempts_exceeded(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + cursor_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + ) + + # Mock when check_availability is run on the persons incremental stream + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_persons_event_record(event_type="person.created")).with_record( + record=_create_persons_event_record(event_type="person.created")).build(), + ) + + http_mocker.get( + _create_events_request().with_created_gte(cursor_datetime).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + a_response_with_status(429), # Returns 429 on all subsequent requests to test the maximum number of retries + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog(sync_mode=SyncMode.incremental)) + actual_messages = read( + source, + config=_CONFIG, + catalog=_create_catalog(sync_mode=SyncMode.incremental), + state=StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(actual_messages.errors) == 1 + + @HttpMocker() + def test_server_error_parent_stream_accounts(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + [ + a_response_with_status(500), + _create_response().with_record(record=_create_record("accounts")).build(), + ], + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + ) + + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog()) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses(_STREAM_NAME)) + assert len(actual_messages.records) == 2 + + @HttpMocker() + def test_server_error_substream_persons(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + [ + a_response_with_status(500), + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + ] + ) + + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog()) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses(_STREAM_NAME)) + assert len(actual_messages.records) == 2 + + @HttpMocker() + def test_server_error_max_attempts_exceeded(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _create_accounts_request().with_limit(100).build(), + _create_response().with_record(record=_create_record("accounts")).build(), + ) + + http_mocker.get( + _create_persons_request().with_limit(100).build(), + [ + # Used to pass the initial check_availability before starting the sync + _create_response().with_record(record=_create_record("persons")).with_record(record=_create_record("persons")).build(), + a_response_with_status(500), # Returns 429 on all subsequent requests to test the maximum number of retries + ] + ) + + http_mocker.get( + _create_events_request().with_created_gte(_NOW - timedelta(days=30)).with_created_lte(_NOW).with_limit(100).with_types( + ["person.created", "person.updated", "person.deleted"]).build(), + _create_response().with_record(record=_create_record("events")).with_record(record=_create_record("events")).build(), + ) + + source = SourceStripe(config=_CONFIG, catalog=_create_catalog()) + actual_messages = read(source, config=_CONFIG, catalog=_create_catalog()) + + assert len(actual_messages.errors) == 1 diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_reviews.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_reviews.py new file mode 100644 index 000000000000..45ee0219f8da --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_reviews.py @@ -0,0 +1,374 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_EVENT_TYPES = ["review.closed", "review.opened"] + +_DATA_FIELD = NestedPath(["data", "object"]) +_STREAM_NAME = "reviews" +_ENDPOINT_TEMPLATE_NAME = "reviews" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _reviews_request() -> StripeRequestBuilder: + return StripeRequestBuilder.reviews_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=75)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any]) -> SourceStripe: + return SourceStripe(catalog, config) + + +def _an_event() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _events_response() -> HttpResponseBuilder: + return create_response_builder( + find_template("events", __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _a_review() -> RecordBuilder: + return create_record_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _reviews_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _given_reviews_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.reviews_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _reviews_response().build() + ) + + +def _given_events_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().build() + ) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config), config, catalog, state, expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _reviews_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _reviews_response().with_record(_a_review()).with_record(_a_review()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _reviews_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _reviews_response().with_pagination().with_record(_a_review().with_id("last_record_id_from_first_page")).build(), + ) + http_mocker.get( + _reviews_request().with_starting_after("last_record_id_from_first_page").with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _reviews_response().with_record(_a_review()).with_record(_a_review()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_no_state_when_read_then_return_ignore_lookback(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _reviews_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _reviews_response().with_record(_a_review()).build(), + ) + + self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + # request matched http_mocker + + @HttpMocker() + def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _reviews_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _reviews_response().with_record(_a_review()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + assert output.records[0].record.data["updated"] == output.records[0].record.data["created"] + + @HttpMocker() + def test_given_slice_range_when_read_then_perform_multiple_requests(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=30) + slice_range = timedelta(days=20) + slice_datetime = start_date + slice_range + + _given_events_availability_check(http_mocker) + http_mocker.get( + _reviews_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(), + _reviews_response().build(), + ) + http_mocker.get( + _reviews_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).build(), + _reviews_response().build(), + ) + + self._read(_config().with_start_date(start_date).with_slice_range_in_days(slice_range.days)) + + # request matched http_mocker + + @HttpMocker() + def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _reviews_request().with_any_query_params().build(), + a_response_with_status(400), + ) + output = self._read(_config()) + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + @HttpMocker() + def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _reviews_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _reviews_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _reviews_response().with_record(_a_review()).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _reviews_request().with_any_query_params().build(), + [a_response_with_status(500), _reviews_response().with_record(_a_review()).build()], + ) + output = self._read(_config()) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _reviews_request().with_any_query_params().build(), + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None: + # see https://github.com/airbytehq/airbyte/issues/33499 + events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build() + http_mocker.get( + events_requests, + _events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now + ) + http_mocker.get( + _reviews_request().with_any_query_params().build(), + _reviews_response().build(), + ) + + self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1)) + + http_mocker.assert_number_of_calls(events_requests, 30) + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_state_when_read_then_use_reviews_endpoint(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + cursor_value = int(_A_START_DATE.timestamp()) + 1 + http_mocker.get( + _reviews_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _reviews_response().with_record(_a_review().with_cursor(cursor_value)).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_when_read_then_query_events_using_types_and_state_value_plus_1(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + cursor_value = int(state_datetime.timestamp()) + 1 + + _given_reviews_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_cursor(cursor_value).with_field(_DATA_FIELD, _a_review().build()) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_reviews_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_pagination().with_record( + _an_event().with_id("last_record_id_from_first_page").with_field(_DATA_FIELD, _a_review().build()) + ).build(), + ) + http_mocker.get( + _events_request().with_starting_after("last_record_id_from_first_page").with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_review_event()).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_state_and_small_slice_range_when_read_then_perform_multiple_queries(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + slice_range = timedelta(days=3) + slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range + + _given_reviews_availability_check(http_mocker) + _given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_review_event()).build(), + ) + http_mocker.get( + _events_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_review_event()).with_record(self._a_review_event()).build(), + ) + + output = self._read( + _config().with_start_date(_NOW - timedelta(days=30)).with_slice_range_in_days(slice_range.days), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None: + # this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the + # reviews endpoint + _given_reviews_availability_check(http_mocker) + start_date = _NOW - timedelta(days=40) + state_value = _NOW - timedelta(days=39) + events_lower_boundary = _NOW - timedelta(days=30) + http_mocker.get( + _events_request().with_created_gte(events_lower_boundary).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_review_event()).build(), + ) + + self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_value.timestamp())}).build(), + ) + + # request matched http_mocker + + def _a_review_event(self) -> RecordBuilder: + return _an_event().with_field(_DATA_FIELD, _a_review().build()) + + def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_transactions.py b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_transactions.py new file mode 100644 index 000000000000..8c4db0697223 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/integration/test_transactions.py @@ -0,0 +1,374 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional +from unittest import TestCase + +import freezegun +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + NestedPath, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from airbyte_protocol.models import ConfiguredAirbyteCatalog, FailureType, SyncMode +from integration.config import ConfigBuilder +from integration.pagination import StripePaginationStrategy +from integration.request_builder import StripeRequestBuilder +from integration.response_builder import a_response_with_status +from source_stripe import SourceStripe + +_EVENT_TYPES = ["issuing_transaction.created", "issuing_transaction.updated"] + +_DATA_FIELD = NestedPath(["data", "object"]) +_STREAM_NAME = "transactions" +_ENDPOINT_TEMPLATE_NAME = "issuing_transactions" +_NOW = datetime.now(timezone.utc) +_A_START_DATE = _NOW - timedelta(days=60) +_ACCOUNT_ID = "account_id" +_CLIENT_SECRET = "client_secret" +_NO_STATE = {} +_AVOIDING_INCLUSIVE_BOUNDARIES = timedelta(seconds=1) + + +def _transactions_request() -> StripeRequestBuilder: + return StripeRequestBuilder.issuing_transactions_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _events_request() -> StripeRequestBuilder: + return StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET) + + +def _config() -> ConfigBuilder: + return ConfigBuilder().with_start_date(_NOW - timedelta(days=75)).with_account_id(_ACCOUNT_ID).with_client_secret(_CLIENT_SECRET) + + +def _catalog(sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(_STREAM_NAME, sync_mode).build() + + +def _source(catalog: ConfiguredAirbyteCatalog, config: Dict[str, Any]) -> SourceStripe: + return SourceStripe(catalog, config) + + +def _an_event() -> RecordBuilder: + return create_record_builder( + find_template("events", __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _events_response() -> HttpResponseBuilder: + return create_response_builder( + find_template("events", __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _a_transaction() -> RecordBuilder: + return create_record_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + record_id_path=FieldPath("id"), + record_cursor_path=FieldPath("created"), + ) + + +def _transactions_response() -> HttpResponseBuilder: + return create_response_builder( + find_template(_ENDPOINT_TEMPLATE_NAME, __file__), + FieldPath("data"), + pagination_strategy=StripePaginationStrategy() + ) + + +def _given_transactions_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.issuing_transactions_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _transactions_response().build() + ) + + +def _given_events_availability_check(http_mocker: HttpMocker) -> None: + http_mocker.get( + StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build(), + _events_response().build() + ) + + +def _read( + config_builder: ConfigBuilder, + sync_mode: SyncMode, + state: Optional[Dict[str, Any]] = None, + expecting_exception: bool = False +) -> EntrypointOutput: + catalog = _catalog(sync_mode) + config = config_builder.build() + return read(_source(catalog, config), config, catalog, state, expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class FullRefreshTest(TestCase): + + @HttpMocker() + def test_given_one_page_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _transactions_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _transactions_response().with_record(_a_transaction()).with_record(_a_transaction()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_many_pages_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _transactions_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _transactions_response().with_pagination().with_record(_a_transaction().with_id("last_record_id_from_first_page")).build(), + ) + http_mocker.get( + _transactions_request().with_starting_after("last_record_id_from_first_page").with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _transactions_response().with_record(_a_transaction()).with_record(_a_transaction()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE)) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_no_state_when_read_then_return_ignore_lookback(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _transactions_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _transactions_response().with_record(_a_transaction()).build(), + ) + + self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + # request matched http_mocker + + @HttpMocker() + def test_when_read_then_add_cursor_field(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _transactions_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _transactions_response().with_record(_a_transaction()).build(), + ) + + output = self._read(_config().with_start_date(_A_START_DATE).with_lookback_window_in_days(10)) + + assert output.records[0].record.data["updated"] == output.records[0].record.data["created"] + + @HttpMocker() + def test_given_slice_range_when_read_then_perform_multiple_requests(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=30) + slice_range = timedelta(days=20) + slice_datetime = start_date + slice_range + + _given_events_availability_check(http_mocker) + http_mocker.get( + _transactions_request().with_created_gte(start_date).with_created_lte(slice_datetime).with_limit(100).build(), + _transactions_response().build(), + ) + http_mocker.get( + _transactions_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).build(), + _transactions_response().build(), + ) + + self._read(_config().with_start_date(start_date).with_slice_range_in_days(slice_range.days)) + + # request matched http_mocker + + @HttpMocker() + def test_given_http_status_400_when_read_then_stream_is_ignored(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _transactions_request().with_any_query_params().build(), + a_response_with_status(400), + ) + output = self._read(_config()) + assert len(output.get_stream_statuses(_STREAM_NAME)) == 0 + + @HttpMocker() + def test_given_http_status_401_when_read_then_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _transactions_request().with_any_query_params().build(), + a_response_with_status(401), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_rate_limited_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _transactions_request().with_any_query_params().build(), + [ + a_response_with_status(429), + _transactions_response().with_record(_a_transaction()).build(), + ], + ) + output = self._read(_config().with_start_date(_A_START_DATE)) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_once_before_200_when_read_then_retry_and_return_records(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + http_mocker.get( + _transactions_request().with_any_query_params().build(), + [a_response_with_status(500), _transactions_response().with_record(_a_transaction()).build()], + ) + output = self._read(_config()) + assert len(output.records) == 1 + + @HttpMocker() + def test_given_http_status_500_on_availability_when_read_then_raise_system_error(self, http_mocker: HttpMocker) -> None: + http_mocker.get( + _transactions_request().with_any_query_params().build(), + a_response_with_status(500), + ) + output = self._read(_config(), expecting_exception=True) + assert output.errors[-1].trace.error.failure_type == FailureType.system_error + + @HttpMocker() + def test_given_small_slice_range_when_read_then_availability_check_performs_too_many_queries(self, http_mocker: HttpMocker) -> None: + # see https://github.com/airbytehq/airbyte/issues/33499 + events_requests = StripeRequestBuilder.events_endpoint(_ACCOUNT_ID, _CLIENT_SECRET).with_any_query_params().build() + http_mocker.get( + events_requests, + _events_response().build() # it is important that the event response does not have a record. This is not far fetched as this is what would happend 30 days before now + ) + http_mocker.get( + _transactions_request().with_any_query_params().build(), + _transactions_response().build(), + ) + + self._read(_config().with_start_date(_NOW - timedelta(days=60)).with_slice_range_in_days(1)) + + http_mocker.assert_number_of_calls(events_requests, 30) + + def _read(self, config: ConfigBuilder, expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.full_refresh, expecting_exception=expecting_exception) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IncrementalTest(TestCase): + + @HttpMocker() + def test_given_no_state_when_read_then_use_transactions_endpoint(self, http_mocker: HttpMocker) -> None: + _given_events_availability_check(http_mocker) + cursor_value = int(_A_START_DATE.timestamp()) + 1 + http_mocker.get( + _transactions_request().with_created_gte(_A_START_DATE).with_created_lte(_NOW).with_limit(100).build(), + _transactions_response().with_record(_a_transaction().with_cursor(cursor_value)).build(), + ) + output = self._read(_config().with_start_date(_A_START_DATE), _NO_STATE) + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_when_read_then_query_events_using_types_and_state_value_plus_1(self, http_mocker: HttpMocker) -> None: + start_date = _NOW - timedelta(days=40) + state_datetime = _NOW - timedelta(days=5) + cursor_value = int(state_datetime.timestamp()) + 1 + + _given_transactions_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record( + _an_event().with_cursor(cursor_value).with_field(_DATA_FIELD, _a_transaction().build()) + ).build(), + ) + + output = self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert output.most_recent_state == {_STREAM_NAME: {"updated": cursor_value}} + + @HttpMocker() + def test_given_state_and_pagination_when_read_then_return_records(self, http_mocker: HttpMocker) -> None: + _given_transactions_availability_check(http_mocker) + _given_events_availability_check(http_mocker) + state_datetime = _NOW - timedelta(days=5) + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_pagination().with_record( + _an_event().with_id("last_record_id_from_first_page").with_field(_DATA_FIELD, _a_transaction().build()) + ).build(), + ) + http_mocker.get( + _events_request().with_starting_after("last_record_id_from_first_page").with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_transaction_event()).build(), + ) + + output = self._read( + _config(), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 2 + + @HttpMocker() + def test_given_state_and_small_slice_range_when_read_then_perform_multiple_queries(self, http_mocker: HttpMocker) -> None: + state_datetime = _NOW - timedelta(days=5) + slice_range = timedelta(days=3) + slice_datetime = state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES + slice_range + + _given_transactions_availability_check(http_mocker) + _given_events_availability_check(http_mocker) # the availability check does not consider the state so we need to define a generic availability check + http_mocker.get( + _events_request().with_created_gte(state_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(slice_datetime).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_transaction_event()).build(), + ) + http_mocker.get( + _events_request().with_created_gte(slice_datetime + _AVOIDING_INCLUSIVE_BOUNDARIES).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_transaction_event()).with_record(self._a_transaction_event()).build(), + ) + + output = self._read( + _config().with_start_date(_NOW - timedelta(days=30)).with_slice_range_in_days(slice_range.days), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_datetime.timestamp())}).build(), + ) + + assert len(output.records) == 3 + + @HttpMocker() + def test_given_state_earlier_than_30_days_when_read_then_query_events_using_types_and_event_lower_boundary(self, http_mocker: HttpMocker) -> None: + # this seems odd as we would miss some data between start_date and events_lower_boundary. In that case, we should hit the + # transactions endpoint + _given_transactions_availability_check(http_mocker) + start_date = _NOW - timedelta(days=40) + state_value = _NOW - timedelta(days=39) + events_lower_boundary = _NOW - timedelta(days=30) + http_mocker.get( + _events_request().with_created_gte(events_lower_boundary).with_created_lte(_NOW).with_limit(100).with_types(_EVENT_TYPES).build(), + _events_response().with_record(self._a_transaction_event()).build(), + ) + + self._read( + _config().with_start_date(start_date), + StateBuilder().with_stream_state(_STREAM_NAME, {"updated": int(state_value.timestamp())}).build(), + ) + + # request matched http_mocker + + def _a_transaction_event(self) -> RecordBuilder: + return _an_event().with_field(_DATA_FIELD, _a_transaction().build()) + + def _read(self, config: ConfigBuilder, state: Optional[Dict[str, Any]], expecting_exception: bool = False) -> EntrypointOutput: + return _read(config, SyncMode.incremental, state, expecting_exception) diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/400.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/400.json new file mode 100644 index 000000000000..4ded0c0a8919 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/400.json @@ -0,0 +1,7 @@ +{ + "error": { + "message": "Your account is not set up to use Issuing. Please visit https://dashboard.stripe.com/issuing/overview to get started.", + "request_log_url": "https://dashboard.stripe.com/test/logs/req_OzHOvvVQ4ALtKm?t=1702476901", + "type": "invalid_request_error" + } +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/401.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/401.json new file mode 100644 index 000000000000..67f5dfd22e07 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/401.json @@ -0,0 +1,6 @@ +{ + "error": { + "message": "Invalid API Key provided: sk_test_*****************************************************mFeM", + "type": "invalid_request_error" + } +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/403.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/403.json new file mode 100644 index 000000000000..9fadb9f2fe1f --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/403.json @@ -0,0 +1,8 @@ +{ + "error": { + "code": "oauth_not_supported", + "message": "This application does not have the required permissions for this endpoint on account 'acct_1G9HZLIEn49ers'.", + "request_log_url": "https://dashboard.stripe.com/acct_1IB2IIRPz4Eoy76F/test/logs/req_yfhmzM1ChMWuhX?t=1703806215", + "type": "invalid_request_error" + } +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/429.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/429.json new file mode 100644 index 000000000000..249f882eecc0 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/429.json @@ -0,0 +1,8 @@ +{ + "error": { + "message": "Request rate limit exceeded. Learn more about rate limits here https://stripe.com/docs/rate-limits.", + "type": "invalid_request_error", + "code": "rate_limit", + "doc_url": "https://stripe.com/docs/error-codes/rate-limit" + } +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/500.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/500.json new file mode 100644 index 000000000000..0077e9a45a61 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/500.json @@ -0,0 +1,3 @@ +{ + "unknown": "maxi297: I could not reproduce the issue hence this response will not look like the actual 500 status response" +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/accounts.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/accounts.json new file mode 100644 index 000000000000..475961a4ed4f --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/accounts.json @@ -0,0 +1,136 @@ +{ + "object": "list", + "url": "/v1/accounts", + "has_more": false, + "data": [ + { + "id": "acct_1G9HZLIEn49ers", + "object": "account", + "business_profile": { + "mcc": null, + "name": null, + "product_description": null, + "support_address": null, + "support_email": null, + "support_phone": null, + "support_url": null, + "url": null + }, + "business_type": null, + "capabilities": { + "card_payments": "inactive", + "transfers": "inactive" + }, + "charges_enabled": false, + "country": "US", + "created": 1695830751, + "default_currency": "usd", + "details_submitted": false, + "email": "john.lynch@49ers.com", + "external_accounts": { + "object": "list", + "data": [], + "has_more": false, + "total_count": 0, + "url": "/v1/accounts/acct_1G9HZLIEn49ers/external_accounts" + }, + "future_requirements": { + "alternatives": [], + "current_deadline": null, + "currently_due": [], + "disabled_reason": null, + "errors": [], + "eventually_due": [], + "past_due": [], + "pending_verification": [] + }, + "metadata": {}, + "payouts_enabled": false, + "requirements": { + "alternatives": [], + "current_deadline": null, + "currently_due": [ + "business_profile.mcc", + "business_profile.url", + "business_type", + "external_account", + "representative.first_name", + "representative.last_name", + "tos_acceptance.date", + "tos_acceptance.ip" + ], + "disabled_reason": "requirements.past_due", + "errors": [], + "eventually_due": [ + "business_profile.mcc", + "business_profile.url", + "business_type", + "external_account", + "representative.first_name", + "representative.last_name", + "tos_acceptance.date", + "tos_acceptance.ip" + ], + "past_due": [ + "business_profile.mcc", + "business_profile.url", + "business_type", + "external_account", + "representative.first_name", + "representative.last_name", + "tos_acceptance.date", + "tos_acceptance.ip" + ], + "pending_verification": [] + }, + "settings": { + "bacs_debit_payments": {}, + "branding": { + "icon": null, + "logo": null, + "primary_color": null, + "secondary_color": null + }, + "card_issuing": { + "tos_acceptance": { + "date": null, + "ip": null + } + }, + "card_payments": { + "decline_on": { + "avs_failure": false, + "cvc_failure": false + }, + "statement_descriptor_prefix": null, + "statement_descriptor_prefix_kana": null, + "statement_descriptor_prefix_kanji": null + }, + "dashboard": { + "display_name": null, + "timezone": "Etc/UTC" + }, + "payments": { + "statement_descriptor": null, + "statement_descriptor_kana": null, + "statement_descriptor_kanji": null + }, + "payouts": { + "debit_negative_balances": false, + "schedule": { + "delay_days": 2, + "interval": "daily" + }, + "statement_descriptor": null + }, + "sepa_debit_payments": {} + }, + "tos_acceptance": { + "date": null, + "ip": null, + "user_agent": null + }, + "type": "custom" + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/application_fees.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/application_fees.json new file mode 100644 index 000000000000..97bb806e6bbe --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/application_fees.json @@ -0,0 +1,138 @@ +{ + "object": "list", + "url": "/v1/application_fees", + "has_more": false, + "data": [ + { + "id": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "object": "application_fee", + "account": "acct_164wxjKbnvuxQXGu", + "amount": 105, + "amount_refunded": 105, + "application": "ca_32D88BD1qLklliziD7gYQvctJIhWBSQ7", + "balance_transaction": "txn_1032HU2eZvKYlo2CEPtcnUvl", + "charge": "ch_1B73DOKbnvuxQXGurbwPqzsu", + "created": 1506609734, + "currency": "gbp", + "livemode": false, + "originating_transaction": null, + "refunded": true, + "refunds": { + "object": "list", + "data": [ + { + "id": "fr_1MBoV6KbnvuxQXGucP0PaPPO", + "object": "fee_refund", + "amount": 0, + "balance_transaction": null, + "created": 1670284508, + "currency": "usd", + "fee": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "metadata": {} + }, + { + "id": "fr_1MBoU0KbnvuxQXGu2wCCz4Bb", + "object": "fee_refund", + "amount": 0, + "balance_transaction": null, + "created": 1670284441, + "currency": "usd", + "fee": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "metadata": {} + }, + { + "id": "fr_1MBoRzKbnvuxQXGuvKkBKkSR", + "object": "fee_refund", + "amount": 0, + "balance_transaction": null, + "created": 1670284315, + "currency": "usd", + "fee": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "metadata": {} + }, + { + "id": "fr_1MBoPOKbnvuxQXGueOBnke22", + "object": "fee_refund", + "amount": 0, + "balance_transaction": null, + "created": 1670284154, + "currency": "usd", + "fee": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "metadata": {} + }, + { + "id": "fr_1MBoOGKbnvuxQXGu6EPQI2Zp", + "object": "fee_refund", + "amount": 0, + "balance_transaction": null, + "created": 1670284084, + "currency": "usd", + "fee": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "metadata": {} + }, + { + "id": "fr_1MBoMUKbnvuxQXGu8Y0Peaoy", + "object": "fee_refund", + "amount": 0, + "balance_transaction": null, + "created": 1670283974, + "currency": "usd", + "fee": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "metadata": {} + }, + { + "id": "fr_1MAgZBKbnvuxQXGuLTUrgGeq", + "object": "fee_refund", + "amount": 0, + "balance_transaction": null, + "created": 1670015681, + "currency": "usd", + "fee": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "metadata": {} + }, + { + "id": "fr_1JAu9EKbnvuxQXGuRdZYkxVW", + "object": "fee_refund", + "amount": 0, + "balance_transaction": null, + "created": 1625738880, + "currency": "usd", + "fee": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "metadata": { + "order_id": "6735" + } + }, + { + "id": "fr_1HZK0UKbnvuxQXGuS428gH0W", + "object": "fee_refund", + "amount": 0, + "balance_transaction": null, + "created": 1602005482, + "currency": "usd", + "fee": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "metadata": {} + }, + { + "id": "fr_D0s7fGBKB40Twy", + "object": "fee_refund", + "amount": 138, + "balance_transaction": "txn_1CaqNg2eZvKYlo2C75cA3Euk", + "created": 1528486576, + "currency": "usd", + "fee": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "metadata": {} + } + ], + "has_more": false, + "url": "/v1/application_fees/fee_1B73DOKbnvuxQXGuhY8Aw0TN/refunds" + }, + "source": { + "fee_type": "charge_application_fee", + "resource": { + "charge": "ch_1B73DOKbnvuxQXGurbwPqzsu", + "type": "charge" + } + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/application_fees_refunds.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/application_fees_refunds.json new file mode 100644 index 000000000000..47eacf5fad9f --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/application_fees_refunds.json @@ -0,0 +1,17 @@ +{ + "object": "list", + "url": "/v1/application_fees/fr_1MtJRpKbnvuxQXGuM6Ww0D24/refunds", + "has_more": false, + "data": [ + { + "id": "fr_1MtJRpKbnvuxQXGuM6Ww0D24", + "object": "fee_refund", + "amount": 100, + "balance_transaction": null, + "created": 1680651573, + "currency": "usd", + "fee": "fee_1B73DOKbnvuxQXGuhY8Aw0TN", + "metadata": {} + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/bank_accounts.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/bank_accounts.json new file mode 100644 index 000000000000..bad75c218964 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/bank_accounts.json @@ -0,0 +1,23 @@ +{ + "object": "list", + "url": "/v1/customers/cus_9s6XI9OFIdpjIg/bank_accounts", + "has_more": false, + "data": [ + { + "id": "ba_1MvoIJ2eZvKYlo2CO9f0MabO", + "object": "bank_account", + "account_holder_name": "Jane Austen", + "account_holder_type": "company", + "account_type": null, + "bank_name": "STRIPE TEST BANK", + "country": "US", + "currency": "usd", + "customer": "cus_9s6XI9OFIdpjIg", + "fingerprint": "1JWtPxqbdX5Gamtc", + "last4": "6789", + "metadata": {}, + "routing_number": "110000000", + "status": "new" + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/customers_expand_data_source.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/customers_expand_data_source.json new file mode 100644 index 000000000000..182aa0e44e9f --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/customers_expand_data_source.json @@ -0,0 +1,43 @@ +{ + "object": "list", + "url": "/v1/customers", + "has_more": false, + "data": [ + { + "id": "cus_NffrFeUfNV2Hib", + "object": "customer", + "address": null, + "balance": 0, + "created": 1680893993, + "currency": null, + "default_source": null, + "delinquent": false, + "description": null, + "discount": null, + "email": "jennyrosen@example.com", + "invoice_prefix": "0759376C", + "invoice_settings": { + "custom_fields": null, + "default_payment_method": null, + "footer": null, + "rendering_options": null + }, + "livemode": false, + "metadata": {}, + "name": "Jenny Rosen", + "next_invoice_sequence": 1, + "phone": null, + "preferred_locales": [], + "shipping": null, + "sources": { + "object": "list", + "data": [], + "has_more": false, + "total_count": 0, + "url": "/v1/customers/cus_NffrFeUfNV2Hib/sources" + }, + "tax_exempt": "none", + "test_clock": null + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/events.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/events.json new file mode 100644 index 000000000000..7f62598ea161 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/events.json @@ -0,0 +1,58 @@ +{ + "object": "list", + "data": [ + { + "id": "evt_1OEiWvEcXtiJtvvhLaQOew6V", + "object": "event", + "api_version": "2020-08-27", + "created": 1700529213, + "data": { + "object": { + "object": "balance", + "available": [ + { + "amount": 518686, + "currency": "usd", + "source_types": { + "card": 518686 + } + } + ], + "connect_reserved": [ + { + "amount": 0, + "currency": "usd" + } + ], + "issuing": { + "available": [ + { + "amount": 150000, + "currency": "usd" + } + ] + }, + "livemode": false, + "pending": [ + { + "amount": 0, + "currency": "usd", + "source_types": { + "card": 0 + } + } + ] + } + }, + "livemode": false, + "pending_webhooks": 0, + "request": { + "id": null, + "idempotency_key": null + }, + "type": "balance.available" + } + ], + "has_more": false, + "url": "/v1/events" +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/external_account_cards.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/external_account_cards.json new file mode 100644 index 000000000000..c26bc36461cd --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/external_account_cards.json @@ -0,0 +1,34 @@ +{ + "object": "list", + "url": "/v1/accounts/acct_1032D82eZvKYlo2C/external_accounts", + "has_more": false, + "data": [ + { + "id": "card_1NAz2x2eZvKYlo2C75wJ1YUs", + "object": "card", + "address_city": null, + "address_country": null, + "address_line1": null, + "address_line1_check": null, + "address_line2": null, + "address_state": null, + "address_zip": null, + "address_zip_check": null, + "brand": "Visa", + "country": "US", + "cvc_check": "pass", + "dynamic_last4": null, + "exp_month": 8, + "exp_year": 2024, + "fingerprint": "Xt5EWLLDS7FJjR1c", + "funding": "credit", + "last4": "4242", + "metadata": {}, + "name": null, + "redaction": null, + "tokenization_method": null, + "wallet": null, + "account": "acct_1032D82eZvKYlo2C" + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/external_bank_accounts.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/external_bank_accounts.json new file mode 100644 index 000000000000..a8704270de2f --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/external_bank_accounts.json @@ -0,0 +1,23 @@ +{ + "object": "list", + "url": "/v1/accounts/acct_1032D82eZvKYlo2C/external_accounts", + "has_more": false, + "data": [ + { + "id": "ba_1NB1IV2eZvKYlo2CByiLrMWv", + "object": "bank_account", + "account_holder_name": "Jane Austen", + "account_holder_type": "company", + "account_type": null, + "bank_name": "STRIPE TEST BANK", + "country": "US", + "currency": "usd", + "fingerprint": "1JWtPxqbdX5Gamtc", + "last4": "6789", + "metadata": {}, + "routing_number": "110000000", + "status": "new", + "account": "acct_1032D82eZvKYlo2C" + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/issuing_authorizations.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/issuing_authorizations.json new file mode 100644 index 000000000000..cdf281ee2cd7 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/issuing_authorizations.json @@ -0,0 +1,142 @@ +{ + "object": "list", + "url": "/v1/issuing/authorizations", + "has_more": false, + "data": [ + { + "id": "iauth_1JVXl82eZvKYlo2CPIiWlzrn", + "object": "issuing.authorization", + "amount": 382, + "amount_details": { + "atm_fee": null + }, + "approved": false, + "authorization_method": "online", + "balance_transactions": [], + "card": { + "id": "ic_1JDmgz2eZvKYlo2CRXlTsXj6", + "object": "issuing.card", + "brand": "Visa", + "cancellation_reason": null, + "cardholder": { + "id": "ich_1JDmfb2eZvKYlo2CwHUgaAxU", + "object": "issuing.cardholder", + "billing": { + "address": { + "city": "San Francisco", + "country": "US", + "line1": "123 Main Street", + "line2": null, + "postal_code": "94111", + "state": "CA" + } + }, + "company": null, + "created": 1626425119, + "email": "jenny.rosen@example.com", + "individual": null, + "livemode": false, + "metadata": {}, + "name": "Jenny Rosen", + "phone_number": "+18008675309", + "redaction": null, + "requirements": { + "disabled_reason": null, + "past_due": [] + }, + "spending_controls": { + "allowed_categories": [], + "blocked_categories": [], + "spending_limits": [], + "spending_limits_currency": null + }, + "status": "active", + "type": "individual" + }, + "created": 1626425206, + "currency": "usd", + "exp_month": 6, + "exp_year": 2024, + "last4": "8693", + "livemode": false, + "metadata": {}, + "redaction": null, + "replaced_by": null, + "replacement_for": null, + "replacement_reason": null, + "shipping": null, + "spending_controls": { + "allowed_categories": null, + "blocked_categories": null, + "spending_limits": [ + { + "amount": 50000, + "categories": [], + "interval": "daily" + } + ], + "spending_limits_currency": "usd" + }, + "status": "active", + "type": "virtual", + "wallets": { + "apple_pay": { + "eligible": true, + "ineligible_reason": null + }, + "google_pay": { + "eligible": true, + "ineligible_reason": null + }, + "primary_account_identifier": null + } + }, + "cardholder": "ich_1JDmfb2eZvKYlo2CwHUgaAxU", + "created": 1630657706, + "currency": "usd", + "livemode": false, + "merchant_amount": 382, + "merchant_currency": "usd", + "merchant_data": { + "category": "computer_software_stores", + "category_code": "5734", + "city": "SAN FRANCISCO", + "country": "US", + "name": "STRIPE", + "network_id": "1234567890", + "postal_code": "94103", + "state": "CA" + }, + "metadata": { + "order_id": "6735" + }, + "network_data": null, + "pending_request": null, + "redaction": null, + "request_history": [ + { + "amount": 382, + "amount_details": { + "atm_fee": null + }, + "approved": false, + "created": 1630657706, + "currency": "usd", + "merchant_amount": 382, + "merchant_currency": "usd", + "reason": "verification_failed", + "reason_message": null + } + ], + "status": "closed", + "transactions": [], + "verification_data": { + "address_line1_check": "not_provided", + "address_postal_code_check": "not_provided", + "cvc_check": "mismatch", + "expiry_check": "match" + }, + "wallet": null + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/issuing_cards.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/issuing_cards.json new file mode 100644 index 000000000000..1d5027df5457 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/issuing_cards.json @@ -0,0 +1,82 @@ +{ + "object": "list", + "url": "/v1/issuing/cards", + "has_more": false, + "data": [ + { + "id": "ic_1MvSieLkdIwHu7ixn6uuO0Xu", + "object": "issuing.card", + "brand": "Visa", + "cancellation_reason": null, + "cardholder": { + "id": "ich_1MsKAB2eZvKYlo2C3eZ2BdvK", + "object": "issuing.cardholder", + "billing": { + "address": { + "city": "Anytown", + "country": "US", + "line1": "123 Main Street", + "line2": null, + "postal_code": "12345", + "state": "CA" + } + }, + "company": null, + "created": 1680415995, + "email": null, + "individual": null, + "livemode": false, + "metadata": {}, + "name": "John Doe", + "phone_number": null, + "requirements": { + "disabled_reason": "requirements.past_due", + "past_due": [ + "individual.card_issuing.user_terms_acceptance.ip", + "individual.card_issuing.user_terms_acceptance.date", + "individual.first_name", + "individual.last_name" + ] + }, + "spending_controls": { + "allowed_categories": [], + "blocked_categories": [], + "spending_limits": [], + "spending_limits_currency": null + }, + "status": "active", + "type": "individual" + }, + "created": 1681163868, + "currency": "usd", + "exp_month": 8, + "exp_year": 2024, + "last4": "4242", + "livemode": false, + "metadata": {}, + "replaced_by": null, + "replacement_for": null, + "replacement_reason": null, + "shipping": null, + "spending_controls": { + "allowed_categories": null, + "blocked_categories": null, + "spending_limits": [], + "spending_limits_currency": null + }, + "status": "active", + "type": "virtual", + "wallets": { + "apple_pay": { + "eligible": false, + "ineligible_reason": "missing_cardholder_contact" + }, + "google_pay": { + "eligible": false, + "ineligible_reason": "missing_cardholder_contact" + }, + "primary_account_identifier": null + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/issuing_transactions.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/issuing_transactions.json new file mode 100644 index 000000000000..bbd790f318eb --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/issuing_transactions.json @@ -0,0 +1,38 @@ +{ + "object": "list", + "url": "/v1/issuing/transactions", + "has_more": false, + "data": [ + { + "id": "ipi_1MzFN1K8F4fqH0lBmFq8CjbU", + "object": "issuing.transaction", + "amount": -100, + "amount_details": { + "atm_fee": null + }, + "authorization": "iauth_1MzFMzK8F4fqH0lBc9VdaZUp", + "balance_transaction": "txn_1MzFN1K8F4fqH0lBQPtqUmJN", + "card": "ic_1MzFMxK8F4fqH0lBjIUITRYi", + "cardholder": "ich_1MzFMxK8F4fqH0lBXnFW0ROG", + "created": 1682065867, + "currency": "usd", + "dispute": null, + "livemode": false, + "merchant_amount": -100, + "merchant_currency": "usd", + "merchant_data": { + "category": "computer_software_stores", + "category_code": "5734", + "city": "SAN FRANCISCO", + "country": "US", + "name": "WWWW.BROWSEBUG.BIZ", + "network_id": "1234567890", + "postal_code": "94103", + "state": "CA" + }, + "metadata": {}, + "type": "capture", + "wallet": null + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/payment_methods.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/payment_methods.json new file mode 100644 index 000000000000..59ced3939e62 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/payment_methods.json @@ -0,0 +1,52 @@ +{ + "object": "list", + "url": "/v1/payment_methods", + "has_more": false, + "data": [ + { + "id": "pm_1NO6mA2eZvKYlo2CEydeHsKT", + "object": "payment_method", + "billing_details": { + "address": { + "city": null, + "country": null, + "line1": null, + "line2": null, + "postal_code": null, + "state": null + }, + "email": null, + "name": null, + "phone": null + }, + "card": { + "brand": "visa", + "checks": { + "address_line1_check": null, + "address_postal_code_check": null, + "cvc_check": "unchecked" + }, + "country": "US", + "exp_month": 8, + "exp_year": 2024, + "fingerprint": "Xt5EWLLDS7FJjR1c", + "funding": "credit", + "generated_from": null, + "last4": "4242", + "networks": { + "available": ["visa"], + "preferred": null + }, + "three_d_secure_usage": { + "supported": true + }, + "wallet": null + }, + "created": 1687991030, + "customer": "cus_9s6XKzkNRiz8i3", + "livemode": false, + "metadata": {}, + "type": "card" + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/persons.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/persons.json new file mode 100644 index 000000000000..f7ede1e42817 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/persons.json @@ -0,0 +1,65 @@ +{ + "object": "list", + "url": "/v1/accounts/acct_1G9HZLIEn49ers/persons", + "has_more": false, + "data": [ + { + "id": "person_1MqjB62eZvKYlo2CaeEJzK13", + "person": "person_1MqjB62eZvKYlo2CaeEJzK13", + "object": "person", + "account": "acct_1G9HZLIEn49ers", + "created": 1680035496, + "dob": { + "day": null, + "month": null, + "year": null + }, + "first_name": "Brock", + "future_requirements": { + "alternatives": [], + "currently_due": [], + "errors": [], + "eventually_due": [], + "past_due": [], + "pending_verification": [] + }, + "id_number_provided": false, + "last_name": "Purdy", + "metadata": {}, + "relationship": { + "director": false, + "executive": true, + "owner": false, + "percent_ownership": null, + "representative": false, + "title": null + }, + "requirements": { + "alternatives": [], + "currently_due": [], + "errors": [], + "eventually_due": [], + "past_due": [], + "pending_verification": [] + }, + "ssn_last_4_provided": false, + "verification": { + "additional_document": { + "back": null, + "details": null, + "details_code": null, + "front": null + }, + "details": null, + "details_code": null, + "document": { + "back": null, + "details": null, + "details_code": null, + "front": null + }, + "status": "verified" + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/radar_early_fraud_warnings.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/radar_early_fraud_warnings.json new file mode 100644 index 000000000000..8da264f5b475 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/radar_early_fraud_warnings.json @@ -0,0 +1,16 @@ +{ + "object": "list", + "url": "/v1/radar/early_fraud_warnings", + "has_more": false, + "data": [ + { + "id": "issfr_1NnrwHBw2dPENLoi9lnhV3RQ", + "object": "radar.early_fraud_warning", + "actionable": true, + "charge": "ch_1234", + "created": 123456789, + "fraud_type": "misc", + "livemode": false + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/refunds.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/refunds.json new file mode 100644 index 000000000000..af20ee7480d3 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/refunds.json @@ -0,0 +1,32 @@ +{ + "object": "list", + "url": "/v1/refunds", + "has_more": false, + "data": [ + { + "id": "re_1Nispe2eZvKYlo2Cd31jOCgZ", + "object": "refund", + "amount": 1000, + "balance_transaction": "txn_1Nispe2eZvKYlo2CYezqFhEx", + "charge": "ch_1NirD82eZvKYlo2CIvbtLWuY", + "created": 1692942318, + "currency": "usd", + "destination_details": { + "card": { + "reference": "123456789012", + "reference_status": "available", + "reference_type": "acquirer_reference_number", + "type": "refund" + }, + "type": "card" + }, + "metadata": {}, + "payment_intent": "pi_1GszsK2eZvKYlo2CfhZyoZLp", + "reason": null, + "receipt_number": null, + "source_transfer_reversal": null, + "status": "succeeded", + "transfer_reversal": null + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/reviews.json b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/reviews.json new file mode 100644 index 000000000000..0e41d57d3bb1 --- /dev/null +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/resource/http/response/reviews.json @@ -0,0 +1,23 @@ +{ + "object": "list", + "url": "/v1/reviews", + "has_more": false, + "data": [ + { + "id": "prv_1NVyFt2eZvKYlo2CjubqF1xm", + "object": "review", + "billing_zip": null, + "charge": null, + "closed_reason": null, + "created": 1689864901, + "ip_address": null, + "ip_address_location": null, + "livemode": false, + "open": true, + "opened_reason": "rule", + "payment_intent": "pi_3NVy8c2eZvKYlo2C055h7pkd", + "reason": "rule", + "session": null + } + ] +} diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/test_availability_strategy.py b/airbyte-integrations/connectors/source-stripe/unit_tests/test_availability_strategy.py index 76b6d68c556d..ee41b71dd049 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/test_availability_strategy.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/test_availability_strategy.py @@ -2,78 +2,87 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import pendulum +import logging +import urllib.parse + +import pytest from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy -from source_stripe.availability_strategy import StripeSubStreamAvailabilityStrategy -from source_stripe.streams import InvoiceLineItems, Invoices +from source_stripe.availability_strategy import STRIPE_ERROR_CODES, StripeSubStreamAvailabilityStrategy +from source_stripe.streams import IncrementalStripeStream, StripeLazySubStream + +@pytest.fixture() +def stream_mock(mocker): + def _mocker(): + return mocker.Mock(stream_slices=mocker.Mock(return_value=[{}]), read_records=mocker.Mock(return_value=[{}])) + return _mocker -def test_traverse_over_substreams(mocker): + +def test_traverse_over_substreams(stream_mock, mocker): # Mock base HttpAvailabilityStrategy to capture all the check_availability method calls - check_availability_mock = mocker.MagicMock() - check_availability_mock.return_value = (True, None) + check_availability_mock = mocker.MagicMock(return_value=(True, None)) + cdk_check_availability_mock = mocker.MagicMock(return_value=(True, None)) mocker.patch( - "airbyte_cdk.sources.streams.http.availability_strategy.HttpAvailabilityStrategy.check_availability", - check_availability_mock + "source_stripe.availability_strategy.StripeAvailabilityStrategy.check_availability", check_availability_mock + ) + mocker.patch( + "airbyte_cdk.sources.streams.http.availability_strategy.HttpAvailabilityStrategy.check_availability", cdk_check_availability_mock ) - # Prepare tree of nested objects - root = mocker.Mock() + root = stream_mock() root.availability_strategy = HttpAvailabilityStrategy() root.parent = None - child_1 = mocker.Mock() + child_1 = stream_mock() child_1.availability_strategy = StripeSubStreamAvailabilityStrategy() - child_1.get_parent_stream_instance.return_value = root + child_1.parent = root - child_1_1 = mocker.Mock() + child_1_1 = stream_mock() child_1_1.availability_strategy = StripeSubStreamAvailabilityStrategy() - child_1_1.get_parent_stream_instance.return_value = child_1 + child_1_1.parent = child_1 - child_1_1_1 = mocker.Mock() + child_1_1_1 = stream_mock() child_1_1_1.availability_strategy = StripeSubStreamAvailabilityStrategy() - child_1_1_1.get_parent_stream_instance.return_value = child_1_1 + child_1_1_1.parent = child_1_1 # Start traverse is_available, reason = child_1_1_1.availability_strategy.check_availability(child_1_1_1, mocker.Mock(), mocker.Mock()) assert is_available and reason is None - # Check availability strategy was called once for every nested object - assert check_availability_mock.call_count == 4 + assert check_availability_mock.call_count == 3 + assert cdk_check_availability_mock.call_count == 1 # Check each availability strategy was called with proper instance argument - assert id(check_availability_mock.call_args_list[0].args[0]) == id(root) - assert id(check_availability_mock.call_args_list[1].args[0]) == id(child_1) - assert id(check_availability_mock.call_args_list[2].args[0]) == id(child_1_1) - assert id(check_availability_mock.call_args_list[3].args[0]) == id(child_1_1_1) + assert id(cdk_check_availability_mock.call_args_list[0].args[0]) == id(root) + assert id(check_availability_mock.call_args_list[0].args[0]) == id(child_1) + assert id(check_availability_mock.call_args_list[1].args[0]) == id(child_1_1) + assert id(check_availability_mock.call_args_list[2].args[0]) == id(child_1_1_1) -def test_traverse_over_substreams_failure(mocker): +def test_traverse_over_substreams_failure(stream_mock, mocker): # Mock base HttpAvailabilityStrategy to capture all the check_availability method calls - check_availability_mock = mocker.MagicMock() - check_availability_mock.side_effect = [(True, None), (False, "child_1")] + check_availability_mock = mocker.MagicMock(side_effect=[(True, None), (False, "child_1")]) mocker.patch( - "airbyte_cdk.sources.streams.http.availability_strategy.HttpAvailabilityStrategy.check_availability", - check_availability_mock + "source_stripe.availability_strategy.StripeAvailabilityStrategy.check_availability", check_availability_mock ) # Prepare tree of nested objects - root = mocker.Mock() + root = stream_mock() root.availability_strategy = HttpAvailabilityStrategy() root.parent = None - child_1 = mocker.Mock() + child_1 = stream_mock() child_1.availability_strategy = StripeSubStreamAvailabilityStrategy() - child_1.get_parent_stream_instance.return_value = root + child_1.parent = root - child_1_1 = mocker.Mock() + child_1_1 = stream_mock() child_1_1.availability_strategy = StripeSubStreamAvailabilityStrategy() - child_1_1.get_parent_stream_instance.return_value = child_1 + child_1_1.parent = child_1 - child_1_1_1 = mocker.Mock() + child_1_1_1 = stream_mock() child_1_1_1.availability_strategy = StripeSubStreamAvailabilityStrategy() - child_1_1_1.get_parent_stream_instance.return_value = child_1_1 + child_1_1_1.parent = child_1_1 # Start traverse is_available, reason = child_1_1_1.availability_strategy.check_availability(child_1_1_1, mocker.Mock(), mocker.Mock()) @@ -84,39 +93,133 @@ def test_traverse_over_substreams_failure(mocker): assert check_availability_mock.call_count == 2 # Check each availability strategy was called with proper instance argument - assert id(check_availability_mock.call_args_list[0].args[0]) == id(root) - assert id(check_availability_mock.call_args_list[1].args[0]) == id(child_1) + assert id(check_availability_mock.call_args_list[0].args[0]) == id(child_1) + assert id(check_availability_mock.call_args_list[1].args[0]) == id(child_1_1) -def test_substream_availability(mocker): +def test_substream_availability(mocker, stream_by_name): check_availability_mock = mocker.MagicMock() check_availability_mock.return_value = (True, None) mocker.patch( - "airbyte_cdk.sources.streams.http.availability_strategy.HttpAvailabilityStrategy.check_availability", - check_availability_mock + "source_stripe.availability_strategy.StripeAvailabilityStrategy.check_availability", check_availability_mock ) - - stream = InvoiceLineItems(start_date=pendulum.today().subtract(days=3).int_timestamp, account_id="None") + stream = stream_by_name("invoice_line_items") is_available, reason = stream.availability_strategy.check_availability(stream, mocker.Mock(), mocker.Mock()) assert is_available and reason is None assert check_availability_mock.call_count == 2 - assert isinstance(check_availability_mock.call_args_list[0].args[0], Invoices) - assert isinstance(check_availability_mock.call_args_list[1].args[0], InvoiceLineItems) + assert isinstance(check_availability_mock.call_args_list[0].args[0], IncrementalStripeStream) + assert isinstance(check_availability_mock.call_args_list[1].args[0], StripeLazySubStream) -def test_substream_availability_no_parent(mocker): +def test_substream_availability_no_parent(mocker, stream_by_name): check_availability_mock = mocker.MagicMock() check_availability_mock.return_value = (True, None) mocker.patch( - "airbyte_cdk.sources.streams.http.availability_strategy.HttpAvailabilityStrategy.check_availability", - check_availability_mock + "source_stripe.availability_strategy.StripeAvailabilityStrategy.check_availability", check_availability_mock ) - - stream = InvoiceLineItems(start_date=pendulum.today().subtract(days=3).int_timestamp, account_id="None") + stream = stream_by_name("invoice_line_items") stream.parent = None stream.availability_strategy.check_availability(stream, mocker.Mock(), mocker.Mock()) assert check_availability_mock.call_count == 1 - assert isinstance(check_availability_mock.call_args_list[0].args[0], InvoiceLineItems) + assert isinstance(check_availability_mock.call_args_list[0].args[0], StripeLazySubStream) + + +def test_403_error_handling(stream_by_name, requests_mock): + stream = stream_by_name("invoices") + logger = logging.getLogger("airbyte") + for error_code in STRIPE_ERROR_CODES: + requests_mock.get(f"{stream.url_base}{stream.path()}", status_code=403, json={"error": {"code": f"{error_code}"}}) + available, message = stream.check_availability(logger) + assert not available + assert STRIPE_ERROR_CODES[error_code] in message + + +@pytest.mark.parametrize( + "stream_name, endpoints, expected_calls", + ( + ( + "accounts", + { + "/v1/accounts": {"data": []} + }, + 1 + ), + ( + "refunds", + { + "/v1/refunds": {"data": []} + }, + 2 + ), + ( + "credit_notes", + { + "/v1/credit_notes": {"data": []}, "/v1/events": {"data": []} + }, + 2 + ), + ( + "charges", + { + "/v1/charges": {"data": []}, "/v1/events": {"data": []} + }, + 2 + ), + ( + "subscription_items", + { + "/v1/subscriptions": {"data": [{"id": 1}]}, + "/v1/events": {"data": []} + }, + 3 + ), + ( + "bank_accounts", + { + "/v1/customers": {"data": [{"id": 1}]}, + "/v1/events": {"data": []} + }, + 2 + ), + ( + "customer_balance_transactions", + { + "/v1/events": {"data": [{"data":{"object": {"id": 1}}, "created": 1, "type": "customer.updated"}]}, + "/v1/customers": {"data": [{"id": 1}]}, + "/v1/customers/1/balance_transactions": {"data": []} + }, + 4 + ), + ( + "transfer_reversals", + { + "/v1/transfers": {"data": [{"id": 1}]}, + "/v1/events": {"data": [{"data":{"object": {"id": 1}}, "created": 1, "type": "transfer.updated"}]}, + "/v1/transfers/1/reversals": {"data": []} + }, + 4 + ), + ( + "persons", + { + "/v1/accounts": {"data": [{"id": 1}]}, + "/v1/events": {"data": []}, + "/v1/accounts/1/persons": {"data": []} + }, + 4 + ) + ) +) +def test_availability_strategy_visits_endpoints(stream_by_name, stream_name, endpoints, expected_calls, requests_mock, mocker, config): + for endpoint, data in endpoints.items(): + requests_mock.get(endpoint, json=data) + stream = stream_by_name(stream_name, config) + is_available, reason = stream.check_availability(mocker.Mock(), mocker.Mock()) + assert (is_available, reason) == (True, None) + assert len(requests_mock.request_history) == expected_calls + + for call in requests_mock.request_history: + assert urllib.parse.urlparse(call.url).path in endpoints.keys() diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py b/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py index f5ea6c77b31c..2f2a1c0acd1e 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/test_source.py @@ -1,63 +1,152 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +import datetime +import logging +from contextlib import nullcontext as does_not_raise +from unittest.mock import patch -import json -from unittest.mock import Mock, patch - -import pendulum import pytest import source_stripe +import stripe +from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode +from airbyte_cdk.sources.streams.call_rate import CachedLimiterSession, LimiterSession, Rate +from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.utils import AirbyteTracedException from source_stripe import SourceStripe -from source_stripe.source import Invoices -now_dt = pendulum.now() +logger = logging.getLogger("airbyte") +_ANY_CATALOG = ConfiguredAirbyteCatalog.parse_obj({"streams": []}) +_ANY_CONFIG = {} -SECONDS_IN_DAY = 24 * 60 * 60 +class CatalogBuilder: + def __init__(self) -> None: + self._streams = [] -@pytest.mark.parametrize( - "lookback_window_days, current_state, expected, message", - [ - (None, now_dt.timestamp(), now_dt.timestamp(), - "if lookback_window_days is not set should not affect cursor value"), - (0, now_dt.timestamp(), now_dt.timestamp(), - "if lookback_window_days is not set should not affect cursor value"), - (10, now_dt.timestamp(), int(now_dt.timestamp() - SECONDS_IN_DAY * 10), - "Should calculate cursor value as expected"), - # ignore sign - (-10, now_dt.timestamp(), int(now_dt.timestamp() - SECONDS_IN_DAY * 10), - "Should not care for the sign, use the module"), - ], -) -def test_lookback_window(lookback_window_days, current_state, expected, message): - inv_stream = Invoices(account_id=213, start_date=1577836800, - lookback_window_days=lookback_window_days) - inv_stream.cursor_field = "created" - assert inv_stream.get_start_timestamp( - {"created": current_state}) == expected, message + def with_stream(self, name: str, sync_mode: SyncMode) -> "CatalogBuilder": + self._streams.append( + { + "stream": { + "name": name, + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]], + }, + "primary_key": [["id"]], + "sync_mode": sync_mode.name, + "destination_sync_mode": "overwrite", + } + ) + return self + + def build(self) -> ConfiguredAirbyteCatalog: + return ConfiguredAirbyteCatalog.parse_obj({"streams": self._streams}) -def test_source_streams(): - with open("sample_files/config.json") as f: - config = json.load(f) - streams = SourceStripe().streams(config=config) - assert len(streams) == 46 +def _a_valid_config(): + return {"account_id": 1, "client_secret": "secret"} @patch.object(source_stripe.source, "stripe") def test_source_check_connection_ok(mocked_client, config): - assert SourceStripe().check_connection(None, config=config) == (True, None) + assert SourceStripe(_ANY_CATALOG, _ANY_CONFIG).check_connection(logger, config=config) == (True, None) -@patch.object(source_stripe.source, "stripe") -def test_source_check_connection_failure(mocked_client, config): - exception = Exception("Test") - mocked_client.Account.retrieve = Mock(side_effect=exception) - assert SourceStripe().check_connection(None, config=config) == (False, exception) +def test_streams_are_unique(config): + stream_names = [s.name for s in SourceStripe(_ANY_CATALOG, _ANY_CONFIG).streams(config=config)] + assert len(stream_names) == len(set(stream_names)) == 46 -@patch.object(source_stripe.source, "stripe") -def test_streams_are_unique(mocked_client, config): - streams = [s.name for s in SourceStripe().streams(config)] - assert sorted(streams) == sorted(set(streams)) +@pytest.mark.parametrize( + "input_config, expected_error_msg", + ( + ({"lookback_window_days": "month"}, "Invalid lookback window month. Please use only positive integer values or 0."), + ({"start_date": "January First, 2022"}, "Invalid start date January First, 2022. Please use YYYY-MM-DDTHH:MM:SSZ format."), + ({"slice_range": -10}, "Invalid slice range value -10. Please use positive integer values only."), + (_a_valid_config(), None), + ), +) +@patch.object(source_stripe.source.stripe, "Account") +def test_config_validation(mocked_client, input_config, expected_error_msg): + context = pytest.raises(AirbyteTracedException, match=expected_error_msg) if expected_error_msg else does_not_raise() + with context: + SourceStripe(_ANY_CATALOG, _ANY_CONFIG).check_connection(logger, config=input_config) + + +@pytest.mark.parametrize( + "exception", + ( + stripe.error.AuthenticationError, + stripe.error.PermissionError, + ), +) +@patch.object(source_stripe.source.stripe, "Account") +def test_given_stripe_error_when_check_connection_then_connection_not_available(mocked_client, exception): + mocked_client.retrieve.side_effect = exception + is_available, _ = SourceStripe(_ANY_CATALOG, _ANY_CONFIG).check_connection(logger, config=_a_valid_config()) + assert not is_available + + +def test_when_streams_return_full_refresh_as_concurrent(): + streams = SourceStripe( + CatalogBuilder().with_stream("bank_accounts", SyncMode.full_refresh).with_stream("customers", SyncMode.incremental).build(), + _a_valid_config(), + ).streams(_a_valid_config()) + + assert len(list(filter(lambda stream: isinstance(stream, StreamFacade), streams))) == 1 + + +@pytest.mark.parametrize( + "input_config, default_call_limit", + ( + ({"account_id": 1, "client_secret": "secret"}, 100), + ({"account_id": 1, "client_secret": "secret", "call_rate_limit": 10}, 10), + ({"account_id": 1, "client_secret": "secret", "call_rate_limit": 110}, 100), + ({"account_id": 1, "client_secret": "sk_test_some_secret"}, 25), + ({"account_id": 1, "client_secret": "sk_test_some_secret", "call_rate_limit": 10}, 10), + ({"account_id": 1, "client_secret": "sk_test_some_secret", "call_rate_limit": 30}, 25), + ), +) +def test_call_budget_creation(mocker, input_config, default_call_limit): + """Test that call_budget was created with specific config i.e., that first policy has specific matchers.""" + + policy_mock = mocker.patch("source_stripe.source.MovingWindowCallRatePolicy") + matcher_mock = mocker.patch("source_stripe.source.HttpRequestMatcher") + source = SourceStripe(catalog=None, config=input_config) + + source.get_api_call_budget(input_config) + + policy_mock.assert_has_calls( + calls=[ + mocker.call(matchers=[mocker.ANY, mocker.ANY], rates=[Rate(limit=20, interval=datetime.timedelta(seconds=1))]), + mocker.call(matchers=[], rates=[Rate(limit=default_call_limit, interval=datetime.timedelta(seconds=1))]), + ], + ) + + matcher_mock.assert_has_calls( + calls=[ + mocker.call(url="https://api.stripe.com/v1/files"), + mocker.call(url="https://api.stripe.com/v1/file_links"), + ] + ) + + +def test_call_budget_passed_to_every_stream(mocker): + """Test that each stream has call_budget passed and creates a proper session""" + + prod_config = {"account_id": 1, "client_secret": "secret"} + source = SourceStripe(catalog=None, config=prod_config) + get_api_call_budget_mock = mocker.patch.object(source, "get_api_call_budget") + + streams = source.streams(prod_config) + + assert streams + get_api_call_budget_mock.assert_called_once() + + for stream in streams: + assert isinstance(stream, HttpStream) + session = stream.request_session() + assert isinstance(session, (CachedLimiterSession, LimiterSession)) + assert session._api_budget == get_api_call_budget_mock.return_value diff --git a/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py index 1fd42de179e9..55da589609e7 100644 --- a/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-stripe/unit_tests/test_streams.py @@ -2,280 +2,1094 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import logging +from urllib.parse import urlencode +import freezegun import pendulum import pytest -from airbyte_cdk.models import SyncMode -from source_stripe.availability_strategy import STRIPE_ERROR_CODES -from source_stripe.streams import ( - ApplicationFees, - ApplicationFeesRefunds, - BalanceTransactions, - BankAccounts, - Charges, - CheckoutSessions, - CheckoutSessionsLineItems, - Coupons, - CustomerBalanceTransactions, - Customers, - Disputes, - EarlyFraudWarnings, - Events, - ExternalAccount, - ExternalAccountBankAccounts, - ExternalAccountCards, - InvoiceItems, - InvoiceLineItems, - Invoices, - PaymentIntents, - Payouts, - Persons, - Plans, - Prices, - Products, - PromotionCodes, - Refunds, - SetupIntents, - ShippingRates, - SubscriptionItems, - Subscriptions, - SubscriptionSchedule, - Transfers, -) +from source_stripe.streams import CustomerBalanceTransactions, Persons, SetupAttempts -def test_missed_id_child_stream(requests_mock): +def read_from_stream(stream, sync_mode, state): + records = [] + for slice_ in stream.stream_slices(sync_mode=sync_mode, stream_state=state): + for record in stream.read_records(sync_mode=sync_mode, stream_slice=slice_, stream_state=state): + records.append(record) + return records - session_id_missed = "cs_test_a165K4wNihuJlp2u3tknuohrvjAxyXFUB7nxZH3lwXRKJsadNEvIEWMUJ9" - session_id_exists = "cs_test_a1RjRHNyGUQOFVF3OkL8V8J0lZUASyVoCtsnZYG74VrBv3qz4245BLA1BP" - response_sessions = { - "data": [{"id": session_id_missed, "expires_at": 100_000}, {"id": session_id_exists, "expires_at": 100_000}], - "has_more": False, - "object": "list", - "url": "/v1/checkout/sessions", - } +def test_request_headers(stream_by_name): + stream = stream_by_name("accounts") + headers = stream.request_headers() + assert headers["Stripe-Version"] == "2022-11-15" - response_sessions_line_items = { - "data": [{"id": "li_1JpAUUIEn5WyEQxnfGJT5MbL"}], - "has_more": False, - "object": "list", - "url": "/v1/checkout/sessions/{}/line_items".format(session_id_exists), - } - response_error = { - "error": { - "code": "resource_missing", - "doc_url": "https://stripe.com/docs/error-codes/resource-missing", - "message": "No such checkout session: '{}'".format(session_id_missed), - "param": "session", - "type": "invalid_request_error", +bank_accounts_full_refresh_test_case = ( + { + "https://api.stripe.com/v1/customers?expand%5B%5D=data.sources": { + "has_more": False, + "object": "list", + "url": "/v1/customers", + "data": [ + { + "created": 1641038947, + "id": "cus_HezytZRkaQJC8W", + "object": "customer", + "total": 1, + "sources": { + "data": [ + { + "id": "cs_1", + "object": "card", + }, + { + "id": "cs_2", + "object": "bank_account", + }, + ], + "has_more": True, + "object": "list", + "total_count": 4, + "url": "/v1/customers/cus_HezytZRkaQJC8W/sources", + }, + } + ], + }, + "https://api.stripe.com/v1/customers/cus_HezytZRkaQJC8W/bank_accounts?starting_after=cs_2": { + "data": [ + { + "id": "cs_4", + "object": "bank_account", + }, + ], + "has_more": False, + "object": "list", + "url": "/v1/customers/cus_HezytZRkaQJC8W/bank_accounts", + }, + }, + "bank_accounts", + [ + {"id": "cs_2", "object": "bank_account", "updated": 1692802815}, + {"id": "cs_4", "object": "bank_account", "updated": 1692802815}, + ], + "full_refresh", + {}, +) + + +bank_accounts_incremental_test_case = ( + { + "https://api.stripe.com/v1/events?types%5B%5D=customer.source.created&types%5B%5D=customer.source.expiring&types" + "%5B%5D=customer.source.updated&types%5B%5D=customer.source.deleted": { + "data": [ + { + "id": "evt_1NdNFoEcXtiJtvvhBP5mxQmL", + "object": "event", + "api_version": "2020-08-27", + "created": 1692802016, + "data": {"object": {"object": "bank_account", "bank_account": "cs_1K9GK0EcXtiJtvvhSo2LvGqT", "created": 1653341716}}, + "type": "customer.source.created", + }, + { + "id": "evt_1NdNFoEcXtiJtvvhBP5mxQmL", + "object": "event", + "api_version": "2020-08-27", + "created": 1692802017, + "data": {"object": {"object": "card", "card": "cs_1K9GK0EcXtiJtvvhSo2LvGqT", "created": 1653341716}}, + "type": "customer.source.updated", + }, + ], + "has_more": False, } - } + }, + "bank_accounts", + [{"object": "bank_account", "bank_account": "cs_1K9GK0EcXtiJtvvhSo2LvGqT", "created": 1653341716, "updated": 1692802016}], + "incremental", + {"updated": 1692802015}, +) + + +@pytest.mark.parametrize( + "requests_mock_map, stream_cls, expected_records, sync_mode, state", + (bank_accounts_incremental_test_case, bank_accounts_full_refresh_test_case), +) +@freezegun.freeze_time("2023-08-23T15:00:15Z") +def test_lazy_substream_data_cursor_value_is_populated( + requests_mock, stream_by_name, config, requests_mock_map, stream_cls, expected_records, sync_mode, state +): + config["start_date"] = str(pendulum.today().subtract(days=3)) + stream = stream_by_name(stream_cls, config) + for url, body in requests_mock_map.items(): + requests_mock.get(url, json=body) + + records = read_from_stream(stream, sync_mode, state) + assert records == expected_records + for record in records: + assert bool(record[stream.cursor_field]) + + +@pytest.mark.parametrize("requests_mock_map, stream_cls, expected_records, sync_mode, state", (bank_accounts_full_refresh_test_case,)) +@freezegun.freeze_time("2023-08-23T15:00:15Z") +def test_lazy_substream_data_is_expanded( + requests_mock, stream_by_name, config, requests_mock_map, stream_cls, expected_records, sync_mode, state +): + + config["start_date"] = str(pendulum.today().subtract(days=3)) + stream = stream_by_name("bank_accounts", config) + for url, body in requests_mock_map.items(): + requests_mock.get(url, json=body) + + records = read_from_stream(stream, sync_mode, state) + + assert list(records) == expected_records + assert len(requests_mock.request_history) == 2 + assert urlencode({"expand[]": "data.sources"}) in requests_mock.request_history[0].url + + +@pytest.mark.parametrize( + "requests_mock_map, stream_cls, expected_records, sync_mode, state, expected_object", + ((*bank_accounts_full_refresh_test_case, "bank_account"), (*bank_accounts_incremental_test_case, "bank_account")), +) +@freezegun.freeze_time("2023-08-23T15:00:15Z") +def test_lazy_substream_data_is_filtered( + requests_mock, stream_by_name, config, requests_mock_map, stream_cls, expected_records, sync_mode, state, expected_object +): + config["start_date"] = str(pendulum.today().subtract(days=3)) + stream = stream_by_name(stream_cls, config) + for url, body in requests_mock_map.items(): + requests_mock.get(url, json=body) + + records = read_from_stream(stream, sync_mode, state) + assert records == expected_records + for record in records: + assert record["object"] == expected_object + + +balance_transactions_api_objects = [ + {"id": "txn_1KVQhfEcXtiJtvvhF7ox3YEm", "object": "balance_transaction", "amount": 435, "created": 1653299388, "status": "available"}, + {"id": "txn_tiJtvvhF7ox3YEmKvVQhfEcX", "object": "balance_transaction", "amount": -9164, "created": 1679568588, "status": "available"}, +] + + +refunds_api_objects = [ + { + "id": "re_3NYB8LAHLf1oYfwN3EZRDIfF", + "object": "refund", + "amount": 100, + "charge": "ch_3NYB8LAHLf1oYfwN3P6BxdKj", + "created": 1653299388, + "currency": "usd", + }, + { + "id": "re_Lf1oYfwN3EZRDIfF3NYB8LAH", + "object": "refund", + "amount": 15, + "charge": "ch_YfwN3P6BxdKj3NYB8LAHLf1o", + "created": 1679568588, + "currency": "eur", + }, +] + + +@pytest.mark.parametrize( + "requests_mock_map, expected_records, expected_slices, stream_name, sync_mode, state", + ( + ( + { + "/v1/balance_transactions": [ + { + "json": { + "data": [balance_transactions_api_objects[0]], + "has_more": False, + } + }, + { + "json": { + "data": [balance_transactions_api_objects[-1]], + "has_more": False, + } + }, + ], + }, + [ + { + "id": "txn_1KVQhfEcXtiJtvvhF7ox3YEm", + "object": "balance_transaction", + "amount": 435, + "created": 1653299388, + "status": "available", + }, + { + "id": "txn_tiJtvvhF7ox3YEmKvVQhfEcX", + "object": "balance_transaction", + "amount": -9164, + "created": 1679568588, + "status": "available", + }, + ], + [{"created[gte]": 1631199615, "created[lte]": 1662735615}, {"created[gte]": 1662735616, "created[lte]": 1692802815}], + "balance_transactions", + "full_refresh", + {}, + ), + ( + { + "/v1/balance_transactions": [ + { + "json": { + "data": [balance_transactions_api_objects[-1]], + "has_more": False, + } + }, + ], + }, + [ + { + "id": "txn_tiJtvvhF7ox3YEmKvVQhfEcX", + "object": "balance_transaction", + "amount": -9164, + "created": 1679568588, + "status": "available", + }, + ], + [{"created[gte]": 1665308989, "created[lte]": 1692802815}], + "balance_transactions", + "incremental", + {"created": 1666518588}, + ), + ( + { + "/v1/refunds": [ + { + "json": { + "data": [refunds_api_objects[0]], + "has_more": False, + } + }, + { + "json": { + "data": [refunds_api_objects[-1]], + "has_more": False, + } + }, + ], + }, + [ + { + "id": "re_3NYB8LAHLf1oYfwN3EZRDIfF", + "object": "refund", + "amount": 100, + "charge": "ch_3NYB8LAHLf1oYfwN3P6BxdKj", + "created": 1653299388, + "currency": "usd", + }, + { + "id": "re_Lf1oYfwN3EZRDIfF3NYB8LAH", + "object": "refund", + "amount": 15, + "charge": "ch_YfwN3P6BxdKj3NYB8LAHLf1o", + "created": 1679568588, + "currency": "eur", + }, + ], + [{"created[gte]": 1631199615, "created[lte]": 1662735615}, {"created[gte]": 1662735616, "created[lte]": 1692802815}], + "refunds", + "full_refresh", + {}, + ), + ( + { + "/v1/refunds": [ + { + "json": { + "data": [refunds_api_objects[-1]], + "has_more": False, + } + }, + ], + }, + [ + { + "id": "re_Lf1oYfwN3EZRDIfF3NYB8LAH", + "object": "refund", + "amount": 15, + "charge": "ch_YfwN3P6BxdKj3NYB8LAHLf1o", + "created": 1679568588, + "currency": "eur", + } + ], + [{"created[gte]": 1665308989, "created[lte]": 1692802815}], + "refunds", + "incremental", + {"created": 1666518588}, + ), + ), +) +@freezegun.freeze_time("2023-08-23T15:00:15Z") +def test_created_cursor_incremental_stream( + requests_mock, requests_mock_map, stream_by_name, expected_records, expected_slices, stream_name, sync_mode, state, config +): + config["start_date"] = str(pendulum.now().subtract(months=23)) + stream = stream_by_name(stream_name, {"lookback_window_days": 14, **config}) + for url, response in requests_mock_map.items(): + requests_mock.get(url, response) + + slices = list(stream.stream_slices(sync_mode, stream_state=state)) + assert slices == expected_slices + records = read_from_stream(stream, sync_mode, state) + assert records == expected_records + for record in records: + assert bool(record[stream.cursor_field]) + call_history = iter(requests_mock.request_history) + for slice_ in slices: + call = next(call_history) + assert urlencode(slice_) in call.url + + +@pytest.mark.parametrize( + "start_date, lookback_window, max_days_from_now, stream_state, expected_start_timestamp", + ( + ("2020-01-01T00:00:00Z", 0, 0, {}, "2020-01-01T00:00:00Z"), + ("2020-01-01T00:00:00Z", 14, 0, {}, "2019-12-18T00:00:00Z"), + ("2020-01-01T00:00:00Z", 0, 30, {}, "2023-07-24T15:00:15Z"), + ("2020-01-01T00:00:00Z", 14, 30, {}, "2023-07-24T15:00:15Z"), + ("2020-01-01T00:00:00Z", 0, 0, {"created": pendulum.parse("2022-07-17T00:00:00Z").int_timestamp}, "2022-07-17T00:00:01Z"), + ("2020-01-01T00:00:00Z", 14, 0, {"created": pendulum.parse("2022-07-17T00:00:00Z").int_timestamp}, "2022-07-03T00:00:01Z"), + ("2020-01-01T00:00:00Z", 0, 30, {"created": pendulum.parse("2022-07-17T00:00:00Z").int_timestamp}, "2023-07-24T15:00:15Z"), + ("2020-01-01T00:00:00Z", 14, 30, {"created": pendulum.parse("2022-07-17T00:00:00Z").int_timestamp}, "2023-07-24T15:00:15Z"), + ), +) +@freezegun.freeze_time("2023-08-23T15:00:15Z") +def test_get_start_timestamp( + stream_by_name, config, start_date, lookback_window, max_days_from_now, stream_state, expected_start_timestamp +): + config["start_date"] = start_date + config["lookback_window_days"] = lookback_window + stream = stream_by_name("balance_transactions", config) + stream.start_date_max_days_from_now = max_days_from_now + assert stream.get_start_timestamp(stream_state) == pendulum.parse(expected_start_timestamp).int_timestamp + + +@pytest.mark.parametrize("sync_mode", ("full_refresh", "incremental")) +def test_updated_cursor_incremental_stream_slices(stream_by_name, sync_mode): + stream = stream_by_name("credit_notes") + assert list(stream.stream_slices(sync_mode)) == [{}] + + +@pytest.mark.parametrize( + "last_record, stream_state, expected_state", + (({"updated": 110}, {"updated": 111}, {"updated": 111}), ({"created": 110}, {"updated": 111}, {"updated": 111})), +) +def test_updated_cursor_incremental_stream_get_updated_state(stream_by_name, last_record, stream_state, expected_state): + stream = stream_by_name("credit_notes") + assert stream.get_updated_state(last_record, stream_state) == expected_state + + +@pytest.mark.parametrize("sync_mode", ("full_refresh", "incremental")) +def test_updated_cursor_incremental_stream_read_wo_state(requests_mock, sync_mode, stream_by_name): + requests_mock.get( + "/v1/credit_notes", + [ + { + "json": { + "data": [ + { + "id": "cn_1NGPwmEcXtiJtvvhNXwHpgJF", + "object": "credit_note", + "amount": 8400, + "amount_shipping": 0, + "created": 1686158100, + }, + { + "id": "cn_JtvvhNXwHpgJF1NGPwmEcXti", + "object": "credit_note", + "amount": 350, + "amount_shipping": 150, + "created": 1685861100, + }, + ], + "has_more": False, + } + } + ], + ) + stream = stream_by_name("credit_notes") + records = [record for record in stream.read_records(sync_mode)] + assert records == [ + { + "id": "cn_1NGPwmEcXtiJtvvhNXwHpgJF", + "object": "credit_note", + "amount": 8400, + "amount_shipping": 0, + "updated": 1686158100, + "created": 1686158100, + }, + { + "id": "cn_JtvvhNXwHpgJF1NGPwmEcXti", + "object": "credit_note", + "amount": 350, + "amount_shipping": 150, + "created": 1685861100, + "updated": 1685861100, + }, + ] + + +@freezegun.freeze_time("2023-08-23T00:00:00") +def test_updated_cursor_incremental_stream_read_w_state(requests_mock, stream_by_name): + requests_mock.get( + "/v1/events", + [ + { + "json": { + "data": [ + { + "id": "evt_1NdNFoEcXtiJtvvhBP5mxQmL", + "object": "event", + "api_version": "2020-08-27", + "created": 1691629292, + "data": {"object": {"object": "credit_note", "invoice": "in_1K9GK0EcXtiJtvvhSo2LvGqT", "created": 1653341716}}, + "type": "credit_note.voided", + } + ], + "has_more": False, + } + } + ], + ) + + stream = stream_by_name("credit_notes") + records = [ + record + for record in stream.read_records("incremental", stream_state={"updated": pendulum.parse("2023-01-01T15:00:15Z").int_timestamp}) + ] + assert records == [{"object": "credit_note", "invoice": "in_1K9GK0EcXtiJtvvhSo2LvGqT", "created": 1653341716, "updated": 1691629292}] + + +def test_customer_balance_transactions_stream_slices(requests_mock, stream_args): + stream_args["start_date"] = pendulum.now().subtract(days=1).int_timestamp + requests_mock.get( + "/v1/customers", + json={ + "data": [ + {"id": 1, "next_invoice_sequence": 1, "balance": 0, "created": 1653341716}, + {"id": 2, "created": 1653341000}, + {"id": 3, "next_invoice_sequence": 13, "balance": 343.43, "created": 1651716334}, + ] + }, + ) + stream = CustomerBalanceTransactions(**stream_args) + assert list(stream.stream_slices("full_refresh")) == [ + {"id": 2, "created": 1653341000, "updated": 1653341000}, + {"id": 3, "next_invoice_sequence": 13, "balance": 343.43, "created": 1651716334, "updated": 1651716334}, + ] + - requests_mock.get("https://api.stripe.com/v1/checkout/sessions", json=response_sessions) +@freezegun.freeze_time("2023-08-23T15:00:15Z") +def test_setup_attempts(requests_mock, incremental_stream_args): requests_mock.get( - "https://api.stripe.com/v1/checkout/sessions/{}/line_items".format(session_id_exists), json=response_sessions_line_items + "/v1/setup_intents", + [ + {"json": {"data": [{"id": 1, "created": 111, "object": "setup_intent"}]}}, + {"json": {"data": [{"id": 2, "created": 222, "object": "setup_intent"}]}}, + ], ) requests_mock.get( - "https://api.stripe.com/v1/checkout/sessions/{}/line_items".format(session_id_missed), json=response_error, status_code=404 + "/v1/setup_attempts", + [ + {"json": {"data": [{"id": 1, "created": 112, "object": "setup_attempt"}]}}, + {"json": {"data": [{"id": 2, "created": 230, "object": "setup_attempt"}]}}, + {"json": {"data": [{"id": 3, "created": 345, "object": "setup_attempt"}]}}, + {"json": {"data": [{"id": 4, "created": 450, "object": "setup_attempt"}]}}, + ], ) + incremental_stream_args["slice_range"] = 1 + incremental_stream_args["lookback_window_days"] = 0 + incremental_stream_args["start_date"] = pendulum.now().subtract(days=2).int_timestamp + stream = SetupAttempts(**incremental_stream_args) + slices = list(stream.stream_slices("full_refresh")) + assert slices == [ + { + "created[gte]": 1692630015, + "created[lte]": 1692716415, + "parent": {"id": 1, "created": 111, "updated": 111, "object": "setup_intent"}, + }, + { + "created[gte]": 1692716416, + "created[lte]": 1692802815, + "parent": {"id": 1, "created": 111, "updated": 111, "object": "setup_intent"}, + }, + { + "created[gte]": 1692630015, + "created[lte]": 1692716415, + "parent": {"id": 2, "created": 222, "updated": 222, "object": "setup_intent"}, + }, + { + "created[gte]": 1692716416, + "created[lte]": 1692802815, + "parent": {"id": 2, "created": 222, "updated": 222, "object": "setup_intent"}, + }, + ] + records = [] + for slice_ in slices: + for record in stream.read_records("full_refresh", stream_slice=slice_): + records.append(record) + assert records == [ + {"id": 1, "created": 112, "object": "setup_attempt"}, + {"id": 2, "created": 230, "object": "setup_attempt"}, + {"id": 3, "created": 345, "object": "setup_attempt"}, + {"id": 4, "created": 450, "object": "setup_attempt"}, + ] - stream = CheckoutSessionsLineItems(start_date=100_100, account_id=None) + +def test_persons_wo_state(requests_mock, stream_args): + requests_mock.get("/v1/accounts", json={"data": [{"id": 1, "object": "account", "created": 111}]}) + stream = Persons(**stream_args) + slices = list(stream.stream_slices("full_refresh")) + assert slices == [{"parent": {"id": 1, "object": "account", "created": 111}}] + requests_mock.get("/v1/accounts/1/persons", json={"data": [{"id": 11, "object": "person", "created": 222}]}) records = [] - for slice_ in stream.stream_slices(sync_mode=SyncMode.full_refresh): - records.extend(stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice_)) - assert len(records) == 1 + for slice_ in slices: + for record in stream.read_records("full_refresh", stream_slice=slice_): + records.append(record) + assert records == [{"id": 11, "object": "person", "created": 222, "updated": 222}] -def test_sub_stream(requests_mock): +@freezegun.freeze_time("2023-08-23T15:00:15") +def test_persons_w_state(requests_mock, stream_args): + requests_mock.get( + "/v1/events", + json={ + "data": [ + { + "id": "evt_1NdNFoEcXtiJtvvhBP5mxQmL", + "object": "event", + "api_version": "2020-08-27", + "created": 1691629292, + "data": {"object": {"object": "person", "name": "John", "created": 1653341716}}, + "type": "person.updated", + } + ], + "has_more": False, + }, + ) + stream = Persons(**stream_args) + slices = list(stream.stream_slices("incremental", stream_state={"updated": pendulum.parse("2023-08-20T00:00:00").int_timestamp})) + assert slices == [{}] + records = [ + record + for record in stream.read_records("incremental", stream_state={"updated": pendulum.parse("2023-08-20T00:00:00").int_timestamp}) + ] + assert records == [{"object": "person", "name": "John", "created": 1653341716, "updated": 1691629292}] - # First initial request to parent stream + +@pytest.mark.parametrize("sync_mode, stream_state", (("full_refresh", {}), ("incremental", {}), ("incremental", {"updated": 1693987430}))) +def test_cursorless_incremental_stream(requests_mock, stream_by_name, sync_mode, stream_state): + # Testing streams that *only* have the cursor field value in incremental mode because of API discrepancies, + # e.g. /bank_accounts does not return created/updated date, however /events?type=bank_account.updated returns the update date. + # Key condition here is that the underlying stream has legacy cursor field set to None. + stream = stream_by_name("external_account_bank_accounts") requests_mock.get( - "https://api.stripe.com/v1/invoices", + "/v1/accounts//external_accounts", json={ + "data": [ + { + "id": "ba_1Nncwa2eZvKYlo2CDILv1Q7N", + "object": "bank_account", + "account": "acct_1032D82eZvKYlo2C", + "bank_name": "STRIPE TEST BANK", + "country": "US", + } + ] + }, + ) + requests_mock.get( + "/v1/events", + json={ + "data": [ + { + "id": "evt_1NdNFoEcXtiJtvvhBP5mxQmL", + "object": "event", + "api_version": "2020-08-27", + "created": 1691629292, + "data": { + "object": { + "id": "ba_1Nncwa2eZvKYlo2CDILv1Q7N", + "object": "bank_account", + "account": "acct_1032D82eZvKYlo2C", + "bank_name": "STRIPE TEST BANK", + "country": "US", + } + }, + "type": "account.external_account.updated", + } + ], "has_more": False, + }, + ) + for slice_ in stream.stream_slices(sync_mode=sync_mode, stream_state=stream_state): + for record in stream.read_records(sync_mode=sync_mode, stream_state=stream_state, stream_slice=slice_): + stream.get_updated_state(stream_state, record) + # no assertions, this should be just a successful sync + + +@pytest.mark.parametrize("sync_mode, stream_state", (("full_refresh", {}), ("incremental", {}), ("incremental", {"updated": 1693987430}))) +def test_cursorless_incremental_substream(requests_mock, stream_by_name, sync_mode, stream_state): + # same for substreams + stream = stream_by_name("bank_accounts") + requests_mock.get( + "/v1/customers", + json={ + "data": [ + {"id": 1, "created": 1, "object": "customer", "sources": {"data": [{"id": 1, "object": "bank_account"}], "has_more": True}} + ], + "has_more": False, + }, + ) + requests_mock.get("/v1/customers/1/bank_accounts", json={"has_more": False, "data": [{"id": 2, "object": "bank_account"}]}) + requests_mock.get( + "/v1/events", + json={ + "data": [ + { + "id": "evt_1NdNFoEcXtiJtvvhBP5mxQmL", + "object": "event", + "api_version": "2020-08-27", + "created": 1691629292, + "data": { + "object": { + "id": "ba_1Nncwa2eZvKYlo2CDILv1Q7N", + "object": "bank_account", + "account": "acct_1032D82eZvKYlo2C", + "bank_name": "STRIPE TEST BANK", + "country": "US", + } + }, + "type": "account.external_account.updated", + } + ] + }, + ) + for slice_ in stream.stream_slices(sync_mode=sync_mode, stream_state=stream_state): + for record in stream.read_records(sync_mode=sync_mode, stream_state=stream_state, stream_slice=slice_): + stream.get_updated_state(stream_state, record) + + +@pytest.mark.parametrize("stream_name", ("bank_accounts",)) +def test_get_updated_state(stream_name, stream_by_name, requests_mock): + stream = stream_by_name(stream_name) + response = {"data": [{"id": 1, stream.cursor_field: 1695292083}]} + requests_mock.get("/v1/credit_notes", json=response) + requests_mock.get("/v1/balance_transactions", json=response) + requests_mock.get("/v1/invoices", json=response) + requests_mock.get( + "/v1/customers", + json={"data": [{"id": 1, "created": 1695292083, "sources": {"data": [{"id": 1, "object": "bank_account"}], "has_more": False}}]}, + ) + state = {} + for slice_ in stream.stream_slices(sync_mode="incremental", stream_state=state): + for record in stream.read_records(sync_mode="incremental", stream_slice=slice_, stream_state=state): + state = stream.get_updated_state(state, record) + assert state + + +@freezegun.freeze_time("2023-08-23T15:00:15Z") +def test_subscription_items_extra_request_params(requests_mock, stream_by_name, config): + requests_mock.get( + "/v1/subscriptions", + json={ "object": "list", - "url": "/v1/checkout/sessions", + "url": "/v1/subscriptions", + "has_more": False, "data": [ { - "created": 1641038947, - "customer": "cus_HezytZRkaQJC8W", - "id": "in_1KD6OVIEn5WyEQxn9xuASHsD", - "object": "invoice", - "total": 1, - "lines": { + "id": "sub_1OApco2eZvKYlo2CEDCzwLrE", + "object": "subscription", + "created": 1699603174, + "items": { + "object": "list", "data": [ { - "id": "il_1", - "object": "line_item", - }, - { - "id": "il_2", - "object": "line_item", - }, + "id": "si_OynDmET1kQPTbI", + "object": "subscription_item", + "created": 1699603175, + "quantity": 1, + "subscription": "sub_1OApco2eZvKYlo2CEDCzwLrE", + } ], "has_more": True, - "object": "list", - "total_count": 3, - "url": "/v1/invoices/in_1KD6OVIEn5WyEQxn9xuASHsD/lines", }, + "latest_invoice": None, + "livemode": False, } ], }, ) - - # Second pagination request to main stream requests_mock.get( - "https://api.stripe.com/v1/invoices/in_1KD6OVIEn5WyEQxn9xuASHsD/lines", + "/v1/subscription_items?subscription=sub_1OApco2eZvKYlo2CEDCzwLrE", json={ + "object": "list", + "url": "/v1/subscription_items", + "has_more": False, "data": [ { - "id": "il_3", - "object": "line_item", - }, + "id": "si_OynPdzMZykmCWm", + "object": "subscription_item", + "created": 1699603884, + "quantity": 2, + "subscription": "sub_1OApco2eZvKYlo2CEDCzwLrE", + } ], - "has_more": False, - "object": "list", - "total_count": 3, - "url": "/v1/invoices/in_1KD6OVIEn5WyEQxn9xuASHsD/lines", }, ) - # make start date a recent date so there's just one slice in a parent stream - start_date = pendulum.today().subtract(days=3).int_timestamp - stream = InvoiceLineItems(start_date=start_date, account_id="None") - records = [] - for slice_ in stream.stream_slices(sync_mode=SyncMode.full_refresh): - records.extend(stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice_)) - assert list(records) == [ - {"id": "il_1", "invoice_id": "in_1KD6OVIEn5WyEQxn9xuASHsD", "object": "line_item"}, - {"id": "il_2", "invoice_id": "in_1KD6OVIEn5WyEQxn9xuASHsD", "object": "line_item"}, - {"id": "il_3", "invoice_id": "in_1KD6OVIEn5WyEQxn9xuASHsD", "object": "line_item"}, + config["start_date"] = str(pendulum.now().subtract(days=3)) + stream = stream_by_name("subscription_items", config) + records = read_from_stream(stream, "full_refresh", {}) + assert records == [ + { + "id": "si_OynDmET1kQPTbI", + "object": "subscription_item", + "created": 1699603175, + "quantity": 1, + "subscription": "sub_1OApco2eZvKYlo2CEDCzwLrE", + }, + { + "id": "si_OynPdzMZykmCWm", + "object": "subscription_item", + "created": 1699603884, + "quantity": 2, + "subscription": "sub_1OApco2eZvKYlo2CEDCzwLrE", + }, ] + assert len(requests_mock.request_history) == 2 + assert "subscription=sub_1OApco2eZvKYlo2CEDCzwLrE" in requests_mock.request_history[-1].url -@pytest.mark.parametrize( - "stream_cls, kwargs, expected", - [ - (ApplicationFees, {}, "application_fees"), - (ApplicationFeesRefunds, {"stream_slice": {"refund_id": "fr"}}, "application_fees/fr/refunds"), - (Customers, {}, "customers"), - (BalanceTransactions, {}, "balance_transactions"), - (Charges, {}, "charges"), - (CustomerBalanceTransactions, {"stream_slice": {"id": "C1"}}, "customers/C1/balance_transactions"), - (Coupons, {}, "coupons"), - (Disputes, {}, "disputes"), - (EarlyFraudWarnings, {}, "radar/early_fraud_warnings"), - (Events, {}, "events"), - (Invoices, {}, "invoices"), - (InvoiceLineItems, {"stream_slice": {"invoice_id": "I1"}}, "invoices/I1/lines"), - (InvoiceItems, {}, "invoiceitems"), - (Payouts, {}, "payouts"), - (Persons, {"stream_slice": {"id": "A1"}}, "accounts/A1/persons"), - (Plans, {}, "plans"), - (Prices, {}, "prices"), - (Products, {}, "products"), - (Subscriptions, {}, "subscriptions"), - (SubscriptionItems, {}, "subscription_items"), - (SubscriptionSchedule, {}, "subscription_schedules"), - (Transfers, {}, "transfers"), - (Refunds, {}, "refunds"), - (PaymentIntents, {}, "payment_intents"), - (BankAccounts, {"stream_slice": {"customer_id": "C1"}}, "customers/C1/sources"), - (CheckoutSessions, {}, "checkout/sessions"), - (CheckoutSessionsLineItems, {"stream_slice": {"checkout_session_id": "CS1"}}, "checkout/sessions/CS1/line_items"), - (PromotionCodes, {}, "promotion_codes"), - (ExternalAccount, {}, "accounts//external_accounts"), - (SetupIntents, {}, "setup_intents"), - (ShippingRates, {}, "shipping_rates"), - ], -) -def test_path_and_headers( - stream_cls, - kwargs, - expected, - stream_args, -): - stream = stream_cls(**stream_args) - assert stream.path(**kwargs) == expected - headers = stream.request_headers(**kwargs) - assert headers["Stripe-Version"] == "2022-11-15" +checkout_session_api_response = { + "/v1/checkout/sessions": { + "object": "list", + "url": "/v1/checkout/sessions", + "has_more": False, + "data": [ + { + "id": "cs_test_a1yxusdFIgDDkWTaKn6JTYniMDBzrmnBiXH8oRSExZt7tcbIzIEoZk1Lre", + "object": "checkout.session", + "created": 1699647441, + "expires_at": 1699647441, + "payment_intent": "pi_1Gt0KQ2eZvKYlo2CeWXUgmhy", + "status": "open", + "line_items": { + "object": "list", + "has_more": False, + "url": "/v1/checkout/sessions", + "data": [ + { + "id": "li_1OB18o2eZvKYlo2CObYam50U", + "object": "item", + "amount_discount": 0, + "amount_subtotal": 0, + "amount_tax": 0, + "amount_total": 0, + "currency": "usd", + } + ], + }, + }, + { + "id": "cs_test_XH8oRSExZt7tcbIzIEoZk1Lrea1yxusdFIgDDkWTaKn6JTYniMDBzrmnBi", + "object": "checkout.session", + "created": 1699744164, + "expires_at": 1699644174, + "payment_intent": "pi_lo2CeWXUgmhy1Gt0KQ2eZvKY", + "status": "open", + "line_items": { + "object": "list", + "has_more": False, + "url": "/v1/checkout/sessions", + "data": [ + { + "id": "li_KYlo2CObYam50U1OB18o2eZv", + "object": "item", + "amount_discount": 0, + "amount_subtotal": 0, + "amount_tax": 0, + "amount_total": 0, + "currency": "usd", + } + ], + }, + }, + ], + } +} + + +checkout_session_line_items_api_response = { + "/v1/checkout/sessions/cs_test_a1yxusdFIgDDkWTaKn6JTYniMDBzrmnBiXH8oRSExZt7tcbIzIEoZk1Lre/line_items": { + "object": "list", + "has_more": False, + "data": [ + { + "id": "li_1OB18o2eZvKYlo2CObYam50U", + "object": "item", + "amount_discount": 0, + "amount_subtotal": 0, + "amount_tax": 0, + "amount_total": 0, + "currency": "usd", + } + ], + "link": "/v1/checkout/sessions/cs_test_a1yxusdFIgDDkWTaKn6JTYniMDBzrmnBiXH8oRSExZt7tcbIzIEoZk1Lre/line_items", + }, + "/v1/checkout/sessions/cs_test_XH8oRSExZt7tcbIzIEoZk1Lrea1yxusdFIgDDkWTaKn6JTYniMDBzrmnBi/line_items": { + "object": "list", + "has_more": False, + "url": "/v1/checkout/sessions/cs_test_XH8oRSExZt7tcbIzIEoZk1Lrea1yxusdFIgDDkWTaKn6JTYniMDBzrmnBi/line_items", + "data": [ + { + "id": "li_KYlo2CObYam50U1OB18o2eZv", + "object": "item", + "amount_discount": 0, + "amount_subtotal": 0, + "amount_tax": 0, + "amount_total": 0, + "currency": "usd", + } + ], + }, +} + + +checkout_session_events_response = { + "/v1/events": { + "data": [ + { + "id": "evt_1NdNFoEcXtiJtvvhBP5mxQmL", + "object": "event", + "api_version": "2020-08-27", + "created": 1699902016, + "data": { + "object": { + "object": "checkout_session", + "checkout_session": "cs_test_a1yxusdFIgDDkWTaKn6JTYniMDBzrmnBiXH8oRSExZt7tcbIzIEoZk1Lre", + "created": 1653341716, + "id": "cs_test_a1yxusdFIgDDkWTaKn6JTYniMDBzrmnBiXH8oRSExZt7tcbIzIEoZk1Lre", + "expires_at": 1692896410, + } + }, + "type": "checkout.session.completed", + }, + { + "id": "evt_XtiJtvvhBP5mxQmL1NdNFoEc", + "object": "event", + "api_version": "2020-08-27", + "created": 1699901630, + "data": { + "object": { + "object": "checkout_session", + "checkout_session": "cs_test_XH8oRSExZt7tcbIzIEoZk1Lrea1yxusdFIgDDkWTaKn6JTYniMDBzrmnBi", + "created": 1653341716, + "id": "cs_test_XH8oRSExZt7tcbIzIEoZk1Lrea1yxusdFIgDDkWTaKn6JTYniMDBzrmnBi", + "expires_at": 1692896410, + } + }, + "type": "checkout.session.completed", + }, + ], + "has_more": False, + }, +} @pytest.mark.parametrize( - "stream, kwargs, expected", - [ + "requests_mock_map, stream_name, sync_mode, state, expected_slices", + ( ( - CustomerBalanceTransactions, - {"stream_state": {}, "stream_slice": {"created[gte]": 1596466368, "created[lte]": 1596552768}}, - {"limit": 100, "created[gte]": 1596466368, "created[lte]": 1596552768}, + checkout_session_api_response, + "checkout_sessions_line_items", + "full_refresh", + {}, + [ + { + "parent": { + "id": "cs_test_a1yxusdFIgDDkWTaKn6JTYniMDBzrmnBiXH8oRSExZt7tcbIzIEoZk1Lre", + "object": "checkout.session", + "created": 1699647441, + "updated": 1699647441, + "expires_at": 1699647441, + "payment_intent": "pi_1Gt0KQ2eZvKYlo2CeWXUgmhy", + "status": "open", + "line_items": { + "object": "list", + "has_more": False, + "url": "/v1/checkout/sessions", + "data": [ + { + "id": "li_1OB18o2eZvKYlo2CObYam50U", + "object": "item", + "amount_discount": 0, + "amount_subtotal": 0, + "amount_tax": 0, + "amount_total": 0, + "currency": "usd", + } + ], + }, + } + }, + { + "parent": { + "id": "cs_test_XH8oRSExZt7tcbIzIEoZk1Lrea1yxusdFIgDDkWTaKn6JTYniMDBzrmnBi", + "object": "checkout.session", + "created": 1699744164, + "updated": 1699744164, + "expires_at": 1699644174, + "payment_intent": "pi_lo2CeWXUgmhy1Gt0KQ2eZvKY", + "status": "open", + "line_items": { + "object": "list", + "has_more": False, + "url": "/v1/checkout/sessions", + "data": [ + { + "id": "li_KYlo2CObYam50U1OB18o2eZv", + "object": "item", + "amount_discount": 0, + "amount_subtotal": 0, + "amount_tax": 0, + "amount_total": 0, + "currency": "usd", + } + ], + }, + } + }, + ], ), ( - Customers, - {"stream_state": {}, "stream_slice": {"created[gte]": 1596466368, "created[lte]": 1596552768}}, - {"created[gte]": 1596466368, "created[lte]": 1596552768, "limit": 100}, + checkout_session_events_response, + "checkout_sessions_line_items", + "incremental", + {"checkout_session_updated": 1685898010}, + [ + { + "parent": { + "object": "checkout_session", + "checkout_session": "cs_test_a1yxusdFIgDDkWTaKn6JTYniMDBzrmnBiXH8oRSExZt7tcbIzIEoZk1Lre", + "created": 1653341716, + "id": "cs_test_a1yxusdFIgDDkWTaKn6JTYniMDBzrmnBiXH8oRSExZt7tcbIzIEoZk1Lre", + "expires_at": 1692896410, + "updated": 1699902016, + } + }, + { + "parent": { + "object": "checkout_session", + "checkout_session": "cs_test_XH8oRSExZt7tcbIzIEoZk1Lrea1yxusdFIgDDkWTaKn6JTYniMDBzrmnBi", + "created": 1653341716, + "updated": 1699901630, + "id": "cs_test_XH8oRSExZt7tcbIzIEoZk1Lrea1yxusdFIgDDkWTaKn6JTYniMDBzrmnBi", + "expires_at": 1692896410, + } + }, + ], ), - (InvoiceLineItems, {"stream_state": {}, "stream_slice": {"starting_after": "2030"}}, {"limit": 100, "starting_after": "2030"}), + ), +) +@freezegun.freeze_time("2023-08-23T15:00:15") +def test_parent_incremental_substream_stream_slices( + requests_mock, requests_mock_map, stream_by_name, stream_name, sync_mode, state, expected_slices +): + for url, response in requests_mock_map.items(): + requests_mock.get(url, json=response) + + stream = stream_by_name(stream_name) + slices = stream.stream_slices(sync_mode, stream_state=state) + assert list(slices) == expected_slices + + +checkout_session_line_items_slice_to_record_data_map = { + "id": "checkout_session_id", + "expires_at": "checkout_session_expires_at", + "created": "checkout_session_created", + "updated": "checkout_session_updated", +} + + +@pytest.mark.parametrize( + "requests_mock_map, stream_name, sync_mode, state, mapped_fields", + ( ( - Subscriptions, - {"stream_slice": {"created[gte]": 1596466368, "created[lte]": 1596552768}}, - {"created[gte]": 1596466368, "limit": 100, "status": "all", "created[lte]": 1596552768}, + {**checkout_session_api_response, **checkout_session_line_items_api_response}, + "checkout_sessions_line_items", + "full_refresh", + {}, + checkout_session_line_items_slice_to_record_data_map, ), - (SubscriptionItems, {"stream_state": {}, "stream_slice": {"subscription_id": "SI"}}, {"limit": 100, "subscription": "SI"}), - (BankAccounts, {"stream_state": {}, "stream_slice": {"subscription_id": "SI"}}, {"limit": 100, "object": "bank_account"}), - (CheckoutSessions, {"stream_state": None, "stream_slice": {}}, {"limit": 100}), ( - CheckoutSessionsLineItems, - {"stream_state": None, "stream_slice": {}}, - {"limit": 100, "expand[]": ["data.discounts", "data.taxes"]}, + {**checkout_session_events_response, **checkout_session_line_items_api_response}, + "checkout_sessions_line_items", + "incremental", + {"checkout_session_updated": 1685898010}, + checkout_session_line_items_slice_to_record_data_map, ), - (ExternalAccountBankAccounts, {"stream_state": None, "stream_slice": {}}, {"limit": 100, "object": "bank_account"}), - (ExternalAccountCards, {"stream_state": None, "stream_slice": {}}, {"limit": 100, "object": "card"}), - ], + ), ) -def test_request_params( - stream, - kwargs, - expected, - stream_args, +def test_parent_incremental_substream_records_contain_data_from_slice( + requests_mock, requests_mock_map, stream_by_name, stream_name, sync_mode, state, mapped_fields ): - assert stream(**stream_args).request_params(**kwargs) == expected + for url, response in requests_mock_map.items(): + requests_mock.get(url, json=response) + + stream = stream_by_name(stream_name) + for slice_ in stream.stream_slices(sync_mode, stream_state=state): + for record in stream.read_records(sync_mode, stream_slice=slice_, stream_state=state): + for key, value in mapped_fields.items(): + assert slice_["parent"][key] == record[value] @pytest.mark.parametrize( - "stream_cls", + "requests_mock_map, stream_name, state", ( - ApplicationFees, - Customers, - BalanceTransactions, - Charges, - Coupons, - Disputes, - Events, - Invoices, - InvoiceItems, - Payouts, - Plans, - Prices, - Products, - Subscriptions, - SubscriptionSchedule, - Transfers, - Refunds, - PaymentIntents, - CheckoutSessions, - PromotionCodes, - ExternalAccount, - SetupIntents, - ShippingRates - ) + ( + { + "/v1/events": ( + { + "data": [ + { + "id": "evt_1NdNFoEcXtiJtvvhBP5mxQmL", + "object": "event", + "api_version": "2020-08-27", + "created": 1699902016, + "data": { + "object": { + "object": "checkout_session", + "checkout_session": "cs_1K9GK0EcXtiJtvvhSo2LvGqT", + "created": 1653341716, + "id": "cs_1K9GK0EcXtiJtvvhSo2LvGqT", + "expires_at": 1692896410, + } + }, + "type": "checkout.session.completed", + } + ], + "has_more": False, + }, + 200, + ), + "/v1/checkout/sessions/cs_1K9GK0EcXtiJtvvhSo2LvGqT/line_items": ({}, 404), + }, + "checkout_sessions_line_items", + {"checkout_session_updated": 1686934810}, + ), + ), ) -def test_403_error_handling(stream_args, stream_cls, requests_mock): - stream = stream_cls(**stream_args) - logger = logging.getLogger("airbyte") - for error_code in STRIPE_ERROR_CODES: - requests_mock.get(f"{stream.url_base}{stream.path()}", status_code=403, json={"error": {"code": f"{error_code}"}}) - available, message = stream.check_availability(logger) - assert not available - assert STRIPE_ERROR_CODES[error_code] in message +@freezegun.freeze_time("2023-08-23T15:00:15") +def test_parent_incremental_substream_handles_404(requests_mock, requests_mock_map, stream_by_name, stream_name, state, caplog): + for url, (response, status) in requests_mock_map.items(): + requests_mock.get(url, json=response, status_code=status) + + stream = stream_by_name(stream_name) + records = read_from_stream(stream, "incremental", state) + assert records == [] + assert "Data was not found for URL" in caplog.text diff --git a/airbyte-integrations/connectors/source-survey-sparrow/README.md b/airbyte-integrations/connectors/source-survey-sparrow/README.md index 888bfbbbf128..ade52dd01f82 100644 --- a/airbyte-integrations/connectors/source-survey-sparrow/README.md +++ b/airbyte-integrations/connectors/source-survey-sparrow/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-survey-sparrow:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/survey-sparrow) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_survey_sparrow/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-survey-sparrow:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-survey-sparrow build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-survey-sparrow:airbyteDocker +An image will be built with the tag `airbyte/source-survey-sparrow:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-survey-sparrow:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-survey-sparrow:dev che docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-survey-sparrow:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-survey-sparrow:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-survey-sparrow test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-survey-sparrow:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-survey-sparrow:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-survey-sparrow test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/survey-sparrow.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-survey-sparrow/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-survey-sparrow/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-survey-sparrow/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-survey-sparrow/build.gradle b/airbyte-integrations/connectors/source-survey-sparrow/build.gradle deleted file mode 100644 index 620affb88541..000000000000 --- a/airbyte-integrations/connectors/source-survey-sparrow/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_survey_sparrow' -} diff --git a/airbyte-integrations/connectors/source-surveycto/README.md b/airbyte-integrations/connectors/source-surveycto/README.md index e718795db84d..e5879b99a770 100644 --- a/airbyte-integrations/connectors/source-surveycto/README.md +++ b/airbyte-integrations/connectors/source-surveycto/README.md @@ -44,14 +44,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-surveycto:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/surveycto) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_surveycto/spec.yaml` file. @@ -71,18 +63,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-surveycto:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-surveycto build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-surveycto:airbyteDocker +An image will be built with the tag `airbyte/source-surveycto:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-surveycto:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -92,44 +85,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-surveycto:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-surveycto:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-surveycto:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-surveycto test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-surveycto:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-surveycto:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -139,8 +104,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-surveycto test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/surveycto.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-surveycto/acceptance-test-config.yml b/airbyte-integrations/connectors/source-surveycto/acceptance-test-config.yml index 76059e318f17..002bd1124ddf 100644 --- a/airbyte-integrations/connectors/source-surveycto/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-surveycto/acceptance-test-config.yml @@ -18,7 +18,7 @@ tests: full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - incremental: + incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" \ No newline at end of file + future_state_path: "integration_tests/abnormal_state.json" diff --git a/airbyte-integrations/connectors/source-surveycto/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-surveycto/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-surveycto/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-surveycto/build.gradle b/airbyte-integrations/connectors/source-surveycto/build.gradle deleted file mode 100644 index bc6c05284a28..000000000000 --- a/airbyte-integrations/connectors/source-surveycto/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_surveycto' -} diff --git a/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py b/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py index 0e3396d52a68..dbf4fe4d6e47 100644 --- a/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-surveycto/unit_tests/test_source.py @@ -8,27 +8,27 @@ from source_surveycto.source import SourceSurveycto, SurveyctoStream -@pytest.fixture(name='config') +@pytest.fixture(name="config") def config_fixture(): return { - 'server_name': 'server_name', - 'form_id': ['form_id_1', 'form_id_2'], - 'start_date': 'Jan 09, 2022 00:00:00 AM', - 'password': 'password', - 'username': 'username' + "server_name": "server_name", + "form_id": ["form_id_1", "form_id_2"], + "start_date": "Jan 09, 2022 00:00:00 AM", + "password": "password", + "username": "username", } -@pytest.fixture(name='source') +@pytest.fixture(name="source") def source_fixture(): return SourceSurveycto() -@pytest.fixture(name='mock_survey_cto') +@pytest.fixture(name="mock_survey_cto") def mock_survey_cto_fixture(): - with patch('source_surveycto.source.Helpers.call_survey_cto', return_value="value") as mock_call_survey_cto, \ - patch('source_surveycto.source.Helpers.get_filter_data', return_value="value") as mock_filter_data, \ - patch('source_surveycto.source.Helpers.get_json_schema', return_value="value") as mock_json_schema: + with patch("source_surveycto.source.Helpers.call_survey_cto", return_value="value") as mock_call_survey_cto, patch( + "source_surveycto.source.Helpers.get_filter_data", return_value="value" + ) as mock_filter_data, patch("source_surveycto.source.Helpers.get_json_schema", return_value="value") as mock_json_schema: yield mock_call_survey_cto, mock_filter_data, mock_json_schema @@ -36,13 +36,13 @@ def test_check_connection_valid(mock_survey_cto, source, config): logger_mock = MagicMock() records = iter(["record1", "record2"]) - with patch.object(SurveyctoStream, 'read_records', return_value=records): + with patch.object(SurveyctoStream, "read_records", return_value=records): assert source.check_connection(logger_mock, config) == (True, None) def test_check_connection_failure(mock_survey_cto, source, config): logger_mock = MagicMock() - expected_outcome = 'Unable to connect - 400 Client Error: 400 for url: https://server_name.surveycto.com/api/v2/forms/data/wide/json/form_id_1?date=Jan+09%2C+2022+00%3A00%3A00+AM' + expected_outcome = "Unable to connect - 400 Client Error: 400 for url: https://server_name.surveycto.com/api/v2/forms/data/wide/json/form_id_1?date=Jan+09%2C+2022+00%3A00%3A00+AM" assert source.check_connection(logger_mock, config) == (False, expected_outcome) @@ -51,7 +51,7 @@ def test_generate_streams(mock_survey_cto, source, config): assert len(streams) == 2 -@patch('source_surveycto.source.SourceSurveycto.generate_streams', return_value=['stream_1', 'stream2']) +@patch("source_surveycto.source.SourceSurveycto.generate_streams", return_value=["stream_1", "stream2"]) def test_streams(mock_generate_streams, source, config): streams = source.streams(config) assert len(streams) == 2 diff --git a/airbyte-integrations/connectors/source-surveycto/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-surveycto/unit_tests/test_streams.py index 2a3497a95941..6818528e170a 100644 --- a/airbyte-integrations/connectors/source-surveycto/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-surveycto/unit_tests/test_streams.py @@ -8,9 +8,15 @@ from source_surveycto.helpers import Helpers -@pytest.fixture(name='config') +@pytest.fixture(name="config") def config_fixture(): - return {'server_name': 'server_name', 'form_id': 'form_id', 'start_date': 'Jan 09, 2022 00:00:00 AM', 'password': 'password', 'username': 'username'} + return { + "server_name": "server_name", + "form_id": "form_id", + "start_date": "Jan 09, 2022 00:00:00 AM", + "password": "password", + "username": "username", + } @pytest.fixture @@ -32,7 +38,7 @@ def json_response(): def expected_json_schema(): return { "$schema": "http://json-schema.org/draft-07/schema#", - "properties": {'fields': {'name': 'test'}, 'id': 'abc'}, + "properties": {"fields": {"name": "test"}, "id": "abc"}, "type": "object", } diff --git a/airbyte-integrations/connectors/source-surveymonkey/Dockerfile b/airbyte-integrations/connectors/source-surveymonkey/Dockerfile deleted file mode 100644 index 043acddb543a..000000000000 --- a/airbyte-integrations/connectors/source-surveymonkey/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY source_surveymonkey ./source_surveymonkey -COPY main.py ./ -COPY setup.py ./ -RUN pip install . - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - - -LABEL io.airbyte.version=0.2.2 -LABEL io.airbyte.name=airbyte/source-surveymonkey diff --git a/airbyte-integrations/connectors/source-surveymonkey/README.md b/airbyte-integrations/connectors/source-surveymonkey/README.md index fe5582caa5ea..0ef47e0bcc4c 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/README.md +++ b/airbyte-integrations/connectors/source-surveymonkey/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-surveymonkey:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/surveymonkey) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_surveymonkey/spec.json` file. @@ -56,19 +48,70 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-surveymonkey:dev + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-surveymonkey build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-surveymonkey:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. -You can also build the connector image via Gradle: +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-surveymonkey:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-surveymonkey:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-surveymonkey:dev . +# Running the spec command against your patched connector +docker run airbyte/source-surveymonkey:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -77,44 +120,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-surveymonkey:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-surveymonkey:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-surveymonkey:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-surveymonkey test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-surveymonkey:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-surveymonkey:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -124,14 +139,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. - -### Performance considerations +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-surveymonkey test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/surveymonkey.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. -The SurveyMonkey API applies heavy API quotas for default private apps, which have the following limits: -* 125 requests per minute -* 500 requests per day \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-surveymonkey/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-surveymonkey/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-surveymonkey/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-surveymonkey/build.gradle b/airbyte-integrations/connectors/source-surveymonkey/build.gradle deleted file mode 100644 index 56a271660fbd..000000000000 --- a/airbyte-integrations/connectors/source-surveymonkey/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_surveymonkey' -} diff --git a/airbyte-integrations/connectors/source-surveymonkey/metadata.yaml b/airbyte-integrations/connectors/source-surveymonkey/metadata.yaml index 1c5c6fc423a9..4ef8ecb31f1a 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/metadata.yaml +++ b/airbyte-integrations/connectors/source-surveymonkey/metadata.yaml @@ -1,12 +1,18 @@ data: + ab_internal: + ql: 200 + sl: 200 allowedHosts: hosts: - api.surveymonkey.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: api connectorType: source definitionId: badc5925-0485-42be-8caa-b34096cb71b5 - dockerImageTag: 0.2.2 + dockerImageTag: 0.2.3 dockerRepository: airbyte/source-surveymonkey + documentationUrl: https://docs.airbyte.com/integrations/sources/surveymonkey githubIssueLabel: source-surveymonkey icon: surveymonkey.svg license: MIT @@ -17,11 +23,7 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/surveymonkey + supportLevel: certified tags: - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_source.py b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_source.py index 855f9eb5a67e..d76e50f555d1 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_source.py @@ -42,6 +42,22 @@ def test_source_check_connection_failed_missing_scopes(requests_mock): results = SourceSurveymonkey().check_connection(logger=None, config=new_source_config) assert results == (False, "missed required scopes: responses_read_detail") +def test_source_check_connection_config_with_survey_id_errors(requests_mock): + mock_status_code = 404 + mock_survey_id = "1234567890" + mock_msg = f"{mock_status_code} None: {mock_survey_id}" + + new_source_config['survey_ids'] = [mock_survey_id] + requests_mock.get( + "https://api.surveymonkey.com/v3/users/me", json={"scopes": {"granted": ["responses_read_detail", "surveys_read", "users_read"]}} + ) + + requests_mock.head( + f"https://api.surveymonkey.com/v3/surveys/{mock_survey_id}/details", status_code=mock_status_code + ) + + results = SourceSurveymonkey().check_connection(logger=None, config=new_source_config) + assert results == (False, mock_msg) @pytest.mark.parametrize( "config, err_msg", diff --git a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py index 50de2ced472f..b2148397749b 100644 --- a/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-surveymonkey/unit_tests/test_streams.py @@ -327,24 +327,20 @@ def test_survey_questions(requests_mock): def test_survey_collectors(requests_mock): - requests_mock.get("https://api.surveymonkey.com/v3/surveys/307785415/collectors", json={ - "data": [{"name": "Teams Poll", "id": "1", "href": "https://api.surveymonkey.com/v3/collectors/1"}], - "per_page": 50, - "page": 1, - "total": 1, - "links": { - "self": "https://api.surveymonkey.com/v3/surveys/307785415/collectors?page=1&per_page=50" - } - }) + requests_mock.get( + "https://api.surveymonkey.com/v3/surveys/307785415/collectors", + json={ + "data": [{"name": "Teams Poll", "id": "1", "href": "https://api.surveymonkey.com/v3/collectors/1"}], + "per_page": 50, + "page": 1, + "total": 1, + "links": {"self": "https://api.surveymonkey.com/v3/surveys/307785415/collectors?page=1&per_page=50"}, + }, + ) args = {**args_mock, **{"survey_ids": ["307785415"]}} records = SurveyCollectors(**args).read_records(sync_mode=SyncMode.full_refresh, stream_slice={"survey_id": "307785415"}) assert list(records) == [ - { - "name": "Teams Poll", - "id": "1", - "href": "https://api.surveymonkey.com/v3/collectors/1", - "survey_id": "307785415" - } + {"name": "Teams Poll", "id": "1", "href": "https://api.surveymonkey.com/v3/collectors/1", "survey_id": "307785415"} ] @@ -400,7 +396,12 @@ def test_surveys_responses_get_updated_state(current_stream_state, latest_record ), ( {}, - {"sort_order": "ASC", "sort_by": "date_modified", "per_page": 100, "start_modified_at": "2000-01-01T00:00:00"}, # return start_date + { + "sort_order": "ASC", + "sort_by": "date_modified", + "per_page": 100, + "start_modified_at": "2000-01-01T00:00:00", + }, # return start_date ), ], ) diff --git a/airbyte-integrations/connectors/source-talkdesk-explore/README.md b/airbyte-integrations/connectors/source-talkdesk-explore/README.md index 7b42072e3ad2..8f8fa650ed71 100644 --- a/airbyte-integrations/connectors/source-talkdesk-explore/README.md +++ b/airbyte-integrations/connectors/source-talkdesk-explore/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-talkdesk-explore-explore:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/talkdesk-explore) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_talkdesk_explore/spec.json` file. @@ -54,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-talkdesk-explore-explore:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-talkdesk-explore build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-talkdesk-explore-explore:airbyteDocker +An image will be built with the tag `airbyte/source-talkdesk-explore:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-talkdesk-explore:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -75,44 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-talkdesk-explore:dev c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-talkdesk-explore:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-talkdesk-explore:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-talkdesk-explore test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-talkdesk-explore:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-talkdesk-explore:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -122,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-talkdesk-explore test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/talkdesk-explore.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-talkdesk-explore/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-talkdesk-explore/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-talkdesk-explore/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-talkdesk-explore/build.gradle b/airbyte-integrations/connectors/source-talkdesk-explore/build.gradle deleted file mode 100644 index c78ee4737128..000000000000 --- a/airbyte-integrations/connectors/source-talkdesk-explore/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_talkdesk_singer' -} diff --git a/airbyte-integrations/connectors/source-tempo/README.md b/airbyte-integrations/connectors/source-tempo/README.md index c00f9698fa81..7ae7456e2be0 100644 --- a/airbyte-integrations/connectors/source-tempo/README.md +++ b/airbyte-integrations/connectors/source-tempo/README.md @@ -1,108 +1,67 @@ -# Tempo Source +# Fullstory Source -This is the repository for the Tempo source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/tempo). +This is the repository for the Fullstory configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/fullstory). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-tempo:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/tempo) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_tempo/spec.json` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/fullstory) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_fullstory/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source tempo test creds` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source fullstory test creds` and place them into `secrets/config.json`. - -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-tempo:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name source-fullstory build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-tempo:airbyteDocker +An image will be built with the tag `airbyte/source-fullstory:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-fullstory:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-tempo:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-tempo:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-tempo:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-tempo:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm airbyte/source-fullstory:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-fullstory:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-fullstory:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-fullstory:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-tempo:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-tempo test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-tempo test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/tempo.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-tempo/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-tempo/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-tempo/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-tempo/build.gradle b/airbyte-integrations/connectors/source-tempo/build.gradle deleted file mode 100644 index 117ebdc5cb2c..000000000000 --- a/airbyte-integrations/connectors/source-tempo/build.gradle +++ /dev/null @@ -1,10 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_tempo' -} - diff --git a/airbyte-integrations/connectors/source-tempo/metadata.yaml b/airbyte-integrations/connectors/source-tempo/metadata.yaml index a3bc6a870eba..d29e9d18b11c 100644 --- a/airbyte-integrations/connectors/source-tempo/metadata.yaml +++ b/airbyte-integrations/connectors/source-tempo/metadata.yaml @@ -1,4 +1,7 @@ data: + ab_internal: + ql: 300 + sl: 100 allowedHosts: hosts: - api.tempo.io @@ -7,6 +10,7 @@ data: definitionId: d1aa448b-7c54-498e-ad95-263cbebcd2db dockerImageTag: 0.3.1 dockerRepository: airbyte/source-tempo + documentationUrl: https://docs.airbyte.com/integrations/sources/tempo githubIssueLabel: source-tempo icon: tempo.svg license: MIT @@ -17,12 +21,8 @@ data: oss: enabled: true releaseStage: beta - documentationUrl: https://docs.airbyte.com/integrations/sources/tempo + supportLevel: community tags: - language:low-code - language:python - ab_internal: - sl: 200 - ql: 300 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-teradata/Dockerfile b/airbyte-integrations/connectors/source-teradata/Dockerfile deleted file mode 100644 index 88631917ad75..000000000000 --- a/airbyte-integrations/connectors/source-teradata/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-teradata - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-teradata - -COPY --from=build /airbyte /airbyte - -# Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile. -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-teradata diff --git a/airbyte-integrations/connectors/source-teradata/README.md b/airbyte-integrations/connectors/source-teradata/README.md index ad9a67489aa1..c80424c6b39b 100644 --- a/airbyte-integrations/connectors/source-teradata/README.md +++ b/airbyte-integrations/connectors/source-teradata/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:source-teradata:airbyteDocker +./gradlew :airbyte-integrations:connectors:source-teradata:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/source-teradata:dev`. the Dockerfile. #### Run @@ -62,8 +63,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-teradata test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/teradata.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-teradata/acceptance-test-config.yml b/airbyte-integrations/connectors/source-teradata/acceptance-test-config.yml deleted file mode 100644 index 49f39cead0ef..000000000000 --- a/airbyte-integrations/connectors/source-teradata/acceptance-test-config.yml +++ /dev/null @@ -1,8 +0,0 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) -# for more information about how to configure these tests -connector_image: airbyte/source-teradata:dev -acceptance_tests: - spec: - tests: - - spec_path: "src/test-integration/resources/expected_spec.json" - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-teradata/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-teradata/acceptance-test-docker.sh deleted file mode 100644 index 394835b93b09..000000000000 --- a/airbyte-integrations/connectors/source-teradata/acceptance-test-docker.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env sh - -# Build latest connector image -docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) - -# Pull latest acctest image -docker pull airbyte/connector-acceptance-test:latest - -# Run -docker run --rm -it \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /tmp:/tmp \ - -v $(pwd):/test_input \ - airbyte/connector-acceptance-test \ - --acceptance-test-config /test_input diff --git a/airbyte-integrations/connectors/source-teradata/build.gradle b/airbyte-integrations/connectors/source-teradata/build.gradle index 47b083feee8a..985aaa4b48fc 100644 --- a/airbyte-integrations/connectors/source-teradata/build.gradle +++ b/airbyte-integrations/connectors/source-teradata/build.gradle @@ -1,30 +1,34 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' - id 'airbyte-connector-acceptance-test' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.7.7' + features = ['db-sources'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileTestJava { + options.compilerArgs.remove("-Werror") + } + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.source.teradata.TeradataSource' } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation project(':airbyte-integrations:connectors:source-relational-db') implementation 'com.teradata.jdbc:terajdbc:20.00.00.06' - testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) - testImplementation 'org.apache.commons:commons-lang3:3.11' - - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-teradata') - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + testImplementation libs.testcontainers.jdbc } diff --git a/airbyte-integrations/connectors/source-teradata/metadata.yaml b/airbyte-integrations/connectors/source-teradata/metadata.yaml index fc063a148c33..cd03a4a562c1 100644 --- a/airbyte-integrations/connectors/source-teradata/metadata.yaml +++ b/airbyte-integrations/connectors/source-teradata/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: database connectorType: source definitionId: aa8ba6fd-4875-d94e-fc8d-4e1e09aa2503 - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-teradata githubIssueLabel: source-teradata icon: teradata.svg diff --git a/airbyte-integrations/connectors/source-teradata/src/main/java/io/airbyte/integrations/source/teradata/TeradataSource.java b/airbyte-integrations/connectors/source-teradata/src/main/java/io/airbyte/integrations/source/teradata/TeradataSource.java index 129b289328c7..d9410b75cdd1 100644 --- a/airbyte-integrations/connectors/source-teradata/src/main/java/io/airbyte/integrations/source/teradata/TeradataSource.java +++ b/airbyte-integrations/connectors/source-teradata/src/main/java/io/airbyte/integrations/source/teradata/TeradataSource.java @@ -6,18 +6,18 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.factory.DataSourceFactory; +import io.airbyte.cdk.db.jdbc.JdbcDatabase; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.jdbc.StreamingJdbcDatabase; +import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; +import io.airbyte.cdk.integrations.source.jdbc.JdbcDataSourceUtils; +import io.airbyte.cdk.integrations.source.relationaldb.TableInfo; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.map.MoreMaps; -import io.airbyte.db.factory.DataSourceFactory; -import io.airbyte.db.jdbc.JdbcDatabase; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.StreamingJdbcDatabase; -import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.JdbcDataSourceUtils; -import io.airbyte.integrations.source.relationaldb.TableInfo; import io.airbyte.protocol.models.CommonField; import java.io.IOException; import java.io.PrintWriter; @@ -110,13 +110,15 @@ public JdbcDatabase createDatabase(JsonNode sourceConfig) throws SQLException { JdbcDataSourceUtils.assertCustomParametersDontOverwriteDefaultParameters(customProperties, sslConnectionProperties); final JsonNode jdbcConfig = toDatabaseConfig(sourceConfig); + final Map connectionProperties = MoreMaps.merge(customProperties, sslConnectionProperties); // Create the data source final DataSource dataSource = DataSourceFactory.create( jdbcConfig.has(JdbcUtils.USERNAME_KEY) ? jdbcConfig.get(JdbcUtils.USERNAME_KEY).asText() : null, jdbcConfig.has(JdbcUtils.PASSWORD_KEY) ? jdbcConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, - driverClass, + driverClassName, jdbcConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), - MoreMaps.merge(customProperties, sslConnectionProperties)); + connectionProperties, + getConnectionTimeout(connectionProperties, driverClassName)); // Record the data source so that it can be closed. dataSources.add(dataSource); diff --git a/airbyte-integrations/connectors/source-teradata/src/main/java/io/airbyte/integrations/source/teradata/TeradataSourceOperations.java b/airbyte-integrations/connectors/source-teradata/src/main/java/io/airbyte/integrations/source/teradata/TeradataSourceOperations.java index c6309853d818..1d1d96f6de0d 100644 --- a/airbyte-integrations/connectors/source-teradata/src/main/java/io/airbyte/integrations/source/teradata/TeradataSourceOperations.java +++ b/airbyte-integrations/connectors/source-teradata/src/main/java/io/airbyte/integrations/source/teradata/TeradataSourceOperations.java @@ -4,12 +4,12 @@ package io.airbyte.integrations.source.teradata; -import static io.airbyte.db.DataTypeUtils.TIMESTAMPTZ_FORMATTER; +import static io.airbyte.cdk.db.DataTypeUtils.TIMESTAMPTZ_FORMATTER; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.airbyte.db.DataTypeUtils; -import io.airbyte.db.jdbc.DateTimeConverter; -import io.airbyte.db.jdbc.JdbcSourceOperations; +import io.airbyte.cdk.db.DataTypeUtils; +import io.airbyte.cdk.db.jdbc.DateTimeConverter; +import io.airbyte.cdk.db.jdbc.JdbcSourceOperations; import java.sql.Date; import java.sql.PreparedStatement; import java.sql.ResultSet; diff --git a/airbyte-integrations/connectors/source-teradata/src/main/java/io/airbyte/integrations/source/teradata/envclient/dto/DeleteEnvironmentRequest.java b/airbyte-integrations/connectors/source-teradata/src/main/java/io/airbyte/integrations/source/teradata/envclient/dto/DeleteEnvironmentRequest.java index 98622f2cc227..7fc7f6eaafaf 100644 --- a/airbyte-integrations/connectors/source-teradata/src/main/java/io/airbyte/integrations/source/teradata/envclient/dto/DeleteEnvironmentRequest.java +++ b/airbyte-integrations/connectors/source-teradata/src/main/java/io/airbyte/integrations/source/teradata/envclient/dto/DeleteEnvironmentRequest.java @@ -4,8 +4,4 @@ package io.airbyte.integrations.source.teradata.envclient.dto; -public record DeleteEnvironmentRequest( - - String name - -) {} +public record DeleteEnvironmentRequest(String name) {} diff --git a/airbyte-integrations/connectors/source-teradata/src/test-integration/java/io/airbyte/integrations/source/teradata/TeradataSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-teradata/src/test-integration/java/io/airbyte/integrations/source/teradata/TeradataSourceAcceptanceTest.java index ccf89017b8c8..4f04fcaf1e3d 100644 --- a/airbyte-integrations/connectors/source-teradata/src/test-integration/java/io/airbyte/integrations/source/teradata/TeradataSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-teradata/src/test-integration/java/io/airbyte/integrations/source/teradata/TeradataSourceAcceptanceTest.java @@ -6,14 +6,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.integrations.source.teradata.envclient.TeradataHttpClient; import io.airbyte.integrations.source.teradata.envclient.dto.CreateEnvironmentRequest; import io.airbyte.integrations.source.teradata.envclient.dto.DeleteEnvironmentRequest; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.CatalogHelpers; @@ -32,8 +32,10 @@ import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.TestInstance; +@Disabled @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class TeradataSourceAcceptanceTest extends SourceAcceptanceTest { diff --git a/airbyte-integrations/connectors/source-teradata/src/test/java/io/airbyte/integrations/source/teradata/TeradataJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-teradata/src/test/java/io/airbyte/integrations/source/teradata/TeradataJdbcSourceAcceptanceTest.java index f00c338098fe..e755e01af383 100644 --- a/airbyte-integrations/connectors/source-teradata/src/test/java/io/airbyte/integrations/source/teradata/TeradataJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-teradata/src/test/java/io/airbyte/integrations/source/teradata/TeradataJdbcSourceAcceptanceTest.java @@ -6,51 +6,42 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.commons.io.IOs; import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.integrations.source.teradata.envclient.TeradataHttpClient; import io.airbyte.integrations.source.teradata.envclient.dto.CreateEnvironmentRequest; import io.airbyte.integrations.source.teradata.envclient.dto.DeleteEnvironmentRequest; import java.nio.file.Path; import java.sql.Connection; import java.sql.DriverManager; -import java.sql.JDBCType; import java.sql.SQLException; import java.sql.Statement; import java.util.List; import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.TestInstance; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +@Disabled @TestInstance(TestInstance.Lifecycle.PER_CLASS) -class TeradataJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { +class TeradataJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { - private static final Logger LOGGER = LoggerFactory.getLogger(TeradataJdbcSourceAcceptanceTest.class); + private static JsonNode staticConfig; - private JsonNode staticConfig; - - static { - COLUMN_CLAUSE_WITH_PK = "id INTEGER NOT NULL, name VARCHAR(200) NOT NULL, updated_at DATE NOT NULL"; - - CREATE_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s ST_Geometry) NO PRIMARY INDEX;"; - INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES('POLYGON((1 1, 1 3, 6 3, 6 0, 1 1))');"; - - COL_TIMESTAMP = "tmstmp"; - INSERT_TABLE_NAME_AND_TIMESTAMP_QUERY = "INSERT INTO %s (name, tmstmp) VALUES ('%s', '%s')"; - COL_TIMESTAMP_TYPE = "TIMESTAMP(0)"; + public static void cleanUpBeforeStarting() { + try { + cleanupEnvironment(); + } catch (final Exception ignored) {} } @BeforeAll - public void initEnvironment() throws ExecutionException, InterruptedException { + static void initEnvironment() throws ExecutionException, InterruptedException { staticConfig = Jsons.deserialize(IOs.readFile(Path.of("secrets/config.json"))); - TeradataHttpClient teradataHttpClient = new TeradataHttpClient(staticConfig.get("env_host").asText()); + cleanUpBeforeStarting(); + final TeradataHttpClient teradataHttpClient = new TeradataHttpClient(staticConfig.get("env_host").asText()); var request = new CreateEnvironmentRequest( staticConfig.get("env_name").asText(), staticConfig.get("env_region").asText(), @@ -62,25 +53,25 @@ public void initEnvironment() throws ExecutionException, InterruptedException { } catch (ClassNotFoundException e) { throw new RuntimeException(e); } + + COLUMN_CLAUSE_WITH_PK = "id INTEGER NOT NULL, name VARCHAR(200) NOT NULL, updated_at DATE NOT NULL"; + + CREATE_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "CREATE TABLE %s (%s ST_Geometry) NO PRIMARY INDEX;"; + INSERT_TABLE_WITHOUT_CURSOR_TYPE_QUERY = "INSERT INTO %s VALUES('POLYGON((1 1, 1 3, 6 3, 6 0, 1 1))');"; + + COL_TIMESTAMP = "tmstmp"; + INSERT_TABLE_NAME_AND_TIMESTAMP_QUERY = "INSERT INTO %s (name, tmstmp) VALUES ('%s', '%s')"; + COL_TIMESTAMP_TYPE = "TIMESTAMP(0)"; } @AfterAll - public void cleanupEnvironment() throws ExecutionException, InterruptedException { - TeradataHttpClient teradataHttpClient = new TeradataHttpClient(staticConfig.get("env_host").asText()); - var request = new DeleteEnvironmentRequest(staticConfig.get("env_name").asText()); + public static void cleanupEnvironment() throws ExecutionException, InterruptedException { + final TeradataHttpClient teradataHttpClient = new TeradataHttpClient(staticConfig.get("env_host").asText()); + final var request = new DeleteEnvironmentRequest(staticConfig.get("env_name").asText()); teradataHttpClient.deleteEnvironment(request, staticConfig.get("env_token").asText()).get(); } - @BeforeEach - public void setup() throws Exception { - executeStatements(List.of( - statement -> statement.executeUpdate("CREATE DATABASE \"database_name\" AS PERMANENT = 120e6, SPOOL = 120e6;")), - staticConfig.get("host").asText(), staticConfig.get("username").asText(), staticConfig.get("password").asText()); - super.setup(); - } - - @AfterEach - public void tearDown() { + static void deleteDatabase() { executeStatements(List.of( statement -> statement.executeUpdate("DELETE DATABASE \"database_name\";"), statement -> statement.executeUpdate("DROP DATABASE \"database_name\";")), staticConfig.get("host").asText(), @@ -88,34 +79,31 @@ public void tearDown() { } @Override - public AbstractJdbcSource getSource() { - return new TeradataSource(); + protected TeradataTestDatabase createTestDatabase() { + executeStatements(List.of( + statement -> statement.executeUpdate("CREATE DATABASE \"database_name\" AS PERMANENT = 120e6, SPOOL = 120e6;")), + staticConfig.get("host").asText(), staticConfig.get("username").asText(), staticConfig.get("password").asText()); + return new TeradataTestDatabase(source().toDatabaseConfig(Jsons.clone(staticConfig))).initialized(); } @Override public boolean supportsSchemas() { - // TODO check if your db supports it and update method accordingly return false; } @Override - public JsonNode getConfig() { + public JsonNode config() { return Jsons.clone(staticConfig); } @Override - public String getDriverClass() { - return TeradataSource.DRIVER_CLASS; - } - - @Override - public AbstractJdbcSource getJdbcSource() { + protected TeradataSource source() { return new TeradataSource(); } @Override public String getFullyQualifiedTableName(String tableName) { - return "database_name." + tableName; + return staticConfig.get(JdbcUtils.DATABASE_KEY).asText() + "." + tableName; } private static void executeStatements(List consumers, String host, String username, String password) { diff --git a/airbyte-integrations/connectors/source-teradata/src/test/java/io/airbyte/integrations/source/teradata/TeradataTestDatabase.java b/airbyte-integrations/connectors/source-teradata/src/test/java/io/airbyte/integrations/source/teradata/TeradataTestDatabase.java new file mode 100644 index 000000000000..09d9eca44028 --- /dev/null +++ b/airbyte-integrations/connectors/source-teradata/src/test/java/io/airbyte/integrations/source/teradata/TeradataTestDatabase.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.teradata; + +import static io.airbyte.cdk.db.factory.DatabaseDriver.TERADATA; +import static io.airbyte.integrations.source.teradata.TeradataJdbcSourceAcceptanceTest.deleteDatabase; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.NonContainer; +import io.airbyte.cdk.testutils.TestDatabase; +import java.util.stream.Stream; +import org.jooq.SQLDialect; + +public class TeradataTestDatabase extends TestDatabase { + + private final String username; + private final String password; + private final String jdbcUrl; + private final String databaseName; + + protected TeradataTestDatabase(final JsonNode teradataConfig) { + super(new NonContainer(teradataConfig.get(JdbcUtils.USERNAME_KEY).asText(), + teradataConfig.has(JdbcUtils.PASSWORD_KEY) ? teradataConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null, + teradataConfig.get(JdbcUtils.JDBC_URL_KEY).asText(), TERADATA.getDriverClassName(), "")); + this.username = teradataConfig.get(JdbcUtils.USERNAME_KEY).asText(); + this.password = teradataConfig.has(JdbcUtils.PASSWORD_KEY) ? teradataConfig.get(JdbcUtils.PASSWORD_KEY).asText() : null; + this.jdbcUrl = teradataConfig.get(JdbcUtils.JDBC_URL_KEY).asText(); + this.databaseName = teradataConfig.get(JdbcUtils.SCHEMA_KEY).asText(); + } + + @Override + public String getDatabaseName() { + return databaseName; + } + + @Override + public String getJdbcUrl() { + return jdbcUrl; + } + + @Override + public String getPassword() { + return password; + } + + @Override + public String getUserName() { + return username; + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.empty(); + } + + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return TERADATA; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.DEFAULT; + } + + @Override + public void close() { + deleteDatabase(); + } + + static public class TeradataDbConfigBuilder extends TestDatabase.ConfigBuilder { + + protected TeradataDbConfigBuilder(final TeradataTestDatabase testdb) { + super(testdb); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-the-guardian-api/README.md b/airbyte-integrations/connectors/source-the-guardian-api/README.md index 036410fdf74d..0d36dec3168f 100644 --- a/airbyte-integrations/connectors/source-the-guardian-api/README.md +++ b/airbyte-integrations/connectors/source-the-guardian-api/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-the-guardian-api:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/the-guardian-api) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_the_guardian_api/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-the-guardian-api:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-the-guardian-api build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-the-guardian-api:airbyteDocker +An image will be built with the tag `airbyte/source-the-guardian-api:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-the-guardian-api:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-the-guardian-api:dev c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-the-guardian-api:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-the-guardian-api:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-the-guardian-api test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-the-guardian-api:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-the-guardian-api:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-the-guardian-api test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/the-guardian-api.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-the-guardian-api/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-the-guardian-api/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-the-guardian-api/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-the-guardian-api/build.gradle b/airbyte-integrations/connectors/source-the-guardian-api/build.gradle deleted file mode 100644 index f52157bc6e06..000000000000 --- a/airbyte-integrations/connectors/source-the-guardian-api/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_the_guardian_api' -} diff --git a/airbyte-integrations/connectors/source-tidb/Dockerfile b/airbyte-integrations/connectors/source-tidb/Dockerfile deleted file mode 100755 index 26069fa25844..000000000000 --- a/airbyte-integrations/connectors/source-tidb/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -### WARNING ### -# The Java connector Dockerfiles will soon be deprecated. -# This Dockerfile is not used to build the connector image we publish to DockerHub. -# The new logic to build the connector image is declared with Dagger here: -# https://github.com/airbytehq/airbyte/blob/master/tools/ci_connector_ops/ci_connector_ops/pipelines/actions/environments.py#L649 - -# If you need to add a custom logic to build your connector image, you can do it by adding a finalize_build.sh or finalize_build.py script in the connector folder. -# Please reach out to the Connectors Operations team if you have any question. -FROM airbyte/integration-base-java:dev AS build - -WORKDIR /airbyte - -ENV APPLICATION source-tidb - -COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar - -RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar - -FROM airbyte/integration-base-java:dev - -WORKDIR /airbyte - -ENV APPLICATION source-tidb - -COPY --from=build /airbyte /airbyte - -# Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile. -LABEL io.airbyte.version=0.2.5 -LABEL io.airbyte.name=airbyte/source-tidb diff --git a/airbyte-integrations/connectors/source-tidb/README.md b/airbyte-integrations/connectors/source-tidb/README.md index b1dfac6bb1b1..c8d82b58d917 100755 --- a/airbyte-integrations/connectors/source-tidb/README.md +++ b/airbyte-integrations/connectors/source-tidb/README.md @@ -21,10 +21,11 @@ Note that the `secrets` directory is git-ignored by default, so there is no dang #### Build Build the connector image via Gradle: + ``` -./gradlew :airbyte-integrations:connectors:source-tidb:airbyteDocker +./gradlew :airbyte-integrations:connectors:source-tidb:buildConnectorImage ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +Once built, the docker image name and tag on your host will be `airbyte/source-tidb:dev`. the Dockerfile. #### Run @@ -62,8 +63,11 @@ To run acceptance and custom integration tests: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-tidb test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/tidb.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-tidb/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-tidb/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-tidb/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-tidb/build.gradle b/airbyte-integrations/connectors/source-tidb/build.gradle index 6f41920bc5ca..22914cf6a720 100644 --- a/airbyte-integrations/connectors/source-tidb/build.gradle +++ b/airbyte-integrations/connectors/source-tidb/build.gradle @@ -1,38 +1,39 @@ plugins { id 'application' - id 'airbyte-docker' - id 'airbyte-integration-test-java' - id 'airbyte-connector-acceptance-test' + id 'airbyte-java-connector' } +airbyteJavaConnector { + cdkVersionRequired = '0.7.7' + features = ['db-sources'] + useLocalCdk = false +} + +//remove once upgrading the CDK version to 0.4.x or later +java { + compileTestJava { + options.compilerArgs.remove("-Werror") + } + compileJava { + options.compilerArgs.remove("-Werror") + } +} + +airbyteJavaConnector.addCdkDependencies() + application { mainClass = 'io.airbyte.integrations.source.tidb.TiDBSource' applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] } dependencies { - implementation project(':airbyte-db:db-lib') - implementation project(':airbyte-integrations:bases:base-java') - implementation libs.airbyte.protocol - implementation project(':airbyte-integrations:connectors:source-jdbc') - implementation project(':airbyte-integrations:connectors:source-relational-db') //TODO Add jdbc driver import here. Ex: implementation 'com.microsoft.sqlserver:mssql-jdbc:8.4.1.jre14' implementation 'mysql:mysql-connector-java:8.0.22' - // Add testcontainers and use GenericContainer for TiDB - implementation libs.connectors.testcontainers.tidb - - testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) + testImplementation libs.testcontainers.tidb.source testImplementation 'org.apache.commons:commons-lang3:3.11' - integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-tidb') - integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') - - integrationTestJavaImplementation libs.connectors.testcontainers.tidb - - implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) - integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + integrationTestJavaImplementation libs.testcontainers.tidb.source } - diff --git a/airbyte-integrations/connectors/source-tidb/metadata.yaml b/airbyte-integrations/connectors/source-tidb/metadata.yaml index 47a33d680ae8..fe15803049ee 100644 --- a/airbyte-integrations/connectors/source-tidb/metadata.yaml +++ b/airbyte-integrations/connectors/source-tidb/metadata.yaml @@ -6,7 +6,7 @@ data: connectorSubtype: database connectorType: source definitionId: 0dad1a35-ccf8-4d03-b73e-6788c00b13ae - dockerImageTag: 0.2.5 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-tidb githubIssueLabel: source-tidb icon: tidb.svg diff --git a/airbyte-integrations/connectors/source-tidb/src/main/java/io/airbyte/integrations/source/tidb/TiDBSource.java b/airbyte-integrations/connectors/source-tidb/src/main/java/io/airbyte/integrations/source/tidb/TiDBSource.java index ac7a151bd7f8..79413a0148a2 100644 --- a/airbyte-integrations/connectors/source-tidb/src/main/java/io/airbyte/integrations/source/tidb/TiDBSource.java +++ b/airbyte-integrations/connectors/source-tidb/src/main/java/io/airbyte/integrations/source/tidb/TiDBSource.java @@ -7,14 +7,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.mysql.cj.MysqlType; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.db.jdbc.streaming.AdaptiveStreamingQueryConfig; +import io.airbyte.cdk.integrations.base.IntegrationRunner; +import io.airbyte.cdk.integrations.base.Source; +import io.airbyte.cdk.integrations.base.ssh.SshWrappedSource; +import io.airbyte.cdk.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; -import io.airbyte.integrations.base.IntegrationRunner; -import io.airbyte.integrations.base.Source; -import io.airbyte.integrations.base.ssh.SshWrappedSource; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import java.util.List; import java.util.Set; import org.slf4j.Logger; diff --git a/airbyte-integrations/connectors/source-tidb/src/main/java/io/airbyte/integrations/source/tidb/TiDBSourceOperations.java b/airbyte-integrations/connectors/source-tidb/src/main/java/io/airbyte/integrations/source/tidb/TiDBSourceOperations.java index 9e86422b765f..4331b40230f8 100644 --- a/airbyte-integrations/connectors/source-tidb/src/main/java/io/airbyte/integrations/source/tidb/TiDBSourceOperations.java +++ b/airbyte-integrations/connectors/source-tidb/src/main/java/io/airbyte/integrations/source/tidb/TiDBSourceOperations.java @@ -5,7 +5,7 @@ package io.airbyte.integrations.source.tidb; import static com.mysql.cj.MysqlType.*; -import static io.airbyte.db.jdbc.JdbcConstants.*; +import static io.airbyte.cdk.db.jdbc.JdbcConstants.*; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.NullNode; @@ -13,9 +13,9 @@ import com.mysql.cj.MysqlType; import com.mysql.cj.jdbc.result.ResultSetMetaData; import com.mysql.cj.result.Field; -import io.airbyte.db.DataTypeUtils; -import io.airbyte.db.SourceOperations; -import io.airbyte.db.jdbc.AbstractJdbcCompatibleSourceOperations; +import io.airbyte.cdk.db.DataTypeUtils; +import io.airbyte.cdk.db.SourceOperations; +import io.airbyte.cdk.db.jdbc.AbstractJdbcCompatibleSourceOperations; import io.airbyte.protocol.models.JsonSchemaType; import java.sql.PreparedStatement; import java.sql.ResultSet; diff --git a/airbyte-integrations/connectors/source-tidb/src/test-integration/java/io/airbyte/integrations/source/tidb/TiDBSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-tidb/src/test-integration/java/io/airbyte/integrations/source/tidb/TiDBSourceAcceptanceTest.java index 56ac8e4c7917..7c402f0b86fb 100644 --- a/airbyte-integrations/connectors/source-tidb/src/test-integration/java/io/airbyte/integrations/source/tidb/TiDBSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-tidb/src/test-integration/java/io/airbyte/integrations/source/tidb/TiDBSourceAcceptanceTest.java @@ -7,15 +7,15 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import io.airbyte.cdk.db.Database; +import io.airbyte.cdk.db.factory.DSLContextFactory; +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.integrations.base.ssh.SshHelpers; +import io.airbyte.cdk.integrations.standardtest.source.SourceAcceptanceTest; +import io.airbyte.cdk.integrations.standardtest.source.TestDestinationEnv; +import io.airbyte.cdk.integrations.util.HostPortResolver; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.Database; -import io.airbyte.db.factory.DSLContextFactory; -import io.airbyte.db.factory.DatabaseDriver; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; -import io.airbyte.integrations.standardtest.source.TestDestinationEnv; -import io.airbyte.integrations.util.HostPortResolver; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; import io.airbyte.protocol.models.v0.CatalogHelpers; @@ -27,9 +27,11 @@ import java.util.HashMap; import org.jooq.DSLContext; import org.jooq.SQLDialect; +import org.junit.jupiter.api.Disabled; import org.testcontainers.containers.GenericContainer; import org.testcontainers.utility.DockerImageName; +@Disabled public class TiDBSourceAcceptanceTest extends SourceAcceptanceTest { private static final String STREAM_NAME = "id_and_name"; diff --git a/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBJdbcSourceAcceptanceTest.java index 631a87c728da..e3e2728f7aaf 100755 --- a/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBJdbcSourceAcceptanceTest.java @@ -5,50 +5,14 @@ package io.airbyte.integrations.source.tidb; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ImmutableMap; -import com.mysql.cj.MysqlType; +import io.airbyte.cdk.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; -import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; -import org.junit.jupiter.api.*; -import org.testcontainers.containers.GenericContainer; +import org.junit.jupiter.api.Disabled; +import org.testcontainers.tidb.TiDBContainer; import org.testcontainers.utility.DockerImageName; -class TiDBJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { - - protected static GenericContainer container; - protected static String USER = "root"; - protected static String DATABASE = "test"; - - @BeforeEach - public void setup() throws Exception { - container = new GenericContainer(DockerImageName.parse("pingcap/tidb:nightly")) - .withExposedPorts(4000); - container.start(); - - config = Jsons.jsonNode(ImmutableMap.builder() - .put(JdbcUtils.HOST_KEY, "127.0.0.1") - .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) - .put(JdbcUtils.USERNAME_KEY, USER) - .put(JdbcUtils.DATABASE_KEY, DATABASE) - // .put(JdbcUtils.SSL_KEY, true) - .build()); - - super.setup(); - } - - @AfterEach - void tearDownTiDB() throws Exception { - container.close(); - container.stop(); - super.tearDown(); - } - - @Override - public AbstractJdbcSource getSource() { - return new TiDBSource(); - } +@Disabled +class TiDBJdbcSourceAcceptanceTest extends JdbcSourceAcceptanceTest { @Override public boolean supportsSchemas() { @@ -56,23 +20,21 @@ public boolean supportsSchemas() { } @Override - public JsonNode getConfig() { - return Jsons.clone(config); + public JsonNode config() { + return Jsons.clone(testdb.configBuilder().build()); } @Override - public String getDriverClass() { - return TiDBSource.DRIVER_CLASS; - } - - @Override - public AbstractJdbcSource getJdbcSource() { + protected TiDBSource source() { return new TiDBSource(); } - @AfterAll - static void cleanUp() { - container.close(); + @Override + protected TiDBTestDatabase createTestDatabase() { + final TiDBContainer container = new TiDBContainer(DockerImageName.parse("pingcap/tidb:nightly")) + .withExposedPorts(4000); + container.start(); + return new TiDBTestDatabase(container).initialized(); } } diff --git a/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBSourceTests.java b/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBSourceTests.java index 5e96bf7c27c8..d2ceb40e6bc4 100644 --- a/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBSourceTests.java +++ b/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBSourceTests.java @@ -8,13 +8,15 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import io.airbyte.cdk.db.jdbc.JdbcUtils; import io.airbyte.commons.json.Jsons; -import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.protocol.models.v0.AirbyteConnectionStatus; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.testcontainers.containers.GenericContainer; import org.testcontainers.utility.DockerImageName; +@Disabled public class TiDBSourceTests { private JsonNode config; diff --git a/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBTestDatabase.java b/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBTestDatabase.java new file mode 100644 index 000000000000..26da58c7eff1 --- /dev/null +++ b/airbyte-integrations/connectors/source-tidb/src/test/java/io/airbyte/integrations/source/tidb/TiDBTestDatabase.java @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.tidb; + +import io.airbyte.cdk.db.factory.DatabaseDriver; +import io.airbyte.cdk.db.jdbc.JdbcUtils; +import io.airbyte.cdk.testutils.TestDatabase; +import java.util.stream.Stream; +import org.jooq.SQLDialect; +import org.testcontainers.tidb.TiDBContainer; + +public class TiDBTestDatabase extends + TestDatabase { + + protected static final String USER = "root"; + protected static final String DATABASE = "test"; + private final TiDBContainer container; + + protected TiDBTestDatabase(final TiDBContainer container) { + super(container); + this.container = container; + } + + @Override + public String getJdbcUrl() { + return container.getJdbcUrl(); + } + + @Override + public String getDatabaseName() { + return DATABASE; + } + + @Override + public String getUserName() { + return container.getUsername(); + } + + @Override + public String getPassword() { + return container.getPassword(); + } + + @Override + protected Stream> inContainerBootstrapCmd() { + return Stream.empty(); + } + + @Override + protected Stream inContainerUndoBootstrapCmd() { + return Stream.empty(); + } + + @Override + public DatabaseDriver getDatabaseDriver() { + return DatabaseDriver.MYSQL; + } + + @Override + public SQLDialect getSqlDialect() { + return SQLDialect.MYSQL; + } + + @Override + public void close() { + container.close(); + } + + @Override + public TiDBDbConfigBuilder configBuilder() { + return new TiDBDbConfigBuilder(this) + .with(JdbcUtils.HOST_KEY, "127.0.0.1") + .with(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) + .with(JdbcUtils.USERNAME_KEY, USER) + .with(JdbcUtils.DATABASE_KEY, DATABASE); + } + + static public class TiDBDbConfigBuilder extends TestDatabase.ConfigBuilder { + + protected TiDBDbConfigBuilder(final TiDBTestDatabase testdb) { + super(testdb); + } + + } + +} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile b/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile deleted file mode 100644 index 4b62a8e5fe52..000000000000 --- a/airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone -# Bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_tiktok_marketing ./source_tiktok_marketing - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=3.4.1 -LABEL io.airbyte.name=airbyte/source-tiktok-marketing diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/README.md b/airbyte-integrations/connectors/source-tiktok-marketing/README.md index 24d0b76f17ca..f787ae62f943 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/README.md +++ b/airbyte-integrations/connectors/source-tiktok-marketing/README.md @@ -29,12 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-tiktok-marketing:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/tiktok-marketing) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_tiktok_marketing/spec.json` file. @@ -54,19 +48,70 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-tiktok-marketing:dev + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-tiktok-marketing build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-tiktok-marketing:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations -You can also build the connector image via Gradle: +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-tiktok-marketing:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-tiktok-marketing:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-tiktok-marketing:dev . +# Running the spec command against your patched connector +docker run airbyte/source-tiktok-marketing:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -75,45 +120,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-tiktok-marketing:dev c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-tiktok-marketing:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-tiktok-marketing:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .'[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-tiktok-marketing test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-tiktok-marketing:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-tiktok-marketing:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-tiktok-marketing:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -123,8 +139,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-tiktok-marketing test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/tiktok-marketing.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-config.yml b/airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-config.yml index 49ecf788ab14..f1be50fe8cff 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-config.yml @@ -6,102 +6,136 @@ test_strictness_level: "high" acceptance_tests: spec: tests: - - spec_path: "source_tiktok_marketing/spec.json" - # the spec was changed with the `format: date` for start/end dates input, - # to satisfy the DatePicker requirements - backward_compatibility_tests_config: - disable_for_version: "3.1.0" # attribution windows: add min/max values; change default 0 -> 3 + - spec_path: "source_tiktok_marketing/spec.json" + # the spec was changed with the `format: date` for start/end dates input, + # to satisfy the DatePicker requirements + backward_compatibility_tests_config: + disable_for_version: "3.1.0" # attribution windows: add min/max values; change default 0 -> 3 connection: tests: - - config_path: "secrets/prod_config.json" - status: "succeed" - - config_path: "secrets/prod_config_with_day_granularity.json" - status: "succeed" - - config_path: "secrets/prod_config_with_lifetime_granularity.json" - status: "succeed" - - config_path: "secrets/new_config_prod.json" - status: "succeed" - - config_path: "secrets/config_oauth.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" - - config_path: "integration_tests/invalid_config_access_token.json" - status: "failed" - - config_path: "integration_tests/invalid_config_oauth.json" - status: "failed" + - config_path: "secrets/prod_config.json" + status: "succeed" + - config_path: "secrets/prod_config_with_day_granularity.json" + status: "succeed" + - config_path: "secrets/prod_config_with_lifetime_granularity.json" + status: "succeed" + - config_path: "secrets/new_config_prod.json" + status: "succeed" + - config_path: "secrets/config_oauth.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + - config_path: "integration_tests/invalid_config_access_token.json" + status: "failed" + - config_path: "integration_tests/invalid_config_oauth.json" + status: "failed" discovery: tests: - - config_path: "secrets/prod_config.json" - - config_path: "secrets/prod_config_with_day_granularity.json" - - config_path: "secrets/prod_config_with_lifetime_granularity.json" - - config_path: "secrets/config.json" - - config_path: "secrets/new_config_prod.json" - - config_path: "secrets/config_oauth.json" + - config_path: "secrets/prod_config.json" + - config_path: "secrets/prod_config_with_day_granularity.json" + - config_path: "secrets/prod_config_with_lifetime_granularity.json" + - config_path: "secrets/config.json" + - config_path: "secrets/new_config_prod.json" + - config_path: "secrets/config_oauth.json" basic_read: tests: - - config_path: "secrets/config.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: no - extra_records: yes - timeout_seconds: 1200 - empty_streams: - - name: ads_reports_hourly - bypass_reason: "Tested with daily granularity." - - name: ad_groups_reports_hourly - bypass_reason: "Tested with daily granularity." - - name: advertisers_reports_hourly - bypass_reason: "Tested with daily granularity." - - name: campaigns_reports_hourly - bypass_reason: "Tested with daily granularity." - - name: campaigns_audience_reports_daily - bypass_reason: "Stream is added by the Contributor. We should seed the sandbox later on." - - name: advertisers_audience_reports_by_country_daily - bypass_reason: "Stream is added by the Contributor. We should seed the sandbox later on." - - name: ads_audience_reports_by_country_daily - bypass_reason: "Stream is added by the Contributor. We should seed the sandbox later on." - - name: campaigns_audience_reports_by_platform_daily - bypass_reason: "Stream is added by the Contributor. We should seed the sandbox later on." - - name: ad_group_audience_reports_by_platform_daily - bypass_reason: "Stream is added by the Contributor. We should seed the sandbox later on." - - name: ad_group_audience_reports_by_country_daily - bypass_reason: "Stream is added by the Contributor. We should seed the sandbox later on." - - name: advertisers_audience_reports_by_platform_daily - bypass_reason: "Stream is added by the Contributor. We should seed the sandbox later on." - - name: ads_audience_reports_by_platform_daily - bypass_reason: "Stream is added by the Contributor. We should seed the sandbox later on." - # Old style streams with granularity config (for < 0.1.13) - # Note: not needed to be tested separately in full and incremental tests, because code of - # these streams is called directly in new style streams - - config_path: "secrets/prod_config_with_day_granularity.json" - expect_records: - path: "integration_tests/expected_records2.jsonl" - extra_fields: no - exact_order: no - extra_records: yes - timeout_seconds: 1200 - empty_streams: - - name: ad_groups - bypass_reason: "This stream is tested on the new style config." - - name: ads - bypass_reason: "This stream is tested on the new style config." - - name: advertiser_ids - bypass_reason: "This stream is tested on the new style config." - - name: advertisers - bypass_reason: "This stream is tested on the new style config." - - name: campaigns - bypass_reason: "This stream is tested on the new style config." + - config_path: "secrets/config.json" + expect_records: + path: "integration_tests/expected_records.jsonl" + extra_fields: no + exact_order: no + extra_records: yes + timeout_seconds: 1200 + ignored_fields: + ads: + - name: "profile_image_url" + bypass_reason: "Volatile URLs" + creative_assets_images: + - name: "image_url" + bypass_reason: "Volatile URL params" + creative_assets_videos: + - name: "preview_url" + bypass_reason: "Volatile URL params" + - name: "video_cover_url" + bypass_reason: "Volatile URL params" + - name: "preview_url_expire_time" + bypass_reason: "Changes over time" + empty_streams: + - name: ads_reports_hourly + bypass_reason: "Tested with daily granularity." + - name: ad_groups_reports_hourly + bypass_reason: "Tested with daily granularity." + - name: advertisers_reports_hourly + bypass_reason: "Tested with daily granularity." + - name: campaigns_reports_hourly + bypass_reason: "Tested with daily granularity." + - name: creative_assets_portfolios + bypass_reason: "No data in the integration test account. We should seed the sandbox later on." + - name: creative_assets_music + bypass_reason: "System music provided by TikTok - very volatile data." + # Old style streams with granularity config (for < 0.1.13) + # Note: not needed to be tested separately in full and incremental tests, because code of + # these streams is called directly in new style streams + - config_path: "secrets/prod_config_with_day_granularity.json" + expect_records: + path: "integration_tests/expected_records2.jsonl" + extra_fields: no + exact_order: no + extra_records: yes + timeout_seconds: 1200 + ignored_fields: + ads: + - name: "profile_image_url" + bypass_reason: "Volatile URLs" + creative_assets_images: + - name: "image_url" + bypass_reason: "Volatile URL params" + creative_assets_videos: + - name: "preview_url" + bypass_reason: "Volatile URL params" + - name: "video_cover_url" + bypass_reason: "Volatile URL params" + - name: "preview_url_expire_time" + bypass_reason: "Changes over time" + empty_streams: + - name: ad_groups + bypass_reason: "This stream is tested on the new style config." + - name: ads + bypass_reason: "This stream is tested on the new style config." + - name: advertiser_ids + bypass_reason: "This stream is tested on the new style config." + - name: advertisers + bypass_reason: "This stream is tested on the new style config." + - name: campaigns + bypass_reason: "This stream is tested on the new style config." + - name: creative_assets_portfolios + bypass_reason: "No data in the integration test account. We should seed the sandbox later on." + - name: creative_assets_music + bypass_reason: "System music provided by TikTok - very volatile data." full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 7200 + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 7200 + ignored_fields: + ads: + - name: "profile_image_url" + bypass_reason: "Volatile URLs" + creative_assets_images: + - name: "image_url" + bypass_reason: "Volatile URL params" + creative_assets_videos: + - name: "preview_url" + bypass_reason: "Volatile URL params" + - name: "video_cover_url" + bypass_reason: "Volatile URL params" + - name: "preview_url_expire_time" + bypass_reason: "Changes over time" incremental: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 7200 - skip_comprehensive_incremental_tests: true - future_state: - future_state_path: "integration_tests/abnormal_state.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 7200 + skip_comprehensive_incremental_tests: true + future_state: + future_state_path: "integration_tests/abnormal_state.json" diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-tiktok-marketing/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/build.gradle b/airbyte-integrations/connectors/source-tiktok-marketing/build.gradle deleted file mode 100644 index 186dbde7c8cd..000000000000 --- a/airbyte-integrations/connectors/source-tiktok-marketing/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_tiktok_marketing_singer' -} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state.json index 1ddb68e19891..38b317323132 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/abnormal_state.json @@ -10,6 +10,17 @@ } } }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "ads_audience_reports_by_province_daily" + }, + "stream_state": { + "stat_time_day": "2099-01-01" + } + } + }, { "type": "STREAM", "stream": { @@ -119,5 +130,115 @@ "modify_time": "2099-01-01 01:00:0" } } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "creative_assets_images" + }, + "stream_state": { + "modify_time": "2099-01-01 01:00:0" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "creative_assets_videos" + }, + "stream_state": { + "modify_time": "2099-01-01 01:00:0" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "ad_group_audience_reports_by_platform_daily" + }, + "stream_state": { + "stat_time_day": "2099-01-01" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "campaigns_audience_reports_daily" + }, + "stream_state": { + "stat_time_day": "2099-01-01" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "campaigns_audience_reports_by_platform_daily" + }, + "stream_state": { + "stat_time_day": "2099-01-01" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "ad_group_audience_reports_by_country_daily" + }, + "stream_state": { + "stat_time_day": "2099-01-01" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "advertisers_audience_reports_by_platform_daily" + }, + "stream_state": { + "stat_time_day": "2099-01-01" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "ads_audience_reports_by_country_daily" + }, + "stream_state": { + "stat_time_day": "2099-01-01" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "ads_audience_reports_by_platform_daily" + }, + "stream_state": { + "stat_time_day": "2099-01-01" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "advertisers_audience_reports_by_country_daily" + }, + "stream_state": { + "stat_time_day": "2099-01-01" + } + } } ] diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/configured_catalog.json index 90acbeb472d2..468bcccf3b4b 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/configured_catalog.json @@ -147,6 +147,18 @@ "cursor_field": ["stat_time_day"], "primary_key": [["adgroup_id"], ["stat_time_day"], ["gender"], ["age"]] }, + { + "stream": { + "name": "ads_audience_reports_by_province_daily", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["ad_id"], ["province_id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "cursor_field": ["stat_time_day"], + "primary_key": [["ad_id"], ["province_id"]] + }, { "stream": { "name": "campaigns_audience_reports_by_country_daily", @@ -183,6 +195,201 @@ "destination_sync_mode": "overwrite", "cursor_field": ["stat_time_day"], "primary_key": [["advertiser_id"], ["stat_time_day"], ["gender"], ["age"]] + }, + { + "stream": { + "name": "audiences", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["audience_id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "primary_key": [["audience_id"]] + }, + { + "stream": { + "name": "creative_assets_portfolios", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["creative_portfolio_id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "primary_key": [["creative_portfolio_id"]] + }, + { + "stream": { + "name": "creative_assets_images", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["modify_time"], + "source_defined_primary_key": [["image_id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "cursor_field": ["modify_time"], + "primary_key": [["image_id"]] + }, + { + "stream": { + "name": "creative_assets_videos", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["modify_time"], + "source_defined_primary_key": [["video_id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite", + "cursor_field": ["modify_time"], + "primary_key": [["video_id"]] + }, + { + "stream": { + "name": "campaigns_audience_reports_daily", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [ + ["campaign_id"], + ["stat_time_day"], + ["gender"], + ["age"] + ] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["stat_time_day"], + "primary_key": [["campaign_id"], ["stat_time_day"], ["gender"], ["age"]] + }, + { + "stream": { + "name": "advertisers_audience_reports_by_country_daily", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [ + ["advertiser_id"], + ["stat_time_day"], + ["country_code"] + ] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["stat_time_day"], + "primary_key": [["advertiser_id"], ["stat_time_day"], ["country_code"]] + }, + { + "stream": { + "name": "ads_audience_reports_by_country_daily", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [ + ["ad_id"], + ["stat_time_day"], + ["country_code"] + ] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["stat_time_day"], + "primary_key": [["ad_id"], ["stat_time_day"], ["country_code"]] + }, + { + "stream": { + "name": "campaigns_audience_reports_by_platform_daily", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [ + ["campaign_id"], + ["stat_time_day"], + ["platform"] + ] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["stat_time_day"], + "primary_key": [["campaign_id"], ["stat_time_day"], ["platform"]] + }, + { + "stream": { + "name": "ad_group_audience_reports_by_platform_daily", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [ + ["adgroup_id"], + ["stat_time_day"], + ["platform"] + ] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["stat_time_day"], + "primary_key": [["adgroup_id"], ["stat_time_day"], ["platform"]] + }, + { + "stream": { + "name": "ad_group_audience_reports_by_country_daily", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [ + ["adgroup_id"], + ["stat_time_day"], + ["country_code"] + ] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["stat_time_day"], + "primary_key": [["adgroup_id"], ["stat_time_day"], ["country_code"]] + }, + { + "stream": { + "name": "advertisers_audience_reports_by_platform_daily", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [ + ["advertiser_id"], + ["stat_time_day"], + ["platform"] + ] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["stat_time_day"], + "primary_key": [["advertiser_id"], ["stat_time_day"], ["platform"]] + }, + { + "stream": { + "name": "ads_audience_reports_by_platform_daily", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["stat_time_day"], + "source_defined_primary_key": [ + ["ad_id"], + ["stat_time_day"], + ["platform"] + ] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["stat_time_day"], + "primary_key": [["ad_id"], ["stat_time_day"], ["platform"]] } ] } diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records.jsonl index dc6fbcbf35a0..0efab7420dd8 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records.jsonl @@ -1,50 +1,86 @@ -{"stream": "advertisers", "data": {"description": "https://", "contacter": "Ai***te", "license_city": null, "timezone": "Etc/GMT+8", "promotion_center_province": null, "address": "350 29th avenue, San Francisco", "country": "US", "brand": null, "status": "STATUS_ENABLE", "role": "ROLE_ADVERTISER", "rejection_reason": "", "email": "i***************@**********", "license_province": null, "industry": "291905", "license_no": "", "name": "Airbyte0830", "create_time": 1630335591, "promotion_area": "0", "advertiser_account_type": "AUCTION", "cellphone_number": "+13477****53", "company": "Airbyte", "advertiser_id": 7002238017842757633, "promotion_center_city": null, "telephone_number": "+14156****85", "display_timezone": "America/Los_Angeles", "license_url": null, "currency": "USD", "language": "en", "balance": 10}, "emitted_at": 1691143342127} -{"stream": "ads", "data": {"app_name": "", "fallback_type": "UNSET", "advertiser_id": 7002238017842757633, "landing_page_urls": null, "brand_safety_postbid_partner": "UNSET", "campaign_id": 1728545382536225, "adgroup_id": 1728545385226289, "campaign_name": "CampaignVadimTraffic", "page_id": null, "avatar_icon_web_uri": "ad-site-i18n-sg/202203285d0de5c114d0690a462bb6a4", "ad_texts": null, "create_time": "2022-03-28 12:09:09", "profile_image_url": "https://p21-ad-sg.ibyteimg.com/large/ad-site-i18n-sg/202203285d0de5c114d0690a462bb6a4", "playable_url": "", "click_tracking_url": null, "is_aco": false, "impression_tracking_url": null, "call_to_action_id": "7080120957230238722", "modify_time": "2022-03-28 21:34:26", "image_ids": ["v0201/7f371ff6f0764f8b8ef4f37d7b980d50"], "operation_status": "ENABLE", "viewability_postbid_partner": "UNSET", "identity_type": "CUSTOMIZED_USER", "creative_type": null, "deeplink": "", "adgroup_name": "AdGroupVadim", "ad_name": "AdVadim-Optimized Version 3_202203281449_2022-03-28 05:03:44", "display_name": "airbyte", "viewability_vast_url": null, "brand_safety_vast_url": null, "identity_id": "7080121820963422209", "secondary_status": "AD_STATUS_CAMPAIGN_DISABLE", "deeplink_type": "NORMAL", "creative_authorized": false, "optimization_event": null, "video_id": "v10033g50000c90q1d3c77ub6e96fvo0", "ad_format": "SINGLE_VIDEO", "ad_id": 1728545390695442, "ad_text": "Open-source\ndata integration for modern data teams", "card_id": null, "landing_page_url": "https://airbyte.com", "vast_moat_enabled": false, "music_id": null, "is_new_structure": true}, "emitted_at": 1691143343208} -{"stream": "ads", "data": {"app_name": "", "fallback_type": "UNSET", "advertiser_id": 7002238017842757633, "landing_page_urls": null, "brand_safety_postbid_partner": "UNSET", "campaign_id": 1714125042508817, "adgroup_id": 1714125049901106, "campaign_name": "Website Traffic20211020010104", "page_id": null, "avatar_icon_web_uri": "ad-site-i18n-sg/202110205d0d488b68ead898460bad74", "ad_texts": null, "create_time": "2021-10-20 08:04:06", "profile_image_url": "https://p21-ad-sg.ibyteimg.com/large/ad-site-i18n-sg/202110205d0d488b68ead898460bad74", "playable_url": "", "click_tracking_url": null, "is_aco": false, "impression_tracking_url": null, "call_to_action_id": null, "modify_time": "2021-10-21 17:50:11", "image_ids": ["v0201/8f77082a1f3c40c586f8282356490c58"], "call_to_action": "LEARN_MORE", "operation_status": "ENABLE", "viewability_postbid_partner": "UNSET", "identity_type": "UNSET", "creative_type": null, "deeplink": "", "adgroup_name": "Ad Group20211020010107", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "display_name": "Airbyte", "viewability_vast_url": null, "brand_safety_vast_url": null, "identity_id": "", "secondary_status": "AD_STATUS_CAMPAIGN_DISABLE", "deeplink_type": "NORMAL", "creative_authorized": true, "optimization_event": null, "video_id": "v10033g50000c5nsqcbc77ubdn136b70", "ad_format": "SINGLE_VIDEO", "ad_id": 1714125051115569, "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "card_id": null, "landing_page_url": "https://airbyte.io", "vast_moat_enabled": false, "music_id": null, "is_new_structure": true}, "emitted_at": 1691143343209} -{"stream": "ads", "data": {"app_name": "", "fallback_type": "UNSET", "advertiser_id": 7002238017842757633, "landing_page_urls": null, "brand_safety_postbid_partner": "UNSET", "campaign_id": 1714124576938033, "adgroup_id": 1714124588896305, "campaign_name": "Website Traffic20211020005342", "page_id": null, "avatar_icon_web_uri": "ad-site-i18n-sg/202110205d0d4ce78859d72d409eb82d", "ad_texts": null, "create_time": "2021-10-20 07:56:39", "profile_image_url": "https://p21-ad-sg.ibyteimg.com/large/ad-site-i18n-sg/202110205d0d4ce78859d72d409eb82d", "playable_url": "", "click_tracking_url": null, "is_aco": false, "impression_tracking_url": null, "call_to_action_id": null, "modify_time": "2021-10-20 08:05:12", "image_ids": ["ad-site-i18n-sg/202110195d0db51e12a222ee4334a396"], "call_to_action": "LEARN_MORE", "operation_status": "DISABLE", "viewability_postbid_partner": "UNSET", "identity_type": "UNSET", "creative_type": null, "deeplink": "", "adgroup_name": "Ad Group20211020005346", "ad_name": "1200x1200 logo_1634667070143.png_2021-10-20 10:54:51", "display_name": "Airbyte", "viewability_vast_url": null, "brand_safety_vast_url": null, "identity_id": "", "secondary_status": "AD_STATUS_CAMPAIGN_DISABLE", "deeplink_type": "NORMAL", "creative_authorized": false, "optimization_event": null, "video_id": null, "ad_format": "SINGLE_IMAGE", "ad_id": 1714124564763650, "ad_text": "Airbyte - open source data portability platform - from nywhere to anywhere!", "card_id": null, "landing_page_url": "https://airbyte.io", "vast_moat_enabled": false, "music_id": null, "is_new_structure": true}, "emitted_at": 1691143343210} -{"stream": "ad_groups", "data": {"secondary_optimization_event": null, "operating_systems": [], "interest_keyword_ids": [], "campaign_name": "CampaignVadimTraffic", "pixel_id": null, "device_price_ranges": [], "excluded_custom_actions": [], "skip_learning_phase": 0, "delivery_mode": null, "pacing": "PACING_MODE_SMOOTH", "languages": [], "age_groups": ["AGE_25_34", "AGE_35_44"], "adgroup_app_profile_page_state": null, "search_result_enabled": false, "placement_type": "PLACEMENT_TYPE_AUTOMATIC", "video_download_disabled": false, "billing_event": "CPC", "app_download_url": null, "conversion_window": null, "optimization_goal": "CLICK", "adgroup_id": 1728545385226289, "excluded_audience_ids": [], "auto_targeting_enabled": false, "gender": "GENDER_UNLIMITED", "is_new_structure": true, "brand_safety_type": "NO_BRAND_SAFETY", "adgroup_name": "AdGroupVadim", "budget": 20, "schedule_end_time": "2032-03-25 13:02:23", "statistic_type": null, "schedule_start_time": "2022-03-28 13:02:23", "schedule_type": "SCHEDULE_FROM_NOW", "category_exclusion_ids": [], "operation_status": "ENABLE", "rf_estimated_cpr": null, "network_types": [], "deep_cpa_bid": 0, "frequency_schedule": null, "bid_display_mode": "CPMV", "inventory_filter_enabled": false, "next_day_retention": null, "rf_estimated_frequency": null, "share_disabled": false, "bid_price": 0, "keywords": null, "promotion_type": "WEBSITE", "comment_disabled": false, "budget_mode": "BUDGET_MODE_DAY", "creative_material_mode": "CUSTOM", "category_id": 0, "deep_bid_type": null, "dayparting": "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", "secondary_status": "ADGROUP_STATUS_CAMPAIGN_DISABLE", "optimization_event": null, "location_ids": [6252001], "actions": [], "app_type": null, "brand_safety_partner": null, "bid_type": "BID_TYPE_NO_BID", "placements": ["PLACEMENT_TIKTOK", "PLACEMENT_TOPBUZZ", "PLACEMENT_PANGLE"], "campaign_id": 1728545382536225, "modify_time": "2022-03-31 08:13:30", "rf_purchased_type": null, "audience_ids": [], "advertiser_id": 7002238017842757633, "conversion_bid_price": 0, "app_id": null, "is_hfss": false, "feed_type": null, "device_model_ids": [], "schedule_infos": null, "included_custom_actions": [], "ios14_quota_type": "UNOCCUPIED", "scheduled_budget": 0, "frequency": null, "is_smart_performance_campaign": false, "create_time": "2022-03-28 12:09:07", "interest_category_ids": [15], "purchased_impression": null, "purchased_reach": null}, "emitted_at": 1691143344341} -{"stream": "ad_groups", "data": {"secondary_optimization_event": null, "operating_systems": [], "interest_keyword_ids": [], "campaign_name": "Website Traffic20211020010104", "pixel_id": null, "device_price_ranges": [], "excluded_custom_actions": [], "skip_learning_phase": 0, "delivery_mode": null, "pacing": "PACING_MODE_SMOOTH", "languages": ["en"], "age_groups": ["AGE_25_34", "AGE_35_44", "AGE_45_54"], "adgroup_app_profile_page_state": null, "search_result_enabled": false, "placement_type": "PLACEMENT_TYPE_AUTOMATIC", "video_download_disabled": false, "billing_event": "CPC", "app_download_url": null, "conversion_window": null, "optimization_goal": "CLICK", "adgroup_id": 1714125049901106, "excluded_audience_ids": [], "auto_targeting_enabled": false, "gender": "GENDER_UNLIMITED", "is_new_structure": true, "brand_safety_type": "NO_BRAND_SAFETY", "adgroup_name": "Ad Group20211020010107", "budget": 20, "schedule_end_time": "2021-10-31 09:01:07", "statistic_type": null, "schedule_start_time": "2021-10-20 09:01:07", "schedule_type": "SCHEDULE_START_END", "category_exclusion_ids": [], "operation_status": "ENABLE", "rf_estimated_cpr": null, "network_types": [], "deep_cpa_bid": 0, "frequency_schedule": null, "bid_display_mode": "CPMV", "inventory_filter_enabled": false, "next_day_retention": null, "rf_estimated_frequency": null, "share_disabled": false, "bid_price": 0, "keywords": null, "promotion_type": "WEBSITE", "comment_disabled": false, "budget_mode": "BUDGET_MODE_DAY", "creative_material_mode": "CUSTOM", "category_id": 0, "deep_bid_type": null, "dayparting": "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", "secondary_status": "ADGROUP_STATUS_CAMPAIGN_DISABLE", "optimization_event": null, "location_ids": [6252001], "actions": [], "app_type": null, "brand_safety_partner": null, "bid_type": "BID_TYPE_NO_BID", "placements": ["PLACEMENT_TIKTOK", "PLACEMENT_TOPBUZZ", "PLACEMENT_PANGLE"], "campaign_id": 1714125042508817, "modify_time": "2022-03-24 12:06:54", "rf_purchased_type": null, "audience_ids": [], "advertiser_id": 7002238017842757633, "conversion_bid_price": 0, "app_id": null, "is_hfss": false, "feed_type": null, "device_model_ids": [], "schedule_infos": null, "included_custom_actions": [], "ios14_quota_type": "UNOCCUPIED", "scheduled_budget": 0, "frequency": null, "is_smart_performance_campaign": false, "create_time": "2021-10-20 08:04:05", "interest_category_ids": [], "purchased_impression": null, "purchased_reach": null}, "emitted_at": 1691143344343} -{"stream": "ad_groups", "data": {"secondary_optimization_event": null, "operating_systems": [], "interest_keyword_ids": [], "campaign_name": "Website Traffic20211020005342", "pixel_id": null, "device_price_ranges": [], "excluded_custom_actions": [], "skip_learning_phase": 0, "delivery_mode": null, "pacing": "PACING_MODE_SMOOTH", "languages": [], "age_groups": null, "adgroup_app_profile_page_state": null, "search_result_enabled": false, "placement_type": "PLACEMENT_TYPE_AUTOMATIC", "video_download_disabled": false, "billing_event": "CPC", "app_download_url": null, "conversion_window": null, "optimization_goal": "CLICK", "adgroup_id": 1714124588896305, "excluded_audience_ids": [], "auto_targeting_enabled": false, "gender": "GENDER_UNLIMITED", "is_new_structure": true, "brand_safety_type": "NO_BRAND_SAFETY", "adgroup_name": "Ad Group20211020005346", "budget": 20, "schedule_end_time": "2021-10-31 08:53:46", "statistic_type": null, "schedule_start_time": "2021-10-20 08:53:46", "schedule_type": "SCHEDULE_START_END", "category_exclusion_ids": [], "operation_status": "DISABLE", "rf_estimated_cpr": null, "network_types": [], "deep_cpa_bid": 0, "frequency_schedule": null, "bid_display_mode": "CPMV", "inventory_filter_enabled": false, "next_day_retention": null, "rf_estimated_frequency": null, "share_disabled": false, "bid_price": 0, "keywords": null, "promotion_type": "WEBSITE", "comment_disabled": false, "budget_mode": "BUDGET_MODE_DAY", "creative_material_mode": "CUSTOM", "category_id": 0, "deep_bid_type": null, "dayparting": "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", "secondary_status": "ADGROUP_STATUS_CAMPAIGN_DISABLE", "optimization_event": null, "location_ids": [6252001], "actions": [], "app_type": null, "brand_safety_partner": null, "bid_type": "BID_TYPE_NO_BID", "placements": ["PLACEMENT_TIKTOK", "PLACEMENT_TOPBUZZ", "PLACEMENT_HELO", "PLACEMENT_PANGLE"], "campaign_id": 1714124576938033, "modify_time": "2021-10-20 08:08:14", "rf_purchased_type": null, "audience_ids": [], "advertiser_id": 7002238017842757633, "conversion_bid_price": 0, "app_id": null, "is_hfss": false, "feed_type": null, "device_model_ids": [], "schedule_infos": null, "included_custom_actions": [], "ios14_quota_type": "UNOCCUPIED", "scheduled_budget": 0, "frequency": null, "is_smart_performance_campaign": false, "create_time": "2021-10-20 07:56:39", "interest_category_ids": [], "purchased_impression": null, "purchased_reach": null}, "emitted_at": 1691143344345} -{"stream": "campaigns", "data": {"budget": 0, "campaign_type": "REGULAR_CAMPAIGN", "is_smart_performance_campaign": false, "budget_mode": "BUDGET_MODE_INFINITE", "campaign_name": "CampaignVadimTraffic", "deep_bid_type": null, "objective": "LANDING_PAGE", "operation_status": "DISABLE", "objective_type": "TRAFFIC", "modify_time": "2022-03-30 21:23:52", "create_time": "2022-03-28 12:09:05", "roas_bid": 0, "advertiser_id": 7002238017842757633, "secondary_status": "CAMPAIGN_STATUS_DISABLE", "campaign_id": 1728545382536225, "is_new_structure": true}, "emitted_at": 1691143345193} -{"stream": "campaigns", "data": {"budget": 0, "campaign_type": "REGULAR_CAMPAIGN", "is_smart_performance_campaign": false, "budget_mode": "BUDGET_MODE_INFINITE", "campaign_name": "Website Traffic20211020010104", "deep_bid_type": null, "objective": "LANDING_PAGE", "operation_status": "DISABLE", "objective_type": "TRAFFIC", "modify_time": "2022-03-24 12:08:29", "create_time": "2021-10-20 08:04:04", "roas_bid": 0, "advertiser_id": 7002238017842757633, "secondary_status": "CAMPAIGN_STATUS_DISABLE", "campaign_id": 1714125042508817, "is_new_structure": true}, "emitted_at": 1691143345193} -{"stream": "campaigns", "data": {"budget": 0, "campaign_type": "REGULAR_CAMPAIGN", "is_smart_performance_campaign": false, "budget_mode": "BUDGET_MODE_INFINITE", "campaign_name": "Website Traffic20211020005342", "deep_bid_type": null, "objective": "LANDING_PAGE", "operation_status": "DISABLE", "objective_type": "TRAFFIC", "modify_time": "2021-10-20 08:01:18", "create_time": "2021-10-20 07:56:38", "roas_bid": 0, "advertiser_id": 7002238017842757633, "secondary_status": "CAMPAIGN_STATUS_DISABLE", "campaign_id": 1714124576938033, "is_new_structure": true}, "emitted_at": 1691143345194} -{"stream": "advertiser_ids", "data": {"advertiser_id": 7001035076276387841, "advertiser_name": "Airbyte0827"}, "emitted_at": 1691143345771} -{"stream": "advertiser_ids", "data": {"advertiser_id": 7001040009704833026, "advertiser_name": "Airbyte08270"}, "emitted_at": 1691143345772} -{"stream": "advertiser_ids", "data": {"advertiser_id": 7002238017842757633, "advertiser_name": "Airbyte0830"}, "emitted_at": 1691143345772} -{"stream": "ads_reports_daily", "data": {"metrics": {"mobile_app_id": "0", "result": "69", "follows": "0", "cost_per_secondary_goal_result": null, "ctr": "1.18", "promotion_type": "Website", "clicks_on_music_disc": "0", "real_time_conversion": "0", "real_time_result": "69", "total_complete_payment_rate": "0.000", "frequency": "1.21", "placement_type": "Automatic Placement", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "shares": "0", "video_views_p75": "140", "spend": "20.000", "cost_per_1000_reached": "4.161", "likes": "36", "video_watched_2s": "686", "video_views_p50": "214", "complete_payment": "0", "cpc": "0.290", "vta_purchase": "0", "real_time_result_rate": "1.18", "real_time_conversion_rate": "0.00", "comments": "0", "conversion": "0", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "average_video_play": "1.52", "onsite_shopping": "0", "campaign_id": 1714125042508817, "cta_purchase": "0", "video_views_p25": "513", "video_watched_6s": "180", "conversion_rate": "0.00", "cost_per_conversion": "0.000", "cpm": "3.430", "secondary_goal_result_rate": null, "impressions": "5830", "video_views_p100": "92", "total_purchase_value": "0.000", "vta_conversion": "0", "reach": "4806", "adgroup_id": 1714125049901106, "profile_visits": "0", "dpa_target_audience_type": null, "clicks": "69", "real_time_app_install_cost": "0.000", "real_time_cost_per_result": "0.2899", "value_per_complete_payment": "0.000", "secondary_goal_result": null, "campaign_name": "Website Traffic20211020010104", "tt_app_id": 0, "average_video_play_per_user": "1.64", "real_time_app_install": "0", "adgroup_name": "Ad Group20211020010107", "tt_app_name": "0", "video_play_actions": "5173", "cost_per_result": "0.2899", "app_install": "0", "real_time_cost_per_conversion": "0.000", "total_onsite_shopping_value": "0.000", "cta_conversion": "0", "total_pageview": "0", "result_rate": "1.18"}, "dimensions": {"stat_time_day": "2021-10-25 00:00:00", "ad_id": 1714125051115569}, "stat_time_day": "2021-10-25 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1691143872903} -{"stream": "ads_reports_daily", "data": {"metrics": {"mobile_app_id": "0", "result": "53", "follows": "0", "cost_per_secondary_goal_result": null, "ctr": "1.41", "promotion_type": "Website", "clicks_on_music_disc": "0", "real_time_conversion": "0", "real_time_result": "53", "total_complete_payment_rate": "0.000", "frequency": "1.20", "placement_type": "Automatic Placement", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "shares": "0", "video_views_p75": "74", "spend": "20.000", "cost_per_1000_reached": "6.382", "likes": "36", "video_watched_2s": "408", "video_views_p50": "130", "complete_payment": "0", "cpc": "0.380", "vta_purchase": "0", "real_time_result_rate": "1.41", "real_time_conversion_rate": "0.00", "comments": "1", "conversion": "0", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "average_video_play": "1.45", "onsite_shopping": "0", "campaign_id": 1714125042508817, "cta_purchase": "0", "video_views_p25": "295", "video_watched_6s": "106", "conversion_rate": "0.00", "cost_per_conversion": "0.000", "cpm": "5.310", "secondary_goal_result_rate": null, "impressions": "3765", "video_views_p100": "52", "total_purchase_value": "0.000", "vta_conversion": "0", "reach": "3134", "adgroup_id": 1714125049901106, "profile_visits": "0", "dpa_target_audience_type": null, "clicks": "53", "real_time_app_install_cost": "0.000", "real_time_cost_per_result": "0.3774", "value_per_complete_payment": "0.000", "secondary_goal_result": null, "campaign_name": "Website Traffic20211020010104", "tt_app_id": 0, "average_video_play_per_user": "1.55", "real_time_app_install": "0", "adgroup_name": "Ad Group20211020010107", "tt_app_name": "0", "video_play_actions": "3344", "cost_per_result": "0.3774", "app_install": "0", "real_time_cost_per_conversion": "0.000", "total_onsite_shopping_value": "0.000", "cta_conversion": "0", "total_pageview": "0", "result_rate": "1.41"}, "dimensions": {"stat_time_day": "2021-10-20 00:00:00", "ad_id": 1714125051115569}, "stat_time_day": "2021-10-20 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1691143872907} -{"stream": "ads_reports_daily", "data": {"metrics": {"mobile_app_id": "0", "result": "46", "follows": "0", "cost_per_secondary_goal_result": null, "ctr": "1.23", "promotion_type": "Website", "clicks_on_music_disc": "0", "real_time_conversion": "0", "real_time_result": "46", "total_complete_payment_rate": "0.000", "frequency": "1.20", "placement_type": "Automatic Placement", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "shares": "0", "video_views_p75": "90", "spend": "20.000", "cost_per_1000_reached": "6.412", "likes": "25", "video_watched_2s": "413", "video_views_p50": "142", "complete_payment": "0", "cpc": "0.430", "vta_purchase": "0", "real_time_result_rate": "1.23", "real_time_conversion_rate": "0.00", "comments": "1", "conversion": "0", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "average_video_play": "1.50", "onsite_shopping": "0", "campaign_id": 1714125042508817, "cta_purchase": "0", "video_views_p25": "297", "video_watched_6s": "112", "conversion_rate": "0.00", "cost_per_conversion": "0.000", "cpm": "5.330", "secondary_goal_result_rate": null, "impressions": "3750", "video_views_p100": "71", "total_purchase_value": "0.000", "vta_conversion": "0", "reach": "3119", "adgroup_id": 1714125049901106, "profile_visits": "0", "dpa_target_audience_type": null, "clicks": "46", "real_time_app_install_cost": "0.000", "real_time_cost_per_result": "0.4348", "value_per_complete_payment": "0.000", "secondary_goal_result": null, "campaign_name": "Website Traffic20211020010104", "tt_app_id": 0, "average_video_play_per_user": "1.61", "real_time_app_install": "0", "adgroup_name": "Ad Group20211020010107", "tt_app_name": "0", "video_play_actions": "3344", "cost_per_result": "0.4348", "app_install": "0", "real_time_cost_per_conversion": "0.000", "total_onsite_shopping_value": "0.000", "cta_conversion": "0", "total_pageview": "0", "result_rate": "1.23"}, "dimensions": {"stat_time_day": "2021-10-26 00:00:00", "ad_id": 1714125051115569}, "stat_time_day": "2021-10-26 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1691143872911} -{"stream": "ads_reports_lifetime", "data": {"dimensions": {"ad_id": 1728545390695442}, "metrics": {"app_install": "0", "average_video_play": "1.26", "complete_payment": "0", "video_watched_2s": "1364", "clicks": "145", "clicks_on_music_disc": "0", "total_onsite_shopping_value": "0.000", "real_time_app_install_cost": "0.000", "video_views_p100": "402", "result_rate": "0.92", "result": "145", "value_per_complete_payment": "0.000", "secondary_goal_result_rate": null, "ctr": "0.92", "total_pageview": "0", "cpc": "0.410", "campaign_name": "CampaignVadimTraffic", "conversion": "0", "tt_app_id": 0, "real_time_cost_per_conversion": "0.000", "real_time_cost_per_result": "0.4138", "video_watched_6s": "402", "cost_per_conversion": "0.000", "follows": "0", "comments": "0", "impressions": "15689", "cta_purchase": "0", "dpa_target_audience_type": null, "total_complete_payment_rate": "0.000", "promotion_type": "Website", "video_play_actions": "14333", "cta_conversion": "0", "real_time_result": "145", "tt_app_name": "0", "mobile_app_id": "0", "spend": "60.000", "ad_name": "AdVadim-Optimized Version 3_202203281449_2022-03-28 05:03:44", "real_time_conversion_rate": "0.00", "cpm": "3.820", "shares": "0", "frequency": "1.20", "reach": "13052", "adgroup_id": 1728545385226289, "video_views_p50": "907", "likes": "11", "vta_purchase": "0", "onsite_shopping": "0", "conversion_rate": "0.00", "real_time_app_install": "0", "total_purchase_value": "0.000", "average_video_play_per_user": "1.39", "cost_per_secondary_goal_result": null, "adgroup_name": "AdGroupVadim", "campaign_id": 1728545382536225, "real_time_result_rate": "0.92", "cost_per_1000_reached": "4.597", "video_views_p75": "522", "ad_text": "Open-source\ndata integration for modern data teams", "secondary_goal_result": null, "profile_visits": "0", "vta_conversion": "0", "placement_type": "Automatic Placement", "video_views_p25": "3339", "real_time_conversion": "0", "cost_per_result": "0.4138"}, "ad_id": 1728545390695442}, "emitted_at": 1691143894042} -{"stream": "ads_reports_lifetime", "data": {"dimensions": {"ad_id": 1714125051115569}, "metrics": {"app_install": "0", "average_video_play": "1.48", "complete_payment": "0", "video_watched_2s": "5100", "clicks": "540", "clicks_on_music_disc": "0", "total_onsite_shopping_value": "0.000", "real_time_app_install_cost": "0.000", "video_views_p100": "723", "result_rate": "1.17", "result": "540", "value_per_complete_payment": "0.000", "secondary_goal_result_rate": null, "ctr": "1.17", "total_pageview": "0", "cpc": "0.370", "campaign_name": "Website Traffic20211020010104", "conversion": "0", "tt_app_id": 0, "real_time_cost_per_conversion": "0.000", "real_time_cost_per_result": "0.3704", "video_watched_6s": "1295", "cost_per_conversion": "0.000", "follows": "0", "comments": "2", "impressions": "46116", "cta_purchase": "0", "dpa_target_audience_type": null, "total_complete_payment_rate": "0.000", "promotion_type": "Website", "video_play_actions": "40753", "cta_conversion": "0", "real_time_result": "540", "tt_app_name": "0", "mobile_app_id": "0", "spend": "200.000", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "real_time_conversion_rate": "0.00", "cpm": "4.340", "shares": "0", "frequency": "1.37", "reach": "33556", "adgroup_id": 1714125049901106, "video_views_p50": "1588", "likes": "263", "vta_purchase": "0", "onsite_shopping": "0", "conversion_rate": "0.00", "real_time_app_install": "0", "total_purchase_value": "0.000", "average_video_play_per_user": "1.80", "cost_per_secondary_goal_result": null, "adgroup_name": "Ad Group20211020010107", "campaign_id": 1714125042508817, "real_time_result_rate": "1.17", "cost_per_1000_reached": "5.960", "video_views_p75": "998", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "secondary_goal_result": null, "profile_visits": "0", "vta_conversion": "0", "placement_type": "Automatic Placement", "video_views_p25": "3674", "real_time_conversion": "0", "cost_per_result": "0.3704"}, "ad_id": 1714125051115569}, "emitted_at": 1691143894045} -{"stream": "ads_reports_lifetime", "data": {"dimensions": {"ad_id": 1714124564763650}, "metrics": {"app_install": "0", "average_video_play": "0.00", "complete_payment": "0", "video_watched_2s": "0", "clicks": "0", "clicks_on_music_disc": "0", "total_onsite_shopping_value": "0.000", "real_time_app_install_cost": "0.000", "video_views_p100": "0", "result_rate": "0.00", "result": "0", "value_per_complete_payment": "0.000", "secondary_goal_result_rate": null, "ctr": "0.00", "total_pageview": "0", "cpc": "0.000", "campaign_name": "Website Traffic20211020005342", "conversion": "0", "tt_app_id": 0, "real_time_cost_per_conversion": "0.000", "real_time_cost_per_result": "0.0000", "video_watched_6s": "0", "cost_per_conversion": "0.000", "follows": "0", "comments": "0", "impressions": "0", "cta_purchase": "0", "dpa_target_audience_type": null, "total_complete_payment_rate": "0.000", "promotion_type": "Website", "video_play_actions": "0", "cta_conversion": "0", "real_time_result": "0", "tt_app_name": "0", "mobile_app_id": "0", "spend": "0.000", "ad_name": "1200x1200 logo_1634667070143.png_2021-10-20 10:54:51", "real_time_conversion_rate": "0.00", "cpm": "0.000", "shares": "0", "frequency": "0.00", "reach": "0", "adgroup_id": 1714124588896305, "video_views_p50": "0", "likes": "0", "vta_purchase": "0", "onsite_shopping": "0", "conversion_rate": "0.00", "real_time_app_install": "0", "total_purchase_value": "0.000", "average_video_play_per_user": "0.00", "cost_per_secondary_goal_result": null, "adgroup_name": "Ad Group20211020005346", "campaign_id": 1714124576938033, "real_time_result_rate": "0.00", "cost_per_1000_reached": "0.000", "video_views_p75": "0", "ad_text": "Airbyte - open source data portability platform - from nywhere to anywhere!", "secondary_goal_result": null, "profile_visits": "0", "vta_conversion": "0", "placement_type": "Automatic Placement", "video_views_p25": "0", "real_time_conversion": "0", "cost_per_result": "0.0000"}, "ad_id": 1714124564763650}, "emitted_at": 1691143894049} -{"stream": "ad_groups_reports_daily", "data": {"metrics": {"conversion": "0", "conversion_rate": "0.00", "real_time_cost_per_conversion": "0.000", "comments": "1", "campaign_name": "Website Traffic20211020010104", "real_time_cost_per_result": "0.3774", "cost_per_1000_reached": "6.382", "cpc": "0.380", "average_video_play_per_user": "1.55", "likes": "36", "real_time_conversion_rate": "0.00", "dpa_target_audience_type": null, "mobile_app_id": "0", "video_views_p25": "295", "video_watched_2s": "408", "cpm": "5.310", "app_install": "0", "frequency": "1.20", "adgroup_name": "Ad Group20211020010107", "clicks_on_music_disc": "0", "real_time_app_install_cost": "0.000", "tt_app_id": 0, "cost_per_conversion": "0.000", "video_views_p75": "74", "real_time_app_install": "0", "shares": "0", "secondary_goal_result": null, "promotion_type": "Website", "clicks": "53", "cost_per_result": "0.3774", "result_rate": "1.41", "video_views_p50": "130", "video_play_actions": "3344", "placement_type": "Automatic Placement", "real_time_result_rate": "1.41", "cost_per_secondary_goal_result": null, "real_time_result": "53", "reach": "3134", "video_watched_6s": "106", "average_video_play": "1.45", "tt_app_name": "0", "impressions": "3765", "follows": "0", "real_time_conversion": "0", "campaign_id": 1714125042508817, "ctr": "1.41", "secondary_goal_result_rate": null, "profile_visits": "0", "result": "53", "video_views_p100": "52", "spend": "20.000"}, "dimensions": {"stat_time_day": "2021-10-20 00:00:00", "adgroup_id": 1714125049901106}, "stat_time_day": "2021-10-20 00:00:00", "adgroup_id": 1714125049901106}, "emitted_at": 1691144407230} -{"stream": "ad_groups_reports_daily", "data": {"metrics": {"conversion": "0", "conversion_rate": "0.00", "real_time_cost_per_conversion": "0.000", "comments": "0", "campaign_name": "Website Traffic20211020010104", "real_time_cost_per_result": "0.2899", "cost_per_1000_reached": "4.161", "cpc": "0.290", "average_video_play_per_user": "1.64", "likes": "36", "real_time_conversion_rate": "0.00", "dpa_target_audience_type": null, "mobile_app_id": "0", "video_views_p25": "513", "video_watched_2s": "686", "cpm": "3.430", "app_install": "0", "frequency": "1.21", "adgroup_name": "Ad Group20211020010107", "clicks_on_music_disc": "0", "real_time_app_install_cost": "0.000", "tt_app_id": 0, "cost_per_conversion": "0.000", "video_views_p75": "140", "real_time_app_install": "0", "shares": "0", "secondary_goal_result": null, "promotion_type": "Website", "clicks": "69", "cost_per_result": "0.2899", "result_rate": "1.18", "video_views_p50": "214", "video_play_actions": "5173", "placement_type": "Automatic Placement", "real_time_result_rate": "1.18", "cost_per_secondary_goal_result": null, "real_time_result": "69", "reach": "4806", "video_watched_6s": "180", "average_video_play": "1.52", "tt_app_name": "0", "impressions": "5830", "follows": "0", "real_time_conversion": "0", "campaign_id": 1714125042508817, "ctr": "1.18", "secondary_goal_result_rate": null, "profile_visits": "0", "result": "69", "video_views_p100": "92", "spend": "20.000"}, "dimensions": {"stat_time_day": "2021-10-25 00:00:00", "adgroup_id": 1714125049901106}, "stat_time_day": "2021-10-25 00:00:00", "adgroup_id": 1714125049901106}, "emitted_at": 1691144407234} -{"stream": "ad_groups_reports_daily", "data": {"metrics": {"conversion": "0", "conversion_rate": "0.00", "real_time_cost_per_conversion": "0.000", "comments": "0", "campaign_name": "Website Traffic20211020010104", "real_time_cost_per_result": "0.5000", "cost_per_1000_reached": "5.523", "cpc": "0.500", "average_video_play_per_user": "1.50", "likes": "13", "real_time_conversion_rate": "0.00", "dpa_target_audience_type": null, "mobile_app_id": "0", "video_views_p25": "306", "video_watched_2s": "436", "cpm": "4.550", "app_install": "0", "frequency": "1.21", "adgroup_name": "Ad Group20211020010107", "clicks_on_music_disc": "0", "real_time_app_install_cost": "0.000", "tt_app_id": 0, "cost_per_conversion": "0.000", "video_views_p75": "85", "real_time_app_install": "0", "shares": "0", "secondary_goal_result": null, "promotion_type": "Website", "clicks": "40", "cost_per_result": "0.5000", "result_rate": "0.91", "video_views_p50": "130", "video_play_actions": "3852", "placement_type": "Automatic Placement", "real_time_result_rate": "0.91", "cost_per_secondary_goal_result": null, "real_time_result": "40", "reach": "3621", "video_watched_6s": "104", "average_video_play": "1.41", "tt_app_name": "0", "impressions": "4394", "follows": "0", "real_time_conversion": "0", "campaign_id": 1714125042508817, "ctr": "0.91", "secondary_goal_result_rate": null, "profile_visits": "0", "result": "40", "video_views_p100": "66", "spend": "20.000"}, "dimensions": {"stat_time_day": "2021-10-29 00:00:00", "adgroup_id": 1714125049901106}, "stat_time_day": "2021-10-29 00:00:00", "adgroup_id": 1714125049901106}, "emitted_at": 1691144407237} -{"stream": "ad_groups_reports_lifetime", "data": {"metrics": {"secondary_goal_result_rate": null, "follows": "0", "dpa_target_audience_type": null, "campaign_name": "CampaignVadimTraffic", "video_views_p75": "522", "video_views_p25": "3339", "real_time_app_install_cost": "0.000", "cost_per_conversion": "0.000", "placement_type": "Automatic Placement", "tt_app_name": "0", "spend": "60.000", "clicks_on_music_disc": "0", "cost_per_secondary_goal_result": null, "result": "145", "app_install": "0", "video_views_p100": "402", "shares": "0", "secondary_goal_result": null, "profile_visits": "0", "mobile_app_id": "0", "average_video_play_per_user": "1.39", "average_video_play": "1.26", "conversion_rate": "0.00", "real_time_result": "145", "frequency": "1.20", "real_time_conversion_rate": "0.00", "cpm": "3.820", "adgroup_name": "AdGroupVadim", "cpc": "0.410", "reach": "13052", "campaign_id": 1728545382536225, "video_watched_6s": "402", "cost_per_result": "0.4138", "result_rate": "0.92", "video_watched_2s": "1364", "real_time_conversion": "0", "real_time_cost_per_result": "0.4138", "real_time_result_rate": "0.92", "real_time_app_install": "0", "tt_app_id": 0, "promotion_type": "Website", "video_play_actions": "14333", "cost_per_1000_reached": "4.597", "impressions": "15689", "video_views_p50": "907", "comments": "0", "likes": "11", "conversion": "0", "real_time_cost_per_conversion": "0.000", "clicks": "145", "ctr": "0.92"}, "dimensions": {"adgroup_id": 1728545385226289}, "adgroup_id": 1728545385226289}, "emitted_at": 1691144426151} -{"stream": "ad_groups_reports_lifetime", "data": {"metrics": {"secondary_goal_result_rate": null, "follows": "0", "dpa_target_audience_type": null, "campaign_name": "Website Traffic20211020010104", "video_views_p75": "998", "video_views_p25": "3674", "real_time_app_install_cost": "0.000", "cost_per_conversion": "0.000", "placement_type": "Automatic Placement", "tt_app_name": "0", "spend": "200.000", "clicks_on_music_disc": "0", "cost_per_secondary_goal_result": null, "result": "540", "app_install": "0", "video_views_p100": "723", "shares": "0", "secondary_goal_result": null, "profile_visits": "0", "mobile_app_id": "0", "average_video_play_per_user": "1.80", "average_video_play": "1.48", "conversion_rate": "0.00", "real_time_result": "540", "frequency": "1.37", "real_time_conversion_rate": "0.00", "cpm": "4.340", "adgroup_name": "Ad Group20211020010107", "cpc": "0.370", "reach": "33556", "campaign_id": 1714125042508817, "video_watched_6s": "1295", "cost_per_result": "0.3704", "result_rate": "1.17", "video_watched_2s": "5100", "real_time_conversion": "0", "real_time_cost_per_result": "0.3704", "real_time_result_rate": "1.17", "real_time_app_install": "0", "tt_app_id": 0, "promotion_type": "Website", "video_play_actions": "40753", "cost_per_1000_reached": "5.960", "impressions": "46116", "video_views_p50": "1588", "comments": "2", "likes": "263", "conversion": "0", "real_time_cost_per_conversion": "0.000", "clicks": "540", "ctr": "1.17"}, "dimensions": {"adgroup_id": 1714125049901106}, "adgroup_id": 1714125049901106}, "emitted_at": 1691144426155} -{"stream": "ad_groups_reports_lifetime", "data": {"metrics": {"secondary_goal_result_rate": null, "follows": "0", "dpa_target_audience_type": null, "campaign_name": "Website Traffic20211020005342", "video_views_p75": "0", "video_views_p25": "0", "real_time_app_install_cost": "0.000", "cost_per_conversion": "0.000", "placement_type": "Automatic Placement", "tt_app_name": "0", "spend": "0.000", "clicks_on_music_disc": "0", "cost_per_secondary_goal_result": null, "result": "0", "app_install": "0", "video_views_p100": "0", "shares": "0", "secondary_goal_result": null, "profile_visits": "0", "mobile_app_id": "0", "average_video_play_per_user": "0.00", "average_video_play": "0.00", "conversion_rate": "0.00", "real_time_result": "0", "frequency": "0.00", "real_time_conversion_rate": "0.00", "cpm": "0.000", "adgroup_name": "Ad Group20211020005346", "cpc": "0.000", "reach": "0", "campaign_id": 1714124576938033, "video_watched_6s": "0", "cost_per_result": "0.0000", "result_rate": "0.00", "video_watched_2s": "0", "real_time_conversion": "0", "real_time_cost_per_result": "0.0000", "real_time_result_rate": "0.00", "real_time_app_install": "0", "tt_app_id": 0, "promotion_type": "Website", "video_play_actions": "0", "cost_per_1000_reached": "0.000", "impressions": "0", "video_views_p50": "0", "comments": "0", "likes": "0", "conversion": "0", "real_time_cost_per_conversion": "0.000", "clicks": "0", "ctr": "0.00"}, "dimensions": {"adgroup_id": 1714124588896305}, "adgroup_id": 1714124588896305}, "emitted_at": 1691144426159} -{"stream": "campaigns_reports_daily", "data": {"metrics": {"video_watched_2s": "493", "campaign_name": "Website Traffic20211020010104", "video_views_p50": "164", "average_video_play_per_user": "1.61", "cpc": "0.390", "impressions": "4696", "follows": "0", "video_views_p100": "76", "real_time_app_install": "0", "ctr": "1.09", "clicks_on_music_disc": "0", "comments": "0", "frequency": "1.23", "reach": "3822", "average_video_play": "1.48", "shares": "0", "profile_visits": "0", "video_play_actions": "4179", "video_views_p25": "355", "video_views_p75": "108", "app_install": "0", "clicks": "51", "spend": "20.000", "real_time_app_install_cost": "0.000", "cost_per_1000_reached": "5.233", "video_watched_6s": "132", "likes": "18", "cpm": "4.260"}, "dimensions": {"campaign_id": 1714125042508817, "stat_time_day": "2021-10-27 00:00:00"}, "stat_time_day": "2021-10-27 00:00:00", "campaign_id": 1714125042508817}, "emitted_at": 1691144967333} -{"stream": "campaigns_reports_daily", "data": {"metrics": {"video_watched_2s": "390", "campaign_name": "Website Traffic20211020010104", "video_views_p50": "112", "average_video_play_per_user": "1.57", "cpc": "0.480", "impressions": "3520", "follows": "0", "video_views_p100": "59", "real_time_app_install": "0", "ctr": "1.19", "clicks_on_music_disc": "0", "comments": "0", "frequency": "1.21", "reach": "2908", "average_video_play": "1.46", "shares": "0", "profile_visits": "0", "video_play_actions": "3118", "video_views_p25": "277", "video_views_p75": "74", "app_install": "0", "clicks": "42", "spend": "20.000", "real_time_app_install_cost": "0.000", "cost_per_1000_reached": "6.878", "video_watched_6s": "92", "likes": "17", "cpm": "5.680"}, "dimensions": {"campaign_id": 1714125042508817, "stat_time_day": "2021-10-22 00:00:00"}, "stat_time_day": "2021-10-22 00:00:00", "campaign_id": 1714125042508817}, "emitted_at": 1691144967337} -{"stream": "campaigns_reports_daily", "data": {"metrics": {"video_watched_2s": "471", "campaign_name": "Website Traffic20211020010104", "video_views_p50": "144", "average_video_play_per_user": "1.54", "cpc": "0.420", "impressions": "4787", "follows": "0", "video_views_p100": "70", "real_time_app_install": "0", "ctr": "1.00", "clicks_on_music_disc": "0", "comments": "0", "frequency": "1.22", "reach": "3938", "average_video_play": "1.42", "shares": "0", "profile_visits": "0", "video_play_actions": "4253", "video_views_p25": "328", "video_views_p75": "100", "app_install": "0", "clicks": "48", "spend": "20.000", "real_time_app_install_cost": "0.000", "cost_per_1000_reached": "5.079", "video_watched_6s": "120", "likes": "18", "cpm": "4.180"}, "dimensions": {"campaign_id": 1714125042508817, "stat_time_day": "2021-10-28 00:00:00"}, "stat_time_day": "2021-10-28 00:00:00", "campaign_id": 1714125042508817}, "emitted_at": 1691144967339} -{"stream": "campaigns_reports_lifetime", "data": {"metrics": {"cpm": "3.820", "campaign_name": "CampaignVadimTraffic", "video_views_p25": "3339", "impressions": "15689", "frequency": "1.20", "cpc": "0.410", "follows": "0", "video_play_actions": "14333", "profile_visits": "0", "real_time_app_install": "0", "ctr": "0.92", "cost_per_1000_reached": "4.597", "average_video_play_per_user": "1.39", "likes": "11", "comments": "0", "app_install": "0", "average_video_play": "1.26", "real_time_app_install_cost": "0.000", "video_watched_2s": "1364", "clicks": "145", "video_views_p50": "907", "spend": "60.000", "video_watched_6s": "402", "video_views_p75": "522", "video_views_p100": "402", "shares": "0", "clicks_on_music_disc": "0", "reach": "13052"}, "dimensions": {"campaign_id": 1728545382536225}, "campaign_id": 1728545382536225}, "emitted_at": 1691144987206} -{"stream": "campaigns_reports_lifetime", "data": {"metrics": {"cpm": "4.340", "campaign_name": "Website Traffic20211020010104", "video_views_p25": "3674", "impressions": "46116", "frequency": "1.37", "cpc": "0.370", "follows": "0", "video_play_actions": "40753", "profile_visits": "0", "real_time_app_install": "0", "ctr": "1.17", "cost_per_1000_reached": "5.960", "average_video_play_per_user": "1.80", "likes": "263", "comments": "2", "app_install": "0", "average_video_play": "1.48", "real_time_app_install_cost": "0.000", "video_watched_2s": "5100", "clicks": "540", "video_views_p50": "1588", "spend": "200.000", "video_watched_6s": "1295", "video_views_p75": "998", "video_views_p100": "723", "shares": "0", "clicks_on_music_disc": "0", "reach": "33556"}, "dimensions": {"campaign_id": 1714125042508817}, "campaign_id": 1714125042508817}, "emitted_at": 1691144987209} -{"stream": "campaigns_reports_lifetime", "data": {"metrics": {"cpm": "0.000", "campaign_name": "Website Traffic20211020005342", "video_views_p25": "0", "impressions": "0", "frequency": "0.00", "cpc": "0.000", "follows": "0", "video_play_actions": "0", "profile_visits": "0", "real_time_app_install": "0", "ctr": "0.00", "cost_per_1000_reached": "0.000", "average_video_play_per_user": "0.00", "likes": "0", "comments": "0", "app_install": "0", "average_video_play": "0.00", "real_time_app_install_cost": "0.000", "video_watched_2s": "0", "clicks": "0", "video_views_p50": "0", "spend": "0.000", "video_watched_6s": "0", "video_views_p75": "0", "video_views_p100": "0", "shares": "0", "clicks_on_music_disc": "0", "reach": "0"}, "dimensions": {"campaign_id": 1714124576938033}, "campaign_id": 1714124576938033}, "emitted_at": 1691144987213} -{"stream": "advertisers_reports_daily", "data": {"dimensions": {"stat_time_day": "2021-10-28 00:00:00", "advertiser_id": 7002238017842757633}, "metrics": {"cash_spend": "20.000", "video_views_p75": "100", "spend": "20.000", "app_install": "0", "average_video_play": "1.42", "average_video_play_per_user": "1.54", "frequency": "1.22", "clicks": "48", "video_views_p100": "70", "comments": "0", "real_time_app_install": "0", "likes": "18", "video_watched_6s": "120", "video_views_p25": "328", "ctr": "1.00", "follows": "0", "shares": "0", "cost_per_1000_reached": "5.079", "video_watched_2s": "471", "reach": "3938", "real_time_app_install_cost": "0.000", "cpc": "0.420", "cpm": "4.180", "voucher_spend": "0.000", "video_play_actions": "4253", "profile_visits": "0", "impressions": "4787", "clicks_on_music_disc": "0", "video_views_p50": "144"}, "stat_time_day": "2021-10-28 00:00:00", "advertiser_id": 7002238017842757633}, "emitted_at": 1691145506565} -{"stream": "advertisers_reports_daily", "data": {"dimensions": {"stat_time_day": "2021-10-23 00:00:00", "advertiser_id": 7002238017842757633}, "metrics": {"cash_spend": "20.000", "video_views_p75": "95", "spend": "20.000", "app_install": "0", "average_video_play": "1.53", "average_video_play_per_user": "1.65", "frequency": "1.23", "clicks": "67", "video_views_p100": "65", "comments": "0", "real_time_app_install": "0", "likes": "19", "video_watched_6s": "124", "video_views_p25": "338", "ctr": "1.64", "follows": "0", "shares": "0", "cost_per_1000_reached": "6.020", "video_watched_2s": "463", "reach": "3322", "real_time_app_install_cost": "0.000", "cpc": "0.300", "cpm": "4.910", "voucher_spend": "0.000", "video_play_actions": "3590", "profile_visits": "0", "impressions": "4077", "clicks_on_music_disc": "0", "video_views_p50": "146"}, "stat_time_day": "2021-10-23 00:00:00", "advertiser_id": 7002238017842757633}, "emitted_at": 1691145506568} -{"stream": "advertisers_reports_daily", "data": {"dimensions": {"stat_time_day": "2021-10-26 00:00:00", "advertiser_id": 7002238017842757633}, "metrics": {"cash_spend": "20.000", "video_views_p75": "90", "spend": "20.000", "app_install": "0", "average_video_play": "1.50", "average_video_play_per_user": "1.61", "frequency": "1.20", "clicks": "46", "video_views_p100": "71", "comments": "1", "real_time_app_install": "0", "likes": "25", "video_watched_6s": "112", "video_views_p25": "297", "ctr": "1.23", "follows": "0", "shares": "0", "cost_per_1000_reached": "6.412", "video_watched_2s": "413", "reach": "3119", "real_time_app_install_cost": "0.000", "cpc": "0.430", "cpm": "5.330", "voucher_spend": "0.000", "video_play_actions": "3344", "profile_visits": "0", "impressions": "3750", "clicks_on_music_disc": "0", "video_views_p50": "142"}, "stat_time_day": "2021-10-26 00:00:00", "advertiser_id": 7002238017842757633}, "emitted_at": 1691145506571} -{"stream": "advertisers_reports_lifetime", "data": {"metrics": {"follows": "0", "video_views_p50": "2665", "reach": "50418", "shares": "0", "clicks_on_music_disc": "0", "likes": "328", "video_play_actions": "59390", "spend": "280.000", "video_views_p75": "1636", "app_install": "0", "video_watched_6s": "1838", "video_views_p25": "7364", "video_views_p100": "1205", "ctr": "1.12", "clicks": "750", "cpc": "0.370", "profile_visits": "0", "video_watched_2s": "6941", "comments": "2", "real_time_app_install_cost": "0.000", "cpm": "4.200", "impressions": "66691", "average_video_play": "1.43", "cost_per_1000_reached": "5.554", "real_time_app_install": "0", "average_video_play_per_user": "1.68", "frequency": "1.32"}, "dimensions": {"advertiser_id": 7002238017842757633}, "advertiser_id": 7002238017842757633}, "emitted_at": 1691145526253} -{"stream": "ads_audience_reports_daily", "data": {"metrics": {"spend": "3.690", "ctr": "0.98", "cost_per_conversion": "0.000", "result": "12", "adgroup_name": "Ad Group20211020010107", "cpm": "3.020", "result_rate": "0.98", "real_time_conversion": "0", "placement_type": "Automatic Placement", "impressions": "1222", "real_time_cost_per_result": "0.3075", "real_time_result_rate": "0.98", "promotion_type": "Website", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "campaign_id": 1714125042508817, "dpa_target_audience_type": null, "real_time_result": "12", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "real_time_cost_per_conversion": "0.000", "tt_app_id": "0", "mobile_app_id": "0", "real_time_conversion_rate": "0.00", "conversion_rate": "0.00", "conversion": "0", "cpc": "0.310", "campaign_name": "Website Traffic20211020010104", "cost_per_result": "0.3075", "tt_app_name": "0", "clicks": "12", "adgroup_id": 1714125049901106}, "dimensions": {"stat_time_day": "2021-10-25 00:00:00", "ad_id": 1714125051115569, "gender": "MALE", "age": "AGE_25_34"}, "stat_time_day": "2021-10-25 00:00:00", "ad_id": 1714125051115569, "gender": "MALE", "age": "AGE_25_34"}, "emitted_at": 1691145528615} -{"stream": "ads_audience_reports_daily", "data": {"metrics": {"spend": "1.770", "ctr": "1.52", "cost_per_conversion": "0.000", "result": "5", "adgroup_name": "Ad Group20211020010107", "cpm": "5.380", "result_rate": "1.52", "real_time_conversion": "0", "placement_type": "Automatic Placement", "impressions": "329", "real_time_cost_per_result": "0.3540", "real_time_result_rate": "1.52", "promotion_type": "Website", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "campaign_id": 1714125042508817, "dpa_target_audience_type": null, "real_time_result": "5", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "real_time_cost_per_conversion": "0.000", "tt_app_id": "0", "mobile_app_id": "0", "real_time_conversion_rate": "0.00", "conversion_rate": "0.00", "conversion": "0", "cpc": "0.350", "campaign_name": "Website Traffic20211020010104", "cost_per_result": "0.3540", "tt_app_name": "0", "clicks": "5", "adgroup_id": 1714125049901106}, "dimensions": {"stat_time_day": "2021-10-21 00:00:00", "ad_id": 1714125051115569, "gender": "MALE", "age": "AGE_45_54"}, "stat_time_day": "2021-10-21 00:00:00", "ad_id": 1714125051115569, "gender": "MALE", "age": "AGE_45_54"}, "emitted_at": 1691145528618} -{"stream": "ads_audience_reports_daily", "data": {"metrics": {"spend": "0.000", "ctr": "0.00", "cost_per_conversion": "0.000", "result": "0", "adgroup_name": "Ad Group20211019111040", "cpm": "0.000", "result_rate": "0.00", "real_time_conversion": "0", "placement_type": "Automatic Placement", "impressions": "56", "real_time_cost_per_result": "0.0000", "real_time_result_rate": "0.00", "promotion_type": "Website", "ad_text": "Open Source ETL", "campaign_id": 1714073078669329, "dpa_target_audience_type": null, "real_time_result": "0", "ad_name": "Optimized Version 1_202110192111_2021-10-19 21:11:39", "real_time_cost_per_conversion": "0.000", "tt_app_id": "0", "mobile_app_id": "0", "real_time_conversion_rate": "0.00", "conversion_rate": "0.00", "conversion": "0", "cpc": "0.000", "campaign_name": "Website Traffic20211019110444", "cost_per_result": "0.0000", "tt_app_name": "0", "clicks": "0", "adgroup_id": 1714073022392322}, "dimensions": {"stat_time_day": "2021-10-19 00:00:00", "ad_id": 1714073085256738, "gender": "MALE", "age": "AGE_25_34"}, "stat_time_day": "2021-10-19 00:00:00", "ad_id": 1714073085256738, "gender": "MALE", "age": "AGE_25_34"}, "emitted_at": 1691145528621} -{"stream": "ad_group_audience_reports_daily", "data": {"dimensions": {"age": "AGE_45_54", "gender": "MALE", "adgroup_id": 1714125049901106, "stat_time_day": "2021-10-27 00:00:00"}, "metrics": {"clicks": "5", "tt_app_name": "0", "campaign_id": 1714125042508817, "result_rate": "1.34", "dpa_target_audience_type": null, "impressions": "373", "mobile_app_id": "0", "conversion_rate": "0.00", "campaign_name": "Website Traffic20211020010104", "real_time_cost_per_conversion": "0.000", "ctr": "1.34", "real_time_conversion": "0", "real_time_cost_per_result": "0.4280", "cpm": "5.740", "cost_per_conversion": "0.000", "real_time_conversion_rate": "0.00", "tt_app_id": "0", "real_time_result_rate": "1.34", "placement_type": "Automatic Placement", "promotion_type": "Website", "result": "5", "real_time_result": "5", "cpc": "0.430", "cost_per_result": "0.4280", "conversion": "0", "spend": "2.140", "adgroup_name": "Ad Group20211020010107"}, "stat_time_day": "2021-10-27 00:00:00", "adgroup_id": 1714125049901106, "gender": "MALE", "age": "AGE_45_54"}, "emitted_at": 1691145591995} -{"stream": "ad_group_audience_reports_daily", "data": {"dimensions": {"age": "AGE_35_44", "gender": "FEMALE", "adgroup_id": 1714125049901106, "stat_time_day": "2021-10-22 00:00:00"}, "metrics": {"clicks": "9", "tt_app_name": "0", "campaign_id": 1714125042508817, "result_rate": "1.41", "dpa_target_audience_type": null, "impressions": "638", "mobile_app_id": "0", "conversion_rate": "0.00", "campaign_name": "Website Traffic20211020010104", "real_time_cost_per_conversion": "0.000", "ctr": "1.41", "real_time_conversion": "0", "real_time_cost_per_result": "0.4789", "cpm": "6.760", "cost_per_conversion": "0.000", "real_time_conversion_rate": "0.00", "tt_app_id": "0", "real_time_result_rate": "1.41", "placement_type": "Automatic Placement", "promotion_type": "Website", "result": "9", "real_time_result": "9", "cpc": "0.480", "cost_per_result": "0.4789", "conversion": "0", "spend": "4.310", "adgroup_name": "Ad Group20211020010107"}, "stat_time_day": "2021-10-22 00:00:00", "adgroup_id": 1714125049901106, "gender": "FEMALE", "age": "AGE_35_44"}, "emitted_at": 1691145591998} -{"stream": "ad_group_audience_reports_daily", "data": {"dimensions": {"age": "AGE_35_44", "gender": "FEMALE", "adgroup_id": 1714073022392322, "stat_time_day": "2021-10-19 00:00:00"}, "metrics": {"clicks": "0", "tt_app_name": "0", "campaign_id": 1714073078669329, "result_rate": "0.00", "dpa_target_audience_type": null, "impressions": "41", "mobile_app_id": "0", "conversion_rate": "0.00", "campaign_name": "Website Traffic20211019110444", "real_time_cost_per_conversion": "0.000", "ctr": "0.00", "real_time_conversion": "0", "real_time_cost_per_result": "0.0000", "cpm": "0.000", "cost_per_conversion": "0.000", "real_time_conversion_rate": "0.00", "tt_app_id": "0", "real_time_result_rate": "0.00", "placement_type": "Automatic Placement", "promotion_type": "Website", "result": "0", "real_time_result": "0", "cpc": "0.000", "cost_per_result": "0.0000", "conversion": "0", "spend": "0.000", "adgroup_name": "Ad Group20211019111040"}, "stat_time_day": "2021-10-19 00:00:00", "adgroup_id": 1714073022392322, "gender": "FEMALE", "age": "AGE_35_44"}, "emitted_at": 1691145592001} -{"stream": "campaigns_audience_reports_by_country_daily", "data": {"dimensions": {"campaign_id": 1714073078669329, "country_code": "US", "stat_time_day": "2021-10-19 00:00:00"}, "metrics": {"clicks": "65", "impressions": "4874", "campaign_name": "Website Traffic20211019110444", "ctr": "1.33", "cpm": "4.100", "cpc": "0.310", "spend": "20.000"}, "stat_time_day": "2021-10-19 00:00:00", "campaign_id": 1714073078669329, "country_code": "US"}, "emitted_at": 1691145665950} -{"stream": "campaigns_audience_reports_by_country_daily", "data": {"dimensions": {"campaign_id": 1714073078669329, "country_code": "US", "stat_time_day": "2021-10-20 00:00:00"}, "metrics": {"clicks": "0", "impressions": "12", "campaign_name": "Website Traffic20211019110444", "ctr": "0.00", "cpm": "0.000", "cpc": "0.000", "spend": "0.000"}, "stat_time_day": "2021-10-20 00:00:00", "campaign_id": 1714073078669329, "country_code": "US"}, "emitted_at": 1691145665953} -{"stream": "campaigns_audience_reports_by_country_daily", "data": {"dimensions": {"campaign_id": 1714073078669329, "country_code": "US", "stat_time_day": "2021-10-22 00:00:00"}, "metrics": {"clicks": "0", "impressions": "0", "campaign_name": "Website Traffic20211019110444", "ctr": "0.00", "cpm": "0.000", "cpc": "0.000", "spend": "0.000"}, "stat_time_day": "2021-10-22 00:00:00", "campaign_id": 1714073078669329, "country_code": "US"}, "emitted_at": 1691145665956} -{"stream": "advertisers_audience_reports_daily", "data": {"dimensions": {"advertiser_id": 7002238017842757633, "age": "AGE_13_17", "gender": "MALE", "stat_time_day": "2021-10-19 00:00:00"}, "metrics": {"clicks": "22", "impressions": "1814", "ctr": "1.21", "cpm": "3.930", "cpc": "0.320", "spend": "7.130"}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_13_17"}, "emitted_at": 1691145699763} -{"stream": "advertisers_audience_reports_daily", "data": {"dimensions": {"advertiser_id": 7002238017842757633, "age": "AGE_13_17", "gender": "MALE", "stat_time_day": "2021-10-20 00:00:00"}, "metrics": {"clicks": "0", "impressions": "4", "ctr": "0.00", "cpm": "0.000", "cpc": "0.000", "spend": "0.000"}, "stat_time_day": "2021-10-20 00:00:00", "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_13_17"}, "emitted_at": 1691145699766} -{"stream": "advertisers_audience_reports_daily", "data": {"dimensions": {"advertiser_id": 7002238017842757633, "age": "AGE_13_17", "gender": "FEMALE", "stat_time_day": "2021-10-19 00:00:00"}, "metrics": {"clicks": "29", "impressions": "2146", "ctr": "1.35", "cpm": "3.880", "cpc": "0.290", "spend": "8.320"}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "gender": "FEMALE", "age": "AGE_13_17"}, "emitted_at": 1691145699769} -{"stream": "advertisers_audience_reports_lifetime", "data": {"metrics": {"cpm": "4.580", "impressions": "6897", "cpc": "0.360", "ctr": "1.26", "clicks": "87", "spend": "31.560"}, "dimensions": {"advertiser_id": 7002238017842757633, "age": "AGE_35_44", "gender": "MALE"}, "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_35_44"}, "emitted_at": 1691145715370} -{"stream": "advertisers_audience_reports_lifetime", "data": {"metrics": {"cpm": "4.280", "impressions": "3450", "cpc": "0.380", "ctr": "1.13", "clicks": "39", "spend": "14.770"}, "dimensions": {"advertiser_id": 7002238017842757633, "age": "AGE_45_54", "gender": "MALE"}, "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_45_54"}, "emitted_at": 1691145715374} -{"stream": "advertisers_audience_reports_lifetime", "data": {"metrics": {"cpm": "3.920", "impressions": "1818", "cpc": "0.320", "ctr": "1.21", "clicks": "22", "spend": "7.130"}, "dimensions": {"advertiser_id": 7002238017842757633, "age": "AGE_13_17", "gender": "MALE"}, "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_13_17"}, "emitted_at": 1691145715376} +{"stream": "advertisers", "data": {"country": "US", "promotion_center_province": null, "description": "https://", "display_timezone": "America/Los_Angeles", "telephone_number": "+14156****85", "brand": null, "promotion_center_city": null, "timezone": "Etc/GMT+8", "advertiser_account_type": "AUCTION", "industry": "291905", "advertiser_id": 7002238017842757633, "role": "ROLE_ADVERTISER", "address": "350 29th avenue, San Francisco", "name": "Airbyte0830", "license_url": null, "balance": 0, "license_province": null, "status": "STATUS_ENABLE", "contacter": "Ai***te", "create_time": 1630335591, "company": "Airbyte", "license_no": "", "cellphone_number": "+13477****53", "rejection_reason": "", "currency": "USD", "language": "en", "promotion_area": "0", "email": "i***************@**********", "license_city": null}, "emitted_at": 1698061644722} +{"stream": "ads", "data": {"creative_authorized": true, "viewability_vast_url": null, "secondary_status": "AD_STATUS_CAMPAIGN_DISABLE", "ad_name": "Video16974675946002_Heartwarming Atmosphere Pops with Piano Main(827850)_2023-10-16 07:46:35", "app_name": "", "vast_moat_enabled": false, "profile_image_url": "https://p21-ad-sg.ibyteimg.com/obj/ad-site-i18n-sg/202110205d0d488b68ead898460bad74", "call_to_action_id": "7290567590173363202", "landing_page_urls": null, "ad_texts": null, "identity_type": "CUSTOMIZED_USER", "modify_time": "2023-10-16 14:53:29", "campaign_name": "UTM_PARAMSTraffic20231016173112", "click_tracking_url": null, "is_new_structure": true, "page_id": null, "avatar_icon_web_uri": "ad-site-i18n-sg/202110205d0d488b68ead898460bad74", "ad_format": "SINGLE_VIDEO", "is_aco": false, "landing_page_url": "https://airbyte.com/?utm_source=tiktok&utm_medium=paid-social&utm_campaign=_tiktok_ads", "identity_id": "7244085252222255106", "adgroup_id": 1779923881029666, "brand_safety_postbid_partner": "UNSET", "brand_safety_vast_url": null, "creative_type": null, "deeplink_type": "NORMAL", "advertiser_id": 7002238017842757633, "fallback_type": "UNSET", "ad_id": 1779923894511665, "video_id": "v10033g50000ckmkpnbc77ucmin3t88g", "create_time": "2023-10-16 14:48:30", "deeplink": "", "campaign_id": 1779923887578145, "card_id": null, "playable_url": "", "viewability_postbid_partner": "UNSET", "image_ids": ["tos-alisg-p-0051c001-sg/oEnxmUxanQZbiaZzA9eAQfNdQBb1lzBIDRgVDL"], "display_name": "Airbyte", "operation_status": "ENABLE", "optimization_event": null, "music_id": null, "tracking_pixel_id": 0, "ad_text": "airbyte", "adgroup_name": "Ad group 20231016073545", "impression_tracking_url": null}, "emitted_at": 1698061646871} +{"stream": "ads", "data": {"creative_authorized": true, "viewability_vast_url": null, "secondary_status": "AD_STATUS_CAMPAIGN_DISABLE", "ad_name": "Video16974675945951_Whale, sea, electronica(859574)_2023-10-16 07:46:35", "app_name": "", "vast_moat_enabled": false, "profile_image_url": "https://p21-ad-sg.ibyteimg.com/obj/ad-site-i18n-sg/202110205d0d488b68ead898460bad74", "call_to_action_id": "7290567668606716929", "landing_page_urls": null, "ad_texts": null, "identity_type": "CUSTOMIZED_USER", "modify_time": "2023-10-16 14:53:29", "campaign_name": "UTM_PARAMSTraffic20231016173112", "click_tracking_url": null, "is_new_structure": true, "page_id": null, "avatar_icon_web_uri": "ad-site-i18n-sg/202110205d0d488b68ead898460bad74", "ad_format": "SINGLE_VIDEO", "is_aco": false, "landing_page_url": "https://airbyte.com/?utm_source=tiktok&utm_medium=paid-social&utm_campaign=_tiktok_ads", "identity_id": "7244085252222255106", "adgroup_id": 1779923881029666, "brand_safety_postbid_partner": "UNSET", "brand_safety_vast_url": null, "creative_type": null, "deeplink_type": "NORMAL", "advertiser_id": 7002238017842757633, "fallback_type": "UNSET", "ad_id": 1779923894506609, "video_id": "v10033g50000ckmkplrc77u30n4dkdb0", "create_time": "2023-10-16 14:48:30", "deeplink": "", "campaign_id": 1779923887578145, "card_id": null, "playable_url": "", "viewability_postbid_partner": "UNSET", "image_ids": ["tos-alisg-p-0051c001-sg/oEaaNUsAlIQwDnBGmBLecBzDQGnACMA0Xgbfb5"], "display_name": "Airbyte", "operation_status": "ENABLE", "optimization_event": null, "music_id": null, "tracking_pixel_id": 0, "ad_text": "airbyte", "adgroup_name": "Ad group 20231016073545", "impression_tracking_url": null}, "emitted_at": 1698061646872} +{"stream": "ads", "data": {"creative_authorized": false, "viewability_vast_url": null, "secondary_status": "AD_STATUS_CAMPAIGN_DISABLE", "ad_name": "AdVadim-Optimized Version 3_202203281449_2022-03-28 05:03:44", "app_name": "", "vast_moat_enabled": false, "profile_image_url": "https://p21-ad-sg.ibyteimg.com/obj/ad-site-i18n-sg/202203285d0de5c114d0690a462bb6a4", "call_to_action_id": "7080120957230238722", "landing_page_urls": null, "ad_texts": null, "identity_type": "CUSTOMIZED_USER", "modify_time": "2023-10-17 08:50:32", "campaign_name": "CampaignVadimTraffic", "click_tracking_url": null, "is_new_structure": true, "page_id": null, "avatar_icon_web_uri": "ad-site-i18n-sg/202203285d0de5c114d0690a462bb6a4", "ad_format": "SINGLE_VIDEO", "is_aco": false, "landing_page_url": "https://airbyte.com", "identity_id": "7080121820963422209", "adgroup_id": 1728545385226289, "brand_safety_postbid_partner": "UNSET", "brand_safety_vast_url": null, "creative_type": null, "deeplink_type": "NORMAL", "advertiser_id": 7002238017842757633, "fallback_type": "UNSET", "ad_id": 1728545390695442, "video_id": "v10033g50000c90q1d3c77ub6e96fvo0", "create_time": "2022-03-28 12:09:09", "deeplink": "", "campaign_id": 1728545382536225, "card_id": null, "playable_url": "", "viewability_postbid_partner": "UNSET", "image_ids": ["v0201/7f371ff6f0764f8b8ef4f37d7b980d50"], "display_name": "airbyte", "operation_status": "ENABLE", "optimization_event": null, "music_id": null, "ad_text": "Open-source\ndata integration for modern data teams", "adgroup_name": "AdGroupVadim", "impression_tracking_url": null}, "emitted_at": 1698061646873} +{"stream": "ad_groups", "data": {"schedule_infos": null, "adgroup_app_profile_page_state": null, "billing_event": "CPC", "skip_learning_phase": 0, "secondary_status": "ADGROUP_STATUS_CAMPAIGN_DISABLE", "rf_purchased_type": null, "excluded_custom_actions": [], "gender": "GENDER_UNLIMITED", "is_hfss": false, "conversion_window": null, "video_download_disabled": false, "rf_estimated_cpr": null, "budget_mode": "BUDGET_MODE_DYNAMIC_DAILY_BUDGET", "network_types": [], "auto_targeting_enabled": false, "schedule_end_time": "2033-10-13 15:35:45", "contextual_tag_ids": [], "isp_ids": [], "zipcode_ids": [], "adgroup_name": "Ad group 20231016073545", "inventory_filter_enabled": false, "modify_time": "2023-10-16 21:47:26", "campaign_name": "UTM_PARAMSTraffic20231016173112", "household_income": [], "bid_price": 0, "dayparting": "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", "comment_disabled": false, "next_day_retention": null, "interest_category_ids": [], "purchased_reach": null, "is_new_structure": true, "app_download_url": null, "device_price_ranges": [], "pacing": "PACING_MODE_SMOOTH", "purchased_impression": null, "audience_ids": [], "category_exclusion_ids": [], "interest_keyword_ids": [], "delivery_mode": null, "statistic_type": null, "budget": 30, "pixel_id": null, "device_model_ids": [], "optimization_goal": "CLICK", "excluded_audience_ids": [], "share_disabled": false, "location_ids": [6252001], "scheduled_budget": 0, "creative_material_mode": "CUSTOM", "keywords": null, "app_id": null, "brand_safety_type": "NO_BRAND_SAFETY", "advertiser_id": 7002238017842757633, "frequency": null, "languages": [], "age_groups": null, "feed_type": null, "frequency_schedule": null, "placement_type": "PLACEMENT_TYPE_AUTOMATIC", "adgroup_id": 1779923881029666, "promotion_type": "WEBSITE", "rf_estimated_frequency": null, "secondary_optimization_event": null, "create_time": "2023-10-16 14:48:28", "campaign_id": 1779923887578145, "brand_safety_partner": null, "ios14_quota_type": "UNOCCUPIED", "app_type": null, "category_id": 0, "conversion_bid_price": 0, "deep_bid_type": null, "bid_type": "BID_TYPE_NO_BID", "operation_status": "ENABLE", "operating_systems": [], "optimization_event": null, "placements": ["PLACEMENT_TIKTOK", "PLACEMENT_GLOBAL_APP_BUNDLE", "PLACEMENT_PANGLE"], "bid_display_mode": "CPMV", "spending_power": "ALL", "search_result_enabled": true, "deep_cpa_bid": 0, "included_custom_actions": [], "schedule_type": "SCHEDULE_FROM_NOW", "is_smart_performance_campaign": false, "schedule_start_time": "2023-10-16 15:35:45", "actions": []}, "emitted_at": 1698061648761} +{"stream": "ad_groups", "data": {"schedule_infos": null, "adgroup_app_profile_page_state": null, "billing_event": "CPC", "skip_learning_phase": 0, "secondary_status": "ADGROUP_STATUS_CAMPAIGN_DISABLE", "rf_purchased_type": null, "excluded_custom_actions": [], "gender": "GENDER_UNLIMITED", "is_hfss": false, "conversion_window": null, "video_download_disabled": false, "rf_estimated_cpr": null, "budget_mode": "BUDGET_MODE_DAY", "network_types": [], "auto_targeting_enabled": false, "schedule_end_time": "2032-03-25 13:02:23", "adgroup_name": "AdGroupVadim", "inventory_filter_enabled": false, "modify_time": "2023-10-17 14:31:10", "campaign_name": "CampaignVadimTraffic", "bid_price": 0, "dayparting": "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", "comment_disabled": false, "next_day_retention": null, "interest_category_ids": [15], "purchased_reach": null, "is_new_structure": true, "app_download_url": null, "device_price_ranges": [], "pacing": "PACING_MODE_SMOOTH", "purchased_impression": null, "audience_ids": [], "category_exclusion_ids": [], "interest_keyword_ids": [], "delivery_mode": null, "statistic_type": null, "budget": 20, "pixel_id": null, "device_model_ids": [], "optimization_goal": "CLICK", "excluded_audience_ids": [], "share_disabled": false, "location_ids": [6252001], "scheduled_budget": 0, "creative_material_mode": "CUSTOM", "keywords": null, "app_id": null, "brand_safety_type": "NO_BRAND_SAFETY", "advertiser_id": 7002238017842757633, "frequency": null, "languages": [], "age_groups": ["AGE_25_34", "AGE_35_44"], "feed_type": null, "frequency_schedule": null, "placement_type": "PLACEMENT_TYPE_AUTOMATIC", "adgroup_id": 1728545385226289, "promotion_type": "WEBSITE", "rf_estimated_frequency": null, "secondary_optimization_event": null, "create_time": "2022-03-28 12:09:07", "campaign_id": 1728545382536225, "brand_safety_partner": null, "ios14_quota_type": "UNOCCUPIED", "app_type": null, "category_id": 0, "conversion_bid_price": 0, "deep_bid_type": null, "bid_type": "BID_TYPE_NO_BID", "operation_status": "ENABLE", "operating_systems": [], "optimization_event": null, "placements": ["PLACEMENT_TIKTOK", "PLACEMENT_TOPBUZZ", "PLACEMENT_PANGLE"], "bid_display_mode": "CPMV", "search_result_enabled": false, "deep_cpa_bid": 0, "included_custom_actions": [], "schedule_type": "SCHEDULE_FROM_NOW", "is_smart_performance_campaign": false, "schedule_start_time": "2022-03-28 13:02:23", "actions": []}, "emitted_at": 1698061648763} +{"stream": "ad_groups", "data": {"schedule_infos": null, "adgroup_app_profile_page_state": null, "billing_event": "CPC", "skip_learning_phase": 0, "secondary_status": "ADGROUP_STATUS_CAMPAIGN_DISABLE", "rf_purchased_type": null, "excluded_custom_actions": [], "gender": "GENDER_UNLIMITED", "is_hfss": false, "conversion_window": null, "video_download_disabled": false, "rf_estimated_cpr": null, "budget_mode": "BUDGET_MODE_DAY", "network_types": [], "auto_targeting_enabled": false, "schedule_end_time": "2021-10-31 09:01:07", "adgroup_name": "Ad Group20211020010107", "inventory_filter_enabled": false, "modify_time": "2022-03-24 12:06:54", "campaign_name": "Website Traffic20211020010104", "bid_price": 0, "dayparting": "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", "comment_disabled": false, "next_day_retention": null, "interest_category_ids": [], "purchased_reach": null, "is_new_structure": true, "app_download_url": null, "device_price_ranges": [], "pacing": "PACING_MODE_SMOOTH", "purchased_impression": null, "audience_ids": [], "category_exclusion_ids": [], "interest_keyword_ids": [], "delivery_mode": null, "statistic_type": null, "budget": 20, "pixel_id": null, "device_model_ids": [], "optimization_goal": "CLICK", "excluded_audience_ids": [], "share_disabled": false, "location_ids": [6252001], "scheduled_budget": 0, "creative_material_mode": "CUSTOM", "keywords": null, "app_id": null, "brand_safety_type": "NO_BRAND_SAFETY", "advertiser_id": 7002238017842757633, "frequency": null, "languages": ["en"], "age_groups": ["AGE_25_34", "AGE_35_44", "AGE_45_54"], "feed_type": null, "frequency_schedule": null, "placement_type": "PLACEMENT_TYPE_AUTOMATIC", "adgroup_id": 1714125049901106, "promotion_type": "WEBSITE", "rf_estimated_frequency": null, "secondary_optimization_event": null, "create_time": "2021-10-20 08:04:05", "campaign_id": 1714125042508817, "brand_safety_partner": null, "ios14_quota_type": "UNOCCUPIED", "app_type": null, "category_id": 0, "conversion_bid_price": 0, "deep_bid_type": null, "bid_type": "BID_TYPE_NO_BID", "operation_status": "ENABLE", "operating_systems": [], "optimization_event": null, "placements": ["PLACEMENT_TIKTOK", "PLACEMENT_TOPBUZZ", "PLACEMENT_PANGLE"], "bid_display_mode": "CPMV", "search_result_enabled": false, "deep_cpa_bid": 0, "included_custom_actions": [], "schedule_type": "SCHEDULE_START_END", "is_smart_performance_campaign": false, "schedule_start_time": "2021-10-20 09:01:07", "actions": []}, "emitted_at": 1698061648765} +{"stream": "audiences", "data": {"create_time": "2021-10-20 07:26:39", "shared": false, "expired_time": "2022-10-20 07:26:39", "is_expiring": true, "is_creator": true, "is_valid": false, "audience_id": "125451003", "cover_num": 0, "name": "Airbyte2", "audience_type": "Lead Generation", "calculate_type": null}, "emitted_at": 1698061649885} +{"stream": "audiences", "data": {"create_time": "2021-10-20 07:10:04", "shared": false, "expired_time": "2022-10-20 07:10:04", "is_expiring": true, "is_creator": true, "is_valid": false, "audience_id": "125450951", "cover_num": 0, "name": "Airbyte", "audience_type": "Website Audience", "calculate_type": null}, "emitted_at": 1698061649886} +{"stream": "campaigns", "data": {"campaign_id": 1779923887578145, "is_search_campaign": false, "create_time": "2023-10-16 14:48:26", "objective_type": "TRAFFIC", "secondary_status": "CAMPAIGN_STATUS_DISABLE", "app_promotion_type": "UNSET", "operation_status": "DISABLE", "rf_campaign_type": "STANDARD", "campaign_name": "UTM_PARAMSTraffic20231016173112", "campaign_type": "REGULAR_CAMPAIGN", "modify_time": "2023-10-17 14:33:35", "advertiser_id": 7002238017842757633, "is_new_structure": true, "objective": "LANDING_PAGE", "budget_mode": "BUDGET_MODE_INFINITE", "roas_bid": 0, "deep_bid_type": null, "budget": 0, "is_smart_performance_campaign": false}, "emitted_at": 1698061651114} +{"stream": "campaigns", "data": {"campaign_id": 1728545382536225, "is_search_campaign": false, "create_time": "2022-03-28 12:09:05", "objective_type": "TRAFFIC", "secondary_status": "CAMPAIGN_STATUS_DISABLE", "operation_status": "DISABLE", "campaign_name": "CampaignVadimTraffic", "campaign_type": "REGULAR_CAMPAIGN", "modify_time": "2022-03-30 21:23:52", "advertiser_id": 7002238017842757633, "is_new_structure": true, "objective": "LANDING_PAGE", "budget_mode": "BUDGET_MODE_INFINITE", "roas_bid": 0, "deep_bid_type": null, "budget": 0, "is_smart_performance_campaign": false}, "emitted_at": 1698061651115} +{"stream": "campaigns", "data": {"campaign_id": 1714125042508817, "is_search_campaign": false, "create_time": "2021-10-20 08:04:04", "objective_type": "TRAFFIC", "secondary_status": "CAMPAIGN_STATUS_DISABLE", "operation_status": "DISABLE", "campaign_name": "Website Traffic20211020010104", "campaign_type": "REGULAR_CAMPAIGN", "modify_time": "2022-03-24 12:08:29", "advertiser_id": 7002238017842757633, "is_new_structure": true, "objective": "LANDING_PAGE", "budget_mode": "BUDGET_MODE_INFINITE", "roas_bid": 0, "deep_bid_type": null, "budget": 0, "is_smart_performance_campaign": false}, "emitted_at": 1698061651115} +{"stream": "creative_assets_images", "data": {"image_id": "v0201/7f371ff6f0764f8b8ef4f37d7b980d50", "signature": "2fb70f79f5b11f1bf102039bdd9315df", "create_time": "2022-03-28T12:11:34Z", "format": "jpeg", "modify_time": "2022-03-28T12:09:10Z", "displayable": false, "size": 33342, "file_name": "7080121373767221250", "image_url": "https://p19-ad-site-sign-sg.ibyteimg.com/v0201/7f371ff6f0764f8b8ef4f37d7b980d50~tplv-d5opwmad15-image.jpeg?x-expires=2013421654&x-signature=OJAJlm6UitTmAfSoLs8TfwubL30%3D", "height": 1280, "is_carousel_usable": false, "width": 720, "material_id": "7080121373767221250"}, "emitted_at": 1698061654288} +{"stream": "creative_assets_images", "data": {"image_id": "ad-site-i18n-sg/10623349cd96274a8bf6650030e15214", "signature": "f564bd399410d9c271f79f0c785414e1", "create_time": "2022-03-28T11:50:49Z", "format": "jpeg", "modify_time": "2022-03-28T11:50:50Z", "displayable": false, "size": 117823, "file_name": "a2_1648468243469.png", "image_url": "https://p21-ad-sg.ibyteimg.com/obj/ad-site-i18n-sg/10623349cd96274a8bf6650030e15214", "height": 1280, "is_carousel_usable": true, "width": 720, "material_id": "7080116242086625281"}, "emitted_at": 1698061654289} +{"stream": "creative_assets_images", "data": {"image_id": "ad-site-i18n-sg/31e1ea531418f2526783eebea4d43ae3", "signature": "68fa13980c042ab6f998bf771375ae15", "create_time": "2022-03-28T11:42:28Z", "format": "jpeg", "modify_time": "2022-03-28T11:42:29Z", "displayable": false, "size": 216662, "file_name": "air.png", "image_url": "https://p21-ad-sg.ibyteimg.com/obj/ad-site-i18n-sg/31e1ea531418f2526783eebea4d43ae3", "height": 720, "is_carousel_usable": true, "width": 1280, "material_id": "7080114557281419265"}, "emitted_at": 1698061654290} +{"stream": "creative_assets_videos", "data": {"size": 13605359, "signature": "ded8dcda6363d0219764ba5246a673ad", "file_name": "Video16974675945951_Whale, sea, electronica(859574)", "width": 720, "create_time": "2023-10-16T14:46:50Z", "bit_rate": 3626524, "allow_download": true, "modify_time": "2023-10-16T14:46:50Z", "height": 1280, "material_id": "7290567851409981441", "preview_url_expire_time": "2023-10-23 17:48:23", "video_cover_url": "http://p16-sign-sg.tiktokcdn.com/tos-alisg-p-0051c001-sg/oEaaNUsAlIQwDnBGmBLecBzDQGnACMA0Xgbfb5~tplv-noop.image?x-expires=1698083303&x-signature=%2Fq08ZTEll8KgfYXlg%2B1r8oqCgBk%3D", "allowed_placements": ["PLACEMENT_TOPBUZZ", "PLACEMENT_TIKTOK", "PLACEMENT_HELO", "PLACEMENT_PANGLE", "PLACEMENT_GLOBAL_APP_BUNDLE"], "video_id": "v10033g50000ckmkplrc77u30n4dkdb0", "preview_url": "http://v16m-default.akamaized.net/4a8d0298528bfaa92661ed8b31e822b7/6536b1e7/video/tos/alisg/tos-alisg-ve-0051c001-sg/oQABgVatYClIbB2FGEnU0QXmQLTNDzegfWGDs5/?a=0&ch=0&cr=0&dr=0&lr=ad&cd=0%7C0%7C0%7C0&cv=1&br=1630&bt=815&bti=Njs0Zi8tOg%3D%3D&cs=0&ds=3&ft=dl9~j-Inz7T8KqFZiyq8Z&mime_type=video_mp4&qs=0&rc=ZTw5M2c4ZDhoODllM2VmNkBpM2dncTY6Zm9ubjMzODYzNEAxXy40YDUyXzAxYmMwMGItYSNfaDEtcjRvbWpgLS1kMC1zcw%3D%3D&l=202310231147525B364A25545CD403DD31&btag=e00088000", "duration": 30.013, "displayable": true, "format": "mp4"}, "emitted_at": 1698061673747} +{"stream": "creative_assets_videos", "data": {"size": 2150312, "signature": "61c2035644dbcea75ef315af73a120ea", "file_name": "Optimized Version 3_202203281449", "width": 720, "create_time": "2022-03-28T11:49:10Z", "bit_rate": 3430892, "allow_download": true, "modify_time": "2022-03-28T11:49:12Z", "height": 1280, "material_id": "7080116206691549186", "preview_url_expire_time": "2023-10-23 17:47:58", "video_cover_url": "http://p16-sign-sg.tiktokcdn.com/v0201/7f371ff6f0764f8b8ef4f37d7b980d50~tplv-noop.image?x-expires=1698083278&x-signature=1Gq16fQpsQPQsQwk5nzMZoQ5GcY%3D", "allowed_placements": ["PLACEMENT_TOPBUZZ", "PLACEMENT_TIKTOK", "PLACEMENT_HELO", "PLACEMENT_PANGLE", "PLACEMENT_GLOBAL_APP_BUNDLE"], "video_id": "v10033g50000c90q1d3c77ub6e96fvo0", "preview_url": "http://v16m-default.akamaized.net/47236ddfa95a07a9b22fc7b37b96568b/6536b1ce/video/tos/alisg/tos-alisg-v-0000/8968c64a5dc6489a89fe324a6156634d/?a=0&ch=0&cr=0&dr=0&lr=ad&cd=0%7C0%7C0%7C0&cv=1&br=1664&bt=832&bti=Njs0Zi8tOg%3D%3D&cs=0&ds=3&ft=dl9~j-Inz7T8KqFZiyq8Z&mime_type=video_mp4&qs=0&rc=NjY6aDpoNmY0aTRnNmczOEBpM3k5aGU6Zmd0PDMzODYzNEAvYzZiYTAuNTYxMzUxYDZjYSNsYzYzcjQwLi1gLS1kMC1zcw%3D%3D&l=202310231147525B364A25545CD403DD31&btag=e00088000", "duration": 5.014, "displayable": true, "format": "mp4"}, "emitted_at": 1698061673748} +{"stream": "creative_assets_videos", "data": {"size": 2338817, "signature": "f95bfbd2c3d6f6c16d3bd048447b3b73", "file_name": "7021053237617754114", "width": 720, "create_time": "2021-10-20T08:04:10Z", "bit_rate": 1867878, "allow_download": false, "modify_time": "2022-03-28T12:08:49Z", "height": 1280, "material_id": "7021053237617754114", "preview_url_expire_time": "2023-10-23 17:48:03", "video_cover_url": "http://p16-sign-sg.tiktokcdn.com/v0201/8f77082a1f3c40c586f8282356490c58~tplv-noop.image?x-expires=1698083283&x-signature=LCEIxKOuPrFQFqfrpoGE2K6tK00%3D", "allowed_placements": ["PLACEMENT_TOPBUZZ", "PLACEMENT_TIKTOK", "PLACEMENT_HELO", "PLACEMENT_PANGLE", "PLACEMENT_GLOBAL_APP_BUNDLE"], "video_id": "v10033g50000c5nsqcbc77ubdn136b70", "preview_url": "http://v16m-default.akamaized.net/cb4fc19071c911726067bfea11fe06ec/6536b1d3/video/tos/alisg/tos-alisg-v-0000/43f1d52d089a4e428e31dcf356d66906/?a=0&ch=0&cr=0&dr=0&lr=ad&cd=0%7C0%7C0%7C0&cv=1&br=1642&bt=821&bti=Njs0Zi8tOg%3D%3D&cs=0&ds=3&ft=dl9~j-Inz7T8KqFZiyq8Z&mime_type=video_mp4&qs=0&rc=OzpoOWc2OzdpN2hnOjtoOEBpM2U2cWU6ZmZ2ODMzODYzNEAzLTMtNGAuNl8xMmE2MGA1YSM0My5hcjRfbmtgLS1kMC1zcw%3D%3D&l=202310231147525B364A25545CD403DD31&btag=e00088000", "duration": 10.017, "displayable": true, "format": "mp4"}, "emitted_at": 1698061673749} +{"stream": "advertiser_ids", "data": {"advertiser_id": 7001035076276387841, "advertiser_name": "Airbyte0827"}, "emitted_at": 1698061674290} +{"stream": "advertiser_ids", "data": {"advertiser_id": 7001040009704833026, "advertiser_name": "Airbyte08270"}, "emitted_at": 1698061674291} +{"stream": "advertiser_ids", "data": {"advertiser_id": 7002238017842757633, "advertiser_name": "Airbyte0830"}, "emitted_at": 1698061674292} +{"stream": "ads_reports_daily", "data": {"dimensions": {"ad_id": 1714125051115569, "stat_time_day": "2021-10-25 00:00:00"}, "metrics": {"cpm": "3.43", "shares": 0, "real_time_cost_per_result": "0.290", "video_views_p75": 140, "follows": 0, "comments": 0, "mobile_app_id": "0", "tt_app_id": 0, "video_watched_6s": 180, "cost_per_result": "0.290", "average_video_play_per_user": 1.64, "cta_purchase": "0", "real_time_conversion": "0", "real_time_cost_per_conversion": "0.00", "promotion_type": "Website", "video_views_p50": 214, "cost_per_secondary_goal_result": null, "ctr": "1.18", "real_time_result_rate": "1.18", "real_time_app_install_cost": 0, "impressions": "5830", "conversion": "0", "cta_conversion": "0", "placement_type": "Automatic Placement", "profile_visits": 0, "result": "69", "cost_per_1000_reached": "4.16", "video_views_p25": 513, "campaign_id": 1714125042508817, "vta_purchase": "0", "tt_app_name": "0", "onsite_shopping": "0", "total_pageview": "0", "cpc": "0.29", "complete_payment": "0", "dpa_target_audience_type": null, "total_onsite_shopping_value": "0.00", "vta_conversion": "0", "spend": "20.00", "real_time_result": "69", "secondary_goal_result_rate": null, "conversion_rate": "0.00", "secondary_goal_result": null, "adgroup_name": "Ad Group20211020010107", "total_purchase_value": "0.00", "result_rate": "1.18", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "likes": 36, "video_watched_2s": 686, "real_time_app_install": 0, "reach": "4806", "total_complete_payment_rate": "0.00", "clicks": "69", "cost_per_conversion": "0.00", "app_install": 0, "real_time_conversion_rate": "0.00", "video_play_actions": 5173, "value_per_complete_payment": "0.00", "frequency": "1.21", "average_video_play": 1.52, "video_views_p100": 92, "clicks_on_music_disc": 0, "adgroup_id": 1714125049901106, "campaign_name": "Website Traffic20211020010104"}, "stat_time_day": "2021-10-25 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1698062442958} +{"stream": "ads_reports_daily", "data": {"dimensions": {"ad_id": 1714125051115569, "stat_time_day": "2021-10-20 00:00:00"}, "metrics": {"cpm": "5.31", "shares": 0, "real_time_cost_per_result": "0.377", "video_views_p75": 74, "follows": 0, "comments": 1, "mobile_app_id": "0", "tt_app_id": 0, "video_watched_6s": 106, "cost_per_result": "0.377", "average_video_play_per_user": 1.55, "cta_purchase": "0", "real_time_conversion": "0", "real_time_cost_per_conversion": "0.00", "promotion_type": "Website", "video_views_p50": 130, "cost_per_secondary_goal_result": null, "ctr": "1.41", "real_time_result_rate": "1.41", "real_time_app_install_cost": 0, "impressions": "3765", "conversion": "0", "cta_conversion": "0", "placement_type": "Automatic Placement", "profile_visits": 0, "result": "53", "cost_per_1000_reached": "6.38", "video_views_p25": 295, "campaign_id": 1714125042508817, "vta_purchase": "0", "tt_app_name": "0", "onsite_shopping": "0", "total_pageview": "0", "cpc": "0.38", "complete_payment": "0", "dpa_target_audience_type": null, "total_onsite_shopping_value": "0.00", "vta_conversion": "0", "spend": "20.00", "real_time_result": "53", "secondary_goal_result_rate": null, "conversion_rate": "0.00", "secondary_goal_result": null, "adgroup_name": "Ad Group20211020010107", "total_purchase_value": "0.00", "result_rate": "1.41", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "likes": 36, "video_watched_2s": 408, "real_time_app_install": 0, "reach": "3134", "total_complete_payment_rate": "0.00", "clicks": "53", "cost_per_conversion": "0.00", "app_install": 0, "real_time_conversion_rate": "0.00", "video_play_actions": 3344, "value_per_complete_payment": "0.00", "frequency": "1.20", "average_video_play": 1.45, "video_views_p100": 52, "clicks_on_music_disc": 0, "adgroup_id": 1714125049901106, "campaign_name": "Website Traffic20211020010104"}, "stat_time_day": "2021-10-20 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1698062442963} +{"stream": "ads_reports_daily", "data": {"dimensions": {"ad_id": 1714125051115569, "stat_time_day": "2021-10-26 00:00:00"}, "metrics": {"cpm": "5.33", "shares": 0, "real_time_cost_per_result": "0.435", "video_views_p75": 90, "follows": 0, "comments": 1, "mobile_app_id": "0", "tt_app_id": 0, "video_watched_6s": 112, "cost_per_result": "0.435", "average_video_play_per_user": 1.61, "cta_purchase": "0", "real_time_conversion": "0", "real_time_cost_per_conversion": "0.00", "promotion_type": "Website", "video_views_p50": 142, "cost_per_secondary_goal_result": null, "ctr": "1.23", "real_time_result_rate": "1.23", "real_time_app_install_cost": 0, "impressions": "3750", "conversion": "0", "cta_conversion": "0", "placement_type": "Automatic Placement", "profile_visits": 0, "result": "46", "cost_per_1000_reached": "6.41", "video_views_p25": 297, "campaign_id": 1714125042508817, "vta_purchase": "0", "tt_app_name": "0", "onsite_shopping": "0", "total_pageview": "0", "cpc": "0.43", "complete_payment": "0", "dpa_target_audience_type": null, "total_onsite_shopping_value": "0.00", "vta_conversion": "0", "spend": "20.00", "real_time_result": "46", "secondary_goal_result_rate": null, "conversion_rate": "0.00", "secondary_goal_result": null, "adgroup_name": "Ad Group20211020010107", "total_purchase_value": "0.00", "result_rate": "1.23", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "likes": 25, "video_watched_2s": 413, "real_time_app_install": 0, "reach": "3119", "total_complete_payment_rate": "0.00", "clicks": "46", "cost_per_conversion": "0.00", "app_install": 0, "real_time_conversion_rate": "0.00", "video_play_actions": 3344, "value_per_complete_payment": "0.00", "frequency": "1.20", "average_video_play": 1.5, "video_views_p100": 71, "clicks_on_music_disc": 0, "adgroup_id": 1714125049901106, "campaign_name": "Website Traffic20211020010104"}, "stat_time_day": "2021-10-26 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1698062442968} +{"stream": "ads_reports_lifetime", "data": {"metrics": {"result_rate": "0.00", "campaign_name": "UTM_PARAMSTraffic20231016173112", "real_time_app_install": 0, "average_video_play": 0, "adgroup_id": 1779923881029666, "cpc": "0.00", "dpa_target_audience_type": null, "promotion_type": "Website", "complete_payment": "0", "video_views_p25": 0, "tt_app_name": "0", "cta_purchase": "0", "adgroup_name": "Ad group 20231016073545", "likes": 0, "clicks_on_music_disc": 0, "frequency": "0.00", "video_watched_6s": 0, "conversion": "0", "real_time_cost_per_result": "0.000", "cpm": "0.00", "mobile_app_id": "0", "video_play_actions": 0, "cost_per_conversion": "0.00", "comments": 0, "real_time_result_rate": "0.00", "cost_per_result": "0.000", "follows": 0, "result": "0", "impressions": "0", "campaign_id": 1779923887578145, "tt_app_id": 0, "profile_visits": 0, "shares": 0, "vta_conversion": "0", "placement_type": "Automatic Placement", "cost_per_secondary_goal_result": null, "video_watched_2s": 0, "video_views_p50": 0, "reach": "0", "ctr": "0.00", "total_onsite_shopping_value": "0.00", "video_views_p75": 0, "cost_per_1000_reached": "0.00", "cta_conversion": "0", "real_time_conversion": "0", "total_complete_payment_rate": "0.00", "real_time_cost_per_conversion": "0.00", "secondary_goal_result": null, "real_time_result": "0", "secondary_goal_result_rate": null, "real_time_conversion_rate": "0.00", "ad_name": "Video16974675945951_Whale, sea, electronica(859574)_2023-10-16 07:46:35", "video_views_p100": 0, "clicks": "0", "vta_purchase": "0", "app_install": 0, "onsite_shopping": "0", "spend": "0.00", "ad_text": "airbyte", "total_pageview": "0", "real_time_app_install_cost": 0, "value_per_complete_payment": "0.00", "average_video_play_per_user": 0, "conversion_rate": "0.00", "total_purchase_value": "0.00"}, "dimensions": {"ad_id": 1779923894506609}, "ad_id": 1779923894506609}, "emitted_at": 1698062474785} +{"stream": "ads_reports_lifetime", "data": {"metrics": {"result_rate": "0.00", "campaign_name": "UTM_PARAMSTraffic20231016173112", "real_time_app_install": 0, "average_video_play": 0, "adgroup_id": 1779923881029666, "cpc": "0.00", "dpa_target_audience_type": null, "promotion_type": "Website", "complete_payment": "0", "video_views_p25": 0, "tt_app_name": "0", "cta_purchase": "0", "adgroup_name": "Ad group 20231016073545", "likes": 0, "clicks_on_music_disc": 0, "frequency": "0.00", "video_watched_6s": 0, "conversion": "0", "real_time_cost_per_result": "0.000", "cpm": "0.00", "mobile_app_id": "0", "video_play_actions": 0, "cost_per_conversion": "0.00", "comments": 0, "real_time_result_rate": "0.00", "cost_per_result": "0.000", "follows": 0, "result": "0", "impressions": "0", "campaign_id": 1779923887578145, "tt_app_id": 0, "profile_visits": 0, "shares": 0, "vta_conversion": "0", "placement_type": "Automatic Placement", "cost_per_secondary_goal_result": null, "video_watched_2s": 0, "video_views_p50": 0, "reach": "0", "ctr": "0.00", "total_onsite_shopping_value": "0.00", "video_views_p75": 0, "cost_per_1000_reached": "0.00", "cta_conversion": "0", "real_time_conversion": "0", "total_complete_payment_rate": "0.00", "real_time_cost_per_conversion": "0.00", "secondary_goal_result": null, "real_time_result": "0", "secondary_goal_result_rate": null, "real_time_conversion_rate": "0.00", "ad_name": "Video16974675946002_Heartwarming Atmosphere Pops with Piano Main(827850)_2023-10-16 07:46:35", "video_views_p100": 0, "clicks": "0", "vta_purchase": "0", "app_install": 0, "onsite_shopping": "0", "spend": "0.00", "ad_text": "airbyte", "total_pageview": "0", "real_time_app_install_cost": 0, "value_per_complete_payment": "0.00", "average_video_play_per_user": 0, "conversion_rate": "0.00", "total_purchase_value": "0.00"}, "dimensions": {"ad_id": 1779923894511665}, "ad_id": 1779923894511665}, "emitted_at": 1698062474802} +{"stream": "ads_reports_lifetime", "data": {"metrics": {"result_rate": "0.92", "campaign_name": "CampaignVadimTraffic", "real_time_app_install": 0, "average_video_play": 1.26, "adgroup_id": 1728545385226289, "cpc": "0.41", "dpa_target_audience_type": null, "promotion_type": "Website", "complete_payment": "0", "video_views_p25": 3339, "tt_app_name": "0", "cta_purchase": "0", "adgroup_name": "AdGroupVadim", "likes": 11, "clicks_on_music_disc": 0, "frequency": "1.20", "video_watched_6s": 402, "conversion": "0", "real_time_cost_per_result": "0.414", "cpm": "3.82", "mobile_app_id": "0", "video_play_actions": 14333, "cost_per_conversion": "0.00", "comments": 0, "real_time_result_rate": "0.92", "cost_per_result": "0.414", "follows": 0, "result": "145", "impressions": "15689", "campaign_id": 1728545382536225, "tt_app_id": 0, "profile_visits": 0, "shares": 0, "vta_conversion": "0", "placement_type": "Automatic Placement", "cost_per_secondary_goal_result": null, "video_watched_2s": 1364, "video_views_p50": 907, "reach": "13052", "ctr": "0.92", "total_onsite_shopping_value": "0.00", "video_views_p75": 522, "cost_per_1000_reached": "4.60", "cta_conversion": "0", "real_time_conversion": "0", "total_complete_payment_rate": "0.00", "real_time_cost_per_conversion": "0.00", "secondary_goal_result": null, "real_time_result": "145", "secondary_goal_result_rate": null, "real_time_conversion_rate": "0.00", "ad_name": "AdVadim-Optimized Version 3_202203281449_2022-03-28 05:03:44", "video_views_p100": 402, "clicks": "145", "vta_purchase": "0", "app_install": 0, "onsite_shopping": "0", "spend": "60.00", "ad_text": "Open-source\ndata integration for modern data teams", "total_pageview": "0", "real_time_app_install_cost": 0, "value_per_complete_payment": "0.00", "average_video_play_per_user": 1.39, "conversion_rate": "0.00", "total_purchase_value": "0.00"}, "dimensions": {"ad_id": 1728545390695442}, "ad_id": 1728545390695442}, "emitted_at": 1698062474816} +{"stream": "ad_groups_reports_daily", "data": {"metrics": {"shares": 0, "real_time_result": "53", "result": "53", "clicks": "53", "cpc": "0.38", "real_time_result_rate": "1.41", "conversion": "0", "average_video_play_per_user": 1.55, "cost_per_conversion": "0.00", "cost_per_secondary_goal_result": null, "spend": "20.00", "likes": 36, "profile_visits": 0, "clicks_on_music_disc": 0, "video_play_actions": 3344, "secondary_goal_result": null, "tt_app_name": "0", "ctr": "1.41", "promotion_type": "Website", "video_views_p50": 130, "cpm": "5.31", "real_time_app_install_cost": 0, "video_views_p75": 74, "mobile_app_id": "0", "cost_per_1000_reached": "6.38", "video_watched_6s": 106, "average_video_play": 1.45, "adgroup_name": "Ad Group20211020010107", "video_watched_2s": 408, "real_time_conversion_rate": "0.00", "video_views_p100": 52, "placement_type": "Automatic Placement", "conversion_rate": "0.00", "cost_per_result": "0.377", "real_time_cost_per_result": "0.377", "tt_app_id": 0, "secondary_goal_result_rate": null, "dpa_target_audience_type": null, "result_rate": "1.41", "real_time_app_install": 0, "comments": 1, "campaign_name": "Website Traffic20211020010104", "app_install": 0, "real_time_cost_per_conversion": "0.00", "impressions": "3765", "reach": "3134", "real_time_conversion": "0", "follows": 0, "video_views_p25": 295, "frequency": "1.20", "campaign_id": 1714125042508817}, "dimensions": {"adgroup_id": 1714125049901106, "stat_time_day": "2021-10-20 00:00:00"}, "stat_time_day": "2021-10-20 00:00:00", "adgroup_id": 1714125049901106}, "emitted_at": 1698063197412} +{"stream": "ad_groups_reports_daily", "data": {"metrics": {"shares": 0, "real_time_result": "69", "result": "69", "clicks": "69", "cpc": "0.29", "real_time_result_rate": "1.18", "conversion": "0", "average_video_play_per_user": 1.64, "cost_per_conversion": "0.00", "cost_per_secondary_goal_result": null, "spend": "20.00", "likes": 36, "profile_visits": 0, "clicks_on_music_disc": 0, "video_play_actions": 5173, "secondary_goal_result": null, "tt_app_name": "0", "ctr": "1.18", "promotion_type": "Website", "video_views_p50": 214, "cpm": "3.43", "real_time_app_install_cost": 0, "video_views_p75": 140, "mobile_app_id": "0", "cost_per_1000_reached": "4.16", "video_watched_6s": 180, "average_video_play": 1.52, "adgroup_name": "Ad Group20211020010107", "video_watched_2s": 686, "real_time_conversion_rate": "0.00", "video_views_p100": 92, "placement_type": "Automatic Placement", "conversion_rate": "0.00", "cost_per_result": "0.290", "real_time_cost_per_result": "0.290", "tt_app_id": 0, "secondary_goal_result_rate": null, "dpa_target_audience_type": null, "result_rate": "1.18", "real_time_app_install": 0, "comments": 0, "campaign_name": "Website Traffic20211020010104", "app_install": 0, "real_time_cost_per_conversion": "0.00", "impressions": "5830", "reach": "4806", "real_time_conversion": "0", "follows": 0, "video_views_p25": 513, "frequency": "1.21", "campaign_id": 1714125042508817}, "dimensions": {"adgroup_id": 1714125049901106, "stat_time_day": "2021-10-25 00:00:00"}, "stat_time_day": "2021-10-25 00:00:00", "adgroup_id": 1714125049901106}, "emitted_at": 1698063197417} +{"stream": "ad_groups_reports_daily", "data": {"metrics": {"shares": 0, "real_time_result": "40", "result": "40", "clicks": "40", "cpc": "0.50", "real_time_result_rate": "0.91", "conversion": "0", "average_video_play_per_user": 1.5, "cost_per_conversion": "0.00", "cost_per_secondary_goal_result": null, "spend": "20.00", "likes": 13, "profile_visits": 0, "clicks_on_music_disc": 0, "video_play_actions": 3852, "secondary_goal_result": null, "tt_app_name": "0", "ctr": "0.91", "promotion_type": "Website", "video_views_p50": 130, "cpm": "4.55", "real_time_app_install_cost": 0, "video_views_p75": 85, "mobile_app_id": "0", "cost_per_1000_reached": "5.52", "video_watched_6s": 104, "average_video_play": 1.41, "adgroup_name": "Ad Group20211020010107", "video_watched_2s": 436, "real_time_conversion_rate": "0.00", "video_views_p100": 66, "placement_type": "Automatic Placement", "conversion_rate": "0.00", "cost_per_result": "0.500", "real_time_cost_per_result": "0.500", "tt_app_id": 0, "secondary_goal_result_rate": null, "dpa_target_audience_type": null, "result_rate": "0.91", "real_time_app_install": 0, "comments": 0, "campaign_name": "Website Traffic20211020010104", "app_install": 0, "real_time_cost_per_conversion": "0.00", "impressions": "4394", "reach": "3621", "real_time_conversion": "0", "follows": 0, "video_views_p25": 306, "frequency": "1.21", "campaign_id": 1714125042508817}, "dimensions": {"adgroup_id": 1714125049901106, "stat_time_day": "2021-10-29 00:00:00"}, "stat_time_day": "2021-10-29 00:00:00", "adgroup_id": 1714125049901106}, "emitted_at": 1698063197422} +{"stream": "ad_groups_reports_lifetime", "data": {"metrics": {"cpc": "0.00", "likes": 0, "clicks_on_music_disc": 0, "result": "0", "placement_type": "Automatic Placement", "real_time_app_install": 0, "cost_per_secondary_goal_result": null, "clicks": "0", "average_video_play": 0, "video_views_p75": 0, "cost_per_1000_reached": "0.00", "result_rate": "0.00", "average_video_play_per_user": 0, "video_watched_2s": 0, "frequency": "0.00", "video_play_actions": 0, "spend": "0.00", "promotion_type": "Website", "app_install": 0, "mobile_app_id": "0", "secondary_goal_result": null, "real_time_result_rate": "0.00", "real_time_app_install_cost": 0, "secondary_goal_result_rate": null, "comments": 0, "tt_app_name": "0", "conversion": "0", "video_views_p100": 0, "cpm": "0.00", "cost_per_conversion": "0.00", "real_time_conversion_rate": "0.00", "tt_app_id": 0, "cost_per_result": "0.000", "campaign_id": 1779923887578145, "video_watched_6s": 0, "profile_visits": 0, "real_time_conversion": "0", "video_views_p50": 0, "campaign_name": "UTM_PARAMSTraffic20231016173112", "ctr": "0.00", "adgroup_name": "Ad group 20231016073545", "real_time_result": "0", "shares": 0, "real_time_cost_per_result": "0.000", "impressions": "0", "conversion_rate": "0.00", "reach": "0", "real_time_cost_per_conversion": "0.00", "video_views_p25": 0, "dpa_target_audience_type": null, "follows": 0}, "dimensions": {"adgroup_id": 1779923881029666}, "adgroup_id": 1779923881029666}, "emitted_at": 1698063226007} +{"stream": "ad_groups_reports_lifetime", "data": {"metrics": {"cpc": "0.41", "likes": 11, "clicks_on_music_disc": 0, "result": "145", "placement_type": "Automatic Placement", "real_time_app_install": 0, "cost_per_secondary_goal_result": null, "clicks": "145", "average_video_play": 1.26, "video_views_p75": 522, "cost_per_1000_reached": "4.60", "result_rate": "0.92", "average_video_play_per_user": 1.39, "video_watched_2s": 1364, "frequency": "1.20", "video_play_actions": 14333, "spend": "60.00", "promotion_type": "Website", "app_install": 0, "mobile_app_id": "0", "secondary_goal_result": null, "real_time_result_rate": "0.92", "real_time_app_install_cost": 0, "secondary_goal_result_rate": null, "comments": 0, "tt_app_name": "0", "conversion": "0", "video_views_p100": 402, "cpm": "3.82", "cost_per_conversion": "0.00", "real_time_conversion_rate": "0.00", "tt_app_id": 0, "cost_per_result": "0.414", "campaign_id": 1728545382536225, "video_watched_6s": 402, "profile_visits": 0, "real_time_conversion": "0", "video_views_p50": 907, "campaign_name": "CampaignVadimTraffic", "ctr": "0.92", "adgroup_name": "AdGroupVadim", "real_time_result": "145", "shares": 0, "real_time_cost_per_result": "0.414", "impressions": "15689", "conversion_rate": "0.00", "reach": "13052", "real_time_cost_per_conversion": "0.00", "video_views_p25": 3339, "dpa_target_audience_type": null, "follows": 0}, "dimensions": {"adgroup_id": 1728545385226289}, "adgroup_id": 1728545385226289}, "emitted_at": 1698063226012} +{"stream": "ad_groups_reports_lifetime", "data": {"metrics": {"cpc": "0.37", "likes": 263, "clicks_on_music_disc": 0, "result": "540", "placement_type": "Automatic Placement", "real_time_app_install": 0, "cost_per_secondary_goal_result": null, "clicks": "540", "average_video_play": 1.48, "video_views_p75": 998, "cost_per_1000_reached": "5.96", "result_rate": "1.17", "average_video_play_per_user": 1.8, "video_watched_2s": 5100, "frequency": "1.37", "video_play_actions": 40753, "spend": "200.00", "promotion_type": "Website", "app_install": 0, "mobile_app_id": "0", "secondary_goal_result": null, "real_time_result_rate": "1.17", "real_time_app_install_cost": 0, "secondary_goal_result_rate": null, "comments": 2, "tt_app_name": "0", "conversion": "0", "video_views_p100": 723, "cpm": "4.34", "cost_per_conversion": "0.00", "real_time_conversion_rate": "0.00", "tt_app_id": 0, "cost_per_result": "0.370", "campaign_id": 1714125042508817, "video_watched_6s": 1295, "profile_visits": 0, "real_time_conversion": "0", "video_views_p50": 1588, "campaign_name": "Website Traffic20211020010104", "ctr": "1.17", "adgroup_name": "Ad Group20211020010107", "real_time_result": "540", "shares": 0, "real_time_cost_per_result": "0.370", "impressions": "46116", "conversion_rate": "0.00", "reach": "33556", "real_time_cost_per_conversion": "0.00", "video_views_p25": 3674, "dpa_target_audience_type": null, "follows": 0}, "dimensions": {"adgroup_id": 1714125049901106}, "adgroup_id": 1714125049901106}, "emitted_at": 1698063226016} +{"stream": "campaigns_reports_daily", "data": {"metrics": {"video_watched_2s": 493, "profile_visits": 0, "ctr": "1.09", "likes": 18, "video_play_actions": 4179, "shares": 0, "cpc": "0.39", "cpm": "4.26", "video_views_p75": 108, "video_views_p100": 76, "real_time_app_install_cost": 0, "video_views_p25": 355, "spend": "20.00", "video_watched_6s": 132, "reach": "3822", "impressions": "4696", "real_time_app_install": 0, "campaign_name": "Website Traffic20211020010104", "app_install": 0, "video_views_p50": 164, "comments": 0, "follows": 0, "cost_per_1000_reached": "5.23", "average_video_play": 1.48, "average_video_play_per_user": 1.61, "clicks": "51", "clicks_on_music_disc": 0, "frequency": "1.23"}, "dimensions": {"campaign_id": 1714125042508817, "stat_time_day": "2021-10-27 00:00:00"}, "stat_time_day": "2021-10-27 00:00:00", "campaign_id": 1714125042508817}, "emitted_at": 1698063912579} +{"stream": "campaigns_reports_daily", "data": {"metrics": {"video_watched_2s": 390, "profile_visits": 0, "ctr": "1.19", "likes": 17, "video_play_actions": 3118, "shares": 0, "cpc": "0.48", "cpm": "5.68", "video_views_p75": 74, "video_views_p100": 59, "real_time_app_install_cost": 0, "video_views_p25": 277, "spend": "20.00", "video_watched_6s": 92, "reach": "2908", "impressions": "3520", "real_time_app_install": 0, "campaign_name": "Website Traffic20211020010104", "app_install": 0, "video_views_p50": 112, "comments": 0, "follows": 0, "cost_per_1000_reached": "6.88", "average_video_play": 1.46, "average_video_play_per_user": 1.57, "clicks": "42", "clicks_on_music_disc": 0, "frequency": "1.21"}, "dimensions": {"campaign_id": 1714125042508817, "stat_time_day": "2021-10-22 00:00:00"}, "stat_time_day": "2021-10-22 00:00:00", "campaign_id": 1714125042508817}, "emitted_at": 1698063912584} +{"stream": "campaigns_reports_daily", "data": {"metrics": {"video_watched_2s": 471, "profile_visits": 0, "ctr": "1.00", "likes": 18, "video_play_actions": 4253, "shares": 0, "cpc": "0.42", "cpm": "4.18", "video_views_p75": 100, "video_views_p100": 70, "real_time_app_install_cost": 0, "video_views_p25": 328, "spend": "20.00", "video_watched_6s": 120, "reach": "3938", "impressions": "4787", "real_time_app_install": 0, "campaign_name": "Website Traffic20211020010104", "app_install": 0, "video_views_p50": 144, "comments": 0, "follows": 0, "cost_per_1000_reached": "5.08", "average_video_play": 1.42, "average_video_play_per_user": 1.54, "clicks": "48", "clicks_on_music_disc": 0, "frequency": "1.22"}, "dimensions": {"campaign_id": 1714125042508817, "stat_time_day": "2021-10-28 00:00:00"}, "stat_time_day": "2021-10-28 00:00:00", "campaign_id": 1714125042508817}, "emitted_at": 1698063912590} +{"stream": "campaigns_reports_lifetime", "data": {"metrics": {"impressions": "0", "cpc": "0.00", "video_watched_2s": 0, "video_views_p25": 0, "video_views_p50": 0, "ctr": "0.00", "video_play_actions": 0, "profile_visits": 0, "app_install": 0, "average_video_play": 0, "video_views_p100": 0, "video_watched_6s": 0, "clicks_on_music_disc": 0, "comments": 0, "frequency": "0.00", "follows": 0, "real_time_app_install_cost": 0, "video_views_p75": 0, "cost_per_1000_reached": "0.00", "shares": 0, "real_time_app_install": 0, "cpm": "0.00", "average_video_play_per_user": 0, "likes": 0, "campaign_name": "UTM_PARAMSTraffic20231016173112", "clicks": "0", "reach": "0", "spend": "0.00"}, "dimensions": {"campaign_id": 1779923887578145}, "campaign_id": 1779923887578145}, "emitted_at": 1698063939714} +{"stream": "campaigns_reports_lifetime", "data": {"metrics": {"impressions": "15689", "cpc": "0.41", "video_watched_2s": 1364, "video_views_p25": 3339, "video_views_p50": 907, "ctr": "0.92", "video_play_actions": 14333, "profile_visits": 0, "app_install": 0, "average_video_play": 1.26, "video_views_p100": 402, "video_watched_6s": 402, "clicks_on_music_disc": 0, "comments": 0, "frequency": "1.20", "follows": 0, "real_time_app_install_cost": 0, "video_views_p75": 522, "cost_per_1000_reached": "4.60", "shares": 0, "real_time_app_install": 0, "cpm": "3.82", "average_video_play_per_user": 1.39, "likes": 11, "campaign_name": "CampaignVadimTraffic", "clicks": "145", "reach": "13052", "spend": "60.00"}, "dimensions": {"campaign_id": 1728545382536225}, "campaign_id": 1728545382536225}, "emitted_at": 1698063939718} +{"stream": "campaigns_reports_lifetime", "data": {"metrics": {"impressions": "46116", "cpc": "0.37", "video_watched_2s": 5100, "video_views_p25": 3674, "video_views_p50": 1588, "ctr": "1.17", "video_play_actions": 40753, "profile_visits": 0, "app_install": 0, "average_video_play": 1.48, "video_views_p100": 723, "video_watched_6s": 1295, "clicks_on_music_disc": 0, "comments": 2, "frequency": "1.37", "follows": 0, "real_time_app_install_cost": 0, "video_views_p75": 998, "cost_per_1000_reached": "5.96", "shares": 0, "real_time_app_install": 0, "cpm": "4.34", "average_video_play_per_user": 1.8, "likes": 263, "campaign_name": "Website Traffic20211020010104", "clicks": "540", "reach": "33556", "spend": "200.00"}, "dimensions": {"campaign_id": 1714125042508817}, "campaign_id": 1714125042508817}, "emitted_at": 1698063939723} +{"stream": "advertisers_reports_daily", "data": {"metrics": {"cpm": "4.18", "impressions": "4787", "video_watched_6s": 120, "profile_visits": 0, "average_video_play_per_user": 1.54, "shares": 0, "app_install": 0, "clicks_on_music_disc": 0, "likes": 18, "cpc": "0.42", "real_time_app_install": 0, "average_video_play": 1.42, "cash_spend": "20.00", "reach": "3938", "real_time_app_install_cost": 0, "video_play_actions": 4253, "video_views_p75": 100, "video_views_p100": 70, "video_watched_2s": 471, "comments": 0, "cost_per_1000_reached": "5.08", "follows": 0, "spend": "20.00", "ctr": "1.00", "video_views_p25": 328, "frequency": "1.22", "video_views_p50": 144, "voucher_spend": "0.00", "clicks": "48"}, "dimensions": {"advertiser_id": 7002238017842757633, "stat_time_day": "2021-10-28 00:00:00"}, "stat_time_day": "2021-10-28 00:00:00", "advertiser_id": 7002238017842757633}, "emitted_at": 1698064649491} +{"stream": "advertisers_reports_daily", "data": {"metrics": {"cpm": "4.91", "impressions": "4077", "video_watched_6s": 124, "profile_visits": 0, "average_video_play_per_user": 1.65, "shares": 0, "app_install": 0, "clicks_on_music_disc": 0, "likes": 19, "cpc": "0.30", "real_time_app_install": 0, "average_video_play": 1.53, "cash_spend": "20.00", "reach": "3322", "real_time_app_install_cost": 0, "video_play_actions": 3590, "video_views_p75": 95, "video_views_p100": 65, "video_watched_2s": 463, "comments": 0, "cost_per_1000_reached": "6.02", "follows": 0, "spend": "20.00", "ctr": "1.64", "video_views_p25": 338, "frequency": "1.23", "video_views_p50": 146, "voucher_spend": "0.00", "clicks": "67"}, "dimensions": {"advertiser_id": 7002238017842757633, "stat_time_day": "2021-10-23 00:00:00"}, "stat_time_day": "2021-10-23 00:00:00", "advertiser_id": 7002238017842757633}, "emitted_at": 1698064649496} +{"stream": "advertisers_reports_daily", "data": {"metrics": {"cpm": "5.33", "impressions": "3750", "video_watched_6s": 112, "profile_visits": 0, "average_video_play_per_user": 1.61, "shares": 0, "app_install": 0, "clicks_on_music_disc": 0, "likes": 25, "cpc": "0.43", "real_time_app_install": 0, "average_video_play": 1.5, "cash_spend": "20.00", "reach": "3119", "real_time_app_install_cost": 0, "video_play_actions": 3344, "video_views_p75": 90, "video_views_p100": 71, "video_watched_2s": 413, "comments": 1, "cost_per_1000_reached": "6.41", "follows": 0, "spend": "20.00", "ctr": "1.23", "video_views_p25": 297, "frequency": "1.20", "video_views_p50": 142, "voucher_spend": "0.00", "clicks": "46"}, "dimensions": {"advertiser_id": 7002238017842757633, "stat_time_day": "2021-10-26 00:00:00"}, "stat_time_day": "2021-10-26 00:00:00", "advertiser_id": 7002238017842757633}, "emitted_at": 1698064649500} +{"stream": "advertisers_reports_lifetime", "data": {"metrics": {"impressions": "66691", "cpc": "0.37", "video_watched_2s": 6941, "video_views_p25": 7364, "video_views_p50": 2665, "ctr": "1.12", "video_play_actions": 59390, "profile_visits": 0, "app_install": 0, "average_video_play": 1.43, "video_views_p100": 1205, "video_watched_6s": 1838, "clicks_on_music_disc": 0, "comments": 2, "frequency": "1.32", "follows": 0, "real_time_app_install_cost": 0, "video_views_p75": 1636, "cost_per_1000_reached": "5.55", "shares": 0, "real_time_app_install": 0, "cpm": "4.20", "average_video_play_per_user": 1.68, "likes": 328, "clicks": "750", "reach": "50418", "spend": "280.00"}, "dimensions": {"advertiser_id": 7002238017842757633}, "advertiser_id": 7002238017842757633}, "emitted_at": 1698064675577} +{"stream": "advertisers_reports_lifetime", "data": {"dimensions": {"advertiser_id": 7002238017842757633}, "metrics": {"follows": 0, "video_watched_2s": 220, "cpc": "0.15", "frequency": "1.01", "video_views_p75": 12, "average_video_play": 1.26, "shares": 0, "clicks_on_music_disc": 0, "spend": "10.00", "video_views_p25": 53, "impressions": "2725", "likes": 0, "real_time_app_install": 0, "cpm": "3.67", "reach": "2687", "video_watched_6s": 67, "app_install": 0, "clicks": "66", "video_play_actions": 2695, "video_views_p100": 9, "ctr": "2.42", "average_video_play_per_user": 1.27, "profile_visits": 0, "video_views_p50": 22, "comments": 0, "real_time_app_install_cost": 0, "cost_per_1000_reached": "3.72"}, "advertiser_id": 7002238017842757633}, "emitted_at": 1698064677941} +{"stream": "ads_audience_reports_daily", "data": {"dimensions": {"ad_id": 1714125051115569, "gender": "MALE", "age": "AGE_45_54", "stat_time_day": "2021-10-29 00:00:00"}, "metrics": {"campaign_name": "Website Traffic20211020010104", "clicks": "3", "conversion_rate": "0.00", "conversion": "0", "result_rate": "0.98", "dpa_target_audience_type": null, "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "impressions": "305", "real_time_result": "3", "promotion_type": "Website", "adgroup_id": 1714125049901106, "real_time_cost_per_conversion": "0.00", "cost_per_conversion": "0.00", "tt_app_id": "0", "real_time_conversion": "0", "campaign_id": 1714125042508817, "cpm": "5.02", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "cpc": "0.51", "real_time_conversion_rate": "0.00", "ctr": "0.98", "tt_app_name": "0", "real_time_cost_per_result": "0.510", "cost_per_result": "0.510", "adgroup_name": "Ad Group20211020010107", "result": "3", "placement_type": "Automatic Placement", "real_time_result_rate": "0.98", "spend": "1.53", "mobile_app_id": "0"}, "stat_time_day": "2021-10-29 00:00:00", "ad_id": 1714125051115569, "gender": "MALE", "age": "AGE_45_54"}, "emitted_at": 1698064682391} +{"stream": "ads_audience_reports_daily", "data": {"dimensions": {"ad_id": 1714073085256738, "gender": "MALE", "age": "AGE_18_24", "stat_time_day": "2021-10-20 00:00:00"}, "metrics": {"campaign_name": "Website Traffic20211019110444", "clicks": "0", "conversion_rate": "0.00", "conversion": "0", "result_rate": "0.00", "dpa_target_audience_type": null, "ad_text": "Open Source ETL", "impressions": "1", "real_time_result": "0", "promotion_type": "Website", "adgroup_id": 1714073022392322, "real_time_cost_per_conversion": "0.00", "cost_per_conversion": "0.00", "tt_app_id": "0", "real_time_conversion": "0", "campaign_id": 1714073078669329, "cpm": "0.00", "ad_name": "Optimized Version 1_202110192111_2021-10-19 21:11:39", "cpc": "0.00", "real_time_conversion_rate": "0.00", "ctr": "0.00", "tt_app_name": "0", "real_time_cost_per_result": "0.000", "cost_per_result": "0.000", "adgroup_name": "Ad Group20211019111040", "result": "0", "placement_type": "Automatic Placement", "real_time_result_rate": "0.00", "spend": "0.00", "mobile_app_id": "0"}, "stat_time_day": "2021-10-20 00:00:00", "ad_id": 1714073085256738, "gender": "MALE", "age": "AGE_18_24"}, "emitted_at": 1698064682394} +{"stream": "ads_audience_reports_daily", "data": {"dimensions": {"ad_id": 1714125051115569, "gender": "MALE", "age": "AGE_35_44", "stat_time_day": "2021-10-21 00:00:00"}, "metrics": {"campaign_name": "Website Traffic20211020010104", "clicks": "5", "conversion_rate": "0.00", "conversion": "0", "result_rate": "1.10", "dpa_target_audience_type": null, "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "impressions": "454", "real_time_result": "5", "promotion_type": "Website", "adgroup_id": 1714125049901106, "real_time_cost_per_conversion": "0.00", "cost_per_conversion": "0.00", "tt_app_id": "0", "real_time_conversion": "0", "campaign_id": 1714125042508817, "cpm": "5.02", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "cpc": "0.46", "real_time_conversion_rate": "0.00", "ctr": "1.10", "tt_app_name": "0", "real_time_cost_per_result": "0.456", "cost_per_result": "0.456", "adgroup_name": "Ad Group20211020010107", "result": "5", "placement_type": "Automatic Placement", "real_time_result_rate": "1.10", "spend": "2.28", "mobile_app_id": "0"}, "stat_time_day": "2021-10-21 00:00:00", "ad_id": 1714125051115569, "gender": "MALE", "age": "AGE_35_44"}, "emitted_at": 1698064682397} +{"stream": "ads_audience_reports_by_province_daily", "data": {"dimensions": {"ad_id": 1714125051115569, "province_id": "5279468", "stat_time_day": "2021-10-26 00:00:00"}, "metrics": {"ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "real_time_conversion": "0", "mobile_app_id": "0", "impressions": "26", "adgroup_id": 1714125049901106, "real_time_cost_per_conversion": "0.00", "tt_app_id": "0", "ctr": "0.00", "cpc": "0.00", "campaign_id": 1714125042508817, "result": "0", "conversion": "0", "conversion_rate": "0.00", "clicks": "0", "real_time_conversion_rate": "0.00", "adgroup_name": "Ad Group20211020010107", "tt_app_name": "0", "promotion_type": "Website", "real_time_result": "0", "cost_per_conversion": "0.00", "real_time_result_rate": "0.00", "cost_per_result": "0.000", "dpa_target_audience_type": null, "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "result_rate": "0.00", "spend": "0.01", "cpm": "0.38", "campaign_name": "Website Traffic20211020010104", "placement_type": "Automatic Placement", "real_time_cost_per_result": "0.000"}, "stat_time_day": "2021-10-26 00:00:00", "ad_id": 1714125051115569, "province_id": "5279468"}, "emitted_at": 1698064768612} +{"stream": "ads_audience_reports_by_province_daily", "data": {"dimensions": {"ad_id": 1714125051115569, "province_id": "5596512", "stat_time_day": "2021-10-29 00:00:00"}, "metrics": {"ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "real_time_conversion": "0", "mobile_app_id": "0", "impressions": "0", "adgroup_id": 1714125049901106, "real_time_cost_per_conversion": "0.00", "tt_app_id": "0", "ctr": "0.00", "cpc": "0.00", "campaign_id": 1714125042508817, "result": "0", "conversion": "0", "conversion_rate": "0.00", "clicks": "0", "real_time_conversion_rate": "0.00", "adgroup_name": "Ad Group20211020010107", "tt_app_name": "0", "promotion_type": "Website", "real_time_result": "0", "cost_per_conversion": "0.00", "real_time_result_rate": "0.00", "cost_per_result": "0.000", "dpa_target_audience_type": null, "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "result_rate": "0.00", "spend": "0.00", "cpm": "0.00", "campaign_name": "Website Traffic20211020010104", "placement_type": "Automatic Placement", "real_time_cost_per_result": "0.000"}, "stat_time_day": "2021-10-29 00:00:00", "ad_id": 1714125051115569, "province_id": "5596512"}, "emitted_at": 1698064768615} +{"stream": "ads_audience_reports_by_province_daily", "data": {"dimensions": {"ad_id": 1714125051115569, "province_id": "4138106", "stat_time_day": "2021-10-28 00:00:00"}, "metrics": {"ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "real_time_conversion": "0", "mobile_app_id": "0", "impressions": "0", "adgroup_id": 1714125049901106, "real_time_cost_per_conversion": "0.00", "tt_app_id": "0", "ctr": "0.00", "cpc": "0.00", "campaign_id": 1714125042508817, "result": "0", "conversion": "0", "conversion_rate": "0.00", "clicks": "0", "real_time_conversion_rate": "0.00", "adgroup_name": "Ad Group20211020010107", "tt_app_name": "0", "promotion_type": "Website", "real_time_result": "0", "cost_per_conversion": "0.00", "real_time_result_rate": "0.00", "cost_per_result": "0.000", "dpa_target_audience_type": null, "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "result_rate": "0.00", "spend": "0.00", "cpm": "0.00", "campaign_name": "Website Traffic20211020010104", "placement_type": "Automatic Placement", "real_time_cost_per_result": "0.000"}, "stat_time_day": "2021-10-28 00:00:00", "ad_id": 1714125051115569, "province_id": "4138106"}, "emitted_at": 1698064768619} +{"stream": "ad_group_audience_reports_daily", "data": {"metrics": {"tt_app_name": "0", "real_time_conversion": "0", "conversion": "0", "promotion_type": "Website", "conversion_rate": "0.00", "cpm": "7.66", "result_rate": "1.91", "mobile_app_id": "0", "clicks": "12", "spend": "4.81", "cost_per_conversion": "0.00", "campaign_id": 1714125042508817, "ctr": "1.91", "tt_app_id": "0", "campaign_name": "Website Traffic20211020010104", "adgroup_name": "Ad Group20211020010107", "cpc": "0.40", "dpa_target_audience_type": null, "real_time_cost_per_result": "0.401", "cost_per_result": "0.401", "real_time_conversion_rate": "0.00", "impressions": "628", "result": "12", "real_time_result_rate": "1.91", "real_time_result": "12", "real_time_cost_per_conversion": "0.00", "placement_type": "Automatic Placement"}, "dimensions": {"age": "AGE_35_44", "adgroup_id": 1714125049901106, "stat_time_day": "2021-10-26 00:00:00", "gender": "FEMALE"}, "stat_time_day": "2021-10-26 00:00:00", "adgroup_id": 1714125049901106, "gender": "FEMALE", "age": "AGE_35_44"}, "emitted_at": 1698064814719} +{"stream": "ad_group_audience_reports_daily", "data": {"metrics": {"tt_app_name": "0", "real_time_conversion": "0", "conversion": "0", "promotion_type": "Website", "conversion_rate": "0.00", "cpm": "3.91", "result_rate": "0.82", "mobile_app_id": "0", "clicks": "12", "spend": "5.69", "cost_per_conversion": "0.00", "campaign_id": 1714125042508817, "ctr": "0.82", "tt_app_id": "0", "campaign_name": "Website Traffic20211020010104", "adgroup_name": "Ad Group20211020010107", "cpc": "0.47", "dpa_target_audience_type": null, "real_time_cost_per_result": "0.474", "cost_per_result": "0.474", "real_time_conversion_rate": "0.00", "impressions": "1455", "result": "12", "real_time_result_rate": "0.82", "real_time_result": "12", "real_time_cost_per_conversion": "0.00", "placement_type": "Automatic Placement"}, "dimensions": {"age": "AGE_25_34", "adgroup_id": 1714125049901106, "stat_time_day": "2021-10-29 00:00:00", "gender": "FEMALE"}, "stat_time_day": "2021-10-29 00:00:00", "adgroup_id": 1714125049901106, "gender": "FEMALE", "age": "AGE_25_34"}, "emitted_at": 1698064814723} +{"stream": "ad_group_audience_reports_daily", "data": {"metrics": {"tt_app_name": "0", "real_time_conversion": "0", "conversion": "0", "promotion_type": "Website", "conversion_rate": "0.00", "cpm": "7.84", "result_rate": "1.63", "mobile_app_id": "0", "clicks": "4", "spend": "1.92", "cost_per_conversion": "0.00", "campaign_id": 1714125042508817, "ctr": "1.63", "tt_app_id": "0", "campaign_name": "Website Traffic20211020010104", "adgroup_name": "Ad Group20211020010107", "cpc": "0.48", "dpa_target_audience_type": null, "real_time_cost_per_result": "0.480", "cost_per_result": "0.480", "real_time_conversion_rate": "0.00", "impressions": "245", "result": "4", "real_time_result_rate": "1.63", "real_time_result": "4", "real_time_cost_per_conversion": "0.00", "placement_type": "Automatic Placement"}, "dimensions": {"age": "AGE_45_54", "adgroup_id": 1714125049901106, "stat_time_day": "2021-10-22 00:00:00", "gender": "MALE"}, "stat_time_day": "2021-10-22 00:00:00", "adgroup_id": 1714125049901106, "gender": "MALE", "age": "AGE_45_54"}, "emitted_at": 1698064814726} +{"stream": "campaigns_audience_reports_by_country_daily", "data": {"metrics": {"cpc": "0.31", "impressions": "4874", "cpm": "4.10", "spend": "20.00", "clicks": "65", "campaign_name": "Website Traffic20211019110444", "ctr": "1.33"}, "dimensions": {"country_code": "US", "campaign_id": 1714073078669329, "stat_time_day": "2021-10-19 00:00:00"}, "stat_time_day": "2021-10-19 00:00:00", "campaign_id": 1714073078669329, "country_code": "US"}, "emitted_at": 1698064909477} +{"stream": "campaigns_audience_reports_by_country_daily", "data": {"metrics": {"cpc": "0.38", "impressions": "3765", "cpm": "5.31", "spend": "20.00", "clicks": "53", "campaign_name": "Website Traffic20211020010104", "ctr": "1.41"}, "dimensions": {"country_code": "US", "campaign_id": 1714125042508817, "stat_time_day": "2021-10-20 00:00:00"}, "stat_time_day": "2021-10-20 00:00:00", "campaign_id": 1714125042508817, "country_code": "US"}, "emitted_at": 1698064909481} +{"stream": "campaigns_audience_reports_by_country_daily", "data": {"metrics": {"cpc": "0.00", "impressions": "12", "cpm": "0.00", "spend": "0.00", "clicks": "0", "campaign_name": "Website Traffic20211019110444", "ctr": "0.00"}, "dimensions": {"country_code": "US", "campaign_id": 1714073078669329, "stat_time_day": "2021-10-20 00:00:00"}, "stat_time_day": "2021-10-20 00:00:00", "campaign_id": 1714073078669329, "country_code": "US"}, "emitted_at": 1698064909484} +{"stream": "advertisers_audience_reports_daily", "data": {"metrics": {"cpc": "0.38", "clicks": "6", "spend": "2.26", "cpm": "6.75", "ctr": "1.79", "impressions": "335"}, "dimensions": {"advertiser_id": 7002238017842757633, "age": "AGE_18_24", "gender": "MALE", "stat_time_day": "2021-10-19 00:00:00"}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_18_24"}, "emitted_at": 1698064951205} +{"stream": "advertisers_audience_reports_daily", "data": {"metrics": {"cpc": "0.00", "clicks": "1", "spend": "0.00", "cpm": "0.00", "ctr": "2.86", "impressions": "35"}, "dimensions": {"advertiser_id": 7002238017842757633, "age": "AGE_35_44", "gender": "MALE", "stat_time_day": "2021-10-19 00:00:00"}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_35_44"}, "emitted_at": 1698064951208} +{"stream": "advertisers_audience_reports_daily", "data": {"metrics": {"cpc": "0.29", "clicks": "29", "spend": "8.32", "cpm": "3.88", "ctr": "1.35", "impressions": "2146"}, "dimensions": {"advertiser_id": 7002238017842757633, "age": "AGE_13_17", "gender": "FEMALE", "stat_time_day": "2021-10-19 00:00:00"}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "gender": "FEMALE", "age": "AGE_13_17"}, "emitted_at": 1698064951211} +{"stream": "advertisers_audience_reports_lifetime", "data": {"metrics": {"cpm": "4.58", "cpc": "0.36", "spend": "31.56", "ctr": "1.26", "clicks": "87", "impressions": "6897"}, "dimensions": {"age": "AGE_35_44", "advertiser_id": 7002238017842757633, "gender": "MALE"}, "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_35_44"}, "emitted_at": 1698064974164} +{"stream": "advertisers_audience_reports_lifetime", "data": {"metrics": {"cpm": "0.00", "cpc": "0.00", "spend": "0.00", "ctr": "0.00", "clicks": "0", "impressions": "17"}, "dimensions": {"age": "AGE_55_100", "advertiser_id": 7002238017842757633, "gender": "FEMALE"}, "advertiser_id": 7002238017842757633, "gender": "FEMALE", "age": "AGE_55_100"}, "emitted_at": 1698064974168} +{"stream": "advertisers_audience_reports_lifetime", "data": {"metrics": {"cpm": "3.92", "cpc": "0.32", "spend": "7.13", "ctr": "1.21", "clicks": "22", "impressions": "1818"}, "dimensions": {"age": "AGE_13_17", "advertiser_id": 7002238017842757633, "gender": "MALE"}, "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_13_17"}, "emitted_at": 1698064974171} +{"stream": "ad_group_audience_reports_by_country_daily", "data": {"metrics": {"campaign_id": 1714073078669329, "clicks": "65", "conversion": "0", "result": "65", "impressions": "4874", "cpc": "0.31", "real_time_result": "65", "real_time_conversion_rate": "0.00", "real_time_conversion": "0", "real_time_cost_per_conversion": "0.00", "dpa_target_audience_type": null, "spend": "20.00", "cost_per_result": "0.308", "real_time_cost_per_result": "0.308", "cost_per_conversion": "0.00", "tt_app_name": "0", "cpm": "4.10", "result_rate": "1.33", "placement_type": "Automatic Placement", "conversion_rate": "0.00", "mobile_app_id": "0", "campaign_name": "Website Traffic20211019110444", "tt_app_id": "0", "promotion_type": "Website", "real_time_result_rate": "1.33", "ctr": "1.33", "adgroup_name": "Ad Group20211019111040"}, "dimensions": {"stat_time_day": "2021-10-19 00:00:00", "country_code": "US", "adgroup_id": 1714073022392322}, "stat_time_day": "2021-10-19 00:00:00", "adgroup_id": 1714073022392322, "country_code": "US"}, "emitted_at": 1698913315398} +{"stream": "ad_group_audience_reports_by_country_daily", "data": {"metrics": {"campaign_id": 1714125042508817, "clicks": "69", "conversion": "0", "result": "69", "impressions": "5830", "cpc": "0.29", "real_time_result": "69", "real_time_conversion_rate": "0.00", "real_time_conversion": "0", "real_time_cost_per_conversion": "0.00", "dpa_target_audience_type": null, "spend": "20.00", "cost_per_result": "0.290", "real_time_cost_per_result": "0.290", "cost_per_conversion": "0.00", "tt_app_name": "0", "cpm": "3.43", "result_rate": "1.18", "placement_type": "Automatic Placement", "conversion_rate": "0.00", "mobile_app_id": "0", "campaign_name": "Website Traffic20211020010104", "tt_app_id": "0", "promotion_type": "Website", "real_time_result_rate": "1.18", "ctr": "1.18", "adgroup_name": "Ad Group20211020010107"}, "dimensions": {"stat_time_day": "2021-10-25 00:00:00", "country_code": "US", "adgroup_id": 1714125049901106}, "stat_time_day": "2021-10-25 00:00:00", "adgroup_id": 1714125049901106, "country_code": "US"}, "emitted_at": 1698913315402} +{"stream": "ad_group_audience_reports_by_country_daily", "data": {"metrics": {"campaign_id": 1714125042508817, "clicks": "53", "conversion": "0", "result": "53", "impressions": "3765", "cpc": "0.38", "real_time_result": "53", "real_time_conversion_rate": "0.00", "real_time_conversion": "0", "real_time_cost_per_conversion": "0.00", "dpa_target_audience_type": null, "spend": "20.00", "cost_per_result": "0.377", "real_time_cost_per_result": "0.377", "cost_per_conversion": "0.00", "tt_app_name": "0", "cpm": "5.31", "result_rate": "1.41", "placement_type": "Automatic Placement", "conversion_rate": "0.00", "mobile_app_id": "0", "campaign_name": "Website Traffic20211020010104", "tt_app_id": "0", "promotion_type": "Website", "real_time_result_rate": "1.41", "ctr": "1.41", "adgroup_name": "Ad Group20211020010107"}, "dimensions": {"stat_time_day": "2021-10-20 00:00:00", "country_code": "US", "adgroup_id": 1714125049901106}, "stat_time_day": "2021-10-20 00:00:00", "adgroup_id": 1714125049901106, "country_code": "US"}, "emitted_at": 1698913315406} +{"stream": "ad_group_audience_reports_by_platform_daily", "data": {"dimensions": {"platform": "ANDROID", "stat_time_day": "2021-10-25 00:00:00", "adgroup_id": 1714125049901106}, "metrics": {"mobile_app_id": "0", "spend": "0.00", "real_time_cost_per_result": "0.000", "real_time_result": "0", "cost_per_conversion": "0.00", "cpm": "0.00", "adgroup_name": "Ad Group20211020010107", "conversion_rate": "0.00", "clicks": "0", "campaign_name": "Website Traffic20211020010104", "dpa_target_audience_type": null, "ctr": "0.00", "tt_app_name": "0", "placement_type": "Automatic Placement", "result_rate": "0.00", "conversion": "0", "promotion_type": "Website", "real_time_conversion": "0", "real_time_cost_per_conversion": "0.00", "campaign_id": 1714125042508817, "real_time_conversion_rate": "0.00", "impressions": "5", "tt_app_id": "0", "result": "0", "real_time_result_rate": "0.00", "cpc": "0.00", "cost_per_result": "0.000"}, "stat_time_day": "2021-10-25 00:00:00", "adgroup_id": 1714125049901106, "platform": "ANDROID"}, "emitted_at": 1698913290573} +{"stream": "ad_group_audience_reports_by_platform_daily", "data": {"dimensions": {"platform": "ANDROID", "stat_time_day": "2021-10-20 00:00:00", "adgroup_id": 1714125049901106}, "metrics": {"mobile_app_id": "0", "spend": "0.00", "real_time_cost_per_result": "0.000", "real_time_result": "0", "cost_per_conversion": "0.00", "cpm": "0.00", "adgroup_name": "Ad Group20211020010107", "conversion_rate": "0.00", "clicks": "0", "campaign_name": "Website Traffic20211020010104", "dpa_target_audience_type": null, "ctr": "0.00", "tt_app_name": "0", "placement_type": "Automatic Placement", "result_rate": "0.00", "conversion": "0", "promotion_type": "Website", "real_time_conversion": "0", "real_time_cost_per_conversion": "0.00", "campaign_id": 1714125042508817, "real_time_conversion_rate": "0.00", "impressions": "10", "tt_app_id": "0", "result": "0", "real_time_result_rate": "0.00", "cpc": "0.00", "cost_per_result": "0.000"}, "stat_time_day": "2021-10-20 00:00:00", "adgroup_id": 1714125049901106, "platform": "ANDROID"}, "emitted_at": 1698913290580} +{"stream": "ad_group_audience_reports_by_platform_daily", "data": {"dimensions": {"platform": "IPAD", "stat_time_day": "2021-10-29 00:00:00", "adgroup_id": 1714125049901106}, "metrics": {"mobile_app_id": "0", "spend": "6.77", "real_time_cost_per_result": "0.521", "real_time_result": "13", "cost_per_conversion": "0.00", "cpm": "5.05", "adgroup_name": "Ad Group20211020010107", "conversion_rate": "0.00", "clicks": "13", "campaign_name": "Website Traffic20211020010104", "dpa_target_audience_type": null, "ctr": "0.97", "tt_app_name": "0", "placement_type": "Automatic Placement", "result_rate": "0.97", "conversion": "0", "promotion_type": "Website", "real_time_conversion": "0", "real_time_cost_per_conversion": "0.00", "campaign_id": 1714125042508817, "real_time_conversion_rate": "0.00", "impressions": "1340", "tt_app_id": "0", "result": "13", "real_time_result_rate": "0.97", "cpc": "0.52", "cost_per_result": "0.521"}, "stat_time_day": "2021-10-29 00:00:00", "adgroup_id": 1714125049901106, "platform": "IPAD"}, "emitted_at": 1698913290584} +{"stream": "ads_audience_reports_by_country_daily", "data": {"dimensions": {"country_code": "US", "ad_id": 1714073085256738, "stat_time_day": "2021-10-20 00:00:00"}, "metrics": {"spend": "0.00", "real_time_result_rate": "0.00", "result_rate": "0.00", "real_time_conversion_rate": "0.00", "ad_name": "Optimized Version 1_202110192111_2021-10-19 21:11:39", "impressions": "12", "cpc": "0.00", "campaign_id": 1714073078669329, "dpa_target_audience_type": null, "cost_per_conversion": "0.00", "conversion": "0", "result": "0", "adgroup_id": 1714073022392322, "adgroup_name": "Ad Group20211019111040", "real_time_cost_per_result": "0.000", "real_time_conversion": "0", "real_time_result": "0", "clicks": "0", "placement_type": "Automatic Placement", "tt_app_id": "0", "conversion_rate": "0.00", "mobile_app_id": "0", "cpm": "0.00", "real_time_cost_per_conversion": "0.00", "cost_per_result": "0.000", "campaign_name": "Website Traffic20211019110444", "ad_text": "Open Source ETL", "tt_app_name": "0", "promotion_type": "Website", "ctr": "0.00"}, "stat_time_day": "2021-10-20 00:00:00", "ad_id": 1714073085256738, "country_code": "US"}, "emitted_at": 1698913236298} +{"stream": "ads_audience_reports_by_country_daily", "data": {"dimensions": {"country_code": "US", "ad_id": 1714125051115569, "stat_time_day": "2021-10-28 00:00:00"}, "metrics": {"spend": "20.00", "real_time_result_rate": "1.00", "result_rate": "1.00", "real_time_conversion_rate": "0.00", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "impressions": "4787", "cpc": "0.42", "campaign_id": 1714125042508817, "dpa_target_audience_type": null, "cost_per_conversion": "0.00", "conversion": "0", "result": "48", "adgroup_id": 1714125049901106, "adgroup_name": "Ad Group20211020010107", "real_time_cost_per_result": "0.417", "real_time_conversion": "0", "real_time_result": "48", "clicks": "48", "placement_type": "Automatic Placement", "tt_app_id": "0", "conversion_rate": "0.00", "mobile_app_id": "0", "cpm": "4.18", "real_time_cost_per_conversion": "0.00", "cost_per_result": "0.417", "campaign_name": "Website Traffic20211020010104", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "tt_app_name": "0", "promotion_type": "Website", "ctr": "1.00"}, "stat_time_day": "2021-10-28 00:00:00", "ad_id": 1714125051115569, "country_code": "US"}, "emitted_at": 1698913236302} +{"stream": "ads_audience_reports_by_country_daily", "data": {"dimensions": {"country_code": "US", "ad_id": 1714125051115569, "stat_time_day": "2021-10-23 00:00:00"}, "metrics": {"spend": "20.00", "real_time_result_rate": "1.64", "result_rate": "1.64", "real_time_conversion_rate": "0.00", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "impressions": "4077", "cpc": "0.30", "campaign_id": 1714125042508817, "dpa_target_audience_type": null, "cost_per_conversion": "0.00", "conversion": "0", "result": "67", "adgroup_id": 1714125049901106, "adgroup_name": "Ad Group20211020010107", "real_time_cost_per_result": "0.299", "real_time_conversion": "0", "real_time_result": "67", "clicks": "67", "placement_type": "Automatic Placement", "tt_app_id": "0", "conversion_rate": "0.00", "mobile_app_id": "0", "cpm": "4.91", "real_time_cost_per_conversion": "0.00", "cost_per_result": "0.299", "campaign_name": "Website Traffic20211020010104", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "tt_app_name": "0", "promotion_type": "Website", "ctr": "1.64"}, "stat_time_day": "2021-10-23 00:00:00", "ad_id": 1714125051115569, "country_code": "US"}, "emitted_at": 1698913236305} +{"stream": "ads_audience_reports_by_platform_daily", "data": {"dimensions": {"platform": "ANDROID", "stat_time_day": "2021-10-28 00:00:00", "ad_id": 1714125051115569}, "metrics": {"result": "0", "real_time_cost_per_conversion": "0.00", "tt_app_name": "0", "dpa_target_audience_type": null, "conversion": "0", "result_rate": "0.00", "cost_per_conversion": "0.00", "real_time_conversion": "0", "clicks": "0", "spend": "0.00", "real_time_conversion_rate": "0.00", "real_time_result": "0", "impressions": "3", "campaign_name": "Website Traffic20211020010104", "cpm": "0.00", "campaign_id": 1714125042508817, "cost_per_result": "0.000", "promotion_type": "Website", "mobile_app_id": "0", "real_time_cost_per_result": "0.000", "real_time_result_rate": "0.00", "tt_app_id": "0", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "ctr": "0.00", "cpc": "0.00", "adgroup_id": 1714125049901106, "placement_type": "Automatic Placement", "conversion_rate": "0.00", "adgroup_name": "Ad Group20211020010107", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!"}, "stat_time_day": "2021-10-28 00:00:00", "ad_id": 1714125051115569, "platform": "ANDROID"}, "emitted_at": 1698913365906} +{"stream": "ads_audience_reports_by_platform_daily", "data": {"dimensions": {"platform": "ANDROID", "stat_time_day": "2021-10-23 00:00:00", "ad_id": 1714125051115569}, "metrics": {"result": "0", "real_time_cost_per_conversion": "0.00", "tt_app_name": "0", "dpa_target_audience_type": null, "conversion": "0", "result_rate": "0.00", "cost_per_conversion": "0.00", "real_time_conversion": "0", "clicks": "0", "spend": "0.00", "real_time_conversion_rate": "0.00", "real_time_result": "0", "impressions": "1", "campaign_name": "Website Traffic20211020010104", "cpm": "0.00", "campaign_id": 1714125042508817, "cost_per_result": "0.000", "promotion_type": "Website", "mobile_app_id": "0", "real_time_cost_per_result": "0.000", "real_time_result_rate": "0.00", "tt_app_id": "0", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "ctr": "0.00", "cpc": "0.00", "adgroup_id": 1714125049901106, "placement_type": "Automatic Placement", "conversion_rate": "0.00", "adgroup_name": "Ad Group20211020010107", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!"}, "stat_time_day": "2021-10-23 00:00:00", "ad_id": 1714125051115569, "platform": "ANDROID"}, "emitted_at": 1698913365908} +{"stream": "ads_audience_reports_by_platform_daily", "data": {"dimensions": {"platform": "IPAD", "stat_time_day": "2021-10-27 00:00:00", "ad_id": 1714125051115569}, "metrics": {"result": "18", "real_time_cost_per_conversion": "0.00", "tt_app_name": "0", "dpa_target_audience_type": null, "conversion": "0", "result_rate": "1.27", "cost_per_conversion": "0.00", "real_time_conversion": "0", "clicks": "18", "spend": "6.99", "real_time_conversion_rate": "0.00", "real_time_result": "18", "impressions": "1412", "campaign_name": "Website Traffic20211020010104", "cpm": "4.95", "campaign_id": 1714125042508817, "cost_per_result": "0.388", "promotion_type": "Website", "mobile_app_id": "0", "real_time_cost_per_result": "0.388", "real_time_result_rate": "1.27", "tt_app_id": "0", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "ctr": "1.27", "cpc": "0.39", "adgroup_id": 1714125049901106, "placement_type": "Automatic Placement", "conversion_rate": "0.00", "adgroup_name": "Ad Group20211020010107", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!"}, "stat_time_day": "2021-10-27 00:00:00", "ad_id": 1714125051115569, "platform": "IPAD"}, "emitted_at": 1698913365909} +{"stream": "advertisers_audience_reports_by_country_daily", "data": {"metrics": {"cpc": "0.31", "cpm": "4.10", "clicks": "65", "spend": "20.00", "impressions": "4874", "ctr": "1.33"}, "dimensions": {"country_code": "US", "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "country_code": "US"}, "emitted_at": 1698913221257} +{"stream": "advertisers_audience_reports_by_country_daily", "data": {"metrics": {"cpc": "0.38", "cpm": "5.30", "clicks": "53", "spend": "20.00", "impressions": "3777", "ctr": "1.40"}, "dimensions": {"country_code": "US", "stat_time_day": "2021-10-20 00:00:00", "advertiser_id": 7002238017842757633}, "stat_time_day": "2021-10-20 00:00:00", "advertiser_id": 7002238017842757633, "country_code": "US"}, "emitted_at": 1698913221263} +{"stream": "advertisers_audience_reports_by_country_daily", "data": {"metrics": {"cpc": "0.44", "cpm": "5.03", "clicks": "45", "spend": "20.00", "impressions": "3977", "ctr": "1.13"}, "dimensions": {"country_code": "US", "stat_time_day": "2021-10-21 00:00:00", "advertiser_id": 7002238017842757633}, "stat_time_day": "2021-10-21 00:00:00", "advertiser_id": 7002238017842757633, "country_code": "US"}, "emitted_at": 1698913221267} +{"stream": "advertisers_audience_reports_by_platform_daily", "data": {"dimensions": {"stat_time_day": "2021-10-19 00:00:00", "platform": "IPHONE", "advertiser_id": 7002238017842757633}, "metrics": {"spend": "14.33", "clicks": "45", "impressions": "3549", "cpc": "0.32", "ctr": "1.27", "cpm": "4.04"}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "platform": "IPHONE"}, "emitted_at": 1698913340224} +{"stream": "advertisers_audience_reports_by_platform_daily", "data": {"dimensions": {"stat_time_day": "2021-10-19 00:00:00", "platform": "IPAD", "advertiser_id": 7002238017842757633}, "metrics": {"spend": "5.67", "clicks": "20", "impressions": "1298", "cpc": "0.28", "ctr": "1.54", "cpm": "4.37"}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "platform": "IPAD"}, "emitted_at": 1698913340226} +{"stream": "advertisers_audience_reports_by_platform_daily", "data": {"dimensions": {"stat_time_day": "2021-10-19 00:00:00", "platform": "ANDROID", "advertiser_id": 7002238017842757633}, "metrics": {"spend": "0.00", "clicks": "0", "impressions": "27", "cpc": "0.00", "ctr": "0.00", "cpm": "0.00"}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "platform": "ANDROID"}, "emitted_at": 1698913340227} +{"stream": "campaigns_audience_reports_by_platform_daily", "data": {"metrics": {"clicks": "45", "campaign_name": "Website Traffic20211019110444", "cpm": "4.04", "cpc": "0.32", "impressions": "3549", "ctr": "1.27", "spend": "14.33"}, "dimensions": {"campaign_id": 1714073078669329, "stat_time_day": "2021-10-19 00:00:00", "platform": "IPHONE"}, "stat_time_day": "2021-10-19 00:00:00", "campaign_id": 1714073078669329, "platform": "IPHONE"}, "emitted_at": 1698913264525} +{"stream": "campaigns_audience_reports_by_platform_daily", "data": {"metrics": {"clicks": "20", "campaign_name": "Website Traffic20211019110444", "cpm": "4.37", "cpc": "0.28", "impressions": "1298", "ctr": "1.54", "spend": "5.67"}, "dimensions": {"campaign_id": 1714073078669329, "stat_time_day": "2021-10-19 00:00:00", "platform": "IPAD"}, "stat_time_day": "2021-10-19 00:00:00", "campaign_id": 1714073078669329, "platform": "IPAD"}, "emitted_at": 1698913264532} +{"stream": "campaigns_audience_reports_by_platform_daily", "data": {"metrics": {"clicks": "0", "campaign_name": "Website Traffic20211019110444", "cpm": "0.00", "cpc": "0.00", "impressions": "27", "ctr": "0.00", "spend": "0.00"}, "dimensions": {"campaign_id": 1714073078669329, "stat_time_day": "2021-10-19 00:00:00", "platform": "ANDROID"}, "stat_time_day": "2021-10-19 00:00:00", "campaign_id": 1714073078669329, "platform": "ANDROID"}, "emitted_at": 1698913264537} +{"stream": "campaigns_audience_reports_daily", "data": {"dimensions": {"gender": "MALE", "campaign_id": 1714073078669329, "stat_time_day": "2021-10-19 00:00:00", "age": "AGE_18_24"}, "metrics": {"campaign_name": "Website Traffic20211019110444", "cpm": "6.75", "cpc": "0.38", "clicks": "6", "spend": "2.26", "ctr": "1.79", "impressions": "335"}, "stat_time_day": "2021-10-19 00:00:00", "campaign_id": 1714073078669329, "gender": "MALE", "age": "AGE_18_24"}, "emitted_at": 1698913199242} +{"stream": "campaigns_audience_reports_daily", "data": {"dimensions": {"gender": "MALE", "campaign_id": 1714073078669329, "stat_time_day": "2021-10-19 00:00:00", "age": "AGE_35_44"}, "metrics": {"campaign_name": "Website Traffic20211019110444", "cpm": "0.00", "cpc": "0.00", "clicks": "1", "spend": "0.00", "ctr": "2.86", "impressions": "35"}, "stat_time_day": "2021-10-19 00:00:00", "campaign_id": 1714073078669329, "gender": "MALE", "age": "AGE_35_44"}, "emitted_at": 1698913199250} +{"stream": "campaigns_audience_reports_daily", "data": {"dimensions": {"gender": "FEMALE", "campaign_id": 1714073078669329, "stat_time_day": "2021-10-19 00:00:00", "age": "AGE_13_17"}, "metrics": {"campaign_name": "Website Traffic20211019110444", "cpm": "3.88", "cpc": "0.29", "clicks": "29", "spend": "8.32", "ctr": "1.35", "impressions": "2146"}, "stat_time_day": "2021-10-19 00:00:00", "campaign_id": 1714073078669329, "gender": "FEMALE", "age": "AGE_13_17"}, "emitted_at": 1698913199254} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records2.jsonl b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records2.jsonl index 1e6c7a91d969..c01f830dbef7 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records2.jsonl +++ b/airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/expected_records2.jsonl @@ -1,40 +1,32 @@ -{"stream": "ads_reports", "data": {"metrics": {"cpm": "3.430", "real_time_result": "69", "vta_conversion": "0", "real_time_conversion": "0", "comments": "0", "impressions": "5830", "campaign_id": 1714125042508817, "real_time_result_rate": "1.18", "campaign_name": "Website Traffic20211020010104", "cpc": "0.290", "app_install": "0", "placement_type": "Automatic Placement", "tt_app_name": "0", "dpa_target_audience_type": null, "real_time_conversion_rate": "0.00", "video_views_p75": "140", "video_views_p100": "92", "clicks": "69", "cost_per_result": "0.2899", "real_time_cost_per_result": "0.2899", "reach": "4806", "secondary_goal_result_rate": null, "total_onsite_shopping_value": "0.000", "frequency": "1.21", "shares": "0", "vta_purchase": "0", "real_time_cost_per_conversion": "0.000", "tt_app_id": 0, "result_rate": "1.18", "value_per_complete_payment": "0.000", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "average_video_play_per_user": "1.64", "adgroup_name": "Ad Group20211020010107", "total_pageview": "0", "total_complete_payment_rate": "0.000", "real_time_app_install_cost": "0.000", "cta_conversion": "0", "clicks_on_music_disc": "0", "likes": "36", "secondary_goal_result": null, "ctr": "1.18", "video_watched_2s": "686", "video_views_p50": "214", "video_watched_6s": "180", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "follows": "0", "onsite_shopping": "0", "complete_payment": "0", "cost_per_secondary_goal_result": null, "conversion_rate": "0.00", "mobile_app_id": "0", "profile_visits": "0", "conversion": "0", "promotion_type": "Website", "average_video_play": "1.52", "adgroup_id": 1714125049901106, "total_purchase_value": "0.000", "cost_per_conversion": "0.000", "cta_purchase": "0", "video_views_p25": "513", "result": "69", "real_time_app_install": "0", "spend": "20.000", "cost_per_1000_reached": "4.161", "video_play_actions": "5173"}, "dimensions": {"ad_id": 1714125051115569, "stat_time_day": "2021-10-25 00:00:00"}, "stat_time_day": "2021-10-25 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1690215563238} -{"stream": "ads_reports", "data": {"metrics": {"cpm": "5.310", "real_time_result": "53", "vta_conversion": "0", "real_time_conversion": "0", "comments": "1", "impressions": "3765", "campaign_id": 1714125042508817, "real_time_result_rate": "1.41", "campaign_name": "Website Traffic20211020010104", "cpc": "0.380", "app_install": "0", "placement_type": "Automatic Placement", "tt_app_name": "0", "dpa_target_audience_type": null, "real_time_conversion_rate": "0.00", "video_views_p75": "74", "video_views_p100": "52", "clicks": "53", "cost_per_result": "0.3774", "real_time_cost_per_result": "0.3774", "reach": "3134", "secondary_goal_result_rate": null, "total_onsite_shopping_value": "0.000", "frequency": "1.20", "shares": "0", "vta_purchase": "0", "real_time_cost_per_conversion": "0.000", "tt_app_id": 0, "result_rate": "1.41", "value_per_complete_payment": "0.000", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "average_video_play_per_user": "1.55", "adgroup_name": "Ad Group20211020010107", "total_pageview": "0", "total_complete_payment_rate": "0.000", "real_time_app_install_cost": "0.000", "cta_conversion": "0", "clicks_on_music_disc": "0", "likes": "36", "secondary_goal_result": null, "ctr": "1.41", "video_watched_2s": "408", "video_views_p50": "130", "video_watched_6s": "106", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "follows": "0", "onsite_shopping": "0", "complete_payment": "0", "cost_per_secondary_goal_result": null, "conversion_rate": "0.00", "mobile_app_id": "0", "profile_visits": "0", "conversion": "0", "promotion_type": "Website", "average_video_play": "1.45", "adgroup_id": 1714125049901106, "total_purchase_value": "0.000", "cost_per_conversion": "0.000", "cta_purchase": "0", "video_views_p25": "295", "result": "53", "real_time_app_install": "0", "spend": "20.000", "cost_per_1000_reached": "6.382", "video_play_actions": "3344"}, "dimensions": {"ad_id": 1714125051115569, "stat_time_day": "2021-10-20 00:00:00"}, "stat_time_day": "2021-10-20 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1690215563242} -{"stream": "ads_reports", "data": {"metrics": {"cpm": "4.910", "real_time_result": "67", "vta_conversion": "0", "real_time_conversion": "0", "comments": "0", "impressions": "4077", "campaign_id": 1714125042508817, "real_time_result_rate": "1.64", "campaign_name": "Website Traffic20211020010104", "cpc": "0.300", "app_install": "0", "placement_type": "Automatic Placement", "tt_app_name": "0", "dpa_target_audience_type": null, "real_time_conversion_rate": "0.00", "video_views_p75": "95", "video_views_p100": "65", "clicks": "67", "cost_per_result": "0.2985", "real_time_cost_per_result": "0.2985", "reach": "3322", "secondary_goal_result_rate": null, "total_onsite_shopping_value": "0.000", "frequency": "1.23", "shares": "0", "vta_purchase": "0", "real_time_cost_per_conversion": "0.000", "tt_app_id": 0, "result_rate": "1.64", "value_per_complete_payment": "0.000", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "average_video_play_per_user": "1.65", "adgroup_name": "Ad Group20211020010107", "total_pageview": "0", "total_complete_payment_rate": "0.000", "real_time_app_install_cost": "0.000", "cta_conversion": "0", "clicks_on_music_disc": "0", "likes": "19", "secondary_goal_result": null, "ctr": "1.64", "video_watched_2s": "463", "video_views_p50": "146", "video_watched_6s": "124", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "follows": "0", "onsite_shopping": "0", "complete_payment": "0", "cost_per_secondary_goal_result": null, "conversion_rate": "0.00", "mobile_app_id": "0", "profile_visits": "0", "conversion": "0", "promotion_type": "Website", "average_video_play": "1.53", "adgroup_id": 1714125049901106, "total_purchase_value": "0.000", "cost_per_conversion": "0.000", "cta_purchase": "0", "video_views_p25": "338", "result": "67", "real_time_app_install": "0", "spend": "20.000", "cost_per_1000_reached": "6.020", "video_play_actions": "3590"}, "dimensions": {"ad_id": 1714125051115569, "stat_time_day": "2021-10-23 00:00:00"}, "stat_time_day": "2021-10-23 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1690215563247} -{"stream": "ads_reports", "data": {"metrics": {"cpm": "5.330", "real_time_result": "46", "vta_conversion": "0", "real_time_conversion": "0", "comments": "1", "impressions": "3750", "campaign_id": 1714125042508817, "real_time_result_rate": "1.23", "campaign_name": "Website Traffic20211020010104", "cpc": "0.430", "app_install": "0", "placement_type": "Automatic Placement", "tt_app_name": "0", "dpa_target_audience_type": null, "real_time_conversion_rate": "0.00", "video_views_p75": "90", "video_views_p100": "71", "clicks": "46", "cost_per_result": "0.4348", "real_time_cost_per_result": "0.4348", "reach": "3119", "secondary_goal_result_rate": null, "total_onsite_shopping_value": "0.000", "frequency": "1.20", "shares": "0", "vta_purchase": "0", "real_time_cost_per_conversion": "0.000", "tt_app_id": 0, "result_rate": "1.23", "value_per_complete_payment": "0.000", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "average_video_play_per_user": "1.61", "adgroup_name": "Ad Group20211020010107", "total_pageview": "0", "total_complete_payment_rate": "0.000", "real_time_app_install_cost": "0.000", "cta_conversion": "0", "clicks_on_music_disc": "0", "likes": "25", "secondary_goal_result": null, "ctr": "1.23", "video_watched_2s": "413", "video_views_p50": "142", "video_watched_6s": "112", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "follows": "0", "onsite_shopping": "0", "complete_payment": "0", "cost_per_secondary_goal_result": null, "conversion_rate": "0.00", "mobile_app_id": "0", "profile_visits": "0", "conversion": "0", "promotion_type": "Website", "average_video_play": "1.50", "adgroup_id": 1714125049901106, "total_purchase_value": "0.000", "cost_per_conversion": "0.000", "cta_purchase": "0", "video_views_p25": "297", "result": "46", "real_time_app_install": "0", "spend": "20.000", "cost_per_1000_reached": "6.412", "video_play_actions": "3344"}, "dimensions": {"ad_id": 1714125051115569, "stat_time_day": "2021-10-26 00:00:00"}, "stat_time_day": "2021-10-26 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1690215563249} -{"stream": "ads_reports", "data": {"metrics": {"cpm": "5.030", "real_time_result": "45", "vta_conversion": "0", "real_time_conversion": "0", "comments": "0", "impressions": "3977", "campaign_id": 1714125042508817, "real_time_result_rate": "1.13", "campaign_name": "Website Traffic20211020010104", "cpc": "0.440", "app_install": "0", "placement_type": "Automatic Placement", "tt_app_name": "0", "dpa_target_audience_type": null, "real_time_conversion_rate": "0.00", "video_views_p75": "77", "video_views_p100": "54", "clicks": "45", "cost_per_result": "0.4444", "real_time_cost_per_result": "0.4444", "reach": "3227", "secondary_goal_result_rate": null, "total_onsite_shopping_value": "0.000", "frequency": "1.23", "shares": "0", "vta_purchase": "0", "real_time_cost_per_conversion": "0.000", "tt_app_id": 0, "result_rate": "1.13", "value_per_complete_payment": "0.000", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "average_video_play_per_user": "1.57", "adgroup_name": "Ad Group20211020010107", "total_pageview": "0", "total_complete_payment_rate": "0.000", "real_time_app_install_cost": "0.000", "cta_conversion": "0", "clicks_on_music_disc": "0", "likes": "25", "secondary_goal_result": null, "ctr": "1.13", "video_watched_2s": "422", "video_views_p50": "128", "video_watched_6s": "107", "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "follows": "0", "onsite_shopping": "0", "complete_payment": "0", "cost_per_secondary_goal_result": null, "conversion_rate": "0.00", "mobile_app_id": "0", "profile_visits": "0", "conversion": "0", "promotion_type": "Website", "average_video_play": "1.46", "adgroup_id": 1714125049901106, "total_purchase_value": "0.000", "cost_per_conversion": "0.000", "cta_purchase": "0", "video_views_p25": "301", "result": "45", "real_time_app_install": "0", "spend": "20.000", "cost_per_1000_reached": "6.198", "video_play_actions": "3460"}, "dimensions": {"ad_id": 1714125051115569, "stat_time_day": "2021-10-21 00:00:00"}, "stat_time_day": "2021-10-21 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1690215563252} -{"stream":"ad_groups_reports","data":{"metrics":{"dpa_target_audience_type":null,"average_video_play":"1.45","conversion_rate":"0.00","mobile_app_id":"0","cost_per_result":"0.3774","video_views_p50":"130","comments":"1","video_watched_6s":"106","placement_type":"Automatic Placement","secondary_goal_result":null,"shares":"0","app_install":"0","result_rate":"1.41","result":"53","real_time_app_install":"0","likes":"36","video_views_p100":"52","impressions":"3765","profile_visits":"0","adgroup_name":"Ad Group20211020010107","clicks_on_music_disc":"0","cost_per_conversion":"0.000","video_views_p75":"74","cpc":"0.380","average_video_play_per_user":"1.55","cost_per_1000_reached":"6.382","promotion_type":"Website","video_views_p25":"295","real_time_cost_per_conversion":"0.000","cost_per_secondary_goal_result":null,"frequency":"1.20","follows":"0","clicks":"53","ctr":"1.41","conversion":"0","video_play_actions":"3344","cpm":"5.310","reach":"3134","video_watched_2s":"408","campaign_name":"Website Traffic20211020010104","spend":"20.000","tt_app_id":0,"real_time_conversion_rate":"0.00","real_time_cost_per_result":"0.3774","secondary_goal_result_rate":null,"real_time_app_install_cost":"0.000","real_time_conversion":"0","real_time_result":"53","real_time_result_rate":"1.41","campaign_id":1714125042508817,"tt_app_name":"0"},"dimensions":{"adgroup_id":1714125049901106,"stat_time_day":"2021-10-20 00:00:00"},"stat_time_day":"2021-10-20 00:00:00","adgroup_id":1714125049901106},"emitted_at":1680792355223} -{"stream":"ad_groups_reports","data":{"metrics":{"dpa_target_audience_type":null,"average_video_play":"1.52","conversion_rate":"0.00","mobile_app_id":"0","cost_per_result":"0.2899","video_views_p50":"214","comments":"0","video_watched_6s":"180","placement_type":"Automatic Placement","secondary_goal_result":null,"shares":"0","app_install":"0","result_rate":"1.18","result":"69","real_time_app_install":"0","likes":"36","video_views_p100":"92","impressions":"5830","profile_visits":"0","adgroup_name":"Ad Group20211020010107","clicks_on_music_disc":"0","cost_per_conversion":"0.000","video_views_p75":"140","cpc":"0.290","average_video_play_per_user":"1.64","cost_per_1000_reached":"4.161","promotion_type":"Website","video_views_p25":"513","real_time_cost_per_conversion":"0.000","cost_per_secondary_goal_result":null,"frequency":"1.21","follows":"0","clicks":"69","ctr":"1.18","conversion":"0","video_play_actions":"5173","cpm":"3.430","reach":"4806","video_watched_2s":"686","campaign_name":"Website Traffic20211020010104","spend":"20.000","tt_app_id":0,"real_time_conversion_rate":"0.00","real_time_cost_per_result":"0.2899","secondary_goal_result_rate":null,"real_time_app_install_cost":"0.000","real_time_conversion":"0","real_time_result":"69","real_time_result_rate":"1.18","campaign_id":1714125042508817,"tt_app_name":"0"},"dimensions":{"adgroup_id":1714125049901106,"stat_time_day":"2021-10-25 00:00:00"},"stat_time_day":"2021-10-25 00:00:00","adgroup_id":1714125049901106},"emitted_at":1680792355227} -{"stream":"ad_groups_reports","data":{"metrics":{"dpa_target_audience_type":null,"average_video_play":"1.48","conversion_rate":"0.00","mobile_app_id":"0","cost_per_result":"0.3922","video_views_p50":"164","comments":"0","video_watched_6s":"132","placement_type":"Automatic Placement","secondary_goal_result":null,"shares":"0","app_install":"0","result_rate":"1.09","result":"51","real_time_app_install":"0","likes":"18","video_views_p100":"76","impressions":"4696","profile_visits":"0","adgroup_name":"Ad Group20211020010107","clicks_on_music_disc":"0","cost_per_conversion":"0.000","video_views_p75":"108","cpc":"0.390","average_video_play_per_user":"1.61","cost_per_1000_reached":"5.233","promotion_type":"Website","video_views_p25":"355","real_time_cost_per_conversion":"0.000","cost_per_secondary_goal_result":null,"frequency":"1.23","follows":"0","clicks":"51","ctr":"1.09","conversion":"0","video_play_actions":"4179","cpm":"4.260","reach":"3822","video_watched_2s":"493","campaign_name":"Website Traffic20211020010104","spend":"20.000","tt_app_id":0,"real_time_conversion_rate":"0.00","real_time_cost_per_result":"0.3922","secondary_goal_result_rate":null,"real_time_app_install_cost":"0.000","real_time_conversion":"0","real_time_result":"51","real_time_result_rate":"1.09","campaign_id":1714125042508817,"tt_app_name":"0"},"dimensions":{"adgroup_id":1714125049901106,"stat_time_day":"2021-10-27 00:00:00"},"stat_time_day":"2021-10-27 00:00:00","adgroup_id":1714125049901106},"emitted_at":1680792355230} -{"stream":"ad_groups_reports","data":{"metrics":{"dpa_target_audience_type":null,"average_video_play":"1.46","conversion_rate":"0.00","mobile_app_id":"0","cost_per_result":"0.4762","video_views_p50":"112","comments":"0","video_watched_6s":"92","placement_type":"Automatic Placement","secondary_goal_result":null,"shares":"0","app_install":"0","result_rate":"1.19","result":"42","real_time_app_install":"0","likes":"17","video_views_p100":"59","impressions":"3520","profile_visits":"0","adgroup_name":"Ad Group20211020010107","clicks_on_music_disc":"0","cost_per_conversion":"0.000","video_views_p75":"74","cpc":"0.480","average_video_play_per_user":"1.57","cost_per_1000_reached":"6.878","promotion_type":"Website","video_views_p25":"277","real_time_cost_per_conversion":"0.000","cost_per_secondary_goal_result":null,"frequency":"1.21","follows":"0","clicks":"42","ctr":"1.19","conversion":"0","video_play_actions":"3118","cpm":"5.680","reach":"2908","video_watched_2s":"390","campaign_name":"Website Traffic20211020010104","spend":"20.000","tt_app_id":0,"real_time_conversion_rate":"0.00","real_time_cost_per_result":"0.4762","secondary_goal_result_rate":null,"real_time_app_install_cost":"0.000","real_time_conversion":"0","real_time_result":"42","real_time_result_rate":"1.19","campaign_id":1714125042508817,"tt_app_name":"0"},"dimensions":{"adgroup_id":1714125049901106,"stat_time_day":"2021-10-22 00:00:00"},"stat_time_day":"2021-10-22 00:00:00","adgroup_id":1714125049901106},"emitted_at":1680792355232} -{"stream":"ad_groups_reports","data":{"metrics":{"dpa_target_audience_type":null,"average_video_play":"1.55","conversion_rate":"0.00","mobile_app_id":"0","cost_per_result":"0.2532","video_views_p50":"278","comments":"0","video_watched_6s":"218","placement_type":"Automatic Placement","secondary_goal_result":null,"shares":"0","app_install":"0","result_rate":"1.08","result":"79","real_time_app_install":"0","likes":"56","video_views_p100":"118","impressions":"7310","profile_visits":"0","adgroup_name":"Ad Group20211020010107","clicks_on_music_disc":"0","cost_per_conversion":"0.000","video_views_p75":"155","cpc":"0.250","average_video_play_per_user":"1.66","cost_per_1000_reached":"3.318","promotion_type":"Website","video_views_p25":"663","real_time_cost_per_conversion":"0.000","cost_per_secondary_goal_result":null,"frequency":"1.21","follows":"0","clicks":"79","ctr":"1.08","conversion":"0","video_play_actions":"6433","cpm":"2.740","reach":"6028","video_watched_2s":"917","campaign_name":"Website Traffic20211020010104","spend":"20.000","tt_app_id":0,"real_time_conversion_rate":"0.00","real_time_cost_per_result":"0.2532","secondary_goal_result_rate":null,"real_time_app_install_cost":"0.000","real_time_conversion":"0","real_time_result":"79","real_time_result_rate":"1.08","campaign_id":1714125042508817,"tt_app_name":"0"},"dimensions":{"adgroup_id":1714125049901106,"stat_time_day":"2021-10-24 00:00:00"},"stat_time_day":"2021-10-24 00:00:00","adgroup_id":1714125049901106},"emitted_at":1680792355235} -{"stream":"campaigns_reports","data":{"metrics":{"real_time_app_install":"0","cpm":"2.740","impressions":"7310","cost_per_1000_reached":"3.318","ctr":"1.08","average_video_play_per_user":"1.66","video_views_p100":"118","follows":"0","video_views_p50":"278","clicks_on_music_disc":"0","video_watched_6s":"218","comments":"0","reach":"6028","shares":"0","campaign_name":"Website Traffic20211020010104","video_watched_2s":"917","likes":"56","app_install":"0","average_video_play":"1.55","spend":"20.000","profile_visits":"0","video_play_actions":"6433","real_time_app_install_cost":"0.000","cpc":"0.250","video_views_p25":"663","video_views_p75":"155","clicks":"79","frequency":"1.21"},"dimensions":{"stat_time_day":"2021-10-24 00:00:00","campaign_id":1714125042508817},"stat_time_day":"2021-10-24 00:00:00","campaign_id":1714125042508817},"emitted_at":1680792415636} -{"stream":"campaigns_reports","data":{"metrics":{"real_time_app_install":"0","cpm":"4.260","impressions":"4696","cost_per_1000_reached":"5.233","ctr":"1.09","average_video_play_per_user":"1.61","video_views_p100":"76","follows":"0","video_views_p50":"164","clicks_on_music_disc":"0","video_watched_6s":"132","comments":"0","reach":"3822","shares":"0","campaign_name":"Website Traffic20211020010104","video_watched_2s":"493","likes":"18","app_install":"0","average_video_play":"1.48","spend":"20.000","profile_visits":"0","video_play_actions":"4179","real_time_app_install_cost":"0.000","cpc":"0.390","video_views_p25":"355","video_views_p75":"108","clicks":"51","frequency":"1.23"},"dimensions":{"stat_time_day":"2021-10-27 00:00:00","campaign_id":1714125042508817},"stat_time_day":"2021-10-27 00:00:00","campaign_id":1714125042508817},"emitted_at":1680792415639} -{"stream":"campaigns_reports","data":{"metrics":{"real_time_app_install":"0","cpm":"5.680","impressions":"3520","cost_per_1000_reached":"6.878","ctr":"1.19","average_video_play_per_user":"1.57","video_views_p100":"59","follows":"0","video_views_p50":"112","clicks_on_music_disc":"0","video_watched_6s":"92","comments":"0","reach":"2908","shares":"0","campaign_name":"Website Traffic20211020010104","video_watched_2s":"390","likes":"17","app_install":"0","average_video_play":"1.46","spend":"20.000","profile_visits":"0","video_play_actions":"3118","real_time_app_install_cost":"0.000","cpc":"0.480","video_views_p25":"277","video_views_p75":"74","clicks":"42","frequency":"1.21"},"dimensions":{"stat_time_day":"2021-10-22 00:00:00","campaign_id":1714125042508817},"stat_time_day":"2021-10-22 00:00:00","campaign_id":1714125042508817},"emitted_at":1680792415641} -{"stream":"campaigns_reports","data":{"metrics":{"real_time_app_install":"0","cpm":"3.430","impressions":"5830","cost_per_1000_reached":"4.161","ctr":"1.18","average_video_play_per_user":"1.64","video_views_p100":"92","follows":"0","video_views_p50":"214","clicks_on_music_disc":"0","video_watched_6s":"180","comments":"0","reach":"4806","shares":"0","campaign_name":"Website Traffic20211020010104","video_watched_2s":"686","likes":"36","app_install":"0","average_video_play":"1.52","spend":"20.000","profile_visits":"0","video_play_actions":"5173","real_time_app_install_cost":"0.000","cpc":"0.290","video_views_p25":"513","video_views_p75":"140","clicks":"69","frequency":"1.21"},"dimensions":{"stat_time_day":"2021-10-25 00:00:00","campaign_id":1714125042508817},"stat_time_day":"2021-10-25 00:00:00","campaign_id":1714125042508817},"emitted_at":1680792415644} -{"stream":"campaigns_reports","data":{"metrics":{"real_time_app_install":"0","cpm":"5.310","impressions":"3765","cost_per_1000_reached":"6.382","ctr":"1.41","average_video_play_per_user":"1.55","video_views_p100":"52","follows":"0","video_views_p50":"130","clicks_on_music_disc":"0","video_watched_6s":"106","comments":"1","reach":"3134","shares":"0","campaign_name":"Website Traffic20211020010104","video_watched_2s":"408","likes":"36","app_install":"0","average_video_play":"1.45","spend":"20.000","profile_visits":"0","video_play_actions":"3344","real_time_app_install_cost":"0.000","cpc":"0.380","video_views_p25":"295","video_views_p75":"74","clicks":"53","frequency":"1.20"},"dimensions":{"stat_time_day":"2021-10-20 00:00:00","campaign_id":1714125042508817},"stat_time_day":"2021-10-20 00:00:00","campaign_id":1714125042508817},"emitted_at":1680792415646} -{"stream":"ads_audience_reports","data":{"metrics":{"real_time_result":"12","cost_per_result":"0.3075","conversion_rate":"0.00","adgroup_name":"Ad Group20211020010107","clicks":"12","campaign_id":1714125042508817,"tt_app_name":"0","impressions":"1222","mobile_app_id":"0","real_time_result_rate":"0.98","ad_name":"Optimized Version 4_202110201102_2021-10-20 11:02:00","dpa_target_audience_type":null,"cpm":"3.020","cpc":"0.310","ctr":"0.98","result_rate":"0.98","real_time_conversion":"0","placement_type":"Automatic Placement","cost_per_conversion":"0.000","tt_app_id":"0","real_time_cost_per_result":"0.3075","result":"12","campaign_name":"Website Traffic20211020010104","real_time_cost_per_conversion":"0.000","adgroup_id":1714125049901106,"real_time_conversion_rate":"0.00","conversion":"0","spend":"3.690","promotion_type":"Website","ad_text":"Airbyte - data portabioolity platform - from anywhere to anywhere!"},"dimensions":{"gender":"MALE","stat_time_day":"2021-10-25 00:00:00","ad_id":1714125051115569,"age":"AGE_25_34"},"stat_time_day":"2021-10-25 00:00:00","ad_id":1714125051115569,"gender":"MALE","age":"AGE_25_34"},"emitted_at":1680792474442} -{"stream":"ads_audience_reports","data":{"metrics":{"real_time_result":"5","cost_per_result":"0.3540","conversion_rate":"0.00","adgroup_name":"Ad Group20211020010107","clicks":"5","campaign_id":1714125042508817,"tt_app_name":"0","impressions":"329","mobile_app_id":"0","real_time_result_rate":"1.52","ad_name":"Optimized Version 4_202110201102_2021-10-20 11:02:00","dpa_target_audience_type":null,"cpm":"5.380","cpc":"0.350","ctr":"1.52","result_rate":"1.52","real_time_conversion":"0","placement_type":"Automatic Placement","cost_per_conversion":"0.000","tt_app_id":"0","real_time_cost_per_result":"0.3540","result":"5","campaign_name":"Website Traffic20211020010104","real_time_cost_per_conversion":"0.000","adgroup_id":1714125049901106,"real_time_conversion_rate":"0.00","conversion":"0","spend":"1.770","promotion_type":"Website","ad_text":"Airbyte - data portabioolity platform - from anywhere to anywhere!"},"dimensions":{"gender":"MALE","stat_time_day":"2021-10-21 00:00:00","ad_id":1714125051115569,"age":"AGE_45_54"},"stat_time_day":"2021-10-21 00:00:00","ad_id":1714125051115569,"gender":"MALE","age":"AGE_45_54"},"emitted_at":1680792474445} -{"stream":"ads_audience_reports","data":{"metrics":{"real_time_result":"0","cost_per_result":"0.0000","conversion_rate":"0.00","adgroup_name":"Ad Group20211019111040","clicks":"0","campaign_id":1714073078669329,"tt_app_name":"0","impressions":"56","mobile_app_id":"0","real_time_result_rate":"0.00","ad_name":"Optimized Version 1_202110192111_2021-10-19 21:11:39","dpa_target_audience_type":null,"cpm":"0.000","cpc":"0.000","ctr":"0.00","result_rate":"0.00","real_time_conversion":"0","placement_type":"Automatic Placement","cost_per_conversion":"0.000","tt_app_id":"0","real_time_cost_per_result":"0.0000","result":"0","campaign_name":"Website Traffic20211019110444","real_time_cost_per_conversion":"0.000","adgroup_id":1714073022392322,"real_time_conversion_rate":"0.00","conversion":"0","spend":"0.000","promotion_type":"Website","ad_text":"Open Source ETL"},"dimensions":{"gender":"MALE","stat_time_day":"2021-10-19 00:00:00","ad_id":1714073085256738,"age":"AGE_25_34"},"stat_time_day":"2021-10-19 00:00:00","ad_id":1714073085256738,"gender":"MALE","age":"AGE_25_34"},"emitted_at":1680792474448} -{"stream":"ads_audience_reports","data":{"metrics":{"real_time_result":"10","cost_per_result":"0.2550","conversion_rate":"0.00","adgroup_name":"Ad Group20211020010107","clicks":"10","campaign_id":1714125042508817,"tt_app_name":"0","impressions":"736","mobile_app_id":"0","real_time_result_rate":"1.36","ad_name":"Optimized Version 4_202110201102_2021-10-20 11:02:00","dpa_target_audience_type":null,"cpm":"3.460","cpc":"0.260","ctr":"1.36","result_rate":"1.36","real_time_conversion":"0","placement_type":"Automatic Placement","cost_per_conversion":"0.000","tt_app_id":"0","real_time_cost_per_result":"0.2550","result":"10","campaign_name":"Website Traffic20211020010104","real_time_cost_per_conversion":"0.000","adgroup_id":1714125049901106,"real_time_conversion_rate":"0.00","conversion":"0","spend":"2.550","promotion_type":"Website","ad_text":"Airbyte - data portabioolity platform - from anywhere to anywhere!"},"dimensions":{"gender":"FEMALE","stat_time_day":"2021-10-25 00:00:00","ad_id":1714125051115569,"age":"AGE_45_54"},"stat_time_day":"2021-10-25 00:00:00","ad_id":1714125051115569,"gender":"FEMALE","age":"AGE_45_54"},"emitted_at":1680792474451} -{"stream":"ads_audience_reports","data":{"metrics":{"real_time_result":"9","cost_per_result":"0.4789","conversion_rate":"0.00","adgroup_name":"Ad Group20211020010107","clicks":"9","campaign_id":1714125042508817,"tt_app_name":"0","impressions":"638","mobile_app_id":"0","real_time_result_rate":"1.41","ad_name":"Optimized Version 4_202110201102_2021-10-20 11:02:00","dpa_target_audience_type":null,"cpm":"6.760","cpc":"0.480","ctr":"1.41","result_rate":"1.41","real_time_conversion":"0","placement_type":"Automatic Placement","cost_per_conversion":"0.000","tt_app_id":"0","real_time_cost_per_result":"0.4789","result":"9","campaign_name":"Website Traffic20211020010104","real_time_cost_per_conversion":"0.000","adgroup_id":1714125049901106,"real_time_conversion_rate":"0.00","conversion":"0","spend":"4.310","promotion_type":"Website","ad_text":"Airbyte - data portabioolity platform - from anywhere to anywhere!"},"dimensions":{"gender":"FEMALE","stat_time_day":"2021-10-22 00:00:00","ad_id":1714125051115569,"age":"AGE_35_44"},"stat_time_day":"2021-10-22 00:00:00","ad_id":1714125051115569,"gender":"FEMALE","age":"AGE_35_44"},"emitted_at":1680792474455} -{"stream":"ad_group_audience_reports","data":{"dimensions":{"adgroup_id":1714125049901106,"age":"AGE_45_54","gender":"MALE","stat_time_day":"2021-10-27 00:00:00"},"metrics":{"result_rate":"1.34","promotion_type":"Website","campaign_id":1714125042508817,"cost_per_conversion":"0.000","placement_type":"Automatic Placement","impressions":"373","conversion_rate":"0.00","real_time_cost_per_conversion":"0.000","real_time_conversion":"0","real_time_cost_per_result":"0.4280","conversion":"0","real_time_result_rate":"1.34","tt_app_id":"0","cpc":"0.430","dpa_target_audience_type":null,"clicks":"5","cpm":"5.740","campaign_name":"Website Traffic20211020010104","real_time_conversion_rate":"0.00","ctr":"1.34","spend":"2.140","cost_per_result":"0.4280","adgroup_name":"Ad Group20211020010107","mobile_app_id":"0","tt_app_name":"0","result":"5","real_time_result":"5"},"stat_time_day":"2021-10-27 00:00:00","adgroup_id":1714125049901106,"gender":"MALE","age":"AGE_45_54"},"emitted_at":1680792537240} -{"stream":"ad_group_audience_reports","data":{"dimensions":{"adgroup_id":1714125049901106,"age":"AGE_35_44","gender":"FEMALE","stat_time_day":"2021-10-22 00:00:00"},"metrics":{"result_rate":"1.41","promotion_type":"Website","campaign_id":1714125042508817,"cost_per_conversion":"0.000","placement_type":"Automatic Placement","impressions":"638","conversion_rate":"0.00","real_time_cost_per_conversion":"0.000","real_time_conversion":"0","real_time_cost_per_result":"0.4789","conversion":"0","real_time_result_rate":"1.41","tt_app_id":"0","cpc":"0.480","dpa_target_audience_type":null,"clicks":"9","cpm":"6.760","campaign_name":"Website Traffic20211020010104","real_time_conversion_rate":"0.00","ctr":"1.41","spend":"4.310","cost_per_result":"0.4789","adgroup_name":"Ad Group20211020010107","mobile_app_id":"0","tt_app_name":"0","result":"9","real_time_result":"9"},"stat_time_day":"2021-10-22 00:00:00","adgroup_id":1714125049901106,"gender":"FEMALE","age":"AGE_35_44"},"emitted_at":1680792537243} -{"stream":"ad_group_audience_reports","data":{"dimensions":{"adgroup_id":1714073022392322,"age":"AGE_35_44","gender":"FEMALE","stat_time_day":"2021-10-19 00:00:00"},"metrics":{"result_rate":"0.00","promotion_type":"Website","campaign_id":1714073078669329,"cost_per_conversion":"0.000","placement_type":"Automatic Placement","impressions":"41","conversion_rate":"0.00","real_time_cost_per_conversion":"0.000","real_time_conversion":"0","real_time_cost_per_result":"0.0000","conversion":"0","real_time_result_rate":"0.00","tt_app_id":"0","cpc":"0.000","dpa_target_audience_type":null,"clicks":"0","cpm":"0.000","campaign_name":"Website Traffic20211019110444","real_time_conversion_rate":"0.00","ctr":"0.00","spend":"0.000","cost_per_result":"0.0000","adgroup_name":"Ad Group20211019111040","mobile_app_id":"0","tt_app_name":"0","result":"0","real_time_result":"0"},"stat_time_day":"2021-10-19 00:00:00","adgroup_id":1714073022392322,"gender":"FEMALE","age":"AGE_35_44"},"emitted_at":1680792537246} -{"stream":"ad_group_audience_reports","data":{"dimensions":{"adgroup_id":1714073022392322,"age":"AGE_13_17","gender":"MALE","stat_time_day":"2021-10-20 00:00:00"},"metrics":{"result_rate":"0.00","promotion_type":"Website","campaign_id":1714073078669329,"cost_per_conversion":"0.000","placement_type":"Automatic Placement","impressions":"4","conversion_rate":"0.00","real_time_cost_per_conversion":"0.000","real_time_conversion":"0","real_time_cost_per_result":"0.0000","conversion":"0","real_time_result_rate":"0.00","tt_app_id":"0","cpc":"0.000","dpa_target_audience_type":null,"clicks":"0","cpm":"0.000","campaign_name":"Website Traffic20211019110444","real_time_conversion_rate":"0.00","ctr":"0.00","spend":"0.000","cost_per_result":"0.0000","adgroup_name":"Ad Group20211019111040","mobile_app_id":"0","tt_app_name":"0","result":"0","real_time_result":"0"},"stat_time_day":"2021-10-20 00:00:00","adgroup_id":1714073022392322,"gender":"MALE","age":"AGE_13_17"},"emitted_at":1680792537249} -{"stream":"ad_group_audience_reports","data":{"dimensions":{"adgroup_id":1714073022392322,"age":"AGE_25_34","gender":"MALE","stat_time_day":"2021-10-19 00:00:00"},"metrics":{"result_rate":"0.00","promotion_type":"Website","campaign_id":1714073078669329,"cost_per_conversion":"0.000","placement_type":"Automatic Placement","impressions":"56","conversion_rate":"0.00","real_time_cost_per_conversion":"0.000","real_time_conversion":"0","real_time_cost_per_result":"0.0000","conversion":"0","real_time_result_rate":"0.00","tt_app_id":"0","cpc":"0.000","dpa_target_audience_type":null,"clicks":"0","cpm":"0.000","campaign_name":"Website Traffic20211019110444","real_time_conversion_rate":"0.00","ctr":"0.00","spend":"0.000","cost_per_result":"0.0000","adgroup_name":"Ad Group20211019111040","mobile_app_id":"0","tt_app_name":"0","result":"0","real_time_result":"0"},"stat_time_day":"2021-10-19 00:00:00","adgroup_id":1714073022392322,"gender":"MALE","age":"AGE_25_34"},"emitted_at":1680792537252} -{"stream":"campaigns_audience_reports_by_country","data":{"dimensions":{"country_code":"US","stat_time_day":"2021-10-19 00:00:00","campaign_id":1714073078669329},"metrics":{"cpm":"4.100","campaign_name":"Website Traffic20211019110444","ctr":"1.33","clicks":"65","spend":"20.000","cpc":"0.310","impressions":"4874"},"stat_time_day":"2021-10-19 00:00:00","campaign_id":1714073078669329,"country_code":"US"},"emitted_at":1680792595276} -{"stream":"campaigns_audience_reports_by_country","data":{"dimensions":{"country_code":"US","stat_time_day":"2021-10-20 00:00:00","campaign_id":1714073078669329},"metrics":{"cpm":"0.000","campaign_name":"Website Traffic20211019110444","ctr":"0.00","clicks":"0","spend":"0.000","cpc":"0.000","impressions":"12"},"stat_time_day":"2021-10-20 00:00:00","campaign_id":1714073078669329,"country_code":"US"},"emitted_at":1680792595279} -{"stream":"campaigns_audience_reports_by_country","data":{"dimensions":{"country_code":"US","stat_time_day":"2021-10-22 00:00:00","campaign_id":1714073078669329},"metrics":{"cpm":"0.000","campaign_name":"Website Traffic20211019110444","ctr":"0.00","clicks":"0","spend":"0.000","cpc":"0.000","impressions":"0"},"stat_time_day":"2021-10-22 00:00:00","campaign_id":1714073078669329,"country_code":"US"},"emitted_at":1680792595281} -{"stream":"campaigns_audience_reports_by_country","data":{"dimensions":{"country_code":"US","stat_time_day":"2021-10-20 00:00:00","campaign_id":1714125042508817},"metrics":{"cpm":"5.310","campaign_name":"Website Traffic20211020010104","ctr":"1.41","clicks":"53","spend":"20.000","cpc":"0.380","impressions":"3765"},"stat_time_day":"2021-10-20 00:00:00","campaign_id":1714125042508817,"country_code":"US"},"emitted_at":1680792595284} -{"stream":"campaigns_audience_reports_by_country","data":{"dimensions":{"country_code":"US","stat_time_day":"2021-10-21 00:00:00","campaign_id":1714125042508817},"metrics":{"cpm":"5.030","campaign_name":"Website Traffic20211020010104","ctr":"1.13","clicks":"45","spend":"20.000","cpc":"0.440","impressions":"3977"},"stat_time_day":"2021-10-21 00:00:00","campaign_id":1714125042508817,"country_code":"US"},"emitted_at":1680792595286} -{"stream":"advertisers_reports","data":{"metrics":{"shares":"0","cpm":"4.910","ctr":"1.64","spend":"20.000","video_views_p100":"65","clicks":"67","video_watched_6s":"124","impressions":"4077","likes":"19","video_views_p25":"338","average_video_play":"1.53","app_install":"0","average_video_play_per_user":"1.65","profile_visits":"0","video_views_p50":"146","clicks_on_music_disc":"0","follows":"0","comments":"0","video_watched_2s":"463","real_time_app_install_cost":"0.000","video_views_p75":"95","cash_spend":"20.000","real_time_app_install":"0","video_play_actions":"3590","reach":"3322","cpc":"0.300","voucher_spend":"0.000","frequency":"1.23","cost_per_1000_reached":"6.020"},"dimensions":{"stat_time_day":"2021-10-23 00:00:00","advertiser_id":7002238017842757633},"stat_time_day":"2021-10-23 00:00:00","advertiser_id":7002238017842757633},"emitted_at":1680792695016} -{"stream":"advertisers_reports","data":{"metrics":{"shares":"0","cpm":"5.330","ctr":"1.23","spend":"20.000","video_views_p100":"71","clicks":"46","video_watched_6s":"112","impressions":"3750","likes":"25","video_views_p25":"297","average_video_play":"1.50","app_install":"0","average_video_play_per_user":"1.61","profile_visits":"0","video_views_p50":"142","clicks_on_music_disc":"0","follows":"0","comments":"1","video_watched_2s":"413","real_time_app_install_cost":"0.000","video_views_p75":"90","cash_spend":"20.000","real_time_app_install":"0","video_play_actions":"3344","reach":"3119","cpc":"0.430","voucher_spend":"0.000","frequency":"1.20","cost_per_1000_reached":"6.412"},"dimensions":{"stat_time_day":"2021-10-26 00:00:00","advertiser_id":7002238017842757633},"stat_time_day":"2021-10-26 00:00:00","advertiser_id":7002238017842757633},"emitted_at":1680792695019} -{"stream":"advertisers_reports","data":{"metrics":{"shares":"0","cpm":"5.030","ctr":"1.13","spend":"20.000","video_views_p100":"54","clicks":"45","video_watched_6s":"107","impressions":"3977","likes":"25","video_views_p25":"301","average_video_play":"1.46","app_install":"0","average_video_play_per_user":"1.57","profile_visits":"0","video_views_p50":"128","clicks_on_music_disc":"0","follows":"0","comments":"0","video_watched_2s":"422","real_time_app_install_cost":"0.000","video_views_p75":"77","cash_spend":"20.000","real_time_app_install":"0","video_play_actions":"3460","reach":"3227","cpc":"0.440","voucher_spend":"0.000","frequency":"1.23","cost_per_1000_reached":"6.198"},"dimensions":{"stat_time_day":"2021-10-21 00:00:00","advertiser_id":7002238017842757633},"stat_time_day":"2021-10-21 00:00:00","advertiser_id":7002238017842757633},"emitted_at":1680792695021} -{"stream":"advertisers_reports","data":{"metrics":{"shares":"0","cpm":"3.430","ctr":"1.18","spend":"20.000","video_views_p100":"92","clicks":"69","video_watched_6s":"180","impressions":"5830","likes":"36","video_views_p25":"513","average_video_play":"1.52","app_install":"0","average_video_play_per_user":"1.64","profile_visits":"0","video_views_p50":"214","clicks_on_music_disc":"0","follows":"0","comments":"0","video_watched_2s":"686","real_time_app_install_cost":"0.000","video_views_p75":"140","cash_spend":"20.000","real_time_app_install":"0","video_play_actions":"5173","reach":"4806","cpc":"0.290","voucher_spend":"0.000","frequency":"1.21","cost_per_1000_reached":"4.161"},"dimensions":{"stat_time_day":"2021-10-25 00:00:00","advertiser_id":7002238017842757633},"stat_time_day":"2021-10-25 00:00:00","advertiser_id":7002238017842757633},"emitted_at":1680792695024} -{"stream":"advertisers_reports","data":{"metrics":{"shares":"0","cpm":"5.300","ctr":"1.40","spend":"20.000","video_views_p100":"52","clicks":"53","video_watched_6s":"106","impressions":"3777","likes":"38","video_views_p25":"297","average_video_play":"1.46","app_install":"0","average_video_play_per_user":"1.55","profile_visits":"0","video_views_p50":"130","clicks_on_music_disc":"0","follows":"0","comments":"1","video_watched_2s":"411","real_time_app_install_cost":"0.000","video_views_p75":"74","cash_spend":"20.000","real_time_app_install":"0","video_play_actions":"3355","reach":"3145","cpc":"0.380","voucher_spend":"0.000","frequency":"1.20","cost_per_1000_reached":"6.359"},"dimensions":{"stat_time_day":"2021-10-20 00:00:00","advertiser_id":7002238017842757633},"stat_time_day":"2021-10-20 00:00:00","advertiser_id":7002238017842757633},"emitted_at":1680792695026} -{"stream":"advertisers_audience_reports","data":{"dimensions":{"stat_time_day":"2021-10-19 00:00:00","advertiser_id":7002238017842757633,"gender":"MALE","age":"AGE_13_17"},"metrics":{"ctr":"1.21","cpc":"0.320","clicks":"22","impressions":"1814","cpm":"3.930","spend":"7.130"},"stat_time_day":"2021-10-19 00:00:00","advertiser_id":7002238017842757633,"gender":"MALE","age":"AGE_13_17"},"emitted_at":1680792775318} -{"stream":"advertisers_audience_reports","data":{"dimensions":{"stat_time_day":"2021-10-20 00:00:00","advertiser_id":7002238017842757633,"gender":"MALE","age":"AGE_13_17"},"metrics":{"ctr":"0.00","cpc":"0.000","clicks":"0","impressions":"4","cpm":"0.000","spend":"0.000"},"stat_time_day":"2021-10-20 00:00:00","advertiser_id":7002238017842757633,"gender":"MALE","age":"AGE_13_17"},"emitted_at":1680792775321} -{"stream":"advertisers_audience_reports","data":{"dimensions":{"stat_time_day":"2021-10-19 00:00:00","advertiser_id":7002238017842757633,"gender":"FEMALE","age":"AGE_13_17"},"metrics":{"ctr":"1.35","cpc":"0.290","clicks":"29","impressions":"2146","cpm":"3.880","spend":"8.320"},"stat_time_day":"2021-10-19 00:00:00","advertiser_id":7002238017842757633,"gender":"FEMALE","age":"AGE_13_17"},"emitted_at":1680792775324} -{"stream":"advertisers_audience_reports","data":{"dimensions":{"stat_time_day":"2021-10-20 00:00:00","advertiser_id":7002238017842757633,"gender":"FEMALE","age":"AGE_13_17"},"metrics":{"ctr":"0.00","cpc":"0.000","clicks":"0","impressions":"6","cpm":"0.000","spend":"0.000"},"stat_time_day":"2021-10-20 00:00:00","advertiser_id":7002238017842757633,"gender":"FEMALE","age":"AGE_13_17"},"emitted_at":1680792775326} -{"stream":"advertisers_audience_reports","data":{"dimensions":{"stat_time_day":"2021-10-19 00:00:00","advertiser_id":7002238017842757633,"gender":"MALE","age":"AGE_18_24"},"metrics":{"ctr":"1.79","cpc":"0.380","clicks":"6","impressions":"335","cpm":"6.750","spend":"2.260"},"stat_time_day":"2021-10-19 00:00:00","advertiser_id":7002238017842757633,"gender":"MALE","age":"AGE_18_24"},"emitted_at":1680792775329} +{"stream": "audiences", "data": {"create_time": "2021-10-20 07:26:39", "is_valid": false, "audience_id": "125451003", "cover_num": 0, "shared": false, "is_expiring": true, "audience_type": "Lead Generation", "calculate_type": null, "name": "Airbyte2", "expired_time": "2022-10-20 07:26:39", "is_creator": true}, "emitted_at": 1698065374424} +{"stream": "audiences", "data": {"create_time": "2021-10-20 07:10:04", "is_valid": false, "audience_id": "125450951", "cover_num": 0, "shared": false, "is_expiring": true, "audience_type": "Website Audience", "calculate_type": null, "name": "Airbyte", "expired_time": "2022-10-20 07:10:04", "is_creator": true}, "emitted_at": 1698065374425} +{"stream": "creative_assets_images", "data": {"image_url": "https://p21-ad-sg.ibyteimg.com/obj/ad-site-i18n-sg/202108305d0dfc5fa236678842faa2e0", "is_carousel_usable": true, "file_name": "imgonline-com-ua-Resize-xODPXZ4mmUGvUpJ_1630293661041.jpg", "material_id": "7002056724858683393", "signature": "794ea6266f30750ff0158651a42c8e4c", "create_time": "2021-08-30T03:21:16Z", "format": "jpeg", "width": 640, "modify_time": "2021-08-30T03:21:17Z", "size": 30364, "displayable": true, "height": 640, "image_id": "ad-site-i18n-sg/202108305d0dfc5fa236678842faa2e0"}, "emitted_at": 1698065377209} +{"stream": "creative_assets_images", "data": {"width": 720, "file_name": "7080121373767221250", "displayable": false, "is_carousel_usable": false, "material_id": "7080121373767221250", "create_time": "2022-03-28T12:11:34Z", "size": 33342, "modify_time": "2022-03-28T12:09:10Z", "format": "jpeg", "height": 1280, "image_url": "https://p16-ad-site-sign-sg.ibyteimg.com/v0201/7f371ff6f0764f8b8ef4f37d7b980d50~tplv-d5opwmad15-image.jpeg?x-expires=2013425378&x-signature=mN6uOUjxW82AQ25O%2F2xVDe89xqE%3D", "image_id": "v0201/7f371ff6f0764f8b8ef4f37d7b980d50", "signature": "2fb70f79f5b11f1bf102039bdd9315df"}, "emitted_at": 1698065378369} +{"stream": "creative_assets_images", "data": {"width": 720, "file_name": "a2_1648468243469.png", "displayable": false, "is_carousel_usable": true, "material_id": "7080116242086625281", "create_time": "2022-03-28T11:50:49Z", "size": 117823, "modify_time": "2022-03-28T11:50:50Z", "format": "jpeg", "height": 1280, "image_url": "https://p21-ad-sg.ibyteimg.com/obj/ad-site-i18n-sg/10623349cd96274a8bf6650030e15214", "image_id": "ad-site-i18n-sg/10623349cd96274a8bf6650030e15214", "signature": "f564bd399410d9c271f79f0c785414e1"}, "emitted_at": 1698065378369} +{"stream": "creative_assets_videos", "data": {"file_name": "7002057781092417537", "duration": 10.017, "displayable": true, "video_id": "v10033g50000c4m4trjc77u9bvs72i40", "modify_time": "2021-08-30T03:25:10Z", "allow_download": false, "size": 2719857, "video_cover_url": "http://p16-sign-sg.tiktokcdn.com/v0201/14597664b6954c9cbcce349b6a2fbdb1~tplv-noop.image?x-expires=1698087038&x-signature=JRyNaYQHUOKbA3e8uYxeJRqZN%2FQ%3D", "bit_rate": 2172192, "preview_url": "http://v16m-default.akamaized.net/cd0e25a45833118b9c63d3d5ad7768bf/6536c07e/video/tos/alisg/tos-alisg-v-0000/563db5dbe3bc4dc4ae19f8ae85a2c1c1/?a=0&ch=0&cr=0&dr=0&lr=ad&cd=0%7C0%7C0%7C0&br=1738&bt=869&bti=Njs0Zi8tOg%3D%3D&cs=0&ds=3&ft=dl9~j-Inz7TaaGFZiyq8Z&mime_type=video_mp4&qs=0&rc=aDdlMzUzNDdmO2k5PDhmNEBpM2w6eTw6ZnU3NzMzODYzNEA0Nl5hLTY2XmAxYy8xL2FfYSMxL3BfcjRncWpgLS1kMC1zcw%3D%3D&l=2023102312502799AA8A4C2423390BC15B&btag=e00088000", "material_id": "7002057781092417537", "allowed_placements": ["PLACEMENT_TOPBUZZ", "PLACEMENT_TIKTOK", "PLACEMENT_HELO", "PLACEMENT_PANGLE", "PLACEMENT_GLOBAL_APP_BUNDLE"], "create_time": "2021-08-30T03:25:09Z", "width": 720, "signature": "322637074c8b57ab85bbdf0c0cebc0b3", "format": "mp4", "preview_url_expire_time": "2023-10-23 18:50:38", "height": 1280}, "emitted_at": 1698065428541} +{"stream": "creative_assets_videos", "data": {"size": 13657417, "height": 1280, "allow_download": true, "modify_time": "2023-10-16T14:46:55Z", "format": "mp4", "displayable": true, "allowed_placements": ["PLACEMENT_TOPBUZZ", "PLACEMENT_TIKTOK", "PLACEMENT_HELO", "PLACEMENT_PANGLE", "PLACEMENT_GLOBAL_APP_BUNDLE"], "create_time": "2023-10-16T14:46:55Z", "video_id": "v10033g50000ckmkpnbc77ucmin3t88g", "duration": 30.013, "bit_rate": 3640400, "preview_url": "http://v16m-default.akamaized.net/b5768ccc75fcd6e15ee15cc9e1a80cd2/6536c093/video/tos/alisg/tos-alisg-ve-0051c001-sg/oUQJeNBlILamze9UZixnqggDQQBbAdaGxIGVDU/?a=0&ch=0&cr=0&dr=0&lr=ad&cd=0%7C0%7C0%7C0&cv=1&br=1666&bt=833&bti=Njs0Zi8tOg%3D%3D&cs=0&ds=3&ft=dl9~j-Inz7TFaGFZiyq8Z&mime_type=video_mp4&qs=0&rc=ZGc8ZmQ7Omg2Ozs7ZzhpZ0Bpajs2bGY6ZnFubjMzODYzNEAxYS8wLmMyNjQxMy0uNS8uYSM1cWtqcjRfbWpgLS1kMC1zcw%3D%3D&l=2023102312502899AA8A4C2423390BC195&btag=e00088000", "video_cover_url": "http://p16-sign-sg.tiktokcdn.com/tos-alisg-p-0051c001-sg/oEnxmUxanQZbiaZzA9eAQfNdQBb1lzBIDRgVDL~tplv-noop.image?x-expires=1698087059&x-signature=FbPt%2FI8vfpeVFwzp%2FH9v%2BuHJ0aY%3D", "material_id": "7290567877181833218", "width": 720, "preview_url_expire_time": "2023-10-23 18:50:59", "signature": "62a98cced4a71e2fe6e1edb77ca9dfb7", "file_name": "Video16974675946002_Heartwarming Atmosphere Pops with Piano Main(827850)"}, "emitted_at": 1698065429406} +{"stream": "creative_assets_videos", "data": {"size": 14116114, "height": 1280, "allow_download": true, "modify_time": "2023-10-16T14:46:54Z", "format": "mp4", "displayable": true, "allowed_placements": ["PLACEMENT_TOPBUZZ", "PLACEMENT_TIKTOK", "PLACEMENT_HELO", "PLACEMENT_PANGLE", "PLACEMENT_GLOBAL_APP_BUNDLE"], "create_time": "2023-10-16T14:46:54Z", "video_id": "v10033g50000ckmkpmbc77u1rm8g4jg0", "duration": 30.013, "bit_rate": 3762666, "preview_url": "http://v16m-default.akamaized.net/7c7721438d67a1d486c546ec3e3356dd/6536c093/video/tos/alisg/tos-alisg-ve-0051c001-sg/o4tZwlzQdEyGfICZsotAm3sOQBTlTqBEAhAgYI/?a=0&ch=0&cr=0&dr=0&lr=ad&cd=0%7C0%7C0%7C0&br=3524&bt=1762&bti=Njs0Zi8tOg%3D%3D&cs=0&ds=3&ft=dl9~j-Inz7TFaGFZiyq8Z&mime_type=video_mp4&qs=0&rc=aGY5PDxnO2U6ZDQ6NzYzaEBpM21qcDQ6ZnBubjMzODYzNEAvYzYtXjMtNjQxMmJgNGBfYSNkMTVvcjRfbWpgLS1kMC1zcw%3D%3D&l=2023102312502899AA8A4C2423390BC195&btag=e00088000", "video_cover_url": "http://p16-sign-sg.tiktokcdn.com/tos-alisg-p-0051c001-sg/ogy2o5t3sATEqAfmBACIZNrAWlBEwWt8SZTzhg~tplv-noop.image?x-expires=1698087059&x-signature=DX2gZxniolPWoQkl4pYVCj7msrk%3D", "material_id": "7290567861675278338", "width": 720, "preview_url_expire_time": "2023-10-23 18:50:59", "signature": "45aa9a03c4b7485e7b98251de9c1e48b", "file_name": "Video16974675945900_Whale, sea, electronica(859574)"}, "emitted_at": 1698065429407} +{"stream": "ads_reports", "data": {"metrics": {"clicks": "69", "cpc": "0.29", "vta_conversion": "0", "cost_per_secondary_goal_result": null, "spend": "20.00", "video_play_actions": 5173, "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "real_time_app_install_cost": 0, "average_video_play": 1.52, "video_views_p100": 92, "adgroup_id": 1714125049901106, "real_time_app_install": 0, "comments": 0, "app_install": 0, "impressions": "5830", "real_time_conversion": "0", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "follows": 0, "campaign_id": 1714125042508817, "conversion": "0", "cost_per_conversion": "0.00", "value_per_complete_payment": "0.00", "profile_visits": 0, "clicks_on_music_disc": 0, "video_views_p50": 214, "mobile_app_id": "0", "complete_payment": "0", "cost_per_1000_reached": "4.16", "total_complete_payment_rate": "0.00", "conversion_rate": "0.00", "tt_app_id": 0, "dpa_target_audience_type": null, "result_rate": "1.18", "total_onsite_shopping_value": "0.00", "campaign_name": "Website Traffic20211020010104", "real_time_cost_per_conversion": "0.00", "promotion_type": "Website", "frequency": "1.21", "shares": 0, "onsite_shopping": "0", "result": "69", "average_video_play_per_user": 1.64, "cta_conversion": "0", "tt_app_name": "0", "vta_purchase": "0", "video_views_p75": 140, "video_watched_2s": 686, "real_time_conversion_rate": "0.00", "cost_per_result": "0.290", "real_time_cost_per_result": "0.290", "reach": "4806", "video_views_p25": 513, "real_time_result": "69", "real_time_result_rate": "1.18", "total_pageview": "0", "likes": 36, "secondary_goal_result": null, "ctr": "1.18", "cpm": "3.43", "cta_purchase": "0", "video_watched_6s": 180, "adgroup_name": "Ad Group20211020010107", "total_purchase_value": "0.00", "placement_type": "Automatic Placement", "secondary_goal_result_rate": null}, "dimensions": {"ad_id": 1714125051115569, "stat_time_day": "2021-10-25 00:00:00"}, "stat_time_day": "2021-10-25 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1698065478693} +{"stream": "ads_reports", "data": {"metrics": {"clicks": "53", "cpc": "0.38", "vta_conversion": "0", "cost_per_secondary_goal_result": null, "spend": "20.00", "video_play_actions": 3344, "ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "real_time_app_install_cost": 0, "average_video_play": 1.45, "video_views_p100": 52, "adgroup_id": 1714125049901106, "real_time_app_install": 0, "comments": 1, "app_install": 0, "impressions": "3765", "real_time_conversion": "0", "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "follows": 0, "campaign_id": 1714125042508817, "conversion": "0", "cost_per_conversion": "0.00", "value_per_complete_payment": "0.00", "profile_visits": 0, "clicks_on_music_disc": 0, "video_views_p50": 130, "mobile_app_id": "0", "complete_payment": "0", "cost_per_1000_reached": "6.38", "total_complete_payment_rate": "0.00", "conversion_rate": "0.00", "tt_app_id": 0, "dpa_target_audience_type": null, "result_rate": "1.41", "total_onsite_shopping_value": "0.00", "campaign_name": "Website Traffic20211020010104", "real_time_cost_per_conversion": "0.00", "promotion_type": "Website", "frequency": "1.20", "shares": 0, "onsite_shopping": "0", "result": "53", "average_video_play_per_user": 1.55, "cta_conversion": "0", "tt_app_name": "0", "vta_purchase": "0", "video_views_p75": 74, "video_watched_2s": 408, "real_time_conversion_rate": "0.00", "cost_per_result": "0.377", "real_time_cost_per_result": "0.377", "reach": "3134", "video_views_p25": 295, "real_time_result": "53", "real_time_result_rate": "1.41", "total_pageview": "0", "likes": 36, "secondary_goal_result": null, "ctr": "1.41", "cpm": "5.31", "cta_purchase": "0", "video_watched_6s": 106, "adgroup_name": "Ad Group20211020010107", "total_purchase_value": "0.00", "placement_type": "Automatic Placement", "secondary_goal_result_rate": null}, "dimensions": {"ad_id": 1714125051115569, "stat_time_day": "2021-10-20 00:00:00"}, "stat_time_day": "2021-10-20 00:00:00", "ad_id": 1714125051115569}, "emitted_at": 1698065478699} +{"stream": "ads_reports", "data": {"metrics": {"clicks": "0", "cpc": "0.00", "vta_conversion": "0", "cost_per_secondary_goal_result": null, "spend": "0.00", "video_play_actions": 11, "ad_text": "Open Source ETL", "real_time_app_install_cost": 0, "average_video_play": 1.68, "video_views_p100": 0, "adgroup_id": 1714073022392322, "real_time_app_install": 0, "comments": 0, "app_install": 0, "impressions": "12", "real_time_conversion": "0", "ad_name": "Optimized Version 1_202110192111_2021-10-19 21:11:39", "follows": 0, "campaign_id": 1714073078669329, "conversion": "0", "cost_per_conversion": "0.00", "value_per_complete_payment": "0.00", "profile_visits": 0, "clicks_on_music_disc": 0, "video_views_p50": 0, "mobile_app_id": "0", "complete_payment": "0", "cost_per_1000_reached": "0.00", "total_complete_payment_rate": "0.00", "conversion_rate": "0.00", "tt_app_id": 0, "dpa_target_audience_type": null, "result_rate": "0.00", "total_onsite_shopping_value": "0.00", "campaign_name": "Website Traffic20211019110444", "real_time_cost_per_conversion": "0.00", "promotion_type": "Website", "frequency": "1.20", "shares": 0, "onsite_shopping": "0", "result": "0", "average_video_play_per_user": 1.85, "cta_conversion": "0", "tt_app_name": "0", "vta_purchase": "0", "video_views_p75": 0, "video_watched_2s": 3, "real_time_conversion_rate": "0.00", "cost_per_result": "0.000", "real_time_cost_per_result": "0.000", "reach": "10", "video_views_p25": 2, "real_time_result": "0", "real_time_result_rate": "0.00", "total_pageview": "0", "likes": 2, "secondary_goal_result": null, "ctr": "0.00", "cpm": "0.00", "cta_purchase": "0", "video_watched_6s": 0, "adgroup_name": "Ad Group20211019111040", "total_purchase_value": "0.00", "placement_type": "Automatic Placement", "secondary_goal_result_rate": null}, "dimensions": {"ad_id": 1714073085256738, "stat_time_day": "2021-10-20 00:00:00"}, "stat_time_day": "2021-10-20 00:00:00", "ad_id": 1714073085256738}, "emitted_at": 1698065478704} +{"stream": "ad_groups_reports", "data": {"metrics": {"secondary_goal_result_rate": null, "video_watched_6s": 106, "real_time_result": "53", "real_time_result_rate": "1.41", "real_time_conversion": "0", "real_time_app_install_cost": 0, "clicks": "53", "spend": "20.00", "ctr": "1.41", "comments": 1, "cost_per_conversion": "0.00", "promotion_type": "Website", "campaign_name": "Website Traffic20211020010104", "placement_type": "Automatic Placement", "follows": 0, "adgroup_name": "Ad Group20211020010107", "shares": 0, "impressions": "3765", "cost_per_1000_reached": "6.38", "video_views_p75": 74, "likes": 36, "real_time_cost_per_conversion": "0.00", "video_views_p50": 130, "reach": "3134", "clicks_on_music_disc": 0, "video_play_actions": 3344, "tt_app_name": "0", "average_video_play": 1.45, "cost_per_secondary_goal_result": null, "real_time_cost_per_result": "0.377", "video_views_p25": 295, "result": "53", "conversion_rate": "0.00", "frequency": "1.20", "conversion": "0", "tt_app_id": 0, "cost_per_result": "0.377", "secondary_goal_result": null, "dpa_target_audience_type": null, "cpm": "5.31", "real_time_app_install": 0, "mobile_app_id": "0", "real_time_conversion_rate": "0.00", "campaign_id": 1714125042508817, "result_rate": "1.41", "video_views_p100": 52, "profile_visits": 0, "cpc": "0.38", "video_watched_2s": 408, "average_video_play_per_user": 1.55, "app_install": 0}, "dimensions": {"stat_time_day": "2021-10-20 00:00:00", "adgroup_id": 1714125049901106}, "stat_time_day": "2021-10-20 00:00:00", "adgroup_id": 1714125049901106}, "emitted_at": 1698065555282} +{"stream": "ad_groups_reports", "data": {"metrics": {"secondary_goal_result_rate": null, "video_watched_6s": 180, "real_time_result": "69", "real_time_result_rate": "1.18", "real_time_conversion": "0", "real_time_app_install_cost": 0, "clicks": "69", "spend": "20.00", "ctr": "1.18", "comments": 0, "cost_per_conversion": "0.00", "promotion_type": "Website", "campaign_name": "Website Traffic20211020010104", "placement_type": "Automatic Placement", "follows": 0, "adgroup_name": "Ad Group20211020010107", "shares": 0, "impressions": "5830", "cost_per_1000_reached": "4.16", "video_views_p75": 140, "likes": 36, "real_time_cost_per_conversion": "0.00", "video_views_p50": 214, "reach": "4806", "clicks_on_music_disc": 0, "video_play_actions": 5173, "tt_app_name": "0", "average_video_play": 1.52, "cost_per_secondary_goal_result": null, "real_time_cost_per_result": "0.290", "video_views_p25": 513, "result": "69", "conversion_rate": "0.00", "frequency": "1.21", "conversion": "0", "tt_app_id": 0, "cost_per_result": "0.290", "secondary_goal_result": null, "dpa_target_audience_type": null, "cpm": "3.43", "real_time_app_install": 0, "mobile_app_id": "0", "real_time_conversion_rate": "0.00", "campaign_id": 1714125042508817, "result_rate": "1.18", "video_views_p100": 92, "profile_visits": 0, "cpc": "0.29", "video_watched_2s": 686, "average_video_play_per_user": 1.64, "app_install": 0}, "dimensions": {"stat_time_day": "2021-10-25 00:00:00", "adgroup_id": 1714125049901106}, "stat_time_day": "2021-10-25 00:00:00", "adgroup_id": 1714125049901106}, "emitted_at": 1698065555289} +{"stream": "ad_groups_reports", "data": {"metrics": {"secondary_goal_result_rate": null, "video_watched_6s": 132, "real_time_result": "51", "real_time_result_rate": "1.09", "real_time_conversion": "0", "real_time_app_install_cost": 0, "clicks": "51", "spend": "20.00", "ctr": "1.09", "comments": 0, "cost_per_conversion": "0.00", "promotion_type": "Website", "campaign_name": "Website Traffic20211020010104", "placement_type": "Automatic Placement", "follows": 0, "adgroup_name": "Ad Group20211020010107", "shares": 0, "impressions": "4696", "cost_per_1000_reached": "5.23", "video_views_p75": 108, "likes": 18, "real_time_cost_per_conversion": "0.00", "video_views_p50": 164, "reach": "3822", "clicks_on_music_disc": 0, "video_play_actions": 4179, "tt_app_name": "0", "average_video_play": 1.48, "cost_per_secondary_goal_result": null, "real_time_cost_per_result": "0.392", "video_views_p25": 355, "result": "51", "conversion_rate": "0.00", "frequency": "1.23", "conversion": "0", "tt_app_id": 0, "cost_per_result": "0.392", "secondary_goal_result": null, "dpa_target_audience_type": null, "cpm": "4.26", "real_time_app_install": 0, "mobile_app_id": "0", "real_time_conversion_rate": "0.00", "campaign_id": 1714125042508817, "result_rate": "1.09", "video_views_p100": 76, "profile_visits": 0, "cpc": "0.39", "video_watched_2s": 493, "average_video_play_per_user": 1.61, "app_install": 0}, "dimensions": {"stat_time_day": "2021-10-27 00:00:00", "adgroup_id": 1714125049901106}, "stat_time_day": "2021-10-27 00:00:00", "adgroup_id": 1714125049901106}, "emitted_at": 1698065555294} +{"stream": "campaigns_reports", "data": {"metrics": {"spend": "20.00", "clicks_on_music_disc": 0, "average_video_play_per_user": 1.66, "video_views_p25": 663, "video_views_p100": 118, "average_video_play": 1.55, "app_install": 0, "real_time_app_install_cost": 0, "impressions": "7310", "real_time_app_install": 0, "shares": 0, "cpc": "0.25", "likes": 56, "profile_visits": 0, "video_watched_6s": 218, "video_views_p50": 278, "follows": 0, "reach": "6028", "campaign_name": "Website Traffic20211020010104", "video_views_p75": 155, "clicks": "79", "cost_per_1000_reached": "3.32", "ctr": "1.08", "video_play_actions": 6433, "cpm": "2.74", "comments": 0, "frequency": "1.21", "video_watched_2s": 917}, "dimensions": {"campaign_id": 1714125042508817, "stat_time_day": "2021-10-24 00:00:00"}, "stat_time_day": "2021-10-24 00:00:00", "campaign_id": 1714125042508817}, "emitted_at": 1698065632296} +{"stream": "campaigns_reports", "data": {"metrics": {"spend": "20.00", "clicks_on_music_disc": 0, "average_video_play_per_user": 1.61, "video_views_p25": 355, "video_views_p100": 76, "average_video_play": 1.48, "app_install": 0, "real_time_app_install_cost": 0, "impressions": "4696", "real_time_app_install": 0, "shares": 0, "cpc": "0.39", "likes": 18, "profile_visits": 0, "video_watched_6s": 132, "video_views_p50": 164, "follows": 0, "reach": "3822", "campaign_name": "Website Traffic20211020010104", "video_views_p75": 108, "clicks": "51", "cost_per_1000_reached": "5.23", "ctr": "1.09", "video_play_actions": 4179, "cpm": "4.26", "comments": 0, "frequency": "1.23", "video_watched_2s": 493}, "dimensions": {"campaign_id": 1714125042508817, "stat_time_day": "2021-10-27 00:00:00"}, "stat_time_day": "2021-10-27 00:00:00", "campaign_id": 1714125042508817}, "emitted_at": 1698065632302} +{"stream": "campaigns_reports", "data": {"metrics": {"spend": "20.00", "clicks_on_music_disc": 0, "average_video_play_per_user": 1.57, "video_views_p25": 277, "video_views_p100": 59, "average_video_play": 1.46, "app_install": 0, "real_time_app_install_cost": 0, "impressions": "3520", "real_time_app_install": 0, "shares": 0, "cpc": "0.48", "likes": 17, "profile_visits": 0, "video_watched_6s": 92, "video_views_p50": 112, "follows": 0, "reach": "2908", "campaign_name": "Website Traffic20211020010104", "video_views_p75": 74, "clicks": "42", "cost_per_1000_reached": "6.88", "ctr": "1.19", "video_play_actions": 3118, "cpm": "5.68", "comments": 0, "frequency": "1.21", "video_watched_2s": 390}, "dimensions": {"campaign_id": 1714125042508817, "stat_time_day": "2021-10-22 00:00:00"}, "stat_time_day": "2021-10-22 00:00:00", "campaign_id": 1714125042508817}, "emitted_at": 1698065632313} +{"stream": "ads_audience_reports", "data": {"dimensions": {"ad_id": 1714073085256738, "stat_time_day": "2021-10-20 00:00:00", "gender": "MALE", "age": "AGE_18_24"}, "metrics": {"ad_text": "Open Source ETL", "real_time_conversion": "0", "mobile_app_id": "0", "impressions": "1", "adgroup_id": 1714073022392322, "real_time_cost_per_conversion": "0.00", "tt_app_id": "0", "ctr": "0.00", "cpc": "0.00", "campaign_id": 1714073078669329, "result": "0", "conversion": "0", "conversion_rate": "0.00", "clicks": "0", "real_time_conversion_rate": "0.00", "adgroup_name": "Ad Group20211019111040", "tt_app_name": "0", "promotion_type": "Website", "real_time_result": "0", "cost_per_conversion": "0.00", "real_time_result_rate": "0.00", "cost_per_result": "0.000", "dpa_target_audience_type": null, "ad_name": "Optimized Version 1_202110192111_2021-10-19 21:11:39", "result_rate": "0.00", "spend": "0.00", "cpm": "0.00", "campaign_name": "Website Traffic20211019110444", "placement_type": "Automatic Placement", "real_time_cost_per_result": "0.000"}, "stat_time_day": "2021-10-20 00:00:00", "ad_id": 1714073085256738, "gender": "MALE", "age": "AGE_18_24"}, "emitted_at": 1698065704216} +{"stream": "ads_audience_reports", "data": {"dimensions": {"ad_id": 1714125051115569, "stat_time_day": "2021-10-21 00:00:00", "gender": "MALE", "age": "AGE_35_44"}, "metrics": {"ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "real_time_conversion": "0", "mobile_app_id": "0", "impressions": "454", "adgroup_id": 1714125049901106, "real_time_cost_per_conversion": "0.00", "tt_app_id": "0", "ctr": "1.10", "cpc": "0.46", "campaign_id": 1714125042508817, "result": "5", "conversion": "0", "conversion_rate": "0.00", "clicks": "5", "real_time_conversion_rate": "0.00", "adgroup_name": "Ad Group20211020010107", "tt_app_name": "0", "promotion_type": "Website", "real_time_result": "5", "cost_per_conversion": "0.00", "real_time_result_rate": "1.10", "cost_per_result": "0.456", "dpa_target_audience_type": null, "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "result_rate": "1.10", "spend": "2.28", "cpm": "5.02", "campaign_name": "Website Traffic20211020010104", "placement_type": "Automatic Placement", "real_time_cost_per_result": "0.456"}, "stat_time_day": "2021-10-21 00:00:00", "ad_id": 1714125051115569, "gender": "MALE", "age": "AGE_35_44"}, "emitted_at": 1698065704221} +{"stream": "ads_audience_reports", "data": {"dimensions": {"ad_id": 1714125051115569, "stat_time_day": "2021-10-27 00:00:00", "gender": "FEMALE", "age": "AGE_25_34"}, "metrics": {"ad_text": "Airbyte - data portabioolity platform - from anywhere to anywhere!", "real_time_conversion": "0", "mobile_app_id": "0", "impressions": "1462", "adgroup_id": 1714125049901106, "real_time_cost_per_conversion": "0.00", "tt_app_id": "0", "ctr": "1.09", "cpc": "0.39", "campaign_id": 1714125042508817, "result": "16", "conversion": "0", "conversion_rate": "0.00", "clicks": "16", "real_time_conversion_rate": "0.00", "adgroup_name": "Ad Group20211020010107", "tt_app_name": "0", "promotion_type": "Website", "real_time_result": "16", "cost_per_conversion": "0.00", "real_time_result_rate": "1.09", "cost_per_result": "0.391", "dpa_target_audience_type": null, "ad_name": "Optimized Version 4_202110201102_2021-10-20 11:02:00", "result_rate": "1.09", "spend": "6.26", "cpm": "4.28", "campaign_name": "Website Traffic20211020010104", "placement_type": "Automatic Placement", "real_time_cost_per_result": "0.391"}, "stat_time_day": "2021-10-27 00:00:00", "ad_id": 1714125051115569, "gender": "FEMALE", "age": "AGE_25_34"}, "emitted_at": 1698065704224} +{"stream": "ad_group_audience_reports", "data": {"dimensions": {"stat_time_day": "2021-10-26 00:00:00", "adgroup_id": 1714125049901106, "gender": "FEMALE", "age": "AGE_35_44"}, "metrics": {"campaign_id": 1714125042508817, "real_time_result_rate": "1.91", "conversion_rate": "0.00", "promotion_type": "Website", "real_time_conversion_rate": "0.00", "clicks": "12", "adgroup_name": "Ad Group20211020010107", "campaign_name": "Website Traffic20211020010104", "dpa_target_audience_type": null, "real_time_cost_per_result": "0.401", "real_time_conversion": "0", "impressions": "628", "real_time_cost_per_conversion": "0.00", "result_rate": "1.91", "real_time_result": "12", "placement_type": "Automatic Placement", "cpm": "7.66", "spend": "4.81", "tt_app_id": "0", "ctr": "1.91", "conversion": "0", "cost_per_result": "0.401", "mobile_app_id": "0", "result": "12", "tt_app_name": "0", "cpc": "0.40", "cost_per_conversion": "0.00"}, "stat_time_day": "2021-10-26 00:00:00", "adgroup_id": 1714125049901106, "gender": "FEMALE", "age": "AGE_35_44"}, "emitted_at": 1698065776951} +{"stream": "ad_group_audience_reports", "data": {"dimensions": {"stat_time_day": "2021-10-22 00:00:00", "adgroup_id": 1714125049901106, "gender": "MALE", "age": "AGE_45_54"}, "metrics": {"campaign_id": 1714125042508817, "real_time_result_rate": "1.63", "conversion_rate": "0.00", "promotion_type": "Website", "real_time_conversion_rate": "0.00", "clicks": "4", "adgroup_name": "Ad Group20211020010107", "campaign_name": "Website Traffic20211020010104", "dpa_target_audience_type": null, "real_time_cost_per_result": "0.480", "real_time_conversion": "0", "impressions": "245", "real_time_cost_per_conversion": "0.00", "result_rate": "1.63", "real_time_result": "4", "placement_type": "Automatic Placement", "cpm": "7.84", "spend": "1.92", "tt_app_id": "0", "ctr": "1.63", "conversion": "0", "cost_per_result": "0.480", "mobile_app_id": "0", "result": "4", "tt_app_name": "0", "cpc": "0.48", "cost_per_conversion": "0.00"}, "stat_time_day": "2021-10-22 00:00:00", "adgroup_id": 1714125049901106, "gender": "MALE", "age": "AGE_45_54"}, "emitted_at": 1698065776956} +{"stream": "ad_group_audience_reports", "data": {"dimensions": {"stat_time_day": "2021-10-23 00:00:00", "adgroup_id": 1714125049901106, "gender": "FEMALE", "age": "AGE_45_54"}, "metrics": {"campaign_id": 1714125042508817, "real_time_result_rate": "1.54", "conversion_rate": "0.00", "promotion_type": "Website", "real_time_conversion_rate": "0.00", "clicks": "8", "adgroup_name": "Ad Group20211020010107", "campaign_name": "Website Traffic20211020010104", "dpa_target_audience_type": null, "real_time_cost_per_result": "0.286", "real_time_conversion": "0", "impressions": "519", "real_time_cost_per_conversion": "0.00", "result_rate": "1.54", "real_time_result": "8", "placement_type": "Automatic Placement", "cpm": "4.41", "spend": "2.29", "tt_app_id": "0", "ctr": "1.54", "conversion": "0", "cost_per_result": "0.286", "mobile_app_id": "0", "result": "8", "tt_app_name": "0", "cpc": "0.29", "cost_per_conversion": "0.00"}, "stat_time_day": "2021-10-23 00:00:00", "adgroup_id": 1714125049901106, "gender": "FEMALE", "age": "AGE_45_54"}, "emitted_at": 1698065776960} +{"stream": "campaigns_audience_reports_by_country", "data": {"dimensions": {"country_code": "US", "campaign_id": 1714073078669329, "stat_time_day": "2021-10-19 00:00:00"}, "metrics": {"clicks": "65", "spend": "20.00", "campaign_name": "Website Traffic20211019110444", "impressions": "4874", "ctr": "1.33", "cpc": "0.31", "cpm": "4.10"}, "stat_time_day": "2021-10-19 00:00:00", "campaign_id": 1714073078669329, "country_code": "US"}, "emitted_at": 1698065845197} +{"stream": "campaigns_audience_reports_by_country", "data": {"dimensions": {"country_code": "US", "campaign_id": 1714125042508817, "stat_time_day": "2021-10-20 00:00:00"}, "metrics": {"clicks": "53", "spend": "20.00", "campaign_name": "Website Traffic20211020010104", "impressions": "3765", "ctr": "1.41", "cpc": "0.38", "cpm": "5.31"}, "stat_time_day": "2021-10-20 00:00:00", "campaign_id": 1714125042508817, "country_code": "US"}, "emitted_at": 1698065845201} +{"stream": "campaigns_audience_reports_by_country", "data": {"dimensions": {"country_code": "US", "campaign_id": 1714073078669329, "stat_time_day": "2021-10-20 00:00:00"}, "metrics": {"clicks": "0", "spend": "0.00", "campaign_name": "Website Traffic20211019110444", "impressions": "12", "ctr": "0.00", "cpc": "0.00", "cpm": "0.00"}, "stat_time_day": "2021-10-20 00:00:00", "campaign_id": 1714073078669329, "country_code": "US"}, "emitted_at": 1698065845204} +{"stream": "advertisers_reports", "data": {"dimensions": {"stat_time_day": "2021-10-23 00:00:00", "advertiser_id": 7002238017842757633}, "metrics": {"video_views_p50": 146, "ctr": "1.64", "reach": "3322", "follows": 0, "app_install": 0, "video_views_p100": 65, "cpc": "0.30", "real_time_app_install": 0, "frequency": "1.23", "cost_per_1000_reached": "6.02", "cash_spend": "20.00", "cpm": "4.91", "average_video_play": 1.53, "impressions": "4077", "shares": 0, "spend": "20.00", "video_play_actions": 3590, "clicks": "67", "video_watched_2s": 463, "clicks_on_music_disc": 0, "video_views_p25": 338, "comments": 0, "average_video_play_per_user": 1.65, "real_time_app_install_cost": 0, "likes": 19, "video_watched_6s": 124, "profile_visits": 0, "video_views_p75": 95, "voucher_spend": "0.00"}, "stat_time_day": "2021-10-23 00:00:00", "advertiser_id": 7002238017842757633}, "emitted_at": 1698065930500} +{"stream": "advertisers_reports", "data": {"dimensions": {"stat_time_day": "2021-10-26 00:00:00", "advertiser_id": 7002238017842757633}, "metrics": {"video_views_p50": 142, "ctr": "1.23", "reach": "3119", "follows": 0, "app_install": 0, "video_views_p100": 71, "cpc": "0.43", "real_time_app_install": 0, "frequency": "1.20", "cost_per_1000_reached": "6.41", "cash_spend": "20.00", "cpm": "5.33", "average_video_play": 1.5, "impressions": "3750", "shares": 0, "spend": "20.00", "video_play_actions": 3344, "clicks": "46", "video_watched_2s": 413, "clicks_on_music_disc": 0, "video_views_p25": 297, "comments": 1, "average_video_play_per_user": 1.61, "real_time_app_install_cost": 0, "likes": 25, "video_watched_6s": 112, "profile_visits": 0, "video_views_p75": 90, "voucher_spend": "0.00"}, "stat_time_day": "2021-10-26 00:00:00", "advertiser_id": 7002238017842757633}, "emitted_at": 1698065930505} +{"stream": "advertisers_reports", "data": {"dimensions": {"stat_time_day": "2021-10-21 00:00:00", "advertiser_id": 7002238017842757633}, "metrics": {"video_views_p50": 128, "ctr": "1.13", "reach": "3227", "follows": 0, "app_install": 0, "video_views_p100": 54, "cpc": "0.44", "real_time_app_install": 0, "frequency": "1.23", "cost_per_1000_reached": "6.20", "cash_spend": "20.00", "cpm": "5.03", "average_video_play": 1.46, "impressions": "3977", "shares": 0, "spend": "20.00", "video_play_actions": 3460, "clicks": "45", "video_watched_2s": 422, "clicks_on_music_disc": 0, "video_views_p25": 301, "comments": 0, "average_video_play_per_user": 1.57, "real_time_app_install_cost": 0, "likes": 25, "video_watched_6s": 107, "profile_visits": 0, "video_views_p75": 77, "voucher_spend": "0.00"}, "stat_time_day": "2021-10-21 00:00:00", "advertiser_id": 7002238017842757633}, "emitted_at": 1698065930510} +{"stream": "advertisers_audience_reports", "data": {"metrics": {"cpm": "6.75", "ctr": "1.79", "impressions": "335", "spend": "2.26", "clicks": "6", "cpc": "0.38"}, "dimensions": {"age": "AGE_18_24", "gender": "MALE", "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_18_24"}, "emitted_at": 1698066026190} +{"stream": "advertisers_audience_reports", "data": {"metrics": {"cpm": "0.00", "ctr": "2.86", "impressions": "35", "spend": "0.00", "clicks": "1", "cpc": "0.00"}, "dimensions": {"age": "AGE_35_44", "gender": "MALE", "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "gender": "MALE", "age": "AGE_35_44"}, "emitted_at": 1698066026193} +{"stream": "advertisers_audience_reports", "data": {"metrics": {"cpm": "3.88", "ctr": "1.35", "impressions": "2146", "spend": "8.32", "clicks": "29", "cpc": "0.29"}, "dimensions": {"age": "AGE_13_17", "gender": "FEMALE", "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633}, "stat_time_day": "2021-10-19 00:00:00", "advertiser_id": 7002238017842757633, "gender": "FEMALE", "age": "AGE_13_17"}, "emitted_at": 1698066026196} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml b/airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml index 3ab49d44427f..e18fca8923b2 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml +++ b/airbyte-integrations/connectors/source-tiktok-marketing/metadata.yaml @@ -1,13 +1,19 @@ data: + ab_internal: + ql: 400 + sl: 300 allowedHosts: hosts: - sandbox-ads.tiktok.com - business-api.tiktok.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: api connectorType: source definitionId: 4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35 - dockerImageTag: 3.4.1 + dockerImageTag: 3.9.2 dockerRepository: airbyte/source-tiktok-marketing + documentationUrl: https://docs.airbyte.com/integrations/sources/tiktok-marketing githubIssueLabel: source-tiktok-marketing icon: tiktok.svg license: MIT @@ -18,11 +24,16 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/tiktok-marketing + supportLevel: certified tags: - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified + suggestedStreams: + streams: + - ads_reports_daily + - ads + - campaigns + - campaigns_reports_daily + - ad_groups + - ad_groups_reports_daily + - advertisers_reports_daily metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/ad_groups.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/ad_groups.json index d958b38cf2d2..c62bfa875b4a 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/ad_groups.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/ad_groups.json @@ -437,6 +437,33 @@ "items": { "type": "string" } + }, + "contextual_tag_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "household_income": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "isp_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "spending_power": { + "type": ["null", "string"] + }, + "zipcode_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } } } } diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/ads.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/ads.json index 83cf9fe8164f..be26308e38f8 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/ads.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/ads.json @@ -22,6 +22,15 @@ "ad_name": { "type": "string" }, + "tracking_app_id": { + "type": ["null", "string"] + }, + "tracking_offline_event_set_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, "call_to_action": { "type": ["null", "string"] }, @@ -116,6 +125,9 @@ "click_tracking_url": { "type": ["null", "string"] }, + "tracking_pixel_id": { + "type": ["null", "integer"] + }, "deeplink": { "type": ["null", "string"] }, @@ -186,9 +198,24 @@ "identity_type": { "type": ["null", "string"] }, + "identity_authorized_bc_id": { + "type": ["null", "string"] + }, + "phone_region_code": { + "type": ["null", "string"] + }, + "phone_region_calling_code": { + "type": ["null", "string"] + }, "optimization_event": { "type": ["null", "string"] }, + "phone_number": { + "type": ["null", "string"] + }, + "carousel_image_index": { + "type": ["null", "integer"] + }, "viewability_postbid_partner": { "type": ["null", "string"] }, @@ -197,6 +224,76 @@ }, "music_id": { "type": ["null", "string"] + }, + "utm_params": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "key": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + }, + "shopping_ads_deeplink_type": { + "type": ["null", "string"] + }, + "dark_post_status": { + "type": ["null", "string"] + }, + "branded_content_disabled": { + "type": ["null", "string"] + }, + "product_specific_type": { + "type": ["null", "string"] + }, + "catalog_id": { + "type": ["null", "string"] + }, + "item_group_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "product_set_id": { + "type": ["null", "string"] + }, + "sku_ids": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "dynamic_format": { + "type": ["null", "string"] + }, + "vertical_video_strategy": { + "type": ["null", "string"] + }, + "dynamic_destination": { + "type": ["null", "string"] + }, + "showcase_products": { + "type": ["null", "object"], + "properties": { + "item_group_id": { + "type": ["null", "string"] + }, + "store_id": { + "type": ["null", "string"] + }, + "catalog_id": { + "type": ["null", "string"] + } + } + }, + "tiktok_page_category": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/audience_reports.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/audience_reports.json index 36fb4ba8e099..a98ade3a031e 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/audience_reports.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/audience_reports.json @@ -36,6 +36,9 @@ "age": { "type": ["null", "string"] }, + "province_id": { + "type": ["null", "string"] + }, "metrics": { "type": ["null", "object"], "properties": { @@ -134,6 +137,9 @@ }, "real_time_result_rate": { "type": ["null", "string"] + }, + "province_id": { + "type": ["null", "string"] } } }, diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/audiences.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/audiences.json new file mode 100644 index 000000000000..3559ce877e3a --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/audiences.json @@ -0,0 +1,40 @@ +{ + "type": "object", + "properties": { + "shared": { + "type": ["null", "boolean"] + }, + "is_creator": { + "type": ["null", "boolean"] + }, + "audience_id": { + "type": ["null", "string"] + }, + "cover_num": { + "type": ["null", "integer"] + }, + "create_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "is_valid": { + "type": ["null", "boolean"] + }, + "is_expiring": { + "type": ["null", "boolean"] + }, + "expired_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "name": { + "type": ["null", "string"] + }, + "audience_type": { + "type": ["null", "string"] + }, + "calculate_type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/basic_reports.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/basic_reports.json index 6acd7fe268e4..98ba2a45b9bb 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/basic_reports.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/basic_reports.json @@ -173,6 +173,102 @@ }, "total_complete_payment_rate": { "type": ["null", "string"] + }, + "video_play_actions": { + "type": ["null", "number"] + }, + "video_watched_2s": { + "type": ["null", "number"] + }, + "video_watched_6s": { + "type": ["null", "number"] + }, + "average_video_play": { + "type": ["null", "number"] + }, + "average_video_play_per_user": { + "type": ["null", "number"] + }, + "video_views_p25": { + "type": ["null", "number"] + }, + "video_views_p50": { + "type": ["null", "number"] + }, + "video_views_p75": { + "type": ["null", "number"] + }, + "video_views_p100": { + "type": ["null", "number"] + }, + "profile_visits": { + "type": ["null", "number"] + }, + "likes": { + "type": ["null", "number"] + }, + "comments": { + "type": ["null", "number"] + }, + "shares": { + "type": ["null", "number"] + }, + "follows": { + "type": ["null", "number"] + }, + "clicks_on_music_disc": { + "type": ["null", "number"] + }, + "real_time_app_install": { + "type": ["null", "number"] + }, + "real_time_app_install_cost": { + "type": ["null", "number"] + }, + "app_install": { + "type": ["null", "number"] + }, + "profile_visits_rate": { + "type": ["null", "number"] + }, + "purchase": { + "type": ["null", "number"] + }, + "purchase_rate": { + "type": ["null", "number"] + }, + "registration": { + "type": ["null", "number"] + }, + "registration_rate": { + "type": ["null", "number"] + }, + "sales_lead": { + "type": ["null", "number"] + }, + "sales_lead_rate": { + "type": ["null", "number"] + }, + "cost_per_app_install": { + "type": ["null", "number"] + }, + "cost_per_purchase": { + "type": ["null", "number"] + }, + "cost_per_registration": { + "type": ["null", "number"] + }, + "cost_per_sales_lead": { + "type": ["null", "number"] + }, + "cost_per_total_sales_lead": { + "type": ["null", "number"] + }, + "cost_per_total_app_event_add_to_cart": { + "type": ["null", "number"] + }, + "total_app_event_add_to_cart": { + "type": ["null", "number"] } } }, diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/campaigns.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/campaigns.json index c6644b1baadf..81291b0f55a1 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/campaigns.json +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/campaigns.json @@ -64,6 +64,15 @@ }, "is_smart_performance_campaign": { "type": ["null", "boolean"] + }, + "is_search_campaign": { + "type": ["null", "boolean"] + }, + "app_promotion_type": { + "type": ["null", "string"] + }, + "rf_campaign_type": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_images.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_images.json new file mode 100644 index 000000000000..5bee3b1723ca --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_images.json @@ -0,0 +1,46 @@ +{ + "type": "object", + "properties": { + "image_id": { + "type": ["null", "string"] + }, + "format": { + "type": ["null", "string"] + }, + "image_url": { + "type": ["null", "string"] + }, + "height": { + "type": ["null", "integer"] + }, + "width": { + "type": ["null", "integer"] + }, + "signature": { + "type": ["null", "string"] + }, + "size": { + "type": ["null", "integer"] + }, + "material_id": { + "type": ["null", "string"] + }, + "is_carousel_usable": { + "type": ["null", "boolean"] + }, + "file_name": { + "type": ["null", "string"] + }, + "create_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "modify_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "displayable": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_music.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_music.json new file mode 100644 index 000000000000..06bb4ff374e2 --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_music.json @@ -0,0 +1,55 @@ +{ + "type": "object", + "properties": { + "music_id": { + "type": ["null", "string"] + }, + "material_id": { + "type": ["null", "string"] + }, + "sources": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "author": { + "type": ["null", "string"] + }, + "liked": { + "type": ["null", "boolean"] + }, + "cover_url": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "duration": { + "type": ["null", "number"] + }, + "style": { + "type": ["null", "string"] + }, + "signature": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "file_name": { + "type": ["null", "string"] + }, + "copyright": { + "type": ["null", "string"] + }, + "create_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "modify_time": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_portfolios.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_portfolios.json new file mode 100644 index 000000000000..2e9c74de2e8b --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_portfolios.json @@ -0,0 +1,14 @@ +{ + "type": "object", + "properties": { + "creative_portfolio_id": { + "type": ["null", "string"] + }, + "creative_portfolio_type": { + "type": ["null", "string"] + }, + "creative_portfolio_preview_url": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_videos.json b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_videos.json new file mode 100644 index 000000000000..c4c20bb0cb75 --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/schemas/creative_assets_videos.json @@ -0,0 +1,66 @@ +{ + "type": "object", + "properties": { + "video_id": { + "type": ["null", "string"] + }, + "video_cover_url": { + "type": ["null", "string"] + }, + "format": { + "type": ["null", "string"] + }, + "preview_url": { + "type": ["null", "string"] + }, + "preview_url_expire_time": { + "type": ["null", "string"], + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "duration": { + "type": ["null", "number"] + }, + "height": { + "type": ["null", "integer"] + }, + "width": { + "type": ["null", "integer"] + }, + "bit_rate": { + "type": ["null", "number"] + }, + "signature": { + "type": ["null", "string"] + }, + "size": { + "type": ["null", "integer"] + }, + "material_id": { + "type": ["null", "string"] + }, + "allowed_placements": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "allow_download": { + "type": ["null", "boolean"] + }, + "file_name": { + "type": ["null", "string"] + }, + "create_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "modify_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "displayable": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py index 61d8aff8c89c..e59489196a20 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py @@ -1,9 +1,10 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - +import logging from typing import Any, List, Mapping, Tuple +import pendulum from airbyte_cdk.logger import AirbyteLogger from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource @@ -13,6 +14,7 @@ from .streams import ( DEFAULT_END_DATE, DEFAULT_START_DATE, + MINIMUM_START_DATE, AdGroupAudienceReports, AdGroupAudienceReportsByCountry, AdGroupAudienceReportsByPlatform, @@ -22,6 +24,7 @@ AdsAudienceReports, AdsAudienceReportsByCountry, AdsAudienceReportsByPlatform, + AdsAudienceReportsByProvince, AdsReports, AdvertiserIds, Advertisers, @@ -29,18 +32,24 @@ AdvertisersAudienceReportsByCountry, AdvertisersAudienceReportsByPlatform, AdvertisersReports, + Audiences, BasicReports, Campaigns, CampaignsAudienceReports, CampaignsAudienceReportsByCountry, CampaignsAudienceReportsByPlatform, CampaignsReports, + CreativeAssetsImages, + CreativeAssetsMusic, + CreativeAssetsPortfolios, + CreativeAssetsVideos, Daily, Hourly, Lifetime, ReportGranularity, ) +logger = logging.getLogger("airbyte") DOCUMENTATION_URL = "https://docs.airbyte.com/integrations/sources/tiktok-marketing" @@ -86,9 +95,13 @@ def _prepare_stream_args(config: Mapping[str, Any]) -> Mapping[str, Any]: app_id = int(config.get("environment", {}).get("app_id", 0)) advertiser_id = config.get("environment", {}).get("advertiser_id") + start_date = config.get("start_date") or DEFAULT_START_DATE + if pendulum.parse(start_date) < pendulum.parse(MINIMUM_START_DATE): + logger.warning(f"The start date is too far in the past. Setting it to {MINIMUM_START_DATE}.") + start_date = MINIMUM_START_DATE stream_args = { "authenticator": TiktokTokenAuthenticator(access_token), - "start_date": config.get("start_date") or DEFAULT_START_DATE, + "start_date": start_date, "end_date": config.get("end_date") or DEFAULT_END_DATE, "app_id": app_id, "secret": secret, @@ -127,7 +140,12 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Advertisers(**args), Ads(**args), AdGroups(**args), + Audiences(**args), Campaigns(**args), + CreativeAssetsImages(**args), + CreativeAssetsMusic(**args), + CreativeAssetsPortfolios(**args), + CreativeAssetsVideos(**args), ] if is_production: @@ -184,6 +202,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: AdsAudienceReports, AdsAudienceReportsByCountry, AdsAudienceReportsByPlatform, + AdsAudienceReportsByProvince, AdGroupAudienceReports, AdGroupAudienceReportsByCountry, AdGroupAudienceReportsByPlatform, diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py index 3e9724eee853..d38cc2f1759b 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py @@ -15,7 +15,6 @@ import pydantic import requests from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.core import package_name_from_class from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader @@ -23,6 +22,7 @@ # TikTok Initial release date is September 2016 DEFAULT_START_DATE = "2016-09-01" +MINIMUM_START_DATE = "2012-01-01" DEFAULT_END_DATE = str(datetime.now().date()) NOT_AUDIENCE_METRICS = [ "reach", @@ -62,6 +62,21 @@ "complete_payment", "value_per_complete_payment", "total_complete_payment_rate", + "profile_visits_rate", + "purchase", + "purchase_rate", + "registration", + "registration_rate", + "sales_lead", + "sales_lead_rate", + "cost_per_app_install", + "cost_per_purchase", + "cost_per_registration", + "total_purchase_value", + "cost_per_sales_lead", + "cost_per_total_sales_lead", + "cost_per_total_app_event_add_to_cart", + "total_app_event_add_to_cart", ] T = TypeVar("T") @@ -87,7 +102,8 @@ # | └─AdGroupAudienceReportsByPlatform (11 ad_group_audience_reports_by_platform) # ├─AdsAudienceReports (12 ads_audience_reports) # | ├─AdsAudienceReportsByCountry (13 ads_audience_reports_by_country) -# | └─AdsAudienceReportsByPlatform (14 ads_audience_reports_by_platform) +# | ├─AdsAudienceReportsByPlatform (14 ads_audience_reports_by_platform) +# | ├─AdsAudienceReportsByProvince (14 ads_audience_reports_by_platform) # ├─AdvertisersAudienceReports (15 advertisers_audience_reports) # | ├─AdvertisersAudienceReportsByCountry (16 advertisers_audience_reports_by_country) # | └─AdvertisersAudienceReportsByPlatform (17 advertisers_audience_reports_by_platform) @@ -171,10 +187,6 @@ def __init__(self, **kwargs): self._advertiser_id = kwargs.get("advertiser_id") self.is_sandbox = kwargs.get("is_sandbox") - @property - def availability_strategy(self) -> Optional["AvailabilityStrategy"]: - return None - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: """All responses have the similar structure: { @@ -277,7 +289,7 @@ class FullRefreshTiktokStream(TiktokStream, ABC): @transformer.registerCustomTransform def transform_function(original_value: Any, field_schema: Dict[str, Any]) -> Any: - """Custom traun""" + """Custom transformation""" if original_value == "-": return None elif isinstance(original_value, float): @@ -327,23 +339,6 @@ def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: def is_finished(self): return len(self._advertiser_ids) == 0 - def request_params( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - params = {"page_size": self.page_size} - if self.fields: - params["fields"] = self.convert_array_param(self.fields) - if stream_slice: - params.update(stream_slice) - return params - - -class IncrementalTiktokStream(FullRefreshTiktokStream, ABC): - cursor_field = "modify_time" - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: """All responses have the following pagination data: { @@ -359,17 +354,32 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, } """ - page_info = response.json()["data"]["page_info"] + page_info = response.json().get("data", {}).get("page_info", {}) + if not page_info: + return None if page_info["page"] < page_info["total_page"]: return {"page": page_info["page"] + 1} return None - def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: - params = super().request_params(next_page_token=next_page_token, **kwargs) + def request_params( + self, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + params = {"page_size": self.page_size} + if self.fields: + params["fields"] = self.convert_array_param(self.fields) + if stream_slice: + params.update(stream_slice) if next_page_token: params.update(next_page_token) return params + +class IncrementalTiktokStream(FullRefreshTiktokStream, ABC): + cursor_field = "modify_time" + def select_cursor_field_value(self, data: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None) -> str: if not data or not self.cursor_field: return None @@ -406,17 +416,17 @@ def parse_response( self, response: requests.Response, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs ) -> Iterable[Mapping]: """Additional data filtering""" - state = self.select_cursor_field_value(stream_state) or self._start_time + state_cursor_value = self.select_cursor_field_value(stream_state) or self._start_time for record in super().parse_response(response=response, stream_state=stream_state, **kwargs): record = self.unnest_cursor_and_pk(record) - updated = self.select_cursor_field_value(record, stream_slice) - if updated is None: + updated_cursor_value = self.select_cursor_field_value(record, stream_slice) + if updated_cursor_value is None: yield record - elif updated <= state: + elif updated_cursor_value < state_cursor_value: continue else: - if not self.max_cursor_date or self.max_cursor_date < updated: - self.max_cursor_date = updated + if not self.max_cursor_date or self.max_cursor_date < updated_cursor_value: + self.max_cursor_date = updated_cursor_value yield record def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: @@ -461,6 +471,37 @@ def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: yield {"advertiser_ids": ids[i : min(end, i + step)]} +class Audiences(FullRefreshTiktokStream): + """Docs: https://business-api.tiktok.com/portal/docs?id=1739940506015746""" + + page_size = 100 + primary_key = "audience_id" + + def path(self, *args, **kwargs) -> str: + return "dmp/custom_audience/list/" + + +class CreativeAssetsMusic(FullRefreshTiktokStream): + """Docs: https://business-api.tiktok.com/portal/docs?id=1740053909509122""" + + primary_key = "music_id" + response_list_field = "musics" + + def path(self, *args, **kwargs) -> str: + return "file/music/get/" + + +class CreativeAssetsPortfolios(FullRefreshTiktokStream): + """Docs: https://business-api.tiktok.com/portal/docs?id=1766324010279938""" + + page_size = 100 + primary_key = "creative_portfolio_id" + response_list_field = "creative_portfolios" + + def path(self, *args, **kwargs) -> str: + return "creative/portfolio/list/" + + class Campaigns(IncrementalTiktokStream): """Docs: https://ads.tiktok.com/marketing_api/docs?id=1739315828649986""" @@ -488,6 +529,26 @@ def path(self, *args, **kwargs) -> str: return "ad/get/" +class CreativeAssetsImages(IncrementalTiktokStream): + """Docs: https://business-api.tiktok.com/portal/docs?id=1740052016789506""" + + page_size = 100 + primary_key = "image_id" + + def path(self, *args, **kwargs) -> str: + return "file/image/ad/search/" + + +class CreativeAssetsVideos(IncrementalTiktokStream): + """Docs: https://business-api.tiktok.com/portal/docs?id=1740050472224769""" + + page_size = 100 + primary_key = "video_id" + + def path(self, *args, **kwargs) -> str: + return "file/video/ad/search/" + + class BasicReports(IncrementalTiktokStream, ABC): """Docs: https://ads.tiktok.com/marketing_api/docs?id=1738864915188737""" @@ -802,25 +863,21 @@ def request_params( class CampaignsAudienceReports(AudienceReport): - ref_pk = "campaign_id" report_level = ReportLevel.CAMPAIGN class AdGroupAudienceReports(AudienceReport): - ref_pk = "adgroup_id" report_level = ReportLevel.ADGROUP class AdsAudienceReports(AudienceReport): - ref_pk = "ad_id" report_level = ReportLevel.AD class AdvertisersAudienceReports(AudienceReport): - ref_pk = "advertiser_id" report_level = ReportLevel.ADVERTISER @@ -871,3 +928,9 @@ class AdvertisersAudienceReportsByPlatform(AdvertisersAudienceReports): """Custom reports for advertisers by platform""" audience_dimensions = ["platform"] + + +class AdsAudienceReportsByProvince(AdsAudienceReports): + """Custom reports for ads by province""" + + audience_dimensions = ["province_id"] diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py new file mode 100644 index 000000000000..969410ee24e1 --- /dev/null +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/conftest.py @@ -0,0 +1,18 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import os + +import pytest +from airbyte_cdk.utils.constants import ENV_REQUEST_CACHE_PATH + +os.environ[ENV_REQUEST_CACHE_PATH] = ENV_REQUEST_CACHE_PATH + + +@pytest.fixture(autouse=True) +def patch_sleep(mocker): + mocker.patch("time.sleep") + + +@pytest.fixture(autouse=True) +def disable_cache(mocker): + mocker.patch("source_tiktok_marketing.streams.AdvertiserIds.use_cache", new_callable=mocker.PropertyMock, return_value=False) diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py index 6d98cd6c569e..eca654baa6f7 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py @@ -6,6 +6,7 @@ import pendulum import pytest +import requests from source_tiktok_marketing.source import get_report_stream from source_tiktok_marketing.streams import ( AdGroupsReports, @@ -69,15 +70,13 @@ def advertiser_ids_fixture(): ], ) def test_get_time_interval(pendulum_now_mock, granularity, intervals_len): - intervals = BasicReports._get_time_interval( - start_date="2020-01-01", ending_date="2020-03-01", granularity=granularity) + intervals = BasicReports._get_time_interval(start_date="2020-01-01", ending_date="2020-03-01", granularity=granularity) assert len(list(intervals)) == intervals_len @patch.object(pendulum, "now", return_value=pendulum.parse("2018-12-25")) def test_get_time_interval_past(pendulum_now_mock_past): - intervals = BasicReports._get_time_interval( - start_date="2020-01-01", ending_date="2020-01-01", granularity=ReportGranularity.DAY) + intervals = BasicReports._get_time_interval(start_date="2020-01-01", ending_date="2020-01-01", granularity=ReportGranularity.DAY) assert len(list(intervals)) == 1 @@ -112,25 +111,18 @@ def test_stream_slices_basic_sandbox(advertiser_ids, config_name, slices_expecte ( Daily, [ - {"advertiser_id": 1, "end_date": "2020-01-30", - "start_date": "2020-01-01"}, - {"advertiser_id": 1, "end_date": "2020-02-29", - "start_date": "2020-01-31"}, - {"advertiser_id": 1, "end_date": "2020-03-01", - "start_date": "2020-03-01"}, - {"advertiser_id": 2, "end_date": "2020-01-30", - "start_date": "2020-01-01"}, - {"advertiser_id": 2, "end_date": "2020-02-29", - "start_date": "2020-01-31"}, - {"advertiser_id": 2, "end_date": "2020-03-01", - "start_date": "2020-03-01"}, + {"advertiser_id": 1, "end_date": "2020-01-30", "start_date": "2020-01-01"}, + {"advertiser_id": 1, "end_date": "2020-02-29", "start_date": "2020-01-31"}, + {"advertiser_id": 1, "end_date": "2020-03-01", "start_date": "2020-03-01"}, + {"advertiser_id": 2, "end_date": "2020-01-30", "start_date": "2020-01-01"}, + {"advertiser_id": 2, "end_date": "2020-02-29", "start_date": "2020-01-31"}, + {"advertiser_id": 2, "end_date": "2020-03-01", "start_date": "2020-03-01"}, ], ), ], ) def test_stream_slices_report(advertiser_ids, granularity, slices_expected, pendulum_now_mock): - slices = get_report_stream(AdsReports, granularity)( - **CONFIG).stream_slices() + slices = get_report_stream(AdsReports, granularity)(**CONFIG).stream_slices() assert list(slices) == slices_expected @@ -176,8 +168,7 @@ def test_basic_reports_get_metrics_lifetime(stream, metrics_number): ], ) def test_basic_reports_get_reporting_dimensions_lifetime(stream, dimensions_expected): - dimensions = get_report_stream(stream, Lifetime)( - **CONFIG)._get_reporting_dimensions() + dimensions = get_report_stream(stream, Lifetime)(**CONFIG)._get_reporting_dimensions() assert dimensions == dimensions_expected @@ -188,13 +179,11 @@ def test_basic_reports_get_reporting_dimensions_lifetime(stream, dimensions_expe (AdGroupsReports, ["adgroup_id", "stat_time_day"]), (AdvertisersReports, ["advertiser_id", "stat_time_day"]), (CampaignsReports, ["campaign_id", "stat_time_day"]), - (AdvertisersAudienceReports, [ - "advertiser_id", "stat_time_day", "gender", "age"]), + (AdvertisersAudienceReports, ["advertiser_id", "stat_time_day", "gender", "age"]), ], ) def test_basic_reports_get_reporting_dimensions_day(stream, dimensions_expected): - dimensions = get_report_stream(stream, Daily)( - **CONFIG)._get_reporting_dimensions() + dimensions = get_report_stream(stream, Daily)(**CONFIG)._get_reporting_dimensions() assert dimensions == dimensions_expected @@ -212,11 +201,23 @@ def test_basic_reports_cursor_field(granularity, cursor_field_expected): assert cursor_field == cursor_field_expected +@pytest.mark.parametrize( + "granularity, cursor_field_expected", + [ + (Daily, ["dimensions", "stat_time_day"]), + (Hourly, ["dimensions", "stat_time_hour"]), + (Lifetime, ["dimensions", "stat_time_day"]), + ], +) +def test_basic_reports_deprecated_cursor_field(granularity, cursor_field_expected): + ads_reports = get_report_stream(AdsReports, granularity)(**CONFIG) + deprecated_cursor_field = ads_reports.deprecated_cursor_field + assert deprecated_cursor_field == cursor_field_expected + + def test_request_params(): - stream_slice = {"advertiser_id": 1, - "start_date": "2020", "end_date": "2021"} - params = get_report_stream(AdvertisersAudienceReports, Daily)( - **CONFIG).request_params(stream_slice=stream_slice) + stream_slice = {"advertiser_id": 1, "start_date": "2020", "end_date": "2021"} + params = get_report_stream(AdvertisersAudienceReports, Daily)(**CONFIG).request_params(stream_slice=stream_slice) assert params == { "advertiser_id": 1, "data_level": "AUCTION_ADVERTISER", @@ -242,14 +243,33 @@ def test_get_updated_state(): # state should be empty while stream is reading records ads.max_cursor_date = "2020-01-08 00:00:00" is_finished.return_value = False - state1 = ads.get_updated_state( - current_stream_state=state, latest_record={}) + state1 = ads.get_updated_state(current_stream_state=state, latest_record={}) assert state1 == {"modify_time": ""} # state should be updated only when all records have been read (is_finished = True) is_finished.return_value = True - state2 = ads.get_updated_state( - current_stream_state=state, latest_record={}) + state2 = ads.get_updated_state(current_stream_state=state, latest_record={}) # state2_modify_time is JsonUpdatedState object state2_modify_time = state2["modify_time"] assert state2_modify_time.dict() == "2020-01-08 00:00:00" + + +@pytest.mark.parametrize( + "value, expected", + [ + (["str1", "str2", "str3"], '["str1", "str2", "str3"]'), + ([1, 2, 3], "[1, 2, 3]"), + ], +) +def test_convert_array_param(value, expected): + stream = Advertisers("2021-01-01", "2021-01-02") + test = stream.convert_array_param(value) + assert test == expected + + +def test_no_next_page_token(requests_mock): + stream = Advertisers("2021-01-01", "2021-01-02") + url = stream.url_base + stream.path() + requests_mock.get(url, json={"data": {"page_info": {}}}) + test_response = requests.get(url) + assert stream.next_page_token(test_response) is None diff --git a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py index 073b318e006f..6377e8eae84f 100644 --- a/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py @@ -134,8 +134,8 @@ def test_random_items(prepared_prod_args): @pytest.mark.parametrize( "config, stream_len", [ - (PROD_CONFIG_FILE, 30), - (SANDBOX_CONFIG_FILE, 22), + (PROD_CONFIG_FILE, 36), + (SANDBOX_CONFIG_FILE, 28), ], ) def test_source_streams(config, stream_len): @@ -186,3 +186,13 @@ def test_source_prepare_stream_args(config_file): config = json.load(f) args = SourceTiktokMarketing._prepare_stream_args(config) assert "authenticator" in args + + +def test_minimum_start_date(config, caplog): + config["start_date"] = "2000-01-01" + source = SourceTiktokMarketing() + streams = source.streams(config) + + for stream in streams: + assert stream._start_time == "2012-01-01 00:00:00" + assert "The start date is too far in the past. Setting it to 2012-01-01" in caplog.text diff --git a/airbyte-integrations/connectors/source-timely/Dockerfile b/airbyte-integrations/connectors/source-timely/Dockerfile index 82cb0a6003e4..09470ad4f373 100644 --- a/airbyte-integrations/connectors/source-timely/Dockerfile +++ b/airbyte-integrations/connectors/source-timely/Dockerfile @@ -34,5 +34,5 @@ COPY source_timely ./source_timely ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.3.0 LABEL io.airbyte.name=airbyte/source-timely diff --git a/airbyte-integrations/connectors/source-timely/README.md b/airbyte-integrations/connectors/source-timely/README.md index a64bb8add0da..fefa316371f7 100644 --- a/airbyte-integrations/connectors/source-timely/README.md +++ b/airbyte-integrations/connectors/source-timely/README.md @@ -1,45 +1,12 @@ # Timely Source -This is the repository for the Timely source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/timely). +This is the repository for the Timely configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/timely). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-timely:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/timely) +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/timely) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_timely/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. @@ -47,28 +14,21 @@ See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source timely test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-timely:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-timely build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-timely:airbyteDocker +An image will be built with the tag `airbyte/source-timely:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-timely:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-timely:dev check --con docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-timely:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-timely:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-timely test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-timely:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-timely:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-timely test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/timely.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-timely/__init__.py b/airbyte-integrations/connectors/source-timely/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-timely/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-timely/acceptance-test-config.yml b/airbyte-integrations/connectors/source-timely/acceptance-test-config.yml index feeee984bd84..a9911c73472b 100644 --- a/airbyte-integrations/connectors/source-timely/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-timely/acceptance-test-config.yml @@ -1,20 +1,25 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-timely:dev -tests: +acceptance_tests: spec: - - spec_path: "source_timely/spec.json" + tests: + - spec_path: "source_timely/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-timely/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-timely/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-timely/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-timely/build.gradle b/airbyte-integrations/connectors/source-timely/build.gradle deleted file mode 100644 index 83b94682be2d..000000000000 --- a/airbyte-integrations/connectors/source-timely/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_timely_integration' -} diff --git a/airbyte-integrations/connectors/source-timely/integration_tests/__init__.py b/airbyte-integrations/connectors/source-timely/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-timely/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-timely/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-timely/metadata.yaml b/airbyte-integrations/connectors/source-timely/metadata.yaml index f0783a97a8b5..f9b1f4fbbcfb 100644 --- a/airbyte-integrations/connectors/source-timely/metadata.yaml +++ b/airbyte-integrations/connectors/source-timely/metadata.yaml @@ -1,24 +1,28 @@ data: + allowedHosts: + hosts: + - api.timelyapp.com + registries: + oss: + enabled: true + cloud: + enabled: false connectorSubtype: api connectorType: source definitionId: bc617b5f-1b9e-4a2d-bebe-782fd454a771 - dockerImageTag: 0.1.0 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-timely githubIssueLabel: source-timely icon: timely.svg license: MIT name: Timely - registries: - cloud: - enabled: false - oss: - enabled: true + releaseDate: 2022-06-22 releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/timely tags: - - language:python + - language:low-code ab_internal: sl: 100 ql: 100 - supportLevel: community metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-timely/setup.py b/airbyte-integrations/connectors/source-timely/setup.py index f4579a7b9c50..6dc55b4722ef 100644 --- a/airbyte-integrations/connectors/source-timely/setup.py +++ b/airbyte-integrations/connectors/source-timely/setup.py @@ -5,24 +5,18 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", -] +MAIN_REQUIREMENTS = ["airbyte-cdk"] -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest~=6.1", - "pytest-mock~=3.6.1", -] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( name="source_timely", - description="Source implementation for Timely Integration.", + description="Source implementation for Timely.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-timely/source_timely/__init__.py b/airbyte-integrations/connectors/source-timely/source_timely/__init__.py index ca546fb9f12b..55ee5b8be476 100644 --- a/airbyte-integrations/connectors/source-timely/source_timely/__init__.py +++ b/airbyte-integrations/connectors/source-timely/source_timely/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-timely/source_timely/manifest.yaml b/airbyte-integrations/connectors/source-timely/source_timely/manifest.yaml new file mode 100644 index 000000000000..9557f77b50e5 --- /dev/null +++ b/airbyte-integrations/connectors/source-timely/source_timely/manifest.yaml @@ -0,0 +1,46 @@ +version: "0.29.0" +type: DeclarativeSource +check: + type: CheckStream + stream_names: + - events +streams: + - type: DeclarativeStream + name: events + primary_key: + - id + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.timelyapp.com/1.1/ + path: "{{ config.account_id }}/events" + http_method: GET + request_parameters: + account_id: "{{ config.account_id }}" + since: "{{ config.start_date }}" + upto: "{{ now_utc().strftime('%Y-%m-%d') }}" + request_headers: + Content-Type: application/json + authenticator: + type: BearerAuthenticator + api_token: "{{ config['bearer_token'] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + inject_into: request_parameter + field_name: per_page + type: RequestOption + pagination_strategy: + type: PageIncrement + start_from_page: 1 + page_size: 1000 diff --git a/airbyte-integrations/connectors/source-timely/source_timely/schemas/events.json b/airbyte-integrations/connectors/source-timely/source_timely/schemas/events.json index 29b5f586b99c..3eb8707a5863 100644 --- a/airbyte-integrations/connectors/source-timely/source_timely/schemas/events.json +++ b/airbyte-integrations/connectors/source-timely/source_timely/schemas/events.json @@ -1,6 +1,7 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "invoice_id": { "type": ["null", "string"] @@ -11,6 +12,19 @@ "locked": { "type": ["null", "boolean"] }, + "creator_id": { + "type": ["null", "integer"] + }, + "state": { + "type": ["null", "object"], + "additionalProperties": true + }, + "timestamps": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, "created_id": { "type": ["null", "integer"] }, @@ -26,7 +40,8 @@ "airbyte_type": "timestamp_with_timezone" }, "estimated_cost": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "amount": { "type": ["null", "number"] @@ -46,13 +61,19 @@ "type": ["null", "integer"] }, "label_ids": { - "type": ["null", "array"] + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, "forecast_id": { "type": ["null", "string"] }, "user_ids": { - "type": ["null", "array"] + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, "timer_stopped_on": { "type": ["null", "integer"] @@ -65,7 +86,8 @@ "type": ["null", "integer"] }, "cost": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "amount": { "type": ["null", "number"] @@ -90,13 +112,20 @@ "type": ["null", "integer"] }, "project": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "required_label_ids": { - "type": ["null", "array"] + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, "labels": { - "type": ["null", "array"] + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, "invoice_by_budget": { "type": ["null", "boolean"] @@ -123,7 +152,10 @@ "type": ["null", "boolean"] }, "label_ids": { - "type": ["null", "array"] + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, "enable_labels": { "type": ["null", "string"] @@ -135,7 +167,8 @@ "type": ["null", "boolean"] }, "client": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "external_id": { "type": ["null", "string"] @@ -195,7 +228,8 @@ } }, "user": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "name": { "type": ["null", "string"] @@ -209,7 +243,8 @@ "airbyte_type": "timestamp_with_timezone" }, "avatar": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "medium": { "type": ["null", "string"] @@ -240,7 +275,8 @@ "type": ["null", "string"] }, "estimated_duration": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "total_minutes": { "type": ["null", "integer"] @@ -286,7 +322,8 @@ "type": ["null", "integer"] }, "duration": { - "type": "object", + "type": ["null", "object"], + "additionalProperties": true, "properties": { "total_minutes": { "type": ["null", "integer"] @@ -321,7 +358,10 @@ "type": ["null", "string"] }, "entry_ids": { - "type": ["null", "array"] + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, "hour_rate": { "type": ["null", "number"] @@ -334,6 +374,9 @@ }, "timer_started_on": { "type": ["null", "integer"] + }, + "updated_at": { + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-timely/source_timely/source.py b/airbyte-integrations/connectors/source-timely/source_timely/source.py index a911484b15a6..d7cd0f1c1870 100644 --- a/airbyte-integrations/connectors/source-timely/source_timely/source.py +++ b/airbyte-integrations/connectors/source-timely/source_timely/source.py @@ -2,92 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from abc import ABC -from datetime import datetime -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple -from urllib.parse import parse_qs, urlparse +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. +WARNING: Do not modify this file. +""" -# Basic full refresh stream -class TimelyIntegrationStream(HttpStream, ABC): - FIRST_PAGE = 1 - primary_key = "id" - url_base = "https://api.timelyapp.com/1.1/" - def __init__(self, account_id: str, start_date: str, bearer_token: str, **kwargs): - super().__init__(**kwargs) - self.account_id = account_id - self.start_date = start_date - self.bearer_token = bearer_token - - def request_params( - self, stream_state: Mapping[str, any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - - if next_page_token is None: - return {"page": self.FIRST_PAGE, "per_page": "1000", "account_id": self.account_id} - - return next_page_token - - def request_headers(self, **kwargs) -> Mapping[str, Any]: - bearer_token = self.bearer_token - event_headers = {"Authorization": f"Bearer {bearer_token}", "Content-Type": "application/json"} - return event_headers - - def parse_response( - self, - response: requests.Response, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[Mapping]: - return response.json() - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - results = response.json() - if results: - if len(results) > 0: - url_query = urlparse(response.url).query - query_params = parse_qs(url_query) - - new_params = {param_name: param_value[0] for param_name, param_value in query_params.items()} - if "page" in new_params: - new_params["page"] = int(new_params["page"]) + 1 - return new_params - - -class Events(TimelyIntegrationStream): - # https://dev.timelyapp.com/#list-all-events - primary_key = "id" - - def path(self, **kwargs) -> str: - account_id = self.account_id - start_date = self.start_date - upto = datetime.today().strftime("%Y-%m-%d") - return f"{account_id}/events?since={start_date}&upto={upto}" - - -class SourceTimely(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - account_id = config["account_id"] - start_date = config["start_date"] - bearer_token = config["bearer_token"] - - headers = {"Authorization": f"Bearer {bearer_token}", "Content-Type": "application/json"} - url = f"https://api.timelyapp.com/1.1/{account_id}/events?since={start_date}&upto=2022-05-01" - - try: - session = requests.get(url, headers=headers) - session.raise_for_status() - return True, None - except requests.exceptions.RequestException as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - args = {"bearer_token": config["bearer_token"], "account_id": config["account_id"], "start_date": config["start_date"]} - return [Events(**args)] +# Declarative Source +class SourceTimely(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-timely/source_timely/spec.json b/airbyte-integrations/connectors/source-timely/source_timely/spec.json deleted file mode 100644 index 98b085bdcf35..000000000000 --- a/airbyte-integrations/connectors/source-timely/source_timely/spec.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "documentationUrl": "https://docsurl.com", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Timely Integration Spec", - "type": "object", - "required": ["account_id", "start_date", "bearer_token"], - "additionalProperties": false, - "properties": { - "account_id": { - "title": "account_id", - "type": "string", - "description": "Timely account id" - }, - "start_date": { - "title": "startDate", - "type": "string", - "description": "start date", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", - "example": "2022-05-06" - }, - "bearer_token": { - "title": "Bearer token", - "type": "string", - "description": "Timely bearer token" - } - } - } -} diff --git a/airbyte-integrations/connectors/source-timely/source_timely/spec.yaml b/airbyte-integrations/connectors/source-timely/source_timely/spec.yaml new file mode 100644 index 000000000000..ccaf47579243 --- /dev/null +++ b/airbyte-integrations/connectors/source-timely/source_timely/spec.yaml @@ -0,0 +1,25 @@ +documentationUrl: https://docs.airbyte.io/integrations/sources/timely +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Timely Integration Spec + type: object + required: + - account_id + - start_date + - bearer_token + additionalProperties: true + properties: + account_id: + title: account_id + type: string + description: Timely account id + start_date: + title: startDate + type: string + description: start date + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + example: "2022-05-06" + bearer_token: + title: Bearer token + type: string + description: Timely bearer token diff --git a/airbyte-integrations/connectors/source-timely/unit_tests/__init__.py b/airbyte-integrations/connectors/source-timely/unit_tests/__init__.py deleted file mode 100644 index 46b7376756ec..000000000000 --- a/airbyte-integrations/connectors/source-timely/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-timely/unit_tests/test_source.py b/airbyte-integrations/connectors/source-timely/unit_tests/test_source.py deleted file mode 100644 index d746d37b3a06..000000000000 --- a/airbyte-integrations/connectors/source-timely/unit_tests/test_source.py +++ /dev/null @@ -1,30 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -from pytest import fixture -from source_timely.source import SourceTimely - - -@fixture() -def config(request): - args = {"account_id": "123", "start_date": "2022-04-01", "bearer_token": "51UWRAsFuIbeygfIY3XfucQUGiX"} - return args - - -def test_check_connection(mocker, config): - source = SourceTimely() - logger_mock = MagicMock() - (connection_status, error) = source.check_connection(logger_mock, config) - expected_status = False - assert connection_status == expected_status - - -def test_streams(mocker, config): - source = SourceTimely() - streams = source.streams(config) - # TODO: replace this with your streams number - expected_streams_number = 1 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-timely/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-timely/unit_tests/test_streams.py deleted file mode 100644 index fd25b2aea1a9..000000000000 --- a/airbyte-integrations/connectors/source-timely/unit_tests/test_streams.py +++ /dev/null @@ -1,73 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_timely.source import TimelyIntegrationStream - - -@pytest.fixture() -def config(request): - args = {"account_id": "123", "start_date": "2022-04-01", "bearer_token": "51UWRAsFuIbeygfIY3XfucQUGiX"} - return args - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(TimelyIntegrationStream, "path", "v0/example_endpoint") - mocker.patch.object(TimelyIntegrationStream, "primary_key", "test_primary_key") - mocker.patch.object(TimelyIntegrationStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class, config): - stream = TimelyIntegrationStream(**config) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {"account_id": "123", "page": 1, "per_page": "1000"} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class, config): - stream = TimelyIntegrationStream(**config) - inputs = {"response": MagicMock()} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_request_headers(patch_base_class, config): - stream = TimelyIntegrationStream(**config) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {"Authorization": "Bearer 51UWRAsFuIbeygfIY3XfucQUGiX", "Content-Type": "application/json"} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class, config): - stream = TimelyIntegrationStream(**config) - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry, config): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = TimelyIntegrationStream(**config) - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class, config): - response_mock = MagicMock() - stream = TimelyIntegrationStream(**config) - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-tmdb/README.md b/airbyte-integrations/connectors/source-tmdb/README.md index 25b3dbddd3ec..a4ed1e3439f9 100644 --- a/airbyte-integrations/connectors/source-tmdb/README.md +++ b/airbyte-integrations/connectors/source-tmdb/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-tmdb:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/tmdb) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_tmdb/spec.yaml` file. @@ -49,18 +41,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-tmdb:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-tmdb build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-tmdb:airbyteDocker +An image will be built with the tag `airbyte/source-tmdb:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-tmdb:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -70,25 +63,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-tmdb:dev check --confi docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-tmdb:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-tmdb:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-tmdb test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-tmdb:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-tmdb:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -97,8 +82,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-tmdb test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/tmdb.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-tmdb/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-tmdb/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-tmdb/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-tmdb/build.gradle b/airbyte-integrations/connectors/source-tmdb/build.gradle deleted file mode 100644 index e46f30c1d7df..000000000000 --- a/airbyte-integrations/connectors/source-tmdb/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_tmdb' -} diff --git a/airbyte-integrations/connectors/source-todoist/.dockerignore b/airbyte-integrations/connectors/source-todoist/.dockerignore deleted file mode 100644 index d853b63208a0..000000000000 --- a/airbyte-integrations/connectors/source-todoist/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -* -!Dockerfile -!main.py -!source_todoist -!setup.py -!secrets diff --git a/airbyte-integrations/connectors/source-todoist/Dockerfile b/airbyte-integrations/connectors/source-todoist/Dockerfile deleted file mode 100644 index c30309061741..000000000000 --- a/airbyte-integrations/connectors/source-todoist/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.13-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_todoist ./source_todoist - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-todoist diff --git a/airbyte-integrations/connectors/source-todoist/README.md b/airbyte-integrations/connectors/source-todoist/README.md index a3882df6c753..b68dcdc0a9bf 100644 --- a/airbyte-integrations/connectors/source-todoist/README.md +++ b/airbyte-integrations/connectors/source-todoist/README.md @@ -1,45 +1,13 @@ # Todoist Source -This is the repository for the Todoist source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/todoist). +This is the repository for the Todoist configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/todoist). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-todoist:build -``` #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/todoist) +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/todoist) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_todoist/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. @@ -47,28 +15,69 @@ See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source todoist test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-todoist:dev +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name source-todoist build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-todoist:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container -You can also build the connector image via Gradle: + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-todoist:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-todoist:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. + +2. Build your image: +```bash +docker build -t airbyte/source-todoist:dev . +# Running the spec command against your patched connector +docker run airbyte/source-todoist:dev spec #### Run Then run any of the connector commands as follows: @@ -79,42 +88,13 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-todoist:dev discover - docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-todoist:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-todoist:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-todoist:integrationTest +Please run acceptance tests via [airbyte-ci](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#connectors-test-command): +```bash +airbyte-ci connectors --name source-todoist test ``` ## Dependency Management @@ -125,8 +105,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-todoist test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/todoist.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-todoist/__init__.py b/airbyte-integrations/connectors/source-todoist/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-todoist/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-todoist/acceptance-test-config.yml b/airbyte-integrations/connectors/source-todoist/acceptance-test-config.yml index 88600ec88177..25c0dc8dc1d4 100644 --- a/airbyte-integrations/connectors/source-todoist/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-todoist/acceptance-test-config.yml @@ -1,7 +1,6 @@ -# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-todoist:dev -test_strictness_level: low acceptance_tests: spec: tests: @@ -20,19 +19,20 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.txt" -# extra_fields: no -# exact_order: no -# extra_records: yes - incremental: + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.jsonl" + # extra_fields: no + # exact_order: no + # extra_records: yes + incremental: bypass_reason: "This connector does not implement incremental sync" -# TODO uncomment this block this block if your connector implements incremental sync: -# tests: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# future_state_path: "integration_tests/abnormal_state.json" + # TODO uncomment this block this block if your connector implements incremental sync: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-todoist/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-todoist/acceptance-test-docker.sh deleted file mode 100755 index c51577d10690..000000000000 --- a/airbyte-integrations/connectors/source-todoist/acceptance-test-docker.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env sh - -# Build latest connector image -docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) - -# Pull latest acctest image -docker pull airbyte/source-acceptance-test:latest - -# Run -docker run --rm -it \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /tmp:/tmp \ - -v $(pwd):/test_input \ - airbyte/source-acceptance-test \ - --acceptance-test-config /test_input - diff --git a/airbyte-integrations/connectors/source-todoist/build.gradle b/airbyte-integrations/connectors/source-todoist/build.gradle deleted file mode 100644 index 08a1b8687274..000000000000 --- a/airbyte-integrations/connectors/source-todoist/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_todoist' -} diff --git a/airbyte-integrations/connectors/source-todoist/icon.svg b/airbyte-integrations/connectors/source-todoist/icon.svg index dbf417c99c8d..c4df54834efe 100644 --- a/airbyte-integrations/connectors/source-todoist/icon.svg +++ b/airbyte-integrations/connectors/source-todoist/icon.svg @@ -1,6 +1,14 @@ - - - - - - + + + + + + + + + + + + + + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/__init__.py b/airbyte-integrations/connectors/source-todoist/integration_tests/__init__.py index 1100c1c58cf5..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-todoist/integration_tests/abnormal_state.json index 09f16c3ccf2a..52b0f2c2118f 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/abnormal_state.json @@ -1,37 +1,5 @@ { - "tasks": { - "id": false, - "project_id": 10, - "section_id": -10, - "content": 10, - "description": true, - "is_completed": "not so true", - "labels": [true, false], - "parent_id": -50, - "order": true, - "priority": "10", - "due": true, - "url": 50, - "comment_count": "10", - "created_at": { "when": true }, - "creator_id": -1, - "assignee_id": 20, - "assigner_id": 50 - }, - "projects": { - "id": { - "number": "2203306141" - }, - "name": 50, - "comment_count": false, - "order": "1", - "color": 100, - "is_shared": [false], - "is_favorite": "world", - "parent_id": 100, - "is_inbox_project": [true], - "is_team_inbox": "false", - "view_style": ["list"], - "url": ["https://todoist.com/showProject?id=2203306141"] + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" } } diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json index 275f28ac14f5..8fed59fd4d9e 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/configured_catalog.json @@ -1,34 +1,22 @@ { "streams": [ { - "cursor_field": null, - "destination_sync_mode": "append", - "primary_key": null, "stream": { - "default_cursor_field": null, - "json_schema": {}, "name": "tasks", - "namespace": null, - "source_defined_cursor": null, - "source_defined_primary_key": [["id"]], + "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "full_refresh" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" }, { - "cursor_field": null, - "destination_sync_mode": "append", - "primary_key": null, "stream": { - "default_cursor_field": null, - "json_schema": {}, "name": "projects", - "namespace": null, - "source_defined_cursor": null, - "source_defined_primary_key": [["id"]], + "json_schema": {}, "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "full_refresh" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json index 37cbe64ae615..244ec5755c74 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/invalid_config.json @@ -1,3 +1,3 @@ { - "token": "INVALID TOKEN" + "token": "INVALID_API_TOKEN" } diff --git a/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json index 55e640b7e5c3..6dbfca1a0354 100644 --- a/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-todoist/integration_tests/sample_config.json @@ -1,3 +1,3 @@ { - "token": "VALID TOKEN" + "token": "API_TOKEN" } diff --git a/airbyte-integrations/connectors/source-todoist/metadata.yaml b/airbyte-integrations/connectors/source-todoist/metadata.yaml index f32560daec66..b578c8f07755 100644 --- a/airbyte-integrations/connectors/source-todoist/metadata.yaml +++ b/airbyte-integrations/connectors/source-todoist/metadata.yaml @@ -1,24 +1,30 @@ data: + allowedHosts: + hosts: + - api.todoist.com/rest/v2 + registries: + oss: + enabled: true + cloud: + enabled: false + connectorBuildOptions: + # Please update to the latest version of the connector base image. + # https://hub.docker.com/r/airbyte/python-connector-base + # Please use the full address with sha256 hash to guarantee build reproducibility. + baseImage: docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27 connectorSubtype: api connectorType: source - definitionId: 7d272065-c316-4c04-a433-cd4ee143f83e - dockerImageTag: 0.1.0 + definitionId: 1a3d38e4-dc6b-4154-b56b-582f9e978ecd + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-todoist githubIssueLabel: source-todoist icon: todoist.svg license: MIT name: Todoist - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: 2023-12-10 releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/todoist tags: - - language:python - ab_internal: - sl: 100 - ql: 100 - supportLevel: community + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-todoist/requirements.txt b/airbyte-integrations/connectors/source-todoist/requirements.txt index ecf975e2fa63..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-todoist/requirements.txt +++ b/airbyte-integrations/connectors/source-todoist/requirements.txt @@ -1 +1 @@ --e . \ No newline at end of file +-e . diff --git a/airbyte-integrations/connectors/source-todoist/setup.py b/airbyte-integrations/connectors/source-todoist/setup.py index cf672aeb4515..3add92262189 100644 --- a/airbyte-integrations/connectors/source-todoist/setup.py +++ b/airbyte-integrations/connectors/source-todoist/setup.py @@ -6,14 +6,13 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", - "pytest~=6.1", - "pytest-mock~=3.6", - "requests_mock~=1.8", + "pytest~=6.2", + "pytest-mock~=3.6.1", ] setup( diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/__init__.py b/airbyte-integrations/connectors/source-todoist/source_todoist/__init__.py index f04f17ce4a6f..62d0c357fa93 100644 --- a/airbyte-integrations/connectors/source-todoist/source_todoist/__init__.py +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/manifest.yaml b/airbyte-integrations/connectors/source-todoist/source_todoist/manifest.yaml new file mode 100644 index 000000000000..8460493fe200 --- /dev/null +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/manifest.yaml @@ -0,0 +1,62 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + requester: + type: HttpRequester + url_base: "https://api.todoist.com/rest/v2" + http_method: "GET" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['token'] }}" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester" + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + tasks_stream: + $ref: "#/definitions/base_stream" + name: "tasks" + $parameters: + path: "/tasks" + projects_stream: + $ref: "#/definitions/base_stream" + name: "projects" + $parameters: + path: "/projects" + +streams: + - "#/definitions/tasks_stream" + - "#/definitions/projects_stream" + +check: + type: CheckStream + stream_names: + - "tasks" + - "projects" + +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/source-todolist + connection_specification: + title: Source Todolist Spec + type: object + required: + - token + additionalProperties: true + properties: + token: + type: string + description: API authorization bearer token for authenticating the API + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/employees.json b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/employees.json new file mode 100644 index 000000000000..c9bce00c9315 --- /dev/null +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/employees.json @@ -0,0 +1,82 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "type": "object", + "properties": { + "assignee_id": { + "type": ["null", "string"] + }, + "assigner_id": { + "type": ["null", "string"] + }, + "comment_count": { + "type": ["null", "integer"] + }, + "content": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "creator_id": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "due": { + "anyOf": [ + { + "type": ["null", "object"] + }, + { + "properties": { + "date": { + "type": ["null", "string"] + }, + "is_recurring": { + "type": ["null", "boolean"] + }, + "lang": { + "type": ["null", "string"] + }, + "string": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + } + ] + }, + "id": { + "type": ["null", "string"] + }, + "is_completed": { + "type": ["null", "boolean"] + }, + "labels": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "order": { + "type": ["null", "integer"] + }, + "parent_id": { + "type": ["null", "string"] + }, + "priority": { + "type": ["null", "integer"] + }, + "project_id": { + "type": ["null", "string"] + }, + "section_id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/tasks.json b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/tasks.json index cf22f8da8a4b..f5d926e1d087 100644 --- a/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/tasks.json +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/schemas/tasks.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/schema#", + "$schema": "http://json-schema.org/draft-07/schema#", "additionalProperties": true, "type": "object", "properties": { @@ -51,6 +51,9 @@ "id": { "type": ["null", "string"] }, + "duration": { + "type": ["null", "string"] + }, "is_completed": { "type": ["null", "boolean"] }, diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/source.py b/airbyte-integrations/connectors/source-todoist/source_todoist/source.py index 517c10ae7383..b363f5fb2623 100644 --- a/airbyte-integrations/connectors/source-todoist/source_todoist/source.py +++ b/airbyte-integrations/connectors/source-todoist/source_todoist/source.py @@ -2,66 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. +WARNING: Do not modify this file. +""" -# Basic full refresh stream -class TodoistStream(HttpStream): - """ - Stream for Todoist REST API : https://developer.todoist.com/rest/v2/#overview - """ - @property - def url_base(self) -> str: - return "https://api.todoist.com/rest/v2/" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return {} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield from response.json() - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return self.name.title().lower() - - -class Tasks(TodoistStream): - - primary_key = "id" - - -class Projects(TodoistStream): - - primary_key = "id" - - -# Source -class SourceTodoist(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - try: - token = config["token"] - authenticator = TokenAuthenticator(token=token) - task_stream = Tasks(authenticator) - task_records = task_stream.read_records(sync_mode="full_refresh") - next(task_records) - return True, None - except Exception as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - token = config["token"] - auth = TokenAuthenticator(token=token) # Oauth2Authenticator is also available if you need oauth support - return [Tasks(authenticator=auth), Projects(authenticator=auth)] +# Declarative Source +class SourceTodoist(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-todoist/source_todoist/spec.yaml b/airbyte-integrations/connectors/source-todoist/source_todoist/spec.yaml deleted file mode 100644 index 19e7f9dc2e42..000000000000 --- a/airbyte-integrations/connectors/source-todoist/source_todoist/spec.yaml +++ /dev/null @@ -1,15 +0,0 @@ -documentationUrl: https://docs.airbyte.io/integrations/sources/todoist -connectionSpecification: - $schema: http://json-schema.org/draft-07/schema# - title: Todoist Spec - type: object - required: - - token - properties: - token: - type: string - description: >- - Your API Token. See here. The token is - case sensitive. - airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-todoist/unit_tests/__init__.py b/airbyte-integrations/connectors/source-todoist/unit_tests/__init__.py deleted file mode 100644 index 1100c1c58cf5..000000000000 --- a/airbyte-integrations/connectors/source-todoist/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-todoist/unit_tests/test_source.py b/airbyte-integrations/connectors/source-todoist/unit_tests/test_source.py deleted file mode 100644 index d4dbafa573be..000000000000 --- a/airbyte-integrations/connectors/source-todoist/unit_tests/test_source.py +++ /dev/null @@ -1,25 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock, patch - -from source_todoist.source import SourceTodoist - - -def test_check_connection(mocker): - source = SourceTodoist() - fake_info_record = {"collection": "is_mocked"} - with patch("source_todoist.source.Tasks.read_records", MagicMock(return_value=iter([fake_info_record]))): - logger_mock = MagicMock() - config_mock = {"token": "test"} - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -def test_streams(mocker): - source = SourceTodoist() - config_mock = MagicMock() - streams = source.streams(config_mock) - # TODO: replace this with your streams number - expected_streams_number = 2 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-todoist/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-todoist/unit_tests/test_streams.py deleted file mode 100644 index af1ed528da42..000000000000 --- a/airbyte-integrations/connectors/source-todoist/unit_tests/test_streams.py +++ /dev/null @@ -1,65 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_todoist.source import TodoistStream - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(TodoistStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = TodoistStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = TodoistStream() - inputs = {"response": MagicMock()} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_request_headers(patch_base_class): - stream = TodoistStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = TodoistStream() - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = TodoistStream() - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = TodoistStream() - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-toggl/README.md b/airbyte-integrations/connectors/source-toggl/README.md index e4623b653ad8..d196b009f6d9 100644 --- a/airbyte-integrations/connectors/source-toggl/README.md +++ b/airbyte-integrations/connectors/source-toggl/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-toggl:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/toggl) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_toggl/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-toggl:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-toggl build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-toggl:airbyteDocker +An image will be built with the tag `airbyte/source-toggl:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-toggl:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-toggl:dev check --conf docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-toggl:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-toggl:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-toggl test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-toggl:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-toggl:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-toggl test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/toggl.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-toggl/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-toggl/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-toggl/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-toggl/build.gradle b/airbyte-integrations/connectors/source-toggl/build.gradle deleted file mode 100644 index 3cca4b4c1c4d..000000000000 --- a/airbyte-integrations/connectors/source-toggl/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_toggl' -} diff --git a/airbyte-integrations/connectors/source-tplcentral/README.md b/airbyte-integrations/connectors/source-tplcentral/README.md index 822a6a1cbf0d..ba7a0aa252bf 100644 --- a/airbyte-integrations/connectors/source-tplcentral/README.md +++ b/airbyte-integrations/connectors/source-tplcentral/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-tplcentral:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/tplcentral) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_tplcentral/spec.json` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-tplcentral:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-tplcentral build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-tplcentral:airbyteDocker +An image will be built with the tag `airbyte/source-tplcentral:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-tplcentral:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-tplcentral:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-tplcentral:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-tplcentral:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-tplcentral test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-tplcentral:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-tplcentral:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-tplcentral test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/tplcentral.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-tplcentral/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-tplcentral/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-tplcentral/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-tplcentral/build.gradle b/airbyte-integrations/connectors/source-tplcentral/build.gradle deleted file mode 100644 index 8c3c297045e7..000000000000 --- a/airbyte-integrations/connectors/source-tplcentral/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_tplcentral' -} diff --git a/airbyte-integrations/connectors/source-trello/.dockerignore b/airbyte-integrations/connectors/source-trello/.dockerignore index 4ab13fc2e420..5fda561be30c 100644 --- a/airbyte-integrations/connectors/source-trello/.dockerignore +++ b/airbyte-integrations/connectors/source-trello/.dockerignore @@ -1,6 +1,5 @@ * !Dockerfile -!Dockerfile.test !main.py !source_trello !setup.py diff --git a/airbyte-integrations/connectors/source-trello/Dockerfile b/airbyte-integrations/connectors/source-trello/Dockerfile index ab1cf6ecde2d..c82e892c2289 100644 --- a/airbyte-integrations/connectors/source-trello/Dockerfile +++ b/airbyte-integrations/connectors/source-trello/Dockerfile @@ -6,7 +6,9 @@ WORKDIR /airbyte/integration_code # upgrade pip to the latest version RUN apk --no-cache upgrade \ - && pip install --upgrade pip + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + COPY setup.py ./ # install necessary packages to a temporary folder @@ -18,8 +20,11 @@ WORKDIR /airbyte/integration_code # copy all loaded and built libraries to a pure basic image COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone -# Bash is installed for more convenient debugging. +# bash is installed for more convenient debugging. RUN apk --no-cache add bash # copy payload code only @@ -29,5 +34,6 @@ COPY source_trello ./source_trello ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.3.4 + +LABEL io.airbyte.version=1.0.2 LABEL io.airbyte.name=airbyte/source-trello diff --git a/airbyte-integrations/connectors/source-trello/README.md b/airbyte-integrations/connectors/source-trello/README.md index c713b949a5ad..debe2e6038f2 100644 --- a/airbyte-integrations/connectors/source-trello/README.md +++ b/airbyte-integrations/connectors/source-trello/README.md @@ -1,73 +1,34 @@ # Trello Source -This is the repository for the Trello source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/trello). +This is the repository for the Trello configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/trello). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-trello:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/trello) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_trello/spec.json` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/trello) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_trello/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source trello test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-trello:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-trello build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-trello:airbyteDocker +An image will be built with the tag `airbyte/source-trello:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-trello:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -77,44 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-trello:dev check --con docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-trello:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-trello:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-trello test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-trello:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-trello:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -124,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -2. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -3. Create a Pull Request. -4. Pat yourself on the back for being an awesome contributor. -5. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-trello test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/trello.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-trello/__init__.py b/airbyte-integrations/connectors/source-trello/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-trello/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-trello/acceptance-test-config.yml b/airbyte-integrations/connectors/source-trello/acceptance-test-config.yml index 66a0278cb620..7be14f26455a 100644 --- a/airbyte-integrations/connectors/source-trello/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-trello/acceptance-test-config.yml @@ -5,38 +5,36 @@ test_strictness_level: "high" acceptance_tests: spec: tests: - - spec_path: "source_trello/spec.json" - backward_compatibility_tests_config: - disable_for_version: 0.2.0 + - spec_path: "source_trello/spec.yaml" + backward_compatibility_tests_config: + disable_for_version: 0.2.0 connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: tests: - - config_path: "secrets/config.json" - backward_compatibility_tests_config: - disable_for_version: 0.2.0 + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: 0.2.0 basic_read: tests: - - config_path: "secrets/config.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: no - extra_records: yes - fail_on_extra_columns: false + - config_path: "secrets/config.json" + expect_records: + path: "integration_tests/expected_records.jsonl" + extra_fields: no + exact_order: no + extra_records: yes + fail_on_extra_columns: false incremental: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" - cursor_paths: - actions: ["611aa586ef5f2c8e1deec8b6", "date"] + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-trello/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-trello/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-trello/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-trello/build.gradle b/airbyte-integrations/connectors/source-trello/build.gradle deleted file mode 100644 index 6b4df2291f12..000000000000 --- a/airbyte-integrations/connectors/source-trello/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_trello' -} diff --git a/airbyte-integrations/connectors/source-trello/integration_tests/__init__.py b/airbyte-integrations/connectors/source-trello/integration_tests/__init__.py index e69de29bb2d1..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-trello/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-trello/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-trello/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-trello/integration_tests/abnormal_state.json index 4d239734099d..752578573c81 100644 --- a/airbyte-integrations/connectors/source-trello/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-trello/integration_tests/abnormal_state.json @@ -6,12 +6,16 @@ "name": "actions" }, "stream_state": { - "611aa586ef5f2c8e1deec8b6": { - "date": "2099-03-16T20:10:54.641Z" - }, - "611aa0ef37acd675af67dc9b": { - "date": "2099-03-16T18:47:16.404Z" - } + "states": [ + { + "partition": { "id": "611aa586ef5f2c8e1deec8b6" }, + "cursor": { "date": "2099-03-16T20:10:54.641Z" } + }, + { + "partition": { "id": "611aa0ef37acd675af67dc9b" }, + "cursor": { "date": "2099-03-16T18:47:16.404Z" } + } + ] } } } diff --git a/airbyte-integrations/connectors/source-trello/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-trello/integration_tests/acceptance.py index d49b55882333..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-trello/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-trello/integration_tests/acceptance.py @@ -10,4 +10,7 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-trello/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-trello/integration_tests/expected_records.jsonl index 6951286fd9f5..d030301c8e9f 100644 --- a/airbyte-integrations/connectors/source-trello/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-trello/integration_tests/expected_records.jsonl @@ -13,4 +13,4 @@ {"stream": "lists", "data": {"id": "611aa0ef37acd675af67dc9e", "name": "Done", "closed": false, "idBoard": "611aa0ef37acd675af67dc9b", "pos": 49152, "subscribed": false, "softLimit": null, "status": null}, "emitted_at": 1683406411203} {"stream": "users", "data": {"id": "610be3762899a26d04256dae", "fullName": "integration test", "username": "integrationtest19"}, "emitted_at": 1683406412874} {"stream": "users", "data": {"id": "610be3762899a26d04256dae", "fullName": "integration test", "username": "integrationtest19"}, "emitted_at": 1683406413051} -{"stream": "organizations", "data": {"id": "610be50a0537086c571a5684", "creationMethod": null, "name": "airbyteworkspace", "credits": [], "displayName": "Airbyte Workspace", "desc": "", "descData": {"emoji": {}}, "domainName": "airbyte.io", "idBoards": ["611aa0ef37acd675af67dc9b", "611aa586ef5f2c8e1deec8b6"], "idEnterprise": null, "idMemberCreator": "610be3762899a26d04256dae", "invited": false, "invitations": [], "limits": {"orgs": {"totalMembersPerOrg": {"status": "ok", "disableAt": 4000, "warnAt": 3200}, "freeBoardsPerOrg": {"status": "ok", "disableAt": 10, "warnAt": 3}}}, "membersCount": 1, "prefs": {"permissionLevel": "private", "orgInviteRestrict": [], "boardInviteRestrict": "any", "externalMembersDisabled": false, "associatedDomain": null, "googleAppsVersion": 1, "boardVisibilityRestrict": {"private": "org", "org": "org", "enterprise": "org", "public": "org"}, "boardDeleteRestrict": {"private": "org", "org": "org", "enterprise": "org", "public": "org"}, "attachmentRestrictions": null, "seatAutomation": null, "seatAutomationActiveDays": null, "seatAutomationInactiveDays": null, "newLicenseInviteRestrict": null, "newLicenseInviteRestrictUrl": null, "atlassianIntelligenceEnabled": false}, "powerUps": [], "products": [], "billableMemberCount": 1, "billableCollaboratorCount": 0, "url": "https://trello.com/w/airbyteworkspace", "website": null, "logoHash": null, "logoUrl": null, "premiumFeatures": ["additionalBoardBackgrounds", "additionalStickers", "customBoardBackgrounds", "customEmoji", "customStickers", "plugins"], "promotions": [], "enterpriseJoinRequest": {}, "standardVariation": null, "availableLicenseCount": null, "maximumLicenseCount": null, "ixUpdate": "6", "teamType": null, "dateLastActivity": "2023-03-16T20:10:54.595Z", "memberships": [{"idMember": "610be3762899a26d04256dae", "memberType": "admin", "unconfirmed": false, "deactivated": false, "id": "610be50a0537086c571a5685"}]}, "emitted_at": 1690381821566} +{"stream": "organizations", "data": {"id": "610be50a0537086c571a5684", "creationMethod": null, "name": "airbyteworkspace", "credits": [], "displayName": "Airbyte Workspace", "desc": "", "descData": {"emoji": {}}, "domainName": "airbyte.io", "idBoards": ["611aa0ef37acd675af67dc9b", "611aa586ef5f2c8e1deec8b6"], "idEnterprise": null, "idMemberCreator": "610be3762899a26d04256dae", "invited": false, "invitations": [], "limits": {"orgs": {"totalMembersPerOrg": {"status": "ok", "disableAt": 4000, "warnAt": 3200}, "freeBoardsPerOrg": {"status": "ok", "disableAt": 10, "warnAt": 3}}}, "membersCount": 1, "nodeId": "ari:cloud:trello::workspace/610be50a0537086c571a5684", "prefs": {"permissionLevel": "private", "orgInviteRestrict": [], "boardInviteRestrict": "any", "externalMembersDisabled": false, "associatedDomain": null, "googleAppsVersion": 1, "boardVisibilityRestrict": {"private": "org", "org": "org", "enterprise": "org", "public": "org"}, "boardDeleteRestrict": {"private": "org", "org": "org", "enterprise": "org", "public": "org"}, "attachmentRestrictions": null, "newLicenseInviteRestrict": null, "newLicenseInviteRestrictUrl": null, "atlassianIntelligenceEnabled": false}, "powerUps": [], "products": [], "billableMemberCount": 1, "billableCollaboratorCount": 0, "url": "https://trello.com/w/airbyteworkspace", "website": null, "logoHash": null, "logoUrl": null, "premiumFeatures": ["additionalBoardBackgrounds", "additionalStickers", "customBoardBackgrounds", "customEmoji", "customStickers", "plugins"], "promotions": [], "enterpriseJoinRequest": {}, "standardVariation": null, "availableLicenseCount": null, "maximumLicenseCount": null, "ixUpdate": "6", "teamType": null, "dateLastActivity": "2023-03-16T20:10:54.595Z", "jwmLink": null, "memberships": [{"idMember": "610be3762899a26d04256dae", "memberType": "admin", "unconfirmed": false, "deactivated": false, "id": "610be50a0537086c571a5685"}]}, "emitted_at": 1697218944777} diff --git a/airbyte-integrations/connectors/source-trello/metadata.yaml b/airbyte-integrations/connectors/source-trello/metadata.yaml index 8d72b0bc1636..51d7da393c99 100644 --- a/airbyte-integrations/connectors/source-trello/metadata.yaml +++ b/airbyte-integrations/connectors/source-trello/metadata.yaml @@ -1,27 +1,33 @@ data: + ab_internal: + ql: 300 + sl: 100 allowedHosts: hosts: - api.trello.com + registries: + oss: + enabled: true + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: 8da67652-004c-11ec-9a03-0242ac130003 - dockerImageTag: 0.3.4 + dockerImageTag: 1.0.2 dockerRepository: airbyte/source-trello + documentationUrl: https://docs.airbyte.com/integrations/sources/trello githubIssueLabel: source-trello icon: trello.svg license: MIT name: Trello - registries: - cloud: - enabled: true - oss: - enabled: true + releases: + breakingChanges: + 1.0.0: + upgradeDeadline: "2023-09-28" + message: "The verison migrates the Trello connector to the low-code framework for greater maintainability. This introduces a breaking change to the state format for the `response` stream. If you are using the incremental sync mode for this stream, you will need to reset the affected connection after upgrading to prevent sync failures." + releaseDate: 2021-08-18 releaseStage: beta - documentationUrl: https://docs.airbyte.com/integrations/sources/trello + supportLevel: community tags: - - language:python - ab_internal: - sl: 200 - ql: 300 - supportLevel: certified + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-trello/setup.py b/airbyte-integrations/connectors/source-trello/setup.py index 4209a8aabf1b..38d12ba64785 100644 --- a/airbyte-integrations/connectors/source-trello/setup.py +++ b/airbyte-integrations/connectors/source-trello/setup.py @@ -5,12 +5,14 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1"] +MAIN_REQUIREMENTS = [ + "airbyte-cdk", +] TEST_REQUIREMENTS = [ - "pytest~=6.1", + "requests-mock~=1.9.3", + "pytest~=6.2", "pytest-mock~=3.6.1", - "requests-mock", ] setup( @@ -20,7 +22,7 @@ author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-trello/source_trello/__init__.py b/airbyte-integrations/connectors/source-trello/source_trello/__init__.py index 1267f6961f64..b0f5075e713e 100644 --- a/airbyte-integrations/connectors/source-trello/source_trello/__init__.py +++ b/airbyte-integrations/connectors/source-trello/source_trello/__init__.py @@ -1,25 +1,5 @@ # -# MIT License -# -# Copyright (c) 2020 Airbyte -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-trello/source_trello/auth.py b/airbyte-integrations/connectors/source-trello/source_trello/auth.py deleted file mode 100644 index e34c9a208f70..000000000000 --- a/airbyte-integrations/connectors/source-trello/source_trello/auth.py +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator - - -class TrelloAuthenticator(AbstractHeaderAuthenticator): - """ - https://developer.atlassian.com/cloud/trello/guides/rest-api/authorization/#passing-token-and-key-in-api-requests - """ - - def __init__(self, apiKey: str, apiToken: str): - self.apiKey = apiKey - self.apiToken = apiToken - - @property - def auth_header(self) -> str: - return "Authorization" - - @property - def token(self) -> str: - return f'OAuth oauth_consumer_key="{self.apiKey}", oauth_token="{self.apiToken}"' diff --git a/airbyte-integrations/connectors/source-trello/source_trello/components.py b/airbyte-integrations/connectors/source-trello/source_trello/components.py new file mode 100644 index 000000000000..2f3dd3721704 --- /dev/null +++ b/airbyte-integrations/connectors/source-trello/source_trello/components.py @@ -0,0 +1,44 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import dataclass +from typing import Iterable + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import SubstreamPartitionRouter +from airbyte_cdk.sources.declarative.types import StreamSlice +from airbyte_cdk.sources.streams.core import Stream + + +@dataclass +class OrderIdsPartitionRouter(SubstreamPartitionRouter): + def stream_slices(self) -> Iterable[StreamSlice]: + + stream_map = {stream_config.stream.name: stream_config.stream for stream_config in self.parent_stream_configs} + + board_ids = set(self.config.get("board_ids", [])) + if not board_ids: + board_ids = self.read_all_boards(stream_boards=stream_map["boards"], stream_organizations=stream_map["organizations"]) + + for board_id in board_ids: + yield {"id": board_id} + + def read_all_boards(self, stream_boards: Stream, stream_organizations: Stream): + """ + Method to get id of each board in the boards stream, + get ids of boards associated with each organization in the organizations stream + and yield each unique board id + """ + board_ids = set() + + for record in stream_boards.read_records(sync_mode=SyncMode.full_refresh): + if record["id"] not in board_ids: + board_ids.add(record["id"]) + yield record["id"] + + for record in stream_organizations.read_records(sync_mode=SyncMode.full_refresh): + for board_id in record["idBoards"]: + if board_id not in board_ids: + board_ids.add(board_id) + yield board_id diff --git a/airbyte-integrations/connectors/source-trello/source_trello/manifest.yaml b/airbyte-integrations/connectors/source-trello/source_trello/manifest.yaml new file mode 100644 index 000000000000..35318588768f --- /dev/null +++ b/airbyte-integrations/connectors/source-trello/source_trello/manifest.yaml @@ -0,0 +1,270 @@ +version: 0.51.2 +type: DeclarativeSource + +check: + type: CheckStream + stream_names: + - boards + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + boards_selector: + $ref: "#/definitions/selector" + record_filter: + condition: "{{ record.id in config.board_ids or not config.board_ids }}" + requester: + type: HttpRequester + url_base: https://api.trello.com/1/ + http_method: GET + request_headers: {} + authenticator: + type: ApiKeyAuthenticator + api_token: 'OAuth oauth_consumer_key="{{ config[''key''] }}", oauth_token="{{ config[''token''] }}"' + inject_into: + type: RequestOption + field_name: Authorization + inject_into: header + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + backoff_strategies: + - type: ConstantBackoffStrategy + backoff_time_in_seconds: 10 + request_body_json: {} + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: before + page_size_option: + type: RequestOption + field_name: limit + inject_into: request_parameter + pagination_strategy: + type: CursorPagination + page_size: 500 + cursor_value: "{{ (last_records|last)['id'] }}" + stop_condition: "{{ not last_records }}" + board_id_partition_router: + - type: CustomPartitionRouter + class_name: source_trello.components.OrderIdsPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: id + stream: + $ref: "#/definitions/boards_stream" + - type: ParentStreamConfig + parent_key: idBoards + partition_field: id + stream: + $ref: "#/definitions/organizations_stream" + boards_stream: + type: DeclarativeStream + name: boards + primary_key: id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: members/me/boards + request_parameters: + since: "{{ config['start_date'] }}" + record_selector: + $ref: "#/definitions/boards_selector" + organizations_stream: + type: DeclarativeStream + name: organizations + primary_key: id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: members/me/organizations + request_parameters: + since: "{{ config['start_date'] }}" + record_selector: + $ref: "#/definitions/selector" + actions_stream: + type: DeclarativeStream + name: actions + primary_key: id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: boards/{{ stream_partition.id }}/actions + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/paginator" + partition_router: + $ref: "#/definitions/board_id_partition_router" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: date + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%S.%fZ" + datetime_format: "%Y-%m-%dT%H:%M:%S.%fZ" + start_datetime: + type: MinMaxDatetime + datetime: "{{ config['start_date'] }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: since + inject_into: request_parameter + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + cards_stream: + type: DeclarativeStream + name: cards + primary_key: id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: boards/{{ stream_partition.id }}/cards/all + request_parameters: + list: "true" + sort: "-id" + since: "{{ config['start_date'] }}" + members: "true" + pluginData: "true" + actions_display: "true" + customFieldItems: "true" + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/paginator" + partition_router: + $ref: "#/definitions/board_id_partition_router" + checklists_stream: + type: DeclarativeStream + name: checklists + primary_key: id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: boards/{{ stream_partition.id }}/checklists + request_parameters: + since: "{{ config['start_date'] }}" + fields: all + checkItem_fields: all + record_selector: + $ref: "#/definitions/selector" + partition_router: + $ref: "#/definitions/board_id_partition_router" + lists_stream: + type: DeclarativeStream + name: lists + primary_key: id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: boards/{{ stream_partition.id }}/lists + request_parameters: + since: "{{ config['start_date'] }}" + record_selector: + $ref: "#/definitions/selector" + partition_router: + $ref: "#/definitions/board_id_partition_router" + users_stream: + type: DeclarativeStream + name: users + primary_key: id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: boards/{{ stream_partition.id }}/members + request_parameters: + since: "{{ config['start_date'] }}" + record_selector: + $ref: "#/definitions/selector" + partition_router: + $ref: "#/definitions/board_id_partition_router" + +streams: + - "#/definitions/boards_stream" + - "#/definitions/organizations_stream" + - "#/definitions/actions_stream" + - "#/definitions/cards_stream" + - "#/definitions/checklists_stream" + - "#/definitions/lists_stream" + - "#/definitions/users_stream" + +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/trello + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + type: object + additionalProperties: true + required: + - key + - token + - start_date + properties: + key: + type: string + title: API key + description: Trello API key. See the docs for instructions on how to generate it. + airbyte_secret: true + order: 0 + token: + type: string + title: API token + description: Trello API token. See the docs for instructions on how to generate it. + airbyte_secret: true + order: 1 + start_date: + type: string + title: Start Date + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + description: UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. + examples: + - "2021-03-01T00:00:00Z" + format: date-time + order: 2 + board_ids: + type: array + items: + type: string + pattern: "^[0-9a-fA-F]{24}$" + title: Trello Board IDs + description: IDs of the boards to replicate data from. If left empty, data from all boards to which you have access will be replicated. Please note that this is not the 8-character ID in the board's shortLink (URL of the board). Rather, what is required here is the 24-character ID usually returned by the API + order: 3 + advanced_auth: + auth_flow_type: oauth2.0 + oauth_config_specification: + complete_oauth_output_specification: + type: object + additionalProperties: false + properties: + token: + type: string + path_in_connector_config: + - token + key: + type: string + path_in_connector_config: + - key + complete_oauth_server_input_specification: + type: object + additionalProperties: false + properties: + client_id: + type: string + client_secret: + type: string diff --git a/airbyte-integrations/connectors/source-trello/source_trello/schemas/cards.json b/airbyte-integrations/connectors/source-trello/source_trello/schemas/cards.json index ec2b0187043c..83d139c78bb4 100644 --- a/airbyte-integrations/connectors/source-trello/source_trello/schemas/cards.json +++ b/airbyte-integrations/connectors/source-trello/source_trello/schemas/cards.json @@ -189,6 +189,7 @@ }, "cover": { "type": ["null", "object"], + "additionalProperties": true, "properties": { "idAttachment": { "type": ["null", "string"] @@ -197,13 +198,22 @@ "type": ["null", "string"] }, "idUploadedBackground": { - "type": ["null", "boolean"] + "type": ["null", "string"] }, "size": { "type": ["null", "string"] }, "brightness": { "type": ["null", "string"] + }, + "edgeColor": { + "type": ["null", "string"] + }, + "sharedSourceUrl": { + "type": ["null", "string"] + }, + "idPlugin": { + "type": ["null", "string"] } } }, diff --git a/airbyte-integrations/connectors/source-trello/source_trello/source.py b/airbyte-integrations/connectors/source-trello/source_trello/source.py index 5caaea8444e8..591073e589be 100644 --- a/airbyte-integrations/connectors/source-trello/source_trello/source.py +++ b/airbyte-integrations/connectors/source-trello/source_trello/source.py @@ -2,219 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import datetime -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +WARNING: Do not modify this file. +""" -from .auth import TrelloAuthenticator -from .utils import TrelloRequestRateLimits as balancer -from .utils import read_all_boards - -class TrelloStream(HttpStream, ABC): - url_base = "https://api.trello.com/1/" - - # Define primary key as sort key for full_refresh, or very first sync for incremental_refresh - primary_key = "id" - - # Page size - limit = None - - extra_params = None - - transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) - - def __init__(self, config: Mapping[str, Any]): - super().__init__(authenticator=config["authenticator"]) - self.start_date = config["start_date"] - self.config = config - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params = {"limit": self.limit, "since": self.start_date} - if next_page_token: - params.update(**next_page_token) - if self.extra_params: - params.update(self.extra_params) - return params - - @balancer.balance_rate_limit() - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - json_response = response.json() - for record in json_response: - yield record - - -class ChildStreamMixin: - def stream_slices(self, sync_mode, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - board_ids = set(self.config.get("board_ids", [])) - for board_id in read_all_boards(Boards(self.config), Organizations(self.config)): - if not board_ids or board_id in board_ids: - yield {"id": board_id} - - -class IncrementalTrelloStream(TrelloStream, ABC): - cursor_field = "date" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - response_json = response.json() - if response_json: - return {"before": response_json[-1]["id"]} - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state, stream_slice, next_page_token) - if stream_state: - board_id = stream_slice["id"] - since = stream_state.get(board_id, {}).get(self.cursor_field) - if since: - params["since"] = since - return params - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - board_id = latest_record["data"]["board"]["id"] - updated_state = latest_record[self.cursor_field] - stream_state_value = current_stream_state.get(board_id, {}).get(self.cursor_field) - if stream_state_value: - updated_state = max(updated_state, stream_state_value) - current_stream_state.setdefault(board_id, {})[self.cursor_field] = updated_state - return current_stream_state - - -class Boards(TrelloStream): - """Return list of all boards. - API Docs: https://developer.atlassian.com/cloud/trello/rest/api-group-members/#api-members-id-boards-get - Endpoint: https://api.trello.com/1/members/me/boards - """ - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return "members/me/boards" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - board_ids = self.config.get("board_ids", []) - for record in super().parse_response(response, **kwargs): - if not board_ids or record["id"] in board_ids: - yield record - - -class Cards(ChildStreamMixin, TrelloStream): - """Return list of all cards of a boards. - API Docs: https://developer.atlassian.com/cloud/trello/rest/api-group-boards/#api-boards-id-cards-get - Endpoint: https://api.trello.com/1/boards//cards/all - """ - - limit = 500 - extra_params = { - "customFieldItems": "true", - "pluginData": "true", - "actions_display": "true", - "members": "true", - "list": "true", - "sort": "-id", - } - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - response_json = response.json() - if response_json: - return {"before": response_json[-1]["id"]} - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return f"boards/{stream_slice['id']}/cards/all" - - -class Checklists(ChildStreamMixin, TrelloStream): - """Return list of all checklists of a boards. - API Docs: https://developer.atlassian.com/cloud/trello/rest/api-group-boards/#api-boards-id-checklists-get - Endpoint: https://api.trello.com/1/boards//checklists - """ - - extra_params = {"fields": "all", "checkItem_fields": "all"} - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return f"boards/{stream_slice['id']}/checklists" - - -class Lists(ChildStreamMixin, TrelloStream): - """Return list of all lists of a boards. - API Docs: https://developer.atlassian.com/cloud/trello/rest/api-group-boards/#api-boards-id-lists-get - Endpoint: https://api.trello.com/1/boards//lists - """ - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return f"boards/{stream_slice['id']}/lists" - - -class Organizations(TrelloStream): - """Return list of all member's organizations - API Docs: https://developer.atlassian.com/cloud/trello/rest/api-group-members/#api-members-id-organizations-get - Endpoint: https://api.trello.com/1/members/me/organizations - """ - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return "members/me/organizations" - - -class Users(ChildStreamMixin, TrelloStream): - """Return list of all members of a boards. - API Docs: https://developer.atlassian.com/cloud/trello/rest/api-group-boards/#api-boards-id-members-get - Endpoint: https://api.trello.com/1/boards//members - """ - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return f"boards/{stream_slice['id']}/members" - - -class Actions(ChildStreamMixin, IncrementalTrelloStream): - """Return list of all actions of a boards. - API Docs: https://developer.atlassian.com/cloud/trello/rest/api-group-boards/#api-boards-boardid-actions-get - Endpoint: https://api.trello.com/1/boards//actions - """ - - limit = 1000 - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return f"boards/{stream_slice['id']}/actions" - - -class SourceTrello(AbstractSource): - """ - Source Trello fetch date from web-based, Kanban-style, list-making application. - """ - - def _validate_and_transform(self, config: Mapping[str, Any]): - datetime.datetime.strptime(config["start_date"], "%Y-%m-%dT%H:%M:%SZ") - return config - - @staticmethod - def _get_authenticator(config: dict): - return TrelloAuthenticator(apiKey=config["key"], apiToken=config["token"]) - - def check_connection(self, logger, config) -> Tuple[bool, any]: - """ - Testing connection availability for the connector by granting the credentials. - """ - - config = self._validate_and_transform(config) - config["authenticator"] = self._get_authenticator(config) - available_boards = set(read_all_boards(Boards({**config, "board_ids": []}), Organizations(config))) - unknown_boards = set(config.get("board_ids", [])) - available_boards - if unknown_boards: - unknown_boards = ", ".join(sorted(unknown_boards)) - return False, f"Board ID(s): {unknown_boards} not found" - return True, None - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - config["authenticator"] = self._get_authenticator(config) - return [Actions(config), Boards(config), Cards(config), Checklists(config), Lists(config), Users(config), Organizations(config)] +# Declarative Source +class SourceTrello(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-trello/source_trello/spec.json b/airbyte-integrations/connectors/source-trello/source_trello/spec.json deleted file mode 100644 index 5576ac580c4f..000000000000 --- a/airbyte-integrations/connectors/source-trello/source_trello/spec.json +++ /dev/null @@ -1,76 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/trello", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Trello Spec", - "type": "object", - "required": ["key", "token", "start_date"], - "additionalProperties": true, - "properties": { - "key": { - "type": "string", - "title": "API key", - "description": "Trello API key. See the docs for instructions on how to generate it.", - "airbyte_secret": true, - "order": 0 - }, - "token": { - "type": "string", - "title": "API token", - "description": "Trello API token. See the docs for instructions on how to generate it.", - "airbyte_secret": true, - "order": 1 - }, - "start_date": { - "type": "string", - "title": "Start Date", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", - "examples": ["2021-03-01T00:00:00Z"], - "format": "date-time", - "order": 2 - }, - "board_ids": { - "type": "array", - "items": { - "type": "string", - "pattern": "^[0-9a-fA-F]{24}$" - }, - "title": "Trello Board IDs", - "description": "IDs of the boards to replicate data from. If left empty, data from all boards to which you have access will be replicated.", - "order": 3 - } - } - }, - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "token": { - "type": "string", - "path_in_connector_config": ["token"] - }, - "key": { - "type": "string", - "path_in_connector_config": ["key"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "additionalProperties": false, - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-trello/source_trello/utils.py b/airbyte-integrations/connectors/source-trello/source_trello/utils.py deleted file mode 100644 index 9e3545574baf..000000000000 --- a/airbyte-integrations/connectors/source-trello/source_trello/utils.py +++ /dev/null @@ -1,102 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from functools import wraps -from time import sleep - -import requests -from airbyte_cdk import AirbyteLogger -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams import Stream - - -class TrelloRequestRateLimits: - @staticmethod - def balance_rate_limit(threshold: float = 0.5, rate_limits_headers=None): - """ - To avoid reaching Trello API Rate Limits, use the rate limits header value, - to determine the current rate limits and load and handle sleep time based on load %. - Recommended sleep time between each request is 9 sec. - Header example: - { - x-rate-limit-api-token-interval-ms: 10000 - x-rate-limit-api-token-max: 100 - x-rate-limit-api-token-remaining: 80 - x-rate-limit-api-key-interval-ms: 10000 - x-rate-limit-api-key-max: 300 - x-rate-limit-api-key-remaining: 100 - } - More information: https://developer.atlassian.com/cloud/trello/guides/rest-api/rate-limits/ - """ - - # Define standard timings in seconds - if rate_limits_headers is None: - rate_limits_headers = [ - ("x-rate-limit-api-key-remaining", "x-rate-limit-api-key-max"), - ("x-rate-limit-api-token-remaining", "x-rate-limit-api-token-max"), - ] - - sleep_on_high_load: float = 9.0 - - def decorator(func): - @wraps(func) - def wrapper_balance_rate_limit(*args, **kwargs): - sleep_time = 0 - free_load = float("inf") - # find the Response inside args list - for arg in args: - response = arg if type(arg) is requests.models.Response else None - - # Get the rate_limits from response - rate_limits = ( - [ - (response.headers.get(rate_remaining_limit_header), response.headers.get(rate_max_limit_header)) - for rate_remaining_limit_header, rate_max_limit_header in rate_limits_headers - ] - if response - else None - ) - - # define current load from rate_limits - if rate_limits: - for current_rate, max_rate_limit in rate_limits: - free_load = min(free_load, int(current_rate) / int(max_rate_limit)) - - # define sleep time based on load conditions - if free_load <= threshold: - sleep_time = sleep_on_high_load - - # sleep based on load conditions - sleep(sleep_time) - AirbyteLogger().info(f"Sleep {sleep_time} seconds based on load conditions.") - - return func(*args, **kwargs) - - return wrapper_balance_rate_limit - - return decorator - - -def read_full_refresh(stream_instance: Stream): - slices = stream_instance.stream_slices(sync_mode=SyncMode.full_refresh) - for _slice in slices: - records = stream_instance.read_records(stream_slice=_slice, sync_mode=SyncMode.full_refresh) - for record in records: - yield record - - -def read_all_boards(stream_boards: Stream, stream_organizations: Stream): - board_ids = set() - - for record in read_full_refresh(stream_boards): - if record["id"] not in board_ids: - board_ids.add(record["id"]) - yield record["id"] - - for record in read_full_refresh(stream_organizations): - for board_id in record["idBoards"]: - if board_id not in board_ids: - board_ids.add(board_id) - yield board_id diff --git a/airbyte-integrations/connectors/source-trello/unit_tests/__init__.py b/airbyte-integrations/connectors/source-trello/unit_tests/__init__.py index 9db886e0930f..3f8e5f92f4c7 100644 --- a/airbyte-integrations/connectors/source-trello/unit_tests/__init__.py +++ b/airbyte-integrations/connectors/source-trello/unit_tests/__init__.py @@ -1,7 +1,7 @@ # # MIT License # -# Copyright (c) 2020 Airbyte +# Copyright (c) 2023 Airbyte # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/airbyte-integrations/connectors/source-trello/unit_tests/conftest.py b/airbyte-integrations/connectors/source-trello/unit_tests/conftest.py deleted file mode 100644 index 92c382d52cf8..000000000000 --- a/airbyte-integrations/connectors/source-trello/unit_tests/conftest.py +++ /dev/null @@ -1,11 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from airbyte_cdk.sources.streams.http.auth import NoAuth -from pytest import fixture - - -@fixture -def config(): - return {"start_date": "start_date", "authenticator": NoAuth} diff --git a/airbyte-integrations/connectors/source-trello/unit_tests/helpers.py b/airbyte-integrations/connectors/source-trello/unit_tests/helpers.py deleted file mode 100644 index e1989be16e65..000000000000 --- a/airbyte-integrations/connectors/source-trello/unit_tests/helpers.py +++ /dev/null @@ -1,11 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -NO_SLEEP_HEADERS = { - "x-rate-limit-api-token-max": "1", - "x-rate-limit-api-token-remaining": "1", - "x-rate-limit-api-key-max": "1", - "x-rate-limit-api-key-remaining": "1", -} diff --git a/airbyte-integrations/connectors/source-trello/unit_tests/test_control_rate_limit.py b/airbyte-integrations/connectors/source-trello/unit_tests/test_control_rate_limit.py deleted file mode 100644 index 0bf094625d0c..000000000000 --- a/airbyte-integrations/connectors/source-trello/unit_tests/test_control_rate_limit.py +++ /dev/null @@ -1,89 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from functools import wraps - -import requests - -# Define standard timings in seconds -SLEEP_ON_HIGH_LOAD: float = 9.0 - -TEST_DATA_FIELD = "some_data_field" -TEST_RATE_LIMIT_THRESHOLD = 0.1 -TEST_HEADERS_NAME = [ - ("x-rate-limit-api-key-remaining", "x-rate-limit-api-key-max"), - ("x-rate-limit-api-token-remaining", "x-rate-limit-api-token-max"), -] - - -def control_request_rate_limit_decorator(threshold: float = 0.05, limit_headers=None): - """ - This decorator was replicated completely, as separeted function in order to be tested. - The only difference is: - :: the real one inside utils.py sleeps the actual defined time and returns the function back, - :: and this fake one simply sleeps and returns the wait_time as actual sleep time in order to be tested. - """ - - def decorator(func): - @wraps(func) - def wrapper_control_request_rate_limit(*args, **kwargs): - sleep_time = 0 - free_load = float("inf") - # find the Response inside args list - for arg in args: - response = arg if type(arg) is requests.models.Response else None - - # Get the rate_limits from response - rate_limits = ( - [ - (response.headers.get(rate_remaining_limit_header), response.headers.get(rate_max_limit_header)) - for rate_remaining_limit_header, rate_max_limit_header in limit_headers - ] - if response - else None - ) - - # define current load from rate_limits - if rate_limits: - for current_rate, max_rate_limit in rate_limits: - free_load = min(free_load, int(current_rate) / int(max_rate_limit)) - - # define sleep time based on load conditions - if free_load <= threshold: - sleep_time = SLEEP_ON_HIGH_LOAD - - # for this test RETURN sleep_time based on load conditions - return sleep_time - - return wrapper_control_request_rate_limit - - return decorator - - -# Simulating real function call based CDK's parse_response() method -@control_request_rate_limit_decorator(TEST_RATE_LIMIT_THRESHOLD, TEST_HEADERS_NAME) -def fake_parse_response(response: requests.Response, **kwargs): - json_response = response.json() - records = json_response.get(TEST_DATA_FIELD, []) if TEST_DATA_FIELD is not None else json_response - yield from records - - -def test_with_load(requests_mock): - """ - Test simulates high load of rate limit. - In this case we should wait at least 9 sec before next API call. - """ - test_response_header = { - "x-rate-limit-api-token-max": "300", - "x-rate-limit-api-token-remaining": "10", - "x-rate-limit-api-key-max": "300", - "x-rate-limit-api-key-remaining": "100", - } - - requests_mock.get("https://test.trello.com/", headers=test_response_header) - test_response = requests.get("https://test.trello.com/") - - actual_sleep_time = fake_parse_response(test_response) - - assert SLEEP_ON_HIGH_LOAD == actual_sleep_time diff --git a/airbyte-integrations/connectors/source-trello/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-trello/unit_tests/test_incremental_streams.py deleted file mode 100644 index b598330cf496..000000000000 --- a/airbyte-integrations/connectors/source-trello/unit_tests/test_incremental_streams.py +++ /dev/null @@ -1,62 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from airbyte_cdk.models import SyncMode -from pytest import fixture -from source_trello.source import IncrementalTrelloStream - - -@fixture -def patch_incremental_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(IncrementalTrelloStream, "path", "v0/example_endpoint") - mocker.patch.object(IncrementalTrelloStream, "primary_key", "test_primary_key") - mocker.patch.object(IncrementalTrelloStream, "__abstractmethods__", set()) - - -def test_cursor_field(patch_incremental_base_class, config): - stream = IncrementalTrelloStream(config) - expected_cursor_field = "date" - assert stream.cursor_field == expected_cursor_field - - -def test_get_updated_state(patch_incremental_base_class, config): - stream = IncrementalTrelloStream(config) - expected_cursor_field = "date" - inputs = { - "current_stream_state": {"611aa0ef37acd675af67dc9b": {expected_cursor_field: "2021-07-12T10:44:09+00:00"}}, - "latest_record": {"data": {"board": {"id": "611aa0ef37acd675af67dc9b"}}, expected_cursor_field: "2021-07-15T10:44:09+00:00"}, - } - expected_state = {"611aa0ef37acd675af67dc9b": {expected_cursor_field: "2021-07-15T10:44:09+00:00"}} - assert stream.get_updated_state(**inputs) == expected_state - - -def test_stream_slices(patch_incremental_base_class, config): - stream = IncrementalTrelloStream(config) - expected_cursor_field = "date" - inputs = { - "sync_mode": SyncMode.incremental, - "cursor_field": expected_cursor_field, - "stream_state": {expected_cursor_field: "2021-07-15T10:44:09+00:00"}, - } - expected_stream_slice = [None] - assert stream.stream_slices(**inputs) == expected_stream_slice - - -def test_supports_incremental(patch_incremental_base_class, mocker, config): - mocker.patch.object(IncrementalTrelloStream, "cursor_field", "dummy_field") - stream = IncrementalTrelloStream(config) - assert stream.supports_incremental - - -def test_source_defined_cursor(patch_incremental_base_class, config): - stream = IncrementalTrelloStream(config) - assert stream.source_defined_cursor - - -def test_stream_checkpoint_interval(patch_incremental_base_class, config): - stream = IncrementalTrelloStream(config) - expected_checkpoint_interval = None - assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-trello/unit_tests/test_order_ids_partition_router.py b/airbyte-integrations/connectors/source-trello/unit_tests/test_order_ids_partition_router.py new file mode 100644 index 000000000000..439e5148ef2b --- /dev/null +++ b/airbyte-integrations/connectors/source-trello/unit_tests/test_order_ids_partition_router.py @@ -0,0 +1,69 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest +from airbyte_cdk.sources.streams.core import Stream +from source_trello.components import OrderIdsPartitionRouter + + +class MockStream(Stream): + def __init__(self, records): + self.records = records + + def primary_key(self): + return + + def read_records(self, sync_mode): + return self.records + + +# test cases as a list of tuples (boards_records, organizations_records, expected_board_ids) +test_cases = [ + ( + # test same ids in both boards and organizations + [{"id": "b11111111111111111111111", "name": "board_1"}, {"id": "b22222222222222222222222", "name": "board_2"}], + [{"id": "org111111111111111111111", "idBoards": ["b11111111111111111111111", "b22222222222222222222222"]}], + ["b11111111111111111111111", "b22222222222222222222222"], + ), + ( + # test one different id in organizations + [{"id": "b11111111111111111111111", "name": "board_1"}, {"id": "b22222222222222222222222", "name": "board_2"}], + [{"id": "org111111111111111111111", "idBoards": ["b11111111111111111111111", "b33333333333333333333333"]}], + ["b11111111111111111111111", "b22222222222222222222222", "b33333333333333333333333"], + ), + ( + # test different ids in multiple boards and organizations + [{"id": "b11111111111111111111111", "name": "board_1"}, {"id": "b22222222222222222222222", "name": "board_2"}], + [ + {"id": "org111111111111111111111", "idBoards": ["b11111111111111111111111", "b33333333333333333333333"]}, + {"id": "org222222222222222222222", "idBoards": ["b00000000000000000000000", "b44444444444444444444444"]}, + ], + [ + "b11111111111111111111111", + "b22222222222222222222222", + "b33333333333333333333333", + "b00000000000000000000000", + "b44444444444444444444444", + ], + ), + ( + # test empty boards and organizations + [], + [], + [], + ), +] + + +@pytest.mark.parametrize("boards_records, organizations_records, expected_board_ids", test_cases) +def test_read_all_boards(boards_records, organizations_records, expected_board_ids): + # Set up mock streams with provided records + partition_router = OrderIdsPartitionRouter(parent_stream_configs=[None], config=None, parameters=None) + boards_stream = MockStream(records=boards_records) + organizations_stream = MockStream(records=organizations_records) + + # Call the function and check the result + board_ids = list(partition_router.read_all_boards(boards_stream, organizations_stream)) + assert board_ids == expected_board_ids diff --git a/airbyte-integrations/connectors/source-trello/unit_tests/test_source.py b/airbyte-integrations/connectors/source-trello/unit_tests/test_source.py deleted file mode 100644 index c86f790ba865..000000000000 --- a/airbyte-integrations/connectors/source-trello/unit_tests/test_source.py +++ /dev/null @@ -1,51 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -from source_trello.source import SourceTrello - -from .helpers import NO_SLEEP_HEADERS - - -def test_streams(mocker): - source = SourceTrello() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 7 - assert len(streams) == expected_streams_number - - -def test_check_connection(requests_mock): - config = { - "start_date": "2020-01-01T00:00:00Z", - "key": "key", - "token": "token", - } - - logger = MagicMock() - - requests_mock.get( - "https://api.trello.com/1/members/me/boards", - headers=NO_SLEEP_HEADERS, - json=[ - {"id": "b11111111111111111111111", "name": "board_1"}, - {"id": "b22222222222222222222222", "name": "board_2"} - ], - ) - - requests_mock.get( - "https://api.trello.com/1/members/me/organizations", - headers=NO_SLEEP_HEADERS, - json=[{"id": "org111111111111111111111", "idBoards": ["b11111111111111111111111", "b22222222222222222222222"]}], - ) - - source = SourceTrello() - status, error = source.check_connection(logger, config) - assert status is True - assert error is None - config["board_ids"] = ["b11111111111111111111111", "b33333333333333333333333", "b44444444444444444444444"] - status, error = source.check_connection(logger, config) - assert status is False - assert error == 'Board ID(s): b33333333333333333333333, b44444444444444444444444 not found' diff --git a/airbyte-integrations/connectors/source-trello/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-trello/unit_tests/test_streams.py deleted file mode 100644 index 9254e616ec06..000000000000 --- a/airbyte-integrations/connectors/source-trello/unit_tests/test_streams.py +++ /dev/null @@ -1,118 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from itertools import cycle -from unittest.mock import MagicMock - -from airbyte_cdk.sources.streams.http.auth.core import NoAuth -from pytest import fixture -from source_trello.source import Boards, Cards, TrelloStream -from source_trello.utils import read_full_refresh - -from .helpers import NO_SLEEP_HEADERS - - -@fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(TrelloStream, "path", "v0/example_endpoint") - mocker.patch.object(TrelloStream, "primary_key", "test_primary_key") - mocker.patch.object(TrelloStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class, config): - stream = TrelloStream(config) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": {"before": "id"}} - expected_params = {"limit": None, "since": "start_date", "before": "id"} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class, config): - stream = TrelloStream(config) - inputs = {"response": MagicMock()} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_boards_stream(requests_mock): - mock_boards_request = requests_mock.get( - "https://api.trello.com/1/members/me/boards", - headers=NO_SLEEP_HEADERS, - json=[{"id": "b11111111111111111111111", "name": "board_1"}, {"id": "b22222222222222222222222", "name": "board_2"}], - ) - - config = {"authenticator": NoAuth(), "start_date": "2021-02-11T08:35:49.540Z"} - stream1 = Boards(config=config) - records = list(read_full_refresh(stream1)) - assert records == [{"id": "b11111111111111111111111", "name": "board_1"}, {"id": "b22222222222222222222222", "name": "board_2"}] - - stream2 = Boards(config={**config, "board_ids": ["b22222222222222222222222"]}) - records = list(read_full_refresh(stream2)) - assert records == [{"id": "b22222222222222222222222", "name": "board_2"}] - - stream3 = Boards(config={**config, "board_ids": ["not-found"]}) - records = list(read_full_refresh(stream3)) - assert records == [] - - assert mock_boards_request.call_count == 3 - - -def test_cards_stream(requests_mock): - mock_boards_request = requests_mock.get( - "https://api.trello.com/1/members/me/boards", - headers=NO_SLEEP_HEADERS, - json=[{"id": "b11111111111111111111111", "name": "board_1"}, {"id": "b22222222222222222222222", "name": "board_2"}], - ) - - mock_organizations_request = requests_mock.get( - "https://api.trello.com/1/members/me/organizations", - headers=NO_SLEEP_HEADERS, - json=[{"id": "org111111111111111111111", "idBoards": ["b11111111111111111111111", "b22222222222222222222222"]}], - ) - - json_responses1 = cycle([ - [{"id": "c11111111111111111111111", "name": "card_1"}, {"id": "c22222222222222222222222", "name": "card_2"}], - [], - ]) - - mock_cards_request_1 = requests_mock.get( - "https://api.trello.com/1/boards/b11111111111111111111111/cards/all", - headers=NO_SLEEP_HEADERS, - json=lambda request, context: next(json_responses1), - ) - - json_responses2 = cycle([ - [{"id": "c33333333333333333333333", "name": "card_3"}, {"id": "c44444444444444444444444", "name": "card_4"}], - [], - ]) - - mock_cards_request_2 = requests_mock.get( - "https://api.trello.com/1/boards/b22222222222222222222222/cards/all", - headers=NO_SLEEP_HEADERS, - json=lambda request, context: next(json_responses2), - ) - - config = {"authenticator": NoAuth(), "start_date": "2021-02-11T08:35:49.540Z"} - stream1 = Cards(config=config) - records = list(read_full_refresh(stream1)) - assert records == [ - {"id": "c11111111111111111111111", "name": "card_1"}, - {"id": "c22222222222222222222222", "name": "card_2"}, - {"id": "c33333333333333333333333", "name": "card_3"}, - {"id": "c44444444444444444444444", "name": "card_4"}, - ] - - stream2 = Cards(config={**config, "board_ids": ["b22222222222222222222222"]}) - records = list(read_full_refresh(stream2)) - assert records == [{"id": "c33333333333333333333333", "name": "card_3"}, {"id": "c44444444444444444444444", "name": "card_4"}] - - stream3 = Cards(config={**config, "board_ids": ["not-found"]}) - records = list(read_full_refresh(stream3)) - assert records == [] - - assert mock_boards_request.call_count == 3 - assert mock_organizations_request.call_count == 3 - assert mock_cards_request_1.call_count == 2 - assert mock_cards_request_2.call_count == 4 diff --git a/airbyte-integrations/connectors/source-trustpilot/README.md b/airbyte-integrations/connectors/source-trustpilot/README.md index b86f013b6e00..7745d8a7f18e 100644 --- a/airbyte-integrations/connectors/source-trustpilot/README.md +++ b/airbyte-integrations/connectors/source-trustpilot/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-trustpilot:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/trustpilot) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_trustpilot/spec.yaml` file. @@ -57,24 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-trustpilot:dev -``` -If you want to build the Docker image with the CDK on your local machine (rather than the most recent package published to pypi), from the airbyte base directory run: +#### Build +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** ```bash -CONNECTOR_TAG= CONNECTOR_NAME= sh airbyte-integrations/scripts/build-connector-image-with-local-cdk.sh +airbyte-ci connectors --name=source-trustpilot build ``` +An image will be built with the tag `airbyte/source-trustpilot:dev`. -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-trustpilot:airbyteDocker +**Via `docker build`:** +```bash +docker build -t airbyte/source-trustpilot:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -84,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-trustpilot:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-trustpilot:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-trustpilot:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-trustpilot test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-trustpilot:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-trustpilot:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -131,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-trustpilot test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/trustpilot.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-trustpilot/acceptance-test-config.yml b/airbyte-integrations/connectors/source-trustpilot/acceptance-test-config.yml index 9041b42c7b1b..6ee6dc17190a 100644 --- a/airbyte-integrations/connectors/source-trustpilot/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-trustpilot/acceptance-test-config.yml @@ -19,12 +19,10 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - incremental: + incremental: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - cursor_paths: - private_reviews: ["5f5e954ec15b2700017c834f_createdAt"] future_state: future_state_path: "integration_tests/abnormal_state.json" full_refresh: diff --git a/airbyte-integrations/connectors/source-trustpilot/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-trustpilot/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-trustpilot/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-trustpilot/build.gradle b/airbyte-integrations/connectors/source-trustpilot/build.gradle deleted file mode 100644 index 3a8453b104f4..000000000000 --- a/airbyte-integrations/connectors/source-trustpilot/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_trustpilot' -} diff --git a/airbyte-integrations/connectors/source-trustpilot/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-trustpilot/unit_tests/test_incremental_streams.py index 7c2a183601a9..c8723363e522 100644 --- a/airbyte-integrations/connectors/source-trustpilot/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-trustpilot/unit_tests/test_incremental_streams.py @@ -14,33 +14,25 @@ def patch_incremental_base_class(mocker): # Mock abstract methods to enable instantiating abstract class mocker.patch.object(TrustpilotIncrementalStream, "path", "v0/example_endpoint") mocker.patch.object(TrustpilotIncrementalStream, "primary_key", "test_primary_key") - mocker.patch.object(TrustpilotIncrementalStream, "_start_date", - pendulum.now("UTC").add(years=-1)) - mocker.patch.object(TrustpilotIncrementalStream, "_current_stream_slice", - {'business_unit_id': '5f5e954ec15b2700017c834f'}) + mocker.patch.object(TrustpilotIncrementalStream, "_start_date", pendulum.now("UTC").add(years=-1)) + mocker.patch.object(TrustpilotIncrementalStream, "_current_stream_slice", {"business_unit_id": "5f5e954ec15b2700017c834f"}) mocker.patch.object(TrustpilotIncrementalStream, "__abstractmethods__", set()) def test_cursor_field(patch_incremental_base_class): stream = TrustpilotIncrementalStream() - expected_cursor_field = 'createdAt' + expected_cursor_field = "createdAt" assert stream.cursor_field == expected_cursor_field def test_get_updated_state(patch_incremental_base_class): stream = TrustpilotIncrementalStream() inputs = { - "current_stream_state": { - '5f5e954ec15b2700017c834f_createdAt': '2023-03-01T00:00:00+00:00' - }, - "latest_record": { - 'createdAt': '2023-03-23T15:12:17Z' - } - } - expected_state = { - '5f5e954ec15b2700017c834f_createdAt': '2023-03-23T15:12:17+00:00' + "current_stream_state": {"5f5e954ec15b2700017c834f_createdAt": "2023-03-01T00:00:00+00:00"}, + "latest_record": {"createdAt": "2023-03-23T15:12:17Z"}, } + expected_state = {"5f5e954ec15b2700017c834f_createdAt": "2023-03-23T15:12:17+00:00"} assert stream.get_updated_state(**inputs) == expected_state @@ -48,12 +40,8 @@ def test_stream_slices(patch_incremental_base_class): stream = TrustpilotIncrementalStream() inputs = { "sync_mode": SyncMode.incremental, - "cursor_field": [ - 'createdAt' - ], - "stream_state": { - '5f5e954ec15b2700017c834f_createdAt': '2023-03-01T00:00:00+00:00' - } + "cursor_field": ["createdAt"], + "stream_state": {"5f5e954ec15b2700017c834f_createdAt": "2023-03-01T00:00:00+00:00"}, } # TODO: replace this with your expected stream slices list expected_stream_slice = [None] diff --git a/airbyte-integrations/connectors/source-trustpilot/unit_tests/test_source.py b/airbyte-integrations/connectors/source-trustpilot/unit_tests/test_source.py index 28ef9319e39e..1b64655ca03a 100644 --- a/airbyte-integrations/connectors/source-trustpilot/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-trustpilot/unit_tests/test_source.py @@ -10,7 +10,7 @@ def test_check_connection(mocker): source = SourceTrustpilot() - with open('secrets/config.json') as f: + with open("secrets/config.json") as f: logger_mock, config_mock = MagicMock(), json.load(f) assert source.check_connection(logger_mock, config_mock) == (True, None) @@ -18,14 +18,9 @@ def test_check_connection(mocker): def test_streams(mocker): source = SourceTrustpilot() config_mock = { - 'credentials': { - 'auth_type': '__api_key__', - 'client_id': '__client_id__' - }, - 'business_units': [ - 'my_domain.com' - ], - 'start_date': '2023-01-01T00:00:00Z' + "credentials": {"auth_type": "__api_key__", "client_id": "__client_id__"}, + "business_units": ["my_domain.com"], + "start_date": "2023-01-01T00:00:00Z", } streams = source.streams(config_mock) expected_streams_number = 2 diff --git a/airbyte-integrations/connectors/source-trustpilot/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-trustpilot/unit_tests/test_streams.py index 7df1d0c75022..abb559d7276d 100644 --- a/airbyte-integrations/connectors/source-trustpilot/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-trustpilot/unit_tests/test_streams.py @@ -37,18 +37,18 @@ def test_next_page_token(patch_base_class): def test_parse_response(patch_base_class): stream = TrustpilotStream() response = MagicMock() - response.json = lambda: json.loads(b"""{ + response.json = lambda: json.loads( + b"""{ "links": { "rel": "next-page", "href": "http://..." }, "id": "12351241" -}""") +}""" + ) inputs = {"response": response} - expected_parsed_object = { - "id": "12351241" - } + expected_parsed_object = {"id": "12351241"} assert next(stream.parse_response(**inputs)) == expected_parsed_object diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/README.md b/airbyte-integrations/connectors/source-tvmaze-schedule/README.md index 9feb6bc29b36..fdf98ecdf7d0 100644 --- a/airbyte-integrations/connectors/source-tvmaze-schedule/README.md +++ b/airbyte-integrations/connectors/source-tvmaze-schedule/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-tvmaze-schedule:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/tvmaze-schedule) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_tvmaze_schedule/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-tvmaze-schedule:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-tvmaze-schedule build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-tvmaze-schedule:airbyteDocker +An image will be built with the tag `airbyte/source-tvmaze-schedule:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-tvmaze-schedule:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-tvmaze-schedule:dev ch docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-tvmaze-schedule:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-tvmaze-schedule:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-tvmaze-schedule test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-tvmaze-schedule:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-tvmaze-schedule:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-tvmaze-schedule test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/tvmaze-schedule.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-tvmaze-schedule/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-tvmaze-schedule/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-tvmaze-schedule/build.gradle b/airbyte-integrations/connectors/source-tvmaze-schedule/build.gradle deleted file mode 100644 index df1d329b1e1a..000000000000 --- a/airbyte-integrations/connectors/source-tvmaze-schedule/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_tvmaze_schedule' -} diff --git a/airbyte-integrations/connectors/source-twilio-taskrouter/README.md b/airbyte-integrations/connectors/source-twilio-taskrouter/README.md index 2011185ec41d..8269ff16c6ec 100644 --- a/airbyte-integrations/connectors/source-twilio-taskrouter/README.md +++ b/airbyte-integrations/connectors/source-twilio-taskrouter/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-twilio-taskrouter:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/twilio-taskrouter) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_twilio_taskrouter/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-twilio-taskrouter:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-twilio-taskrouter build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-twilio-taskrouter:airbyteDocker +An image will be built with the tag `airbyte/source-twilio-taskrouter:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-twilio-taskrouter:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-twilio-taskrouter:dev docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-twilio-taskrouter:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-twilio-taskrouter:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-twilio-taskrouter test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-twilio-taskrouter:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-twilio-taskrouter:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-twilio-taskrouter test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/twilio-taskrouter.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-twilio-taskrouter/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-twilio-taskrouter/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-twilio-taskrouter/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-twilio-taskrouter/build.gradle b/airbyte-integrations/connectors/source-twilio-taskrouter/build.gradle deleted file mode 100644 index 618253f6701e..000000000000 --- a/airbyte-integrations/connectors/source-twilio-taskrouter/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_twilio_taskrouter' -} diff --git a/airbyte-integrations/connectors/source-twilio/Dockerfile b/airbyte-integrations/connectors/source-twilio/Dockerfile deleted file mode 100644 index b8e2f02135cb..000000000000 --- a/airbyte-integrations/connectors/source-twilio/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY setup.py ./ -RUN pip install . -COPY source_twilio ./source_twilio -COPY main.py ./ - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.10.0 -LABEL io.airbyte.name=airbyte/source-twilio diff --git a/airbyte-integrations/connectors/source-twilio/README.md b/airbyte-integrations/connectors/source-twilio/README.md index c25f4c2f2db0..95b155fb069d 100644 --- a/airbyte-integrations/connectors/source-twilio/README.md +++ b/airbyte-integrations/connectors/source-twilio/README.md @@ -27,14 +27,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-twilio:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/twilio) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_twilio/spec.json` file. @@ -54,19 +46,71 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-twilio:dev + + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name source-twilio build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-twilio:dev`. -You can also build the connector image via Gradle: +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-twilio:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-twilio:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-twilio:dev . +# Running the spec command against your patched connector +docker run airbyte/source-twilio:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -75,45 +119,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-twilio:dev check --con docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-twilio:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-twilio:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .'[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-twilio test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-twilio:dev \ -&& python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-twilio:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-twilio:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -123,8 +138,10 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-twilio test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/twilio.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-twilio/acceptance-test-config.yml b/airbyte-integrations/connectors/source-twilio/acceptance-test-config.yml index 9c1849dc0b9a..adbf72dc66c2 100644 --- a/airbyte-integrations/connectors/source-twilio/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-twilio/acceptance-test-config.yml @@ -63,7 +63,6 @@ acceptance_tests: configured_catalog_path: "integration_tests/incremental_catalog.json" future_state: future_state_path: "integration_tests/abnormal_state.json" - threshold_days: 30 timeout_seconds: 3600 full_refresh: tests: diff --git a/airbyte-integrations/connectors/source-twilio/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-twilio/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-twilio/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-twilio/build.gradle b/airbyte-integrations/connectors/source-twilio/build.gradle deleted file mode 100644 index 0a2470c60adc..000000000000 --- a/airbyte-integrations/connectors/source-twilio/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_twilio' -} diff --git a/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.jsonl index 330566a9ea1a..ba744d425a95 100644 --- a/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-twilio/integration_tests/expected_records.jsonl @@ -7,9 +7,9 @@ {"stream": "available_phone_number_countries", "data": {"country_code": "AU", "country": "Australia", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AU.json", "beta": false, "subresource_uris": {"local": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AU/Local.json", "toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AU/TollFree.json", "mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/AU/Mobile.json"}}, "emitted_at": 1691419684730} {"stream": "available_phone_number_countries", "data": {"country_code": "BE", "country": "Belgium", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BE.json", "beta": false, "subresource_uris": {"toll_free": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/BE/TollFree.json"}}, "emitted_at": 1691419684732} {"stream": "available_phone_number_countries", "data": {"country_code": "SE", "country": "Sweden", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SE.json", "beta": false, "subresource_uris": {"mobile": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/AvailablePhoneNumbers/SE/Mobile.json"}}, "emitted_at": 1691419684732} -{"stream": "calls", "data": {"date_updated": "2022-07-13T19:44:06Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 62, "from": "+13392299964", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CA0bc0fdb5783917659002fe37ba581e11", "queue_time": 0, "price": -0.017, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-07-13T19:43:04Z", "date_created": "2022-07-13T19:43:04Z", "from_formatted": "(339) 229-9964", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0bc0fdb5783917659002fe37ba581e11.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-07-13T19:44:06Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0bc0fdb5783917659002fe37ba581e11/Feedback.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0bc0fdb5783917659002fe37ba581e11/UserDefinedMessages.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0bc0fdb5783917659002fe37ba581e11/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0bc0fdb5783917659002fe37ba581e11/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0bc0fdb5783917659002fe37ba581e11/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0bc0fdb5783917659002fe37ba581e11/Payments.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0bc0fdb5783917659002fe37ba581e11/UserDefinedMessageSubscriptions.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0bc0fdb5783917659002fe37ba581e11/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA0bc0fdb5783917659002fe37ba581e11/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json"}}, "emitted_at": 1691419833550} -{"stream": "calls", "data": {"date_updated": "2022-07-19T20:11:13Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 12, "from": "+16613444179", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CA22893fba469912ccdf127472f69b301d", "queue_time": 0, "price": -0.0085, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-07-19T20:11:01Z", "date_created": "2022-07-19T20:11:01Z", "from_formatted": "(661) 344-4179", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA22893fba469912ccdf127472f69b301d.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-07-19T20:11:13Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA22893fba469912ccdf127472f69b301d/Feedback.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA22893fba469912ccdf127472f69b301d/UserDefinedMessages.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA22893fba469912ccdf127472f69b301d/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA22893fba469912ccdf127472f69b301d/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA22893fba469912ccdf127472f69b301d/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA22893fba469912ccdf127472f69b301d/Payments.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA22893fba469912ccdf127472f69b301d/UserDefinedMessageSubscriptions.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA22893fba469912ccdf127472f69b301d/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA22893fba469912ccdf127472f69b301d/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json"}}, "emitted_at": 1691419833553} -{"stream": "calls", "data": {"date_updated": "2022-07-20T18:24:12Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 11, "from": "+16613444179", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CAcf07b73a4f6c8e50942ca59b7d0468e9", "queue_time": 0, "price": -0.0085, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2022-07-20T18:24:01Z", "date_created": "2022-07-20T18:24:01Z", "from_formatted": "(661) 344-4179", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAcf07b73a4f6c8e50942ca59b7d0468e9.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2022-07-20T18:24:12Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": {"feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAcf07b73a4f6c8e50942ca59b7d0468e9/Feedback.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAcf07b73a4f6c8e50942ca59b7d0468e9/UserDefinedMessages.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAcf07b73a4f6c8e50942ca59b7d0468e9/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAcf07b73a4f6c8e50942ca59b7d0468e9/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAcf07b73a4f6c8e50942ca59b7d0468e9/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAcf07b73a4f6c8e50942ca59b7d0468e9/Payments.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAcf07b73a4f6c8e50942ca59b7d0468e9/UserDefinedMessageSubscriptions.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAcf07b73a4f6c8e50942ca59b7d0468e9/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAcf07b73a4f6c8e50942ca59b7d0468e9/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json"}}, "emitted_at": 1691419833554} +{"stream": "calls", "data": { "date_updated": "2023-06-15T19:57:59Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 13, "from": "+12056890337", "to": "+12056561170", "annotation": null, "answered_by": null, "sid": "CAb70f3b70167dd8d4ee2e1dc15db64e02", "queue_time": 0, "price": -0.0085, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2023-06-15T19:57:46Z", "date_created": "2023-06-15T19:57:46Z", "from_formatted": "(205) 689-0337", "group_sid": null, "trunk_sid": "", "forwarded_from": "+12056561170", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAb70f3b70167dd8d4ee2e1dc15db64e02.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2023-06-15T19:57:59Z", "to_formatted": "(205) 656-1170", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": { "feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAb70f3b70167dd8d4ee2e1dc15db64e02/Feedback.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAb70f3b70167dd8d4ee2e1dc15db64e02/UserDefinedMessages.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAb70f3b70167dd8d4ee2e1dc15db64e02/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAb70f3b70167dd8d4ee2e1dc15db64e02/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAb70f3b70167dd8d4ee2e1dc15db64e02/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAb70f3b70167dd8d4ee2e1dc15db64e02/Payments.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAb70f3b70167dd8d4ee2e1dc15db64e02/UserDefinedMessageSubscriptions.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAb70f3b70167dd8d4ee2e1dc15db64e02/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CAb70f3b70167dd8d4ee2e1dc15db64e02/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json" } }, "emitted_at": 1694639568884} +{"stream": "calls", "data": { "date_updated": "2023-03-15T11:35:20Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": "", "duration": 0, "from": "+12056561170", "to": "+14156236785", "annotation": null, "answered_by": null, "sid": "CA651f21262d4308879ea685e704dd0384", "queue_time": 0, "price": null, "api_version": "2010-04-01", "status": "busy", "direction": "outbound-api", "start_time": "2023-03-15T11:35:03Z", "date_created": "2023-03-15T11:35:03Z", "from_formatted": "(205) 656-1170", "group_sid": null, "trunk_sid": "", "forwarded_from": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA651f21262d4308879ea685e704dd0384.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2023-03-15T11:35:20Z", "to_formatted": "(415) 623-6785", "phone_number_sid": "PNe40bd7f3ac343b32fd51275d2d5b3dcc", "subresource_uris": { "feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA651f21262d4308879ea685e704dd0384/Feedback.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA651f21262d4308879ea685e704dd0384/UserDefinedMessages.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA651f21262d4308879ea685e704dd0384/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA651f21262d4308879ea685e704dd0384/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA651f21262d4308879ea685e704dd0384/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA651f21262d4308879ea685e704dd0384/Payments.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA651f21262d4308879ea685e704dd0384/UserDefinedMessageSubscriptions.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA651f21262d4308879ea685e704dd0384/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA651f21262d4308879ea685e704dd0384/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json" } }, "emitted_at": 1694639568886} +{"stream": "calls", "data": { "date_updated": "2023-02-16T14:37:32Z", "price_unit": "USD", "parent_call_sid": null, "caller_name": null, "duration": 14, "from": "+380636306253", "to": "+13603004201", "annotation": null, "answered_by": null, "sid": "CA9121cd06fb7a1c0c96664c089621c979", "queue_time": 0, "price": -0.0085, "api_version": "2010-04-01", "status": "completed", "direction": "inbound", "start_time": "2023-02-16T14:37:18Z", "date_created": "2023-02-16T14:37:18Z", "from_formatted": "+380636306253", "group_sid": null, "trunk_sid": "", "forwarded_from": "+13603004201", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA9121cd06fb7a1c0c96664c089621c979.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "end_time": "2023-02-16T14:37:32Z", "to_formatted": "(360) 300-4201", "phone_number_sid": "PN1fe31291fa81c17bf71cd128bc649e68", "subresource_uris": { "feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA9121cd06fb7a1c0c96664c089621c979/Feedback.json", "user_defined_messages": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA9121cd06fb7a1c0c96664c089621c979/UserDefinedMessages.json", "notifications": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA9121cd06fb7a1c0c96664c089621c979/Notifications.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA9121cd06fb7a1c0c96664c089621c979/Recordings.json", "streams": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA9121cd06fb7a1c0c96664c089621c979/Streams.json", "payments": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA9121cd06fb7a1c0c96664c089621c979/Payments.json", "user_defined_message_subscriptions": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA9121cd06fb7a1c0c96664c089621c979/UserDefinedMessageSubscriptions.json", "siprec": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA9121cd06fb7a1c0c96664c089621c979/Siprec.json", "events": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/CA9121cd06fb7a1c0c96664c089621c979/Events.json", "feedback_summaries": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Calls/FeedbackSummary.json" } }, "emitted_at": 1694639568887} {"stream": "conferences", "data": {"status": "completed", "reason_conference_ended": "last-participant-left", "date_updated": "2022-09-23T14:44:41Z", "region": "us1", "friendly_name": "test_conference", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CFca0fa08200f55a6d60779d18b644a675.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "call_sid_ending_conference": "CA8858f240bdccfb3393def1682c2dbdf0", "sid": "CFca0fa08200f55a6d60779d18b644a675", "date_created": "2022-09-23T14:44:11Z", "api_version": "2010-04-01", "subresource_uris": {"participants": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CFca0fa08200f55a6d60779d18b644a675/Participants.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CFca0fa08200f55a6d60779d18b644a675/Recordings.json"}}, "emitted_at": 1691419855153} {"stream": "conferences", "data": {"status": "completed", "reason_conference_ended": "last-participant-left", "date_updated": "2023-02-15T14:49:37Z", "region": "us1", "friendly_name": "Conference2", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CF15e8707d15e02c1af88809b159ff8b42.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "call_sid_ending_conference": "CA04ae9210566d36c425bae2087736f6ac", "sid": "CF15e8707d15e02c1af88809b159ff8b42", "date_created": "2023-02-15T14:49:21Z", "api_version": "2010-04-01", "subresource_uris": {"participants": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CF15e8707d15e02c1af88809b159ff8b42/Participants.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CF15e8707d15e02c1af88809b159ff8b42/Recordings.json"}}, "emitted_at": 1691419855509} {"stream": "conferences", "data": {"status": "completed", "reason_conference_ended": "last-participant-left", "date_updated": "2023-02-16T09:57:39Z", "region": "us1", "friendly_name": "Conference2", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CF33199d5a9a0b202b3bd9558438a052d8.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "call_sid_ending_conference": "CAf8464ca5eda3ab7cc3e2d86cdb3c720f", "sid": "CF33199d5a9a0b202b3bd9558438a052d8", "date_created": "2023-02-16T09:57:11Z", "api_version": "2010-04-01", "subresource_uris": {"participants": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CF33199d5a9a0b202b3bd9558438a052d8/Participants.json", "recordings": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Conferences/CF33199d5a9a0b202b3bd9558438a052d8/Recordings.json"}}, "emitted_at": 1691419855510} @@ -22,10 +22,10 @@ {"stream": "incoming_phone_numbers", "data": {"origin": "twilio", "status": "in-use", "address_requirements": "none", "date_updated": "2023-03-27T07:57:03Z", "voice_url": "https://handler.twilio.com/twiml/EH5793263d703ad674bbcdeb31ac80e359", "sms_application_sid": "", "voice_fallback_method": "POST", "emergency_address_status": "unregistered", "identity_sid": null, "emergency_status": "Active", "voice_application_sid": "", "capabilities": {"fax": false, "voice": true, "sms": true, "mms": true}, "api_version": "2010-04-01", "sid": "PNf2eb05a16e73094f891b01076b830a6a", "status_callback_method": "POST", "voice_fallback_url": "", "phone_number": "+16508997708", "emergency_address_sid": null, "beta": false, "address_sid": "AD07820b628d536f40af85140c67e108f0", "sms_url": "https://webhooks.twilio.com/v1/Accounts/ACdade166c12e160e9ed0a6088226718fb/Flows/FWbd726b7110b21294a9f27a47f4ab0080", "voice_method": "POST", "voice_caller_id_lookup": false, "friendly_name": "Test phone number 8", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/IncomingPhoneNumbers/PNf2eb05a16e73094f891b01076b830a6a.json", "sms_fallback_url": "", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "sms_method": "POST", "trunk_sid": null, "sms_fallback_method": "POST", "date_created": "2023-02-16T14:31:29Z", "bundle_sid": null, "status_callback": "", "subresource_uris": {"assigned_add_ons": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/IncomingPhoneNumbers/PNf2eb05a16e73094f891b01076b830a6a/AssignedAddOns.json"}}, "emitted_at": 1691419867845} {"stream": "incoming_phone_numbers", "data": {"origin": "twilio", "status": "in-use", "address_requirements": "none", "date_updated": "2023-03-27T07:58:14Z", "voice_url": "https://handler.twilio.com/twiml/EHb6471af720e8b66baa14e7226227893b", "sms_application_sid": "", "voice_fallback_method": "POST", "emergency_address_status": "unregistered", "identity_sid": null, "emergency_status": "Active", "voice_application_sid": "", "capabilities": {"fax": false, "voice": true, "sms": true, "mms": true}, "api_version": "2010-04-01", "sid": "PNd74715bab1be123cc9004f03b85bb067", "status_callback_method": "POST", "voice_fallback_url": "", "phone_number": "+14246220939", "emergency_address_sid": null, "beta": false, "address_sid": "AD0164001bc0f84d9bc29e17378fe47c20", "sms_url": "https://webhooks.twilio.com/v1/Accounts/ACdade166c12e160e9ed0a6088226718fb/Flows/FWbd726b7110b21294a9f27a47f4ab0080", "voice_method": "POST", "voice_caller_id_lookup": false, "friendly_name": "Test phone number 9", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/IncomingPhoneNumbers/PNd74715bab1be123cc9004f03b85bb067.json", "sms_fallback_url": "", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "sms_method": "POST", "trunk_sid": null, "sms_fallback_method": "POST", "date_created": "2023-02-16T14:34:00Z", "bundle_sid": null, "status_callback": "", "subresource_uris": {"assigned_add_ons": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/IncomingPhoneNumbers/PNd74715bab1be123cc9004f03b85bb067/AssignedAddOns.json"}}, "emitted_at": 1691419867848} {"stream": "incoming_phone_numbers", "data": {"origin": "twilio", "status": "in-use", "address_requirements": "none", "date_updated": "2023-03-27T07:58:40Z", "voice_url": "https://handler.twilio.com/twiml/EHb77bc7c1f889b6c9fe5202d0463edfc4", "sms_application_sid": "", "voice_fallback_method": "POST", "emergency_address_status": "unregistered", "identity_sid": null, "emergency_status": "Active", "voice_application_sid": "", "capabilities": {"fax": false, "voice": true, "sms": true, "mms": true}, "api_version": "2010-04-01", "sid": "PN99400a65bf5a4305d5420060842d4d2c", "status_callback_method": "POST", "voice_fallback_url": "", "phone_number": "+19125901057", "emergency_address_sid": null, "beta": false, "address_sid": "AD0e69bf9110f766787a88f99b507c9eeb", "sms_url": "https://webhooks.twilio.com/v1/Accounts/ACdade166c12e160e9ed0a6088226718fb/Flows/FWbd726b7110b21294a9f27a47f4ab0080", "voice_method": "POST", "voice_caller_id_lookup": false, "friendly_name": "Test phone number 2", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/IncomingPhoneNumbers/PN99400a65bf5a4305d5420060842d4d2c.json", "sms_fallback_url": "", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "sms_method": "POST", "trunk_sid": null, "sms_fallback_method": "POST", "date_created": "2023-02-15T09:31:24Z", "bundle_sid": null, "status_callback": "", "subresource_uris": {"assigned_add_ons": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/IncomingPhoneNumbers/PN99400a65bf5a4305d5420060842d4d2c/AssignedAddOns.json"}}, "emitted_at": 1691419867849} -{"stream": "message_media", "data": {"sid": "MEa79c9e10f96b3c6018caa5e9a62567cc", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "parent_sid": "MM9170a757ee3976ecb8ebeb4e9580f9c0", "content_type": "image/png", "date_created": "2022-09-23T20:17:19Z", "date_updated": "2022-09-23T20:17:20Z", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM9170a757ee3976ecb8ebeb4e9580f9c0/Media/MEa79c9e10f96b3c6018caa5e9a62567cc.json"}, "emitted_at": 1691419887396} +{"stream": "message_media", "data": { "sid": "ME66ee8039997ee13231f5bd4a9121162c", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "parent_sid": "MMf491b7a98d00cdf54afc20b1839cea4e", "content_type": "image/png", "date_created": "2023-07-19T07:03:14Z", "date_updated": "2023-07-19T07:03:14Z", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MMf491b7a98d00cdf54afc20b1839cea4e/Media/ME66ee8039997ee13231f5bd4a9121162c.json" }, "emitted_at": 1691419887396} {"stream": "message_media", "data": {"sid": "ME34324546a4398b36fc96fd36500038c3", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "parent_sid": "MM56662e159d1a5d1f1c6e2d43202b7940", "content_type": "image/png", "date_created": "2023-02-14T14:02:28Z", "date_updated": "2023-02-14T14:02:28Z", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM56662e159d1a5d1f1c6e2d43202b7940/Media/ME34324546a4398b36fc96fd36500038c3.json"}, "emitted_at": 1691419915272} {"stream": "message_media", "data": {"sid": "ME45c86c927aa3eb6749bac07b9bc6f418", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "parent_sid": "MM5e9241ae9a444f8061b28e3de05fe818", "content_type": "image/png", "date_created": "2023-02-14T14:02:59Z", "date_updated": "2023-02-14T14:02:59Z", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM5e9241ae9a444f8061b28e3de05fe818/Media/ME45c86c927aa3eb6749bac07b9bc6f418.json"}, "emitted_at": 1691419915437} -{"stream": "messages", "data": {"body": "Hello there!", "num_segments": 1, "direction": "outbound-api", "from": "+12056561170", "date_updated": "2022-09-23T20:17:20Z", "price": -0.02, "error_message": "Unknown error", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM9170a757ee3976ecb8ebeb4e9580f9c0.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "num_media": 1, "to": "+19142793086", "date_created": "2022-09-23T20:17:19Z", "status": "undelivered", "sid": "MM9170a757ee3976ecb8ebeb4e9580f9c0", "date_sent": "2022-09-23T20:17:20Z", "messaging_service_sid": null, "error_code": "30008", "price_unit": "USD", "api_version": "2010-04-01", "subresource_uris": {"media": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM9170a757ee3976ecb8ebeb4e9580f9c0/Media.json", "feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM9170a757ee3976ecb8ebeb4e9580f9c0/Feedback.json"}}, "emitted_at": 1691419956845} +{"stream": "messages", "data": { "body": "Hi there, Test 4!", "num_segments": 1, "direction": "outbound-api", "from": "+12056561170", "date_updated": "2023-02-14T16:03:17Z", "price": -0.02, "error_message": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM70ec51fd8ba9408302cdd16b98a47c81.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "num_media": 1, "to": "+14156236785", "date_created": "2023-02-14T14:03:37Z", "status": "sent", "sid": "MM70ec51fd8ba9408302cdd16b98a47c81", "date_sent": "2023-02-14T14:03:38Z", "messaging_service_sid": null, "error_code": null, "price_unit": "USD", "api_version": "2010-04-01", "subresource_uris": { "media": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM70ec51fd8ba9408302cdd16b98a47c81/Media.json", "feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/MM70ec51fd8ba9408302cdd16b98a47c81/Feedback.json" } }, "emitted_at": 1691419956845} {"stream": "messages", "data": {"body": "Test", "num_segments": 1, "direction": "inbound", "from": "+12025502908", "date_updated": "2022-12-16T18:58:29Z", "price": -0.0079, "error_message": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/SMf73a453514f0a1d8bd4d0121713a8be9.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "num_media": 0, "to": "+12056561170", "date_created": "2022-12-16T18:58:28Z", "status": "received", "sid": "SMf73a453514f0a1d8bd4d0121713a8be9", "date_sent": "2022-12-16T18:58:29Z", "messaging_service_sid": null, "error_code": null, "price_unit": "USD", "api_version": "2010-04-01", "subresource_uris": {"media": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/SMf73a453514f0a1d8bd4d0121713a8be9/Media.json", "feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/SMf73a453514f0a1d8bd4d0121713a8be9/Feedback.json"}}, "emitted_at": 1691419957399} {"stream": "messages", "data": {"body": "Airbyte", "num_segments": 1, "direction": "inbound", "from": "+12025502908", "date_updated": "2022-12-16T18:58:47Z", "price": -0.0079, "error_message": null, "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/SM0ae34204de318609bd2801af5396442d.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "num_media": 0, "to": "+12056561170", "date_created": "2022-12-16T18:58:47Z", "status": "received", "sid": "SM0ae34204de318609bd2801af5396442d", "date_sent": "2022-12-16T18:58:47Z", "messaging_service_sid": null, "error_code": null, "price_unit": "USD", "api_version": "2010-04-01", "subresource_uris": {"media": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/SM0ae34204de318609bd2801af5396442d/Media.json", "feedback": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/Messages/SM0ae34204de318609bd2801af5396442d/Feedback.json"}}, "emitted_at": 1691419957401} {"stream": "outgoing_caller_ids", "data": {"phone_number": "+14153597503", "date_updated": "2020-11-17T04:17:37Z", "friendly_name": "(415) 359-7503", "uri": "/2010-04-01/Accounts/ACdade166c12e160e9ed0a6088226718fb/OutgoingCallerIds/PN16ba111c0df5756cfe37044ed0ee3136.json", "account_sid": "ACdade166c12e160e9ed0a6088226718fb", "sid": "PN16ba111c0df5756cfe37044ed0ee3136", "date_created": "2020-11-17T04:17:37Z"}, "emitted_at": 1691419960444} diff --git a/airbyte-integrations/connectors/source-twilio/metadata.yaml b/airbyte-integrations/connectors/source-twilio/metadata.yaml index 34410a3acd58..db8a6e53432d 100644 --- a/airbyte-integrations/connectors/source-twilio/metadata.yaml +++ b/airbyte-integrations/connectors/source-twilio/metadata.yaml @@ -1,15 +1,21 @@ data: + ab_internal: + ql: 200 + sl: 200 allowedHosts: hosts: - api.twilio.com - monitor.twilio.com - chat.twilio.com - trunking.twilio.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: b9dc6155-672e-42ea-b10d-9f1f1fb95ab1 - dockerImageTag: 0.10.0 + dockerImageTag: 0.10.1 dockerRepository: airbyte/source-twilio + documentationUrl: https://docs.airbyte.com/integrations/sources/twilio githubIssueLabel: source-twilio icon: twilio.svg license: MIT @@ -20,11 +26,7 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/twilio + supportLevel: certified tags: - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py b/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py index 0d1c3b44b6b6..2b47141ca77d 100644 --- a/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py +++ b/airbyte-integrations/connectors/source-twilio/source_twilio/streams.py @@ -71,8 +71,9 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp for record in records: for field in self.changeable_fields: record.pop(field, None) - yield record - yield from records + yield record + else: + yield from records def backoff_time(self, response: requests.Response) -> Optional[float]: """This method is called if we run into the rate limit. diff --git a/airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py index ad6d7dad4569..10d64697b22e 100644 --- a/airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-twilio/unit_tests/test_streams.py @@ -22,6 +22,7 @@ Messages, Recordings, TwilioNestedStream, + TwilioStream, UsageRecords, UsageTriggers, ) @@ -59,13 +60,14 @@ def test_data_field(self, stream_cls, expected): @pytest.mark.parametrize( "stream_cls, expected", [ - (Accounts, []), + (Accounts, ['name']), ], ) def test_changeable_fields(self, stream_cls, expected): - stream = stream_cls(**self.CONFIG) - result = stream.changeable_fields - assert result == expected + with patch.object(Accounts, "changeable_fields", ['name']): + stream = stream_cls(**self.CONFIG) + result = stream.changeable_fields + assert result == expected @pytest.mark.parametrize( "stream_cls, expected", @@ -101,16 +103,17 @@ def test_next_page_token(self, requests_mock, stream_cls, test_response, expecte @pytest.mark.parametrize( "stream_cls, test_response, expected", [ - (Accounts, {"accounts": {"id": "123"}}, ["id"]), + (Accounts, {"accounts": [{"id": "123", "name": "test"}]}, [{"id": "123"}]), ], ) def test_parse_response(self, requests_mock, stream_cls, test_response, expected): - stream = stream_cls(**self.CONFIG) - url = f"{stream.url_base}{stream.path()}" - requests_mock.get(url, json=test_response) - response = requests.get(url) - result = stream.parse_response(response) - assert list(result) == expected + with patch.object(TwilioStream, "changeable_fields", ["name"]): + stream = stream_cls(**self.CONFIG) + url = f"{stream.url_base}{stream.path()}" + requests_mock.get(url, json=test_response) + response = requests.get(url) + result = list(stream.parse_response(response)) + assert result[0]['id'] == expected[0]['id'] @pytest.mark.parametrize( "stream_cls, expected", @@ -218,32 +221,24 @@ def test_stream_slices(self, mocker, stream_cls, parent_cls_records, extra_slice Messages, {"date_sent": "2022-11-13 23:39:00"}, [ - {'DateSent>': '2022-11-13 23:39:00Z', 'DateSent<': '2022-11-14 23:39:00Z'}, - {'DateSent>': '2022-11-14 23:39:00Z', 'DateSent<': '2022-11-15 23:39:00Z'}, - {'DateSent>': '2022-11-15 23:39:00Z', 'DateSent<': '2022-11-16 12:03:11Z'} - ] + {"DateSent>": "2022-11-13 23:39:00Z", "DateSent<": "2022-11-14 23:39:00Z"}, + {"DateSent>": "2022-11-14 23:39:00Z", "DateSent<": "2022-11-15 23:39:00Z"}, + {"DateSent>": "2022-11-15 23:39:00Z", "DateSent<": "2022-11-16 12:03:11Z"}, + ], ), + (UsageRecords, {"start_date": "2021-11-16 00:00:00"}, [{"StartDate": "2021-11-16", "EndDate": "2022-11-16"}]), ( - UsageRecords, - {"start_date": "2021-11-16 00:00:00"}, + Recordings, + {"date_created": "2021-11-16 00:00:00"}, [ - {'StartDate': '2021-11-16', 'EndDate': '2022-11-16'} - ] + {"DateCreated>": "2021-11-16 00:00:00Z", "DateCreated<": "2022-11-16 00:00:00Z"}, + {"DateCreated>": "2022-11-16 00:00:00Z", "DateCreated<": "2022-11-16 12:03:11Z"}, + ], ), - ( - Recordings, {"date_created": "2021-11-16 00:00:00"}, - [ - {'DateCreated>': '2021-11-16 00:00:00Z', 'DateCreated<': '2022-11-16 00:00:00Z'}, - {'DateCreated>': '2022-11-16 00:00:00Z', 'DateCreated<': '2022-11-16 12:03:11Z'} - ] - ) - ) + ), ) def test_generate_dt_ranges(self, stream_cls, state, expected_dt_ranges): - stream = stream_cls( - authenticator=TEST_CONFIG.get("authenticator"), - start_date="2000-01-01 00:00:00" - ) + stream = stream_cls(authenticator=TEST_CONFIG.get("authenticator"), start_date="2000-01-01 00:00:00") stream.state = state dt_ranges = list(stream.generate_date_ranges()) assert dt_ranges == expected_dt_ranges diff --git a/airbyte-integrations/connectors/source-twitter/README.md b/airbyte-integrations/connectors/source-twitter/README.md index 59078caa9682..4dffdf658200 100644 --- a/airbyte-integrations/connectors/source-twitter/README.md +++ b/airbyte-integrations/connectors/source-twitter/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-twitter:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/twitter) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_twitter/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-twitter:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-twitter build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-twitter:airbyteDocker +An image will be built with the tag `airbyte/source-twitter:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-twitter:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,26 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-twitter:dev check --co docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-twitter:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-twitter:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. -If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -``` -python -m pytest integration_tests -p integration_tests.acceptance +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-twitter test ``` -To run your integration tests with docker -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-twitter:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-twitter:integrationTest -``` +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -74,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-twitter test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/twitter.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-twitter/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-twitter/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-twitter/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-twitter/build.gradle b/airbyte-integrations/connectors/source-twitter/build.gradle deleted file mode 100644 index 2feb3d98cee8..000000000000 --- a/airbyte-integrations/connectors/source-twitter/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_twitter' -} diff --git a/airbyte-integrations/connectors/source-twitter/metadata.yaml b/airbyte-integrations/connectors/source-twitter/metadata.yaml index 11ff23358334..b80e85f83dbf 100644 --- a/airbyte-integrations/connectors/source-twitter/metadata.yaml +++ b/airbyte-integrations/connectors/source-twitter/metadata.yaml @@ -1,4 +1,7 @@ data: + ab_internal: + ql: 300 + sl: 100 allowedHosts: hosts: - api.twitter.com @@ -7,6 +10,7 @@ data: definitionId: d7fd4f40-5e5a-4b8b-918f-a73077f8c131 dockerImageTag: 0.1.2 dockerRepository: airbyte/source-twitter + documentationUrl: https://docs.airbyte.com/integrations/sources/twitter githubIssueLabel: source-twitter icon: twitter.svg license: MIT @@ -17,12 +21,8 @@ data: oss: enabled: true releaseStage: beta - documentationUrl: https://docs.airbyte.com/integrations/sources/twitter + supportLevel: community tags: - language:low-code - language:python - ab_internal: - sl: 200 - ql: 300 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-tyntec-sms/README.md b/airbyte-integrations/connectors/source-tyntec-sms/README.md index d0f274f425ae..fe952c25d8c4 100644 --- a/airbyte-integrations/connectors/source-tyntec-sms/README.md +++ b/airbyte-integrations/connectors/source-tyntec-sms/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-tyntec-sms:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/tyntec-sms) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_tyntec_sms/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-tyntec-sms:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-tyntec-sms build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-tyntec-sms:airbyteDocker +An image will be built with the tag `airbyte/source-tyntec-sms:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-tyntec-sms:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-tyntec-sms:dev check - docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-tyntec-sms:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-tyntec-sms:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-tyntec-sms test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-tyntec-sms:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-tyntec-sms:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-tyntec-sms test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/tyntec-sms.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-tyntec-sms/acceptance-test-config.yml b/airbyte-integrations/connectors/source-tyntec-sms/acceptance-test-config.yml index 941b685e1e8d..6af682c1b675 100644 --- a/airbyte-integrations/connectors/source-tyntec-sms/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-tyntec-sms/acceptance-test-config.yml @@ -21,7 +21,7 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - incremental: + incremental: bypass_reason: "This connector does not implement incremental sync" full_refresh: tests: diff --git a/airbyte-integrations/connectors/source-tyntec-sms/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-tyntec-sms/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-tyntec-sms/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-tyntec-sms/build.gradle b/airbyte-integrations/connectors/source-tyntec-sms/build.gradle deleted file mode 100644 index 080863eb9787..000000000000 --- a/airbyte-integrations/connectors/source-tyntec-sms/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_tyntec_sms' -} diff --git a/airbyte-integrations/connectors/source-typeform/.coveragerc b/airbyte-integrations/connectors/source-typeform/.coveragerc new file mode 100644 index 000000000000..2c98af4de877 --- /dev/null +++ b/airbyte-integrations/connectors/source-typeform/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + source_typeform/run.py \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-typeform/.dockerignore b/airbyte-integrations/connectors/source-typeform/.dockerignore index cb57facccb8d..138d52c577c0 100644 --- a/airbyte-integrations/connectors/source-typeform/.dockerignore +++ b/airbyte-integrations/connectors/source-typeform/.dockerignore @@ -1,6 +1,5 @@ * !Dockerfile -!Dockerfile.test !main.py !source_typeform !setup.py diff --git a/airbyte-integrations/connectors/source-typeform/Dockerfile b/airbyte-integrations/connectors/source-typeform/Dockerfile deleted file mode 100644 index 82ab07c099d5..000000000000 --- a/airbyte-integrations/connectors/source-typeform/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY setup.py ./ -RUN pip install . -COPY source_typeform ./source_typeform -COPY main.py ./ - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=1.0.0 -LABEL io.airbyte.name=airbyte/source-typeform diff --git a/airbyte-integrations/connectors/source-typeform/README.md b/airbyte-integrations/connectors/source-typeform/README.md index acbe4c6db7b3..b414ad7e136a 100644 --- a/airbyte-integrations/connectors/source-typeform/README.md +++ b/airbyte-integrations/connectors/source-typeform/README.md @@ -1,74 +1,85 @@ # Typeform Source -This is the repository for the Typeform source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/typeform). +This is the repository for the Typeform configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/typeform). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/typeform) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_typeform/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. -#### Minimum Python version required `= 3.7.0` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source typeform test creds` +and place them into `secrets/config.json`. -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` +### Locally running the connector docker image -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-typeform:build +```bash +airbyte-ci connectors --name=source-typeform build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-typeform:dev`. -#### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/typeform) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_typeform/spec.json` file. -Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. -See `integration_tests/sample_config.json` for a sample config file. +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source typeform test creds` -and place them into `secrets/config.json`. +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` +from typing import TYPE_CHECKING -### Locally running the connector docker image +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-typeform:dev -``` -You can also build the connector image via Gradle: +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-typeform:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-typeform:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-typeform:dev . +# Running the spec command against your patched connector +docker run airbyte/source-typeform:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -77,44 +88,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-typeform:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-typeform:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-typeform:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-typeform test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-typeform:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-typeform:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -124,8 +107,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-typeform test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/typeform.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-typeform/__init__.py b/airbyte-integrations/connectors/source-typeform/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-typeform/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-typeform/acceptance-test-config.yml b/airbyte-integrations/connectors/source-typeform/acceptance-test-config.yml index c8f7582adcad..ebcb571f701e 100644 --- a/airbyte-integrations/connectors/source-typeform/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-typeform/acceptance-test-config.yml @@ -1,42 +1,43 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests connector_image: airbyte/source-typeform:1.0.0 test_strictness_level: "high" acceptance_tests: spec: tests: - - spec_path: "source_typeform/spec.json" - backward_compatibility_tests_config: - disable_for_version: "0.3.0" + - spec_path: "source_typeform/spec.yaml" + backward_compatibility_tests_config: + disable_for_version: "0.3.0" connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "secrets/config_oauth.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "secrets/config_oauth.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: tests: - - config_path: "secrets/config.json" - backward_compatibility_tests_config: - disable_for_version: "0.3.0" + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.3.0" basic_read: tests: - - config_path: "secrets/config.json" - empty_streams: - - name: webhooks - bypass_reason: "no data" - expect_records: - path: "integration_tests/expected_records.jsonl" - fail_on_extra_columns: true + - config_path: "secrets/config.json" + empty_streams: + - name: webhooks + bypass_reason: "no data" + expect_records: + path: "integration_tests/expected_records.jsonl" + fail_on_extra_columns: true incremental: tests: - - config_path: "secrets/incremental_config.json" - configured_catalog_path: "integration_tests/configured_catalog_incremental.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" - cursor_paths: - "responses": ["SdMKQYkv", "submitted_at"] + - config_path: "secrets/incremental_config.json" + configured_catalog_path: "integration_tests/configured_catalog_incremental.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" + skip_comprehensive_incremental_tests: true full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-typeform/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-typeform/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-typeform/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-typeform/build.gradle b/airbyte-integrations/connectors/source-typeform/build.gradle deleted file mode 100644 index 8098789cc3b0..000000000000 --- a/airbyte-integrations/connectors/source-typeform/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_typeform' -} diff --git a/airbyte-integrations/connectors/source-typeform/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-typeform/integration_tests/abnormal_state.json index 3b04806c8ce9..d09ec8ec208c 100644 --- a/airbyte-integrations/connectors/source-typeform/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-typeform/integration_tests/abnormal_state.json @@ -2,21 +2,15 @@ { "type": "STREAM", "stream": { + "stream_descriptor": { "name": "responses" }, "stream_state": { - "SdMKQYkv": { - "submitted_at": 9999999999 - }, - "XtrcGoGJ": { - "submitted_at": 9999999999 - }, - "kRt99jlK": { - "submitted_at": 9999999999 - }, - "VWO7mLtl": { - "submitted_at": 9999999999 - } - }, - "stream_descriptor": { "name": "responses" } + "states": [ + { + "partition": { "form_id": "SdMKQYkv" }, + "cursor": { "submitted_at": "2050-09-04T16:39:47Z" } + } + ] + } } } ] diff --git a/airbyte-integrations/connectors/source-typeform/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-typeform/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-typeform/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-typeform/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-typeform/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-typeform/integration_tests/expected_records.jsonl index bf37dab0f628..c24d2b0cbe90 100644 --- a/airbyte-integrations/connectors/source-typeform/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-typeform/integration_tests/expected_records.jsonl @@ -1,18 +1,9 @@ {"stream": "forms", "data": {"id": "VWO7mLtl", "type": "quiz", "title": "Connector Extensibility meetup", "workspace": {"href": "https://api.typeform.com/workspaces/sDaAqs"}, "theme": {"href": "https://api.typeform.com/themes/qHWOQ7"}, "settings": {"language": "en", "progress_bar": "proportion", "meta": {"allow_indexing": false}, "hide_navigation": false, "is_public": true, "is_trial": false, "show_progress_bar": true, "show_typeform_branding": true, "are_uploads_public": false, "show_time_to_complete": true, "show_number_of_submissions": false, "show_cookie_consent": false, "show_question_number": true, "show_key_hint_on_choices": true, "autosave_progress": true, "free_form_navigation": false, "use_lead_qualification": false, "pro_subdomain_enabled": false, "capabilities": {"e2e_encryption": {"enabled": false, "modifiable": false}}}, "thankyou_screens": [{"id": "qvDqCNAHuIC8", "ref": "01GHC6KQ5Y0M8VN6XHVAG75J0G", "title": "", "type": "thankyou_screen", "properties": {"show_button": true, "share_icons": true, "button_mode": "default_redirect", "button_text": "Create a typeform"}}, {"id": "DefaultTyScreen", "ref": "default_tys", "title": "Thanks for completing this typeform\nNow *create your own* \u2014 it's free, easy, & beautiful", "type": "thankyou_screen", "properties": {"show_button": true, "share_icons": false, "button_mode": "default_redirect", "button_text": "Create a *typeform*"}, "attachment": {"type": "image", "href": "https://images.typeform.com/images/2dpnUBBkz2VN"}}], "fields": [{"id": "ZdzF0rrvsVdB", "title": "What times work for you to visit San Francisco to work with the team?", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM", "properties": {"randomize": false, "allow_multiple_selection": true, "allow_other_choice": true, "vertical_alignment": true, "choices": [{"id": "nLpt4rvNjFB3", "ref": "01GHC6KQ5Y155J0F550BGYYS1A", "label": "Dec 12-16"}, {"id": "4xpK9sqA06eL", "ref": "01GHC6KQ5YBATX0CFENVVB5BYG", "label": "Dec 19-23"}, {"id": "jQHb3mqslOsZ", "ref": "1c392fa3-e693-49fe-b334-3a5cddc1db6f", "label": "Jan 9-14"}, {"id": "wS5FKMUnMgqR", "ref": "2ac396a3-1b8e-4e56-b36d-d1f27c1b834d", "label": "Jan 16-20"}, {"id": "uvmLX80Loava", "ref": "8fffd3a8-1e96-421d-a605-a7029bd55e97", "label": "Jan 22-26"}, {"id": "7ubtgCrW2meb", "ref": "17403cc9-74cd-49d1-856a-be6662b3b497", "label": "Jan30 - Feb3"}, {"id": "51q0g4fTFtYc", "ref": "3a1295b4-97b9-4986-9c37-f1af1d72501d", "label": "Feb 6 - 11"}, {"id": "vi3iwtpETqlb", "ref": "54edf52a-c9c7-4bc4-a5a6-bd86115f5adb", "label": "Feb 13-17"}, {"id": "iI0hDpta14Kk", "ref": "e149c19f-8b61-4ff0-a17a-e9e65c3a8fee", "label": "Feb 19-24"}]}, "validations": {"required": false}, "type": "multiple_choice", "attachment": {"type": "image", "href": "https://images.typeform.com/images/WMALzu59xbXQ"}, "layout": {"type": "split", "attachment": {"type": "image", "href": "https://images.typeform.com/images/WMALzu59xbXQ"}}}], "created_at": "2022-11-08T18:04:03+00:00", "last_updated_at": "2022-11-08T21:10:54+00:00", "published_at": "2022-11-08T21:10:54+00:00", "_links": {"display": "https://xe03v5buli4.typeform.com/to/VWO7mLtl", "responses": "https://api.typeform.com/forms/VWO7mLtl/responses"}}, "emitted_at": 1686590629013} -{"stream": "responses", "data": {"landing_id": "fr2wm964fnyxpdx9a8tfr2wmlph34hqi", "token": "fr2wm964fnyxpdx9a8tfr2wmlph34hqi", "response_id": "fr2wm964fnyxpdx9a8tfr2wmlph34hqi", "landed_at": "2022-11-08T21:59:53Z", "submitted_at": "2022-11-08T22:00:24Z", "metadata": {"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/VWO7mLtl", "network_id": "8a0111039f", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "ZdzF0rrvsVdB", "type": "multiple_choice", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM"}, "type": "choices", "choices": {"ids": ["nLpt4rvNjFB3", "4xpK9sqA06eL", "jQHb3mqslOsZ", "wS5FKMUnMgqR", "uvmLX80Loava", "7ubtgCrW2meb", "iI0hDpta14Kk"], "refs": ["01GHC6KQ5Y155J0F550BGYYS1A", "01GHC6KQ5YBATX0CFENVVB5BYG", "1c392fa3-e693-49fe-b334-3a5cddc1db6f", "2ac396a3-1b8e-4e56-b36d-d1f27c1b834d", "8fffd3a8-1e96-421d-a605-a7029bd55e97", "17403cc9-74cd-49d1-856a-be6662b3b497", "e149c19f-8b61-4ff0-a17a-e9e65c3a8fee"], "labels": ["Dec 12-16", "Dec 19-23", "Jan 9-14", "Jan 16-20", "Jan 22-26", "Jan30 - Feb3", "Feb 19-24"]}}], "form_id": "VWO7mLtl"}, "emitted_at": 1687522222458} -{"stream": "responses", "data": {"landing_id": "0dc8djmlrkmxuwu7s7mmia0dc8dj4a1r", "token": "0dc8djmlrkmxuwu7s7mmia0dc8dj4a1r", "response_id": "0dc8djmlrkmxuwu7s7mmia0dc8dj4a1r", "landed_at": "2022-11-08T22:08:39Z", "submitted_at": "2022-11-08T22:10:04Z", "metadata": {"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/VWO7mLtl", "network_id": "d4b74277d2", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "ZdzF0rrvsVdB", "type": "multiple_choice", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM"}, "type": "choices", "choices": {"ids": ["nLpt4rvNjFB3", "wS5FKMUnMgqR", "jQHb3mqslOsZ", "51q0g4fTFtYc", "vi3iwtpETqlb", "iI0hDpta14Kk"], "refs": ["01GHC6KQ5Y155J0F550BGYYS1A", "2ac396a3-1b8e-4e56-b36d-d1f27c1b834d", "1c392fa3-e693-49fe-b334-3a5cddc1db6f", "3a1295b4-97b9-4986-9c37-f1af1d72501d", "54edf52a-c9c7-4bc4-a5a6-bd86115f5adb", "e149c19f-8b61-4ff0-a17a-e9e65c3a8fee"], "labels": ["Dec 12-16", "Jan 16-20", "Jan 9-14", "Feb 6 - 11", "Feb 13-17", "Feb 19-24"]}}], "form_id": "VWO7mLtl"}, "emitted_at": 1687522222461} -{"stream": "responses", "data": {"landing_id": "ng2hh3i6cy7ikeyorbnl0ng2hh3icyvq", "token": "ng2hh3i6cy7ikeyorbnl0ng2hh3icyvq", "response_id": "ng2hh3i6cy7ikeyorbnl0ng2hh3icyvq", "landed_at": "2022-11-09T06:16:08Z", "submitted_at": "2022-11-09T06:16:10Z", "metadata": {"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/VWO7mLtl", "network_id": "2be9dd4bab", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "ZdzF0rrvsVdB", "type": "multiple_choice", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM"}, "type": "choices", "choices": {"ids": ["nLpt4rvNjFB3", "wS5FKMUnMgqR", "uvmLX80Loava", "7ubtgCrW2meb", "51q0g4fTFtYc", "vi3iwtpETqlb", "iI0hDpta14Kk"], "refs": ["01GHC6KQ5Y155J0F550BGYYS1A", "2ac396a3-1b8e-4e56-b36d-d1f27c1b834d", "8fffd3a8-1e96-421d-a605-a7029bd55e97", "17403cc9-74cd-49d1-856a-be6662b3b497", "3a1295b4-97b9-4986-9c37-f1af1d72501d", "54edf52a-c9c7-4bc4-a5a6-bd86115f5adb", "e149c19f-8b61-4ff0-a17a-e9e65c3a8fee"], "labels": ["Dec 12-16", "Jan 16-20", "Jan 22-26", "Jan30 - Feb3", "Feb 6 - 11", "Feb 13-17", "Feb 19-24"]}}], "form_id": "VWO7mLtl"}, "emitted_at": 1687522222826} -{"stream": "responses", "data": {"landing_id": "e7hli3wynwfkiwaebwe7h2aeeso4xrum", "token": "e7hli3wynwfkiwaebwe7h2aeeso4xrum", "response_id": "e7hli3wynwfkiwaebwe7h2aeeso4xrum", "landed_at": "2022-11-09T08:09:11Z", "submitted_at": "2022-11-09T08:33:38Z", "metadata": {"user_agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 15_6_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.6.1 Mobile/15E148 Safari/604.1", "platform": "mobile", "referer": "https://xe03v5buli4.typeform.com/to/VWO7mLtl?typeform-source=www.linkedin.com", "network_id": "fec8bf87e1", "browser": "touch"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "ZdzF0rrvsVdB", "type": "multiple_choice", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM"}, "type": "choices", "choices": {"ids": ["nLpt4rvNjFB3", "wS5FKMUnMgqR", "uvmLX80Loava"], "refs": ["01GHC6KQ5Y155J0F550BGYYS1A", "2ac396a3-1b8e-4e56-b36d-d1f27c1b834d", "8fffd3a8-1e96-421d-a605-a7029bd55e97"], "labels": ["Dec 12-16", "Jan 16-20", "Jan 22-26"]}}], "form_id": "VWO7mLtl"}, "emitted_at": 1687522222827} -{"stream": "responses", "data": {"landing_id": "r4epuzzxlonggr4epp07wenb3a58sm6h", "token": "r4epuzzxlonggr4epp07wenb3a58sm6h", "response_id": "r4epuzzxlonggr4epp07wenb3a58sm6h", "landed_at": "2022-11-09T14:33:46Z", "submitted_at": "2022-11-09T14:35:10Z", "metadata": {"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/VWO7mLtl", "network_id": "7e338d2504", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "ZdzF0rrvsVdB", "type": "multiple_choice", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM"}, "type": "choices", "choices": {"ids": ["51q0g4fTFtYc", "vi3iwtpETqlb", "iI0hDpta14Kk"], "refs": ["3a1295b4-97b9-4986-9c37-f1af1d72501d", "54edf52a-c9c7-4bc4-a5a6-bd86115f5adb", "e149c19f-8b61-4ff0-a17a-e9e65c3a8fee"], "labels": ["Feb 6 - 11", "Feb 13-17", "Feb 19-24"]}}], "form_id": "VWO7mLtl"}, "emitted_at": 1687522223051} -{"stream": "responses", "data": {"landing_id": "ic7ydv73zomudp1p9ic7yp9spye7h72b", "token": "ic7ydv73zomudp1p9ic7yp9spye7h72b", "response_id": "ic7ydv73zomudp1p9ic7yp9spye7h72b", "landed_at": "2022-11-15T02:31:04Z", "submitted_at": "2022-11-15T02:34:53Z", "metadata": {"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/VWO7mLtl", "network_id": "8284380108", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "ZdzF0rrvsVdB", "type": "multiple_choice", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM"}, "type": "choices", "choices": {"ids": ["jQHb3mqslOsZ", "wS5FKMUnMgqR", "uvmLX80Loava", "7ubtgCrW2meb", "51q0g4fTFtYc", "iI0hDpta14Kk"], "refs": ["1c392fa3-e693-49fe-b334-3a5cddc1db6f", "2ac396a3-1b8e-4e56-b36d-d1f27c1b834d", "8fffd3a8-1e96-421d-a605-a7029bd55e97", "17403cc9-74cd-49d1-856a-be6662b3b497", "3a1295b4-97b9-4986-9c37-f1af1d72501d", "e149c19f-8b61-4ff0-a17a-e9e65c3a8fee"], "labels": ["Jan 9-14", "Jan 16-20", "Jan 22-26", "Jan30 - Feb3", "Feb 6 - 11", "Feb 19-24"]}}], "form_id": "VWO7mLtl"}, "emitted_at": 1687522223053} -{"stream": "responses", "data": {"landing_id": "trdyqvm2wmf9b0dhoostrdtugo8fdcoa", "token": "trdyqvm2wmf9b0dhoostrdtugo8fdcoa", "response_id": "trdyqvm2wmf9b0dhoostrdtugo8fdcoa", "landed_at": "2022-11-15T02:36:40Z", "submitted_at": "2022-11-15T02:39:52Z", "metadata": {"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/VWO7mLtl", "network_id": "8284380108", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "ZdzF0rrvsVdB", "type": "multiple_choice", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM"}, "type": "choices", "choices": {"ids": ["jQHb3mqslOsZ", "wS5FKMUnMgqR", "uvmLX80Loava", "7ubtgCrW2meb", "51q0g4fTFtYc", "iI0hDpta14Kk"], "refs": ["1c392fa3-e693-49fe-b334-3a5cddc1db6f", "2ac396a3-1b8e-4e56-b36d-d1f27c1b834d", "8fffd3a8-1e96-421d-a605-a7029bd55e97", "17403cc9-74cd-49d1-856a-be6662b3b497", "3a1295b4-97b9-4986-9c37-f1af1d72501d", "e149c19f-8b61-4ff0-a17a-e9e65c3a8fee"], "labels": ["Jan 9-14", "Jan 16-20", "Jan 22-26", "Jan30 - Feb3", "Feb 6 - 11", "Feb 19-24"]}}], "form_id": "VWO7mLtl"}, "emitted_at": 1687522223249} -{"stream": "responses", "data": {"landing_id": "w9yrjygpz00o1vop20h68tlw9yriwyqr", "token": "w9yrjygpz00o1vop20h68tlw9yriwyqr", "response_id": "w9yrjygpz00o1vop20h68tlw9yriwyqr", "landed_at": "2021-06-27T15:16:09Z", "submitted_at": "2021-06-27T15:18:12Z", "metadata": {"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/SdMKQYkv", "network_id": "abd4cbf203", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "63WvXUnvSCa9", "type": "short_text", "ref": "ef34b985c51e4131"}, "type": "text", "text": "Mr X"}, {"field": {"id": "kwpFrd2lI3ok", "type": "email", "ref": "0c3cabd70157cf16"}, "type": "email", "email": "mrx@airbyte.com"}, {"field": {"id": "1Ua3d1mzhJwj", "type": "picture_choice", "ref": "7207397713e2b5e3"}, "type": "choice", "choice": {"id": "z5hxxjpJl07L", "ref": "bfcc3fbf608583f7", "label": "Yes"}}, {"field": {"id": "MmrPLXSaCF5B", "type": "short_text", "ref": "9aaaeeebe70858c4"}, "type": "text", "text": "water"}, {"field": {"id": "gurSOcuvNnvb", "type": "long_text", "ref": "18842abd9aa9ded4"}, "type": "text", "text": "do you know who I am ?"}], "form_id": "SdMKQYkv"}, "emitted_at": 1687522223916} -{"stream": "responses", "data": {"landing_id": "x9ege1s9u758nla0bx9ege1bg63u7daz", "token": "x9ege1s9u758nla0bx9ege1bg63u7daz", "response_id": "x9ege1s9u758nla0bx9ege1bg63u7daz", "landed_at": "2021-07-01T10:03:21Z", "submitted_at": "2021-07-01T10:04:01Z", "metadata": {"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/kRt99jlK", "network_id": "32d3e45763", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "qthblBc7InVU", "type": "multiple_choice", "ref": "8267768033031e53"}, "type": "choice", "choice": {"id": "3XbsOhLiGFkv", "ref": "fc3c3f5dbe75a01e", "label": "Center-right party"}}, {"field": {"id": "rB7FJUThFlu4", "type": "picture_choice", "ref": "b13b02912db6f287"}, "type": "choice", "choice": {"id": "Jt8FTorS35Sb", "ref": "b99bb45c5b8c25be", "label": "Going in the wrong direction"}}, {"field": {"id": "vV7ISYSgZ94I", "type": "opinion_scale", "ref": "f1939629f760be75"}, "type": "number", "number": 1}, {"field": {"id": "Mrq4qNeRInni", "type": "picture_choice", "ref": "c52566d91c5052e2"}, "type": "choice", "choice": {"id": "szBe0vqnkCUK", "ref": "3f1692f85c3cd73b", "label": "Military & Defense"}}, {"field": {"id": "x9myjwStSn9a", "type": "rating", "ref": "55c2e5c15f7dccec"}, "type": "number", "number": 1}, {"field": {"id": "zaP8jDAArI5x", "type": "rating", "ref": "f853e99096a32208"}, "type": "number", "number": 1}, {"field": {"id": "VFmcjbHlFTzg", "type": "rating", "ref": "6f0fd734177ecf27"}, "type": "number", "number": 1}, {"field": {"id": "DgGh4ZkRBAyH", "type": "rating", "ref": "b4171ed292dc3cee"}, "type": "number", "number": 1}, {"field": {"id": "yC3UrwN1LKT8", "type": "rating", "ref": "c8dd7d63c26777d9"}, "type": "number", "number": 5}], "form_id": "kRt99jlK"}, "emitted_at": 1687522224466} -{"stream": "responses", "data": {"landing_id": "ohgpotzjg8w852pohgpo1ub0gq76tks2", "token": "ohgpotzjg8w852pohgpo1ub0gq76tks2", "response_id": "ohgpotzjg8w852pohgpo1ub0gq76tks2", "landed_at": "2021-06-20T16:49:16Z", "submitted_at": "2021-06-20T16:49:20Z", "metadata": {"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/XtrcGoGJ", "network_id": "1866502915", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "8VK4KwNd0DgB", "type": "short_text", "ref": "01F8N53B7KPZ2A1DWGZTTK9SKG"}, "type": "text", "text": "11"}, {"field": {"id": "X6dq0mumvtKq", "type": "multiple_choice", "ref": "01F8N53B8293QHVDDHT84RZR6K"}, "type": "choice", "choice": {"id": "FWQrVLFdHroI", "ref": "01F8N53B82JXPXZ1B53BMJY0X2", "label": "Terrific!"}}], "form_id": "XtrcGoGJ"}, "emitted_at": 1687522225029} -{"stream": "responses", "data": {"landing_id": "74zhlkhbmspze5nllpl143674zhlkh81", "token": "74zhlkhbmspze5nllpl143674zhlkh81", "response_id": "74zhlkhbmspze5nllpl143674zhlkh81", "landed_at": "2021-06-27T15:32:07Z", "submitted_at": "2021-06-27T15:32:14Z", "metadata": {"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/XtrcGoGJ", "network_id": "abd4cbf203", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "8VK4KwNd0DgB", "type": "short_text", "ref": "01F8N53B7KPZ2A1DWGZTTK9SKG"}, "type": "text", "text": "222"}, {"field": {"id": "X6dq0mumvtKq", "type": "multiple_choice", "ref": "01F8N53B8293QHVDDHT84RZR6K"}, "type": "choice", "choice": {"id": "7jNEfjJ2cDAl", "ref": "01F8N53B82RE3YZK7RR50KNRQ0", "label": "Not so well..."}}], "form_id": "XtrcGoGJ"}, "emitted_at": 1687522225029} -{"stream": "responses", "data": {"landing_id": "9f7s89wh4wagc4qbr1c9f7s89tr0cu6g", "token": "9f7s89wh4wagc4qbr1c9f7s89tr0cu6g", "response_id": "9f7s89wh4wagc4qbr1c9f7s89tr0cu6g", "landed_at": "2021-06-27T15:32:33Z", "submitted_at": "2021-06-27T15:32:39Z", "metadata": {"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/XtrcGoGJ", "network_id": "abd4cbf203", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "8VK4KwNd0DgB", "type": "short_text", "ref": "01F8N53B7KPZ2A1DWGZTTK9SKG"}, "type": "text", "text": "There is a new library called furl. I find this library to be most pythonic for doing url algebra. To install:"}, {"field": {"id": "X6dq0mumvtKq", "type": "multiple_choice", "ref": "01F8N53B8293QHVDDHT84RZR6K"}, "type": "choice", "choice": {"id": "FWQrVLFdHroI", "ref": "01F8N53B82JXPXZ1B53BMJY0X2", "label": "Terrific!"}}], "form_id": "XtrcGoGJ"}, "emitted_at": 1687522225225} -{"stream": "responses", "data": {"landing_id": "zn4d1osa0ou5gitzn4k3e3am0c0q7vgd", "token": "zn4d1osa0ou5gitzn4k3e3am0c0q7vgd", "response_id": "zn4d1osa0ou5gitzn4k3e3am0c0q7vgd", "landed_at": "2021-07-01T14:06:57Z", "submitted_at": "2021-07-01T14:07:03Z", "metadata": {"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/XtrcGoGJ", "network_id": "32d3e45763", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "8VK4KwNd0DgB", "type": "short_text", "ref": "01F8N53B7KPZ2A1DWGZTTK9SKG"}, "type": "text", "text": "124125125"}, {"field": {"id": "X6dq0mumvtKq", "type": "multiple_choice", "ref": "01F8N53B8293QHVDDHT84RZR6K"}, "type": "choice", "choice": {"id": "7jNEfjJ2cDAl", "ref": "01F8N53B82RE3YZK7RR50KNRQ0", "label": "Not so well..."}}], "form_id": "XtrcGoGJ"}, "emitted_at": 1687522225226} -{"stream": "responses", "data": {"landing_id": "s3ah741anof3uot3340qs3ah7q62w544", "token": "s3ah741anof3uot3340qs3ah7q62w544", "response_id": "s3ah741anof3uot3340qs3ah7q62w544", "landed_at": "2021-09-04T14:35:19Z", "submitted_at": "2021-09-04T14:35:30Z", "metadata": {"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/XtrcGoGJ", "network_id": "8c4966ac74", "browser": "default"}, "hidden": {}, "calculated": {"score": 0}, "answers": [{"field": {"id": "8VK4KwNd0DgB", "type": "short_text", "ref": "01F8N53B7KPZ2A1DWGZTTK9SKG"}, "type": "text", "text": "test123"}, {"field": {"id": "6lGZzhNfrqwB", "type": "multiple_choice", "ref": "43153da3-fbbc-443e-b66f-1752770c0e0a"}, "type": "choices", "choices": {"ids": ["03VP9UxCwCLT", "ELm7HbFr0OOq", "acwDGU8NeO2A", "3HfyxDo5JoXf"], "refs": ["27b8dfcb-ef16-4ad7-b2be-734ec24c34ca", "ce51ab49-2cce-490d-b831-309337c79fa0", "74ef0411-0c8a-4c09-a6f3-7a62b0745f68", "f83999f6-c869-47cc-af2f-f22b628a0fdb"], "labels": ["choice 4", "choice2", "choice1", "choice 3"]}}, {"field": {"id": "X6dq0mumvtKq", "type": "multiple_choice", "ref": "01F8N53B8293QHVDDHT84RZR6K"}, "type": "choice", "choice": {"id": "FWQrVLFdHroI", "ref": "01F8N53B82JXPXZ1B53BMJY0X2", "label": "Terrific!"}}], "form_id": "XtrcGoGJ"}, "emitted_at": 1687522225440} +{"stream": "responses", "data": { "landing_id": "ic7ydv73zomudp1p9ic7yp9spye7h72b", "token": "ic7ydv73zomudp1p9ic7yp9spye7h72b", "response_id": "ic7ydv73zomudp1p9ic7yp9spye7h72b", "response_type": "completed", "landed_at": "2022-11-15T02:31:04Z", "submitted_at": "2022-11-15T02:34:53Z", "metadata": { "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/VWO7mLtl", "network_id": "8284380108", "browser": "default" }, "hidden": {}, "calculated": { "score": 0 }, "answers": [ { "field": { "id": "ZdzF0rrvsVdB", "type": "multiple_choice", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM" }, "type": "choices", "choices": { "ids": [ "jQHb3mqslOsZ", "wS5FKMUnMgqR", "uvmLX80Loava", "7ubtgCrW2meb", "51q0g4fTFtYc", "iI0hDpta14Kk" ], "refs": [ "1c392fa3-e693-49fe-b334-3a5cddc1db6f", "2ac396a3-1b8e-4e56-b36d-d1f27c1b834d", "8fffd3a8-1e96-421d-a605-a7029bd55e97", "17403cc9-74cd-49d1-856a-be6662b3b497", "3a1295b4-97b9-4986-9c37-f1af1d72501d", "e149c19f-8b61-4ff0-a17a-e9e65c3a8fee" ], "labels": [ "Jan 9-14", "Jan 16-20", "Jan 22-26", "Jan30 - Feb3", "Feb 6 - 11", "Feb 19-24" ] } } ], "form_id": "VWO7mLtl" }, "emitted_at": 1687522222458} +{"stream": "responses", "data": { "landing_id": "0dc8djmlrkmxuwu7s7mmia0dc8dj4a1r", "token": "0dc8djmlrkmxuwu7s7mmia0dc8dj4a1r", "response_id": "0dc8djmlrkmxuwu7s7mmia0dc8dj4a1r", "response_type": "completed", "landed_at": "2022-11-08T22:08:39Z", "submitted_at": "2022-11-08T22:10:04Z", "metadata": { "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/VWO7mLtl", "network_id": "d4b74277d2", "browser": "default" }, "hidden": {}, "calculated": { "score": 0 }, "answers": [ { "field": { "id": "ZdzF0rrvsVdB", "type": "multiple_choice", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM" }, "type": "choices", "choices": { "ids": [ "nLpt4rvNjFB3", "wS5FKMUnMgqR", "jQHb3mqslOsZ", "51q0g4fTFtYc", "vi3iwtpETqlb", "iI0hDpta14Kk" ], "refs": [ "01GHC6KQ5Y155J0F550BGYYS1A", "2ac396a3-1b8e-4e56-b36d-d1f27c1b834d", "1c392fa3-e693-49fe-b334-3a5cddc1db6f", "3a1295b4-97b9-4986-9c37-f1af1d72501d", "54edf52a-c9c7-4bc4-a5a6-bd86115f5adb", "e149c19f-8b61-4ff0-a17a-e9e65c3a8fee" ], "labels": [ "Dec 12-16", "Jan 16-20", "Jan 9-14", "Feb 6 - 11", "Feb 13-17", "Feb 19-24" ] } } ], "form_id": "VWO7mLtl" }, "emitted_at": 1687522222461} +{"stream": "responses", "data": { "landing_id": "ng2hh3i6cy7ikeyorbnl0ng2hh3icyvq", "token": "ng2hh3i6cy7ikeyorbnl0ng2hh3icyvq", "response_id": "ng2hh3i6cy7ikeyorbnl0ng2hh3icyvq", "response_type": "completed", "landed_at": "2022-11-09T06:16:08Z", "submitted_at": "2022-11-09T06:16:10Z", "metadata": { "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/VWO7mLtl", "network_id": "2be9dd4bab", "browser": "default" }, "hidden": {}, "calculated": { "score": 0 }, "answers": [ { "field": { "id": "ZdzF0rrvsVdB", "type": "multiple_choice", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM" }, "type": "choices", "choices": { "ids": [ "nLpt4rvNjFB3", "wS5FKMUnMgqR", "uvmLX80Loava", "7ubtgCrW2meb", "51q0g4fTFtYc", "vi3iwtpETqlb", "iI0hDpta14Kk" ], "refs": [ "01GHC6KQ5Y155J0F550BGYYS1A", "2ac396a3-1b8e-4e56-b36d-d1f27c1b834d", "8fffd3a8-1e96-421d-a605-a7029bd55e97", "17403cc9-74cd-49d1-856a-be6662b3b497", "3a1295b4-97b9-4986-9c37-f1af1d72501d", "54edf52a-c9c7-4bc4-a5a6-bd86115f5adb", "e149c19f-8b61-4ff0-a17a-e9e65c3a8fee" ], "labels": [ "Dec 12-16", "Jan 16-20", "Jan 22-26", "Jan30 - Feb3", "Feb 6 - 11", "Feb 13-17", "Feb 19-24" ] } } ], "form_id": "VWO7mLtl" }, "emitted_at": 1687522222826} +{"stream": "responses", "data": { "landing_id": "r4epuzzxlonggr4epp07wenb3a58sm6h", "token": "r4epuzzxlonggr4epp07wenb3a58sm6h", "response_id": "r4epuzzxlonggr4epp07wenb3a58sm6h", "response_type": "completed", "landed_at": "2022-11-09T14:33:46Z", "submitted_at": "2022-11-09T14:35:10Z", "metadata": { "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/VWO7mLtl", "network_id": "7e338d2504", "browser": "default" }, "hidden": {}, "calculated": { "score": 0 }, "answers": [ { "field": { "id": "ZdzF0rrvsVdB", "type": "multiple_choice", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM" }, "type": "choices", "choices": { "ids": [ "51q0g4fTFtYc", "vi3iwtpETqlb", "iI0hDpta14Kk" ], "refs": [ "3a1295b4-97b9-4986-9c37-f1af1d72501d", "54edf52a-c9c7-4bc4-a5a6-bd86115f5adb", "e149c19f-8b61-4ff0-a17a-e9e65c3a8fee" ], "labels": [ "Feb 6 - 11", "Feb 13-17", "Feb 19-24" ] } } ], "form_id": "VWO7mLtl" }, "emitted_at": 1687522222827} +{"stream": "responses", "data": { "landing_id": "trdyqvm2wmf9b0dhoostrdtugo8fdcoa", "token": "trdyqvm2wmf9b0dhoostrdtugo8fdcoa", "response_id": "trdyqvm2wmf9b0dhoostrdtugo8fdcoa", "response_type": "completed", "landed_at": "2022-11-15T02:36:40Z", "submitted_at": "2022-11-15T02:39:52Z", "metadata": { "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36", "platform": "other", "referer": "https://xe03v5buli4.typeform.com/to/VWO7mLtl", "network_id": "8284380108", "browser": "default" }, "hidden": {}, "calculated": { "score": 0 }, "answers": [ { "field": { "id": "ZdzF0rrvsVdB", "type": "multiple_choice", "ref": "01GHC6KQ5Y6S9ZQH5CHKZPT1RM" }, "type": "choices", "choices": { "ids": [ "jQHb3mqslOsZ", "wS5FKMUnMgqR", "uvmLX80Loava", "7ubtgCrW2meb", "51q0g4fTFtYc", "iI0hDpta14Kk" ], "refs": [ "1c392fa3-e693-49fe-b334-3a5cddc1db6f", "2ac396a3-1b8e-4e56-b36d-d1f27c1b834d", "8fffd3a8-1e96-421d-a605-a7029bd55e97", "17403cc9-74cd-49d1-856a-be6662b3b497", "3a1295b4-97b9-4986-9c37-f1af1d72501d", "e149c19f-8b61-4ff0-a17a-e9e65c3a8fee" ], "labels": [ "Jan 9-14", "Jan 16-20", "Jan 22-26", "Jan30 - Feb3", "Feb 6 - 11", "Feb 19-24" ] } } ], "form_id": "VWO7mLtl" }, "emitted_at": 1687522223051} {"stream":"workspaces","data":{"default":false,"forms":{"count":4,"href":"https://api.typeform.com/forms?workspace_id=sDaAqs"},"id":"sDaAqs","name":"My workspace","account_id":"01F8CZR731ZFGBGBEKHMFD5J6Y","self":{"href":"https://api.typeform.com/workspaces/sDaAqs"},"shared":false},"emitted_at":1673035162976} {"stream":"images","data":{"id":"JD76sXLuakwc","src":"https://images.typeform.com/images/JD76sXLuakwc","file_name":"1200x1200 logo.png","width":1200,"height":1200,"media_type":"image/png","has_alpha":true,"avg_color":"d1cbfe"},"emitted_at":1673035163945} {"stream":"images","data":{"id":"D7r8BDHAa5ac","src":"https://images.typeform.com/images/D7r8BDHAa5ac","file_name":"1200x1200 logo.png","width":1200,"height":1200,"media_type":"image/png","has_alpha":true,"avg_color":"d1cbfe"},"emitted_at":1673035163946} diff --git a/airbyte-integrations/connectors/source-typeform/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-typeform/integration_tests/sample_config.json new file mode 100644 index 000000000000..ecc4913b84c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-typeform/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "fix-me": "TODO" +} diff --git a/airbyte-integrations/connectors/source-typeform/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-typeform/integration_tests/sample_state.json new file mode 100644 index 000000000000..129af8dec641 --- /dev/null +++ b/airbyte-integrations/connectors/source-typeform/integration_tests/sample_state.json @@ -0,0 +1,16 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_descriptor": { "name": "responses" }, + "stream_state": { + "states": [ + { + "partition": { "form_id": "SdMKQYkv" }, + "cursor": { "submitted_at": "2021-09-04T16:39:47Z" } + } + ] + } + } + } +] diff --git a/airbyte-integrations/connectors/source-typeform/main.py b/airbyte-integrations/connectors/source-typeform/main.py index 332857e87cdd..126dc556ff7d 100644 --- a/airbyte-integrations/connectors/source-typeform/main.py +++ b/airbyte-integrations/connectors/source-typeform/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_typeform import SourceTypeform +from source_typeform.run import run if __name__ == "__main__": - source = SourceTypeform() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-typeform/metadata.yaml b/airbyte-integrations/connectors/source-typeform/metadata.yaml index 8c8904a2e6a7..f8996479b011 100644 --- a/airbyte-integrations/connectors/source-typeform/metadata.yaml +++ b/airbyte-integrations/connectors/source-typeform/metadata.yaml @@ -1,12 +1,18 @@ data: + ab_internal: + ql: 200 + sl: 200 allowedHosts: hosts: - api.typeform.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: e7eff203-90bf-43e5-a240-19ea3056c474 - dockerImageTag: 1.0.0 + dockerImageTag: 1.2.3 dockerRepository: airbyte/source-typeform + documentationUrl: https://docs.airbyte.com/integrations/sources/typeform githubIssueLabel: source-typeform icon: typeform.svg license: MIT @@ -14,15 +20,21 @@ data: registries: cloud: enabled: true - dockerImageTag: 1.0.0 oss: enabled: true + releaseDate: 2021-07-10 releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/typeform - tags: - - language:python - ab_internal: - sl: 200 - ql: 400 + releases: + breakingChanges: + 1.1.0: + message: + This version migrates the Typeform connector to the low-code framework + for greater maintainability. This introduces a breaking change to the state + format for the `responses` stream. If you are using the incremental sync + mode for this stream, you will need to reset affected connections after + upgrading to prevent sync failures. + upgradeDeadline: "2023-09-25" supportLevel: certified + tags: + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-typeform/setup.py b/airbyte-integrations/connectors/source-typeform/setup.py index 720f7ea64da1..09b119b93869 100644 --- a/airbyte-integrations/connectors/source-typeform/setup.py +++ b/airbyte-integrations/connectors/source-typeform/setup.py @@ -5,20 +5,23 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = [ - "airbyte-cdk", -] +MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1"] -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6", "requests_mock~=1.8"] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( + entry_points={ + "console_scripts": [ + "source-typeform=source_typeform.run:run", + ], + }, name="source_typeform", description="Source implementation for Typeform.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, - package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, }, diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/__init__.py b/airbyte-integrations/connectors/source-typeform/source_typeform/__init__.py index 4398c5361697..2cf08f1a5d12 100644 --- a/airbyte-integrations/connectors/source-typeform/source_typeform/__init__.py +++ b/airbyte-integrations/connectors/source-typeform/source_typeform/__init__.py @@ -1,29 +1,7 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -""" -MIT License -Copyright (c) 2020 Airbyte - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -""" from .source import SourceTypeform diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/components.py b/airbyte-integrations/connectors/source-typeform/source_typeform/components.py new file mode 100644 index 000000000000..d7e12b70396d --- /dev/null +++ b/airbyte-integrations/connectors/source-typeform/source_typeform/components.py @@ -0,0 +1,39 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import dataclass +from typing import Any, Iterable, Mapping + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator +from airbyte_cdk.sources.declarative.auth.oauth import DeclarativeSingleUseRefreshTokenOauth2Authenticator +from airbyte_cdk.sources.declarative.auth.token import BearerAuthenticator +from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import SubstreamPartitionRouter +from airbyte_cdk.sources.declarative.types import StreamSlice + + +@dataclass +class TypeformAuthenticator(DeclarativeAuthenticator): + config: Mapping[str, Any] + token_auth: BearerAuthenticator + oauth2: DeclarativeSingleUseRefreshTokenOauth2Authenticator + + def __new__(cls, token_auth, oauth2, config, *args, **kwargs): + return token_auth if config["credentials"]["auth_type"] == "access_token" else oauth2 + + +@dataclass +class FormIdPartitionRouter(SubstreamPartitionRouter): + def stream_slices(self) -> Iterable[StreamSlice]: + form_ids = self.config.get("form_ids", []) + + if form_ids: + for item in form_ids: + yield {"form_id": item} + else: + for parent_stream_config in self.parent_stream_configs: + for item in parent_stream_config.stream.read_records(sync_mode=SyncMode.full_refresh): + yield {"form_id": item["id"]} + + yield from [] diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/manifest.yaml b/airbyte-integrations/connectors/source-typeform/source_typeform/manifest.yaml new file mode 100644 index 000000000000..3fa818405d3f --- /dev/null +++ b/airbyte-integrations/connectors/source-typeform/source_typeform/manifest.yaml @@ -0,0 +1,334 @@ +version: 0.50.0 +type: DeclarativeSource +check: + type: CheckStream + stream_names: + - forms + +definitions: + items_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - items + no_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + token_auth: + type: BearerAuthenticator + api_token: "{{ config['credentials']['access_token'] }}" + oauth2: + type: OAuthAuthenticator + token_refresh_endpoint: https://api.typeform.com/oauth/token + client_id: "{{ config['credentials']['client_id'] }}" + client_secret: "{{ config['credentials']['client_secret'] }}" + refresh_token: "{{ config['credentials']['refresh_token'] }}" + refresh_token_updater: {} + requester: + type: HttpRequester + url_base: https://api.typeform.com/ + http_method: GET + request_parameters: {} + request_headers: {} + error_handler: + type: CompositeErrorHandler + error_handlers: + - type: DefaultErrorHandler + response_filters: + - http_codes: [499] + action: FAIL + error_message: "Could not complete the stream: Source Typeform has been waiting for too long for a response from Typeform API. Please try again later." + authenticator: + class_name: source_typeform.components.TypeformAuthenticator + token_auth: "#/definitions/token_auth" + oauth2: "#/definitions/oauth2" + request_body_json: {} + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: page + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + pagination_strategy: + type: PageIncrement + page_size: 200 + start_from_page: 1 + paginated_stream: + type: DeclarativeStream + primary_key: id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: "{{ parameters.path }}" + record_selector: + $ref: "#/definitions/items_selector" + paginator: + $ref: "#/definitions/paginator" + trim_forms_stream: + $ref: "#/definitions/paginated_stream" + name: trim_forms + $parameters: + path: forms + form_id_partition_router: + - type: CustomPartitionRouter + class_name: source_typeform.components.FormIdPartitionRouter + parent_stream_configs: + - type: ParentStreamConfig + parent_key: id + partition_field: form_id + stream: + $ref: "#/definitions/trim_forms_stream" + + forms_stream: + type: DeclarativeStream + name: forms + primary_key: id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: forms/{{ stream_partition.form_id }} + record_selector: + $ref: "#/definitions/no_selector" + partition_router: + $ref: "#/definitions/form_id_partition_router" + + responses_stream: + type: DeclarativeStream + name: responses + primary_key: response_id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: forms/{{ stream_partition.form_id }}/responses + record_selector: + $ref: "#/definitions/items_selector" + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + field_name: page_size + inject_into: request_parameter + page_token_option: + type: RequestOption + field_name: before + inject_into: request_parameter + pagination_strategy: + type: CursorPagination + cursor_value: "{{ last_records[-1]['token'] }}" + stop_condition: "{{ not response['total_items'] }}" + page_size: 1000 + partition_router: + $ref: "#/definitions/form_id_partition_router" + transformations: + - type: AddFields + fields: + - path: + - form_id + value: "{{ stream_partition.form_id }}" + incremental_sync: + type: DatetimeBasedCursor + cursor_field: submitted_at + cursor_datetime_formats: + - "%Y-%m-%dT%H:%M:%SZ" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_datetime: + type: MinMaxDatetime + datetime: >- + {{ format_datetime((config.start_date if config.start_date else + now_utc() - duration('P1Y')), '%Y-%m-%dT%H:%M:%SZ') }} + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + start_time_option: + type: RequestOption + field_name: since + inject_into: request_parameter + end_datetime: + type: MinMaxDatetime + datetime: "{{ now_utc().strftime('%Y-%m-%dT%H:%M:%SZ') }}" + datetime_format: "%Y-%m-%dT%H:%M:%SZ" + + webhooks_stream: + type: DeclarativeStream + name: webhooks + primary_key: id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + path: forms/{{ stream_partition.form_id }}/webhooks + record_selector: + $ref: "#/definitions/items_selector" + partition_router: + $ref: "#/definitions/form_id_partition_router" + workspaces_stream: + $ref: "#/definitions/paginated_stream" + name: workspaces + $parameters: + path: workspaces + images_stream: + type: DeclarativeStream + name: images + primary_key: id + retriever: + type: SimpleRetriever + requester: + $ref: "#/definitions/requester" + record_selector: + $ref: "#/definitions/no_selector" + $parameters: + path: images + themes_stream: + $ref: "#/definitions/paginated_stream" + name: themes + $parameters: + path: themes + +streams: + - "#/definitions/forms_stream" + - "#/definitions/responses_stream" + - "#/definitions/webhooks_stream" + - "#/definitions/workspaces_stream" + - "#/definitions/images_stream" + - "#/definitions/themes_stream" +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/typeform + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + type: object + additionalProperties: true + required: + - credentials + properties: + credentials: + title: Authorization Method + type: object + order: 0 + oneOf: + - type: object + title: OAuth2.0 + required: + - client_id + - client_secret + - refresh_token + - access_token + - token_expiry_date + properties: + auth_type: + type: string + const: oauth2.0 + client_id: + type: string + description: The Client ID of the Typeform developer application. + airbyte_secret: true + client_secret: + type: string + description: The Client Secret the Typeform developer application. + airbyte_secret: true + access_token: + type: string + description: Access Token for making authenticated requests. + airbyte_secret: true + token_expiry_date: + type: string + description: The date-time when the access token should be refreshed. + format: date-time + refresh_token: + type: string + description: The key to refresh the expired access_token. + airbyte_secret: true + - title: Private Token + type: object + required: + - access_token + properties: + auth_type: + type: string + const: access_token + access_token: + type: string + title: Private Token + description: + Log into your Typeform account and then generate a personal + Access Token. + airbyte_secret: true + start_date: + type: string + title: Start Date + description: + The date from which you'd like to replicate data for Typeform API, + in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will + be replicated. + examples: + - "2021-03-01T00:00:00Z" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + order: 2 + format: date-time + form_ids: + title: Form IDs to replicate + description: + When this parameter is set, the connector will replicate data only + from the input forms. Otherwise, all forms in your Typeform account will be + replicated. You can find form IDs in your form URLs. For example, in the URL + "https://mysite.typeform.com/to/u6nXL7" the form_id is u6nXL7. You can find + form URLs on Share panel + type: array + items: + type: string + uniqueItems: true + order: 3 + advanced_auth: + auth_flow_type: oauth2.0 + predicate_key: + - credentials + - auth_type + predicate_value: oauth2.0 + oauth_config_specification: + complete_oauth_output_specification: + type: object + properties: + access_token: + type: string + path_in_connector_config: + - credentials + - access_token + refresh_token: + type: string + path_in_connector_config: + - credentials + - refresh_token + token_expiry_date: + type: string + format: date-time + path_in_connector_config: + - credentials + - token_expiry_date + complete_oauth_server_input_specification: + type: object + properties: + client_id: + type: string + client_secret: + type: string + complete_oauth_server_output_specification: + type: object + properties: + client_id: + type: string + path_in_connector_config: + - credentials + - client_id + client_secret: + type: string + path_in_connector_config: + - credentials + - client_secret diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/run.py b/airbyte-integrations/connectors/source-typeform/source_typeform/run.py new file mode 100644 index 000000000000..2ebf804b4940 --- /dev/null +++ b/airbyte-integrations/connectors/source-typeform/source_typeform/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_typeform import SourceTypeform + + +def run(): + source = SourceTypeform() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/responses.json b/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/responses.json index 6097f2e5d155..ee18a6040efe 100644 --- a/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/responses.json +++ b/airbyte-integrations/connectors/source-typeform/source_typeform/schemas/responses.json @@ -4,6 +4,9 @@ "response_id": { "type": ["null", "string"] }, + "response_type": { + "type": ["null", "string"] + }, "landed_at": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/source.py b/airbyte-integrations/connectors/source-typeform/source_typeform/source.py index 17c48ae7b27a..58ec8391d7d7 100644 --- a/airbyte-integrations/connectors/source-typeform/source_typeform/source.py +++ b/airbyte-integrations/connectors/source-typeform/source_typeform/source.py @@ -2,303 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import urllib.parse as urlparse -from abc import ABC, abstractmethod -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple -from urllib.parse import parse_qs +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import pendulum -import requests -from airbyte_cdk import AirbyteLogger -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -from airbyte_cdk.sources.streams.http.requests_native_auth.oauth import SingleUseRefreshTokenOauth2Authenticator -from pendulum.datetime import DateTime -from requests.auth import AuthBase +WARNING: Do not modify this file. +""" -class TypeformStream(HttpStream, ABC): - url_base = "https://api.typeform.com/" - # maximum number of entities in API response per single page - limit: int = 200 - date_format: str = "YYYY-MM-DDTHH:mm:ss[Z]" - - def __init__(self, **kwargs: Mapping[str, Any]): - super().__init__(authenticator=kwargs["authenticator"]) - self.config: Mapping[str, Any] = kwargs - # if start_date is not provided during setup, use date from a year ago instead - self.start_date: DateTime = pendulum.today().subtract(years=1) - if kwargs.get("start_date"): - self.start_date: DateTime = pendulum.from_format(kwargs["start_date"], self.date_format) - - # changes page limit, this param is using for development and debugging - if kwargs.get("page_size"): - self.limit = kwargs.get("page_size") - - def next_page_token(self, response: requests.Response) -> Optional[Any]: - return None - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield from response.json()["items"] - - -class PaginatedStream(TypeformStream): - def next_page_token(self, response: requests.Response) -> Optional[Any]: - page = self.get_current_page_token(response.url) - # stop pagination if current page equals to total pages - return None if not page or response.json()["page_count"] <= page else page + 1 - - def get_current_page_token(self, url: str) -> Optional[int]: - """ - Fetches page query parameter from URL - """ - parsed = urlparse.urlparse(url) - page = parse_qs(parsed.query).get("page") - return int(page[0]) if page else None - - def request_params(self, next_page_token: Optional[Any] = None, **kwargs) -> MutableMapping[str, Any]: - params = {"page_size": self.limit} - params["page"] = next_page_token or 1 - return params - - -class TrimForms(PaginatedStream): - """ - This stream is responsible for fetching list of from_id(s) which required to process data from Forms and Responses. - API doc: https://developer.typeform.com/create/reference/retrieve-forms/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "forms" - - -class TrimFormsMixin: - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - form_ids = self.config.get("form_ids", []) - if form_ids: - for item in form_ids: - yield {"form_id": item} - else: - for item in TrimForms(**self.config).read_records(sync_mode=SyncMode.full_refresh): - yield {"form_id": item["id"]} - - yield from [] - - -class Forms(TrimFormsMixin, TypeformStream): - """ - This stream is responsible for detailed information about Form. - API doc: https://developer.typeform.com/create/reference/retrieve-form/ - """ - - primary_key = "id" - - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - return f"forms/{stream_slice['form_id']}" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield response.json() - - -class IncrementalTypeformStream(TypeformStream, ABC): - cursor_field: str = "submitted_at" - token_field: str = "token" - - @property - def limit(self): - return super().limit - - state_checkpoint_interval = limit - - @abstractmethod - def get_updated_state( - self, - current_stream_state: MutableMapping[str, Any], - latest_record: Mapping[str, Any], - ) -> Mapping[str, Any]: - pass - - def next_page_token(self, response: requests.Response) -> Optional[Any]: - items = response.json()["items"] - if items and len(items) == self.limit: - return items[-1][self.token_field] - return None - - -class Responses(TrimFormsMixin, IncrementalTypeformStream): - """ - This stream is responsible for fetching responses for particular form_id. - API doc: https://developer.typeform.com/responses/reference/retrieve-responses/ - """ - - primary_key = "response_id" - limit: int = 1000 - - def path(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> str: - return f"forms/{stream_slice['form_id']}/responses" - - def get_form_id(self, record: Mapping[str, Any]) -> Optional[str]: - """ - Fetches form id to which current record belongs. - """ - referer = record.get("metadata", {}).get("referer") - return urlparse.urlparse(referer).path.split("/")[-1] if referer else None - - def current_state_value_int(self, current_stream_state: MutableMapping[str, Any], form_id: str) -> int: - # state used to be stored as int, now we store it as str, so need to handle both cases - value = current_stream_state.get(form_id, {}).get(self.cursor_field, self.start_date.int_timestamp) - if isinstance(value, str): - value = pendulum.from_format(value, self.date_format).int_timestamp - return value - - def get_updated_state( - self, - current_stream_state: MutableMapping[str, Any], - latest_record: Mapping[str, Any], - ) -> Mapping[str, Any]: - form_id = self.get_form_id(latest_record) - if not form_id or not latest_record.get(self.cursor_field): - return current_stream_state - - current_stream_state[form_id] = current_stream_state.get(form_id, {}) - new_state_value = max( - pendulum.from_format(latest_record[self.cursor_field], self.date_format).int_timestamp, - self.current_state_value_int(current_stream_state, form_id), - ) - current_stream_state[form_id][self.cursor_field] = pendulum.from_timestamp(new_state_value).format(self.date_format) - return current_stream_state - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, any] = None, - next_page_token: Optional[Any] = None, - ) -> MutableMapping[str, Any]: - params = {"page_size": self.limit} - stream_state = stream_state or {} - - if not next_page_token: - # use state for first request in incremental sync - params["sort"] = "submitted_at,asc" - # start from last state or from start date - since = max(self.start_date.int_timestamp, self.current_state_value_int(stream_state, stream_slice["form_id"])) - if since: - params["since"] = pendulum.from_timestamp(since).format(self.date_format) - else: - # use response token for pagination after first request - # this approach allow to avoid data duplication within single sync - params["after"] = next_page_token - - return params - - def parse_response(self, response: requests.Response, stream_slice: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: - responses = response.json()["items"] - for response in responses: - response["form_id"] = stream_slice["form_id"] - return responses - - -class Webhooks(TrimFormsMixin, TypeformStream): - """ - This stream is responsible for fetching webhooks for particular form_id. - API doc: https://developer.typeform.com/webhooks/reference/retrieve-webhooks/ - """ - - primary_key = "id" - - def path(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> str: - return f"forms/{stream_slice['form_id']}/webhooks" - - -class Workspaces(PaginatedStream): - """ - This stream is responsible for fetching workspaces. - API doc: https://developer.typeform.com/create/reference/retrieve-workspaces/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "workspaces" - - -class Images(TypeformStream): - """ - This stream is responsible for fetching images. - API doc: https://developer.typeform.com/create/reference/retrieve-images-collection/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "images" - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - return response.json() - - -class Themes(PaginatedStream): - """ - This stream is responsible for fetching themes. - API doc: https://developer.typeform.com/create/reference/retrieve-themes/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "themes" - - -class SourceTypeform(AbstractSource): - def get_auth(self, config: MutableMapping) -> AuthBase: - credentials = config.get("credentials") - if credentials and credentials.get("access_token"): - return TokenAuthenticator(token=credentials["access_token"]) - return SingleUseRefreshTokenOauth2Authenticator(config, token_refresh_endpoint="https://api.typeform.com/oauth/token") - - def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, any]: - try: - form_ids = config.get("form_ids", []).copy() - auth = self.get_auth(config) - # verify if form inputted by user is valid - try: - url = urlparse.urljoin(TypeformStream.url_base, "me") - auth_headers = auth.get_auth_header() - session = requests.get(url, headers=auth_headers) - session.raise_for_status() - except Exception as e: - return False, f"Cannot authenticate, please verify token. Error: {e}" - if form_ids: - for form in form_ids: - try: - url = urlparse.urljoin(TypeformStream.url_base, f"forms/{form}") - response = requests.get(url, headers=auth_headers) - response.raise_for_status() - except Exception as e: - return ( - False, - f"Cannot find forms with ID: {form}. Please make sure they are valid form IDs and try again. Error: {e}", - ) - return True, None - else: - return True, None - - except Exception as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - auth = self.get_auth(config) - return [ - Forms(authenticator=auth, **config), - Responses(authenticator=auth, **config), - Webhooks(authenticator=auth, **config), - Workspaces(authenticator=auth, **config), - Images(authenticator=auth, **config), - Themes(authenticator=auth, **config), - ] +# Declarative Source +class SourceTypeform(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-typeform/source_typeform/spec.json b/airbyte-integrations/connectors/source-typeform/source_typeform/spec.json deleted file mode 100644 index ea424a86dc97..000000000000 --- a/airbyte-integrations/connectors/source-typeform/source_typeform/spec.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.com/integrations/sources/typeform", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Source Typeform Spec", - "type": "object", - "required": ["credentials"], - "additionalProperties": true, - "properties": { - "credentials": { - "title": "Authorization Method", - "type": "object", - "order": 0, - "oneOf": [ - { - "type": "object", - "title": "OAuth2.0", - "required": [ - "client_id", - "client_secret", - "refresh_token", - "access_token", - "token_expiry_date" - ], - "properties": { - "auth_type": { - "type": "string", - "const": "oauth2.0" - }, - "client_id": { - "type": "string", - "description": "The Client ID of the Typeform developer application.", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "description": "The Client Secret the Typeform developer application.", - "airbyte_secret": true - }, - "access_token": { - "type": "string", - "description": "Access Token for making authenticated requests.", - "airbyte_secret": true - }, - "token_expiry_date": { - "type": "string", - "description": "The date-time when the access token should be refreshed.", - "format": "date-time" - }, - "refresh_token": { - "type": "string", - "description": "The key to refresh the expired access_token.", - "airbyte_secret": true - } - } - }, - { - "title": "Private Token", - "type": "object", - "required": ["access_token"], - "properties": { - "auth_type": { - "type": "string", - "const": "access_token" - }, - "access_token": { - "type": "string", - "title": "Private Token", - "description": "Log into your Typeform account and then generate a personal Access Token.", - "airbyte_secret": true - } - } - } - ] - }, - "start_date": { - "type": "string", - "title": "Start Date", - "description": "The date from which you'd like to replicate data for Typeform API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", - "examples": ["2021-03-01T00:00:00Z"], - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "order": 2, - "format": "date-time" - }, - "form_ids": { - "title": "Form IDs to replicate", - "description": "When this parameter is set, the connector will replicate data only from the input forms. Otherwise, all forms in your Typeform account will be replicated. You can find form IDs in your form URLs. For example, in the URL \"https://mysite.typeform.com/to/u6nXL7\" the form_id is u6nXL7. You can find form URLs on Share panel", - "type": "array", - "items": { - "type": "string" - }, - "uniqueItems": true, - "order": 3 - } - } - }, - "advanced_auth": { - "auth_flow_type": "oauth2.0", - "predicate_key": ["credentials", "auth_type"], - "predicate_value": "oauth2.0", - "oauth_config_specification": { - "complete_oauth_output_specification": { - "type": "object", - "properties": { - "access_token": { - "type": "string", - "path_in_connector_config": ["credentials", "access_token"] - }, - "refresh_token": { - "type": "string", - "path_in_connector_config": ["credentials", "refresh_token"] - }, - "token_expiry_date": { - "type": "string", - "format": "date-time", - "path_in_connector_config": ["credentials", "token_expiry_date"] - } - } - }, - "complete_oauth_server_input_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string" - }, - "client_secret": { - "type": "string" - } - } - }, - "complete_oauth_server_output_specification": { - "type": "object", - "properties": { - "client_id": { - "type": "string", - "path_in_connector_config": ["credentials", "client_id"] - }, - "client_secret": { - "type": "string", - "path_in_connector_config": ["credentials", "client_secret"] - } - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-typeform/unit_tests/__init__.py b/airbyte-integrations/connectors/source-typeform/unit_tests/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-typeform/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-typeform/unit_tests/conftest.py b/airbyte-integrations/connectors/source-typeform/unit_tests/conftest.py deleted file mode 100644 index ef5db854a4a9..000000000000 --- a/airbyte-integrations/connectors/source-typeform/unit_tests/conftest.py +++ /dev/null @@ -1,547 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import pytest - - -@pytest.fixture -def config(): - return {"start_date": "2020-01-01T00:00:00Z", "credentials": {"access_token": "7607999ef26581e81726777b7b79f20e70e75602"}, "form_ids": ["u6nXL7", "k9xNV4"]} - - -@pytest.fixture -def config_without_forms(): - return {"start_date": "2020-01-01T00:00:00Z", "credentials":{"access_token": "7607999ef26581e81726777b7b79f20e70e75602"}} - - -@pytest.fixture -def form_response(): - return setup_response( - 200, - { - "id": "id", - "title": "title", - "language": "en", - "fields": [{}], - "hidden": ["string"], - "variables": {"score": 0, "price": 0}, - "welcome_screens": [ - { - "ref": "nice-readable-welcome-ref", - "title": "Welcome Title", - "properties": {"description": "Cool description for the welcome", "show_button": True, "button_text": "start"}, - "attachment": { - "type": "image", - "href": { - "image": {"value": "https://images.typeform.com/images/4bcd3"}, - "Pexels": {"value": "https://www.pexels.com/video/people-traveling-in-the-desert-1739011"}, - "Vimeo": {"value": "https://vimeo.com/245714980"}, - "YouTube": {"value": "https://www.youtube.com/watch?v=cGk3tZIIpXE"}, - }, - "scale": 0, - "properties": {"description": "description"}, - }, - "layout": { - "type": "float", - "placement": "left", - "attachment": { - "type": "image", - "href": { - "image": {"value": "https://images.typeform.com/images/4bcd3"}, - "Pexels": {"value": "https://www.pexels.com/video/people-traveling-in-the-desert-1739011"}, - "Vimeo": {"value": "https://vimeo.com/245714980"}, - "YouTube": {"value": "https://www.youtube.com/watch?v=cGk3tZIIpXE"}, - }, - "scale": 0, - "properties": {"brightness": 0, "description": "description", "focal_point": {"x": 0, "y": 0}}, - }, - }, - } - ], - "thankyou_screens": [ - { - "ref": "nice-readable-thank-you-ref", - "title": "Thank you Title", - "properties": { - "show_button": True, - "button_text": "start", - "button_mode": "redirect", - "redirect_url": "https://www.typeform.com", - "share_icons": True, - }, - "attachment": { - "type": "image", - "href": { - "image": {"value": "https://images.typeform.com/images/4bcd3"}, - "Pexels": {"value": "https://www.pexels.com/video/people-traveling-in-the-desert-1739011"}, - "Vimeo": {"value": "https://vimeo.com/245714980"}, - "YouTube": {"value": "https://www.youtube.com/watch?v=cGk3tZIIpXE"}, - }, - "scale": 0, - "properties": {"description": "description"}, - }, - "layout": { - "type": "float", - "placement": "left", - "attachment": { - "type": "image", - "href": { - "image": {"value": "https://images.typeform.com/images/4bcd3"}, - "Pexels": {"value": "https://www.pexels.com/video/people-traveling-in-the-desert-1739011"}, - "Vimeo": {"value": "https://vimeo.com/245714980"}, - "YouTube": {"value": "https://www.youtube.com/watch?v=cGk3tZIIpXE"}, - }, - "scale": 0, - "properties": {"brightness": 0, "description": "description", "focal_point": {"x": 0, "y": 0}}, - }, - }, - } - ], - "logic": [ - { - "type": "type", - "ref": "ref", - "actions": [ - { - "action": "action", - "details": { - "to": {"type": "type", "value": "value"}, - "target": {"type": "type", "value": "value"}, - "value": {"type": "type", "value": 0}, - }, - "condition": {"op": "op", "vars": [{"type": "type", "value": {}}]}, - } - ], - } - ], - "theme": {"href": "https://api.typeform.com/themes/Fs24as"}, - "workspace": {"href": "https://api.typeform.com/workspaces/Aw33bz"}, - "_links": {"display": "https://subdomain.typeform.com/to/abc123"}, - "settings": { - "language": "language", - "is_public": True, - "progress_bar": "proportion", - "show_progress_bar": True, - "show_typeform_branding": True, - "show_time_to_complete": True, - "hide_navigation": True, - "meta": {"title": "title", "allow_indexing": True, "description": "description", "image": {"href": "href"}}, - "redirect_after_submit_url": "redirect_after_submit_url", - "google_analytics": "google_analytics", - "facebook_pixel": "facebook_pixel", - "google_tag_manager": "google_tag_manager", - }, - "cui_settings": { - "avatar": "https://images.typeform.com/images/4bcd3", - "is_typing_emulation_disabled": True, - "typing_emulation_speed": "fast", - }, - }, - ) - - -@pytest.fixture -def forms_response(): - return setup_response(200, {"total_items": 2, "page_count": 1, "items": [{"id": "u6nXL7"}, {"id": "k9xNV4"}]}) - - -@pytest.fixture -def response_response(): - return setup_response( - 200, - { - "items": [ - { - "answers": [ - { - "field": {"id": "hVONkQcnSNRj", "ref": "my_custom_dropdown_reference", "type": "dropdown"}, - "text": "Job opportunities", - "type": "text", - }, - { - "boolean": False, - "field": {"id": "RUqkXSeXBXSd", "ref": "my_custom_yes_no_reference", "type": "yes_no"}, - "type": "boolean", - }, - { - "boolean": True, - "field": {"id": "gFFf3xAkJKsr", "ref": "my_custom_legal_reference", "type": "legal"}, - "type": "boolean", - }, - { - "field": {"id": "JwWggjAKtOkA", "ref": "my_custom_short_text_reference", "type": "short_text"}, - "text": "Lian", - "type": "text", - }, - { - "email": "lian1078@other.com", - "field": {"id": "SMEUb7VJz92Q", "ref": "my_custom_email_reference", "type": "email"}, - "type": "email", - }, - { - "field": {"id": "pn48RmPazVdM", "ref": "my_custom_number_reference", "type": "number"}, - "number": 1, - "type": "number", - }, - { - "field": {"id": "Q7M2XAwY04dW", "ref": "my_custom_number2_reference", "type": "number"}, - "number": 1, - "type": "number", - }, - { - "field": {"id": "WOTdC00F8A3h", "ref": "my_custom_rating_reference", "type": "rating"}, - "number": 3, - "type": "number", - }, - { - "field": {"id": "DlXFaesGBpoF", "ref": "my_custom_long_text_reference", "type": "long_text"}, - "text": "It's a big, busy city. I moved here for a job, but I like it, so I am planning to stay. I have made good friends here.", - "type": "text", - }, - { - "field": {"id": "NRsxU591jIW9", "ref": "my_custom_opinion_scale_reference", "type": "opinion_scale"}, - "number": 1, - "type": "number", - }, - { - "choices": {"labels": ["New York", "Tokyo"]}, - "field": {"id": "PNe8ZKBK8C2Q", "ref": "my_custom_picture_choice_reference", "type": "picture_choice"}, - "type": "choices", - }, - { - "date": "2012-03-20T00:00:00Z", - "field": {"id": "KoJxDM3c6x8h", "ref": "my_custom_date_reference", "type": "date"}, - "type": "date", - }, - { - "choice": {"label": "A friend's experience in Sydney"}, - "field": {"id": "ceIXxpbP3t2q", "ref": "my_custom_multiple_choice_reference", "type": "multiple_choice"}, - "type": "choice", - }, - { - "choices": {"labels": ["New York", "Tokyo"]}, - "field": {"id": "abISxvbD5t1p", "ref": "my_custom_ranking_reference", "type": "ranking"}, - "type": "choices", - }, - { - "choice": {"label": "Tokyo"}, - "field": {"id": "k6TP9oLGgHjl", "ref": "my_custom_multiple_choice2_reference", "type": "multiple_choice"}, - "type": "choice", - }, - ], - "calculated": {"score": 2}, - "hidden": {}, - "landed_at": "2017-09-14T22:33:59Z", - "landing_id": "21085286190ffad1248d17c4135ee56f", - "metadata": { - "browser": "default", - "network_id": "responsdent_network_id", - "platform": "other", - "referer": "https://user_id.typeform.com/to/lR6F4j", - "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8", - }, - "response_id": "21085286190ffad1248d17c4135ee56f", - "submitted_at": "2017-09-14T22:38:22Z", - "token": "test21085286190ffad1248d17c4135ee56f", - "variables": [{"key": "score", "number": 2, "type": "number"}, {"key": "name", "text": "typeform", "type": "text"}], - }, - { - "answers": [ - { - "choice": {"label": "New York"}, - "field": {"id": "k6TP9oLGgHjl", "ref": "my_custom_multiple_choice2_reference", "type": "multiple_choice"}, - "type": "choice", - }, - { - "field": {"id": "X4BgU2f1K6tG", "ref": "my_custom_file_upload_reference", "type": "file_upload"}, - "file_url": "https://api.typeform.com/forms/lT9Z2j/responses/7f46165474d11ee5836777d85df2cdab/fields/X4BgU2f1K6tG/files/afd8258fd453-aerial_view_rural_city_latvia_valmiera_urban_district_48132860.jpg", - "type": "file_url", - }, - { - "choice": {"label": "Other"}, - "field": {"id": "ceIXxpbP3t2q", "ref": "my_custom_multiple_choice_reference", "type": "multiple_choice"}, - "type": "choice", - }, - { - "field": {"id": "hVONkQcnSNRj", "ref": "my_custom_dropdown_reference", "type": "dropdown"}, - "text": "Cost of living", - "type": "text", - }, - { - "field": {"id": "JwWggjAKtOkA", "ref": "my_custom_short_text_reference", "type": "short_text"}, - "text": "Sarah", - "type": "text", - }, - { - "boolean": True, - "field": {"id": "RUqkXSeXBXSd", "ref": "my_custom_yes_no_reference", "type": "yes_no"}, - "type": "boolean", - }, - { - "field": {"id": "Fep7sEoBsnvC", "ref": "my_custom_long_text_reference", "type": "long_text"}, - "text": "I read a magazine article about traveling to Sydney", - "type": "text", - }, - { - "boolean": True, - "field": {"id": "gFFf3xAkJKsr", "ref": "my_custom_legal_reference", "type": "legal"}, - "type": "boolean", - }, - { - "field": {"id": "BFcpoPU5yJPM", "ref": "my_custom_short_text_reference", "type": "short_text"}, - "text": "San Francisco", - "type": "text", - }, - { - "email": "sarahbsmith@example.com", - "field": {"id": "SMEUb7VJz92Q", "ref": "my_custom_rmail_reference", "type": "email"}, - "type": "email", - }, - { - "field": {"id": "pn48RmPazVdM", "ref": "my_custom_number_reference", "type": "number"}, - "number": 1, - "type": "number", - }, - { - "field": {"id": "WOTdC00F8A3h", "ref": "my_custom_rating_reference", "type": "rating"}, - "number": 3, - "type": "number", - }, - { - "field": {"id": "Q7M2XAwY04dW", "ref": "my_custom_number2_reference", "type": "number"}, - "number": 3, - "type": "number", - }, - { - "field": {"id": "DlXFaesGBpoF", "ref": "my_custom_long_text_reference", "type": "long_text"}, - "text": "It's a rural area. Very quiet. There are a lot of farms...farming is the major industry here.", - "type": "text", - }, - { - "field": {"id": "NRsxU591jIW9", "ref": "my_custom_opinion_scale_reference", "type": "opinion_scale"}, - "number": 1, - "type": "number", - }, - { - "date": "2016-05-13T00:00:00Z", - "field": {"id": "KoJxDM3c6x8h", "ref": "my_custom_date_reference", "type": "date"}, - "type": "date", - }, - { - "choices": {"labels": ["London", "New York"]}, - "field": {"id": "PNe8ZKBK8C2Q", "ref": "my_custom_picture_choice_reference", "type": "picture_choice"}, - "type": "choices", - }, - ], - "calculated": {"score": 4}, - "hidden": {}, - "landed_at": "2017-09-14T22:27:38Z", - "landing_id": "610fc266478b41e4927945e20fe54ad2", - "metadata": { - "browser": "default", - "network_id": "responsdent_network_id", - "platform": "other", - "referer": "https://user_id.typeform.com/to/lR6F4j", - "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36", - }, - "submitted_at": "2017-09-14T22:33:56Z", - "token": "test610fc266478b41e4927945e20fe54ad2", - }, - { - "answers": [ - { - "boolean": False, - "field": {"id": "RUqkXSeXBXSd", "ref": "my_custom_yes_no_reference", "type": "yes_no"}, - "type": "boolean", - }, - { - "boolean": False, - "field": {"id": "gFFf3xAkJKsr", "ref": "my_custom_legal_reference", "type": "legal"}, - "type": "boolean", - }, - { - "field": {"id": "JwWggjAKtOkA", "ref": "my_custom_short_text_reference", "type": "short_text"}, - "text": "Paolo", - "type": "text", - }, - { - "field": {"id": "pn48RmPazVdM", "ref": "my_custom_number_reference", "type": "number"}, - "number": 5, - "type": "number", - }, - { - "field": {"id": "Q7M2XAwY04dW", "ref": "my_custom_number2_reference", "type": "number"}, - "number": 5, - "type": "number", - }, - { - "choices": {"labels": ["Barcelona", "Sydney"]}, - "field": {"id": "PNe8ZKBK8C2Q", "ref": "my_custom_picture_choice_reference", "type": "picture_choice"}, - "type": "choices", - }, - { - "field": {"id": "WOTdC00F8A3h", "ref": "my_custom_rating_reference", "type": "rating"}, - "number": 5, - "type": "number", - }, - { - "field": {"id": "DlXFaesGBpoF", "ref": "my_custom_long_text_reference", "type": "long_text"}, - "text": "I live in a medium-sized European city. It's not too crowded, and the people are nice. I like the weather. It's also easy to travel to many beautiful and interesting vacation destinations from where I live.", - "type": "text", - }, - { - "field": {"id": "NRsxU591jIW9", "ref": "my_custom_opinion_scale_reference", "type": "opinion_scale"}, - "number": 4, - "type": "number", - }, - { - "date": "1999-08-01T00:00:00Z", - "field": {"id": "KoJxDM3c6x8h", "ref": "my_custom_date_reference", "type": "date"}, - "type": "date", - }, - { - "choice": {"label": "Barcelona"}, - "field": {"id": "k6TP9oLGgHjl", "ref": "my_custom_multiple_choice_reference", "type": "multiple_choice"}, - "type": "choice", - }, - ], - "calculated": {"score": 10}, - "hidden": {}, - "landed_at": "2017-09-14T22:24:49Z", - "landing_id": "9ba5db11ec6c63d22f08aade805bd363", - "metadata": { - "browser": "default", - "network_id": "responsdent_network_id", - "platform": "other", - "referer": "https://user_id.typeform.com/to/lR6F4j", - "user_agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 10_2_1 like Mac OS X) AppleWebKit/602.4.6 (KHTML, like Gecko) Version/10.0 Mobile/14D27 Safari/602.1", - }, - "submitted_at": "2017-09-14T22:27:34Z", - "token": "9ba5db11ec6c63d22f08aade805bd363", - }, - { - "answers": [], - "calculated": {"score": 0}, - "hidden": {}, - "landed_at": "2017-09-15T09:09:30Z", - "landing_id": "5fcb3f9c162e1fcdaadff4405b741080", - "metadata": { - "browser": "default", - "network_id": "responsdent_network_id", - "platform": "other", - "referer": "https://user_id.typeform.com/to/lR6F4j", - "user_agent": "Mozilla/5.0 (Linux; Android 4.1.2; GT-N7000 Build/JZO54K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.91 Mobile Safari/537.36", - }, - "submitted_at": "0001-01-01T00:00:00Z", - "token": "test5fcb3f9c162e1fcdaadff4405b741080", - }, - ], - "page_count": 1, - "total_items": 4, - }, - ) - - -@pytest.fixture -def webhooks_response(): - return setup_response( - 200, - { - "items": [ - { - "created_at": "2016-11-21T12:23:28Z", - "enabled": True, - "form_id": "abc123", - "id": "yRtagDm8AT", - "tag": "phoenix", - "updated_at": "2016-11-21T12:23:28Z", - "url": "https://test.com", - "verify_ssl": True, - } - ] - }, - ) - - -@pytest.fixture -def images_response(): - return setup_response( - 200, [{"file_name": "file_name1", "id": "id1", "src": "src1"}, {"file_name": "file_name2", "id": "id2", "src": "src2"}] - ) - - -@pytest.fixture -def workspaces_response(): - return setup_response( - 200, - { - "items": [ - { - "forms": [{"count": 12, "href": "https://api.typeform.com/workspaces/a1b2c3/forms"}], - "id": "a1b2c3", - "name": "My Workspace1", - "self": [{"href": "https://api.typeform.com/workspaces/a1b2c3"}], - "shared": False, - }, - { - "forms": [{"count": 10, "href": "https://api.typeform.com/workspaces/a1b2c3/forms"}], - "id": "a1b2c3d4", - "name": "My Workspace2", - "self": [{"href": "https://api.typeform.com/workspaces/a1b2c3"}], - "shared": True, - }, - ], - "page_count": 1, - "total_items": 2, - }, - ) - - -@pytest.fixture -def themes_response(): - return setup_response( - 200, - { - "items": [ - { - "background": [{"brightness": 12, "href": "https://api.typeform.com/workspaces/a1b2c3/forms", "layout": "fullscreen"}], - "colors": [{"answer": "answer1", "background": "background1", "button": "button1", "question": "question1"}], - "fields": [{"alignment": "left", "font_size": "medium"}], - "font": "Helvetica Neue", - "has_transparent_button": True, - "id": "a1b2c3", - "name": "name1", - "screens": [{"alignment": "left", "font_size": "medium"}], - "visibility": "public", - }, - { - "background": [{"brightness": 13, "href": "https://api.typeform.com/workspaces/a1b2c3/forms", "layout": "fullscreen"}], - "colors": [{"answer": "answer2", "background": "background2", "button": "button2", "question": "question2"}], - "fields": [{"alignment": "left", "font_size": "medium"}], - "font": "Helvetica Neue", - "has_transparent_button": True, - "id": "a1b2c3", - "name": "name1", - "screens": [{"alignment": "left", "font_size": "medium"}], - "visibility": "public", - }, - ], - "page_count": 1, - "total_items": 2, - }, - ) - - -@pytest.fixture -def empty_response_ok(): - return setup_response(200, {}) - - -@pytest.fixture -def empty_response_bad(): - return setup_response(400, {}) - - -def setup_response(status, body): - return [{"json": body, "status_code": status}] diff --git a/airbyte-integrations/connectors/source-typeform/unit_tests/test_authenticator.py b/airbyte-integrations/connectors/source-typeform/unit_tests/test_authenticator.py new file mode 100644 index 000000000000..a841b1d266cf --- /dev/null +++ b/airbyte-integrations/connectors/source-typeform/unit_tests/test_authenticator.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from airbyte_cdk.sources.declarative.auth.oauth import DeclarativeSingleUseRefreshTokenOauth2Authenticator +from airbyte_cdk.sources.declarative.auth.token import BearerAuthenticator +from source_typeform.components import TypeformAuthenticator + + +def test_typeform_authenticator(): + config = {"credentials": {"auth_type": "access_token", "access_token": "access_token"}} + oauth_config = {"credentials": {"auth_type": "oauth2.0", "access_token": None, "client_id": "client_id", "client_secret": "client_secret"}} + + class TokenProvider: + def get_token(self) -> str: + return "test token" + + auth = TypeformAuthenticator( + token_auth=BearerAuthenticator(config=config, token_provider=TokenProvider(), parameters={}), + config=config, + oauth2=DeclarativeSingleUseRefreshTokenOauth2Authenticator(connector_config=oauth_config, token_refresh_endpoint="/new_token") + ) + assert isinstance(auth, BearerAuthenticator) + + oauth = TypeformAuthenticator( + token_auth=BearerAuthenticator(config=config, token_provider=TokenProvider(), parameters={}), + config=oauth_config, + oauth2=DeclarativeSingleUseRefreshTokenOauth2Authenticator(connector_config=oauth_config, token_refresh_endpoint="/new_token") + ) + assert isinstance(oauth, DeclarativeSingleUseRefreshTokenOauth2Authenticator) diff --git a/airbyte-integrations/connectors/source-typeform/unit_tests/test_form_id_partition_router.py b/airbyte-integrations/connectors/source-typeform/unit_tests/test_form_id_partition_router.py new file mode 100644 index 000000000000..c5c3f1508e76 --- /dev/null +++ b/airbyte-integrations/connectors/source-typeform/unit_tests/test_form_id_partition_router.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from unittest.mock import Mock + +import pytest +from airbyte_cdk.sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig +from source_typeform.components import FormIdPartitionRouter, TypeformAuthenticator + +# test cases as a list of tuples (form_ids, parent_stream_configs, expected_slices) +test_cases = [ + ( + # test form ids present in config + ["form_id_1", "form_id_2"], + [{"stream": Mock(read_records=Mock(return_value=[{"id": "form_id_3"}, {"id": "form_id_4"}]))}], + [{"form_id": "form_id_1"}, {"form_id": "form_id_2"}], + ), + ( + # test no form ids in config + [], + [ + {"stream": Mock(read_records=Mock(return_value=[{"id": "form_id_3"}, {"id": "form_id_4"}]))}, + {"stream": Mock(read_records=Mock(return_value=[{"id": "form_id_5"}, {"id": "form_id_6"}]))}, + ], + [{"form_id": "form_id_3"}, {"form_id": "form_id_4"}, {"form_id": "form_id_5"}, {"form_id": "form_id_6"}], + ), +] + + +@pytest.mark.parametrize("form_ids, parent_stream_configs, expected_slices", test_cases) +def test_stream_slices(form_ids, parent_stream_configs, expected_slices): + stream_configs = [] + + for parent_stream_config in parent_stream_configs: + stream_config = ParentStreamConfig( + stream=parent_stream_config["stream"], parent_key=None, partition_field=None, config=None, parameters=None + ) + stream_configs.append(stream_config) + if not stream_configs: + stream_configs = [None] + + router = FormIdPartitionRouter(config={"form_ids": form_ids}, parent_stream_configs=stream_configs, parameters=None) + slices = list(router.stream_slices()) + + assert slices == expected_slices diff --git a/airbyte-integrations/connectors/source-typeform/unit_tests/test_responses_stream.py b/airbyte-integrations/connectors/source-typeform/unit_tests/test_responses_stream.py deleted file mode 100644 index 53aa38990976..000000000000 --- a/airbyte-integrations/connectors/source-typeform/unit_tests/test_responses_stream.py +++ /dev/null @@ -1,65 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from typing import Any, Mapping, Optional - -import pendulum -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -from pendulum.datetime import DateTime -from source_typeform.source import Responses - -start_date_str = "2020-06-27T15:32:38Z" -start_date = pendulum.parse(start_date_str) -start_date_ts = start_date.int_timestamp -config = {"token": "10", "start_date": start_date_str, "page_size": 2} - -UTC = pendulum.timezone("UTC") -responses = Responses(authenticator=TokenAuthenticator(token=config["token"]), **config) - - -def get_last_record(last_record_cursor: DateTime, form_id: Optional[str] = "form1") -> Mapping[str, Any]: - metadata = {"referer": f"http://134/{form_id}"} if form_id else {} - return {Responses.cursor_field: last_record_cursor.format(Responses.date_format), "metadata": metadata} - - -def test_get_updated_state_new(): - # current record cursor greater than current state - current_state = {"form1": {Responses.cursor_field: start_date_ts + 100000}} - last_record_cursor = pendulum.now(UTC) - last_record = get_last_record(last_record_cursor) - - new_state = responses.get_updated_state(current_state, last_record) - assert new_state["form1"][Responses.cursor_field] == last_record_cursor.format(Responses.date_format) - - -def test_get_updated_state_not_changed(): - # current record cursor less than current state - current_state = {"form1": {Responses.cursor_field: start_date_ts + 100000}} - last_record_cursor = pendulum.from_timestamp(start_date_ts + 100) - last_record = get_last_record(last_record_cursor) - - new_state = responses.get_updated_state(current_state, last_record) - assert new_state["form1"][Responses.cursor_field] == pendulum.from_timestamp(start_date_ts + 100000).format(Responses.date_format) - - -def test_get_updated_state_form_id_is_new(): - # current record has new form id which is not exists in current state - current_state = {"form1": {Responses.cursor_field: start_date_ts + 100000}} - last_record_cursor = pendulum.from_timestamp(start_date_ts + 100) - last_record = get_last_record(last_record_cursor, form_id="form2") - - new_state = responses.get_updated_state(current_state, last_record) - assert new_state["form2"][Responses.cursor_field] == last_record_cursor.format(Responses.date_format) - assert new_state["form1"][Responses.cursor_field] == start_date_ts + 100000 - - -def test_get_updated_state_form_id_not_found_in_record(): - # current record doesn't have form_id - current_state = {"form1": {Responses.cursor_field: 100000}} - last_record_cursor = pendulum.now(UTC) - last_record = get_last_record(last_record_cursor, form_id=None) - - new_state = responses.get_updated_state(current_state, last_record) - assert new_state["form1"][Responses.cursor_field] == 100000 diff --git a/airbyte-integrations/connectors/source-typeform/unit_tests/test_source.py b/airbyte-integrations/connectors/source-typeform/unit_tests/test_source.py deleted file mode 100644 index 50e4ce49a0e7..000000000000 --- a/airbyte-integrations/connectors/source-typeform/unit_tests/test_source.py +++ /dev/null @@ -1,63 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import re - -from airbyte_cdk import AirbyteLogger -from source_typeform import SourceTypeform -from source_typeform.source import TypeformStream - -logger = AirbyteLogger() - -TYPEFORM_BASE_URL = TypeformStream.url_base - - -def test_check_connection_success(requests_mock, config, empty_response_ok): - requests_mock.register_uri("GET", TYPEFORM_BASE_URL + "me", empty_response_ok) - requests_mock.register_uri("GET", TYPEFORM_BASE_URL + "forms/u6nXL7", empty_response_ok) - requests_mock.register_uri("GET", TYPEFORM_BASE_URL + "forms/k9xNV4", empty_response_ok) - - ok, error = SourceTypeform().check_connection(logger, config) - - assert ok and not error - - -def test_check_connection_bad_request_me(requests_mock, config, empty_response_bad): - requests_mock.register_uri("GET", TYPEFORM_BASE_URL + "me", empty_response_bad) - - ok, error = SourceTypeform().check_connection(logger, config) - - assert not ok and error and re.match("Cannot authenticate, please verify token.", error) - - -def test_check_connection_bad_request_forms(requests_mock, config, empty_response_ok, empty_response_bad): - requests_mock.register_uri("GET", TYPEFORM_BASE_URL + "me", empty_response_ok) - requests_mock.register_uri("GET", TYPEFORM_BASE_URL + "forms/u6nXL7", empty_response_bad) - - ok, error = SourceTypeform().check_connection(logger, config) - - assert not ok and error and re.match("Cannot find forms with ID: u6nXL7", error) - - -def test_check_connection_empty(): - config = {} - - ok, error = SourceTypeform().check_connection(logger, config) - - assert not ok and error - - -def test_check_connection_incomplete(config): - credentials = config["credentials"] - credentials.pop("access_token") - - ok, error = SourceTypeform().check_connection(logger, config) - - assert not ok and error - - -def test_streams(config): - streams = SourceTypeform().streams(config) - - assert len(streams) == 6 diff --git a/airbyte-integrations/connectors/source-typeform/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-typeform/unit_tests/test_streams.py deleted file mode 100644 index 125db0d02d9a..000000000000 --- a/airbyte-integrations/connectors/source-typeform/unit_tests/test_streams.py +++ /dev/null @@ -1,112 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from typing import Any, List, Mapping -from unittest.mock import MagicMock - -from airbyte_cdk import AirbyteLogger -from airbyte_cdk.models import SyncMode -from source_typeform.source import Forms, Images, Responses, Themes, TypeformStream, Webhooks, Workspaces - -logger = AirbyteLogger() - -TYPEFORM_BASE_URL = TypeformStream.url_base - - -def merge_records(stream: TypeformStream, sync_mode: SyncMode) -> List[Mapping[str, Any]]: - merged_records = [] - for stream_slice in stream.stream_slices(sync_mode=sync_mode): - records = stream.read_records(sync_mode=sync_mode, stream_slice=stream_slice) - for record in records: - merged_records.append(record) - - return merged_records - - -def test_stream_forms_configured(requests_mock, config, form_response): - requests_mock.register_uri("GET", TYPEFORM_BASE_URL + "forms/u6nXL7", form_response) - requests_mock.register_uri("GET", TYPEFORM_BASE_URL + "forms/k9xNV4", form_response) - - stream = Forms(authenticator=MagicMock(), **config) - - merged_records = merge_records(stream, SyncMode.full_refresh) - - assert len(merged_records) == 2 - - -def test_stream_forms_unconfigured(requests_mock, config_without_forms, forms_response, form_response): - requests_mock.register_uri("GET", TYPEFORM_BASE_URL + "forms/u6nXL7", form_response) - requests_mock.register_uri("GET", TYPEFORM_BASE_URL + "forms/k9xNV4", form_response) - requests_mock.register_uri("GET", TYPEFORM_BASE_URL + "forms?page_size=200&page=1", forms_response) - - stream = Forms(authenticator=MagicMock(), **config_without_forms) - - merged_records = merge_records(stream, SyncMode.full_refresh) - - assert len(merged_records) == 2 - - -def test_stream_responses_configured(requests_mock, config, response_response): - requests_mock.register_uri("GET", TYPEFORM_BASE_URL + "forms/u6nXL7/responses", response_response) - requests_mock.register_uri("GET", TYPEFORM_BASE_URL + "forms/k9xNV4/responses", response_response) - - stream = Responses(authenticator=MagicMock(), **config) - - merged_records = merge_records(stream, SyncMode.incremental) - - assert len(merged_records) == 8 - - -def test_stream_responses_unconfigured(requests_mock, config_without_forms, forms_response, response_response): - requests_mock.register_uri("GET", TYPEFORM_BASE_URL + "forms/u6nXL7/responses", response_response) - requests_mock.register_uri("GET", TYPEFORM_BASE_URL + "forms/k9xNV4/responses", response_response) - requests_mock.register_uri("GET", TYPEFORM_BASE_URL + "forms?page_size=200&page=1", forms_response) - - stream = Responses(authenticator=MagicMock(), **config_without_forms) - - merged_records = merge_records(stream, SyncMode.incremental) - - assert len(merged_records) == 8 - - -def test_stream_webhooks(requests_mock, config, forms_response, webhooks_response): - requests_mock.register_uri("GET", TYPEFORM_BASE_URL + "forms/u6nXL7/webhooks", webhooks_response) - requests_mock.register_uri("GET", TYPEFORM_BASE_URL + "forms/k9xNV4/webhooks", webhooks_response) - requests_mock.register_uri("GET", TYPEFORM_BASE_URL + "forms?page_size=200&page=1", forms_response) - - stream = Webhooks(authenticator=MagicMock(), **config) - - merged_records = merge_records(stream, SyncMode.full_refresh) - - assert len(merged_records) == 2 - - -def test_stream_workspaces(requests_mock, config, workspaces_response): - requests_mock.register_uri("GET", TYPEFORM_BASE_URL + "workspaces?page=1&page_size=200", workspaces_response) - - stream = Workspaces(authenticator=MagicMock(), **config) - - merged_records = merge_records(stream, SyncMode.full_refresh) - - assert len(merged_records) == 2 - - -def test_stream_images(requests_mock, config, images_response): - requests_mock.register_uri("GET", TYPEFORM_BASE_URL + "images", images_response) - - stream = Images(authenticator=MagicMock(), **config) - - merged_records = merge_records(stream, SyncMode.full_refresh) - - assert len(merged_records) == 2 - - -def test_stream_themes(requests_mock, config, themes_response): - requests_mock.register_uri("GET", TYPEFORM_BASE_URL + "themes?page=1&page_size=200", themes_response) - - stream = Themes(authenticator=MagicMock(), **config) - - merged_records = merge_records(stream, SyncMode.full_refresh) - - assert len(merged_records) == 2 diff --git a/airbyte-integrations/connectors/source-unleash/README.md b/airbyte-integrations/connectors/source-unleash/README.md index 9b45755ef151..aefd0e9eac58 100644 --- a/airbyte-integrations/connectors/source-unleash/README.md +++ b/airbyte-integrations/connectors/source-unleash/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-unleash:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/unleash) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_unleash/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-unleash:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-unleash build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-unleash:airbyteDocker +An image will be built with the tag `airbyte/source-unleash:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-unleash:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-unleash:dev check --co docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-unleash:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-unleash:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-unleash test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-unleash:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-unleash:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-unleash test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/unleash.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-unleash/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-unleash/acceptance-test-docker.sh deleted file mode 100755 index a8d6ac4bb608..000000000000 --- a/airbyte-integrations/connectors/source-unleash/acceptance-test-docker.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env sh - -# Build latest connector image -docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) - -# Pull latest acctest image -docker pull airbyte/connector-acceptance-test:latest - -# Run -docker run --rm -it \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /tmp:/tmp \ - -v $(pwd):/test_input \ - airbyte/connector-acceptance-test \ - --acceptance-test-config /test_input - diff --git a/airbyte-integrations/connectors/source-unleash/build.gradle b/airbyte-integrations/connectors/source-unleash/build.gradle deleted file mode 100644 index a79828b196e0..000000000000 --- a/airbyte-integrations/connectors/source-unleash/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_unleash' -} diff --git a/airbyte-integrations/connectors/source-us-census/Dockerfile b/airbyte-integrations/connectors/source-us-census/Dockerfile index 8ce00031e8b7..49be16577813 100644 --- a/airbyte-integrations/connectors/source-us-census/Dockerfile +++ b/airbyte-integrations/connectors/source-us-census/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/source-us-census diff --git a/airbyte-integrations/connectors/source-us-census/README.md b/airbyte-integrations/connectors/source-us-census/README.md index d4899f44d8bc..ad9c105dbf3b 100644 --- a/airbyte-integrations/connectors/source-us-census/README.md +++ b/airbyte-integrations/connectors/source-us-census/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-us-census:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/us-census) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_us_census/spec.json` file. @@ -56,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-us-census:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-us-census build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-us-census:airbyteDocker +An image will be built with the tag `airbyte/source-us-census:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-us-census:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -77,44 +70,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-us-census:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-us-census:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-us-census:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-us-census test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-us-census:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-us-census:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -124,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-us-census test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/us-census.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-us-census/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-us-census/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-us-census/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-us-census/build.gradle b/airbyte-integrations/connectors/source-us-census/build.gradle deleted file mode 100644 index ebb4c6cc2957..000000000000 --- a/airbyte-integrations/connectors/source-us-census/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_us_census' -} diff --git a/airbyte-integrations/connectors/source-us-census/metadata.yaml b/airbyte-integrations/connectors/source-us-census/metadata.yaml index 47c5e9e77e8b..e08c93411869 100644 --- a/airbyte-integrations/connectors/source-us-census/metadata.yaml +++ b/airbyte-integrations/connectors/source-us-census/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: c4cfaeda-c757-489a-8aba-859fb08b6970 - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 dockerRepository: airbyte/source-us-census githubIssueLabel: source-us-census icon: uscensus.svg diff --git a/airbyte-integrations/connectors/source-us-census/setup.py b/airbyte-integrations/connectors/source-us-census/setup.py index 6b00267e73c8..7b2b4ebc83ec 100644 --- a/airbyte-integrations/connectors/source-us-census/setup.py +++ b/airbyte-integrations/connectors/source-us-census/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ diff --git a/airbyte-integrations/connectors/source-us-census/source_us_census/spec.json b/airbyte-integrations/connectors/source-us-census/source_us_census/spec.json index b8c67bbff6af..1c2115ecdd30 100644 --- a/airbyte-integrations/connectors/source-us-census/source_us_census/spec.json +++ b/airbyte-integrations/connectors/source-us-census/source_us_census/spec.json @@ -5,7 +5,7 @@ "title": "https://api.census.gov/ Source Spec", "type": "object", "required": ["api_key", "query_path"], - "additionalProperties": false, + "additionalProperties": true, "properties": { "query_params": { "type": "string", diff --git a/airbyte-integrations/connectors/source-vantage/README.md b/airbyte-integrations/connectors/source-vantage/README.md index 4647fb5e9427..fc037e05b323 100644 --- a/airbyte-integrations/connectors/source-vantage/README.md +++ b/airbyte-integrations/connectors/source-vantage/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-vantage:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/vantage) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_vantage/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-vantage:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-vantage build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-vantage:airbyteDocker +An image will be built with the tag `airbyte/source-vantage:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-vantage:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-vantage:dev check --co docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-vantage:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-vantage:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-vantage test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-vantage:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-vantage:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-vantage test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/vantage.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-vantage/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-vantage/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-vantage/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-vantage/build.gradle b/airbyte-integrations/connectors/source-vantage/build.gradle deleted file mode 100644 index 226502c81507..000000000000 --- a/airbyte-integrations/connectors/source-vantage/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_vantage' -} diff --git a/airbyte-integrations/connectors/source-visma-economic/BOOTSTRAP.md b/airbyte-integrations/connectors/source-visma-economic/BOOTSTRAP.md deleted file mode 100644 index be8113df80fa..000000000000 --- a/airbyte-integrations/connectors/source-visma-economic/BOOTSTRAP.md +++ /dev/null @@ -1,5 +0,0 @@ -# Visma e-conomic -Visma e-conomic is an accounting program. -Using the api it is possible interact with most of the functionality in the software. -The streams implemented allows you to do reporting based on invoices. -For more information about use cases of e-conomic please visit [this page](https://developer.visma.com/api/e-conomic/). \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-visma-economic/Dockerfile b/airbyte-integrations/connectors/source-visma-economic/Dockerfile index f85cd6906f4a..3c6fa9a2e487 100644 --- a/airbyte-integrations/connectors/source-visma-economic/Dockerfile +++ b/airbyte-integrations/connectors/source-visma-economic/Dockerfile @@ -34,5 +34,5 @@ COPY source_visma_economic ./source_visma_economic ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-visma-economic diff --git a/airbyte-integrations/connectors/source-visma-economic/README.md b/airbyte-integrations/connectors/source-visma-economic/README.md index 4ade1f0ae26e..34b7ba870bfb 100644 --- a/airbyte-integrations/connectors/source-visma-economic/README.md +++ b/airbyte-integrations/connectors/source-visma-economic/README.md @@ -1,76 +1,34 @@ # Visma Economic Source -This is the repository for the Visma Economic source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/visma-economic). +This is the repository for the Visma Economic configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/visma-economic). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-visma-economic:build -``` - #### Create credentials -For demo credentials see `sample_files/demo_config.json`. - -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/visma-economic) +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/visma-economic) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_visma_economic/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. - See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source visma-economic test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-visma-economic:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-visma-economic build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-visma-economic:airbyteDocker +An image will be built with the tag `airbyte/source-visma-economic:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-visma-economic:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -80,44 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-visma-economic:dev che docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-visma-economic:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-visma-economic:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-visma-economic test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-visma-economic:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-visma-economic:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -127,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-visma-economic test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/visma-economic.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-visma-economic/__init__.py b/airbyte-integrations/connectors/source-visma-economic/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-visma-economic/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-visma-economic/acceptance-test-config.yml b/airbyte-integrations/connectors/source-visma-economic/acceptance-test-config.yml index b94809fceaac..5d227254bb41 100644 --- a/airbyte-integrations/connectors/source-visma-economic/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-visma-economic/acceptance-test-config.yml @@ -1,26 +1,27 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-visma-economic:dev -tests: +acceptance_tests: spec: - - spec_path: "source_visma_economic/spec.yaml" + tests: + - spec_path: "source_visma_economic/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - timeout_seconds: 120 - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: ["invoices_total"] - expect_records: - path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: no - extra_records: yes + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + incremental: + bypass_reason: "This connector does not implement incremental sync" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-visma-economic/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-visma-economic/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-visma-economic/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-visma-economic/build.gradle b/airbyte-integrations/connectors/source-visma-economic/build.gradle deleted file mode 100644 index 2fc62653da4f..000000000000 --- a/airbyte-integrations/connectors/source-visma-economic/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_visma_economic' -} diff --git a/airbyte-integrations/connectors/source-visma-economic/integration_tests/__init__.py b/airbyte-integrations/connectors/source-visma-economic/integration_tests/__init__.py index 46b7376756ec..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-visma-economic/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-visma-economic/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-visma-economic/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-visma-economic/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-visma-economic/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-visma-economic/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-visma-economic/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-visma-economic/integration_tests/expected_records.jsonl deleted file mode 100644 index c6787903abcf..000000000000 --- a/airbyte-integrations/connectors/source-visma-economic/integration_tests/expected_records.jsonl +++ /dev/null @@ -1,12 +0,0 @@ -{"stream": "accounts", "data": {"accountNumber": 1000, "accountType": "heading", "balance": 0.0, "blockDirectEntries": false, "debitCredit": "debit", "name": "RESULTATOPG\u00d8RELSE", "accountingYears": "https://restapi.e-conomic.com/accounts/1000/accounting-years", "self": "https://restapi.e-conomic.com/accounts/1000"}, "emitted_at": 1667233244356} -{"stream": "accounts", "data": {"accountNumber": 1010, "accountType": "profitAndLoss", "balance": -6010.0, "blockDirectEntries": false, "debitCredit": "credit", "name": "Salg af varer/ydelser m/moms", "vatAccount": {"vatCode": "U25", "self": "https://restapi.e-conomic.com/vat-accounts/U25"}, "accountingYears": "https://restapi.e-conomic.com/accounts/1010/accounting-years", "self": "https://restapi.e-conomic.com/accounts/1010"}, "emitted_at": 1667233244360} -{"stream": "customers", "data": {"customerNumber": 1, "currency": "DKK", "paymentTerms": {"paymentTermsNumber": 1, "self": "https://restapi.e-conomic.com/payment-terms/1"}, "customerGroup": {"customerGroupNumber": 1, "self": "https://restapi.e-conomic.com/customer-groups/1"}, "address": "Avenue des Arts No 5", "balance": -1200.0, "dueAmount": 0.0, "city": "Brussels", "country": "Belgium", "email": "customerone@mailinator.com", "name": "Decathlon", "zip": "1040", "telephoneAndFaxNumber": "08343242525", "vatZone": {"vatZoneNumber": 1, "self": "https://restapi.e-conomic.com/vat-zones/1"}, "attention": {"customerContactNumber": 1, "customer": {"customerNumber": 1, "self": "https://restapi.e-conomic.com/customers/1"}, "self": "https://restapi.e-conomic.com/customers/1/contacts/1"}, "customerContact": {"customerContactNumber": 2, "customer": {"customerNumber": 1, "self": "https://restapi.e-conomic.com/customers/1"}, "self": "https://restapi.e-conomic.com/customers/1/contacts/2"}, "salesPerson": {"employeeNumber": 1, "self": "https://restapi.e-conomic.com/employees/1"}, "lastUpdated": "2022-06-02T08:53:29Z", "contacts": "https://restapi.e-conomic.com/customers/1/contacts", "defaultDeliveryLocation": {"deliveryLocationNumber": 1, "self": "https://restapi.e-conomic.com/customers/1/delivery-locations/1"}, "templates": {"invoice": "https://restapi.e-conomic.com/customers/1/templates/invoice", "invoiceLine": "https://restapi.e-conomic.com/customers/1/templates/invoiceline", "self": "https://restapi.e-conomic.com/customers/1/templates"}, "totals": {"drafts": "https://restapi.e-conomic.com/invoices/totals/drafts/customers/1", "booked": "https://restapi.e-conomic.com/invoices/totals/booked/customers/1", "self": "https://restapi.e-conomic.com/customers/1/totals"}, "deliveryLocations": "https://restapi.e-conomic.com/customers/1/delivery-locations", "invoices": {"drafts": "https://restapi.e-conomic.com/customers/1/invoices/drafts", "booked": "https://restapi.e-conomic.com/customers/1/invoices/booked", "self": "https://restapi.e-conomic.com/customers/1/invoices"}, "mobilePhone": "066567657657575", "eInvoicingDisabledByDefault": false, "self": "https://restapi.e-conomic.com/customers/1"}, "emitted_at": 1667233244985} -{"stream": "customers", "data": {"customerNumber": 5, "currency": "EUR", "paymentTerms": {"paymentTermsNumber": 1, "self": "https://restapi.e-conomic.com/payment-terms/1"}, "customerGroup": {"customerGroupNumber": 4, "self": "https://restapi.e-conomic.com/customer-groups/4"}, "address": "Avenue Centrale No 3", "balance": 0.0, "dueAmount": 0.0, "city": "Paris", "country": "France", "email": "freres@mailinator.com", "name": "Les Freres Heureux", "zip": "1231", "telephoneAndFaxNumber": "5475685685", "vatZone": {"vatZoneNumber": 17, "self": "https://restapi.e-conomic.com/vat-zones/17"}, "attention": {"customerContactNumber": 15, "customer": {"customerNumber": 5, "self": "https://restapi.e-conomic.com/customers/5"}, "self": "https://restapi.e-conomic.com/customers/5/contacts/15"}, "customerContact": {"customerContactNumber": 16, "customer": {"customerNumber": 5, "self": "https://restapi.e-conomic.com/customers/5"}, "self": "https://restapi.e-conomic.com/customers/5/contacts/16"}, "salesPerson": {"employeeNumber": 2, "self": "https://restapi.e-conomic.com/employees/2"}, "lastUpdated": "2022-06-07T12:09:44Z", "contacts": "https://restapi.e-conomic.com/customers/5/contacts", "defaultDeliveryLocation": {"deliveryLocationNumber": 5, "self": "https://restapi.e-conomic.com/customers/5/delivery-locations/5"}, "templates": {"invoice": "https://restapi.e-conomic.com/customers/5/templates/invoice", "invoiceLine": "https://restapi.e-conomic.com/customers/5/templates/invoiceline", "self": "https://restapi.e-conomic.com/customers/5/templates"}, "totals": {"drafts": "https://restapi.e-conomic.com/invoices/totals/drafts/customers/5", "booked": "https://restapi.e-conomic.com/invoices/totals/booked/customers/5", "self": "https://restapi.e-conomic.com/customers/5/totals"}, "deliveryLocations": "https://restapi.e-conomic.com/customers/5/delivery-locations", "invoices": {"drafts": "https://restapi.e-conomic.com/customers/5/invoices/drafts", "booked": "https://restapi.e-conomic.com/customers/5/invoices/booked", "self": "https://restapi.e-conomic.com/customers/5/invoices"}, "mobilePhone": "09675675685", "eInvoicingDisabledByDefault": false, "self": "https://restapi.e-conomic.com/customers/5"}, "emitted_at": 1667233244987} -{"stream": "invoices_booked", "data": {"bookedInvoiceNumber": 1, "orderNumber": 1, "date": "2022-06-02", "currency": "DKK", "exchangeRate": 100.0, "netAmount": 70.0, "netAmountInBaseCurrency": 70.0, "grossAmount": 87.5, "grossAmountInBaseCurrency": 87.5, "vatAmount": 17.5, "roundingAmount": 0.0, "remainder": 0.0, "remainderInBaseCurrency": 0.0, "dueDate": "2022-06-10", "paymentTerms": {"paymentTermsNumber": 1, "daysOfCredit": 8, "description": "Netto 8 dage", "name": "Netto 8 dage", "paymentTermsType": "net", "self": "https://restapi.e-conomic.com/payment-terms/1"}, "customer": {"customerNumber": 1, "self": "https://restapi.e-conomic.com/customers/1"}, "recipient": {"name": "Customer 1", "address": "Avenue des Arts No 5", "zip": "1040", "city": "Brussels", "country": "Belgium", "attention": {"customerContactNumber": 1, "self": "https://restapi.e-conomic.com/customers/1/contacts/1"}, "vatZone": {"name": "Domestic", "vatZoneNumber": 1, "enabledForCustomer": true, "enabledForSupplier": true, "self": "https://restapi.e-conomic.com/vat-zones/1"}}, "deliveryLocation": {"deliveryLocationNumber": 1, "self": "https://restapi.e-conomic.com/customers/1/delivery-locations/1"}, "delivery": {"address": "Langebrogade 1", "zip": "1411", "city": "K\u00f8benhavn K", "country": "Denmark"}, "references": {"customerContact": {"customerContactNumber": 2, "self": "https://restapi.e-conomic.com/customers/1/contacts/2"}, "salesPerson": {"employeeNumber": 1, "self": "https://restapi.e-conomic.com/employees/1"}}, "layout": {"layoutNumber": 21, "self": "https://restapi.e-conomic.com/layouts/21"}, "pdf": {"download": "https://restapi.e-conomic.com/invoices/booked/1/pdf"}, "sent": "https://restapi.e-conomic.com/invoices/booked/1/sent", "self": "https://restapi.e-conomic.com/invoices/booked/1"}, "emitted_at": 1667233245263} -{"stream": "invoices_booked", "data": {"bookedInvoiceNumber": 2, "orderNumber": 2, "date": "2022-06-02", "currency": "DKK", "exchangeRate": 100.0, "netAmount": 950.0, "netAmountInBaseCurrency": 950.0, "grossAmount": 1187.5, "grossAmountInBaseCurrency": 1187.5, "vatAmount": 237.5, "roundingAmount": 0.0, "remainder": 0.0, "remainderInBaseCurrency": 0.0, "dueDate": "2022-06-10", "paymentTerms": {"paymentTermsNumber": 1, "daysOfCredit": 8, "description": "Netto 8 dage", "name": "Netto 8 dage", "paymentTermsType": "net", "self": "https://restapi.e-conomic.com/payment-terms/1"}, "customer": {"customerNumber": 4, "self": "https://restapi.e-conomic.com/customers/4"}, "recipient": {"name": "Customer with EAN", "address": "Avenue des arts 4", "zip": "3221", "city": "Brussels", "country": "Belgium", "ean": "9773365646824", "attention": {"customerContactNumber": 7, "self": "https://restapi.e-conomic.com/customers/4/contacts/7"}, "vatZone": {"name": "Domestic", "vatZoneNumber": 1, "enabledForCustomer": true, "enabledForSupplier": true, "self": "https://restapi.e-conomic.com/vat-zones/1"}}, "deliveryLocation": {"deliveryLocationNumber": 4, "self": "https://restapi.e-conomic.com/customers/4/delivery-locations/4"}, "delivery": {"address": "Avenue des arts no 3", "zip": "1212", "city": "K\u00f8benhavn K", "country": "Denmark"}, "notes": {"heading": "Customer 4"}, "references": {"customerContact": {"customerContactNumber": 8, "self": "https://restapi.e-conomic.com/customers/4/contacts/8"}, "salesPerson": {"employeeNumber": 6, "self": "https://restapi.e-conomic.com/employees/6"}}, "layout": {"layoutNumber": 21, "self": "https://restapi.e-conomic.com/layouts/21"}, "project": {"projectNumber": 1, "self": "https://restapi.e-conomic.com/projects/1"}, "pdf": {"download": "https://restapi.e-conomic.com/invoices/booked/2/pdf"}, "sent": "https://restapi.e-conomic.com/invoices/booked/2/sent", "self": "https://restapi.e-conomic.com/invoices/booked/2"}, "emitted_at": 1667233245266} -{"stream": "invoices_booked_document", "data": {"bookedInvoiceNumber": 1, "orderNumber": 1, "date": "2022-06-02", "currency": "DKK", "exchangeRate": 100.0, "netAmount": 70.0, "netAmountInBaseCurrency": 70.0, "grossAmount": 87.5, "grossAmountInBaseCurrency": 87.5, "vatAmount": 17.5, "roundingAmount": 0.0, "remainder": 0.0, "remainderInBaseCurrency": 0.0, "dueDate": "2022-06-10", "paymentTerms": {"paymentTermsNumber": 1, "daysOfCredit": 8, "description": "Netto 8 dage", "name": "Netto 8 dage", "paymentTermsType": "net", "self": "https://restapi.e-conomic.com/payment-terms/1"}, "customer": {"customerNumber": 1, "self": "https://restapi.e-conomic.com/customers/1"}, "recipient": {"name": "Customer 1", "address": "Avenue des Arts No 5", "zip": "1040", "city": "Brussels", "country": "Belgium", "attention": {"customerContactNumber": 1, "self": "https://restapi.e-conomic.com/customers/1/contacts/1"}, "vatZone": {"name": "Domestic", "vatZoneNumber": 1, "enabledForCustomer": true, "enabledForSupplier": true, "self": "https://restapi.e-conomic.com/vat-zones/1"}}, "deliveryLocation": {"deliveryLocationNumber": 1, "self": "https://restapi.e-conomic.com/customers/1/delivery-locations/1"}, "delivery": {"address": "Langebrogade 1", "zip": "1411", "city": "K\u00f8benhavn K", "country": "Denmark"}, "references": {"customerContact": {"customerContactNumber": 2, "self": "https://restapi.e-conomic.com/customers/1/contacts/2"}, "salesPerson": {"employeeNumber": 1, "self": "https://restapi.e-conomic.com/employees/1"}}, "layout": {"layoutNumber": 21, "self": "https://restapi.e-conomic.com/layouts/21"}, "pdf": {"download": "https://restapi.e-conomic.com/invoices/booked/1/pdf"}, "lines": [{"lineNumber": 1, "sortKey": 1, "description": "T-shirts", "quantity": 1.0, "unitNetPrice": 70.0, "discountPercentage": 0.0, "unitCostPrice": 40.0, "vatRate": 25.0, "vatAmount": 17.5, "totalNetAmount": 70.0, "product": {"productNumber": "1", "self": "https://restapi.e-conomic.com/products/1"}, "unit": {"unitNumber": 1, "name": "stk.", "products": "https://restapi.e-conomic.com/units/1/products", "self": "https://restapi.e-conomic.com/units/1"}}], "sent": "https://restapi.e-conomic.com/invoices/booked/1/sent", "self": "https://restapi.e-conomic.com/invoices/booked/1"}, "emitted_at": 1667233245881} -{"stream": "invoices_booked_document", "data": {"bookedInvoiceNumber": 2, "orderNumber": 2, "date": "2022-06-02", "currency": "DKK", "exchangeRate": 100.0, "netAmount": 950.0, "netAmountInBaseCurrency": 950.0, "grossAmount": 1187.5, "grossAmountInBaseCurrency": 1187.5, "vatAmount": 237.5, "roundingAmount": 0.0, "remainder": 0.0, "remainderInBaseCurrency": 0.0, "dueDate": "2022-06-10", "paymentTerms": {"paymentTermsNumber": 1, "daysOfCredit": 8, "description": "Netto 8 dage", "name": "Netto 8 dage", "paymentTermsType": "net", "self": "https://restapi.e-conomic.com/payment-terms/1"}, "customer": {"customerNumber": 4, "self": "https://restapi.e-conomic.com/customers/4"}, "recipient": {"name": "Customer with EAN", "address": "Avenue des arts 4", "zip": "3221", "city": "Brussels", "country": "Belgium", "ean": "9773365646824", "attention": {"customerContactNumber": 7, "self": "https://restapi.e-conomic.com/customers/4/contacts/7"}, "vatZone": {"name": "Domestic", "vatZoneNumber": 1, "enabledForCustomer": true, "enabledForSupplier": true, "self": "https://restapi.e-conomic.com/vat-zones/1"}}, "deliveryLocation": {"deliveryLocationNumber": 4, "self": "https://restapi.e-conomic.com/customers/4/delivery-locations/4"}, "delivery": {"address": "Avenue des arts no 3", "zip": "1212", "city": "K\u00f8benhavn K", "country": "Denmark"}, "notes": {"heading": "Customer 4"}, "references": {"customerContact": {"customerContactNumber": 8, "self": "https://restapi.e-conomic.com/customers/4/contacts/8"}, "salesPerson": {"employeeNumber": 6, "self": "https://restapi.e-conomic.com/employees/6"}}, "layout": {"layoutNumber": 21, "self": "https://restapi.e-conomic.com/layouts/21"}, "project": {"projectNumber": 1, "self": "https://restapi.e-conomic.com/projects/1"}, "pdf": {"download": "https://restapi.e-conomic.com/invoices/booked/2/pdf"}, "lines": [{"lineNumber": 1, "sortKey": 1, "description": "T-shirts", "quantity": 10.0, "unitNetPrice": 70.0, "discountPercentage": 0.0, "unitCostPrice": 40.0, "vatRate": 25.0, "vatAmount": 175.0, "totalNetAmount": 700.0, "product": {"productNumber": "1", "self": "https://restapi.e-conomic.com/products/1"}, "unit": {"unitNumber": 1, "name": "stk.", "products": "https://restapi.e-conomic.com/units/1/products", "self": "https://restapi.e-conomic.com/units/1"}}, {"lineNumber": 2, "sortKey": 2, "description": "T-shirts", "quantity": 1.0, "unitNetPrice": 250.0, "discountPercentage": 0.0, "unitCostPrice": 40.0, "vatRate": 25.0, "vatAmount": 62.5, "totalNetAmount": 250.0, "product": {"productNumber": "1", "self": "https://restapi.e-conomic.com/products/1"}, "unit": {"unitNumber": 1, "name": "stk.", "products": "https://restapi.e-conomic.com/units/1/products", "self": "https://restapi.e-conomic.com/units/1"}}], "sent": "https://restapi.e-conomic.com/invoices/booked/2/sent", "self": "https://restapi.e-conomic.com/invoices/booked/2"}, "emitted_at": 1667233246020} -{"stream": "invoices_paid", "data": {"bookedInvoiceNumber": 1, "orderNumber": 1, "date": "2022-06-02", "currency": "DKK", "exchangeRate": 100.0, "netAmount": 70.0, "netAmountInBaseCurrency": 70.0, "grossAmount": 87.5, "grossAmountInBaseCurrency": 87.5, "vatAmount": 17.5, "roundingAmount": 0.0, "remainder": 0.0, "remainderInBaseCurrency": 0.0, "dueDate": "2022-06-10", "paymentTerms": {"paymentTermsNumber": 1, "daysOfCredit": 8, "description": "Netto 8 dage", "name": "Netto 8 dage", "paymentTermsType": "net", "self": "https://restapi.e-conomic.com/payment-terms/1"}, "customer": {"customerNumber": 1, "self": "https://restapi.e-conomic.com/customers/1"}, "recipient": {"name": "Customer 1", "address": "Avenue des Arts No 5", "zip": "1040", "city": "Brussels", "country": "Belgium", "attention": {"customerContactNumber": 1, "self": "https://restapi.e-conomic.com/customers/1/contacts/1"}, "vatZone": {"name": "Domestic", "vatZoneNumber": 1, "enabledForCustomer": true, "enabledForSupplier": true, "self": "https://restapi.e-conomic.com/vat-zones/1"}}, "deliveryLocation": {"deliveryLocationNumber": 1, "self": "https://restapi.e-conomic.com/customers/1/delivery-locations/1"}, "delivery": {"address": "Langebrogade 1", "zip": "1411", "city": "K\u00f8benhavn K", "country": "Denmark"}, "references": {"customerContact": {"customerContactNumber": 2, "self": "https://restapi.e-conomic.com/customers/1/contacts/2"}, "salesPerson": {"employeeNumber": 1, "self": "https://restapi.e-conomic.com/employees/1"}}, "layout": {"layoutNumber": 21, "self": "https://restapi.e-conomic.com/layouts/21"}, "pdf": {"download": "https://restapi.e-conomic.com/invoices/booked/1/pdf"}, "sent": "https://restapi.e-conomic.com/invoices/booked/1/sent", "self": "https://restapi.e-conomic.com/invoices/booked/1"}, "emitted_at": 1667233246938} -{"stream": "invoices_paid", "data": {"bookedInvoiceNumber": 2, "orderNumber": 2, "date": "2022-06-02", "currency": "DKK", "exchangeRate": 100.0, "netAmount": 950.0, "netAmountInBaseCurrency": 950.0, "grossAmount": 1187.5, "grossAmountInBaseCurrency": 1187.5, "vatAmount": 237.5, "roundingAmount": 0.0, "remainder": 0.0, "remainderInBaseCurrency": 0.0, "dueDate": "2022-06-10", "paymentTerms": {"paymentTermsNumber": 1, "daysOfCredit": 8, "description": "Netto 8 dage", "name": "Netto 8 dage", "paymentTermsType": "net", "self": "https://restapi.e-conomic.com/payment-terms/1"}, "customer": {"customerNumber": 4, "self": "https://restapi.e-conomic.com/customers/4"}, "recipient": {"name": "Customer with EAN", "address": "Avenue des arts 4", "zip": "3221", "city": "Brussels", "country": "Belgium", "ean": "9773365646824", "attention": {"customerContactNumber": 7, "self": "https://restapi.e-conomic.com/customers/4/contacts/7"}, "vatZone": {"name": "Domestic", "vatZoneNumber": 1, "enabledForCustomer": true, "enabledForSupplier": true, "self": "https://restapi.e-conomic.com/vat-zones/1"}}, "deliveryLocation": {"deliveryLocationNumber": 4, "self": "https://restapi.e-conomic.com/customers/4/delivery-locations/4"}, "delivery": {"address": "Avenue des arts no 3", "zip": "1212", "city": "K\u00f8benhavn K", "country": "Denmark"}, "notes": {"heading": "Customer 4"}, "references": {"customerContact": {"customerContactNumber": 8, "self": "https://restapi.e-conomic.com/customers/4/contacts/8"}, "salesPerson": {"employeeNumber": 6, "self": "https://restapi.e-conomic.com/employees/6"}}, "layout": {"layoutNumber": 21, "self": "https://restapi.e-conomic.com/layouts/21"}, "project": {"projectNumber": 1, "self": "https://restapi.e-conomic.com/projects/1"}, "pdf": {"download": "https://restapi.e-conomic.com/invoices/booked/2/pdf"}, "sent": "https://restapi.e-conomic.com/invoices/booked/2/sent", "self": "https://restapi.e-conomic.com/invoices/booked/2"}, "emitted_at": 1667233246940} -{"stream": "products", "data": {"productNumber": "1", "description": "V-cut T-shirt, size S-XXL", "name": "Noname T-shirt Black", "costPrice": 40.0, "recommendedPrice": 90.0, "salesPrice": 70.0, "barred": false, "minimumStock": 0.0, "lastUpdated": "2022-06-24T08:25:00Z", "productGroup": {"productGroupNumber": 1, "name": "Varer m/moms", "salesAccounts": "https://restapi.e-conomic.com/product-groups/1/sales-accounts", "products": "https://restapi.e-conomic.com/product-groups/1/products", "self": "https://restapi.e-conomic.com/product-groups/1"}, "unit": {"unitNumber": 1, "name": "stk.", "products": "https://restapi.e-conomic.com/units/1/products", "self": "https://restapi.e-conomic.com/units/1"}, "invoices": {"drafts": "https://restapi.e-conomic.com/products/1/invoices/drafts", "booked": "https://restapi.e-conomic.com/products/1/invoices/booked", "self": "https://restapi.e-conomic.com/products/1/invoices"}, "pricing": {"currencySpecificSalesPrices": "https://restapi.e-conomic.com/products/1/pricing/currency-specific-sales-prices"}, "self": "https://restapi.e-conomic.com/products/1"}, "emitted_at": 1667233247352} -{"stream": "products", "data": {"productNumber": "2", "description": "This product has been barred", "name": "Barred product", "costPrice": 20.0, "recommendedPrice": 50.0, "salesPrice": 50.0, "barred": true, "minimumStock": 0.0, "lastUpdated": "2022-06-24T08:25:00Z", "productGroup": {"productGroupNumber": 1, "name": "Varer m/moms", "salesAccounts": "https://restapi.e-conomic.com/product-groups/1/sales-accounts", "products": "https://restapi.e-conomic.com/product-groups/1/products", "self": "https://restapi.e-conomic.com/product-groups/1"}, "unit": {"unitNumber": 1, "name": "stk.", "products": "https://restapi.e-conomic.com/units/1/products", "self": "https://restapi.e-conomic.com/units/1"}, "invoices": {"drafts": "https://restapi.e-conomic.com/products/2/invoices/drafts", "booked": "https://restapi.e-conomic.com/products/2/invoices/booked", "self": "https://restapi.e-conomic.com/products/2/invoices"}, "pricing": {"currencySpecificSalesPrices": "https://restapi.e-conomic.com/products/2/pricing/currency-specific-sales-prices"}, "self": "https://restapi.e-conomic.com/products/2"}, "emitted_at": 1667233247354} diff --git a/airbyte-integrations/connectors/source-visma-economic/metadata.yaml b/airbyte-integrations/connectors/source-visma-economic/metadata.yaml index 07e591456923..56e6d7330ce5 100644 --- a/airbyte-integrations/connectors/source-visma-economic/metadata.yaml +++ b/airbyte-integrations/connectors/source-visma-economic/metadata.yaml @@ -1,24 +1,25 @@ data: + allowedHosts: + hosts: + - "restapi.e-conomic.com" + registries: + oss: + enabled: true + cloud: + enabled: false connectorSubtype: api connectorType: source definitionId: 42495935-95de-4f5c-ae08-8fac00f6b308 - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-visma-economic githubIssueLabel: source-visma-economic - icon: visma-e-conomic.svg + icon: visma-economic.svg license: MIT - name: Visma E-conomic - registries: - cloud: - enabled: false - oss: - enabled: true + name: Visma Economic + releaseDate: "2022-11-08" releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/visma-economic tags: - - language:python - ab_internal: - sl: 100 - ql: 100 - supportLevel: community + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-visma-economic/requirements.txt b/airbyte-integrations/connectors/source-visma-economic/requirements.txt index ecf975e2fa63..cf563bcab685 100644 --- a/airbyte-integrations/connectors/source-visma-economic/requirements.txt +++ b/airbyte-integrations/connectors/source-visma-economic/requirements.txt @@ -1 +1,2 @@ --e . \ No newline at end of file +-e . +-e ../../bases/connector-acceptance-test \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-visma-economic/setup.py b/airbyte-integrations/connectors/source-visma-economic/setup.py index 4f72cf569184..9574bffc86ff 100644 --- a/airbyte-integrations/connectors/source-visma-economic/setup.py +++ b/airbyte-integrations/connectors/source-visma-economic/setup.py @@ -9,7 +9,11 @@ "airbyte-cdk~=0.1", ] -TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.1", "pytest-mock~=3.6.1", "responses~=0.13.3"] +TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest~=6.2", + "pytest-mock~=3.6.1", +] setup( name="source_visma_economic", diff --git a/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/__init__.py b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/__init__.py index 4b60788947b2..18146c2bff0e 100644 --- a/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/__init__.py +++ b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/manifest.yaml b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/manifest.yaml new file mode 100644 index 000000000000..969c91b8a939 --- /dev/null +++ b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/manifest.yaml @@ -0,0 +1,143 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["collection"] + requester: + type: HttpRequester + url_base: "https://restapi.e-conomic.com/" + http_method: "GET" + request_headers: + X-AppSecretToken: "{{config['app_secret_token']}}" + X-AgreementGrantToken: "{{config['agreement_grant_token']}}" + request_parameters: + pagesize: "1000" + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ response['pagination']['nextPage'] }}" + page_token_option: + type: "RequestPath" + requester: + $ref: "#/definitions/requester" + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + accounts_stream: + $ref: "#/definitions/base_stream" + name: "accounts" + primary_key: "accountNumber" + $parameters: + path: "accounts" + + customers_stream: + $ref: "#/definitions/base_stream" + name: "customers" + primary_key: "customerNumber" + $parameters: + path: "customers" + + products_stream: + $ref: "#/definitions/base_stream" + name: "products" + primary_key: "productNumber" + $parameters: + path: "products" + + invoices_total_stream: + $ref: "#/definitions/base_stream" + name: "invoices_total" + primary_key: "" + retriever: + $ref: "#/definitions/retriever" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: NoPagination + $parameters: + path: "invoices/totals" + + invoices_paid_stream: + $ref: "#/definitions/base_stream" + name: "invoices_paid" + primary_key: "bookedInvoiceNumber" + $parameters: + path: "invoices/paid" + + invoices_booked_stream: + $ref: "#/definitions/base_stream" + name: "invoices_booked" + primary_key: "bookedInvoiceNumber" + $parameters: + path: "invoices/booked" + + invoices_booked_document_stream: + name: "invoices_booked_document" + primary_key: "bookedInvoiceNumber" + retriever: + $ref: "#/definitions/retriever" + requester: + $ref: "#/definitions/requester" + path: "invoices/booked/{{ stream_slice.parent_id }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: [] + paginator: + type: NoPagination + partition_router: + type: SubstreamPartitionRouter + parent_stream_configs: + - stream: "#/definitions/invoices_booked_stream" + parent_key: "bookedInvoiceNumber" + partition_field: "parent_id" + +streams: + - "#/definitions/accounts_stream" + - "#/definitions/customers_stream" + - "#/definitions/products_stream" + - "#/definitions/invoices_total_stream" + - "#/definitions/invoices_paid_stream" + - "#/definitions/invoices_booked_stream" + - "#/definitions/invoices_booked_document_stream" + +check: + type: CheckStream + stream_names: + - "accounts" + +spec: + type: Spec + documentationUrl: https://docs.airbyte.com/integrations/sources/visma-economic + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + title: Visma E-conomic Spec + type: object + required: + - app_secret_token + - agreement_grant_token + properties: + app_secret_token: + title: App Secret Token + type: string + description: Identification token for app accessing data + airbyte_secret: true + agreement_grant_token: + title: Agreement Grant Token + type: string + description: Identifier for the grant issued by an agreement + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/accounts.json b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/accounts.json index 657c6bc3cf24..6df49ef8625b 100644 --- a/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/accounts.json +++ b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/accounts.json @@ -1,9 +1,8 @@ { - "$schema": "http://json-schema.org/draft-03/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "title": "Accounts collection schema", "description": "A schema for retrieving the accounts of the chart of accounts.", "type": "object", - "restdocs": "http://restdocs.e-conomic.com/#get-accounts", "properties": { "accountNumber": { "type": "integer", @@ -42,6 +41,12 @@ "filterable": true, "description": "Determines if the account can be manually updated with entries." }, + "department": { + "type": ["null", "object"] + }, + "departmentalDistribution": { + "type": ["null", "object"] + }, "contraAccount": { "type": "object", "description": "The default contra account of the account.", @@ -149,6 +154,19 @@ "type": "string", "format": "uri", "description": "A unique reference to the account resource." + }, + "openingAccount": { + "type": ["null", "object"], + "properties": { + "accountNumber": { + "type": "integer" + }, + "self": { + "type": "string", + "format": "uri", + "description": "The unique self link of the VAT code." + } + } } } } diff --git a/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/customers.json b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/customers.json index 02ee90159f9c..15210c6970cd 100644 --- a/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/customers.json +++ b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/customers.json @@ -1,9 +1,8 @@ { - "$schema": "http://json-schema.org/draft-03/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "title": "Customer collection GET schema", "description": "A schema for fetching a collection of customer, aka. Debtor.", "type": "object", - "restdocs": "http://restdocs.e-conomic.com/#get-customers", "properties": { "address": { "type": "string", @@ -18,6 +17,9 @@ "filterable": true, "description": "The outstanding amount for this customer." }, + "dueAmount": { + "type": ["null", "number"] + }, "barred": { "type": "boolean", "filterable": true, diff --git a/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/invoices_booked.json b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/invoices_booked.json index 4f8f16dbbf86..83b5ae0fca5f 100644 --- a/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/invoices_booked.json +++ b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/invoices_booked.json @@ -1,9 +1,8 @@ { - "$schema": "http://json-schema.org/draft-03/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "title": "Booked invoice collection schema", "description": "A schema for retrieving a collection of booked invoices.", "type": "object", - "restdocs": "http://restdocs.e-conomic.com/#get-invoices-booked", "properties": { "bookedInvoiceNumber": { "type": "integer", @@ -12,6 +11,9 @@ "sortable": true, "description": "A reference number for the booked invoice document." }, + "orderNumber": { + "type": ["null", "integer"] + }, "date": { "type": "string", "format": "full-date", diff --git a/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/invoices_booked_document.json b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/invoices_booked_document.json index 165e1157a460..5b7e97461d51 100644 --- a/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/invoices_booked_document.json +++ b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/invoices_booked_document.json @@ -1,9 +1,8 @@ { - "$schema": "http://json-schema.org/draft-03/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "title": "Booked invoice schema", "description": "A schema for retrieving a booked invoice.", "type": "object", - "restdocs": "http://restdocs.e-conomic.com/#get-invoices-booked-bookedinvoicenumber", "properties": { "bookedInvoiceNumber": { "type": "integer", @@ -12,6 +11,9 @@ "sortable": true, "description": "A reference number for the booked invoice document." }, + "orderNumber": { + "type": ["null", "integer"] + }, "date": { "type": "string", "format": "full-date", diff --git a/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/invoices_paid.json b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/invoices_paid.json index 44a7a0045b69..53fe57db1619 100644 --- a/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/invoices_paid.json +++ b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/invoices_paid.json @@ -1,8 +1,7 @@ { - "$schema": "http://json-schema.org/draft-03/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "title": "Paid invoice", "type": "object", - "restdocs": "http://restdocs.e-conomic.com/#get-invoices-paid", "properties": { "bookedInvoiceNumber": { "type": "integer", @@ -11,6 +10,15 @@ "sortable": true, "description": "A reference number for the booked invoice document." }, + "exchangeRate": { + "type": ["null", "number"] + }, + "orderNumber": { + "type": ["null", "integer"] + }, + "grossAmountInBaseCurrency": { + "type": ["null", "number"] + }, "date": { "type": "string", "format": "full-date", diff --git a/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/invoices_total.json b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/invoices_total.json index 76b0626e41f4..e4f29a6ab4e7 100644 --- a/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/invoices_total.json +++ b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/invoices_total.json @@ -1,9 +1,8 @@ { - "$schema": "http://json-schema.org/draft-03/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "title": "Invoice totals GET schema", "description": "A schema for retrieval of the totals of invoices.", "type": "object", - "restdocs": "http://restdocs.e-conomic.com/#get-invoices-totals", "properties": { "drafts": { "type": "object", diff --git a/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/products.json b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/products.json index 23ddd9767dda..5344d6701880 100644 --- a/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/products.json +++ b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/schemas/products.json @@ -1,7 +1,6 @@ { - "$schema": "http://json-schema.org/draft-03/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "title": "Products collection GET schema", - "restdocs": "http://restdocs.e-conomic.com/#get-products", "type": "object", "description": "A schema for retrieval of a collection of products.", "properties": { @@ -46,6 +45,12 @@ "maxDecimal": 2, "description": "This is the unit net price that will appear on invoice lines when a product is added to an invoice line." }, + "minimumStock": { + "type": ["null", "number"] + }, + "pricing": { + "type": ["null", "object"] + }, "barCode": { "type": "string", "filterable": true, diff --git a/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/source.py b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/source.py index 7194c6104ba5..bd4ee0857492 100644 --- a/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/source.py +++ b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/source.py @@ -2,146 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple -from urllib.parse import parse_qs, urlparse +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. +WARNING: Do not modify this file. +""" -class VismaEconomicStream(HttpStream, ABC): - url_base: str = "https://restapi.e-conomic.com/" - page_size: int = 1000 - def __init__(self, app_secret_token: str = None, agreement_grant_token: str = None): - self.app_secret_token: str = app_secret_token - self.agreement_grant_token: str = agreement_grant_token - super().__init__() - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - response_json = response.json() - if "nextPage" in response_json.get("pagination", {}).keys(): - parsed_url = urlparse(response_json["pagination"]["nextPage"]) - query_params = parse_qs(parsed_url.query) - return query_params - else: - return None - - def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: - if next_page_token: - return dict(next_page_token) - else: - return {"skippages": 0, "pagesize": self.page_size} - - def request_headers(self, **kwargs) -> Mapping[str, Any]: - return {"X-AppSecretToken": self.app_secret_token, "X-AgreementGrantToken": self.agreement_grant_token} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - yield from response.json().get("collection", []) - - -class Accounts(VismaEconomicStream): - primary_key = "accountNumber" - - def path(self, **kwargs) -> str: - return "accounts" - - -class Customers(VismaEconomicStream): - primary_key = "customerNumber" - - def path(self, **kwargs) -> str: - return "customers" - - -class Products(VismaEconomicStream): - primary_key = "productNumber" - - def path(self, **kwargs) -> str: - return "products" - - -class InvoicesTotal(VismaEconomicStream): - primary_key = None - - def path(self, **kwargs) -> str: - return "invoices/totals" - - -class InvoicesPaid(VismaEconomicStream): - primary_key = "bookedInvoiceNumber" - - def path(self, **kwargs) -> str: - return "invoices/paid" - - -class InvoicesBooked(VismaEconomicStream): - primary_key = "bookedInvoiceNumber" - - def path(self, **kwargs) -> str: - return "invoices/booked" - - -class InvoicesBookedDocument(HttpSubStream, VismaEconomicStream): - primary_key = "bookedInvoiceNumber" - - def __init__(self, **kwargs): - super().__init__(InvoicesBooked(**kwargs), **kwargs) - - def path(self, stream_slice: Mapping[str, Any], **kwargs) -> str: - booked_invoice_number = stream_slice["parent"]["bookedInvoiceNumber"] - return f"invoices/booked/{booked_invoice_number}" - - def __is_missing_booked_invoice_number(self, response: requests.Response) -> bool: - try: - response.raise_for_status() - except requests.HTTPError as exc: - response_json = response.json() - if "error_code" in response_json and response_json.get("error_code") == "NO_SUCH_BOOKED_INVOICE_NUMBER": - self.logger.info(response.text) - return True - else: - self.logger.error(response.text) - raise exc - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - if not self.__is_missing_booked_invoice_number(response): - yield response.json() - - -class SourceVismaEconomic(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - """ - :param config: the user-input config object conforming to the connector's spec.yaml - :param logger: logger object - :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. - """ - - try: - stream = Accounts(**config) - stream.page_size = 1 - _ = list(stream.read_records(sync_mode=SyncMode.full_refresh)) - return True, None - except Exception as e: - logger.error(e) - return False, repr(e) - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - stream_list = [ - Accounts(**config), - Customers(**config), - InvoicesBooked(**config), - InvoicesPaid(**config), - InvoicesTotal(**config), - Products(**config), - InvoicesBookedDocument(**config), - ] - - return stream_list +# Declarative Source +class SourceVismaEconomic(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/spec.yaml b/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/spec.yaml deleted file mode 100644 index c0c56c93ba7a..000000000000 --- a/airbyte-integrations/connectors/source-visma-economic/source_visma_economic/spec.yaml +++ /dev/null @@ -1,19 +0,0 @@ -documentationUrl: https://docs.airbyte.com/integrations/sources/visma-economic -connectionSpecification: - $schema: http://json-schema.org/draft-07/schema# - title: Visma E-conomic Spec - type: object - required: - - app_secret_token - - agreement_grant_token - properties: - app_secret_token: - title: App Secret Token - type: string - description: Identification token for app accessing data - airbyte_secret: true - agreement_grant_token: - title: Agreement Grant Token - type: string - description: Identifier for the grant issued by an agreement - airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-visma-economic/unit_tests/__init__.py b/airbyte-integrations/connectors/source-visma-economic/unit_tests/__init__.py deleted file mode 100644 index 46b7376756ec..000000000000 --- a/airbyte-integrations/connectors/source-visma-economic/unit_tests/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-integrations/connectors/source-visma-economic/unit_tests/test_source.py b/airbyte-integrations/connectors/source-visma-economic/unit_tests/test_source.py deleted file mode 100644 index 4cece02d3354..000000000000 --- a/airbyte-integrations/connectors/source-visma-economic/unit_tests/test_source.py +++ /dev/null @@ -1,26 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -import responses -from source_visma_economic.source import SourceVismaEconomic - - -@responses.activate -def test_check_connection(mocker): - responses.add(responses.GET, "https://restapi.e-conomic.com/accounts?skippages=0&pagesize=1", json={"collection": []}) - - source = SourceVismaEconomic() - logger_mock, config_mock = MagicMock(), MagicMock() - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -def test_streams(mocker): - source = SourceVismaEconomic() - config_mock = MagicMock() - streams = source.streams(config_mock) - # TODO: replace this with your streams number - expected_streams_number = 7 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-visma-economic/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-visma-economic/unit_tests/test_streams.py deleted file mode 100644 index 7c6d4962febb..000000000000 --- a/airbyte-integrations/connectors/source-visma-economic/unit_tests/test_streams.py +++ /dev/null @@ -1,92 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -import requests -from source_visma_economic.source import VismaEconomicStream - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(VismaEconomicStream, "path", "v0/example_endpoint") - mocker.patch.object(VismaEconomicStream, "primary_key", "test_primary_key") - mocker.patch.object(VismaEconomicStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = VismaEconomicStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {"pagesize": 1000, "skippages": 0} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class): - stream = VismaEconomicStream() - response = MagicMock(requests.Response) - json = { - "pagination": { - "maxPageSizeAllowed": 1000, - "skipPages": 0, - "pageSize": 100, - "results": 200, - "resultsWithoutFilter": 200, - "firstPage": "https://restapi.e-conomic.com/stream?skippages=0&pagesize=100", - "nextPage": "https://restapi.e-conomic.com/stream?skippages=1&pagesize=100", - "lastPage": "https://restapi.e-conomic.com/stream?skippages=1&pagesize=100", - } - } - response.json = MagicMock(return_value=json) - inputs = {"response": response} - - expected_token = {"skippages": ["1"], "pagesize": ["100"]} - assert stream.next_page_token(**inputs) == expected_token - - -def test_no_next_page_token(patch_base_class): - stream = VismaEconomicStream() - response = MagicMock(requests.Response) - response.json = MagicMock(return_value={}) - inputs = {"response": response} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_request_headers(patch_base_class): - stream = VismaEconomicStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {"X-AgreementGrantToken": None, "X-AppSecretToken": None} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(patch_base_class): - stream = VismaEconomicStream() - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = VismaEconomicStream() - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = VismaEconomicStream() - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-vitally/README.md b/airbyte-integrations/connectors/source-vitally/README.md index c0990380f1fa..db65f39f5b6a 100644 --- a/airbyte-integrations/connectors/source-vitally/README.md +++ b/airbyte-integrations/connectors/source-vitally/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-vitally:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/vitally) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_vitally/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-vitally:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-vitally build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-vitally:airbyteDocker +An image will be built with the tag `airbyte/source-vitally:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-vitally:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-vitally:dev check --co docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-vitally:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-vitally:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-vitally test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-vitally:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-vitally:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-vitally test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/vitally.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-vitally/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-vitally/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-vitally/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-vitally/build.gradle b/airbyte-integrations/connectors/source-vitally/build.gradle deleted file mode 100644 index 02bcdac3d9eb..000000000000 --- a/airbyte-integrations/connectors/source-vitally/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_vitally' -} diff --git a/airbyte-integrations/connectors/source-waiteraid/README.md b/airbyte-integrations/connectors/source-waiteraid/README.md index 6877dcf249fa..c7d29eb67e18 100644 --- a/airbyte-integrations/connectors/source-waiteraid/README.md +++ b/airbyte-integrations/connectors/source-waiteraid/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-waiteraid:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/waiteraid) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_waiteraid/spec.yaml` file. @@ -56,18 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-waiteraid:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-waiteraid build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-waiteraid:airbyteDocker +An image will be built with the tag `airbyte/source-waiteraid:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-waiteraid:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -77,43 +70,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-waiteraid:dev check -- docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-waiteraid:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-waiteraid:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-waiteraid test ``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-waiteraid:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-waiteraid:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -122,8 +89,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-waiteraid test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/waiteraid.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-waiteraid/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-waiteraid/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-waiteraid/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-waiteraid/build.gradle b/airbyte-integrations/connectors/source-waiteraid/build.gradle deleted file mode 100644 index 820d7f19dcbc..000000000000 --- a/airbyte-integrations/connectors/source-waiteraid/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_waiteraid' -} diff --git a/airbyte-integrations/connectors/source-weatherstack/README.md b/airbyte-integrations/connectors/source-weatherstack/README.md index 26fff4155a22..531fb57a99eb 100644 --- a/airbyte-integrations/connectors/source-weatherstack/README.md +++ b/airbyte-integrations/connectors/source-weatherstack/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-weatherstack:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/weatherstack) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_weatherstack/spec.yaml` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-weatherstack:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-weatherstack build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-weatherstack:airbyteDocker +An image will be built with the tag `airbyte/source-weatherstack:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-weatherstack:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-weatherstack:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-weatherstack:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-weatherstack:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-weatherstack test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-weatherstack:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-weatherstack:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-weatherstack test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/weatherstack.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-weatherstack/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-weatherstack/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-weatherstack/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-weatherstack/build.gradle b/airbyte-integrations/connectors/source-weatherstack/build.gradle deleted file mode 100644 index 180cc4231fa1..000000000000 --- a/airbyte-integrations/connectors/source-weatherstack/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_weatherstack' -} diff --git a/airbyte-integrations/connectors/source-webflow/Dockerfile b/airbyte-integrations/connectors/source-webflow/Dockerfile index c5d8ef311742..487098ab5c11 100644 --- a/airbyte-integrations/connectors/source-webflow/Dockerfile +++ b/airbyte-integrations/connectors/source-webflow/Dockerfile @@ -34,5 +34,5 @@ COPY source_webflow ./source_webflow ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/source-webflow diff --git a/airbyte-integrations/connectors/source-webflow/README.md b/airbyte-integrations/connectors/source-webflow/README.md index c19fdf5b8221..807cec095b47 100644 --- a/airbyte-integrations/connectors/source-webflow/README.md +++ b/airbyte-integrations/connectors/source-webflow/README.md @@ -10,7 +10,7 @@ A detailed tutorial has been written about this implementation. See: [Build a co ## Local development ### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** +- Webflow v1 API Key #### Minimum Python version required `= 3.9.11` @@ -34,14 +34,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-webflow:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/webflow) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_webflow/spec.yaml` file. @@ -63,19 +55,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image. Execute the following from -the source-webflow project directory (where Dockerfile can be found): -``` -docker build . -t airbyte/source-webflow:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-webflow build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-webflow:airbyteDocker +An image will be built with the tag `airbyte/source-webflow:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-webflow:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -85,49 +77,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-webflow:dev check --co docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-webflow:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-webflow:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -Or if you are running in OSX with zsh, you may need to execute the following instead -``` -pip install .'[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-webflow test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-webflow:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-webflow:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -137,9 +96,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-webflow test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/webflow.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-webflow/acceptance-test-config.yml b/airbyte-integrations/connectors/source-webflow/acceptance-test-config.yml index 4b5c6775e316..172751e263c1 100644 --- a/airbyte-integrations/connectors/source-webflow/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-webflow/acceptance-test-config.yml @@ -15,5 +15,3 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - - diff --git a/airbyte-integrations/connectors/source-webflow/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-webflow/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-webflow/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-webflow/build.gradle b/airbyte-integrations/connectors/source-webflow/build.gradle deleted file mode 100644 index 2f59b3cde4e1..000000000000 --- a/airbyte-integrations/connectors/source-webflow/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_webflow' -} diff --git a/airbyte-integrations/connectors/source-webflow/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-webflow/integration_tests/configured_catalog.json index c2887e81f620..3891ee24dd07 100644 --- a/airbyte-integrations/connectors/source-webflow/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-webflow/integration_tests/configured_catalog.json @@ -2,8 +2,9 @@ "streams": [ { "stream": { - "name": "Blog Authors", - "json_schema": {} + "name": "Ebooks", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" diff --git a/airbyte-integrations/connectors/source-webflow/metadata.yaml b/airbyte-integrations/connectors/source-webflow/metadata.yaml index 2f37cd559298..0ff0f88db000 100644 --- a/airbyte-integrations/connectors/source-webflow/metadata.yaml +++ b/airbyte-integrations/connectors/source-webflow/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: api connectorType: source definitionId: ef580275-d9a9-48bb-af5e-db0f5855be04 - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 dockerRepository: airbyte/source-webflow githubIssueLabel: source-webflow icon: webflow.svg diff --git a/airbyte-integrations/connectors/source-webflow/setup.py b/airbyte-integrations/connectors/source-webflow/setup.py index 8c550837d124..6912dcae7d7c 100644 --- a/airbyte-integrations/connectors/source-webflow/setup.py +++ b/airbyte-integrations/connectors/source-webflow/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ diff --git a/airbyte-integrations/connectors/source-webflow/source_webflow/auth.py b/airbyte-integrations/connectors/source-webflow/source_webflow/auth.py index 722c06f4080d..132db64bf5b8 100644 --- a/airbyte-integrations/connectors/source-webflow/source_webflow/auth.py +++ b/airbyte-integrations/connectors/source-webflow/source_webflow/auth.py @@ -23,6 +23,6 @@ def get_auth_header(self) -> Mapping[str, Any]: class WebflowTokenAuthenticator(WebflowAuthMixin, TokenAuthenticator): """ - Auth class for Personal Access Token - https://help.getharvest.com/api-v2/authentication-api/authentication/authentication/#personal-access-tokens + Authentication class information + https://docs.developers.webflow.com/reference/authorization """ diff --git a/airbyte-integrations/connectors/source-webflow/source_webflow/source.py b/airbyte-integrations/connectors/source-webflow/source_webflow/source.py index 21f7d0bc63dd..f0209639ae09 100644 --- a/airbyte-integrations/connectors/source-webflow/source_webflow/source.py +++ b/airbyte-integrations/connectors/source-webflow/source_webflow/source.py @@ -61,10 +61,10 @@ def request_params( class CollectionSchema(WebflowStream): """ - Gets the schema of the current collection - see: https://developers.webflow.com/#get-collection-with-full-schema, and + Gets the schema of the current collection - see: https://docs.developers.webflow.com/v1.0.0/reference/get-collection, and then converts that schema to a json-schema.org-compatible schema that uses supported Airbyte types. - More info about Webflow schema: https://developers.webflow.com/#get-collection-with-full-schema + More info about Webflow schema: https://docs.developers.webflow.com/v1.0.0/reference/get-collection Airbyte data types: https://docs.airbyte.com/understanding-airbyte/supported-data-types/ """ @@ -77,8 +77,8 @@ def __init__(self, collection_id: str = None, **kwargs): def path(self, **kwargs) -> str: """ - See: https://developers.webflow.com/#list-collections - Returns a list which contains high-level information about each collection. + See: https://docs.developers.webflow.com/v1.0.0/reference/get-collection + Returns a collection with full schema by collection_id """ path = f"collections/{self.collection_id}" @@ -243,6 +243,7 @@ def get_json_schema(self) -> Mapping[str, Any]: extra_fields = { "_id": {"type": ["null", "string"]}, "_cid": {"type": ["null", "string"]}, + "_locale": {"type": ["null", "string"]}, } json_schema.update(extra_fields) @@ -283,7 +284,7 @@ def get_authenticator(config): which overloads that standard authentication to include additional headers that are required by Webflow. """ api_key = config.get("api_key", None) - accept_version = WEBFLOW_ACCEPT_VERSION + accept_version = config.get("accept_version", WEBFLOW_ACCEPT_VERSION) if not api_key: raise Exception("Config validation error: 'api_key' is a required property") diff --git a/airbyte-integrations/connectors/source-webflow/source_webflow/spec.yaml b/airbyte-integrations/connectors/source-webflow/source_webflow/spec.yaml index 0fd66a820c1a..5d3be8b1984d 100644 --- a/airbyte-integrations/connectors/source-webflow/source_webflow/spec.yaml +++ b/airbyte-integrations/connectors/source-webflow/source_webflow/spec.yaml @@ -6,7 +6,7 @@ connectionSpecification: required: - api_key - site_id - additionalProperties: false + additionalProperties: true properties: site_id: title: Site id @@ -21,3 +21,9 @@ connectionSpecification: example: "a very long hex sequence" order: 1 airbyte_secret: true + accept_version: + title: Accept Version + type: string + description: "The version of the Webflow API to use. See https://developers.webflow.com/#versioning" + example: "1.0.0" + order: 2 diff --git a/airbyte-integrations/connectors/source-webflow/source_webflow/webflow_to_airbyte_mapping.py b/airbyte-integrations/connectors/source-webflow/source_webflow/webflow_to_airbyte_mapping.py index 39c3e709b4ff..ea40dc0ab320 100644 --- a/airbyte-integrations/connectors/source-webflow/source_webflow/webflow_to_airbyte_mapping.py +++ b/airbyte-integrations/connectors/source-webflow/source_webflow/webflow_to_airbyte_mapping.py @@ -30,4 +30,5 @@ class WebflowToAirbyteMapping: "RichText": {"type": ["null", "string"]}, "User": {"type": ["null", "string"]}, "Video": {"type": ["null", "string"]}, + "FileRef": {"type": ["null", "object"]}, } diff --git a/airbyte-integrations/connectors/source-whisky-hunter/README.md b/airbyte-integrations/connectors/source-whisky-hunter/README.md index 670da14b0c68..48a699198d28 100644 --- a/airbyte-integrations/connectors/source-whisky-hunter/README.md +++ b/airbyte-integrations/connectors/source-whisky-hunter/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-whisky-hunter:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/whisky-hunter) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_whisky_hunter/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-whisky-hunter:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-whisky-hunter build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-whisky-hunter:airbyteDocker +An image will be built with the tag `airbyte/source-whisky-hunter:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-whisky-hunter:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-whisky-hunter:dev chec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-whisky-hunter:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-whisky-hunter:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-whisky-hunter test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-whisky-hunter:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-whisky-hunter:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-whisky-hunter test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/whisky-hunter.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-whisky-hunter/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-whisky-hunter/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-whisky-hunter/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-whisky-hunter/build.gradle b/airbyte-integrations/connectors/source-whisky-hunter/build.gradle deleted file mode 100644 index 591b9d144d57..000000000000 --- a/airbyte-integrations/connectors/source-whisky-hunter/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_whisky_hunter' -} diff --git a/airbyte-integrations/connectors/source-wikipedia-pageviews/README.md b/airbyte-integrations/connectors/source-wikipedia-pageviews/README.md index c19bbb5bde63..6ffa64ddd6e2 100755 --- a/airbyte-integrations/connectors/source-wikipedia-pageviews/README.md +++ b/airbyte-integrations/connectors/source-wikipedia-pageviews/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-wikipedia-pageviews:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/wikipedia-pageviews) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_wikipedia_pageviews/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-wikipedia-pageviews:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-wikipedia-pageviews build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-wikipedia-pageviews:airbyteDocker +An image will be built with the tag `airbyte/source-wikipedia-pageviews:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-wikipedia-pageviews:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-wikipedia-pageviews:de docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-wikipedia-pageviews:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-wikipedia-pageviews:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-wikipedia-pageviews test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-wikipedia-pageviews:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-wikipedia-pageviews:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-wikipedia-pageviews test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/wikipedia-pageviews.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-wikipedia-pageviews/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-wikipedia-pageviews/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-wikipedia-pageviews/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-wikipedia-pageviews/build.gradle b/airbyte-integrations/connectors/source-wikipedia-pageviews/build.gradle deleted file mode 100755 index b3356aafea73..000000000000 --- a/airbyte-integrations/connectors/source-wikipedia-pageviews/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_wikipedia_pageviews' -} diff --git a/airbyte-integrations/connectors/source-woocommerce/README.md b/airbyte-integrations/connectors/source-woocommerce/README.md index 0f0c6010cfaf..153cbfd90c59 100644 --- a/airbyte-integrations/connectors/source-woocommerce/README.md +++ b/airbyte-integrations/connectors/source-woocommerce/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-woocommerce:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/woocommerce) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_woocommerce/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-woocommerce:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-woocommerce build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-woocommerce:airbyteDocker +An image will be built with the tag `airbyte/source-woocommerce:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-woocommerce:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-woocommerce:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-woocommerce:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-woocommerce:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-woocommerce test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-woocommerce:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-woocommerce:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-woocommerce test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/woocommerce.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-woocommerce/acceptance-test-config.yml b/airbyte-integrations/connectors/source-woocommerce/acceptance-test-config.yml index 7acd33c6dcdf..886de23dc003 100644 --- a/airbyte-integrations/connectors/source-woocommerce/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-woocommerce/acceptance-test-config.yml @@ -4,34 +4,34 @@ connector_image: airbyte/source-woocommerce:dev acceptance_tests: spec: tests: - - spec_path: "source_woocommerce/spec.yaml" + - spec_path: "source_woocommerce/spec.yaml" connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" - timeout_seconds: 300 + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + timeout_seconds: 300 discovery: tests: - - config_path: "secrets/config.json" - backward_compatibility_tests_config: - disable_for_version: "0.1.2" + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.2" basic_read: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 3600 - expect_records: - path: "integration_tests/expected_records.jsonl" - fail_on_extra_columns: false + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 3600 + expect_records: + path: "integration_tests/expected_records.jsonl" + fail_on_extra_columns: false incremental: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-woocommerce/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-woocommerce/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-woocommerce/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-woocommerce/build.gradle b/airbyte-integrations/connectors/source-woocommerce/build.gradle deleted file mode 100644 index 236fc10758d6..000000000000 --- a/airbyte-integrations/connectors/source-woocommerce/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_woocommerce' -} diff --git a/airbyte-integrations/connectors/source-woocommerce/metadata.yaml b/airbyte-integrations/connectors/source-woocommerce/metadata.yaml index 1415b8c68efc..9e34f36b4e55 100644 --- a/airbyte-integrations/connectors/source-woocommerce/metadata.yaml +++ b/airbyte-integrations/connectors/source-woocommerce/metadata.yaml @@ -1,4 +1,7 @@ data: + ab_internal: + ql: 300 + sl: 100 allowedHosts: hosts: - ${domain} @@ -7,6 +10,7 @@ data: definitionId: 2a2552ca-9f78-4c1c-9eb7-4d0dc66d72df dockerImageTag: 0.2.3 dockerRepository: airbyte/source-woocommerce + documentationUrl: https://docs.airbyte.com/integrations/sources/woocommerce githubIssueLabel: source-woocommerce icon: woocommerce.svg license: MIT @@ -17,12 +21,8 @@ data: oss: enabled: true releaseStage: beta - documentationUrl: https://docs.airbyte.com/integrations/sources/woocommerce + supportLevel: community tags: - language:low-code - language:python - ab_internal: - sl: 200 - ql: 300 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-workable/README.md b/airbyte-integrations/connectors/source-workable/README.md index c5ca0894e104..183ef638217c 100644 --- a/airbyte-integrations/connectors/source-workable/README.md +++ b/airbyte-integrations/connectors/source-workable/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-workable:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/workable) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_workable/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-workable:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-workable build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-workable:airbyteDocker +An image will be built with the tag `airbyte/source-workable:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-workable:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-workable:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-workable:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-workable:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-workable test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-workable:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-workable:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-workable test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/workable.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-workable/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-workable/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-workable/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-workable/build.gradle b/airbyte-integrations/connectors/source-workable/build.gradle deleted file mode 100644 index 3367afb130cd..000000000000 --- a/airbyte-integrations/connectors/source-workable/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_workable' -} diff --git a/airbyte-integrations/connectors/source-workramp/README.md b/airbyte-integrations/connectors/source-workramp/README.md index e9a496631aea..3382c28a060b 100644 --- a/airbyte-integrations/connectors/source-workramp/README.md +++ b/airbyte-integrations/connectors/source-workramp/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-workramp:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/workramp) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_workramp/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-workramp:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-workramp build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-workramp:airbyteDocker +An image will be built with the tag `airbyte/source-workramp:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-workramp:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-workramp:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-workramp:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-workramp:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-workramp test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-workramp:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-workramp:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-workramp test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/workramp.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-workramp/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-workramp/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-workramp/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-workramp/build.gradle b/airbyte-integrations/connectors/source-workramp/build.gradle deleted file mode 100644 index f021cccc88b7..000000000000 --- a/airbyte-integrations/connectors/source-workramp/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_workramp' -} diff --git a/airbyte-integrations/connectors/source-wrike/Dockerfile b/airbyte-integrations/connectors/source-wrike/Dockerfile index eae358751af0..3a494af37721 100644 --- a/airbyte-integrations/connectors/source-wrike/Dockerfile +++ b/airbyte-integrations/connectors/source-wrike/Dockerfile @@ -34,5 +34,5 @@ COPY source_wrike ./source_wrike ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-wrike diff --git a/airbyte-integrations/connectors/source-wrike/README.md b/airbyte-integrations/connectors/source-wrike/README.md index 5773325e756d..25b2da5fd3ce 100644 --- a/airbyte-integrations/connectors/source-wrike/README.md +++ b/airbyte-integrations/connectors/source-wrike/README.md @@ -1,89 +1,34 @@ # Wrike Source -This is the repository for the Wrike source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/wrike). +This is the repository for the Wrike configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/wrike). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-wrike:build -``` - -### Create credentials - -#### Generating access token - -First get a Wrike account, you can get a trial here: Register for trial: https://www.wrike.com/free-trial/ - -To generate a token: - -1. Navigate to the ‘API apps’ section. -2. Select the required application from the list. -3. Click ‘Configure’. -4. Click ‘Obtain token’ in the “Permanent access token” section. -5. You will see a pop-up with a warning about using the permanent token, and after confirming, you will be able to copy and paste it into a secure storage (Wrike will not display it again). If your permanent token gets lost, you should generate a new one. - - -### Configuration files - -Then create a file `secrets/config.json` conforming to the `source_wrike/spec.yaml` file. +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/wrike) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_wrike/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source wrike test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-wrike:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-wrike build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-wrike:airbyteDocker +An image will be built with the tag `airbyte/source-wrike:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-wrike:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -93,44 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-wrike:dev check --conf docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-wrike:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-wrike:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-wrike test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-wrike:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-wrike:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -140,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-wrike test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/wrike.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-wrike/__init__.py b/airbyte-integrations/connectors/source-wrike/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-wrike/acceptance-test-config.yml b/airbyte-integrations/connectors/source-wrike/acceptance-test-config.yml index bd5daa45a4a6..05e94454ef6a 100644 --- a/airbyte-integrations/connectors/source-wrike/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-wrike/acceptance-test-config.yml @@ -1,20 +1,43 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-wrike:dev -tests: +acceptance_tests: spec: - - spec_path: "source_wrike/spec.yaml" + tests: + - spec_path: "source_wrike/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: comments + bypass_reason: "Sandbox accounts dont have permission for accessing the stream" + - name: workflows + bypass_reason: "Sandbox accounts can't seed the stream" + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.jsonl" + # extra_fields: no + # exact_order: no + # extra_records: yes + incremental: + bypass_reason: "This connector does not implement incremental sync" + # TODO uncomment this block this block if your connector implements incremental sync: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-wrike/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-wrike/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-wrike/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-wrike/bootstrap.md b/airbyte-integrations/connectors/source-wrike/bootstrap.md deleted file mode 100644 index 1ba4e4fdcee2..000000000000 --- a/airbyte-integrations/connectors/source-wrike/bootstrap.md +++ /dev/null @@ -1,28 +0,0 @@ -# Wrike - -The connector uses the v4 API documented here: https://developers.wrike.com/overview/ . It is -straightforward HTTP REST API with Bearer token authentication. - -## Generating access token - -First get a Wrike account, you can get a trial here: Register for trial: https://www.wrike.com/free-trial/ - -To generate a token: - -1. Navigate to the ‘API apps’ section. -2. Select the required application from the list. -3. Click ‘Configure’. -4. Click ‘Obtain token’ in the “Permanent access token” section. -5. You will see a pop-up with a warning about using the permanent token, and after confirming, you will be able to copy and paste it into a secure storage (Wrike will not display it again). If your permanent token gets lost, you should generate a new one. - -Auth is done by TokenAuthenticator. - -## Implementation details - -I wrote a longer blog post on the implementation details: https://medium.com/starschema-blog/extending-airbyte-creating-a-source-connector-for-wrike-8e6c1337365a - -In a nutshell: - * We use only GET methods, all endpoints are straightforward. We emit what we receive as HTTP response. - * The `comments` endpoint is the only trick one: it allows only 7 days data to be retrieved once. Thus, the codes creates 7 days slices, starting from start replication date. - * It uses cursor based pagination. If you provide the next_page_token then no other parameters needed, the API will assume all parameters from the first page. - diff --git a/airbyte-integrations/connectors/source-wrike/build.gradle b/airbyte-integrations/connectors/source-wrike/build.gradle deleted file mode 100644 index 28757d1855be..000000000000 --- a/airbyte-integrations/connectors/source-wrike/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_wrike' -} diff --git a/airbyte-integrations/connectors/source-wrike/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-wrike/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..8e2fc74c73c0 --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/integration_tests/abnormal_state.json @@ -0,0 +1,35 @@ +[ + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "comments" + }, + "stream_state": { + "updatedDate": "2099-01-01T00:00:00" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "folders" + }, + "stream_state": { + "createdDate": "2099-01-01T00:00:00" + } + } + }, + { + "type": "STREAM", + "stream": { + "stream_descriptor": { + "name": "tasks" + }, + "stream_state": { + "createdDate": "2099-01-01T00:00:00" + } + } + } +] diff --git a/airbyte-integrations/connectors/source-wrike/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-wrike/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-wrike/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-wrike/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-wrike/integration_tests/catalog.json b/airbyte-integrations/connectors/source-wrike/integration_tests/catalog.json deleted file mode 100644 index 0bbacc5360a5..000000000000 --- a/airbyte-integrations/connectors/source-wrike/integration_tests/catalog.json +++ /dev/null @@ -1,320 +0,0 @@ -{ - "streams": [ - [ - { - "name": "tasks", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "accountId": { - "type": "string" - }, - "title": { - "type": "string" - }, - "status": { - "type": "string" - }, - "importance": { - "type": ["null", "string"] - }, - "scope": { - "type": ["null", "string"] - }, - "briefDescription": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "customStatusId": { - "type": ["null", "string"] - }, - "permalink": { - "type": ["null", "string"] - }, - "priority": { - "type": ["null", "string"] - }, - "createdDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "responsibleIds": { - "type": ["array", "null"], - "items": { - "type": ["string", "null"] - } - }, - "parentIds": { - "type": ["array", "null"], - "items": { - "type": ["string", "null"] - } - }, - "superTaskIds": { - "type": ["array", "null"], - "items": { - "type": ["string", "null"] - } - }, - "authorIds": { - "type": ["array", "null"], - "items": { - "type": ["string", "null"] - } - }, - "customFields": { - "type": ["array", "null"], - "items": { - "type": ["object", "null"], - "properties": { - "id": { - "type": ["string"] - }, - "value": { - "type": ["string", "null"] - } - } - } - }, - "dates": { - "type": ["object", "null"], - "properties": { - "type": { - "type": ["string", "null"] - } - } - }, - "updatedDate": { - "type": ["null", "string"], - "format": "date-time" - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - }, - { - "name": "customfields", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "accountId": { - "type": ["null", "string"] - }, - "type": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "spaceId": { - "type": ["null", "string"] - }, - "sharedIds": { - "type": ["array", "null"], - "items": { - "type": ["string", "null"] - } - }, - "settings": { - "type": ["object", "null"], - "properties": { - "inheritanceType": { - "type": ["string", "null"] - }, - "values": { - "type": ["array", "null"], - "items": { - "type": ["string", "integer", "number", "boolean", "null"] - } - }, - "decimalPlaces": { - "type": ["integer", "null"] - }, - "useThousandsSeparator": { - "type": ["boolean", "null"] - }, - "readOnly": { - "type": ["boolean", "null"] - }, - "allowTime": { - "type": ["boolean", "null"] - }, - "allowOtherValues": { - "type": ["boolean", "null"] - }, - "aggregation": { - "type": ["string", "null"] - }, - "currency": { - "type": ["string", "null"] - } - } - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - }, - { - "name": "contacts", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "firstName": { - "type": ["null", "string"] - }, - "lastName": { - "type": ["null", "string"] - }, - "avatarUrl": { - "type": ["null", "string"] - }, - "timezone": { - "type": ["null", "string"] - }, - "locale": { - "type": ["null", "string"] - }, - "deleted": { - "type": ["null", "boolean"] - }, - "type": { - "type": ["null", "string"] - }, - "profiles": { - "type": ["array", "null"], - "items": { - "type": ["object", "null"], - "properties": { - "accountId": { - "type": ["string"] - }, - "role": { - "type": ["string"] - }, - "external": { - "type": ["boolean", "null"] - }, - "admin": { - "type": ["boolean", "null"] - }, - "owner": { - "type": ["boolean", "null"] - }, - "email": { - "type": ["string", "null"] - } - } - } - }, - "title": { - "type": ["null", "string"] - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - }, - { - "name": "folders", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "scope": { - "type": ["null", "string"] - }, - "customStatusId": { - "type": ["null", "string"] - }, - "createdDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "childIds": { - "type": ["array", "null"], - "items": { - "type": ["string", "null"] - } - }, - "title": { - "type": ["null", "string"] - }, - "project": { - "type": ["object", "null"], - "properties": { - "authorId": { - "type": ["string", "null"] - }, - "customStatusId": { - "type": ["string", "null"] - }, - "createdDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "ownerIds": { - "type": ["array", "null"], - "items": { - "type": ["string", "null"] - } - } - } - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - }, - { - "name": "comments", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "authorId": { - "type": ["null", "string"] - }, - "text": { - "type": ["null", "string"] - }, - "taskId": { - "type": ["null", "string"] - }, - "createdDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "updatedDate": { - "type": ["null", "string"], - "format": "date-time" - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] - } - ] - ] -} diff --git a/airbyte-integrations/connectors/source-wrike/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-wrike/integration_tests/configured_catalog.json index 350d75ef56f1..ab5643e89b8d 100644 --- a/airbyte-integrations/connectors/source-wrike/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-wrike/integration_tests/configured_catalog.json @@ -3,37 +3,8 @@ { "stream": { "name": "tasks", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "accountId": { - "type": "string" - }, - "title": { - "type": "string" - }, - "status": { - "type": "string" - }, - "importance": { - "type": ["null", "string"] - }, - "createdDate": { - "type": ["null", "string"], - "format": "date-time" - }, - "updatedDate": { - "type": ["null", "string"], - "format": "date-time" - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -41,27 +12,8 @@ { "stream": { "name": "customfields", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "years_of_service": { - "type": ["null", "integer"] - }, - "start_date": { - "type": ["null", "string"], - "format": "date-time" - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -69,20 +21,8 @@ { "stream": { "name": "folders", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "accountId": { - "type": ["null", "string"] - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -90,20 +30,8 @@ { "stream": { "name": "contacts", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "firstName": { - "type": ["null", "string"] - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" @@ -111,20 +39,17 @@ { "stream": { "name": "comments", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "text": { - "type": ["null", "string"] - } - } - }, - "supported_sync_modes": ["full_refresh"], - "source_defined_primary_key": [["id"]] + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "workflows", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" diff --git a/airbyte-integrations/connectors/source-wrike/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-wrike/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-wrike/metadata.yaml b/airbyte-integrations/connectors/source-wrike/metadata.yaml index eae619a4402a..7a6309f77cb9 100644 --- a/airbyte-integrations/connectors/source-wrike/metadata.yaml +++ b/airbyte-integrations/connectors/source-wrike/metadata.yaml @@ -1,24 +1,27 @@ data: + allowedHosts: + hosts: + - app-us*.wrike.com + - app-eu*.wrike.com + - www.wrike.com + registries: + oss: + enabled: true + cloud: + enabled: false connectorSubtype: api connectorType: source definitionId: 9c13f986-a13b-4988-b808-4705badf71c2 - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-wrike githubIssueLabel: source-wrike icon: wrike.svg license: MIT name: Wrike - registries: - cloud: - enabled: false - oss: - enabled: true + releaseDate: 2023-10-10 releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/wrike tags: - - language:python - ab_internal: - sl: 100 - ql: 100 - supportLevel: community + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-wrike/setup.py b/airbyte-integrations/connectors/source-wrike/setup.py index d41b776a27b3..a3c314147ca7 100644 --- a/airbyte-integrations/connectors/source-wrike/setup.py +++ b/airbyte-integrations/connectors/source-wrike/setup.py @@ -9,11 +9,7 @@ "airbyte-cdk~=0.1", ] -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest~=6.1", - "pytest-mock~=3.6.1", -] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( name="source_wrike", diff --git a/airbyte-integrations/connectors/source-wrike/source_wrike/manifest.yaml b/airbyte-integrations/connectors/source-wrike/source_wrike/manifest.yaml new file mode 100644 index 000000000000..ae388543e80e --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/source_wrike/manifest.yaml @@ -0,0 +1,99 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["data"] + + requester: + type: HttpRequester + url_base: "https://{{ config['wrike_instance'] }}/api/v4/" + http_method: "GET" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['access_token'] }}" + + base_paginator: + type: "DefaultPaginator" + pagination_strategy: + type: "CursorPagination" + cursor_value: "{{ last_records['nextPageToken'] }}" + page_token_option: + type: "RequestPath" + field_name: "nextPageToken" + inject_into: "request_parameter" + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + $ref: "#/definitions/base_paginator" + requester: + $ref: "#/definitions/requester" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + customfields_stream: + $ref: "#/definitions/base_stream" + name: "customfields" + primary_key: "id" + $parameters: + path: "customfields" + + contacts_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "contacts" + path: "contacts" + + workflows_stream: + $ref: "#/definitions/base_stream" + name: "workflows" + primary_key: "id" + $parameters: + path: "workflows" + + comments_stream: + $ref: "#/definitions/base_stream" + name: "comments" + primary_key: "id" + $parameters: + path: "comments" + + folders_stream: + $ref: "#/definitions/base_stream" + name: "folders" + primary_key: "id" + $parameters: + path: "folders" + + tasks_stream: + $ref: "#/definitions/base_stream" + name: "tasks" + primary_key: "id" + $parameters: + path: "tasks" + +streams: + - "#/definitions/customfields_stream" + - "#/definitions/contacts_stream" + - "#/definitions/workflows_stream" + - "#/definitions/comments_stream" + - "#/definitions/folders_stream" + - "#/definitions/tasks_stream" + +check: + type: CheckStream + stream_names: + - "customfields" + - "contacts" + - "folders" + - "tasks" + - "workflows" diff --git a/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/comments.json b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/comments.json index 1a5d17b5d3ae..af92c8e4fe41 100644 --- a/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/comments.json +++ b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/comments.json @@ -1,21 +1,22 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", + "type": ["object", "null"], + "additionalProperties": true, "properties": { "id": { - "type": "string" + "type": ["string", "null"] }, "authorId": { - "type": "string" + "type": ["string", "null"] }, "text": { - "type": "string" + "type": ["string", "null"] }, "taskId": { - "type": "string" + "type": ["string", "null"] }, "createdDate": { - "type": "string", + "type": ["string", "null"], "format": "date-time" }, "updatedDate": { diff --git a/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/contacts.json b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/contacts.json index 08709d869df3..4e21e1cffffd 100644 --- a/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/contacts.json @@ -1,41 +1,42 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", + "type": ["object", "null"], + "additionalProperties": true, "properties": { "id": { - "type": "string" + "type": ["string", "null"] }, "firstName": { - "type": "string" + "type": ["string", "null"] }, "lastName": { - "type": "string" + "type": ["string", "null"] }, "avatarUrl": { - "type": "string" + "type": ["string", "null"] }, "timezone": { - "type": "string" + "type": ["string", "null"] }, "locale": { - "type": "string" + "type": ["string", "null"] }, "deleted": { - "type": "boolean" + "type": ["boolean", "null"] }, "type": { - "type": "string" + "type": ["string", "null"] }, "profiles": { - "type": "array", + "type": ["array", "null"], "items": { "type": ["object", "null"], "properties": { "accountId": { - "type": "string" + "type": ["string", "null"] }, "role": { - "type": "string" + "type": ["string", "null"] }, "external": { "type": ["boolean", "null"] @@ -54,6 +55,24 @@ }, "title": { "type": ["null", "string"] + }, + "me": { + "type": ["null", "boolean"] + }, + "companyName": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "myTeam": { + "type": ["null", "boolean"] + }, + "memberIds": { + "type": ["array", "null"], + "items": { + "type": ["string", "null"] + } } } } diff --git a/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/customfields.json b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/customfields.json index e31b380b4d1e..059188f1a194 100644 --- a/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/customfields.json +++ b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/customfields.json @@ -1,26 +1,27 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", + "type": ["object", "null"], + "additionalProperties": true, "properties": { "id": { - "type": "string" + "type": ["string", "null"] }, "accountId": { - "type": "string" + "type": ["string", "null"] }, "type": { - "type": "string" + "type": ["string", "null"] }, "title": { - "type": "string" + "type": ["string", "null"] }, "spaceId": { "type": ["null", "string"] }, "sharedIds": { - "type": "array", + "type": ["array", "null"], "items": { - "type": "string" + "type": ["string", "null"] } }, "settings": { diff --git a/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/folders.json b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/folders.json index 77a093303914..58d16333cc13 100644 --- a/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/folders.json +++ b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/folders.json @@ -1,12 +1,13 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", + "type": ["object", "null"], + "additionalProperties": true, "properties": { "id": { - "type": "string" + "type": ["string", "null"] }, "scope": { - "type": "string" + "type": ["string", "null"] }, "createdDate": { "type": ["null", "string"], @@ -19,10 +20,10 @@ } }, "space": { - "type": "boolean" + "type": ["boolean", "null"] }, "title": { - "type": "string" + "type": ["string", "null"] }, "project": { "type": ["object", "null"], diff --git a/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/tasks.json b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/tasks.json index aaf086c8b645..3f63eb94773a 100644 --- a/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/tasks.json +++ b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/tasks.json @@ -1,24 +1,25 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", + "type": ["object", "null"], + "additionalProperties": true, "properties": { "id": { - "type": "string" + "type": ["string", "null"] }, "accountId": { - "type": "string" + "type": ["string", "null"] }, "title": { - "type": "string" + "type": ["string", "null"] }, "status": { - "type": "string" + "type": ["string", "null"] }, "importance": { - "type": "string" + "type": ["string", "null"] }, "scope": { - "type": "string" + "type": ["string", "null"] }, "briefDescription": { "type": ["null", "string"] @@ -27,16 +28,16 @@ "type": ["null", "string"] }, "customStatusId": { - "type": "string" + "type": ["string", "null"] }, "permalink": { - "type": "string" + "type": ["string", "null"] }, "priority": { - "type": "string" + "type": ["string", "null"] }, "createdDate": { - "type": "string", + "type": ["string", "null"], "format": "date-time" }, "responsibleIds": { @@ -69,7 +70,7 @@ "type": ["object", "null"], "properties": { "id": { - "type": "string" + "type": ["string", "null"] }, "value": { "type": ["string", "null"] @@ -78,16 +79,20 @@ } }, "dates": { - "type": "object", + "type": ["object", "null"], "properties": { "type": { - "type": "string" + "type": ["string", "null"] } } }, "updatedDate": { "type": "string", "format": "date-time" + }, + "completedDate": { + "type": "string", + "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/workflows.json b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/workflows.json index 264a755be4ab..26174d5f2638 100644 --- a/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/workflows.json +++ b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/workflows.json @@ -1,50 +1,51 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", + "type": ["object", "null"], + "additionalProperties": true, "properties": { "id": { - "type": "string" + "type": ["string", "null"] }, "name": { - "type": "string" + "type": ["string", "null"] }, "standard": { - "type": "string" + "type": ["string", "null"] }, "hidden": { - "type": "string" + "type": ["string", "null"] }, "customStatuses": { - "type": "array", + "type": ["array", "null"], "items": { "type": ["object", "null"], "properties": { "id": { - "type": "string" + "type": ["string", "null"] }, "name": { - "type": "string" + "type": ["string", "null"] }, "standardName": { - "type": "boolean" + "type": ["boolean", "null"] }, "color": { "type": ["string", "null"] }, "standard": { - "type": "string" + "type": ["string", "null"] }, "group": { - "type": "string" + "type": ["string", "null"] }, "hidden": { - "type": "string" + "type": ["string", "null"] } } } }, "title": { - "type": "string" + "type": ["string", "null"] } } } diff --git a/airbyte-integrations/connectors/source-wrike/source_wrike/source.py b/airbyte-integrations/connectors/source-wrike/source_wrike/source.py index ac69b2c03e37..def244ee6e77 100644 --- a/airbyte-integrations/connectors/source-wrike/source_wrike/source.py +++ b/airbyte-integrations/connectors/source-wrike/source_wrike/source.py @@ -2,150 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import pendulum -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from pendulum import DateTime +WARNING: Do not modify this file. +""" -# Basic full refresh stream -class WrikeStream(HttpStream, ABC): - """ - Wrike API Reference: https://developers.wrike.com/overview/ - """ - - primary_key = "id" - url_base = "" - - def __init__(self, wrike_instance: str, **kwargs): - super().__init__(**kwargs) - self.url_base = f"https://{wrike_instance}/api/v4/" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - nextPageToken = response.json().get("nextPageToken") - - if nextPageToken: - return {"nextPageToken": nextPageToken} - else: - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - - return next_page_token - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - - for record in response.json()["data"]: - yield record - - def path(self, **kwargs) -> str: - """ - This one is tricky, the API path is the class name by default. Airbyte will load `url_base`/`classname` by - default, like https://app-us2.wrike.com/api/v4/tasks if the class name is Tasks - """ - return self.__class__.__name__.lower() - - -class Tasks(WrikeStream): - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - - return next_page_token or {"fields": "[customFields,parentIds,authorIds,responsibleIds,description,briefDescription,superTaskIds]"} - - -class Customfields(WrikeStream): - pass - - -class Contacts(WrikeStream): - pass - - -class Workflows(WrikeStream): - pass - - -def to_utc_z(date: DateTime): - return date.strftime("%Y-%m-%dT%H:%M:%SZ") - - -class Comments(WrikeStream): - def __init__(self, start_date: DateTime, **kwargs): - self._start_date = start_date - super().__init__(**kwargs) - - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - """ - Yields a list of the beginning timestamps of each 7 days period between the start date and now, - as the comments endpoint limits the requests for 7 days intervals. - """ - start_date = self._start_date - now = pendulum.now() - - while start_date <= now: - end_date = start_date + pendulum.duration(days=7) - yield {"start": to_utc_z(start_date)} - start_date = end_date - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - - slice_params = {"updatedDate": '{"start":"' + stream_slice["start"] + '"}'} - return next_page_token or slice_params - - -class Folders(WrikeStream): - pass - - -# Source - - -class SourceWrike(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - """ - :param config: the user-input config object conforming to the connector's spec.yaml - :param logger: logger object - :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. - """ - - try: - headers = { - "Accept": "application/json", - } | TokenAuthenticator(token=config["access_token"]).get_auth_header() - - resp = requests.get(f"https://{config['wrike_instance']}/api/v4/version", headers=headers) - resp.raise_for_status() - return True, None - - except requests.exceptions.RequestException as e: - error = e.response.json() - message = error.get("errorDescription") or error.get("error") - return False, message - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - start_date = pendulum.parse(config.get("start_date")) if config.get("start_date") else pendulum.now().subtract(days=7) - - args = {"authenticator": TokenAuthenticator(token=config["access_token"]), "wrike_instance": config["wrike_instance"]} - return [ - Tasks(**args), - Customfields(**args), - Contacts(**args), - Workflows(**args), - Folders(**args), - Comments(start_date=start_date, **args), - ] +# Declarative Source +class SourceWrike(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-wrike/unit_tests/test_source.py b/airbyte-integrations/connectors/source-wrike/unit_tests/test_source.py deleted file mode 100644 index 8b13af5b5d75..000000000000 --- a/airbyte-integrations/connectors/source-wrike/unit_tests/test_source.py +++ /dev/null @@ -1,39 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -import pendulum -from pytest import fixture -from source_wrike.source import SourceWrike - - -@fixture() -def config(request): - args = {"access_token": "foo", "wrike_instance": "app-us2.wrike.com"} - return args - - -def test_check_connection(mocker, config): - source = SourceWrike() - logger_mock = MagicMock() - (connection_status, _) = source.check_connection(logger_mock, config) - expected_status = False - assert connection_status == expected_status - - -def test_streams_without_date(mocker, config): - source = SourceWrike() - streams = source.streams(config) - expected_streams_number = 6 - assert len(streams) == expected_streams_number - assert streams[-1]._start_date is not None - - -def test_streams_with_date(mocker, config): - source = SourceWrike() - streams = source.streams(config | {"start_date": "2022-05-01T00:00:00Z"}) - expected_streams_number = 6 - assert len(streams) == expected_streams_number - assert streams[-1]._start_date == pendulum.parse("2022-05-01T00:00:00Z") diff --git a/airbyte-integrations/connectors/source-wrike/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-wrike/unit_tests/test_streams.py deleted file mode 100644 index a4cf0c9dff67..000000000000 --- a/airbyte-integrations/connectors/source-wrike/unit_tests/test_streams.py +++ /dev/null @@ -1,122 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pendulum -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from pytest import fixture, mark -from source_wrike.source import Comments, Tasks, WrikeStream, to_utc_z - - -@fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(WrikeStream, "__abstractmethods__", set()) - - -@fixture() -def args(request): - args = {"wrike_instance": "app-us2.wrike.com", "authenticator": TokenAuthenticator(token="tokkk")} - return args - - -def test_request_params(args): - stream = WrikeStream(**args) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = None - assert stream.request_params(**inputs) == expected_params - - -def test_tasks_request_params(args): - stream = Tasks(**args) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - assert stream.request_params(**inputs).get("fields") - - -def test_comments_slices(args): - stream = Comments(start_date=pendulum.parse("2022-05-01"), **args) - inputs = {"stream_state": None} - slices = list(stream.stream_slices(**inputs)) - assert slices[0].get("start") == "2022-05-01T00:00:00Z" - assert len(slices) > 1 - - -def test_next_page_token(args): - stream = WrikeStream(**args) - - response = MagicMock() - # first page - response.json.return_value = { - "kind": "tasks", - "nextPageToken": "ADE7SXYAAAAAUAAAAAAQAAAAMAAAAAABVB5K4QPE7SXKM", - "responseSize": 96, - "data": [{"id": "IEAFHZ6ZKQ233LQK"}], - } - inputs = {"response": response} - expected_token = {"nextPageToken": "ADE7SXYAAAAAUAAAAAAQAAAAMAAAAAABVB5K4QPE7SXKM"} - assert stream.next_page_token(**inputs) == expected_token - # next page - response.json.return_value = { - "kind": "tasks", - "responseSize": 96, - "data": [{"id": "IEAFHZ6ZKQ233LQK"}], - } - inputs = {"response": response} - expected_token = None - assert stream.next_page_token(**inputs) == expected_token - - -def test_parse_response(args): - stream = WrikeStream(**args) - response = MagicMock() - response.json.return_value = { - "kind": "tasks", - "responseSize": 96, - "data": [{"id": "IEAFHZ6ZKQ233LQK"}], - } - inputs = {"response": response} - expected_parsed_object = {"id": "IEAFHZ6ZKQ233LQK"} - assert next(stream.parse_response(**inputs)) == expected_parsed_object - - -def test_request_headers(args): - stream = WrikeStream(**args) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {} - assert stream.request_headers(**inputs) == expected_headers - - -def test_http_method(args): - stream = WrikeStream(**args) - expected_method = "GET" - assert stream.http_method == expected_method - - -@mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(args, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = WrikeStream(**args) - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(args): - response_mock = MagicMock() - stream = WrikeStream(**args) - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time - - -def test_to_utc_z(): - assert to_utc_z(pendulum.parse("2022-05-01")) == "2022-05-01T00:00:00Z" diff --git a/airbyte-integrations/connectors/source-xero/Dockerfile b/airbyte-integrations/connectors/source-xero/Dockerfile index a97eba3cca01..b634bf69f6a1 100644 --- a/airbyte-integrations/connectors/source-xero/Dockerfile +++ b/airbyte-integrations/connectors/source-xero/Dockerfile @@ -34,5 +34,5 @@ COPY source_xero ./source_xero ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.3 +LABEL io.airbyte.version=0.2.5 LABEL io.airbyte.name=airbyte/source-xero diff --git a/airbyte-integrations/connectors/source-xero/README.md b/airbyte-integrations/connectors/source-xero/README.md index 0f252db95a00..5fb499eaa440 100644 --- a/airbyte-integrations/connectors/source-xero/README.md +++ b/airbyte-integrations/connectors/source-xero/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-xero:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/xero) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_xero/spec.yaml` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-xero:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-xero build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-xero:airbyteDocker +An image will be built with the tag `airbyte/source-xero:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-xero:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-xero:dev check --confi docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-xero:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-xero:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-xero test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-xero:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-xero:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-xero test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/xero.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-xero/acceptance-test-config.yml b/airbyte-integrations/connectors/source-xero/acceptance-test-config.yml index b55a783cf14b..c9e52f62abbf 100644 --- a/airbyte-integrations/connectors/source-xero/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-xero/acceptance-test-config.yml @@ -5,51 +5,60 @@ test_strictness_level: "high" acceptance_tests: spec: tests: - - spec_path: "source_xero/spec.yaml" - backward_compatibility_tests_config: - disable_for_version: "0.1.0" + - spec_path: "source_xero/spec.yaml" + backward_compatibility_tests_config: + disable_for_version: "0.1.0" connection: tests: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: tests: - - config_path: "secrets/config.json" - backward_compatibility_tests_config: - disable_for_version: "0.1.0" + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.0" basic_read: tests: - - config_path: "secrets/config.json" - expect_records: - path: "integration_tests/expected_records.jsonl" - extra_fields: no - exact_order: no - extra_records: yes - empty_streams: - - name: "bank_transfers" - bypass_reason: "Empty stream, further investigation is required" - - name: "employees" - bypass_reason: "Empty stream, further investigation is required" - - name: "manual_journals" - bypass_reason: "Empty stream, further investigation is required" - - name: "overpayments" - bypass_reason: "Empty stream, further investigation is required" - - name: "prepayments" - bypass_reason: "Empty stream, further investigation is required" - - name: "repeating_invoices" - bypass_reason: "Empty stream, further investigation is required" - - name: "tracking_categories" - bypass_reason: "Empty stream, further investigation is required" - fail_on_extra_columns: false + - config_path: "secrets/config.json" + expect_records: + path: "integration_tests/expected_records.jsonl" + extra_fields: no + exact_order: no + extra_records: yes + empty_streams: + - name: "bank_transfers" + bypass_reason: "Empty stream, further investigation is required" + - name: "employees" + bypass_reason: "Empty stream, further investigation is required" + - name: "manual_journals" + bypass_reason: "Empty stream, further investigation is required" + - name: "overpayments" + bypass_reason: "Empty stream, further investigation is required" + - name: "prepayments" + bypass_reason: "Empty stream, further investigation is required" + - name: "repeating_invoices" + bypass_reason: "Empty stream, further investigation is required" + - name: "tracking_categories" + bypass_reason: "Empty stream, further investigation is required" + fail_on_extra_columns: false + ignored_fields: + accounts: + - name: "TaxType" + bypass_reason: "empty may change from empty string to none" + items: + - name: "SalesDetails" + bypass_reason: "empty may change from empty string to none" + - name: "PurchaseDetails" + bypass_reason: "empty may change from empty string to none" incremental: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-xero/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-xero/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-xero/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-xero/build.gradle b/airbyte-integrations/connectors/source-xero/build.gradle deleted file mode 100644 index a40bff1932ad..000000000000 --- a/airbyte-integrations/connectors/source-xero/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_xero' -} diff --git a/airbyte-integrations/connectors/source-xero/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-xero/integration_tests/expected_records.jsonl index 08f96ab1c914..1d971399e3b4 100644 --- a/airbyte-integrations/connectors/source-xero/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-xero/integration_tests/expected_records.jsonl @@ -11,9 +11,9 @@ {"stream": "invoices", "data": {"Type": "ACCPAY", "InvoiceID": "dfe48ca9-b4c8-4441-93c9-30b5818eaa6d", "InvoiceNumber": "RPT-DD", "Reference": "", "Payments": [], "CreditNotes": [], "Prepayments": [], "Overpayments": [], "AmountDue": 1.5, "AmountPaid": 0.0, "AmountCredited": 0.0, "CurrencyRate": 1.0, "IsDiscounted": false, "HasAttachments": false, "InvoiceAddresses": [], "HasErrors": false, "InvoicePaymentServices": [], "Contact": {"ContactID": "fac713dd-25b1-48c9-889e-b268590b6736", "Name": "Paragorn", "Addresses": [], "Phones": [], "ContactGroups": [], "ContactPersons": [], "HasValidationErrors": false}, "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "DueDateString": "2021-08-07T00:00:00", "DueDate": "2021-08-07T00:00:00+00:00", "Status": "DRAFT", "LineAmountTypes": "NoTax", "LineItems": [{"ItemCode": "10005", "Description": "Pen 'Airbyte'", "UnitAmount": 1.5, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 1.5, "AccountCode": "123451234", "Item": {"ItemID": "e6f28cdb-2fa3-4b4d-af19-c77497211cec", "Name": "Pen 'Airbyte'", "Code": "10005"}, "Tracking": [], "Quantity": 1.0, "LineItemID": "4fe37d11-b6a9-42c4-80ad-c6cd21286014", "AccountID": "482c2ee9-ec40-48c2-92b7-47e52ba58a57"}], "SubTotal": 1.5, "TotalTax": 0.0, "Total": 1.5, "UpdatedDateUTC": "2021-08-31T11:58:21+00:00", "CurrencyCode": "USD"}, "emitted_at": 1691572846166} {"stream": "purchase_orders", "data": {"PurchaseOrderID": "5e634e70-837c-41c6-a924-c5f43370f613", "PurchaseOrderNumber": "PO-0001", "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "DeliveryDateString": "2021-08-10T00:00:00", "DeliveryDate": "2021-08-10T00:00:00+00:00", "DeliveryAddress": "350 29th Avenue\nSan Francisco\nCalifornia\n94121\nUnited States", "AttentionTo": "Airbyte Testing", "Telephone": " +1 925-949-5463", "DeliveryInstructions": "FedEx", "HasErrors": false, "IsDiscounted": false, "Reference": "RPT-DD", "Type": "PURCHASEORDER", "CurrencyRate": 1.0, "CurrencyCode": "USD", "Contact": {"ContactID": "faf2b539-535c-4bee-9a76-30340354aaa6", "ContactStatus": "ACTIVE", "Name": "Petra Shop", "FirstName": "Mickle", "LastName": "Born", "Addresses": [{"AddressType": "STREET", "AddressLine1": "1071 Santa Rosa Plz", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Santa Rosa", "Region": "CA", "PostalCode": "95401", "Country": "USA", "AttentionTo": "1071 Santa Rosa Plz, Santa Rosa, CA, 95401"}, {"AddressType": "POBOX", "AddressLine1": "1071 Santa Rosa Plz", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Santa Rosa", "Region": "CA", "PostalCode": "95401", "Country": "USA", "AttentionTo": "1071 Santa Rosa Plz, Santa Rosa, CA, 95401"}], "Phones": [{"PhoneType": "DEFAULT", "PhoneNumber": "2056400", "PhoneAreaCode": "707", "PhoneCountryCode": "1"}, {"PhoneType": "MOBILE", "PhoneNumber": "2056400", "PhoneAreaCode": "707", "PhoneCountryCode": "1"}], "UpdatedDateUTC": "2021-08-30T21:29:18+00:00", "ContactGroups": [], "DefaultCurrency": "USD", "ContactPersons": [], "HasValidationErrors": false}, "BrandingThemeID": "58e361fe-3867-478d-aa2a-eb91641088b8", "Status": "DRAFT", "LineAmountTypes": "NoTax", "LineItems": [{"ItemCode": "10004", "Description": "Notebook 'Airbyte'", "UnitAmount": 6.0, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 6.0, "AccountCode": "123451234", "Tracking": [], "Quantity": 1.0, "LineItemID": "2435b710-f070-444c-ad74-1ab284bde68f"}], "SubTotal": 6.0, "TotalTax": 0.0, "Total": 6.0, "UpdatedDateUTC": "2021-08-31T11:52:32+00:00", "HasAttachments": false}, "emitted_at": 1691572848638} {"stream": "purchase_orders", "data": {"PurchaseOrderID": "23b4ba7d-c171-46d6-90b8-8985ddedf116", "PurchaseOrderNumber": "PO-0002", "DateString": "2021-08-31T00:00:00", "Date": "2021-08-31T00:00:00+00:00", "DeliveryAddress": "350 29th Avenue\nSan Francisco\nCalifornia\n94121\nUnited States", "AttentionTo": "Airbyte Testing", "Telephone": "", "DeliveryInstructions": "", "HasErrors": false, "IsDiscounted": false, "Reference": "", "Type": "PURCHASEORDER", "CurrencyRate": 1.0, "CurrencyCode": "USD", "Contact": {"ContactID": "55fa44bb-3060-485c-88c7-11021b15e753", "ContactStatus": "ACTIVE", "Name": "Milly", "FirstName": "Ashly", "LastName": "Simon", "Addresses": [{"AddressType": "STREET", "AddressLine1": "350 29th Ave", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Columbus", "Region": "GA", "PostalCode": "31903", "Country": "USA", "AttentionTo": ""}, {"AddressType": "POBOX", "AddressLine1": "350 29th Ave", "AddressLine2": "", "AddressLine3": "", "AddressLine4": "", "City": "Columbus", "Region": "GA", "PostalCode": "31903", "Country": "United States", "AttentionTo": ""}], "Phones": [{"PhoneType": "DEFAULT", "PhoneNumber": "6179800", "PhoneAreaCode": "323", "PhoneCountryCode": "1"}, {"PhoneType": "DDI", "PhoneNumber": "6179804", "PhoneAreaCode": "323", "PhoneCountryCode": "1"}, {"PhoneType": "FAX", "PhoneNumber": "6179802", "PhoneAreaCode": "323", "PhoneCountryCode": "1"}, {"PhoneType": "MOBILE", "PhoneNumber": "6179803", "PhoneAreaCode": "323", "PhoneCountryCode": "1"}], "UpdatedDateUTC": "2021-08-30T21:30:20+00:00", "ContactGroups": [], "DefaultCurrency": "USD", "ContactPersons": [], "HasValidationErrors": false}, "BrandingThemeID": "58e361fe-3867-478d-aa2a-eb91641088b8", "Status": "DRAFT", "LineAmountTypes": "Inclusive", "LineItems": [{"ItemCode": "10004", "Description": "Notebook 'Airbyte'", "UnitAmount": 6.0, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 6.0, "AccountCode": "123451234", "Tracking": [], "Quantity": 1.0, "LineItemID": "646a2f78-2175-483c-868b-95c2d8b1445d"}, {"ItemCode": "10005", "Description": "Pen 'Airbyte'", "UnitAmount": 1.5, "TaxType": "NONE", "TaxAmount": 0.0, "LineAmount": 1.5, "AccountCode": "123451234", "Tracking": [], "Quantity": 1.0, "LineItemID": "10138b09-066c-4405-98bb-b6a67c41391a"}], "SubTotal": 7.5, "TotalTax": 0.0, "Total": 7.5, "UpdatedDateUTC": "2021-08-31T11:54:19+00:00", "HasAttachments": false}, "emitted_at": 1691572848639} -{"stream": "accounts", "data": {"AccountID": "492d0a31-bde9-426b-aef7-12c4ae5bf784", "Name": "Business Account", "Status": "ACTIVE", "Type": "BANK", "TaxType": "NONE", "Class": "ASSET", "EnablePaymentsToAccount": false, "ShowInExpenseClaims": false, "BankAccountNumber": "1234567890", "BankAccountType": "BANK", "CurrencyCode": "USD", "ReportingCode": "ASS.CUR.CAS.CAS", "ReportingCodeName": "Cash on hand", "HasAttachments": false, "UpdatedDateUTC": "2021-08-31T12:09:02+00:00", "AddToWatchlist": false}, "emitted_at": 1691572849400} -{"stream": "accounts", "data": {"AccountID": "7dc83cc5-4a75-40aa-abe6-806b934bbce0", "Code": "1200", "Name": "Accounts Receivable", "Status": "ACTIVE", "Type": "CURRENT", "TaxType": "NONE", "Description": "Outstanding invoices the company has issued out to the client but has not yet received in cash at balance date.", "Class": "ASSET", "SystemAccount": "DEBTORS", "EnablePaymentsToAccount": false, "ShowInExpenseClaims": false, "BankAccountType": "", "ReportingCode": "ASS.CUR.REC.ACR", "HasAttachments": false, "UpdatedDateUTC": "2021-08-27T14:03:33+00:00", "AddToWatchlist": false}, "emitted_at": 1691572849401} -{"stream": "accounts", "data": {"AccountID": "5043da8e-9646-4ffa-a235-707ea08e23f0", "Code": "1231231230", "Name": "Current Asset account", "Status": "ACTIVE", "Type": "CURRENT", "TaxType": "NONE", "Description": "Current Asset account", "Class": "ASSET", "EnablePaymentsToAccount": false, "ShowInExpenseClaims": false, "BankAccountType": "", "ReportingCode": "ASS", "ReportingCodeName": "Assets", "HasAttachments": false, "UpdatedDateUTC": "2021-08-31T10:41:57+00:00", "AddToWatchlist": true}, "emitted_at": 1691572849401} +{"stream": "accounts", "data": {"AccountID": "492d0a31-bde9-426b-aef7-12c4ae5bf784", "Name": "Business Account", "Status": "ACTIVE", "Type": "BANK", "TaxType": "", "Class": "ASSET", "EnablePaymentsToAccount": false, "ShowInExpenseClaims": false, "BankAccountNumber": "1234567890", "BankAccountType": "BANK", "CurrencyCode": "USD", "ReportingCode": "ASS.CUR.CAS.CAS", "ReportingCodeName": "Cash on hand", "HasAttachments": false, "UpdatedDateUTC": "2021-08-31T12:09:02+00:00", "AddToWatchlist": false}, "emitted_at": 1691572849400} +{"stream": "accounts", "data": {"AccountID": "7dc83cc5-4a75-40aa-abe6-806b934bbce0", "Code": "1200", "Name": "Accounts Receivable", "Status": "ACTIVE", "Type": "CURRENT", "TaxType": "", "Description": "Outstanding invoices the company has issued out to the client but has not yet received in cash at balance date.", "Class": "ASSET", "SystemAccount": "DEBTORS", "EnablePaymentsToAccount": false, "ShowInExpenseClaims": false, "BankAccountType": "", "ReportingCode": "ASS.CUR.REC.ACR", "HasAttachments": false, "UpdatedDateUTC": "2021-08-27T14:03:33+00:00", "AddToWatchlist": false}, "emitted_at": 1691572849401} +{"stream": "accounts", "data": {"AccountID": "5043da8e-9646-4ffa-a235-707ea08e23f0", "Code": "1231231230", "Name": "Current Asset account", "Status": "ACTIVE", "Type": "CURRENT", "TaxType": "", "Description": "Current Asset account", "Class": "ASSET", "EnablePaymentsToAccount": false, "ShowInExpenseClaims": false, "BankAccountType": "", "ReportingCode": "ASS", "ReportingCodeName": "Assets", "HasAttachments": false, "UpdatedDateUTC": "2021-08-31T10:41:57+00:00", "AddToWatchlist": true}, "emitted_at": 1691572849401} {"stream": "items", "data": {"ItemID": "6ac509d9-89e7-44a9-b630-4a27a2b2287a", "Code": "10001", "Description": "T-shirt 'Airbyte'", "PurchaseDescription": "T-shirt 'Airbyte'", "UpdatedDateUTC": "2021-08-31T11:18:51+00:00", "PurchaseDetails": {"UnitPrice": 10.0, "COGSAccountCode": "5100", "TaxType": "NONE"}, "SalesDetails": {"UnitPrice": 12.0, "AccountCode": "4000", "TaxType": "NONE"}, "Name": "T-shirt 'Airbyte'", "IsTrackedAsInventory": true, "InventoryAssetAccountCode": "123451234", "TotalCostPool": 40.0, "QuantityOnHand": 4.0, "IsSold": true, "IsPurchased": true}, "emitted_at": 1691572851425} {"stream": "items", "data": {"ItemID": "b0684a92-7379-4a65-bf71-9118cc75e381", "Code": "10002", "Description": "Cap 'Airbyte'", "PurchaseDescription": "Cap 'Airbyte'", "UpdatedDateUTC": "2021-08-31T10:51:03+00:00", "PurchaseDetails": {"UnitPrice": 18.0, "COGSAccountCode": "5000", "TaxType": "NONE"}, "SalesDetails": {"UnitPrice": 15.0, "AccountCode": "4000", "TaxType": "NONE"}, "Name": "Cap 'Airbyte'", "IsTrackedAsInventory": true, "InventoryAssetAccountCode": "123451234", "TotalCostPool": 36.0, "QuantityOnHand": 2.0, "IsSold": true, "IsPurchased": true}, "emitted_at": 1691572851426} {"stream": "items", "data": {"ItemID": "27212cb0-4d3e-4655-8d90-f2895452cd23", "Code": "10004", "Description": "Notebook", "PurchaseDescription": "Notebook 'Airbyte'", "UpdatedDateUTC": "2021-08-31T11:51:30+00:00", "PurchaseDetails": {"UnitPrice": 6.0, "COGSAccountCode": "5000", "TaxType": "NONE"}, "SalesDetails": {"UnitPrice": 7.5, "AccountCode": "4000", "TaxType": "NONE"}, "Name": "Notebook 'Airbyte'", "IsTrackedAsInventory": true, "InventoryAssetAccountCode": "123451234", "TotalCostPool": 12.0, "QuantityOnHand": 2.0, "IsSold": true, "IsPurchased": true}, "emitted_at": 1691572851427} diff --git a/airbyte-integrations/connectors/source-xero/main.py b/airbyte-integrations/connectors/source-xero/main.py index ecb627ec90dd..d765f10d2093 100644 --- a/airbyte-integrations/connectors/source-xero/main.py +++ b/airbyte-integrations/connectors/source-xero/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_xero import SourceXero +from source_xero.run import run if __name__ == "__main__": - source = SourceXero() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-xero/metadata.yaml b/airbyte-integrations/connectors/source-xero/metadata.yaml index f70a0bf641f2..6edc99384fe7 100644 --- a/airbyte-integrations/connectors/source-xero/metadata.yaml +++ b/airbyte-integrations/connectors/source-xero/metadata.yaml @@ -5,7 +5,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6fd1e833-dd6e-45ec-a727-ab917c5be892 - dockerImageTag: 0.2.3 + dockerImageTag: 0.2.5 dockerRepository: airbyte/source-xero githubIssueLabel: source-xero icon: xero.svg @@ -13,7 +13,7 @@ data: name: Xero registries: cloud: - enabled: true + enabled: false oss: enabled: true releaseStage: beta diff --git a/airbyte-integrations/connectors/source-xero/setup.py b/airbyte-integrations/connectors/source-xero/setup.py index b93015ee5795..89541436cfd3 100644 --- a/airbyte-integrations/connectors/source-xero/setup.py +++ b/airbyte-integrations/connectors/source-xero/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.40", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ @@ -16,6 +16,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-xero=source_xero.run:run", + ], + }, name="source_xero", description="Source implementation for Xero.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-xero/source_xero/run.py b/airbyte-integrations/connectors/source-xero/source_xero/run.py new file mode 100644 index 000000000000..fb8d5955af03 --- /dev/null +++ b/airbyte-integrations/connectors/source-xero/source_xero/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_xero import SourceXero + + +def run(): + source = SourceXero() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-xero/source_xero/source.py b/airbyte-integrations/connectors/source-xero/source_xero/source.py index 1e1ccf3f7727..a125790cec2f 100644 --- a/airbyte-integrations/connectors/source-xero/source_xero/source.py +++ b/airbyte-integrations/connectors/source-xero/source_xero/source.py @@ -5,6 +5,7 @@ from typing import Any, List, Mapping, Tuple import pendulum +import requests from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -41,11 +42,22 @@ def _validate_and_transform(self, config: Mapping[str, Any]): return config def check_connection(self, logger, config) -> Tuple[bool, any]: - config = self._validate_and_transform(config) - stream = Organisations(authenticator=self.get_authenticator(config), tenant_id=config["tenant_id"]) - records = stream.read_records(sync_mode=SyncMode.full_refresh) - record = next(records) - return record["OrganisationID"] == config["tenant_id"], None + try: + config = self._validate_and_transform(config) + stream = Organisations(authenticator=self.get_authenticator(config), tenant_id=config["tenant_id"]) + records = stream.read_records(sync_mode=SyncMode.full_refresh) + record = next(records) + return record["OrganisationID"] == config["tenant_id"], None + except requests.exceptions.HTTPError as e: + error_message = str(e) + if e.response.status_code == 403: + error_message = ( + "For oauth2 authentication try to re-authenticate and allow all requested scopes, for token authentication please update " + "access token with all required scopes mentioned in prerequisites. Full error message: " + error_message + ) + return False, error_message + except Exception as e: + return False, str(e) def streams(self, config: Mapping[str, Any]) -> List[Stream]: stream_kwargs = { diff --git a/airbyte-integrations/connectors/source-xero/unit_tests/conftest.py b/airbyte-integrations/connectors/source-xero/unit_tests/conftest.py index 8cce48c72dd0..d5c57aec0a3e 100644 --- a/airbyte-integrations/connectors/source-xero/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-xero/unit_tests/conftest.py @@ -13,7 +13,7 @@ def config_fixture(): "client_secret": "client_secret", "refresh_token": "refresh_token", "access_token": "access_token", - "token_expiry_date": "2099-01-01T12:00:00.000000+00:00" + "token_expiry_date": "2099-01-01T12:00:00.000000+00:00", }, "tenant_id": "tenant_id", "start_date": "2020-01-01T00:00:00Z", diff --git a/airbyte-integrations/connectors/source-xero/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-xero/unit_tests/test_incremental_streams.py index 039cc4a08285..1532b333391b 100644 --- a/airbyte-integrations/connectors/source-xero/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-xero/unit_tests/test_incremental_streams.py @@ -59,19 +59,21 @@ def test_stream_checkpoint_interval(patch_incremental_base_class): def test_read_incremental(requests_mock): - json_responses = iter([ - { - 'BankTransactions': [ - {'BankTransactionID': '4848c602-aeba-4e01-a533-8eae3e090633', 'UpdatedDateUTC': '/Date(1630412754013+0000)/'}, - {'BankTransactionID': '550c811d-66d3-4b72-9334-4555d22c85b5', 'UpdatedDateUTC': '/Date(1630413087633+0000)/'}, - ] - }, - { - 'BankTransactions': [ - {'BankTransactionID': '9a704749-8084-4eed-9554-4edccaa1b6ce', 'UpdatedDateUTC': '/Date(1630413149867+0000)/'} - ] - } - ]) + json_responses = iter( + [ + { + "BankTransactions": [ + {"BankTransactionID": "4848c602-aeba-4e01-a533-8eae3e090633", "UpdatedDateUTC": "/Date(1630412754013+0000)/"}, + {"BankTransactionID": "550c811d-66d3-4b72-9334-4555d22c85b5", "UpdatedDateUTC": "/Date(1630413087633+0000)/"}, + ] + }, + { + "BankTransactions": [ + {"BankTransactionID": "9a704749-8084-4eed-9554-4edccaa1b6ce", "UpdatedDateUTC": "/Date(1630413149867+0000)/"} + ] + }, + ] + ) requests_mock.get( "https://api.xero.com/api.xro/2.0/bankTransactions", @@ -83,12 +85,10 @@ def test_read_incremental(requests_mock): stream_state = {} records = read_incremental(stream, stream_state) assert records == [ - {'BankTransactionID': '4848c602-aeba-4e01-a533-8eae3e090633', 'UpdatedDateUTC': '2021-08-31T12:25:54+00:00'}, - {'BankTransactionID': '550c811d-66d3-4b72-9334-4555d22c85b5', 'UpdatedDateUTC': '2021-08-31T12:31:27+00:00'} + {"BankTransactionID": "4848c602-aeba-4e01-a533-8eae3e090633", "UpdatedDateUTC": "2021-08-31T12:25:54+00:00"}, + {"BankTransactionID": "550c811d-66d3-4b72-9334-4555d22c85b5", "UpdatedDateUTC": "2021-08-31T12:31:27+00:00"}, ] - assert stream_state == {'UpdatedDateUTC': '2021-08-31T12:31:27+00:00'} + assert stream_state == {"UpdatedDateUTC": "2021-08-31T12:31:27+00:00"} records = read_incremental(stream, stream_state) - assert stream_state == {'UpdatedDateUTC': '2021-08-31T12:32:29+00:00'} - assert records == [ - {'BankTransactionID': '9a704749-8084-4eed-9554-4edccaa1b6ce', 'UpdatedDateUTC': '2021-08-31T12:32:29+00:00'} - ] + assert stream_state == {"UpdatedDateUTC": "2021-08-31T12:32:29+00:00"} + assert records == [{"BankTransactionID": "9a704749-8084-4eed-9554-4edccaa1b6ce", "UpdatedDateUTC": "2021-08-31T12:32:29+00:00"}] diff --git a/airbyte-integrations/connectors/source-xero/unit_tests/test_source.py b/airbyte-integrations/connectors/source-xero/unit_tests/test_source.py index cc67d20a9935..b77916b1a1fa 100644 --- a/airbyte-integrations/connectors/source-xero/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-xero/unit_tests/test_source.py @@ -15,6 +15,18 @@ def test_check_connection(mock_auth, mock_stream, mock_response, config): assert source.check_connection(logger_mock, config_mock) == (True, None) +def test_check_connection_failed(mock_auth, mock_stream, mock_response, config, requests_mock): + mock_stream("Organisation", response={"Organisations": [{"OrganisationID": "tenant_id"}]}) + mock_auth({"access_token": "TOKEN", "expires_in": 123}) + + requests_mock.get(url="https://api.xero.com/api.xro/2.0/Organisation", status_code=403, content= b'{"status": 403, "code": "restricted_resource"}') + + source = SourceXero() + check_succeeded, error = source.check_connection(MagicMock(), config) + assert check_succeeded is False + assert 'For oauth2 authentication try to re-authenticate and allow all requested scopes' in error + + def test_streams(config): source = SourceXero() streams = source.streams(config) diff --git a/airbyte-integrations/connectors/source-xero/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-xero/unit_tests/test_streams.py index 16aec57385d3..232a8293d566 100644 --- a/airbyte-integrations/connectors/source-xero/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-xero/unit_tests/test_streams.py @@ -3,11 +3,12 @@ # import datetime +import logging from http import HTTPStatus from unittest.mock import MagicMock import pytest -from source_xero.streams import XeroStream, parse_date +from source_xero.streams import Organisations, XeroStream, parse_date @pytest.fixture @@ -93,3 +94,35 @@ def test_parse_date(): assert parse_date("/Date(1656792775000)/") == datetime.datetime(2022, 7, 2, 20, 12, 55, tzinfo=datetime.timezone.utc) # Not a date assert parse_date("not a date") is None + + +@pytest.mark.parametrize( + "stream,url,status_code,response_content,expected_availability,expected_reason_substring", + [ + ( + Organisations, + "https://api.xero.com/api.xro/2.0/Organisation", + 403, + b'{"object": "error", "status": 403, "code": "restricted_resource"}', + False, + "Unable to read organisations stream. The endpoint https://api.xero.com/api.xro/2.0/Organisation returned 403: None. This is most likely due to insufficient permissions on the credentials in use.", + ), + + ], +) +def test_403_error_handling( + requests_mock, stream, url, status_code, response_content, expected_availability, expected_reason_substring +): + """ + Test that availability strategy flags streams with 403 error as unavailable + and returns custom Notion integration message. + """ + + requests_mock.get(url=url, status_code=status_code, content=response_content) + + stream = stream(tenant_id='tenant_id') + + is_available, reason = stream.check_availability(logger=logging.Logger, source=MagicMock()) + + assert is_available is expected_availability + assert expected_reason_substring in reason diff --git a/airbyte-integrations/connectors/source-xkcd/README.md b/airbyte-integrations/connectors/source-xkcd/README.md index ddb3e997e24e..19c7ae821398 100644 --- a/airbyte-integrations/connectors/source-xkcd/README.md +++ b/airbyte-integrations/connectors/source-xkcd/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-xkcd:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/xkcd) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_xkcd/spec.yaml` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-xkcd:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-xkcd build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-xkcd:airbyteDocker +An image will be built with the tag `airbyte/source-xkcd:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-xkcd:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-xkcd:dev check --confi docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-xkcd:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-xkcd:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-xkcd test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-xkcd:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-xkcd:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-xkcd test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/xkcd.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-xkcd/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-xkcd/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-xkcd/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-xkcd/build.gradle b/airbyte-integrations/connectors/source-xkcd/build.gradle deleted file mode 100644 index f554f36059a0..000000000000 --- a/airbyte-integrations/connectors/source-xkcd/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_xkcd' -} diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/README.md b/airbyte-integrations/connectors/source-yahoo-finance-price/README.md index 305a00be0afe..65c9768e7518 100644 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/README.md +++ b/airbyte-integrations/connectors/source-yahoo-finance-price/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-yahoo-finance-price:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/yahoo-finance-price) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_yahoo_finance_price/spec.json` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-yahoo-finance-price:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-yahoo-finance-price build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-yahoo-finance-price:airbyteDocker +An image will be built with the tag `airbyte/source-yahoo-finance-price:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-yahoo-finance-price:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-yahoo-finance-price:de docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-yahoo-finance-price:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-yahoo-finance-price:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-yahoo-finance-price test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-yahoo-finance-price:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-yahoo-finance-price:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-yahoo-finance-price test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/yahoo-finance-price.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-yahoo-finance-price/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-yahoo-finance-price/build.gradle b/airbyte-integrations/connectors/source-yahoo-finance-price/build.gradle deleted file mode 100644 index 3a0cfc286d84..000000000000 --- a/airbyte-integrations/connectors/source-yahoo-finance-price/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_yahoo_finance_price' -} diff --git a/airbyte-integrations/connectors/source-yandex-metrica/README.md b/airbyte-integrations/connectors/source-yandex-metrica/README.md index be6d0812cc87..85449337f5ca 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/README.md +++ b/airbyte-integrations/connectors/source-yandex-metrica/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-yandex-metrica:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/yandex-metrica) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_yandex_metrica/spec.yaml` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-yandex-metrica:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-yandex-metrica build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-yandex-metrica:airbyteDocker +An image will be built with the tag `airbyte/source-yandex-metrica:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-yandex-metrica:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-yandex-metrica:dev che docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-yandex-metrica:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-yandex-metrica:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-yandex-metrica test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-yandex-metrica:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-yandex-metrica:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-yandex-metrica test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/yandex-metrica.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-yandex-metrica/acceptance-test-config.yml b/airbyte-integrations/connectors/source-yandex-metrica/acceptance-test-config.yml index c20762bdfbd5..a0aa317a1a8d 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-yandex-metrica/acceptance-test-config.yml @@ -28,4 +28,4 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" - timeout_seconds: 3600 \ No newline at end of file + timeout_seconds: 3600 diff --git a/airbyte-integrations/connectors/source-yandex-metrica/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-yandex-metrica/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-yandex-metrica/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-yandex-metrica/build.gradle b/airbyte-integrations/connectors/source-yandex-metrica/build.gradle deleted file mode 100644 index 099e755bcf23..000000000000 --- a/airbyte-integrations/connectors/source-yandex-metrica/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_yandex_metrica' -} diff --git a/airbyte-integrations/connectors/source-yandex-metrica/metadata.yaml b/airbyte-integrations/connectors/source-yandex-metrica/metadata.yaml index 7fa1a9be1deb..31c05d9a4759 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/metadata.yaml +++ b/airbyte-integrations/connectors/source-yandex-metrica/metadata.yaml @@ -1,4 +1,7 @@ data: + ab_internal: + ql: 300 + sl: 100 allowedHosts: hosts: - api-metrica.yandex.net @@ -7,6 +10,7 @@ data: definitionId: 7865dce4-2211-4f6a-88e5-9d0fe161afe7 dockerImageTag: 1.0.0 dockerRepository: airbyte/source-yandex-metrica + documentationUrl: https://docs.airbyte.com/integrations/sources/yandex-metrica githubIssueLabel: source-yandex-metrica icon: yandexmetrica.svg license: MIT @@ -17,11 +21,7 @@ data: oss: enabled: true releaseStage: beta - documentationUrl: https://docs.airbyte.com/integrations/sources/yandex-metrica + supportLevel: community tags: - language:python - ab_internal: - sl: 200 - ql: 300 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/conftest.py b/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/conftest.py index 79f731e629df..4508f0f63e53 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/conftest.py @@ -5,37 +5,29 @@ import pytest -@pytest.fixture(name='config') +@pytest.fixture(name="config") def config_fixture(): - return { - "auth_token": "test_token", - "counter_id": "00000000", - "start_date": "2022-07-01", - "end_date": "2022-07-02" - } + return {"auth_token": "test_token", "counter_id": "00000000", "start_date": "2022-07-01", "end_date": "2022-07-02"} -@pytest.fixture(name='config_wrong_date') +@pytest.fixture(name="config_wrong_date") def config_wrong_date_fixture(): - return { - "auth_token": "test_token", - "counter_id": "00000000", - "start_date": "2022-07-02", - "end_date": "2022-07-01" - } + return {"auth_token": "test_token", "counter_id": "00000000", "start_date": "2022-07-02", "end_date": "2022-07-01"} @pytest.fixture(name="mock_all_requests") def mock_all_requests_fixture(requests_mock): - requests_mock.register_uri("POST", - "https://api-metrica.yandex.net/management/v1/counter/00000000/logrequests", - json={"log_request": {"request_id": 1}}) - requests_mock.register_uri("GET", - "https://api-metrica.yandex.net/management/v1/counter/00000000/logrequest/1", - json={"log_request": {"status": "processed", "parts":[{"part_number": 0}]}}) - requests_mock.register_uri("GET", - 'https://api-metrica.yandex.net/management/v1/counter/00000000/logrequest/1/part/0/download', - text="watchID\tdateTime\n00000000\t2022-09-01 12:00:00\n00000001\t2022-08-01 12:00:10") - requests_mock.register_uri("POST", - 'https://api-metrica.yandex.net/management/v1/counter/00000000/logrequest/1/clean', - status_code=200) + requests_mock.register_uri( + "POST", "https://api-metrica.yandex.net/management/v1/counter/00000000/logrequests", json={"log_request": {"request_id": 1}} + ) + requests_mock.register_uri( + "GET", + "https://api-metrica.yandex.net/management/v1/counter/00000000/logrequest/1", + json={"log_request": {"status": "processed", "parts": [{"part_number": 0}]}}, + ) + requests_mock.register_uri( + "GET", + "https://api-metrica.yandex.net/management/v1/counter/00000000/logrequest/1/part/0/download", + text="watchID\tdateTime\n00000000\t2022-09-01 12:00:00\n00000001\t2022-08-01 12:00:10", + ) + requests_mock.register_uri("POST", "https://api-metrica.yandex.net/management/v1/counter/00000000/logrequest/1/clean", status_code=200) diff --git a/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_source.py b/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_source.py index f2222eff19b9..3993932b1c69 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_source.py @@ -19,16 +19,16 @@ def test_streams(config): def test_check_connection_invalid_api_key(config, requests_mock): config["auth_token"] = "invalid_token" - requests_mock.register_uri("GET", - "https://api-metrica.yandex.net/management/v1/counter/00000000/logrequests/evaluate", - [{"status_code": 400}]) + requests_mock.register_uri( + "GET", "https://api-metrica.yandex.net/management/v1/counter/00000000/logrequests/evaluate", [{"status_code": 400}] + ) ok, error_msg = SourceYandexMetrica().check_connection(logger, config=config) assert not ok and error_msg @freezegun.freeze_time("2023-01-02") def test_check_connection_no_end_date(config): - config.pop('end_date') + config.pop("end_date") assert SourceYandexMetrica().get_end_date(config=config) == "2023-01-01" diff --git a/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_streams.py index 839cf5d00777..afd233b11790 100644 --- a/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-yandex-metrica/unit_tests/test_streams.py @@ -8,7 +8,7 @@ EXPECTED_RECORDS = [ {"watchID": "00000000", "dateTime": "2022-09-01T12:00:00+00:00"}, - {"watchID": "00000001", "dateTime": "2022-08-01T12:00:10+00:00"} + {"watchID": "00000001", "dateTime": "2022-08-01T12:00:10+00:00"}, ] @@ -22,4 +22,4 @@ def test_download_parse_response(config, mock_all_requests): def test_state(config, mock_all_requests): stream = Sessions(config) list(stream.read_records(sync_mode=SyncMode.incremental)) - assert stream.state == {'dateTime': '2022-09-01T12:00:00+00:00'} + assert stream.state == {"dateTime": "2022-09-01T12:00:00+00:00"} diff --git a/airbyte-integrations/connectors/source-yotpo/README.md b/airbyte-integrations/connectors/source-yotpo/README.md index d0ba27b612ed..a0454fb315c5 100644 --- a/airbyte-integrations/connectors/source-yotpo/README.md +++ b/airbyte-integrations/connectors/source-yotpo/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-yotpo:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/yotpo) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_yotpo/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-yotpo:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-yotpo build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-yotpo:airbyteDocker +An image will be built with the tag `airbyte/source-yotpo:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-yotpo:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-yotpo:dev check --conf docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-yotpo:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-yotpo:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-yotpo test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-yotpo:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-yotpo:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-yotpo test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/yotpo.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-yotpo/acceptance-test-config.yml b/airbyte-integrations/connectors/source-yotpo/acceptance-test-config.yml index 1d6d15b88725..c06e358431a7 100644 --- a/airbyte-integrations/connectors/source-yotpo/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-yotpo/acceptance-test-config.yml @@ -18,7 +18,7 @@ acceptance_tests: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: + empty_streams: - name: "raw_data" bypass_reason: "Sandbox account cannot seed the endpoint" - name: "unsubscribers" @@ -32,12 +32,12 @@ acceptance_tests: extra_fields: no exact_order: no extra_records: yes - incremental: - tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" + incremental: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-yotpo/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-yotpo/acceptance-test-docker.sh deleted file mode 100755 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-yotpo/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-yotpo/build.gradle b/airbyte-integrations/connectors/source-yotpo/build.gradle deleted file mode 100644 index 83058c6777f4..000000000000 --- a/airbyte-integrations/connectors/source-yotpo/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_yotpo' -} diff --git a/airbyte-integrations/connectors/source-younium/.dockerignore b/airbyte-integrations/connectors/source-younium/.dockerignore deleted file mode 100644 index cebe350aeec3..000000000000 --- a/airbyte-integrations/connectors/source-younium/.dockerignore +++ /dev/null @@ -1,6 +0,0 @@ -* -!Dockerfile -!main.py -!source_younium -!setup.py -!secrets \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-younium/Dockerfile b/airbyte-integrations/connectors/source-younium/Dockerfile deleted file mode 100644 index 92fb46be3954..000000000000 --- a/airbyte-integrations/connectors/source-younium/Dockerfile +++ /dev/null @@ -1,38 +0,0 @@ -FROM python:3.9.13-alpine3.15 as base - -# build and load all requirements -FROM base as builder -WORKDIR /airbyte/integration_code - -# upgrade pip to the latest version -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - - -COPY setup.py ./ -# install necessary packages to a temporary folder -RUN pip install --prefix=/install . - -# build a clean environment -FROM base -WORKDIR /airbyte/integration_code - -# copy all loaded and built libraries to a pure basic image -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -# bash is installed for more convenient debugging. -RUN apk --no-cache add bash - -# copy payload code only -COPY main.py ./ -COPY source_younium ./source_younium - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.0 -LABEL io.airbyte.name=airbyte/source-younium \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-younium/README.md b/airbyte-integrations/connectors/source-younium/README.md index e9f0407ff831..bd076ad1fa20 100644 --- a/airbyte-integrations/connectors/source-younium/README.md +++ b/airbyte-integrations/connectors/source-younium/README.md @@ -1,121 +1,53 @@ -# Younium Source +# Zapier Supported Storage Source -This is the repository for the Younium source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/younium). +This is the repository for the Zapier Supported Storage configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/zapier-supported-storage). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-younium:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/younium) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_younium/spec.yaml` file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/zapier-supported-storage) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zapier_supported_storage/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source younium test creds` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source zapier-supported-storage test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-younium:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name source-zapier-supported-storage build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-younium:airbyteDocker +An image will be built with the tag `airbyte/source-zapier-supported-storage:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-zapier-supported-storage:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-younium:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-younium:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-younium:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-younium:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +docker run --rm airbyte/source-zapier-supported-storage:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zapier-supported-storage:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zapier-supported-storage:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-zapier-supported-storage:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-younium test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-younium:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-younium:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. \ No newline at end of file +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-younium test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/younium.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-younium/__init__.py b/airbyte-integrations/connectors/source-younium/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-younium/acceptance-test-config.yml b/airbyte-integrations/connectors/source-younium/acceptance-test-config.yml index 2e04f110fc78..a8a1f5ebcfb4 100644 --- a/airbyte-integrations/connectors/source-younium/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-younium/acceptance-test-config.yml @@ -1,26 +1,39 @@ # See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-younium:dev -tests: +acceptance_tests: spec: - - spec_path: "source_younium/spec.yaml" + tests: + - spec_path: "source_younium/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file - # expect_records: - # path: "integration_tests/expected_records.jsonl" - # extra_fields: no - # exact_order: no - # extra_records: yes + # expect_records: + # path: "integration_tests/expected_records.jsonl" + # extra_fields: no + # exact_order: no + # extra_records: yes + incremental: + bypass_reason: "This connector does not implement incremental sync" + # TODO uncomment this block this block if your connector implements incremental sync: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-younium/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-younium/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-younium/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-younium/build.gradle b/airbyte-integrations/connectors/source-younium/build.gradle deleted file mode 100644 index 4573f552bb29..000000000000 --- a/airbyte-integrations/connectors/source-younium/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_younium' -} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-younium/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-younium/integration_tests/configured_catalog.json index a9a071c44c44..cc33475c5661 100644 --- a/airbyte-integrations/connectors/source-younium/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-younium/integration_tests/configured_catalog.json @@ -1,5 +1,23 @@ { "streams": [ + { + "stream": { + "name": "account", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "booking", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, { "stream": { "name": "invoice", diff --git a/airbyte-integrations/connectors/source-younium/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-younium/integration_tests/expected_records.jsonl new file mode 100644 index 000000000000..6ac78e102510 --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/integration_tests/expected_records.jsonl @@ -0,0 +1,14 @@ +{"stream": "account", "data": {"id": "53abf01b-5942-4bed-b61f-08da9ad6fb8c", "accountNumber": "A-000001", "accountType": "Customer", "inactive": false, "invoiceDeliveryMethod": "Print", "accountsReceivable": "1510", "currencyCode": "SEK", "defaultPaymentTerm": "30", "name": "example customer", "taxTemplate": "Standard", "invoiceTemplateId": "f6b8e099-332f-44a1-56b6-08da9aea9fec", "defaultInvoiceAddress": {"id": "4c595cad-00ea-47ec-b697-5f6a04900411", "description": "Invoice address", "country": "Sweden"}, "addresses": [{"id": "4c595cad-00ea-47ec-b697-5f6a04900411", "description": "Invoice address", "country": "Sweden"}], "customFields": {}, "created": "2022-09-21T13:22:16.366372", "modified": "2022-09-21T13:22:16.3881969", "cmrr": {"amount": 0.0, "baseCurrencyAmount": 0.0}, "acv": {"amount": 0.0, "baseCurrencyAmount": 0.0}, "emrr": {"amount": 0.0, "baseCurrencyAmount": 0.0}, "oneTimeFees": {"amount": 0.0, "baseCurrencyAmount": 0.0}, "tcv": {"amount": 0.0, "baseCurrencyAmount": 0.0}}, "emitted_at": 1696446765299} +{"stream": "account", "data": {"id": "0901e0a9-cf4f-48c8-8cba-08da9f9a6eb8", "accountNumber": "A-000002", "accountType": "Customer", "inactive": false, "invoiceDeliveryMethod": "Print", "accountsReceivable": "1510", "currencyCode": "EUR", "defaultPaymentTerm": "30", "name": "Example Customer 1", "organizationNumber": "1", "ourReference": "John", "taxRegistrationNumber": "1", "taxTemplate": "Standard", "invoiceTemplateId": "f6b8e099-332f-44a1-56b6-08da9aea9fec", "defaultInvoiceAddress": {"id": "40c16354-6688-4640-b822-57fbccd7704a", "street": "Street 1", "city": "City name 1", "zip": "11111", "country": "United States of America"}, "defaultDeliveryAddress": {"id": "87d15064-1180-4aaa-b82e-ba5bc9cdb474", "country": "United States of America"}, "addresses": [{"id": "40c16354-6688-4640-b822-57fbccd7704a", "street": "Street 1", "city": "City name 1", "zip": "11111", "country": "United States of America"}, {"id": "87d15064-1180-4aaa-b82e-ba5bc9cdb474", "country": "United States of America"}], "customFields": {}, "created": "2022-09-26T17:51:34.2600917", "modified": "2022-09-26T18:04:53.2056541", "cmrr": {"amount": 9000.0, "currencyCode": "EUR", "baseCurrencyAmount": 97504.2, "baseCurrencyCode": "SEK"}, "acv": {"amount": 108000.0, "currencyCode": "EUR", "baseCurrencyAmount": 1170050.4, "baseCurrencyCode": "SEK"}, "emrr": {"amount": 9000.0, "currencyCode": "EUR", "baseCurrencyAmount": 97504.2, "baseCurrencyCode": "SEK"}, "oneTimeFees": {"amount": 0.0, "currencyCode": "EUR", "baseCurrencyAmount": 0.0, "baseCurrencyCode": "SEK"}, "tcv": {"amount": 324000.0, "currencyCode": "EUR", "baseCurrencyAmount": 3510151.2, "baseCurrencyCode": "SEK"}}, "emitted_at": 1696446765301} +{"stream": "account", "data": {"id": "1dc25080-543d-4dc6-8cbb-08da9f9a6eb8", "accountNumber": "A-000003", "accountType": "Customer", "inactive": false, "invoiceDeliveryMethod": "Print", "accountsReceivable": "1510", "currencyCode": "EUR", "defaultPaymentTerm": "30", "name": "Example Customer 2", "organizationNumber": "2", "ourReference": "John", "taxRegistrationNumber": "2", "taxTemplate": "Standard", "invoiceTemplateId": "f6b8e099-332f-44a1-56b6-08da9aea9fec", "defaultInvoiceAddress": {"id": "2223c2bd-0940-4f81-88fc-a2eee02f26e6", "street": "Street 2", "city": "City name 2", "zip": "11112", "country": "United States of America"}, "defaultDeliveryAddress": {"id": "8fdae644-4b66-4156-9ade-47e2e511ab5c", "country": "United States of America"}, "addresses": [{"id": "8fdae644-4b66-4156-9ade-47e2e511ab5c", "country": "United States of America"}, {"id": "2223c2bd-0940-4f81-88fc-a2eee02f26e6", "street": "Street 2", "city": "City name 2", "zip": "11112", "country": "United States of America"}], "customFields": {}, "created": "2022-09-26T17:51:34.8362407", "modified": "2022-09-26T18:04:53.2056493", "cmrr": {"amount": 1500.0, "currencyCode": "EUR", "baseCurrencyAmount": 16250.7, "baseCurrencyCode": "SEK"}, "acv": {"amount": 18000.0, "currencyCode": "EUR", "baseCurrencyAmount": 195008.4, "baseCurrencyCode": "SEK"}, "emrr": {"amount": 1500.0, "currencyCode": "EUR", "baseCurrencyAmount": 16250.7, "baseCurrencyCode": "SEK"}, "oneTimeFees": {"amount": 0.0, "currencyCode": "EUR", "baseCurrencyAmount": 0.0, "baseCurrencyCode": "SEK"}, "tcv": {"amount": 54000.0, "currencyCode": "EUR", "baseCurrencyAmount": 585025.2, "baseCurrencyCode": "SEK"}}, "emitted_at": 1696446765302} +{"stream": "booking", "data": {"id": "66dbd7eb-5ccf-44a6-1d8c-08da9f9a72e8", "bookingType": "New", "accountCategory": "New", "changeType": "None", "cmrr": {"amount": 3000.0, "amountInBaseCurrency": 32501.4}, "acv": {"amount": 36000.0, "amountInBaseCurrency": 390016.8}, "emrr": {"amount": 3000.0, "amountInBaseCurrency": 32501.4}, "oneTimeFees": {"amount": 0.0, "amountInBaseCurrency": 0.0}, "tcv": {"amount": 108000.0, "amountInBaseCurrency": 1170050.4}, "fmrr": {"amount": 0.0, "amountInBaseCurrency": 0.0}, "effectiveDate": "2022-09-26T00:00:00", "bookingLines": [{"charge": {"name": "Product simple 2", "chargeNumber": "OPC-000185", "description": "Product simple 2", "id": "f87905e7-eb9e-4200-8fce-08da9f9a7175"}, "cmrr": {"amount": 3000.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 32501.4, "baseCurrencyCode": "SEK"}, "acv": {"amount": 36000.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 390016.8, "baseCurrencyCode": "SEK"}, "emrr": {"amount": 3000.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 32501.4, "baseCurrencyCode": "SEK"}, "oneTimeFees": {"amount": 0.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 0.0, "baseCurrencyCode": "SEK"}, "tcv": {"amount": 108000.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 1170050.4, "baseCurrencyCode": "SEK"}, "fmrr": {"amount": 0.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 0.0, "baseCurrencyCode": "SEK"}, "created": "2022-09-26T18:30:58.6124354", "modified": "2022-09-26T18:30:58.6124354"}], "classification": {"name": "New customer", "description": "New customer", "chartColor": "#81c784", "isSystemClassification": true, "classificationType": "NewCustomer"}, "created": "2022-09-26T18:30:58.6124374", "modified": "2022-09-26T18:30:58.6124374", "order": {"orderNumber": "O-000185", "id": "d846f8f6-df83-4aa5-c53c-08da9f9a7136"}}, "emitted_at": 1696446766271} +{"stream": "booking", "data": {"id": "b9e3249c-2794-4a6a-1d8d-08da9f9a72e8", "bookingType": "New", "accountCategory": "New", "changeType": "None", "cmrr": {"amount": 500.0, "amountInBaseCurrency": 5416.9}, "acv": {"amount": 6000.0, "amountInBaseCurrency": 65002.8}, "emrr": {"amount": 500.0, "amountInBaseCurrency": 5416.9}, "oneTimeFees": {"amount": 0.0, "amountInBaseCurrency": 0.0}, "tcv": {"amount": 18000.0, "amountInBaseCurrency": 195008.4}, "fmrr": {"amount": 0.0, "amountInBaseCurrency": 0.0}, "effectiveDate": "2022-09-26T00:00:00", "bookingLines": [{"charge": {"name": "Product simple 2", "chargeNumber": "OPC-000186", "description": "Product simple 2", "id": "43e9f3a0-ac1b-4dc7-8fcf-08da9f9a7175"}, "cmrr": {"amount": 500.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 5416.9, "baseCurrencyCode": "SEK"}, "acv": {"amount": 6000.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 65002.8, "baseCurrencyCode": "SEK"}, "emrr": {"amount": 500.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 5416.9, "baseCurrencyCode": "SEK"}, "oneTimeFees": {"amount": 0.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 0.0, "baseCurrencyCode": "SEK"}, "tcv": {"amount": 18000.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 195008.4, "baseCurrencyCode": "SEK"}, "fmrr": {"amount": 0.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 0.0, "baseCurrencyCode": "SEK"}, "created": "2022-09-26T18:31:00.3292686", "modified": "2022-09-26T18:31:00.3292686"}], "classification": {"name": "New customer", "description": "New customer", "chartColor": "#81c784", "isSystemClassification": true, "classificationType": "NewCustomer"}, "created": "2022-09-26T18:31:00.329271", "modified": "2022-09-26T18:31:00.329271", "order": {"orderNumber": "O-000186", "id": "a216f4e7-e7ac-4f97-c53d-08da9f9a7136"}}, "emitted_at": 1696446766273} +{"stream": "booking", "data": {"id": "514726c4-b995-4802-1d8e-08da9f9a72e8", "bookingType": "New", "accountCategory": "New", "changeType": "None", "cmrr": {"amount": 750.0, "amountInBaseCurrency": 8125.35}, "acv": {"amount": 9000.0, "amountInBaseCurrency": 97504.2}, "emrr": {"amount": 750.0, "amountInBaseCurrency": 8125.35}, "oneTimeFees": {"amount": 0.0, "amountInBaseCurrency": 0.0}, "tcv": {"amount": 27000.0, "amountInBaseCurrency": 292512.6}, "fmrr": {"amount": 0.0, "amountInBaseCurrency": 0.0}, "effectiveDate": "2022-09-26T00:00:00", "bookingLines": [{"charge": {"name": "Product simple 2", "chargeNumber": "OPC-000187", "description": "Product simple 2", "id": "36df068f-ccf0-4f23-8fd0-08da9f9a7175"}, "cmrr": {"amount": 750.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 8125.35, "baseCurrencyCode": "SEK"}, "acv": {"amount": 9000.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 97504.2, "baseCurrencyCode": "SEK"}, "emrr": {"amount": 750.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 8125.35, "baseCurrencyCode": "SEK"}, "oneTimeFees": {"amount": 0.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 0.0, "baseCurrencyCode": "SEK"}, "tcv": {"amount": 27000.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 292512.6, "baseCurrencyCode": "SEK"}, "fmrr": {"amount": 0.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 0.0, "baseCurrencyCode": "SEK"}, "created": "2022-09-26T18:31:01.9532166", "modified": "2022-09-26T18:31:01.9532166"}], "classification": {"name": "New customer", "description": "New customer", "chartColor": "#81c784", "isSystemClassification": true, "classificationType": "NewCustomer"}, "created": "2022-09-26T18:31:01.9532189", "modified": "2022-09-26T18:31:01.9532189", "order": {"orderNumber": "O-000187", "id": "b366d29e-4156-46a4-c53e-08da9f9a7136"}}, "emitted_at": 1696446766274} +{"stream": "invoice", "data": {"id": "5eaf6136-0926-47cc-b798-08da9f8c8f90", "invoiceNumber": "I-000003", "status": "Posted", "account": {"name": "Example Customer 1", "accountNumber": "A-000002", "id": "0901e0a9-cf4f-48c8-8cba-08da9f9a6eb8"}, "invoiceDate": "2022-09-26T00:00:00", "dueDate": "2022-10-26T00:00:00", "daysPastDue": 343, "nrOfReminders": 0, "paymentTerm": {"id": "54c4a8ef-6425-4eef-065b-08da9af238cb", "days": 30, "name": "30"}, "currency": "EUR", "subtotal": 36000.0, "tax": 0.0, "totalAmount": 36000.0, "totalRoundingAmount": 0.0, "settledAmount": 0.0, "balancedAmount": 36000.0, "taxIncluded": false, "invoiceAddress": {"id": "10c20059-ccc7-49d4-197e-08da9f8abe33", "street": "Street 1", "city": "City name 1", "zip": "11111", "country": "United States of America"}, "deliveryAddress": {"id": "127c869d-6d95-4e26-197d-08da9f8abe33", "country": "United States of America"}, "invoiceBatchId": "2d101aed-f114-4556-ca90-08da9f8c8f8a", "invoiceLines": [{"id": "d2e2cd62-9e58-48d5-5b0d-08da9f8c8f93", "invoiceLineNumber": 1, "productNumber": "OP-000185", "productName": "Product simple 2", "chargeDescription": "Product simple 2", "chargeNumber": "OPC-000185", "price": 0.0, "subtotal": 36000.0, "total": 36000.0, "tax": 0.0, "subtotalPreDiscount": 36000.0, "totalPreDiscount": 36000.0, "taxPreDiscount": 0.0, "servicePeriodStartDate": "2022-01-01T00:00:00", "servicePeriodEndDate": "2022-12-31T00:00:00", "orderChargeId": "f87905e7-eb9e-4200-8fce-08da9f9a7175", "orderId": "d846f8f6-df83-4aa5-c53c-08da9f9a7136", "accountId": "0901e0a9-cf4f-48c8-8cba-08da9f9a6eb8", "chargePlanId": "9d096381-44b1-4b41-0b24-08da9d2d7d44", "orderProductId": "4fb45ef5-612c-4475-2dac-08da9f9a715f", "customFields": {}, "deferredRevenue": {"id": "2a2bbabc-bd73-4276-9503-08da9af23710", "code": "2970", "name": "Deferred Revenue", "description": "Deferred Revenue"}, "recognizedRevenue": {"id": "4db1e524-c02d-4f9b-9504-08da9af23710", "code": "3000", "name": "Revenue", "description": "Revenue"}, "created": "2022-09-26T18:37:08.0510778", "modified": "2022-09-26T18:37:08.0510778", "taxCategoryName": "Export", "taxRate": 0.0}], "ourReference": "John", "invoiceType": "Invoice", "sendMethod": "No send method", "exchangeRate": 10.8338, "invoiceDeliveryMethod": "Print", "disableAutomaticInvoiceReminder": false, "accountsReceivable": {"id": "a5b0dfd2-7b18-4311-9500-08da9af23710", "code": "1510", "name": "Accounts Receivable", "description": "Accounts Receivable"}, "customFields": {}, "created": "2022-09-26T18:37:08.0510742", "modified": "2022-09-26T18:37:18.9886116"}, "emitted_at": 1696446767262} +{"stream": "invoice", "data": {"id": "81b0e11f-b141-4659-b799-08da9f8c8f90", "invoiceNumber": "I-000004", "status": "Posted", "account": {"name": "Example Customer 2", "accountNumber": "A-000003", "id": "1dc25080-543d-4dc6-8cbb-08da9f9a6eb8"}, "invoiceDate": "2022-09-26T00:00:00", "dueDate": "2022-10-26T00:00:00", "daysPastDue": 343, "nrOfReminders": 0, "paymentTerm": {"id": "54c4a8ef-6425-4eef-065b-08da9af238cb", "days": 30, "name": "30"}, "currency": "EUR", "subtotal": 6000.0, "tax": 0.0, "totalAmount": 6000.0, "totalRoundingAmount": 0.0, "settledAmount": 0.0, "balancedAmount": 6000.0, "taxIncluded": false, "invoiceAddress": {"id": "dc378c81-2625-4e03-1980-08da9f8abe33", "street": "Street 2", "city": "City name 2", "zip": "11112", "country": "United States of America"}, "deliveryAddress": {"id": "8b4e0007-ebe4-4233-197f-08da9f8abe33", "country": "United States of America"}, "invoiceBatchId": "2d101aed-f114-4556-ca90-08da9f8c8f8a", "invoiceLines": [{"id": "627dd5b5-addd-4dae-5b0e-08da9f8c8f93", "invoiceLineNumber": 1, "productNumber": "OP-000186", "productName": "Product simple 2", "chargeDescription": "Product simple 2", "chargeNumber": "OPC-000186", "price": 0.0, "subtotal": 6000.0, "total": 6000.0, "tax": 0.0, "subtotalPreDiscount": 6000.0, "totalPreDiscount": 6000.0, "taxPreDiscount": 0.0, "servicePeriodStartDate": "2022-01-01T00:00:00", "servicePeriodEndDate": "2022-12-31T00:00:00", "orderChargeId": "43e9f3a0-ac1b-4dc7-8fcf-08da9f9a7175", "orderId": "a216f4e7-e7ac-4f97-c53d-08da9f9a7136", "accountId": "1dc25080-543d-4dc6-8cbb-08da9f9a6eb8", "chargePlanId": "9d096381-44b1-4b41-0b24-08da9d2d7d44", "orderProductId": "4545707b-16c6-4b6c-2dad-08da9f9a715f", "customFields": {}, "deferredRevenue": {"id": "2a2bbabc-bd73-4276-9503-08da9af23710", "code": "2970", "name": "Deferred Revenue", "description": "Deferred Revenue"}, "recognizedRevenue": {"id": "4db1e524-c02d-4f9b-9504-08da9af23710", "code": "3000", "name": "Revenue", "description": "Revenue"}, "created": "2022-09-26T18:37:08.0510837", "modified": "2022-09-26T18:37:08.0510837", "taxCategoryName": "Export", "taxRate": 0.0}], "ourReference": "John", "invoiceType": "Invoice", "sendMethod": "No send method", "exchangeRate": 10.8338, "invoiceDeliveryMethod": "Print", "disableAutomaticInvoiceReminder": false, "accountsReceivable": {"id": "a5b0dfd2-7b18-4311-9500-08da9af23710", "code": "1510", "name": "Accounts Receivable", "description": "Accounts Receivable"}, "customFields": {}, "created": "2022-09-26T18:37:08.0510802", "modified": "2022-09-26T18:37:19.3210414"}, "emitted_at": 1696446767263} +{"stream": "invoice", "data": {"id": "45145766-dfd7-4cfc-b79a-08da9f8c8f90", "invoiceNumber": "I-000005", "status": "Posted", "account": {"name": "Example Customer 3", "accountNumber": "A-000004", "id": "013a0b63-7183-4450-8cbc-08da9f9a6eb8"}, "invoiceDate": "2022-09-26T00:00:00", "dueDate": "2022-10-26T00:00:00", "daysPastDue": 343, "nrOfReminders": 0, "paymentTerm": {"id": "54c4a8ef-6425-4eef-065b-08da9af238cb", "days": 30, "name": "30"}, "currency": "EUR", "subtotal": 9000.0, "tax": 0.0, "totalAmount": 9000.0, "totalRoundingAmount": 0.0, "settledAmount": 0.0, "balancedAmount": 9000.0, "taxIncluded": false, "invoiceAddress": {"id": "5a3ad10c-e82a-4bdb-1982-08da9f8abe33", "street": "Street 3", "city": "City name 3", "zip": "11113", "country": "United States of America"}, "deliveryAddress": {"id": "6e43cbe3-628f-4c85-1981-08da9f8abe33", "country": "United States of America"}, "invoiceBatchId": "2d101aed-f114-4556-ca90-08da9f8c8f8a", "invoiceLines": [{"id": "6ef472d5-cabc-4483-5b0f-08da9f8c8f93", "invoiceLineNumber": 1, "productNumber": "OP-000187", "productName": "Product simple 2", "chargeDescription": "Product simple 2", "chargeNumber": "OPC-000187", "price": 0.0, "subtotal": 9000.0, "total": 9000.0, "tax": 0.0, "subtotalPreDiscount": 9000.0, "totalPreDiscount": 9000.0, "taxPreDiscount": 0.0, "servicePeriodStartDate": "2022-02-01T00:00:00", "servicePeriodEndDate": "2023-01-31T00:00:00", "orderChargeId": "36df068f-ccf0-4f23-8fd0-08da9f9a7175", "orderId": "b366d29e-4156-46a4-c53e-08da9f9a7136", "accountId": "013a0b63-7183-4450-8cbc-08da9f9a6eb8", "chargePlanId": "9d096381-44b1-4b41-0b24-08da9d2d7d44", "orderProductId": "9ee429b6-2fa8-476f-2dae-08da9f9a715f", "customFields": {}, "deferredRevenue": {"id": "2a2bbabc-bd73-4276-9503-08da9af23710", "code": "2970", "name": "Deferred Revenue", "description": "Deferred Revenue"}, "recognizedRevenue": {"id": "4db1e524-c02d-4f9b-9504-08da9af23710", "code": "3000", "name": "Revenue", "description": "Revenue"}, "created": "2022-09-26T18:37:08.0510894", "modified": "2022-09-26T18:37:08.0510894", "taxCategoryName": "Export", "taxRate": 0.0}], "ourReference": "John", "invoiceType": "Invoice", "sendMethod": "No send method", "exchangeRate": 10.8338, "invoiceDeliveryMethod": "Print", "disableAutomaticInvoiceReminder": false, "accountsReceivable": {"id": "a5b0dfd2-7b18-4311-9500-08da9af23710", "code": "1510", "name": "Accounts Receivable", "description": "Accounts Receivable"}, "customFields": {}, "created": "2022-09-26T18:37:08.051086", "modified": "2022-09-26T18:37:19.8131564"}, "emitted_at": 1696446767263} +{"stream": "product", "data": {"id": "f6c380a7-b4f0-4f96-689c-08da9ae1caef", "productNumber": "P-000001", "name": "Product simple", "productType": "Simple", "category": "Core service", "isFrameworkProduct": false, "chargePlans": [{"id": "eb03f071-6e0e-41ad-9120-08da9ae1caef", "chargePlanNumber": "CP-000001", "name": "Product simple", "effectiveStartDate": "2022-09-21T13:22:26.3750684", "charges": [{"id": "110d6e9a-c374-44ca-2839-08da9ae1caf0", "chargeNumber": "C-000001", "name": "Product simple", "model": "Flat", "chargeType": "Recurring", "defaultQuantity": 0.0, "pricePeriod": "Monthly", "usageRating": "Sum", "createInvoiceLinesPerTier": false, "billingDay": "FromOrder", "specificBillingDay": 0, "billingPeriod": "Monthly", "periodAlignment": "AlignToOrder", "billingTiming": "InAdvance", "taxTemplate": "Standard", "taxIncluded": false, "created": "2022-09-21T13:22:26.4477862", "modified": "2022-09-21T13:22:26.4477862", "deferredRevenueAccount": "2970", "recognizedRevenueAccount": "3000", "customFields": {}, "priceDetails": [{"currency": "SEK", "price": 1000.0}]}], "customFields": {}, "created": "2022-09-21T13:22:26.4477847", "modified": "2022-09-21T13:22:26.4477847"}], "created": "2022-09-21T13:22:26.4477821", "modified": "2022-09-21T13:22:26.4477821", "customFields": {}}, "emitted_at": 1696446767963} +{"stream": "product", "data": {"id": "74d7e920-84ad-4743-92e1-08da9d2d7d43", "productNumber": "P-000002", "name": "Product simple 2", "productType": "Simple", "category": "Core service", "isFrameworkProduct": false, "chargePlans": [{"id": "9d096381-44b1-4b41-0b24-08da9d2d7d44", "chargePlanNumber": "CP-000002", "name": "Product simple 2", "effectiveStartDate": "2022-09-24T13:17:06.0110839", "charges": [{"id": "9dfd63cb-cb0e-40e7-1dc5-08da9d2d7d45", "chargeNumber": "C-000002", "name": "Product simple 2", "model": "Flat", "chargeType": "Recurring", "defaultQuantity": 0.0, "pricePeriod": "Monthly", "usageRating": "Sum", "createInvoiceLinesPerTier": false, "billingDay": "FromOrder", "specificBillingDay": 0, "billingPeriod": "Monthly", "periodAlignment": "AlignToOrder", "billingTiming": "InAdvance", "taxTemplate": "Standard", "taxIncluded": false, "created": "2022-09-24T13:17:06.5267131", "modified": "2022-09-24T13:17:06.5267131", "deferredRevenueAccount": "2970", "recognizedRevenueAccount": "3000", "customFields": {}, "priceDetails": [{"currency": "SEK", "price": 1000.0}, {"currency": "EUR", "price": 100.0}]}], "customFields": {}, "created": "2022-09-24T13:17:06.5267117", "modified": "2022-09-24T13:17:06.5267117"}], "created": "2022-09-24T13:17:06.526709", "modified": "2022-09-24T13:17:06.526709", "customFields": {}}, "emitted_at": 1696446767965} +{"stream": "subscription", "data": {"id": "d846f8f6-df83-4aa5-c53c-08da9f9a7136", "orderNumber": "O-000185", "version": 1, "isLastVersion": true, "status": "Active", "effectiveStartDate": "2022-01-01T00:00:00", "effectiveEndDate": "2024-12-31T00:00:00", "orderDate": "2021-01-01T00:00:00", "noticePeriodDate": "2024-09-30T00:00:00", "created": "2022-09-26T18:30:58.494057", "modified": "2022-09-26T18:30:58.494057", "noticePeriod": 3, "term": 36, "renewalTerm": 12, "isAutoRenewed": true, "orderType": "Subscription", "termType": "Termed", "orderPaymentMethod": "Invoice", "invoiceSeparatly": true, "ourReference": "John", "invoiceAddress": {"id": "40c16354-6688-4640-b822-57fbccd7704a", "street": "Street 1", "city": "City name 1", "zip": "11111", "country": "United States of America"}, "deliveryAddress": {"id": "87d15064-1180-4aaa-b82e-ba5bc9cdb474", "country": "United States of America"}, "orderBillingPeriod": "Monthly", "setOrderBillingPeriod": false, "paymentTerm": "30", "useAccountInvoiceBatchGroup": false, "account": {"name": "Example Customer 1", "accountNumber": "A-000002", "id": "0901e0a9-cf4f-48c8-8cba-08da9f9a6eb8"}, "invoiceAccount": {"name": "Example Customer 1", "accountNumber": "A-000002", "id": "0901e0a9-cf4f-48c8-8cba-08da9f9a6eb8"}, "products": [{"id": "4fb45ef5-612c-4475-2dac-08da9f9a715f", "productNumber": "OP-000185", "chargePlanId": "9d096381-44b1-4b41-0b24-08da9d2d7d44", "chargePlanName": "Product simple 2", "chargePlanNumber": "CP-000002", "productLineNumber": 0, "name": "Product simple 2", "charges": [{"id": "f87905e7-eb9e-4200-8fce-08da9f9a7175", "chargeNumber": "OPC-000185", "version": 1, "isLastVersion": true, "name": "Product simple 2", "chargeType": "Recurring", "priceModel": "Flat", "effectiveStartDate": "2022-01-01T00:00:00", "effectiveEndDate": "2024-12-31T00:00:00", "quantity": 0.0, "startOn": "AlignToSubscription", "endOn": "AlignToSubscription", "chargedThroughDate": "2023-01-01T00:00:00", "pricePeriod": "Monthly", "usageRating": "Sum", "revenueRecognitionRule": "Recognized monthly over time", "billingDay": "FromOrder", "specificBillingDay": 0, "billingPeriod": "Annual", "billingTiming": "InAdvance", "periodAlignment": "AlignToOrder", "taxTemplate": "Standard", "taxIncluded": false, "createInvoiceLinesPerTier": false, "estimatedUsage": 0, "estimatedQuantity": 0, "deferredRevenueAccount": "2970", "recognizedRevenueAccount": "3000", "changeState": "New", "displayPrice": 3000.0, "customFields": {}, "priceDetails": [{"tier": 0, "price": 3000.0, "listPrice": 100.0, "fromQuantity": 0.0, "toQuantity": 0.0, "priceBase": "Flat", "lineDiscountPercent": 0.0, "lineDiscountAmount": 0.0}], "recurringMonthlyAmount": 3000.0, "recurringMonthlyAmountBase": 32501.4, "features": [], "orderDiscounts": [], "cmrr": {"amount": 3000.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 32501.4, "baseCurrencyCode": "SEK"}, "acv": {"amount": 36000.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 390016.8, "baseCurrencyCode": "SEK"}, "tcv": {"amount": 108000.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 1170050.4, "baseCurrencyCode": "SEK"}, "emrr": {"amount": 3000.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 32501.4, "baseCurrencyCode": "SEK"}, "oneTimeFees": {"amount": 0.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 0.0, "baseCurrencyCode": "SEK"}, "orderProductId": "4fb45ef5-612c-4475-2dac-08da9f9a715f", "orderId": "d846f8f6-df83-4aa5-c53c-08da9f9a7136", "created": "2022-09-26T18:30:58.4940602", "modified": "2022-09-26T18:37:18.5434643"}], "customFields": {}, "cmrr": {"amount": 3000.0, "currencyCode": "EUR", "baseCurrencyAmount": 32501.4, "baseCurrencyCode": "SEK"}, "acv": {"amount": 36000.0, "currencyCode": "EUR", "baseCurrencyAmount": 390016.8, "baseCurrencyCode": "SEK"}, "emrr": {"amount": 3000.0, "currencyCode": "EUR", "baseCurrencyAmount": 32501.4, "baseCurrencyCode": "SEK"}, "oneTimeFees": {"amount": 0.0, "currencyCode": "EUR", "baseCurrencyAmount": 0.0, "baseCurrencyCode": "SEK"}, "tcv": {"amount": 108000.0, "currencyCode": "EUR", "baseCurrencyAmount": 1170050.4, "baseCurrencyCode": "SEK"}, "created": "2022-09-26T18:30:58.4940591", "modified": "2022-09-26T18:30:58.4940591"}], "milestones": [], "orderDiscounts": [], "currency": "EUR", "customFields": {}, "cmrr": {"amount": 3000.0, "currencyCode": "EUR", "baseCurrencyAmount": 32501.4, "baseCurrencyCode": "SEK"}, "acv": {"amount": 36000.0, "currencyCode": "EUR", "baseCurrencyAmount": 390016.8, "baseCurrencyCode": "SEK"}, "emrr": {"amount": 3000.0, "currencyCode": "EUR", "baseCurrencyAmount": 32501.4, "baseCurrencyCode": "SEK"}, "oneTimeFees": {"amount": 0.0, "currencyCode": "EUR", "baseCurrencyAmount": 0.0, "baseCurrencyCode": "SEK"}, "tcv": {"amount": 108000.0, "currencyCode": "EUR", "baseCurrencyAmount": 1170050.4, "baseCurrencyCode": "SEK"}}, "emitted_at": 1696446768826} +{"stream": "subscription", "data": {"id": "a216f4e7-e7ac-4f97-c53d-08da9f9a7136", "orderNumber": "O-000186", "version": 1, "isLastVersion": true, "status": "Active", "effectiveStartDate": "2022-01-01T00:00:00", "effectiveEndDate": "2024-12-31T00:00:00", "orderDate": "2021-01-01T00:00:00", "noticePeriodDate": "2024-09-30T00:00:00", "created": "2022-09-26T18:31:00.1883267", "modified": "2022-09-26T18:31:00.1883267", "noticePeriod": 3, "term": 36, "renewalTerm": 12, "isAutoRenewed": true, "orderType": "Subscription", "termType": "Termed", "orderPaymentMethod": "Invoice", "invoiceSeparatly": true, "ourReference": "John", "invoiceAddress": {"id": "2223c2bd-0940-4f81-88fc-a2eee02f26e6", "street": "Street 2", "city": "City name 2", "zip": "11112", "country": "United States of America"}, "deliveryAddress": {"id": "8fdae644-4b66-4156-9ade-47e2e511ab5c", "country": "United States of America"}, "orderBillingPeriod": "Monthly", "setOrderBillingPeriod": false, "paymentTerm": "30", "useAccountInvoiceBatchGroup": false, "account": {"name": "Example Customer 2", "accountNumber": "A-000003", "id": "1dc25080-543d-4dc6-8cbb-08da9f9a6eb8"}, "invoiceAccount": {"name": "Example Customer 2", "accountNumber": "A-000003", "id": "1dc25080-543d-4dc6-8cbb-08da9f9a6eb8"}, "products": [{"id": "4545707b-16c6-4b6c-2dad-08da9f9a715f", "productNumber": "OP-000186", "chargePlanId": "9d096381-44b1-4b41-0b24-08da9d2d7d44", "chargePlanName": "Product simple 2", "chargePlanNumber": "CP-000002", "productLineNumber": 0, "name": "Product simple 2", "charges": [{"id": "43e9f3a0-ac1b-4dc7-8fcf-08da9f9a7175", "chargeNumber": "OPC-000186", "version": 1, "isLastVersion": true, "name": "Product simple 2", "chargeType": "Recurring", "priceModel": "Flat", "effectiveStartDate": "2022-01-01T00:00:00", "effectiveEndDate": "2024-12-31T00:00:00", "quantity": 0.0, "startOn": "AlignToSubscription", "endOn": "AlignToSubscription", "chargedThroughDate": "2023-01-01T00:00:00", "pricePeriod": "Monthly", "usageRating": "Sum", "revenueRecognitionRule": "Recognized monthly over time", "billingDay": "FromOrder", "specificBillingDay": 0, "billingPeriod": "Annual", "billingTiming": "InAdvance", "periodAlignment": "AlignToOrder", "taxTemplate": "Standard", "taxIncluded": false, "createInvoiceLinesPerTier": false, "estimatedUsage": 0, "estimatedQuantity": 0, "deferredRevenueAccount": "2970", "recognizedRevenueAccount": "3000", "changeState": "New", "displayPrice": 500.0, "customFields": {}, "priceDetails": [{"tier": 0, "price": 500.0, "listPrice": 100.0, "fromQuantity": 0.0, "toQuantity": 0.0, "priceBase": "Flat", "lineDiscountPercent": 0.0, "lineDiscountAmount": 0.0}], "recurringMonthlyAmount": 500.0, "recurringMonthlyAmountBase": 5416.9, "features": [], "orderDiscounts": [], "cmrr": {"amount": 500.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 5416.9, "baseCurrencyCode": "SEK"}, "acv": {"amount": 6000.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 65002.8, "baseCurrencyCode": "SEK"}, "tcv": {"amount": 18000.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 195008.4, "baseCurrencyCode": "SEK"}, "emrr": {"amount": 500.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 5416.9, "baseCurrencyCode": "SEK"}, "oneTimeFees": {"amount": 0.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 0.0, "baseCurrencyCode": "SEK"}, "orderProductId": "4545707b-16c6-4b6c-2dad-08da9f9a715f", "orderId": "a216f4e7-e7ac-4f97-c53d-08da9f9a7136", "created": "2022-09-26T18:31:00.1883306", "modified": "2022-09-26T18:37:19.168274"}], "customFields": {}, "cmrr": {"amount": 500.0, "currencyCode": "EUR", "baseCurrencyAmount": 5416.9, "baseCurrencyCode": "SEK"}, "acv": {"amount": 6000.0, "currencyCode": "EUR", "baseCurrencyAmount": 65002.8, "baseCurrencyCode": "SEK"}, "emrr": {"amount": 500.0, "currencyCode": "EUR", "baseCurrencyAmount": 5416.9, "baseCurrencyCode": "SEK"}, "oneTimeFees": {"amount": 0.0, "currencyCode": "EUR", "baseCurrencyAmount": 0.0, "baseCurrencyCode": "SEK"}, "tcv": {"amount": 18000.0, "currencyCode": "EUR", "baseCurrencyAmount": 195008.4, "baseCurrencyCode": "SEK"}, "created": "2022-09-26T18:31:00.1883293", "modified": "2022-09-26T18:31:00.1883293"}], "milestones": [], "orderDiscounts": [], "currency": "EUR", "customFields": {}, "cmrr": {"amount": 500.0, "currencyCode": "EUR", "baseCurrencyAmount": 5416.9, "baseCurrencyCode": "SEK"}, "acv": {"amount": 6000.0, "currencyCode": "EUR", "baseCurrencyAmount": 65002.8, "baseCurrencyCode": "SEK"}, "emrr": {"amount": 500.0, "currencyCode": "EUR", "baseCurrencyAmount": 5416.9, "baseCurrencyCode": "SEK"}, "oneTimeFees": {"amount": 0.0, "currencyCode": "EUR", "baseCurrencyAmount": 0.0, "baseCurrencyCode": "SEK"}, "tcv": {"amount": 18000.0, "currencyCode": "EUR", "baseCurrencyAmount": 195008.4, "baseCurrencyCode": "SEK"}}, "emitted_at": 1696446768827} +{"stream": "subscription", "data": {"id": "b366d29e-4156-46a4-c53e-08da9f9a7136", "orderNumber": "O-000187", "version": 1, "isLastVersion": true, "status": "Active", "effectiveStartDate": "2022-02-01T00:00:00", "effectiveEndDate": "2025-01-31T00:00:00", "orderDate": "2021-02-01T00:00:00", "noticePeriodDate": "2024-10-31T00:00:00", "created": "2022-09-26T18:31:01.8373875", "modified": "2022-09-26T18:31:01.8373875", "noticePeriod": 3, "term": 36, "renewalTerm": 12, "isAutoRenewed": true, "orderType": "Subscription", "termType": "Termed", "orderPaymentMethod": "Invoice", "invoiceSeparatly": true, "ourReference": "John", "invoiceAddress": {"id": "036f70d0-b0e1-4b8b-bca8-1b679caaf435", "street": "Street 3", "city": "City name 3", "zip": "11113", "country": "United States of America"}, "deliveryAddress": {"id": "4fb9c18f-93db-42c4-9e98-5e422b62066d", "country": "United States of America"}, "orderBillingPeriod": "Monthly", "setOrderBillingPeriod": false, "paymentTerm": "30", "useAccountInvoiceBatchGroup": false, "account": {"name": "Example Customer 3", "accountNumber": "A-000004", "id": "013a0b63-7183-4450-8cbc-08da9f9a6eb8"}, "invoiceAccount": {"name": "Example Customer 3", "accountNumber": "A-000004", "id": "013a0b63-7183-4450-8cbc-08da9f9a6eb8"}, "products": [{"id": "9ee429b6-2fa8-476f-2dae-08da9f9a715f", "productNumber": "OP-000187", "chargePlanId": "9d096381-44b1-4b41-0b24-08da9d2d7d44", "chargePlanName": "Product simple 2", "chargePlanNumber": "CP-000002", "productLineNumber": 0, "name": "Product simple 2", "charges": [{"id": "36df068f-ccf0-4f23-8fd0-08da9f9a7175", "chargeNumber": "OPC-000187", "version": 1, "isLastVersion": true, "name": "Product simple 2", "chargeType": "Recurring", "priceModel": "Flat", "effectiveStartDate": "2022-02-01T00:00:00", "effectiveEndDate": "2025-01-31T00:00:00", "quantity": 0.0, "startOn": "AlignToSubscription", "endOn": "AlignToSubscription", "chargedThroughDate": "2023-02-01T00:00:00", "pricePeriod": "Monthly", "usageRating": "Sum", "revenueRecognitionRule": "Recognized monthly over time", "billingDay": "FromOrder", "specificBillingDay": 0, "billingPeriod": "Annual", "billingTiming": "InAdvance", "periodAlignment": "AlignToOrder", "taxTemplate": "Standard", "taxIncluded": false, "createInvoiceLinesPerTier": false, "estimatedUsage": 0, "estimatedQuantity": 0, "deferredRevenueAccount": "2970", "recognizedRevenueAccount": "3000", "changeState": "New", "displayPrice": 750.0, "customFields": {}, "priceDetails": [{"tier": 0, "price": 750.0, "listPrice": 100.0, "fromQuantity": 0.0, "toQuantity": 0.0, "priceBase": "Flat", "lineDiscountPercent": 0.0, "lineDiscountAmount": 0.0}], "recurringMonthlyAmount": 750.0, "recurringMonthlyAmountBase": 8125.35, "features": [], "orderDiscounts": [], "cmrr": {"amount": 750.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 8125.35, "baseCurrencyCode": "SEK"}, "acv": {"amount": 9000.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 97504.2, "baseCurrencyCode": "SEK"}, "tcv": {"amount": 27000.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 292512.6, "baseCurrencyCode": "SEK"}, "emrr": {"amount": 750.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 8125.35, "baseCurrencyCode": "SEK"}, "oneTimeFees": {"amount": 0.0, "currencyCode": "EUR", "currencyConversionDate": "2022-09-26T00:00:00", "baseCurrencyAmount": 0.0, "baseCurrencyCode": "SEK"}, "orderProductId": "9ee429b6-2fa8-476f-2dae-08da9f9a715f", "orderId": "b366d29e-4156-46a4-c53e-08da9f9a7136", "created": "2022-09-26T18:31:01.8373913", "modified": "2022-09-26T18:37:19.5818297"}], "customFields": {}, "cmrr": {"amount": 750.0, "currencyCode": "EUR", "baseCurrencyAmount": 8125.35, "baseCurrencyCode": "SEK"}, "acv": {"amount": 9000.0, "currencyCode": "EUR", "baseCurrencyAmount": 97504.2, "baseCurrencyCode": "SEK"}, "emrr": {"amount": 750.0, "currencyCode": "EUR", "baseCurrencyAmount": 8125.35, "baseCurrencyCode": "SEK"}, "oneTimeFees": {"amount": 0.0, "currencyCode": "EUR", "baseCurrencyAmount": 0.0, "baseCurrencyCode": "SEK"}, "tcv": {"amount": 27000.0, "currencyCode": "EUR", "baseCurrencyAmount": 292512.6, "baseCurrencyCode": "SEK"}, "created": "2022-09-26T18:31:01.8373899", "modified": "2022-09-26T18:31:01.8373899"}], "milestones": [], "orderDiscounts": [], "currency": "EUR", "customFields": {}, "cmrr": {"amount": 750.0, "currencyCode": "EUR", "baseCurrencyAmount": 8125.35, "baseCurrencyCode": "SEK"}, "acv": {"amount": 9000.0, "currencyCode": "EUR", "baseCurrencyAmount": 97504.2, "baseCurrencyCode": "SEK"}, "emrr": {"amount": 750.0, "currencyCode": "EUR", "baseCurrencyAmount": 8125.35, "baseCurrencyCode": "SEK"}, "oneTimeFees": {"amount": 0.0, "currencyCode": "EUR", "baseCurrencyAmount": 0.0, "baseCurrencyCode": "SEK"}, "tcv": {"amount": 27000.0, "currencyCode": "EUR", "baseCurrencyAmount": 292512.6, "baseCurrencyCode": "SEK"}}, "emitted_at": 1696446768827} diff --git a/airbyte-integrations/connectors/source-younium/metadata.yaml b/airbyte-integrations/connectors/source-younium/metadata.yaml index 5eeb118c878d..c8d91423df43 100644 --- a/airbyte-integrations/connectors/source-younium/metadata.yaml +++ b/airbyte-integrations/connectors/source-younium/metadata.yaml @@ -1,24 +1,27 @@ data: + registries: + oss: + enabled: true + cloud: + enabled: false + connectorBuildOptions: + # Please update to the latest version of the connector base image. + # https://hub.docker.com/r/airbyte/python-connector-base + # Please use the full address with sha256 hash to guarantee build reproducibility. + baseImage: docker.io/airbyte/python-connector-base:1.0.0@sha256:dd17e347fbda94f7c3abff539be298a65af2d7fc27a307d89297df1081a45c27 connectorSubtype: api connectorType: source definitionId: 9c74c2d7-531a-4ebf-b6d8-6181f805ecdc - dockerImageTag: 0.1.0 + dockerImageTag: 0.3.0 dockerRepository: airbyte/source-younium githubIssueLabel: source-younium icon: younium.svg license: MIT name: Younium - registries: - cloud: - enabled: true - oss: - enabled: true + releaseDate: 2022-11-09 releaseStage: alpha + supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/younium tags: - - language:python - ab_internal: - sl: 100 - ql: 100 - supportLevel: community + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-younium/requirements.txt b/airbyte-integrations/connectors/source-younium/requirements.txt index ecf975e2fa63..d6e1198b1ab1 100644 --- a/airbyte-integrations/connectors/source-younium/requirements.txt +++ b/airbyte-integrations/connectors/source-younium/requirements.txt @@ -1 +1 @@ --e . \ No newline at end of file +-e . diff --git a/airbyte-integrations/connectors/source-younium/setup.py b/airbyte-integrations/connectors/source-younium/setup.py index b2a5f3f95a39..2a8872be5287 100644 --- a/airbyte-integrations/connectors/source-younium/setup.py +++ b/airbyte-integrations/connectors/source-younium/setup.py @@ -6,14 +6,13 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.2", + "airbyte-cdk", ] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", - "pytest~=6.1", + "pytest~=6.2", "pytest-mock~=3.6.1", - "responses~=0.22.0", ] setup( diff --git a/airbyte-integrations/connectors/source-younium/source_younium/components.py b/airbyte-integrations/connectors/source-younium/source_younium/components.py new file mode 100644 index 000000000000..7b9bcaa42a5f --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/source_younium/components.py @@ -0,0 +1,86 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from dataclasses import dataclass +from http import HTTPStatus +from typing import Any, Mapping, Union + +import requests +from airbyte_cdk.sources.declarative.auth.declarative_authenticator import NoAuth +from airbyte_cdk.sources.declarative.interpolation import InterpolatedString +from airbyte_cdk.sources.declarative.types import Config +from requests import HTTPError + +# https://developers.zoom.us/docs/internal-apps/s2s-oauth/#successful-response +# The Bearer token generated by server-to-server token will expire in one hour + + +@dataclass +class CustomYouniumAuthenticator(NoAuth): + config: Config + + username: Union[InterpolatedString, str] + password: Union[InterpolatedString, str] + legal_entity: Union[InterpolatedString, str] + grant_type: Union[InterpolatedString, str] + client_id: Union[InterpolatedString, str] + scope: Union[InterpolatedString, str] + + _access_token = None + _token_type = None + + def __post_init__(self, parameters: Mapping[str, Any]): + self._username = InterpolatedString.create(self.username, parameters=parameters).eval(self.config) + self._password = InterpolatedString.create(self.password, parameters=parameters).eval(self.config) + self._legal_entity = InterpolatedString.create(self.legal_entity, parameters=parameters).eval(self.config) + self._grant_type = InterpolatedString.create(self.grant_type, parameters=parameters).eval(self.config) + self._client_id = InterpolatedString.create(self.client_id, parameters=parameters).eval(self.config) + self._scope = InterpolatedString.create(self.scope, parameters=parameters).eval(self.config) + + def __call__(self, request: requests.PreparedRequest) -> requests.PreparedRequest: + """Attach the page access token to params to authenticate on the HTTP request""" + if self._access_token is None or self._token_type is None: + self._access_token, self._token_type = self.generate_access_token() + + headers = {self.auth_header: f"{self._token_type} {self._access_token}", "Content-type": "application/json"} + + request.headers.update(headers) + + return request + + @property + def auth_header(self) -> str: + return "Authorization" + + @property + def token(self) -> str: + return self._access_token + + def generate_access_token(self) -> (str, str): + # return (str("token123"), str("Bearer")) + try: + headers = {"Content-Type": "application/x-www-form-urlencoded"} + + data = { + "username": self._username, + "password": self._password, + "legal_entity": self._legal_entity, + "grant_type": self._grant_type, + "client_id": self._client_id, + "scope": self._scope, + } + + if self.config.get("playground"): + url = "https://younium-identity-server-sandbox.azurewebsites.net/connect/token" + # url = "http://localhost:3000/playground/auth/token" + else: + url = "https://younium-identity-server.azurewebsites.net/connect/token" + # url = "http://localhost:3000/auth/token" + + rest = requests.post(url, headers=headers, data=data) + if rest.status_code != HTTPStatus.OK: + raise HTTPError(rest.text) + return (rest.json().get("access_token"), rest.json().get("token_type")) + except Exception as e: + raise Exception(f"Error while generating access token: {e}") from e diff --git a/airbyte-integrations/connectors/source-younium/source_younium/manifest.yaml b/airbyte-integrations/connectors/source-younium/source_younium/manifest.yaml new file mode 100644 index 000000000000..46d10c904aab --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/source_younium/manifest.yaml @@ -0,0 +1,124 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + requester: + type: HttpRequester + url_base: "{{ 'https://apisandbox.younium.com' if config['playground'] else 'https://api.younium.com' }}" + http_method: "GET" + + authenticator: + class_name: source_younium.components.CustomYouniumAuthenticator + username: "{{ config['username'] }}" + password: "{{ config['password'] }}" + legal_entity: "{{ config['legal_entity'] }}" + grant_type: password + client_id: apiclient + scope: openid youniumapi profile + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + pagination_strategy: + type: CursorPagination + page_size: 100 + cursor_value: '{{ response.get("nextPage", {}) }}' + stop_condition: '{{ not response.get("nextPage", {}) }}' + page_size_option: + inject_into: request_parameter + type: RequestOption + field_name: PageSize + + requester: + $ref: "#/definitions/requester" + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + account_stream: + $ref: "#/definitions/base_stream" + name: account + $parameters: + path: Accounts + + booking_stream: + $ref: "#/definitions/base_stream" + name: booking + $parameters: + path: Bookings + + invoice_stream: + $ref: "#/definitions/base_stream" + name: invoice + $parameters: + path: Invoices + + product_stream: + $ref: "#/definitions/base_stream" + name: product + $parameters: + path: Products + + subscription_stream: + $ref: "#/definitions/base_stream" + name: subscription + $parameters: + path: Subscriptions + +streams: + - "#/definitions/account_stream" + - "#/definitions/booking_stream" + - "#/definitions/invoice_stream" + - "#/definitions/product_stream" + - "#/definitions/subscription_stream" + +check: + type: CheckStream + stream_names: + - account + - booking + - invoice + - product + - subscription +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/younium + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + title: Younium Spec + type: object + additionalProperties: true + required: + - username + - password + - legal_entity + properties: + username: + title: Username + type: string + description: Username for Younium account + password: + title: Password + type: string + description: Account password for younium account API key + airbyte_secret: true + legal_entity: + title: Legal Entity + type: string + description: Legal Entity that data should be pulled from + playground: + title: Playground environment + type: boolean + description: Property defining if connector is used against playground or production environment + default: false diff --git a/airbyte-integrations/connectors/source-younium/source_younium/schemas/account.json b/airbyte-integrations/connectors/source-younium/source_younium/schemas/account.json new file mode 100644 index 000000000000..fc6fda30bf42 --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/source_younium/schemas/account.json @@ -0,0 +1,212 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "accountNumber": { + "type": ["null", "string"] + }, + "accountType": { + "type": ["null", "string"] + }, + "accountsReceivable": { + "type": ["null", "string"] + }, + "acv": { + "additionalProperties": true, + "properties": { + "amount": { + "type": ["null", "number"] + }, + "baseCurrencyAmount": { + "type": ["null", "number"] + }, + "baseCurrencyCode": { + "type": ["null", "string"] + }, + "currencyCode": { + "type": ["null", "string"] + } + }, + "type": "object" + }, + "addresses": { + "items": { + "additionalProperties": true, + "properties": { + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "street": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": "object" + }, + "type": "array" + }, + "cmrr": { + "additionalProperties": true, + "properties": { + "amount": { + "type": ["null", "number"] + }, + "baseCurrencyAmount": { + "type": ["null", "number"] + }, + "baseCurrencyCode": { + "type": ["null", "string"] + }, + "currencyCode": { + "type": ["null", "string"] + } + }, + "type": "object" + }, + "created": { + "type": ["null", "string"] + }, + "currencyCode": { + "type": ["null", "string"] + }, + "customFields": { + "additionalProperties": true, + "type": "object" + }, + "defaultDeliveryAddress": { + "additionalProperties": true, + "properties": { + "country": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + } + }, + "type": "object" + }, + "defaultInvoiceAddress": { + "additionalProperties": true, + "properties": { + "city": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "street": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": "object" + }, + "defaultPaymentTerm": { + "type": ["null", "string"] + }, + "emrr": { + "additionalProperties": true, + "properties": { + "amount": { + "type": ["null", "number"] + }, + "baseCurrencyAmount": { + "type": ["null", "number"] + }, + "baseCurrencyCode": { + "type": ["null", "string"] + }, + "currencyCode": { + "type": ["null", "string"] + } + }, + "type": "object" + }, + "id": { + "type": ["null", "string"] + }, + "inactive": { + "type": ["null", "boolean"] + }, + "invoiceDeliveryMethod": { + "type": ["null", "string"] + }, + "invoiceTemplateId": { + "type": ["null", "string"] + }, + "modified": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "oneTimeFees": { + "additionalProperties": true, + "properties": { + "amount": { + "type": ["null", "number"] + }, + "baseCurrencyAmount": { + "type": ["null", "number"] + }, + "baseCurrencyCode": { + "type": ["null", "string"] + }, + "currencyCode": { + "type": ["null", "string"] + } + }, + "type": "object" + }, + "organizationNumber": { + "type": ["null", "string"] + }, + "ourReference": { + "type": ["null", "string"] + }, + "taxRegistrationNumber": { + "type": ["null", "string"] + }, + "taxTemplate": { + "type": ["null", "string"] + }, + "tcv": { + "additionalProperties": true, + "properties": { + "amount": { + "type": ["null", "number"] + }, + "baseCurrencyAmount": { + "type": ["null", "number"] + }, + "baseCurrencyCode": { + "type": ["null", "string"] + }, + "currencyCode": { + "type": ["null", "string"] + } + }, + "type": "object" + } + }, + "type": "object" +} diff --git a/airbyte-integrations/connectors/source-younium/source_younium/schemas/booking.json b/airbyte-integrations/connectors/source-younium/source_younium/schemas/booking.json new file mode 100644 index 000000000000..c60a06749428 --- /dev/null +++ b/airbyte-integrations/connectors/source-younium/source_younium/schemas/booking.json @@ -0,0 +1,292 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": true, + "properties": { + "accountCategory": { + "type": ["null", "string"] + }, + "acv": { + "additionalProperties": true, + "properties": { + "amount": { + "type": ["null", "number"] + }, + "amountInBaseCurrency": { + "type": ["null", "number"] + } + }, + "type": "object" + }, + "bookingLines": { + "items": { + "additionalProperties": true, + "properties": { + "acv": { + "additionalProperties": true, + "properties": { + "amount": { + "type": ["null", "number"] + }, + "baseCurrencyAmount": { + "type": ["null", "number"] + }, + "baseCurrencyCode": { + "type": ["null", "string"] + }, + "currencyCode": { + "type": ["null", "string"] + }, + "currencyConversionDate": { + "type": ["null", "string"] + } + }, + "type": "object" + }, + "charge": { + "additionalProperties": true, + "properties": { + "chargeNumber": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + }, + "type": "object" + }, + "cmrr": { + "additionalProperties": true, + "properties": { + "amount": { + "type": ["null", "number"] + }, + "baseCurrencyAmount": { + "type": ["null", "number"] + }, + "baseCurrencyCode": { + "type": ["null", "string"] + }, + "currencyCode": { + "type": ["null", "string"] + }, + "currencyConversionDate": { + "type": ["null", "string"] + } + }, + "type": "object" + }, + "created": { + "type": ["null", "string"] + }, + "emrr": { + "additionalProperties": true, + "properties": { + "amount": { + "type": ["null", "number"] + }, + "baseCurrencyAmount": { + "type": ["null", "number"] + }, + "baseCurrencyCode": { + "type": ["null", "string"] + }, + "currencyCode": { + "type": ["null", "string"] + }, + "currencyConversionDate": { + "type": ["null", "string"] + } + }, + "type": "object" + }, + "fmrr": { + "additionalProperties": true, + "properties": { + "amount": { + "type": ["null", "number"] + }, + "baseCurrencyAmount": { + "type": ["null", "number"] + }, + "baseCurrencyCode": { + "type": ["null", "string"] + }, + "currencyCode": { + "type": ["null", "string"] + }, + "currencyConversionDate": { + "type": ["null", "string"] + } + }, + "type": "object" + }, + "modified": { + "type": ["null", "string"] + }, + "oneTimeFees": { + "additionalProperties": true, + "properties": { + "amount": { + "type": ["null", "number"] + }, + "baseCurrencyAmount": { + "type": ["null", "number"] + }, + "baseCurrencyCode": { + "type": ["null", "string"] + }, + "currencyCode": { + "type": ["null", "string"] + }, + "currencyConversionDate": { + "type": ["null", "string"] + } + }, + "type": "object" + }, + "tcv": { + "additionalProperties": true, + "properties": { + "amount": { + "type": ["null", "number"] + }, + "baseCurrencyAmount": { + "type": ["null", "number"] + }, + "baseCurrencyCode": { + "type": ["null", "string"] + }, + "currencyCode": { + "type": ["null", "string"] + }, + "currencyConversionDate": { + "type": ["null", "string"] + } + }, + "type": "object" + } + }, + "type": "object" + }, + "type": "array" + }, + "bookingType": { + "type": ["null", "string"] + }, + "changeType": { + "type": ["null", "string"] + }, + "classification": { + "additionalProperties": true, + "properties": { + "chartColor": { + "type": ["null", "string"] + }, + "classificationType": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "isSystemClassification": { + "type": ["null", "boolean"] + }, + "name": { + "type": ["null", "string"] + } + }, + "type": "object" + }, + "cmrr": { + "additionalProperties": true, + "properties": { + "amount": { + "type": ["null", "number"] + }, + "amountInBaseCurrency": { + "type": ["null", "number"] + } + }, + "type": "object" + }, + "created": { + "type": ["null", "string"] + }, + "effectiveDate": { + "type": ["null", "string"] + }, + "emrr": { + "additionalProperties": true, + "properties": { + "amount": { + "type": ["null", "number"] + }, + "amountInBaseCurrency": { + "type": ["null", "number"] + } + }, + "type": "object" + }, + "fmrr": { + "additionalProperties": true, + "properties": { + "amount": { + "type": ["null", "number"] + }, + "amountInBaseCurrency": { + "type": ["null", "number"] + } + }, + "type": "object" + }, + "id": { + "type": ["null", "string"] + }, + "modified": { + "type": ["null", "string"] + }, + "oneTimeFees": { + "additionalProperties": true, + "properties": { + "amount": { + "type": ["null", "number"] + }, + "amountInBaseCurrency": { + "type": ["null", "number"] + } + }, + "type": "object" + }, + "order": { + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "string"] + }, + "orderNumber": { + "type": ["null", "string"] + } + }, + "type": "object" + }, + "tcv": { + "additionalProperties": true, + "properties": { + "amount": { + "type": ["null", "number"] + }, + "amountInBaseCurrency": { + "type": ["null", "number"] + } + }, + "type": "object" + } + }, + "type": "object" +} diff --git a/airbyte-integrations/connectors/source-younium/source_younium/schemas/invoice.json b/airbyte-integrations/connectors/source-younium/source_younium/schemas/invoice.json index 96c65fe57d0c..26de8b306f75 100644 --- a/airbyte-integrations/connectors/source-younium/source_younium/schemas/invoice.json +++ b/airbyte-integrations/connectors/source-younium/source_younium/schemas/invoice.json @@ -1,121 +1,131 @@ { "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": true, "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "invoiceNumber": { - "type": "string" + "type": ["null", "string"] }, "status": { - "type": "string" + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"] + }, + "invoiceDeliveryMethod": { + "type": ["null", "string"] + }, + "modified": { + "type": ["null", "string"] }, "account": { "type": "object", "properties": { "name": { - "type": "string" + "type": ["null", "string"] }, "accountNumber": { - "type": "string" + "type": ["null", "string"] }, "id": { - "type": "string" + "type": ["null", "string"] }, "externalERPId": { - "type": "string" + "type": ["null", "string"] }, "externalCRMId": { - "type": "string" + "type": ["null", "string"] } } }, "notes": { - "type": "string" + "type": ["null", "string"] }, "invoiceDate": { - "type": "string" + "type": ["null", "string"] }, "dueDate": { - "type": "string" + "type": ["null", "string"] }, "daysPastDue": { - "type": "number" + "type": ["null", "number"] }, "nrOfReminders": { - "type": "number" + "type": ["null", "number"] }, "paymentTerm": { "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "days": { - "type": "number" + "type": ["null", "number"] }, "name": { - "type": "string" + "type": ["null", "string"] } } }, "currency": { - "type": "string" + "type": ["null", "string"] }, "subtotal": { - "type": "number" + "type": ["null", "number"] }, "tax": { - "type": "number" + "type": ["null", "number"] }, "totalAmount": { - "type": "number" + "type": ["null", "number"] }, "totalRoundingAmount": { - "type": "number" + "type": ["null", "number"] }, "settledAmount": { - "type": "number" + "type": ["null", "number"] }, "balancedAmount": { - "type": "number" + "type": ["null", "number"] }, "taxIncluded": { - "type": "boolean" + "type": ["null", "boolean"] }, "invoiceAddress": { "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "description": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "street": { - "type": "string" + "type": ["null", "string"] }, "street2": { - "type": "string" + "type": ["null", "string"] }, "city": { - "type": "string" + "type": ["null", "string"] }, "county": { - "type": "string" + "type": ["null", "string"] }, "state": { - "type": "string" + "type": ["null", "string"] }, "zip": { - "type": "string" + "type": ["null", "string"] }, "country": { - "type": "string" + "type": ["null", "string"] } } }, @@ -123,39 +133,39 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "description": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "street": { - "type": "string" + "type": ["null", "string"] }, "street2": { - "type": "string" + "type": ["null", "string"] }, "city": { - "type": "string" + "type": ["null", "string"] }, "county": { - "type": "string" + "type": ["null", "string"] }, "state": { - "type": "string" + "type": ["null", "string"] }, "zip": { - "type": "string" + "type": ["null", "string"] }, "country": { - "type": "string" + "type": ["null", "string"] } } }, "invoiceBatchId": { - "type": "string" + "type": ["null", "string"] }, "invoiceLines": { "type": "array", @@ -163,96 +173,96 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "invoiceLineNumber": { - "type": "number" + "type": ["null", "number"] }, "productNumber": { - "type": "string" + "type": ["null", "string"] }, "productName": { - "type": "string" + "type": ["null", "string"] }, "chargeDescription": { - "type": "string" + "type": ["null", "string"] }, "chargeNumber": { - "type": "string" + "type": ["null", "string"] }, "quantity": { - "type": "number" + "type": ["null", "number"] }, "unitOfMeasure": { "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "unitCode": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "displayName": { - "type": "string" + "type": ["null", "string"] } } }, "price": { - "type": "number" + "type": ["null", "number"] }, "subtotal": { - "type": "number" + "type": ["null", "number"] }, "total": { - "type": "number" + "type": ["null", "number"] }, "tax": { - "type": "number" + "type": ["null", "number"] }, "servicePeriodStartDate": { - "type": "string" + "type": ["null", "string"] }, "servicePeriodEndDate": { - "type": "string" + "type": ["null", "string"] }, "notes": { - "type": "string" + "type": ["null", "string"] }, "orderChargeId": { - "type": "string" + "type": ["null", "string"] }, "orderId": { - "type": "string" + "type": ["null", "string"] }, "accountId": { - "type": "string" + "type": ["null", "string"] }, "customFields": { - "type": "object" + "type": ["null", "object"] }, "accountsReceivable": { "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "code": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "description": { - "type": "string" + "type": ["null", "string"] }, "externalERPId": { - "type": "string" + "type": ["null", "string"] }, "externalCRMId": { - "type": "string" + "type": ["null", "string"] } } }, @@ -260,22 +270,22 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "code": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "description": { - "type": "string" + "type": ["null", "string"] }, "externalERPId": { - "type": "string" + "type": ["null", "string"] }, "externalCRMId": { - "type": "string" + "type": ["null", "string"] } } }, @@ -283,104 +293,104 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "code": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "description": { - "type": "string" + "type": ["null", "string"] }, "externalERPId": { - "type": "string" + "type": ["null", "string"] }, "externalCRMId": { - "type": "string" + "type": ["null", "string"] } } }, "externalERPId": { - "type": "string" + "type": ["null", "string"] }, "externalCRMId": { - "type": "string" + "type": ["null", "string"] }, "taxCategoryName": { - "type": "string" + "type": ["null", "string"] }, "taxRate": { - "type": "number" + "type": ["null", "number"] } } } }, "yourReference": { - "type": "string" + "type": ["null", "string"] }, "ourReference": { - "type": "string" + "type": ["null", "string"] }, "yourOrderNumber": { - "type": "string" + "type": ["null", "string"] }, "buyerReference": { - "type": "string" + "type": ["null", "string"] }, "invoiceType": { - "type": "string" + "type": ["null", "string"] }, "sendMethod": { - "type": "string" + "type": ["null", "string"] }, "exchangeRate": { - "type": "number" + "type": ["null", "number"] }, "settledNotes": { - "type": "string" + "type": ["null", "string"] }, "invoiceTemplateId": { - "type": "string" + "type": ["null", "string"] }, "disableAutomaticInvoiceReminder": { - "type": "boolean" + "type": ["null", "boolean"] }, "onlinePaymentLink": { - "type": "string" + "type": ["null", "string"] }, "accountsReceivable": { "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "code": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "description": { - "type": "string" + "type": ["null", "string"] }, "externalERPId": { - "type": "string" + "type": ["null", "string"] }, "externalCRMId": { - "type": "string" + "type": ["null", "string"] } } }, "customFields": { - "type": "object" + "type": ["null", "object"] }, "externalERPId": { - "type": "string" + "type": ["null", "string"] }, "externalCRMId": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-younium/source_younium/schemas/product.json b/airbyte-integrations/connectors/source-younium/source_younium/schemas/product.json index 296946ccd16d..d3f55669708d 100644 --- a/airbyte-integrations/connectors/source-younium/source_younium/schemas/product.json +++ b/airbyte-integrations/connectors/source-younium/source_younium/schemas/product.json @@ -1,36 +1,43 @@ { "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": true, "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "productNumber": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"] + }, + "modified": { + "type": ["null", "string"] }, "productType": { - "type": "string" + "type": ["null", "string"] }, "category": { - "type": "string" + "type": ["null", "string"] }, "activationDate": { - "type": "string" + "type": ["null", "string"] }, "endOfNewSalesDate": { - "type": "string" + "type": ["null", "string"] }, "endOfRenewalDate": { - "type": "string" + "type": ["null", "string"] }, "endOfLifeDate": { - "type": "string" + "type": ["null", "string"] }, "isFrameworkProduct": { - "type": "boolean" + "type": ["null", "boolean"] }, "chargePlans": { "type": "array", @@ -38,22 +45,22 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "chargePlanNumber": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "effectiveStartDate": { - "type": "string" + "type": ["null", "string"] }, "endOfNewSalesDate": { - "type": "string" + "type": ["null", "string"] }, "effectiveEndDate": { - "type": "string" + "type": ["null", "string"] }, "charges": { "type": "array", @@ -61,70 +68,70 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "chargeNumber": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "model": { - "type": "string" + "type": ["null", "string"] }, "chargeType": { - "type": "string" + "type": ["null", "string"] }, "unitCode": { - "type": "string" + "type": ["null", "string"] }, "defaultQuantity": { - "type": "number" + "type": ["null", "number"] }, "pricePeriod": { - "type": "string" + "type": ["null", "string"] }, "usageRating": { - "type": "string" + "type": ["null", "string"] }, "createInvoiceLinesPerTier": { - "type": "boolean" + "type": ["null", "boolean"] }, "billingDay": { - "type": "string" + "type": ["null", "string"] }, "specificBillingDay": { - "type": "number" + "type": ["null", "number"] }, "billingPeriod": { - "type": "string" + "type": ["null", "string"] }, "periodAlignment": { - "type": "string" + "type": ["null", "string"] }, "billingTiming": { - "type": "string" + "type": ["null", "string"] }, "taxTemplate": { - "type": "string" + "type": ["null", "string"] }, "taxIncluded": { - "type": "boolean" + "type": ["null", "boolean"] }, "externalERPId": { - "type": "string" + "type": ["null", "string"] }, "externalCRMId": { - "type": "string" + "type": ["null", "string"] }, "deferredRevenueAccount": { - "type": "string" + "type": ["null", "string"] }, "recognizedRevenueAccount": { - "type": "string" + "type": ["null", "string"] }, "customFields": { - "type": "object" + "type": ["null", "object"] }, "priceDetails": { "type": "array", @@ -132,25 +139,25 @@ "type": "object", "properties": { "currency": { - "type": "string" + "type": ["null", "string"] }, "price": { - "type": "number" + "type": ["null", "number"] }, "tier": { - "type": "number" + "type": ["null", "number"] }, "description": { - "type": "string" + "type": ["null", "string"] }, "fromQuantity": { - "type": "number" + "type": ["null", "number"] }, "toQuantity": { - "type": "number" + "type": ["null", "number"] }, "priceBase": { - "type": "string" + "type": ["null", "string"] } } } @@ -162,13 +169,13 @@ } }, "externalERPId": { - "type": "string" + "type": ["null", "string"] }, "externalCRMId": { - "type": "string" + "type": ["null", "string"] }, "customFields": { - "type": "object" + "type": ["null", "object"] } } } diff --git a/airbyte-integrations/connectors/source-younium/source_younium/schemas/subscription.json b/airbyte-integrations/connectors/source-younium/source_younium/schemas/subscription.json index c4ea6eeca3ff..60a664830f81 100644 --- a/airbyte-integrations/connectors/source-younium/source_younium/schemas/subscription.json +++ b/airbyte-integrations/connectors/source-younium/source_younium/schemas/subscription.json @@ -1,114 +1,127 @@ { "$schema": "http://json-schema.org/draft-04/schema#", + "additionalProperties": true, "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "orderNumber": { - "type": "string" + "type": ["null", "string"] + }, + "created": { + "type": ["null", "string"] + }, + "modified": { + "type": ["null", "string"] + }, + "orderBillingPeriod": { + "type": ["null", "string"] + }, + "setOrderBillingPeriod": { + "type": ["null", "boolean"] }, "version": { - "type": "number" + "type": ["null", "number"] }, "isLastVersion": { - "type": "boolean" + "type": ["null", "boolean"] }, "status": { - "type": "string" + "type": ["null", "string"] }, "description": { - "type": "string" + "type": ["null", "string"] }, "remarks": { - "type": "string" + "type": ["null", "string"] }, "effectiveStartDate": { - "type": "string" + "type": ["null", "string"] }, "effectiveEndDate": { - "type": "string" + "type": ["null", "string"] }, "cancellationDate": { - "type": "string" + "type": ["null", "string"] }, "effectiveChangeDate": { - "type": "string" + "type": ["null", "string"] }, "orderDate": { - "type": "string" + "type": ["null", "string"] }, "noticePeriodDate": { - "type": "string" + "type": ["null", "string"] }, "lastRenewalDate": { - "type": "string" + "type": ["null", "string"] }, "noticePeriod": { - "type": "number" + "type": ["null", "number"] }, "term": { - "type": "number" + "type": ["null", "number"] }, "renewalTerm": { - "type": "number" + "type": ["null", "number"] }, "isAutoRenewed": { - "type": "boolean" + "type": ["null", "boolean"] }, "orderType": { - "type": "string" + "type": ["null", "string"] }, "termType": { - "type": "string" + "type": ["null", "string"] }, "orderPaymentMethod": { - "type": "string" + "type": ["null", "string"] }, "invoiceSeparatly": { - "type": "boolean" + "type": ["null", "boolean"] }, "yourReference": { - "type": "string" + "type": ["null", "string"] }, "ourReference": { - "type": "string" + "type": ["null", "string"] }, "yourOrderNumber": { - "type": "string" + "type": ["null", "string"] }, "invoiceAddress": { "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "description": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "street": { - "type": "string" + "type": ["null", "string"] }, "street2": { - "type": "string" + "type": ["null", "string"] }, "city": { - "type": "string" + "type": ["null", "string"] }, "county": { - "type": "string" + "type": ["null", "string"] }, "state": { - "type": "string" + "type": ["null", "string"] }, "zip": { - "type": "string" + "type": ["null", "string"] }, "country": { - "type": "string" + "type": ["null", "string"] } } }, @@ -116,34 +129,34 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "description": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "street": { - "type": "string" + "type": ["null", "string"] }, "street2": { - "type": "string" + "type": ["null", "string"] }, "city": { - "type": "string" + "type": ["null", "string"] }, "county": { - "type": "string" + "type": ["null", "string"] }, "state": { - "type": "string" + "type": ["null", "string"] }, "zip": { - "type": "string" + "type": ["null", "string"] }, "country": { - "type": "string" + "type": ["null", "string"] } } }, @@ -151,39 +164,39 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "code": { - "type": "string" + "type": ["null", "string"] }, "description": { - "type": "string" + "type": ["null", "string"] } } }, "paymentTerm": { - "type": "string" + "type": ["null", "string"] }, "useAccountInvoiceBatchGroup": { - "type": "boolean" + "type": ["null", "boolean"] }, "account": { "type": "object", "properties": { "name": { - "type": "string" + "type": ["null", "string"] }, "accountNumber": { - "type": "string" + "type": ["null", "string"] }, "id": { - "type": "string" + "type": ["null", "string"] }, "externalERPId": { - "type": "string" + "type": ["null", "string"] }, "externalCRMId": { - "type": "string" + "type": ["null", "string"] } } }, @@ -191,19 +204,19 @@ "type": "object", "properties": { "name": { - "type": "string" + "type": ["null", "string"] }, "accountNumber": { - "type": "string" + "type": ["null", "string"] }, "id": { - "type": "string" + "type": ["null", "string"] }, "externalERPId": { - "type": "string" + "type": ["null", "string"] }, "externalCRMId": { - "type": "string" + "type": ["null", "string"] } } }, @@ -213,25 +226,25 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "productNumber": { - "type": "string" + "type": ["null", "string"] }, "chargePlanId": { - "type": "string" + "type": ["null", "string"] }, "chargePlanName": { - "type": "string" + "type": ["null", "string"] }, "chargePlanNumber": { - "type": "string" + "type": ["null", "string"] }, "productLineNumber": { - "type": "number" + "type": ["null", "number"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "charges": { "type": "array", @@ -239,112 +252,112 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "chargeNumber": { - "type": "string" + "type": ["null", "string"] }, "version": { - "type": "number" + "type": ["null", "number"] }, "isLastVersion": { - "type": "boolean" + "type": ["null", "boolean"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "chargeType": { - "type": "string" + "type": ["null", "string"] }, "priceModel": { - "type": "string" + "type": ["null", "string"] }, "effectiveStartDate": { - "type": "string" + "type": ["null", "string"] }, "effectiveEndDate": { - "type": "string" + "type": ["null", "string"] }, "quantity": { - "type": "number" + "type": ["null", "number"] }, "unitCode": { - "type": "string" + "type": ["null", "string"] }, "startOn": { - "type": "string" + "type": ["null", "string"] }, "endOn": { - "type": "string" + "type": ["null", "string"] }, "chargedThroughDate": { - "type": "string" + "type": ["null", "string"] }, "lastRenewalDate": { - "type": "string" + "type": ["null", "string"] }, "lastPriceAdjustmentDate": { - "type": "string" + "type": ["null", "string"] }, "pricePeriod": { - "type": "string" + "type": ["null", "string"] }, "usageRating": { - "type": "string" + "type": ["null", "string"] }, "revenueRecognitionRule": { - "type": "string" + "type": ["null", "string"] }, "billingDay": { - "type": "string" + "type": ["null", "string"] }, "specificBillingDay": { - "type": "number" + "type": ["null", "number"] }, "billingPeriod": { - "type": "string" + "type": ["null", "string"] }, "billingTiming": { - "type": "string" + "type": ["null", "string"] }, "periodAlignment": { - "type": "string" + "type": ["null", "string"] }, "taxTemplate": { - "type": "string" + "type": ["null", "string"] }, "taxIncluded": { - "type": "boolean" + "type": ["null", "boolean"] }, "createInvoiceLinesPerTier": { - "type": "boolean" + "type": ["null", "boolean"] }, "estimatedUsage": { - "type": "number" + "type": ["null", "number"] }, "estimatedQuantity": { - "type": "number" + "type": ["null", "number"] }, "remarks": { - "type": "string" + "type": ["null", "string"] }, "accountsReceivableAccount": { - "type": "string" + "type": ["null", "string"] }, "deferredRevenueAccount": { - "type": "string" + "type": ["null", "string"] }, "recognizedRevenueAccount": { - "type": "string" + "type": ["null", "string"] }, "changeState": { - "type": "string" + "type": ["null", "string"] }, "displayPrice": { - "type": "number" + "type": ["null", "number"] }, "customFields": { - "type": "object" + "type": ["null", "object"] }, "priceDetails": { "type": "array", @@ -352,40 +365,40 @@ "type": "object", "properties": { "tier": { - "type": "number" + "type": ["null", "number"] }, "price": { - "type": "number" + "type": ["null", "number"] }, "listPrice": { - "type": "number" + "type": ["null", "number"] }, "description": { - "type": "string" + "type": ["null", "string"] }, "fromQuantity": { - "type": "number" + "type": ["null", "number"] }, "toQuantity": { - "type": "number" + "type": ["null", "number"] }, "priceBase": { - "type": "string" + "type": ["null", "string"] }, "lineDiscountPercent": { - "type": "number" + "type": ["null", "number"] }, "lineDiscountAmount": { - "type": "number" + "type": ["null", "number"] } } } }, "recurringMonthlyAmount": { - "type": "number" + "type": ["null", "number"] }, "recurringMonthlyAmountBase": { - "type": "number" + "type": ["null", "number"] }, "features": { "type": "array", @@ -393,10 +406,10 @@ "type": "object", "properties": { "code": { - "type": "string" + "type": ["null", "string"] }, "description": { - "type": "string" + "type": ["null", "string"] } } } @@ -407,37 +420,37 @@ "type": "object", "properties": { "orderDiscountId": { - "type": "string" + "type": ["null", "string"] }, "chargeId": { - "type": "string" + "type": ["null", "string"] } } } }, "externalERPId": { - "type": "string" + "type": ["null", "string"] }, "externalCRMId": { - "type": "string" + "type": ["null", "string"] }, "cmrr": { "type": "object", "properties": { "amount": { - "type": "number" + "type": ["null", "number"] }, "currencyCode": { - "type": "string" + "type": ["null", "string"] }, "currencyConversionDate": { - "type": "string" + "type": ["null", "string"] }, "baseCurrencyAmount": { - "type": "number" + "type": ["null", "number"] }, "baseCurrencyCode": { - "type": "string" + "type": ["null", "string"] } } }, @@ -445,19 +458,19 @@ "type": "object", "properties": { "amount": { - "type": "number" + "type": ["null", "number"] }, "currencyCode": { - "type": "string" + "type": ["null", "string"] }, "currencyConversionDate": { - "type": "string" + "type": ["null", "string"] }, "baseCurrencyAmount": { - "type": "number" + "type": ["null", "number"] }, "baseCurrencyCode": { - "type": "string" + "type": ["null", "string"] } } }, @@ -465,19 +478,19 @@ "type": "object", "properties": { "amount": { - "type": "number" + "type": ["null", "number"] }, "currencyCode": { - "type": "string" + "type": ["null", "string"] }, "currencyConversionDate": { - "type": "string" + "type": ["null", "string"] }, "baseCurrencyAmount": { - "type": "number" + "type": ["null", "number"] }, "baseCurrencyCode": { - "type": "string" + "type": ["null", "string"] } } }, @@ -485,19 +498,19 @@ "type": "object", "properties": { "amount": { - "type": "number" + "type": ["null", "number"] }, "currencyCode": { - "type": "string" + "type": ["null", "string"] }, "currencyConversionDate": { - "type": "string" + "type": ["null", "string"] }, "baseCurrencyAmount": { - "type": "number" + "type": ["null", "number"] }, "baseCurrencyCode": { - "type": "string" + "type": ["null", "string"] } } }, @@ -505,57 +518,57 @@ "type": "object", "properties": { "amount": { - "type": "number" + "type": ["null", "number"] }, "currencyCode": { - "type": "string" + "type": ["null", "string"] }, "currencyConversionDate": { - "type": "string" + "type": ["null", "string"] }, "baseCurrencyAmount": { - "type": "number" + "type": ["null", "number"] }, "baseCurrencyCode": { - "type": "string" + "type": ["null", "string"] } } }, "orderProductId": { - "type": "string" + "type": ["null", "string"] }, "orderId": { - "type": "string" + "type": ["null", "string"] } } } }, "customFields": { - "type": "object" + "type": ["null", "object"] }, "externalERPId": { - "type": "string" + "type": ["null", "string"] }, "externalCRMId": { - "type": "string" + "type": ["null", "string"] }, "cmrr": { "type": "object", "properties": { "amount": { - "type": "number" + "type": ["null", "number"] }, "currencyCode": { - "type": "string" + "type": ["null", "string"] }, "currencyConversionDate": { - "type": "string" + "type": ["null", "string"] }, "baseCurrencyAmount": { - "type": "number" + "type": ["null", "number"] }, "baseCurrencyCode": { - "type": "string" + "type": ["null", "string"] } } }, @@ -563,19 +576,19 @@ "type": "object", "properties": { "amount": { - "type": "number" + "type": ["null", "number"] }, "currencyCode": { - "type": "string" + "type": ["null", "string"] }, "currencyConversionDate": { - "type": "string" + "type": ["null", "string"] }, "baseCurrencyAmount": { - "type": "number" + "type": ["null", "number"] }, "baseCurrencyCode": { - "type": "string" + "type": ["null", "string"] } } }, @@ -583,19 +596,19 @@ "type": "object", "properties": { "amount": { - "type": "number" + "type": ["null", "number"] }, "currencyCode": { - "type": "string" + "type": ["null", "string"] }, "currencyConversionDate": { - "type": "string" + "type": ["null", "string"] }, "baseCurrencyAmount": { - "type": "number" + "type": ["null", "number"] }, "baseCurrencyCode": { - "type": "string" + "type": ["null", "string"] } } }, @@ -603,19 +616,19 @@ "type": "object", "properties": { "amount": { - "type": "number" + "type": ["null", "number"] }, "currencyCode": { - "type": "string" + "type": ["null", "string"] }, "currencyConversionDate": { - "type": "string" + "type": ["null", "string"] }, "baseCurrencyAmount": { - "type": "number" + "type": ["null", "number"] }, "baseCurrencyCode": { - "type": "string" + "type": ["null", "string"] } } }, @@ -623,19 +636,19 @@ "type": "object", "properties": { "amount": { - "type": "number" + "type": ["null", "number"] }, "currencyCode": { - "type": "string" + "type": ["null", "string"] }, "currencyConversionDate": { - "type": "string" + "type": ["null", "string"] }, "baseCurrencyAmount": { - "type": "number" + "type": ["null", "number"] }, "baseCurrencyCode": { - "type": "string" + "type": ["null", "string"] } } } @@ -648,22 +661,22 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "orderId": { - "type": "string" + "type": ["null", "string"] }, "name": { - "type": "string" + "type": ["null", "string"] }, "description": { - "type": "string" + "type": ["null", "string"] }, "milestoneDate": { - "type": "string" + "type": ["null", "string"] }, "plannedDate": { - "type": "string" + "type": ["null", "string"] } } } @@ -674,28 +687,28 @@ "type": "object", "properties": { "id": { - "type": "string" + "type": ["null", "string"] }, "orderId": { - "type": "string" + "type": ["null", "string"] }, "startOn": { - "type": "string" + "type": ["null", "string"] }, "endOn": { - "type": "string" + "type": ["null", "string"] }, "startDate": { - "type": "string" + "type": ["null", "string"] }, "endDate": { - "type": "string" + "type": ["null", "string"] }, "percent": { - "type": "number" + "type": ["null", "number"] }, "discountType": { - "type": "string" + "type": ["null", "string"] }, "orderProductCharges": { "type": "array", @@ -703,52 +716,52 @@ "type": "object", "properties": { "orderDiscountId": { - "type": "string" + "type": ["null", "string"] }, "chargeId": { - "type": "string" + "type": ["null", "string"] } } } }, "onSpecificCharges": { - "type": "boolean" + "type": ["null", "boolean"] } } } }, "currency": { - "type": "string" + "type": ["null", "string"] }, "externalERPId": { - "type": "string" + "type": ["null", "string"] }, "externalCRMId": { - "type": "string" + "type": ["null", "string"] }, "currencyCodeToUseWhenInvoice": { - "type": "string" + "type": ["null", "string"] }, "customFields": { - "type": "object" + "type": ["null", "object"] }, "cmrr": { "type": "object", "properties": { "amount": { - "type": "number" + "type": ["null", "number"] }, "currencyCode": { - "type": "string" + "type": ["null", "string"] }, "currencyConversionDate": { - "type": "string" + "type": ["null", "string"] }, "baseCurrencyAmount": { - "type": "number" + "type": ["null", "number"] }, "baseCurrencyCode": { - "type": "string" + "type": ["null", "string"] } } }, @@ -756,19 +769,19 @@ "type": "object", "properties": { "amount": { - "type": "number" + "type": ["null", "number"] }, "currencyCode": { - "type": "string" + "type": ["null", "string"] }, "currencyConversionDate": { - "type": "string" + "type": ["null", "string"] }, "baseCurrencyAmount": { - "type": "number" + "type": ["null", "number"] }, "baseCurrencyCode": { - "type": "string" + "type": ["null", "string"] } } }, @@ -776,19 +789,19 @@ "type": "object", "properties": { "amount": { - "type": "number" + "type": ["null", "number"] }, "currencyCode": { - "type": "string" + "type": ["null", "string"] }, "currencyConversionDate": { - "type": "string" + "type": ["null", "string"] }, "baseCurrencyAmount": { - "type": "number" + "type": ["null", "number"] }, "baseCurrencyCode": { - "type": "string" + "type": ["null", "string"] } } }, @@ -796,19 +809,19 @@ "type": "object", "properties": { "amount": { - "type": "number" + "type": ["null", "number"] }, "currencyCode": { - "type": "string" + "type": ["null", "string"] }, "currencyConversionDate": { - "type": "string" + "type": ["null", "string"] }, "baseCurrencyAmount": { - "type": "number" + "type": ["null", "number"] }, "baseCurrencyCode": { - "type": "string" + "type": ["null", "string"] } } }, @@ -816,19 +829,19 @@ "type": "object", "properties": { "amount": { - "type": "number" + "type": ["null", "number"] }, "currencyCode": { - "type": "string" + "type": ["null", "string"] }, "currencyConversionDate": { - "type": "string" + "type": ["null", "string"] }, "baseCurrencyAmount": { - "type": "number" + "type": ["null", "number"] }, "baseCurrencyCode": { - "type": "string" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-younium/source_younium/source.py b/airbyte-integrations/connectors/source-younium/source_younium/source.py index f76f99fad38c..2b8e59669dbf 100644 --- a/airbyte-integrations/connectors/source-younium/source_younium/source.py +++ b/airbyte-integrations/connectors/source-younium/source_younium/source.py @@ -2,130 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +WARNING: Do not modify this file. +""" -# Basic full refresh stream -class YouniumStream(HttpStream, ABC): - # url_base = "https://apisandbox.younium.com" - - # https://api.younium.com - def __init__(self, authenticator=TokenAuthenticator, playground: bool = False, *args, **kwargs): - super().__init__(authenticator=authenticator) - self.page_size = 100 - self.playground: bool = playground - - @property - def url_base(self) -> str: - if self.playground: - endpoint = "https://apisandbox.younium.com" - else: - endpoint = "https://api.younium.com" - return endpoint - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - response = response.json() - current_page = response.get("pageNumber", 1) - total_rows = response.get("totalCount", 0) - - total_pages = total_rows // self.page_size - - if current_page <= total_pages: - return {"pageNumber": current_page + 1} - else: - return None - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - if next_page_token: - return {"pageNumber": next_page_token["pageNumber"], "PageSize": self.page_size} - else: - return {"PageSize": self.page_size} - - def parse_response( - self, - response: requests.Response, - *, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[Mapping]: - response_results = response.json() - yield from response_results.get("data", []) - - -class Invoice(YouniumStream): - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "Invoices" - - -class Product(YouniumStream): - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "Products" - - -class Subscription(YouniumStream): - primary_key = "id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - return "Subscriptions" - - -class SourceYounium(AbstractSource): - def get_auth(self, config): - scope = "openid youniumapi profile" - - if config.get("playground"): - url = "https://younium-identity-server-sandbox.azurewebsites.net/connect/token" - else: - url = "https://younium-identity-server.azurewebsites.net/connect/token" - - payload = f"grant_type=password&client_id=apiclient&username={config['username']}&password={config['password']}&scope={scope}" - headers = {"Content-Type": "application/x-www-form-urlencoded"} - response = requests.request("POST", url, headers=headers, data=payload) - response.raise_for_status() - access_token = response.json()["access_token"] - - auth = TokenAuthenticator(token=access_token) - return auth - - def check_connection(self, logger, config) -> Tuple[bool, any]: - - try: - stream = Invoice(authenticator=self.get_auth(config), **config) - stream.next_page_token = lambda response: None - stream.page_size = 1 - # auth = self.get_auth(config) - _ = list(stream.read_records(sync_mode=SyncMode.full_refresh)) - return True, None - except Exception as e: - logger.error(e) - return False, repr(e) - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - auth = self.get_auth(config) - return [Invoice(authenticator=auth, **config), Product(authenticator=auth, **config), Subscription(authenticator=auth, **config)] +# Declarative Source +class SourceYounium(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-younium/source_younium/spec.yaml b/airbyte-integrations/connectors/source-younium/source_younium/spec.yaml deleted file mode 100644 index 0cf38ff69a03..000000000000 --- a/airbyte-integrations/connectors/source-younium/source_younium/spec.yaml +++ /dev/null @@ -1,28 +0,0 @@ -documentationUrl: https://docs.airbyte.com/integrations/sources/younium -connectionSpecification: - $schema: http://json-schema.org/draft-07/schema# - title: Younium Spec - type: object - required: - - username - - password - - legal_entity - properties: - username: - title: Username - type: string - description: Username for Younium account - password: - title: Password - type: string - description: Account password for younium account API key - airbyte_secret: true - legal_entity: - title: Legal Entity - type: string - description: Legal Entity that data should be pulled from - playground: - title: Playground environment - type: boolean - description: Property defining if connector is used against playground or production environment - default: false diff --git a/airbyte-integrations/connectors/source-younium/unit_tests/test_source.py b/airbyte-integrations/connectors/source-younium/unit_tests/test_source.py deleted file mode 100644 index d1b501fcdb75..000000000000 --- a/airbyte-integrations/connectors/source-younium/unit_tests/test_source.py +++ /dev/null @@ -1,51 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import responses -from source_younium.source import SourceYounium - - -@responses.activate -def test_check_connection(mocker): - sandbox = False - - source = SourceYounium() - # mock the post request - - if sandbox: - mock_url1 = "https://younium-identity-server-sandbox.azurewebsites.net/connect/token" - mock_url2 = "https://apisandbox.younium.com/Invoices?PageSize=1" - else: - mock_url1 = "https://younium-identity-server.azurewebsites.net/connect/token" - mock_url2 = "https://api.younium.com/Invoices?PageSize=1" - # Mock the POST to get the access token - responses.add( - responses.POST, - mock_url1, - json={ - "access_token": "dummy_token", - }, - status=HTTPStatus.OK, - ) - - # Mock the GET to get the first page of the stream - responses.add(responses.GET, mock_url2, json={}, status=HTTPStatus.OK) - - logger_mock = MagicMock() - config_mock = {"playground": sandbox, "username": "dummy_username", "password": "dummy_password"} - - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -def test_streams(mocker): - source = SourceYounium() - mocker.patch.object(source, "get_auth", return_value="dummy_token") - config_mock = {"playground": False, "username": "dummy_username", "password": "dummy_password"} - streams = source.streams(config_mock) - - expected_streams_number = 3 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-younium/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-younium/unit_tests/test_streams.py deleted file mode 100644 index ab0226d7c9b2..000000000000 --- a/airbyte-integrations/connectors/source-younium/unit_tests/test_streams.py +++ /dev/null @@ -1,73 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_younium.source import YouniumStream - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(YouniumStream, "path", "v0/example_endpoint") - mocker.patch.object(YouniumStream, "primary_key", "test_primary_key") - mocker.patch.object(YouniumStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = YouniumStream(authenticator=None) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {"PageSize": 100} - assert stream.request_params(**inputs) == expected_params - - -def test_request_params_with_next_page_token(patch_base_class): - stream = YouniumStream(authenticator=None) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": {"pageNumber": 2}} - expected_params = {"PageSize": 100, "pageNumber": 2} - assert stream.request_params(**inputs) == expected_params - - -def test_playground_url_base(patch_base_class): - stream = YouniumStream(authenticator=None, playground=True) - expected_url_base = "https://apisandbox.younium.com" - assert stream.url_base == expected_url_base - - -def test_use_playground_url_base(patch_base_class): - stream = YouniumStream(authenticator=None, playground=True) - expected_url_base = "https://apisandbox.younium.com" - assert stream.url_base == expected_url_base - - -def test_http_method(patch_base_class): - stream = YouniumStream(authenticator=None) - # TODO: replace this with your expected http request method - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = YouniumStream(authenticator=None) - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = YouniumStream(authenticator=None) - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-youtube-analytics/README.md b/airbyte-integrations/connectors/source-youtube-analytics/README.md index 9dee076608fb..57c9e04b195a 100644 --- a/airbyte-integrations/connectors/source-youtube-analytics/README.md +++ b/airbyte-integrations/connectors/source-youtube-analytics/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-youtube-analytics:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/youtube-analytics) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_youtube_analytics/spec.json` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-youtube-analytics:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-youtube-analytics build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-youtube-analytics:airbyteDocker +An image will be built with the tag `airbyte/source-youtube-analytics:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-youtube-analytics:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-youtube-analytics:dev docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-youtube-analytics:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-youtube-analytics:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-youtube-analytics test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-youtube-analytics:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-youtube-analytics:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-youtube-analytics test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/youtube-analytics.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-youtube-analytics/acceptance-test-config.yml b/airbyte-integrations/connectors/source-youtube-analytics/acceptance-test-config.yml index 9be17892d97b..e521392e533d 100644 --- a/airbyte-integrations/connectors/source-youtube-analytics/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-youtube-analytics/acceptance-test-config.yml @@ -3,40 +3,40 @@ connector_image: airbyte/source-youtube-analytics:dev tests: spec: - - spec_path: "source_youtube_analytics/spec.json" + - spec_path: "source_youtube_analytics/spec.json" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "exception" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "exception" discovery: - - config_path: "secrets/config.json" - backward_compatibility_tests_config: - disable_for_version: "0.1.0" + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.0" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 1200 - empty_streams: - - channel_annotations_a1 - - channel_cards_a1 - - channel_demographics_a1 - - channel_end_screens_a1 - - channel_sharing_service_a1 - - channel_province_a2 - - playlist_basic_a1 - - playlist_combined_a1 - - playlist_device_os_a1 - - playlist_playback_location_a1 - - playlist_province_a1 - - playlist_traffic_source_a1 - fail_on_extra_columns: false + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 1200 + empty_streams: + - channel_annotations_a1 + - channel_cards_a1 + - channel_demographics_a1 + - channel_end_screens_a1 + - channel_sharing_service_a1 + - channel_province_a2 + - playlist_basic_a1 + - playlist_combined_a1 + - playlist_device_os_a1 + - playlist_playback_location_a1 + - playlist_province_a1 + - playlist_traffic_source_a1 + fail_on_extra_columns: false incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" - timeout_seconds: 7200 + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + timeout_seconds: 7200 full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - timeout_seconds: 3600 + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + timeout_seconds: 3600 diff --git a/airbyte-integrations/connectors/source-youtube-analytics/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-youtube-analytics/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-youtube-analytics/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-youtube-analytics/build.gradle b/airbyte-integrations/connectors/source-youtube-analytics/build.gradle deleted file mode 100644 index 9f1060506a1f..000000000000 --- a/airbyte-integrations/connectors/source-youtube-analytics/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_youtube_analytics' -} diff --git a/airbyte-integrations/connectors/source-youtube-analytics/metadata.yaml b/airbyte-integrations/connectors/source-youtube-analytics/metadata.yaml index 0562428d329f..9cd1fe065eaa 100644 --- a/airbyte-integrations/connectors/source-youtube-analytics/metadata.yaml +++ b/airbyte-integrations/connectors/source-youtube-analytics/metadata.yaml @@ -1,4 +1,7 @@ data: + ab_internal: + ql: 300 + sl: 100 allowedHosts: hosts: - "*.googleapis.com" @@ -7,6 +10,7 @@ data: definitionId: afa734e4-3571-11ec-991a-1e0031268139 dockerImageTag: 0.1.4 dockerRepository: airbyte/source-youtube-analytics + documentationUrl: https://docs.airbyte.com/integrations/sources/youtube-analytics githubIssueLabel: source-youtube-analytics icon: youtube-analytics.svg license: MIT @@ -17,11 +21,7 @@ data: oss: enabled: true releaseStage: beta - documentationUrl: https://docs.airbyte.com/integrations/sources/youtube-analytics + supportLevel: community tags: - language:python - ab_internal: - sl: 200 - ql: 300 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-zapier-supported-storage/README.md b/airbyte-integrations/connectors/source-zapier-supported-storage/README.md index 1b0070ea187b..31199979ea24 100644 --- a/airbyte-integrations/connectors/source-zapier-supported-storage/README.md +++ b/airbyte-integrations/connectors/source-zapier-supported-storage/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-zapier-supported-storage:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/zapier-supported-storage) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zapier_supported_storage/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-zapier-supported-storage:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-zapier-supported-storage build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-zapier-supported-storage:airbyteDocker +An image will be built with the tag `airbyte/source-zapier-supported-storage:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-zapier-supported-storage:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zapier-supported-stora docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zapier-supported-storage:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-zapier-supported-storage:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-zapier-supported-storage test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-zapier-supported-storage:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-zapier-supported-storage:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-zapier-supported-storage test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/zapier-supported-storage.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-zapier-supported-storage/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-zapier-supported-storage/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-zapier-supported-storage/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-zapier-supported-storage/build.gradle b/airbyte-integrations/connectors/source-zapier-supported-storage/build.gradle deleted file mode 100644 index 7cda4118e40a..000000000000 --- a/airbyte-integrations/connectors/source-zapier-supported-storage/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_zapier_supported_storage' -} diff --git a/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile b/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile deleted file mode 100644 index c7764ec2d2e5..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-chat/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -ENV CODE_PATH="source_zendesk_chat" -ENV AIRBYTE_IMPL_MODULE="source_zendesk_chat" -ENV AIRBYTE_IMPL_PATH="SourceZendeskChat" -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main_dev.py" - -WORKDIR /airbyte/integration_code -COPY $CODE_PATH ./$CODE_PATH -COPY main_dev.py ./ -COPY setup.py ./ -RUN pip install . - -ENTRYPOINT ["python", "/airbyte/integration_code/main_dev.py"] - -LABEL io.airbyte.version=0.1.14 -LABEL io.airbyte.name=airbyte/source-zendesk-chat diff --git a/airbyte-integrations/connectors/source-zendesk-chat/README.md b/airbyte-integrations/connectors/source-zendesk-chat/README.md index 018a20fda017..5e8b7fd035dc 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/README.md +++ b/airbyte-integrations/connectors/source-zendesk-chat/README.md @@ -1,19 +1,19 @@ -# Zendesk Chat Source +# XKCD Source -This is the repository for the Zendesk Chat source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/zendesk-chat). +This is the repository for the Xkcd source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/xkcd). ## Local development ### Prerequisites **To iterate on this connector, make sure to complete this prerequisites section.** -#### Minimum Python version required `= 3.7.0` +#### Minimum Python version required `= 3.9.0` #### Build & Activate Virtual Environment and install dependencies From this connector directory, create a virtual environment: ``` -python -m venv .venv +python3 -m venv .venv ``` This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your @@ -21,6 +21,7 @@ development environment of choice. To activate it from the terminal, run: ``` source .venv/bin/activate pip install -r requirements.txt +pip install '.[tests]' ``` If you are in an IDE, follow your IDE's instructions to activate the virtualenv. @@ -29,82 +30,71 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-zendesk-chat:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/zendesk-chat) -to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zendesk_chat/spec.json` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. -See `sample_files/sample_config.json` for a sample config file. +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/xkcd) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_xkcd/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. -**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source zendesk-chat test creds` +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source xkcd test creds` and place them into `secrets/config.json`. - ### Locally running the connector ``` -python main_dev.py spec -python main_dev.py check --config secrets/config.json -python main_dev.py discover --config secrets/config.json -python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. -If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-zendesk-chat:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` -To run your integration tests with docker - -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json ``` ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-zendesk-chat:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name source-xkcd build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-zendesk-chat:airbyteDocker +An image will be built with the tag `airbyte/source-xkcd:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-xkcd:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: ``` -docker run --rm airbyte/source-zendesk-chat:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zendesk-chat:dev check --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zendesk-chat:dev discover --config /secrets/config.json -docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-zendesk-chat:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json +docker run --rm airbyte/source-xkcd:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-xkcd:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-xkcd:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-xkcd:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-zendesk-chat:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-zendesk-chat test +``` + +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-zendesk-chat test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/zendesk-chat.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml index b00d731dd5f3..37352d28dabe 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-config.yml @@ -3,44 +3,39 @@ test_strictness_level: high acceptance_tests: spec: tests: - - spec_path: "source_zendesk_chat/spec.json" + - spec_path: "source_zendesk_chat/spec.json" connection: tests: - - config_path: "secrets/config_old.json" - status: "succeed" - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "secrets/config_oauth.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + - config_path: "secrets/config_old.json" + status: "succeed" + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "secrets/config_oauth.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: tests: - - config_path: "secrets/config_old.json" - - config_path: "secrets/config.json" - - config_path: "secrets/config_oauth.json" + - config_path: "secrets/config_old.json" + - config_path: "secrets/config.json" + - config_path: "secrets/config_oauth.json" basic_read: tests: - - config_path: "secrets/config.json" - expect_records: - path: "integration_tests/expected_records.txt" - fail_on_extra_columns: false - - config_path: "secrets/config_oauth.json" - expect_records: - path: "integration_tests/expected_records.txt" - fail_on_extra_columns: false + - config_path: "secrets/config.json" + expect_records: + path: "integration_tests/expected_records.txt" + fail_on_extra_columns: false + - config_path: "secrets/config_oauth.json" + expect_records: + path: "integration_tests/expected_records.txt" + fail_on_extra_columns: false incremental: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state: - bypass_reason: "Unable to use 'future_state_path' because Zendesk Chat API returns an error when specifying a date in the future." - cursor_paths: - agents: ["id"] - bans: ["id"] - agent_timeline: ["start_time"] - chats: ["update_timestamp"] + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state: + bypass_reason: "Unable to use 'future_state_path' because Zendesk Chat API returns an error when specifying a date in the future." full_refresh: tests: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-chat/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-zendesk-chat/build.gradle b/airbyte-integrations/connectors/source-zendesk-chat/build.gradle deleted file mode 100644 index 20c14ce65c4f..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-chat/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_zendesk_chat' -} diff --git a/airbyte-integrations/connectors/source-zendesk-chat/build_customization.py b/airbyte-integrations/connectors/source-zendesk-chat/build_customization.py new file mode 100644 index 000000000000..e85c96df52a9 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-chat/build_customization.py @@ -0,0 +1,21 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from dagger import Container + +MAIN_FILE_NAME = "main_dev.py" + + +async def pre_connector_install(base_image_container: Container) -> Container: + """This function will run before the connector installation. + We set these environment variable to match what was originally in the Dockerfile. + Disclaimer: I have no idea if these env vars are actually needed. + """ + return base_image_container.with_env_variable("AIRBYTE_IMPL_MODULE", "source_zendesk_chat").with_env_variable( + "AIRBYTE_IMPL_PATH", "SourceZendeskChat" + ) diff --git a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.txt index 33c52b2289bf..10f75ba0af99 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.txt +++ b/airbyte-integrations/connectors/source-zendesk-chat/integration_tests/expected_records.txt @@ -8,17 +8,17 @@ {"stream": "bans", "data": {"type": "visitor", "id": 75411401, "reason": "Spammer", "created_at": "2021-04-27T15:52:32Z", "visitor_name": "Visitor 62959049", "visitor_id": "10414779.13ojzHu7at4VKcG"}, "emitted_at": 1672828433831} {"stream": "bans", "data": {"created_at": "2021-04-27T15:52:32Z", "visitor_id": "10414779.13ojzHu7at4VKcG", "id": 75411401, "reason": "Spammer", "visitor_name": "Visitor 62959049", "type": "visitor"}, "emitted_at": 1672828434000} {"stream": "bans", "data": {"created_at": "2021-04-27T15:52:33Z", "visitor_id": "10414779.13ojzHu7s9YwIjz", "id": 75411441, "reason": "Spammer", "visitor_name": "Visitor 97350211", "type": "visitor"}, "emitted_at": 1672828434001} -{"stream": "chats", "data": {"visitor": {"phone": "", "notes": "", "id": "0.83900", "name": "Fake user - chat 116", "email": "fake_user_chat_116@doe.com"}, "type": "offline_msg", "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2022-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "timestamp": "2021-04-30T13:36:28Z", "deleted": false, "tags": [], "department_name": null, "update_timestamp": "2021-04-30T13:36:28Z", "unread": true, "department_id": null, "message": "Hi there!", "id": "2104.10414779.SW4WsV9Tu0Ynj", "zendesk_ticket_id": null}, "emitted_at": 1672828434384} -{"stream": "chats", "data": {"visitor": {"phone": "", "notes": "", "id": "6.42465", "name": "Fake user - chat 117", "email": "fake_user_chat_117@doe.com"}, "type": "offline_msg", "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2022-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "timestamp": "2021-04-30T13:36:29Z", "deleted": false, "tags": [], "department_name": null, "update_timestamp": "2021-04-30T13:36:29Z", "unread": true, "department_id": null, "message": "Hi there!", "id": "2104.10414779.SW4WsbJTqVJsF", "zendesk_ticket_id": null}, "emitted_at": 1672828434384} -{"stream": "chats", "data": {"visitor": {"phone": "", "notes": "", "id": "8.89712", "name": "Fake user - chat 118", "email": "fake_user_chat_118@doe.com"}, "type": "offline_msg", "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2022-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "timestamp": "2021-04-30T13:36:29Z", "deleted": false, "tags": [], "department_name": null, "update_timestamp": "2021-04-30T13:36:29Z", "unread": true, "department_id": null, "message": "Hi there!", "id": "2104.10414779.SW4WsgcJUJbVN", "zendesk_ticket_id": null}, "emitted_at": 1672828434384} -{"stream": "chats", "data": {"visitor": {"phone": "", "notes": "", "id": "9.61246", "name": "Fake user - chat 119", "email": "fake_user_chat_119@doe.com"}, "type": "offline_msg", "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2022-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "timestamp": "2021-04-30T13:36:29Z", "deleted": false, "tags": [], "department_name": null, "update_timestamp": "2021-04-30T13:36:29Z", "unread": true, "department_id": null, "message": "Hi there!", "id": "2104.10414779.SW4WslzhLr3zm", "zendesk_ticket_id": null}, "emitted_at": 1672828434385} +{"stream": "chats", "data": {"department_id": null, "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2014-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "visitor": {"phone": "+32178763521", "notes": "Test 2", "id": "3.45678", "name": "Jiny", "email": "visitor_jiny@doe.com"}, "update_timestamp": "2021-04-27T15:09:17Z", "department_name": null, "type": "offline_msg", "deleted": false, "tags": [], "timestamp": "2021-04-26T13:54:02Z", "unread": false, "id": "2104.10414779.SVhDCJ9flq79a", "message": "Hi there!", "zendesk_ticket_id": null}, "emitted_at": 1701452730189} +{"stream": "chats", "data": {"department_id": null, "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2014-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "visitor": {"phone": "", "notes": "", "id": "1.12345", "name": "John", "email": "visitor_john@doe.com"}, "update_timestamp": "2021-04-30T11:06:19Z", "department_name": null, "type": "offline_msg", "deleted": false, "tags": [], "timestamp": "2021-04-21T14:36:55Z", "unread": false, "id": "2104.10414779.SVE9Mo9bE4wR8", "message": "Hi there!", "zendesk_ticket_id": null}, "emitted_at": 1701452730190} +{"stream": "chats", "data": {"department_id": null, "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2014-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "visitor": {"phone": "+78763521", "notes": "Test", "id": "2.34567", "name": "Tiny", "email": "visitor_tiny@doe.com"}, "update_timestamp": "2021-04-30T11:08:12Z", "department_name": null, "type": "offline_msg", "deleted": false, "tags": [], "timestamp": "2021-04-26T13:53:30Z", "unread": false, "id": "2104.10414779.SVhD3v7I1LBOq", "message": "Hi there!", "zendesk_ticket_id": null}, "emitted_at": 1701452730190} +{"stream": "chats", "data": {"department_id": null, "webpath": [], "session": {"browser": "Safari", "city": "Orlando", "country_code": "US", "country_name": "United States", "end_date": "2022-10-09T05:46:47Z", "id": "141109.654464.1KhqS0Nw", "ip": "67.32.299.96", "platform": "Mac OS", "region": "Florida", "start_date": "2014-10-09T05:28:31Z", "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10) AppleWebKit/600.1.25 (KHTML, like Gecko) Version/8.0 Safari/600.1.25"}, "visitor": {"phone": "", "notes": "", "id": "7.34502", "name": "Fake user - chat 2", "email": "fake_user_chat_2@doe.com"}, "update_timestamp": "2021-04-30T13:32:27Z", "department_name": null, "type": "offline_msg", "deleted": false, "tags": [], "timestamp": "2021-04-30T13:32:27Z", "unread": true, "id": "2104.10414779.SW4VrjJpOq6gk", "message": "Hi there!", "zendesk_ticket_id": null}, "emitted_at": 1701452730191} {"stream": "departments", "data": {"settings": {"chat_enabled": true, "support_group_id": 7282640316815}, "members": [361084605116], "name": "Airbyte Department 1", "enabled": true, "description": "A sample department", "id": 7282640316815}, "emitted_at": 1688547521914} {"stream": "departments", "data": {"settings": {"chat_enabled": true, "support_group_id": 7282618889231}, "members": [360786799676], "name": "Department 1", "enabled": true, "description": "A sample department", "id": 7282618889231}, "emitted_at": 1688547521914} {"stream": "departments", "data": {"settings": {"chat_enabled": true, "support_group_id": 7282630247567}, "members": [361089721035, 361084605116], "name": "Department 2", "enabled": true, "description": "A sample department 2", "id": 7282630247567}, "emitted_at": 1688547521914} -{"stream": "goals", "data": {"description": "A new goal", "id": 513481, "attribution_model": "first_touch", "attribution_period": 15, "name": "Goal 3", "enabled": true, "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1672828434873} -{"stream": "goals", "data": {"description": "A new goal - 1", "id": 529641, "attribution_model": "first_touch", "attribution_period": 15, "name": "Goal one", "enabled": false, "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1672828434874} -{"stream": "goals", "data": {"description": "A new goal - 2", "id": 529681, "attribution_model": "first_touch", "attribution_period": 15, "name": "Goal two", "enabled": false, "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1672828434874} -{"stream": "goals", "data": {"description": "Test goal", "id": 537121, "attribution_model": "last_touch", "attribution_period": 30, "name": "Test goal", "enabled": true, "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://zendesk.com/thanks"}]}}, "emitted_at": 1672828434874} +{"stream": "goals", "data": {"enabled": true, "id": 513481, "attribution_period": 15, "attribution_model": "first_touch", "name": "Goal 3", "description": "A new goal", "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1701453031915} +{"stream": "goals", "data": {"enabled": false, "id": 529641, "attribution_period": 5, "attribution_model": "first_touch", "name": "Goal one", "description": "A new goal - 1", "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1701453031916} +{"stream": "goals", "data": {"enabled": false, "id": 529681, "attribution_period": 15, "attribution_model": "first_touch", "name": "Goal two", "description": "A new goal - 2", "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://mysite.com/"}]}}, "emitted_at": 1701453031916} +{"stream": "goals", "data": {"enabled": true, "id": 537121, "attribution_period": 30, "attribution_model": "last_touch", "name": "Test goal", "description": "Test goal", "settings": {"conditions": [{"operator": "equals", "type": "url", "value": "http://zendesk.com/thanks"}]}}, "emitted_at": 1701453031916} {"stream": "roles", "data": {"permissions": {"visitors_seen": "account", "proactive_chatting": "listen-join", "edit_visitor_information": true, "edit_visitor_notes": true, "view_past_chats": "account", "edit_chat_tags": true, "manage_bans": "account", "access_analytics": "account", "view_monitor": "account", "edit_department_agents": "account", "set_agent_chat_limit": "account", "manage_shortcuts": "account"}, "enabled": true, "description": "In addition to regular agent privileges, administrators can edit widget and accounts settings, manage agents, roles and permissions, and more. Permissions for the administrator role cannot be modified.", "id": 360002848996, "name": "Administrator", "members_count": 1}, "emitted_at": 1672828435141} {"stream": "roles", "data": {"permissions": {"visitors_seen": "account", "proactive_chatting": "listen-join", "edit_visitor_information": true, "edit_visitor_notes": true, "view_past_chats": "account", "edit_chat_tags": false, "manage_bans": "account", "access_analytics": "none", "view_monitor": "account", "edit_department_agents": "none", "set_agent_chat_limit": "none", "manage_shortcuts": "account"}, "enabled": true, "description": "Agent is the most basic role in an account, and their primary responsibility is to serve chats. Permissions for the agent role can be modified.", "id": 360002848976, "name": "Agent", "members_count": 2}, "emitted_at": 1672828435142} {"stream": "shortcuts", "data": {"name": "goodbye", "id": "goodbye", "options": "Yes/No", "tags": ["goodbye_survey"], "scope": "all", "message": "Thanks for chatting with us. Have we resolved your question(s)?"}, "emitted_at": 1672828435386} @@ -31,4 +31,4 @@ {"stream": "triggers", "data": {"name": "Product Discounts", "enabled": true, "description": "Offer your returning customers a discount on one of your products or services. This Trigger will need to be customized based on the page.", "id": 66052801, "definition": {"event": "chat_requested", "condition": ["and", ["icontains", "@visitor_page_url", "[product name]"], ["stillOnPage", 30], ["eq", "@visitor_requesting_chat", false], ["eq", "@visitor_served", false], ["not", ["firedBefore"]]], "actions": [["sendMessageToVisitor", "Customer Service", "Hi, are you interested in [insert product name]? We're offering a one-time 20% discount. Chat with me to find out more."]], "version": 1, "editor": "advanced"}}, "emitted_at": 1688547525543} {"stream": "triggers", "data": {"name": "Request Contact Details", "enabled": true, "description": "When your account is set to away, ask customer's requesting a chat to leave their email address.", "id": 66052841, "definition": {"event": "chat_requested", "condition": ["and", ["eq", "@account_status", "away"], ["not", ["firedBefore"]]], "actions": [["addTag", "Away_request"], ["sendMessageToVisitor", "Customer Service", "Hi, sorry we are away at the moment. Please leave your email address and we will get back to you as soon as possible."]], "version": 1, "editor": "advanced"}}, "emitted_at": 1688547525543} {"stream": "triggers", "data": {"name": "Tag Repeat Visitors", "enabled": true, "description": "Add a tag to a visitor that has visited your site 5 or more times. This helps you identify potential customers who are very interested in your brand.", "id": 66052881, "definition": {"event": "page_enter", "condition": ["and", ["gte", "@visitor_previous_visits", 5]], "actions": [["addTag", "5times"]], "version": 1, "editor": "advanced"}}, "emitted_at": 1688547525543} -{"stream": "routing_settings", "data": {"routing_mode": "assigned", "chat_limit": {"enabled": false, "limit": 3, "limit_type": "account", "allow_agent_override": false}, "skill_routing": {"enabled": true, "max_wait_time": 30}, "reassignment": {"enabled": true, "timeout": 30}, "auto_idle": {"enabled": false, "reassignments_before_idle": 3, "new_status": "away"}}, "emitted_at": 1688547526146} +{"stream": "routing_settings", "data": {"routing_mode": "assigned", "chat_limit": {"enabled": false, "limit": 3, "limit_type": "account", "allow_agent_override": false}, "skill_routing": {"enabled": true, "max_wait_time": 30}, "reassignment": {"enabled": true, "timeout": 30}, "auto_idle": {"enabled": false, "reassignments_before_idle": 3, "new_status": "away"}, "auto_accept": {"enabled": false}}, "emitted_at": 1701453336379} diff --git a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml index 5d9c006e0853..5b8c27ff5ce9 100644 --- a/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-chat/metadata.yaml @@ -1,12 +1,18 @@ data: + ab_internal: + ql: 200 + sl: 200 allowedHosts: hosts: - zopim.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c connectorSubtype: api connectorType: source definitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4 - dockerImageTag: 0.1.14 + dockerImageTag: 0.2.1 dockerRepository: airbyte/source-zendesk-chat + documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-chat githubIssueLabel: source-zendesk-chat icon: zendesk-chat.svg license: MIT @@ -17,11 +23,7 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-chat + supportLevel: certified tags: - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-zendesk-sell/Dockerfile b/airbyte-integrations/connectors/source-zendesk-sell/Dockerfile index 381fff970479..3eb58532e2df 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/Dockerfile +++ b/airbyte-integrations/connectors/source-zendesk-sell/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.13-alpine3.15 as base +FROM python:3.9.11-alpine3.15 as base # build and load all requirements FROM base as builder @@ -34,5 +34,5 @@ COPY source_zendesk_sell ./source_zendesk_sell ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-zendesk-sell diff --git a/airbyte-integrations/connectors/source-zendesk-sell/README.md b/airbyte-integrations/connectors/source-zendesk-sell/README.md index c529f9a7b76e..5b6fabc3b1a2 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/README.md +++ b/airbyte-integrations/connectors/source-zendesk-sell/README.md @@ -1,75 +1,34 @@ # Zendesk Sell Source -This is the repository for the Zendesk Sell source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/zendesk-sell). +This is the repository for the Zendesk Sell configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/zendesk-sell). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.9.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-zendesk-sell:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/zendesk-sell) +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/zendesk-sell) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zendesk_sell/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. -Note that the full process to generate api tokens is available [here](https://developer.zendesk.com/documentation/sales-crm/first-call/#1-generate-an-access-token) **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source zendesk-sell test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-zendesk-sell:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-zendesk-sell build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-zendesk-sell:airbyteDocker +An image will be built with the tag `airbyte/source-zendesk-sell:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-zendesk-sell:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -79,44 +38,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zendesk-sell:dev check docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zendesk-sell:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-zendesk-sell:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-zendesk-sell test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-zendesk-sell:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-zendesk-sell:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -126,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-zendesk-sell test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/zendesk-sell.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-zendesk-sell/__init__.py b/airbyte-integrations/connectors/source-zendesk-sell/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-zendesk-sell/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zendesk-sell/acceptance-test-config.yml index bc6eee9d6a46..6022d768ea2c 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-zendesk-sell/acceptance-test-config.yml @@ -1,30 +1,39 @@ -# See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) # for more information about how to configure these tests connector_image: airbyte/source-zendesk-sell:dev -tests: +acceptance_tests: spec: - - spec_path: "source_zendesk_sell/spec.yaml" + tests: + - spec_path: "source_zendesk_sell/spec.yaml" connection: - - config_path: "secrets/config.json" - status: "succeed" - - config_path: "integration_tests/invalid_config.json" - status: "failed" + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" discovery: - - config_path: "secrets/config.json" + tests: + - config_path: "secrets/config.json" basic_read: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] - # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file - # expect_records: - # path: "integration_tests/expected_records.jsonl" - # extra_fields: no - # exact_order: no - # extra_records: yes - incremental: # TODO if your connector does not implement incremental sync, remove this block - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - future_state_path: "integration_tests/abnormal_state.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: + - name: text_messages + bypass_reason: "Is empty in test data" + - name: visit_outcomes + bypass_reason: "Is empty in test data" + - name: visits + bypass_reason: "Is empty in test data" + incremental: + bypass_reason: "This connector does not implement incremental sync" + # TODO uncomment this block this block if your connector implements incremental sync: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-zendesk-sell/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-zendesk-sell/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-sell/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-zendesk-sell/build.gradle b/airbyte-integrations/connectors/source-zendesk-sell/build.gradle deleted file mode 100644 index f81364cb959c..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-sell/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_zendesk_sell' -} diff --git a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/acceptance.py index 82823254d266..9e6409236281 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/integration_tests/acceptance.py @@ -11,4 +11,6 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-zendesk-sell/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-sell/metadata.yaml index 5920a672a86f..5f11fc4305bf 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-sell/metadata.yaml @@ -1,22 +1,25 @@ data: + allowedHosts: + hosts: + - api.getbase.com + registries: + oss: + enabled: false + cloud: + enabled: true connectorSubtype: api connectorType: source definitionId: 982eaa4c-bba1-4cce-a971-06a41f700b8c - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-zendesk-sell githubIssueLabel: source-zendesk-sell icon: zendesk.svg license: MIT name: Zendesk Sell - registries: - cloud: - enabled: false - oss: - enabled: true releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-sell tags: - - language:python + - language:low-code ab_internal: sl: 100 ql: 100 diff --git a/airbyte-integrations/connectors/source-zendesk-sell/setup.py b/airbyte-integrations/connectors/source-zendesk-sell/setup.py index 4f3427a1a107..054742fb41d4 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/setup.py @@ -5,13 +5,11 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.4", -] +MAIN_REQUIREMENTS = ["airbyte-cdk"] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", - "pytest~=6.1", + "pytest~=6.2", "pytest-mock~=3.6.1", ] diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/manifest.yaml b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/manifest.yaml new file mode 100644 index 000000000000..bcebfd8e78f3 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/manifest.yaml @@ -0,0 +1,233 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["items", "*", "data"] + + requester: + type: HttpRequester + url_base: "https://api.getbase.com/v2" + http_method: "GET" + authenticator: + type: BearerAuthenticator + api_token: "{{ config['api_token'] }}" + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + requester: + $ref: "#/definitions/requester" + paginator: + type: DefaultPaginator + page_size_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "per_page" + pagination_strategy: + type: "PageIncrement" + page_size: 2 + start_from_page: 1 + page_token_option: + type: "RequestOption" + inject_into: "request_parameter" + field_name: "page" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + call_outcomes_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "call_outcomes" + path: "/call_outcomes" + calls_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "calls" + path: "/calls" + collaborations_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "collaborations" + path: "/collaborations" + contacts_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "contacts" + path: "/contacts" + + deal_sources_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "deal_sources" + path: "/deal_sources" + deal_unqualified_reasons_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "deal_unqualified_reasons" + path: "/deal_unqualified_reasons" + deals_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "deals" + path: "/deals" + + leads_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "leads" + path: "/leads" + lead_conversions_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "lead_conversions" + path: "/lead_conversions" + lead_sources_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "lead_sources" + path: "/lead_sources" + lead_unqualified_reasons_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "lead_unqualified_reasons" + path: "/lead_unqualified_reasons" + loss_reasons_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "loss_reasons" + path: "/loss_reasons" + notes_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "notes" + path: "/notes" + orders_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "orders" + path: "/orders" + pipelines_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "pipelines" + path: "/pipelines" + products_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "products" + path: "/products" + stages_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "stages" + path: "/stages" + tags_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "tags" + path: "/tags" + tasks_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "tasks" + path: "/tasks" + text_messages_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "text_messages" + path: "/text_messages" + users_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "users" + path: "/users" + visit_outcomes_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "visit_outcomes" + path: "/visit_outcomes" + visits_stream: + $ref: "#/definitions/base_stream" + primary_key: "id" + $parameters: + name: "visits" + path: "/visits" + +streams: + - "#/definitions/call_outcomes_stream" + - "#/definitions/calls_stream" + - "#/definitions/collaborations_stream" + - "#/definitions/contacts_stream" + - "#/definitions/leads_stream" + - "#/definitions/lead_conversions_stream" + - "#/definitions/lead_sources_stream" + - "#/definitions/lead_unqualified_reasons_stream" + - "#/definitions/deal_sources_stream" + - "#/definitions/deal_unqualified_reasons_stream" + - "#/definitions/deals_stream" + - "#/definitions/loss_reasons_stream" + - "#/definitions/notes_stream" + - "#/definitions/orders_stream" + - "#/definitions/pipelines_stream" + - "#/definitions/products_stream" + - "#/definitions/stages_stream" + - "#/definitions/tags_stream" + - "#/definitions/tasks_stream" + - "#/definitions/text_messages_stream" + - "#/definitions/users_stream" + - "#/definitions/visit_outcomes_stream" + - "#/definitions/visits_stream" + +check: + type: CheckStream + stream_names: + - leads + +spec: + type: Spec + documentation_url: https://docs.airbyte.com/integrations/sources/zendesk-sell + connection_specification: + $schema: http://json-schema.org/draft-07/schema# + title: Source Zendesk Sell Spec + type: object + additionalProperties: true + required: + - api_token + properties: + api_token: + title: API token + type: string + description: "The API token for authenticating to Zendesk Sell" + examples: + - "f23yhd630otl94y85a8bf384958473pto95847fd006da49382716or937ruw059" + airbyte_secret: true + order: 1 diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/call_outcomes.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/call_outcomes.json index 0ca39aa5f870..8774f5027a58 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/call_outcomes.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/call_outcomes.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/calls.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/calls.json index 94612ce6a249..d8326c3692ce 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/calls.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/calls.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { @@ -21,7 +21,7 @@ "type": ["null", "number"] }, "phone_number": { - "type": ["null", "number"] + "type": ["null", "string"] }, "incoming": { "type": ["null", "boolean"] diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/collaborations.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/collaborations.json index 89e857aa5031..934bc0fa824f 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/collaborations.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/collaborations.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/contacts.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/contacts.json index 022461fce783..6935c6d4df58 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/contacts.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { @@ -72,13 +72,19 @@ "type": ["null", "string"] }, "address": { - "type": ["null", "string"] + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} }, "billing_address": { - "type": ["null", "string"] + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} }, "shipping_address": { - "type": ["null", "string"] + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} }, "tags": { "type": ["null", "array"] diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deal_sources.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deal_sources.json index 4c1d21218947..41aab6f4f682 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deal_sources.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deal_sources.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deal_unqualified_reasons.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deal_unqualified_reasons.json index 0ca39aa5f870..8774f5027a58 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deal_unqualified_reasons.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deal_unqualified_reasons.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deals.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deals.json index c0f3104de75a..b8f9eb527d88 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deals.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/deals.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_conversions.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_conversions.json index 0f973d1cd534..291a3b6c73af 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_conversions.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_conversions.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_sources.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_sources.json index 4c1d21218947..41aab6f4f682 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_sources.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_sources.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_unqualified_reasons.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_unqualified_reasons.json index f93853b912d9..71ad22f3caa5 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_unqualified_reasons.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/lead_unqualified_reasons.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/leads.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/leads.json index d31d0c53d6a0..46fbddb21b93 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/leads.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/leads.json @@ -1,6 +1,7 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "number"] @@ -63,13 +64,20 @@ "type": ["null", "string"] }, "address": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} + }, + "unqualified_reason_id": { "type": ["null", "string"] }, "tags": { "type": ["null", "array"] }, "custom_fields": { - "type": ["null", "object"] + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} }, "created_at": { "type": ["null", "string"], diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/loss_reasons.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/loss_reasons.json index f93853b912d9..71ad22f3caa5 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/loss_reasons.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/loss_reasons.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/notes.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/notes.json index 3ac4eb3fe957..a56878675334 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/notes.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/notes.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/orders.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/orders.json index d789fb749c46..246b2d5b1cb6 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/orders.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/orders.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/pipelines.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/pipelines.json index 6d25e0550eef..16035d118fae 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/pipelines.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/pipelines.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { @@ -8,6 +8,9 @@ "name": { "type": ["null", "string"] }, + "locale_key": { + "type": ["null", "string"] + }, "created_at": { "type": ["null", "string"], "format": "date-time", diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/products.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/products.json index 47f7873f7ec0..31c5c9febd16 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/products.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/products.json @@ -1,6 +1,7 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": ["null", "number"] @@ -24,13 +25,21 @@ "type": ["null", "number"] }, "cost": { - "type": ["null", "number"] + "type": ["null", "string"] + }, + "version": { + "type": ["null", "integer"] }, "cost_currency": { "type": ["null", "string"] }, "prices": { - "type": ["null", "array"] + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": {} + } }, "created_at": { "type": ["null", "string"], diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/stages.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/stages.json index 0a826d2e8eae..9449e57f8f97 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/stages.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/stages.json @@ -11,9 +11,18 @@ "category": { "type": ["null", "string"] }, + "locale_key": { + "type": ["null", "string"] + }, + "position_hash": { + "type": ["null", "string"] + }, "active": { "type": ["null", "boolean"] }, + "has_original_name": { + "type": ["null", "boolean"] + }, "position": { "type": ["null", "number"] }, diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tags.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tags.json index 4c1d21218947..41aab6f4f682 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tags.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tags.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tasks.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tasks.json index 66a456a01b3e..14088a9c663d 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tasks.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/tasks.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/text_messages.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/text_messages.json index 2fbbc8db5c0e..a71327aefe5d 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/text_messages.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/text_messages.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/users.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/users.json index 1c8c43003290..362e2850bf16 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/users.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/users.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { @@ -17,6 +17,12 @@ "invited": { "type": ["null", "boolean"] }, + "detached": { + "type": ["null", "boolean"] + }, + "sell_login_disabled": { + "type": ["null", "boolean"] + }, "confirmed": { "type": ["null", "boolean"] }, @@ -38,6 +44,12 @@ "reports_to": { "type": ["null", "number"] }, + "system_tags": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, "timezone": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visit_outcomes.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visit_outcomes.json index 0ca39aa5f870..8774f5027a58 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visit_outcomes.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visit_outcomes.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visits.json b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visits.json index 5118a43d1929..22cbdcfb9707 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visits.json +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/schemas/visits.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { "id": { diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py index 014a66ead128..0dd712d81c73 100644 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py +++ b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/source.py @@ -2,345 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import re -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +WARNING: Do not modify this file. +""" -# Basic full refresh stream -class ZendeskSellStream(HttpStream, ABC): - """ - This class represents a stream output by the connector. - This is an abstract base class meant to contain all the common functionality at the API level e.g: the API base URL, pagination strategy, - parsing responses etc.. - """ - - url_base = "https://api.getbase.com/v2/" - primary_key = None - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - try: - meta_links = {} - regex_page = r"[=?/]page[_=/-]?(\d{1,3})" - data = response.json() - if data: - meta_links = data.get("meta", {}).get("links") - if "next_page" in meta_links.keys(): - return {"page": int(re.findall(regex_page, meta_links["next_page"])[0])} - return None - except Exception as e: - self.logger.error(f"{e.__class__} occurred, while trying to get next page information from the following dict {meta_links}") - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - if next_page_token: - return {"page": next_page_token["page"]} - else: - return {} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - items = response.json()["items"] - yield from [item["data"] for item in items] - - -class Pipelines(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/pipelines/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "pipelines" - - -class Stages(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/stages/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "stages" - - -class Contacts(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/contacts/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "contacts" - - -class Deals(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/deals/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "deals" - - -class Leads(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/leads/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "leads" - - -class CallOutcomes(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/call-outcomes/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "call_outcomes" - - -class Calls(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/calls/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "calls" - - -class Collaborations(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/collaborations/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "collaborations" - - -class DealSources(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/deal-sources/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "deal_sources" - - -class DealUnqualifiedReasons(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/deal-unqualified-reasons/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "deal_unqualified_reasons" - - -class LeadConversions(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/lead-conversions/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "lead_conversions" - - -class LeadSources(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/lead-sources/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "lead_sources" - - -class LeadUnqualifiedReasons(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/lead-unqualified-reasons/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "lead_unqualified_reasons" - - -class LossReasons(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/loss-reasons/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "loss_reasons" - - -class Notes(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/notes/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "notes" - - -class Orders(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/orders/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "orders" - - -class Products(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/products/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "products" - - -class Tags(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/tags/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "tags" - - -class Tasks(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/tasks/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "tasks" - - -class TextMessages(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/text-messages/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "text_messages" - - -class Users(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/users/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "users" - - -class VisitOutcomes(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/visit-outcomes/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "visit_outcomes" - - -class Visits(ZendeskSellStream): - """ - Docs: https://developer.zendesk.com/api-reference/sales-crm/resources/visits/ - """ - - primary_key = "id" - - def path(self, **kwargs) -> str: - return "visits" - - -# Source -class SourceZendeskSell(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - try: - authenticator = TokenAuthenticator(token=config["api_token"]) - stream = Contacts(authenticator=authenticator) - records = stream.read_records(sync_mode=SyncMode.full_refresh) - next(records) - return True, None - except Exception as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - auth = TokenAuthenticator(token=config["api_token"]) - return [ - Contacts(authenticator=auth), - Deals(authenticator=auth), - Leads(authenticator=auth), - Pipelines(authenticator=auth), - Stages(authenticator=auth), - CallOutcomes(authenticator=auth), - Calls(authenticator=auth), - Collaborations(authenticator=auth), - DealSources(authenticator=auth), - DealUnqualifiedReasons(authenticator=auth), - LeadConversions(authenticator=auth), - LeadSources(authenticator=auth), - LeadUnqualifiedReasons(authenticator=auth), - LossReasons(authenticator=auth), - Notes(authenticator=auth), - Orders(authenticator=auth), - Products(authenticator=auth), - Tags(authenticator=auth), - Tasks(authenticator=auth), - TextMessages(authenticator=auth), - Users(authenticator=auth), - VisitOutcomes(authenticator=auth), - Visits(authenticator=auth), - ] +# Declarative Source +class SourceZendeskSell(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml b/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml deleted file mode 100644 index eb92f5a7083e..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-sell/source_zendesk_sell/spec.yaml +++ /dev/null @@ -1,15 +0,0 @@ -documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-sell -connectionSpecification: - $schema: http://json-schema.org/draft-07/schema# - title: Source Zendesk Sell Spec - type: object - required: - - api_token - properties: - api_token: - title: API token - type: string - description: "The API token for authenticating to Zendesk Sell" - examples: - - "f23yhd630otl94y85a8bf384958473pto95847fd006da49382716or937ruw059" - airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_source.py b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_source.py deleted file mode 100644 index 38920cb55102..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_source.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -from pytest import fixture -from source_zendesk_sell.source import SourceZendeskSell - - -@fixture -def config(): - return {"config": {"user_auth_key": "", "start_date": "2021-01-01T00:00:00Z", "outcome_names": ""}} - - -def test_check_connection(mocker, requests_mock, config): - source = SourceZendeskSell() - logger_mock, config_mock = MagicMock(), MagicMock() - requests_mock.get("https://api.getbase.com/v2/contacts", json={"items": [{"data": {}}]}) - - assert source.check_connection(logger_mock, config_mock) == (True, None) - - -def test_streams(mocker): - source = SourceZendeskSell() - config_mock = MagicMock() - streams = source.streams(config_mock) - expected_streams_number = 23 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py deleted file mode 100644 index e4f2b99b1e07..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-sell/unit_tests/test_streams.py +++ /dev/null @@ -1,151 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus -from unittest.mock import MagicMock - -import pytest -from source_zendesk_sell.source import ZendeskSellStream - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(ZendeskSellStream, "path", "v0/example_endpoint") - mocker.patch.object(ZendeskSellStream, "primary_key", "test_primary_key") - mocker.patch.object(ZendeskSellStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class): - stream = ZendeskSellStream() - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {} - assert stream.request_params(**inputs) == expected_params - - -@pytest.mark.parametrize( - ("inputs", "expected_token"), - [ - ( - { - "items": [], - "meta": { - "type": "collection", - "count": 25, - "links": { - "self": "https://api.getbase.com/v2/contacts?page=2&per_page=25", - "first_page": "https://api.getbase.com/v2/contacts?page=1&per_page=25", - "prev_page": "https://api.getbase.com/v2/contacts?page=1&per_page=25", - "next_page": "https://api.getbase.com/v2/contacts?page=3&per_page=25", - }, - }, - }, - {"page": 3}, - ), - ( - { - "items": [], - "meta": { - "type": "collection", - "count": 25, - "links": { - "self": "https://api.getbase.com/v2/contacts?page=2&per_page=25", - "first_page": "https://api.getbase.com/v2/contacts?page=1&per_page=25", - "prev_page": "https://api.getbase.com/v2/contacts?page=1&per_page=25", - }, - }, - }, - None, - ), - ({None}, None), - ], -) -def test_next_page_token(mocker, requests_mock, patch_base_class, inputs, expected_token): - stream = ZendeskSellStream() - response = mocker.MagicMock() - response.json.return_value = inputs - assert stream.next_page_token(response) == expected_token - - -def test_parse_response(patch_base_class, mocker): - stream = ZendeskSellStream() - response = mocker.MagicMock() - response.json.return_value = { - "items": [ - { - "data": { - "id": 302488228, - "creator_id": 2393211, - "contact_id": 302488227, - "created_at": "2020-11-12T09:05:47Z", - "updated_at": "2022-03-23T16:53:22Z", - "title": None, - "name": "Octavia Squidington", - "first_name": "Octavia", - "last_name": "Squidington", - }, - "meta": {"version": 36, "type": "contact"}, - } - ], - "meta": { - "type": "collection", - "count": 25, - "links": { - "self": "https://api.getbase.com/v2/contacts?page=2&per_page=25", - "first_page": "https://api.getbase.com/v2/contacts?page=1&per_page=25", - "prev_page": "https://api.getbase.com/v2/contacts?page=1&per_page=25", - "next_page": "https://api.getbase.com/v2/contacts?page=3&per_page=25", - }, - }, - } - expected_parsed_object = { - "id": 302488228, - "creator_id": 2393211, - "contact_id": 302488227, - "created_at": "2020-11-12T09:05:47Z", - "updated_at": "2022-03-23T16:53:22Z", - "title": None, - "name": "Octavia Squidington", - "first_name": "Octavia", - "last_name": "Squidington", - } - assert next(stream.parse_response(response)) == expected_parsed_object - - -def test_request_headers(patch_base_class): - stream = ZendeskSellStream() - stream_slice = None - stream_state = None - next_page_token = {"page": 2} - expected_headers = {"page": 2} - assert stream.request_params(stream_slice, stream_state, next_page_token) == expected_headers - - -def test_http_method(patch_base_class): - stream = ZendeskSellStream() - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = ZendeskSellStream() - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class): - response_mock = MagicMock() - stream = ZendeskSellStream() - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/README.md b/airbyte-integrations/connectors/source-zendesk-sunshine/README.md index 452bbe11f615..983968e07ae5 100644 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/README.md +++ b/airbyte-integrations/connectors/source-zendesk-sunshine/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-zendesk-sunshine:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/zendesk-sunshine) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zendesk_sunshine/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-zendesk-sunshine:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-zendesk-sunshine build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-zendesk-sunshine:airbyteDocker +An image will be built with the tag `airbyte/source-zendesk-sunshine:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-zendesk-sunshine:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,28 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zendesk-sunshine:dev c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zendesk-sunshine:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-zendesk-sunshine:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-zendesk-sunshine test +``` -#### Acceptance Tests +### Customizing acceptance Tests Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with Docker, run: -``` -./acceptance-test-docker.sh -``` - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-zendesk-sunshine:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-zendesk-sunshine:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -75,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-zendesk-sunshine test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/zendesk-sunshine.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-docker.sh deleted file mode 100644 index b6d65deeccb4..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/acceptance-test-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-zendesk-sunshine/build.gradle b/airbyte-integrations/connectors/source-zendesk-sunshine/build.gradle deleted file mode 100644 index f20109efb7bb..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-sunshine/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_zendesk_sunshine' -} diff --git a/airbyte-integrations/connectors/source-zendesk-support/Dockerfile b/airbyte-integrations/connectors/source-zendesk-support/Dockerfile deleted file mode 100644 index 35c4c32298d4..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-support/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -FROM python:3.9.11-alpine3.15 as base -FROM base as builder - - -RUN apk --no-cache upgrade \ - && pip install --upgrade pip \ - && apk --no-cache add tzdata build-base - -WORKDIR /airbyte/integration_code -COPY setup.py ./ -RUN pip install --prefix=/install . - - -FROM base -COPY --from=builder /install /usr/local -# add default timezone settings -COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN echo "Etc/UTC" > /etc/timezone - -WORKDIR /airbyte/integration_code -COPY main.py ./ -COPY source_zendesk_support ./source_zendesk_support - - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=1.0.0 -LABEL io.airbyte.name=airbyte/source-zendesk-support \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-support/README.md b/airbyte-integrations/connectors/source-zendesk-support/README.md index 9d259a28a0e2..11b8aa2fd4cb 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/README.md +++ b/airbyte-integrations/connectors/source-zendesk-support/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-zendesk-support:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/zendesk-support) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zendesk_support/spec.json` file. @@ -56,19 +48,70 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-zendesk-support:dev + + +#### Use `airbyte-ci` to build your connector +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-zendesk-support build ``` +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-zendesk-support:dev`. + +##### Customizing our build process +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. -You can also build the connector image via Gradle: +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -./gradlew :airbyte-integrations:connectors:source-zendesk-support:airbyteDocker + +#### Build your own connector image +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. +```Dockerfile +FROM airbyte/source-zendesk-support:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +Please use this as an example. This is not optimized. +2. Build your image: +```bash +docker build -t airbyte/source-zendesk-support:dev . +# Running the spec command against your patched connector +docker run airbyte/source-zendesk-support:dev spec +``` #### Run Then run any of the connector commands as follows: ``` @@ -77,45 +120,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zendesk-support:dev ch docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zendesk-support:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-zendesk-support:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .'[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-zendesk-support test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-zendesk-support:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-zendesk-support:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-zendesk-support:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +139,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-zendesk-support test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/zendesk-support.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-zendesk-support/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zendesk-support/acceptance-test-config.yml index 02fbf382affb..fb956e46e31c 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-zendesk-support/acceptance-test-config.yml @@ -31,7 +31,6 @@ acceptance_tests: extra_fields: no exact_order: no extra_records: yes - fail_on_extra_columns: false empty_streams: - name: "post_comments" bypass_reason: "not available in current subscription plan" @@ -39,15 +38,14 @@ acceptance_tests: bypass_reason: "not available in current subscription plan" - name: "post_comment_votes" bypass_reason: "not available in current subscription plan" + - name: "tags" + bypass_reason: "API issue" # TODO: remove this after all changes being merged incremental: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/incremental_catalog.json" future_state: future_state_path: "integration_tests/abnormal_state.json" - cursor_paths: - ticket_comments: ["created_at"] - threshold_days: 100 full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-zendesk-support/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-zendesk-support/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-support/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-zendesk-support/build.gradle b/airbyte-integrations/connectors/source-zendesk-support/build.gradle deleted file mode 100644 index 89517b26eeb8..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-support/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_zendesk_support' -} diff --git a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/abnormal_state.json index 1f2f9af814dd..ffacf434d37b 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/abnormal_state.json @@ -124,5 +124,75 @@ "stream_state": { "updated_at": "2222-07-19T22:21:26Z" }, "stream_descriptor": { "name": "ticket_skips" } } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-07-19T22:21:26Z" }, + "stream_descriptor": { "name": "custom_roles" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-07-19T22:21:26Z" }, + "stream_descriptor": { "name": "schedules" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-07-19T22:21:26Z" }, + "stream_descriptor": { "name": "sla_policies" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-07-19T22:21:26Z" }, + "stream_descriptor": { "name": "post_votes" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-07-19T22:21:26Z" }, + "stream_descriptor": { "name": "post_comments" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-07-19T22:21:26Z" }, + "stream_descriptor": { "name": "post_comment_votes" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-07-19T22:21:26Z" }, + "stream_descriptor": { "name": "articles" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-07-19T22:21:26Z" }, + "stream_descriptor": { "name": "article_votes" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-07-19T22:21:26Z" }, + "stream_descriptor": { "name": "article_comments" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2222-07-19T22:21:26Z" }, + "stream_descriptor": { "name": "article_comment_votes" } + } } ] diff --git a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/configured_catalog.json index 0354f87101be..7992e9dabc2e 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/configured_catalog.json @@ -60,6 +60,18 @@ "sync_mode": "full_refresh", "destination_sync_mode": "append" }, + { + "stream": { + "name": "organization_fields", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, { "stream": { "name": "satisfaction_ratings", @@ -226,7 +238,7 @@ }, { "stream": { - "name": "posts", + "name": "organization_memberships", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -238,21 +250,10 @@ }, { "stream": { - "name": "post_comments", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "post_votes", + "name": "account_attributes", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", @@ -260,18 +261,17 @@ }, { "stream": { - "name": "post_comment_votes", + "name": "attribute_definitions", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "source_defined_primary_key": [["id"]] + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false }, "sync_mode": "full_refresh", "destination_sync_mode": "append" }, { "stream": { - "name": "organization_memberships", + "name": "user_fields", "json_schema": {}, "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, @@ -280,27 +280,6 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "append" - }, - { - "stream": { - "name": "account_attributes", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false, - "source_defined_primary_key": [["id"]] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "attribute_definitions", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl index 5931bfa47ac0..4123754c3b14 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl @@ -1,62 +1,73 @@ -{"stream": "audit_logs", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/audit_logs/7502393054223.json", "id": 7502393054223, "action_label": "Signed in", "actor_id": 360786799676, "source_id": 360786799676, "source_type": "user", "source_label": "Team member: Team Airbyte", "action": "login", "change_description": "Successful sign-in using Zendesk password from https://d3v-airbyte.zendesk.com/access/login", "ip_address": "109.86.166.58", "created_at": "2023-07-24T10:56:28Z", "actor_name": "Team Airbyte"}, "emitted_at": 1690888150345} -{"stream": "audit_logs", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/audit_logs/7465455408271.json", "id": 7465455408271, "action_label": "Signed in", "actor_id": 360786799676, "source_id": 360786799676, "source_type": "user", "source_label": "Team member: Team Airbyte", "action": "login", "change_description": "Successful sign-in using Zendesk password from https://d3v-airbyte.zendesk.com/access/login", "ip_address": "109.86.166.58", "created_at": "2023-07-21T08:03:28Z", "actor_name": "Team Airbyte"}, "emitted_at": 1690888150346} -{"stream": "audit_logs", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/audit_logs/7453133196303.json", "id": 7453133196303, "action_label": "Signed in", "actor_id": 360786799676, "source_id": 360786799676, "source_type": "user", "source_label": "Team member: Team Airbyte", "action": "login", "change_description": "Successful sign-in using Zendesk password from https://d3v-airbyte.zendesk.com/access/login", "ip_address": "136.24.229.166", "created_at": "2023-07-19T19:09:32Z", "actor_name": "Team Airbyte"}, "emitted_at": 1690888150346} -{"stream": "group_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/group_memberships/360007820916.json", "id": 360007820916, "user_id": 360786799676, "group_id": 360003074836, "default": true, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z"}, "emitted_at": 1690888151470} -{"stream": "group_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/group_memberships/360011727976.json", "id": 360011727976, "user_id": 361084605116, "group_id": 360003074836, "default": true, "created_at": "2021-04-23T14:33:11Z", "updated_at": "2021-04-23T14:33:11Z"}, "emitted_at": 1690888151471} -{"stream": "group_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/group_memberships/360011812655.json", "id": 360011812655, "user_id": 361089721035, "group_id": 360003074836, "default": true, "created_at": "2021-04-23T14:34:20Z", "updated_at": "2021-04-23T14:34:20Z"}, "emitted_at": 1690888151471} -{"stream": "groups", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/groups/7282640316815.json", "id": 7282640316815, "is_public": true, "name": "Airbyte Department 1", "description": "A sample department", "default": false, "deleted": false, "created_at": "2023-06-26T10:09:12Z", "updated_at": "2023-06-26T10:09:12Z"}, "emitted_at": 1690888152597} -{"stream": "groups", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/groups/7282618889231.json", "id": 7282618889231, "is_public": true, "name": "Department 1", "description": "A sample department", "default": false, "deleted": false, "created_at": "2023-06-26T10:09:14Z", "updated_at": "2023-06-26T10:09:14Z"}, "emitted_at": 1690888152598} -{"stream": "groups", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/groups/7282630247567.json", "id": 7282630247567, "is_public": true, "name": "Department 2", "description": "A sample department 2", "default": false, "deleted": false, "created_at": "2023-06-26T10:09:14Z", "updated_at": "2023-06-26T10:09:14Z"}, "emitted_at": 1690888152598} -{"stream": "macros", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/macros/360011363556.json", "id": 360011363556, "title": "Customer not responding", "active": true, "updated_at": "2020-12-11T18:34:06Z", "created_at": "2020-12-11T18:34:06Z", "default": false, "position": 9999, "description": null, "actions": [{"field": "status", "value": "pending"}, {"field": "comment_value", "value": "Hello {{ticket.requester.name}}. Our agent {{current_user.name}} has tried to contact you about this request but we haven't heard back from you yet. Please let us know if we can be of further assistance. Thanks. "}], "restriction": null, "raw_title": "Customer not responding"}, "emitted_at": 1690888153534} -{"stream": "macros", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/macros/360011363536.json", "id": 360011363536, "title": "Downgrade and inform", "active": true, "updated_at": "2020-12-11T18:34:06Z", "created_at": "2020-12-11T18:34:06Z", "default": false, "position": 9999, "description": null, "actions": [{"field": "priority", "value": "low"}, {"field": "comment_value", "value": "We're currently experiencing unusually high traffic. We'll get back to you as soon as possible."}], "restriction": null, "raw_title": "Downgrade and inform"}, "emitted_at": 1690888153535} -{"stream": "organizations", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/organizations/360033549136.json", "id": 360033549136, "name": "Airbyte", "shared_tickets": true, "shared_comments": true, "external_id": null, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2023-04-13T14:51:21Z", "domain_names": ["cloud.airbyte.com"], "details": "test", "notes": "test", "group_id": 6770788212111, "tags": ["test"], "organization_fields": {"test_check_box_field_1": false, "test_drop_down_field_1": null, "test_number_field_1": null}}, "emitted_at": 1690888154541} -{"stream": "organizations", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/organizations/360045373216.json", "id": 360045373216, "name": "ressssssss", "shared_tickets": false, "shared_comments": false, "external_id": null, "created_at": "2021-07-15T18:29:14Z", "updated_at": "2021-07-15T18:29:14Z", "domain_names": [], "details": "", "notes": "", "group_id": null, "tags": [], "organization_fields": {"test_check_box_field_1": false, "test_drop_down_field_1": null, "test_number_field_1": null}}, "emitted_at": 1690888154543} -{"stream": "organization_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/organization_memberships/360057705196.json", "id": 360057705196, "user_id": 360786799676, "organization_id": 360033549136, "default": true, "created_at": "2020-12-11T18:34:05Z", "organization_name": "Airbyte", "updated_at": "2020-12-11T18:34:05Z", "view_tickets": true}, "emitted_at": 1690888156003} -{"stream": "organization_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/organization_memberships/7282880134671.json", "id": 7282880134671, "user_id": 7282634891791, "organization_id": 360033549136, "default": true, "created_at": "2023-06-26T11:03:38Z", "organization_name": "Airbyte", "updated_at": "2023-06-26T11:03:38Z", "view_tickets": true}, "emitted_at": 1690888156004} -{"stream": "posts", "data": {"id": 7253351904271, "title": "How do I get around the community?", "details": "

      You can use search to find answers. You can also browse topics and posts using views and filters. See Getting around the community.

      ", "author_id": 360786799676, "vote_sum": 0, "vote_count": 0, "comment_count": 0, "follower_count": 0, "topic_id": 7253351897871, "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/community/posts/7253351904271-How-do-I-get-around-the-community-", "created_at": "2023-06-22T00:32:21Z", "updated_at": "2023-06-22T00:32:21Z", "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/posts/7253351904271-How-do-I-get-around-the-community-.json", "featured": false, "pinned": false, "closed": false, "frozen": false, "status": "none", "non_author_editor_id": null, "non_author_updated_at": null, "content_tag_ids": []}, "emitted_at": 1692700567985} -{"stream": "posts", "data": {"id": 7253375870607, "title": "Which topics should I add to my community?", "details": "

      That depends. If you support several products, you might add a topic for each product. If you have one big product, you might add a topic for each major feature area or task. If you have different types of users (for example, end users and API developers), you might add a topic or topics for each type of user.

      A General Discussion topic is a place for users to discuss issues that don't quite fit in the other topics. You could monitor this topic for emerging issues that might need their own topics.

      \n\n

      To create your own topics, see Adding community discussion topics.

      ", "author_id": 360786799676, "vote_sum": 0, "vote_count": 0, "comment_count": 0, "follower_count": 0, "topic_id": 7253351897871, "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/community/posts/7253375870607-Which-topics-should-I-add-to-my-community-", "created_at": "2023-06-22T00:32:21Z", "updated_at": "2023-06-22T00:32:21Z", "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/posts/7253375870607-Which-topics-should-I-add-to-my-community-.json", "featured": false, "pinned": false, "closed": false, "frozen": false, "status": "none", "non_author_editor_id": null, "non_author_updated_at": null, "content_tag_ids": []}, "emitted_at": 1692700567986} -{"stream": "posts", "data": {"id": 7253375879055, "title": "I'd like a way for users to submit feature requests", "details": "

      You can add a topic like this one in your community. End users can add feature requests and describe their use cases. Other users can comment on the requests and vote for them. Product managers can review feature requests and provide feedback.

      ", "author_id": 360786799676, "vote_sum": 0, "vote_count": 0, "comment_count": 0, "follower_count": 0, "topic_id": 7253394974479, "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/community/posts/7253375879055-I-d-like-a-way-for-users-to-submit-feature-requests", "created_at": "2023-06-22T00:32:21Z", "updated_at": "2023-06-22T00:32:21Z", "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/posts/7253375879055-I-d-like-a-way-for-users-to-submit-feature-requests.json", "featured": false, "pinned": false, "closed": false, "frozen": false, "status": "none", "non_author_editor_id": null, "non_author_updated_at": null, "content_tag_ids": []}, "emitted_at": 1692700567986} -{"stream": "satisfaction_ratings", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/satisfaction_ratings/7235633102607.json", "id": 7235633102607, "assignee_id": null, "group_id": null, "requester_id": 361089721035, "ticket_id": 146, "score": "offered", "created_at": "2023-06-19T18:01:40Z", "updated_at": "2023-06-19T18:01:40Z", "comment": null}, "emitted_at": 1690888165601} -{"stream": "satisfaction_ratings", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/satisfaction_ratings/5909514818319.json", "id": 5909514818319, "assignee_id": null, "group_id": null, "requester_id": 360786799676, "ticket_id": 25, "score": "offered", "created_at": "2022-11-22T17:02:04Z", "updated_at": "2022-11-22T17:02:04Z", "comment": null}, "emitted_at": 1690888165602} -{"stream": "satisfaction_ratings", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/satisfaction_ratings/5527212710799.json", "id": 5527212710799, "assignee_id": null, "group_id": null, "requester_id": 5527080499599, "ticket_id": 144, "score": "offered", "created_at": "2022-09-19T16:01:43Z", "updated_at": "2022-09-19T16:01:43Z", "comment": null}, "emitted_at": 1690888165602} -{"stream": "sla_policies", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/slas/policies/360001110696.json", "id": 360001110696, "title": "test police", "description": "for tests", "position": 1, "filter": {"all": [{"field": "assignee_id", "operator": "is", "value": 361089721035}], "any": []}, "policy_metrics": [{"priority": "high", "metric": "first_reply_time", "target": 61, "business_hours": false}], "created_at": "2021-07-16T11:05:31Z", "updated_at": "2021-07-16T11:05:31Z"}, "emitted_at": 1690888166730} -{"stream": "sla_policies", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/slas/policies/360001113715.json", "id": 360001113715, "title": "test police 2", "description": "test police 2", "position": 2, "filter": {"all": [{"field": "organization_id", "operator": "is", "value": 360033549136}], "any": []}, "policy_metrics": [{"priority": "high", "metric": "first_reply_time", "target": 121, "business_hours": false}], "created_at": "2021-07-16T11:06:01Z", "updated_at": "2021-07-16T11:06:01Z"}, "emitted_at": 1690888166731} -{"stream": "tags", "data": {"name": "test", "count": 6}, "emitted_at": 1690888168471} -{"stream": "tags", "data": {"name": "tag2", "count": 3}, "emitted_at": 1690888168472} -{"stream": "tags", "data": {"name": "tag1", "count": 2}, "emitted_at": 1690888168472} -{"stream": "ticket_audits", "data": {"id": 7429253845903, "ticket_id": 152, "created_at": "2023-07-16T12:01:39Z", "author_id": -1, "metadata": {"system": {}, "custom": {}}, "events": [{"id": 7429253846031, "type": "Change", "value": "closed", "field_name": "status", "previous_value": "solved"}], "via": {"channel": "rule", "source": {"to": {}, "from": {"deleted": false, "title": "Close ticket 4 days after status is set to solved", "id": 6241378811151}, "rel": "automation"}}}, "emitted_at": 1690888174095} -{"stream": "ticket_audits", "data": {"id": 7283194465039, "ticket_id": 141, "created_at": "2023-06-26T12:15:34Z", "author_id": 360786799676, "metadata": {"system": {"client": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.58", "ip_address": "85.209.47.207", "location": "Kyiv, 30, Ukraine", "latitude": 50.458, "longitude": 30.5303}, "custom": {}}, "events": [{"id": 7283194465167, "type": "Change", "value": "7282634891791", "field_name": "assignee_id", "previous_value": null}, {"id": 7283194465295, "type": "Change", "value": "360006394556", "field_name": "group_id", "previous_value": null}, {"id": 7283194465423, "type": "Notification", "via": {"channel": "rule", "source": {"from": {"deleted": false, "title": "Notify assignee of assignment", "id": 360011363256, "revision_id": 1}, "rel": "trigger"}}, "subject": "[{{ticket.account}}] Assignment: {{ticket.title}}", "body": "You have been assigned to this ticket (#{{ticket.id}}).\n\n{{ticket.comments_formatted}}", "recipients": [7282634891791]}], "via": {"channel": "web", "source": {"from": {}, "to": {}, "rel": null}}}, "emitted_at": 1690888174096} -{"stream": "ticket_audits", "data": {"id": 7283163099535, "ticket_id": 153, "created_at": "2023-06-26T12:13:42Z", "author_id": 360786799676, "metadata": {"system": {"client": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.58", "ip_address": "85.209.47.207", "location": "Kyiv, 30, Ukraine", "latitude": 50.458, "longitude": 30.5303}, "custom": {}}, "events": [{"id": 7283163099663, "type": "Change", "value": "7282634891791", "field_name": "assignee_id", "previous_value": "360786799676"}, {"id": 7283163099791, "type": "Change", "value": "360006394556", "field_name": "group_id", "previous_value": "6770788212111"}, {"id": 7283163099919, "type": "Notification", "via": {"channel": "rule", "source": {"from": {"deleted": false, "title": "Notify assignee of assignment", "id": 360011363256, "revision_id": 1}, "rel": "trigger"}}, "subject": "[{{ticket.account}}] Assignment: {{ticket.title}}", "body": "You have been assigned to this ticket (#{{ticket.id}}).\n\n{{ticket.comments_formatted}}", "recipients": [7282634891791]}], "via": {"channel": "web", "source": {"from": {}, "to": {}, "rel": null}}}, "emitted_at": 1690888174097} -{"stream": "ticket_comments", "data": {"id": 5162146653071, "via": {"channel": "web", "source": {"from": {}, "to": {"name": "Team Airbyte", "address": "integration-test@airbyte.io"}, "rel": null}}, "via_reference_id": null, "type": "Comment", "author_id": 360786799676, "body": " 163748", "html_body": "
       163748
      ", "plain_body": " 163748", "public": true, "attachments": [], "audit_id": 5162146652943, "created_at": "2022-07-18T09:58:23Z", "event_type": "Comment", "ticket_id": 124, "timestamp": 1658138303}, "emitted_at": 1690888176621} -{"stream": "ticket_comments", "data": {"id": 5162208963983, "via": {"channel": "web", "source": {"from": {}, "to": {}, "rel": null}}, "via_reference_id": null, "type": "Comment", "author_id": 360786799676, "body": "238473846", "html_body": "
      238473846
      ", "plain_body": "238473846", "public": false, "attachments": [], "audit_id": 5162208963855, "created_at": "2022-07-18T10:16:53Z", "event_type": "Comment", "ticket_id": 125, "timestamp": 1658139413}, "emitted_at": 1690888176622} -{"stream": "ticket_comments", "data": {"id": 5162223308559, "via": {"channel": "web", "source": {"from": {}, "to": {}, "rel": null}}, "via_reference_id": null, "type": "Comment", "author_id": 360786799676, "body": "Airbyte", "html_body": "", "plain_body": "Airbyte", "public": false, "attachments": [], "audit_id": 5162223308431, "created_at": "2022-07-18T10:25:21Z", "event_type": "Comment", "ticket_id": 125, "timestamp": 1658139921}, "emitted_at": 1690888176622} -{"stream": "ticket_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_fields/360002833076.json", "id": 360002833076, "type": "subject", "title": "Subject", "raw_title": "Subject", "description": "", "raw_description": "", "position": 1, "active": true, "required": false, "collapsed_for_agents": false, "regexp_for_validation": null, "title_in_portal": "Subject", "raw_title_in_portal": "Subject", "visible_in_portal": true, "editable_in_portal": true, "required_in_portal": true, "tag": null, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z", "removable": false, "key": null, "agent_description": null}, "emitted_at": 1690888178196} -{"stream": "ticket_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_fields/360002833096.json", "id": 360002833096, "type": "description", "title": "Description", "raw_title": "Description", "description": "Please enter the details of your request. A member of our support staff will respond as soon as possible.", "raw_description": "Please enter the details of your request. A member of our support staff will respond as soon as possible.", "position": 2, "active": true, "required": false, "collapsed_for_agents": false, "regexp_for_validation": null, "title_in_portal": "Description", "raw_title_in_portal": "Description", "visible_in_portal": true, "editable_in_portal": true, "required_in_portal": true, "tag": null, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z", "removable": false, "key": null, "agent_description": null}, "emitted_at": 1690888178197} -{"stream": "ticket_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_fields/360002833116.json", "id": 360002833116, "type": "status", "title": "Status", "raw_title": "Status", "description": "Request status", "raw_description": "Request status", "position": 3, "active": true, "required": false, "collapsed_for_agents": false, "regexp_for_validation": null, "title_in_portal": "Status", "raw_title_in_portal": "Status", "visible_in_portal": false, "editable_in_portal": false, "required_in_portal": false, "tag": null, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z", "removable": false, "key": null, "agent_description": null, "system_field_options": [{"name": "Open", "value": "open"}, {"name": "Pending", "value": "pending"}, {"name": "Solved", "value": "solved"}], "sub_type_id": 0}, "emitted_at": 1690888178198} -{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7283000498191.json", "id": 7283000498191, "ticket_id": 153, "created_at": "2023-06-26T11:31:48Z", "updated_at": "2023-06-26T12:13:42Z", "group_stations": 2, "assignee_stations": 2, "reopens": 0, "replies": 0, "assignee_updated_at": "2023-06-26T11:31:48Z", "requester_updated_at": "2023-06-26T11:31:48Z", "status_updated_at": "2023-06-26T11:31:48Z", "initially_assigned_at": "2023-06-26T11:31:48Z", "assigned_at": "2023-06-26T12:13:42Z", "solved_at": null, "latest_comment_added_at": "2023-06-26T11:31:48Z", "reply_time_in_minutes": {"calendar": null, "business": null}, "first_resolution_time_in_minutes": {"calendar": null, "business": null}, "full_resolution_time_in_minutes": {"calendar": null, "business": null}, "agent_wait_time_in_minutes": {"calendar": null, "business": null}, "requester_wait_time_in_minutes": {"calendar": null, "business": null}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "custom_status_updated_at": "2023-06-26T11:31:48Z"}, "emitted_at": 1690888179326} -{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7282909551759.json", "id": 7282909551759, "ticket_id": 152, "created_at": "2023-06-26T11:10:33Z", "updated_at": "2023-06-26T11:25:43Z", "group_stations": 1, "assignee_stations": 1, "reopens": 0, "replies": 1, "assignee_updated_at": "2023-06-26T11:25:43Z", "requester_updated_at": "2023-06-26T11:10:33Z", "status_updated_at": "2023-07-16T12:01:39Z", "initially_assigned_at": "2023-06-26T11:10:33Z", "assigned_at": "2023-06-26T11:10:33Z", "solved_at": "2023-06-26T11:25:43Z", "latest_comment_added_at": "2023-06-26T11:21:06Z", "reply_time_in_minutes": {"calendar": 11, "business": 0}, "first_resolution_time_in_minutes": {"calendar": 15, "business": 0}, "full_resolution_time_in_minutes": {"calendar": 15, "business": 0}, "agent_wait_time_in_minutes": {"calendar": 15, "business": 0}, "requester_wait_time_in_minutes": {"calendar": 0, "business": 0}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "custom_status_updated_at": "2023-06-26T11:25:43Z"}, "emitted_at": 1690888179326} -{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7282901696015.json", "id": 7282901696015, "ticket_id": 151, "created_at": "2023-06-26T11:09:33Z", "updated_at": "2023-06-26T12:03:38Z", "group_stations": 1, "assignee_stations": 1, "reopens": 0, "replies": 1, "assignee_updated_at": "2023-06-26T12:03:37Z", "requester_updated_at": "2023-06-26T11:09:33Z", "status_updated_at": "2023-06-26T11:09:33Z", "initially_assigned_at": "2023-06-26T11:09:33Z", "assigned_at": "2023-06-26T11:09:33Z", "solved_at": null, "latest_comment_added_at": "2023-06-26T12:03:37Z", "reply_time_in_minutes": {"calendar": 54, "business": 0}, "first_resolution_time_in_minutes": {"calendar": null, "business": null}, "full_resolution_time_in_minutes": {"calendar": null, "business": null}, "agent_wait_time_in_minutes": {"calendar": null, "business": null}, "requester_wait_time_in_minutes": {"calendar": null, "business": null}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "custom_status_updated_at": "2023-06-26T11:09:33Z"}, "emitted_at": 1690888179327} -{"stream": "ticket_metric_events", "data": {"id": 4992797383183, "ticket_id": 121, "metric": "agent_work_time", "instance_id": 0, "type": "measure", "time": "2022-06-17T14:49:20Z"}, "emitted_at": 1690888180347} -{"stream": "ticket_metric_events", "data": {"id": 4992797383311, "ticket_id": 121, "metric": "pausable_update_time", "instance_id": 0, "type": "measure", "time": "2022-06-17T14:49:20Z"}, "emitted_at": 1690888180348} -{"stream": "ticket_metric_events", "data": {"id": 4992797383439, "ticket_id": 121, "metric": "reply_time", "instance_id": 0, "type": "measure", "time": "2022-06-17T14:49:20Z"}, "emitted_at": 1690888180348} -{"stream": "ticket_skips", "data": {"id": 7290033348623, "ticket_id": 121, "user_id": 360786799676, "reason": "I have no idea.", "created_at": "2023-06-27T08:24:02Z", "updated_at": "2023-06-27T08:24:02Z", "ticket": {"url": "https://d3v-airbyte.zendesk.com/api/v2/tickets/121.json", "id": 121, "external_id": null, "via": {"channel": "voice", "source": {"rel": "voicemail", "from": {"formatted_phone": "+1 (689) 689-8023", "phone": "+16896898023", "name": "Caller +1 (689) 689-8023"}, "to": {"formatted_phone": "+1 (205) 953-1462", "phone": "+12059531462", "name": "Airbyte", "brand_id": 360000358316}}}, "created_at": "2022-06-17T14:49:20Z", "updated_at": "2022-06-17T16:01:42Z", "type": null, "subject": "Voicemail from: Caller +1 (689) 689-8023", "raw_subject": "Voicemail from: Caller +1 (689) 689-8023", "description": "Call from: +1 (689) 689-8023\\nTime of call: June 17, 2022 at 2:48:27 PM", "priority": null, "status": "new", "recipient": null, "requester_id": 4992781783439, "submitter_id": 4992781783439, "assignee_id": null, "organization_id": null, "group_id": null, "collaborator_ids": [], "follower_ids": [], "email_cc_ids": [], "forum_topic_id": null, "problem_id": null, "has_incidents": false, "is_public": false, "due_at": null, "tags": [], "custom_fields": [], "satisfaction_rating": {"score": "offered"}, "sharing_agreement_ids": [], "custom_status_id": 4044356, "fields": [], "followup_ids": [], "ticket_form_id": 360000084116, "deleted_ticket_form_id": null, "brand_id": 360000358316, "allow_channelback": false, "allow_attachments": true, "from_messaging_channel": false}}, "emitted_at": 1690888182191} -{"stream": "ticket_skips", "data": {"id": 7290088475023, "ticket_id": 125, "user_id": 360786799676, "reason": "Another test skip.", "created_at": "2023-06-27T08:30:01Z", "updated_at": "2023-06-27T08:30:01Z", "ticket": {"url": "https://d3v-airbyte.zendesk.com/api/v2/tickets/125.json", "id": 125, "external_id": null, "via": {"channel": "web", "source": {"from": {}, "to": {}, "rel": null}}, "created_at": "2022-07-18T10:16:53Z", "updated_at": "2022-07-18T10:36:02Z", "type": "question", "subject": "Ticket Test 2", "raw_subject": "Ticket Test 2", "description": "238473846", "priority": "urgent", "status": "open", "recipient": null, "requester_id": 360786799676, "submitter_id": 360786799676, "assignee_id": 361089721035, "organization_id": 360033549136, "group_id": 5059439464079, "collaborator_ids": [360786799676], "follower_ids": [360786799676], "email_cc_ids": [], "forum_topic_id": null, "problem_id": null, "has_incidents": false, "is_public": false, "due_at": null, "tags": [], "custom_fields": [], "satisfaction_rating": {"score": "unoffered"}, "sharing_agreement_ids": [], "custom_status_id": 4044376, "fields": [], "followup_ids": [], "ticket_form_id": 360000084116, "deleted_ticket_form_id": null, "brand_id": 360000358316, "allow_channelback": false, "allow_attachments": true, "from_messaging_channel": false}}, "emitted_at": 1690888182192} -{"stream": "tickets", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/tickets/121.json", "id": 121, "external_id": null, "via": {"channel": "voice", "source": {"rel": "voicemail", "from": {"formatted_phone": "+1 (689) 689-8023", "phone": "+16896898023", "name": "Caller +1 (689) 689-8023"}, "to": {"formatted_phone": "+1 (205) 953-1462", "phone": "+12059531462", "name": "Airbyte", "brand_id": 360000358316}}}, "created_at": "2022-06-17T14:49:20Z", "updated_at": "2022-06-17T16:01:42Z", "type": null, "subject": "Voicemail from: Caller +1 (689) 689-8023", "raw_subject": "Voicemail from: Caller +1 (689) 689-8023", "description": "Call from: +1 (689) 689-8023\\nTime of call: June 17, 2022 at 2:48:27 PM", "priority": null, "status": "new", "recipient": null, "requester_id": 4992781783439, "submitter_id": 4992781783439, "assignee_id": null, "organization_id": null, "group_id": null, "collaborator_ids": [], "follower_ids": [], "email_cc_ids": [], "forum_topic_id": null, "problem_id": null, "has_incidents": false, "is_public": false, "due_at": null, "tags": [], "custom_fields": [], "satisfaction_rating": {"score": "offered"}, "sharing_agreement_ids": [], "custom_status_id": 4044356, "fields": [], "followup_ids": [], "ticket_form_id": 360000084116, "brand_id": 360000358316, "allow_channelback": false, "allow_attachments": true, "from_messaging_channel": false, "generated_timestamp": 1655481702}, "emitted_at": 1690888183377} -{"stream": "tickets", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/tickets/122.json", "id": 122, "external_id": null, "via": {"channel": "voice", "source": {"rel": "voicemail", "from": {"formatted_phone": "+1 (912) 420-0314", "phone": "+19124200314", "name": "Caller +1 (912) 420-0314"}, "to": {"formatted_phone": "+1 (205) 953-1462", "phone": "+12059531462", "name": "Airbyte", "brand_id": 360000358316}}}, "created_at": "2022-06-17T19:52:39Z", "updated_at": "2022-06-17T21:01:41Z", "type": null, "subject": "Voicemail from: Caller +1 (912) 420-0314", "raw_subject": "Voicemail from: Caller +1 (912) 420-0314", "description": "Call from: +1 (912) 420-0314\\nTime of call: June 17, 2022 at 7:52:02 PM", "priority": null, "status": "new", "recipient": null, "requester_id": 4993467856015, "submitter_id": 4993467856015, "assignee_id": null, "organization_id": null, "group_id": null, "collaborator_ids": [], "follower_ids": [], "email_cc_ids": [], "forum_topic_id": null, "problem_id": null, "has_incidents": false, "is_public": false, "due_at": null, "tags": [], "custom_fields": [], "satisfaction_rating": {"score": "offered"}, "sharing_agreement_ids": [], "custom_status_id": 4044356, "fields": [], "followup_ids": [], "ticket_form_id": 360000084116, "brand_id": 360000358316, "allow_channelback": false, "allow_attachments": true, "from_messaging_channel": false, "generated_timestamp": 1655499701}, "emitted_at": 1690888183379} -{"stream": "tickets", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/tickets/123.json", "id": 123, "external_id": null, "via": {"channel": "voice", "source": {"rel": "voicemail", "from": {"formatted_phone": "+1 (607) 210-9549", "phone": "+16072109549", "name": "Caller +1 (607) 210-9549"}, "to": {"formatted_phone": "+1 (205) 953-1462", "phone": "+12059531462", "name": "Airbyte", "brand_id": 360000358316}}}, "created_at": "2022-07-13T14:34:05Z", "updated_at": "2022-07-13T16:02:03Z", "type": null, "subject": "Voicemail from: Caller +1 (607) 210-9549", "raw_subject": "Voicemail from: Caller +1 (607) 210-9549", "description": "Call from: +1 (607) 210-9549\\nTime of call: July 13, 2022 at 2:33:27 PM", "priority": null, "status": "new", "recipient": null, "requester_id": 5137812260495, "submitter_id": 5137812260495, "assignee_id": null, "organization_id": null, "group_id": null, "collaborator_ids": [], "follower_ids": [], "email_cc_ids": [], "forum_topic_id": null, "problem_id": null, "has_incidents": false, "is_public": false, "due_at": null, "tags": [], "custom_fields": [], "satisfaction_rating": {"score": "offered"}, "sharing_agreement_ids": [], "custom_status_id": 4044356, "fields": [], "followup_ids": [], "ticket_form_id": 360000084116, "brand_id": 360000358316, "allow_channelback": false, "allow_attachments": true, "from_messaging_channel": false, "generated_timestamp": 1657728123}, "emitted_at": 1690888183380} -{"stream": "users", "data": {"id": 4992781783439, "url": "https://d3v-airbyte.zendesk.com/api/v2/users/4992781783439.json", "name": "Caller +1 (689) 689-8023", "email": null, "created_at": "2022-06-17T14:49:19Z", "updated_at": "2022-06-17T14:49:19Z", "time_zone": "Pacific/Noumea", "iana_time_zone": "Pacific/Noumea", "phone": "+16896898023", "shared_phone_number": false, "photo": null, "locale_id": 1, "locale": "en-US", "organization_id": null, "role": "end-user", "verified": true, "external_id": null, "tags": [], "alias": null, "active": true, "shared": false, "shared_agent": false, "last_login_at": null, "two_factor_auth_enabled": false, "signature": null, "details": null, "notes": null, "role_type": null, "custom_role_id": null, "moderator": false, "ticket_restriction": "requested", "only_private_comments": false, "restricted_agent": true, "suspended": false, "default_group_id": null, "report_csv": false, "user_fields": {}}, "emitted_at": 1690888188031} -{"stream": "users", "data": {"id": 4993467856015, "url": "https://d3v-airbyte.zendesk.com/api/v2/users/4993467856015.json", "name": "Caller +1 (912) 420-0314", "email": null, "created_at": "2022-06-17T19:52:38Z", "updated_at": "2022-06-17T19:52:38Z", "time_zone": "Pacific/Noumea", "iana_time_zone": "Pacific/Noumea", "phone": "+19124200314", "shared_phone_number": false, "photo": null, "locale_id": 1, "locale": "en-US", "organization_id": null, "role": "end-user", "verified": true, "external_id": null, "tags": [], "alias": null, "active": true, "shared": false, "shared_agent": false, "last_login_at": null, "two_factor_auth_enabled": false, "signature": null, "details": null, "notes": null, "role_type": null, "custom_role_id": null, "moderator": false, "ticket_restriction": "requested", "only_private_comments": false, "restricted_agent": true, "suspended": false, "default_group_id": null, "report_csv": false, "user_fields": {}}, "emitted_at": 1690888188032} -{"stream": "users", "data": {"id": 5137812260495, "url": "https://d3v-airbyte.zendesk.com/api/v2/users/5137812260495.json", "name": "Caller +1 (607) 210-9549", "email": null, "created_at": "2022-07-13T14:34:04Z", "updated_at": "2022-07-13T14:34:04Z", "time_zone": "Pacific/Noumea", "iana_time_zone": "Pacific/Noumea", "phone": "+16072109549", "shared_phone_number": false, "photo": null, "locale_id": 1, "locale": "en-US", "organization_id": null, "role": "end-user", "verified": true, "external_id": null, "tags": [], "alias": null, "active": true, "shared": false, "shared_agent": false, "last_login_at": null, "two_factor_auth_enabled": false, "signature": null, "details": null, "notes": null, "role_type": null, "custom_role_id": null, "moderator": false, "ticket_restriction": "requested", "only_private_comments": false, "restricted_agent": true, "suspended": false, "default_group_id": null, "report_csv": false, "user_fields": {}}, "emitted_at": 1690888188033} -{"stream": "brands", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/brands/360000358316.json", "id": 360000358316, "name": "Airbyte", "brand_url": "https://d3v-airbyte.zendesk.com", "subdomain": "d3v-airbyte", "host_mapping": null, "has_help_center": true, "help_center_state": "enabled", "active": true, "default": true, "is_deleted": false, "logo": null, "ticket_form_ids": [360000084116], "signature_template": "{{agent.signature}}", "created_at": "2020-12-11T18:34:04Z", "updated_at": "2020-12-11T18:34:09Z"}, "emitted_at": 1690888190028} -{"stream": "custom_roles", "data": {"id": 360000210636, "name": "Advisor", "description": "Can automate ticket workflows, manage channels and make private comments on tickets", "role_type": 0, "created_at": "2020-12-11T18:34:36Z", "updated_at": "2020-12-11T18:34:36Z", "configuration": {"chat_access": true, "end_user_list_access": "full", "forum_access_restricted_content": false, "light_agent": false, "manage_business_rules": true, "manage_dynamic_content": false, "manage_extensions_and_channels": true, "manage_facebook": true, "moderate_forums": false, "side_conversation_create": true, "ticket_access": "within-groups", "ticket_comment_access": "none", "ticket_deletion": false, "ticket_tag_editing": true, "twitter_search_access": false, "view_deleted_tickets": false, "voice_access": true, "group_access": false, "organization_editing": false, "organization_notes_editing": false, "assign_tickets_to_any_group": false, "end_user_profile_access": "readonly", "explore_access": "readonly", "forum_access": "readonly", "macro_access": "full", "report_access": "none", "ticket_editing": true, "ticket_merge": false, "user_view_access": "full", "view_access": "full", "voice_dashboard_access": false, "manage_automations": true, "manage_contextual_workspaces": false, "manage_organization_fields": false, "manage_skills": true, "manage_slas": true, "manage_ticket_fields": false, "manage_ticket_forms": false, "manage_user_fields": false, "ticket_redaction": false, "manage_roles": "none", "manage_groups": false, "manage_group_memberships": false, "manage_organizations": false, "manage_suspended_tickets": false, "manage_triggers": true, "manage_team_members": "readonly"}, "team_member_count": 1}, "emitted_at": 1692702313283} -{"stream": "custom_roles", "data": {"id": 360000210596, "name": "Staff", "description": "Can edit tickets within their groups", "role_type": 0, "created_at": "2020-12-11T18:34:36Z", "updated_at": "2020-12-11T18:34:36Z", "configuration": {"chat_access": true, "end_user_list_access": "full", "forum_access_restricted_content": false, "light_agent": false, "manage_business_rules": false, "manage_dynamic_content": false, "manage_extensions_and_channels": false, "manage_facebook": false, "moderate_forums": false, "side_conversation_create": true, "ticket_access": "within-groups", "ticket_comment_access": "public", "ticket_deletion": false, "ticket_tag_editing": false, "twitter_search_access": false, "view_deleted_tickets": false, "voice_access": true, "group_access": false, "organization_editing": false, "organization_notes_editing": false, "assign_tickets_to_any_group": false, "end_user_profile_access": "readonly", "explore_access": "readonly", "forum_access": "readonly", "macro_access": "manage-personal", "report_access": "readonly", "ticket_editing": true, "ticket_merge": false, "user_view_access": "manage-personal", "view_access": "manage-personal", "voice_dashboard_access": false, "manage_automations": false, "manage_contextual_workspaces": false, "manage_organization_fields": false, "manage_skills": false, "manage_slas": false, "manage_ticket_fields": false, "manage_ticket_forms": false, "manage_user_fields": false, "ticket_redaction": false, "manage_roles": "none", "manage_groups": false, "manage_group_memberships": false, "manage_organizations": false, "manage_suspended_tickets": false, "manage_triggers": false, "manage_team_members": "readonly"}, "team_member_count": 1}, "emitted_at": 1692702313283} -{"stream": "custom_roles", "data": {"id": 360000210616, "name": "Team lead", "description": "Can manage all tickets and forums", "role_type": 0, "created_at": "2020-12-11T18:34:36Z", "updated_at": "2023-06-26T11:06:24Z", "configuration": {"chat_access": true, "end_user_list_access": "full", "forum_access_restricted_content": false, "light_agent": false, "manage_business_rules": true, "manage_dynamic_content": true, "manage_extensions_and_channels": true, "manage_facebook": true, "moderate_forums": false, "side_conversation_create": true, "ticket_access": "all", "ticket_comment_access": "public", "ticket_deletion": true, "ticket_tag_editing": true, "twitter_search_access": false, "view_deleted_tickets": true, "voice_access": true, "group_access": true, "organization_editing": true, "organization_notes_editing": true, "assign_tickets_to_any_group": false, "end_user_profile_access": "full", "explore_access": "edit", "forum_access": "full", "macro_access": "full", "report_access": "full", "ticket_editing": true, "ticket_merge": true, "user_view_access": "full", "view_access": "playonly", "voice_dashboard_access": true, "manage_automations": true, "manage_contextual_workspaces": true, "manage_organization_fields": true, "manage_skills": true, "manage_slas": true, "manage_ticket_fields": true, "manage_ticket_forms": true, "manage_user_fields": true, "ticket_redaction": true, "manage_roles": "all-except-self", "manage_groups": true, "manage_group_memberships": true, "manage_organizations": true, "manage_suspended_tickets": true, "manage_triggers": true, "manage_team_members": "readonly"}, "team_member_count": 2}, "emitted_at": 1692702313284} -{"stream": "schedules", "data": {"id": 4567312249615, "name": "Test Schedule", "time_zone": "New Caledonia", "created_at": "2022-03-25T10:23:34Z", "updated_at": "2022-03-25T10:23:34Z", "intervals": [{"start_time": 1980, "end_time": 2460}, {"start_time": 3420, "end_time": 3900}, {"start_time": 4860, "end_time": 5340}, {"start_time": 6300, "end_time": 6780}, {"start_time": 7740, "end_time": 8220}]}, "emitted_at": 1690888192224} -{"stream": "ticket_forms", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_forms/360000084116.json", "name": "Default Ticket Form", "display_name": "Default Ticket Form", "id": 360000084116, "raw_name": "Default Ticket Form", "raw_display_name": "Default Ticket Form", "end_user_visible": true, "position": 1, "ticket_field_ids": [360002833076, 360002833096, 360002833116, 360002833136, 360002833156, 360002833176, 360002833196], "active": true, "default": true, "created_at": "2020-12-11T18:34:37Z", "updated_at": "2020-12-11T18:34:37Z", "in_all_brands": true, "restricted_brand_ids": [], "end_user_conditions": [], "agent_conditions": []}, "emitted_at": 1690888193249} -{"stream": "account_attributes", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/routing/attributes/ac43b460-0ebd-11ee-85a3-4750db6aa722.json", "id": "ac43b460-0ebd-11ee-85a3-4750db6aa722", "name": "Language", "created_at": "2023-06-19T16:23:49Z", "updated_at": "2023-06-19T16:23:49Z"}, "emitted_at": 1690888194272} -{"stream": "account_attributes", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/routing/attributes/c15cdb76-0ebd-11ee-a37f-f315f48c0150.json", "id": "c15cdb76-0ebd-11ee-a37f-f315f48c0150", "name": "Quality", "created_at": "2023-06-19T16:24:25Z", "updated_at": "2023-06-19T16:24:25Z"}, "emitted_at": 1690888194273} -{"stream": "attribute_definitions", "data": {"title": "Number of incidents", "subject": "number_of_incidents", "type": "text", "group": "ticket", "nullable": false, "repeatable": false, "operators": [{"value": "less_than", "title": "Less than", "terminal": false}, {"value": "greater_than", "title": "Greater than", "terminal": false}, {"value": "is", "title": "Is", "terminal": false}, {"value": "less_than_equal", "title": "Less than or equal to", "terminal": false}, {"value": "greater_than_equal", "title": "Greater than or equal to", "terminal": false}], "condition": "all"}, "emitted_at": 1690888195504} -{"stream": "attribute_definitions", "data": {"title": "Brand", "subject": "brand_id", "type": "list", "group": "ticket", "nullable": false, "repeatable": false, "operators": [{"value": "is", "title": "Is", "terminal": false}, {"value": "is_not", "title": "Is not", "terminal": false}], "values": [{"value": "360000358316", "title": "Airbyte", "enabled": true}], "condition": "all"}, "emitted_at": 1690888195504} -{"stream": "attribute_definitions", "data": {"title": "Form", "subject": "ticket_form_id", "type": "list", "group": "ticket", "nullable": false, "repeatable": false, "operators": [{"value": "is", "title": "Is", "terminal": false}, {"value": "is_not", "title": "Is not", "terminal": false}], "values": [{"value": "360000084116", "title": "Default Ticket Form", "enabled": true}], "condition": "all"}, "emitted_at": 1690888195505} -{"stream":"topics","data":{"id":7253394974479,"url":"https://d3v-airbyte.zendesk.com/api/v2/help_center/community/topics/7253394974479.json","html_url":"https://d3v-airbyte.zendesk.com/hc/en-us/community/topics/7253394974479-Feature-Requests","name":"Feature Requests","description":null,"position":0,"follower_count":1,"community_id":7253391140495,"created_at":"2023-06-22T00:32:21Z","updated_at":"2023-06-22T00:32:21Z","manageable_by":"managers","user_segment_id":null},"emitted_at":1687861697934} -{"stream":"topics","data":{"id":7253351897871,"url":"https://d3v-airbyte.zendesk.com/api/v2/help_center/community/topics/7253351897871.json","html_url":"https://d3v-airbyte.zendesk.com/hc/en-us/community/topics/7253351897871-General-Discussion","name":"General Discussion","description":null,"position":0,"follower_count":1,"community_id":7253391140495,"created_at":"2023-06-22T00:32:20Z","updated_at":"2023-06-22T00:32:20Z","manageable_by":"managers","user_segment_id":null},"emitted_at":1687861697934} +{"stream": "articles", "data": {"id": 7253351877519, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/en-us/articles/7253351877519.json", "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/articles/7253351877519-Sample-article-Stellar-Skyonomy-refund-policies", "author_id": 360786799676, "comments_disabled": false, "draft": true, "promoted": false, "position": 0, "vote_sum": 0, "vote_count": 0, "section_id": 7253394933775, "created_at": "2023-06-22T00:32:20Z", "updated_at": "2023-06-22T00:32:20Z", "name": "Sample article: Stellar Skyonomy refund policies", "title": "Sample article: Stellar Skyonomy refund policies", "source_locale": "en-us", "locale": "en-us", "outdated": false, "outdated_locales": [], "edited_at": "2023-06-22T00:32:20Z", "user_segment_id": null, "permission_group_id": 7253379449487, "content_tag_ids": [], "label_names": [], "body": "

      All Stellar Skyonomy merchandise purchases are backed by our 30-day satisfaction guarantee, no questions asked. We even pay to have it shipped back to us. Additionally, you can cancel your Stellar Skyonomy subscription at any time. Before you cancel, review our refund policies in this article.


      Refund policy

      We automatically issue a full refund when you initiate a return within 30 days of delivery.

      To cancel an annual website subscription you can do so at any time and your refund will be prorated based on the cancellation date.


      Request a refund

      If you believe you\u2019re eligible for a refund but haven\u2019t received one, contact us by completing a refund request form. We review every refund and aim to respond within two business days.

      If you haven't received a refund you're expecting, note that it can take up to 10 business days to appear on your card statement.

      "}, "emitted_at": 1697714809846} +{"stream": "articles", "data": {"id": 7253391134863, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/en-us/articles/7253391134863.json", "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/articles/7253391134863-How-can-agents-leverage-knowledge-to-help-customers-", "author_id": 360786799676, "comments_disabled": false, "draft": false, "promoted": false, "position": 0, "vote_sum": 0, "vote_count": 0, "section_id": 7253394947215, "created_at": "2023-06-22T00:32:20Z", "updated_at": "2023-06-22T00:32:20Z", "name": "How can agents leverage knowledge to help customers?", "title": "How can agents leverage knowledge to help customers?", "source_locale": "en-us", "locale": "en-us", "outdated": false, "outdated_locales": [], "edited_at": "2023-06-22T00:32:20Z", "user_segment_id": null, "permission_group_id": 7253379449487, "content_tag_ids": [], "label_names": [], "body": "

      You can use our Knowledge Capture app to leverage your team\u2019s collective knowledge.

      \n

      Using the app, agents can:\n

        \n
      • Search the Help Center without leaving the ticket
      • \n
      • Insert links to relevant Help Center articles in ticket comments
      • \n
      • Add inline feedback to existing articles that need updates
      • \n
      • Create new articles while answering tickets using a pre-defined template
      • \n
      \n\n\n

      Agents never have to leave the ticket interface to share, flag, or create knowledge, so they can help the customer, while also improving your self-service offerings for other customers.

      \n\n

      To get started, see our Knowledge Capture documentation.

      \n\n

      And before your agents can start creating new knowledge directly from tickets, you\u2019ll need to create a template for them to use. To help you along, we\u2019ve provided some template ideas below. You can copy and paste any sample template below into a new article, add the KCTemplate label to the article, and you\u2019ll be all set.

      \n\n

      Q&A template:

      \n\n
      \n\n

      \n

      \n

      [Title]

      \n\n\n

      \n

      \n

      Question

      \nwrite the question here.\n\n\n

      \n

      \n

      Answer

      \nwrite the answer here.\n\n\n
      \n\n

      Solution template:

      \n\n
      \n\n

      \n

      \n

      [Title]

      \n\n\n

      \n

      \n

      Symptoms

      \nwrite the symptoms here.\n\n\n

      \n

      \n

      Resolution

      \nwrite the resolution here.\n\n\n

      \n

      \n

      Cause

      \nwrite the cause here.\n\n\n
      \n\n

      How-to template:

      \n\n
      \n\n

      \n

      \n

      [Title]

      \n\n\n

      \n

      \n

      Objective

      \nwrite the purpose or task here.\n\n\n

      \n

      \n

      Procedure

      \nwrite the steps here.\n\n\n
      \n"}, "emitted_at": 1697714809848} +{"stream": "articles", "data": {"id": 7253394952591, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/en-us/articles/7253394952591.json", "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/articles/7253394952591-How-do-I-customize-my-Help-Center-", "author_id": 360786799676, "comments_disabled": false, "draft": false, "promoted": false, "position": 0, "vote_sum": 0, "vote_count": 0, "section_id": 7253394947215, "created_at": "2023-06-22T00:32:20Z", "updated_at": "2023-06-22T00:32:20Z", "name": "How do I customize my Help Center?", "title": "How do I customize my Help Center?", "source_locale": "en-us", "locale": "en-us", "outdated": false, "outdated_locales": [], "edited_at": "2023-06-22T00:32:20Z", "user_segment_id": null, "permission_group_id": 7253379449487, "content_tag_ids": [], "label_names": [], "body": "

      You can modify the look and feel of your Help Center by changing colors and fonts. See Branding your Help Center to learn how.

      \n\n

      You can also change the design of your Help Center. If you're comfortable working with page code, you can dig in to the site's HTML, CSS, and Javascript to customize your theme. To get started, see Customizing the Help Center.

      "}, "emitted_at": 1697714809849} +{"stream": "article_comments", "data": {"id": 7253381447311, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/en-us/articles/7253394935055/comments/7253381447311.json", "body": "

      Test comment 2

      ", "author_id": 360786799676, "source_id": 7253394935055, "source_type": "Article", "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/articles/7253394935055/comments/7253381447311", "locale": "en-us", "created_at": "2023-06-22T00:33:36Z", "updated_at": "2023-06-22T00:33:42Z", "vote_sum": -1, "vote_count": 1, "non_author_editor_id": null, "non_author_updated_at": null}, "emitted_at": 1697714814160} +{"stream": "article_comments", "data": {"id": 7253366869647, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/en-us/articles/7253394935055/comments/7253366869647.json", "body": "

      Test comment

      ", "author_id": 360786799676, "source_id": 7253394935055, "source_type": "Article", "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/articles/7253394935055/comments/7253366869647", "locale": "en-us", "created_at": "2023-06-22T00:33:29Z", "updated_at": "2023-06-22T00:33:40Z", "vote_sum": 1, "vote_count": 1, "non_author_editor_id": null, "non_author_updated_at": null}, "emitted_at": 1697714814162} +{"stream": "article_comment_votes", "data": {"id": 7253393200655, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/votes/7253393200655.json", "user_id": 360786799676, "value": -1, "item_id": 7253381447311, "item_type": "Comment", "created_at": "2023-06-22T00:33:42Z", "updated_at": "2023-06-22T00:33:42Z"}, "emitted_at": 1697714823072} +{"stream": "article_comment_votes", "data": {"id": 7253381522703, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/votes/7253381522703.json", "user_id": 360786799676, "value": 1, "item_id": 7253366869647, "item_type": "Comment", "created_at": "2023-06-22T00:33:40Z", "updated_at": "2023-06-22T00:33:40Z"}, "emitted_at": 1697714823501} +{"stream": "article_votes", "data": {"id": 7816935174287, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/votes/7816935174287.json", "user_id": 360786799676, "value": 1, "item_id": 7253394935055, "item_type": "Article", "created_at": "2023-09-04T13:52:38Z", "updated_at": "2023-09-04T13:52:38Z"}, "emitted_at": 1697714827544} +{"stream": "article_votes", "data": {"id": 7816935384335, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/votes/7816935384335.json", "user_id": 360786799676, "value": 1, "item_id": 7253391120527, "item_type": "Article", "created_at": "2023-09-04T13:52:58Z", "updated_at": "2023-09-04T13:52:58Z"}, "emitted_at": 1697714828540} +{"stream": "audit_logs", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/audit_logs/8170722077839.json", "id": 8170722077839, "action_label": "Updated", "actor_id": 360786799676, "source_id": 360786799676, "source_type": "user", "source_label": "Team member: Team Airbyte", "action": "update", "change_description": "Password: Changed", "ip_address": null, "created_at": "2023-10-19T11:20:04Z", "actor_name": "Team Airbyte"}, "emitted_at": 1697714829754} +{"stream": "audit_logs", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/audit_logs/8156156463759.json", "id": 8156156463759, "action_label": "Updated", "actor_id": -1, "source_id": 8156194806799, "source_type": "account_setting", "source_label": "Agent Workspace Auto Activation opt out", "action": "create", "change_description": "Turned on", "ip_address": null, "created_at": "2023-10-17T22:00:13Z", "actor_name": "Zendesk"}, "emitted_at": 1697714829755} +{"stream": "audit_logs", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/audit_logs/8154367957263.json", "id": 8154367957263, "action_label": "Updated", "actor_id": 360786799676, "source_id": 360786799676, "source_type": "user", "source_label": "Team member: Team Airbyte", "action": "update", "change_description": "Password: Changed", "ip_address": null, "created_at": "2023-10-17T14:01:52Z", "actor_name": "Team Airbyte"}, "emitted_at": 1697714829755} +{"stream": "group_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/group_memberships/360007820916.json", "id": 360007820916, "user_id": 360786799676, "group_id": 360003074836, "default": true, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z"}, "emitted_at": 1697714830912} +{"stream": "group_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/group_memberships/360011727976.json", "id": 360011727976, "user_id": 361084605116, "group_id": 360003074836, "default": true, "created_at": "2021-04-23T14:33:11Z", "updated_at": "2021-04-23T14:33:11Z"}, "emitted_at": 1697714830913} +{"stream": "group_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/group_memberships/360011812655.json", "id": 360011812655, "user_id": 361089721035, "group_id": 360003074836, "default": true, "created_at": "2021-04-23T14:34:20Z", "updated_at": "2021-04-23T14:34:20Z"}, "emitted_at": 1697714830914} +{"stream": "groups", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/groups/7282640316815.json", "id": 7282640316815, "is_public": true, "name": "Airbyte Department 1", "description": "A sample department", "default": false, "deleted": false, "created_at": "2023-06-26T10:09:12Z", "updated_at": "2023-06-26T10:09:12Z"}, "emitted_at": 1697714832511} +{"stream": "groups", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/groups/7282618889231.json", "id": 7282618889231, "is_public": true, "name": "Department 1", "description": "A sample department", "default": false, "deleted": false, "created_at": "2023-06-26T10:09:14Z", "updated_at": "2023-06-26T10:09:14Z"}, "emitted_at": 1697714832513} +{"stream": "groups", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/groups/7282630247567.json", "id": 7282630247567, "is_public": true, "name": "Department 2", "description": "A sample department 2", "default": false, "deleted": false, "created_at": "2023-06-26T10:09:14Z", "updated_at": "2023-06-26T10:09:14Z"}, "emitted_at": 1697714832514} +{"stream": "macros", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/macros/360011363556.json", "id": 360011363556, "title": "Customer not responding", "active": true, "updated_at": "2020-12-11T18:34:06Z", "created_at": "2020-12-11T18:34:06Z", "default": false, "position": 9999, "description": null, "actions": [{"field": "status", "value": "pending"}, {"field": "comment_value", "value": "Hello {{ticket.requester.name}}. Our agent {{current_user.name}} has tried to contact you about this request but we haven't heard back from you yet. Please let us know if we can be of further assistance. Thanks. "}], "restriction": null, "raw_title": "Customer not responding"}, "emitted_at": 1697714834209} +{"stream": "macros", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/macros/360011363536.json", "id": 360011363536, "title": "Downgrade and inform", "active": true, "updated_at": "2020-12-11T18:34:06Z", "created_at": "2020-12-11T18:34:06Z", "default": false, "position": 9999, "description": null, "actions": [{"field": "priority", "value": "low"}, {"field": "comment_value", "value": "We're currently experiencing unusually high traffic. We'll get back to you as soon as possible."}], "restriction": null, "raw_title": "Downgrade and inform"}, "emitted_at": 1697714834212} +{"stream": "organizations", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/organizations/360033549136.json", "id": 360033549136, "name": "Airbyte", "shared_tickets": true, "shared_comments": true, "external_id": null, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2023-04-13T14:51:21Z", "domain_names": ["cloud.airbyte.com"], "details": "test", "notes": "test", "group_id": 6770788212111, "tags": ["test"], "organization_fields": {"test_check_box_field_1": false, "test_drop_down_field_1": null, "test_number_field_1": null}, "deleted_at": null}, "emitted_at": 1697714835264} +{"stream": "organization_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/organization_fields/7376684398223.json", "id": 7376684398223, "type": "dropdown", "key": "test_drop_down_field_1", "title": "Test Drop-Down field 1", "description": "Description for a Test Drop-Down field", "raw_title": "Test Drop-Down field 1", "raw_description": "Description for a Test Drop-Down field", "position": 0, "active": true, "system": false, "regexp_for_validation": null, "created_at": "2023-07-10T08:35:43Z", "updated_at": "2023-07-10T08:35:43Z", "custom_field_options": [{"id": 7376695621007, "name": "Test 1", "raw_name": "Test 1", "value": "test_1"}, {"id": 7376695621135, "name": "Test 2", "raw_name": "Test 2", "value": "test_2"}, {"id": 7376695621263, "name": "12", "raw_name": "12", "value": "12"}, {"id": 7376695621391, "name": "154", "raw_name": "154", "value": "154"}]}, "emitted_at": 1697714836208} +{"stream": "organization_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/organization_fields/7376684841999.json", "id": 7376684841999, "type": "integer", "key": "test_number_field_1", "title": "Test Number field 1", "description": "Description for a Test Number field", "raw_title": "Test Number field 1", "raw_description": "Description for a Test Number field", "position": 1, "active": true, "system": false, "regexp_for_validation": null, "created_at": "2023-07-10T08:36:13Z", "updated_at": "2023-07-10T08:36:13Z"}, "emitted_at": 1697714836211} +{"stream": "organization_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/organization_fields/7376673274511.json", "id": 7376673274511, "type": "checkbox", "key": "test_check_box_field_1", "title": "Test Check box field 1", "description": "Description for a Test Check box field", "raw_title": "Test Check box field 1", "raw_description": "Description for a Test Check box field", "position": 2, "active": true, "system": false, "regexp_for_validation": null, "created_at": "2023-07-10T08:36:58Z", "updated_at": "2023-07-10T08:36:58Z", "tag": "check_box_1"}, "emitted_at": 1697714836211} +{"stream": "organization_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/organization_memberships/360057705196.json", "id": 360057705196, "user_id": 360786799676, "organization_id": 360033549136, "default": true, "created_at": "2020-12-11T18:34:05Z", "organization_name": "Airbyte", "updated_at": "2020-12-11T18:34:05Z", "view_tickets": true}, "emitted_at": 1697714837426} +{"stream": "organization_memberships", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/organization_memberships/7282880134671.json", "id": 7282880134671, "user_id": 7282634891791, "organization_id": 360033549136, "default": true, "created_at": "2023-06-26T11:03:38Z", "organization_name": "Airbyte", "updated_at": "2023-06-26T11:03:38Z", "view_tickets": true}, "emitted_at": 1697714837428} +{"stream": "posts", "data": {"id": 7253351904271, "title": "How do I get around the community?", "details": "

      You can use search to find answers. You can also browse topics and posts using views and filters. See Getting around the community.

      ", "author_id": 360786799676, "vote_sum": 0, "vote_count": 0, "comment_count": 0, "follower_count": 0, "topic_id": 7253351897871, "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/community/posts/7253351904271-How-do-I-get-around-the-community-", "created_at": "2023-06-22T00:32:21Z", "updated_at": "2023-06-22T00:32:21Z", "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/posts/7253351904271-How-do-I-get-around-the-community-.json", "featured": false, "pinned": false, "closed": false, "frozen": false, "status": "none", "non_author_editor_id": null, "non_author_updated_at": null, "content_tag_ids": []}, "emitted_at": 1697714838032} +{"stream": "posts", "data": {"id": 7253375870607, "title": "Which topics should I add to my community?", "details": "

      That depends. If you support several products, you might add a topic for each product. If you have one big product, you might add a topic for each major feature area or task. If you have different types of users (for example, end users and API developers), you might add a topic or topics for each type of user.

      A General Discussion topic is a place for users to discuss issues that don't quite fit in the other topics. You could monitor this topic for emerging issues that might need their own topics.

      \n\n

      To create your own topics, see Adding community discussion topics.

      ", "author_id": 360786799676, "vote_sum": 0, "vote_count": 0, "comment_count": 0, "follower_count": 0, "topic_id": 7253351897871, "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/community/posts/7253375870607-Which-topics-should-I-add-to-my-community-", "created_at": "2023-06-22T00:32:21Z", "updated_at": "2023-06-22T00:32:21Z", "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/posts/7253375870607-Which-topics-should-I-add-to-my-community-.json", "featured": false, "pinned": false, "closed": false, "frozen": false, "status": "none", "non_author_editor_id": null, "non_author_updated_at": null, "content_tag_ids": []}, "emitted_at": 1697714838034} +{"stream": "posts", "data": {"id": 7253375879055, "title": "I'd like a way for users to submit feature requests", "details": "

      You can add a topic like this one in your community. End users can add feature requests and describe their use cases. Other users can comment on the requests and vote for them. Product managers can review feature requests and provide feedback.

      ", "author_id": 360786799676, "vote_sum": 0, "vote_count": 0, "comment_count": 0, "follower_count": 0, "topic_id": 7253394974479, "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/community/posts/7253375879055-I-d-like-a-way-for-users-to-submit-feature-requests", "created_at": "2023-06-22T00:32:21Z", "updated_at": "2023-06-22T00:32:21Z", "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/posts/7253375879055-I-d-like-a-way-for-users-to-submit-feature-requests.json", "featured": false, "pinned": false, "closed": false, "frozen": false, "status": "none", "non_author_editor_id": null, "non_author_updated_at": null, "content_tag_ids": []}, "emitted_at": 1697714838034} +{"stream": "satisfaction_ratings", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/satisfaction_ratings/7235633102607.json", "id": 7235633102607, "assignee_id": null, "group_id": null, "requester_id": 361089721035, "ticket_id": 146, "score": "offered", "created_at": "2023-06-19T18:01:40Z", "updated_at": "2023-06-19T18:01:40Z", "comment": null}, "emitted_at": 1697714848277} +{"stream": "satisfaction_ratings", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/satisfaction_ratings/5909514818319.json", "id": 5909514818319, "assignee_id": null, "group_id": null, "requester_id": 360786799676, "ticket_id": 25, "score": "offered", "created_at": "2022-11-22T17:02:04Z", "updated_at": "2022-11-22T17:02:04Z", "comment": null}, "emitted_at": 1697714848279} +{"stream": "satisfaction_ratings", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/satisfaction_ratings/5527212710799.json", "id": 5527212710799, "assignee_id": null, "group_id": null, "requester_id": 5527080499599, "ticket_id": 144, "score": "offered", "created_at": "2022-09-19T16:01:43Z", "updated_at": "2022-09-19T16:01:43Z", "comment": null}, "emitted_at": 1697714848279} +{"stream": "sla_policies", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/slas/policies/360001110696.json", "id": 360001110696, "title": "test police", "description": "for tests", "position": 1, "filter": {"all": [{"field": "assignee_id", "operator": "is", "value": 361089721035}], "any": []}, "policy_metrics": [{"priority": "high", "metric": "first_reply_time", "target": 61, "business_hours": false}], "created_at": "2021-07-16T11:05:31Z", "updated_at": "2021-07-16T11:05:31Z"}, "emitted_at": 1697714849344} +{"stream": "sla_policies", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/slas/policies/360001113715.json", "id": 360001113715, "title": "test police 2", "description": "test police 2", "position": 2, "filter": {"all": [{"field": "organization_id", "operator": "is", "value": 360033549136}], "any": []}, "policy_metrics": [{"priority": "high", "metric": "first_reply_time", "target": 121, "business_hours": false}], "created_at": "2021-07-16T11:06:01Z", "updated_at": "2021-07-16T11:06:01Z"}, "emitted_at": 1697714849345} +{"stream": "ticket_audits", "data": {"id": 8154471428239, "ticket_id": 154, "created_at": "2023-10-17T14:28:25Z", "author_id": -1, "metadata": {"system": {}, "custom": {}}, "events": [{"id": 8154455112079, "type": "Notification", "via": {"channel": "rule", "source": {"from": {"deleted": false, "title": "Notify assignee of comment update", "id": 360011363236, "revision_id": 1}, "rel": "trigger"}}, "subject": "[{{ticket.account}}] Re: {{ticket.title}}", "body": "This ticket (#{{ticket.id}}) has been updated.\n\n{{ticket.comments_formatted}}", "recipients": [360786799676]}, {"id": 8154459400847, "type": "ChatEndedEvent", "value": {"chat_id": "2310.10414779.TsxCzMf0jqtMS", "chat_started_event_id": 8154502502927, "visitor_id": "10414779-1INmCdTWKtlu4ct", "is_served": true, "tags": ["business_messaging_slack_connect_chat"]}, "attachments": []}, {"id": 8154474154895, "type": "Comment", "author_id": -1, "body": "(01:28:24) Team Airbyte: Nope", "html_body": "

      (01:28:24) Team Airbyte: Nope

      ", "plain_body": "(01:28:24) Team Airbyte: Nope", "public": true, "attachments": [], "audit_id": 8154471428239}], "via": {"channel": "chat_transcript", "source": {"from": {}, "to": {}, "rel": null}}}, "emitted_at": 1697714855596} +{"stream": "ticket_audits", "data": {"id": 8154455109135, "ticket_id": 154, "created_at": "2023-10-17T14:28:25Z", "author_id": -1, "metadata": {"system": {}, "custom": {}}, "events": [{"id": 8154502502927, "type": "ChatStartedEvent", "value": {"chat_id": "2310.10414779.TsxCzMf0jqtMS", "visitor_id": "10414779-1INmCdTWKtlu4ct", "conversation_id": "dc3b91c917014c6095d8123a", "user_id": 8154469488271, "authenticated": false, "tags": ["business_messaging_slack_connect_chat"], "initiator": 2, "backend": "chat", "history": [{"actor_name": "Erica D'Souza", "timestamp": 1697552904528, "chat_index": 0, "actor_type": "end-user", "actor_id": "8154469488271", "type": "ChatJoin"}, {"actor_name": "Team Airbyte", "timestamp": 1697552904538, "chat_index": 1, "actor_type": "agent", "actor_id": "360786799676", "type": "ChatJoin"}, {"actor_name": "Team Airbyte", "timestamp": 1697552904542, "chat_index": 2, "actor_type": "agent", "actor_id": "360786799676", "type": "ChatMessage", "message": "Nope", "message_id": "6daa2df0-6cf9-11ee-9286-1500c615ce3a"}, {"actor_name": "Erica D'Souza", "timestamp": 1697552905076, "chat_index": 3, "actor_type": "end-user", "actor_id": "8154469488271", "type": "ChatMessageStatus", "status": "SEND_SUCCESS", "status_ts": 1697552904542, "parent_message_id": "2310.10414779.TsxCzMf0jqtMS_2", "external_message_id": "652e9a092e0208aaf19929fd"}, {"actor_name": "Erica D'Souza", "timestamp": 1697552905301, "chat_index": 4, "actor_type": "end-user", "actor_id": "8154469488271", "type": "ChatLeave", "reason": "agent_workspace_ticket_status_initiate_chat_end"}], "webpath": [], "channel": "business_messaging_slack_connect"}, "attachments": []}], "via": {"channel": "business_messaging_slack_connect", "source": {"from": {}, "to": {}, "rel": null}}}, "emitted_at": 1697714855600} +{"stream": "ticket_audits", "data": {"id": 8154502294415, "ticket_id": 154, "created_at": "2023-10-17T14:27:53Z", "author_id": -1, "metadata": {"system": {}, "custom": {}}, "events": [{"id": 8154502294543, "type": "ChatEndedEvent", "value": {"chat_id": "2310.10414779.TsxC6Q5zdSu1f", "chat_started_event_id": 8154438755727, "visitor_id": "10414779-1INmCdTWKtlu4ct", "is_served": false, "tags": ["business_messaging_slack_connect_chat"]}, "attachments": []}, {"id": 8154502294671, "type": "Comment", "author_id": -1, "body": "(01:24:53) Erica D'Souza: what is the <#C061EKCHEJZ|zendesk-chat-integration-test> channel for?\n(01:25:39) Erica D'Souza: for what purpose?\n(01:26:45) Erica D'Souza: ahh gotcha, was just making sure no one is asking for ZD chat lol", "html_body": "

      (01:24:53) Erica D'Souza: what is the <#C061EKCHEJZ|zendesk-chat-integration-test> channel for?\n
      (01:25:39) Erica D'Souza: for what purpose?\n
      (01:26:45) Erica D'Souza: ahh gotcha, was just making sure no one is asking for ZD chat lol

      ", "plain_body": "(01:24:53) Erica D'Souza: what is the <#C061EKCHEJZ|zendesk-chat-integration-test> channel for?\n\n(01:25:39) Erica D'Souza: for what purpose?\n\n(01:26:45) Erica D'Souza: ahh gotcha, was just making sure no one is asking for ZD chat lol", "public": true, "attachments": [], "audit_id": 8154502294415}, {"id": 8154502294799, "type": "Change", "value": "1", "field_name": "is_public", "previous_value": "0"}, {"id": 8154502294927, "type": "Notification", "via": {"channel": "rule", "source": {"from": {"deleted": false, "title": "Notify assignee of comment update", "id": 360011363236, "revision_id": 1}, "rel": "trigger"}}, "subject": "[{{ticket.account}}] Re: {{ticket.title}}", "body": "This ticket (#{{ticket.id}}) has been updated.\n\n{{ticket.comments_formatted}}", "recipients": [360786799676]}], "via": {"channel": "chat_transcript", "source": {"from": {}, "to": {}, "rel": null}}}, "emitted_at": 1697714855601} +{"stream": "ticket_comments", "data": {"id": 5162146653071, "via": {"channel": "web", "source": {"from": {}, "to": {"name": "Team Airbyte", "address": "integration-test@airbyte.io"}, "rel": null}}, "via_reference_id": null, "type": "Comment", "author_id": 360786799676, "body": " 163748", "html_body": "
       163748
      ", "plain_body": " 163748", "public": true, "attachments": [], "audit_id": 5162146652943, "created_at": "2022-07-18T09:58:23Z", "event_type": "Comment", "ticket_id": 124, "timestamp": 1658138303}, "emitted_at": 1697714859038} +{"stream": "ticket_comments", "data": {"id": 5162208963983, "via": {"channel": "web", "source": {"from": {}, "to": {}, "rel": null}}, "via_reference_id": null, "type": "Comment", "author_id": 360786799676, "body": "238473846", "html_body": "
      238473846
      ", "plain_body": "238473846", "public": false, "attachments": [], "audit_id": 5162208963855, "created_at": "2022-07-18T10:16:53Z", "event_type": "Comment", "ticket_id": 125, "timestamp": 1658139413}, "emitted_at": 1697714859039} +{"stream": "ticket_comments", "data": {"id": 5162223308559, "via": {"channel": "web", "source": {"from": {}, "to": {}, "rel": null}}, "via_reference_id": null, "type": "Comment", "author_id": 360786799676, "body": "Airbyte", "html_body": "", "plain_body": "Airbyte", "public": false, "attachments": [], "audit_id": 5162223308431, "created_at": "2022-07-18T10:25:21Z", "event_type": "Comment", "ticket_id": 125, "timestamp": 1658139921}, "emitted_at": 1697714859040} +{"stream": "ticket_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_fields/360002833076.json", "id": 360002833076, "type": "subject", "title": "Subject", "raw_title": "Subject", "description": "", "raw_description": "", "position": 1, "active": true, "required": false, "collapsed_for_agents": false, "regexp_for_validation": null, "title_in_portal": "Subject", "raw_title_in_portal": "Subject", "visible_in_portal": true, "editable_in_portal": true, "required_in_portal": true, "tag": null, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z", "removable": false, "key": null, "agent_description": null}, "emitted_at": 1697714860081} +{"stream": "ticket_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_fields/360002833096.json", "id": 360002833096, "type": "description", "title": "Description", "raw_title": "Description", "description": "Please enter the details of your request. A member of our support staff will respond as soon as possible.", "raw_description": "Please enter the details of your request. A member of our support staff will respond as soon as possible.", "position": 2, "active": true, "required": false, "collapsed_for_agents": false, "regexp_for_validation": null, "title_in_portal": "Description", "raw_title_in_portal": "Description", "visible_in_portal": true, "editable_in_portal": true, "required_in_portal": true, "tag": null, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z", "removable": false, "key": null, "agent_description": null}, "emitted_at": 1697714860083} +{"stream": "ticket_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_fields/360002833116.json", "id": 360002833116, "type": "status", "title": "Status", "raw_title": "Status", "description": "Request status", "raw_description": "Request status", "position": 3, "active": true, "required": false, "collapsed_for_agents": false, "regexp_for_validation": null, "title_in_portal": "Status", "raw_title_in_portal": "Status", "visible_in_portal": false, "editable_in_portal": false, "required_in_portal": false, "tag": null, "created_at": "2020-12-11T18:34:05Z", "updated_at": "2020-12-11T18:34:05Z", "removable": false, "key": null, "agent_description": null, "system_field_options": [{"name": "Open", "value": "open"}, {"name": "Pending", "value": "pending"}, {"name": "Solved", "value": "solved"}], "sub_type_id": 0}, "emitted_at": 1697714860085} +{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/8154457562767.json", "id": 8154457562767, "ticket_id": 154, "created_at": "2023-10-17T14:24:53Z", "updated_at": "2023-10-17T14:28:25Z", "group_stations": 2, "assignee_stations": 1, "reopens": 0, "replies": 1, "assignee_updated_at": "2023-10-17T14:27:52Z", "requester_updated_at": "2023-10-17T14:26:45Z", "status_updated_at": "2023-11-06T15:01:40Z", "initially_assigned_at": "2023-10-17T14:26:33Z", "assigned_at": "2023-10-17T14:26:33Z", "solved_at": "2023-10-17T14:27:52Z", "latest_comment_added_at": "2023-10-17T14:28:25Z", "reply_time_in_minutes": {"calendar": 4, "business": 0}, "first_resolution_time_in_minutes": {"calendar": 3, "business": 0}, "full_resolution_time_in_minutes": {"calendar": 3, "business": 0}, "agent_wait_time_in_minutes": {"calendar": 0, "business": 0}, "requester_wait_time_in_minutes": {"calendar": 3, "business": 0}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "reply_time_in_seconds": {"calendar": 212}, "custom_status_updated_at": "2023-10-17T14:27:52Z"}, "emitted_at": 1699646404810} +{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7283000498191.json", "id": 7283000498191, "ticket_id": 153, "created_at": "2023-06-26T11:31:48Z", "updated_at": "2023-06-26T12:13:42Z", "group_stations": 2, "assignee_stations": 2, "reopens": 0, "replies": 0, "assignee_updated_at": "2023-06-26T11:31:48Z", "requester_updated_at": "2023-06-26T11:31:48Z", "status_updated_at": "2023-06-26T11:31:48Z", "initially_assigned_at": "2023-06-26T11:31:48Z", "assigned_at": "2023-06-26T12:13:42Z", "solved_at": null, "latest_comment_added_at": "2023-06-26T11:31:48Z", "reply_time_in_minutes": {"calendar": null, "business": null}, "first_resolution_time_in_minutes": {"calendar": null, "business": null}, "full_resolution_time_in_minutes": {"calendar": null, "business": null}, "agent_wait_time_in_minutes": {"calendar": null, "business": null}, "requester_wait_time_in_minutes": {"calendar": null, "business": null}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "custom_status_updated_at": "2023-06-26T11:31:48Z"}, "emitted_at": 1699646404810} +{"stream": "ticket_metrics", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_metrics/7282901696015.json", "id": 7282901696015, "ticket_id": 151, "created_at": "2023-06-26T11:09:33Z", "updated_at": "2023-06-26T12:03:38Z", "group_stations": 1, "assignee_stations": 1, "reopens": 0, "replies": 1, "assignee_updated_at": "2023-06-26T12:03:37Z", "requester_updated_at": "2023-06-26T11:09:33Z", "status_updated_at": "2023-06-26T11:09:33Z", "initially_assigned_at": "2023-06-26T11:09:33Z", "assigned_at": "2023-06-26T11:09:33Z", "solved_at": null, "latest_comment_added_at": "2023-06-26T12:03:37Z", "reply_time_in_minutes": {"calendar": 54, "business": 0}, "first_resolution_time_in_minutes": {"calendar": null, "business": null}, "full_resolution_time_in_minutes": {"calendar": null, "business": null}, "agent_wait_time_in_minutes": {"calendar": null, "business": null}, "requester_wait_time_in_minutes": {"calendar": null, "business": null}, "on_hold_time_in_minutes": {"calendar": 0, "business": 0}, "custom_status_updated_at": "2023-06-26T11:09:33Z"}, "emitted_at": 1700040843806} +{"stream": "ticket_metric_events", "data": {"id": 4992797383183, "ticket_id": 121, "metric": "agent_work_time", "instance_id": 0, "type": "measure", "time": "2022-06-17T14:49:20Z"}, "emitted_at": 1697714863384} +{"stream": "ticket_metric_events", "data": {"id": 4992797383311, "ticket_id": 121, "metric": "pausable_update_time", "instance_id": 0, "type": "measure", "time": "2022-06-17T14:49:20Z"}, "emitted_at": 1697714863386} +{"stream": "ticket_metric_events", "data": {"id": 4992797383439, "ticket_id": 121, "metric": "reply_time", "instance_id": 0, "type": "measure", "time": "2022-06-17T14:49:20Z"}, "emitted_at": 1697714863386} +{"stream": "ticket_skips", "data": {"id": 7290033348623, "ticket_id": 121, "user_id": 360786799676, "reason": "I have no idea.", "created_at": "2023-06-27T08:24:02Z", "updated_at": "2023-06-27T08:24:02Z", "ticket": {"url": "https://d3v-airbyte.zendesk.com/api/v2/tickets/121.json", "id": 121, "external_id": null, "via": {"channel": "voice", "source": {"rel": "voicemail", "from": {"formatted_phone": "+1 (689) 689-8023", "phone": "+16896898023", "name": "Caller +1 (689) 689-8023"}, "to": {"formatted_phone": "+1 (205) 953-1462", "phone": "+12059531462", "name": "Airbyte", "brand_id": 360000358316}}}, "created_at": "2022-06-17T14:49:20Z", "updated_at": "2022-06-17T16:01:42Z", "type": null, "subject": "Voicemail from: Caller +1 (689) 689-8023", "raw_subject": "Voicemail from: Caller +1 (689) 689-8023", "description": "Call from: +1 (689) 689-8023\\nTime of call: June 17, 2022 at 2:48:27 PM", "priority": null, "status": "new", "recipient": null, "requester_id": 4992781783439, "submitter_id": 4992781783439, "assignee_id": null, "organization_id": null, "group_id": null, "collaborator_ids": [], "follower_ids": [], "email_cc_ids": [], "forum_topic_id": null, "problem_id": null, "has_incidents": false, "is_public": false, "due_at": null, "tags": [], "custom_fields": [], "satisfaction_rating": {"score": "offered"}, "sharing_agreement_ids": [], "custom_status_id": 4044356, "fields": [], "followup_ids": [], "ticket_form_id": 360000084116, "deleted_ticket_form_id": null, "brand_id": 360000358316, "allow_channelback": false, "allow_attachments": true, "from_messaging_channel": false}}, "emitted_at": 1697714864517} +{"stream": "ticket_skips", "data": {"id": 7290088475023, "ticket_id": 125, "user_id": 360786799676, "reason": "Another test skip.", "created_at": "2023-06-27T08:30:01Z", "updated_at": "2023-06-27T08:30:01Z", "ticket": {"url": "https://d3v-airbyte.zendesk.com/api/v2/tickets/125.json", "id": 125, "external_id": null, "via": {"channel": "web", "source": {"from": {}, "to": {}, "rel": null}}, "created_at": "2022-07-18T10:16:53Z", "updated_at": "2022-07-18T10:36:02Z", "type": "question", "subject": "Ticket Test 2", "raw_subject": "Ticket Test 2", "description": "238473846", "priority": "urgent", "status": "open", "recipient": null, "requester_id": 360786799676, "submitter_id": 360786799676, "assignee_id": 361089721035, "organization_id": 360033549136, "group_id": 5059439464079, "collaborator_ids": [360786799676], "follower_ids": [360786799676], "email_cc_ids": [], "forum_topic_id": null, "problem_id": null, "has_incidents": false, "is_public": false, "due_at": null, "tags": [], "custom_fields": [], "satisfaction_rating": {"score": "unoffered"}, "sharing_agreement_ids": [], "custom_status_id": 4044376, "fields": [], "followup_ids": [], "ticket_form_id": 360000084116, "deleted_ticket_form_id": null, "brand_id": 360000358316, "allow_channelback": false, "allow_attachments": true, "from_messaging_channel": false}}, "emitted_at": 1697714864519} +{"stream": "tickets", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/tickets/121.json", "id": 121, "external_id": null, "via": {"channel": "voice", "source": {"rel": "voicemail", "from": {"formatted_phone": "+1 (689) 689-8023", "phone": "+16896898023", "name": "Caller +1 (689) 689-8023"}, "to": {"formatted_phone": "+1 (205) 953-1462", "phone": "+12059531462", "name": "Airbyte", "brand_id": 360000358316}}}, "created_at": "2022-06-17T14:49:20Z", "updated_at": "2022-06-17T16:01:42Z", "type": null, "subject": "Voicemail from: Caller +1 (689) 689-8023", "raw_subject": "Voicemail from: Caller +1 (689) 689-8023", "description": "Call from: +1 (689) 689-8023\\nTime of call: June 17, 2022 at 2:48:27 PM", "priority": null, "status": "new", "recipient": null, "requester_id": 4992781783439, "submitter_id": 4992781783439, "assignee_id": null, "organization_id": null, "group_id": null, "collaborator_ids": [], "follower_ids": [], "email_cc_ids": [], "forum_topic_id": null, "problem_id": null, "has_incidents": false, "is_public": false, "due_at": null, "tags": [], "custom_fields": [], "satisfaction_rating": {"score": "offered"}, "sharing_agreement_ids": [], "custom_status_id": 4044356, "fields": [], "followup_ids": [], "ticket_form_id": 360000084116, "brand_id": 360000358316, "allow_channelback": false, "allow_attachments": true, "from_messaging_channel": false, "generated_timestamp": 1655481702}, "emitted_at": 1697714865818} +{"stream": "tickets", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/tickets/122.json", "id": 122, "external_id": null, "via": {"channel": "voice", "source": {"rel": "voicemail", "from": {"formatted_phone": "+1 (912) 420-0314", "phone": "+19124200314", "name": "Caller +1 (912) 420-0314"}, "to": {"formatted_phone": "+1 (205) 953-1462", "phone": "+12059531462", "name": "Airbyte", "brand_id": 360000358316}}}, "created_at": "2022-06-17T19:52:39Z", "updated_at": "2022-06-17T21:01:41Z", "type": null, "subject": "Voicemail from: Caller +1 (912) 420-0314", "raw_subject": "Voicemail from: Caller +1 (912) 420-0314", "description": "Call from: +1 (912) 420-0314\\nTime of call: June 17, 2022 at 7:52:02 PM", "priority": null, "status": "new", "recipient": null, "requester_id": 4993467856015, "submitter_id": 4993467856015, "assignee_id": null, "organization_id": null, "group_id": null, "collaborator_ids": [], "follower_ids": [], "email_cc_ids": [], "forum_topic_id": null, "problem_id": null, "has_incidents": false, "is_public": false, "due_at": null, "tags": [], "custom_fields": [], "satisfaction_rating": {"score": "offered"}, "sharing_agreement_ids": [], "custom_status_id": 4044356, "fields": [], "followup_ids": [], "ticket_form_id": 360000084116, "brand_id": 360000358316, "allow_channelback": false, "allow_attachments": true, "from_messaging_channel": false, "generated_timestamp": 1655499701}, "emitted_at": 1697714865822} +{"stream": "tickets", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/tickets/125.json", "id": 125, "external_id": null, "via": {"channel": "web", "source": {"from": {}, "to": {}, "rel": null}}, "created_at": "2022-07-18T10:16:53Z", "updated_at": "2022-07-18T10:36:02Z", "type": "question", "subject": "Ticket Test 2", "raw_subject": "Ticket Test 2", "description": "238473846", "priority": "urgent", "status": "open", "recipient": null, "requester_id": 360786799676, "submitter_id": 360786799676, "assignee_id": 361089721035, "organization_id": 360033549136, "group_id": 5059439464079, "collaborator_ids": [360786799676], "follower_ids": [360786799676], "email_cc_ids": [], "forum_topic_id": null, "problem_id": null, "has_incidents": false, "is_public": false, "due_at": null, "tags": [], "custom_fields": [], "satisfaction_rating": {"score": "unoffered"}, "sharing_agreement_ids": [], "custom_status_id": 4044376, "fields": [], "followup_ids": [], "ticket_form_id": 360000084116, "brand_id": 360000358316, "allow_channelback": false, "allow_attachments": true, "from_messaging_channel": false, "generated_timestamp": 1658140562}, "emitted_at": 1697714865824} +{"stream": "topics", "data": {"id": 7253394974479, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/topics/7253394974479.json", "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/community/topics/7253394974479-Feature-Requests", "name": "Feature Requests", "description": null, "position": 0, "follower_count": 1, "community_id": 7253391140495, "created_at": "2023-06-22T00:32:21Z", "updated_at": "2023-06-22T00:32:21Z", "manageable_by": "managers", "user_segment_id": null}, "emitted_at": 1697714866838} +{"stream": "topics", "data": {"id": 7253351897871, "url": "https://d3v-airbyte.zendesk.com/api/v2/help_center/community/topics/7253351897871.json", "html_url": "https://d3v-airbyte.zendesk.com/hc/en-us/community/topics/7253351897871-General-Discussion", "name": "General Discussion", "description": null, "position": 0, "follower_count": 1, "community_id": 7253391140495, "created_at": "2023-06-22T00:32:20Z", "updated_at": "2023-06-22T00:32:20Z", "manageable_by": "managers", "user_segment_id": null}, "emitted_at": 1697714866839} +{"stream":"users","data":{"id":4992781783439,"url":"https://d3v-airbyte.zendesk.com/api/v2/users/4992781783439.json","name":"Caller +1 (689) 689-8023","email":null,"created_at":"2022-06-17T14:49:19Z","updated_at":"2022-06-17T14:49:19Z","time_zone":"Pacific/Noumea","iana_time_zone":"Pacific/Noumea","phone":"+16896898023","shared_phone_number":false,"photo":null,"locale_id":1,"locale":"en-US","organization_id":null,"role":"end-user","verified":true,"external_id":null,"tags":[],"alias":null,"active":true,"shared":false,"shared_agent":false,"last_login_at":null,"two_factor_auth_enabled":null,"signature":null,"details":null,"notes":null,"role_type":null,"custom_role_id":null,"moderator":false,"ticket_restriction":"requested","only_private_comments":false,"restricted_agent":true,"suspended":false,"default_group_id":null,"report_csv":false,"user_fields":{"test_display_name_checkbox_field":false,"test_display_name_decimal_field":null,"test_display_name_text_field":null}},"emitted_at":1704976960493} +{"stream":"users","data":{"id":4993467856015,"url":"https://d3v-airbyte.zendesk.com/api/v2/users/4993467856015.json","name":"Caller +1 (912) 420-0314","email":null,"created_at":"2022-06-17T19:52:38Z","updated_at":"2022-06-17T19:52:38Z","time_zone":"Pacific/Noumea","iana_time_zone":"Pacific/Noumea","phone":"+19124200314","shared_phone_number":false,"photo":null,"locale_id":1,"locale":"en-US","organization_id":null,"role":"end-user","verified":true,"external_id":null,"tags":[],"alias":null,"active":true,"shared":false,"shared_agent":false,"last_login_at":null,"two_factor_auth_enabled":null,"signature":null,"details":null,"notes":null,"role_type":null,"custom_role_id":null,"moderator":false,"ticket_restriction":"requested","only_private_comments":false,"restricted_agent":true,"suspended":false,"default_group_id":null,"report_csv":false,"user_fields":{"test_display_name_checkbox_field":false,"test_display_name_decimal_field":null,"test_display_name_text_field":null}},"emitted_at":1704976960494} +{"stream":"users","data":{"id":5137812260495,"url":"https://d3v-airbyte.zendesk.com/api/v2/users/5137812260495.json","name":"Caller +1 (607) 210-9549","email":null,"created_at":"2022-07-13T14:34:04Z","updated_at":"2022-07-13T14:34:04Z","time_zone":"Pacific/Noumea","iana_time_zone":"Pacific/Noumea","phone":"+16072109549","shared_phone_number":false,"photo":null,"locale_id":1,"locale":"en-US","organization_id":null,"role":"end-user","verified":true,"external_id":null,"tags":[],"alias":null,"active":true,"shared":false,"shared_agent":false,"last_login_at":null,"two_factor_auth_enabled":null,"signature":null,"details":null,"notes":null,"role_type":null,"custom_role_id":null,"moderator":false,"ticket_restriction":"requested","only_private_comments":false,"restricted_agent":true,"suspended":false,"default_group_id":null,"report_csv":false,"user_fields":{"test_display_name_checkbox_field":false,"test_display_name_decimal_field":null,"test_display_name_text_field":null}},"emitted_at":1704976960494} +{"stream": "brands", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/brands/360000358316.json", "id": 360000358316, "name": "Airbyte", "brand_url": "https://d3v-airbyte.zendesk.com", "subdomain": "d3v-airbyte", "host_mapping": null, "has_help_center": true, "help_center_state": "enabled", "active": true, "default": true, "is_deleted": false, "logo": null, "ticket_form_ids": [360000084116], "signature_template": "{{agent.signature}}", "created_at": "2020-12-11T18:34:04Z", "updated_at": "2020-12-11T18:34:09Z"}, "emitted_at": 1697714873604} +{"stream": "custom_roles", "data": {"id": 360000210636, "name": "Advisor", "description": "Can automate ticket workflows, manage channels and make private comments on tickets", "role_type": 0, "created_at": "2020-12-11T18:34:36Z", "updated_at": "2020-12-11T18:34:36Z", "configuration": {"chat_access": true, "end_user_list_access": "full", "forum_access_restricted_content": false, "light_agent": false, "manage_business_rules": true, "manage_dynamic_content": false, "manage_extensions_and_channels": true, "manage_facebook": true, "moderate_forums": false, "side_conversation_create": true, "ticket_access": "within-groups", "ticket_comment_access": "none", "ticket_deletion": false, "ticket_tag_editing": true, "twitter_search_access": false, "view_deleted_tickets": false, "voice_access": true, "group_access": false, "organization_editing": false, "organization_notes_editing": false, "assign_tickets_to_any_group": false, "end_user_profile_access": "readonly", "explore_access": "readonly", "forum_access": "readonly", "macro_access": "full", "report_access": "none", "ticket_editing": true, "ticket_merge": false, "user_view_access": "full", "view_access": "full", "voice_dashboard_access": false, "manage_automations": true, "manage_contextual_workspaces": false, "manage_organization_fields": false, "manage_skills": true, "manage_slas": true, "manage_ticket_fields": false, "manage_ticket_forms": false, "manage_user_fields": false, "ticket_redaction": false, "manage_roles": "none", "manage_groups": false, "manage_group_memberships": false, "manage_organizations": false, "manage_suspended_tickets": false, "manage_triggers": true, "view_reduced_count": false, "view_filter_tickets": true, "manage_macro_content_suggestions": false, "read_macro_content_suggestions": false, "custom_objects": {}}, "team_member_count": 1}, "emitted_at": 1698749854337} +{"stream": "custom_roles", "data": {"id": 360000210596, "name": "Staff", "description": "Can edit tickets within their groups", "role_type": 0, "created_at": "2020-12-11T18:34:36Z", "updated_at": "2020-12-11T18:34:36Z", "configuration": {"chat_access": true, "end_user_list_access": "full", "forum_access_restricted_content": false, "light_agent": false, "manage_business_rules": false, "manage_dynamic_content": false, "manage_extensions_and_channels": false, "manage_facebook": false, "moderate_forums": false, "side_conversation_create": true, "ticket_access": "within-groups", "ticket_comment_access": "public", "ticket_deletion": false, "ticket_tag_editing": false, "twitter_search_access": false, "view_deleted_tickets": false, "voice_access": true, "group_access": false, "organization_editing": false, "organization_notes_editing": false, "assign_tickets_to_any_group": false, "end_user_profile_access": "readonly", "explore_access": "readonly", "forum_access": "readonly", "macro_access": "manage-personal", "report_access": "readonly", "ticket_editing": true, "ticket_merge": false, "user_view_access": "manage-personal", "view_access": "manage-personal", "voice_dashboard_access": false, "manage_automations": false, "manage_contextual_workspaces": false, "manage_organization_fields": false, "manage_skills": false, "manage_slas": false, "manage_ticket_fields": false, "manage_ticket_forms": false, "manage_user_fields": false, "ticket_redaction": false, "manage_roles": "none", "manage_groups": false, "manage_group_memberships": false, "manage_organizations": false, "manage_suspended_tickets": false, "manage_triggers": false, "view_reduced_count": false, "view_filter_tickets": true, "manage_macro_content_suggestions": false, "read_macro_content_suggestions": false, "custom_objects": {}}, "team_member_count": 1}, "emitted_at": 1698749854338} +{"stream": "custom_roles", "data": {"id": 360000210616, "name": "Team lead", "description": "Can manage all tickets and forums", "role_type": 0, "created_at": "2020-12-11T18:34:36Z", "updated_at": "2023-06-26T11:06:24Z", "configuration": {"chat_access": true, "end_user_list_access": "full", "forum_access_restricted_content": false, "light_agent": false, "manage_business_rules": true, "manage_dynamic_content": true, "manage_extensions_and_channels": true, "manage_facebook": true, "moderate_forums": false, "side_conversation_create": true, "ticket_access": "all", "ticket_comment_access": "public", "ticket_deletion": true, "ticket_tag_editing": true, "twitter_search_access": false, "view_deleted_tickets": true, "voice_access": true, "group_access": true, "organization_editing": true, "organization_notes_editing": true, "assign_tickets_to_any_group": false, "end_user_profile_access": "full", "explore_access": "edit", "forum_access": "full", "macro_access": "full", "report_access": "full", "ticket_editing": true, "ticket_merge": true, "user_view_access": "full", "view_access": "playonly", "voice_dashboard_access": true, "manage_automations": true, "manage_contextual_workspaces": true, "manage_organization_fields": true, "manage_skills": true, "manage_slas": true, "manage_ticket_fields": true, "manage_ticket_forms": true, "manage_user_fields": true, "ticket_redaction": true, "manage_roles": "all-except-self", "manage_groups": true, "manage_group_memberships": true, "manage_organizations": true, "manage_suspended_tickets": true, "manage_triggers": true, "view_reduced_count": false, "view_filter_tickets": true, "manage_macro_content_suggestions": false, "read_macro_content_suggestions": false, "custom_objects": {}}, "team_member_count": 2}, "emitted_at": 1698749854339} +{"stream": "schedules", "data": {"id": 4567312249615, "name": "Test Schedule", "time_zone": "New Caledonia", "created_at": "2022-03-25T10:23:34Z", "updated_at": "2022-03-25T10:23:34Z", "intervals": [{"start_time": 1980, "end_time": 2460}, {"start_time": 3420, "end_time": 3900}, {"start_time": 4860, "end_time": 5340}, {"start_time": 6300, "end_time": 6780}, {"start_time": 7740, "end_time": 8220}]}, "emitted_at": 1697714875775} +{"stream": "user_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/user_fields/7761239926287.json", "id": 7761239926287, "type": "text", "key": "test_display_name_text_field", "title": "test Display Name text field", "description": "test Display Name text field", "raw_title": "test Display Name text field", "raw_description": "test Display Name text field", "position": 0, "active": true, "system": false, "regexp_for_validation": null, "created_at": "2023-08-28T10:10:46Z", "updated_at": "2023-08-28T10:10:46Z"}, "emitted_at": 1697714876719} +{"stream": "user_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/user_fields/7761264848527.json", "id": 7761264848527, "type": "checkbox", "key": "test_display_name_checkbox_field", "title": "test Display Name Checkbox field", "description": "", "raw_title": "test Display Name Checkbox field", "raw_description": "", "position": 1, "active": true, "system": false, "regexp_for_validation": null, "created_at": "2023-08-28T10:11:16Z", "updated_at": "2023-08-28T10:11:16Z", "tag": null}, "emitted_at": 1697714876720} +{"stream": "user_fields", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/user_fields/7761256026127.json", "id": 7761256026127, "type": "decimal", "key": "test_display_name_decimal_field", "title": "test Display Name Decimal field", "description": "", "raw_title": "test Display Name Decimal field", "raw_description": "", "position": 2, "active": true, "system": false, "regexp_for_validation": null, "created_at": "2023-08-28T10:11:30Z", "updated_at": "2023-08-28T10:11:30Z"}, "emitted_at": 1697714876721} +{"stream": "ticket_forms", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/ticket_forms/360000084116.json", "name": "Default Ticket Form", "display_name": "Default Ticket Form", "id": 360000084116, "raw_name": "Default Ticket Form", "raw_display_name": "Default Ticket Form", "end_user_visible": true, "position": 1, "ticket_field_ids": [360002833076, 360002833096, 360002833116, 360002833136, 360002833156, 360002833176, 360002833196], "active": true, "default": true, "created_at": "2020-12-11T18:34:37Z", "updated_at": "2020-12-11T18:34:37Z", "in_all_brands": true, "restricted_brand_ids": [], "end_user_conditions": [], "agent_conditions": []}, "emitted_at": 1697714877576} +{"stream": "account_attributes", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/routing/attributes/ac43b460-0ebd-11ee-85a3-4750db6aa722.json", "id": "ac43b460-0ebd-11ee-85a3-4750db6aa722", "name": "Language", "created_at": "2023-06-19T16:23:49Z", "updated_at": "2023-06-19T16:23:49Z"}, "emitted_at": 1697714879176} +{"stream": "account_attributes", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/routing/attributes/c15cdb76-0ebd-11ee-a37f-f315f48c0150.json", "id": "c15cdb76-0ebd-11ee-a37f-f315f48c0150", "name": "Quality", "created_at": "2023-06-19T16:24:25Z", "updated_at": "2023-06-19T16:24:25Z"}, "emitted_at": 1697714879178} +{"stream": "attribute_definitions", "data": {"title": "Number of incidents", "subject": "number_of_incidents", "type": "text", "group": "ticket", "nullable": false, "repeatable": false, "operators": [{"value": "less_than", "title": "Less than", "terminal": false}, {"value": "greater_than", "title": "Greater than", "terminal": false}, {"value": "is", "title": "Is", "terminal": false}, {"value": "less_than_equal", "title": "Less than or equal to", "terminal": false}, {"value": "greater_than_equal", "title": "Greater than or equal to", "terminal": false}], "condition": "all"}, "emitted_at": 1697714880365} +{"stream": "attribute_definitions", "data": {"title": "Brand", "subject": "brand_id", "type": "list", "group": "ticket", "nullable": false, "repeatable": false, "operators": [{"value": "is", "title": "Is", "terminal": false}, {"value": "is_not", "title": "Is not", "terminal": false}], "values": [{"value": "360000358316", "title": "Airbyte", "enabled": true}], "condition": "all"}, "emitted_at": 1697714880367} +{"stream": "attribute_definitions", "data": {"title": "Form", "subject": "ticket_form_id", "type": "list", "group": "ticket", "nullable": false, "repeatable": false, "operators": [{"value": "is", "title": "Is", "terminal": false}, {"value": "is_not", "title": "Is not", "terminal": false}], "values": [{"value": "360000084116", "title": "Default Ticket Form", "enabled": true}], "condition": "all"}, "emitted_at": 1697714880367} diff --git a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/incremental_catalog.json b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/incremental_catalog.json index 42aa125638b1..08b3d403363c 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/integration_tests/incremental_catalog.json +++ b/airbyte-integrations/connectors/source-zendesk-support/integration_tests/incremental_catalog.json @@ -1,5 +1,17 @@ { "streams": [ + { + "stream": { + "name": "articles", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, { "stream": { "name": "audit_logs", @@ -12,6 +24,18 @@ "sync_mode": "incremental", "destination_sync_mode": "append" }, + { + "stream": { + "name": "custom_roles", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, { "stream": { "name": "group_memberships", @@ -204,6 +228,30 @@ "sync_mode": "incremental", "destination_sync_mode": "append" }, + { + "stream": { + "name": "schedules", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "sla_policies", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, { "stream": { "name": "ticket_skips", @@ -215,6 +263,54 @@ }, "sync_mode": "incremental", "destination_sync_mode": "append" + }, + { + "stream": { + "name": "posts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "article_votes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "article_comments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "article_comment_votes", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-zendesk-support/main.py b/airbyte-integrations/connectors/source-zendesk-support/main.py index d3b005c42d35..88eed5ec56af 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/main.py +++ b/airbyte-integrations/connectors/source-zendesk-support/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_zendesk_support import SourceZendeskSupport +from source_zendesk_support.run import run if __name__ == "__main__": - source = SourceZendeskSupport() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-zendesk-support/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-support/metadata.yaml index 9a082be9b570..30befddb885f 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-support/metadata.yaml @@ -1,17 +1,23 @@ data: + ab_internal: + ql: 400 + sl: 300 allowedHosts: hosts: - ${subdomain}.zendesk.com - zendesk.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source - maxSecondsBetweenMessages: 10800 definitionId: 79c1aa37-dae3-42ae-b333-d1c105477715 - dockerImageTag: 1.0.0 + dockerImageTag: 2.2.6 dockerRepository: airbyte/source-zendesk-support + documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-support githubIssueLabel: source-zendesk-support icon: zendesk-support.svg license: ELv2 + maxSecondsBetweenMessages: 10800 name: Zendesk Support registries: cloud: @@ -19,16 +25,32 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-support - tags: - - language:python - ab_internal: - sl: 300 - ql: 400 - supportLevel: certified releases: breakingChanges: 1.0.0: message: "`cursor_field` for `Tickets` stream is changed to `generated_timestamp`" upgradeDeadline: "2023-07-19" + 2.0.0: + message: + The `Deleted Tickets` stream was removed. Deleted tickets are still + available from the Tickets stream. + upgradeDeadline: "2023-10-04" + suggestedStreams: + streams: + - brands + - groups + - organizations + - satisfaction_ratings + - tags + - ticket_audits + - ticket_comments + - ticket_fields + - ticket_forms + - ticket_metric_events + - ticket_metrics + - tickets + - users + supportLevel: certified + tags: + - language:python metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-zendesk-support/setup.py b/airbyte-integrations/connectors/source-zendesk-support/setup.py index 58dc32bc30d0..e0466040c015 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-support/setup.py @@ -7,9 +7,14 @@ MAIN_REQUIREMENTS = ["airbyte-cdk", "pytz"] -TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock~=3.6", "requests-mock==1.9.3"] +TEST_REQUIREMENTS = ["freezegun", "pytest~=6.1", "pytest-mock~=3.6", "requests-mock==1.9.3"] setup( + entry_points={ + "console_scripts": [ + "source-zendesk-support=source_zendesk_support.run:run", + ], + }, version="0.1.0", name="source_zendesk_support", description="Source implementation for Zendesk Support.", diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/ZendeskSupportAvailabilityStrategy.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/ZendeskSupportAvailabilityStrategy.py deleted file mode 100644 index 42b60b6a13bd..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/ZendeskSupportAvailabilityStrategy.py +++ /dev/null @@ -1,15 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import requests -from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy - - -class ZendeskSupportAvailabilityStrategy(HttpAvailabilityStrategy): - def check_availability(self, stream, logger, source): - try: - stream.get_api_records_count() - except requests.HTTPError as error: - return self.handle_http_error(stream, logger, source, error) - return True, None diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/run.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/run.py new file mode 100644 index 000000000000..95b88323a18c --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zendesk_support import SourceZendeskSupport + + +def run(): + source = SourceZendeskSupport() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/article_comments.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/article_comments.json new file mode 100644 index 000000000000..09b85f7b0952 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/article_comments.json @@ -0,0 +1,52 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "Post Comments", + "type": ["null", "object"], + "properties": { + "author_id": { + "type": ["null", "integer"] + }, + "body": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "html_url": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "locale": { + "type": ["null", "string"] + }, + "non_author_editor_id": { + "type": ["null", "integer"] + }, + "non_author_updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "source_id": { + "type": ["null", "integer"] + }, + "source_type": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "url": { + "type": ["null", "string"] + }, + "vote_count": { + "type": ["null", "integer"] + }, + "vote_sum": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/articles.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/articles.json new file mode 100644 index 000000000000..19ae6f1d6e04 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/articles.json @@ -0,0 +1,85 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "Articles", + "type": ["null", "object"], + "properties": { + "author_id": { + "type": ["null", "integer"] + }, + "body": { + "type": ["null", "string"] + }, + "comments_disabled": { + "type": ["null", "boolean"] + }, + "content_tag_ids": { + "type": ["null", "array"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "draft": { + "type": ["null", "boolean"] + }, + "edited_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "html_url": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "label_names": { + "type": ["null", "array"] + }, + "locale": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "outdated": { + "type": ["null", "boolean"] + }, + "outdated_locales": { + "type": ["null", "array"] + }, + "permission_group_id": { + "type": ["null", "integer"] + }, + "position": { + "type": ["null", "integer"] + }, + "promoted": { + "type": ["null", "boolean"] + }, + "section_id": { + "type": ["null", "integer"] + }, + "source_locale": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "url": { + "type": ["null", "string"] + }, + "user_segment_id": { + "type": ["null", "integer"] + }, + "vote_count": { + "type": ["null", "integer"] + }, + "vote_sum": { + "type": ["null", "integer"] + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/attribute_definitions.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/attribute_definitions.json index 11aae8663482..b2167cc3f446 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/attribute_definitions.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/attribute_definitions.json @@ -1,4 +1,6 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "Attribute Definitions", "type": ["null", "object"], "properties": { "title": { @@ -55,6 +57,9 @@ }, "condition": { "type": ["null", "string"] + }, + "confition": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/audit_logs.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/audit_logs.json index 3e9c2950e43d..726f87ca93b6 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/audit_logs.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/audit_logs.json @@ -1,12 +1,14 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Audit_logs Schema", - "additionalProperties": true, + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "Audit Logs", "type": ["null", "object"], "properties": { "action": { "type": ["null", "string"] }, + "action_label": { + "type": ["null", "string"] + }, "actor_id": { "type": ["null", "number"] }, diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/custom_roles.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/custom_roles.json index dbc7ebd9cdf2..442a7c65f39a 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/custom_roles.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/custom_roles.json @@ -142,6 +142,12 @@ "updated_at": { "type": ["null", "string"], "format": "date-time" + }, + "manage_macro_content_suggestions": { + "type": ["null", "boolean"] + }, + "read_macro_content_suggestions": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/deleted_tickets.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/deleted_tickets.json new file mode 100644 index 000000000000..6f370225b0aa --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/deleted_tickets.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "Deleted Tickets", + "type": ["null", "object"], + "properties": { + "actor": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "id": { + "type": ["null", "integer"] + }, + "subject": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "deleted_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "previous_state": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/groups.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/groups.json index b10e430d0375..6c884abc17dd 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/groups.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/groups.json @@ -15,11 +15,20 @@ "type": ["null", "string"], "format": "date-time" }, + "default": { + "type": ["null", "boolean"] + }, "deleted": { "type": ["null", "boolean"] }, + "description": { + "type": ["null", "string"] + }, "id": { "type": ["null", "integer"] + }, + "is_public": { + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/macros.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/macros.json index 1110d1e1bbb9..283936900422 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/macros.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/macros.json @@ -1,4 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "Macros", + "type": ["null", "object"], "properties": { "id": { "type": ["null", "integer"] @@ -36,6 +39,9 @@ "description": { "type": ["null", "string"] }, + "default": { + "type": ["null", "boolean"] + }, "updated_at": { "type": ["null", "string"], "format": "date-time" @@ -43,6 +49,9 @@ "active": { "type": ["null", "boolean"] }, + "raw_title": { + "type": ["null", "string"] + }, "actions": { "items": { "properties": { @@ -57,6 +66,5 @@ }, "type": ["null", "array"] } - }, - "type": ["null", "object"] + } } diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/organization_fields.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/organization_fields.json new file mode 100644 index 000000000000..1f10ccf1c6cb --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/organization_fields.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "Ticket Activities", + "type": ["null", "object"], + "properties": { + "active": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "custom_field_options": { + "type": ["null", "array"] + }, + "description": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "key": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "integer"] + }, + "raw_description": { + "type": ["null", "string"] + }, + "raw_title": { + "type": ["null", "string"] + }, + "regexp_for_validation": { + "type": ["null", "string"] + }, + "relationship_filter": { + "type": ["null", "object"] + }, + "relationship_target_type": { + "type": ["null", "string"] + }, + "system": { + "type": ["null", "boolean"] + }, + "tag": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "url": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/posts.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/posts.json index cf3bf282738f..b27e0b3c5373 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/posts.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/posts.json @@ -1,27 +1,66 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Posts Schema", - "additionalProperties": true, + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "Posts", "type": ["null", "object"], "properties": { "author_id": { "type": ["null", "number"] }, + "closed": { + "type": ["null", "boolean"] + }, + "comment_count": { + "type": ["null", "integer"] + }, "content_tag_ids": { "type": ["null", "array"], "items": { "type": ["null", "number"] } }, + "details": { + "type": ["null", "string"] + }, "featured": { "type": ["null", "boolean"] }, + "follower_count": { + "type": ["null", "integer"] + }, + "frozen": { + "type": ["null", "boolean"] + }, + "html_url": { + "type": ["null", "string"] + }, "id": { "type": ["null", "number"] }, + "non_author_editor_id": { + "type": ["null", "integer"] + }, + "non_author_updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "pinned": { + "type": ["null", "boolean"] + }, + "status": { + "type": ["null", "string"] + }, "title": { "type": ["null", "string"] }, + "topic_id": { + "type": ["null", "integer"] + }, + "vote_count": { + "type": ["null", "integer"] + }, + "vote_sum": { + "type": ["null", "integer"] + }, "created_at": { "type": ["null", "string"], "format": "date-time" diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/tickets.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/tickets.json index 95534c91f74e..cea500a4e061 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/tickets.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/shared/tickets.json @@ -1,4 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "Tickets", + "type": ["null", "object"], "properties": { "organization_id": { "type": ["null", "integer"] @@ -71,6 +74,9 @@ "type": ["null", "string"], "format": "date-time" }, + "fields": { + "type": ["null", "array"] + }, "custom_fields": { "items": { "properties": { @@ -186,6 +192,5 @@ "from_messaging_channel": { "type": ["null", "boolean"] } - }, - "type": ["null", "object"] + } } diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_comments.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_comments.json index 4fb30ea5cb38..cc5a241d034b 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_comments.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_comments.json @@ -1,4 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "Ticket Comments", + "type": ["null", "object"], "properties": { "created_at": { "type": ["null", "string"], @@ -16,6 +19,9 @@ "ticket_id": { "type": ["null", "integer"] }, + "event_type": { + "type": ["null", "string"] + }, "type": { "type": ["null", "string"] }, @@ -37,9 +43,17 @@ "author_id": { "type": ["null", "integer"] }, - "via": { "$ref": "via.json" }, - "metadata": { "$ref": "metadata.json" }, - "attachments": { "$ref": "attachments.json" } - }, - "type": ["null", "object"] + "via": { + "$ref": "via.json" + }, + "metadata": { + "$ref": "metadata.json" + }, + "attachments": { + "$ref": "attachments.json" + }, + "uploads": { + "type": ["null", "array"] + } + } } diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_fields.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_fields.json index 3c9df5a7335e..c5d1d7a765bd 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_fields.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_fields.json @@ -1,4 +1,7 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "Ticket Fields", + "type": ["null", "object"], "properties": { "created_at": { "type": ["null", "string"], @@ -94,9 +97,14 @@ "system_field_options": { "type": ["null", "array"] }, + "custom_statuses": { + "type": ["null", "array"] + }, + "key": { + "type": ["null", "string"] + }, "sub_type_id": { "type": ["null", "integer"] } - }, - "type": ["null", "object"] + } } diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_forms.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_forms.json index 0c94cb05c689..8d4d889625ab 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_forms.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_forms.json @@ -1,5 +1,14 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "Ticket Forms", + "type": ["null", "object"], "properties": { + "agent_conditions": { + "type": ["null", "array"] + }, + "end_user_conditions": { + "type": ["null", "array"] + }, "created_at": { "type": ["null", "string"], "format": "date-time" @@ -53,6 +62,5 @@ "type": ["null", "integer"] } } - }, - "type": ["null", "object"] + } } diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_metrics.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_metrics.json index 893519aa0201..c91ca0d1a9a9 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_metrics.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/ticket_metrics.json @@ -149,6 +149,17 @@ "custom_status_updated_at": { "type": ["null", "string"], "format": "date-time" + }, + "reply_time_in_seconds": { + "type": ["null", "object"], + "properties": { + "calendar": { + "type": ["null", "integer"] + }, + "business": { + "type": ["null", "integer"] + } + } } }, "type": ["null", "object"] diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/topics.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/topics.json index 43f8439b48ab..908706bf1ae5 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/topics.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/topics.json @@ -1,7 +1,6 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Topics Schema", - "additionalProperties": true, + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "Topics", "type": ["null", "object"], "properties": { "html_url": { @@ -20,6 +19,9 @@ "type": ["null", "string"], "format": "date-time" }, + "community_id": { + "type": ["null", "integer"] + }, "updated_at": { "type": ["null", "string"], "format": "date-time" diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/user_fields.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/user_fields.json new file mode 100644 index 000000000000..31f7d26c9215 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/user_fields.json @@ -0,0 +1,54 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "User Fields", + "type": "object", + "properties": { + "active": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "description": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "number"] + }, + "key": { + "type": ["null", "string"] + }, + "position": { + "type": ["null", "number"] + }, + "raw_description": { + "type": ["null", "string"] + }, + "raw_title": { + "type": ["null", "string"] + }, + "regexp_for_validation": { + "type": ["null", "string"] + }, + "system": { + "type": ["null", "boolean"] + }, + "tag": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "url": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/users.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/users.json index 34ba660e8659..df993ff2cdc2 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/users.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/schemas/users.json @@ -1,4 +1,6 @@ { + "$schema": "https://json-schema.org/draft-07/schema#", + "title": "Users", "type": ["null", "object"], "properties": { "verified": { @@ -192,6 +194,9 @@ }, "report_csv": { "type": ["null", "boolean"] + }, + "iana_time_zone": { + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py index 2aafd9029aa6..bdb140387ffc 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py @@ -7,14 +7,19 @@ from datetime import datetime from typing import Any, List, Mapping, Tuple +import pendulum from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from source_zendesk_support.streams import DATETIME_FORMAT, SourceZendeskException +from source_zendesk_support.streams import DATETIME_FORMAT, ZendeskConfigException from .streams import ( AccountAttributes, + ArticleComments, + ArticleCommentVotes, + Articles, + ArticleVotes, AttributeDefinitions, AuditLogs, Brands, @@ -22,6 +27,7 @@ GroupMemberships, Groups, Macros, + OrganizationFields, OrganizationMemberships, Organizations, PostComments, @@ -41,6 +47,7 @@ Tickets, TicketSkips, Topics, + UserFields, Users, UserSettingsStream, ) @@ -64,8 +71,22 @@ class SourceZendeskSupport(AbstractSource): """ @classmethod - def get_authenticator(cls, config: Mapping[str, Any]) -> [TokenAuthenticator, BasicApiTokenAuthenticator]: + def get_default_start_date(cls) -> str: + """ + Gets the default start date for data retrieval. + + The default date is set to the current date and time in UTC minus 2 years. + Returns: + str: The default start date in 'YYYY-MM-DDTHH:mm:ss[Z]' format. + + Note: + Start Date is a required request parameter for Zendesk Support API streams. + """ + return pendulum.now(tz="UTC").subtract(years=2).format("YYYY-MM-DDTHH:mm:ss[Z]") + + @classmethod + def get_authenticator(cls, config: Mapping[str, Any]) -> [TokenAuthenticator, BasicApiTokenAuthenticator]: # old authentication flow support auth_old = config.get("auth_method") if auth_old: @@ -79,7 +100,7 @@ def get_authenticator(cls, config: Mapping[str, Any]) -> [TokenAuthenticator, Ba elif auth.get("credentials") == "api_token": return BasicApiTokenAuthenticator(config["credentials"]["email"], config["credentials"]["api_token"]) else: - raise SourceZendeskException(f"Not implemented authorization method: {config['credentials']}") + raise ZendeskConfigException(message=f"Not implemented authorization method: {config['credentials']}") def check_connection(self, logger, config) -> Tuple[bool, any]: """Connection check to validate that the user-provided config can be used to connect to the underlying API @@ -90,17 +111,18 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: (False, error) otherwise. """ auth = self.get_authenticator(config) - settings = None try: datetime.strptime(config["start_date"], DATETIME_FORMAT) settings = UserSettingsStream(config["subdomain"], authenticator=auth, start_date=None).get_settings() except Exception as e: return False, e - active_features = [k for k, v in settings.get("active_features", {}).items() if v] - # logger.info("available features: %s" % active_features) if "organization_access_enabled" not in active_features: - return False, "Organization access is not enabled. Please check admin permission of the current account" + return ( + False, + "Please verify that the account linked to the API key has admin permissions and try again." + "For more information visit https://support.zendesk.com/hc/en-us/articles/4408832171034-About-team-member-product-roles-and-access.", + ) return True, None @classmethod @@ -110,7 +132,7 @@ def convert_config2stream_args(cls, config: Mapping[str, Any]) -> Mapping[str, A """ return { "subdomain": config["subdomain"], - "start_date": config["start_date"], + "start_date": config.get("start_date", cls.get_default_start_date()), "authenticator": cls.get_authenticator(config), "ignore_pagination": config.get("ignore_pagination", False), } @@ -121,11 +143,16 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ args = self.convert_config2stream_args(config) streams = [ + Articles(**args), + ArticleComments(**args), + ArticleCommentVotes(**args), + ArticleVotes(**args), AuditLogs(**args), GroupMemberships(**args), Groups(**args), Macros(**args), Organizations(**args), + OrganizationFields(**args), OrganizationMemberships(**args), Posts(**args), PostComments(**args), @@ -146,6 +173,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Brands(**args), CustomRoles(**args), Schedules(**args), + UserFields(**args), ] ticket_forms_stream = TicketForms(**args) account_attributes = AccountAttributes(**args) diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/spec.json b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/spec.json index 8ad90bb867de..a90fb7bf6fc5 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/spec.json +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/spec.json @@ -4,7 +4,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Source Zendesk Support Spec", "type": "object", - "required": ["start_date", "subdomain"], + "required": ["subdomain"], "additionalProperties": true, "properties": { "start_date": { @@ -13,17 +13,21 @@ "description": "The UTC date and time from which you'd like to replicate data, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", "examples": ["2020-10-15T00:00:00Z"], "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", - "format": "date-time" + "pattern_descriptor": "YYYY-MM-DDTHH:mm:ssZ", + "format": "date-time", + "order": 2 }, "subdomain": { "type": "string", "title": "Subdomain", - "description": "This is your unique Zendesk subdomain that can be found in your account URL. For example, in https://MY_SUBDOMAIN.zendesk.com/, MY_SUBDOMAIN is the value of your subdomain." + "description": "This is your unique Zendesk subdomain that can be found in your account URL. For example, in https://MY_SUBDOMAIN.zendesk.com/, MY_SUBDOMAIN is the value of your subdomain.", + "order": 0 }, "credentials": { "title": "Authentication", "type": "object", "description": "Zendesk allows two authentication methods. We recommend using `OAuth2.0` for Airbyte Cloud users and `API token` for Airbyte Open Source users.", + "order": 1, "oneOf": [ { "title": "OAuth2.0", diff --git a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py index 6dacf68b0148..ab5316725f1a 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py @@ -8,19 +8,18 @@ from abc import ABC from datetime import datetime from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union -from urllib.parse import parse_qsl, urljoin, urlparse +from urllib.parse import parse_qsl, urlparse import pendulum import pytz import requests from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy -from airbyte_cdk.sources.streams.core import package_name_from_class +from airbyte_cdk.sources.streams.core import StreamData, package_name_from_class from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream -from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from source_zendesk_support.ZendeskSupportAvailabilityStrategy import ZendeskSupportAvailabilityStrategy +from airbyte_cdk.utils import AirbyteTracedException +from airbyte_protocol.models import FailureType DATETIME_FORMAT: str = "%Y-%m-%dT%H:%M:%SZ" LAST_END_TIME_KEY: str = "_last_end_time" @@ -38,8 +37,12 @@ def to_int(s): return s -class SourceZendeskException(Exception): - """default exception of custom SourceZendesk logic""" +class ZendeskConfigException(AirbyteTracedException): + """default config exception to custom SourceZendesk logic""" + + def __init__(self, **kwargs): + failure_type: FailureType = FailureType.config_error + super(ZendeskConfigException, self).__init__(failure_type=failure_type, **kwargs) class BaseZendeskSupportStream(HttpStream, ABC): @@ -53,8 +56,8 @@ def __init__(self, subdomain: str, start_date: str, ignore_pagination: bool = Fa self._ignore_pagination = ignore_pagination @property - def availability_strategy(self) -> Optional[AvailabilityStrategy]: - return HttpAvailabilityStrategy() + def max_retries(self) -> Union[int, None]: + return 10 def backoff_time(self, response: requests.Response) -> Union[int, float]: """ @@ -126,11 +129,29 @@ def should_retry(self, response: requests.Response) -> bool: except requests.exceptions.JSONDecodeError: reason = response.reason error = {"title": f"{reason}", "message": "Received empty JSON response"} - self.logger.error(f"Skipping stream {self.name}: Check permissions, error message: {error}.") + self.logger.error( + f"Skipping stream {self.name}, error message: {error}. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." + ) setattr(self, "raise_on_http_errors", False) return False return super().should_retry(response) + def read_records( + self, + sync_mode: SyncMode, + cursor_field: Optional[List[str]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + stream_state: Optional[Mapping[str, Any]] = None, + ) -> Iterable[StreamData]: + try: + yield from super().read_records( + sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + except requests.exceptions.JSONDecodeError: + self.logger.error( + f"Skipping stream {self.name}: Non-JSON response received. Please ensure that you have enough permissions for this stream." + ) + class SourceZendeskSupportStream(BaseZendeskSupportStream): """Basic Zendesk class""" @@ -148,10 +169,6 @@ class SourceZendeskSupportStream(BaseZendeskSupportStream): def url_base(self) -> str: return f"https://{self._subdomain}.zendesk.com/api/v2/" - @property - def availability_strategy(self) -> Optional["AvailabilityStrategy"]: - return ZendeskSupportAvailabilityStrategy() - def path(self, **kwargs): return self.name @@ -164,26 +181,6 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late return {self.cursor_field: max(latest_benchmark, current_stream_state[self.cursor_field])} return {self.cursor_field: latest_benchmark} - def get_api_records_count(self, stream_slice: Mapping[str, Any] = None, stream_state: Mapping[str, Any] = None): - """ - Count stream records before generating the future requests - to then correctly generate the pagination parameters. - """ - - count_url = urljoin(self.url_base, f"{self.path(stream_state=stream_state, stream_slice=stream_slice)}/count.json") - - start_date = self._start_date - params = {} - if self.cursor_field and stream_state: - start_date = stream_state.get(self.cursor_field) - if start_date: - params["start_time"] = self.str2datetime(start_date) - - response = self._session.request("get", count_url) - records_count = response.json().get("count", {}).get("value", 0) - - return records_count - def request_params( self, stream_state: Mapping[str, Any], @@ -259,7 +256,7 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late new_value = str((latest_record or {}).get(self.cursor_field, "")) return {self.cursor_field: max(new_value, old_value)} - def check_stream_state(self, stream_state: Mapping[str, Any] = None) -> int: + def get_stream_state_value(self, stream_state: Mapping[str, Any] = None) -> int: """ Returns the state value, if exists. Otherwise, returns user defined `Start Date`. """ @@ -284,7 +281,7 @@ def request_params( next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: params = { - "start_time": self.check_stream_state(stream_state), + "start_time": self.get_stream_state_value(stream_state), "page[size]": self.page_size, } if next_page_token: @@ -309,7 +306,7 @@ def request_params( next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: next_page_token = next_page_token or {} - parsed_state = self.check_stream_state(stream_state) + parsed_state = self.get_stream_state_value(stream_state) if self.cursor_field: params = {"start_time": next_page_token.get(self.cursor_field, parsed_state)} else: @@ -322,15 +319,19 @@ class SourceZendeskIncrementalExportStream(IncrementalZendeskSupportStream): https://developer.zendesk.com/api-reference/ticketing/ticket-management/incremental_exports/#incremental-ticket-export-time-based @ param response_list_name: the main nested entity to look at inside of response, default = response_list_name - @ param sideload_param : parameter variable to include various information to response more info: https://developer.zendesk.com/documentation/ticketing/using-the-zendesk-api/side_loading/#supported-endpoints """ - response_list_name: str = None - sideload_param: str = None + @property + def response_list_name() -> str: + raise NotImplementedError("The `response_list_name` must be implemented") + + @property + def next_page_field() -> str: + raise NotImplementedError("The `next_page_field` varies depending on stream and must be set individually") @staticmethod - def check_start_time_param(requested_start_time: int, value: int = 1) -> int: + def validate_start_time(requested_start_time: int, value: int = 1) -> int: """ Requesting tickets in the future is not allowed, hits 400 - bad request. We get current UNIX timestamp minus `value` from now(), default = 1 (minute). @@ -350,7 +351,9 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, if self._ignore_pagination: return None response_json = response.json() - return None if response_json.get(END_OF_STREAM_KEY, True) else {"cursor": response_json.get("after_cursor")} + if END_OF_STREAM_KEY in response_json and response_json[END_OF_STREAM_KEY]: + return None + return dict(parse_qsl(urlparse(response_json.get(self.next_page_field, "")).query)) def request_params( self, @@ -358,15 +361,14 @@ def request_params( stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: - params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - # check "start_time" is not in the future - params["start_time"] = self.check_start_time_param(params["start_time"]) - if self.sideload_param: - params["include"] = self.sideload_param + """ + Request params are based on parsed query params of next page url. + `start_time` will be included as the initial request parameter and will never be changed unless it is itself a next page token. + """ if next_page_token: - params.pop("start_time", None) - params.update(next_page_token) - return params + return next_page_token + start_time = self.get_stream_state_value(stream_state) + return {"start_time": self.validate_start_time(start_time)} def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: for record in response.json().get(self.response_list_name, []): @@ -381,48 +383,23 @@ class SourceZendeskSupportTicketEventsExportStream(SourceZendeskIncrementalExpor @ param response_target_entity: nested property inside of `response_list_name`, default = "child_events" @ param list_entities_from_event : the list of nested child_events entities to include from parent record @ param event_type : specific event_type to check ["Audit", "Change", "Comment", etc] + @ param sideload_param : parameter variable to include various information to response """ cursor_field = "created_at" + event_type: str = None + list_entities_from_event: List[str] = None response_list_name: str = "ticket_events" response_target_entity: str = "child_events" - list_entities_from_event: List[str] = None - event_type: str = None - - def path( - self, - *, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> str: - return f"incremental/{self.response_list_name}.json" - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - Returns next_page_token based on `end_of_stream` parameter inside of response - """ - response_json = response.json() - return None if response_json.get(END_OF_STREAM_KEY, True) else {"start_time": response_json.get("end_time")} + sideload_param: str = None + next_page_field: str = "next_page" def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - next_page_token = next_page_token or {} - parsed_state = self.check_stream_state(stream_state) - if self.cursor_field: - params = {"start_time": next_page_token.get(self.cursor_field, parsed_state)} - else: - params = {"start_time": calendar.timegm(pendulum.parse(self._start_date).utctimetuple())} - # check "start_time" is not in the future - params["start_time"] = self.check_start_time_param(params["start_time"]) + params = super().request_params(stream_state, stream_slice, next_page_token) if self.sideload_param: params["include"] = self.sideload_param - if next_page_token: - params.update(next_page_token) return params @property @@ -444,6 +421,10 @@ class OrganizationMemberships(CursorPaginationZendeskSupportStream): """OrganizationMemberships stream: https://developer.zendesk.com/api-reference/ticketing/organizations/organization_memberships/""" +class OrganizationFields(CursorPaginationZendeskSupportStream): + """OrganizationMemberships stream: https://developer.zendesk.com/api-reference/ticketing/organizations/organization_fields/#list-organization-fields""" + + class AuditLogs(CursorPaginationZendeskSupportStream): """AuditLogs stream: https://developer.zendesk.com/api-reference/ticketing/account-configuration/audit_logs/#list-audit-logs""" @@ -457,34 +438,18 @@ class Users(SourceZendeskIncrementalExportStream): """Users stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/incremental_exports/#incremental-user-export""" response_list_name: str = "users" + next_page_field: str = "after_url" def path(self, **kwargs) -> str: return "incremental/users/cursor.json" - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - next_page_token = next_page_token or {} - parsed_state = self.check_stream_state(stream_state) - if self.cursor_field: - params = {"start_time": next_page_token.get(self.cursor_field, parsed_state)} - else: - params = {"start_time": calendar.timegm(pendulum.parse(self._start_date).utctimetuple())} - # check "start_time" is not in the future - params["start_time"] = self.check_start_time_param(params["start_time"]) - if self.sideload_param: - params["include"] = self.sideload_param - if next_page_token: - params.update(next_page_token) - return params - -class Organizations(SourceZendeskSupportStream): +class Organizations(SourceZendeskIncrementalExportStream): """Organizations stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/incremental_exports/""" + response_list_name: str = "organizations" + next_page_field: str = "next_page" + class Posts(CursorPaginationZendeskSupportStream): """Posts stream: https://developer.zendesk.com/api-reference/help_center/help-center-api/posts/#list-posts""" @@ -504,41 +469,28 @@ class Tickets(SourceZendeskIncrementalExportStream): transformer: TypeTransformer = TypeTransformer(TransformConfig.DefaultSchemaNormalization) cursor_field = "generated_timestamp" + next_page_field = "after_url" def path(self, **kwargs) -> str: return "incremental/tickets/cursor.json" - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - parsed_state = self.check_stream_state(stream_state) - params = {"start_time": self.check_start_time_param(parsed_state)} - if self.sideload_param: - params["include"] = self.sideload_param - if next_page_token: - params.update(next_page_token) - return params - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: old_value = (current_stream_state or {}).get(self.cursor_field, pendulum.parse(self._start_date).int_timestamp) new_value = (latest_record or {}).get(self.cursor_field, pendulum.parse(self._start_date).int_timestamp) return {self.cursor_field: max(new_value, old_value)} - def check_stream_state(self, stream_state: Mapping[str, Any] = None) -> int: + def get_stream_state_value(self, stream_state: Mapping[str, Any] = None) -> int: """ Returns the state value, if exists. Otherwise, returns user defined `Start Date`. """ return stream_state.get(self.cursor_field) if stream_state else pendulum.parse(self._start_date).int_timestamp - def check_start_time_param(self, requested_start_time: int, value: int = 1) -> int: + def validate_start_time(self, requested_start_time: int, value: int = 1) -> int: """ The stream returns 400 Bad Request StartTimeTooRecent when requesting tasks 1 second before now. Figured out during experiments that the most recent time needed for request to be successful is 3 seconds before now. """ - return super().check_start_time_param(requested_start_time, value=3) + return super().validate_start_time(requested_start_time, value=3) class TicketComments(SourceZendeskSupportTicketEventsExportStream): @@ -630,7 +582,7 @@ def request_params( next_page_token: Mapping[str, Any] = None, ) -> MutableMapping[str, Any]: params = { - "start_time": self.check_stream_state(stream_state), + "start_time": self.get_stream_state_value(stream_state), "page[size]": self.page_size, } if next_page_token: # need keep start_time for this stream @@ -646,7 +598,7 @@ class TicketAudits(IncrementalZendeskSupportStream): """TicketAudits stream: https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_audits/""" # can request a maximum of 1,000 results - page_size = 1000 + page_size = 200 # ticket audits doesn't have the 'updated_by' field cursor_field = "created_at" @@ -674,6 +626,23 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, response_json = response.json() return {"cursor": response.json().get("before_cursor")} if response_json.get("before_cursor") else None + def read_records( + self, + sync_mode: SyncMode, + cursor_field: Optional[List[str]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + stream_state: Optional[Mapping[str, Any]] = None, + ) -> Iterable[StreamData]: + try: + yield from super().read_records( + sync_mode=sync_mode, cursor_field=cursor_field, stream_slice=stream_slice, stream_state=stream_state + ) + except requests.exceptions.HTTPError as e: + if e.response.status_code == requests.codes.GATEWAY_TIMEOUT: + self.logger.error(f"Skipping stream `{self.name}`. Timed out waiting for response: {e.response.text}...") + else: + raise e + class Tags(FullRefreshZendeskSupportStream): """Tags stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/tags/""" @@ -693,7 +662,7 @@ def path(self, **kwargs): return "community/topics" -class SlaPolicies(FullRefreshZendeskSupportStream): +class SlaPolicies(IncrementalZendeskSupportStream): """SlaPolicies stream: https://developer.zendesk.com/api-reference/ticketing/business-rules/sla_policies/""" def path(self, *args, **kwargs) -> str: @@ -712,7 +681,7 @@ class Brands(FullRefreshZendeskSupportStream): """Brands stream: https://developer.zendesk.com/api-reference/ticketing/account-configuration/brands/#list-brands""" -class CustomRoles(FullRefreshZendeskSupportStream): +class CustomRoles(IncrementalZendeskSupportStream): """CustomRoles stream: https://developer.zendesk.com/api-reference/ticketing/account-configuration/custom_roles/#list-custom-roles""" def request_params( @@ -724,7 +693,7 @@ def request_params( return {} -class Schedules(FullRefreshZendeskSupportStream): +class Schedules(IncrementalZendeskSupportStream): """Schedules stream: https://developer.zendesk.com/api-reference/ticketing/ticket-management/schedules/#list-schedules""" def path(self, *args, **kwargs) -> str: @@ -758,7 +727,7 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str, definition["condition"] = "all" yield definition for definition in response.json()["definitions"]["conditions_any"]: - definition["confition"] = "any" + definition["condition"] = "any" yield definition def path(self, *args, **kwargs) -> str: @@ -792,7 +761,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp def get_settings(self) -> Mapping[str, Any]: for resp in self.read_records(SyncMode.full_refresh): return resp - raise SourceZendeskException("not found settings") + raise ZendeskConfigException(message="Can not get access to settings endpoint; Please check provided credentials") def request_params( self, @@ -803,7 +772,16 @@ def request_params( return {} -class PostComments(FullRefreshZendeskSupportStream, HttpSubStream): +class UserFields(FullRefreshZendeskSupportStream): + """User Fields stream: https://developer.zendesk.com/api-reference/ticketing/users/user_fields/#list-user-fields""" + + def path(self, *args, **kwargs) -> str: + return "user_fields" + + +class PostComments(CursorPaginationZendeskSupportStream, HttpSubStream): + """Post Comments Stream: https://developer.zendesk.com/api-reference/help_center/help-center-api/post_comments/""" + response_list_name = "comments" def __init__(self, **kwargs): @@ -821,7 +799,7 @@ def path( return f"community/posts/{post_id}/comments" -class AbstractVotes(FullRefreshZendeskSupportStream, ABC): +class AbstractVotes(CursorPaginationZendeskSupportStream, ABC): response_list_name = "votes" def get_json_schema(self) -> Mapping[str, Any]: @@ -859,3 +837,72 @@ def path( post_id = stream_slice.get("parent").get("post_id") comment_id = stream_slice.get("parent").get("id") return f"community/posts/{post_id}/comments/{comment_id}/votes" + + +class Articles(SourceZendeskIncrementalExportStream): + """Articles Stream: https://developer.zendesk.com/api-reference/help_center/help-center-api/articles/#list-articles""" + + response_list_name: str = "articles" + next_page_field: str = "next_page" + + def path(self, **kwargs) -> str: + return "help_center/incremental/articles" + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + return {"sort_by": "updated_at", "sort_order": "asc", **super().request_params(stream_state, stream_slice, next_page_token)} + + +class ArticleVotes(AbstractVotes, HttpSubStream): + def __init__(self, **kwargs): + parent = Articles(**kwargs) + super().__init__(parent=parent, **kwargs) + + def path( + self, + *, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> str: + article_id = stream_slice.get("parent").get("id") + return f"help_center/articles/{article_id}/votes" + + +class ArticleComments(CursorPaginationZendeskSupportStream, HttpSubStream): + response_list_name = "comments" + + def __init__(self, **kwargs): + parent = Articles(**kwargs) + super().__init__(parent=parent, **kwargs) + + def path( + self, + *, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> str: + article_id = stream_slice.get("parent").get("id") + return f"help_center/articles/{article_id}/comments" + + +class ArticleCommentVotes(AbstractVotes, HttpSubStream): + def __init__(self, **kwargs): + parent = ArticleComments(**kwargs) + super().__init__(parent=parent, **kwargs) + + def path( + self, + *, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> str: + article_id = stream_slice.get("parent").get("source_id") + comment_id = stream_slice.get("parent").get("id") + return f"help_center/articles/{article_id}/comments/{comment_id}/votes" diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py new file mode 100644 index 000000000000..c3d9c1c98188 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/conftest.py @@ -0,0 +1,5 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import os + +os.environ["REQUEST_CACHE_PATH"] = "REQUEST_CACHE_PATH" diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_backoff_on_rate_limit.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_backoff_on_rate_limit.py index 1dbdfe7bb0ad..0c17e1bcc722 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_backoff_on_rate_limit.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/test_backoff_on_rate_limit.py @@ -25,10 +25,13 @@ def prepare_config(config: Dict): return SourceZendeskSupport().convert_config2stream_args(config) -@pytest.mark.parametrize("retry_after, expected", [("5", 5), ("5, 4", 5)]) -def test_backoff(requests_mock, config, retry_after, expected): +@pytest.mark.parametrize( + "x_rate_limit, retry_after, expected", + [("60", {}, 1), ("0", {}, None), ("0", {"Retry-After": "5"}, 5), ("0", {"Retry-After": "5, 4"}, 5)], +) +def test_backoff(requests_mock, config, x_rate_limit, retry_after, expected): """ """ - test_response_header = {"Retry-After": retry_after, "X-Rate-Limit": "0"} + test_response_header = {"X-Rate-Limit": x_rate_limit} | retry_after test_response_json = {"count": {"value": 1, "refreshed_at": "2022-03-29T10:10:51+00:00"}} # create client diff --git a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py index 8037215adb23..6dde9b4f99e8 100644 --- a/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-zendesk-support/unit_tests/unit_test.py @@ -10,6 +10,7 @@ from unittest.mock import patch from urllib.parse import parse_qsl, urlparse +import freezegun import pendulum import pytest import pytz @@ -21,6 +22,10 @@ END_OF_STREAM_KEY, LAST_END_TIME_KEY, AccountAttributes, + ArticleComments, + ArticleCommentVotes, + Articles, + ArticleVotes, AttributeDefinitions, AuditLogs, BaseZendeskSupportStream, @@ -33,6 +38,7 @@ Organizations, PostCommentVotes, Posts, + PostVotes, SatisfactionRatings, Schedules, SlaPolicies, @@ -47,6 +53,7 @@ Tickets, TicketSkips, Topics, + UserFields, Users, UserSettingsStream, ) @@ -67,6 +74,19 @@ "credentials": {"credentials": "api_token", "email": "integration-test@airbyte.io", "api_token": "api_token"}, } +# raw old config +TEST_OLD_CONFIG = { + "auth_method": {"auth_method": "api_token", "email": "integration-test@airbyte.io", "api_token": "api_token"}, + "subdomain": "sandbox", + "start_date": "2021-06-01T00:00:00Z", +} + +TEST_CONFIG_WITHOUT_START_DATE = { + "subdomain": "sandbox", + "credentials": {"credentials": "api_token", "email": "integration-test@airbyte.io", "api_token": "api_token"}, +} + + # raw config oauth TEST_CONFIG_OAUTH = { "subdomain": "sandbox", @@ -114,10 +134,20 @@ def test_convert_config2stream_args(config): assert "authenticator" in result +@freezegun.freeze_time("2022-01-01") +def test_default_start_date(): + result = SourceZendeskSupport().convert_config2stream_args(TEST_CONFIG_WITHOUT_START_DATE) + assert result["start_date"] == "2020-01-01T00:00:00Z" + + @pytest.mark.parametrize( "config, expected", - [(TEST_CONFIG, "aW50ZWdyYXRpb24tdGVzdEBhaXJieXRlLmlvL3Rva2VuOmFwaV90b2tlbg=="), (TEST_CONFIG_OAUTH, "test_access_token")], - ids=["api_token", "oauth"], + [ + (TEST_CONFIG, "aW50ZWdyYXRpb24tdGVzdEBhaXJieXRlLmlvL3Rva2VuOmFwaV90b2tlbg=="), + (TEST_CONFIG_OAUTH, "test_access_token"), + (TEST_OLD_CONFIG, "aW50ZWdyYXRpb24tdGVzdEBhaXJieXRlLmlvL3Rva2VuOmFwaV90b2tlbg=="), + ], + ids=["api_token", "oauth", "old_config"], ) def test_get_authenticator(config, expected): # we expect base64 from creds input @@ -143,20 +173,24 @@ def test_check(response, start_date, check_passed): @pytest.mark.parametrize( "ticket_forms_response, status_code, expected_n_streams, expected_warnings, reason", [ - ('{"ticket_forms": [{"id": 1, "updated_at": "2021-07-08T00:05:45Z"}]}', 200, 28, [], None), + ('{"ticket_forms": [{"id": 1, "updated_at": "2021-07-08T00:05:45Z"}]}', 200, 34, [], None), ( - '{"error": "Not sufficient permissions"}', - 403, - 25, - ["Skipping stream ticket_forms: Check permissions, error message: Not sufficient permissions."], - None + '{"error": "Not sufficient permissions"}', + 403, + 31, + [ + "Skipping stream ticket_forms, error message: Not sufficient permissions. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." + ], + None, ), ( - '', - 404, - 25, - ["Skipping stream ticket_forms: Check permissions, error message: {'title': 'Not Found', 'message': 'Received empty JSON response'}."], - 'Not Found' + "", + 404, + 31, + [ + "Skipping stream ticket_forms, error message: {'title': 'Not Found', 'message': 'Received empty JSON response'}. Please ensure the authenticated user has access to this stream. If the issue persists, contact Zendesk support." + ], + "Not Found", ), ], ids=["forms_accessible", "forms_inaccessible", "forms_not_exists"], @@ -165,7 +199,7 @@ def test_full_access_streams(caplog, requests_mock, ticket_forms_response, statu requests_mock.get("/api/v2/ticket_forms", status_code=status_code, text=ticket_forms_response, reason=reason) result = SourceZendeskSupport().streams(config=TEST_CONFIG) assert len(result) == expected_n_streams - logged_warnings = iter([record for record in caplog.records if record.levelname == "ERROR"]) + logged_warnings = (record for record in caplog.records if record.levelname == "ERROR") for msg in expected_warnings: assert msg in next(logged_warnings).message @@ -197,7 +231,7 @@ def test_str2unixtime(): def test_check_start_time_param(): expected = 1626936955 start_time = calendar.timegm(pendulum.parse(DATETIME_STR).utctimetuple()) - output = SourceZendeskIncrementalExportStream.check_start_time_param(start_time) + output = SourceZendeskIncrementalExportStream.validate_start_time(start_time) assert output == expected @@ -211,7 +245,7 @@ def test_check_start_time_param(): ids=["state present", "state is None"], ) def test_check_stream_state(stream_state, expected): - result = Tickets(**STREAM_ARGS).check_stream_state(stream_state) + result = Tickets(**STREAM_ARGS).get_stream_state_value(stream_state) assert result == expected @@ -262,6 +296,7 @@ class TestAllStreams: (Schedules), (AccountAttributes), (AttributeDefinitions), + (UserFields), ], ids=[ "AuditLogs", @@ -289,6 +324,7 @@ class TestAllStreams: "Schedules", "AccountAttributes", "AttributeDefinitions", + "UserFields", ], ) def test_streams(self, expected_stream_cls): @@ -299,6 +335,12 @@ def test_streams(self, expected_stream_cls): if expected_stream_cls in streams: assert isinstance(stream, expected_stream_cls) + def test_ticket_forms_exception_stream(self): + with patch.object(TicketForms, "read_records", return_value=[{}]) as mocked_records: + mocked_records.side_effect = Exception("The error") + streams = SourceZendeskSupport().streams(TEST_CONFIG) + assert not any([isinstance(stream, TicketForms) for stream in streams]) + @pytest.mark.parametrize( "stream_cls, expected", [ @@ -306,7 +348,7 @@ def test_streams(self, expected_stream_cls): (GroupMemberships, "group_memberships"), (Groups, "groups"), (Macros, "macros"), - (Organizations, "organizations"), + (Organizations, "incremental/organizations.json"), (Posts, "community/posts"), (OrganizationMemberships, "organization_memberships"), (SatisfactionRatings, "satisfaction_ratings"), @@ -327,6 +369,7 @@ def test_streams(self, expected_stream_cls): (Schedules, "business_hours/schedules.json"), (AccountAttributes, "routing/attributes"), (AttributeDefinitions, "routing/attributes/definitions"), + (UserFields, "user_fields"), ], ids=[ "AuditLogs", @@ -354,6 +397,7 @@ def test_streams(self, expected_stream_cls): "Schedules", "AccountAttributes", "AttributeDefinitions", + "UserFields", ], ) def test_path(self, stream_cls, expected): @@ -365,26 +409,8 @@ def test_path(self, stream_cls, expected): class TestSourceZendeskSupportStream: @pytest.mark.parametrize( "stream_cls", - [ - (Macros), - (Organizations), - (Posts), - (Groups), - (SatisfactionRatings), - (TicketFields), - (TicketMetrics), - (Topics) - ], - ids=[ - "Macros", - "Organizations", - "Posts", - "Groups", - "SatisfactionRatings", - "TicketFields", - "TicketMetrics", - "Topics" - ], + [(Macros), (Posts), (Groups), (SatisfactionRatings), (TicketFields), (TicketMetrics), (Topics)], + ids=["Macros", "Posts", "Groups", "SatisfactionRatings", "TicketFields", "TicketMetrics", "Topics"], ) def test_parse_response(self, requests_mock, stream_cls): stream = stream_cls(**STREAM_ARGS) @@ -395,28 +421,24 @@ def test_parse_response(self, requests_mock, stream_cls): output = list(stream.parse_response(test_response, None)) assert expected == output + def test_attribute_definition_parse_response(self, requests_mock): + stream = AttributeDefinitions(**STREAM_ARGS) + conditions_all = {"subject": "number_of_incidents", "title": "Number of incidents"} + conditions_any = {"subject": "brand", "title": "Brand"} + response_json = {"definitions": {"conditions_all": [conditions_all], "conditions_any": [conditions_any]}} + requests_mock.get(STREAM_URL, json=response_json) + test_response = requests.get(STREAM_URL) + output = list(stream.parse_response(test_response, None)) + expected_records = [ + {"condition": "all", "subject": "number_of_incidents", "title": "Number of incidents"}, + {"condition": "any", "subject": "brand", "title": "Brand"}, + ] + assert expected_records == output + @pytest.mark.parametrize( "stream_cls", - [ - (Macros), - (Organizations), - (Posts), - (Groups), - (SatisfactionRatings), - (TicketFields), - (TicketMetrics), - (Topics) - ], - ids=[ - "Macros", - "Organizations", - "Posts", - "Groups", - "SatisfactionRatings", - "TicketFields", - "TicketMetrics", - "Topics" - ], + [(Macros), (Organizations), (Posts), (Groups), (SatisfactionRatings), (TicketFields), (TicketMetrics), (Topics)], + ids=["Macros", "Organizations", "Posts", "Groups", "SatisfactionRatings", "TicketFields", "TicketMetrics", "Topics"], ) def test_url_base(self, stream_cls): stream = stream_cls(**STREAM_ARGS) @@ -429,10 +451,10 @@ def test_url_base(self, stream_cls): (Macros, {}, {"updated_at": "2022-03-17T16:03:07Z"}, {"updated_at": "2022-03-17T16:03:07Z"}), (Posts, {}, {"updated_at": "2022-03-17T16:03:07Z"}, {"updated_at": "2022-03-17T16:03:07Z"}), ( - Organizations, - {"updated_at": "2022-03-17T16:03:07Z"}, - {"updated_at": "2023-03-17T16:03:07Z"}, - {"updated_at": "2023-03-17T16:03:07Z"}, + Organizations, + {"updated_at": "2022-03-17T16:03:07Z"}, + {"updated_at": "2023-03-17T16:03:07Z"}, + {"updated_at": "2023-03-17T16:03:07Z"}, ), (Groups, {}, {"updated_at": "2022-03-17T16:03:07Z"}, {"updated_at": "2022-03-17T16:03:07Z"}), (SatisfactionRatings, {}, {"updated_at": "2022-03-17T16:03:07Z"}, {"updated_at": "2022-03-17T16:03:07Z"}), @@ -440,16 +462,7 @@ def test_url_base(self, stream_cls): (TicketMetrics, {}, {"updated_at": "2022-03-17T16:03:07Z"}, {"updated_at": "2022-03-17T16:03:07Z"}), (Topics, {}, {"updated_at": "2022-03-17T16:03:07Z"}, {"updated_at": "2022-03-17T16:03:07Z"}), ], - ids=[ - "Macros", - "Posts", - "Organizations", - "Groups", - "SatisfactionRatings", - "TicketFields", - "TicketMetrics", - "Topics" - ], + ids=["Macros", "Posts", "Organizations", "Groups", "SatisfactionRatings", "TicketFields", "TicketMetrics", "Topics"], ) def test_get_updated_state(self, stream_cls, current_state, last_record, expected): stream = stream_cls(**STREAM_ARGS) @@ -461,7 +474,7 @@ def test_get_updated_state(self, stream_cls, current_state, last_record, expecte [ (Macros, None), (Posts, None), - (Organizations, None), + (Organizations, {}), (Groups, None), (TicketFields, None), ], @@ -504,16 +517,7 @@ def test_request_params(self, stream_cls, expected): class TestSourceZendeskSupportFullRefreshStream: @pytest.mark.parametrize( "stream_cls", - [ - (Tags), - (SlaPolicies), - (Brands), - (CustomRoles), - (Schedules), - (UserSettingsStream), - (AccountAttributes), - (AttributeDefinitions) - ], + [(Tags), (SlaPolicies), (Brands), (CustomRoles), (Schedules), (UserSettingsStream), (AccountAttributes), (AttributeDefinitions)], ids=[ "Tags", "SlaPolicies", @@ -620,50 +624,52 @@ def test_get_updated_state(self, stream_cls, current_state, last_record, expecte [ (GroupMemberships, {}, None), (TicketForms, {}, None), - (TicketMetricEvents, { - "meta": {"has_more": True, "after_cursor": "", "before_cursor": ""}, - "links": { - "prev": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bbefore%5D=%3D&page%5Bsize%5D=2", - "next": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bafter%5D=%3D&page%5Bsize%5D=2", + ( + TicketMetricEvents, + { + "meta": {"has_more": True, "after_cursor": "", "before_cursor": ""}, + "links": { + "prev": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bbefore%5D=%3D&page%5Bsize%5D=2", + "next": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bafter%5D=%3D&page%5Bsize%5D=2", + }, }, - }, - {"page[after]": ""}), + {"page[after]": ""}, + ), (TicketAudits, {}, None), ( - TicketMetrics, - { - "meta": {"has_more": True, "after_cursor": "", "before_cursor": ""}, - "links": { - "prev": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bbefore%5D=%3D&page%5Bsize%5D=2", - "next": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bafter%5D=%3D&page%5Bsize%5D=2", - }, + TicketMetrics, + { + "meta": {"has_more": True, "after_cursor": "", "before_cursor": ""}, + "links": { + "prev": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bbefore%5D=%3D&page%5Bsize%5D=2", + "next": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bafter%5D=%3D&page%5Bsize%5D=2", }, - {"page[after]": ""}, + }, + {"page[after]": ""}, ), (SatisfactionRatings, {}, None), ( - OrganizationMemberships, - { - "meta": {"has_more": True, "after_cursor": "", "before_cursor": ""}, - "links": { - "prev": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bbefore%5D=%3D&page%5Bsize%5D=2", - "next": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bafter%5D=%3D&page%5Bsize%5D=2", - }, + OrganizationMemberships, + { + "meta": {"has_more": True, "after_cursor": "", "before_cursor": ""}, + "links": { + "prev": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bbefore%5D=%3D&page%5Bsize%5D=2", + "next": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bafter%5D=%3D&page%5Bsize%5D=2", }, - {"page[after]": ""}, + }, + {"page[after]": ""}, ), ( - TicketSkips, - { - "meta": {"has_more": True, "after_cursor": "", "before_cursor": ""}, - "links": { - "prev": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bbefore%5D=%3D&page%5Bsize%5D=2", - "next": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bafter%5D=%3D&page%5Bsize%5D=2", - }, + TicketSkips, + { + "meta": {"has_more": True, "after_cursor": "", "before_cursor": ""}, + "links": { + "prev": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bbefore%5D=%3D&page%5Bsize%5D=2", + "next": "https://subdomain.zendesk.com/api/v2/ticket_metrics.json?page%5Bafter%5D=%3D&page%5Bsize%5D=2", }, - {"page[after]": ""}, + }, + {"page[after]": ""}, ), - ], ids=[ "GroupMemberships", @@ -693,28 +699,21 @@ def test_next_page_token(self, requests_mock, stream_cls, response, expected): (OrganizationMemberships, 1622505600), (TicketSkips, 1622505600), ], - ids=[ - "GroupMemberships", - "TicketForms", - "TicketMetricEvents", - "TicketAudits", - "OrganizationMemberships", - "TicketSkips" - ], + ids=["GroupMemberships", "TicketForms", "TicketMetricEvents", "TicketAudits", "OrganizationMemberships", "TicketSkips"], ) def test_check_stream_state(self, stream_cls, expected): stream = stream_cls(**STREAM_ARGS) - result = stream.check_stream_state() + result = stream.get_stream_state_value() assert result == expected @pytest.mark.parametrize( "stream_cls, expected", [ - (GroupMemberships, {'page[size]': 100, 'sort_by': 'asc', 'start_time': 1622505600}), + (GroupMemberships, {"page[size]": 100, "sort_by": "asc", "start_time": 1622505600}), (TicketForms, {"start_time": 1622505600}), - (TicketMetricEvents, {'page[size]': 100, 'start_time': 1622505600}), - (TicketAudits, {"sort_by": "created_at", "sort_order": "desc", "limit": 1000}), - (SatisfactionRatings, {'page[size]': 100, 'sort_by': 'asc', 'start_time': 1622505600}), + (TicketMetricEvents, {"page[size]": 100, "start_time": 1622505600}), + (TicketAudits, {"sort_by": "created_at", "sort_order": "desc", "limit": 200}), + (SatisfactionRatings, {"page[size]": 100, "sort_by": "asc", "start_time": 1622505600}), (TicketMetrics, {"page[size]": 100, "start_time": 1622505600}), (OrganizationMemberships, {"page[size]": 100, "start_time": 1622505600}), (TicketSkips, {"page[size]": 100, "start_time": 1622505600}), @@ -751,7 +750,7 @@ class TestSourceZendeskIncrementalExportStream: def test_check_start_time_param(self, stream_cls): expected = int(dict(parse_qsl(urlparse(STREAM_URL).query)).get("start_time")) stream = stream_cls(**STREAM_ARGS) - result = stream.check_start_time_param(expected) + result = stream.validate_start_time(expected) assert result == expected @pytest.mark.parametrize( @@ -771,17 +770,19 @@ def test_next_page_token(self, requests_mock, stream_cls): requests_mock.get(STREAM_URL, json={stream_name: {}}) test_response = requests.get(STREAM_URL) output = stream.next_page_token(test_response) - assert output is None + assert output == {} @pytest.mark.parametrize( "stream_cls, expected", [ (Users, {"start_time": 1622505600}), (Tickets, {"start_time": 1622505600}), + (Articles, {"sort_by": "updated_at", "sort_order": "asc", "start_time": 1622505600}), ], ids=[ "Users", "Tickets", + "Articles", ], ) def test_request_params(self, stream_cls, expected): @@ -809,6 +810,23 @@ def test_parse_response(self, requests_mock, stream_cls): output = list(stream.parse_response(test_response)) assert expected == output + @pytest.mark.parametrize( + "stream_cls, stream_slice, expected_path", + [ + (ArticleVotes, {"parent": {"id": 1}}, "help_center/articles/1/votes"), + (ArticleComments, {"parent": {"id": 1}}, "help_center/articles/1/comments"), + (ArticleCommentVotes, {"parent": {"id": 1, "source_id": 1}}, "help_center/articles/1/comments/1/votes"), + ], + ids=[ + "ArticleVotes_path", + "ArticleComments_path", + "ArticleCommentVotes_path", + ], + ) + def test_path(self, stream_cls, stream_slice, expected_path): + stream = stream_cls(**STREAM_ARGS) + assert stream.path(stream_slice=stream_slice) == expected_path + class TestSourceZendeskSupportTicketEventsExportStream: @pytest.mark.parametrize( @@ -929,7 +947,7 @@ def test_read_tickets_stream(requests_mock): ] }, ], - "end_of_stream": True + "end_of_stream": True, }, ) @@ -949,24 +967,51 @@ def test_read_tickets_stream(requests_mock): ] -def test_read_post_comment_votes_stream(requests_mock): +def test_read_post_votes_stream(requests_mock): post_response = { - "posts": [ - {"id": 7253375870607, "title": "Test_post", "created_at": "2023-01-01T00:00:00Z", "updated_at": "2023-01-01T00:00:00Z"} + "posts": [{"id": 7253375870607, "title": "Test_post", "created_at": "2023-01-01T00:00:00Z", "updated_at": "2023-01-01T00:00:00Z"}] + } + requests_mock.get("https://subdomain.zendesk.com/api/v2/community/posts", json=post_response) + + post_votes_response = { + "votes": [ + { + "author_id": 89567, + "body": "Test_comment for Test_post", + "id": 35467, + "post_id": 7253375870607, + "updated_at": "2023-01-02T00:00:00Z", + } ] } + requests_mock.get("https://subdomain.zendesk.com/api/v2/community/posts/7253375870607/votes", json=post_votes_response) + + stream = PostVotes(subdomain="subdomain", start_date="2020-01-01T00:00:00Z") + records = read_full_refresh(stream) + assert records == post_votes_response.get("votes") + + +def test_read_post_comment_votes_stream(requests_mock): + post_response = { + "posts": [{"id": 7253375870607, "title": "Test_post", "created_at": "2023-01-01T00:00:00Z", "updated_at": "2023-01-01T00:00:00Z"}] + } requests_mock.get("https://subdomain.zendesk.com/api/v2/community/posts", json=post_response) post_comments_response = { "comments": [ - {"author_id": 89567, "body": "Test_comment for Test_post", "id": 35467, "post_id": 7253375870607} + { + "author_id": 89567, + "body": "Test_comment for Test_post", + "id": 35467, + "post_id": 7253375870607, + "updated_at": "2023-01-02T00:00:00Z", + } ] } requests_mock.get("https://subdomain.zendesk.com/api/v2/community/posts/7253375870607/comments", json=post_comments_response) - votes = [{"id": 35467, "user_id": 888887, "value": -1}] - requests_mock.get("https://subdomain.zendesk.com/api/v2/community/posts/7253375870607/comments/35467/votes", - json={"votes": votes}) + votes = [{"id": 35467, "user_id": 888887, "value": -1, "updated_at": "2023-01-03T00:00:00Z"}] + requests_mock.get("https://subdomain.zendesk.com/api/v2/community/posts/7253375870607/comments/35467/votes", json={"votes": votes}) stream = PostCommentVotes(subdomain="subdomain", start_date="2020-01-01T00:00:00Z") records = read_full_refresh(stream) assert records == votes @@ -976,47 +1021,69 @@ def test_read_ticket_metric_events_request_params(requests_mock): first_page_response = { "ticket_metric_events": [ {"id": 1, "ticket_id": 123, "metric": "agent_work_time", "instance_id": 0, "type": "measure", "time": "2020-01-01T01:00:00Z"}, - {"id": 2, "ticket_id": 123, "metric": "pausable_update_time", "instance_id": 0, "type": "measure", "time": "2020-01-01T01:00:00Z"}, + { + "id": 2, + "ticket_id": 123, + "metric": "pausable_update_time", + "instance_id": 0, + "type": "measure", + "time": "2020-01-01T01:00:00Z", + }, {"id": 3, "ticket_id": 123, "metric": "reply_time", "instance_id": 0, "type": "measure", "time": "2020-01-01T01:00:00Z"}, - {"id": 4, "ticket_id": 123, "metric": "requester_wait_time", "instance_id": 1, "type": "activate", "time": "2020-01-01T01:00:00Z"} + { + "id": 4, + "ticket_id": 123, + "metric": "requester_wait_time", + "instance_id": 1, + "type": "activate", + "time": "2020-01-01T01:00:00Z", + }, ], - "meta": { - "has_more": True, - "after_cursor": "", - "before_cursor": "" - }, + "meta": {"has_more": True, "after_cursor": "", "before_cursor": ""}, "links": { "prev": "https://subdomain.zendesk.com/api/v2/incremental/ticket_metric_events.json?page%5Bbefore%5D=&page%5Bsize%5D=100&start_time=1577836800", - "next": "https://subdomain.zendesk.com/api/v2/incremental/ticket_metric_events.json?page%5Bafter%5D=&page%5Bsize%5D=100&start_time=1577836800" + "next": "https://subdomain.zendesk.com/api/v2/incremental/ticket_metric_events.json?page%5Bafter%5D=&page%5Bsize%5D=100&start_time=1577836800", }, - "end_of_stream": False + "end_of_stream": False, } second_page_response = { "ticket_metric_events": [ - {"id": 5163373143183, "ticket_id": 130, "metric": "reply_time", "instance_id": 1, "type": "fulfill", "time": "2022-07-18T16:39:48Z"}, - {"id": 5163373143311, "ticket_id": 130, "metric": "requester_wait_time", "instance_id": 0, "type": "measure", "time": "2022-07-18T16:39:48Z"} + { + "id": 5163373143183, + "ticket_id": 130, + "metric": "reply_time", + "instance_id": 1, + "type": "fulfill", + "time": "2022-07-18T16:39:48Z", + }, + { + "id": 5163373143311, + "ticket_id": 130, + "metric": "requester_wait_time", + "instance_id": 0, + "type": "measure", + "time": "2022-07-18T16:39:48Z", + }, ], - "meta": { - "has_more": False, - "after_cursor": "", - "before_cursor": "" - }, + "meta": {"has_more": False, "after_cursor": "", "before_cursor": ""}, "links": { "prev": "https://subdomain.zendesk.com/api/v2/incremental/ticket_metric_events.json?page%5Bbefore%5D=&page%5Bsize%5D=100&start_time=1577836800", - "next": "https://subdomain.zendesk.com/api/v2/incremental/ticket_metric_events.json?page%5Bbefore%5D=&page%5Bsize%5D=100&start_time=1577836800" + "next": "https://subdomain.zendesk.com/api/v2/incremental/ticket_metric_events.json?page%5Bbefore%5D=&page%5Bsize%5D=100&start_time=1577836800", }, - "end_of_stream": True + "end_of_stream": True, } - request_history = requests_mock.get("https://subdomain.zendesk.com/api/v2/incremental/ticket_metric_events", [ - {"json": first_page_response}, {"json": second_page_response}]) + request_history = requests_mock.get( + "https://subdomain.zendesk.com/api/v2/incremental/ticket_metric_events", + [{"json": first_page_response}, {"json": second_page_response}], + ) stream = TicketMetricEvents(subdomain="subdomain", start_date="2020-01-01T00:00:00Z") read_full_refresh(stream) assert request_history.call_count == 2 assert request_history.last_request.qs == {"page[after]": [""], "page[size]": ["100"], "start_time": ["1577836800"]} -@pytest.mark.parametrize("status_code",[(403),(404)]) +@pytest.mark.parametrize("status_code", [(403), (404)]) def test_read_tickets_comment(requests_mock, status_code): request_history = requests_mock.get( "https://subdomain.zendesk.com/api/v2/incremental/ticket_events.json", status_code=status_code, json={"error": "wrong permissions"} @@ -1024,3 +1091,23 @@ def test_read_tickets_comment(requests_mock, status_code): stream = TicketComments(subdomain="subdomain", start_date="2020-01-01T00:00:00Z") read_full_refresh(stream) assert request_history.call_count == 1 + + +def test_read_non_json_error(requests_mock, caplog): + requests_mock.get("https://subdomain.zendesk.com/api/v2/incremental/tickets/cursor.json", text="not_json_response") + stream = Tickets(subdomain="subdomain", start_date="2020-01-01T00:00:00Z") + expected_message = ( + "Skipping stream tickets: Non-JSON response received. Please ensure that you have enough permissions for this stream." + ) + read_full_refresh(stream) + assert expected_message in (record.message for record in caplog.records if record.levelname == "ERROR") + + +def test_read_ticket_audits_504_error(requests_mock, caplog): + requests_mock.get("https://subdomain.zendesk.com/api/v2/ticket_audits", status_code=504, text="upstream request timeout") + stream = TicketAudits(subdomain="subdomain", start_date="2020-01-01T00:00:00Z") + expected_message = ( + "Skipping stream `ticket_audits`. Timed out waiting for response: upstream request timeout..." + ) + read_full_refresh(stream) + assert expected_message in (record.message for record in caplog.records if record.levelname == "ERROR") diff --git a/airbyte-integrations/connectors/source-zendesk-talk/.coveragerc b/airbyte-integrations/connectors/source-zendesk-talk/.coveragerc new file mode 100644 index 000000000000..753140399d72 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-talk/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + source_zendesk_talk/run.py \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-talk/Dockerfile b/airbyte-integrations/connectors/source-zendesk-talk/Dockerfile deleted file mode 100644 index eb5a50eb68fa..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-talk/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.9-slim - -# Bash is installed for more convenient debugging. -RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* - -WORKDIR /airbyte/integration_code -COPY source_zendesk_talk ./source_zendesk_talk -COPY main.py ./ -COPY setup.py ./ -RUN pip install . - -ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" -ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] - -LABEL io.airbyte.version=0.1.9 -LABEL io.airbyte.name=airbyte/source-zendesk-talk diff --git a/airbyte-integrations/connectors/source-zendesk-talk/README.md b/airbyte-integrations/connectors/source-zendesk-talk/README.md index 887971c4ebdf..eb3f34750b87 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/README.md +++ b/airbyte-integrations/connectors/source-zendesk-talk/README.md @@ -6,22 +6,27 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development ### Prerequisites + **To iterate on this connector, make sure to complete this prerequisites section.** #### Minimum Python version required `= 3.7.0` #### Build & Activate Virtual Environment and install dependencies + From this connector directory, create a virtual environment: -``` + +```bash python -m venv .venv ``` This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your development environment of choice. To activate it from the terminal, run: -``` + +```bash source .venv/bin/activate pip install -r requirements.txt ``` + If you are in an IDE, follow your IDE's instructions to activate the virtualenv. Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is @@ -29,13 +34,8 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -From the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-zendesk-talk:build -``` - #### Create credentials + **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/zendesk-talk) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zendesk_talk/spec.json` file. Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. @@ -45,7 +45,8 @@ See `sample_files/sample_config.json` for a sample config file. and place them into `secrets/config.json`. ### Locally running the connector -``` + +```bash python main.py spec python main.py check --config secrets/config.json python main.py discover --config secrets/config.json @@ -54,76 +55,117 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image -#### Build -First, make sure you build the latest Docker image: +#### Use `airbyte-ci` to build your connector + +The Airbyte way of building this connector is to use our `airbyte-ci` tool. +You can follow install instructions [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1). +Then running the following command will build your connector: + +```bash +airbyte-ci connectors --name=source-zendesk-talk build ``` -docker build . -t airbyte/source-zendesk-talk:dev + +Once the command is done, you will find your connector image in your local docker registry: `airbyte/source-zendesk-talk:dev`. + +##### Customizing our build process + +When contributing on our connector you might need to customize the build process to add a system dependency or set an env var. +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: + +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") ``` -You can also build the connector image via Gradle: +#### Build your own connector image + +This connector is built using our dynamic built process in `airbyte-ci`. +The base image used to build it is defined within the metadata.yaml file under the `connectorBuildOptions`. +The build logic is defined using [Dagger](https://dagger.io/) [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/builds/python_connectors.py). +It does not rely on a Dockerfile. + +If you would like to patch our connector and build your own a simple approach would be to: + +1. Create your own Dockerfile based on the latest version of the connector image. + +```Dockerfile +FROM airbyte/source-zendesk-talk:latest + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -./gradlew :airbyte-integrations:connectors:source-zendesk-talk:airbyteDocker + +Please use this as an example. This is not optimized. + +2. Build your image: + +```bash +docker build -t airbyte/source-zendesk-talk:dev . +# Running the spec command against your patched connector +docker run airbyte/source-zendesk-talk:dev spec ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run + Then run any of the connector commands as follows: -``` + +```bash docker run --rm airbyte/source-zendesk-talk:dev spec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zendesk-talk:dev check --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zendesk-talk:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-zendesk-talk:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json ``` + ## Testing - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .'[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. -If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unittest run: -``` -./gradlew :airbyte-integrations:connectors:source-zendesk-talk:unitTest -``` -To run acceptance and custom integration tests run: -``` -./gradlew :airbyte-integrations:connectors:source-zendesk-talk:IntegrationTest +```bash +airbyte-ci connectors --name=source-zendesk-talk test ``` +### Customizing acceptance Tests + +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + ## Dependency Management + All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: + * required for your connector to work need to go to `MAIN_REQUIREMENTS` list. * required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector + You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master + +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-zendesk-talk test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/zendesk-talk.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-zendesk-talk/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zendesk-talk/acceptance-test-config.yml index 220f3dbdaea5..29f7464fc66c 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-zendesk-talk/acceptance-test-config.yml @@ -6,7 +6,7 @@ test_strictness_level: "high" acceptance_tests: spec: tests: - - spec_path: "source_zendesk_talk/spec.json" + - spec_path: "source_zendesk_talk/spec.json" connection: tests: - config_path: "secrets/config.json" @@ -17,12 +17,12 @@ acceptance_tests: status: "succeed" discovery: tests: - - config_path: "secrets/config.json" - backward_compatibility_tests_config: - disable_for_version: "0.1.3" - - config_path: "secrets/config_old.json" - backward_compatibility_tests_config: - disable_for_version: "0.1.3" + - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.3" + - config_path: "secrets/config_old.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.3" basic_read: tests: - config_path: "secrets/config.json" @@ -44,9 +44,9 @@ acceptance_tests: fail_on_extra_columns: false incremental: tests: - - config_path: "secrets/config.json" - future_state: - future_state_path: "integration_tests/abnormal_state.json" + - config_path: "secrets/config.json" + future_state: + future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-zendesk-talk/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-zendesk-talk/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-talk/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-zendesk-talk/build.gradle b/airbyte-integrations/connectors/source-zendesk-talk/build.gradle deleted file mode 100644 index 51aa0c759152..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-talk/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_zendesk_talk' -} diff --git a/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/expected_records.jsonl index 056bf233dc49..2620572a5677 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/expected_records.jsonl +++ b/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/expected_records.jsonl @@ -4,14 +4,14 @@ {"stream": "addresses", "data": {"id": 360000047915, "name": "Fake Zendesk 998", "street": "1019 Market Street", "zip": "94103", "city": "San Francisco", "state": null, "province": "California", "country_code": "US", "provider_reference": "ADa89d87601b4f38b45ca172ba36bc4c36"}, "emitted_at": 1674159463525} {"stream": "addresses", "data": {"id": 360000049276, "name": "Fake Zendesk 997", "street": "1019 Market Street", "zip": "94103", "city": "San Francisco", "state": null, "province": "California", "country_code": "US", "provider_reference": "AD1b03f6250ae793c562f9290a47404e5b"}, "emitted_at": 1674159463525} {"stream": "agents_activity", "data": {"name": "Team Airbyte", "agent_id": 360786799676, "via": "client", "avatar_url": "https://d3v-airbyte.zendesk.com/system/photos/7282824912911/Airbyte_logo_220x220_thumb.png", "forwarding_number": null, "average_talk_time": 0, "calls_accepted": 0, "calls_denied": 0, "calls_missed": 0, "online_time": 0, "available_time": 0, "total_call_duration": 0, "total_talk_time": 0, "total_wrap_up_time": 0, "away_time": 0, "call_status": null, "agent_state": "offline", "transfers_only_time": 0, "average_wrap_up_time": 0, "accepted_transfers": 0, "started_transfers": 0, "calls_put_on_hold": 0, "average_hold_time": 0, "total_hold_time": 0, "started_third_party_conferences": 0, "accepted_third_party_conferences": 0}, "emitted_at": 1688470692771} -{"stream": "calls", "data": {"id": 360088814475, "created_at": "2021-04-01T13:42:47Z", "updated_at": "2021-04-01T14:23:15Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "failed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 18, "exceeded_queue_wait_time": null, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": null, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": null, "ivr_routed_to": null, "callback": null, "callback_source": null, "default_group": false, "ivr_action": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["silence"], "line": "+12059531462", "line_id": 360000121575, "line_type": "phone"}, "emitted_at": 1674159472100} -{"stream": "calls", "data": {"id": 360120314196, "created_at": "2021-10-20T15:16:31Z", "updated_at": "2021-10-20T15:56:54Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 8, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"], "line": "+12059531462", "line_id": 360000121575, "line_type": "phone"}, "emitted_at": 1674159472101} -{"stream": "calls", "data": {"id": 360121169675, "created_at": "2021-10-20T15:16:42Z", "updated_at": "2021-10-20T15:57:03Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 7, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"], "line": "+12059531462", "line_id": 360000121575, "line_type": "phone"}, "emitted_at": 1674159472101} -{"stream": "calls", "data": {"id": 360121166995, "created_at": "2021-10-20T13:22:25Z", "updated_at": "2021-10-20T16:22:34Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 6, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"], "line": "+12059531462", "line_id": 360000121575, "line_type": "phone"}, "emitted_at": 1674159472102} -{"stream": "calls", "data": {"id": 360120313416, "created_at": "2021-10-20T14:34:26Z", "updated_at": "2021-10-20T17:34:36Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 349, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 6, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"], "line": "+12059531462", "line_id": 360000121575, "line_type": "phone"}, "emitted_at": 1674159472102} -{"stream": "calls", "data": {"id": 360121168815, "created_at": "2021-10-20T14:40:05Z", "updated_at": "2021-10-20T17:40:25Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 17, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"], "line": "+12059531462", "line_id": 360000121575, "line_type": "phone"}, "emitted_at": 1674159472102} -{"stream": "calls", "data": {"id": 360120313656, "created_at": "2021-10-20T14:49:50Z", "updated_at": "2021-10-20T17:50:02Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 5, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"], "line": "+12059531462", "line_id": 360000121575, "line_type": "phone"}, "emitted_at": 1674159472103} -{"stream": "calls", "data": {"id": 360120313676, "created_at": "2021-10-20T14:50:08Z", "updated_at": "2021-10-20T17:50:16Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 6, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"], "line": "+12059531462", "line_id": 360000121575, "line_type": "phone"}, "emitted_at": 1674159472103} +{"stream": "calls", "data": {"id": 360088814475, "created_at": "2021-04-01T13:42:47Z", "updated_at": "2021-04-01T14:23:15Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "failed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 18, "exceeded_queue_wait_time": null, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": null, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": null, "ivr_routed_to": null, "callback": null, "callback_source": null, "default_group": false, "ivr_action": null, "line": null, "line_id": null, "line_type": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["silence"]}, "emitted_at": 1701476700438} +{"stream": "calls", "data": {"id": 360120314196, "created_at": "2021-10-20T15:16:31Z", "updated_at": "2021-10-20T15:56:54Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 8, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "line": null, "line_id": null, "line_type": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"]}, "emitted_at": 1701476700440} +{"stream": "calls", "data": {"id": 360121169675, "created_at": "2021-10-20T15:16:42Z", "updated_at": "2021-10-20T15:57:03Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 7, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "line": null, "line_id": null, "line_type": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"]}, "emitted_at": 1701476700441} +{"stream": "calls", "data": {"id": 360120313416, "created_at": "2021-10-20T14:34:26Z", "updated_at": "2021-10-20T17:34:36Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 349, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 6, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "line": null, "line_id": null, "line_type": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"]}, "emitted_at": 1701476700441} +{"stream": "calls", "data": {"id": 360120313416, "created_at": "2021-10-20T14:34:26Z", "updated_at": "2021-10-20T17:34:36Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 349, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 6, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "line": null, "line_id": null, "line_type": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"]}, "emitted_at": 1701476700441} +{"stream": "calls", "data": {"id": 360121168815, "created_at": "2021-10-20T14:40:05Z", "updated_at": "2021-10-20T17:40:25Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 17, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "line": null, "line_id": null, "line_type": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"]}, "emitted_at": 1701476700442} +{"stream": "calls", "data": {"id": 360120313656, "created_at": "2021-10-20T14:49:50Z", "updated_at": "2021-10-20T17:50:02Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 5, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 1, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "line": null, "line_id": null, "line_type": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"]}, "emitted_at": 1701476700442} +{"stream": "calls", "data": {"id": 360120313696, "created_at": "2021-10-20T14:50:28Z", "updated_at": "2021-10-20T17:50:38Z", "agent_id": null, "call_charge": "0.003", "consultation_time": 0, "completion_status": "completed", "customer_id": null, "customer_requested_voicemail": false, "direction": "outbound", "duration": 288, "exceeded_queue_wait_time": false, "hold_time": 0, "minutes_billed": 5, "outside_business_hours": false, "phone_number_id": 360000121575, "phone_number": "+12059531462", "ticket_id": null, "time_to_answer": null, "voicemail": false, "wait_time": 0, "wrap_up_time": 0, "ivr_time_spent": null, "ivr_hops": null, "ivr_destination_group_name": null, "talk_time": 0, "ivr_routed_to": null, "callback": false, "callback_source": null, "default_group": false, "ivr_action": null, "line": null, "line_id": null, "line_type": null, "overflowed": false, "overflowed_to": null, "recording_control_interactions": 0, "recording_time": 0, "not_recording_time": 0, "call_recording_consent": "always", "call_recording_consent_action": null, "call_recording_consent_keypress": null, "call_group_id": null, "call_channel": null, "quality_issues": ["none"]}, "emitted_at": 1701476700442} {"stream": "call_legs", "data": {"id": 360167085115, "created_at": "2021-04-01T13:42:47Z", "updated_at": "2021-04-01T14:23:14Z", "agent_id": 360786799676, "user_id": 0, "duration": 18, "hold_time": 0, "wrap_up_time": 0, "available_via": "browser", "forwarded_to": null, "consultation_from": null, "transferred_from": null, "transferred_to": null, "minutes_billed": 1, "call_charge": "0.003", "completion_status": "completed", "consultation_time": null, "talk_time": 0, "consultation_to": null, "conference_time": null, "conference_from": null, "conference_to": null, "quality_issues": ["silence"], "call_id": 360088814475, "type": "agent"}, "emitted_at": 1674159475493} {"stream": "call_legs", "data": {"id": 360222253536, "created_at": "2021-10-20T13:22:27Z", "updated_at": "2021-10-20T14:02:47Z", "agent_id": 0, "user_id": null, "duration": 4, "hold_time": 0, "wrap_up_time": null, "available_via": null, "forwarded_to": null, "consultation_from": null, "transferred_from": null, "transferred_to": null, "minutes_billed": null, "call_charge": "0.0", "completion_status": "completed", "consultation_time": null, "talk_time": 0, "consultation_to": null, "conference_time": null, "conference_from": null, "conference_to": null, "quality_issues": ["none"], "call_id": 360121166995, "type": "customer"}, "emitted_at": 1674159475493} {"stream": "call_legs", "data": {"id": 360223248835, "created_at": "2021-10-20T14:34:28Z", "updated_at": "2021-10-20T14:40:20Z", "agent_id": 0, "user_id": null, "duration": 347, "hold_time": 0, "wrap_up_time": null, "available_via": null, "forwarded_to": null, "consultation_from": null, "transferred_from": null, "transferred_to": null, "minutes_billed": null, "call_charge": "0.0", "completion_status": "completed", "consultation_time": null, "talk_time": 0, "consultation_to": null, "conference_time": null, "conference_from": null, "conference_to": null, "quality_issues": ["information_not_available"], "call_id": 360120313416, "type": "customer"}, "emitted_at": 1674159475494} diff --git a/airbyte-integrations/connectors/source-zendesk-talk/main.py b/airbyte-integrations/connectors/source-zendesk-talk/main.py index 88d4616c2155..679ec2c79a78 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/main.py +++ b/airbyte-integrations/connectors/source-zendesk-talk/main.py @@ -2,12 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - -import sys - -from airbyte_cdk.entrypoint import launch -from source_zendesk_talk import SourceZendeskTalk +from source_zendesk_talk.run import run if __name__ == "__main__": - source = SourceZendeskTalk() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-zendesk-talk/metadata.yaml b/airbyte-integrations/connectors/source-zendesk-talk/metadata.yaml index 98b742f21721..8fb1cde85c38 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/metadata.yaml +++ b/airbyte-integrations/connectors/source-zendesk-talk/metadata.yaml @@ -1,13 +1,19 @@ data: + ab_internal: + ql: 200 + sl: 200 allowedHosts: hosts: - ${subdomain}.zendesk.com - zendesk.com + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorSubtype: api connectorType: source definitionId: c8630570-086d-4a40-99ae-ea5b18673071 - dockerImageTag: 0.1.9 + dockerImageTag: 0.1.11 dockerRepository: airbyte/source-zendesk-talk + documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-talk githubIssueLabel: source-zendesk-talk icon: zendesk-talk.svg license: MIT @@ -18,11 +24,7 @@ data: oss: enabled: true releaseStage: generally_available - documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-talk + supportLevel: certified tags: - language:python - ab_internal: - sl: 200 - ql: 400 - supportLevel: certified metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-zendesk-talk/setup.py b/airbyte-integrations/connectors/source-zendesk-talk/setup.py index e0e910f6461b..204a1c5cded5 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/setup.py +++ b/airbyte-integrations/connectors/source-zendesk-talk/setup.py @@ -15,6 +15,11 @@ ] setup( + entry_points={ + "console_scripts": [ + "source-zendesk-talk=source_zendesk_talk.run:run", + ], + }, name="source_zendesk_talk", description="Source implementation for Zendesk Talk.", author="Airbyte", diff --git a/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/run.py b/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/run.py new file mode 100644 index 000000000000..154690ce67d1 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/run.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_zendesk_talk import SourceZendeskTalk + + +def run(): + source = SourceZendeskTalk() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-zenefits/Dockerfile b/airbyte-integrations/connectors/source-zenefits/Dockerfile index 428b41031501..ed00456a6ec6 100644 --- a/airbyte-integrations/connectors/source-zenefits/Dockerfile +++ b/airbyte-integrations/connectors/source-zenefits/Dockerfile @@ -34,5 +34,5 @@ COPY source_zenefits ./source_zenefits ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-zenefits diff --git a/airbyte-integrations/connectors/source-zenefits/README.md b/airbyte-integrations/connectors/source-zenefits/README.md index 111b2c614f45..ce0e576087c5 100644 --- a/airbyte-integrations/connectors/source-zenefits/README.md +++ b/airbyte-integrations/connectors/source-zenefits/README.md @@ -7,46 +7,13 @@ Follow the steps in the given link [Here](https://developers.zenefits.com/docs/s # Zenefits Source -This is the repository for the Zenefits source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/zenefits). +This is the repository for the Zenefits configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/zenefits). ## Local development -### Prerequisites -**To iterate on this connector, make sure to complete this prerequisites section.** - -#### Minimum Python version required `= 3.7.0` - -#### Build & Activate Virtual Environment and install dependencies -From this connector directory, create a virtual environment: -``` -python -m venv .venv -``` - -This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your -development environment of choice. To activate it from the terminal, run: -``` -source .venv/bin/activate -pip install -r requirements.txt -pip install '.[tests]' -``` -If you are in an IDE, follow your IDE's instructions to activate the virtualenv. - -Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is -used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. -If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything -should work as you expect. - -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-zenefits:build -``` - #### Create credentials -**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/zenefits) +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/zenefits) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zenefits/spec.yaml` file. Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. @@ -54,28 +21,21 @@ See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source zenefits test creds` and place them into `secrets/config.json`. -### Locally running the connector -``` -python main.py spec -python main.py check --config secrets/config.json -python main.py discover --config secrets/config.json -python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json -``` - ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-zenefits:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-zenefits build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-zenefits:airbyteDocker +An image will be built with the tag `airbyte/source-zenefits:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-zenefits:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -85,44 +45,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zenefits:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zenefits:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-zenefits:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-zenefits test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-zenefits:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-zenefits:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -132,8 +64,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-zenefits test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/zenefits.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-zenefits/__init__.py b/airbyte-integrations/connectors/source-zenefits/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-zenefits/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-zenefits/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zenefits/acceptance-test-config.yml index 327fb42674ae..eef49e8d987d 100644 --- a/airbyte-integrations/connectors/source-zenefits/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-zenefits/acceptance-test-config.yml @@ -3,7 +3,7 @@ connector_image: airbyte/source-zenefits:dev tests: spec: - - spec_path: "secrets/spec.json" + - spec_path: "source_zenefits/spec.yaml" connection: - config_path: "secrets/config.json" status: "succeed" diff --git a/airbyte-integrations/connectors/source-zenefits/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-zenefits/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-zenefits/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-zenefits/build.gradle b/airbyte-integrations/connectors/source-zenefits/build.gradle deleted file mode 100644 index 7a67e811b0b0..000000000000 --- a/airbyte-integrations/connectors/source-zenefits/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_zenefits' -} diff --git a/airbyte-integrations/connectors/source-zenefits/integration_tests/__init__.py b/airbyte-integrations/connectors/source-zenefits/integration_tests/__init__.py index cb65d40dabd2..c941b3045795 100644 --- a/airbyte-integrations/connectors/source-zenefits/integration_tests/__init__.py +++ b/airbyte-integrations/connectors/source-zenefits/integration_tests/__init__.py @@ -1,3 +1,3 @@ # -# Copyright (c) 2022s Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # diff --git a/airbyte-integrations/connectors/source-zenefits/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-zenefits/integration_tests/acceptance.py index ea1ca1161ee2..82823254d266 100644 --- a/airbyte-integrations/connectors/source-zenefits/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-zenefits/integration_tests/acceptance.py @@ -10,5 +10,5 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - + """This fixture is a placeholder for external resources that acceptance test might require.""" yield diff --git a/airbyte-integrations/connectors/source-zenefits/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-zenefits/integration_tests/sample_config.json new file mode 100644 index 000000000000..0bfb2f4fac21 --- /dev/null +++ b/airbyte-integrations/connectors/source-zenefits/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "token": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +} diff --git a/airbyte-integrations/connectors/source-zenefits/metadata.yaml b/airbyte-integrations/connectors/source-zenefits/metadata.yaml index 9c96ce0d6f3d..b5eec18af058 100644 --- a/airbyte-integrations/connectors/source-zenefits/metadata.yaml +++ b/airbyte-integrations/connectors/source-zenefits/metadata.yaml @@ -1,24 +1,28 @@ data: + allowedHosts: + hosts: + - api.zenefits.com + registries: + cloud: + enabled: false + oss: + enabled: true connectorSubtype: api connectorType: source definitionId: 8baba53d-2fe3-4e33-bc85-210d0eb62884 - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-zenefits githubIssueLabel: source-zenefits - icon: zenefits.svg + icon: icon.svg license: MIT name: Zenefits - registries: - cloud: - enabled: false - oss: - enabled: true + releaseDate: 2022-08-24 releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/zenefits - tags: - - language:python ab_internal: sl: 100 ql: 100 supportLevel: community + tags: + - language:low-code metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-zenefits/setup.py b/airbyte-integrations/connectors/source-zenefits/setup.py index cdd39928c1ce..5cc7df205794 100644 --- a/airbyte-integrations/connectors/source-zenefits/setup.py +++ b/airbyte-integrations/connectors/source-zenefits/setup.py @@ -6,14 +6,10 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk", ] -TEST_REQUIREMENTS = [ - "requests-mock~=1.9.3", - "pytest~=6.1", - "pytest-mock~=3.6.1", -] +TEST_REQUIREMENTS = ["requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1"] setup( name="source_zenefits", diff --git a/airbyte-integrations/connectors/source-zenefits/source_zenefits/__init__.py b/airbyte-integrations/connectors/source-zenefits/source_zenefits/__init__.py index 69bcc1a443fd..096b7e754b6e 100644 --- a/airbyte-integrations/connectors/source-zenefits/source_zenefits/__init__.py +++ b/airbyte-integrations/connectors/source-zenefits/source_zenefits/__init__.py @@ -1,7 +1,8 @@ # -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. # + from .source import SourceZenefits __all__ = ["SourceZenefits"] diff --git a/airbyte-integrations/connectors/source-zenefits/source_zenefits/manifest.yaml b/airbyte-integrations/connectors/source-zenefits/source_zenefits/manifest.yaml new file mode 100644 index 000000000000..89c4774b1186 --- /dev/null +++ b/airbyte-integrations/connectors/source-zenefits/source_zenefits/manifest.yaml @@ -0,0 +1,444 @@ +version: 0.35.0 +type: DeclarativeSource +check: + type: CheckStream + stream_names: + - people +streams: + - type: DeclarativeStream + name: people + primary_key: [] + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.zenefits.com/ + path: core/people + http_method: GET + request_parameters: {} + request_headers: + Content-Type: application/json + Accept: application/json + request_body_json: {} + authenticator: + type: BearerAuthenticator + api_token: "{{ config['token'] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + inject_into: request_parameter + type: RequestOption + field_name: limit + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.data.next_url }}" + page_size: 100 + stop_condition: '{{ response.data.next_url == "null" }}' + - type: DeclarativeStream + name: employments + primary_key: [] + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.zenefits.com/ + path: core/employments + http_method: GET + request_parameters: {} + request_headers: + Content-Type: application/json + Accept: application/json + request_body_json: {} + authenticator: + type: BearerAuthenticator + api_token: "{{ config['token'] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + inject_into: request_parameter + field_name: limit + type: RequestOption + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.data.next_url }}" + stop_condition: '{{ response.data.next_url == "null" }}' + page_size: 100 + - type: DeclarativeStream + name: departments + primary_key: [] + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.zenefits.com/ + path: core/departments + http_method: GET + request_parameters: {} + request_headers: + Content-Type: application/json + Accept: application/json + request_body_json: {} + authenticator: + type: BearerAuthenticator + api_token: "{{ config['token'] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + inject_into: request_parameter + field_name: limit + type: RequestOption + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.data.next_url }}" + stop_condition: '{{ response.data.next_url == "null" }}' + page_size: 100 + - type: DeclarativeStream + name: locations + primary_key: [] + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.zenefits.com/ + path: core/locations + http_method: GET + request_parameters: {} + request_headers: + Content-Type: application/json + Accept: application/json + request_body_json: {} + authenticator: + type: BearerAuthenticator + api_token: "{{ config['token'] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + inject_into: request_parameter + field_name: limit + type: RequestOption + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.data.next_url }}" + stop_condition: '{{ response.data.next_url == "null" }}' + page_size: 100 + - type: DeclarativeStream + name: labor_groups + primary_key: [] + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.zenefits.com/ + path: core/labor_groups + http_method: GET + request_parameters: {} + request_headers: + Content-Type: application/json + Accept: application/json + request_body_json: {} + authenticator: + type: BearerAuthenticator + api_token: "{{ config['token'] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + inject_into: request_parameter + field_name: limit + type: RequestOption + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.data.next_url }}" + stop_condition: '{{ response.data.next_url == "null" }}' + page_size: 100 + - type: DeclarativeStream + name: labor_group_types + primary_key: [] + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.zenefits.com/ + path: core/labor_group_types + http_method: GET + request_parameters: {} + request_headers: + Content-Type: application/json + Accept: application/json + request_body_json: {} + authenticator: + type: BearerAuthenticator + api_token: "{{ config['token'] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + inject_into: request_parameter + field_name: limit + type: RequestOption + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.data.next_url }}" + stop_condition: '{{ response.data.next_url == "null" }}' + page_size: 100 + - type: DeclarativeStream + name: custom_fields + primary_key: [] + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.zenefits.com/ + path: core/custom_fields + http_method: GET + request_parameters: {} + request_headers: + Content-Type: application/json + Accept: application/json + request_body_json: {} + authenticator: + type: BearerAuthenticator + api_token: "{{ config['token'] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + inject_into: request_parameter + field_name: limit + type: RequestOption + pagination_strategy: + type: CursorPagination + page_size: 100 + cursor_value: "{{ response.data.next_url }}" + stop_condition: '{{ response.data.next_url == "null" }}' + - type: DeclarativeStream + name: custom_field_values + primary_key: [] + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.zenefits.com/ + path: core/custom_field_values + http_method: GET + request_parameters: {} + request_headers: + Content-Type: application/json + Accept: application/json + request_body_json: {} + authenticator: + type: BearerAuthenticator + api_token: "{{ config['token'] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + inject_into: request_parameter + field_name: limit + type: RequestOption + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.data.next_url }}" + stop_condition: '{{ response.data.next_url == "null" }}' + page_size: 100 + - type: DeclarativeStream + name: vacation_requests + primary_key: [] + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.zenefits.com/ + path: time_off/vacation_requests + http_method: GET + request_parameters: {} + request_headers: + Content-Type: application/json + Accept: application/json + request_body_json: {} + authenticator: + type: BearerAuthenticator + api_token: "{{ config['token'] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + inject_into: request_parameter + field_name: limit + type: RequestOption + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.data.next_url }}" + stop_condition: '{{ response.data.next_url == "null" }}' + page_size: 100 + - type: DeclarativeStream + name: vacation_types + primary_key: [] + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.zenefits.com/ + path: time_off/vacation_types + http_method: GET + request_parameters: {} + request_headers: + Content-Type: application/json + Accept: application/json + request_body_json: {} + authenticator: + type: BearerAuthenticator + api_token: "{{ config['token'] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + inject_into: request_parameter + field_name: limit + type: RequestOption + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.data.next_url }}" + stop_condition: '{{ response.data.next_url == "null" }}' + page_size: 100 + - type: DeclarativeStream + name: time_durations + primary_key: [] + retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: https://api.zenefits.com/ + path: time_attendance/time_durations + http_method: GET + request_parameters: {} + request_headers: + Content-Type: application/json + Accept: application/json + request_body_json: {} + authenticator: + type: BearerAuthenticator + api_token: "{{ config['token'] }}" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - data + - data + paginator: + type: DefaultPaginator + page_token_option: + type: RequestPath + page_size_option: + inject_into: request_parameter + field_name: limit + type: RequestOption + pagination_strategy: + type: CursorPagination + cursor_value: "{{ response.data.next_url }}" + stop_condition: '{{ response.data.next_url == "null" }}' + page_size: 100 + +spec: + type: Spec + documentation_url: https://docs.airbyte.io/integrations/sources/zenefits + connection_specification: + "$schema": http://json-schema.org/draft-07/schema# + title: Zenefits Spec + type: object + required: + - token + additionalProperties: true + properties: + token: + title: token + type: string + description: + Use Sync with Zenefits button on the link given on the readme file, + and get the token to access the api + airbyte_secret: true diff --git a/airbyte-integrations/connectors/source-zenefits/source_zenefits/schemas/employments.json b/airbyte-integrations/connectors/source-zenefits/source_zenefits/schemas/employments.json index 0ae59f207732..b307f072c879 100644 --- a/airbyte-integrations/connectors/source-zenefits/source_zenefits/schemas/employments.json +++ b/airbyte-integrations/connectors/source-zenefits/source_zenefits/schemas/employments.json @@ -19,6 +19,18 @@ "hire_date": { "type": ["string", "null"] }, + "amount_type": { + "type": ["string", "null"] + }, + "annual_salary": { + "type": ["string", "null"] + }, + "comp_type": { + "type": ["string", "null"] + }, + "pay_rate": { + "type": ["string", "null"] + }, "employment_type": { "type": ["string", "null"] }, diff --git a/airbyte-integrations/connectors/source-zenefits/source_zenefits/schemas/people.json b/airbyte-integrations/connectors/source-zenefits/source_zenefits/schemas/people.json index 610ecefb31a5..86da2d58ff9b 100644 --- a/airbyte-integrations/connectors/source-zenefits/source_zenefits/schemas/people.json +++ b/airbyte-integrations/connectors/source-zenefits/source_zenefits/schemas/people.json @@ -138,9 +138,53 @@ } } }, + "banks": { + "type": "object", + "properties": { + "ref_object": { + "type": ["string", "null"] + }, + "object": { + "type": ["string", "null"] + }, + "url": { + "type": ["string", "null"] + } + } + }, + "date_of_birth": { + "type": ["string", "null"] + }, "status": { "type": ["string", "null"] }, + "federal_filing_status": { + "type": ["string", "null"] + }, + "gender": { + "type": ["string", "null"] + }, + "personal_email": { + "type": ["string", "null"] + }, + "personal_phone": { + "type": ["string", "null"] + }, + "personal_pronoun": { + "type": ["string", "null"] + }, + "photo_thumbnail_url": { + "type": ["string", "null"] + }, + "photo_url": { + "type": ["string", "null"] + }, + "social_security_number": { + "type": ["string", "null"] + }, + "is_full_admin": { + "type": ["boolean", "null"] + }, "last_name": { "type": ["string", "null"] }, diff --git a/airbyte-integrations/connectors/source-zenefits/source_zenefits/schemas/vacation_types.json b/airbyte-integrations/connectors/source-zenefits/source_zenefits/schemas/vacation_types.json index d7ca01218948..be0ac648836b 100644 --- a/airbyte-integrations/connectors/source-zenefits/source_zenefits/schemas/vacation_types.json +++ b/airbyte-integrations/connectors/source-zenefits/source_zenefits/schemas/vacation_types.json @@ -11,6 +11,9 @@ "url": { "type": ["string", "null"] }, + "counts_as": { + "type": ["string", "null"] + }, "company": { "type": "object", "properties": { diff --git a/airbyte-integrations/connectors/source-zenefits/source_zenefits/source.py b/airbyte-integrations/connectors/source-zenefits/source_zenefits/source.py index e4609ae4037c..756264647ed1 100644 --- a/airbyte-integrations/connectors/source-zenefits/source_zenefits/source.py +++ b/airbyte-integrations/connectors/source-zenefits/source_zenefits/source.py @@ -2,181 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple -from urllib.parse import parse_qs, urlparse +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource -import requests -from airbyte_cdk.sources import AbstractSource -from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. +WARNING: Do not modify this file. +""" -# Basic full refresh stream -class ZenefitsStream(HttpStream, ABC): - LIMIT = 100 - url_base = "https://api.zenefits.com/" - - def __init__(self, token: str, **kwargs): - super().__init__(**kwargs) - self.token = token - - def request_params( - self, stream_state: Mapping[str, any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - - params = {"limit": self.LIMIT} - if next_page_token: - params = next_page_token - return params - - def request_headers(self, **kwargs) -> Mapping[str, Any]: - return {"Authorization": f"Bearer {self.token}", "Content-Type": "application/json", "Accept": "application/json"} - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - response_json = response.json().get("data") - next_page_url = response_json.get("next_url") - if next_page_url: - next_url = urlparse(next_page_url) - next_params = parse_qs(next_url.query) - return next_params - - return None - - def parse_response( - self, - response: requests.Response, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[Mapping]: - - return response.json().get("data", {}).get("data") - - -# Employee -class People(ZenefitsStream): - - primary_key = None - - def path(self, **kwargs) -> str: - - return "core/people" - - -# Employee Employment details -class Employments(ZenefitsStream): - - primary_key = None - - def path(self, **kwargs) -> str: - return "core/employments" - - -# Departments -class Departments(ZenefitsStream): - - primary_key = None - - def path(self, **kwargs) -> str: - return "core/departments" - - -# locations -class Locations(ZenefitsStream): - - primary_key = None - - def path(self, **kwargs) -> str: - return "core/locations" - - -# labor_groups -class Labor_groups(ZenefitsStream): - - primary_key = None - - def path(self, **kwargs) -> str: - return "core/labor_groups" - - -# labor_groups_types -class Labor_group_types(ZenefitsStream): - - primary_key = None - - def path(self, **kwargs) -> str: - return "core/labor_group_types" - - -# custom_fields -class Custom_fields(ZenefitsStream): - - primary_key = None - - def path(self, **kwargs) -> str: - return "core/custom_fields" - - -# custom_field_values -class Custom_field_values(ZenefitsStream): - - primary_key = None - - def path(self, **kwargs) -> str: - return "core/custom_field_values" - - -# Vacation Requested -class Vacation_requests(ZenefitsStream): - primary_key = None - - def path(self, **kwargs) -> str: - return "time_off/vacation_requests" - - -# Vacation Types -class Vacation_types(ZenefitsStream): - primary_key = None - - def path(self, **kwargs) -> str: - return "time_off/vacation_types" - - -# Time Durations -class Time_durations(ZenefitsStream): - primary_key = None - - def path(self, **kwargs) -> str: - return "time_attendance/time_durations" - - -class SourceZenefits(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - token = config["token"] - headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json", "Accept": "application/json"} - url = "https://api.zenefits.com/core/people" - - try: - session = requests.get(url, headers=headers) - session.raise_for_status() - return True, None - except requests.exceptions.RequestException as e: - return False, e - - def streams(self, config: Mapping[str, Any]) -> List[Stream]: - args = {"token": config["token"]} - return [ - People(**args), - Employments(**args), - Vacation_requests(**args), - Vacation_types(**args), - Time_durations(**args), - Departments(**args), - Locations(**args), - Labor_group_types(**args), - Custom_fields(**args), - Custom_field_values(**args), - Labor_groups(**args), - ] +# Declarative Source +class SourceZenefits(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-zenefits/source_zenefits/spec.json b/airbyte-integrations/connectors/source-zenefits/source_zenefits/spec.json deleted file mode 100644 index c30e4056ffd2..000000000000 --- a/airbyte-integrations/connectors/source-zenefits/source_zenefits/spec.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "documentationUrl": "https://docsurl.com", - "connectionSpecification": { - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Zenefits Integration Spec", - "type": "object", - "required": ["token"], - "additionalProperties": false, - "properties": { - "token": { - "title": "token", - "type": "string", - "description": "Use Sync with Zenefits button on the link given on the readme file, and get the token to access the api" - } - } - } -} diff --git a/airbyte-integrations/connectors/source-zenefits/unit_tests/test_source.py b/airbyte-integrations/connectors/source-zenefits/unit_tests/test_source.py deleted file mode 100644 index d2efa8f77d0c..000000000000 --- a/airbyte-integrations/connectors/source-zenefits/unit_tests/test_source.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import MagicMock - -from pytest import fixture -from source_zenefits.source import SourceZenefits - - -@fixture() -def config(request): - args = {"token": "YXNmnhkjf"} - return args - - -def test_check_connection(mocker, config): - source = SourceZenefits() - logger_mock = MagicMock() - (connection_status, error) = source.check_connection(logger_mock, config) - expected_status = False - assert connection_status == expected_status - - -def test_streams(mocker, config): - source = SourceZenefits() - streams = source.streams(config) - expected_streams_number = 11 - assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-zenefits/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-zenefits/unit_tests/test_streams.py deleted file mode 100644 index 6c737978849f..000000000000 --- a/airbyte-integrations/connectors/source-zenefits/unit_tests/test_streams.py +++ /dev/null @@ -1,86 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from http import HTTPStatus - -# from unittest import result -from unittest.mock import MagicMock - -import pytest -from source_zenefits.source import ZenefitsStream - - -@pytest.fixture() -def config(request): - args = {"token": "YXNmnhkjf"} - return args - - -@pytest.fixture -def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class - mocker.patch.object(ZenefitsStream, "path", "v0/example_endpoint") - mocker.patch.object(ZenefitsStream, "primary_key", "test_primary_key") - mocker.patch.object(ZenefitsStream, "__abstractmethods__", set()) - - -def test_request_params(patch_base_class, config): - stream = ZenefitsStream(**config) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {"limit": 100} - assert stream.request_params(**inputs) == expected_params - - -def test_next_page_token(patch_base_class, config): - stream = ZenefitsStream(**config) - response = MagicMock() - response.json.return_value = {"data": {"next_url": "https://api.zenefits.com/core/people?limit=100&starting_after=22334422"}} - inputs = {"response": response} - expected_token = {"limit": ["100"], "starting_after": ["22334422"]} - assert stream.next_page_token(**inputs) == expected_token - - -def test_request_headers(patch_base_class, config): - stream = ZenefitsStream(**config) - inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = {"Authorization": "Bearer YXNmnhkjf", "Content-Type": "application/json", "Accept": "application/json"} - assert stream.request_headers(**inputs) == expected_headers - - -def test_parse_response(patch_base_class, config): - stream = ZenefitsStream(**config) - response = MagicMock() - response.json.return_value = {"people": [{"id": "123", "name": "John Doe"}]} - inputs = {"response": response, "stream_state": MagicMock()} - expected_parsed_object = [{"id": "123", "name": "John Doe"}] - return stream.parse_response(**inputs) == expected_parsed_object - - -def test_http_method(patch_base_class, config): - stream = ZenefitsStream(**config) - expected_method = "GET" - assert stream.http_method == expected_method - - -@pytest.mark.parametrize( - ("http_status", "should_retry"), - [ - (HTTPStatus.OK, False), - (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), - (HTTPStatus.INTERNAL_SERVER_ERROR, True), - ], -) -def test_should_retry(patch_base_class, http_status, should_retry, config): - response_mock = MagicMock() - response_mock.status_code = http_status - stream = ZenefitsStream(**config) - assert stream.should_retry(response_mock) == should_retry - - -def test_backoff_time(patch_base_class, config): - response_mock = MagicMock() - stream = ZenefitsStream(**config) - expected_backoff_time = None - assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/airbyte-integrations/connectors/source-zenloop/README.md b/airbyte-integrations/connectors/source-zenloop/README.md index 0b1ab33403bd..a06504a69373 100644 --- a/airbyte-integrations/connectors/source-zenloop/README.md +++ b/airbyte-integrations/connectors/source-zenloop/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-zenloop:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/zenloop) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zenloop/spec.json` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-zenloop:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-zenloop build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-zenloop:airbyteDocker +An image will be built with the tag `airbyte/source-zenloop:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-zenloop:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zenloop:dev check --co docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zenloop:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-zenloop:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .'[tests]' -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-zenloop test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-zenloop:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-zenloop:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-zenloop test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/zenloop.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-zenloop/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zenloop/acceptance-test-config.yml index e5b5f839bec9..de6f12368cf2 100644 --- a/airbyte-integrations/connectors/source-zenloop/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-zenloop/acceptance-test-config.yml @@ -24,9 +24,6 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" - cursor_paths: - "answers": [ "states", 0, "cursor", "inserted_at" ] - "answers_survey_group": [ "states", 0, "cursor", "inserted_at" ] full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-zenloop/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-zenloop/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-zenloop/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-zenloop/build.gradle b/airbyte-integrations/connectors/source-zenloop/build.gradle deleted file mode 100644 index 7a306385ce64..000000000000 --- a/airbyte-integrations/connectors/source-zenloop/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_zenloop' -} diff --git a/airbyte-integrations/connectors/source-zoho-crm/README.md b/airbyte-integrations/connectors/source-zoho-crm/README.md index ecd4ce422d56..2a63cefbbc46 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/README.md +++ b/airbyte-integrations/connectors/source-zoho-crm/README.md @@ -30,14 +30,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-zoho-crm:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/zoho-crm) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zoho_crm/spec.json` file. @@ -57,18 +49,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-zoho-crm:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-zoho-crm build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-zoho-crm:airbyteDocker +An image will be built with the tag `airbyte/source-zoho-crm:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-zoho-crm:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -78,44 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zoho-crm:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zoho-crm:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-zoho-crm:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-zoho-crm test ``` -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests -``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -python -m pytest integration_tests -p integration_tests.acceptance -``` -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-zoho-crm:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-zoho-crm:integrationTest -``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -125,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-zoho-crm test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/zoho-crm.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-zoho-crm/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-zoho-crm/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-zoho-crm/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-zoho-crm/build.gradle b/airbyte-integrations/connectors/source-zoho-crm/build.gradle deleted file mode 100644 index 17109fe40df6..000000000000 --- a/airbyte-integrations/connectors/source-zoho-crm/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_zoho_crm' -} diff --git a/airbyte-integrations/connectors/source-zoho-crm/unit_tests/parametrize.py b/airbyte-integrations/connectors/source-zoho-crm/unit_tests/parametrize.py index 5a78794dc57c..31ad48a1a41c 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/unit_tests/parametrize.py +++ b/airbyte-integrations/connectors/source-zoho-crm/unit_tests/parametrize.py @@ -6,7 +6,9 @@ import pytest -TestCase = namedtuple("TestCase", ("json_type", "data_type", "length", "decimal_place", "api_name", "pick_list_values", "autonumber", "expected_values")) +TestCase = namedtuple( + "TestCase", ("json_type", "data_type", "length", "decimal_place", "api_name", "pick_list_values", "autonumber", "expected_values") +) datatype_inputs = pytest.mark.parametrize( @@ -171,7 +173,7 @@ "Field", [], None, - {"items": {"airbyte_type": "big_integer", "type": "string"}, "title": "Field", "type": "array"} + {"items": {"airbyte_type": "big_integer", "type": "string"}, "title": "Field", "type": "array"}, ), TestCase( "jsonarray", @@ -236,8 +238,8 @@ "Field", [], { - 'prefix': 'prefix', - 'suffix': 'suffix', + "prefix": "prefix", + "suffix": "suffix", }, { "format": "string", diff --git a/airbyte-integrations/connectors/source-zoho-crm/unit_tests/test_types.py b/airbyte-integrations/connectors/source-zoho-crm/unit_tests/test_types.py index 01dda0cb7438..a3740bbaf8bc 100644 --- a/airbyte-integrations/connectors/source-zoho-crm/unit_tests/test_types.py +++ b/airbyte-integrations/connectors/source-zoho-crm/unit_tests/test_types.py @@ -88,6 +88,7 @@ def test_field_schema(json_type, data_type, length, decimal_place, api_name, pic system_mandatory=True, display_label=api_name, pick_list_values=pick_list_values, - auto_number=autonumber or {'prefix': '', 'suffix': ''}) + auto_number=autonumber or {"prefix": "", "suffix": ""}, + ) assert field.schema == expected_values diff --git a/airbyte-integrations/connectors/source-zoom/README.md b/airbyte-integrations/connectors/source-zoom/README.md index e971c05d4264..d2146cd51da7 100644 --- a/airbyte-integrations/connectors/source-zoom/README.md +++ b/airbyte-integrations/connectors/source-zoom/README.md @@ -5,14 +5,6 @@ For information about how to use this connector within Airbyte, see [the documen ## Local development -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-zoom:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/zoom) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_survey_sparrow/spec.yaml` file. @@ -24,18 +16,19 @@ and place them into `secrets/config.json`. ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build . -t airbyte/source-zoom:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-zoom build ``` -You can also build the connector image via Gradle: -``` -./gradlew :airbyte-integrations:connectors:source-zoom:airbyteDocker +An image will be built with the tag `airbyte/source-zoom:dev`. + +**Via `docker build`:** +```bash +docker build -t airbyte/source-zoom:dev . ``` -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. #### Run Then run any of the connector commands as follows: @@ -45,25 +38,17 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zoom:dev check --confi docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zoom:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-zoom:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` + ## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-zoom test +``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-zoom:unitTest -``` -To run acceptance and custom integration tests: -``` -./gradlew :airbyte-integrations:connectors:source-zoom:integrationTest -``` - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. We split dependencies between two groups, dependencies that are: @@ -72,8 +57,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-zoom test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/zoom.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-zoom/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zoom/acceptance-test-config.yml index 621b45ee1f8f..0401b49598b8 100644 --- a/airbyte-integrations/connectors/source-zoom/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-zoom/acceptance-test-config.yml @@ -40,5 +40,5 @@ tests: "meetings": - "start_url" "webinars": - - "start_url" + - "start_url" timeout_seconds: 3600 diff --git a/airbyte-integrations/connectors/source-zoom/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-zoom/acceptance-test-docker.sh deleted file mode 100755 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-zoom/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-zoom/build.gradle b/airbyte-integrations/connectors/source-zoom/build.gradle deleted file mode 100644 index 60cdae821331..000000000000 --- a/airbyte-integrations/connectors/source-zoom/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_zoom' -} diff --git a/airbyte-integrations/connectors/source-zoom/unit_tests/test_zoom_authenticator.py b/airbyte-integrations/connectors/source-zoom/unit_tests/test_zoom_authenticator.py index 7706d1eaa079..c10eb35a6e6a 100755 --- a/airbyte-integrations/connectors/source-zoom/unit_tests/test_zoom_authenticator.py +++ b/airbyte-integrations/connectors/source-zoom/unit_tests/test_zoom_authenticator.py @@ -21,23 +21,24 @@ def test_generate_access_token(self): "client_id": "rc-123456789", "client_secret": "rc-test-secret", "authorization_endpoint": "https://example.zoom.com/oauth/token", - "grant_type": "account_credentials" + "grant_type": "account_credentials", } parameters = config - client = ServerToServerOauthAuthenticator(config=config, - account_id=config["account_id"], - client_id=config["client_id"], - client_secret=config["client_secret"], - grant_type=config["grant_type"], - authorization_endpoint=config["authorization_endpoint"], - parameters=parameters) + client = ServerToServerOauthAuthenticator( + config=config, + account_id=config["account_id"], + client_id=config["client_id"], + client_secret=config["client_secret"], + grant_type=config["grant_type"], + authorization_endpoint=config["authorization_endpoint"], + parameters=parameters, + ) # Encode the client credentials in base64 - token = base64.b64encode(f'{config.get("client_id")}:{config.get("client_secret")}'.encode('ascii')).decode('utf-8') + token = base64.b64encode(f'{config.get("client_id")}:{config.get("client_secret")}'.encode("ascii")).decode("utf-8") # Define the headers that should be sent in the request - headers = {'Authorization': f'Basic {token}', - 'Content-type': 'application/json'} + headers = {"Authorization": f"Basic {token}", "Content-type": "application/json"} # Define the URL containing the grant_type and account_id as query parameters url = f'{config.get("authorization_endpoint")}?grant_type={config.get("grant_type")}&account_id={config.get("account_id")}' diff --git a/airbyte-integrations/connectors/source-zuora/README.md b/airbyte-integrations/connectors/source-zuora/README.md index d576daa73e64..e0e56908c9d3 100644 --- a/airbyte-integrations/connectors/source-zuora/README.md +++ b/airbyte-integrations/connectors/source-zuora/README.md @@ -29,14 +29,6 @@ used for editable installs (`pip install -e`) to pull in Python dependencies fro If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything should work as you expect. -#### Building via Gradle -You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. - -To build using Gradle, from the Airbyte repository root, run: -``` -./gradlew :airbyte-integrations:connectors:source-zuora:build -``` - #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/zuora) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_zuora/spec.json` file. @@ -56,19 +48,19 @@ python main.py read --config secrets/config.json --catalog integration_tests/con ### Locally running the connector docker image + #### Build -First, make sure you build the latest Docker image: -``` -docker build --no-cache . -t airbyte/source-zuora:dev +**Via [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) (recommended):** +```bash +airbyte-ci connectors --name=source-zuora build ``` -You can also build the connector image via Gradle: -``` -./gradlew clean :airbyte-integrations:connectors:source-zuora:airbyteDocker -``` +An image will be built with the tag `airbyte/source-zuora:dev`. -When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in -the Dockerfile. +**Via `docker build`:** +```bash +docker build -t airbyte/source-zuora:dev . +``` #### Run Then run any of the connector commands as follows: @@ -79,95 +71,16 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-zuora:dev discover --c docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-zuora:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` -## Testing -Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. -First install test dependencies into your virtual environment: -``` -pip install .[tests] -``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` - -### Integration Tests -There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). - -#### Custom Integration tests -Place custom tests inside `integration_tests/` folder, then, from the connector root, run -``` -python -m pytest integration_tests +## Testing +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-zuora test ``` -#### Acceptance Tests -Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +### Customizing acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. -To run your integration tests with acceptance tests, from the connector root, run -``` -docker build . --no-cache -t airbyte/source-zuora:dev \ -&& python -m pytest -p connector_acceptance_test.plugin -``` - -To run your integration tests with docker - -### Using gradle to run tests -All commands should be run from airbyte project root. -To run unit tests: -``` -./gradlew :airbyte-integrations:connectors:source-zuora:unitTest -``` - -To run acceptance and custom integration tests: -``` -./gradlew clean :airbyte-integrations:connectors:source-zuora:integrationTest -``` - -### Run the actual test using with DBT Normalisation in action -## Add the Zuora Connector you just built: - -# Run Airbyte using docker-compose -Under Airbyte's root directory: -``` -docker compose up -d -``` -Open the web-page of Airbyte: -``` -http://localhost:8000/ -``` -Proceed the `First-Steps` on the web-page, use some test information for this, for now you can skipp the `onboarding` part. - -# Add New Source Connector -Add the SOURCE by going to `Admin` panel and press: -``` -+ New Connector -``` -Complete the New Connector's form by entering: -``` -Name : Test Zuora Connector -Image: airbyte/source-zuora -Tag: dev -Documentation: http://SomeTestDocumentation.com/ -``` -Save the form. - -# Add Destination -Use the following steps to build and test the custom destination on the Postgres example: -``` -docker run --rm --name airbyte-destination -e POSTGRES_PASSWORD=password -p 3000:5432 -d postgres -``` -After the docker has run the local postgres database in the background for your needs, use the following credentials to set up the DESTINATION on the Airbyte's web-page, on your local machine. -``` -Host: localhost -Port: 3000 -User: postgres -Password: password -DB Name: postgres -``` - -# Run the Connection Sync - ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. @@ -177,8 +90,11 @@ We split dependencies between two groups, dependencies that are: ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests. -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). -1. Create a Pull Request. -1. Pat yourself on the back for being an awesome contributor. -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-zuora test` +2. Bump the connector version in `metadata.yaml`: increment the `dockerImageTag` value. Please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors). +3. Make sure the `metadata.yaml` content is up to date. +4. Make the connector documentation and its changelog is up to date (`docs/integrations/sources/zuora.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. + diff --git a/airbyte-integrations/connectors/source-zuora/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zuora/acceptance-test-config.yml index f584243cf9df..d833a920c49e 100644 --- a/airbyte-integrations/connectors/source-zuora/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-zuora/acceptance-test-config.yml @@ -25,4 +25,3 @@ tests: configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" timeout_seconds: 3600 - diff --git a/airbyte-integrations/connectors/source-zuora/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-zuora/acceptance-test-docker.sh deleted file mode 100644 index 5797d20fe9a7..000000000000 --- a/airbyte-integrations/connectors/source-zuora/acceptance-test-docker.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-zuora/build.gradle b/airbyte-integrations/connectors/source-zuora/build.gradle deleted file mode 100644 index ad86953edf0e..000000000000 --- a/airbyte-integrations/connectors/source-zuora/build.gradle +++ /dev/null @@ -1,9 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' - id 'airbyte-connector-acceptance-test' -} - -airbytePython { - moduleDirectory 'source_zuora' -} diff --git a/airbyte-integrations/connectors/third-party/farosai/airbyte-customer-io-source/icon.svg b/airbyte-integrations/connectors/third-party/farosai/airbyte-customer-io-source/icon.svg deleted file mode 100644 index f308dc9eb77a..000000000000 --- a/airbyte-integrations/connectors/third-party/farosai/airbyte-customer-io-source/icon.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/airbyte-integrations/connectors/third-party/farosai/airbyte-customer-io-source/metadata.yaml b/airbyte-integrations/connectors/third-party/farosai/airbyte-customer-io-source/metadata.yaml deleted file mode 100644 index 6c0b76c003d1..000000000000 --- a/airbyte-integrations/connectors/third-party/farosai/airbyte-customer-io-source/metadata.yaml +++ /dev/null @@ -1,24 +0,0 @@ -data: - registries: - cloud: - enabled: false - oss: - enabled: true - connectorSubtype: api - connectorType: source - definitionId: c47d6804-8b98-449f-970a-5ddb5cb5d7aa - dockerImageTag: 0.1.23 - dockerRepository: farosai/airbyte-customer-io-source - githubIssueLabel: farosai/airbyte-customer-io-source - icon: customer-io.svg - license: MIT - name: Customer.io - releaseStage: alpha - documentationUrl: https://docs.airbyte.com/integrations/sources/customer-io - tags: - - language:unknown - ab_internal: - sl: 100 - ql: 100 - supportLevel: community -metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/scripts/build-connector-image-with-local-cdk.sh b/airbyte-integrations/scripts/build-connector-image-with-local-cdk.sh deleted file mode 100755 index afe118997393..000000000000 --- a/airbyte-integrations/scripts/build-connector-image-with-local-cdk.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -set -e - -ROOT_DIR="$(git rev-parse --show-toplevel)" -source "$ROOT_DIR/airbyte-integrations/scripts/utils.sh" - -[ -n "$CONNECTOR_TAG" ] || die "Missing CONNECTOR_TAG" -[ -n "$CONNECTOR_NAME" ] || die "Missing CONNECTOR_NAME" - -echo "Building docker image for $CONNECTOR_NAME with local CDK" - -CDK_DIR="$ROOT_DIR/airbyte-cdk/python" -CONNECTOR_DIR="$ROOT_DIR/airbyte-integrations/connectors/$CONNECTOR_NAME" -CONNECTOR_SUBDIR="$CONNECTOR_DIR/$(echo $CONNECTOR_NAME | sed 's/-/_/g')" -BUILD_DIR=$(mktemp -d) - -# Copy the connector files & CDK to the build directory -cd "$BUILD_DIR" -cp "$CONNECTOR_DIR/setup.py" . -cp "$CONNECTOR_DIR/main.py" . -cp "$CONNECTOR_DIR/Dockerfile" . -cp -r "$CONNECTOR_SUBDIR" . -mkdir airbyte-cdk -rsync -a --exclude "build/" --exclude ".venv/" "$CDK_DIR/" airbyte-cdk/ - -# Insert an instruction to the Dockerfile to copy the local CDK -awk 'NR==1 {print; print "COPY airbyte-cdk /airbyte-cdk"} NR!=1' Dockerfile > Dockerfile.copy -mv Dockerfile.copy Dockerfile - -# Modify setup.py so it uses the local CDK -sed -iE 's,"airbyte-cdk[^"]*","airbyte-cdk @ file://localhost/airbyte-cdk",' setup.py - -# Build the connector image -if [ -n "$QUIET_BUILD" ]; then - docker build -t "$CONNECTOR_TAG" -q . -else - docker build -t "$CONNECTOR_TAG" . -fi - -cd - - -# Clean up now that the image has been created -rm -rf "$BUILD_DIR" diff --git a/airbyte-integrations/scripts/run-acceptance-test-docker.sh b/airbyte-integrations/scripts/run-acceptance-test-docker.sh deleted file mode 100755 index 93009e637b10..000000000000 --- a/airbyte-integrations/scripts/run-acceptance-test-docker.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env sh - -CONNECTOR_NAME=$1 -OUTPUT_DIR=$2 - -ROOT_DIR="$(git rev-parse --show-toplevel)" -source "$ROOT_DIR/airbyte-integrations/scripts/utils.sh" - -[ -n "$CONNECTOR_NAME" ] || die "Missing CONNECTOR_NAME" -[ -n "$OUTPUT_DIR" ] || die "Missing OUTPUT_DIR" - -CONNECTOR_DIR=$ROOT_DIR/airbyte-integrations/connectors/$CONNECTOR_NAME -CONNECTOR_OUTPUT_DIR=$OUTPUT_DIR/$CONNECTOR_NAME - -cd $CONNECTOR_DIR - -if [ -f acceptance-test-docker.sh ] && [ -f setup.py ] && grep -q "airbyte-cdk" setup.py; then - mkdir $CONNECTOR_OUTPUT_DIR - echo "Building docker image for $CONNECTOR_NAME." - LOCAL_CDK=1 FETCH_SECRETS=1 QUIET_BUILD=1 sh acceptance-test-docker.sh > $CONNECTOR_OUTPUT_DIR/$CONNECTOR_NAME.out 2> $CONNECTOR_OUTPUT_DIR/$CONNECTOR_NAME.err - echo $? > $CONNECTOR_OUTPUT_DIR/$CONNECTOR_NAME.exit-code -fi diff --git a/airbyte-json-validation/build.gradle b/airbyte-json-validation/build.gradle deleted file mode 100644 index f59ad1824523..000000000000 --- a/airbyte-json-validation/build.gradle +++ /dev/null @@ -1,11 +0,0 @@ -plugins { - id "java-library" -} - -dependencies { - implementation 'com.networknt:json-schema-validator:1.0.72' - // needed so that we can follow $ref when parsing json. jackson does not support this natively. - implementation 'me.andrz.jackson:jackson-json-reference-core:0.3.2' -} - -Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project) diff --git a/airbyte-lib/.gitattributes b/airbyte-lib/.gitattributes new file mode 100644 index 000000000000..7af38cfbe107 --- /dev/null +++ b/airbyte-lib/.gitattributes @@ -0,0 +1,2 @@ +# Hide diffs in auto-generated files +docs/generated/**/* linguist-generated=true diff --git a/airbyte-lib/.gitignore b/airbyte-lib/.gitignore new file mode 100644 index 000000000000..3eb9954af8d1 --- /dev/null +++ b/airbyte-lib/.gitignore @@ -0,0 +1 @@ +.venv* \ No newline at end of file diff --git a/airbyte-lib/README.md b/airbyte-lib/README.md new file mode 100644 index 000000000000..b30ced523d93 --- /dev/null +++ b/airbyte-lib/README.md @@ -0,0 +1,26 @@ +# airbyte-lib + +airbyte-lib is a library that allows to run Airbyte syncs embedded into any Python application, without the need to run Airbyte server. + +## Development + +* Make sure [Poetry is installed](https://python-poetry.org/docs/#). +* Run `poetry install` +* For examples, check out the `examples` folder. They can be run via `poetry run python examples/` +* Unit tests and type checks can be run via `poetry run pytest` + +## Documentation + +Regular documentation lives in the `/docs` folder. Based on the doc strings of public methods, we generate API documentation using [pdoc](https://pdoc.dev). To generate the documentation, run `poetry run generate-docs`. The documentation will be generated in the `docs/generate` folder. This needs to be done manually when changing the public interface of the library. + +A unit test validates the documentation is up to date. + +## Validating source connectors + +To validate a source connector for compliance, the `airbyte-lib-validate-source` script can be used. It can be used like this: + +``` +airbyte-lib-validate-source —connector-dir . -—sample-config secrets/config.json +``` + +The script will install the python package in the provided directory, and run the connector against the provided config. The config should be a valid JSON file, with the same structure as the one that would be provided to the connector in Airbyte. The script will exit with a non-zero exit code if the connector fails to run. diff --git a/airbyte-lib/airbyte_lib/__init__.py b/airbyte-lib/airbyte_lib/__init__.py new file mode 100644 index 000000000000..895849f19771 --- /dev/null +++ b/airbyte-lib/airbyte_lib/__init__.py @@ -0,0 +1,17 @@ +"""AirbyteLib brings Airbyte ELT to every Python developer.""" + +from airbyte_lib._factories.cache_factories import get_default_cache, new_local_cache +from airbyte_lib._factories.connector_factories import get_connector +from airbyte_lib.datasets import CachedDataset +from airbyte_lib.results import ReadResult +from airbyte_lib.source import Source + + +__all__ = [ + "get_connector", + "get_default_cache", + "new_local_cache", + "CachedDataset", + "ReadResult", + "Source", +] diff --git a/airbyte-lib/airbyte_lib/_executor.py b/airbyte-lib/airbyte_lib/_executor.py new file mode 100644 index 000000000000..1a816cc46848 --- /dev/null +++ b/airbyte-lib/airbyte_lib/_executor.py @@ -0,0 +1,234 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +import subprocess +import sys +from abc import ABC, abstractmethod +from contextlib import contextmanager +from pathlib import Path +from typing import IO, TYPE_CHECKING, Any, NoReturn + +from airbyte_lib.telemetry import SourceTelemetryInfo, SourceType + + +if TYPE_CHECKING: + from collections.abc import Generator, Iterable, Iterator + + from airbyte_lib.registry import ConnectorMetadata + + +_LATEST_VERSION = "latest" + + +class Executor(ABC): + def __init__( + self, + metadata: ConnectorMetadata, + target_version: str | None = None, + ) -> None: + self.metadata = metadata + self.enforce_version = target_version is not None + if target_version is None or target_version == _LATEST_VERSION: + self.target_version = metadata.latest_available_version + else: + self.target_version = target_version + + @abstractmethod + def execute(self, args: list[str]) -> Iterator[str]: + pass + + @abstractmethod + def ensure_installation(self) -> None: + pass + + @abstractmethod + def install(self) -> None: + pass + + @abstractmethod + def get_telemetry_info(self) -> SourceTelemetryInfo: + pass + + @abstractmethod + def uninstall(self) -> None: + pass + + +@contextmanager +def _stream_from_subprocess(args: list[str]) -> Generator[Iterable[str], None, None]: + process = subprocess.Popen( + args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + ) + + def _stream_from_file(file: IO[str]) -> Generator[str, Any, None]: + while True: + line = file.readline() + if not line: + break + yield line + + if process.stdout is None: + raise Exception("Failed to start subprocess") + try: + yield _stream_from_file(process.stdout) + finally: + # Close the stdout stream + if process.stdout: + process.stdout.close() + + # Terminate the process if it is still running + if process.poll() is None: # Check if the process is still running + process.terminate() + try: + # Wait for a short period to allow process to terminate gracefully + process.wait(timeout=10) + except subprocess.TimeoutExpired: + # If the process does not terminate within the timeout, force kill it + process.kill() + + # Now, the process is either terminated or killed. Check the exit code. + exit_code = process.wait() + + # If the exit code is not 0 or -15 (SIGTERM), raise an exception + if exit_code not in (0, -15): + raise Exception(f"Process exited with code {exit_code}") + + +class VenvExecutor(Executor): + def __init__( + self, + metadata: ConnectorMetadata, + target_version: str | None = None, + pip_url: str | None = None, + *, + install_if_missing: bool = False, + ) -> None: + super().__init__(metadata, target_version) + self.install_if_missing = install_if_missing + + # This is a temporary install path that will be replaced with a proper package + # name once they are published. + # TODO: Replace with `f"airbyte-{self.metadata.name}"` + self.pip_url = pip_url or f"../airbyte-integrations/connectors/{self.metadata.name}" + + def _get_venv_name(self) -> str: + return f".venv-{self.metadata.name}" + + def _get_connector_path(self) -> Path: + return Path(self._get_venv_name(), "bin", self.metadata.name) + + def _run_subprocess_and_raise_on_failure(self, args: list[str]) -> None: + result = subprocess.run(args, check=False) + if result.returncode != 0: + raise Exception(f"Install process exited with code {result.returncode}") + + def uninstall(self) -> None: + venv_name = self._get_venv_name() + if Path(venv_name).exists(): + self._run_subprocess_and_raise_on_failure(["rm", "-rf", venv_name]) + + def install(self) -> None: + venv_name = self._get_venv_name() + self._run_subprocess_and_raise_on_failure([sys.executable, "-m", "venv", venv_name]) + + pip_path = str(Path(venv_name) / "bin" / "pip") + + self._run_subprocess_and_raise_on_failure([pip_path, "install", "-e", self.pip_url]) + + def _get_installed_version(self) -> str: + """Detect the version of the connector installed. + + In the venv, we run the following: + > python -c "from importlib.metadata import version; print(version(''))" + """ + venv_name = self._get_venv_name() + connector_name = self.metadata.name + return subprocess.check_output( + [ + Path(venv_name) / "bin" / "python", + "-c", + f"from importlib.metadata import version; print(version('{connector_name}'))", + ], + universal_newlines=True, + ).strip() + + def ensure_installation( + self, + ) -> None: + """Ensure that the connector is installed in a virtual environment. + + If not yet installed and if install_if_missing is True, then install. + + Optionally, verify that the installed version matches the target version. + + Note: Version verification is not supported for connectors installed from a + local path. + """ + venv_name = f".venv-{self.metadata.name}" + venv_path = Path(venv_name) + if not venv_path.exists(): + if not self.install_if_missing: + raise Exception( + f"Connector {self.metadata.name} is not available - " + f"venv {venv_name} does not exist" + ) + self.install() + + connector_path = self._get_connector_path() + if not connector_path.exists(): + raise FileNotFoundError( + f"Could not find connector '{self.metadata.name}' in venv '{venv_name}' with " + f"connector path '{connector_path}'.", + ) + + if self.enforce_version: + installed_version = self._get_installed_version() + if installed_version != self.target_version: + # If the version doesn't match, reinstall + self.install() + + # Check the version again + version_after_install = self._get_installed_version() + if version_after_install != self.target_version: + raise Exception( + f"Failed to install connector {self.metadata.name} version " + f"{self.target_version}. Installed version is {version_after_install}", + ) + + def execute(self, args: list[str]) -> Iterator[str]: + connector_path = self._get_connector_path() + + with _stream_from_subprocess([str(connector_path), *args]) as stream: + yield from stream + + def get_telemetry_info(self) -> SourceTelemetryInfo: + return SourceTelemetryInfo(self.metadata.name, SourceType.VENV, self.target_version) + + +class PathExecutor(Executor): + def ensure_installation(self) -> None: + try: + self.execute(["spec"]) + except Exception as e: + raise Exception( + f"Connector {self.metadata.name} is not available - executing it failed" + ) from e + + def install(self) -> NoReturn: + raise Exception(f"Connector {self.metadata.name} is not available - cannot install it") + + def uninstall(self) -> NoReturn: + raise Exception( + f"Connector {self.metadata.name} is installed manually and not managed by airbyte-lib -" + " please remove it manually" + ) + + def execute(self, args: list[str]) -> Iterator[str]: + with _stream_from_subprocess([self.metadata.name, *args]) as stream: + yield from stream + + def get_telemetry_info(self) -> SourceTelemetryInfo: + return SourceTelemetryInfo(self.metadata.name, SourceType.LOCAL_INSTALL, version=None) diff --git a/airbyte-lib/airbyte_lib/_factories/__init__.py b/airbyte-lib/airbyte_lib/_factories/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-lib/airbyte_lib/_factories/cache_factories.py b/airbyte-lib/airbyte_lib/_factories/cache_factories.py new file mode 100644 index 000000000000..5a95dce2db7b --- /dev/null +++ b/airbyte-lib/airbyte_lib/_factories/cache_factories.py @@ -0,0 +1,59 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +from pathlib import Path + +import ulid + +from airbyte_lib.caches.duckdb import DuckDBCache, DuckDBCacheConfig + + +def get_default_cache() -> DuckDBCache: + """Get a local cache for storing data, using the default database path. + + Cache files are stored in the `.cache` directory, relative to the current + working directory. + """ + config = DuckDBCacheConfig( + db_path="./.cache/default_cache_db.duckdb", + ) + return DuckDBCache(config=config) + + +def new_local_cache( + cache_name: str | None = None, + cache_dir: str | Path | None = None, + *, + cleanup: bool = True, +) -> DuckDBCache: + """Get a local cache for storing data, using a name string to seed the path. + + Args: + cache_name: Name to use for the cache. Defaults to None. + cache_dir: Root directory to store the cache in. Defaults to None. + cleanup: Whether to clean up temporary files. Defaults to True. + + Cache files are stored in the `.cache` directory, relative to the current + working directory. + """ + if cache_name: + if " " in cache_name: + raise ValueError(f"Cache name '{cache_name}' cannot contain spaces") + + if not cache_name.replace("_", "").isalnum(): + raise ValueError( + f"Cache name '{cache_name}' can only contain alphanumeric " + "characters and underscores." + ) + + cache_name = cache_name or str(ulid.ULID()) + cache_dir = cache_dir or Path(f"./.cache/{cache_name}") + if not isinstance(cache_dir, Path): + cache_dir = Path(cache_dir) + + config = DuckDBCacheConfig( + db_path=cache_dir / f"db_{cache_name}.duckdb", + cache_dir=cache_dir, + cleanup=cleanup, + ) + return DuckDBCache(config=config) diff --git a/airbyte-lib/airbyte_lib/_factories/connector_factories.py b/airbyte-lib/airbyte_lib/_factories/connector_factories.py new file mode 100644 index 000000000000..347710f20824 --- /dev/null +++ b/airbyte-lib/airbyte_lib/_factories/connector_factories.py @@ -0,0 +1,59 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +from typing import Any + +from airbyte_lib._executor import Executor, PathExecutor, VenvExecutor +from airbyte_lib.registry import get_connector_metadata +from airbyte_lib.source import Source + + +def get_connector( + name: str, + version: str | None = None, + pip_url: str | None = None, + config: dict[str, Any] | None = None, + *, + use_local_install: bool = False, + install_if_missing: bool = True, +) -> Source: + """Get a connector by name and version. + + Args: + name: connector name + version: connector version - if not provided, the currently installed version will be used. + If no version is installed, the latest available version will be used. The version can + also be set to "latest" to force the use of the latest available version. + pip_url: connector pip URL - if not provided, the pip url will be inferred from the + connector name. + config: connector config - if not provided, you need to set it later via the set_config + method. + use_local_install: whether to use a virtual environment to run the connector. If True, the + connector is expected to be available on the path (e.g. installed via pip). If False, + the connector will be installed automatically in a virtual environment. + install_if_missing: whether to install the connector if it is not available locally. This + parameter is ignored if use_local_install is True. + """ + metadata = get_connector_metadata(name) + if use_local_install: + if pip_url: + raise ValueError("Param 'pip_url' is not supported when 'use_local_install' is True") + if version: + raise ValueError("Param 'version' is not supported when 'use_local_install' is True") + executor: Executor = PathExecutor( + metadata=metadata, + target_version=version, + ) + + else: + executor = VenvExecutor( + metadata=metadata, + target_version=version, + install_if_missing=install_if_missing, + pip_url=pip_url, + ) + return Source( + executor=executor, + name=name, + config=config, + ) diff --git a/airbyte-lib/airbyte_lib/_file_writers/__init__.py b/airbyte-lib/airbyte_lib/_file_writers/__init__.py new file mode 100644 index 000000000000..007dde832434 --- /dev/null +++ b/airbyte-lib/airbyte_lib/_file_writers/__init__.py @@ -0,0 +1,11 @@ +from .base import FileWriterBase, FileWriterBatchHandle, FileWriterConfigBase +from .parquet import ParquetWriter, ParquetWriterConfig + + +__all__ = [ + "FileWriterBatchHandle", + "FileWriterBase", + "FileWriterConfigBase", + "ParquetWriter", + "ParquetWriterConfig", +] diff --git a/airbyte-lib/airbyte_lib/_file_writers/base.py b/airbyte-lib/airbyte_lib/_file_writers/base.py new file mode 100644 index 000000000000..3f16953f12f5 --- /dev/null +++ b/airbyte-lib/airbyte_lib/_file_writers/base.py @@ -0,0 +1,111 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""Define abstract base class for File Writers, which write and read from file storage.""" + +from __future__ import annotations + +import abc +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, cast, final + +from overrides import overrides + +from airbyte_lib._processors import BatchHandle, RecordProcessor +from airbyte_lib.config import CacheConfigBase + + +if TYPE_CHECKING: + import pyarrow as pa + + +DEFAULT_BATCH_SIZE = 10000 + + +# The batch handle for file writers is a list of Path objects. +@dataclass +class FileWriterBatchHandle(BatchHandle): + """The file writer batch handle is a list of Path objects.""" + + files: list[Path] = field(default_factory=list) + + +class FileWriterConfigBase(CacheConfigBase): + """Configuration for the Snowflake cache.""" + + cache_dir: Path = Path("./.cache/files/") + """The directory to store cache files in.""" + cleanup: bool = True + """Whether to clean up temporary files after processing a batch.""" + + +class FileWriterBase(RecordProcessor, abc.ABC): + """A generic base implementation for a file-based cache.""" + + config_class = FileWriterConfigBase + config: FileWriterConfigBase + + @abc.abstractmethod + @overrides + def _write_batch( + self, + stream_name: str, + batch_id: str, + record_batch: pa.Table | pa.RecordBatch, + ) -> FileWriterBatchHandle: + """Process a record batch. + + Return a list of paths to one or more cache files. + """ + ... + + @final + def write_batch( + self, + stream_name: str, + batch_id: str, + record_batch: pa.Table | pa.RecordBatch, + ) -> FileWriterBatchHandle: + """Write a batch of records to the cache. + + This method is final because it should not be overridden. + + Subclasses should override `_write_batch` instead. + """ + return self._write_batch(stream_name, batch_id, record_batch) + + @overrides + def _cleanup_batch( + self, + stream_name: str, + batch_id: str, + batch_handle: BatchHandle, + ) -> None: + """Clean up the cache. + + For file writers, this means deleting the files created and declared in the batch. + + This method is a no-op if the `cleanup` config option is set to False. + """ + if self.config.cleanup: + batch_handle = cast(FileWriterBatchHandle, batch_handle) + _ = stream_name, batch_id + for file_path in batch_handle.files: + file_path.unlink() + + @final + def cleanup_batch( + self, + stream_name: str, + batch_id: str, + batch_handle: BatchHandle, + ) -> None: + """Clean up the cache. + + For file writers, this means deleting the files created and declared in the batch. + + This method is final because it should not be overridden. + + Subclasses should override `_cleanup_batch` instead. + """ + self._cleanup_batch(stream_name, batch_id, batch_handle) diff --git a/airbyte-lib/airbyte_lib/_file_writers/parquet.py b/airbyte-lib/airbyte_lib/_file_writers/parquet.py new file mode 100644 index 000000000000..aeb2113f2a28 --- /dev/null +++ b/airbyte-lib/airbyte_lib/_file_writers/parquet.py @@ -0,0 +1,59 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""A Parquet cache implementation.""" +from __future__ import annotations + +from pathlib import Path +from typing import cast + +import pyarrow as pa +import ulid +from overrides import overrides +from pyarrow import parquet + +from .base import FileWriterBase, FileWriterBatchHandle, FileWriterConfigBase + + +class ParquetWriterConfig(FileWriterConfigBase): + """Configuration for the Snowflake cache.""" + + # Inherits `cache_dir` from base class + + +class ParquetWriter(FileWriterBase): + """A Parquet cache implementation.""" + + config_class = ParquetWriterConfig + + def get_new_cache_file_path( + self, + stream_name: str, + batch_id: str | None = None, # ULID of the batch + ) -> Path: + """Return a new cache file path for the given stream.""" + batch_id = batch_id or str(ulid.ULID()) + config: ParquetWriterConfig = cast(ParquetWriterConfig, self.config) + target_dir = Path(config.cache_dir) + target_dir.mkdir(parents=True, exist_ok=True) + return target_dir / f"{stream_name}_{batch_id}.parquet" + + @overrides + def _write_batch( + self, + stream_name: str, + batch_id: str, + record_batch: pa.Table | pa.RecordBatch, + ) -> FileWriterBatchHandle: + """Process a record batch. + + Return the path to the cache file. + """ + _ = batch_id # unused + output_file_path = self.get_new_cache_file_path(stream_name) + + with parquet.ParquetWriter(output_file_path, record_batch.schema) as writer: + writer.write_table(cast(pa.Table, record_batch)) + + batch_handle = FileWriterBatchHandle() + batch_handle.files.append(output_file_path) + return batch_handle diff --git a/airbyte-lib/airbyte_lib/_processors.py b/airbyte-lib/airbyte_lib/_processors.py new file mode 100644 index 000000000000..f0f94c30c512 --- /dev/null +++ b/airbyte-lib/airbyte_lib/_processors.py @@ -0,0 +1,310 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""Define abstract base class for Processors, including Caches and File writers. + +Processors can all take input from STDIN or a stream of Airbyte messages. + +Caches will pass their input to the File Writer. They share a common base class so certain +abstractions like "write" and "finalize" can be handled in either layer, or both. +""" + +from __future__ import annotations + +import abc +import contextlib +import io +import sys +from collections import defaultdict +from typing import TYPE_CHECKING, Any, cast, final + +import pyarrow as pa +import ulid + +from airbyte_protocol.models import ( + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStateMessage, + AirbyteStateType, + AirbyteStreamState, + ConfiguredAirbyteCatalog, + Type, +) + +from airbyte_lib._util import protocol_util # Internal utility functions + + +if TYPE_CHECKING: + from collections.abc import Generator, Iterable, Iterator + + from airbyte_lib.config import CacheConfigBase + + +DEFAULT_BATCH_SIZE = 10000 + + +class BatchHandle: + pass + + +class AirbyteMessageParsingError(Exception): + """Raised when an Airbyte message is invalid or cannot be parsed.""" + + +class RecordProcessor(abc.ABC): + """Abstract base class for classes which can process input records.""" + + config_class: type[CacheConfigBase] + skip_finalize_step: bool = False + + def __init__( + self, + config: CacheConfigBase | dict | None, + ) -> None: + if isinstance(config, dict): + config = self.config_class(**config) + + self.config = config or self.config_class() + if not isinstance(self.config, self.config_class): + err_msg = ( + f"Expected config class of type '{self.config_class.__name__}'. " + f"Instead found '{type(self.config).__name__}'." + ) + raise TypeError(err_msg) + + self.source_catalog: ConfiguredAirbyteCatalog | None = None + + self._pending_batches: dict[str, dict[str, Any]] = defaultdict(lambda: {}, {}) + self._finalized_batches: dict[str, dict[str, Any]] = defaultdict(lambda: {}, {}) + + self._pending_state_messages: dict[str, list[AirbyteStateMessage]] = defaultdict(list, {}) + self._finalized_state_messages: dict[ + str, + list[AirbyteStateMessage], + ] = defaultdict(list, {}) + + self._setup() + + def register_source( + self, + source_name: str, + source_catalog: ConfiguredAirbyteCatalog, + ) -> None: + """Register the source name and catalog. + + For now, only one source at a time is supported. + If this method is called multiple times, the last call will overwrite the previous one. + + TODO: Expand this to handle mutliple sources. + """ + _ = source_name + self.source_catalog = source_catalog + + @final + def process_stdin( + self, + max_batch_size: int = DEFAULT_BATCH_SIZE, + ) -> None: + """Process the input stream from stdin. + + Return a list of summaries for testing. + """ + input_stream = io.TextIOWrapper(sys.stdin.buffer, encoding="utf-8") + self.process_input_stream(input_stream, max_batch_size) + + @final + def _airbyte_messages_from_buffer( + self, + buffer: io.TextIOBase, + ) -> Iterator[AirbyteMessage]: + """Yield messages from a buffer.""" + yield from (AirbyteMessage.parse_raw(line) for line in buffer) + + @final + def process_input_stream( + self, + input_stream: io.TextIOBase, + max_batch_size: int = DEFAULT_BATCH_SIZE, + ) -> None: + """Parse the input stream and process data in batches. + + Return a list of summaries for testing. + """ + messages = self._airbyte_messages_from_buffer(input_stream) + self.process_airbyte_messages(messages, max_batch_size) + + @final + def process_airbyte_messages( + self, + messages: Iterable[AirbyteMessage], + max_batch_size: int = DEFAULT_BATCH_SIZE, + ) -> None: + stream_batches: dict[str, list[dict]] = defaultdict(list, {}) + + # Process messages, writing to batches as we go + for message in messages: + if message.type is Type.RECORD: + record_msg = cast(AirbyteRecordMessage, message.record) + stream_name = record_msg.stream + stream_batch = stream_batches[stream_name] + stream_batch.append(protocol_util.airbyte_record_message_to_dict(record_msg)) + + if len(stream_batch) >= max_batch_size: + record_batch = pa.Table.from_pylist(stream_batch) + self._process_batch(stream_name, record_batch) + stream_batch.clear() + + elif message.type is Type.STATE: + state_msg = cast(AirbyteStateMessage, message.state) + if state_msg.type in [AirbyteStateType.GLOBAL, AirbyteStateType.LEGACY]: + self._pending_state_messages[f"_{state_msg.type}"].append(state_msg) + else: + stream_state = cast(AirbyteStreamState, state_msg.stream) + stream_name = stream_state.stream_descriptor.name + self._pending_state_messages[stream_name].append(state_msg) + + elif message.type in [Type.LOG, Type.TRACE]: + pass + + else: + raise ValueError(f"Unexpected message type: {message.type}") + + # We are at the end of the stream. Process whatever else is queued. + for stream_name, batch in stream_batches.items(): + if batch: + record_batch = pa.Table.from_pylist(batch) + self._process_batch(stream_name, record_batch) + + # Finalize any pending batches + for stream_name in list(self._pending_batches.keys()): + self._finalize_batches(stream_name) + + @final + def _process_batch( + self, + stream_name: str, + record_batch: pa.Table, + ) -> tuple[str, Any, Exception | None]: + """Process a single batch. + + Returns a tuple of the batch ID, batch handle, and an exception if one occurred. + """ + batch_id = self._new_batch_id() + batch_handle = self._write_batch( + stream_name, + batch_id, + record_batch, + ) or self._get_batch_handle(stream_name, batch_id) + + if self.skip_finalize_step: + self._finalized_batches[stream_name][batch_id] = batch_handle + else: + self._pending_batches[stream_name][batch_id] = batch_handle + + return batch_id, batch_handle, None + + @abc.abstractmethod + def _write_batch( + self, + stream_name: str, + batch_id: str, + record_batch: pa.Table | pa.RecordBatch, + ) -> BatchHandle: + """Process a single batch. + + Returns a batch handle, such as a path or any other custom reference. + """ + + def _cleanup_batch( # noqa: B027 # Intentionally empty, not abstract + self, + stream_name: str, + batch_id: str, + batch_handle: BatchHandle, + ) -> None: + """Clean up the cache. + + This method is called after the given batch has been finalized. + + For instance, file writers can override this method to delete the files created. Caches, + similarly, can override this method to delete any other temporary artifacts. + """ + pass + + def _new_batch_id(self) -> str: + """Return a new batch handle.""" + return str(ulid.ULID()) + + def _get_batch_handle( + self, + stream_name: str, + batch_id: str | None = None, # ULID of the batch + ) -> str: + """Return a new batch handle. + + By default this is a concatenation of the stream name and batch ID. + However, any Python object can be returned, such as a Path object. + """ + batch_id = batch_id or self._new_batch_id() + return f"{stream_name}_{batch_id}" + + def _finalize_batches(self, stream_name: str) -> dict[str, BatchHandle]: + """Finalize all uncommitted batches. + + Returns a mapping of batch IDs to batch handles, for processed batches. + + This is a generic implementation, which can be overridden. + """ + with self._finalizing_batches(stream_name) as batches_to_finalize: + if batches_to_finalize and not self.skip_finalize_step: + raise NotImplementedError( + "Caches need to be finalized but no _finalize_batch() method " + f"exists for class {self.__class__.__name__}", + ) + + return batches_to_finalize + + @final + @contextlib.contextmanager + def _finalizing_batches( + self, + stream_name: str, + ) -> Generator[dict[str, BatchHandle], str, None]: + """Context manager to use for finalizing batches, if applicable. + + Returns a mapping of batch IDs to batch handles, for those processed batches. + """ + batches_to_finalize = self._pending_batches[stream_name].copy() + state_messages_to_finalize = self._pending_state_messages[stream_name].copy() + self._pending_batches[stream_name].clear() + self._pending_state_messages[stream_name].clear() + yield batches_to_finalize + + self._finalized_batches[stream_name].update(batches_to_finalize) + self._finalized_state_messages[stream_name] += state_messages_to_finalize + + for batch_id, batch_handle in batches_to_finalize.items(): + self._cleanup_batch(stream_name, batch_id, batch_handle) + + def _setup(self) -> None: # noqa: B027 # Intentionally empty, not abstract + """Create the database. + + By default this is a no-op but subclasses can override this method to prepare + any necessary resources. + """ + + def _teardown(self) -> None: + """Teardown the processor resources. + + By default, the base implementation simply calls _cleanup_batch() for all pending batches. + """ + for stream_name, pending_batches in self._pending_batches.items(): + for batch_id, batch_handle in pending_batches.items(): + self._cleanup_batch( + stream_name=stream_name, + batch_id=batch_id, + batch_handle=batch_handle, + ) + + @final + def __del__(self) -> None: + """Teardown temporary resources when instance is unloaded from memory.""" + self._teardown() diff --git a/airbyte-lib/airbyte_lib/_util/__init__.py b/airbyte-lib/airbyte_lib/_util/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-lib/airbyte_lib/_util/protocol_util.py b/airbyte-lib/airbyte_lib/_util/protocol_util.py new file mode 100644 index 000000000000..58ada9f5435b --- /dev/null +++ b/airbyte-lib/airbyte_lib/_util/protocol_util.py @@ -0,0 +1,71 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""Internal utility functions, especially for dealing with Airbyte Protocol.""" +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, cast + +from airbyte_protocol.models import ( + AirbyteMessage, + AirbyteRecordMessage, + ConfiguredAirbyteCatalog, + Type, +) + + +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator + + +def airbyte_messages_to_record_dicts( + messages: Iterable[AirbyteMessage], +) -> Iterator[dict[str, Any]]: + """Convert an AirbyteMessage to a dictionary.""" + yield from ( + cast(dict[str, Any], airbyte_message_to_record_dict(message)) + for message in messages + if message is not None + ) + + +def airbyte_message_to_record_dict(message: AirbyteMessage) -> dict[str, Any] | None: + """Convert an AirbyteMessage to a dictionary. + + Return None if the message is not a record message. + """ + if message.type != Type.RECORD: + return None + + return airbyte_record_message_to_dict(message.record) + + +def airbyte_record_message_to_dict( + record_message: AirbyteRecordMessage, +) -> dict[str, Any]: + """Convert an AirbyteMessage to a dictionary. + + Return None if the message is not a record message. + """ + result = record_message.data + + # TODO: Add the metadata columns (this breaks tests) + # result["_airbyte_extracted_at"] = datetime.datetime.fromtimestamp( + # record_message.emitted_at + # ) + + return result # noqa: RET504 # unnecessary assignment and then return (see TODO above) + + +def get_primary_keys_from_stream( + stream_name: str, + configured_catalog: ConfiguredAirbyteCatalog, +) -> set[str]: + """Get the primary keys from a stream in the configured catalog.""" + stream = next( + (stream for stream in configured_catalog.streams if stream.stream.name == stream_name), + None, + ) + if stream is None: + raise ValueError(f"Stream {stream_name} not found in catalog.") + + return set(stream.stream.source_defined_primary_key or []) diff --git a/airbyte-lib/airbyte_lib/caches/__init__.py b/airbyte-lib/airbyte_lib/caches/__init__.py new file mode 100644 index 000000000000..d14980332458 --- /dev/null +++ b/airbyte-lib/airbyte_lib/caches/__init__.py @@ -0,0 +1,18 @@ +"""Base module for all caches.""" + +from airbyte_lib.caches.base import SQLCacheBase +from airbyte_lib.caches.duckdb import DuckDBCache, DuckDBCacheConfig +from airbyte_lib.caches.postgres import PostgresCache, PostgresCacheConfig +from airbyte_lib.caches.snowflake import SnowflakeCacheConfig, SnowflakeSQLCache + + +# We export these classes for easy access: `airbyte_lib.caches...` +__all__ = [ + "DuckDBCache", + "DuckDBCacheConfig", + "PostgresCache", + "PostgresCacheConfig", + "SQLCacheBase", + "SnowflakeCacheConfig", + "SnowflakeSQLCache", +] diff --git a/airbyte-lib/airbyte_lib/caches/base.py b/airbyte-lib/airbyte_lib/caches/base.py new file mode 100644 index 000000000000..0e3ad837b194 --- /dev/null +++ b/airbyte-lib/airbyte_lib/caches/base.py @@ -0,0 +1,746 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""A SQL Cache implementation.""" +from __future__ import annotations + +import abc +import enum +from collections.abc import Generator, Iterator, Mapping +from contextlib import contextmanager +from functools import cached_property +from typing import TYPE_CHECKING, Any, cast, final + +import pandas as pd +import pyarrow as pa +import sqlalchemy +import ulid +from overrides import overrides +from sqlalchemy import create_engine, text +from sqlalchemy.pool import StaticPool +from sqlalchemy.sql.elements import TextClause + +from airbyte_lib._file_writers.base import FileWriterBase, FileWriterBatchHandle +from airbyte_lib._processors import BatchHandle, RecordProcessor +from airbyte_lib.config import CacheConfigBase +from airbyte_lib.types import SQLTypeConverter + + +if TYPE_CHECKING: + from pathlib import Path + + from sqlalchemy.engine import Connection, Engine + from sqlalchemy.engine.cursor import CursorResult + from sqlalchemy.engine.reflection import Inspector + from sqlalchemy.sql.base import Executable + + from airbyte_protocol.models import ConfiguredAirbyteStream + + from airbyte_lib.datasets._base import DatasetBase + from airbyte_lib.telemetry import CacheTelemetryInfo + + +DEBUG_MODE = False # Set to True to enable additional debug logging. + + +class RecordDedupeMode(enum.Enum): + APPEND = "append" + REPLACE = "replace" + + +class SQLRuntimeError(Exception): + """Raised when an SQL operation fails.""" + + +class SQLCacheConfigBase(CacheConfigBase): + """Same as a regular config except it exposes the 'get_sql_alchemy_url()' method.""" + + dedupe_mode = RecordDedupeMode.REPLACE + schema_name: str = "airbyte_raw" + + table_prefix: str | None = None + """ A prefix to add to all table names. + If 'None', a prefix will be created based on the source name. + """ + + table_suffix: str = "" + """A suffix to add to all table names.""" + + @abc.abstractmethod + def get_sql_alchemy_url(self) -> str: + """Returns a SQL Alchemy URL.""" + ... + + @abc.abstractmethod + def get_database_name(self) -> str: + """Return the name of the database.""" + ... + + +class GenericSQLCacheConfig(SQLCacheConfigBase): + """Allows configuring 'sql_alchemy_url' directly.""" + + sql_alchemy_url: str + + @overrides + def get_sql_alchemy_url(self) -> str: + """Returns a SQL Alchemy URL.""" + return self.sql_alchemy_url + + +class SQLCacheBase(RecordProcessor): + """A base class to be used for SQL Caches. + + Optionally we can use a file cache to store the data in parquet files. + """ + + type_converter_class: type[SQLTypeConverter] = SQLTypeConverter + config_class: type[SQLCacheConfigBase] + file_writer_class: type[FileWriterBase] + + supports_merge_insert = False + use_singleton_connection = False # If true, the same connection is used for all operations. + + # Constructor: + + @final # We don't want subclasses to have to override the constructor. + def __init__( + self, + config: SQLCacheConfigBase | None = None, + file_writer: FileWriterBase | None = None, + **kwargs: dict[str, Any], # Added for future proofing purposes. + ) -> None: + self.config: SQLCacheConfigBase + self._engine: Engine | None = None + self._connection_to_reuse: Connection | None = None + super().__init__(config, **kwargs) + self._ensure_schema_exists() + + self.file_writer = file_writer or self.file_writer_class(config) + self.type_converter = self.type_converter_class() + + # Public interface: + + def get_sql_alchemy_url(self) -> str: + """Return the SQLAlchemy URL to use.""" + return self.config.get_sql_alchemy_url() + + @final + @cached_property + def database_name(self) -> str: + """Return the name of the database.""" + return self.config.get_database_name() + + @final + def get_sql_engine(self) -> Engine: + """Return a new SQL engine to use.""" + if self._engine: + return self._engine + + sql_alchemy_url = self.get_sql_alchemy_url() + if self.use_singleton_connection: + if self._connection_to_reuse is None: + # This temporary bootstrap engine will be created once and is needed to + # create the long-lived connection object. + bootstrap_engine = create_engine( + sql_alchemy_url, + ) + self._connection_to_reuse = bootstrap_engine.connect() + + self._engine = create_engine( + sql_alchemy_url, + creator=lambda: self._connection_to_reuse, + poolclass=StaticPool, + echo=DEBUG_MODE, + # isolation_level="AUTOCOMMIT", + ) + else: + # Regular engine creation for new connections + self._engine = create_engine( + sql_alchemy_url, + echo=DEBUG_MODE, + # isolation_level="AUTOCOMMIT", + ) + + return self._engine + + @contextmanager + def get_sql_connection(self) -> Generator[sqlalchemy.engine.Connection, None, None]: + """A context manager which returns a new SQL connection for running queries. + + If the connection needs to close, it will be closed automatically. + """ + if self.use_singleton_connection and self._connection_to_reuse is not None: + connection = self._connection_to_reuse + yield connection + + else: + with self.get_sql_engine().begin() as connection: + yield connection + + if not self.use_singleton_connection: + connection.close() + del connection + + def get_sql_table_name( + self, + stream_name: str, + ) -> str: + """Return the name of the SQL table for the given stream.""" + table_prefix = self.config.table_prefix or "" + + # TODO: Add default prefix based on the source name. + + return self._normalize_table_name( + f"{table_prefix}{stream_name}{self.config.table_suffix}", + ) + + @final + def get_sql_table( + self, + stream_name: str, + ) -> sqlalchemy.Table: + """Return a temporary table name.""" + table_name = self.get_sql_table_name(stream_name) + return sqlalchemy.Table( + table_name, + sqlalchemy.MetaData(schema=self.config.schema_name), + autoload_with=self.get_sql_engine(), + ) + + @final + @property + def streams( + self, + ) -> dict[str, DatasetBase]: + """Return a temporary table name.""" + # TODO: Add support for streams map, based on the cached catalog. + raise NotImplementedError("Streams map is not yet supported.") + + # Read methods: + + def get_records( + self, + stream_name: str, + ) -> Iterator[Mapping[str, Any]]: + """Uses SQLAlchemy to select all rows from the table. + + # TODO: Refactor to return a LazyDataset here. + """ + table_ref = self.get_sql_table(stream_name) + stmt = table_ref.select() + with self.get_sql_connection() as conn: + for row in conn.execute(stmt): + # Access to private member required because SQLAlchemy doesn't expose a public API. + # https://pydoc.dev/sqlalchemy/latest/sqlalchemy.engine.row.RowMapping.html + yield cast(Mapping[str, Any], row._mapping) # noqa: SLF001 + + def get_pandas_dataframe( + self, + stream_name: str, + ) -> pd.DataFrame: + """Return a Pandas data frame with the stream's data.""" + table_name = self.get_sql_table_name(stream_name) + engine = self.get_sql_engine() + return pd.read_sql_table(table_name, engine) + + # Protected members (non-public interface): + + def _ensure_schema_exists( + self, + ) -> None: + """Return a new (unique) temporary table name.""" + schema_name = self.config.schema_name + if schema_name in self._get_schemas_list(): + return + + sql = f"CREATE SCHEMA IF NOT EXISTS {schema_name}" + + try: + self._execute_sql(sql) + except Exception as ex: + # Ignore schema exists errors. + if "already exists" not in str(ex): + raise + + if DEBUG_MODE: + found_schemas = self._get_schemas_list() + assert ( + schema_name in found_schemas + ), f"Schema {schema_name} was not created. Found: {found_schemas}" + + @final + def _get_temp_table_name( + self, + stream_name: str, + batch_id: str | None = None, # ULID of the batch + ) -> str: + """Return a new (unique) temporary table name.""" + batch_id = batch_id or str(ulid.ULID()) + return self._normalize_table_name(f"{stream_name}_{batch_id}") + + def _fully_qualified( + self, + table_name: str, + ) -> str: + """Return the fully qualified name of the given table.""" + return f"{self.config.schema_name}.{table_name}" + + @final + def _create_table_for_loading( + self, + /, + stream_name: str, + batch_id: str, + ) -> str: + """Create a new table for loading data.""" + temp_table_name = self._get_temp_table_name(stream_name, batch_id) + column_definition_str = ",\n ".join( + f"{column_name} {sql_type}" + for column_name, sql_type in self._get_sql_column_definitions(stream_name).items() + ) + self._create_table(temp_table_name, column_definition_str) + + return temp_table_name + + def _get_tables_list( + self, + ) -> list[str]: + """Return a list of all tables in the database.""" + with self.get_sql_connection() as conn: + inspector: Inspector = sqlalchemy.inspect(conn) + return inspector.get_table_names(schema=self.config.schema_name) + + def _get_schemas_list( + self, + database_name: str | None = None, + ) -> list[str]: + """Return a list of all tables in the database.""" + inspector: Inspector = sqlalchemy.inspect(self.get_sql_engine()) + database_name = database_name or self.database_name + found_schemas = inspector.get_schema_names() + return [ + found_schema.split(".")[-1].strip('"') + for found_schema in found_schemas + if "." not in found_schema + or (found_schema.split(".")[0].lower().strip('"') == database_name.lower()) + ] + + def _ensure_final_table_exists( + self, + stream_name: str, + *, + create_if_missing: bool = True, + ) -> str: + """Create the final table if it doesn't already exist. + + Return the table name. + """ + table_name = self.get_sql_table_name(stream_name) + did_exist = self._table_exists(table_name) + if not did_exist and create_if_missing: + column_definition_str = ",\n ".join( + f"{column_name} {sql_type}" + for column_name, sql_type in self._get_sql_column_definitions( + stream_name, + ).items() + ) + self._create_table(table_name, column_definition_str) + + return table_name + + def _ensure_compatible_table_schema( + self, + stream_name: str, + table_name: str, + *, + raise_on_error: bool = False, + ) -> bool: + """Return true if the given table is compatible with the stream's schema. + + If raise_on_error is true, raise an exception if the table is not compatible. + + TODO: Expand this to check for column types and sizes, and to add missing columns. + + Returns true if the table is compatible, false if it is not. + """ + json_schema = self._get_stream_json_schema(stream_name) + stream_column_names: list[str] = json_schema["properties"].keys() + table_column_names: list[str] = self.get_sql_table(table_name).columns.keys() + + missing_columns: set[str] = set(stream_column_names) - set(table_column_names) + if missing_columns: + if raise_on_error: + raise RuntimeError( + f"Table {table_name} is missing columns: {missing_columns}", + ) + return False # Some columns are missing. + + return True # All columns exist. + + @final + def _create_table( + self, + table_name: str, + column_definition_str: str, + ) -> None: + if DEBUG_MODE: + assert table_name not in self._get_tables_list(), f"Table {table_name} already exists." + + cmd = f""" + CREATE TABLE {self._fully_qualified(table_name)} ( + {column_definition_str} + ) + """ + _ = self._execute_sql(cmd) + if DEBUG_MODE: + tables_list = self._get_tables_list() + assert ( + table_name in tables_list + ), f"Table {table_name} was not created. Found: {tables_list}" + + def _normalize_column_name( + self, + raw_name: str, + ) -> str: + return raw_name.lower().replace(" ", "_").replace("-", "_") + + def _normalize_table_name( + self, + raw_name: str, + ) -> str: + return raw_name.lower().replace(" ", "_").replace("-", "_") + + @final + def _get_sql_column_definitions( + self, + stream_name: str, + ) -> dict[str, sqlalchemy.types.TypeEngine]: + """Return the column definitions for the given stream.""" + columns: dict[str, sqlalchemy.types.TypeEngine] = {} + properties = self._get_stream_json_schema(stream_name)["properties"] + for property_name, json_schema_property_def in properties.items(): + clean_prop_name = self._normalize_column_name(property_name) + columns[clean_prop_name] = self.type_converter.to_sql_type( + json_schema_property_def, + ) + + # TODO: Add the metadata columns (this breaks tests) + # columns["_airbyte_extracted_at"] = sqlalchemy.TIMESTAMP() + # columns["_airbyte_loaded_at"] = sqlalchemy.TIMESTAMP() + return columns + + @final + def _get_stream_config( + self, + stream_name: str, + ) -> ConfiguredAirbyteStream: + """Return the column definitions for the given stream.""" + if not self.source_catalog: + raise RuntimeError("Cannot get stream JSON schema without a catalog.") + + matching_streams: list[ConfiguredAirbyteStream] = [ + stream for stream in self.source_catalog.streams if stream.stream.name == stream_name + ] + if not matching_streams: + raise RuntimeError(f"Stream '{stream_name}' not found in catalog.") + + if len(matching_streams) > 1: + raise RuntimeError(f"Multiple streams found with name '{stream_name}'.") + + return matching_streams[0] + + @final + def _get_stream_json_schema( + self, + stream_name: str, + ) -> dict[str, Any]: + """Return the column definitions for the given stream.""" + return self._get_stream_config(stream_name).stream.json_schema + + @overrides + def _write_batch( + self, + stream_name: str, + batch_id: str, + record_batch: pa.Table | pa.RecordBatch, + ) -> FileWriterBatchHandle: + """Process a record batch. + + Return the path to the cache file. + """ + return self.file_writer.write_batch(stream_name, batch_id, record_batch) + + def _cleanup_batch( + self, + stream_name: str, + batch_id: str, + batch_handle: BatchHandle, + ) -> None: + """Clean up the cache. + + For SQL caches, we only need to call the cleanup operation on the file writer. + + Subclasses should call super() if they override this method. + """ + self.file_writer.cleanup_batch(stream_name, batch_id, batch_handle) + + @final + @overrides + def _finalize_batches(self, stream_name: str) -> dict[str, BatchHandle]: + """Finalize all uncommitted batches. + + This is a generic 'final' implementation, which should not be overridden. + + Returns a mapping of batch IDs to batch handles, for those processed batches. + + TODO: Add a dedupe step here to remove duplicates from the temp table. + Some sources will send us duplicate records within the same stream, + although this is a fairly rare edge case we can ignore in V1. + """ + with self._finalizing_batches(stream_name) as batches_to_finalize: + if not batches_to_finalize: + return {} + + files: list[Path] = [] + # Get a list of all files to finalize from all pending batches. + for batch_handle in batches_to_finalize.values(): + batch_handle = cast(FileWriterBatchHandle, batch_handle) + files += batch_handle.files + # Use the max batch ID as the batch ID for table names. + max_batch_id = max(batches_to_finalize.keys()) + + # Make sure the target schema and target table exist. + self._ensure_schema_exists() + final_table_name = self._ensure_final_table_exists( + stream_name, + create_if_missing=True, + ) + self._ensure_compatible_table_schema( + stream_name=stream_name, + table_name=final_table_name, + raise_on_error=True, + ) + + try: + temp_table_name = self._write_files_to_new_table( + files, + stream_name, + max_batch_id, + ) + self._write_temp_table_to_final_table( + stream_name, + temp_table_name, + final_table_name, + ) + finally: + self._drop_temp_table(temp_table_name, if_exists=True) + + # Return the batch handles as measure of work completed. + return batches_to_finalize + + def _execute_sql(self, sql: str | TextClause | Executable) -> CursorResult: + """Execute the given SQL statement.""" + if isinstance(sql, str): + sql = text(sql) + if isinstance(sql, TextClause): + sql = sql.execution_options( + autocommit=True, + ) + + with self.get_sql_connection() as conn: + try: + result = conn.execute(sql) + except ( + sqlalchemy.exc.ProgrammingError, + sqlalchemy.exc.SQLAlchemyError, + ) as ex: + msg = f"Error when executing SQL:\n{sql}\n{type(ex).__name__}{ex!s}" + raise SQLRuntimeError(msg) from None # from ex + + return result + + def _drop_temp_table( + self, + table_name: str, + *, + if_exists: bool = True, + ) -> None: + """Drop the given table.""" + exists_str = "IF EXISTS" if if_exists else "" + self._execute_sql(f"DROP TABLE {exists_str} {self._fully_qualified(table_name)}") + + def _write_files_to_new_table( + self, + files: list[Path], + stream_name: str, + batch_id: str, + ) -> str: + """Write a file(s) to a new table. + + This is a generic implementation, which can be overridden by subclasses + to improve performance. + """ + temp_table_name = self._create_table_for_loading(stream_name, batch_id) + for file_path in files: + with pa.parquet.ParquetFile(file_path) as pf: + record_batch = pf.read() + dataframe = record_batch.to_pandas() + + # Pandas will auto-create the table if it doesn't exist, which we don't want. + if not self._table_exists(temp_table_name): + raise RuntimeError(f"Table {temp_table_name} does not exist after creation.") + + dataframe.to_sql( + temp_table_name, + self.get_sql_alchemy_url(), + schema=self.config.schema_name, + if_exists="append", + index=False, + dtype=self._get_sql_column_definitions(stream_name), + ) + return temp_table_name + + @final + def _write_temp_table_to_final_table( + self, + stream_name: str, + temp_table_name: str, + final_table_name: str, + ) -> None: + """Merge the temp table into the final table.""" + if self.config.dedupe_mode == RecordDedupeMode.REPLACE: + if not self.supports_merge_insert: + raise NotImplementedError( + "Deduping was requested but merge-insert is not yet supported.", + ) + + if not self._get_primary_keys(stream_name): + self._swap_temp_table_with_final_table( + stream_name, + temp_table_name, + final_table_name, + ) + else: + self._merge_temp_table_to_final_table( + stream_name, + temp_table_name, + final_table_name, + ) + + else: + self._append_temp_table_to_final_table( + stream_name=stream_name, + temp_table_name=temp_table_name, + final_table_name=final_table_name, + ) + + def _append_temp_table_to_final_table( + self, + temp_table_name: str, + final_table_name: str, + stream_name: str, + ) -> None: + nl = "\n" + columns = self._get_sql_column_definitions(stream_name).keys() + self._execute_sql( + f""" + INSERT INTO {self._fully_qualified(final_table_name)} ( + {f',{nl} '.join(columns)} + ) + SELECT + {f',{nl} '.join(columns)} + FROM {self._fully_qualified(temp_table_name)} + """, + ) + + def _get_primary_keys( + self, + stream_name: str, + ) -> list[str]: + pks = self._get_stream_config(stream_name).primary_key + if not pks: + return [] + + joined_pks = [".".join(pk) for pk in pks] + for pk in joined_pks: + if "." in pk: + msg = "Nested primary keys are not yet supported. Found: {pk}" + raise NotImplementedError(msg) + + return joined_pks + + def _swap_temp_table_with_final_table( + self, + stream_name: str, + temp_table_name: str, + final_table_name: str, + ) -> None: + """Merge the temp table into the main one. + + This implementation requires MERGE support in the SQL DB. + Databases that do not support this syntax can override this method. + """ + if final_table_name is None: + raise ValueError("Arg 'final_table_name' cannot be None.") + if temp_table_name is None: + raise ValueError("Arg 'temp_table_name' cannot be None.") + + _ = stream_name + deletion_name = f"{final_table_name}_deleteme" + commands = [ + f"ALTER TABLE {final_table_name} RENAME TO {deletion_name}", + f"ALTER TABLE {temp_table_name} RENAME TO {final_table_name}", + f"DROP TABLE {deletion_name}", + ] + for cmd in commands: + self._execute_sql(cmd) + + def _merge_temp_table_to_final_table( + self, + stream_name: str, + temp_table_name: str, + final_table_name: str, + ) -> None: + """Merge the temp table into the main one. + + This implementation requires MERGE support in the SQL DB. + Databases that do not support this syntax can override this method. + """ + nl = "\n" + columns = self._get_sql_column_definitions(stream_name).keys() + pk_columns = self._get_primary_keys(stream_name) + non_pk_columns = columns - pk_columns + join_clause = "{nl} AND ".join(f"tmp.{pk_col} = final.{pk_col}" for pk_col in pk_columns) + set_clause = "{nl} ".join(f"{col} = tmp.{col}" for col in non_pk_columns) + self._execute_sql( + f""" + MERGE INTO {self._fully_qualified(final_table_name)} final + USING ( + SELECT * + FROM {self._fully_qualified(temp_table_name)} + ) AS tmp + ON {join_clause} + WHEN MATCHED THEN UPDATE + SET + {set_clause} + WHEN NOT MATCHED THEN INSERT + ( + {f',{nl} '.join(columns)} + ) + VALUES ( + tmp.{f',{nl} tmp.'.join(columns)} + ); + """, + ) + + @final + def _table_exists( + self, + table_name: str, + ) -> bool: + """Return true if the given table exists.""" + return table_name in self._get_tables_list() + + @abc.abstractmethod + def get_telemetry_info(self) -> CacheTelemetryInfo: + pass diff --git a/airbyte-lib/airbyte_lib/caches/duckdb.py b/airbyte-lib/airbyte_lib/caches/duckdb.py new file mode 100644 index 000000000000..0d6ba6efe38a --- /dev/null +++ b/airbyte-lib/airbyte_lib/caches/duckdb.py @@ -0,0 +1,157 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""A DuckDB implementation of the cache.""" + +from __future__ import annotations + +from pathlib import Path +from typing import cast + +from overrides import overrides + +from airbyte_lib._file_writers import ParquetWriter, ParquetWriterConfig +from airbyte_lib.caches.base import SQLCacheBase, SQLCacheConfigBase +from airbyte_lib.telemetry import CacheTelemetryInfo + + +class DuckDBCacheConfig(SQLCacheConfigBase, ParquetWriterConfig): + """Configuration for the DuckDB cache. + + Also inherits config from the ParquetWriter, which is responsible for writing files to disk. + """ + + db_path: Path | str + """Normally db_path is a Path object. + + There are some cases, such as when connecting to MotherDuck, where it could be a string that + is not also a path, such as "md:" to connect the user's default MotherDuck DB. + """ + schema_name: str = "main" + """The name of the schema to write to. Defaults to "main".""" + + @overrides + def get_sql_alchemy_url(self) -> str: + """Return the SQLAlchemy URL to use.""" + # return f"duckdb:///{self.db_path}?schema={self.schema_name}" + return f"duckdb:///{self.db_path!s}" + + def get_database_name(self) -> str: + """Return the name of the database.""" + if self.db_path == ":memory:": + return "memory" + + # Return the file name without the extension + return str(self.db_path).split("/")[-1].split(".")[0] + + +class DuckDBCacheBase(SQLCacheBase): + """A DuckDB implementation of the cache. + + Parquet is used for local file storage before bulk loading. + Unlike the Snowflake implementation, we can't use the COPY command to load data + so we insert as values instead. + """ + + config_class = DuckDBCacheConfig + supports_merge_insert = True + + @overrides + def get_telemetry_info(self) -> CacheTelemetryInfo: + return CacheTelemetryInfo("duckdb") + + @overrides + def _setup(self) -> None: + """Create the database parent folder if it doesn't yet exist.""" + config = cast(DuckDBCacheConfig, self.config) + + if config.db_path == ":memory:": + return + + Path(config.db_path).parent.mkdir(parents=True, exist_ok=True) + + +class DuckDBCache(DuckDBCacheBase): + """A DuckDB implementation of the cache. + + Parquet is used for local file storage before bulk loading. + Unlike the Snowflake implementation, we can't use the COPY command to load data + so we insert as values instead. + """ + + file_writer_class = ParquetWriter + + @overrides + def _merge_temp_table_to_final_table( + self, + stream_name: str, + temp_table_name: str, + final_table_name: str, + ) -> None: + """Merge the temp table into the main one. + + This implementation requires MERGE support in the SQL DB. + Databases that do not support this syntax can override this method. + """ + if not self._get_primary_keys(stream_name): + raise RuntimeError( + f"Primary keys not found for stream {stream_name}. " + "Cannot run merge updates without primary keys." + ) + + _ = stream_name + final_table = self._fully_qualified(final_table_name) + staging_table = self._fully_qualified(temp_table_name) + self._execute_sql( + # https://duckdb.org/docs/sql/statements/insert.html + # NOTE: This depends on primary keys being set properly in the final table. + f""" + INSERT OR REPLACE INTO {final_table} BY NAME + (SELECT * FROM {staging_table}) + """ + ) + + @overrides + def _ensure_compatible_table_schema( + self, + stream_name: str, + table_name: str, + *, + raise_on_error: bool = True, + ) -> bool: + """Return true if the given table is compatible with the stream's schema. + + In addition to the base implementation, this also checks primary keys. + """ + # call super + if not super()._ensure_compatible_table_schema( + stream_name=stream_name, + table_name=table_name, + raise_on_error=raise_on_error, + ): + return False + + pk_cols = self._get_primary_keys(stream_name) + table = self.get_sql_table(table_name) + table_pk_cols = table.primary_key.columns.keys() + if set(pk_cols) != set(table_pk_cols): + if raise_on_error: + raise RuntimeError( + f"Primary keys do not match for table {table_name}. " + f"Expected: {pk_cols}. " + f"Found: {table_pk_cols}.", + ) + return False + + return True + + def _write_files_to_new_table( + self, + files: list[Path], + stream_name: str, + batch_id: str, + ) -> str: + """Write a file(s) to a new table. + + TODO: Optimize this for DuckDB instead of calling the base implementation. + """ + return super()._write_files_to_new_table(files, stream_name, batch_id) diff --git a/airbyte-lib/airbyte_lib/caches/postgres.py b/airbyte-lib/airbyte_lib/caches/postgres.py new file mode 100644 index 000000000000..72fe8291bf54 --- /dev/null +++ b/airbyte-lib/airbyte_lib/caches/postgres.py @@ -0,0 +1,57 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""A Postgres implementation of the cache.""" + +from __future__ import annotations + +from overrides import overrides + +from airbyte_lib._file_writers import ParquetWriter, ParquetWriterConfig +from airbyte_lib.caches.base import SQLCacheBase, SQLCacheConfigBase +from airbyte_lib.telemetry import CacheTelemetryInfo + + +class PostgresCacheConfig(SQLCacheConfigBase, ParquetWriterConfig): + """Configuration for the Postgres cache. + + Also inherits config from the ParquetWriter, which is responsible for writing files to disk. + """ + + host: str + port: int + username: str + password: str + database: str + + # Already defined in base class: `schema_name` + + @overrides + def get_sql_alchemy_url(self) -> str: + """Return the SQLAlchemy URL to use.""" + return ( + f"postgresql://{self.username}:{self.password}@{self.host}:{self.port}/{self.database}" + ) + + def get_database_name(self) -> str: + """Return the name of the database.""" + return self.database + + +class PostgresCache(SQLCacheBase): + """A Postgres implementation of the cache. + + Parquet is used for local file storage before bulk loading. + Unlike the Snowflake implementation, we can't use the COPY command to load data + so we insert as values instead. + + TOOD: Add optimized bulk load path for Postgres. Could use an alternate file writer + or another import method. (Relatively low priority, since for now it works fine as-is.) + """ + + config_class = PostgresCacheConfig + file_writer_class = ParquetWriter + supports_merge_insert = True + + @overrides + def get_telemetry_info(self) -> CacheTelemetryInfo: + return CacheTelemetryInfo("postgres") diff --git a/airbyte-lib/airbyte_lib/caches/snowflake.py b/airbyte-lib/airbyte_lib/caches/snowflake.py new file mode 100644 index 000000000000..86c30c995cf6 --- /dev/null +++ b/airbyte-lib/airbyte_lib/caches/snowflake.py @@ -0,0 +1,86 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""A Snowflake implementation of the cache. + +TODO: FIXME: Snowflake Cache doesn't work yet. It's a work in progress. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from overrides import overrides +from snowflake.sqlalchemy import URL + +from airbyte_lib._file_writers import ParquetWriter, ParquetWriterConfig +from airbyte_lib.caches.base import RecordDedupeMode, SQLCacheBase, SQLCacheConfigBase +from airbyte_lib.telemetry import CacheTelemetryInfo + + +if TYPE_CHECKING: + from pathlib import Path + + +class SnowflakeCacheConfig(SQLCacheConfigBase, ParquetWriterConfig): + """Configuration for the Snowflake cache. + + Also inherits config from the ParquetWriter, which is responsible for writing files to disk. + """ + + account: str + username: str + password: str + warehouse: str + database: str + role: str + + dedupe_mode = RecordDedupeMode.APPEND + + # Already defined in base class: + # schema_name: str + + @overrides + def get_sql_alchemy_url(self) -> str: + """Return the SQLAlchemy URL to use.""" + return str( + URL( + account=self.account, + user=self.username, + password=self.password, + database=self.database, + warehouse=self.warehouse, + role=self.role, + ) + ) + + def get_database_name(self) -> str: + """Return the name of the database.""" + return self.database + + +class SnowflakeSQLCache(SQLCacheBase): + """A Snowflake implementation of the cache. + + Parquet is used for local file storage before bulk loading. + """ + + config_class = SnowflakeCacheConfig + file_writer_class = ParquetWriter + + @overrides + def _write_files_to_new_table( + self, + files: list[Path], + stream_name: str, + batch_id: str, + ) -> str: + """Write a file(s) to a new table. + + TODO: Override the base implementation to use the COPY command. + TODO: Make sure this works for all data types. + """ + return super()._write_files_to_new_table(files, stream_name, batch_id) + + @overrides + def get_telemetry_info(self) -> CacheTelemetryInfo: + return CacheTelemetryInfo("snowflake") diff --git a/airbyte-lib/airbyte_lib/config.py b/airbyte-lib/airbyte_lib/config.py new file mode 100644 index 000000000000..4401ca72f85f --- /dev/null +++ b/airbyte-lib/airbyte_lib/config.py @@ -0,0 +1,13 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""Define base Config interface, used by Caches and also File Writers (Processors).""" + +from __future__ import annotations + +from pydantic import BaseModel + + +class CacheConfigBase( + BaseModel +): # TODO: meta=EnforceOverrides (Pydantic doesn't like it currently) + pass diff --git a/airbyte-lib/airbyte_lib/datasets/__init__.py b/airbyte-lib/airbyte_lib/datasets/__init__.py new file mode 100644 index 000000000000..862eee3e8baf --- /dev/null +++ b/airbyte-lib/airbyte_lib/datasets/__init__.py @@ -0,0 +1,10 @@ +from airbyte_lib.datasets._base import DatasetBase +from airbyte_lib.datasets._cached import CachedDataset +from airbyte_lib.datasets._map import DatasetMap + + +__all__ = [ + "CachedDataset", + "DatasetBase", + "DatasetMap", +] diff --git a/airbyte-lib/airbyte_lib/datasets/_base.py b/airbyte-lib/airbyte_lib/datasets/_base.py new file mode 100644 index 000000000000..b42e5258e559 --- /dev/null +++ b/airbyte-lib/airbyte_lib/datasets/_base.py @@ -0,0 +1,28 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from abc import ABC, abstractmethod +from collections.abc import Iterator, Mapping +from typing import Any, cast + +from pandas import DataFrame +from typing_extensions import Self + + +class DatasetBase(ABC, Iterator[Mapping[str, Any]]): + """Base implementation for all datasets.""" + + def __iter__(self) -> Self: + """Return the iterator object (usually self).""" + return self + + @abstractmethod + def __next__(self) -> Mapping[str, Any]: + """Return the next value from the iterator.""" + raise NotImplementedError + + def to_pandas(self) -> DataFrame: + """Return a pandas DataFrame representation of the dataset.""" + # Technically, we return an iterator of Mapping objects. However, pandas + # expects an iterator of dict objects. This cast is safe because we know + # duck typing is correct for this use case. + return DataFrame(cast(Iterator[dict[str, Any]], self)) diff --git a/airbyte-lib/airbyte_lib/datasets/_cached.py b/airbyte-lib/airbyte_lib/datasets/_cached.py new file mode 100644 index 000000000000..37aed0458312 --- /dev/null +++ b/airbyte-lib/airbyte_lib/datasets/_cached.py @@ -0,0 +1,34 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any + +from typing_extensions import Self + +from airbyte_lib.datasets._base import DatasetBase + + +if TYPE_CHECKING: + from pandas import DataFrame + from sqlalchemy import Table + + from airbyte_lib.caches import SQLCacheBase + + +class CachedDataset(DatasetBase): + def __init__(self, cache: "SQLCacheBase", stream: str) -> None: + self._cache = cache + self._stream = stream + self._iterator = iter(self._cache.get_records(self._stream)) + + def __iter__(self) -> Self: + return self + + def __next__(self) -> Mapping[str, Any]: + return next(self._iterator) + + def to_pandas(self) -> "DataFrame": + return self._cache.get_pandas_dataframe(self._stream) + + def to_sql_table(self) -> "Table": + return self._cache.get_sql_table(self._stream) diff --git a/airbyte-lib/airbyte_lib/datasets/_lazy.py b/airbyte-lib/airbyte_lib/datasets/_lazy.py new file mode 100644 index 000000000000..e08f844c4078 --- /dev/null +++ b/airbyte-lib/airbyte_lib/datasets/_lazy.py @@ -0,0 +1,47 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from overrides import overrides +from typing_extensions import Self + +from airbyte_lib.datasets import DatasetBase + + +if TYPE_CHECKING: + from collections.abc import Callable, Iterator + + +class LazyDataset(DatasetBase): + """A dataset that is loaded incrementally from a source or a SQL query. + + TODO: Test and debug this. It is not yet implemented anywhere in the codebase. + For now it servers as a placeholder. + """ + + def __init__( + self, + iterator: Iterator, + on_open: Callable | None = None, + on_close: Callable | None = None, + ) -> None: + self._iterator = iterator + self._on_open = on_open + self._on_close = on_close + raise NotImplementedError("This class is not implemented yet.") + + @overrides + def __iter__(self) -> Self: + raise NotImplementedError("This class is not implemented yet.") + # Pseudocode: + # if self._on_open is not None: + # self._on_open() + + # yield from self._iterator + + # if self._on_close is not None: + # self._on_close() + + def __next__(self) -> dict[str, Any]: + return next(self._iterator) diff --git a/airbyte-lib/airbyte_lib/datasets/_map.py b/airbyte-lib/airbyte_lib/datasets/_map.py new file mode 100644 index 000000000000..3881e1d33da8 --- /dev/null +++ b/airbyte-lib/airbyte_lib/datasets/_map.py @@ -0,0 +1,27 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""A generic interface for a set of streams. + +TODO: This is a work in progress. It is not yet used by any other code. +TODO: Implement before release, or delete. +""" + +from collections.abc import Iterator, Mapping + +from airbyte_lib.datasets._base import DatasetBase + + +class DatasetMap(Mapping): + """A generic interface for a set of streams or datasets.""" + + def __init__(self) -> None: + self._datasets: dict[str, DatasetBase] = {} + + def __getitem__(self, key: str) -> DatasetBase: + return self._datasets[key] + + def __iter__(self) -> Iterator[str]: + return iter(self._datasets) + + def __len__(self) -> int: + return len(self._datasets) diff --git a/airbyte-lib/airbyte_lib/py.typed b/airbyte-lib/airbyte_lib/py.typed new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-lib/airbyte_lib/registry.py b/airbyte-lib/airbyte_lib/registry.py new file mode 100644 index 000000000000..e0afdbaf2c3a --- /dev/null +++ b/airbyte-lib/airbyte_lib/registry.py @@ -0,0 +1,51 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from pathlib import Path + +import requests + +from airbyte_lib.version import get_version + + +@dataclass +class ConnectorMetadata: + name: str + latest_available_version: str + + +_cache: dict[str, ConnectorMetadata] | None = None + +REGISTRY_URL = "https://connectors.airbyte.com/files/registries/v0/oss_registry.json" + + +def _update_cache() -> None: + global _cache + if os.environ.get("AIRBYTE_LOCAL_REGISTRY"): + with Path(str(os.environ.get("AIRBYTE_LOCAL_REGISTRY"))).open() as f: + data = json.load(f) + else: + response = requests.get( + REGISTRY_URL, headers={"User-Agent": f"airbyte-lib-{get_version()}"} + ) + response.raise_for_status() + data = response.json() + _cache = {} + for connector in data["sources"]: + name = connector["dockerRepository"].replace("airbyte/", "") + _cache[name] = ConnectorMetadata(name, connector["dockerImageTag"]) + + +def get_connector_metadata(name: str) -> ConnectorMetadata: + """Check the cache for the connector. + + If the cache is empty, populate by calling update_cache. + """ + if not _cache: + _update_cache() + if not _cache or name not in _cache: + raise Exception(f"Connector {name} not found") + return _cache[name] diff --git a/airbyte-lib/airbyte_lib/results.py b/airbyte-lib/airbyte_lib/results.py new file mode 100644 index 000000000000..3261416c130b --- /dev/null +++ b/airbyte-lib/airbyte_lib/results.py @@ -0,0 +1,22 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from sqlalchemy.engine import Engine + +from airbyte_lib.caches import SQLCacheBase +from airbyte_lib.datasets import CachedDataset + + +class ReadResult: + def __init__(self, processed_records: int, cache: SQLCacheBase) -> None: + self.processed_records = processed_records + self._cache = cache + + def __getitem__(self, stream: str) -> CachedDataset: + return CachedDataset(self._cache, stream) + + def get_sql_engine(self) -> Engine: + return self._cache.get_sql_engine() + + @property + def cache(self) -> SQLCacheBase: + return self._cache diff --git a/airbyte-lib/airbyte_lib/source.py b/airbyte-lib/airbyte_lib/source.py new file mode 100644 index 000000000000..415284994510 --- /dev/null +++ b/airbyte-lib/airbyte_lib/source.py @@ -0,0 +1,355 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +import json +import tempfile +from contextlib import contextmanager, suppress +from typing import TYPE_CHECKING, Any + +import jsonschema + +from airbyte_protocol.models import ( + AirbyteCatalog, + AirbyteMessage, + AirbyteRecordMessage, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + ConnectorSpecification, + DestinationSyncMode, + Status, + SyncMode, + Type, +) + +from airbyte_lib._factories.cache_factories import get_default_cache +from airbyte_lib._util import protocol_util # Internal utility functions +from airbyte_lib.results import ReadResult +from airbyte_lib.telemetry import ( + CacheTelemetryInfo, + SyncState, + send_telemetry, + streaming_cache_info, +) + + +if TYPE_CHECKING: + from collections.abc import Generator, Iterable, Iterator + + from airbyte_lib._executor import Executor + from airbyte_lib.caches import SQLCacheBase + + +@contextmanager +def as_temp_files(files: list[Any]) -> Generator[list[Any], Any, None]: + temp_files: list[Any] = [] + try: + for content in files: + temp_file = tempfile.NamedTemporaryFile(mode="w+t", delete=True) + temp_file.write( + json.dumps(content) if isinstance(content, dict) else content, + ) + temp_file.flush() + temp_files.append(temp_file) + yield [file.name for file in temp_files] + finally: + for temp_file in temp_files: + with suppress(Exception): + temp_file.close() + + +class Source: + """A class representing a source that can be called.""" + + def __init__( + self, + executor: Executor, + name: str, + config: dict[str, Any] | None = None, + streams: list[str] | None = None, + ) -> None: + self._processed_records = 0 + self.executor = executor + self.name = name + self.streams: list[str] | None = None + self._processed_records = 0 + self._config_dict: dict[str, Any] | None = None + self._last_log_messages: list[str] = [] + self._discovered_catalog: AirbyteCatalog | None = None + self._spec: ConnectorSpecification | None = None + if config is not None: + self.set_config(config) + if streams is not None: + self.set_streams(streams) + + def set_streams(self, streams: list[str]) -> None: + available_streams = self.get_available_streams() + for stream in streams: + if stream not in available_streams: + raise Exception( + f"Stream {stream} is not available for connector {self.name}. " + f"Choose from: {available_streams}", + ) + self.streams = streams + + def set_config(self, config: dict[str, Any]) -> None: + self._validate_config(config) + self._config_dict = config + + @property + def _config(self) -> dict[str, Any]: + if self._config_dict is None: + raise Exception( + "Config is not set, either set in get_connector or via source.set_config", + ) + return self._config_dict + + def _discover(self) -> AirbyteCatalog: + """Call discover on the connector. + + This involves the following steps: + * Write the config to a temporary file + * execute the connector with discover --config + * Listen to the messages and return the first AirbyteCatalog that comes along. + * Make sure the subprocess is killed when the function returns. + """ + with as_temp_files([self._config]) as [config_file]: + for msg in self._execute(["discover", "--config", config_file]): + if msg.type == Type.CATALOG and msg.catalog: + return msg.catalog + raise Exception( + f"Connector did not return a catalog. Last logs: {self._last_log_messages}", + ) + + def _validate_config(self, config: dict[str, Any]) -> None: + """Validate the config against the spec.""" + spec = self._get_spec(force_refresh=False) + jsonschema.validate(config, spec.connectionSpecification) + + def get_available_streams(self) -> list[str]: + """Get the available streams from the spec.""" + return [s.name for s in self._discover().streams] + + def _get_spec(self, *, force_refresh: bool = False) -> ConnectorSpecification: + """Call spec on the connector. + + This involves the following steps: + * execute the connector with spec + * Listen to the messages and return the first AirbyteCatalog that comes along. + * Make sure the subprocess is killed when the function returns. + """ + if force_refresh or self._spec is None: + for msg in self._execute(["spec"]): + if msg.type == Type.SPEC and msg.spec: + self._spec = msg.spec + break + + if self._spec: + return self._spec + + raise Exception( + f"Connector did not return a spec. Last logs: {self._last_log_messages}", + ) + + @property + def raw_catalog(self) -> AirbyteCatalog: + """Get the raw catalog for the given streams.""" + return self._discover() + + @property + def configured_catalog(self) -> ConfiguredAirbyteCatalog: + """Get the configured catalog for the given streams.""" + if self._discovered_catalog is None: + self._discovered_catalog = self._discover() + + return ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=s, + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + primary_key=None, + ) + for s in self._discovered_catalog.streams + if self.streams is None or s.name in self.streams + ], + ) + + def get_records(self, stream: str) -> Iterator[dict[str, Any]]: + """Read a stream from the connector. + + This involves the following steps: + * Call discover to get the catalog + * Generate a configured catalog that syncs the given stream in full_refresh mode + * Write the configured catalog and the config to a temporary file + * execute the connector with read --config --catalog + * Listen to the messages and return the first AirbyteRecordMessages that come along. + * Make sure the subprocess is killed when the function returns. + """ + catalog = self._discover() + configured_catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=s, + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + for s in catalog.streams + if s.name == stream + ], + ) + if len(configured_catalog.streams) == 0: + raise ValueError( + f"Stream {stream} is not available for connector {self.name}, " + f"choose from {self.get_available_streams()}", + ) + + iterator: Iterable[dict[str, Any]] = protocol_util.airbyte_messages_to_record_dicts( + self._read_with_catalog(streaming_cache_info, configured_catalog), + ) + yield from iterator # TODO: Refactor to use LazyDataset here + + def check(self) -> None: + """Call check on the connector. + + This involves the following steps: + * Write the config to a temporary file + * execute the connector with check --config + * Listen to the messages and return the first AirbyteCatalog that comes along. + * Make sure the subprocess is killed when the function returns. + """ + with as_temp_files([self._config]) as [config_file]: + for msg in self._execute(["check", "--config", config_file]): + if msg.type == Type.CONNECTION_STATUS and msg.connectionStatus: + if msg.connectionStatus.status != Status.FAILED: + return # Success! + + raise Exception( + f"Connector returned failed status: {msg.connectionStatus.message}", + ) + raise Exception( + f"Connector did not return check status. Last logs: {self._last_log_messages}", + ) + + def install(self) -> None: + """Install the connector if it is not yet installed.""" + self.executor.install() + + def uninstall(self) -> None: + """Uninstall the connector if it is installed. + + This only works if the use_local_install flag wasn't used and installation is managed by + airbyte-lib. + """ + self.executor.uninstall() + + def _read(self, cache_info: CacheTelemetryInfo) -> Iterable[AirbyteRecordMessage]: + """ + Call read on the connector. + + This involves the following steps: + * Call discover to get the catalog + * Generate a configured catalog that syncs all streams in full_refresh mode + * Write the configured catalog and the config to a temporary file + * execute the connector with read --config --catalog + * Listen to the messages and return the AirbyteRecordMessages that come along. + """ + catalog = self._discover() + configured_catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=s, + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + ) + for s in catalog.streams + if self.streams is None or s.name in self.streams + ], + ) + yield from self._read_with_catalog(cache_info, configured_catalog) + + def _read_with_catalog( + self, + cache_info: CacheTelemetryInfo, + catalog: ConfiguredAirbyteCatalog, + ) -> Iterator[AirbyteMessage]: + """Call read on the connector. + + This involves the following steps: + * Write the config to a temporary file + * execute the connector with read --config --catalog + * Listen to the messages and return the AirbyteRecordMessages that come along. + * Send out telemetry on the performed sync (with information about which source was used and + the type of the cache) + """ + source_tracking_information = self.executor.get_telemetry_info() + send_telemetry(source_tracking_information, cache_info, SyncState.STARTED) + try: + with as_temp_files([self._config, catalog.json()]) as [ + config_file, + catalog_file, + ]: + yield from self._execute( + ["read", "--config", config_file, "--catalog", catalog_file], + ) + except Exception: + send_telemetry( + source_tracking_information, cache_info, SyncState.FAILED, self._processed_records + ) + raise + finally: + send_telemetry( + source_tracking_information, + cache_info, + SyncState.SUCCEEDED, + self._processed_records, + ) + + def _add_to_logs(self, message: str) -> None: + self._last_log_messages.append(message) + self._last_log_messages = self._last_log_messages[-10:] + + def _execute(self, args: list[str]) -> Iterator[AirbyteMessage]: + """Execute the connector with the given arguments. + + This involves the following steps: + * Locate the right venv. It is called ".venv-" + * Spawn a subprocess with .venv-/bin/ + * Read the output line by line of the subprocess and serialize them AirbyteMessage objects. + Drop if not valid. + """ + self.executor.ensure_installation() + + try: + self._last_log_messages = [] + for line in self.executor.execute(args): + try: + message = AirbyteMessage.parse_raw(line) + yield message + if message.type == Type.LOG: + self._add_to_logs(message.log.message) + except Exception: + self._add_to_logs(line) + except Exception as e: + raise Exception(f"Execution failed. Last logs: {self._last_log_messages}") from e + + def _tally_records( + self, + messages: Iterable[AirbyteRecordMessage], + ) -> Generator[AirbyteRecordMessage, Any, None]: + """This method simply tallies the number of records processed and yields the messages.""" + self._processed_records = 0 # Reset the counter before we start + for message in messages: + self._processed_records += 1 + yield message + + def read(self, cache: SQLCacheBase | None = None) -> ReadResult: + if cache is None: + cache = get_default_cache() + + cache.register_source(source_name=self.name, source_catalog=self.configured_catalog) + cache.process_airbyte_messages(self._tally_records(self._read(cache.get_telemetry_info()))) + + return ReadResult( + processed_records=self._processed_records, + cache=cache, + ) diff --git a/airbyte-lib/airbyte_lib/telemetry.py b/airbyte-lib/airbyte_lib/telemetry.py new file mode 100644 index 000000000000..5ab816208383 --- /dev/null +++ b/airbyte-lib/airbyte_lib/telemetry.py @@ -0,0 +1,80 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from __future__ import annotations + +import datetime +import os +from contextlib import suppress +from dataclasses import asdict, dataclass +from enum import Enum +from typing import Any + +import requests + +from airbyte_lib.version import get_version + + +# TODO: Use production tracking key +# TODO: This 'or' is a no-op. Intentional? Should we switch order to prefer env var if available? +TRACKING_KEY = "jxT1qP9WEKwR3vtKMwP9qKhfQEGFtIM1" or str(os.environ.get("AIRBYTE_TRACKING_KEY")) # noqa: SIM222 + + +class SourceType(str, Enum): + VENV = "venv" + LOCAL_INSTALL = "local_install" + + +@dataclass +class CacheTelemetryInfo: + type: str + + +streaming_cache_info = CacheTelemetryInfo("streaming") + + +class SyncState(str, Enum): + STARTED = "started" + FAILED = "failed" + SUCCEEDED = "succeeded" + + +@dataclass +class SourceTelemetryInfo: + name: str + type: SourceType + version: str | None + + +def send_telemetry( + source_info: SourceTelemetryInfo, + cache_info: CacheTelemetryInfo, + state: SyncState, + number_of_records: int | None = None, +) -> None: + # If DO_NOT_TRACK is set, we don't send any telemetry + if os.environ.get("DO_NOT_TRACK"): + return + + current_time: str = datetime.datetime.utcnow().isoformat() # noqa: DTZ003 # prefer now() over utcnow() + payload: dict[str, Any] = { + "anonymousId": "airbyte-lib-user", + "event": "sync", + "properties": { + "version": get_version(), + "source": asdict(source_info), + "state": state, + "cache": asdict(cache_info), + # explicitly set to 0.0.0.0 to avoid leaking IP addresses + "ip": "0.0.0.0", + "flags": { + "CI": bool(os.environ.get("CI")), + }, + }, + "timestamp": current_time, + } + if number_of_records is not None: + payload["properties"]["number_of_records"] = number_of_records + + # Suppress exceptions if host is unreachable or network is unavailable + with suppress(Exception): + # Do not handle the response, we don't want to block the execution + _ = requests.post("https://api.segment.io/v1/track", auth=(TRACKING_KEY, ""), json=payload) diff --git a/airbyte-lib/airbyte_lib/types.py b/airbyte-lib/airbyte_lib/types.py new file mode 100644 index 000000000000..ca34a5801e0b --- /dev/null +++ b/airbyte-lib/airbyte_lib/types.py @@ -0,0 +1,111 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""Type conversion methods for SQL Caches.""" +from __future__ import annotations + +from typing import cast + +import sqlalchemy + + +# Compare to documentation here: https://docs.airbyte.com/understanding-airbyte/supported-data-types +CONVERSION_MAP = { + "string": sqlalchemy.types.VARCHAR, + "integer": sqlalchemy.types.BIGINT, + "number": sqlalchemy.types.DECIMAL, + "boolean": sqlalchemy.types.BOOLEAN, + "date": sqlalchemy.types.DATE, + "timestamp_with_timezone": sqlalchemy.types.TIMESTAMP, + "timestamp_without_timezone": sqlalchemy.types.TIMESTAMP, + "time_with_timezone": sqlalchemy.types.TIME, + "time_without_timezone": sqlalchemy.types.TIME, + # Technically 'object' and 'array' as JSON Schema types, not airbyte types. + # We include them here for completeness. + "object": sqlalchemy.types.VARCHAR, + "array": sqlalchemy.types.VARCHAR, +} + + +class SQLTypeConversionError(Exception): + """An exception to be raised when a type conversion fails.""" + + +def _get_airbyte_type( + json_schema_property_def: dict[str, str | dict], +) -> tuple[str, str | None]: + """Get the airbyte type and subtype from a JSON schema property definition. + + Subtype is only used for array types. Otherwise, subtype will return None. + """ + airbyte_type = cast(str, json_schema_property_def.get("airbyte_type", None)) + if airbyte_type: + return airbyte_type, None + + json_schema_type = json_schema_property_def.get("type", None) + json_schema_format = json_schema_property_def.get("format", None) + + if json_schema_type == "string": + if json_schema_format == "date": + return "date", None + + if json_schema_format == "date-time": + return "timestamp_with_timezone", None + + if json_schema_format == "time": + return "time_without_timezone", None + + if json_schema_type in ["string", "number", "boolean", "integer"]: + return cast(str, json_schema_type), None + + if json_schema_type == "object" and "properties" in json_schema_property_def: + return "object", None + + err_msg = f"Could not determine airbyte type from JSON schema type: {json_schema_property_def}" + raise SQLTypeConversionError(err_msg) + + +class SQLTypeConverter: + """A base class to perform type conversions.""" + + def __init__( + self, + conversion_map: dict | None = None, + ) -> None: + self.conversion_map = conversion_map or CONVERSION_MAP + + @staticmethod + def get_failover_type() -> sqlalchemy.types.TypeEngine: + """Get the 'last resort' type to use if no other type is found.""" + return sqlalchemy.types.VARCHAR() + + def to_sql_type( + self, + json_schema_property_def: dict[str, str | dict], + ) -> sqlalchemy.types.TypeEngine: + """Convert a value to a SQL type.""" + try: + airbyte_type, airbyte_subtype = _get_airbyte_type(json_schema_property_def) + return self.conversion_map[airbyte_type]() + except SQLTypeConversionError: + print(f"Could not determine airbyte type from JSON schema: {json_schema_property_def}") + except KeyError: + print(f"Could not find SQL type for airbyte type: {airbyte_type}") + + json_schema_type = json_schema_property_def.get("type", None) + json_schema_format = json_schema_property_def.get("format", None) + + if json_schema_type == "string" and json_schema_format == "date": + return sqlalchemy.types.DATE() + + if json_schema_type == "string" and json_schema_format == "date-time": + return sqlalchemy.types.TIMESTAMP() + + if json_schema_type == "array": + # TODO: Implement array type conversion. + return self.get_failover_type() + + if json_schema_type == "object": + # TODO: Implement object type handling. + return self.get_failover_type() + + return self.get_failover_type() diff --git a/airbyte-lib/airbyte_lib/validate.py b/airbyte-lib/airbyte_lib/validate.py new file mode 100644 index 000000000000..8eac20e1692b --- /dev/null +++ b/airbyte-lib/airbyte_lib/validate.py @@ -0,0 +1,121 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +"""Defines the `airbyte-lib-validate-source` CLI. + +This tool checks if connectors are compatible with airbyte-lib. +""" + +import argparse +import json +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +import yaml + +import airbyte_lib as ab + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Validate a connector") + parser.add_argument( + "--connector-dir", + type=str, + required=True, + help="Path to the connector directory", + ) + parser.add_argument( + "--sample-config", + type=str, + required=True, + help="Path to the sample config.json file", + ) + return parser.parse_args() + + +def _run_subprocess_and_raise_on_failure(args: list[str]) -> None: + result = subprocess.run(args, check=False) + if result.returncode != 0: + raise Exception(f"{args} exited with code {result.returncode}") + + +def tests(connector_name: str, sample_config: str) -> None: + print("Creating source and validating spec and version...") + source = ab.get_connector( + # TODO: FIXME: noqa: SIM115, PTH123 + connector_name, + config=json.load(open(sample_config)), # noqa: SIM115, PTH123 + ) + + print("Running check...") + source.check() + + print("Fetching streams...") + streams = source.get_available_streams() + + # try to peek all streams - if one works, stop, if none works, throw exception + for stream in streams: + try: + print(f"Trying to read from stream {stream}...") + record = next(source.get_records(stream)) + assert record, "No record returned" + break + except Exception as e: + print(f"Could not read from stream {stream}: {e}") + else: + raise Exception(f"Could not read from any stream from {streams}") + + +def run() -> None: + """Handle CLI entrypoint for the `airbyte-lib-validate-source` command. + + It's called like this: + > airbyte-lib-validate-source —connector-dir . -—sample-config secrets/config.json + + It performs a basic smoke test to make sure the connector in question is airbyte-lib compliant: + * Can be installed into a venv + * Can be called via cli entrypoint + * Answers according to the Airbyte protocol when called with spec, check, discover and read. + """ + # parse args + args = _parse_args() + connector_dir = args.connector_dir + sample_config = args.sample_config + validate(connector_dir, sample_config) + + +def validate(connector_dir: str, sample_config: str) -> None: + # read metadata.yaml + metadata_path = Path(connector_dir) / "metadata.yaml" + with Path(metadata_path).open() as stream: + metadata = yaml.safe_load(stream)["data"] + + # TODO: Use remoteRegistries.pypi.packageName once set for connectors + connector_name = metadata["dockerRepository"].replace("airbyte/", "") + + # create a venv and install the connector + venv_name = f".venv-{connector_name}" + venv_path = Path(venv_name) + if not venv_path.exists(): + _run_subprocess_and_raise_on_failure([sys.executable, "-m", "venv", venv_name]) + + pip_path = str(venv_path / "bin" / "pip") + + _run_subprocess_and_raise_on_failure([pip_path, "install", "-e", connector_dir]) + + # write basic registry to temp json file + registry = { + "sources": [ + { + "dockerRepository": f"airbyte/{connector_name}", + "dockerImageTag": "0.0.1", + }, + ], + } + + with tempfile.NamedTemporaryFile(mode="w+t", delete=True) as temp_file: + temp_file.write(json.dumps(registry)) + temp_file.seek(0) + os.environ["AIRBYTE_LOCAL_REGISTRY"] = str(temp_file.name) + tests(connector_name, sample_config) diff --git a/airbyte-lib/airbyte_lib/version.py b/airbyte-lib/airbyte_lib/version.py new file mode 100644 index 000000000000..9ed83a5ef456 --- /dev/null +++ b/airbyte-lib/airbyte_lib/version.py @@ -0,0 +1,10 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import importlib.metadata + + +airbyte_lib_version = importlib.metadata.version("airbyte-lib") + + +def get_version() -> str: + return airbyte_lib_version diff --git a/airbyte-lib/docs.py b/airbyte-lib/docs.py new file mode 100644 index 000000000000..bfd30c05e554 --- /dev/null +++ b/airbyte-lib/docs.py @@ -0,0 +1,30 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import os +import pathlib +import shutil + +import pdoc + + +def run() -> None: + """Generate docs for all public modules in airbyte_lib and save them to docs/generated. + + Public modules are: + * The main airbyte_lib module + * All directory modules in airbyte_lib that don't start with an underscore. + """ + public_modules = ["airbyte_lib"] + + # recursively delete the docs/generated folder if it exists + if pathlib.Path("docs/generated").exists(): + shutil.rmtree("docs/generated") + + # All folders in `airbyte_lib` that don't start with "_" are treated as public modules. + for d in os.listdir("airbyte_lib"): + dir_path = pathlib.Path(f"airbyte_lib/{d}") + if dir_path.is_dir() and not d.startswith("_"): + public_modules.append(dir_path) + + pdoc.render.configure(template_directory="docs", show_source=False, search=False) + pdoc.pdoc(*public_modules, output_directory=pathlib.Path("docs/generated")) diff --git a/airbyte-lib/docs/frame.html.jinja2 b/airbyte-lib/docs/frame.html.jinja2 new file mode 100644 index 000000000000..379ae376725f --- /dev/null +++ b/airbyte-lib/docs/frame.html.jinja2 @@ -0,0 +1,14 @@ + +
      + {% block module_contents %}{% endblock %} +
      + +{% filter minify_css %} + {% block style %} + {# The same CSS files as in pdoc's default template, except for layout.css. + You may leave out Bootstrap Reboot, which corrects inconsistences across browsers + but may conflict with you website's stylesheet. #} + + + {% endblock %} +{% endfilter %} diff --git a/airbyte-lib/docs/generated/airbyte_lib.html b/airbyte-lib/docs/generated/airbyte_lib.html new file mode 100644 index 000000000000..240492002ff8 --- /dev/null +++ b/airbyte-lib/docs/generated/airbyte_lib.html @@ -0,0 +1,416 @@ + +
      +
      +
      + + def + get_connector( name: str, version: str | None = None, pip_url: str | None = None, config: dict[str, typing.Any] | None = None, *, use_local_install: bool = False, install_if_missing: bool = True) -> Source: + + +
      + + +

      Get a connector by name and version.

      + +

      Args: + name: connector name + version: connector version - if not provided, the currently installed version will be used. + If no version is installed, the latest available version will be used. The version can + also be set to "latest" to force the use of the latest available version. + pip_url: connector pip URL - if not provided, the pip url will be inferred from the + connector name. + config: connector config - if not provided, you need to set it later via the set_config + method. + use_local_install: whether to use a virtual environment to run the connector. If True, the + connector is expected to be available on the path (e.g. installed via pip). If False, + the connector will be installed automatically in a virtual environment. + install_if_missing: whether to install the connector if it is not available locally. This + parameter is ignored if use_local_install is True.

      +
      + + +
      +
      +
      + + def + get_default_cache() -> airbyte_lib.caches.duckdb.DuckDBCache: + + +
      + + +

      Get a local cache for storing data, using the default database path.

      + +

      Cache files are stored in the .cache directory, relative to the current +working directory.

      +
      + + +
      +
      +
      + + def + new_local_cache( cache_name: str | None = None, cache_dir: str | pathlib.Path | None = None, *, cleanup: bool = True) -> airbyte_lib.caches.duckdb.DuckDBCache: + + +
      + + +

      Get a local cache for storing data, using a name string to seed the path.

      + +

      Args: + cache_name: Name to use for the cache. Defaults to None. + cache_dir: Root directory to store the cache in. Defaults to None. + cleanup: Whether to clean up temporary files. Defaults to True.

      + +

      Cache files are stored in the .cache directory, relative to the current +working directory.

      +
      + + +
      +
      +
      + + class + CachedDataset(abc.ABC, collections.abc.Iterator[collections.abc.Mapping[str, typing.Any]]): + + +
      + + +

      Base implementation for all datasets.

      +
      + + +
      +
      + + CachedDataset(cache: airbyte_lib.caches.base.SQLCacheBase, stream: str) + + +
      + + + + +
      +
      +
      + + def + to_pandas(self) -> pandas.core.frame.DataFrame: + + +
      + + +

      Return a pandas DataFrame representation of the dataset.

      +
      + + +
      +
      +
      + + def + to_sql_table(self) -> sqlalchemy.sql.schema.Table: + + +
      + + + + +
      +
      +
      +
      + + class + ReadResult: + + +
      + + + + +
      +
      + + ReadResult(processed_records: int, cache: airbyte_lib.caches.base.SQLCacheBase) + + +
      + + + + +
      +
      +
      + processed_records + + +
      + + + + +
      +
      +
      + + def + get_sql_engine(self) -> sqlalchemy.engine.base.Engine: + + +
      + + + + +
      +
      +
      + cache: airbyte_lib.caches.base.SQLCacheBase + + +
      + + + + +
      +
      +
      +
      + + class + Source: + + +
      + + +

      A class representing a source that can be called.

      +
      + + +
      +
      + + Source( executor: airbyte_lib._executor.Executor, name: str, config: dict[str, typing.Any] | None = None, streams: list[str] | None = None) + + +
      + + + + +
      +
      +
      + executor + + +
      + + + + +
      +
      +
      + name + + +
      + + + + +
      +
      +
      + streams: list[str] | None + + +
      + + + + +
      +
      +
      + + def + set_streams(self, streams: list[str]) -> None: + + +
      + + + + +
      +
      +
      + + def + set_config(self, config: dict[str, typing.Any]) -> None: + + +
      + + + + +
      +
      +
      + + def + get_available_streams(self) -> list[str]: + + +
      + + +

      Get the available streams from the spec.

      +
      + + +
      +
      +
      + raw_catalog: airbyte_protocol.models.airbyte_protocol.AirbyteCatalog + + +
      + + +

      Get the raw catalog for the given streams.

      +
      + + +
      +
      +
      + configured_catalog: airbyte_protocol.models.airbyte_protocol.ConfiguredAirbyteCatalog + + +
      + + +

      Get the configured catalog for the given streams.

      +
      + + +
      +
      +
      + + def + get_records(self, stream: str) -> collections.abc.Iterator[dict[str, typing.Any]]: + + +
      + + +

      Read a stream from the connector.

      + +

      This involves the following steps:

      + +
        +
      • Call discover to get the catalog
      • +
      • Generate a configured catalog that syncs the given stream in full_refresh mode
      • +
      • Write the configured catalog and the config to a temporary file
      • +
      • execute the connector with read --config --catalog
      • +
      • Listen to the messages and return the first AirbyteRecordMessages that come along.
      • +
      • Make sure the subprocess is killed when the function returns.
      • +
      +
      + + +
      +
      +
      + + def + check(self) -> None: + + +
      + + +

      Call check on the connector.

      + +

      This involves the following steps:

      + +
        +
      • Write the config to a temporary file
      • +
      • execute the connector with check --config
      • +
      • Listen to the messages and return the first AirbyteCatalog that comes along.
      • +
      • Make sure the subprocess is killed when the function returns.
      • +
      +
      + + +
      +
      +
      + + def + install(self) -> None: + + +
      + + +

      Install the connector if it is not yet installed.

      +
      + + +
      +
      +
      + + def + uninstall(self) -> None: + + +
      + + +

      Uninstall the connector if it is installed.

      + +

      This only works if the use_local_install flag wasn't used and installation is managed by +airbyte-lib.

      +
      + + +
      +
      +
      + + def + read( self, cache: airbyte_lib.caches.base.SQLCacheBase | None = None) -> ReadResult: + + +
      + + + + +
      +
      +
      + + + + \ No newline at end of file diff --git a/airbyte-lib/docs/generated/airbyte_lib/caches.html b/airbyte-lib/docs/generated/airbyte_lib/caches.html new file mode 100644 index 000000000000..dbb675f6d5a2 --- /dev/null +++ b/airbyte-lib/docs/generated/airbyte_lib/caches.html @@ -0,0 +1,947 @@ + +
      +
      +
      + + class + DuckDBCache(airbyte_lib.caches.duckdb.DuckDBCacheBase): + + +
      + + +

      A DuckDB implementation of the cache.

      + +

      Parquet is used for local file storage before bulk loading. +Unlike the Snowflake implementation, we can't use the COPY command to load data +so we insert as values instead.

      +
      + + +
      +
      + file_writer_class = +<class 'airbyte_lib._file_writers.parquet.ParquetWriter'> + + +
      + + + + +
      +
      +
      Inherited Members
      +
      + +
      airbyte_lib.caches.duckdb.DuckDBCacheBase
      +
      config_class
      +
      supports_merge_insert
      +
      get_telemetry_info
      + +
      +
      airbyte_lib._processors.RecordProcessor
      +
      skip_finalize_step
      +
      source_catalog
      +
      register_source
      +
      process_stdin
      +
      process_input_stream
      +
      process_airbyte_messages
      + +
      +
      +
      +
      +
      +
      + + class + DuckDBCacheConfig(airbyte_lib.caches.base.SQLCacheConfigBase, airbyte_lib._file_writers.parquet.ParquetWriterConfig): + + +
      + + +

      Configuration for the DuckDB cache.

      + +

      Also inherits config from the ParquetWriter, which is responsible for writing files to disk.

      +
      + + +
      +
      + db_path: pathlib.Path | str + + +
      + + +

      Normally db_path is a Path object.

      + +

      There are some cases, such as when connecting to MotherDuck, where it could be a string that +is not also a path, such as "md:" to connect the user's default MotherDuck DB.

      +
      + + +
      +
      +
      + schema_name: str + + +
      + + +

      The name of the schema to write to. Defaults to "main".

      +
      + + +
      +
      +
      +
      @overrides
      + + def + get_sql_alchemy_url(self) -> str: + + +
      + + +

      Return the SQLAlchemy URL to use.

      +
      + + +
      +
      +
      + + def + get_database_name(self) -> str: + + +
      + + +

      Return the name of the database.

      +
      + + +
      +
      +
      Inherited Members
      +
      +
      pydantic.main.BaseModel
      +
      BaseModel
      +
      Config
      +
      dict
      +
      json
      +
      parse_obj
      +
      parse_raw
      +
      parse_file
      +
      from_orm
      +
      construct
      +
      copy
      +
      schema
      +
      schema_json
      +
      validate
      +
      update_forward_refs
      + +
      +
      airbyte_lib.caches.base.SQLCacheConfigBase
      +
      dedupe_mode
      +
      table_prefix
      +
      table_suffix
      + +
      +
      airbyte_lib._file_writers.base.FileWriterConfigBase
      +
      cache_dir
      +
      cleanup
      + +
      +
      +
      +
      +
      +
      + + class + PostgresCache(airbyte_lib.caches.SQLCacheBase): + + +
      + + +

      A Postgres implementation of the cache.

      + +

      Parquet is used for local file storage before bulk loading. +Unlike the Snowflake implementation, we can't use the COPY command to load data +so we insert as values instead.

      + +

      TOOD: Add optimized bulk load path for Postgres. Could use an alternate file writer +or another import method. (Relatively low priority, since for now it works fine as-is.)

      +
      + + +
      +
      + config_class = +<class 'PostgresCacheConfig'> + + +
      + + + + +
      +
      +
      + file_writer_class = +<class 'airbyte_lib._file_writers.parquet.ParquetWriter'> + + +
      + + + + +
      +
      +
      + supports_merge_insert = +True + + +
      + + + + +
      +
      +
      +
      @overrides
      + + def + get_telemetry_info(self) -> airbyte_lib.telemetry.CacheTelemetryInfo: + + +
      + + + + +
      +
      +
      Inherited Members
      +
      + +
      airbyte_lib._processors.RecordProcessor
      +
      skip_finalize_step
      +
      source_catalog
      +
      register_source
      +
      process_stdin
      +
      process_input_stream
      +
      process_airbyte_messages
      + +
      +
      +
      +
      +
      +
      + + class + PostgresCacheConfig(airbyte_lib.caches.base.SQLCacheConfigBase, airbyte_lib._file_writers.parquet.ParquetWriterConfig): + + +
      + + +

      Configuration for the Postgres cache.

      + +

      Also inherits config from the ParquetWriter, which is responsible for writing files to disk.

      +
      + + +
      +
      + host: str + + +
      + + + + +
      +
      +
      + port: int + + +
      + + + + +
      +
      +
      + username: str + + +
      + + + + +
      +
      +
      + password: str + + +
      + + + + +
      +
      +
      + database: str + + +
      + + + + +
      +
      +
      +
      @overrides
      + + def + get_sql_alchemy_url(self) -> str: + + +
      + + +

      Return the SQLAlchemy URL to use.

      +
      + + +
      +
      +
      + + def + get_database_name(self) -> str: + + +
      + + +

      Return the name of the database.

      +
      + + +
      +
      +
      Inherited Members
      +
      +
      pydantic.main.BaseModel
      +
      BaseModel
      +
      Config
      +
      dict
      +
      json
      +
      parse_obj
      +
      parse_raw
      +
      parse_file
      +
      from_orm
      +
      construct
      +
      copy
      +
      schema
      +
      schema_json
      +
      validate
      +
      update_forward_refs
      + +
      +
      airbyte_lib.caches.base.SQLCacheConfigBase
      +
      dedupe_mode
      +
      schema_name
      +
      table_prefix
      +
      table_suffix
      + +
      +
      airbyte_lib._file_writers.base.FileWriterConfigBase
      +
      cache_dir
      +
      cleanup
      + +
      +
      +
      +
      +
      +
      + + class + SQLCacheBase(airbyte_lib._processors.RecordProcessor): + + +
      + + +

      A base class to be used for SQL Caches.

      + +

      Optionally we can use a file cache to store the data in parquet files.

      +
      + + +
      +
      + type_converter_class: type[airbyte_lib.types.SQLTypeConverter] = +<class 'airbyte_lib.types.SQLTypeConverter'> + + +
      + + + + +
      +
      +
      + config_class: type[airbyte_lib.caches.base.SQLCacheConfigBase] + + +
      + + + + +
      +
      +
      + file_writer_class: type[airbyte_lib._file_writers.base.FileWriterBase] + + +
      + + + + +
      +
      +
      + supports_merge_insert = +False + + +
      + + + + +
      +
      +
      + use_singleton_connection = +False + + +
      + + + + +
      +
      +
      + config: airbyte_lib.caches.base.SQLCacheConfigBase + + +
      + + + + +
      +
      +
      + file_writer + + +
      + + + + +
      +
      +
      + type_converter + + +
      + + + + +
      +
      +
      + + def + get_sql_alchemy_url(self) -> str: + + +
      + + +

      Return the SQLAlchemy URL to use.

      +
      + + +
      +
      +
      + database_name: str + + +
      + + +

      Return the name of the database.

      +
      + + +
      +
      +
      +
      @final
      + + def + get_sql_engine(self) -> sqlalchemy.engine.base.Engine: + + +
      + + +

      Return a new SQL engine to use.

      +
      + + +
      +
      +
      +
      @contextmanager
      + + def + get_sql_connection( self) -> collections.abc.Generator[sqlalchemy.engine.base.Connection, None, None]: + + +
      + + +

      A context manager which returns a new SQL connection for running queries.

      + +

      If the connection needs to close, it will be closed automatically.

      +
      + + +
      +
      +
      + + def + get_sql_table_name(self, stream_name: str) -> str: + + +
      + + +

      Return the name of the SQL table for the given stream.

      +
      + + +
      +
      +
      +
      @final
      + + def + get_sql_table(self, stream_name: str) -> sqlalchemy.sql.schema.Table: + + +
      + + +

      Return a temporary table name.

      +
      + + +
      +
      +
      + streams: dict[str, airbyte_lib.datasets._base.DatasetBase] + + +
      + + +

      Return a temporary table name.

      +
      + + +
      +
      +
      + + def + get_records( self, stream_name: str) -> collections.abc.Iterator[collections.abc.Mapping[str, typing.Any]]: + + +
      + + +

      Uses SQLAlchemy to select all rows from the table.

      + +

      TODO: Refactor to return a LazyDataset here.

      +
      + + +
      +
      +
      + + def + get_pandas_dataframe(self, stream_name: str) -> pandas.core.frame.DataFrame: + + +
      + + +

      Return a Pandas data frame with the stream's data.

      +
      + + +
      +
      +
      +
      @abc.abstractmethod
      + + def + get_telemetry_info(self) -> airbyte_lib.telemetry.CacheTelemetryInfo: + + +
      + + + + +
      +
      +
      Inherited Members
      +
      +
      airbyte_lib._processors.RecordProcessor
      +
      skip_finalize_step
      +
      source_catalog
      +
      register_source
      +
      process_stdin
      +
      process_input_stream
      +
      process_airbyte_messages
      + +
      +
      +
      +
      +
      +
      + + class + SnowflakeCacheConfig(airbyte_lib.caches.base.SQLCacheConfigBase, airbyte_lib._file_writers.parquet.ParquetWriterConfig): + + +
      + + +

      Configuration for the Snowflake cache.

      + +

      Also inherits config from the ParquetWriter, which is responsible for writing files to disk.

      +
      + + +
      +
      + account: str + + +
      + + + + +
      +
      +
      + username: str + + +
      + + + + +
      +
      +
      + password: str + + +
      + + + + +
      +
      +
      + warehouse: str + + +
      + + + + +
      +
      +
      + database: str + + +
      + + + + +
      +
      +
      + role: str + + +
      + + + + +
      +
      +
      + dedupe_mode + + +
      + + + + +
      +
      +
      +
      @overrides
      + + def + get_sql_alchemy_url(self) -> str: + + +
      + + +

      Return the SQLAlchemy URL to use.

      +
      + + +
      +
      +
      + + def + get_database_name(self) -> str: + + +
      + + +

      Return the name of the database.

      +
      + + +
      +
      +
      Inherited Members
      +
      +
      pydantic.main.BaseModel
      +
      BaseModel
      +
      Config
      +
      dict
      +
      json
      +
      parse_obj
      +
      parse_raw
      +
      parse_file
      +
      from_orm
      +
      construct
      +
      copy
      +
      schema
      +
      schema_json
      +
      validate
      +
      update_forward_refs
      + +
      +
      airbyte_lib.caches.base.SQLCacheConfigBase
      +
      schema_name
      +
      table_prefix
      +
      table_suffix
      + +
      +
      airbyte_lib._file_writers.base.FileWriterConfigBase
      +
      cache_dir
      +
      cleanup
      + +
      +
      +
      +
      +
      +
      + + class + SnowflakeSQLCache(airbyte_lib.caches.SQLCacheBase): + + +
      + + +

      A Snowflake implementation of the cache.

      + +

      Parquet is used for local file storage before bulk loading.

      +
      + + +
      +
      + config_class = +<class 'SnowflakeCacheConfig'> + + +
      + + + + +
      +
      +
      + file_writer_class = +<class 'airbyte_lib._file_writers.parquet.ParquetWriter'> + + +
      + + + + +
      +
      +
      +
      @overrides
      + + def + get_telemetry_info(self) -> airbyte_lib.telemetry.CacheTelemetryInfo: + + +
      + + + + +
      +
      +
      Inherited Members
      +
      + +
      airbyte_lib._processors.RecordProcessor
      +
      skip_finalize_step
      +
      source_catalog
      +
      register_source
      +
      process_stdin
      +
      process_input_stream
      +
      process_airbyte_messages
      + +
      +
      +
      +
      +
      + + + + \ No newline at end of file diff --git a/airbyte-lib/docs/generated/airbyte_lib/datasets.html b/airbyte-lib/docs/generated/airbyte_lib/datasets.html new file mode 100644 index 000000000000..acd3bb5928b6 --- /dev/null +++ b/airbyte-lib/docs/generated/airbyte_lib/datasets.html @@ -0,0 +1,119 @@ + +
      +
      +
      + + class + CachedDataset(abc.ABC, collections.abc.Iterator[collections.abc.Mapping[str, typing.Any]]): + + +
      + + +

      Base implementation for all datasets.

      +
      + + +
      +
      + + CachedDataset(cache: airbyte_lib.caches.base.SQLCacheBase, stream: str) + + +
      + + + + +
      +
      +
      + + def + to_pandas(self) -> pandas.core.frame.DataFrame: + + +
      + + +

      Return a pandas DataFrame representation of the dataset.

      +
      + + +
      +
      +
      + + def + to_sql_table(self) -> sqlalchemy.sql.schema.Table: + + +
      + + + + +
      +
      +
      +
      + + class + DatasetBase(abc.ABC, collections.abc.Iterator[collections.abc.Mapping[str, typing.Any]]): + + +
      + + +

      Base implementation for all datasets.

      +
      + + +
      +
      + + def + to_pandas(self) -> pandas.core.frame.DataFrame: + + +
      + + +

      Return a pandas DataFrame representation of the dataset.

      +
      + + +
      +
      +
      +
      + + class + DatasetMap(collections.abc.Mapping): + + +
      + + +

      A generic interface for a set of streams or datasets.

      +
      + + +
      +
      Inherited Members
      +
      +
      collections.abc.Mapping
      +
      get
      +
      keys
      +
      items
      +
      values
      + +
      +
      +
      +
      +
      + + + + \ No newline at end of file diff --git a/airbyte-lib/docs/generated/airbyte_lib/factories.html b/airbyte-lib/docs/generated/airbyte_lib/factories.html new file mode 100644 index 000000000000..c0d27ca14eaa --- /dev/null +++ b/airbyte-lib/docs/generated/airbyte_lib/factories.html @@ -0,0 +1,7 @@ + +
      +
      + + + + \ No newline at end of file diff --git a/airbyte-lib/docs/generated/index.html b/airbyte-lib/docs/generated/index.html new file mode 100644 index 000000000000..6dfc876b8f9c --- /dev/null +++ b/airbyte-lib/docs/generated/index.html @@ -0,0 +1,7 @@ + + + + + + + diff --git a/airbyte-lib/examples/run_snowflake_faker.py b/airbyte-lib/examples/run_snowflake_faker.py new file mode 100644 index 000000000000..5a6084ce16dc --- /dev/null +++ b/airbyte-lib/examples/run_snowflake_faker.py @@ -0,0 +1,46 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + + +import json +import os + +from google.cloud import secretmanager + +import airbyte_lib as ab +from airbyte_lib.caches import SnowflakeCacheConfig, SnowflakeSQLCache + + +source = ab.get_connector( + "source-faker", + config={"count": 10000, "seed": 0, "parallelism": 1, "always_updated": False}, + install_if_missing=True, +) + +# load secrets from GSM using the GCP_GSM_CREDENTIALS env variable +secret_client = secretmanager.SecretManagerServiceClient.from_service_account_info( + json.loads(os.environ["GCP_GSM_CREDENTIALS"]) +) +secret = json.loads( + secret_client.access_secret_version( + name="projects/dataline-integration-testing/secrets/AIRBYTE_LIB_SNOWFLAKE_CREDS/versions/latest" + ).payload.data.decode("UTF-8") +) + +cache = SnowflakeSQLCache( + SnowflakeCacheConfig( + account=secret["account"], + username=secret["username"], + password=secret["password"], + database=secret["database"], + warehouse=secret["warehouse"], + role=secret["role"], + ) +) + +source.check() + +source.set_streams(["products"]) +result = source.read(cache) + +for name in ["products"]: + print(f"Stream {name}: {len(list(result[name]))} records") diff --git a/airbyte-lib/examples/run_spacex.py b/airbyte-lib/examples/run_spacex.py new file mode 100644 index 000000000000..46c8dee411d8 --- /dev/null +++ b/airbyte-lib/examples/run_spacex.py @@ -0,0 +1,31 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from itertools import islice + +import airbyte_lib as ab + + +# preparation (from airbyte-lib main folder): +# python -m venv .venv-source-spacex-api +# source .venv-source-spacex-api/bin/activate +# pip install -e ../airbyte-integrations/connectors/source-spacex-api +# In separate terminal: +# poetry run python examples/run_spacex.py + +source = ab.get_connector( + "source-spacex-api", + config={"id": "605b4b6aaa5433645e37d03f"}, + install_if_missing=True, +) +cache = ab.new_local_cache() + +source.check() + +source.set_streams(["launches", "rockets", "capsules"]) + +result = source.read(cache) + +print(islice(source.get_records("capsules"), 10)) + +for name, records in result.cache.streams.items(): + print(f"Stream {name}: {len(list(records))} records") diff --git a/airbyte-lib/examples/run_test_source.py b/airbyte-lib/examples/run_test_source.py new file mode 100644 index 000000000000..de57ca8420ff --- /dev/null +++ b/airbyte-lib/examples/run_test_source.py @@ -0,0 +1,27 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import os + +import airbyte_lib as ab + + +# preparation (from airbyte-lib main folder): +# python -m venv .venv-source-test +# source .venv-source-test/bin/activate +# pip install -e ./tests/integration_tests/fixtures/source-test +# In separate terminal: +# poetry run python examples/run_test_source.py + +os.environ["AIRBYTE_LOCAL_REGISTRY"] = "./tests/integration_tests/fixtures/registry.json" + +source = ab.get_connector("source-test", config={"apiKey": "test"}) +cache = ab.new_local_cache() + +source.check() + +print(source.get_available_streams()) + +result = source.read(cache) + +print(result.processed_records) +print(list(result["stream1"])) diff --git a/airbyte-lib/examples/run_test_source_single_stream.py b/airbyte-lib/examples/run_test_source_single_stream.py new file mode 100644 index 000000000000..b1cc55cd1f5c --- /dev/null +++ b/airbyte-lib/examples/run_test_source_single_stream.py @@ -0,0 +1,19 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import os + +import airbyte_lib as ab + + +# preparation (from airbyte-lib main folder): +# python -m venv .venv-source-test +# source .venv-source-test/bin/activate +# pip install -e ./tests/integration_tests/fixtures/source-test +# In separate terminal: +# poetry run python examples/run_test_source.py + +os.environ["AIRBYTE_LOCAL_REGISTRY"] = "./tests/integration_tests/fixtures/registry.json" + +source = ab.get_connector("source-test", config={"apiKey": "test"}) + +print(list(source.read_stream("stream1"))) diff --git a/airbyte-lib/poetry.lock b/airbyte-lib/poetry.lock new file mode 100644 index 000000000000..06576ed330a2 --- /dev/null +++ b/airbyte-lib/poetry.lock @@ -0,0 +1,2612 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "airbyte-cdk" +version = "0.58.9" +description = "A framework for writing Airbyte Connectors." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte-cdk-0.58.9.tar.gz", hash = "sha256:e749bd4aab0911bd93c710e3ab2fcdde45d7a0bed2c0032d873006d3df701478"}, + {file = "airbyte_cdk-0.58.9-py3-none-any.whl", hash = "sha256:45dfbac2d0ae86dd5872c07c140ce16be8481452b7b8f65b228bc9f892843871"}, +] + +[package.dependencies] +airbyte-protocol-models = "0.5.1" +backoff = "*" +cachetools = "*" +Deprecated = ">=1.2,<2.0" +dpath = ">=2.0.1,<2.1.0" +genson = "1.2.2" +isodate = ">=0.6.1,<0.7.0" +Jinja2 = ">=3.1.2,<3.2.0" +jsonref = ">=0.2,<1.0" +jsonschema = ">=3.2.0,<3.3.0" +pendulum = "<3.0.0" +pydantic = ">=1.10.8,<2.0.0" +pyrate-limiter = ">=3.1.0,<3.2.0" +python-dateutil = "*" +PyYAML = ">=6.0.1" +requests = "*" +requests-cache = "*" +wcmatch = "8.4" + +[package.extras] +dev = ["avro (>=1.11.2,<1.12.0)", "cohere (==4.21)", "fastavro (>=1.8.0,<1.9.0)", "freezegun", "langchain (==0.0.271)", "markdown", "mypy", "openai[embeddings] (==0.27.9)", "pandas (==2.0.3)", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "pytest", "pytest-cov", "pytest-httpserver", "pytest-mock", "requests-mock", "tiktoken (==0.4.0)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (==12.0.1)", "pytesseract (==0.3.10)", "unstructured (==0.10.27)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] +sphinx-docs = ["Sphinx (>=4.2,<5.0)", "sphinx-rtd-theme (>=1.0,<2.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] + +[[package]] +name = "airbyte-protocol-models" +version = "0.5.1" +description = "Declares the Airbyte Protocol." +optional = false +python-versions = ">=3.8" +files = [ + {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, + {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, +] + +[package.dependencies] +pydantic = ">=1.9.2,<2.0.0" + +[[package]] +name = "asn1crypto" +version = "1.5.1" +description = "Fast ASN.1 parser and serializer with definitions for private keys, public keys, certificates, CRL, OCSP, CMS, PKCS#3, PKCS#7, PKCS#8, PKCS#12, PKCS#5, X.509 and TSP" +optional = false +python-versions = "*" +files = [ + {file = "asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67"}, + {file = "asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c"}, +] + +[[package]] +name = "attrs" +version = "23.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +files = [ + {file = "attrs-23.2.0-py3-none-any.whl", hash = "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"}, + {file = "attrs-23.2.0.tar.gz", hash = "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30"}, +] + +[package.extras] +cov = ["attrs[tests]", "coverage[toml] (>=5.3)"] +dev = ["attrs[tests]", "pre-commit"] +docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope-interface"] +tests = ["attrs[tests-no-zope]", "zope-interface"] +tests-mypy = ["mypy (>=1.6)", "pytest-mypy-plugins"] +tests-no-zope = ["attrs[tests-mypy]", "cloudpickle", "hypothesis", "pympler", "pytest (>=4.3.0)", "pytest-xdist[psutil]"] + +[[package]] +name = "backoff" +version = "2.2.1" +description = "Function decoration for backoff and retry" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8"}, + {file = "backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba"}, +] + +[[package]] +name = "bracex" +version = "2.4" +description = "Bash style brace expander." +optional = false +python-versions = ">=3.8" +files = [ + {file = "bracex-2.4-py3-none-any.whl", hash = "sha256:efdc71eff95eaff5e0f8cfebe7d01adf2c8637c8c92edaf63ef348c241a82418"}, + {file = "bracex-2.4.tar.gz", hash = "sha256:a27eaf1df42cf561fed58b7a8f3fdf129d1ea16a81e1fadd1d17989bc6384beb"}, +] + +[[package]] +name = "cachetools" +version = "5.3.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cachetools-5.3.2-py3-none-any.whl", hash = "sha256:861f35a13a451f94e301ce2bec7cac63e881232ccce7ed67fab9b5df4d3beaa1"}, + {file = "cachetools-5.3.2.tar.gz", hash = "sha256:086ee420196f7b2ab9ca2db2520aca326318b68fe5ba8bc4d49cca91add450f2"}, +] + +[[package]] +name = "cattrs" +version = "23.2.3" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, + {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +orjson = ["orjson (>=3.9.2)"] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2023.11.17" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +files = [ + {file = "certifi-2023.11.17-py3-none-any.whl", hash = "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"}, + {file = "certifi-2023.11.17.tar.gz", hash = "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1"}, +] + +[[package]] +name = "cffi" +version = "1.16.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "charset-normalizer" +version = "3.3.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "cryptography" +version = "41.0.7" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf"}, + {file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a"}, + {file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1"}, + {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157"}, + {file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406"}, + {file = "cryptography-41.0.7-cp37-abi3-win32.whl", hash = "sha256:f983596065a18a2183e7f79ab3fd4c475205b839e02cbc0efbbf9666c4b3083d"}, + {file = "cryptography-41.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:90452ba79b8788fa380dfb587cca692976ef4e757b194b093d845e8d99f612f2"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:079b85658ea2f59c4f43b70f8119a52414cdb7be34da5d019a77bf96d473b960"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7"}, + {file = "cryptography-41.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c"}, + {file = "cryptography-41.0.7-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248"}, + {file = "cryptography-41.0.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d6c391c021ab1f7a82da5d8d0b3cee2f4b2c455ec86c8aebbc84837a631ff309"}, + {file = "cryptography-41.0.7.tar.gz", hash = "sha256:13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc"}, +] + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +nox = ["nox"] +pep8test = ["black", "check-sdist", "mypy", "ruff"] +sdist = ["build"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test-randomorder = ["pytest-randomly"] + +[[package]] +name = "deprecated" +version = "1.2.14" +description = "Python @deprecated decorator to deprecate old python classes, functions or methods." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, +] + +[package.dependencies] +wrapt = ">=1.10,<2" + +[package.extras] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] + +[[package]] +name = "docker" +version = "7.0.0" +description = "A Python library for the Docker Engine API." +optional = false +python-versions = ">=3.8" +files = [ + {file = "docker-7.0.0-py3-none-any.whl", hash = "sha256:12ba681f2777a0ad28ffbcc846a69c31b4dfd9752b47eb425a274ee269c5e14b"}, + {file = "docker-7.0.0.tar.gz", hash = "sha256:323736fb92cd9418fc5e7133bc953e11a9da04f4483f828b527db553f1e7e5a3"}, +] + +[package.dependencies] +packaging = ">=14.0" +pywin32 = {version = ">=304", markers = "sys_platform == \"win32\""} +requests = ">=2.26.0" +urllib3 = ">=1.26.0" + +[package.extras] +ssh = ["paramiko (>=2.4.3)"] +websockets = ["websocket-client (>=1.3.0)"] + +[[package]] +name = "dpath" +version = "2.0.8" +description = "Filesystem-like pathing and searching for dictionaries" +optional = false +python-versions = ">=3.7" +files = [ + {file = "dpath-2.0.8-py3-none-any.whl", hash = "sha256:f92f595214dd93a00558d75d4b858beee519f4cffca87f02616ad6cd013f3436"}, + {file = "dpath-2.0.8.tar.gz", hash = "sha256:a3440157ebe80d0a3ad794f1b61c571bef125214800ffdb9afc9424e8250fe9b"}, +] + +[[package]] +name = "duckdb" +version = "0.9.2" +description = "DuckDB embedded database" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "duckdb-0.9.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:aadcea5160c586704c03a8a796c06a8afffbefefb1986601104a60cb0bfdb5ab"}, + {file = "duckdb-0.9.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:08215f17147ed83cbec972175d9882387366de2ed36c21cbe4add04b39a5bcb4"}, + {file = "duckdb-0.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ee6c2a8aba6850abef5e1be9dbc04b8e72a5b2c2b67f77892317a21fae868fe7"}, + {file = "duckdb-0.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ff49f3da9399900fd58b5acd0bb8bfad22c5147584ad2427a78d937e11ec9d0"}, + {file = "duckdb-0.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd5ac5baf8597efd2bfa75f984654afcabcd698342d59b0e265a0bc6f267b3f0"}, + {file = "duckdb-0.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:81c6df905589a1023a27e9712edb5b724566587ef280a0c66a7ec07c8083623b"}, + {file = "duckdb-0.9.2-cp310-cp310-win32.whl", hash = "sha256:a298cd1d821c81d0dec8a60878c4b38c1adea04a9675fb6306c8f9083bbf314d"}, + {file = "duckdb-0.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:492a69cd60b6cb4f671b51893884cdc5efc4c3b2eb76057a007d2a2295427173"}, + {file = "duckdb-0.9.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:061a9ea809811d6e3025c5de31bc40e0302cfb08c08feefa574a6491e882e7e8"}, + {file = "duckdb-0.9.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a43f93be768af39f604b7b9b48891f9177c9282a408051209101ff80f7450d8f"}, + {file = "duckdb-0.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ac29c8c8f56fff5a681f7bf61711ccb9325c5329e64f23cb7ff31781d7b50773"}, + {file = "duckdb-0.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b14d98d26bab139114f62ade81350a5342f60a168d94b27ed2c706838f949eda"}, + {file = "duckdb-0.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:796a995299878913e765b28cc2b14c8e44fae2f54ab41a9ee668c18449f5f833"}, + {file = "duckdb-0.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6cb64ccfb72c11ec9c41b3cb6181b6fd33deccceda530e94e1c362af5f810ba1"}, + {file = "duckdb-0.9.2-cp311-cp311-win32.whl", hash = "sha256:930740cb7b2cd9e79946e1d3a8f66e15dc5849d4eaeff75c8788d0983b9256a5"}, + {file = "duckdb-0.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:c28f13c45006fd525001b2011cdf91fa216530e9751779651e66edc0e446be50"}, + {file = "duckdb-0.9.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:fbce7bbcb4ba7d99fcec84cec08db40bc0dd9342c6c11930ce708817741faeeb"}, + {file = "duckdb-0.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:15a82109a9e69b1891f0999749f9e3265f550032470f51432f944a37cfdc908b"}, + {file = "duckdb-0.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9490fb9a35eb74af40db5569d90df8a04a6f09ed9a8c9caa024998c40e2506aa"}, + {file = "duckdb-0.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:696d5c6dee86c1a491ea15b74aafe34ad2b62dcd46ad7e03b1d00111ca1a8c68"}, + {file = "duckdb-0.9.2-cp37-cp37m-win32.whl", hash = "sha256:4f0935300bdf8b7631ddfc838f36a858c1323696d8c8a2cecbd416bddf6b0631"}, + {file = "duckdb-0.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:0aab900f7510e4d2613263865570203ddfa2631858c7eb8cbed091af6ceb597f"}, + {file = "duckdb-0.9.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:7d8130ed6a0c9421b135d0743705ea95b9a745852977717504e45722c112bf7a"}, + {file = "duckdb-0.9.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:974e5de0294f88a1a837378f1f83330395801e9246f4e88ed3bfc8ada65dcbee"}, + {file = "duckdb-0.9.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4fbc297b602ef17e579bb3190c94d19c5002422b55814421a0fc11299c0c1100"}, + {file = "duckdb-0.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1dd58a0d84a424924a35b3772419f8cd78a01c626be3147e4934d7a035a8ad68"}, + {file = "duckdb-0.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11a1194a582c80dfb57565daa06141727e415ff5d17e022dc5f31888a5423d33"}, + {file = "duckdb-0.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:be45d08541002a9338e568dca67ab4f20c0277f8f58a73dfc1435c5b4297c996"}, + {file = "duckdb-0.9.2-cp38-cp38-win32.whl", hash = "sha256:dd6f88aeb7fc0bfecaca633629ff5c986ac966fe3b7dcec0b2c48632fd550ba2"}, + {file = "duckdb-0.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:28100c4a6a04e69aa0f4a6670a6d3d67a65f0337246a0c1a429f3f28f3c40b9a"}, + {file = "duckdb-0.9.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7ae5bf0b6ad4278e46e933e51473b86b4b932dbc54ff097610e5b482dd125552"}, + {file = "duckdb-0.9.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e5d0bb845a80aa48ed1fd1d2d285dd352e96dc97f8efced2a7429437ccd1fe1f"}, + {file = "duckdb-0.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ce262d74a52500d10888110dfd6715989926ec936918c232dcbaddb78fc55b4"}, + {file = "duckdb-0.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6935240da090a7f7d2666f6d0a5e45ff85715244171ca4e6576060a7f4a1200e"}, + {file = "duckdb-0.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5cfb93e73911696a98b9479299d19cfbc21dd05bb7ab11a923a903f86b4d06e"}, + {file = "duckdb-0.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:64e3bc01751f31e7572d2716c3e8da8fe785f1cdc5be329100818d223002213f"}, + {file = "duckdb-0.9.2-cp39-cp39-win32.whl", hash = "sha256:6e5b80f46487636368e31b61461940e3999986359a78660a50dfdd17dd72017c"}, + {file = "duckdb-0.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:e6142a220180dbeea4f341708bd5f9501c5c962ce7ef47c1cadf5e8810b4cb13"}, + {file = "duckdb-0.9.2.tar.gz", hash = "sha256:3843afeab7c3fc4a4c0b53686a4cc1d9cdbdadcbb468d60fef910355ecafd447"}, +] + +[[package]] +name = "duckdb-engine" +version = "0.10.0" +description = "SQLAlchemy driver for duckdb" +optional = false +python-versions = ">=3.7" +files = [ + {file = "duckdb_engine-0.10.0-py3-none-any.whl", hash = "sha256:c408d002e83630b6bbb05fc3b26a43406085b1c22dd43e8cab00bf0b9c011ea8"}, + {file = "duckdb_engine-0.10.0.tar.gz", hash = "sha256:5e3dad3b3513f055a4f5ec5430842249cfe03015743a7597ed1dcc0447dca565"}, +] + +[package.dependencies] +duckdb = ">=0.4.0" +sqlalchemy = ">=1.3.22" + +[[package]] +name = "exceptiongroup" +version = "1.2.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, + {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "faker" +version = "21.0.1" +description = "Faker is a Python package that generates fake data for you." +optional = false +python-versions = ">=3.8" +files = [ + {file = "Faker-21.0.1-py3-none-any.whl", hash = "sha256:0afc67ec898a2d71842a3456e9302620ebc35fab6ad4f3829693fdf151fa4a3a"}, + {file = "Faker-21.0.1.tar.gz", hash = "sha256:bb404bba449b87e6b54a8c50b4602765e9c1a42eaf48abfceb025e42fed01608"}, +] + +[package.dependencies] +python-dateutil = ">=2.4" + +[[package]] +name = "filelock" +version = "3.13.1" +description = "A platform independent file lock." +optional = false +python-versions = ">=3.8" +files = [ + {file = "filelock-3.13.1-py3-none-any.whl", hash = "sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c"}, + {file = "filelock-3.13.1.tar.gz", hash = "sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e"}, +] + +[package.extras] +docs = ["furo (>=2023.9.10)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.24)"] +testing = ["covdefaults (>=2.3)", "coverage (>=7.3.2)", "diff-cover (>=8)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)", "pytest-timeout (>=2.2)"] +typing = ["typing-extensions (>=4.8)"] + +[[package]] +name = "genson" +version = "1.2.2" +description = "GenSON is a powerful, user-friendly JSON Schema generator." +optional = false +python-versions = "*" +files = [ + {file = "genson-1.2.2.tar.gz", hash = "sha256:8caf69aa10af7aee0e1a1351d1d06801f4696e005f06cedef438635384346a16"}, +] + +[[package]] +name = "google-api-core" +version = "2.15.0" +description = "Google API client core library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-api-core-2.15.0.tar.gz", hash = "sha256:abc978a72658f14a2df1e5e12532effe40f94f868f6e23d95133bd6abcca35ca"}, + {file = "google_api_core-2.15.0-py3-none-any.whl", hash = "sha256:2aa56d2be495551e66bbff7f729b790546f87d5c90e74781aa77233bcb395a8a"}, +] + +[package.dependencies] +google-auth = ">=2.14.1,<3.0.dev0" +googleapis-common-protos = ">=1.56.2,<2.0.dev0" +grpcio = [ + {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, + {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, +] +grpcio-status = [ + {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, + {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, +] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" +requests = ">=2.18.0,<3.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0)"] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] + +[[package]] +name = "google-auth" +version = "2.26.2" +description = "Google Authentication Library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-auth-2.26.2.tar.gz", hash = "sha256:97327dbbf58cccb58fc5a1712bba403ae76668e64814eb30f7316f7e27126b81"}, + {file = "google_auth-2.26.2-py2.py3-none-any.whl", hash = "sha256:3f445c8ce9b61ed6459aad86d8ccdba4a9afed841b2d1451a11ef4db08957424"}, +] + +[package.dependencies] +cachetools = ">=2.0.0,<6.0" +pyasn1-modules = ">=0.2.1" +rsa = ">=3.1.4,<5" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] +enterprise-cert = ["cryptography (==36.0.2)", "pyopenssl (==22.0.0)"] +pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] +reauth = ["pyu2f (>=0.1.5)"] +requests = ["requests (>=2.20.0,<3.0.0.dev0)"] + +[[package]] +name = "google-cloud-secret-manager" +version = "2.17.0" +description = "Google Cloud Secret Manager API client library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "google-cloud-secret-manager-2.17.0.tar.gz", hash = "sha256:8254e2960c06a8dc91aeac3c894afba0893a674582f0e0ecfc22f894e6173c2f"}, + {file = "google_cloud_secret_manager-2.17.0-py2.py3-none-any.whl", hash = "sha256:2bf58be569710773f0757d7a788e2860767aa1c148a5255e53b228da3fbcf446"}, +] + +[package.dependencies] +google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} +grpc-google-iam-v1 = ">=0.12.4,<1.0.0dev" +proto-plus = ">=1.22.3,<2.0.0dev" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" + +[[package]] +name = "googleapis-common-protos" +version = "1.62.0" +description = "Common protobufs used in Google APIs" +optional = false +python-versions = ">=3.7" +files = [ + {file = "googleapis-common-protos-1.62.0.tar.gz", hash = "sha256:83f0ece9f94e5672cced82f592d2a5edf527a96ed1794f0bab36d5735c996277"}, + {file = "googleapis_common_protos-1.62.0-py2.py3-none-any.whl", hash = "sha256:4750113612205514f9f6aa4cb00d523a94f3e8c06c5ad2fee466387dc4875f07"}, +] + +[package.dependencies] +grpcio = {version = ">=1.44.0,<2.0.0.dev0", optional = true, markers = "extra == \"grpc\""} +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" + +[package.extras] +grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] + +[[package]] +name = "greenlet" +version = "3.0.3" +description = "Lightweight in-process concurrent programming" +optional = false +python-versions = ">=3.7" +files = [ + {file = "greenlet-3.0.3-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:9da2bd29ed9e4f15955dd1595ad7bc9320308a3b766ef7f837e23ad4b4aac31a"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d353cadd6083fdb056bb46ed07e4340b0869c305c8ca54ef9da3421acbdf6881"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dca1e2f3ca00b84a396bc1bce13dd21f680f035314d2379c4160c98153b2059b"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3ed7fb269f15dc662787f4119ec300ad0702fa1b19d2135a37c2c4de6fadfd4a"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd4f49ae60e10adbc94b45c0b5e6a179acc1736cf7a90160b404076ee283cf83"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73a411ef564e0e097dbe7e866bb2dda0f027e072b04da387282b02c308807405"}, + {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7f362975f2d179f9e26928c5b517524e89dd48530a0202570d55ad6ca5d8a56f"}, + {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:649dde7de1a5eceb258f9cb00bdf50e978c9db1b996964cd80703614c86495eb"}, + {file = "greenlet-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:68834da854554926fbedd38c76e60c4a2e3198c6fbed520b106a8986445caaf9"}, + {file = "greenlet-3.0.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:b1b5667cced97081bf57b8fa1d6bfca67814b0afd38208d52538316e9422fc61"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52f59dd9c96ad2fc0d5724107444f76eb20aaccb675bf825df6435acb7703559"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:afaff6cf5200befd5cec055b07d1c0a5a06c040fe5ad148abcd11ba6ab9b114e"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe754d231288e1e64323cfad462fcee8f0288654c10bdf4f603a39ed923bef33"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2797aa5aedac23af156bbb5a6aa2cd3427ada2972c828244eb7d1b9255846379"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7f009caad047246ed379e1c4dbcb8b020f0a390667ea74d2387be2998f58a22"}, + {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c5e1536de2aad7bf62e27baf79225d0d64360d4168cf2e6becb91baf1ed074f3"}, + {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:894393ce10ceac937e56ec00bb71c4c2f8209ad516e96033e4b3b1de270e200d"}, + {file = "greenlet-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:1ea188d4f49089fc6fb283845ab18a2518d279c7cd9da1065d7a84e991748728"}, + {file = "greenlet-3.0.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:70fb482fdf2c707765ab5f0b6655e9cfcf3780d8d87355a063547b41177599be"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4d1ac74f5c0c0524e4a24335350edad7e5f03b9532da7ea4d3c54d527784f2e"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:149e94a2dd82d19838fe4b2259f1b6b9957d5ba1b25640d2380bea9c5df37676"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:15d79dd26056573940fcb8c7413d84118086f2ec1a8acdfa854631084393efcc"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881b7db1ebff4ba09aaaeae6aa491daeb226c8150fc20e836ad00041bcb11230"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fcd2469d6a2cf298f198f0487e0a5b1a47a42ca0fa4dfd1b6862c999f018ebbf"}, + {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1f672519db1796ca0d8753f9e78ec02355e862d0998193038c7073045899f305"}, + {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2516a9957eed41dd8f1ec0c604f1cdc86758b587d964668b5b196a9db5bfcde6"}, + {file = "greenlet-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:bba5387a6975598857d86de9eac14210a49d554a77eb8261cc68b7d082f78ce2"}, + {file = "greenlet-3.0.3-cp37-cp37m-macosx_11_0_universal2.whl", hash = "sha256:5b51e85cb5ceda94e79d019ed36b35386e8c37d22f07d6a751cb659b180d5274"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:daf3cb43b7cf2ba96d614252ce1684c1bccee6b2183a01328c98d36fcd7d5cb0"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99bf650dc5d69546e076f413a87481ee1d2d09aaaaaca058c9251b6d8c14783f"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dd6e660effd852586b6a8478a1d244b8dc90ab5b1321751d2ea15deb49ed414"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3391d1e16e2a5a1507d83e4a8b100f4ee626e8eca43cf2cadb543de69827c4c"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1f145462f1fa6e4a4ae3c0f782e580ce44d57c8f2c7aae1b6fa88c0b2efdb41"}, + {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1a7191e42732df52cb5f39d3527217e7ab73cae2cb3694d241e18f53d84ea9a7"}, + {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0448abc479fab28b00cb472d278828b3ccca164531daab4e970a0458786055d6"}, + {file = "greenlet-3.0.3-cp37-cp37m-win32.whl", hash = "sha256:b542be2440edc2d48547b5923c408cbe0fc94afb9f18741faa6ae970dbcb9b6d"}, + {file = "greenlet-3.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:01bc7ea167cf943b4c802068e178bbf70ae2e8c080467070d01bfa02f337ee67"}, + {file = "greenlet-3.0.3-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:1996cb9306c8595335bb157d133daf5cf9f693ef413e7673cb07e3e5871379ca"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ddc0f794e6ad661e321caa8d2f0a55ce01213c74722587256fb6566049a8b04"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9db1c18f0eaad2f804728c67d6c610778456e3e1cc4ab4bbd5eeb8e6053c6fc"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7170375bcc99f1a2fbd9c306f5be8764eaf3ac6b5cb968862cad4c7057756506"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b66c9c1e7ccabad3a7d037b2bcb740122a7b17a53734b7d72a344ce39882a1b"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:098d86f528c855ead3479afe84b49242e174ed262456c342d70fc7f972bc13c4"}, + {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:81bb9c6d52e8321f09c3d165b2a78c680506d9af285bfccbad9fb7ad5a5da3e5"}, + {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fd096eb7ffef17c456cfa587523c5f92321ae02427ff955bebe9e3c63bc9f0da"}, + {file = "greenlet-3.0.3-cp38-cp38-win32.whl", hash = "sha256:d46677c85c5ba00a9cb6f7a00b2bfa6f812192d2c9f7d9c4f6a55b60216712f3"}, + {file = "greenlet-3.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:419b386f84949bf0e7c73e6032e3457b82a787c1ab4a0e43732898a761cc9dbf"}, + {file = "greenlet-3.0.3-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:da70d4d51c8b306bb7a031d5cff6cc25ad253affe89b70352af5f1cb68e74b53"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:086152f8fbc5955df88382e8a75984e2bb1c892ad2e3c80a2508954e52295257"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d73a9fe764d77f87f8ec26a0c85144d6a951a6c438dfe50487df5595c6373eac"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7dcbe92cc99f08c8dd11f930de4d99ef756c3591a5377d1d9cd7dd5e896da71"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1551a8195c0d4a68fac7a4325efac0d541b48def35feb49d803674ac32582f61"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64d7675ad83578e3fc149b617a444fab8efdafc9385471f868eb5ff83e446b8b"}, + {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b37eef18ea55f2ffd8f00ff8fe7c8d3818abd3e25fb73fae2ca3b672e333a7a6"}, + {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:77457465d89b8263bca14759d7c1684df840b6811b2499838cc5b040a8b5b113"}, + {file = "greenlet-3.0.3-cp39-cp39-win32.whl", hash = "sha256:57e8974f23e47dac22b83436bdcf23080ade568ce77df33159e019d161ce1d1e"}, + {file = "greenlet-3.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:c5ee858cfe08f34712f548c3c363e807e7186f03ad7a5039ebadb29e8c6be067"}, + {file = "greenlet-3.0.3.tar.gz", hash = "sha256:43374442353259554ce33599da8b692d5aa96f8976d567d4badf263371fbe491"}, +] + +[package.extras] +docs = ["Sphinx", "furo"] +test = ["objgraph", "psutil"] + +[[package]] +name = "grpc-google-iam-v1" +version = "0.13.0" +description = "IAM API client library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "grpc-google-iam-v1-0.13.0.tar.gz", hash = "sha256:fad318608b9e093258fbf12529180f400d1c44453698a33509cc6ecf005b294e"}, + {file = "grpc_google_iam_v1-0.13.0-py2.py3-none-any.whl", hash = "sha256:53902e2af7de8df8c1bd91373d9be55b0743ec267a7428ea638db3775becae89"}, +] + +[package.dependencies] +googleapis-common-protos = {version = ">=1.56.0,<2.0.0dev", extras = ["grpc"]} +grpcio = ">=1.44.0,<2.0.0dev" +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" + +[[package]] +name = "grpcio" +version = "1.60.0" +description = "HTTP/2-based RPC framework" +optional = false +python-versions = ">=3.7" +files = [ + {file = "grpcio-1.60.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:d020cfa595d1f8f5c6b343530cd3ca16ae5aefdd1e832b777f9f0eb105f5b139"}, + {file = "grpcio-1.60.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:b98f43fcdb16172dec5f4b49f2fece4b16a99fd284d81c6bbac1b3b69fcbe0ff"}, + {file = "grpcio-1.60.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:20e7a4f7ded59097c84059d28230907cd97130fa74f4a8bfd1d8e5ba18c81491"}, + {file = "grpcio-1.60.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:452ca5b4afed30e7274445dd9b441a35ece656ec1600b77fff8c216fdf07df43"}, + {file = "grpcio-1.60.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:43e636dc2ce9ece583b3e2ca41df5c983f4302eabc6d5f9cd04f0562ee8ec1ae"}, + {file = "grpcio-1.60.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6e306b97966369b889985a562ede9d99180def39ad42c8014628dd3cc343f508"}, + {file = "grpcio-1.60.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f897c3b127532e6befdcf961c415c97f320d45614daf84deba0a54e64ea2457b"}, + {file = "grpcio-1.60.0-cp310-cp310-win32.whl", hash = "sha256:b87efe4a380887425bb15f220079aa8336276398dc33fce38c64d278164f963d"}, + {file = "grpcio-1.60.0-cp310-cp310-win_amd64.whl", hash = "sha256:a9c7b71211f066908e518a2ef7a5e211670761651039f0d6a80d8d40054047df"}, + {file = "grpcio-1.60.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:fb464479934778d7cc5baf463d959d361954d6533ad34c3a4f1d267e86ee25fd"}, + {file = "grpcio-1.60.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:4b44d7e39964e808b071714666a812049765b26b3ea48c4434a3b317bac82f14"}, + {file = "grpcio-1.60.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:90bdd76b3f04bdb21de5398b8a7c629676c81dfac290f5f19883857e9371d28c"}, + {file = "grpcio-1.60.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:91229d7203f1ef0ab420c9b53fe2ca5c1fbeb34f69b3bc1b5089466237a4a134"}, + {file = "grpcio-1.60.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b36a2c6d4920ba88fa98075fdd58ff94ebeb8acc1215ae07d01a418af4c0253"}, + {file = "grpcio-1.60.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:297eef542156d6b15174a1231c2493ea9ea54af8d016b8ca7d5d9cc65cfcc444"}, + {file = "grpcio-1.60.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:87c9224acba0ad8bacddf427a1c2772e17ce50b3042a789547af27099c5f751d"}, + {file = "grpcio-1.60.0-cp311-cp311-win32.whl", hash = "sha256:95ae3e8e2c1b9bf671817f86f155c5da7d49a2289c5cf27a319458c3e025c320"}, + {file = "grpcio-1.60.0-cp311-cp311-win_amd64.whl", hash = "sha256:467a7d31554892eed2aa6c2d47ded1079fc40ea0b9601d9f79204afa8902274b"}, + {file = "grpcio-1.60.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:a7152fa6e597c20cb97923407cf0934e14224af42c2b8d915f48bc3ad2d9ac18"}, + {file = "grpcio-1.60.0-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:7db16dd4ea1b05ada504f08d0dca1cd9b926bed3770f50e715d087c6f00ad748"}, + {file = "grpcio-1.60.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:b0571a5aef36ba9177e262dc88a9240c866d903a62799e44fd4aae3f9a2ec17e"}, + {file = "grpcio-1.60.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6fd9584bf1bccdfff1512719316efa77be235469e1e3295dce64538c4773840b"}, + {file = "grpcio-1.60.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d6a478581b1a1a8fdf3318ecb5f4d0cda41cacdffe2b527c23707c9c1b8fdb55"}, + {file = "grpcio-1.60.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:77c8a317f0fd5a0a2be8ed5cbe5341537d5c00bb79b3bb27ba7c5378ba77dbca"}, + {file = "grpcio-1.60.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1c30bb23a41df95109db130a6cc1b974844300ae2e5d68dd4947aacba5985aa5"}, + {file = "grpcio-1.60.0-cp312-cp312-win32.whl", hash = "sha256:2aef56e85901c2397bd557c5ba514f84de1f0ae5dd132f5d5fed042858115951"}, + {file = "grpcio-1.60.0-cp312-cp312-win_amd64.whl", hash = "sha256:e381fe0c2aa6c03b056ad8f52f8efca7be29fb4d9ae2f8873520843b6039612a"}, + {file = "grpcio-1.60.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:92f88ca1b956eb8427a11bb8b4a0c0b2b03377235fc5102cb05e533b8693a415"}, + {file = "grpcio-1.60.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:e278eafb406f7e1b1b637c2cf51d3ad45883bb5bd1ca56bc05e4fc135dfdaa65"}, + {file = "grpcio-1.60.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:a48edde788b99214613e440fce495bbe2b1e142a7f214cce9e0832146c41e324"}, + {file = "grpcio-1.60.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:de2ad69c9a094bf37c1102b5744c9aec6cf74d2b635558b779085d0263166454"}, + {file = "grpcio-1.60.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:073f959c6f570797272f4ee9464a9997eaf1e98c27cb680225b82b53390d61e6"}, + {file = "grpcio-1.60.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c826f93050c73e7769806f92e601e0efdb83ec8d7c76ddf45d514fee54e8e619"}, + {file = "grpcio-1.60.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:9e30be89a75ee66aec7f9e60086fadb37ff8c0ba49a022887c28c134341f7179"}, + {file = "grpcio-1.60.0-cp37-cp37m-win_amd64.whl", hash = "sha256:b0fb2d4801546598ac5cd18e3ec79c1a9af8b8f2a86283c55a5337c5aeca4b1b"}, + {file = "grpcio-1.60.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:9073513ec380434eb8d21970e1ab3161041de121f4018bbed3146839451a6d8e"}, + {file = "grpcio-1.60.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:74d7d9fa97809c5b892449b28a65ec2bfa458a4735ddad46074f9f7d9550ad13"}, + {file = "grpcio-1.60.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:1434ca77d6fed4ea312901122dc8da6c4389738bf5788f43efb19a838ac03ead"}, + {file = "grpcio-1.60.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e61e76020e0c332a98290323ecfec721c9544f5b739fab925b6e8cbe1944cf19"}, + {file = "grpcio-1.60.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675997222f2e2f22928fbba640824aebd43791116034f62006e19730715166c0"}, + {file = "grpcio-1.60.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5208a57eae445ae84a219dfd8b56e04313445d146873117b5fa75f3245bc1390"}, + {file = "grpcio-1.60.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:428d699c8553c27e98f4d29fdc0f0edc50e9a8a7590bfd294d2edb0da7be3629"}, + {file = "grpcio-1.60.0-cp38-cp38-win32.whl", hash = "sha256:83f2292ae292ed5a47cdcb9821039ca8e88902923198f2193f13959360c01860"}, + {file = "grpcio-1.60.0-cp38-cp38-win_amd64.whl", hash = "sha256:705a68a973c4c76db5d369ed573fec3367d7d196673fa86614b33d8c8e9ebb08"}, + {file = "grpcio-1.60.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:c193109ca4070cdcaa6eff00fdb5a56233dc7610216d58fb81638f89f02e4968"}, + {file = "grpcio-1.60.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:676e4a44e740deaba0f4d95ba1d8c5c89a2fcc43d02c39f69450b1fa19d39590"}, + {file = "grpcio-1.60.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:5ff21e000ff2f658430bde5288cb1ac440ff15c0d7d18b5fb222f941b46cb0d2"}, + {file = "grpcio-1.60.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c86343cf9ff7b2514dd229bdd88ebba760bd8973dac192ae687ff75e39ebfab"}, + {file = "grpcio-1.60.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fd3b3968ffe7643144580f260f04d39d869fcc2cddb745deef078b09fd2b328"}, + {file = "grpcio-1.60.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:30943b9530fe3620e3b195c03130396cd0ee3a0d10a66c1bee715d1819001eaf"}, + {file = "grpcio-1.60.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b10241250cb77657ab315270b064a6c7f1add58af94befa20687e7c8d8603ae6"}, + {file = "grpcio-1.60.0-cp39-cp39-win32.whl", hash = "sha256:79a050889eb8d57a93ed21d9585bb63fca881666fc709f5d9f7f9372f5e7fd03"}, + {file = "grpcio-1.60.0-cp39-cp39-win_amd64.whl", hash = "sha256:8a97a681e82bc11a42d4372fe57898d270a2707f36c45c6676e49ce0d5c41353"}, + {file = "grpcio-1.60.0.tar.gz", hash = "sha256:2199165a1affb666aa24adf0c97436686d0a61bc5fc113c037701fb7c7fceb96"}, +] + +[package.extras] +protobuf = ["grpcio-tools (>=1.60.0)"] + +[[package]] +name = "grpcio-status" +version = "1.60.0" +description = "Status proto mapping for gRPC" +optional = false +python-versions = ">=3.6" +files = [ + {file = "grpcio-status-1.60.0.tar.gz", hash = "sha256:f10e0b6db3adc0fdc244b71962814ee982996ef06186446b5695b9fa635aa1ab"}, + {file = "grpcio_status-1.60.0-py3-none-any.whl", hash = "sha256:7d383fa36e59c1e61d380d91350badd4d12ac56e4de2c2b831b050362c3c572e"}, +] + +[package.dependencies] +googleapis-common-protos = ">=1.5.5" +grpcio = ">=1.60.0" +protobuf = ">=4.21.6" + +[[package]] +name = "idna" +version = "3.6" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.5" +files = [ + {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, + {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "isodate" +version = "0.6.1" +description = "An ISO 8601 date/time/duration parser and formatter" +optional = false +python-versions = "*" +files = [ + {file = "isodate-0.6.1-py2.py3-none-any.whl", hash = "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96"}, + {file = "isodate-0.6.1.tar.gz", hash = "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "jinja2" +version = "3.1.3" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, + {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "jsonref" +version = "0.3.0" +description = "jsonref is a library for automatic dereferencing of JSON Reference objects for Python." +optional = false +python-versions = ">=3.3,<4.0" +files = [ + {file = "jsonref-0.3.0-py3-none-any.whl", hash = "sha256:9480ad1b500f7e795daeb0ef29f9c55ae3a9ab38fb8d6659b6f4868acb5a5bc8"}, + {file = "jsonref-0.3.0.tar.gz", hash = "sha256:68b330c6815dc0d490dbb3d65ccda265ddde9f7856fd2f3322f971d456ea7549"}, +] + +[[package]] +name = "jsonschema" +version = "3.2.0" +description = "An implementation of JSON Schema validation for Python" +optional = false +python-versions = "*" +files = [ + {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, + {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, +] + +[package.dependencies] +attrs = ">=17.4.0" +pyrsistent = ">=0.14.0" +setuptools = "*" +six = ">=1.11.0" + +[package.extras] +format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] +format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] + +[[package]] +name = "markupsafe" +version = "2.1.3" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win32.whl", hash = "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431"}, + {file = "MarkupSafe-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, + {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win32.whl", hash = "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0"}, + {file = "MarkupSafe-2.1.3-cp37-cp37m-win_amd64.whl", hash = "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win32.whl", hash = "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5"}, + {file = "MarkupSafe-2.1.3-cp38-cp38-win_amd64.whl", hash = "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win32.whl", hash = "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2"}, + {file = "MarkupSafe-2.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba"}, + {file = "MarkupSafe-2.1.3.tar.gz", hash = "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad"}, +] + +[[package]] +name = "mirakuru" +version = "2.5.2" +description = "Process executor (not only) for tests." +optional = false +python-versions = ">=3.8" +files = [ + {file = "mirakuru-2.5.2-py3-none-any.whl", hash = "sha256:90c2d90a8cf14349b2f33e6db30a16acd855499811e0312e56cf80ceacf2d3e5"}, + {file = "mirakuru-2.5.2.tar.gz", hash = "sha256:41ca583d355eb7a6cfdc21c1aea549979d685c27b57239b88725434f115a7132"}, +] + +[package.dependencies] +psutil = {version = ">=4.0.0", markers = "sys_platform != \"cygwin\""} + +[[package]] +name = "mypy" +version = "1.8.0" +description = "Optional static typing for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "mypy-1.8.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:485a8942f671120f76afffff70f259e1cd0f0cfe08f81c05d8816d958d4577d3"}, + {file = "mypy-1.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:df9824ac11deaf007443e7ed2a4a26bebff98d2bc43c6da21b2b64185da011c4"}, + {file = "mypy-1.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2afecd6354bbfb6e0160f4e4ad9ba6e4e003b767dd80d85516e71f2e955ab50d"}, + {file = "mypy-1.8.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8963b83d53ee733a6e4196954502b33567ad07dfd74851f32be18eb932fb1cb9"}, + {file = "mypy-1.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:e46f44b54ebddbeedbd3d5b289a893219065ef805d95094d16a0af6630f5d410"}, + {file = "mypy-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:855fe27b80375e5c5878492f0729540db47b186509c98dae341254c8f45f42ae"}, + {file = "mypy-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4c886c6cce2d070bd7df4ec4a05a13ee20c0aa60cb587e8d1265b6c03cf91da3"}, + {file = "mypy-1.8.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d19c413b3c07cbecf1f991e2221746b0d2a9410b59cb3f4fb9557f0365a1a817"}, + {file = "mypy-1.8.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9261ed810972061388918c83c3f5cd46079d875026ba97380f3e3978a72f503d"}, + {file = "mypy-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:51720c776d148bad2372ca21ca29256ed483aa9a4cdefefcef49006dff2a6835"}, + {file = "mypy-1.8.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:52825b01f5c4c1c4eb0db253ec09c7aa17e1a7304d247c48b6f3599ef40db8bd"}, + {file = "mypy-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f5ac9a4eeb1ec0f1ccdc6f326bcdb464de5f80eb07fb38b5ddd7b0de6bc61e55"}, + {file = "mypy-1.8.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afe3fe972c645b4632c563d3f3eff1cdca2fa058f730df2b93a35e3b0c538218"}, + {file = "mypy-1.8.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:42c6680d256ab35637ef88891c6bd02514ccb7e1122133ac96055ff458f93fc3"}, + {file = "mypy-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:720a5ca70e136b675af3af63db533c1c8c9181314d207568bbe79051f122669e"}, + {file = "mypy-1.8.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:028cf9f2cae89e202d7b6593cd98db6759379f17a319b5faf4f9978d7084cdc6"}, + {file = "mypy-1.8.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4e6d97288757e1ddba10dd9549ac27982e3e74a49d8d0179fc14d4365c7add66"}, + {file = "mypy-1.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f1478736fcebb90f97e40aff11a5f253af890c845ee0c850fe80aa060a267c6"}, + {file = "mypy-1.8.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:42419861b43e6962a649068a61f4a4839205a3ef525b858377a960b9e2de6e0d"}, + {file = "mypy-1.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:2b5b6c721bd4aabaadead3a5e6fa85c11c6c795e0c81a7215776ef8afc66de02"}, + {file = "mypy-1.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5c1538c38584029352878a0466f03a8ee7547d7bd9f641f57a0f3017a7c905b8"}, + {file = "mypy-1.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4ef4be7baf08a203170f29e89d79064463b7fc7a0908b9d0d5114e8009c3a259"}, + {file = "mypy-1.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7178def594014aa6c35a8ff411cf37d682f428b3b5617ca79029d8ae72f5402b"}, + {file = "mypy-1.8.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ab3c84fa13c04aeeeabb2a7f67a25ef5d77ac9d6486ff33ded762ef353aa5592"}, + {file = "mypy-1.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:99b00bc72855812a60d253420d8a2eae839b0afa4938f09f4d2aa9bb4654263a"}, + {file = "mypy-1.8.0-py3-none-any.whl", hash = "sha256:538fd81bb5e430cc1381a443971c0475582ff9f434c16cd46d2c66763ce85d9d"}, + {file = "mypy-1.8.0.tar.gz", hash = "sha256:6ff8b244d7085a0b425b56d327b480c3b29cafbd2eff27316a004f9a7391ae07"}, +] + +[package.dependencies] +mypy-extensions = ">=1.0.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing-extensions = ">=4.1.0" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "1.0.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = false +python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] + +[[package]] +name = "numpy" +version = "1.26.3" +description = "Fundamental package for array computing in Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "numpy-1.26.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:806dd64230dbbfaca8a27faa64e2f414bf1c6622ab78cc4264f7f5f028fee3bf"}, + {file = "numpy-1.26.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02f98011ba4ab17f46f80f7f8f1c291ee7d855fcef0a5a98db80767a468c85cd"}, + {file = "numpy-1.26.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d45b3ec2faed4baca41c76617fcdcfa4f684ff7a151ce6fc78ad3b6e85af0a6"}, + {file = "numpy-1.26.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdd2b45bf079d9ad90377048e2747a0c82351989a2165821f0c96831b4a2a54b"}, + {file = "numpy-1.26.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:211ddd1e94817ed2d175b60b6374120244a4dd2287f4ece45d49228b4d529178"}, + {file = "numpy-1.26.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b1240f767f69d7c4c8a29adde2310b871153df9b26b5cb2b54a561ac85146485"}, + {file = "numpy-1.26.3-cp310-cp310-win32.whl", hash = "sha256:21a9484e75ad018974a2fdaa216524d64ed4212e418e0a551a2d83403b0531d3"}, + {file = "numpy-1.26.3-cp310-cp310-win_amd64.whl", hash = "sha256:9e1591f6ae98bcfac2a4bbf9221c0b92ab49762228f38287f6eeb5f3f55905ce"}, + {file = "numpy-1.26.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b831295e5472954104ecb46cd98c08b98b49c69fdb7040483aff799a755a7374"}, + {file = "numpy-1.26.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9e87562b91f68dd8b1c39149d0323b42e0082db7ddb8e934ab4c292094d575d6"}, + {file = "numpy-1.26.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c66d6fec467e8c0f975818c1796d25c53521124b7cfb760114be0abad53a0a2"}, + {file = "numpy-1.26.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f25e2811a9c932e43943a2615e65fc487a0b6b49218899e62e426e7f0a57eeda"}, + {file = "numpy-1.26.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:af36e0aa45e25c9f57bf684b1175e59ea05d9a7d3e8e87b7ae1a1da246f2767e"}, + {file = "numpy-1.26.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:51c7f1b344f302067b02e0f5b5d2daa9ed4a721cf49f070280ac202738ea7f00"}, + {file = "numpy-1.26.3-cp311-cp311-win32.whl", hash = "sha256:7ca4f24341df071877849eb2034948459ce3a07915c2734f1abb4018d9c49d7b"}, + {file = "numpy-1.26.3-cp311-cp311-win_amd64.whl", hash = "sha256:39763aee6dfdd4878032361b30b2b12593fb445ddb66bbac802e2113eb8a6ac4"}, + {file = "numpy-1.26.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:a7081fd19a6d573e1a05e600c82a1c421011db7935ed0d5c483e9dd96b99cf13"}, + {file = "numpy-1.26.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12c70ac274b32bc00c7f61b515126c9205323703abb99cd41836e8125ea0043e"}, + {file = "numpy-1.26.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f784e13e598e9594750b2ef6729bcd5a47f6cfe4a12cca13def35e06d8163e3"}, + {file = "numpy-1.26.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f24750ef94d56ce6e33e4019a8a4d68cfdb1ef661a52cdaee628a56d2437419"}, + {file = "numpy-1.26.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:77810ef29e0fb1d289d225cabb9ee6cf4d11978a00bb99f7f8ec2132a84e0166"}, + {file = "numpy-1.26.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8ed07a90f5450d99dad60d3799f9c03c6566709bd53b497eb9ccad9a55867f36"}, + {file = "numpy-1.26.3-cp312-cp312-win32.whl", hash = "sha256:f73497e8c38295aaa4741bdfa4fda1a5aedda5473074369eca10626835445511"}, + {file = "numpy-1.26.3-cp312-cp312-win_amd64.whl", hash = "sha256:da4b0c6c699a0ad73c810736303f7fbae483bcb012e38d7eb06a5e3b432c981b"}, + {file = "numpy-1.26.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1666f634cb3c80ccbd77ec97bc17337718f56d6658acf5d3b906ca03e90ce87f"}, + {file = "numpy-1.26.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:18c3319a7d39b2c6a9e3bb75aab2304ab79a811ac0168a671a62e6346c29b03f"}, + {file = "numpy-1.26.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b7e807d6888da0db6e7e75838444d62495e2b588b99e90dd80c3459594e857b"}, + {file = "numpy-1.26.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4d362e17bcb0011738c2d83e0a65ea8ce627057b2fdda37678f4374a382a137"}, + {file = "numpy-1.26.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b8c275f0ae90069496068c714387b4a0eba5d531aace269559ff2b43655edd58"}, + {file = "numpy-1.26.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:cc0743f0302b94f397a4a65a660d4cd24267439eb16493fb3caad2e4389bccbb"}, + {file = "numpy-1.26.3-cp39-cp39-win32.whl", hash = "sha256:9bc6d1a7f8cedd519c4b7b1156d98e051b726bf160715b769106661d567b3f03"}, + {file = "numpy-1.26.3-cp39-cp39-win_amd64.whl", hash = "sha256:867e3644e208c8922a3be26fc6bbf112a035f50f0a86497f98f228c50c607bb2"}, + {file = "numpy-1.26.3-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3c67423b3703f8fbd90f5adaa37f85b5794d3366948efe9a5190a5f3a83fc34e"}, + {file = "numpy-1.26.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46f47ee566d98849323f01b349d58f2557f02167ee301e5e28809a8c0e27a2d0"}, + {file = "numpy-1.26.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a8474703bffc65ca15853d5fd4d06b18138ae90c17c8d12169968e998e448bb5"}, + {file = "numpy-1.26.3.tar.gz", hash = "sha256:697df43e2b6310ecc9d95f05d5ef20eacc09c7c4ecc9da3f235d39e71b7da1e4"}, +] + +[[package]] +name = "orjson" +version = "3.9.12" +description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +optional = false +python-versions = ">=3.8" +files = [ + {file = "orjson-3.9.12-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:6b4e2bed7d00753c438e83b613923afdd067564ff7ed696bfe3a7b073a236e07"}, + {file = "orjson-3.9.12-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd1b8ec63f0bf54a50b498eedeccdca23bd7b658f81c524d18e410c203189365"}, + {file = "orjson-3.9.12-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ab8add018a53665042a5ae68200f1ad14c7953fa12110d12d41166f111724656"}, + {file = "orjson-3.9.12-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12756a108875526b76e505afe6d6ba34960ac6b8c5ec2f35faf73ef161e97e07"}, + {file = "orjson-3.9.12-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:890e7519c0c70296253660455f77e3a194554a3c45e42aa193cdebc76a02d82b"}, + {file = "orjson-3.9.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d664880d7f016efbae97c725b243b33c2cbb4851ddc77f683fd1eec4a7894146"}, + {file = "orjson-3.9.12-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:cfdaede0fa5b500314ec7b1249c7e30e871504a57004acd116be6acdda3b8ab3"}, + {file = "orjson-3.9.12-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6492ff5953011e1ba9ed1bf086835fd574bd0a3cbe252db8e15ed72a30479081"}, + {file = "orjson-3.9.12-cp310-none-win32.whl", hash = "sha256:29bf08e2eadb2c480fdc2e2daae58f2f013dff5d3b506edd1e02963b9ce9f8a9"}, + {file = "orjson-3.9.12-cp310-none-win_amd64.whl", hash = "sha256:0fc156fba60d6b50743337ba09f052d8afc8b64595112996d22f5fce01ab57da"}, + {file = "orjson-3.9.12-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:2849f88a0a12b8d94579b67486cbd8f3a49e36a4cb3d3f0ab352c596078c730c"}, + {file = "orjson-3.9.12-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3186b18754befa660b31c649a108a915493ea69b4fc33f624ed854ad3563ac65"}, + {file = "orjson-3.9.12-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cbbf313c9fb9d4f6cf9c22ced4b6682230457741daeb3d7060c5d06c2e73884a"}, + {file = "orjson-3.9.12-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99e8cd005b3926c3db9b63d264bd05e1bf4451787cc79a048f27f5190a9a0311"}, + {file = "orjson-3.9.12-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:59feb148392d9155f3bfed0a2a3209268e000c2c3c834fb8fe1a6af9392efcbf"}, + {file = "orjson-3.9.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a4ae815a172a1f073b05b9e04273e3b23e608a0858c4e76f606d2d75fcabde0c"}, + {file = "orjson-3.9.12-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ed398f9a9d5a1bf55b6e362ffc80ac846af2122d14a8243a1e6510a4eabcb71e"}, + {file = "orjson-3.9.12-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d3cfb76600c5a1e6be91326b8f3b83035a370e727854a96d801c1ea08b708073"}, + {file = "orjson-3.9.12-cp311-none-win32.whl", hash = "sha256:a2b6f5252c92bcab3b742ddb3ac195c0fa74bed4319acd74f5d54d79ef4715dc"}, + {file = "orjson-3.9.12-cp311-none-win_amd64.whl", hash = "sha256:c95488e4aa1d078ff5776b58f66bd29d628fa59adcb2047f4efd3ecb2bd41a71"}, + {file = "orjson-3.9.12-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:d6ce2062c4af43b92b0221ed4f445632c6bf4213f8a7da5396a122931377acd9"}, + {file = "orjson-3.9.12-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:950951799967558c214cd6cceb7ceceed6f81d2c3c4135ee4a2c9c69f58aa225"}, + {file = "orjson-3.9.12-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2dfaf71499d6fd4153f5c86eebb68e3ec1bf95851b030a4b55c7637a37bbdee4"}, + {file = "orjson-3.9.12-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:659a8d7279e46c97661839035a1a218b61957316bf0202674e944ac5cfe7ed83"}, + {file = "orjson-3.9.12-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af17fa87bccad0b7f6fd8ac8f9cbc9ee656b4552783b10b97a071337616db3e4"}, + {file = "orjson-3.9.12-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd52dec9eddf4c8c74392f3fd52fa137b5f2e2bed1d9ae958d879de5f7d7cded"}, + {file = "orjson-3.9.12-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:640e2b5d8e36b970202cfd0799d11a9a4ab46cf9212332cd642101ec952df7c8"}, + {file = "orjson-3.9.12-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:daa438bd8024e03bcea2c5a92cd719a663a58e223fba967296b6ab9992259dbf"}, + {file = "orjson-3.9.12-cp312-none-win_amd64.whl", hash = "sha256:1bb8f657c39ecdb924d02e809f992c9aafeb1ad70127d53fb573a6a6ab59d549"}, + {file = "orjson-3.9.12-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:f4098c7674901402c86ba6045a551a2ee345f9f7ed54eeffc7d86d155c8427e5"}, + {file = "orjson-3.9.12-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5586a533998267458fad3a457d6f3cdbddbcce696c916599fa8e2a10a89b24d3"}, + {file = "orjson-3.9.12-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:54071b7398cd3f90e4bb61df46705ee96cb5e33e53fc0b2f47dbd9b000e238e1"}, + {file = "orjson-3.9.12-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:67426651faa671b40443ea6f03065f9c8e22272b62fa23238b3efdacd301df31"}, + {file = "orjson-3.9.12-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4a0cd56e8ee56b203abae7d482ac0d233dbfb436bb2e2d5cbcb539fe1200a312"}, + {file = "orjson-3.9.12-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a84a0c3d4841a42e2571b1c1ead20a83e2792644c5827a606c50fc8af7ca4bee"}, + {file = "orjson-3.9.12-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:09d60450cda3fa6c8ed17770c3a88473a16460cd0ff2ba74ef0df663b6fd3bb8"}, + {file = "orjson-3.9.12-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:bc82a4db9934a78ade211cf2e07161e4f068a461c1796465d10069cb50b32a80"}, + {file = "orjson-3.9.12-cp38-none-win32.whl", hash = "sha256:61563d5d3b0019804d782137a4f32c72dc44c84e7d078b89d2d2a1adbaa47b52"}, + {file = "orjson-3.9.12-cp38-none-win_amd64.whl", hash = "sha256:410f24309fbbaa2fab776e3212a81b96a1ec6037259359a32ea79fbccfcf76aa"}, + {file = "orjson-3.9.12-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:e773f251258dd82795fd5daeac081d00b97bacf1548e44e71245543374874bcf"}, + {file = "orjson-3.9.12-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b159baecfda51c840a619948c25817d37733a4d9877fea96590ef8606468b362"}, + {file = "orjson-3.9.12-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:975e72e81a249174840d5a8df977d067b0183ef1560a32998be340f7e195c730"}, + {file = "orjson-3.9.12-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:06e42e899dde61eb1851a9fad7f1a21b8e4be063438399b63c07839b57668f6c"}, + {file = "orjson-3.9.12-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5c157e999e5694475a5515942aebeed6e43f7a1ed52267c1c93dcfde7d78d421"}, + {file = "orjson-3.9.12-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dde1bc7c035f2d03aa49dc8642d9c6c9b1a81f2470e02055e76ed8853cfae0c3"}, + {file = "orjson-3.9.12-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b0e9d73cdbdad76a53a48f563447e0e1ce34bcecef4614eb4b146383e6e7d8c9"}, + {file = "orjson-3.9.12-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:96e44b21fe407b8ed48afbb3721f3c8c8ce17e345fbe232bd4651ace7317782d"}, + {file = "orjson-3.9.12-cp39-none-win32.whl", hash = "sha256:cbd0f3555205bf2a60f8812133f2452d498dbefa14423ba90fe89f32276f7abf"}, + {file = "orjson-3.9.12-cp39-none-win_amd64.whl", hash = "sha256:03ea7ee7e992532c2f4a06edd7ee1553f0644790553a118e003e3c405add41fa"}, + {file = "orjson-3.9.12.tar.gz", hash = "sha256:da908d23a3b3243632b523344403b128722a5f45e278a8343c2bb67538dff0e4"}, +] + +[[package]] +name = "overrides" +version = "7.4.0" +description = "A decorator to automatically detect mismatch when overriding a method." +optional = false +python-versions = ">=3.6" +files = [ + {file = "overrides-7.4.0-py3-none-any.whl", hash = "sha256:3ad24583f86d6d7a49049695efe9933e67ba62f0c7625d53c59fa832ce4b8b7d"}, + {file = "overrides-7.4.0.tar.gz", hash = "sha256:9502a3cca51f4fac40b5feca985b6703a5c1f6ad815588a7ca9e285b9dca6757"}, +] + +[[package]] +name = "packaging" +version = "23.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.7" +files = [ + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, +] + +[[package]] +name = "pandas" +version = "2.1.4" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pandas-2.1.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bdec823dc6ec53f7a6339a0e34c68b144a7a1fd28d80c260534c39c62c5bf8c9"}, + {file = "pandas-2.1.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:294d96cfaf28d688f30c918a765ea2ae2e0e71d3536754f4b6de0ea4a496d034"}, + {file = "pandas-2.1.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b728fb8deba8905b319f96447a27033969f3ea1fea09d07d296c9030ab2ed1d"}, + {file = "pandas-2.1.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00028e6737c594feac3c2df15636d73ace46b8314d236100b57ed7e4b9ebe8d9"}, + {file = "pandas-2.1.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:426dc0f1b187523c4db06f96fb5c8d1a845e259c99bda74f7de97bd8a3bb3139"}, + {file = "pandas-2.1.4-cp310-cp310-win_amd64.whl", hash = "sha256:f237e6ca6421265643608813ce9793610ad09b40154a3344a088159590469e46"}, + {file = "pandas-2.1.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b7d852d16c270e4331f6f59b3e9aa23f935f5c4b0ed2d0bc77637a8890a5d092"}, + {file = "pandas-2.1.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bd7d5f2f54f78164b3d7a40f33bf79a74cdee72c31affec86bfcabe7e0789821"}, + {file = "pandas-2.1.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0aa6e92e639da0d6e2017d9ccff563222f4eb31e4b2c3cf32a2a392fc3103c0d"}, + {file = "pandas-2.1.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d797591b6846b9db79e65dc2d0d48e61f7db8d10b2a9480b4e3faaddc421a171"}, + {file = "pandas-2.1.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d2d3e7b00f703aea3945995ee63375c61b2e6aa5aa7871c5d622870e5e137623"}, + {file = "pandas-2.1.4-cp311-cp311-win_amd64.whl", hash = "sha256:dc9bf7ade01143cddc0074aa6995edd05323974e6e40d9dbde081021ded8510e"}, + {file = "pandas-2.1.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:482d5076e1791777e1571f2e2d789e940dedd927325cc3cb6d0800c6304082f6"}, + {file = "pandas-2.1.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8a706cfe7955c4ca59af8c7a0517370eafbd98593155b48f10f9811da440248b"}, + {file = "pandas-2.1.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0513a132a15977b4a5b89aabd304647919bc2169eac4c8536afb29c07c23540"}, + {file = "pandas-2.1.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9f17f2b6fc076b2a0078862547595d66244db0f41bf79fc5f64a5c4d635bead"}, + {file = "pandas-2.1.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:45d63d2a9b1b37fa6c84a68ba2422dc9ed018bdaa668c7f47566a01188ceeec1"}, + {file = "pandas-2.1.4-cp312-cp312-win_amd64.whl", hash = "sha256:f69b0c9bb174a2342818d3e2778584e18c740d56857fc5cdb944ec8bbe4082cf"}, + {file = "pandas-2.1.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3f06bda01a143020bad20f7a85dd5f4a1600112145f126bc9e3e42077c24ef34"}, + {file = "pandas-2.1.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ab5796839eb1fd62a39eec2916d3e979ec3130509930fea17fe6f81e18108f6a"}, + {file = "pandas-2.1.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edbaf9e8d3a63a9276d707b4d25930a262341bca9874fcb22eff5e3da5394732"}, + {file = "pandas-2.1.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ebfd771110b50055712b3b711b51bee5d50135429364d0498e1213a7adc2be8"}, + {file = "pandas-2.1.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8ea107e0be2aba1da619cc6ba3f999b2bfc9669a83554b1904ce3dd9507f0860"}, + {file = "pandas-2.1.4-cp39-cp39-win_amd64.whl", hash = "sha256:d65148b14788b3758daf57bf42725caa536575da2b64df9964c563b015230984"}, + {file = "pandas-2.1.4.tar.gz", hash = "sha256:fcb68203c833cc735321512e13861358079a96c174a61f5116a1de89c58c0ef7"}, +] + +[package.dependencies] +numpy = [ + {version = ">=1.23.2,<2", markers = "python_version == \"3.11\""}, + {version = ">=1.26.0,<2", markers = "python_version >= \"3.12\""}, + {version = ">=1.22.4,<2", markers = "python_version < \"3.11\""}, +] +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.1" + +[package.extras] +all = ["PyQt5 (>=5.15.6)", "SQLAlchemy (>=1.4.36)", "beautifulsoup4 (>=4.11.1)", "bottleneck (>=1.3.4)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=0.8.1)", "fsspec (>=2022.05.0)", "gcsfs (>=2022.05.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.8.0)", "matplotlib (>=3.6.1)", "numba (>=0.55.2)", "numexpr (>=2.8.0)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pandas-gbq (>=0.17.5)", "psycopg2 (>=2.9.3)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.5)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "pyxlsb (>=1.0.9)", "qtpy (>=2.2.0)", "s3fs (>=2022.05.0)", "scipy (>=1.8.1)", "tables (>=3.7.0)", "tabulate (>=0.8.10)", "xarray (>=2022.03.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)", "zstandard (>=0.17.0)"] +aws = ["s3fs (>=2022.05.0)"] +clipboard = ["PyQt5 (>=5.15.6)", "qtpy (>=2.2.0)"] +compression = ["zstandard (>=0.17.0)"] +computation = ["scipy (>=1.8.1)", "xarray (>=2022.03.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pyxlsb (>=1.0.9)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)"] +feather = ["pyarrow (>=7.0.0)"] +fss = ["fsspec (>=2022.05.0)"] +gcp = ["gcsfs (>=2022.05.0)", "pandas-gbq (>=0.17.5)"] +hdf5 = ["tables (>=3.7.0)"] +html = ["beautifulsoup4 (>=4.11.1)", "html5lib (>=1.1)", "lxml (>=4.8.0)"] +mysql = ["SQLAlchemy (>=1.4.36)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.8.10)"] +parquet = ["pyarrow (>=7.0.0)"] +performance = ["bottleneck (>=1.3.4)", "numba (>=0.55.2)", "numexpr (>=2.8.0)"] +plot = ["matplotlib (>=3.6.1)"] +postgresql = ["SQLAlchemy (>=1.4.36)", "psycopg2 (>=2.9.3)"] +spss = ["pyreadstat (>=1.1.5)"] +sql-other = ["SQLAlchemy (>=1.4.36)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.8.0)"] + +[[package]] +name = "pandas-stubs" +version = "2.1.4.231227" +description = "Type annotations for pandas" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pandas_stubs-2.1.4.231227-py3-none-any.whl", hash = "sha256:211fc23e6ae87073bdf41dbf362c4a4d85e1e3477cb078dbac3da6c7fdaefba8"}, + {file = "pandas_stubs-2.1.4.231227.tar.gz", hash = "sha256:3ea29ef001e9e44985f5ebde02d4413f94891ef6ec7e5056fb07d125be796c23"}, +] + +[package.dependencies] +numpy = {version = ">=1.26.0", markers = "python_version < \"3.13\""} +types-pytz = ">=2022.1.1" + +[[package]] +name = "pdoc" +version = "14.4.0" +description = "API Documentation for Python Projects" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pdoc-14.4.0-py3-none-any.whl", hash = "sha256:6ea4fe07620b1f7601e2708a307a257636ec206e20b5611640b30f2e3cab47d6"}, + {file = "pdoc-14.4.0.tar.gz", hash = "sha256:c92edc425429ccbe287ace2a027953c24f13de53eab484c1a6d31ca72dd2fda9"}, +] + +[package.dependencies] +Jinja2 = ">=2.11.0" +MarkupSafe = "*" +pygments = ">=2.12.0" + +[package.extras] +dev = ["hypothesis", "mypy", "pdoc-pyo3-sample-library (==1.0.11)", "pygments (>=2.14.0)", "pytest", "pytest-cov", "pytest-timeout", "ruff", "tox", "types-pygments"] + +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + +[[package]] +name = "platformdirs" +version = "3.11.0" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +optional = false +python-versions = ">=3.7" +files = [ + {file = "platformdirs-3.11.0-py3-none-any.whl", hash = "sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e"}, + {file = "platformdirs-3.11.0.tar.gz", hash = "sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3"}, +] + +[package.extras] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] + +[[package]] +name = "pluggy" +version = "1.3.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.3.0-py3-none-any.whl", hash = "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"}, + {file = "pluggy-1.3.0.tar.gz", hash = "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "port-for" +version = "0.7.2" +description = "Utility that helps with local TCP ports management. It can find an unused TCP localhost port and remember the association." +optional = false +python-versions = ">=3.8" +files = [ + {file = "port-for-0.7.2.tar.gz", hash = "sha256:074f29335130578aa42fef3726985e57d01c15189e509633a8a1b0b7f9226349"}, + {file = "port_for-0.7.2-py3-none-any.whl", hash = "sha256:16b279ab4f210bad33515c45bd9af0c6e048ab24c3b6bbd9cfc7e451782617df"}, +] + +[[package]] +name = "proto-plus" +version = "1.23.0" +description = "Beautiful, Pythonic protocol buffers." +optional = false +python-versions = ">=3.6" +files = [ + {file = "proto-plus-1.23.0.tar.gz", hash = "sha256:89075171ef11988b3fa157f5dbd8b9cf09d65fffee97e29ce403cd8defba19d2"}, + {file = "proto_plus-1.23.0-py3-none-any.whl", hash = "sha256:a829c79e619e1cf632de091013a4173deed13a55f326ef84f05af6f50ff4c82c"}, +] + +[package.dependencies] +protobuf = ">=3.19.0,<5.0.0dev" + +[package.extras] +testing = ["google-api-core[grpc] (>=1.31.5)"] + +[[package]] +name = "protobuf" +version = "4.25.2" +description = "" +optional = false +python-versions = ">=3.8" +files = [ + {file = "protobuf-4.25.2-cp310-abi3-win32.whl", hash = "sha256:b50c949608682b12efb0b2717f53256f03636af5f60ac0c1d900df6213910fd6"}, + {file = "protobuf-4.25.2-cp310-abi3-win_amd64.whl", hash = "sha256:8f62574857ee1de9f770baf04dde4165e30b15ad97ba03ceac65f760ff018ac9"}, + {file = "protobuf-4.25.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:2db9f8fa64fbdcdc93767d3cf81e0f2aef176284071507e3ede160811502fd3d"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:10894a2885b7175d3984f2be8d9850712c57d5e7587a2410720af8be56cdaf62"}, + {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fc381d1dd0516343f1440019cedf08a7405f791cd49eef4ae1ea06520bc1c020"}, + {file = "protobuf-4.25.2-cp38-cp38-win32.whl", hash = "sha256:33a1aeef4b1927431d1be780e87b641e322b88d654203a9e9d93f218ee359e61"}, + {file = "protobuf-4.25.2-cp38-cp38-win_amd64.whl", hash = "sha256:47f3de503fe7c1245f6f03bea7e8d3ec11c6c4a2ea9ef910e3221c8a15516d62"}, + {file = "protobuf-4.25.2-cp39-cp39-win32.whl", hash = "sha256:5e5c933b4c30a988b52e0b7c02641760a5ba046edc5e43d3b94a74c9fc57c1b3"}, + {file = "protobuf-4.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:d66a769b8d687df9024f2985d5137a337f957a0916cf5464d1513eee96a63ff0"}, + {file = "protobuf-4.25.2-py3-none-any.whl", hash = "sha256:a8b7a98d4ce823303145bf3c1a8bdb0f2f4642a414b196f04ad9853ed0c8f830"}, + {file = "protobuf-4.25.2.tar.gz", hash = "sha256:fe599e175cb347efc8ee524bcd4b902d11f7262c0e569ececcb89995c15f0a5e"}, +] + +[[package]] +name = "psutil" +version = "5.9.7" +description = "Cross-platform lib for process and system monitoring in Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "psutil-5.9.7-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:0bd41bf2d1463dfa535942b2a8f0e958acf6607ac0be52265ab31f7923bcd5e6"}, + {file = "psutil-5.9.7-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:5794944462509e49d4d458f4dbfb92c47539e7d8d15c796f141f474010084056"}, + {file = "psutil-5.9.7-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:fe361f743cb3389b8efda21980d93eb55c1f1e3898269bc9a2a1d0bb7b1f6508"}, + {file = "psutil-5.9.7-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:e469990e28f1ad738f65a42dcfc17adaed9d0f325d55047593cb9033a0ab63df"}, + {file = "psutil-5.9.7-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:3c4747a3e2ead1589e647e64aad601981f01b68f9398ddf94d01e3dc0d1e57c7"}, + {file = "psutil-5.9.7-cp27-none-win32.whl", hash = "sha256:1d4bc4a0148fdd7fd8f38e0498639ae128e64538faa507df25a20f8f7fb2341c"}, + {file = "psutil-5.9.7-cp27-none-win_amd64.whl", hash = "sha256:4c03362e280d06bbbfcd52f29acd79c733e0af33d707c54255d21029b8b32ba6"}, + {file = "psutil-5.9.7-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ea36cc62e69a13ec52b2f625c27527f6e4479bca2b340b7a452af55b34fcbe2e"}, + {file = "psutil-5.9.7-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1132704b876e58d277168cd729d64750633d5ff0183acf5b3c986b8466cd0284"}, + {file = "psutil-5.9.7-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe8b7f07948f1304497ce4f4684881250cd859b16d06a1dc4d7941eeb6233bfe"}, + {file = "psutil-5.9.7-cp36-cp36m-win32.whl", hash = "sha256:b27f8fdb190c8c03914f908a4555159327d7481dac2f01008d483137ef3311a9"}, + {file = "psutil-5.9.7-cp36-cp36m-win_amd64.whl", hash = "sha256:44969859757f4d8f2a9bd5b76eba8c3099a2c8cf3992ff62144061e39ba8568e"}, + {file = "psutil-5.9.7-cp37-abi3-win32.whl", hash = "sha256:c727ca5a9b2dd5193b8644b9f0c883d54f1248310023b5ad3e92036c5e2ada68"}, + {file = "psutil-5.9.7-cp37-abi3-win_amd64.whl", hash = "sha256:f37f87e4d73b79e6c5e749440c3113b81d1ee7d26f21c19c47371ddea834f414"}, + {file = "psutil-5.9.7-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:032f4f2c909818c86cea4fe2cc407f1c0f0cde8e6c6d702b28b8ce0c0d143340"}, + {file = "psutil-5.9.7.tar.gz", hash = "sha256:3f02134e82cfb5d089fddf20bb2e03fd5cd52395321d1c8458a9e58500ff417c"}, +] + +[package.extras] +test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] + +[[package]] +name = "psycopg" +version = "3.1.17" +description = "PostgreSQL database adapter for Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "psycopg-3.1.17-py3-none-any.whl", hash = "sha256:96b7b13af6d5a514118b759a66b2799a8a4aa78675fa6bb0d3f7d52d67eff002"}, + {file = "psycopg-3.1.17.tar.gz", hash = "sha256:437e7d7925459f21de570383e2e10542aceb3b9cb972ce957fdd3826ca47edc6"}, +] + +[package.dependencies] +psycopg-binary = {version = "3.1.17", optional = true, markers = "implementation_name != \"pypy\" and extra == \"binary\""} +psycopg-pool = {version = "*", optional = true, markers = "extra == \"pool\""} +typing-extensions = ">=4.1" +tzdata = {version = "*", markers = "sys_platform == \"win32\""} + +[package.extras] +binary = ["psycopg-binary (==3.1.17)"] +c = ["psycopg-c (==3.1.17)"] +dev = ["black (>=23.1.0)", "codespell (>=2.2)", "dnspython (>=2.1)", "flake8 (>=4.0)", "mypy (>=1.4.1)", "types-setuptools (>=57.4)", "wheel (>=0.37)"] +docs = ["Sphinx (>=5.0)", "furo (==2022.6.21)", "sphinx-autobuild (>=2021.3.14)", "sphinx-autodoc-typehints (>=1.12)"] +pool = ["psycopg-pool"] +test = ["anyio (>=3.6.2,<4.0)", "mypy (>=1.4.1)", "pproxy (>=2.7)", "pytest (>=6.2.5)", "pytest-cov (>=3.0)", "pytest-randomly (>=3.5)"] + +[[package]] +name = "psycopg-binary" +version = "3.1.17" +description = "PostgreSQL database adapter for Python -- C optimisation distribution" +optional = false +python-versions = ">=3.7" +files = [ + {file = "psycopg_binary-3.1.17-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f9ba559eabb0ba1afd4e0504fa0b10e00a212cac0c4028b8a1c3b087b5c1e5de"}, + {file = "psycopg_binary-3.1.17-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2b2a689eaede08cf91a36b10b0da6568dd6e4669200f201e082639816737992b"}, + {file = "psycopg_binary-3.1.17-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a16abab0c1abc58feb6ab11d78d0f8178a67c3586bd70628ec7c0218ec04c4ef"}, + {file = "psycopg_binary-3.1.17-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:73e7097b81cad9ae358334e3cec625246bb3b8013ae6bb287758dd6435e12f65"}, + {file = "psycopg_binary-3.1.17-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:67a5b93101bc85a95a189c0a23d02a29cf06c1080a695a0dedfdd50dd734662a"}, + {file = "psycopg_binary-3.1.17-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:751b31c2faae0348f87f22b45ef58f704bdcfc2abdd680fa0c743c124071157e"}, + {file = "psycopg_binary-3.1.17-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b447ea765e71bc33a82cf070bba814b1efa77967442d116b95ccef8ce5da7631"}, + {file = "psycopg_binary-3.1.17-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:d2e9ed88d9a6a475c67bf70fc8285e88ccece0391727c7701e5a512e0eafbb05"}, + {file = "psycopg_binary-3.1.17-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:a89f36bf7b612ff6ed3e789bd987cbd0787cf0d66c49386fa3bad816dd7bee87"}, + {file = "psycopg_binary-3.1.17-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5ccbe8b2ec444763a51ecb1213befcbb75defc1ef36e7dd5dff501a23d7ce8cf"}, + {file = "psycopg_binary-3.1.17-cp310-cp310-win_amd64.whl", hash = "sha256:adb670031b27949c9dc5cf585c4a5a6b4469d3879fd2fb9d39b6d53e5f66b9bc"}, + {file = "psycopg_binary-3.1.17-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0227885686c2cc0104ceb22d6eebc732766e9ad48710408cb0123237432e5435"}, + {file = "psycopg_binary-3.1.17-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9124b6db07e8d8b11f4512b8b56cbe136bf1b7d0417d1280e62291a9dcad4408"}, + {file = "psycopg_binary-3.1.17-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c8a46f77ba0ca7c5a5449b777170a518fa7820e1710edb40e777c9798f00d033"}, + {file = "psycopg_binary-3.1.17-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5f5f5bcbb772d8c243d605fc7151beec760dd27532d42145a58fb74ef9c5fbf2"}, + {file = "psycopg_binary-3.1.17-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:267a82548c21476120e43dc72b961f1af52c380c0b4c951bdb34cf14cb26bd35"}, + {file = "psycopg_binary-3.1.17-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b20013051f1fd7d02b8d0766cfe8d009e8078babc00a6d39bc7e2d50a7b96af"}, + {file = "psycopg_binary-3.1.17-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:8c5c38129cc79d7e3ba553035b9962a442171e9f97bb1b8795c0885213f206f3"}, + {file = "psycopg_binary-3.1.17-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d01c4faae66de60fcd3afd3720dcc8ffa03bc2087f898106da127774db12aac5"}, + {file = "psycopg_binary-3.1.17-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:e6ae27b0617ad3809449964b5e901b21acff8e306abacb8ba71d5ee7c8c47eeb"}, + {file = "psycopg_binary-3.1.17-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:40af298b209dd77ca2f3e7eb3fbcfb87a25999fc015fcd14140bde030a164c7e"}, + {file = "psycopg_binary-3.1.17-cp311-cp311-win_amd64.whl", hash = "sha256:7b4e4c2b05f3b431e9026e82590b217e87696e7a7548f512ae8059d59fa8af3b"}, + {file = "psycopg_binary-3.1.17-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ea425a8dcd808a7232a5417d2633bfa543da583a2701b5228e9e29989a50deda"}, + {file = "psycopg_binary-3.1.17-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a3f1196d76860e72d338fab0d2b6722e8d47e2285d693e366ae36011c4a5898a"}, + {file = "psycopg_binary-3.1.17-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1e867c2a729348df218a14ba1b862e627177fd57c7b4f3db0b4c708f6d03696"}, + {file = "psycopg_binary-3.1.17-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b0711e46361ea3047cd049868419d030c8236a9dea7e9ed1f053cbd61a853ec9"}, + {file = "psycopg_binary-3.1.17-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d1c0115bdf80cf6c8c9109cb10cf6f650fd1a8d841f884925e8cb12f34eb5371"}, + {file = "psycopg_binary-3.1.17-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d0d154c780cc7b28a3a0886e8a4b18689202a1dbb522b3c771eb3a1289cf7c3"}, + {file = "psycopg_binary-3.1.17-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:f4028443bf25c1e04ecffdc552c0a98d826903dec76a1568dfddf5ebbbb03db7"}, + {file = "psycopg_binary-3.1.17-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bf424d92dd7e94705b31625b02d396297a7c8fab4b6f7de8dba6388323a7b71c"}, + {file = "psycopg_binary-3.1.17-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:00377f6963ee7e4bf71cab17c2c235ef0624df9483f3b615d86aa24cde889d42"}, + {file = "psycopg_binary-3.1.17-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9690a535d9ccd361bbc3590bfce7fe679e847f44fa7cc97f3b885f4744ca8a2c"}, + {file = "psycopg_binary-3.1.17-cp312-cp312-win_amd64.whl", hash = "sha256:6b2ae342d69684555bfe77aed5546d125b4a99012e0b83a8b3da68c8829f0935"}, + {file = "psycopg_binary-3.1.17-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:86bb3656c8d744cc1e42003414cd6c765117d70aa23da6c0f4ff2b826e0fd0fd"}, + {file = "psycopg_binary-3.1.17-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c10b7713e3ed31df7319c2a72d5fea5a2536476d7695a3e1d18a1f289060997c"}, + {file = "psycopg_binary-3.1.17-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12eab8bc91b4ba01b2ecee3b5b80501934b198f6e1f8d4b13596f3f38ba6e762"}, + {file = "psycopg_binary-3.1.17-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6a728beefd89b430ebe2729d04ba10e05036b5e9d01648da60436000d2fcd242"}, + {file = "psycopg_binary-3.1.17-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61104b8e7a43babf2bbaa36c08e31a12023e2f967166e99d6b052b11a4c7db06"}, + {file = "psycopg_binary-3.1.17-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:02cd2eb62ffc56f8c847d68765cbf461b3d11b438fe48951e44b6c563ec27d18"}, + {file = "psycopg_binary-3.1.17-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:ca1757a6e080086f7234dc45684e81a47a66a6dd492a37d6ce38c58a1a93e9ff"}, + {file = "psycopg_binary-3.1.17-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:6e3543edc18553e31a3884af3cd7eea43d6c44532d8b9b16f3e743cdf6cfe6c5"}, + {file = "psycopg_binary-3.1.17-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:914254849486e14aa931b0b3382cd16887f1507068ffba775cbdc5a55fe9ef19"}, + {file = "psycopg_binary-3.1.17-cp37-cp37m-win_amd64.whl", hash = "sha256:92fad8f1aa80a5ab316c0493dc6d1b54c1dba21937e43eea7296ff4a0ccc071e"}, + {file = "psycopg_binary-3.1.17-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6d4f2e15d33ed4f9776fdf23683512d76f4e7825c4b80677e9e3ce6c1b193ff2"}, + {file = "psycopg_binary-3.1.17-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4fa26836ce074a1104249378727e1f239a01530f36bae16e77cf6c50968599b4"}, + {file = "psycopg_binary-3.1.17-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d54bcf2dfc0880bf13f38512d44b194c092794e4ee9e01d804bc6cd3eed9bfb7"}, + {file = "psycopg_binary-3.1.17-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e28024204dc0c61094268c682041d2becfedfea2e3b46bed5f6138239304d98"}, + {file = "psycopg_binary-3.1.17-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0b1ec6895cab887b92c303565617f994c9b9db53befda81fa2a31b76fe8a3ab1"}, + {file = "psycopg_binary-3.1.17-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:420c1eb1626539c261cf3fbe099998da73eb990f9ce1a34da7feda414012ea5f"}, + {file = "psycopg_binary-3.1.17-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:83404a353240fdff5cfe9080665fdfdcaa2d4d0c5112e15b0a2fe2e59200ed57"}, + {file = "psycopg_binary-3.1.17-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a0c4ba73f9e7721dd6cc3e6953016652dbac206f654229b7a1a8ac182b16e689"}, + {file = "psycopg_binary-3.1.17-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:f6898bf1ca5aa01115807643138e3e20ec603b17a811026bc4a49d43055720a7"}, + {file = "psycopg_binary-3.1.17-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6b40fa54a02825d3d6a8009d9a82a2b4fad80387acf2b8fd6d398fd2813cb2d9"}, + {file = "psycopg_binary-3.1.17-cp38-cp38-win_amd64.whl", hash = "sha256:78ebb43dca7d5b41eee543cd005ee5a0256cecc74d84acf0fab4f025997b837e"}, + {file = "psycopg_binary-3.1.17-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:02ac573f5a6e79bb6df512b3a6279f01f033bbd45c47186e8872fee45f6681d0"}, + {file = "psycopg_binary-3.1.17-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:704f6393d758b12a4369887fe956b2a8c99e4aced839d9084de8e3f056015d40"}, + {file = "psycopg_binary-3.1.17-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0340ef87a888fd940796c909e038426f4901046f61856598582a817162c64984"}, + {file = "psycopg_binary-3.1.17-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a880e4113af3ab84d6a0991e3f85a2424924c8a182733ab8d964421df8b5190a"}, + {file = "psycopg_binary-3.1.17-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:93921178b9a40c60c26e47eb44970f88c49fe484aaa3bb7ec02bb8b514eab3d9"}, + {file = "psycopg_binary-3.1.17-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a05400e9314fc30bc1364865ba9f6eaa2def42b5e7e67f71f9a4430f870023e"}, + {file = "psycopg_binary-3.1.17-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:3e2cc2bbf37ff1cf11e8b871c294e3532636a3cf7f0c82518b7537158923d77b"}, + {file = "psycopg_binary-3.1.17-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a343261701a8f63f0d8268f7fd32be40ffe28d24b65d905404ca03e7281f7bb5"}, + {file = "psycopg_binary-3.1.17-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:dceb3930ec426623c0cacc78e447a90882981e8c49d6fea8d1e48850e24a0170"}, + {file = "psycopg_binary-3.1.17-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d613a23f8928f30acb2b6b2398cb7775ba9852e8968e15df13807ba0d3ebd565"}, + {file = "psycopg_binary-3.1.17-cp39-cp39-win_amd64.whl", hash = "sha256:d90c0531e9d591bde8cea04e75107fcddcc56811b638a34853436b23c9a3cb7d"}, +] + +[[package]] +name = "psycopg-pool" +version = "3.2.1" +description = "Connection Pool for Psycopg" +optional = false +python-versions = ">=3.8" +files = [ + {file = "psycopg-pool-3.2.1.tar.gz", hash = "sha256:6509a75c073590952915eddbba7ce8b8332a440a31e77bba69561483492829ad"}, + {file = "psycopg_pool-3.2.1-py3-none-any.whl", hash = "sha256:060b551d1b97a8d358c668be58b637780b884de14d861f4f5ecc48b7563aafb7"}, +] + +[package.dependencies] +typing-extensions = ">=4.4" + +[[package]] +name = "psycopg2" +version = "2.9.9" +description = "psycopg2 - Python-PostgreSQL Database Adapter" +optional = false +python-versions = ">=3.7" +files = [ + {file = "psycopg2-2.9.9-cp310-cp310-win32.whl", hash = "sha256:38a8dcc6856f569068b47de286b472b7c473ac7977243593a288ebce0dc89516"}, + {file = "psycopg2-2.9.9-cp310-cp310-win_amd64.whl", hash = "sha256:426f9f29bde126913a20a96ff8ce7d73fd8a216cfb323b1f04da402d452853c3"}, + {file = "psycopg2-2.9.9-cp311-cp311-win32.whl", hash = "sha256:ade01303ccf7ae12c356a5e10911c9e1c51136003a9a1d92f7aa9d010fb98372"}, + {file = "psycopg2-2.9.9-cp311-cp311-win_amd64.whl", hash = "sha256:121081ea2e76729acfb0673ff33755e8703d45e926e416cb59bae3a86c6a4981"}, + {file = "psycopg2-2.9.9-cp312-cp312-win32.whl", hash = "sha256:d735786acc7dd25815e89cc4ad529a43af779db2e25aa7c626de864127e5a024"}, + {file = "psycopg2-2.9.9-cp312-cp312-win_amd64.whl", hash = "sha256:a7653d00b732afb6fc597e29c50ad28087dcb4fbfb28e86092277a559ae4e693"}, + {file = "psycopg2-2.9.9-cp37-cp37m-win32.whl", hash = "sha256:5e0d98cade4f0e0304d7d6f25bbfbc5bd186e07b38eac65379309c4ca3193efa"}, + {file = "psycopg2-2.9.9-cp37-cp37m-win_amd64.whl", hash = "sha256:7e2dacf8b009a1c1e843b5213a87f7c544b2b042476ed7755be813eaf4e8347a"}, + {file = "psycopg2-2.9.9-cp38-cp38-win32.whl", hash = "sha256:ff432630e510709564c01dafdbe996cb552e0b9f3f065eb89bdce5bd31fabf4c"}, + {file = "psycopg2-2.9.9-cp38-cp38-win_amd64.whl", hash = "sha256:bac58c024c9922c23550af2a581998624d6e02350f4ae9c5f0bc642c633a2d5e"}, + {file = "psycopg2-2.9.9-cp39-cp39-win32.whl", hash = "sha256:c92811b2d4c9b6ea0285942b2e7cac98a59e166d59c588fe5cfe1eda58e72d59"}, + {file = "psycopg2-2.9.9-cp39-cp39-win_amd64.whl", hash = "sha256:de80739447af31525feddeb8effd640782cf5998e1a4e9192ebdf829717e3913"}, + {file = "psycopg2-2.9.9.tar.gz", hash = "sha256:d1454bde93fb1e224166811694d600e746430c006fbb031ea06ecc2ea41bf156"}, +] + +[[package]] +name = "pyarrow" +version = "14.0.2" +description = "Python library for Apache Arrow" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyarrow-14.0.2-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:ba9fe808596c5dbd08b3aeffe901e5f81095baaa28e7d5118e01354c64f22807"}, + {file = "pyarrow-14.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:22a768987a16bb46220cef490c56c671993fbee8fd0475febac0b3e16b00a10e"}, + {file = "pyarrow-14.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dbba05e98f247f17e64303eb876f4a80fcd32f73c7e9ad975a83834d81f3fda"}, + {file = "pyarrow-14.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a898d134d00b1eca04998e9d286e19653f9d0fcb99587310cd10270907452a6b"}, + {file = "pyarrow-14.0.2-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:87e879323f256cb04267bb365add7208f302df942eb943c93a9dfeb8f44840b1"}, + {file = "pyarrow-14.0.2-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:76fc257559404ea5f1306ea9a3ff0541bf996ff3f7b9209fc517b5e83811fa8e"}, + {file = "pyarrow-14.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:b0c4a18e00f3a32398a7f31da47fefcd7a927545b396e1f15d0c85c2f2c778cd"}, + {file = "pyarrow-14.0.2-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:87482af32e5a0c0cce2d12eb3c039dd1d853bd905b04f3f953f147c7a196915b"}, + {file = "pyarrow-14.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:059bd8f12a70519e46cd64e1ba40e97eae55e0cbe1695edd95384653d7626b23"}, + {file = "pyarrow-14.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f16111f9ab27e60b391c5f6d197510e3ad6654e73857b4e394861fc79c37200"}, + {file = "pyarrow-14.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06ff1264fe4448e8d02073f5ce45a9f934c0f3db0a04460d0b01ff28befc3696"}, + {file = "pyarrow-14.0.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:6dd4f4b472ccf4042f1eab77e6c8bce574543f54d2135c7e396f413046397d5a"}, + {file = "pyarrow-14.0.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:32356bfb58b36059773f49e4e214996888eeea3a08893e7dbde44753799b2a02"}, + {file = "pyarrow-14.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:52809ee69d4dbf2241c0e4366d949ba035cbcf48409bf404f071f624ed313a2b"}, + {file = "pyarrow-14.0.2-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:c87824a5ac52be210d32906c715f4ed7053d0180c1060ae3ff9b7e560f53f944"}, + {file = "pyarrow-14.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a25eb2421a58e861f6ca91f43339d215476f4fe159eca603c55950c14f378cc5"}, + {file = "pyarrow-14.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5c1da70d668af5620b8ba0a23f229030a4cd6c5f24a616a146f30d2386fec422"}, + {file = "pyarrow-14.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2cc61593c8e66194c7cdfae594503e91b926a228fba40b5cf25cc593563bcd07"}, + {file = "pyarrow-14.0.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:78ea56f62fb7c0ae8ecb9afdd7893e3a7dbeb0b04106f5c08dbb23f9c0157591"}, + {file = "pyarrow-14.0.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:37c233ddbce0c67a76c0985612fef27c0c92aef9413cf5aa56952f359fcb7379"}, + {file = "pyarrow-14.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:e4b123ad0f6add92de898214d404e488167b87b5dd86e9a434126bc2b7a5578d"}, + {file = "pyarrow-14.0.2-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:e354fba8490de258be7687f341bc04aba181fc8aa1f71e4584f9890d9cb2dec2"}, + {file = "pyarrow-14.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:20e003a23a13da963f43e2b432483fdd8c38dc8882cd145f09f21792e1cf22a1"}, + {file = "pyarrow-14.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc0de7575e841f1595ac07e5bc631084fd06ca8b03c0f2ecece733d23cd5102a"}, + {file = "pyarrow-14.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66e986dc859712acb0bd45601229021f3ffcdfc49044b64c6d071aaf4fa49e98"}, + {file = "pyarrow-14.0.2-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:f7d029f20ef56673a9730766023459ece397a05001f4e4d13805111d7c2108c0"}, + {file = "pyarrow-14.0.2-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:209bac546942b0d8edc8debda248364f7f668e4aad4741bae58e67d40e5fcf75"}, + {file = "pyarrow-14.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:1e6987c5274fb87d66bb36816afb6f65707546b3c45c44c28e3c4133c010a881"}, + {file = "pyarrow-14.0.2-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:a01d0052d2a294a5f56cc1862933014e696aa08cc7b620e8c0cce5a5d362e976"}, + {file = "pyarrow-14.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a51fee3a7db4d37f8cda3ea96f32530620d43b0489d169b285d774da48ca9785"}, + {file = "pyarrow-14.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64df2bf1ef2ef14cee531e2dfe03dd924017650ffaa6f9513d7a1bb291e59c15"}, + {file = "pyarrow-14.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c0fa3bfdb0305ffe09810f9d3e2e50a2787e3a07063001dcd7adae0cee3601a"}, + {file = "pyarrow-14.0.2-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:c65bf4fd06584f058420238bc47a316e80dda01ec0dfb3044594128a6c2db794"}, + {file = "pyarrow-14.0.2-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:63ac901baec9369d6aae1cbe6cca11178fb018a8d45068aaf5bb54f94804a866"}, + {file = "pyarrow-14.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:75ee0efe7a87a687ae303d63037d08a48ef9ea0127064df18267252cfe2e9541"}, + {file = "pyarrow-14.0.2.tar.gz", hash = "sha256:36cef6ba12b499d864d1def3e990f97949e0b79400d08b7cf74504ffbd3eb025"}, +] + +[package.dependencies] +numpy = ">=1.16.6" + +[[package]] +name = "pyarrow-stubs" +version = "10.0.1.7" +description = "Type annotations for pyarrow" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "pyarrow_stubs-10.0.1.7-py3-none-any.whl", hash = "sha256:cccc7a46eddeea4e3cb85330eb8972c116a615da6188b8ae1f7a44cb724b21ac"}, +] + +[[package]] +name = "pyasn1" +version = "0.5.1" +description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1-0.5.1-py2.py3-none-any.whl", hash = "sha256:4439847c58d40b1d0a573d07e3856e95333f1976294494c325775aeca506eb58"}, + {file = "pyasn1-0.5.1.tar.gz", hash = "sha256:6d391a96e59b23130a5cfa74d6fd7f388dbbe26cc8f1edf39fdddf08d9d6676c"}, +] + +[[package]] +name = "pyasn1-modules" +version = "0.3.0" +description = "A collection of ASN.1-based protocols modules" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "pyasn1_modules-0.3.0-py2.py3-none-any.whl", hash = "sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d"}, + {file = "pyasn1_modules-0.3.0.tar.gz", hash = "sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c"}, +] + +[package.dependencies] +pyasn1 = ">=0.4.6,<0.6.0" + +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] + +[[package]] +name = "pydantic" +version = "1.10.13" +description = "Data validation and settings management using python type hints" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pydantic-1.10.13-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:efff03cc7a4f29d9009d1c96ceb1e7a70a65cfe86e89d34e4a5f2ab1e5693737"}, + {file = "pydantic-1.10.13-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3ecea2b9d80e5333303eeb77e180b90e95eea8f765d08c3d278cd56b00345d01"}, + {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1740068fd8e2ef6eb27a20e5651df000978edce6da6803c2bef0bc74540f9548"}, + {file = "pydantic-1.10.13-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84bafe2e60b5e78bc64a2941b4c071a4b7404c5c907f5f5a99b0139781e69ed8"}, + {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bc0898c12f8e9c97f6cd44c0ed70d55749eaf783716896960b4ecce2edfd2d69"}, + {file = "pydantic-1.10.13-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:654db58ae399fe6434e55325a2c3e959836bd17a6f6a0b6ca8107ea0571d2e17"}, + {file = "pydantic-1.10.13-cp310-cp310-win_amd64.whl", hash = "sha256:75ac15385a3534d887a99c713aa3da88a30fbd6204a5cd0dc4dab3d770b9bd2f"}, + {file = "pydantic-1.10.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c553f6a156deb868ba38a23cf0df886c63492e9257f60a79c0fd8e7173537653"}, + {file = "pydantic-1.10.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5e08865bc6464df8c7d61439ef4439829e3ab62ab1669cddea8dd00cd74b9ffe"}, + {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e31647d85a2013d926ce60b84f9dd5300d44535a9941fe825dc349ae1f760df9"}, + {file = "pydantic-1.10.13-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:210ce042e8f6f7c01168b2d84d4c9eb2b009fe7bf572c2266e235edf14bacd80"}, + {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8ae5dd6b721459bfa30805f4c25880e0dd78fc5b5879f9f7a692196ddcb5a580"}, + {file = "pydantic-1.10.13-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f8e81fc5fb17dae698f52bdd1c4f18b6ca674d7068242b2aff075f588301bbb0"}, + {file = "pydantic-1.10.13-cp311-cp311-win_amd64.whl", hash = "sha256:61d9dce220447fb74f45e73d7ff3b530e25db30192ad8d425166d43c5deb6df0"}, + {file = "pydantic-1.10.13-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4b03e42ec20286f052490423682016fd80fda830d8e4119f8ab13ec7464c0132"}, + {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f59ef915cac80275245824e9d771ee939133be38215555e9dc90c6cb148aaeb5"}, + {file = "pydantic-1.10.13-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5a1f9f747851338933942db7af7b6ee8268568ef2ed86c4185c6ef4402e80ba8"}, + {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:97cce3ae7341f7620a0ba5ef6cf043975cd9d2b81f3aa5f4ea37928269bc1b87"}, + {file = "pydantic-1.10.13-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:854223752ba81e3abf663d685f105c64150873cc6f5d0c01d3e3220bcff7d36f"}, + {file = "pydantic-1.10.13-cp37-cp37m-win_amd64.whl", hash = "sha256:b97c1fac8c49be29486df85968682b0afa77e1b809aff74b83081cc115e52f33"}, + {file = "pydantic-1.10.13-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c958d053453a1c4b1c2062b05cd42d9d5c8eb67537b8d5a7e3c3032943ecd261"}, + {file = "pydantic-1.10.13-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4c5370a7edaac06daee3af1c8b1192e305bc102abcbf2a92374b5bc793818599"}, + {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d6f6e7305244bddb4414ba7094ce910560c907bdfa3501e9db1a7fd7eaea127"}, + {file = "pydantic-1.10.13-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3a3c792a58e1622667a2837512099eac62490cdfd63bd407993aaf200a4cf1f"}, + {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:c636925f38b8db208e09d344c7aa4f29a86bb9947495dd6b6d376ad10334fb78"}, + {file = "pydantic-1.10.13-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:678bcf5591b63cc917100dc50ab6caebe597ac67e8c9ccb75e698f66038ea953"}, + {file = "pydantic-1.10.13-cp38-cp38-win_amd64.whl", hash = "sha256:6cf25c1a65c27923a17b3da28a0bdb99f62ee04230c931d83e888012851f4e7f"}, + {file = "pydantic-1.10.13-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8ef467901d7a41fa0ca6db9ae3ec0021e3f657ce2c208e98cd511f3161c762c6"}, + {file = "pydantic-1.10.13-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:968ac42970f57b8344ee08837b62f6ee6f53c33f603547a55571c954a4225691"}, + {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9849f031cf8a2f0a928fe885e5a04b08006d6d41876b8bbd2fc68a18f9f2e3fd"}, + {file = "pydantic-1.10.13-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:56e3ff861c3b9c6857579de282ce8baabf443f42ffba355bf070770ed63e11e1"}, + {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f00790179497767aae6bcdc36355792c79e7bbb20b145ff449700eb076c5f96"}, + {file = "pydantic-1.10.13-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:75b297827b59bc229cac1a23a2f7a4ac0031068e5be0ce385be1462e7e17a35d"}, + {file = "pydantic-1.10.13-cp39-cp39-win_amd64.whl", hash = "sha256:e70ca129d2053fb8b728ee7d1af8e553a928d7e301a311094b8a0501adc8763d"}, + {file = "pydantic-1.10.13-py3-none-any.whl", hash = "sha256:b87326822e71bd5f313e7d3bfdc77ac3247035ac10b0c0618bd99dcf95b1e687"}, + {file = "pydantic-1.10.13.tar.gz", hash = "sha256:32c8b48dcd3b2ac4e78b0ba4af3a2c2eb6048cb75202f0ea7b34feb740efc340"}, +] + +[package.dependencies] +typing-extensions = ">=4.2.0" + +[package.extras] +dotenv = ["python-dotenv (>=0.10.4)"] +email = ["email-validator (>=1.0.3)"] + +[[package]] +name = "pygments" +version = "2.17.2" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pygments-2.17.2-py3-none-any.whl", hash = "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c"}, + {file = "pygments-2.17.2.tar.gz", hash = "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"}, +] + +[package.extras] +plugins = ["importlib-metadata"] +windows-terminal = ["colorama (>=0.4.6)"] + +[[package]] +name = "pyjwt" +version = "2.8.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, + {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, +] + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + +[[package]] +name = "pyopenssl" +version = "23.3.0" +description = "Python wrapper module around the OpenSSL library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pyOpenSSL-23.3.0-py3-none-any.whl", hash = "sha256:6756834481d9ed5470f4a9393455154bc92fe7a64b7bc6ee2c804e78c52099b2"}, + {file = "pyOpenSSL-23.3.0.tar.gz", hash = "sha256:6b2cba5cc46e822750ec3e5a81ee12819850b11303630d575e98108a079c2b12"}, +] + +[package.dependencies] +cryptography = ">=41.0.5,<42" + +[package.extras] +docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx-rtd-theme"] +test = ["flaky", "pretend", "pytest (>=3.0.1)"] + +[[package]] +name = "pyrate-limiter" +version = "3.1.1" +description = "Python Rate-Limiter using Leaky-Bucket Algorithm" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "pyrate_limiter-3.1.1-py3-none-any.whl", hash = "sha256:c51906f1d51d56dc992ff6c26e8300e32151bc6cfa3e6559792e31971dfd4e2b"}, + {file = "pyrate_limiter-3.1.1.tar.gz", hash = "sha256:2f57eda712687e6eccddf6afe8f8a15b409b97ed675fe64a626058f12863b7b7"}, +] + +[package.extras] +all = ["filelock (>=3.0)", "redis (>=5.0.0,<6.0.0)"] +docs = ["furo (>=2022.3.4,<2023.0.0)", "myst-parser (>=0.17)", "sphinx (>=4.3.0,<5.0.0)", "sphinx-autodoc-typehints (>=1.17,<2.0)", "sphinx-copybutton (>=0.5)", "sphinxcontrib-apidoc (>=0.3,<0.4)"] + +[[package]] +name = "pyrsistent" +version = "0.20.0" +description = "Persistent/Functional/Immutable data structures" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pyrsistent-0.20.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8c3aba3e01235221e5b229a6c05f585f344734bd1ad42a8ac51493d74722bbce"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1beb78af5423b879edaf23c5591ff292cf7c33979734c99aa66d5914ead880f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21cc459636983764e692b9eba7144cdd54fdec23ccdb1e8ba392a63666c60c34"}, + {file = "pyrsistent-0.20.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f5ac696f02b3fc01a710427585c855f65cd9c640e14f52abe52020722bb4906b"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win32.whl", hash = "sha256:0724c506cd8b63c69c7f883cc233aac948c1ea946ea95996ad8b1380c25e1d3f"}, + {file = "pyrsistent-0.20.0-cp310-cp310-win_amd64.whl", hash = "sha256:8441cf9616d642c475684d6cf2520dd24812e996ba9af15e606df5f6fd9d04a7"}, + {file = "pyrsistent-0.20.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:0f3b1bcaa1f0629c978b355a7c37acd58907390149b7311b5db1b37648eb6958"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cdd7ef1ea7a491ae70d826b6cc64868de09a1d5ff9ef8d574250d0940e275b8"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cae40a9e3ce178415040a0383f00e8d68b569e97f31928a3a8ad37e3fde6df6a"}, + {file = "pyrsistent-0.20.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6288b3fa6622ad8a91e6eb759cfc48ff3089e7c17fb1d4c59a919769314af224"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win32.whl", hash = "sha256:7d29c23bdf6e5438c755b941cef867ec2a4a172ceb9f50553b6ed70d50dfd656"}, + {file = "pyrsistent-0.20.0-cp311-cp311-win_amd64.whl", hash = "sha256:59a89bccd615551391f3237e00006a26bcf98a4d18623a19909a2c48b8e986ee"}, + {file = "pyrsistent-0.20.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:09848306523a3aba463c4b49493a760e7a6ca52e4826aa100ee99d8d39b7ad1e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a14798c3005ec892bbada26485c2eea3b54109cb2533713e355c806891f63c5e"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b14decb628fac50db5e02ee5a35a9c0772d20277824cfe845c8a8b717c15daa3"}, + {file = "pyrsistent-0.20.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e2c116cc804d9b09ce9814d17df5edf1df0c624aba3b43bc1ad90411487036d"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win32.whl", hash = "sha256:e78d0c7c1e99a4a45c99143900ea0546025e41bb59ebc10182e947cf1ece9174"}, + {file = "pyrsistent-0.20.0-cp312-cp312-win_amd64.whl", hash = "sha256:4021a7f963d88ccd15b523787d18ed5e5269ce57aa4037146a2377ff607ae87d"}, + {file = "pyrsistent-0.20.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:79ed12ba79935adaac1664fd7e0e585a22caa539dfc9b7c7c6d5ebf91fb89054"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f920385a11207dc372a028b3f1e1038bb244b3ec38d448e6d8e43c6b3ba20e98"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5c2d012671b7391803263419e31b5c7c21e7c95c8760d7fc35602353dee714"}, + {file = "pyrsistent-0.20.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ef3992833fbd686ee783590639f4b8343a57f1f75de8633749d984dc0eb16c86"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win32.whl", hash = "sha256:881bbea27bbd32d37eb24dd320a5e745a2a5b092a17f6debc1349252fac85423"}, + {file = "pyrsistent-0.20.0-cp38-cp38-win_amd64.whl", hash = "sha256:6d270ec9dd33cdb13f4d62c95c1a5a50e6b7cdd86302b494217137f760495b9d"}, + {file = "pyrsistent-0.20.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ca52d1ceae015859d16aded12584c59eb3825f7b50c6cfd621d4231a6cc624ce"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b318ca24db0f0518630e8b6f3831e9cba78f099ed5c1d65ffe3e023003043ba0"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed2c3216a605dc9a6ea50c7e84c82906e3684c4e80d2908208f662a6cbf9022"}, + {file = "pyrsistent-0.20.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e14c95c16211d166f59c6611533d0dacce2e25de0f76e4c140fde250997b3ca"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win32.whl", hash = "sha256:f058a615031eea4ef94ead6456f5ec2026c19fb5bd6bfe86e9665c4158cf802f"}, + {file = "pyrsistent-0.20.0-cp39-cp39-win_amd64.whl", hash = "sha256:58b8f6366e152092194ae68fefe18b9f0b4f89227dfd86a07770c3d86097aebf"}, + {file = "pyrsistent-0.20.0-py3-none-any.whl", hash = "sha256:c55acc4733aad6560a7f5f818466631f07efc001fd023f34a6c203f8b6df0f0b"}, + {file = "pyrsistent-0.20.0.tar.gz", hash = "sha256:4c48f78f62ab596c679086084d0dd13254ae4f3d6c72a83ffdf5ebdef8f265a4"}, +] + +[[package]] +name = "pytest" +version = "7.4.4" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, + {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-docker" +version = "2.0.1" +description = "Simple pytest fixtures for Docker and Docker Compose based tests" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-docker-2.0.1.tar.gz", hash = "sha256:1c17e9202a566f85ed5ef269fe2815bd4899e90eb639622e5d14277372ca7524"}, + {file = "pytest_docker-2.0.1-py3-none-any.whl", hash = "sha256:7103f97b8c479c826b63d73cfb83383dc1970d35105ed1ce78a722c90c7fe650"}, +] + +[package.dependencies] +attrs = ">=19.2.0" +pytest = ">=4.0,<8.0" + +[package.extras] +docker-compose-v1 = ["docker-compose (>=1.27.3,<2.0)"] +tests = ["pytest-pycodestyle (>=2.0.0,<3.0)", "pytest-pylint (>=0.14.1,<1.0)", "requests (>=2.22.0,<3.0)"] + +[[package]] +name = "pytest-mypy" +version = "0.10.3" +description = "Mypy static type checker plugin for Pytest" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pytest-mypy-0.10.3.tar.gz", hash = "sha256:f8458f642323f13a2ca3e2e61509f7767966b527b4d8adccd5032c3e7b4fd3db"}, + {file = "pytest_mypy-0.10.3-py3-none-any.whl", hash = "sha256:7638d0d3906848fc1810cb2f5cc7fceb4cc5c98524aafcac58f28620e3102053"}, +] + +[package.dependencies] +attrs = ">=19.0" +filelock = ">=3.0" +mypy = [ + {version = ">=0.900", markers = "python_version >= \"3.11\""}, + {version = ">=0.780", markers = "python_version >= \"3.9\" and python_version < \"3.11\""}, +] +pytest = [ + {version = ">=6.2", markers = "python_version >= \"3.10\""}, + {version = ">=4.6", markers = "python_version >= \"3.6\" and python_version < \"3.10\""}, +] + +[[package]] +name = "pytest-postgresql" +version = "5.0.0" +description = "Postgresql fixtures and fixture factories for Pytest." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-postgresql-5.0.0.tar.gz", hash = "sha256:22edcbafab8995ee85b8d948ddfaad4f70c2c7462303d7477ecd2f77fc9d15bd"}, + {file = "pytest_postgresql-5.0.0-py3-none-any.whl", hash = "sha256:6e8f0773b57c9b8975b6392c241b7b81b7018f32079a533f368f2fbda732ecd3"}, +] + +[package.dependencies] +mirakuru = "*" +port-for = ">=0.6.0" +psycopg = ">=3.0.0" +pytest = ">=6.2" +setuptools = "*" + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "python-ulid" +version = "2.2.0" +description = "Universally unique lexicographically sortable identifier" +optional = false +python-versions = ">=3.9" +files = [ + {file = "python_ulid-2.2.0-py3-none-any.whl", hash = "sha256:ec2e69292c0b7c338a07df5e15b05270be6823675c103383e74d1d531945eab5"}, + {file = "python_ulid-2.2.0.tar.gz", hash = "sha256:9ec777177d396880d94be49ac7eb4ae2cd4a7474448bfdbfe911537add970aeb"}, +] + +[[package]] +name = "pytz" +version = "2023.3.post1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +files = [ + {file = "pytz-2023.3.post1-py2.py3-none-any.whl", hash = "sha256:ce42d816b81b68506614c11e8937d3aa9e41007ceb50bfdcb0749b921bf646c7"}, + {file = "pytz-2023.3.post1.tar.gz", hash = "sha256:7b4fddbeb94a1eba4b557da24f19fdf9db575192544270a9101d8509f9f43d7b"}, +] + +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + +[[package]] +name = "pywin32" +version = "306" +description = "Python for Window Extensions" +optional = false +python-versions = "*" +files = [ + {file = "pywin32-306-cp310-cp310-win32.whl", hash = "sha256:06d3420a5155ba65f0b72f2699b5bacf3109f36acbe8923765c22938a69dfc8d"}, + {file = "pywin32-306-cp310-cp310-win_amd64.whl", hash = "sha256:84f4471dbca1887ea3803d8848a1616429ac94a4a8d05f4bc9c5dcfd42ca99c8"}, + {file = "pywin32-306-cp311-cp311-win32.whl", hash = "sha256:e65028133d15b64d2ed8f06dd9fbc268352478d4f9289e69c190ecd6818b6407"}, + {file = "pywin32-306-cp311-cp311-win_amd64.whl", hash = "sha256:a7639f51c184c0272e93f244eb24dafca9b1855707d94c192d4a0b4c01e1100e"}, + {file = "pywin32-306-cp311-cp311-win_arm64.whl", hash = "sha256:70dba0c913d19f942a2db25217d9a1b726c278f483a919f1abfed79c9cf64d3a"}, + {file = "pywin32-306-cp312-cp312-win32.whl", hash = "sha256:383229d515657f4e3ed1343da8be101000562bf514591ff383ae940cad65458b"}, + {file = "pywin32-306-cp312-cp312-win_amd64.whl", hash = "sha256:37257794c1ad39ee9be652da0462dc2e394c8159dfd913a8a4e8eb6fd346da0e"}, + {file = "pywin32-306-cp312-cp312-win_arm64.whl", hash = "sha256:5821ec52f6d321aa59e2db7e0a35b997de60c201943557d108af9d4ae1ec7040"}, + {file = "pywin32-306-cp37-cp37m-win32.whl", hash = "sha256:1c73ea9a0d2283d889001998059f5eaaba3b6238f767c9cf2833b13e6a685f65"}, + {file = "pywin32-306-cp37-cp37m-win_amd64.whl", hash = "sha256:72c5f621542d7bdd4fdb716227be0dd3f8565c11b280be6315b06ace35487d36"}, + {file = "pywin32-306-cp38-cp38-win32.whl", hash = "sha256:e4c092e2589b5cf0d365849e73e02c391c1349958c5ac3e9d5ccb9a28e017b3a"}, + {file = "pywin32-306-cp38-cp38-win_amd64.whl", hash = "sha256:e8ac1ae3601bee6ca9f7cb4b5363bf1c0badb935ef243c4733ff9a393b1690c0"}, + {file = "pywin32-306-cp39-cp39-win32.whl", hash = "sha256:e25fd5b485b55ac9c057f67d94bc203f3f6595078d1fb3b458c9c28b7153a802"}, + {file = "pywin32-306-cp39-cp39-win_amd64.whl", hash = "sha256:39b61c15272833b5c329a2989999dcae836b1eed650252ab1b7bfbe1d59f30f4"}, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + +[[package]] +name = "referencing" +version = "0.32.1" +description = "JSON Referencing + Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "referencing-0.32.1-py3-none-any.whl", hash = "sha256:7e4dc12271d8e15612bfe35792f5ea1c40970dadf8624602e33db2758f7ee554"}, + {file = "referencing-0.32.1.tar.gz", hash = "sha256:3c57da0513e9563eb7e203ebe9bb3a1b509b042016433bd1e45a2853466c3dd3"}, +] + +[package.dependencies] +attrs = ">=22.2.0" +rpds-py = ">=0.7.0" + +[[package]] +name = "requests" +version = "2.31.0" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.7" +files = [ + {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"}, + {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "requests-cache" +version = "1.1.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.7,<4.0" +files = [ + {file = "requests_cache-1.1.1-py3-none-any.whl", hash = "sha256:c8420cf096f3aafde13c374979c21844752e2694ffd8710e6764685bb577ac90"}, + {file = "requests_cache-1.1.1.tar.gz", hash = "sha256:764f93d3fa860be72125a568c2cc8eafb151cf29b4dc2515433a56ee657e1c60"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=5.4)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.6)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=5.4)"] + +[[package]] +name = "rpds-py" +version = "0.17.1" +description = "Python bindings to Rust's persistent data structures (rpds)" +optional = false +python-versions = ">=3.8" +files = [ + {file = "rpds_py-0.17.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:4128980a14ed805e1b91a7ed551250282a8ddf8201a4e9f8f5b7e6225f54170d"}, + {file = "rpds_py-0.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ff1dcb8e8bc2261a088821b2595ef031c91d499a0c1b031c152d43fe0a6ecec8"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d65e6b4f1443048eb7e833c2accb4fa7ee67cc7d54f31b4f0555b474758bee55"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a71169d505af63bb4d20d23a8fbd4c6ce272e7bce6cc31f617152aa784436f29"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:436474f17733c7dca0fbf096d36ae65277e8645039df12a0fa52445ca494729d"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:10162fe3f5f47c37ebf6d8ff5a2368508fe22007e3077bf25b9c7d803454d921"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:720215373a280f78a1814becb1312d4e4d1077b1202a56d2b0815e95ccb99ce9"}, + {file = "rpds_py-0.17.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:70fcc6c2906cfa5c6a552ba7ae2ce64b6c32f437d8f3f8eea49925b278a61453"}, + {file = "rpds_py-0.17.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:91e5a8200e65aaac342a791272c564dffcf1281abd635d304d6c4e6b495f29dc"}, + {file = "rpds_py-0.17.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:99f567dae93e10be2daaa896e07513dd4bf9c2ecf0576e0533ac36ba3b1d5394"}, + {file = "rpds_py-0.17.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:24e4900a6643f87058a27320f81336d527ccfe503984528edde4bb660c8c8d59"}, + {file = "rpds_py-0.17.1-cp310-none-win32.whl", hash = "sha256:0bfb09bf41fe7c51413f563373e5f537eaa653d7adc4830399d4e9bdc199959d"}, + {file = "rpds_py-0.17.1-cp310-none-win_amd64.whl", hash = "sha256:20de7b7179e2031a04042e85dc463a93a82bc177eeba5ddd13ff746325558aa6"}, + {file = "rpds_py-0.17.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:65dcf105c1943cba45d19207ef51b8bc46d232a381e94dd38719d52d3980015b"}, + {file = "rpds_py-0.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:01f58a7306b64e0a4fe042047dd2b7d411ee82e54240284bab63e325762c1147"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:071bc28c589b86bc6351a339114fb7a029f5cddbaca34103aa573eba7b482382"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ae35e8e6801c5ab071b992cb2da958eee76340e6926ec693b5ff7d6381441745"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:149c5cd24f729e3567b56e1795f74577aa3126c14c11e457bec1b1c90d212e38"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e796051f2070f47230c745d0a77a91088fbee2cc0502e9b796b9c6471983718c"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60e820ee1004327609b28db8307acc27f5f2e9a0b185b2064c5f23e815f248f8"}, + {file = "rpds_py-0.17.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1957a2ab607f9added64478a6982742eb29f109d89d065fa44e01691a20fc20a"}, + {file = "rpds_py-0.17.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8587fd64c2a91c33cdc39d0cebdaf30e79491cc029a37fcd458ba863f8815383"}, + {file = "rpds_py-0.17.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4dc889a9d8a34758d0fcc9ac86adb97bab3fb7f0c4d29794357eb147536483fd"}, + {file = "rpds_py-0.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2953937f83820376b5979318840f3ee47477d94c17b940fe31d9458d79ae7eea"}, + {file = "rpds_py-0.17.1-cp311-none-win32.whl", hash = "sha256:1bfcad3109c1e5ba3cbe2f421614e70439f72897515a96c462ea657261b96518"}, + {file = "rpds_py-0.17.1-cp311-none-win_amd64.whl", hash = "sha256:99da0a4686ada4ed0f778120a0ea8d066de1a0a92ab0d13ae68492a437db78bf"}, + {file = "rpds_py-0.17.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:1dc29db3900cb1bb40353772417800f29c3d078dbc8024fd64655a04ee3c4bdf"}, + {file = "rpds_py-0.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:82ada4a8ed9e82e443fcef87e22a3eed3654dd3adf6e3b3a0deb70f03e86142a"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d36b2b59e8cc6e576f8f7b671e32f2ff43153f0ad6d0201250a7c07f25d570e"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3677fcca7fb728c86a78660c7fb1b07b69b281964673f486ae72860e13f512ad"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:516fb8c77805159e97a689e2f1c80655c7658f5af601c34ffdb916605598cda2"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df3b6f45ba4515632c5064e35ca7f31d51d13d1479673185ba8f9fefbbed58b9"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a967dd6afda7715d911c25a6ba1517975acd8d1092b2f326718725461a3d33f9"}, + {file = "rpds_py-0.17.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dbbb95e6fc91ea3102505d111b327004d1c4ce98d56a4a02e82cd451f9f57140"}, + {file = "rpds_py-0.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:02866e060219514940342a1f84303a1ef7a1dad0ac311792fbbe19b521b489d2"}, + {file = "rpds_py-0.17.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:2528ff96d09f12e638695f3a2e0c609c7b84c6df7c5ae9bfeb9252b6fa686253"}, + {file = "rpds_py-0.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bd345a13ce06e94c753dab52f8e71e5252aec1e4f8022d24d56decd31e1b9b23"}, + {file = "rpds_py-0.17.1-cp312-none-win32.whl", hash = "sha256:2a792b2e1d3038daa83fa474d559acfd6dc1e3650ee93b2662ddc17dbff20ad1"}, + {file = "rpds_py-0.17.1-cp312-none-win_amd64.whl", hash = "sha256:292f7344a3301802e7c25c53792fae7d1593cb0e50964e7bcdcc5cf533d634e3"}, + {file = "rpds_py-0.17.1-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:8ffe53e1d8ef2520ebcf0c9fec15bb721da59e8ef283b6ff3079613b1e30513d"}, + {file = "rpds_py-0.17.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4341bd7579611cf50e7b20bb8c2e23512a3dc79de987a1f411cb458ab670eb90"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2f4eb548daf4836e3b2c662033bfbfc551db58d30fd8fe660314f86bf8510b93"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b686f25377f9c006acbac63f61614416a6317133ab7fafe5de5f7dc8a06d42eb"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4e21b76075c01d65d0f0f34302b5a7457d95721d5e0667aea65e5bb3ab415c25"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b86b21b348f7e5485fae740d845c65a880f5d1eda1e063bc59bef92d1f7d0c55"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f175e95a197f6a4059b50757a3dca33b32b61691bdbd22c29e8a8d21d3914cae"}, + {file = "rpds_py-0.17.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1701fc54460ae2e5efc1dd6350eafd7a760f516df8dbe51d4a1c79d69472fbd4"}, + {file = "rpds_py-0.17.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:9051e3d2af8f55b42061603e29e744724cb5f65b128a491446cc029b3e2ea896"}, + {file = "rpds_py-0.17.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:7450dbd659fed6dd41d1a7d47ed767e893ba402af8ae664c157c255ec6067fde"}, + {file = "rpds_py-0.17.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:5a024fa96d541fd7edaa0e9d904601c6445e95a729a2900c5aec6555fe921ed6"}, + {file = "rpds_py-0.17.1-cp38-none-win32.whl", hash = "sha256:da1ead63368c04a9bded7904757dfcae01eba0e0f9bc41d3d7f57ebf1c04015a"}, + {file = "rpds_py-0.17.1-cp38-none-win_amd64.whl", hash = "sha256:841320e1841bb53fada91c9725e766bb25009cfd4144e92298db296fb6c894fb"}, + {file = "rpds_py-0.17.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:f6c43b6f97209e370124baf2bf40bb1e8edc25311a158867eb1c3a5d449ebc7a"}, + {file = "rpds_py-0.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5e7d63ec01fe7c76c2dbb7e972fece45acbb8836e72682bde138e7e039906e2c"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81038ff87a4e04c22e1d81f947c6ac46f122e0c80460b9006e6517c4d842a6ec"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:810685321f4a304b2b55577c915bece4c4a06dfe38f6e62d9cc1d6ca8ee86b99"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:25f071737dae674ca8937a73d0f43f5a52e92c2d178330b4c0bb6ab05586ffa6"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aa5bfb13f1e89151ade0eb812f7b0d7a4d643406caaad65ce1cbabe0a66d695f"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dfe07308b311a8293a0d5ef4e61411c5c20f682db6b5e73de6c7c8824272c256"}, + {file = "rpds_py-0.17.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a000133a90eea274a6f28adc3084643263b1e7c1a5a66eb0a0a7a36aa757ed74"}, + {file = "rpds_py-0.17.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5d0e8a6434a3fbf77d11448c9c25b2f25244226cfbec1a5159947cac5b8c5fa4"}, + {file = "rpds_py-0.17.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:efa767c220d94aa4ac3a6dd3aeb986e9f229eaf5bce92d8b1b3018d06bed3772"}, + {file = "rpds_py-0.17.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:dbc56680ecf585a384fbd93cd42bc82668b77cb525343170a2d86dafaed2a84b"}, + {file = "rpds_py-0.17.1-cp39-none-win32.whl", hash = "sha256:270987bc22e7e5a962b1094953ae901395e8c1e1e83ad016c5cfcfff75a15a3f"}, + {file = "rpds_py-0.17.1-cp39-none-win_amd64.whl", hash = "sha256:2a7b2f2f56a16a6d62e55354dd329d929560442bd92e87397b7a9586a32e3e76"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:a3264e3e858de4fc601741498215835ff324ff2482fd4e4af61b46512dd7fc83"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:f2f3b28b40fddcb6c1f1f6c88c6f3769cd933fa493ceb79da45968a21dccc920"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9584f8f52010295a4a417221861df9bea4c72d9632562b6e59b3c7b87a1522b7"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c64602e8be701c6cfe42064b71c84ce62ce66ddc6422c15463fd8127db3d8066"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:060f412230d5f19fc8c8b75f315931b408d8ebf56aec33ef4168d1b9e54200b1"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b9412abdf0ba70faa6e2ee6c0cc62a8defb772e78860cef419865917d86c7342"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9737bdaa0ad33d34c0efc718741abaafce62fadae72c8b251df9b0c823c63b22"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9f0e4dc0f17dcea4ab9d13ac5c666b6b5337042b4d8f27e01b70fae41dd65c57"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:1db228102ab9d1ff4c64148c96320d0be7044fa28bd865a9ce628ce98da5973d"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-musllinux_1_2_i686.whl", hash = "sha256:d8bbd8e56f3ba25a7d0cf980fc42b34028848a53a0e36c9918550e0280b9d0b6"}, + {file = "rpds_py-0.17.1-pp310-pypy310_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:be22ae34d68544df293152b7e50895ba70d2a833ad9566932d750d3625918b82"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:bf046179d011e6114daf12a534d874958b039342b347348a78b7cdf0dd9d6041"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:1a746a6d49665058a5896000e8d9d2f1a6acba8a03b389c1e4c06e11e0b7f40d"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0b8bf5b8db49d8fd40f54772a1dcf262e8be0ad2ab0206b5a2ec109c176c0a4"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f7f4cb1f173385e8a39c29510dd11a78bf44e360fb75610594973f5ea141028b"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7fbd70cb8b54fe745301921b0816c08b6d917593429dfc437fd024b5ba713c58"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9bdf1303df671179eaf2cb41e8515a07fc78d9d00f111eadbe3e14262f59c3d0"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fad059a4bd14c45776600d223ec194e77db6c20255578bb5bcdd7c18fd169361"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3664d126d3388a887db44c2e293f87d500c4184ec43d5d14d2d2babdb4c64cad"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:698ea95a60c8b16b58be9d854c9f993c639f5c214cf9ba782eca53a8789d6b19"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-musllinux_1_2_i686.whl", hash = "sha256:c3d2010656999b63e628a3c694f23020322b4178c450dc478558a2b6ef3cb9bb"}, + {file = "rpds_py-0.17.1-pp38-pypy38_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:938eab7323a736533f015e6069a7d53ef2dcc841e4e533b782c2bfb9fb12d84b"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:1e626b365293a2142a62b9a614e1f8e331b28f3ca57b9f05ebbf4cf2a0f0bdc5"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:380e0df2e9d5d5d339803cfc6d183a5442ad7ab3c63c2a0982e8c824566c5ccc"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b760a56e080a826c2e5af09002c1a037382ed21d03134eb6294812dda268c811"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5576ee2f3a309d2bb403ec292d5958ce03953b0e57a11d224c1f134feaf8c40f"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f3c3461ebb4c4f1bbc70b15d20b565759f97a5aaf13af811fcefc892e9197ba"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:637b802f3f069a64436d432117a7e58fab414b4e27a7e81049817ae94de45d8d"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffee088ea9b593cc6160518ba9bd319b5475e5f3e578e4552d63818773c6f56a"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3ac732390d529d8469b831949c78085b034bff67f584559340008d0f6041a049"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:93432e747fb07fa567ad9cc7aaadd6e29710e515aabf939dfbed8046041346c6"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-musllinux_1_2_i686.whl", hash = "sha256:7b7d9ca34542099b4e185b3c2a2b2eda2e318a7dbde0b0d83357a6d4421b5296"}, + {file = "rpds_py-0.17.1-pp39-pypy39_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:0387ce69ba06e43df54e43968090f3626e231e4bc9150e4c3246947567695f68"}, + {file = "rpds_py-0.17.1.tar.gz", hash = "sha256:0210b2668f24c078307260bf88bdac9d6f1093635df5123789bfee4d8d7fc8e7"}, +] + +[[package]] +name = "rsa" +version = "4.9" +description = "Pure-Python RSA implementation" +optional = false +python-versions = ">=3.6,<4" +files = [ + {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"}, + {file = "rsa-4.9.tar.gz", hash = "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"}, +] + +[package.dependencies] +pyasn1 = ">=0.1.3" + +[[package]] +name = "ruff" +version = "0.1.13" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.1.13-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:e3fd36e0d48aeac672aa850045e784673449ce619afc12823ea7868fcc41d8ba"}, + {file = "ruff-0.1.13-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9fb6b3b86450d4ec6a6732f9f60c4406061b6851c4b29f944f8c9d91c3611c7a"}, + {file = "ruff-0.1.13-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b13ba5d7156daaf3fd08b6b993360a96060500aca7e307d95ecbc5bb47a69296"}, + {file = "ruff-0.1.13-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9ebb40442f7b531e136d334ef0851412410061e65d61ca8ce90d894a094feb22"}, + {file = "ruff-0.1.13-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:226b517f42d59a543d6383cfe03cccf0091e3e0ed1b856c6824be03d2a75d3b6"}, + {file = "ruff-0.1.13-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:5f0312ba1061e9b8c724e9a702d3c8621e3c6e6c2c9bd862550ab2951ac75c16"}, + {file = "ruff-0.1.13-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2f59bcf5217c661254bd6bc42d65a6fd1a8b80c48763cb5c2293295babd945dd"}, + {file = "ruff-0.1.13-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e6894b00495e00c27b6ba61af1fc666f17de6140345e5ef27dd6e08fb987259d"}, + {file = "ruff-0.1.13-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a1600942485c6e66119da294c6294856b5c86fd6df591ce293e4a4cc8e72989"}, + {file = "ruff-0.1.13-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:ee3febce7863e231a467f90e681d3d89210b900d49ce88723ce052c8761be8c7"}, + {file = "ruff-0.1.13-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:dcaab50e278ff497ee4d1fe69b29ca0a9a47cd954bb17963628fa417933c6eb1"}, + {file = "ruff-0.1.13-py3-none-musllinux_1_2_i686.whl", hash = "sha256:f57de973de4edef3ad3044d6a50c02ad9fc2dff0d88587f25f1a48e3f72edf5e"}, + {file = "ruff-0.1.13-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:7a36fa90eb12208272a858475ec43ac811ac37e91ef868759770b71bdabe27b6"}, + {file = "ruff-0.1.13-py3-none-win32.whl", hash = "sha256:a623349a505ff768dad6bd57087e2461be8db58305ebd5577bd0e98631f9ae69"}, + {file = "ruff-0.1.13-py3-none-win_amd64.whl", hash = "sha256:f988746e3c3982bea7f824c8fa318ce7f538c4dfefec99cd09c8770bd33e6539"}, + {file = "ruff-0.1.13-py3-none-win_arm64.whl", hash = "sha256:6bbbc3042075871ec17f28864808540a26f0f79a4478c357d3e3d2284e832998"}, + {file = "ruff-0.1.13.tar.gz", hash = "sha256:e261f1baed6291f434ffb1d5c6bd8051d1c2a26958072d38dfbec39b3dda7352"}, +] + +[[package]] +name = "setuptools" +version = "69.0.3" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-69.0.3-py3-none-any.whl", hash = "sha256:385eb4edd9c9d5c17540511303e39a147ce2fc04bc55289c322b9e5904fe2c05"}, + {file = "setuptools-69.0.3.tar.gz", hash = "sha256:be1af57fc409f93647f2e8e4573a142ed38724b8cdd389706a867bb4efcf1e78"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + +[[package]] +name = "snowflake-connector-python" +version = "3.6.0" +description = "Snowflake Connector for Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "snowflake-connector-python-3.6.0.tar.gz", hash = "sha256:15667a918780d79da755e6a60bbf6918051854951e8f56ccdf5692283e9a8479"}, + {file = "snowflake_connector_python-3.6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4093b38cf9abf95c38119f0b23b07e23dc7a8689b956cd5d34975e1875741f20"}, + {file = "snowflake_connector_python-3.6.0-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:cf5a964fe01b177063f8c44d14df3a72715580bcd195788ec2822090f37330a5"}, + {file = "snowflake_connector_python-3.6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55a6418cec585b050e6f05404f25e62b075a3bbea587dc1f903de15640565c58"}, + {file = "snowflake_connector_python-3.6.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7c76aea92b87f6ecd604e9c934aac8a779f2e20f3be1d990d53bb5b6d87b009"}, + {file = "snowflake_connector_python-3.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:9dfcf178271e892e64e4092b9e011239a066ce5de848afd2efe3f13197a9f8b3"}, + {file = "snowflake_connector_python-3.6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4916f9b4a0efd7c96d1fa50a157e05907b6935f91492cca7f200b43cc178a25e"}, + {file = "snowflake_connector_python-3.6.0-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:f15024c66db5e87d359216ec733a2974d7562aa38f3f18c8b6e65489839e00d7"}, + {file = "snowflake_connector_python-3.6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcbd3102f807ebbbae52b1b5683d45cd7b3dcb0eaec131233ba6b156e8d70fa4"}, + {file = "snowflake_connector_python-3.6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7662e2de25b885abe08ab866cf7c7b026ad1af9faa39c25e2c25015ef807abe3"}, + {file = "snowflake_connector_python-3.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:d1fa102f55ee166cc766aeee3f9333b17b4bede6fb088eee1e1f022df15b6d81"}, + {file = "snowflake_connector_python-3.6.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:fde1e0727e2f23c2a07b49b30e1bc0f49977f965d08ddfda10015b24a2beeb76"}, + {file = "snowflake_connector_python-3.6.0-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:1b51fe000c8cf6372d30b73c7136275e52788e6af47010cd1984c9fb03378e86"}, + {file = "snowflake_connector_python-3.6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7a11699689a19916e65794ce58dca72b8a40fe6a7eea06764931ede10b47bcc"}, + {file = "snowflake_connector_python-3.6.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d810be5b180c6f47ce9b6f989fe64b9984383e4b77e30b284a83e33f229a3a82"}, + {file = "snowflake_connector_python-3.6.0-cp38-cp38-win_amd64.whl", hash = "sha256:b5db47d4164d6b7a07c413a46f9edc4a1d687e3df44fd9d5fa89a89aecb94a8e"}, + {file = "snowflake_connector_python-3.6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:bf8c1ad5aab5304fefa2a4178061a24c96da45e3e3db9d901621e9953e005402"}, + {file = "snowflake_connector_python-3.6.0-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:1058ab5c98cc62fde8b3f021f0a5076cb7865b5cdab8a9bccde0df88b9e91334"}, + {file = "snowflake_connector_python-3.6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2b93f55989f80d69278e0f40a7a1c0e737806b7c0ddb0351513a752b837243e8"}, + {file = "snowflake_connector_python-3.6.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50dd954ea5918d3242ded69225b72f701963cd9c043ee7d9ab35dc22211611c8"}, + {file = "snowflake_connector_python-3.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:4ad42613b87f31441d07a8ea242f4c28ed5eb7b6e05986f9e94a7e44b96d3d1e"}, +] + +[package.dependencies] +asn1crypto = ">0.24.0,<2.0.0" +certifi = ">=2017.4.17" +cffi = ">=1.9,<2.0.0" +charset-normalizer = ">=2,<4" +cryptography = ">=3.1.0,<42.0.0" +filelock = ">=3.5,<4" +idna = ">=2.5,<4" +packaging = "*" +platformdirs = ">=2.6.0,<4.0.0" +pyjwt = "<3.0.0" +pyOpenSSL = ">=16.2.0,<24.0.0" +pytz = "*" +requests = "<3.0.0" +sortedcontainers = ">=2.4.0" +tomlkit = "*" +typing-extensions = ">=4.3,<5" +urllib3 = {version = ">=1.21.1,<2.0.0", markers = "python_version < \"3.10\""} + +[package.extras] +development = ["Cython", "coverage", "more-itertools", "numpy (<1.27.0)", "pendulum (!=2.1.1)", "pexpect", "pytest (<7.5.0)", "pytest-cov", "pytest-rerunfailures", "pytest-timeout", "pytest-xdist", "pytzdata"] +pandas = ["pandas (>=1.0.0,<2.2.0)", "pyarrow"] +secure-local-storage = ["keyring (!=16.1.0,<25.0.0)"] + +[[package]] +name = "snowflake-sqlalchemy" +version = "1.5.1" +description = "Snowflake SQLAlchemy Dialect" +optional = false +python-versions = ">=3.7" +files = [ + {file = "snowflake-sqlalchemy-1.5.1.tar.gz", hash = "sha256:4f1383402ffc89311974bd810dee22003aef4af0f312a0fdb55778333ad1abf7"}, + {file = "snowflake_sqlalchemy-1.5.1-py2.py3-none-any.whl", hash = "sha256:df022fb73bc04d68dfb3216ebf7a1bfbd14d22def9c38bbe05275beb258adcd0"}, +] + +[package.dependencies] +snowflake-connector-python = "<4.0.0" +sqlalchemy = ">=1.4.0,<2.0.0" + +[package.extras] +development = ["mock", "numpy", "pytest", "pytest-cov", "pytest-rerunfailures", "pytest-timeout", "pytz"] +pandas = ["snowflake-connector-python[pandas] (<4.0.0)"] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" +optional = false +python-versions = "*" +files = [ + {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"}, + {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, +] + +[[package]] +name = "sqlalchemy" +version = "1.4.51" +description = "Database Abstraction Library" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +files = [ + {file = "SQLAlchemy-1.4.51-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:1a09d5bd1a40d76ad90e5570530e082ddc000e1d92de495746f6257dc08f166b"}, + {file = "SQLAlchemy-1.4.51-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2be4e6294c53f2ec8ea36486b56390e3bcaa052bf3a9a47005687ccf376745d1"}, + {file = "SQLAlchemy-1.4.51-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ca484ca11c65e05639ffe80f20d45e6be81fbec7683d6c9a15cd421e6e8b340"}, + {file = "SQLAlchemy-1.4.51-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0535d5b57d014d06ceeaeffd816bb3a6e2dddeb670222570b8c4953e2d2ea678"}, + {file = "SQLAlchemy-1.4.51-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af55cc207865d641a57f7044e98b08b09220da3d1b13a46f26487cc2f898a072"}, + {file = "SQLAlchemy-1.4.51-cp310-cp310-win32.whl", hash = "sha256:7af40425ac535cbda129d9915edcaa002afe35d84609fd3b9d6a8c46732e02ee"}, + {file = "SQLAlchemy-1.4.51-cp310-cp310-win_amd64.whl", hash = "sha256:8d1d7d63e5d2f4e92a39ae1e897a5d551720179bb8d1254883e7113d3826d43c"}, + {file = "SQLAlchemy-1.4.51-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:eaeeb2464019765bc4340214fca1143081d49972864773f3f1e95dba5c7edc7d"}, + {file = "SQLAlchemy-1.4.51-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7deeae5071930abb3669b5185abb6c33ddfd2398f87660fafdb9e6a5fb0f3f2f"}, + {file = "SQLAlchemy-1.4.51-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0892e7ac8bc76da499ad3ee8de8da4d7905a3110b952e2a35a940dab1ffa550e"}, + {file = "SQLAlchemy-1.4.51-cp311-cp311-win32.whl", hash = "sha256:50e074aea505f4427151c286955ea025f51752fa42f9939749336672e0674c81"}, + {file = "SQLAlchemy-1.4.51-cp311-cp311-win_amd64.whl", hash = "sha256:3b0cd89a7bd03f57ae58263d0f828a072d1b440c8c2949f38f3b446148321171"}, + {file = "SQLAlchemy-1.4.51-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:a33cb3f095e7d776ec76e79d92d83117438b6153510770fcd57b9c96f9ef623d"}, + {file = "SQLAlchemy-1.4.51-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6cacc0b2dd7d22a918a9642fc89840a5d3cee18a0e1fe41080b1141b23b10916"}, + {file = "SQLAlchemy-1.4.51-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:245c67c88e63f1523e9216cad6ba3107dea2d3ee19adc359597a628afcabfbcb"}, + {file = "SQLAlchemy-1.4.51-cp312-cp312-win32.whl", hash = "sha256:8e702e7489f39375601c7ea5a0bef207256828a2bc5986c65cb15cd0cf097a87"}, + {file = "SQLAlchemy-1.4.51-cp312-cp312-win_amd64.whl", hash = "sha256:0525c4905b4b52d8ccc3c203c9d7ab2a80329ffa077d4bacf31aefda7604dc65"}, + {file = "SQLAlchemy-1.4.51-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:1980e6eb6c9be49ea8f89889989127daafc43f0b1b6843d71efab1514973cca0"}, + {file = "SQLAlchemy-1.4.51-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ec7a0ed9b32afdf337172678a4a0e6419775ba4e649b66f49415615fa47efbd"}, + {file = "SQLAlchemy-1.4.51-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:352df882088a55293f621328ec33b6ffca936ad7f23013b22520542e1ab6ad1b"}, + {file = "SQLAlchemy-1.4.51-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:86a22143a4001f53bf58027b044da1fb10d67b62a785fc1390b5c7f089d9838c"}, + {file = "SQLAlchemy-1.4.51-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c37bc677690fd33932182b85d37433845de612962ed080c3e4d92f758d1bd894"}, + {file = "SQLAlchemy-1.4.51-cp36-cp36m-win32.whl", hash = "sha256:d0a83afab5e062abffcdcbcc74f9d3ba37b2385294dd0927ad65fc6ebe04e054"}, + {file = "SQLAlchemy-1.4.51-cp36-cp36m-win_amd64.whl", hash = "sha256:a61184c7289146c8cff06b6b41807c6994c6d437278e72cf00ff7fe1c7a263d1"}, + {file = "SQLAlchemy-1.4.51-cp37-cp37m-macosx_11_0_x86_64.whl", hash = "sha256:3f0ef620ecbab46e81035cf3dedfb412a7da35340500ba470f9ce43a1e6c423b"}, + {file = "SQLAlchemy-1.4.51-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c55040d8ea65414de7c47f1a23823cd9f3fad0dc93e6b6b728fee81230f817b"}, + {file = "SQLAlchemy-1.4.51-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:38ef80328e3fee2be0a1abe3fe9445d3a2e52a1282ba342d0dab6edf1fef4707"}, + {file = "SQLAlchemy-1.4.51-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f8cafa6f885a0ff5e39efa9325195217bb47d5929ab0051636610d24aef45ade"}, + {file = "SQLAlchemy-1.4.51-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e8f2df79a46e130235bc5e1bbef4de0583fb19d481eaa0bffa76e8347ea45ec6"}, + {file = "SQLAlchemy-1.4.51-cp37-cp37m-win32.whl", hash = "sha256:f2e5b6f5cf7c18df66d082604a1d9c7a2d18f7d1dbe9514a2afaccbb51cc4fc3"}, + {file = "SQLAlchemy-1.4.51-cp37-cp37m-win_amd64.whl", hash = "sha256:5e180fff133d21a800c4f050733d59340f40d42364fcb9d14f6a67764bdc48d2"}, + {file = "SQLAlchemy-1.4.51-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:7d8139ca0b9f93890ab899da678816518af74312bb8cd71fb721436a93a93298"}, + {file = "SQLAlchemy-1.4.51-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb18549b770351b54e1ab5da37d22bc530b8bfe2ee31e22b9ebe650640d2ef12"}, + {file = "SQLAlchemy-1.4.51-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55e699466106d09f028ab78d3c2e1f621b5ef2c8694598242259e4515715da7c"}, + {file = "SQLAlchemy-1.4.51-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2ad16880ccd971ac8e570550fbdef1385e094b022d6fc85ef3ce7df400dddad3"}, + {file = "SQLAlchemy-1.4.51-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b97fd5bb6b7c1a64b7ac0632f7ce389b8ab362e7bd5f60654c2a418496be5d7f"}, + {file = "SQLAlchemy-1.4.51-cp38-cp38-win32.whl", hash = "sha256:cecb66492440ae8592797dd705a0cbaa6abe0555f4fa6c5f40b078bd2740fc6b"}, + {file = "SQLAlchemy-1.4.51-cp38-cp38-win_amd64.whl", hash = "sha256:39b02b645632c5fe46b8dd30755682f629ffbb62ff317ecc14c998c21b2896ff"}, + {file = "SQLAlchemy-1.4.51-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:b03850c290c765b87102959ea53299dc9addf76ca08a06ea98383348ae205c99"}, + {file = "SQLAlchemy-1.4.51-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e646b19f47d655261b22df9976e572f588185279970efba3d45c377127d35349"}, + {file = "SQLAlchemy-1.4.51-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3cf56cc36d42908495760b223ca9c2c0f9f0002b4eddc994b24db5fcb86a9e4"}, + {file = "SQLAlchemy-1.4.51-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:0d661cff58c91726c601cc0ee626bf167b20cc4d7941c93c5f3ac28dc34ddbea"}, + {file = "SQLAlchemy-1.4.51-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3823dda635988e6744d4417e13f2e2b5fe76c4bf29dd67e95f98717e1b094cad"}, + {file = "SQLAlchemy-1.4.51-cp39-cp39-win32.whl", hash = "sha256:b00cf0471888823b7a9f722c6c41eb6985cf34f077edcf62695ac4bed6ec01ee"}, + {file = "SQLAlchemy-1.4.51-cp39-cp39-win_amd64.whl", hash = "sha256:a055ba17f4675aadcda3005df2e28a86feb731fdcc865e1f6b4f209ed1225cba"}, + {file = "SQLAlchemy-1.4.51.tar.gz", hash = "sha256:e7908c2025eb18394e32d65dd02d2e37e17d733cdbe7d78231c2b6d7eb20cdb9"}, +] + +[package.dependencies] +greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} + +[package.extras] +aiomysql = ["aiomysql (>=0.2.0)", "greenlet (!=0.4.17)"] +aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] +asyncio = ["greenlet (!=0.4.17)"] +asyncmy = ["asyncmy (>=0.2.3,!=0.2.4)", "greenlet (!=0.4.17)"] +mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2)"] +mssql = ["pyodbc"] +mssql-pymssql = ["pymssql"] +mssql-pyodbc = ["pyodbc"] +mypy = ["mypy (>=0.910)", "sqlalchemy2-stubs"] +mysql = ["mysqlclient (>=1.4.0)", "mysqlclient (>=1.4.0,<2)"] +mysql-connector = ["mysql-connector-python"] +oracle = ["cx_oracle (>=7)", "cx_oracle (>=7,<8)"] +postgresql = ["psycopg2 (>=2.7)"] +postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] +postgresql-pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)"] +postgresql-psycopg2binary = ["psycopg2-binary"] +postgresql-psycopg2cffi = ["psycopg2cffi"] +pymysql = ["pymysql", "pymysql (<1)"] +sqlcipher = ["sqlcipher3_binary"] + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] + +[[package]] +name = "tomlkit" +version = "0.12.3" +description = "Style preserving TOML library" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomlkit-0.12.3-py3-none-any.whl", hash = "sha256:b0a645a9156dc7cb5d3a1f0d4bab66db287fcb8e0430bdd4664a095ea16414ba"}, + {file = "tomlkit-0.12.3.tar.gz", hash = "sha256:75baf5012d06501f07bee5bf8e801b9f343e7aac5a92581f20f80ce632e6b5a4"}, +] + +[[package]] +name = "types-jsonschema" +version = "4.21.0.20240118" +description = "Typing stubs for jsonschema" +optional = false +python-versions = ">=3.8" +files = [ + {file = "types-jsonschema-4.21.0.20240118.tar.gz", hash = "sha256:31aae1b5adc0176c1155c2d4f58348b22d92ae64315e9cc83bd6902168839232"}, + {file = "types_jsonschema-4.21.0.20240118-py3-none-any.whl", hash = "sha256:77a4ac36b0be4f24274d5b9bf0b66208ee771c05f80e34c4641de7d63e8a872d"}, +] + +[package.dependencies] +referencing = "*" + +[[package]] +name = "types-pytz" +version = "2023.3.1.1" +description = "Typing stubs for pytz" +optional = false +python-versions = "*" +files = [ + {file = "types-pytz-2023.3.1.1.tar.gz", hash = "sha256:cc23d0192cd49c8f6bba44ee0c81e4586a8f30204970fc0894d209a6b08dab9a"}, + {file = "types_pytz-2023.3.1.1-py3-none-any.whl", hash = "sha256:1999a123a3dc0e39a2ef6d19f3f8584211de9e6a77fe7a0259f04a524e90a5cf"}, +] + +[[package]] +name = "types-pyyaml" +version = "6.0.12.12" +description = "Typing stubs for PyYAML" +optional = false +python-versions = "*" +files = [ + {file = "types-PyYAML-6.0.12.12.tar.gz", hash = "sha256:334373d392fde0fdf95af5c3f1661885fa10c52167b14593eb856289e1855062"}, + {file = "types_PyYAML-6.0.12.12-py3-none-any.whl", hash = "sha256:c05bc6c158facb0676674b7f11fe3960db4f389718e19e62bd2b84d6205cfd24"}, +] + +[[package]] +name = "types-requests" +version = "2.31.0.4" +description = "Typing stubs for requests" +optional = false +python-versions = "*" +files = [ + {file = "types-requests-2.31.0.4.tar.gz", hash = "sha256:a111041148d7e04bf100c476bc4db3ee6b0a1cd0b4018777f6a660b1c4f1318d"}, + {file = "types_requests-2.31.0.4-py3-none-any.whl", hash = "sha256:c7a9d6b62776f21b169a94a0e9d2dfcae62fa9149f53594ff791c3ae67325490"}, +] + +[package.dependencies] +types-urllib3 = "*" + +[[package]] +name = "types-urllib3" +version = "1.26.25.14" +description = "Typing stubs for urllib3" +optional = false +python-versions = "*" +files = [ + {file = "types-urllib3-1.26.25.14.tar.gz", hash = "sha256:229b7f577c951b8c1b92c1bc2b2fdb0b49847bd2af6d1cc2a2e3dd340f3bda8f"}, + {file = "types_urllib3-1.26.25.14-py3-none-any.whl", hash = "sha256:9683bbb7fb72e32bfe9d2be6e04875fbe1b3eeec3cbb4ea231435aa7fd6b4f0e"}, +] + +[[package]] +name = "typing-extensions" +version = "4.9.0" +description = "Backported and Experimental Type Hints for Python 3.8+" +optional = false +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, +] + +[[package]] +name = "tzdata" +version = "2023.4" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2023.4-py2.py3-none-any.whl", hash = "sha256:aa3ace4329eeacda5b7beb7ea08ece826c28d761cda36e747cfbf97996d39bf3"}, + {file = "tzdata-2023.4.tar.gz", hash = "sha256:dd54c94f294765522c77399649b4fefd95522479a664a0cec87f41bebc6148c9"}, +] + +[[package]] +name = "ulid" +version = "1.1" +description = "Pyhton version of this: https://github.com/alizain/ulid" +optional = false +python-versions = "*" +files = [ + {file = "ulid-1.1.tar.gz", hash = "sha256:0943e8a751ec10dfcdb4df2758f96dffbbfbc055d0b49288caf2f92125900d49"}, +] + +[[package]] +name = "url-normalize" +version = "1.4.3" +description = "URL normalization for Python" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "url-normalize-1.4.3.tar.gz", hash = "sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2"}, + {file = "url_normalize-1.4.3-py2.py3-none-any.whl", hash = "sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "urllib3" +version = "1.26.18" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +files = [ + {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, + {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"}, +] + +[package.extras] +brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "wcmatch" +version = "8.4" +description = "Wildcard/glob file name matcher." +optional = false +python-versions = ">=3.7" +files = [ + {file = "wcmatch-8.4-py3-none-any.whl", hash = "sha256:dc7351e5a7f8bbf4c6828d51ad20c1770113f5f3fd3dfe2a03cfde2a63f03f98"}, + {file = "wcmatch-8.4.tar.gz", hash = "sha256:ba4fc5558f8946bf1ffc7034b05b814d825d694112499c86035e0e4d398b6a67"}, +] + +[package.dependencies] +bracex = ">=2.1.1" + +[[package]] +name = "wrapt" +version = "1.16.0" +description = "Module for decorators, wrappers and monkey patching." +optional = false +python-versions = ">=3.6" +files = [ + {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, + {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, + {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, + {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, + {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, + {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, + {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, + {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, + {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, + {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, + {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, + {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, + {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, + {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, + {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, + {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, + {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, + {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, + {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, + {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, + {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, + {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, + {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, + {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, + {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, + {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, + {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, + {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, + {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, + {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, + {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, + {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, + {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, + {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, + {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, + {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, + {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9" +content-hash = "5457f59d89d371fbdacd1a5ba132de627ad4f7b6bd2c073fe75b54d90d216e5f" diff --git a/airbyte-lib/pyproject.toml b/airbyte-lib/pyproject.toml new file mode 100644 index 000000000000..a1aea3d19bd4 --- /dev/null +++ b/airbyte-lib/pyproject.toml @@ -0,0 +1,237 @@ +[tool.poetry] +name = "airbyte-lib" +description = "AirbyteLib" +version = "0.1.0" +authors = ["Airbyte "] +readme = "README.md" +packages = [{include = "airbyte_lib"}] + +[tool.poetry.dependencies] +python = "^3.9" + +airbyte-cdk = "^0.58.3" +# airbyte-protocol-models = "^1.0.1" # Conflicts with airbyte-cdk # TODO: delete or resolve +jsonschema = "3.2.0" +orjson = "^3.9.10" +overrides = "^7.4.0" +pandas = "^2.1.4" +psycopg = {extras = ["binary", "pool"], version = "^3.1.16"} +python-ulid = "^2.2.0" +types-pyyaml = "^6.0.12.12" +ulid = "^1.1" +sqlalchemy = "1.4.51" +snowflake-connector-python = "3.6.0" +snowflake-sqlalchemy = "^1.5.1" +duckdb-engine = "^0.10.0" +requests = "^2.31.0" +pyarrow = "^14.0.2" +psycopg2 = "^2.9.9" + +[tool.poetry.group.dev.dependencies] +docker = "^7.0.0" +faker = "^21.0.0" +mypy = "^1.7.1" +pandas-stubs = "^2.1.4.231218" +pdoc = "^14.3.0" +pyarrow-stubs = "^10.0.1.7" +pytest = "^7.4.3" +pytest-docker = "^2.0.1" +pytest-mypy = "^0.10.3" +pytest-postgresql = "^5.0.0" +ruff = "^0.1.11" +types-jsonschema = "^4.20.0.0" +google-cloud-secret-manager = "^2.17.0" +types-requests = "2.31.0.4" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.pytest.ini_options] +# addopts = "--mypy" # FIXME: This sometimes blocks test discovery and execution + +[tool.ruff.pylint] +max-args = 8 # Relaxed from default of 5 + +[tool.ruff] +target-version = "py39" +select = [ + # For rules reference, see https://docs.astral.sh/ruff/rules/ + "A", # flake8-builtins + "ANN", # flake8-annotations + "ARG", # flake8-unused-arguments + "ASYNC", # flake8-async + "B", # flake8-bugbear + "FBT", # flake8-boolean-trap + "BLE", # Blind except + "C4", # flake8-comprehensions + "C90", # mccabe (complexity) + "COM", # flake8-commas + "CPY", # missing copyright notice + # "D", # pydocstyle # TODO: Re-enable when adding docstrings + "DTZ", # flake8-datetimez + "E", # pycodestyle (errors) + "ERA", # flake8-eradicate (commented out code) + "EXE", # flake8-executable + "F", # Pyflakes + "FA", # flake8-future-annotations + "FIX", # flake8-fixme + "FLY", # flynt + "FURB", # Refurb + "I", # isort + "ICN", # flake8-import-conventions + "INP", # flake8-no-pep420 + "INT", # flake8-gettext + "ISC", # flake8-implicit-str-concat + "ICN", # flake8-import-conventions + "LOG", # flake8-logging + "N", # pep8-naming + "PD", # pandas-vet + "PERF", # Perflint + "PIE", # flake8-pie + "PGH", # pygrep-hooks + "PL", # Pylint + "PT", # flake8-pytest-style + "PTH", # flake8-use-pathlib + "PYI", # flake8-pyi + "Q", # flake8-quotes + "RET", # flake8-return + "RSE", # flake8-raise + "RUF", # Ruff-specific rules + "SIM", # flake8-simplify + "SLF", # flake8-self + "SLOT", # flake8-slots + "T10", # debugger calls + # "T20", # flake8-print # TODO: Re-enable once we have logging + "TCH", # flake8-type-checking + "TD", # flake8-todos + "TID", # flake8-tidy-imports + "TRY", # tryceratops + "UP", # pyupgrade + "W", # pycodestyle (warnings) + "YTT" # flake8-2020 +] +ignore = [ + # For rules reference, see https://docs.astral.sh/ruff/rules/ + + # These we don't agree with or don't want to prioritize to enforce: + "ANN003", # kwargs missing type annotations + "ANN101", # Type annotations for 'self' args + "COM812", # Because it conflicts with ruff auto-format + "EM", # flake8-errmsgs (may reconsider later) + "DJ", # Django linting + "G", # flake8-logging-format + "ISC001", # Conflicts with ruff auto-format + "NPY", # NumPy-specific rules + "PIE790", # Allow unnecssary 'pass' (sometimes useful for readability) + "PERF203", # exception handling in loop + "S", # flake8-bandit (noisy, security related) + "TD002", # Require author for TODOs + "TRIO", # flake8-trio (opinionated, noisy) + "TRY003", # Exceptions with too-long string descriptions # TODO: re-evaluate once we have our own exception classes + "INP001", # Dir 'examples' is part of an implicit namespace package. Add an __init__.py. + + # TODO: Consider re-enabling these before release: + "A003", # Class attribute 'type' is shadowing a Python builtin + "BLE001", # Do not catch blind exception: Exception + "ERA001", # Remove commented-out code + "FIX002", # Allow "TODO:" until release (then switch to requiring links via TDO003) + "PLW0603", # Using the global statement to update _cache is discouraged + "TD003", # Require links for TODOs # TODO: Re-enable when we disable FIX002 + "TRY002", # TODO: When we have time to tackle exception management ("Create your own exception") +] +fixable = ["ALL"] +unfixable = [ + "ERA001", # Commented-out code (avoid silent loss of code) + "T201" # print() calls (avoid silent loss of code / log messages) +] +line-length = 100 +extend-exclude = ["docs", "test", "tests"] +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +[tool.ruff.isort] +force-sort-within-sections = false +lines-after-imports = 2 +known-first-party = ["airbyte_cdk", "airbyte_protocol"] +known-local-folder = ["airbyte_lib"] +known-third-party = [] +section-order = [ + "future", + "standard-library", + "third-party", + "first-party", + "local-folder" +] + +[tool.ruff.mccabe] +max-complexity = 24 + +[tool.ruff.pycodestyle] +ignore-overlong-task-comments = true + +[tool.ruff.pydocstyle] +convention = "google" + +[tool.ruff.flake8-annotations] +allow-star-arg-any = false +ignore-fully-untyped = false + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +skip-magic-trailing-comma = false +line-ending = "auto" +preview = false +docstring-code-format = true + +[tool.mypy] +# Platform configuration +python_version = "3.9" +# imports related +ignore_missing_imports = true +follow_imports = "silent" +# None and Optional handling +no_implicit_optional = true +strict_optional = true +# Configuring warnings +warn_unused_configs = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_no_return = true +warn_unreachable = true +warn_return_any = false +# Untyped definitions and calls +check_untyped_defs = true +disallow_untyped_calls = false +disallow_untyped_defs = true +disallow_incomplete_defs = true +disallow_untyped_decorators = false +# Disallow dynamic typing +disallow_subclassing_any = true +disallow_any_unimported = false +disallow_any_expr = false +disallow_any_decorated = false +disallow_any_explicit = false +disallow_any_generics = false +# Miscellaneous strictness flags +allow_untyped_globals = false +allow_redefinition = false +local_partial_types = false +implicit_reexport = true +strict_equality = true +# Configuring error messages +show_error_context = false +show_column_numbers = false +show_error_codes = true +exclude = ["docs", "test", "tests"] + +[[tool.mypy.overrides]] +module = [ + "airbyte_protocol", + "airbyte_protocol.models" +] +ignore_missing_imports = true # No stubs yet (😢) + +[tool.poetry.scripts] +generate-docs = "docs:run" +airbyte-lib-validate-source = "airbyte_lib.validate:run" diff --git a/airbyte-lib/tests/conftest.py b/airbyte-lib/tests/conftest.py new file mode 100644 index 000000000000..8defc74b8ba2 --- /dev/null +++ b/airbyte-lib/tests/conftest.py @@ -0,0 +1,131 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +"""Global pytest fixtures.""" + +import json +import logging +import os +import socket +import time +from typing import Optional +from airbyte_lib.caches.snowflake import SnowflakeCacheConfig + +import docker +import psycopg +import pytest +from google.cloud import secretmanager + +from airbyte_lib.caches import PostgresCacheConfig + +logger = logging.getLogger(__name__) + + +PYTEST_POSTGRES_IMAGE = "postgres:13" +PYTEST_POSTGRES_CONTAINER = "postgres_pytest_container" +PYTEST_POSTGRES_PORT = 5432 + + +def is_port_in_use(port): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + return s.connect_ex(("localhost", port)) == 0 + + +@pytest.fixture(scope="session", autouse=True) +def remove_postgres_container(): + client = docker.from_env() + if is_port_in_use(PYTEST_POSTGRES_PORT): + try: + container = client.containers.get( + PYTEST_POSTGRES_CONTAINER, + ) + container.stop() + container.remove() + except docker.errors.NotFound: + pass # Container not found, nothing to do. + + +def test_pg_connection(host) -> bool: + pg_url = f"postgresql://postgres:postgres@{host}:{PYTEST_POSTGRES_PORT}/postgres" + + max_attempts = 10 + for attempt in range(max_attempts): + try: + conn = psycopg.connect(pg_url) + conn.close() + return True + except psycopg.OperationalError: + logger.info(f"Waiting for postgres to start (attempt {attempt + 1}/{max_attempts})") + time.sleep(1.0) + + else: + return False + + +@pytest.fixture(scope="session") +def pg_dsn(): + client = docker.from_env() + try: + client.images.get(PYTEST_POSTGRES_IMAGE) + except docker.errors.ImageNotFound: + # Pull the image if it doesn't exist, to avoid failing our sleep timer + # if the image needs to download on-demand. + client.images.pull(PYTEST_POSTGRES_IMAGE) + + postgres = client.containers.run( + image=PYTEST_POSTGRES_IMAGE, + name=PYTEST_POSTGRES_CONTAINER, + environment={"POSTGRES_USER": "postgres", "POSTGRES_PASSWORD": "postgres", "POSTGRES_DB": "postgres"}, + ports={"5432/tcp": PYTEST_POSTGRES_PORT}, + detach=True, + ) + time.sleep(0.5) + + final_host = None + # Try to connect to the database using localhost and the docker host IP + for host in ["localhost", "172.17.0.1"]: + if test_pg_connection(host): + final_host = host + break + else: + raise Exception("Failed to connect to the PostgreSQL database.") + + yield final_host + # Stop and remove the container after the tests are done + postgres.stop() + postgres.remove() + + +@pytest.fixture +def new_pg_cache_config(pg_dsn): + config = PostgresCacheConfig( + host=pg_dsn, + port=PYTEST_POSTGRES_PORT, + username="postgres", + password="postgres", + database="postgres", + schema_name="public", + ) + yield config + +@pytest.fixture +def snowflake_config(): + if "GCP_GSM_CREDENTIALS" not in os.environ: + raise Exception("GCP_GSM_CREDENTIALS env variable not set, can't fetch secrets for Snowflake. Make sure they are set up as described: https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/ci_credentials/README.md#get-gsm-access") + secret_client = secretmanager.SecretManagerServiceClient.from_service_account_info( + json.loads(os.environ["GCP_GSM_CREDENTIALS"]) + ) + secret = json.loads( + secret_client.access_secret_version( + name="projects/dataline-integration-testing/secrets/AIRBYTE_LIB_SNOWFLAKE_CREDS/versions/latest" + ).payload.data.decode("UTF-8") + ) + config = SnowflakeCacheConfig( + account=secret["account"], + username=secret["username"], + password=secret["password"], + database=secret["database"], + warehouse=secret["warehouse"], + role=secret["role"], + ) + + yield config diff --git a/airbyte-lib/tests/docs_tests/__init__.py b/airbyte-lib/tests/docs_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-lib/tests/docs_tests/test_docs_checked_in.py b/airbyte-lib/tests/docs_tests/test_docs_checked_in.py new file mode 100644 index 000000000000..54614c7cd621 --- /dev/null +++ b/airbyte-lib/tests/docs_tests/test_docs_checked_in.py @@ -0,0 +1,22 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import os + +import docs + + +def test_docs_checked_in(): + """ + Docs need to be generated via `poetry run generate-docs` and checked in to the repo. + + This test runs the docs generation and compares the output with the checked in docs. + It will fail if there are any differences. + """ + + docs.run() + + # compare the generated docs with the checked in docs + diff = os.system("git diff --exit-code docs/generated") + + # if there is a diff, fail the test + assert diff == 0, "Docs are out of date. Please run `poetry run generate-docs` and commit the changes." diff --git a/airbyte-lib/tests/integration_tests/__init__.py b/airbyte-lib/tests/integration_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-lib/tests/integration_tests/fixtures/invalid_config.json b/airbyte-lib/tests/integration_tests/fixtures/invalid_config.json new file mode 100644 index 000000000000..3ce4b45a3209 --- /dev/null +++ b/airbyte-lib/tests/integration_tests/fixtures/invalid_config.json @@ -0,0 +1 @@ +{ "apiKey": "wrong" } diff --git a/airbyte-lib/tests/integration_tests/fixtures/registry.json b/airbyte-lib/tests/integration_tests/fixtures/registry.json new file mode 100644 index 000000000000..f668681a11a9 --- /dev/null +++ b/airbyte-lib/tests/integration_tests/fixtures/registry.json @@ -0,0 +1,40 @@ +{ + "sources": [ + { + "sourceDefinitionId": "9f32dab3-77cb-45a1-9d33-347aa5fbe363", + "name": "Test Source", + "dockerRepository": "airbyte/source-test", + "dockerImageTag": "0.0.1", + "documentationUrl": "https://docs.airbyte.com/integrations/sources/test", + "icon": "test.svg", + "iconUrl": "https://connectors.airbyte.com/files/metadata/airbyte/source-test/latest/icon.svg", + "sourceType": "api", + "spec": { + "documentationUrl": "https://docs.airbyte.com/integrations/sources/test", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "apiKey": { + "type": "string", + "title": "API Key", + "description": "The API key for the service" + } + } + } + }, + "tombstone": false, + "public": true, + "custom": false, + "releaseStage": "alpha", + "supportLevel": "community", + "ab_internal": { + "sl": 100, + "ql": 200 + }, + "tags": ["language:python"], + "githubIssueLabel": "source-test", + "license": "MIT" + } + ] +} diff --git a/airbyte-lib/tests/integration_tests/fixtures/source-test/metadata.yaml b/airbyte-lib/tests/integration_tests/fixtures/source-test/metadata.yaml new file mode 100644 index 000000000000..a2ec10113aa8 --- /dev/null +++ b/airbyte-lib/tests/integration_tests/fixtures/source-test/metadata.yaml @@ -0,0 +1,13 @@ +data: + connectorSubtype: api + connectorType: source + definitionId: 47f17145-fe20-4ef5-a548-e29b048adf84 + dockerImageTag: 0.0.0 + dockerRepository: airbyte/source-test + githubIssueLabel: source-test + name: Test + releaseDate: 2023-08-25 + releaseStage: alpha + supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/sources/apify-dataset +metadataSpecVersion: "1.0" diff --git a/airbyte-lib/tests/integration_tests/fixtures/source-test/setup.py b/airbyte-lib/tests/integration_tests/fixtures/source-test/setup.py new file mode 100644 index 000000000000..b59aca2ec5c3 --- /dev/null +++ b/airbyte-lib/tests/integration_tests/fixtures/source-test/setup.py @@ -0,0 +1,20 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +setup( + name="source_test", + version="0.0.1", + description="Test Soutce", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + entry_points={ + "console_scripts": [ + "source-test=source_test.run:run", + ], + }, +) diff --git a/airbyte-lib/tests/integration_tests/fixtures/source-test/source_test/run.py b/airbyte-lib/tests/integration_tests/fixtures/source-test/source_test/run.py new file mode 100644 index 000000000000..b200e4a84f10 --- /dev/null +++ b/airbyte-lib/tests/integration_tests/fixtures/source-test/source_test/run.py @@ -0,0 +1,133 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import json +import sys + +sample_catalog = { + "type": "CATALOG", + "catalog": { + "streams": [ + { + "name": "stream1", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": True, + "default_cursor_field": ["column1"], + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": {"type": "string"}, + "column2": {"type": "number"}, + }, + }, + }, + { + "name": "stream2", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": False, + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": {"type": "string"}, + "column2": {"type": "number"}, + }, + }, + }, + ] + }, +} + +sample_connection_specification = { + "type": "SPEC", + "spec": { + "documentationUrl": "https://example.com", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "apiKey": { + "type": "string", + "title": "API Key", + "description": "The API key for the service", + } + }, + }, + }, +} + +sample_connection_check_success = { + "type": "CONNECTION_STATUS", + "connectionStatus": {"status": "SUCCEEDED"}, +} + +sample_connection_check_failure = { + "type": "CONNECTION_STATUS", + "connectionStatus": {"status": "FAILED", "message": "An error"}, +} + +sample_record1_stream1 = { + "type": "RECORD", + "record": { + "data": {"column1": "value1", "column2": 1}, + "stream": "stream1", + "emitted_at": 123456789, + }, +} +sample_record2_stream1 = { + "type": "RECORD", + "record": { + "data": {"column1": "value2", "column2": 2}, + "stream": "stream1", + "emitted_at": 123456789, + }, +} +sample_record_stream2 = { + "type": "RECORD", + "record": { + "data": {"column1": "value1", "column2": 1}, + "stream": "stream2", + "emitted_at": 123456789, + }, +} + + +def parse_args(): + arg_dict = {} + args = sys.argv[2:] + for i in range(0, len(args), 2): + arg_dict[args[i]] = args[i + 1] + + return arg_dict + + +def get_json_file(path): + with open(path, "r") as f: + return json.load(f) + + +def run(): + args = sys.argv[1:] + if args[0] == "spec": + print(json.dumps(sample_connection_specification)) + elif args[0] == "discover": + print(json.dumps(sample_catalog)) + elif args[0] == "check": + args = parse_args() + config = get_json_file(args["--config"]) + if config.get("apiKey").startswith("test"): + print(json.dumps(sample_connection_check_success)) + else: + print(json.dumps(sample_connection_check_failure)) + elif args[0] == "read": + args = parse_args() + catalog = get_json_file(args["--catalog"]) + config = get_json_file(args["--config"]) + for stream in catalog["streams"]: + if stream["stream"]["name"] == "stream1": + print(json.dumps(sample_record1_stream1)) + if config.get("apiKey") == "test_fail_during_sync": + raise Exception("An error") + print(json.dumps(sample_record2_stream1)) + elif stream["stream"]["name"] == "stream2": + print(json.dumps(sample_record_stream2)) diff --git a/airbyte-lib/tests/integration_tests/fixtures/valid_config.json b/airbyte-lib/tests/integration_tests/fixtures/valid_config.json new file mode 100644 index 000000000000..fbe094d80a44 --- /dev/null +++ b/airbyte-lib/tests/integration_tests/fixtures/valid_config.json @@ -0,0 +1 @@ +{ "apiKey": "test" } diff --git a/airbyte-lib/tests/integration_tests/test_integration.py b/airbyte-lib/tests/integration_tests/test_integration.py new file mode 100644 index 000000000000..c9bdfbf91415 --- /dev/null +++ b/airbyte-lib/tests/integration_tests/test_integration.py @@ -0,0 +1,390 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import os +import shutil +import subprocess +from unittest.mock import Mock, call, patch +import tempfile +from pathlib import Path + +import airbyte_lib as ab +from airbyte_lib.caches import SnowflakeCacheConfig, SnowflakeSQLCache +import pandas as pd +import pytest + +from airbyte_lib.caches import PostgresCache, PostgresCacheConfig +from airbyte_lib.registry import _update_cache +from airbyte_lib.version import get_version +from airbyte_lib.results import ReadResult + + +@pytest.fixture(scope="module", autouse=True) +def prepare_test_env(): + """ + Prepare test environment. This will pre-install the test source from the fixtures array and set the environment variable to use the local json file as registry. + """ + if os.path.exists(".venv-source-test"): + shutil.rmtree(".venv-source-test") + + os.system("python -m venv .venv-source-test") + os.system(".venv-source-test/bin/pip install -e ./tests/integration_tests/fixtures/source-test") + + os.environ["AIRBYTE_LOCAL_REGISTRY"] = "./tests/integration_tests/fixtures/registry.json" + os.environ["DO_NOT_TRACK"] = "true" + + yield + + shutil.rmtree(".venv-source-test") + +@pytest.fixture +def expected_test_stream_data() -> dict[str, list[dict[str, str | int]]]: + return { + "stream1": [ + {"column1": "value1", "column2": 1}, + {"column1": "value2", "column2": 2}, + ], + "stream2": [ + {"column1": "value1", "column2": 1}, + ], + } + +def test_list_streams(expected_test_stream_data: dict[str, list[dict[str, str | int]]]): + source = ab.get_connector("source-test", config={"apiKey": "test"}) + + assert source.get_available_streams() == list(expected_test_stream_data.keys()) + + +def test_invalid_config(): + with pytest.raises(Exception): + ab.get_connector("source-test", config={"apiKey": 1234}) + + +def test_non_existing_connector(): + with pytest.raises(Exception): + ab.get_connector("source-not-existing", config={"apiKey": "abc"}) + + +@pytest.mark.parametrize( + "latest_available_version, requested_version, raises", + [ + ("0.0.1", None, False), + ("1.2.3", None, False), + ("0.0.1", "latest", False), + ("1.2.3", "latest", True), + ("0.0.1", "0.0.1", False), + ("1.2.3", "1.2.3", True), + ]) +def test_version_enforcement(raises, latest_available_version, requested_version): + """" + Ensures version enforcement works as expected: + * If no version is specified, the current version is accepted + * If the version is specified as "latest", only the latest available version is accepted + * If the version is specified as a semantic version, only the exact version is accepted + + In this test, the actually installed version is 0.0.1 + """ + _update_cache() + from airbyte_lib.registry import _cache + _cache["source-test"].latest_available_version = latest_available_version + if raises: + with pytest.raises(Exception): + ab.get_connector("source-test", version=requested_version, config={"apiKey": "abc"}) + else: + ab.get_connector("source-test", version=requested_version, config={"apiKey": "abc"}) + + # reset + _cache["source-test"].latest_available_version = "0.0.1" + + +def test_check(): + source = ab.get_connector("source-test", config={"apiKey": "test"}) + + source.check() + + +def test_check_fail(): + source = ab.get_connector("source-test", config={"apiKey": "wrong"}) + + with pytest.raises(Exception): + source.check() + + +def test_file_write_and_cleanup() -> None: + """Ensure files are written to the correct location and cleaned up afterwards.""" + with tempfile.TemporaryDirectory() as temp_dir_1, tempfile.TemporaryDirectory() as temp_dir_2: + cache_w_cleanup = ab.new_local_cache(cache_dir=temp_dir_1, cleanup=True) + cache_wo_cleanup = ab.new_local_cache(cache_dir=temp_dir_2, cleanup=False) + + source = ab.get_connector("source-test", config={"apiKey": "test"}) + + _ = source.read(cache_w_cleanup) + _ = source.read(cache_wo_cleanup) + + assert len(list(Path(temp_dir_1).glob("*.parquet"))) == 0, "Expected files to be cleaned up" + assert len(list(Path(temp_dir_2).glob("*.parquet"))) == 2, "Expected files to exist" + + +def test_sync_to_duckdb(expected_test_stream_data: dict[str, list[dict[str, str | int]]]): + source = ab.get_connector("source-test", config={"apiKey": "test"}) + cache = ab.new_local_cache() + + result: ReadResult = source.read(cache) + + assert result.processed_records == 3 + for stream_name, expected_data in expected_test_stream_data.items(): + pd.testing.assert_frame_equal( + result[stream_name].to_pandas(), + pd.DataFrame(expected_data), + check_dtype=False, + ) + + +def test_read_result_as_list(expected_test_stream_data: dict[str, list[dict[str, str | int]]]): + source = ab.get_connector("source-test", config={"apiKey": "test"}) + cache = ab.new_local_cache() + + result: ReadResult = source.read(cache) + stream_1_list = list(result["stream1"]) + stream_2_list = list(result["stream2"]) + assert stream_1_list == expected_test_stream_data["stream1"] + assert stream_2_list == expected_test_stream_data["stream2"] + + +def test_get_records_result_as_list(expected_test_stream_data: dict[str, list[dict[str, str | int]]]): + source = ab.get_connector("source-test", config={"apiKey": "test"}) + cache = ab.new_local_cache() + + stream_1_list = list(source.get_records("stream1")) + stream_2_list = list(source.get_records("stream2")) + assert stream_1_list == expected_test_stream_data["stream1"] + assert stream_2_list == expected_test_stream_data["stream2"] + + + +def test_sync_with_merge_to_duckdb(expected_test_stream_data: dict[str, list[dict[str, str | int]]]): + """Test that the merge strategy works as expected. + + In this test, we sync the same data twice. If the data is not duplicated, we assume + the merge was successful. + + # TODO: Add a check with a primary key to ensure that the merge strategy works as expected. + """ + source = ab.get_connector("source-test", config={"apiKey": "test"}) + cache = ab.new_local_cache() + + # Read twice to test merge strategy + result: ReadResult = source.read(cache) + result: ReadResult = source.read(cache) + + assert result.processed_records == 3 + for stream_name, expected_data in expected_test_stream_data.items(): + pd.testing.assert_frame_equal( + result[stream_name].to_pandas(), + pd.DataFrame(expected_data), + check_dtype=False, + ) + + +@pytest.mark.parametrize( + "method_call", + [ + pytest.param(lambda source: source.check(), id="check"), + pytest.param(lambda source: list(source.get_records("stream1")), id="read_stream"), + pytest.param(lambda source: source.read(), id="read"), + ], +) +def test_check_fail_on_missing_config(method_call): + source = ab.get_connector("source-test") + + with pytest.raises(Exception, match="Config is not set, either set in get_connector or via source.set_config"): + method_call(source) + +def test_sync_with_merge_to_postgres(new_pg_cache_config: PostgresCacheConfig, expected_test_stream_data: dict[str, list[dict[str, str | int]]]): + """Test that the merge strategy works as expected. + + In this test, we sync the same data twice. If the data is not duplicated, we assume + the merge was successful. + + # TODO: Add a check with a primary key to ensure that the merge strategy works as expected. + """ + source = ab.get_connector("source-test", config={"apiKey": "test"}) + cache = PostgresCache(config=new_pg_cache_config) + + # Read twice to test merge strategy + result: ReadResult = source.read(cache) + result: ReadResult = source.read(cache) + + assert result.processed_records == 3 + for stream_name, expected_data in expected_test_stream_data.items(): + pd.testing.assert_frame_equal( + result[stream_name].to_pandas(), + pd.DataFrame(expected_data), + check_dtype=False, + ) + +@patch.dict('os.environ', {'DO_NOT_TRACK': ''}) +@patch('airbyte_lib.telemetry.requests') +@patch('airbyte_lib.telemetry.datetime') +@pytest.mark.parametrize( + "raises, api_key, expected_state, expected_number_of_records, request_call_fails, extra_env, expected_flags", + [ + pytest.param(True, "test_fail_during_sync", "failed", 1, False, {"CI": ""}, {"CI": False}, id="fail_during_sync"), + pytest.param(False, "test", "succeeded", 3, False, {"CI": ""}, {"CI": False}, id="succeed_during_sync"), + pytest.param(False, "test", "succeeded", 3, True, {"CI": ""}, {"CI": False}, id="fail_request_without_propagating"), + pytest.param(False, "test", "succeeded", 3, False, {"CI": ""}, {"CI": False}, id="falsy_ci_flag"), + pytest.param(False, "test", "succeeded", 3, False, {"CI": "true"}, {"CI": True}, id="truthy_ci_flag"), + ], +) +def test_tracking(mock_datetime: Mock, mock_requests: Mock, raises: bool, api_key: str, expected_state: str, expected_number_of_records: int, request_call_fails: bool, extra_env: dict[str, str], expected_flags: dict[str, bool]): + """ + Test that the telemetry is sent when the sync is successful. + This is done by mocking the requests.post method and checking that it is called with the right arguments. + """ + now_date = Mock() + mock_datetime.datetime = Mock() + mock_datetime.datetime.utcnow.return_value = now_date + now_date.isoformat.return_value = "2021-01-01T00:00:00.000000" + + mock_post = Mock() + mock_requests.post = mock_post + + source = ab.get_connector("source-test", config={"apiKey": api_key}) + cache = ab.get_default_cache() + + if request_call_fails: + mock_post.side_effect = Exception("test exception") + + with patch.dict('os.environ', extra_env): + if raises: + with pytest.raises(Exception): + source.read(cache) + else: + source.read(cache) + + + mock_post.assert_has_calls([ + call("https://api.segment.io/v1/track", + auth=("jxT1qP9WEKwR3vtKMwP9qKhfQEGFtIM1", ""), + json={ + "anonymousId": "airbyte-lib-user", + "event": "sync", + "properties": { + "version": get_version(), + "source": {'name': 'source-test', 'version': '0.0.1', 'type': 'venv'}, + "state": "started", + "cache": {"type": "duckdb"}, + "ip": "0.0.0.0", + "flags": expected_flags + }, + "timestamp": "2021-01-01T00:00:00.000000", + } + ), + call( + "https://api.segment.io/v1/track", + auth=("jxT1qP9WEKwR3vtKMwP9qKhfQEGFtIM1", ""), + json={ + "anonymousId": "airbyte-lib-user", + "event": "sync", + "properties": { + "version": get_version(), + "source": {'name': 'source-test', 'version': '0.0.1', 'type': 'venv'}, + "state": expected_state, + "number_of_records": expected_number_of_records, + "cache": {"type": "duckdb"}, + "ip": "0.0.0.0", + "flags": expected_flags + }, + "timestamp": "2021-01-01T00:00:00.000000", + } + ) + ]) + + +def test_sync_to_postgres(new_pg_cache_config: PostgresCacheConfig, expected_test_stream_data: dict[str, list[dict[str, str | int]]]): + source = ab.get_connector("source-test", config={"apiKey": "test"}) + cache = PostgresCache(config=new_pg_cache_config) + + result: ReadResult = source.read(cache) + + assert result.processed_records == 3 + for stream_name, expected_data in expected_test_stream_data.items(): + pd.testing.assert_frame_equal( + result[stream_name].to_pandas(), + pd.DataFrame(expected_data), + check_dtype=False, + ) + +def test_sync_to_snowflake(snowflake_config: SnowflakeCacheConfig, expected_test_stream_data: dict[str, list[dict[str, str | int]]]): + source = ab.get_connector("source-test", config={"apiKey": "test"}) + cache = SnowflakeSQLCache(config=snowflake_config) + + with cache.get_sql_connection() as con: + con.execute("DROP SCHEMA IF EXISTS AIRBYTE_RAW") + + result: ReadResult = source.read(cache) + + assert result.processed_records == 3 + for stream_name, expected_data in expected_test_stream_data.items(): + pd.testing.assert_frame_equal( + result[stream_name].to_pandas(), + pd.DataFrame(expected_data), + check_dtype=False, + ) + +def test_sync_limited_streams(expected_test_stream_data): + source = ab.get_connector("source-test", config={"apiKey": "test"}) + cache = ab.new_local_cache() + + source.set_streams(["stream2"]) + + result = source.read(cache) + + assert result.processed_records == 1 + pd.testing.assert_frame_equal( + result["stream2"].to_pandas(), + pd.DataFrame(expected_test_stream_data["stream2"]), + check_dtype=False, + ) + + +def test_read_stream(): + source = ab.get_connector("source-test", config={"apiKey": "test"}) + + assert list(source.get_records("stream1")) == [{"column1": "value1", "column2": 1}, {"column1": "value2", "column2": 2}] + + +def test_read_stream_nonexisting(): + source = ab.get_connector("source-test", config={"apiKey": "test"}) + + with pytest.raises(Exception): + list(source.get_records("non-existing")) + +def test_failing_path_connector(): + with pytest.raises(Exception): + ab.get_connector("source-test", config={"apiKey": "test"}, use_local_install=True) + +def test_succeeding_path_connector(): + old_path = os.environ["PATH"] + + # set path to include the test venv bin folder + os.environ["PATH"] = f"{os.path.abspath('.venv-source-test/bin')}:{os.environ['PATH']}" + source = ab.get_connector("source-test", config={"apiKey": "test"}, use_local_install=True) + source.check() + + os.environ["PATH"] = old_path + +def test_install_uninstall(): + source = ab.get_connector("source-test", pip_url="./tests/integration_tests/fixtures/source-test", config={"apiKey": "test"}, install_if_missing=False) + + source.uninstall() + + # assert that the venv is gone + assert not os.path.exists(".venv-source-test") + + # assert that the connector is not available + with pytest.raises(Exception): + source.check() + + source.install() + + source.check() \ No newline at end of file diff --git a/airbyte-lib/tests/integration_tests/test_validation.py b/airbyte-lib/tests/integration_tests/test_validation.py new file mode 100644 index 000000000000..75a463592833 --- /dev/null +++ b/airbyte-lib/tests/integration_tests/test_validation.py @@ -0,0 +1,15 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import os +import shutil + +import pytest +from airbyte_lib.validate import validate + + +def test_validate_success(): + validate("./tests/integration_tests/fixtures/source-test", "./tests/integration_tests/fixtures/valid_config.json") + +def test_validate_failure(): + with pytest.raises(Exception): + validate("./tests/integration_tests/fixtures/source-test", "./tests/integration_tests/fixtures/invalid_config.json") diff --git a/airbyte-lib/tests/lint_tests/__init__.py b/airbyte-lib/tests/lint_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-lib/tests/lint_tests/test_mypy.py b/airbyte-lib/tests/lint_tests/test_mypy.py new file mode 100644 index 000000000000..df0997828079 --- /dev/null +++ b/airbyte-lib/tests/lint_tests/test_mypy.py @@ -0,0 +1,21 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import subprocess + +import pytest + + +def test_mypy_typing(): + # Run the check command + check_result = subprocess.run( + ["poetry", "run", "mypy", "."], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + # Assert that the Ruff command exited without errors (exit code 0) + assert check_result.returncode == 0, ( + "MyPy checks failed:\n" + + f"{check_result.stdout.decode()}\n{check_result.stderr.decode()}\n\n" + + "Run `poetry run mypy .` to see all failures." + ) diff --git a/airbyte-lib/tests/lint_tests/test_ruff.py b/airbyte-lib/tests/lint_tests/test_ruff.py new file mode 100644 index 000000000000..57262a8f608c --- /dev/null +++ b/airbyte-lib/tests/lint_tests/test_ruff.py @@ -0,0 +1,51 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import subprocess + +import pytest + + +def test_ruff_linting(): + # Run the check command + check_result = subprocess.run( + ["poetry", "run", "ruff", "check", "."], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + # Assert that the Ruff command exited without errors (exit code 0) + assert check_result.returncode == 0, ( + "Ruff checks failed:\n\n" + + f"{check_result.stdout.decode()}\n{check_result.stderr.decode()}\n\n" + + "Run `poetry run ruff check .` to view all issues." + ) + + +def test_ruff_linting_fixable(): + # Run the check command + fix_diff_result = subprocess.run( + ["poetry", "run", "ruff", "check", "--fix", "--diff", "."], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + + # Assert that the Ruff command exited without errors (exit code 0) + assert fix_diff_result.returncode == 0, ( + "Ruff checks revealed fixable issues:\n\n" + + f"{fix_diff_result.stdout.decode()}\n{fix_diff_result.stderr.decode()}\n\n" + + "Run `poetry run ruff check --fix .` to attempt automatic fixes." + ) + + +def test_ruff_format(): + # Define the command to run Ruff + command = ["poetry", "run", "ruff", "format", "--check", "--diff"] + + # Run the command + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + # Assert that the Ruff command exited without errors (exit code 0) + assert result.returncode == 0, ( + f"Ruff checks failed:\n\n{result.stdout.decode()}\n{result.stderr.decode()}\n\n" + + "Run `poetry run ruff format .` to attempt automatic fixes." + ) diff --git a/airbyte-lib/tests/unit_tests/__init__.py b/airbyte-lib/tests/unit_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-lib/tests/unit_tests/test_caches.py b/airbyte-lib/tests/unit_tests/test_caches.py new file mode 100644 index 000000000000..5bc2ba4186cd --- /dev/null +++ b/airbyte-lib/tests/unit_tests/test_caches.py @@ -0,0 +1,60 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from pathlib import Path + +import pytest + +from airbyte_lib._file_writers import ParquetWriterConfig +from airbyte_lib.caches.base import SQLCacheBase, SQLCacheConfigBase +from airbyte_lib.caches.duckdb import DuckDBCacheBase, DuckDBCacheConfig + + +def test_duck_db_cache_config_initialization(): + config = DuckDBCacheConfig(db_path='test_path', schema_name='test_schema') + assert config.db_path == Path('test_path') + assert config.schema_name == 'test_schema' + +def test_duck_db_cache_config_default_schema_name(): + config = DuckDBCacheConfig(db_path='test_path') + assert config.schema_name == 'main' + +def test_get_sql_alchemy_url(): + config = DuckDBCacheConfig(db_path='test_path', schema_name='test_schema') + assert config.get_sql_alchemy_url() == 'duckdb:///test_path' + +def test_get_sql_alchemy_url_with_default_schema_name(): + config = DuckDBCacheConfig(db_path='test_path') + assert config.get_sql_alchemy_url() == 'duckdb:///test_path' + +def test_duck_db_cache_config_inheritance(): + assert issubclass(DuckDBCacheConfig, SQLCacheConfigBase) + assert issubclass(DuckDBCacheConfig, ParquetWriterConfig) + +def test_duck_db_cache_config_get_sql_alchemy_url(): + config = DuckDBCacheConfig(db_path='test_path', schema_name='test_schema') + assert config.get_sql_alchemy_url() == 'duckdb:///test_path' + +def test_duck_db_cache_config_get_database_name(): + config = DuckDBCacheConfig(db_path='test_path/test_db.duckdb', schema_name='test_schema') + assert config.get_database_name() == 'test_db' + +def test_duck_db_cache_base_inheritance(): + assert issubclass(DuckDBCacheBase, SQLCacheBase) + +def test_duck_db_cache_config_default_schema_name(): + config = DuckDBCacheConfig(db_path='test_path') + assert config.schema_name == 'main' + +def test_duck_db_cache_config_get_sql_alchemy_url_with_default_schema_name(): + config = DuckDBCacheConfig(db_path='test_path') + assert config.get_sql_alchemy_url() == 'duckdb:///test_path' + +def test_duck_db_cache_config_get_database_name_with_default_schema_name(): + config = DuckDBCacheConfig(db_path='test_path/test_db.duckdb') + assert config.get_database_name() == 'test_db' + +def test_duck_db_cache_config_inheritance_from_sql_cache_config_base(): + assert issubclass(DuckDBCacheConfig, SQLCacheConfigBase) + +def test_duck_db_cache_config_inheritance_from_parquet_writer_config(): + assert issubclass(DuckDBCacheConfig, ParquetWriterConfig) diff --git a/airbyte-lib/tests/unit_tests/test_type_translation.py b/airbyte-lib/tests/unit_tests/test_type_translation.py new file mode 100644 index 000000000000..80c1e611c662 --- /dev/null +++ b/airbyte-lib/tests/unit_tests/test_type_translation.py @@ -0,0 +1,27 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import pytest +from sqlalchemy import types +from airbyte_lib.types import SQLTypeConverter + +@pytest.mark.parametrize( + "json_schema_property_def, expected_sql_type", + [ + ({"type": "string"}, types.VARCHAR), + ({"type": "boolean"}, types.BOOLEAN), + ({"type": "string", "format": "date"}, types.DATE), + ({"type": "string", "format": "date-time", "airbyte_type": "timestamp_without_timezone"}, types.TIMESTAMP), + ({"type": "string", "format": "date-time", "airbyte_type": "timestamp_with_timezone"}, types.TIMESTAMP), + ({"type": "string", "format": "time", "airbyte_type": "time_without_timezone"}, types.TIME), + ({"type": "string", "format": "time", "airbyte_type": "time_with_timezone"}, types.TIME), + ({"type": "integer"}, types.BIGINT), + ({"type": "number", "airbyte_type": "integer"}, types.BIGINT), + ({"type": "number"}, types.DECIMAL), + ({"type": "array"}, types.VARCHAR), + ({"type": "object"}, types.VARCHAR), + ], +) +def test_to_sql_type(json_schema_property_def, expected_sql_type): + converter = SQLTypeConverter() + sql_type = converter.to_sql_type(json_schema_property_def) + assert isinstance(sql_type, expected_sql_type) diff --git a/airbyte-lib/tests/unit_tests/test_writers.py b/airbyte-lib/tests/unit_tests/test_writers.py new file mode 100644 index 000000000000..5d0432606b13 --- /dev/null +++ b/airbyte-lib/tests/unit_tests/test_writers.py @@ -0,0 +1,38 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from pathlib import Path +import pytest +from airbyte_lib._file_writers.base import FileWriterBase, FileWriterBatchHandle, FileWriterConfigBase +from airbyte_lib._file_writers.parquet import ParquetWriter, ParquetWriterConfig +from numpy import source + + +def test_parquet_writer_config_initialization(): + config = ParquetWriterConfig(cache_dir='test_path') + assert config.cache_dir == Path('test_path') + +def test_parquet_writer_config_inheritance(): + assert issubclass(ParquetWriterConfig, FileWriterConfigBase) + +def test_parquet_writer_initialization(): + config = ParquetWriterConfig(cache_dir='test_path') + writer = ParquetWriter(config) + assert writer.config == config + +def test_parquet_writer_inheritance(): + assert issubclass(ParquetWriter, FileWriterBase) + +def test_parquet_writer_has_config(): + config = ParquetWriterConfig(cache_dir='test_path') + writer = ParquetWriter(config) + assert hasattr(writer, 'config') + +def test_parquet_writer_has_source_catalog(): + config = ParquetWriterConfig(cache_dir='test_path') + writer = ParquetWriter(config) + assert hasattr(writer, 'source_catalog') + +def test_parquet_writer_source_catalog_is_none(): + config = ParquetWriterConfig(cache_dir='test_path') + writer = ParquetWriter(config) + assert writer.source_catalog is None diff --git a/airbyte-test-utils/LICENSE b/airbyte-test-utils/LICENSE deleted file mode 100644 index ec45d182fcb9..000000000000 --- a/airbyte-test-utils/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2020 Airbyte, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/airbyte-test-utils/build.gradle b/airbyte-test-utils/build.gradle deleted file mode 100644 index 3ec37f539308..000000000000 --- a/airbyte-test-utils/build.gradle +++ /dev/null @@ -1,31 +0,0 @@ -plugins { - id 'java-library' -} - -configurations.all { - exclude group: 'io.micronaut.jaxrs' - exclude group: 'io.micronaut.sql' -} - -dependencies { - api project(':airbyte-db:db-lib') - implementation project(':airbyte-api') - implementation project(':airbyte-connector-test-harnesses:acceptance-test-harness') - - implementation 'io.fabric8:kubernetes-client:5.12.2' - implementation libs.temporal.sdk - - - api libs.junit.jupiter.api - - // Mark as compile only to avoid leaking transitively to connectors - compileOnly libs.platform.testcontainers.jdbc - compileOnly libs.platform.testcontainers.postgresql - compileOnly libs.platform.testcontainers.cockroachdb - - testImplementation libs.platform.testcontainers.jdbc - testImplementation libs.platform.testcontainers.postgresql - testImplementation libs.platform.testcontainers.cockroachdb -} - -Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project) diff --git a/airbyte-test-utils/readme.md b/airbyte-test-utils/readme.md deleted file mode 100644 index f75ba4e74b0b..000000000000 --- a/airbyte-test-utils/readme.md +++ /dev/null @@ -1,3 +0,0 @@ -# airbyte-test-utils - -Shared Java code for executing TestContainers and other helpers. diff --git a/build.gradle b/build.gradle index 3a94a394ec5e..80da7caf07ea 100644 --- a/build.gradle +++ b/build.gradle @@ -1,55 +1,14 @@ -import com.bmuschko.gradle.docker.tasks.image.DockerBuildImage import com.github.spotbugs.snom.SpotBugsTask -import ru.vyarus.gradle.plugin.python.task.PythonTask - -// The buildscript block defines dependencies in order for .gradle file evaluation. -// This is separate from application dependencies. -// See https://stackoverflow.com/questions/17773817/purpose-of-buildscript-block-in-gradle. -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - } - dependencies { - classpath 'com.bmuschko:gradle-docker-plugin:8.0.0' - // 6.x version of OpenApi generator is only compatible with jackson-core 2.13.x onwards. - // This conflicts with the jackson depencneis the bmuschko plugin is pulling in. - // Since api generation is only used in the airbyte-api module and the base gradle files - // are loaded in first, Gradle is not able to intelligently resolve this before loading in - // the bmuschko plugin and thus placing an older jackson version on the class path. - // The alternative is to import the openapi plugin for all modules. - // This might need to be updated when we change openapi plugin versions. - classpath 'com.fasterxml.jackson.core:jackson-core:2.13.0' - - classpath 'org.codehaus.groovy:groovy-yaml:3.0.3' - } -} + plugins { id 'base' - id 'pmd' - id 'com.diffplug.spotless' version '6.20.0' id 'com.github.node-gradle.node' version '3.5.1' - id 'com.github.hierynomus.license' version '0.16.1' id 'com.github.spotbugs' version '5.0.13' - // The distribution plugin has been added to address the an issue with the copyGeneratedTar - // task depending on "distTar". When that dependency has been refactored away, this plugin - // can be removed. - id 'distribution' id 'version-catalog' - id 'maven-publish' id 'ru.vyarus.use-python' } -apply from: "$rootDir/publish-repositories.gradle" - -repositories { - mavenCentral() - maven { - url 'https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/' - } -} Properties env = new Properties() rootProject.file('gradle.properties').withInputStream { env.load(it) } @@ -65,161 +24,23 @@ if (!env.containsKey('VERSION')) { ext { version = System.getenv("VERSION") ?: env.VERSION image_tag = System.getenv("VERSION") ?: 'dev' + skipSlowTests = (System.getProperty('skipSlowTests', 'false') != 'false') } - -def createLicenseWith = { File license, String startComment, String endComment, String lineComment, boolean isPython -> - /* - In java, we don't have a second linter/styling tool other than spotless so it doesn't really - matter if we write a newline or not for startComment/endComment. - - However, in python, we are using black that double-checks and reformats the code. - Thus, writing an extra empty newline (not removed by trimTrailingWhitespace() is actually a - big deal and would be reformatted (removed) because of black's specs. - */ - def tmp = File.createTempFile('tmp', '.tmp') - tmp.withWriter { - def w = it - if (startComment.length() > 0 || !isPython) { - w.writeLine(startComment) - } - license.eachLine { - w << lineComment - w.writeLine(it) - } - if (endComment.length() > 0 || !isPython) { - w.writeLine(endComment) - } - w.writeLine("") - if (isPython) { - w.writeLine("") - } +// Pyenv support. +try { + def pyenvRoot = "pyenv root".execute() + if (pyenvRoot.waitFor() == 0) { + ext.pyenvRoot = pyenvRoot.text.trim() } - return tmp -} - -def createPythonLicenseWith = { license -> - return createLicenseWith(license, '', '', '', true) -} - -def createJavaLicenseWith = { license -> - return createLicenseWith(license, '/*', ' */', ' * ', false) -} - -// We are the spotless exclusions rules using file tree. It seems the excludeTarget option is super finicky in a -// monorepo setup and it doesn't actually exclude directories reliably. This code makes the behavior predictable. -def createSpotlessTarget = { pattern -> - def excludes = [ - '.gradle', - 'node_modules', - '.eggs', - '.mypy_cache', - '.venv', - '*.egg-info', - 'build', - 'dbt-project-template', - 'dbt-project-template-mssql', - 'dbt-project-template-mysql', - 'dbt-project-template-oracle', - 'dbt-project-template-clickhouse', - 'dbt-project-template-snowflake', - 'dbt-project-template-tidb', - 'dbt-project-template-duckdb', - 'dbt_test_config', - 'normalization_test_output', - 'tools', - 'secrets', - 'charts', // Helm charts often have injected template strings that will fail general linting. Helm linting is done separately. - 'resources/seed/*_catalog.json', // Do not remove - this is also necessary to prevent diffs in our github workflows - 'resources/seed/*_registry.json', // Do not remove - this is also necessary to prevent diffs in our github workflows - 'airbyte-integrations/connectors/source-amplitude/unit_tests/api_data/zipped.json', // Zipped file presents as non-UTF-8 making spotless sad - 'airbyte-webapp', // The webapp module uses its own auto-formatter, so spotless is not necessary here - 'airbyte-webapp-e2e-tests', // This module also uses its own auto-formatter - 'airbyte-connector-builder-server/connector_builder/generated', // autogenerated code doesn't need to be formatted - 'airbyte-ci/connectors/metadata_service/lib/tests/fixtures/**/invalid', // These are deliberately invalid and unformattable. - ] - - if (System.getenv().containsKey("SUB_BUILD")) { - excludes.add("airbyte-integrations/connectors") - } - - return fileTree(dir: rootDir, include: pattern, exclude: excludes.collect { "**/${it}" }) +} catch (IOException _) { + // Swallow exception if pyenv is not installed. } -node { - download = true - version = '18.16.1' - npmVersion = '9.5.1' - // when setting both these directories, npm and node will be in separate directories - workDir = file("${buildDir}/nodejs") - npmWorkDir = file("${buildDir}/npm") -} - -spotless { - java { - target createSpotlessTarget('**/*.java') - - importOrder() - - eclipse('4.21.0').configFile(rootProject.file('tools/gradle/codestyle/java-google-style.xml')) - - licenseHeaderFile createJavaLicenseWith(rootProject.file('LICENSE_SHORT')) - removeUnusedImports() - trimTrailingWhitespace() - } - groovyGradle { - target createSpotlessTarget('**/*.gradle') - } - sql { - target createSpotlessTarget('**/*.sql') - - dbeaver().configFile(rootProject.file('tools/gradle/codestyle/sql-dbeaver.properties')) - } - format 'styling', { - target createSpotlessTarget(['**/*.yaml', '**/*.json']) - - prettier() - .npmExecutable("${tasks.named('npmSetup').get().npmDir.get()}/bin/npm") // get the npm executable path from gradle-node-plugin - .nodeExecutable("${tasks.named('nodeSetup').get().nodeDir.get()}/bin/node") // get the node executable path from gradle-node-plugin - } -} - -tasks.named('spotlessStyling').configure { - it.dependsOn('nodeSetup', 'npmSetup') -} - -check.dependsOn 'spotlessApply' - -@SuppressWarnings('GroovyAssignabilityCheck') -def Task getPublishArtifactsTask(String buildVersion, Project subproject) { - // generate a unique task name based on the directory name. - return task("publishArtifact-$subproject.name" { - apply plugin: 'maven-publish' - publishing { - repositories { - publications { - "$subproject.name"(MavenPublication) { - from subproject.components.java - - // use the subproject group and name with the assumption there are no identical subproject - // names, group names or subproject group/name combination. - groupId = "$subproject.group" - artifactId = "$subproject.name" - version = "$buildVersion" - repositories.add(rootProject.repositories.getByName('cloudrepo')) - } - } - } - } - }) -} - -allprojects { - apply plugin: 'com.bmuschko.docker-remote-api' - - task copyDocker(type: Sync) { - from "${project.projectDir}/Dockerfile" - into "build/docker/" +def isConnectorProject = { Project project -> + if (project.parent == null || project.parent.name != 'connectors') { + return false } + return project.name.startsWith("source-") || project.name.startsWith("destination-") } allprojects { @@ -229,17 +50,62 @@ allprojects { // projects clobber each other in an environments with subprojects when projects are in directories named identically. def sub = rootDir.relativePath(projectDir.parentFile).replace('/', '.') group = "io.${rootProject.name}${sub.isEmpty() ? '' : ".$sub"}" - project.archivesBaseName = "${project.group}-${project.name}" + project.base.archivesName = "${project.group}-${project.name}" version = rootProject.ext.version } +// python is required by the root project to run CAT tests for connectors +python { + envPath = '.venv' + minPythonVersion = '3.10' // should be 3.10 for local development + + // Amazon Linux support. + // The airbyte-ci tool runs gradle tasks in AL2023-based containers. + // In AL2023, `python3` is necessarily v3.9, and later pythons need to be installed and named explicitly. + // See https://github.com/amazonlinux/amazon-linux-2023/issues/459 for details. + try { + if ("python3.11 --version".execute().waitFor() == 0) { + // python3.11 definitely exists at this point, use it instead of 'python3'. + pythonBinary "python3.11" + } + } catch (IOException _) { + // Swallow exception if python3.11 is not installed. + } + // Pyenv support. + try { + def pyenvRoot = "pyenv root".execute() + def pyenvLatest = "pyenv latest ${minPythonVersion}".execute() + // Pyenv definitely exists at this point: use 'python' instead of 'python3' in all cases. + pythonBinary "python" + if (pyenvRoot.waitFor() == 0 && pyenvLatest.waitFor() == 0) { + pythonPath "${pyenvRoot.text.trim()}/versions/${pyenvLatest.text.trim()}/bin" + } + } catch (IOException _) { + // Swallow exception if pyenv is not installed. + } + + scope = 'VIRTUALENV' + installVirtualenv = true + // poetry is required for installing and running airbyte-ci + pip 'poetry:1.5.1' +} + +def cleanPythonVenv = rootProject.tasks.register('cleanPythonVenv', Exec) { + commandLine 'rm' + args '-rf', "${rootProject.projectDir.absolutePath}/.venv" +} +rootProject.tasks.named('clean').configure { + dependsOn cleanPythonVenv +} + + def getCDKTargetVersion() { def props = new Properties() file("airbyte-cdk/java/airbyte-cdk/src/main/resources/version.properties").withInputStream { props.load(it) } return props.getProperty('version') } -def getLatestFileModifiedTimeFromFiles(files) { +static def getLatestFileModifiedTimeFromFiles(files) { if (files.isEmpty()) { return null } @@ -278,11 +144,11 @@ def checkCDKJarExists(requiredSnapshotVersion) { } } } -def getCDKSnapshotRequirement(dependenciesList) { +static def getCDKSnapshotRequirement(dependenciesList) { def cdkSnapshotRequirement = dependenciesList.find { it.requested instanceof ModuleComponentSelector && - it.requested.group == 'io.airbyte' && - it.requested.module == 'airbyte-cdk' && + it.requested.group == 'io.airbyte' && + it.requested.module == 'airbyte-cdk' && it.requested.version.endsWith('-SNAPSHOT') } if (cdkSnapshotRequirement == null) { @@ -291,82 +157,46 @@ def getCDKSnapshotRequirement(dependenciesList) { return cdkSnapshotRequirement.requested.version } } -// Java projects common configurations -subprojects { subproj -> +// Common configurations for 'assemble'. +allprojects { - configurations { - runtimeClasspath + tasks.withType(Tar).configureEach { + duplicatesStrategy DuplicatesStrategy.INCLUDE } - - // Common Docker Configuration: - // If subprojects have the dockerImageName property configured in their gradle.properties file, - // register: - // 1) A copyGeneratedTar task to copy generated TARs. Subprojects that produce TARs can use this - // to copy the produced tar into the docker image. - // 2) A buildDockerImage task to build the docker image. - // 3) Make the docker image task depend on the default assemble task. - if (subproj.hasProperty('dockerImageName')) { - project.logger.lifecycle("configuring docker task for $subproj.name") - - // Although this task is defined for every subproject with the dockerImageName property, - // It is not necessarily used for all subprojects. Non-TAR producing projects can ignore this. - tasks.register("copyGeneratedTar", Copy) { - dependsOn copyDocker - dependsOn distTar - from('build/distributions') { - // Assume that tars are named --*.tar. - // Because we only have a handle to the child project, and to keep things simple, - // use a * regex to catch all prefixes. - include "*$subproj.name-*.tar" - } - into 'build/docker/bin' - } - - tasks.register("buildDockerImage", DockerBuildImage) { - // This is currently only used for connectors. - def jdkVersion = System.getenv('JDK_VERSION') ?: '17.0.4' - - def arch = System.getenv('BUILD_ARCH') ?: System.getProperty("os.arch").toLowerCase() - def isArm64 = arch == "aarch64" || arch == "arm64" - def buildArch = System.getenv('DOCKER_BUILD_ARCH') ?: isArm64 ? 'arm64' : 'amd64' - def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: isArm64 ? 'linux/arm64' : 'linux/amd64' - def alpineImage = System.getenv('ALPINE_IMAGE') ?: isArm64 ? 'arm64v8/alpine:3.14' : 'amd64/alpine:3.14' - def nginxImage = System.getenv('NGINX_IMAGE') ?: isArm64 ? 'arm64v8/nginx:alpine' : 'amd64/nginx:alpine' - - // Used by the platform -- Must be an Amazon Corretto-based image for build to work without modification to Dockerfile instructions - def openjdkImage = System.getenv('JDK_IMAGE') ?: "airbyte/airbyte-base-java-image:1.0" - - platform = buildPlatform - images.add("airbyte/$subproj.dockerImageName:$rootProject.ext.image_tag") - buildArgs.put('JDK_VERSION', jdkVersion) - buildArgs.put('DOCKER_BUILD_ARCH', buildArch) - buildArgs.put('ALPINE_IMAGE', alpineImage) - buildArgs.put('NGINX_IMAGE', nginxImage) - buildArgs.put('JDK_IMAGE', openjdkImage) - buildArgs.put('VERSION', rootProject.ext.version) - - } - - tasks.named("assemble") { - dependsOn buildDockerImage - } + tasks.withType(Zip).configureEach { + duplicatesStrategy DuplicatesStrategy.INCLUDE + // Disabling distZip causes the build to break for some reason, so: instead of disabling it, make it fast. + entryCompression ZipEntryCompression.STORED } +} + +// Java projects common configurations. +subprojects { subproj -> - if (subproj.name == 'airbyte-webapp' || subproj.name == 'airbyte-webapp-e2e-tests') { + if (!subproj.file('src/main/java').directory) { return } apply plugin: 'java' apply plugin: 'jacoco' apply plugin: 'com.github.spotbugs' - apply plugin: 'pmd' - sourceCompatibility = JavaVersion.VERSION_17 - targetCompatibility = JavaVersion.VERSION_17 + java { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + compileJava { + options.compilerArgs += ["-Werror", "-Xlint:all,-serial,-processing"] + } + compileTestJava { + //rawtypes and unchecked are necessary for mockito + //deprecation and removal are removed from error since we should still test those constructs. + options.compilerArgs += ["-Werror", "-Xlint:all,-serial,-processing,-rawtypes,-unchecked,-deprecation,-removal"] + } + } - if (subproj.name.startsWith("source-") || subproj.name.startsWith("destination-")) { + if (isConnectorProject(subproj)) { // This is a Java connector project. // Evaluate CDK project before evaluating the connector. @@ -383,38 +213,21 @@ subprojects { subproj -> } } - repositories { - mavenCentral() - maven { - url 'https://jitpack.io' - } - maven { - url 'https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/' - } - mavenLocal() - } - - pmd { - consoleOutput = true - ignoreFailures = false - rulesMinimumPriority = 5 - ruleSets = [] - ruleSetFiles = files(rootProject.file('tools/gradle/pmd/rules.xml')) - toolVersion = '6.51.0' - } - jacoco { toolVersion = "0.8.8" } jacocoTestReport { - dependsOn test reports { html.required = true xml.required = true csv.required = false } } + def jacocoTestReportTask = tasks.named('jacocoTestReport') + jacocoTestReportTask.configure { + dependsOn tasks.named('test') + } jacocoTestCoverageVerification { violationRules { @@ -434,86 +247,98 @@ subprojects { subproj -> } } - // make tag accessible in submodules. - ext { - cloudStorageTestTagName = 'cloud-storage' - numberThreads = project.hasProperty('numberThreads') ? project.getProperty('numberThreads') as int : Runtime.runtime.availableProcessors() ?: 1 - } - spotbugs { ignoreFailures = false effort = 'max' - excludeFilter = rootProject.file('spotbugs-exclude-filter-file.xml') + excludeFilter.set rootProject.file('spotbugs-exclude-filter-file.xml') reportLevel = 'high' showProgress = false toolVersion = '4.7.3' + if (rootProject.ext.skipSlowTests && isConnectorProject(subproj)) { + effort = 'min' + } } test { - //This allows to set up a `gradle.properties` file inside the connector folder to reduce number of threads and reduce parallelization. - //Specially useful for connectors that shares resources (like Redshift or Snowflake). - maxParallelForks = numberThreads - jacoco { - enabled = true - excludes = ['**/*Test*', '**/generated*'] - } - useJUnitPlatform { - excludeTags(cloudStorageTestTagName) - } + useJUnitPlatform() testLogging() { - events "passed", "skipped", "failed" + events 'skipped', 'started', 'passed', 'failed' exceptionFormat 'full' - // uncomment to get the full log output - // showStandardStreams = true + // Swallow the logs when running in airbyte-ci, rely on test reports instead. + showStandardStreams = !System.getenv().containsKey("RUN_IN_AIRBYTE_CI") } - finalizedBy jacocoTestReport - } - task allTests(type: Test) { - useJUnitPlatform() - testLogging() { - events "passed", "failed", "started" - exceptionFormat 'full' - // uncomment to get the full log output - // showStandardStreams = true + // Set the timezone to UTC instead of picking up the host machine's timezone, + // which on a developer's laptop is more likely to be PST. + systemProperty 'user.timezone', 'UTC' + + // Enable parallel test execution in JUnit by default. + // This is to support @Execution(ExecutionMode.CONCURRENT) annotations + // See https://junit.org/junit5/docs/current/user-guide/#writing-tests-parallel-execution for details. + systemProperty 'junit.jupiter.execution.parallel.enabled', 'true' + // Concurrency takes place at the class level. + systemProperty 'junit.jupiter.execution.parallel.mode.classes.default', 'concurrent' + // Within a class, the test methods are still run serially on the same thread. + systemProperty 'junit.jupiter.execution.parallel.mode.default', 'same_thread' + // Effectively disable JUnit concurrency by running tests in only one thread by default. + systemProperty 'junit.jupiter.execution.parallel.config.strategy', 'fixed' + systemProperty 'junit.jupiter.execution.parallel.config.fixed.parallelism', 1 + // Order test classes by annotation. + systemProperty 'junit.jupiter.testclass.order.default', 'org.junit.jupiter.api.ClassOrderer$OrderAnnotation' + + if (!subproj.hasProperty('testExecutionConcurrency')) { + // By default, let gradle spawn as many independent workers as it wants. + maxParallelForks = Runtime.runtime.availableProcessors() + maxHeapSize = '3G' + } else { + // Otherwise, run tests within the same JVM. + // Let gradle spawn only one worker. + maxParallelForks = 1 + maxHeapSize = '8G' + // Manage test execution concurrency in JUnit. + String concurrency = subproj.property('testExecutionConcurrency').toString() + if (concurrency.isInteger() && (concurrency as int) > 0) { + // Define a fixed number of threads when the property is set to a positive integer. + systemProperty 'junit.jupiter.execution.parallel.config.fixed.parallelism', concurrency + } else { + // Otherwise let JUnit manage the concurrency dynamically. + systemProperty 'junit.jupiter.execution.parallel.config.strategy', 'dynamic' + } } - finalizedBy jacocoTestReport - } - dependencies { - if (subproj.name != 'airbyte-commons') { - implementation project(':airbyte-commons') + // Exclude all connector unit tests upon request. + if (rootProject.ext.skipSlowTests) { + exclude '**/io/airbyte/integrations/source/**' + exclude '**/io/airbyte/integrations/destination/**' + } + + jacoco { + enabled = !rootProject.ext.skipSlowTests + excludes = ['**/*Test*', '**/generated*'] } + finalizedBy jacocoTestReportTask + } + + // TODO: These should be added to the CDK or to the individual projects that need them: + dependencies { implementation(platform("com.fasterxml.jackson:jackson-bom:2.13.0")) implementation(platform("org.glassfish.jersey:jersey-bom:2.31")) - // version is handled by "com.fasterxml.jackson:jackson-bom:2.10.4", so we do not explicitly set it here. implementation libs.bundles.jackson - implementation libs.guava - implementation libs.commons.io - implementation libs.bundles.apache - implementation libs.slf4j.api - // SLF4J as a facade over Log4j2 required dependencies implementation libs.bundles.log4j + implementation libs.appender.log4j2 // Bridges from other logging implementations to SLF4J implementation libs.bundles.slf4j - // Dependencies for logging to cloud storage, as well as the various clients used to do so. - implementation libs.appender.log4j2 - implementation libs.aws.java.sdk.s3 - implementation libs.google.cloud.storage - - implementation libs.s3 - // Lombok dependencies compileOnly libs.lombok annotationProcessor libs.lombok @@ -530,17 +355,12 @@ subprojects { subproj -> // adds owasp plugin spotbugsPlugins libs.findsecbugs.plugin implementation libs.spotbugs.annotations - } - tasks.withType(Tar) { - duplicatesStrategy DuplicatesStrategy.INCLUDE + // Airbyte dependencies. + implementation libs.airbyte.protocol } - tasks.withType(Zip) { - duplicatesStrategy DuplicatesStrategy.INCLUDE - } - - tasks.withType(SpotBugsTask) { + tasks.withType(SpotBugsTask).configureEach { // Reports can be found under each subproject in build/spotbugs/ reports { xml.required = false @@ -548,150 +368,109 @@ subprojects { subproj -> } } - tasks.withType(Pmd) { - exclude '**/generated/**' - exclude '**/jooq/**' - } - javadoc.options.addStringOption('Xdoclint:none', '-quiet') - check.dependsOn 'jacocoTestCoverageVerification' } - -// add licenses for python projects. -subprojects { - def pythonFormatTask = project.tasks.findByName('blackFormat') - - if (pythonFormatTask != null) { - apply plugin: "com.github.hierynomus.license" - license { - header rootProject.file("LICENSE_SHORT") - } - task licenseFormatPython(type: com.hierynomus.gradle.license.tasks.LicenseFormat) { - header = createPythonLicenseWith(rootProject.file('LICENSE_SHORT')) - source = fileTree(dir: projectDir) - .include("**/*.py") - .exclude(".venv/**/*.py") - .exclude("**/airbyte_api_client/**/*.py") - .exclude("**/__init__.py") - strictCheck = true - } - def licenseTask = project.tasks.findByName('licenseFormatPython') - def isortFormatTask = project.tasks.findByName('isortFormat') - if (isortFormatTask != null) { - isortFormat.dependsOn licenseTask - } - def flakeCheckTask = project.tasks.findByName('flakeCheck') - if (flakeCheckTask != null) { - flakeCheck.dependsOn licenseTask +// integration and performance test tasks per project +allprojects { + tasks.register('integrationTest') { + dependsOn tasks.matching { + [ + 'integrationTestJava', + 'integrationTestPython', + ].contains(it.name) } - blackFormat.dependsOn licenseTask + } - def generateManifestFilesTask = project.tasks.findByName('generateComponentManifestClassFiles') - if (generateManifestFilesTask != null) { - licenseTask.dependsOn generateManifestFilesTask + tasks.register('performanceTest') { + dependsOn tasks.matching { + [ + 'performanceTestJava', + ].contains(it.name) } } } -python { - envPath = '.venv' - minPythonVersion = '3.9' - scope = 'VIRTUALENV' - installVirtualenv = true - pip 'pip:21.3.1' - // https://github.com/csachs/pyproject-flake8/issues/13 - pip 'flake8:4.0.1' - // flake8 doesn't support pyproject.toml files - // and thus there is the wrapper "pyproject-flake8" for this - pip 'pyproject-flake8:0.0.1a2' - pip 'black:22.3.0' - pip 'mypy:1.4.1' - pip 'isort:5.6.4' - pip 'coverage[toml]:6.3.1' -} -task('generate') { - dependsOn subprojects.collect { it.getTasksByName('generateComponentManifestClassFiles', true) } - dependsOn subprojects.collect { it.getTasksByName('generateJsonSchema2Pojo', true) } -} - -license { - header rootProject.file("LICENSE_SHORT") +// convenience task to list all dependencies per project +subprojects { + tasks.register('listAllDependencies', DependencyReportTask) {} } -task licensePythonGlobalFormat(type: com.hierynomus.gradle.license.tasks.LicenseFormat) { - header = createPythonLicenseWith(rootProject.file('LICENSE_SHORT')) - source = fileTree(dir: rootProject.rootDir) - .include("**/*.py") - .exclude("**/.venv/**") - .exclude("**/build/**") - .exclude("**/node_modules/**") - .exclude("**/airbyte_api_client/**") - .exclude("**/__init__.py") - .exclude("airbyte-cdk/python/airbyte_cdk/sources/declarative/models/declarative_component_schema.py") - .exclude("airbyte-integrations/connectors/source-stock-ticker-api-tutorial/source.py") - .exclude("resources/examples/airflow/superset/docker/pythonpath_dev/superset_config.py") - .exclude("tools/git_hooks/tests/test_spec_linter.py") - .exclude("tools/schema_generator/schema_generator/infer_schemas.py") - strictCheck = true - dependsOn generate +// airbyte-ci tasks for local development +def poetryInstallAirbyteCI = tasks.register('poetryInstallAirbyteCI', Exec) { + workingDir rootProject.file('airbyte-ci/connectors/pipelines') + commandLine rootProject.file('.venv/bin/python').absolutePath + args "-m", "poetry", "install", "--no-cache" } - -task isortGlobalFormat(type: PythonTask) { - module = "isort" - command = "--settings-file=${rootProject.file('pyproject.toml').absolutePath} ./" - dependsOn generate +poetryInstallAirbyteCI.configure { + dependsOn tasks.named('pipInstall') } - -task blackGlobalFormat(type: PythonTask) { - module = "black" - // the line length should match .isort.cfg - command = "--config ${rootProject.file('pyproject.toml').absolutePath} ./" - dependsOn generate +def poetryCleanVirtualenv = tasks.register('cleanVirtualenv', Exec) { + workingDir rootProject.file('airbyte-ci/connectors/pipelines') + commandLine rootProject.file('.venv/bin/python').absolutePath + args "-m", "poetry", "env", "remove", "--all" + onlyIf { + rootProject.file('.venv/bin/python').exists() + } } - -task('format') { - dependsOn generate - dependsOn spotlessApply - dependsOn licensePythonGlobalFormat - dependsOn isortGlobalFormat - dependsOn blackGlobalFormat - tasks.findByName('spotlessApply').mustRunAfter 'generate' - tasks.findByName('licensePythonGlobalFormat').mustRunAfter 'generate' - tasks.findByName('isortGlobalFormat').mustRunAfter 'licensePythonGlobalFormat' - tasks.findByName('blackGlobalFormat').mustRunAfter 'isortGlobalFormat' +cleanPythonVenv.configure { + dependsOn poetryCleanVirtualenv } subprojects { - task listAllDependencies(type: DependencyReportTask) {} + if (!isConnectorProject(project)) { + return + } + def airbyteCIConnectorsTask = { String taskName, String... connectorsArgs -> + def task = tasks.register(taskName, Exec) { + workingDir rootDir + environment "CI", "1" // set to use more suitable logging format + commandLine rootProject.file('.venv/bin/python').absolutePath + args "-m", "poetry" + args "--directory", "${rootProject.file('airbyte-ci/connectors/pipelines').absolutePath}" + args "run" + args "airbyte-ci", "connectors", "--name=${project.name}" + args connectorsArgs + // Forbid these kinds of tasks from running concurrently. + // We can induce serial execution by giving them all a common output directory. + outputs.dir rootProject.file("${rootProject.buildDir}/airbyte-ci-lock") + outputs.upToDateWhen { false } + } + task.configure { dependsOn poetryInstallAirbyteCI } + return task + } + + // Build connector image as part of 'assemble' task. + // This is required for local 'integrationTest' execution. + def buildConnectorImage = airbyteCIConnectorsTask( + 'buildConnectorImage', '--disable-report-auto-open', 'build', '--use-host-gradle-dist-tar') + buildConnectorImage.configure { + // Images for java projects always rely on the distribution tarball. + dependsOn tasks.matching { it.name == 'distTar' } + // Ensure that all files exist beforehand. + dependsOn tasks.matching { it.name == 'generate' } + } + tasks.named('assemble').configure { + // We may revisit the dependency on assemble but the dependency should always be on a base task. + dependsOn buildConnectorImage + } + + // Convenience tasks for local airbyte-ci execution. + airbyteCIConnectorsTask('airbyteCIConnectorBuild', 'build') + airbyteCIConnectorsTask('airbyteCIConnectorTest', 'test') } // produce reproducible archives // (see https://docs.gradle.org/current/userguide/working_with_files.html#sec:reproducible_archives) -tasks.withType(AbstractArchiveTask) { +tasks.withType(AbstractArchiveTask).configureEach { preserveFileTimestamps = false reproducibleFileOrder = true } -// definition for publishing +// pin dependency versions according to ./deps.toml catalog { versionCatalog { from(files("deps.toml")) } } - -publishing { - publications { - // This block is present for dependency catalog publishing. - maven(MavenPublication) { - groupId = 'io.airbyte' - artifactId = 'oss-catalog' - - from components.versionCatalog - // Gradle will by default use the subproject path as the group id and the subproject name as the artifact id. - // e.g. the subproject :airbyte-example:example-models is imported at io.airbyte.airbyte-config-oss:config-persistence:. - } - } - repositories.add(rootProject.repositories.getByName('cloudrepo')) -} diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 6b031e31d4e1..0e2dd4d1f248 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -3,13 +3,23 @@ plugins { } repositories { - jcenter() + // # Gradle looks for dependency artifacts in repositories listed in 'repositories' blocks in descending order. + + // ## Prefer repos controlled by Airbyte. + // TODO: add airbyte-controlled proxy repos here + + // ## Look into other, public repos. + // Gradle plugin portal. + gradlePluginPortal() + // Maven Central has most of everything. + mavenCentral() } dependencies { implementation 'ru.vyarus:gradle-use-python-plugin:2.3.0' + implementation 'org.apache.commons:commons-text:1.10.0' } -tasks.withType(Jar) { +tasks.withType(Jar).configureEach { duplicatesStrategy DuplicatesStrategy.INCLUDE } diff --git a/buildSrc/src/main/groovy/airbyte-connector-acceptance-test.gradle b/buildSrc/src/main/groovy/airbyte-connector-acceptance-test.gradle deleted file mode 100644 index b72aedb83ec2..000000000000 --- a/buildSrc/src/main/groovy/airbyte-connector-acceptance-test.gradle +++ /dev/null @@ -1,71 +0,0 @@ -import org.gradle.api.Plugin -import org.gradle.api.Project - - -class AirbyteConnectorAcceptanceTestPlugin implements Plugin { - - void apply(Project project) { - - project.task('connectorAcceptanceTest') { - dependsOn project.build - dependsOn project.airbyteDocker - - if (!project.hasProperty('connectorAcceptanceTestVersion')) { - project.ext.connectorAcceptanceTestVersion = 'latest' - } - - if (project.connectorAcceptanceTestVersion == 'dev') { - project.connectorAcceptanceTest.dependsOn(':airbyte-integrations:bases:connector-acceptance-test:airbyteDocker') - } - - doFirst { - if (project.connectorAcceptanceTestVersion != 'dev') { - project.exec { - def args = [ - 'docker', 'pull', "airbyte/connector-acceptance-test:${project.connectorAcceptanceTestVersion}" - ] - commandLine args - } - } - project.exec { - def targetMountDirectory = "/test_input" - def args = [ - 'docker', 'run', '--rm', '-i', - // provide access to the docker daemon - '-v', '/var/run/docker.sock:/var/run/docker.sock', - // A container within a container mounts from the host filesystem, not the parent container. - // this forces /tmp to be the same directory for host, parent container, and child container. - '-v', '/tmp:/tmp', - // mount the project dir. all provided input paths must be relative to that dir. - '-v', "${project.projectDir.absolutePath}:${targetMountDirectory}", - '-w', "$targetMountDirectory", - '-e', "AIRBYTE_SAT_CONNECTOR_DIR=${project.projectDir.absolutePath}", - "airbyte/connector-acceptance-test:${project.connectorAcceptanceTestVersion}", - ] - if (project.file('integration_tests/acceptance.py').exists()) { - // if integration_tests/acceptance.py exists, use it for setup/tear down external test dependencies - args.add('-p') - args.add('integration_tests.acceptance') - } - commandLine args - } - } - - outputs.upToDateWhen { false } - } - - - - if (project.hasProperty('airbyteDockerTest')){ - project.connectorAcceptanceTest.dependsOn(project.airbyteDockerTest) - } - - // make sure we create the integrationTest task once - if (!project.tasks.findByName('integrationTest')) { - project.task('integrationTest') - } - - project.integrationTest.dependsOn(project.connectorAcceptanceTest) - } -} - diff --git a/buildSrc/src/main/groovy/airbyte-docker-legacy.gradle b/buildSrc/src/main/groovy/airbyte-docker-legacy.gradle new file mode 100644 index 000000000000..e323cf7c95cc --- /dev/null +++ b/buildSrc/src/main/groovy/airbyte-docker-legacy.gradle @@ -0,0 +1,331 @@ +import java.nio.file.Paths +import java.security.MessageDigest +import java.util.concurrent.ConcurrentHashMap +import org.apache.commons.text.StringSubstitutor +import org.gradle.api.DefaultTask +import org.gradle.api.GradleException +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.file.ConfigurableFileTree +import org.gradle.api.file.FileCollection +import org.gradle.api.tasks.CacheableTask +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFile +import org.gradle.api.tasks.InputFiles +import org.gradle.api.tasks.OutputFile +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity +import org.gradle.api.tasks.TaskAction + +/** + * AirbyteDockerLegacyTask is the task which builds a docker image based on a Dockerfile. + * + * It and the other classes in this file have "Legacy" in their name because we want to get rid of this plugin in favor + * of dagger-pipeline-based tooling like `airbyte-ci`. As of the time of this writing this is already the case for + * connectors. There are still a few remaining usages outside of connectors and they are useful to support a smooth + * local java-centric development experience with gradle, especially around integration tests. + * + * Issue https://github.com/airbytehq/airbyte/issues/30708 tracks the complete removal of this plugin. + */ +@CacheableTask +abstract class AirbyteDockerLegacyTask extends DefaultTask { + + @InputFiles + @PathSensitive(PathSensitivity.RELATIVE) + FileCollection filesInDockerImage + + @Input + Map baseImageHashes + + @InputFile + @PathSensitive(PathSensitivity.RELATIVE) + File dockerFile + + @OutputFile + File idFileOutput + + @InputFile + @PathSensitive(PathSensitivity.RELATIVE) + File buildScript = project.rootProject.file('tools/bin/build_image.sh') + + @TaskAction + def dockerTask() { + project.exec { + commandLine( + buildScript.absolutePath, + project.rootDir.absolutePath, + project.projectDir.absolutePath, + dockerFile.name, + DockerHelpers.getDevTaggedImage(project.projectDir, dockerFile.name), + idFileOutput.absolutePath, + ) + } + } +} + +/** + * AirbyteDockerLegacyTaskFactory is a convenience object to avoid passing the current project around. + */ +class AirbyteDockerLegacyTaskFactory { + + private AirbyteDockerLegacyTaskFactory() {} + + Project project + String dockerFileName + + File dockerFile() { + return project.file(dockerFileName) + } + + // This hash of the full path to the Dockerfile is the name of the task's output file. + String dockerfilePathHash() { + return MessageDigest.getInstance("MD5") + .digest(dockerFile().absolutePath.getBytes()) + .encodeHex() + .toString() + } + + // A superset of the files which are COPYed into the image, defined as the project file set + // with the .dockerignore rules applied to it. + // We could be more precise by parsing the Dockerfile but this is good enough in practice. + FileCollection filteredProjectFiles() { + ConfigurableFileTree files = project.fileTree(project.projectDir) + def dockerignore = project.file('.dockerignore') + if (!dockerignore.exists()) { + return files.filter { + file -> !file.toString().contains(".venv") + } + } + for (def rule : dockerignore.readLines()) { + if (rule.startsWith("#")) { + continue + } + rule = rule.trim() + files = (rule.startsWith("!") ? files.include(rule.substring(1)) : files.exclude(rule)) as ConfigurableFileTree + } + return files + } + + // Queries docker for all images and their hashes. + static synchronized Map collectKnownImageHashes(Project project) { + def stdout = new ByteArrayOutputStream() + project.rootProject.exec { + commandLine "docker", "images", "--no-trunc", "-f", "dangling=false", "--format", "{{.Repository}}:{{.Tag}} {{.ID}}" + standardOutput = stdout + } + Map map = [:] + stdout.toString().eachLine {line -> + def splits = line.split() + map.put(splits[0], splits[1].trim()) + } + return map + } + + // Query all docker images at most once for all tasks, at task creation time. + static def lazyImageHashesAtTaskCreationTime = new LazyImageHashesCache() + + static class LazyImageHashesCache { + private Map lazyValue + + synchronized Map get(Project project) { + if (lazyValue == null) { + lazyValue = collectKnownImageHashes(project) + } + return lazyValue + } + } + + // Global mapping of tagged image name to gradle project. + // This is populated at configuration time and accessed at task creation time. + // All keys verify isTaggedImageOwnedByThisRepo. + static def taggedImageToProject = new ConcurrentHashMap() + + static boolean isTaggedImageOwnedByThisRepo(String taggedImage) { + if (!taggedImage.startsWith("airbyte/")) { + // Airbyte's docker images are all prefixed like this. + // Anything not with this prefix is therefore not owned by this repo. + return false + } + if (taggedImage.startsWith("airbyte/base-airbyte-protocol-python:")) { + // Special case: this image is not built by this repo. + return false + } + if (!taggedImage.endsWith(":dev")) { + // Special case: this image is owned by this repo but built separate. e.g. source-file-secure + return false + } + // Otherwise, assume the image is built by this repo. + return true + } + + // Returns a mapping of each base image referenced in the Dockerfile to the corresponding hash + // in the results of collectKnownImageHashes(). If no hash was found, map to "???" instead. + Map baseTaggedImagesAndHashes(Map allKnownImageHashes) { + def taggedImages = new HashSet() + + // Look for "FROM foo AS bar" directives, and add them to the map with .put("bar", "foo") + Map imageAliases = [:] + dockerFile().eachLine { line -> + def parts = line.split() + if (parts.length >= 4 && parts[0].equals("FROM") && parts[parts.length - 2].equals("AS")) { + imageAliases.put(parts[parts.length - 1], parts[1]) + } + } + + dockerFile().eachLine { line -> + if (line.startsWith("FROM ")) { + def image = line.split()[1] + assert !image.isEmpty() + taggedImages.add(image) + } else if (line.startsWith("COPY --from=")) { + def image = line.substring("COPY --from=".length()).split()[0] + assert !image.isEmpty() + if (imageAliases[image] != null) { + taggedImages.add(imageAliases[image]) + } else { + taggedImages.add(image) + } + } + } + + Map result = [:] + for (def taggedImage : taggedImages) { + // Some image tags rely on environment variables (e.g. "FROM amazoncorretto:${JDK_VERSION}"). + taggedImage = new StringSubstitutor(System.getenv()).replace(taggedImage).trim() + result.put(taggedImage, allKnownImageHashes.getOrDefault(taggedImage, "???")) + } + return result + } + + // Create the task lazily: we shouldn't invoke 'docker' unless the task is created as part of the build. + def createTask(String taskName) { + if (!dockerFile().exists()) { + // This might not actually be necessary. It doesn't seem harmful either. + return project.tasks.register(taskName) { + logger.info "Skipping ${taskName} because ${dockerFile()} does not exist." + } + } + + // Tagged name of the image to be built by this task. + def taggedImage = DockerHelpers.getDevTaggedImage(project.projectDir, dockerFileName) + // Map this project to the tagged name of the image built by this task. + taggedImageToProject.put(taggedImage, project) + // Path to the ID file to be generated by this task. + // The ID file contains the hash of the image. + def idFilePath = Paths.get(project.rootProject.rootDir.absolutePath, '.dockerversions', dockerfilePathHash()) + // Register the task (lazy creation). + def airbyteDockerTask = project.tasks.register(taskName, AirbyteDockerLegacyTask) { task -> + // Set inputs. + task.filesInDockerImage = filteredProjectFiles() + task.dockerFile = this.dockerFile() + task.baseImageHashes = baseTaggedImagesAndHashes(lazyImageHashesAtTaskCreationTime.get(project)) + // Set dependencies on base images built by this repo. + for (String taggedImageDependency : task.baseImageHashes.keySet()) { + if (isTaggedImageOwnedByThisRepo(taggedImageDependency)) { + task.logger.info("adding airbyteDocker task dependency: image ${taggedImage} is based on ${taggedImageDependency}") + def dependentProject = taggedImageToProject.get(taggedImageDependency) + if (dependentProject == null) { + throw new GradleException("no known project for image ${taggedImageDependency}") + } + // Depend on 'assemble' instead of 'airbyteDocker' or 'airbyteDockerTest', it's simpler that way. + task.dependsOn(dependentProject.tasks.named('assemble')) + } + } + // Set outputs. + task.idFileOutput = idFilePath.toFile() + task.outputs.upToDateWhen { + // Because the baseImageHashes is computed at task creation time, it may be stale + // at task execution time. Let's double-check. + + // Missing dependency declarations in the gradle build may result in the airbyteDocker tasks + // to be created in the wrong order. Not worth breaking the build over. + for (Map.Entry e : task.baseImageHashes) { + if (isTaggedImageOwnedByThisRepo(e.key) && e.value == "???") { + task.logger.info "Not up to date: missing at least one airbyte base image in docker" + return false + } + } + // Fetch the hashes of the required based images anew. + def allImageHashes = collectKnownImageHashes(task.project) + // If the image to be built by this task doesn't exist in docker, then it definitely should + // be built regardless of the status of the ID file. + // For instance, it's possible that a `docker image rm` occurred between consecutive + // identical gradle builds: the ID file remains untouched but the image still needs to be rebuilt. + if (!allImageHashes.containsKey(taggedImage)) { + task.logger.info "Not up to date: ID file exists but target image not found in docker" + return false + } + // If the depended-upon base images have changed in the meantime, then it follows that the target + // image needs to be rebuilt regardless of the status of the ID file. + def currentBaseImageHashes = baseTaggedImagesAndHashes(allImageHashes) + if (!task.baseImageHashes.equals(currentBaseImageHashes)) { + task.logger.info "Not up to date: at last one base image has changed in docker since task creation" + return false + } + // In all other cases, if the ID file hasn't been touched, then the task can be skipped. + return true + } + } + + airbyteDockerTask.configure { + // Images for java projects always rely on the distribution tarball. + dependsOn project.tasks.matching { it.name == 'distTar' } + // Ensure that all files exist beforehand. + dependsOn project.tasks.matching { it.name == 'generate' } + } + project.tasks.named('assemble').configure { + // We may revisit the dependency on assemble but the dependency should always be on a base task. + dependsOn airbyteDockerTask + } + // Add a task to clean up when doing a gradle clean. + // Don't actually mess with docker, just delete the output file. + def airbyteDockerCleanTask = project.tasks.register(taskName + "Clean", Delete) { + delete idFilePath + } + project.tasks.named('clean').configure { + dependsOn airbyteDockerCleanTask + } + return airbyteDockerTask + } + + static def build(Project project, String taskName, String dockerFileName) { + def f = new AirbyteDockerLegacyTaskFactory() + f.project = project + f.dockerFileName = dockerFileName + f.createTask(taskName) + } +} + +/** + * AirbyteDockerLegacyPlugin creates an airbyteDocker task for the project when a Dockerfile is present. + * + * Following the same logic, it creates airbyteDockerTest when Dockerfile.test is present, though + * that behavior is not used anywhere except in the source-mongo connector and is therefore deprecated + * through the use of airbyte-ci. + */ +class AirbyteDockerLegacyPlugin implements Plugin { + + void apply(Project project) { + AirbyteDockerLegacyTaskFactory.build(project, 'airbyteDocker', 'Dockerfile') + + // Used only for source-mongodb. Consider removing entirely. + if (project.name.endsWith('source-mongodb')) { + AirbyteDockerLegacyTaskFactory.build(project, 'airbyteDockerTest', 'Dockerfile.test') + } + + // Used for base-normalization. + if (project.name.endsWith('base-normalization')) { + ['airbyteDockerMSSql' : 'mssql', + 'airbyteDockerMySql' : 'mysql', + 'airbyteDockerOracle' : 'oracle', + 'airbyteDockerClickhouse': 'clickhouse', + 'airbyteDockerSnowflake' : 'snowflake', + 'airbyteDockerRedshift' : 'redshift', + 'airbyteDockerTiDB' : 'tidb', + 'airbyteDockerDuckDB' : 'duckdb' + ].forEach {taskName, customConnector -> + AirbyteDockerLegacyTaskFactory.build(project, taskName, "${customConnector}.Dockerfile") + } + } + } +} diff --git a/buildSrc/src/main/groovy/airbyte-docker.gradle b/buildSrc/src/main/groovy/airbyte-docker.gradle deleted file mode 100644 index 275bbcff1bea..000000000000 --- a/buildSrc/src/main/groovy/airbyte-docker.gradle +++ /dev/null @@ -1,251 +0,0 @@ -import org.gradle.api.DefaultTask -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.file.FileCollection -import org.gradle.api.tasks.CacheableTask -import org.gradle.api.tasks.Input -import org.gradle.api.tasks.InputDirectory -import org.gradle.api.tasks.InputFiles -import org.gradle.api.tasks.Internal -import org.gradle.api.tasks.OutputFile -import org.gradle.api.tasks.PathSensitive -import org.gradle.api.tasks.PathSensitivity -import org.gradle.api.tasks.TaskAction -import org.slf4j.Logger - -import java.nio.file.Path -import java.security.MessageDigest -import java.nio.file.Paths - -@CacheableTask -abstract class AirbyteDockerTask extends DefaultTask { - @Internal - abstract File rootDir - - @InputFiles - @PathSensitive(PathSensitivity.RELATIVE) - abstract FileCollection projectFiles - - @Input - abstract Set baseImageHashes - - @InputDirectory - @PathSensitive(PathSensitivity.RELATIVE) - abstract File projectDir - - @Input - String dockerfileName - - @Input - boolean followSymlinks = false - - @OutputFile - abstract File idFileOutput - - def buildDockerfile(String scriptPath, String fileName) { - if (project.file(fileName).exists()) { - def tag = DockerHelpers.getDevTaggedImage(projectDir, dockerfileName) - - def arch = System.getProperty("os.arch").toLowerCase() - def isArm64 = arch == "aarch64" || arch == "arm64" - def buildPlatform = System.getenv('DOCKER_BUILD_PLATFORM') ?: isArm64 ? 'linux/arm64' : 'amd64' - - project.exec { - commandLine scriptPath, rootDir.absolutePath, projectDir.absolutePath, dockerfileName, tag, idFileOutput.absolutePath, followSymlinks, buildPlatform - } - } - } - - def buildDockerfileWithLocalCdk(String scriptPath, String fileName) { - if (project.file(fileName).exists()) { - def tag = DockerHelpers.getDevTaggedImage(projectDir, dockerfileName) - project.exec { - environment "CONNECTOR_TAG", tag - environment "CONNECTOR_NAME", project.findProperty('connectorAcceptanceTest.connectorName') - commandLine scriptPath - } - } - } - - @TaskAction - def dockerTask() { - if ( - project.hasProperty('connectorAcceptanceTest.useLocalCdk') && - project.properties["connectorAcceptanceTest.useLocalCdk"] && - project.parent.project.name.equals("connectors") - ) { - def scriptPath = Paths.get(rootDir.absolutePath, 'airbyte-integrations/scripts/build-connector-image-with-local-cdk.sh').toString() - buildDockerfileWithLocalCdk(scriptPath, dockerfileName) - } - else { - def scriptPath = Paths.get(rootDir.absolutePath, 'tools/bin/build_image.sh').toString() - buildDockerfile(scriptPath, dockerfileName) - } - } -} - -class AirbyteDockerPlugin implements Plugin { - - static def getBaseTaggedImages(File dockerfile) { - def result = [] as Set - - // Look for "FROM foo AS bar" directives, and add them to the map with .put("bar", "foo") - Map imageAliases = [:] - dockerfile.eachLine { line -> - def parts = line.split() - if (parts.length >= 4 && parts[0].equals("FROM") && parts[parts.length - 2].equals("AS")) { - imageAliases.put(parts[parts.length - 1], parts[1]) - } - } - - dockerfile.eachLine { line -> - if (line.startsWith("FROM ")) { - def image = line.split()[1] - assert !image.isEmpty() - result.add(image) - } else if (line.startsWith("COPY --from=")) { - def image = line.substring("COPY --from=".length()).split()[0] - assert !image.isEmpty() - if (imageAliases[image] != null) { - result.add(imageAliases[image]) - } else { - result.add(image) - } - } - } - - return result - } - - static def getBaseImageHashes(Map imageToHash, File dockerfile) { - def result = [] as Set - - getBaseTaggedImages(dockerfile).forEach { taggedImage -> - result.add((String) imageToHash.get(taggedImage)) - } - - return result - } - - static String getImageHash(Project project, String taggedImage) { - def stdout = new ByteArrayOutputStream() - - project.exec { - commandLine "docker", "images", "--no-trunc", "-f", "dangling=false", "--format", "{{.ID}}", resolveEnvironmentVariables(project, taggedImage) - standardOutput = stdout; - } - - return "$stdout".toString().trim() - } - - // Some image tags rely on environment variables (e.g. "FROM amazoncorretto:${JDK_VERSION}"). - // dump those into a "sh -c 'echo ...'" command to resolve them (e.g. "amazoncorretto:17.0.4") - static String resolveEnvironmentVariables(Project project, String str) { - def stdout = new ByteArrayOutputStream() - - project.exec { - commandLine "sh", "-c", "echo " + str - standardOutput = stdout; - } - - return "$stdout".toString().trim() - } - - static boolean isUpToDate(Logger logger, File idFileOutput, Project project, String dockerFile, Path dockerPath) { - if (idFileOutput.exists()) { - def taggedImage = DockerHelpers.getDevTaggedImage(project.projectDir, dockerFile) - logger.debug "taggedImage " + taggedImage - - def current = getImageHash(project, taggedImage) - logger.debug "current " + current - def stored = (String) project.rootProject.imageToHash.get(taggedImage) - logger.debug "stored " + stored - - def notUpToDate = new ArrayList(getBaseTaggedImages(dockerPath.toFile())).any { baseImage -> - logger.debug "checking base image " + baseImage - def storedBase = (String) project.rootProject.imageToHash.get(resolveEnvironmentVariables(project, baseImage)) - def currentBase = getImageHash(project, baseImage) - - logger.debug "storedBase " + storedBase - logger.debug "currentBase " + currentBase - if (!currentBase.equals(storedBase)) { - logger.debug "did not match" - return true - } else { - logger.debug "did match" - return false - } - } - - if (notUpToDate) { - return false; - } - - logger.debug "stored " + stored - - def upToDate = current.equals(stored) - - logger.debug "uptodate " + upToDate.toString() - - return upToDate - } else { - return false - } - } - - static def createTask(Project project, String taskName, String dockerFile) { - if (project.file(dockerFile).exists()) { - def filteredProjectFiles = project.fileTree(project.projectDir).filter { - file -> !file.toString().contains(".venv") - } - - project.task(taskName, type: AirbyteDockerTask) { - def dockerPath = Paths.get(project.projectDir.absolutePath, dockerFile) - def hash = MessageDigest.getInstance("MD5").digest(dockerPath.getBytes()).encodeHex().toString() - dockerfileName = dockerFile - rootDir = project.rootProject.rootDir - projectDir = project.projectDir - projectFiles = filteredProjectFiles - idFileOutput = project.file(Paths.get(project.rootProject.rootDir.absolutePath, '.dockerversions', hash).toString()) - baseImageHashes = getBaseImageHashes(project.rootProject.imageToHash, dockerPath.toFile()) - dependsOn project.assemble - - outputs.upToDateWhen { - return isUpToDate(logger, idFileOutput, project, dockerFile, dockerPath) - } - } - } else { - project.task(taskName) { - logger.info "Skipping ${taskName} because ${dockerFile} does not exist." - } - } - } - - void apply(Project project) { - // set (and cache) global image to version map - project.rootProject.ext.imageToHash = { - if (!project.rootProject.hasProperty("imageToHash")) { - def imageToHash = [:] - def stdout = new ByteArrayOutputStream() - project.exec { - commandLine "docker", "images", "--no-trunc", "-f", "dangling=false", "--format", "{{.Repository}}:{{.Tag}} {{.ID}}" - standardOutput = stdout; - } - - "$stdout".eachLine { line -> - def splits = line.split() - imageToHash.put(splits[0], splits[1].trim()) - } - - return imageToHash - } else { - return project.rootProject.imageToHash - } - }() - - createTask(project, 'airbyteDocker', 'Dockerfile') - createTask(project, 'airbyteDockerTest', 'Dockerfile.test') - - project.build.dependsOn project.airbyteDocker - } -} diff --git a/buildSrc/src/main/groovy/airbyte-integration-test-java.gradle b/buildSrc/src/main/groovy/airbyte-integration-test-java.gradle index e650889c417c..1719bf93596a 100644 --- a/buildSrc/src/main/groovy/airbyte-integration-test-java.gradle +++ b/buildSrc/src/main/groovy/airbyte-integration-test-java.gradle @@ -14,58 +14,48 @@ class AirbyteIntegrationTestJavaPlugin implements Plugin { } } } - project.test.dependsOn('compileIntegrationTestJavaJava') + project.tasks.named('check').configure { + dependsOn project.tasks.matching { it.name == 'compileIntegrationTestJavaJava' } + dependsOn project.tasks.matching { it.name == 'spotbugsIntegrationTestJava' } + } project.configurations { integrationTestJavaImplementation.extendsFrom testImplementation integrationTestJavaRuntimeOnly.extendsFrom testRuntimeOnly } - project.task('integrationTestJava', type: Test) { - mustRunAfter project.test - + def integrationTestJava = project.tasks.register('integrationTestJava', Test) { testClassesDirs = project.sourceSets.integrationTestJava.output.classesDirs classpath += project.sourceSets.integrationTestJava.runtimeClasspath - useJUnitPlatform { - // todo (cgardens) - figure out how to de-dupe this exclusion with the one in build.gradle. - excludeTags 'log4j2-config', 'logger-client', 'cloud-storage' - } - + useJUnitPlatform() testLogging() { - events "passed", "failed", "started" - exceptionFormat "full" - // uncomment to get the full log output - // showStandardStreams = true - } - - outputs.upToDateWhen { false } - - if(project.hasProperty('airbyteDocker')) { - dependsOn project.airbyteDocker + events 'skipped', 'started', 'passed', 'failed' + exceptionFormat 'full' + // Swallow the logs when running in airbyte-ci, rely on test reports instead. + showStandardStreams = !System.getenv().containsKey("RUN_IN_AIRBYTE_CI") } - //This allows to set up a `gradle.properties` file inside the connector folder to reduce number of threads and reduce parallelization. - //Specially useful for connectors that shares resources (like Redshift or Snowflake). - ext.numberThreads = project.hasProperty('numberThreads') ? project.getProperty('numberThreads') as int : Runtime.runtime.availableProcessors() ?: 1 - maxHeapSize = '3g' - maxParallelForks = numberThreads + systemProperties = project.test.systemProperties + maxParallelForks = project.test.maxParallelForks + maxHeapSize = project.test.maxHeapSize - // This is needed to make the destination-snowflake tests succeed - https://github.com/snowflakedb/snowflake-jdbc/issues/589#issuecomment-983944767 - jvmArgs = ["--add-opens=java.base/java.nio=ALL-UNNAMED"] + // Tone down the JIT when running the containerized connector to improve overall performance. + // The JVM default settings are optimized for long-lived processes in steady-state operation. + // Unlike in production, the connector containers in these tests are always short-lived. + // It's very much worth injecting a JAVA_OPTS environment variable into the container with + // flags which will reduce startup time at the detriment of long-term performance. + environment 'JOB_DEFAULT_ENV_JAVA_OPTS', '-XX:TieredStopAtLevel=1' - systemProperties = [ - // Allow tests to set @Execution(ExecutionMode.CONCURRENT) - 'junit.jupiter.execution.parallel.enabled': 'true' - ] + // Always re-run integration tests no matter what. + outputs.upToDateWhen { false } } - - // make sure we create the integrationTest task once in case a standard source test was already initialized - if(!project.hasProperty('integrationTest')) { - project.task('integrationTest') + integrationTestJava.configure { + mustRunAfter project.tasks.named('check') + dependsOn project.tasks.matching { it.name == 'assemble' } + } + project.tasks.named('build').configure { + dependsOn integrationTestJava } - - project.integrationTest.dependsOn(project.integrationTestJava) - project.integrationTest.dependsOn(project.spotbugsMain) } } diff --git a/buildSrc/src/main/groovy/airbyte-java-cdk.gradle b/buildSrc/src/main/groovy/airbyte-java-cdk.gradle new file mode 100644 index 000000000000..68f1c95fdeb6 --- /dev/null +++ b/buildSrc/src/main/groovy/airbyte-java-cdk.gradle @@ -0,0 +1,70 @@ +/* +This class facilites detecting the Java CDK target version via readCdkTargetVersion(). +*/ + +import java.util.Properties +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.tasks.testing.Test + +class AirbyteJavaCdkPlugin implements Plugin { + + static String CDK_VERSION_FILE = "airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties" + + String readCdkTargetVersion(Project project) { + Properties cdkVersionProps = new Properties() + project.file("${project.rootDir}/${CDK_VERSION_FILE}").withInputStream { + cdkVersionProps.load(it) + } + return cdkVersionProps.getProperty('version') ?: 'undefined' + } + + @Override + void apply(Project project) { + project.ext.getCdkTargetVersion = { + return readCdkTargetVersion(project) + } + project.getTasks().create("disableLocalCdkRefs", DisableLocalCdkRefsTask.class) + project.getTasks().create("assertNotUsingLocalCdk", AssertNotUsingLocalCdkTask.class) + } + + public static class DisableLocalCdkRefsTask extends DefaultTask { + @TaskAction + public void disableLocalCdkRefs() { + // Step through the project tree and set useLocalCdk to false on all connectors + getProject().rootProject.fileTree(dir: '.', include: '**/build.gradle').forEach(file -> { + String content = file.getText() + if (content.contains("useLocalCdk = true")) { + content = content.replace("useLocalCdk = true", "useLocalCdk = false") + file.setText(content) + System.out.println("Updated " + file.getPath()) + } + }) + } + } + + public static class AssertNotUsingLocalCdkTask extends DefaultTask { + @TaskAction + public void assertNotUsingLocalCdk() { + List foundPaths = new ArrayList<>() + + for (File file : getProject().rootProject.fileTree(dir: '.', include: '**/build.gradle')) { + String content = file.getText() + if (content.contains("useLocalCdk = true")) { + System.err.println("Found usage of 'useLocalCdk = true' in " + file.getPath()) + foundPaths.add(file.getPath()) + } + } + + if (!foundPaths.isEmpty()) { + String errorMessage = String.format( + "Detected usage of 'useLocalCdk = true' in the following files:\n%s\n" + + "This must be set to 'false' before merging to the main branch. \n" + + "NOTE: You can run './gradlew disableLocalCdkRefs' to automatically set it to 'false' on all projects.", + String.join("\n", foundPaths) + ) + throw new RuntimeException(errorMessage) + } + } + } +} diff --git a/buildSrc/src/main/groovy/airbyte-java-connector.gradle b/buildSrc/src/main/groovy/airbyte-java-connector.gradle new file mode 100644 index 000000000000..9d8a60ed88c8 --- /dev/null +++ b/buildSrc/src/main/groovy/airbyte-java-connector.gradle @@ -0,0 +1,101 @@ +/* +Gradle plugin for Java-based Airbyte connectors. +Also facilitates importing and working with the Java CDK. +*/ + +import org.gradle.api.Plugin +import org.gradle.api.Project + +class AirbyteJavaConnectorExtension { + + boolean useLocalCdk + String cdkVersionRequired + List features = [] // e.g. 'db-sources', 'db-destinations' + Project project + + AirbyteJavaConnectorExtension(Project project) { + this.project = project + } + + void setUseLocalCdk(boolean useLocalCdk) { + this.useLocalCdk = useLocalCdk + addCdkDependencies() + } + + static final List IMPLEMENTATION = [ + 'airbyte-commons', + 'airbyte-json-validation', + 'airbyte-commons-cli', + 'airbyte-api', + 'config-models-oss', + 'init-oss', + ] + + static final List TEST_IMPLEMENTATION = [ + 'airbyte-commons', + 'airbyte-json-validation', + 'airbyte-api', + 'config-models-oss', + ] + + static final List INTEGRATION_TEST_IMPLEMENTATION = [ + 'config-models-oss', + 'init-oss', + 'acceptance-test-harness', + ] + + void addCdkDependencies() { + def projectName = { ":airbyte-cdk:java:airbyte-cdk:${it}" } + def jarName = { "io.airbyte.cdk:airbyte-cdk-${it}:${cdkVersionRequired}" } + project.dependencies { + def dep = { useLocalCdk ? project.project(projectName(it)) : jarName(it) } + def testFixturesDep = { useLocalCdk ? testFixtures(project.project(projectName(it))) : "${jarName(it)}:test-fixtures" } + + IMPLEMENTATION.each { + implementation dep(it) + testFixturesImplementation dep(it) + } + TEST_IMPLEMENTATION.each {testImplementation dep(it) } + INTEGRATION_TEST_IMPLEMENTATION.each {integrationTestJavaImplementation dep(it) } + (["core"] + features).each { + implementation dep(it) + testFixturesImplementation dep(it) + testFixturesImplementation testFixturesDep(it) + testImplementation dep(it) + testImplementation testFixturesDep(it) + integrationTestJavaImplementation dep(it) + integrationTestJavaImplementation testFixturesDep(it) + performanceTestJavaImplementation dep(it) + performanceTestJavaImplementation testFixturesDep(it) + } + } + } +} + + +class AirbyteJavaConnectorPlugin implements Plugin { + + @Override + void apply(Project project) { + + project.plugins.apply('java-test-fixtures') + project.plugins.apply(AirbyteIntegrationTestJavaPlugin) + project.plugins.apply(AirbytePerformanceTestJavaPlugin) + + project.configurations { + testFixturesImplementation.extendsFrom implementation + testFixturesRuntimeOnly.extendsFrom runtimeOnly + } + + project.dependencies { + // Integration and performance tests should automatically + // have access to the project's own main source sets. + integrationTestJavaImplementation project + integrationTestJavaImplementation testFixtures(project) + performanceTestJavaImplementation project + performanceTestJavaImplementation testFixtures(project) + } + + project.extensions.create('airbyteJavaConnector', AirbyteJavaConnectorExtension, project) + } +} diff --git a/buildSrc/src/main/groovy/airbyte-performance-test-java.gradle b/buildSrc/src/main/groovy/airbyte-performance-test-java.gradle index 38e8ab20fbdf..d5ccbbbd0bea 100644 --- a/buildSrc/src/main/groovy/airbyte-performance-test-java.gradle +++ b/buildSrc/src/main/groovy/airbyte-performance-test-java.gradle @@ -14,14 +14,17 @@ class AirbytePerformanceTestJavaPlugin implements Plugin { } } } - project.test.dependsOn('compilePerformanceTestJavaJava') + project.tasks.named('check').configure { + dependsOn project.tasks.matching { it.name == 'compilePerformanceTestJavaJava' } + dependsOn project.tasks.matching { it.name == 'spotbugsPerformanceTestJava' } + } project.configurations { performanceTestJavaImplementation.extendsFrom testImplementation performanceTestJavaRuntimeOnly.extendsFrom testRuntimeOnly } - project.task('performanceTestJava', type: Test) { + def performanceTestJava = project.tasks.register('performanceTestJava', Test) { testClassesDirs = project.sourceSets.performanceTestJava.output.classesDirs classpath += project.sourceSets.performanceTestJava.runtimeClasspath @@ -35,21 +38,11 @@ class AirbytePerformanceTestJavaPlugin implements Plugin { } outputs.upToDateWhen { false } - - if(project.hasProperty('airbyteDocker')) { - dependsOn project.airbyteDocker - } - maxHeapSize = '3g' - - mustRunAfter project.test } - - // make sure we create the performanceTest task once in case a standard source test was already initialized - if(!project.hasProperty('performanceTest')) { - project.task('performanceTest') + performanceTestJava.configure { + mustRunAfter project.tasks.named('check') + dependsOn project.tasks.matching { it.name == 'assemble' } } - - project.performanceTest.dependsOn(project.performanceTestJava) } } diff --git a/buildSrc/src/main/groovy/airbyte-python-docker.gradle b/buildSrc/src/main/groovy/airbyte-python-docker.gradle deleted file mode 100644 index 91eee0eeb355..000000000000 --- a/buildSrc/src/main/groovy/airbyte-python-docker.gradle +++ /dev/null @@ -1,38 +0,0 @@ -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.tasks.Exec - -/* -This plugin exists to build & test a python module via Gradle without requiring the developer to install Python on their machine. It achieves this by mounting the module into a Python Docker image and running the tests. - -Modules consuming this plugin must define two scripts `run_tests.sh` and `run_format.sh`. -*/ - -class AirbytePythonDockerConfiguration { - String moduleDirectory -} - -class AirbytePythonDockerPlugin implements Plugin { - - void apply(Project project) { - def extension = project.extensions.create('airbytePythonDocker', AirbytePythonDockerConfiguration) - - project.task('airbytePythonDockerApply', type: Exec) { - /* - Install the dependencies and run the tests from a docker container - */ - commandLine "docker", "run", "-v", "${project.projectDir.getAbsolutePath()}:/home", "--entrypoint", "/bin/bash", "python:3.9-slim", "-c", "chmod +x /home/run_tests.sh && /home/run_tests.sh /home" - } - - project.task('blackFormat', type: Exec) { - /* - Run formatter and static analysis from a docker container - */ - commandLine "docker", "run", "-v", "${project.projectDir.getAbsolutePath()}:/home", "--entrypoint", "/bin/bash", "python:3.9-slim", "-c", "chmod +x /home/run_tests.sh && /home/run_format.sh /home" - } - - project.assemble.dependsOn project.airbytePythonDockerApply - project.test.dependsOn project.airbytePythonDockerApply - } -} - diff --git a/buildSrc/src/main/groovy/airbyte-python.gradle b/buildSrc/src/main/groovy/airbyte-python.gradle index b7884f661ee2..59f14890c75b 100644 --- a/buildSrc/src/main/groovy/airbyte-python.gradle +++ b/buildSrc/src/main/groovy/airbyte-python.gradle @@ -6,10 +6,6 @@ import org.gradle.api.Project import org.gradle.api.tasks.Exec import ru.vyarus.gradle.plugin.python.task.PythonTask -class AirbytePythonConfiguration { - String moduleDirectory -} - class Helpers { static addTestTaskIfTestFilesFound(Project project, String testFilesDirectory, String taskName, taskDependencies) { """ @@ -30,41 +26,29 @@ class Helpers { See https://github.com/airbytehq/airbyte/issues/4979 for original context """ + boolean requiresTasks = false if (project.file(testFilesDirectory).exists()) { - def outputArg = project.hasProperty('reports_folder') ?"-otemp_coverage.xml" : "--skip-empty" - def coverageFormat = project.hasProperty('reports_folder') ? 'xml' : 'report' - def testConfig = project.file('pytest.ini').exists() ? 'pytest.ini' : project.rootProject.file('pyproject.toml').absolutePath - project.projectDir.toPath().resolve(testFilesDirectory).traverse(type: FileType.FILES, nameFilter: ~/(^test_.*|.*_test)\.py$/) { file -> - project.task("_${taskName}Coverage", type: PythonTask, dependsOn: taskDependencies) { - module = "coverage" - command = "run --data-file=${testFilesDirectory}/.coverage.${taskName} --rcfile=${project.rootProject.file('pyproject.toml').absolutePath} -m pytest -s ${testFilesDirectory} -c ${testConfig}" - } - // generation of coverage report is optional and we should skip it if tests are empty - - project.task(taskName, type: Exec){ - commandLine = ".venv/bin/python" - args "-m", "coverage", coverageFormat, "--data-file=${testFilesDirectory}/.coverage.${taskName}", "--rcfile=${project.rootProject.file('pyproject.toml').absolutePath}", outputArg - dependsOn project.tasks.findByName("_${taskName}Coverage") - setIgnoreExitValue true - doLast { - // try to move a generated report to custom report folder if needed - if (project.file('temp_coverage.xml').exists() && project.hasProperty('reports_folder')) { - project.file('temp_coverage.xml').renameTo(project.file("${project.reports_folder}/coverage.xml")) - } - } - - } + def testDir = project.projectDir.toPath().resolve(testFilesDirectory) + testDir.traverse(type: FileType.FILES, nameFilter: ~/(^test_.*|.*_test)\.py$/) {file -> + requiresTasks = true // If a file is found, terminate the traversal, thus causing this task to be declared at most once return FileVisitResult.TERMINATE } } + if (!requiresTasks) { + return + } - // If the task doesn't exist then we didn't find a matching file. So add an empty task since other tasks will - // probably rely on this one existing. - if (!project.hasProperty(taskName)) { - project.task(taskName) { - logger.info "Skipping task ${taskName} because ${testFilesDirectory} doesn't exist." - } + def coverageTask = project.tasks.register(taskName, PythonTask) { + def dataFile = "${testFilesDirectory}/.coverage.${taskName}" + def rcFile = project.rootProject.file('pyproject.toml').absolutePath + def testConfig = project.file('pytest.ini').exists() ? 'pytest.ini' : project.rootProject.file('pyproject.toml').absolutePath + + module = "coverage" + command = "run --data-file=${dataFile} --rcfile=${rcFile} -m pytest -s ${testFilesDirectory} -c ${testConfig}" + } + coverageTask.configure { + dependsOn taskDependencies } } } @@ -72,79 +56,67 @@ class Helpers { class AirbytePythonPlugin implements Plugin { void apply(Project project) { - def extension = project.extensions.create('airbytePython', AirbytePythonConfiguration) def venvDirectoryName = '.venv' + + // Add a task that allows cleaning up venvs to every python project + def cleanPythonVenv = project.tasks.register('cleanPythonVenv', Exec) { + commandLine 'rm' + args '-rf', "${project.projectDir.absolutePath}/${venvDirectoryName}" + } + project.tasks.named('clean').configure { + dependsOn cleanPythonVenv + } + project.plugins.apply 'ru.vyarus.use-python' + // Configure gradle python plugin. project.python { envPath = venvDirectoryName - minPythonVersion = '3.9' - scope = 'VIRTUALENV' + minPythonVersion '3.10' + + // Amazon Linux support. + // The airbyte-ci tool runs gradle tasks in AL2023-based containers. + // In AL2023, `python3` is necessarily v3.9, and later pythons need to be installed and named explicitly. + // See https://github.com/amazonlinux/amazon-linux-2023/issues/459 for details. + try { + if ("python3.11 --version".execute().waitFor() == 0) { + // python3.11 definitely exists at this point, use it instead of 'python3'. + pythonBinary "python3.11" + } + } catch (IOException _) { + // Swallow exception if python3.11 is not installed. + } + // Pyenv support. + try { + def pyenvRoot = "pyenv root".execute() + def pyenvLatest = "pyenv latest ${minPythonVersion}".execute() + // Pyenv definitely exists at this point: use 'python' instead of 'python3' in all cases. + pythonBinary "python" + if (pyenvRoot.waitFor() == 0 && pyenvLatest.waitFor() == 0) { + pythonPath "${pyenvRoot.text.trim()}/versions/${pyenvLatest.text.trim()}/bin" + } + } catch (IOException _) { + // Swallow exception if pyenv is not installed. + } + + scope 'VIRTUALENV' installVirtualenv = true - pip 'pip:21.3.1' + pip 'pip:23.2.1' pip 'mccabe:0.6.1' // https://github.com/csachs/pyproject-flake8/issues/13 pip 'flake8:4.0.1' // flake8 doesn't support pyproject.toml files // and thus there is the wrapper "pyproject-flake8" for this pip 'pyproject-flake8:0.0.1a2' - pip 'black:22.3.0' - pip 'mypy:1.4.1' - pip 'isort:5.6.4' pip 'pytest:6.2.5' pip 'coverage[toml]:6.3.1' } - - project.task('isortFormat', type: PythonTask) { - module = "isort" - command = "--settings-file=${project.rootProject.file('pyproject.toml').absolutePath} ./" - } - - project.task('isortReport', type: PythonTask) { - module = "isort" - command = "--settings-file=${project.rootProject.file('pyproject.toml').absolutePath} --diff --quiet ./" - outputPrefix = '' - } - - project.task('blackFormat', type: PythonTask) { - module = "black" - // the line length should match .isort.cfg - command = "--config ${project.rootProject.file('pyproject.toml').absolutePath} ./" - dependsOn project.rootProject.licenseFormat - dependsOn project.isortFormat - } - - project.task('blackReport', type: PythonTask) { - module = "black" - command = "--config ${project.rootProject.file('pyproject.toml').absolutePath} --diff --quiet ./" - outputPrefix = '' - } - - project.task('flakeCheck', type: PythonTask, dependsOn: project.blackFormat) { - module = "pflake8" - command = "--config ${project.rootProject.file('pyproject.toml').absolutePath} ./" - } - - project.task('flakeReport', type: PythonTask) { - module = "pflake8" - command = "--exit-zero --config ${project.rootProject.file('pyproject.toml').absolutePath} ./" - outputPrefix = '' - } - - project.task("mypyReport", type: Exec){ - commandLine = ".venv/bin/python" - args "-m", "mypy", "--config-file", "${project.rootProject.file('pyproject.toml').absolutePath}", "./" - setIgnoreExitValue true - } - - - - // attempt to install anything in requirements.txt. by convention this should only be dependencies whose source is located in the project. - + // Attempt to install anything in requirements.txt. + // By convention this should only be dependencies whose source is located in the project. if (project.file('requirements.txt').exists()) { - project.task('installLocalReqs', type: PythonTask) { + project.tasks.register('installLocalReqs', PythonTask) { module = "pip" command = "install -r requirements.txt" inputs.file('requirements.txt') @@ -153,155 +125,61 @@ class AirbytePythonPlugin implements Plugin { } else if (project.file('setup.py').exists()) { // If requirements.txt does not exists, install from setup.py instead, assume a dev or "tests" profile exists. // In this case, there is no need to depend on the base python modules since everything should be contained in the setup.py. - project.task('installLocalReqs', type: PythonTask) { + project.tasks.register('installLocalReqs', PythonTask) { module = "pip" command = "install .[dev,tests]" + inputs.file('setup.py') + outputs.file('build/installedlocalreqs.txt') } } else { - throw new GradleException('Error: Python module lacks requirement.txt and setup.py') + return } - project.task('installReqs', type: PythonTask, dependsOn: project.installLocalReqs) { + def installLocalReqs = project.tasks.named('installLocalReqs') + + def flakeCheck = project.tasks.register('flakeCheck', PythonTask) { + module = "pflake8" + command = "--config ${project.rootProject.file('pyproject.toml').absolutePath} ./" + } + + def installReqs = project.tasks.register('installReqs', PythonTask) { module = "pip" command = "install .[main]" inputs.file('setup.py') outputs.file('build/installedreqs.txt') } + installReqs.configure { + dependsOn installLocalReqs + } - project.task('installTestReqs', type: PythonTask, dependsOn: project.installReqs) { + project.tasks.named('check').configure { + dependsOn installReqs + dependsOn flakeCheck + } + + def installTestReqs = project.tasks.register('installTestReqs', PythonTask) { module = "pip" command = "install .[tests]" inputs.file('setup.py') outputs.file('build/installedtestreqs.txt') } - - Helpers.addTestTaskIfTestFilesFound(project, 'unit_tests', 'unitTest', project.installTestReqs) - - Helpers.addTestTaskIfTestFilesFound(project, 'integration_tests', 'customIntegrationTests', project.installTestReqs) - if (!project.tasks.findByName('integrationTest')) { - project.task('integrationTest') - } - project.integrationTest.dependsOn(project.customIntegrationTests) - - if (extension.moduleDirectory) { - project.task('mypyCheck', type: PythonTask) { - module = "mypy" - command = "-m ${extension.moduleDirectory} --config-file ${project.rootProject.file('pyproject.toml').absolutePath}" - } - - project.check.dependsOn mypyCheck + installTestReqs.configure { + dependsOn installReqs } - project.task('airbytePythonFormat', type: DefaultTask) { - dependsOn project.blackFormat - dependsOn project.isortFormat - dependsOn project.flakeCheck + Helpers.addTestTaskIfTestFilesFound(project, 'unit_tests', 'testPython', installTestReqs) + project.tasks.named('check').configure { + dependsOn project.tasks.matching { it.name == 'testPython' } } - project.task('airbytePythonReport', type: DefaultTask) { - dependsOn project.blackReport - dependsOn project.isortReport - dependsOn project.flakeReport - dependsOn project.mypyReport - doLast { - if (project.hasProperty('reports_folder')) { - // Gradles adds some log messages to files and we must remote them - // examples of these lines: - // :airbyte-integrations:connectors: ... - // [python] .venv/bin/python -m black ... - project.fileTree(project.reports_folder).visit { FileVisitDetails details -> - project.println "Found the report file: " + details.file.path - def tempFile = project.file(details.file.path + ".1") - details.file.eachLine { line -> - if ( !line.startsWith(":airbyte") && !line.startsWith("[python]") ) { - tempFile << line + "\n" - } - } - if (!tempFile.exists()) { - // generate empty file - tempFile << "\n" - } - tempFile.renameTo(details.file) - - } - } - } - } - - project.task('airbytePythonApply', type: DefaultTask) { - dependsOn project.installReqs - dependsOn project.airbytePythonFormat + Helpers.addTestTaskIfTestFilesFound(project, 'integration_tests', 'integrationTestPython', installTestReqs) + def integrationTestTasks = project.tasks.matching { it.name == 'integrationTestPython' } + integrationTestTasks.configureEach { + dependsOn project.tasks.named('assemble') + mustRunAfter project.tasks.named('check') } - - - project.task('airbytePythonTest', type: DefaultTask) { - dependsOn project.airbytePythonApply - dependsOn project.installTestReqs - dependsOn project.unitTest - } - - // Add a task that allows cleaning up venvs to every python project - project.task('cleanPythonVenv', type: Exec) { - commandLine 'rm' - args '-rf', "$project.projectDir.absolutePath/$venvDirectoryName" - } - - // Add a task which can be run at the root project level to delete all python venvs - if (!project.rootProject.hasProperty('cleanPythonVenvs')) { - project.rootProject.task('cleanPythonVenvs') - } - project.rootProject.cleanPythonVenvs.dependsOn(project.cleanPythonVenv) - - project.assemble.dependsOn project.airbytePythonApply - project.assemble.dependsOn project.airbytePythonTest - project.test.dependsOn project.airbytePythonTest - - // saves tools reports to a custom folder - def reportsFolder = project.hasProperty('reports_folder') ? project.reports_folder : '' - if ( reportsFolder != '' ) { - - // clean reports folders - project.file(reportsFolder).deleteDir() - project.file(reportsFolder).mkdirs() - - - - project.tasks.blackReport.configure { - it.logging.addStandardOutputListener(new StandardOutputListener() { - @Override - void onOutput(CharSequence charSequence) { - project.file("$reportsFolder/black.diff") << charSequence - } - }) - } - project.tasks.isortReport.configure { - it.logging.addStandardOutputListener(new StandardOutputListener() { - @Override - void onOutput(CharSequence charSequence) { - project.file("$reportsFolder/isort.diff") << charSequence - } - }) - } - - project.tasks.flakeReport.configure { - it.logging.addStandardOutputListener(new StandardOutputListener() { - @Override - void onOutput(CharSequence charSequence) { - project.file("$reportsFolder/flake.txt") << charSequence - } - }) - } - - project.tasks.mypyReport.configure { - it.logging.addStandardOutputListener(new StandardOutputListener() { - @Override - void onOutput(CharSequence charSequence) { - project.file("$reportsFolder/mypy.log") << charSequence - } - }) - } - + project.tasks.named('build').configure { + dependsOn integrationTestTasks } } } - diff --git a/buildSrc/src/main/groovy/airbyte-standard-source-test-file.gradle b/buildSrc/src/main/groovy/airbyte-standard-source-test-file.gradle deleted file mode 100644 index 3eede9b6321b..000000000000 --- a/buildSrc/src/main/groovy/airbyte-standard-source-test-file.gradle +++ /dev/null @@ -1,83 +0,0 @@ -import org.gradle.api.Plugin -import org.gradle.api.Project - -abstract class AirbyteStandardSourceTestFileConfiguration { - Closure prehook = {} // default no-op - Closure posthook = {} - String configPath - String configuredCatalogPath - String specPath - String statePath -} - -class AirbyteStandardSourceTestFilePlugin implements Plugin { - static void assertNotNull(String param, String paramName) { - if (param == null || param == "") { - throw new IllegalArgumentException("${paramName} parameter must be provided") - } - } - - static void validateConfig(AirbyteStandardSourceTestFileConfiguration config) { - assertNotNull(config.configPath, 'configPath') - assertNotNull(config.specPath, 'specPath') - assertNotNull(config.configuredCatalogPath, 'configuredCatalogPath') - } - - void apply(Project project) { - def config = project.extensions.create('airbyteStandardSourceTestFile', AirbyteStandardSourceTestFileConfiguration) - - project.task('standardSourceTestFile') { - doFirst { - try { - config.prehook.call() - project.exec { - validateConfig(config) - def targetMountDirectory = "/test_input" - def args = [ - 'docker', 'run', '--rm', '-i', - // provide access to the docker daemon - '-v', "/var/run/docker.sock:/var/run/docker.sock", - // A container within a container mounts from the host filesystem, not the parent container. - // this forces /tmp to be the same directory for host, parent container, and child container. - '-v', "/tmp:/tmp", - // mount the project dir. all provided input paths must be relative to that dir. - '-v', "${project.projectDir.absolutePath}:${targetMountDirectory}", - '--name', "std-fs-source-test-${project.name}", - 'airbyte/base-standard-source-test-file:dev', - '--imageName', DockerHelpers.getDevTaggedImage(project.projectDir, 'Dockerfile'), - '--catalog', "${targetMountDirectory}/${config.configuredCatalogPath}", - '--spec', "${targetMountDirectory}/${config.specPath}", - '--config', "${targetMountDirectory}/${config.configPath}", - ] - - if (config.statePath != null) { - args.add("--state") - args.add("${targetMountDirectory}/${config.statePath}") - } - - commandLine args - } - } finally { - config.posthook.call() - } - } - - outputs.upToDateWhen { false } - } - - project.standardSourceTestFile.dependsOn(':airbyte-integrations:bases:base-standard-source-test-file:airbyteDocker') - project.standardSourceTestFile.dependsOn(project.build) - project.standardSourceTestFile.dependsOn(project.airbyteDocker) - if (project.hasProperty('airbyteDockerTest')){ - project.standardSourceTestFile.dependsOn(project.airbyteDockerTest) - } - - // make sure we create the integrationTest task once in case a java integration test was already initialized - if (!project.tasks.findByName('integrationTest')) { - project.task('integrationTest') - } - - project.integrationTest.dependsOn(project.standardSourceTestFile) - } -} - diff --git a/deps.toml b/deps.toml index 4a33456a7e48..fa3943bebe30 100644 --- a/deps.toml +++ b/deps.toml @@ -1,38 +1,29 @@ [versions] -airbyte-protocol = "0.3.6" +airbyte-protocol = "0.5.0" commons_io = "2.7" -connectors-destination-testcontainers-clickhouse = "1.17.3" -connectors-destination-testcontainers-elasticsearch = "1.17.3" -connectors-destination-testcontainers-oracle-xe = "1.17.3" -connectors-source-testcontainers-clickhouse = "1.17.3" -connectors-testcontainers = "1.15.3" -connectors-testcontainers-cassandra = "1.16.0" -connectors-testcontainers-mariadb = "1.16.2" -connectors-testcontainers-pulsar = "1.16.2" -connectors-testcontainers-scylla = "1.16.2" -connectors-testcontainers-tidb = "1.16.3" +testcontainers = "1.19.0" datadog-version = "0.111.0" -fasterxml_version = "2.14.0" +fasterxml_version = "2.15.2" flyway = "7.14.0" glassfish_version = "2.31" hikaricp = "5.0.1" jmh = "1.36" jooq = "3.13.4" junit-jupiter = "5.9.1" -log4j = "2.17.2" +kotlin = "1.9.0" +log4j = "2.21.1" lombok = "1.18.24" micronaut = "3.8.3" micronaut-data = "3.9.4" micronaut-jaxrs = "3.4.0" micronaut-security = "3.9.2" micronaut-test = "3.8.0" -platform-testcontainers = "1.17.3" -postgresql = "42.3.5" +postgresql = "42.6.0" reactor = "3.5.2" segment = "2.1.1" -slf4j = "1.7.36" +slf4j = "2.0.9" temporal = "1.17.0" -debezium = "2.2.0.Final" +debezium = "2.4.0.Final" [libraries] airbyte-protocol = { module = "io.airbyte.airbyte-protocol:protocol-models", version.ref = "airbyte-protocol" } @@ -42,25 +33,24 @@ appender-log4j2 = { module = "com.therealvan:appender-log4j2", version = "3.6.0" assertj-core = { module = "org.assertj:assertj-core", version = "3.21.0" } aws-java-sdk-s3 = { module = "com.amazonaws:aws-java-sdk-s3", version = "1.12.6" } commons-io = { module = "commons-io:commons-io", version.ref = "commons_io" } -connectors-destination-testcontainers-clickhouse = { module = "org.testcontainers:clickhouse", version.ref = "connectors-destination-testcontainers-clickhouse" } -connectors-destination-testcontainers-oracle-xe = { module = "org.testcontainers:oracle-xe", version.ref = "connectors-destination-testcontainers-oracle-xe" } -connectors-source-testcontainers-clickhouse = { module = "org.testcontainers:clickhouse", version.ref = "connectors-source-testcontainers-clickhouse" } -connectors-source-testcontainers-oracle-xe = { module = "org.testcontainers:oracle-xe", version.ref = "connectors-testcontainers" } -connectors-testcontainers = { module = "org.testcontainers:testcontainers", version.ref = "connectors-testcontainers" } -connectors-testcontainers-cassandra = { module = "org.testcontainers:cassandra", version.ref = "connectors-testcontainers-cassandra" } -connectors-testcontainers-cockroachdb = { module = "org.testcontainers:cockroachdb", version.ref = "connectors-testcontainers" } -connectors-testcontainers-db2 = { module = "org.testcontainers:db2", version.ref = "connectors-testcontainers" } -connectors-testcontainers-elasticsearch = { module = "org.testcontainers:elasticsearch", version.ref = "connectors-destination-testcontainers-elasticsearch" } -connectors-testcontainers-jdbc = { module = "org.testcontainers:jdbc", version.ref = "connectors-testcontainers" } -connectors-testcontainers-kafka = { module = "org.testcontainers:kafka", version.ref = "connectors-testcontainers" } -connectors-testcontainers-mariadb = { module = "org.testcontainers:mariadb", version.ref = "connectors-testcontainers-mariadb" } -connectors-testcontainers-mongodb = { module = "org.testcontainers:mongodb", version.ref = "connectors-testcontainers" } -connectors-testcontainers-mssqlserver = { module = "org.testcontainers:mssqlserver", version.ref = "connectors-testcontainers" } -connectors-testcontainers-mysql = { module = "org.testcontainers:mysql", version.ref = "connectors-testcontainers" } -connectors-testcontainers-postgresql = { module = "org.testcontainers:postgresql", version.ref = "connectors-testcontainers" } -connectors-testcontainers-pulsar = { module = "org.testcontainers:pulsar", version.ref = "connectors-testcontainers-pulsar" } -connectors-testcontainers-scylla = { module = "org.testcontainers:testcontainers", version.ref = "connectors-testcontainers-scylla" } -connectors-testcontainers-tidb = { module = "org.testcontainers:testcontainers", version.ref = "connectors-testcontainers-tidb" } +testcontainers = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers" } +testcontainers-cassandra = { module = "org.testcontainers:cassandra", version.ref = "testcontainers" } +testcontainers-clickhouse = { module = "org.testcontainers:clickhouse", version.ref = "testcontainers" } +testcontainers-cockroachdb = { module = "org.testcontainers:cockroachdb", version.ref = "testcontainers" } +testcontainers-db2 = { module = "org.testcontainers:db2", version.ref = "testcontainers" } +testcontainers-elasticsearch = { module = "org.testcontainers:elasticsearch", version.ref = "testcontainers" } +testcontainers-jdbc = { module = "org.testcontainers:jdbc", version.ref = "testcontainers" } +testcontainers-kafka = { module = "org.testcontainers:kafka", version.ref = "testcontainers" } +testcontainers-mariadb = { module = "org.testcontainers:mariadb", version.ref = "testcontainers" } +testcontainers-mongodb = { module = "org.testcontainers:mongodb", version.ref = "testcontainers" } +testcontainers-mssqlserver = { module = "org.testcontainers:mssqlserver", version.ref = "testcontainers" } +testcontainers-mysql = { module = "org.testcontainers:mysql", version.ref = "testcontainers" } +testcontainers-oracle-xe = { module = "org.testcontainers:oracle-xe", version.ref = "testcontainers" } +testcontainers-postgresql = { module = "org.testcontainers:postgresql", version.ref = "testcontainers" } +testcontainers-pulsar = { module = "org.testcontainers:pulsar", version.ref = "testcontainers" } +testcontainers-scylla = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers" } +testcontainers-tidb = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers" } +testcontainers-tidb-source = { module = "org.testcontainers:tidb", version.ref = "testcontainers" } datadog-trace-api = { module = "com.datadoghq:dd-trace-api", version.ref = "datadog-version" } datadog-trace-ot = { module = "com.datadoghq:dd-trace-ot", version.ref = "datadog-version" } fasterxml = { module = "com.fasterxml.jackson:jackson-bom", version.ref = "fasterxml_version" } @@ -76,6 +66,7 @@ jackson-dataformat = { module = "com.fasterxml.jackson.dataformat:jackson-datafo jackson-datatype = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jsr310", version.ref = "fasterxml_version" } jackson-kotlin = { module = "com.fasterxml.jackson.module:jackson-module-kotlin", version.ref = "fasterxml_version" } java-dogstatsd-client = { module = "com.datadoghq:java-dogstatsd-client", version = "4.1.0" } +java-faker = { module = "com.github.javafaker:javafaker", version = "1.0.2" } javax-databind = { module = "javax.xml.bind:jaxb-api", version = "2.4.0-b180830.0359" } jcl-over-slf4j = { module = "org.slf4j:jcl-over-slf4j", version.ref = "slf4j" } jmh-core = { module = "org.openjdk.jmh:jmh-core", version.ref = "jmh" } @@ -89,24 +80,24 @@ junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", vers junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params", version.ref = "junit-jupiter" } junit-jupiter-system-stubs = { module = "uk.org.webcompere:system-stubs-jupiter", version = "2.0.1" } junit-pioneer = { module = "org.junit-pioneer:junit-pioneer", version = "1.7.1" } +kotlin-logging = { module = "io.github.oshai:kotlin-logging-jvm", version = "5.1.0" } +kotlinx-cli = { module = "org.jetbrains.kotlinx:kotlinx-cli", version = "0.3.5" } +kotlinx-cli-jvm = { module = "org.jetbrains.kotlinx:kotlinx-cli-jvm", version = "0.3.5" } launchdarkly = { module = "com.launchdarkly:launchdarkly-java-server-sdk", version = "6.0.1" } log4j-api = { module = "org.apache.logging.log4j:log4j-api", version.ref = "log4j" } log4j-core = { module = "org.apache.logging.log4j:log4j-core", version.ref = "log4j" } -log4j-impl = { module = "org.apache.logging.log4j:log4j-slf4j-impl", version.ref = "log4j" } +log4j-impl = { module = "org.apache.logging.log4j:log4j-slf4j2-impl", version.ref = "log4j" } log4j-over-slf4j = { module = "org.slf4j:log4j-over-slf4j", version.ref = "slf4j" } log4j-web = { module = "org.apache.logging.log4j:log4j-web", version.ref = "log4j" } lombok = { module = "org.projectlombok:lombok", version.ref = "lombok" } micrometer-statsd = { module = "io.micrometer:micrometer-registry-statsd", version = "1.9.3" } mockito-junit-jupiter = { module = "org.mockito:mockito-junit-jupiter", version = "4.6.1" } mockk = { module = "io.mockk:mockk", version = "1.13.3" } +mongo-driver-sync = { module = "org.mongodb:mongodb-driver-sync", version = "4.10.2" } otel-bom = { module = "io.opentelemetry:opentelemetry-bom", version = "1.14.0" } otel-sdk = { module = "io.opentelemetry:opentelemetry-sdk-metrics", version = "1.14.0" } otel-sdk-testing = { module = "io.opentelemetry:opentelemetry-sdk-metrics-testing", version = "1.13.0-alpha" } otel-semconv = { module = "io.opentelemetry:opentelemetry-semconv", version = "1.14.0-alpha" } -platform-testcontainers = { module = "org.testcontainers:testcontainers", version.ref = "platform-testcontainers" } -platform-testcontainers-cockroachdb = { module = "org.testcontainers:cockroachdb", version.ref = "platform-testcontainers" } -platform-testcontainers-jdbc = { module = "org.testcontainers:jdbc", version.ref = "platform-testcontainers" } -platform-testcontainers-postgresql = { module = "org.testcontainers:postgresql", version.ref = "platform-testcontainers" } postgresql = { module = "org.postgresql:postgresql", version.ref = "postgresql" } quartz-scheduler = { module = "org.quartz-scheduler:quartz", version = "2.3.2" } reactor-core = { module = "io.projectreactor:reactor-core", version.ref = "reactor" } @@ -114,6 +105,7 @@ reactor-test = { module = "io.projectreactor:reactor-test", version.ref = "react s3 = { module = "software.amazon.awssdk:s3", version = "2.20.20" } segment-java-analytics = { module = "com.segment.analytics.java:analytics", version.ref = "segment" } slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" } +slf4j-simple = { module = "org.slf4j:slf4j-simple", version.ref = "slf4j" } spotbugs-annotations = { module = "com.github.spotbugs:spotbugs-annotations", version = "4.7.3" } temporal-sdk = { module = "io.temporal:temporal-sdk", version.ref = "temporal" } temporal-serviceclient = { module = "io.temporal:temporal-serviceclient", version.ref = "temporal" } @@ -166,4 +158,6 @@ micronaut-test = ["micronaut-test-core", "micronaut-test-junit5", "h2-database"] micronaut-test-annotation-processor = ["micronaut-inject-java"] slf4j = ["jul-to-slf4j", "jcl-over-slf4j", "log4j-over-slf4j"] temporal = ["temporal-sdk", "temporal-serviceclient"] -debezium-bundle = ["debezium-api", "debezium-embedded", "debezium-sqlserver", "debezium-mysql", "debezium-postgres", "debezium-mongodb"] + +[plugins] +kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } diff --git a/docs/.gitbook/assets/explore_logs.png b/docs/.gitbook/assets/explore_logs.png deleted file mode 100644 index 98d159e8af7a..000000000000 Binary files a/docs/.gitbook/assets/explore_logs.png and /dev/null differ diff --git a/docs/.gitbook/assets/job-state-machine.png b/docs/.gitbook/assets/job-state-machine.png deleted file mode 100644 index 36fb91a22424..000000000000 Binary files a/docs/.gitbook/assets/job-state-machine.png and /dev/null differ diff --git a/docs/.gitbook/assets/orchestrator-lifecycle.png b/docs/.gitbook/assets/orchestrator-lifecycle.png deleted file mode 100644 index 7db52fea872a..000000000000 Binary files a/docs/.gitbook/assets/orchestrator-lifecycle.png and /dev/null differ diff --git a/docs/.gitbook/assets/source/airtable/add_bases.png b/docs/.gitbook/assets/source/airtable/add_bases.png new file mode 100644 index 000000000000..1b46373c9a4f Binary files /dev/null and b/docs/.gitbook/assets/source/airtable/add_bases.png differ diff --git a/docs/.gitbook/assets/source/airtable/add_scopes.png b/docs/.gitbook/assets/source/airtable/add_scopes.png new file mode 100644 index 000000000000..d7cf9ff2640b Binary files /dev/null and b/docs/.gitbook/assets/source/airtable/add_scopes.png differ diff --git a/docs/.gitbook/assets/source/airtable/generate_new_token.png b/docs/.gitbook/assets/source/airtable/generate_new_token.png new file mode 100644 index 000000000000..3173805f3a97 Binary files /dev/null and b/docs/.gitbook/assets/source/airtable/generate_new_token.png differ diff --git a/docs/.gitbook/assets/source/mongodb/mongodb_atlas_connection_string_step_2.png b/docs/.gitbook/assets/source/mongodb/mongodb_atlas_connection_string_step_2.png new file mode 100644 index 000000000000..fab9a232d6fa Binary files /dev/null and b/docs/.gitbook/assets/source/mongodb/mongodb_atlas_connection_string_step_2.png differ diff --git a/docs/.gitbook/assets/source/mongodb/mongodb_atlas_connection_string_step_3.png b/docs/.gitbook/assets/source/mongodb/mongodb_atlas_connection_string_step_3.png new file mode 100644 index 000000000000..e54a1606f4ae Binary files /dev/null and b/docs/.gitbook/assets/source/mongodb/mongodb_atlas_connection_string_step_3.png differ diff --git a/docs/.gitbook/assets/source/mongodb/mongodb_atlas_connection_string_step_4.png b/docs/.gitbook/assets/source/mongodb/mongodb_atlas_connection_string_step_4.png new file mode 100644 index 000000000000..d1fdf5342b43 Binary files /dev/null and b/docs/.gitbook/assets/source/mongodb/mongodb_atlas_connection_string_step_4.png differ diff --git a/docs/.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_2.png b/docs/.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_2.png new file mode 100644 index 000000000000..17e9bba30ea0 Binary files /dev/null and b/docs/.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_2.png differ diff --git a/docs/.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_3.png b/docs/.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_3.png new file mode 100644 index 000000000000..68359ef727ce Binary files /dev/null and b/docs/.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_3.png differ diff --git a/docs/.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_4.png b/docs/.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_4.png new file mode 100644 index 000000000000..7b6a775fbf78 Binary files /dev/null and b/docs/.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_4.png differ diff --git a/docs/.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_5.png b/docs/.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_5.png new file mode 100644 index 000000000000..0e325cc0f787 Binary files /dev/null and b/docs/.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_5.png differ diff --git a/docs/.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_6.png b/docs/.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_6.png new file mode 100644 index 000000000000..0ff9fa570cea Binary files /dev/null and b/docs/.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_6.png differ diff --git a/docs/.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_7.png b/docs/.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_7.png new file mode 100644 index 000000000000..8e844fc29c7e Binary files /dev/null and b/docs/.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_7.png differ diff --git a/docs/.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_8.png b/docs/.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_8.png new file mode 100644 index 000000000000..78a2323e7aef Binary files /dev/null and b/docs/.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_8.png differ diff --git a/docs/.gitbook/assets/worker-lifecycle.png b/docs/.gitbook/assets/worker-lifecycle.png deleted file mode 100644 index 5d9d889433fc..000000000000 Binary files a/docs/.gitbook/assets/worker-lifecycle.png and /dev/null differ diff --git a/docs/assets/docs/okta-create-new-app-integration.png b/docs/access-management/sso-providers/assets/okta-create-new-app-integration.png similarity index 100% rename from docs/assets/docs/okta-create-new-app-integration.png rename to docs/access-management/sso-providers/assets/okta-create-new-app-integration.png diff --git a/docs/access-management/sso-providers/okta.md b/docs/access-management/sso-providers/okta.md new file mode 100644 index 000000000000..241c385cdf05 --- /dev/null +++ b/docs/access-management/sso-providers/okta.md @@ -0,0 +1,108 @@ +--- +sidebar_label: Okta +products: oss-enterprise, cloud-teams +--- + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + +# Setup Single Sign-On via Okta + +This page guides you through setting up Okta for [Single Sign-On](../sso.md) with Airbyte. + +Airbyte will communicate with your Okta account using OpenID Connect (OIDC). + +## Creating an Okta app for Airbyte + +:::info +The following steps need to be executed by an administrator of your company's Okta account. +::: + +You will need to create a new Okta OIDC App Integration for your Airbyte. Documentation on how to do this in Okta can be found [here](https://help.okta.com/en-us/content/topics/apps/apps_app_integration_wizard_oidc.htm). + +You should create an app integration with **OIDC - OpenID Connect** as the sign-in method and **Web Application** as the application type: + +![Screenshot of Okta app integration creation modal](./assets/okta-create-new-app-integration.png) + +On the following screen you'll need to configure all parameters for your Okta application: + + + + You'll require to know your **Company Identifier** to fill in those values. You receive this + from your contact at Airbyte. + + Create the application with the following parameters: + +
      +
      **App integration name**
      +
      A human readable name for the application (e.g. **Airbyte Cloud**). This is only used for identification inside your Okta dashboard.
      +
      **Logo** (optional)
      +
      You can upload an Airbyte logo, which you can find at https://airbyte.com/company/press
      +
      **Grant type**
      +
      Only **Authorization Code** should be selected
      +
      **Sign-in redirect URIs**
      +
      + ``` + https://cloud.airbyte.com/auth/realms//broker/default/endpoint + ``` +
      +
      **Sign-out redirect URIs**
      +
      + ``` + https://cloud.airbyte.com/auth/realms//broker/default/endpoint/logout_response + ``` +
      +
      **Trusted Origins**
      +
      Leave empty
      +
      **Assignments > Controlled Access**
      +
      You can control whether everyone in your Okta organization should be able to access Airbyte using their Okta account or limit it only to a subset of your users by selecting specific groups who should get access.
      +
      + + You'll need to pass your Airbyte contact the following information of the created application. After that we'll setup SSO for you and let you know once it's ready. + + * Your **Okta domain** (it's not specific to this application, see [Find your Okta domain](https://developer.okta.com/docs/guides/find-your-domain/main/)) + * **Client ID** + * **Client Secret** +
      + + Create the application with the following parameters: + +
      +
      **App integration name**
      +
      Please choose a URL-friendly app integraiton name without spaces or special characters, such as `my-airbyte-app`. Screenshot of Okta app integration name Spaces or special characters in this field could result in invalid redirect URIs.
      +
      **Logo** (optional)
      +
      You can upload an Airbyte logo, which you can find at https://airbyte.com/company/press
      +
      **Grant type**
      +
      Only **Authorization Code** should be selected
      +
      **Sign-in redirect URIs**
      +
      + ``` + /auth/realms/airbyte/broker//endpoint + ``` + + `` refers to the domain you access your Airbyte instance at, e.g. `https://airbyte.internal.mycompany.com` + + `` refers to the value you entered in the **App integration name** field +
      +
      **Sign-out redirect URIs**
      +
      + ``` + /auth/realms/airbyte/broker//endpoint/logout_response + ``` +
      +
      **Trusted Origins**
      +
      Leave empty
      +
      **Assignments > Controlled Access**
      +
      You can control whether everyone in your Okta organization should be able to access Airbyte using their Okta account or limit it only to a subset of your users by selecting specific groups who should get access.
      +
      + + Once your Okta app is set up, you're ready to deploy Airbyte with SSO. Take note of the following configuration values, as you will need them to configure Airbyte to use your new Okta SSO app integration: + + * Okta domain ([How to find your Okta domain](https://developer.okta.com/docs/guides/find-your-domain/main/)) + * App Integration Name + * Client ID + * Client Secret + + Visit the [implementation guide](/enterprise-setup/implementation-guide.md) for instructions on how to deploy Airbyte Enterprise using `kubernetes`, `kubectl` and `helm`. +
      +
      diff --git a/docs/access-management/sso.md b/docs/access-management/sso.md new file mode 100644 index 000000000000..065c7ed74e5b --- /dev/null +++ b/docs/access-management/sso.md @@ -0,0 +1,38 @@ +--- +products: oss-enterprise, cloud-teams +--- + +# Single Sign-On (SSO) + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + +Single Sign-On (SSO) allows you to enable logging into Airbyte using your existing Identity Provider (IdP) like Okta or Active Directory. + +SSO is available in Airbyte Enterprise and on Cloud with the Teams add-on. [Talk to us](https://airbyte.com/company/talk-to-sales) if you are interested in setting up SSO for your organization. + +## Set up + +You can find setup explanations for all our supported Identity Providers on the following subpages: + +```mdx-code-block +import DocCardList from '@theme/DocCardList'; + + +``` + +## Logging in + + + + Once we inform you that you’re all set up, you can log into Airbyte using SSO by visiting [cloud.airbyte.com/sso](https://cloud.airbyte.com/sso) or select the **Continue with SSO** option on the login screen. + + Specify your _company identifier_ and hit “Continue with SSO”. You’ll be forwarded to your IdP's login page (e.g. Okta login page). Log into your work account and you’ll be forwarded back to Airbyte Cloud and be logged in. + + *Note:* you were already logged into your company’s Okta account you might not see any login screen and directly get forwarded back to Airbyte Cloud. + + + Accessing your self hosted Airbyte will automatically forward you to your IdP's login page (e.g. Okta login page). Log into your work account and you’ll be forwarded back to your Airbyte and be logged in. + + + diff --git a/docs/airbyte-enterprise.md b/docs/airbyte-enterprise.md deleted file mode 100644 index df04379d8445..000000000000 --- a/docs/airbyte-enterprise.md +++ /dev/null @@ -1,62 +0,0 @@ -# Airbyte Enterprise - -Airbyte Enterprise is a self-hosted version of Airbyte with additional features for enterprise customers. Airbyte Enterprise is currently in early development. Interested in Airbyte Enterprise for your organization? [Learn more](https://airbyte.com/solutions/airbyte-enterprise). - -## Airbyte Enterprise License Key - -A valid license key is required for Airbyte Enterprise. Talk to your Sales Representative to receive your license key before installing Airbyte Enterprise. - -## Single Sign-On (SSO) - -Airbyte Enterprise supports Single Sign-On, allowing an organization to manage user acces to their Airbyte Enterprise instance through the configuration of an Identity Provider (IdP). - -Airbyte Enterprise currently supports SSO via OIDC with [Okta](https://www.okta.com/) as an IdP. - -### Setting up Okta for SSO - -You will need to create a new Okta OIDC App Integration for your Airbyte instance. Documentation on how to do this in Okta can be found [here](https://help.okta.com/en-us/Content/Topics/Apps/Apps_App_Integration_Wizard_OIDC.htm). - -You should create an app integration with **OIDC - OpenID Connect** as the sign-in method and **Web Application** as the application type: - -![Screenshot of Okta app integration creation modal](./assets/docs/okta-create-new-app-integration.png) - -#### App integration name - -Please choose a URL-friendly app integraiton name without spaces or special characters, such as `my-airbyte-app`: - -![Screenshot of Okta app integration name](./assets/docs/okta-app-integration-name.png) - -Spaces or special characters in this field could result in invalid redirect URIs. - -#### Redirect URIs - -In the **Login** section, set the following fields, substituting `` and `` for your own values: - -Sign-in redirect URIs: - -``` -/auth/realms/airbyte/broker//endpoint -``` - -Sign-out redirect URIs - -``` -/auth/realms/airbyte/broker//endpoint/logout_response -``` - -![Okta app integration name screenshot](./assets/docs/okta-login-redirect-uris.png) - -_Example values_ - -`` should point to where your Airbyte instance will be available, including the http/https protocol. - -#### Deploying Airbyte Enterprise with Okta - -Your Okta app is now set up and you're ready to deploy Airbyte with SSO! Take note of the following configuration values, as you will need them to configure Airbyte to use your new Okta SSO app integration: - -- Okta domain ([how to find your Okta domain](https://developer.okta.com/docs/guides/find-your-domain/main/)) -- App integration name -- Client ID -- Client Secret - -Visit [Airbyte Enterprise deployment](/deploying-airbyte/on-kubernetes-via-helm#alpha-airbyte-pro-deployment) for instructions on how to deploy Airbyte Enterprise using `kubernetes`, `kubectl` and `helm`. diff --git a/docs/api-documentation.md b/docs/api-documentation.md index 448a63b2e952..53cf2c7845aa 100644 --- a/docs/api-documentation.md +++ b/docs/api-documentation.md @@ -1,3 +1,7 @@ +--- +products: all +--- + # API documentation Airbyte has two sets of APIs which are intended for different uses. The table below outlines their descriptions, use cases, availability and status. diff --git a/docs/archive/changelog/README.md b/docs/archive/changelog/README.md deleted file mode 100644 index cc854f303e60..000000000000 --- a/docs/archive/changelog/README.md +++ /dev/null @@ -1,645 +0,0 @@ -# Changelog - -## 1/28/2022 Summary - -* New Source: Chartmogul (contributyed by Titas Skrebė) -* New Source: Hellobaton (contributed by Daniel Luftspring) -* New Source: Flexport (contributed by Juozas) -* New Source: PersistIq (contributed by Wadii Zaim) - -* ✨ Postgres Source: Users can now select which schemas they wish to sync before discovery. This makes the discovery stage for large instances much more performant. -* ✨ Shopify Source: Now verifies permissions on the token before accessing resources. -* ✨ Snowflake Destination: Users now have access to an option to purge their staging data. -* ✨ HubSpot Source: Added some more fields for the email_events stream. -* ✨ Amazon Seller Partner Source: Added the GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL report stream. (contributed by @ron-damon) -* ✨ HubSpot Source: Added the form_submission and property_history streams. - -* 🐛 DynamoDB Destination: The parameter dynamodb_table_name is now named dynamodb_table_name_prefix to more accurately represent it. -* 🐛 Intercom Source: The handling of scroll param is now fixed when it is expired. -* 🐛 S3 + GCS Destinations: Now support arrays with unknown item type. -* 🐛 Postgres Source: Now supports handling of the Java SQL date type. -* 🐛 Salesforce Source: No longer fails during schema generation. - -## 1/13/2022 Summary - -⚠️ WARNING ⚠️ - -Snowflake Source: Normalization with Snowflake now produces permanent tables. [If you want to continue creating transient tables, you will need to create a new transient database for Airbyte.] - -* ✨ GitHub Source: PR related streams now support incremental sync. -* ✨ HubSpot Source: We now support ListMemberships in the Contacts stream. -* ✨ Azure Blob Storage Destination: Now has the option to add a BufferedOutputStream to improve performance and fix writing data with over 50GB in a stream. (contributed by @bmatticus) - -* 🐛 Normalization partitioning now works as expected with FLOAT64 and BigQuery. -* 🐛 Normalization now works properly with quoted and case sensitive columns. -* 🐛 Source MSSQL: Added support for some missing data types. -* 🐛 Snowflake Destination: Schema is now not created if it previously exists. -* 🐛 Postgres Source: Now properly reads materialized views. -* 🐛 Delighted Source: Pagination for survey_responses, bounces and unsubscribes streams now works as expected. -* 🐛 Google Search Console Source: Incremental sync now works as expected. -* 🐛 Recurly Source: Now does not load all accounts when importing account coupon redemptions. -* 🐛 Salesforce Source: Now properly handles 400 when streams don't support query or queryAll. - -## 1/6/2022 Summary - -* New Source: 3PL Central (contributed by Juozas) -* New Source: My Hours (contributed by Wisse Jelgersma) -* New Source: Qualaroo (contributed by gunu) -* New Source: SearchMetrics - -* 💎 Salesforce Source: Now supports filtering streams at configuration, making it easier to handle large Salesforce instances. -* 💎 Snowflake Destination: Now supports byte-buffering for staged inserts. -* 💎 Redshift Destination: Now supports byte-buffering for staged inserts. -* ✨ Postgres Source: Now supports all Postgres 14 types. -* ✨ Recurly Source: Now supports incremental sync for all streams. -* ✨ Zendesk Support Source: Added the Brands, CustomRoles, and Schedules streams. -* ✨ Zendesk Support Source: Now uses cursor-based pagination. -* ✨ Kustomer Source: Setup configuration is now more straightforward. -* ✨ Hubspot Source: Now supports incremental sync on all streams where possible. -* ✨ Facebook Marketing Source: Fixed schema for breakdowns fields. -* ✨ Facebook Marketing Source: Added asset_feed_spec to AdCreatives stream. -* ✨ Redshift Destination: Now has an option to toggle the deletion of staging data. - -* 🐛 S3 Destination: Avro and Parquet formats are now processed correctly. -* 🐛 Snowflake Destination: Fixed SQL Compliation error. -* 🐛 Kafka Source: SASL configurations no longer throw null pointer exceptions (contributed by Nitesh Kumar) -* 🐛 Salesforce Source: Now throws a 400 for non-queryable streams. -* 🐛 Amazon Ads Source: Polling for report generation is now much more resilient. (contributed by Juozas) -* 🐛 Jira Source: The filters stream now works as expected. -* 🐛 BigQuery Destination: You can now properly configure the buffer size with the part_size config field. -* 🐛 Snowflake Destination: You can now properly configure the buffer size with the part_size config field. -* 🐛 CockroachDB Source: Now correctly only discovers tables the user has permission to access. -* 🐛 Stripe Source: The date and arrival_date fields are now typed correctly. - -## 12/16/2021 Summary - -🎉 First off... There's a brand new CDK! Menno Hamburg contributed a .NET/C# implementation for our CDK, allowing you to write HTTP API sources and Generic Dotnet sources. Thank you so much Menno, this is huge! - -* New Source: OpenWeather -* New Destination: ClickHouse (contributed by @Bo) -* New Destination: RabbitMQ (contributed by @Luis Gomez) -* New Destination: Amazon SQS (contributed by @Alasdair Brown) -* New Destination: Rockset (contributed by @Steve Baldwin) - -* ✨ Facebook Marketing Source: Updated the campaign schema with more relevant fields. (contributed by @Maxime Lavoie) -* ✨ TikTok Marketing Source: Now supports the Basic Report stream. -* ✨ MySQL Source: Now supports all MySQL 8.0 data types. -* ✨ Klaviyo Source: Improved performance, added incremental sync support to the Global Exclusions stream. -* ✨ Redshift Destination: You can now specify a bucket path to stage your data in before inserting. -* ✨ Kubernetes deployments: Sidecar memory is now 25Mi, up from 6Mi to cover all usage cases. -* ✨ Kubernetes deployments: The Helm chart can now set up writing logs to S3 easily. (contributed by @Valentin Nourdin) - -* 🐛 Python CDK: Now shows the stack trace of unhandled exceptions. -* 🐛 Google Analytics Source: Fix data window input validation, fix date type conversion. -* 🐛 Google Ads Source: Data from the end_date for syncs is now included in a sync. -* 🐛 Marketo Source: Fixed issues around input type conversion and conformation to the schema. -* 🐛 Mailchimp Source: Fixed schema conversion error causing sync failures. -* 🐛 PayPal Transactions Source: Now reports full error message details on failure. -* 🐛 Shopify Source: Normalization now works as expected. - -## 12/9/2021 Summary - -⚠️ WARNING ⚠️ - -v0.33.0 is a minor version with breaking changes. Take the normal precautions with upgrading safely to this version. -v0.33.0 has a bug that affects GCS logs on Kubernetes. Upgrade straight to v0.33.2 if you are running a K8s deployment of Airbyte. - -* New Source: Mailgun - -🎉 Snowflake Destination: You can now stage your inserts, making them much faster. - -* ✨ Google Ads Source: Source configuration is now more clear. -* ✨ Google Analytics Source: Source configuration is now more clear. -* ✨ S3 Destination: You can now write timestamps in Avro and Parquet formats. -* ✨ BigQuery & BigQuery Denormalized Destinations: Now use byte-based buffering for batch inserts. -* ✨ Iterable Source: Now has email validation on the list_users stream. - -* 🐛 Incremental normalization now works properly with empty tables. -* 🐛 LinkedIn Ads Source: 429 response is now properly handled. -* 🐛 Intercom Source: Now handles failed pagination requests with backoffs. -* 🐛 Intercom Source: No longer drops records from the conversation stream. -* 🐛 Google Analytics Source: 400 errors no longer get ignored with custom reports. -* 🐛 Marketo Source: The createdAt and updatedAt fields are now formatted correctly. - -## 12/2/2021 Summary - -🎃 **Hacktoberfest Submissions** 🎃 ------------------------------------------ -* New Destination: Redis (contributed by @Ivica Taseski) -* New Destination: MQTT (contributed by @Mario Molina) -* New Destination: Google Firestore (contributed by @Adam Dobrawy) -* New Destination: Kinesis (contributed by @Ivica Taseski) -* New Source: Zenloop (contributed by @Alexander Batoulis) -* New Source: Outreach (contributed by @Luis Gomez) - -* ✨ Zendesk Source: The chats stream now supports incremental sync and added testing for all streams. -* 🐛 Monday Source: Pagination now works as expected and the schema has been fixed. -* 🐛 Postgres Source: Views are now properly listed during schema discovery. -* 🐛 Postgres Source: Using the money type with an amount greater than 1000 works properly now. -* 🐛 Google Search Console Search: We now set a default end_data value. -* 🐛 Mixpanel Source: Normalization now works as expected and streams are now displayed properly in the UI. -* 🐛 MongoDB Source: The DATE_TIME type now uses milliseconds. - -## 11/25/2021 Summary -Hey Airbyte Community! Let's go over all the changes from v.32.5 and prior! - -🎃 **Hacktoberfest Submissions** 🎃 -* New Source: Airtable (contributed by Tuan Nguyen). -* New Source: Notion (contributed by Bo Lu). -* New Source: Pardot (contributed by Tuan Nguyen). - -* New Source: Youtube analytics. - -* ✨ Source Exchange Rates: add ignore_weekends option. -* ✨ Source Facebook: add the videos stream. -* ✨ Source Freshdesk: removed the limitation in streams pagination. -* ✨ Source Jira: add option to render fields in HTML format. -* ✨ Source MongoDB v2: improve read performance. -* ✨ Source Pipedrive: specify schema for "persons" stream. -* ✨ Source PostgreSQL: exclude tables on which user doesn't have select privileges. -* ✨ Source SurveyMonkey: improve connection check. - -* 🐛 Source Salesforce: improve resiliency of async bulk jobs. -* 🐛 Source Zendesk Support: fix missing ticket_id in ticket_comments stream. -* 🐛 Normalization: optimize incremental normalization runtime with Snowflake. - -As usual, thank you so much to our wonderful contributors this week that have made Airbyte into what it is today: Madison Swain-Bowden, Tuan Nguyen, Bo Lu, Adam Dobrawy, Christopher Wu, Luis Gomez, Ivica Taseski, Mario Molina, Ping Yee, Koji Matsumoto, Sujit Sagar, Shadab, Juozas V.([Labanoras Tech](http://labanoras.io)) and Serhii Chvaliuk! - -## 11/17/2021 Summary - -Hey Airbyte Community! Let's go over all the changes from v.32.1 and prior! But first, there's an important announcement I need to make about upgrading Airbyte to v.32.1. - -⚠️ WARNING ⚠️ -Upgrading to v.32.0 is equivalent to a major version bump. If your current version is v.32.0, you must upgrade to v.32.0 first before upgrading to any later version - -Keep in mind that this upgrade requires your all of your connector Specs to be retrievable, or Airbyte will fail on startup. You can force delete your connector Specs by setting the `VERSION_0_32_0_FORCE_UPGRADE` environment variable to `true`. Steps to specifically check out v.32.0 and details around this breaking change can be found [here](https://docs.airbyte.com/operator-guides/upgrading-airbyte/#mandatory-intermediate-upgrade). - -*Now back to our regularly scheduled programming.* - -🎃 Hacktoberfest Submissions 🎃 - -* New Destination: ScyllaDB (contributed by Ivica Taseski) -* New Source: Azure Table Storage (contributed by geekwhocodes) -* New Source: Linnworks (contributed by Juozas V.([Labanoras Tech](http://labanoras.io))) - -* ✨ Source MySQL: Now has basic performance tests. -* ✨ Source Salesforce: We now automatically transform and handle incorrect data for the anyType and calculated types. - -* 🐛 IBM Db2 Source: Now handles conversion from DECFLOAT to BigDecimal correctly. -* 🐛 MSSQL Source: Now handles VARBINARY correctly. -* 🐛 CockroachDB Source: Improved parsing of various data types. - -As usual, thank you so much to our wonderful contributors this week that have made Airbyte into what it is today: Achmad Syarif Hidayatullah, Tuan Nguyen, Ivica Taseski, Hai To, Juozas, gunu, Shadab, Per-Victor Persson, and Harsha Teja Kanna! - -## 11/11/2021 Summary - -Time to go over changes from v.30.39! And... let's get another update on Hacktoberfest. - -🎃 Hacktoberfest Submissions 🎃 - -* New Destination: Cassandra (contributed by Ivica Taseski) -* New Destination: Pulsar (contributed by Mario Molina) -* New Source: Confluence (contributed by Tuan Nguyen) -* New Source: Monday (contributed by Tuan Nguyen) -* New Source: Commerce Tools (contributed by James Wilson) -* New Source: Pinterest Marketing (contributed by us!) - -* ✨ Shopify Source: Now supports the FulfillmentOrders and Fulfillments streams. -* ✨ Greenhouse Source: Now supports the Demographics stream. -* ✨ Recharge Source: Broken requests should now be re-requested with improved backoff. -* ✨ Stripe Source: Now supports the checkout_sessions, checkout_sessions_line_item, and promotion_codes streams. -* ✨ Db2 Source: Now supports SSL. - -* 🐛 We've made some updates to incremental normalization to fix some outstanding issues. [Details](https://github.com/airbytehq/airbyte/pull/7669) -* 🐛 Airbyte Server no longer crashes due to too many open files. -* 🐛 MSSQL Source: Data type conversion with smalldatetime and smallmoney works correctly now. -* 🐛 Salesforce Source: anyType fields can now be retrieved properly with the BULK API -* 🐛 BigQuery-Denormalized Destination: Fixed JSON parsing with $ref fields. - -As usual, thank you to our awesome contributors that have done awesome work during the last week: Tuan Nguyen, Harsha Teja Kanna, Aaditya S, James Wilson, Vladimir Remar, Yuhui Shi, Mario Molina, Ivica Taseski, Collin Scangarella, and haoranyu! - -## 11/03/2021 Summary - -It's patch notes time. Let's go over the changes from 0.30.24 and before. But before we do, let's get a quick update on how Hacktober is going! - -🎃 Hacktoberfest Submissions 🎃 - -* New Destination: Elasticsearch (contributed by Jeremy Branham) -* New Source: Salesloft (contributed by Pras) -* New Source: OneSignal (contributed by Bo) -* New Source: Strava (contributed by terencecho) -* New Source: Lemlist (contributed by Igli Koxha) -* New Source: Amazon SQS (contributed by Alasdair Brown) -* New Source: Freshservices (contributed by Tuan Nguyen) -* New Source: Freshsales (contributed by Tuan Nguyen) -* New Source: Appsflyer (contributed by Achmad Syarif Hidayatullah) -* New Source: Paystack (contributed by Foluso Ogunlana) -* New Source: Sentry (contributed by koji matsumoto) -* New Source: Retently (contributed by Subhash Gopalakrishnan) -* New Source: Delighted! (contributed by Rodrigo Parra) - -with 18 more currently in review... - -🎉 **Incremental Normalization is here!** 🎉 - -💎 Basic normalization no longer runs on already normalized data, making it way faster and cheaper. :gem: - -🎉 **Airbyte Compiles on M1 Macs!** - -Airbyte developers with M1 chips in their MacBooks can now compile the project and run the server. This is a major step towards being able to fully run Airbyte on M1. (contributed by Harsha Teja Kanna) - -* ✨ BigQuery Destination: You can now run transformations in batches, preventing queries from hitting BigQuery limits. (contributed by Andrés Bravo) -* ✨ S3 Source: Memory and Performance optimizations, also some fancy new PyArrow CSV configuration options. -* ✨ Zuora Source: Now supports Unlimited as an option for the Data Query Live API. -* ✨ Clickhouse Source: Now supports SSL and connection via SSH tunneling. - -* 🐛 Oracle Source: Now handles the LONG RAW data type correctly. -* 🐛 Snowflake Source: Fixed parsing of extreme values for FLOAT and NUMBER data types. -* 🐛 Hubspot Source: No longer fails due to lengthy URI/URLs. -* 🐛 Zendesk Source: The chats stream now pulls data past the first page. -* 🐛 Jira Source: Normalization now works as expected. - -As usual, thank you to our awesome contributors that have done awesome work during this productive spooky season: Tuan Nguyen, Achmad Syarif Hidayatullah, Christopher Wu, Andrés Bravo, Harsha Teja Kanna, Collin Scangarella, haoranyu, koji matsumoto, Subhash Gopalakrishnan, Jeremy Branham, Rodrigo Parra, Foluso Ogunlana, EdBizarro, Gergely Lendvai, Rodeoclash, terencecho, Igli Koxha, Alasdair Brown, bbugh, Pras, Bo, Xiangxuan Liu, Hai To, s-mawjee, Mario Molina, SamyPesse, Yuhui Shi, Maciej Nędza, Matt Hoag, and denis-sokolov! - -## 10/20/2021 Summary - -It's patch notes time! Let's go over changes from 0.30.16! But before we do... I want to remind everyone that Airbyte Hacktoberfest is currently taking place! For every connector that is merged into our codebase, you'll get $500, so make sure to submit before the hackathon ends on November 19th. - -* 🎉 New Source: WooCommerce (contributed by James Wilson) -* 🎉 K8s deployments: Worker image pull policy is now configurable (contributed by Mario Molina) - -* ✨ MSSQL destination: Now supports basic normalization -* 🐛 LinkedIn Ads source: Analytics streams now work as expected. - -We've had a lot of contributors over the last few weeks, so I'd like to thank all of them for their efforts: James Wilson, Mario Molina, Maciej Nędza, Pras, Tuan Nguyen, Andrés Bravo, Christopher Wu, gunu, Harsha Teja Kanna, Jonathan Stacks, darian, Christian Gagnon, Nicolas Moreau, Matt Hoag, Achmad Syarif Hidayatullah, s-mawjee, SamyPesse, heade, zurferr, denis-solokov, and aristidednd! - -## 09/29/2021 Summary - -It's patch notes time, let's go over the changes from our new minor version, v0.30.0. As usual, bug fixes are in the thread. - -* New source: LinkedIn Ads -* New source: Kafka -* New source: Lever Hiring - -* 🎉 New License: Nothing changes for users of Airbyte/contributors. You just can't sell your own Airbyte Cloud! - -* 💎 New API endpoint: You can now call connections/search in the web backend API to search sources and destinations. (contributed by Mario Molina) -* 💎 K8s: Added support for ImagePullSecrets for connector images. -* 💎 MSSQL, Oracle, MySQL sources & destinations: Now support connection via SSH (Bastion server) - -* ✨ MySQL destination: Now supports connection via TLS/SSL -* ✨ BigQuery (denormalized) destination: Supports reading BigQuery types such as date by reading the format field (contributed by Nicolas Moreau) -* ✨ Hubspot source: Added contacts associations to the deals stream. -* ✨ GitHub source: Now supports pulling commits from user-specified branches. -* ✨ Google Search Console source: Now accepts admin email as input when using a service account key. -* ✨ Greenhouse source: Now identifies API streams it has access to if permissions are limited. -* ✨ Marketo source: Now Airbyte native. -* ✨ S3 source: Now supports any source that conforms to the S3 protocol (Non-AWS S3). -* ✨ Shopify source: Now reports pre_tax_price on the line_items stream if you have Shopify Plus. -* ✨ Stripe source: Now actually uses the mandatory start_date config field for incremental syncs. - -* 🏗 Python CDK: Now supports passing custom headers to the requests in OAuth2, enabling token refresh calls. -* 🏗 Python CDK: Parent streams can now be configured to cache data for their child streams. -* 🏗 Python CDK: Now has a Transformer class that can cast record fields to the data type expected by the schema. - -* 🐛 Amplitude source: Fixed schema for date-time objects. -* 🐛 Asana source: Schema fixed for the sections, stories, tasks, and users streams. -* 🐛 GitHub source: Added error handling for streams not applicable to a repo. (contributed by Christopher Wu) -* 🐛 Google Search Console source: Verifies access to sites when performing the connection check. -* 🐛 Hubspot source: Now conforms to the V3 API, with streams such as owners reflecting the new fields. -* 🐛 Intercom source: Fixed data type for the updated_at field. (contributed by Christian Gagnon) -* 🐛 Iterable source: Normalization now works as expected. -* 🐛 Pipedrive source: Schema now reflects the correct types for date/time fields. -* 🐛 Stripe source: Incorrect timestamp formats removed for coupons and subscriptions streams. -* 🐛 Salesforce source: You can now sync more than 10,000 records with the Bulk API. -* 🐛 Snowflake destination: Now accepts any date-time format with normalization. -* 🐛 Snowflake destination: Inserts are now split into batches to accommodate for large data loads. - -Thank you to our awesome contributors. Y'all are amazing: Mario Molina, Pras, Vladimir Remar, Christopher Wu, gunu, Juliano Benvenuto Piovezan, Brian M, Justinas Lukasevicius, Jonathan Stacks, Christian Gagnon, Nicolas Moreau, aristidednd, camro, minimax75, peter-mcconnell, and sashkalife! - -## 09/16/2021 Summary - -Now let's get to the 0.29.19 changelog. As with last time, bug fixes are in the thread! - -* New Destination: Databricks 🎉 -* New Source: Google Search Console -* New Source: Close.com - -* 🏗 Python CDK: Now supports auth workflows involving query params. -* 🏗 Java CDK: You can now run the connector gradle build script on Macs with M1 chips! (contributed by @Harsha Teja Kanna) - -* 💎 Google Ads source: You can now specify user-specified queries in GAQL. -* ✨ GitHub source: All streams with a parent stream use cached parent stream data when possible. -* ✨ Shopify source: Substantial performance improvements to the incremental sync mode. -* ✨ Stripe source: Now supports the PaymentIntents stream. -* ✨ Pipedrive source: Now supports the Organizations stream. -* ✨ Sendgrid source: Now supports the SingleSendStats stream. -* ✨ Bing Ads source: Now supports the Report stream. -* ✨ GitHub source: Now supports the Reactions stream. -* ✨ MongoDB source: Now Airbyte native! -* 🐛 Facebook Marketing source: Numeric values are no longer wrapped into strings. -* 🐛 Facebook Marketing source: Fetching conversion data now works as expected. (contributed by @Manav) -* 🐛 Keen destination: Timestamps are now parsed correctly. -* 🐛 S3 destination: Parquet schema parsing errors are fixed. -* 🐛 Snowflake destination: No longer syncs unnecessary tables with S3. -* 🐛 SurveyMonkey source: Cached responses are now decoded correctly. -* 🐛 Okta source: Incremental sync now works as expected. - -Also, a quick shout out to Jinni Gu and their team who made the DynamoDB destination that we announced last week! - -As usual, thank you to all of our contributors: Harsha Teja Kanna, Manav, Maciej Nędza, mauro, Brian M, Iakov Salikov, Eliziario (Marcos Santos), coeurdestenebres, and mohammadbolt. - -## 09/09/2021 Summary - -We're going over the changes from 0.29.17 and before... and there's a lot of big improvements here, so don't miss them! - -**New Source**: Facebook Pages **New Destination**: MongoDB **New Destination**: DynamoDB - -* 🎉 You can now send notifications via webhook for successes and failures on Airbyte syncs. \(This is a massive contribution by @Pras, thank you\) 🎉 -* 🎉 Scheduling jobs and worker jobs are now separated, allowing for workers to be scaled horizontally. -* 🎉 When developing a connector, you can now preview what your spec looks like in real time with this process. -* 🎉 Oracle destination: Now has basic normalization. -* 🎉 Add XLSB \(binary excel\) support to the Files source \(contributed by Muutech\). -* 🎉 You can now properly cancel K8s deployments. -* ✨ S3 source: Support for Parquet format. -* ✨ Github source: Branches, repositories, organization users, tags, and pull request stats streams added \(contributed by @Christopher Wu\). -* ✨ BigQuery destination: Added GCS upload option. -* ✨ Salesforce source: Now Airbyte native. -* ✨ Redshift destination: Optimized for performance. -* 🏗 CDK: 🎉 We’ve released a tool to generate JSON Schemas from OpenAPI specs. This should make specifying schemas for API connectors a breeze! 🎉 -* 🏗 CDK: Source Acceptance Tests now verify that connectors correctly format strings which are declared as using date-time and date formats. -* 🏗 CDK: Add private options to help in testing: \_limit and \_page\_size are now accepted by any CDK connector to minimze your output size for quick iteration while testing. -* 🐛 Fixed a bug that made it possible for connector definitions to be duplicated, violating uniqueness. -* 🐛 Pipedrive source: Output schemas no longer remove timestamp from fields. -* 🐛 Github source: Empty repos and negative backoff values are now handled correctly. -* 🐛 Harvest source: Normalization now works as expected. -* 🐛 All CDC sources: Removed sleep logic which caused exceptions when loading data from high-volume sources. -* 🐛 Slack source: Increased number of retries to tolerate flaky retry wait times on the API side. -* 🐛 Slack source: Sync operations no longer hang indefinitely. -* 🐛 Jira source: Now uses updated time as the cursor field for incremental sync instead of the created time. -* 🐛 Intercom source: Fixed inconsistency between schema and output data. -* 🐛 HubSpot source: Streams with the items property now have their schemas fixed. -* 🐛 HubSpot source: Empty strings are no longer handled as dates, fixing the deals, companies, and contacts streams. -* 🐛 Typeform source: Allows for multiple choices in responses now. -* 🐛 Shopify source: The type for the amount field is now fixed in the schema. -* 🐛 Postgres destination: \u0000\(NULL\) value processing is now fixed. - -As usual... thank you to our wonderful contributors this week: Pras, Christopher Wu, Brian M, yahu98, Michele Zuccala, jinnig, and luizgribeiro! - -## 09/01/2021 Summary - -Got the changes from 0.29.13... with some other surprises! - -* 🔥 There's a new way to create Airbyte sources! The team at Faros AI has created a Javascript/Typescript CDK which can be found here and in our docs here. This is absolutely awesome and give a huge thanks to Chalenge Masekera, Christopher Wu, eskrm, and Matthew Tovbin! -* ✨ New Destination: Azure Blob Storage ✨ - -**New Source**: Bamboo HR \(contributed by @Oren Haliva\) **New Source**: BigCommerce \(contributed by @James Wilson\) **New Source**: Trello **New Source**: Google Analytics V4 **New Source**: Amazon Ads - -* 💎 Alpine Docker images are the new standard for Python connectors, so image sizes have dropped by around 100 MB! -* ✨ You can now apply tolerations for Airbyte Pods on K8s deployments \(contributed by @Pras\). -* 🐛 Shopify source: Rate limit throttling fixed. -* 📚 We now have a doc on how to deploy Airbyte at scale. Check it out here! -* 🏗 Airbyte CDK: You can now ignore HTTP status errors and override retry parameters. - -As usual, thank you to our awesome contributors: Oren Haliva, Pras, James Wilson, and Muutech. - -## 08/26/2021 Summary - -New Source: Short.io \(contributed by @Apostol Tegko\) - -* 💎 GitHub source: Added support for rotating through multiple API tokens! -* ✨ Syncs are now scheduled with a 3 day timeout \(contributed by @Vladimir Remar\). -* ✨ Google Ads source: Added UserLocationReport stream \(contributed by @Max Krog\). -* ✨ Cart.com source: Added the order\_items stream. -* 🐛 Postgres source: Fixed out-of-memory issue with CDC interacting with large JSON blobs. -* 🐛 Intercom source: Pagination now works as expected. - -As always, thank you to our awesome community contributors this week: Apostol Tegko, Vladimir Remar, Max Krog, Pras, Marco Fontana, Troy Harvey, and damianlegawiec! - -## 08/20/2021 Summary - -Hey Airbyte community, we got some patch notes for y'all. Here's all the changes we've pushed since the last update. - -* **New Source**: S3/Abstract Files -* **New Source**: Zuora -* **New Source**: Kustomer -* **New Source**: Apify -* **New Source**: Chargebee -* **New Source**: Bing Ads - -New Destination: Keen - -* ✨ Shopify source: The `status` property is now in the `Products` stream. -* ✨ Amazon Seller Partner source: Added support for `GET_MERCHANT_LISTINGS_ALL_DATA` and `GET_FBA_INVENTORY_AGED_DATA` stream endpoints. -* ✨ GitHub source: Existing streams now don't minify the user property. -* ✨ HubSpot source: Updated user-defined custom field schema generation. -* ✨ Zendesk source: Migrated from Singer to the Airbyte CDK. -* ✨ Amazon Seller Partner source: Migrated to the Airbyte CDK. -* 🐛 Shopify source: Fixed the `products` schema to be in accordance with the API. -* 🐛 S3 source: Fixed bug where syncs could hang indefinitely. - -And as always... we'd love to shout out the awesome contributors that have helped push Airbyte forward. As a reminder, you can now see your contributions publicly reflected on our [contributors page](https://airbyte.com/contributors). - -Thank you to Rodrigo Parra, Brian Krausz, Max Krog, Apostol Tegko, Matej Hamas, Vladimir Remar, Marco Fontana, Nicholas Bull, @mildbyte, @subhaklp, and Maciej Nędza! - -## 07/30/2021 Summary - -For this week's update, we got... a few new connectors this week in 0.29.0. We found that a lot of sources can pull data directly from the underlying db instance, which we naturally already supported. - -* New Source: PrestaShop ✨ -* New Source: Snapchat Marketing ✨ -* New Source: Drupal -* New Source: Magento -* New Source: Microsoft Dynamics AX -* New Source: Microsoft Dynamics Customer Engagement -* New Source: Microsoft Dynamics GP -* New Source: Microsoft Dynamics NAV -* New Source: Oracle PeopleSoft -* New Source: Oracle Siebel CRM -* New Source: SAP Business One -* New Source: Spree Commerce -* New Source: Sugar CRM -* New Source: Wordpress -* New Source: Zencart -* 🐛 Shopify source: Fixed the products schema to be in accordance with the API -* 🐛 BigQuery source: No longer fails with nested array data types. - -View the full release highlights here: [Platform](platform.md), [Connectors](connectors.md) - -And as always, thank you to our wonderful contributors: Madison Swain-Bowden, Brian Krausz, Apostol Tegko, Matej Hamas, Vladimir Remar, Oren Haliva, satishblotout, jacqueskpoty, wallies - -## 07/23/2021 Summary - -What's going on? We just released 0.28.0 and here's the main highlights. - -* New Destination: Google Cloud Storage ✨ -* New Destination: Kafka ✨ \(contributed by @Mario Molina\) -* New Source: Pipedrive -* New Source: US Census \(contributed by @Daniel Mateus Pires \(Earnest Research\)\) -* ✨ Google Ads source: Now supports Campaigns, Ads, AdGroups, and Accounts streams. -* ✨ Stripe source: All subscription types \(including expired and canceled ones\) are now returned. -* 🐛 Facebook source: Improved rate limit management -* 🐛 Square source: The send\_request method is no longer broken due to CDK changes -* 🐛 MySQL destination: Does not fail on columns with JSON data now. - -View the full release highlights here: [Platform](platform.md), [Connectors](connectors.md) - -And as always, thank you to our wonderful contributors: Mario Molina, Daniel Mateus Pires \(Earnest Research\), gunu, Ankur Adhikari, Vladimir Remar, Madison Swain-Bowden, Maksym Pavlenok, Sam Crowder, mildbyte, avida, and gaart - -## 07/16/2021 Summary - -As for our changes this week... - -* New Source: Zendesk Sunshine -* New Source: Dixa -* New Source: Typeform -* 💎 MySQL destination: Now supports normalization! -* 💎 MSSQL source: Now supports CDC \(Change Data Capture\) -* ✨ Snowflake destination: Data coming from Airbyte is now identifiable -* 🐛 GitHub source: Now uses the correct cursor field for the IssueEvents stream -* 🐛 Square source: The send\_request method is no longer broken due to CDK changes - -View the full release highlights here: [Platform](platform.md), [Connectors](connectors.md) - -As usual, thank you to our awesome community contributors this week: Oliver Meyer, Varun, Brian Krausz, shadabshaukat, Serhii Lazebnyi, Juliano Benvenuto Piovezan, mildbyte, and Sam Crowder! - -## 07/09/2021 Summary - -* New Source: PayPal Transaction -* New Source: Square -* New Source: SurveyMonkey -* New Source: CockroachDB -* New Source: Airbyte-Native GitHub -* New Source: Airbyte-Native GitLab -* New Source: Airbyte-Native Twilio -* ✨ S3 destination: Now supports anyOf, oneOf and allOf schema fields. -* ✨ Instagram source: Migrated to the CDK and has improved error handling. -* ✨ Shopify source: Add support for draft orders. -* ✨ K8s Deployments: Now support logging to GCS. -* 🐛 GitHub source: Fixed issue with locked breaking normalization of the pull\_request stream. -* 🐛 Okta source: Fix endless loop when syncing data from logs stream. -* 🐛 PostgreSQL source: Fixed decimal handling with CDC. -* 🐛 Fixed random silent source failures. -* 📚 New document on how the CDK handles schemas. -* 🏗️ Python CDK: Now allows setting of network adapter args on outgoing HTTP requests. - -View the full release highlights here: [Platform](platform.md), [Connectors](connectors.md) - -As usual, thank you to our awesome community contributors this week: gunu, P.VAD, Rodrigo Parra, Mario Molina, Antonio Grass, sabifranjo, Jaime Farres, shadabshaukat, Rodrigo Menezes, dkelwa, Jonathan Duval, and Augustin Lafanechère. - -## 07/01/2021 Summary - -* New Destination: Google PubSub -* New Source: AWS CloudTrail - -_The risks and issues with upgrading Airbyte are now gone..._ - -* 🎉 Airbyte automatically upgrades versions safely at server startup 🎉 -* 💎 Logs on K8s are now stored in Minio by default, no S3 bucket required -* ✨ Looker Source: Supports the Run Look output stream -* ✨ Slack Source: is now Airbyte native! -* 🐛 Freshdesk Source: No longer fails after 300 pages -* 📚 New tutorial on building Java destinations - -Starting from next week, our weekly office hours will now become demo days! Drop by to get sneak peeks and new feature demos. - -* We added the \#careers channel, so if you're hiring, post your job reqs there! -* We added a \#understanding-airbyte channel to mirror [this](../../understanding-airbyte/) section on our docs site. Ask any questions about our architecture or protocol there. -* We added a \#contributing-to-airbyte channel. A lot of people ask us about how to contribute to the project, so ask away there! - -View the full release highlights here: [Platform](platform.md), [Connectors](connectors.md) - -As usual, thank you to our awesome community contributors this week: Harshith Mullapudi, Michael Irvine, and [sabifranjo](https://github.com/sabifranjo). - -## 06/24/2021 Summary - -* New Source: [IBM Db2](../../integrations/sources/db2.md) -* 💎 We now support Avro and JSONL output for our S3 destination! 💎 -* 💎 Brand new BigQuery destination flavor that now supports denormalized STRUCT types. -* ✨ Looker source now supports self-hosted instances. -* ✨ Facebook Marketing source is now migrated to the CDK, massively improving async job performance and error handling. - -View the full connector release notes [here](connectors.md). - -As usual, thank you to some of our awesome community contributors this week: Harshith Mullapudi, Tyler DeLange, Daniel Mateus Pires, EdBizarro, Tyler Schroeder, and Konrad Schlatte! - -## 06/18/2021 Summary - -* New Source: [Snowflake](../../integrations/sources/snowflake.md) -* 💎 We now support custom dbt transformations! 💎 -* ✨ We now support configuring your destination namespace at the table level when setting up a connection! -* ✨ The S3 destination now supports Minio S3 and Parquet output! - -View the full release notes here: [Platform](platform.md), [Connectors](connectors.md) - -As usual, thank you to some of our awesome community contributors this week: Tyler DeLange, Mario Molina, Rodrigo Parra, Prashanth Patali, Christopher Wu, Itai Admi, Fred Reimer, and Konrad Schlatte! - -## 06/10/2021 Summary - -* New Destination: [S3!!](../../integrations/destinations/s3.md) -* New Sources: [Harvest](../../integrations/sources/harvest.md), [Amplitude](../../integrations/sources/amplitude.md), [Posthog](../../integrations/sources/posthog.md) -* 🐛 Ensure that logs from threads created by replication workers are added to the log file. -* 🐛 Handle TINYINT\(1\) and BOOLEAN correctly and fix target file comparison for MySQL CDC. -* Jira source: now supports all available entities in Jira Cloud. -* 📚 Added a troubleshooting section, a gradle cheatsheet, a reminder on what the reset button does, and a refresh on our docs best practices. - -#### Connector Development: - -* Containerized connector code generator -* Added JDBC source connector bootstrap template. -* Added Java destination generator. - -View the full release notes highlights here: [Platform](platform.md), [Connectors](connectors.md) - -As usual, thank you to some of our awesome community contributors this week \(I've noticed that we've had more contributors to our docs, which we really appreciate\). Ping, Harshith Mullapudi, Michael Irvine, Matheus di Paula, jacqueskpoty and P.VAD. - -## Overview - -Airbyte is comprised of 2 parts: - -* Platform \(The scheduler, workers, api, web app, and the Airbyte protocol\). Here is the [changelog for Platform](platform.md). -* Connectors that run in Docker containers. Here is the [changelog for the connectors](connectors.md). - -## Airbyte Platform Releases - -### Production v. Dev Releases - -The "production" version of Airbyte is the version of the app specified in `.env`. With each production release, we update the version in the `.env` file. This version will always be available for download on DockerHub. It is the version of the app that runs when a user runs `docker compose up`. - -The "development" version of Airbyte is the head of master branch. It is the version of the app that runs when a user runs `./gradlew build && -VERSION=dev docker compose up`. - -### Production Release Schedule - -#### Scheduled Releases - -Airbyte currently releases a new minor version of the application on a weekly basis. Generally this weekly release happens on Monday or Tuesday. - -#### Hotfixes - -Airbyte releases a new version whenever it discovers and fixes a bug that blocks any mission critical functionality. - -**Mission Critical** - -e.g. Non-ASCII characters break the Salesforce source. - -**Non-Mission Critical** - -e.g. Buttons in the UI are offset. - -#### Unscheduled Releases - -We will often release more frequently than the weekly cadence if we complete a feature that we know that a user is waiting on. - -### Development Release Schedule - -As soon as a feature is on master, it is part of the development version of Airbyte. We merge features as soon as they are ready to go \(have been code reviewed and tested\). We attempt to keep the development version of the app working all the time. We are iterating quickly, however, and there may be intermittent periods where the development version is broken. - -If there is ever a feature that is only on the development version, and you need it on the production version, please let us know. We are very happy to do ad-hoc production releases if it unblocks a specific need for one of our users. - -## Airbyte Connector Releases - -Each connector is tracked with its own version. These versions are separate from the versions of Airbyte Platform. We generally will bump the version of a connector anytime we make a change to it. We rely on a large suite of tests to make sure that these changes do not cause regressions in our connectors. - -When we updated the version of a connector, we usually update the connector's version in Airbyte Platform as well. Keep in mind that you might not see the updated version of that connector in the production version of Airbyte Platform until after a production release of Airbyte Platform. - diff --git a/docs/archive/changelog/connectors.md b/docs/archive/changelog/connectors.md deleted file mode 100644 index a1f8b8126e07..000000000000 --- a/docs/archive/changelog/connectors.md +++ /dev/null @@ -1,776 +0,0 @@ ---- -description: Do not miss the new connectors we support! ---- - -# Connectors - -**You can request new connectors directly** [**here**](https://github.com/airbytehq/airbyte/issues/new?assignees=&labels=area%2Fintegration%2C+new-integration&template=new-integration-request.md&title=)**.** - -Note: Airbyte is not built on top of Singer but is compatible with Singer's protocol. Airbyte's ambitions go beyond what Singer enables us to do, so we are building our own protocol that maintains compatibility with Singer's protocol. - -Check out our [connector roadmap](https://github.com/airbytehq/airbyte/projects/3) to see what we're currently working on. - -## 1/28/2022 - -New sources: - -- [**Chartmogul**](https://docs.airbyte.com/integrations/sources/chartmogul) -- [**Hellobaton**](https://docs.airbyte.com/integrations/sources/hellobaton) -- [**Flexport**](https://docs.airbyte.com/integrations/sources/flexport) -- [**PersistIq**](https://docs.airbyte.com/integrations/sources/persistiq) - -## 1/6/2022 - -New sources: - -- [**3PL Central**](https://docs.airbyte.com/integrations/sources/tplcentral) -- [**My Hours**](https://docs.airbyte.com/integrations/sources/my-hours) -- [**Qualaroo**](https://docs.airbyte.com/integrations/sources/qualaroo) -- [**SearchMetrics**](https://docs.airbyte.com/integrations/sources/search-metrics) - -## 12/16/2021 - -New source: - -- [**OpenWeather**](https://docs.airbyte.com/integrations/sources/openweather) - -New destinations: - -- [**ClickHouse**](https://docs.airbyte.com/integrations/destinations/clickhouse) -- [**RabbitMQ**](https://docs.airbyte.com/integrations/destinations/rabbitmq) -- [**Amazon SQS**](https://docs.airbyte.com/integrations/destinations/amazon-sqs) -- [**Rockset**](https://docs.airbyte.com/integrations/destinations/rockset) - -## 12/9/2021 - -New source: - -- [**Mailgun**](https://docs.airbyte.com/integrations/sources/mailgun) - -## 12/2/2021 - -New destinations: - -- [**Redis**](https://docs.airbyte.com/integrations/destinations/redis) -- [**MQTT**](https://docs.airbyte.com/integrations/destinations/mqtt) -- [**Google Firestore**](https://docs.airbyte.com/integrations/destinations/firestore) -- [**Kinesis**](https://docs.airbyte.com/integrations/destinations/kinesis) - -## 11/25/2021 - -New sources: - -- [**Airtable**](https://docs.airbyte.com/integrations/sources/airtable) -- [**Notion**](https://docs.airbyte.com/integrations/sources/notion) -- [**Pardot**](https://docs.airbyte.com/integrations/sources/pardot) -- [**Notion**](https://docs.airbyte.com/integrations/sources/linnworks) -- [**YouTube Analytics**](https://docs.airbyte.com/integrations/sources/youtube-analytics) - -New features: - -- **Exchange Rates** Source: add `ignore_weekends` option. -- **Facebook** Source: add the videos stream. -- **Freshdesk** Source: removed the limitation in streams pagination. -- **Jira** Source: add option to render fields in HTML format. -- **MongoDB v2** Source: improve read performance. -- **Pipedrive** Source: specify schema for "persons" stream. -- **PostgreSQL** Source: exclude tables on which user doesn't have select privileges. -- **SurveyMonkey** Source: improve connection check. - -## 11/17/2021 - -New destination: - -- [**ScyllaDB**](https://docs.airbyte.com/integrations/destinations/scylla) - -New sources: - -- [**Azure Table Storage**](https://docs.airbyte.com/integrations/sources/azure-table) -- [**Linnworks**](https://docs.airbyte.com/integrations/sources/linnworks) - -New features: - -- **MySQL** Source: Now has basic performance tests. -- **Salesforce** Source: We now automatically transform and handle incorrect data for the anyType and calculated types. - -## 11/11/2021 - -New destinations: - -- [**Cassandra**](https://docs.airbyte.com/integrations/destinations/cassandra) -- [**Pulsar**](https://docs.airbyte.com/integrations/destinations/pulsar) - -New sources: - -- [**Confluence**](https://docs.airbyte.com/integrations/sources/confluence) -- [**Monday**](https://docs.airbyte.com/integrations/sources/monday) -- [**Commerce Tools**](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-commercetools) -- [**Pinterest**](https://docs.airbyte.com/integrations/sources/pinterest) - -New features: - -- **Shopify** Source: Now supports the FulfillmentOrders and Fulfillments streams. -- **Greenhouse** Source: Now supports the Demographics stream. -- **Recharge** Source: Broken requests should now be re-requested with improved backoff. -- **Stripe** Source: Now supports the checkout_sessions, checkout_sessions_line_item, and promotion_codes streams. -- **Db2** Source: Now supports SSL. - -## 11/3/2021 - -New destination: - -- [**Elasticsearch**](https://docs.airbyte.com/integrations/destinations/elasticsearch) - -New sources: - -- [**Salesloft**](https://docs.airbyte.com/integrations/sources/salesloft) -- [**OneSignal**](https://docs.airbyte.com/integrations/sources/onesignal) -- [**Strava**](https://docs.airbyte.com/integrations/sources/strava) -- [**Lemlist**](https://docs.airbyte.com/integrations/sources/lemlist) -- [**Amazon SQS**](https://docs.airbyte.com/integrations/sources/amazon-sqs) -- [**Freshservices**](https://docs.airbyte.com/integrations/sources/freshservice/) -- [**Freshsales**](https://docs.airbyte.com/integrations/sources/freshsales) -- [**Appsflyer**](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-appsflyer) -- [**Paystack**](https://docs.airbyte.com/integrations/sources/paystack) -- [**Sentry**](https://docs.airbyte.com/integrations/sources/sentry) -- [**Retently**](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-retently) -- [**Delighted!**](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-delighted) - -New features: - -- **BigQuery** Destination: You can now run transformations in batches, preventing queries from hitting BigQuery limits. (contributed by @Andrés Bravo) -- **S3** Source: Memory and Performance optimizations, also some fancy new PyArrow CSV configuration options. -- **Zuora** Source: Now supports Unlimited as an option for the Data Query Live API. -- **Clickhouse** Source: Now supports SSL and connection via SSH tunneling. - -## 10/20/2021 - -New source: - -- [**WooCommerce**](https://docs.airbyte.com/integrations/sources/woocommerce) - -New feature: - -- **MSSQL** destination: Now supports basic normalization - -## 9/29/2021 - -New sources: - -- [**LinkedIn Ads**](https://docs.airbyte.com/integrations/sources/linkedin-ads) -- [**Kafka**](https://docs.airbyte.com/integrations/sources/kafka) -- [**Lever Hiring**](https://docs.airbyte.com/integrations/sources/lever-hiring) - -New features: - -- **MySQL** destination: Now supports connection via TLS/SSL -- **BigQuery** (denormalized) destination: Supports reading BigQuery types such as date by reading the format field (contributed by @Nicolas Moreau) -- **Hubspot** source: Added contacts associations to the deals stream. -- **GitHub** source: Now supports pulling commits from user-specified branches. -- **Google Search Console** source: Now accepts admin email as input when using a service account key. -- **Greenhouse** source: Now identifies API streams it has access to if permissions are limited. -- **Marketo** source: Now Airbyte native. -- **S3** source: Now supports any source that conforms to the S3 protocol (Non-AWS S3). -- **Shopify** source: Now reports pre_tax_price on the line_items stream if you have Shopify Plus. -- **Stripe** source: Now actually uses the mandatory start_date config field for incremental syncs. - -## 9/16/2021 - -New destinations: - -- [**Databricks**](https://docs.airbyte.com/integrations/destinations/databricks) - -New sources: - -- [**Close.com**](https://docs.airbyte.com/integrations/sources/close-com) -- [**Google Search Console**](https://docs.airbyte.com/integrations/sources/google-search-console) - -New features: - -- **Google Ads** source: You can now specify user-specified queries in GAQL. -- **GitHub** source: All streams with a parent stream use cached parent stream data when possible. -- **Shopify** source: Substantial performance improvements to the incremental sync mode. -- **Stripe** source: Now supports the PaymentIntents stream. -- **Pipedrive** source: Now supports the Organizations stream. -- **Sendgrid** source: Now supports the SingleSendStats stream. -- **Bing Ads** source: Now supports the Report stream. -- **GitHub** source: Now supports the Reactions stream. -- **MongoDB** source: Now Airbyte native! - -## 9/9/2021 - -New source: - -- [**Facebook Pages**](https://docs.airbyte.com/integrations/sources/facebook-pages) - -New destinations: - -- [**MongoDB**](https://docs.airbyte.com/integrations/destinations/mongodb) -- [**DynamoDB**](https://docs.airbyte.com/integrations/destinations/dynamodb) - -New features: - -- **S3** source: Support for Parquet format. -- **Github** source: Branches, repositories, organization users, tags, and pull request stats streams added \(contributed by @Christopher Wu\). -- **BigQuery** destination: Added GCS upload option. -- **Salesforce** source: Now Airbyte native. -- **Redshift** destination: Optimized for performance. - -Bug fixes: - -- **Pipedrive** source: Output schemas no longer remove timestamp from fields. -- **Github** source: Empty repos and negative backoff values are now handled correctly. -- **Harvest** source: Normalization now works as expected. -- **All CDC sources**: Removed sleep logic which caused exceptions when loading data from high-volume sources. -- **Slack** source: Increased number of retries to tolerate flaky retry wait times on the API side. -- **Slack** source: Sync operations no longer hang indefinitely. -- **Jira** source: Now uses updated time as the cursor field for incremental sync instead of the created time. -- **Intercom** source: Fixed inconsistency between schema and output data. -- **HubSpot** source: Streams with the items property now have their schemas fixed. -- **HubSpot** source: Empty strings are no longer handled as dates, fixing the deals, companies, and contacts streams. -- **Typeform** source: Allows for multiple choices in responses now. -- **Shopify** source: The type for the amount field is now fixed in the schema. -- **Postgres** destination: \u0000\(NULL\) value processing is now fixed. - -## 9/1/2021 - -New sources: - -- [**Bamboo HR**](https://docs.airbyte.com/integrations/sources/bamboo-hr) -- [**BigCommerce**](https://docs.airbyte.com/integrations/sources/bigcommerce) -- [**Trello**](https://docs.airbyte.com/integrations/sources/trello) -- [**Google Analytics V4**](https://docs.airbyte.com/integrations/sources/google-analytics-v4) -- [**Amazon Ads**](https://docs.airbyte.com/integrations/sources/google-analytics-v4) - -Bug fixes: - -- **Shopify** source: Rate limit throttling fixed. - -## 8/26/2021 - -New source: - -- [**Short.io**](https://docs.airbyte.com/integrations/sources/shortio) - -New features: - -- **GitHub** source: Add support for rotating through multiple API tokens. -- **Google Ads** source: Added `UserLocationReport` stream. -- **Cart.com** source: Added the `order_items` stream. - -Bug fixes: - -- **Postgres** source: Fix out-of-memory issue with CDC interacting with large JSON blobs. -- **Intercom** source: Pagination now works as expected. - -## 8/18/2021 - -New source: - -- [**Bing Ads**](https://docs.airbyte.com/integrations/sources/bing-ads) - -New destination: - -- [**Keen**](https://docs.airbyte.com/integrations/destinations/keen) - -New features: - -- **Chargebee** source: Adds support for the `items`, `item prices` and `attached items` endpoints. - -Bug fixes: - -- **QuickBooks** source: Now uses the number data type for decimal fields. -- **HubSpot** source: Fixed `empty string` inside of the `number` and `float` datatypes. -- **GitHub** source: Validation fixed on non-required fields. -- **BigQuery** destination: Now supports processing of arrays of records properly. -- **Oracle** destination: Fixed destination check for users without DBA role. - -## 8/9/2021 - -New sources: - -- [**S3/Abstract Files**](https://docs.airbyte.com/integrations/sources/s3) -- [**Zuora**](https://docs.airbyte.com/integrations/sources/zuora) -- [**Kustomer**](https://docs.airbyte.com/integrations/sources/kustomer-singer/) -- [**Apify**](https://docs.airbyte.com/integrations/sources/apify-dataset) -- [**Chargebee**](https://docs.airbyte.com/integrations/sources/chargebee) - -New features: - -- **Shopify** source: The `status` property is now in the `Products` stream. -- **Amazon Seller Partner** source: Added support for `GET_MERCHANT_LISTINGS_ALL_DATA` and `GET_FBA_INVENTORY_AGED_DATA` stream endpoints. -- **GitHub** source: Existing streams now don't minify the `user` property. -- **HubSpot** source: Updated user-defined custom field schema generation. -- **Zendesk** source: Migrated from Singer to the Airbyte CDK. -- **Amazon Seller Partner** source: Migrated to the Airbyte CDK. - -Bug fixes: - -- **HubSpot** source: Casting exceptions are now logged correctly. -- **S3** source: Fixed bug where syncs could hang indefinitely. -- **Shopify** source: Fixed the `products` schema to be in accordance with the API. -- **PayPal Transactions** source: Fixed the start date minimum to be 3 years rather than 45 days. -- **Google Ads** source: Added the `login-customer-id` setting. -- **Intercom** source: Rate limit corrected from 1000 requests/minute from 1000 requests/hour. -- **S3** source: Fixed bug in spec to properly display the `format` field in the UI. - -New CDK features: - -- Now allows for setting request data in non-JSON formats. - -## 7/30/2021 - -New sources: - -- [**PrestaShop**](https://docs.airbyte.com/integrations/sources/prestashop) -- [**Snapchat Marketing**](https://docs.airbyte.com/integrations/sources/snapchat-marketing) -- [**Drupal**](https://docs.airbyte.com/integrations/sources/drupal) -- [**Magento**](https://docs.airbyte.com/integrations/sources/magento) -- [**Microsoft Dynamics AX**](https://docs.airbyte.com/integrations/sources/microsoft-dynamics-ax) -- [**Microsoft Dynamics Customer Engagement**](https://docs.airbyte.com/integrations/sources/microsoft-dynamics-customer-engagement) -- [**Microsoft Dynamics GP**](https://docs.airbyte.com/integrations/sources/microsoft-dynamics-gp) -- [**Microsoft Dynamics NAV**](https://docs.airbyte.com/integrations/sources/microsoft-dynamics-nav) -- [**Oracle PeopleSoft**](https://docs.airbyte.com/integrations/sources/oracle-peoplesoft) -- [**Oracle Siebel CRM**](https://docs.airbyte.com/integrations/sources/oracle-siebel-crm) -- [**SAP Business One**](https://docs.airbyte.com/integrations/sources/sap-business-one) -- [**Spree Commerce**](https://docs.airbyte.com/integrations/sources/spree-commerce) -- [**Sugar CRM**](https://docs.airbyte.com/integrations/sources/sugar-crm) -- [**WooCommerce**](https://docs.airbyte.com/integrations/sources/woocommerce) -- [**Wordpress**](https://docs.airbyte.com/integrations/sources/wordpress) -- [**Zencart**](https://docs.airbyte.com/integrations/sources/zencart) - -Bug fixes: - -- **Shopify** source: Fixed the `products` schema to be in accordance with the API. -- **BigQuery** source: No longer fails with `Array of Records` data types. -- **BigQuery** destination: Improved logging, Job IDs are now filled with location and Project IDs. - -## 7/23/2021 - -New sources: - -- [**Pipedrive**](https://docs.airbyte.com/integrations/sources/pipedrive) -- [**US Census**](https://docs.airbyte.com/integrations/sources/us-census) -- [**BigQuery**](https://docs.airbyte.com/integrations/sources/bigquery) - -New destinations: - -- [**Google Cloud Storage**](https://docs.airbyte.com/integrations/destinations/gcs) -- [**Kafka**](https://docs.airbyte.com/integrations/destinations/kafka) - -New Features: - -- **Java Connectors**: Now have config validators for check, discover, read, and write calls -- **Stripe** source: All subscription types are returnable \(including expired and canceled ones\). -- **Mixpanel** source: Migrated to the CDK. -- **Intercom** source: Migrated to the CDK. -- **Google Ads** source: Now supports the `Campaigns`, `Ads`, `AdGroups`, and `Accounts` streams. - -Bug Fixes: - -- **Facebook** source: Improved rate limit management -- **Instagram** source: Now supports old format for state and automatically updates it to the new format. -- **Sendgrid** source: Now gracefully handles malformed responses from API. -- **Jira** source: Fixed dbt failing to normalize schema for the labels stream. -- **MySQL** destination: Does not fail anymore with columns that contain JSON data. -- **Slack** source: Now does not fail stream slicing on reading threads. - -## 7/16/2021 - -3 new sources: - -- [**Zendesk Sunshine**](https://docs.airbyte.com/integrations/sources/zendesk-sunshine) -- [**Dixa**](https://docs.airbyte.com/integrations/sources/dixa) -- [**Typeform**](https://docs.airbyte.com/integrations/sources/typeform) - -New Features: - -- **MySQL** destination: Now supports normalization! -- **MSSQL** source: Now supports CDC \(Change Data Capture\). -- **Snowflake** destination: Data coming from Airbyte is now identifiable. -- **GitHub** source: Now handles rate limiting. - -Bug Fixes: - -- **GitHub** source: Now uses the correct cursor field for the `IssueEvents` stream. -- **Square** source: `send_request` method is no longer broken. - -## 7/08/2021 - -7 new sources: - -- [**PayPal Transaction**](https://docs.airbyte.com/integrations/sources/paypal-transaction) -- [**Square**](https://docs.airbyte.com/integrations/sources/square) -- [**SurveyMonkey**](https://docs.airbyte.com/integrations/sources/surveymonkey) -- [**CockroachDB**](https://docs.airbyte.com/integrations/sources/cockroachdb) -- [**Airbyte-native GitLab**](https://docs.airbyte.com/integrations/sources/gitlab) -- [**Airbyte-native GitHub**](https://docs.airbyte.com/integrations/sources/github) -- [**Airbyte-native Twilio**](https://docs.airbyte.com/integrations/sources/twilio) - -New Features: - -- **S3** destination: Now supports `anyOf`, `oneOf` and `allOf` schema fields. -- **Instagram** source: Migrated to the CDK and has improved error handling. -- **Snowflake** source: Now has comprehensive data type tests. -- **Shopify** source: Change the default stream cursor field to `update_at` where possible. -- **Shopify** source: Add support for draft orders. -- **MySQL** destination: Now supports normalization. - -Connector Development: - -- **Python CDK**: Now allows setting of network adapter args on outgoing HTTP requests. -- Abstract classes for non-JDBC relational database sources. - -Bugfixes: - -- **GitHub** source: Fixed issue with `locked` breaking normalization of the pull_request stream. -- **PostgreSQL** source: Fixed decimal handling with CDC. -- **Okta** source: Fix endless loop when syncing data from logs stream. - -## 7/01/2021 - -Bugfixes: - -- **Looker** source: Now supports the Run Look stream. -- **Google Adwords**: CI is fixed and new version is published. -- **Slack** source: Now Airbyte native and supports channels, channel members, messages, users, and threads streams. -- **Freshdesk** source: Does not fail after 300 pages anymore. -- **MSSQL** source: Now has comprehensive data type tests. - -## 6/24/2021 - -1 new source: - -- [**Db2**](https://docs.airbyte.com/integrations/sources/db2) - -New features: - -- **S3** destination: supports Avro and Jsonl output! -- **BigQuery** destination: now supports loading JSON data as structured data. -- **Looker** source: Now supports self-hosted instances. -- **Facebook** source: is now migrated to the CDK. - -## 6/18/2021 - -1 new source: - -- [**Snowflake**](https://docs.airbyte.com/integrations/sources/snowflake) - -New features: - -- **Postgres** source: now has comprehensive data type tests. -- **Google Ads** source: now uses the [Google Ads Query Language](https://developers.google.com/google-ads/api/docs/query/overview)! -- **S3** destination: supports Parquet output! -- **S3** destination: supports Minio S3! -- **BigQuery** destination: credentials are now optional. - -## 6/10/2021 - -1 new destination: - -- [**S3**](https://docs.airbyte.com/integrations/destinations/s3) - -3 new sources: - -- [**Harvest**](https://docs.airbyte.com/integrations/sources/harvest) -- [**Amplitude**](https://docs.airbyte.com/integrations/sources/amplitude) -- [**Posthog**](https://docs.airbyte.com/integrations/sources/posthog) - -New features: - -- **Jira** source: now supports all available entities in Jira Cloud. -- **ExchangeRatesAPI** source: clearer messages around unsupported currencies. -- **MySQL** source: Comprehensive core extension to be more compatible with other JDBC sources. -- **BigQuery** destination: Add dataset location. -- **Shopify** source: Add order risks + new attributes to orders schema for native connector - -Bugfixes: - -- **MSSQL** destination: fixed handling of unicode symbols. - -Connector development updates: - -- Containerized connector code generator. -- Added JDBC source connector bootstrap template. -- Added Java destination generator. - -## 06/3/2021 - -2 new sources: - -- [**Okta**](https://docs.airbyte.com/integrations/sources/okta) -- [**Amazon Seller Partner**](https://docs.airbyte.com/integrations/sources/amazon-seller-partner) - -New features: - -- **MySQL CDC** now only polls for 5 minutes if we haven't received any records \([\#3789](https://github.com/airbytehq/airbyte/pull/3789)\) -- **Python CDK** now supports Python 3.7.X \([\#3692](https://github.com/airbytehq/airbyte/pull/3692)\) -- **File** source: now supports Azure Blob Storage \([\#3660](https://github.com/airbytehq/airbyte/pull/3660)\) - -Bugfixes: - -- **Recurly** source: now uses type `number` instead of `integer` \([\#3769](https://github.com/airbytehq/airbyte/pull/3769)\) -- **Stripe** source: fix types in schema \([\#3744](https://github.com/airbytehq/airbyte/pull/3744)\) -- **Stripe** source: output `number` instead of `int` \([\#3728](https://github.com/airbytehq/airbyte/pull/3728)\) -- **MSSQL** destination: fix issue with unicode symbols handling \([\#3671](https://github.com/airbytehq/airbyte/pull/3671)\) - -## 05/25/2021 - -4 new sources: - -- [**Asana**](https://docs.airbyte.com/integrations/sources/asana) -- [**Klaviyo**](https://docs.airbyte.com/integrations/sources/klaviyo) -- [**Recharge**](https://docs.airbyte.com/integrations/sources/recharge) -- [**Tempo**](https://docs.airbyte.com/integrations/sources/tempo) - -Progress on connectors: - -- **CDC for MySQL** is now available! -- **Sendgrid** source: support incremental sync, as rewritten using HTTP CDK \([\#3445](https://github.com/airbytehq/airbyte/pull/3445)\) -- **Github** source bugfix: exception when parsing null date values, use `created_at` as cursor value for issue_milestones \([\#3314](https://github.com/airbytehq/airbyte/pull/3314)\) -- **Slack** source bugfix: don't overwrite thread_ts in threads stream \([\#3483](https://github.com/airbytehq/airbyte/pull/3483)\) -- **Facebook Marketing** source: allow configuring insights lookback window \([\#3396](https://github.com/airbytehq/airbyte/pull/3396)\) -- **Freshdesk** source: fix discovery \([\#3591](https://github.com/airbytehq/airbyte/pull/3591)\) - -## 05/18/2021 - -1 new destination: [**MSSQL**](https://docs.airbyte.com/integrations/destinations/mssql) - -1 new source: [**ClickHouse**](https://docs.airbyte.com/integrations/sources/clickhouse) - -Progress on connectors: - -- **Shopify**: make this source more resilient to timeouts \([\#3409](https://github.com/airbytehq/airbyte/pull/3409)\) -- **Freshdesk** bugfix: output correct schema for various streams \([\#3376](https://github.com/airbytehq/airbyte/pull/3376)\) -- **Iterable**: update to use latest version of CDK \([\#3378](https://github.com/airbytehq/airbyte/pull/3378)\) - -## 05/11/2021 - -1 new destination: [**MySQL**](https://docs.airbyte.com/integrations/destinations/mysql) - -2 new sources: - -- [**Google Search Console**](https://docs.airbyte.com/integrations/sources/google-search-console) -- [**PokeAPI**](https://docs.airbyte.com/integrations/sources/pokeapi) \(talking about long tail and having fun ;\)\) - -Progress on connectors: - -- **Zoom**: bugfix on declaring correct types to match data coming from API \([\#3159](https://github.com/airbytehq/airbyte/pull/3159)\), thanks to [vovavovavovavova](https://github.com/vovavovavovavova) -- **Smartsheets**: bugfix on gracefully handling empty cell values \([\#3337](https://github.com/airbytehq/airbyte/pull/3337)\), thanks to [Nathan Nowack](https://github.com/zzstoatzz) -- **Stripe**: fix date property name, only add connected account header when set, and set primary key \(\#3210\), thanks to [Nathan Yergler](https://github.com/nyergler) - -## 05/04/2021 - -2 new sources: - -- [**Smartsheets**](https://docs.airbyte.com/integrations/sources/smartsheets), thanks to [Nathan Nowack](https://github.com/zzstoatzz) -- [**Zendesk Chat**](https://docs.airbyte.com/integrations/sources/zendesk-chat) - -Progress on connectors: - -- **Appstore**: bugfix private key handling in the UI \([\#3201](https://github.com/airbytehq/airbyte/pull/3201)\) -- **Facebook marketing**: Wait longer \(5 min\) for async jobs to start \([\#3116](https://github.com/airbytehq/airbyte/pull/3116)\), thanks to [Max Krog](https://github.com/MaxKrog) -- **Stripe**: support reading data from connected accounts \(\#3121\), and 2 new streams with Refunds & Bank Accounts \([\#3030](https://github.com/airbytehq/airbyte/pull/3030)\) \([\#3086](https://github.com/airbytehq/airbyte/pull/3086)\) -- **Redshift destination**: Ignore records that are too big \(instead of failing\) \([\#2988](https://github.com/airbytehq/airbyte/pull/2988)\) -- **MongoDB**: add supporting TLS and Replica Sets \([\#3111](https://github.com/airbytehq/airbyte/pull/3111)\) -- **HTTP sources**: bugfix on handling array responses gracefully \([\#3008](https://github.com/airbytehq/airbyte/pull/3008)\) - -## 04/27/2021 - -- **Zendesk Talk**: fix normalization failure \([\#3022](https://github.com/airbytehq/airbyte/pull/3022)\), thanks to [yevhenii-ldv](https://github.com/yevhenii-ldv) -- **Github**: pull_requests stream only incremental syncs \([\#2886](https://github.com/airbytehq/airbyte/pull/2886)\) \([\#3009](https://github.com/airbytehq/airbyte/pull/3009)\), thanks to [Zirochkaa](https://github.com/Zirochkaa) -- Create streaming writes to a file and manage the issuance of copy commands for the destination \([\#2921](https://github.com/airbytehq/airbyte/pull/2921)\) -- **Redshift**: make Redshift part size configurable. \([\#3053](https://github.com/airbytehq/airbyte/pull/23053)\) -- **HubSpot**: fix argument error in log call \(\#3087\) \([\#3087](https://github.com/airbytehq/airbyte/pull/3087)\) , thanks to [Nathan Yergler](https://github.com/nyergler) - -## 04/20/2021 - -3 new source connectors! - -- [**Zendesk Talk**](https://docs.airbyte.com/integrations/sources/zendesk-talk) -- [**Iterable**](https://docs.airbyte.com/integrations/sources/iterable) -- [**QuickBooks**](https://docs.airbyte.com/integrations/sources/quickbooks-singer) - -Other progress on connectors: - -- **Postgres source/destination**: add SSL option, thanks to [Marcos Marx](https://github.com/marcosmarxm) \([\#2757](https://github.com/airbytehq/airbyte/pull/2757)\) -- **Google sheets bugfix**: handle duplicate sheet headers, thanks to [Aneesh Makala](https://github.com/makalaaneesh) \([\#2905](https://github.com/airbytehq/airbyte/pull/2905)\) -- **Source Google Adwords**: support specifying the lookback window for conversions, thanks to [Harshith Mullapudi](https://github.com/harshithmullapudi) \([\#2918](https://github.com/airbytehq/airbyte/pull/2918)\) -- **MongoDB improvement**: speed up mongodb schema discovery, thanks to [Yury Koleda](https://github.com/FUT) \([\#2851](https://github.com/airbytehq/airbyte/pull/2851)\) -- **MySQL bugfix**: parsing Mysql jdbc params, thanks to [Vasily Safronov](https://github.com/gingeard) \([\#2891](https://github.com/airbytehq/airbyte/pull/2891)\) -- **CSV bugfix**: discovery takes too much memory \([\#2089](https://github.com/airbytehq/airbyte/pull/2851)\) -- A lot of work was done on improving the standard tests for the connectors, for better standardization and maintenance! - -## 04/13/2021 - -- New connector: [**Oracle DB**](https://docs.airbyte.com/integrations/sources/oracle), thanks to [Marcos Marx](https://github.com/marcosmarxm) - -## 04/07/2021 - -- New connector: [**Google Workspace Admin Reports**](https://docs.airbyte.com/integrations/sources/google-workspace-admin-reports) \(audit logs\) -- Bugfix in the base python connector library that caused errors to be silently skipped rather than failing the sync -- **Exchangeratesapi.io** bugfix: to point to the updated API URL -- **Redshift destination** bugfix: quote keywords “DATETIME” and “TIME” when used as identifiers -- **GitHub** bugfix: syncs failing when a personal repository doesn’t contain collaborators or team streams available -- **Mixpanel** connector: sync at most the last 90 days of data in the annotations stream to adhere to API limits - -## 03/29/2021 - -- We started measuring throughput of connectors. This will help us improve that point for all connectors. -- **Redshift**: implemented Copy strategy to improve its throughput. -- **Instagram**: bugfix an issue which caused media and media_insights streams to stop syncing prematurely. -- Support NCHAR and NVCHAR types in SQL-based database sources. -- Add the ability to specify a custom JDBC parameters for the MySQL source connector. - -## 03/22/2021 - -- 2 new source connectors: [**Gitlab**](https://docs.airbyte.com/integrations/sources/gitlab) and [**Airbyte-native HubSpot**](https://docs.airbyte.com/integrations/sources/hubspot) -- Developing connectors now requires almost no interaction with Gradle, Airbyte’s monorepo build tool. If you’re building a Python connector, you never have to worry about developing outside your typical flow. See [the updated documentation](https://docs.airbyte.com/connector-development). - -## 03/15/2021 - -- 2 new source connectors: [**Instagram**](https://docs.airbyte.com/integrations/sources/instagram) and [**Google Directory**](https://docs.airbyte.com/integrations/sources/google-directory) -- **Facebook Marketing**: support of API v10 -- **Google Analytics**: support incremental sync -- **Jira**: bug fix to consistently pull all tickets -- **HTTP Source**: bug fix to correctly parse JSON responses consistently - -## 03/08/2021 - -- 1 new source connector: **MongoDB** -- **Google Analytics**: Support chunked syncs to avoid sampling -- **AppStore**: fix bug where the catalog was displayed incorrectly - -## 03/01/2021 - -- **New native HubSpot connector** with schema folder populated -- Facebook Marketing connector: add option to include deleted records - -## 02/22/2021 - -- Bug fixes: - - **Google Analytics:** add the ability to sync custom reports - - **Apple Appstore:** bug fix to correctly run incremental syncs - - **Exchange rates:** UI now correctly validates input date pattern - - **File Source:** Support JSONL \(newline-delimited JSON\) format - - **Freshdesk:** Enable controlling how many requests per minute the connector makes to avoid overclocking rate limits - -## 02/15/2021 - -- 1 new destination connector: [MeiliSearch](https://docs.airbyte.com/integrations/destinations/meilisearch) -- 2 new sources that support incremental append: [Freshdesk](https://docs.airbyte.com/integrations/sources/freshdesk) and [Sendgrid](https://docs.airbyte.com/integrations/sources/sendgrid) -- Other fixes: - - Thanks to [@ns-admetrics](https://github.com/ns-admetrics) for contributing an upgrade to the **Shopify** source connector which now provides the landing_site field containing UTM parameters in the Orders table. - - **Sendgrid** source connector supports most available endpoints available in the API - - **Facebook** Source connector now supports syncing Ad Insights data - - **Freshdesk** source connector now supports syncing satisfaction ratings and conversations - - **Microsoft Teams** source connector now gracefully handles rate limiting - - Bug fix in **Slack** source where the last few records in a sync were sporadically dropped - - Bug fix in **Google Analytics** source where the last few records in sync were sporadically dropped - - In **Redshift source**, support non alpha-numeric table names - - Bug fix in **Github Source** to fix instances where syncs didn’t always fail if there was an error while reading data from the API - -## 02/02/2021 - -- Sources that we improved reliability for \(and that became “certified”\): - - [Certified sources](https://docs.airbyte.com/integrations): Files and Shopify - - Enhanced continuous testing for Tempo and Looker sources -- Other fixes / features: - - Correctly handle boolean types in the File Source - - Add docs for [App Store](https://docs.airbyte.com/integrations/sources/appstore) source - - Fix a bug in Snowflake destination where the connector didn’t check for all needed write permissions, causing some syncs to fail - -## 01/26/2021 - -- Improved reliability with our best practices on : Google Sheets, Google Ads, Marketo, Tempo -- Support incremental for Facebook and Google Ads -- The Facebook connector now supports the FB marketing API v9 - -## 01/19/2021 - -- **Our new** [**Connector Health Grade**](../../integrations/) **page** -- **1 new source:** App Store \(thanks to [@Muriloo](https://github.com/Muriloo)\) -- Fixes on connectors: - - Bug fix writing boolean columns to Redshift - - Bug fix where getting a connector’s input configuration hung indefinitely - - Stripe connector now gracefully handles rate limiting from the Stripe API - -## 01/12/2021 - -- **1 new source:** Tempo \(thanks to [@thomasvl](https://github.com/thomasvl)\) -- **Incremental support for 3 new source connectors:** [Salesforce](../../integrations/sources/salesforce.md), [Slack](../../integrations/sources/slack.md) and [Braintree](../../integrations/sources/braintree.md) -- Fixes on connectors: - - Fix a bug in MSSQL and Redshift source connectors where custom SQL types weren't being handled correctly. - - Improvement of the Snowflake connector from [@hudsondba](https://github.com/hudsondba) \(batch size and timeout sync\) - -## 01/05/2021 - -- **Incremental support for 2 new source connectors:** [Mixpanel](../../integrations/sources/mixpanel.md) and [HubSpot](../../integrations/sources/hubspot.md) -- Fixes on connectors: - - Fixed a bug in the github connector where the connector didn’t verify the provided API token was granted the correct permissions - - Fixed a bug in the Google sheets connector where rate limits were not always respected - - Alpha version of Facebook marketing API v9. This connector is a native Airbyte connector \(current is Singer based\). - -## 12/30/2020 - -**New sources:** [Plaid](../../integrations/sources/plaid.md) \(contributed by [tgiardina](https://github.com/tgiardina)\), [Looker](../../integrations/sources/looker.md) - -## 12/18/2020 - -**New sources:** [Drift](../../integrations/sources/drift.md), [Microsoft Teams](../../integrations/sources/microsoft-teams.md) - -## 12/10/2020 - -**New sources:** [Intercom](../../integrations/sources/intercom.md), [Mixpanel](../../integrations/sources/mixpanel.md), [Jira Cloud](../../integrations/sources/jira.md), [Zoom](../../integrations/sources/zoom.md) - -## 12/07/2020 - -**New sources:** [Slack](../../integrations/sources/slack.md), [Braintree](../../integrations/sources/braintree.md), [Zendesk Support](../../integrations/sources/zendesk-support.md) - -## 12/04/2020 - -**New sources:** [Redshift](../../integrations/sources/redshift.md), [Greenhouse](../../integrations/sources/greenhouse.md) **New destination:** [Redshift](../../integrations/destinations/redshift.md) - -## 11/30/2020 - -**New sources:** [Freshdesk](../../integrations/sources/freshdesk.md), [Twilio](../../integrations/sources/twilio.md) - -## 11/25/2020 - -**New source:** [Recurly](../../integrations/sources/recurly.md) - -## 11/23/2020 - -**New source:** [Sendgrid](../../integrations/sources/sendgrid.md) - -## 11/18/2020 - -**New source:** [Mailchimp](../../integrations/sources/mailchimp.md) - -## 11/13/2020 - -**New source:** [MSSQL](../../integrations/sources/mssql.md) - -## 11/11/2020 - -**New source:** [Shopify](../../integrations/sources/shopify.md) - -## 11/09/2020 - -**New sources:** [Files \(CSV, JSON, HTML...\)](../../integrations/sources/file.md) - -## 11/04/2020 - -**New sources:** [Facebook Ads](connectors.md), [Google Ads](../../integrations/sources/google-ads.md), [Marketo](../../integrations/sources/marketo.md) **New destination:** [Snowflake](../../integrations/destinations/snowflake.md) - -## 10/30/2020 - -**New sources:** [Salesforce](../../integrations/sources/salesforce.md), Google Analytics, [HubSpot](../../integrations/sources/hubspot.md), [GitHub](../../integrations/sources/github.md), [Google Sheets](../../integrations/sources/google-sheets.md), [Rest APIs](connectors.md), and [MySQL](../../integrations/sources/mysql.md) - -## 10/21/2020 - -**New destinations:** we built our own connectors for [BigQuery](../../integrations/destinations/bigquery.md) and [Postgres](../../integrations/destinations/postgres.md), to ensure they are of the highest quality. - -## 09/23/2020 - -**New sources:** [Stripe](../../integrations/sources/stripe.md), [Postgres](../../integrations/sources/postgres.md) **New destinations:** [BigQuery](../../integrations/destinations/bigquery.md), [Postgres](../../integrations/destinations/postgres.md), [local CSV](../../integrations/destinations/csv.md) diff --git a/docs/archive/changelog/platform.md b/docs/archive/changelog/platform.md deleted file mode 100644 index 92bc158dce83..000000000000 --- a/docs/archive/changelog/platform.md +++ /dev/null @@ -1,509 +0,0 @@ ---- -description: Be sure to not miss out on new features and improvements! ---- - -# Platform - -This is the changelog for Airbyte Platform. For our connector changelog, please visit our [Connector Changelog](connectors.md) page. - -## [20-12-2021 - 0.32.5](https://github.com/airbytehq/airbyte/releases/tag/v0.32.5-alpha) -* Add an endpoint that specify that the feedback have been given after the first sync. - -## [18-12-2021 - 0.32.4](https://github.com/airbytehq/airbyte/releases/tag/v0.32.4-alpha) -* No major changes to Airbyte Core. - -## [18-12-2021 - 0.32.3](https://github.com/airbytehq/airbyte/releases/tag/v0.32.3-alpha) -* No major changes to Airbyte Core. - -## [18-12-2021 - 0.32.2](https://github.com/airbytehq/airbyte/releases/tag/v0.32.2-alpha) -* Improve error handling when additional sources/destinations cannot be read. -* Implement connector config dependency for OAuth consent URL. -* Treat oauthFlowInitParameters just as hidden instead of getting rid of them. -* Stop using gentle close with heartbeat. - -## [17-12-2021 - 0.32.1](https://github.com/airbytehq/airbyte/releases/tag/v0.32.1-alpha) -* Add to the new connection flow form with an existing source and destination dropdown. -* Implement protocol change for OAuth outputs. -* Enhance API for use by cloud to provide per-connector billing info. - -## [11-12-2021 - 0.32.0](https://github.com/airbytehq/airbyte/releases/tag/v0.32.0-alpha) -* This is a **MAJOR** version update. You need to [update to this version](../../operator-guides/upgrading-airbyte.md#mandatory-intermediate-upgrade) before updating to any version newer than `0.32.0` - -## [11-11-2021 - 0.31.0](https://github.com/airbytehq/airbyte/releases/tag/v0.31.0-alpha) -* No major changes to Airbyte Core. - -## [11-11-2021 - 0.30.39](https://github.com/airbytehq/airbyte/releases/tag/v0.30.39-alpha) -* We migrated our secret management to Google Secret Manager, allowing us to scale how many connectors we support. - -## [11-09-2021 - 0.30.37](https://github.com/airbytehq/airbyte/releases/tag/v0.30.37-alpha) -* No major changes to Airbyte Core. - -## [11-09-2021 - 0.30.36](https://github.com/airbytehq/airbyte/releases/tag/v0.30.36-alpha) -* No major changes to Airbyte Core. - -## [11-08-2021 - 0.30.35](https://github.com/airbytehq/airbyte/releases/tag/v0.30.35-alpha) -* No major changes to Airbyte Core. - -## [11-06-2021 - 0.30.34](https://github.com/airbytehq/airbyte/releases/tag/v0.30.34-alpha) -* No major changes to Airbyte Core. - -## [11-06-2021 - 0.30.33](https://github.com/airbytehq/airbyte/releases/tag/v0.30.33-alpha) -* No major changes to Airbyte Core. - -## [11-05-2021 - 0.30.32](https://github.com/airbytehq/airbyte/releases/tag/v0.30.32-alpha) -* Airbyte Server no longer crashes from having too many open files. - -## [11-04-2021 - 0.30.31](https://github.com/airbytehq/airbyte/releases/tag/v0.30.31-alpha) -* No major changes to Airbyte Core. - -## [11-01-2021 - 0.30.25](https://github.com/airbytehq/airbyte/releases/tag/v0.30.25-alpha) -* No major changes to Airbyte Core. - -## [11-01-2021 - 0.30.24](https://github.com/airbytehq/airbyte/releases/tag/v0.30.24-alpha) -* Incremental normalization is live. Basic normalization no longer runs on already normalized data, making it way faster and cheaper. - -## [11-01-2021 - 0.30.23](https://github.com/airbytehq/airbyte/releases/tag/v0.30.23-alpha) -* No major changes to Airbyte Core. - -## [10-21-2021 - 0.30.22](https://github.com/airbytehq/airbyte/releases/tag/v0.30.22-alpha) -* We now support experimental deployment of Airbyte on Macbooks with M1 chips! - -:::info - -This interim patch period mostly contained stability changes for Airbyte Cloud, so we skipped from `0.30.16` to `0.30.22`. - -::: - -## [10-07-2021 - 0.30.16](https://github.com/airbytehq/airbyte/releases/tag/v0.30.16-alpha) -* On Kubernetes deployments, you can now configure the Airbyte Worker Pod's image pull policy. - -:::info - -This interim patch period mostly contained stability changes for Airbyte Cloud, so we skipped from `0.30.2` to `0.30.16`. - -::: - -## [09-30-2021 - 0.30.2](https://github.com/airbytehq/airbyte/releases/tag/v0.30.2-alpha) -* Fixed a bug that would fail Airbyte upgrades for deployments with sync notifications. - -## [09-24-2021 - 0.29.22](https://github.com/airbytehq/airbyte/releases/tag/v0.29.22-alpha) -* We now have integration tests for SSH. - -## [09-19-2021 - 0.29.21](https://github.com/airbytehq/airbyte/releases/tag/v0.29.21-alpha) -* You can now [deploy Airbyte on Kubernetes with a Helm Chart](https://github.com/airbytehq/airbyte/pull/5891)! - -## [09-16-2021 - 0.29.19](https://github.com/airbytehq/airbyte/releases/tag/v0.29.19-alpha) -* Fixes a breaking bug that prevents Airbyte upgrading from older versions. - -## [09-15-2021 - 0.29.18](https://github.com/airbytehq/airbyte/releases/tag/v0.29.18-alpha) -* Building images is now optional in the CI build. - -## [09-08-2021 - 0.29.17](https://github.com/airbytehq/airbyte/releases/tag/v0.29.17-alpha) - -* You can now properly cancel deployments when deploying on K8s. - -## [09-08-2021 - 0.29.16](https://github.com/airbytehq/airbyte/releases/tag/v0.29.16-alpha) - -* You can now send notifications via webhook for successes and failures on Airbyte syncs. -* Scheduling jobs and worker jobs are now separated, allowing for workers to be scaled horizontally. - -## [09-04-2021 - 0.29.15](https://github.com/airbytehq/airbyte/releases/tag/v0.29.15-alpha) - -* Fixed a bug that made it possible for connector definitions to be duplicated, violating uniqueness. - -## [09-02-2021 - 0.29.14](https://github.com/airbytehq/airbyte/releases/tag/v0.29.14-alpha) - -* Nothing of note. - -## [08-27-2021 - 0.29.13](https://github.com/airbytehq/airbyte/releases/tag/v0.29.13-alpha) - -* The scheduler now waits for the server before it creates any databases. -* You can now apply tolerations for Airbyte Pods on K8s deployments. - -## [08-23-2021 - 0.29.12](https://github.com/airbytehq/airbyte/releases/tag/v0.29.12-alpha) - -* Syncs now have a `max_sync_timeout` that times them out after 3 days. -* Fixed Kube deploys when logging with Minio. - -## [08-20-2021 - 0.29.11](https://github.com/airbytehq/airbyte/releases/tag/v0.29.11-alpha) - -* Nothing of note. - -## [08-20-2021 - 0.29.10](https://github.com/airbytehq/airbyte/releases/tag/v0.29.10-alpha) - -* Migration of Python connector template images to Alpine Docker images to reduce size. - -## [08-20-2021 - 0.29.9](https://github.com/airbytehq/airbyte/releases/tag/v0.29.9-alpha) - -* Nothing of note. - -## [08-17-2021 - 0.29.8](https://github.com/airbytehq/airbyte/releases/tag/v0.29.8-alpha) - -* Nothing of note. - -## [08-14-2021 - 0.29.7](https://github.com/airbytehq/airbyte/releases/tag/v0.29.7-alpha) - -* Re-release: Fixed errant ENV variable in `0.29.6` - -## [08-14-2021 - 0.29.6](https://github.com/airbytehq/airbyte/releases/tag/v0.29.6-alpha) - -* Connector pods no longer fail with edge case names for the associated Docker images. - -## [08-14-2021 - 0.29.5](https://github.com/airbytehq/airbyte/releases/tag/v0.29.5-alpha) - -* Nothing of note. - -## [08-12-2021 - 0.29.4](https://github.com/airbytehq/airbyte/releases/tag/v0.29.4-alpha) - -* Introduced implementation for date-time support in normalization. - -## [08-9-2021 - 0.29.3](https://github.com/airbytehq/airbyte/releases/tag/v0.29.3-alpha) - -* Importing configuration no longer removes available but unused connectors. - -## [08-6-2021 - 0.29.2](https://github.com/airbytehq/airbyte/releases/tag/v0.29.2-alpha) - -* Fixed nil pointer exception in version migrations. - -## [07-29-2021 - 0.29.1](https://github.com/airbytehq/airbyte/releases/tag/v0.29.1-alpha) - -* When migrating, types represented in the config archive need to be a subset of the types declared in the schema. - -## [07-28-2021 - 0.29.0](https://github.com/airbytehq/airbyte/releases/tag/v0.29.0-alpha) - -* Deprecated `DEFAULT_WORKSPACE_ID`; default workspace no longer exists by default. - -## [07-28-2021 - 0.28.2](https://github.com/airbytehq/airbyte/releases/tag/v0.28.2-alpha) - -* Backend now handles workspaceId for WebBackend operations. - -## [07-26-2021 - 0.28.1](https://github.com/airbytehq/airbyte/releases/tag/v0.28.1-alpha) - -* K8s: Overly-sensitive logs are now silenced. - -## [07-22-2021 - 0.28.0](https://github.com/airbytehq/airbyte/releases/tag/v0.28.0-alpha) - -* Acceptance test dependencies fixed. - -## [07-22-2021 - 0.27.5](https://github.com/airbytehq/airbyte/releases/tag/v0.27.5-alpha) - -* Fixed unreliable logging on Kubernetes deployments. -* Introduced pre-commit to auto-format files on commits. - -## [07-21-2021 - 0.27.4](https://github.com/airbytehq/airbyte/releases/tag/v0.27.4-alpha) - -* Config persistence is now migrated to the internal Airbyte database. -* Source connector ports now properly close when deployed on Kubernetes. -* Missing dependencies added that allow acceptance tests to run. - -## [07-15-2021 - 0.27.3](https://github.com/airbytehq/airbyte/releases/tag/v0.27.3-alpha) - -* Fixed some minor API spec errors. - -## [07-12-2021 - 0.27.2](https://github.com/airbytehq/airbyte/releases/tag/v0.27.2-alpha) - -* GCP environment variable is now stubbed out to prevent noisy and harmless errors. - -## [07-8-2021 - 0.27.1](https://github.com/airbytehq/airbyte/releases/tag/v0.27.1-alpha) - -* New API endpoint: List workspaces -* K8s: Server doesn't start up before Temporal is ready to operate now. -* Silent source failures caused by last patch fixed to throw exceptions. - -## [07-1-2021 - 0.27.0](https://github.com/airbytehq/airbyte/releases/tag/v0.27.0-alpha) - -* Airbyte now automatically upgrades on server startup! - * Airbyte will check whether your `.env` Airbyte version is compatible with the Airbyte version in the database and upgrade accordingly. -* When running Airbyte on K8s logs will automatically be stored in a Minio bucket unless configured otherwise. -* CDC for MySQL now handles decimal types correctly. - -## [06-21-2021 - 0.26.2](https://github.com/airbytehq/airbyte/releases/tag/v0.26.2-alpha) - -* First-Class Kubernetes support! - -## [06-16-2021 - 0.26.0](https://github.com/airbytehq/airbyte/releases/tag/v0.26.0-alpha) - -* Custom dbt transformations! -* You can now configure your destination namespace at the table level when setting up a connection! -* Migrate basic normalization settings to the sync operations. - -## [06-09-2021 - 0.24.8 / 0.25.0](https://github.com/airbytehq/airbyte/releases/tag/v0.24.8-alpha) - -* Bugfix: Handle TINYINT\(1\) and BOOLEAN correctly and fix target file comparison for MySQL CDC. -* Bugfix: Updating the source/destination name in the UI now works as intended. - -## [06-04-2021 - 0.24.7](https://github.com/airbytehq/airbyte/releases/tag/v0.24.7-alpha) - -* Bugfix: Ensure that logs from threads created by replication workers are added to the log file. - -## [06-03-2021 - 0.24.5](https://github.com/airbytehq/airbyte/releases/tag/v0.24.5-alpha) - -* Remove hash from table names when it's not necessary for normalization outputs. - -## [06-03-2021 - 0.24.4](https://github.com/airbytehq/airbyte/releases/tag/v0.24.4-alpha) - -* PythonCDK: change minimum Python version to 3.7.0 - -## [05-28-2021 - 0.24.3](https://github.com/airbytehq/airbyte/releases/tag/v0.24.3-alpha) - -* Minor fixes to documentation -* Reliability updates in preparation for custom transformations -* Limit Docker log size to 500 MB \([\#3702](https://github.com/airbytehq/airbyte/pull/3702)\) - -## [05-26-2021 - 0.24.2](https://github.com/airbytehq/airbyte/releases/tag/v0.24.2-alpha) - -* Fix for file names being too long in Windows deployments \([\#3625](https://github.com/airbytehq/airbyte/pull/3625)\) -* Allow users to access the API and WebApp from the same port \([\#3603](https://github.com/airbytehq/airbyte/pull/3603)\) - -## [05-25-2021 - 0.24.1](https://github.com/airbytehq/airbyte/releases/tag/v0.24.1-alpha) - -* **Checkpointing for incremental syncs** that will now continue where they left off even if they fail! \([\#3290](https://github.com/airbytehq/airbyte/pull/3290)\) - -## [05-25-2021 - 0.24.0](https://github.com/airbytehq/airbyte/releases/tag/v0.24.0-alpha) - -* Avoid dbt runtime exception "maximum recursion depth exceeded" in ephemeral materialization \([\#3470](https://github.com/airbytehq/airbyte/pull/3470)\) - -## [05-18-2021 - 0.23.0](https://github.com/airbytehq/airbyte/releases/tag/v0.23.0-alpha) - -* Documentation to deploy locally on Windows is now available \([\#3425](https://github.com/airbytehq/airbyte/pull/3425)\) -* Connector icons are now displayed in the UI -* Restart core containers if they fail automatically \([\#3423](https://github.com/airbytehq/airbyte/pull/3423)\) -* Progress on supporting custom transformation using dbt. More updates on this soon! - -## [05-11-2021 - 0.22.3](https://github.com/airbytehq/airbyte/releases/tag/v0.22.3-alpha) - -* Bump K8s deployment version to latest stable version, thanks to [Coetzee van Staden](https://github.com/coetzeevs) -* Added tutorial to deploy Airbyte on Azure VM \([\#3171](https://github.com/airbytehq/airbyte/pull/3171)\), thanks to [geekwhocodes](https://github.com/geekwhocodes) -* Progress on checkpointing to support rate limits better -* Upgrade normalization to use dbt from docker images \([\#3186](https://github.com/airbytehq/airbyte/pull/3186)\) - -## [05-04-2021 - 0.22.2](https://github.com/airbytehq/airbyte/releases/tag/v0.22.2-alpha) - -* Split replication and normalization into separate temporal activities \([\#3136](https://github.com/airbytehq/airbyte/pull/3136)\) -* Fix normalization Nesting bug \([\#3110](https://github.com/airbytehq/airbyte/pull/3110)\) - -## [04-27-2021 - 0.22.0](https://github.com/airbytehq/airbyte/releases/tag/v0.22.0-alpha) - -* **Replace timeout for sources** \([\#3031](https://github.com/airbytehq/airbyte/pull/2851)\) -* Fix UI issue where tables with the same name are selected together \([\#3032](https://github.com/airbytehq/airbyte/pull/2851)\) -* Fix feed handling when feeds are unavailable \([\#2964](https://github.com/airbytehq/airbyte/pull/2851)\) -* Export whitelisted tables \([\#3055](https://github.com/airbytehq/airbyte/pull/2851)\) -* Create a contributor bootstrap script \(\#3028\) \([\#3054](https://github.com/airbytehq/airbyte/pull/2851)\), thanks to [nclsbayona](https://github.com/nclsbayona) - -## [04-20-2021 - 0.21.0](https://github.com/airbytehq/airbyte/releases/tag/v0.21.0-alpha) - -* **Namespace support**: supported source-destination pairs will now sync data into the same namespace as the source \(\#2862\) -* Add **“Refresh Schema”** button \([\#2943](https://github.com/airbytehq/airbyte/pull/2943)\) -* In the Settings, you can now **add a webhook to get notified when a sync fails** -* Add destinationSyncModes to connection form -* Add tooltips for connection status icons - -## [04-12-2021 - 0.20.0](https://github.com/airbytehq/airbyte/releases/tag/v0.20.0-alpha) - -* **Change Data Capture \(CDC\)** is now supported for Postgres, thanks to [@jrhizor](https://github.com/jrhizor) and [@cgardens](https://github.com/cgardens). We will now expand it to MySQL and MSSQL in the coming weeks. -* When displaying the schema for a source, you can now search for table names, thanks to [@jamakase](https://github.com/jamakase) -* Better feedback UX when manually triggering a sync with “Sync now” - -## [04-07-2021 - 0.19.0](https://github.com/airbytehq/airbyte/releases/tag/v0.19.0-alpha) - -* New **Connections** page where you can see the list of all your connections and their statuses. -* New **Settings** page to update your preferences. -* Bugfix where very large schemas caused schema discovery to fail. - -## [03-29-2021 - 0.18.1](https://github.com/airbytehq/airbyte/releases/tag/v0.18.1-alpha) - -* Surface the **health of each connection** so that a user can spot any problems at a glance. -* Added support for deduplicating records in the destination using a primary key using incremental dedupe - -* A source’s extraction mode \(incremental, full refresh\) is now decoupled from the destination’s write mode -- so you can repeatedly append full refreshes to get repeated snapshots of data in your source. -* New **Upgrade all** button in Admin to upgrade all your connectors at once -* New **Cancel** job button in Connections Status page when a sync job is running, so you can stop never-ending processes. - -## [03-22-2021 - 0.17.2](https://github.com/airbytehq/airbyte/releases/tag/v0.17.2-alpha) - -* Improved the speed of get spec, check connection, and discover schema by migrating to the Temporal workflow engine. -* Exposed cancellation for sync jobs in the API \(will be exposed in the UI in the next week!\). -* Bug fix: Fix issue where migration app was OOMing. - -## [03-15-2021 - 0.17.1](https://github.com/airbytehq/airbyte/releases/tag/v0.17.1-alpha) - -* **Creating and deleting multiple workspaces** is now supported via the API. Thanks to [@Samuel Gordalina](https://github.com/gordalina) for contributing this feature! -* Normalization now supports numeric types with precision greater than 32 bits -* Normalization now supports union data types -* Support longform text inputs in the UI for cases where you need to preserve formatting on connector inputs like .pem keys -* Expose the latest available connector versions in the API -* Airflow: published a new [tutorial](https://docs.airbyte.com/operator-guides/using-the-airflow-airbyte-operator/) for how to use the Airbyte operator. Thanks [@Marcos Marx](https://github.com/marcosmarxm) for writing the tutorial! -* Connector Contributions: All connectors now describe how to contribute to them without having to touch Airbyte’s monorepo build system -- just work on the connector in your favorite dev setup! - -## [03-08-2021 - 0.17](https://github.com/airbytehq/airbyte/releases/tag/v0.17.0-alpha) - -* **Integration with Airflow** is here. Thanks to @Marcos Marx, you can now run Airbyte jobs from Airflow directly. A tutorial is on the way and should be coming this week! -* Add a prefix for tables, so that tables with the same name don't clobber each other in the destination - -## [03-01-2021 - 0.16](https://github.com/airbytehq/airbyte/milestone/22?closed=1) - -* We made some progress to address **nested tables in our normalization.** - - Previously, basic normalization would output nested tables as-is and append a number for duplicate tables. For example, Stripe’s nested address fields go from: - - ```text - Address - address_1 - ``` - - To - - ```text - Charges_source_owner_755_address - customers_shipping_c70_address - ``` - - After the change, the parent tables are combined with the name of the nested table to show where the nested table originated. **This is a breaking change for the consumers of nested tables. Consumers will need to update to point at the new tables.** - -## [02-19-2021 - 0.15](https://github.com/airbytehq/airbyte/milestone/22?closed=1) - -* We now handle nested tables with the normalization steps. Check out the video below to see how it works. - -{% embed url="https://youtu.be/I4fngMnkJzY" caption="" %} - -## [02-12-2021 - 0.14](https://github.com/airbytehq/airbyte/milestone/21?closed=1) - -* Front-end changes: - * Display Airbyte's version number - * Describe schemas using JsonSchema - * Better feedback on buttons - -## [Beta launch - 0.13](https://github.com/airbytehq/airbyte/milestone/15?closed=1) - Released 02/02/2021 - -* Add connector build status dashboard -* Support Schema Changes in Sources -* Support Import / Export of Airbyte Data in the Admin section of the UI -* Bug fixes: - * If Airbyte is closed during a sync the running job is not marked as failed - * Airbyte should fail when deployment version doesn't match data version - * Upgrade Airbyte Version without losing existing configuration / data - -## [0.12-alpha](https://github.com/airbytehq/airbyte/milestone/14?closed=1) - Released 01/20/2021 - -* Ability to skip onboarding -* Miscellaneous bug fixes: - * A long discovery request causes a timeout in the UI type/bug - * Out of Memory when replicating large table from MySQL - -## 0.11.2-alpha - Released 01/18/2021 - -* Increase timeout for long running catalog discovery operations from 3 minutes to 30 minutes to avoid prematurely failing long-running operations - -## 0.11.1-alpha - Released 01/17/2021 - -### Bugfixes - -* Writing boolean columns to Redshift destination now works correctly - -## [0.11.0-alpha](https://github.com/airbytehq/airbyte/milestone/12?closed=1) - Delivered 01/14/2021 - -### New features - -* Allow skipping the onboarding flow in the UI -* Add the ability to reset a connection's schema when the underlying data source schema changes - -### Bugfixes - -* Fix UI race condition which showed config for the wrong connector when rapidly choosing between different connector -* Fix a bug in MSSQL and Redshift source connectors where custom SQL types weren't being handled correctly. [Pull request](https://github.com/airbytehq/airbyte/pull/1576) -* Support incremental sync for Salesforce, Slack, and Braintree sources -* Gracefully handle invalid nuemric values \(e.g NaN or Infinity\) in MySQL, MSSQL, and Postgtres DB sources -* Fix flashing red sources/destinations fields after success submit -* Fix a bug which caused getting a connector's specification to hang indefinitely if the connector docker image failed to download - -### New connectors - -* Tempo -* Appstore - -## [0.10.0](https://github.com/airbytehq/airbyte/milestone/12?closed=1) - delivered on 01/04/2021 - -* You can now **deploy Airbyte on** [**Kuberbetes**](https://docs.airbyte.com/deploying-airbyte/on-kubernetes) _\*\*_\(alpha version\) -* **Support incremental sync** for Mixpanel and HubSpot sources -* **Fixes on connectors:** - * Fixed a bug in the GitHub connector where the connector didn’t verify the provided API token was granted the correct permissions - * Fixed a bug in the Google Sheets connector where rate limits were not always respected - * Alpha version of Facebook marketing API v9. This connector is a native Airbyte connector \(current is Singer based\). -* **New source:** Plaid \(contributed by [@tgiardina](https://github.com/tgiardina) - thanks Thomas!\) - -## [0.9.0](https://github.com/airbytehq/airbyte/milestone/11?closed=1) - delivered on 12/23/2020 - -* **New chat app from the web app** so you can directly chat with the team for any issues you run into -* **Debugging** has been made easier in the UI, with checks, discover logs, and sync download logs -* Support of **Kubernetes in local**. GKE will come at the next release. -* **New source:** Looker _\*\*_ - -## [0.8.0](https://github.com/airbytehq/airbyte/milestone/10?closed=1) - delivered on 12/17/2020 - -* **Incremental - Append"** - * We now allow sources to replicate only new or modified data. This enables to avoid re-fetching data that you have already replicated from a source. - * The delta from a sync will be _appended_ to the existing data in the data warehouse. - * Here are [all the details of this feature](../../understanding-airbyte/connections/incremental-append.md). - * It has been released for 15 connectors, including Postgres, MySQL, Intercom, Zendesk, Stripe, Twilio, Marketo, Shopify, GitHub, and all the destination connectors. We will expand it to all the connectors in the next couple of weeks. -* **Other features:** - * Improve interface for writing python sources \(should make writing new python sources easier and clearer\). - * Add support for running Standard Source Tests with files \(making them easy to run for any language a source is written in\) - * Add ability to reset data for a connection. -* **Bug fixes:** - * Update version of test containers we use to avoid pull issues while running tests. - * Fix issue where jobs were not sorted by created at in connection detail view. -* **New sources:** Intercom, Mixpanel, Jira Cloud, Zoom, Drift, Microsoft Teams - -## [0.7.0](https://github.com/airbytehq/airbyte/milestone/8?closed=1) - delivered on 12/07/2020 - -* **New destination:** our own **Redshift** warehouse connector. You can also use this connector for Panoply. -* **New sources**: 8 additional source connectors including Recurly, Twilio, Freshdesk. Greenhouse, Redshift \(source\), Braintree, Slack, Zendesk Support -* Bug fixes - -## [0.6.0](https://github.com/airbytehq/airbyte/milestone/6?closed=1) - delivered on 11/23/2020 - -* Support **multiple destinations** -* **New source:** Sendgrid -* Support **basic normalization** -* Bug fixes - -## [0.5.0](https://github.com/airbytehq/airbyte/milestone/5?closed=1) - delivered on 11/18/2020 - -* **New sources:** 10 additional source connectors, including Files \(CSV, HTML, JSON...\), Shopify, MSSQL, Mailchimp - -## [0.4.0](https://github.com/airbytehq/airbyte/milestone/4?closed=1) - delivered on 11/04/2020 - -Here is what we are working on right now: - -* **New destination**: our own **Snowflake** warehouse connector -* **New sources:** Facebook Ads, Google Ads. - -## [0.3.0](https://github.com/airbytehq/airbyte/milestone/3?closed=1) - delivered on 10/30/2020 - -* **New sources:** Salesforce, GitHub, Google Sheets, Google Analytics, HubSpot, Rest APIs, and MySQL -* Integration test suite for sources -* Improve build speed - -## [0.2.0](https://github.com/airbytehq/airbyte/milestone/2?closed=1) - delivered on 10/21/2020 - -* **a new Admin section** to enable users to add their own connectors, in addition to upgrading the ones they currently use -* improve the developer experience \(DX\) for **contributing new connectors** with additional documentation and a connector protocol -* our own **BigQuery** warehouse connector -* our own **Postgres** warehouse connector -* simplify the process of supporting new Singer taps, ideally make it a 1-day process - -## [0.1.0](https://github.com/airbytehq/airbyte/milestone/1?closed=1) - delivered on 09/23/2020 - -This is our very first release after 2 months of work. - -* **New sources:** Stripe, Postgres -* **New destinations:** BigQuery, Postgres -* **Only one destination**: we only support one destination in that 1st release, but you will soon be able to add as many as you need. -* **Logs & monitoring**: you can now see your detailed logs -* **Scheduler:** you now have 10 different frequency options for your recurring syncs -* **Deployment:** you can now deploy Airbyte via a simple Docker image, or directly on AWS and GCP -* **New website**: this is the day we launch our website - airbyte.io. Let us know what you think -* **New documentation:** this is the 1st day for our documentation too -* **New blog:** we published a few articles on our startup journey, but also about our vision to making data integrations a commodity. - -Stay tuned, we will have new sources and destinations very soon! Don't hesitate to subscribe to our [newsletter](https://airbyte.io/#subscribe-newsletter) to receive our product updates and community news. - diff --git a/docs/archive/examples/README.md b/docs/archive/examples/README.md deleted file mode 100644 index e62ee1c8eb21..000000000000 --- a/docs/archive/examples/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# Example Use Cases - diff --git a/docs/archive/examples/build-a-slack-activity-dashboard.md b/docs/archive/examples/build-a-slack-activity-dashboard.md deleted file mode 100644 index b63a2b65babb..000000000000 --- a/docs/archive/examples/build-a-slack-activity-dashboard.md +++ /dev/null @@ -1,424 +0,0 @@ ---- -description: Using Airbyte and Apache Superset ---- - -# Build a Slack Activity Dashboard - -![](../../.gitbook/assets/46.png) - -This article will show how to use [Airbyte](http://airbyte.com) - an open-source data integration platform - and [Apache Superset](https://superset.apache.org/) - an open-source data exploration platform - in order to build a Slack activity dashboard showing: - -* Total number of members of a Slack workspace -* The evolution of the number of Slack workspace members -* Evolution of weekly messages -* Evolution of messages per channel -* Members per time zone - -Before we get started, let’s take a high-level look at how we are going to achieve creating a Slack dashboard using Airbyte and Apache Superset. - -1. We will use the Airbyte’s Slack connector to get the data off a Slack workspace \(we will be using Airbyte’s own Slack workspace for this tutorial\). -2. We will save the data onto a PostgreSQL database. -3. Finally, using Apache Superset, we will implement the various metrics we care about. - -Got it? Now let’s get started. - -## 1. Replicating Data from Slack to Postgres with Airbyte - -### a. Deploying Airbyte - -There are several easy ways to deploy Airbyte, as listed [here](https://docs.airbyte.com/). For this tutorial, I will just use the [Docker Compose method](https://docs.airbyte.com/deploying-airbyte/local-deployment) from my workstation: - -```text -# In your workstation terminal -git clone https://github.com/airbytehq/airbyte.git -cd airbyte -docker-compose up -``` - -The above command will make the Airbyte app available on `localhost:8000`. Visit the URL on your favorite browser, and you should see Airbyte’s dashboard \(if this is your first time, you will be prompted to enter your email to get started\). - -If you haven’t set Docker up, follow the [instructions here](https://docs.docker.com/desktop/) to set it up on your machine. - -### b. Setting Up Airbyte’s Slack Source Connector - -Airbyte’s Slack connector will give us access to the data. So, we are going to kick things off by setting this connector to be our data source in Airbyte’s web app. I am assuming you already have Airbyte and Docker set up on your local machine. We will be using Docker to create our PostgreSQL database container later on. - -Now, let’s proceed. If you already went through the onboarding, click on the “new source” button at the top right of the Sources section. If you're going through the onboarding, then follow the instructions. - -You will be requested to enter a name for the source you are about to create. You can call it “slack-source”. Then, in the Source Type combo box, look for “Slack,” and then select it. Airbyte will then present the configuration fields needed for the Slack connector. So you should be seeing something like this on the Airbyte App: - -![](../../.gitbook/assets/1.png) - -The first thing you will notice is that this connector requires a Slack token. So, we have to obtain one. If you are not a workspace admin, you will need to ask for permission. - -Let’s walk through how we would get the Slack token we need. - -Assuming you are a workspace admin, open the Slack workspace and navigate to \[Workspace Name\] > Administration > Customize \[Workspace Name\]. In our case, it will be Airbyte > Administration > Customize Airbyte \(as shown below\): - -![](../../.gitbook/assets/2.png) - -In the new page that opens up in your browser, you will then need to navigate to **Configure apps**. - -![](../../.gitbook/assets/3.png) - -In the new window that opens up, click on **Build** in the top right corner. - -![](../../.gitbook/assets/4.png) - -Click on the **Create an App** button. - -![](../../.gitbook/assets/5.png) - -In the modal form that follows, give your app a name - you can name it `airbyte_superset`, then select your workspace from the Development Slack Workspace. - -![](../../.gitbook/assets/6.png) - -Next, click on the **Create App** button. You will then be presented with a screen where we are going to set permissions for our `airbyte_superset` app, by clicking on the **Permissions** button on this page. - -![](../../.gitbook/assets/7.png) - -In the next screen, navigate to the scope section. Then, click on the **Add an OAuth Scope** button. This will allow you to add permission scopes for your app. At a minimum, your app should have the following permission scopes: - -![](../../.gitbook/assets/8.png) - -Then, we are going to add our created app to the workspace by clicking the **Install to Workspace** button. - -![](../../.gitbook/assets/9.png) - -Slack will prompt you that your app is requesting permission to access your workspace of choice. Click Allow. - -![](../../.gitbook/assets/10.png) - -After the app has been successfully installed, you will be navigated to Slack’s dashboard, where you will see the Bot User OAuth Access Token. - -This is the token you will provide back on the Airbyte page, where we dropped off to obtain this token. So make sure to copy it and keep it in a safe place. - -Now that we are done with obtaining a Slack token, let’s go back to the Airbyte page we dropped off and add the token in there. - -We will also need to provide Airbyte with `start_date`. This is the date from which we want Airbyte to start replicating data from the Slack API, and we define that in the format: `YYYY-MM-DDT00:00:00Z`. - -We will specify ours as `2020-09-01T00:00:00Z`. We will also tell Airbyte to exclude archived channels and not include private channels, and also to join public channels, so the latter part of the form should look like this: - -![](../../.gitbook/assets/11.png) - -Finally, click on the **Set up source** button for Airbyte to set the Slack source up. - -If the source was set up correctly, you will be taken to the destination section of Airbyte’s dashboard, where you will tell Airbyte where to store the replicated data. - -### c. Setting Up Airbyte’s Postgres Destination Connector - -For our use case, we will be using PostgreSQL as the destination. - -Click the **add destination** button in the top right corner, then click on **add a new destination**. - -![](../../.gitbook/assets/12.png) - -In the next screen, Airbyte will validate the source, and then present you with a form to give your destination a name. We’ll call this destination slack-destination. Then, we will select the Postgres destination type. Your screen should look like this now: - -![](../../.gitbook/assets/13.png) - -Great! We have a form to enter Postgres connection credentials, but we haven’t set up a Postgres database. Let’s do that! - -Since we already have Docker installed, we can spin off a Postgres container with the following command in our terminal: - -```text -docker run --rm --name slack-db -e POSTGRES_PASSWORD=password -p 2000:5432 -d postgres -``` - -\(Note that the Docker compose file for Superset ships with a Postgres database, as you can see [here](https://github.com/apache/superset/blob/master/docker-compose.yml#L40)\). - -The above command will do the following: - -* create a Postgres container with the name slack-db, -* set the password to password, -* expose the container’s port 5432, as our machine’s port 2000. -* create a database and a user, both called postgres. - -With this, we can go back to the Airbyte screen and supply the information needed. Your form should look like this: - -![](../../.gitbook/assets/14.png) - -Then click on the **Set up destination** button. - -### d. Setting Up the Replication - -You should now see the following screen: - -![](../../.gitbook/assets/15.png) - -Airbyte will then fetch the schema for the data coming from the Slack API for your workspace. You should leave all boxes checked and then choose the sync frequency - this is the interval in which Airbyte will sync the data coming from your workspace. Let’s set the sync interval to every 24 hours. - -Then click on the **Set up connection** button. - -Airbyte will now take you to the destination dashboard, where you will see the destination you just set up. Click on it to see more details about this destination. - -![](../../.gitbook/assets/16.png) - -You will see Airbyte running the very first sync. Depending on the size of the data Airbyte is replicating, it might take a while before syncing is complete. - -![](../../.gitbook/assets/17.png) - -When it’s done, you will see the **Running status** change to **Succeeded**, and the size of the data Airbyte replicated as well as the number of records being stored on the Postgres database. - -![](../../.gitbook/assets/18.png) - -To test if the sync worked, run the following in your terminal: - -```text -docker exec slack-source psql -U postgres -c "SELECT * FROM public.users;" -``` - -This should output the rows in the users’ table. - -To get the count of the users’ table as well, you can also run: - -```text -docker exec slack-db psql -U postgres -c "SELECT count(*) FROM public.users;" -``` - -Now that we have the data from the Slack workspace in our Postgres destination, we will head on to creating the Slack dashboard with Apache Superset. - -## 2. Setting Up Apache Superset for the Dashboards - -### a. Installing Apache Superset - -Apache Superset, or simply Superset, is a modern data exploration and visualization platform. To get started using it, we will be cloning the Superset repo. Navigate to a destination in your terminal where you want to clone the Superset repo to and run: - -```text -git clone https://github.com/apache/superset.git -``` - -It’s recommended to check out the latest branch of Superset, so run: - -```text -cd superset -``` - -And then run: - -```text -git checkout latest -``` - -Superset needs you to install and build its frontend dependencies and assets. So, we will start by installing the frontend dependencies: - -```text -npm install -``` - -Note: The above command assumes you have both Node and NPM installed on your machine. - -Finally, for the frontend, we will build the assets by running: - -```text -npm run build -``` - -After that, go back up one directory into the Superset directory by running: - -```text -cd.. -``` - -Then run: - -```text -docker-compose up -``` - -This will download the Docker images Superset needs and build containers and start services Superset needs to run locally on your machine. - -Once that’s done, you should be able to access Superset on your browser by visiting [`http://localhost:8088`](http://localhost:8088), and you should be presented with the Superset login screen. - -Enter username: **admin** and Password: **admin** to be taken to your Superset dashboard. - -Great! You’ve got Superset set up. Now let’s tell Superset about our Postgres Database holding the Slack data from Airbyte. - -### b. Setting Up a Postgres Database in Superset - -To do this, on the top menu in your Superset dashboard, hover on the Data dropdown and click on **Databases**. - -![](../../.gitbook/assets/19.png) - -In the page that opens up, click on the **+ Database** button in the top right corner. - -![](../../.gitbook/assets/20.png) - -Then, you will be presented with a modal to add your Database Name and the connection URI. - -![](../../.gitbook/assets/21.png) - -Let’s call our Database `slack_db`, and then add the following URI as the connection URI: - -```text -postgresql://postgres:password@docker.for.mac.localhost:2000/postgres -``` - -If you are on a Windows Machine, yours will be: - -```text -postgresql://postgres:password@docker.for.win.localhost:2000/postgres -``` - -Note: We are using `docker.for.[mac|win].localhost` in order to access the localhost of your machine, because using just localhost will point to the Docker container network and not your machine’s network. - -Your Superset UI should look like this: - -![](../../.gitbook/assets/22.png) - -We will need to enable some settings on this connection. Click on the **SQL LAB SETTINGS** and check the following boxes: - -![](../../.gitbook/assets/23.png) - -Afterwards, click on the **ADD** button, and you will see your database on the data page of Superset. - -![](../../.gitbook/assets/24.png) - -### c. Importing our dataset - -Now that you’ve added the database, you will need to hover over the data menu again; now click on **Datasets**. - -![](../../.gitbook/assets/25.png) - -Then, you will be taken to the datasets page: - -![](../../.gitbook/assets/26.png) - -We want to only see the datasets that are in our `slack_db` database, so in the Database that is currently showing All, select `slack_db` and you will see that we don’t have any datasets at the moment. - -![](../../.gitbook/assets/27.png) - -![](../../.gitbook/assets/28.png) - -You can fix this by clicking on the **+ DATASET** button and adding the following datasets. - -Note: Make sure you select the public schema under the Schema dropdown. - -![](../../.gitbook/assets/29.png) - -Now that we have set up Superset and given it our Slack data, let’s proceed to creating the visualizations we need. - -Still remember them? Here they are again: - -* Total number of members of a Slack workspace -* The evolution of the number of Slack workspace members -* Evolution of weekly messages -* Evolution of weekly threads created -* Evolution of messages per channel -* Members per time zone - -## 3. Creating Our Dashboards with Superset - -### a. Total number of members of a Slack workspace - -To get this, we will first click on the users’ dataset of our `slack_db` on the Superset dashboard. - -![](../../.gitbook/assets/30.png) - -Next, change **untitled** at the top to **Number of Members**. - -![](../../.gitbook/assets/31.png) - -Now change the **Visualization Type** to **Big Number,** remove the **Time Range** filter, and add a Subheader named “Slack Members.” So your UI should look like this: - -![](../../.gitbook/assets/32.png) - -Then, click on the **RUN QUERY** button, and you should now see the total number of members. - -Pretty cool, right? Now let’s save this chart by clicking on the **SAVE** button. - -![](../../.gitbook/assets/33.png) - -Then, in the **ADD TO DASHBOARD** section, type in “Slack Dashboard”, click on the “Create Slack Dashboard” button, and then click the **Save** button. - -Great! We have successfully created our first Chart, and we also created the Dashboard. Subsequently, we will be following this flow to add the other charts to the created Slack Dashboard. - -### b. Casting the ts column - -Before we proceed with the rest of the charts for our dashboard, if you inspect the **ts** column on either the **messages** table or the **threads** table, you will see it’s of the type `VARCHAR`. We can’t really use this for our charts, so we have to cast both the **messages** and **threads**’ **ts** column as `TIMESTAMP`. Then, we can create our charts from the results of those queries. Let’s do this. - -First, navigate to the **Data** menu, and click on the **Datasets** link. In the list of datasets, click the **Edit** button for the **messages** table. - -![](../../.gitbook/assets/34.png) - -You’re now in the Edit Dataset view. Click the **Lock** button to enable editing of the dataset. Then, navigate to the **Columns** tab, expand the **ts** dropdown, and then tick the **Is Temporal** box. - -![](../../.gitbook/assets/35.png) - -Persist the changes by clicking the Save button. - -### c. The evolution of the number of Slack workspace members - -In the exploration page, let’s first get the chart showing the evolution of the number of Slack members. To do this, make your settings on this page match the screenshot below: - -![](../../.gitbook/assets/36.png) - -Save this chart onto the Slack Dashboard. - -### d. Evolution of weekly messages posted - -Now, we will look at the evolution of weekly messages posted. Let’s configure the chart settings on the same page as the previous one. - -![](../../.gitbook/assets/37.png) - -Remember, your visualization will differ based on the data you have. - -### e. Evolution of weekly threads created - -Now, we are finished with creating the message chart. Let's go over to the thread chart. You will recall that we will need to cast the **ts** column as stated earlier. So, do that and get to the exploration page, and make it match the screenshot below to achieve the required visualization: - -![](../../.gitbook/assets/38.png) - -### f. Evolution of messages per channel - -For this visualization, we will need a more complex SQL query. Here’s the query we used \(as you can see in the screenshot below\): - -```text -SELECT CAST(m.ts as TIMESTAMP), c.name, m.text -FROM public.messages m -INNER JOIN public.channels c -ON m.channel_id = c_id -``` - -![](../../.gitbook/assets/39.png) - -Next, click on **EXPLORE** to be taken to the exploration page; make it match the screenshot below: - -![](../../.gitbook/assets/40.png) - -Save this chart to the dashboard. - -### g. Members per time zone - -Finally, we will be visualizing members per time zone. To do this, instead of casting in the SQL lab as we’ve previously done, we will explore another method to achieve casting by using Superset’s Virtual calculated column feature. This feature allows us to write SQL queries that customize the appearance and behavior of a specific column. - -For our use case, we will need the updated column of the users table to be a `TIMESTAMP`, in order to perform the visualization we need for Members per time zone. Let’s start on clicking the edit icon on the users table in Superset. - -![](../../.gitbook/assets/41.png) - -You will be presented with a modal like so: - -![](../../.gitbook/assets/42.png) - -Click on the **CALCULATED COLUMNS** tab: - -![](../../.gitbook/assets/43.png) - -Then, click on the **+ ADD ITEM** button, and make your settings match the screenshot below. - -![](../../.gitbook/assets/44.png) - -Then, go to the **exploration** page and make it match the settings below: - -![](../../.gitbook/assets/45.png) - -Now save this last chart, and head over to your Slack Dashboard. It should look like this: - -![](../../.gitbook/assets/46.png) - -Of course, you can edit how the dashboard looks to fit what you want on it. - -## Conclusion - -In this article, we looked at using Airbyte’s Slack connector to get the data from a Slack workspace into a Postgres database, and then used Apache Superset to craft a dashboard of visualizations.If you have any questions about Airbyte, don’t hesitate to ask questions on our [Slack](https://slack.airbyte.io)! If you have questions about Superset, you can join the [Superset Community Slack](https://superset.apache.org/community/)! - diff --git a/docs/archive/examples/postgres-replication.md b/docs/archive/examples/postgres-replication.md deleted file mode 100644 index 160da6d20f7a..000000000000 --- a/docs/archive/examples/postgres-replication.md +++ /dev/null @@ -1,116 +0,0 @@ ---- -description: Start syncing data in minutes with Airbyte ---- - -# Postgres Replication - -Let's see how you can spin up a local instance of Airbyte and syncing data from one Postgres database to another. - -Here's a 6-minute video showing you how you can do it. - -{% embed url="https://www.youtube.com/watch?v=Rcpt5SVsMpk" caption="" %} - -First of all, make sure you have Docker and Docker Compose installed. If this isn't the case, follow the [guide](../../deploying-airbyte/local-deployment.md) for the recommended approach to install Docker. - -Once Docker is installed successfully, run the following commands: - -```text -git clone https://github.com/airbytehq/airbyte.git -cd airbyte -docker-compose up -``` - -Once you see an Airbyte banner, the UI is ready to go at [http://localhost:8000/](http://localhost:8000/). - -## 1. Set up your preferences - -You should see an onboarding page. Enter your email and continue. - -![](../../.gitbook/assets/airbyte_get-started.png) - -## 2. Set up your first connection - -We support a growing [list of source connectors](https://docs.airbyte.com/category/sources). For now, we will start out with a Postgres source and destination. - -**If you don't have a readily available Postgres database to sync, here are some quick instructions:** -Run the following commands in a new terminal window to start backgrounded source and destination databases: - -```text -docker run --rm --name airbyte-source -e POSTGRES_PASSWORD=password -p 2000:5432 -d postgres -docker run --rm --name airbyte-destination -e POSTGRES_PASSWORD=password -p 3000:5432 -d postgres -``` - -Add a table with a few rows to the source database: - -```text -docker exec -it airbyte-source psql -U postgres -c "CREATE TABLE users(id SERIAL PRIMARY KEY, col1 VARCHAR(200));" -docker exec -it airbyte-source psql -U postgres -c "INSERT INTO public.users(col1) VALUES('record1');" -docker exec -it airbyte-source psql -U postgres -c "INSERT INTO public.users(col1) VALUES('record2');" -docker exec -it airbyte-source psql -U postgres -c "INSERT INTO public.users(col1) VALUES('record3');" -``` - -You now have a Postgres database ready to be replicated! - -### **Connect the Postgres database** - -In the UI, you will see a wizard that allows you choose the data you want to send through Airbyte. - -![](../../.gitbook/assets/02_set-up-sources.png) - -Use the name `airbyte-source` for the name and `Postgres`as the type. If you used our instructions to create a Postgres database, fill in the configuration fields as follows: - -```text -Host: localhost -Port: 2000 -User: postgres -Password: password -DB Name: postgres -``` - -Click on `Set Up Source` and the wizard should move on to allow you to configure a destination. - -We support a growing list of data warehouses, lakes and databases. For now, use the name `airbyte-destination`, and configure the destination Postgres database: - -```text -Host: localhost -Port: 3000 -User: postgres -Password: password -DB Name: postgres -``` - -After adding the destination, you can choose what tables and columns you want to sync. - -![](../../.gitbook/assets/03_set-up-connection.png) - -For this demo, we recommend leaving the defaults and selecting "Every 5 Minutes" as the frequency. Click `Set Up Connection` to finish setting up the sync. - -## 3. Check the logs of your first sync - -You should now see a list of sources with the source you just added. Click on it to find more information about your connection. This is the page where you can update any settings about this source and how it syncs. There should be a `Completed` job under the history section. If you click on that run, it will show logs from that run. - -![](../../.gitbook/assets/04_source-details.png) - -One of biggest problems we've seen in tools like Fivetran is the lack of visibility when debugging. In Airbyte, allowing full log access and the ability to debug and fix connector problems is one of our highest priorities. We'll be working hard to make these logs accessible and understandable. - -## 4. Check if the syncing actually worked - -Now let's verify that this worked. Let's output the contents of the destination db: - -```text -docker exec airbyte-destination psql -U postgres -c "SELECT * FROM public.users;" -``` - -:::info - -Don't worry about the awkward `public_users` name for now; we are currently working on an update to allow users to configure their destination table names! - -::: - -You should see the rows from the source database inside the destination database! - -And there you have it. You've taken data from one database and replicated it to another. All of the actual configuration for this replication only took place in the UI. - -That's it! This is just the beginning of Airbyte. If you have any questions at all, please reach out to us on [Slack](https://slack.airbyte.io/). We’re still in alpha, so if you see any rough edges or want to request a connector you need, please create an issue on our [Github](https://github.com/airbytehq/airbyte) or leave a thumbs up on an existing issue. - -Thank you and we hope you enjoy using Airbyte. diff --git a/docs/archive/examples/slack-history.md b/docs/archive/examples/slack-history.md deleted file mode 100644 index 6305798bffee..000000000000 --- a/docs/archive/examples/slack-history.md +++ /dev/null @@ -1,109 +0,0 @@ ---- -description: Using Airbyte and MeiliSearch ---- - -# Save and Search Through Your Slack History on a Free Slack Plan - -![](../../.gitbook/assets/slack-history-ui-title.png) - -The [Slack free tier](https://slack.com/pricing/paid-vs-free) saves only the last 10K messages. For social Slack instances, it may be impractical to upgrade to a paid plan to retain these messages. Similarly, for an open-source project like [Airbyte](../../understanding-airbyte/airbyte-protocol.md#catalog) where we interact with our community through a public Slack instance, the cost of paying for a seat for every Slack member is prohibitive. - -However, searching through old messages can be really helpful. Losing that history feels like some advanced form of memory loss. What was that joke about Java 8 Streams? This contributor question sounds familiar—haven't we seen it before? But you just can't remember! - -This tutorial will show you how you can, for free, use Airbyte to save these messages \(even after Slack removes access to them\). It will also provide you a convenient way to search through them. - -Specifically, we will export messages from your Slack instance into an open-source search engine called [MeiliSearch](https://github.com/meilisearch/meilisearch). We will be focusing on getting this setup running from your local workstation. We will mention at the end how you can set up a more productionized version of this pipeline. - -We want to make this process easy, so while we will link to some external documentation for further exploration, we will provide all the instructions you need here to get this up and running. - -## 1. Set Up MeiliSearch - -First, let's get MeiliSearch running on our workstation. MeiliSearch has extensive docs for [getting started](https://docs.meilisearch.com/reference/features/installation.html#download-and-launch). For this tutorial, however, we will give you all the instructions you need to set up MeiliSearch using Docker. - -```text -docker run -it --rm \ - -p 7700:7700 \ - -v $(pwd)/data.ms:/data.ms \ - getmeili/meilisearch -``` - -That's it! - -:::info - -MeiliSearch stores data in $\(pwd\)/data.ms, so if you prefer to store it somewhere else, just adjust this path. - -::: - -## 2. Replicate Your Slack Messages to MeiliSearch - -### a. Set Up Airbyte - -Make sure you have Docker and Docker Compose installed. If you haven’t set Docker up, follow the [instructions here](https://docs.docker.com/desktop/) to set it up on your machine. Then, run the following commands: - -```bash -git clone https://github.com/airbytehq/airbyte.git -cd airbyte -docker-compose up -``` - -If you run into any problems, feel free to check out our more extensive [Getting Started FAQ](https://discuss.airbyte.io/c/faq/15) for help. - -Once you see an Airbyte banner, the UI is ready to go at [http://localhost:8000/](http://localhost:8000/). Once you have set your user preferences, you will be brought to a page that asks you to set up a source. In the next step, we'll go over how to do that. - -### b. Set Up Airbyte’s Slack Source Connector - -In the Airbyte UI, select Slack from the dropdown. We provide step-by-step instructions for setting up the Slack source in Airbyte [here](https://docs.airbyte.com/integrations/sources/slack#setup-guide). These will walk you through how to complete the form on this page. - -![](../../.gitbook/assets/slack-history-setup-wizard.png) - -By the end of these instructions, you should have created a Slack source in the Airbyte UI. For now, just add your Slack app to a single public channel \(you can add it to more channels later\). Only messages from that channel will be replicated. - -The Airbyte app will now prompt you to set up a destination. Next, we will walk through how to set up MeiliSearch. - -### c. Set Up Airbyte’s MeiliSearch Destination Connector - -Head back to the Airbyte UI. It should still be prompting you to set up a destination. Select "MeiliSearch" from the dropdown. For the `host` field, set: `http://localhost:7700`. The `api_key` can be left blank. - -### d. Set Up the Replication - -On the next page, you will be asked to select which streams of data you'd like to replicate. We recommend unchecking "files" and "remote files" since you won't really be able to search them easily in this search engine. - -![](../../.gitbook/assets/airbyte_connection-settings.png) - -For frequency, we recommend every 24 hours. - -## 3. Search MeiliSearch - -After the connection has been saved, Airbyte should start replicating the data immediately. When it completes you should see the following: - -![](../../.gitbook/assets/slack-history-sync.png) - -When the sync is done, you can sanity check that this is all working by making a search request to MeiliSearch. Replication can take several minutes depending on the size of your Slack instance. - -```bash -curl 'http://localhost:7700/indexes/messages/search' --data '{ "q": "" }' -``` - -For example, I have the following message in one of the messages that I replicated: "welcome to airbyte". - -```bash -curl 'http://localhost:7700/indexes/messages/search' --data '{ "q": "welcome to" }' -# => {"hits":[{"_ab_pk":"7ff9a858_6959_45e7_ad6b_16f9e0e91098","channel_id":"C01M2UUP87P","client_msg_id":"77022f01-3846-4b9d-a6d3-120a26b2c2ac","type":"message","text":"welcome to airbyte.","user":"U01AS8LGX41","ts":"2021-02-05T17:26:01.000000Z","team":"T01AB4DDR2N","blocks":[{"type":"rich_text"}],"file_ids":[],"thread_ts":"1612545961.000800"}],"offset":0,"limit":20,"nbHits":2,"exhaustiveNbHits":false,"processingTimeMs":21,"query":"test-72"} -``` - -## 4. Search via a UI - -Making curl requests to search your Slack History is a little clunky, so we have modified the example UI that MeiliSearch provides in [their docs](https://docs.meilisearch.com/learn/tutorials/getting_started.html#integrate-with-your-project) to search through the Slack results. - -Download \(or copy and paste\) this [html file](https://github.com/airbytehq/airbyte/blob/master/docs/examples/slack-history/index.html) to your workstation. Then, open it using a browser. You should now be able to write search terms in the search bar and get results instantly! - -![](../../.gitbook/assets/slack-history-ui.png) - -## 5. "Productionizing" Saving Slack History - -You can find instructions for how to host Airbyte on various cloud platforms [here](../../deploying-airbyte/README.md). - -Documentation on how to host MeiliSearch on cloud platforms can be found [here](https://docs.meilisearch.com/running-production/#a-quick-introduction). - -If you want to use the UI mentioned in the section above, we recommend statically hosting it on S3, GCS, or equivalent. diff --git a/docs/archive/examples/slack-history/index.html b/docs/archive/examples/slack-history/index.html deleted file mode 100644 index 0812368137cd..000000000000 --- a/docs/archive/examples/slack-history/index.html +++ /dev/null @@ -1,77 +0,0 @@ - - - - - - - - -
      - -
      -
      - -
      - - - - - - - diff --git a/docs/archive/examples/zoom-activity-dashboard.md b/docs/archive/examples/zoom-activity-dashboard.md deleted file mode 100644 index a141f2da418a..000000000000 --- a/docs/archive/examples/zoom-activity-dashboard.md +++ /dev/null @@ -1,272 +0,0 @@ ---- -description: Using Airbyte and Tableau ---- - -# Visualizing the Time Spent by Your Team in Zoom Calls - -In this article, we will show you how you can understand how much your team leverages Zoom, or spends time in meetings, in a couple of minutes. We will be using [Airbyte](https://airbyte.com) \(an open-source data integration platform\) and [Tableau](https://www.tableau.com) \(a business intelligence and analytics software\) for this tutorial. - -Here is what we will cover: - -1. Replicating data from Zoom to a PostgreSQL database, using Airbyte -2. Connecting the PostgreSQL database to Tableau -3. Creating charts in Tableau with Zoom data - -We will produce the following charts in Tableau: - -* Meetings per week in a team -* Hours a team spends in meetings per week -* Listing of team members with the number of meetings per week and number of hours spent in meetings, ranked -* Webinars per week in a team -* Hours a team spends in webinars per week -* Participants for all webinars in a team per week -* Listing of team members with the number of webinars per week and number of hours spent in meetings, ranked - -Let’s get started by replicating Zoom data using Airbyte. - -## Step 1: Replicating Zoom data to PostgreSQL - -### Launching Airbyte - -In order to replicate Zoom data, we will need to use [Airbyte’s Zoom connector](https://docs.airbyte.com/integrations/sources/zoom). To do this, you need to start off Airbyte’s web app by opening up your terminal and navigating to Airbyte and running: - -`docker-compose up` - -You can find more details about this in the [Getting Started FAQ](https://discuss.airbyte.io/c/faq/15) on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). - -This will start up Airbyte on `localhost:8000`; open that address in your browser to access the Airbyte dashboard. - -![](../../.gitbook/assets/01_airbyte-dashboard.png) - -If you haven't gone through the onboarding yet, you will be prompted to connect a source and a destination. Then just follow the instructions. If you've gone through it, then you will see the screenshot above. In the top right corner of the Airbyte dashboard, click on the **+ new source** button to add a new Airbyte source. In the screen to set up the new source, enter the source name \(we will use airbyte-zoom\) and select **Zoom** as source type. - -Choosing Zoom as **source type** will cause Airbyte to display the configuration parameters needed to set up the Zoom source. - -![](../../.gitbook/assets/02_setting-zoom-connector-name.png) - -The Zoom connector for Airbyte requires you to provide it with a Zoom JWT token. Let’s take a detour and look at how to obtain one from Zoom. - -### Obtaining a Zoom JWT Token - -To obtain a Zoom JWT Token, login to your Zoom account and go to the [Zoom Marketplace](https://marketplace.zoom.us/). If this is your first time in the marketplace, you will need to agree to the Zoom’s marketplace terms of use. - -Once you are in, you need to click on the **Develop** dropdown and then click on **Build App.** - -![](../../.gitbook/assets/03_click.png) - -Clicking on **Build App** for the first time will display a modal for you to accept the Zoom’s API license and terms of use. Do accept if you agree and you will be presented with the below screen. - -![](../../.gitbook/assets/zoom-marketplace-build-screen%20(3)%20(3).png) - -Select **JWT** as the app you want to build and click on the **Create** button on the card. You will be presented with a modal to enter the app name; type in `airbyte-zoom`. - -![](../../.gitbook/assets/05_app-name-modal.png) - -Next, click on the **Create** button on the modal. - -You will then be taken to the **App Information** page of the app you just created. Fill in the required information. - -![](../../.gitbook/assets/06_app-information.png) - -After filling in the needed information, click on the **Continue** button. You will be taken to the **App Credentials** page. Here, click on the **View JWT Token** dropdown. - -![](../../.gitbook/assets/07_view-jwt-token.png) - -There you can set the expiration time of the token \(we will leave the default 90 minutes\), and then you click on the **Copy** button of the **JWT Token**. - -After copying it, click on the **Continue** button. - -![](../../.gitbook/assets/08_activate-webhook.png) - -You will be taken to a screen to activate **Event Subscriptions**. Just leave it as is, as we won’t be needing Webhooks. Click on **Continue**, and your app should be marked as activated. - -### Connecting Zoom on Airbyte - -So let’s go back to the Airbyte web UI and provide it with the JWT token we copied from our Zoom app. - -Now click on the **Set up source** button. You will see the below success message when the connection is made successfully. - -![](../../.gitbook/assets/setup-successful%20(3)%20(2).png) - -And you will be taken to the page to add your destination. - -### Connecting PostgreSQL on Airbyte - -![](../../.gitbook/assets/10_destination.png) - -For our destination, we will be using a PostgreSQL database, since Tableau supports PostgreSQL as a data source. Click on the **add destination** button, and then in the drop down click on **+ add a new destination**. In the page that presents itself, add the destination name and choose the Postgres destination. - -![](../../.gitbook/assets/11_choose-postgres-destination.png) - -To supply Airbyte with the PostgreSQL configuration parameters needed to make a PostgreSQL destination, we will spin off a PostgreSQL container with Docker using the following command in our terminal. - -`docker run --rm --name airbyte-zoom-db -e POSTGRES_PASSWORD=password -v airbyte_zoom_data:/var/lib/postgresql/data -p 2000:5432 -d postgres` - -This will spin a docker container and persist the data we will be replicating in the PostgreSQL database in a Docker volume `airbyte_zoom_data`. - -Now, let’s supply the above credentials to the Airbyte UI requiring those credentials. - -![](../../.gitbook/assets/postgres_credentials%20(3)%20(3).png) - -Then click on the **Set up destination** button. - -After the connection has been made to your PostgreSQL database successfully, Airbyte will generate the schema of the data to be replicated in your database from the Zoom source. - -Leave all the fields checked. - -![](../../.gitbook/assets/schema%20(3)%20(3).png) - -Select a **Sync frequency** of **manual** and then click on **Set up connection**. - -After successfully making the connection, you will see your PostgreSQL destination. Click on the Launch button to start the data replication. - -![](../../.gitbook/assets/launch%20(3)%20(3).png) - -Then click on the **airbyte-zoom-destination** to see the Sync page. - -![](../../.gitbook/assets/sync-screen%20(3)%20(3).png) - -Syncing should take a few minutes or longer depending on the size of the data being replicated. Once Airbyte is done replicating the data, you will get a **succeeded** status. - -Then, you can run the following SQL command on the PostgreSQL container to confirm that the sync was done successfully. - -`docker exec airbyte-zoom-db psql -U postgres -c "SELECT * FROM public.users;"` - -Now that we have our Zoom data replicated successfully via Airbyte, let’s move on and set up Tableau to make the various visualizations and analytics we want. - -## Step 2: Connect the PostgreSQL database to Tableau - -Tableau helps people and organizations to get answers from their data. It’s a visual analytic platform that makes it easy to explore and manage data. - -To get started with Tableau, you can opt in for a [free trial period](https://www.tableau.com/products/trial) by providing your email and clicking the **DOWNLOAD FREE TRIAL** button to download the Tableau desktop app. The download should automatically detect your machine type \(Windows/Mac\). - -Go ahead and install Tableau on your machine. After the installation is complete, you will need to fill in some more details to activate your free trial. - -Once your activation is successful, you will see your Tableau dashboard. - -![](../../.gitbook/assets/tableau-dashboard%20(3)%20(3).png) - -On the sidebar menu under the **To a Server** section, click on the **More…** menu. You will see a list of datasource connectors you can connect Tableau with. - -![](../../.gitbook/assets/datasources%20(4)%20(4).png) - -Select **PostgreSQL** and you will be presented with a connection credentials modal. - -Fill in the same details of the PostgreSQL database we used as the destination in Airbyte. - -![](../../.gitbook/assets/18_fill-in-connection-details.png) - -Next, click on the **Sign In** button. If the connection was made successfully, you will see the Tableau dashboard for the database you just connected. - -_Note: If you are having trouble connecting PostgreSQL with Tableau, it might be because the driver Tableau comes with for PostgreSQL might not work for newer versions of PostgreSQL. You can download the JDBC driver for PostgreSQL_ [_here_](https://www.tableau.com/support/drivers?_ga=2.62351404.1800241672.1616922684-1838321730.1615100968) _and follow the setup instructions._ - -Now that we have replicated our Zoom data into a PostgreSQL database using Airbyte’s Zoom connector, and connected Tableau with our PostgreSQL database containing our Zoom data, let’s proceed to creating the charts we need to visualize the time spent by a team in Zoom calls. - -## Step 3: Create the charts on Tableau with the Zoom data - -### Meetings per week in a team - -To create this chart, we will need to use the count of the meetings and the **createdAt** field of the **meetings** table. Currently, we haven’t selected a table to work on in Tableau. So you will see a prompt to **Drag tables here**. - -![](../../.gitbook/assets/19_tableau-view-with-all-tables.png) - -Drag the **meetings** table from the sidebar onto the space with the prompt. - -Now that we have the meetings table, we can start building out the chart by clicking on **Sheet 1** at the bottom left of Tableau. - -![](../../.gitbook/assets/20_empty-meeting-sheet.png) - -As stated earlier, we need **Created At**, but currently it’s a String data type. Let’s change that by converting it to a data time. So right click on **Created At**, then select `ChangeDataType` and choose Date & Time. And that’s it! That field is now of type **Date** & **Time**. - -![](../../.gitbook/assets/21_change-to-date-time.png) - -Next, drag **Created At** to **Columns**. - -![](../../.gitbook/assets/22_drag-created-at.png) - -Currently, we get the Created At in **YEAR**, but per our requirement we want them in Weeks, so right click on the **YEAR\(Created At\)** and choose **Week Number**. - -![](../../.gitbook/assets/change-to-per-week%20(3)%20(3).png) - -Tableau should now look like this: - -![](../../.gitbook/assets/24_meetings-per-week.png) - -Now, to finish up, we need to add the **meetings\(Count\) measure** Tableau already calculated for us in the **Rows** section. So drag **meetings\(Count\)** onto the Columns section to complete the chart. - -![](../../.gitbook/assets/evolution-of-meetings-per-week%20(3)%20(3).png) - -And now we are done with the very first chart. Let's save the sheet and create a new Dashboard that we will add this sheet to as well as the others we will be creating. - -Currently the sheet shows **Sheet 1**; right click on **Sheet 1** at the bottom left and rename it to **Weekly Meetings**. - -To create our Dashboard, we can right click on the sheet we just renamed and choose **new Dashboard**. Rename the Dashboard to Zoom Dashboard and drag the sheet into it to have something like this: - -![](../../.gitbook/assets/26_zoom-dashboard.png) - -Now that we have this first chart out of the way, we just need to replicate most of the process we used for this one to create the other charts. Because the steps are so similar, we will mostly be showing the finished screenshots of the charts except when we need to conform to the chart requirements. - -### Hours a team spends in meetings per week - -For this chart, we need the sum of the duration spent in weekly meetings. We already have a Duration field, which is currently displaying durations in minutes. We can derive a calculated field off this field since we want the duration in hours \(we just need to divide the duration field by 60\). - -To do this, right click on the Duration field and select **create**, then click on **calculatedField**. Change the name to **Duration in Hours**, and then the calculation should be **\[Duration\]/60**. Click ok to create the field. - -So now we can drag the Duration in Hours and Created At fields onto your sheet like so: - -![](../../.gitbook/assets/27_hours-spent-in-weekly-meetings.png) - -Note: We are adding a filter on the Duration to filter out null values. You can do this by right clicking on the **SUM\(Duration\)** pill and clicking filter, then make sure the **include null values** checkbox is unchecked. - -### Participants for all meetings per week - -For this chart, we will need to have a calculated field called **\# of meetings attended**, which will be an aggregate of the counts of rows matching a particular user's email in the `report_meeting_participants` table plotted against the **Created At** field of the **meetings** table. To get this done, right click on the **User Email** field. Select **create** and click on **calculatedField**, then enter the title of the field as **\# of meetings attended**. Next, enter the below formula: - -`COUNT(IF [User Email] == [User Email] THEN [Id (Report Meeting Participants)] END)` - -Then click on apply. Finally, drag the **Created At** fields \(make sure it’s on the **Weekly** number\) and the calculated field you just created to match the below screenshot: - -![](../../.gitbook/assets/number_of_participants_per_weekly_meetings.png) - -### Listing of team members with the number of meetings per week and number of hours spent in meetings, ranked. - -To get this chart, we need to create a relationship between the **meetings table** and the `report_meeting_participants` table. You can do this by dragging the `report_meeting_participants` table in as a source alongside the **meetings** table and relate both via the **meeting id**. Then you will be able to create a new worksheet that looks like this: - -![](../../.gitbook/assets/meetings-participant-ranked%20(3)%20(3).png) - -Note: To achieve the ranking, we simply use the sort menu icon on the top menu bar. - -### Webinars per week in a team - -The rest of the charts will be needing the **webinars** and `report_webinar_participants` tables. Similar to the number of meetings per week in a team, we will be plotting the Count of webinars against the **Created At** property. - -![](../../.gitbook/assets/30_weekly-webinars.png) - -### Hours a week spends in webinars per week - -For this chart, as for the meeting’s counterpart, we will get a calculated field off the Duration field to get the **Webinar Duration in Hours**, and then plot **Created At** against the **Sum of Webinar Duration in Hours**, as shown in the screenshot below. Note: Make sure you create a new sheet for each of these graphs. - -### Participants for all webinars per week - -This calculation is the same as the number of participants for all meetings per week, but instead of using the **meetings** and `report_meeting_participants` tables, we will use the webinars and `report_webinar_participants` tables. - -Also, the formula will now be: - -`COUNT(IF [User Email] == [User Email] THEN [Id (Report Webinar Participants)] END)` - -Below is the chart: - -![](../../.gitbook/assets/32_number_of_webinar_attended_per_week.png) - -#### Listing of team members with the number of webinars per week and number of hours spent in meetings, ranked - -Below is the chart with these specs - -![](../../.gitbook/assets/33_number-of-webinars-participants.png) - -## Conclusion - -In this article, we see how we can use Airbyte to get data off the Zoom API onto a PostgreSQL database, and then use that data to create some chart visualizations in Tableau. - -You can leverage Airbyte and Tableau to produce graphs on any collaboration tool. We just used Zoom to illustrate how it can be done. Hope this is helpful! - diff --git a/docs/archive/faq/README.md b/docs/archive/faq/README.md deleted file mode 100644 index 1f6a217b74c7..000000000000 --- a/docs/archive/faq/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# FAQ - -Our FAQ is now a section on our Airbyte Forum. Check it out [here](https://github.com/airbytehq/airbyte/discussions)! - -If you don't see your question answered, feel free to open up a new topic for it. \ No newline at end of file diff --git a/docs/archive/faq/data-loading.md b/docs/archive/faq/data-loading.md deleted file mode 100644 index 4ae20d834edc..000000000000 --- a/docs/archive/faq/data-loading.md +++ /dev/null @@ -1,124 +0,0 @@ -# Data Loading - -## **Why don’t I see any data in my destination yet?** - -It can take a while for Airbyte to load data into your destination. Some sources have restrictive API limits which constrain how much -data we can sync in a given time. Large amounts of data in your source can also make the initial sync take longer. You can check your -sync status in your connection detail page that you can access through the destination detail page or the source one. - -## **Why my final tables are being recreated everytime?** - -Airbyte ingests data into raw tables and applies the process of normalization if you selected it in the connection page. -The normalization runs a full refresh each sync and for some destinations like Snowflake, Redshift, Bigquery this may incur more -resource consumption and more costs. You need to pay attention to the frequency that you're retrieving your data to avoid issues. -For example, if you create a connection to sync every 5 minutes with incremental sync on, it will only retrieve new records into the raw tables but will apply normalization -to *all* the data in every sync! If you have tons of data, this may not be the right sync frequency for you. - -There is a [Github issue](https://github.com/airbytehq/airbyte/issues/4286) to implement normalization using incremental, which will reduce -costs and resources in your destination. - -## **What happens if a sync fails?** - -You won't lose data when a sync fails, however, no data will be added or updated in your destination. - -Airbyte will automatically attempt to replicate data 3 times. You can see and export the logs for those attempts in the connection -detail page. You can access this page through the Source or Destination detail page. - -You can configure a Slack webhook to warn you when a sync fails. - -In the future you will be able to configure other notification method (email, Sentry) and an option to create a -GitHub issue with the logs. We’re still working on it, and the purpose would be to help the community and the Airbyte team to fix the -issue as soon as possible, especially if it is a connector issue. - -Until Airbyte has this system in place, here is what you can do: - -* File a GitHub issue: go [here](https://github.com/airbytehq/airbyte/issues/new?assignees=&labels=type%2Fbug&template=bug-report.md&title=) - and file an issue with the detailed logs copied in the issue’s description. The team will be notified about your issue and will update - it for any progress or comment on it. -* Fix the issue yourself: Airbyte is open source so you don’t need to wait for anybody to fix your issue if it is important to you. - To do so, just fork the [GitHub project](https://github.com/airbytehq/airbyte) and fix the piece of code that need fixing. If you’re okay - with contributing your fix to the community, you can submit a pull request. We will review it ASAP. -* Ask on Slack: don’t hesitate to ping the team on [Slack](https://slack.airbyte.io). - -Once all this is done, Airbyte resumes your sync from where it left off. - -We truly appreciate any contribution you make to help the community. Airbyte will become the open-source standard only if everybody participates. - -## **Can Airbyte support 2-way sync i.e. changes from A go to B and changes from B go to A?** - -Airbyte actually does not support this right now. There are some details around how we handle schema and tables names that isn't going to -work for you in the current iteration. -If you attempt to do a circular dependency between source and destination, you'll end up with the following -A.public.table_foo writes to B.public.public_table_foo to A.public.public_public_table_foo. You won't be writing into your original table, -which I think is your intention. - - -## **What happens to data in the pipeline if the destination gets disconnected? Could I lose data, or wind up with duplicate data when the pipeline is reconnected?** - -Airbyte is architected to prevent data loss or duplication. Airbyte will display a failure for the sync, and re-attempt it at the next syncing, -according to the frequency you set. - -## **How frequently can Airbyte sync data?** - -You can adjust the load time to run as frequent as every hour or as infrequent as once a year using [Cron expressions](https://docs.airbyte.com/cloud/managing-airbyte-cloud/edit-stream-configuration). - -## **Why wouldn’t I choose to load all of my data more frequently?** - -While frequent data loads will give you more up-to-date data, there are a few reasons you wouldn’t want to load your too frequently, including: - -* Higher API usage may cause you to hit a limit that could impact other systems that rely on that API. -* Higher cost of loading data into your warehouse. -* More frequent delays, resulting in increased delay notification emails. For instance, if the data source generally takes several hours to - update but you wanted five-minute increments, you may receive a delay notification every sync. - -Generally is recommended setting the incremental loads to every hour to help limit API calls. - -## **Is there a way to know the estimated time to completion for the first historic sync?** - -Unfortunately not yet. - -## **Do you support change data capture \(CDC\) or logical replication for databases?** - -Airbyte currently supports [CDC for Postgres and Mysql](../../understanding-airbyte/cdc.md). Airbyte is adding support for a few other -databases you can check in the roadmap. - -## Using incremental sync, is it possible to add more fields when some new columns are added to a source table, or when a new table is added? - -For the moment, incremental sync doesn't support schema changes, so you would need to perform a full refresh whenever that happens. -Here’s a related [Github issue](https://github.com/airbytehq/airbyte/issues/1601). - -## There is a limit of how many tables one connection can handle? - -Yes, for more than 6000 thousand tables could be a problem to load the information on UI. - -There are two Github issues about this limitation: [Issue #3942](https://github.com/airbytehq/airbyte/issues/3942) -and [Issue #3943](https://github.com/airbytehq/airbyte/issues/3943). - -## Help, Airbyte is hanging/taking a long time to discover my source's schema! - -This usually happens for database sources that contain a lot of tables. This should resolve itself in half an hour or so. - -If the source contains more than 6k tables, see the [above question](#there-is-a-limit-of-how-many-tables-one-connection-can-handle). - -There is a known issue with [Oracle databases](https://github.com/airbytehq/airbyte/issues/4944). - -## **I see you support a lot of connectors – what about connectors Airbyte doesn’t support yet?** - -You can either: - -* Submit a [connector request](https://github.com/airbytehq/airbyte/issues/new?assignees=&labels=area%2Fintegration%2C+new-integration&template=new-integration-request.md&title=) on our Github project, and be notified once we or the community build a connector for it. -* Build a connector yourself by forking our [GitHub project](https://github.com/airbytehq/airbyte) and submitting a pull request. Here - are the [instructions how to build a connector](../../contributing-to-airbyte/README.md). -* Ask on Slack: don’t hesitate to ping the team on [Slack](https://slack.airbyte.io). - -## **What kind of notifications do I get?** - -For the moment, the UI will only display one kind of notification: when a sync fails, Airbyte will display the failure at the source/destination -level in the list of sources/destinations, and in the connection detail page along with the logs. - -However, there are other types of notifications: - -* When a connector that you use is no longer up to date -* When your connections fails -* When core isn't up to date - diff --git a/docs/archive/faq/deploying-on-other-os.md b/docs/archive/faq/deploying-on-other-os.md deleted file mode 100644 index 0b493c3db200..000000000000 --- a/docs/archive/faq/deploying-on-other-os.md +++ /dev/null @@ -1,40 +0,0 @@ -# Deploying Airbyte on a Non-Standard Operating System - -## CentOS 8 - -From clean install: - -``` -firewall-cmd --zone=public --add-port=8000/tcp --permanent -firewall-cmd --zone=public --add-port=8001/tcp --permanent -firewall-cmd --zone=public --add-port=7233/tcp --permanent -systemctl restart firewalld -``` -OR... if you prefer iptables: -``` -iptables -A INPUT -p tcp -m tcp --dport 8000 -j ACCEPT -iptables -A INPUT -p tcp -m tcp --dport 8001 -j ACCEPT -iptables -A INPUT -p tcp -m tcp --dport 7233 -j ACCEPT -systemctl restart iptables -``` -Setup the docker repo: -``` -dnf config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo` -dnf install docker-ce --nobest -systemctl enable --now docker -usermod -aG docker $USER -``` -You'll need to get docker-compose separately. -``` -dnf install wget git curl -curl -L https://github.com/docker/compose/releases/download/1.25.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose -chmod +x /usr/local/bin/docker-compose -``` -Now we can install Airbyte. In this example, we will install it under `/opt/` -``` -cd /opt -git clone https://github.com/airbytehq/airbyte.git -cd airbyte -docker-compose up -docker-compose ps -``` \ No newline at end of file diff --git a/docs/archive/faq/differences-with/README.md b/docs/archive/faq/differences-with/README.md deleted file mode 100644 index d020cfd1db38..000000000000 --- a/docs/archive/faq/differences-with/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# Differences with - diff --git a/docs/archive/faq/differences-with/fivetran-vs-airbyte.md b/docs/archive/faq/differences-with/fivetran-vs-airbyte.md deleted file mode 100644 index 9a9fe1045660..000000000000 --- a/docs/archive/faq/differences-with/fivetran-vs-airbyte.md +++ /dev/null @@ -1,27 +0,0 @@ -# Fivetran vs Airbyte - -We wrote an article, “[Open-source vs. Commercial Software: How to Solve the Data Integration Problem](https://airbyte.com/articles/data-engineering-thoughts/open-source-vs-commercial-software-how-to-better-solve-data-integration/),” in which we describe the pros and cons of Fivetran’s commercial approach and Airbyte’s open-source approach. Don’t hesitate to check it out for more detailed arguments. As a summary, here are the differences: - -![](https://airbyte.com/wp-content/uploads/2021/01/Airbyte-vs-Fivetran.png) - -## **Fivetran:** - -* **Limited high-quality connectors:** after 8 years in business, Fivetran supports 150 connectors. The more connectors, the more difficult it is for Fivetran to keep the same level of maintenance across all connectors. They will always have a ROI consideration to maintaining long-tailed connectors. -* **Pricing indexed on usage:** Fivetran’s pricing is indexed on the number of active rows \(rows added or edited\) per month. Teams always need to keep that in mind and are not free to move data without thinking about cost, as the costs can grow fast. -* **Security and privacy compliance:** all companies are subject to privacy compliance laws, such as GDPR, CCPA, HIPAA, etc. As a matter of fact, above a certain stage \(about 100 employees\) in a company, all external products need to go through a security compliance process that can take several months. -* **No moving data between internal databases:** Fivetran sits in the cloud, so if you have to replicate data from an internal database to another, it makes no sense to have the data move through them \(Fivetran\) for privacy and cost reasons. - -## **Airbyte:** - -* **Free, as open source, so no more pricing based on usage**: learn more about our [future business model](https://handbook.airbyte.io/strategy/business-model) \(connectors will always remain open source\). -* **Supporting 60 connectors within 8 months from inception**. Our goal is to reach 200+ connectors by the end of 2021. -* **Building new connectors made trivial, in the language of your choice:** Airbyte makes it a lot easier to create your own connector, vs. building them yourself in-house \(with Airflow or other tools\). Scheduling, orchestration, and monitoring comes out of the box with Airbyte. -* **Addressing the long tail of connectors:** with the help of the community, Airbyte ambitions to support thousands of connectors. -* **Adapt existing connectors to your needs:** you can adapt any existing connector to address your own unique edge case. -* **Using data integration in a workflow:** Airbyte’s API lets engineering teams add data integration jobs into their workflow seamlessly. -* **Integrates with your data stack and your needs:** Airflow, Kubernetes, dbt, etc. Its normalization is optional, it gives you a basic version that works out of the box, but also allows you to use dbt to do more complicated things. -* **Debugging autonomy:** if you experience any connector issue, you won’t need to wait for Fivetran’s customer support team to get back to you, if you can fix the issue fast yourself. -* **No more security and privacy compliance, as self-hosted, source-available and open-sourced \(MIT\)**. Any team can directly address their integration needs. - -Your data stays in your cloud. Have full control over your data, and the costs of your data transfers. - diff --git a/docs/archive/faq/differences-with/meltano-vs-airbyte.md b/docs/archive/faq/differences-with/meltano-vs-airbyte.md deleted file mode 100644 index f8e2ff5fba64..000000000000 --- a/docs/archive/faq/differences-with/meltano-vs-airbyte.md +++ /dev/null @@ -1,28 +0,0 @@ -# Meltano vs Airbyte - -We wrote an article, “[The State of Open-Source Data Integration and ETL](https://airbyte.com/articles/data-engineering-thoughts/the-state-of-open-source-data-integration-and-etl/),” in which we list and compare all ETL-related open-source projects, including Meltano and Airbyte. Don’t hesitate to check it out for more detailed arguments. As a summary, here are the differences: - -## **Meltano:** - -* **Meltano is built on top of the Singer protocol, whereas Airbyte is built on top of the Airbyte protocol**. Having initially created Airbyte on top of Singer, we wrote about why we didn't move forward with it [here](https://airbyte.com/blog/why-you-should-not-build-your-data-pipeline-on-top-of-singer) and [here](https://airbyte.com/blog/airbyte-vs-singer-why-airbyte-is-not-built-on-top-of-singer). Summarized, the reasons were: Singer connectors didn't always adhere to the Singer protocol, had poor standardization and visibility in terms of quality, and community governance and support was abandoned by Stitch. By contrast, we aim to make Airbyte a product that ["just works"](https://airbyte.com/blog/our-truth-for-2021-airbyte-just-works) and always plan to maximize engagement within the Airbyte community. -* **CLI-first approach:** Meltano was primarily built with a command line interface in mind. In that sense, they seem to target engineers with a preference for that interface. -* **Integration with Airflow for orchestration:** You can either use Meltano alone for orchestration or with Airflow; Meltano works both ways. -* All connectors must use Python. -* Meltano works with any of Singer's 200+ available connectors. However, in our experience, quality has been hit or miss. - -## **Airbyte:** - -In contrast, Airbyte is a company fully committed to the open-source project and has a [business model](https://handbook.airbyte.io/strategy/business-model) in mind around this project. Our [team](https://airbyte.com/about-us) are data integration experts that have built more than 1,000 integrations collectively at large scale. The team now counts 20 engineers working full-time on Airbyte. - -* **Airbyte supports more than 100 connectors after only 1 year since its inception**, 20% of which were built by the community. Our ambition is to support **200+ connectors by the end of 2021.** -* Airbyte’s connectors are **usable out of the box through a UI and API,** with monitoring, scheduling and orchestration. Airbyte was built on the premise that a user, whatever their background, should be able to move data in 2 minutes. Data engineers might want to use raw data and their own transformation processes, or to use Airbyte’s API to include data integration in their workflows. On the other hand, analysts and data scientists might want to use normalized consolidated data in their database or data warehouses. Airbyte supports all these use cases. -* **One platform, one project with standards:** This will help consolidate the developments behind one single project, some standardization and specific data protocol that can benefit all teams and specific cases. -* **Not limited by Singer’s data protocol:** In contrast to Meltano, Airbyte was not built on top of Singer, but its data protocol is compatible with Singer’s. This means Airbyte can go beyond Singer, but Meltano will remain limited. -* **Connectors can be built in the language of your choice,** as Airbyte runs them as Docker containers. -* **Airbyte integrates with your data stack and your needs:** Airflow, Kubernetes, dbt, etc. Its normalization is optional, it gives you a basic version that works out of the box, but also allows you to use dbt to do more complicated things. - -## **Other noteworthy differences:** - -* In terms of community, Meltano's Slack community got 430 new members in the last 6 months, while Airbyte got 800. -* The difference in velocity in terms of feature progress is easily measurable as both are open-source projects. Meltano closes about 30 issues per month, while Airbyte closes about 120. - diff --git a/docs/archive/faq/differences-with/pipelinewise-vs-airbyte.md b/docs/archive/faq/differences-with/pipelinewise-vs-airbyte.md deleted file mode 100644 index adcc9c2bf376..000000000000 --- a/docs/archive/faq/differences-with/pipelinewise-vs-airbyte.md +++ /dev/null @@ -1,25 +0,0 @@ -# Pipelinewise vs Airbyte - -## **PipelineWise:** - -PipelineWise is an open-source project by Transferwise that was built with the primary goal of serving their own needs. There is no business model attached to the project, and no apparent interest in growing the community. - -* **Supports 21 connectors,** and only adds new ones based on the needs of the mother company, Transferwise. -* **No business model attached to the project,** and no apparent interest from the company in growing the community. -* **As close to the original format as possible:** PipelineWise aims to reproduce the data from the source to an Analytics-Data-Store in as close to the original format as possible. Some minor load time transformations are supported, but complex mapping and joins have to be done in the Analytics-Data-Store to extract meaning. -* **Managed Schema Changes:** When source data changes, PipelineWise detects the change and alters the schema in your Analytics-Data-Store automatically. -* **YAML based configuration:** Data pipelines are defined as YAML files, ensuring that the entire configuration is kept under version control. -* **Lightweight:** No daemons or database setup are required. - -## **Airbyte:** - -In contrast, Airbyte is a company fully committed to the open-source project and has a [business model in mind](https://handbook.airbyte.io/) around this project. - -* Our ambition is to support **300+ connectors by the end of 2021.** We already supported about 50 connectors at the end of 2020, just 5 months after its inception. -* Airbyte’s connectors are **usable out of the box through a UI and API,** with monitoring, scheduling and orchestration. Airbyte was built on the premise that a user, whatever their background, should be able to move data in 2 minutes. Data engineers might want to use raw data and their own transformation processes, or to use Airbyte’s API to include data integration in their workflows. On the other hand, analysts and data scientists might want to use normalized consolidated data in their database or data warehouses. Airbyte supports all these use cases. -* **One platform, one project with standards:** This will help consolidate the developments behind one single project, some standardization and specific data protocol that can benefit all teams and specific cases. -* **Connectors can be built in the language of your choice,** as Airbyte runs them as Docker containers. -* **Airbyte integrates with your data stack and your needs:** Airflow, Kubernetes, dbt, etc. Its normalization is optional, it gives you a basic version that works out of the box, but also allows you to use dbt to do more complicated things. - -The data protocols for both projects are compatible with Singer’s. So it is easy to migrate a Singer tap or target onto Airbyte or PipelineWise. - diff --git a/docs/archive/faq/differences-with/singer-vs-airbyte.md b/docs/archive/faq/differences-with/singer-vs-airbyte.md deleted file mode 100644 index 58edd43eedb0..000000000000 --- a/docs/archive/faq/differences-with/singer-vs-airbyte.md +++ /dev/null @@ -1,28 +0,0 @@ -# Singer vs Airbyte - -If you want to understand the difference between Airbyte and Singer, you might be interested in 2 articles we wrote: - -* “[Airbyte vs. Singer: Why Airbyte is not built on top of Singer](https://airbyte.com/articles/data-engineering-thoughts/airbyte-vs-singer-why-airbyte-is-not-built-on-top-of-singer/).” -* “[The State of Open-Source Data Integration and ETL](https://airbyte.com/articles/data-engineering-thoughts/the-state-of-open-source-data-integration-and-etl/),” in which we list and compare all ETL-related open-source projects, including Singer and Airbyte. As a summary, here are the differences: - -![](https://airbyte.com/wp-content/uploads/2020/10/Landscape-of-open-source-data-integration-platforms-4.png) - -## **Singer:** - -* **Supports 96 connectors after 4 years.** -* **Increasingly outdated connectors:** Talend \(acquirer of StitchData\) seems to have stopped investing in maintaining Singer’s community and connectors. As most connectors see schema changes several times a year, more and more Singer’s taps and targets are not actively maintained and are becoming outdated. -* **Absence of standardization:** each connector is its own open-source project. So you never know the quality of a tap or target until you have actually used it. There is no guarantee whatsoever about what you’ll get. -* **Singer’s connectors are standalone binaries:** you still need to build everything around to make them work \(e.g. UI, configuration validation, state management, normalization, schema migration, monitoring, etc\). -* **No full commitment to open sourcing all connectors,** as some connectors are only offered by StitchData under a paid plan. _\*\*_ - -## **Airbyte:** - -* Our ambition is to support **300+ connectors by the end of 2021.** We already supported about 50 connectors at the end of 2020, just 5 months after its inception. -* Airbyte’s connectors are **usable out of the box through a UI and API**, with monitoring, scheduling and orchestration. Airbyte was built on the premise that a user, whatever their background, should be able to move data in 2 minutes. Data engineers might want to use raw data and their own transformation processes, or to use Airbyte’s API to include data integration in their workflows. On the other hand, analysts and data scientists might want to use normalized consolidated data in their database or data warehouses. Airbyte supports all these use cases. -* **One platform, one project with standards:** This will help consolidate the developments behind one single project, some standardization and specific data protocol that can benefit all teams and specific cases. -* **Connectors can be built in the language of your choice,** as Airbyte runs them as Docker containers. -* **Airbyte integrates with your data stack and your needs:** Airflow, Kubernetes, dbt, etc. Its normalization is optional, it gives you a basic version that works out of the box, but also allows you to use dbt to do more complicated things. -* **A full commitment to the open-source MIT project** with the promise not to hide some connectors behind paid walls. - -Note that Airbyte’s data protocol is compatible with Singer’s. So it is easy to migrate a Singer tap onto Airbyte. - diff --git a/docs/archive/faq/differences-with/stitchdata-vs-airbyte.md b/docs/archive/faq/differences-with/stitchdata-vs-airbyte.md deleted file mode 100644 index ec612ea9b2b1..000000000000 --- a/docs/archive/faq/differences-with/stitchdata-vs-airbyte.md +++ /dev/null @@ -1,29 +0,0 @@ -# StitchData vs Airbyte - -We wrote an article, “[Open-source vs. Commercial Software: How to Solve the Data Integration Problem](https://airbyte.com/articles/data-engineering-thoughts/open-source-vs-commercial-software-how-to-better-solve-data-integration/),” in which we describe the pros and cons of StitchData’s commercial approach and Airbyte’s open-source approach. Don’t hesitate to check it out for more detailed arguments. As a summary, here are the differences: - -![](https://airbyte.com/wp-content/uploads/2020/10/Open-source-vs-commercial-approach-2048x1843.png) - -## StitchData: - -* **Limited deprecating connectors:** Stitch only supports 150 connectors. Talend has stopped investing in StitchData and its connectors. And on Singer, each connector is its own open-source project. So you never know the quality of a tap or target until you have actually used it. There is no guarantee whatsoever about what you’ll get. -* **Pricing indexed on usage:** StitchData’s pricing is indexed on the connectors used and the volume of data transferred. Teams always need to keep that in mind and are not free to move data without thinking about cost. -* **Security and privacy compliance:** all companies are subject to privacy compliance laws, such as GDPR, CCPA, HIPAA, etc. As a matter of fact, above a certain stage \(about 100 employees\) in a company, all external products need to go through a security compliance process that can take several months. -* **No moving data between internal databases:** StitchData sits in the cloud, so if you have to replicate data from an internal database to another, it makes no sense to have the data move through their cloud for privacy and cost reasons. -* **StitchData’s Singer connectors are standalone binaries:** you still need to build everything around to make them work. And it’s hard to update some pre-built connectors, as they are of poor quality. - -## Airbyte: - -* **Free, as open source, so no more pricing based on usage:** learn more about our [future business model](https://handbook.airbyte.io/strategy/business-model) \(connectors will always remain open-source\). -* **Supporting 50+ connectors by the end of 2020** \(so in only 5 months of existence\). Our goal is to reach 300+ connectors by the end of 2021. -* **Building new connectors made trivial, in the language of your choice:** Airbyte makes it a lot easier to create your own connector, vs. building them yourself in-house \(with Airflow or other tools\). Scheduling, orchestration, and monitoring comes out of the box with Airbyte. -* **Maintenance-free connectors you can use in minutes.** Just authenticate your sources and warehouse, and get connectors that adapt to schema and API changes for you. -* **Addressing the long tail of connectors:** with the help of the community, Airbyte ambitions to support thousands of connectors. -* **Adapt existing connectors to your needs:** you can adapt any existing connector to address your own unique edge case. -* **Using data integration in a workflow:** Airbyte’s API lets engineering teams add data integration jobs into their workflow seamlessly. -* **Integrates with your data stack and your needs:** Airflow, Kubernetes, dbt, etc. Its normalization is optional, it gives you a basic version that works out of the box, but also allows you to use dbt to do more complicated things. -* **Debugging autonomy:** if you experience any connector issue, you won’t need to wait for Fivetran’s customer support team to get back to you, if you can fix the issue fast yourself. -* **Your data stays in your cloud.** Have full control over your data, and the costs of your data transfers. -* **No more security and privacy compliance, as self-hosted and open-sourced \(MIT\).** Any team can directly address their integration needs. -* **Premium support directly on our Slack for free**. Our time to resolution is about 3-4 hours in average. - diff --git a/docs/archive/faq/getting-started.md b/docs/archive/faq/getting-started.md deleted file mode 100644 index fd4ce42d47f6..000000000000 --- a/docs/archive/faq/getting-started.md +++ /dev/null @@ -1,50 +0,0 @@ -# Getting Started - -## **What do I need to get started using Airbyte?** - -You can deploy Airbyte in several ways, as [documented here](../../deploying-airbyte/README.md). Airbyte will then help you replicate data between a source and a destination. If you don’t see the connector you need, you can [build your connector yourself](../../connector-development) and benefit from Airbyte’s optional scheduling, orchestration and monitoring modules. - -## **How long does it take to set up Airbyte?** - -It depends on your source and destination. Check our setup guides to see the tasks for your source and destination. Each source and destination also has a list of prerequisites for setup. To make setup faster, get your prerequisites ready before you start to set up your connector. During the setup process, you may need to contact others \(like a database administrator or AWS account owner\) for help, which might slow you down. But if you have access to the connection information, it can take 2 minutes: see [demo video. ](https://www.youtube.com/watch?v=jWVYpUV9vEg) - -## **What data sources does Airbyte offer connectors for?** - -We already offer 100+ connectors, and will focus all our effort in ramping up the number of connectors and strengthening them. If you don’t see a source you need, you can file a [connector request here](https://github.com/airbytehq/airbyte/issues/new?assignees=&labels=area%2Fintegration%2C+new-integration&template=new-integration-request.md&title=). - -## **Where can I see my data in Airbyte?** - -You can’t see your data in Airbyte, because we don’t store it. The sync loads your data into your destination \(data warehouse, data lake, etc.\). While you can’t see your data directly in Airbyte, you can check your schema and sync status on the source detail page in Airbyte. - -## **Can I add multiple destinations?** - -Sure, you can. Just go to the "Destinations" section and click on the top right "+ new destination" button. You can have multiple destinations for the same source, and multiple sources for the same destination. - -## Am I limited to GUI interaction or is there a way to set up / run / interact with Airbyte programmatically? - -You can use the API to do anything you do today from the UI. Though, word of notice, the API is in alpha and may change. You won’t lose any functionality, but you may need to update your code to catch up to any backwards incompatible changes in the API. - -## How does Airbyte handle connecting to databases that are behind a firewall / NAT? - -We don’t. Airbyte is to be self-hosted in your own private cloud. - -## Can I set a start time for my integration? - -[Here](../../understanding-airbyte/connections#sync-schedules) is the link to the docs on scheduling syncs. - -## **Can I disable analytics in Airbyte?** - -Yes, you can control what's sent outside of Airbyte for analytics purposes. - -We added the following telemetry to Airbyte to ensure the best experience for users: - -* Measure usage of features & connectors -* Measure failure rate of connectors to address bugs quickly -* Reach out to our users about Airbyte community updates if they opt-in -* ... - -To disable telemetry, modify the `.env` file and define the two following environment variables: - -```text -TRACKING_STRATEGY=logging -``` diff --git a/docs/archive/faq/security-and-data-audits.md b/docs/archive/faq/security-and-data-audits.md deleted file mode 100644 index e56db4de7ac3..000000000000 --- a/docs/archive/faq/security-and-data-audits.md +++ /dev/null @@ -1,14 +0,0 @@ -# Security & Data Audits - -## **How secure is Airbyte?** - -Airbyte is an open-source self-hosted solution, so let’s say it is as safe as your data infrastructure. _\*\*_ - -## **Is Airbyte GDPR compliant?** - -Airbyte is a self-hosted solution, so it doesn’t bring any security or privacy risk to your infrastructure. We do intend to add data quality and privacy compliance features in the future, in order to give you more visibility on that topic. - -## **How does Airbyte charge?** - -We don’t. All connectors are all under the MIT license. If you are curious about the business model we have in mind, please check our [company handbook](https://handbook.airbyte.io/strategy/business-model). - diff --git a/docs/archive/faq/transformation-and-schemas.md b/docs/archive/faq/transformation-and-schemas.md deleted file mode 100644 index 554b11b558fd..000000000000 --- a/docs/archive/faq/transformation-and-schemas.md +++ /dev/null @@ -1,20 +0,0 @@ -# Transformation and Schemas - -## **Where's the T in Airbyte’s ETL tool?** - -Airbyte is actually an ELT tool, and you have the freedom to use it as an EL-only tool. The transformation part is done by default, but it is optional. You can choose to receive the data in raw \(JSON file for instance\) in your destination. - -We do provide normalization \(if option is still on\) so that data analysts / scientists / any users of the data can use it without much effort. - -We also intend to integrate deeply with dbt to make it easier for your team to continue relying you on them, if this was what you were doing. - -## **How does Airbyte handle replication when a data source changes its schema?** - -Airbyte continues to sync data using the configured schema until that schema is updated. Because Airbyte treats all fields as optional, if a field is renamed or deleted in the source, that field simply will no longer be replicated, but all remaining fields will. The same is true for streams as well. - -For now, the schema can only be updated manually in the UI \(by clicking "Update Schema" in the settings page for the connection\). When a schema is updated Airbyte will re-sync all data for that source using the new schema. - -## **How does Airbyte handle namespaces \(or schemas for the DB-inclined\)?** - -Airbyte respects source-defined namespaces when syncing data with a namespace-supported destination. See [this](../../understanding-airbyte/namespaces.md) for more details. - diff --git a/docs/archive/mongodb.md b/docs/archive/mongodb.md deleted file mode 100644 index d239da867673..000000000000 --- a/docs/archive/mongodb.md +++ /dev/null @@ -1,102 +0,0 @@ -# Mongo DB - -The MongoDB source supports Full Refresh and Incremental sync strategies. - -## Resulting schema - -MongoDB does not have anything like table definition, thus we have to define column types from actual attributes and their values. Discover phase have two steps: - -### Step 1. Find all unique properties - -Connector runs the map-reduce command which returns all unique document props in the collection. Map-reduce approach should be sufficient even for large clusters. - -#### Note - -To work with Atlas MongoDB, a **non-free** tier is required, as the free tier does not support the ability to perform the mapReduce operation. - -### Step 2. Determine property types - -For each property found, connector selects 10k documents from the collection where this property is not empty. If all the selected values have the same type - connector will set appropriate type to the property. In all other cases connector will fallback to `string` type. - -## Features - -| Feature | Supported | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental - Append Sync | Yes | -| Replicate Incremental Deletes | No | -| Namespaces | No | - -### Full Refresh sync - -Works as usual full refresh sync. - -### Incremental sync - -Cursor field can not be nested. Currently only top level document properties are supported. - -Cursor should **never** be blank. In case cursor is blank - the incremental sync results might be unpredictable and will totally rely on MongoDB comparison algorithm. - -Only `datetime` and `integer` cursor types are supported. Cursor type is determined based on the cursor field name: - -* `datetime` - if cursor field name contains a string from: `time`, `date`, `_at`, `timestamp`, `ts` -* `integer` - otherwise - -## Getting started - -This guide describes in details how you can configure MongoDB for integration with Airbyte. - -### Create users - -Run `mongo` shell, switch to `admin` database and create a `READ_ONLY_USER`. `READ_ONLY_USER` will be used for Airbyte integration. Please make sure that user has read-only privileges. - -```javascript -mongo -use admin; -db.createUser({user: "READ_ONLY_USER", pwd: "READ_ONLY_PASSWORD", roles: [{role: "read", db: "TARGET_DATABASE"}]} -``` - -Make sure the user have appropriate access levels. - -### Configure application - -In case your application uses MongoDB without authentication you will have to adjust code base and MongoDB config to enable MongoDB authentication. **Otherwise your application might go down once MongoDB authentication will be enabled.** - -### Enable MongoDB authentication - -Open `/etc/mongod.conf` and add/replace specific keys: - -```yaml -net: - bindIp: 0.0.0.0 - -security: - authorization: enabled -``` - -Binding to `0.0.0.0` will allow to connect to database from any IP address. - -The last line will enable MongoDB security. Now only authenticated users will be able to access the database. - -### Configure firewall - -Make sure that MongoDB is accessible from external servers. Specific commands will depend on the firewall you are using \(UFW/iptables/AWS/etc\). Please refer to appropriate documentation. - -Your `READ_ONLY_USER` should now be ready for use with Airbyte. - - -#### Possible configuration Parameters - -* [Authentication Source](https://docs.mongodb.com/manual/reference/connection-string/#mongodb-urioption-urioption.authSource) -* Host: URL of the database -* Port: Port to use for connecting to the database -* User: username to use when connecting -* Password: used to authenticate the user -* [Replica Set](https://docs.mongodb.com/manual/reference/connection-string/#mongodb-urioption-urioption.replicaSet) -* Whether to enable SSL - - -## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :-------- | :----- | :------ | -| 0.2.3 | 2021-07-20 | [4669](https://github.com/airbytehq/airbyte/pull/4669) | Subscriptions Stream now returns all kinds of subscriptions (including expired and canceled)| diff --git a/docs/archive/securing-airbyte.md b/docs/archive/securing-airbyte.md deleted file mode 100644 index 727ff5043eeb..000000000000 --- a/docs/archive/securing-airbyte.md +++ /dev/null @@ -1,28 +0,0 @@ -# Securing Airbyte access - -## Reporting Vulnerabilities -⚠️ Please do not file GitHub issues or post on our public forum for security vulnerabilities as they are public! ⚠️ - -Airbyte takes security issues very seriously. If you have any concern around Airbyte or believe you have uncovered a vulnerability, please get in touch via the e-mail address security@airbyte.io. In the message, try to provide a description of the issue and ideally a way of reproducing it. The security team will get back to you as soon as possible. - -Note that this security address should be used only for undisclosed vulnerabilities. Dealing with fixed issues or general questions on how to use the security features should be handled regularly via the user and the dev lists. Please report any security problems to us before disclosing it publicly. - -## Access control - -Airbyte, in its open-source version, does not support RBAC to manage access to the UI. - -However, multiple options exist for the operators to implement access control themselves. - -To secure access to Airbyte you have three options: -* Networking restrictions: deploy Airbyte in a private network or use a firewall to filter which IP is allowed to access your host. -* Put Airbyte behind a reverse proxy and handle the access control on the reverse proxy side. -* If you deployed Airbyte on a cloud provider: - * GCP: use the [Identity-Aware proxy](https://cloud.google.com/iap) service - * AWS: use the [AWS Systems Manager Session Manager](https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager.html) service - -**Non exhaustive** online resources list to set up auth on your reverse proxy: -* [Configure HTTP Basic Auth on NGINX for Airbyte](https://shadabshaukat.medium.com/deploy-and-secure-airbyte-with-nginx-reverse-proxy-basic-authentication-lets-encrypt-ssl-72bee223a4d9) -* [Kubernetes: Basic auth on a Nginx ingress controller](https://kubernetes.github.io/ingress-nginx/examples/auth/basic/) -* [How to set up Okta SSO on an NGINX reverse proxy](https://developer.okta.com/blog/2018/08/28/nginx-auth-request) -* [How to enable HTTP Basic Auth on Caddy](https://caddyserver.com/docs/caddyfile/directives/basicauth) -* [SSO for Traefik](https://github.com/thomseddon/traefik-forward-auth) diff --git a/docs/assets/docs/okta-app-integration-name.png b/docs/assets/docs/okta-app-integration-name.png deleted file mode 100644 index 87a9b89d77e4..000000000000 Binary files a/docs/assets/docs/okta-app-integration-name.png and /dev/null differ diff --git a/docs/assets/docs/okta-login-redirect-uris.png b/docs/assets/docs/okta-login-redirect-uris.png deleted file mode 100644 index 40463b1acfd4..000000000000 Binary files a/docs/assets/docs/okta-login-redirect-uris.png and /dev/null differ diff --git a/docs/assets/docs/oneOf-dropdown.png b/docs/assets/docs/oneOf-dropdown.png new file mode 100644 index 000000000000..f90e63335ba4 Binary files /dev/null and b/docs/assets/docs/oneOf-dropdown.png differ diff --git a/docs/assets/docs/oneOf-radio.png b/docs/assets/docs/oneOf-radio.png new file mode 100644 index 000000000000..6de8b302ebbf Binary files /dev/null and b/docs/assets/docs/oneOf-radio.png differ diff --git a/docs/cli-documentation.md b/docs/cli-documentation.md index 17b13dac71bd..f4d2dcd2803c 100644 --- a/docs/cli-documentation.md +++ b/docs/cli-documentation.md @@ -1,3 +1,7 @@ +--- +products: oss-community +--- + # CLI documentation :::caution @@ -173,7 +177,7 @@ headers: Authorization: Bearer ${MY_API_TOKEN} ``` -**Options based headers are overriding file based headers if an header is declared in both.** +**Options based headers are overriding file based headers if a header is declared in both.** ### `octavia` subcommands diff --git a/docs/cloud/core-concepts.md b/docs/cloud/core-concepts.md deleted file mode 100644 index 9383c6ffd036..000000000000 --- a/docs/cloud/core-concepts.md +++ /dev/null @@ -1,165 +0,0 @@ -# Core Concepts - -Airbyte enables you to build data pipelines and replicate data from a source to a destination. You can configure how frequently the data is synced, what data is replicated, what format the data is written to in the destination, and if the data is stored in raw tables format or basic normalized (or JSON) format. - -This page describes the concepts you need to know to use Airbyte. - -## Source - -A source is an API, file, database, or data warehouse that you want to ingest data from. - -## Destination - -A destination is a data warehouse, data lake, database, or an analytics tool where you want to load your ingested data. - -## Connector - -An Airbyte component which pulls data from a source or pushes data to a destination. - -## Connection - -A connection is an automated data pipeline that replicates data from a source to a destination. - -Setting up a connection involves configuring the following parameters: - - - - - - - - - - - - - - - - - - - - - - - - - - -
      Parameter - Description -
      Sync schedule - When should a data sync be triggered? -
      Destination Namespace and stream names - Where should the replicated data be written? -
      Catalog selection - What data should be replicated from the source to the destination? -
      Sync mode - How should the streams be replicated (read and written)? -
      Optional transformations - How should Airbyte protocol messages (raw JSON blob) data be converted into other data representations? -
      - -## Stream - -A stream is a group of related records. - -Examples of streams: - -- A table in a relational database -- A resource or API endpoint for a REST API -- The records from a directory containing many files in a filesystem - -## Field - -A field is an attribute of a record in a stream. - -Examples of fields: - -- A column in the table in a relational database -- A field in an API response - -## Namespace - -Namespace is a group of streams in a source or destination. Common use cases for namespaces are enforcing permissions, segregating test and production data, and general data organization. - -A schema in a relational database system is an example of a namespace. - -In a source, the namespace is the location from where the data is replicated to the destination. - -In a destination, the namespace is the location where the replicated data is stored in the destination. Airbyte supports the following configuration options for destination namespaces: - - - - - - - - - - - - - - - - - - -
      Configuration - Description -
      Mirror source structure - Some sources (for example, databases) provide namespace information for a stream. If a source provides the namespace information, the destination will reproduce the same namespace when this configuration is set. For sources or streams where the source namespace is not known, the behavior will default to the "Destination default" option. -
      Destination default - All streams will be replicated and stored in the default namespace defined on the destination settings page. For settings for popular destinations, see ​​Destination Connector Settings -
      Custom format - All streams will be replicated and stored in a user-defined custom format. See Custom format for more details. -
      - -## Connection sync modes - -A sync mode governs how Airbyte reads from a source and writes to a destination. Airbyte provides different sync modes to account for various use cases. - -- **Full Refresh | Overwrite:** Sync all records from the source and replace data in destination by overwriting it. -- **Full Refresh | Append:** Sync all records from the source and add them to the destination without deleting any data. -- **Incremental Sync | Append:** Sync new records from the source and add them to the destination without deleting any data. -- **Incremental Sync | Append + Deduped:** Sync new records from the source and add them to the destination. Also provides a de-duplicated view mirroring the state of the stream in the source. - -## Normalization - -Normalization is the process of structuring data from the source into a format appropriate for consumption in the destination. For example, when writing data from a nested, dynamically typed source like a JSON API to a relational destination like Postgres, normalization is the process which un-nests JSON from the source into a relational table format which uses the appropriate column types in the destination. - -Note that normalization is only relevant for the following relational database & warehouse destinations: - -- BigQuery -- Snowflake -- Redshift -- Postgres -- Oracle -- MySQL -- MSSQL - -Other destinations do not support normalization as described in this section, though they may normalize data in a format that makes sense for them. For example, the S3 destination connector offers the option of writing JSON files in S3, but also offers the option of writing statically typed files such as Parquet or Avro. - -After a sync is complete, Airbyte normalizes the data. When setting up a connection, you can choose one of the following normalization options: - -- Raw data (no normalization): Airbyte places the JSON blob version of your data in a table called `_airbyte_raw_` -- Basic Normalization: Airbyte converts the raw JSON blob version of your data to the format of your destination. _Note: Not all destinations support normalization._ -- [dbt Cloud integration](https://docs.airbyte.com/cloud/managing-airbyte-cloud/dbt-cloud-integration): Airbyte's dbt Cloud integration allows you to use dbt Cloud for transforming and cleaning your data during the normalization process. - -:::note - -Normalizing data may cause an increase in your destination's compute cost. This cost will vary depending on the amount of data that is normalized and is not related to Airbyte credit usage. - -::: - -## Workspace - -A workspace is a grouping of sources, destinations, connections, and other configurations. It lets you collaborate with team members and share resources across your team under a shared billing account. - -When you [sign up](http://cloud.airbyte.com/signup) for Airbyte Cloud, we automatically create your first workspace where you are the only user with access. You can set up your sources and destinations to start syncing data and invite other users to join your workspace. - -## Glossary of Terms - -You find and extended list of [Airbyte specific terms](https://glossary.airbyte.com/term/airbyte-glossary-of-terms/), [data engineering concepts](https://glossary.airbyte.com/term/data-engineering-concepts) or many [other data related terms](https://glossary.airbyte.com/). diff --git a/docs/cloud/getting-started-with-airbyte-cloud.md b/docs/cloud/getting-started-with-airbyte-cloud.md deleted file mode 100644 index 1071020a92c9..000000000000 --- a/docs/cloud/getting-started-with-airbyte-cloud.md +++ /dev/null @@ -1,221 +0,0 @@ -# Getting Started with Airbyte Cloud - -This page guides you through setting up your Airbyte Cloud account, setting up a source, destination, and connection, verifying the sync, and allowlisting an IP address. - -## Set up your Airbyte Cloud account - -To use Airbyte Cloud: - -1. If you haven't already, [sign up for Airbyte Cloud](https://cloud.airbyte.com/signup?utm_campaign=22Q1_AirbyteCloudSignUpCampaign_Trial&utm_source=Docs&utm_content=SetupGuide) using your email address, Google login, or GitHub login. - - Airbyte Cloud offers a 14-day free trial. For more information, see [Pricing](https://airbyte.com/pricing). - - :::note - If you are invited to a workspace, you cannot use your Google login to create a new Airbyte account. - ::: - -2. If you signed up using your email address, Airbyte will send you an email with a verification link. On clicking the link, you'll be taken to your new workspace. - - :::info - A workspace lets you collaborate with team members and share resources across your team under a shared billing account. - ::: - -## Set up a source - -:::info -A source is an API, file, database, or data warehouse that you want to ingest data from. -::: - -To set up a source: - -:::note - -Set your [default data residency](https://docs.airbyte.com/cloud/managing-airbyte-cloud/manage-data-residency#choose-your-default-data-residency) before creating a new source to ensure your data is processed in the correct region. - -::: - -1. On the Airbyte Cloud dashboard, click **Sources** and then click **+ New source**. -2. On the Set up the source page, select the source you want to set up from the **Source type** dropdown. - - The fields relevant to your source are displayed. The Setup Guide provides information to help you fill out the fields for your selected source. - -3. Click **Set up source**. - -## Set up a destination - -:::info -A destination is a data warehouse, data lake, database, or an analytics tool where you want to load your extracted data. -::: - -To set up a destination: - -1. On the Airbyte Cloud dashboard, click **Destinations** and then click **+ New destination**. -2. On the Set up the destination page, select the destination you want to set up from the **Destination type** dropdown. - - The fields relevant to your destination are displayed. The Setup Guide provides information to help you fill out the fields for your selected destination. - -3. Click **Set up destination**. - -## Set up a connection - -:::info -A connection is an automated data pipeline that replicates data from a source to a destination. -::: - -Setting up a connection involves configuring the following parameters: - -| Parameter | Description | -| ---------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | -| Replication frequency | How often should the data sync? | -| [Data residency](https://docs.airbyte.com/cloud/managing-airbyte-cloud/manage-data-residency#choose-the-data-residency-for-a-connection) | Where should the data be processed? | -| Destination Namespace and stream names | Where should the replicated data be written? | -| Catalog selection | Which streams and fields should be replicated from the source to the destination? | -| Sync mode | How should the streams be replicated (read and written)? | -| Optional transformations | How should Airbyte protocol messages (raw JSON blob) data be converted into other data representations? | - -For more information, see [Connections and Sync Modes](../understanding-airbyte/connections/README.md) and [Namespaces](../understanding-airbyte/namespaces.md) - -If you need to use [cron scheduling](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html): - -1. In the **Replication Frequency** dropdown, click **Cron**. -2. Enter a cron expression and choose a time zone to create a sync schedule. - -:::note - -- Only one sync per connection can run at a time. -- If cron schedules a sync to run before the last one finishes, the scheduled sync will start after the last sync completes. -- Cloud does not allow schedules that sync more than once per hour. - -::: - -To set up a connection: - -:::note - -Set your [default data residency](https://docs.airbyte.com/cloud/managing-airbyte-cloud/manage-data-residency#choose-your-default-data-residency) before creating a new connection to ensure your data is processed in the correct region. - -::: - -1. On the Airbyte Cloud dashboard, click **Connections** and then click **+ New connection**. -2. On the New connection page, select a source: - - - To use an existing source, select your desired source from the **Source** dropdown. Click **Use existing source**. - - - To set up a new source, select the source you want to set up from the **Source type** dropdown. The fields relevant to your source are displayed. The Setup Guide provides information to help you fill out the fields for your selected source. Click **Set up source**. - -3. Select a destination: - - - To use an existing destination, select your desired destination from the **Destination** dropdown. Click **Use existing destination**. - - To set up a new destination, select the destination you want to set up from the **Destination type** dropdown. The fields relevant to your destination are displayed. The Setup Guide provides information to help you fill out the fields for your selected destination. Click **Set up destination**. - - The Set up the connection page is displayed. - -4. From the **Replication frequency** dropdown, select how often you want the data to sync from the source to the destination. - - **Note:** The default replication frequency is **Every 24 hours**. - -5. From the **Destination Namespace** dropdown, select the format in which you want to store the data in the destination: - - **Note:** The default configuration is **Mirror source structure**. - - - - - - - - - - - - - - - - - - -
      Configuration - Description -
      Mirror source structure - Some sources (for example, databases) provide namespace information for a stream. If a source provides the namespace information, the destination will reproduce the same namespace when this configuration is set. For sources or streams where the source namespace is not known, the behavior will default to the "Destination default" option -
      Destination default - All streams will be replicated and stored in the default namespace defined on the Destination Settings page. For more information, see ​​Destination Connector Settings -
      Custom format - All streams will be replicated and stored in a custom format. See Custom format for more details -
      - -:::tip -To better understand the destination namespace configurations, see [Destination Namespace example](../understanding-airbyte/namespaces.md#examples) -::: - -6. (Optional) In the **Destination Stream Prefix (Optional)** field, add a prefix to stream names (for example, adding a prefix `airbyte_` renames `projects` to `airbyte_projects`). -7. (Optional) Click **Refresh schema** if you had previously triggered a sync with a subset of tables in the stream and now want to see all the tables in the stream. -8. Activate the streams you want to sync: - - (Optional) If your source has multiple tables, type the name of the stream you want to enable in the **Search stream name** search box. - - (Optional) To configure the sync settings for multiple streams, select the checkbox next to the desired streams, configure the settings in the purple box, and click **Apply**. -9. Configure the sync settings: - - 1. Toggle the **Sync** button to enable sync for the stream. - 2. **Source:** - 1. **Namespace**: The database schema of your source tables (auto-populated for your source) - 2. **Stream name**: The table name in the source (auto-populated for your source) - 3. **Sync mode**: Select how you want the data to be replicated from the source to the destination: - - For the source: - - - Select **Full Refresh** to copy the entire dataset each time you sync - - Select **Incremental** to replicate only the new or modified data - - For the destination: - - - Select **Overwrite** to erase the old data and replace it completely - - Select **Append** to capture changes to your table - **Note:** This creates duplicate records - - Select **Append + Deduped** to mirror your source while keeping records unique - - **Note:** Some sync modes may not yet be available for your source or destination - - 4. **Cursor field**: Used in **Incremental** sync mode to determine which records to sync. Airbyte pre-selects the cursor field for you (example: updated date). If you have multiple cursor fields, select the one you want. - 5. **Primary key**: Used in **Append + Deduped** sync mode to determine the unique identifier. - 6. **Destination**: - - **Namespace:** The database schema of your destination tables. - - **Stream name:** The final table name in destination. - -10. Click **Set up connection**. -11. Airbyte tests the connection. If the sync is successful, the Connection page is displayed. - -## Verify the connection - -Verify the sync by checking the logs: - -1. On the Airbyte Cloud dashboard, click **Connections**. The list of connections is displayed. Click on the connection you just set up. -2. The Sync History is displayed. Click on the first log in the sync history. -3. Check the data at your destination. If you added a Destination Stream Prefix while setting up the connection, make sure to search for the stream name with the prefix. - -## Allowlist IP addresses - -Depending on your [data residency](https://docs.airbyte.com/cloud/managing-airbyte-cloud/manage-data-residency#choose-your-default-data-residency) location, you may need to allowlist the following IP addresses to enable access to Airbyte: - -### United States and Airbyte Default - -#### GCP region: us-west3 - -[comment]: # "IMPORTANT: if changing the list of IP addresses below, you must also update the connector.airbyteCloudIpAddresses LaunchDarkly flag to show the new list so that the correct list is shown in the Airbyte Cloud UI, then reach out to the frontend team and ask them to update the default value in the useAirbyteCloudIps hook!" - -- 34.106.109.131 -- 34.106.196.165 -- 34.106.60.246 -- 34.106.229.69 -- 34.106.127.139 -- 34.106.218.58 -- 34.106.115.240 -- 34.106.225.141 - -### European Union - -#### AWS region: eu-west-3 - -- 13.37.4.46 -- 13.37.142.60 -- 35.181.124.238 diff --git a/docs/cloud/managing-airbyte-cloud.md b/docs/cloud/managing-airbyte-cloud.md deleted file mode 100644 index 372cfeb5e81c..000000000000 --- a/docs/cloud/managing-airbyte-cloud.md +++ /dev/null @@ -1,455 +0,0 @@ -# Managing Airbyte Cloud - -This page will help you manage your Airbyte Cloud workspaces and understand Airbyte Cloud limitations. - -## Manage your Airbyte Cloud workspace - -An Airbyte workspace allows you to collaborate with other users and manage connections under a shared billing account. - -:::info -Airbyte [credits](https://airbyte.com/pricing) are assigned per workspace and cannot be transferred between workspaces. -::: - -### Add users to your workspace - -To add a user to your workspace: - -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Settings**. - -2. Click **Access Management**. - -3. Click **+ New user**. - -4. On the **Add new users** dialog, enter the email address of the user you want to invite to your workspace. - -5. Click **Send invitation**. - - :::info - The user will have access to only the workspace you invited them to. They will be added as a workspace admin by default. - ::: - -### Remove users from your workspace​ - -To remove a user from your workspace: - -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Settings**. - -2. Click **Access Management**. - -3. Click **Remove** next to the user’s email. - -4. The **Remove user** dialog displays. Click **Remove**. - -### Rename a workspace - -To rename a workspace: - -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Settings**. - -2. Click **General Settings**. - -3. In the **Workspace name** field, enter the new name for your workspace. - -4. Click **Save changes**. - -### Delete a workspace - -To delete a workspace: - -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Settings**. - -2. Click **General Settings**. - -3. In the **Delete your workspace** section, click **Delete**. - -### Single workspace vs. multiple workspaces - -You can use one or multiple workspaces with Airbyte Cloud. - -#### Access -| Number of workspaces | Benefits | Considerations | -|----------------------|-------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------| -| Single | All users in a workspace have access to the same data. | If you add a user to a workspace, you cannot limit their access to specific data within that workspace. | -| Multiple | You can create multiple workspaces to allow certain users to access data in specific workspaces only. | Have to manage user access for each workspace individually. | - -#### Billing -| Number of workspaces | Benefits | Considerations | -|----------------------|-------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------| -| Single | You can use the same payment method for all purchases. | Credits pay for the use of resources in a workspace when you run a sync. Resource usage cannot be divided and paid for separately (for example, you cannot bill different departments in your organization for the usage of some credits in one workspace). | -| Multiple | You can use the same payment method for all purchases. | It is generally recommended to use the same payment method card for each workspace. - -### Switch between multiple workspaces - -To switch between workspaces: - -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click the current workspace name under the Airbyte logo in the navigation bar. - -2. Click **View all workspaces**. - -3. Click the name of the workspace you want to switch to. - -### Choose your default data residency - -Default data residency allows you to choose where your data is processed. - -:::note - -Configuring default data residency only applies to new connections and does not affect existing connections. - -::: - -For individual connections, you can choose a data residency that is different from the default through [connection settings](#choose-the-data-residency-for-a-connection) or when you create a [new connection](https://docs.airbyte.com/cloud/getting-started-with-airbyte-cloud#set-up-a-connection). - -:::note - -While the data is processed in a data plane in the chosen residency, the cursor and primary key data is stored in the US control plane. If you have data that cannot be stored in the US, do not use it as a cursor or primary key. -All your account information such as the name and email addresses of the Airbyte users are stored in the US as well. - -::: - -To choose your default data residency: - -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Settings**. - -2. Click **Data Residency**. - -3. Click the dropdown and choose the location for your default data residency. - -4. Click **Save changes**. - -:::info - -Depending on your network configuration, you may need to add [IP addresses](https://docs.airbyte.com/cloud/getting-started-with-airbyte-cloud/#allowlist-ip-addresses) to your allowlist. - -::: - -## Manage Airbyte Cloud notifications - -To set up Slack notifications: - -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Settings**. - -2. Click **Notifications**. - -3. [Create an Incoming Webhook for Slack](https://api.slack.com/messaging/webhooks). - -4. Navigate back to the Airbyte Cloud dashboard > Settings > Notifications and enter the Webhook URL. - -5. Toggle the **When sync fails** and **When sync succeeds** buttons as required. - -6. Click **Save changes**. - -## Understand Airbyte Cloud limits - -Airbyte Cloud limitations within the Cloud UI (does not apply to customers using [Powered by Airbyte](https://airbyte.com/embed-airbyte-connectors-with-api)) - -* Max number of workspaces per user: 5* -* Max number of sources in a workspace: 20* -* Max number of destinations in a workspace: 20* -* Max number of connections in a workspace: 20* -* Max number of streams that can be returned by a source in a discover call: 1K -* Max number of streams that can be configured to sync in a single connection: 1K -* Size of a single record: 100MB -* Shortest sync schedule: Every 60 min -* Schedule accuracy: +/- 30 min - -*Limits on workspaces, sources, destinations and connections do not apply to customers of [Powered by Airbyte](https://airbyte.com/embed-airbyte-connectors-with-api). To learn more [contact us](https://airbyte.com/talk-to-sales)! - - -## View the sync summary -The sync summary displays information about the data moved during a sync. - -To view the sync summary: -1. On the [Airbyte Cloud](http://cloud.airbyte.com/) dashboard, click **Connections**. - -2. Click a connection in the list to view its sync history. - - Sync History displays the sync status or [reset](https://docs.airbyte.com/operator-guides/reset/) status (Succeeded, Partial Success, Failed, Cancelled, or Running) and the [sync summary](#sync-summary). - - :::note - - Airbyte will try to sync your data three times. After a third failure, it will stop attempting to sync. - - ::: - -3. To view the full sync log, click the sync summary dropdown. - -### Sync summary - -| Data | Description | -|--------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------| -| x GB (also measured in KB, MB) | Amount of data moved during the sync. If basic normalization is on, the amount of data would not change since normalization occurs in the destination. | -| x emitted records | Number of records read from the source during the sync. | -| x committed records | Number of records the destination confirmed it received. | -| xh xm xs | Total time (hours, minutes, seconds) for the sync and basic normalization, if enabled, to complete. | - -:::note - -In a successful sync, the number of emitted records and committed records should be the same. - -::: - -## Edit stream configuration - -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Connections** and then click the connection you want to change. - -2. Click the **Replication** tab. - -The **Transfer** and **Streams** settings include the following parameters: - -| Parameter | Description | -|--------------------------------------|-------------------------------------------------------------------------------------| -| Replication frequency | How often the data syncs | -| [Non-breaking schema updates](#review-non-breaking-schema-changes) detected | How Airbyte handles syncs when it detects non-breaking schema changes in the source | -| Destination Namespace | Where the replicated data is written | -| Destination Stream Prefix | Helps you identify streams from different connectors | - -:::note - -These parameters apply to all streams in the connection. - -::: - -If you need to use [cron scheduling](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html): -1. In the **Replication Frequency** dropdown, click **Cron**. -2. Enter a cron expression and choose a time zone to create a sync schedule. - -:::note - -* Only one sync per connection can run at a time. -* If cron schedules a sync to run before the last one finishes, the scheduled sync will start after the last sync completes. -* Airbyte Cloud does not allow schedules that sync more than once per hour. - -::: - -In the **Activate the streams you want to sync section**, you can make changes to any stream you choose. - -To search for a stream: - -1. Click the **Search stream name** search box. - -2. Type the name of the stream you want to find. - -3. Streams matching your search are displayed in the list. - -To change individual stream configuration: - -![Single Edit Gif 7](https://user-images.githubusercontent.com/106352739/187313088-85c61a6d-1025-45fa-b14e-a7fe86defea4.gif) - -1. In the **Sync** column of the stream, toggle the sync on or off. - -2. Click the dropdown arrow in the **Sync mode** column and select the sync mode you want to apply. - -:::note - -Depending on the sync mode you select, you may need to choose a cursor or primary key. - -::: - -3. If there is a dropdown arrow in the **Cursor** or **Primary key** fields, click the dropdown arrow and choose the cursor or primary key. - -To change multiple stream configurations: - -![Batch Edit gif 5](https://user-images.githubusercontent.com/106352739/187312110-d16b4f9a-9d43-4b23-b644-b64004f33b58.gif) - -1. Click the first checkbox in the table header to select all streams in the connection. - -2. Deselect the checkboxes of streams you do not want to apply these changes to. - -3. In the highlighted header of the table, toggle the sync on or off. - -4. Click the dropdown arrow in the **Sync mode** column and select the sync mode you want to apply to these streams. - -5. If there is a dropdown arrow in the **Cursor** or **Primary key** fields of the highlighted table header, click the dropdown arrow and choose the cursor or primary key. - -6. Click **Apply** to apply these changes to the streams you selected, or click **Cancel** to discard the changes. - -To save the changes: -1. Click **Save changes**, or click **Cancel** to discard the changes. - -2. The **Stream configuration changed** dialog displays. This gives you the option to reset streams when you save the changes. - -:::caution - -Airbyte recommends that you reset streams. A reset will delete data in the destination of the affected streams and then re-sync that data. Skipping a reset is discouraged and might lead to unexpected behavior. - -::: - -3. Click **Save connection**, or click **Cancel** to close the dialog. - -To refresh the source schema: -1. Click **Refresh source schema** to fetch the schema of your data source. - -2. If the schema has changed, the **Refreshed source schema** dialog displays them. - -## Manage schema changes - -Once every 24 hours, Airbyte checks for changes in your source schema and allows you to review the changes and fix breaking changes. - -:::note - -Schema changes are flagged in your connection but are not propagated to your destination. - -::: - -### Review non-breaking schema changes - -To review non-breaking schema changes: -1. On the [Airbyte Cloud](http://cloud.airbyte.com/) dashboard, click **Connections** and select the connection with non-breaking changes (indicated by a **yellow exclamation mark** icon). - -2. Click **Review changes**. - -3. The **Refreshed source schema** dialog displays the changes. - -4. Review the changes and click **OK** to close the dialog. - -5. Scroll to the bottom of the page and click **Save changes**. - -:::note - - By default, Airbyte ignores non-breaking changes and continues syncing. You can configure how Airbyte handles syncs when it detects non-breaking changes by [editing the stream configuration](#edit-stream-configuration). - -::: - -### Fix breaking schema changes - -:::note - -Breaking changes can only occur in the **Cursor** or **Primary key** fields. - -::: - -To review and fix breaking schema changes: -1. On the [Airbyte Cloud](http://cloud.airbyte.com/) dashboard, click **Connections** and select the connection with breaking changes (indicated by a **red exclamation mark** icon). - -2. Click **Review changes**. - -3. The **Refreshed source schema** dialog displays the changes. - -4. Review the changes and click **OK** to close the dialog. - -5. In the streams table, the stream with a breaking change is highlighted. - -6. Fix the breaking change by selecting a new **Cursor** or **Primary key**. - -7. Scroll to the bottom of the page and click **Save changes**. - -:::note - -If a connection’s source schema has breaking changes, it will stop syncing. You must review and fix the changes before editing the connection or resuming syncs. - -::: - -### Enable schema update notifications - -To get notified when your source schema changes: -1. Make sure you have [webhook notifications](https://docs.airbyte.com/cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications#set-up-slack-notifications) set up. - -2. On the [Airbyte Cloud](http://cloud.airbyte.com/) dashboard, click **Connections** and select the connection you want to receive notifications for. - -3. Click the **Settings** tab on the Connection page. - -4. Toggle **Schema update notifications**. - -## Display the connection state - -Connection state provides additional information about incremental syncs. It includes the most recent values for the global or stream-level cursors, which can aid in debugging or determining which data will be included in the next syncs. - -To display the connection state: -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Connections** and then click the connection you want to display. - -2. Click the **Settings** tab on the Connection page. - -3. Click the **Advanced** dropdown arrow. - - **Connection State** displays. - -## Choose the data residency for a connection -You can choose the data residency for your connection in the connection settings. You can also choose data residency when creating a [new connection](https://docs.airbyte.com/cloud/getting-started-with-airbyte-cloud#set-up-a-connection), or you can set the [default data residency](#choose-your-default-data-residency) for your workspace. - -To choose the data residency for your connection: - -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Connections** and then click the connection that you want to change. - -2. Click the **Settings** tab. - -3. Click the **Data residency** dropdown and choose the location for your default data residency. - -4. Click **Save changes** - -:::note - -Changes to data residency will not affect any sync in progress. - -::: - -## Manage credits - -### Enroll in the Free Connector Program - -The Free Connector Program allows you to sync connections with [alpha](https://docs.airbyte.com/project-overview/product-release-stages#alpha) or [beta](https://docs.airbyte.com/project-overview/product-release-stages/#beta) connectors at no cost. - -:::note - -You must be enrolled in the program to use alpha and beta connectors for free. If either the source or destination is in alpha or beta, the whole connection is free to sync. When both the source and destination of a connection become [generally available](https://docs.airbyte.com/project-overview/product-release-stages/#general-availability-ga) (GA), the connection will no longer be free. We will email you two weeks before both connectors in a connection move to GA. - -::: - -Before enrolling in the program, [set up](https://docs.airbyte.com/cloud/getting-started-with-airbyte-cloud#set-up-a-source) at least one alpha or beta connector and verify your email if you haven't already. - -To enroll in the program: -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Credits** in the navigation bar. - -2. Click **Enroll now** in the **Free Connector Program** banner. - -3. Click **Enroll now**. - -4. Input your credit card information and click **Save card**. - -:::note - -Credit card information is required, even if you previously bought credits on Airbyte Cloud. This ensures uninterrupted syncs when both connectors move to GA. - -::: - -Since alpha and beta connectors are still in development, support is not provided. For additional resources, check out our [Connector Catalog](https://docs.airbyte.com/integrations/), [Troubleshooting & FAQ](https://docs.airbyte.com/troubleshooting/), and our [Community Slack](https://slack.airbyte.io/). - -### Buy credits - -This section guides you through purchasing credits on Airbyte Cloud. An Airbyte [credit](https://airbyte.com/pricing) is a unit of measure used to pay for Airbyte resources when you run a sync. - -To buy credits: - -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Credits** in the navigation bar. - -2. If you are unsure of how many credits you need, click **Talk to Sales** to find the right amount for your team. - -3. Click **Buy credits**. - -4. The Stripe payment page displays. If you want to change the amount of credits, click Qty **200**. The **Update quantity** dialog displays, and you can either type the amount or use minus (**-**) or plus (**+**) to change the quantity. Click **Update**. - - :::note - - Purchase limits: - * Minimum: 100 credits - * Maximum: 999 credits - - ::: - - To buy more credits or a subscription plan, reach out to [Sales](https://airbyte.com/talk-to-sales). - -5. Fill out the payment information. - - After you enter your billing address, sales tax is calculated and added to the total. - -6. Click **Pay**. - - Your payment is processed. The Credits page displays the updated quantity of credits, total credit usage, and the credit usage per connection. - - A receipt for your purchase is sent to your email. [Email us](mailto:ar@airbyte.io) for an invoice. - - :::note - - Credits expire after one year if they are not used. - - ::: diff --git a/docs/cloud/managing-airbyte-cloud/assets/connection-job-history.png b/docs/cloud/managing-airbyte-cloud/assets/connection-job-history.png new file mode 100644 index 000000000000..a9df65156a02 Binary files /dev/null and b/docs/cloud/managing-airbyte-cloud/assets/connection-job-history.png differ diff --git a/docs/cloud/managing-airbyte-cloud/assets/connection-status-page.png b/docs/cloud/managing-airbyte-cloud/assets/connection-status-page.png new file mode 100644 index 000000000000..a382786e3804 Binary files /dev/null and b/docs/cloud/managing-airbyte-cloud/assets/connection-status-page.png differ diff --git a/docs/cloud/managing-airbyte-cloud/configuring-connections.md b/docs/cloud/managing-airbyte-cloud/configuring-connections.md new file mode 100644 index 000000000000..269b68723ce6 --- /dev/null +++ b/docs/cloud/managing-airbyte-cloud/configuring-connections.md @@ -0,0 +1,90 @@ +--- +products: all +--- + +# Configuring Connections + +A connection links a source to a destination and defines how your data will sync. After you have created a connection, you can modify any of the configuration settings or stream settings. + +## Configure Connection Settings + +Configuring the connection settings allows you to manage various aspects of the sync, such as how often data syncs and where data is written. + +To configure these settings: + +1. In the Airbyte UI, click **Connections** and then click the connection you want to change. + +2. Click the **Replication** tab. + +3. Click the **Configuration** dropdown to expand the options. + +:::note + +These settings apply to all streams in the connection. + +::: + +You can configure the following settings: + +| Setting | Description | +|--------------------------------------|-------------------------------------------------------------------------------------| +| Connection Name | A custom name for your connection | +| [Replication frequency](/using-airbyte/core-concepts/sync-schedules.md) | How often data syncs (can be scheduled, cron, API-triggered or manual) | +| [Destination namespace](/using-airbyte/core-concepts/namespaces.md) | Where the replicated data is written to in the destination | +| Destination stream prefix | A prefix added to each table name in the destination | +| [Detect and propagate schema changes](/cloud/managing-airbyte-cloud/manage-schema-changes.md) | How Airbyte handles schema changes in the source | +| [Connection Data Residency](/cloud/managing-airbyte-cloud/manage-data-residency.md) | Where data will be processed (Cloud only) | + +## Modify streams in your connection + +In the **Activate the streams you want to sync** table, you choose which streams to sync and how they are loaded to the destination. + +:::info +A connection's schema consists of one or many streams. Each stream is most commonly associated with a database table or an API endpoint. Within a stream, there can be one or many fields or columns. +::: + +To modify streams: + +1. In the Airbyte UI, click **Connections** and then click the connection you want to change. + +2. Click the **Replication** tab. + +3. Scroll down to the **Activate the streams you want to sync** table. + +Modify an individual stream: + +1. In the **Activate the streams you want to sync** table, toggle **Sync** on or off for your selected stream. To select or deselect all streams, click the checkbox in the table header. To deselect an individual stream, deselect its checkbox in the table. + +2. Click the **Sync mode** dropdown and select the sync mode you want to apply. Depending on the sync mode you select, you may need to choose a cursor or primary key. + +:::info + +Source-defined cursors and primary keys are selected automatically and cannot be changed in the table. + +::: + +3. Click on a stream to display the stream details panel. You'll see each column we detect from the source. + +4. Column selection is available to protect PII or sensitive data from being synced to the destination. Toggle individual fields to include or exclude them in the sync, or use the toggle in the table header to select all fields at once. + +:::info + +* You can only deselect top-level fields. You cannot deselect nested fields. +* The Airbyte platform may read all data from the source (depending on the source), but it will only write data to the destination from fields you selected. Deselecting fields will not prevent the Airbyte platform from reading them. +* When you refresh the schema, newly added fields will be selected by default, even if you have previously deselected fields in that stream. + +::: + +5. Click the **X** to close the stream details panel. + +6. Click **Save changes**, or click **Cancel** to discard the changes. + +7. The **Stream configuration changed** dialog displays. This gives you the option to reset streams when you save the changes. + +:::tip + +When editing the stream configuration, Airbyte recommends that you reset streams. A reset will delete data in the destination of the affected streams and then re-sync that data. Skipping a reset is discouraged and might lead to unexpected behavior. + +::: + +8. Click **Save connection**. diff --git a/docs/cloud/managing-airbyte-cloud/dbt-cloud-integration.md b/docs/cloud/managing-airbyte-cloud/dbt-cloud-integration.md index 96510918cfd9..a98dd3beac86 100644 --- a/docs/cloud/managing-airbyte-cloud/dbt-cloud-integration.md +++ b/docs/cloud/managing-airbyte-cloud/dbt-cloud-integration.md @@ -1,7 +1,17 @@ +--- +products: cloud +--- + # Use the dbt Cloud integration By using the dbt Cloud integration, you can create and run dbt transformations during syncs in Airbyte Cloud. This allows you to transform raw data into a format that is suitable for analysis and reporting, including cleaning and enriching the data. +:::note + +Normalizing data may cause an increase in your destination's compute cost. This cost will vary depending on the amount of data that is normalized and is not related to Airbyte credit usage. + +::: + ## Step 1: Generate a service token Generate a [service token](https://docs.getdbt.com/docs/dbt-cloud-apis/service-tokens#generating-service-account-tokens) for your dbt Cloud transformation. @@ -17,7 +27,7 @@ Generate a [service token](https://docs.getdbt.com/docs/dbt-cloud-apis/service-t To set up the dbt Cloud integration in Airbyte Cloud: -1. On the Airbyte Cloud dashboard, click **Settings**. +1. In the Airbyte UI, click **Settings**. 2. Click **dbt Cloud integration**. diff --git a/docs/cloud/managing-airbyte-cloud/edit-stream-configuration.md b/docs/cloud/managing-airbyte-cloud/edit-stream-configuration.md deleted file mode 100644 index 4ccab5d12f38..000000000000 --- a/docs/cloud/managing-airbyte-cloud/edit-stream-configuration.md +++ /dev/null @@ -1,162 +0,0 @@ -# Manage syncs - -After you have created a connection, you can change how your data syncs to the destination by modifying the [configuration settings](#configure-connection-settings) and the [stream settings](#modify-streams-in-your-connection). - -## Configure connection settings - -Configuring the connection settings allows you to manage various aspects of the sync, such as how often data syncs and where data is written. - -To configure these settings: - -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Connections** and then click the connection you want to change. - -2. Click the **Replication** tab. - -3. Click the **Configuration** dropdown. - -You can configure the following settings: - -:::note - -These settings apply to all streams in the connection. - -::: - -| Setting | Description | -|--------------------------------------|-------------------------------------------------------------------------------------| -| Replication frequency | How often the data syncs | -| Destination namespace | Where the replicated data is written | -| Destination stream prefix | How you identify streams from different connectors | -| [Non-breaking schema updates](https://docs.airbyte.com/cloud/managing-airbyte-cloud/manage-schema-changes/#review-non-breaking-schema-changes) detected | How Airbyte handles syncs when it detects non-breaking schema changes in the source | - -To use [cron scheduling](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html): - -1. In the **Replication Frequency** dropdown, click **Cron**. - -2. Enter a cron expression and choose a time zone to create a sync schedule. - -:::note - -* Only one sync per connection can run at a time. -* If a sync is scheduled to run before the previous sync finishes, the scheduled sync will start after the completion of the previous sync. -* Airbyte Cloud does not support schedules that sync more frequently than once per hour. - -::: - -## Modify streams in your connection - -In the **Activate the streams you want to sync** table, you can choose which streams to sync and how they are loaded to the destination. - -:::note -A connection's schema consists of one or many streams. Each stream is most commonly associated with a database table or an API endpoint. Within a stream, there can be one or many fields or columns. -::: - -To modify streams: - -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Connections** and then click the connection you want to change. - -2. Click the **Replication** tab. - -3. Scroll down to the **Activate the streams you want to sync** table. - -:::note - -You can modify a single stream, or you can modify multiple streams at once. - -::: - -Modify an individual stream: - -![gif-single-edit-march-2023](https://user-images.githubusercontent.com/106352739/226917693-068256da-c948-4f22-bdce-49f5bad95bf6.gif) - -1. In the **Activate the streams you want to sync** table, toggle **Sync** on or off for your selected stream. - -2. Click the **Sync mode** dropdown and select the sync mode you want to apply. - -:::note - -Depending on the sync mode you select, you may need to choose a cursor or primary key. - -::: - -3. Select the **Cursor** or **Primary keys** when they are required by the stream sync mode. - -:::note - -Source-defined cursors and primary keys are selected automatically and cannot be changed in the table. - -::: - -4. Click on a stream to display the stream details panel. - -5. Toggle individual fields to include or exclude them in the sync, or use the toggle in the table header to select all fields at once. - -:::note - -* You can only deselect top-level fields. You cannot deselect nested fields. -* The Airbyte platform may read all data from the source (depending on the source), but it will only write data to the destination from fields you selected. Deselecting fields will not prevent the Airbyte platform from reading them. -* When you refresh the schema, newly added fields will be selected by default, even if you have previously deselected fields in that stream. - -::: - -6. Depending on the sync mode you chose for your connection, you can select the **Cursor** or **Primary keys** for individual fields in this table. - -7. Click the **X** to close the stream details panel. - -8. Click **Save changes**, or click **Cancel** to discard the changes. - -9. The **Stream configuration changed** dialog displays. This gives you the option to reset streams when you save the changes. - -:::caution - -Airbyte recommends that you reset streams. A reset will delete data in the destination of the affected streams and then re-sync that data. Skipping a reset is discouraged and might lead to unexpected behavior. - -::: - -10. Click **Save connection**. - -Modify multiple streams: - -![gif-batch-edit-march-2023](https://user-images.githubusercontent.com/106352739/226917994-c43941db-bb54-4a12-8270-f24fc4e2e6a7.gif) - -1. In the **Activate the streams you want to sync** table, select the checkboxes of streams that you want to apply changes to. - -:::note - -To select or deselect all streams, click the checkbox in the table header. To deselect an individual stream, deselect its checkbox in the table. - -::: - -* In the highlighted footer of the table: - - 1. Toggle **Sync** on or off. - - 2. Click the **Sync mode** dropdown and select the sync mode you want to apply. - - :::note - - Depending on the sync mode you select, you may need to choose a cursor or primary key. - - ::: - - 3. Select the **Cursor** and **Primary keys** if there are dropdowns in those fields. - - :::note - - Source-defined cursors and primary keys cannot be changed while configuring multiple streams. - - ::: - - 4. Click **Apply** to apply these changes to the streams you selected, or click **Cancel** to discard the changes. - -2. Click **Save changes**, or click **Cancel** to discard the changes. - -3. The **Stream configuration changed** dialog displays. This gives you the option to reset streams when you save the changes. - -:::caution - -Airbyte recommends that you reset streams. A reset will delete data in the destination of the affected streams and then re-sync that data. Skipping a reset is discouraged and might lead to unexpected behavior. - -::: - -4. Click **Save connection**. diff --git a/docs/cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications.md b/docs/cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications.md index 1e6e8e0eb3e1..b03e0d24d6e9 100644 --- a/docs/cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications.md +++ b/docs/cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications.md @@ -1,38 +1,89 @@ +--- +products: all +--- + # Manage notifications -This page provides guidance on how to manage notifications for Airbyte Cloud, allowing you to stay up-to-date on the activities in your workspace. +This page provides guidance on how to manage notifications for Airbyte, allowing you to stay up-to-date on the activities in your workspace. +## Notification Event Types -## Configure Notification Settings +| Type of Notification | Description | +|------------------------|---------------------------------------------------------------------------------------------------------------------| +| Failed Syncs | A sync from any of your connections fails. Note that if sync runs frequently or if there are many syncs in the workspace these types of events can be noisy | +| Successful Syncs | A sync from any of your connections succeeds. Note that if sync runs frequently or if there are many syncs in the workspace these types of events can be noisy +| Automated Connection Updates | A connection is updated automatically (ex. a source schema is automatically updated) | +| Connection Updates Requiring Action | A connection update requires you to take action (ex. a breaking schema change is detected) | +| Warning - Repeated Failures | A connection will be disabled soon due to repeated failures. It has failed 50 times consecutively or there were only failed jobs in the past 7 days | +| Sync Disabled - Repeated Failures | A connection was automatically disabled due to repeated failures. It will be disabled when it has failed 100 times consecutively or has been failing for 14 days in a row | +| Warning - Upgrade Required (Cloud only) | A new connector version is available and requires manual upgrade | +| Sync Disabled - Upgrade Required (Cloud only) | One or more connections were automatically disabled due to a connector upgrade deadline passing -To set up Webhook notifications: +## Configure Email Notification Settings -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Settings**. + -2. Click **Notifications**. +To set up email notifications: -3. Have a webhook URL ready if you plan to use webhook notifications. Using a Slack webook is recommended. [Create an Incoming Webhook for Slack](https://api.slack.com/messaging/webhooks). +1. In the Airbyte UI, click **Settings** and navigate to **Notifications**. -4. Toggle the type of events you are interested to receive notifications for. - 1. To enable webhook notifications, the webhook URL is required. For your convenience, we provide a 'test' function to send a test message to your webhook URL so you can make sure it's working as expected. +2. Toggle which messages you'd like to receive from Airbyte. All email notifications will be sent by default to the creator of the workspace. To change the recipient, edit and save the **notification email recipient**. If you would like to send email notifications to more than one recipient, you can enter an email distribution list (ie Google Group) as the recipient. -5. Click **Save changes**. +3. Click **Save changes**. -## Notification Event Types +:::note +All email notifications except for Successful Syncs are enabled by default. +::: + +## Configure Slack Notification settings + +To set up Slack notifications: + +If you're more of a visual learner, just head over to [this video](https://www.youtube.com/watch?v=NjYm8F-KiFc&ab_channel=Airbyte) to learn how to do this. You can also refer to the Slack documentation on how to [create an incoming webhook for Slack](https://api.slack.com/messaging/webhooks). + +### Create a Slack app + +1. **Create a Slack App**: Navigate to https://api.slack.com/apps/. Select `Create an App`. + +![](../../.gitbook/assets/notifications_create_slack_app.png) + +2. Select `From Scratch`. Enter your App Name (e.g. Airbyte Sync Notifications) and pick your desired Slack workspace. + +3. **Set up the webhook URL.**: in the left sidebar, click on `Incoming Webhooks`. Click the slider button in the top right to turn the feature on. Then click `Add New Webhook to Workspace`. + +![](../../.gitbook/assets/notifications_add_new_webhook.png) + +4. Pick the channel that you want to receive Airbyte notifications in (ideally a dedicated one), and click `Allow` to give it permissions to access the channel. You should see the bot show up in the selected channel now. You will see an active webhook right above the `Add New Webhook to Workspace` button. + +![](../../.gitbook/assets/notifications_webhook_url.png) + +5. Click `Copy.` to copy the link to your clipboard, which you will need to enter into Airbyte. + +Your Webhook URL should look something like this: + +![](../../.gitbook/assets/notifications_airbyte_notification_settings.png) + + +### Enable the Slack notification in Airbyte + +1. In the Airbyte UI, click **Settings** and navigate to **Notifications**. + +2. Paste the copied webhook URL to `Webhook URL`. Using a Slack webook is recommended. On this page, you can toggle each slider decide whether you want notifications on each notification type. + +3. **Test it out.**: you can click `Test` to send a test message to the channel. Or, just run a sync now and try it out! If all goes well, you should receive a notification in your selected channel that looks like this: + +![](../../.gitbook/assets/notifications_slack_message.png) -1. Failed syncs and successful syncs: When a connection sync has finished, you can choose to be notified whether the sync has failed, succeeded or both. Note that if sync runs frequently or if there are many syncs in the workspace these types of events can be noisy. -1. Automated Connection Updates: when Airbyte detects that the connection's source schema has changed and can be updated automatically, Airbyte will update your connection and send you a notification message. -1. Connection Updates Requiring Action: When Airbyte detects some updates that requires your action to run syncs. Since this will affect your sync from running as scheduled, you cannot disable this type of email notification. -1. Sync Disabled and Sync Disabled Warning: If a sync has been failing for multiple days or many times consecutively, Airbyte will disable the connection to prevent it from running further. Airbyte will send a Sync Disabled Warning notification when we detect the trend, and once the failure counts hits a threshold Airbyte will send a Sync Disabled notification and will actually disable the connection. Again, because the sync will not continue to run as you have configured, the Sync Disabled notification cannot be disabled. +You're done! - +4. Click **Save changes**. ## Enable schema update notifications -To get notified when your source schema changes: -1. Make sure you have `Automatic Connection Updates` and `Connection Updates Requiring Action` turned on for your desired notification channels; If these are off, even if you turned on schema update notifications in a connection's settings, Airbyte will *NOT* send out any notifications related to these types of events. +To be notified of any source schema changes: +1. Make sure you have enabled `Automatic Connection Updates` and `Connection Updates Requiring Action` notifications. If these are off, even if you turned on schema update notifications in a connection's settings, Airbyte will *NOT* send out any notifications related to these types of events. -2. On the [Airbyte Cloud](http://cloud.airbyte.com/) dashboard, click **Connections** and select the connection you want to receive notifications for. +2. In the Airbyte UI, click **Connections** and select the connection you want to receive notifications for. 3. Click the **Settings** tab on the Connection page. diff --git a/docs/cloud/managing-airbyte-cloud/manage-airbyte-cloud-workspace.md b/docs/cloud/managing-airbyte-cloud/manage-airbyte-cloud-workspace.md deleted file mode 100644 index 1db3697191a5..000000000000 --- a/docs/cloud/managing-airbyte-cloud/manage-airbyte-cloud-workspace.md +++ /dev/null @@ -1,85 +0,0 @@ -# Manage your workspace - -An Airbyte Cloud workspace allows you to collaborate with other users and manage connections under a shared billing account. - -:::info -Airbyte [credits](https://airbyte.com/pricing) are assigned per workspace and cannot be transferred between workspaces. -::: - -## Add users to your workspace - -To add a user to your workspace: - -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Settings**. - -2. Click **Access Management**. - -3. Click **+ New user**. - -4. On the **Add new users** dialog, enter the email address of the user you want to invite to your workspace. - -5. Click **Send invitation**. - - :::info - The user will have access to only the workspace you invited them to. They will be added as a workspace admin by default. - ::: - -## Remove users from your workspace​ - -To remove a user from your workspace: - -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Settings**. - -2. Click **Access Management**. - -3. Click **Remove** next to the user’s email. - -4. The **Remove user** dialog displays. Click **Remove**. - -## Rename a workspace - -To rename a workspace: - -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Settings**. - -2. Click **General Settings**. - -3. In the **Workspace name** field, enter the new name for your workspace. - -4. Click **Save changes**. - -## Delete a workspace - -To delete a workspace: - -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Settings**. - -2. Click **General Settings**. - -3. In the **Delete your workspace** section, click **Delete**. - -## Single workspace vs. multiple workspaces - -You can use one or multiple workspaces with Airbyte Cloud, which gives you flexibility in managing user access and billing. - -### Access -| Number of workspaces | Benefits | Considerations | -|----------------------|-------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------| -| Single | All users in a workspace have access to the same data. | If you add a user to a workspace, you cannot limit their access to specific data within that workspace. | -| Multiple | You can create multiple workspaces to allow certain users to access the data. | Since you have to manage user access for each workspace individually, it can get complicated if you have many users in multiple workspaces. | - -### Billing -| Number of workspaces | Benefits | Considerations | -|----------------------|-------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------| -| Single | You can use the same payment method for all purchases. | Credits pay for the use of resources in a workspace when you run a sync. Resource usage cannot be divided and paid for separately (for example, you cannot bill different departments in your organization for the usage of some credits in one workspace). | -| Multiple | Workspaces are independent of each other, so you can use a different payment method card for each workspace (for example, different credit cards per department in your organization). | You can use the same payment method for different workspaces, but each workspace is billed separately. Managing billing for each workspace can become complicated if you have many workspaces. | - -## Switch between multiple workspaces - -To switch between workspaces: - -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click the current workspace name under the Airbyte logo in the navigation bar. - -2. Click **View all workspaces**. - -3. Click the name of the workspace you want to switch to. diff --git a/docs/cloud/managing-airbyte-cloud/manage-connection-state.md b/docs/cloud/managing-airbyte-cloud/manage-connection-state.md new file mode 100644 index 000000000000..a745288fbe8c --- /dev/null +++ b/docs/cloud/managing-airbyte-cloud/manage-connection-state.md @@ -0,0 +1,28 @@ +--- +products: all +--- + +# Modifying connection state + +The connection state provides additional information about incremental syncs. It includes the most recent values for the global or stream-level cursors, which can aid in debugging or determining which data will be included in the next sync. + +To review the connection state: +1. In the Airbyte UI, click **Connections** and then click the connection you want to display. + +2. Click the **Settings** tab on the Connection page. + +3. Click the **Advanced** dropdown arrow. + + **Connection State** displays. + +Editing the connection state allows the sync to start from any date in the past. If the state is edited, Airbyte will start syncing incrementally from the new date. This is helpful if you do not want to fully resync your data. To edit the connection state: + +:::warning +Updates to connection state should be handled with extreme care. Updates may break your syncs, requiring a reset to fix. Make changes only as directed by the Airbyte team. +::: + +1. Click anywhere in the Connection state to start editing. + +2. Confirm changes by clicking "Update state". Discard any changes by clikcing "Revert changes". + +3. Confirm the changes to the connection state update. \ No newline at end of file diff --git a/docs/cloud/managing-airbyte-cloud/manage-credits.md b/docs/cloud/managing-airbyte-cloud/manage-credits.md index 7246e674a0cd..8f04f6ffa788 100644 --- a/docs/cloud/managing-airbyte-cloud/manage-credits.md +++ b/docs/cloud/managing-airbyte-cloud/manage-credits.md @@ -1,72 +1,72 @@ +--- +products: cloud +--- + # Manage credits -This page provides guidance on enrolling in Airbyte Cloud’s [Free Connector Program](https://airbyte.com/free-connector-program) and purchasing credits. +Airbyte [credits](https://airbyte.com/pricing) are used to pay for Airbyte resources when you run a sync. You can purchase credits on Airbyte Cloud to keep your data flowing without interruption. -## Enroll in the Free Connector Program +## Buy credits -The Free Connector Program allows you to sync connections with [alpha](https://docs.airbyte.com/project-overview/product-release-stages#alpha) or [beta](https://docs.airbyte.com/project-overview/product-release-stages/#beta) connectors at no cost. +To purchase credits directly through the UI, -:::note - -You must be enrolled in the program to use alpha and beta connectors for free. If either the source or destination is in alpha or beta, the whole connection is free to sync. When both the source and destination of a connection become [generally available](https://docs.airbyte.com/project-overview/product-release-stages/#general-availability-ga) (GA), the connection will no longer be free. We will email you two weeks before both connectors in a connection move to GA. - -::: +1. Click **Billing** in the left-hand sidebar. -Before enrolling in the program, [set up](https://docs.airbyte.com/cloud/getting-started-with-airbyte-cloud#set-up-a-source) at least one alpha or beta connector and verify your email if you haven't already. +2. If you are unsure of how many credits you need, use our [Cost Estimator](https://www.airbyte.com/pricing) or click **Talk to Sales** to find the right amount for your team. -To enroll in the program: -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Credits** in the navigation bar. +3. Click **Buy credits**. -2. Click **Enroll now** in the **Free Connector Program** banner. +4. Determine the quantity of credits you intend to purchase. Adjust the **credit quantity**. When you're ready, click **Checkout**. -3. Click **Enroll now**. + :::note -4. Input your credit card information and click **Save card**. + Purchase limits: + * Minimum: 20 credits + * Maximum: 6,000 credits -:::note - -Credit card information is required, even if you previously bought credits on Airbyte Cloud. This ensures uninterrupted syncs when both connectors move to GA. + ::: + + To buy more credits or discuss a custom plan, reach out to [Sales](https://airbyte.com/talk-to-sales). + +5. You'll be renavigated to a Stripe payment page. If this is your first time purchasing, you'll be asked for payment details. After you enter your billing address, sales tax (if applicable) is calculated and added to the total. + +6. Click **Pay**. -::: + Your payment is processed. The Billing page displays the available credits, total credit usage, and the credit usage per connection. -Since alpha and beta connectors are still in development, support is not provided. For additional resources, check out our [Connector Catalog](https://docs.airbyte.com/integrations/), [Troubleshooting & FAQ](https://docs.airbyte.com/troubleshooting/), and our [Community Slack](https://slack.airbyte.io/). + A receipt for your purchase is sent to your email. -## Buy credits + :::note -Airbyte [credits](https://airbyte.com/pricing) are used to pay for Airbyte resources when you run a sync. You can purchase credits on Airbyte Cloud to keep your data flowing without interruption. + Credits expire after one year if they are not used. -To buy credits: + ::: -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Billing** in the navigation bar. +## Automatic reload of credits (Beta) -2. If you are unsure of how many credits you need, click **Talk to Sales** to find the right amount for your team. +You can enroll in automatic top-ups of your credit balance. This is a beta feature for those who do not want to manually add credits each time. -3. Click **Buy credits**. +To enroll, [email us](mailto:billing@airbyte.io) with: -4. The Stripe payment page displays. If you want to change the amount of credits, click the **Qty 200** dropdown. The **Update quantity** dialog displays, and you can either type the amount or use minus (**–**) or plus (**+**) to change the quantity. Click **Update**. +1. A link to your workspace that you'd like to enable this feature for. +2. **Recharge threshold** The number under what credit balance you would like the automatic top up to occur. +3. **Recharge balance** The amount of credits you would like to refill to. - :::note - - Purchase limits: - * Minimum: 100 credits - * Maximum: 999 credits - - ::: +As an example, if the recharge threshold is 10 credits and recharge balance is 30 credits, anytime your workspace's credit balance dipped below 10 credits, Airbyte will automatically add enough credits to bring the balance back to 30 credits by charging the difference between your credit balance and 30 credits. - To buy more credits or a subscription plan, reach out to [Sales](https://airbyte.com/talk-to-sales). +To take a real example, if: +1. The credit balance reached 3 credits. +2. 27 credits are automatically charged to the card on file and added to the balance. +3. The ending credit balance is 30 credits. -5. Fill out the payment information. - - After you enter your billing address, sales tax is calculated and added to the total. +Note that the difference between the recharge credit amount and recharge threshold must be at least 20 as our minimum purchase is 20 credits. -6. Click **Pay**. - - Your payment is processed. The Billing page displays the available credits, total credit usage, and the credit usage per connection. +If you are enrolled and want to change your limits or cancel your enrollment, [email us](mailto:billing@airbyte.io). - A receipt for your purchase is sent to your email. [Email us](mailto:ar@airbyte.io) for an invoice. +## View invoice history - :::note - - Credits expire after one year if they are not used. - - ::: +1. In the Airbyte UI, click **Billing** in the navigation bar. + +2. Click **Invoice History**. You will be redirected to a Stripe portal. + +3. Enter the email address used to make the purchase to see your invoice history. [Email us](mailto:ar@airbyte.io) for an invoice. diff --git a/docs/cloud/managing-airbyte-cloud/manage-data-residency.md b/docs/cloud/managing-airbyte-cloud/manage-data-residency.md index da02874006ce..478c6c5862b5 100644 --- a/docs/cloud/managing-airbyte-cloud/manage-data-residency.md +++ b/docs/cloud/managing-airbyte-cloud/manage-data-residency.md @@ -1,10 +1,14 @@ -# Manage data residency +--- +products: cloud +--- -In Airbyte Cloud, you can set the default data residency and choose the data residency for individual connections, which can help you comply with data localization requirements. +# Setting data residency -## Choose your default data residency +In Airbyte Cloud, you can set the default data residency for your workspace and also set the the data residency for individual connections, which can help you comply with data localization requirements. -Default data residency allows you to choose where your data is processed. Set the default data residency before creating a new source or connection so workflows that rely on the default data residency, such as fetching the schema or testing the source or destination, can process data in the correct region. +## Choose your workspace default data residency + +Setting a default data residency allows you to choose where your data is processed. Set the default data residency **before** creating a new source or connection so that subsequent workflows that rely on the default data residency, such as fetching the schema or testing the source or destination, can process data in the correct region. :::note @@ -12,11 +16,11 @@ While the data is processed in a data plane of the chosen residency, the cursor ::: -When you set the default data residency, it applies to new connections only. If you do not set the default data residency, the [Airbyte Default](https://docs.airbyte.com/cloud/getting-started-with-airbyte-cloud/#united-states-and-airbyte-default) region is used. If you want to change the data residency for a connection, you can do so in its [connection settings](#choose-the-data-residency-for-a-connection). +When you set the default data residency, it applies your preference to new connections only. If you do not adjust the default data residency, the [Airbyte Default](configuring-connections.md) region is used (United States). If you want to change the data residency for an individual connection, you can do so in its [connection settings](configuring-connections.md). To choose your default data residency: -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Settings**. +1. In the Airbyte UI, click **Settings**. 2. Click **Data Residency**. @@ -26,16 +30,16 @@ To choose your default data residency: :::info -Depending on your network configuration, you may need to add [IP addresses](https://docs.airbyte.com/cloud/getting-started-with-airbyte-cloud/#allowlist-ip-addresses) to your allowlist. +Depending on your network configuration, you may need to add [IP addresses](/operating-airbyte/security.md#network-security-1) to your allowlist. ::: ## Choose the data residency for a connection -You can choose the data residency for your connection in the connection settings. You can also choose data residency when creating a [new connection](https://docs.airbyte.com/cloud/getting-started-with-airbyte-cloud#set-up-a-connection), or you can set the [default data residency](#choose-your-default-data-residency) for your workspace. +You can additionally choose the data residency for your connection in the connection settings. You can choose the data residency when creating a new connection, or you can set the default data residency for your workspace so that it applies for any new connections moving forward. -To choose the data residency for your connection: +To choose a custom data residency for your connection: -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Connections** and then click the connection that you want to change. +1. In the Airbyte UI, click **Connections** and then click the connection that you want to change. 2. Click the **Settings** tab. diff --git a/docs/cloud/managing-airbyte-cloud/manage-schema-changes.md b/docs/cloud/managing-airbyte-cloud/manage-schema-changes.md index 0d494e3b0526..5865c43f8a1e 100644 --- a/docs/cloud/managing-airbyte-cloud/manage-schema-changes.md +++ b/docs/cloud/managing-airbyte-cloud/manage-schema-changes.md @@ -1,86 +1,93 @@ -# Manage schema changes +--- +products: all +--- + +# Schema Change Management You can specify for each connection how Airbyte should handle any change of schema in the source. This process helps ensure accurate and efficient data syncs, minimizing errors and saving you time and effort in managing your data pipelines. -Airbyte checks for any changes in your source schema before syncing, at most once every 24 hours. +Airbyte checks for any changes in your source schema immediately before syncing, at most once every 24 hours. + +## Detection and Propagate Schema Changes +Based on your configured settings for **Detect and propagate schema changes**, Airbyte will automatically sync those changes or ignore them: -Based on your configured settings for "Detect and propagate schema changes", Airbyte can automatically sync those changes or ignore them: -* **Propagate all changes** automatically propagates stream changes (additions or deletions) or column changes (additions or deletions) detected in the source -* **Propagate column changes only** automatically propagates column changes detected in the source -* **Ignore** any schema change, in which case the schema you’ve set up will not change even if the source schema changes until you approve the changes manually -* **Pause connection** disables the connection from syncing further once a change is detected +| Setting | Description | +|---------------------|---------------------------------------------------------------------------------------------------------------------| +| Propagate all field and stream changes | All new tables and column changes from the source will automatically be propagated and reflected in the destination. This includes stream changes (additions or deletions), column changes (additions or deletions) and data type changes +| Propagate field changes only | Only column changes will be propagated +| Detect changes and manually approve | Schema changes will be detected, but not propagated. Syncs will continue running with the schema you've set up. To propagate the detected schema changes, you will need to approve the changes manually | +| Detect changes and pause connection | Connections will be automatically disabled as soon as any schema changes are detected | -When a new column is detected and propagated, values for that column will be filled in for the updated rows. If you are missing values for rows not updated, a backfill can be done by completing a full refresh. +## Types of Schema Changes +When propagation is enabled, your data in the destination will automatically shift to bring in the new changes. -When a column is deleted, the values for that column will stop updating for the updated rows and be filled with Null values. +| Type of Schema Change | Propagation Behavior | +|---------------------|---------------------------------------------------------------------------------------------------------------------| +| New Column | The new colummn will be created in the destination. Values for the column will be filled in for the updated rows. If you are missing values for rows not updated, a backfill can be done by completing a full resync. +| Removal of column | The old column will be removed from the destination. +| New stream | The first sync will create the new stream in the destination and fill all data in as if it is a historical sync. | +| Removal of stream | The stream will stop updating, and any existing data in the destination will remain. | +| Column data type changes | The data in the destination will remain the same. Any new or updated rows with incompatible data types will result in a row error in the raw Airbyte tables. You will need to refresh the schema and do a full resync to ensure the data types are consistent. -When a new stream is detected and propagated, the first sync will fill all data in as if it is a historical sync. When a stream is deleted from the source, the stream will stop updating, and we leave any existing data in the destination. The rest of the enabled streams will continue syncing. +:::tip +Ensure you receive webhook notifications for your connection by enabling `Schema update notifications` in the connection's settings. +::: -In all cases, if a breaking change is detected, the connection will be paused for manual review to prevent future syncs from failing. Breaking schema changes occur when: +In all cases, if a breaking schema change is detected, the connection will be paused immediately for manual review to prevent future syncs from failing. Breaking schema changes occur when: * An existing primary key is removed from the source * An existing cursor is removed from the source -See "Fix breaking schema changes" to understand how to resolve these types of changes. +To re-enable the streams, ensure the correct **Primary Key** and **Cursor** are selected for each stream and save the connection. ## Review non-breaking schema changes -To review non-breaking schema changes: -1. On the [Airbyte Cloud](http://cloud.airbyte.com/) dashboard, click **Connections** and select the connection with non-breaking changes (indicated by a **yellow exclamation mark** icon). +If the connection is set to **Detect any changes and manually approve** schema changes, Airbyte continues syncing according to your last saved schema. You need to manually approve any detected schema changes for the schema in the destination to change. + +1. In the Airbyte UI, click **Connections**. Select a connection and navigate to the **Replication** tab. If schema changes are detected, you'll see a blue "i" icon next to the Replication ab. 2. Click **Review changes**. -3. The **Refreshed source schema** dialog displays the changes. +3. The **Refreshed source schema** dialog displays the changes detected. 4. Review the changes and click **OK** to close the dialog. 5. Scroll to the bottom of the page and click **Save changes**. -:::note - - By default, Airbyte ignores non-breaking changes and continues syncing. You can configure how Airbyte handles syncs when it detects non-breaking changes by [editing the stream configuration](https://docs.airbyte.com/cloud/managing-airbyte-cloud/edit-stream-configuration). - -::: +## Resolving breaking changes -## Fix breaking schema changes +Breaking changes require your attention to resolve. They may immediately cause the connection to be disabled, or you can upgrade the connector manually within a time period once reviewing the changes. -Breaking schema changes occur when: -* An existing primary key is removed from the source -* An existing cursor is removed from the source +A connection will always automatically be disabled if an existing primary key or cursor field is removed. You must review and fix the changes before editing the connection or resuming syncs. -To review and fix breaking schema changes: -1. On the [Airbyte Cloud](http://cloud.airbyte.com/) dashboard, click **Connections** and select the connection with breaking changes (indicated by a **red exclamation mark** icon). +Breaking changes can also occur when a new version of the connector is released. In these cases, the connection will alert you of a breaking change but continue to sync until the cutoff date for upgrade. On the cutoff date, the connection will automatically be disabled on that date to prevent failure or unexpected behavior. It is **highly recommended** to upgrade before the cutoff date to ensure you continue syncing without interruption. -2. Click **Review changes**. - -3. The **Refreshed source schema** dialog displays the changes. - -4. Review the changes and click **OK** to close the dialog. +A major version upgrade will include a breaking change if any of these apply: -5. In the streams table, the stream with a breaking change is highlighted. +| Type of Change | Description | +|------------------|---------------------------------------------------------------------------------------------------------------------| +| Connector Spec Change | The configuration has been changed and syncs will fail until users reconfigure or re-authenticate. | +| Schema Change | The type of property previously present within a record has changed and a refresh of the source schema is required. +| Stream or Property Removal | Data that was previously being synced is no longer going to be synced | +| Destination Format / Normalization Change | The way the destination writes the final data or how Airbyte cleans that data is changing in a way that requires a full refresh | +| State Changes | The format of the source’s state has changed, and the full dataset will need to be re-synced | -6. Fix the breaking change by selecting a new **Cursor** or **Primary key**. +To review and fix breaking schema changes: +1. In the Airbyte UI, click **Connections** and select the connection with breaking changes. -7. Scroll to the bottom of the page and click **Save changes**. +2. Review the description of what has changed in the new version. The breaking change will require you to upgrade your source or destination to a new version by a specific cutoff date. -:::note - -If a connection’s source schema has breaking changes, it will stop syncing. You must review and fix the changes before editing the connection or resuming syncs. - -::: +3. Update the source or destination to the new version to continue syncing. ### Manually refresh the source schema -In addition to Airbyte Cloud’s automatic schema change detection, you can manually refresh the source schema to stay up to date with changes in your schema. +In addition to Airbyte's automatic schema change detection, you can manually refresh the source schema to stay up to date with changes in your schema. To manually refresh the source schema: - 1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Connections** and then click the connection you want to refresh. + 1. In the Airbyte UI, click **Connections** and then click the connection you want to refresh. 2. Click the **Replication** tab. 3. In the **Activate the streams you want to sync** table, click **Refresh source schema** to fetch the schema of your data source. - 4. If there are changes to the schema, you can review them in the **Refreshed source schema** dialog. - -## Manage Schema Change Notifications -[Refer to our notification documentation](https://docs.airbyte.com/cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications#enable-schema-update-notifications) to understand how to stay updated on any schema updates to your connections. \ No newline at end of file + 4. If there are changes to the schema, you can review them in the **Refreshed source schema** dialog. \ No newline at end of file diff --git a/docs/cloud/managing-airbyte-cloud/review-connection-state.md b/docs/cloud/managing-airbyte-cloud/review-connection-state.md deleted file mode 100644 index e2596f39ec2e..000000000000 --- a/docs/cloud/managing-airbyte-cloud/review-connection-state.md +++ /dev/null @@ -1,12 +0,0 @@ -# Review the connection state - -The connection state provides additional information about incremental syncs. It includes the most recent values for the global or stream-level cursors, which can aid in debugging or determining which data will be included in the next sync. - -To review the connection state: -1. On the [Airbyte Cloud](http://cloud.airbyte.com) dashboard, click **Connections** and then click the connection you want to display. - -2. Click the **Settings** tab on the Connection page. - -3. Click the **Advanced** dropdown arrow. - - **Connection State** displays. diff --git a/docs/cloud/managing-airbyte-cloud/review-connection-status.md b/docs/cloud/managing-airbyte-cloud/review-connection-status.md new file mode 100644 index 000000000000..c93a94d3bb1d --- /dev/null +++ b/docs/cloud/managing-airbyte-cloud/review-connection-status.md @@ -0,0 +1,50 @@ +--- +products: all +--- + +# Review the connection status +The connection status displays information about the connection and of each stream being synced. Reviewing this summary allows you to assess the connection's current status and understand when the next sync will be run. + +![Connection Status](./assets/connection-status-page.png) + +To review the connection status: +1. In the Airbyte UI, click **Connections**. + +2. Click a connection in the list to view its status. + +| Status | Description | +|------------------|---------------------------------------------------------------------------------------------------------------------| +| On time | The connection is operating within the expected timeframe expectations set by the replication frequency | +| On track | The connection is slightly delayed but is expected to catch up before the next sync. | +| Delayed | The connection has not loaded data within the scheduled replication frequency. For example, if the replication frequency is 1 hour, the connection has not loaded data for more than 1 hour | +| Error | The connection has not loaded data in more than two times the scheduled replication frequency. For example, if the replication frequency is 1 hour, the connection has not loaded data for more than 2 hours | +| Action Required | A breaking change related to the source or destination requires attention to resolve | +| In Progress | The connection is currently extracting or loading data | +| Disabled | The connection has been disabled and is not scheduled to run | +| Pending | The connection has not been run yet, so no status exists | + +If the most recent sync failed, you'll see the error message that will help diagnose if the failure is due to a source or destination configuration error. [Reach out](/community/getting-support.md) to us if you need any help to ensure you data continues syncing. + +:::info +If a sync starts to fail, it will automatically be disabled after 100 consecutive failures or 14 consecutive days of failure. +::: + +If a new major version of the connector has been released, you will also see a banner on this page indicating the cutoff date for the version. Airbyte recommends upgrading before the cutoff date to ensure your data continues syncing. If you do not upgrade before the cutoff date, Airbyte will automatically disable your connection. + +Learn more about version upgrades in our [resolving breaking change documentation](/cloud/managing-airbyte-cloud/manage-schema-changes#resolving-breaking-changes). + +## Review the stream status +The stream status allows you to monitor each stream's latest status. The stream will be highlighted with a grey pending bar to indicate the sync is actively extracting or loading data. + +| Status | Description | +|------------------|---------------------------------------------------------------------------------------------------------------------| +| On time | The stream is operating within the expected timeframe expectations set by the replication frequency | +| Error | The most recent sync for this stream failed +| Pending | The stream has not been synced yet, so not status exists | + +Each stream shows the last record loaded to the destination. Toggle the header to display the exact datetime the last record was loaded. + +You can [reset](/operator-guides/reset.md) an individual stream without resetting all streams in a connection by clicking the three grey dots next to any stream. + +You can also navigate directly to the stream's configuration by click the three grey dots next to any stream and selecting "Open details" to be redirected to the stream configuration. + diff --git a/docs/cloud/managing-airbyte-cloud/review-sync-history.md b/docs/cloud/managing-airbyte-cloud/review-sync-history.md new file mode 100644 index 000000000000..dae49ab3c7ac --- /dev/null +++ b/docs/cloud/managing-airbyte-cloud/review-sync-history.md @@ -0,0 +1,39 @@ +--- +products: all +--- + +# Review the sync history + +The job history displays information about synced data, such as the amount of data moved, the number of records read and committed, and the total sync time. Reviewing this summary can help you monitor the sync performance and identify any potential issues. + +![Job History](./assets/connection-job-history.png) + +To review the sync history, click a connection in the list to view its sync history. Sync History displays the sync status or [reset](/operator-guides/reset.md) status. The sync status is defined as: + +| Status | Description | +|---------------------|---------------------------------------------------------------------------------------------------------------------| +| Succeeded | 100% of the data has been extracted and loaded to the destination | +| Partially Succeeded | A subset of the data has been loaded to the destination +| Failed | None of the data has been loaded to the destination | +| Cancelled | The sync was cancelled manually before finishing | +| Running | The sync is currently running | + +## Sync summary + +Each sync shows the time the sync was initiated and additional metadata. This information can help in understanding sync performance over time. + +| Data | Description | +|------------------------------------------|--------------------------------------------------------------------------------------| +| x GB (also measured in KB, MB) | Amount of data moved during the sync | +| x extracted records | Number of records read from the source during the sync | +| x loaded records | Number of records the destination confirmed it received. | +| xh xm xs | Total time (hours, minutes, seconds) for the sync to complete | + + +:::note + +In the event of a failure, Airbyte will make several attempts to sync your data before waiting for the next sync to retry. The latest rules can be read about [here](../../understanding-airbyte/jobs.md#retry-rules). + +::: + +On this page, you can also view the complete logs and find any relevant errors, find a link to the job to share with Support, or download a copy of the logs locally. \ No newline at end of file diff --git a/docs/cloud/managing-airbyte-cloud/review-sync-summary.md b/docs/cloud/managing-airbyte-cloud/review-sync-summary.md deleted file mode 100644 index 7d6da7b286a0..000000000000 --- a/docs/cloud/managing-airbyte-cloud/review-sync-summary.md +++ /dev/null @@ -1,32 +0,0 @@ -# Review the sync summary -The sync summary displays information about synced data, such as the amount of data moved, the number of records read and committed, and the total sync time. Reviewing this summary can help you monitor the sync performance and identify any potential issues. - -To review the sync summary: -1. On the [Airbyte Cloud](http://cloud.airbyte.com/) dashboard, click **Connections**. - -2. Click a connection in the list to view its sync history. - - Sync History displays the sync status or [reset](https://docs.airbyte.com/operator-guides/reset/) status (Succeeded, Partial Success, Failed, Cancelled, or Running) and the [sync summary](#sync-summary). - - :::note - - In the event of a failure, Airbyte will make several attempts to sync your data before giving up. The latest rules can be read about [here](../../understanding-airbyte/jobs.md#retry-rules). - - ::: - -3. To view the full sync log, click the sync summary dropdown. - -## Sync summary - -| Data | Description | -|--------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------| -| x GB (also measured in KB, MB) | Amount of data moved during the sync. If basic normalization is on, the amount of data would not change since normalization occurs in the destination. | -| x extracted records | Number of records read from the source during the sync. | -| x loaded records | Number of records the destination confirmed it received. | -| xh xm xs | Total time (hours, minutes, seconds) for the sync and basic normalization, if enabled, to complete. | - -:::note - -In a successful sync, the number of extracted records and loaded records should be the same. - -::: diff --git a/docs/cloud/managing-airbyte-cloud/understand-airbyte-cloud-limits.md b/docs/cloud/managing-airbyte-cloud/understand-airbyte-cloud-limits.md index 470a50cc64d2..f9e4b5467a84 100644 --- a/docs/cloud/managing-airbyte-cloud/understand-airbyte-cloud-limits.md +++ b/docs/cloud/managing-airbyte-cloud/understand-airbyte-cloud-limits.md @@ -1,17 +1,16 @@ -# Understand Airbyte Cloud limits +--- +products: cloud +--- + +# Airbyte Cloud limits Understanding the following limitations will help you more effectively manage Airbyte Cloud. -* Max number of workspaces per user: 5* -* Max number of sources in a workspace: 20* +* Max number of workspaces per user: 3* +* Max number of instances of the same source connector: 10* * Max number of destinations in a workspace: 20* -* Max number of connections in a workspace: 20* -* Max number of consecutive sync failures before a connection is paused: 100 -* Max number of days with consecutive sync failures before a connection is paused: 14 days * Max number of streams that can be returned by a source in a discover call: 1K * Max number of streams that can be configured to sync in a single connection: 1K -* Size of a single record: 100MB -* Shortest sync schedule: Every 60 min -* Schedule accuracy: +/- 30 min +* Size of a single record: 20MB -*Limits on workspaces, sources, destinations and connections do not apply to customers of [Powered by Airbyte](https://airbyte.com/solutions/powered-by-airbyte). To learn more [contact us](https://airbyte.com/talk-to-sales)! +*Limits on workspaces, sources, and destinations do not apply to customers of [Powered by Airbyte](https://airbyte.com/solutions/powered-by-airbyte). To learn more [contact us](https://airbyte.com/talk-to-sales)! diff --git a/docs/community/code-of-conduct.md b/docs/community/code-of-conduct.md new file mode 100644 index 000000000000..4cb81d4468fc --- /dev/null +++ b/docs/community/code-of-conduct.md @@ -0,0 +1,91 @@ +--- +description: Our Community Code of Conduct +--- + +# Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to make participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others’ private information, such as a physical or electronic address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies within all project spaces, and it also applies when an individual is representing the project or its community in public spaces. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at [conduct@airbyte.io](mailto:conduct@airbyte.io). All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project’s leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/), version 1.4, available at [https://www.contributor-covenant.org/version/1/4/code-of-conduct.html](https://www.contributor-covenant.org/version/1/4/code-of-conduct.html) + +## Slack Code of Conduct + +Airbyte's Slack community is growing incredibly fast. We're home to over 1500 data professionals and are growing at an awesome pace. We are proud of our community, and have provided these guidelines to support new members in maintaining the wholesome spirit we have developed here. We appreciate your continued commitment to making this a community we are all excited to be a part of. + +### Rule 1: Be respectful. + +Our desire is for everyone to have a positive, fulfilling experience in Airbyte Slack, and we sincerely appreciate your help in making this happen. +All of the guidelines we provide below are important, but there’s a reason respect is the first rule. We take it seriously, and while the occasional breach of etiquette around Slack is forgivable, we cannot condone disrespectful behavior. + +### Rule 2: Use the most relevant channels. + +We deliberately use topic-specific Slack channels so members of the community can opt-in on various types of conversations. Our members take care to post their messages in the most relevant channel, and you’ll often see reminders about the best place to post a message (respectfully written, of course!). If you're looking for help directly from the Community Assistance Team or other Airbyte employees, please stick to posting in the airbyte-help channel, so we know you're asking us specifically! + +### Rule 3: Don’t double-post. + +Please be considerate of our community members’ time. We know your question is important, but please keep in mind that Airbyte Slack is not a customer service platform but a community of volunteers who will help you as they are able around their own work schedule. You have access to all the history, so it’s easy to check if your question has already been asked. + +### Rule 4: Check question for clarity and thoughtfulness. + +Airbyte Slack is a community of volunteers. Our members enjoy helping others; they are knowledgeable, gracious, and willing to give their time and expertise for free. Putting some effort into a well-researched and thoughtful post shows consideration for their time and will gain more responses. + +### Rule 5: Keep it public. + +This is a public forum; please do not contact individual members of this community without their express permission, regardless of whether you are trying to recruit someone, sell a product, or solicit help. + +### Rule 6: No soliciting! + +The purpose of the Airbyte Slack community is to provide a forum for data practitioners to discuss their work and share their ideas and learnings. It is not intended as a place to generate leads for vendors or recruiters, and may not be used as such. + +If you’re a vendor, you may advertise your product in #shameless-plugs. Advertising your product anywhere else is strictly against the rules. + +### Rule 7: Don't spam tags, or use @here or @channel. + +Using the @here and @channel keywords in a post will not help, as they are disabled in Slack for everyone excluding admins. Nonetheless, if you use them we will remind you with a link to this rule, to help you better understand the way Airbyte Slack operates. + +Do not tag specific individuals for help on your questions. If someone chooses to respond to your question, they will do so. You will find that our community of volunteers is generally very responsive and amazingly helpful! + +### Rule 8: Use threads for discussion. + +The simplest way to keep conversations on track in Slack is to use threads. The Airbyte Slack community relies heavily on threads, and if you break from this convention, rest assured one of our community members will respectfully inform you quickly! + +_If you see a message or receive a direct message that violates any of these rules, please contact an Airbyte team member and we will take the appropriate moderation action immediately. We have zero tolerance for intentional rule-breaking and hate speech._ + diff --git a/docs/community/getting-support.md b/docs/community/getting-support.md new file mode 100644 index 000000000000..339bd08399c4 --- /dev/null +++ b/docs/community/getting-support.md @@ -0,0 +1,71 @@ +--- +products: all +--- + +# Getting Support + +Hold up! Have you looked at [our docs](https://docs.airbyte.com/) yet? We recommend searching the wealth of knowledge in our documentation as many times the answer you are looking for is there! + +## Airbyte Open Source Support + +Running Airbyte Open Source and have questions that our docs could not clear up? Post your questions on our [Github Discussions](https://github.com/airbytehq/airbyte/discussions?_gl=1*70s0c6*_ga*MTc1OTkyOTYzNi4xNjQxMjQyMjA0*_ga_HDBMVFQGBH*MTY4OTY5MDQyOC4zNDEuMC4xNjg5NjkwNDI4LjAuMC4w) and also join our community Slack to connect with other Airbyte users. + +### Community Slack + +**Join our Slack community** [HERE](https://slack.airbyte.com/?_gl=1*1h8mjfe*_gcl_au*MTc4MjAxMDQzOS4xNjgyOTczMDYy*_ga*MTc1OTkyOTYzNi4xNjQxMjQyMjA0*_ga_HDBMVFQGBH*MTY4Nzg4OTQ4MC4zMjUuMS4xNjg3ODkwMjE1LjAuMC4w&_ga=2.58571491.813788522.1687789276-1759929636.1641242204)! + +Ask your questions first in the #ask-ai channel and if our bot can not assist you, reach out to our community in the #ask-community-for-troubleshooting channel. + +If you require personalized support, reach out to our sales team to inquire about [Airbyte Enterprise](https://airbyte.com/airbyte-enterprise). + +### Airbyte Forum + +We are driving our community support from our [forum](https://github.com/airbytehq/airbyte/discussions) on GitHub. + + +## Airbyte Cloud Support + +If you have questions about connector setup, error resolution, or want to report a bug, Airbyte Support is available to assist you. We recommend checking [our documentation](https://docs.airbyte.com/) and searching our [Help Center](https://support.airbyte.com/hc/en-us) before opening a support ticket. + +If you couldn't find the information you need in our docs or Help Center, open a ticket within the Airbyte Cloud platform by selecting the "Support" icon in the lower left navigation bar. Alternatively, you can submit a ticket through our [Help Center](https://support.airbyte.com/hc/en-us) by completing an Airbyte Cloud Support Request. Our team is online and availible to assist from 7AM - 7PM Eastern. + +**If you're unsure about the supported connectors, refer to our [Connector Support Levels](https://docs.airbyte.com/project-overview/product-support-levels/) & [Connector Catalog](https://docs.airbyte.com/integrations/).** + +For account or credit-related inquiries, contact our [sales team](https://airbyte.com/talk-to-sales). + +If you don't see a connector you need, you can submit a [connector request](https://airbyte.com/connector-requests). + +To stay updated on Airbyte's future plans, take a look at [our roadmap](https://github.com/orgs/airbytehq/projects/37/views/1). + +Please be sure to sign up for Airbyte with your company email address, as we do not support personal accounts. + +## Airbyte Enterprise (self-hosted) Support + +If you're running Airbyte Open Source with Airbyte Enterprise or have an OSS support package, we're here to help you with upgrading Airbyte versions, debugging connector issues, or troubleshooting schema changes. + +Before opening a support ticket, we recommend consulting [our documentation](https://docs.airbyte.com/) and searching our [Help Center](https://support.airbyte.com/hc/en-us). If your question remains unanswered, please submit a ticket through our Help Center. We suggest creating an [Airbyte Help Center account](https://airbyte1416.zendesk.com/auth/v2/login/signin?return_to=https%3A%2F%2Fsupport.airbyte.com%2Fhc%2Fen-us&theme=hc&locale=en-us&brand_id=15365055240347&auth_origin=15365055240347%2Ctrue%2Ctrue) to access your organization's support requests. Our team is online and availible to assist from 7AM - 7PM Eastern. + +**Connector support is based on certification status of the connector.** Please see our [Connector Support Levels](https://docs.airbyte.com/project-overview/product-support-levels) if you have any questions on support provided for one of your connectors. + +Submitting a Pull Request for review? + +* Be sure to follow our [contribution guidelines](https://docs.airbyte.com/contributing-to-airbyte/) laid out here on our doc. Highlights include: + * PRs should be limited to a single change-set +* Submit the PR as a PR Request through the Help Center Open Source Enterprise Support Request form +* If you are submitting a Platform PR we accept Platform PRs in the areas below: + * Helm + * Environment variable configurations + * Bug Fixes + * Security version bumps + * **If outside these areas, please open up an issue to help the team understand the need and if we are able to consider a PR** + +Submitting a PR does not guarantee its merge. The Airbyte support team will conduct an initial review, and if the PR aligns with Airbyte's roadmap, it will be prioritized based on team capacities and priorities. + +Although we strive to offer our utmost assistance, there are certain requests that we are unable to support. Currently, we do not provide assistance for these particular items: +* Question/troubleshooting assistance with forked versions of Airbyte +* Configuring using Octavia CLI +* Creating and configuring custom transformation using dbt +* Curating unique documentation and training materials +* Configuring Airbyte to meet security requirements + +If you think you will need assistance when upgrading, we recommend upgrading during our support hours, Monday-Friday 7AM - 7PM ET so we can assist if support is needed. If you upgrade outside of support hours, please submit a ticket and we will assist when we are back online. diff --git a/docs/connector-development/README.md b/docs/connector-development/README.md index 9a011cb3f4e3..d33f9d148df5 100644 --- a/docs/connector-development/README.md +++ b/docs/connector-development/README.md @@ -14,7 +14,6 @@ If you need help from our team for connector development, we offer premium suppo The [connector builder UI](connector-builder-ui/overview.md) is based on the low-code development framework below and allows to develop and use connectors without leaving the Airbyte UI (no local development environment required). - ### Low-code Connector-Development Framework You can use the [low-code framework](config-based/low-code-cdk-overview.md) to build source connectors for REST APIs by modifying boilerplate YAML files. @@ -24,10 +23,11 @@ You can use the [low-code framework](config-based/low-code-cdk-overview.md) to b You can build a connector very quickly in Python with the [Airbyte CDK](cdk-python/), which generates 75% of the code required for you. ### Community maintained CDKs + The Airbyte community also maintains some CDKs: -* The [Typescript CDK](https://github.com/faros-ai/airbyte-connectors) is actively maintained by Faros.ai for use in their product. -* The [Airbyte Dotnet CDK](cdk-dotnet/) comes with C# templates which can be used to generate 75% of the code required for you +- The [Typescript CDK](https://github.com/faros-ai/airbyte-connectors) is actively maintained by Faros.ai for use in their product. +- The [Airbyte Dotnet CDK](cdk-dotnet/) comes with C# templates which can be used to generate 75% of the code required for you ## The Airbyte specification @@ -52,14 +52,14 @@ If you are building a connector in any of the following languages/frameworks, th #### Sources -* **Python Source Connector** -* [**Singer**](https://singer.io)**-based Python Source Connector**. [Singer.io](https://singer.io/) is an open source framework with a large community and many available connectors \(known as taps & targets\). To build an Airbyte connector from a Singer tap, wrap the tap in a thin Python package to make it Airbyte Protocol-compatible. See the [Github Connector](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-github-singer) for an example of an Airbyte Connector implemented on top of a Singer tap. -* **Generic Connector**: This template provides a basic starting point for any language. +- **Python Source Connector** +- [**Singer**](https://singer.io)**-based Python Source Connector**. [Singer.io](https://singer.io/) is an open source framework with a large community and many available connectors \(known as taps & targets\). To build an Airbyte connector from a Singer tap, wrap the tap in a thin Python package to make it Airbyte Protocol-compatible. See the [Github Connector](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-github) for an example of an Airbyte Connector implemented on top of a Singer tap. +- **Generic Connector**: This template provides a basic starting point for any language. #### Destinations -* **Java Destination Connector** -* **Python Destination Connector** +- **Java Destination Connector** +- **Python Destination Connector** #### Creating a connector from a template @@ -74,11 +74,10 @@ and choose the relevant template by using the arrow keys. This will generate a n Search the generated directory for "TODO"s and follow them to implement your connector. For more detailed walkthroughs and instructions, follow the relevant tutorial: -* [Speedrun: Building a HTTP source with the CDK](tutorials/cdk-speedrun.md) -* [Building a HTTP source with the CDK](tutorials/cdk-tutorial-python-http/getting-started.md) -* [Building a Python source](tutorials/building-a-python-source.md) -* [Building a Python destination](tutorials/building-a-python-destination.md) -* [Building a Java destination](tutorials/building-a-java-destination.md) +- [Speedrun: Building a HTTP source with the CDK](tutorials/cdk-speedrun.md) +- [Building a HTTP source with the CDK](tutorials/cdk-tutorial-python-http/getting-started.md) +- [Building a Python source](tutorials/building-a-python-source.md) +- [Building a Java destination](tutorials/building-a-java-destination.md) As you implement your connector, make sure to review the [Best Practices for Connector Development](best-practices.md) guide. Following best practices is not a requirement for merging your contribution to Airbyte, but it certainly doesn't hurt ;\) @@ -117,30 +116,17 @@ The steps for updating an existing connector are the same as for building a new 3. Add any needed docs updates 4. Create a PR to get the connector published -## Adding normalization to a connector - -In order to enable normalization for a destination connector, you'll need to set some fields on the destination definitions entry for the connector. This is done in the [metadata.yaml](connector-metadata-file.md) file found at the root of each connector. - -Here's an example of normalization fields being set to enable normalization for the Postgres destination: +## Adding Typing and Deduplication to a connector -```yaml -data: - # ... other fields - normalizationConfig: - normalizationRepository: airbyte/normalization - normalizationTag: 0.2.25 - normalizationIntegrationType: postgres -``` - -For more information about what these fields mean, see the [NormalizationDestinationDefinitionConfig](https://github.com/airbytehq/airbyte/blob/master/airbyte-config-oss/config-models-oss/src/main/resources/types/NormalizationDestinationDefinitionConfig.yaml) schema. +_Coming soon._ -The presence of these fields will enable normalization for the connector, and determine which docker image will run. +Typing and Deduplication is how Airbyte transforms the raw data which is transmitted during a sync into easy-to-use final tables for database and data warehouse destinations. For more information on how typing and deduplication works, see [this doc](/understanding-airbyte/typing-deduping). ## Publishing a connector Once you've finished iterating on the changes to a connector as specified in its `README.md`, follow these instructions to ship the new version of the connector with Airbyte out of the box. -1. Bump the version in the `Dockerfile` of the connector \(`LABEL io.airbyte.version=X.X.X`\). +1. Bump the version in the `Dockerfile` of the connector \(`LABEL io.airbyte.version=X.X.X`\). 2. Bump the docker image version in the [metadata.yaml](connector-metadata-file.md) of the connector. 3. Submit a PR containing the changes you made. 4. One of Airbyte maintainers will review the change in the new version and make sure the tests are passing. @@ -150,10 +136,11 @@ Once you've finished iterating on the changes to a connector as specified in its ### Updating Connector Metadata When a new (or updated version) of a connector is ready, our automations will check your branch for a few things: -* Does the connector have an icon? -* Does the connector have documentation and is it in the proper format? -* Does the connector have a changelog entry for this version? -* The [metadata.yaml](connector-metadata-file.md) file is valid. + +- Does the connector have an icon? +- Does the connector have documentation and is it in the proper format? +- Does the connector have a changelog entry for this version? +- The [metadata.yaml](connector-metadata-file.md) file is valid. If any of the above are failing, you won't be able to merge your PR or publish your connector. @@ -161,23 +148,25 @@ Connector icons should be square SVGs and be located in [this directory](https:/ Connector documentation and changelogs are markdown files living either [here for sources](https://github.com/airbytehq/airbyte/tree/master/docs/integrations/sources), or [here for destinations](https://github.com/airbytehq/airbyte/tree/master/docs/integrations/destinations). - ## Using credentials in CI In order to run integration tests in CI, you'll often need to inject credentials into CI. There are a few steps for doing this: + 1. **Place the credentials into Google Secret Manager(GSM)**: Airbyte uses a project 'Google Secret Manager' service as the source of truth for all CI secrets. Place the credentials **exactly as they should be used by the connector** into a GSM secret [here](https://console.cloud.google.com/security/secret-manager?referrer=search&orgonly=true&project=dataline-integration-testing&supportedpurview=organizationId) i.e.: it should basically be a copy paste of the `config.json` passed into a connector via the `--config` flag. We use the following naming pattern: `SECRET__CREDS` e.g: `SECRET_SOURCE-S3_CREDS` or `SECRET_DESTINATION-SNOWFLAKE_CREDS`. 2. **Add the GSM secret's labels**: - * `connector` (required) -- unique connector's name or set of connectors' names with '_' as delimiter i.e.: `connector=source-s3`, `connector=destination-snowflake` - * `filename` (optional) -- custom target secret file. Unfortunately Google doesn't use '.' into labels' values and so Airbyte CI scripts will add '.json' to the end automatically. By default secrets will be saved to `./secrets/config.json` i.e: `filename=config_auth` => `secrets/config_auth.json` + - `connector` (required) -- unique connector's name or set of connectors' names with '\_' as delimiter i.e.: `connector=source-s3`, `connector=destination-snowflake` + - `filename` (optional) -- custom target secret file. Unfortunately Google doesn't use '.' into labels' values and so Airbyte CI scripts will add '.json' to the end automatically. By default secrets will be saved to `./secrets/config.json` i.e: `filename=config_auth` => `secrets/config_auth.json` 3. **Save a necessary JSON value** [Example](https://user-images.githubusercontent.com/11213273/146040653-4a76c371-a00e-41fe-8300-cbd411f10b2e.png). 4. That should be it. #### Access CI secrets on GSM + Access to GSM storage is limited to Airbyte employees. To give an employee permissions to the project: + 1. Go to the permissions' [page](https://console.cloud.google.com/iam-admin/iam?project=dataline-integration-testing) 2. Add a new principal to `dataline-integration-testing`: + - input their login email - select the role `Development_CI_Secrets` -3. Save - +3. Save diff --git a/docs/connector-development/cdk-python/README.md b/docs/connector-development/cdk-python/README.md index 5e2c63af32e2..76872e0186c6 100644 --- a/docs/connector-development/cdk-python/README.md +++ b/docs/connector-development/cdk-python/README.md @@ -1,35 +1,54 @@ # Connector Development Kit :::info -Developer updates will be announced via our #help-connector-development Slack channel. If you are using the CDK, please join to stay up to date on changes and issues. +Over the next few months, the project will only accept connector contributions that are made using the +[Low-Code CDK](https://docs.airbyte.com/connector-development/config-based/low-code-cdk-overview) or the +[Connector Builder](https://docs.airbyte.com/connector-development/connector-builder-ui/overview). + +New pull requests made with the Python CDK will be closed, but we will inquire to understand why it wasn't done with +Low-Code/Connector Builder so we can address missing features. This decision is aimed at improving maintenance and +providing a larger catalog with high-quality connectors. + +You can continue to use the Python CDK to build connectors to help your company or projects. +::: + +:::info +Developer updates will be announced via +[#help-connector-development](https://airbytehq.slack.com/archives/C027KKE4BCZ) Slack channel. If you are using the +CDK, please join to stay up to date on changes and issues. ::: :::info -This section is for the Python CDK. See our [community-maintained CDKs section](../README.md#community-maintained-cdks) -if you want to write connectors in other languages. +This section is for the Python CDK. See our +[community-maintained CDKs section](../README.md#community-maintained-cdks) if you want to write connectors in other +languages. ::: -The Airbyte Python CDK is a framework for rapidly developing production-grade Airbyte connectors. The CDK currently offers helpers specific for creating Airbyte source connectors for: + +The Airbyte Python CDK is a framework for rapidly developing production-grade Airbyte connectors. The CDK currently +offers helpers specific for creating Airbyte source connectors for: - HTTP APIs \(REST APIs, GraphQL, etc..\) - Generic Python sources \(anything not covered by the above\) -- Singer Taps (Note: The CDK supports building Singer taps but Airbyte no longer access contributions of this type) - -The CDK provides an improved developer experience by providing basic implementation structure and abstracting away low-level glue boilerplate. -This document is a general introduction to the CDK. Readers should have basic familiarity with the [Airbyte Specification](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/) before proceeding. +This document is a general introduction to the CDK. Readers should have basic familiarity with the +[Airbyte Specification](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/) before proceeding. -If you have any issues with troubleshooting or want to learn more about the CDK from the Airbyte team, head to [the Connector Development section of our Airbyte Forum](https://github.com/airbytehq/airbyte/discussions) to inquire further! +If you have any issues with troubleshooting or want to learn more about the CDK from the Airbyte team, head to +[the Connector Development section of our Airbyte Forum](https://github.com/airbytehq/airbyte/discussions) to +inquire further! ## Getting Started -Generate an empty connector using the code generator. First clone the Airbyte repository then from the repository root run +Generate an empty connector using the code generator. First clone the Airbyte repository, then from the repository +root run -```text +```bash cd airbyte-integrations/connector-templates/generator ./generate.sh ``` -then follow the interactive prompt. Next, find all `TODO`s in the generated project directory -- they're accompanied by lots of comments explaining what you'll need to do in order to implement your connector. Upon completing all TODOs properly, you should have a functioning connector. +Next, find all `TODO`s in the generated project directory. They're accompanied by comments explaining what you'll +need to do in order to implement your connector. Upon completing all TODOs properly, you should have a functioning connector. Additionally, you can follow [this tutorial](../tutorials/cdk-tutorial-python-http/getting-started.md) for a complete walkthrough of creating an HTTP connector using the Airbyte CDK. @@ -59,24 +78,23 @@ You can find a complete tutorial for implementing an HTTP source connector in [t **HTTP Connectors**: -- [Exchangerates API](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-exchange-rates/source_exchange_rates/source.py) - [Stripe](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py) - [Slack](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-slack/source_slack/source.py) **Simple Python connectors using the barebones `Source` abstraction**: -- [Google Sheets](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/google_sheets_source.py) +- [Google Sheets](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-google-sheets/source_google_sheets/source.py) - [Mailchimp](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-mailchimp/source_mailchimp/source.py) ## Contributing ### First time setup -We assume `python` points to python >=3.9. +We assume `python` points to Python 3.9 or higher. Setup a virtual env: -```text +```bash python -m venv .venv source .venv/bin/activate pip install -e ".[tests]" # [tests] installs test-only dependencies @@ -93,7 +111,7 @@ pip install -e ".[tests]" # [tests] installs test-only dependencies While developing your connector, you can print detailed debug information during a sync by specifying the `--debug` flag. This allows you to get a better picture of what is happening during each step of your sync. -```text +```bash python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json --debug ``` @@ -111,11 +129,3 @@ All tests are located in the `unit_tests` directory. Run `pytest --cov=airbyte_c 1. Open a PR 2. Once it is approved and merge, an Airbyte member must run the `Publish CDK Manually` workflow using `release-type=major|manor|patch` and setting the changelog message. - -## Coming Soon - -- Full OAuth 2.0 support \(including refresh token issuing flow via UI or CLI\) -- Airbyte Java HTTP CDK -- CDK for Async HTTP endpoints \(request-poll-wait style endpoints\) -- CDK for other protocols -- Don't see a feature you need? [Create an issue and let us know how we can help!](https://github.com/airbytehq/airbyte/issues/new?assignees=&labels=type%2Fenhancement&template=feature-request.md&title=) diff --git a/docs/connector-development/config-based/advanced-topics.md b/docs/connector-development/config-based/advanced-topics.md index 8914a1260540..cd9b70f4549a 100644 --- a/docs/connector-development/config-based/advanced-topics.md +++ b/docs/connector-development/config-based/advanced-topics.md @@ -105,7 +105,7 @@ In this example, outer.inner.k2 will evaluate to "MyKey is MyValue" Strings can contain references to previously defined values. The parser will dereference these values to produce a complete object definition. -References can be defined using a "#/{arg}" string. +References can be defined using a `#/{arg}` string. ```yaml key: 1234 diff --git a/docs/connector-development/config-based/low-code-cdk-overview.md b/docs/connector-development/config-based/low-code-cdk-overview.md index a012de473c24..da246f031775 100644 --- a/docs/connector-development/config-based/low-code-cdk-overview.md +++ b/docs/connector-development/config-based/low-code-cdk-overview.md @@ -7,7 +7,7 @@ Developer updates will be announced via our #help-connector-development Slack ch ::: :::note -The low-code framework is in [beta](https://docs.airbyte.com/project-overview/product-release-stages/#beta), which means that while it will be backwards compatible, it’s still in active development. Share feedback and requests with us on our [Slack channel](https://slack.airbyte.com/) or email us at [feedback@airbyte.io](mailto:feedback@airbyte.io) +The low-code framework is in **beta**, which means that while it will be backwards compatible, it’s still in active development. Share feedback and requests with us on our [Slack channel](https://slack.airbyte.com/) or email us at [feedback@airbyte.io](mailto:feedback@airbyte.io) ::: ## Why low-code? @@ -138,7 +138,7 @@ For a deep dive into each of the components, refer to [Understanding the YAML fi ## Tutorial -This section a tutorial that will guide you through the end-to-end process of implementing a low-code connector. +This section is a tutorial that will guide you through the end-to-end process of implementing a low-code connector. 0. [Getting started](tutorial/0-getting-started.md) 1. [Creating a source](tutorial/1-create-source.md) @@ -155,3 +155,6 @@ For examples of production-ready config-based connectors, refer to: - [Greenhouse](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/manifest.yaml) - [Sendgrid](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-sendgrid/source_sendgrid/manifest.yaml) - [Sentry](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-sentry/source_sentry/manifest.yaml) + +## Reference +The full schema definition for the YAML file can be found [here](https://raw.githubusercontent.com/airbytehq/airbyte/master/airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_component_schema.yaml). diff --git a/docs/connector-development/config-based/tutorial/0-getting-started.md b/docs/connector-development/config-based/tutorial/0-getting-started.md index da65aad47914..5a8a940c2973 100644 --- a/docs/connector-development/config-based/tutorial/0-getting-started.md +++ b/docs/connector-development/config-based/tutorial/0-getting-started.md @@ -1,6 +1,6 @@ # Getting Started -:warning: This framework is in [alpha](https://docs.airbyte.com/project-overview/product-release-stages/#alpha). It is still in active development and may include backward-incompatible changes. Please share feedback and requests directly with us at feedback@airbyte.io :warning: +:warning: This framework is in **alpha**. It is still in active development and may include backward-incompatible changes. Please share feedback and requests directly with us at feedback@airbyte.io :warning: ## Summary @@ -31,12 +31,12 @@ The output schema of our stream will look like the following: ## Exchange Rates API Setup Before we get started, you'll need to generate an API access key for the Exchange Rates API. -This can be done by signing up for the Free tier plan on [Exchange Rates API](https://exchangeratesapi.io/): +This can be done by signing up for the Free tier plan on [Exchange Rates Data API](https://apilayer.com/marketplace/exchangerates_data-api), not [Exchange Rates API](https://exchangeratesapi.io/): -1. Visit https://exchangeratesapi.io and click "Get free API key" on the top right -2. You'll be taken to https://apilayer.com -- finish the sign up process, signing up for the free tier -3. Once you're signed in, visit https://apilayer.com/marketplace/exchangerates_data-api#documentation-tab and click "Live Demo" -4. Inside that editor, you'll see an API key. This is your API key. +1. Visit https://apilayer.com/ and click "Sign In" on the top +2. Finish the sign up process, signing up for the free tier +3. Once you're signed in, visit https://apilayer.com/marketplace/exchangerates_data-api and click "Subscribe" for free +4. On the top right, you'll see an API key. This is your API key. ## Requirements @@ -44,6 +44,7 @@ This can be done by signing up for the Free tier plan on [Exchange Rates API](ht - Python >= 3.9 - Docker must be running - NodeJS +- [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1) CLI ## Next Steps diff --git a/docs/connector-development/config-based/tutorial/6-testing.md b/docs/connector-development/config-based/tutorial/6-testing.md index 5aaa3ef2814e..4bbb90e8ed01 100644 --- a/docs/connector-development/config-based/tutorial/6-testing.md +++ b/docs/connector-development/config-based/tutorial/6-testing.md @@ -27,11 +27,10 @@ and `integration_tests/abnormal_state.json` with } ``` -You can run the acceptance tests with the following commands: +You can run the [acceptance tests](https://github.com/airbytehq/airbyte/blob/master/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md#L1) with the following commands using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1): ```bash -docker build . -t airbyte/source-exchange-rates-tutorial:dev -python -m pytest integration_tests -p integration_tests.acceptance +airbyte-ci connectors --use-remote-secrets=false --name source-exchange-rates-tutorial test ``` ## Next steps: diff --git a/docs/connector-development/config-based/understanding-the-yaml-file/authentication.md b/docs/connector-development/config-based/understanding-the-yaml-file/authentication.md index 477d8ea3e782..1fdb87165bed 100644 --- a/docs/connector-development/config-based/understanding-the-yaml-file/authentication.md +++ b/docs/connector-development/config-based/understanding-the-yaml-file/authentication.md @@ -51,7 +51,7 @@ authenticator: ### BearerAuthenticator -The `BearerAuthenticator` is a specialized `ApiKeyAuthenticator` that always sets the header "Authorization" with the value "Bearer {token}". +The `BearerAuthenticator` is a specialized `ApiKeyAuthenticator` that always sets the header "Authorization" with the value `Bearer {token}`. The following definition will set the header "Authorization" with a value "Bearer hello" Schema: @@ -82,7 +82,7 @@ More information on bearer authentication can be found [here](https://swagger.io ### BasicHttpAuthenticator The `BasicHttpAuthenticator` set the "Authorization" header with a (USER ID/password) pair, encoded using base64 as per [RFC 7617](https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication#basic_authentication_scheme). -The following definition will set the header "Authorization" with a value "Basic {encoded credentials}" +The following definition will set the header "Authorization" with a value `Basic {encoded credentials}` Schema: diff --git a/docs/connector-development/connector-builder-ui/authentication.md b/docs/connector-development/connector-builder-ui/authentication.md index 0a6cb8b67763..b57aa440cc89 100644 --- a/docs/connector-development/connector-builder-ui/authentication.md +++ b/docs/connector-development/connector-builder-ui/authentication.md @@ -90,7 +90,7 @@ In this case the injection mechanism is `header` and the field name is `X-CoinAP ### OAuth -The OAuth authentication method implements authentication using an OAuth2.0 flow with a [refresh token grant type](https://oauth.net/2/grant-types/refresh-token/) and [client credentiuals grant type](https://oauth.net/2/grant-types/client-credentials/). +The OAuth authentication method implements authentication using an OAuth2.0 flow with a [refresh token grant type](https://oauth.net/2/grant-types/refresh-token/) and [client credentials grant type](https://oauth.net/2/grant-types/client-credentials/). In this scheme, the OAuth endpoint of an API is called with client id and client secret and/or a long-lived refresh token that's provided by the end user when configuring this connector as a Source. These credentials are used to obtain a short-lived access token that's used to make requests actually extracting records. If the access token expires, the connection will automatically request a new one. diff --git a/docs/connector-development/connector-builder-ui/incremental-sync.md b/docs/connector-development/connector-builder-ui/incremental-sync.md index c6f780fe12cc..4b05f8d48ba9 100644 --- a/docs/connector-development/connector-builder-ui/incremental-sync.md +++ b/docs/connector-development/connector-builder-ui/incremental-sync.md @@ -11,9 +11,8 @@ To use incremental syncs, the API endpoint needs to fullfil the following requir - Records contain a top-level date/time field that defines when this record was last updated (the "cursor field") - If the record's cursor field is nested, you can use an "Add Field" transformation to copy it to the top-level, and a Remove Field to remove it from the object. This will effectively move the field to the top-level of the record - It's possible to filter/request records by the cursor field -- The records are sorted in ascending order based on their cursor field -The knowledge of a cursor value also allows the Airbyte system to automatically keep a history of changes to records in the destination. To learn more about how different modes of incremental syncs, check out the [Incremental Sync - Append](/understanding-airbyte/connections/incremental-append/) and [Incremental Sync - Append + Deduped](/understanding-airbyte/connections/incremental-append-deduped) pages. +The knowledge of a cursor value also allows the Airbyte system to automatically keep a history of changes to records in the destination. To learn more about how different modes of incremental syncs, check out the [Incremental Sync - Append](/using-airbyte/core-concepts/sync-modes/incremental-append/) and [Incremental Sync - Append + Deduped](/using-airbyte/core-concepts/sync-modes/incremental-append-deduped) pages. ## Configuration @@ -59,19 +58,13 @@ As this fulfills the requirements for incremental syncs, we can configure the "I -This API orders records by default from new to old, which is not optimal for a reliable sync as the last encountered cursor value will be the most recent date even if some older records did not get synced (for example if a sync fails halfway through). It's better to start with the oldest records and work your way up to make sure that all older records are synced already once a certain date is encountered on a record. In this case the API can be configured to behave like this by setting an additional parameter: - -- Add a new "Query Parameter" near the top of the page -- Set the key to `order-by` -- Set the value to `oldest` - Setting the start date in the "Testing values" to a date in the past like **2023-04-09T00:00:00Z** results in the following request:
      -curl 'https://content.guardianapis.com/search?order-by=oldest&from-date=2023-04-09T00:00:00Z&to-date={`now`}'
      +curl 'https://content.guardianapis.com/search?from-date=2023-04-09T00:00:00Z&to-date={`now`}'
       
      -The last encountered date will be saved as part of the connection - when the next sync is running, it picks up from the last record. Let's assume the last ecountered article looked like this: +The most recent encountered date will be saved as part of the connection - when the next sync is running, it picks up from that date as the new start date. Let's assume the last ecountered article looked like this:
       {`{
      @@ -86,13 +79,17 @@ The last encountered date will be saved as part of the connection - when the nex
       Then when a sync is triggered for the same connection the next day, the following request is made:
       
       
      -curl 'https://content.guardianapis.com/search?order-by=oldest&from-date=2023-04-15T07:30:58Z&to-date={``}'
      +curl 'https://content.guardianapis.com/search?from-date=2023-04-15T07:30:58Z&to-date={``}'
       
      +:::info +If the last record read has a datetime earlier than the end time of the stream interval, the end time of the interval will be stored in the state. +::: + The `from-date` is set to the cutoff date of articles synced already and the `to-date` is set to the current date. :::info -In some cases, it's helpful to reference the start and end date of the interval that's currently synced, for example if it needs to be injected into the URL path of the current stream. In these cases it can be referenced using the `{{ stream_interval.start_date }}` and `{{ stream_interval.end_date }}` [placeholders](/connector-development/config-based/understanding-the-yaml-file/reference#variables). Check out [the tutorial](./tutorial.mdx#adding-incremental-reads) for such a case. +In some cases, it's helpful to reference the start and end date of the interval that's currently synced, for example if it needs to be injected into the URL path of the current stream. In these cases it can be referenced using the `{{ stream_interval.start_time }}` and `{{ stream_interval.end_time }}` [placeholders](/connector-development/config-based/understanding-the-yaml-file/reference#variables). Check out [the tutorial](./tutorial.mdx#adding-incremental-reads) for such a case. ::: ## Incremental sync without time filtering @@ -118,9 +115,9 @@ The "cursor granularity" also needs to be set to an ISO 8601 duration - it repre For example if the "Step" is set to 10 days (`P10D`) and the "Cursor granularity" set to second (`PT1S`) for the Guardian articles stream described above and a longer time range, then the following requests will be performed:
      -curl 'https://content.guardianapis.com/search?order-by=oldest&from-date=2023-01-01T00:00:00Z&to-date=2023-01-10T00:00:00Z'{`\n`}
      -curl 'https://content.guardianapis.com/search?order-by=oldest&from-date=2023-01-10T00:00:00Z&to-date=2023-01-20T00:00:00Z'{`\n`}
      -curl 'https://content.guardianapis.com/search?order-by=oldest&from-date=2023-01-20T00:00:00Z&to-date=2023-01-30T00:00:00Z'{`\n`}
      +curl 'https://content.guardianapis.com/search?from-date=2023-01-01T00:00:00Z&to-date=2023-01-10T00:00:00Z'{`\n`}
      +curl 'https://content.guardianapis.com/search?from-date=2023-01-10T00:00:00Z&to-date=2023-01-20T00:00:00Z'{`\n`}
      +curl 'https://content.guardianapis.com/search?from-date=2023-01-20T00:00:00Z&to-date=2023-01-30T00:00:00Z'{`\n`}
       ...
       
      @@ -139,7 +136,7 @@ Some APIs update records over time but do not allow to filter or search by modif In these cases, there are two options: -- **Do not use incremental sync** and always sync the full set of records to always have a consistent state, losing the advantages of reduced load and [automatic history keeping in the destination](/understanding-airbyte/connections/incremental-append-deduped) +- **Do not use incremental sync** and always sync the full set of records to always have a consistent state, losing the advantages of reduced load and [automatic history keeping in the destination](/using-airbyte/core-concepts/sync-modes/incremental-append-deduped) - **Configure the "Lookback window"** to not only sync exclusively new records, but resync some portion of records before the cutoff date to catch changes that were made to existing records, trading off data consistency and the amount of synced records. In the case of the API of The Guardian, news articles tend to only be updated for a few days after the initial release date, so this strategy should be able to catch most updates without having to resync all articles. Reiterating the example from above with a "Lookback window" of 2 days configured, let's assume the last encountered article looked like this: @@ -157,7 +154,7 @@ Reiterating the example from above with a "Lookback window" of 2 days configured Then when a sync is triggered for the same connection the next day, the following request is made:
      -curl 'https://content.guardianapis.com/search?order-by=oldest&from-date=2023-04-13T07:30:58Z&to-date={``}'
      +curl 'https://content.guardianapis.com/search?from-date=2023-04-13T07:30:58Z&to-date={``}'
       
      ## Custom parameter injection diff --git a/docs/connector-development/connector-builder-ui/overview.md b/docs/connector-development/connector-builder-ui/overview.md index f3d2bfd43ddf..81141aba9f61 100644 --- a/docs/connector-development/connector-builder-ui/overview.md +++ b/docs/connector-development/connector-builder-ui/overview.md @@ -1,6 +1,6 @@ # Connector Builder Intro -The connector builder UI provides an intuitive UI on top of the [low-code YAML format](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/yaml-overview) and to use built connectors for syncs within the same workspace directly from within the UI. We recommend using it to iterate on your low-code connectors. +The connector builder UI provides an intuitive UI on top of the [low-code YAML format](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/yaml-overview) and uses built connectors for syncs within the same workspace directly from within the UI. We recommend using it to iterate on your low-code connectors. :::note The connector builder UI is in beta, which means it’s still in active development and may include backward-incompatible changes. Share feedback and requests with us on our Slack channel or email us at feedback@airbyte.io @@ -21,27 +21,27 @@ The connector builder is the right tool if the following points are met: The high level flow for using the connector builder is as follows: -1. Access the connector builder in the Airbyte webapp +1. Access the connector builder in the Airbyte web app 2. Use the connector builder to iterate on your low-code connector 3. Once the connector is ready, publish it to the local workspace 4. Configure a Source based on the released connector 5. Use the Source in a connection to sync data -Follow [the tutorial](./tutorial.mdx) for an example of this flow. The concept pages in the side bar to the left go into greater detail of more complex configurations. +Follow [the tutorial](./tutorial.mdx) for an example of this flow. The concept pages in the sidebar to the left go into greater detail of more complex configurations. ## Connector vs. configured source vs. connection -When building a connector, it's important to differentiate between the connector, the configured source based on a connector and the connection: +When building a connector, it's important to differentiate between the connector, the configured source based on a connector, and the connection: -The **connector** defines the functionality how to access an API or a database, for example protocol, URL paths to access, the way requests need to be structured and how to extract records from responses. +The **connector** defines the functionality of how to access an API or a database, for example protocol, URL paths to access, the way requests need to be structured, and how to extract records from responses. :::info While configuring a connector in the builder, make sure to not hardcode things like API keys or passwords - these should be passed in as user input when configuring a Source based on your connector. -Follow [the tutorial](./tutorial.mdx) for an example how this looks like in practice. +Follow [the tutorial](./tutorial.mdx) for an example of how this looks like in practice. ::: -The **configured source** is configuring a connector to actually extract records. The exact fields of the configuration depend on the connector, but in most cases it provides authentication information (username and password, api key) and information about which data to extract, for example start date to sync records from, a search query records have to match. +The **configured source** is configuring a connector to actually extract records. The exact fields of the configuration depend on the connector, but in most cases, it provides authentication information (username and password, API key) and information about which data to extract, for example, the start date to sync records from, a search query records have to match. The **connection** links up a configured source and a configured destination to perform syncs. It defines things like the replication frequency (e.g. hourly, daily, manually) and which streams to replicate. @@ -82,7 +82,7 @@ To do so, follow these steps: * Change and test the connector The following connectors are good showcases for real-world use cases: -* The [Pendo.io API](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-pendo/source_pendo/manifest.yaml) is a simple connector implementing multiple streams and api-key based authentication +* The [Pendo.io API](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-pendo/source_pendo/manifest.yaml) is a simple connector implementing multiple streams and API-key based authentication * The [News API](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-news-api/source_news_api/manifest.yaml) implements pagination and user-configurable request parameters * The [CoinGecko API](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-coingecko-coins/source_coingecko_coins/manifest.yaml) implements incremental syncs diff --git a/docs/connector-development/connector-builder-ui/partitioning.md b/docs/connector-development/connector-builder-ui/partitioning.md index 3ac5afcb862a..57dc0d5d9307 100644 --- a/docs/connector-development/connector-builder-ui/partitioning.md +++ b/docs/connector-development/connector-builder-ui/partitioning.md @@ -12,27 +12,29 @@ There are some cases that require multiple requests to fetch all records as well * If your records are spread out across multiple pages that need to be requested individually if there are too many records, use the Pagination feature. * If your records are spread out over time and multiple requests are necessary to fetch all data (for example one request per day), use the Incremental sync feature. -## Dynamic and static partition routing +## Dynamic and static partitioning -There are three possible sources for the partitions that need to be queried - the connector itself, supplied by the end user when configuring a Source based on the connector, or the API provides the list of partitions on another endpoint (for example the Woocommerce API also includes an `/orders` endpoint that returns all orders). +There are three possible sources for the partitions that need to be queried - the connector itself, supplied by the end user when configuring a Source based on the connector, or the API provides the list of partitions via another endpoint (for example the Woocommerce API also includes an `/orders` endpoint that returns all orders). -The first two options are a "static" form of partition routing (because the partitions won't change as long as the Airbyte configuration isn't changed). The API providing the partitions via one or multiple separate requests is a "dynamic" form of partition routing because the partitions can change any time. +The first two options are a "static" form of partition routing (because the partitions won't change as long as the Airbyte configuration isn't changed). This can be achieved by configuring the Parameterized Requests component in the Connector Builder. -### List partition router +The API providing the partitions via one or multiple separate requests is a "dynamic" form of partitioning because the partitions can change any time. This can be achieved by configuring the Parent Stream partition component in the Connector Builder. -To configure static partitioning, choose the "List" method for the partition router. The following fields have to be configured: -* The "partition values" can either be set to a list of strings, making the partitions part of the connector itself or delegated to a user input so the end user configuring a Source based on the connector can control which partitions to fetch. When using "user input" mode for the partition values, create a user input of type array and reference it as the value using the [placeholder](/connector-development/config-based/understanding-the-yaml-file/reference#variables) value using `{{ config[''] }}` -* The "Current partition value identifier" can be freely choosen and is the identifier of the variable holding the current partition value. It can for example be used in the path of the stream using the `{{ stream_partition. }}` syntax. -* The "Inject partition value into outgoing HTTP request" option allows you to configure how to add the current partition value to the requests +### Parameterized Requests + +To configure static partitioning, enable the `Parameterized Requests` component. The following fields have to be configured: +* The "Parameter Values" can either be set to a list of strings, making the partitions part of the connector itself, or delegated to a user input so the end user configuring a Source based on the connector can control which partitions to fetch. When using "user input" mode for the parameter values, create a user input of type array and reference it as the value using the [placeholder](/connector-development/config-based/understanding-the-yaml-file/reference#variables) value using `{{ config[''] }}` +* The "Current Parameter Value Identifier" can be freely choosen and is the identifier of the variable holding the current parameter value. It can for example be used in the path of the stream using the `{{ stream_partition. }}` syntax. +* The "Inject Parameter Value into outgoing HTTP Request" option allows you to configure how to add the current parameter value to the requests #### Example -To enable static partition routing defined as part of the connector for the [SurveySparrow API](https://developers.surveysparrow.com/rest-apis/response#getV3Responses) responses, the list partition router needs to be configured as following: -* "Partition values" are set to the list of survey ids to fetch -* "Current partition value identifier" is set to `survey` (this is not used for this example) -* "Inject partition value into outgoing HTTP request" is set to `request_parameter` for the field name `survey_id` +To enable static partitioning defined as part of the connector for the [SurveySparrow API](https://developers.surveysparrow.com/rest-apis/response#getV3Responses) responses, the Parameterized Requests component needs to be configured as following: +* "Parameter Values" are set to the list of survey ids to fetch +* "Current Parameter Value Identifier" is set to `survey` (this is not used for this example) +* "Inject Parameter Value into outgoing HTTP Request" is set to `request_parameter` for the field name `survey_id` -When partition values were set to `123`, `456` and `789`, the following requests will be executed: +When parameter values were set to `123`, `456` and `789`, the following requests will be executed: ``` curl -X GET https://api.surveysparrow.com/v3/responses?survey_id=123 curl -X GET https://api.surveysparrow.com/v3/responses?survey_id=456 @@ -40,39 +42,39 @@ curl -X GET https://api.surveysparrow.com/v3/responses?survey_id=789 ``` To enable user-configurable static partitions for the [Woocommerce API](https://woocommerce.github.io/woocommerce-rest-api-docs/#order-notes) order notes, the configuration would look like this: -* Set "Partition values" to "User input" -* In the "Value" input, click the blue user icon and create a new user input +* Set "Parameter Values" to "User input" +* In the "Value" input, click the user icon and create a new user input * Name it `Order IDs`, set type to `array` and click create -* Set "Current partition value identifier" to `order` -* "Inject partition value into outgoing HTTP request" is disabled, because the order id needs to be injected into the path +* Set "Current Parameter Value Identifier" to `order` +* "Inject Parameter Value into outgoing HTTP Request" is disabled, because the order id needs to be injected into the path * In the general section of the stream configuration, the "URL Path" is set to `/orders/{{ stream_partition.order }}/notes` - + -When order IDs were set to `123`, `456` and `789` in the testing values, the following requests will be executed: +When Order IDs were set to `123`, `456` and `789` in the testing values, the following requests will be executed: ``` curl -X GET https://example.com/wp-json/wc/v3/orders/123/notes curl -X GET https://example.com/wp-json/wc/v3/orders/456/notes curl -X GET https://example.com/wp-json/wc/v3/orders/789/notes ``` -### Substream partition router +### Parent Stream -To fetch the list of partitions (in this example surveys or orders) from the API itself, the "Substream" partition router has to be used. It allows you to select another stream of the same connector to serve as the source for partitions to fetch. Each record of the parent stream is used as a partition for the current stream. +To fetch the list of partitions (in this example surveys or orders) from the API itself, the "Parent Stream" component has to be used. It allows you to select another stream of the same connector to serve as the source for partitions to fetch. Each record of the parent stream is used as a partition for the current stream. -The following fields have to be configured to use the substream partition router: -* The "Parent stream" defines the records of which stream should be used as partitions -* The "Parent key" is the property on the parent stream record that should become the partition value (in most cases this is some form of id) -* The "Current partition value identifier" can be freely choosen and is the identifier of the variable holding the current partition value. It can for example be used in the path of the stream using the `{{ stream_partition. }}` [interpolation placeholder](/connector-development/config-based/understanding-the-yaml-file/reference#variables). +The following fields have to be configured to use the Parent Stream component: +* The "Parent Stream" defines the records of which stream should be used as partitions +* The "Parent Key" is the property on the parent stream record that should become the partition value (in most cases this is some form of id) +* The "Current Parent Key Value Identifier" can be freely choosen and is the identifier of the variable holding the current partition value. It can for example be used in the path of the stream using the `{{ stream_partition. }}` [interpolation placeholder](/connector-development/config-based/understanding-the-yaml-file/reference#variables). #### Example -To enable dynamic partition routing for the [Woocommerce API](https://woocommerce.github.io/woocommerce-rest-api-docs/#order-notes) order notes, first an "orders" stream needs to be configured for the `/orders` endpoint to fetch a list of orders. Once this is done, the partition router for responses has be configured like this: -* "Parent key" is set to `id` -* "Current partition value identifier" is set to `order` +To enable dynamic partitioning for the [Woocommerce API](https://woocommerce.github.io/woocommerce-rest-api-docs/#order-notes) order notes, first an orders stream needs to be configured for the `/orders` endpoint to fetch a list of orders. Once this is done, the Parent Stream component for the responses stream has be configured like this: +* "Parent Key" is set to `id` +* "Current Parent Key Value Identifier" is set to `order` * In the general section of the stream configuration, the "URL Path" is set to `/orders/{{ stream_partition.order }}/notes` - + When triggering a sync, the connector will first fetch all records of the orders stream. The records will look like this: ``` @@ -90,9 +92,9 @@ curl -X GET https://example.com/wp-json/wc/v3/orders/789/notes ## Multiple partition routers -It is possible to configure multiple partition routers on a single stream - if this is the case, all possible combinations of partition values are requested separately. +It is possible to configure multiple partitioning mechanisms on a single stream - if this is the case, all possible combinations of partition values are requested separately. -For example, the [Google Pagespeed API](https://developers.google.com/speed/docs/insights/v5/reference/pagespeedapi/runpagespeed) allows to specify the URL and the "strategy" to run an analysis for. To allow a user to trigger an analysis for multiple URLs and strategies at the same time, two list partition routers can be used (one injecting the partition value into the `url` parameter, one injecting it into the `strategy` parameter). +For example, the [Google Pagespeed API](https://developers.google.com/speed/docs/insights/v5/reference/pagespeedapi/runpagespeed) allows to specify the URL and the "strategy" to run an analysis for. To allow a user to trigger an analysis for multiple URLs and strategies at the same time, two Parameterized Request lists can be used (one injecting the parameter value into the `url` parameter, one injecting it into the `strategy` parameter). If a user configures the URLs `example.com` and `example.org` and the strategies `desktop` and `mobile`, then the following requests will be triggered ``` @@ -112,19 +114,20 @@ For example when fetching the order notes via the [Woocommerce API](https://wooc ``` However the order id can be added by taking the following steps: -* Making sure the "Current partition value identifier" is set to `order` +* Making sure the "Current Parameter Value Identifier" is set to `order` * Add an "Add field" transformation with "Path" `order_id` and "Value" `{{ stream_partition.order }}` Using this configuration, the notes record looks like this: ``` { "id": 999, "author": "Jon Doe", "note": "Great product!", "order_id": 123 } ``` + ## Custom parameter injection -Using the "Inject partition value into outgoing HTTP request" option in the partitioning form works for most cases, but sometimes the API has special requirements that can't be handled this way: +Using the "Inject Parameter / Parent Key Value into outgoing HTTP Request" option in the Parameterized Requests and Parent Stream components works for most cases, but sometimes the API has special requirements that can't be handled this way: * The API requires to add a prefix or a suffix to the actual value * Multiple values need to be put together in a single parameter * The value needs to be injected into the URL path * Some conditional logic needs to be applied -To handle these cases, disable injection in the partitioning form and use the generic parameter section at the bottom of the stream configuration form to freely configure query parameters, headers and properties of the JSON body, by using jinja expressions and [available variables](/connector-development/config-based/understanding-the-yaml-file/reference/#/variables). You can also use these variables (like `stream_partition`) as part of the URL path as shown in the Woocommerce example above. +To handle these cases, disable injection in the component and use the generic parameter section at the bottom of the stream configuration form to freely configure query parameters, headers and properties of the JSON body, by using jinja expressions and [available variables](/connector-development/config-based/understanding-the-yaml-file/reference/#/variables). You can also use these variables (like `stream_partition`) as part of the URL path as shown in the Woocommerce example above. diff --git a/docs/connector-development/connector-builder-ui/record-processing.mdx b/docs/connector-development/connector-builder-ui/record-processing.mdx index d5ac0dbb88de..41a57d2351a9 100644 --- a/docs/connector-development/connector-builder-ui/record-processing.mdx +++ b/docs/connector-development/connector-builder-ui/record-processing.mdx @@ -321,7 +321,7 @@ Besides bringing the records in the right shape, it's important to communicate s ### Primary key -The "Primary key" field specifies how to uniquely identify a record. This is important for downstream de-duplication of records (e.g. by the [incremental sync - Append + Deduped sync mode](/understanding-airbyte/connections/incremental-append-deduped)). +The "Primary key" field specifies how to uniquely identify a record. This is important for downstream de-duplication of records (e.g. by the [incremental sync - Append + Deduped sync mode](/using-airbyte/core-concepts/sync-modes/incremental-append-deduped)). In a lot of cases, like for the EmailOctopus example from above, there is a dedicated id field that can be used for this purpose. It's important that the value of the id field is guaranteed to only occur once for a single record. diff --git a/docs/connector-development/connector-builder-ui/tutorial.mdx b/docs/connector-development/connector-builder-ui/tutorial.mdx index 3be949c1c23f..d8b1b7b1c3a7 100644 --- a/docs/connector-development/connector-builder-ui/tutorial.mdx +++ b/docs/connector-development/connector-builder-ui/tutorial.mdx @@ -29,12 +29,11 @@ The output schema of our stream will look like the following: ### Setting up Exchange Rates API key Before we get started, you'll need to generate an API access key for the Exchange Rates API. -This can be done by signing up for the Free tier plan on [Exchange Rates API](https://exchangeratesapi.io/): +This can be done by signing up for the Free tier plan on [Exchange Rates API](https://apilayer.com/marketplace/exchangerates_data-api): -1. Visit https://exchangeratesapi.io and click "Get free API key" on the top right -2. You'll be taken to https://apilayer.com -- finish the sign up process, signing up for the free tier -3. Once you're signed in, visit https://apilayer.com/marketplace/exchangerates_data-api#documentation-tab and click "Live Demo" -4. Inside that editor, you'll see an API key. This is your API key. +1. Visit https://apilayer.com and click Sign In on the top right - either sign into an existing account or sign up for a new account on the free tier. +2. Once you're signed in, visit https://apilayer.com/marketplace/exchangerates_data-api +3. You should see an API Key displayed with a `Copy API Key` button next to it. This is your API key. ### Setting up the environment diff --git a/docs/connector-development/connector-metadata-file.md b/docs/connector-development/connector-metadata-file.md index bef8204a61a8..1b9fed5380b6 100644 --- a/docs/connector-development/connector-metadata-file.md +++ b/docs/connector-development/connector-metadata-file.md @@ -88,14 +88,14 @@ In the example above, the connector has three tags. Tags are used for two primar These are just examples of how tags can be used. As a free-form field, the tags list can be customized as required for each connector. This flexibility allows tags to be a powerful tool for managing and discovering connectors. ## The `icon` Field -This denotes the name of the icon file for the connector. At this time the icon file is located in the `platform-internal` repository. So please ensure that the icon file is present in the `platform-internal` repository at [oss/airbyte-config/init/src/main/resources/icons](https://github.com/airbytehq/airbyte-platform-internal/tree/master/oss/airbyte-config/init/src/main/resources/icons) before adding the icon name to the `metadata.yaml` file. +This denotes the name of the icon file for the connector. At this time the icon file is located in the `airbyte-platform` repository. So please ensure that the icon file is present in the `airbyte-platform` repository at [https://github.com/airbytehq/airbyte-platform/tree/main/airbyte-config/init/src/main/resources/icons](https://github.com/airbytehq/airbyte-platform/tree/main/airbyte-config/init/src/main/resources/icons) before adding the icon name to the `metadata.yaml` file. ### Future Plans _⚠️ This property is in the process of being refactored to be a file in the connector folder_ You may notice a `icon.svg` file in the connectors folder. -This is because we are transitioning away from icons being stored in the `platform-internal` repository. Instead, we will be storing them in the connector folder itself. This will allow us to have a single source of truth for all connector-related information. +This is because we are transitioning away from icons being stored in the `airbyte-platform` repository. Instead, we will be storing them in the connector folder itself. This will allow us to have a single source of truth for all connector-related information. This transition is currently in progress. Once it is complete, the `icon` field in the `metadata.yaml` file will be removed, and the `icon.svg` file will be used instead. @@ -108,13 +108,14 @@ The `releases` section contains extra information about certain types of release The `breakingChanges` section of `releases` contains a dictionary of version numbers (usually major versions, i.e. `1.0.0`) and information about their associated breaking changes. Each entry must contain the following parameters: * `message`: A description of the breaking change, written in a user-friendly format. This message should briefly describe - * What the breaking change is - * How it affects the user (or which users it will affect) - * What the user should do to fix the issue + * What the breaking change is, and which users it effects (e.g. all users of the source, or only those using a certain stream) + * Why the change is better for the user (fixed a bug, something got faster, etc) + * What the user should do to fix the issue (e.g. a full reset, run a SQL query in the destinaton, etc) * `upgradeDeadline`: (`YYYY-MM-DD`) The date by which the user should upgrade to the new version. -Note that the `message` should be brief no matter how involved the fix is - the user will be redirected to the migration documentation for the -full upgrade/migration instructions. +When considering what the `upgradeDeadline` should be, target the amount of time which would be reasonable for the user to make the required changes described in the `message` and upgrade giude. If the required changes are _simple_ (e.g. "do a full reset"), 2 weeks is recommended. Note that you do *not* want to link the duration of `upgradeDeadline` to an upstream API's deprecation date. While it is true that the older version of a connector will continue to work for that period of time, it means that users who are pinned to the older version of the connector will not benefit from future updates and fixes. + +Without all 3 of these points, the breaking change message is not helpful to users. Here is an example: ```yaml @@ -122,5 +123,47 @@ releases: breakingChanges: 1.0.0: message: "This version changes the connector’s authentication by removing ApiKey authentication, which is now deprecated by the [upstream source](upsteam-docs-url.com). Users currently using ApiKey auth will need to reauthenticate with OAuth after upgrading to continue syncing." - upgradeDeadline: "2023-12-31" # The date that the upstream API stops support for ApiKey authentication -``` \ No newline at end of file + upgradeDeadline: "2023-12-31" +``` + +#### `scopedImpact` +The optional `scopedImpact` property allows you to provide a list of scopes for which the change is breaking. +This allows you to reduce the scope of the change; it's assumed that any scopes not listed are unaffected by the breaking change. + +For example, consider the following `scopedImpact` definition: + +```yaml +releases: + breakingChanges: + 1.0.0: + message: "This version changes the cursor for the `Users` stream. After upgrading, please reset the stream." + upgradeDeadline: "2023-12-31" + scopedImpact: + - scopeType: stream + impactedScopes: ["users"] +``` + +This change only breaks the `users` stream - all other streams are unaffected. A user can safely ignore the breaking change +if they are not syncing the `users` stream. + +The supported scope types are listed below. + +| Scope Type | Value Type | Value Description | +|------------|------------|------------------| +| stream | `list[str]` | List of stream names | + +#### `remoteRegistries` +The optional `remoteRegistries` property allows you to configure how a connector should be published to registries like Pypi. + +**Important note**: Currently no automated publishing will occur. + +```yaml +remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-connector-name +``` + +The `packageName` property of the `pypi` section is the name of the installable package in the PyPi registry. + +If not specified, all remote registry configurations are disabled by default. diff --git a/docs/connector-development/connector-specification-reference.md b/docs/connector-development/connector-specification-reference.md index 6bdb79b2aef4..76d95e76b761 100644 --- a/docs/connector-development/connector-specification-reference.md +++ b/docs/connector-development/connector-specification-reference.md @@ -4,7 +4,11 @@ The [connector specification](../understanding-airbyte/airbyte-protocol.md#spec) ## Demoing your specification -While iterating on your specification, you can preview what it will look like in the UI in realtime by following the instructions [here](https://github.com/airbytehq/airbyte-platform/blob/master/airbyte-webapp/docs/HowTo-ConnectionSpecification.md). +While iterating on your specification, you can preview what it will look like in the UI in realtime by following the instructions below. +1. Open the `ConnectorForm` preview component in our deployed Storybook at: https://components.airbyte.dev/?path=/story/connector-connectorform--preview +2. Press `raw` on the `connectionSpecification` property, so you will be able to paste a JSON structured string +3. Set the string you want to preview the UI for +4. When submitting the form you can see a preview of the values in the "Actions" tab ### Secret obfuscation @@ -327,6 +331,61 @@ In each item in the `oneOf` array, the `option_title` string field exists with t } ``` +#### oneOf display type +You can also configure the way that oneOf fields are displayed in the Airbyte UI through the `display_type` property. Valid values for this property are: +- `dropdown` + - Renders a dropdown menu containing the title of each option for the user to select + - This is a compact look that works well in most cases + - The descriptions of the options can be found in the oneOf field's tooltip +- `radio` + - Renders radio-button cards side-by-side containing the title and description of each option for the user to select + - This choice draws more attention to the field and shows the descriptions of each option at all times, which can be useful for important or complicated fields + +Here is an example of setting the `display_type` of a oneOf field to `dropdown`, along with how it looks in the Airbyte UI: +``` +"update_method": { + "type": "object", + "title": "Update Method", + "display_type": "dropdown", + "oneOf": [ + { + "title": "Read Changes using Binary Log (CDC)", + "description": "Recommended - Incrementally reads new inserts, updates, and deletes using the MySQL binary log. This must be enabled on your database.", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "CDC", + "order": 0 + }, + "initial_waiting_seconds": { + ... + }, + "server_time_zone": { + ... + } + } + }, + { + "title": "Scan Changes with User Defined Cursor", + "description": "Incrementally detects new inserts and updates using the cursor column chosen when configuring a connection (e.g. created_at, updated_at).", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "STANDARD", + "order": 0 + } + } + } + ] +} +``` +![dropdown oneOf](../assets/docs/oneOf-dropdown.png) + +And here is how it looks if the `display_type` property is set to `radio` instead: +![radio oneOf](../assets/docs/oneOf-radio.png) + ### Using `enum` In regular `jsonschema`, some drafts enforce that `enum` lists must contain distinct values, while others do not. For consistency, Airbyte enforces this restriction. diff --git a/docs/connector-development/migration-to-base-image.md b/docs/connector-development/migration-to-base-image.md new file mode 100644 index 000000000000..63299ddc06b0 --- /dev/null +++ b/docs/connector-development/migration-to-base-image.md @@ -0,0 +1,78 @@ +# Migration guide: How to make a connector use our base image + +We currently enforce our certified connectors to use our [base image](https://hub.docker.com/r/airbyte/python-connector-base). +This guide will help connector developers to migrate their connector to use our base image. + +N.B: This guide currently only applies to python connectors. + +## Prerequisite +[Install the airbyte-ci tool](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md#L1) + + +## Definition of a successful migration +1. The connector `Dockerfile` is removed from the connector folder +2. The connector `metadata.yaml` is referencing the latest base image in the `data.connectorBuildOptions.baseImage` key +3. The connector version is bumped by a patch increment +4. A changelog entry is added to the connector documentation file +5. The connector is successfully built and tested by our CI +6. If you add `build_customization.py` to your connector, the Connector Operations team has reviewed and approved your changes. + +## Semi automated migration +- Run `airbyte-ci connectors --name= migrate_to_base_image ` +- Commit and push the changes on your PR + +## Manual migration + +In order for a connector to use our base image it has to declare it in its `metadata.yaml` file under the `data.connectorBuildOptions.baseImage` key: + +Example: + +```yaml + connectorBuildOptions: + baseImage: docker.io/airbyte/python-connector-base:1.1.0@sha256:bd98f6505c6764b1b5f99d3aedc23dfc9e9af631a62533f60eb32b1d3dbab20c +``` + +### Why are we using long addresses instead of tags? +**For build reproducibility!**. +Using full image address allows us to have a more deterministic build process. +If we used tags our connector could get built with a different base image if the tag was overwritten. +In other word, using the image digest (sha256), we have the guarantee that a build, on the same commit, will always use the same base image. + +### What if my connector needs specific system dependencies? +Declaring the base image in the metadata.yaml file makes the Dockerfile obselete and the connector will be built using our internal build process declared [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py#L55). +If your connector has specific system dependencies, or has to set environment variables, we have a pre/post build hook framework for that. + +You can customize our build process by adding a `build_customization.py` module to your connector. +This module should contain a `pre_connector_install` and `post_connector_install` async function that will mutate the base image and the connector container respectively. +It will be imported at runtime by our build process and the functions will be called if they exist. + +Here is an example of a `build_customization.py` module: +```python +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Feel free to check the dagger documentation for more information on the Container object and its methods. + # https://dagger-io.readthedocs.io/en/sdk-python-v0.6.4/ + from dagger import Container + + +async def pre_connector_install(base_image_container: Container) -> Container: + return await base_image_container.with_env_variable("MY_PRE_BUILD_ENV_VAR", "my_pre_build_env_var_value") + +async def post_connector_install(connector_container: Container) -> Container: + return await connector_container.with_env_variable("MY_POST_BUILD_ENV_VAR", "my_post_build_env_var_value") +``` + +### Listing migrated / non migrated connectors: + +To list all migrated certified connectors you can ran: +```bash +airbyte-ci connectors --support-level=certified --metadata-query="data.connectorBuildOptions.baseImage is not None" list +``` + +To list all non migrated certified connectors you can ran: +```bash +airbyte-ci connectors --metadata-query="data.supportLevel == 'certified' and 'connectorBuildOptions' not in data.keys()" list +``` diff --git a/docs/connector-development/testing-connectors/README.md b/docs/connector-development/testing-connectors/README.md index 4c0abe0e51fc..90842bccbccb 100644 --- a/docs/connector-development/testing-connectors/README.md +++ b/docs/connector-development/testing-connectors/README.md @@ -7,7 +7,7 @@ Connector specific tests declared in the connector code directory: * Integration tests Tests common to all connectors: -* [QA checks](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/connector_ops/ci_connector_ops/qa_checks.py#L1) +* [QA checks](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/connector_ops/connector_ops/qa_checks.py) * [Connector Acceptance tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference/) ## Running tests @@ -41,4 +41,4 @@ Connector Acceptance tests require connector configuration to be provided as a ` ## Tests on pull requests Our CI infrastructure runs the connector tests with [`airbyte-ci` CLI](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md). Connectors tests are automatically and remotely triggered on your branch according to the changes made in your branch. -**Passing tests are required to merge a connector pull request.** \ No newline at end of file +**Passing tests are required to merge a connector pull request.** diff --git a/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md b/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md index 2d9196626f3a..7d650d089415 100644 --- a/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md +++ b/docs/connector-development/testing-connectors/connector-acceptance-tests-reference.md @@ -32,34 +32,60 @@ _Note: Not all types of tests work for all connectors, only configure the ones t Build your connector image if needed. -```text -docker build . +**Option A (Preferred): Building the docker image with `airbyte-ci`** + +This is the preferred method for building and testing connectors. + +If you want to open source your connector we encourage you to use our [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) tool to build your connector. +It will not use a Dockerfile but will build the connector image from our [base image](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/base_images/README.md) and use our internal build logic to build an image from your Python connector code. + +Running `airbyte-ci connectors --name source- build` will build your connector image. +Once the command is done, you will find your connector image in your local docker host: `airbyte/source-:dev`. + +**Option B: Building the docker image with a Dockerfile** + +If you don't want to rely on `airbyte-ci` to build your connector, you can build the docker image using your own Dockerfile. This method is not preferred, and is not supported for certified connectors. + +Create a `Dockerfile` in the root of your connector directory. The `Dockerfile` should look something like this: + +```Dockerfile + +FROM airbyte/python-connector-base:1.1.0 + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] ``` -And test via one of the two following Options +Please use this as an example. This is not optimized. -### (Prefered) Option 1: Run against the production acceptance test image +Build your image: -From the root of your connector run: ```bash -./acceptance-test-docker.sh +docker build . -t airbyte/source-example-python:dev ``` -This will run you local connector image against the same test suite that Airbyte uses in production +And test via one of the two following Options + +### Option 1 (Preferred): Run against the Airbyte CI test suite + +Learn how to use and install [`airbyte-ci` here](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md). Once installed, `airbyte-ci connectors test` command will run unit, integration, and acceptance tests against your connector. Pass `--name ` to test just one connector. -### Option 2: Run against the Airbyte CI test suite ```bash -pipx install airbyte-ci/connectors/pipelines/ airbyte-ci connectors --name= --use-remote-secrets=false test ``` -### (Debugging) Option 3: Run against the acceptance tests on your branch +### Option 2 (Debugging): Run against the acceptance tests on your branch This will run the acceptance test suite directly with pytest. Allowing you to set breakpoints and debug your connector locally. The only pre-requisite is that you have [Poetry](https://python-poetry.org/docs/#installation) installed. Afterwards you do the following from the root of the `airbyte` repo: + ```bash cd airbyte-integrations/bases/connector-acceptance-test/ poetry install @@ -155,7 +181,7 @@ Configuring all streams in the input catalog to full refresh mode verifies that Set `validate_data_points=True` if possible. This validation is going to be enabled by default and won't be configurable in future releases. | Input | Type | Default | Note | -|:------------------------------------------| :--------------- |:--------------------------------------------|:--------------------------------------------------------------------------------------------------------------| +| :---------------------------------------- | :--------------- | :------------------------------------------ | :------------------------------------------------------------------------------------------------------------ | | `config_path` | string | `secrets/config.json` | Path to a JSON object representing a valid connector configuration | | `configured_catalog_path` | string | `integration_tests/configured_catalog.json` | Path to configured catalog | | `empty_streams` | array of objects | \[\] | List of streams that might be empty with a `bypass_reason` | @@ -209,7 +235,7 @@ In general, the expected_records.jsonl should contain the subset of output of th This test performs two read operations on all streams which support full refresh syncs. It then verifies that the RECORD messages output from both were identical or the former is a strict subset of the latter. | Input | Type | Default | Note | -|:------------------------------------------|:-------|:--------------------------------------------|:-----------------------------------------------------------------------| +| :---------------------------------------- | :----- | :------------------------------------------ | :--------------------------------------------------------------------- | | `config_path` | string | `secrets/config.json` | Path to a JSON object representing a valid connector configuration | | `configured_catalog_path` | string | `integration_tests/configured_catalog.json` | Path to configured catalog | | `timeout_seconds` | int | 20\*60 | Test execution timeout in seconds | @@ -223,26 +249,22 @@ This test performs two read operations on all streams which support full refresh This test verifies that all streams in the input catalog which support incremental sync can do so correctly. It does this by running two read operations: the first takes the configured catalog and config provided to this test as input. It then verifies that the sync produced a non-zero number of `RECORD` and `STATE` messages. The second read takes the same catalog and config used in the first test, plus the last `STATE` message output by the first read operation as the input state file. It verifies that either no records are produced \(since we read all records in the first sync\) or all records that produced have cursor value greater or equal to cursor value from `STATE` message. This test is performed only for streams that support incremental. Streams that do not support incremental sync are ignored. If no streams in the input catalog support incremental sync, this test is skipped. -| Input | Type | Default | Note | -| :------------------------ | :----- | :------------------------------------------ | :------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `config_path` | string | `secrets/config.json` | Path to a JSON object representing a valid connector configuration | -| `configured_catalog_path` | string | `integration_tests/configured_catalog.json` | Path to configured catalog | -| `cursor_paths` | dict | {} | For each stream, the path of its cursor field in the output state messages. If omitted the path will be taken from the last piece of path from stream cursor_field. | -| `timeout_seconds` | int | 20\*60 | Test execution timeout in seconds | -| `threshold_days` | int | 0 | For date-based cursors, allow records to be emitted with a cursor value this number of days before the state value. | +| Input | Type | Default | Note | +| :------------------------ | :----- | :------------------------------------------ | :----------------------------------------------------------------- | +| `config_path` | string | `secrets/config.json` | Path to a JSON object representing a valid connector configuration | +| `configured_catalog_path` | string | `integration_tests/configured_catalog.json` | Path to configured catalog | +| `timeout_seconds` | int | 20\*60 | Test execution timeout in seconds | ### TestReadSequentialSlices This test offers more comprehensive verification that all streams in the input catalog which support incremental syncs perform the sync correctly. It does so in two phases. The first phase uses the configured catalog and config provided to this test as input to make a request to the partner API and assemble the complete set of messages to be synced. It then verifies that the sync produced a non-zero number of `RECORD` and `STATE` messages. This set of messages is partitioned into batches of a `STATE` message followed by zero or more `RECORD` messages. For each batch of messages, the initial `STATE` message is used as input for a read operation to get records with respect to the cursor. The test then verifies that all of the `RECORDS` retrieved have a cursor value greater or equal to the cursor from the current `STATE` message. This test is performed only for streams that support incremental. Streams that do not support incremental sync are ignored. If no streams in the input catalog support incremental sync, this test is skipped. -| Input | Type | Default | Note | -| :------------------------------------- | :----- | :------------------------------------------ | :------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `config_path` | string | `secrets/config.json` | Path to a JSON object representing a valid connector configuration | -| `configured_catalog_path` | string | `integration_tests/configured_catalog.json` | Path to configured catalog | -| `cursor_paths` | dict | {} | For each stream, the path of its cursor field in the output state messages. If omitted the path will be taken from the last piece of path from stream cursor_field. | -| `timeout_seconds` | int | 20\*60 | Test execution timeout in seconds | -| `threshold_days` | int | 0 | For date-based cursors, allow records to be emitted with a cursor value this number of days before the state value. | -| `skip_comprehensive_incremental_tests` | bool | false | For non-GA and in-development connectors, control whether the more comprehensive incremental tests will be skipped | +| Input | Type | Default | Note | +| :------------------------------------- | :----- | :------------------------------------------ | :----------------------------------------------------------------------------------------------------------------- | +| `config_path` | string | `secrets/config.json` | Path to a JSON object representing a valid connector configuration | +| `configured_catalog_path` | string | `integration_tests/configured_catalog.json` | Path to configured catalog | +| `timeout_seconds` | int | 20\*60 | Test execution timeout in seconds | +| `skip_comprehensive_incremental_tests` | bool | false | For non-GA and in-development connectors, control whether the more comprehensive incremental tests will be skipped | **Note that this test samples a fraction of stream slices across an incremental sync in order to reduce test duration and avoid spamming partner APIs** @@ -258,6 +280,16 @@ This test verifies that sync produces no records when run with the STATE with ab | `timeout_seconds` | int | 20\*60 | Test execution timeout in seconds | | | `bypass_reason` | string | None | Explain why this test is bypassed | | +## Test Connector Attributes + +Verifies that certain properties of the connector and its streams guarantee a higher level of usability standards for certified connectors. +Some examples of the types of tests covered are verification that streams define primary keys, correct OAuth spec configuration, or a connector emits the correct stream status during a read. + +| Input | Type | Default | Note | +|:------------------------------------------|:-----------------|:----------------------|:-----------------------------------------------------------------------| +| `config_path` | string | `secrets/config.json` | Path to a JSON object representing a valid connector configuration | +| `streams_without_primary_key` | array of objects | None | List of streams that do not support a primary key like reports streams | + ## Strictness level To enforce maximal coverage of acceptances tests we expose a `test_strictness_level` field at the root of the `acceptance-test-config.yml` configuration. @@ -367,7 +399,6 @@ acceptance_tests: tests: - config_path: secrets/config.json configured_catalog_path: integration_tests/configured_catalog.json - cursor_paths: ... future_state: future_state_path: integration_tests/abnormal_state.json @@ -386,31 +417,35 @@ You can disable this behavior by setting `cached_discovered_catalog: False` at t Breaking changes are modifications that make previous versions of the connector incompatible, requiring a major version bump. Here are the various types of changes that we consider breaking: 1. **Changes to Stream Schema** - - **Removing a Field**: If a field is removed from the stream's schema, it's a breaking change. Clients expecting the field may fail when it's absent. - - **Changing Field Type**: If the data type of a field is changed, it could break clients expecting the original type. For instance, changing a field from string to integer would be a breaking change. - - **Renaming a Field**: If a field is renamed, it can break existing clients that expect the field by its original name. + + - **Removing a Field**: If a field is removed from the stream's schema, it's a breaking change. Clients expecting the field may fail when it's absent. + - **Changing Field Type**: If the data type of a field is changed, it could break clients expecting the original type. For instance, changing a field from string to integer would be a breaking change. + - **Renaming a Field**: If a field is renamed, it can break existing clients that expect the field by its original name. 2. **Changes to Stream Behaviour** - - **Changing the Cursor**: Changing the cursor field for incremental streams can cause data discrepancies or synchronization issues. Therefore, it's considered a breaking change. - - **Renaming a Stream**: If a stream is renamed, it could cause failures for clients expecting the stream with its original name. Hence, this is a breaking change. - - **Changing Sync Mechanism**: If a stream's sync mechanism changes, such as switching from full refresh sync to incremental sync (or vice versa), it's a breaking change. Existing workflows may fail or behave unexpectedly due to this change. + + - **Changing the Cursor**: Changing the cursor field for incremental streams can cause data discrepancies or synchronization issues. Therefore, it's considered a breaking change. + - **Renaming a Stream**: If a stream is renamed, it could cause failures for clients expecting the stream with its original name. Hence, this is a breaking change. + - **Changing Sync Mechanism**: If a stream's sync mechanism changes, such as switching from full refresh sync to incremental sync (or vice versa), it's a breaking change. Existing workflows may fail or behave unexpectedly due to this change. 3. **Changes to Configuration Options** - - **Removing or Renaming Options**: If configuration options are removed or renamed, it could break clients using those options, hence, is considered a breaking change. - - **Changing Default Values or Behaviours**: Altering default values or behaviours of configuration options can break existing clients that rely on previous defaults. + + - **Removing or Renaming Options**: If configuration options are removed or renamed, it could break clients using those options, hence, is considered a breaking change. + - **Changing Default Values or Behaviours**: Altering default values or behaviours of configuration options can break existing clients that rely on previous defaults. 4. **Changes to Authentication Mechanism** - - Any change to the connector's authentication mechanism that isn't backwards compatible is a breaking change. For example, switching from API key authentication to OAuth without supporting both is a breaking change. + + - Any change to the connector's authentication mechanism that isn't backwards compatible is a breaking change. For example, switching from API key authentication to OAuth without supporting both is a breaking change. 5. **Changes to Error Handling** - - Altering the way errors are handled can be a breaking change. For example, if a certain type of error was previously ignored and now causes the connector to fail, it could break user's existing workflows. + + - Altering the way errors are handled can be a breaking change. For example, if a certain type of error was previously ignored and now causes the connector to fail, it could break user's existing workflows. 6. **Changes That Require User Intervention** - - If a change requires user intervention, such as manually updating settings or reconfiguring workflows, it would be considered a breaking change. + - If a change requires user intervention, such as manually updating settings or reconfiguring workflows, it would be considered a breaking change. Please note that this is an exhaustive but not an exclusive list. Other changes could be considered breaking if they disrupt the functionality of the connector or alter user expectations in a significant way. - ## Additional Checks While not necessarily related to Connector Acceptance Testing, Airbyte employs a number of additional checks which run on connector Pull Requests which check the following items: @@ -457,9 +492,9 @@ data: ## Custom environment variable The connector under tests can be run with custom environment variables: + ```yaml connector_image: "airbyte/source-pokeapi" custom_environment_variables: my_custom_environment_variable: value -... -``` \ No newline at end of file +``` diff --git a/docs/connector-development/testing-connectors/testing-a-local-catalog-in-development.md b/docs/connector-development/testing-connectors/testing-a-local-catalog-in-development.md deleted file mode 100644 index d8c9fd4af67c..000000000000 --- a/docs/connector-development/testing-connectors/testing-a-local-catalog-in-development.md +++ /dev/null @@ -1,36 +0,0 @@ -# Testing A Custom Registry - -## Purpose of this document -This document describes how to -1. Modify the connector catalog used by the platform -2. Use the newly modified catalog in the platform - -## Why you might need to -1. You've added/updated/deleted a generally available connector and want to test it in the platform UI -1. You've added/updated/deleted a generally available connector and want to test it in the platform API - -## Method 1: Edit the registry by hand (easiest) - -### 1. Download the current OSS Registry -Download the current registry from [here](https://connectors.airbyte.com/files/registries/v0/oss_registry.json) to somewhere on your local machine. - -### 2. Modify the registry -Modify the registry as you see fit. For example, you can add a new connector, update an existing connector, or delete a connector. - -### 3. Upload the modified registry to a public location -Upload the modified registry to a public location. For example, you can upload it to a public S3 bucket, or you can upload it to a public GitHub repo, or a service like file.io - -### 4. Point the platform to the modified registry -Run the platform with the following environment variable set: -``` -REMOTE_CONNECTOR_CATALOG_URL = -``` - -## Method 2: Use the registry generator (more involved) - -Follow the steps in the [Metadata Orchestrator Readme](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/metadata_service/orchestrator/README.md)) to setup the orchestrator. - -You can then use the public GCS url of the registry created by the orchestrator to point the platform to the modified registry. -``` -REMOTE_CONNECTOR_CATALOG_URL = -``` \ No newline at end of file diff --git a/docs/connector-development/tutorials/adding-incremental-sync.md b/docs/connector-development/tutorials/adding-incremental-sync.md index 992c9d9ed4b5..8a454049a7dd 100644 --- a/docs/connector-development/tutorials/adding-incremental-sync.md +++ b/docs/connector-development/tutorials/adding-incremental-sync.md @@ -2,7 +2,7 @@ ## Overview -This tutorial will assume that you already have a working source. If you do not, feel free to refer to the [Building a Toy Connector](building-a-python-source.md) tutorial. This tutorial will build directly off the example from that article. We will also assume that you have a basic understanding of how Airbyte's Incremental-Append replication strategy works. We have a brief explanation of it [here](../../understanding-airbyte/connections/incremental-append.md). +This tutorial will assume that you already have a working source. If you do not, feel free to refer to the [Building a Toy Connector](build-a-connector-the-hard-way.md) tutorial. This tutorial will build directly off the example from that article. We will also assume that you have a basic understanding of how Airbyte's Incremental-Append replication strategy works. We have a brief explanation of it [here](/using-airbyte/core-concepts/sync-modes/incremental-append.md). ## Update Catalog in `discover` @@ -293,6 +293,6 @@ Bonus points: go to Airbyte UI and reconfigure the connection to use incremental Incremental definitely requires more configurability than full refresh, so your implementation may deviate slightly depending on whether your cursor field is source defined or user-defined. If you think you are running into one of those cases, check out -our [incremental](../../understanding-airbyte/connections/incremental-append.md) documentation for more information on different types of +our [incremental](/using-airbyte/core-concepts/sync-modes/incremental-append.md) documentation for more information on different types of configuration. diff --git a/docs/connector-development/tutorials/build-a-connector-the-hard-way.md b/docs/connector-development/tutorials/build-a-connector-the-hard-way.md index 75fd0a0e6772..5f9edd2d0d58 100644 --- a/docs/connector-development/tutorials/build-a-connector-the-hard-way.md +++ b/docs/connector-development/tutorials/build-a-connector-the-hard-way.md @@ -4,77 +4,67 @@ description: Building a source connector without using any helpers to learn the # Building a Source Connector: The Hard Way -This tutorial walks you through building a simple Airbyte source without using any helpers to demonstrate the following concepts in Action: +This tutorial walks you through building a simple Airbyte source without using any helpers to demonstrate the following concepts in action: -* [The Airbyte Specification](../../understanding-airbyte/airbyte-protocol.md) and the interface implemented by a source connector -* [The AirbyteCatalog](../../understanding-airbyte/beginners-guide-to-catalog.md) -* [Packaging your connector](https://docs.airbyte.com/connector-development#1.-implement-and-package-the-connector) -* [Testing your connector](../testing-connectors/connector-acceptance-tests-reference.md) +- [The Airbyte Specification](../../understanding-airbyte/airbyte-protocol.md) and the interface implemented by a source connector +- [The AirbyteCatalog](../../understanding-airbyte/beginners-guide-to-catalog.md) +- [Packaging your connector](https://docs.airbyte.com/connector-development#1.-implement-and-package-the-connector) +- [Testing your connector](../testing-connectors/connector-acceptance-tests-reference.md) -At the end of this tutorial, you will have a working source that you will be able to use in the Airbyte UI. +:::warning +**This tutorial is meant for those interested in learning how the Airbyte Specification works in detail, +not for creating production connectors**. +If you're building a real source, you should start with using the [Connector Builder](../connector-builder-ui/overview), or +the [Connector Development Kit](https://github.com/airbytehq/airbyte/tree/master/airbyte-cdk/python/docs/tutorials). +::: -**This tutorial is meant for those interested in learning how the Airbyte Specification works in detail, not for creating production connectors**. We intentionally don't use helper libraries provided by Airbyte so that this tutorial is self-contained. If you were building a "real" source, you'll want to use the helper modules such as the [Connector Development Kit](https://github.com/airbytehq/airbyte/tree/master/airbyte-cdk/python/docs/tutorials). - -This tutorial can be done entirely on your local workstation. - -### Requirements +## Requirements To run this tutorial, you'll need: -* Docker, Python, and Java with the versions listed in the [tech stack section](../../understanding-airbyte/tech-stack.md). -* The `requests` Python package installed via `pip install requests` \(or `pip3` if `pip` is linked to a Python2 installation on your system\) - -**A note on running Python**: all the commands below assume that `python` points to a version of Python 3.9 or greater. Verify this by running - -```bash -$ python --version -Python 3.9.11 -``` - -On some systems, `python` points to a Python2 installation and `python3` points to Python3. If this is the case on your machine, substitute all `python` commands in this guide with `python3` . Otherwise, make sure to install Python 3 before beginning. - -You need also to install `requests` python library: -````bash -pip install requests -```` +- Docker, Python, and Java with the versions listed in the [tech stack section](../../understanding-airbyte/tech-stack.md). +- The `requests` Python package installed via `pip install requests` \(or `pip3` if `pip` is linked to a Python2 installation on your system\) ## Our connector: a stock ticker API -Our connector will output the daily price of a stock since a given date. We'll leverage the free [Polygon.io API](https://polygon.io/pricing) for this. We'll use Python to implement the connector because its syntax is accessible to most programmers, but the process described here can be applied to any language. +The connector will output the daily price of a stock since a given date. +We'll leverage [Polygon.io API](https://polygon.io/) for this. -Here's the outline of what we'll do to build our connector: +:::info +We'll use Python to implement the connector, but you could build an Airbyte +connector in any language. +::: + +Here's the outline of what we'll do to build the connector: 1. Use the Airbyte connector template to bootstrap the connector package 2. Implement the methods required by the Airbyte Specification for our connector: 1. `spec`: declares the user-provided credentials or configuration needed to run the connector - 2. `check`: tests if the connector can connect with the underlying data source with the user-provided configuration + 2. `check`: tests if the connector can connect with the underlying data source with the user-provided configuration 3. `discover`: declares the different streams of data that this connector can output 4. `read`: reads data from the underlying data source \(The stock ticker API\) 3. Package the connector in a Docker image -4. Test the connector using Airbyte's Standard Test Suite +4. Test the connector using Airbyte's Connector Acceptance Test Suite 5. Use the connector to create a new Connection and run a sync in Airbyte UI -Once we've completed the above steps, we will have built a functioning connector. Then, we'll add some optional functionality: +[Part 2 of this article](adding-incremental-sync.md) covers: + +- Support [incremental sync](../../using-airbyte/core-concepts/sync-modes/incremental-append.md) +- Add custom integration tests -* Support [incremental sync](../../understanding-airbyte/connections/incremental-append.md) -* Add custom integration tests +Let's get started! + +--- ### 1. Bootstrap the connector package -We'll start the process from the Airbyte repository root: +Start the process from the Airbyte repository root: ```bash $ pwd /Users/sherifnada/code/airbyte ``` -First, let's create a new branch: - -```bash -$ git checkout -b $(whoami)/source-connector-tutorial -Switched to a new branch 'sherifnada/source-connector-tutorial' -``` - Airbyte provides a code generator which bootstraps the scaffolding for our connector. Let's use it by running: ```bash @@ -82,22 +72,23 @@ $ cd airbyte-integrations/connector-templates/generator $ ./generate.sh ``` -We'll select the `generic` template and call the connector `stock-ticker-api`: +Select the `Generic Source` template and call the connector `stock-ticker-api`: ![](../../.gitbook/assets/newsourcetutorial_plop.gif) -Note: The generic template is very bare. If you are planning on developing a Python source, we recommend using the `python` template. It provides some convenience code to help reduce boilerplate. This tutorial uses the bare-bones version because it makes it easier to see how all the pieces of a connector work together. You can find a walk through on how to build a Python connector here \(**coming soon**\). +:::info +This tutorial uses the bare-bones `Generic Source` template to illustrate how all the pieces of a connector +work together. For real connectors, the generator provides `Python` and `Python HTTP API` source templates, they use +[Airbyte CDK](../cdk-python/README.md). +::: -Head to the connector directory and we should see the following files have been generated: ```bash $ cd ../../connectors/source-stock-ticker-api $ ls -Dockerfile README.md acceptance-test-config.yml acceptance-test-docker.sh build.gradle +Dockerfile README.md acceptance-test-config.yml metadata.yaml ``` -We'll use each of these files later. But first, let's write some code! - ### 2. Implement the connector in line with the Airbyte Specification In the connector package directory, create a single Python file `source.py` that will hold our implementation: @@ -108,17 +99,18 @@ touch source.py #### Implement the spec operation -At this stage in the tutorial, we just want to implement the `spec` operation as described in the [Airbyte Protocol](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#spec). This involves a couple of steps: - -1. Decide which inputs we need from the user in order to connect to the stock ticker API \(i.e: the connector's specification\) and encode it as a JSON file. -2. Identify when the connector has been invoked with the `spec` operation and return the specification as an `AirbyteMessage` +The `spec` operation is described in the [Airbyte Protocol](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#spec). +It's a way for the connector to tell Airbyte what user inputs it needs in order to connecto to the source (the stock +ticker API in our case). Airbyte expects the command to output a connector specification in `AirbyteMessage` format. To contact the stock ticker API, we need two things: 1. Which stock ticker we're interested in 2. The API key to use when contacting the API \(you can obtain a free API token from [Polygon.io](https://polygon.io/dashboard/signup) free plan\) +:::info For reference, the API docs we'll be using [can be found here](https://polygon.io/docs/stocks/get_v2_aggs_ticker__stocksticker__range__multiplier___timespan___from___to). +::: Let's create a [JSONSchema](http://json-schema.org/) file `spec.json` encoding these two requirements: @@ -147,21 +139,19 @@ Let's create a [JSONSchema](http://json-schema.org/) file `spec.json` encoding t } ``` -* `documentationUrl` is the URL that will appear in the UI for the user to gain more info about this connector. Typically this points to `docs.airbyte.com/integrations/sources/source-` but to keep things simple we won't show adding documentation -* `title` is the "human readable" title displayed in the UI. Without this field, The Stock Ticker field will have the title `stock_ticker` in the UI -* `description` will be shown in the Airbyte UI under each field to help the user understand it -* `airbyte_secret` used by Airbyte to determine if the field should be displayed as a password \(e.g: `********`\) in the UI and not readable from the API +- `documentationUrl` is the URL that will appear in the UI for the user to gain more info about this connector. Typically this points to `docs.airbyte.com/integrations/sources/source-` but to keep things simple we won't show adding documentation +- `title` is the "human readable" title displayed in the UI. Without this field, The Stock Ticker field will have the title `stock_ticker` in the UI +- `description` will be shown in the Airbyte UI under each field to help the user understand it +- `airbyte_secret` used by Airbyte to determine if the field should be displayed as a password \(e.g: `********`\) in the UI and not readable from the API -We'll save this file in the root directory of our connector. Now we have the following files: ```bash $ ls -1 Dockerfile README.md acceptance-test-config.yml -acceptance-test-docker.sh -build.gradle source.py +metadata.yaml spec.json ``` @@ -173,6 +163,7 @@ import argparse # helps parse commandline arguments import json import sys import os +from datetime import datetime def read_json(filepath): @@ -185,6 +176,12 @@ def log(message): print(json.dumps(log_json)) +def log_error(error_message): + current_time_in_ms = int(datetime.now().timestamp()) * 1000 + log_json = {"type": "TRACE", "trace": {"type": "ERROR", "emitted_at": current_time_in_ms, "error": {"message": error_message}}} + print(json.dumps(log_json)) + + def spec(): # Read the file named spec.json from the module directory as a JSON file current_script_directory = os.path.dirname(os.path.realpath(__file__)) @@ -231,10 +228,13 @@ if __name__ == "__main__": Some notes on the above code: -1. As described in the [specification](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#key-takeaways), Airbyte connectors are CLIs which communicate via stdout, so the output of the command is simply a JSON string formatted according to the Airbyte Specification. So to "return" a value we use `print` to output the return value to stdout -2. All Airbyte commands can output log messages that take the form `{"type":"LOG", "log":"message"}`, so we create a helper method `log(message)` to allow logging -3. All Airbyte commands can output error messages that take the form `{"type":"TRACE", "trace": {"type": "ERROR", "emitted_at": current_time_in_ms, "error": {"message": error_message}}}}`, so we create a helper method `log_error(message)` to allow error messages - +1. As described in the [specification](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#key-takeaways), + Airbyte connectors are CLIs which communicate via stdout, so the output of the command is simply a JSON string + formatted according to the Airbyte Specification. So to "return" a value we use `print` to output the return value to stdout. +2. All Airbyte commands can output log messages that take the form `{"type":"LOG", "log":"message"}`, so we create a helper method `log(message)` to allow logging. +3. All Airbyte commands can output error messages that take the form + `{"type":"TRACE", "trace": {"type": "ERROR", "emitted_at": current_time_in_ms, "error": {"message": error_message}}}}`, + so we create a helper method `log_error(message)` to allow error messages. Now if we run `python source.py spec` we should see the specification printed out: @@ -243,17 +243,19 @@ python source.py spec {"type": "SPEC", "spec": {"documentationUrl": "https://polygon.io/docs/stocks/get_v2_aggs_ticker__stocksticker__range__multiplier___timespan___from___to", "connectionSpecification": {"$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "required": ["stock_ticker", "api_key"], "properties": {"stock_ticker": {"type": "string", "title": "Stock Ticker", "description": "The stock ticker to track", "examples": ["AAPL", "TSLA", "AMZN"]}, "api_key": {"type": "string", "description": "The Polygon.io Stocks API key to use to hit the API.", "airbyte_secret": true}}}}} ``` -We've implemented the first command! Three more and we'll have a working connector. - #### Implementing check connection -The second command to implement is the [check operation](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#check) `check --config `, which tells the user whether a config file they gave us is correct. In our case, "correct" means they input a valid stock ticker and a correct API key like we declare via the `spec` operation. +The second command to implement is the [check operation](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#check) `check --config `, +which tells the user whether a config file they gave us is correct. In our case, "correct" means they input a valid +stock ticker and a correct API key like we declare via the `spec` operation. To achieve this, we'll: -1. Create valid and invalid configuration files to test the success and failure cases with our connector. We'll place config files in the `secrets/` directory which is gitignored everywhere in the Airbyte monorepo by default to avoid accidentally checking in API keys -2. Add a `check` method which calls the Polygon.io API to verify if the provided token & stock ticker are correct and output the correct airbyte message -3. Extend the argument parser to recognize the `check --config ` command and call the `check` method when the `check` command is invoked +1. Create valid and invalid configuration files to test the success and failure cases with our connector. + We'll place config files in the `secrets/` directory which is gitignored everywhere in the Airbyte monorepo by + default to avoid accidentally checking in API keys. +2. Add a `check` method which calls the Polygon.io API to verify if the provided token & stock ticker are correct and output the correct airbyte message. +3. Extend the argument parser to recognize the `check --config ` command and call the `check` method when the `check` command is invoked. Let's first add the configuration files: @@ -265,7 +267,7 @@ $ echo '{"api_key": "not_a_real_key", "stock_ticker": "TSLA"}' > secrets/invalid Make sure to add your actual API key instead of the placeholder value `` when following the tutorial. -Then we'll add the `check_method`: +Then we'll add the `check` method: ```python import requests @@ -295,7 +297,8 @@ def check(config): print(json.dumps(output_message)) ``` -Lastly we'll extend the `run` method to accept the `check` command and call the `check` method. First we'll add a helper method for reading input: +In Airbyte, the contract for input files is that they will be available in the current working directory if they are not provided as an absolute path. +This method helps us achieve that: ```python def get_input_file_path(path): @@ -305,33 +308,7 @@ def get_input_file_path(path): return os.path.join(os.getcwd(), path) ``` -In Airbyte, the contract for input files is that they will be available in the current working directory if they are not provided as an absolute path. This method helps us achieve that. - -We also need to extend the arguments parser by adding the following two blocks to the `run` method: - -```python - # Accept the check command - check_parser = subparsers.add_parser("check", help="checks the config used to connect", parents=[parent_parser]) - required_check_parser = check_parser.add_argument_group("required named arguments") - required_check_parser.add_argument("--config", type=str, required=True, help="path to the json configuration file") -``` - -and - -```python -elif command == "check": - config_file_path = get_input_file_path(parsed_args.config) - config = read_json(config_file_path) - check(config) -``` - -Then we need to update our list of available commands: - -```python - log("Invalid command. Allowable commands: [spec, check]") -``` - -This results in the following `run` method. +We'll then add the `check` command support to `run`: ```python def run(args): @@ -366,8 +343,6 @@ def run(args): sys.exit(0) ``` -and that should be it. - Let's test our new method: ```bash @@ -383,7 +358,8 @@ Our connector is able to detect valid and invalid configs correctly. Two methods The `discover` command outputs a Catalog, a struct that declares the Streams and Fields \(Airbyte's equivalents of tables and columns\) output by the connector. It also includes metadata around which features a connector supports \(e.g. which sync modes\). In other words it describes what data is available in the source. If you'd like to read a bit more about this concept check out our [Beginner's Guide to the Airbyte Catalog](../../understanding-airbyte/beginners-guide-to-catalog.md) or for a more detailed treatment read the [Airbyte Specification](../../understanding-airbyte/airbyte-protocol.md). -The data output by this connector will be structured in a very simple way. This connector outputs records belonging to exactly one Stream \(table\). Each record contains three Fields \(columns\): `date`, `price`, and `stock_ticker`, corresponding to the price of a stock on a given day. +The stock ticker connector outputs records belonging to exactly one Stream \(table\). +Each record contains three Fields \(columns\): `date`, `price`, and `stock_ticker`, corresponding to the price of a stock on a given day. To implement `discover`, we'll: @@ -440,8 +416,9 @@ We need to update our list of available commands: ```python log("Invalid command. Allowable commands: [spec, check, discover]") ``` - +:::info You may be wondering why `config` is a required input to `discover` if it's not used. This is done for consistency: the Airbyte Specification requires `--config` as an input to `discover` because many sources require it \(e.g: to discover the tables available in a Postgres database, you must supply a password\). So instead of guessing whether the flag is required depending on the connector, we always assume it is required, and the connector can choose whether to use it. +::: The full run method is now below: @@ -504,10 +481,11 @@ python source.py read --config --catalog --use-remote-secrets=false test ``` -After tests have run, you should see a test summary like: - -```text -collecting ... - test_core.py ✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓✓ 95% █████████▌ - test_full_refresh.py ✓ 100% ██████████ - -================== short test summary info ================== -SKIPPED [1] connector_acceptance_test/plugin.py:56: Skipping TestIncremental.test_two_sequential_reads because not found in the config - -Results (8.91s): - 20 passed -``` +`airbyte-ci` will build and then test your connector, and provide a report on the test results. That's it! We've created a fully functioning connector. Now let's get to the exciting part: using it from the Airbyte UI. +--- + ### Use the connector in the Airbyte UI Let's recap what we've achieved so far: 1. Implemented a connector 2. Packaged it in a Docker image -3. Integrated it with the Airbyte Standard Test suite +3. Ran Connector Acceptance Tests for the connector with `airbyte-ci` To use it from the Airbyte UI, we need to: @@ -1057,7 +998,8 @@ To use it from the Airbyte UI, we need to: #### 1. Publish the Docker image -Since we're running this tutorial locally, Airbyte will have access to any Docker images available to your local `docker` daemon. So all we need to do is build & tag our connector. If you want your connector to be available to everyone using Airbyte, you'll need to publish it to `Dockerhub`. [Open a PR](https://github.com/airbytehq/airbyte) or visit our [Slack](https://slack.airbyte.io) for help with this. +Since we're running this tutorial locally, Airbyte will have access to any Docker images available to your local `docker` daemon. So all we need to do is build & tag our connector. +For real production connectors to be available on Airbyte Cloud, you'd need to publish them on DockerHub. Airbyte's build system builds and tags your connector's image correctly by default as part of the connector's standard `build` process. **From the Airbyte repo root**, run: @@ -1091,7 +1033,7 @@ docker compose up When Airbyte server is done starting up, it prints the following banner in the log output \(it can take 10-20 seconds for the server to start\): ```bash -airbyte-server | 2022-03-11 18:38:33 INFO i.a.s.ServerApp(start):121 - +airbyte-server | 2022-03-11 18:38:33 INFO i.a.s.ServerApp(start):121 - airbyte-server | ___ _ __ __ airbyte-server | / | (_)____/ /_ __ __/ /____ airbyte-server | / /| | / / ___/ __ \/ / / / __/ _ \ @@ -1102,7 +1044,7 @@ airbyte-server | -------------------------------------- airbyte-server | Now ready at http://localhost:8000/ airbyte-server | -------------------------------------- airbyte-server | Version: dev -airbyte-server | +airbyte-server | ``` After you see the above banner printed out in the terminal window where you are running `docker compose up`, visit [http://localhost:8000](http://localhost:8000) in your browser and log in with the default credentials: username `airbyte` and password `password`. @@ -1144,12 +1086,12 @@ to find your connector by typing part of its name: After you select your connector in the Source type dropdown, the modal will show two more fields: API Key and Stock Ticker. Remember that `spec.json` file you created at the very beginning of this tutorial? These fields should correspond to the `properties` section of that file. Copy-paste your Polygon.io API key and a stock ticker into these fields and then click "Set up source" -button at the bottom right of the modal. +button at the bottom right of the modal. ![](../../.gitbook/assets/newsourcetutorial_source_config.png) Once you click "Set up source", Airbyte will spin up your connector and run "check" method to verify the configuration. -You will see a progress bar briefly and if the configuration is valid, you will see a success message, +You will see a progress bar briefly and if the configuration is valid, you will see a success message, the modal will close and you will see your connector on the updated Sources page. ![](../../.gitbook/assets/newsourcetutorial_sources_stock_ticker.png) @@ -1171,7 +1113,7 @@ Select "Mirror source structure" in the Destination Namespace, check the checkbo ![](../../.gitbook/assets/newsourcetutorial_configure_connection.png) -Ta-da! Your connection is now configured to sync once a day. You will see your new connection on the next screen: +Ta-da! Your connection is now configured to sync once a day. You will see your new connection on the next screen: ![](../../.gitbook/assets/newsourcetutorial_connection_done.png) @@ -1192,23 +1134,12 @@ $ cat /tmp/airbyte_local/tutorial_json/_airbyte_raw_stock_prices.jsonl Congratulations! We've successfully written a fully functioning Airbyte connector. You're an Airbyte contributor now ;\) -Armed with the knowledge you gained in this guide, here are some places you can go from here: - -1. Implement Incremental Sync for your connector \(described in the sections below\) -2. Implement another connector using the language specific helpers listed below -3. While not required, we love contributions! if you end up creating a new connector, we're here to help you make it available to everyone using Airbyte. Remember that you're never expected to maintain a connector by yourself if you merge it to Airbyte -- we're committed to supporting connectors if you can't do it yourself - -## Optional additions - -This section is not yet complete and will be completed soon. Please reach out to us on [Slack](https://slack.airbyte.io) or [Github](https://github.com/airbytehq/airbyte) if you need the information promised by these sections immediately. - -### Incremental sync -Follow the [next tutorial](adding-incremental-sync.md) to implement incremental sync. +1. Follow the [next tutorial](adding-incremental-sync.md) to implement incremental sync. +2. Implement another connector using the Low-code CDK, [Connector Builder](../connector-builder-ui/overview), or [Connector Development Kit](https://github.com/airbytehq/airbyte/tree/master/airbyte-cdk/python/docs/tutorials) +3. We welcome low-code configuration based connector contributions! If you make a connector in the connector builder + and want to share it with everyone using Airbyte, pull requests are welcome! -### Connector Development Kit -Like we mention at the beginning of the tutorial, this guide is meant more for understanding than as a blueprint for implementing production connectors. See the [Connector Development Kit](https://github.com/airbytehq/airbyte/tree/master/airbyte-cdk/python/docs/tutorials) for the frameworks you should use to build production-ready connectors. +## Additional guides -### Language specific helpers - * [Building a Python Source](https://docs.airbyte.com/connector-development/tutorials/building-a-python-source) - * [Building a Python Destination](https://docs.airbyte.com/connector-development/tutorials/building-a-python-destination) - * [Building a Java Destination](https://docs.airbyte.com/connector-development/tutorials/building-a-java-destination) +- [Building a Python Source](https://docs.airbyte.com/connector-development/tutorials/building-a-python-source) +- [Building a Java Destination](https://docs.airbyte.com/connector-development/tutorials/building-a-java-destination) diff --git a/docs/connector-development/tutorials/building-a-python-destination.md b/docs/connector-development/tutorials/building-a-python-destination.md deleted file mode 100644 index 73c9325a9e8a..000000000000 --- a/docs/connector-development/tutorials/building-a-python-destination.md +++ /dev/null @@ -1,210 +0,0 @@ -# Building a Python Destination - -:::warning -Airbyte has a Python Destination CDK that allows you to create quick and simple destinations for particular use cases. -Currently, the project is not accepting new Python Destinations in the official catalog. -However, you can still build the connector and use it for your projects locally or import it into Airbyte Cloud. -::: - -## Summary - -This article provides a checklist for how to create a Python destination. Each step in the checklist has a link to a more detailed explanation below. - -## Requirements - -Docker and Python with the versions listed in the [tech stack section](../../understanding-airbyte/tech-stack.md). You can use any Python version between 3.7 and 3.9, but this tutorial was tested with 3.7. - -## Checklist - -### Creating a destination - -* Step 1: Create the destination using the template generator -* Step 2: Setup the virtual environment -* Step 3: Implement `spec` to define the configuration required to run the connector -* Step 4: Implement `check` to provide a way to validate configurations provided to the connector -* Step 5: Implement `write` to write data to the destination -* Step 6: Set up Acceptance Tests -* Step 7: Write unit tests or integration tests -* Step 8: Update the docs \(in `docs/integrations/destinations/.md`\) - -:::info - -If you need help with any step of the process, feel free to submit a PR with your progress and any questions you have, or ask us on [slack](https://slack.airbyte.io). Also reference the KvDB python destination implementation if you want to see an example of a working destination. - -::: - -## Explaining Each Step - -### Step 1: Create the destination using the template - -Airbyte provides a code generator which bootstraps the scaffolding for our connector. - -```bash -$ cd airbyte-integrations/connector-templates/generator # assumes you are starting from the root of the Airbyte project. -$ ./generate.sh -``` - -Select the `Python Destination` template and then input the name of your connector. We'll refer to the destination as `destination-` in this tutorial, but you should replace `` with the actual name you used for your connector e.g: `redis` or `google-sheets`. - -### Step 2: Setup the dev environment - -Setup your Python virtual environment: - -```bash -cd airbyte-integrations/connectors/destination- - -# Create a virtual environment in the .venv directory -python -m venv .venv - -# activate the virtualenv -source .venv/bin/activate - -# Install with the "tests" extra which provides test requirements -pip install '.[tests]' -``` - -This step sets up the initial python environment. **All** subsequent `python` or `pip` commands assume you have activated your virtual environment. - -If you want your IDE to auto complete and resolve dependencies properly, point it at the python binary in `airbyte-integrations/connectors/destination-/.venv/bin/python`. Also anytime you change the dependencies in the `setup.py` make sure to re-run the build command. The build system will handle installing all dependencies in the `setup.py` into the virtual environment. - -Let's quickly get a few housekeeping items out of the way. - -#### Dependencies - -Python dependencies for your destination should be declared in `airbyte-integrations/connectors/destination-/setup.py` in the `install_requires` field. You might notice that a couple of Airbyte dependencies are already declared there \(mainly the Airbyte CDK and potentially some testing libraries or helpers\). Keep those as they will be useful during development. - -You may notice that there is a `requirements.txt` in your destination's directory as well. Do not touch this. It is autogenerated and used to install local Airbyte dependencies which are not published to PyPI. All your dependencies should be declared in `setup.py`. - -#### Iterating on your implementation - -Pretty much all it takes to create a destination is to implement the `Destination` interface. Let's briefly recap the three methods implemented by a Destination: - -1. `spec`: declares the user-provided credentials or configuration needed to run the connector -2. `check`: tests if the user-provided configuration can be used to connect to the underlying data destination, and with the correct write permissions -3. `write`: writes data to the underlying destination by reading a configuration, a stream of records from stdin, and a configured catalog describing the schema of the data and how it should be written to the destination - -The destination interface is described in detail in the [Airbyte Specification](../../understanding-airbyte/airbyte-protocol.md) reference. - -The generated files fill in a lot of information for you and have docstrings describing what you need to do to implement each method. The next few steps are just implementing that interface. - -:::info - -All logging should be done through the `self.logger` object available in the `Destination` class. Otherwise, logs will not be shown properly in the Airbyte UI. - -::: - -Everyone develops differently but here are 3 ways that we recommend iterating on a destination. Consider using whichever one matches your style. - -**Run the destination using Python** - -You'll notice in your destination's directory that there is a python file called `main.py`. This file is the entrypoint for the connector: - -```bash -# from airbyte-integrations/connectors/destination- -python main.py spec -python main.py check --config secrets/config.json -# messages.jsonl should contain AirbyteMessages (described in the Airbyte spec) -cat messages.jsonl | python main.py write --config secrets/config.json --catalog sample_files/configured_catalog.json -``` - -The nice thing about this approach is that you can iterate completely within in python. The downside is that you are not quite running your destination as it will actually be run by Airbyte. Specifically you're not running it from within the docker container that will house it. - -**Run using Docker** If you want to run your destination exactly as it will be run by Airbyte \(i.e. within a docker container\), you can use the following commands from the connector module directory \(`airbyte-integrations/connectors/destination-`\): - -```bash -# First build the container -docker build . -t airbyte/destination-:dev - -# Then use the following commands to run it -docker run --rm airbyte/destination-:dev spec -docker run --rm -v $(pwd)/secrets:/secrets airbyte/destination-:dev check --config /secrets/config.json -cat messages.jsonl | docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/destination-:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json -``` - -Note: Each time you make a change to your implementation you need to re-build the connector image. `docker build . -t airbyte/destination-:dev`. This ensures the new python code is added into the docker container. - -The nice thing about this approach is that you are running your source exactly as it will be run by Airbyte. The tradeoff is that iteration is slightly slower, because you need to re-build the connector between each change. - -**TDD using standard tests** - -_note: these tests aren't yet available for Python connectors but will be very soon. Until then you should use custom unit or integration tests for TDD_. - -Airbyte provides a standard test suite that is run against every destination. The objective of these tests is to provide some "free" tests that can sanity check that the basic functionality of the destination works. One approach to developing your connector is to simply run the tests between each change and use the feedback from them to guide your development. - -If you want to try out this approach, check out Step 6 which describes what you need to do to set up the standard tests for your destination. - -The nice thing about this approach is that you are running your destination exactly as Airbyte will run it in the CI. The downside is that the tests do not run very quickly. - -### Step 3: Implement `spec` - -Each destination contains a specification written in JsonSchema that describes the inputs it requires and accepts. Defining the specification is a good place to start development. To do this, find the spec file generated in `airbyte-integrations/connectors/destination-/src/main/resources/spec.json`. Edit it and you should be done with this step. The generated connector will take care of reading this file and converting it to the correct output. - -Some notes about fields in the output spec: - -* `supportsNormalization` is a boolean which indicates if this connector supports [basic normalization via DBT](https://docs.airbyte.com/understanding-airbyte/basic-normalization). If true, `supportsDBT` must also be true. -* `supportsDBT` is a boolean which indicates whether this destination is compatible with DBT. If set to true, the user can define custom DBT transformations that run on this destination after each successful sync. This must be true if `supportsNormalization` is set to true. -* `supported_destination_sync_modes`: An array of strings declaring the sync modes supported by this connector. The available options are: - * `overwrite`: The connector can be configured to wipe any existing data in a stream before writing new data - * `append`: The connector can be configured to append new data to existing data - * `append_dedup`: The connector can be configured to deduplicate \(i.e: UPSERT\) data in the destination based on the new data and primary keys -* `supportsIncremental`: Whether the connector supports any `append` sync mode. Must be set to true if `append` or `append_dedupe` are included in the `supported_destination_sync_modes`. - -Some helpful resources: - -* [**JSONSchema website**](https://json-schema.org/) -* [**Definition of Airbyte Protocol data models**](https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml). The output of `spec` is described by the `ConnectorSpecification` model \(which is wrapped in an `AirbyteConnectionStatus` message\). -* [**Postgres Destination's spec.json file**](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/destination-postgres/src/main/resources/spec.json) as an example `spec.json`. - -Once you've edited the file, see the `spec` operation in action: - -```bash -python main.py spec -``` - -### Step 4: Implement `check` - -The check operation accepts a JSON object conforming to the `spec.json`. In other words if the `spec.json` said that the destination requires a `username` and `password`, the config object might be `{ "username": "airbyte", "password": "password123" }`. It returns a json object that reports, given the credentials in the config, whether we were able to connect to the destination. - -While developing, we recommend storing any credentials in `secrets/config.json`. Any `secrets` directory in the Airbyte repo is gitignored by default. - -Implement the `check` method in the generated file `destination_/destination.py`. Here's an [example implementation](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/destination.py) from the KvDB destination. - -Verify that the method is working by placing your config in `secrets/config.json` then running: - -```bash -python main.py check --config secrets/config.json -``` - -### Step 5: Implement `write` - -The `write` operation is the main workhorse of a destination connector: it reads input data from the source and writes it to the underlying destination. It takes as input the config file used to run the connector as well as the configured catalog: the file used to describe the schema of the incoming data and how it should be written to the destination. Its "output" is two things: - -1. Data written to the underlying destination -2. `AirbyteMessage`s of type `AirbyteStateMessage`, written to stdout to indicate which records have been written so far during a sync. It's important to output these messages when possible in order to avoid re-extracting messages from the source. See the [write operation protocol reference](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol#write) for more information. - -To implement the `write` Airbyte operation, implement the `write` method in your generated `destination.py` file. [Here is an example implementation](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/destination-kvdb/destination_kvdb/destination.py) from the KvDB destination connector. - -### Step 6: Set up Acceptance Tests - -_Coming soon. These tests are not yet available for Python destinations but will be very soon. For now please skip this step and rely on copious amounts of integration and unit testing_. - -### Step 7: Write unit tests and/or integration tests - -The Acceptance Tests are meant to cover the basic functionality of a destination. Think of it as the bare minimum required for us to add a destination to Airbyte. You should probably add some unit testing or custom integration testing in case you need to test additional functionality of your destination. - -Add unit tests in `unit_tests/` directory and integration tests in the `integration_tests/` directory. Run them via - -```bash -python -m pytest -s -vv integration_tests/ -``` - -See the [KvDB integration tests](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/destination-kvdb/integration_tests/integration_test.py) for an example of tests you can implement. - -#### Step 8: Update the docs - -Each connector has its own documentation page. By convention, that page should have the following path: in `docs/integrations/destinations/.md`. For the documentation to get packaged with the docs, make sure to add a link to it in `docs/SUMMARY.md`. You can pattern match doing that from existing connectors. - -## Wrapping up - -Well done on making it this far! If you'd like your connector to ship with Airbyte by default, create a PR against the Airbyte repo and we'll work with you to get it across the finish line. - diff --git a/docs/connector-development/tutorials/building-a-python-source.md b/docs/connector-development/tutorials/building-a-python-source.md index dc86631782be..57d292680918 100644 --- a/docs/connector-development/tutorials/building-a-python-source.md +++ b/docs/connector-development/tutorials/building-a-python-source.md @@ -25,22 +25,18 @@ All the commands below assume that `python` points to a version of python >3. * Step 5: Implement `check` * Step 6: Implement `discover` * Step 7: Implement `read` -* Step 8: Set up Standard Tests +* Step 8: Set up Connector Acceptance Tests * Step 9: Write unit tests or integration tests * Step 10: Update the `README.md` \(If API credentials are required to run the integration, please document how they can be obtained or link to a how-to guide.\) * Step 11: Update the `metadata.yaml` file with accurate information about your connector. These metadata will be used to add the connector to Airbyte's connector registry. * Step 12: Add docs \(in `docs/integrations/sources/.md`\) :::info - Each step of the Creating a Source checklist is explained in more detail below. - ::: :::info - All `./gradlew` commands must be run from the root of the airbyte project. - ::: ### Submitting a Source to Airbyte @@ -52,9 +48,7 @@ All `./gradlew` commands must be run from the root of the airbyte project. * Edit the `airbyte/tools/bin/ci_credentials.sh` script to pull the script from the build environment and write it to `secrets/config.json` during the build. :::info - -If you have a question about a step the Submitting a Source to Airbyte checklist include it in your PR or ask it on [slack](https://slack.airbyte.io). - +If you have a question about a step the Submitting a Source to Airbyte checklist include it in your PR or ask it on [#help-connector-development channel on Slack](https://airbytehq.slack.com/archives/C027KKE4BCZ). ::: ## Explaining Each Step @@ -74,7 +68,7 @@ Select the `python` template and then input the name of your connector. For this Build the source by running: -```text +```bash cd airbyte-integrations/connectors/source- python -m venv .venv # Create a virtual environment in the .venv directory source .venv/bin/activate # enable the venv @@ -105,9 +99,7 @@ The commands we ran above created a virtual environment for your source. If you Pretty much all it takes to create a source is to implement the `Source` interface. The template fills in a lot of information for you and has extensive docstrings describing what you need to do to implement each method. The next 4 steps are just implementing that interface. :::info - All logging should be done through the `logger` object passed into each method. Otherwise, logs will not be shown in the Airbyte UI. - ::: #### Iterating on your implementation @@ -118,7 +110,7 @@ Everyone develops differently but here are 3 ways that we recommend iterating on You'll notice in your source's directory that there is a python file called `main.py`. This file exists as convenience for development. You can call it from within the virtual environment mentioned above `. ./.venv/bin/activate` to test out that your source works. -```text +```bash # from airbyte-integrations/connectors/source- python main.py spec python main.py check --config secrets/config.json @@ -128,33 +120,73 @@ python main.py read --config secrets/config.json --catalog sample_files/configur The nice thing about this approach is that you can iterate completely within in python. The downside is that you are not quite running your source as it will actually be run by Airbyte. Specifically you're not running it from within the docker container that will house it. -**Run the source using docker** -If you want to run your source exactly as it will be run by Airbyte \(i.e. within a docker container\), you can use the following commands from the connector module directory \(`airbyte-integrations/connectors/source-example-python`\): +**Build the source docker image** + +You have to build a docker image for your connector if you want to run your source exactly as it will be run by Airbyte. + +**Option A: Building the docker image with `airbyte-ci`** + +This is the preferred method for building and testing connectors. + +If you want to open source your connector we encourage you to use our [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) tool to build your connector. +It will not use a Dockerfile but will build the connector image from our [base image](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/base_images/README.md) and use our internal build logic to build an image from your Python connector code. + +Running `airbyte-ci connectors --name source- build` will build your connector image. +Once the command is done, you will find your connector image in your local docker host: `airbyte/source-:dev`. + + + +**Option B: Building the docker image with a Dockerfile** + +If you don't want to rely on `airbyte-ci` to build your connector, you can build the docker image using your own Dockerfile. This method is not preferred, and is not supported for certified connectors. -```text -# First build the container +Create a `Dockerfile` in the root of your connector directory. The `Dockerfile` should look something like this: + +```Dockerfile + +FROM airbyte/python-connector-base:1.1.0 + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] +``` + +Please use this as an example. This is not optimized. + +Build your image: +```bash docker build . -t airbyte/source-example-python:dev +``` -# Then use the following commands to run it +**Run the source docker image** + +```bash docker run --rm airbyte/source-example-python:dev spec docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-example-python:dev check --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-example-python:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files airbyte/source-example-python:dev read --config /secrets/config.json --catalog /sample_files/configured_catalog.json ``` -Note: Each time you make a change to your implementation you need to re-build the connector image. `docker build . -t airbyte/source-example-python:dev`. This ensures the new python code is added into the docker container. +:::info +Each time you make a change to your implementation you need to re-build the connector image. This ensures the new python code is added into the docker container. +::: The nice thing about this approach is that you are running your source exactly as it will be run by Airbyte. The tradeoff is that iteration is slightly slower, because you need to re-build the connector between each change. **Detailed Debug Messages** During development of your connector, you can enable the printing of detailed debug information during a sync by specifying the `--debug` flag. This will allow you to get a better picture of what is happening during each step of your sync. -```text + +```bash python main.py read --config secrets/config.json --catalog sample_files/configured_catalog.json --debug ``` In addition to the preset CDK debug statements, you can also emit custom debug information from your connector by introducing your own debug statements: + ```python self.logger.debug( "your debug message here", @@ -165,9 +197,9 @@ self.logger.debug( ) ``` -**TDD using standard tests** +**TDD using acceptance tests & integration tests** -Airbyte provides a standard test suite that is run against every source. The objective of these tests is to provide some "free" tests that can sanity check that the basic functionality of the source works. One approach to developing your connector is to simply run the tests between each change and use the feedback from them to guide your development. +Airbyte provides an acceptance test suite that is run against every source. The objective of these tests is to provide some "free" tests that can sanity check that the basic functionality of the source works. One approach to developing your connector is to simply run the tests between each change and use the feedback from them to guide your development. If you want to try out this approach, check out Step 8 which describes what you need to do to set up the standard tests for your source. @@ -197,21 +229,19 @@ For a brief overview on the catalog check out [Beginner's Guide to the Airbyte C As described in the template code, this method takes in the same config object as the previous methods. It also takes in a "configured catalog". This object wraps the catalog emitted by the `discover` step and includes configuration on how the data should be replicated. For a brief overview on the configured catalog check out [Beginner's Guide to the Airbyte Catalog](../../understanding-airbyte/beginners-guide-to-catalog.md). It then returns a generator which returns each record in the stream. -### Step 8: Set up Standard Tests +### Step 8: Set up Connector Acceptance Tests (CATs) -The Standard Tests are a set of tests that run against all sources. These tests are run in the Airbyte CI to prevent regressions. They also can help you sanity check that your source works as expected. The following [article](../testing-connectors/connector-acceptance-tests-reference.md) explains Standard Tests and how to run them. +The Connector Acceptance Tests are a set of tests that run against all sources. These tests are run in the Airbyte CI to prevent regressions. They also can help you sanity check that your source works as expected. The following [article](../testing-connectors/connector-acceptance-tests-reference.md) explains Connector Acceptance Tests and how to run them. You can run the tests using `./gradlew :airbyte-integrations:connectors:source-:integrationTest`. Make sure to run this command from the Airbyte repository root. :::info - In some rare cases we make exceptions and allow a source to not need to pass all the standard tests. If for some reason you think your source cannot reasonably pass one of the tests cases, reach out to us on github or slack, and we can determine whether there's a change we can make so that the test will pass or if we should skip that test for your source. - ::: ### Step 9: Write unit tests and/or integration tests -The Standard Tests are meant to cover the basic functionality of a source. Think of it as the bare minimum required for us to add a source to Airbyte. In case you need to test additional functionality of your source, write unit or integration tests. +The connector acceptance tests are meant to cover the basic functionality of a source. Think of it as the bare minimum required for us to add a source to Airbyte. In case you need to test additional functionality of your source, write unit or integration tests. #### Unit Tests @@ -237,7 +267,7 @@ If you are self hosting Airbyte (OSS) you are able to use the Custom Connector f If you are using Airbyte Cloud (or OSS), you can submit a PR to add your connector to the Airbyte repository. Once the PR is merged, the connector will be available to all Airbyte Cloud users. You can read more about it [here](https://docs.airbyte.com/contributing-to-airbyte/submit-new-connector). Note that when submitting an Airbyte connector, you will need to ensure that -1. The connector passes the standard test suite. See [Set up Standard Tests](#step-8-set-up-standard-tests). +1. The connector passes the CAT suite. See [Set up Connector Acceptance Tests](#step-8-set-up-connector-acceptance-tests-\(cats\)). 2. The metadata.yaml file (created by our generator) is filed out and valid. See [Connector Metadata File](https://docs.airbyte.com/connector-development/connector-metadata-file). 3. You have created appropriate documentation for the connector. See [Add docs](#step-12-add-docs). diff --git a/docs/connector-development/tutorials/cdk-speedrun.md b/docs/connector-development/tutorials/cdk-speedrun.md index d32544898191..d6caac36974f 100644 --- a/docs/connector-development/tutorials/cdk-speedrun.md +++ b/docs/connector-development/tutorials/cdk-speedrun.md @@ -231,10 +231,44 @@ python main.py read --config sample_files/config.json --catalog sample_files/con If all goes well, containerize it so you can use it in the UI: + +**Option A: Building the docker image with `airbyte-ci`** + +This is the preferred method for building and testing connectors. + +If you want to open source your connector we encourage you to use our [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) tool to build your connector. +It will not use a Dockerfile but will build the connector image from our [base image](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/base_images/README.md) and use our internal build logic to build an image from your Python connector code. + +Running `airbyte-ci connectors --name source- build` will build your connector image. +Once the command is done, you will find your connector image in your local docker host: `airbyte/source-:dev`. + + + +**Option B: Building the docker image with a Dockerfile** + +If you don't want to rely on `airbyte-ci` to build your connector, you can build the docker image using your own Dockerfile. This method is not preferred, and is not supported for certified connectors. + +Create a `Dockerfile` in the root of your connector directory. The `Dockerfile` should look something like this: +```Dockerfile + +FROM airbyte/python-connector-base:1.1.0 + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] +``` + +Please use this as an example. This is not optimized. + +Build your image: ```bash -docker build . -t airbyte/source-python-http-example:dev +docker build . -t airbyte/source-example-python:dev ``` + You're done. Stop the clock :\) ## Further reading diff --git a/docs/connector-development/tutorials/cdk-tutorial-python-http/getting-started.md b/docs/connector-development/tutorials/cdk-tutorial-python-http/getting-started.md index 6a813f32895a..dce0f253bbec 100644 --- a/docs/connector-development/tutorials/cdk-tutorial-python-http/getting-started.md +++ b/docs/connector-development/tutorials/cdk-tutorial-python-http/getting-started.md @@ -14,7 +14,7 @@ All the commands below assume that `python` points to a version of python >=3 ## Exchange Rates API Setup -For this guide we will be making API calls to the Exchange Rates API. In order to generate the API access key that will be used by the new connector, you will have to follow steps on the [Exchange Rates API](https://exchangeratesapi.io/) by signing up for the Free tier plan. Once you have an API access key, you can continue with the guide. +For this guide we will be making API calls to the Exchange Rates API. In order to generate the API access key that will be used by the new connector, you will have to follow steps on the [Exchange Rates Data API](https://apilayer.com/marketplace/exchangerates_data-api/) by signing up for the Free tier plan. Once you have an API access key, you can continue with the guide. ## Checklist diff --git a/docs/connector-development/tutorials/cdk-tutorial-python-http/read-data.md b/docs/connector-development/tutorials/cdk-tutorial-python-http/read-data.md index 711880cb0460..8cdee893e5ab 100644 --- a/docs/connector-development/tutorials/cdk-tutorial-python-http/read-data.md +++ b/docs/connector-development/tutorials/cdk-tutorial-python-http/read-data.md @@ -132,7 +132,7 @@ To add incremental sync, we'll do a few things: 6. Update the `path` method to specify the date to pull exchange rates for. 7. Update the configured catalog to use `incremental` sync when we're testing the stream. -We'll describe what each of these methods do below. Before we begin, it may help to familiarize yourself with how incremental sync works in Airbyte by reading the [docs on incremental](../../../understanding-airbyte/connections/incremental-append.md). +We'll describe what each of these methods do below. Before we begin, it may help to familiarize yourself with how incremental sync works in Airbyte by reading the [docs on incremental](/using-airbyte/core-concepts/sync-modes/incremental-append.md). To keep things concise, we'll only show functions as we edit them one by one. diff --git a/docs/connector-development/tutorials/cdk-tutorial-python-http/use-connector-in-airbyte.md b/docs/connector-development/tutorials/cdk-tutorial-python-http/use-connector-in-airbyte.md index 19f204275b50..db190ea87d3e 100644 --- a/docs/connector-development/tutorials/cdk-tutorial-python-http/use-connector-in-airbyte.md +++ b/docs/connector-development/tutorials/cdk-tutorial-python-http/use-connector-in-airbyte.md @@ -1,6 +1,46 @@ # Step 7: Use the Connector in Airbyte -To use your connector in your own installation of Airbyte, build the docker image for your container by running `docker build . -t airbyte/source-python-http-example:dev`. Then, follow the instructions from the [building a Python source tutorial](../building-a-python-source.md#step-11-add-the-connector-to-the-api-ui) for using the connector in the Airbyte UI, replacing the name as appropriate. +To use your connector in your own installation of Airbyte you have to build the docker image for your connector. + + + +**Option A: Building the docker image with `airbyte-ci`** + +This is the preferred method for building and testing connectors. + +If you want to open source your connector we encourage you to use our [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) tool to build your connector. +It will not use a Dockerfile but will build the connector image from our [base image](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/base_images/README.md) and use our internal build logic to build an image from your Python connector code. + +Running `airbyte-ci connectors --name source- build` will build your connector image. +Once the command is done, you will find your connector image in your local docker host: `airbyte/source-:dev`. + + + +**Option B: Building the docker image with a Dockerfile** + +If you don't want to rely on `airbyte-ci` to build your connector, you can build the docker image using your own Dockerfile. This method is not preferred, and is not supported for certified connectors. + +Create a `Dockerfile` in the root of your connector directory. The `Dockerfile` should look something like this: +```Dockerfile + +FROM airbyte/python-connector-base:1.1.0 + +COPY . ./airbyte/integration_code +RUN pip install ./airbyte/integration_code + +# The entrypoint and default env vars are already set in the base image +# ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +# ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] +``` + +Please use this as an example. This is not optimized. + +Build your image: +```bash +docker build . -t airbyte/source-example-python:dev +``` + +Then, follow the instructions from the [building a Python source tutorial](../building-a-python-source.md#step-11-add-the-connector-to-the-api-ui) for using the connector in the Airbyte UI, replacing the name as appropriate. Note: your built docker image must be accessible to the `docker` daemon running on the Airbyte node. If you're doing this tutorial locally, these instructions are sufficient. Otherwise you may need to push your Docker image to Dockerhub. diff --git a/docs/contributing-to-airbyte/README.md b/docs/contributing-to-airbyte/README.md index b8dda6f8b28c..59b7d3e59599 100644 --- a/docs/contributing-to-airbyte/README.md +++ b/docs/contributing-to-airbyte/README.md @@ -4,45 +4,52 @@ description: 'We love contributions to Airbyte, big or small.' # Contributing to Airbyte -Thank you for your interest in contributing! We love community contributions. +Thank you for your interest in contributing! We love community contributions. Read on to learn how to contribute to Airbyte. We appreciate first time contributors and we are happy to assist you in getting started. In case of questions, just reach out to us via [email](mailto:hey@airbyte.io) or [Slack](https://slack.airbyte.io)! -Before getting started, please review Airbyte's Code of Conduct. Everyone interacting in Slack, codebases, mailing lists, events, or other Airbyte activities is expected to follow [Code of Conduct](../project-overview/code-of-conduct.md). +Before getting started, please review Airbyte's Code of Conduct. Everyone interacting in Slack, codebases, mailing lists, events, or other Airbyte activities is expected to follow [Code of Conduct](../community/code-of-conduct.md). ## Code Contributions -Most of the issues that are open for contributions are tagged with `good first issue` or `help-welcome`. +Most of the issues that are open for contributions are tagged with `good first issue` or `help-welcome`. A great place to start looking will be our GitHub projects for: [**Community Connector Issues Project**](https://github.com/orgs/airbytehq/projects/50) -Due to project priorities, we may not be able to accept all contributions at this time. -We are prioritizing the following contributions: -* Bug fixes, features, and enhancements to existing API source connectors -* New connector sources built with the Low-Code CDK and Connector Builder, as these connectors are easier to maintain. -* Bug fixes, features, and enhancements to the following database sources: MongoDB, Postgres, MySQL, MSSQL -* Bug fixes to the following destinations: BigQuery, Snowflake, Redshift, S3, and Postgres -* Helm Charts features, bug fixes, and other platform bug fixes +Due to project priorities, we may not be able to accept all contributions at this time. +We are prioritizing the following contributions: +* Bug fixes, features, and enhancements to existing API source connectors. +* Migrating Python CDK to Low-code or No-Code Framework. +* New connector sources built with the Low-Code CDK or Connector Builder, as these connectors are easier to maintain. +* Bug fixes, features, and enhancements to the following database sources: Postgres, MySQL, MSSQL. +* Bug fixes to the following destinations: BigQuery, Snowflake, Redshift, S3, and Postgres. +* Helm Charts features, bug fixes, and other platform bug fixes. +:::warning +Airbyte is undergoing a major revamp of the shared core Java destinations codebase, with plans to release a new CDK in 2024. +We are actively working on improving usability, speed (through asynchronous loading), and implementing [Typing and Deduplication](/using-airbyte/core-concepts/typing-deduping) (Destinations V2). +For this reason, Airbyte is not reviewing/accepting new Java connectors for now. +::: :::warning Contributions outside of these will be evaluated on a case-by-case basis by our engineering team. ::: The usual workflow of code contribution is: -1. Fork the Airbyte repository -2. Clone the repository locally -3. Make changes and commit them -4. Push your local branch to your fork -5. Submit a Pull Request so that we can review your changes -6. [Link an existing Issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue) without `needs triage` label to your Pull Request (PR without this will be closed) -7. Write a commit message -8. An Airbyte maintainer will trigger the CI tests for you and review the code -9. Update the comments and review -10. Merge the contribution - -Pull Request reviews are done on a regular basis. +1. Fork the Airbyte repository. +2. Clone the repository locally. +3. Create a branch for your feature/bug fix with the format `{YOUR_USERNAME}/{FEATURE/BUG}` (e.g. `jdoe/source-stock-api-stream-fix`) +4. Make and commit changes. +5. Push your local branch to your fork. +6. Submit a Pull Request so that we can review your changes. +7. [Link an existing Issue](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue) that does not include the `needs triage` label to your Pull Request. A pull request without a linked issue will be closed, otherwise. +8. Write a PR title and description that follows the [Pull Request Handbook](./resources/pull-requests-handbook.md) and [Pull Request Template](https://github.com/airbytehq/airbyte/blob/master/.github/pull_request_template.md). +9. An Airbyte maintainer will trigger the CI tests for you and review the code. +10. Review and respond to feedback and questions by Airbyte maintainers. +11. Merge the contribution. + +Pull Request reviews are done on a regular basis. :::info Please make sure you respond to our feedback/questions and sign our CLA. @@ -58,13 +65,13 @@ Guidelines to common code contributions: We welcome Pull Requests that enhance the grammar, structure, or fix typos in our documentation. -- Check the [guidelines](writing-docs.md) to submit documentation changes +- Check the [Updating Documentation](writing-docs.md) guide for submitting documentation changes. ## Community Content We welcome contributions as new tutorials / showcases / articles, or to any of the existing guides on our tutorials page. -We have a repo dedicated to community content. Everything is documented [there](https://github.com/airbytehq/community-content/). +We have a repo dedicated to community content: [Write for the Community](https://github.com/airbytehq/write-for-the-community). Feel free to submit a pull request in this repo, if you have something to add even if it's not related to anything mentioned above. @@ -72,6 +79,6 @@ Feel free to submit a pull request in this repo, if you have something to add ev Another crucial way to contribute is by reporting bugs and helping other users in the community. -You're welcome to enter the Community Slack and help other users or report bugs in Github. +You're welcome to enter the [Community Slack](https://slack.airbyte.io) and help other users or report bugs in Github. -- How to report a bug [guideline](issues-and-requests.md) +- Refer to the [Issues and Requests](issues-and-requests.md) guide to learn about reporting bugs. diff --git a/docs/contributing-to-airbyte/change-cdk-connector.md b/docs/contributing-to-airbyte/change-cdk-connector.md index f4becce2492f..ffb3c5151983 100644 --- a/docs/contributing-to-airbyte/change-cdk-connector.md +++ b/docs/contributing-to-airbyte/change-cdk-connector.md @@ -55,7 +55,7 @@ When we review, we look at: ## Breaking Changes to Connectors -Often times, changes to connectors can be made without impacting the user experience.  However, there are some changes that will require users to take action before they can continue to sync data.  These changes are considered **Breaking Changes** and require a +Often times, changes to connectors can be made without impacting the user experience.  However, there are some changes that will require users to take action before they can continue to sync data.  These changes are considered **Breaking Changes** and require: 1. A **Major Version** increase  2. A [`breakingChanges` entry](https://docs.airbyte.com/connector-development/connector-metadata-file/) in the `releases` section of the `metadata.yaml` file @@ -66,7 +66,12 @@ Often times, changes to connectors can be made without impacting the user experi A breaking change is any change that will require users to take action before they can continue to sync data. The following are examples of breaking changes: - **Spec Change** - The configuration required by users of this connector have been changed and syncs will fail until users reconfigure or re-authenticate.  This change is not possible via a Config Migration  -- **Schema Change** - The type of a property previously present within a record has changed +- **Schema Change** - The type of property previously present within a record has changed - **Stream or Property Removal** - Data that was previously being synced is no longer going to be synced. - **Destination Format / Normalization Change** - The way the destination writes the final data or how normalization cleans that data is changing in a way that requires a full-refresh. -- **State Changes** - The format of the source’s state has changed, and the full dataset will need to be re-synced \ No newline at end of file +- **State Changes** - The format of the source’s state has changed, and the full dataset will need to be re-synced + +### Limiting the Impact of Breaking Changes +Some of the changes listed above may not impact all users of the connector. For example, a change to the schema of a specific stream only impacts users who are syncing that stream. + +The breaking change metadata allows you to specify narrowed scopes that are specifically affected by a breaking change. See the [`breakingChanges` entry](https://docs.airbyte.com/connector-development/connector-metadata-file/) documentation for supported scopes. diff --git a/docs/contributing-to-airbyte/resources/code-formatting.md b/docs/contributing-to-airbyte/resources/code-formatting.md new file mode 100644 index 000000000000..f2e4ab359fa5 --- /dev/null +++ b/docs/contributing-to-airbyte/resources/code-formatting.md @@ -0,0 +1,38 @@ +# Code formatting + +## Tools + +### 🐍 Python +We format our Python code using: +* [Black](https://github.com/psf/black) for code formatting +* [isort](https://pycqa.github.io/isort/) for import sorting + +Our configuration for both tools is in the [pyproject.toml](https://github.com/airbytehq/airbyte/blob/master/pyproject.toml) file. + +### ☕ Java +We format our Java code using [Spotless](https://github.com/diffplug/spotless). +Our configuration for Spotless is in the [spotless-maven-pom.xml](https://github.com/airbytehq/airbyte/blob/master/spotless-maven-pom.xml) file. + +### Json and Yaml +We format our Json and Yaml files using [prettier](https://prettier.io/). + +## Pre-push hooks and CI +We wrapped all our code formatting tools in [airbyte-ci](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md). + +### Local formatting +You can run `airbyte-ci format fix all` to format all the code in the repository. +We wrapped this command in a pre-push hook so that you can't push code that is not formatted. + +To install the pre-push hook, run: +```bash +make tools.pre-commit.setup +``` +This will install `airbyte-ci` and the pre-push hook. + +The pre-push hook runs formatting on all the repo files. +If the hook attempts to format a file that is not part of your contribution, it means that formatting is also broken in the master branch. Please open a separate PR to fix the formatting in the master branch. + +### CI checks +In the CI we run the `airbyte-ci format check all` command to check that all the code is formatted. +If it is not, the CI will fail and you will have to run `airbyte-ci format fix all` locally to fix the formatting issues. +Failure on the CI is not expected if you installed the pre-push hook. diff --git a/docs/contributing-to-airbyte/resources/developing-locally.md b/docs/contributing-to-airbyte/resources/developing-locally.md index 2dee4f5de14e..3ca71c9dbd4f 100644 --- a/docs/contributing-to-airbyte/resources/developing-locally.md +++ b/docs/contributing-to-airbyte/resources/developing-locally.md @@ -68,7 +68,7 @@ A good rule of thumb is to set this to \(\# of cores - 1\). On Mac, if you run into an error while compiling openssl \(this happens when running pip install\), you may need to explicitly add these flags to your bash profile so that the C compiler can find the appropriate libraries. -```text +```bash export LDFLAGS="-L/usr/local/opt/openssl/lib" export CPPFLAGS="-I/usr/local/opt/openssl/include" ``` @@ -107,9 +107,11 @@ In your local `airbyte` repository, run the following command: ``` - Then, build the connector image: -``` -docker build ./airbyte-integrations/connectors/ -t airbyte/:dev -``` + - Install our [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) tool to build your connector. + - Running `airbyte-ci connectors --name source- build` will build your connector image. + - Once the command is done, you will find your connector image in your local docker host: `airbyte/source-:dev`. + + :::info @@ -131,7 +133,7 @@ Now when you run a sync with that connector, it will use your local docker image In your local `airbyte-platform` repository, run the following commands to run acceptance \(end-to-end\) tests for the platform: - +```bash SUB_BUILD=PLATFORM ./gradlew clean build SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:acceptanceTests ``` diff --git a/docs/contributing-to-airbyte/resources/gradle.md b/docs/contributing-to-airbyte/resources/gradle.md index 2b061a2c2c42..84adf04695da 100644 --- a/docs/contributing-to-airbyte/resources/gradle.md +++ b/docs/contributing-to-airbyte/resources/gradle.md @@ -1,4 +1,4 @@ -# Gradle Cheatsheet +# (DEPRECATED) Gradle Cheatsheet ## Overview @@ -135,7 +135,7 @@ Unit Tests can be run using the `:test` task on any submodule. These test class- We split Acceptance Tests into 2 different test suites: -* Platform Acceptance Tests: These tests are a coarse test to sanity check that each major feature in the platform. They are run with the following command: `SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:acceptanceTests`. These tests expect to find a local version of Airbyte running. For testing the docker version start Airbyte locally. For an example, see the [acceptance_test script](../../tools/bin/acceptance_test.sh) that is used by the CI. For Kubernetes, see the [acceptance_test_helm script](../../tools/bin/acceptance_test_kube_helm.sh) that is used by the CI. +* Platform Acceptance Tests: These tests are a coarse test to sanity check that each major feature in the platform. They are run with the following command: `SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:acceptanceTests`. These tests expect to find a local version of Airbyte running. For testing the docker version start Airbyte locally. For an example, see the [acceptance_test script](https://github.com/airbytehq/airbyte-platform/blob/main/tools/bin/acceptance_test.sh) that is used by the CI. For Kubernetes, see the [acceptance_test_helm script](https://github.com/airbytehq/airbyte-platform/blob/main/tools/bin/acceptance_test_kube_helm.sh) that is used by the CI. * Migration Acceptance Tests: These tests make sure the end-to-end process of migrating from one version of Airbyte to the next works. These tests are run with the following command: `SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan`. These tests do not expect there to be a separate deployment of Airbyte running. These tests currently all live in [airbyte-tests](https://github.com/airbytehq/airbyte/airbyte-tests) @@ -223,22 +223,23 @@ The TOML file consists of 4 major sections: - the [bundles] section is used to declare dependency bundles - the [plugins] section is used to declare plugins -> TOML file Example: -> ```gradle -> [versions] -> groovy = "3.0.5" -> -> [libraries] -> groovy-core = { module = "org.codehaus.groovy:groovy", version.ref = "groovy" } -> -> [bundles] -> groovy = ["groovy-core", "groovy-json", "groovy-nio"] -> -> [plugins] -> jmh = { id = "me.champeau.jmh", version = "0.6.5" } -> ``` -> NOTE: for more information please follow [this](https://docs.gradle.org/current/userguide/platforms.html#:~:text=The%20version%20catalog%20TOML%20file%20format -) link. +TOML file Example: + +```gradle +[versions] +groovy = "3.0.5" + +[libraries] +groovy-core = { module = "org.codehaus.groovy:groovy", version.ref = "groovy" } + +[bundles] +groovy = ["groovy-core", "groovy-json", "groovy-nio"] + +[plugins] +jmh = { id = "me.champeau.jmh", version = "0.6.5" } +``` + +NOTE: for more information please follow [this](https://docs.gradle.org/current/userguide/platforms.html#:~:text=The%20version%20catalog%20TOML%20file%20format) link. As described above this project contains TOML file `deps.toml` which is fully fulfilled with respect to [official](https://docs.gradle.org/current/userguide/platforms.html#sub::toml-dependencies-format) documentation. In case when new versions should be used please update `deps.toml` accordingly. @@ -246,6 +247,7 @@ In case when new versions should be used please update `deps.toml` accordingly.
      deps.toml +``` [versions] fasterxml_version = "2.13.0" glassfish_version = "2.31" @@ -293,37 +295,41 @@ apache = ["apache-commons", "apache-commons-lang"] log4j = ["log4j-api", "log4j-core", "log4j-impl", "log4j-web"] slf4j = ["jul-to-slf4j", "jcl-over-slf4j", "log4j-over-slf4j"] junit = ["junit-jupiter-api", "junit-jupiter-params", "mockito-junit-jupiter"] +```
      #### Declaring a version catalog Version catalogs can be declared in the settings.gradle file. There should be specified section `dependencyResolutionManagement` which uses `deps.toml` file as a declared catalog. -> Example: -> ```gradle -> dependencyResolutionManagement { -> repositories { -> maven { -> url 'https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/' -> } -> } -> versionCatalogs { -> libs { -> from(files("deps.toml")) -> } -> } -> } -> ``` +Example: + +```gradle +dependencyResolutionManagement { + repositories { + maven { + url 'https://airbyte.mycloudrepo.io/public/repositories/airbyte-public-jars/' + } + } + versionCatalogs { + libs { + from(files("deps.toml")) + } + } +} +``` #### Sharing Catalogs To share this catalog for further usage by other Projects, we do the following 2 steps: - Define `version-catalog` plugin in `build.gradle` file (ignore if this record exists) + ```gradle plugins { id '...' id 'version-catalog' ``` - Prepare Catalog for Publishing + ```gradle catalog { versionCatalog { @@ -334,6 +340,7 @@ To share this catalog for further usage by other Projects, we do the following 2 #### Configure the Plugin Publishing Plugin To **Publishing**, first define the `maven-publish` plugin in `build.gradle` file (ignore if this already exists): + ```gradle plugins { id '...' @@ -341,29 +348,30 @@ plugins { } ``` After that, describe the publishing section. Please use [this](https://docs.gradle.org/current/userguide/publishing_gradle_plugins.html) official documentation for more details. -> Example: -> ```gradle -> publishing { -> publications { -> maven(MavenPublication) { -> groupId = 'io.airbyte' -> artifactId = 'oss-catalog' -> -> from components.versionCatalog -> } -> } -> -> repositories { -> maven { -> url 'https://airbyte.mycloudrepo.io/repositories/airbyte-public-jars' -> credentials { -> name 'cloudrepo' -> username System.getenv('CLOUDREPO_USER') -> password System.getenv('CLOUDREPO_PASSWORD') -> } -> } -> -> mavenLocal() -> } -> } -> ``` +Example: + +```gradle +publishing { + publications { + maven(MavenPublication) { + groupId = 'io.airbyte' + artifactId = 'oss-catalog' + + from components.versionCatalog + } + } + + repositories { + maven { + url 'https://airbyte.mycloudrepo.io/repositories/airbyte-public-jars' + credentials { + name 'cloudrepo' + username System.getenv('CLOUDREPO_USER') + password System.getenv('CLOUDREPO_PASSWORD') + } + } + + mavenLocal() + } +} +``` diff --git a/docs/contributing-to-airbyte/resources/pull-requests-handbook.md b/docs/contributing-to-airbyte/resources/pull-requests-handbook.md index b4517652e67e..b38d7606bb52 100644 --- a/docs/contributing-to-airbyte/resources/pull-requests-handbook.md +++ b/docs/contributing-to-airbyte/resources/pull-requests-handbook.md @@ -43,6 +43,7 @@ When creating or updating connectors, we spend a lot of time manually transcribi Changes to connector behavior should always be accompanied by a version bump and a changelog entry. We use [semantic versioning](https://semver.org/) to version changes to connectors. Since connectors are a bit different from APIs, we have our own take on semantic versioning, focusing on maintaining the best user experience of using a connector. - Major: a version in which a change is made which requires manual intervention (update to config or configured catalog) for an existing connection to continue to succeed, or one in which data that was previously being synced will no longer be synced + - Note that a category of "user intervention" is a schema change in the destination, as users will be required to update downstream reports and tools. A change that leads to a differnt final table in the destination is a breaking change - Minor: a version that introduces user-facing functionality in a backwards compatible manner - Patch: a version that introduces backwards compatible bug fixes or performance improvements diff --git a/docs/contributing-to-airbyte/resources/python-gradle-setup.md b/docs/contributing-to-airbyte/resources/python-gradle-setup.md index 37740a3765bb..34ef885a9f64 100644 --- a/docs/contributing-to-airbyte/resources/python-gradle-setup.md +++ b/docs/contributing-to-airbyte/resources/python-gradle-setup.md @@ -1,4 +1,4 @@ -# Monorepo Python Development +# (DEPRECATED) Monorepo Python Development This guide contains instructions on how to setup Python with Gradle within the Airbyte Monorepo. If you are a contributor working on one or two connectors, this page is most likely not relevant to you. Instead, you should use your standard Python development flow. @@ -30,21 +30,7 @@ python tools/bin/update_intellij_venv.py --all-modules --install-venv This will create a `virtualenv` and install dependencies for the connector you want to work on as well as any internal Airbyte python packages it depends on. -When iterating on a single connector, you will often iterate by running -```text -./gradlew :airbyte-integrations:connectors:your-connector-dir:build -``` - -This command will: - -1. Install a virtual environment at `airbyte-integrations/connectors//.venv` -2. Install local development dependencies specified in `airbyte-integrations/connectors/your-connector-dir/requirements.txt` -3. Runs the following pip modules: - 1. [Black](https://pypi.org/project/black/) to lint the code - 2. [isort](https://pypi.org/project/isort/) to sort imports - 3. [Flake8](https://pypi.org/project/flake8/) to check formatting - 4. [MyPy](https://pypi.org/project/mypy/) to check type usage ## Formatting/linting diff --git a/docs/contributing-to-airbyte/writing-docs.md b/docs/contributing-to-airbyte/writing-docs.md index b3a927b36dea..3f2f20926547 100644 --- a/docs/contributing-to-airbyte/writing-docs.md +++ b/docs/contributing-to-airbyte/writing-docs.md @@ -1,3 +1,6 @@ +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + # Updating Documentation We welcome contributions to the Airbyte documentation! @@ -13,10 +16,10 @@ The Docs team maintains a list of [#good-first-issues](https://github.com/airbyt ## Contributing to Airbyte docs -Before contributing to Airbyte docs, read the Airbyte Community +Before contributing to Airbyte docs, read the Airbyte Community [Code of Conduct](../community/code-of-conduct.md). :::tip -If you're new to GitHub and Markdown, complete [the First Contributions tutorial](https://github.com/firstcontributions/first-contributions) and learn [Markdown basics](https://guides.github.com/features/mastering-markdown/) before contributing to Airbyte documentation. +If you're new to GitHub and Markdown, complete [the First Contributions tutorial](https://github.com/firstcontributions/first-contributions) and learn [Markdown basics](https://guides.github.com/features/mastering-markdown/) before contributing to Airbyte documentation. Even if you're familiar with the basics, you may be interested in Airbyte's [custom markdown extensions for connector docs](#custom-markdown-extensions-for-connector-docs). ::: You can contribute to Airbyte docs in two ways: @@ -55,22 +58,22 @@ To make complex changes or edit multiple files, edit the files on your local mac ```bash cd docusaurus - yarn install + pnpm install ``` To see changes as you make them, run: ```bash - yarn start + pnpm start ``` - Then navigate to [http://localhost:3000/](http://localhost:3000/). Whenever you make and save changes, you will see them reflected in the server. You can stop the running server in OSX/Linux by pressing `Ctrl-C` in the terminal. + Then navigate to [http://localhost:3005/](http://localhost:3005/). Whenever you make and save changes, you will see them reflected in the server. You can stop the running server in OSX/Linux by pressing `Ctrl-C` in the terminal. You can also build the docs locally and see the resulting changes. This is useful if you introduce changes that need to be run at build-time (e.g. adding a docs plug-in). To do so, run: ```bash - yarn build - yarn serve + pnpm build + pnpm serve ``` Then navigate to [http://localhost:3000/](http://localhost:3000/) to see your changes. You can stop the running server in OSX/Linux by pressing `Ctrl-C` in the terminal. @@ -84,6 +87,243 @@ To make complex changes or edit multiple files, edit the files on your local mac 5. Assign `airbytehq/docs` as a Reviewer for your pull request. +### Custom markdown extensions for connector docs +Airbyte's markdown documentation—particularly connector-specific documentation—needs to gracefully support multiple different contexts: key details may differ between open-source builds and Airbyte Cloud, and the more exhaustive explanations appropriate for https://docs.airbyte.com may bury key details when rendered as inline documentation within the Airbyte application. In order to support all these different contexts without resorting to multiple overlapping files that must be maintained in parallel, Airbyte's documentation tooling supports multiple nonstandard features. + +Please familiarize yourself with all the tools available to you when writing documentation for a connector, so that you can provide appropriately tailored information to your readers in whichever context they see it. + +:::note +As a general rule, features that introduce new behavior or prevent certain content from rendering will affect how the Airbyte UI displays markdown content, but have no impact on https://docs.airbyte.com. If you want to test out these in-app features in [a local Airbyte build](https://docs.airbyte.com/contributing-to-airbyte/resources/developing-locally/#develop-on-airbyte-webapp), ensure that you have the `airbyte` git repository checked out to the same parent directory as the airbyte platform repository: if so, development builds will by default fetch connector documentation from your local filesystem, allowing you to freely edit their content and view the rendered output. +::: + +#### Select between mutually-exclusive content options with `` +Tabs are a built-in feature of Docusaurus, the tool we use to build `https://docs.airbyte.com`; please refer to [their documentation](https://docusaurus.io/docs/markdown-features/tabs) for their options and behavior in this context. For better site-agnostic documentation, and because we like the feature, we maintain a separate `Tabs` implementation with limited, one-way API compatibility: all usage options we document should behave the same in-app and on `https://docs.airbyte.com`. If you find a discrepancy or breakage, we would appreciate if you [report it as a bug](https://github.com/airbytehq/airbyte/issues/new?assignees=&labels=type%2Fenhancement%2Carea%2Fdocumentation+needs-triage&projects=&template=8-documentation.yaml)! The reverse is not necessarily true, however: Docusaurus supports many use cases besides ours, so supporting its every usage pattern is a deliberate non-goal. + +:::info +Because Docusaurus uses an mdx component, you must include the following import lines in any markdown file which uses tabs: + +```js +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; +``` + +This is not optional: if these lines are missing, the documentation site will have errors. They won't show up in the rendered document, however. +::: + +Here's an example set of tabs; note that you can put any number of `...` tags inside a ``. + +```md + + + +When configuring this hypothetical connector using basic HTTP auth, you should mind some tricky security considerations! This just a hypothetical, though, so I never bothered to come up with any. + +As the first tab, this would be shown by default if no `TabItem` were marked as `default`. + + + + +When configuring this hypothetical connector using OAuth authentication, you should do a dance. Good for you! Since it's not the first `TabItem` in its set, we had to explicitly mark this tab as `default` for it to get top billing. + + + +``` + +That renders as the following: + + + + +When configuring this hypothetical connector using basic HTTP auth, you should mind some tricky security considerations! This just a hypothetical, though, so I never bothered to come up with any. + +As the first tab, this would be shown by default if no `TabItem` were marked as `default`. + + + + +When configuring this hypothetical connector using OAuth authentication, you should do a dance. Good for you! Since it's not the first `TabItem` in its set, we had to explicitly mark this tab as `default` for it to get top billing. + + + + +- You don't need to mark any tab as `default` +- If you don't, the first tab (here, Basic HTTP) will be the initial selection instead +- You can use ordinary markdown syntax inside a `TabItem` +- **however**, due to bugs in our in-app markdown rendering library, you should be dilligent about using empty lines to separate different formatting-related things (surrounding tags and their contents, paragraphs vs lists, etc) +- You should also avoid indenting `TabItem` tags and their content according to html conventions, since text indented by four spaces (common for html nested inside two levels of tags) can be interpreted as a code block; different markdown rendering tools can handle this inconsistently. + +#### Jump to the relevant documentation section when specific connector setup inputs are focused with `` +In the documentation, the relevant section needs to be wrapped in a `` component. When a user focuses the field identified by the `field` attribute in the connector setup UI, the documentation pane will automatically scroll to the associated section of the documentation, highlighting all content contained inside the `` tag. These are rendered as regular divs in the documentation site, so they have no effect in places other than the in-app documentation panel—however, note that there must be blank lines between a custom tag like `FieldAnchor` the content it wraps for the documentation site to render markdown syntax inside the custom tag to html. + +The `field` attribute must be a valid json path to one of the properties nested under `connectionSpecification.properties` in that connector's `spec.json` or `spec.yaml` file. For example, if the connector spec contains a `connectionSpecification.properties.replication_method.replication_slot`, you would mark the start of the related documentation section with `` and its end with ``. It's also possible to highlight the same section for multiple fields by separating them with commas, like ``. To mark a section as highlighted after the user picks an option from a `oneOf`: use a `field` prop like `path.to.field[value-of-selection-key]`, where the `value-of-selection-key` is the value of a `const` field nested inside that `oneOf`. For example, if the specification of the `oneOf` field is: + +```json +"replication_method": { + "type": "object", + "title": "Update Method", + "oneOf": [ + { + "title": "Read Changes using Binary Log (CDC)", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "CDC", + "order": 0 + }, + "initial_waiting_seconds": { + "type": "integer", + "title": "Initial Waiting Time in Seconds (Advanced)", + }, + } + }, + { + "title": "Scan Changes with User Defined Cursor", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "STANDARD", + "order": 0 + } + } + } + ] +} +``` + +The selection keys are `CDC` and `STANDARD`, so you can wrap a specific replication method's documentation section with a `...` tag, and it will be highlighted if the user selects CDC replication in the UI. + +:::tip +Because of their close connection with the connector setup form fields, `` tags are only enabled for the source and destination setup pages. +::: + +#### Prevent specific content from rendering in the UI with `` +Certain content is important to document, but unhelpful in the context of the Airbyte UI's inline documentation views: +- background information that helps users understand a connector but doesn't affect configuration +- edge cases that are unusual but time-consuming to solve +- context for readers on the documentation site about environment-specific content (see [below](#environment-specific-in-app-content-with-magic-html-comments)) + +Wrapping such content in a pair of `...` tags will prevent it from being rendered within the Airbyte UI without affecting its presentation on https://docs.airbyte.com. This allows a single markdown file to be the source of truth for both a streamlined in-app reference and a more thorough treatment on the documentation website. + +#### Environment-specific in-app content with magic html comments +Sometimes, there are connector setup instructions which differ between open-source Airbyte builds and Airbyte Cloud. Document both cases, but wrap each in a pair of special HTML comments: +```md + + + +## For open source: + + + +Only open-source builds of the Airbyte UI will render this content. + + + + + +## For Airbyte Cloud: + + + +Only cloud builds of the Airbyte UI will render this content. + + +Content outside of the magic-comment-delimited blocks will be rendered everywhere. +``` +Note that the documentation site will render _all_ environment-specific content, so please introduce environment-specific variants with some documentation-site-only context (like the hidden subheadings in the example above) to disambiguate. + +#### Contextually-styled callouts with admonition blocks +We have added support for [Docusaurus' admonition syntax](https://docusaurus.io/docs/markdown-features/admonitions) to Airbyte's in-app markdown renderer. + +To make an admonition, wrap text with lines of three colons, with the first colons immediately followed (no space) by a tag specifying the callout's semantic styling, which will be one of `tip`, `warning`, `caution`, `danger`, `note`, or `info`. The syntax parallells a code block's, but with colons instead of backticks. + +Examples of the different admonition types: + +```md +:::note + +A **note** with _Markdown_ `syntax`. + +::: +``` + +:::note + +A **note** with _Markdown_ `syntax`. + +::: + +```md +:::tip + +A **tip** with _Markdown_ `syntax`. + +::: +``` + +:::tip + +A **tip** with _Markdown_ `syntax`. + +::: + +```md +:::info + +Some **info** with _Markdown_ `syntax`. + +::: +``` + +:::info + +Some **info** with _Markdown_ `syntax`. + +::: + +```md +:::caution + +A **caution** with _Markdown_ `syntax`. + +::: +``` + +:::caution + +A **caution** with _Markdown_ `syntax`. + +::: + +```md +:::danger + +Some **dangerous** content with _Markdown_ `syntax`. + +::: +``` + +:::danger + +Some **dangerous** content with _Markdown_ `syntax`. + +::: + +#### Collapsible content with `
      ` and `` + +```md +## Ordinary markdown content + +
      + Here is an expandible section! Everything but this title is hidden by default. + Here is the dropdown content; if users expand this section, they will be able to read your valuable but perhaps nonessential content. +
      + +Back to ordinary markdown content. +``` +Eagle-eyed readers may note that _all_ markdown should support this feature since it's part of the html spec. However, it's worth special mention since these dropdowns have been styled to be a graceful visual fit within our rendered documentation in all environments. + ## Additional guidelines - If you're updating a connector doc, follow the [Connector documentation template](https://hackmd.io/Bz75cgATSbm7DjrAqgl4rw) @@ -99,16 +339,7 @@ To make complex changes or edit multiple files, edit the files on your local mac ### Adding a redirect -To add a redirect, open the [`docusaurus.config.js`](https://github.com/airbytehq/airbyte/blob/master/docusaurus/docusaurus.config.js#L22) file and locate the following commented section: - -```js -// { -// from: '/some-lame-path', -// to: '/a-much-cooler-uri', -// }, -``` - -Copy this section, replace the values, and [test the changes locally](#editing-on-your-local-machine) by going to the path you created a redirect for and verify that the address changes to the new one. +To add a redirect, open the [`docusaurus/redirects.yml`](https://github.com/airbytehq/airbyte/blob/master/docusaurus/redirects.yml) file and add an entry from which old path to which new path a redirect should happen. :::note Your path **needs** a leading slash `/` to work @@ -149,3 +380,39 @@ cd airbyte git checkout ./tools/bin/deploy_docusaurus ``` + +### Adding a diagram +We have the docusaurus [Mermaid](https://mermaid.js.org/) plugin which has a variety of diagram +types and syntaxes available. + +:::danger + The connector specific docs do **not** currently support this, only use this for general docs. +::: + +Here is an example from the [Mermaid docs](https://mermaid.js.org/syntax/entityRelationshipDiagram.html) +you would add the following to your markdown wrapped in a code block. + +```md + --- + title: Order example + --- + erDiagram + CUSTOMER ||--o{ ORDER : places + ORDER ||--|{ LINE-ITEM : contains + CUSTOMER }|..|{ DELIVERY-ADDRESS : uses +``` + +which produces the following diagram + +```mermaid +--- +title: Order example +--- +erDiagram + CUSTOMER ||--o{ ORDER : places + ORDER ||--|{ LINE-ITEM : contains + CUSTOMER }|..|{ DELIVERY-ADDRESS : uses +``` + +check out the rest of the Mermaid documentation for its capabilities just be aware that not all +the features are available to the docusaurus plugin. diff --git a/docs/deploying-airbyte/README.md b/docs/deploying-airbyte/README.md deleted file mode 100644 index 2f8a6e290a36..000000000000 --- a/docs/deploying-airbyte/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# Deploy Airbyte where you want to - -![not all who wander are lost](https://user-images.githubusercontent.com/2591516/170351002-0d054d06-c901-4794-8719-97569060408f.png) - -- [Local Deployment](local-deployment.md) -- [On Airbyte Cloud](on-cloud.md) -- [On Aws](on-aws-ec2.md) -- [On Azure VM Cloud Shell](on-azure-vm-cloud-shell.md) -- [On Digital Ocean Droplet](on-digitalocean-droplet.md) -- [On GCP.md](on-gcp-compute-engine.md) -- [On Kubernetes](on-kubernetes-via-helm.md) -- [On OCI VM](on-oci-vm.md) -- [On Restack](on-restack.md) -- [On Plural](on-plural.md) -- [On AWS ECS (spoiler alert: it doesn't work)](on-aws-ecs.md) diff --git a/docs/deploying-airbyte/local-deployment.md b/docs/deploying-airbyte/local-deployment.md index 3be4e715984f..d3247a86668c 100644 --- a/docs/deploying-airbyte/local-deployment.md +++ b/docs/deploying-airbyte/local-deployment.md @@ -12,7 +12,7 @@ These instructions have been tested on MacOS, Windows 10 and Ubuntu 22.04. ```bash # clone Airbyte from GitHub -git clone https://github.com/airbytehq/airbyte.git +git clone --depth=1 https://github.com/airbytehq/airbyte.git # switch into Airbyte directory cd airbyte @@ -21,8 +21,8 @@ cd airbyte ./run-ab-platform.sh ``` -- In your browser, just visit [http://localhost:8000](http://localhost:8000) -- You will be asked for a username and password. By default, that's username `airbyte` and password `password`. Once you deploy Airbyte to your servers, be sure to change these: +- In your browser, visit [http://localhost:8000](http://localhost:8000) +- You will be asked for a username and password. By default, that's username `airbyte` and password `password`. Once you deploy Airbyte to your servers, be sure to change these in your `.env` file: ```yaml # Proxy Configuration @@ -56,7 +56,7 @@ Make sure to select the options: **3. You're done!** ```bash -git clone https://github.com/airbytehq/airbyte.git +git clone --depth=1 https://github.com/airbytehq/airbyte.git cd airbyte bash run-ab-platform.sh ``` @@ -66,5 +66,11 @@ bash run-ab-platform.sh - Start moving some data! ## Troubleshooting +If you have any questions about the local setup and deployment process, head over to our [Getting Started FAQ](https://github.com/airbytehq/airbyte/discussions/categories/questions) on our Airbyte Forum that answers the following questions and more: + +- How long does it take to set up Airbyte? +- Where can I see my data once I've run a sync? +- Can I set a start time for my sync? -If you encounter any issues, just connect to our [Slack](https://slack.airbyte.io). Our community will help! We also have a [troubleshooting](../troubleshooting.md) section in our docs for common problems. +If you encounter any issues, check out [Getting Support](/community/getting-support) documentation +for options how to get in touch with the community or us. diff --git a/docs/deploying-airbyte/on-aws-ec2.md b/docs/deploying-airbyte/on-aws-ec2.md index 909bcfa391df..3b352ce59015 100644 --- a/docs/deploying-airbyte/on-aws-ec2.md +++ b/docs/deploying-airbyte/on-aws-ec2.md @@ -41,6 +41,7 @@ sudo usermod -a -G docker $USER sudo yum install -y docker-compose-plugin docker compose version ``` +If you encounter an error on this part, you might prefer to follow the documentation to [install the docker compose plugin manually](https://docs.docker.com/compose/install/linux/#install-the-plugin-manually) (_make sure to do it for all users_). 4. To close the SSH connection, run the following command in your SSH session on the instance terminal: @@ -90,7 +91,7 @@ ssh -i $SSH_KEY -L 8000:localhost:8000 -N -f ec2-user@$INSTANCE_IP ## Get Airbyte logs in CloudWatch -Follow this [guide](https://aws.amazon.com/pt/premiumsupport/knowledge-center/cloudwatch-docker-container-logs-proxy/) to get your logs from your Airbyte Docker containers in CloudWatch. +Follow this [guide](https://aws.amazon.com/en/premiumsupport/knowledge-center/cloudwatch-docker-container-logs-proxy/) to get your logs from your Airbyte Docker containers in CloudWatch. ## Troubleshooting diff --git a/docs/deploying-airbyte/on-kubernetes-via-helm.md b/docs/deploying-airbyte/on-kubernetes-via-helm.md index 1efea587491b..375a59b8576d 100644 --- a/docs/deploying-airbyte/on-kubernetes-via-helm.md +++ b/docs/deploying-airbyte/on-kubernetes-via-helm.md @@ -10,6 +10,10 @@ If you don't want to configure your own Kubernetes cluster and Airbyte instance, Alternatively, you can deploy Airbyte on [Restack](https://www.restack.io) to provision your Kubernetes cluster on AWS. Follow [this guide](on-restack.md) to get started. +:::note +Airbyte running on Self-Hosted Kubernetes doesn't support DBT Transformations. Please refer to [#5901](https://github.com/airbytehq/airbyte/issues/5091) +::: + ## Getting Started ### Cluster Setup @@ -79,17 +83,21 @@ After adding the repo, perform the repo indexing process by running `helm repo u After this you can browse all charts uploaded to repository by running `helm search repo airbyte` -It'll produce the output below: +It'll produce output similar to below: ```text -NAME CHART VERSION APP VERSION DESCRIPTION -airbyte-oss/airbyte 0.30.23 0.39.37-alpha Helm chart to deploy airbyte -airbyte-oss/airbyte-bootloader 0.30.23 0.39.37-alpha Helm chart to deploy airbyte-bootloader -airbyte-oss/pod-sweeper 0.30.23 0.39.37-alpha Helm chart to deploy airbyte-pod-sweeper -airbyte-oss/server 0.30.23 0.39.37-alpha Helm chart to deploy airbyte-server -airbyte-oss/temporal 0.30.23 0.39.37-alpha Helm chart to deploy airbyte-temporal -airbyte-oss/webapp 0.30.23 0.39.37-alpha Helm chart to deploy airbyte-webapp -airbyte-oss/worker 0.30.23 0.39.37-alpha Helm chart to deploy airbyte-worker +NAME CHART VERSION APP VERSION DESCRIPTION +airbyte/airbyte 0.49.9 0.50.33 Helm chart to deploy airbyte +airbyte/airbyte-api-server 0.49.9 0.50.33 Helm chart to deploy airbyte-api-server +airbyte/airbyte-bootloader 0.49.9 0.50.33 Helm chart to deploy airbyte-bootloader +airbyte/connector-builder-server 0.49.9 0.50.33 Helm chart to deploy airbyte-connector-builder-... +airbyte/cron 0.49.9 0.50.33 Helm chart to deploy airbyte-cron +airbyte/metrics 0.49.9 0.50.33 Helm chart to deploy airbyte-metrics +airbyte/pod-sweeper 0.49.9 0.50.33 Helm chart to deploy airbyte-pod-sweeper +airbyte/server 0.49.9 0.50.33 Helm chart to deploy airbyte-server +airbyte/temporal 0.49.9 0.50.33 Helm chart to deploy airbyte-temporal +airbyte/webapp 0.49.9 0.50.33 Helm chart to deploy airbyte-webapp +airbyte/worker 0.49.9 0.50.33 Helm chart to deploy airbyte-worker ``` ## Deploy Airbyte @@ -104,6 +112,8 @@ In order to do so, run the command: helm install %release_name% airbyte/airbyte ``` +**Note**: `release_name` should only contain lowercase letters and optionally dashes (`release_name` must start with a letter). + ### Custom deployment In order to customize your deployment, you need to create `values.yaml` file in the local folder and populate it with default configuration override values. @@ -116,40 +126,6 @@ After specifying your own configuration, run the following command: helm install --values path/to/values.yaml %release_name% airbyte/airbyte ``` -### (Alpha) Airbyte Enterprise deployment - -[Airbyte Enterprise](/airbyte-enterprise) is in early alpha stages, so this section will likely evolve. That said, if you have an Airbyte Enterprise license key and wish to install Airbyte Enterprise via helm, follow these steps: - -1. Checkout the latest revision of the [airbyte-platform repository](https://github.com/airbytehq/airbyte-platform) - -2. Add your Airbyte Enterprise license key and [auth configuration details](/airbyte-enterprise#single-sign-on-sso) to a file called `airbyte.yml` in the root directory of `airbyte-platform`. You can copy `airbyte.sample.yml` to use as a template: - -```text -cp airbyte.sample.yml airbyte.yml -``` - -Then, open up `airbyte.yml` in your text editor to fill in the indicated fields. - -:::caution - -For now, auth configurations aren't easy to modify once initially installed, so please double check them to make sure they're accurate before proceeding! This will be improved in the near future. - -::: - -3. Make sure your helm repository is up to date: - -```text -helm repo update -``` - -4. Install Airbyte Enterprise on helm using the following command: - -```text -RELEASE_NAME=./tools/bin/install_airbyte_pro_on_helm.sh -``` - -If unspecified, the default release name is `airbyte-pro`. You can change this by editing the `install_airbyte_pro_on_helm.sh` script. - ## Migrate from old charts to new ones Starting from `0.39.37-alpha` we've revisited helm charts structure and separated all components of airbyte into their own independent charts, thus by allowing our developers to test single component without deploying airbyte as a whole and by upgrading single component at a time. diff --git a/docs/project-overview/licenses/README.md b/docs/developer-guides/licenses/README.md similarity index 100% rename from docs/project-overview/licenses/README.md rename to docs/developer-guides/licenses/README.md diff --git a/docs/project-overview/licenses/elv2-license.md b/docs/developer-guides/licenses/elv2-license.md similarity index 100% rename from docs/project-overview/licenses/elv2-license.md rename to docs/developer-guides/licenses/elv2-license.md diff --git a/docs/project-overview/licenses/examples.md b/docs/developer-guides/licenses/examples.md similarity index 94% rename from docs/project-overview/licenses/examples.md rename to docs/developer-guides/licenses/examples.md index 5b9305dccee6..0a160a520dbb 100644 --- a/docs/project-overview/licenses/examples.md +++ b/docs/developer-guides/licenses/examples.md @@ -17,5 +17,3 @@ Let's start with the list of projects that falls under ELv2 and for which you ca * Creating an analytics or attribution platform for which you want to use Airbyte to bring data in on behalf of your customers. * Creating any kind of platform on which you offer Airbyte's connectors to your customers to bring their data in, unless you're selling some ELT / ETL solution. * Building your internal data stack and configuring pipelines through Airbyte's UI or API. - -**Contact us on the chat on the bottom right to add your project example!** diff --git a/docs/project-overview/licenses/license-faq.md b/docs/developer-guides/licenses/license-faq.md similarity index 86% rename from docs/project-overview/licenses/license-faq.md rename to docs/developer-guides/licenses/license-faq.md index 837ae5a5fd3d..6865094e4ba4 100644 --- a/docs/project-overview/licenses/license-faq.md +++ b/docs/developer-guides/licenses/license-faq.md @@ -1,16 +1,19 @@ # License FAQ ## Airbyte Licensing Overview -* **Airbyte Connectors** are open sourced and available under the MIT License. -* **Airbyte Protocol** is open sourced and available under the MIT License. -* **Airbyte CDK** (Connector Development Kit) is open sourced and available under the MIT License. -* **Airbyte Core** is licensed under the Elastic License 2.0 (ELv2). -* **Airbyte Cloud & Airbyte Enterprise** are both closed source and require a commercial license from Airbyte. + +- **Airbyte Connectors** are open sourced and available under the [MIT](https://opensource.org/license/mit/) or [Elastic License 2.0 (ELv2)](https://www.elastic.co/licensing/elastic-license/faq) License. Each connector's `metadata.yaml` file contains more information. +- **Airbyte Protocol** is open sourced and available under the MIT License. +- **Airbyte CDK** (Connector Development Kit) is open sourced and available under the MIT License. +- **Airbyte Core** is licensed under the Elastic License 2.0 (ELv2). +- **Airbyte Cloud & Airbyte Enterprise** are both closed source and require a commercial license from Airbyte. ![Diagram of license structure](../../.gitbook/assets/license_faq_diagram.png) ## About Elastic License 2.0 (ELv2) + ELv2 is a simple, non-copyleft license, allowing for the right to “use, copy, distribute, make available, and prepare derivative works of the software”. Anyone can use Airbyte, free of charge. You can run the software at scale on your infrastructure. There are only three high-level limitations. You cannot: + 1. Provide the products to others as a managed service ([read more](#what-is-the-managed-service-use-case-that-is-not-allowed-under-elv2)); 2. Circumvent the license key functionality or remove/obscure features protected by license keys; or 3. Remove or obscure any licensing, copyright, or other notices. @@ -20,60 +23,75 @@ In case you want to work with Airbyte without these limitations, we offer altern [View License](elv2-license.md) ## FAQ + ### What limitations does ELv2 impose on my use of Airbyte? + If you are an Airbyte Cloud customer, nothing changes for you. For open-source users, everyone can continue to use Airbyte as they are doing today: no limitations on volume, number of users, number of connections… There are only a few high-level limitations. You cannot: + 1. Provide the products to others as a managed service. For example, you cannot sell a cloud service that provides users with direct access to Airbyte. You can sell access to applications built and run using Airbyte ([read more](#what-is-the-managed-service-use-case-that-is-not-allowed-under-elv2)). 2. Circumvent the license key functionality or remove/obscure features protected by license keys. For example, our code may contain watermarks or keys to unlock proprietary functionality. Those elements of our code will be marked in our source code. You can’t remove or change them. ### Why did Airbyte adopt ELv2? + We are releasing Airbyte Cloud, a managed version of Airbyte that will offer alternatives to how our users operate Airbyte, including additional features and new execution models. We want to find a great way to execute our mission to commoditize data integration with open source and our ambition to create a sustainable business. -ELv2 gives us the best of both worlds. +ELv2 gives us the best of both worlds. On one hand, our users can continue to use Airbyte freely, and on the other hand, we can safely create a sustainable business and continue to invest in our community, project and product. We don’t have to worry about other large companies taking the product to monetize it for themselves, thus hurting our community. ### Will Airbyte connectors continue to be open source? + Our own connectors remain open-source, and our contributors can also develop their own connectors and continue to choose whichever license they prefer. This is our way to accomplish Airbyte’s vision of commoditizing data integration: access to data shouldn’t be behind a paywall. Also, we want Airbyte’s licensing to work well with applications that are integrated using connectors. We are continuously investing in Airbyte's data protocol and all the tooling around it. The Connector Development Kit (CDK), which helps our community and our team build and maintain connectors at scale, is a cornerstone of our commoditization strategy and also remains open-source. ### How do I continue to contribute to Airbyte under ELv2? + Airbyte’s projects are available here. Anyone can contribute to any of these projects (including those licensed with ELv2). We are introducing a Contributor License Agreement that you will have to sign with your first contribution. ### When will ELv2 be effective? + ELv2 will apply from the following Airbyte core version as of September 27, 2021: version 0.30.0. ### What is the “managed service” use case that is not allowed under ELv2? -We chose ELv2 because it is very permissive with what you can do with the software. + +We chose ELv2 because it is very permissive with what you can do with the software. You can basically build ANY product on top of Airbyte as long as you don’t: -* Host Airbyte yourself and sell it as an ELT/ETL tool, or a replacement for the Airbyte solution. -* Sell a product that directly exposes Airbyte’s UI or API. + +- Host Airbyte yourself and sell it as an ELT/ETL tool, or a replacement for the Airbyte solution. +- Sell a product that directly exposes Airbyte’s UI or API. Here is a non-exhaustive list of what you can do (without providing your customers direct access to Airbyte functionality): -* I am creating an analytics platform and I want to use Airbyte to bring data in on behalf of my customers. -* I am building my internal data stack and I want my team to be able to interact with Airbyte to configure the pipelines through the UI or the API. -* ... + +- I am creating an analytics platform and I want to use Airbyte to bring data in on behalf of my customers. +- I am building my internal data stack and I want my team to be able to interact with Airbyte to configure the pipelines through the UI or the API. +- ... ### My company has a policy against using code that restricts commercial use – can I still use Airbyte under ELv2? -You can use software under ELv2 for your commercial business, you simply cannot offer it as a managed service. + +You can use software under ELv2 for your commercial business, you simply cannot offer it as a managed service. ### As a Data Agency, I currently use Airbyte to fulfill my customer needs. How does ELv2 affect me? + You can continue to use Airbyte, as long as you don’t offer it as a managed service. ### I started to use Airbyte to ingest my customer’s data. What should I do? + You can continue to use Airbyte, as long as you don’t offer it as a managed service. ### Can I customize ELv2 software? + Yes, you can customize ELv2 software. ELv2 is similar in this sense to permissive open-source licenses. You can modify the software, integrate the variant into your application, and operate the modified application, as long as you don’t go against any of the limitations. ### Why didn’t you use a closed-source license for Airbyte Core? + We want to provide developers with free access to our Airbyte Core source code — including rights to modify it. Since this wouldn’t be possible with a closed-source license, we decided to use the more permissive ELv2. ### Is there any revenue sharing for those who create Airbyte connectors? -We will be introducing a new participative model in the next few months. There are still a lot of details to figure out, but the general idea is that maintainers of connectors would have the option to obtain a share of revenue when the connectors are being used in the paid version of Airbyte. In exchange, maintainers would be responsible for SLAs, new features, and bug fixes for the said connector. +We will be introducing a new participative model in the next few months. There are still a lot of details to figure out, but the general idea is that maintainers of connectors would have the option to obtain a share of revenue when the connectors are being used in the paid version of Airbyte. In exchange, maintainers would be responsible for SLAs, new features, and bug fixes for the said connector. diff --git a/docs/project-overview/licenses/mit-license.md b/docs/developer-guides/licenses/mit-license.md similarity index 100% rename from docs/project-overview/licenses/mit-license.md rename to docs/developer-guides/licenses/mit-license.md diff --git a/docs/enterprise-setup/README.md b/docs/enterprise-setup/README.md new file mode 100644 index 000000000000..c46542f240be --- /dev/null +++ b/docs/enterprise-setup/README.md @@ -0,0 +1,21 @@ +--- +products: oss-enterprise +--- + +# Airbyte Self-Managed Enterprise + +[Airbyte Self-Managed Enterprise](https://airbyte.com/product/airbyte-enterprise) is the best way to run Airbyte yourself. You get all 300+ pre-built connectors, data never leaves your environment, and Airbyte becomes self-serve in your organization with new tools to manage multiple users, and multiple teams using Airbyte all in one place. + +A valid license key is required to get started with Airbyte Self-Managed Enterprise. [Talk to sales](https://airbyte.com/company/talk-to-sales) to receive your license key. + +The following pages outline how to: +1. [Deploy Airbyte Enterprise using Kubernetes](./implementation-guide.md) +2. [Configure Okta for Single Sign-On (SSO) with Airbyte Self-Managed Self-Managed Enterprise](/access-management/sso.md) + +| Feature | Description | +|---------------------------|--------------------------------------------------------------------------------------------------------------| +| Premium Support | [Priority assistance](https://docs.airbyte.com/operator-guides/contact-support/#airbyte-enterprise-self-hosted-support) with deploying, managing and upgrading Airbyte or troubleshooting any connection issues. | +| User Management | [Okta SSO](/access-management/sso.md) to extend each Airbyte workspace to multiple users | +| Multiple Workspaces | Ability to create + manage multiple workspaces on one Airbyte instance | +| Role-Based Access Control | Isolate workspaces from one another with users roles scoped to individual workspaces | + diff --git a/docs/enterprise-setup/assets/okta-create-new-app-integration.png b/docs/enterprise-setup/assets/okta-create-new-app-integration.png new file mode 100644 index 000000000000..bff936656aad Binary files /dev/null and b/docs/enterprise-setup/assets/okta-create-new-app-integration.png differ diff --git a/docs/enterprise-setup/implementation-guide.md b/docs/enterprise-setup/implementation-guide.md new file mode 100644 index 000000000000..a12843e3c19e --- /dev/null +++ b/docs/enterprise-setup/implementation-guide.md @@ -0,0 +1,347 @@ +--- +products: oss-enterprise +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Implementation Guide + +[Airbyte Self-Managed Enterprise](./README.md) is in an early access stage for select priority users. Once you [are qualified for a Self-Managed Enterprise license key](https://airbyte.com/company/talk-to-sales), you can deploy Airbyte with the following instructions. + +Airbyte Self-Managed Enterprise must be deployed using Kubernetes. This is to enable Airbyte's best performance and scale. The core components \(api server, scheduler, etc\) run as deployments while the scheduler launches connector-related pods on different nodes. + +## Prerequisites + +There are three prerequisites to deploying: installing [helm](https://helm.sh/docs/intro/install/), a Kubernetes cluster, and having configured `kubectl` to connect to the cluster. + +For production, we recommend deploying to EKS, GKE or AKS. If you are doing some local testing, follow the cluster setup instructions outlined [here](/deploying-airbyte/on-kubernetes-via-helm.md#cluster-setup). + +To install `kubectl`, please follow [these instructions](https://kubernetes.io/docs/tasks/tools/). To configure `kubectl` to connect to your cluster by using `kubectl use-context my-cluster-name`, see the following: + +
      + Configure kubectl to connect to your cluster + + +
        +
      1. Configure gcloud with gcloud auth login.
      2. +
      3. On the Google Cloud Console, the cluster page will have a "Connect" button, with a command to run locally: gcloud container clusters get-credentials $CLUSTER_NAME --zone $ZONE_NAME --project $PROJECT_NAME
      4. +
      5. Use kubectl config get-contexts to show the contexts available.
      6. +
      7. Run kubectl config use-context $GKE_CONTEXT to access the cluster from kubectl.
      8. +
      +
      + +
        +
      1. Configure your AWS CLI to connect to your project.
      2. +
      3. Install eksctl.
      4. +
      5. Run eksctl utils write-kubeconfig --cluster=$CLUSTER_NAME to make the context available to kubectl.
      6. +
      7. Use kubectl config get-contexts to show the contexts available.
      8. +
      9. Run kubectl config use-context $EKS_CONTEXT to access the cluster with kubectl.
      10. +
      +
      +
      +
      + +## Deploy Airbyte Enterprise + +### Add Airbyte Helm Repository + +Follow these instructions to add the Airbyte helm repository: +1. Run `helm repo add airbyte https://airbytehq.github.io/helm-charts`, where `airbyte` is the name of the repository that will be indexed locally. +2. Perform the repo indexing process, and ensure your helm repository is up-to-date by running `helm repo update`. +3. You can then browse all charts uploaded to your repository by running `helm search repo airbyte`. + +### Clone & Configure Airbyte + +1. `git clone` the latest revision of the [airbyte-platform repository](https://github.com/airbytehq/airbyte-platform) + +2. Create a new `airbyte.yml` file in the `configs` directory of the `airbyte-platform` folder. You may also copy `airbyte.sample.yml` to use as a template: + +```sh +cp configs/airbyte.sample.yml configs/airbyte.yml +``` + +3. Add your Airbyte Self-Managed Enterprise license key to your `airbyte.yml`. + +4. Add your [auth details](/enterprise-setup/sso) to your `airbyte.yml`. Auth configurations aren't easy to modify after Airbyte is installed, so please double check them to make sure they're accurate before proceeding. + +
      + Configuring auth in your airbyte.yml file + +To configure SSO with Okta, add the following at the end of your `airbyte.yml` file: + +```yaml +auth: + identity-providers: + - type: okta + domain: $OKTA_DOMAIN + app-name: $OKTA_APP_INTEGRATION_NAME + client-id: $OKTA_CLIENT_ID + client-secret: $OKTA_CLIENT_SECRET +``` + +To configure basic auth (deploy without SSO), remove the entire `auth:` section from your airbyte.yml config file. You will authenticate with the instance admin user and password included in the your `airbyte.yml`. + +
      + +#### Configuring the Airbyte Database + +For Self-Managed Enterprise deployments, we recommend using a dedicated database instance for better reliability, and backups (such as AWS RDS or GCP Cloud SQL) instead of the default internal Postgres database (`airbyte/db`) that Airbyte spins up within the Kubernetes cluster. + +We assume in the following that you've already configured a Postgres instance: + +
      +External database setup steps + +1. In the `charts/airbyte/values.yaml` file, disable the default Postgres database (`airbyte/db`): + +```yaml +postgresql: + enabled: false +``` + +2. In the `charts/airbyte/values.yaml` file, enable and configure the external Postgres database: + +```yaml +externalDatabase: + host: ## Database host + user: ## Non-root username for the Airbyte database + database: db-airbyte ## Database name + port: 5432 ## Database port number +``` + +For the non-root user's password which has database access, you may use `password`, `existingSecret` or `jdbcUrl`. We recommend using `existingSecret`, or injecting sensitive fields from your own external secret store. Each of these parameters is mutually exclusive: + +```yaml +externalDatabase: + ... + password: ## Password for non-root database user + existingSecret: ## The name of an existing Kubernetes secret containing the password. + existingSecretPasswordKey: ## The Kubernetes secret key containing the password. + jdbcUrl: "jdbc:postgresql://:@localhost:5432/db-airbyte" ## Full database JDBC URL. You can also add additional arguments. +``` + +The optional `jdbcUrl` field should be entered in the following format: `jdbc:postgresql://localhost:5432/db-airbyte`. We recommend against using this unless you need to add additional extra arguments can be passed to the JDBC driver at this time (e.g. to handle SSL). + +
      + +#### Configuring External Logging + +For Self-Managed Enterprise deployments, we recommend spinning up standalone log storage for additional reliability using tools such as S3 and GCS instead of against using the defaul internal Minio storage (`airbyte/minio`). It's then a common practice to configure additional log forwarding from external log storage into your observability tool. + +
      +External log storage setup steps + +1. In the `charts/airbyte/values.yaml` file, disable the default Minio instance (`airbyte/minio`): + +```yaml +minio: + enabled: false +``` + +2. In the `charts/airbyte/values.yaml` file, enable and configure external log storage: + + + + + +```yaml +global: + ... + logs: + storage: + type: "S3" + + minio: + enabled: false + + s3: + enabled: true + bucket: "" ## S3 bucket name that you've created. + bucketRegion: "" ## e.g. us-east-1 + + accessKey: ## AWS Access Key. + password: "" + existingSecret: "" ## The name of an existing Kubernetes secret containing the AWS Access Key. + existingSecretKey: "" ## The Kubernetes secret key containing the AWS Access Key. + + secretKey: ## AWS Secret Access Key + password: + existingSecret: "" ## The name of an existing Kubernetes secret containing the AWS Secret Access Key. + existingSecretKey: "" ## The name of an existing Kubernetes secret containing the AWS Secret Access Key. +``` + +For each of `accessKey` and `secretKey`, the `password` and `existingSecret` fields are mutually exclusive. + +3. Ensure your access key is tied to an IAM user with the [following policies](https://docs.aws.amazon.com/AmazonS3/latest/userguide/example-policies-s3.html#iam-policy-ex0), allowing the user access to S3 storage: + +```yaml +{ + "Version":"2012-10-17", + "Statement":[ + { + "Effect":"Allow", + "Action": "s3:ListAllMyBuckets", + "Resource":"*" + }, + { + "Effect":"Allow", + "Action":["s3:ListBucket","s3:GetBucketLocation"], + "Resource":"arn:aws:s3:::YOUR-S3-BUCKET-NAME" + }, + { + "Effect":"Allow", + "Action":[ + "s3:PutObject", + "s3:PutObjectAcl", + "s3:GetObject", + "s3:GetObjectAcl", + "s3:DeleteObject" + ], + "Resource":"arn:aws:s3:::YOUR-S3-BUCKET-NAME/*" + } + ] +} +``` + + + + + +```yaml +global: + ... + logs: + storage: + type: "GCS" + + minio: + enabled: false + + gcs: + bucket: airbyte-dev-logs # GCS bucket name that you've created. + credentials: "" + credentialsJson: "" ## Base64 encoded json GCP credentials file contents +``` + +Note that the `credentials` and `credentialsJson` fields are mutually exclusive. + + + +
      + + +#### Configuring Ingress + +To access the Airbyte UI, you will need to manually attach an ingress configuration to your deployment. The following is a skimmed down definition of an ingress resource you could use for Self-Managed Enterprise: + +
      +Ingress configuration setup steps + + + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: # ingress name, example: enterprise-demo + annotations: + ingress.kubernetes.io/ssl-redirect: "false" +spec: + rules: + - host: # host, example: enterprise-demo.airbyte.com + http: + paths: + - backend: + service: + # format is ${RELEASE_NAME}-airbyte-webapp-svc + name: airbyte-pro-airbyte-webapp-svc + port: + number: # service port, example: 8080 + path: / + pathType: Prefix + - backend: + service: + # format is ${RELEASE_NAME}-airbyte-keycloak-svc + name: airbyte-pro-airbyte-keycloak-svc + port: + number: # service port, example: 8180 + path: /auth + pathType: Prefix +``` + + + + +If you are intending on using Amazon Application Load Balancer (ALB) for ingress, this ingress definition will be close to what's needed to get up and running: + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: + annotations: + # Specifies that the Ingress should use an AWS ALB. + kubernetes.io/ingress.class: "alb" + # Redirects HTTP traffic to HTTPS. + ingress.kubernetes.io/ssl-redirect: "true" + # Creates an internal ALB, which is only accessible within your VPC or through a VPN. + alb.ingress.kubernetes.io/scheme: internal + # Specifies the ARN of the SSL certificate managed by AWS ACM, essential for HTTPS. + alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-x:xxxxxxxxx:certificate/xxxxxxxxx-xxxxx-xxxx-xxxx-xxxxxxxxxxx + # Sets the idle timeout value for the ALB. + alb.ingress.kubernetes.io/load-balancer-attributes: idle_timeout.timeout_seconds=30 + # [If Applicable] Specifies the VPC subnets and security groups for the ALB + # alb.ingress.kubernetes.io/subnets: '' e.g. 'subnet-12345, subnet-67890' + # alb.ingress.kubernetes.io/security-groups: +spec: + rules: + - host: e.g. enterprise-demo.airbyte.com + http: + paths: + - backend: + service: + name: airbyte-pro-airbyte-webapp-svc + port: + number: 80 + path: / + pathType: Prefix + - backend: + service: + name: airbyte-pro-airbyte-keycloak-svc + port: + number: 8180 + path: /auth + pathType: Prefix +``` + +The ALB controller will use a `ServiceAccount` that requires the [following IAM policy](https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/main/docs/install/iam_policy.json) to be attached. + + + +
      + +Once this is complete, ensure that the value of the `webapp-url` field in your `airbyte.yml` is configured to match the ingress URL. + +You may configure ingress using a load balancer or an API Gateway. We do not currently support most service meshes (such as Istio). If you are having networking issues after fully deploying Airbyte, please verify that firewalls or lacking permissions are not interfering with pod-pod communication. Please also verify that deployed pods have the right permissions to make requests to your external database. + +### Install Airbyte Enterprise + +Install Airbyte Enterprise on helm using the following command: + +```sh +./tools/bin/install_airbyte_pro_on_helm.sh +``` + +The default release name is `airbyte-pro`. You can change this via the `RELEASE_NAME` environment +variable. + +### Customizing your Airbyte Enterprise Deployment + +In order to customize your deployment, you need to create `values.yaml` file in a local folder and populate it with default configuration override values. A `values.yaml` example can be located in [charts/airbyte](https://github.com/airbytehq/airbyte-platform/blob/main/charts/airbyte/values.yaml) folder of the Airbyte repository. + +After specifying your own configuration, run the following command: + +```sh +./tools/bin/install_airbyte_pro_on_helm.sh --values path/to/values.yaml +``` diff --git a/docs/enterprise-setup/upgrading-from-community.md b/docs/enterprise-setup/upgrading-from-community.md new file mode 100644 index 000000000000..15217913cc17 --- /dev/null +++ b/docs/enterprise-setup/upgrading-from-community.md @@ -0,0 +1,105 @@ +--- +products: oss-enterprise +--- + +# Existing Instance Upgrades + +This page supplements the [Self-Managed Enterprise implementation guide](./implementation-guide.md). It highlights the steps to take if you are currently using Airbyte Self-Managed Community, our free open source offering, and are ready to upgrade to [Airbyte Self-Managed Enterprise](./README.md). + +A valid license key is required to get started with Airbyte Enterprise. [Talk to sales](https://airbyte.com/company/talk-to-sales) to receive your license key. + +These instructions are for you if: +* You want your Self-Managed Enterprise instance to inherit state from your existing deployment. +* You are currently deploying Airbyte on Kubernetes. +* You are comfortable with an in-place upgrade. This guide does not dual-write to a new Airbyte deployment. + +### Step 1: Update Airbyte Open Source + +You must first update to the latest Open Source community release. We assume you are running the following steps from the root of the `airbytehq/airbyte-platform` cloned repo. + +1. Determine your current helm release name by running `helm list`. This will now be referred to as `[RELEASE_NAME]` for the rest of this guide. +2. Upgrade to the latest Open Source community release. The output will now be refered to as `[RELEASE_VERSION]` for the rest of this guide: + +```sh +helm upgrade [RELEASE_NAME] airbyte/airbyte +``` + +### Step 2: Configure Self-Managed Enterprise + +At this step, please create and fill out the `airbyte.yml` as explained in the [Self-Managed Enterprise implementation guide](./implementation-guide.md#clone--configure-airbyte) in the `configs` directory. You should avoid making any changes to your Airbyte database or log storage at this time. When complete, you should have a completed file matching the following skeleton: + +
      +Configuring your airbyte.yml file + +```yml +webapp-url: # example: localhost:8080 + +initial-user: + email: + first-name: + last-name: + username: # your existing Airbyte instance username + password: # your existing Airbyte instance password + +license-key: + +auth: + identity-providers: + - type: okta + domain: + app-name: + client-id: + client-secret: +``` + +
      + +### Step 3: Deploy Self-Managed Enterprise + +1. You can now run the following command to upgrade your instance to Self-Managed Enterprise. If you previously included additional `values` files on your existing deployment, be sure to add these here as well: + +```sh +helm upgrade [RELEASE_NAME] airbyte/airbyte \ +--version [RELEASE_VERSION] \ +--set-file airbyteYml=./configs/airbyte.yml \ +--values ./charts/airbyte/airbyte-pro-values.yaml [... additional --values] +``` + +2. Once this is complete, you will need to upgrade your ingress to include the new `/auth` path. The following is a skimmed down definition of an ingress resource you could use for Self-Managed Enterprise: + +
      +Configuring your Airbyte ingress + +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: # ingress name, example: enterprise-demo + annotations: + ingress.kubernetes.io/ssl-redirect: "false" +spec: + rules: + - host: # host, example: enterprise-demo.airbyte.com + http: + paths: + - backend: + service: + # format is ${RELEASE_NAME}-airbyte-webapp-svc + name: airbyte-pro-airbyte-webapp-svc + port: + number: # service port, example: 8080 + path: / + pathType: Prefix + - backend: + service: + # format is ${RELEASE_NAME}-airbyte-keycloak-svc + name: airbyte-pro-airbyte-keycloak-svc + port: + number: # service port, example: 8180 + path: /auth + pathType: Prefix +``` + +
      + +All set! When you log in, you should expect all connections, sources and destinations to be present, and configured as prior. \ No newline at end of file diff --git a/docs/integrations/README.md b/docs/integrations/README.md index 01da3eae1fb2..fe41578bacf5 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -2,17 +2,15 @@ import ConnectorRegistry from '@site/src/components/ConnectorRegistry'; # Connector Catalog -## Connector Release Stages +## Connector Support Levels -Airbyte uses a grading system for connectors to help you understand what to expect from a connector: +Airbyte uses a two tiered system for connectors to help you understand what to expect from a connector: -**Generally Available**: A generally available connector has been deemed ready for use in a production environment and is officially supported by Airbyte. Its documentation is considered sufficient to support widespread adoption. +**Certified**: A certified connector is actively maintained and supported by the Airbyte team and maintains a high quality bar. It is production ready. -**Beta**: A beta connector is considered stable with no backwards incompatible changes but has not been validated by a broader group of users. We expect to find and fix a few issues and bugs in the release before it’s ready for GA. +**Community**: A community connector is maintained by the Airbyte community until it becomes Certified. Airbyte has over 800 code contributors and 15,000 people in the Slack community to help. The Airbyte team is continually certifying Community connectors as usage grows. As these connectors are not maintained by Airbyte, we do not offer support SLAs around them, and we encourage caution when using them in production. -**Alpha**: An alpha connector signifies a connector under development and helps Airbyte gather early feedback and issues reported by early adopters. We strongly discourage using alpha releases for production use cases and do not offer Cloud Support SLAs around these products, features, or connectors. - -For more information about the grading system, see [Product Release Stages](https://docs.airbyte.com/project-overview/product-release-stages) +For more information about the system, see [Connector Support Levels](./connector-support-levels.md) _[View the connector registries in full](https://connectors.airbyte.com/files/generated_reports/connector_registry_report.html)_ diff --git a/docs/integrations/connector-support-levels.md b/docs/integrations/connector-support-levels.md new file mode 100644 index 000000000000..9baeebc9785c --- /dev/null +++ b/docs/integrations/connector-support-levels.md @@ -0,0 +1,43 @@ +--- +products: all +--- + +# Connector Support Levels + +The following table describes the support levels of Airbyte connectors. + +| | Certified | Community | Custom | +| ------------------------------------ | ----------------------------------------- | ------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Availability** | Available to all users | Available to all users | Available to all users | +| **Who builds them?** | Either the community or the Airbyte team. | Typically they are built by the community. The Airbyte team may upgrade them to Certified at any time. | Anyone can build custom connectors. We recommend using our [Connector Builder](https://docs.airbyte.com/connector-development/connector-builder-ui/overview) or [Low-code CDK](https://docs.airbyte.com/connector-development/config-based/low-code-cdk-overview). | +| **Who maintains them?** | The Airbyte team | Users | Users | +| **Production Readiness** | Guaranteed by Airbyte | Not guaranteed | Not guaranteed | +| **Support: Cloud** | Supported* | No Support | Supported** | +| **Support: Powered by Airbyte** | Supported* | No Support | Supported** | +| **Support: Self-Managed Enterprise** | Supported* | No Support | Supported** | +| **Support: Community (OSS)** | Slack Support only | No Support | Slack Support only | + +\*For Certified connectors, Official Support SLAs are only available to customers with Premium Support included in their contract. Otherwise, please use our support portal and we will address your issues as soon as possible. + +\*\*For Custom connectors, Official Support SLAs are only available to customers with Premium Support included in their contract. This support is provided with best efforts, and maintenance/upgrades are owned by the customer. + +## Certified + +A **Certified** connector is actively maintained and supported by the Airbyte team and maintains a high quality bar. It is production ready. + +### What you should know about Certified connectors: + +- Certified connectors are available to all users. +- These connectors have been tested and vetted in order to be certified and are production ready. +- Certified connectors should go through minimal breaking change but in the event an upgrade is needed users will be given an adequate upgrade window. + +## Community + +A **Community** connector is maintained by the Airbyte community until it becomes Certified. Airbyte has over 800 code contributors and 15,000 people in the Slack community to help. The Airbyte team is continually certifying Community connectors as usage grows. As these connectors are not maintained by Airbyte, we do not offer support SLAs around them, and we encourage caution when using them in production. + +### What you should know about Community connectors: + +- Community connectors are available to all users. +- Community connectors may be upgraded to Certified at any time, and we will notify users of these upgrades via our Slack Community and in our Connector Catalog. +- Community connectors might not be feature-complete (features planned for release are under development or not prioritized) and may include backward-incompatible/breaking API changes with no or short notice. +- Community connectors have no Support SLAs. diff --git a/docs/integrations/custom-connectors.md b/docs/integrations/custom-connectors.md index bdd14e56a251..aef3132e0be7 100644 --- a/docs/integrations/custom-connectors.md +++ b/docs/integrations/custom-connectors.md @@ -4,7 +4,7 @@ description: Missing a connector? # Custom or New Connector -If you'd like to **ask for a new connector,** you can request it directly [here](https://github.com/airbytehq/airbyte/issues/new?assignees=&labels=area%2Fintegration%2C+new-integration&template=new-integration-request.md&title=). +If you'd like to **ask for a new connector,** you can request it directly [here](https://github.com/airbytehq/airbyte/discussions/new?category=new-connector-request). If you'd like to build new connectors and **make them part of the pool of pre-built connectors on Airbyte,** first a big thank you. We invite you to check our [contributing guide on building connectors](../contributing-to-airbyte/README.md). @@ -12,6 +12,10 @@ If you'd like to build new connectors, or update existing ones, **for your own u ## Developing your own connector +:::info +Custom connectors are currently exclusive to Airbyte Open-Source deployments. However, there are plans for their release on Airbyte Cloud, scheduled for January 2024. You can track the progress on this development [here](https://github.com/orgs/airbytehq/projects/37?pane=issue&itemId=45471174). +::: + It's easy to code your own connectors on Airbyte. Here is a link to instruct on how to code new sources and destinations: [building new connectors](../connector-development/README.md) While the guides in the link above are specific to the languages used most frequently to write integrations, **Airbyte connectors can be written in any language**. Please reach out to us if you'd like help developing connectors in other languages. diff --git a/docs/integrations/destinations/amazon-sqs.md b/docs/integrations/destinations/amazon-sqs.md index 6178d690e0f0..1cf727ff6a34 100644 --- a/docs/integrations/destinations/amazon-sqs.md +++ b/docs/integrations/destinations/amazon-sqs.md @@ -2,7 +2,8 @@ ## Overview -The Airbyte SQS destination allows you to sync data to Amazon SQS. It currently supports sending all streams to a single Queue. +The Airbyte SQS destination allows you to sync data to Amazon SQS. It currently supports sending all +streams to a single Queue. ### Sync overview @@ -10,63 +11,73 @@ The Airbyte SQS destination allows you to sync data to Amazon SQS. It currently All streams will be output into a single SQS Queue. -Amazon SQS messages can only contain JSON, XML or text, and this connector supports writing messages in all three formats. See the **Writing Text or XML messages** section for more info. +Amazon SQS messages can only contain JSON, XML or text, and this connector supports writing messages +in all three formats. See the **Writing Text or XML messages** section for more info. #### Features -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | No | | -| Incremental - Append Sync | Yes | | -| Incremental - Append + Deduped | No | | -| Namespaces | No | | +| Feature | Supported?\(Yes/No\) | Notes | +| :----------------------------- | :------------------- | :---- | +| Full Refresh Sync | No | | +| Incremental - Append Sync | Yes | | +| Incremental - Append + Deduped | No | | +| Namespaces | No | | ## Getting started ### Requirements -* AWS IAM Access Key -* AWS IAM Secret Key -* AWS SQS Queue +- AWS IAM Access Key +- AWS IAM Secret Key +- AWS SQS Queue #### Permissions If the target SQS Queue is not public, you will need the following permissions on the Queue: -* `sqs:SendMessage` +- `sqs:SendMessage` ### Properties Required properties are 'Queue URL' and 'AWS Region' as noted in **bold** below. -* **Queue URL** (STRING) - * The full AWS endpoint URL of the queue e.g.`https://sqs.eu-west-1.amazonaws.com/1234567890/example-queue-url` -* **AWS Region** (STRING) - * The region code for the SQS Queue e.g. eu-west-1 -* Message Delay (INT) - * Time in seconds that this message should be hidden from consumers. - * See the [AWS SQS documentation](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-timers.html) for more detail. -* AWS IAM Access Key ID (STRING) - * The Access Key for the IAM User with permissions on this Queue - * Permission `sqs:SendMessage` is required -* AWS IAM Secret Key (STRING) - * The Secret Key for the IAM User with permissions on this Queue -* Message Body Key (STRING) - * Rather than sending the entire Record as the Message Body, use this property to reference a Key in the Record to use as the message body. The value of this property should be the Key name in the input Record. The key must be at the top level of the Record, nested Keys are not supported. -* Message Group Id (STRING) - * When using a FIFO queue, this property is **required**. - * See the [AWS SQS documentation](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/using-messagegroupid-property.html) for more detail. +- **Queue URL** (STRING) + - The full AWS endpoint URL of the queue + e.g.`https://sqs.eu-west-1.amazonaws.com/1234567890/example-queue-url` +- **AWS Region** (STRING) + - The region code for the SQS Queue e.g. eu-west-1 +- Message Delay (INT) + - Time in seconds that this message should be hidden from consumers. + - See the + [AWS SQS documentation](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-timers.html) + for more detail. +- AWS IAM Access Key ID (STRING) + - The Access Key for the IAM User with permissions on this Queue + - Permission `sqs:SendMessage` is required +- AWS IAM Secret Key (STRING) + - The Secret Key for the IAM User with permissions on this Queue +- Message Body Key (STRING) + - Rather than sending the entire Record as the Message Body, use this property to reference a Key + in the Record to use as the message body. The value of this property should be the Key name in + the input Record. The key must be at the top level of the Record, nested Keys are not supported. +- Message Group Id (STRING) + - When using a FIFO queue, this property is **required**. + - See the + [AWS SQS documentation](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/using-messagegroupid-property.html) + for more detail. ### Setup guide -* [Create IAM Keys](https://aws.amazon.com/premiumsupport/knowledge-center/create-access-key/) -* [Create SQS Queue](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-getting-started.html#step-create-queue) +- [Create IAM Keys](https://aws.amazon.com/premiumsupport/knowledge-center/create-access-key/) +- [Create SQS Queue](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-getting-started.html#step-create-queue) #### Using the Message Body Key -This property allows you to reference a Key within the input Record as using that properties Value as the SQS Message Body. +This property allows you to reference a Key within the input Record as using that properties Value +as the SQS Message Body. For example, with the input Record: + ``` { "parent_with_child": { @@ -76,7 +87,9 @@ For example, with the input Record: } ``` -To send *only* the `parent_with_child` object, we can set `Message Body Key` to `parent_with_child`. Giving an output SQS Message of: +To send _only_ the `parent_with_child` object, we can set `Message Body Key` to `parent_with_child`. +Giving an output SQS Message of: + ``` { "child": "child_value" @@ -85,9 +98,11 @@ To send *only* the `parent_with_child` object, we can set `Message Body Key` to #### Writing Text or XML messages -To output Text or XML, the data must be contained within a String field in the input data, and then referenced by setting the `Message Body Key` property. +To output Text or XML, the data must be contained within a String field in the input data, and then +referenced by setting the `Message Body Key` property. For example, with an input Record as: + ``` { "my_xml_field": "value" @@ -102,9 +117,9 @@ The output SQS message would contain: value ``` - ## CHANGELOG -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| `0.1.0` | 2021-10-27 | [\#0000](https://github.com/airbytehq/airbyte/pull/0000) | `Initial version` | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :-------------------------------------------------------- | :-------------------------------- | +| 0.1.1 | 2024-01-03 | [#33924](https://github.com/airbytehq/airbyte/pull/33924) | Add new ap-southeast-3 AWS region | +| 0.1.0 | 2021-10-27 | [\#0000](https://github.com/airbytehq/airbyte/pull/0000) | Initial version | diff --git a/docs/integrations/destinations/aws-datalake.md b/docs/integrations/destinations/aws-datalake.md index 50cda4825fe2..d0323ea35dd8 100644 --- a/docs/integrations/destinations/aws-datalake.md +++ b/docs/integrations/destinations/aws-datalake.md @@ -1,77 +1,98 @@ # AWS Datalake -This page contains the setup guide and reference information for the AWS Datalake destination connector. +This page contains the setup guide and reference information for the AWS Datalake destination +connector. -The AWS Datalake destination connector allows you to sync data to AWS. It will write data as JSON files in S3 and -will make it available through a [Lake Formation Governed Table](https://docs.aws.amazon.com/lake-formation/latest/dg/governed-tables.html) in the Glue Data Catalog so that the data is available throughout other AWS services such as Athena, Glue jobs, EMR, Redshift, etc. +The AWS Datalake destination connector allows you to sync data to AWS. It will write data as JSON +files in S3 and will make it available through a +[Lake Formation Governed Table](https://docs.aws.amazon.com/lake-formation/latest/dg/governed-tables.html) +in the Glue Data Catalog so that the data is available throughout other AWS services such as Athena, +Glue jobs, EMR, Redshift, etc. ## Prerequisites To use this destination connector, you will need: -* An AWS account -* An S3 bucket where the data will be written -* An AWS Lake Formation database where tables will be created (one per stream) -* AWS credentials in the form of either the pair Access key ID / Secret key ID or a role with the following permissions: - * Writing objects in the S3 bucket - * Updating of the Lake Formation database +- An AWS account +- An S3 bucket where the data will be written +- An AWS Lake Formation database where tables will be created (one per stream) +- AWS credentials in the form of either the pair Access key ID / Secret key ID or a role with the + following permissions: + + - Writing objects in the S3 bucket + - Updating of the Lake Formation database Please check the Setup guide below if you need guidance creating those. ## Setup guide -You should now have all the requirements needed to configure AWS Datalake as a destination in the UI. You'll need the -following information to configure the destination: +You should now have all the requirements needed to configure AWS Datalake as a destination in the +UI. You'll need the following information to configure the destination: -- Aws Account Id : The account ID of your AWS account. You will find the instructions to setup a new AWS account [here](https://aws.amazon.com/premiumsupport/knowledge-center/create-and-activate-aws-account/). +- Aws Account Id : The account ID of your AWS account. You will find the instructions to setup a new + AWS account + [here](https://aws.amazon.com/premiumsupport/knowledge-center/create-and-activate-aws-account/). - Aws Region : The region in which your resources are deployed -- Authentication mode : The AWS Datalake connector lets you authenticate with either a user or a role. In both case, you will have to make sure -that appropriate policies are in place. Select "ROLE" if you are using a role, "USER" if using a user with Access key / Secret Access key. -- Target Role Arn : The name of the role, if "Authentication mode" was "ROLE". You will find the instructions to create a new role [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-service.html). -- Access Key Id : The Access Key ID of the user if "Authentication mode" was "USER". You will find the instructions to create a new user [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html). Make sure to select "Programmatic Access" so that you get secret access keys. +- Authentication mode : The AWS Datalake connector lets you authenticate with either a user or a + role. In both case, you will have to make sure that appropriate policies are in place. Select + "ROLE" if you are using a role, "USER" if using a user with Access key / Secret Access key. +- Target Role Arn : The name of the role, if "Authentication mode" was "ROLE". You will find the + instructions to create a new role + [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-service.html). +- Access Key Id : The Access Key ID of the user if "Authentication mode" was "USER". You will find + the instructions to create a new user + [here](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html). Make sure to select + "Programmatic Access" so that you get secret access keys. - Secret Access Key : The Secret Access Key ID of the user if "Authentication mode" was "USER" -- S3 Bucket Name : The bucket in which the data will be written. You will find the instructions to create a new S3 bucket [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html). +- S3 Bucket Name : The bucket in which the data will be written. You will find the instructions to + create a new S3 bucket + [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html). - Target S3 Bucket Prefix : A prefix to prepend to the file name when writing to the bucket -- Database : The database in which the tables will be created. You will find the instructions to create a new Lakeformation Database [here](https://docs.aws.amazon.com/lake-formation/latest/dg/creating-database.html). +- Database : The database in which the tables will be created. You will find the instructions to + create a new Lakeformation Database + [here](https://docs.aws.amazon.com/lake-formation/latest/dg/creating-database.html). **Assigning proper permissions** The policy used by the user or the role must have access to the following services: -* AWS Lake Formation -* AWS Glue -* AWS S3 +- AWS Lake Formation +- AWS Glue +- AWS S3 -You can use [the AWS policy generator](https://awspolicygen.s3.amazonaws.com/policygen.html) to help you generate an appropriate policy. +You can use [the AWS policy generator](https://awspolicygen.s3.amazonaws.com/policygen.html) to help +you generate an appropriate policy. -Please also make sure that the role or user you will use has appropriate permissions on the database in AWS Lakeformation. You will find more information about Lake Formation permissions in the [AWS Lake Formation Developer Guide](https://docs.aws.amazon.com/lake-formation/latest/dg/lake-formation-permissions.html). +Please also make sure that the role or user you will use has appropriate permissions on the database +in AWS Lakeformation. You will find more information about Lake Formation permissions in the +[AWS Lake Formation Developer Guide](https://docs.aws.amazon.com/lake-formation/latest/dg/lake-formation-permissions.html). ## Supported sync modes -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental - Append Sync | Yes | | -| Namespaces | No | | - +| Feature | Supported?\(Yes/No\) | Notes | +| :------------------------ | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | Yes | | +| Namespaces | No | | ## Data type map -The Glue tables will be created with schema information provided by the source, i.e : You will find the same columns -and types in the destination table as in the source except for the following types which will be translated for compatibility with the Glue Data Catalog: - -|Type in the source| Type in the destination| -| :--- | :--- | -| number | float | -| integer | int | - +The Glue tables will be created with schema information provided by the source, i.e : You will find +the same columns and types in the destination table as in the source except for the following types +which will be translated for compatibility with the Glue Data Catalog: +| Type in the source | Type in the destination | +| :----------------- | :---------------------- | +| number | float | +| integer | int | ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.3 | 2023-03-28 | [\#24642](https://github.com/airbytehq/airbyte/pull/24642) | Prefer airbyte type for complex types when available | -| 0.1.2 | 2022-09-26 | [\#17193](https://github.com/airbytehq/airbyte/pull/17193) | Fix schema keyerror and add parquet support | -| 0.1.1 | 2022-04-20 | [\#11811](https://github.com/airbytehq/airbyte/pull/11811) | Fix name of required param in specification | -| 0.1.0 | 2022-03-29 | [\#10760](https://github.com/airbytehq/airbyte/pull/10760) | Initial release | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :--------------------------------------------------------- | :--------------------------------------------------- | +| 0.1.5 | 2024-01-03 | [#33924](https://github.com/airbytehq/airbyte/pull/33924) | Add new ap-southeast-3 AWS region | +| 0.1.4 | 2023-10-25 | [\#29221](https://github.com/airbytehq/airbyte/pull/29221) | Upgrade AWSWrangler | +| 0.1.3 | 2023-03-28 | [\#24642](https://github.com/airbytehq/airbyte/pull/24642) | Prefer airbyte type for complex types when available | +| 0.1.2 | 2022-09-26 | [\#17193](https://github.com/airbytehq/airbyte/pull/17193) | Fix schema keyerror and add parquet support | +| 0.1.1 | 2022-04-20 | [\#11811](https://github.com/airbytehq/airbyte/pull/11811) | Fix name of required param in specification | +| 0.1.0 | 2022-03-29 | [\#10760](https://github.com/airbytehq/airbyte/pull/10760) | Initial release | diff --git a/docs/integrations/destinations/azure-blob-storage.md b/docs/integrations/destinations/azure-blob-storage.md index d93f132cfb99..6d1e2e27c836 100644 --- a/docs/integrations/destinations/azure-blob-storage.md +++ b/docs/integrations/destinations/azure-blob-storage.md @@ -143,7 +143,8 @@ They will be like this in the output file: | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 0.2.0 | 2023-01-18 | [\#15318](https://github.com/airbytehq/airbyte/pull/21467) | Support spilling of objects exceeding configured size threshold | +| 0.2.1 | 2023-09-13 | [\#30412](https://github.com/airbytehq/airbyte/pull/30412) | Switch noisy logging to debug | +| 0.2.0 | 2023-01-18 | [\#21467](https://github.com/airbytehq/airbyte/pull/21467) | Support spilling of objects exceeding configured size threshold | | 0.1.6 | 2022-08-08 | [\#15318](https://github.com/airbytehq/airbyte/pull/15318) | Support per-stream state | | 0.1.5 | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors | | 0.1.4 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | diff --git a/docs/integrations/destinations/bigquery-denormalized.md b/docs/integrations/destinations/bigquery-denormalized.md deleted file mode 100644 index a69a2c5ac896..000000000000 --- a/docs/integrations/destinations/bigquery-denormalized.md +++ /dev/null @@ -1,83 +0,0 @@ -# BigQuery Denormalized - -See [destinations/bigquery](https://docs.airbyte.com/integrations/destinations/bigquery) - -## Changelog - -### bigquery-denormalized - -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :--------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------- | -| 1.5.3 | 2023-07-14 | [\#28345](https://github.com/airbytehq/airbyte/pull/28345) | Increment patch to trigger a rebuild | -| 1.5.2 | 2023-07-05 | [\#27936](https://github.com/airbytehq/airbyte/pull/27936) | Internal code change | -| 1.5.1 | 2023-06-30 | [\#27891](https://github.com/airbytehq/airbyte/pull/27891) | Revert bugged update | -| 1.5.0 | 2023-06-27 | [\#27781](https://github.com/airbytehq/airbyte/pull/27781) | License Update: Elv2 | -| 1.4.1 | 2023-05-17 | [\#26213](https://github.com/airbytehq/airbyte/pull/26213) | Fix bug in parsing file buffer config count | -| 1.4.0 | 2023-04-28 | [\#25570](https://github.com/airbytehq/airbyte/pull/25570) | Fix: all integer schemas should be converted to Avro longs | -| 1.3.3 | 2023-04-27 | [\#25346](https://github.com/airbytehq/airbyte/pull/25346) | Internal code cleanup | -| 1.3.0 | 2023-04-19 | [\#25287](https://github.com/airbytehq/airbyte/pull/25287) | Add parameter to configure the number of file buffers when GCS is used as the loading method | -| 1.2.20 | 2023-04-12 | [\#25122](https://github.com/airbytehq/airbyte/pull/25122) | Add additional data centers | -| 1.2.19 | 2023-03-29 | [\#24671](https://github.com/airbytehq/airbyte/pull/24671) | Fail faster in certain error cases | -| 1.2.18 | 2023-03-23 | [\#24447](https://github.com/airbytehq/airbyte/pull/24447) | Set the Service Account Key JSON field to always_show: true so that it isn't collapsed into an optional fields section | -| 1.2.17 | 2023-03-17 | [\#23788](https://github.com/airbytehq/airbyte/pull/23788) | S3-Parquet: added handler to process null values in arrays | -| 1.2.16 | 2023-03-10 | [\#23931](https://github.com/airbytehq/airbyte/pull/23931) | Added support for periodic buffer flush | -| 1.2.15 | 2023-03-10 | [\#23466](https://github.com/airbytehq/airbyte/pull/23466) | Changed S3 Avro type from Int to Long | -| 1.2.14 | 2023-02-08 | [\#22497](https://github.com/airbytehq/airbyte/pull/22497) | Fixed table already exists error | -| 1.2.13 | 2023-01-26 | [\#20631](https://github.com/airbytehq/airbyte/pull/20631) | Added support for destination checkpointing with staging | -| 1.2.12 | 2023-01-18 | [\#21087](https://github.com/airbytehq/airbyte/pull/21087) | Wrap Authentication Errors as Config Exceptions | -| 1.2.11 | 2023-01-18 | [\#21144](https://github.com/airbytehq/airbyte/pull/21144) | Added explicit error message if sync fails due to a config issue | -| 1.2.10 | 2023-01-04 | [\#20730](https://github.com/airbytehq/airbyte/pull/20730) | An incoming source Number type will create a big query integer rather than a float. | -| 1.2.9 | 2022-12-14 | [\#20501](https://github.com/airbytehq/airbyte/pull/20501) | Report GCS staging failures that occur during connection check | -| 1.2.8 | 2022-11-22 | [\#19489](https://github.com/airbytehq/airbyte/pull/19489) | Added non-billable projects handle to check connection stage | -| 1.2.7 | 2022-11-11 | [\#19358](https://github.com/airbytehq/airbyte/pull/19358) | Fixed check method to capture mismatch dataset location | -| 1.2.6 | 2022-11-10 | [\#18554](https://github.com/airbytehq/airbyte/pull/18554) | Improve check connection method to handle more errors | -| 1.2.5 | 2022-10-19 | [\#18162](https://github.com/airbytehq/airbyte/pull/18162) | Improve error logs | -| 1.2.4 | 2022-09-26 | [\#16890](https://github.com/airbytehq/airbyte/pull/16890) | Add user-agent header | -| 1.2.3 | 2022-09-22 | [\#17054](https://github.com/airbytehq/airbyte/pull/17054) | Respect stream namespace | -| 1.2.2 | 2022-09-14 | [\#15668](https://github.com/airbytehq/airbyte/pull/15668) | (bugged, do not use) Wrap logs in AirbyteLogMessage | -| 1.2.1 | 2022-09-10 | [\#16401](https://github.com/airbytehq/airbyte/pull/16401) | (bugged, do not use) Wrapping string objects with TextNode | -| 1.2.0 | 2022-09-09 | [\#14023](https://github.com/airbytehq/airbyte/pull/14023) | (bugged, do not use) Cover arrays only if they are nested | -| 1.1.16 | 2022-09-01 | [\#16243](https://github.com/airbytehq/airbyte/pull/16243) | Fix Json to Avro conversion when there is field name clash from combined restrictions (`anyOf`, `oneOf`, `allOf` fields) | -| 1.1.15 | 2022-08-03 | [\#14784](https://github.com/airbytehq/airbyte/pull/14784) | Enabling Application Default Credentials | -| 1.1.14 | 2022-08-02 | [\#14801](https://github.com/airbytehq/airbyte/pull/14801) | Fix multiple log bindings | -| 1.1.13 | 2022-08-02 | [\#15180](https://github.com/airbytehq/airbyte/pull/15180) | Fix standard loading mode | -| 1.1.12 | 2022-06-29 | [\#14079](https://github.com/airbytehq/airbyte/pull/14079) | Map "airbyte_type": "big_integer" to INT64 | -| 1.1.11 | 2022-06-24 | [\#14114](https://github.com/airbytehq/airbyte/pull/14114) | Remove "additionalProperties": false from specs for connectors with staging | -| 1.1.10 | 2022-06-16 | [\#13852](https://github.com/airbytehq/airbyte/pull/13852) | Updated stacktrace format for any trace message errors | -| 1.1.9 | 2022-06-17 | [\#13753](https://github.com/airbytehq/airbyte/pull/13753) | Deprecate and remove PART_SIZE_MB fields from connectors based on StreamTransferManager | -| 1.1.8 | 2022-06-07 | [\#13579](https://github.com/airbytehq/airbyte/pull/13579) | Always check GCS bucket for GCS loading method to catch invalid HMAC keys. | -| 1.1.7 | 2022-06-07 | [\#13424](https://github.com/airbytehq/airbyte/pull/13424) | Reordered fields for specification. | -| 1.1.6 | 2022-05-15 | [\#12768](https://github.com/airbytehq/airbyte/pull/12768) | Clarify that the service account key json field is required on cloud. | -| 0.3.5 | 2022-05-12 | [\#12805](https://github.com/airbytehq/airbyte/pull/12805) | Updated to latest base-java to emit AirbyteTraceMessage on error. | -| 0.3.4 | 2022-05-04 | [\#12578](https://github.com/airbytehq/airbyte/pull/12578) | In JSON to Avro conversion, log JSON field values that do not follow Avro schema for debugging. | -| 0.3.3 | 2022-05-02 | [\#12528](https://github.com/airbytehq/airbyte/pull/12528) | Update Dataset location field description | -| 0.3.2 | 2022-04-29 | [\#12477](https://github.com/airbytehq/airbyte/pull/12477) | Dataset location is a required field | -| 0.3.1 | 2022-04-15 | [\#11978](https://github.com/airbytehq/airbyte/pull/11978) | Fixed emittedAt timestamp. | -| 0.3.0 | 2022-04-06 | [\#11776](https://github.com/airbytehq/airbyte/pull/11776) | Use serialized buffering strategy to reduce memory consumption. | -| 0.2.15 | 2022-04-05 | [\#11166](https://github.com/airbytehq/airbyte/pull/11166) | Fixed handling of anyOf and allOf fields | -| 0.2.14 | 2022-04-02 | [\#11620](https://github.com/airbytehq/airbyte/pull/11620) | Updated spec | -| 0.2.13 | 2022-04-01 | [\#11636](https://github.com/airbytehq/airbyte/pull/11636) | Added new unit tests | -| 0.2.12 | 2022-03-28 | [\#11454](https://github.com/airbytehq/airbyte/pull/11454) | Integration test enhancement for picking test-data and schemas | -| 0.2.11 | 2022-03-18 | [\#10793](https://github.com/airbytehq/airbyte/pull/10793) | Fix namespace with invalid characters | -| 0.2.10 | 2022-03-03 | [\#10755](https://github.com/airbytehq/airbyte/pull/10755) | Make sure to kill children threads and stop JVM | -| 0.2.8 | 2022-02-14 | [\#10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.2.7 | 2022-02-01 | [\#9959](https://github.com/airbytehq/airbyte/pull/9959) | Fix null pointer exception from buffered stream consumer. | -| 0.2.6 | 2022-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | -| 0.2.5 | 2022-01-18 | [\#9573](https://github.com/airbytehq/airbyte/pull/9573) | BigQuery Destination : update description for some input fields | -| 0.2.4 | 2022-01-17 | [\#8383](https://github.com/airbytehq/airbyte/issues/8383) | BigQuery/BiqQuery denorm Destinations : Support dataset-id prefixed by project-id | -| 0.2.3 | 2022-01-12 | [\#9415](https://github.com/airbytehq/airbyte/pull/9415) | BigQuery Destination : Fix GCS processing of Facebook data | -| 0.2.2 | 2021-12-22 | [\#9039](https://github.com/airbytehq/airbyte/pull/9039) | Added part_size configuration to UI for GCS staging | -| 0.2.1 | 2021-12-21 | [\#8574](https://github.com/airbytehq/airbyte/pull/8574) | Added namespace to Avro and Parquet record types | -| 0.2.0 | 2021-12-17 | [\#8788](https://github.com/airbytehq/airbyte/pull/8788) | BigQuery/BiqQuery denorm Destinations : Add possibility to use different types of GCS files | -| 0.1.11 | 2021-12-16 | [\#8816](https://github.com/airbytehq/airbyte/issues/8816) | Update dataset locations | -| 0.1.10 | 2021-11-09 | [\#7804](https://github.com/airbytehq/airbyte/pull/7804) | handle null values in fields described by a $ref definition | -| 0.1.9 | 2021-11-08 | [\#7736](https://github.com/airbytehq/airbyte/issues/7736) | Fixed the handling of ObjectNodes with $ref definition key | -| 0.1.8 | 2021-10-27 | [\#7413](https://github.com/airbytehq/airbyte/issues/7413) | Fixed DATETIME conversion for BigQuery | -| 0.1.7 | 2021-10-26 | [\#7240](https://github.com/airbytehq/airbyte/issues/7240) | Output partitioned/clustered tables | -| 0.1.6 | 2021-09-16 | [\#6145](https://github.com/airbytehq/airbyte/pull/6145) | BigQuery Denormalized support for date, datetime & timestamp types through the json "format" key | -| 0.1.5 | 2021-09-07 | [\#5881](https://github.com/airbytehq/airbyte/pull/5881) | BigQuery Denormalized NPE fix | -| 0.1.4 | 2021-09-04 | [\#5813](https://github.com/airbytehq/airbyte/pull/5813) | fix Stackoverflow error when receive a schema from source where "Array" type doesn't contain a required "items" element | -| 0.1.3 | 2021-08-07 | [\#5261](https://github.com/airbytehq/airbyte/pull/5261) | 🐛 Destination BigQuery\(Denormalized\): Fix processing arrays of records | -| 0.1.2 | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | -| 0.1.1 | 2021-06-21 | [\#3555](https://github.com/airbytehq/airbyte/pull/3555) | Partial Success in BufferedStreamConsumer | -| 0.1.0 | 2021-06-21 | [\#4176](https://github.com/airbytehq/airbyte/pull/4176) | Destination using Typed Struct and Repeated fields | diff --git a/docs/integrations/destinations/bigquery-migrations.md b/docs/integrations/destinations/bigquery-migrations.md new file mode 100644 index 000000000000..7f62d4880b4c --- /dev/null +++ b/docs/integrations/destinations/bigquery-migrations.md @@ -0,0 +1,14 @@ +# BigQuery Migration Guide + +## Upgrading to 2.0.0 + +This version introduces [Destinations V2](/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2), which provides better error handling, incremental delivery of data for large syncs, and improved final table structures. To review the breaking changes, and how to upgrade, see [here](/release_notes/upgrading_to_destinations_v2/#quick-start-to-upgrading). These changes will likely require updates to downstream dbt / SQL models, which we walk through [here](/release_notes/upgrading_to_destinations_v2/#updating-downstream-transformations). Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync. You can manually sync existing connections prior to the next scheduled sync to start the upgrade early. + +Worthy of specific mention, this version includes: + +- Per-record error handling +- Clearer table structure +- Removal of sub-tables for nested properties +- Removal of SCD tables + +Learn more about what's new in Destinations V2 [here](/understanding-airbyte/typing-deduping). \ No newline at end of file diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index e6d94ce03776..09fc5de2a0f1 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -1,75 +1,115 @@ # BigQuery -Setting up the BigQuery destination connector involves setting up the data loading method (BigQuery Standard method and Google Cloud Storage bucket) and configuring the BigQuery destination connector using the Airbyte UI. +Setting up the BigQuery destination connector involves setting up the data loading method (BigQuery +Standard method and Google Cloud Storage bucket) and configuring the BigQuery destination connector +using the Airbyte UI. This page guides you through setting up the BigQuery destination connector. ## Prerequisites -- For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your BigQuery connector to version `1.1.14` or newer +- For Airbyte Open Source users using the + [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, + [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to + version `v0.40.0-alpha` or newer and upgrade your BigQuery connector to version `1.1.14` or newer - [A Google Cloud project with BigQuery enabled](https://cloud.google.com/bigquery/docs/quickstarts/query-public-dataset-console) -- [A BigQuery dataset](https://cloud.google.com/bigquery/docs/quickstarts/quickstart-web-ui#create_a_dataset) to sync data to. +- [A BigQuery dataset](https://cloud.google.com/bigquery/docs/quickstarts/quickstart-web-ui#create_a_dataset) + to sync data to. - **Note:** Queries written in BigQuery can only reference datasets in the same physical location. If you plan on combining the data that Airbyte syncs with data from other datasets in your queries, create the datasets in the same location on Google Cloud. For more information, read [Introduction to Datasets](https://cloud.google.com/bigquery/docs/datasets-intro) + **Note:** Queries written in BigQuery can only reference datasets in the same physical location. + If you plan on combining the data that Airbyte syncs with data from other datasets in your + queries, create the datasets in the same location on Google Cloud. For more information, read + [Introduction to Datasets](https://cloud.google.com/bigquery/docs/datasets-intro) -- (Required for Airbyte Cloud; Optional for Airbyte Open Source) A Google Cloud [Service Account](https://cloud.google.com/iam/docs/service-accounts) with the [`BigQuery User`](https://cloud.google.com/bigquery/docs/access-control#bigquery) and [`BigQuery Data Editor`](https://cloud.google.com/bigquery/docs/access-control#bigquery) roles and the [Service Account Key in JSON format](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). - -## Connector modes - -While setting up the connector, you can configure it in the following modes: - -- **BigQuery**: Produces a normalized output by storing the JSON blob data in `_airbyte_raw_*` tables and then transforming and normalizing the data into separate tables, potentially `exploding` nested streams into their own tables if basic normalization is configured. -- **BigQuery (Denormalized)**: Leverages BigQuery capabilities with Structured and Repeated fields to produce a single "big" table per stream. Airbyte does not support normalization for this option at this time. +- (Required for Airbyte Cloud; Optional for Airbyte Open Source) A Google Cloud + [Service Account](https://cloud.google.com/iam/docs/service-accounts) with the + [`BigQuery User`](https://cloud.google.com/bigquery/docs/access-control#bigquery) and + [`BigQuery Data Editor`](https://cloud.google.com/bigquery/docs/access-control#bigquery) roles and + the + [Service Account Key in JSON format](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). ## Setup guide ### Step 1: Set up a data loading method -Although you can load data using BigQuery's [`INSERTS`](https://cloud.google.com/bigquery/docs/reference/standard-sql/dml-syntax), we highly recommend using a [Google Cloud Storage bucket](https://cloud.google.com/storage/docs/introduction) not only for performance and cost but reliability since larger datasets are prone to more failures when using standard inserts. +Although you can load data using BigQuery's +[`INSERTS`](https://cloud.google.com/bigquery/docs/reference/standard-sql/dml-syntax), we highly +recommend using a [Google Cloud Storage bucket](https://cloud.google.com/storage/docs/introduction) +not only for performance and cost but reliability since larger datasets are prone to more failures +when using standard inserts. #### (Recommended) Using a Google Cloud Storage bucket To use a Google Cloud Storage bucket: -1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) with the Protection Tools set to `none` or `Object versioning`. Make sure the bucket does not have a [retention policy](https://cloud.google.com/storage/docs/samples/storage-set-retention-policy). +1. [Create a Cloud Storage bucket](https://cloud.google.com/storage/docs/creating-buckets) with the + Protection Tools set to `none` or `Object versioning`. Make sure the bucket does not have a + [retention policy](https://cloud.google.com/storage/docs/samples/storage-set-retention-policy). 2. [Create an HMAC key and access ID](https://cloud.google.com/storage/docs/authentication/managing-hmackeys#create). -3. Grant the [`Storage Object Admin` role](https://cloud.google.com/storage/docs/access-control/iam-roles#standard-roles) to the Google Cloud [Service Account](https://cloud.google.com/iam/docs/service-accounts). -4. Make sure your Cloud Storage bucket is accessible from the machine running Airbyte. The easiest way to verify if Airbyte is able to connect to your bucket is via the check connection tool in the UI. - -Your bucket must be encrypted using a Google-managed encryption key (this is the default setting when creating a new bucket). We currently do not support buckets using customer-managed encryption keys (CMEK). You can view this setting under the "Configuration" tab of your GCS bucket, in the `Encryption type` row. +3. Grant the + [`Storage Object Admin` role](https://cloud.google.com/storage/docs/access-control/iam-roles#standard-roles) + to the Google Cloud [Service Account](https://cloud.google.com/iam/docs/service-accounts). This + must be the same service account as the one you configure for BigQuery access in the + [BigQuery connector setup step](#step-2-set-up-the-bigquery-connector). +4. Make sure your Cloud Storage bucket is accessible from the machine running Airbyte. The easiest + way to verify if Airbyte is able to connect to your bucket is via the check connection tool in + the UI. + +Your bucket must be encrypted using a Google-managed encryption key (this is the default setting +when creating a new bucket). We currently do not support buckets using customer-managed encryption +keys (CMEK). You can view this setting under the "Configuration" tab of your GCS bucket, in the +`Encryption type` row. #### Using `INSERT` -You can use BigQuery's [`INSERT`](https://cloud.google.com/bigquery/docs/reference/standard-sql/dml-syntax) statement to upload data directly from your source to BigQuery. While this is faster to set up initially, we strongly recommend not using this option for anything other than a quick demo. Due to the Google BigQuery SDK client limitations, using `INSERT` is 10x slower than using a Google Cloud Storage bucket, and you may see some failures for big datasets and slow sources (For example, if reading from a source takes more than 10-12 hours). For more details, refer to https://github.com/airbytehq/airbyte/issues/3549 +You can use BigQuery's +[`INSERT`](https://cloud.google.com/bigquery/docs/reference/standard-sql/dml-syntax) statement to +upload data directly from your source to BigQuery. While this is faster to set up initially, we +strongly recommend not using this option for anything other than a quick demo. Due to the Google +BigQuery SDK client limitations, using `INSERT` is 10x slower than using a Google Cloud Storage +bucket, and you may see some failures for big datasets and slow sources (For example, if reading +from a source takes more than 10-12 hours). For more details, refer to +https://github.com/airbytehq/airbyte/issues/3549 ### Step 2: Set up the BigQuery connector -1. Log into your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account. +1. Log into your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source + account. 2. Click **Destinations** and then click **+ New destination**. -3. On the Set up the destination page, select **BigQuery** or **BigQuery (denormalized typed struct)** from the **Destination type** dropdown depending on whether you want to set up the connector in [BigQuery](#connector-modes) or [BigQuery (Denormalized)](#connector-modes) mode. +3. On the Set up the destination page, select **BigQuery** from the **Destination type** dropdown. 4. Enter the name for the BigQuery connector. -5. For **Project ID**, enter your [Google Cloud project ID](https://cloud.google.com/resource-manager/docs/creating-managing-projects#identifying_projects). -6. For **Dataset Location**, select the location of your BigQuery dataset. - :::warning - You cannot change the location later. - ::: -7. For **Default Dataset ID**, enter the BigQuery [Dataset ID](https://cloud.google.com/bigquery/docs/datasets#create-dataset). -8. For **Loading Method**, select [Standard Inserts](#using-insert) or [GCS Staging](#recommended-using-a-google-cloud-storage-bucket). - :::tip - We recommend using the GCS Staging option. - ::: -9. For **Service Account Key JSON (Required for cloud, optional for open-source)**, enter the Google Cloud [Service Account Key in JSON format](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). -10. For **Transformation Query Run Type (Optional)**, select **interactive** to have [BigQuery run interactive query jobs](https://cloud.google.com/bigquery/docs/running-queries#queries) or **batch** to have [BigQuery run batch queries](https://cloud.google.com/bigquery/docs/running-queries#batch). - - :::note - Interactive queries are executed as soon as possible and count towards daily concurrent quotas and limits, while batch queries are executed as soon as idle resources are available in the BigQuery shared resource pool. If BigQuery hasn't started the query within 24 hours, BigQuery changes the job priority to interactive. Batch queries don't count towards your concurrent rate limit, making it easier to start many queries at once. - ::: - -11. For **Google BigQuery Client Chunk Size (Optional)**, use the default value of 15 MiB. Later, if you see networking or memory management problems with the sync (specifically on the destination), try decreasing the chunk size. In that case, the sync will be slower but more likely to succeed. +5. For **Project ID**, enter your + [Google Cloud project ID](https://cloud.google.com/resource-manager/docs/creating-managing-projects#identifying_projects). +6. For **Dataset Location**, select the location of your BigQuery dataset. :::warning You cannot + change the location later. ::: +7. For **Default Dataset ID**, enter the BigQuery + [Dataset ID](https://cloud.google.com/bigquery/docs/datasets#create-dataset). +8. For **Loading Method**, select [Standard Inserts](#using-insert) or + [GCS Staging](#recommended-using-a-google-cloud-storage-bucket). :::tip We recommend using the + GCS Staging option. ::: +9. For **Service Account Key JSON (Required for cloud, optional for open-source)**, enter the Google + Cloud + [Service Account Key in JSON format](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). +10. For **Transformation Query Run Type (Optional)**, select **interactive** to have + [BigQuery run interactive query jobs](https://cloud.google.com/bigquery/docs/running-queries#queries) + or **batch** to have + [BigQuery run batch queries](https://cloud.google.com/bigquery/docs/running-queries#batch). + + :::note Interactive queries are executed as soon as possible and count towards daily concurrent + quotas and limits, while batch queries are executed as soon as idle resources are available in + the BigQuery shared resource pool. If BigQuery hasn't started the query within 24 hours, + BigQuery changes the job priority to interactive. Batch queries don't count towards your + concurrent rate limit, making it easier to start many queries at once. ::: + +11. For **Google BigQuery Client Chunk Size (Optional)**, use the default value of 15 MiB. Later, if + you see networking or memory management problems with the sync (specifically on the + destination), try decreasing the chunk size. In that case, the sync will be slower but more + likely to succeed. ## Supported sync modes -The BigQuery destination connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): +The BigQuery destination connector supports the following +[sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): - Full Refresh Sync - Incremental - Append Sync @@ -77,52 +117,89 @@ The BigQuery destination connector supports the following [sync modes](https://d ## Output schema -Airbyte outputs each stream into its own table in BigQuery. Each table contains three columns: - -- `_airbyte_ab_id`: A UUID assigned by Airbyte to each event that is processed. The column type in BigQuery is `String`. -- `_airbyte_emitted_at`: A timestamp representing when the event was pulled from the data source. The column type in BigQuery is `Timestamp`. -- `_airbyte_data`: A JSON blob representing the event data. The column type in BigQuery is `String`. - -The output tables in BigQuery are partitioned and clustered by the Time-unit column `_airbyte_emitted_at` at a daily granularity. Partitions boundaries are based on UTC time. -This is useful to limit the number of partitions scanned when querying these partitioned tables, by using a predicate filter (a `WHERE` clause). Filters on the partitioning column are used to prune the partitions and reduce the query cost. (The parameter **Require partition filter** is not enabled by Airbyte, but you may toggle it by updating the produced tables.) +Airbyte outputs each stream into its own raw table in `airbyte_internal` dataset by default (can be +overriden by user) and a final table with Typed columns. Contents in raw table are _NOT_ +deduplicated. + +### Raw Table schema + +| Airbyte field | Description | Column type | +| ---------------------- | ------------------------------------------------------------------ | ----------- | +| \_airbyte_raw_id | A UUID assigned to each processed event | STRING | +| \_airbyte_extracted_at | A timestamp for when the event was pulled from the data source | TIMESTAMP | +| \_airbyte_loaded_at | Timestamp to indicate when the record was loaded into Typed tables | TIMESTAMP | +| \_airbyte_data | A JSON blob with the event data. | STRING | + +**Note:** Although the contents of the `_airbyte_data` are fairly stable, schema of the raw table +could be subject to change in future versions. + +### Final Table schema + +- `airbyte_raw_id`: A UUID assigned by Airbyte to each event that is processed. The column type in + BigQuery is `String`. +- `airbyte_extracted_at`: A timestamp representing when the event was pulled from the data source. + The column type in BigQuery is `Timestamp`. +- `_airbyte_meta`: A JSON blob representing typing errors. You can query these results to audit + misformatted or unexpected data. The column type in BigQuery is `JSON`. ... and a column of the + proper data type for each of the top-level properties from your source's schema. Arrays and + Objects will remain as JSON columns in BigQuery. Learn more about Typing and Deduping + [here](/understanding-airbyte/typing-deduping) + +The output tables in BigQuery are partitioned by the Time-unit column `airbyte_extracted_at` at a +daily granularity and clustered by `airbyte_extracted_at` and the table Primary Keys. Partitions +boundaries are based on UTC time. This is useful to limit the number of partitions scanned when +querying these partitioned tables, by using a predicate filter (a `WHERE` clause). Filters on the +partitioning column are used to prune the partitions and reduce the query cost. (The parameter +**Require partition filter** is not enabled by Airbyte, but you may toggle it by updating the +produced tables.) ## BigQuery Naming Conventions -Follow [BigQuery Datasets Naming conventions](https://cloud.google.com/bigquery/docs/datasets#dataset-naming). +Follow +[BigQuery Datasets Naming conventions](https://cloud.google.com/bigquery/docs/datasets#dataset-naming). -Airbyte converts any invalid characters into `_` characters when writing data. However, since datasets that begin with `_` are hidden on the BigQuery Explorer panel, Airbyte prepends the namespace with `n` for converted namespaces. +Airbyte converts any invalid characters into `_` characters when writing data. However, since +datasets that begin with `_` are hidden on the BigQuery Explorer panel, Airbyte prepends the +namespace with `n` for converted namespaces. ## Data type map -| Airbyte type | BigQuery type | BigQuery denormalized type | -| :---------------------------------- | :------------ | :------------------------- | -| DATE | DATE | DATE | -| STRING (BASE64) | STRING | STRING | -| NUMBER | FLOAT | NUMBER | -| OBJECT | STRING | RECORD | -| STRING | STRING | STRING | -| BOOLEAN | BOOLEAN | BOOLEAN | -| INTEGER | INTEGER | INTEGER | -| STRING (BIG_NUMBER) | STRING | STRING | -| STRING (BIG_INTEGER) | STRING | STRING | -| ARRAY | REPEATED | REPEATED | -| STRING (TIMESTAMP_WITH_TIMEZONE) | TIMESTAMP | DATETIME | -| STRING (TIMESTAMP_WITHOUT_TIMEZONE) | TIMESTAMP | DATETIME | +| Airbyte type | BigQuery type | +| :---------------------------------- | :------------ | +| STRING | STRING | +| STRING (BASE64) | STRING | +| STRING (BIG_NUMBER) | STRING | +| STRING (BIG_INTEGER) | STRING | +| NUMBER | NUMERIC | +| INTEGER | INT64 | +| BOOLEAN | BOOL | +| STRING (TIMESTAMP_WITH_TIMEZONE) | TIMESTAMP | +| STRING (TIMESTAMP_WITHOUT_TIMEZONE) | DATETIME | +| STRING (TIME_WITH_TIMEZONE) | STRING | +| STRING (TIME_WITHOUT_TIMEZONE) | TIME | +| DATE | DATE | +| OBJECT | JSON | +| ARRAY | JSON | ## Troubleshooting permission issues The service account does not have the proper permissions. -- Make sure the BigQuery service account has `BigQuery User` and `BigQuery Data Editor` roles or equivalent permissions as those two roles. -- If the GCS staging mode is selected, ensure the BigQuery service account has the right permissions to the GCS bucket and path or the `Cloud Storage Admin` role, which includes a superset of the required permissions. +- Make sure the BigQuery service account has `BigQuery User` and `BigQuery Data Editor` roles or + equivalent permissions as those two roles. +- If the GCS staging mode is selected, ensure the BigQuery service account has the right permissions + to the GCS bucket and path or the `Cloud Storage Admin` role, which includes a superset of the + required permissions. The HMAC key is wrong. -- Make sure the HMAC key is created for the BigQuery service account, and the service account has permission to access the GCS bucket and path. +- Make sure the HMAC key is created for the BigQuery service account, and the service account has + permission to access the GCS bucket and path. ## Tutorials -Now that you have set up the BigQuery destination connector, check out the following BigQuery tutorials: +Now that you have set up the BigQuery destination connector, check out the following BigQuery +tutorials: - [Export Google Analytics data to BigQuery](https://airbyte.com/tutorials/export-google-analytics-to-bigquery) - [Load data from Facebook Ads to BigQuery](https://airbyte.com/tutorials/facebook-ads-to-bigquery) @@ -131,10 +208,74 @@ Now that you have set up the BigQuery destination connector, check out the follo ## Changelog -### bigquery - | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 2.3.30 | 2024-01-12 | [34226](https://github.com/airbytehq/airbyte/pull/34226) | Upgrade CDK to 0.12.0; Cleanup dependencies | +| 2.3.29 | 2024-01-09 | [34003](https://github.com/airbytehq/airbyte/pull/34003) | Fix loading credentials from GCP Env | +| 2.3.28 | 2024-01-08 | [34021](https://github.com/airbytehq/airbyte/pull/34021) | Add idempotency ids in dummy insert for check call | +| 2.3.27 | 2024-01-05 | [\#33948](https://github.com/airbytehq/airbyte/pull/33948) | Skip retrieving initial table state when setup fails | +| 2.3.26 | 2024-01-04 | [\#33730](https://github.com/airbytehq/airbyte/pull/33730) | Internal code structure changes | +| 2.3.25 | 2023-12-20 | [\#33704](https://github.com/airbytehq/airbyte/pull/33704) | Update to java CDK 0.10.0 (no changes) | +| 2.3.24 | 2023-12-20 | [\#33697](https://github.com/airbytehq/airbyte/pull/33697) | Stop creating unnecessary tmp tables | +| 2.3.23 | 2023-12-18 | [\#33124](https://github.com/airbytehq/airbyte/pull/33124) | Make Schema Creation Separate from Table Creation | +| 2.3.22 | 2023-12-14 | [\#33451](https://github.com/airbytehq/airbyte/pull/33451) | Remove old spec option | +| 2.3.21 | 2023-12-13 | [\#33232](https://github.com/airbytehq/airbyte/pull/33232) | Only run typing+deduping for a stream if the stream had any records | +| 2.3.20 | 2023-12-08 | [\#33263](https://github.com/airbytehq/airbyte/pull/33263) | Adopt java CDK version 0.7.0 | +| 2.3.19 | 2023-12-07 | [\#32326](https://github.com/airbytehq/airbyte/pull/32326) | Update common T&D interfaces | +| 2.3.18 | 2023-12-04 | [\#33084](https://github.com/airbytehq/airbyte/pull/33084) | T&D SQL statements moved to debug log level | +| 2.3.17 | 2023-12-04 | [\#33078](https://github.com/airbytehq/airbyte/pull/33078) | Further increase gcs COPY timeout | +| 2.3.16 | 2023-11-14 | [\#32526](https://github.com/airbytehq/airbyte/pull/32526) | Clean up memory manager logs. | +| 2.3.15 | 2023-11-13 | [\#32468](https://github.com/airbytehq/airbyte/pull/32468) | Further error grouping enhancements | +| 2.3.14 | 2023-11-06 | [\#32234](https://github.com/airbytehq/airbyte/pull/32234) | Remove unused config option. | +| 2.3.13 | 2023-11-08 | [\#32125](https://github.com/airbytehq/airbyte/pull/32125) | fix compiler warnings | +| 2.3.12 | 2023-11-08 | [\#32309](https://github.com/airbytehq/airbyte/pull/32309) | Revert: Use Typed object for connection config | +| 2.3.11 | 2023-11-07 | [\#32147](https://github.com/airbytehq/airbyte/pull/32147) | Use Typed object for connection config | +| 2.3.10 | 2023-11-07 | [\#32261](https://github.com/airbytehq/airbyte/pull/32261) | Further improve error reporting | +| 2.3.9 | 2023-11-07 | [\#32112](https://github.com/airbytehq/airbyte/pull/32112) | GCS staging mode: reduce flush frequency to use rate limit more efficiently | +| 2.3.8 | 2023-11-06 | [\#32026](https://github.com/airbytehq/airbyte/pull/32026) | Move SAFE_CAST transaction to separate transactions | +| 2.3.7 | 2023-11-06 | [\#32190](https://github.com/airbytehq/airbyte/pull/32190) | Further improve error reporting | +| 2.3.6 | 2023-11-06 | [\#32193](https://github.com/airbytehq/airbyte/pull/32193) | Adopt java CDK version 0.4.1. | +| 2.3.5 | 2023-11-02 | [\#31983](https://github.com/airbytehq/airbyte/pull/31983) | Improve error reporting | +| 2.3.4 | 2023-10-31 | [\#32010](https://github.com/airbytehq/airbyte/pull/32010) | Add additional data centers. | +| 2.3.3 | 2023-10-30 | [\#31985](https://github.com/airbytehq/airbyte/pull/31985) | Delay upgrade deadline to Nov 7 | +| 2.3.2 | 2023-10-30 | [\#31960](https://github.com/airbytehq/airbyte/pull/31960) | Adopt java CDK version 0.2.0. | +| 2.3.1 | 2023-10-27 | [\#31529](https://github.com/airbytehq/airbyte/pull/31529) | Performance enhancement (switch to a `merge` statement for incremental-dedup syncs) | +| 2.3.0 | 2023-10-25 | [\#31686](https://github.com/airbytehq/airbyte/pull/31686) | Opt out flag for typed and deduped tables | +| 2.2.0 | 2023-10-25 | [\#31520](https://github.com/airbytehq/airbyte/pull/31520) | Stop deduping raw table | +| 2.1.6 | 2023-10-23 | [\#31717](https://github.com/airbytehq/airbyte/pull/31717) | Remove inadvertent Destination v2 check | +| 2.1.5 | 2023-10-17 | [\#30069](https://github.com/airbytehq/airbyte/pull/30069) | Staging destination async | +| 2.1.4 | 2023-10-17 | [\#31191](https://github.com/airbytehq/airbyte/pull/31191) | Improve typing+deduping performance by filtering new raw records on extracted_at | +| 2.1.3 | 2023-10-10 | [\#31358](https://github.com/airbytehq/airbyte/pull/31358) | Stringify array and object types for type:string column in final table | +| 2.1.2 | 2023-10-10 | [\#31194](https://github.com/airbytehq/airbyte/pull/31194) | Deallocate unused per stream buffer memory when empty | +| 2.1.1 | 2023-10-10 | [\#31083](https://github.com/airbytehq/airbyte/pull/31083) | Fix precision of numeric values in async destinations | +| 2.1.0 | 2023-10-09 | [\#31149](https://github.com/airbytehq/airbyte/pull/31149) | No longer fail syncs when PKs are null - try do dedupe anyway | +| 2.0.26 | 2023-10-09 | [\#31198](https://github.com/airbytehq/airbyte/pull/31198) | Clarify configuration groups | +| 2.0.25 | 2023-10-09 | [\#31185](https://github.com/airbytehq/airbyte/pull/31185) | Increase staging file upload timeout to 5 minutes | +| 2.0.24 | 2023-10-06 | [\#31139](https://github.com/airbytehq/airbyte/pull/31139) | Bump CDK version | +| 2.0.23 | 2023-10-06 | [\#31129](https://github.com/airbytehq/airbyte/pull/31129) | Reduce async buffer size | +| 2.0.22 | 2023-10-04 | [\#31082](https://github.com/airbytehq/airbyte/pull/31082) | Revert null PK checks | +| 2.0.21 | 2023-10-03 | [\#31028](https://github.com/airbytehq/airbyte/pull/31028) | Update timeout | +| 2.0.20 | 2023-09-26 | [\#30779](https://github.com/airbytehq/airbyte/pull/30779) | Final table PK columns become non-null and skip check for null PKs in raw records (performance) | +| 2.0.19 | 2023-09-26 | [\#30775](https://github.com/airbytehq/airbyte/pull/30775) | Increase async block size | +| 2.0.18 | 2023-09-27 | [\#30739](https://github.com/airbytehq/airbyte/pull/30739) | Fix column name collision detection | +| 2.0.17 | 2023-09-26 | [\#30696](https://github.com/airbytehq/airbyte/pull/30696) | Attempt unsafe typing operations with an exception clause | +| 2.0.16 | 2023-09-22 | [\#30697](https://github.com/airbytehq/airbyte/pull/30697) | Improve resiliency to unclean exit during schema change | +| 2.0.15 | 2023-09-21 | [\#30640](https://github.com/airbytehq/airbyte/pull/30640) | Handle streams with identical name and namespace | +| 2.0.14 | 2023-09-20 | [\#30069](https://github.com/airbytehq/airbyte/pull/30069) | Staging destination async | +| 2.0.13 | 2023-09-19 | [\#30592](https://github.com/airbytehq/airbyte/pull/30592) | Internal code changes | +| 2.0.12 | 2023-09-19 | [\#30319](https://github.com/airbytehq/airbyte/pull/30319) | Improved testing | +| 2.0.11 | 2023-09-18 | [\#30551](https://github.com/airbytehq/airbyte/pull/30551) | GCS Staging is first loading method option | +| 2.0.10 | 2023-09-15 | [\#30491](https://github.com/airbytehq/airbyte/pull/30491) | Improve error message display | +| 2.0.9 | 2023-09-14 | [\#30439](https://github.com/airbytehq/airbyte/pull/30439) | Fix a transient error | +| 2.0.8 | 2023-09-12 | [\#30364](https://github.com/airbytehq/airbyte/pull/30364) | Add log message | +| 2.0.7 | 2023-08-29 | [\#29878](https://github.com/airbytehq/airbyte/pull/29878) | Internal code changes | +| 2.0.6 | 2023-09-05 | [\#29917](https://github.com/airbytehq/airbyte/pull/29917) | Improve performance by changing metadata error array construction from ARRAY_CONCAT to ARRAY_AGG | +| 2.0.5 | 2023-08-31 | [\#30020](https://github.com/airbytehq/airbyte/pull/30020) | Run typing and deduping tasks in parallel | +| 2.0.4 | 2023-09-05 | [\#30117](https://github.com/airbytehq/airbyte/pull/30117) | Type and Dedupe at sync start and then every 6 hours | +| 2.0.3 | 2023-09-01 | [\#30056](https://github.com/airbytehq/airbyte/pull/30056) | Internal refactor, no behavior change | +| 2.0.2 | 2023-09-01 | [\#30120](https://github.com/airbytehq/airbyte/pull/30120) | Improve performance on very wide streams by skipping SAFE_CAST on strings | +| 2.0.1 | 2023-08-29 | [\#29972](https://github.com/airbytehq/airbyte/pull/29972) | Publish a new version to supersede old v2.0.0 | +| 2.0.0 | 2023-08-27 | [\#29783](https://github.com/airbytehq/airbyte/pull/29783) | Destinations V2 | | 1.10.2 | 2023-08-24 | [\#29805](https://github.com/airbytehq/airbyte/pull/29805) | Destinations v2: Don't soft reset in migration | | 1.10.1 | 2023-08-23 | [\#29774](https://github.com/airbytehq/airbyte/pull/29774) | Destinations v2: Don't soft reset overwrite syncs | | 1.10.0 | 2023-08-21 | [\#29636](https://github.com/airbytehq/airbyte/pull/29636) | Destinations v2: Several Critical Bug Fixes (cursorless dedup, improved floating-point handling, improved special characters handling; improved error handling) | @@ -231,4 +372,4 @@ Now that you have set up the BigQuery destination connector, check out the follo | 0.3.10 | 2021-07-28 | [\#3549](https://github.com/airbytehq/airbyte/issues/3549) | Add extended logs and made JobId filled with region and projectId | | 0.3.9 | 2021-07-28 | [\#5026](https://github.com/airbytehq/airbyte/pull/5026) | Add sanitized json fields in raw tables to handle quotes in column names | | 0.3.6 | 2021-06-18 | [\#3947](https://github.com/airbytehq/airbyte/issues/3947) | Service account credentials are now optional. | -| 0.3.4 | 2021-06-07 | [\#3277](https://github.com/airbytehq/airbyte/issues/3277) | Add dataset location option | +| 0.3.4 | 2021-06-07 | [\#3277](https://github.com/airbytehq/airbyte/issues/3277) | Add dataset location option | \ No newline at end of file diff --git a/docs/integrations/destinations/chroma.md b/docs/integrations/destinations/chroma.md new file mode 100644 index 000000000000..10d65af7170a --- /dev/null +++ b/docs/integrations/destinations/chroma.md @@ -0,0 +1,87 @@ +# Chroma +This page guides you through the process of setting up the [Chroma](https://docs.trychroma.com/?lang=py) destination connector. + + + +## Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :----------------------------- | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | Yes | | +| Incremental - Append + Deduped | Yes | | + +#### Output Schema + +Only one stream will exist to collect data from all source streams. This will be in a [collection](https://docs.trychroma.com/usage-guide#using-collections) in [Chroma](https://docs.trychroma.com/?lang=py) whose name will be defined by the user, and validated and corrected by Airbyte. + +For each record, a UUID string is generated and used as the document id. The embeddings generated as defined will be stored as embeddings. Data in the text fields will be stored as documents and those in the metadata fields will be stored as metadata. + +## Getting Started \(Airbyte Open Source\) + + +You can connect to a Chroma instance either in client/server mode or in a local persistent mode. For the local persistent mode, the database file will be saved in the path defined in the `path` config parameter. Note that `path` must be an absolute path, prefixed with `/local`. + +:::danger + +Persistent Client mode is not supported on Kubernetes + +::: + +By default, the `LOCAL_ROOT` env variable in the `.env` file is set `/tmp/airbyte_local`. + +The local mount is mounted by Docker onto `LOCAL_ROOT`. This means the `/local` is substituted by `/tmp/airbyte_local` by default. + +:::caution + +Please make sure that Docker Desktop has access to `/tmp` (and `/private` on a MacOS, as /tmp has a symlink that points to /private. It will not work otherwise). You allow it with "File sharing" in `Settings -> Resources -> File sharing -> add the one or two above folder` and hit the "Apply & restart" button. + +::: + +#### Requirements + +To use the Chroma destination, you'll need: +- An account with API access for OpenAI, Cohere (depending on which embedding method you want to use) or neither (if you want to use the [default chroma embedding function](https://docs.trychroma.com/embeddings#default-all-minilm-l6-v2)) +- A Chroma db instance (client/server mode or persistent mode) +- Credentials (for cient/server mode) +- Local File path (for Persistent mode) + +#### Configure Network Access + +Make sure your Chroma database can be accessed by Airbyte. If your database is within a VPC, you may need to allow access from the IP you're using to expose Airbyte. + + +### Setup the Chroma Destination in Airbyte + +You should now have all the requirements needed to configure Chroma as a destination in the UI. You'll need the following information to configure the Chroma destination: + +- (Required) **Text fields to embed** +- (Optional) **Text splitter** Options around configuring the chunking process provided by the [Langchain Python library](https://python.langchain.com/docs/get_started/introduction). +- (Required) **Fields to store as metadata** +- (Required) **Collection** The name of the collection in Chroma db to store your data +- (Required) Authentication method + - For client/server mode + - **Host** for example localhost + - **Port** for example 8000 + - **Username** (Optional) + - **Password** (Optional) + - For persistent mode + - **Path** The path to the local database file. Note that `path` must be an absolute path, prefixed with `/local`. +- (Optional) Embedding + - **OpenAI API key** if using OpenAI for embedding + - **Cohere API key** if using Cohere for embedding + - Embedding **Field name** and **Embedding dimensions** if getting the embeddings from stream records + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :--------------------------------------------------------- | :----------------------------------------- | +| 0.0.9 | 2023-12-11 | [33303](https://github.com/airbytehq/airbyte/pull/33303) | Fix bug with embedding special tokens | +| 0.0.8 | 2023-12-01 | [32697](https://github.com/airbytehq/airbyte/pull/32697) | Allow omitting raw text | +| 0.0.7 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Support deleting records for CDC sources | +| 0.0.6 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | +| 0.0.5 | 2023-10-23 | [#31563](https://github.com/airbytehq/airbyte/pull/31563) | Add field mapping option | +| 0.0.4 | 2023-10-15 | [#31329](https://github.com/airbytehq/airbyte/pull/31329) | Add OpenAI-compatible embedder option | +| 0.0.3 | 2023-10-04 | [#31075](https://github.com/airbytehq/airbyte/pull/31075) | Fix OpenAI embedder batch size | +| 0.0.2 | 2023-09-29 | [#30820](https://github.com/airbytehq/airbyte/pull/30820) | Update CDK | +| 0.0.1 | 2023-09-08 | [#30023](https://github.com/airbytehq/airbyte/pull/30023) | 🎉 New Destination: Chroma (Vector Database) | diff --git a/docs/integrations/destinations/clickhouse.md b/docs/integrations/destinations/clickhouse.md index 75da81407f48..02446ba825f6 100644 --- a/docs/integrations/destinations/clickhouse.md +++ b/docs/integrations/destinations/clickhouse.md @@ -21,7 +21,7 @@ Each stream will be output into its own table in ClickHouse. Each table will con Airbyte Cloud only supports connecting to your ClickHouse instance with SSL or TLS encryption, which is supported by [ClickHouse JDBC driver](https://github.com/ClickHouse/clickhouse-jdbc). -## Getting Started \(Airbyte Open-Source\) +## Getting Started \(Airbyte Open Source\) #### Requirements diff --git a/docs/integrations/destinations/csv.md b/docs/integrations/destinations/csv.md index 4cc00f440c79..223c618b8f8b 100644 --- a/docs/integrations/destinations/csv.md +++ b/docs/integrations/destinations/csv.md @@ -69,7 +69,7 @@ You can also copy the output file to your host machine, the following command wi docker cp airbyte-server:/tmp/airbyte_local/{destination_path}/{filename}.csv . ``` -Note: If you are running Airbyte on Windows with Docker backed by WSL2, you have to use similar step as above or refer to this [link](../../operator-guides/locating-files-local-destination.md) for an alternative approach. +Note: If you are running Airbyte on Windows with Docker backed by WSL2, you have to use similar step as above or refer to this [link](/integrations/locating-files-local-destination.md) for an alternative approach. ## Changelog diff --git a/docs/integrations/destinations/databend.md b/docs/integrations/destinations/databend.md index e25a80f7ec88..444a47473a6d 100644 --- a/docs/integrations/destinations/databend.md +++ b/docs/integrations/destinations/databend.md @@ -20,7 +20,7 @@ Each stream will be output into its own table in Databend. Each table will conta ## Getting Started (Airbyte Cloud) Coming soon... -## Getting Started (Airbyte Open-Source) +## Getting Started (Airbyte Open Source) You can follow the [Connecting to a Warehouse docs](https://docs.databend.com/using-databend-cloud/warehouses/connecting-a-warehouse) to get the user, password, host etc. Or you can create such a user by running: diff --git a/docs/integrations/destinations/databricks.md b/docs/integrations/destinations/databricks.md index c8a6a0516d49..d39e64084c44 100644 --- a/docs/integrations/destinations/databricks.md +++ b/docs/integrations/destinations/databricks.md @@ -2,11 +2,20 @@ ## Overview -This destination syncs data to Delta Lake on Databricks Lakehouse. Each stream is written to its own [delta-table](https://delta.io/). +This destination syncs data to Delta Lake on Databricks Lakehouse. Each stream is written to its own +[delta-table](https://delta.io/). -This connector requires a JDBC driver to connect to the Databricks cluster. By using the driver and the connector, you must agree to the [JDBC ODBC driver license](https://databricks.com/jdbc-odbc-driver-license). This means that you can only use this connector to connect third party applications to Apache Spark SQL within a Databricks offering using the ODBC and/or JDBC protocols. +This connector requires a JDBC driver to connect to the Databricks cluster. By using the driver and +the connector, you must agree to the +[JDBC ODBC driver license](https://databricks.com/jdbc-odbc-driver-license). This means that you can +only use this connector to connect third party applications to Apache Spark SQL within a Databricks +offering using the ODBC and/or JDBC protocols. -Currently, this connector requires 30+MB of memory for each stream. When syncing multiple streams, it may run into an out-of-memory error if the allocated memory is too small. This performance bottleneck is tracked in [this issue](https://github.com/airbytehq/airbyte/issues/11424). Once this issue is resolved, the connector should be able to sync an almost infinite number of streams with less than 500MB of memory. +Currently, this connector requires 30+MB of memory for each stream. When syncing multiple streams, +it may run into an out-of-memory error if the allocated memory is too small. This performance +bottleneck is tracked in [this issue](https://github.com/airbytehq/airbyte/issues/11424). Once this +issue is resolved, the connector should be able to sync an almost infinite number of streams with +less than 500MB of memory. ## Getting started @@ -14,23 +23,31 @@ Currently, this connector requires 30+MB of memory for each stream. When syncing ### 1. Create a Databricks Workspace -- Follow Databricks guide [Create a workspace using the account console](https://docs.databricks.com/administration-guide/workspace/create-workspace.html#create-a-workspace-using-the-account-console). - > **_IMPORTANT:_** Don't forget to create a [cross-account IAM role](https://docs.databricks.com/administration-guide/cloud-configurations/aws/iam-role.html#create-a-cross-account-iam-role) for workspaces +- Follow Databricks guide + [Create a workspace using the account console](https://docs.databricks.com/administration-guide/workspace/create-workspace.html#create-a-workspace-using-the-account-console). + > **_IMPORTANT:_** Don't forget to create a + > [cross-account IAM role](https://docs.databricks.com/administration-guide/cloud-configurations/aws/iam-role.html#create-a-cross-account-iam-role) + > for workspaces > **_TIP:_** Alternatively use Databricks quickstart for new workspace > ![](../../.gitbook/assets/destination/databricks/databricks_workspace_quciksetup.png) ### 2. Create a metastore and attach it to workspace -> **_IMPORTANT:_** The metastore should be in the same region as the workspaces you want to use to access the data. Make sure that this matches the region of the cloud storage bucket you created earlier. +> **_IMPORTANT:_** The metastore should be in the same region as the workspaces you want to use to +> access the data. Make sure that this matches the region of the cloud storage bucket you created +> earlier. #### Setup storage bucket and IAM role in AWS -Follow [Configure a storage bucket and IAM role in AWS](https://docs.databricks.com/data-governance/unity-catalog/get-started.html#configure-a-storage-bucket-and-iam-role-in-aws) to setup AWS bucket with necessary permissions. +Follow +[Configure a storage bucket and IAM role in AWS](https://docs.databricks.com/data-governance/unity-catalog/get-started.html#configure-a-storage-bucket-and-iam-role-in-aws) +to setup AWS bucket with necessary permissions. #### Create metastore -- Login into Databricks [account console](https://accounts.cloud.databricks.com/login) with admin permissions. +- Login into Databricks [account console](https://accounts.cloud.databricks.com/login) with admin + permissions. - Go to Data tab and hit Create metastore button: ![](../../.gitbook/assets/destination/databricks/databricks_new_metastore.png) @@ -41,8 +58,11 @@ Follow [Configure a storage bucket and IAM role in AWS](https://docs.databricks. - `Name` - `Region` The metastore should be in same region as the workspace. - - `S3 bucket path` created at [Setup storage bucket and IAM role in AWS](#setup-storage-bucket-and-iam-role-in-aws) step. - - `IAM role ARN` created at [Setup storage bucket and IAM role in AWS](#setup-storage-bucket-and-iam-role-in-aws) step. Example: `arn:aws:iam:::role/` + - `S3 bucket path` created at + [Setup storage bucket and IAM role in AWS](#setup-storage-bucket-and-iam-role-in-aws) step. + - `IAM role ARN` created at + [Setup storage bucket and IAM role in AWS](#setup-storage-bucket-and-iam-role-in-aws) step. + Example: `arn:aws:iam:::role/` - Select the workspaces in `Assign to workspaces` tab and click Assign. @@ -134,16 +154,24 @@ Follow [Configure a storage bucket and IAM role in AWS](https://docs.databricks. ![](../../.gitbook/assets/destination/databricks/databricks_new_external_location.png) -> **_TIP:_** The new `Storage credential` can be added in the `Storage Credentials` tab or use same as for Metastore. +> **_TIP:_** The new `Storage credential` can be added in the `Storage Credentials` tab or use same +> as for Metastore. ## Airbyte Setup ### Databricks fields -- `Agree to the Databricks JDBC Driver Terms & Conditions` - [Databricks JDBC ODBC driver license](https://www.databricks.com/legal/jdbc-odbc-driver-license). -- `Server Hostname` - can be taken from [4. Databricks SQL Warehouse connection details](#4-databricks-sql-warehouse-connection-details) or [6. Databricks Cluster connection details](#6-databricks-cluster-connection-details) steps. -- `HTTP Path` - can be taken from [4. Databricks SQL Warehouse connection details](#4-databricks-sql-warehouse-connection-details) or [6. Databricks Cluster connection details](#6-databricks-cluster-connection-details) steps. -- `Port` - can be taken from [4. Databricks SQL Warehouse connection details](#4-databricks-sql-warehouse-connection-details) or [6. Databricks Cluster connection details](#6-databricks-cluster-connection-details) steps. +- `Agree to the Databricks JDBC Driver Terms & Conditions` - + [Databricks JDBC ODBC driver license](https://www.databricks.com/legal/jdbc-odbc-driver-license). +- `Server Hostname` - can be taken from + [4. Databricks SQL Warehouse connection details](#4-databricks-sql-warehouse-connection-details) + or [6. Databricks Cluster connection details](#6-databricks-cluster-connection-details) steps. +- `HTTP Path` - can be taken from + [4. Databricks SQL Warehouse connection details](#4-databricks-sql-warehouse-connection-details) + or [6. Databricks Cluster connection details](#6-databricks-cluster-connection-details) steps. +- `Port` - can be taken from + [4. Databricks SQL Warehouse connection details](#4-databricks-sql-warehouse-connection-details) + or [6. Databricks Cluster connection details](#6-databricks-cluster-connection-details) steps. - `Access Token` - can be taken from [7. Create Databricks Token](#7-create-databricks-token) step. ### Data Source @@ -156,25 +184,39 @@ You could choose a data source type #### Managed tables data source type -Please check Databricks documentation about [What is managed tables](https://docs.databricks.com/lakehouse/data-objects.html#what-is-a-managed-table) +Please check Databricks documentation about +[What is managed tables](https://docs.databricks.com/lakehouse/data-objects.html#what-is-a-managed-table) > **_TIP:_** There is no addition setup should be done for this type. #### Amazon S3 data source type (External storage) -> **_IMPORTANT:_** Make sure the `External Locations` has been added to the workspace. Check [Adding External Locations](#8-adding-external-locations-optional) step. +> **_IMPORTANT:_** Make sure the `External Locations` has been added to the workspace. Check +> [Adding External Locations](#8-adding-external-locations-optional) step. Provide your Amazon S3 data: - `S3 Bucket Name` - The bucket name - `S3 Bucket Path` - Subdirectory under the above bucket to sync the data into -- `S3 Bucket Region` - See [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes. - > **_IMPORTANT:_** The metastore should be in the same region as the workspaces you want to use to access the data. Make sure that this matches the region of the cloud storage bucket you created earlier. +- `S3 Bucket Region` - See + [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) + for all region codes. + > **_IMPORTANT:_** The metastore should be in the same region as the workspaces you want to use to + > access the data. Make sure that this matches the region of the cloud storage bucket you created + > earlier. - `S3 Access Key ID` - Corresponding key to the above key id - `S3 Secret Access Key` - - - See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key. - - We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the bucket. -- `S3 Filename pattern` - The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: {date}, {date:yyyy_MM}, {timestamp}, {timestamp:millis}, {timestamp:micros}, {part_number}, {sync_id}, {format_extension}. Please, don't use empty space and not supportable placeholders, as they won't be recognized + - See + [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) + on how to generate an access key. + - We recommend creating an Airbyte-specific user. This user will require + [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) + to objects in the bucket. +- `S3 Filename pattern` - The pattern allows you to set the file-name format for the S3 staging + file(s), next placeholders combinations are currently supported: `{date}`, `{date:yyyy_MM}`, + `{timestamp}`, `{timestamp:millis}`, `{timestamp:micros}`, `{part_number}`, `{sync_id}`, + `{format_extension}`. Please, don't use empty space and not supportable placeholders, as they + won't be recognized #### Azure Blob Storage data source type (External storage) @@ -191,34 +233,39 @@ Provide your Amazon S3 data: ## Configuration -| Category | Parameter | Type | Notes | -| :------------------ | :-------------------- | :-----: | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Databricks | Server Hostname | string | Required. Example: `abc-12345678-wxyz.cloud.databricks.com`. See [documentation](https://docs.databricks.com/integrations/bi/jdbc-odbc-bi.html#get-server-hostname-port-http-path-and-jdbc-url). Please note that this is the server for the Databricks Cluster. It is different from the SQL Endpoint Cluster. | -| | HTTP Path | string | Required. Example: `sql/protocolvx/o/1234567489/0000-1111111-abcd90`. See [documentation](https://docs.databricks.com/integrations/bi/jdbc-odbc-bi.html#get-server-hostname-port-http-path-and-jdbc-url). | -| | Port | string | Optional. Default to "443". See [documentation](https://docs.databricks.com/integrations/bi/jdbc-odbc-bi.html#get-server-hostname-port-http-path-and-jdbc-url). | -| | Personal Access Token | string | Required. Example: `dapi0123456789abcdefghij0123456789AB`. See [documentation](https://docs.databricks.com/sql/user/security/personal-access-tokens.html). | -| General | Databricks catalog | string | Optional. The name of the catalog. If not specified otherwise, the "hive_metastore" will be used. | -| | Database schema | string | Optional. The default schema tables are written. If not specified otherwise, the "default" will be used. | -| | Schema evolution | boolean | Optional. The connector enables automatic schema evolution in the destination tables. | -| | Purge Staging Data | boolean | The connector creates staging files and tables on S3 or Azure. By default, they will be purged when the data sync is complete. Set it to `false` for debugging purposes. | -| Data Source - S3 | Bucket Name | string | Name of the bucket to sync data into. | -| | Bucket Path | string | Subdirectory under the above bucket to sync the data into. | -| | Region | string | See [documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes. | -| | Access Key ID | string | AWS/Minio credential. | -| | Secret Access Key | string | AWS/Minio credential. | -| | S3 Filename pattern | string | The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: {date}, {date:yyyy_MM}, {timestamp}, {timestamp:millis}, {timestamp:micros}, {part_number}, {sync_id}, {format_extension}. Please, don't use empty space and not supportable placeholders, as they won't recognized. | -| Data Source - Azure | Account Name | string | Name of the account to sync data into. | -| | Container Name | string | Container under the above account to sync the data into. | -| | SAS token | string | Shared-access signature token for the above account. | -| | Endpoint domain name | string | Usually blob.core.windows.net. | - -⚠️ Please note that under "Full Refresh Sync" mode, data in the configured bucket and path will be wiped out before each sync. We recommend you provision a dedicated S3 or Azure resource for this sync to prevent unexpected data deletion from misconfiguration. ⚠️ +| Category | Parameter | Type | Notes | +| :------------------ | :-------------------- | :-----: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Databricks | Server Hostname | string | Required. Example: `abc-12345678-wxyz.cloud.databricks.com`. See [documentation](https://docs.databricks.com/integrations/bi/jdbc-odbc-bi.html#get-server-hostname-port-http-path-and-jdbc-url). Please note that this is the server for the Databricks Cluster. It is different from the SQL Endpoint Cluster. | +| | HTTP Path | string | Required. Example: `sql/protocolvx/o/1234567489/0000-1111111-abcd90`. See [documentation](https://docs.databricks.com/integrations/bi/jdbc-odbc-bi.html#get-server-hostname-port-http-path-and-jdbc-url). | +| | Port | string | Optional. Default to "443". See [documentation](https://docs.databricks.com/integrations/bi/jdbc-odbc-bi.html#get-server-hostname-port-http-path-and-jdbc-url). | +| | Personal Access Token | string | Required. Example: `dapi0123456789abcdefghij0123456789AB`. See [documentation](https://docs.databricks.com/sql/user/security/personal-access-tokens.html). | +| General | Databricks catalog | string | Optional. The name of the catalog. If not specified otherwise, the "hive_metastore" will be used. | +| | Database schema | string | Optional. The default schema tables are written. If not specified otherwise, the "default" will be used. | +| | Schema evolution | boolean | Optional. The connector enables automatic schema evolution in the destination tables. | +| | Purge Staging Data | boolean | The connector creates staging files and tables on S3 or Azure. By default, they will be purged when the data sync is complete. Set it to `false` for debugging purposes. | +| Data Source - S3 | Bucket Name | string | Name of the bucket to sync data into. | +| | Bucket Path | string | Subdirectory under the above bucket to sync the data into. | +| | Region | string | See [documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes. | +| | Access Key ID | string | AWS/Minio credential. | +| | Secret Access Key | string | AWS/Minio credential. | +| | S3 Filename pattern | string | The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: `{date}`, `{date:yyyy_MM}`, `{timestamp}`, `{timestamp:millis}`, `{timestamp:micros}`, `{part_number}`, `{sync_id}`, `{format_extension}`. Please, don't use empty space and not supportable placeholders, as they won't recognized. | +| Data Source - Azure | Account Name | string | Name of the account to sync data into. | +| | Container Name | string | Container under the above account to sync the data into. | +| | SAS token | string | Shared-access signature token for the above account. | +| | Endpoint domain name | string | Usually blob.core.windows.net. | + +⚠️ Please note that under "Full Refresh Sync" mode, data in the configured bucket and path will be +wiped out before each sync. We recommend you provision a dedicated S3 or Azure resource for this +sync to prevent unexpected data deletion from misconfiguration. ⚠️ ## Staging Files (Delta Format) ### S3 -Data streams are first written as staging delta-table ([Parquet](https://parquet.apache.org/) + [Transaction Log](https://databricks.com/blog/2019/08/21/diving-into-delta-lake-unpacking-the-transaction-log.html)) files on S3, and then loaded into Databricks delta-tables. All the staging files will be deleted after the sync is done. For debugging purposes, here is the full path for a staging file: +Data streams are first written as staging delta-table ([Parquet](https://parquet.apache.org/) + +[Transaction Log](https://databricks.com/blog/2019/08/21/diving-into-delta-lake-unpacking-the-transaction-log.html)) +files on S3, and then loaded into Databricks delta-tables. All the staging files will be deleted +after the sync is done. For debugging purposes, here is the full path for a staging file: ```text s3:///// @@ -238,11 +285,15 @@ s3://testing_bucket/data_output_path/98c450be-5b1c-422d-b8b5-6ca9903727d9/users/ ### Azure -Similarly, streams are first written to a staging location, but the Azure option uses CSV format. A staging table is created from the CSV files. +Similarly, streams are first written to a staging location, but the Azure option uses CSV format. A +staging table is created from the CSV files. ## Unmanaged Spark SQL Table -Currently, all streams are synced into unmanaged Spark SQL tables. See [documentation](https://docs.databricks.com/data/tables.html#managed-and-unmanaged-tables) for details. In summary, you have full control of the location of the data underlying an unmanaged table. In S3, the full path of each data stream is: +Currently, all streams are synced into unmanaged Spark SQL tables. See +[documentation](https://docs.databricks.com/data/tables.html#managed-and-unmanaged-tables) for +details. In summary, you have full control of the location of the data underlying an unmanaged +table. In S3, the full path of each data stream is: ```text s3:///// @@ -265,28 +316,36 @@ In Azure, the full path of each data stream is: abfss://@.dfs.core.windows.net// ``` -Please keep these data directories on S3/Azure. Otherwise, the corresponding tables will have no data in Databricks. +Please keep these data directories on S3/Azure. Otherwise, the corresponding tables will have no +data in Databricks. ## Output Schema Each table will have the following columns: -| Column | Type | Notes | -| :--------------------------------- | :-------: | :------------------------------------------------------------- | -| `_airbyte_ab_id` | string | UUID. | -| `_airbyte_emitted_at` | timestamp | Data emission timestamp. | -| Data fields from the source stream | various | All fields in the staging files will be expanded in the table. | +| Column | Type | Notes | +| :-------------------- | :-------: | :----------------------------------------------- | +| `_airbyte_ab_id` | string | UUID. | +| `_airbyte_emitted_at` | timestamp | Data emission timestamp. | +| `_airbyte_data` | JSON | The data from your source will be in this column | -Under the hood, an Airbyte data stream in Json schema is first converted to an Avro schema, then the Json object is converted to an Avro record, and finally the Avro record is outputted to the Parquet format. Because the data stream can come from any data source, the Json to Avro conversion process has arbitrary rules and limitations. Learn more about how source data is converted to Avro and the current limitations [here](https://docs.airbyte.com/understanding-airbyte/json-avro-conversion). +Under the hood, an Airbyte data stream in Json schema is first converted to an Avro schema, then the +Json object is converted to an Avro record, and finally the Avro record is outputted to the Parquet +format. Because the data stream can come from any data source, the Json to Avro conversion process +has arbitrary rules and limitations. Learn more about how source data is converted to Avro and the +current limitations [here](https://docs.airbyte.com/understanding-airbyte/json-avro-conversion). ## Related tutorial -Suppose you are interested in learning more about the Databricks connector or details on how the Delta Lake tables are created. You may want to consult the tutorial on [How to Load Data into Delta Lake on Databricks Lakehouse](https://airbyte.com/tutorials/load-data-into-delta-lake-on-databricks-lakehouse). +Suppose you are interested in learning more about the Databricks connector or details on how the +Delta Lake tables are created. You may want to consult the tutorial on +[How to Load Data into Delta Lake on Databricks Lakehouse](https://airbyte.com/tutorials/load-data-into-delta-lake-on-databricks-lakehouse). ## CHANGELOG | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------------------------------------------------------------------ | :----------------------------------------------------------------------------------------------------------------------- | +| 1.1.1 | 2024-01-03 | [#33924](https://github.com/airbytehq/airbyte/pull/33924) | (incompatible with CDK, do not use) Add new ap-southeast-3 AWS region | | 1.1.0 | 2023-06-02 | [\#26942](https://github.com/airbytehq/airbyte/pull/26942) | Support schema evolution | | 1.0.2 | 2023-04-20 | [\#25366](https://github.com/airbytehq/airbyte/pull/25366) | Fix default catalog to be `hive_metastore` | | 1.0.1 | 2023-03-30 | [\#24657](https://github.com/airbytehq/airbyte/pull/24657) | Fix support for external tables on S3 | diff --git a/docs/integrations/destinations/duckdb-migrations.md b/docs/integrations/destinations/duckdb-migrations.md new file mode 100644 index 000000000000..53ae60158669 --- /dev/null +++ b/docs/integrations/destinations/duckdb-migrations.md @@ -0,0 +1,7 @@ +# DuckDB Migration Guide + +## Upgrading to 0.3.0 + +This version updates the DuckDB libraries from `v0.8.1` to `v0.9.1`. Note that DuckDB `0.9.x` is not backwards compatible with prior versions of DuckDB. Please see the [DuckDB 0.9.0 release notes](https://github.com/duckdb/duckdb/releases/tag/v0.9.0) for more information and for upgrade instructions. + +MotherDuck users will need to log into the MotherDuck UI at https://app.motherduck.com/ and click "Start Upgrade". The upgrade prompt will automatically appear the next time the user logs in. If the prompt does not appear, then your database has been upgraded automatically, and in this case you are ready to to use the latest version of the connector. diff --git a/docs/integrations/destinations/duckdb.md b/docs/integrations/destinations/duckdb.md index 352975308ec4..47691837c989 100644 --- a/docs/integrations/destinations/duckdb.md +++ b/docs/integrations/destinations/duckdb.md @@ -1,29 +1,53 @@ # DuckDB -:::danger + -This destination is meant to be used on a local workstation and won't work on Kubernetes +:::caution + +Local file-based DBs will not work in Airbyte Cloud or Kubernetes. Please use MotherDuck when running in Airbyte Cloud. ::: + + ## Overview -[DuckDB](https://duckdb.org/) is an in-process SQL OLAP database management system and this destination is meant to use locally if you have multiple smaller sources such as GitHub repos, some social media and local CSVs or files you want to run analytics workloads on. +[DuckDB](https://duckdb.org/) is an in-process SQL OLAP database management system and this destination is meant to use locally if you have multiple smaller sources such as GitHub repos, some social media and local CSVs or files you want to run analytics workloads on. This destination writes data to the [MotherDuck](https://motherduck.com) service, or to a file on the _local_ filesystem on the host running Airbyte. + +For file-based DBs, data is written to `/tmp/airbyte_local` by default. To change this location, modify the `LOCAL_ROOT` environment variable for Airbyte. + +## Use with MotherDuck + +This DuckDB destination is compatible with [MotherDuck](https://motherduck.com). -This destination writes data to a file on the _local_ filesystem on the host running Airbyte. By default, data is written to `/tmp/airbyte_local`. To change this location, modify the `LOCAL_ROOT` environment variable for Airbyte. +### Specifying a MotherDuck Database + +To specify a MotherDuck-hosted database as your destination, simply provide your database uri with the normal `md:` database prefix in the `destination_path` configuration option. + +:::caution + +We do not recommend providing your API token in the `md:` connection string, as this may cause your token to be printed to execution logs. Please use the `MotherDuck API Key` setting instead. + +::: + +### Authenticating to MotherDuck + +For authentication, you can can provide your [MotherDuck Service Credential](https://motherduck.com/docs/authenticating-to-motherduck/#syntax) as the `motherduck_api_key` configuration option. ### Sync Overview #### Output schema -If you set [Normalization](https://docs.airbyte.com/understanding-airbyte/basic-normalization/), source data will be normalized to a tabular form. Let's say you have a source such as GitHub with nested JSONs; the Normalization ensures you end up with tables and columns. Suppose you have a many-to-many relationship between the users and commits. Normalization will create separate tables for it. The end state is the [third normal form](https://en.wikipedia.org/wiki/Third_normal_form) (3NF). - Each table will contain 3 columns: - `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. - `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. - `_airbyte_data`: a json blob representing with the event data. +### Normalization + +If you set [Normalization](https://docs.airbyte.com/understanding-airbyte/basic-normalization/), source data will be normalized to a tabular form. Let's say you have a source such as GitHub with nested JSONs; the Normalization ensures you end up with tables and columns. Suppose you have a many-to-many relationship between the users and commits. Normalization will create separate tables for it. The end state is the [third normal form](https://en.wikipedia.org/wiki/Third_normal_form) (3NF). + #### Features | Feature | Supported | | @@ -37,7 +61,9 @@ Each table will contain 3 columns: This integration will be constrained by the speed at which your filesystem accepts writes. -## Getting Started + + +## Getting Started with Local Database Files The `destination_path` will always start with `/local` whether it is specified by the user or not. Any directory nesting within local will be mapped onto the local mount. @@ -72,10 +98,16 @@ You can also copy the output file to your host machine, the following command wi docker cp airbyte-server:/tmp/airbyte_local/{destination_path} . ``` -Note: If you are running Airbyte on Windows with Docker backed by WSL2, you have to use similar step as above or refer to this [link](../../operator-guides/locating-files-local-destination.md) for an alternative approach. +Note: If you are running Airbyte on Windows with Docker backed by WSL2, you have to use similar step as above or refer to this [link](/integrations/locating-files-local-destination.md) for an alternative approach. + + ## Changelog | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :--------------------- | +| 0.3.1 | 2023-11-18 | [#32635](https://github.com/airbytehq/airbyte/pull/32635) | Upgrade DuckDB version to [`v0.9.2`](https://github.com/duckdb/duckdb/releases/tag/v0.9.2). | +| 0.3.0 | 2022-10-23 | [#31744](https://github.com/airbytehq/airbyte/pull/31744) | Upgrade DuckDB version to [`v0.9.1`](https://github.com/duckdb/duckdb/releases/tag/v0.9.1). **Required update for all MotherDuck users.** Note, this is a **BREAKING CHANGE** for users who may have other connections using versions of DuckDB prior to 0.9.x. See the [0.9.0 release notes](https://github.com/duckdb/duckdb/releases/tag/v0.9.0) for more information and for upgrade instructions. | +| 0.2.1 | 2022-10-20 | [#30600](https://github.com/airbytehq/airbyte/pull/30600) | Fix: schema name mapping | +| 0.2.0 | 2022-10-19 | [#29428](https://github.com/airbytehq/airbyte/pull/29428) | Add support for MotherDuck. Upgrade DuckDB version to `v0.8``. | | 0.1.0 | 2022-10-14 | [17494](https://github.com/airbytehq/airbyte/pull/17494) | New DuckDB destination | diff --git a/docs/integrations/destinations/dynamodb.md b/docs/integrations/destinations/dynamodb.md index 08000797ba53..5143a177f287 100644 --- a/docs/integrations/destinations/dynamodb.md +++ b/docs/integrations/destinations/dynamodb.md @@ -2,17 +2,22 @@ This destination writes data to AWS DynamoDB. -The Airbyte DynamoDB destination allows you to sync data to AWS DynamoDB. Each stream is written to its own table under the DynamoDB. +The Airbyte DynamoDB destination allows you to sync data to AWS DynamoDB. Each stream is written to +its own table under the DynamoDB. ## Prerequisites -- For Airbyte Open Source users using the [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer and upgrade your DynamoDB connector to version `0.1.5` or newer +- For Airbyte Open Source users using the + [Postgres](https://docs.airbyte.com/integrations/sources/postgres) source connector, + [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to + version `v0.40.0-alpha` or newer and upgrade your DynamoDB connector to version `0.1.5` or newer ## Sync overview ### Output schema -Each stream will be output into its own DynamoDB table. Each table will a collections of `json` objects containing 4 fields: +Each stream will be output into its own DynamoDB table. Each table will a collections of `json` +objects containing 4 fields: - `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. - `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. @@ -30,13 +35,15 @@ Each stream will be output into its own DynamoDB table. Each table will a collec ### Performance considerations -This connector by default uses 10 capacity units for both Read and Write in DynamoDB tables. Please provision more capacity units in the DynamoDB console when there are performance constraints. +This connector by default uses 10 capacity units for both Read and Write in DynamoDB tables. Please +provision more capacity units in the DynamoDB console when there are performance constraints. ## Getting started ### Requirements -1. Allow connections from Airbyte server to your AWS DynamoDB tables \(if they exist in separate VPCs\). +1. Allow connections from Airbyte server to your AWS DynamoDB tables \(if they exist in separate + VPCs\). 2. The credentials for AWS DynamoDB \(for the COPY strategy\). ### Setup guide @@ -49,19 +56,27 @@ This connector by default uses 10 capacity units for both Read and Write in Dyna - **DynamoDB Region** - The region of the DynamoDB. - **Access Key Id** - - See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key. - - We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_dynamodb_specific-table.html) to the DynamoDB table. + - See + [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) + on how to generate an access key. + - We recommend creating an Airbyte-specific user. This user will require + [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_dynamodb_specific-table.html) + to the DynamoDB table. - **Secret Access Key** - Corresponding key to the above key id. - Make sure your DynamoDB tables are accessible from the machine running Airbyte. - This depends on your networking setup. - - You can check AWS DynamoDB documentation with a tutorial on how to properly configure your DynamoDB's access [here](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/access-control-overview.html). - - The easiest way to verify if Airbyte is able to connect to your DynamoDB tables is via the check connection tool in the UI. + - You can check AWS DynamoDB documentation with a tutorial on how to properly configure your + DynamoDB's access + [here](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/access-control-overview.html). + - The easiest way to verify if Airbyte is able to connect to your DynamoDB tables is via the check + connection tool in the UI. ## CHANGELOG | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :---------------------------------------------------------- | +| 0.1.8 | 2024-01-03 | [#33924](https://github.com/airbytehq/airbyte/pull/33924) | Add new ap-southeast-3 AWS region | | 0.1.7 | 2022-11-03 | [\#18672](https://github.com/airbytehq/airbyte/pull/18672) | Added strict-encrypt cloud runner | | 0.1.6 | 2022-11-01 | [\#18672](https://github.com/airbytehq/airbyte/pull/18672) | Enforce to use ssl connection | | 0.1.5 | 2022-08-05 | [\#15350](https://github.com/airbytehq/airbyte/pull/15350) | Added per-stream handling | diff --git a/docs/integrations/destinations/firestore.md b/docs/integrations/destinations/firestore.md index c82a9f12068e..94a6002a70c4 100644 --- a/docs/integrations/destinations/firestore.md +++ b/docs/integrations/destinations/firestore.md @@ -1,6 +1,35 @@ # Firestore -The Firestore destination for Airbyte +This destination writes data to Google Firestore. + +Google Firestore, officially known as Cloud Firestore, is a flexible, scalable database for mobile, web, and server development from Firebase and Google Cloud. It is commonly used for developing applications as a NoSQL database that provides real-time data syncing across user devices. + +## Getting started + +### Requiremnets + +- An existing GCP project +- A role with permissions to create a Service Account Key in GCP + +### Step 1: Create a Service Account +1. Log in to the Google Cloud Console. Select the project where your Firestore database is located. +2. Navigate to "IAM & Admin" and select "Service Accounts". Create a Service Account and assign appropriate roles. Ensure “Cloud Datastore User” or “Firebase Rules System” are enabled. +3. Navigate to the service account and generate the JSON key. Download and copy the contents to the configuration. + +## Sync overview + +### Output schema + +Each stream will be output into a BigQuery table. + +#### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :----------------------------- | :------------------- | :---- | +| Full Refresh Sync | ✅ | | +| Incremental - Append Sync | ✅ | | +| Incremental - Append + Deduped | ✅ | | +| Namespaces | ✅ | | ## Changelog diff --git a/docs/integrations/destinations/gcs.md b/docs/integrations/destinations/gcs.md index df8405a3448d..f272b77a9d6c 100644 --- a/docs/integrations/destinations/gcs.md +++ b/docs/integrations/destinations/gcs.md @@ -13,7 +13,7 @@ The Airbyte GCS destination allows you to sync data to cloud storage buckets. Ea | Feature | Support | Notes | | :----------------------------- | :-----: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Full Refresh Sync | ✅ | Warning: this mode deletes all previously synced data in the configured bucket path. | -| Incremental - Append Sync | ✅ | Warning: Airbyte provides at-least-once delivery. Depending on your source, you may see duplicated data. Learn more [here](/understanding-airbyte/connections/incremental-append#inclusive-cursors) | +| Incremental - Append Sync | ✅ | Warning: Airbyte provides at-least-once delivery. Depending on your source, you may see duplicated data. Learn more [here](/using-airbyte/core-concepts/sync-modes/incremental-append#inclusive-cursors) | | Incremental - Append + Deduped | ❌ | | | Namespaces | ❌ | Setting a specific bucket path is equivalent to having separate namespaces. | diff --git a/docs/integrations/destinations/google-sheets.md b/docs/integrations/destinations/google-sheets.md index 14c148a957e7..1bf21c51b225 100644 --- a/docs/integrations/destinations/google-sheets.md +++ b/docs/integrations/destinations/google-sheets.md @@ -1,7 +1,14 @@ # Google Sheets -The Google Sheets Destination is configured to push data to a single Google Sheets spreadsheet with multiple Worksheets as streams. To replicate data to multiple spreadsheets, you can create multiple instances of the Google Sheets Destination in your Airbyte instance. -This page guides you through the process of setting up the Google Sheets destination connector. +The Google Sheets Destination is configured to push data to a single Google Sheets spreadsheet with multiple Worksheets as streams. To replicate data to multiple spreadsheets, you can create multiple instances of the Google Sheets Destination in your Airbyte instance. + +:::warning + +Google Sheets imposes rate limits and hard limits on the amount of data it can receive, which results in sync failure. Only use Google Sheets as a destination for small, non-production use cases, as it is not designed for handling large-scale data operations. + +Read more about the [limitations](#limitations) of using Google Sheets below. + +::: ## Prerequisites @@ -12,41 +19,80 @@ This page guides you through the process of setting up the Google Sheets destina ### Google Account -#### If you don't have a Google Account - -Visit the [Google Support](https://support.google.com/accounts/answer/27441?hl=en) and create your Google Account. +To create a Google account, visit [Google](https://support.google.com/accounts/answer/27441?hl=en) and create a Google Account. ### Google Sheets (Google Spreadsheets) -1. Once you acquire your Google Account, simply open the [Google Support](https://support.google.com/docs/answer/6000292?hl=en&co=GENIE.Platform%3DDesktop) to create the fresh empty Google to be used as a destination for your data replication, or if already have one - follow the next step. -2. You will need the link of the Spreadsheet you'd like to sync. To get it, click Share button in the top right corner of Google Sheets interface, and then click Copy Link in the dialog that pops up. - These two steps are highlighted in the screenshot below: - -![](../../.gitbook/assets/google_spreadsheet_url.png) +1. Once you are logged into your Google account, create a new Google Sheet. [Follow this guide](https://support.google.com/docs/answer/6000292?hl=en&co=GENIE.Platform%3DDesktop) to create a new sheet. You may use an existing Google Sheet. +2. You will need the link of the Google Sheet you'd like to sync. To get it, click "Share" in the top right corner of the Google Sheets interface, and then click Copy Link in the dialog that pops up. ## Step 2: Set up the Google Sheets destination connector in Airbyte + **For Airbyte Cloud:** -1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. -2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new destination**. -3. On the source setup page, select **Google Sheets** from the Source type dropdown and enter a name for this connector. -4. Select `Sign in with Google`. -5. Log in and Authorize to the Google account and click `Set up source`. +1. Select **Google Sheets** from the Source type dropdown and enter a name for this connector. +2. Select `Sign in with Google`. +3. Log in and Authorize to the Google account and click `Set up source`. +4. Copy the Google Sheet link to **Spreadsheet Link** + + **For Airbyte Open Source:** -At this moment the `Google Sheets Destination` works only with Airbyte Cloud. +Authentication to Google Sheets is only available using OAuth for authentication. + +1. Create a new [Google Cloud project](https://console.cloud.google.com/projectcreate). +2. Enable the [Google Sheets API](https://console.cloud.google.com/apis/library/sheets.googleapis.com). +3. Create a new [OAuth client ID](https://console.cloud.google.com/apis/credentials/oauthclient). Select `Web application` as the Application type, give it a `name` and add `https://developers.google.com/oauthplayground` as an Authorized redirect URI. +4. Add a `Client Secret` (Add secret), and take note of both the `Client Secret` and `Client ID`. +5. Go to [Google OAuth Playground](https://developers.google.com/oauthplayground/) +6. Click the cog in the top-right corner, select `Use your own OAuth credentials` and enter the `OAuth Client ID` and `OAuth Client secret` from the previous step. +7. In the left sidebar, find and select `Google Sheets API v4`, then choose the `https://www.googleapis.com/auth/spreadsheets` scope. Click `Authorize APIs`. +8. In **step 2**, click `Exchange authorization code for tokens`. Take note of the `Refresh token`. +9. Set up a new destination in Airbyte, select `Google Sheets` and enter the `Client ID`, `Client Secret`, `Refresh Token` and `Spreadsheet Link` from the previous steps. + ### Output schema -Each worksheet in the selected spreadsheet will be the output as a separate source-connector stream. The data is coerced to string before the output to the spreadsheet. The nested data inside of the source connector data is normalized to the `first-level-nesting` and represented as string, this produces nested lists and objects to be a string rather than normal lists and objects, the further data processing is required if you need to analyze the data. +Each worksheet in the selected spreadsheet will be the output as a separate source-connector stream. + +The output columns are re-ordered in alphabetical order. The output columns should **not** be reordered manually after the sync, as this could cause future syncs to fail. + +All data is coerced to a `string` format in Google Sheets. +Any nested lists and objects will be formatted as a string rather than normal lists and objects. Further data processing is required if you require the data for downstream analysis. + +Airbyte only supports replicating `Grid Sheets`, which means only text is replicated. Objects like charts or images cannot be synced. See the [Google Sheets API docs](https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#SheetType) for more info on all available sheet types. + +### Rate Limiting & Performance Considerations + +The [Google API rate limit](https://developers.google.com/sheets/api/limits) is 60 requests per 60 seconds per user and 300 requests per 60 seconds per project, which will result in slow sync speeds. Airbyte batches requests to the API in order to efficiently pull data and respects these rate limits. + +### Limitations + +Google Sheets imposes hard limits on the amount of data that can be synced. If you attempt to sync more data than is allowed, the sync will fail. + +**Maximum of 10 Million Cells** + +A Google Sheets document can contain a maximum of 10 million cells. These can be in a single worksheet or in multiple sheets. +If you already have reached the 10 million limit, it will not allow you to add more columns (and vice versa, i.e., if the 10 million cells limit is reached with a certain number of rows, it will not allow more rows). + +**Maximum of 50,000 characters per cell** + +There can be at most 50,000 characters per cell. Do not use Google Sheets if you have fields with long text in your source. -Airbyte only supports replicating `Grid Sheets`, which means the text raw data only could be replicated to the target spreadsheet. See the [Google Sheets API docs](https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#SheetType) for more info on all available sheet types. +**Maximum of 18,278 Columns** + +There can be at most 18,278 columns in Google Sheets in a worksheet. + +**Maximum of 200 Worksheets in a Spreadsheet** + +You cannot create more than 200 worksheets within single spreadsheet. + +Syncs will fail if any of these limits are reached. #### Note: -- The output columns are ordered alphabetically. The output columns should not be reordered manually after the sync, this could cause the data corruption for all next syncs. - The underlying process of record normalization is applied to avoid data corruption during the write process. This handles two scenarios: 1. UnderSetting - when record has less keys (columns) than catalog declares @@ -100,36 +146,11 @@ EXAMPLE: | Incremental Append | Yes | | Incremental Append-Deduplicate | Yes | -### Rate Limiting & Performance Considerations - -At the time of writing, the [Google API rate limit](https://developers.google.com/sheets/api/limits) is 100 requests per 100 seconds per user and 500 requests per 100 seconds per project. Airbyte batches requests to the API in order to efficiently pull data and respects these rate limits. It is recommended that you use the same service user \(see the "Creating a service user" section below for more information on how to create one\) for no more than 3 instances of the Google Sheets Destination to ensure high transfer speeds. -Please be aware of the [Google Spreadsheet limitations](#limitations) before you configure your airbyte data replication using Destination Google Sheets - -### Google Sheets Limitations - -During the upload process and from the data storage perspective there are some limitations that should be considered beforehand as [determined by Google here](https://support.google.com/drive/answer/37603): - -- **Maximum of 10 Million Cells** - -A Google Sheets document can have a maximum of 10 million cells. These can be in a single worksheet or in multiple sheets. -In case you already have the 10 million limit reached in fewer columns, it will not allow you to add more columns (and vice versa, i.e., if 10 million cells limit is reached with a certain number of rows, it will not allow more rows). - -- **Maximum of 18,278 Columns** - -At max, you can have 18,278 columns in Google Sheets in a worksheet. - -- **Up to 200 Worksheets in a Spreadsheet** - -You cannot create more than 200 worksheets within single spreadsheet. - -#### Future improvements: - -- Handle multiple spreadsheets to split big amount of data into parts, once the main spreadsheet is full and cannot be extended more, due to [limitations](#limitations). - ## Changelog | Version | Date | Pull Request | Subject | |---------|------------|----------------------------------------------------------|------------------------------------------------| +| 0.2.3 | 2023-09-25 | [30748](https://github.com/airbytehq/airbyte/pull/30748) | Performance testing - include socat binary in docker image | | 0.2.2 | 2023-07-06 | [28035](https://github.com/airbytehq/airbyte/pull/28035) | Migrate from authSpecification to advancedAuth | | 0.2.1 | 2023-06-26 | [27782](https://github.com/airbytehq/airbyte/pull/27782) | Only allow HTTPS urls | | 0.2.0 | 2023-06-26 | [27780](https://github.com/airbytehq/airbyte/pull/27780) | License Update: Elv2 | diff --git a/docs/integrations/destinations/iceberg.md b/docs/integrations/destinations/iceberg.md index 6b48df61743d..512e7964b649 100644 --- a/docs/integrations/destinations/iceberg.md +++ b/docs/integrations/destinations/iceberg.md @@ -6,10 +6,10 @@ This page guides you through the process of setting up the Iceberg destination c ### Output schema -The incoming airbyte data is structured in keyspaces and tables and is partitioned and replicated across different nodes -in the cluster. This connector maps an incoming `stream` to an Iceberg `table` and a `namespace` to an -Iceberg `database`. Fields in the airbyte message become different columns in the Iceberg tables. Each table will -contain the following columns. +The incoming airbyte data is structured in keyspaces and tables and is partitioned and replicated +across different nodes in the cluster. This connector maps an incoming `stream` to an Iceberg +`table` and a `namespace` to an Iceberg `database`. Fields in the airbyte message become different +columns in the Iceberg tables. Each table will contain the following columns. - `_airbyte_ab_id`: A random generated uuid. - `_airbyte_emitted_at`: a timestamp representing when the event was received from the data source. @@ -28,14 +28,13 @@ This section should contain a table with the following format: ### Performance considerations -Every ten thousand pieces of incoming airbyte data in a stream ————we call it a batch, would produce one data file( -Parquet/Avro) in an Iceberg table. This batch size can be configurabled by `Data file flushing batch size` -property. -As the quantity of Iceberg data files grows, it causes an unnecessary amount of metadata and less efficient queries from -file open costs. -Iceberg provides data file compaction action to improve this case, you can read more about -compaction [HERE](https://iceberg.apache.org/docs/latest/maintenance/#compact-data-files). -This connector also provides auto compact action when stream closes, by `Auto compact data files` property. Any you can +Every ten thousand pieces of incoming airbyte data in a stream ————we call it a batch, would produce +one data file( Parquet/Avro) in an Iceberg table. This batch size can be configurabled by +`Data file flushing batch size` property. As the quantity of Iceberg data files grows, it causes an +unnecessary amount of metadata and less efficient queries from file open costs. Iceberg provides +data file compaction action to improve this case, you can read more about compaction +[HERE](https://iceberg.apache.org/docs/latest/maintenance/#compact-data-files). This connector also +provides auto compact action when stream closes, by `Auto compact data files` property. Any you can specify the target size of compacted Iceberg data file. ## Getting started @@ -43,24 +42,28 @@ specify the target size of compacted Iceberg data file. ### Requirements - **Iceberg catalog** : Iceberg uses `catalog` to manage tables. this connector already supports: - - [HiveCatalog](https://iceberg.apache.org/docs/latest/hive/#global-hive-catalog) connects to a **Hive metastore** - to keep track of Iceberg tables. - - [HadoopCatalog](https://iceberg.apache.org/docs/latest/java-api-quickstart/#using-a-hadoop-catalog) doesn’t need - to connect to a Hive MetaStore, but can only be used with **HDFS or similar file systems** that support atomic - rename. For `HadoopCatalog`, this connector use **Storage Config** (S3 or HDFS) to manage Iceberg tables. - - [JdbcCatalog](https://iceberg.apache.org/docs/latest/jdbc/) uses a table in a relational database to manage - Iceberg tables through JDBC. So far, this connector supports **PostgreSQL** only. - - [RESTCatalog](https://iceberg.apache.org/docs/latest/spark-configuration/#catalog-configuration) connects to a REST - server, which manages Iceberg tables. -- **Storage medium** means where Iceberg data files storages in. So far, this connector supports **S3/S3N/S3N** - object-storage. When using the RESTCatalog, it is possible to have storage be managed by the server. + - [HiveCatalog](https://iceberg.apache.org/docs/latest/hive/#global-hive-catalog) connects to a + **Hive metastore** to keep track of Iceberg tables. + - [HadoopCatalog](https://iceberg.apache.org/docs/latest/java-api-quickstart/#using-a-hadoop-catalog) + doesn’t need to connect to a Hive MetaStore, but can only be used with **HDFS or similar file + systems** that support atomic rename. For `HadoopCatalog`, this connector use **Storage Config** + (S3 or HDFS) to manage Iceberg tables. + - [JdbcCatalog](https://iceberg.apache.org/docs/latest/jdbc/) uses a table in a relational + database to manage Iceberg tables through JDBC. So far, this connector supports **PostgreSQL** + only. + - [RESTCatalog](https://iceberg.apache.org/docs/latest/spark-configuration/#catalog-configuration) + connects to a REST server, which manages Iceberg tables. +- **Storage medium** means where Iceberg data files storages in. So far, this connector supports + **S3/S3N/S3N** object-storage. When using the RESTCatalog, it is possible to have storage be + managed by the server. ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------| :------------------------------------------------------- | :------------- | -| 0.1.4 | 2023-07-20 | [28506](https://github.com/airbytehq/airbyte/pull/28506) | Support server-managed storage config | -| 0.1.3 | 2023-07-12 | [28158](https://github.com/airbytehq/airbyte/pull/28158) | Bump Iceberg library to 1.3.0 and add REST catalog support | -| 0.1.2 | 2023-07-14 | [28345](https://github.com/airbytehq/airbyte/pull/28345) | Trigger rebuild of image | -| 0.1.1 | 2023-02-27 | [23201](https://github.com/airbytehq/airbyte/pull/23301) | Bump Iceberg library to 1.1.0 | -| 0.1.0 | 2022-11-01 | [18836](https://github.com/airbytehq/airbyte/pull/18836) | Initial Commit | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :-------------------------------------------------------- | :--------------------------------------------------------- | +| 0.1.5 | 2024-01-03 | [#33924](https://github.com/airbytehq/airbyte/pull/33924) | Add new ap-southeast-3 AWS region | +| 0.1.4 | 2023-07-20 | [28506](https://github.com/airbytehq/airbyte/pull/28506) | Support server-managed storage config | +| 0.1.3 | 2023-07-12 | [28158](https://github.com/airbytehq/airbyte/pull/28158) | Bump Iceberg library to 1.3.0 and add REST catalog support | +| 0.1.2 | 2023-07-14 | [28345](https://github.com/airbytehq/airbyte/pull/28345) | Trigger rebuild of image | +| 0.1.1 | 2023-02-27 | [23201](https://github.com/airbytehq/airbyte/pull/23301) | Bump Iceberg library to 1.1.0 | +| 0.1.0 | 2022-11-01 | [18836](https://github.com/airbytehq/airbyte/pull/18836) | Initial Commit | diff --git a/docs/integrations/destinations/langchain-migrations.md b/docs/integrations/destinations/langchain-migrations.md new file mode 100644 index 000000000000..005845d0382c --- /dev/null +++ b/docs/integrations/destinations/langchain-migrations.md @@ -0,0 +1,9 @@ +# Vector Database (powered by LangChain) Migration Guide + +## Upgrading to 0.1.0 + +This version changes the way record ids are tracked internally. If you are using a stream in **append-dedup** mode, you need to reset the connection after doing the upgrade to avoid duplicates. + +Prior to this version, deduplication only considered the primary key per record, without disambiugating between streams. This could lead to data loss if records from two different streams had the same primary key. + +The problem is fixed by appending the namespace and stream name to the `_ab_record_id` field to disambiguate between records originating from different streams. If a connection using **append-dedup** mode is not reset after the upgrade, it will consider all records as new and will not deduplicate them, leading to stale vectors in the destination. \ No newline at end of file diff --git a/docs/integrations/destinations/langchain.md b/docs/integrations/destinations/langchain.md index 8f3c9a8625d5..4ac1fe151906 100644 --- a/docs/integrations/destinations/langchain.md +++ b/docs/integrations/destinations/langchain.md @@ -1,5 +1,17 @@ # Vector Database (powered by LangChain) +:::warning +The vector db destination destination has been split into separate destinations per vector database. This destination will not receive any further updates and is not subject to SLAs. The separate destinations support all features of this destination and are actively maintained. Please migrate to the respective destination as soon as possible. + +Please use the respective destination for the vector database you want to use to ensure you receive updates and support. + +To following databases are supported: +* [Pinecone](https://docs.airbyte.com/integrations/destinations/pinecone) +* [Weaviate](https://docs.airbyte.com/integrations/destinations/weaviate) +* [Milvus](https://docs.airbyte.com/integrations/destinations/milvus) +* [Chroma](https://docs.airbyte.com/integrations/destinations/chroma) +* [Qdrant](https://docs.airbyte.com/integrations/destinations/qdrant) +::: ## Overview @@ -18,7 +30,7 @@ When specifying text fields, you can access nested fields in the record by using The chunk length is measured in tokens produced by the `tiktoken` library. The maximum is 8191 tokens, which is the maximum length supported by the `text-embedding-ada-002` model. -The stream name gets added as a metadata field `_airbyte_stream` to each document. If available, the primary key of the record is used to identify the document to avoid duplications when updated versions of records are indexed. It is added as the `_natural_id` metadata field. +The stream name gets added as a metadata field `_airbyte_stream` to each document. If available, the primary key of the record is used to identify the document to avoid duplications when updated versions of records are indexed. It is added as the `_record_id` metadata field. ### Embedding @@ -59,6 +71,7 @@ For Pinecone pods of type starter, only up to 10,000 chunks can be indexed. For ::: + #### Chroma vector store The [Chroma vector store](https://trychroma.com) is running the Chroma embedding database as persistent client and stores the vectors in a local file. @@ -133,12 +146,15 @@ DocArrayHnswSearch is meant to be used on a local workstation and won't work on Please make sure that Docker Desktop has access to `/tmp` (and `/private` on a MacOS, as /tmp has a symlink that points to /private. It will not work otherwise). You allow it with "File sharing" in `Settings -> Resources -> File sharing -> add the one or two above folder` and hit the "Apply & restart" button. ::: - + ## CHANGELOG | Version | Date | Pull Request | Subject | |:--------| :--------- |:--------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.1.2 | 2023-11-13 | [#32455](https://github.com/airbytehq/airbyte/pull/32455) | Fix build | +| 0.1.1 | 2023-09-01 | [#30282](https://github.com/airbytehq/airbyte/pull/30282) | Use embedders from CDK | +| 0.1.0 | 2023-09-01 | [#30080](https://github.com/airbytehq/airbyte/pull/30080) | Fix bug with potential data loss on append+dedup syncing. 🚨 Streams using append+dedup mode need to be reset after upgrade. | | 0.0.8 | 2023-08-21 | [#29515](https://github.com/airbytehq/airbyte/pull/29515) | Clean up generated schema spec | | 0.0.7 | 2023-08-18 | [#29513](https://github.com/airbytehq/airbyte/pull/29513) | Fix for starter pods | | 0.0.6 | 2023-08-02 | [#28977](https://github.com/airbytehq/airbyte/pull/28977) | Validate pinecone index dimensions during check | diff --git a/docs/integrations/destinations/local-json.md b/docs/integrations/destinations/local-json.md index 11870a8d5177..45ddda3fb757 100644 --- a/docs/integrations/destinations/local-json.md +++ b/docs/integrations/destinations/local-json.md @@ -69,7 +69,7 @@ You can also copy the output file to your host machine, the following command wi docker cp airbyte-server:/tmp/airbyte_local/{destination_path}/{filename}.jsonl . ``` -Note: If you are running Airbyte on Windows with Docker backed by WSL2, you have to use similar step as above or refer to this [link](../../operator-guides/locating-files-local-destination.md) for an alternative approach. +Note: If you are running Airbyte on Windows with Docker backed by WSL2, you have to use similar step as above or refer to this [link](/integrations/locating-files-local-destination.md) for an alternative approach. ## Changelog diff --git a/docs/integrations/destinations/meilisearch.md b/docs/integrations/destinations/meilisearch.md index d7f40201b775..f788f9613057 100644 --- a/docs/integrations/destinations/meilisearch.md +++ b/docs/integrations/destinations/meilisearch.md @@ -33,7 +33,9 @@ The setup only requires two fields. First is the `host` which is the address at | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :----------------------------------------------------- | +| 1.0.1 | 2023-12-19 | [27692](https://github.com/airbytehq/airbyte/pull/27692) | Fix incomplete data indexing | | 1.0.0 | 2022-10-26 | [18036](https://github.com/airbytehq/airbyte/pull/18036) | Migrate MeiliSearch to Python CDK | | 0.2.13 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.2.12 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.2.11 | 2021-12-28 | [9156](https://github.com/airbytehq/airbyte/pull/9156) | Update connector fields title/description | + diff --git a/docs/integrations/destinations/milvus.md b/docs/integrations/destinations/milvus.md new file mode 100644 index 000000000000..d2cff5caf09b --- /dev/null +++ b/docs/integrations/destinations/milvus.md @@ -0,0 +1,123 @@ +# Milvus + +## Overview + +This page guides you through the process of setting up the [Milvus](https://milvus.io/) destination connector. + +There are three parts to this: +* Processing - split up individual records in chunks so they will fit the context window and decide which fields to use as context and which are supplementary metadata. +* Embedding - convert the text into a vector representation using a pre-trained model (Currently, OpenAI's `text-embedding-ada-002` and Cohere's `embed-english-light-v2.0` are supported.) +* Indexing - store the vectors in a vector database for similarity search + +## Prerequisites + +To use the Milvus destination, you'll need: + +- An account with API access for OpenAI or Cohere (depending on which embedding method you want to use) +- Either a running self-managed Milvus instance or a [Zilliz](https://zilliz.com/) account + +You'll need the following information to configure the destination: + +- **Embedding service API Key** - The API key for your OpenAI or Cohere account +- **Milvus Endpoint URL** - The URL of your Milvus instance +- Either **Milvus API token** or **Milvus Instance Username and Password** +- **Milvus Collection name** - The name of the collection to load data into + +## Features + +| Feature | Supported? | Notes | +| :----------------------------- | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | Yes | | +| Incremental - Append + Deduped | Yes | | +| Partitions | No | | +| Record-defined ID | No | Auto-id needs to be enabled | + +## Configuration + +### Processing + +Each record will be split into text fields and meta fields as configured in the "Processing" section. All text fields are concatenated into a single string and then split into chunks of configured length. If specified, the metadata fields are stored as-is along with the embedded text chunks. Options around configuring the chunking process use the [Langchain Python library](https://python.langchain.com/docs/get_started/introduction). + +When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. + +The chunk length is measured in tokens produced by the `tiktoken` library. The maximum is 8191 tokens, which is the maximum length supported by the `text-embedding-ada-002` model. + +The stream name gets added as a metadata field `_ab_stream` to each document. If available, the primary key of the record is used to identify the document to avoid duplications when updated versions of records are indexed. It is added as the `_ab_record_id` metadata field. + +### Embedding + +The connector can use one of the following embedding methods: + +1. OpenAI - using [OpenAI API](https://beta.openai.com/docs/api-reference/text-embedding) , the connector will produce embeddings using the `text-embedding-ada-002` model with **1536 dimensions**. This integration will be constrained by the [speed of the OpenAI embedding API](https://platform.openai.com/docs/guides/rate-limits/overview). + +2. Cohere - using the [Cohere API](https://docs.cohere.com/reference/embed), the connector will produce embeddings using the `embed-english-light-v2.0` model with **1024 dimensions**. + +For testing purposes, it's also possible to use the [Fake embeddings](https://python.langchain.com/docs/modules/data_connection/text_embedding/integrations/fake) integration. It will generate random embeddings and is suitable to test a data pipeline without incurring embedding costs. + +### Indexing + +If the specified collection doesn't exist, the connector will create it for you with a primary key field `pk` and the configured vector field matching the embedding configuration. Dynamic fields will be enabled. The vector field will have an L2 IVF_FLAT index with an `nlist` parameter of 1024. + +If you want to change any of these settings, create a new collection in your Milvus instance yourself. Make sure that +* The primary key field is set to [auto_id](https://milvus.io/docs/create_collection.md) +* There is a vector field with the correct dimensionality (1536 for OpenAI, 1024 for Cohere) and [a configured index](https://milvus.io/docs/build_index.md) + +If the record contains a field with the same name as the primary key, it will be prefixed with an underscore so Milvus can control the primary key internally. + +### Setting up a collection + +When using the Zilliz cloud, this can be done using the UI - in this case only the collection name and the vector dimensionality needs to be configured, the vector field with index will be automatically created under the name `vector`. Using the REST API, the following command will create the index: +``` +POST /v1/vector/collections/create +{ + "collectionName": "my-collection", + "dimension": 1536, + "metricType": "L2", + "vectorField": "vector", + “primaryField”: “pk” +} +``` + +When using a self-hosted Milvus cluster, the collection needs to be created using the Milvus CLI or Python client. The following commands will create a collection set up for loading data via Airbyte: +```python +from pymilvus import CollectionSchema, FieldSchema, DataType, connections, Collection + +connections.connect() # connect to locally running Milvus instance without authentication + +pk = FieldSchema(name="pk",dtype=DataType.INT64, is_primary=True, auto_id=True) +vector = FieldSchema(name="vector",dtype=DataType.FLOAT_VECTOR,dim=1536) +schema = CollectionSchema(fields=[pk, vector], enable_dynamic_field=True) +collection = Collection(name="test_collection", schema=schema) +collection.create_index(field_name="vector", index_params={ "metric_type":"L2", "index_type":"IVF_FLAT", "params":{"nlist":1024} }) +``` + +### Langchain integration + +To initialize a langchain vector store based on the indexed data, use the following code: +```python +embeddings = OpenAIEmbeddings(openai_api_key="my-key") +vector_store = Milvus(embeddings=embeddings, collection_name="my-collection", connection_args={"uri": "my-zilliz-endpoint", "token": "my-api-key"}) +vector_store.fields.append("text") +# call vs.fields.append() for all fields you need from the metadata + +vector_store.similarity_search("test") +``` + + +## CHANGELOG + +| Version | Date | Pull Request | Subject | +|:--------| :--------- |:--------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.0.12 | 2023-12-11 | [33303](https://github.com/airbytehq/airbyte/pull/33303) | Fix bug with embedding special tokens | +| 0.0.11 | 2023-12-01 | [32697](https://github.com/airbytehq/airbyte/pull/32697) | Allow omitting raw text | +| 0.0.10 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Support deleting records for CDC sources | +| 0.0.9 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | +| 0.0.8 | 2023-11-08 | [#31563](https://github.com/airbytehq/airbyte/pull/32262) | Auto-create collection if it doesn't exist | +| 0.0.7 | 2023-10-23 | [#31563](https://github.com/airbytehq/airbyte/pull/31563) | Add field mapping option | +| 0.0.6 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 0.0.5 | 2023-10-15 | [#31329](https://github.com/airbytehq/airbyte/pull/31329) | Add OpenAI-compatible embedder option | +| 0.0.4 | 2023-10-04 | [#31075](https://github.com/airbytehq/airbyte/pull/31075) | Fix OpenAI embedder batch size | +| 0.0.3 | 2023-09-29 | [#30820](https://github.com/airbytehq/airbyte/pull/30820) | Update CDK | +| 0.0.2 | 2023-08-25 | [#30689](https://github.com/airbytehq/airbyte/pull/30689) | Update CDK to support azure OpenAI embeddings and text splitting options, make sure primary key field is not accidentally set, promote to certified | +| 0.0.1 | 2023-08-12 | [#29442](https://github.com/airbytehq/airbyte/pull/29442) | Milvus connector with some embedders | \ No newline at end of file diff --git a/docs/integrations/destinations/mongodb.md b/docs/integrations/destinations/mongodb.md index 51bd94cb8c46..6df8e95f929c 100644 --- a/docs/integrations/destinations/mongodb.md +++ b/docs/integrations/destinations/mongodb.md @@ -25,7 +25,7 @@ Each stream will be output into its own collection in MongoDB. Each collection w Airbyte Cloud only supports connecting to your MongoDB instance with TLS encryption. Other than that, you can proceed with the open-source instructions below. -## Getting Started \(Airbyte Open-Source\) +## Getting Started \(Airbyte Open Source\) #### Requirements diff --git a/docs/integrations/destinations/mssql.md b/docs/integrations/destinations/mssql.md index c48261be1a0b..2a4bfd50bf5a 100644 --- a/docs/integrations/destinations/mssql.md +++ b/docs/integrations/destinations/mssql.md @@ -33,7 +33,7 @@ Airbyte Cloud only supports connecting to your MSSQL instance with TLS encryptio | Incremental - Append + Deduped | Yes | | | Namespaces | Yes | | -## Getting Started \(Airbyte Open-Source\) +## Getting Started \(Airbyte Open Source\) ### Requirements diff --git a/docs/integrations/destinations/mysql.md b/docs/integrations/destinations/mysql.md index 58826d631142..2a19352e8ad7 100644 --- a/docs/integrations/destinations/mysql.md +++ b/docs/integrations/destinations/mysql.md @@ -11,7 +11,7 @@ There are two flavors of connectors for this destination: | :----------------------------- | :------------------- | :---- | | Full Refresh Sync | Yes | | | Incremental - Append Sync | Yes | | -| Incremental - Append + Deduped | Yes | | +| Incremental - Append + Deduped | No | | | Namespaces | Yes | | | SSH Tunnel Connection | Yes | | @@ -27,7 +27,7 @@ Each stream will be output into its own table in MySQL. Each table will contain Airbyte Cloud only supports connecting to your MySQL instance with TLS encryption. Other than that, you can proceed with the open-source instructions below. -## Getting Started \(Airbyte Open-Source\) +## Getting Started \(Airbyte Open Source\) ### Requirements @@ -116,6 +116,7 @@ Using this feature requires additional configuration, when creating the destinat | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------------------- | +| 0.3.0 | 2023-12-18 | [33468](https://github.com/airbytehq/airbyte/pull/33468) | Upgrade to latest Java CDK | | 0.2.0 | 2023-06-27 | [27781](https://github.com/airbytehq/airbyte/pull/27781) | License Update: Elv2 | | 0.1.21 | 2022-09-14 | [15668](https://github.com/airbytehq/airbyte/pull/15668) | Wrap logs in AirbyteLogMessage | | 0.1.20 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | diff --git a/docs/integrations/destinations/oracle.md b/docs/integrations/destinations/oracle.md index 2b26a69cbf6c..d2e9867eb04a 100644 --- a/docs/integrations/destinations/oracle.md +++ b/docs/integrations/destinations/oracle.md @@ -26,7 +26,7 @@ Enabling normalization will also create normalized, strongly typed tables. The Oracle connector is currently in Alpha on Airbyte Cloud. Only TLS encrypted connections to your DB can be made from Airbyte Cloud. Other than that, follow the open-source instructions below. -## Getting Started \(Airbyte Open-Source\) +## Getting Started \(Airbyte Open Source\) #### Requirements diff --git a/docs/integrations/destinations/pinecone.md b/docs/integrations/destinations/pinecone.md new file mode 100644 index 000000000000..51dae798ab8e --- /dev/null +++ b/docs/integrations/destinations/pinecone.md @@ -0,0 +1,98 @@ +# Pinecone + +## Overview + +This page guides you through the process of setting up the [Pinecone](https://pinecone.io/) destination connector. + +There are three parts to this: +* Processing - split up individual records in chunks so they will fit the context window and decide which fields to use as context and which are supplementary metadata. +* Embedding - convert the text into a vector representation using a pre-trained model (Currently, OpenAI's `text-embedding-ada-002` and Cohere's `embed-english-light-v2.0` are supported.) +* Indexing - store the vectors in a vector database for similarity search + +## Prerequisites + +To use the Pinecone destination, you'll need: + +- An account with API access for OpenAI or Cohere (depending on which embedding method you want to use) +- A Pinecone project with a pre-created index with the correct dimensionality based on your embedding method + +You'll need the following information to configure the destination: + +- **Embedding service API Key** - The API key for your OpenAI or Cohere account +- **Pinecone API Key** - The API key for your Pinecone account +- **Pinecone Environment** - The name of the Pinecone environment to use +- **Pinecone Index name** - The name of the Pinecone index to load data into + +## Features + +| Feature | Supported? | Notes | +| :----------------------------- | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | Yes | | +| Incremental - Append + Deduped | Yes | Deleting records via CDC is not supported (see issue [#29827](https://github.com/airbytehq/airbyte/issues/29827)) | +| Namespaces | Yes | | + +## Data type mapping + +All fields specified as metadata fields will be stored in the metadata object of the document and can be used for filtering. The following data types are allowed for metadata fields: +* String +* Number (integer or floating point, gets converted to a 64 bit floating point) +* Booleans (true, false) +* List of String + +All other fields are ignored. + +## Configuration + +### Processing + +Each record will be split into text fields and meta fields as configured in the "Processing" section. All text fields are concatenated into a single string and then split into chunks of configured length. If specified, the metadata fields are stored as-is along with the embedded text chunks. Please note that meta data fields can only be used for filtering and not for retrieval and have to be of type string, number, boolean (all other values are ignored). Please note that there's a 40kb limit on the _total_ size of the metadata saved for each entry. Options around configuring the chunking process use the [Langchain Python library](https://python.langchain.com/docs/get_started/introduction). + +When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. + +The chunk length is measured in tokens produced by the `tiktoken` library. The maximum is 8191 tokens, which is the maximum length supported by the `text-embedding-ada-002` model. + +The stream name gets added as a metadata field `_ab_stream` to each document. If available, the primary key of the record is used to identify the document to avoid duplications when updated versions of records are indexed. It is added as the `_ab_record_id` metadata field. + +### Embedding + +The connector can use one of the following embedding methods: + +1. OpenAI - using [OpenAI API](https://beta.openai.com/docs/api-reference/text-embedding) , the connector will produce embeddings using the `text-embedding-ada-002` model with **1536 dimensions**. This integration will be constrained by the [speed of the OpenAI embedding API](https://platform.openai.com/docs/guides/rate-limits/overview). + +2. Cohere - using the [Cohere API](https://docs.cohere.com/reference/embed), the connector will produce embeddings using the `embed-english-light-v2.0` model with **1024 dimensions**. + +For testing purposes, it's also possible to use the [Fake embeddings](https://python.langchain.com/docs/modules/data_connection/text_embedding/integrations/fake) integration. It will generate random embeddings and is suitable to test a data pipeline without incurring embedding costs. + +### Indexing + +To get started, use the [Pinecone web UI or API](https://docs.pinecone.io/docs/quickstart) to create a project and an index before running the destination. All streams will be indexed into the same index, the `_ab_stream` metadata field is used to distinguish between streams. Overall, the size of the metadata fields is limited to 30KB per document. + +OpenAI and Fake embeddings produce vectors with 1536 dimensions, and the Cohere embeddings produce vectors with 1024 dimensions. Make sure to configure the index accordingly. + +## CHANGELOG + +| Version | Date | Pull Request | Subject | +|:--------| :--------- |:--------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.0.22 | 2023-12-11 | [33303](https://github.com/airbytehq/airbyte/pull/33303) | Fix bug with embedding special tokens | +| 0.0.21 | 2023-12-01 | [32697](https://github.com/airbytehq/airbyte/pull/32697) | Allow omitting raw text | +| 0.0.20 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | +| 0.0.19 | 2023-10-20 | [#31329](https://github.com/airbytehq/airbyte/pull/31373) | Improve error messages | +| 0.0.18 | 2023-10-20 | [#31329](https://github.com/airbytehq/airbyte/pull/31373) | Add support for namespaces and fix index cleaning when namespace is defined | +| 0.0.17 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 0.0.16 | 2023-10-15 | [#31329](https://github.com/airbytehq/airbyte/pull/31329) | Add OpenAI-compatible embedder option | +| 0.0.15 | 2023-10-04 | [#31075](https://github.com/airbytehq/airbyte/pull/31075) | Fix OpenAI embedder batch size | +| 0.0.14 | 2023-09-29 | [#30820](https://github.com/airbytehq/airbyte/pull/30820) | Update CDK | +| 0.0.13 | 2023-09-26 | [#30649](https://github.com/airbytehq/airbyte/pull/30649) | Allow more text splitting options | +| 0.0.12 | 2023-09-25 | [#30649](https://github.com/airbytehq/airbyte/pull/30649) | Fix bug with stale documents left on starter pods | +| 0.0.11 | 2023-09-22 | [#30649](https://github.com/airbytehq/airbyte/pull/30649) | Set visible certified flag | +| 0.0.10 | 2023-09-20 | [#30514](https://github.com/airbytehq/airbyte/pull/30514) | Fix bug with failing embedding step on large records | +| 0.0.9 | 2023-09-18 | [#30510](https://github.com/airbytehq/airbyte/pull/30510) | Fix bug with overwrite mode on starter pods | +| 0.0.8 | 2023-09-14 | [#30296](https://github.com/airbytehq/airbyte/pull/30296) | Add Azure embedder | +| 0.0.7 | 2023-09-13 | [#30382](https://github.com/airbytehq/airbyte/pull/30382) | Promote to certified/beta | +| 0.0.6 | 2023-09-09 | [#30193](https://github.com/airbytehq/airbyte/pull/30193) | Improve documentation | +| 0.0.5 | 2023-09-07 | [#30133](https://github.com/airbytehq/airbyte/pull/30133) | Refactor internal structure of connector | +| 0.0.4 | 2023-09-05 | [#30086](https://github.com/airbytehq/airbyte/pull/30079) | Switch to GRPC client for improved performance. | +| 0.0.3 | 2023-09-01 | [#30079](https://github.com/airbytehq/airbyte/pull/30079) | Fix bug with potential data loss on append+dedup syncing. 🚨 Streams using append+dedup mode need to be reset after upgrade. | +| 0.0.2 | 2023-08-31 | [#29442](https://github.com/airbytehq/airbyte/pull/29946) | Improve test coverage | +| 0.0.1 | 2023-08-29 | [#29539](https://github.com/airbytehq/airbyte/pull/29539) | Pinecone connector with some embedders | \ No newline at end of file diff --git a/docs/integrations/destinations/postgres.md b/docs/integrations/destinations/postgres.md index a05718c145e2..b8fffc91ab93 100644 --- a/docs/integrations/destinations/postgres.md +++ b/docs/integrations/destinations/postgres.md @@ -169,7 +169,13 @@ Now that you have set up the Postgres destination connector, check out the follo ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :--------------------------------------------------------- | :-------------------------------------------------------------------------------------------------- | +|:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------| +| 0.5.5 | 2024-01-18 | [34236](https://github.com/airbytehq/airbyte/pull/34236) | Upgrade CDK to 0.13.1; Add indexes in raw table for query optimization | +| 0.5.4 | 2024-01-11 | [34177](https://github.com/airbytehq/airbyte/pull/34177) | Add code for DV2 beta (no user-visible changes) | +| 0.5.3 | 2024-01-10 | [34135](https://github.com/airbytehq/airbyte/pull/34135) | Use published CDK missed in previous release | +| 0.5.2 | 2024-01-08 | [33875](https://github.com/airbytehq/airbyte/pull/33875) | Update CDK to get Tunnel heartbeats feature | +| 0.5.1 | 2024-01-04 | [33873](https://github.com/airbytehq/airbyte/pull/33873) | Install normalization to enable DV2 beta | +| 0.5.0 | 2023-12-18 | [33507](https://github.com/airbytehq/airbyte/pull/33507) | Upgrade to latest CDK; Fix DATs and tests | | 0.4.0 | 2023-06-27 | [\#27781](https://github.com/airbytehq/airbyte/pull/27781) | License Update: Elv2 | | 0.3.27 | 2023-04-04 | [\#24604](https://github.com/airbytehq/airbyte/pull/24604) | Support for destination checkpointing | | 0.3.26 | 2022-09-27 | [\#17299](https://github.com/airbytehq/airbyte/pull/17299) | Improve error handling for strict-encrypt postgres destination | @@ -186,4 +192,4 @@ Now that you have set up the Postgres destination connector, check out the follo | 0.3.13 | 2021-12-01 | [\#8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | | 0.3.12 | 2021-11-08 | [\#7719](https://github.com/airbytehq/airbyte/pull/7719) | Improve handling of wide rows by buffering records based on their byte size rather than their count | | 0.3.11 | 2021-09-07 | [\#5743](https://github.com/airbytehq/airbyte/pull/5743) | Add SSH Tunnel support | -| 0.3.10 | 2021-08-11 | [\#5336](https://github.com/airbytehq/airbyte/pull/5336) | Destination Postgres: fix \u0000\(NULL\) value processing | +| 0.3.10 | 2021-08-11 | [\#5336](https://github.com/airbytehq/airbyte/pull/5336) | Destination Postgres: fix \u0000\(NULL\) value processing | \ No newline at end of file diff --git a/docs/integrations/destinations/qdrant.md b/docs/integrations/destinations/qdrant.md new file mode 100644 index 000000000000..537df671fbd5 --- /dev/null +++ b/docs/integrations/destinations/qdrant.md @@ -0,0 +1,83 @@ +# Qdrant +This page guides you through the process of setting up the [Qdrant](https://qdrant.tech/documentation/) destination connector. + + + +## Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :----------------------------- | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | Yes | | +| Incremental - Append + Deduped | Yes | | + +#### Output Schema + +Only one stream will exist to collect payload and vectors (optional) from all source streams. This will be in a [collection](https://qdrant.tech/documentation/concepts/collections/) in [Qdrant](https://qdrant.tech/documentation/) whose name will be defined by the user. If the collection does not already exist in the Qdrant instance, a new collection with the same name will be created. + +For each [point](https://qdrant.tech/documentation/concepts/points/) in the collection, a UUID string is generated and used as the [point id](https://qdrant.tech/documentation/concepts/points/#point-ids). The embeddings generated as defined or extracted from the source stream will be stored as the point vectors. The point payload will contain primarily the record metadata. The text field will then be stored in a field (as defined in the config) in the point payload. + +## Getting Started + +You can connect to a Qdrant instance either in local mode or cloud mode. + - For the local mode, you will need to set it up using Docker. Check the Qdrant docs [here](https://qdrant.tech/documentation/guides/installation/#docker) for an official guide. After setting up, you would need your host, port and if applicable, your gRPC port. + - To setup to an instance in Qdrant cloud, check out [this official guide](https://qdrant.tech/documentation/cloud/) to get started. After setting up the instance, you would need the instance url and an API key to connect. + +Note that this connector does not support a local persistent mode. To test, use the docker option. + + +#### Requirements + +To use the Qdrant destination, you'll need: +- An account with API access for OpenAI, Cohere (depending on which embedding method you want to use) or neither (if you want to extract the vectors from the source stream) +- A Qdrant db instance (local mode or cloud mode) +- Qdrant API Credentials (for cloud mode) +- Host and Port (for local mode) +- gRPC port (if applicable in local mode) + +#### Configure Network Access + +Make sure your Qdrant database can be accessed by Airbyte. If your database is within a VPC, you may need to allow access from the IP you're using to expose Airbyte. + + +### Setup the Qdrant Destination in Airbyte + +You should now have all the requirements needed to configure Qdrant as a destination in the UI. You'll need the following information to configure the Qdrant destination: + +- (Required) **Text fields to embed** +- (Optional) **Text splitter** Options around configuring the chunking process provided by the [Langchain Python library](https://python.langchain.com/docs/get_started/introduction). +- (Required) **Fields to store as metadata** +- (Required) **Collection** The name of the collection in Qdrant db to store your data +- (Required) **The field in the payload that contains the embedded text** +- (Required) **Prefer gRPC** Whether to prefer gRPC over HTTP. +- (Required) **Distance Metric** The Distance metrics used to measure similarities among vectors. Select from: + - [Dot product](https://en.wikipedia.org/wiki/Dot_product) + - [Cosine similarity](https://en.wikipedia.org/wiki/Cosine_similarity) + - [Euclidean distance](https://en.wikipedia.org/wiki/Euclidean_distance) +- (Required) Authentication method + - For local mode + - **Host** for example localhost + - **Port** for example 8000 + - **gRPC Port** (Optional) + - For cloud mode + - **Url** The url of the cloud Qdrant instance. + - **API Key** The API Key for the cloud Qdrant instance +- (Optional) Embedding + - **OpenAI API key** if using OpenAI for embedding + - **Cohere API key** if using Cohere for embedding + - Embedding **Field name** and **Embedding dimensions** if getting the embeddings from stream records + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :--------------------------------------------------------- | :----------------------------------------- | +| 0.0.10 | 2023-12-11 | [33303](https://github.com/airbytehq/airbyte/pull/33303) | Fix bug with embedding special tokens | +| 0.0.9 | 2023-12-01 | [32697](https://github.com/airbytehq/airbyte/pull/32697) | Allow omitting raw text | +| 0.0.8 | 2023-11-29 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Support deleting records for CDC sources and fix spec schema | +| 0.0.7 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | +| 0.0.6 | 2023-10-23 | [#31563](https://github.com/airbytehq/airbyte/pull/31563) | Add field mapping option | +| 0.0.5 | 2023-10-15 | [#31329](https://github.com/airbytehq/airbyte/pull/31329) | Add OpenAI-compatible embedder option | +| 0.0.4 | 2023-10-04 | [#31075](https://github.com/airbytehq/airbyte/pull/31075) | Fix OpenAI embedder batch size | +| 0.0.3 | 2023-09-29 | [#30820](https://github.com/airbytehq/airbyte/pull/30820) | Update CDK | +| 0.0.2 | 2023-09-25 | [#30689](https://github.com/airbytehq/airbyte/pull/30689) | Update CDK to support Azure OpenAI embeddings and text splitting options | +| 0.0.1 | 2023-09-22 | [#30332](https://github.com/airbytehq/airbyte/pull/30332) | 🎉 New Destination: Qdrant (Vector Database) | diff --git a/docs/integrations/destinations/r2.md b/docs/integrations/destinations/r2.md index 6ed9c67deebc..287e7c5de23e 100644 --- a/docs/integrations/destinations/r2.md +++ b/docs/integrations/destinations/r2.md @@ -46,7 +46,7 @@ to create an S3 bucket, or you can create bucket via R2 module of [dashboard](ht _${EPOCH}_`. - **R2 Filename pattern** - The pattern allows you to set the file-name format for the R2 staging file(s), next placeholders combinations are currently supported: - {date}, {date:yyyy_MM}, {timestamp}, {timestamp:millis}, {timestamp:micros}, {part_number}, {sync_id}, {format_extension}. Please, don't use empty space and not supportable placeholders, as they won't recognized. + `{date}`, `{date:yyyy_MM}`, `{timestamp}`, `{timestamp:millis}`, `{timestamp:micros}`, `{part_number}`, `{sync_id}`, `{format_extension}`. Please, don't use empty space and not supportable placeholders, as they won't recognized. 5. Click `Set up destination`. **For Airbyte OSS:** @@ -73,7 +73,7 @@ _${EPOCH}_`. _${EPOCH}_`. - **R2 Filename pattern** - The pattern allows you to set the file-name format for the R2 staging file(s), next placeholders combinations are currently supported: - {date}, {date:yyyy_MM}, {timestamp}, {timestamp:millis}, {timestamp:micros}, {part_number}, {sync_id}, {format_extension}. Please, don't use empty space and not supportable placeholders, as they won't recognized. + `{date}`, `{date:yyyy_MM}`, `{timestamp}`, `{timestamp:millis}`, `{timestamp:micros}`, `{part_number}`, `{sync_id}`, `{format_extension}`. Please, don't use empty space and not supportable placeholders, as they won't recognized. 5. Click `Set up destination`. diff --git a/docs/integrations/destinations/redshift.md b/docs/integrations/destinations/redshift.md index 4dfedfd02f34..7a551b8ba696 100644 --- a/docs/integrations/destinations/redshift.md +++ b/docs/integrations/destinations/redshift.md @@ -8,7 +8,12 @@ The Airbyte Redshift destination allows you to sync data to Redshift. This Redshift destination connector has two replication strategies: -1. INSERT: Replicates data via SQL INSERT queries. This is built on top of the destination-jdbc code base and is configured to rely on JDBC 4.2 standard drivers provided by Amazon via Mulesoft [here](https://mvnrepository.com/artifact/com.amazon.redshift/redshift-jdbc42) as described in Redshift documentation [here](https://docs.aws.amazon.com/redshift/latest/mgmt/jdbc20-install.html). **Not recommended for production workloads as this does not scale well**. +1. INSERT: Replicates data via SQL INSERT queries. This is built on top of the destination-jdbc code + base and is configured to rely on JDBC 4.2 standard drivers provided by Amazon via Mulesoft + [here](https://mvnrepository.com/artifact/com.amazon.redshift/redshift-jdbc42) as described in + Redshift documentation + [here](https://docs.aws.amazon.com/redshift/latest/mgmt/jdbc20-install.html). **Not recommended + for production workloads as this does not scale well**. For INSERT strategy: @@ -21,77 +26,129 @@ For INSERT strategy: - This database needs to exist within the cluster provided. - **JDBC URL Params** (optional) -2. COPY: Replicates data by first uploading data to an S3 bucket and issuing a COPY command. This is the recommended loading approach described by Redshift [best practices](https://docs.aws.amazon.com/redshift/latest/dg/c_loading-data-best-practices.html). Requires an S3 bucket and credentials. +2. COPY: Replicates data by first uploading data to an S3 bucket and issuing a COPY command. This is + the recommended loading approach described by Redshift + [best practices](https://docs.aws.amazon.com/redshift/latest/dg/c_loading-data-best-practices.html). + Requires an S3 bucket and credentials. -Airbyte automatically picks an approach depending on the given configuration - if S3 configuration is present, Airbyte will use the COPY strategy and vice versa. +Airbyte automatically picks an approach depending on the given configuration - if S3 configuration +is present, Airbyte will use the COPY strategy and vice versa. For COPY strategy: - **S3 Bucket Name** - - See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket. + - See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to + create an S3 bucket. - **S3 Bucket Region** - Place the S3 bucket and the Redshift cluster in the same region to save on networking costs. - **Access Key Id** - - See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key. - - We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the staging bucket. + - See + [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) + on how to generate an access key. + - We recommend creating an Airbyte-specific user. This user will require + [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) + to objects in the staging bucket. - **Secret Access Key** - Corresponding key to the above key id. - **Part Size** - - Affects the size limit of an individual Redshift table. Optional. Increase this if syncing tables larger than 100GB. Files are streamed to S3 in parts. This determines the size of each part, in MBs. As S3 has a limit of 10,000 parts per file, part size affects the table size. This is 10MB by default, resulting in a default table limit of 100GB. Note, a larger part size will result in larger memory requirements. A rule of thumb is to multiply the part size by 10 to get the memory requirement. Modify this with care. + - Affects the size limit of an individual Redshift table. Optional. Increase this if syncing + tables larger than 100GB. Files are streamed to S3 in parts. This determines the size of each + part, in MBs. As S3 has a limit of 10,000 parts per file, part size affects the table size. This + is 10MB by default, resulting in a default table limit of 100GB. Note, a larger part size will + result in larger memory requirements. A rule of thumb is to multiply the part size by 10 to get + the memory requirement. Modify this with care. - **S3 Filename pattern** - - The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: {date}, {date:yyyy_MM}, {timestamp}, {timestamp:millis}, {timestamp:micros}, {part_number}, {sync_id}, {format_extension}. Please, don't use empty space and not supportable placeholders, as they won't recognized. + - The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders + combinations are currently supported: `{date}`, `{date:yyyy_MM}`, `{timestamp}`, + `{timestamp:millis}`, `{timestamp:micros}`, `{part_number}`, `{sync_id}`, `{format_extension}`. + Please, don't use empty space and not supportable placeholders, as they won't recognized. Optional parameters: - **Bucket Path** - - The directory within the S3 bucket to place the staging data. For example, if you set this to `yourFavoriteSubdirectory`, we will place the staging data inside `s3://yourBucket/yourFavoriteSubdirectory`. If not provided, defaults to the root directory. + - The directory within the S3 bucket to place the staging data. For example, if you set this to + `yourFavoriteSubdirectory`, we will place the staging data inside + `s3://yourBucket/yourFavoriteSubdirectory`. If not provided, defaults to the root directory. - **Purge Staging Data** - - Whether to delete the staging files from S3 after completing the sync. Specifically, the connector will create CSV files named `bucketPath/namespace/streamName/syncDate_epochMillis_randomUuid.csv` containing three columns (`ab_id`, `data`, `emitted_at`). Normally these files are deleted after the `COPY` command completes; if you want to keep them for other purposes, set `purge_staging_data` to `false`. + - Whether to delete the staging files from S3 after completing the sync. Specifically, the + connector will create CSV files named + `bucketPath/namespace/streamName/syncDate_epochMillis_randomUuid.csv` containing three columns + (`ab_id`, `data`, `emitted_at`). Normally these files are deleted after the `COPY` command + completes; if you want to keep them for other purposes, set `purge_staging_data` to `false`. -NOTE: S3 staging does not use the SSH Tunnel option, if configured. SSH Tunnel supports the SQL connection only. S3 is secured through public HTTPS access only. +NOTE: S3 staging does not use the SSH Tunnel option, if configured. SSH Tunnel supports the SQL +connection only. S3 is secured through public HTTPS access only. ## Step 1: Set up Redshift -1. [Log in](https://aws.amazon.com/console/) to AWS Management console. - If you don't have a AWS account already, you’ll need to [create](https://aws.amazon.com/premiumsupport/knowledge-center/create-and-activate-aws-account/) one in order to use the API. -2. Go to the AWS Redshift service -3. [Create](https://docs.aws.amazon.com/ses/latest/dg/event-publishing-redshift-cluster.html) and activate AWS Redshift cluster if you don't have one ready -4. (Optional) [Allow](https://aws.amazon.com/premiumsupport/knowledge-center/cannot-connect-redshift-cluster/) connections from Airbyte to your Redshift cluster \(if they exist in separate VPCs\) -5. (Optional) [Create](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) a staging S3 bucket \(for the COPY strategy\). -6. Create a user with at least create table permissions for the schema. If the schema does not exist you need to add permissions for that, too. Something like this: - -``` -GRANT CREATE ON DATABASE database_name TO airflow_user; -- add create schema permission -GRANT usage, create on schema my_schema TO airflow_user; -- add create table permission +1. [Log in](https://aws.amazon.com/console/) to AWS Management console. If you don't have a AWS + account already, you’ll need to + [create](https://aws.amazon.com/premiumsupport/knowledge-center/create-and-activate-aws-account/) + one in order to use the API. +2. Go to the AWS Redshift service. +3. [Create](https://docs.aws.amazon.com/ses/latest/dg/event-publishing-redshift-cluster.html) and + activate AWS Redshift cluster if you don't have one ready. +4. (Optional) + [Allow](https://aws.amazon.com/premiumsupport/knowledge-center/cannot-connect-redshift-cluster/) + connections from Airbyte to your Redshift cluster \(if they exist in separate VPCs\). +5. (Optional) + [Create](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) a + staging S3 bucket \(for the COPY strategy\). + +### Permissions in Redshift +Airbyte writes data into two schemas, whichever schema you want your data to land in, e.g. `my_schema` +and a "Raw Data" schema that Airbyte uses to improve ELT reliability. By default, this raw data schema +is `airbyte_internal` but this can be overridden in the Redshift Destination's advanced settings. +Airbyte also needs to query Redshift's +[SVV_TABLE_INFO](https://docs.aws.amazon.com/redshift/latest/dg/r_SVV_TABLE_INFO.html) table for +metadata about the tables airbyte manages. + +To ensure the `airbyte_user` has the correction permissions to: +- create schemas in your database +- grant usage to any existing schemas you want Airbyte to use +- grant select to the `svv_table_info` table + +You can execute the following SQL statements + +```sql +GRANT CREATE ON DATABASE database_name TO airbyte_user; -- add create schema permission +GRANT usage, create on schema my_schema TO airbyte_user; -- add create table permission +GRANT SELECT ON TABLE SVV_TABLE_INFO TO airbyte_user; -- add select permission for svv_table_info ``` ### Optional Use of SSH Bastion Host -This connector supports the use of a Bastion host as a gateway to a private Redshift cluster via SSH Tunneling. -Setup of the host is beyond the scope of this document but several tutorials are available online to fascilitate this task. -Enter the bastion host, port and credentials in the destination configuration. +This connector supports the use of a Bastion host as a gateway to a private Redshift cluster via SSH +Tunneling. Setup of the host is beyond the scope of this document but several tutorials are +available online to fascilitate this task. Enter the bastion host, port and credentials in the +destination configuration. ## Step 2: Set up the destination connector in Airbyte **For Airbyte Cloud:** 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. -2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new destination**. -3. On the destination setup page, select **Redshift** from the Destination type dropdown and enter a name for this connector. -4. Fill in all the required fields to use the INSERT or COPY strategy +2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new + destination**. +3. On the destination setup page, select **Redshift** from the Destination type dropdown and enter a + name for this connector. +4. Fill in all the required fields to use the INSERT or COPY strategy. 5. Click `Set up destination`. **For Airbyte Open Source:** 1. Go to local Airbyte page. -2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new destination**. -3. On the destination setup page, select **Redshift** from the Destination type dropdown and enter a name for this connector. -4. Fill in all the required fields to use the INSERT or COPY strategy +2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new + destination**. +3. On the destination setup page, select **Redshift** from the Destination type dropdown and enter a + name for this connector. +4. Fill in all the required fields to use the INSERT or COPY strategy. 5. Click `Set up destination`. ## Supported sync modes -The Redshift destination connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-mode): +The Redshift destination connector supports the following +[sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-mode): - Full Refresh - Incremental - Append Sync @@ -99,8 +156,8 @@ The Redshift destination connector supports the following [sync modes](https://d ## Performance considerations -Synchronization performance depends on the amount of data to be transferred. -Cluster scaling issues can be resolved directly using the cluster settings in the AWS Redshift console +Synchronization performance depends on the amount of data to be transferred. Cluster scaling issues +can be resolved directly using the cluster settings in the AWS Redshift console. ## Connector-specific features & highlights @@ -110,33 +167,49 @@ From [Redshift Names & Identifiers](https://docs.aws.amazon.com/redshift/latest/ #### Standard Identifiers -- Begin with an ASCII single-byte alphabetic character or underscore character, or a UTF-8 multibyte character two to four bytes long. -- Subsequent characters can be ASCII single-byte alphanumeric characters, underscores, or dollar signs, or UTF-8 multibyte characters two to four bytes long. +- Begin with an ASCII single-byte alphabetic character or underscore character, or a UTF-8 multibyte + character two to four bytes long. +- Subsequent characters can be ASCII single-byte alphanumeric characters, underscores, or dollar + signs, or UTF-8 multibyte characters two to four bytes long. - Be between 1 and 127 bytes in length, not including quotation marks for delimited identifiers. - Contain no quotation marks and no spaces. #### Delimited Identifiers -Delimited identifiers \(also known as quoted identifiers\) begin and end with double quotation marks \("\). If you use a delimited identifier, you must use the double quotation marks for every reference to that object. The identifier can contain any standard UTF-8 printable characters other than the double quotation mark itself. Therefore, you can create column or table names that include otherwise illegal characters, such as spaces or the percent symbol. ASCII letters in delimited identifiers are case-insensitive and are folded to lowercase. To use a double quotation mark in a string, you must precede it with another double quotation mark character. +Delimited identifiers \(also known as quoted identifiers\) begin and end with double quotation marks +\("\). If you use a delimited identifier, you must use the double quotation marks for every +reference to that object. The identifier can contain any standard UTF-8 printable characters other +than the double quotation mark itself. Therefore, you can create column or table names that include +otherwise illegal characters, such as spaces or the percent symbol. ASCII letters in delimited +identifiers are case-insensitive and are folded to lowercase. To use a double quotation mark in a +string, you must precede it with another double quotation mark character. -Therefore, Airbyte Redshift destination will create tables and schemas using the Unquoted identifiers when possible or fallback to Quoted Identifiers if the names are containing special characters. +Therefore, Airbyte Redshift destination will create tables and schemas using the Unquoted +identifiers when possible or fallback to Quoted Identifiers if the names are containing special +characters. ### Data Size Limitations -Redshift specifies a maximum limit of 1MB (and 65535 bytes for any VARCHAR fields within the JSON record) to store the raw JSON record data. Thus, when a row is too big to fit, the Redshift destination fails to load such data and currently ignores that record. -See docs for [SUPER](https://docs.aws.amazon.com/redshift/latest/dg/r_SUPER_type.html) and [SUPER limitations](https://docs.aws.amazon.com/redshift/latest/dg/limitations-super.html) +Redshift specifies a maximum limit of 16MB (and 65535 bytes for any VARCHAR fields within the JSON +record) to store the raw JSON record data. Thus, when a row is too big to fit, the Redshift +destination fails to load such data and currently ignores that record. See docs for +[SUPER](https://docs.aws.amazon.com/redshift/latest/dg/r_SUPER_type.html) and +[SUPER limitations](https://docs.aws.amazon.com/redshift/latest/dg/limitations-super.html). ### Encryption -All Redshift connections are encrypted using SSL +All Redshift connections are encrypted using SSL. ### Output schema Each stream will be output into its own raw table in Redshift. Each table will contain 3 columns: -- `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in Redshift is `VARCHAR`. -- `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. The column type in Redshift is `TIMESTAMP WITH TIME ZONE`. -- `_airbyte_data`: a json blob representing with the event data. The column type in Redshift is `SUPER`. +- `_airbyte_ab_id`: a uuid assigned by Airbyte to each event that is processed. The column type in + Redshift is `VARCHAR`. +- `_airbyte_emitted_at`: a timestamp representing when the event was pulled from the data source. + The column type in Redshift is `TIMESTAMP WITH TIME ZONE`. +- `_airbyte_data`: a json blob representing with the event data. The column type in Redshift is + `SUPER`. ## Data type mapping @@ -156,6 +229,29 @@ Each stream will be output into its own raw table in Redshift. Each table will c | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.8.0 | 2024-01-18 | [34236](https://github.com/airbytehq/airbyte/pull/34236) | Upgrade CDK to 0.13.0 | +| 0.7.15 | 2024-01-11 | [\#34186](https://github.com/airbytehq/airbyte/pull/34186) | Update check method with svv_table_info permission check, fix bug where s3 staging files were not being deleted. | +| 0.7.14 | 2024-01-08 | [\#34014](https://github.com/airbytehq/airbyte/pull/34014) | Update order of options in spec | +| 0.7.13 | 2024-01-05 | [\#33948](https://github.com/airbytehq/airbyte/pull/33948) | Fix NPE when prepare tables fail; Add case sensitive session for super; Bastion heartbeats added | +| 0.7.12 | 2024-01-03 | [\#33924](https://github.com/airbytehq/airbyte/pull/33924) | Add new ap-southeast-3 AWS region | +| 0.7.11 | 2024-01-04 | [\#33730](https://github.com/airbytehq/airbyte/pull/33730) | Internal code structure changes | +| 0.7.10 | 2024-01-04 | [\#33728](https://github.com/airbytehq/airbyte/pull/33728) | Allow users to disable final table creation | +| 0.7.9 | 2024-01-03 | [\#33877](https://github.com/airbytehq/airbyte/pull/33877) | Fix Jooq StackOverflowError | +| 0.7.8 | 2023-12-28 | [\#33788](https://github.com/airbytehq/airbyte/pull/33788) | Thread-safe fix for file part names (s3 staging files) | +| 0.7.7 | 2024-01-04 | [\#33728](https://github.com/airbytehq/airbyte/pull/33728) | Add option to only type and dedupe at the end of the sync | +| 0.7.6 | 2023-12-20 | [\#33704](https://github.com/airbytehq/airbyte/pull/33704) | Only run T+D on a stream if it had any records during the sync | +| 0.7.5 | 2023-12-18 | [\#33124](https://github.com/airbytehq/airbyte/pull/33124) | Make Schema Creation Separate from Table Creation | +| 0.7.4 | 2023-12-13 | [\#33369](https://github.com/airbytehq/airbyte/pull/33369) | Use jdbc common sql implementation | +| 0.7.3 | 2023-12-12 | [\#33367](https://github.com/airbytehq/airbyte/pull/33367) | DV2: fix migration logic | +| 0.7.2 | 2023-12-11 | [\#33335](https://github.com/airbytehq/airbyte/pull/33335) | DV2: improve data type mapping | +| 0.7.1 | 2023-12-11 | [\#33307](https://github.com/airbytehq/airbyte/pull/33307) | ~DV2: improve data type mapping~ No changes | +| 0.7.0 | 2023-12-05 | [\#32326](https://github.com/airbytehq/airbyte/pull/32326) | Opt in beta for v2 destination | +| 0.6.11 | 2023-11-29 | [\#32888](https://github.com/airbytehq/airbyte/pull/32888) | Use the new async framework. | +| 0.6.10 | 2023-11-06 | [\#32193](https://github.com/airbytehq/airbyte/pull/32193) | Adopt java CDK version 0.4.1. | +| 0.6.9 | 2023-10-10 | [\#31083](https://github.com/airbytehq/airbyte/pull/31083) | Fix precision of numeric values in async destinations | +| 0.6.8 | 2023-10-10 | [\#31218](https://github.com/airbytehq/airbyte/pull/31218) | Clarify configuration groups | +| 0.6.7 | 2023-10-06 | [\#31153](https://github.com/airbytehq/airbyte/pull/31153) | Increase jvm GC retries | +| 0.6.6 | 2023-10-06 | [\#31129](https://github.com/airbytehq/airbyte/pull/31129) | Reduce async buffer size | | 0.6.5 | 2023-08-18 | [\#28619](https://github.com/airbytehq/airbyte/pull/29640) | Fix duplicate staging object names in concurrent environment (e.g. async) | | 0.6.4 | 2023-08-10 | [\#28619](https://github.com/airbytehq/airbyte/pull/28619) | Use async method for staging | | 0.6.3 | 2023-08-07 | [\#29188](https://github.com/airbytehq/airbyte/pull/29188) | Internal code refactoring | @@ -197,7 +293,7 @@ Each stream will be output into its own raw table in Redshift. Each table will c | 0.3.33 | 2022-05-04 | [\#12601](https://github.com/airbytehq/airbyte/pull/12601) | Apply buffering strategy for S3 staging | | 0.3.32 | 2022-04-20 | [\#12085](https://github.com/airbytehq/airbyte/pull/12085) | Fixed bug with switching between INSERT and COPY config | | 0.3.31 | 2022-04-19 | [\#12064](https://github.com/airbytehq/airbyte/pull/12064) | Added option to support SUPER datatype in \_airbyte_raw\*\*\* table | -| 0.3.29 | 2022-04-05 | [\#11729](https://github.com/airbytehq/airbyte/pull/11729) | Fixed bug with dashes in schema name | | +| 0.3.29 | 2022-04-05 | [\#11729](https://github.com/airbytehq/airbyte/pull/11729) | Fixed bug with dashes in schema name | | 0.3.28 | 2022-03-18 | [\#11254](https://github.com/airbytehq/airbyte/pull/11254) | Fixed missing records during S3 staging | | 0.3.27 | 2022-02-25 | [\#10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | | 0.3.25 | 2022-02-14 | [\#9920](https://github.com/airbytehq/airbyte/pull/9920) | Updated the size of staging files for S3 staging. Also, added closure of S3 writers to staging files when data has been written to an staging file. | @@ -212,4 +308,4 @@ Each stream will be output into its own raw table in Redshift. Each table will c | 0.3.14 | 2021-10-08 | [\#5924](https://github.com/airbytehq/airbyte/pull/5924) | Fixed AWS S3 Staging COPY is writing records from different table in the same raw table | | 0.3.13 | 2021-09-02 | [\#5745](https://github.com/airbytehq/airbyte/pull/5745) | Disable STATUPDATE flag when using S3 staging to speed up performance | | 0.3.12 | 2021-07-21 | [\#3555](https://github.com/airbytehq/airbyte/pull/3555) | Enable partial checkpointing for halfway syncs | -| 0.3.11 | 2021-07-20 | [\#4874](https://github.com/airbytehq/airbyte/pull/4874) | allow `additionalProperties` in connector spec | +| 0.3.11 | 2021-07-20 | [\#4874](https://github.com/airbytehq/airbyte/pull/4874) | allow `additionalProperties` in connector spec | \ No newline at end of file diff --git a/docs/integrations/destinations/rockset.md b/docs/integrations/destinations/rockset.md index 0ab1709a68b6..bf685f3e4ce9 100644 --- a/docs/integrations/destinations/rockset.md +++ b/docs/integrations/destinations/rockset.md @@ -23,7 +23,7 @@ | api_server | string | api URL to rockset, specifying http protocol | | workspace | string | workspace under which rockset collections will be added/modified | -## Getting Started \(Airbyte Open-Source / Airbyte Cloud\) +## Getting Started \(Airbyte Open Source / Airbyte Cloud\) #### Requirements diff --git a/docs/integrations/destinations/s3-glue.md b/docs/integrations/destinations/s3-glue.md index 5e66cf7d6e70..ec0a2ac1bd6c 100644 --- a/docs/integrations/destinations/s3-glue.md +++ b/docs/integrations/destinations/s3-glue.md @@ -14,52 +14,74 @@ List of required fields: - **Glue database** - **Glue serialization library** -1. Allow connections from Airbyte server to your AWS S3/ Minio S3 cluster \(if they exist in separate VPCs\). -2. An S3 bucket with credentials or an instance profile with read/write permissions configured for the host (ec2, eks). +1. Allow connections from Airbyte server to your AWS S3/ Minio S3 cluster \(if they exist in + separate VPCs\). +2. An S3 bucket with credentials or an instance profile with read/write permissions configured for + the host (ec2, eks). 3. [Enforce encryption of data in transit](https://docs.aws.amazon.com/AmazonS3/latest/userguide/security-best-practices.html#transit) 4. Allow permissions to access the AWS Glue service from the Airbyte connector ## Step 1: Set up S3 -[Sign in](https://console.aws.amazon.com/iam/) to your AWS account. -Use an existing or create new [Access Key ID and Secret Access Key](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#:~:text=IAM%20User%20Guide.-,Programmatic%20access,-You%20must%20provide). +[Sign in](https://console.aws.amazon.com/iam/) to your AWS account. Use an existing or create new +[Access Key ID and Secret Access Key](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#:~:text=IAM%20User%20Guide.-,Programmatic%20access,-You%20must%20provide). -Prepare S3 bucket that will be used as destination, see [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket. +Prepare S3 bucket that will be used as destination, see +[this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create +an S3 bucket. -NOTE: If the S3 cluster is not configured to use TLS, the connection to Amazon S3 silently reverts to an unencrypted connection. Airbyte recommends all connections be configured to use TLS/SSL as support for AWS's [shared responsibility model](https://aws.amazon.com/compliance/shared-responsibility-model/) +NOTE: If the S3 cluster is not configured to use TLS, the connection to Amazon S3 silently reverts +to an unencrypted connection. Airbyte recommends all connections be configured to use TLS/SSL as +support for AWS's +[shared responsibility model](https://aws.amazon.com/compliance/shared-responsibility-model/) ## Step 2: Set up Glue -[Sign in](https://console.aws.amazon.com/iam/) to your AWS account. -Use an existing or create new [Access Key ID and Secret Access Key](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#:~:text=IAM%20User%20Guide.-,Programmatic%20access,-You%20must%20provide). +[Sign in](https://console.aws.amazon.com/iam/) to your AWS account. Use an existing or create new +[Access Key ID and Secret Access Key](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#:~:text=IAM%20User%20Guide.-,Programmatic%20access,-You%20must%20provide). -Prepare the Glue database that will be used as destination, see [this](https://docs.aws.amazon.com/glue/latest/dg/console-databases.html) to create a Glue database +Prepare the Glue database that will be used as destination, see +[this](https://docs.aws.amazon.com/glue/latest/dg/console-databases.html) to create a Glue database ## Step 3: Set up the S3-Glue destination connector in Airbyte **For Airbyte Cloud:** 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. -2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new destination**. -3. On the destination setup page, select **S3** from the Destination type dropdown and enter a name for this connector. +2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new + destination**. +3. On the destination setup page, select **S3** from the Destination type dropdown and enter a name + for this connector. 4. Configure fields: - **Access Key Id** - - See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key. - - We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the bucket. + - See + [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) + on how to generate an access key. + - We recommend creating an Airbyte-specific user. This user will require + [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) + to objects in the bucket. - **Secret Access Key** - Corresponding key to the above key id. - **S3 Bucket Name** - - See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket. + - See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) + to create an S3 bucket. - **S3 Bucket Path** - Subdirectory under the above bucket to sync the data into. - **S3 Bucket Region** - - See [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes. + - See + [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) + for all region codes. - **S3 Path Format** - - Additional string format on how to store data under S3 Bucket Path. Default value is `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_`. + - Additional string format on how to store data under S3 Bucket Path. Default value is + `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_`. - **S3 Endpoint** - Leave empty if using AWS S3, fill in S3 URL if using Minio S3. - **S3 Filename pattern** - - The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: {date}, {date:yyyy_MM}, {timestamp}, {timestamp:millis}, {timestamp:micros}, {part_number}, {sync_id}, {format_extension}. Please, don't use empty space and not supportable placeholders, as they won't recognized. + - The pattern allows you to set the file-name format for the S3 staging file(s), next + placeholders combinations are currently supported: `{date}`, `{date:yyyy_MM}`, `{timestamp}`, + `{timestamp:millis}`, `{timestamp:micros}`, `{part_number}`, `{sync_id}`, + `{format_extension}`. Please, don't use empty space and not supportable placeholders, as they + won't recognized. - **Glue database** - The Glue database name that was previously created through the management console or the cli. - **Glue serialization library** @@ -69,34 +91,55 @@ Prepare the Glue database that will be used as destination, see [this](https://d **For Airbyte Open Source:** 1. Go to local Airbyte page. -2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new destination**. -3. On the destination setup page, select **S3** from the Destination type dropdown and enter a name for this connector. +2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new + destination**. +3. On the destination setup page, select **S3** from the Destination type dropdown and enter a name + for this connector. 4. Configure fields: - **Access Key Id** - - See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key. - - See [this](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html) on how to create a instanceprofile. - - We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the staging bucket. - - If the Access Key and Secret Access Key are not provided, the authentication will rely on the instanceprofile. + - See + [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) + on how to generate an access key. + - See + [this](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html) + on how to create a instanceprofile. + - We recommend creating an Airbyte-specific user. This user will require + [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) + to objects in the staging bucket. + - If the Access Key and Secret Access Key are not provided, the authentication will rely on the + instanceprofile. - **Secret Access Key** - Corresponding key to the above key id. - Make sure your S3 bucket is accessible from the machine running Airbyte. - This depends on your networking setup. - - You can check AWS S3 documentation with a tutorial on how to properly configure your S3's access [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-control-overview.html). - - If you use instance profile authentication, make sure the role has permission to read/write on the bucket. - - The easiest way to verify if Airbyte is able to connect to your S3 bucket is via the check connection tool in the UI. + - You can check AWS S3 documentation with a tutorial on how to properly configure your S3's + access + [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-control-overview.html). + - If you use instance profile authentication, make sure the role has permission to read/write + on the bucket. + - The easiest way to verify if Airbyte is able to connect to your S3 bucket is via the check + connection tool in the UI. - **S3 Bucket Name** - - See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket. + - See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) + to create an S3 bucket. - **S3 Bucket Path** - Subdirectory under the above bucket to sync the data into. - **S3 Bucket Region** - - See [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes. + - See + [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) + for all region codes. - **S3 Path Format** - - Additional string format on how to store data under S3 Bucket Path. Default value is `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_`. + - Additional string format on how to store data under S3 Bucket Path. Default value is + `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_`. - **S3 Endpoint** - Leave empty if using AWS S3, fill in S3 URL if using Minio S3. - **S3 Filename pattern** - - The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: {date}, {date:yyyy_MM}, {timestamp}, {timestamp:millis}, {timestamp:micros}, {part_number}, {sync_id}, {format_extension}. Please, don't use empty space and not supportable placeholders, as they won't recognized. + - The pattern allows you to set the file-name format for the S3 staging file(s), next + placeholders combinations are currently supported: `{date}`, `{date:yyyy_MM}`, `{timestamp}`, + `{timestamp:millis}`, `{timestamp:micros}`, `{part_number}`, `{sync_id}`, + `{format_extension}`. Please, don't use empty space and not supportable placeholders, as they + won't recognized. - **Glue database** - The Glue database name that was previously created through the management console or the cli. - **Glue serialization library** @@ -104,7 +147,8 @@ Prepare the Glue database that will be used as destination, see [this](https://d 5. Click `Set up destination`. -In order for everything to work correctly, it is also necessary that the user whose "S3 Key Id" and "S3 Access Key" are used have access to both the bucket and its contents. Policies to use: +In order for everything to work correctly, it is also necessary that the user whose "S3 Key Id" and +"S3 Access Key" are used have access to both the bucket and its contents. Policies to use: ```json { @@ -113,18 +157,18 @@ In order for everything to work correctly, it is also necessary that the user wh { "Effect": "Allow", "Action": "s3:*", - "Resource": [ - "arn:aws:s3:::YOUR_BUCKET_NAME/*", - "arn:aws:s3:::YOUR_BUCKET_NAME" - ] + "Resource": ["arn:aws:s3:::YOUR_BUCKET_NAME/*", "arn:aws:s3:::YOUR_BUCKET_NAME"] } ] } ``` -For setting up the necessary Glue policies see [this](https://docs.aws.amazon.com/glue/latest/dg/glue-resource-policies.html) and [this](https://docs.aws.amazon.com/glue/latest/dg/create-service-policy.html) +For setting up the necessary Glue policies see +[this](https://docs.aws.amazon.com/glue/latest/dg/glue-resource-policies.html) and +[this](https://docs.aws.amazon.com/glue/latest/dg/create-service-policy.html) -The full path of the output data with the default S3 Path Format `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_` is: +The full path of the output data with the default S3 Path Format +`${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_` is: ```text ///__. @@ -153,7 +197,8 @@ The rationales behind this naming pattern are: But it is possible to further customize by using the available variables to format the bucket path: -- `${NAMESPACE}`: Namespace where the stream comes from or configured by the connection namespace fields. +- `${NAMESPACE}`: Namespace where the stream comes from or configured by the connection namespace + fields. - `${STREAM_NAME}`: Name of the stream - `${YEAR}`: Year in which the sync was writing the output data in. - `${MONTH}`: Month in which the sync was writing the output data in. @@ -168,33 +213,41 @@ But it is possible to further customize by using the available variables to form Note: - Multiple `/` characters in the S3 path are collapsed into a single `/` character. -- If the output bucket contains too many files, the part id variable is using a `UUID` instead. It uses sequential ID otherwise. +- If the output bucket contains too many files, the part id variable is using a `UUID` instead. It + uses sequential ID otherwise. -Please note that the stream name may contain a prefix, if it is configured on the connection. -A data sync may create multiple files as the output files can be partitioned by size (targeting a size of 200MB compressed or lower) . +Please note that the stream name may contain a prefix, if it is configured on the connection. A data +sync may create multiple files as the output files can be partitioned by size (targeting a size of +200MB compressed or lower) . ## Supported sync modes -| Feature | Support | Notes | -| :----------------------------- | :-----: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Full Refresh Sync | ✅ | Warning: this mode deletes all previously synced data in the configured bucket path. | -| Incremental - Append Sync | ✅ | Warning: Airbyte provides at-least-once delivery. Depending on your source, you may see duplicated data. Learn more [here](/understanding-airbyte/connections/incremental-append#inclusive-cursors) | -| Incremental - Append + Deduped | ❌ | | -| Namespaces | ❌ | Setting a specific bucket path is equivalent to having separate namespaces. | +| Feature | Support | Notes | +| :----------------------------- | :-----: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Full Refresh Sync | ✅ | Warning: this mode deletes all previously synced data in the configured bucket path. | +| Incremental - Append Sync | ✅ | Warning: Airbyte provides at-least-once delivery. Depending on your source, you may see duplicated data. Learn more [here](/using-airbyte/core-concepts/sync-modes/incremental-append#inclusive-cursors) | +| Incremental - Append + Deduped | ❌ | | +| Namespaces | ❌ | Setting a specific bucket path is equivalent to having separate namespaces. | -The Airbyte S3 destination allows you to sync data to AWS S3 or Minio S3. Each stream is written to its own directory under the bucket. -⚠️ Please note that under "Full Refresh Sync" mode, data in the configured bucket and path will be wiped out before each sync. We recommend you to provision a dedicated S3 resource for this sync to prevent unexpected data deletion from misconfiguration. ⚠️ +The Airbyte S3 destination allows you to sync data to AWS S3 or Minio S3. Each stream is written to +its own directory under the bucket. ⚠️ Please note that under "Full Refresh Sync" mode, data in the +configured bucket and path will be wiped out before each sync. We recommend you to provision a +dedicated S3 resource for this sync to prevent unexpected data deletion from misconfiguration. ⚠️ ## Supported Output schema -Each stream will be outputted to its dedicated directory according to the configuration. The complete datastore of each stream includes all the output files under that directory. You can think of the directory as equivalent of a Table in the database world. +Each stream will be outputted to its dedicated directory according to the configuration. The +complete datastore of each stream includes all the output files under that directory. You can think +of the directory as equivalent of a Table in the database world. - Under Full Refresh Sync mode, old output files will be purged before new files are created. -- Under Incremental - Append Sync mode, new output files will be added that only contain the new data. +- Under Incremental - Append Sync mode, new output files will be added that only contain the new + data. ### JSON Lines \(JSONL\) -[JSON Lines](https://jsonlines.org/) is a text format with one JSON per line. Each line has a structure as follows: +[JSON Lines](https://jsonlines.org/) is a text format with one JSON per line. Each line has a +structure as follows: ```json { @@ -225,7 +278,8 @@ For example, given the following two json objects from a source: ] ``` -depending on whether you want to flatten your data or not (**_available as a configuration option_**) +depending on whether you want to flatten your data or not (**_available as a configuration +option_**) The json objects can have the following formats: @@ -239,17 +293,19 @@ The json objects can have the following formats: { "_airbyte_ab_id": "0a61de1b-9cdd-4455-a739-93572c9a5f20", "_airbyte_emitted_at": "1631948170000", "user_id": 456, "name": { "first": "Jane", "last": "Roe" } } ``` -Output files can be compressed. The default option is GZIP compression. If compression is selected, the output filename will have an extra extension (GZIP: `.jsonl.gz`). +Output files can be compressed. The default option is GZIP compression. If compression is selected, +the output filename will have an extra extension (GZIP: `.jsonl.gz`). ## CHANGELOG -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------- | -| 0.1.7 | 2023-05-01 | [25724](https://github.com/airbytehq/airbyte/pull/25724) | Fix decimal type creation syntax to avoid overflow | -| 0.1.6 | 2023-04-13 | [25178](https://github.com/airbytehq/airbyte/pull/25178) | Fix decimal precision and scale to allow for a wider range of numeric values | -| 0.1.5 | 2023-04-11 | [25048](https://github.com/airbytehq/airbyte/pull/25048) | Fix config schema to support new JSONL flattening configuration interface | -| 0.1.4 | 2023-03-10 | [23950](https://github.com/airbytehq/airbyte/pull/23950) | Fix schema syntax error for struct fields and handle missing `items` in array fields | -| 0.1.3 | 2023-02-10 | [22822](https://github.com/airbytehq/airbyte/pull/22822) | Fix data type for \_ab_emitted_at column in table definition | -| 0.1.2 | 2023-02-01 | [22220](https://github.com/airbytehq/airbyte/pull/22220) | Fix race condition in test, table metadata, add Airbyte sync fields to table definition | -| 0.1.1 | 2022-12-13 | [19907](https://github.com/airbytehq/airbyte/pull/19907) | Fix parsing empty object in schema | -| 0.1.0 | 2022-11-17 | [18695](https://github.com/airbytehq/airbyte/pull/18695) | Initial Commit | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :-------------------------------------------------------- | :-------------------------------------------------------------------------------------- | +| 0.1.8 | 2024-01-03 | [#33924](https://github.com/airbytehq/airbyte/pull/33924) | Add new ap-southeast-3 AWS region | +| 0.1.7 | 2023-05-01 | [25724](https://github.com/airbytehq/airbyte/pull/25724) | Fix decimal type creation syntax to avoid overflow | +| 0.1.6 | 2023-04-13 | [25178](https://github.com/airbytehq/airbyte/pull/25178) | Fix decimal precision and scale to allow for a wider range of numeric values | +| 0.1.5 | 2023-04-11 | [25048](https://github.com/airbytehq/airbyte/pull/25048) | Fix config schema to support new JSONL flattening configuration interface | +| 0.1.4 | 2023-03-10 | [23950](https://github.com/airbytehq/airbyte/pull/23950) | Fix schema syntax error for struct fields and handle missing `items` in array fields | +| 0.1.3 | 2023-02-10 | [22822](https://github.com/airbytehq/airbyte/pull/22822) | Fix data type for \_ab_emitted_at column in table definition | +| 0.1.2 | 2023-02-01 | [22220](https://github.com/airbytehq/airbyte/pull/22220) | Fix race condition in test, table metadata, add Airbyte sync fields to table definition | +| 0.1.1 | 2022-12-13 | [19907](https://github.com/airbytehq/airbyte/pull/19907) | Fix parsing empty object in schema | +| 0.1.0 | 2022-11-17 | [18695](https://github.com/airbytehq/airbyte/pull/18695) | Initial Commit | diff --git a/docs/integrations/destinations/s3.md b/docs/integrations/destinations/s3.md index d7c5291fca0c..dca7fc450427 100644 --- a/docs/integrations/destinations/s3.md +++ b/docs/integrations/destinations/s3.md @@ -12,20 +12,27 @@ List of required fields: - **S3 Bucket Path** - **S3 Bucket Region** -1. Allow connections from Airbyte server to your AWS S3/ Minio S3 cluster \(if they exist in separate VPCs\). -2. An S3 bucket with credentials or an instance profile with read/write permissions configured for the host (ec2, eks). +1. Allow connections from Airbyte server to your AWS S3/ Minio S3 cluster \(if they exist in + separate VPCs\). +2. An S3 bucket with credentials or an instance profile with read/write permissions configured for + the host (ec2, eks). 3. [Enforce encryption of data in transit](https://docs.aws.amazon.com/AmazonS3/latest/userguide/security-best-practices.html#transit) ## Setup guide ### Step 1: Set up S3 -[Sign in](https://console.aws.amazon.com/iam/) to your AWS account. -Use an existing or create new [Access Key ID and Secret Access Key](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#:~:text=IAM%20User%20Guide.-,Programmatic%20access,-You%20must%20provide). +[Sign in](https://console.aws.amazon.com/iam/) to your AWS account. Use an existing or create new +[Access Key ID and Secret Access Key](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#:~:text=IAM%20User%20Guide.-,Programmatic%20access,-You%20must%20provide). -Prepare S3 bucket that will be used as destination, see [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket. +Prepare S3 bucket that will be used as destination, see +[this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create +an S3 bucket. -NOTE: If the S3 cluster is not configured to use TLS, the connection to Amazon S3 silently reverts to an unencrypted connection. Airbyte recommends all connections be configured to use TLS/SSL as support for AWS's [shared responsibility model](https://aws.amazon.com/compliance/shared-responsibility-model/) +NOTE: If the S3 cluster is not configured to use TLS, the connection to Amazon S3 silently reverts +to an unencrypted connection. Airbyte recommends all connections be configured to use TLS/SSL as +support for AWS's +[shared responsibility model](https://aws.amazon.com/compliance/shared-responsibility-model/) ### Step 2: Set up the S3 destination connector in Airbyte @@ -34,26 +41,40 @@ NOTE: If the S3 cluster is not configured to use TLS, the connection to Amazon S **For Airbyte Cloud:** 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. -2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new destination**. -3. On the destination setup page, select **S3** from the Destination type dropdown and enter a name for this connector. +2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new + destination**. +3. On the destination setup page, select **S3** from the Destination type dropdown and enter a name + for this connector. 4. Configure fields: - **Access Key Id** - - See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key. - - We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the bucket. + - See + [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) + on how to generate an access key. + - We recommend creating an Airbyte-specific user. This user will require + [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) + to objects in the bucket. - **Secret Access Key** - Corresponding key to the above key id. - **S3 Bucket Name** - - See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket. + - See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) + to create an S3 bucket. - **S3 Bucket Path** - Subdirectory under the above bucket to sync the data into. - - **S3 Bucket Region** - - See [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes. + - **S3 Bucket Region**: + - See + [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) + for all region codes. - **S3 Path Format** - - Additional string format on how to store data under S3 Bucket Path. Default value is `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_`. + - Additional string format on how to store data under S3 Bucket Path. Default value is + `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_`. - **S3 Endpoint** - Leave empty if using AWS S3, fill in S3 URL if using Minio S3. - **S3 Filename pattern** - - The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: {date}, {date:yyyy_MM}, {timestamp}, {timestamp:millis}, {timestamp:micros}, {part_number}, {sync_id}, {format_extension}. Please, don't use empty space and not supportable placeholders, as they won't be recognized. + - The pattern allows you to set the file-name format for the S3 staging file(s), next + placeholders combinations are currently supported: `{date}`, `{date:yyyy_MM}`, `{timestamp}`, + `{timestamp:millis}`, `{timestamp:micros}`, `{part_number}`, `{sync_id}`, + `{format_extension}`. Please, don't use empty space and not supportable placeholders, as they + won't be recognized. 5. Click `Set up destination`. @@ -62,38 +83,46 @@ NOTE: If the S3 cluster is not configured to use TLS, the connection to Amazon S **For Airbyte Open Source:** 1. Go to local Airbyte page. -2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new destination**. -3. On the destination setup page, select **S3** from the Destination type dropdown and enter a name for this connector. -4. Configure fields: - _ **Access Key Id** - _ See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key. - _ See [this](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html) on how to create a instanceprofile. - _ We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the staging bucket. - _ If the Access Key and Secret Access Key are not provided, the authentication will rely on the instanceprofile. - _ **Secret Access Key** - _ Corresponding key to the above key id. - _ Make sure your S3 bucket is accessible from the machine running Airbyte. - _ This depends on your networking setup. - _ You can check AWS S3 documentation with a tutorial on how to properly configure your S3's access [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-control-overview.html). - _ If you use instance profile authentication, make sure the role has permission to read/write on the bucket. - _ The easiest way to verify if Airbyte is able to connect to your S3 bucket is via the check connection tool in the UI. - _ **S3 Bucket Name** - _ See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket. - _ **S3 Bucket Path** - _ Subdirectory under the above bucket to sync the data into. - _ **S3 Bucket Region** - _ See [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) for all region codes. - _ **S3 Path Format** - _ Additional string format on how to store data under S3 Bucket Path. Default value is `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_`. - _ **S3 Endpoint** - _ Leave empty if using AWS S3, fill in S3 URL if using Minio S3. - - - **S3 Filename pattern** \* The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: {date}, {date:yyyy_MM}, {timestamp}, {timestamp:millis}, {timestamp:micros}, {part_number}, {sync_id}, {format_extension}. Please, don't use empty space and not supportable placeholders, as they won't recognized. +2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new + destination**. +3. On the destination setup page, select **S3** from the Destination type dropdown and enter a name + for this connector. +4. Configure fields: _ **Access Key Id** _ See + [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) + on how to generate an access key. _ See + [this](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html) + on how to create a instanceprofile. _ We recommend creating an Airbyte-specific user. This user + will require + [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) + to objects in the staging bucket. _ If the Access Key and Secret Access Key are not provided, the + authentication will rely on the instanceprofile. _ **Secret Access Key** _ Corresponding key to + the above key id. _ Make sure your S3 bucket is accessible from the machine running Airbyte. _ + This depends on your networking setup. _ You can check AWS S3 documentation with a tutorial on + how to properly configure your S3's access + [here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-control-overview.html). _ If + you use instance profile authentication, make sure the role has permission to read/write on the + bucket. _ The easiest way to verify if Airbyte is able to connect to your S3 bucket is via the + check connection tool in the UI. _ **S3 Bucket Name** _ See + [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to + create an S3 bucket. _ **S3 Bucket Path** _ Subdirectory under the above bucket to sync the data + into. _ **S3 Bucket Region** _ See + [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html#concepts-available-regions) + for all region codes. _ **S3 Path Format** _ Additional string format on how to store data under + S3 Bucket Path. Default value is `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_`. + _ **S3 Endpoint** _ Leave empty if using AWS S3, fill in S3 URL if using Minio S3. + + - **S3 Filename pattern** \* The pattern allows you to set the file-name format for the S3 + staging file(s), next placeholders combinations are currently supported: `{date}`, + `{date:yyyy_MM}`, `{timestamp}`, `{timestamp:millis}`, `{timestamp:micros}`, `{part_number}`, + `{sync_id}`, `{format_extension}`. Please, don't use empty space and not supportable + placeholders, as they won't recognized. 5. Click `Set up destination`. -In order for everything to work correctly, it is also necessary that the user whose "S3 Key Id" and "S3 Access Key" are used have access to both the bucket and its contents. Minimum required Policies to use: +In order for everything to work correctly, it is also necessary that the user whose "S3 Key Id" and +"S3 Access Key" are used have access to both the bucket and its contents. Minimum required Policies +to use: ```json { @@ -111,16 +140,14 @@ In order for everything to work correctly, it is also necessary that the user wh "s3:AbortMultipartUpload", "s3:GetBucketLocation" ], - "Resource": [ - "arn:aws:s3:::YOUR_BUCKET_NAME/*", - "arn:aws:s3:::YOUR_BUCKET_NAME" - ] + "Resource": ["arn:aws:s3:::YOUR_BUCKET_NAME/*", "arn:aws:s3:::YOUR_BUCKET_NAME"] } ] } ``` -The full path of the output data with the default S3 Path Format `${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_` is: +The full path of the output data with the default S3 Path Format +`${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_` is: ```text ///__. @@ -149,7 +176,8 @@ The rationales behind this naming pattern are: But it is possible to further customize by using the available variables to format the bucket path: -- `${NAMESPACE}`: Namespace where the stream comes from or configured by the connection namespace fields. +- `${NAMESPACE}`: Namespace where the stream comes from or configured by the connection namespace + fields. - `${STREAM_NAME}`: Name of the stream - `${YEAR}`: Year in which the sync was writing the output data in. - `${MONTH}`: Month in which the sync was writing the output data in. @@ -164,34 +192,45 @@ But it is possible to further customize by using the available variables to form Note: - Multiple `/` characters in the S3 path are collapsed into a single `/` character. -- If the output bucket contains too many files, the part id variable is using a `UUID` instead. It uses sequential ID otherwise. +- If the output bucket contains too many files, the part id variable is using a `UUID` instead. It + uses sequential ID otherwise. -Please note that the stream name may contain a prefix, if it is configured on the connection. -A data sync may create multiple files as the output files can be partitioned by size (targeting a size of 200MB compressed or lower) . +Please note that the stream name may contain a prefix, if it is configured on the connection. A data +sync may create multiple files as the output files can be partitioned by size (targeting a size of +200MB compressed or lower) . ## Supported sync modes -| Feature | Support | Notes | -| :----------------------------- | :-----: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Full Refresh Sync | ✅ | Warning: this mode deletes all previously synced data in the configured bucket path. | -| Incremental - Append Sync | ✅ | Warning: Airbyte provides at-least-once delivery. Depending on your source, you may see duplicated data. Learn more [here](/understanding-airbyte/connections/incremental-append#inclusive-cursors) | -| Incremental - Append + Deduped | ❌ | | -| Namespaces | ❌ | Setting a specific bucket path is equivalent to having separate namespaces. | +| Feature | Support | Notes | +| :----------------------------- | :-----: | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Full Refresh Sync | ✅ | Warning: this mode deletes all previously synced data in the configured bucket path. | +| Incremental - Append Sync | ✅ | Warning: Airbyte provides at-least-once delivery. Depending on your source, you may see duplicated data. Learn more [here](/using-airbyte/core-concepts/sync-modes/incremental-append#inclusive-cursors) | +| Incremental - Append + Deduped | ❌ | | +| Namespaces | ❌ | Setting a specific bucket path is equivalent to having separate namespaces. | -The Airbyte S3 destination allows you to sync data to AWS S3 or Minio S3. Each stream is written to its own directory under the bucket. +The Airbyte S3 destination allows you to sync data to AWS S3 or Minio S3. Each stream is written to +its own directory under the bucket. -⚠️ Please note that under "Full Refresh Sync" mode, data in the configured bucket and path will be wiped out before each sync. We recommend you to provision a dedicated S3 resource for this sync to prevent unexpected data deletion from misconfiguration. ⚠️ +⚠️ Please note that under "Full Refresh Sync" mode, data in the configured bucket and path will be +wiped out before each sync. We recommend you to provision a dedicated S3 resource for this sync to +prevent unexpected data deletion from misconfiguration. ⚠️ ## Supported Output schema -Each stream will be outputted to its dedicated directory according to the configuration. The complete datastore of each stream includes all the output files under that directory. You can think of the directory as equivalent of a Table in the database world. +Each stream will be outputted to its dedicated directory according to the configuration. The +complete datastore of each stream includes all the output files under that directory. You can think +of the directory as equivalent of a Table in the database world. - Under Full Refresh Sync mode, old output files will be purged before new files are created. -- Under Incremental - Append Sync mode, new output files will be added that only contain the new data. +- Under Incremental - Append Sync mode, new output files will be added that only contain the new + data. ### Avro -[Apache Avro](https://avro.apache.org/) serializes data in a compact binary format. Currently, the Airbyte S3 Avro connector always uses the [binary encoding](http://avro.apache.org/docs/current/spec.html#binary_encoding), and assumes that all data records follow the same schema. +[Apache Avro](https://avro.apache.org/) serializes data in a compact binary format. Currently, the +Airbyte S3 Avro connector always uses the +[binary encoding](http://avro.apache.org/docs/current/spec.html#binary_encoding), and assumes that +all data records follow the same schema. #### Configuration @@ -209,7 +248,9 @@ Here is the available compression codecs: - Range `[0, 9]`. Default to 6. - Level 0-3 are fast with medium compression. - Level 4-6 are fairly slow with high compression. - - Level 7-9 are like level 6 but use bigger dictionaries and have higher memory requirements. Unless the uncompressed size of the file exceeds 8 MiB, 16 MiB, or 32 MiB, it is waste of memory to use the presets 7, 8, or 9, respectively. + - Level 7-9 are like level 6 but use bigger dictionaries and have higher memory requirements. + Unless the uncompressed size of the file exceeds 8 MiB, 16 MiB, or 32 MiB, it is waste of + memory to use the presets 7, 8, or 9, respectively. - `zstandard` - Compression level - Range `[-5, 22]`. Default to 3. @@ -222,11 +263,17 @@ Here is the available compression codecs: #### Data schema -Under the hood, an Airbyte data stream in JSON schema is first converted to an Avro schema, then the JSON object is converted to an Avro record. Because the data stream can come from any data source, the JSON to Avro conversion process has arbitrary rules and limitations. Learn more about how source data is converted to Avro and the current limitations [here](https://docs.airbyte.com/understanding-airbyte/json-avro-conversion). +Under the hood, an Airbyte data stream in JSON schema is first converted to an Avro schema, then the +JSON object is converted to an Avro record. Because the data stream can come from any data source, +the JSON to Avro conversion process has arbitrary rules and limitations. Learn more about how source +data is converted to Avro and the current limitations +[here](https://docs.airbyte.com/understanding-airbyte/json-avro-conversion). ### CSV -Like most of the other Airbyte destination connectors, usually the output has three columns: a UUID, an emission timestamp, and the data blob. With the CSV output, it is possible to normalize \(flatten\) the data blob to multiple columns. +Like most of the other Airbyte destination connectors, usually the output has three columns: a UUID, +an emission timestamp, and the data blob. With the CSV output, it is possible to normalize +\(flatten\) the data blob to multiple columns. | Column | Condition | Description | | :-------------------- | :------------------------------------------------------------------------------------------------ | :----------------------------------------------------------------------- | @@ -259,11 +306,13 @@ With root level normalization, the output CSV is: | :------------------------------------- | :-------------------- | :-------- | :----------------------------------- | | `26d73cde-7eb1-4e1e-b7db-a4c03b4cf206` | 1622135805000 | 123 | `{ "first": "John", "last": "Doe" }` | -Output files can be compressed. The default option is GZIP compression. If compression is selected, the output filename will have an extra extension (GZIP: `.csv.gz`). +Output files can be compressed. The default option is GZIP compression. If compression is selected, +the output filename will have an extra extension (GZIP: `.csv.gz`). ### JSON Lines \(JSONL\) -[JSON Lines](https://jsonlines.org/) is a text format with one JSON per line. Each line has a structure as follows: +[JSON Lines](https://jsonlines.org/) is a text format with one JSON per line. Each line has a +structure as follows: ```json { @@ -301,7 +350,8 @@ They will be like this in the output file: { "_airbyte_ab_id": "0a61de1b-9cdd-4455-a739-93572c9a5f20", "_airbyte_emitted_at": "1631948170000", "_airbyte_data": { "user_id": 456, "name": { "first": "Jane", "last": "Roe" } } } ``` -Output files can be compressed. The default option is GZIP compression. If compression is selected, the output filename will have an extra extension (GZIP: `.jsonl.gz`). +Output files can be compressed. The default option is GZIP compression. If compression is selected, +the output filename will have an extra extension (GZIP: `.jsonl.gz`). ### Parquet @@ -318,13 +368,22 @@ The following configuration is available to configure the Parquet output: | `dictionary_page_size_kb` | integer | 1024 \(KB\) | **Dictionary Page Size** in KB. There is one dictionary page per column per row group when dictionary encoding is used. The dictionary page size works like the page size but for dictionary. | | `dictionary_encoding` | boolean | `true` | **Dictionary encoding**. This parameter controls whether dictionary encoding is turned on. | -These parameters are related to the `ParquetOutputFormat`. See the [Java doc](https://www.javadoc.io/doc/org.apache.parquet/parquet-hadoop/1.12.0/org/apache/parquet/hadoop/ParquetOutputFormat.html) for more details. Also see [Parquet documentation](https://parquet.apache.org/docs/file-format/configurations/) for their recommended configurations \(512 - 1024 MB block size, 8 KB page size\). +These parameters are related to the `ParquetOutputFormat`. See the +[Java doc](https://www.javadoc.io/doc/org.apache.parquet/parquet-hadoop/1.12.0/org/apache/parquet/hadoop/ParquetOutputFormat.html) +for more details. Also see +[Parquet documentation](https://parquet.apache.org/docs/file-format/configurations/) for their +recommended configurations \(512 - 1024 MB block size, 8 KB page size\). #### Data schema -Under the hood, an Airbyte data stream in JSON schema is first converted to an Avro schema, then the JSON object is converted to an Avro record, and finally the Avro record is outputted to the Parquet format. Because the data stream can come from any data source, the JSON to Avro conversion process has arbitrary rules and limitations. Learn more about how source data is converted to Avro and the current limitations [here](https://docs.airbyte.com/understanding-airbyte/json-avro-conversion). +Under the hood, an Airbyte data stream in JSON schema is first converted to an Avro schema, then the +JSON object is converted to an Avro record, and finally the Avro record is outputted to the Parquet +format. Because the data stream can come from any data source, the JSON to Avro conversion process +has arbitrary rules and limitations. Learn more about how source data is converted to Avro and the +current limitations [here](https://docs.airbyte.com/understanding-airbyte/json-avro-conversion). -In order for everything to work correctly, it is also necessary that the user whose "S3 Key Id" and "S3 Access Key" are used have access to both the bucket and its contents. Policies to use: +In order for everything to work correctly, it is also necessary that the user whose "S3 Key Id" and +"S3 Access Key" are used have access to both the bucket and its contents. Policies to use: ```json { @@ -333,10 +392,7 @@ In order for everything to work correctly, it is also necessary that the user wh { "Effect": "Allow", "Action": "s3:*", - "Resource": [ - "arn:aws:s3:::YOUR_BUCKET_NAME/*", - "arn:aws:s3:::YOUR_BUCKET_NAME" - ] + "Resource": ["arn:aws:s3:::YOUR_BUCKET_NAME/*", "arn:aws:s3:::YOUR_BUCKET_NAME"] } ] } @@ -346,6 +402,12 @@ In order for everything to work correctly, it is also necessary that the user wh | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------- | +| 0.5.8 | 2024-01-03 | [#33924](https://github.com/airbytehq/airbyte/pull/33924) | Add new ap-southeast-3 AWS region | +| 0.5.7 | 2023-12-28 | [#33788](https://github.com/airbytehq/airbyte/pull/33788) | Thread-safe fix for file part names | +| 0.5.6 | 2023-12-08 | [#33263](https://github.com/airbytehq/airbyte/pull/33263) | (incorrect filename format, do not use) Adopt java CDK version 0.7.0. | +| 0.5.5 | 2023-12-08 | [#33264](https://github.com/airbytehq/airbyte/pull/33264) | Update UI options with common defaults. | +| 0.5.4 | 2023-11-06 | [#32193](https://github.com/airbytehq/airbyte/pull/32193) | (incorrect filename format, do not use) Adopt java CDK version 0.4.1. | +| 0.5.3 | 2023-11-03 | [#32050](https://github.com/airbytehq/airbyte/pull/32050) | (incorrect filename format, do not use) Adopt java CDK version 0.4.0. This updates filenames to include a UUID. | | 0.5.1 | 2023-06-26 | [#27786](https://github.com/airbytehq/airbyte/pull/27786) | Fix build | | 0.5.0 | 2023-06-26 | [#27725](https://github.com/airbytehq/airbyte/pull/27725) | License Update: Elv2 | | 0.4.2 | 2023-06-21 | [#27555](https://github.com/airbytehq/airbyte/pull/27555) | Reduce image size | diff --git a/docs/integrations/destinations/snowflake-migrations.md b/docs/integrations/destinations/snowflake-migrations.md index 3076e9e9f77d..2da5fe727be8 100644 --- a/docs/integrations/destinations/snowflake-migrations.md +++ b/docs/integrations/destinations/snowflake-migrations.md @@ -1,4 +1,18 @@ # Snowflake Migration Guide +## Upgrading to 3.0.0 + +This version introduces [Destinations V2](/release_notes/upgrading_to_destinations_v2/#what-is-destinations-v2), which provides better error handling, incremental delivery of data for large syncs, and improved final table structures. To review the breaking changes, and how to upgrade, see [here](/release_notes/upgrading_to_destinations_v2/#quick-start-to-upgrading). These changes will likely require updates to downstream dbt / SQL models, which we walk through [here](/release_notes/upgrading_to_destinations_v2/#updating-downstream-transformations). Selecting `Upgrade` will upgrade **all** connections using this destination at their next sync. You can manually sync existing connections prior to the next scheduled sync to start the upgrade early. + +Worthy of specific mention, this version includes: + +- Per-record error handling +- Clearer table structure +- Removal of sub-tables for nested properties +- Removal of SCD tables + +Learn more about what's new in Destinations V2 [here](/understanding-airbyte/typing-deduping). + ## Upgrading to 2.0.0 + Snowflake no longer supports GCS/S3. Please migrate to the Internal Staging option. This is recommended by Snowflake and is cheaper and faster. diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index 5a80360e48ae..aad1612b10c8 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -1,6 +1,6 @@ # Snowflake -Setting up the Snowflake destination connector involves setting up Snowflake entities (warehouse, database, schema, user, and role) in the Snowflake console, setting up the data loading method (internal stage, AWS S3, or Google Cloud Storage bucket), and configuring the Snowflake destination connector using the Airbyte UI. +Setting up the Snowflake destination connector involves setting up Snowflake entities (warehouse, database, schema, user, and role) in the Snowflake console and configuring the Snowflake destination connector using the Airbyte UI. This page describes the step-by-step process of setting up the Snowflake destination connector. @@ -19,11 +19,15 @@ To determine whether a network policy is set on your account or for a specific u **Account** - SHOW PARAMETERS LIKE 'network_policy' IN ACCOUNT; +``` +SHOW PARAMETERS LIKE 'network_policy' IN ACCOUNT; +``` **User** - SHOW PARAMETERS LIKE 'network_policy' IN USER ; +``` +SHOW PARAMETERS LIKE 'network_policy' IN USER ; +``` To read more please check official [Snowflake documentation](https://docs.snowflake.com/en/user-guide/network-policies.html#) @@ -113,64 +117,32 @@ You can use the following script in a new [Snowflake worksheet](https://docs.sno ### Step 2: Set up a data loading method -By default, Airbyte uses Snowflake’s [Internal Stage](https://docs.snowflake.com/en/user-guide/data-load-local-file-system-create-stage.html) to load data. You can also load data using an [Amazon S3 bucket](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html), or [Google Cloud Storage bucket](https://cloud.google.com/storage/docs/introduction). +Airbyte uses Snowflake’s [Internal Stage](https://docs.snowflake.com/en/user-guide/data-load-local-file-system-create-stage.html) to load data. Make sure the database and schema have the `USAGE` privilege. -#### Using an Amazon S3 bucket - -To use an Amazon S3 bucket, [create a new Amazon S3 bucket](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) with read/write access for Airbyte to stage data to Snowflake. - -#### Using a Google Cloud Storage bucket - -To use a Google Cloud Storage bucket: - -1. Navigate to the Google Cloud Console and [create a new bucket](https://cloud.google.com/storage/docs/creating-buckets) with read/write access for Airbyte to stage data to Snowflake. -2. [Generate a JSON key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys) for your service account. -3. Edit the following script to replace `AIRBYTE_ROLE` with the role you used for Airbyte's Snowflake configuration and `YOURBUCKETNAME` with your bucket name. - - ```text - create storage INTEGRATION gcs_airbyte_integration - TYPE = EXTERNAL_STAGE - STORAGE_PROVIDER = GCS - ENABLED = TRUE - STORAGE_ALLOWED_LOCATIONS = ('gcs://YOURBUCKETNAME'); - - create stage gcs_airbyte_stage - url = 'gcs://YOURBUCKETNAME' - storage_integration = gcs_airbyte_integration; - - GRANT USAGE ON integration gcs_airbyte_integration TO ROLE AIRBYTE_ROLE; - GRANT USAGE ON stage gcs_airbyte_stage TO ROLE AIRBYTE_ROLE; - - DESC STORAGE INTEGRATION gcs_airbyte_integration; - ``` - - The final query should show a `STORAGE_GCP_SERVICE_ACCOUNT` property with an email as the property value. Add read/write permissions to your bucket with that email. - -4. Navigate to the Snowflake UI and run the script as a [Snowflake account admin](https://docs.snowflake.com/en/user-guide/security-access-control-considerations.html) using the [Worksheet page](https://docs.snowflake.com/en/user-guide/ui-worksheet.html) or [Snowsight](https://docs.snowflake.com/en/user-guide/ui-snowsight-gs.html). - ### Step 3: Set up Snowflake as a destination in Airbyte Navigate to the Airbyte UI to set up Snowflake as a destination. You can authenticate using username/password or OAuth 2.0: ### Login and Password -| Field | Description | -| ----------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [Host](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html) | The host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com). Example: `accountname.us-east-2.aws.snowflakecomputing.com` | -| [Role](https://docs.snowflake.com/en/user-guide/security-access-control-overview.html#roles) | The role you created in Step 1 for Airbyte to access Snowflake. Example: `AIRBYTE_ROLE` | -| [Warehouse](https://docs.snowflake.com/en/user-guide/warehouses-overview.html#overview-of-warehouses) | The warehouse you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_WAREHOUSE` | -| [Database](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl) | The database you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_DATABASE` | -| [Schema](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl) | The default schema used as the target schema for all statements issued from the connection that do not explicitly specify a schema name. | -| Username | The username you created in Step 1 to allow Airbyte to access the database. Example: `AIRBYTE_USER` | -| Password | The password associated with the username. | -| [JDBC URL Params](https://docs.snowflake.com/en/user-guide/jdbc-parameters.html) (Optional) | Additional properties to pass to the JDBC URL string when connecting to the database formatted as `key=value` pairs separated by the symbol `&`. Example: `key1=value1&key2=value2&key3=value3` | +| Field | Description | +|-------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| [Host](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html) | The host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com). Example: `accountname.us-east-2.aws.snowflakecomputing.com` | +| [Role](https://docs.snowflake.com/en/user-guide/security-access-control-overview.html#roles) | The role you created in Step 1 for Airbyte to access Snowflake. Example: `AIRBYTE_ROLE` | +| [Warehouse](https://docs.snowflake.com/en/user-guide/warehouses-overview.html#overview-of-warehouses) | The warehouse you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_WAREHOUSE` | +| [Database](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl) | The database you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_DATABASE` | +| [Schema](https://docs.snowflake.com/en/sql-reference/ddl-database.html#database-schema-share-ddl) | The default schema used as the target schema for all statements issued from the connection that do not explicitly specify a schema name. | +| Username | The username you created in Step 1 to allow Airbyte to access the database. Example: `AIRBYTE_USER` | +| Password | The password associated with the username. | +| [JDBC URL Params](https://docs.snowflake.com/en/user-guide/jdbc-parameters.html) (Optional) | Additional properties to pass to the JDBC URL string when connecting to the database formatted as `key=value` pairs separated by the symbol `&`. Example: `key1=value1&key2=value2&key3=value3` | +| Disable Final Tables (Optional) | Disables writing final Typed tables See [output schema](#output-schema). WARNING! The data format in \_airbyte_data is likely stable but there are no guarantees that other metadata columns will remain the same in future versions | ### OAuth 2.0 | Field | Description | -| :---------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +|:------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | [Host](https://docs.snowflake.com/en/user-guide/admin-account-identifier.html) | The host domain of the snowflake instance (must include the account, region, cloud environment, and end with snowflakecomputing.com). Example: `accountname.us-east-2.aws.snowflakecomputing.com` | | [Role](https://docs.snowflake.com/en/user-guide/security-access-control-overview.html#roles) | The role you created in Step 1 for Airbyte to access Snowflake. Example: `AIRBYTE_ROLE` | | [Warehouse](https://docs.snowflake.com/en/user-guide/warehouses-overview.html#overview-of-warehouses) | The warehouse you created in Step 1 for Airbyte to sync data into. Example: `AIRBYTE_WAREHOUSE` | @@ -202,41 +174,44 @@ Navigate to the Airbyte UI to set up Snowflake as a destination. You can authent `alter user set rsa_public_key=;` - and replace with your user name and with your public key. - -To use AWS S3 as the cloud storage, enter the information for the S3 bucket you created in Step 2: - -| Field | Description | -| ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| S3 Bucket Name | The name of the staging S3 bucket (Example: `airbyte.staging`). Airbyte will write files to this bucket and read them via statements on Snowflake. | -| S3 Bucket Region | The S3 staging bucket region used. | -| S3 Key Id \* | The Access Key ID granting access to the S3 staging bucket. Airbyte requires Read and Write permissions for the bucket. | -| S3 Access Key \* | The corresponding secret to the S3 Key ID. | -| Stream Part Size (Optional) | Increase this if syncing tables larger than 100GB. Files are streamed to S3 in parts. This determines the size of each part, in MBs. As S3 has a limit of 10,000 parts per file, part size affects the table size. This is 5MB by default, resulting in a default limit of 100GB tables.
      Note, a larger part size will result in larger memory requirements. A rule of thumb is to multiply the part size by 10 to get the memory requirement. Modify this with care. (e.g. 5) | -| Purge Staging Files and Tables | Determines whether to delete the staging files from S3 after completing the sync. Specifically, the connector will create CSV files named `bucketPath/namespace/streamName/syncDate_epochMillis_randomUuid.csv` containing three columns (`ab_id`, `data`, `emitted_at`). Normally these files are deleted after sync; if you want to keep them for other purposes, set `purge_staging_data` to false. | -| Encryption | Whether files on S3 are encrypted. You probably don't need to enable this, but it can provide an additional layer of security if you are sharing your data storage with other applications. If you do use encryption, you must choose between ephemeral keys (Airbyte will automatically generate a new key for each sync, and nobody but Airbyte and Snowflake will be able to read the data on S3) or providing your own key (if you have the "Purge staging files and tables" option disabled, and you want to be able to decrypt the data yourself) | -| S3 Filename pattern (Optional) | The pattern allows you to set the file-name format for the S3 staging file(s), next placeholders combinations are currently supported: {date}, {date:yyyy_MM}, {timestamp}, {timestamp:millis}, {timestamp:micros}, {part_number}, {sync_id}, {format_extension}. Please, don't use empty space and not supportable placeholders, as they won't recognized. | + and replace `` with your user name and `` with your public key. -To use a Google Cloud Storage bucket, enter the information for the bucket you created in Step 2: +## Output schema -| Field | Description | -| ------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| GCP Project ID | The name of the GCP project ID for your credentials. (Example: `my-project`) | -| GCP Bucket Name | The name of the staging bucket. Airbyte will write files to this bucket and read them via statements on Snowflake. (Example: `airbyte-staging`) | -| Google Application Credentials | The contents of the JSON key file that has read/write permissions to the staging GCS bucket. You will separately need to grant bucket access to your Snowflake GCP service account. See the [Google Cloud docs](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys) for more information on how to generate a JSON key for your service account. | +Airbyte outputs each stream into its own raw table in `airbyte_internal` schema by default (can be overriden by user) and a final table with Typed columns. Contents in raw table are _NOT_ deduplicated. -## Output schema +### Raw Table schema -Airbyte outputs each stream into its own table with the following columns in Snowflake: +| Airbyte field | Description | Column type | +|------------------------|--------------------------------------------------------------------|--------------------------| +| \_airbyte_raw_id | A UUID assigned to each processed event | VARCHAR | +| \_airbyte_extracted_at | A timestamp for when the event was pulled from the data source | TIMESTAMP WITH TIME ZONE | +| \_airbyte_loaded_at | Timestamp to indicate when the record was loaded into Typed tables | TIMESTAMP WITH TIME ZONE | +| \_airbyte_data | A JSON blob with the event data. | VARIANT | -| Airbyte field | Description | Column type | -| -------------------- | -------------------------------------------------------------- | ------------------------ | -| \_airbyte_ab_id | A UUID assigned to each processed event | VARCHAR | -| \_airbyte_emitted_at | A timestamp for when the event was pulled from the data source | TIMESTAMP WITH TIME ZONE | -| \_airbyte_data | A JSON blob with the event data. | VARIANT | +**Note:** Although the contents of the `_airbyte_data` are fairly stable, schema of the raw table could be subject to change in future versions. **Note:** By default, Airbyte creates permanent tables. If you prefer transient tables, create a dedicated transient database for Airbyte. For more information, refer to[ Working with Temporary and Transient Tables](https://docs.snowflake.com/en/user-guide/tables-temp-transient.html) +## Data type map + +| Airbyte type | Snowflake type | +|:------------------------------------|:---------------| +| STRING | TEXT | +| STRING (BASE64) | TEXT | +| STRING (BIG_NUMBER) | TEXT | +| STRING (BIG_INTEGER) | TEXT | +| NUMBER | FLOAT | +| INTEGER | NUMBER | +| BOOLEAN | BOOLEAN | +| STRING (TIMESTAMP_WITH_TIMEZONE) | TIMESTAMP_TZ | +| STRING (TIMESTAMP_WITHOUT_TIMEZONE) | TIMESTAMP_NTZ | +| STRING (TIME_WITH_TIMEZONE) | TEXT | +| STRING (TIME_WITHOUT_TIMEZONE) | TIME | +| DATE | DATE | +| OBJECT | OBJECT | +| ARRAY | ARRAY | + ## Supported sync modes The Snowflake destination supports the following sync modes: @@ -271,6 +246,64 @@ Otherwise, make sure to grant the role the required permissions in the desired n | Version | Date | Pull Request | Subject | |:----------------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.4.22 | 2024-01-12 | [34227](https://github.com/airbytehq/airbyte/pull/34227) | Upgrade CDK to 0.12.0; Cleanup unused dependencies | +| 3.4.21 | 2024-01-10 | [\#34083](https://github.com/airbytehq/airbte/pull/34083) | Emit destination stats as part of the state message | +| 3.4.20 | 2024-01-05 | [\#33948](https://github.com/airbytehq/airbyte/pull/33948) | Skip retrieving initial table state when setup fails | +| 3.4.19 | 2024-01-04 | [\#33730](https://github.com/airbytehq/airbyte/pull/33730) | Internal code structure changes | +| 3.4.18 | 2024-01-02 | [\#33728](https://github.com/airbytehq/airbyte/pull/33728) | Add option to only type and dedupe at the end of the sync | +| 3.4.17 | 2023-12-20 | [\#33704](https://github.com/airbytehq/airbyte/pull/33704) | Update to java CDK 0.10.0 (no changes) | +| 3.4.16 | 2023-12-18 | [\#33124](https://github.com/airbytehq/airbyte/pull/33124) | Make Schema Creation Seperate from Table Creation | +| 3.4.15 | 2023-12-13 | [\#33232](https://github.com/airbytehq/airbyte/pull/33232) | Only run typing+deduping for a stream if the stream had any records | +| 3.4.14 | 2023-12-08 | [\#33263](https://github.com/airbytehq/airbyte/pull/33263) | Adopt java CDK version 0.7.0 | +| 3.4.13 | 2023-12-05 | [\#32326](https://github.com/airbytehq/airbyte/pull/32326) | Use jdbc metadata for table existence check | +| 3.4.12 | 2023-12-04 | [\#33084](https://github.com/airbytehq/airbyte/pull/33084) | T&D SQL statements moved to debug log level | +| 3.4.11 | 2023-11-14 | [\#32526](https://github.com/airbytehq/airbyte/pull/32526) | Clean up memory manager logs. | +| 3.4.10 | 2023-11-08 | [\#32125](https://github.com/airbytehq/airbyte/pull/32125) | Fix compilation warnings. | +| 3.4.9 | 2023-11-06 | [\#32026](https://github.com/airbytehq/airbyte/pull/32026) | Add separate TRY_CAST transaction to reduce compute usage | +| 3.4.8 | 2023-11-06 | [\#32190](https://github.com/airbytehq/airbyte/pull/32190) | Further improve error reporting | +| 3.4.7 | 2023-11-06 | [\#32193](https://github.com/airbytehq/airbyte/pull/32193) | Adopt java CDK version 0.4.1. | +| 3.4.6 | 2023-11-02 | [\#32124](https://github.com/airbytehq/airbyte/pull/32124) | Revert `merge` statement | +| 3.4.5 | 2023-11-02 | [\#31983](https://github.com/airbytehq/airbyte/pull/31983) | Improve error reporting | +| 3.4.4 | 2023-10-30 | [\#31985](https://github.com/airbytehq/airbyte/pull/31985) | Delay upgrade deadline to Nov 7 | +| 3.4.3 | 2023-10-30 | [\#31960](https://github.com/airbytehq/airbyte/pull/31960) | Adopt java CDK version 0.2.0. | +| 3.4.2 | 2023-10-27 | [\#31897](https://github.com/airbytehq/airbyte/pull/31897) | Further filtering on extracted_at | +| 3.4.1 | 2023-10-27 | [\#31683](https://github.com/airbytehq/airbyte/pull/31683) | Performance enhancement (switch to a `merge` statement for incremental-dedup syncs) | +| 3.4.0 | 2023-10-25 | [\#31686](https://github.com/airbytehq/airbyte/pull/31686) | Opt out flag for typed and deduped tables | +| 3.3.0 | 2023-10-25 | [\#31520](https://github.com/airbytehq/airbyte/pull/31520) | Stop deduping raw table | +| 3.2.3 | 2023-10-17 | [\#31191](https://github.com/airbytehq/airbyte/pull/31191) | Improve typing+deduping performance by filtering new raw records on extracted_at | +| 3.2.2 | 2023-10-10 | [\#31194](https://github.com/airbytehq/airbyte/pull/31194) | Deallocate unused per stream buffer memory when empty | +| 3.2.1 | 2023-10-10 | [\#31083](https://github.com/airbytehq/airbyte/pull/31083) | Fix precision of numeric values in async destinations | +| 3.2.0 | 2023-10-09 | [\#31149](https://github.com/airbytehq/airbyte/pull/31149) | No longer fail syncs when PKs are null - try do dedupe anyway | +| 3.1.22 | 2023-10-06 | [\#31153](https://github.com/airbytehq/airbyte/pull/31153) | Increase jvm GC retries | +| 3.1.21 | 2023-10-06 | [\#31139](https://github.com/airbytehq/airbyte/pull/31139) | Bump CDK version | +| 3.1.20 | 2023-10-06 | [\#31129](https://github.com/airbytehq/airbyte/pull/31129) | Reduce async buffer size | +| 3.1.19 | 2023-10-04 | [\#31082](https://github.com/airbytehq/airbyte/pull/31082) | Revert null PK checks | +| 3.1.18 | 2023-10-01 | [\#30779](https://github.com/airbytehq/airbyte/pull/30779) | Final table PK columns become non-null and skip check for null PKs in raw records (performance) | +| 3.1.17 | 2023-09-29 | [\#30938](https://github.com/airbytehq/airbyte/pull/30938) | Upgrade snowflake-jdbc driver | +| 3.1.16 | 2023-09-28 | [\#30835](https://github.com/airbytehq/airbyte/pull/30835) | Fix regression from 3.1.15 in supporting concurrent syncs with identical stream name but different namespace | +| 3.1.15 | 2023-09-26 | [\#30775](https://github.com/airbytehq/airbyte/pull/30775) | Increase async block size | +| 3.1.14 | 2023-09-27 | [\#30739](https://github.com/airbytehq/airbyte/pull/30739) | Fix column name collision detection | +| 3.1.13 | 2023-09-19 | [\#30599](https://github.com/airbytehq/airbyte/pull/30599) | Support concurrent syncs with identical stream name but different namespace | +| 3.1.12 | 2023-09-21 | [\#30671](https://github.com/airbytehq/airbyte/pull/30671) | Reduce async buffer size | +| 3.1.11 | 2023-09-19 | [\#30592](https://github.com/airbytehq/airbyte/pull/30592) | Internal code changes | +| 3.1.10 | 2023-09-18 | [\#30546](https://github.com/airbytehq/airbyte/pull/30546) | Make sure that the async buffer are flush every 5 minutes | +| 3.1.9 | 2023-09-19 | [\#30319](https://github.com/airbytehq/airbyte/pull/30319) | Support column names that are reserved | +| 3.1.8 | 2023-09-18 | [\#30479](https://github.com/airbytehq/airbyte/pull/30479) | Fix async memory management | +| 3.1.7 | 2023-09-15 | [\#30491](https://github.com/airbytehq/airbyte/pull/30491) | Improve error message display | +| 3.1.6 | 2023-09-14 | [\#30439](https://github.com/airbytehq/airbyte/pull/30439) | Fix a transient error | +| 3.1.5 | 2023-09-13 | [\#30416](https://github.com/airbytehq/airbyte/pull/30416) | Support `${` in stream name/namespace, and in column names | +| 3.1.4 | 2023-09-12 | [\#30364](https://github.com/airbytehq/airbyte/pull/30364) | Add log message | +| 3.1.3 | 2023-08-29 | [\#29878](https://github.com/airbytehq/airbyte/pull/29878) | Reenable incremental typing and deduping | +| 3.1.2 | 2023-08-31 | [\#30020](https://github.com/airbytehq/airbyte/pull/30020) | Run typing and deduping tasks in parallel | +| 3.1.1 | 2023-09-05 | [\#30117](https://github.com/airbytehq/airbyte/pull/30117) | Type and Dedupe at sync start and then every 6 hours | +| 3.1.0 | 2023-09-01 | [\#30056](https://github.com/airbytehq/airbyte/pull/30056) | Upcase final table names to allow case-insensitive references | +| 3.0.2 | 2023-09-01 | [\#30121](https://github.com/airbytehq/airbyte/pull/30121) | Improve performance on very wide streams by skipping TRY_CAST on strings | +| 3.0.1 | 2023-08-27 | [\#30065](https://github.com/airbytehq/airbyte/pull/30065) | Clearer error thrown when records are missing a primary key | +| 3.0.0 | 2023-08-27 | [\#29783](https://github.com/airbytehq/airbyte/pull/29783) | Destinations V2 | +| 2.1.7 | 2023-08-29 | [\#29949](https://github.com/airbytehq/airbyte/pull/29949) | Destinations V2: Fix checking for empty table by ensuring upper-case DB names | +| 2.1.6 | 2023-08-28 | [\#29878](https://github.com/airbytehq/airbyte/pull/29878) | Destinations V2: Fix detection of existing table by ensuring upper-case DB names | +| 2.1.5 | 2023-08-28 | [\#29903](https://github.com/airbytehq/airbyte/pull/29917) | Destinations V2: Performance Improvement, Changing Metadata error array construction from ARRAY_CAT to ARRAY_CONSTRUCT_COMPACT | +| 2.1.4 | 2023-08-28 | [\#29903](https://github.com/airbytehq/airbyte/pull/29903) | Abort queries on crash | | 2.1.3 | 2023-08-25 | [\#29881](https://github.com/airbytehq/airbyte/pull/29881) | Destinations v2: Only run T+D once at end of sync, to prevent data loss under async conditions | | 2.1.2 | 2023-08-24 | [\#29805](https://github.com/airbytehq/airbyte/pull/29805) | Destinations v2: Don't soft reset in migration | | 2.1.1 | 2023-08-23 | [\#29774](https://github.com/airbytehq/airbyte/pull/29774) | Destinations v2: Don't soft reset overwrite syncs | @@ -370,4 +403,4 @@ Otherwise, make sure to grant the role the required permissions in the desired n | 0.3.13 | 2021-09-01 | [\#5784](https://github.com/airbytehq/airbyte/pull/5784) | Updated query timeout from 30 minutes to 3 hours | | 0.3.12 | 2021-07-30 | [\#5125](https://github.com/airbytehq/airbyte/pull/5125) | Enable `additionalPropertities` in spec.json | | 0.3.11 | 2021-07-21 | [\#3555](https://github.com/airbytehq/airbyte/pull/3555) | Partial Success in BufferedStreamConsumer | -| 0.3.10 | 2021-07-12 | [\#4713](https://github.com/airbytehq/airbyte/pull/4713) | Tag traffic with `airbyte` label to enable optimization opportunities from Snowflake | +| 0.3.10 | 2021-07-12 | [\#4713](https://github.com/airbytehq/airbyte/pull/4713) | Tag traffic with `airbyte` label to enable optimization opportunities from Snowflake | \ No newline at end of file diff --git a/docs/integrations/destinations/sqlite.md b/docs/integrations/destinations/sqlite.md index eb266b61eee8..f5c2a3193780 100644 --- a/docs/integrations/destinations/sqlite.md +++ b/docs/integrations/destinations/sqlite.md @@ -68,7 +68,7 @@ You can also copy the output file to your host machine, the following command wi docker cp airbyte-server:/tmp/airbyte_local/{destination_path} . ``` -Note: If you are running Airbyte on Windows with Docker backed by WSL2, you have to use similar step as above or refer to this [link](../../operator-guides/locating-files-local-destination.md) for an alternative approach. +Note: If you are running Airbyte on Windows with Docker backed by WSL2, you have to use similar step as above or refer to this [link](/integrations/locating-files-local-destination.md) for an alternative approach. ## Changelog diff --git a/docs/integrations/destinations/teradata.md b/docs/integrations/destinations/teradata.md index 213f4d0692da..74f2b89fd598 100644 --- a/docs/integrations/destinations/teradata.md +++ b/docs/integrations/destinations/teradata.md @@ -42,6 +42,8 @@ following[ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-s | Incremental - Append + Deduped | No | | | Namespaces | Yes | | +The Teradata destination connector supports [ DBT custom transformation](https://docs.airbyte.com/operator-guides/transformation-and-normalization/transformations-with-airbyte/) type. Teradata DBT Docker image is available at https://hub.docker.com/r/teradata/dbt-teradata. + ### Performance considerations ## Getting started @@ -84,6 +86,7 @@ You can also use a pre-existing user but we highly recommend creating a dedicate | Version | Date | Pull Request | Subject | | :------ | :--------- | :---------------------------------------------- | :------------------------------- | +| 0.1.3 | 2023-08-17 | https://github.com/airbytehq/airbyte/pull/30740 | Enable custom DBT transformation | | 0.1.2 | 2023-08-09 | https://github.com/airbytehq/airbyte/pull/29174 | Small internal refactor | | 0.1.1 | 2023-03-03 | https://github.com/airbytehq/airbyte/pull/21760 | Added SSL support | | 0.1.0 | 2022-12-13 | https://github.com/airbytehq/airbyte/pull/20428 | New Destination Teradata Vantage | diff --git a/docs/integrations/destinations/timeplus.md b/docs/integrations/destinations/timeplus.md index dcf43cc48225..d883fc1b3726 100644 --- a/docs/integrations/destinations/timeplus.md +++ b/docs/integrations/destinations/timeplus.md @@ -16,7 +16,7 @@ Each stream will be output into its own stream in Timeplus, with corresponding s ## Getting Started (Airbyte Cloud) Coming soon... -## Getting Started (Airbyte Open-Source) +## Getting Started (Airbyte Open Source) You can follow the [Quickstart with Timeplus Ingestion API](https://docs.timeplus.com/quickstart-ingest-api) to createa a workspace and API key. ### Setup the Timeplus Destination in Airbyte diff --git a/docs/integrations/destinations/typesense.md b/docs/integrations/destinations/typesense.md index f9dfff351c60..7185a69013df 100644 --- a/docs/integrations/destinations/typesense.md +++ b/docs/integrations/destinations/typesense.md @@ -37,5 +37,6 @@ The setup only requires two fields. First is the `host` which is the address at | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :---------------------------- | +| 0.1.2 | 2023-08-25 | [29817](https://github.com/airbytehq/airbyte/pull/29817) | Fix writing multiple streams | +| 0.1.1 | 2023-08-24 | [29555](https://github.com/airbytehq/airbyte/pull/29555) | Increasing connection timeout | | 0.1.0 | 2022-10-28 | [18349](https://github.com/airbytehq/airbyte/pull/18349) | New Typesense destination | -| 0.1.1 | 2023-22-17 | [29555](https://github.com/airbytehq/airbyte/pull/29555) | Increasing connection timeout | diff --git a/docs/integrations/destinations/vectara.md b/docs/integrations/destinations/vectara.md new file mode 100644 index 000000000000..19d19baa7402 --- /dev/null +++ b/docs/integrations/destinations/vectara.md @@ -0,0 +1,65 @@ +# Vectara + +This page contains the setup guide and reference information for the Vectara destination connector. + +[Vectara](https://vectara.com/) is the trusted GenAI platform that provides Retrieval Augmented Generation or [RAG](https://vectara.com/grounded-generation/) as a service. + +The Vectara destination connector allows you to connect any Airbyte source to Vectara and ingest data into Vectara for your RAG pipeline. + +:::info +In case of issues, the following public channels are available for support: + +* For Airbyte related issues such as data source or processing: [Open a Github issue](https://github.com/airbytehq/airbyte/issues/new?assignees=&labels=type%2Fbug%2Carea%2Fconnectors%2Cneeds-triage&projects=&template=1-issue-connector.yaml) +* For Vectara related issues such as data indexing or RAG: Create a post in the [Vectara forum](https://discuss.vectara.com/) or reach out on [Vectara's Discord server](https://discord.gg/GFb8gMz6UH) + +::: + +## Overview + +The Vectara destination connector supports Full Refresh Overwrite, Full Refresh Append, and Incremental Append. + +### Output schema + +All streams will be output into a corpus in Vectara whose name must be specified in the config. + +Note that there are no restrictions in naming the Vectara corpus and if a corpus with the specified name is not found, a new corpus with that name will be created. Also, if multiple corpora exists with the same name, an error will be returned as Airbyte will be unable to determine the prefered corpus. + +### Features + +| Feature | Supported? | +| :---------------------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | Yes | +| Incremental - Dedupe Sync | Yes | + +## Getting started + +You will need a Vectara account to use Vectara with Airbyte. To get started, use the following steps: +1. [Sign up](https://vectara.com/integrations/airbyte) for a Vectara account if you don't already have one. Once you have completed your sign up you will have a Vectara customer ID. You can find your customer ID by clicking on your name, on the top-right of the Vectara console window. +2. Within your account you can create your corpus, which represents an area that stores text data you want to ingest into Vectara. + * To create a corpus, use the **"Create Corpus"** button in the console. You then provide a name to your corpus as well as a description. If you click on your created corpus, you can see its name and corpus ID right on the top. You can see more details in this [guide](https://docs.vectara.com/docs/console-ui/creating-a-corpus). + * Optionally you can define filtering attributes and apply some advanced options. + * For the Vectara connector to work properly you **must** define a special meta-data field called `_ab_stream` (string typed) which the connector uses to identify source streams. +3. The Vectara destination connector uses [OAuth2.0 Credentials](https://docs.vectara.com/docs/learn/authentication/oauth-2). You will need your `Client ID` and `Client Secret` handy for your connector setup. + +### Setup the Vectara Destination in Airbyte + +You should now have all the requirements needed to configure Vectara as a destination in the UI. + +You'll need the following information to configure the Vectara destination: + +- (Required) OAuth2.0 Credentials + - (Required) **Client ID** + - (Required) **Client Secret** +- (Required) **Customer ID** +- (Required) **Corpus Name**. You can specify a corpus name you've setup manually given the instructions above, or if you specify a corpus name that does not exist, the connector will generate a new corpus in this name and setup the required meta-data filtering fields within that corpus. + +In addition, in the connector UI you define two set of fields for this connector: +* `text_fields` define the source fields which are turned into text in the Vectara side and are used for query or summarization. +* `metadata_fields` define the source fields which will be added to each document as meta-data. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------- | +| 0.1.0 | 2023-11-10 | [31958](https://github.com/airbytehq/airbyte/pull/31958) | 🎉 New Destination: Vectara (Vector Database) | diff --git a/docs/integrations/destinations/weaviate-migrations.md b/docs/integrations/destinations/weaviate-migrations.md new file mode 100644 index 000000000000..1d54ca39f45f --- /dev/null +++ b/docs/integrations/destinations/weaviate-migrations.md @@ -0,0 +1,18 @@ +# Weaviate Migration Guide + +## Upgrading to 0.2.0 + +This version adds several new features like flexible embedding options, overwrite and append+dedup sync modes. When upgrading from prior versions on this connector, a one-time migration of existing connections is required. This is done to align the behavior of vector database destinations in Airbyte. The following changes are included: + +### Changed configuration object structure + +Due to a change of the configuration structure, it's necessary to reconfigure existing destinations with the same information (e.g. credentials). + +### Auto-generated ids + +It's no longer possible to configure `id` fields in the destination. Instead, the destination will generate a UUID for each Weaviate object. The `id` for each record is stored in the `_ab_record_id` property and can be used to identify Weaviate objects by Airbyte record. + +### Vector fields + +It's not possible anymore to configure separate vector fields per stream. To load embedding vectors from the records itself, the embedding method `From Field` can be used and configured with a single field name that has to be available in records from all streams. If your records contain multiple vector fields, you need to configure separate destinations and connections to configure separate vector field names. + diff --git a/docs/integrations/destinations/weaviate.md b/docs/integrations/destinations/weaviate.md index 3b54c45a0d6e..c8acb02b50de 100644 --- a/docs/integrations/destinations/weaviate.md +++ b/docs/integrations/destinations/weaviate.md @@ -1,80 +1,104 @@ # Weaviate +## Overview + +This page guides you through the process of setting up the [Weaviate](https://weaviate.io/) destination connector. + +There are three parts to this: +* Processing - split up individual records in chunks so they will fit the context window and decide which fields to use as context and which are supplementary metadata. +* Embedding - convert the text into a vector representation using a pre-trained model (Currently, OpenAI's `text-embedding-ada-002` and Cohere's `embed-english-light-v2.0` are supported.) +* Indexing - store the vectors in a vector database for similarity search + +## Prerequisites + +To use the Weaviate destination, you'll need: + +* Access to a running Weaviate instance (either self-hosted or via Weaviate Cloud Services), minimum version 1.21.2 +* Either + * An account with API access for OpenAI or Cohere (depending on which embedding method you want to use) + * Pre-calculated embeddings stored in a field in your source database + +You'll need the following information to configure the destination: + +- **Embedding service API Key** - The API key for your OpenAI or Cohere account +- **Weaviate cluster URL** - The URL of the Weaviate cluster to load data into. Airbyte Cloud only supports connecting to your Weaviate Instance instance with TLS encryption. +- **Weaviate credentials** - The credentials for your Weaviate instance (either API token or username/password) + ## Features | Feature | Supported?\(Yes/No\) | Notes | | :----------------------------- | :------------------- | :---- | -| Full Refresh Sync | No | | +| Full Refresh Sync | Yes | | | Incremental - Append Sync | Yes | | -| Incremental - Append + Deduped | No | | +| Incremental - Append + Deduped | Yes | | | Namespaces | No | | -| Provide vector | Yes | | +| Provide vector | Yes | Either from field are calculated during the load process | -#### Output Schema +## Data type mapping -Each stream will be output into its [own class](https://weaviate.io/developers/weaviate/current/core-knowledge/basics.html#class-collections) in [Weaviate](https://weaviate.io). The record fields will be stored as fields -in the Weaviate class. +All fields specified as metadata fields will be stored as properties in the object can be used for filtering. The following data types are allowed for metadata fields: +* String +* Number (integer or floating point, gets converted to a 64 bit floating point) +* Booleans (true, false) +* List of String -**Uploading Vectors:** Use the vectors configuration if you want to upload -vectors from a source database into Weaviate. You can do this by specifying -the stream name and vector field name in the following format: +All other fields are serialized into their JSON representation. -``` -., . -``` +## Configuration -For example, if you have a table named `my_table` and the vector is stored using the column `vector` then -you should use the following `vectors`configuration: `my_table.vector`. +### Processing -Dynamic Schema: Weaviate will automatically create a schema for the stream if no class was defined unless -you have disabled the Dynamic Schema feature in Weaviate. You can also create the class in Weaviate in advance -if you need more control over the schema in Weaviate. +Each record will be split into text fields and metadata fields as configured in the "Processing" section. All text fields are concatenated into a single string and then split into chunks of configured length. If specified, the metadata fields are stored as-is along with the embedded text chunks. Options around configuring the chunking process use the [Langchain Python library](https://python.langchain.com/docs/get_started/introduction). -IDs: If your source table has an int based id stored as field name `id` then the -ID will automatically be converted to a UUID. Weaviate only supports the ID to be a UUID. -For example, if the record has `id=1` then this would become a uuid of -`00000000-0000-0000-0000-000000000001`. +When specifying text fields, you can access nested fields in the record by using dot notation, e.g. `user.name` will access the `name` field in the `user` object. It's also possible to use wildcards to access all fields in an object, e.g. `users.*.name` will access all `names` fields in all entries of the `users` array. -Any field name starting with an upper case letter will be converted to lower case. For example, -if you have a field name `USD` then that field will become `uSD`. This is due to a limitation -in Weaviate, see [this issue in Weaviate](https://github.com/semi-technologies/weaviate/issues/2438). +The chunk length is measured in tokens produced by the `tiktoken` library. The maximum is 8191 tokens, which is the maximum length supported by the `text-embedding-ada-002` model. -## Getting Started +The stream name gets added as a metadata field `_ab_stream` to each document. If available, the primary key of the record is used to identify the document to avoid duplications when updated versions of records are indexed. It is added as the `_ab_record_id` metadata field. -Airbyte Cloud only supports connecting to your Weaviate Instance instance with TLS encryption and with a username and -password. +### Embedding -## Getting Started \(Airbyte Open-Source\) +The connector can use one of the following embedding methods: -#### Requirements +1. OpenAI - using [OpenAI API](https://beta.openai.com/docs/api-reference/text-embedding) , the connector will produce embeddings using the `text-embedding-ada-002` model with **1536 dimensions**. This integration will be constrained by the [speed of the OpenAI embedding API](https://platform.openai.com/docs/guides/rate-limits/overview). -To use the Weaviate destination, you'll need: +2. Cohere - using the [Cohere API](https://docs.cohere.com/reference/embed), the connector will produce embeddings using the `embed-english-light-v2.0` model with **1024 dimensions**. -- A Weaviate cluster version 21.8.10.19 or above +3. From field - if you have pre-calculated embeddings stored in a field in your source database, you can use the `From field` integration to load them into Weaviate. The field must be a JSON array of numbers, e.g. `[0.1, 0.2, 0.3]`. -#### Configure Network Access +4. No embedding - if you don't want to use embeddings or have configured a [vectorizer](https://weaviate.io/developers/weaviate/modules/retriever-vectorizer-modules) for your class, you can use the `No embedding` integration. -Make sure your Weaviate database can be accessed by Airbyte. If your database is within a VPC, you may need to allow access from the IP you're using to expose Airbyte. +For testing purposes, it's also possible to use the [Fake embeddings](https://python.langchain.com/docs/modules/data_connection/text_embedding/integrations/fake) integration. It will generate random embeddings and is suitable to test a data pipeline without incurring embedding costs. -#### **Permissions** +### Indexing -You need a Weaviate user or use a Weaviate instance that's accessible to all +All streams will be indexed into separate classes derived from the stream name. +If a class doesn't exist in the schema of the cluster, it will be created using the configure vectorizer configuration. In this case, dynamic schema has to be enabled on the server. -### Setup the Weaviate Destination in Airbyte +You can also create the class in Weaviate in advance if you need more control over the schema in Weaviate. In this case, the text properies `_ab_stream` and `_ab_record_id` need to be created for bookkeeping reasons. In case a sync is run in `Overwrite` mode, the class will be deleted and recreated. -You should now have all the requirements needed to configure Weaviate as a destination in the UI. You'll need the following information to configure the Weaviate destination: +As properties have to start will a lowercase letter in Weaviate and can't contain spaces or special characters. Field names might be updated during the loading process. The field names `id`, `_id` and `_additional` are reserved keywords in Weaviate, so they will be renamed to `raw_id`, `raw__id` and `raw_additional` respectively. -- **URL** for example http://localhost:8080 or https://my-wcs.semi.network -- **Username** (Optional) -- **Password** (Optional) -- **Batch Size** (Optional, defaults to 100) -- **Vectors** a comma separated list of `` to specify the field -- **ID Schema** a comma separated list of `` to specify the field - name that contains the ID of a record +When using [multi-tenancy](https://weaviate.io/developers/weaviate/manage-data/multi-tenancy), the tenant id can be configured in the connector configuration. If not specified, multi-tenancy will be disabled. In case you want to index into an already created class, you need to make sure the class is created with multi-tenancy enabled. In case the class doesn't exist, it will be created with multi-tenancy properly configured. If the class already exists but the tenant id is not associated with the class, the connector will automatically add the tenant id to the class. This allows you to configure multiple connections for different tenants on the same schema. ## Changelog | Version | Date | Pull Request | Subject | | :------ | :--------- | :--------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------- | +| 0.2.14 | 2023-01-15 | [34229](https://github.com/airbytehq/airbyte/pull/34229) | Allow configuring tenant id | +| 0.2.13 | 2023-12-11 | [33303](https://github.com/airbytehq/airbyte/pull/33303) | Fix bug with embedding special tokens | +| 0.2.12 | 2023-12-07 | [33218](https://github.com/airbytehq/airbyte/pull/33218) | Normalize metadata field names | +| 0.2.11 | 2023-12-01 | [32697](https://github.com/airbytehq/airbyte/pull/32697) | Allow omitting raw text | +| 0.2.10 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Support deleting records for CDC sources | +| 0.2.9 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | +| 0.2.8 | 2023-11-03 | [#32134](https://github.com/airbytehq/airbyte/pull/32134) | Improve test coverage | +| 0.2.7 | 2023-11-03 | [#32134](https://github.com/airbytehq/airbyte/pull/32134) | Upgrade weaviate client library | +| 0.2.6 | 2023-11-01 | [#32038](https://github.com/airbytehq/airbyte/pull/32038) | Retry failed object loads | +| 0.2.5 | 2023-10-24 | [#31953](https://github.com/airbytehq/airbyte/pull/31953) | Fix memory leak | +| 0.2.4 | 2023-10-23 | [#31563](https://github.com/airbytehq/airbyte/pull/31563) | Add field mapping option, improve append+dedupe sync performance and remove unnecessary retry logic | +| 0.2.3 | 2023-10-19 | [#31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 0.2.2 | 2023-10-15 | [#31329](https://github.com/airbytehq/airbyte/pull/31329) | Add OpenAI-compatible embedder option | +| 0.2.1 | 2023-10-04 | [#31075](https://github.com/airbytehq/airbyte/pull/31075) | Fix OpenAI embedder batch size and conflict field name handling | +| 0.2.0 | 2023-09-22 | [#30151](https://github.com/airbytehq/airbyte/pull/30151) | Add embedding capabilities, overwrite and dedup support and API key auth mode, make certified. 🚨 Breaking changes - check migrations guide. | | 0.1.1 | 2022-02-08 | [\#22527](https://github.com/airbytehq/airbyte/pull/22527) | Multiple bug fixes: Support String based IDs, arrays of uknown type and additionalProperties of type object and array of objects | -| 0.1.0 | 2022-12-06 | [\#20094](https://github.com/airbytehq/airbyte/pull/20094) | Add Weaviate destination | +| 0.1.0 | 2022-12-06 | [\#20094](https://github.com/airbytehq/airbyte/pull/20094) | Add Weaviate destination | \ No newline at end of file diff --git a/docs/integrations/getting-started/destination-redshift.md b/docs/integrations/getting-started/destination-redshift.md deleted file mode 100644 index ae59b0eeff95..000000000000 --- a/docs/integrations/getting-started/destination-redshift.md +++ /dev/null @@ -1,70 +0,0 @@ -# Getting Started: Destination Redshift - -## Requirements - -1. Active Redshift cluster -2. Allow connections from Airbyte to your Redshift cluster \(if they exist in separate VPCs\) -3. A staging S3 bucket with credentials \(for the COPY strategy\). - -## Setup guide - -### 1. Make sure your cluster is active and accessible from the machine running Airbyte - -This is dependent on your networking setup. The easiest way to verify if Airbyte is able to connect to your Redshift cluster is via the check connection tool in the UI. You can check AWS Redshift documentation with a tutorial on how to properly configure your cluster's access [here](https://docs.aws.amazon.com/redshift/latest/gsg/rs-gsg-authorize-cluster-access.html) - -### 2. Fill up connection info - -Next is to provide the necessary information on how to connect to your cluster such as the `host` whcih is part of the connection string or Endpoint accessible [here](https://docs.aws.amazon.com/redshift/latest/gsg/rs-gsg-connect-to-cluster.html#rs-gsg-how-to-get-connection-string) without the `port` and `database` name \(it typically includes the cluster-id, region and end with `.redshift.amazonaws.com`\). - -You should have all the requirements needed to configure Redshift as a destination in the UI. You'll need the following information to configure the destination: - -* **Host** -* **Port** -* **Username** -* **Password** -* **Schema** -* **Database** - * This database needs to exist within the cluster provided. - -### 2a. Fill up S3 info \(for COPY strategy\) - -Provide the required S3 info. - -* **S3 Bucket Name** - * See [this](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) to create an S3 bucket. -* **S3 Bucket Region** - * Place the S3 bucket and the Redshift cluster in the same region to save on networking costs. -* **Access Key Id** - * See [this](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) on how to generate an access key. - * We recommend creating an Airbyte-specific user. This user will require [read and write permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_examples_s3_rw-bucket.html) to objects in the staging bucket. -* **Secret Access Key** - * Corresponding key to the above key id. -* **Part Size** - * Affects the size limit of an individual Redshift table. Optional. Increase this if syncing tables larger than 100GB. Files are streamed to S3 in parts. This determines the size of each part, in MBs. As S3 has a limit of 10,000 parts per file, part size affects the table size. This is 10MB by default, resulting in a default table limit of 100GB. Note, a larger part size will result in larger memory requirements. A rule of thumb is to multiply the part size by 10 to get the memory requirement. Modify this with care. - -Optional parameters: -* **Bucket Path** - * The directory within the S3 bucket to place the staging data. For example, if you set this to `yourFavoriteSubdirectory`, staging data will be placed inside `s3://yourBucket/yourFavoriteSubdirectory`. If not provided, defaults to the root directory. - -## Notes about Redshift Naming Conventions - -From [Redshift Names & Identifiers](https://docs.aws.amazon.com/redshift/latest/dg/r_names.html): - -### Standard Identifiers - -* Begin with an ASCII single-byte alphabetic character or underscore character, or a UTF-8 multibyte character two to four bytes long. -* Subsequent characters can be ASCII single-byte alphanumeric characters, underscores, or dollar signs, or UTF-8 multibyte characters two to four bytes long. -* Be between 1 and 127 bytes in length, not including quotation marks for delimited identifiers. -* Contain no quotation marks and no spaces. - -### Delimited Identifiers - -Delimited identifiers \(also known as quoted identifiers\) begin and end with double quotation marks \("\). If you use a delimited identifier, you must use the double quotation marks for every reference to that object. The identifier can contain any standard UTF-8 printable characters other than the double quotation mark itself. Therefore, you can create column or table names that include otherwise illegal characters, such as spaces or the percent symbol. ASCII letters in delimited identifiers are case-insensitive and are folded to lowercase. To use a double quotation mark in a string, you must precede it with another double quotation mark character. - -Therefore, Airbyte Redshift destination will create tables and schemas using the Unquoted identifiers when possible or fallback to Quoted Identifiers if the names are containing special characters. - -## Data Size Limitations - -Redshift specifies a maximum limit of 65535 bytes to store the raw JSON record data. Thus, when a row is too big to fit, the Redshift destination fails to load such data and currently ignores that record. - -For more information, see the [docs here.](https://docs.aws.amazon.com/redshift/latest/dg/r_Character_types.html) diff --git a/docs/integrations/getting-started/source-facebook-marketing.md b/docs/integrations/getting-started/source-facebook-marketing.md deleted file mode 100644 index cb0303519372..000000000000 --- a/docs/integrations/getting-started/source-facebook-marketing.md +++ /dev/null @@ -1,42 +0,0 @@ -# Getting Started: Source Facebook Marketing - -## Requirements - -Google Ads Account with an approved Developer Token \(note: In order to get API access to Google Ads, you must have a "manager" account. This must be created separately from your standard account. You can find more information about this distinction in the [google ads docs](https://ads.google.com/home/tools/manager-accounts/).\) - -* developer_token -* client_id -* client_secret -* refresh_token -* start_date -* customer_id - -## Setup guide - -This guide will provide information as if starting from scratch. Please skip over any steps you have already completed. - -* Create an Google Ads Account. Here are [Google's instruction](https://support.google.com/google-ads/answer/6366720) on how to create one. -* Create an Google Ads MANAGER Account. Here are [Google's instruction](https://ads.google.com/home/tools/manager-accounts/) on how to create one. -* You should now have two Google Ads accounts: a normal account and a manager account. Link the Manager account to the normal account following [Google's documentation](https://support.google.com/google-ads/answer/7459601). -* Apply for a developer token \(**make sure you follow our** [**instructions**](#how-to-apply-for-the-developer-token)\) on your Manager account. This token allows you to access your data from the Google Ads API. Here are [Google's instructions](https://developers.google.com/google-ads/api/docs/first-call/dev-token). The docs are a little unclear on this point, but you will _not_ be able to access your data via the Google Ads API until this token is approved. You cannot use a test developer token, it has to be at least a basic developer token. It usually takes Google 24 hours to respond to these applications. This developer token is the value you will use in the `developer_token` field. -* Fetch your `client_id`, `client_secret`, and `refresh_token`. Google provides [instructions](https://developers.google.com/google-ads/api/docs/first-call/overview) on how to do this. -* Select your `customer_id`. The `customer_is` refer to the id of each of your Google Ads accounts. This is the 10 digit number in the top corner of the page when you are in google ads ui. The source will only pull data from the accounts for which you provide an id. If you are having trouble finding it, check out [Google's instructions](https://support.google.com/google-ads/answer/1704344). - -Wow! That was a lot of steps. We are working on making the OAuth flow for all of our connectors simpler \(allowing you to skip needing to get a `developer_token` and a `refresh_token` which are the most painful / time-consuming steps in this walkthrough\). - -## How to apply for the developer token - -Google is very picky about which software and which use case can get access to a developer token. The Airbyte team has worked with the Google Ads team to whitelist Airbyte and make sure you can get one \(see [issue 1981](https://github.com/airbytehq/airbyte/issues/1981) for more information\). - -When you apply for a token, you need to mention: - -* Why you need the token \(eg: want to run some internal analytics...\) -* That you will be using the Airbyte Open Source project -* That you have full access to the code base \(because we're open source\) -* That you have full access to the server running the code \(because you're self-hosting Airbyte\) - -If for any reason the request gets denied, let us know and we will be able to unblock you. - -## Understanding Google Ads Query Language - -The Google Ads Query Language can query the Google Ads API. Check out [Google Ads Query Language](https://developers.google.com/google-ads/api/docs/query/overview) diff --git a/docs/integrations/getting-started/source-github.md b/docs/integrations/getting-started/source-github.md deleted file mode 100644 index 6ae7f442aade..000000000000 --- a/docs/integrations/getting-started/source-github.md +++ /dev/null @@ -1,12 +0,0 @@ -## Getting Started: Source GitHub - -### Requirements - -* Github Account -* Github Personal Access Token wih the necessary permissions \(described below\) - -### Setup guide - -Log into Github and then generate a [personal access token](https://github.com/settings/tokens). - -Your token should have at least the `repo` scope. Depending on which streams you want to sync, the user generating the token needs more permissions: diff --git a/docs/operator-guides/locating-files-local-destination.md b/docs/integrations/locating-files-local-destination.md similarity index 98% rename from docs/operator-guides/locating-files-local-destination.md rename to docs/integrations/locating-files-local-destination.md index e514f3a92ebd..d401d7952455 100644 --- a/docs/operator-guides/locating-files-local-destination.md +++ b/docs/integrations/locating-files-local-destination.md @@ -1,3 +1,7 @@ +--- +displayed_sidebar: docs +--- + # Windows - Browsing Local File Output ## Overview diff --git a/docs/integrations/missing-an-integration.md b/docs/integrations/missing-an-integration.md deleted file mode 100644 index e52613182866..000000000000 --- a/docs/integrations/missing-an-integration.md +++ /dev/null @@ -1,14 +0,0 @@ -# Missing an Integration? - -If you'd like to ask for a new connector, or build a new connectors and make them part of the pool of pre-built connectors on Airbyte, first a big thank you. We invite you to check our [contributing guide](../contributing-to-airbyte/). - -If you'd like to build new connectors, or update existing ones, for your own usage, without contributing to the Airbyte codebase, read along. - -## Developing your own connectors - -It's easy to code your own integrations on Airbyte. Here are some links to instruct on how to code new sources and destinations. - -* [Building new connectors](../contributing-to-airbyte/README.md) - -While the guides above are specific to the languages used most frequently to write integrations, **Airbyte integrations can be written in any language**. Please reach out to us if you'd like help developing integrations in other languages. - diff --git a/docs/integrations/sources/airtable-migrations.md b/docs/integrations/sources/airtable-migrations.md new file mode 100644 index 000000000000..66a0d6526f01 --- /dev/null +++ b/docs/integrations/sources/airtable-migrations.md @@ -0,0 +1,4 @@ +# Airtable Migration Guide + +## Upgrading to 4.0.0 +Columns with Formulas are narrowing from `array` to `string` or `number`. You may need to refresh the connection schema (with the reset), and run a sync. \ No newline at end of file diff --git a/docs/integrations/sources/airtable.inapp.md b/docs/integrations/sources/airtable.inapp.md deleted file mode 100644 index 200b8f65ec25..000000000000 --- a/docs/integrations/sources/airtable.inapp.md +++ /dev/null @@ -1,22 +0,0 @@ -:::info -Currently, this source connector works with `Standard` subscription plan only. `Enterprise` level accounts are not supported yet. -::: - -## Prerequisites - -* An active Airtable account - -## Setup guide -1. Name your source. -2. You can use OAuth or a Personal Access Token to authenticate your Airtable account. We recommend using OAuth for Airbyte Cloud. - - To authenticate using OAuth, select **OAuth2.0** from the Authentication dropdown click **Authenticate your Airtable account** to sign in with Airtable, select required workspaces you want to sync and authorize your account. - - To authenticate using a [Personal Access Token](https://airtable.com/developers/web/guides/personal-access-tokens), select **Personal Access Token** from the Authentication dropdown and enter the Access Token for your Airtable account. The following scopes are required: - - `data.records:read` - - `data.recordComments:read` - - `schema.bases:read` - -:::info -When using OAuth, you may see a `400` or `401` error causing a failed sync. You can re-authenticate your Airtable connector to solve the issue temporarily. We are working on a permanent fix that you can follow [here](https://github.com/airbytehq/airbyte/issues/25278). -::: - -3. Click **Set up source**. diff --git a/docs/integrations/sources/airtable.md b/docs/integrations/sources/airtable.md index 59d427c2f6a3..5e314da9f1f7 100644 --- a/docs/integrations/sources/airtable.md +++ b/docs/integrations/sources/airtable.md @@ -2,10 +2,6 @@ This page contains the setup guide and reference information for the [Airtable](https://airtable.com/api) source connector. -:::caution -Currently, this source connector works with `Standard` subscription plan only. `Enterprise` level accounts are not supported yet. -::: - ## Prerequisites * An active Airtable account @@ -15,9 +11,25 @@ Currently, this source connector works with `Standard` subscription plan only. ` - `schema.bases:read` ## Setup guide - ### Step 1: Set up Airtable + +#### For Airbyte Open Source: +1. Go to https://airtable.com/create/tokens to create new token. + ![Generate new Token](../../.gitbook/assets/source/airtable/generate_new_token.png) +2. Add following scopes: + - `data.records:read` + - `data.recordComments:read` + - `schema.bases:read` + + ![Add Scopes](../../.gitbook/assets/source/airtable/add_scopes.png) +3. Select required bases or allow access to all available and press the `Create Token` button. + ![Add Bases](../../.gitbook/assets/source/airtable/add_bases.png) +4. Save token from the popup window. + + +### Step 2: Set up Airtable connector in Airbyte + ### For Airbyte Cloud: @@ -51,51 +63,52 @@ Please keep in mind that if you start syncing a table via Airbyte, then rename i The airtable source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): -| Feature | Supported?\(Yes/No\) | Notes | -|:------------------|:---------------------|:------| -| Full Refresh Sync | Yes | | -| Incremental Sync | No | | - +- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/glossary#full-refresh-sync) +- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append) ## Supported Tables -This source allows you to pull all available tables and bases using `Metadata API` for a given authenticated user. In case you rename or add a column to any existing table, you will need to recreate the source to update the Airbyte catalog. +This source allows you to pull all available tables and bases using `Metadata API` for a given authenticated user. In case you rename or add a column to any existing table, you will need to recreate the source to update the Airbyte catalog. + +### Performance Considerations + +See information about rate limits [here](https://airtable.com/developers/web/api/rate-limits). ## Data type map -| Integration Type | Airbyte Type | Nullable | -|:------------------------|:--------------------------------|----------| -| `multipleAttachments` | `string` | Yes | -| `autoNumber` | `string` | Yes | -| `barcode` | `string` | Yes | -| `button` | `string` | Yes | -| `checkbox` | `boolean` | Yes | -| `singleCollaborator` | `string` | Yes | -| `count` | `number` | Yes | -| `createdBy` | `string` | Yes | -| `createdTime` | `datetime`, `format: date-time` | Yes | -| `currency` | `number` | Yes | -| `email` | `string` | Yes | -| `date` | `string`, `format: date` | Yes | -| `duration` | `number` | Yes | -| `lastModifiedBy` | `string` | Yes | -| `lastModifiedTime` | `datetime`, `format: date-time` | Yes | -| `multipleRecordLinks` | `array with strings` | Yes | -| `multilineText` | `string` | Yes | -| `multipleCollaborators` | `array with strings` | Yes | -| `multipleSelects` | `array with strings` | Yes | -| `number` | `number` | Yes | -| `percent` | `number` | Yes | -| `phoneNumber` | `string` | Yes | -| `rating` | `number` | Yes | -| `richText` | `string` | Yes | -| `singleLineText` | `string` | Yes | -| `externalSyncSource` | `string` | Yes | -| `url` | `string` | Yes | -| `formula` | `array with any` | Yes | -| `lookup` | `array with any` | Yes | -| `multipleLookupValues` | `array with any` | Yes | -| `rollup` | `array with any` | Yes | +| Integration Type | Airbyte Type | Nullable | +|:------------------------|:---------------------------------------|----------| +| `multipleAttachments` | `string` | Yes | +| `autoNumber` | `string` | Yes | +| `barcode` | `string` | Yes | +| `button` | `string` | Yes | +| `checkbox` | `boolean` | Yes | +| `singleCollaborator` | `string` | Yes | +| `count` | `number` | Yes | +| `createdBy` | `string` | Yes | +| `createdTime` | `datetime`, `format: date-time` | Yes | +| `currency` | `number` | Yes | +| `email` | `string` | Yes | +| `date` | `string`, `format: date` | Yes | +| `duration` | `number` | Yes | +| `lastModifiedBy` | `string` | Yes | +| `lastModifiedTime` | `datetime`, `format: date-time` | Yes | +| `multipleRecordLinks` | `array with strings` | Yes | +| `multilineText` | `string` | Yes | +| `multipleCollaborators` | `array with strings` | Yes | +| `multipleSelects` | `array with strings` | Yes | +| `number` | `number` | Yes | +| `percent` | `number` | Yes | +| `phoneNumber` | `string` | Yes | +| `rating` | `number` | Yes | +| `richText` | `string` | Yes | +| `singleLineText` | `string` | Yes | +| `externalSyncSource` | `string` | Yes | +| `url` | `string` | Yes | +| `formula` | `string`, `number` or `array with any` | Yes | +| `lookup` | `array with any` | Yes | +| `multipleLookupValues` | `array with any` | Yes | +| `rollup` | `array with any` | Yes | * All the fields are `nullable` by default, meaning that the field could be empty. * The `array with any` - represents the classic array with one of the other Airtable data types inside, such as: @@ -103,24 +116,27 @@ This source allows you to pull all available tables and bases using `Metadata AP - number/integer - nested lists/objects -### Performance Considerations (Airbyte Open-Source) - -See information about rate limits [here](https://airtable.com/developers/web/api/rate-limits). - ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------| -| 3.0.1 | 2023-05-10 | [25946](https://github.com/airbytehq/airbyte/pull/25946) | Skip stream if it does not appear in catalog | -| 3.0.0 | 2023-03-20 | [22704](https://github.com/airbytehq/airbyte/pull/22704) | Fix for stream name uniqueness | -| 2.0.4 | 2023-03-15 | [24093](https://github.com/airbytehq/airbyte/pull/24093) | Update spec and doc | -| 2.0.3 | 2023-02-02 | [22311](https://github.com/airbytehq/airbyte/pull/22311) | Fix for `singleSelect` types when discovering the schema | -| 2.0.2 | 2023-02-01 | [22245](https://github.com/airbytehq/airbyte/pull/22245) | Fix for empty `result` object when discovering the schema | -| 2.0.1 | 2023-02-01 | [22224](https://github.com/airbytehq/airbyte/pull/22224) | Fixed broken `API Key` authentication | -| 2.0.0 | 2023-01-27 | [21962](https://github.com/airbytehq/airbyte/pull/21962) | Added casting of native Airtable data types to JsonSchema types | -| 1.0.2 | 2023-01-25 | [20934](https://github.com/airbytehq/airbyte/pull/20934) | Added `OAuth2.0` authentication support | -| 1.0.1 | 2023-01-10 | [21215](https://github.com/airbytehq/airbyte/pull/21215) | Fix field names | -| 1.0.0 | 2022-12-22 | [20846](https://github.com/airbytehq/airbyte/pull/20846) | Migrated to Metadata API for dynamic schema generation | -| 0.1.3 | 2022-10-26 | [18491](https://github.com/airbytehq/airbyte/pull/18491) | Improve schema discovery logic | -| 0.1.2 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | -| 0.1.1 | 2021-12-06 | [8425](https://github.com/airbytehq/airbyte/pull/8425) | Update title, description fields in spec | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------| +| 4.1.5 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 4.1.4 | 2023-10-19 | [31360](https://github.com/airbytehq/airbyte/pull/31360) | Update docstings | +| 4.1.3 | 2023-10-13 | [31360](https://github.com/airbytehq/airbyte/pull/31360) | Update error message for invalid permissions | +| 4.1.2 | 2023-10-10 | [31215](https://github.com/airbytehq/airbyte/pull/31215) | Exclude bases without permission | +| 4.1.1 | 2023-10-10 | [31119](https://github.com/airbytehq/airbyte/pull/31119) | Add user-friendly error message when refresh token has expired | +| 4.1.0 | 2023-10-10 | [31044](https://github.com/airbytehq/airbyte/pull/31044) | Add source table name to output records | +| 4.0.0 | 2023-10-09 | [31181](https://github.com/airbytehq/airbyte/pull/31181) | Additional schema processing for the FORMULA schema type: Convert to simple data types | +| 3.0.1 | 2023-05-10 | [25946](https://github.com/airbytehq/airbyte/pull/25946) | Skip stream if it does not appear in catalog | +| 3.0.0 | 2023-03-20 | [22704](https://github.com/airbytehq/airbyte/pull/22704) | Fix for stream name uniqueness | +| 2.0.4 | 2023-03-15 | [24093](https://github.com/airbytehq/airbyte/pull/24093) | Update spec and doc | +| 2.0.3 | 2023-02-02 | [22311](https://github.com/airbytehq/airbyte/pull/22311) | Fix for `singleSelect` types when discovering the schema | +| 2.0.2 | 2023-02-01 | [22245](https://github.com/airbytehq/airbyte/pull/22245) | Fix for empty `result` object when discovering the schema | +| 2.0.1 | 2023-02-01 | [22224](https://github.com/airbytehq/airbyte/pull/22224) | Fixed broken `API Key` authentication | +| 2.0.0 | 2023-01-27 | [21962](https://github.com/airbytehq/airbyte/pull/21962) | Added casting of native Airtable data types to JsonSchema types | +| 1.0.2 | 2023-01-25 | [20934](https://github.com/airbytehq/airbyte/pull/20934) | Added `OAuth2.0` authentication support | +| 1.0.1 | 2023-01-10 | [21215](https://github.com/airbytehq/airbyte/pull/21215) | Fix field names | +| 1.0.0 | 2022-12-22 | [20846](https://github.com/airbytehq/airbyte/pull/20846) | Migrated to Metadata API for dynamic schema generation | +| 0.1.3 | 2022-10-26 | [18491](https://github.com/airbytehq/airbyte/pull/18491) | Improve schema discovery logic | +| 0.1.2 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | +| 0.1.1 | 2021-12-06 | [8425](https://github.com/airbytehq/airbyte/pull/8425) | Update title, description fields in spec | \ No newline at end of file diff --git a/docs/integrations/sources/alloydb.md b/docs/integrations/sources/alloydb.md deleted file mode 100644 index 1768d08fc432..000000000000 --- a/docs/integrations/sources/alloydb.md +++ /dev/null @@ -1,363 +0,0 @@ -# AlloyDB for PostgreSQL - -This page contains the setup guide and reference information for the AlloyDB for PostgreSQL. - -## Prerequisites - -- For Airbyte Open Source users, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer -- For Airbyte Cloud (and optionally for Airbyte Open Source), ensure SSL is enabled in your environment - -## Setup guide - -## When to use AlloyDB with CDC - -Configure AlloyDB with CDC if: - -- You need a record of deletions -- Your table has a primary key but doesn't have a reasonable cursor field for incremental syncing (`updated_at`). CDC allows you to sync your table incrementally - -If your goal is to maintain a snapshot of your table in the destination but the limitations prevent you from using CDC, consider using [non-CDC incremental sync](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) and occasionally reset the data and re-sync. - -If your dataset is small and you just want a snapshot of your table in the destination, consider using [Full Refresh replication](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite) for your table instead of CDC. - -### Step 1: (Optional) Create a dedicated read-only user - -We recommend creating a dedicated read-only user for better permission control and auditing. Alternatively, you can use an existing AlloyDB user in your database. - -To create a dedicated user, run the following command: - -``` -CREATE USER PASSWORD 'your_password_here'; -``` - -Grant access to the relevant schema: - -``` -GRANT USAGE ON SCHEMA TO -``` - -:::note -To replicate data from multiple AlloyDB schemas, re-run the command to grant access to all the relevant schemas. Note that you'll need to set up multiple Airbyte sources connecting to the same AlloyDB database on multiple schemas. -::: - -Grant the user read-only access to the relevant tables: - -``` -GRANT SELECT ON ALL TABLES IN SCHEMA TO ; -``` - -Allow user to see tables created in the future: - -``` -ALTER DEFAULT PRIVILEGES IN SCHEMA GRANT SELECT ON TABLES TO ; -``` - -Additionally, if you plan to configure CDC for the AlloyDB source connector, grant `REPLICATION` permissions to the user: - -``` -ALTER USER REPLICATION; -``` - -**Syncing a subset of columns​** - -Currently, there is no way to sync a subset of columns using the AlloyDB source connector: - -- When setting up a connection, you can only choose which tables to sync, but not columns. -- If the user can only access a subset of columns, the connection check will pass. However, the data sync will fail with a permission denied exception. - -The workaround for partial table syncing is to create a view on the specific columns, and grant the user read access to that view: - -``` -CREATE VIEW as SELECT FROM ; -``` - -``` -GRANT SELECT ON TABLE IN SCHEMA to ; -``` - -**Note:** The workaround works only for non-CDC setups since CDC requires data to be in tables and not views. -This issue is tracked in [#9771](https://github.com/airbytehq/airbyte/issues/9771). - -### Step 2: Set up the AlloyDB connector in Airbyte - -1. Log into your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account. -2. Click **Sources** and then click **+ New source**. -3. On the Set up the source page, select **AlloyDB** from the Source type dropdown. -4. Enter a name for your source. -5. For the **Host**, **Port**, and **DB Name**, enter the hostname, port number, and name for your AlloyDB database. -6. List the **Schemas** you want to sync. - :::note - The schema names are case sensitive. The 'public' schema is set by default. Multiple schemas may be used at one time. No schemas set explicitly - will sync all of existing. - ::: -7. For **User** and **Password**, enter the username and password you created in [Step 1](#step-1-optional-create-a-dedicated-read-only-user). -8. To customize the JDBC connection beyond common options, specify additional supported [JDBC URL parameters](https://jdbc.postgresql.org/documentation/head/connect.html) as key-value pairs separated by the symbol & in the **JDBC URL Parameters (Advanced)** field. - - Example: key1=value1&key2=value2&key3=value3 - - These parameters will be added at the end of the JDBC URL that the AirByte will use to connect to your AlloyDB database. - - The connector now supports `connectTimeout` and defaults to 60 seconds. Setting connectTimeout to 0 seconds will set the timeout to the longest time available. - - **Note:** Do not use the following keys in JDBC URL Params field as they will be overwritten by Airbyte: - `currentSchema`, `user`, `password`, `ssl`, and `sslmode`. - - :::warning - This is an advanced configuration option. Users are advised to use it with caution. - ::: - -9. For Airbyte Open Source, toggle the switch to connect using SSL. Airbyte Cloud uses SSL by default. -10. For Replication Method, select Standard or [Logical CDC](https://www.postgresql.org/docs/10/logical-replication.html) from the dropdown. Refer to [Configuring AlloyDB connector with Change Data Capture (CDC)](#configuring-alloydb-connector-with-change-data-capture-cdc) for more information. -11. For SSH Tunnel Method, select: - - No Tunnel for a direct connection to the database - - SSH Key Authentication to use an RSA Private as your secret for establishing the SSH tunnel - - Password Authentication to use a password as your secret for establishing the SSH tunnel - Refer to [Connect via SSH Tunnel](#connect-via-ssh-tunnel​) for more information. -12. Click **Set up source**. - -### Connect via SSH Tunnel​ - -You can connect to a AlloyDB instance via an SSH tunnel. - -When using an SSH tunnel, you are configuring Airbyte to connect to an intermediate server (also called a bastion server) that has direct access to the database. Airbyte connects to the bastion and then asks the bastion to connect directly to the server. - -To connect to a AlloyDB instance via an SSH tunnel: - -1. While [setting up](#setup-guide) the AlloyDB source connector, from the SSH tunnel dropdown, select: - - SSH Key Authentication to use an RSA Private as your secret for establishing the SSH tunnel - - Password Authentication to use a password as your secret for establishing the SSH Tunnel -2. For **SSH Tunnel Jump Server Host**, enter the hostname or IP address for the intermediate (bastion) server that Airbyte will connect to. -3. For **SSH Connection Port**, enter the port on the bastion server. The default port for SSH connections is 22. -4. For **SSH Login Username**, enter the username to use when connecting to the bastion server. **Note:** This is the operating system username and not the AlloyDB username. -5. For authentication: - - If you selected **SSH Key Authentication**, set the **SSH Private Key** to the [RSA Private Key](#generating-an-rsa-private-key​) that you are using to create the SSH connection. - - If you selected **Password Authentication**, enter the password for the operating system user to connect to the bastion server. **Note:** This is the operating system password and not the AlloyDB password. - -#### Generating an RSA Private Key​ - -The connector expects an RSA key in PEM format. To generate this key, run: - -``` -ssh-keygen -t rsa -m PEM -f myuser_rsa -``` - -The command produces the private key in PEM format and the public key remains in the standard format used by the `authorized_keys` file on your bastion server. Add the public key to your bastion host to the user you want to use with Airbyte. The private key is provided via copy-and-paste to the Airbyte connector configuration screen to allow it to log into the bastion server. - -## Configuring AlloyDB connector with Change Data Capture (CDC) - -Airbyte uses [logical replication](https://www.postgresql.org/docs/10/logical-replication.html) of the Postgres write-ahead log (WAL) to incrementally capture deletes using a replication plugin. To learn more how Airbyte implements CDC, refer to [Change Data Capture (CDC)](https://docs.airbyte.com/understanding-airbyte/cdc/) - -### CDC Considerations - -- Incremental sync is only supported for tables with primary keys. For tables without primary keys, use [Full Refresh sync](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite). -- Data must be in tables and not views. -- The modifications you want to capture must be made using `DELETE`/`INSERT`/`UPDATE`. For example, changes made using `TRUNCATE`/`ALTER` will not appear in logs and therefore in your destination. -- Schema changes are not supported automatically for CDC sources. Reset and resync data if you make a schema change. -- The records produced by `DELETE` statements only contain primary keys. All other data fields are unset. -- Log-based replication only works for master instances of AlloyDB. -- Using logical replication increases disk space used on the database server. The additional data is stored until it is consumed. - - Set frequent syncs for CDC to ensure that the data doesn't fill up your disk space. - - If you stop syncing a CDC-configured AlloyDB instance with Airbyte, delete the replication slot. Otherwise, it may fill up your disk space. - -### Setting up CDC for AlloyDB - -Airbyte requires a replication slot configured only for its use. Only one source should be configured that uses this replication slot. See Setting up CDC for AlloyDB for instructions. - -#### Step 2: Select a replication plugin​ - -We recommend using a [pgoutput](https://www.postgresql.org/docs/9.6/logicaldecoding-output-plugin.html) plugin (the standard logical decoding plugin in AlloyDB). If the replication table contains multiple JSON blobs and the table size exceeds 1 GB, we recommend using a [wal2json](https://github.com/eulerto/wal2json) instead. Note that wal2json may require additional installation for Bare Metal, VMs (EC2/GCE/etc), Docker, etc. For more information read the [wal2json documentation](https://github.com/eulerto/wal2json). - -#### Step 3: Create replication slot​ - -To create a replication slot called `airbyte_slot` using pgoutput, run: - -``` -SELECT pg_create_logical_replication_slot('airbyte_slot', 'pgoutput'); -``` - -To create a replication slot called `airbyte_slot` using wal2json, run: - -``` -SELECT pg_create_logical_replication_slot('airbyte_slot', 'wal2json'); -``` - -#### Step 4: Create publications and replication identities for tables​ - -For each table you want to replicate with CDC, add the replication identity (the method of distinguishing between rows) first: - -To use primary keys to distinguish between rows, run: - -``` -ALTER TABLE tbl1 REPLICA IDENTITY DEFAULT; -``` - -After setting the replication identity, run: - -``` -CREATE PUBLICATION airbyte_publication FOR TABLE ;` -``` - -The publication name is customizable. Refer to the [Postgres docs](https://www.postgresql.org/docs/10/sql-alterpublication.html) if you need to add or remove tables from your publication in the future. - -:::note -You must add the replication identity before creating the publication. Otherwise, `ALTER`/`UPDATE`/`DELETE` statements may fail if AlloyDB cannot determine how to uniquely identify rows. -Also, the publication should include all the tables and only the tables that need to be synced. Otherwise, data from these tables may not be replicated correctly. -::: - -:::warning -The Airbyte UI currently allows selecting any tables for CDC. If a table is selected that is not part of the publication, it will not be replicated even though it is selected. If a table is part of the publication but does not have a replication identity, that replication identity will be created automatically on the first run if the Airbyte user has the necessary permissions. -::: - -#### Step 5: [Optional] Set up initial waiting time - -:::warning -This is an advanced feature. Use it if absolutely necessary. -::: - -The AlloyDB connector may need some time to start processing the data in the CDC mode in the following scenarios: - -- When the connection is set up for the first time and a snapshot is needed -- When the connector has a lot of change logs to process - -The connector waits for the default initial wait time of 5 minutes (300 seconds). Setting the parameter to a longer duration will result in slower syncs, while setting it to a shorter duration may cause the connector to not have enough time to create the initial snapshot or read through the change logs. The valid range is 120 seconds to 1200 seconds. - -If you know there are database changes to be synced, but the connector cannot read those changes, the root cause may be insufficient waiting time. In that case, you can increase the waiting time (example: set to 600 seconds) to test if it is indeed the root cause. On the other hand, if you know there are no database changes, you can decrease the wait time to speed up the zero record syncs. - -#### Step 6: Set up the AlloyDB source connector - -In [Step 2](#step-2-set-up-the-alloydb-connector-in-airbyte) of the connector setup guide, enter the replication slot and publication you just created. - -## Supported sync modes - -The AlloyDB source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): - -- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/) -- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append) -- [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) -- [Incremental Sync - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped) - -## Supported cursors - -- `TIMESTAMP` -- `TIMESTAMP_WITH_TIMEZONE` -- `TIME` -- `TIME_WITH_TIMEZONE` -- `DATE` -- `BIT` -- `BOOLEAN` -- `TINYINT/SMALLINT` -- `INTEGER` -- `BIGINT` -- `FLOAT/DOUBLE` -- `REAL` -- `NUMERIC/DECIMAL` -- `CHAR/NCHAR/NVARCHAR/VARCHAR/LONGVARCHAR` -- `BINARY/BLOB` - -## Data type mapping - -The AlloyDb is a fully managed PostgreSQL-compatible database service. - -According to Postgres [documentation](https://www.postgresql.org/docs/14/datatype.html), Postgres data types are mapped to the following data types when synchronizing data. You can check the test values examples [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java). If you can't find the data type you are looking for or have any problems feel free to add a new test! - -| Postgres Type | Resulting Type | Notes | -| :------------------------------------ | :------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------- | -| `bigint` | number | | -| `bigserial`, `serial8` | number | | -| `bit` | string | Fixed-length bit string (e.g. "0100"). | -| `bit varying`, `varbit` | string | Variable-length bit string (e.g. "0100"). | -| `boolean`, `bool` | boolean | | -| `box` | string | | -| `bytea` | string | Variable length binary string with hex output format prefixed with "\x" (e.g. "\x6b707a"). | -| `character`, `char` | string | | -| `character varying`, `varchar` | string | | -| `cidr` | string | | -| `circle` | string | | -| `date` | string | Parsed as ISO8601 date time at midnight. CDC mode doesn't support era indicators. Issue: [#14590](https://github.com/airbytehq/airbyte/issues/14590) | -| `double precision`, `float`, `float8` | number | `Infinity`, `-Infinity`, and `NaN` are not supported and converted to `null`. Issue: [#8902](https://github.com/airbytehq/airbyte/issues/8902). | -| `hstore` | string | | -| `inet` | string | | -| `integer`, `int`, `int4` | number | | -| `interval` | string | | -| `json` | string | | -| `jsonb` | string | | -| `line` | string | | -| `lseg` | string | | -| `macaddr` | string | | -| `macaddr8` | string | | -| `money` | number | | -| `numeric`, `decimal` | number | `Infinity`, `-Infinity`, and `NaN` are not supported and converted to `null`. Issue: [#8902](https://github.com/airbytehq/airbyte/issues/8902). | -| `path` | string | | -| `pg_lsn` | string | | -| `point` | string | | -| `polygon` | string | | -| `real`, `float4` | number | | -| `smallint`, `int2` | number | | -| `smallserial`, `serial2` | number | | -| `serial`, `serial4` | number | | -| `text` | string | | -| `time` | string | Parsed as a time string without a time-zone in the ISO-8601 calendar system. | -| `timetz` | string | Parsed as a time string with time-zone in the ISO-8601 calendar system. | -| `timestamp` | string | Parsed as a date-time string without a time-zone in the ISO-8601 calendar system. | -| `timestamptz` | string | Parsed as a date-time string with time-zone in the ISO-8601 calendar system. | -| `tsquery` | string | | -| `tsvector` | string | | -| `uuid` | string | | -| `xml` | string | | -| `enum` | string | | -| `tsrange` | string | | -| `array` | array | E.g. "[\"10001\",\"10002\",\"10003\",\"10004\"]". | -| composite type | string | | - -## Limitations - -- The AlloyDB source connector currently does not handle schemas larger than 4MB. -- The AlloyDB source connector does not alter the schema present in your database. Depending on the destination connected to this source, however, the schema may be altered. See the destination's documentation for more details. -- The following two schema evolution actions are currently supported: - - Adding/removing tables without resetting the entire connection at the destination - Caveat: In the CDC mode, adding a new table to a connection may become a temporary bottleneck. When a new table is added, the next sync job takes a full snapshot of the new table before it proceeds to handle any changes. - - Resetting a single table within the connection without resetting the rest of the destination tables in that connection -- Changing a column data type or removing a column might break connections. - -## Changelog - -| Version | Date | Pull Request | Subject | -|:--------|:-----------| :------------------------------------------------------- |:------------------------------------------------------------------------------------------------------------------------------------------| -| 3.1.5 | 2023-08-22 | [29534](https://github.com/airbytehq/airbyte/pull/29534) | Support "options" JDBC URL parameter | -| 3.1.3 | 2023-08-03 | [28708](https://github.com/airbytehq/airbyte/pull/28708) | Enable checkpointing snapshots in CDC connections | -| 3.1.2 | 2023-08-01 | [28954](https://github.com/airbytehq/airbyte/pull/28954) | Fix an issue that prevented use of tables with names containing uppercase letters | -| 3.1.1 | 2023-07-31 | [28892](https://github.com/airbytehq/airbyte/pull/28892) | Fix an issue that prevented use of cursor columns with names containing uppercase letters | -| 3.1.0 | 2023-07-25 | [28339](https://github.com/airbytehq/airbyte/pull/28339) | Checkpointing initial load for incremental syncs: enabled for xmin and cursor based only. | -| 2.0.28 | 2023-04-26 | [25401](https://github.com/airbytehq/airbyte/pull/25401) | CDC : Upgrade Debezium to version 2.2.0 | -| 2.0.23 | 2023-04-19 | [24582](https://github.com/airbytehq/airbyte/pull/24582) | CDC : Enable frequent state emission during incremental syncs + refactor for performance improvement | -| 2.0.22 | 2023-04-17 | [25220](https://github.com/airbytehq/airbyte/pull/25220) | Logging changes : Log additional metadata & clean up noisy logs | -| 2.0.21 | 2023-04-12 | [25131](https://github.com/airbytehq/airbyte/pull/25131) | Make Client Certificate and Client Key always show | -| 2.0.19 | 2023-04-11 | [24656](https://github.com/airbytehq/airbyte/pull/24656) | CDC minor refactor | -| 2.0.17 | 2023-04-05 | [24622](https://github.com/airbytehq/airbyte/pull/24622) | Allow streams not in CDC publication to be synced in Full-refresh mode | -| 2.0.15 | 2023-04-04 | [24833](https://github.com/airbytehq/airbyte/pull/24833) | Disallow the "disable" SSL Modes; fix Debezium retry policy configuration | -| 2.0.13 | 2023-03-28 | [24166](https://github.com/airbytehq/airbyte/pull/24166) | Fix InterruptedException bug during Debezium shutdown | -| 2.0.11 | 2023-03-27 | [24529](https://github.com/airbytehq/airbyte/pull/24373) | Preparing the connector for CDC checkpointing | -| 2.0.10 | 2023-03-24 | [24529](https://github.com/airbytehq/airbyte/pull/24529) | Set SSL Mode to required on strict-encrypt variant | -| 2.0.9 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | -| 2.0.6 | 2023-03-21 | [24271](https://github.com/airbytehq/airbyte/pull/24271) | Fix NPE in CDC mode | -| 2.0.3 | 2023-03-21 | [24147](https://github.com/airbytehq/airbyte/pull/24275) | Fix error with CDC checkpointing | -| 2.0.2 | 2023-03-13 | [23112](https://github.com/airbytehq/airbyte/pull/21727) | Add state checkpointing for CDC sync. | -| 2.0.1 | 2023-03-08 | [23596](https://github.com/airbytehq/airbyte/pull/23596) | For network isolation, source connector accepts a list of hosts it is allowed to connect | -| 2.0.0 | 2023-03-06 | [23112](https://github.com/airbytehq/airbyte/pull/23112) | Upgrade Debezium version to 2.1.2 | -| 1.0.51 | 2023-03-02 | [23642](https://github.com/airbytehq/airbyte/pull/23642) | Revert : Support JSONB datatype for Standard sync mode | -| 1.0.49 | 2023-02-27 | [21695](https://github.com/airbytehq/airbyte/pull/21695) | Support JSONB datatype for Standard sync mode | -| 1.0.48 | 2023-02-24 | [23383](https://github.com/airbytehq/airbyte/pull/23383) | Fixed bug with non readable double-quoted values within a database name or column name | -| 1.0.47 | 2023-02-22 | [22221](https://github.com/airbytehq/airbyte/pull/23138) | Fix previous versions which doesn't verify privileges correctly, preventing CDC syncs to run. | -| 1.0.46 | 2023-02-21 | [23105](https://github.com/airbytehq/airbyte/pull/23105) | Include log levels and location information (class, method and line number) with source connector logs published to Airbyte Platform. | -| 1.0.45 | 2023-02-09 | [22221](https://github.com/airbytehq/airbyte/pull/22371) | Ensures that user has required privileges for CDC syncs. | -| | 2023-02-15 | [23028](https://github.com/airbytehq/airbyte/pull/23028) | | -| 1.0.44 | 2023-02-06 | [22221](https://github.com/airbytehq/airbyte/pull/22221) | Exclude new set of system tables when using `pg_stat_statements` extension. | -| 1.0.43 | 2023-02-06 | [21634](https://github.com/airbytehq/airbyte/pull/21634) | Improve Standard sync performance by caching objects. | -| 1.0.36 | 2023-01-24 | [21825](https://github.com/airbytehq/airbyte/pull/21825) | Put back the original change that will cause an incremental sync to error if table contains a NULL value in cursor column. | -| 1.0.35 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | -| 1.0.34 | 2022-12-13 | [20378](https://github.com/airbytehq/airbyte/pull/20378) | Improve descriptions | -| 1.0.17 | 2022-10-31 | [18538](https://github.com/airbytehq/airbyte/pull/18538) | Encode database name | -| 1.0.16 | 2022-10-25 | [18256](https://github.com/airbytehq/airbyte/pull/18256) | Disable allow and prefer ssl modes in CDC mode | -| | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | -| 1.0.15 | 2022-10-11 | [17782](https://github.com/airbytehq/airbyte/pull/17782) | Align with Postgres source v.1.0.15 | -| 1.0.0 | 2022-09-15 | [16776](https://github.com/airbytehq/airbyte/pull/16776) | Align with strict-encrypt version | -| 0.1.0 | 2022-09-05 | [16323](https://github.com/airbytehq/airbyte/pull/16323) | Initial commit. Based on source-postgres v.1.0.7 | diff --git a/docs/integrations/sources/amazon-ads-migrations.md b/docs/integrations/sources/amazon-ads-migrations.md index 6b95c39af7b9..11f3e15cb5ef 100644 --- a/docs/integrations/sources/amazon-ads-migrations.md +++ b/docs/integrations/sources/amazon-ads-migrations.md @@ -1,5 +1,31 @@ # Amazon Ads Migration Guide +## Upgrading to 4.0.0 + +Streams `SponsoredBrandsAdGroups` and `SponsoredBrandsKeywords` now have updated schemas. + +### Refresh affected schemas and reset data + +1. Select **Connections** in the main navbar. + 1. Select the connection(s) affected by the update. +2. Select the **Replication** tab. + 1. Select **Refresh source schema**. + 2. Select **OK**. +```note +Any detected schema changes will be listed for your review. +``` +3. Select **Save changes** at the bottom of the page. + 1. Ensure the **Reset affected streams** option is checked. +```note +Depending on destination type you may not be prompted to reset your data. +``` +4. Select **Save connection**. +```note +This will reset the data in your destination and initiate a fresh sync. +``` + +For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). + ## Upgrading to 3.0.0 A major update of attribution report stream schemas. diff --git a/docs/integrations/sources/amazon-ads.inapp.md b/docs/integrations/sources/amazon-ads.inapp.md deleted file mode 100644 index d501c7be4642..000000000000 --- a/docs/integrations/sources/amazon-ads.inapp.md +++ /dev/null @@ -1,20 +0,0 @@ -## Prerequisites - -- An [Amazon user](https://www.amazon.com) with access to an [Amazon Ads account](https://advertising.amazon.com) - -## Setup Guide - -1. Click `Authenticate your Amazon Ads account`. Log in and authorize access to the Amazon account. -2. Select **Region** to pull data from **North America (NA)**, **Europe (EU)**, **Far East (FE)**. See [Amazon Ads documentation](https://advertising.amazon.com/API/docs/en-us/info/api-overview#api-endpoints) for more details. -3. (Optional) **Start Date** can be used to generate reports starting from the specified start date in the format YYYY-MM-DD. The date should not be more than 60 days in the past. If not specified, today's date is used. The date is treated in the timezone of the processed profile. -4. (Optional) **Profile ID(s)** you want to fetch data for. A profile is an advertiser's account in a specific marketplace. See [Amazon Ads docs](https://advertising.amazon.com/API/docs/en-us/concepts/authorization/profiles) for more details. If not specified, data from all Profiles will be synced. -5. (Optional) **State Filter** Filter for Display, Product, and Brand Campaign streams with a state of enabled, paused, or archived. If not specified, all streams regardless of state will be synced. -6. (Optional) **Look Back Window** The amount of days to go back in time to get the updated data from Amazon Ads. After the first sync, data from this date will be synced. -7. (Optional) **Report Record Types** Optional configuration which accepts an array of string of record types. Leave blank for default behaviour to pull all report types. Use this config option only if you want to pull specific report type(s). See [Amazon Ads docs](https://advertising.amazon.com/API/docs/en-us/reporting/v2/report-types) for more details -9. Click `Set up source`. - -### Report Timezones - -All the reports are generated relative to the target profile' timezone. - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Amazon Ads](https://docs.airbyte.com/integrations/sources/amazon-ads). diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md index 0955fe1e7906..ecef322972b8 100644 --- a/docs/integrations/sources/amazon-ads.md +++ b/docs/integrations/sources/amazon-ads.md @@ -9,14 +9,15 @@ This page contains the setup guide and reference information for the Amazon Ads * Region * Start Date (Optional) * Profile IDs (Optional) +* Marketplace IDs (Optional) ## Setup guide ### Step 1: Set up Amazon Ads -Create an [Amazon user](https://www.amazon.com) with access to [Amazon Ads account](https://advertising.amazon.com). +Create an [Amazon user](https://www.amazon.com) with access to an [Amazon Ads account](https://advertising.amazon.com). **For Airbyte Open Source:** -To use the [Amazon Ads API](https://advertising.amazon.com/API/docs/en-us), you must first complete the [onboarding process](https://advertising.amazon.com/API/docs/en-us/setting-up/overview). The onboarding process has several steps and may take several days to complete. After completing all steps you will have to get Amazon client application `Client ID`, `Client Secret` and `Refresh Token`. +To use the [Amazon Ads API](https://advertising.amazon.com/API/docs/en-us), you must first complete the [onboarding process](https://advertising.amazon.com/API/docs/en-us/setting-up/overview). The onboarding process has several steps and may take several days to complete. After completing all steps you will have to get the Amazon client application's `Client ID`, `Client Secret` and `Refresh Token`. ### Step 2: Set up the Amazon Ads connector in Airbyte @@ -27,12 +28,13 @@ To use the [Amazon Ads API](https://advertising.amazon.com/API/docs/en-us), you 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**. 3. On the source setup page, select **Amazon Ads** from the Source type dropdown and enter a name for this connector. -4. Click `Authenticate your Amazon Ads account`. +4. Click **Authenticate your Amazon Ads account**. 5. Log in and Authorize to the Amazon account. 6. Select **Region** to pull data from **North America (NA)**, **Europe (EU)**, **Far East (FE)**. See [docs](https://advertising.amazon.com/API/docs/en-us/info/api-overview#api-endpoints) for more details. -7. **Start Date (Optional)** is used for generating reports starting from the specified start date. Should be in YYYY-MM-DD format and not more than 60 days in the past. If not specified today's date is used. The date is treated in the timezone of the processed profile. +7. **Start Date (Optional)** is used for generating reports starting from the specified start date. This should be in YYYY-MM-DD format and not more than 60 days in the past. If a date is not specified, today's date is used. The date is treated in the timezone of the processed profile. 8. **Profile IDs (Optional)** you want to fetch data for. See [docs](https://advertising.amazon.com/API/docs/en-us/concepts/authorization/profiles) for more details. -9. Click `Set up source`. +9. **Marketplace IDs (Optional)** you want to fetch data for. _Note: If Profile IDs are also selected, profiles will be selected if they match the Profile ID **OR** the Marketplace ID._ +10. Click **Set up source**. @@ -41,6 +43,10 @@ To use the [Amazon Ads API](https://advertising.amazon.com/API/docs/en-us), you 1. **Client ID** of your Amazon Ads developer application. See [onboarding process](https://advertising.amazon.com/API/docs/en-us/setting-up/overview) for more details. 2. **Client Secret** of your Amazon Ads developer application. See [onboarding process](https://advertising.amazon.com/API/docs/en-us/setting-up/overview) for more details. 3. **Refresh Token**. See [onboarding process](https://advertising.amazon.com/API/docs/en-us/setting-up/overview) for more details. +4. Select **Region** to pull data from **North America (NA)**, **Europe (EU)**, **Far East (FE)**. See [docs](https://advertising.amazon.com/API/docs/en-us/info/api-overview#api-endpoints) for more details. +5. **Start Date (Optional)** is used for generating reports starting from the specified start date. This should be in YYYY-MM-DD format and not more than 60 days in the past. If a date is not specified, today's date is used. The date is treated in the timezone of the processed profile. +6. **Profile IDs (Optional)** you want to fetch data for. See [docs](https://advertising.amazon.com/API/docs/en-us/concepts/authorization/profiles) for more details. +7. **Marketplace IDs (Optional)** you want to fetch data for. _Note: If Profile IDs are also selected, profiles will be selected if they match the Profile ID **OR** the Marketplace ID._ ## Supported sync modes @@ -60,6 +66,7 @@ This source is capable of syncing the following streams: * [Sponsored Display Ad groups](https://advertising.amazon.com/API/docs/en-us/sponsored-display/3-0/openapi#/Ad%20groups) * [Sponsored Display Product Ads](https://advertising.amazon.com/API/docs/en-us/sponsored-display/3-0/openapi#/Product%20ads) * [Sponsored Display Targetings](https://advertising.amazon.com/API/docs/en-us/sponsored-display/3-0/openapi#/Targeting) +* [Sponsored Display Creatives](https://advertising.amazon.com/API/docs/en-us/sponsored-display/3-0/openapi#/Creatives) * [Sponsored Display Budget Rules](https://advertising.amazon.com/API/docs/en-us/sponsored-display/3-0/openapi/prod#/BudgetRules/GetSDBudgetRulesForAdvertiser) * [Sponsored Products Campaigns](https://advertising.amazon.com/API/docs/en-us/sponsored-display/3-0/openapi#/Campaigns) * [Sponsored Products Ad groups](https://advertising.amazon.com/API/docs/en-us/sponsored-products/2-0/openapi#/Ad%20groups) @@ -78,15 +85,15 @@ This source is capable of syncing the following streams: ## Connector-specific features and highlights -All the reports are generated relative to the target profile' timezone. +All the reports are generated relative to the target profile's timezone. -Campaign reports may sometimes have no data or not presenting in records. This can occur when there are no clicks or views associated with the campaigns on the requested day - [details](https://advertising.amazon.com/API/docs/en-us/guides/reporting/v2/faq#why-is-my-report-empty). +Campaign reports may sometimes have no data or may not be presenting in records. This can occur when there are no clicks or views associated with the campaigns on the requested day - [details](https://advertising.amazon.com/API/docs/en-us/guides/reporting/v2/faq#why-is-my-report-empty). -Report data synchronization only cover the last 60 days - [details](https://advertising.amazon.com/API/docs/en-us/reference/1/reports#parameters). +Report data synchronization only covers the last 60 days - [details](https://advertising.amazon.com/API/docs/en-us/reference/1/reports#parameters). ## Performance considerations -Information about expected report generation waiting time you may find [here](https://advertising.amazon.com/API/docs/en-us/get-started/developer-notes). +Information about expected report generation waiting time can be found [here](https://advertising.amazon.com/API/docs/en-us/get-started/developer-notes). ### Data type mapping @@ -103,6 +110,15 @@ Information about expected report generation waiting time you may find [here](ht | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| 4.0.1 | 2023-12-28 | [33833](https://github.com/airbytehq/airbyte/pull/33833) | Updated oauth spec to put region, so we can choose oauth consent url based on it | +| 4.0.0 | 2023-12-28 | [33817](https://github.com/airbytehq/airbyte/pull/33817) | Fix schema for streams: `SponsoredBrandsAdGroups` and `SponsoredBrandsKeywords` | +| 3.4.2 | 2023-12-12 | [33361](https://github.com/airbytehq/airbyte/pull/33361) | Fix unexpected crash when handling error messages which don't have `requestId` field | +| 3.4.1 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 3.4.0 | 2023-06-09 | [25913](https://github.com/airbytehq/airbyte/pull/26203) | Add Stream `DisplayCreatives` | +| 3.3.0 | 2023-09-22 | [30679](https://github.com/airbytehq/airbyte/pull/30679) | Fix unexpected column for `SponsoredProductCampaigns` and `SponsoredBrandsKeywords` | +| 3.2.0 | 2023-09-18 | [30517](https://github.com/airbytehq/airbyte/pull/30517) | Add suggested streams; fix unexpected column issue | +| 3.1.2 | 2023-08-16 | [29233](https://github.com/airbytehq/airbyte/pull/29233) | Add filter for Marketplace IDs | +| 3.1.1 | 2023-08-28 | [29900](https://github.com/airbytehq/airbyte/pull/29900) | Add 404 handling for no assotiated with bid ad groups | | 3.1.0 | 2023-08-08 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Add `T00030` tactic support for `sponsored_display_report_stream` | | 3.0.0 | 2023-07-24 | [27868](https://github.com/airbytehq/airbyte/pull/27868) | Fix attribution report stream schemas | | 2.3.1 | 2023-07-11 | [28155](https://github.com/airbytehq/airbyte/pull/28155) | Bugfix: validation error when record values are missing | @@ -146,4 +162,4 @@ Information about expected report generation waiting time you may find [here](ht | 0.1.3 | 2021-12-28 | [8388](https://github.com/airbytehq/airbyte/pull/8388) | Add retry if recoverable error occured for reporting stream processing | | 0.1.2 | 2021-10-01 | [6367](https://github.com/airbytehq/airbyte/pull/6461) | Add option to pull data for different regions. Add option to choose profiles we want to pull data. Add lookback | | 0.1.1 | 2021-09-22 | [6367](https://github.com/airbytehq/airbyte/pull/6367) | Add seller and vendor filters to profiles stream | -| 0.1.0 | 2021-08-13 | [5023](https://github.com/airbytehq/airbyte/pull/5023) | Initial version | +| 0.1.0 | 2021-08-13 | [5023](https://github.com/airbytehq/airbyte/pull/5023) | Initial version | \ No newline at end of file diff --git a/docs/integrations/sources/amazon-seller-partner-migrations.md b/docs/integrations/sources/amazon-seller-partner-migrations.md new file mode 100644 index 000000000000..5dc3da1be61a --- /dev/null +++ b/docs/integrations/sources/amazon-seller-partner-migrations.md @@ -0,0 +1,65 @@ +# Amazon Seller Partner Migration Guide + +## Upgrading to 3.0.0 + +Streams `GET_FLAT_FILE_ALL_ORDERS_DATA_BY_ORDER_DATE_GENERAL` and `GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL` now have updated schemas. + +The following streams now have date-time formatted fields: + +| Stream | Affected fields | Format change | +|-----------------------------------------------|-------------------------------------------------------------------------------|----------------------------------------------------------------------| +| `GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL` | `estimated-arrival-date` | `string YYYY-MM-DDTHH:mm:ssZ` -> `date-time YYYY-MM-DDTHH:mm:ssZ` | +| `GET_LEDGER_DETAIL_VIEW_DATA` | `Date and Time` | `string YYYY-MM-DDTHH:mm:ssZ` -> `date-time YYYY-MM-DDTHH:mm:ssZ` | +| `GET_MERCHANTS_LISTINGS_FYP_REPORT` | `Status Change Date` | `string MMM D[,] YYYY` -> `date-time YYYY-MM-DD` | +| `GET_STRANDED_INVENTORY_UI_DATA` | `Date-to-take-auto-removal` | `string YYYY-MM-DDTHH:mm:ssZ` -> `date-time YYYY-MM-DDTHH:mm:ssZ` | +| `GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE` | `settlement-start-date`, `settlement-end-date`, `deposit-date`, `posted-date` | `string YYYY-MM-DDTHH:mm:ssZ` -> `date-time YYYY-MM-DDTHH:mm:ssZ` | +| `GET_MERCHANT_LISTINGS_ALL_DATA` | `open-date` | `string YYYY-MM-DD HH:mm:ss ZZZ` -> `date-time YYYY-MM-DDTHH:mm:ssZ` | +| `GET_MERCHANT_LISTINGS_DATA` | `open-date` | `string YYYY-MM-DD HH:mm:ss ZZZ` -> `date-time YYYY-MM-DDTHH:mm:ssZ` | +| `GET_MERCHANT_LISTINGS_INACTIVE_DATA` | `open-date` | `string YYYY-MM-DD HH:mm:ss ZZZ` -> `date-time YYYY-MM-DDTHH:mm:ssZ` | +| `GET_MERCHANT_LISTINGS_DATA_BACK_COMPAT` | `open-date` | `string YYYY-MM-DD HH:mm:ss ZZZ` -> `date-time YYYY-MM-DDTHH:mm:ssZ` | + + +Users will need to refresh the source schemas and reset these streams after upgrading. + +### Refresh affected schemas and reset data + +1. Select **Connections** in the main navbar. + 1. Select the connection(s) affected by the update. +2. Select the **Replication** tab. + 1. Select **Refresh source schema**. + 2. Select **OK**. +```note +Any detected schema changes will be listed for your review. +``` +3. Select **Save changes** at the bottom of the page. + 1. Ensure the **Reset affected streams** option is checked. +```note +Depending on destination type you may not be prompted to reset your data. +``` +4. Select **Save connection**. +```note +This will reset the data in your destination and initiate a fresh sync. +``` + +For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). + + +## Upgrading to 2.0.0 + +This change removes Brand Analytics and permanently removes deprecated FBA reports (from Airbyte Cloud). +Customers who have those streams must refresh their schema OR disable the following streams: +* `GET_BRAND_ANALYTICS_MARKET_BASKET_REPORT` +* `GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT` +* `GET_BRAND_ANALYTICS_REPEAT_PURCHASE_REPORT` +* `GET_BRAND_ANALYTICS_ALTERNATE_PURCHASE_REPORT` +* `GET_BRAND_ANALYTICS_ITEM_COMPARISON_REPORT` +* `GET_SALES_AND_TRAFFIC_REPORT` +* `GET_VENDOR_SALES_REPORT` +* `GET_VENDOR_INVENTORY_REPORT` + +Customers, who have the following streams, will have to disable them: +* `GET_FBA_FULFILLMENT_INVENTORY_ADJUSTMENTS_DATA` +* `GET_FBA_FULFILLMENT_CURRENT_INVENTORY_DATA` +* `GET_FBA_FULFILLMENT_INVENTORY_RECEIPTS_DATA` +* `GET_FBA_FULFILLMENT_INVENTORY_SUMMARY_DATA` +* `GET_FBA_FULFILLMENT_MONTHLY_INVENTORY_DATA` diff --git a/docs/integrations/sources/amazon-seller-partner.md b/docs/integrations/sources/amazon-seller-partner.md index bdef35431f89..7bcb6d06f3ed 100644 --- a/docs/integrations/sources/amazon-seller-partner.md +++ b/docs/integrations/sources/amazon-seller-partner.md @@ -1,47 +1,70 @@ # Amazon Seller Partner -This page guides you through the process of setting up the Amazon Seller Partner source connector. +This page contains the setup guide and reference information for the Amazon Seller Partner source connector. ## Prerequisites +- Amazon Seller Partner account + + + +**For Airbyte Cloud:** + - AWS Environment - AWS Region -- AWS Access Key -- AWS Secret Key -- Role ARN -- LWA Client ID (LWA App ID)** -- LWA Client Secret** -- Refresh token** -- Replication Start Date +- AWS Seller Partner Account Type +- Granted OAuth access + + + -**not required for Airbyte Cloud +**For Airbyte Open Source:** + +- AWS Environment +- AWS Region +- AWS Seller Partner Account Type +- LWA Client Id +- LWA Client Secret +- Refresh Token + + +## Setup Guide ## Step 1: Set up Amazon Seller Partner -1. [Register](https://developer-docs.amazon.com/sp-api/docs/registering-your-application) Amazon Seller Partner application. - - The application must be published as Amazon does not allow external parties such as Airbyte to access draft applications. -2. [Create](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html) IAM user. + + +**Airbyte Open Source setup steps** + +- [Register](https://developer-docs.amazon.com/sp-api/docs/registering-your-application) Amazon Seller Partner application. The application must be published as Amazon does not allow external parties such as Airbyte to access draft applications. + + ## Step 2: Set up the source connector in Airbyte **For Airbyte Cloud:** -1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. -2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**. -3. On the source setup page, select **Amazon Seller Partner** from the Source type dropdown and enter a name for this connector. -4. Click `Authenticate your account`. -5. Log in and Authorize to your Amazon Seller Partner account. -6. Paste all other data to required fields using your IAM user. -7. Click `Set up source`. +1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. +2. Click **Sources** and then click **+ New source**. +3. On the Set up the source page, select **Amazon Seller Partner** from the **Source type** dropdown. +4. Enter a name for the Amazon Seller Partner connector. +5. Click `Authenticate your account`. +6. Log in and Authorize to your Amazon Seller Partner account. +7. For `Start Date`, enter the date in `YYYY-MM-DD` format. The data added on and after this date will be replicated. This field is optional - if not provided, the date 2 years ago from today will be used. +8. For `End Date`, enter the date in `YYYY-MM-DD` format. Any data after this date will not be replicated. This field is optional - if not provided, today's date will be used. +9. You can specify report options for each stream using **Report Options** section. Available options can be found in corresponding category [here](https://developer-docs.amazon.com/sp-api/docs/report-type-values). +10. Click `Set up source`. **For Airbyte Open Source:** 1. Using developer application from Step 1, [generate](https://developer-docs.amazon.com/sp-api/docs/self-authorization) refresh token. 2. Go to local Airbyte page. -3. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**. -4. On the Set up the source page, enter the name for the Amazon Seller Partner connector and select **Amazon Seller Partner** from the Source type dropdown. -5. Paste all data to required fields using your IAM user and developer account. -6. Click `Set up source`. +3. On the Set up the source page, select **Amazon Seller Partner** from the **Source type** dropdown. +4. Enter a name for the Amazon Seller Partner connector. +5. For Start Date, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. This field is optional - if not provided, the date 2 years ago from today will be used. +6. For End Date, enter the date in YYYY-MM-DD format. Any data after this date will not be replicated. This field is optional - if not provided, today's date will be used. +7. You can specify report options for each stream using **Report Options** section. Available options can be found in corresponding category [here](https://developer-docs.amazon.com/sp-api/docs/report-type-values). +8. Click `Set up source`. ## Supported sync modes @@ -49,71 +72,73 @@ The Amazon Seller Partner source connector supports the following [sync modes](h - Full Refresh - Incremental -## Performance considerations - -Information about rate limits you may find [here](https://developer-docs.amazon.com/sp-api/docs/usage-plans-and-rate-limits-in-the-sp-api). - ## Supported streams -This source is capable of syncing the following tables and their data: -- [Manage FBA Inventory Reports](https://sellercentral.amazon.com/gp/help/200740930) -- [Removal FBA Order Details Reports](https://sellercentral.amazon.com/gp/help/help.html?itemID=200989110) -- [FBA Shipments Reports](https://sellercentral.amazon.com/gp/help/help.html?itemID=200989100) -- [FBA Replacements Reports](https://sellercentral.amazon.com/help/hub/reference/200453300) -- [FBA Storage Fees Report](https://sellercentral.amazon.com/help/hub/reference/G202086720) -- [Restock Inventory Reports](https://sellercentral.amazon.com/help/hub/reference/202105670) -- [Flat File Actionable Order Data Shipping](https://developer-docs.amazon.com/sp-api/docs/order-reports-attributes#get_flat_file_actionable_order_data_shipping) -- [Flat File Open Listings Reports](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [Flat File Orders Reports](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [Flat File Orders Reports By Last Update](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) \(incremental\) -- [Amazon-Fulfilled Shipments Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [Merchant Listings Reports](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [Vendor Direct Fulfillment Shipping](https://developer-docs.amazon.com/sp-api/docs/vendor-direct-fulfillment-shipping-api-v1-reference) -- [Vendor Inventory Health Reports](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) +- [Active Listings Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-inventory) \(incremental\) +- [All Listings Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-inventory) \(incremental\) +- [Amazon Search Terms Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#brand-analytics-reports) \(only available in OSS, incremental\) +- [Browse Tree Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-browse-tree) \(incremental\) +- [Canceled Listings Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-inventory) \(incremental\) +- [FBA Amazon Fulfilled Inventory Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) \(incremental\) +- [FBA Amazon Fulfilled Shipments Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-sales-reports) \(incremental\) +- [FBA Fee Preview Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-payments-reports) \(incremental\) +- [FBA Manage Inventory](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) \(incremental\) +- [FBA Manage Inventory Health Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) \(incremental\) +- [FBA Multi-Country Inventory Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) \(incremental\) +- [FBA Promotions Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-sales-reports) \(incremental\) +- [FBA Reimbursements Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-payments-reports) \(incremental\) +- [FBA Removal Order Detail Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-removals-reports) \(incremental\) +- [FBA Removal Shipment Detail Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-removals-reports) \(incremental\) +- [FBA Replacements Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-concessions-reports) \(incremental\) +- [FBA Returns Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-concessions-reports) \(incremental\) +- [FBA Storage Fees Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) \(incremental\) +- [FBA Stranded Inventory Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) \(incremental\) +- [Financial Events](https://developer-docs.amazon.com/sp-api/docs/finances-api-reference#get-financesv0financialevents) \(incremental\) +- [Financial Event Groups](https://developer-docs.amazon.com/sp-api/docs/finances-api-reference#get-financesv0financialeventgroups) \(incremental\) +- [Flat File Archived Orders Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-order#order-tracking-reports) \(incremental\) +- [Flat File Feedback Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-performance) \(incremental\) +- [Flat File Orders By Last Update Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-order#order-tracking-reports) \(incremental\) +- [Flat File Orders By Order Date Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-order#order-tracking-reports) \(incremental\) +- [Flat File Returns Report by Return Date](https://developer-docs.amazon.com/sp-api/docs/report-type-values-returns) \(incremental\) +- [Flat File Settlement Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-settlement) \(incremental\) +- [Inactive Listings Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-inventory) \(incremental\) +- [Inventory Ledger Report - Detailed View](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) \(incremental\) +- [Inventory Ledger Report - Summary View](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) \(incremental\) +- [Inventory Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-inventory) \(incremental\) +- [Market Basket Analysis Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#brand-analytics-reports) \(only available in OSS, incremental\) +- [Net Pure Product Margin Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#vendor-retail-analytics-reports) \(incremental\) +- [Open Listings Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-inventory) \(incremental\) - [Orders](https://developer-docs.amazon.com/sp-api/docs/orders-api-v0-reference) \(incremental\) -- [Orders Items](https://developer-docs.amazon.com/sp-api/docs/orders-api-v0-reference#getorderitems) \(incremental\) -- [Seller Feedback Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) \(incremental\) -- [Brand Analytics Alternate Purchase Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values#brand-analytics-reports) -- [Brand Analytics Item Comparison Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values#brand-analytics-reports) -- [Brand Analytics Market Basket Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values#brand-analytics-reports) -- [Brand Analytics Repeat Purchase Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values#brand-analytics-reports) -- [Brand Analytics Search Terms Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values#brand-analytics-reports) -- [Browse tree report](https://github.com/amzn/selling-partner-api-docs/blob/main/references/reports-api/reporttype-values.md#browse-tree-report) -- [Financial Event Groups](https://developer-docs.amazon.com/sp-api/docs/finances-api-reference#get-financesv0financialeventgroups) -- [Financial Events](https://developer-docs.amazon.com/sp-api/docs/finances-api-reference#get-financesv0financialevents) -- [FBA Fee Preview Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [FBA Daily Inventory History Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [FBA Promotions Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [FBA Inventory Adjustments Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [FBA Received Inventory Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [FBA Inventory Event Detail Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [FBA Monthly Inventory History Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [FBA Manage Inventory](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [Subscribe and Save Forecast Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [Subscribe and Save Performance Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [Flat File Archived Orders Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [Flat File Returns Report by Return Date](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [Canceled Listings Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [Active Listings Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [Open Listings Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [Suppressed Listings Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [Inactive Listings Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [FBA Stranded Inventory Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [XML Orders By Order Date Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [Inventory Ledger Report - Detailed View](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [FBA Manage Inventory Health Report](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [Inventory Ledger Report - Summary View](https://developer-docs.amazon.com/sp-api/docs/reports-api-v2021-06-30-reference) -- [FBA Reimbursements Report](https://sellercentral.amazon.com/help/hub/reference/G200732720) -- [Order Data Shipping Report](https://developer-docs.amazon.com/sp-api/docs/order-reports-attributes#get_order_report_data_shipping) +- [Order Items](https://developer-docs.amazon.com/sp-api/docs/orders-api-v0-reference#getorderitems) \(incremental\) +- [Rapid Retail Analytics Inventory Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#vendor-retail-analytics-reports) \(incremental\) +- [Repeat Purchase](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#brand-analytics-reports) \(only available in OSS, incremental\) +- [Restock Inventory Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-inventory-reports) \(incremental\) +- [Sales and Traffic Business Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#seller-retail-analytics-reports) \(incremental\) +- [Scheduled XML Order Report (Shipping)](https://developer-docs.amazon.com/sp-api/docs/report-type-values-order#order-reports) \(incremental\) +- [Subscribe and Save Forecast Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-subscribe-and-save-reports) \(incremental\) +- [Subscribe and Save Performance Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-fba#fba-subscribe-and-save-reports) \(incremental\) +- [Suppressed Listings Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-inventory) \(incremental\) +- [Unshipped Orders Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-order#order-reports) \(incremental\) +- [Vendor Direct Fulfillment Shipping](https://developer-docs.amazon.com/sp-api/docs/vendor-direct-fulfillment-shipping-api-v1-reference) \(incremental\) +- [Vendor Inventory Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#vendor-retail-analytics-reports) \(incremental\) +- [Vendor Sales Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#vendor-retail-analytics-reports) \(incremental\) +- [Vendor Traffic Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-analytics#vendor-retail-analytics-reports) \(incremental\) +- [XML Orders By Order Date Report](https://developer-docs.amazon.com/sp-api/docs/report-type-values-order#order-tracking-reports) \(incremental\) ## Report options -Make sure to configure the [required parameters](https://developer-docs.amazon.com/sp-api/docs/report-type-values) in the report options setting for the reports configured. +Report options can be assigned on a per-stream basis that alter the behavior when generating a report. +For the full list, refer to Amazon’s report type values [documentation](https://developer-docs.amazon.com/sp-api/docs/report-type-values). -For `GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL` and `GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE` streams maximum value for `period_in_days` 30 days and 60 days. +Certain report types have required parameters that must be defined. +For `GET_AMAZON_FULFILLED_SHIPMENTS_DATA_GENERAL` and `GET_FLAT_FILE_RETURNS_DATA_BY_RETURN_DATE` streams maximum value for `period_in_days` 30 days and 60 days. So, for any value that exceeds the limit, the `period_in_days` will be automatically reduced to the limit for the stream. -## Data type mapping +## Performance considerations + +Information about rate limits you may find [here](https://developer-docs.amazon.com/sp-api/docs/usage-plans-and-rate-limits-in-the-sp-api). + +## Data type map | Integration Type | Airbyte Type | | :----------------------- | :----------- | @@ -124,52 +149,66 @@ So, for any value that exceeds the limit, the `period_in_days` will be automatic | `array` | `array` | | `object` | `object` | - ## Changelog -| Version | Date | Pull Request | Subject | -|:---------|:-----------|:--------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `1.5.1` | 2023-08-18 | [\#29255](https://github.com/airbytehq/airbyte/pull/29255) | role_arn is optional on UI but not really on the backend blocking connector set up using oauth | -| `1.5.0` | 2023-08-08 | [\#29054](https://github.com/airbytehq/airbyte/pull/29054) | Add new stream `OrderItems` | -| `1.4.1` | 2023-07-25 | [\#27050](https://github.com/airbytehq/airbyte/pull/27050) | Fix - non vendor accounts connector create/check issue | -| `1.4.0` | 2023-07-21 | [\#27110](https://github.com/airbytehq/airbyte/pull/27110) | Add `GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING` and `GET_ORDER_REPORT_DATA_SHIPPING` streams | -| `1.3.0` | 2023-06-09 | [\#27110](https://github.com/airbytehq/airbyte/pull/27110) | Removed `app_id` from `InputConfiguration`, refactored `spec` | -| `1.2.0` | 2023-05-23 | [\#22503](https://github.com/airbytehq/airbyte/pull/22503) | Enabled stream attribute customization from Source configuration | -| `1.1.0` | 2023-04-21 | [\#23605](https://github.com/airbytehq/airbyte/pull/23605) | Add FBA Reimbursement Report stream | -| `1.0.1` | 2023-03-15 | [\#24098](https://github.com/airbytehq/airbyte/pull/24098) | Add Belgium Marketplace | -| `1.0.0` | 2023-03-13 | [\#23980](https://github.com/airbytehq/airbyte/pull/23980) | Make `app_id` required. Increase `end_date` gap up to 5 minutes from now for Finance streams. Fix connection check failure when trying to connect to Amazon Vendor Central accounts | -| `0.2.33` | 2023-03-01 | [\#23606](https://github.com/airbytehq/airbyte/pull/23606) | Implement reportOptions for all missing reports and refactor | -| `0.2.32` | 2022-02-21 | [\#23300](https://github.com/airbytehq/airbyte/pull/23300) | Make AWS Access Key, AWS Secret Access and Role ARN optional | -| `0.2.31` | 2022-01-10 | [\#16430](https://github.com/airbytehq/airbyte/pull/16430) | Implement slicing for report streams | -| `0.2.30` | 2022-12-28 | [\#20896](https://github.com/airbytehq/airbyte/pull/20896) | Validate connections without orders data | -| `0.2.29` | 2022-11-18 | [\#19581](https://github.com/airbytehq/airbyte/pull/19581) | Use user provided end date for GET_SALES_AND_TRAFFIC_REPORT | -| `0.2.28` | 2022-10-20 | [\#18283](https://github.com/airbytehq/airbyte/pull/18283) | Added multiple (22) report types | -| `0.2.26` | 2022-09-24 | [\#16629](https://github.com/airbytehq/airbyte/pull/16629) | Report API version to 2021-06-30, added multiple (5) report types | -| `0.2.25` | 2022-07-27 | [\#15063](https://github.com/airbytehq/airbyte/pull/15063) | Add Restock Inventory Report | -| `0.2.24` | 2022-07-12 | [\#14625](https://github.com/airbytehq/airbyte/pull/14625) | Add FBA Storage Fees Report | -| `0.2.23` | 2022-06-08 | [\#13604](https://github.com/airbytehq/airbyte/pull/13604) | Add new streams: Fullfiments returns and Settlement reports | -| `0.2.22` | 2022-06-15 | [\#13633](https://github.com/airbytehq/airbyte/pull/13633) | Fix - handle start date for financial stream | -| `0.2.21` | 2022-06-01 | [\#13364](https://github.com/airbytehq/airbyte/pull/13364) | Add financial streams | -| `0.2.20` | 2022-05-30 | [\#13059](https://github.com/airbytehq/airbyte/pull/13059) | Add replication end date to config | -| `0.2.19` | 2022-05-24 | [\#13119](https://github.com/airbytehq/airbyte/pull/13119) | Add OAuth2.0 support | -| `0.2.18` | 2022-05-06 | [\#12663](https://github.com/airbytehq/airbyte/pull/12663) | Add GET_XML_BROWSE_TREE_DATA report | -| `0.2.17` | 2022-05-19 | [\#12946](https://github.com/airbytehq/airbyte/pull/12946) | Add throttling exception managing in Orders streams | -| `0.2.16` | 2022-05-04 | [\#12523](https://github.com/airbytehq/airbyte/pull/12523) | allow to use IAM user arn or IAM role | -| `0.2.15` | 2022-01-25 | [\#9789](https://github.com/airbytehq/airbyte/pull/9789) | Add stream FbaReplacementsReports | -| `0.2.14` | 2022-01-19 | [\#9621](https://github.com/airbytehq/airbyte/pull/9621) | Add GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL report | -| `0.2.13` | 2022-01-18 | [\#9581](https://github.com/airbytehq/airbyte/pull/9581) | Change createdSince parameter to dataStartTime | -| `0.2.12` | 2022-01-05 | [\#9312](https://github.com/airbytehq/airbyte/pull/9312) | Add all remaining brand analytics report streams | -| `0.2.11` | 2022-01-05 | [\#9115](https://github.com/airbytehq/airbyte/pull/9115) | Fix reading only 100 orders | -| `0.2.10` | 2021-12-31 | [\#9236](https://github.com/airbytehq/airbyte/pull/9236) | Fix NoAuth deprecation warning | -| `0.2.9` | 2021-12-30 | [\#9212](https://github.com/airbytehq/airbyte/pull/9212) | Normalize GET_SELLER_FEEDBACK_DATA header field names | -| `0.2.8` | 2021-12-22 | [\#8810](https://github.com/airbytehq/airbyte/pull/8810) | Fix GET_SELLER_FEEDBACK_DATA Date cursor field format | -| `0.2.7` | 2021-12-21 | [\#9002](https://github.com/airbytehq/airbyte/pull/9002) | Extract REPORTS_MAX_WAIT_SECONDS to configurable parameter | -| `0.2.6` | 2021-12-10 | [\#8179](https://github.com/airbytehq/airbyte/pull/8179) | Add GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT report | -| `0.2.5` | 2021-12-06 | [\#8425](https://github.com/airbytehq/airbyte/pull/8425) | Update title, description fields in spec | -| `0.2.4` | 2021-11-08 | [\#8021](https://github.com/airbytehq/airbyte/pull/8021) | Added GET_SELLER_FEEDBACK_DATA report with incremental sync capability | -| `0.2.3` | 2021-11-08 | [\#7828](https://github.com/airbytehq/airbyte/pull/7828) | Remove datetime format from all streams | -| `0.2.2` | 2021-11-08 | [\#7752](https://github.com/airbytehq/airbyte/pull/7752) | Change `check_connection` function to use stream Orders | -| `0.2.1` | 2021-09-17 | [\#5248](https://github.com/airbytehq/airbyte/pull/5248) | Added `extra stream` support. Updated `reports streams` logics | -| `0.2.0` | 2021-08-06 | [\#4863](https://github.com/airbytehq/airbyte/pull/4863) | Rebuild source with `airbyte-cdk` | -| `0.1.3` | 2021-06-23 | [\#4288](https://github.com/airbytehq/airbyte/pull/4288) | Bugfix failing `connection check` | -| `0.1.2` | 2021-06-15 | [\#4108](https://github.com/airbytehq/airbyte/pull/4108) | Fixed: Sync fails with timeout when create report is CANCELLED` | +| Version | Date | Pull Request | Subject | +|:---------|:-----------|:------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `3.1.0` | 2024-01-17 | [\#34283](https://github.com/airbytehq/airbyte/pull/34283) | Delete deprecated streams | +| `3.0.1` | 2023-12-22 | [\#33741](https://github.com/airbytehq/airbyte/pull/33741) | Improve report streams performance | +| `3.0.0` | 2023-12-12 | [\#32977](https://github.com/airbytehq/airbyte/pull/32977) | Make all streams incremental | +| `2.5.0` | 2023-11-27 | [\#32505](https://github.com/airbytehq/airbyte/pull/32505) | Make report options configurable via UI | +| `2.4.0` | 2023-11-23 | [\#32738](https://github.com/airbytehq/airbyte/pull/32738) | Add `GET_VENDOR_NET_PURE_PRODUCT_MARGIN_REPORT`, `GET_VENDOR_REAL_TIME_INVENTORY_REPORT`, and `GET_VENDOR_TRAFFIC_REPORT` streams | +| `2.3.0` | 2023-11-22 | [\#32541](https://github.com/airbytehq/airbyte/pull/32541) | Make `GET_AFN_INVENTORY_DATA`, `GET_AFN_INVENTORY_DATA_BY_COUNTRY`, and `GET_V2_SETTLEMENT_REPORT_DATA_FLAT_FILE` streams incremental | +| `2.2.0` | 2023-11-21 | [\#32639](https://github.com/airbytehq/airbyte/pull/32639) | Make start date optional, if start date is not provided, date 2 years ago from today will be used | +| `2.1.1` | 2023-11-21 | [\#32560](https://github.com/airbytehq/airbyte/pull/32560) | Silently exit sync if the retry attempts were unsuccessful | +| `2.1.0` | 2023-11-21 | [\#32591](https://github.com/airbytehq/airbyte/pull/32591) | Add new fields to GET_LEDGER_DETAIL_VIEW_DATA, GET_FBA_INVENTORY_PLANNING_DATA and Orders schemas | +| `2.0.2` | 2023-11-17 | [\#32462](https://github.com/airbytehq/airbyte/pull/32462) | Remove Max time option from specification; set default waiting time for reports to 1 hour | +| `2.0.1` | 2023-11-16 | [\#32550](https://github.com/airbytehq/airbyte/pull/32550) | Fix the OAuth flow | +| `2.0.0` | 2023-11-23 | [\#32355](https://github.com/airbytehq/airbyte/pull/32355) | Remove Brand Analytics from Airbyte Cloud, permanently remove deprecated FBA reports | +| `1.6.2` | 2023-11-14 | [\#32508](https://github.com/airbytehq/airbyte/pull/32508) | Do not use AWS signature as it is no longer required by the Amazon API | +| `1.6.1` | 2023-11-13 | [\#32457](https://github.com/airbytehq/airbyte/pull/32457) | Fix report decompression | +| `1.6.0` | 2023-11-09 | [\#32259](https://github.com/airbytehq/airbyte/pull/32259) | mark "aws_secret_key" and "aws_access_key" as required in specification; update schema for stream `Orders` | +| `1.5.1` | 2023-08-18 | [\#29255](https://github.com/airbytehq/airbyte/pull/29255) | role_arn is optional on UI but not really on the backend blocking connector set up using oauth | +| `1.5.0` | 2023-08-08 | [\#29054](https://github.com/airbytehq/airbyte/pull/29054) | Add new stream `OrderItems` | +| `1.4.1` | 2023-07-25 | [\#27050](https://github.com/airbytehq/airbyte/pull/27050) | Fix - non vendor accounts connector create/check issue | +| `1.4.0` | 2023-07-21 | [\#27110](https://github.com/airbytehq/airbyte/pull/27110) | Add `GET_FLAT_FILE_ACTIONABLE_ORDER_DATA_SHIPPING` and `GET_ORDER_REPORT_DATA_SHIPPING` streams | +| `1.3.0` | 2023-06-09 | [\#27110](https://github.com/airbytehq/airbyte/pull/27110) | Removed `app_id` from `InputConfiguration`, refactored `spec` | +| `1.2.0` | 2023-05-23 | [\#22503](https://github.com/airbytehq/airbyte/pull/22503) | Enabled stream attribute customization from Source configuration | +| `1.1.0` | 2023-04-21 | [\#23605](https://github.com/airbytehq/airbyte/pull/23605) | Add FBA Reimbursement Report stream | +| `1.0.1` | 2023-03-15 | [\#24098](https://github.com/airbytehq/airbyte/pull/24098) | Add Belgium Marketplace | +| `1.0.0` | 2023-03-13 | [\#23980](https://github.com/airbytehq/airbyte/pull/23980) | Make `app_id` required. Increase `end_date` gap up to 5 minutes from now for Finance streams. Fix connection check failure when trying to connect to Amazon Vendor Central accounts | +| `0.2.33` | 2023-03-01 | [\#23606](https://github.com/airbytehq/airbyte/pull/23606) | Implement reportOptions for all missing reports and refactor | +| `0.2.32` | 2022-02-21 | [\#23300](https://github.com/airbytehq/airbyte/pull/23300) | Make AWS Access Key, AWS Secret Access and Role ARN optional | +| `0.2.31` | 2022-01-10 | [\#16430](https://github.com/airbytehq/airbyte/pull/16430) | Implement slicing for report streams | +| `0.2.30` | 2022-12-28 | [\#20896](https://github.com/airbytehq/airbyte/pull/20896) | Validate connections without orders data | +| `0.2.29` | 2022-11-18 | [\#19581](https://github.com/airbytehq/airbyte/pull/19581) | Use user provided end date for GET_SALES_AND_TRAFFIC_REPORT | +| `0.2.28` | 2022-10-20 | [\#18283](https://github.com/airbytehq/airbyte/pull/18283) | Added multiple (22) report types | +| `0.2.26` | 2022-09-24 | [\#16629](https://github.com/airbytehq/airbyte/pull/16629) | Report API version to 2021-06-30, added multiple (5) report types | +| `0.2.25` | 2022-07-27 | [\#15063](https://github.com/airbytehq/airbyte/pull/15063) | Add Restock Inventory Report | +| `0.2.24` | 2022-07-12 | [\#14625](https://github.com/airbytehq/airbyte/pull/14625) | Add FBA Storage Fees Report | +| `0.2.23` | 2022-06-08 | [\#13604](https://github.com/airbytehq/airbyte/pull/13604) | Add new streams: Fullfiments returns and Settlement reports | +| `0.2.22` | 2022-06-15 | [\#13633](https://github.com/airbytehq/airbyte/pull/13633) | Fix - handle start date for financial stream | +| `0.2.21` | 2022-06-01 | [\#13364](https://github.com/airbytehq/airbyte/pull/13364) | Add financial streams | +| `0.2.20` | 2022-05-30 | [\#13059](https://github.com/airbytehq/airbyte/pull/13059) | Add replication end date to config | +| `0.2.19` | 2022-05-24 | [\#13119](https://github.com/airbytehq/airbyte/pull/13119) | Add OAuth2.0 support | +| `0.2.18` | 2022-05-06 | [\#12663](https://github.com/airbytehq/airbyte/pull/12663) | Add GET_XML_BROWSE_TREE_DATA report | +| `0.2.17` | 2022-05-19 | [\#12946](https://github.com/airbytehq/airbyte/pull/12946) | Add throttling exception managing in Orders streams | +| `0.2.16` | 2022-05-04 | [\#12523](https://github.com/airbytehq/airbyte/pull/12523) | allow to use IAM user arn or IAM role | +| `0.2.15` | 2022-01-25 | [\#9789](https://github.com/airbytehq/airbyte/pull/9789) | Add stream FbaReplacementsReports | +| `0.2.14` | 2022-01-19 | [\#9621](https://github.com/airbytehq/airbyte/pull/9621) | Add GET_FLAT_FILE_ALL_ORDERS_DATA_BY_LAST_UPDATE_GENERAL report | +| `0.2.13` | 2022-01-18 | [\#9581](https://github.com/airbytehq/airbyte/pull/9581) | Change createdSince parameter to dataStartTime | +| `0.2.12` | 2022-01-05 | [\#9312](https://github.com/airbytehq/airbyte/pull/9312) | Add all remaining brand analytics report streams | +| `0.2.11` | 2022-01-05 | [\#9115](https://github.com/airbytehq/airbyte/pull/9115) | Fix reading only 100 orders | +| `0.2.10` | 2021-12-31 | [\#9236](https://github.com/airbytehq/airbyte/pull/9236) | Fix NoAuth deprecation warning | +| `0.2.9` | 2021-12-30 | [\#9212](https://github.com/airbytehq/airbyte/pull/9212) | Normalize GET_SELLER_FEEDBACK_DATA header field names | +| `0.2.8` | 2021-12-22 | [\#8810](https://github.com/airbytehq/airbyte/pull/8810) | Fix GET_SELLER_FEEDBACK_DATA Date cursor field format | +| `0.2.7` | 2021-12-21 | [\#9002](https://github.com/airbytehq/airbyte/pull/9002) | Extract REPORTS_MAX_WAIT_SECONDS to configurable parameter | +| `0.2.6` | 2021-12-10 | [\#8179](https://github.com/airbytehq/airbyte/pull/8179) | Add GET_BRAND_ANALYTICS_SEARCH_TERMS_REPORT report | +| `0.2.5` | 2021-12-06 | [\#8425](https://github.com/airbytehq/airbyte/pull/8425) | Update title, description fields in spec | +| `0.2.4` | 2021-11-08 | [\#8021](https://github.com/airbytehq/airbyte/pull/8021) | Added GET_SELLER_FEEDBACK_DATA report with incremental sync capability | +| `0.2.3` | 2021-11-08 | [\#7828](https://github.com/airbytehq/airbyte/pull/7828) | Remove datetime format from all streams | +| `0.2.2` | 2021-11-08 | [\#7752](https://github.com/airbytehq/airbyte/pull/7752) | Change `check_connection` function to use stream Orders | +| `0.2.1` | 2021-09-17 | [\#5248](https://github.com/airbytehq/airbyte/pull/5248) | Added `extra stream` support. Updated `reports streams` logics | +| `0.2.0` | 2021-08-06 | [\#4863](https://github.com/airbytehq/airbyte/pull/4863) | Rebuild source with `airbyte-cdk` | +| `0.1.3` | 2021-06-23 | [\#4288](https://github.com/airbytehq/airbyte/pull/4288) | Bugfix failing `connection check` | +| `0.1.2` | 2021-06-15 | [\#4108](https://github.com/airbytehq/airbyte/pull/4108) | Fixed: Sync fails with timeout when create report is CANCELLED` | diff --git a/docs/integrations/sources/amazon-sqs.md b/docs/integrations/sources/amazon-sqs.md index a71586ffced3..78fa87bc627f 100644 --- a/docs/integrations/sources/amazon-sqs.md +++ b/docs/integrations/sources/amazon-sqs.md @@ -6,19 +6,20 @@ This source will sync messages from an [SQS Queue](https://docs.aws.amazon.com/s ### Output schema -This source will output one stream for the configured SQS Queue. -The stream record data will have three fields: -* id (a UUIDv4 as a STRING) -* body (message body as a STRING) -* attributes (attributes of the messages as an OBJECT or NULL) +This source will output one stream for the configured SQS Queue. The stream record data will have +three fields: + +- id (a UUIDv4 as a STRING) +- body (message body as a STRING) +- attributes (attributes of the messages as an OBJECT or NULL) ### Features -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | yes | | -| Incremental Sync | no | | -| Namespaces | no | | +| Feature | Supported?\(Yes/No\) | Notes | +| :---------------- | :------------------- | :---- | +| Full Refresh Sync | yes | | +| Incremental Sync | no | | +| Namespaces | no | | ### Performance considerations @@ -26,63 +27,77 @@ The stream record data will have three fields: ### Requirements -* AWS IAM Access Key -* AWS IAM Secret Key -* AWS SQS Queue +- AWS IAM Access Key +- AWS IAM Secret Key +- AWS SQS Queue ### Properties -Required properties are 'Queue URL', 'AWS Region' and 'Delete Messages After Read' as noted in **bold** below. - -* **Queue URL** (STRING) - * The full AWS endpoint URL of the queue e.g. `https://sqs.eu-west-1.amazonaws.com/1234567890/example-queue-url` -* **AWS Region** (STRING) - * The region code for the SQS Queue e.g. eu-west-1 -* **Delete Messages After Read** (BOOLEAN) - * **WARNING:** Setting this option to TRUE can result in data loss, do not enable this option unless you understand the risk. See the **Data loss warning** section below. - * Should the message be deleted from the SQS Queue after being read? This prevents the message being read more than once - * By default messages are NOT deleted, thus can be re-read after the `Message Visibility Timeout` - * Default: False -* Max Batch Size (INTEGER) - * The max amount of messages to consume in a single poll e.g. 5 - * Minimum of 1, maximum of 10 - * Default: 10 -* Max Wait Time (INTEGER) - * The max amount of time (in seconds) to poll for messages before commiting a batch (or timing out) unless we fill a batch (as per `Max Batch Size`) - * Minimum of 1, maximum of 20 - * Default: 20 -* Message Attributes To Return (STRING) - * A comma separated list of Attributes to return for each message - * Default: All -* Message Visibility Timeout (INTEGER) - * After a message is read, how much time (in seconds) should the message be hidden from other consumers - * After this timeout, the message is not deleted and can be re-read - * Default: 30 -* AWS IAM Access Key ID (STRING) - * The Access Key for the IAM User with permissions on this Queue - * If `Delete Messages After Read` is `false` then only `sqs:ReceiveMessage` - * If `Delete Messages After Read` is `true` then `sqs:DeleteMessage` is also needed -* AWS IAM Secret Key (STRING) - * The Secret Key for the IAM User with permissions on this Queue +Required properties are 'Queue URL', 'AWS Region' and 'Delete Messages After Read' as noted in +**bold** below. + +- **Queue URL** (STRING) + - The full AWS endpoint URL of the queue e.g. + `https://sqs.eu-west-1.amazonaws.com/1234567890/example-queue-url` +- **AWS Region** (STRING) + - The region code for the SQS Queue e.g. eu-west-1 +- **Delete Messages After Read** (BOOLEAN) + - **WARNING:** Setting this option to TRUE can result in data loss, do not enable this option + unless you understand the risk. See the **Data loss warning** section below. + - Should the message be deleted from the SQS Queue after being read? This prevents the message + being read more than once + - By default messages are NOT deleted, thus can be re-read after the `Message Visibility Timeout` + - Default: False +- Max Batch Size (INTEGER) + - The max amount of messages to consume in a single poll e.g. 5 + - Minimum of 1, maximum of 10 + - Default: 10 +- Max Wait Time (INTEGER) + - The max amount of time (in seconds) to poll for messages before commiting a batch (or timing + out) unless we fill a batch (as per `Max Batch Size`) + - Minimum of 1, maximum of 20 + - Default: 20 +- Message Attributes To Return (STRING) + - A comma separated list of Attributes to return for each message + - Default: All +- Message Visibility Timeout (INTEGER) + - After a message is read, how much time (in seconds) should the message be hidden from other + consumers + - After this timeout, the message is not deleted and can be re-read + - Default: 30 +- AWS IAM Access Key ID (STRING) + - The Access Key for the IAM User with permissions on this Queue + - If `Delete Messages After Read` is `false` then only `sqs:ReceiveMessage` + - If `Delete Messages After Read` is `true` then `sqs:DeleteMessage` is also needed +- AWS IAM Secret Key (STRING) + - The Secret Key for the IAM User with permissions on this Queue ### Data loss warning -When enabling **Delete Messages After Read**, the Source will delete messages from the SQS Queue after reading them. The message is deleted *after* the configured Destination takes the message from this Source, but makes no guarentee that the downstream destination has commited/persisted the message. This means that it is possible for the Airbyte Destination to read the message from the Source, the Source deletes the message, then the downstream application fails - resulting in the message being lost permanently. +When enabling **Delete Messages After Read**, the Source will delete messages from the SQS Queue +after reading them. The message is deleted _after_ the configured Destination takes the message from +this Source, but makes no guarentee that the downstream destination has commited/persisted the +message. This means that it is possible for the Airbyte Destination to read the message from the +Source, the Source deletes the message, then the downstream application fails - resulting in the +message being lost permanently. Extra care should be taken to understand this risk before enabling this option. ### Setup guide -* [Create IAM Keys](https://aws.amazon.com/premiumsupport/knowledge-center/create-access-key/) -* [Create SQS Queue](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-getting-started.html#step-create-queue) +- [Create IAM Keys](https://aws.amazon.com/premiumsupport/knowledge-center/create-access-key/) +- [Create SQS Queue](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-getting-started.html#step-create-queue) > **NOTE**: -> * If `Delete Messages After Read` is `false` then the IAM User needs only `sqs:ReceiveMessage` in the AWS IAM Policy -> * If `Delete Messages After Read` is `true` then both `sqs:ReceiveMessage` and `sqs:DeleteMessage` are needed in the AWS IAM Policy +> +> - If `Delete Messages After Read` is `false` then the IAM User needs only `sqs:ReceiveMessage` in +> the AWS IAM Policy +> - If `Delete Messages After Read` is `true` then both `sqs:ReceiveMessage` and `sqs:DeleteMessage` +> are needed in the AWS IAM Policy ## CHANGELOG -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| `0.1.0` | 2021-10-10 | [\#0000](https://github.com/airbytehq/airbyte/pull/0000) | `Initial version` | - +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :-------------------------------------------------------- | :-------------------------------- | +| 0.1.1 | 2024-01-03 | [#33924](https://github.com/airbytehq/airbyte/pull/33924) | Add new ap-southeast-3 AWS region | +| 0.1.0 | 2021-10-10 | [\#0000](https://github.com/airbytehq/airbyte/pull/0000) | Initial version | diff --git a/docs/integrations/sources/amplitude.md b/docs/integrations/sources/amplitude.md index c729b8ba854d..2c789ed146cd 100644 --- a/docs/integrations/sources/amplitude.md +++ b/docs/integrations/sources/amplitude.md @@ -1,6 +1,6 @@ # Amplitude -This page guides you through setting up the Amplitude source connector to sync data for the [Amplitude API](https://developers.amplitude.com/docs/http-api-v2). +This page guides you through setting up the Amplitude source connector to sync data for the [Amplitude API](https://www.docs.developers.amplitude.com/analytics/apis/http-v2-api/). ## Prerequisite @@ -20,11 +20,11 @@ To set up the Amplitude source connector, you'll need your Amplitude [`API Key` The Amplitude source connector supports the following streams: -* [Active Users Counts](https://developers.amplitude.com/docs/dashboard-rest-api#active-and-new-user-counts) \(Incremental sync\) -* [Annotations](https://developers.amplitude.com/docs/chart-annotations-api#get-all-annotations) -* [Average Session Length](https://developers.amplitude.com/docs/dashboard-rest-api#average-session-length) \(Incremental sync\) -* [Cohorts](https://developers.amplitude.com/docs/behavioral-cohorts-api#listing-all-cohorts) -* [Events](https://developers.amplitude.com/docs/export-api#export-api---export-your-projects-event-data) \(Incremental sync\) +* [Active Users Counts](https://www.docs.developers.amplitude.com/analytics/apis/dashboard-rest-api/#get-active-and-new-user-counts) \(Incremental sync\) +* [Annotations](https://www.docs.developers.amplitude.com/analytics/apis/chart-annotations-api/#get-all-chart-annotations) +* [Average Session Length](https://www.docs.developers.amplitude.com/analytics/apis/dashboard-rest-api/#get-average-session-length) \(Incremental sync\) +* [Cohorts](https://www.docs.developers.amplitude.com/analytics/apis/behavioral-cohorts-api/#get-all-cohorts-response) +* [Events](https://www.docs.developers.amplitude.com/analytics/apis/export-api/#response-schema) \(Incremental sync\) If there are more endpoints you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose) @@ -35,6 +35,15 @@ The Amplitude source connector supports the following [sync modes](https://docs. - Full Refresh - Incremental +## Connector-specific features + +There are two data region servers supported by Airbyte: + +- Standard Server +- EU Residency Server + +The `Standard Server` will be the default option until you change it in the Optional fields. + ## Performance considerations The Amplitude connector ideally should gracefully handle Amplitude API limitations under normal usage. [Create an issue](https://github.com/airbytehq/airbyte/issues/new/choose) if you see any rate limit issues that are not automatically retried successfully. @@ -43,6 +52,13 @@ The Amplitude connector ideally should gracefully handle Amplitude API limitatio | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------| +| 0.3.6 | 2023-10-23 | [31702](https://github.com/airbytehq/airbyte/pull/31702) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 0.3.5 | 2023-09-28 | [30846](https://github.com/airbytehq/airbyte/pull/30846) | Add support of multiple cursor date formats | +| 0.3.4 | 2023-09-28 | [30831](https://github.com/airbytehq/airbyte/pull/30831) | Add user friendly error description on 403 error | +| 0.3.3 | 2023-09-21 | [30652](https://github.com/airbytehq/airbyte/pull/30652) | Update spec: declare `start_date` type as `date-time` | +| 0.3.2 | 2023-09-18 | [30525](https://github.com/airbytehq/airbyte/pull/30525) | Fix `KeyError` while getting `data_region` from config | +| 0.3.1 | 2023-09-15 | [30471](https://github.com/airbytehq/airbyte/pull/30471) | Fix `Event` stream: Use `start_time` instead of cursor in the case of more recent | +| 0.3.0 | 2023-09-13 | [30378](https://github.com/airbytehq/airbyte/pull/30378) | Switch to latest CDK version | | 0.2.4 | 2023-05-05 | [25842](https://github.com/airbytehq/airbyte/pull/25842) | added missing attrs in events schema, enabled default availability strategy | | 0.2.3 | 2023-04-20 | [25317](https://github.com/airbytehq/airbyte/pull/25317) | Refactor Events Stream, use pre-YAML version based on Python CDK | | 0.2.2 | 2023-04-19 | [25315](https://github.com/airbytehq/airbyte/pull/25315) | Refactor to only fetch date_time_fields once per request | diff --git a/docs/integrations/sources/apify-dataset-migrations.md b/docs/integrations/sources/apify-dataset-migrations.md new file mode 100644 index 000000000000..f4bb1ed7c329 --- /dev/null +++ b/docs/integrations/sources/apify-dataset-migrations.md @@ -0,0 +1,14 @@ +# Apify Dataset Migration Guide + +## Upgrading to 2.0.0 + +Major update: The old broken Item Collection stream has been removed and replaced with a new Item Collection (WCC) stream specific for the datasets produced by [Website Content Crawler](https://apify.com/apify/website-content-crawler) Actor. In a follow-up release 2.1.0, a generic item collection stream will be added to support all other datasets. + +After upgrading, users should: +- Reconfigure dataset id and API key +- Reset all streams + +## Upgrading to 1.0.0 + +A major update fixing the data ingestion to retrieve properly data from Apify. +Please update your connector configuration setup. diff --git a/docs/integrations/sources/apify-dataset.md b/docs/integrations/sources/apify-dataset.md index 6793c861387b..a6546160709d 100644 --- a/docs/integrations/sources/apify-dataset.md +++ b/docs/integrations/sources/apify-dataset.md @@ -6,47 +6,76 @@ description: Web scraping and automation platform. ## Overview -[Apify](https://www.apify.com) is a web scraping and web automation platform providing both ready-made and custom solutions, an open-source [SDK](https://sdk.apify.com/) for web scraping, proxies, and many other tools to help you build and run web automation jobs at scale. +[Apify](https://apify.com/) is a web scraping and web automation platform providing both ready-made and custom solutions, an open-source [JavaScript SDK](https://docs.apify.com/sdk/js/) and [Python SDK](https://docs.apify.com/sdk/python/) for web scraping, proxies, and many other tools to help you build and run web automation jobs at scale. -The results of a scraping job are usually stored in [Apify Dataset](https://docs.apify.com/storage/dataset). This Airbyte connector allows you to automatically sync the contents of a dataset to your chosen destination using Airbyte. + +The results of a scraping job are usually stored in the [Apify Dataset](https://docs.apify.com/storage/dataset). This Airbyte connector provides streams to work with the datasets, including syncing their content to your chosen destination using Airbyte. + -To sync data from a dataset, all you need to know is its ID. You will find it in [Apify console](https://my.apify.com/) under storages. +To sync data from a dataset, all you need to know is your API token and dataset ID. + + +You can find your personal API token in the Apify Console in the [Settings -> Integrations](https://console.apify.com/account/integrations) and the dataset ID in the [Storage -> Datasets](https://console.apify.com/storage/datasets). + ### Running Airbyte sync from Apify webhook -When your Apify job \(aka [actor run](https://docs.apify.com/actors/running)\) finishes, it can trigger an Airbyte sync by calling the Airbyte [API](https://airbyte-public-api-docs.s3.us-east-2.amazonaws.com/rapidoc-api-docs.html#post-/v1/connections/sync) manual connection trigger \(`POST /v1/connections/sync`\). The API can be called from Apify [webhook](https://docs.apify.com/webhooks) which is executed when your Apify run finishes. +When your Apify job (aka [Actor run](https://docs.apify.com/platform/actors/running)) finishes, it can trigger an Airbyte sync by calling the Airbyte [API](https://airbyte-public-api-docs.s3.us-east-2.amazonaws.com/rapidoc-api-docs.html#post-/v1/connections/sync) manual connection trigger (`POST /v1/connections/sync`). The API can be called from Apify [webhook](https://docs.apify.com/platform/integrations/webhooks) which is executed when your Apify run finishes. ![](../../.gitbook/assets/apify_trigger_airbyte_connection.png) -### Output schema - -Since the dataset items do not have strongly typed schema, they are synced as objects stored in the `data` field, without any assumption on their content. - ### Features -| Feature | Supported? | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental Sync | No | +| Feature | Supported? | +| :---------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental Sync | Yes | ### Performance considerations The Apify dataset connector uses [Apify Python Client](https://docs.apify.com/apify-client-python) under the hood and should handle any API limitations under normal usage. -## Getting started +## Streams + +### `dataset_collection` + +- Calls `api.apify.com/v2/datasets` ([docs](https://docs.apify.com/api/v2#/reference/datasets/dataset-collection/get-list-of-datasets)) +- Properties: + - Apify Personal API token (you can find it [here](https://console.apify.com/account/integrations)) + +### `dataset` + +- Calls `https://api.apify.com/v2/datasets/{datasetId}` ([docs](https://docs.apify.com/api/v2#/reference/datasets/dataset/get-dataset)) +- Properties: + - Apify Personal API token (you can find it [here](https://console.apify.com/account/integrations)) + - Dataset ID (check the [docs](https://docs.apify.com/platform/storage/dataset)) + +### `item_collection` -### Requirements +- Calls `api.apify.com/v2/datasets/{datasetId}/items` ([docs](https://docs.apify.com/api/v2#/reference/datasets/item-collection/get-items)) +- Properties: + - Apify Personal API token (you can find it [here](https://console.apify.com/account/integrations)) + - Dataset ID (check the [docs](https://docs.apify.com/platform/storage/dataset)) +- Limitations: + - The stream uses a dynamic schema (all the data are stored under the `"data"` key), so it should support all the Apify Datasets (produced by whatever Actor). -* Apify [dataset](https://docs.apify.com/storage/dataset) ID +### `item_collection_website_content_crawler` -### Changelog +- Calls the same endpoint and uses the same properties as the `item_collection` stream. +- Limitations: + - The stream uses a static schema which corresponds to the datasets produced by [Website Content Crawler](https://apify.com/apify/website-content-crawler) Actor. So only datasets produced by this Actor are supported. -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.2.0 | 2022-06-20 | [28290](https://github.com/airbytehq/airbyte/pull/28290) | Make connector work with platform changes not syncing empty stream schemas. | -| 0.1.11 | 2022-04-27 | [12397](https://github.com/airbytehq/airbyte/pull/12397) | No changes. Used connector to test publish workflow changes. | -| 0.1.9 | 2022-04-05 | [PR\#11712](https://github.com/airbytehq/airbyte/pull/11712) | No changes from 0.1.4. Used connector to test publish workflow changes. | -| 0.1.4 | 2021-12-23 | [PR\#8434](https://github.com/airbytehq/airbyte/pull/8434) | Update fields in source-connectors specifications | -| 0.1.2 | 2021-11-08 | [PR\#7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | -| 0.1.0 | 2021-07-29 | [PR\#5069](https://github.com/airbytehq/airbyte/pull/5069) | Initial version of the connector | +## Changelog +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :----------------------------------------------------------- | :-------------------------------------------------------------------------- | +| 2.1.1 | 2023-12-14 | [33414](https://github.com/airbytehq/airbyte/pull/33414) | Prepare for airbyte-lib | +| 2.1.0 | 2023-10-13 | [31333](https://github.com/airbytehq/airbyte/pull/31333) | Add stream for arbitrary datasets | +| 2.0.0 | 2023-09-18 | [30428](https://github.com/airbytehq/airbyte/pull/30428) | Fix broken stream, manifest refactor | +| 1.0.0 | 2023-08-25 | [29859](https://github.com/airbytehq/airbyte/pull/29859) | Migrate to lowcode | +| 0.2.0 | 2022-06-20 | [28290](https://github.com/airbytehq/airbyte/pull/28290) | Make connector work with platform changes not syncing empty stream schemas. | +| 0.1.11 | 2022-04-27 | [12397](https://github.com/airbytehq/airbyte/pull/12397) | No changes. Used connector to test publish workflow changes. | +| 0.1.9 | 2022-04-05 | [PR\#11712](https://github.com/airbytehq/airbyte/pull/11712) | No changes from 0.1.4. Used connector to test publish workflow changes. | +| 0.1.4 | 2021-12-23 | [PR\#8434](https://github.com/airbytehq/airbyte/pull/8434) | Update fields in source-connectors specifications | +| 0.1.2 | 2021-11-08 | [PR\#7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | +| 0.1.0 | 2021-07-29 | [PR\#5069](https://github.com/airbytehq/airbyte/pull/5069) | Initial version of the connector | diff --git a/docs/integrations/sources/appstore-singer.md b/docs/integrations/sources/appstore.md similarity index 100% rename from docs/integrations/sources/appstore-singer.md rename to docs/integrations/sources/appstore.md diff --git a/docs/integrations/sources/asana.inapp.md b/docs/integrations/sources/asana.inapp.md deleted file mode 100644 index 887635811141..000000000000 --- a/docs/integrations/sources/asana.inapp.md +++ /dev/null @@ -1,13 +0,0 @@ -## Prerequisites - -* OAuth access or a Personal Access Token - -## Setup guide -1. Enter a name for your source -2. Authenticate using OAuth (recommended) or enter your `personal_access_token`. Please follow these [steps](https://developers.asana.com/docs/personal-access-token) to obtain a Personal Access Token for your account. -3. Click **Set up source** - -### Syncing Multiple Projects -If you have access to multiple projects, Airbyte will sync data related to all projects you have access to. The ability to filter to specific projects is not available at this time. - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Asana](https://docs.airbyte.com/integrations/sources/asana). \ No newline at end of file diff --git a/docs/integrations/sources/asana.md b/docs/integrations/sources/asana.md index dede4313e1a3..a53215a84ac0 100644 --- a/docs/integrations/sources/asana.md +++ b/docs/integrations/sources/asana.md @@ -1,30 +1,45 @@ # Asana -This page contains the setup guide and reference information for the Asana source connector. + + +This page contains the setup guide and reference information for the [Asana](https://www.asana.com) source connector. + + ## Prerequisites -Please follow these [steps](https://developers.asana.com/docs/personal-access-token) to obtain Personal Access Token for your account. +This connector supports **OAuth** and **Personal Access Tokens**. Please follow these [steps](https://developers.asana.com/docs/personal-access-token) to obtain Personal Access Token for your account. ## Setup guide -## Step 1: Set up the Asana connector in Airbyte + -### For Airbyte Cloud: +**For Airbyte Cloud:** 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**. -3. Set the name for your source -4. Enter your `personal_access_token` -5. Click **Set up source** +3. Set the name for your source. +4. Authenticate using OAuth (recommended) or enter your `personal_access_token`. +5. Click **Set up source**. + +#### Syncing Multiple Projects +If you have access to multiple projects, Airbyte will sync data related to all projects you have access to. The ability to filter to specific projects is not available at this time. + + + + -### For Airbyte OSS: +**For Airbyte Open Source:** -1. Navigate to the Airbyte Open Source dashboard +1. Navigate to the Airbyte Open Source dashboard. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**. -3. Set the name for your source -4. Enter your `personal_access_token` -5. Click **Set up source** +3. Set the name for your source. +4. Enter your `personal_access_token`. +5. Click **Set up source**. + + + + ## Supported sync modes @@ -37,9 +52,11 @@ The Asana source connector supports the following [sync modes](https://docs.airb | Namespaces | No | ## Supported Streams - +- [Attachments](https://developers.asana.com/docs/attachments) - [Custom fields](https://developers.asana.com/docs/custom-fields) - [Projects](https://developers.asana.com/docs/projects) +- [Portfolios](https://developers.asana.com/docs/portfolios) +- [PortfolioMemberships](https://developers.asana.com/reference/portfolio-memberships) - [Sections](https://developers.asana.com/docs/sections) - [Stories](https://developers.asana.com/docs/stories) - [Tags](https://developers.asana.com/docs/tags) @@ -49,10 +66,6 @@ The Asana source connector supports the following [sync modes](https://docs.airb - [Users](https://developers.asana.com/docs/users) - [Workspaces](https://developers.asana.com/docs/workspaces) -## Performance considerations - -The connector is restricted by normal Asana [requests limitation](https://developers.asana.com/docs/rate-limits). - ## Data type map | Integration Type | Airbyte Type | @@ -64,15 +77,45 @@ The connector is restricted by normal Asana [requests limitation](https://develo | `array` | `array` | | `object` | `object` | +## Limitations & Troubleshooting + +
      + +Expand to see details about Asana connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting + +The connector is restricted by [Asana rate limits](https://developers.asana.com/docs/rate-limits). + +### Troubleshooting + +* If you encounter access errors while using **OAuth** authentication, please make sure you've followed this [Asana Article](https://developers.asana.com/docs/oauth). +* Check out common troubleshooting issues for the Asana source connector on our Airbyte Forum [here](https://github.com/airbytehq/airbyte/discussions). + +
      + ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------- | -| 0.1.7 | 2023-05-29 | [26716](https://github.com/airbytehq/airbyte/pull/26716) | Remove authSpecification from spec.json, use advancedAuth instead | -| 0.1.6 | 2023-05-26 | [26653](https://github.com/airbytehq/airbyte/pull/26653) | Fix order of authentication methods | -| 0.1.5 | 2022-11-16 | [19561](https://github.com/airbytehq/airbyte/pull/19561) | Added errors handling, updated SAT with new format | -| 0.1.4 | 2022-08-18 | [15749](https://github.com/airbytehq/airbyte/pull/15749) | Add cache to project stream | -| 0.1.3 | 2021-10-06 | [6832](https://github.com/airbytehq/airbyte/pull/6832) | Add oauth init flow parameters support | -| 0.1.2 | 2021-09-24 | [6402](https://github.com/airbytehq/airbyte/pull/6402) | Fix SAT tests: update schemas and invalid_config.json file | -| 0.1.1 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add entrypoint and bump version for connector | -| 0.1.0 | 2021-05-25 | [3510](https://github.com/airbytehq/airbyte/pull/3510) | New Source: Asana | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------- | +| 0.6.1 | 2023-11-13 | [31110](https://github.com/airbytehq/airbyte/pull/31110) | Fix hidden config access | +| 0.6.0 | 2023-11-03 | [31110](https://github.com/airbytehq/airbyte/pull/31110) | Add new stream Portfolio Memberships with Parent Portfolio | +| 0.5.0 | 2023-10-30 | [31114](https://github.com/airbytehq/airbyte/pull/31114) | Add Portfolios stream | +| 0.4.0 | 2023-10-24 | [31084](https://github.com/airbytehq/airbyte/pull/31084) | Add StoriesCompact stream | +| 0.3.0 | 2023-10-24 | [31634](https://github.com/airbytehq/airbyte/pull/31634) | Add OrganizationExports stream | +| 0.2.0 | 2023-10-17 | [31090](https://github.com/airbytehq/airbyte/pull/31090) | Add Attachments stream | +| 0.1.9 | 2023-10-16 | [31089](https://github.com/airbytehq/airbyte/pull/31089) | Add Events stream | +| 0.1.8 | 2023-10-16 | [31009](https://github.com/airbytehq/airbyte/pull/31009) | Add SectionsCompact stream | +| 0.1.7 | 2023-05-29 | [26716](https://github.com/airbytehq/airbyte/pull/26716) | Remove authSpecification from spec.json, use advancedAuth instead | +| 0.1.6 | 2023-05-26 | [26653](https://github.com/airbytehq/airbyte/pull/26653) | Fix order of authentication methods | +| 0.1.5 | 2022-11-16 | [19561](https://github.com/airbytehq/airbyte/pull/19561) | Added errors handling, updated SAT with new format | +| 0.1.4 | 2022-08-18 | [15749](https://github.com/airbytehq/airbyte/pull/15749) | Add cache to project stream | +| 0.1.3 | 2021-10-06 | [6832](https://github.com/airbytehq/airbyte/pull/6832) | Add oauth init flow parameters support | +| 0.1.2 | 2021-09-24 | [6402](https://github.com/airbytehq/airbyte/pull/6402) | Fix SAT tests: update schemas and invalid_config.json file | +| 0.1.1 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add entrypoint and bump version for connector | +| 0.1.0 | 2021-05-25 | [3510](https://github.com/airbytehq/airbyte/pull/3510) | New Source: Asana | + +
      \ No newline at end of file diff --git a/docs/integrations/sources/auth0.md b/docs/integrations/sources/auth0.md index e3e63e390f29..724a4669fd52 100644 --- a/docs/integrations/sources/auth0.md +++ b/docs/integrations/sources/auth0.md @@ -6,8 +6,8 @@ The source connector fetches data from [Auth0 Management API](https://auth0.com/ ## Prerequisites -* You own an Auth0 account, free or paid. -* Follow the [Setup guide](#setup-guide) to authorize Airbyte to read data from your account. +- You own an Auth0 account, free or paid. +- Follow the [Setup guide](#setup-guide) to authorize Airbyte to read data from your account. ## Setup guide @@ -30,14 +30,15 @@ The source connector fetches data from [Auth0 Management API](https://auth0.com/ 2. In Auth0, go to [Dashboard > Applications > Applications](https://manage.auth0.com/?#/applications). 3. Create a new application, name it **Airbyte**. Choose the application type **Machine to Machine Applications** 4. Select the Management API V2, this is the api you want call from Airbyte. -5. Each M2M app that accesses an API must be granted a set of permissions (or scopes). Here, we only need permissions starting with `read` (e.g. *read:users*). Under the [API doc](https://auth0.com/docs/api/management/v2#!/Users/get_users), each api will list the required scopes. +5. Each M2M app that accesses an API must be granted a set of permissions (or scopes). Here, we only need permissions starting with `read` (e.g. _read:users_). Under the [API doc](https://auth0.com/docs/api/management/v2#!/Users/get_users), each api will list the required scopes. 6. More details can be found from [this documentation](https://auth0.com/docs/secure/tokens/access-tokens/get-management-api-access-tokens-for-production). ## Supported sync modes The Auth0 source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): - - Full Refresh - - Incremental + +- Full Refresh +- Incremental ## Supported Streams @@ -54,10 +55,12 @@ The connector is restricted by Auth0 [rate limits](https://auth0.com/docs/troubl ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------| -| 0.4.0 | 2023-08-03 | [28972](https://github.com/airbytehq/airbyte/pull/28972) | Migrate to Low-Code CDK | -| 0.3.0 | 2023-06-20 | [29001](https://github.com/airbytehq/airbyte/pull/29001) | Add Organizations, OrganizationMembers, OrganizationMemberRoles streams | -| 0.2.0 | 2023-05-23 | [26445](https://github.com/airbytehq/airbyte/pull/26445) | Add Clients stream | -| 0.1.0 | 2022-10-21 | [18338](https://github.com/airbytehq/airbyte/pull/18338) | Add Auth0 and Users stream | - +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------------- | +| 0.5.1 | 2023-10-20 | [31643](https://github.com/airbytehq/airbyte/pull/31643) | Upgrade base image to airbyte/python-connector-base:1.1.0 | +| 0.5.0 | 2023-10-11 | [30467](https://github.com/airbytehq/airbyte/pull/30467) | Use Python base image | +| 0.4.1 | 2023-08-24 | [29804](https://github.com/airbytehq/airbyte/pull/29804) | Fix low code migration bugs | +| 0.4.0 | 2023-08-03 | [28972](https://github.com/airbytehq/airbyte/pull/28972) | Migrate to Low-Code CDK | +| 0.3.0 | 2023-06-20 | [29001](https://github.com/airbytehq/airbyte/pull/29001) | Add Organizations, OrganizationMembers, OrganizationMemberRoles streams | +| 0.2.0 | 2023-05-23 | [26445](https://github.com/airbytehq/airbyte/pull/26445) | Add Clients stream | +| 0.1.0 | 2022-10-21 | [18338](https://github.com/airbytehq/airbyte/pull/18338) | Add Auth0 and Users stream | \ No newline at end of file diff --git a/docs/integrations/sources/azure-blob-storage.md b/docs/integrations/sources/azure-blob-storage.md index 3c0896f97c55..0a6dc5e5e8f7 100644 --- a/docs/integrations/sources/azure-blob-storage.md +++ b/docs/integrations/sources/azure-blob-storage.md @@ -14,46 +14,191 @@ Cloud storage may incur egress costs. Egress refers to data that is transferred ### Step 2: Set up the Azure Blob Storage connector in Airbyte - -1. Create a new Azure Blob Storage source with a suitable name. -2. Set `container` appropriately. This will be the name of the container where the blobs are located. -3. If you are only interested in blobs containing some prefix in the container set the blobs prefix property -4. Set schema inference limit if you want to limit the number of blobs being considered for constructing the schema -5. Choose the format corresponding to the format of your files. - +1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account, or navigate to your Airbyte Open Source dashboard. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. +3. Find and select **Azure Blob Storage** from the list of available sources. +4. Enter the name of your Azure **Account**. +5. Enter the *Azure Blob Storage account key* which grants access to your account. +6. Enter the name of the **Container** containing your files to replicate. +7. Add a stream + 1. Write the **File Type** + 2. In the **Format** box, use the dropdown menu to select the format of the files you'd like to replicate. The supported formats are **CSV**, **Parquet**, **Avro** and **JSONL**. Toggling the **Optional fields** button within the **Format** box will allow you to enter additional configurations based on the selected format. For a detailed breakdown of these settings, refer to the [File Format section](#file-format-settings) below. + 3. Give a **Name** to the stream + 4. (Optional) - If you want to enforce a specific schema, you can enter a **Input schema**. By default, this value is set to `{}` and will automatically infer the schema from the file\(s\) you are replicating. For details on providing a custom schema, refer to the [User Schema section](#user-schema). + 5. Optionally, enter the **Globs** which dictates which files to be synced. This is a regular expression that allows Airbyte to pattern match the specific files to replicate. If you are replicating all the files within your bucket, use `**` as the pattern. For more precise pattern matching options, refer to the [Path Patterns section](#path-patterns) below. +8. Optionally, enter the endpoint to use for the data replication. +9. Optionally, enter the desired start date from which to begin replicating data. ## Supported sync modes The Azure Blob Storage source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): | Feature | Supported? | -|:-----------------------------------------------| :--------- | +| :--------------------------------------------- |:-----------| | Full Refresh Sync | Yes | | Incremental Sync | Yes | | Replicate Incremental Deletes | No | -| Replicate Multiple Files \(blob prefix\) | Yes | -| Replicate Multiple Streams \(distinct tables\) | No | +| Replicate Multiple Files \(pattern matching\) | Yes | +| Replicate Multiple Streams \(distinct tables\) | Yes | | Namespaces | No | +## File Compressions + +| Compression | Supported? | +| :---------- | :--------- | +| Gzip | Yes | +| Zip | No | +| Bzip2 | Yes | +| Lzma | No | +| Xz | No | +| Snappy | No | + +Please let us know any specific compressions you'd like to see support for next! + +## Path Patterns + +\(tl;dr -> path pattern syntax using [wcmatch.glob](https://facelessuser.github.io/wcmatch/glob/). GLOBSTAR and SPLIT flags are enabled.\) + +This connector can sync multiple files by using glob-style patterns, rather than requiring a specific path for every file. This enables: + +- Referencing many files with just one pattern, e.g. `**` would indicate every file in the bucket. +- Referencing future files that don't exist yet \(and therefore don't have a specific path\). + +You must provide a path pattern. You can also provide many patterns split with \| for more complex directory layouts. + +Each path pattern is a reference from the _root_ of the bucket, so don't include the bucket name in the pattern\(s\). + +Some example patterns: + +- `**` : match everything. +- `**/*.csv` : match all files with specific extension. +- `myFolder/**/*.csv` : match all csv files anywhere under myFolder. +- `*/**` : match everything at least one folder deep. +- `*/*/*/**` : match everything at least three folders deep. +- `**/file.*|**/file` : match every file called "file" with any extension \(or no extension\). +- `x/*/y/*` : match all files that sit in folder x -> any folder -> folder y. +- `**/prefix*.csv` : match all csv files with specific prefix. +- `**/prefix*.parquet` : match all parquet files with specific prefix. + +Let's look at a specific example, matching the following bucket layout: + +```text +myBucket + -> log_files + -> some_table_files + -> part1.csv + -> part2.csv + -> images + -> more_table_files + -> part3.csv + -> extras + -> misc + -> another_part1.csv +``` + +We want to pick up part1.csv, part2.csv and part3.csv \(excluding another_part1.csv for now\). We could do this a few different ways: + +- We could pick up every csv file called "partX" with the single pattern `**/part*.csv`. +- To be a bit more robust, we could use the dual pattern `some_table_files/*.csv|more_table_files/*.csv` to pick up relevant files only from those exact folders. +- We could achieve the above in a single pattern by using the pattern `*table_files/*.csv`. This could however cause problems in the future if new unexpected folders started being created. +- We can also recursively wildcard, so adding the pattern `extras/**/*.csv` would pick up any csv files nested in folders below "extras", such as "extras/misc/another_part1.csv". + +As you can probably tell, there are many ways to achieve the same goal with path patterns. We recommend using a pattern that ensures clarity and is robust against future additions to the directory structure. + +## User Schema + +Providing a schema allows for more control over the output of this stream. Without a provided schema, columns and datatypes will be inferred from the first created file in the bucket matching your path pattern and suffix. This will probably be fine in most cases but there may be situations you want to enforce a schema instead, e.g.: + +- You only care about a specific known subset of the columns. The other columns would all still be included, but packed into the `_ab_additional_properties` map. +- Your initial dataset is quite small \(in terms of number of records\), and you think the automatic type inference from this sample might not be representative of the data in the future. +- You want to purposely define types for every column. +- You know the names of columns that will be added to future data and want to include these in the core schema as columns rather than have them appear in the `_ab_additional_properties` map. + +Or any other reason! The schema must be provided as valid JSON as a map of `{"column": "datatype"}` where each datatype is one of: -## Azure Blob Storage Settings +- string +- number +- integer +- object +- array +- boolean +- null -* `azure_blob_storage_endpoint` : azure blob storage endpoint to connect to -* `azure_blob_storage_container_name` : name of the container where your blobs are located -* `azure_blob_storage_account_name` : name of your account -* `azure_blob_storage_account_key` : key of your account -* `azure_blob_storage_blobs_prefix` : prefix for getting files which contain that prefix i.e. FolderA/FolderB/ will get files named FolderA/FolderB/blob.json but not FolderA/blob.json -* `azure_blob_storage_schema_inference_limit` : Limits the number of files being scanned for schema inference and can increase speed and efficiency -* `format` : File format of the blobs in the container +For example: + +- `{"id": "integer", "location": "string", "longitude": "number", "latitude": "number"}` +- `{"username": "string", "friends": "array", "information": "object"}` + +## File Format Settings + +### CSV + +Since CSV files are effectively plain text, providing specific reader options is often required for correct parsing of the files. These settings are applied when a CSV is created or exported so please ensure that this process happens consistently over time. + +- **Header Definition**: How headers will be defined. `User Provided` assumes the CSV does not have a header row and uses the headers provided and `Autogenerated` assumes the CSV does not have a header row and the CDK will generate headers using for `f{i}` where `i` is the index starting from 0. Else, the default behavior is to use the header from the CSV file. If a user wants to autogenerate or provide column names for a CSV having headers, they can set a value for the "Skip rows before header" option to ignore the header row. +- **Delimiter**: Even though CSV is an acronym for Comma Separated Values, it is used more generally as a term for flat file data that may or may not be comma separated. The delimiter field lets you specify which character acts as the separator. To use [tab-delimiters](https://en.wikipedia.org/wiki/Tab-separated_values), you can set this value to `\t`. By default, this value is set to `,`. +- **Double Quote**: This option determines whether two quotes in a quoted CSV value denote a single quote in the data. Set to True by default. +- **Encoding**: Some data may use a different character set \(typically when different alphabets are involved\). See the [list of allowable encodings here](https://docs.python.org/3/library/codecs.html#standard-encodings). By default, this is set to `utf8`. +- **Escape Character**: An escape character can be used to prefix a reserved character and ensure correct parsing. A commonly used character is the backslash (`\`). For example, given the following data: + +``` +Product,Description,Price +Jeans,"Navy Blue, Bootcut, 34\"",49.99 +``` + +The backslash (`\`) is used directly before the second double quote (`"`) to indicate that it is _not_ the closing quote for the field, but rather a literal double quote character that should be included in the value (in this example, denoting the size of the jeans in inches: `34"` ). + +Leaving this field blank (default option) will disallow escaping. + +- **False Values**: A set of case-sensitive strings that should be interpreted as false values. +- **Null Values**: A set of case-sensitive strings that should be interpreted as null values. For example, if the value 'NA' should be interpreted as null, enter 'NA' in this field. +- **Quote Character**: In some cases, data values may contain instances of reserved characters \(like a comma, if that's the delimiter\). CSVs can handle this by wrapping a value in defined quote characters so that on read it can parse it correctly. By default, this is set to `"`. +- **Skip Rows After Header**: The number of rows to skip after the header row. +- **Skip Rows Before Header**: The number of rows to skip before the header row. +- **Strings Can Be Null**: Whether strings can be interpreted as null values. If true, strings that match the null_values set will be interpreted as null. If false, strings that match the null_values set will be interpreted as the string itself. +- **True Values**: A set of case-sensitive strings that should be interpreted as true values. + + +### Parquet + +Apache Parquet is a column-oriented data storage format of the Apache Hadoop ecosystem. It provides efficient data compression and encoding schemes with enhanced performance to handle complex data in bulk. At the moment, partitioned parquet datasets are unsupported. The following settings are available: + +- **Convert Decimal Fields to Floats**: Whether to convert decimal fields to floats. There is a loss of precision when converting decimals to floats, so this is not recommended. + +### Avro + +The Avro parser uses the [Fastavro library](https://fastavro.readthedocs.io/en/latest/). The following settings are available: +- **Convert Double Fields to Strings**: Whether to convert double fields to strings. This is recommended if you have decimal numbers with a high degree of precision because there can be a loss precision when handling floating point numbers. + +### JSONL + +There are currently no options for JSONL parsing. + + + +### Document File Type Format (Experimental) + +:::warning +The Document File Type Format is currently an experimental feature and not subject to SLAs. Use at your own risk. +::: -**File Format Settings** +The Document File Type Format is a special format that allows you to extract text from Markdown, TXT, PDF, Word and Powerpoint documents. If selected, the connector will extract text from the documents and output it as a single field named `content`. The `document_key` field will hold a unique identifier for the processed file which can be used as a primary key. The content of the document will contain markdown formatting converted from the original file format. Each file matching the defined glob pattern needs to either be a markdown (`md`), PDF (`pdf`), Word (`docx`) or Powerpoint (`.pptx`) file. -### Jsonl +One record will be emitted for each document. Keep in mind that large files can emit large records that might not fit into every destination as each destination has different limitations for string fields. -Only the line-delimited [JSON](https://jsonlines.org/) format is supported for now +To perform the text extraction from PDF and Docx files, the connector uses the [Unstructured](https://pypi.org/project/unstructured/) Python library. + -## Changelog 21210 +## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:-------------------------------------------------|:------------------------------------------------------------------------| -| 0.1.0 | 2023-02-17 | https://github.com/airbytehq/airbyte/pull/23222 | Initial release with full-refresh and incremental sync with JSONL files | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------| +| 0.3.1 | 2024-01-10 | [34084](https://github.com/airbytehq/airbyte/pull/34084) | Fix bug for running check with document file format | +| 0.3.0 | 2023-12-14 | [33411](https://github.com/airbytehq/airbyte/pull/33411) | Bump CDK version to auto-set primary key for document file streams and support raw txt files | +| 0.2.5 | 2023-12-06 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Bump CDK version to hide source-defined primary key | +| 0.2.4 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Improve document file type parser | +| 0.2.3 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | +| 0.2.2 | 2023-10-30 | [31904](https://github.com/airbytehq/airbyte/pull/31904) | Update CDK to support document file types | +| 0.2.1 | 2023-10-18 | [31543](https://github.com/airbytehq/airbyte/pull/31543) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 0.2.0 | 2023-10-10 | https://github.com/airbytehq/airbyte/pull/31336 | Migrate to File-based CDK. Add support of CSV, Parquet and Avro files | +| 0.1.0 | 2023-02-17 | https://github.com/airbytehq/airbyte/pull/23222 | Initial release with full-refresh and incremental sync with JSONL files | \ No newline at end of file diff --git a/docs/integrations/sources/azure-table.md b/docs/integrations/sources/azure-table.md index 108b90361c94..6ea26ec6c34e 100644 --- a/docs/integrations/sources/azure-table.md +++ b/docs/integrations/sources/azure-table.md @@ -9,11 +9,11 @@ The Azure table storage supports Full Refresh and Incremental syncs. You can cho This Source have generic schema for all streams. Azure Table storage is a service that stores non-relational structured data (also known as structured NoSQL data). There is no efficient way to read schema for the given table. We use `data` property to have all the properties for any given row. -- data - This property contain all values +- data - This property contains all values - additionalProperties - This property denotes that all the values are in `data` property. -` - { +``` +{ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { @@ -25,7 +25,7 @@ Azure Table storage is a service that stores non-relational structured data (als } } } -` +``` ### Data type mapping @@ -58,7 +58,7 @@ The Azure table storage connector should not run into API limitations under norm Visit the [Azure Portal](https://portal.azure.com). Go to your storage account, you can find : - Azure Storage Account - under the overview tab - Azure Storage Account Key - under the Access keys tab - - Azure Storage Endpoint Suffix - under the Enpoint tab + - Azure Storage Endpoint Suffix - under the Endpoint tab We recommend creating a restricted key specifically for Airbyte access. This will allow you to control which resources Airbyte should be able to access. However, shared access key authentication is not supported by this connector yet. diff --git a/docs/integrations/sources/bamboo-hr.inapp.md b/docs/integrations/sources/bamboo-hr.inapp.md deleted file mode 100644 index f7d75a171f76..000000000000 --- a/docs/integrations/sources/bamboo-hr.inapp.md +++ /dev/null @@ -1,14 +0,0 @@ -## Prerequisites - -* BambooHR [API Key](https://documentation.bamboohr.com/docs#authentication) - - -## Setup guide -1. Name your BambooHR connector -2. Enter your `api_key`. To generate an API key, log in and click your name in the upper right-hand corner of any page to get to the user context menu. If you have sufficient administrator permissions, there will be an "API Keys" option in that menu to go to the page. -3. Enter your `subdomain`. If you access BambooHR at https://mycompany.bamboohr.com, then the subdomain is "mycompany" -4. (Optional) Enter any `Custom Report Fields` as a comma-separated list of fields to include in your custom reports. Example: `firstName,lastName`. If none are listed, then the [default fields](https://documentation.bamboohr.com/docs/list-of-field-names) will be returned. -5. Toggle `Custom Reports Include Default Fields`. If true, then the [default fields](https://documentation.bamboohr.com/docs/list-of-field-names) will be returned. If false, then the values defined in `Custom Report Fields` will be returned. -6. Click **Set up source** - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [BambooHR](https://docs.airbyte.com/integrations/sources/bamboo-hr). diff --git a/docs/integrations/sources/bamboo-hr.md b/docs/integrations/sources/bamboo-hr.md index 597a92647354..3cd19e3d9e5a 100644 --- a/docs/integrations/sources/bamboo-hr.md +++ b/docs/integrations/sources/bamboo-hr.md @@ -1,71 +1,54 @@ -# Bamboo HR +# BambooHR -## Overview + -The BambooHr source supports Full Refresh sync. You can choose if this connector will overwrite the old records or duplicate old ones. +This page contains the setup guide and reference information for the [BambooHR](https://www.bamboohr.com/) source connector. -### Output schema + -This connector outputs the following stream: - -* [Custom Reports](https://documentation.bamboohr.com/reference/request-custom-report-1) - -### Features - -| Feature | Supported? | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental - Append Sync | No | -| SSL connection | Yes | -| Namespaces | No | - -### Performance considerations - -BambooHR has the [rate limits](https://documentation.bamboohr.com/docs/api-details), but the connector should not run into API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. +## Prerequisites -## Getting started +* BambooHR Account +* BambooHR [API key](https://documentation.bamboohr.com/docs) -### Requirements +## Setup Guide -* BambooHr Account -* BambooHr [Api key](https://documentation.bamboohr.com/docs) +## Step 1: Set up the BambooHR connector in Airbyte -# Bamboo HR + -This page contains the setup guide and reference information for the Bamboo HR source connector. +**For Airbyte Cloud:** -## Prerequisites +1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. +3. On the Set up the source page, enter the name for the BambooHR connector and select **BambooHR** from the Source type dropdown. +3. Enter your `subdomain`. If you access BambooHR at https://mycompany.bamboohr.com, then the subdomain is "mycompany". +4. Enter your `api_key`. To generate an API key, log in and click your name in the upper right-hand corner of any page to get to the user context menu. If you have sufficient administrator permissions, there will be an "API Keys" option in that menu to go to the page. +5. (Optional) Enter any `Custom Report Fields` as a comma-separated list of fields to include in your custom reports. Example: `firstName,lastName`. If none are listed, then the [default fields](https://documentation.bamboohr.com/docs/list-of-field-names) will be returned. +6. Toggle `Custom Reports Include Default Fields`. If true, then the [default fields](https://documentation.bamboohr.com/docs/list-of-field-names) will be returned. If false, then the values defined in `Custom Report Fields` will be returned. +7. Click **Set up source** -* BambooHr Account -* BambooHr [Api key](https://documentation.bamboohr.com/docs) + -## Setup guide -## Step 1: Set up the Bamboo HR connector in Airbyte + -### For Airbyte Cloud: +**For Airbyte OSS:** -1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. -2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**. -3. On the Set up the source page, enter the name for the Bamboo HR connector and select **Bamboo HR** from the Source type dropdown. -3. Enter your `subdomain` -4. Enter your `api_key` -5. Enter your `custom_reports_fields` if need -6. Choose `custom_reports_include_default_fields` flag value +1. Navigate to the Airbyte Open Source dashboard. +2. Set the name for your source. +3. Enter your `subdomain`. If you access BambooHR at https://mycompany.bamboohr.com, then the subdomain is "mycompany". +4. Enter your `api_key`. To generate an API key, log in and click your name in the upper right-hand corner of any page to get to the user context menu. If you have sufficient administrator permissions, there will be an "API Keys" option in that menu to go to the page. +5. (Optional) Enter any `Custom Report Fields` as a comma-separated list of fields to include in your custom reports. Example: `firstName,lastName`. If none are listed, then the [default fields](https://documentation.bamboohr.com/docs/list-of-field-names) will be returned. +6. Toggle `Custom Reports Include Default Fields`. If true, then the [default fields](https://documentation.bamboohr.com/docs/list-of-field-names) will be returned. If false, then the values defined in `Custom Report Fields` will be returned. 7. Click **Set up source** -### For Airbyte OSS: + -1. Navigate to the Airbyte Open Source dashboard -2. Set the name for your source -3. Enter your `subdomain` -4. Enter your `api_key` -5. Enter your `custom_reports_fields` if need -6. Choose `custom_reports_include_default_fields` flag value -7. Click **Set up source** + ## Supported sync modes -The Bamboo HR source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): +The BambooHR source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): | Feature | Supported? | | :--- | :--- | @@ -79,11 +62,27 @@ The Bamboo HR source connector supports the following [sync modes](https://docs. * [Custom Reports](https://documentation.bamboohr.com/reference/request-custom-report-1) -## Performance considerations +## Limitations & Troubleshooting + +
      + +Expand to see details about BambooHR connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting + +BambooHR has the [rate limits](https://documentation.bamboohr.com/docs/api-details), but the connector should not run into API limitations under normal usage. -BambooHR has the [rate limits](https://documentation.bamboohr.com/docs/api-details), but the connector should not run into API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. +### Troubleshooting + +* Check out common troubleshooting issues for the BambooHR source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +
      + ## Changelog | Version | Date | Pull Request | Subject | @@ -92,3 +91,5 @@ Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see | 0.2.1 | 2022-09-16 | [16826](https://github.com/airbytehq/airbyte/pull/16826) | Add custom fields validation during check | | 0.2.0 | 2022-03-24 | [11326](https://github.com/airbytehq/airbyte/pull/11326) | Add support for Custom Reports endpoint | | 0.1.0 | 2021-08-27 | [5054](https://github.com/airbytehq/airbyte/pull/5054) | Initial release with Employees API | + +
      \ No newline at end of file diff --git a/docs/integrations/sources/bigcommerce.md b/docs/integrations/sources/bigcommerce.md index 968597242761..251d0e4376eb 100644 --- a/docs/integrations/sources/bigcommerce.md +++ b/docs/integrations/sources/bigcommerce.md @@ -55,6 +55,7 @@ BigCommerce has some [rate limit restrictions](https://developer.bigcommerce.com | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------| +| 0.2.0 | 2023-08-16 | [29469](https://github.com/airbytehq/airbyte/pull/29469) | Migrate Python CDK to Low Code | | 0.1.10 | 2022-12-16 | [20518](https://github.com/airbytehq/airbyte/pull/20518) | Add brands and categories streams | | 0.1.9 | 2022-12-15 | [20540](https://github.com/airbytehq/airbyte/pull/20540) | Rebuild on CDK 0.15.0 | | 0.1.8 | 2022-12-15 | [20090](https://github.com/airbytehq/airbyte/pull/20090) | Add order_products stream | diff --git a/docs/integrations/sources/bigquery.md b/docs/integrations/sources/bigquery.md index c4b3d042b519..1777869527d9 100644 --- a/docs/integrations/sources/bigquery.md +++ b/docs/integrations/sources/bigquery.md @@ -87,7 +87,8 @@ Once you've configured BigQuery as a source, delete the Service Account Key from ### source-bigquery | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------- | +|:--------|:-----------| :------------------------------------------------------- |:------------------------------------------------------------------------------------------------------------------------------------------| +| 0.4.0 | 2023-12-18 | [33484](https://github.com/airbytehq/airbyte/pull/33484) | Remove LEGACY state | | 0.3.0 | 2023-06-26 | [27737](https://github.com/airbytehq/airbyte/pull/27737) | License Update: Elv2 | | 0.2.3 | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | | 0.2.2 | 2022-09-22 | [16902](https://github.com/airbytehq/airbyte/pull/16902) | Source BigQuery: added user agent header | diff --git a/docs/integrations/sources/bing-ads-migrations.md b/docs/integrations/sources/bing-ads-migrations.md new file mode 100644 index 000000000000..c078d1d0cb56 --- /dev/null +++ b/docs/integrations/sources/bing-ads-migrations.md @@ -0,0 +1,47 @@ +# Bing Ads Migration Guide + +## Upgrading to 2.0.0 + +This version update affects all hourly reports (end in report_hourly) and the following streams: + +- Accounts +- Campaigns +- Search Query Performance Report +- AppInstallAds +- AppInstallAdLabels +- Labels +- Campaign Labels +- Keyword Labels +- Ad Group Labels +- Keywords +- Budget Summary Report + +All `date` and `date-time` fields will be converted to standard `RFC3339`. Stream state format will be updated as well. + +For the changes to take effect, please refresh the source schema and reset affected streams after you have applied the upgrade. + +| Stream field | Current Airbyte Type | New Airbyte Type | +|-----------------------------|----------------------|-------------------| +| LinkedAgencies | string | object | +| BiddingScheme.MaxCpc.Amount | string | number | +| CostPerConversion | integer | number | +| Modified Time | string | timestamp with tz | +| Date | string | date | +| TimePeriod | string | timestamp with tz | + +Detailed date-time field change examples: + +| Affected streams | Field_name | Old type | New type (`RFC3339`) | +|----------------------------------------------------------------------------------------------------------------------|-----------------|---------------------------|---------------------------------| +| `AppInstallAds`, `AppInstallAdLabels`, `Labels`, `Campaign Labels`, `Keyword Labels`, `Ad Group Labels`, `Keywords` | `Modified Time` | `04/27/2023 18:00:14.970` | `2023-04-27T16:00:14.970+00:00` | +| `Budget Summary Report` | `Date` | `6/10/2021` | `2021-06-10` | +| `* Report Hourly` | `TimePeriod` | `2023-11-04\|11` | `2023-11-04T11:00:00+00:00` | + +## Upgrading to 1.0.0 + +This version update only affects the geographic performance reports streams. + +Version 1.0.0 prevents the data loss by removing the primary keys from the `GeographicPerformanceReportMonthly`, `GeographicPerformanceReportWeekly`, `GeographicPerformanceReportDaily`, `GeographicPerformanceReportHourly` streams. +Due to multiple records with the same primary key, users could experience data loss in the incremental append+dedup mode because of deduplication. + +For the changes to take effect, please reset your data and refresh the stream schemas after you have applied the upgrade. \ No newline at end of file diff --git a/docs/integrations/sources/bing-ads.inapp.md b/docs/integrations/sources/bing-ads.inapp.md deleted file mode 100644 index 225e16d1352a..000000000000 --- a/docs/integrations/sources/bing-ads.inapp.md +++ /dev/null @@ -1,33 +0,0 @@ -## Prerequisites -* [Microsoft developer token](https://docs.microsoft.com/en-us/advertising/guides/get-started?view=bingads-13#get-developer-token) of an authorized Bing Ads OAuth application - -To use the Bing Ads API, you must have a developer token and valid user credentials. You will need to register an OAuth app to get a refresh token. -1. [Register your application](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-register?view=bingads-13) in the Azure portal. -2. [Request user consent](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-consent?view=bingads-13l) to get the authorization code. -3. Use the authorization code to [get a refresh token](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-get-tokens?view=bingads-13). - -:::note - -The refresh token expires every 90 days. Repeat the authorization process to get a new refresh token. The full authentication process described [here](https://docs.microsoft.com/en-us/advertising/guides/get-started?view=bingads-13#access-token). -Please be sure to authenticate with the email (personal or work) that you used to sign in to the Bing ads/Microsoft ads platform. -::: - -4. Request your [Microsoft developer token](https://docs.microsoft.com/en-us/advertising/guides/get-started?view=bingads-13#get-developer-token) in the Microsoft Advertising Developer Portal account tab. - - -## Setup guide -1. Enter a name for your source. -2. Enter the developer token -3. For **Tenant ID**, enter the custom tenant or use the common tenant. If your OAuth app has a custom tenant and you cannot use Microsoft’s recommended common tenant, use the custom tenant in the Tenant ID field when you set up the connector. - -:::info -The custom tenant is used in the authentication URL, for example: `https://login.microsoftonline.com//oauth2/v2.0/authorize` - -::: - -4. For **Replication Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. -5. For **Lookback window** (also known as attribution or conversion window), enter the number of **days** to look into the past. If your conversion window has an hours/minutes granularity, round it up to the number of days exceeding. If you're not using performance report streams in incremental mode, let it with 0 default value. -6. Click **Authenticate your Bing Ads account** and authorize your account. -8. Click **Set up source**. - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Bing Ads](https://docs.airbyte.com/integrations/sources/bing-ads). diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md index 20910a85be35..fe74163adf14 100644 --- a/docs/integrations/sources/bing-ads.md +++ b/docs/integrations/sources/bing-ads.md @@ -1,10 +1,20 @@ # Bing Ads + + This page contains the setup guide and reference information for the Bing Ads source connector. + + +## Prerequisites +- Microsoft Advertising account +- Microsoft Developer Token + ## Setup guide -### Step 1: Set up Bing Ads + + +For Airbyte Open Source set up your application to get **Client ID**, **Client Secret**, **Refresh Token** 1. [Register your application](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-register?view=bingads-13) in the Azure portal. 2. [Request user consent](https://docs.microsoft.com/en-us/advertising/guides/authentication-oauth-consent?view=bingads-13l) to get the authorization code. @@ -16,8 +26,16 @@ The refresh token expires in 90 days. Repeat the authorization process to get a Please be sure to authenticate with the email (personal or work) that you used to sign in to the Bing ads/Microsoft ads platform. ::: -4. Get your [Microsoft developer token](https://docs.microsoft.com/en-us/advertising/guides/get-started?view=bingads-13#get-developer-token). -5. If your OAuth app has a custom tenant and you cannot use Microsoft’s recommended common tenant, use the custom tenant in the **Tenant ID** field when you set up the connector. + + +### Step 1: Set up Bing Ads + +1. Get your [Microsoft developer token](https://docs.microsoft.com/en-us/advertising/guides/get-started?view=bingads-13#get-developer-token). To use Bing Ads APIs, you must have a developer token and valid user credentials. See [Microsoft Advertising docs](https://docs.microsoft.com/en-us/advertising/guides/get-started?view=bingads-13#get-developer-token) for more info. + 1. Sign in with [Super Admin](https://learn.microsoft.com/en-us/advertising/guides/account-hierarchy-permissions?view=bingads-13#user-roles-permissions) credentials at the [Microsoft Advertising Developer Portal](https://developers.ads.microsoft.com/Account) account tab. + 2. Choose the user that you want associated with the developer token. Typically an application only needs one universal token regardless how many users will be supported. + 3. Click on the Request Token button. + +2. If your OAuth app has a custom tenant, and you cannot use Microsoft’s recommended common tenant, use the custom tenant in the **Tenant ID** field when you set up the connector. :::info @@ -37,11 +55,17 @@ The tenant is used in the authentication URL, for example: `https://login.micros 4. Enter a name for your source. 5. For **Tenant ID**, enter the custom tenant or use the common tenant. 6. Add the developer token from [Step 1](#step-1-set-up-bing-ads). -7. For **Replication Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. -8. For **Lookback window** (also known as attribution or conversion window) enter the number of **days** to look into the past. If your conversion window has an hours/minutes granularity, round it up to the number of days exceeding. If you're not using performance report streams in incremental mode, let it with 0 default value. -9. Click **Authenticate your Bing Ads account**. -10. Log in and authorize the Bing Ads account. -11. Click **Set up source**. +7. For **Account Names Predicates** - see [predicates](https://learn.microsoft.com/en-us/advertising/customer-management-service/predicate?view=bingads-13) in bing ads docs. Will be used to filter your accounts by specified operator and account name. You can use multiple predicates pairs. The **Operator** is a one of Contains or Equals. The **Account Name** is a value to compare Accounts Name field in rows by specified operator. For example, for operator=Contains and name=Dev, all accounts where name contains dev will be replicated. And for operator=Equals and name=Airbyte, all accounts where name is equal to Airbyte will be replicated. Account Name value is not case-sensitive. +8. For **Reports Replication Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data from previous and current calendar years. +9. For **Lookback window** (also known as attribution or conversion window) enter the number of **days** to look into the past. If your conversion window has an hours/minutes granularity, round it up to the number of days exceeding. If you're not using performance report streams in incremental mode and Reports Start Date is not provided, let it with 0 default value. +10. For *Custom Reports* - see [custom reports](#custom-reports) section, list of custom reports object: + 1. For *Report Name* enter the name that you want for your custom report. + 2. For *Reporting Data Object* add the Bing Ads Reporting Object that you want to sync in the custom report. + 3. For *Columns* add list columns of Reporting Data Object that you want to see in the custom report. + 4. For *Aggregation* add time aggregation. See [report aggregation](#report-aggregation) section. +11. Click **Authenticate your Bing Ads account**. +12. Log in and authorize the Bing Ads account. +13. Click **Set up source**. @@ -54,11 +78,20 @@ The tenant is used in the authentication URL, for example: `https://login.micros 4. Enter a name for your source. 5. For **Tenant ID**, enter the custom tenant or use the common tenant. 6. Enter the **Client ID**, **Client Secret**, **Refresh Token**, and **Developer Token** from [Step 1](#step-1-set-up-bing-ads). -7. For **Replication Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. -8. For **Lookback window** (also known as attribution or conversion window) enter the number of **days** to look into the past. If your conversion window has an hours/minutes granularity, round it up to the number of days exceeding. If you're not using performance report streams in incremental mode, let it with 0 default value. -9. Click **Set up source**. +7. For **Account Names Predicates** - see [predicates](https://learn.microsoft.com/en-us/advertising/customer-management-service/predicate?view=bingads-13) in bing ads docs. Will be used to filter your accounts by specified operator and account name. You can use multiple predicates pairs. The **Operator** is a one of Contains or Equals. The **Account Name** is a value to compare Accounts Name field in rows by specified operator. For example, for operator=Contains and name=Dev, all accounts where name contains dev will be replicated. And for operator=Equals and name=Airbyte, all accounts where name is equal to Airbyte will be replicated. Account Name value is not case-sensitive. +8. For **Reports Replication Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data from previous and current calendar years. +9. For **Lookback window** (also known as attribution or conversion window) enter the number of **days** to look into the past. If your conversion window has an hours/minutes granularity, round it up to the number of days exceeding. If you're not using performance report streams in incremental mode and Reports Start Date is not provided, let it with 0 default value. +10. For *Custom Reports* - see [custom reports](#custom-reports) section: + 1. For *Report Name* enter the name that you want for your custom report. + 2. For *Reporting Data Object* add the Bing Ads Reporting Object that you want to sync in the custom report. + 3. For *Columns* add columns of Reporting Data Object that you want to see in the custom report. + 4. For *Aggregation* select time aggregation. See [report aggregation](#report-aggregation) section. + +11. Click **Set up source**. + + ## Supported sync modes The Bing Ads source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): @@ -74,38 +107,93 @@ The Bing Ads source connector supports the following streams. For more informati ### Basic streams -- [accounts](https://docs.microsoft.com/en-us/advertising/customer-management-service/searchaccounts?view=bingads-13) -- [ad_groups](https://docs.microsoft.com/en-us/advertising/campaign-management-service/getadgroupsbycampaignid?view=bingads-13) -- [ads](https://docs.microsoft.com/en-us/advertising/campaign-management-service/getadsbyadgroupid?view=bingads-13) -- [campaigns](https://docs.microsoft.com/en-us/advertising/campaign-management-service/getcampaignsbyaccountid?view=bingads-13) +- [Accounts](https://docs.microsoft.com/en-us/advertising/customer-management-service/searchaccounts?view=bingads-13) +- [Ad Groups](https://docs.microsoft.com/en-us/advertising/campaign-management-service/getadgroupsbycampaignid?view=bingads-13) +- [Ad Group Labels](https://learn.microsoft.com/en-us/advertising/bulk-service/ad-group-label?view=bingads-13) +- [Ads](https://docs.microsoft.com/en-us/advertising/campaign-management-service/getadsbyadgroupid?view=bingads-13) +- [App Install Ads](https://learn.microsoft.com/en-us/advertising/bulk-service/app-install-ad?view=bingads-13) +- [App Install Ad Labels](https://learn.microsoft.com/en-us/advertising/bulk-service/app-install-ad-label?view=bingads-13) +- [Campaigns](https://docs.microsoft.com/en-us/advertising/campaign-management-service/getcampaignsbyaccountid?view=bingads-13) +- [Campaign Labels](https://learn.microsoft.com/en-us/advertising/bulk-service/campaign-label?view=bingads-13) +- [Keywords](https://learn.microsoft.com/en-us/advertising/bulk-service/keyword?view=bingads-13) +- [Keyword Labels](https://learn.microsoft.com/en-us/advertising/bulk-service/keyword-label?view=bingads-13) +- [Labels](https://learn.microsoft.com/en-us/advertising/bulk-service/label?view=bingads-13) ### Report Streams -- [account_performance_report_hourly](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13) -- [account_performance_report_daily](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13) -- [account_performance_report_weekly](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13) -- [account_performance_report_monthly](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13) -- [ad_group_performance_report_hourly](https://docs.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13) -- [ad_group_performance_report_daily](https://docs.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13) -- [ad_group_performance_report_weekly](https://docs.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13) -- [ad_group_performance_report_monthly](https://docs.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13) -- [ad_performance_report_hourly](https://docs.microsoft.com/en-us/advertising/reporting-service/adperformancereportrequest?view=bingads-13) -- [ad_performance_report_daily](https://docs.microsoft.com/en-us/advertising/reporting-service/adperformancereportrequest?view=bingads-13) -- [ad_performance_report_weekly](https://docs.microsoft.com/en-us/advertising/reporting-service/adperformancereportrequest?view=bingads-13) -- [ad_performance_report_monthly](https://docs.microsoft.com/en-us/advertising/reporting-service/adperformancereportrequest?view=bingads-13) -- [geographic_performance_report_hourly](https://learn.microsoft.com/en-us/advertising/reporting-service/geographicperformancereportrequest?view=bingads-13) -- [geographic_performance_report_daily](https://learn.microsoft.com/en-us/advertising/reporting-service/geographicperformancereportrequest?view=bingads-13) -- [geographic_performance_report_weekly](https://learn.microsoft.com/en-us/advertising/reporting-service/geographicperformancereportrequest?view=bingads-13) -- [geographic_performance_report_monthly](https://learn.microsoft.com/en-us/advertising/reporting-service/geographicperformancereportrequest?view=bingads-13) -- [budget_summary_report](https://docs.microsoft.com/en-us/advertising/reporting-service/budgetsummaryreportrequest?view=bingads-13) -- [campaign_performance_report_hourly](https://docs.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13) -- [campaign_performance_report_daily](https://docs.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13) -- [campaign_performance_report_weekly](https://docs.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13) -- [campaign_performance_report_monthly](https://docs.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13) -- [keyword_performance_report_hourly](https://docs.microsoft.com/en-us/advertising/reporting-service/keywordperformancereportrequest?view=bingads-13) -- [keyword_performance_report_daily](https://docs.microsoft.com/en-us/advertising/reporting-service/keywordperformancereportrequest?view=bingads-13) -- [keyword_performance_report_weekly](https://docs.microsoft.com/en-us/advertising/reporting-service/keywordperformancereportrequest?view=bingads-13) -- [keyword_performance_report_monthly](https://docs.microsoft.com/en-us/advertising/reporting-service/keywordperformancereportrequest?view=bingads-13) +- [Account Performance Report Hourly](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13) +- [Account Performance Report Daily](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13) +- [Account Performance Report Weekly](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13) +- [Account Performance Report Monthly](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13) +- [Account Impression Performance Report Hourly](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13) +- [Account Impression Performance Report Daily](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13) +- [Account Impression Performance Report Weekly](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13) +- [Account Impression Performance Report Monthly](https://docs.microsoft.com/en-us/advertising/reporting-service/accountperformancereportrequest?view=bingads-13) +- [Ad Group Performance Report Hourly](https://docs.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13) +- [Ad Group Performance Report Daily](https://docs.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13) +- [Ad Group Performance Report Weekly](https://docs.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13) +- [Ad Group Performance Report Monthly](https://docs.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13) +- [Ad Group Impression Performance Report Hourly](https://learn.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13) +- [Ad Group Impression Performance Report Daily](https://learn.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13) +- [Ad Group Impression Performance Report Weekly](https://learn.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13) +- [Ad Group Impression Performance Report Monthly](https://learn.microsoft.com/en-us/advertising/reporting-service/adgroupperformancereportrequest?view=bingads-13) +- [Ad Performance Report Hourly](https://docs.microsoft.com/en-us/advertising/reporting-service/adperformancereportrequest?view=bingads-13) +- [Ad Performance Report Daily](https://docs.microsoft.com/en-us/advertising/reporting-service/adperformancereportrequest?view=bingads-13) +- [Ad Performance Report Weekly](https://docs.microsoft.com/en-us/advertising/reporting-service/adperformancereportrequest?view=bingads-13) +- [Ad Performance Report Monthly](https://docs.microsoft.com/en-us/advertising/reporting-service/adperformancereportrequest?view=bingads-13) +- [Age Gender Audience Report Hourly](https://learn.microsoft.com/en-us/advertising/reporting-service/agegenderaudiencereportrequest?view=bingads-13) +- [Age Gender Audience Report Daily](https://learn.microsoft.com/en-us/advertising/reporting-service/agegenderaudiencereportrequest?view=bingads-13) +- [Age Gender Audience Report Weekly](https://learn.microsoft.com/en-us/advertising/reporting-service/agegenderaudiencereportrequest?view=bingads-13) +- [Age Gender Audience Report Monthly](https://learn.microsoft.com/en-us/advertising/reporting-service/agegenderaudiencereportrequest?view=bingads-13) +- [Geographic Performance Report Hourly](https://learn.microsoft.com/en-us/advertising/reporting-service/geographicperformancereportrequest?view=bingads-13) +- [Geographic Performance Report Daily](https://learn.microsoft.com/en-us/advertising/reporting-service/geographicperformancereportrequest?view=bingads-13) +- [Geographic Performance Report Weekly](https://learn.microsoft.com/en-us/advertising/reporting-service/geographicperformancereportrequest?view=bingads-13) +- [Geographic Performance Report Monthly](https://learn.microsoft.com/en-us/advertising/reporting-service/geographicperformancereportrequest?view=bingads-13) +- [Budget Summary Report](https://docs.microsoft.com/en-us/advertising/reporting-service/budgetsummaryreportrequest?view=bingads-13) +- [Campaign Performance Report Hourly](https://docs.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13) +- [Campaign Performance Report Daily](https://docs.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13) +- [Campaign Performance Report Weekly](https://docs.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13) +- [Campaign Performance Report Monthly](https://docs.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13) +- [Campaign Impression Performance Report Hourly](https://learn.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13) +- [Campaign Impression Performance Report Daily](https://learn.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13) +- [Campaign Impression Performance Report Weekly](https://learn.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13) +- [Campaign Impression Performance Report Monthly](https://learn.microsoft.com/en-us/advertising/reporting-service/campaignperformancereportrequest?view=bingads-13) +- [Keyword Performance Report Hourly](https://docs.microsoft.com/en-us/advertising/reporting-service/keywordperformancereportrequest?view=bingads-13) +- [Keyword Performance Report Daily](https://docs.microsoft.com/en-us/advertising/reporting-service/keywordperformancereportrequest?view=bingads-13) +- [Keyword Performance Report Weekly](https://docs.microsoft.com/en-us/advertising/reporting-service/keywordperformancereportrequest?view=bingads-13) +- [Keyword Performance Report Monthly](https://docs.microsoft.com/en-us/advertising/reporting-service/keywordperformancereportrequest?view=bingads-13) +- [User Location Performance Report Hourly](https://learn.microsoft.com/en-us/advertising/reporting-service/userlocationperformancereportrequest?view=bingads-13) +- [User Location Performance Report Daily](https://learn.microsoft.com/en-us/advertising/reporting-service/userlocationperformancereportrequest?view=bingads-13) +- [User Location Performance Report Weekly](https://learn.microsoft.com/en-us/advertising/reporting-service/userlocationperformancereportrequest?view=bingads-13) +- [User Location Performance Report Monthly](https://learn.microsoft.com/en-us/advertising/reporting-service/userlocationperformancereportrequest?view=bingads-13) +- [Search Query Performance Report Hourly](https://learn.microsoft.com/en-us/advertising/reporting-service/searchqueryperformancereportrequest?view=bingads-13) +- [Search Query Performance Report Daily](https://learn.microsoft.com/en-us/advertising/reporting-service/searchqueryperformancereportrequest?view=bingads-13) +- [Search Query Performance Report Weekly](https://learn.microsoft.com/en-us/advertising/reporting-service/searchqueryperformancereportrequest?view=bingads-13) +- [Search Query Performance Report Monthly](https://learn.microsoft.com/en-us/advertising/reporting-service/searchqueryperformancereportrequest?view=bingads-13) + +:::info + +Ad Group Impression Performance Report, Geographic Performance Report, Account Impression Performance Report have user-defined primary key. +This means that you can define your own primary key in Replication tab in your connection for these streams. + +Example pk: +Ad Group Impression Performance Report: composite pk - [AdGroupId, Status, TimePeriod, AccountId] +Geographic Performance Report: composite pk - [AdGroupId, Country, State, MetroArea, City] +Account Impression Performance Report: composite pk - [AccountName, AccountNumber, AccountId, TimePeriod] + +Note: These are just examples, and you should consider your own data and needs in order to correctly define the primary key. + +See more info about user-defined pk [here](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped#user-defined-primary-key). + +::: + +### Custom Reports +You can build your own report by providing: +- *Report Name* - name of the stream +- *Reporting Data Object* - Bing Ads reporting data object that you can find [here](https://learn.microsoft.com/en-us/advertising/reporting-service/reporting-data-objects?view=bingads-13). All data object with ending ReportRequest can be used as data object in custom reports. +- *Columns* - Reporting object columns that you want to sync. You can find it on ReportRequest data object page by clicking the ...ReportColumn link in [Bing Ads docs](https://learn.microsoft.com/en-us/advertising/reporting-service/reporting-value-sets?view=bingads-13). +The report must include the Required Columns (you can find it under list of all columns of reporting object) at a minimum. As a general rule, each report must include at least one attribute column and at least one non-impression share performance statistics column. Be careful you can't add extra columns that not specified in Bing Ads docs and not all fields can be skipped. +- *Aggregation* - Hourly, Daily, Weekly, Monthly, DayOfWeek, HourOfDay, WeeklyStartingMonday, Summary. See [report aggregation](#report-aggregation). ### Report aggregation @@ -115,37 +203,80 @@ For example, if you select a report with daily aggregation, the report will cont A report's aggregation window is indicated in its name. For example, `account_performance_report_hourly` is the Account Performance Reported aggregated using an hourly window. -## Performance considerations +## Limitations & Troubleshooting + +
      + +Expand to see details about Bing Ads connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting The Bing Ads API limits the number of requests for all Microsoft Advertising clients. You can find detailed info [here](https://docs.microsoft.com/en-us/advertising/guides/services-protocol?view=bingads-13#throttling). +### Troubleshooting + +* Check out common troubleshooting issues for the Bing Ads source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +
      + ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------- | -| 0.2.0 | 2023-08-17 | [27619](https://github.com/airbytehq/airbyte/pull/27619) | Add Geographic Performance Report | -| 0.1.24 | 2023-06-22 | [27619](https://github.com/airbytehq/airbyte/pull/27619) | Retry request after facing temporary name resolution error. | -| 0.1.23 | 2023-05-11 | [25996](https://github.com/airbytehq/airbyte/pull/25996) | Implement a retry logic if SSL certificate validation fails. | -| 0.1.22 | 2023-05-08 | [24223](https://github.com/airbytehq/airbyte/pull/24223) | Add CampaignLabels report column in campaign performance report | -| 0.1.21 | 2023-04-28 | [25668](https://github.com/airbytehq/airbyte/pull/25668) | Add undeclared fields to accounts, campaigns, campaign_performance_report, keyword_performance_report and account_performance_report streams | -| 0.1.20 | 2023-03-09 | [23663](https://github.com/airbytehq/airbyte/pull/23663) | Add lookback window for performance reports in incremental mode | -| 0.1.19 | 2023-03-08 | [23868](https://github.com/airbytehq/airbyte/pull/23868) | Add dimensional-type columns for reports. | -| 0.1.18 | 2023-01-30 | [22073](https://github.com/airbytehq/airbyte/pull/22073) | Fix null values in the `Keyword` column of `keyword_performance_report` streams | -| 0.1.17 | 2022-12-10 | [20005](https://github.com/airbytehq/airbyte/pull/20005) | Add `Keyword` to `keyword_performance_report` stream | -| 0.1.16 | 2022-10-12 | [17873](https://github.com/airbytehq/airbyte/pull/17873) | Fix: added missing campaign types in (Audience, Shopping and DynamicSearchAds) in campaigns stream | -| 0.1.15 | 2022-10-03 | [17505](https://github.com/airbytehq/airbyte/pull/17505) | Fix: limit cache size for ServiceClient instances | -| 0.1.14 | 2022-09-29 | [17403](https://github.com/airbytehq/airbyte/pull/17403) | Fix: limit cache size for ReportingServiceManager instances | -| 0.1.13 | 2022-09-29 | [17386](https://github.com/airbytehq/airbyte/pull/17386) | Migrate to per-stream states. | -| 0.1.12 | 2022-09-05 | [16335](https://github.com/airbytehq/airbyte/pull/16335) | Added backoff for socket.timeout | -| 0.1.11 | 2022-08-25 | [15684](https://github.com/airbytehq/airbyte/pull/15684) (published in [15987](https://github.com/airbytehq/airbyte/pull/15987)) | Fixed log messages being unreadable | -| 0.1.10 | 2022-08-12 | [15602](https://github.com/airbytehq/airbyte/pull/15602) | Fixed bug caused Hourly Reports to crash due to invalid fields set | -| 0.1.9 | 2022-08-02 | [14862](https://github.com/airbytehq/airbyte/pull/14862) | Added missing columns | -| 0.1.8 | 2022-06-15 | [13801](https://github.com/airbytehq/airbyte/pull/13801) | All reports `hourly/daily/weekly/monthly` will be generated by default, these options are removed from input configuration | -| 0.1.7 | 2022-05-17 | [12937](https://github.com/airbytehq/airbyte/pull/12937) | Added OAuth2.0 authentication method, removed `redirect_uri` from input configuration | -| 0.1.6 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | -| 0.1.5 | 2022-01-01 | [11652](https://github.com/airbytehq/airbyte/pull/11652) | Rebump attempt after DockerHub failure at registring the 0.1.4 | -| 0.1.4 | 2022-03-22 | [11311](https://github.com/airbytehq/airbyte/pull/11311) | Added optional Redirect URI & Tenant ID to spec | -| 0.1.3 | 2022-01-14 | [9510](https://github.com/airbytehq/airbyte/pull/9510) | Fixed broken dependency that blocked connector's operations | -| 0.1.2 | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Update titles and descriptions | -| 0.1.1 | 2021-08-31 | [5750](https://github.com/airbytehq/airbyte/pull/5750) | Added reporting streams\) | -| 0.1.0 | 2021-07-22 | [4911](https://github.com/airbytehq/airbyte/pull/4911) | Initial release supported core streams \(Accounts, Campaigns, Ads, AdGroups\) | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------| +| 2.1.2 | 2024-01-09 | [34045](https://github.com/airbytehq/airbyte/pull/34045) | Speed up record transformation | +| 2.1.1 | 2023-12-15 | [33500](https://github.com/airbytehq/airbyte/pull/33500) | Fix state setter when state was provided | +| 2.1.0 | 2023-12-05 | [33095](https://github.com/airbytehq/airbyte/pull/33095) | Add account filtering | +| 2.0.1 | 2023-11-16 | [32597](https://github.com/airbytehq/airbyte/pull/32597) | Fix start date parsing from stream state | +| 2.0.0 | 2023-11-07 | [31995](https://github.com/airbytehq/airbyte/pull/31995) | Schema update for Accounts, Campaigns and Search Query Performance Report streams. Convert `date` and `date-time` fields to standard `RFC3339` | +| 1.13.0 | 2023-11-13 | [32306](https://github.com/airbytehq/airbyte/pull/32306) | Add Custom reports and decrease backoff max tries number | +| 1.12.1 | 2023-11-10 | [32422](https://github.com/airbytehq/airbyte/pull/32422) | Normalize numeric values in reports | +| 1.12.0 | 2023-11-09 | [32340](https://github.com/airbytehq/airbyte/pull/32340) | Remove default start date in favor of Time Period - Last Year and This Year, if start date is not provided | +| 1.11.0 | 2023-11-06 | [32201](https://github.com/airbytehq/airbyte/pull/32201) | Skip broken CSV report files | +| 1.10.0 | 2023-11-06 | [32148](https://github.com/airbytehq/airbyte/pull/32148) | Add new fields to stream Ads: "BusinessName", "CallToAction", "Headline", "Images", "Videos", "Text" | +| 1.9.0 | 2023-11-03 | [32131](https://github.com/airbytehq/airbyte/pull/32131) | Add "CampaignId", "AccountId", "CustomerId" fields to Ad Groups, Ads and Campaigns streams. | +| 1.8.0 | 2023-11-02 | [32059](https://github.com/airbytehq/airbyte/pull/32059) | Add new streams `CampaignImpressionPerformanceReport` (daily, hourly, weekly, monthly) | +| 1.7.1 | 2023-11-02 | [32088](https://github.com/airbytehq/airbyte/pull/32088) | Raise config error when user does not have accounts | +| 1.7.0 | 2023-11-01 | [32027](https://github.com/airbytehq/airbyte/pull/32027) | Add new streams `AdGroupImpressionPerformanceReport` | +| 1.6.0 | 2023-10-31 | [32008](https://github.com/airbytehq/airbyte/pull/32008) | Add new streams `Keywords` | +| 1.5.0 | 2023-10-30 | [31952](https://github.com/airbytehq/airbyte/pull/31952) | Add new streams `Labels`, `App install ads`, `Keyword Labels`, `Campaign Labels`, `App Install Ad Labels`, `Ad Group Labels` | +| 1.4.0 | 2023-10-27 | [31885](https://github.com/airbytehq/airbyte/pull/31885) | Add new stream: `AccountImpressionPerformanceReport` (daily, hourly, weekly, monthly) | +| 1.3.0 | 2023-10-26 | [31837](https://github.com/airbytehq/airbyte/pull/31837) | Add new stream: `UserLocationPerformanceReport` (daily, hourly, weekly, monthly) | +| 1.2.0 | 2023-10-24 | [31783](https://github.com/airbytehq/airbyte/pull/31783) | Add new stream: `SearchQueryPerformanceReport` (daily, hourly, weekly, monthly) | +| 1.1.0 | 2023-10-24 | [31712](https://github.com/airbytehq/airbyte/pull/31712) | Add new stream: `AgeGenderAudienceReport` (daily, hourly, weekly, monthly) | +| 1.0.2 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 1.0.1 | 2023-10-16 | [31432](https://github.com/airbytehq/airbyte/pull/31432) | Remove primary keys from the geographic performance reports - complete what was missed in version 1.0.0 | +| 1.0.0 | 2023-10-11 | [31277](https://github.com/airbytehq/airbyte/pull/31277) | Remove primary keys from the geographic performance reports. | +| 0.2.3 | 2023-09-28 | [30834](https://github.com/airbytehq/airbyte/pull/30834) | Wrap auth error with the config error. | +| 0.2.2 | 2023-09-27 | [30791](https://github.com/airbytehq/airbyte/pull/30791) | Fix missing fields for geographic performance reports. | +| 0.2.1 | 2023-09-04 | [30128](https://github.com/airbytehq/airbyte/pull/30128) | Add increasing download timeout if ReportingDownloadException occurs | +| 0.2.0 | 2023-08-17 | [27619](https://github.com/airbytehq/airbyte/pull/27619) | Add Geographic Performance Report | +| 0.1.24 | 2023-06-22 | [27619](https://github.com/airbytehq/airbyte/pull/27619) | Retry request after facing temporary name resolution error. | +| 0.1.23 | 2023-05-11 | [25996](https://github.com/airbytehq/airbyte/pull/25996) | Implement a retry logic if SSL certificate validation fails. | +| 0.1.22 | 2023-05-08 | [24223](https://github.com/airbytehq/airbyte/pull/24223) | Add CampaignLabels report column in campaign performance report | +| 0.1.21 | 2023-04-28 | [25668](https://github.com/airbytehq/airbyte/pull/25668) | Add undeclared fields to accounts, campaigns, campaign_performance_report, keyword_performance_report and account_performance_report streams | +| 0.1.20 | 2023-03-09 | [23663](https://github.com/airbytehq/airbyte/pull/23663) | Add lookback window for performance reports in incremental mode | +| 0.1.19 | 2023-03-08 | [23868](https://github.com/airbytehq/airbyte/pull/23868) | Add dimensional-type columns for reports. | +| 0.1.18 | 2023-01-30 | [22073](https://github.com/airbytehq/airbyte/pull/22073) | Fix null values in the `Keyword` column of `keyword_performance_report` streams | +| 0.1.17 | 2022-12-10 | [20005](https://github.com/airbytehq/airbyte/pull/20005) | Add `Keyword` to `keyword_performance_report` stream | +| 0.1.16 | 2022-10-12 | [17873](https://github.com/airbytehq/airbyte/pull/17873) | Fix: added missing campaign types in (Audience, Shopping and DynamicSearchAds) in campaigns stream | +| 0.1.15 | 2022-10-03 | [17505](https://github.com/airbytehq/airbyte/pull/17505) | Fix: limit cache size for ServiceClient instances | +| 0.1.14 | 2022-09-29 | [17403](https://github.com/airbytehq/airbyte/pull/17403) | Fix: limit cache size for ReportingServiceManager instances | +| 0.1.13 | 2022-09-29 | [17386](https://github.com/airbytehq/airbyte/pull/17386) | Migrate to per-stream states. | +| 0.1.12 | 2022-09-05 | [16335](https://github.com/airbytehq/airbyte/pull/16335) | Added backoff for socket.timeout | +| 0.1.11 | 2022-08-25 | [15684](https://github.com/airbytehq/airbyte/pull/15684) (published in [15987](https://github.com/airbytehq/airbyte/pull/15987)) | Fixed log messages being unreadable | +| 0.1.10 | 2022-08-12 | [15602](https://github.com/airbytehq/airbyte/pull/15602) | Fixed bug caused Hourly Reports to crash due to invalid fields set | +| 0.1.9 | 2022-08-02 | [14862](https://github.com/airbytehq/airbyte/pull/14862) | Added missing columns | +| 0.1.8 | 2022-06-15 | [13801](https://github.com/airbytehq/airbyte/pull/13801) | All reports `hourly/daily/weekly/monthly` will be generated by default, these options are removed from input configuration | +| 0.1.7 | 2022-05-17 | [12937](https://github.com/airbytehq/airbyte/pull/12937) | Added OAuth2.0 authentication method, removed `redirect_uri` from input configuration | +| 0.1.6 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | +| 0.1.5 | 2022-01-01 | [11652](https://github.com/airbytehq/airbyte/pull/11652) | Rebump attempt after DockerHub failure at registring the 0.1.4 | +| 0.1.4 | 2022-03-22 | [11311](https://github.com/airbytehq/airbyte/pull/11311) | Added optional Redirect URI & Tenant ID to spec | +| 0.1.3 | 2022-01-14 | [9510](https://github.com/airbytehq/airbyte/pull/9510) | Fixed broken dependency that blocked connector's operations | +| 0.1.2 | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Update titles and descriptions | +| 0.1.1 | 2021-08-31 | [5750](https://github.com/airbytehq/airbyte/pull/5750) | Added reporting streams\) | +| 0.1.0 | 2021-07-22 | [4911](https://github.com/airbytehq/airbyte/pull/4911) | Initial release supported core streams \(Accounts, Campaigns, Ads, AdGroups\) | + +
      \ No newline at end of file diff --git a/docs/integrations/sources/braintree.md b/docs/integrations/sources/braintree.md index fb4663ce73b2..f16c2cabae2d 100644 --- a/docs/integrations/sources/braintree.md +++ b/docs/integrations/sources/braintree.md @@ -71,6 +71,7 @@ The Braintree connector should not run into Braintree API limitations under norm | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.2.1 | 2023-11-08 | [31489](https://github.com/airbytehq/airbyte/pull/31489) | Fix transaction stream custom fields | | 0.2.0 | 2023-07-17 | [29200](https://github.com/airbytehq/airbyte/pull/29200) | Migrate connector to low-code framework | | 0.1.5 | 2023-05-24 | [26340](https://github.com/airbytehq/airbyte/pull/26340) | Fix error in `check_connection` in integration tests | | 0.1.4 | 2023-03-13 | [23548](https://github.com/airbytehq/airbyte/pull/23548) | Update braintree python library version to 4.18.1 | diff --git a/docs/integrations/sources/braze.md b/docs/integrations/sources/braze.md index f4c2ccdc42ca..9f97ff8a5951 100644 --- a/docs/integrations/sources/braze.md +++ b/docs/integrations/sources/braze.md @@ -51,6 +51,9 @@ Rate limits table: https://www.braze.com/docs/api/api_limits/#rate-limits-by-req | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------|:-----------------------------------| +| 0.3.0 | 2023-11-04 | [31857](https://github.com/airbytehq/airbyte/pull/31857) | Add Campaigns, Canvases, Segments Details Streams | +| 0.2.0 | 2023-10-28 | [31607](https://github.com/airbytehq/airbyte/pull/31607) | Fix CanvasAnalytics Stream Null Data for step_stats, variant_stats | +| 0.1.4 | 2023-11-03 | [20520](https://github.com/airbytehq/airbyte/pull/20520) | Fix integration tests | | 0.1.3 | 2022-12-15 | [20520](https://github.com/airbytehq/airbyte/pull/20520) | The Braze connector born | diff --git a/docs/integrations/sources/cart.md b/docs/integrations/sources/cart.md index 0559e6c7f3a3..4e79d99f0761 100644 --- a/docs/integrations/sources/cart.md +++ b/docs/integrations/sources/cart.md @@ -50,6 +50,8 @@ Please follow these [steps](https://developers.cart.com/docs/rest-api/docs/READM | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------- | +| 0.3.1 | 2023-11-21 | [32705](https://github.com/airbytehq/airbyte/pull/32705) | Update CDK version | +| 0.3.0 | 2023-11-14 | [23317](https://github.com/airbytehq/airbyte/pull/23317) | Update schemas | | 0.2.1 | 2023-02-22 | [23317](https://github.com/airbytehq/airbyte/pull/23317) | Remove support for incremental for `order_statuses` stream | | 0.2.0 | 2022-09-21 | [16612](https://github.com/airbytehq/airbyte/pull/16612) | Source Cart.com: implement Central API Router access method and improve backoff policy | | 0.1.6 | 2022-07-15 | [14752](https://github.com/airbytehq/airbyte/pull/14752) | Add `order_statuses` stream | diff --git a/docs/integrations/sources/chargebee.md b/docs/integrations/sources/chargebee.md index 81a64c2a707c..3bce7a464f78 100644 --- a/docs/integrations/sources/chargebee.md +++ b/docs/integrations/sources/chargebee.md @@ -4,7 +4,11 @@ This page contains the setup guide and reference information for the Chargebee s ## Prerequisites -To set up the Chargebee source connector, you'll need the [Chargebee API key](https://apidocs.chargebee.com/docs/api?prod_cat_ver=2#api_authentication) and the [Product Catalog version](https://apidocs.chargebee.com/docs/api?prod_cat_ver=2). +To set up the Chargebee source connector, you will need a valid [Chargebee API key](https://apidocs.chargebee.com/docs/api?prod_cat_ver=2#api_authentication) and the [Product Catalog version](https://www.chargebee.com/docs/1.0/upgrade-product-catalog.html) of the Chargebee site you are syncing data from. + +:::info +All Chargebee sites created from May 5, 2021 onward will have [Product Catalog 2.0](https://www.chargebee.com/docs/2.0/product-catalog.html) enabled by default. Sites created prior to this date will use [Product Catalog 1.0](https://www.chargebee.com/docs/1.0/product-catalog.html). +::: ## Set up the Chargebee connector in Airbyte @@ -26,45 +30,42 @@ The Chargebee source connector supports the following [sync modes](https://docs. * [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append) * [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) -## Supported Streams - -* [Subscriptions](https://apidocs.chargebee.com/docs/api/subscriptions?prod_cat_ver=2#list_subscriptions) -* [Customers](https://apidocs.chargebee.com/docs/api/customers?prod_cat_ver=2#list_customers) -* [Invoices](https://apidocs.chargebee.com/docs/api/invoices?prod_cat_ver=2#list_invoices) -* [Orders](https://apidocs.chargebee.com/docs/api/orders?prod_cat_ver=2#list_orders) -* [Plans](https://apidocs.chargebee.com/docs/api/plans?prod_cat_ver=1&lang=curl#list_plans) -* [Addons](https://apidocs.chargebee.com/docs/api/addons?prod_cat_ver=1&lang=curl#list_addons) -* [Items](https://apidocs.chargebee.com/docs/api/items?prod_cat_ver=2#list_items) -* [Item Prices](https://apidocs.chargebee.com/docs/api/item_prices?prod_cat_ver=2#list_item_prices) -* [Attached Items](https://apidocs.chargebee.com/docs/api/attached_items?prod_cat_ver=2#list_attached_items) - -Some streams are available only for specific on Product Catalog versions: - -1. Available in `Product Catalog 1.0` and `Product Catalog 2.0`: - * Customers - * Events - * Invoices - * Credit Notes - * Orders - * Coupons - * Subscriptions - * Transactions -2. Available only in `Product Catalog 1.0`: - * Plans - * Addons -3. Available only in `Product Catalog 2.0`: - * Items - * Item Prices - * Attached Items - -Note that except the `Attached Items` stream, all the streams listed above are incremental streams, which means they: - -* Read only new records -* Output only new records - -The `Attached Items` stream is also incremental but it reads _all_ records and outputs only new records, which is why syncing the `Attached Items` stream, even in incremental mode, is expensive in terms of your Chargebee API quota. - -Generally speaking, it incurs a number of API calls equal to the total number of attached items in your chargebee instance divided by 100, regardless of how many `AttachedItems` were actually changed or synced in a particular sync job. +## Supported streams + +Most streams are supported regardless of your Chargebee site's [Product Catalog version](https://www.chargebee.com/docs/1.0/upgrade-product-catalog.html), with a few version-specific exceptions. + +| Stream | Product Catalog 1.0 | Product Catalog 2.0 | +| ------------------------------------------------------------------------------------------------------ | ------------------- | ------------------- | +| [Addons](https://apidocs.chargebee.com/docs/api/addons?prod_cat_ver=1) | ✔ | | +| [Attached Items](https://apidocs.chargebee.com/docs/api/attached_items?prod_cat_ver=2) | | ✔ | +| [Comments](https://apidocs.chargebee.com/docs/api/comments?prod_cat_ver=2) | ✔ | ✔ | +| [Contacts](https://apidocs.chargebee.com/docs/api/customers?lang=curl#list_of_contacts_for_a_customer) | ✔ | ✔ | +| [Coupons](https://apidocs.chargebee.com/docs/api/coupons) | ✔ | ✔ | +| [Credit Notes](https://apidocs.chargebee.com/docs/api/credit_notes) | ✔ | ✔ | +| [Customers](https://apidocs.chargebee.com/docs/api/customers) | ✔ | ✔ | +| [Differential Prices](https://apidocs.chargebee.com/docs/api/differential_prices) | ✔ | ✔ | +| [Events](https://apidocs.chargebee.com/docs/api/events) | ✔ | ✔ | +| [Gifts](https://apidocs.chargebee.com/docs/api/gifts) | ✔ | ✔ | +| [Hosted Pages](https://apidocs.chargebee.com/docs/api/hosted_pages) | ✔ | ✔ | +| [Invoices](https://apidocs.chargebee.com/docs/api/invoices) | ✔ | ✔ | +| [Items](https://apidocs.chargebee.com/docs/api/items?prod_cat_ver=2) | | ✔ | +| [Item Prices](https://apidocs.chargebee.com/docs/api/item_prices?prod_cat_ver=2) | | ✔ | +| [Item Families](https://apidocs.chargebee.com/docs/api/item_families?prod_cat_ver=2) | | ✔ | +| [Orders](https://apidocs.chargebee.com/docs/api/orders) | ✔ | ✔ | +| [Payment Sources](https://apidocs.chargebee.com/docs/api/payment_sources) | ✔ | ✔ | +| [Plans](https://apidocs.chargebee.com/docs/api/plans?prod_cat_ver=1) | ✔ | | +| [Promotional Credits](https://apidocs.chargebee.com/docs/api/promotional_credits) | ✔ | ✔ | +| [Quotes](https://apidocs.chargebee.com/docs/api/quotes) | ✔ | ✔ | +| [Quote Line Groups](https://apidocs.chargebee.com/docs/api/quote_line_groups) | ✔ | ✔ | +| [Site Migration Details](https://apidocs.chargebee.com/docs/api/site_migration_details) | ✔ | ✔ | +| [Subscriptions](https://apidocs.chargebee.com/docs/api/subscriptions) | ✔ | ✔ | +| [Transactions](https://apidocs.chargebee.com/docs/api/transactions) | ✔ | ✔ | +| [Unbilled Charges](https://apidocs.chargebee.com/docs/api/unbilled_charges) | ✔ | ✔ | +| [Virtual Bank Accounts](https://apidocs.chargebee.com/docs/api/virtual_bank_accounts) | ✔ | ✔ | + +:::note +When using incremental sync mode, the `Attached Items` stream behaves differently than the other streams. Whereas other incremental streams read and output _only new_ records, the `Attached Items` stream reads _all_ records but only outputs _new_ records, making it more demanding on your Chargebee API quota. Each sync incurs API calls equal to the total number of attached items in your Chargebee instance divided by 100, regardless of the actual number of `Attached Items` changed or synced. +::: ## Performance considerations @@ -73,8 +74,11 @@ The Chargebee connector should not run into [Chargebee API](https://apidocs.char ## Changelog | Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------| -| 0.2.4 | 2023-08-01 | [28905](https://github.com/airbytehq/airbyte/pull/28905) | Updated the connector to use latest CDK version | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------------------- | +| 0.3.0 | 2023-12-26 | [33696](https://github.com/airbytehq/airbyte/pull/33696) | Add new stream, add fields to existing streams | +| 0.2.6 | 2023-12-19 | [32100](https://github.com/airbytehq/airbyte/pull/32100) | Add new fields in streams | +| 0.2.5 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 0.2.4 | 2023-08-01 | [28905](https://github.com/airbytehq/airbyte/pull/28905) | Updated the connector to use latest CDK version | | 0.2.3 | 2023-03-22 | [24370](https://github.com/airbytehq/airbyte/pull/24370) | Ignore 404 errors for `Contact` stream | | 0.2.2 | 2023-02-17 | [21688](https://github.com/airbytehq/airbyte/pull/21688) | Migrate to CDK beta 0.29; fix schemas | | 0.2.1 | 2023-02-17 | [23207](https://github.com/airbytehq/airbyte/pull/23207) | Edited stream schemas to get rid of unnecessary `enum` | @@ -95,4 +99,4 @@ The Chargebee connector should not run into [Chargebee API](https://apidocs.char | 0.1.3 | 2021-08-17 | [5421](https://github.com/airbytehq/airbyte/pull/5421) | Add support for "Product Catalog 2.0" specific streams: `Items`, `Item prices` and `Attached Items` | | 0.1.2 | 2021-07-30 | [5067](https://github.com/airbytehq/airbyte/pull/5067) | Prepare connector for publishing | | 0.1.1 | 2021-07-07 | [4539](https://github.com/airbytehq/airbyte/pull/4539) | Add entrypoint and bump version for connector | -| 0.1.0 | 2021-06-30 | [3410](https://github.com/airbytehq/airbyte/pull/3410) | New Source: Chargebee | +| 0.1.0 | 2021-06-30 | [3410](https://github.com/airbytehq/airbyte/pull/3410) | New Source: Chargebee | \ No newline at end of file diff --git a/docs/integrations/sources/chargify.md b/docs/integrations/sources/chargify.md index f533658f4f7e..42ea9fb9bedc 100644 --- a/docs/integrations/sources/chargify.md +++ b/docs/integrations/sources/chargify.md @@ -42,6 +42,7 @@ Please follow the [Chargify documentation for generating an API key](https://dev | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :----------------------------- | +| 0.4.0 | 2023-10-16 | [31116](https://github.com/airbytehq/airbyte/pull/31116) | Add Coupons, Transactions, Invoices Streams | | 0.3.0 | 2023-08-10 | [29130](https://github.com/airbytehq/airbyte/pull/29130) | Migrate Python CDK to Low Code | | 0.2.0 | 2023-08-08 | [29218](https://github.com/airbytehq/airbyte/pull/29218) | Fix schema | | 0.1.0 | 2022-03-16 | [10853](https://github.com/airbytehq/airbyte/pull/10853) | Initial release | diff --git a/docs/integrations/sources/chartmogul-migrations.md b/docs/integrations/sources/chartmogul-migrations.md new file mode 100644 index 000000000000..a0294bf0e818 --- /dev/null +++ b/docs/integrations/sources/chartmogul-migrations.md @@ -0,0 +1,7 @@ +# Chartmogul Migration Guide + +## Upgrading to 1.0.0 + +Version 1.0.0 refactors and separates the `customer_count` stream into multiple streams (daily, weekly, monthly, quarterly). + +Users that have this stream enabled will need to refresh the schema and run a reset to use the new streams in affected connections to continue syncing. diff --git a/docs/integrations/sources/chartmogul.md b/docs/integrations/sources/chartmogul.md index 8d7726109a28..f92aad3361fb 100644 --- a/docs/integrations/sources/chartmogul.md +++ b/docs/integrations/sources/chartmogul.md @@ -4,7 +4,6 @@ This page contains the setup guide and reference information for the [Chartmogul ## Prerequisites - A Chartmogul API Key. - A desired start date from which to begin replicating data. -- A desired interval period for the `CustomerCount` stream. The available options are **day**, **week**, **month**, and **quarter**. ## Setup guide ### Step 1: Set up a Chartmogul API key @@ -30,8 +29,7 @@ For further reading on Chartmogul API Key creation and maintenance, please refer The **Start date** will only apply to the `Activities` stream. The `Customers` endpoint does not provide a way to filter by the creation or update dates. ::: -6. From the **Interval** dropdown menu, select an interval period for the `CustomerCount` stream. -7. Click **Set up source** and wait for the tests to complete. +6. Click **Set up source** and wait for the tests to complete. ## Supported sync modes @@ -45,7 +43,10 @@ The Chartmogul source connector supports the following [sync modes](https://docs This connector outputs the following full refresh streams: * [Activities](https://dev.chartmogul.com/reference/list-activities) -* [CustomerCount](https://dev.chartmogul.com/reference/retrieve-customer-count) +* [CustomerCountDaily](https://dev.chartmogul.com/reference/retrieve-customer-count) +* [CustomerCountWeekly](https://dev.chartmogul.com/reference/retrieve-customer-count) +* [CustomerCountMonthly](https://dev.chartmogul.com/reference/retrieve-customer-count) +* [CustomerCountQuarterly](https://dev.chartmogul.com/reference/retrieve-customer-count) * [Customers](https://dev.chartmogul.com/reference/list-customers) ## Performance considerations @@ -56,6 +57,7 @@ The Chartmogul connector should not run into Chartmogul API limitations under no | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 1.0.0 | 2023-11-09 | [23075](https://github.com/airbytehq/airbyte/pull/23075) | Refactor CustomerCount stream into CustomerCountDaily, CustomerCountWeekly, CustomerCountMonthly, CustomerCountQuarterly Streams | | 0.2.1 | 2023-02-15 | [23075](https://github.com/airbytehq/airbyte/pull/23075) | Specified date formatting in specification | | 0.2.0 | 2022-11-15 | [19276](https://github.com/airbytehq/airbyte/pull/19276) | Migrate connector from Alpha (Python) to Beta (YAML) | | 0.1.1 | 2022-03-02 | [10756](https://github.com/airbytehq/airbyte/pull/10756) | Add new stream: customer-count | diff --git a/docs/integrations/sources/clickhouse.md b/docs/integrations/sources/clickhouse.md index 912065139d93..513180093b18 100644 --- a/docs/integrations/sources/clickhouse.md +++ b/docs/integrations/sources/clickhouse.md @@ -97,18 +97,19 @@ Using this feature requires additional configuration, when creating the source. ## CHANGELOG source-clickhouse-strict-encrypt -| Version | Date | Pull Request | Subject | -|:---| :--- |:---------------------------------------------------------|:---------------------------------------------------------------------------| -| 0.1.17 | 2022-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | -| 0.1.16 |2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to| -| 0.1.15 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------| +| 0.2.0 | 2023-12-18 | [33485](https://github.com/airbytehq/airbyte/pull/33485) | Remove LEGACY state | +| 0.1.17 | 2022-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | +| 0.1.16 | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to | +| 0.1.15 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | -| 0.1.14 | 2022-09-27 | [17031](https://github.com/airbytehq/airbyte/pull/17031) | Added custom jdbc url parameters field | -| 0.1.13 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | -| 0.1.9 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | -| 0.1.6 | 2022-02-09 | [\#10214](https://github.com/airbytehq/airbyte/pull/10214) | Fix exception in case `password` field is not provided | -| 0.1.5 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.1.3 | 2021-12-29 | [\#9182](https://github.com/airbytehq/airbyte/pull/9182) [\#8958](https://github.com/airbytehq/airbyte/pull/8958) | Add support for JdbcType.ARRAY. Fixed tests | -| 0.1.2 | 2021-12-01 | [\#8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | -| 0.1.1 | 20.10.2021 | [\#7327](https://github.com/airbytehq/airbyte/pull/7327) | Added support for connection via SSH tunnel(aka Bastion server). | -| 0.1.0 | 20.10.2021 | [\#7127](https://github.com/airbytehq/airbyte/pull/7127) | Added source-clickhouse-strict-encrypt that supports SSL connections only. | +| 0.1.14 | 2022-09-27 | [17031](https://github.com/airbytehq/airbyte/pull/17031) | Added custom jdbc url parameters field | +| 0.1.13 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | +| 0.1.9 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | +| 0.1.6 | 2022-02-09 | [\#10214](https://github.com/airbytehq/airbyte/pull/10214) | Fix exception in case `password` field is not provided | +| 0.1.5 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.1.3 | 2021-12-29 | [\#9182](https://github.com/airbytehq/airbyte/pull/9182) [\#8958](https://github.com/airbytehq/airbyte/pull/8958) | Add support for JdbcType.ARRAY. Fixed tests | +| 0.1.2 | 2021-12-01 | [\#8371](https://github.com/airbytehq/airbyte/pull/8371) | Fixed incorrect handling "\n" in ssh key | +| 0.1.1 | 20.10.2021 | [\#7327](https://github.com/airbytehq/airbyte/pull/7327) | Added support for connection via SSH tunnel(aka Bastion server). | +| 0.1.0 | 20.10.2021 | [\#7127](https://github.com/airbytehq/airbyte/pull/7127) | Added source-clickhouse-strict-encrypt that supports SSL connections only. | diff --git a/docs/integrations/sources/clockify.md b/docs/integrations/sources/clockify.md index ec3b68c6ee93..5a4a2e22ace8 100644 --- a/docs/integrations/sources/clockify.md +++ b/docs/integrations/sources/clockify.md @@ -6,6 +6,7 @@ The Airbyte Source for [Clockify](https://clockify.me) | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------- | +| 0.3.0 | 2023-08-27 | [TBD](https://github.com/airbytehq/airbyte/pull/TBD) | ✨ Source Clockify: Migrate to LowCode CDK | | 0.2.1 | 2023-08-01 | [27881](https://github.com/airbytehq/airbyte/pull/27881) | 🐛 Source Clockify: Source Clockify: Fix pagination logic | | 0.2.0 | 2023-08-01 | [27689](https://github.com/airbytehq/airbyte/pull/27689) | ✨ Source Clockify: Add Optional API Url parameter | | 0.1.0 | 2022-10-26 | [17767](https://github.com/airbytehq/airbyte/pull/17767) | 🎉 New Connector: Clockify [python cdk] | diff --git a/docs/integrations/sources/close-com.md b/docs/integrations/sources/close-com.md index 9cb94dcc2de1..d152f845fe9c 100644 --- a/docs/integrations/sources/close-com.md +++ b/docs/integrations/sources/close-com.md @@ -105,7 +105,9 @@ The Close.com connector is subject to rate limits. For more information on this | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------| -| 0.4.2 | 2023-08-08 | [29206](https://github.com/airbytehq/airbyte/pull/29206) | Fixed the issue with `DatePicker` format for `start date` | +| 0.5.0 | 2023-11-30 | [32984](https://github.com/airbytehq/airbyte/pull/32984) | Add support for custom fields | +| 0.4.3 | 2023-10-28 | [31534](https://github.com/airbytehq/airbyte/pull/31534) | Fixed Email Activities Stream Pagination | +| 0.4.2 | 2023-08-08 | [29206](https://github.com/airbytehq/airbyte/pull/29206) | Fixed the issue with `DatePicker` format for `start date` | | 0.4.1 | 2023-07-04 | [27950](https://github.com/airbytehq/airbyte/pull/27950) | Add human readable titles to API Key and Start Date fields | | 0.4.0 | 2023-06-27 | [27776](https://github.com/airbytehq/airbyte/pull/27776) | Update the `Email Followup Tasks` stream schema | | 0.3.0 | 2023-05-12 | [26024](https://github.com/airbytehq/airbyte/pull/26024) | Update the `Email sequences` stream schema | diff --git a/docs/integrations/sources/cockroachdb.md b/docs/integrations/sources/cockroachdb.md index a32fddf184d9..5f7a4c843d1a 100644 --- a/docs/integrations/sources/cockroachdb.md +++ b/docs/integrations/sources/cockroachdb.md @@ -93,25 +93,26 @@ Your database user should now be ready for use with Airbyte. ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------| :--- | :--- | -| 0.1.22 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | -| 0.1.21 | 2023-03-14 | [24000](https://github.com/airbytehq/airbyte/pull/24000) | Removed check method call on read. | -| 0.1.20 | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect | -| 0.1.19 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :--- |:------------------------------------------------------------------------------------------------------------------------------------------| +| 0.2.0 | 2023-12-18 | [33485](https://github.com/airbytehq/airbyte/pull/33485) | Removed LEGACY state | +| 0.1.22 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | +| 0.1.21 | 2023-03-14 | [24000](https://github.com/airbytehq/airbyte/pull/24000) | Removed check method call on read. | +| 0.1.20 | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect | +| 0.1.19 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | -| 0.1.18 | 2022-09-01 | [16394](https://github.com/airbytehq/airbyte/pull/16394) | Added custom jdbc properties field | -| 0.1.17 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | -| 0.1.16 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | -| 0.1.13 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | -| 0.1.12 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | -| 0.1.11 | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | -| 0.1.10 | 2022-02-24 | [10235](https://github.com/airbytehq/airbyte/pull/10235) | Fix Replication Failure due Multiple portal opens | -| 0.1.9 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | -| 0.1.8 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | -| 0.1.7 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | -| 0.1.6 | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved discovering tables in case if user does not have permissions to any table | -| 0.1.5 | 2021-12-24 | [9004](https://github.com/airbytehq/airbyte/pull/9004) | User can see only permmited tables during discovery | -| 0.1.4 | 2021-12-24 | [8958](https://github.com/airbytehq/airbyte/pull/8958) | Add support for JdbcType.ARRAY | -| 0.1.3 | 2021-10-10 | [7819](https://github.com/airbytehq/airbyte/pull/7819) | Fixed Datatype errors during Cockroach DB parsing | -| 0.1.2 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator | +| 0.1.18 | 2022-09-01 | [16394](https://github.com/airbytehq/airbyte/pull/16394) | Added custom jdbc properties field | +| 0.1.17 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | +| 0.1.16 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | +| 0.1.13 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | +| 0.1.12 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | +| 0.1.11 | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | +| 0.1.10 | 2022-02-24 | [10235](https://github.com/airbytehq/airbyte/pull/10235) | Fix Replication Failure due Multiple portal opens | +| 0.1.9 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | +| 0.1.8 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | +| 0.1.7 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | +| 0.1.6 | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved discovering tables in case if user does not have permissions to any table | +| 0.1.5 | 2021-12-24 | [9004](https://github.com/airbytehq/airbyte/pull/9004) | User can see only permmited tables during discovery | +| 0.1.4 | 2021-12-24 | [8958](https://github.com/airbytehq/airbyte/pull/8958) | Add support for JdbcType.ARRAY | +| 0.1.3 | 2021-10-10 | [7819](https://github.com/airbytehq/airbyte/pull/7819) | Fixed Datatype errors during Cockroach DB parsing | +| 0.1.2 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator | diff --git a/docs/integrations/sources/convex.md b/docs/integrations/sources/convex.md index c0dd127574aa..d643940939a0 100644 --- a/docs/integrations/sources/convex.md +++ b/docs/integrations/sources/convex.md @@ -70,8 +70,10 @@ In the Data tab, you should see the tables and a sample of the data that will be ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :-------------------- | -| 0.2.0 | 2023-06-21 | [27226](https://github.com/airbytehq/airbyte/pull/27226) | 🐛 Convex source fix skipped records | -| 0.1.1 | 2023-03-06 | [23797](https://github.com/airbytehq/airbyte/pull/23797) | 🐛 Convex source connector error messages | -| 0.1.0 | 2022-10-24 | [18403](https://github.com/airbytehq/airbyte/pull/18403) | 🎉 New Source: Convex | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------- | +| 0.4.0 | 2023-12-13 | [33431](https://github.com/airbytehq/airbyte/pull/33431) | 🐛 Convex source fix bug where full_refresh stops after one page | +| 0.3.0 | 2023-09-28 | [30853](https://github.com/airbytehq/airbyte/pull/30853) | 🐛 Convex source switch to clean JSON format | +| 0.2.0 | 2023-06-21 | [27226](https://github.com/airbytehq/airbyte/pull/27226) | 🐛 Convex source fix skipped records | +| 0.1.1 | 2023-03-06 | [23797](https://github.com/airbytehq/airbyte/pull/23797) | 🐛 Convex source connector error messages | +| 0.1.0 | 2022-10-24 | [18403](https://github.com/airbytehq/airbyte/pull/18403) | 🎉 New Source: Convex | diff --git a/docs/integrations/sources/copper.md b/docs/integrations/sources/copper.md index 4b5827c5b117..021e14db8fa0 100644 --- a/docs/integrations/sources/copper.md +++ b/docs/integrations/sources/copper.md @@ -35,11 +35,12 @@ The Copper source connector supports the following [sync modes](https://docs.air - [People](https://developer.copper.com/people/list-people-search.html) - [Companies](https://developer.copper.com/companies/list-companies-search.html) - [Projects](https://developer.copper.com/projects/list-projects-search.html) +- [Oppurtunities](https://developer.copper.com/opportunities/list-opportunities-search.html) ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :-------------------------------------------------------- | :--------------------------------- | -| 0.2.1 | 2023-08-16 | [24824](https://github.com/airbytehq/airbyte/pull/24824) | Fix schemas and tests | -| 0.2.0 | 2023-04-17 | [24824](https://github.com/airbytehq/airbyte/pull/24824) | Add `opportunities` stream | -| 0.1.0 | 2022-11-17 | [18848](https://github.com/airbytehq/airbyte/pull/18848) | 🎉 New Source: Copper [python cdk] | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :-------------------------------------------------------- | :---------------------------------- | +| 0.3.0 | 2023-08-10 | [*****](https://github.com/airbytehq/airbyte/pull/*****) | Migrate to low code | +| 0.2.0 | 2023-04-17 | [24824](https://github.com/airbytehq/airbyte/pull/24824) | Add `opportunities` stream | +| 0.1.0 | 2022-11-17 | [18848](https://github.com/airbytehq/airbyte/pull/18848) | 🎉 New Source: Copper [python cdk] | diff --git a/docs/integrations/sources/customer-io.md b/docs/integrations/sources/customer-io.md index 3150b814ce52..768332dd99ad 100644 --- a/docs/integrations/sources/customer-io.md +++ b/docs/integrations/sources/customer-io.md @@ -47,6 +47,7 @@ Please follow the [their documentation for generating an App API Key](https://cu ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.23 | 2021-11-09 | [126](https://github.com/faros-ai/airbyte-connectors/pull/126) | Add Customer.io source | +| Version | Date | Pull Request | Subject | +| :-------- | :----------- | :------------------------------------------------------------- | :-------------------------------------------- | +| 0.2.0 | 2021-11-09 | [29385](https://github.com/airbytehq/airbyte/pull/29385) | Migrate TS CDK to Low code | +| 0.1.23 | 2021-11-09 | [126](https://github.com/faros-ai/airbyte-connectors/pull/126) | Add Customer.io source | diff --git a/docs/integrations/sources/datadog.md b/docs/integrations/sources/datadog.md index c2799e5cf1e0..be4deee99bcf 100644 --- a/docs/integrations/sources/datadog.md +++ b/docs/integrations/sources/datadog.md @@ -24,7 +24,7 @@ An API key is required as well as an API application key. See the [Datadog API a 10. Enter your `queries` - Optional. Multiple queries resulting in multiple streams. 1. Enter the `name`- Optional. Query Name. 2. Select the `data_source` from dropdown - Optional. Supported data sources - metrics, cloud_cost, logs, rum. - 3. Enter the `query`- Optional. A classic query string. Example - _"kubernetes_state.node.count{*}"_, _"@type:resource @resource.status_code:>=400 @resource.type:(xhr OR fetch)"_ + 3. Enter the `query`- Optional. A classic query string. Example - `"kubernetes_state.node.count{*}"`, `"@type:resource @resource.status_code:>=400 @resource.type:(xhr OR fetch)"` 11. Click **Set up source**. ### For Airbyte OSS: @@ -40,7 +40,7 @@ An API key is required as well as an API application key. See the [Datadog API a 10. Enter your `queries` - Optional. Multiple queries resulting in multiple streams. 1. Enter the `name`- Required. Query Name. 2. Select the `data_source` - Required. Supported data sources - metrics, cloud_cost, logs, rum. - 3. Enter the `query`- Required. A classic query string. Example - _"kubernetes_state.node.count{*}"_, _"@type:resource @resource.status_code:>=400 @resource.type:(xhr OR fetch)"_ + 3. Enter the `query`- Required. A classic query string. Example - `"kubernetes_state.node.count{*}"`, `"@type:resource @resource.status_code:>=400 @resource.type:(xhr OR fetch)"` 10. Click **Set up source**. ## Supported sync modes @@ -63,6 +63,8 @@ The Datadog source connector supports the following [sync modes](https://docs.ai * [Incidents](https://docs.datadoghq.com/api/latest/incidents/#get-a-list-of-incidents) * [Logs](https://docs.datadoghq.com/api/latest/logs/#search-logs) * [Metrics](https://docs.datadoghq.com/api/latest/metrics/#get-a-list-of-metrics) +* [Monitors](https://docs.datadoghq.com/api/latest/monitors/#get-all-monitor-details) +* [ServiceLevelObjectives](https://docs.datadoghq.com/api/latest/service-level-objectives/#get-all-slos) * [SyntheticTests](https://docs.datadoghq.com/api/latest/synthetics/#get-the-list-of-all-tests) * [Users](https://docs.datadoghq.com/api/latest/users/#list-all-users) * [Series](https://docs.datadoghq.com/api/latest/metrics/?code-lang=curl#query-timeseries-data-across-multiple-products) @@ -71,6 +73,8 @@ The Datadog source connector supports the following [sync modes](https://docs.ai | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------|:-----------------------------------------------------------------------------| +| 0.4.0 | 2023-12-04 | [30999](https://github.com/airbytehq/airbyte/pull/30999) | Add `monitors` and `service_level_objectives` Streams | +| 0.3.0 | 2023-08-27 | [29885](https://github.com/airbytehq/airbyte/pull/29885) | Migrate to low code | | 0.2.2 | 2023-07-10 | [28089](https://github.com/airbytehq/airbyte/pull/28089) | Additional stream and query details in response | | 0.2.1 | 2023-06-28 | [26534](https://github.com/airbytehq/airbyte/pull/26534) | Support multiple query streams and pulling data from different datadog sites | | 0.2.0 | 2023-06-28 | [27784](https://github.com/airbytehq/airbyte/pull/27784) | Add necessary fields to schemas | diff --git a/docs/integrations/sources/db2.md b/docs/integrations/sources/db2.md index ff3f706265e6..e742560e4a1a 100644 --- a/docs/integrations/sources/db2.md +++ b/docs/integrations/sources/db2.md @@ -58,27 +58,28 @@ You can also enter your own password for the keystore, but if you don't, the pas ## Changelog -| Version | Date | Pull Request | Subject | -|:--------| :--- | :--- | :--- | -| 0.1.20 | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212) | Fix silent exception swallowing in StreamingJdbcDatabase | -| 0.1.19 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | -| 0.1.18 | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to | -| 0.1.17 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :--- |:------------------------------------------------------------------------------------------------------------------------------------------| +| 0.2.0 | 2023-12-18 | [33485](https://github.com/airbytehq/airbyte/pull/33485) | Remove LEGACY state | +| 0.1.20 | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212) | Fix silent exception swallowing in StreamingJdbcDatabase | +| 0.1.19 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | +| 0.1.18 | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to | +| 0.1.17 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | -| 0.1.16 | 2022-09-06 | [16354](https://github.com/airbytehq/airbyte/pull/16354) | Add custom JDBC params | -| 0.1.15 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | -| 0.1.14 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | -| 0.1.13 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | -| 0.1.12 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | -| 0.1.11 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | -| 0.1.10 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | -| 0.1.9 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | -| 0.1.8 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | -| 0.1.7 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option |**** -| 0.1.6 | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved discovering tables in case if user does not have permissions to any table | -| 0.1.5 | 2022-02-01 | [9875](https://github.com/airbytehq/airbyte/pull/9875) | Discover only permitted for user tables | -| 0.1.4 | 2021-12-30 | [9187](https://github.com/airbytehq/airbyte/pull/9187) [8749](https://github.com/airbytehq/airbyte/pull/8749) | Add support of JdbcType.ARRAY to JdbcSourceOperations. | -| 0.1.3 | 2021-11-05 | [7670](https://github.com/airbytehq/airbyte/pull/7670) | Updated unique DB2 types transformation | -| 0.1.2 | 2021-10-25 | [7355](https://github.com/airbytehq/airbyte/pull/7355) | Added ssl support | -| 0.1.1 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator | -| 0.1.0 | 2021-06-22 | [4197](https://github.com/airbytehq/airbyte/pull/4197) | New Source: IBM DB2 | +| 0.1.16 | 2022-09-06 | [16354](https://github.com/airbytehq/airbyte/pull/16354) | Add custom JDBC params | +| 0.1.15 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | +| 0.1.14 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | +| 0.1.13 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | +| 0.1.12 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | +| 0.1.11 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | +| 0.1.10 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | +| 0.1.9 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | +| 0.1.8 | 2022-02-18 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Updated timestamp transformation with microseconds | +| 0.1.7 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option |**** +| 0.1.6 | 2022-02-08 | [10173](https://github.com/airbytehq/airbyte/pull/10173) | Improved discovering tables in case if user does not have permissions to any table | +| 0.1.5 | 2022-02-01 | [9875](https://github.com/airbytehq/airbyte/pull/9875) | Discover only permitted for user tables | +| 0.1.4 | 2021-12-30 | [9187](https://github.com/airbytehq/airbyte/pull/9187) [8749](https://github.com/airbytehq/airbyte/pull/8749) | Add support of JdbcType.ARRAY to JdbcSourceOperations. | +| 0.1.3 | 2021-11-05 | [7670](https://github.com/airbytehq/airbyte/pull/7670) | Updated unique DB2 types transformation | +| 0.1.2 | 2021-10-25 | [7355](https://github.com/airbytehq/airbyte/pull/7355) | Added ssl support | +| 0.1.1 | 2021-08-13 | [4699](https://github.com/airbytehq/airbyte/pull/4699) | Added json config validator | +| 0.1.0 | 2021-06-22 | [4197](https://github.com/airbytehq/airbyte/pull/4197) | New Source: IBM DB2 | diff --git a/docs/integrations/sources/delighted.md b/docs/integrations/sources/delighted.md index 5491a650b605..02e6c9f7b366 100644 --- a/docs/integrations/sources/delighted.md +++ b/docs/integrations/sources/delighted.md @@ -50,9 +50,10 @@ This source is capable of syncing the following core streams: ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------- | -| 0.2.2 | 2023-03-09 | [23909](https://github.com/airbytehq/airbyte/pull/23909) | Updated the input config pattern to accept both `RFC3339` and `datetime string` formats in UI | -| 0.2.1 | 2023-02-14 | [23009](https://github.com/airbytehq/airbyte/pull/23009) | Specified date formatting in specification | +|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------| +| 0.2.3 | 2023-09-08 | [27946](https://github.com/airbytehq/airbyte/pull/27946) | Changed `Date Since` input field title to `Replication Start Date` | +| 0.2.2 | 2023-03-09 | [23909](https://github.com/airbytehq/airbyte/pull/23909) | Updated the input config pattern to accept both `RFC3339` and `datetime string` formats in UI | +| 0.2.1 | 2023-02-14 | [23009](https://github.com/airbytehq/airbyte/pull/23009) |Specified date formatting in specification | | 0.2.0 | 2022-11-22 | [19822](https://github.com/airbytehq/airbyte/pull/19822) | Migrate to Low code + certify to Beta | | 0.1.4 | 2022-06-10 | [13439](https://github.com/airbytehq/airbyte/pull/13439) | Change since parameter input to iso date | | 0.1.3 | 2022-01-31 | [9550](https://github.com/airbytehq/airbyte/pull/9550) | Output only records in which cursor field is greater than the value in state for incremental streams | diff --git a/docs/integrations/sources/dixa.md b/docs/integrations/sources/dixa.md index ced877c4a6b3..23e5f2cbc120 100644 --- a/docs/integrations/sources/dixa.md +++ b/docs/integrations/sources/dixa.md @@ -51,6 +51,7 @@ When using the connector, keep in mind that increasing the `batch_size` paramete | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------- | +| 0.3.0 | 2023-10-17 | [30994](https://github.com/airbytehq/airbyte/pull/30994) | Migrate to Low-code Framework | | 0.2.0 | 2023-06-08 | [25103](https://github.com/airbytehq/airbyte/pull/25103) | Add fields to `conversation_export` stream | | 0.1.3 | 2022-07-07 | [14437](https://github.com/airbytehq/airbyte/pull/14437) | 🎉 Source Dixa: bump version 0.1.3 | | 0.1.2 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | diff --git a/docs/integrations/sources/dv-360.md b/docs/integrations/sources/dv-360.md index 9e4341f1d847..b3c095f4691c 100644 --- a/docs/integrations/sources/dv-360.md +++ b/docs/integrations/sources/dv-360.md @@ -36,7 +36,7 @@ Available filters and metrics are provided in this [page](https://developers.goo 3. Fill out a start date, and optionally, an end date and filters (check the [Queries documentation](https://developers.google.com/bid-manager/v1.1/queries)) . 4. You're done. -## Getting Started \(Airbyte Open-Source\) +## Getting Started \(Airbyte Open Source\) #### Requirements diff --git a/docs/integrations/sources/dynamodb.md b/docs/integrations/sources/dynamodb.md index 9ffddd23ac7a..e27494295b2c 100644 --- a/docs/integrations/sources/dynamodb.md +++ b/docs/integrations/sources/dynamodb.md @@ -1,6 +1,7 @@ # Dynamodb -The Dynamodb source allows you to sync data from Dynamodb. The source supports Full Refresh and Incremental sync strategies. +The Dynamodb source allows you to sync data from Dynamodb. The source supports Full Refresh and +Incremental sync strategies. ## Resulting schema @@ -8,27 +9,29 @@ Dynamodb doesn't have table schemas. The discover phase has three steps: ### Step 1. Retrieve items -The connector scans the table with a scan limit of 1k and if the data set size is > 1MB it will initiate another -scan with the same limit until it has >= 1k items. +The connector scans the table with a scan limit of 1k and if the data set size is > 1MB it will +initiate another scan with the same limit until it has >= 1k items. ### Step 2. Combining attributes -After retrieving the items it will combine all the different top level attributes found in the retrieved items. The implementation -assumes that the same attribute present in different items has the same type and possibly nested attributes values. +After retrieving the items it will combine all the different top level attributes found in the +retrieved items. The implementation assumes that the same attribute present in different items has +the same type and possibly nested attributes values. ### Step 3. Determine property types -For each item attribute found the connector determines its type by calling AttributeValue.type(), depending on the received type it will map the -attribute to one of the supported Airbyte types in the schema. +For each item attribute found the connector determines its type by calling AttributeValue.type(), +depending on the received type it will map the attribute to one of the supported Airbyte types in +the schema. ## Features -| Feature | Supported | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental - Append Sync | Yes | -| Replicate Incremental Deletes | No | -| Namespaces | No | +| Feature | Supported | +| :---------------------------- | :-------- | +| Full Refresh Sync | Yes | +| Incremental - Append Sync | Yes | +| Replicate Incremental Deletes | No | +| Namespaces | No | ### Full Refresh sync @@ -38,12 +41,15 @@ Works as usual full refresh sync. Cursor field can't be nested, and it needs to be top level attribute in the item. -Cursor should **never** be blank. and it needs to be either a string or integer type - the incremental sync results might be unpredictable and will totally rely on Dynamodb comparison algorithm. +Cursor should **never** be blank. and it needs to be either a string or integer type - the +incremental sync results might be unpredictable and will totally rely on Dynamodb comparison +algorithm. -Only `ISO 8601` and `epoch` cursor types are supported. Cursor type is determined based on the property type present in the previously generated schema: +Only `ISO 8601` and `epoch` cursor types are supported. Cursor type is determined based on the +property type present in the previously generated schema: -* `ISO 8601` - if cursor type is string -* `epoch` - if cursor type is integer +- `ISO 8601` - if cursor type is string +- `epoch` - if cursor type is integer ## Getting started @@ -51,18 +57,20 @@ This guide describes in details how you can configure the connector to connect w ### Сonfiguration Parameters -* **_endpoint_**: aws endpoint of the dynamodb instance -* **_region_**: the region code of the dynamodb instance -* **_access_key_id_**: the access key for the IAM user with the required permissions -* **_secret_access_key_**: the secret key for the IAM user with the required permissions -* **_reserved_attribute_names_**: comma separated list of attribute names present in the replication tables which contain reserved words or special characters. https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.ExpressionAttributeNames.html +- **_endpoint_**: aws endpoint of the dynamodb instance +- **_region_**: the region code of the dynamodb instance +- **_access_key_id_**: the access key for the IAM user with the required permissions +- **_secret_access_key_**: the secret key for the IAM user with the required permissions +- **_reserved_attribute_names_**: comma separated list of attribute names present in the replication + tables which contain reserved words or special characters. + https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Expressions.ExpressionAttributeNames.html ## Changelog - -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:------------------------------------------------|:---------------------------------------------------------------------| -| 0.1.2 | 01-19-2023 | https://github.com/airbytehq/airbyte/pull/20172 | Fix reserved words in projection expression & make them configurable | -| 0.1.1 | 02-09-2023 | https://github.com/airbytehq/airbyte/pull/22682 | Fix build | -| 0.1.0 | 11-14-2022 | https://github.com/airbytehq/airbyte/pull/18750 | Initial version | - +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :-------------------------------------------------------- | :------------------------------------------------------------------- | +| 0.2.1 | 2024-01-03 | [#33924](https://github.com/airbytehq/airbyte/pull/33924) | Add new ap-southeast-3 AWS region | +| 0.2.0 | 18-12-2023 | https://github.com/airbytehq/airbyte/pull/33485 | Remove LEGACY state | +| 0.1.2 | 01-19-2023 | https://github.com/airbytehq/airbyte/pull/20172 | Fix reserved words in projection expression & make them configurable | +| 0.1.1 | 02-09-2023 | https://github.com/airbytehq/airbyte/pull/22682 | Fix build | +| 0.1.0 | 11-14-2022 | https://github.com/airbytehq/airbyte/pull/18750 | Initial version | diff --git a/docs/integrations/sources/e2e-test-cloud.md b/docs/integrations/sources/e2e-test-cloud.md index f83816364112..633e65c3e548 100644 --- a/docs/integrations/sources/e2e-test-cloud.md +++ b/docs/integrations/sources/e2e-test-cloud.md @@ -2,7 +2,7 @@ ## Overview -This is a mock source for testing the Airbyte pipeline. It can generate arbitrary data streams. It is a subset of what is in [End-to-End Testing Source](e2e-test.md) in Open-Source to avoid Airbyte Cloud users accidentally in curring a huge bill. +This is a mock source for testing the Airbyte pipeline. It can generate arbitrary data streams. It is a subset of what is in [End-to-End Testing Source](e2e-test.md) in Open Source to avoid Airbyte Cloud users accidentally in curring a huge bill. ## Mode @@ -30,5 +30,6 @@ The OSS and Cloud variants have the same version number. The Cloud variant was i | Version | Date | Pull request | Notes | |---------|------------|----------------------------------------------------------|-----------------------------------------------------| +| 2.1.5 | 2023-10-06 | [31092](https://github.com/airbytehq/airbyte/pull/31092) | Bring in changes from oss | | 2.1.4 | 2023-03-01 | [23656](https://github.com/airbytehq/airbyte/pull/23656) | Fix inheritance between e2e-test and e2e-test-cloud | | 0.1.0 | 2021-07-23 | [9720](https://github.com/airbytehq/airbyte/pull/9720) | Initial release. | diff --git a/docs/integrations/sources/e2e-test.js b/docs/integrations/sources/e2e-test.js new file mode 100644 index 000000000000..c2d26a553d32 --- /dev/null +++ b/docs/integrations/sources/e2e-test.js @@ -0,0 +1,87 @@ +import React, {useState} from 'react'; +import CodeBlock from '@theme/CodeBlock'; + +function simpleProperty(name, type) { + return [name, { type }]; +} + +function dateProperty(name, format, airbyteType) { + const definition = { + type: ['null', 'string'], + format, + }; + if (airbyteType !== null) { + definition.airbyte_type = airbyteType; + } + return [name, definition]; +} + +const allSupportedColumnTypePropertyGenerators = [ + (i) => simpleProperty(`string_${i}`, 'string'), + (i) => simpleProperty(`boolean_${i}`, 'boolean'), + (i) => dateProperty(`date_${i}`, 'date', null), + (i) => dateProperty(`timestamp_wo_tz_${i}`, 'date-time', 'timestamp_without_timezone'), + (i) => dateProperty(`timestamp_w_tz_${i}`, 'date-time', 'timestamp_with_timezone'), + (i) => dateProperty(`time_wo_tz_${i}`, 'time', 'time_without_timezone'), + (i) => dateProperty(`time_w_tz_${i}`, 'time', 'time_with_timezone'), + (i) => simpleProperty(`integer_${i}`, 'integer'), + (i) => simpleProperty(`number_${i}`, 'number'), + (i) => simpleProperty(`array_${i}`, 'array'), + (i) => simpleProperty(`object_${i}`, 'object'), +]; + +function generateWideSchema(columns) { + + const fullSchema = { type: 'object' }; + const properties = {}; + + // Special case id and updated_at column + const id = simpleProperty('id', 'integer'); + properties[id[0]] = id[1]; + properties['updated_at'] = { type: 'string', format: 'date-time', airbyte_type: 'timestamp_with_timezone' }; + + let columnCount = 2; + let propertyGeneratorIndex = 0; + + while (columnCount < columns) { + const propertyInfo = allSupportedColumnTypePropertyGenerators[propertyGeneratorIndex](columnCount); + properties[propertyInfo[0]] = propertyInfo[1]; + + propertyGeneratorIndex += 1; + if (propertyGeneratorIndex === allSupportedColumnTypePropertyGenerators.length) { + propertyGeneratorIndex = 0; + } + + columnCount += 1; + } + + fullSchema.properties = properties; + + return fullSchema; +} + + + + +export const SchemaGenerator = () => { + + const [generatedSchema, updateGeneratedSchema] = useState({ + 'schema' : JSON.stringify(generateWideSchema(10)) + }) + function updateSchema(event) { + const columns = parseInt(document.getElementById("schema-generator-column-count").value) + const schema = JSON.stringify(generateWideSchema(columns)) + updateGeneratedSchema({'schema': schema}) + } + + return ( +
      + + +

      Generated Schema:

      + + { generatedSchema['schema'] } + +
      + ) +} diff --git a/docs/integrations/sources/e2e-test.md b/docs/integrations/sources/e2e-test.md index 514de7d9253d..5342116444fa 100644 --- a/docs/integrations/sources/e2e-test.md +++ b/docs/integrations/sources/e2e-test.md @@ -1,3 +1,5 @@ +import {SchemaGenerator} from './e2e-test.js' + # End-to-End Testing Source ## Overview @@ -26,6 +28,13 @@ Here is its configuration: | | random seed | integer | no | current time millis | The seed is used in random Json object generation. Min 0. Max 1 million. | | | message interval | integer | no | 0 | The time interval between messages in millisecond. Min 0 ms. Max 60000 ms (1 minute). | + +#### Example Stream Schemas +If you need a stream for testing performance simulating a wide table, we have an example [500 column stream](https://gist.github.com/jbfbell/9b7db8fdf0de0187c7da92df2f699502) +or use the form below to generate your own with an arbitrary width, then copy+paste the resulting schema into your configuration. + + + ### Legacy Infinite Feed This is a legacy mode used in Airbyte integration tests. It has been removed since `2.0.0`. It has a simple catalog with one `data` stream that has the following schema: @@ -61,15 +70,17 @@ This mode is also excluded from the Cloud variant of this connector. The OSS and Cloud variants have the same version number. The Cloud variant was initially released at version `1.0.0`. -| Version | Date | Pull request | Notes | -|---------|------------| ----------------------------------------------------------------------------------------------------------------- |-------------------------------------------------------------------------------------------------------| -| 2.1.4 | 2023-03-01 | [23656](https://github.com/airbytehq/airbyte/pull/23656) | Add speed benchmark mode to e2e test | -| 2.1.3 | 2022-08-25 | [15591](https://github.com/airbytehq/airbyte/pull/15591) | Declare supported sync modes in catalogs | -| 2.1.1 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | -| 2.1.0 | 2021-02-12 | [\#10298](https://github.com/airbytehq/airbyte/pull/10298) | Support stream duplication to quickly create a multi-stream catalog. | -| 2.0.0 | 2021-02-01 | [\#9954](https://github.com/airbytehq/airbyte/pull/9954) | Remove legacy modes. Use more efficient Json generator. | -| 1.0.1 | 2021-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | -| 1.0.0 | 2021-01-23 | [\#9720](https://github.com/airbytehq/airbyte/pull/9720) | Add new continuous feed mode that supports arbitrary catalog specification. Initial release to cloud. | -| 0.1.2 | 2022-10-18 | [\#18100](https://github.com/airbytehq/airbyte/pull/18100) | Set supported sync mode on streams | -| 0.1.1 | 2021-12-16 | [\#8217](https://github.com/airbytehq/airbyte/pull/8217) | Fix sleep time in infinite feed mode. | +| Version | Date | Pull request | Notes | +|---------|------------| ------------------------------------------------------------------ |-------------------------------------------------------------------------------------------------------| +| 2.2.0 | 2023-12-18 | [33485](https://github.com/airbytehq/airbyte/pull/33485) | Remove LEGACY state | +| 2.1.5 | 2023-10-04 | [31092](https://github.com/airbytehq/airbyte/pull/31092) | Bump jsonschemafriend dependency version to fix bug | +| 2.1.4 | 2023-03-01 | [23656](https://github.com/airbytehq/airbyte/pull/23656) | Add speed benchmark mode to e2e test | +| 2.1.3 | 2022-08-25 | [15591](https://github.com/airbytehq/airbyte/pull/15591) | Declare supported sync modes in catalogs | +| 2.1.1 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | +| 2.1.0 | 2021-02-12 | [\#10298](https://github.com/airbytehq/airbyte/pull/10298) | Support stream duplication to quickly create a multi-stream catalog. | +| 2.0.0 | 2021-02-01 | [\#9954](https://github.com/airbytehq/airbyte/pull/9954) | Remove legacy modes. Use more efficient Json generator. | +| 1.0.1 | 2021-01-29 | [\#9745](https://github.com/airbytehq/airbyte/pull/9745) | Integrate with Sentry. | +| 1.0.0 | 2021-01-23 | [\#9720](https://github.com/airbytehq/airbyte/pull/9720) | Add new continuous feed mode that supports arbitrary catalog specification. Initial release to cloud. | +| 0.1.2 | 2022-10-18 | [\#18100](https://github.com/airbytehq/airbyte/pull/18100) | Set supported sync mode on streams | +| 0.1.1 | 2021-12-16 | [\#8217](https://github.com/airbytehq/airbyte/pull/8217) | Fix sleep time in infinite feed mode. | | 0.1.0 | 2021-07-23 | [\#3290](https://github.com/airbytehq/airbyte/pull/3290) [\#4939](https://github.com/airbytehq/airbyte/pull/4939) | Initial release. | diff --git a/docs/integrations/sources/everhour.md b/docs/integrations/sources/everhour.md index 3475806ec1f0..d8a99925fc5e 100644 --- a/docs/integrations/sources/everhour.md +++ b/docs/integrations/sources/everhour.md @@ -1,6 +1,6 @@ # Everhour -This page contains the setup guide and reference information for the Everhour source connector. +This page contains the setup guide and reference information for the [Everhour](https://everhour.com/) source connector. ## Prerequisites @@ -25,5 +25,4 @@ This project supports the following streams: | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------| - | 0.1.0 | 2023-02-28 | [23593](https://github.com/airbytehq/airbyte/pull/23593) | Initial Release | diff --git a/docs/integrations/sources/exchange-rates.inapp.md b/docs/integrations/sources/exchange-rates.inapp.md deleted file mode 100644 index a883ff6a7054..000000000000 --- a/docs/integrations/sources/exchange-rates.inapp.md +++ /dev/null @@ -1,25 +0,0 @@ -## Prerequisites - -- API Access Key - -In order to get a free `API Access Key` please go to [this](https://manage.exchangeratesapi.io/signup/free) page and enter the required information. After registration and login, you will see your `API Access Key`. You can also locate it [here](https://manage.exchangeratesapi.io/dashboard). - -If you have a `free` subscription plan, you will have two limitations to the plan: - -1. Limit of 1,000 API calls per month -2. You won't be able to specify the `base` parameter, meaning that you will be only be allowed to use the default base value which is EUR. - -## Setup guide -1. Enter a **Name** for your source. -2. Enter your **API key** as the `access_key` from the prerequisites. -3. Enter the **Start Date** in YYYY-MM-DD format. The data added on and after this date will be replicated. -4. (Optional) Enter a **base** currency. For those on the free plan, `EUR` is the only option available. If none are specified, `EUR` will be used. -5. Click **Set up source**. - -### Exchange Rates data output -- The sync will include one stream: `exchange_rates` -- Each record in the stream contains many fields: - - The date of the record - - One field for every supported [currency](https://www.ecb.europa.eu/stats/policy_and_exchange_rates/euro_reference_exchange_rates/html/index.en.html) which contain the value of that currency on that date. - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Exchange Rates](https://docs.airbyte.com/integrations/sources/exchange-rates/). diff --git a/docs/integrations/sources/exchange-rates.md b/docs/integrations/sources/exchange-rates.md index 3d5f8d623131..635cea691220 100644 --- a/docs/integrations/sources/exchange-rates.md +++ b/docs/integrations/sources/exchange-rates.md @@ -1,25 +1,45 @@ # Exchange Rates API + + +This page contains the setup guide and reference information for the [Exchange Rates API](https://exchangeratesapi.io/) source connector. + + + ## Overview -The exchange rates integration is a toy integration to demonstrate how Airbyte works with a very simple source. +The Exchange Rates API integration is a toy integration to demonstrate how Airbyte works with a very simple source. -It pulls all its data from [https://exchangeratesapi.io](https://exchangeratesapi.io) +## Prerequisites -#### Output schema +- Exchange Rates API account +- API Access Key -It contains one stream: `exchange_rates` +## Setup Guide -Each record in the stream contains many fields: +### Step 1: Set up Exchange Rates API -- The date of the record -- One field for every supported [currency](https://www.ecb.europa.eu/stats/policy_and_exchange_rates/euro_reference_exchange_rates/html/index.en.html) which contain the value of that currency on that date. +1. Create an account with [Exchange Rates API](https://manage.exchangeratesapi.io/signup/). +2. Navigate to the [Exchange Rates API Dashboard](https://manage.exchangeratesapi.io/dashboard) to find your `API Access Key`. + +:::note +If you have a `free` subscription plan, you will have two limitations to the plan: + +1. Limit of 1,000 API calls per month +2. You won't be able to specify the `base` parameter, meaning that you will be only be allowed to use the default base value which is `EUR`. +::: -#### Data type mapping +### Step 2: Set up the Exchange Rates connector in Airbyte -Currencies are `number` and the date is a `string`. +1. Enter a **Name** for your source. +2. Enter your **API key** as the `access_key` from the prerequisites. +3. Enter the **Start Date** in YYYY-MM-DD format. The data added on and after this date will be replicated. +4. (Optional) Enter a **base** currency. For those on the free plan, `EUR` is the only option available. If none are specified, `EUR` will be used. +5. Click **Set up source**. -#### Features + + +## Supported sync modes | Feature | Supported? | | :------------------------ | :--------- | @@ -27,20 +47,41 @@ Currencies are `number` and the date is a `string`. | Incremental - Append Sync | Yes | | Namespaces | No | -### Getting started +## Supported streams + +It contains one stream: `exchange_rates` -### Requirements +Each record in the stream contains many fields: -- API Access Key +- The date of the record. +- One field for every supported [currency](https://www.ecb.europa.eu/stats/policy_and_exchange_rates/euro_reference_exchange_rates/html/index.en.html) which contain the value of that currency on that date. + +## Data type map + +| Field | Airbyte Type | +| :------------------------ | :----------- | +| Currency | `number` | +| Date | `string` | -### Setup guide +## Limitations & Troubleshooting -In order to get an `API Access Key` please go to [this](https://manage.exchangeratesapi.io/signup/free) page and enter needed info. After registration and login you will see your `API Access Key`, also you may find it [here](https://manage.exchangeratesapi.io/dashboard). +
      + +Expand to see details about Exchange Rates API connector limitations and troubleshooting. + -If you have `free` subscription plan \(you may check it [here](https://manage.exchangeratesapi.io/plan)\) this means that you will have 2 limitations: +### Connector limitations -1. 1000 API calls per month. -2. You won't be able to specify the `base` parameter, meaning that you will be dealing only with default base value which is EUR. +#### Rate limiting + +The Exchange Rates API has rate limits that vary per pricing plan. The free plan is subject to rate limiting of 1,000 requests per month. Review the [Exchange Rates API Pricing Plans](https://exchangeratesapi.io/#pricing_plan) for more information. + +### Troubleshooting + +* With the free plan, you won't be able to specify the `base` parameter, meaning that you will be only be allowed to use the default base value which is `EUR`. +* Check out common troubleshooting issues for the Exchange Rates API source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +
      ## Changelog @@ -58,3 +99,5 @@ If you have `free` subscription plan \(you may check it [here](https://manage.ex | 0.2.2 | 2021-05-28 | [3677](https://github.com/airbytehq/airbyte/pull/3677) | Adding clearer error message when a currency isn't supported. access_key field in spec.json was marked as sensitive | | 0.2.0 | 2021-05-26 | [3566](https://github.com/airbytehq/airbyte/pull/3566) | Move from `api.ratesapi.io/` to `api.exchangeratesapi.io/`. Add required field `access_key` to `config.json`. | | 0.1.0 | 2021-04-19 | [2942](https://github.com/airbytehq/airbyte/pull/2942) | Implement Exchange API using the CDK | + +
      \ No newline at end of file diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md index 2f8055158b75..75eff14ff67c 100644 --- a/docs/integrations/sources/facebook-marketing.md +++ b/docs/integrations/sources/facebook-marketing.md @@ -5,22 +5,41 @@ This page guides you through the process of setting up the Facebook Marketing so ## Prerequisites - A [Facebook Ad Account ID](https://www.facebook.com/business/help/1492627900875762) + + + +#### For Airbyte Cloud + +If you are not the owner/admin of the Ad account, you must be granted [permissions to access the Ad account](https://www.facebook.com/business/help/155909647811305?id=829106167281625) by an admin. + + + -- (For Airbyte Open Source) A [Facebook app](https://developers.facebook.com/apps/) with the Marketing API enabled + +#### For Airbyte Open Source + +A [Facebook app](https://developers.facebook.com/apps/) with the Marketing API enabled and the following permissions: + + - [ads_management](https://developers.facebook.com/docs/permissions#a) + - [ads_read](https://developers.facebook.com/docs/permissions#a) + - [business_management](https://developers.facebook.com/docs/permissions#b) + - [read_insights](https://developers.facebook.com/docs/permissions#r) + ## Setup guide -### (For Airbyte Open Source) Generate an access token and request a rate limit increase: +### (For Airbyte Open Source) Generate an access token and request a rate limit increase To set up Facebook Marketing as a source in Airbyte Open Source, you will first need to create a Facebook app and generate a Marketing API access token. You will then need to request a rate limit increase from Facebook. The following steps will guide you through this process: -1. Navigate to [Meta for Developers](https://developers.facebook.com/apps/) and follow the steps provided in the [Facebook documentation](https://developers.facebook.com/docs/development/create-an-app/) to create a Facebook app. Set the app type to **Business** when prompted. -2. From your App’s dashboard, [set up the Marketing API](https://developers.facebook.com/docs/marketing-apis/get-started). -3. Generate a Marketing API access token: From your App’s Dashboard, click **Marketing API** --> **Tools**. Select all the available token permissions (`ads_management`, `ads_read`, `read_insights`, `business_management`) and click **Get token**. Copy the generated token for later use. -4. Request a rate limit increase: Facebook [heavily throttles](https://developers.facebook.com/docs/marketing-api/overview/authorization#limits) API tokens generated from Facebook apps with the default Standard Access tier, making it infeasible to use the token for syncs with Airbyte. You'll need to request an upgrade to Advanced Access for your app on the following permissions: +1. Navigate to [Meta for Developers](https://developers.facebook.com/apps/) and follow the steps provided in the [Facebook documentation](https://developers.facebook.com/docs/development/create-an-app/) to create a Facebook app. +2. While creating the app, when you are prompted for "What do you want your app to do?", select **Other**. You will also need to set the app type to **Business** when prompted. +3. From your App’s dashboard, [set up the Marketing API](https://developers.facebook.com/docs/marketing-apis/get-started). +4. Generate a Marketing API access token: From your App’s Dashboard, click **Marketing API** --> **Tools**. Select all the available token permissions (`ads_management`, `ads_read`, `read_insights`, `business_management`) and click **Get token**. Copy the generated token for later use. +5. Request a rate limit increase: Facebook [heavily throttles](https://developers.facebook.com/docs/marketing-api/overview/authorization#limits) API tokens generated from Facebook apps with the default Standard Access tier, making it infeasible to use the token for syncs with Airbyte. You'll need to request an upgrade to Advanced Access for your app on the following permissions: - Ads Management Standard Access - ads_read @@ -28,9 +47,13 @@ To set up Facebook Marketing as a source in Airbyte Open Source, you will first See the Facebook [documentation on Authorization](https://developers.facebook.com/docs/marketing-api/overview/authorization/#access-levels) to request Advanced Access to the relevant permissions. +:::tip +You can use the [Access Token Tool](https://developers.facebook.com/tools/accesstoken) at any time to view your existing access tokens, including their assigned permissions and lifecycles. +::: + -### Set up Facebook Marketing as a source in Airbyte: +### Set up Facebook Marketing as a source in Airbyte 1. [Log in to your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account, or navigate to your Airbyte Open Source dashboard. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. @@ -45,10 +68,10 @@ To set up Facebook Marketing as a source in Airbyte Open Source, you will first **For Airbyte Open Source**: In the **Access Token** field, enter the access token you generated with your Facebook app. -#### Facebook Marketing Source Settings: +#### Facebook Marketing Source Settings -1. For **Account ID**, enter the [Facebook Ad Account ID Number](https://www.facebook.com/business/help/1492627900875762) to use when pulling data from the Facebook Marketing API. To find this ID, open your Meta Ads Manager. The Ad Account ID number is in the **Account** dropdown menu or in your browser's address bar. Refer to the [Facebook docs](https://www.facebook.com/business/help/1492627900875762) for more information. -2. For **Start Date**, use the provided datepicker, or enter the date programmatically in the `YYYY-MM-DDTHH:mm:ssZ` format. The data added on and after this date will be replicated. If this field is left blank, Airbyte will replicate all data. +1. For **Account ID(s)**, enter one or multiple comma-separated [Facebook Ad Account ID Numbers](https://www.facebook.com/business/help/1492627900875762) to use when pulling data from the Facebook Marketing API. To find this ID, open your Meta Ads Manager. The Ad Account ID number is in the **Account** dropdown menu or in your browser's address bar. Refer to the [Facebook docs](https://www.facebook.com/business/help/1492627900875762) for more information. +2. (Optional) For **Start Date**, use the provided datepicker, or enter the date programmatically in the `YYYY-MM-DDTHH:mm:ssZ` format. If not set then all data will be replicated for usual streams and only last 2 years for insight streams. :::warning Insight tables are only able to pull data from the last 37 months. If you are syncing insight tables and your start date is older than 37 months, your sync will fail. @@ -92,8 +115,10 @@ To set up Facebook Marketing as a source in Airbyte Open Source, you will first 7. (Optional) For **Page Size of Requests**, you can specify the number of records per page for paginated responses. Most users do not need to set this field unless specific issues arise or there are unique use cases that require tuning the connector's settings. The default value is set to retrieve 100 records per page. 8. (Optional) For **Insights Window Lookback**, you may set a window in days to revisit data during syncing to capture updated conversion data from the API. Facebook allows for attribution windows of up to 28 days, during which time a conversion can be attributed to an ad. If you have set a custom attribution window in your Facebook account, please set the same value here. Otherwise, you may leave it at the default value of 28. For more information on action attributions, please refer to [the Meta Help Center](https://www.facebook.com/business/help/458681590974355?id=768381033531365). -9. (Optional) You can set a **Maximum size of Batched Requests** for the connector. This is the maximum number of records that will be sent in a single request to the Facebook Marketing API. Most users do not need to configure this field, unless specific issues arise of there are unique use cases that require tuning the connector's settings. The maximum number of requests per batch allowed by the API is 50. More information on this topic can be found in the [Facebook documentation](https://developers.facebook.com/docs/graph-api/batch-requests). -10. Click **Set up source** and wait for the tests to complete. +8. (Optional) For **Insights Job Timeout**, you may set a custom value in range from 10 to 60. It establishes the maximum amount of time (in minutes) of waiting for the report job to complete. +9. Click **Set up source** and wait for the tests to complete. + + ## Supported sync modes @@ -178,6 +203,21 @@ The Facebook Marketing connector uses the `lookback_window` parameter to repeate | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.3.0 | 2024-01-09 | [33538](https://github.com/airbytehq/airbyte/pull/33538) | Updated the `Ad Account ID(s)` property to support multiple IDs | +| 1.2.3 | 2024-01-04 | [33934](https://github.com/airbytehq/airbyte/pull/33828) | Make ready for airbyte-lib | +| 1.2.2 | 2024-01-02 | [33828](https://github.com/airbytehq/airbyte/pull/33828) | Add insights job timeout to be an option, so a user can specify their own value | +| 1.2.1 | 2023-11-22 | [32731](https://github.com/airbytehq/airbyte/pull/32731) | Removed validation that blocked personal ad accounts during `check` | +| 1.2.0 | 2023-10-31 | [31999](https://github.com/airbytehq/airbyte/pull/31999) | Extend the `AdCreatives` stream schema | +| 1.1.17 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 1.1.16 | 2023-10-11 | [31284](https://github.com/airbytehq/airbyte/pull/31284) | Fix error occurring when trying to access the `funding_source_details` field of the `AdAccount` stream | +| 1.1.15 | 2023-10-06 | [31132](https://github.com/airbytehq/airbyte/pull/31132) | Fix permission error for `AdAccount` stream | +| 1.1.14 | 2023-09-26 | [30758](https://github.com/airbytehq/airbyte/pull/30758) | Exception should not be raises if a stream is not found | +| 1.1.13 | 2023-09-22 | [30706](https://github.com/airbytehq/airbyte/pull/30706) | Performance testing - include socat binary in docker image | +| 1.1.12 | 2023-09-22 | [30655](https://github.com/airbytehq/airbyte/pull/30655) | Updated doc; improved schema for custom insight streams; updated SAT or custom insight streams; removed obsolete optional max_batch_size option from spec | +| 1.1.11 | 2023-09-21 | [30650](https://github.com/airbytehq/airbyte/pull/30650) | Fix None issue since start_date is optional | +| 1.1.10 | 2023-09-15 | [30485](https://github.com/airbytehq/airbyte/pull/30485) | added 'status' and 'configured_status' fields for campaigns stream schema | +| 1.1.9 | 2023-08-31 | [29994](https://github.com/airbytehq/airbyte/pull/29994) | Removed batch processing, updated description in specs, added user-friendly error message, removed start_date from required attributes | +| 1.1.8 | 2023-09-04 | [29666](https://github.com/airbytehq/airbyte/pull/29666) | Adding custom field `boosted_object_id` to a streams schema in `campaigns` catalog `CustomAudiences` | | 1.1.7 | 2023-08-21 | [29674](https://github.com/airbytehq/airbyte/pull/29674) | Exclude `rule` from stream `CustomAudiences` | | 1.1.6 | 2023-08-18 | [29642](https://github.com/airbytehq/airbyte/pull/29642) | Stop batch requests if only 1 left in a batch | | 1.1.5 | 2023-08-18 | [29610](https://github.com/airbytehq/airbyte/pull/29610) | Automatically reduce batch size | @@ -290,3 +330,5 @@ The Facebook Marketing connector uses the `lookback_window` parameter to repeate | 0.1.3 | 2021-02-15 | [1990](https://github.com/airbytehq/airbyte/pull/1990) | Support Insights stream via async queries | | 0.1.2 | 2021-01-22 | [1699](https://github.com/airbytehq/airbyte/pull/1699) | Add incremental support | | 0.1.1 | 2021-01-15 | [1552](https://github.com/airbytehq/airbyte/pull/1552) | Release Native Facebook Marketing Connector | + + diff --git a/docs/integrations/sources/facebook-pages.md b/docs/integrations/sources/facebook-pages.md index 4f60c987df4d..7055a1f798a9 100644 --- a/docs/integrations/sources/facebook-pages.md +++ b/docs/integrations/sources/facebook-pages.md @@ -1,5 +1,9 @@ # Facebook Pages +:::danger +The Facebook Pages API utilized by this connector has been deprecated. You will not be able to make a successful connection. If you would like to make a community contribution or track API upgrade status, visit: https://github.com/airbytehq/airbyte/issues/25515. +::: + This page contains the setup guide and reference information for the Facebook Pages source connector. ## Prerequisites diff --git a/docs/integrations/sources/faker.md b/docs/integrations/sources/faker.md index 8e2feb4f99b1..39a58897e124 100644 --- a/docs/integrations/sources/faker.md +++ b/docs/integrations/sources/faker.md @@ -2,11 +2,13 @@ ## Sync overview -The Sample Data (Faker) source generates sample data using the python [`mimesis`](https://mimesis.name/en/master/) package. +The Sample Data (Faker) source generates sample data using the python +[`mimesis`](https://mimesis.name/en/master/) package. ### Output schema -This source will generate an "e-commerce-like" dataset with users, products, and purchases. Here's what is produced at a Postgres destination connected to this source: +This source will generate an "e-commerce-like" dataset with users, products, and purchases. Here's +what is produced at a Postgres destination connected to this source: ```sql CREATE TABLE "public"."users" ( @@ -84,9 +86,12 @@ CREATE TABLE "public"."purchases" ( | Incremental Sync | Yes | | | Namespaces | No | | -Of note, if you choose `Incremental Sync`, state will be maintained between syncs, and once you hit `count` records, no new records will be added. +Of note, if you choose `Incremental Sync`, state will be maintained between syncs, and once you hit +`count` records, no new records will be added. -You can choose a specific `seed` (integer) as an option for this connector which will guarantee that the same fake records are generated each time. Otherwise, random data will be created on each subsequent sync. +You can choose a specific `seed` (integer) as an option for this connector which will guarantee that +the same fake records are generated each time. Otherwise, random data will be created on each +subsequent sync. ### Requirements @@ -95,7 +100,9 @@ None! ## Changelog | Version | Date | Pull Request | Subject | -|:--------|:-----------|:----------------------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| :------ | :--------- | :-------------------------------------------------------------------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------- | +| 5.0.2 | 2024-01-17 | [34344](https://github.com/airbytehq/airbyte/pull/34344) | Ensure unique state messages | +| 5.0.1 | 2023-01-08 | [34033](https://github.com/airbytehq/airbyte/pull/34033) | Add standard entrypoints for usage with AirbyteLib | | 5.0.0 | 2023-08-08 | [29213](https://github.com/airbytehq/airbyte/pull/29213) | Change all `*id` fields and `products.year` to be integer | | 4.0.0 | 2023-07-19 | [28485](https://github.com/airbytehq/airbyte/pull/28485) | Bump to test publication | | 3.0.2 | 2023-07-07 | [27807](https://github.com/airbytehq/airbyte/pull/28060) | Bump to test publication | diff --git a/docs/integrations/sources/file.md b/docs/integrations/sources/file.md index 99ee0add09bd..8afdbf2c912f 100644 --- a/docs/integrations/sources/file.md +++ b/docs/integrations/sources/file.md @@ -185,22 +185,22 @@ Here are a list of examples of possible file inputs: | Dataset Name | Storage | URL | Reader Impl | Service Account | Description | | ----------------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------ | -------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | epidemiology | HTTPS | [https://storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv](https://storage.googleapis.com/covid19-open-data/v2/latest/epidemiology.csv) | | | [COVID-19 Public dataset](https://console.cloud.google.com/marketplace/product/bigquery-public-datasets/covid19-public-data-program?filter=solution-type:dataset&id=7d6cc408-53c8-4485-a187-b8cb9a5c0b56) on BigQuery | -| hr_and_financials | GCS | gs://airbyte-vault/financial.csv | smart_open or gcfs | {"type": "service_account", "private_key_id": "XXXXXXXX", ...} | data from a private bucket, a service account is necessary | +| hr_and_financials | GCS | gs://airbyte-vault/financial.csv | smart_open or gcfs | `{"type": "service_account", "private_key_id": "XXXXXXXX", ...}` | data from a private bucket, a service account is necessary | | landsat_index | GCS | gcp-public-data-landsat/index.csv.gz | smart_open | | Using smart_open, we don't need to specify the compression (note the gs:// is optional too, same for other providers) | Examples with reader options: | Dataset Name | Storage | URL | Reader Impl | Reader Options | Description | | ------------- | ------- | ----------------------------------------------- | ----------- | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | -| landsat_index | GCS | gs://gcp-public-data-landsat/index.csv.gz | GCFS | {"compression": "gzip"} | Additional reader options to specify a compression option to `read_csv` | -| GDELT | S3 | s3://gdelt-open-data/events/20190914.export.csv | | {"sep": "\t", "header": null} | Here is TSV data separated by tabs without header row from [AWS Open Data](https://registry.opendata.aws/gdelt/) | -| server_logs | local | /local/logs.log | | {"sep": ";"} | After making sure a local text file exists at `/tmp/airbyte_local/logs.log` with logs file from some server that are delimited by ';' delimiters | +| landsat_index | GCS | gs://gcp-public-data-landsat/index.csv.gz | GCFS | `{"compression": "gzip"}` | Additional reader options to specify a compression option to `read_csv` | +| GDELT | S3 | s3://gdelt-open-data/events/20190914.export.csv | | `{"sep": "\t", "header": null}` | Here is TSV data separated by tabs without header row from [AWS Open Data](https://registry.opendata.aws/gdelt/) | +| server_logs | local | /local/logs.log | | `{"sep": ";"}` | After making sure a local text file exists at `/tmp/airbyte_local/logs.log` with logs file from some server that are delimited by ';' delimiters | Example for SFTP: | Dataset Name | Storage | User | Password | Host | URL | Reader Options | Description | | ------------ | ------- | ---- | -------- | --------------- | ----------------------- | ----------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| Test Rebext | SFTP | demo | password | test.rebext.net | /pub/example/readme.txt | {"sep": "\r\n", "header": null, "names": \["text"], "engine": "python"} | We use `python` engine for `read_csv` in order to handle delimiter of more than 1 character while providing our own column names. | +| Test Rebext | SFTP | demo | password | test.rebext.net | /pub/example/readme.txt | `{"sep": "\r\n", "header": null, "names": \["text"], "engine": "python"}` | We use `python` engine for `read_csv` in order to handle delimiter of more than 1 character while providing our own column names. | Please see (or add) more at `airbyte-integrations/connectors/source-file/integration_tests/integration_source_test.py` for further usages examples. @@ -214,59 +214,63 @@ In order to read large files from a remote location, this connector uses the [sm ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:-----------------------------------------------------------|:--------------------------------------------------------------------------------------------------------| -| 0.3.11 | 2023-06-08 | [27157](https://github.com/airbytehq/airbyte/pull/27157) | Force smart open log level to ERROR | -| 0.3.10 | 2023-06-07 | [27107](https://github.com/airbytehq/airbyte/pull/27107) | Make source-file testable in our new airbyte-ci pipelines | -| 0.3.9 | 2023-05-18 | [26275](https://github.com/airbytehq/airbyte/pull/26275) | Add ParserError handling | -| 0.3.8 | 2023-05-17 | [26210](https://github.com/airbytehq/airbyte/pull/26210) | Bugfix for https://github.com/airbytehq/airbyte/pull/26115 | -| 0.3.7 | 2023-05-16 | [26131](https://github.com/airbytehq/airbyte/pull/26131) | Re-release source-file to be in sync with source-file-secure | -| 0.3.6 | 2023-05-16 | [26115](https://github.com/airbytehq/airbyte/pull/26115) | Add retry on SSHException('Error reading SSH protocol banner') | -| 0.3.5 | 2023-05-16 | [26117](https://github.com/airbytehq/airbyte/pull/26117) | Check if reader options is a valid JSON object | -| 0.3.4 | 2023-05-10 | [25965](https://github.com/airbytehq/airbyte/pull/25965) | fix Pandas date-time parsing to airbyte type | -| 0.3.3 | 2023-05-04 | [25819](https://github.com/airbytehq/airbyte/pull/25819) | GCP service_account_json is a secret | -| 0.3.2 | 2023-05-01 | [25641](https://github.com/airbytehq/airbyte/pull/25641) | Handle network errors | -| 0.3.1 | 2023-04-27 | [25575](https://github.com/airbytehq/airbyte/pull/25575) | Fix OOM; read Excel files in chunks using `openpyxl` | -| 0.3.0 | 2023-04-24 | [25445](https://github.com/airbytehq/airbyte/pull/25445) | Add datatime format parsing support for csv files | -| 0.2.38 | 2023-04-12 | [23759](https://github.com/airbytehq/airbyte/pull/23759) | Fix column data types for numerical values | -| 0.2.37 | 2023-04-06 | [24525](https://github.com/airbytehq/airbyte/pull/24525) | Fix examples in spec | -| 0.2.36 | 2023-03-27 | [24588](https://github.com/airbytehq/airbyte/pull/24588) | Remove traceback from user messages. | -| 0.2.35 | 2023-03-03 | [24278](https://github.com/airbytehq/airbyte/pull/24278) | Read only file header when checking connectivity; read only a single chunk when discovering the schema. | -| 0.2.34 | 2023-03-03 | [23723](https://github.com/airbytehq/airbyte/pull/23723) | Update description in spec, make user-friendly error messages and docs. | -| 0.2.33 | 2023-01-04 | [21012](https://github.com/airbytehq/airbyte/pull/21012) | Fix special characters bug | -| 0.2.32 | 2022-12-21 | [20740](https://github.com/airbytehq/airbyte/pull/20740) | Source File: increase SSH timeout to 60s | -| 0.2.31 | 2022-11-17 | [19567](https://github.com/airbytehq/airbyte/pull/19567) | Source File: bump 0.2.31 | -| 0.2.30 | 2022-11-10 | [19222](https://github.com/airbytehq/airbyte/pull/19222) | Use AirbyteConnectionStatus for "check" command | -| 0.2.29 | 2022-11-08 | [18587](https://github.com/airbytehq/airbyte/pull/18587) | Fix pandas read_csv header none issue. | -| 0.2.28 | 2022-10-27 | [18428](https://github.com/airbytehq/airbyte/pull/18428) | Add retry logic for `Connection reset error - 104` | -| 0.2.27 | 2022-10-26 | [18481](https://github.com/airbytehq/airbyte/pull/18481) | Fix check for wrong format | -| 0.2.26 | 2022-10-18 | [18116](https://github.com/airbytehq/airbyte/pull/18116) | Transform Dropbox shared link | -| 0.2.25 | 2022-10-14 | [17994](https://github.com/airbytehq/airbyte/pull/17994) | Handle `UnicodeDecodeError` during discover step. | -| 0.2.24 | 2022-10-03 | [17504](https://github.com/airbytehq/airbyte/pull/17504) | Validate data for `HTTPS` while `check_connection` | -| 0.2.23 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state. | -| 0.2.22 | 2022-09-15 | [16772](https://github.com/airbytehq/airbyte/pull/16772) | Fix schema generation for JSON files containing arrays | -| 0.2.21 | 2022-08-26 | [15568](https://github.com/airbytehq/airbyte/pull/15568) | Specify `pyxlsb` library for Excel Binary Workbook files | -| 0.2.20 | 2022-08-23 | [15870](https://github.com/airbytehq/airbyte/pull/15870) | Fix CSV schema discovery | -| 0.2.19 | 2022-08-19 | [15768](https://github.com/airbytehq/airbyte/pull/15768) | Convert 'nan' to 'null' | -| 0.2.18 | 2022-08-16 | [15698](https://github.com/airbytehq/airbyte/pull/15698) | Cache binary stream to file for discover | -| 0.2.17 | 2022-08-11 | [15501](https://github.com/airbytehq/airbyte/pull/15501) | Cache binary stream to file | -| 0.2.16 | 2022-08-10 | [15293](https://github.com/airbytehq/airbyte/pull/15293) | Add support for encoding reader option | -| 0.2.15 | 2022-08-05 | [15269](https://github.com/airbytehq/airbyte/pull/15269) | Bump `smart-open` version to 6.0.0 | -| 0.2.12 | 2022-07-12 | [14535](https://github.com/airbytehq/airbyte/pull/14535) | Fix invalid schema generation for JSON files | -| 0.2.11 | 2022-07-12 | [9974](https://github.com/airbytehq/airbyte/pull/14588) | Add support to YAML format | -| 0.2.9 | 2022-02-01 | [9974](https://github.com/airbytehq/airbyte/pull/9974) | Update airbyte-cdk 0.1.47 | -| 0.2.8 | 2021-12-06 | [8524](https://github.com/airbytehq/airbyte/pull/8524) | Update connector fields title/description | -| 0.2.7 | 2021-10-28 | [7387](https://github.com/airbytehq/airbyte/pull/7387) | Migrate source to CDK structure, add SAT testing. | -| 0.2.6 | 2021-08-26 | [5613](https://github.com/airbytehq/airbyte/pull/5613) | Add support to xlsb format | -| 0.2.5 | 2021-07-26 | [4953](https://github.com/airbytehq/airbyte/pull/4953) | Allow non-default port for SFTP type | -| 0.2.4 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add AIRBYTE_ENTRYPOINT for Kubernetes support | -| 0.2.3 | 2021-06-01 | [3771](https://github.com/airbytehq/airbyte/pull/3771) | Add Azure Storage Blob Files option | -| 0.2.2 | 2021-04-16 | [2883](https://github.com/airbytehq/airbyte/pull/2883) | Fix CSV discovery memory consumption | -| 0.2.1 | 2021-04-03 | [2726](https://github.com/airbytehq/airbyte/pull/2726) | Fix base connector versioning | -| 0.2.0 | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238) | Protocol allows future/unknown properties | -| 0.1.10 | 2021-02-18 | [2118](https://github.com/airbytehq/airbyte/pull/2118) | Support JSONL format | -| 0.1.9 | 2021-02-02 | [1768](https://github.com/airbytehq/airbyte/pull/1768) | Add test cases for all formats | -| 0.1.8 | 2021-01-27 | [1738](https://github.com/airbytehq/airbyte/pull/1738) | Adopt connector best practices | -| 0.1.7 | 2020-12-16 | [1331](https://github.com/airbytehq/airbyte/pull/1331) | Refactor Python base connector | -| 0.1.6 | 2020-12-08 | [1249](https://github.com/airbytehq/airbyte/pull/1249) | Handle NaN values | -| 0.1.5 | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046) | Add connectors using an index YAML file | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------| +| 0.3.15 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Upgrade to airbyte/python-connector-base:1.0.1 | +| 0.3.14 | 2023-10-13 | [30984](https://github.com/airbytehq/airbyte/pull/30984) | Prevent local file usage on cloud | +| 0.3.13 | 2023-10-12 | [31341](https://github.com/airbytehq/airbyte/pull/31341) | Build from airbyte/python-connector-base:1.0.0 | +| 0.3.12 | 2023-09-19 | [30579](https://github.com/airbytehq/airbyte/pull/30579) | Add ParserError handling for `discovery` | +| 0.3.11 | 2023-06-08 | [27157](https://github.com/airbytehq/airbyte/pull/27157) | Force smart open log level to ERROR | +| 0.3.10 | 2023-06-07 | [27107](https://github.com/airbytehq/airbyte/pull/27107) | Make source-file testable in our new airbyte-ci pipelines | +| 0.3.9 | 2023-05-18 | [26275](https://github.com/airbytehq/airbyte/pull/26275) | Add ParserError handling | +| 0.3.8 | 2023-05-17 | [26210](https://github.com/airbytehq/airbyte/pull/26210) | Bugfix for https://github.com/airbytehq/airbyte/pull/26115 | +| 0.3.7 | 2023-05-16 | [26131](https://github.com/airbytehq/airbyte/pull/26131) | Re-release source-file to be in sync with source-file-secure | +| 0.3.6 | 2023-05-16 | [26115](https://github.com/airbytehq/airbyte/pull/26115) | Add retry on SSHException('Error reading SSH protocol banner') | +| 0.3.5 | 2023-05-16 | [26117](https://github.com/airbytehq/airbyte/pull/26117) | Check if reader options is a valid JSON object | +| 0.3.4 | 2023-05-10 | [25965](https://github.com/airbytehq/airbyte/pull/25965) | fix Pandas date-time parsing to airbyte type | +| 0.3.3 | 2023-05-04 | [25819](https://github.com/airbytehq/airbyte/pull/25819) | GCP service_account_json is a secret | +| 0.3.2 | 2023-05-01 | [25641](https://github.com/airbytehq/airbyte/pull/25641) | Handle network errors | +| 0.3.1 | 2023-04-27 | [25575](https://github.com/airbytehq/airbyte/pull/25575) | Fix OOM; read Excel files in chunks using `openpyxl` | +| 0.3.0 | 2023-04-24 | [25445](https://github.com/airbytehq/airbyte/pull/25445) | Add datatime format parsing support for csv files | +| 0.2.38 | 2023-04-12 | [23759](https://github.com/airbytehq/airbyte/pull/23759) | Fix column data types for numerical values | +| 0.2.37 | 2023-04-06 | [24525](https://github.com/airbytehq/airbyte/pull/24525) | Fix examples in spec | +| 0.2.36 | 2023-03-27 | [24588](https://github.com/airbytehq/airbyte/pull/24588) | Remove traceback from user messages. | +| 0.2.35 | 2023-03-03 | [24278](https://github.com/airbytehq/airbyte/pull/24278) | Read only file header when checking connectivity; read only a single chunk when discovering the schema. | +| 0.2.34 | 2023-03-03 | [23723](https://github.com/airbytehq/airbyte/pull/23723) | Update description in spec, make user-friendly error messages and docs. | +| 0.2.33 | 2023-01-04 | [21012](https://github.com/airbytehq/airbyte/pull/21012) | Fix special characters bug | +| 0.2.32 | 2022-12-21 | [20740](https://github.com/airbytehq/airbyte/pull/20740) | Source File: increase SSH timeout to 60s | +| 0.2.31 | 2022-11-17 | [19567](https://github.com/airbytehq/airbyte/pull/19567) | Source File: bump 0.2.31 | +| 0.2.30 | 2022-11-10 | [19222](https://github.com/airbytehq/airbyte/pull/19222) | Use AirbyteConnectionStatus for "check" command | +| 0.2.29 | 2022-11-08 | [18587](https://github.com/airbytehq/airbyte/pull/18587) | Fix pandas read_csv header none issue. | +| 0.2.28 | 2022-10-27 | [18428](https://github.com/airbytehq/airbyte/pull/18428) | Add retry logic for `Connection reset error - 104` | +| 0.2.27 | 2022-10-26 | [18481](https://github.com/airbytehq/airbyte/pull/18481) | Fix check for wrong format | +| 0.2.26 | 2022-10-18 | [18116](https://github.com/airbytehq/airbyte/pull/18116) | Transform Dropbox shared link | +| 0.2.25 | 2022-10-14 | [17994](https://github.com/airbytehq/airbyte/pull/17994) | Handle `UnicodeDecodeError` during discover step. | +| 0.2.24 | 2022-10-03 | [17504](https://github.com/airbytehq/airbyte/pull/17504) | Validate data for `HTTPS` while `check_connection` | +| 0.2.23 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state. | +| 0.2.22 | 2022-09-15 | [16772](https://github.com/airbytehq/airbyte/pull/16772) | Fix schema generation for JSON files containing arrays | +| 0.2.21 | 2022-08-26 | [15568](https://github.com/airbytehq/airbyte/pull/15568) | Specify `pyxlsb` library for Excel Binary Workbook files | +| 0.2.20 | 2022-08-23 | [15870](https://github.com/airbytehq/airbyte/pull/15870) | Fix CSV schema discovery | +| 0.2.19 | 2022-08-19 | [15768](https://github.com/airbytehq/airbyte/pull/15768) | Convert 'nan' to 'null' | +| 0.2.18 | 2022-08-16 | [15698](https://github.com/airbytehq/airbyte/pull/15698) | Cache binary stream to file for discover | +| 0.2.17 | 2022-08-11 | [15501](https://github.com/airbytehq/airbyte/pull/15501) | Cache binary stream to file | +| 0.2.16 | 2022-08-10 | [15293](https://github.com/airbytehq/airbyte/pull/15293) | Add support for encoding reader option | +| 0.2.15 | 2022-08-05 | [15269](https://github.com/airbytehq/airbyte/pull/15269) | Bump `smart-open` version to 6.0.0 | +| 0.2.12 | 2022-07-12 | [14535](https://github.com/airbytehq/airbyte/pull/14535) | Fix invalid schema generation for JSON files | +| 0.2.11 | 2022-07-12 | [9974](https://github.com/airbytehq/airbyte/pull/14588) | Add support to YAML format | +| 0.2.9 | 2022-02-01 | [9974](https://github.com/airbytehq/airbyte/pull/9974) | Update airbyte-cdk 0.1.47 | +| 0.2.8 | 2021-12-06 | [8524](https://github.com/airbytehq/airbyte/pull/8524) | Update connector fields title/description | +| 0.2.7 | 2021-10-28 | [7387](https://github.com/airbytehq/airbyte/pull/7387) | Migrate source to CDK structure, add SAT testing. | +| 0.2.6 | 2021-08-26 | [5613](https://github.com/airbytehq/airbyte/pull/5613) | Add support to xlsb format | +| 0.2.5 | 2021-07-26 | [4953](https://github.com/airbytehq/airbyte/pull/4953) | Allow non-default port for SFTP type | +| 0.2.4 | 2021-06-09 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add AIRBYTE_ENTRYPOINT for Kubernetes support | +| 0.2.3 | 2021-06-01 | [3771](https://github.com/airbytehq/airbyte/pull/3771) | Add Azure Storage Blob Files option | +| 0.2.2 | 2021-04-16 | [2883](https://github.com/airbytehq/airbyte/pull/2883) | Fix CSV discovery memory consumption | +| 0.2.1 | 2021-04-03 | [2726](https://github.com/airbytehq/airbyte/pull/2726) | Fix base connector versioning | +| 0.2.0 | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238) | Protocol allows future/unknown properties | +| 0.1.10 | 2021-02-18 | [2118](https://github.com/airbytehq/airbyte/pull/2118) | Support JSONL format | +| 0.1.9 | 2021-02-02 | [1768](https://github.com/airbytehq/airbyte/pull/1768) | Add test cases for all formats | +| 0.1.8 | 2021-01-27 | [1738](https://github.com/airbytehq/airbyte/pull/1738) | Adopt connector best practices | +| 0.1.7 | 2020-12-16 | [1331](https://github.com/airbytehq/airbyte/pull/1331) | Refactor Python base connector | +| 0.1.6 | 2020-12-08 | [1249](https://github.com/airbytehq/airbyte/pull/1249) | Handle NaN values | +| 0.1.5 | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046) | Add connectors using an index YAML file | \ No newline at end of file diff --git a/docs/integrations/sources/freshcaller.md b/docs/integrations/sources/freshcaller.md index 3af2e263107a..72e9dd857f71 100644 --- a/docs/integrations/sources/freshcaller.md +++ b/docs/integrations/sources/freshcaller.md @@ -41,7 +41,9 @@ Please read [How to find your API key](https://support.freshdesk.com/en/support/ ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------| -| 0.2.0 | 2023-05-15 | [26065](https://github.com/airbytehq/airbyte/pull/26065) | Fix spec type check for `start_date` | -| 0.1.0 | 2022-08-11 | [14759](https://github.com/airbytehq/airbyte/pull/14759) | 🎉 New Source: Freshcaller | \ No newline at end of file +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------- | +| 0.3.1 | 2023-11-28 | [32874](https://github.com/airbytehq/airbyte/pull/32874) | 🐛 Source: fix page_size_option parameter in spec | +| 0.3.0 | 2023-10-24 | [31102](https://github.com/airbytehq/airbyte/pull/14759) | ✨ Source: Migrate to Low Code CDK | +| 0.2.0 | 2023-05-15 | [26065](https://github.com/airbytehq/airbyte/pull/26065) | Fix spec type check for `start_date` | +| 0.1.0 | 2022-08-11 | [14759](https://github.com/airbytehq/airbyte/pull/14759) | 🎉 New Source: Freshcaller | diff --git a/docs/integrations/sources/freshdesk.md b/docs/integrations/sources/freshdesk.md index 126173fefbbe..95b855e3b96d 100644 --- a/docs/integrations/sources/freshdesk.md +++ b/docs/integrations/sources/freshdesk.md @@ -68,6 +68,8 @@ If you don't use the start date Freshdesk will retrieve only the last 30 days. M | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------ | +| 3.0.6 | 2024-01-10 | [34101](https://github.com/airbytehq/airbyte/pull/34101) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 3.0.5 | 2023-11-30 | [33000](https://github.com/airbytehq/airbyte/pull/33000) | Base image migration: remove Dockerfile and use the python-connector-base image | | 3.0.4 | 2023-06-24 | [27680](https://github.com/airbytehq/airbyte/pull/27680) | Fix formatting | | 3.0.3 | 2023-06-02 | [26978](https://github.com/airbytehq/airbyte/pull/26978) | Skip the stream if subscription level had changed during sync | | 3.0.2 | 2023-02-06 | [21970](https://github.com/airbytehq/airbyte/pull/21970) | Enable availability strategy for all streams | diff --git a/docs/integrations/sources/freshsales-migrations.md b/docs/integrations/sources/freshsales-migrations.md new file mode 100644 index 000000000000..42b98fbb668d --- /dev/null +++ b/docs/integrations/sources/freshsales-migrations.md @@ -0,0 +1,7 @@ +# Freshsales Migration Guide + +## Upgrading to 1.0.0 + +This version migrates the Freshsales connector to our low-code framework for greater maintainability. + +As part of this release, we've also updated data types across streams to match the correct return types from the upstream API. You will need to run a reset on connections using this connector after upgrading to continue syncing. diff --git a/docs/integrations/sources/freshsales.md b/docs/integrations/sources/freshsales.md index 685398c4eae1..b8b100a0301c 100644 --- a/docs/integrations/sources/freshsales.md +++ b/docs/integrations/sources/freshsales.md @@ -69,6 +69,7 @@ The Freshsales connector should not run into Freshsales API limitations under no | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------| +| 1.0.0 | 2023-10-21 | [31685](https://github.com/airbytehq/airbyte/pull/31685) | Migrate to Low-Code CDK | | 0.1.4 | 2023-03-23 | [24396](https://github.com/airbytehq/airbyte/pull/24396) | Certify to Beta | | 0.1.3 | 2023-03-16 | [24155](https://github.com/airbytehq/airbyte/pull/24155) | Set `additionalProperties` to `True` in `spec` to support BC | | 0.1.2 | 2022-07-14 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Tune the `get_view_id` function | diff --git a/docs/integrations/sources/freshservice.md b/docs/integrations/sources/freshservice.md index 6481eb888c3f..1a9a30aebdf0 100644 --- a/docs/integrations/sources/freshservice.md +++ b/docs/integrations/sources/freshservice.md @@ -54,6 +54,7 @@ Please read [How to find your API key](https://api.freshservice.com/#authenticat | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 1.3.0 | 2024-01-15 | [29126](https://github.com/airbytehq/airbyte/pull/29126) | Add `Requested Items` stream | | 1.2.0 | 2023-08-06 | [29126](https://github.com/airbytehq/airbyte/pull/29126) | Migrated to Low-Code CDK | | 1.1.0 | 2023-05-09 | [25929](https://github.com/airbytehq/airbyte/pull/25929) | Add stream for customer satisfaction survey responses endpoint | | 1.0.0 | 2023-05-02 | [25743](https://github.com/airbytehq/airbyte/pull/25743) | Correct data types in tickets, agents and requesters schemas to match Freshservice API | diff --git a/docs/integrations/sources/gcs.md b/docs/integrations/sources/gcs.md index 8b309e911197..1dcb6d735fc0 100644 --- a/docs/integrations/sources/gcs.md +++ b/docs/integrations/sources/gcs.md @@ -35,7 +35,12 @@ Use the service account ID from above, grant read access to your target bucket. ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :------------------- | -| 0.2.0 | 2023-06-26 | [27725](https://github.com/airbytehq/airbyte/pull/27725) | License Update: Elv2 | -| 0.1.0 | 2023-02-16 | [23186](https://github.com/airbytehq/airbyte/pull/23186) | New Source: GCS | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------| +| 0.3.4 | 2024-01-11 | [34158](https://github.com/airbytehq/airbyte/pull/34158) | Fix issue in stream reader for document file type parser | +| 0.3.3 | 2023-12-06 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Bump CDK version to hide source-defined primary key | +| 0.3.2 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Improve document file type parser | +| 0.3.1 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | +| 0.3.0 | 2023-10-11 | [31212](https://github.com/airbytehq/airbyte/pull/31212) | Migrated to file based CDK | +| 0.2.0 | 2023-06-26 | [27725](https://github.com/airbytehq/airbyte/pull/27725) | License Update: Elv2 | +| 0.1.0 | 2023-02-16 | [23186](https://github.com/airbytehq/airbyte/pull/23186) | New Source: GCS | diff --git a/docs/integrations/sources/getlago.md b/docs/integrations/sources/getlago.md index 056ad3c566c8..33c296afd364 100644 --- a/docs/integrations/sources/getlago.md +++ b/docs/integrations/sources/getlago.md @@ -1,8 +1,8 @@ -# getLago API +# Lago API ## Sync overview -This source can sync data from the [getLago API](https://doc.getlago.com/docs/guide/intro/welcome). At present this connector only supports full refresh syncs meaning that each time you use the connector it will sync all available records from scratch. Please use cautiously if you expect your API to have a lot of records. +This source can sync data from the [Lago API](https://doc.getlago.com/docs/guide/intro/welcome). At present this connector only supports full refresh syncs meaning that each time you use the connector it will sync all available records from scratch. Please use cautiously if you expect your API to have a lot of records. ## This Source Supports the Following Streams @@ -25,11 +25,13 @@ This source can sync data from the [getLago API](https://doc.getlago.com/docs/gu ## Getting started ### Requirements - -* getLago API KEY +* Lago API URL +* Lago API KEY ## Changelog | Version | Date | Pull Request | Subject | | :------ | :--------- | :-------------------------------------------------------- | :----------------------------------------- | +| 0.3.0 | 2023-10-05 | [#31099](https://github.com/airbytehq/airbyte/pull/31099) | Added customer_usage and wallet stream | +| 0.2.0 | 2023-09-19 | [#30572](https://github.com/airbytehq/airbyte/pull/30572) | Source GetLago: Support API URL | | 0.1.0 | 2022-10-26 | [#18727](https://github.com/airbytehq/airbyte/pull/18727) | 🎉 New Source: getLago API [low-code CDK] | \ No newline at end of file diff --git a/docs/integrations/sources/github.inapp.md b/docs/integrations/sources/github.inapp.md deleted file mode 100644 index 5bd87e059bf4..000000000000 --- a/docs/integrations/sources/github.inapp.md +++ /dev/null @@ -1,47 +0,0 @@ -## Prerequisites - -- Access to a Github repository - -## Setup guide - -1. Name your source. -2. Click `Authenticate your GitHub account` or use a [Personal Access Token](https://github.com/settings/tokens) for Authentication. For Personal Access Tokens, refer to the list of required [permissions and scopes](https://docs.airbyte.com/integrations/sources/github#permissions-and-scopes). -3. **Start date** Enter the date you'd like to replicate data from. - -These streams will only sync records generated on or after the **Start Date**: - -`comments`, `commit_comment_reactions`, `commit_comments`, `commits`, `deployments`, `events`, `issue_comment_reactions`, `issue_events`, `issue_milestones`, `issue_reactions`, `issues`, `project_cards`, `project_columns`, `projects`, `pull_request_comment_reactions`, `pull_requests`, `pull_requeststats`, `releases`, `review_comments`, `reviews`, `stargazers`, `workflow_runs`, `workflows`. - -The **Start Date** does not apply to the streams below and all data will be synced for these streams: - -`assignees`, `branches`, `collaborators`, `issue_labels`, `organizations`, `pull_request_commits`, `pull_request_stats`, `repositories`, `tags`, `teams`, `users` - -4. **GitHub Repositories** - Enter a space-delimited list of GitHub organizations or repositories. - -Example of a single repository: -``` -airbytehq/airbyte -``` -Example of multiple repositories: -``` -airbytehq/airbyte airbytehq/another-repo -``` -Example of an organization to receive data from all of its repositories: -``` -airbytehq/* -``` -Repositories which have a misspelled name, do not exist, or have the wrong name format will return an error. - -5. (Optional) **Branch** - Enter a space-delimited list of GitHub repository branches to pull commits for, e.g. `airbytehq/airbyte/master`. If no branches are specified for a repository, the default branch will be pulled. (e.g. `airbytehq/airbyte/master airbytehq/airbyte/my-branch`). -6. (Optional) **Max requests per hour** - The GitHub API allows for a maximum of 5000 requests per hour (15,000 for Github Enterprise). You can specify a lower value to limit your use of the API quota. - -### Incremental Sync Methods -Incremental sync is offered for most streams, with some differences in sync behavior. - -1. `comments`, `commits`, `issues` and `review comments` only syncs new records. Only new records will be synced. - -2. `workflow_runs` and `worflow_jobs` syncs new records and any records run in the [last 30 days](https://docs.github.com/en/actions/managing-workflow-runs/re-running-workflows-and-jobs) - -3. All other incremental streams sync all historical records and output any updated or new records. - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [GitHub](https://docs.airbyte.com/integrations/sources/github/). diff --git a/docs/integrations/sources/github.md b/docs/integrations/sources/github.md index aca33b1c377d..6fdf34bba108 100644 --- a/docs/integrations/sources/github.md +++ b/docs/integrations/sources/github.md @@ -1,19 +1,20 @@ # GitHub -This page contains the setup guide and reference information for the GitHub source connector. + + +This page contains the setup guide and reference information for the [GitHub](https://www.github.com) source connector. + + ## Prerequisites -- Start date -- GitHub Repositories -- Branch (Optional) -- Page size for large streams (Optional) +- List of GitHub Repositories (and access for them in case they are private) **For Airbyte Cloud:** -- Personal Access Token (see [Permissions and scopes](https://docs.airbyte.com/integrations/sources/github#permissions-and-scopes)) - OAuth +- Personal Access Token (see [Permissions and scopes](https://docs.airbyte.com/integrations/sources/github#permissions-and-scopes)) @@ -34,86 +35,101 @@ Create a [GitHub Account](https://github.com). Log into [GitHub](https://github.com) and then generate a [personal access token](https://github.com/settings/tokens). To load balance your API quota consumption across multiple API tokens, input multiple tokens separated with `,`. - ### Step 2: Set up the GitHub connector in Airbyte + **For Airbyte Cloud:** 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. -2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ new source**. -3. On the source setup page, select **GitHub** from the Source type dropdown and enter a name for this connector. -4. Click `Authenticate your GitHub account` by selecting Oauth or Personal Access Token for Authentication. -5. Log in and Authorize to the GitHub account. -6. **Start date** - The date from which you'd like to replicate data for streams: `comments`, `commit_comment_reactions`, `commit_comments`, `commits`, `deployments`, `events`, `issue_comment_reactions`, `issue_events`, `issue_milestones`, `issue_reactions`, `issues`, `project_cards`, `project_columns`, `projects`, `pull_request_comment_reactions`, `pull_requests`, `pull_requeststats`, `releases`, `review_comments`, `reviews`, `stargazers`, `workflow_runs`, `workflows`. -7. **GitHub Repositories** - Space-delimited list of GitHub organizations/repositories, e.g. `airbytehq/airbyte` for single repository, `airbytehq/airbyte airbytehq/another-repo` for multiple repositories. If you want to specify the organization to receive data from all its repositories, then you should specify it according to the following example: `airbytehq/*`. Repositories with the wrong name, or repositories that do not exist, or have the wrong name format are not allowed. -8. **Branch (Optional)** - Space-delimited list of GitHub repository branches to pull commits for, e.g. `airbytehq/airbyte/master`. If no branches are specified for a repository, the default branch will be pulled. (e.g. `airbytehq/airbyte/master airbytehq/airbyte/my-branch`). -9. **Max requests per hour (Optional)** - The GitHub API allows for a maximum of 5000 requests per hour (15000 for Github Enterprise). You can specify a lower value to limit your use of the API quota. - +2. In the left navigation bar, click **Sources**. +3. On the source selection page, select **GitHub** from the list of Sources. +4. Add a name for your GitHub connector. +5. To authenticate: + + - **For Airbyte Cloud:** **Authenticate your GitHub account** to authorize your GitHub account. Airbyte will authenticate the GitHub account you are already logged in to. Please make sure you are logged into the right account. + -**For Airbyte Open Source:** -1. Authenticate with **Personal Access Token**. + - **For Airbyte Open Source:** Authenticate with **Personal Access Token**. To generate a personal access token, log into [GitHub](https://github.com) and then generate a [personal access token](https://github.com/settings/tokens). Enter your GitHub personal access token. To load balance your API quota consumption across multiple API tokens, input multiple tokens separated with `,`. +6. **GitHub Repositories** - Enter a list of GitHub organizations/repositories, e.g. `airbytehq/airbyte` for single repository, `airbytehq/airbyte airbytehq/another-repo` for multiple repositories. If you want to specify the organization to receive data from all its repositories, then you should specify it according to the following example: `airbytehq/*`. + +:::caution +Repositories with the wrong name or repositories that do not exist or have the wrong name format will be skipped with `WARN` message in the logs. +::: + +7. **Start date (Optional)** - The date from which you'd like to replicate data for streams. For streams which support this configuration, only data generated on or after the start date will be replicated. + +- These streams will only sync records generated on or after the **Start Date**: `comments`, `commit_comment_reactions`, `commit_comments`, `commits`, `deployments`, `events`, `issue_comment_reactions`, `issue_events`, `issue_milestones`, `issue_reactions`, `issues`, `project_cards`, `project_columns`, `projects`, `pull_request_comment_reactions`, `pull_requests`, `pull_requeststats`, `releases`, `review_comments`, `reviews`, `stargazers`, `workflow_runs`, `workflows`. + +- The **Start Date** does not apply to the streams below and all data will be synced for these streams: `assignees`, `branches`, `collaborators`, `issue_labels`, `organizations`, `pull_request_commits`, `pull_request_stats`, `repositories`, `tags`, `teams`, `users` + +8. **Branch (Optional)** - List of GitHub repository branches to pull commits from, e.g. `airbytehq/airbyte/master`. If no branches are specified for a repository, the default branch will be pulled. (e.g. `airbytehq/airbyte/master airbytehq/airbyte/my-branch`). +9. **Max requests per hour (Optional)** - The GitHub API allows for a maximum of 5,000 requests per hour (15,000 for Github Enterprise). You can specify a lower value to limit your use of the API quota. Refer to GitHub article [Rate limits for the REST API](https://docs.github.com/en/rest/overview/rate-limits-for-the-rest-api). + + + ## Supported sync modes The GitHub source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-modes): -| Feature | Supported? | -| :---------------------------- | :---------- | -| Full Refresh Sync | Yes | -| Incremental - Append Sync | Yes | -| Replicate Incremental Deletes | Coming soon | -| SSL connection | Yes | -| Namespaces | No | +- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/) +- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append) +- [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) +- [Incremental Sync - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped) ## Supported Streams This connector outputs the following full refresh streams: -- [Assignees](https://docs.github.com/en/rest/reference/issues#list-assignees) -- [Branches](https://docs.github.com/en/rest/reference/repos#list-branches) -- [Collaborators](https://docs.github.com/en/rest/reference/repos#list-repository-collaborators) -- [Issue labels](https://docs.github.com/en/rest/issues/labels#list-labels-for-a-repository) -- [Organizations](https://docs.github.com/en/rest/reference/orgs#get-an-organization) -- [Pull request commits](https://docs.github.com/en/rest/reference/pulls#list-commits-on-a-pull-request) -- [Tags](https://docs.github.com/en/rest/reference/repos#list-repository-tags) -- [TeamMembers](https://docs.github.com/en/rest/teams/members#list-team-members) -- [TeamMemberships](https://docs.github.com/en/rest/reference/teams#get-team-membership-for-a-user) -- [Teams](https://docs.github.com/en/rest/reference/teams#list-teams) -- [Users](https://docs.github.com/en/rest/reference/orgs#list-organization-members) +- [Assignees](https://docs.github.com/en/rest/issues/assignees?apiVersion=2022-11-28#list-assignees) +- [Branches](https://docs.github.com/en/rest/branches/branches?apiVersion=2022-11-28#list-branches) +- [Contributor Activity](https://docs.github.com/en/rest/metrics/statistics?apiVersion=2022-11-28#get-all-contributor-commit-activity) +- [Collaborators](https://docs.github.com/en/rest/collaborators/collaborators?apiVersion=2022-11-28#list-repository-collaborators) +- [Issue labels](https://docs.github.com/en/rest/issues/labels?apiVersion=2022-11-28#list-labels-for-a-repository) +- [Organizations](https://docs.github.com/en/rest/orgs/orgs?apiVersion=2022-11-28#list-organizations) +- [Pull request commits](https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28#list-commits-on-a-pull-request) +- [Tags](https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-repository-tags) +- [TeamMembers](https://docs.github.com/en/rest/teams/members?apiVersion=2022-11-28#list-team-members) +- [TeamMemberships](https://docs.github.com/en/rest/teams/members?apiVersion=2022-11-28#get-team-membership-for-a-user) +- [Teams](https://docs.github.com/en/rest/teams/teams?apiVersion=2022-11-28#list-teams) +- [Users](https://docs.github.com/en/rest/orgs/members?apiVersion=2022-11-28#list-organization-members) +- [Issue timeline events](https://docs.github.com/en/rest/issues/timeline?apiVersion=2022-11-28#list-timeline-events-for-an-issue) This connector outputs the following incremental streams: -- [Comments](https://docs.github.com/en/rest/reference/issues#list-issue-comments-for-a-repository) -- [Commit comment reactions](https://docs.github.com/en/rest/reference/reactions#list-reactions-for-a-commit-comment) -- [Commit comments](https://docs.github.com/en/rest/reference/repos#list-commit-comments-for-a-repository) -- [Commits](https://docs.github.com/en/rest/reference/repos#list-commits) -- [Deployments](https://docs.github.com/en/rest/reference/deployments#list-deployments) -- [Events](https://docs.github.com/en/rest/reference/activity#list-repository-events) -- [Issue comment reactions](https://docs.github.com/en/rest/reference/reactions#list-reactions-for-an-issue-comment) -- [Issue events](https://docs.github.com/en/rest/reference/issues#list-issue-events-for-a-repository) -- [Issue milestones](https://docs.github.com/en/rest/reference/issues#list-milestones) -- [Issue reactions](https://docs.github.com/en/rest/reference/reactions#list-reactions-for-an-issue) -- [Issues](https://docs.github.com/en/rest/reference/issues#list-repository-issues) -- [Project cards](https://docs.github.com/en/rest/reference/projects#list-project-cards) -- [Project columns](https://docs.github.com/en/rest/reference/projects#list-project-columns) -- [Projects](https://docs.github.com/en/rest/reference/projects#list-repository-projects) -- [Pull request comment reactions](https://docs.github.com/en/rest/reference/reactions#list-reactions-for-a-pull-request-review-comment) -- [Pull request stats](https://docs.github.com/en/rest/reference/pulls#get-a-pull-request) -- [Pull requests](https://docs.github.com/en/rest/reference/pulls#list-pull-requests) -- [Releases](https://docs.github.com/en/rest/reference/repos#list-releases) -- [Repositories](https://docs.github.com/en/rest/reference/repos#list-organization-repositories) -- [Review comments](https://docs.github.com/en/rest/reference/pulls#list-review-comments-in-a-repository) -- [Reviews](https://docs.github.com/en/rest/reference/pulls#list-reviews-for-a-pull-request) -- [Stargazers](https://docs.github.com/en/rest/reference/activity#list-stargazers) -- [WorkflowRuns](https://docs.github.com/en/rest/actions/workflow-runs#list-workflow-runs-for-a-repository) -- [Workflows](https://docs.github.com/en/rest/reference/actions#workflows) +- [Comments](https://docs.github.com/en/rest/issues/comments?apiVersion=2022-11-28#list-issue-comments-for-a-repository) +- [Commit comment reactions](https://docs.github.com/en/rest/reference/reactions?apiVersion=2022-11-28#list-reactions-for-a-commit-comment) +- [Commit comments](https://docs.github.com/en/rest/commits/comments?apiVersion=2022-11-28#list-commit-comments-for-a-repository) +- [Commits](https://docs.github.com/en/rest/commits/commits?apiVersion=2022-11-28#list-commits) +- [Deployments](https://docs.github.com/en/rest/deployments/deployments?apiVersion=2022-11-28#list-deployments) +- [Events](https://docs.github.com/en/rest/activity/events?apiVersion=2022-11-28#list-repository-events) +- [Issue comment reactions](https://docs.github.com/en/rest/reactions/reactions?apiVersion=2022-11-28#list-reactions-for-an-issue-comment) +- [Issue events](https://docs.github.com/en/rest/issues/events?apiVersion=2022-11-28#list-issue-events-for-a-repository) +- [Issue milestones](https://docs.github.com/en/rest/issues/milestones?apiVersion=2022-11-28#list-milestones) +- [Issue reactions](https://docs.github.com/en/rest/reactions/reactions?apiVersion=2022-11-28#list-reactions-for-an-issue) +- [Issues](https://docs.github.com/en/rest/issues/issues?apiVersion=2022-11-28#list-repository-issues) +- [Project (Classic) cards](https://docs.github.com/en/rest/projects/cards?apiVersion=2022-11-28#list-project-cards) +- [Project (Classic) columns](https://docs.github.com/en/rest/projects/columns?apiVersion=2022-11-28#list-project-columns) +- [Projects (Classic)](https://docs.github.com/en/rest/projects/projects?apiVersion=2022-11-28#list-repository-projects) +- [ProjectsV2](https://docs.github.com/en/graphql/reference/objects#projectv2) +- [Pull request comment reactions](https://docs.github.com/en/rest/reactions/reactions?apiVersion=2022-11-28#list-reactions-for-a-pull-request-review-comment) +- [Pull request stats](https://docs.github.com/en/graphql/reference/objects#pullrequest) +- [Pull requests](https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28#list-pull-requests) +- [Releases](https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#list-releases) +- [Repositories](https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-organization-repositories) +- [Review comments](https://docs.github.com/en/rest/pulls/comments?apiVersion=2022-11-28#list-review-comments-in-a-repository) +- [Reviews](https://docs.github.com/en/rest/pulls/reviews?apiVersion=2022-11-28#list-reviews-for-a-pull-request) +- [Stargazers](https://docs.github.com/en/rest/activity/starring?apiVersion=2022-11-28#list-stargazers) +- [WorkflowJobs](https://docs.github.com/pt/rest/actions/workflow-jobs?apiVersion=2022-11-28#list-jobs-for-a-workflow-run) +- [WorkflowRuns](https://docs.github.com/en/rest/actions/workflow-runs?apiVersion=2022-11-28#list-workflow-runs-for-a-repository) +- [Workflows](https://docs.github.com/en/rest/actions/workflows?apiVersion=2022-11-28#list-repository-workflows) ### Notes -1. Only 4 streams \(`comments`, `commits`, `issues` and `review comments`\) from the above 24 incremental streams are pure incremental meaning that they: +1. Only 4 streams \(`comments`, `commits`, `issues` and `review comments`\) from the listed above streams are pure incremental meaning that they: - read only new records; - output only new records. @@ -130,7 +146,7 @@ This connector outputs the following incremental streams: - output only new records. Please, consider this behaviour when using those 19 incremental streams because it may affect you API call limits. -4. We are passing few parameters \(`since`, `sort` and `direction`\) to GitHub in order to filter records and sometimes for large streams specifying very distant `start_date` in the past may result in keep on getting error from GitHub instead of records \(respective `WARN` log message will be outputted\). In this case Specifying more recent `start_date` may help. +4. Sometimes for large streams specifying very distant `start_date` in the past may result in keep on getting error from GitHub instead of records \(respective `WARN` log message will be outputted\). In this case Specifying more recent `start_date` may help. **The "Start date" configuration option does not apply to the streams below, because the GitHub API does not include dates which can be used for filtering:** - `assignees` @@ -145,9 +161,21 @@ This connector outputs the following incremental streams: - `teams` - `users` -### Permissions and scopes +## Limitations & Troubleshooting + +
      + +Expand to see details about GitHub connector limitations and troubleshooting. + -If you use OAuth authentication method, the oauth2.0 application requests the next list of [scopes](https://docs.github.com/en/developers/apps/building-oauth-apps/scopes-for-oauth-apps#available-scopes): **repo**, **read:org**, **read:repo_hook**, **read:user**, **read:discussion**, **workflow**. For [personal access token](https://github.com/settings/tokens) it need to manually select needed scopes. +### Connector limitations + +#### Rate limiting +The GitHub connector should not run into GitHub API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. Refer to GitHub article [Rate limits for the REST API](https://docs.github.com/en/rest/overview/rate-limits-for-the-rest-api). + +#### Permissions and scopes + +If you use OAuth authentication method, the OAuth2.0 application requests the next list of [scopes](https://docs.github.com/en/developers/apps/building-oauth-apps/scopes-for-oauth-apps#available-scopes): **repo**, **read:org**, **read:repo_hook**, **read:user**, **read:discussion**, **workflow**. For [personal access token](https://github.com/settings/tokens) you need to manually select needed scopes. Your token should have at least the `repo` scope. Depending on which streams you want to sync, the user generating the token needs more permissions: @@ -155,17 +183,38 @@ Your token should have at least the `repo` scope. Depending on which streams you - Syncing [Teams](https://docs.github.com/en/organizations/organizing-members-into-teams/about-teams) is only available to authenticated members of a team's [organization](https://docs.github.com/en/rest/orgs). [Personal user accounts](https://docs.github.com/en/get-started/learning-about-github/types-of-github-accounts) and repositories belonging to them don't have access to Teams features. In this case no records will be synced. - To sync the Projects stream, the repository must have the Projects feature enabled. -### Performance considerations +### Troubleshooting -The GitHub connector should not run into GitHub API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. +* Check out common troubleshooting issues for the GitHub source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions) + +
      ## Changelog | Version | Date | Pull Request | Subject | |:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 1.0.4 | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes | -| 1.0.3 | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references | -| 1.0.2 | 2023-07-11 | [28144](https://github.com/airbytehq/airbyte/pull/28144) | Add `archived_at` property to `Organizations` schema parameter | +| 1.5.5 | 2023-12-26 | [33783](https://github.com/airbytehq/airbyte/pull/33783) | Fix retry for 504 error in GraphQL based streams | +| 1.5.4 | 2023-11-20 | [32679](https://github.com/airbytehq/airbyte/pull/32679) | Return AirbyteMessage if max retry exeeded for 202 status code | +| 1.5.3 | 2023-10-23 | [31702](https://github.com/airbytehq/airbyte/pull/31702) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 1.5.2 | 2023-10-13 | [31386](https://github.com/airbytehq/airbyte/pull/31386) | Handle `ContributorActivity` continuous `ACCEPTED` response | +| 1.5.1 | 2023-10-12 | [31307](https://github.com/airbytehq/airbyte/pull/31307) | Increase backoff_time for stream `ContributorActivity` | +| 1.5.0 | 2023-10-11 | [31300](https://github.com/airbytehq/airbyte/pull/31300) | Update Schemas: Add date-time format to fields | +| 1.4.6 | 2023-10-04 | [31056](https://github.com/airbytehq/airbyte/pull/31056) | Migrate spec properties' `repository` and `branch` type to \ | +| 1.4.5 | 2023-10-02 | [31023](https://github.com/airbytehq/airbyte/pull/31023) | Increase backoff for stream `Contributor Activity` | +| 1.4.4 | 2023-10-02 | [30971](https://github.com/airbytehq/airbyte/pull/30971) | Mark `start_date` as optional. | +| 1.4.3 | 2023-10-02 | [30979](https://github.com/airbytehq/airbyte/pull/30979) | Fetch archived records in `Project Cards` | +| 1.4.2 | 2023-09-30 | [30927](https://github.com/airbytehq/airbyte/pull/30927) | Provide actionable user error messages | +| 1.4.1 | 2023-09-30 | [30839](https://github.com/airbytehq/airbyte/pull/30839) | Update CDK to Latest version | +| 1.4.0 | 2023-09-29 | [30823](https://github.com/airbytehq/airbyte/pull/30823) | Add new stream `issue Timeline Events` | +| 1.3.1 | 2023-09-28 | [30824](https://github.com/airbytehq/airbyte/pull/30824) | Handle empty response in stream `ContributorActivity` | +| 1.3.0 | 2023-09-25 | [30731](https://github.com/airbytehq/airbyte/pull/30731) | Add new stream `ProjectsV2` | +| 1.2.1 | 2023-09-22 | [30693](https://github.com/airbytehq/airbyte/pull/30693) | Handle 404 error in `TeamMemberShips` | +| 1.2.0 | 2023-09-22 | [30647](https://github.com/airbytehq/airbyte/pull/30647) | Add support for self-hosted GitHub instances | +| 1.1.1 | 2023-09-21 | [30654](https://github.com/airbytehq/airbyte/pull/30654) | Rewrite source connection error messages | +| 1.1.0 | 2023-08-03 | [30615](https://github.com/airbytehq/airbyte/pull/30615) | Add new stream `Contributor Activity` | +| 1.0.4 | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes | +| 1.0.3 | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references | +| 1.0.2 | 2023-07-11 | [28144](https://github.com/airbytehq/airbyte/pull/28144) | Add `archived_at` property to `Organizations` schema parameter | | 1.0.1 | 2023-05-22 | [25838](https://github.com/airbytehq/airbyte/pull/25838) | Deprecate "page size" input parameter | | 1.0.0 | 2023-05-19 | [25778](https://github.com/airbytehq/airbyte/pull/25778) | Improve repo(s) name validation on UI | | 0.5.0 | 2023-05-16 | [25793](https://github.com/airbytehq/airbyte/pull/25793) | Implement client-side throttling of requests | @@ -252,3 +301,5 @@ The GitHub connector should not run into GitHub API limitations under normal usa | 0.1.2 | 2021-07-13 | [4708](https://github.com/airbytehq/airbyte/pull/4708) | Fix bug with IssueEvents stream and add handling for rate limiting | | 0.1.1 | 2021-07-07 | [4590](https://github.com/airbytehq/airbyte/pull/4590) | Fix schema in the `pull_request` stream | | 0.1.0 | 2021-07-06 | [4174](https://github.com/airbytehq/airbyte/pull/4174) | New Source: GitHub | + +
      \ No newline at end of file diff --git a/docs/integrations/sources/gitlab-migrations.md b/docs/integrations/sources/gitlab-migrations.md new file mode 100644 index 000000000000..e1a597fb8a77 --- /dev/null +++ b/docs/integrations/sources/gitlab-migrations.md @@ -0,0 +1,8 @@ +# Gitlab Migration Guide + +## Upgrading to 2.0.0 + + +In the 2.0.0 config change, several streams were updated to date-time field format, as declared in the Gitlab API. +These changes impact `pipeline.created_at` and` pipeline.updated_at` fields for stream Deployments and `expires_at` field for stream Group Members and stream Project Members. +You will need to refresh the source schema and reset affected streams after upgrading. \ No newline at end of file diff --git a/docs/integrations/sources/gitlab.md b/docs/integrations/sources/gitlab.md index caf45350163f..170f4b94c38d 100644 --- a/docs/integrations/sources/gitlab.md +++ b/docs/integrations/sources/gitlab.md @@ -5,9 +5,6 @@ This page contains the setup guide and reference information for the Gitlab Sour ## Prerequisites - Gitlab instance or an account at [Gitlab](https://gitlab.com) -- Start date -- GitLab Groups (Optional) -- GitLab Projects (Optional) @@ -49,10 +46,10 @@ Log into [GitLab](https://gitlab.com) and then generate a [personal access token 3. On the source setup page, select **GitLab** from the Source type dropdown and enter a name for this connector. 4. Click `Authenticate your GitLab account` by selecting Oauth or Personal Access Token for Authentication. 5. Log in and Authorize to the GitLab account. -6. **Start date** - The date from which you'd like to replicate data for streams. -7. **API URL** - The URL to access you self-hosted GitLab instance or `gitlab.com` (default). -8. **Groups (Optional)** - Space-delimited list of GitLab group IDs, e.g. `airbytehq` for single group, `airbytehq another-repo` for multiple groups. -9. **Projects (Optional)** - Space-delimited list of GitLab projects to pull data for, e.g. `airbytehq/airbyte`. +6. **API URL (Optional)** - The URL to access your self-hosted GitLab instance or `gitlab.com` (default). +7. **Start date (Optional)** - The date from which you'd like to replicate data for streams. +8. **Groups (Optional)** - List of GitLab group IDs, e.g. `airbytehq` for single group, `airbytehq another-repo` for multiple groups. +9. **Projects (Optional)** - List of GitLab projects to pull data for, e.g. `airbytehq/airbyte`. 10. Click **Set up source**. **Note:** You can specify either Group IDs or Project IDs in the source configuration. If both fields are blank, the connector will retrieve a list of all the groups that are accessible to the configured token and ingest as normal. @@ -94,9 +91,10 @@ This connector outputs the following streams: - [Group and Project members](https://docs.gitlab.com/ee/api/members.html) - [Tags](https://docs.gitlab.com/ee/api/tags.html) - [Releases](https://docs.gitlab.com/ee/api/releases/index.html) +- [Deployments](https://docs.gitlab.com/ee/api/deployments/index.html) - [Group Labels](https://docs.gitlab.com/ee/api/group_labels.html) - [Project Labels](https://docs.gitlab.com/ee/api/labels.html) -- [Epics](https://docs.gitlab.com/ee/api/epics.html) \(only available for GitLab Ultimate and GitLab.com Gold accounts\) +- [Epics](https://docs.gitlab.com/ee/api/epics.html) \(only available for GitLab Ultimate and GitLab.com Gold accounts. Stream Epics uses iid field as primary key for more convenient search and matching with UI. Iid is the internal ID of the epic, number of Epic on UI.\) - [Epic Issues](https://docs.gitlab.com/ee/api/epic_issues.html) \(only available for GitLab Ultimate and GitLab.com Gold accounts\) ## Additional information @@ -109,35 +107,45 @@ Gitlab has the [rate limits](https://docs.gitlab.com/ee/user/gitlab_com/index.ht ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :----------------------------------------------------------------------------------------- | -| 1.6.0 | 2023-06-30 | [27869](https://github.com/airbytehq/airbyte/pull/27869) | Add `shared_runners_setting` field to groups | -| 1.5.1 | 2023-06-24 | [27679](https://github.com/airbytehq/airbyte/pull/27679) | Fix formatting | -| 1.5.0 | 2023-06-15 | [27392](https://github.com/airbytehq/airbyte/pull/27392) | Make API URL an optional parameter in spec. | -| 1.4.2 | 2023-06-15 | [27346](https://github.com/airbytehq/airbyte/pull/27346) | Partially revert changes made in version 1.0.4, disallow http calls in cloud. | -| 1.4.1 | 2023-06-13 | [27351](https://github.com/airbytehq/airbyte/pull/27351) | Fix OAuth token expiry date. | -| 1.4.0 | 2023-06-12 | [27234](https://github.com/airbytehq/airbyte/pull/27234) | Skip stream slices on 403/404 errors, do not fail syncs. | -| 1.3.1 | 2023-06-08 | [27147](https://github.com/airbytehq/airbyte/pull/27147) | Improve connectivity check for connections with no projects/groups | -| 1.3.0 | 2023-06-08 | [27150](https://github.com/airbytehq/airbyte/pull/27150) | Update stream schemas | -| 1.2.1 | 2023-06-02 | [26947](https://github.com/airbytehq/airbyte/pull/26947) | New field `name` added to `Pipelines` and `PipelinesExtended` stream schema | -| 1.2.0 | 2023-05-17 | [22293](https://github.com/airbytehq/airbyte/pull/22293) | Preserve data in records with flattened keys | -| 1.1.1 | 2023-05-23 | [26422](https://github.com/airbytehq/airbyte/pull/26422) | Fix error `404 Repository Not Found` when syncing project with Repository feature disabled | -| 1.1.0 | 2023-05-10 | [25948](https://github.com/airbytehq/airbyte/pull/25948) | Introduce two new fields in the `Projects` stream schema | -| 1.0.4 | 2023-04-20 | [21373](https://github.com/airbytehq/airbyte/pull/21373) | Accept api_url with or without scheme | -| 1.0.3 | 2023-02-14 | [22992](https://github.com/airbytehq/airbyte/pull/22992) | Specified date formatting in specification | -| 1.0.2 | 2023-01-27 | [22001](https://github.com/airbytehq/airbyte/pull/22001) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 1.0.1 | 2023-01-23 | [21713](https://github.com/airbytehq/airbyte/pull/21713) | Fix missing data issue | -| 1.0.0 | 2022-12-05 | [7506](https://github.com/airbytehq/airbyte/pull/7506) | Add `OAuth2.0` authentication option | -| 0.1.12 | 2022-12-15 | [20542](https://github.com/airbytehq/airbyte/pull/20542) | Revert HttpAvailability changes, run on cdk 0.15.0 | -| 0.1.11 | 2022-12-14 | [20479](https://github.com/airbytehq/airbyte/pull/20479) | Use HttpAvailabilityStrategy + add unit tests | -| 0.1.10 | 2022-12-12 | [20384](https://github.com/airbytehq/airbyte/pull/20384) | Fetch groups along with their subgroups | -| 0.1.9 | 2022-12-11 | [20348](https://github.com/airbytehq/airbyte/pull/20348) | Fix 403 error when syncing `EpicIssues` stream | -| 0.1.8 | 2022-12-02 | [20023](https://github.com/airbytehq/airbyte/pull/20023) | Fix duplicated records issue for `Projects` stream | -| 0.1.7 | 2022-12-01 | [19986](https://github.com/airbytehq/airbyte/pull/19986) | Fix `GroupMilestones` stream schema | -| 0.1.6 | 2022-06-23 | [13252](https://github.com/airbytehq/airbyte/pull/13252) | Add GroupIssueBoards stream | -| 0.1.5 | 2022-05-02 | [11907](https://github.com/airbytehq/airbyte/pull/11907) | Fix null projects param and `container_expiration_policy` | -| 0.1.4 | 2022-03-23 | [11140](https://github.com/airbytehq/airbyte/pull/11140) | Ingest All Accessible Groups if not Specified in Config | -| 0.1.3 | 2021-12-21 | [8991](https://github.com/airbytehq/airbyte/pull/8991) | Update connector fields title/description | -| 0.1.2 | 2021-10-18 | [7108](https://github.com/airbytehq/airbyte/pull/7108) | Allow all domains to be used as `api_url` | -| 0.1.1 | 2021-10-12 | [6932](https://github.com/airbytehq/airbyte/pull/6932) | Fix pattern field in spec file, remove unused fields from config files, use cache from CDK | -| 0.1.0 | 2021-07-06 | [4174](https://github.com/airbytehq/airbyte/pull/4174) | Initial Release | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 2.1.1 | 2024-01-12 | [34203](https://github.com/airbytehq/airbyte/pull/34203) | prepare for airbyte-lib | +| 2.1.0 | 2023-12-20 | [33676](https://github.com/airbytehq/airbyte/pull/33676) | Add fields to Commits (extended_trailers), Groups (emails_enabled, service_access_tokens_expiration_enforced) and Projects (code_suggestions, model_registry_access_level) streams | +| 2.0.0 | 2023-10-23 | [31700](https://github.com/airbytehq/airbyte/pull/31700) | Add correct date-time format for Deployments, Projects and Groups Members streams | +| 1.8.4 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 1.8.3 | 2023-10-18 | [31547](https://github.com/airbytehq/airbyte/pull/31547) | Add validation for invalid `groups_list` and/or `projects_list` | +| 1.8.2 | 2023-10-17 | [31492](https://github.com/airbytehq/airbyte/pull/31492) | Expand list of possible error status codes when handling expired `access_token` | +| 1.8.1 | 2023-10-12 | [31375](https://github.com/airbytehq/airbyte/pull/31375) | Mark `start_date` as optional, migrate `groups` and `projects` to array | +| 1.8.0 | 2023-10-12 | [31339](https://github.com/airbytehq/airbyte/pull/31339) | Add undeclared fields to streams schemas, validate date/date-time format in stream schemas | +| 1.7.1 | 2023-10-10 | [31210](https://github.com/airbytehq/airbyte/pull/31210) | Added expired `access_token` handling, while checking the connection | +| 1.7.0 | 2023-08-08 | [27869](https://github.com/airbytehq/airbyte/pull/29203) | Add Deployments stream | +| 1.6.0 | 2023-06-30 | [27869](https://github.com/airbytehq/airbyte/pull/27869) | Add `shared_runners_setting` field to groups | +| 1.5.1 | 2023-06-24 | [27679](https://github.com/airbytehq/airbyte/pull/27679) | Fix formatting | +| 1.5.0 | 2023-06-15 | [27392](https://github.com/airbytehq/airbyte/pull/27392) | Make API URL an optional parameter in spec. | +| 1.4.2 | 2023-06-15 | [27346](https://github.com/airbytehq/airbyte/pull/27346) | Partially revert changes made in version 1.0.4, disallow http calls in cloud. | +| 1.4.1 | 2023-06-13 | [27351](https://github.com/airbytehq/airbyte/pull/27351) | Fix OAuth token expiry date. | +| 1.4.0 | 2023-06-12 | [27234](https://github.com/airbytehq/airbyte/pull/27234) | Skip stream slices on 403/404 errors, do not fail syncs. | +| 1.3.1 | 2023-06-08 | [27147](https://github.com/airbytehq/airbyte/pull/27147) | Improve connectivity check for connections with no projects/groups | +| 1.3.0 | 2023-06-08 | [27150](https://github.com/airbytehq/airbyte/pull/27150) | Update stream schemas | +| 1.2.1 | 2023-06-02 | [26947](https://github.com/airbytehq/airbyte/pull/26947) | New field `name` added to `Pipelines` and `PipelinesExtended` stream schema | +| 1.2.0 | 2023-05-17 | [22293](https://github.com/airbytehq/airbyte/pull/22293) | Preserve data in records with flattened keys | +| 1.1.1 | 2023-05-23 | [26422](https://github.com/airbytehq/airbyte/pull/26422) | Fix error `404 Repository Not Found` when syncing project with Repository feature disabled | +| 1.1.0 | 2023-05-10 | [25948](https://github.com/airbytehq/airbyte/pull/25948) | Introduce two new fields in the `Projects` stream schema | +| 1.0.4 | 2023-04-20 | [21373](https://github.com/airbytehq/airbyte/pull/21373) | Accept api_url with or without scheme | +| 1.0.3 | 2023-02-14 | [22992](https://github.com/airbytehq/airbyte/pull/22992) | Specified date formatting in specification | +| 1.0.2 | 2023-01-27 | [22001](https://github.com/airbytehq/airbyte/pull/22001) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 1.0.1 | 2023-01-23 | [21713](https://github.com/airbytehq/airbyte/pull/21713) | Fix missing data issue | +| 1.0.0 | 2022-12-05 | [7506](https://github.com/airbytehq/airbyte/pull/7506) | Add `OAuth2.0` authentication option | +| 0.1.12 | 2022-12-15 | [20542](https://github.com/airbytehq/airbyte/pull/20542) | Revert HttpAvailability changes, run on cdk 0.15.0 | +| 0.1.11 | 2022-12-14 | [20479](https://github.com/airbytehq/airbyte/pull/20479) | Use HttpAvailabilityStrategy + add unit tests | +| 0.1.10 | 2022-12-12 | [20384](https://github.com/airbytehq/airbyte/pull/20384) | Fetch groups along with their subgroups | +| 0.1.9 | 2022-12-11 | [20348](https://github.com/airbytehq/airbyte/pull/20348) | Fix 403 error when syncing `EpicIssues` stream | +| 0.1.8 | 2022-12-02 | [20023](https://github.com/airbytehq/airbyte/pull/20023) | Fix duplicated records issue for `Projects` stream | +| 0.1.7 | 2022-12-01 | [19986](https://github.com/airbytehq/airbyte/pull/19986) | Fix `GroupMilestones` stream schema | +| 0.1.6 | 2022-06-23 | [13252](https://github.com/airbytehq/airbyte/pull/13252) | Add GroupIssueBoards stream | +| 0.1.5 | 2022-05-02 | [11907](https://github.com/airbytehq/airbyte/pull/11907) | Fix null projects param and `container_expiration_policy` | +| 0.1.4 | 2022-03-23 | [11140](https://github.com/airbytehq/airbyte/pull/11140) | Ingest All Accessible Groups if not Specified in Config | +| 0.1.3 | 2021-12-21 | [8991](https://github.com/airbytehq/airbyte/pull/8991) | Update connector fields title/description | +| 0.1.2 | 2021-10-18 | [7108](https://github.com/airbytehq/airbyte/pull/7108) | Allow all domains to be used as `api_url` | +| 0.1.1 | 2021-10-12 | [6932](https://github.com/airbytehq/airbyte/pull/6932) | Fix pattern field in spec file, remove unused fields from config files, use cache from CDK | +| 0.1.0 | 2021-07-06 | [4174](https://github.com/airbytehq/airbyte/pull/4174) | Initial Release | \ No newline at end of file diff --git a/docs/integrations/sources/glassfrog.md b/docs/integrations/sources/glassfrog.md index b12151215226..703963ff2e94 100644 --- a/docs/integrations/sources/glassfrog.md +++ b/docs/integrations/sources/glassfrog.md @@ -48,6 +48,7 @@ This Source is capable of syncing the following Streams: | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.2.0 | 2023-08-10 | [29306](https://github.com/airbytehq/airbyte/pull/29306) | Migrated to LowCode CDK | | 0.1.1 | 2023-08-15 | [13868](https://github.com/airbytehq/airbyte/pull/13868) | Fix schema and tests | | 0.1.0 | 2022-06-16 | [13868](https://github.com/airbytehq/airbyte/pull/13868) | Add Native Glassfrog Source Connector | diff --git a/docs/integrations/sources/google-ads-migrations.md b/docs/integrations/sources/google-ads-migrations.md new file mode 100644 index 000000000000..22dcc734b26c --- /dev/null +++ b/docs/integrations/sources/google-ads-migrations.md @@ -0,0 +1,55 @@ +# Google Ads Migration Guide + +## Upgrading to 3.0.0 + +This release upgrades the Google Ads API from Version 13 to Version 15 which causes the following changes in the schemas: + +| Stream | Current field name | New field name | +|----------------------------|----------------------------------------------------------------------------|--------------------------------------------------------------------------| +| ad_listing_group_criterion | ad_group_criterion.listing_group.case_value.product_bidding_category.id | ad_group_criterion.listing_group.case_value.product_category.category_id | +| ad_listing_group_criterion | ad_group_criterion.listing_group.case_value.product_bidding_category.level | ad_group_criterion.listing_group.case_value.product_category.level | +| shopping_performance_view | segments.product_bidding_category_level1 | segments.product_category_level1 | +| shopping_performance_view | segments.product_bidding_category_level2 | segments.product_category_level2 | +| shopping_performance_view | segments.product_bidding_category_level3 | segments.product_category_level3 | +| shopping_performance_view | segments.product_bidding_category_level4 | segments.product_category_level4 | +| shopping_performance_view | segments.product_bidding_category_level5 | segments.product_category_level5 | +| campaign | campaign.shopping_setting.sales_country | This field has been deleted | + +Users should: +- Refresh the source schema +- Reset affected streams after upgrading to ensure uninterrupted syncs. + +### Refresh affected schemas and reset data + +1. Select **Connections** in the main navbar. + 1. Select the connection(s) affected by the update. +2. Select the **Replication** tab. + 1. Select **Refresh source schema**. + 2. Select **OK**. +```note +Any detected schema changes will be listed for your review. +``` +3. Select **Save changes** at the bottom of the page. + 1. Ensure the **Reset affected streams** option is checked. +```note +Depending on destination type you may not be prompted to reset your data. +``` +4. Select **Save connection**. +```note +This will reset the data in your destination and initiate a fresh sync. +``` + +For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). + + +## Upgrading to 2.0.0 + +This release updates the Source Google Ads connector so that its default streams and stream names match the related resources in [Google Ads API](https://developers.google.com/google-ads/api/fields/v14/ad_group_ad). + +Users should: +- Refresh the source schema +- And reset affected streams after upgrading to ensure uninterrupted syncs. + +## Upgrading to 1.0.0 + +This release introduced fixes to the creation of custom query schemas. For instance, the field ad_group_ad.ad.final_urls in the custom query has had its type changed from `{"type": "string"}` to `{"type": ["null", "array"], "items": {"type": "string"}}`. Users should refresh the source schema and reset affected streams after upgrading to ensure uninterrupted syncs. diff --git a/docs/integrations/sources/google-ads.inapp.md b/docs/integrations/sources/google-ads.inapp.md deleted file mode 100644 index ff2b18e3edc9..000000000000 --- a/docs/integrations/sources/google-ads.inapp.md +++ /dev/null @@ -1,72 +0,0 @@ -## Prerequisites - -- A [Google Ads Account](https://support.google.com/google-ads/answer/6366720) [linked](https://support.google.com/google-ads/answer/7459601) to a Google Ads Manager account - -- (For Airbyte Open Source): - - A Developer Token - - OAuth credentials to authenticate your Google account - - -## Setup guide - - - -To set up the Google Ads source connector with Airbyte Open Source, you will first need to obtain a developer token, as well as credentials for OAuth authentication. For more information on the steps involved, please refer to our [full documentation](https://docs.airbyte.com/integrations/sources/google-ads#setup-guide). - - - - -### For Airbyte Cloud: - -1. Enter a **Source name** of your choosing. -2. Click **Sign in with Google** to authenticate your Google Ads account. In the pop-up, select the appropriate Google account and click **Continue** to proceed. -3. Enter a comma-separated list of the **Customer ID(s)** for your account. These IDs are 10-digit numbers that uniquely identify your account. To find your Customer ID, please follow [Google's instructions](https://support.google.com/google-ads/answer/1704344). -4. Enter a **Start Date** using the provided datepicker, or by programmatically entering the date in YYYY-MM-DD format. The data added on and after this date will be replicated. -5. (Optional) You can use the **Custom GAQL Queries** field to enter a custom query using Google Ads Query Language. Click **Add** and enter your query, as well as the desired name of the table for this data in the destination. Multiple queries can be provided. For more information on formulating these queries, refer to our [guide below](#custom-query-understanding-google-ads-query-language). -6. (Required for Manager accounts) If accessing your account through a Google Ads Manager account, you must enter the [**Customer ID**](https://developers.google.com/google-ads/api/docs/concepts/call-structure#cid) of the Manager account. -7. (Optional) Enter a **Conversion Window**. This is the number of days after an ad interaction during which a conversion is recorded in Google Ads. For more information on this topic, refer to the [Google Ads Help Center](https://support.google.com/google-ads/answer/3123169?hl=en). This field defaults to 14 days. -8. (Optional) Enter an **End Date** in YYYY-MM-DD format. Any data added after this date will not be replicated. Leaving this field blank will replicate all data from the start date onward. -9. Click **Set up source** and wait for the tests to complete. - - - - -### For Airbyte Open Source: - -1. Enter a **Source name** of your choosing. -2. Enter the **Developer Token** you obtained from Google. -3. To authenticate your Google account, enter your Google application's **Client ID**, **Client Secret**, **Refresh Token**, and optionally, the **Access Token**. -4. Enter a comma-separated list of the **Customer ID(s)** for your account. These IDs are 10-digit numbers that uniquely identify your account. To find your Customer ID, please follow [Google's instructions](https://support.google.com/google-ads/answer/1704344). -5. Enter a **Start Date** using the provided datepicker, or by programmatically entering the date in YYYY-MM-DD format. The data added on and after this date will be replicated. -6. (Optional) You can use the **Custom GAQL Queries** field to enter a custom query using Google Ads Query Language. Click **Add** and enter your query, as well as the desired name of the table for this data in the destination. Multiple queries can be provided. For more information on formulating these queries, refer to our [guide below](#custom-query-understanding-google-ads-query-language). -7. (Required for Manager accounts) If accessing your account through a Google Ads Manager account, you must enter the [**Customer ID**](https://developers.google.com/google-ads/api/docs/concepts/call-structure#cid) of the Manager account. -8. (Optional) Enter a **Conversion Window**. This is the number of days after an ad interaction during which a conversion is recorded in Google Ads. For more information on this topic, refer to the [Google Ads Help Center](https://support.google.com/google-ads/answer/3123169?hl=en). This field defaults to 14 days. -9. (Optional) Enter an **End Date** in YYYY-MM-DD format. Any data added after this date will not be replicated. Leaving this field blank will replicate all data from the start date onward. -10. Click **Set up source** and wait for the tests to complete. - - - -## Custom Query: Understanding Google Ads Query Language -Additional streams for Google Ads can be dynamically created using custom queries. - -The Google Ads Query Language queries the Google Ads API. Review the [Google Ads Query Language](https://developers.google.com/google-ads/api/docs/query/overview) and the [query builder](https://developers.google.com/google-ads/api/fields/v13/query_validator) to validate your query. You can then add these as custom queries when configuring the Google Ads source. - -Example GAQL Custom Query: -``` -SELECT - campaign.name, - metrics.conversions, - metrics.conversions_by_conversion_date -FROM ad_group -``` -Note the segments.date is automatically added to the output, and does not need to be specified in the custom query. All custom reports will by synced by day. - -Each custom query in the input configuration must work for all the customer account IDs. Otherwise, the customer ID will be skipped for every query that fails the validation test. For example, if your query contains metrics fields in the select clause, it will not be executed against manager accounts. - -Follow Google's guidance on [Selectability between segments and metrics](https://developers.google.com/google-ads/api/docs/reporting/segmentation#selectability_between_segments_and_metrics) when editing custom queries or default stream schemas (which will also be turned into GAQL queries by the connector). Fields like `segments.keyword.info.text`, `segments.keyword.info.match_type`, `segments.keyword.ad_group_criterion` in the `SELECT` clause tell the query to only get the rows of data that have keywords and remove any row that is not associated with a keyword. This is often unobvious and undesired behavior and can lead to missing data records. If you need this field in the stream, add a new stream instead of editing the existing ones. - -:::info -For an existing Google Ads source, when you are updating or removing Custom GAQL Queries, you should also subsequently refresh your source schema to pull in any changes. -::: - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the [full documentation for Google Ads](https://docs.airbyte.com/integrations/sources/google-ads/). diff --git a/docs/integrations/sources/google-ads.md b/docs/integrations/sources/google-ads.md index 13b35b9277d9..da2195352073 100644 --- a/docs/integrations/sources/google-ads.md +++ b/docs/integrations/sources/google-ads.md @@ -43,6 +43,11 @@ If you are using Airbyte Open Source, you will need to obtain the following OAut Please refer to [Google's documentation](https://developers.google.com/identity/protocols/oauth2) for detailed instructions on how to obtain these credentials. +A single access token can grant varying degrees of access to multiple APIs. A variable parameter called scope controls the set of resources and operations that an access token permits. During the access token request, your app sends one or more values in the scope parameter. + +The scope for the Google Ads API is: https://www.googleapis.com/auth/adwords + +Each Google Ads API developer token is assigned an access level and "permissible use". The access level determines whether you can affect production accounts and the number of operations and requests that you can execute daily. Permissible use determines the specific Google Ads API features that the developer token is allowed to use. Read more about it and apply for higher access [here](https://developers.google.com/google-ads/api/docs/access-levels#access_levels_2). ### Step 3: Set up the Google Ads connector in Airbyte @@ -57,13 +62,14 @@ To set up Google Ads as a source in Airbyte Cloud: 3. Find and select **Google Ads** from the list of available sources. 4. Enter a **Source name** of your choosing. 5. Click **Sign in with Google** to authenticate your Google Ads account. In the pop-up, select the appropriate Google account and click **Continue** to proceed. -6. Enter a comma-separated list of the **Customer ID(s)** for your account. These IDs are 10-digit numbers that uniquely identify your account. To find your Customer ID, please follow [Google's instructions](https://support.google.com/google-ads/answer/1704344). -7. Enter a **Start Date** using the provided datepicker, or by programmatically entering the date in YYYY-MM-DD format. The data added on and after this date will be replicated. -8. (Optional) You can use the **Custom GAQL Queries** field to enter a custom query using Google Ads Query Language. Click **Add** and enter your query, as well as the desired name of the table for this data in the destination. Multiple queries can be provided. For more information on formulating these queries, refer to our [guide below](#custom-query-understanding-google-ads-query-language). -9. (Required for Manager accounts) If accessing your account through a Google Ads Manager account, you must enter the [**Customer ID**](https://developers.google.com/google-ads/api/docs/concepts/call-structure#cid) of the Manager account. -10. (Optional) Enter a **Conversion Window**. This is the number of days after an ad interaction during which a conversion is recorded in Google Ads. For more information on this topic, refer to the [Google Ads Help Center](https://support.google.com/google-ads/answer/3123169?hl=en). This field defaults to 14 days. -11. (Optional) Enter an **End Date** in YYYY-MM-DD format. Any data added after this date will not be replicated. Leaving this field blank will replicate all data from the start date onward. -12. Click **Set up source** and wait for the tests to complete. +6. (Optional) Enter a comma-separated list of the **Customer ID(s)** for your account. These IDs are 10-digit numbers that uniquely identify your account. To find your Customer ID, please follow [Google's instructions](https://support.google.com/google-ads/answer/1704344). Leaving this field blank will replicate data from all connected accounts. +7. (Optional) Enter customer statuses to filter customers. Leaving this field blank will replicate data from all accounts. Check [Google Ads documentation](https://developers.google.com/google-ads/api/reference/rpc/v15/CustomerStatusEnum.CustomerStatus) for more info. +8. (Optional) Enter a **Start Date** using the provided datepicker, or by programmatically entering the date in YYYY-MM-DD format. The data added on and after this date will be replicated. (Default start date is 2 years ago) +9. (Optional) You can use the **Custom GAQL Queries** field to enter a custom query using Google Ads Query Language. Click **Add** and enter your query, as well as the desired name of the table for this data in the destination. Multiple queries can be provided. For more information on formulating these queries, refer to our [guide below](#custom-query-understanding-google-ads-query-language). +10. (Required for Manager accounts) If accessing your account through a Google Ads Manager account, you must enter the [**Customer ID**](https://developers.google.com/google-ads/api/docs/concepts/call-structure#cid) of the Manager account. +11. (Optional) Enter a **Conversion Window**. This is the number of days after an ad interaction during which a conversion is recorded in Google Ads. For more information on this topic, refer to the [Google Ads Help Center](https://support.google.com/google-ads/answer/3123169?hl=en). This field defaults to 14 days. +12. (Optional) Enter an **End Date** in YYYY-MM-DD format. Any data added after this date will not be replicated. Leaving this field blank will replicate all data from the start date onward. +13. Click **Set up source** and wait for the tests to complete. @@ -78,17 +84,18 @@ To set up Google Ads as a source in Airbyte Open Source: 4. Enter a **Source name** of your choosing. 5. Enter the **Developer Token** you obtained from Google. 6. To authenticate your Google account, enter your Google application's **Client ID**, **Client Secret**, **Refresh Token**, and optionally, the **Access Token**. -7. Enter a comma-separated list of the **Customer ID(s)** for your account. These IDs are 10-digit numbers that uniquely identify your account. To find your Customer ID, please follow [Google's instructions](https://support.google.com/google-ads/answer/1704344). -8. Enter a **Start Date** using the provided datepicker, or by programmatically entering the date in YYYY-MM-DD format. The data added on and after this date will be replicated. -9. (Optional) You can use the **Custom GAQL Queries** field to enter a custom query using Google Ads Query Language. Click **Add** and enter your query, as well as the desired name of the table for this data in the destination. Multiple queries can be provided. For more information on formulating these queries, refer to our [guide below](#custom-query-understanding-google-ads-query-language). -10. (Required for Manager accounts) If accessing your account through a Google Ads Manager account, you must enter the [**Customer ID**](https://developers.google.com/google-ads/api/docs/concepts/call-structure#cid) of the Manager account. -11. (Optional) Enter a **Conversion Window**. This is the number of days after an ad interaction during which a conversion is recorded in Google Ads. For more information on this topic, see the section on [Conversion Windows](#note-on-conversion-windows) below, or refer to the [Google Ads Help Center](https://support.google.com/google-ads/answer/3123169?hl=en). This field defaults to 14 days. -12. (Optional) Enter an **End Date** in YYYY-MM-DD format. Any data added after this date will not be replicated. Leaving this field blank will replicate all data from the start date onward. -13. Click **Set up source** and wait for the tests to complete. +7. (Optional) Enter a comma-separated list of the **Customer ID(s)** for your account. These IDs are 10-digit numbers that uniquely identify your account. To find your Customer ID, please follow [Google's instructions](https://support.google.com/google-ads/answer/1704344). Leaving this field blank will replicate data from all connected accounts. +8. (Optional) Enter customer statuses to filter customers. Leaving this field blank will replicate data from all accounts. Check [Google Ads documentation](https://developers.google.com/google-ads/api/reference/rpc/v15/CustomerStatusEnum.CustomerStatus) for more info. +9. (Optional) Enter a **Start Date** using the provided datepicker, or by programmatically entering the date in YYYY-MM-DD format. The data added on and after this date will be replicated. (Default start date is 2 years ago) +10. (Optional) You can use the **Custom GAQL Queries** field to enter a custom query using Google Ads Query Language. Click **Add** and enter your query, as well as the desired name of the table for this data in the destination. Multiple queries can be provided. For more information on formulating these queries, refer to our [guide below](#custom-query-understanding-google-ads-query-language). +11. (Required for Manager accounts) If accessing your account through a Google Ads Manager account, you must enter the [**Customer ID**](https://developers.google.com/google-ads/api/docs/concepts/call-structure#cid) of the Manager account. +12. (Optional) Enter a **Conversion Window**. This is the number of days after an ad interaction during which a conversion is recorded in Google Ads. For more information on this topic, see the section on [Conversion Windows](#note-on-conversion-windows) below, or refer to the [Google Ads Help Center](https://support.google.com/google-ads/answer/3123169?hl=en). This field defaults to 14 days. +13. (Optional) Enter an **End Date** in YYYY-MM-DD format. Any data added after this date will not be replicated. Leaving this field blank will replicate all data from the start date onward. +14. Click **Set up source** and wait for the tests to complete. -## Supported sync modes +## Supported Sync Modes The Google Ads source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): @@ -97,58 +104,123 @@ The Google Ads source connector supports the following [sync modes](https://docs - [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) - [Incremental Sync - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped) +#### Incremental Events Streams +List of stream: +- [ad_group_criterions](https://developers.google.com/google-ads/api/fields/v15/ad_group_criterion) +- [ad_listing_group_criterions](https://developers.google.com/google-ads/api/fields/v15/ad_group_criterion) +- [campaign_criterion](https://developers.google.com/google-ads/api/fields/v15/campaign_criterion) + +These streams support incremental updates, including deletions, leveraging the Change Status stream. However, they only capture updates from the most recent three months. + +The initial sync operates as a full refresh. Subsequent syncs begin by reading updates from the Change Status stream, followed by syncing records based on their IDs. + +:::warning +It's important to note that the Google Ads API resource ChangeStatus has a limit of 10,000 records per request. That's why you cannot sync stream with more than 10,000 updates in a single microsecond. In such cases, it's recommended to use a full refresh sync to ensure all updates are captured. +::: ## Supported Streams The Google Ads source connector can sync the following tables. It can also sync custom queries using GAQL. ### Main Tables -- [accounts](https://developers.google.com/google-ads/api/fields/v11/customer) -- [ad_group_ads](https://developers.google.com/google-ads/api/fields/v11/ad_group_ad) -- [ad_group_ad_labels](https://developers.google.com/google-ads/api/fields/v11/ad_group_ad_label) -- [ad_groups](https://developers.google.com/google-ads/api/fields/v11/ad_group) -- [ad_group_labels](https://developers.google.com/google-ads/api/fields/v11/ad_group_label) -- [campaign_labels](https://developers.google.com/google-ads/api/fields/v11/campaign_label) -- [click_view](https://developers.google.com/google-ads/api/reference/rpc/v11/ClickView) -- [geographic](https://developers.google.com/google-ads/api/fields/v11/geographic_view) -- [keyword](https://developers.google.com/google-ads/api/fields/v11/keyword_view) +- [customer](https://developers.google.com/google-ads/api/fields/v15/customer) + +Highlights the setup and configurations of a Google Ads account. It encompasses features like call reporting and conversion tracking, giving a clear picture of the account's operational settings and features. +- [customer_label](https://developers.google.com/google-ads/api/fields/v15/customer_label) +- [campaign_criterion](https://developers.google.com/google-ads/api/fields/v15/campaign_criterion) + +Targeting option for a campaign, such as a keyword, placement, or audience. +- [campaign_bidding_strategy](https://developers.google.com/google-ads/api/fields/v15/campaign) + +Represents the bidding strategy at the campaign level. +- [campaign_label](https://developers.google.com/google-ads/api/fields/v15/campaign_label) +- [label](https://developers.google.com/google-ads/api/fields/v15/label) + +Represents labels that can be attached to different entities such as campaigns or ads. +- [ad_group_ad](https://developers.google.com/google-ads/api/fields/v15/ad_group_ad) + +Different attributes of ads from ad groups segmented by date. +- [ad_group_ad_label](https://developers.google.com/google-ads/api/fields/v15/ad_group_ad_label) +- [ad_group](https://developers.google.com/google-ads/api/fields/v15/ad_group) + +Represents an ad group within a campaign. Ad groups contain one or more ads which target a shared set of keywords. +- [ad_group_label](https://developers.google.com/google-ads/api/fields/v15/ad_group_label) +- [ad_group_bidding_strategy](https://developers.google.com/google-ads/api/fields/v15/ad_group) + +Represents the bidding strategy at the ad group level. +- [ad_group_criterion](https://developers.google.com/google-ads/api/fields/v15/ad_group_criterion) + +Represents criteria in an ad group, such as keywords or placements. +- [ad_listing_group_criterion](https://developers.google.com/google-ads/api/fields/v15/ad_group_criterion) + +Represents criteria for listing group ads. +- [ad_group_criterion_label](https://developers.google.com/google-ads/api/fields/v15/ad_group_criterion_label) +- [audience](https://developers.google.com/google-ads/api/fields/v15/audience) -Note that `ad_groups`, `ad_group_ads`, and `campaigns` contain a `labels` field, which should be joined against their respective `*_labels` streams if you want to view the actual labels. For example, the `ad_groups` stream contains an `ad_group.labels` field, which you would join against the `ad_group_labels` stream's `label.resource_name` field. +Represents user lists that are defined by the advertiser to target specific users. +- [user_interest](https://developers.google.com/google-ads/api/fields/v15/user_interest) +A particular interest-based vertical to be targeted. +- [click_view](https://developers.google.com/google-ads/api/reference/rpc/v15/ClickView) + +A click view with metrics aggregated at each click level, including both valid and invalid clicks. + +Note that `ad_group`, `ad_group_ad`, and `campaign` contain a `labels` field, which should be joined against their respective `*_label` streams if you want to view the actual labels. For example, the `ad_group` stream contains an `ad_group.labels` field, which you would join against the `ad_group_label` stream's `label.resource_name` field. ### Report Tables - [account_performance_report](https://developers.google.com/google-ads/api/docs/migration/mapping#account_performance) -- [ad_groups](https://developers.google.com/google-ads/api/fields/v14/ad_group) -- [ad_group_ad_report](https://developers.google.com/google-ads/api/docs/migration/mapping#ad_performance) -- [ad_group_criterions](https://developers.google.com/google-ads/api/fields/v14/ad_group_criterion) -- [ad_group_criterion_labels](https://developers.google.com/google-ads/api/fields/v14/ad_group_criterion_label) -- [campaigns](https://developers.google.com/google-ads/api/fields/v11/campaign) -- [campaign_budget](https://developers.google.com/google-ads/api/fields/v13/campaign_budget) -- [customer_labels](https://developers.google.com/google-ads/api/fields/v14/customer_label) -- [display_keyword_report](https://developers.google.com/google-ads/api/docs/migration/mapping#display_keyword_performance) -- [display_topics_report](https://developers.google.com/google-ads/api/docs/migration/mapping#display_topics_performance) -- [labels](https://developers.google.com/google-ads/api/fields/v14/label) -- [shopping_performance_report](https://developers.google.com/google-ads/api/docs/migration/mapping#shopping_performance) -- [user_location_report](https://developers.google.com/google-ads/api/fields/v11/user_location_view) + +Provides in-depth metrics related to ads interactions, including viewability, click-through rates, and conversions. Segments data by various factors, offering a granular look into how ads perform across different contexts. +- [campaign](https://developers.google.com/google-ads/api/fields/v15/campaign) + +Represents a campaign in Google Ads. +- [campaign_budget](https://developers.google.com/google-ads/api/fields/v15/campaign_budget) + +Represents the budget settings of a campaign. +- [geographic_view](https://developers.google.com/google-ads/api/fields/v15/geographic_view) + +Geographic View includes all metrics aggregated at the country level. It reports metrics at either actual physical location of the user or an area of interest. +- [user_location_view](https://developers.google.com/google-ads/api/fields/v15/user_location_view) + +User Location View includes all metrics aggregated at the country level. It reports metrics at the actual physical location of the user by targeted or not targeted location. +- [display_keyword_view](https://developers.google.com/google-ads/api/fields/v15/display_keyword_view) + +Metrics for display keywords, which are keywords that are targeted in display campaigns. +- [topic_view](https://developers.google.com/google-ads/api/fields/v15/topic_view) + +Reporting view that shows metrics aggregated by topic, which are broad categories of interests that users have. +- [shopping_performance_view](https://developers.google.com/google-ads/api/fields/v15/shopping_performance_view) + +Provides Shopping campaign statistics aggregated at several product dimension levels. Product dimension values from Merchant Center such as brand, category, custom attributes, product condition and product type will reflect the state of each dimension as of the date and time when the corresponding event was recorded. +- [keyword_view](https://developers.google.com/google-ads/api/fields/v15/keyword_view) + +Provides metrics related to the performance of keywords in the campaign. +- [ad_group_ad_legacy](https://developers.google.com/google-ads/api/fields/v15/ad_group_ad) + +Metrics and attributes of legacy ads from ad groups. :::note -Due to Google Ads API constraints, the `click_view` stream retrieves data one day at a time and can only retrieve data newer than 90 days ago. Also, [metrics](https://developers.google.com/google-ads/api/fields/v11/metrics) cannot be requested for a Google Ads Manager account. Report streams are only available when pulling data from a non-manager account. +Due to Google Ads API constraints, the `click_view` stream retrieves data one day at a time and can only retrieve data newer than 90 days ago. Also, [metrics](https://developers.google.com/google-ads/api/fields/v15/metrics) cannot be requested for a Google Ads Manager account. Report streams are only available when pulling data from a non-manager account. ::: :::warning -Google Ads doesn't support `PERFORMACE_MAX` campaigns on `ad_group` or `ad` stream level, only on `campaign` level. +Google Ads doesn't support `PERFORMANCE_MAX` campaigns on `ad_group` or `ad` stream level, only on `campaign` level. If you have this type of campaign Google will remove them from the results for the `ads` reports. More [info](https://github.com/airbytehq/airbyte/issues/11062) and [Google Discussions](https://groups.google.com/g/adwords-api/c/_mxbgNckaLQ). ::: -For incremental streams, data is synced up to the previous day using your Google Ads account time zone since Google Ads can filter data only by [date](https://developers.google.com/google-ads/api/fields/v11/ad_group_ad#segments.date) without time. Also, some reports cannot load data real-time due to Google Ads [limitations](https://support.google.com/google-ads/answer/2544985?hl=en). +For incremental streams, data is synced up to the previous day using your Google Ads account time zone since Google Ads can filter data only by [date](https://developers.google.com/google-ads/api/fields/v15/ad_group_ad#segments.date) without time. Also, some reports cannot load data real-time due to Google Ads [limitations](https://support.google.com/google-ads/answer/2544985?hl=en). + +### Reasoning Behind Primary Key Selection + +Primary keys are chosen to uniquely identify records within streams. In this selection, we considered the scope of ID uniqueness as detailed in [the Google Ads API structure documentation](https://developers.google.com/google-ads/api/docs/concepts/api-structure#object_ids). This approach guarantees that each record remains unique across various scopes and contexts. Moreover, in the Google Ads API, segmentation is crucial for dissecting performance data. As pointed out in [the Google Ads support documentation](https://developers.google.com/google-ads/api/docs/reporting/segmentation), segments offer a granular insight into data based on specific criteria, like device type or click interactions. ## Custom Query: Understanding Google Ads Query Language Additional streams for Google Ads can be dynamically created using custom queries. -The Google Ads Query Language queries the Google Ads API. Review the [Google Ads Query Language](https://developers.google.com/google-ads/api/docs/query/overview) and the [query builder](https://developers.google.com/google-ads/api/fields/v13/query_validator) to validate your query. You can then add these as custom queries when configuring the Google Ads source. +The Google Ads Query Language queries the Google Ads API. Review the [Google Ads Query Language](https://developers.google.com/google-ads/api/docs/query/overview) and the [query builder](https://developers.google.com/google-ads/api/fields/v15/query_validator) to validate your query. You can then add these as custom queries when configuring the Google Ads source. Example GAQL Custom Query: @@ -160,7 +232,7 @@ SELECT FROM ad_group ``` -Note the segments.date is automatically added to the output, and does not need to be specified in the custom query. All custom reports will by synced by day. +Note that `segments.date` is automatically added to the `WHERE` clause if it is included in the `SELECT` clause. Custom reports including `segments.date` in the `SELECT` clause will be synced by day. Each custom query in the input configuration must work for all the customer account IDs. Otherwise, the customer ID will be skipped for every query that fails the validation test. For example, if your query contains metrics fields in the select clause, it will not be executed against manager accounts. @@ -170,6 +242,23 @@ Follow Google's guidance on [Selectability between segments and metrics](https:/ For an existing Google Ads source, when you are updating or removing Custom GAQL Queries, you should also subsequently refresh your source schema to pull in any changes. ::: + +## Difference between manager and client accounts + +A manager account isn't an "upgrade" of your Google Ads account. Instead, it's an entirely new Google Ads account you create. Think of a manager account as an umbrella Google Ads account with several individual Google Ads accounts linked to it. You can link new and existing Google Ads accounts, as well as other manager accounts. + +You can then monitor ad performance, update campaigns, and manage other account tasks for those client accounts. Your manager account can also be given ownership of a client account. This allows you to manage user access for the client account. + +[Link](https://support.google.com/google-ads/answer/6139186?hl=en#) for more details on how it works and how you can create it. + +**Manager Accounts (MCC)** primarily focus on account management and oversight. They can access and manage multiple client accounts, view shared resources, and handle invitations to link with client accounts. + +**Client Accounts** are more operationally focused. They deal with campaign management, bidding, keywords, targeting, extensions, metrics, reporting, billing, and other ad-specific functionalities. + +While both types of accounts can access a wide range of resources in the API, the difference lies in their scope and purpose. Manager accounts have a broader oversight, while client accounts delve into the specifics of advertising operations. + +For detailed information, refer to the [official documentation.](https://developers.google.com/google-ads/api/fields/v15/overview) + ## Note on Conversion Windows In digital advertising, a 'conversion' typically refers to a user undertaking a desired action after viewing or interacting with an ad. This could be anything from clicking through to the advertiser's website, signing up for a newsletter, making a purchase, and so on. The conversion window is the period of time after a user sees or clicks on an ad during which their actions can still be credited to that ad. @@ -190,7 +279,27 @@ Due to a limitation in the Google Ads API which does not allow getting performan ## Changelog | Version | Date | Pull Request | Subject | -| :------- | :--------- | :------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------- | +|:---------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------| +| `3.3.1` | 2024-01-16 | [34007](https://github.com/airbytehq/airbyte/pull/34007) | prepare for airbyte-lib | +| `3.3.0` | 2024-01-12 | [34212](https://github.com/airbytehq/airbyte/pull/34212) | Remove metric from query in Ad Group stream for non-manager account | +| `3.2.1` | 2024-01-12 | [34200](https://github.com/airbytehq/airbyte/pull/34200) | Disable raising error for not enabled accounts | +| `3.2.0` | 2024-01-09 | [33707](https://github.com/airbytehq/airbyte/pull/33707) | Add possibility to sync all connected accounts | +| `3.1.0` | 2024-01-09 | [33603](https://github.com/airbytehq/airbyte/pull/33603) | Fix two issues in the custom queries: automatic addition of `segments.date` in the query; incorrect field type for `DATE` fields. | +| `3.0.2` | 2024-01-08 | [33494](https://github.com/airbytehq/airbyte/pull/33494) | Add handling for 401 error while parsing response. Add `metrics.cost_micros` field to Ad Group stream. | +| `3.0.1` | 2023-12-26 | [33769](https://github.com/airbytehq/airbyte/pull/33769) | Run a read function in a separate thread to enforce a time limit for its execution | +| `3.0.0` | 2023-12-07 | [33120](https://github.com/airbytehq/airbyte/pull/33120) | Upgrade API version to v15 | +| `2.0.4` | 2023-11-10 | [32414](https://github.com/airbytehq/airbyte/pull/32414) | Add backoff strategy for read_records method | +| `2.0.3` | 2023-11-02 | [32102](https://github.com/airbytehq/airbyte/pull/32102) | Fix incremental events streams | +| `2.0.2` | 2023-10-31 | [32001](https://github.com/airbytehq/airbyte/pull/32001) | Added handling (retry) for `InternalServerError` while reading the streams | +| `2.0.1` | 2023-10-27 | [31908](https://github.com/airbytehq/airbyte/pull/31908) | Base image migration: remove Dockerfile and use the python-connector-base image | +| `2.0.0` | 2023-10-04 | [31048](https://github.com/airbytehq/airbyte/pull/31048) | Fix schem default streams, change names of streams. | +| `1.0.0` | 2023-09-28 | [30705](https://github.com/airbytehq/airbyte/pull/30705) | Fix schemas for custom queries | +| `0.11.1` | 2023-09-26 | [30758](https://github.com/airbytehq/airbyte/pull/30758) | Exception should not be raises if a stream is not found | +| `0.11.0` | 2023-09-23 | [30704](https://github.com/airbytehq/airbyte/pull/30704) | Update error handling | +| `0.10.0` | 2023-09-19 | [30091](https://github.com/airbytehq/airbyte/pull/30091) | Fix schemas for correct primary and foreign keys | +| `0.9.0` | 2023-09-14 | [28970](https://github.com/airbytehq/airbyte/pull/28970) | Add incremental deletes for Campaign and Ad Group Criterion streams | +| `0.8.1` | 2023-09-13 | [30376](https://github.com/airbytehq/airbyte/pull/30376) | Revert pagination changes from 0.8.0 | +| `0.8.0` | 2023-09-01 | [30071](https://github.com/airbytehq/airbyte/pull/30071) | Delete start_date from required parameters and fix pagination | | `0.7.4` | 2023-07-28 | [28832](https://github.com/airbytehq/airbyte/pull/28832) | Update field descriptions | | `0.7.3` | 2023-07-24 | [28510](https://github.com/airbytehq/airbyte/pull/28510) | Set dates with client's timezone | | `0.7.2` | 2023-07-20 | [28535](https://github.com/airbytehq/airbyte/pull/28535) | UI improvement: Make the query field in custom reports a multi-line string field | diff --git a/docs/integrations/sources/google-analytics-data-api-migrations.md b/docs/integrations/sources/google-analytics-data-api-migrations.md new file mode 100644 index 000000000000..84ac3684a6dc --- /dev/null +++ b/docs/integrations/sources/google-analytics-data-api-migrations.md @@ -0,0 +1,25 @@ +# Google Analytics 4 (GA4) Migration Guide + +## Upgrading to 2.0.0 + +This version update only affects the schema of GA4 connections that sync more than one property. + +Version 2.0.0 prevents the duplication of stream names by renaming some property streams with a new stream name that includes the property ID. + + If you only are syncing from one property, no changes will occur when you upgrade to the new version. The stream names will continue to appear as: + - "daily_active_users", + - "weekly_active_users" + +If you are syncing more than one property, any property after the first will have the property ID appended to the stream name. + +For example, if your property IDs are: `0001`, `0002`, `0003`, the streams related to properties `0002` and `0003` will have the property ID appended to the end of the stream name. + - "daily_active_users", + - "daily_active_users_property_0002", + - "daily_active_users_property_0003", + - "weekly_active_users", + - "weekly_active_users_property_0002" + - "weekly_active_users_property_0003" + +If you are syncing more than one property ID, you will need to reset those streams to ensure syncing continues accurately. + +In the future, if you add an additional property ID, all new streams will append the property ID to the stream name without affecting existing streams. A reset is not required if you add the consecutive property after upgrading to 2.0.0. \ No newline at end of file diff --git a/docs/integrations/sources/google-analytics-data-api.md b/docs/integrations/sources/google-analytics-data-api.md index 857918a2bd6b..17b18cc3a460 100644 --- a/docs/integrations/sources/google-analytics-data-api.md +++ b/docs/integrations/sources/google-analytics-data-api.md @@ -29,8 +29,8 @@ For **Airbyte Cloud** users, we highly recommend using OAuth for authentication, If the Property Settings shows a "Tracking Id" such as "UA-123...-1", this denotes that the property is a Universal Analytics property, and the Analytics data for that property cannot be reported on using this connector. You can create a new Google Analytics 4 property by following [these instructions](https://support.google.com/analytics/answer/9744165?hl=en). ::: -7. In the **Start Date** field, use the provided datepicker or enter a date programmatically in the format `YYYY-MM-DD`. All data added from this date onward will be replicated. Note that this setting is _not_ applied to custom Cohort reports. -8. (Optional) In the **Custom Reports** field, you may optionally provide a JSON array describing any custom reports you want to sync from Google Analytics. See the [Custom Reports](#custom-reports) section below for more information on formulating these reports. +7. (Optional) In the **Start Date** field, use the provided datepicker or enter a date programmatically in the format `YYYY-MM-DD`. All data added from this date onward will be replicated. Note that this setting is _not_ applied to custom Cohort reports. +8. (Optional) In the **Custom Reports** field, you may optionally describe any custom reports you want to sync from Google Analytics. See the [Custom Reports](#custom-reports) section below for more information on formulating these reports. 9. (Optional) In the **Data Request Interval (Days)** field, you can specify the interval in days (ranging from 1 to 364) used when requesting data from the Google Analytics API. The bigger this value is, the faster the sync will be, but the more likely that sampling will be applied to your data, potentially causing inaccuracies in the returned results. We recommend setting this to 1 unless you have a hard requirement to make the sync faster at the expense of accuracy. This field does not apply to custom Cohort reports. See the [Data Sampling](#data-sampling-and-data-request-intervals) section below for more context on this field. :::caution @@ -68,6 +68,7 @@ Before you can use the service account to access Google Analytics data, you need 1. Go to the [Google Analytics Reporting API dashboard](https://console.developers.google.com/apis/api/analyticsreporting.googleapis.com/overview). Make sure you have selected the associated project for your service account, and enable the API. You can also set quotas and check usage. 2. Go to the [Google Analytics API dashboard](https://console.developers.google.com/apis/api/analytics.googleapis.com/overview). Make sure you have selected the associated project for your service account, and enable the API. +3. Go to the [Google Analytics Data API dashboard](https://console.developers.google.com/apis/api/analyticsdata.googleapis.com/overview). Make sure you have selected the associated project for your service account, and enable the API. #### Set up the Google Analytics connector in Airbyte @@ -80,9 +81,19 @@ Before you can use the service account to access Google Analytics data, you need If the Property Settings shows a "Tracking Id" such as "UA-123...-1", this denotes that the property is a Universal Analytics property, and the Analytics data for that property cannot be reported on in the Data API. You can create a new Google Analytics 4 property by following [these instructions](https://support.google.com/analytics/answer/9744165?hl=en). ::: -6. In the **Start Date** field, use the provided datepicker or enter a date programmatically in the format `YYYY-MM-DD`. All data added from this date onward will be replicated. Note that this setting is _not_ applied to custom Cohort reports. -7. (Optional) In the **Custom Reports** field, you may optionally provide a JSON array describing any custom reports you want to sync from Google Analytics. See the [Custom Reports](#custom-reports) section below for more information on formulating these reports. -8. (Optional) In the **Data Request Interval (Days)** field, you can specify the interval in days (ranging from 1 to 364) used when requesting data from the Google Analytics API. The bigger this value is, the faster the sync will be, but the more likely that sampling will be applied to your data, potentially causing inaccuracies in the returned results. We recommend setting this to 1 unless you have a hard requirement to make the sync faster at the expense of accuracy. This field does not apply to custom Cohort reports. See the [Data Sampling](#data-sampling-and-data-request-intervals) section below for more context on this field. +6. (Optional) In the **Start Date** field, use the provided datepicker or enter a date programmatically in the format `YYYY-MM-DD`. All data added from this date onward will be replicated. Note that this setting is _not_ applied to custom Cohort reports. + +:::note +If the start date is not provided, the default value will be used, which is two years from the initial sync. +::: + +:::caution +Many analyses and data investigations may require 24-48 hours to process information from your website or app. To ensure the accuracy of the data, we subtract two days from the starting date. For more details, please refer to [Google's documentation](https://support.google.com/analytics/answer/9333790?hl=en). +::: + +7. (Optional) Toggle the switch **Keep Empty Rows** if you want each row with all metrics equal to 0 to be returned. +8. (Optional) In the **Custom Reports** field, you may optionally describe any custom reports you want to sync from Google Analytics. See the [Custom Reports](#custom-reports) section below for more information on formulating these reports. +9. (Optional) In the **Data Request Interval (Days)** field, you can specify the interval in days (ranging from 1 to 364) used when requesting data from the Google Analytics API. The bigger this value is, the faster the sync will be, but the more likely that sampling will be applied to your data, potentially causing inaccuracies in the returned results. We recommend setting this to 1 unless you have a hard requirement to make the sync faster at the expense of accuracy. This field does not apply to custom Cohort reports. See the [Data Sampling](#data-sampling-and-data-request-intervals) section below for more context on this field. :::caution @@ -117,6 +128,55 @@ This connector outputs the following incremental streams: - [traffic_sources](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) - [website_overview](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) - [weekly_active_users](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [user_acquisition_first_user_medium_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [user_acquisition_first_user_source_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [user_acquisition_first_user_source_medium_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [user_acquisition_first_user_source_platform_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [user_acquisition_first_user_campaign_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [user_acquisition_first_user_google_ads_ad_network_type_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [user_acquisition_first_user_google_ads_ad_group_name_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [traffic_acquisition_session_source_medium_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [traffic_acquisition_session_medium_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [traffic_acquisition_session_source_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [traffic_acquisition_session_campaign_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [traffic_acquisition_session_default_channel_grouping_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [traffic_acquisition_session_source_platform_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [events_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [weekly_events_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [conversions_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [pages_title_and_screen_class_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [pages_path_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [pages_title_and_screen_name_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [content_group_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [ecommerce_purchases_item_name_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [ecommerce_purchases_item_id_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [ecommerce_purchases_item_category_report_combined](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [ecommerce_purchases_item_category_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [ecommerce_purchases_item_category_2_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [ecommerce_purchases_item_category_3_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [ecommerce_purchases_item_category_4_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [ecommerce_purchases_item_category_5_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [ecommerce_purchases_item_brand_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [publisher_ads_ad_unit_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [publisher_ads_page_path_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [publisher_ads_ad_format_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [publisher_ads_ad_source_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [demographic_country_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [demographic_region_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [demographic_city_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [demographic_language_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [demographic_age_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [demographic_gender_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [demographic_interest_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [tech_browser_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [tech_device_category_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [tech_device_model_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [tech_screen_resolution_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [tech_app_version_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [tech_platform_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [tech_platform_device_category_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [tech_operating_system_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) + - [tech_os_with_version_report](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) - [Custom stream\(s\)](https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/properties/runReport) ## Connector-specific features @@ -133,19 +193,6 @@ Custom reports in Google Analytics allow for flexibility in querying specific da A full list of dimensions and metrics supported in the API can be found [here](https://developers.google.com/analytics/devguides/reporting/data/v1/api-schema). To ensure your dimensions and metrics are compatible for your GA4 property, you can use the [GA4 Dimensions & Metrics Explorer](https://ga-dev-tools.google/ga4/dimensions-metrics-explorer/). -Custom reports should be constructed as an array of JSON objects in the following format: - -```json -[ - { - "name": "", - "dimensions": ["", ...], - "metrics": ["", ...], - "cohortSpec": {/* cohortSpec object */}, - "pivots": [{/* pivot object */}, ...] - } -] -``` The following is an example of a basic User Engagement report to track sessions and bounce rate, segmented by city: @@ -206,34 +253,50 @@ The Google Analytics connector is subject to Google Analytics Data API quotas. P ## Data type map -| Integration Type | Airbyte Type | Notes | -| :--------------- | :----------- | :---- | -| `string` | `string` | | -| `number` | `number` | | -| `array` | `array` | | -| `object` | `object` | | +| Integration Type | Airbyte Type | +|:-----------------|:-------------| +| `string` | `string` | +| `number` | `number` | +| `array` | `array` | +| `object` | `object` | ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------| -| 1.1.3 | 2023-08-04 | [29103](https://github.com/airbytehq/airbyte/pull/29103) | Update input field descriptions | -| 1.1.2 | 2023-07-03 | [27909](https://github.com/airbytehq/airbyte/pull/27909) | Limit the page size of custom report streams | -| 1.1.1 | 2023-06-26 | [27718](https://github.com/airbytehq/airbyte/pull/27718) | Limit the page size when calling `check()` | -| 1.1.0 | 2023-06-26 | [27738](https://github.com/airbytehq/airbyte/pull/27738) | License Update: Elv2 | -| 1.0.0 | 2023-06-22 | [26283](https://github.com/airbytehq/airbyte/pull/26283) | Added primary_key and lookback window | -| 0.2.7 | 2023-06-21 | [27531](https://github.com/airbytehq/airbyte/pull/27531) | Fix formatting | -| 0.2.6 | 2023-06-09 | [27207](https://github.com/airbytehq/airbyte/pull/27207) | Improve api rate limit messages | -| 0.2.5 | 2023-06-08 | [27175](https://github.com/airbytehq/airbyte/pull/27175) | Improve Error Messages | -| 0.2.4 | 2023-06-01 | [26887](https://github.com/airbytehq/airbyte/pull/26887) | Remove `authSpecification` from connector spec in favour of `advancedAuth` | -| 0.2.3 | 2023-05-16 | [26126](https://github.com/airbytehq/airbyte/pull/26126) | Fix pagination | -| 0.2.2 | 2023-05-12 | [25987](https://github.com/airbytehq/airbyte/pull/25987) | Categorized Config Errors Accurately | -| 0.2.1 | 2023-05-11 | [26008](https://github.com/airbytehq/airbyte/pull/26008) | Added handling for `429 - potentiallyThresholdedRequestsPerHour` error | -| 0.2.0 | 2023-04-13 | [25179](https://github.com/airbytehq/airbyte/pull/25179) | Implement support for custom Cohort and Pivot reports | -| 0.1.3 | 2023-03-10 | [23872](https://github.com/airbytehq/airbyte/pull/23872) | Fix parse + cursor for custom reports | -| 0.1.2 | 2023-03-07 | [23822](https://github.com/airbytehq/airbyte/pull/23822) | Improve `rate limits` customer faced error messages and retry logic for `429` | -| 0.1.1 | 2023-01-10 | [21169](https://github.com/airbytehq/airbyte/pull/21169) | Slicer updated, unit tests added | -| 0.1.0 | 2023-01-08 | [20889](https://github.com/airbytehq/airbyte/pull/20889) | Improved config validation, SAT | -| 0.0.3 | 2022-08-15 | [15229](https://github.com/airbytehq/airbyte/pull/15229) | Source Google Analytics Data Api: code refactoring | -| 0.0.2 | 2022-07-27 | [15087](https://github.com/airbytehq/airbyte/pull/15087) | fix documentationUrl | -| 0.0.1 | 2022-05-09 | [12701](https://github.com/airbytehq/airbyte/pull/12701) | Introduce Google Analytics Data API source | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------| +| 2.2.1 | 2024-01-18 | [34352](https://github.com/airbytehq/airbyte/pull/34352) | Add incorrect custom reports config handling | +| 2.2.0 | 2024-01-10 | [34176](https://github.com/airbytehq/airbyte/pull/34176) | Add a report option keepEmptyRows | +| 2.1.1 | 2024-01-08 | [34018](https://github.com/airbytehq/airbyte/pull/34018) | prepare for airbyte-lib | +| 2.1.0 | 2023-12-28 | [33802](https://github.com/airbytehq/airbyte/pull/33802) | Add `CohortSpec` to custom report in specification | +| 2.0.3 | 2023-11-03 | [32149](https://github.com/airbytehq/airbyte/pull/32149) | Fixed bug with missing `metadata` when the credentials are not valid | +| 2.0.2 | 2023-11-02 | [32094](https://github.com/airbytehq/airbyte/pull/32094) | Added handling for `JSONDecodeError` while checking for `api qouta` limits | +| 2.0.1 | 2023-10-18 | [31543](https://github.com/airbytehq/airbyte/pull/31543) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 2.0.0 | 2023-09-29 | [30930](https://github.com/airbytehq/airbyte/pull/30930) | Use distinct stream naming in case there are multiple properties in the config. | +| 1.6.0 | 2023-09-19 | [30460](https://github.com/airbytehq/airbyte/pull/30460) | Migrated custom reports from string to array; add `FilterExpressions` support | +| 1.5.1 | 2023-09-20 | [30608](https://github.com/airbytehq/airbyte/pull/30608) | Revert `:` auto replacement name to underscore | +| 1.5.0 | 2023-09-18 | [30421](https://github.com/airbytehq/airbyte/pull/30421) | Add `yearWeek`, `yearMonth`, `year` dimensions cursor | +| 1.4.1 | 2023-09-17 | [30506](https://github.com/airbytehq/airbyte/pull/30506) | Fix None type error when metrics or dimensions response does not have name | +| 1.4.0 | 2023-09-15 | [30417](https://github.com/airbytehq/airbyte/pull/30417) | Change start date to optional; add suggested streams and update errors handling | +| 1.3.1 | 2023-09-14 | [30424](https://github.com/airbytehq/airbyte/pull/30424) | Fixed duplicated stream issue | +| 1.3.0 | 2023-09-13 | [30152](https://github.com/airbytehq/airbyte/pull/30152) | Ability to add multiple property ids | +| 1.2.0 | 2023-09-11 | [30290](https://github.com/airbytehq/airbyte/pull/30290) | Add new preconfigured reports | +| 1.1.3 | 2023-08-04 | [29103](https://github.com/airbytehq/airbyte/pull/29103) | Update input field descriptions | +| 1.1.2 | 2023-07-03 | [27909](https://github.com/airbytehq/airbyte/pull/27909) | Limit the page size of custom report streams | +| 1.1.1 | 2023-06-26 | [27718](https://github.com/airbytehq/airbyte/pull/27718) | Limit the page size when calling `check()` | +| 1.1.0 | 2023-06-26 | [27738](https://github.com/airbytehq/airbyte/pull/27738) | License Update: Elv2 | +| 1.0.0 | 2023-06-22 | [26283](https://github.com/airbytehq/airbyte/pull/26283) | Added primary_key and lookback window | +| 0.2.7 | 2023-06-21 | [27531](https://github.com/airbytehq/airbyte/pull/27531) | Fix formatting | +| 0.2.6 | 2023-06-09 | [27207](https://github.com/airbytehq/airbyte/pull/27207) | Improve api rate limit messages | +| 0.2.5 | 2023-06-08 | [27175](https://github.com/airbytehq/airbyte/pull/27175) | Improve Error Messages | +| 0.2.4 | 2023-06-01 | [26887](https://github.com/airbytehq/airbyte/pull/26887) | Remove `authSpecification` from connector spec in favour of `advancedAuth` | +| 0.2.3 | 2023-05-16 | [26126](https://github.com/airbytehq/airbyte/pull/26126) | Fix pagination | +| 0.2.2 | 2023-05-12 | [25987](https://github.com/airbytehq/airbyte/pull/25987) | Categorized Config Errors Accurately | +| 0.2.1 | 2023-05-11 | [26008](https://github.com/airbytehq/airbyte/pull/26008) | Added handling for `429 - potentiallyThresholdedRequestsPerHour` error | +| 0.2.0 | 2023-04-13 | [25179](https://github.com/airbytehq/airbyte/pull/25179) | Implement support for custom Cohort and Pivot reports | +| 0.1.3 | 2023-03-10 | [23872](https://github.com/airbytehq/airbyte/pull/23872) | Fix parse + cursor for custom reports | +| 0.1.2 | 2023-03-07 | [23822](https://github.com/airbytehq/airbyte/pull/23822) | Improve `rate limits` customer faced error messages and retry logic for `429` | +| 0.1.1 | 2023-01-10 | [21169](https://github.com/airbytehq/airbyte/pull/21169) | Slicer updated, unit tests added | +| 0.1.0 | 2023-01-08 | [20889](https://github.com/airbytehq/airbyte/pull/20889) | Improved config validation, SAT | +| 0.0.3 | 2022-08-15 | [15229](https://github.com/airbytehq/airbyte/pull/15229) | Source Google Analytics Data Api: code refactoring | +| 0.0.2 | 2022-07-27 | [15087](https://github.com/airbytehq/airbyte/pull/15087) | fix documentationUrl | +| 0.0.1 | 2022-05-09 | [12701](https://github.com/airbytehq/airbyte/pull/12701) | Introduce Google Analytics Data API source | \ No newline at end of file diff --git a/docs/integrations/sources/google-analytics-v4.inapp.md b/docs/integrations/sources/google-analytics-v4.inapp.md deleted file mode 100644 index 305fd218af5a..000000000000 --- a/docs/integrations/sources/google-analytics-v4.inapp.md +++ /dev/null @@ -1,101 +0,0 @@ -:::caution - -**The Google Analytics (Universal Analytics) connector will be deprecated soon.** - -Google is phasing out Universal Analytics in favor of Google Analytics 4 (GA4). In consequence, we are deprecating the Google Analytics (Universal Analytics) connector and recommend that you migrate to the [Google Analytics 4 (GA4) connector](https://docs.airbyte.com/integrations/sources/google-analytics-data-api) as soon as possible to ensure your syncs are not affected. - -Due to this deprecation, we will not be accepting new contributions for this source. - -For more information, see ["Universal Analytics is going away"](https://support.google.com/analytics/answer/11583528). - -::: - -## Prerequisite - -* Administrator access to a Google Analytics 4 (GA4) property - -## Setup guide - -1. Click **Authenticate your account** by selecting Oauth (recommended). - * If you select Service Account Key Authentication, follow the instructions in our [full documentation](https://docs.airbyte.com/integrations/sources/google-analytics-v4). -2. Log in and Authorize the Google Analytics account. -3. Enter your [Property ID](https://developers.google.com/analytics/devguides/reporting/data/v1/property-id#what_is_my_property_id) -4. Enter the **Start Date** from which to replicate report data in the format YYYY-MM-DD. -5. (Optional) Airbyte generates 8 default reports. To add more reports, you need to add **Custom Reports** as a JSON array describing the custom reports you want to sync from Google Analytics. See below for more information. -6. (Optional) Enter the **Data request time increment in days**. The bigger this value is, the faster the sync will be, but the more likely that sampling will be applied to your data, potentially causing inaccuracies in the returned results. We recommend setting this to 1 unless you have a hard requirement to make the sync faster at the expense of accuracy. The minimum allowed value for this field is 1, and the maximum is 364. - -## (Optional) Custom Reports -Custom Reports allow for flexibility in the reporting dimensions and metrics to meet your specific use case. Use the [GA4 Query Explorer](https://ga-dev-tools.google/ga4/query-explorer/) to help build your report. To ensure your dimensions and metrics are compatible, you can also refer to the [GA4 Dimensions & Metrics Explorer](https://ga-dev-tools.google/ga4/dimensions-metrics-explorer/). - -A custom report is formatted as: `[{"name": "", "dimensions": ["", ...], "metrics": ["", ...]}]` - -Example of a custom report: -``` -[{ - "name" : "page_views_and_users", - "dimensions" :[ - "ga:date", - "ga:pagePath", - "ga:sessionDefaultChannelGrouping" - ], - "metrics" :[ - "ga:screenPageViews", - "ga:totalUsers" - ] -}] -``` -Multiple custom reports should be entered with a comma separator. Each custom report is created as it's own stream. -Example of multiple custom reports: -``` -[ - { - "name" : "page_views_and_users", - "dimensions" :[ - "ga:date", - "ga:pagePath" - ], - "metrics" :[ - "ga:screenPageViews", - "ga:totalUsers" - ] - }, - { - "name" : "sessions_by_region", - "dimensions" :[ - "ga:date", - "ga:region" - ], - "metrics" :[ - "ga:totalUsers", - "ga:sessions" - ] - } -] -``` - -Custom reports can also include segments and filters to pull a subset of your data. The report should be formatted as: `[{"name": "", "dimensions": ["", ...], "metrics": ["", ...], "segments": [""}]` - -* When using segments, make sure you also add the `ga:segment` dimension. - -Example of a custom report with segments and/or filters: -``` -[{ "name" : "page_views_and_users", - "dimensions" :[ - "ga:date", - "ga:pagePath", - "ga:segment" - ], - "metrics" :[ - "ga:sessions", - "ga:totalUsers" - ], - "segments" :[ - "ga:sessionSource!=(direct)" - ], - "filter" :[ - "ga:sessionSource!=(direct);ga:sessionSource!=(not set)" - ] -}] -``` - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Google Analytics 4 (GA4)](https://docs.airbyte.com/integrations/sources/google-analytics-v4). diff --git a/docs/integrations/sources/google-analytics-v4.md b/docs/integrations/sources/google-analytics-v4.md index 2b32a9d7d7a6..48b65cbae456 100644 --- a/docs/integrations/sources/google-analytics-v4.md +++ b/docs/integrations/sources/google-analytics-v4.md @@ -1,9 +1,13 @@ # Google Analytics (Universal Analytics) + + This page contains the setup guide and reference information for the Google Analytics (Universal Analytics) source connector. This connector supports Universal Analytics properties through the [Reporting API v4](https://developers.google.com/analytics/devguides/reporting/core/v4). + + :::caution **The Google Analytics (Universal Analytics) connector will be deprecated soon.** @@ -34,8 +38,6 @@ A Google Cloud account with [Viewer permissions](https://support.google.com/anal **For Airbyte Cloud:** -To set up Google Analytics as a source in Airbyte Cloud: - 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. 3. On the Set up the source page, select **Google Analytics** from the **Source type** dropdown. @@ -52,19 +54,20 @@ To set up Google Analytics as a source in Airbyte Cloud: **For Airbyte Open Source:** -To set up Google Analytics as a source in Airbyte Open Source: - -1. Go to the Airbyte UI and click **Sources** and then click **+ New source**. -2. On the Set up the source page, select **Google Analytics** from the **Source type** dropdown. -3. Enter a name for the Google Analytics connector. -4. Authenticate your Google account via OAuth or Service Account Key Authentication: +1. Navigate to the Airbyte Open Source dashboard. +2. Go to the Airbyte UI and click **Sources** and then click **+ New source**. +3. On the Set up the source page, select **Google Analytics** from the **Source type** dropdown. +4. Enter a name for the Google Analytics connector. +5. Authenticate your Google account via OAuth or Service Account Key Authentication: - To authenticate your Google account via OAuth, enter your Google application's [client ID, client secret, and refresh token](https://developers.google.com/identity/protocols/oauth2). - To authenticate your Google account via Service Account Key Authentication, enter your [Google Cloud service account key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys#creating_service_account_keys) in JSON format. Use the service account email address to [add a user](https://support.google.com/analytics/answer/1009702) to the Google analytics view you want to access via the API and grant [Read and Analyze permissions](https://support.google.com/analytics/answer/2884495). 5. Enter the **Replication Start Date** in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. + 6. Enter the [**View ID**](https://ga-dev-tools.appspot.com/account-explorer/) for the Google Analytics View you want to fetch data from. 7. Optionally, enter a JSON object as a string in the **Custom Reports** field. For details, refer to [Requesting custom reports](#requesting-custom-reports) 8. Leave **Data request time increment in days (Optional)** blank or set to 1. For faster syncs, set this value to more than 1 but that might result in the Google Analytics API returning [sampled data](#sampled-data-in-reports), potentially causing inaccuracies in the returned results. The maximum allowed value is 364. - + + ## Supported sync modes @@ -86,7 +89,7 @@ You need to add the service account email address on the account level, not the The Google Analytics (Universal Analytics) source connector can sync the following tables: | Stream name | Schema | -| :----------------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +|:-------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | website_overview | `{"ga_date":"2021-02-11","ga_users":1,"ga_newUsers":0,"ga_sessions":9,"ga_sessionsPerUser":9.0,"ga_avgSessionDuration":28.77777777777778,"ga_pageviews":63,"ga_pageviewsPerSession":7.0,"ga_avgTimeOnPage":4.685185185185185,"ga_bounceRate":0.0,"ga_exitRate":14.285714285714285,"view_id":"211669975"}` | | traffic_sources | `{"ga_date":"2021-02-11","ga_source":"(direct)","ga_medium":"(none)","ga_socialNetwork":"(not set)","ga_users":1,"ga_newUsers":0,"ga_sessions":9,"ga_sessionsPerUser":9.0,"ga_avgSessionDuration":28.77777777777778,"ga_pageviews":63,"ga_pageviewsPerSession":7.0,"ga_avgTimeOnPage":4.685185185185185,"ga_bounceRate":0.0,"ga_exitRate":14.285714285714285,"view_id":"211669975"}` | | pages | `{"ga_date":"2021-02-11","ga_hostname":"mydemo.com","ga_pagePath":"/home5","ga_pageviews":63,"ga_uniquePageviews":9,"ga_avgTimeOnPage":4.685185185185185,"ga_entrances":9,"ga_entranceRate":14.285714285714285,"ga_bounceRate":0.0,"ga_exits":9,"ga_exitRate":14.285714285714285,"view_id":"211669975"}` | @@ -101,7 +104,7 @@ The Google Analytics (Universal Analytics) source connector can sync the followi Reach out to us on Slack or [create an issue](https://github.com/airbytehq/airbyte/issues) if you need to send custom Google Analytics report data with Airbyte. -## Rate Limits and Performance Considerations \(Airbyte Open-Source\) +## Rate Limits and Performance Considerations \(Airbyte Open Source\) [Analytics Reporting API v4](https://developers.google.com/analytics/devguides/reporting/core/v4/limits-quotas) @@ -119,28 +122,90 @@ If you are not on the Google Analytics 360 tier, the Google Analytics API may re In order to minimize the chances of sampling being applied to your data, Airbyte makes data requests to Google in one day increments (the smallest allowed date increment). This reduces the amount of data the Google API processes per request, thus minimizing the chances of sampling being applied. The downside of requesting data in one day increments is that it increases the time it takes to export your Google Analytics data. If sampling is not a concern, you can override this behavior by setting the optional `window_in_day` parameter to specify the number of days to look back and avoid sampling. When sampling occurs, a warning is logged to the sync log. -## Data processing latency - -According to the [Google Analytics API documentation](https://support.google.com/analytics/answer/1070983?hl=en#DataProcessingLatency&zippy=%2Cin-this-article), all report data may continue to be updated 48 hours after it appears in the Google Analytics API. This means if you request the same report twice within 48 hours of that data being sent to Google Analytics, the report data might be different across the two requests. This happens when Google Analytics is still processing all events it received. - -When this occurs, the returned data will set the flag `isDataGolden` to false. As mentioned in the [Google Analytics API docs](https://developers.google.com/analytics/devguides/reporting/core/v4/rest/v4/reports/batchGet#reportdata), the `isDataGolden` flag indicates if [data] is golden or not. Data is golden when the exact same request [for a report] will not produce any new results if asked at a later point in time. - -To address this issue, the connector adds a lookback window of 2 days to ensure any previously synced non-golden data is re-synced with its potential updates. For example: If your last sync occurred 5 days ago and a sync is initiated today, the connector will attempt to sync data from 7 days ago up to the latest data available. - -To determine whether data is finished processing or not, the `isDataGolden` flag is exposed and should be used. - ## Requesting Custom Reports -To replicate Google Analytics [Custom Reports](https://support.google.com/analytics/answer/1033013?hl=en) using this connector, input a JSON object as a string in the **Custom Reports** field when setting up the connector. The JSON is an array of objects where each object has the following schema: - -```text -{"name": string, "dimensions": [string], "metrics": [string]} +Custom Reports allow for flexibility in the reporting dimensions and metrics to meet your specific use case. Use the [GA4 Query Explorer](https://ga-dev-tools.google/ga4/query-explorer/) to help build your report. To ensure your dimensions and metrics are compatible, you can also refer to the [GA4 Dimensions & Metrics Explorer](https://ga-dev-tools.google/ga4/dimensions-metrics-explorer/). + +A custom report is formatted as: `[{"name": "", "dimensions": ["", ...], "metrics": ["", ...]}]` + +Example of a custom report: +```json +[{ + "name" : "page_views_and_users", + "dimensions" :[ + "ga:date", + "ga:pagePath", + "ga:sessionDefaultChannelGrouping" + ], + "metrics" :[ + "ga:screenPageViews", + "ga:totalUsers" + ] +}] +``` +Multiple custom reports should be entered with a comma separator. Each custom report is created as it's own stream. +Example of multiple custom reports: +```json +[ + { + "name" : "page_views_and_users", + "dimensions" :[ + "ga:date", + "ga:pagePath" + ], + "metrics" :[ + "ga:screenPageViews", + "ga:totalUsers" + ] + }, + { + "name" : "sessions_by_region", + "dimensions" :[ + "ga:date", + "ga:region" + ], + "metrics" :[ + "ga:totalUsers", + "ga:sessions" + ] + } +] ``` -Here is an example input "Custom Reports" field: +Custom reports can also include segments and filters to pull a subset of your data. The report should be formatted as: +```json +[ + { + "name": "", + "dimensions": ["", ...], + "metrics": ["", ...], + "segments": ["", ...], + "filter": "" + } +] +``` -```text -[{"name": "new_users_per_day", "dimensions": ["ga:date","ga:country","ga:region"], "metrics": ["ga:newUsers"]}, {"name": "users_per_city", "dimensions": ["ga:city"], "metrics": ["ga:users"]}] +* When using segments, make sure you also add the `ga:segment` dimension. + +Example of a custom report with segments and/or filters: +```json +[{ "name" : "page_views_and_users", + "dimensions" :[ + "ga:date", + "ga:pagePath", + "ga:segment" + ], + "metrics" :[ + "ga:sessions", + "ga:totalUsers" + ], + "segments" :[ + "ga:sessionSource!=(direct)" + ], + "filter" :[ + "ga:sessionSource!=(direct);ga:sessionSource!=(not set)" + ] +}] ``` To create a list of dimensions, you can use default Google Analytics dimensions (listed below) or custom dimensions if you have some defined. Each report can contain no more than 7 dimensions, and they must all be unique. The default Google Analytics dimensions are: @@ -186,10 +251,40 @@ A custom report can contain no more than 10 unique metrics. The default availabl Incremental sync is supported only if you add `ga:date` dimension to your custom report. +## Limitations & Troubleshooting + +
      + +Expand to see details about Google Analytics v4 connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting + +[Analytics Reporting API v4](https://developers.google.com/analytics/devguides/reporting/core/v4/limits-quotas) + +- Number of requests per day per project: 50,000 +- Number of requests per view (profile) per day: 10,000 (cannot be increased) +- Number of requests per 100 seconds per project: 2,000 +- Number of requests per 100 seconds per user per project: 100 (can be increased in Google API Console to 1,000). + +The Google Analytics connector should not run into the "requests per 100 seconds" limitation under normal usage. [Create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully and try increasing the `window_in_days` value. + +### Troubleshooting + + + +* Check out common troubleshooting issues for the Google Analytics v4 source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +
      + ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------- | +|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------| +| 0.2.3 | 2024-01-18 | [34353](https://github.com/airbytehq/airbyte/pull/34353) | Add End date option | +| 0.2.2 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | | 0.2.1 | 2023-07-11 | [28149](https://github.com/airbytehq/airbyte/pull/28149) | Specify date format to support datepicker in UI | | 0.2.0 | 2023-06-26 | [27738](https://github.com/airbytehq/airbyte/pull/27738) | License Update: Elv2 | | 0.1.36 | 2023-04-13 | [22223](https://github.com/airbytehq/airbyte/pull/22223) | Fix custom report with Segments dimensions | @@ -227,3 +322,5 @@ Incremental sync is supported only if you add `ga:date` dimension to your custom | 0.1.2 | 2021-09-20 | [6306](https://github.com/airbytehq/airbyte/pull/6306) | Support of Airbyte OAuth initialization flow | | 0.1.1 | 2021-08-25 | [5655](https://github.com/airbytehq/airbyte/pull/5655) | Corrected validation of empty custom report | | 0.1.0 | 2021-08-10 | [5290](https://github.com/airbytehq/airbyte/pull/5290) | Initial Release | + +
      \ No newline at end of file diff --git a/docs/integrations/sources/google-directory.md b/docs/integrations/sources/google-directory.md index b0e570f7544f..d263d9efc93e 100644 --- a/docs/integrations/sources/google-directory.md +++ b/docs/integrations/sources/google-directory.md @@ -40,7 +40,7 @@ This connector attempts to back off gracefully when it hits Directory API's rate 1. Click `OAuth2.0 authorization` then `Authenticate your Google Directory account`. 2. You're done. -## Getting Started \(Airbyte Open-Source\) +## Getting Started \(Airbyte Open Source\) Google APIs use the OAuth 2.0 protocol for authentication and authorization. This connector supports [Web server application](https://developers.google.com/identity/protocols/oauth2#webserver) and [Service accounts](https://developers.google.com/identity/protocols/oauth2#serviceaccount) scenarios. Therefore, there are 2 options of setting up authorization for this source: diff --git a/docs/integrations/sources/google-drive.md b/docs/integrations/sources/google-drive.md new file mode 100644 index 000000000000..cc0a92099b4e --- /dev/null +++ b/docs/integrations/sources/google-drive.md @@ -0,0 +1,256 @@ +# Google Drive + +This page contains the setup guide and reference information for the Google Drive source connector. + +:::info +The Google Drive source connector pulls data from a single folder in Google Drive. Subfolders are recursively included in the sync. All files in the specified folder and all sub folders will be considered. +::: + +## Prerequisites + +- Drive folder link - The link to the Google Drive folder you want to sync files from (includes files located in subfolders) + +- **For Airbyte Cloud** A Google Workspace user with access to the spreadsheet + + +- **For Airbyte Open Source:** + - A GCP project + - Enable the Google Drive API in your GCP project + - Service Account Key with access to the Spreadsheet you want to replicate + + +## Setup guide + +The Google Drive source connector supports authentication via either OAuth or Service Account Key Authentication. + +For **Airbyte Cloud** users, we highly recommend using OAuth, as it significantly simplifies the setup process and allows you to authenticate [directly from the Airbyte UI](#set-up-the-google-drive-source-connector-in-airbyte). + + + + + +For **Airbyte Open Source** users, we recommend using Service Account Key Authentication. Follow the steps below to create a service account, generate a key, and enable the Google Drive API. + +:::note +If you prefer to use OAuth for authentication with **Airbyte Open Source**, you can follow [Google's OAuth instructions](https://developers.google.com/identity/protocols/oauth2) to create an authentication app. Be sure to set the scopes to `https://www.googleapis.com/auth/drive.readonly`. You will need to obtain your client ID, client secret, and refresh token for the connector setup. +::: + +### Set up the service account key (Airbyte Open Source) + +#### Create a service account + +1. Open the [Service Accounts page](https://console.cloud.google.com/projectselector2/iam-admin/serviceaccounts) in your Google Cloud console. +2. Select an existing project, or create a new project. +3. At the top of the page, click **+ Create service account**. +4. Enter a name and description for the service account, then click **Create and Continue**. +5. Under **Service account permissions**, select the roles to grant to the service account, then click **Continue**. We recommend the **Viewer** role. + +#### Generate a key + +1. Go to the [API Console/Credentials](https://console.cloud.google.com/apis/credentials) page and click on the email address of the service account you just created. +2. In the **Keys** tab, click **+ Add key**, then click **Create new key**. +3. Select **JSON** as the Key type. This will generate and download the JSON key file that you'll use for authentication. Click **Continue**. + +#### Enable the Google Drive API + +1. Go to the [API Console/Library](https://console.cloud.google.com/apis/library) page. +2. Make sure you have selected the correct project from the top. +3. Find and select the **Google Drive API**. +4. Click **ENABLE**. + +If your folder is viewable by anyone with its link, no further action is needed. If not, give your Service account access to your folder. Check out [this video](https://youtu.be/GyomEw5a2NQ%22) for how to do this. + + + +### Set up the Google Drive source connector in Airbyte + +To set up Google Drive as a source in Airbyte Cloud: + +1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. +3. Find and select **Google Drive** from the list of available sources. +4. For **Source name**, enter a name to help you identify this source. +5. Select your authentication method: + + + +#### For Airbyte Cloud + +- **(Recommended)** Select **Authenticate via Google (OAuth)** from the Authentication dropdown, click **Sign in with Google** and complete the authentication workflow. + + + + +#### For Airbyte Open Source + +- **(Recommended)** Select **Service Account Key Authentication** from the dropdown and enter your Google Cloud service account key in JSON format: + + ```js + { "type": "service_account", "project_id": "YOUR_PROJECT_ID", "private_key_id": "YOUR_PRIVATE_KEY", ... } + ``` + +- To authenticate your Google account via OAuth, select **Authenticate via Google (OAuth)** from the dropdown and enter your Google application's client ID, client secret, and refresh token. + + + +6. For **Folder Link**, enter the link to the Google Drive folder. To get the link, navigate to the folder you want to sync in the Google Drive UI, and copy the current URL. +7. Configure the optional **Start Date** parameter that marks a starting date and time in UTC for data replication. Any files that have _not_ been modified since this specified date/time will _not_ be replicated. Use the provided datepicker (recommended) or enter the desired date programmatically in the format `YYYY-MM-DDTHH:mm:ssZ`. Leaving this field blank will replicate data from all files that have not been excluded by the **Path Pattern** and **Path Prefix**. +8. Click **Set up source** and wait for the tests to complete. + +## Supported sync modes + +The Google Drive source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): + +| Feature | Supported? | +| :--------------------------------------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental Sync | Yes | +| Replicate Incremental Deletes | No | +| Replicate Multiple Files \(pattern matching\) | Yes | +| Replicate Multiple Streams \(distinct tables\) | Yes | +| Namespaces | No | + +## Path Patterns + +\(tl;dr -> path pattern syntax using [wcmatch.glob](https://facelessuser.github.io/wcmatch/glob/). GLOBSTAR and SPLIT flags are enabled.\) + +This connector can sync multiple files by using glob-style patterns, rather than requiring a specific path for every file. This enables: + +- Referencing many files with just one pattern, e.g. `**` would indicate every file in the folder. +- Referencing future files that don't exist yet \(and therefore don't have a specific path\). + +You must provide a path pattern. You can also provide many patterns split with \| for more complex directory layouts. + +Each path pattern is a reference from the _root_ of the folder, so don't include the root folder name itself in the pattern\(s\). + +Some example patterns: + +- `**` : match everything. +- `**/*.csv` : match all files with specific extension. +- `myFolder/**/*.csv` : match all csv files anywhere under myFolder. +- `*/**` : match everything at least one folder deep. +- `*/*/*/**` : match everything at least three folders deep. +- `**/file.*|**/file` : match every file called "file" with any extension \(or no extension\). +- `x/*/y/*` : match all files that sit in sub-folder x -> any folder -> folder y. +- `**/prefix*.csv` : match all csv files with specific prefix. +- `**/prefix*.parquet` : match all parquet files with specific prefix. + +Let's look at a specific example, matching the following folder layout (`MyFolder` is the folder specified in the connector config as the root folder, which the patterns are relative to): + +```text +MyFolder + -> log_files + -> some_table_files + -> part1.csv + -> part2.csv + -> images + -> more_table_files + -> part3.csv + -> extras + -> misc + -> another_part1.csv +``` + +We want to pick up part1.csv, part2.csv and part3.csv \(excluding another_part1.csv for now\). We could do this a few different ways: + +- We could pick up every csv file called "partX" with the single pattern `**/part*.csv`. +- To be a bit more robust, we could use the dual pattern `some_table_files/*.csv|more_table_files/*.csv` to pick up relevant files only from those exact folders. +- We could achieve the above in a single pattern by using the pattern `*table_files/*.csv`. This could however cause problems in the future if new unexpected folders started being created. +- We can also recursively wildcard, so adding the pattern `extras/**/*.csv` would pick up any csv files nested in folders below "extras", such as "extras/misc/another_part1.csv". + +As you can probably tell, there are many ways to achieve the same goal with path patterns. We recommend using a pattern that ensures clarity and is robust against future additions to the directory structure. + +## User Schema + +When using the Avro, Jsonl, CSV or Parquet format, you can provide a schema to use for the output stream. **Note that this doesn't apply to the experimental Document file type format.** + +Providing a schema allows for more control over the output of this stream. Without a provided schema, columns and datatypes will be inferred from the first created file in the bucket matching your path pattern and suffix. This will probably be fine in most cases but there may be situations you want to enforce a schema instead, e.g.: + +- You only care about a specific known subset of the columns. The other columns would all still be included, but packed into the `_ab_additional_properties` map. +- Your initial dataset is quite small \(in terms of number of records\), and you think the automatic type inference from this sample might not be representative of the data in the future. +- You want to purposely define types for every column. +- You know the names of columns that will be added to future data and want to include these in the core schema as columns rather than have them appear in the `_ab_additional_properties` map. + +Or any other reason! The schema must be provided as valid JSON as a map of `{"column": "datatype"}` where each datatype is one of: + +- string +- number +- integer +- object +- array +- boolean +- null + +For example: + +- `{"id": "integer", "location": "string", "longitude": "number", "latitude": "number"}` +- `{"username": "string", "friends": "array", "information": "object"}` + +## File Format Settings + +### CSV + +Since CSV files are effectively plain text, providing specific reader options is often required for correct parsing of the files. These settings are applied when a CSV is created or exported so please ensure that this process happens consistently over time. + +- **Header Definition**: How headers will be defined. `User Provided` assumes the CSV does not have a header row and uses the headers provided and `Autogenerated` assumes the CSV does not have a header row and the CDK will generate headers using for `f{i}` where `i` is the index starting from 0. Else, the default behavior is to use the header from the CSV file. If a user wants to autogenerate or provide column names for a CSV having headers, they can set a value for the "Skip rows before header" option to ignore the header row. +- **Delimiter**: Even though CSV is an acronym for Comma Separated Values, it is used more generally as a term for flat file data that may or may not be comma separated. The delimiter field lets you specify which character acts as the separator. To use [tab-delimiters](https://en.wikipedia.org/wiki/Tab-separated_values), you can set this value to `\t`. By default, this value is set to `,`. +- **Double Quote**: This option determines whether two quotes in a quoted CSV value denote a single quote in the data. Set to True by default. +- **Encoding**: Some data may use a different character set \(typically when different alphabets are involved\). See the [list of allowable encodings here](https://docs.python.org/3/library/codecs.html#standard-encodings). By default, this is set to `utf8`. +- **Escape Character**: An escape character can be used to prefix a reserved character and ensure correct parsing. A commonly used character is the backslash (`\`). For example, given the following data: + +``` +Product,Description,Price +Jeans,"Navy Blue, Bootcut, 34\"",49.99 +``` + +The backslash (`\`) is used directly before the second double quote (`"`) to indicate that it is _not_ the closing quote for the field, but rather a literal double quote character that should be included in the value (in this example, denoting the size of the jeans in inches: `34"` ). + +Leaving this field blank (default option) will disallow escaping. + +- **False Values**: A set of case-sensitive strings that should be interpreted as false values. +- **Null Values**: A set of case-sensitive strings that should be interpreted as null values. For example, if the value 'NA' should be interpreted as null, enter 'NA' in this field. +- **Quote Character**: In some cases, data values may contain instances of reserved characters \(like a comma, if that's the delimiter\). CSVs can handle this by wrapping a value in defined quote characters so that on read it can parse it correctly. By default, this is set to `"`. +- **Skip Rows After Header**: The number of rows to skip after the header row. +- **Skip Rows Before Header**: The number of rows to skip before the header row. +- **Strings Can Be Null**: Whether strings can be interpreted as null values. If true, strings that match the null_values set will be interpreted as null. If false, strings that match the null_values set will be interpreted as the string itself. +- **True Values**: A set of case-sensitive strings that should be interpreted as true values. + + +### Parquet + +Apache Parquet is a column-oriented data storage format of the Apache Hadoop ecosystem. It provides efficient data compression and encoding schemes with enhanced performance to handle complex data in bulk. At the moment, partitioned parquet datasets are unsupported. The following settings are available: + +- **Convert Decimal Fields to Floats**: Whether to convert decimal fields to floats. There is a loss of precision when converting decimals to floats, so this is not recommended. + +### Avro + +The Avro parser uses the [Fastavro library](https://fastavro.readthedocs.io/en/latest/). The following settings are available: +- **Convert Double Fields to Strings**: Whether to convert double fields to strings. This is recommended if you have decimal numbers with a high degree of precision because there can be a loss precision when handling floating point numbers. + +### JSONL + +There are currently no options for JSONL parsing. + +### Document File Type Format (Experimental) + +:::warning +The Document file type format is currently an experimental feature and not subject to SLAs. Use at your own risk. +::: + +The Document file type format is a special format that allows you to extract text from Markdown, TXT, PDF, Word, Powerpoint and Google documents. If selected, the connector will extract text from the documents and output it as a single field named `content`. The `document_key` field will hold a unique identifier for the processed file which can be used as a primary key. The content of the document will contain markdown formatting converted from the original file format. Each file matching the defined glob pattern needs to either be a markdown (`md`), PDF (`pdf`) or Docx (`docx`) file. + +One record will be emitted for each document. Keep in mind that large files can emit large records that might not fit into every destination as each destination has different limitations for string fields. + +Before parsing each document, the connector exports Google Document files to Docx format internally. Google Sheets, Google Slides, and drawings are internally exported and parsed by the connector as PDFs. + +## Changelog + +| Version | Date | Pull Request | Subject | +|---------|------------|-----------------------------------------------------------|--------------------------------------------------------------| +| 0.0.6 | 2023-12-16 | [33414](https://github.com/airbytehq/airbyte/pull/33414) | Prepare for airbyte-lib | +| 0.0.5 | 2023-12-14 | [33411](https://github.com/airbytehq/airbyte/pull/33411) | Bump CDK version to auto-set primary key for document file streams and support raw txt files | +| 0.0.4 | 2023-12-06 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Bump CDK version to hide source-defined primary key | +| 0.0.3 | 2023-11-16 | [31458](https://github.com/airbytehq/airbyte/pull/31458) | Improve folder id input and update document file type parser | +| 0.0.2 | 2023-11-02 | [31458](https://github.com/airbytehq/airbyte/pull/31458) | Allow syncs on shared drives | +| 0.0.1 | 2023-11-02 | [31458](https://github.com/airbytehq/airbyte/pull/31458) | Initial Google Drive source | + diff --git a/docs/integrations/sources/google-search-console.inapp.md b/docs/integrations/sources/google-search-console.inapp.md deleted file mode 100644 index 5dbd50cc87b0..000000000000 --- a/docs/integrations/sources/google-search-console.inapp.md +++ /dev/null @@ -1,28 +0,0 @@ -## Prerequisite - -- A verified property in Google Search Console - -- Google Search Console API enabled for your project (**Airbyte Open Source** only) - - -## Setup guide - -1. For **Source name**, enter a name to help you identify this source. -2. For **Website URL Property**, enter the specific website property in Google Seach Console with data you want to replicate. -3. For **Start Date**, by default the `2021-01-01` is set, use the provided datepicker or enter a date in the format `YYYY-MM-DD`. Any data created on or after this date will be replicated. -4. To authenticate the connection: - - - - **For Airbyte Cloud**: Select **Oauth** from the Authentication dropdown, then click **Sign in with Google** to authorize your account. More information on authentication methods can be found in our [full Google Search Console documentation](https://docs.airbyte.io/integrations/sources/google-search-console#setup-guide). - - - - (Recommended) To authenticate with a service account, select **Service Account Key Authorization** from the Authentication dropdown, then enter the **Admin Email** and **Service Account JSON Key**. For the key, copy and paste the JSON key you obtained during the service account setup. It should begin with `{"type": "service account", "project_id": YOUR_PROJECT_ID, "private_key_id": YOUR_PRIVATE_KEY, ...}`. - - To authenticate with OAuth, select **Oauth** from the Authentication dropdown, then enter your **Client ID**, **Client Secret**, **Access Token** and **Refresh Token**. More information on authentication methods for Airbyte Open Source can be found in our [full Google Search Console documentation](https://docs.airbyte.io/integrations/sources/google-search-console#setup-guide). - - -5. (Optional) For **End Date**, you may optionally provide a date in the format `YYYY-MM-DD`. Any data created between the defined Start Date and End Date will be replicated. Leaving this field blank will replicate all data created on or after the Start Date to the present. -6. (Optional) For **Custom Reports**, you may optionally provide an array of JSON objects representing any custom reports you wish to query the API with. Refer to the [Custom reports](https://docs.airbyte.com/integrations/sources/google-search-console#custom-reports) section in our full documentation for more information on formulating these reports. -7. (Optional) For **Data Freshness**, you may choose whether to include "fresh" data that has not been finalized by Google, and may be subject to change. Please note that if you are using Incremental sync mode, we highly recommend leaving this option to its default value of `final`. Refer to the [Data Freshness](https://docs.airbyte.com/integrations/sources/google-search-console#data-freshness) section in our full documentation for more information on this parameter. -8. Click **Set up source** and wait for the tests to complete. - -For detailed information on supported sync modes, supported streams, and performance considerations, refer to the full documentation for [Google Search Console](https://docs.airbyte.com/integrations/sources/google-search-console/). diff --git a/docs/integrations/sources/google-search-console.md b/docs/integrations/sources/google-search-console.md index 79ccf752a790..12d4ba6128b1 100644 --- a/docs/integrations/sources/google-search-console.md +++ b/docs/integrations/sources/google-search-console.md @@ -1,7 +1,11 @@ # Google Search Console + + This page contains the setup guide and reference information for the Google Search Console source connector. + + ## Prerequisites - A verified property in Google Search Console (or the list of the `Site URLs` (Website URL Properties)) @@ -11,13 +15,13 @@ This page contains the setup guide and reference information for the Google Sear ## Setup guide -### Step 1: Set up Google Search Console authentication +### Step 1: Set up Google Search Console To authenticate the Google Search Console connector, you will need to use one of the following methods: -#### I: OAuth (Recommended for Airbyte Cloud) - +#### OAuth (Recommended for Airbyte Cloud) + You can authenticate using your Google Account with OAuth if you are the owner of the Google Search Console property or have view permissions. Follow [Google's instructions](https://support.google.com/webmasters/answer/7687615?sjid=11103698321670173176-NA) to ensure that your account has the necessary permissions (**Owner** or **Full User**) to view the Google Search Console property. This option is recommended for **Airbyte Cloud** users, as it significantly simplifies the setup process and allows you to authenticate the connection [directly from the Airbyte UI](#step-2-set-up-the-google-search-console-connector-in-airbyte). @@ -31,7 +35,7 @@ To authenticate with OAuth in **Airbyte Open Source**, you will need to create a More information on the steps to create an OAuth app to access Google APIs and obtain these credentials can be found [in Google's documentation](https://developers.google.com/identity/protocols/oauth2). -#### II: Google service account with JSON key file (Recommended for Airbyte Open Source) +#### Google service account with JSON key file (Recommended for Airbyte Open Source) You can authenticate the connection using a JSON key file associated with a Google service account. This option is recommended for **Airbyte Open Source** users. Follow the steps below to create a service account and generate the JSON key file: @@ -70,28 +74,32 @@ For more information on this topic, please refer to [this Google article](https: ### Step 2: Set up the Google Search Console connector in Airbyte -1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account. + +**For Airbyte Cloud:** + +1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. 3. Find and select **Google Search Console** from the list of available sources. 4. For **Source name**, enter a name to help you identify this source. 5. For **Website URL Property**, enter the specific website property in Google Seach Console with data you want to replicate. 6. For **Start Date**, by default the `2021-01-01` is set, use the provided datepicker or enter a date in the format `YYYY-MM-DD`. Any data created on or after this date will be replicated. 7. To authenticate the connection: - - - - **For Airbyte Cloud**: Select **Oauth** from the Authentication dropdown, then click **Sign in with Google** to authorize your account. - - - - **For Airbyte Open Source**: - - (Recommended) Select **Service Account Key Authorization** from the Authentication dropdown, then enter the **Admin Email** and **Service Account JSON Key**. For the key, copy and paste the JSON key you obtained during the service account setup. It should begin with `{"type": "service account", "project_id": YOUR_PROJECT_ID, "private_key_id": YOUR_PRIVATE_KEY, ...}` - - Select **Oauth** from the Authentication dropdown, then enter your **Client ID**, **Client Secret**, **Access Token** and **Refresh Token**. - - + +- **For Airbyte Cloud:** + - Select **Oauth** from the Authentication dropdown, then click **Sign in with Google** to authorize your account. + + +- **For Airbyte Open Source:** + - (Recommended) Select **Service Account Key Authorization** from the Authentication dropdown, then enter the **Admin Email** and **Service Account JSON Key**. For the key, copy and paste the JSON key you obtained during the service account setup. It should begin with `{"type": "service account", "project_id": YOUR_PROJECT_ID, "private_key_id": YOUR_PRIVATE_KEY, ...}` + - Select **Oauth** from the Authentication dropdown, then enter your **Client ID**, **Client Secret**, **Access Token** and **Refresh Token**. + 8. (Optional) For **End Date**, you may optionally provide a date in the format `YYYY-MM-DD`. Any data created between the defined Start Date and End Date will be replicated. Leaving this field blank will replicate all data created on or after the Start Date to the present. 9. (Optional) For **Custom Reports**, you may optionally provide an array of JSON objects representing any custom reports you wish to query the API with. Refer to the [Custom reports](#custom-reports) section below for more information on formulating these reports. 10. (Optional) For **Data Freshness**, you may choose whether to include "fresh" data that has not been finalized by Google, and may be subject to change. Please note that if you are using Incremental sync mode, we highly recommend leaving this option to its default value of `final`. Refer to the [Data Freshness](#data-freshness) section below for more information on this parameter. 11. Click **Set up source** and wait for the tests to complete. + + ## Supported sync modes The Google Search Console Source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): @@ -127,34 +135,28 @@ The granularity for the cursor is 1 day, so Incremental Sync in Append mode may ### Custom reports -Custom reports allow you to query the API with a custom set of dimensions to group results by. Results are grouped in the order that you supply these dimensions. Each custom report should be constructed as a JSON object in the following format: +Custom reports allow you to query the API with a custom set of dimensions to group results by. Results are grouped in the order that you supply these dimensions. Each custom report should be constructed like following: -```json -{ - "name": "", - "dimensions": ["", "", ...] - } -``` +1. Click `Add` under the `Custom Reports` section +2. Enter the `Name` of the report, this will be the name of the stream +3. Select one or more `Dimensions` from the available dropdown list -The available dimensions are: +The available `Dimensions` are: - `country` - `date` - `device` - `page` - `query` -- `searchAppearance` For example, to query the API for a report that groups results by country, then by date, you could enter the following custom report: -```json -[ - { - "name": "country_date", - "dimensions": ["country", "date"] - } -] -``` +* Name: country_date +* Dimensions: ["country", "date"] + +Please note, that for technical reasons `date` is the default dimension which will be included in your query whether you specify it or not. By specifying it you can change the order the results are grouped in. Primary key will consist of your custom dimensions and the default dimension along with `site_url` and `search_type`. + +The information you provide via UI Custom report builder will then be transformed into the custom stream by it's `Name` You can use the [Google APIS Explorer](https://developers.google.com/webmaster-tools/v1/searchanalytics/query) to build and test the reports you want to use. @@ -169,10 +171,6 @@ The **Data Freshness** parameter deals with the "freshness", or finality of the When using Incremental Sync mode, we recommend leaving this parameter to its default state of `final`, as the `all` option may cause discrepancies between the data in your destination table and the finalized data in Google Search Console. ::: -## Performance considerations - -This connector attempts to back off gracefully when it hits Reports API's rate limits. To find more information about limits, see [Usage Limits](https://developers.google.com/webmaster-tools/limits) documentation. - ## Data type map | Integration Type | Airbyte Type | Notes | @@ -182,14 +180,36 @@ This connector attempts to back off gracefully when it hits Reports API's rate l | `array` | `array` | | | `object` | `object` | | +## Limitations & Troubleshooting + +
      + +Expand to see details about Google Search Console connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting +This connector attempts to back off gracefully when it hits Reports API's rate limits. To find more information about limits, see [Usage Limits](https://developers.google.com/webmaster-tools/limits) documentation. + +### Troubleshooting + +* Check out common troubleshooting issues for the Google Search Console source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +
      + ## Changelog | Version | Date | Pull Request | Subject | -| :------- | :--------- | :------------------------------------------------------------------------------------------------------------ | :----------------------------------------------------------------------------------------------------------------------------- | -| `1.3.2` | 2023-08-25 | [29829](https://github.com/airbytehq/airbyte/pull/29829) | Make `Start Date` a non-required, added the `suggested streams`, corrected public docs | -| `1.3.1` | 2023-08-24 | [29329](https://github.com/airbytehq/airbyte/pull/29329) | Update tooltip descriptions | -| `1.3.0` | 2023-08-24 | [29750](https://github.com/airbytehq/airbyte/pull/29750) | Add new `Keyword-Site-Report-By-Site` stream | -| `1.2.2` | 2023-08-23 | [29741](https://github.com/airbytehq/airbyte/pull/29741) | Handle `HTTP-401`, `HTTP-403` errors | +|:---------|:-----------|:--------------------------------------------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------| +| `1.3.6` | 2023-10-26 | [31863](https://github.com/airbytehq/airbyte/pull/31863) | Base image migration: remove Dockerfile and use the python-connector-base image | +| `1.3.5` | 2023-09-28 | [30822](https://github.com/airbytehq/airbyte/pull/30822) | Fix primary key for custom reports | +| `1.3.4` | 2023-09-27 | [30785](https://github.com/airbytehq/airbyte/pull/30785) | Do not migrate config for the newly created connections | +| `1.3.3` | 2023-08-29 | [29941](https://github.com/airbytehq/airbyte/pull/29941) | Added `primary key` to each stream, added `custom_report` config migration | +| `1.3.2` | 2023-08-25 | [29829](https://github.com/airbytehq/airbyte/pull/29829) | Make `Start Date` a non-required, added the `suggested streams`, corrected public docs | +| `1.3.1` | 2023-08-24 | [29329](https://github.com/airbytehq/airbyte/pull/29329) | Update tooltip descriptions | +| `1.3.0` | 2023-08-24 | [29750](https://github.com/airbytehq/airbyte/pull/29750) | Add new `Keyword-Site-Report-By-Site` stream | +| `1.2.2` | 2023-08-23 | [29741](https://github.com/airbytehq/airbyte/pull/29741) | Handle `HTTP-401`, `HTTP-403` errors | | `1.2.1` | 2023-07-04 | [27952](https://github.com/airbytehq/airbyte/pull/27952) | Removed deprecated `searchType`, added `discover`(Discover results) and `googleNews`(Results from news.google.com, etc.) types | | `1.2.0` | 2023-06-29 | [27831](https://github.com/airbytehq/airbyte/pull/27831) | Add new streams | | `1.1.0` | 2023-06-26 | [27738](https://github.com/airbytehq/airbyte/pull/27738) | License Update: Elv2 | @@ -218,3 +238,5 @@ This connector attempts to back off gracefully when it hits Reports API's rate l | `0.1.2` | 2021-09-17 | [6222](https://github.com/airbytehq/airbyte/pull/6222) | Correct Spec File | | `0.1.1` | 2021-09-22 | [6315](https://github.com/airbytehq/airbyte/pull/6315) | Verify access to all sites when performing connection check | | `0.1.0` | 2021-09-03 | [5350](https://github.com/airbytehq/airbyte/pull/5350) | Initial Release | + +
      \ No newline at end of file diff --git a/docs/integrations/sources/google-sheets.inapp.md b/docs/integrations/sources/google-sheets.inapp.md deleted file mode 100644 index 3e8374aef9d2..000000000000 --- a/docs/integrations/sources/google-sheets.inapp.md +++ /dev/null @@ -1,45 +0,0 @@ -## Prerequisites -- Spreadsheet Link - The link to the Google spreadsheet you want to sync. -- A Google Workspace user with access to the spreadsheet - -:::info -The Google Sheets source connector pulls data from a single Google Sheets spreadsheet. To replicate multiple spreadsheets, set up multiple Google Sheets source connectors in your Airbyte instance. -::: - -## Setup guide - -1. For **Source name**, enter a name to help you identify this source. -2. Select your authentication method: - - - -#### For Airbyte Cloud - -- **(Recommended)** Select **Authenticate via Google (OAuth)** from the Authentication dropdown, click **Sign in with Google** and complete the authentication workflow. - - - - -#### For Airbyte Open Source - -- **(Recommended)** Select **Service Account Key Authentication** from the dropdown and enter your Google Cloud service account key in JSON format: - - ```js - { "type": "service_account", "project_id": "YOUR_PROJECT_ID", "private_key_id": "YOUR_PRIVATE_KEY", ... } - ``` - -- To authenticate your Google account via OAuth, select **Authenticate via Google (OAuth)** from the dropdown and enter your Google application's client ID, client secret, and refresh token. - -For detailed instructions on how to generate a service account key or OAuth credentials, refer to the [full documentation](https://docs.airbyte.io/integrations/sources/google-sheets#setup-guide). - - - -3. For **Spreadsheet Link**, enter the link to the Google spreadsheet. To get the link, go to the Google spreadsheet you want to sync, click **Share** in the top right corner, and click **Copy Link**. -4. (Optional) You may enable the option to **Convert Column Names to SQL-Compliant Format**. Enabling this option will allow the connector to convert column names to a standardized, SQL-friendly format. For example, a column name of `Café Earnings 2022` will be converted to `cafe_earnings_2022`. We recommend enabling this option if your target destination is SQL-based (ie Postgres, MySQL). Set to false by default. -5. Click **Set up source** and wait for the tests to complete. - -### Output schema - -- Airbyte only supports replicating [Grid](https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#SheetType) sheets. - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Google Sheets](https://docs.airbyte.com/integrations/sources/google-sheets/). diff --git a/docs/integrations/sources/google-sheets.md b/docs/integrations/sources/google-sheets.md index b40cbf6a2317..6a3575451028 100644 --- a/docs/integrations/sources/google-sheets.md +++ b/docs/integrations/sources/google-sheets.md @@ -1,41 +1,49 @@ # Google Sheets + + This page contains the setup guide and reference information for the Google Sheets source connector. + + :::info -The Google Sheets source connector pulls data from a single Google Sheets spreadsheet. Each sheet (tab) within a spreadsheet can be replicated. To replicate multiple spreadsheets, set up multiple Google Sheets source connectors in your Airbyte instance. No other files in your Google Drive are accessed. +The Google Sheets source connector pulls data from a single Google Sheets spreadsheet. Each sheet within a spreadsheet can be replicated. To replicate multiple spreadsheets, set up multiple Google Sheets source connectors in your Airbyte instance. No other files in your Google Drive are accessed. ::: ### Prerequisites - Spreadsheet Link - The link to the Google spreadsheet you want to sync. - -- **For Airbyte Cloud** A Google Workspace user with access to the spreadsheet - - -- **For Airbyte Open Source:** + +- **For Airbyte Cloud** A Google Workspace user with access to the spreadsheet + + +- **For Airbyte Open Source:** - A GCP project - Enable the Google Sheets API in your GCP project - Service Account Key with access to the Spreadsheet you want to replicate - + ## Setup guide The Google Sheets source connector supports authentication via either OAuth or Service Account Key Authentication. -For **Airbyte Cloud** users, we highly recommend using OAuth, as it significantly simplifies the setup process and allows you to authenticate [directly from the Airbyte UI](#set-up-the-google-sheets-source-connector-in-airbyte). +**For Airbyte Cloud:** + +We highly recommend using OAuth, as it significantly simplifies the setup process and allows you to authenticate [directly from the Airbyte UI](#set-up-the-google-sheets-source-connector-in-airbyte). -For **Airbyte Open Source** users, we recommend using Service Account Key Authentication. Follow the steps below to create a service account, generate a key, and enable the Google Sheets API. +**For Airbyte Open Source:** + +We recommend using Service Account Key Authentication. Follow the steps below to create a service account, generate a key, and enable the Google Sheets API. :::note If you prefer to use OAuth for authentication with **Airbyte Open Source**, you can follow [Google's OAuth instructions](https://developers.google.com/identity/protocols/oauth2) to create an authentication app. Be sure to set the scopes to `https://www.googleapis.com/auth/spreadsheets.readonly`. You will need to obtain your client ID, client secret, and refresh token for the connector setup. ::: -### Set up the service account key (Airbyte Open Source) +### Set up the service account key #### Create a service account @@ -64,39 +72,36 @@ If your spreadsheet is viewable by anyone with its link, no further action is ne ### Set up the Google Sheets source connector in Airbyte -To set up Google Sheets as a source in Airbyte Cloud: -1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account. + +1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. 3. Find and select **Google Sheets** from the list of available sources. 4. For **Source name**, enter a name to help you identify this source. 5. Select your authentication method: - - -#### For Airbyte Cloud - -- **(Recommended)** Select **Authenticate via Google (OAuth)** from the Authentication dropdown, click **Sign in with Google** and complete the authentication workflow. - + - **For Airbyte Cloud: (Recommended)** Select **Authenticate via Google (OAuth)** from the Authentication dropdown, click **Sign in with Google** and complete the authentication workflow. - -#### For Airbyte Open Source - -- **(Recommended)** Select **Service Account Key Authentication** from the dropdown and enter your Google Cloud service account key in JSON format: - - ```js - { "type": "service_account", "project_id": "YOUR_PROJECT_ID", "private_key_id": "YOUR_PRIVATE_KEY", ... } - ``` - -- To authenticate your Google account via OAuth, select **Authenticate via Google (OAuth)** from the dropdown and enter your Google application's client ID, client secret, and refresh token. - + - **For Airbyte Open Source: (Recommended)** Select **Service Account Key Authentication** from the dropdown and enter your Google Cloud service account key in JSON format: + + ```json + { + "type": "service_account", + "project_id": "YOUR_PROJECT_ID", + "private_key_id": "YOUR_PRIVATE_KEY", + ... + } + ``` + + - To authenticate your Google account via OAuth, select **Authenticate via Google (OAuth)** from the dropdown and enter your Google application's client ID, client secret, and refresh token. - 6. For **Spreadsheet Link**, enter the link to the Google spreadsheet. To get the link, go to the Google spreadsheet you want to sync, click **Share** in the top right corner, and click **Copy Link**. 7. (Optional) You may enable the option to **Convert Column Names to SQL-Compliant Format**. Enabling this option will allow the connector to convert column names to a standardized, SQL-friendly format. For example, a column name of `Café Earnings 2022` will be converted to `cafe_earnings_2022`. We recommend enabling this option if your target destination is SQL-based (ie Postgres, MySQL). Set to false by default. 8. Click **Set up source** and wait for the tests to complete. + + ### Output schema Each sheet in the selected spreadsheet is synced as a separate stream. Each selected column in the sheet is synced as a string field. @@ -116,7 +121,16 @@ The Google Sheets source connector supports the following sync modes: |:-----------------|:-------------|:------| | any type | `string` | | -## Performance consideration +## Limitations & Troubleshooting + +
      + +Expand to see details about Google Sheets connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting The [Google API rate limits](https://developers.google.com/sheets/api/limits) are: @@ -125,13 +139,24 @@ The [Google API rate limits](https://developers.google.com/sheets/api/limits) ar Airbyte batches requests to the API in order to efficiently pull data and respect these rate limits. We recommend not using the same user or service account for more than 3 instances of the Google Sheets source connector to ensure high transfer speeds. -## Troubleshooting -- If your sheet is completely empty(no header rows) or deleted, Airbyte will not delete the table in the destination. If this happens, the sync logs will contain a message saying the sheet has been skipped when syncing the full spreadsheet. +### Troubleshooting + +* If your sheet is completely empty (no header rows) or deleted, Airbyte will not delete the table in the destination. If this happens, the sync logs will contain a message saying the sheet has been skipped when syncing the full spreadsheet. +* Connector setup will fail if the speadsheet is not a Google Sheets file. If the file was saved or imported as another file type the setup could fail. +* Check out common troubleshooting issues for the Google Sheets source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +
      ## Changelog | Version | Date | Pull Request | Subject | |---------|------------|----------------------------------------------------------|-----------------------------------------------------------------------------------| +| 0.3.13 | 2024-01-19 | [34376](https://github.com/airbytehq/airbyte/pull/34376) | Fix names conversion | +| 0.3.12 | 2023-12-14 | [33414](https://github.com/airbytehq/airbyte/pull/33414) | Prepare for airbyte-lib | +| 0.3.11 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 0.3.10 | 2023-09-27 | [30487](https://github.com/airbytehq/airbyte/pull/30487) | Fix bug causing rows to be skipped when batch size increased due to rate limits. | +| 0.3.9 | 2023-09-25 | [30749](https://github.com/airbytehq/airbyte/pull/30749) | Performance testing - include socat binary in docker image | +| 0.3.8 | 2023-09-25 | [30747](https://github.com/airbytehq/airbyte/pull/30747) | Performance testing - include socat binary in docker image | | 0.3.7 | 2023-08-25 | [29826](https://github.com/airbytehq/airbyte/pull/29826) | Remove row batch size from spec, add auto increase this value when rate limits | | 0.3.6 | 2023-08-16 | [29491](https://github.com/airbytehq/airbyte/pull/29491) | Update to latest CDK | | 0.3.5 | 2023-08-16 | [29427](https://github.com/airbytehq/airbyte/pull/29427) | Add stop reading in case of 429 error | @@ -174,3 +199,5 @@ Airbyte batches requests to the API in order to efficiently pull data and respec | 0.1.6 | 2021-01-27 | [1668](https://github.com/airbytehq/airbyte/pull/1668) | Adopt connector best practices | | 0.1.5 | 2020-12-30 | [1438](https://github.com/airbytehq/airbyte/pull/1438) | Implement backoff | | 0.1.4 | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046) | Add connectors using an index YAML file | + +
      \ No newline at end of file diff --git a/docs/integrations/sources/greenhouse.md b/docs/integrations/sources/greenhouse.md index 50ef4dc63de0..02984acb4850 100644 --- a/docs/integrations/sources/greenhouse.md +++ b/docs/integrations/sources/greenhouse.md @@ -62,18 +62,20 @@ The Greenhouse connector should not run into Greenhouse API limitations under no ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 0.4.2 | 2023-08-02 | [28969](https://github.com/airbytehq/airbyte/pull/28969) | Update CDK version | -| 0.4.1 | 2023-06-28 | [27773](https://github.com/airbytehq/airbyte/pull/27773) | Update following state breaking changes | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------- | +| 0.4.4 | 2023-11-29 | [32397](https://github.com/airbytehq/airbyte/pull/32397) | Increase test coverage and migrate to base image | +| 0.4.3 | 2023-09-20 | [30648](https://github.com/airbytehq/airbyte/pull/30648) | Update candidates.json | +| 0.4.2 | 2023-08-02 | [28969](https://github.com/airbytehq/airbyte/pull/28969) | Update CDK version | +| 0.4.1 | 2023-06-28 | [27773](https://github.com/airbytehq/airbyte/pull/27773) | Update following state breaking changes | | 0.4.0 | 2023-04-26 | [25332](https://github.com/airbytehq/airbyte/pull/25332) | Add new streams: `ActivityFeed`, `Approvals`, `Disciplines`, `Eeoc`, `EmailTemplates`, `Offices`, `ProspectPools`, `Schools`, `Tags`, `UserPermissions`, `UserRoles` | -| 0.3.1 | 2023-03-06 | [23231](https://github.com/airbytehq/airbyte/pull/23231) | Publish using low-code CDK Beta version | -| 0.3.0 | 2022-10-19 | [18154](https://github.com/airbytehq/airbyte/pull/18154) | Extend `Users` stream schema | -| 0.2.11 | 2022-09-27 | [17239](https://github.com/airbytehq/airbyte/pull/17239) | Always install the latest version of Airbyte CDK | -| 0.2.10 | 2022-09-05 | [16338](https://github.com/airbytehq/airbyte/pull/16338) | Implement incremental syncs & fix SATs | -| 0.2.9 | 2022-08-22 | [15800](https://github.com/airbytehq/airbyte/pull/15800) | Bugfix to allow reading sentry.yaml and schemas at runtime | -| 0.2.8 | 2022-08-10 | [15344](https://github.com/airbytehq/airbyte/pull/15344) | Migrate connector to config-based framework | -| 0.2.7 | 2022-04-15 | [11941](https://github.com/airbytehq/airbyte/pull/11941) | Correct Schema data type for Applications, Candidates, Scorecards and Users | -| 0.2.6 | 2021-11-08 | [7607](https://github.com/airbytehq/airbyte/pull/7607) | Implement demographics streams support. Update SAT for demographics streams | -| 0.2.5 | 2021-09-22 | [6377](https://github.com/airbytehq/airbyte/pull/6377) | Refactor the connector to use CDK. Implement additional stream support | -| 0.2.4 | 2021-09-15 | [6238](https://github.com/airbytehq/airbyte/pull/6238) | Add identification of accessible streams for API keys with limited permissions | +| 0.3.1 | 2023-03-06 | [23231](https://github.com/airbytehq/airbyte/pull/23231) | Publish using low-code CDK Beta version | +| 0.3.0 | 2022-10-19 | [18154](https://github.com/airbytehq/airbyte/pull/18154) | Extend `Users` stream schema | +| 0.2.11 | 2022-09-27 | [17239](https://github.com/airbytehq/airbyte/pull/17239) | Always install the latest version of Airbyte CDK | +| 0.2.10 | 2022-09-05 | [16338](https://github.com/airbytehq/airbyte/pull/16338) | Implement incremental syncs & fix SATs | +| 0.2.9 | 2022-08-22 | [15800](https://github.com/airbytehq/airbyte/pull/15800) | Bugfix to allow reading sentry.yaml and schemas at runtime | +| 0.2.8 | 2022-08-10 | [15344](https://github.com/airbytehq/airbyte/pull/15344) | Migrate connector to config-based framework | +| 0.2.7 | 2022-04-15 | [11941](https://github.com/airbytehq/airbyte/pull/11941) | Correct Schema data type for Applications, Candidates, Scorecards and Users | +| 0.2.6 | 2021-11-08 | [7607](https://github.com/airbytehq/airbyte/pull/7607) | Implement demographics streams support. Update SAT for demographics streams | +| 0.2.5 | 2021-09-22 | [6377](https://github.com/airbytehq/airbyte/pull/6377) | Refactor the connector to use CDK. Implement additional stream support | +| 0.2.4 | 2021-09-15 | [6238](https://github.com/airbytehq/airbyte/pull/6238) | Add identification of accessible streams for API keys with limited permissions | diff --git a/docs/integrations/sources/gutendex.md b/docs/integrations/sources/gutendex.md index fa026dd5da4b..434276e2db2c 100644 --- a/docs/integrations/sources/gutendex.md +++ b/docs/integrations/sources/gutendex.md @@ -34,12 +34,14 @@ ___ Lists of book information in the Project Gutenberg database are queried using the API at /books (e.g. gutendex.com/books). Book data will be returned in the format:- - { - "count": , - "next": , - "previous": , - "results": - } +``` +{ + "count": , + "next": , + "previous": , + "results": +} +``` where `results` is an array of 0-32 book objects, next and previous are URLs to the next and previous pages of results, and count in the total number of books for the query on all pages combined. diff --git a/docs/integrations/sources/harness.md b/docs/integrations/sources/harness.md index f858f3617dbf..b6433e30483a 100644 --- a/docs/integrations/sources/harness.md +++ b/docs/integrations/sources/harness.md @@ -2,7 +2,7 @@ ## Overview -The Harness source is maintained by [Faros +The Harness source is migrated from [Faros AI](https://github.com/faros-ai/airbyte-connectors/tree/main/sources/harness-source). Please file any support requests on that repo to minimize response time from the maintainers. The source supports both Full Refresh and Incremental syncs. You @@ -13,19 +13,19 @@ the tables and columns you set up for replication, every time a sync is run. Only one stream is currently available from this source: -* [Executions](https://docs.harness.io/article/ba4vs50071-use-workflows-api) \(Incremental\) +* [Organization](https://apidocs.harness.io/tag/Organization#operation/getOrganizationList) If there are more endpoints you'd like Faros AI to support, please [create an issue.](https://github.com/faros-ai/airbyte-connectors/issues/new) ### Features -| Feature | Supported? | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental Sync | Yes | -| SSL connection | Yes | -| Namespaces | No | +| Feature | Supported? | +| :----------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental Sync | No | +| SSL connection | No | +| Namespaces | No | ### Performance considerations @@ -47,6 +47,7 @@ Key](https://ngdocs.harness.io/article/tdoad7xrh9-add-and-manage-api-keys#harnes ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.23 | 2021-11-16 | [153](https://github.com/faros-ai/airbyte-connectors/pull/153) | Add Harness source and Faros destination's converter | +| Version | Date | Pull Request | Subject | +| :--------- | :--------- | :------------------------------------------------------------------ | :---------------------------------------------------- | +| 0.1.0 | 2023-10-10 | [31103](https://github.com/airbytehq/airbyte/pull/31103) | Migrate to low code | +| 0.1.23 | 2021-11-16 | [153](https://github.com/faros-ai/airbyte-connectors/pull/153) | Add Harness source and Faros destination's converter | diff --git a/docs/integrations/sources/harvest.md b/docs/integrations/sources/harvest.md index e53ff9afbe96..ac19b7dffdb7 100644 --- a/docs/integrations/sources/harvest.md +++ b/docs/integrations/sources/harvest.md @@ -81,6 +81,9 @@ The connector is restricted by the [Harvest rate limits](https://help.getharvest | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------- | +| 0.1.21 | 2023-11-30 | [33003](https://github.com/airbytehq/airbyte/pull/33003) | Update expected records | +| 0.1.20 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 0.1.19 | 2023-07-26 | [28755](https://github.com/airbytehq/airbyte/pull/28755) | Changed parameters for Time Reports to use 365 days as opposed to 1 year | | 0.1.18 | 2023-05-29 | [26714](https://github.com/airbytehq/airbyte/pull/26714) | Remove `authSpecification` from spec in favour of `advancedAuth` | | 0.1.17 | 2023-03-03 | [22983](https://github.com/airbytehq/airbyte/pull/22983) | Specified date formatting in specification | | 0.1.16 | 2023-02-07 | [22417](https://github.com/airbytehq/airbyte/pull/22417) | Turn on default HttpAvailabilityStrategy | diff --git a/docs/integrations/sources/hubplanner.md b/docs/integrations/sources/hubplanner.md index 4256522aacb4..429f71763413 100644 --- a/docs/integrations/sources/hubplanner.md +++ b/docs/integrations/sources/hubplanner.md @@ -39,4 +39,5 @@ The Okta source connector supports the following [sync modes](https://docs.airby | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------| +| 0.2.0 | 2021-09-31 | [29311](https://github.com/airbytehq/airbyte/pull/29311) | Migrated to LowCode CDK | | 0.1.0 | 2021-08-10 | [12145](https://github.com/airbytehq/airbyte/pull/12145) | Initial Release | diff --git a/docs/integrations/sources/hubspot-migrations.md b/docs/integrations/sources/hubspot-migrations.md new file mode 100644 index 000000000000..498959ee5884 --- /dev/null +++ b/docs/integrations/sources/hubspot-migrations.md @@ -0,0 +1,35 @@ +# HubSpot Migration Guide + +## Upgrading to 2.0.0 + +:::note +This change is only breaking if you are syncing the Property History stream. +::: + +With this update, you can now access historical property changes for Deals and Companies, in addition to Contacts. To facilitate this change, the Property History stream has been renamed to Contacts Property History (since it contained historical property changes from Contacts) and two new streams have been added: Deals Property History and Companies Property History. + +This constitutes a breaking change as the Property History stream has been deprecated and replaced with the Contacts Property History. Please follow the instructions below to migrate to version 2.0.0: + +1. Select **Connections** in the main navbar. + 1. Select the connection(s) affected by the update. +2. Select the **Replication** tab. + 1. Select **Refresh source schema**. + +:::note +Any detected schema changes will be listed for your review. Select **OK** to proceed. +::: + +3. Select **Save changes** at the bottom of the page. + 1. Ensure the **Reset affected streams** option is checked. + +:::note +Depending on destination type you may not be prompted to reset your data +::: + +4. Select **Save connection**. + +:::note +This will reset the data in your destination and initiate a fresh sync. +::: + +For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). diff --git a/docs/integrations/sources/hubspot.inapp.md b/docs/integrations/sources/hubspot.inapp.md deleted file mode 100644 index fdafd906607e..000000000000 --- a/docs/integrations/sources/hubspot.inapp.md +++ /dev/null @@ -1,57 +0,0 @@ -## Prerequisites - -- HubSpot Account -- **For Airbyte Open Source**: Private App with Access Token - -### Authentication method -You can use OAuth or a Private App to authenticate your HubSpot account. For Airbyte Cloud users, we highly recommend you use OAuth rather than Private App authentication, as it significantly simplifies the setup process. - -For more information on which authentication method to choose and the required setup steps, see our full -[Hubspot documentation](https://docs.airbyte.com/integrations/sources/hubspot/). - -### Scopes Required (for Private App and Open Source OAuth) -Unless you are authenticating via OAuth on **Airbyte Cloud**, you must manually configure scopes to ensure Airbyte can sync all available data. To see a breakdown of the specific scopes each stream uses, see our full [Hubspot documentation](https://docs.airbyte.com/integrations/sources/hubspot/). - -* `content` -* `forms` -* `tickets` -* `automation` -* `e-commerce` -* `sales-email-read` -* `crm.objects.companies.read` -* `crm.schemas.companies.read` -* `crm.objects.lists.read` -* `crm.objects.contacts.read` -* `crm.objects.deals.read` -* `crm.schemas.deals.read` -* `crm.objects.goals.read` -* `crm.objects.owners.read` -* `crm.objects.custom.read` - -## Setup guide - -1. Log in to your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account. -2. From the Airbyte UI, click **Sources**, then click on **+ New Source** and select **HubSpot** from the list of available sources. -3. Enter a **Source name** of your choosing. -4. From the **Authentication** dropdown, select your chosen authentication method: - - -#### For Airbyte Cloud users: -- To authenticate using OAuth, select **OAuth** and click **Authenticate your HubSpot account** to sign in with HubSpot and authorize your account. -- To authenticate using a Private App, select **Private App** and enter the Access Token for your HubSpot account. - - - -#### For Airbyte Open Source users: -- To authenticate using OAuth, select **OAuth** and enter your Client ID, Client Secret, and Refresh Token. -- To authenticate using a Private App, select **Private App** and enter the Access Token for your HubSpot account. - - -5. For **Start date**, use the provided datepicker or enter the date programmatically in the following format: -`yyyy-mm-ddThh:mm:ssZ`. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. -6. Click **Set up source** and wait for the tests to complete. - -## Supported Objects -Airbyte supports syncing standard and custom CRM objects. Custom CRM objects will appear as streams available for sync, alongside the standard objects. - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Hubspot](https://docs.airbyte.com/integrations/sources/hubspot/). diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md index ebf4ce938044..97d00e2a9b94 100644 --- a/docs/integrations/sources/hubspot.md +++ b/docs/integrations/sources/hubspot.md @@ -1,44 +1,63 @@ # HubSpot + + This page contains the setup guide and reference information for the [HubSpot](https://www.hubspot.com/) source connector. + + ## Prerequisites - HubSpot Account + + - **For Airbyte Open Source**: Private App with Access Token + ## Setup guide -**For Airbyte Cloud** users, we highly recommend you use OAuth rather than Private App authentication, as it significantly simplifies the setup process. + +**For Airbyte Cloud:** + +We highly recommend you use OAuth rather than Private App authentication, as it significantly simplifies the setup process. + -**For Airbyte Open Source** users we recommend Private App authentication. + +**For Airbyte Open Source:** + +We recommend Private App authentication. + More information on HubSpot authentication methods can be found [here](https://developers.hubspot.com/docs/api/intro-to-auth). -### Step 1: Set up the authentication method +### Step 1: Set up Hubspot + + +**For Airbyte Cloud:** -#### Private App setup (Recommended for Airbyte Open Source) +**- OAuth** (Recommended) -If you are authenticating via a Private App, you will need to use your Access Token to set up the connector. Please refer to the -[official HubSpot documentation](https://developers.hubspot.com/docs/api/private-apps) for a detailed guide. +**- Private App:** If you are using a Private App, you will need to use your Access Token to set up the connector. Please refer to the [official HubSpot documentation](https://developers.hubspot.com/docs/api/private-apps) for a detailed guide. + +**For Airbyte Open Source:** -#### OAuth setup for Airbyte Open Source (Not recommended) +**- Private App setup** (Recommended): If you are authenticating via a Private App, you will need to use your Access Token to set up the connector. Please refer to the [official HubSpot documentation](https://developers.hubspot.com/docs/api/private-apps) for a detailed guide. -If you are using Oauth to authenticate on Airbyte Open Source, please refer to [Hubspot's detailed walkthrough](https://developers.hubspot.com/docs/api/working-with-oauth). To set up the connector, you will need to acquire your: +**- OAuth setup:** If you are using Oauth to authenticate on Airbyte Open Source, please refer to [Hubspot's detailed walkthrough](https://developers.hubspot.com/docs/api/working-with-oauth). To set up the connector, you will need to acquire your: - Client ID - Client Secret - Refresh Token - ### Step 2: Configure the scopes for your streams -Next, you need to configure the appropriate scopes for the following streams. Please refer to -[Hubspot's page on scopes](https://legacydocs.hubspot.com/docs/methods/oauth2/initiate-oauth-integration#scopes) for instructions. +Unless you are authenticating via OAuth on **Airbyte Cloud**, you must manually configure scopes to ensure Airbyte can sync all available data. To see a breakdown of the specific scopes each stream uses, see our full [Hubspot documentation](https://docs.airbyte.com/integrations/sources/hubspot/). + +Next, you need to configure the appropriate scopes for the following streams. Please refer to [Hubspot's page on scopes](https://legacydocs.hubspot.com/docs/methods/oauth2/initiate-oauth-integration#scopes) for instructions. | Stream | Required Scope | | :-------------------------- | :----------------------------------------------------------------------------------------------------------- | @@ -47,7 +66,7 @@ Next, you need to configure the appropriate scopes for the following streams. Pl | `contact_lists` | `crm.objects.lists.read` | | `contacts` | `crm.objects.contacts.read` | | `contacts_list_memberships` | `crm.objects.contacts.read` | -| Custom CRM OBjects | `crm.objects.custom.read` | +| Custom CRM Objects | `crm.objects.custom.read` | | `deal_pipelines` | `crm.objects.contacts.read` | | `deals` | `crm.objects.deals.read`, `crm.schemas.deals.read` | | `deals_archived` | `crm.objects.deals.read`, `crm.schemas.deals.read` | @@ -68,31 +87,61 @@ Next, you need to configure the appropriate scopes for the following streams. Pl ### Step 3: Set up the HubSpot source connector in Airbyte -1. Log in to your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account. + +**For Airbyte Cloud:** + +1. Log in to your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. 2. From the Airbyte UI, click **Sources**, then click on **+ New Source** and select **HubSpot** from the list of available sources. 3. Enter a **Source name** of your choosing. 4. From the **Authentication** dropdown, select your chosen authentication method: + - **Recommended:** To authenticate using OAuth, select **OAuth** and click **Authenticate your HubSpot account** to sign in with HubSpot and authorize your account. + :::tip HubSpot Authentication issues + You might encounter errors during the connection process in the popup window, such as `An invalid scope name was provided`. + To resolve this, close the window and attempt authentication again. + ::: + - **Not Recommended:**To authenticate using a Private App, select **Private App** and enter the Access Token for your HubSpot account. +5. For **Start date**, use the provided datepicker or enter the date programmatically in the following format: + `yyyy-mm-ddThh:mm:ssZ`. The data added on and after this date will be replicated. +6. Click **Set up source** and wait for the tests to complete. + - + +#### For Airbyte Open Source: -#### For Airbyte Cloud users: +1. Navigate to the Airbyte Open Source dashboard. +2. From the Airbyte UI, click **Sources**, then click on **+ New Source** and select **HubSpot** from the list of available sources. +3. Enter a **Source name** of your choosing. +4. From the **Authentication** dropdown, select your chosen authentication method: + - **Recommended:** To authenticate using a Private App, select **Private App** and enter the Access Token for your HubSpot account. + - **Not Recommended:**To authenticate using OAuth, select **OAuth** and enter your Client ID, Client Secret, and Refresh Token. +5. For **Start date**, use the provided datepicker or enter the date programmatically in the following format: + `yyyy-mm-ddThh:mm:ssZ`. The data added on and after this date will be replicated. +6. Click **Set up source** and wait for the tests to complete. -- **Recommended:** To authenticate using OAuth, select **OAuth** and click **Authenticate your HubSpot account** to sign in with HubSpot and authorize your account. -- **Not Recommended:**To authenticate using a Private App, select **Private App** and enter the Access Token for your HubSpot account. - + - +### Experimental streams -#### For Airbyte Open Source users: +[Web Analytics](https://developers.hubspot.com/docs/api/events/web-analytics) streams may be enabled as an experimental feature but please note that they are based on API which is currently in beta and may change at some point of time or be unstable. -- **Recommended:** To authenticate using a Private App, select **Private App** and enter the Access Token for your HubSpot account. -- **Not Recommended:**To authenticate using OAuth, select **OAuth** and enter your Client ID, Client Secret, and Refresh Token. + - +### Custom CRM Objects -5. For **Start date**, use the provided datepicker or enter the date programmatically in the following format: - `yyyy-mm-ddThh:mm:ssZ`. The data added on and after this date will be replicated. -6. Click **Set up source** and wait for the tests to complete. +Custom CRM Objects and Custom Web Analytics will appear as streams available for sync, alongside the standard objects listed above. + +If you set up your connections before April 15th, 2023 (on Airbyte Cloud) or before 0.8.0 (OSS) then you'll need to do some additional work to sync custom CRM objects. + +First you need to give the connector some additional permissions: + + +- **If you are using OAuth on Airbyte Cloud** go to the Hubspot source settings page in the Airbyte UI and re-authenticate via OAuth to allow Airbyte the permissions to access custom objects. + +- **If you are using OAuth on OSS or Private App auth** go into the Hubspot UI where you created your Private App or OAuth application and add the `crm.objects.custom.read` scope to your app's scopes. See HubSpot's instructions [here](https://developers.hubspot.com/docs/api/working-with-oauth#scopes). + +Then, go to the replication settings of your connection and click **refresh source schema** to pull in those new streams for syncing. + + ## Supported sync modes @@ -117,6 +166,7 @@ The HubSpot source connector supports the following streams: - [Contact Lists](http://developers.hubspot.com/docs/methods/lists/get_lists) \(Incremental\) - [Contacts](https://developers.hubspot.com/docs/methods/contacts/get_contacts) \(Incremental\) - [Contacts List Memberships](https://legacydocs.hubspot.com/docs/methods/contacts/get_contacts) +- [Contacts Merged Audit](https://legacydocs.hubspot.com/docs/methods/contacts/get_batch_by_vid) - [Deal Pipelines](https://developers.hubspot.com/docs/methods/pipelines/get_pipelines_for_object_type) \(Client-Side Incremental\) - [Deals](https://developers.hubspot.com/docs/api/crm/deals) \(including Contact associations\) \(Incremental\) - Records that have been deleted (archived) and stored in HubSpot's recycle bin will only be kept for 90 days, see [response from HubSpot Team](https://community.hubspot.com/t5/APIs-Integrations/Archived-deals-deleted-or-different/m-p/714157) @@ -135,25 +185,27 @@ The HubSpot source connector supports the following streams: - [Line Items](https://developers.hubspot.com/docs/api/crm/line-items) \(Incremental\) - [Marketing Emails](https://legacydocs.hubspot.com/docs/methods/cms_email/get-all-marketing-email-statistics) - [Owners](https://developers.hubspot.com/docs/methods/owners/get_owners) \(Client-Side Incremental\) +- [Owners Archived](https://legacydocs.hubspot.com/docs/methods/owners/get_owners) \(Client-Side Incremental) - [Products](https://developers.hubspot.com/docs/api/crm/products) \(Incremental\) -- [Property History](https://legacydocs.hubspot.com/docs/methods/contacts/get_contacts) \(Incremental\) +- [Contacts Property History](https://legacydocs.hubspot.com/docs/methods/contacts/get_contacts) \(Client-Side Incremental\) +- [Companies Property History](https://legacydocs.hubspot.com/docs/methods/companies/get-all-companies) \(Client-Side Incremental\) +- [Deals Property History](https://legacydocs.hubspot.com/docs/methods/deals/get-all-deals) \(Client-Side Incremental\) - [Subscription Changes](https://developers.hubspot.com/docs/methods/email/get_subscriptions_timeline) \(Incremental\) - [Tickets](https://developers.hubspot.com/docs/api/crm/tickets) \(Incremental\) - [Ticket Pipelines](https://developers.hubspot.com/docs/api/crm/pipelines) \(Client-Side Incremental\) - [Workflows](https://legacydocs.hubspot.com/docs/methods/workflows/v3/get_workflows) \(Client-Side Incremental\) - -### Custom CRM Objects - -Custom CRM Objects will appear as streams available for sync, alongside the standard objects listed above. - -If you set up your connections before April 15th, 2023 (on Cloud) or before 0.8.0 (OSS) then you'll need to do some additional work to sync custom CRM objects. - -First you need to give the connector some additional permissions: - -- **If you are using OAuth on Cloud** go to the Hubspot source settings page in the Airbyte UI and reauthenticate via Oauth to allow Airbyte the permissions to access custom objects. -- **If you are using OAuth on OSS or Private App auth (on OSS or Cloud)**: you'll need to go into the Hubspot UI where you created your Private App or OAuth application and add the `crm.objects.custom.read` scope to your app's scopes. See HubSpot's instructions [here](https://developers.hubspot.com/docs/api/working-with-oauth#scopes). - -Then, go to the replication settings of your connection and click **refresh source schema** to pull in those new streams for syncing. +- [ContactsWebAnalytics](https://developers.hubspot.com/docs/api/events/web-analytics) \(Incremental\) +- [CompaniesWebAnalytics](https://developers.hubspot.com/docs/api/events/web-analytics) \(Incremental\) +- [DealsWebAnalytics](https://developers.hubspot.com/docs/api/events/web-analytics) \(Incremental\) +- [TicketsWebAnalytics](https://developers.hubspot.com/docs/api/events/web-analytics) \(Incremental\) +- [EngagementsCallsWebAnalytics](https://developers.hubspot.com/docs/api/events/web-analytics) \(Incremental\) +- [EngagementsEmailsWebAnalytics](https://developers.hubspot.com/docs/api/events/web-analytics) \(Incremental\) +- [EngagementsMeetingsWebAnalytics](https://developers.hubspot.com/docs/api/events/web-analytics) \(Incremental\) +- [EngagementsNotesWebAnalytics](https://developers.hubspot.com/docs/api/events/web-analytics) \(Incremental\) +- [EngagementsTasksWebAnalytics](https://developers.hubspot.com/docs/api/events/web-analytics) \(Incremental\) +- [GoalsWebAnalytics](https://developers.hubspot.com/docs/api/events/web-analytics) \(Incremental\) +- [LineItemsWebAnalytics](https://developers.hubspot.com/docs/api/events/web-analytics) \(Incremental\) +- [ProductsWebAnalytics](https://developers.hubspot.com/docs/api/events/web-analytics) \(Incremental\) ### Notes on the `engagements` stream @@ -174,42 +226,100 @@ Then, go to the replication settings of your connection and click **refresh sour Because of this, the `engagements` stream can be slow to sync if it hasn't synced within the last 30 days and/or is generating large volumes of new data. We therefore recommend scheduling frequent syncs. -## Performance considerations +## Limitations & Troubleshooting + +
      + +Expand to see details about Hubspot connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting The connector is restricted by normal HubSpot [rate limitations](https://legacydocs.hubspot.com/apps/api_guidelines). -Some streams, such as `workflows`, need to be enabled before they can be read using a connector authenticated using an `API Key`. If reading a stream that is not enabled, a log message returned to the output and the sync operation only sync the other streams available. +### Troubleshooting + +* Consider checking out the following Hubspot tutorial: [Build a single customer view with open-source tools](https://airbyte.com/tutorials/single-customer-view). +* **Enabling streams:** Some streams, such as `workflows`, need to be enabled before they can be read using a connector authenticated using an `API Key`. If reading a stream that is not enabled, a log message returned to the output and the sync operation will only sync the other streams available. + + Example of the output message when trying to read `workflows` stream with missing permissions for the `API Key`: + + ```json + { + "type": "LOG", + "log": { + "level": "WARN", + "message": "Stream `workflows` cannot be proceed. This API Key (EXAMPLE_API_KEY) does not have proper permissions! (requires any of [automation-access])" + } + } + ``` + +* **Unnesting top level properties**: Since version 1.5.0, in order to not make the users query their destinations for complicated json fields, we duplicate most of nested data as top level fields. -Example of the output message when trying to read `workflows` stream with missing permissions for the `API Key`: + For instance: -```text -{ - "type": "LOG", - "log": { - "level": "WARN", - "message": 'Stream `workflows` cannot be proceed. This API Key (EXAMPLE_API_KEY) does not have proper permissions! (requires any of [automation-access])' + ```json + { + "id": 1, + "updatedAt": "2020-01-01", + "properties": { + "hs_note_body": "World's best boss", + "hs_created_by": "Michael Scott" + } } -} -``` + ``` + + becomes + + ```json + { + "id": 1, + "updatedAt": "2020-01-01", + "properties": { + "hs_note_body": "World's best boss", + "hs_created_by": "Michael Scott" + }, + "properties_hs_note_body": "World's best boss", + "properties_hs_created_by": "Michael Scott" + } + ``` +* **403 Forbidden Error** + * Hubspot has **scopes** for each API call. + * Each stream is tied to a scope and will need access to that scope to sync data. + * Review the Hubspot OAuth scope documentation [here](https://developers.hubspot.com/docs/api/working-with-oauth#scopes). + * Additional permissions: -HubSpot's API will [rate limit](https://developers.hubspot.com/docs/api/usage-details) the amount of records you can sync daily, so make sure that you are on the appropriate plan if you are planning on syncing more than 250,000 records per day. + `feedback_submissions`: Service Hub Professional account -## Tutorials + `marketing_emails`: Market Hub Starter account -Now that you have set up the Hubspot source connector, check out the following Hubspot tutorial: + `workflows`: Sales, Service, and Marketing Hub Professional accounts +* Check out common troubleshooting issues for the Hubspot source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). -[Build a single customer view with open-source tools](https://airbyte.com/tutorials/single-customer-view) +
      ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1.4.1 | 2023-08-22 | [29715](https://github.com/airbytehq/airbyte/pull/29715) | Fix python package configuration stream | -| 1.4.0 | 2023-08-11 | [29249](https://github.com/airbytehq/airbyte/pull/29249) | Add `OwnersArchived` stream | -| 1.3.3 | 2023-08-10 | [29248](https://github.com/airbytehq/airbyte/pull/29248) | Specify `threadId` in `engagements` stream to type string | -| 1.3.2 | 2023-08-10 | [29326](https://github.com/airbytehq/airbyte/pull/29326) | Add primary keys to streams `ContactLists` and `PropertyHistory` | -| 1.3.1 | 2023-08-08 | [29211](https://github.com/airbytehq/airbyte/pull/29211) | Handle 400 and 403 errors without interruption of the sync | -| 1.3.0 | 2023-08-01 | [28909](https://github.com/airbytehq/airbyte/pull/28909) | Add handling of source connection errors | +|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 2.0.2 | 2023-12-15 | [33844](https://github.com/airbytehq/airbyte/pull/33844) | Make property_history PK combined to support Incremental/Deduped sync type | +| 2.0.1 | 2023-12-15 | [33527](https://github.com/airbytehq/airbyte/pull/33527) | Make query string calculated correctly for ProertyHistory streams to avoid 414 HTTP Errors | +| 2.0.0 | 2023-12-08 | [33266](https://github.com/airbytehq/airbyte/pull/33266) | Added ContactsPropertyHistory, CompaniesPropertyHistory, DealsPropertyHistory streams | +| 1.9.0 | 2023-12-04 | [33042](https://github.com/airbytehq/airbyte/pull/33042) | Added Web Analytics streams | +| 1.8.0 | 2023-11-23 | [32778](https://github.com/airbytehq/airbyte/pull/32778) | Extend `PropertyHistory` stream to support incremental sync | +| 1.7.0 | 2023-11-01 | [32035](https://github.com/airbytehq/airbyte/pull/32035) | Extend the `Forms` stream schema | +| 1.6.1 | 2023-10-20 | [31644](https://github.com/airbytehq/airbyte/pull/31644) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 1.6.0 | 2023-10-19 | [31606](https://github.com/airbytehq/airbyte/pull/31606) | Added new field `aifeatures` to the `marketing emails` stream schema | +| 1.5.1 | 2023-10-04 | [31050](https://github.com/airbytehq/airbyte/pull/31050) | Add type transformer for `Engagements` stream | +| 1.5.0 | 2023-09-11 | [30322](https://github.com/airbytehq/airbyte/pull/30322) | Unnest stream schemas | +| 1.4.1 | 2023-08-22 | [29715](https://github.com/airbytehq/airbyte/pull/29715) | Fix python package configuration stream | +| 1.4.0 | 2023-08-11 | [29249](https://github.com/airbytehq/airbyte/pull/29249) | Add `OwnersArchived` stream | +| 1.3.3 | 2023-08-10 | [29248](https://github.com/airbytehq/airbyte/pull/29248) | Specify `threadId` in `engagements` stream to type string | +| 1.3.2 | 2023-08-10 | [29326](https://github.com/airbytehq/airbyte/pull/29326) | Add primary keys to streams `ContactLists` and `PropertyHistory` | +| 1.3.1 | 2023-08-08 | [29211](https://github.com/airbytehq/airbyte/pull/29211) | Handle 400 and 403 errors without interruption of the sync | +| 1.3.0 | 2023-08-01 | [28909](https://github.com/airbytehq/airbyte/pull/28909) | Add handling of source connection errors | | 1.2.0 | 2023-07-27 | [27091](https://github.com/airbytehq/airbyte/pull/27091) | Add new stream `ContactsMergedAudit` | | 1.1.2 | 2023-07-27 | [28558](https://github.com/airbytehq/airbyte/pull/28558) | Improve error messages during connector setup | | 1.1.1 | 2023-07-25 | [28705](https://github.com/airbytehq/airbyte/pull/28705) | Fix retry handler for token expired error | @@ -315,3 +425,5 @@ Now that you have set up the Hubspot source connector, check out the following H | 0.1.9 | 2021-08-11 | [5334](https://github.com/airbytehq/airbyte/pull/5334) | Fix empty strings inside float datatype | | 0.1.8 | 2021-08-06 | [5250](https://github.com/airbytehq/airbyte/pull/5250) | Fix issue with printing exceptions | | 0.1.7 | 2021-07-27 | [4913](https://github.com/airbytehq/airbyte/pull/4913) | Update fields schema | + +
      diff --git a/docs/integrations/sources/insightly.md b/docs/integrations/sources/insightly.md index b51287925972..24c5aa71dba9 100644 --- a/docs/integrations/sources/insightly.md +++ b/docs/integrations/sources/insightly.md @@ -71,6 +71,7 @@ The connector is restricted by Insightly [requests limitation](https://api.na1.i | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------- | +| 0.2.0 | 2023-10-23 |[31162](https://github.com/airbytehq/airbyte/pull/31162) | Migrate to low-code framework | | 0.1.3 | 2023-05-15 |[26079](https://github.com/airbytehq/airbyte/pull/26079) | Make incremental syncs timestamp inclusive | | 0.1.2 | 2023-03-23 |[24422](https://github.com/airbytehq/airbyte/pull/24422) | Fix incremental timedelta causing missing records | | 0.1.1 | 2022-11-11 |[19356](https://github.com/airbytehq/airbyte/pull/19356) | Fix state date parse bug | diff --git a/docs/integrations/sources/instagram-migrations.md b/docs/integrations/sources/instagram-migrations.md new file mode 100644 index 000000000000..49326bc1e4f8 --- /dev/null +++ b/docs/integrations/sources/instagram-migrations.md @@ -0,0 +1,74 @@ +# Instagram Migration Guide + +## Upgrading to 3.0.0 + +The Instagram connector has been upgrade to API v18 (following the deprecation of v11). Connector will be upgraded to API v18. Affected Streams and their corresponding changes are listed below: + +- `Media Insights` + + Old metric will be replaced with the new ones, refer to the [IG Media Insights](https://developers.facebook.com/docs/instagram-api/reference/ig-media/insights#metrics) for more info. + + | Old metric | New metric | + |----------------------------|--------------------| + | carousel_album_engagement | total_interactions | + | carousel_album_impressions | impressions | + | carousel_album_reach | reach | + | carousel_album_saved | saved | + | carousel_album_video_views | video_views | + | engagement | total_interactions | + +:::note + +You may see different results: `engagement` count includes likes, comments, and saves while `total_interactions` count includes likes, comments, and saves, as well as shares. + +::: + + New metrics for Reels: `ig_reels_avg_watch_time`, `ig_reels_video_view_total_time` + +- `User Lifetime Insights` + + - Metric `audience_locale` will become unavailable. + - Metrics `audience_city`, `audience_country`, and `audience_gender_age` will be consolidated into a single metric named `follower_demographics`, featuring respective breakdowns for `city`, `country`, and `age,gender`. + - Primary key will be changed to `["business_account_id", "breakdown"]`. + +:::note + +Due to Instagram limitations, the "Metric Type" will be set to `total_value` for `follower_demographics` metric. Refer to the [docs](https://developers.facebook.com/docs/instagram-api/reference/ig-user/insights#metric-type) for more info. + +::: + + +- `Story Insights` + + Metrics: `exits`, `taps_back`, `taps_forward` will become unavailable. + + +Please follow the instructions below to migrate to version 3.0.0: + +1. Select **Connections** in the main navbar. +1.1 Select the connection(s) affected by the update. +2. Select the **Replication** tab. +2.1 Select **Refresh source schema**. + ```note + Any detected schema changes will be listed for your review. + ``` +2.2 Select **OK**. +3. Select **Save changes** at the bottom of the page. +3.1 Ensure the **Reset affected streams** option is checked. + ```note + Depending on destination type you may not be prompted to reset your data + ``` +4. Select **Save connection**. + ```note + This will reset the data in your destination and initiate a fresh sync. + ``` + +For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). + +## Upgrading to 2.0.0 + +This release adds a default primary key for the streams UserLifetimeInsights and UserInsights, and updates the format of timestamp fields in the UserLifetimeInsights, UserInsights, Media and Stories streams to include timezone information. + +To ensure uninterrupted syncs, users should: +- Refresh the source schema +- Reset affected streams \ No newline at end of file diff --git a/docs/integrations/sources/instagram.inapp.md b/docs/integrations/sources/instagram.inapp.md deleted file mode 100644 index 6be4f2da558e..000000000000 --- a/docs/integrations/sources/instagram.inapp.md +++ /dev/null @@ -1,17 +0,0 @@ -## Prerequisite - -* [Instagram business account](https://www.facebook.com/business/help/898752960195806) to your Facebook page - -:::info -The Instagram connector syncs data related to Users, Media, and Stories and their insights from the [Instagram Graph API](https://developers.facebook.com/docs/instagram-api/). For performance data related to Instagram Ads, use the Facebook Marketing source. -::: - -## Setup guide - -1. Click Authenticate your Instagram account. -2. Log in and authorize the Instagram account. -3. (Optional) Select a start date date. All data generated after this date will be replicated. If this field is blank, Airbyte will replicate all data. -4. Click Set up source. -​ - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Instagram](https://docs.airbyte.com/integrations/sources/instagram). \ No newline at end of file diff --git a/docs/integrations/sources/instagram.md b/docs/integrations/sources/instagram.md index cfe5d8b7c9a6..9d6e0eb3e4ca 100644 --- a/docs/integrations/sources/instagram.md +++ b/docs/integrations/sources/instagram.md @@ -1,14 +1,20 @@ # Instagram -This page contains the setup guide and reference information for the Instagram source connector. + + +This page contains the setup guide and reference information for the [Instagram](https://www.instagram.com/) source connector. + + ## Prerequisites - [Meta for Developers account](https://developers.facebook.com) - [Instagram business account](https://www.facebook.com/business/help/898752960195806) to your Facebook page +- [Facebook ad account ID number](https://www.facebook.com/business/help/1492627900875762) (you'll use this to configure Instagram as a source in Airbyte + - [Instagram Graph API](https://developers.facebook.com/docs/instagram-api/) to your Facebook app -- [Facebook OAuth Reference](https://developers.facebook.com/docs/instagram-basic-display-api/reference) -- [Facebook ad account ID number](https://www.facebook.com/business/help/1492627900875762) (you'll use this to configure Instagram as a source in Airbyte) +- [Facebook Instagram OAuth Reference](https://developers.facebook.com/docs/instagram-basic-display-api/reference) + ## Setup Guide @@ -24,7 +30,7 @@ This page contains the setup guide and reference information for the Instagram s 4. Enter a name for your source. 5. Click **Authenticate your Instagram account**. 6. Log in and authorize the Instagram account. -7. Enter the **Start Date** in YYYY-MM-DDTHH:mm:ssZ format. All data generated after this date will be replicated. If this field is blank, Airbyte will replicate all data. +7. (Optional) Enter the **Start Date** in YYYY-MM-DDTHH:mm:ssZ format. All data generated after this date will be replicated. If left blank, the start date will be set to 2 years before the present date. 8. Click **Set up source**. @@ -36,12 +42,13 @@ This page contains the setup guide and reference information for the Instagram s 2. Click **Sources** and then click **+ New source**. 3. On the Set up the source page, select **Instagram** from the **Source type** dropdown. 4. Enter a name for your source. -5. Click **Authenticate your Instagram account**. -6. Log in and authorize the Instagram account. -7. Enter the **Start Date** in YYYY-MM-DDTHH:mm:ssZ format. All data generated after this date will be replicated. If this field is blank, Airbyte will replicate all data. -8. Click **Set up source**. +5. Enter **Access Token** generated using [Graph API Explorer](https://developers.facebook.com/tools/explorer/) or [by using an app you can create on Facebook](https://developers.facebook.com/docs/instagram-api/getting-started) with the required permissions: instagram_basic, instagram_manage_insights, pages_show_list, pages_read_engagement. +6. (Optional) Enter the **Start Date** in YYYY-MM-DDTHH:mm:ssZ format. All data generated after this date will be replicated. If left blank, the start date will be set to 2 years before the present date. +7. Click **Set up source**. + + ## Supported sync modes The Instagram source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): @@ -68,25 +75,54 @@ The Instagram source connector supports the following streams. For more informat - [Stories](https://developers.facebook.com/docs/instagram-api/reference/ig-user/stories/) - [Story Insights](https://developers.facebook.com/docs/instagram-api/reference/ig-media/insights) -### Rate Limiting and Performance Considerations - -Instagram limits the number of requests that can be made at a time, but the Instagram connector gracefully handles rate limiting. See Facebook's [documentation on rate limiting](https://developers.facebook.com/docs/graph-api/overview/rate-limiting/#instagram-graph-api) for more information. +:::info +The Instagram connector syncs data related to Users, Media, and Stories and their insights from the [Instagram Graph API](https://developers.facebook.com/docs/instagram-api/). For performance data related to Instagram Ads, use the Facebook Marketing source. +::: ## Data type map AirbyteRecords are required to conform to the [Airbyte type](https://docs.airbyte.com/understanding-airbyte/supported-data-types/) system. This means that all sources must produce schemas and records within these types and all destinations must handle records that conform to this type system. | Integration Type | Airbyte Type | -| :--------------- | :----------- | +|:-----------------|:-------------| | `string` | `string` | | `number` | `number` | | `array` | `array` | | `object` | `object` | +## Limitations & Troubleshooting + +
      + +Expand to see details about Instagram connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting + +Instagram limits the number of requests that can be made at a time. See Facebook's [documentation on rate limiting](https://developers.facebook.com/docs/graph-api/overview/rate-limiting/#instagram-graph-api) for more information. + +### Troubleshooting + +* Check out common troubleshooting issues for the Instagram source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +
      + ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------ | +|:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------| +| 3.0.2 | 2024-01-15 | [34254](https://github.com/airbytehq/airbyte/pull/34254) | prepare for airbyte-lib | +| 3.0.1 | 2024-01-08 | [33989](https://github.com/airbytehq/airbyte/pull/33989) | Remove metrics from video feed | +| 3.0.0 | 2024-01-05 | [33930](https://github.com/airbytehq/airbyte/pull/33930) | Upgrade to API v18.0 | +| 2.0.1 | 2024-01-03 | [33889](https://github.com/airbytehq/airbyte/pull/33889) | Change requested metrics for stream `media_insights` | +| 2.0.0 | 2023-11-17 | [32500](https://github.com/airbytehq/airbyte/pull/32500) | Add primary keys for UserLifetimeInsights and UserInsights; add airbyte_type to timestamp fields | +| 1.0.16 | 2023-11-17 | [32627](https://github.com/airbytehq/airbyte/pull/32627) | Fix start_date type; fix docs | +| 1.0.15 | 2023-11-14 | [32494](https://github.com/airbytehq/airbyte/pull/32494) | Marked start_date as optional; set max retry time to 10 minutes; add suggested streams | +| 1.0.14 | 2023-11-13 | [32423](https://github.com/airbytehq/airbyte/pull/32423) | Capture media_product_type column in media and stories stream | +| 1.0.13 | 2023-11-10 | [32245](https://github.com/airbytehq/airbyte/pull/32245) | Add skipping reading MediaInsights stream if an error code 10 is received | +| 1.0.12 | 2023-11-07 | [32200](https://github.com/airbytehq/airbyte/pull/32200) | The backoff strategy has been updated to make some errors retriable | | 1.0.11 | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes | | 1.0.10 | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references | | 1.0.9 | 2023-07-01 | [27908](https://github.com/airbytehq/airbyte/pull/27908) | Fix bug when `user_lifetime_insights` stream returns `Key Error (end_time)`, refactored `state` to use `IncrementalMixin` | @@ -105,3 +141,5 @@ AirbyteRecords are required to conform to the [Airbyte type](https://docs.airbyt | 0.1.8 | 2021-08-11 | [5354](https://github.com/airbytehq/airbyte/pull/5354) | Added check for empty state and fixed tests | | 0.1.7 | 2021-07-19 | [4805](https://github.com/airbytehq/airbyte/pull/4805) | Add support for previous `STATE` format | | 0.1.6 | 2021-07-07 | [4210](https://github.com/airbytehq/airbyte/pull/4210) | Refactor connector to use CDK: - improve error handling - fix sync fail with HTTP status 400 - integrate SAT | + +
      diff --git a/docs/integrations/sources/intercom.md b/docs/integrations/sources/intercom.md index 82bbdc6e68cf..e3bffdb9aa56 100644 --- a/docs/integrations/sources/intercom.md +++ b/docs/integrations/sources/intercom.md @@ -73,7 +73,10 @@ The Intercom connector should not run into Intercom API limitations under normal ## Changelog | Version | Date | Pull Request | Subject | -|:--------| :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------- | +|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------| +| 0.4.0 | 2024-01-11 | [33882](https://github.com/airbytehq/airbyte/pull/33882) | Add new stream `Activity Logs` | +| 0.3.2 | 2023-12-07 | [33223](https://github.com/airbytehq/airbyte/pull/33223) | Ignore 404 error for `Conversation Parts` | +| 0.3.1 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | | 0.3.0 | 2023-05-25 | [29598](https://github.com/airbytehq/airbyte/pull/29598) | Update custom components to make them compatible with latest cdk version, simplify logic, update schemas | | 0.2.1 | 2023-05-25 | [26571](https://github.com/airbytehq/airbyte/pull/26571) | Remove authSpecification from spec.json in favour of advancedAuth | | 0.2.0 | 2023-04-05 | [23013](https://github.com/airbytehq/airbyte/pull/23013) | Migrated to Low-code (YAML Frramework) | @@ -110,4 +113,4 @@ The Intercom connector should not run into Intercom API limitations under normal | 0.1.3 | 2021-09-08 | [5908](https://github.com/airbytehq/airbyte/pull/5908) | Corrected timestamp and arrays in schemas | | 0.1.2 | 2021-08-19 | [5531](https://github.com/airbytehq/airbyte/pull/5531) | Corrected pagination | | 0.1.1 | 2021-07-31 | [5123](https://github.com/airbytehq/airbyte/pull/5123) | Corrected rate limit | -| 0.1.0 | 2021-07-19 | [4676](https://github.com/airbytehq/airbyte/pull/4676) | Release Intercom CDK Connector | +| 0.1.0 | 2021-07-19 | [4676](https://github.com/airbytehq/airbyte/pull/4676) | Release Intercom CDK Connector | \ No newline at end of file diff --git a/docs/integrations/sources/iterable.md b/docs/integrations/sources/iterable.md index ec0bb73ea0fb..0b68d7856f2f 100644 --- a/docs/integrations/sources/iterable.md +++ b/docs/integrations/sources/iterable.md @@ -80,6 +80,9 @@ The Iterable source connector supports the following [sync modes](https://docs.a | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :----------------------------------------------------------------------------------------------------------- | +| 0.2.1 | 2024-01-12 | [1234](https://github.com/airbytehq/airbyte/pull/1234) | prepare for airbyte-lib | +| 0.2.0 | 2023-09-29 | [28457](https://github.com/airbytehq/airbyte/pull/30931) | Added `userId` to `email_bounce`, `email_click`, `email_complaint`, `email_open`, `email_send` `email_send_skip`, `email_subscribe`, `email_unsubscribe`, `events` streams | +| 0.1.31 | 2023-12-06 | [33106](https://github.com/airbytehq/airbyte/pull/33106) | Base image migration: remove Dockerfile and use the python-connector-base image | | 0.1.30 | 2023-07-19 | [28457](https://github.com/airbytehq/airbyte/pull/28457) | Fixed TypeError for StreamSlice in debug mode | | 0.1.29 | 2023-05-24 | [26459](https://github.com/airbytehq/airbyte/pull/26459) | Added requests reading timeout 300 seconds | | 0.1.28 | 2023-05-12 | [26014](https://github.com/airbytehq/airbyte/pull/26014) | Improve 500 handling for Events stream | diff --git a/docs/integrations/sources/jira-migrations.md b/docs/integrations/sources/jira-migrations.md new file mode 100644 index 000000000000..9dc0955b49d2 --- /dev/null +++ b/docs/integrations/sources/jira-migrations.md @@ -0,0 +1,27 @@ +# Jira Migration Guide + +## Upgrading to 1.0.0 + +Note: this change is only breaking if you are using the `Boards Issues` stream in Incremental Sync mode. + +This is a breaking change because Stream State for `Boards Issues` will be changed, so please follow the instructions below to migrate to version 1.0.0: + +1. Select **Connections** in the main navbar. +1.1 Select the connection(s) affected by the update. +2. Select the **Replication** tab. +2.1 Select **Refresh source schema**. + ```note + Any detected schema changes will be listed for your review. + ``` +2.2 Select **OK**. +3. Select **Save changes** at the bottom of the page. +3.1 Ensure the **Reset affected streams** option is checked. + ```note + Depending on destination type you may not be prompted to reset your data + ``` +4. Select **Save connection**. + ```note + This will reset the data in your destination and initiate a fresh sync. + ``` + +For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). \ No newline at end of file diff --git a/docs/integrations/sources/jira.inapp.md b/docs/integrations/sources/jira.inapp.md deleted file mode 100644 index 671fba287cfc..000000000000 --- a/docs/integrations/sources/jira.inapp.md +++ /dev/null @@ -1,20 +0,0 @@ -## Prerequisites - -- Access to a JIRA account -- [JIRA API Token](https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/) -- JIRA Account Domain - -## Setup guide - -1. Enter a name for the connector. -2. Enter the **API Token** that you have created. The **API Token** is used for Authorization to your account. -2. Enter the **Domain** for your Jira account, e.g. `airbyte.atlassian.net`. -3. Enter the **Email** for your Jira account which you used to generate the API token. This field is used for Authorization to your account. -4. (Optional) Enter the list of **Projects** for which you need to replicate data. If empty, data from all projects will be replicated. -5. (Optional) Enter the **Start Date** from which you'd like to replicate data for Jira in the format YYYY-MM-DDTHH:MM:SSZ. All data generated after this date will be replicated. If empty, all data will be replicated. Note that it will be used only in the following streams: `BoardIssues`, `IssueComments`, `IssueProperties`, `IssueRemoteLinks`, `IssueVotes`, `IssueWatchers`, `IssueWorklogs`, `Issues`, `PullRequests`, `SprintIssues`. For other streams, it will replicate all data. -9. Toggle **Expand Issue Changelog** to get a list of updates to every issue in the Issues stream. If the toggle is off, the changelog will not be pulled. -10. Toggle **Render Issue Fields** to additionally return field values rendered in HTML format in the Issues stream. Issue fields will always be returned in JSON format. -11. Toggle **Enable Experimental Streams** to enable syncing for undocumented internal JIRA API endpoints and may stop working if those enpoints undergo major changes. Currently, this only applies to the PullRequests stream. -10. Click **Set up source** - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [JIRA](https://docs.airbyte.com/integrations/sources/jira). diff --git a/docs/integrations/sources/jira.md b/docs/integrations/sources/jira.md index aea4d3cac20d..1e579b009d0b 100644 --- a/docs/integrations/sources/jira.md +++ b/docs/integrations/sources/jira.md @@ -25,10 +25,7 @@ This page contains the setup guide and reference information for the Jira source 5. Enter the **Domain** for your Jira account, e.g. `airbyteio.atlassian.net`. 6. Enter the **Email** for your Jira account which you used to generate the API token. This field is used for Authorization to your account by BasicAuth. 7. Enter the list of **Projects (Optional)** for which you need to replicate data, or leave it empty if you want to replicate data for all projects. -8. Enter the **Start Date (Optional)** from which you'd like to replicate data for Jira in the format YYYY-MM-DDTHH:MM:SSZ. All data generated after this date will be replicated, or leave it empty if you want to replicate all data. Note that it will be used only in the following streams:BoardIssues, IssueComments, IssueProperties, IssueRemoteLinks, IssueVotes, IssueWatchers, IssueWorklogs, Issues, PullRequests, SprintIssues. For other streams it will replicate all data. -9. Toggle **Expand Issue Changelog** allows you to get a list of recent updates to every issue in the Issues stream. -10. Toggle **Render Issue Fields** allows returning field values rendered in HTML format in the Issues stream. -11. Toggle **Enable Experimental Streams** enables experimental PullRequests stream. +8. Enter the **Start Date (Optional)** from which you'd like to replicate data for Jira in the format YYYY-MM-DDTHH:MM:SSZ. All data generated after this date will be replicated, or leave it empty if you want to replicate all data. Note that it will be used only in the following streams: Board Issues, Issue Comments, Issue Properties, Issue Remote Links, Issue Votes, Issue Watchers, Issue Worklogs, Issues, Pull Requests, Sprint Issues. For other streams it will replicate all data. ## Supported sync modes @@ -39,10 +36,6 @@ The Jira source connector supports the following [sync modes](https://docs.airby - [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) - [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped) -## Troubleshooting - -Check out common troubleshooting issues for the Jira connector on our Airbyte Forum [here](https://github.com/airbytehq/airbyte/discussions). - ## Supported Streams This connector outputs the following full refresh streams: @@ -57,6 +50,7 @@ This connector outputs the following full refresh streams: - [Issue fields](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-fields/#api-rest-api-3-field-get) - [Issue field configurations](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-field-configurations/#api-rest-api-3-fieldconfiguration-get) - [Issue custom field contexts](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-custom-field-contexts/#api-rest-api-3-field-fieldid-context-get) +- [Issue custom field options](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-custom-field-options/#api-rest-api-3-field-fieldid-context-contextid-option-get) - [Issue link types](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-link-types/#api-rest-api-3-issuelinktype-get) - [Issue navigator settings](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-navigator-settings/#api-rest-api-3-settings-columns-get) - [Issue notification schemes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-notification-schemes/#api-rest-api-3-notificationscheme-get) @@ -65,8 +59,10 @@ This connector outputs the following full refresh streams: - [Issue remote links](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-remote-links/#api-rest-api-3-issue-issueidorkey-remotelink-get) - [Issue resolutions](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-resolutions/#api-rest-api-3-resolution-search-get) - [Issue security schemes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-security-schemes/#api-rest-api-3-issuesecurityschemes-get) +- [Issue transitions](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issues/#api-rest-api-3-issue-issueidorkey-transitions-get) - [Issue type schemes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-type-schemes/#api-rest-api-3-issuetypescheme-get) - [Issue type screen schemes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-type-screen-schemes/#api-rest-api-3-issuetypescreenscheme-get) +- [Issue types](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-types/#api-group-issue-types) - [Issue votes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-votes/#api-group-issue-votes) - [Issue watchers](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-watchers/#api-rest-api-3-issue-issueidorkey-watchers-get) - [Jira settings](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-jira-settings/#api-rest-api-3-application-properties-get) @@ -79,6 +75,7 @@ This connector outputs the following full refresh streams: - [Project components](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-components/#api-rest-api-3-project-projectidorkey-component-get) - [Project email](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-email/#api-rest-api-3-project-projectid-email-get) - [Project permission schemes](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-permission-schemes/#api-group-project-permission-schemes) +- [Project roles](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-roles#api-rest-api-3-role-get) - [Project types](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-types/#api-rest-api-3-project-type-get) - [Project versions](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-project-versions/#api-rest-api-3-project-projectidorkey-version-get) - [Screens](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-screens/#api-rest-api-3-screens-get) @@ -101,6 +98,7 @@ This connector outputs the following incremental streams: - [Issue worklogs](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-worklogs/#api-rest-api-3-issue-issueidorkey-worklog-get) - [Issues](https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-issue-search/#api-rest-api-3-search-get) - [Sprint issues](https://developer.atlassian.com/cloud/jira/software/rest/api-group-sprint/#api-rest-agile-1-0-sprint-sprintid-issue-get) +- [PullRequests](https://docs.airbyte.com/integrations/sources/jira#experimental-tables) If there are more endpoints you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose) @@ -125,7 +123,28 @@ The Jira connector should not run into Jira API limitations under normal usage. ## CHANGELOG | Version | Date | Pull Request | Subject | -| :------ | :--------- | :--------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------- | +|:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------| +| 1.0.0 | 2024-01-01 | [33682](https://github.com/airbytehq/airbyte/pull/33682) | Save state for stream `Board Issues` per `board` | +| 0.14.1 | 2023-12-19 | [33625](https://github.com/airbytehq/airbyte/pull/33625) | Skip 404 error | +| 0.14.0 | 2023-12-15 | [33532](https://github.com/airbytehq/airbyte/pull/33532) | Add lookback window | +| 0.13.0 | 2023-12-12 | [33353](https://github.com/airbytehq/airbyte/pull/33353) | Fix check command to check access for all available streams | +| 0.12.0 | 2023-12-01 | [33011](https://github.com/airbytehq/airbyte/pull/33011) | Fix BoardIssues stream; increase number of retries for backoff policy to 10 | +| 0.11.0 | 2023-11-29 | [32927](https://github.com/airbytehq/airbyte/pull/32927) | Fix incremental syncs for stream Issues | +| 0.10.2 | 2023-10-26 | [31896](https://github.com/airbytehq/airbyte/pull/31896) | Provide better guidance when configuring the connector with an invalid domain | +| 0.10.1 | 2023-10-23 | [31702](https://github.com/airbytehq/airbyte/pull/31702) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 0.10.0 | 2023-10-13 | [\#31385](https://github.com/airbytehq/airbyte/pull/31385) | Fixed `aggregatetimeoriginalestimate, timeoriginalestimate` field types for the `Issues` stream schema | +| 0.9.0 | 2023-09-26 | [\#30688](https://github.com/airbytehq/airbyte/pull/30688) | Added `createdDate` field to sprints schema, Removed `Expand Issues stream` from spec | +| 0.8.0 | 2023-09-26 | [\#30755](https://github.com/airbytehq/airbyte/pull/30755) | Add new streams: `Issue custom field options`, `IssueTypes`, `Project Roles` | +| 0.7.2 | 2023-09-19 | [\#30675](https://github.com/airbytehq/airbyte/pull/30675) | Ensure invalid URL does not trigger Sentry alert | +| 0.7.1 | 2023-09-19 | [\#30585](https://github.com/airbytehq/airbyte/pull/30585) | Add skip for 404 error in issue properties steam | +| 0.7.0 | 2023-09-17 | [\#30532](https://github.com/airbytehq/airbyte/pull/30532) | Add foreign key to stream record where it misseing | +| 0.6.3 | 2023-09-19 | [\#30515](https://github.com/airbytehq/airbyte/pull/30515) | Add transform for invalid date-time format, add 404 handling for check | +| 0.6.2 | 2023-09-19 | [\#30578](https://github.com/airbytehq/airbyte/pull/30578) | Fetch deleted and archived Projects | +| 0.6.1 | 2023-09-17 | [\#30550](https://github.com/airbytehq/airbyte/pull/30550) | Update `Issues` expand settings | +| 0.6.0 | 2023-09-17 | [\#30507](https://github.com/airbytehq/airbyte/pull/30507) | Add new stream `IssueTransitions` | +| 0.5.0 | 2023-09-14 | [\#29960](https://github.com/airbytehq/airbyte/pull/29960) | Add `boardId` to `sprints` stream | +| 0.3.14 | 2023-09-11 | [\#30297](https://github.com/airbytehq/airbyte/pull/30297) | Remove `requests` and `pendulum` from setup dependencies | +| 0.3.13 | 2023-09-01 | [\#30108](https://github.com/airbytehq/airbyte/pull/30108) | Skip 404 error for stream `IssueWatchers` | | 0.3.12 | 2023-06-01 | [\#26652](https://github.com/airbytehq/airbyte/pull/26652) | Expand on `leads` for `projects` stream | | 0.3.11 | 2023-06-01 | [\#26906](https://github.com/airbytehq/airbyte/pull/26906) | Handle project permissions error | | 0.3.10 | 2023-05-26 | [\#26652](https://github.com/airbytehq/airbyte/pull/26652) | Fixed bug when `board` doesn't support `sprints` | @@ -158,4 +177,4 @@ The Jira connector should not run into Jira API limitations under normal usage. | 0.2.6 | 2021-06-15 | [\#4113](https://github.com/airbytehq/airbyte/pull/4113) | Fixed `user` stream with the correct endpoint and query param. | | 0.2.5 | 2021-06-09 | [\#3973](https://github.com/airbytehq/airbyte/pull/3973) | Added `AIRBYTE_ENTRYPOINT` in base Docker image for Kubernetes support. | | 0.2.4 | | | Implementing base_read acceptance test dived by stream groups. | -| 0.2.3 | | | Implementing incremental sync. Migrated to airbyte-cdk. Adding all available entities in Jira Cloud. | +| 0.2.3 | | | Implementing incremental sync. Migrated to airbyte-cdk. Adding all available entities in Jira Cloud. | \ No newline at end of file diff --git a/docs/integrations/sources/klarna.md b/docs/integrations/sources/klarna.md index a7699f9322f0..ff809bbb2a35 100644 --- a/docs/integrations/sources/klarna.md +++ b/docs/integrations/sources/klarna.md @@ -58,4 +58,5 @@ Connector will handle an issue with rate limiting as Klarna returns 429 status c | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------| +| 0.2.0 | 2023-10-23 | [31003](https://github.com/airbytehq/airbyte/pull/31003) | Migrate to low-code | | 0.1.0 | 2022-10-24 | [18385](https://github.com/airbytehq/airbyte/pull/18385) | Klarna Settlements Payout and Transactions API | diff --git a/docs/integrations/sources/klaus-api.md b/docs/integrations/sources/klaus-api.md new file mode 100644 index 000000000000..bf38b64ea60e --- /dev/null +++ b/docs/integrations/sources/klaus-api.md @@ -0,0 +1,34 @@ +# Klaus API + +## Sync overview + +The Klaus API source supports both Full Refresh only. + +This source can sync data for the [Klaus API](https://support.klausapp.com/en/collections/2212726-integrating-manually), +[Klaus Swagger](https://pub.klausapp.com/?urls.primaryName=Public%20API) + +### Output schema + +This Source is capable of syncing the following core Streams: + +- [users](https://pub.klausapp.com/?urls.primaryName=Public%20API#/PublicApi/PublicApi_UsersV2) +- [categories](https://pub.klausapp.com/?urls.primaryName=Public%20API#/PublicApi/PublicApi_RatingCategoriesV2) +- [reviews](https://pub.klausapp.com/?urls.primaryName=Public%20API#/PublicApi/PublicApi_ReviewsV2) + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :------------------------ |:---------------------| :---- | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | Yes | | +| Namespaces | No | | + +## Requirements + +- **Klaus API keys**. See the [Klaus API docs](https://support.klausapp.com/en/articles/4027272-setting-up-a-custom-integration) for information on how to obtain the API keys. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ |:-----------| :------------------------------------------------------- |:-------------------------------| +| 0.1.0 | 2023-05-04 | [25790](https://github.com/airbytehq/airbyte/pull/25790) | Add Klaus API Source Connector | diff --git a/docs/integrations/sources/klaviyo-migrations.md b/docs/integrations/sources/klaviyo-migrations.md new file mode 100644 index 000000000000..9dc27b89ad88 --- /dev/null +++ b/docs/integrations/sources/klaviyo-migrations.md @@ -0,0 +1,23 @@ +# Klaviyo Migration Guide + +## Upgrading to 2.0.0 + +Streams `campaigns`, `email_templates`, `events`, `flows`, `global_exclusions`, `lists`, and `metrics` are now pulling +data using latest API which has a different schema. Users will need to refresh the source schemas and reset these +streams after upgrading. See the chart below for the API version change. + +| Stream | Current API version | New API version | +|-------------------|---------------------|-----------------| +| campaigns | v1 | 2023-06-15 | +| email_templates | v1 | 2023-10-15 | +| events | v1 | 2023-10-15 | +| flows | v1 | 2023-10-15 | +| global_exclusions | v1 | 2023-10-15 | +| lists | v1 | 2023-10-15 | +| metrics | v1 | 2023-10-15 | +| profiles | 2023-02-22 | 2023-02-22 | + +## Upgrading to 1.0.0 + +`event_properties/items/quantity` for `Events` stream is changed from `integer` to `number`. +For a smooth migration, data reset and schema refresh are needed. \ No newline at end of file diff --git a/docs/integrations/sources/klaviyo.md b/docs/integrations/sources/klaviyo.md index e8d1ce170503..56aefb605150 100644 --- a/docs/integrations/sources/klaviyo.md +++ b/docs/integrations/sources/klaviyo.md @@ -4,16 +4,24 @@ This page contains the setup guide and reference information for the Klaviyo sou ## Prerequisites -To set up the Klaviyo source connector, you'll need the [Klaviyo Private API key](https://help.klaviyo.com/hc/en-us/articles/115005062267-How-to-Manage-Your-Account-s-API-Keys#your-private-api-keys3). +- Klaviyo [account](https://www.klaviyo.com) +- [Klaviyo Private API key](https://help.klaviyo.com/hc/en-us/articles/115005062267-How-to-Manage-Your-Account-s-API-Keys#your-private-api-keys3) -## Set up the Klaviyo connector in Airbyte +## Setup guide -1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) or navigate to the Airbyte Open Source dashboard. -2. Click **Sources** and then click **+ New source**. -3. On the Set up the source page, select **Klaviyo** from the Source type dropdown. -4. Enter the name for the Klaviyo connector. +### Step 1: Set up Klaviyo + +1. Create a [Klaviyo account](https://www.klaviyo.com) +2. Create a [Private API key](https://help.klaviyo.com/hc/en-us/articles/115005062267-How-to-Manage-Your-Account-s-API-Keys#your-private-api-keys3). Make sure you selected all [scopes](https://help.klaviyo.com/hc/en-us/articles/7423954176283) corresponding to the streams you would like to replicate. You can find which scope is required for a specific stream by navigating to the relevant API documentation for the streams Airbyte supports. + +### Step 2: Set up the Klaviyo connector in Airbyte + +1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account. +2. Click **Sources** and then click **+ new source**. +3. On the Set up the source page, select **Klaviyo** from the **Source type** dropdown. +4. Enter a name for the Klaviyo connector. 5. For **Api Key**, enter the Klaviyo [Private API key](https://help.klaviyo.com/hc/en-us/articles/115005062267-How-to-Manage-Your-Account-s-API-Keys#your-private-api-keys3). -6. For **Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. +6. For **Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. This field is optional - if not provided, all data will be replicated. 7. Click **Set up source**. ## Supported sync modes @@ -27,13 +35,14 @@ The Klaviyo source connector supports the following [sync modes](https://docs.ai ## Supported Streams -- [Campaigns](https://developers.klaviyo.com/en/v1-2/reference/get-campaigns#get-campaigns) -- [Events](https://developers.klaviyo.com/en/v1-2/reference/metrics-timeline) -- [GlobalExclusions](https://developers.klaviyo.com/en/v1-2/reference/get-global-exclusions) -- [Lists](https://developers.klaviyo.com/en/v1-2/reference/get-lists) -- [Metrics](https://developers.klaviyo.com/en/v1-2/reference/get-metrics) +- [Campaigns](https://developers.klaviyo.com/en/v2023-06-15/reference/get_campaigns) +- [Email Templates](https://developers.klaviyo.com/en/reference/get_templates) +- [Events](https://developers.klaviyo.com/en/reference/get_events) - [Flows](https://developers.klaviyo.com/en/reference/get_flows) -- [Profiles](https://developers.klaviyo.com/en/reference/get_profiles) +- [GlobalExclusions](https://developers.klaviyo.com/en/v2023-02-22/reference/get_profiles) +- [Lists](https://developers.klaviyo.com/en/reference/get_lists) +- [Metrics](https://developers.klaviyo.com/en/reference/get_metrics) +- [Profiles](https://developers.klaviyo.com/en/v2023-02-22/reference/get_profiles) ## Performance considerations @@ -52,19 +61,28 @@ The Klaviyo connector should not run into Klaviyo API limitations under normal u ## Changelog -| Version | Date | Pull Request | Subject | -| :------- | :--------- | :--------------------------------------------------------- | :---------------------------------------------------------------------------------------- | -| `0.3.2` | 2023-06-20 | [27498](https://github.com/airbytehq/airbyte/pull/27498) | Do not store state in the future | -| `0.3.1` | 2023-06-08 | [27162](https://github.com/airbytehq/airbyte/pull/27162) | Anonymize check connection error message | -| `0.3.0` | 2023-02-18 | [23236](https://github.com/airbytehq/airbyte/pull/23236) | Add ` Email Templates` stream | -| `0.2.0` | 2023-03-13 | [22942](https://github.com/airbytehq/airbyte/pull/23968) | Add `Profiles` stream | -| `0.1.13` | 2023-02-13 | [22942](https://github.com/airbytehq/airbyte/pull/22942) | Specified date formatting in specification | -| `0.1.12` | 2023-01-30 | [22071](https://github.com/airbytehq/airbyte/pull/22071) | Fix `Events` stream schema | -| `0.1.11` | 2023-01-27 | [22012](https://github.com/airbytehq/airbyte/pull/22012) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| `0.1.10` | 2022-09-29 | [17422](https://github.com/airbytehq/airbyte/issues/17422) | Update CDK dependency | -| `0.1.9` | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/issues/17304) | Migrate to per-stream state. | -| `0.1.6` | 2022-07-20 | [14872](https://github.com/airbytehq/airbyte/issues/14872) | Increase test coverage | -| `0.1.5` | 2022-07-12 | [14617](https://github.com/airbytehq/airbyte/issues/14617) | Set max_retries = 10 for `lists` stream. | -| `0.1.4` | 2022-04-15 | [11723](https://github.com/airbytehq/airbyte/issues/11723) | Enhance klaviyo source for flows stream and update to events stream. | -| `0.1.3` | 2021-12-09 | [8592](https://github.com/airbytehq/airbyte/pull/8592) | Improve performance, make Global Exclusions stream incremental and enable Metrics stream. | -| `0.1.2` | 2021-10-19 | [6952](https://github.com/airbytehq/airbyte/pull/6952) | Update schema validation in SAT | +| Version | Date | Pull Request | Subject | +|:---------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------| +| `2.1.0` | 2023-12-07 | [33237](https://github.com/airbytehq/airbyte/pull/33237) | Continue syncing streams even when one of the stream fails | +| `2.0.2` | 2023-12-05 | [33099](https://github.com/airbytehq/airbyte/pull/33099) | Fix filtering for archived records stream | +| `2.0.1` | 2023-11-08 | [32291](https://github.com/airbytehq/airbyte/pull/32291) | Add logic to have regular checkpointing schedule | +| `2.0.0` | 2023-11-03 | [32128](https://github.com/airbytehq/airbyte/pull/32128) | Use the latest API for streams `campaigns`, `email_templates`, `events`, `flows`, `global_exclusions`, `lists`, and `metrics` | +| `1.1.0` | 2023-10-23 | [31710](https://github.com/airbytehq/airbyte/pull/31710) | Make `start_date` config field optional | +| `1.0.0` | 2023-10-18 | [31565](https://github.com/airbytehq/airbyte/pull/31565) | added new known fields for 'events' stream | +| `0.5.0` | 2023-10-19 | [31611](https://github.com/airbytehq/airbyte/pull/31611) | Add `date-time` format for `datetime` field in `Events` stream | +| `0.4.0` | 2023-10-18 | [31562](https://github.com/airbytehq/airbyte/pull/31562) | Add `archived` field to `Flows` stream | +| `0.3.3` | 2023-10-13 | [31379](https://github.com/airbytehq/airbyte/pull/31379) | Skip streams that the connector no longer has access to | +| `0.3.2` | 2023-06-20 | [27498](https://github.com/airbytehq/airbyte/pull/27498) | Do not store state in the future | +| `0.3.1` | 2023-06-08 | [27162](https://github.com/airbytehq/airbyte/pull/27162) | Anonymize check connection error message | +| `0.3.0` | 2023-02-18 | [23236](https://github.com/airbytehq/airbyte/pull/23236) | Add ` Email Templates` stream | +| `0.2.0` | 2023-03-13 | [22942](https://github.com/airbytehq/airbyte/pull/23968) | Add `Profiles` stream | +| `0.1.13` | 2023-02-13 | [22942](https://github.com/airbytehq/airbyte/pull/22942) | Specified date formatting in specification | +| `0.1.12` | 2023-01-30 | [22071](https://github.com/airbytehq/airbyte/pull/22071) | Fix `Events` stream schema | +| `0.1.11` | 2023-01-27 | [22012](https://github.com/airbytehq/airbyte/pull/22012) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| `0.1.10` | 2022-09-29 | [17422](https://github.com/airbytehq/airbyte/issues/17422) | Update CDK dependency | +| `0.1.9` | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/issues/17304) | Migrate to per-stream state. | +| `0.1.6` | 2022-07-20 | [14872](https://github.com/airbytehq/airbyte/issues/14872) | Increase test coverage | +| `0.1.5` | 2022-07-12 | [14617](https://github.com/airbytehq/airbyte/issues/14617) | Set max_retries = 10 for `lists` stream. | +| `0.1.4` | 2022-04-15 | [11723](https://github.com/airbytehq/airbyte/issues/11723) | Enhance klaviyo source for flows stream and update to events stream. | +| `0.1.3` | 2021-12-09 | [8592](https://github.com/airbytehq/airbyte/pull/8592) | Improve performance, make Global Exclusions stream incremental and enable Metrics stream. | +| `0.1.2` | 2021-10-19 | [6952](https://github.com/airbytehq/airbyte/pull/6952) | Update schema validation in SAT | diff --git a/docs/integrations/sources/kyve.md b/docs/integrations/sources/kyve.md index fbb3ff6f694d..4fd218ec4e55 100644 --- a/docs/integrations/sources/kyve.md +++ b/docs/integrations/sources/kyve.md @@ -1,4 +1,4 @@ -# Kyve Source +# KYVE This page contains the setup guide and reference information for the **KYVE** source connector. @@ -9,15 +9,21 @@ For information about how to setup an end to end pipeline with this connector, s ## Source configuration setup -1. In order to create an ELT pipeline with KYVE source you should specify the **`Pool-ID`** of [Kyve storage pool](https://app.kyve.network/#/pools) from which you want to retrieve data. +1. In order to create an ELT pipeline with KYVE source you should specify the **`Pool-ID`** of [KYVE storage pool](https://app.kyve.network/#/pools) from which you want to retrieve data. -2. You can specify a specific **`Bundle-Start-ID`** in case you want to narrow the records that will be retrieved from the pool. You can find the valid bundles of in the KYVE app (e.g. [Moonbeam pool bundles](https://app.kyve.network/#/pools/0/bundles)). +2. You can specify a specific **`Bundle-Start-ID`** in case you want to narrow the records that will be retrieved from the pool. You can find the valid bundles of in the KYVE app (e.g. [Cosmos Hub pool](https://app.kyve.network/#/pools/0/bundles)). + +3. In order to extract the validated from KYVE, you can specify the endpoint which will be requested **`KYVE-API URL Base`**. By default, the official KYVE **`mainnet`** endpoint will be used, providing the data of [these pools](https://app.kyve.network/#/pools). + + ***Note:*** + KYVE Network consists of three individual networks: *Korellia* is the `devnet` used for development purposes, *Kaon* is the `testnet` used for testing purposes, and **`mainnet`** is the official network. Although through Kaon and Korellia validated data can be used for development purposes, it is recommended to only trust the data validated on Mainnet. ## Multiple pools You can fetch with one source configuration more than one pool simultaneously. You just need to specify the **`Pool-IDs`** and the **`Bundle-Start-ID`** for the KYVE storage pool you want to archive separated with comma. ## Changelog -| Version | Date | Pull Request | Subject | -|:--------| :--- | :----------- | :--------------- | -| 0.1.0 | 25-05-29 | [26299](https://github.com/airbytehq/airbyte/pull/26299) | Initial release of KYVE source connector| +| Version | Date | Subject | +| :------ |:---------|:----------------------------------------------------| +| 0.1.0 | 25-05-23 | Initial release of KYVE source connector | +| 0.2.0 | 10-11-23 | Update KYVE source to support to Mainnet and Testnet| \ No newline at end of file diff --git a/docs/integrations/sources/lemlist.md b/docs/integrations/sources/lemlist.md index 1fea1df4cec7..f55473da227b 100644 --- a/docs/integrations/sources/lemlist.md +++ b/docs/integrations/sources/lemlist.md @@ -37,5 +37,6 @@ The Lemlist connector should not run into Lemlist API limitations under normal u | Version | Date | Pull Request | Subject | | :------ | :--------- | :----------------------------------------------------- | :-------------- | -| 0.1.1 | Unknown | Unknown | Bump Version | -| 0.1.0 | 2021-10-14 | [7062](https://github.com/airbytehq/airbyte/pull/7062) | Initial Release | +| 0.2.0 | 2023-08-14 | [29406](https://github.com/airbytehq/airbyte/pull/29406) | Migrated to LowCode Cdk | +| 0.1.1 | Unknown | Unknown | Bump Version | +| 0.1.0 | 2021-10-14 | [7062](https://github.com/airbytehq/airbyte/pull/7062) | Initial Release | diff --git a/docs/integrations/sources/linkedin-ads.md b/docs/integrations/sources/linkedin-ads.md index 0b752afae000..7bb11b6c5100 100644 --- a/docs/integrations/sources/linkedin-ads.md +++ b/docs/integrations/sources/linkedin-ads.md @@ -171,6 +171,12 @@ After 5 unsuccessful attempts - the connector will stop the sync operation. In s | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| 0.6.7 | 2024-01-11 | [34152](https://github.com/airbytehq/airbyte/pull/34152) | prepare for airbyte-lib | +| 0.6.6 | 2024-01-15 | [34222](https://github.com/airbytehq/airbyte/pull/34222) | Use stream slices for Analytics streams | +| 0.6.5 | 2023-12-15 | [33530](https://github.com/airbytehq/airbyte/pull/33530) | Fix typo in `Pivot Category` list | +| 0.6.4 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 0.6.3 | 2023-10-13 | [31396](https://github.com/airbytehq/airbyte/pull/31396) | Fix pagination for reporting | +| 0.6.2 | 2023-08-23 | [31221](https://github.com/airbytehq/airbyte/pull/31221) | Increase max time between messages to 24 hours | | 0.6.1 | 2023-08-23 | [29600](https://github.com/airbytehq/airbyte/pull/29600) | Update field descriptions | | 0.6.0 | 2023-08-22 | [29721](https://github.com/airbytehq/airbyte/pull/29721) | Add `Conversions` stream | | 0.5.0 | 2023-08-14 | [29175](https://github.com/airbytehq/airbyte/pull/29175) | Add Custom report Constructor | @@ -195,4 +201,4 @@ After 5 unsuccessful attempts - the connector will stop the sync operation. In s | 0.1.3 | 2021-11-11 | [7839](https://github.com/airbytehq/airbyte/pull/7839) | Added OAuth support | | 0.1.2 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | | 0.1.1 | 2021-10-02 | [6610](https://github.com/airbytehq/airbyte/pull/6610) | Fix for `Campaigns/targetingCriteria` transformation, coerced `Creatives/variables/values` to string by default | -| 0.1.0 | 2021-09-05 | [5285](https://github.com/airbytehq/airbyte/pull/5285) | Initial release of Native LinkedIn Ads connector for Airbyte | +| 0.1.0 | 2021-09-05 | [5285](https://github.com/airbytehq/airbyte/pull/5285) | Initial release of Native LinkedIn Ads connector for Airbyte | \ No newline at end of file diff --git a/docs/integrations/sources/mailchimp-migrations.md b/docs/integrations/sources/mailchimp-migrations.md new file mode 100644 index 000000000000..c236c549eef2 --- /dev/null +++ b/docs/integrations/sources/mailchimp-migrations.md @@ -0,0 +1,85 @@ +# Mailchimp Migration Guide + +## Upgrading to 1.0.0 + +Version 1.0.0 of the Source Mailchimp connector introduces a number of breaking changes to the schemas of all incremental streams. A full schema refresh and data reset are required when upgrading to this version. + +### Upgrade steps + +1. Select **Connections** in the main navbar. +2. From the list of your existing connections, select the connection(s) affected by the update. +3. Select the **Replication** tab, then select **Refresh source schema**. + +:::note +Any detected schema changes will be listed for your review. Select **OK** when you are ready to proceed. +::: + +4. At the bottom of the page, select **Save changes**. Ensure the **Reset all streams** option is checked. + +:::note +Depending on the destination type, you may not be prompted to reset your data +::: + +5. Select **Save connection**. This will reset the data in your destination (if applicable) and initiate a fresh sync. + +## Changes + +- The `._links` field, which contained non-user-relevant Mailchimp metadata, has been removed from all streams. +- All instances of datetime fields have had their type changed from `string` to airbyte-type `timestamp-with-timezone`. This change should ensure greater precision and consistency in how datetime information is represented and processed by destinations. +- The Mailchimp API returns many fields without data as empty strings. To accomodate the above changes, empty strings are now converted to null values: + +```md +{"id": "record_id", "last_opened": ""} -> {"id": "record_id", "last_opened": null} +``` + +### Updated datetime fields + +- Automations: + - `create_time` + - `send_time` + +- Campaigns: + - `create_time` + - `send_time` + - `rss_opts.last_sent` + - `ab_split_opts.send_time_a` + - `ab_split_opts.send_time_b` + - `variate_settings.send_times` (Array of datetime fields) + +- Email Activity: + - `timestamp` + +- List Members: + - `timestamp_signup` + - `timestamp_opt` + - `last_changed` + - `created_at` + +- Lists: + - `date_created` + - `stats.campaign_last_sent` + - `stats.last_sub_date` + - `stats.last_unsub_date` + +- Reports: + - `send_time` + - `rss_last_send` + - `opens.last_open` + - `clicks.last_click` + - `ab_split.a.last_open` + - `ab_split.b.last_open` + - `timewarp.last_open` + - `timeseries.timestamp` + +- Segment Members: + - `timestamp_signup` + - `timestamp_opt` + - `last_changed` + - `last_note.created_at` + +- Segments: + - `created_at` + - `updated_at` + +- Unsubscribes: + - `timestamp` diff --git a/docs/integrations/sources/mailchimp.md b/docs/integrations/sources/mailchimp.md index c8a0fed4f046..106112d4dc7c 100644 --- a/docs/integrations/sources/mailchimp.md +++ b/docs/integrations/sources/mailchimp.md @@ -1,224 +1,117 @@ # Mailchimp -This page guides you through setting up the Mailchimp source connector. +This page guides you through setting up the [Mailchimp](https://mailchimp.com/) source connector. -## Prerequisite +## Prerequisites -You can use [OAuth](https://mailchimp.com/developer/marketing/guides/access-user-data-oauth-2/) or an API key to authenticate your Mailchimp account. If you choose to authenticate with OAuth, [register](https://mailchimp.com/developer/marketing/guides/access-user-data-oauth-2/#register-your-application) your Mailchimp account. + + +#### For Airbyte Cloud + +- Access to a valid Mailchimp account. If you are not an Owner/Admin of the account, you must be [granted Admin access](https://mailchimp.com/help/manage-user-levels-in-your-account/#Grant_account_access) by the account's Owner/Admin. + + + + + +#### For Airbyte Open Source + +- A valid Mailchimp **API Key** (recommended) or OAuth credentials: **Client ID**, **Client Secret** and **Access Token** + + + +## Setup guide + + + +### Airbyte Open Source: Generate a Mailchimp API key + +1. Navigate to the API Keys section of your Mailchimp account. +2. Click **Create New Key**, and give the key a name to help you identify it. You won't be able to see or copy the key once you finish generating it, so be sure to copy the key and store it in a secure location. + +For more information on Mailchimp API Keys, please refer to the [official Mailchimp docs](https://mailchimp.com/help/about-api-keys/#api+key+security). If you want to use OAuth authentication with Airbyte Open Source, please follow the steps laid out [here](https://mailchimp.com/developer/marketing/guides/access-user-data-oauth-2/) to obtain your OAuth **Client ID**, **Client Secret** and **Access Token**. + + ## Set up the Mailchimp source connector 1. Log into your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account. -2. Click **Sources** and then click **+ New source**. -3. On the Set up the source page, select **Mailchimp** from the Source type dropdown. +2. Click **Sources** and then click **+ New source**. +3. Find and select **Mailchimp** from the list of available sources. 4. Enter a name for your source. +5. You can use OAuth or an API key to authenticate your Mailchimp account. We recommend using OAuth for Airbyte Cloud and an API key for Airbyte Open Source. -6. You can use OAuth or an API key to authenticate your Mailchimp account. We recommend using OAuth for Airbyte Cloud and an API key for Airbyte Open Source. - - To authenticate using OAuth for Airbyte Cloud, ensure you have [registered your Mailchimp account](#prerequisite) and then click **Authenticate your Mailchimp account** to sign in with Mailchimp and authorize your account. - - To authenticate using an API key for Airbyte Open Source, select **API key** from the Authentication dropdown and enter the [API key](https://mailchimp.com/developer/marketing/guides/quick-start/#generate-your-api-key) for your Mailchimp account. - :::note - Check the [performance considerations](#performance-considerations) before using an API key. - ::: -7. Click **Set up source**. + -## Supported sync modes +- To authenticate using OAuth for Airbyte Cloud, click **Authenticate your Mailchimp account** and follow the instructions to sign in with Mailchimp and authorize your account. -The Mailchimp source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-mode): + - - Full Refresh - - Incremental + -Airbyte doesn't support Incremental Deletes for the `Campaigns`, `Lists`, and `Email Activity` streams because Mailchimp doesn't provide any information about deleted data in these streams. +- To authenticate using an API key for Airbyte Open Source, select **API key** from the Authentication dropdown and enter the [API key](https://mailchimp.com/developer/marketing/guides/quick-start/#generate-your-api-key) for your Mailchimp account. +- To authenticate using OAuth credentials, select **Oauth2.0** from the dropdown and enter the **Client ID**, **Client Secret** and **Access Token** you obtained. -## Performance considerations + -[Mailchimp does not impose rate limits](https://mailchimp.com/developer/guides/marketing-api-conventions/#throttling) on how much data is read from its API in a single sync process. However, Mailchimp enforces a maximum of 10 simultaneous connections to its API, which means that Airbyte is unable to run more than 10 concurrent syncs from Mailchimp using API keys generated from the same account. +6. (Optional) You may optionally provide an **Incremental Sync Start Date** using the provided datepicker, or by programmatically entering a UTC date-time in the format `YYYY-MM-DDThh:mm:ss.sssZ`. If set, only data generated on or after the configured date-time will be synced. Leaving this field blank will sync all data returned from the API. +7. Click **Set up source** and wait for the tests to complete. + + ## Supported streams -The Mailchimp source connector supports the following streams: - -**[Lists](https://mailchimp.com/developer/api/marketing/lists/get-list-info) Stream** - -``` -{ - "id": "q1w2e3r4t5", - "web_id": 000001, - "name": "Newsletter Subscribers", - "contact": { - "company": "", - "address1": "", - "address2": "", - "city": "San Francisco", - "state": "CA", - "zip": "00000-1111", - "country": "US", - "phone": "" - }, - "permission_reminder": "You are receiving this email because you opted in via our website.", - "use_archive_bar": true, - "campaign_defaults": { - "from_name": "Airbyte Community", - "from_email": "hey@email.com", - "subject": "", - "language": "en" - }, - "notify_on_subscribe": "", - "notify_on_unsubscribe": "", - "date_created": "2020-09-17T04:48:49+00:00", - "list_rating": 3, - "email_type_option": false, - "subscribe_url_short": "http://eepurl.com/hfpWAr", - "subscribe_url_long": "https://daxtarity.us2.list-manage.com/subscribe?u=q1q1q1q1q1q1q1q1q1q&id=q1w2e3r4t5", - "beamer_address": "us2-00000000-qqqqqqqqq@inbound.mailchimp.com", - "visibility": "prv", - "double_optin": false, - "has_welcome": false, - "marketing_permissions": false, - "modules": [], - "stats": { - "member_count": 4204, - "unsubscribe_count": 194, - "cleaned_count": 154, - "member_count_since_send": 91, - "unsubscribe_count_since_send": 19, - "cleaned_count_since_send": 23, - "campaign_count": 27, - "campaign_last_sent": "2022-04-01T14:29:31+00:00", - "merge_field_count": 5, - "avg_sub_rate": 219, - "avg_unsub_rate": 10, - "target_sub_rate": 18, - "open_rate": 39.478173607626694, - "click_rate": 8.504017780817234, - "last_sub_date": "2022-04-12T07:39:29+00:00", - "last_unsub_date": "2022-04-11T08:08:07+00:00" - }, - "_links": [ - { - "rel": "self", - "href": "https://us2.api.mailchimp.com/3.0/lists/q1w2e3r4t5", - "method": "GET", - "targetSchema": "https://us2.api.mailchimp.com/schema/3.0/Definitions/Lists/Response.json" - } - ] -} -``` - -**[Campaigns](https://mailchimp.com/developer/api/marketing/campaigns/get-campaign-info/) Stream** - -``` -{ - "id": "q1w2e3r4t5", - "web_id": 0000000, - "type": "regular", - "create_time": "2020-11-03T22:46:43+00:00", - "archive_url": "http://eepurl.com/hhSLxH", - "long_archive_url": "https://mailchi.mp/xxxxxxxx/weekly-bytes-learnings-from-soft-launch-and-our-vision-0000000", - "status": "sent", - "emails_sent": 89, - "send_time": "2020-11-05T16:15:00+00:00", - "content_type": "template", - "needs_block_refresh": false, - "resendable": true, - "recipients": { - "list_id": "1q2w3e4r", - "list_is_active": true, - "list_name": "Newsletter Subscribers", - "segment_text": "", - "recipient_count": 89 - }, - "settings": { - "subject_line": "Some subject", - "preview_text": "Text", - "title": "Newsletter", - "from_name": "Weekly Bytes from Airbyte", - "reply_to": "hey@email.com", - "use_conversation": false, - "to_name": "", - "folder_id": "", - "authenticate": true, - "auto_footer": false, - "inline_css": false, - "auto_tweet": false, - "fb_comments": true, - "timewarp": false, - "template_id": 0000000, - "drag_and_drop": false - }, - "tracking": { - "opens": true, - "html_clicks": true, - "text_clicks": false, - "goal_tracking": false, - "ecomm360": false, - "google_analytics": "", - "clicktale": "" - }, - "report_summary": { - "opens": 46, - "unique_opens": 33, - "open_rate": 0.0128372, - "clicks": 13, - "subscriber_clicks": 7, - "click_rate": 0.0383638, - "ecommerce": { - "total_orders": 0, - "total_spent": 0, - "total_revenue": 0 - } - }, - "delivery_status": { - "enabled": false - }, - "_links": [ - { - "rel": "parent", - "href": "https://us2.api.mailchimp.com/3.0/campaigns", - "method": "GET", - "targetSchema": "https://us2.api.mailchimp.com/schema/3.0/Definitions/Campaigns/CollectionResponse.json", - "schema": "https://us2.api.mailchimp.com/schema/3.0/Paths/Campaigns/Collection.json" - } - ] -} -``` - -**[Email Activity](https://mailchimp.com/developer/marketing/api/email-activity-reports/) Stream** - -``` -{ - "campaign_id": "q1w2q1w2q1w2", - "list_id": "123qwe", - "list_is_active": true, - "email_id": "qwerty123456", - "email_address": "email@email.com", - "_links": [ - { - "rel": "parent", - "href": "https://us2.api.mailchimp.com/3.0/reports/q1w2q1w2q1w2/email-activity", - "method": "GET", - "targetSchema": "https://us2.api.mailchimp.com/schema/3.0/Definitions/Reports/EmailActivity/CollectionResponse.json" - } - ], - "action": "open", - "timestamp": "2020-10-08T22:15:43+00:00", - "ip": "00.000.00.5" -} -``` - -### A note on the primary keys - -The `Lists` and `Campaigns` streams have `id` as the primary key. The `Email Activity` stream doesn't have a primary key because Mailchimp does not provide one. +The Mailchimp source connector supports the following streams and [sync modes](https://docs.airbyte.com/cloud/core-concepts/#connection-sync-mode): + +| Stream | Full Refresh | Incremental | +|:-------------------------------------------------------------------------------------------------------------------|:-------------|:------------| +| [Automations](https://mailchimp.com/developer/marketing/api/automation/list-automations/) | ✓ | ✓ | +| [Campaigns](https://mailchimp.com/developer/marketing/api/campaigns/get-campaign-info/) | ✓ | ✓ | +| [Email Activity](https://mailchimp.com/developer/marketing/api/email-activity-reports/list-email-activity/) | ✓ | ✓ | +| [Interests](https://mailchimp.com/developer/marketing/api/interests/list-interests-in-category/) | ✓ | | +| [Interest Categories](https://mailchimp.com/developer/marketing/api/interest-categories/list-interest-categories/) | ✓ | | +| [Lists](https://mailchimp.com/developer/api/marketing/lists/get-list-info) | ✓ | ✓ | +| [List Members](https://mailchimp.com/developer/marketing/api/list-members/list-members-info/) | ✓ | ✓ | +| [Reports](https://mailchimp.com/developer/marketing/api/reports/list-campaign-reports/) | ✓ | ✓ | +| [Segments](https://mailchimp.com/developer/marketing/api/list-segments/list-segments/) | ✓ | ✓ | +| [Segment Members](https://mailchimp.com/developer/marketing/api/list-segment-members/list-members-in-segment/) | ✓ | ✓ | +| [Tags](https://mailchimp.com/developer/marketing/api/lists-tags-search/search-for-tags-on-a-list-by-name/) | ✓ | | +| [Unsubscribes](https://mailchimp.com/developer/marketing/api/unsub-reports/list-unsubscribed-members/) | ✓ | ✓ | + +### A note on primary keys + +The `EmailActivity` and `Unsubscribes` streams do not have an `id` primary key, and therefore use the following composite keys as unique identifiers: + +- EmailActivity [`email_id`, `action`, `timestamp`] +- Unsubscribes [`campaign_id`, `email_id`, `timestamp`] + +All other streams contain an `id` primary key. ## Data type mapping -| Integration Type | Airbyte Type | Notes | -|:---------------------------|:-------------|:------------------------------------------------------------------------------------| -| `array` | `array` | the type of elements in the array is determined based on the mappings in this table | -| `date`, `time`, `datetime` | `string` | | -| `int`, `float`, `number` | `number` | | -| `object` | `object` | properties within objects are mapped based on the mappings in this table | -| `string` | `string` | | +| Integration Type | Airbyte Type | Notes | +|:---------------------------|:--------------------------|:------------------------------------------------------------------------------------| +| `array` | `array` | the type of elements in the array is determined based on the mappings in this table | +| `string` | `string` | | +| `float`, `number` | `number` | | +| `integer` | `integer` | | +| `object` | `object` | properties within objects are mapped based on the mappings in this table | +| `string` (timestamp) | `timestamp_with_timezone` | Mailchimp timestamps are formatted as `YYYY-MM-DDTHH:MM:SS+00:00` | + +## Limitations & Troubleshooting + +
      + + +Expand to see details about Mailchimp connector limitations and troubleshooting + + + +### Connector limitations + +[Mailchimp does not impose rate limits](https://mailchimp.com/developer/guides/marketing-api-conventions/#throttling) on how much data is read from its API in a single sync process. However, Mailchimp enforces a maximum of 10 simultaneous connections to its API, which means that Airbyte is unable to run more than 10 concurrent syncs from Mailchimp using API keys generated from the same account. + +
      ## Tutorials @@ -230,6 +123,18 @@ Now that you have set up the Mailchimp source connector, check out the following | Version | Date | Pull Request | Subject | |---------|------------|----------------------------------------------------------|----------------------------------------------------------------------------| +| 1.1.1 | 2024-01-11 | [34157](https://github.com/airbytehq/airbyte/pull/34157) | Prepare for airbyte-lib | +| 1.1.0 | 2023-12-20 | [32852](https://github.com/airbytehq/airbyte/pull/32852) | Add optional start_date for incremental streams | +| 1.0.0 | 2023-12-19 | [32836](https://github.com/airbytehq/airbyte/pull/32836) | Add airbyte-type to `datetime` columns and remove `._links` column | +| 0.10.0 | 2023-11-23 | [32782](https://github.com/airbytehq/airbyte/pull/32782) | Add SegmentMembers stream | +| 0.9.0 | 2023-11-17 | [32218](https://github.com/airbytehq/airbyte/pull/32218) | Add Interests, InterestCategories, Tags streams | +| 0.8.3 | 2023-11-15 | [32543](https://github.com/airbytehq/airbyte/pull/32543) | Handle empty datetime fields in Reports stream | +| 0.8.2 | 2023-11-13 | [32466](https://github.com/airbytehq/airbyte/pull/32466) | Improve error handling during connection check | +| 0.8.1 | 2023-11-06 | [32226](https://github.com/airbytehq/airbyte/pull/32226) | Unmute expected records test after data anonymisation | +| 0.8.0 | 2023-11-01 | [32032](https://github.com/airbytehq/airbyte/pull/32032) | Add ListMembers stream | +| 0.7.0 | 2023-10-27 | [31940](https://github.com/airbytehq/airbyte/pull/31940) | Implement availability strategy | +| 0.6.0 | 2023-10-27 | [31922](https://github.com/airbytehq/airbyte/pull/31922) | Add Segments stream | +| 0.5.0 | 2023-10-20 | [31675](https://github.com/airbytehq/airbyte/pull/31675) | Add Unsubscribes stream | | 0.4.1 | 2023-05-02 | [25717](https://github.com/airbytehq/airbyte/pull/25717) | Handle unknown error in EmailActivity | | 0.4.0 | 2023-04-11 | [23290](https://github.com/airbytehq/airbyte/pull/23290) | Add Automations stream | | 0.3.5 | 2023-02-28 | [23464](https://github.com/airbytehq/airbyte/pull/23464) | Add Reports stream | @@ -255,3 +160,5 @@ Now that you have set up the Mailchimp source connector, check out the following | 0.2.1 | 2021-04-03 | [2726](https://github.com/airbytehq/airbyte/pull/2726) | Fix base connector versioning | | 0.2.0 | 2021-03-09 | [2238](https://github.com/airbytehq/airbyte/pull/2238) | Protocol allows future/unknown properties | | 0.1.4 | 2020-11-30 | [1046](https://github.com/airbytehq/airbyte/pull/1046) | Add connectors using an index YAML file | + +
      diff --git a/docs/integrations/sources/mailgun.md b/docs/integrations/sources/mailgun.md index 5afbe7eaf37c..c5d0c391afb3 100644 --- a/docs/integrations/sources/mailgun.md +++ b/docs/integrations/sources/mailgun.md @@ -14,8 +14,8 @@ Just pass the generated API key for establishing the connection. - Generate an API key (Example: 12345) - Params (If specific info is needed) - Available params - - domain_region: Domain region code. 'EU' or 'US' are possible values. The default is 'US'. - - start_date: UTC date and time in the format 2020-10-01 00:00:00. Any data before this date will not be replicated. If omitted, defaults to 3 days ago. + - **Domain Region Code**: Domain region code. 'EU' or 'US' are possible values. The default is 'US'. + - **Replication Start Date**: UTC date and time in the format 2020-10-01 00:00:00. Any data before this date will not be replicated. If omitted, defaults to 90 days ago. ## Step 2: Set up the MailGun connector in Airbyte @@ -25,7 +25,7 @@ Just pass the generated API key for establishing the connection. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**. 3. On the Set up the source page, enter the name for the MailGun connector and select **MailGun** from the Source type dropdown. 4. Enter your api_key as `private_key`. -5. Enter the params configuration if needed. Supported params are: domain_region, start_date. +5. Enter the optional params configuration if needed. Supported params are: **Domain Region Code**, **Replication Start Date**. 6. Click **Set up source**. ### For Airbyte OSS: @@ -33,7 +33,7 @@ Just pass the generated API key for establishing the connection. 1. Navigate to the Airbyte Open Source dashboard. 2. Set the name for your source. 3. Enter your api_key as `pivate_key`. -4. Enter the params configuration if needed. Supported params are: domain_region, start_date. +4. Enter the optional params configuration if needed. Supported params are: **Domain Region Code**, **Replication Start Date**. 5. Click **Set up source**. ## Supported sync modes @@ -63,8 +63,9 @@ MailGun's [API reference](https://documentation.mailgun.com/en/latest/api_refere ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------ | :------------------------------------------ | -| 0.2.0 | 2023-08-05 | [29122](https://github.com/airbytehq/airbyte/pull/29122) | Migrate to Low Code | -| 0.1.1 | 2023-02-13 | [22939](https://github.com/airbytehq/airbyte/pull/22939) | Specified date formatting in specification | -| 0.1.0 | 2021-11-09 | [8056](https://github.com/airbytehq/airbyte/pull/8056) | New Source: Mailgun | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------ | :--------------------------------------------------- | +| 0.2.1 | 2023-10-16 | [31405](https://github.com/airbytehq/airbyte/pull/31405) | Fixed test connection failure if date field is empty | +| 0.2.0 | 2023-08-05 | [29122](https://github.com/airbytehq/airbyte/pull/29122) | Migrate to Low Code | +| 0.1.1 | 2023-02-13 | [22939](https://github.com/airbytehq/airbyte/pull/22939) | Specified date formatting in specification | +| 0.1.0 | 2021-11-09 | [8056](https://github.com/airbytehq/airbyte/pull/8056) | New Source: Mailgun | diff --git a/docs/integrations/sources/mailjet-mail.md b/docs/integrations/sources/mailjet-mail.md index 09709ad06495..85c89b0fda51 100644 --- a/docs/integrations/sources/mailjet-mail.md +++ b/docs/integrations/sources/mailjet-mail.md @@ -34,5 +34,6 @@ Mailjet APIs are under rate limits for the number of API calls allowed per API k | Version | Date | Pull Request | Subject | | :------ | :--------- | :-------------------------------------------------------- | :----------------------------------------- | -| 0.1.1 | 2022-04-19 | [#24689](https://github.com/airbytehq/airbyte/pull/24689) | Add listrecipient stream | +| 0.1.2 | 2022-12-18 | [#30924](https://github.com/airbytehq/airbyte/pull/30924) | Adds Subject field to `message` stream | +| 0.1.1 | 2022-04-19 | [#24689](https://github.com/airbytehq/airbyte/pull/24689) | Add listrecipient stream | | 0.1.0 | 2022-10-26 | [#18332](https://github.com/airbytehq/airbyte/pull/18332) | 🎉 New Source: Mailjet Mail API [low-code CDK] | diff --git a/docs/integrations/sources/marketo.md b/docs/integrations/sources/marketo.md index b179e2515660..5e5f7ad883fd 100644 --- a/docs/integrations/sources/marketo.md +++ b/docs/integrations/sources/marketo.md @@ -81,13 +81,19 @@ The Marketo source connector supports the following[ sync modes](https://docs.ai This connector can be used to sync the following tables from Marketo: -- **activities_X** where X is an activity type contains information about lead activities of the type X. For example, activities_send_email contains information about lead activities related to the activity type `send_email`. See the [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Activities/getLeadActivitiesUsingGET) for a detailed explanation of what each column means. -- **activity_types.** Contains metadata about activity types. See the [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Activities/getAllActivityTypesUsingGET) for a detailed explanation of columns. -- **campaigns.** Contains info about your Marketo campaigns. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Campaigns/getCampaignsUsingGET). -- **leads.** Contains info about your Marketo leads. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Leads/getLeadByIdUsingGET). -- **lists.** Contains info about your Marketo static lists. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Static_Lists/getListByIdUsingGET). -- **programs.** Contains info about your Marketo programs. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/asset-endpoint-reference/#!/Programs/browseProgramsUsingGET). -- **segmentations.** Contains info about your Marketo programs. [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/asset-endpoint-reference/#!/Segments/getSegmentationUsingGET). +- **Activities_X** where X is an activity type contains information about lead activities of the type X. For example, activities_send_email contains information about lead activities related to the activity type `send_email`. See the [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Activities/getLeadActivitiesUsingGET) for a detailed explanation of what each column means. +- **Activity types** Contains metadata about activity types. See the [Marketo docs](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Activities/getAllActivityTypesUsingGET) for a detailed explanation of columns. +- **[Campaigns](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Campaigns/getCampaignsUsingGET)**: Contains info about your Marketo campaigns. +- **[Leads](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Leads/getLeadByIdUsingGET)**: Contains info about your Marketo leads. + +:::caution + +Available fields are limited by what is presented in the static schema. + +::: +- **[Lists](https://developers.marketo.com/rest-api/endpoint-reference/lead-database-endpoint-reference/#!/Static_Lists/getListByIdUsingGET)**: Contains info about your Marketo static lists. +- **[Programs](https://developers.marketo.com/rest-api/endpoint-reference/asset-endpoint-reference/#!/Programs/browseProgramsUsingGET)**: Contains info about your Marketo programs. +- **[Segmentations](https://developers.marketo.com/rest-api/endpoint-reference/asset-endpoint-reference/#!/Segments/getSegmentationUsingGET)**: Contains info about your Marketo programs. ## Performance considerations @@ -100,7 +106,7 @@ If the 50,000 limit is too stringent, contact Marketo support for a quota increa ## Data type map | Integration Type | Airbyte Type | Notes | -| :--------------- | :----------- | :------------------------------------------------------------------------------ | +|:-----------------|:-------------|:--------------------------------------------------------------------------------| | `array` | `array` | primitive arrays are converted into arrays of the types described in this table | | `int`, `long` | `number` | | | `object` | `object` | | @@ -109,24 +115,29 @@ If the 50,000 limit is too stringent, contact Marketo support for a quota increa ## Changelog -| Version | Date | Pull Request | Subject | -| :------- | :--------- | :------------------------------------------------------- | :----------------------------------------------------------------------------------------------- | -| `1.2.0` | 2023-06-26 | [27726](https://github.com/airbytehq/airbyte/pull/27726) | License Update: Elv2 | -| `1.1.0` | 2023-04-18 | [23956](https://github.com/airbytehq/airbyte/pull/23956) | Add `Segmentations` Stream | -| `1.0.4` | 2023-04-25 | [25481](https://github.com/airbytehq/airbyte/pull/25481) | Minor fix for bug caused by `<=` producing additional API call when there is a single date slice | -| `1.0.3` | 2023-02-13 | [22938](https://github.com/airbytehq/airbyte/pull/22938) | Specified date formatting in specification | -| `1.0.2` | 2023-02-01 | [22203](https://github.com/airbytehq/airbyte/pull/22203) | Handle Null cursor values | -| `1.0.1` | 2023-01-31 | [22015](https://github.com/airbytehq/airbyte/pull/22015) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| `1.0.0` | 2023-01-25 | [21790](https://github.com/airbytehq/airbyte/pull/21790) | Fix `activities_*` stream schemas | -| `0.1.12` | 2023-01-19 | [20973](https://github.com/airbytehq/airbyte/pull/20973) | Fix encoding error (note: this change is not in version 1.0.0, but is in later versions | -| `0.1.11` | 2022-09-30 | [17445](https://github.com/airbytehq/airbyte/pull/17445) | Do not use temporary files for memory optimization | -| `0.1.10` | 2022-09-30 | [17445](https://github.com/airbytehq/airbyte/pull/17445) | Optimize memory consumption | -| `0.1.9` | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream sate. | -| `0.1.7` | 2022-08-23 | [15817](https://github.com/airbytehq/airbyte/pull/15817) | Improved unit test coverage | -| `0.1.6` | 2022-08-21 | [15824](https://github.com/airbytehq/airbyte/pull/15824) | Fix semi incremental streams: do not ignore start date, make one api call instead of multiple | -| `0.1.5` | 2022-08-16 | [15683](https://github.com/airbytehq/airbyte/pull/15683) | Retry failed creation of a job instead of skipping it | -| `0.1.4` | 2022-06-20 | [13930](https://github.com/airbytehq/airbyte/pull/13930) | Process failing creation of export jobs | -| `0.1.3` | 2021-12-10 | [8429](https://github.com/airbytehq/airbyte/pull/8578) | Updated titles and descriptions | -| `0.1.2` | 2021-12-03 | [8483](https://github.com/airbytehq/airbyte/pull/8483) | Improve field conversion to conform schema | -| `0.1.1` | 2021-11-29 | [0000](https://github.com/airbytehq/airbyte/pull/0000) | Fix timestamp value format issue | -| `0.1.0` | 2021-09-06 | [5863](https://github.com/airbytehq/airbyte/pull/5863) | Release Marketo CDK Connector | +| Version | Date | Pull Request | Subject | +|:---------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------| +| 1.2.5 | 2024-01-15 | [34246](https://github.com/airbytehq/airbyte/pull/34246) | prepare for airbyte-lib | +| `1.2.4` | 2024-01-08 | [33999](https://github.com/airbytehq/airbyte/pull/33999) | Fix for `Export daily quota exceeded` | +| `1.2.3` | 2023-08-02 | [28999](https://github.com/airbytehq/airbyte/pull/28999) | Fix for ` _csv.Error: line contains NUL` | +| `1.2.2` | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| `1.2.1` | 2023-09-18 | [30533](https://github.com/airbytehq/airbyte/pull/30533) | Fix `json_schema` for stream `Leads` | +| `1.2.0` | 2023-06-26 | [27726](https://github.com/airbytehq/airbyte/pull/27726) | License Update: Elv2 | +| `1.1.0` | 2023-04-18 | [23956](https://github.com/airbytehq/airbyte/pull/23956) | Add `Segmentations` Stream | +| `1.0.4` | 2023-04-25 | [25481](https://github.com/airbytehq/airbyte/pull/25481) | Minor fix for bug caused by `<=` producing additional API call when there is a single date slice | +| `1.0.3` | 2023-02-13 | [22938](https://github.com/airbytehq/airbyte/pull/22938) | Specified date formatting in specification | +| `1.0.2` | 2023-02-01 | [22203](https://github.com/airbytehq/airbyte/pull/22203) | Handle Null cursor values | +| `1.0.1` | 2023-01-31 | [22015](https://github.com/airbytehq/airbyte/pull/22015) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| `1.0.0` | 2023-01-25 | [21790](https://github.com/airbytehq/airbyte/pull/21790) | Fix `activities_*` stream schemas | +| `0.1.12` | 2023-01-19 | [20973](https://github.com/airbytehq/airbyte/pull/20973) | Fix encoding error (note: this change is not in version 1.0.0, but is in later versions | +| `0.1.11` | 2022-09-30 | [17445](https://github.com/airbytehq/airbyte/pull/17445) | Do not use temporary files for memory optimization | +| `0.1.10` | 2022-09-30 | [17445](https://github.com/airbytehq/airbyte/pull/17445) | Optimize memory consumption | +| `0.1.9` | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream sate. | +| `0.1.7` | 2022-08-23 | [15817](https://github.com/airbytehq/airbyte/pull/15817) | Improved unit test coverage | +| `0.1.6` | 2022-08-21 | [15824](https://github.com/airbytehq/airbyte/pull/15824) | Fix semi incremental streams: do not ignore start date, make one api call instead of multiple | +| `0.1.5` | 2022-08-16 | [15683](https://github.com/airbytehq/airbyte/pull/15683) | Retry failed creation of a job instead of skipping it | +| `0.1.4` | 2022-06-20 | [13930](https://github.com/airbytehq/airbyte/pull/13930) | Process failing creation of export jobs | +| `0.1.3` | 2021-12-10 | [8429](https://github.com/airbytehq/airbyte/pull/8578) | Updated titles and descriptions | +| `0.1.2` | 2021-12-03 | [8483](https://github.com/airbytehq/airbyte/pull/8483) | Improve field conversion to conform schema | +| `0.1.1` | 2021-11-29 | [0000](https://github.com/airbytehq/airbyte/pull/0000) | Fix timestamp value format issue | +| `0.1.0` | 2021-09-06 | [5863](https://github.com/airbytehq/airbyte/pull/5863) | Release Marketo CDK Connector | diff --git a/docs/integrations/sources/metabase.md b/docs/integrations/sources/metabase.md index 40e05a5f6223..a09eacaa239d 100644 --- a/docs/integrations/sources/metabase.md +++ b/docs/integrations/sources/metabase.md @@ -70,6 +70,7 @@ The Metabase source connector supports the following [sync modes](https://docs.a | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------| +| 1.1.0 | 2023-10-31 | [31909](https://github.com/airbytehq/airbyte/pull/31909) | Add `databases` and `native_query_snippets` streams | | 1.0.1 | 2023-07-20 | [28470](https://github.com/airbytehq/airbyte/pull/27777) | Update CDK to 0.47.0 | | 1.0.0 | 2023-06-27 | [27777](https://github.com/airbytehq/airbyte/pull/27777) | Remove Activity Stream | | 0.3.1 | 2022-12-15 | [20535](https://github.com/airbytehq/airbyte/pull/20535) | Run on CDK 0.15.0 | diff --git a/docs/integrations/sources/microsoft-onedrive.md b/docs/integrations/sources/microsoft-onedrive.md new file mode 100644 index 000000000000..e3f1335d2571 --- /dev/null +++ b/docs/integrations/sources/microsoft-onedrive.md @@ -0,0 +1,126 @@ +# Microsoft OneDrive + +This page contains the setup guide and reference information for the Microsoft OneDrive source connector. + +### Requirements + +* Application \(client\) ID +* Directory \(tenant\) ID +* Drive name +* Folder Path +* Client secrets + +## Setup guide + + + +**For Airbyte Cloud:** + +1. Navigate to the Airbyte Open Source dashboard. +2. Click **Sources** and then click **+ New source**. +3. On the Set up the source page, select **Microsoft OneDrive** from the Source type dropdown. +4. Enter the name for the Microsoft OneDrive connector. +5. Enter **Drive Name**. To find your drive name go to settings and at the top of setting menu you can find the name of your drive. +6. Enter **Folder Path**. +7. The **OAuth2.0** authorization method is selected by default. Click **Authenticate your Microsoft OneDrive account**. Log in and authorize your Microsoft account. +8. For **Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. +9. Add a stream: + 1. Write the **File Type** + 2. In the **Format** box, use the dropdown menu to select the format of the files you'd like to replicate. The supported formats are **CSV**, **Parquet**, **Avro** and **JSONL**. Toggling the **Optional fields** button within the **Format** box will allow you to enter additional configurations based on the selected format. For a detailed breakdown of these settings, refer to the [File Format section](#file-format-settings) below. + 3. Give a **Name** to the stream + 4. (Optional) - If you want to enforce a specific schema, you can enter a **Input schema**. By default, this value is set to `{}` and will automatically infer the schema from the file\(s\) you are replicating. For details on providing a custom schema, refer to the [User Schema section](#user-schema). + 5. Optionally, enter the **Globs** which dictates which files to be synced. This is a regular expression that allows Airbyte to pattern match the specific files to replicate. If you are replicating all the files within your bucket, use `**` as the pattern. For more precise pattern matching options, refer to the [Path Patterns section](#path-patterns) below. +10. Click **Set up source** + + + + +**For Airbyte Open Source:** + +### Step 1: Set up OneDrive application + +The Microsoft Graph API uses OAuth for authentication. Microsoft Graph exposes granular permissions that control the access that apps have to resources, like users, groups, and mail. When a user signs in to your app they, or, in some cases, an administrator, are given a chance to consent to these permissions. If the user consents, your app is given access to the resources and APIs that it has requested. For apps that don't take a signed-in user, permissions can be pre-consented to by an administrator when the app is installed. + +Microsoft Graph has two types of permissions: + +* **Delegated permissions** are used by apps that have a signed-in user present. For these apps, either the user or an administrator consents to the permissions that the app requests, and the app can act as the signed-in user when making calls to Microsoft Graph. Some delegated permissions can be consented by non-administrative users, but some higher-privileged permissions require administrator consent. +* **Application permissions** are used by apps that run without a signed-in user present; for example, apps that run as background services or daemons. Application permissions can only be consented by an administrator. + +This source requires **Application permissions**. Follow these [instructions](https://docs.microsoft.com/en-us/graph/auth-v2-service?context=graph%2Fapi%2F1.0&view=graph-rest-1.0) for creating an app in the Azure portal. This process will produce the `client_id`, `client_secret`, and `tenant_id` needed for the tap configuration file. + +1. Login to [Azure Portal](https://portal.azure.com/#home) +2. Click upper-left menu icon and select **Azure Active Directory** +3. Select **App Registrations** +4. Click **New registration** +5. Register an application + 1. Name: + 2. Supported account types: Accounts in this organizational directory only + 3. Register \(button\) +6. Record the client\_id and tenant\_id which will be used by the tap for authentication and API integration. +7. Select **Certificates & secrets** +8. Provide **Description and Expires** + 1. Description: tap-microsoft-onedrive client secret + 2. Expires: 1-year + 3. Add +9. Copy the client secret value, this will be the client\_secret +10. Select **API permissions** + 1. Click **Add a permission** +11. Select **Microsoft Graph** +12. Select **Application permissions** +13. Select the following permissions: + 1. Files + * Files.Read.All +14. Click **Add permissions** +15. Click **Grant admin consent** + +### Step 2: Set up the Microsoft OneDrive connector in Airbyte + +1. Navigate to the Airbyte Open Source dashboard. +2. Click **Sources** and then click **+ New source**. +3. On the **Set up** the source page, select **Microsoft OneDrive** from the Source type dropdown. +4. Enter the name for the Microsoft OneDrive connector. +5. Enter **Drive Name**. To find your drive name go to settings and at the top of setting menu you can find the name of your drive. +6. Enter **Folder Path**. +7. Switch to **Service Key Authentication** +8. For **User Practical Name**, enter the [UPN](https://learn.microsoft.com/en-us/sharepoint/list-onedrive-urls) for your user. +9. Enter **Tenant ID**, **Client ID** and **Client secret**. +10. For **Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. +11. Add a stream: + 1. Write the **File Type** + 2. In the **Format** box, use the dropdown menu to select the format of the files you'd like to replicate. The supported formats are **CSV**, **Parquet**, **Avro** and **JSONL**. Toggling the **Optional fields** button within the **Format** box will allow you to enter additional configurations based on the selected format. For a detailed breakdown of these settings, refer to the [File Format section](#file-format-settings) below. + 3. Give a **Name** to the stream + 4. (Optional) - If you want to enforce a specific schema, you can enter a **Input schema**. By default, this value is set to `{}` and will automatically infer the schema from the file\(s\) you are replicating. For details on providing a custom schema, refer to the [User Schema section](#user-schema). + 5. Optionally, enter the **Globs** which dictates which files to be synced. This is a regular expression that allows Airbyte to pattern match the specific files to replicate. If you are replicating all the files within your bucket, use `**` as the pattern. For more precise pattern matching options, refer to the [Path Patterns section](#path-patterns) below. +12. Click **Set up source** + + + +## Sync overview + +### Data type mapping + +| Integration Type | Airbyte Type | +|:-----------------|:-------------| +| `string` | `string` | +| `number` | `number` | +| `array` | `array` | +| `object` | `object` | + +### Features + +| Feature | Supported?\(Yes/No\) | +|:------------------------------|:---------------------| +| Full Refresh Sync | Yes | +| Incremental Sync | Yes | + +### Performance considerations + +The connector is restricted by normal Microsoft Graph [requests limitation](https://docs.microsoft.com/en-us/graph/throttling). + +## Changelog + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:--------------------------| +| 0.1.2 | 2021-12-22 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Add ql and sl to metadata | +| 0.1.1 | 2021-12-15 | [33539](https://github.com/airbytehq/airbyte/pull/33539) | Fix for docs name | +| 0.1.0 | 2021-12-06 | [32655](https://github.com/airbytehq/airbyte/pull/32655) | New source | diff --git a/docs/integrations/sources/microsoft-teams-migrations.md b/docs/integrations/sources/microsoft-teams-migrations.md new file mode 100644 index 000000000000..5610ecd721ad --- /dev/null +++ b/docs/integrations/sources/microsoft-teams-migrations.md @@ -0,0 +1,38 @@ +# Microsoft teams Migration Guide + +## Upgrading to 1.0.0 + +Version 1.0.0 of the Microsoft Teams source connector introduces breaking changes to the schemas of all streams. A full schema refresh is required to ensure a seamless upgrade to this version. + +### Refresh schemas and reset data + +1. Select **Connections** in the main navbar. +2. From the list of your existing connections, select the connection(s) affected by the update. +3. Select the **Replication** tab, then select **Refresh source schema**. + +:::note +Any detected schema changes will be listed for your review. Select **OK** when you are ready to proceed. +::: + +4. At the bottom of the page, select **Save changes**. + +:::caution +Depending on your destination, you may be prompted to **Reset all streams**. Although this step is not required to proceed, it is highly recommended for users who have selected `Full Refresh | Append` sync mode, as the updated schema may lead to inconsistencies in the data structure within the destination. +::: + +5. Select **Save connection**. This will reset the data in your destination (if selected) and initiate a fresh sync. + +For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). + +### Changes in 1.0.0 + +- The naming convention for field names in previous versions used "snake_case", which is not aligned with the "camelCase" convention used by the Microsoft Graph API. For example: + +`user_id` -> `userId` +`created_date` -> `createdDate` + +With the update to "camelCase", fields that may have been unrecognized or omitted in earlier versions will now be properly mapped and included in the data synchronization process, enhancing the accuracy and completeness of your data. + +- The `team_device_usage_report` stream contained a fatal bug that could lead to crashes during syncs. You should now be able to reliably use this stream during syncs. + +- `Date` and `date-time` fields have been typed as airbyte_type `date` and `timestamp_without_timezone`, respectively. diff --git a/docs/integrations/sources/microsoft-teams.md b/docs/integrations/sources/microsoft-teams.md index 1adde3295778..cc3846a489d3 100644 --- a/docs/integrations/sources/microsoft-teams.md +++ b/docs/integrations/sources/microsoft-teams.md @@ -29,22 +29,24 @@ Some APIs aren't supported in v1.0, e.g. channel messages and channel messages r ### Data type mapping -| Integration Type | Airbyte Type | Notes | -| :--- | :--- | :--- | -| `string` | `string` | | -| `number` | `number` | | -| `array` | `array` | | -| `object` | `object` | | +| Integration Type | Airbyte Type | +| :--------------- | :--------------------------- | +| `string` | `string` | +| `number` | `number` | +| `date` | `date` | +| `datetime` | `timestamp_without_timezone` | +| `array` | `array` | +| `object` | `object` | ### Features -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental Sync | Coming soon | | -| Replicate Incremental Deletes | Coming soon | | -| SSL connection | Yes | | -| Namespaces | No | | +| Feature | Supported? | +| :---------------------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental Sync | No | +| Replicate Incremental Deletes | No | +| SSL connection | Yes | +| Namespaces | No | ### Performance considerations @@ -54,9 +56,9 @@ The connector is restricted by normal Microsoft Graph [requests limitation](http ### Requirements -* Application \(client\) ID +* Application \(client\) ID * Directory \(tenant\) ID -* Client secrets +* Client secrets ### Setup guide @@ -157,8 +159,9 @@ Token acquiring implemented by [instantiate](https://docs.microsoft.com/en-us/az ## CHANGELOG -| Version | Date | Pull Request | Subject | -|:--------|:-----------| :--- | :--- | -| 0.2.5 | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Update titles and descriptions | -| 0.2.4 | 2021-12-07 | [7807](https://github.com/airbytehq/airbyte/pull/7807) | Implement OAuth support | -| 0.2.3 | 2021-12-06 | [8469](https://github.com/airbytehq/airbyte/pull/8469) | Migrate to the CDK | +| Version | Date | Pull Request | Subject | +|:------- |:---------- | :------------------------------------------------------- | :----------------------------- | +| 1.0.0 | 2024-01-04 | [33959](https://github.com/airbytehq/airbyte/pull/33959) | Schema updates | +| 0.2.5 | 2021-12-14 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Update titles and descriptions | +| 0.2.4 | 2021-12-07 | [7807](https://github.com/airbytehq/airbyte/pull/7807) | Implement OAuth support | +| 0.2.3 | 2021-12-06 | [8469](https://github.com/airbytehq/airbyte/pull/8469) | Migrate to the CDK | diff --git a/docs/integrations/sources/mixpanel-migrations.md b/docs/integrations/sources/mixpanel-migrations.md new file mode 100644 index 000000000000..424628775cf4 --- /dev/null +++ b/docs/integrations/sources/mixpanel-migrations.md @@ -0,0 +1,9 @@ +# Mixpanel Migration Guide + +## Upgrading to 2.0.0 + +In this release, the default primary key for stream Export has been deleted, allowing users to select the key that best fits their data. Refreshing the source schema and resetting affected streams is necessary only if new primary keys are to be applied following the upgrade. + +## Upgrading to 1.0.0 + +In this release, the datetime field of stream engage has had its type changed from date-time to string due to inconsistent data from Mixpanel. Additionally, the primary key for stream export has been fixed to uniquely identify records. Users will need to refresh the source schema and reset affected streams after upgrading. diff --git a/docs/integrations/sources/mixpanel.md b/docs/integrations/sources/mixpanel.md index b0f31a65d224..b4cc8d6cfaae 100644 --- a/docs/integrations/sources/mixpanel.md +++ b/docs/integrations/sources/mixpanel.md @@ -43,6 +43,10 @@ Note: Incremental sync returns duplicated \(old records\) for the state date due - [Cohorts](https://developer.mixpanel.com/reference/cohorts-list) \(Incremental\) - [Cohort Members](https://developer.mixpanel.com/reference/engage-query) \(Incremental\) +### Primary key selection for Export stream + +Mixpanel recommends using `[insert_id, event_time, event_name, distinct_id]` as the primary key. However, note that some rows might lack an `insert_id` for certain users. Ensure you select a primary key that aligns with your data. + ## Performance considerations Syncing huge date windows may take longer due to Mixpanel's low API rate-limits \(**60 reqs per hour**\). @@ -50,11 +54,19 @@ Syncing huge date windows may take longer due to Mixpanel's low API rate-limits ## CHANGELOG | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------- | -| 0.1.37 | 2022-07-20 | [27932](https://github.com/airbytehq/airbyte/pull/27932) | Fix spec: change start/end date format to `date` | -| 0.1.36 | 2022-06-27 | [27752](https://github.com/airbytehq/airbyte/pull/27752) | Partially revert version 0.1.32; Use exponential backoff; | -| 0.1.35 | 2022-06-12 | [27252](https://github.com/airbytehq/airbyte/pull/27252) | Add should_retry False for 402 error | -| 0.1.34 | 2022-05-15 | [21837](https://github.com/airbytehq/airbyte/pull/21837) | Add "insert_id" field to "export" stream schema | +|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------------------------| +| 2.0.1 | 2024-01-11 | [34147](https://github.com/airbytehq/airbyte/pull/34147) | prepare for airbyte-lib | +| 2.0.0 | 2023-10-30 | [31955](https://github.com/airbytehq/airbyte/pull/31955) | Delete the default primary key for the Export stream | +| 1.0.1 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 1.0.0 | 2023-09-27 | [30025](https://github.com/airbytehq/airbyte/pull/30025) | Fix type of datetime field in engage stream; fix primary key for export stream. | +| 0.1.41 | 2023-09-26 | [30149](https://github.com/airbytehq/airbyte/pull/30149) | Change config schema; set checkpointing interval; add suggested streams; add casting datetime fields. | +| 0.1.40 | 2022-09-20 | [30090](https://github.com/airbytehq/airbyte/pull/30090) | Handle 400 error when the credentials become expired | +| 0.1.39 | 2023-09-15 | [30469](https://github.com/airbytehq/airbyte/pull/30469) | Add default primary key `distinct_id` to `Export` stream | +| 0.1.38 | 2023-08-31 | [30028](https://github.com/airbytehq/airbyte/pull/30028) | Handle gracefully project timezone mismatch | +| 0.1.37 | 2023-07-20 | [27932](https://github.com/airbytehq/airbyte/pull/27932) | Fix spec: change start/end date format to `date` | +| 0.1.36 | 2023-06-27 | [27752](https://github.com/airbytehq/airbyte/pull/27752) | Partially revert version 0.1.32; Use exponential backoff; | +| 0.1.35 | 2023-06-12 | [27252](https://github.com/airbytehq/airbyte/pull/27252) | Add should_retry False for 402 error | +| 0.1.34 | 2023-05-15 | [21837](https://github.com/airbytehq/airbyte/pull/21837) | Add "insert_id" field to "export" stream schema | | 0.1.33 | 2023-04-25 | [25543](https://github.com/airbytehq/airbyte/pull/25543) | Set should_retry for 104 error in stream export | | 0.1.32 | 2023-04-11 | [25056](https://github.com/airbytehq/airbyte/pull/25056) | Set HttpAvailabilityStrategy, add exponential backoff, streams export and annotations add undeclared fields | | 0.1.31 | 2023-02-13 | [22936](https://github.com/airbytehq/airbyte/pull/22936) | Specified date formatting in specification | @@ -87,4 +99,4 @@ Syncing huge date windows may take longer due to Mixpanel's low API rate-limits | 0.1.3 | 2021-10-30 | [7505](https://github.com/airbytehq/airbyte/issues/7505) | Guarantee that standard and custom mixpanel properties in the `Engage` stream are written as strings | | 0.1.2 | 2021-11-02 | [7439](https://github.com/airbytehq/airbyte/issues/7439) | Added delay for all streams to match API limitation of requests rate | | 0.1.1 | 2021-09-16 | [6075](https://github.com/airbytehq/airbyte/issues/6075) | Added option to select project region | -| 0.1.0 | 2021-07-06 | [3698](https://github.com/airbytehq/airbyte/issues/3698) | Created CDK native mixpanel connector | +| 0.1.0 | 2021-07-06 | [3698](https://github.com/airbytehq/airbyte/issues/3698) | Created CDK native mixpanel connector | \ No newline at end of file diff --git a/docs/integrations/sources/monday-migrations.md b/docs/integrations/sources/monday-migrations.md new file mode 100644 index 000000000000..9d095b9e127f --- /dev/null +++ b/docs/integrations/sources/monday-migrations.md @@ -0,0 +1,77 @@ +# Monday Migration Guide + +## Upgrading to 2.0.0 + +Source Monday has deprecated API version 2023-07. We have upgraded the connector to the latest API version 2024-01. In this new version, the Id field has changed from an integer to a string in the streams Boards, Items, Tags, Teams, Updates, Users and Workspaces. Please reset affected streams. + +## Connector Upgrade Guide + +### For Airbyte Open Source: Update the local connector image + +Airbyte Open Source users must manually update the connector image in their local registry before proceeding with the migration. To do so: + +1. Select **Settings** in the main navbar. + 1. Select **Sources**. +2. Find Monday in the list of connectors. + +:::note +You will see two versions listed, the current in-use version and the latest version available. +::: + +3. Select **Change** to update your OSS version to the latest available version. + +### Update the connector version + +1. Select **Sources** in the main navbar. +2. Select the instance of the connector you wish to upgrade. + +:::note +Each instance of the connector must be updated separately. If you have created multiple instances of a connector, updating one will not affect the others. +::: + +3. Select **Upgrade** + 1. Follow the prompt to confirm you are ready to upgrade to the new version. + + +### Refresh schemas and reset data + +1. Select **Connections** in the main navbar. +2. Select the connection(s) affected by the update. +3. Select the **Replication** tab. + 1. Select **Refresh source schema**. + 2. Select **OK**. +:::note +Any detected schema changes will be listed for your review. +::: +4. Select **Save changes** at the bottom of the page. + 1. Ensure the **Reset all streams** option is checked. +5. Select **Save connection**. +:::note +This will reset the data in your destination and initiate a fresh sync. +::: + +For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). + + +### Refresh affected schemas and reset data + +1. Select **Connections** in the main navb nar. + 1. Select the connection(s) affected by the update. +2. Select the **Replication** tab. + 1. Select **Refresh source schema**. + 2. Select **OK**. +:::note +Any detected schema changes will be listed for your review. +::: +3. Select **Save changes** at the bottom of the page. + 1. Ensure the **Reset affected streams** option is checked. +:::note +Depending on destination type you may not be prompted to reset your data. +::: +4. Select **Save connection**. +:::note +This will reset the data in your destination and initiate a fresh sync. +::: + +For more information on resetting your data in Airbyte, see [this page](https://docs.airbyte.com/operator-guides/reset). + diff --git a/docs/integrations/sources/monday.md b/docs/integrations/sources/monday.md index 1b9694f43172..f3c8ab44f4fd 100644 --- a/docs/integrations/sources/monday.md +++ b/docs/integrations/sources/monday.md @@ -1,5 +1,7 @@ # Monday +This page contains the setup guide and reference information for the [Monday](https://monday.com/) source connector. + ## Prerequisites * Monday API Token / Monday Access Token @@ -15,18 +17,20 @@ You can get the API token for Monday by going to Profile picture (bottom left co 3. On the Set up the source page, enter the name for the Monday connector and select **Monday** from the Source type dropdown. 4. Fill in your API Key or authenticate using OAuth and then click **Set up source**. -### Connect using `OAuth 2.0` option: +### Connect using `OAuth 2.0` option + 1. Select `OAuth2.0` in `Authorization Method`. 2. Click on `authenticate your Monday account`. -2. Proceed the authentication using your credentials for your Monday account. +3. Proceed with the authentication using the credentials for your Monday account. + +### Connect using `API Token` option -### Connect using `API Token` option: 1. Generate an API Token as described [here](https://developer.monday.com/api-reference/docs/authentication). 2. Use the generated `api_token` in the Airbyte connection. ## Supported sync modes -The Monday supports full refresh syncs +The Monday source connector supports the following features: | Feature | Supported? | |:------------------|:-----------| @@ -49,6 +53,7 @@ Several output streams are available from this source: * [Workspaces](https://developer.monday.com/api-reference/docs/workspaces) Important Notes: + * `Columns` are available from the `Boards` stream. By syncing the `Boards` stream you will get the `Columns` for each `Board` synced in the database The typical name of the table depends on the `destination` you use like `boards.columns`, for instance. @@ -61,16 +66,17 @@ Ids of boards and items are extracted from activity logs events and used to sele Some data may be lost if the time between incremental syncs is longer than the activity logs retention time for your plan. Check your Monday plan at https://monday.com/pricing. - ## Performance considerations The Monday connector should not run into Monday API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. - ## Changelog | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------|:------------------------------------------------------------------------| +| 2.0.0 | 2024-01-12 | [34108](https://github.com/airbytehq/airbyte/pull/34108) | Migrated to the latest API version: 2024-01 | +| 1.1.4 | 2023-12-13 | [33448](https://github.com/airbytehq/airbyte/pull/33448) | Increase test coverage and migrate to base image | +| 1.1.3 | 2023-09-23 | [30248](https://github.com/airbytehq/airbyte/pull/30248) | Add new field "type" to board stream | | 1.1.2 | 2023-08-23 | [29777](https://github.com/airbytehq/airbyte/pull/29777) | Add retry for `502` error | | 1.1.1 | 2023-08-15 | [29429](https://github.com/airbytehq/airbyte/pull/29429) | Ignore `null` records in response | | 1.1.0 | 2023-07-05 | [27944](https://github.com/airbytehq/airbyte/pull/27944) | Add incremental sync for Items and Boards streams | diff --git a/docs/integrations/sources/mongodb-v2-migrations.md b/docs/integrations/sources/mongodb-v2-migrations.md new file mode 100644 index 000000000000..93211e70e93f --- /dev/null +++ b/docs/integrations/sources/mongodb-v2-migrations.md @@ -0,0 +1,25 @@ +# MongoDb Migration Guide + +## Upgrading to 1.0.0 + +This version introduces a general availability version of the MongoDB V2 source connector, which leverages +[Change Data Capture (CDC)](https://docs.airbyte.com/understanding-airbyte/cdc) to improve the performance and +reliability of syncs. This version provides better error handling, incremental delivery of data and improved +reliability of large syncs via frequent checkpointing. + +**THIS VERSION INCLUDES BREAKING CHANGES FROM PREVIOUS VERSIONS OF THE CONNECTOR!** + +The changes will require you to reconfigure your existing MongoDB V2 configured source connectors. To review the +breaking changes and to learn how to upgrade the connector, refer to the [MongoDB V2 source connector documentation](mongodb-v2#upgrade-from-previous-version). +Additionally, you can manually update existing connections prior to the next scheduled sync to perform the upgrade or +re-create the source using the new configuration. + +Worthy of specific mention, this version includes: + +- Support for MongoDB replica sets only +- Use of Change Data Capture for incremental delivery of changes +- Frequent checkpointing of synced data +- Sampling of fields for schema discovery +- Required SSL/TLS connections + +Learn more about what's new in the connection, view the updated documentation [here](mongodb-v2). \ No newline at end of file diff --git a/docs/integrations/sources/mongodb-v2.md b/docs/integrations/sources/mongodb-v2.md index 9fca57ea3805..4ff1a9a2f0fb 100644 --- a/docs/integrations/sources/mongodb-v2.md +++ b/docs/integrations/sources/mongodb-v2.md @@ -1,63 +1,86 @@ # Mongo DB -The MongoDB source allows to sync data from MongoDb. Source supports Full Refresh and Incremental sync strategies. +Airbyte's certified MongoDB connector offers the following features: -## Resulting schema +* [Change Data Capture (CDC)](https://docs.airbyte.com/understanding-airbyte/cdc) via [MongoDB's change streams](https://www.mongodb.com/docs/manual/changeStreams/)/[Replica Set Oplog](https://www.mongodb.com/docs/manual/core/replica-set-oplog/). +* Reliable replication of any collection size with [checkpointing](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#state--checkpointing) and chunking of data reads. -MongoDB does not have anything like table definition, thus we have to define column types from actual attributes and their values. Discover phase have two steps: +## Quick Start -### Step 1. Find all unique properties +This section provides information about configuring the MongoDB V2 source connector. If you are upgrading from a +previous version of the MongoDB V2 source connector, please refer to the [upgrade](#upgrade-from-previous-version) instructions +in this document. -Connector select 10k documents to collect all distinct field. +### New Installation/New Source Connector Configuration -### Step 2. Determine property types +Here is an outline of the minimum required steps to configure a new MongoDB V2 source connector: -For each property found, connector determines its type, if all the selected values have the same type - connector will set appropriate type to the property. In all other cases connector will fallback to `string` type. +1. Create or discover the configuration of a [MongoDB replica set](https://www.mongodb.com/docs/manual/replication/), either hosted in [MongoDB Atlas](https://www.mongodb.com/atlas/database) or self-hosted. +2. Create a new MongoDB source in the Airbyte UI +3. (Airbyte Cloud Only) Allow inbound traffic from Airbyte IPs -## Features +Once this is complete, you will be able to select MongoDB as a source for replicating data. -| Feature | Supported | -| :---------------------------- | :-------- | -| Full Refresh Sync | Yes | -| Incremental - Append Sync | Yes | -| Replicate Incremental Deletes | No | -| Namespaces | No | +#### Step 1: Create a dedicated read-only MongoDB user -### Full Refresh sync +These steps create a dedicated, read-only user for replicating data. Alternatively, you can use an existing MongoDB user with +access to the database. -Works as usual full refresh sync. +##### MongoDB Atlas -### Incremental sync +1. Log in to the MongoDB Atlas dashboard. +2. From the dashboard, click on "Database Access" under "Security" -Cursor field can not be nested. Currently only top level document properties are supported. +![Security Database Access](../../.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_2.png) -Cursor should **never** be blank. In case cursor is blank - the incremental sync results might be unpredictable and will totally rely on MongoDB comparison algorithm. +3. Click on the "+ ADD NEW DATABASE USER" button. -Only `datetime` and `number` cursor types are supported. Cursor type is determined based on the cursor field name: +![Add New Database User](../../.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_3.png) -- `datetime` - if cursor field name contains a string from: `time`, `date`, `_at`, `timestamp`, `ts` -- `number` - otherwise +4. On the "Add new Database User" modal dialog, choose "Password" for the "Authentication Method". -## Getting started +![Authentication Method](../../.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_4.png) -This guide describes in details how you can configure MongoDB for integration with Airbyte. +5. In the "Password Authentication" section, set the username to `READ_ONLY_USER` in the first text box and set a password in the second text box. -### Create users +![Username and Password](../../.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_5.png) -Run `mongo` shell, switch to `admin` database and create a `READ_ONLY_USER`. `READ_ONLY_USER` will be used for Airbyte integration. Please make sure that user has read-only privileges. +6. Under "Database User Privileges", click on "Select one built-in role for this user" under "Built-in Role" and choose "Only read any database". -```javascript -mongo -use admin; -db.createUser({user: "READ_ONLY_USER", pwd: "READ_ONLY_PASSWORD", roles: [{role: "read", db: "TARGET_DATABASE"}]}) -``` +![Database User Privileges](../../.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_6.png) + +7. Enable "Restrict Access to Specific Clusters/Federated Database instances" and enable only those clusters/database that you wish to replicate. + +![Restrict Access](../../.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_7.png) + +8. Click on "Add User" at the bottom to save the user. + +![Add User](../../.gitbook/assets/source/mongodb/mongodb_atlas_database_user_step_8.png) -**Make sure the user have appropriate access levels, a user with higher access levels may throw an exception.** +##### Self Hosted -### Enable MongoDB authentication +These instructions assume that the [MongoDB shell](https://www.mongodb.com/docs/mongodb-shell/) is installed. To +install the MongoDB shell, please follow [these instructions](https://www.mongodb.com/docs/mongodb-shell/install/#std-label-mdb-shell-install). -Open `/etc/mongod.conf` and add/replace specific keys: +1. From a terminal window, launch the MongoDB shell: +```shell +> mongosh --username ; +``` +2. Switch to the `admin` database: +```shell +test> use admin +switched to db admin +``` +3. Create the `READ_ONLY_USER` user with the `read` role: +```shell +admin> db.createUser({user: "READ_ONLY_USER", pwd: "READ_ONLY_PASSWORD", roles: [{role: "read", db: "TARGET_DATABASE"}]}) +``` + +:::note +Replace `READ_ONLY_PASSWORD` with a password of your choice and `TARGET_DATABASE` with the name of the database to be replicated. +::: +4. Next, enable authentication, if not already enabled. Start by editing the `/etc/mongodb.conf` by adding/editing these specific keys: ```yaml net: bindIp: 0.0.0.0 @@ -66,42 +89,148 @@ security: authorization: enabled ``` -Binding to `0.0.0.0` will allow to connect to database from any IP address. +:::note +Setting the `bindIp` key to `0.0.0.0` will allow connections to database from any IP address. Setting the `security.authorization` key to `enabled` will enable security and only allow authenticated users to access the database. +::: + +#### Step 2: Discover the MongoDB cluster connection string + +These steps outline how to discover the connection string of your MongoDB instance. + +##### MongoDB Atlas + +Atlas is MongoDB's [cloud-hosted offering](https://www.mongodb.com/atlas/database). Below are the steps to discover +the connection configuration for a MongoDB Atlas-hosted replica set cluster: + +1. Log in to the [MongoDB Atlas dashboard](https://cloud.mongodb.com/). +2. From the dashboard, click on the "Connect" button of the source cluster. + +![Connect to Source Cluster](../../.gitbook/assets/source/mongodb/mongodb_atlas_connection_string_step_2.png) + +3. On the "Connect to <cluster name>" modal dialog, select "Shell" under the "Access your data through tools" section. + +![Shell Connect](../../.gitbook/assets/source/mongodb/mongodb_atlas_connection_string_step_3.png) + +4. Copy the connection string from the entry labeled "2. Run your connection string in your command line" on the modal dialog, removing/avoiding the quotation marks. + +![Copy Connection String](../../.gitbook/assets/source/mongodb/mongodb_atlas_connection_string_step_4.png) + +##### Self Hosted Cluster + +Self-hosted clusters are MongoDB instances that are hosted outside of [MongoDB Atlas](https://www.mongodb.com/atlas/database). Below are the steps to discover +the connection string for a MongoDB self-hosted replica set cluster. -The last line will enable MongoDB security. Now only authenticated users will be able to access the database. +1. Refer to the [MongoDB connection string documentation](https://www.mongodb.com/docs/manual/reference/connection-string/#find-your-self-hosted-deployment-s-connection-string) for instructions +on discovering a self-hosted deployment connection string. + +#### Step 3: Configure the Airbyte MongoDB Source + +To configure the Airbyte MongoDB source, use the database credentials and connection string from steps 1 and 2, respectively. +The source will test the connection to the MongoDB instance upon creation. + +## Replication Methods + +The MongoDB source utilizes change data capture (CDC) as a reliable way to keep your data up to date. + +### CDC + +Airbyte utilizes [the change streams feature](https://www.mongodb.com/docs/manual/changeStreams/) of a [MongoDB replica set](https://www.mongodb.com/docs/manual/replication/) to incrementally capture inserts, updates and deletes using a replication plugin. To learn more how Airbyte implements CDC, refer to [Change Data Capture (CDC)](https://docs.airbyte.com/understanding-airbyte/cdc/). + +### Schema Enforcement + +By default the MongoDB V2 source connector enforces a schema. This means that while setting up a connector it will sample a configureable number of docuemnts and will create a set of fields to sync. From that set of fields, an admin can then deselect specific fields from the Replication screen to filter them out from the sync. + +When the schema enforced option is disabled, MongoDB collections are read in schema-less mode which doesn't assume documents share the same structure. +This allows for greater flexibility in reading data that is unstructured or vary a lot in between documents in a single collection. +When schema is not enforced, each document will generate a record that only contains the following top-level fields: +```json +{ + "_id": , + "data": {} +} +``` +The contents of `data` will vary according to the contents of each document read from MongoDB. +Unlike in Schema enforced mode, the same field can vary in type between document. For example field `"xyz"` may be a String on one document and a Date on another. +As a result no field will be omitted and no document will be rejected. +When Schema is not enforced there is not way to deselect fields as all fields are read for every document. + +## Limitations & Troubleshooting + +### MongoDB Oplog and Change Streams + +[MongoDB's Change Streams](https://www.mongodb.com/docs/manual/changeStreams/) are based on the [Replica Set Oplog](https://www.mongodb.com/docs/manual/core/replica-set-oplog/). This has retention limitations. Syncs that run less frequently than the retention period of the Oplog may encounter issues with missing data. + +We recommend adjusting the Oplog size for your MongoDB cluster to ensure it holds at least 24 hours of changes. For optimal results, we suggest expanding it to maintain a week's worth of data. To adjust your Oplog size, see the corresponding tutorials for [MongoDB Atlas](https://www.mongodb.com/docs/atlas/cluster-additional-settings/#set-oplog-size) (fully-managed) and [MongoDB shell](https://www.mongodb.com/docs/manual/tutorial/change-oplog-size/) (self-hosted). + +If you are running into an issue similar to "invalid resume token", it may mean you need to: +1. Increase the Oplog retention period. +2. Increase the Oplog size. +3. Increase the Airbyte sync frequency. + +You can run the commands outlined [in this tutorial](https://www.mongodb.com/docs/manual/tutorial/troubleshoot-replica-sets/#check-the-size-of-the-oplog) to verify the current of your Oplog. The expect output is: + +```yaml +configured oplog size: 10.10546875MB +log length start to end: 94400 (26.22hrs) +oplog first event time: Mon Mar 19 2012 13:50:38 GMT-0400 (EDT) +oplog last event time: Wed Oct 03 2012 14:59:10 GMT-0400 (EDT) +now: Wed Oct 03 2012 15:00:21 GMT-0400 (EDT) +``` -### Configure firewall +When importing a large MongoDB collection for the first time, the import duration might exceed the Oplog retention period. The Oplog is crucial for incremental updates, and an invalid resume token will require the MongoDB collection to be re-imported to ensure no source updates were missed. -Make sure that MongoDB is accessible from external servers. Specific commands will depend on the firewall you are using \(UFW/iptables/AWS/etc\). Please refer to appropriate documentation. +### Supported MongoDB Clusters -Your `READ_ONLY_USER` should now be ready for use with Airbyte. +* Only supports [replica set](https://www.mongodb.com/docs/manual/replication/) cluster type. +* TLS/SSL is required by this connector. TLS/SSL is enabled by default for MongoDB Atlas clusters. To enable TSL/SSL connection for a self-hosted MongoDB instance, please refer to [MongoDb Documentation](https://docs.mongodb.com/manual/tutorial/configure-ssl/). +* Views, capped collections and clustered collections are not supported. +* Empty collections are excluded from schema discovery. +* Collections with different data types for the values in the `_id` field among the documents in a collection are not supported. All `_id` values within the collection must be the same data type. +* Atlas DB cluster are only supported in a dedicated M10 tier and above. Lower tiers may fail during connection setup. -### TLS/SSL on a Connection +### Schema Discovery & Enforcement -It is recommended to use encrypted connection. Connection with TLS/SSL security protocol for MongoDb Atlas Cluster and Replica Set instances is enabled by default. To enable TSL/SSL connection with Standalone MongoDb instance, please refer to [MongoDb Documentation](https://docs.mongodb.com/manual/tutorial/configure-ssl/). +* Schema discovery uses [sampling](https://www.mongodb.com/docs/manual/reference/operator/aggregation/sample/) of the documents to collect all distinct top-level fields. This value is universally applied to all collections discovered in the target database. The approach is modelled after [MongoDB Compass sampling](https://www.mongodb.com/docs/compass/current/sampling/) and is used for efficiency. By default, 10,000 documents are sampled. This value can be increased up to 100,000 documents to increase the likelihood that all fields will be discovered. However, the trade-off is time, as a higher value will take the process longer to sample the collection. +* When Running with Schema Enforced set to `false` there is no attempt to discover any schema. See more in [Schema Enforcement](#Schema-Enforcement). -### Сonfiguration Parameters +## Configuration Parameters -- Database: database name -- Authentication Source: specifies the database that the supplied credentials should be validated against. Defaults to `admin`. -- User: username to use when connecting -- Password: used to authenticate the user -- **Standalone MongoDb instance** - - Host: URL of the database - - Port: Port to use for connecting to the database - - TLS: indicates whether to create encrypted connection -- **Replica Set** - - Server addresses: the members of a replica set - - Replica Set: A replica set name -- **MongoDb Atlas Cluster** - - Cluster URL: URL of a cluster to connect to +| Parameter Name | Description | +|:-------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Cluster Type | The type of the MongoDB cluster ([MongoDB Atlas](https://www.mongodb.com/atlas/database) replica set or self-hosted replica set). | +| Connection String | The connection string of the source MongoDB cluster. For Atlas hosted clusters, see [the quick start guide](#step-2-discover-the-mongodb-cluster-connection-string) for steps to find the connection string. For self-hosted clusters, refer to the [MongoDB connection string documentation](https://www.mongodb.com/docs/manual/reference/connection-string/#find-your-self-hosted-deployment-s-connection-string) for more information. | +| Database Name | The name of the database that contains the source collection(s) to sync. | +| Username | The username which is used to access the database. Required for MongoDB Atlas clusters. | +| Password | The password associated with this username. Required for MongoDB Atlas clusters. | +| Authentication Source | (MongoDB Atlas clusters only) Specifies the database that the supplied credentials should be validated against. Defaults to `admin`. See the [MongoDB documentation](https://www.mongodb.com/docs/manual/reference/connection-string/#mongodb-urioption-urioption.authSource) for more details. | +| Schema Enforced | Controls whether schema is discovered and enforced. See discussion in [Schema Enforcement](#Schema-Enforcement). | +| Initial Waiting Time in Seconds (Advanced) | The amount of time the connector will wait when it launches to determine if there is new data to sync or not. Defaults to 300 seconds. Valid range: 120 seconds to 1200 seconds. | +| Size of the queue (Advanced) | The size of the internal queue. This may interfere with memory consumption and efficiency of the connector, please be careful. | +| Discovery Sample Size (Advanced) | The maximum number of documents to sample when attempting to discover the unique fields for a collection. Default is 10,000 with a valid range of 1,000 to 100,000. See the [MongoDB sampling method](https://www.mongodb.com/docs/compass/current/sampling/#sampling-method) for more details. | -For more information regarding configuration parameters, please see [MongoDb Documentation](https://docs.mongodb.com/drivers/java/sync/v4.3/fundamentals/connection/). +For more information regarding configuration parameters, please see [MongoDb Documentation](https://docs.mongodb.com/drivers/java/sync/v4.10/fundamentals/connection/). ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- |:---------------------------------------------------------| :-------------------------------------------------------------------------------------------------------- | +|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------| +| 1.2.3 | 2024-01-18 | [34364](https://github.com/airbytehq/airbyte/pull/34364) | Add additional logging for resume token + reduce discovery size to 10. | +| 1.2.2 | 2024-01-16 | [34314](https://github.com/airbytehq/airbyte/pull/34314) | Reduce minimum document discovery size to 100. | +| 1.2.1 | 2023-12-18 | [33549](https://github.com/airbytehq/airbyte/pull/33549) | Add logging to understand op log size. | +| 1.2.0 | 2023-12-18 | [33438](https://github.com/airbytehq/airbyte/pull/33438) | Remove LEGACY state flag | +| 1.1.0 | 2023-12-14 | [32328](https://github.com/airbytehq/airbyte/pull/32328) | Schema less mode in mongodb. | +| 1.0.12 | 2023-12-13 | [33430](https://github.com/airbytehq/airbyte/pull/33430) | Add more verbose logging. | +| 1.0.11 | 2023-11-28 | [33356](https://github.com/airbytehq/airbyte/pull/33356) | Support for better debugging tools. | +| 1.0.10 | 2023-11-28 | [32886](https://github.com/airbytehq/airbyte/pull/32886) | Handle discover phase OOMs | +| 1.0.9 | 2023-11-08 | [32285](https://github.com/airbytehq/airbyte/pull/32285) | Additional support to read UUIDs | +| 1.0.8 | 2023-11-08 | [32125](https://github.com/airbytehq/airbyte/pull/32125) | Fix compilation warnings | +| 1.0.7 | 2023-11-07 | [32250](https://github.com/airbytehq/airbyte/pull/32250) | Add support to read UUIDs. | +| 1.0.6 | 2023-11-06 | [32193](https://github.com/airbytehq/airbyte/pull/32193) | Adopt java CDK version 0.4.1. | +| 1.0.5 | 2023-10-31 | [32028](https://github.com/airbytehq/airbyte/pull/32028) | url encode username and password.
      Handle a case of document update and delete in a single sync. | +| 1.0.3 | 2023-10-19 | [31629](https://github.com/airbytehq/airbyte/pull/31629) | Allow discover operation use of disk file when an operation goes over max allowed mem | +| 1.0.2 | 2023-10-19 | [31596](https://github.com/airbytehq/airbyte/pull/31596) | Allow use of temp disk file when an operation goes over max allowed mem | +| 1.0.1 | 2023-10-03 | [31034](https://github.com/airbytehq/airbyte/pull/31034) | Fix field filtering logic related to nested documents | +| 1.0.0 | 2023-10-03 | [29969](https://github.com/airbytehq/airbyte/pull/29969) | General availability release using Change Data Capture (CDC) | | 0.2.5 | 2023-07-27 | [28815](https://github.com/airbytehq/airbyte/pull/28815) | Revert back to version 0.2.0 | | 0.2.4 | 2023-07-26 | [28760](https://github.com/airbytehq/airbyte/pull/28760) | Fix bug preventing some syncs from succeeding when collecting stats | | 0.2.3 | 2023-07-26 | [28733](https://github.com/airbytehq/airbyte/pull/28733) | Fix bug preventing syncs from discovering field types | diff --git a/docs/integrations/sources/mssql-migrations.md b/docs/integrations/sources/mssql-migrations.md index c2271160bab3..d637f94fe073 100644 --- a/docs/integrations/sources/mssql-migrations.md +++ b/docs/integrations/sources/mssql-migrations.md @@ -1,4 +1,22 @@ # Microsoft SQL Server (MSSQL) Migration Guide +## Upgrading to 3.0.0 +This change remapped date, datetime, datetime2, datetimeoffset, smalldatetime, and time data type to their correct Airbyte types. Customers whose streams have columns with the affected datatype must refresh their schema and reset their data. See chart below for the mapping change. + +| Mssql type | Current Airbyte Type | New Airbyte Type | +|----------------|----------------------|-------------------| +| date | string | date | +| datetime | string | timestamp | +| datetime2 | string | timestamp | +| datetimeoffset | string | timestamp with tz | +| smalldatetime | string | timestamp | +| time | string | time | + +For current source-mssql users: +- If your streams do not contain any column of an affected data type, your connection will be unaffected. No further action is required from you. +- If your streams contain at least one column of an affected data type, you can opt in, refresh your schema, but *do not* reset your stream data. Once the sync starts, the Airbyte platform will trigger a schema change that will propagate to the destination tables. *Note:* In the case that your sync fails, please reset your data and rerun the sync. This will drop, recreate all the necessary tables, and reread the source data from the beginning. + +If resetting your stream data is an issue, please reach out to Airbyte Cloud support for assistance. + ## Upgrading to 2.0.0 -CDC syncs now has default cursor field called `_ab_cdc_cursor`. You will need to force normalization to rebuild your destination tables by manually dropping the SCD tables, refreshing the connection schema (skipping the reset), and running a sync. Alternatively, you can just run a reset. \ No newline at end of file +CDC syncs now has default cursor field called `_ab_cdc_cursor`. You will need to force normalization to rebuild your destination tables by manually dropping the SCD tables, refreshing the connection schema (skipping the reset), and running a sync. Alternatively, you can just run a reset. diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md index 28d3fe16e5c0..1ff83f409190 100644 --- a/docs/integrations/sources/mssql.md +++ b/docs/integrations/sources/mssql.md @@ -25,7 +25,7 @@ Note: Currently hierarchyid and sql_variant are not processed in CDC migration t On Airbyte Cloud, only TLS connections to your MSSQL instance are supported in source configuration. Other than that, you can proceed with the open-source instructions below. -## Getting Started \(Airbyte Open-Source\) +## Getting Started \(Airbyte Open Source\) #### Requirements @@ -265,39 +265,39 @@ This produces the private key in pem format, and the public key remains in the s MSSQL data types are mapped to the following data types when synchronizing data. You can check the test values examples [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/MssqlSourceComprehensiveTest.java). If you can't find the data type you are looking for or have any problems feel free to add a new test! -| MSSQL Type | Resulting Type | Notes | -| :------------------------------------------------------ | :------------- | :---- | -| `bigint` | number | | -| `binary` | string | | -| `bit` | boolean | | -| `char` | string | | -| `date` | number | | -| `datetime` | string | | -| `datetime2` | string | | -| `datetimeoffset` | string | | -| `decimal` | number | | -| `int` | number | | -| `float` | number | | -| `geography` | string | | -| `geometry` | string | | -| `money` | number | | -| `numeric` | number | | -| `ntext` | string | | -| `nvarchar` | string | | -| `nvarchar(max)` | string | | -| `real` | number | | -| `smalldatetime` | string | | -| `smallint` | number | | -| `smallmoney` | number | | -| `sql_variant` | string | | -| `uniqueidentifier` | string | | -| `text` | string | | -| `time` | string | | -| `tinyint` | number | | -| `varbinary` | string | | -| `varchar` | string | | -| `varchar(max) COLLATE Latin1_General_100_CI_AI_SC_UTF8` | string | | -| `xml` | string | | +| MSSQL Type | Resulting Type | Notes | +| :------------------------------------------------------ |:------------------------| :---- | +| `bigint` | number | | +| `binary` | string | | +| `bit` | boolean | | +| `char` | string | | +| `date` | date | | +| `datetime` | timestamp | | +| `datetime2` | timestamp | | +| `datetimeoffset` | timestamp with timezone | | +| `decimal` | number | | +| `int` | number | | +| `float` | number | | +| `geography` | string | | +| `geometry` | string | | +| `money` | number | | +| `numeric` | number | | +| `ntext` | string | | +| `nvarchar` | string | | +| `nvarchar(max)` | string | | +| `real` | number | | +| `smalldatetime` | timestamp | | +| `smallint` | number | | +| `smallmoney` | number | | +| `sql_variant` | string | | +| `uniqueidentifier` | string | | +| `text` | string | | +| `time` | time | | +| `tinyint` | number | | +| `varbinary` | string | | +| `varchar` | string | | +| `varchar(max) COLLATE Latin1_General_100_CI_AI_SC_UTF8` | string | | +| `xml` | string | | If you do not see a type in this list, assume that it is coerced into a string. We are happy to take feedback on preferred mappings. @@ -341,7 +341,25 @@ WHERE actor_definition_id ='b5ea17b1-f170-46dc-bc31-cc744ca984c1' AND (configura ## Changelog | Version | Date | Pull Request | Subject | -|:--------|:-----------|:------------------------------------------------------------------------------------------------------------------| :---------------------------------------------------------------------------------------------------------------------------------------------- | +|:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.6.0 | 2024-01-10 | [33700](https://github.com/airbytehq/airbyte/pull/33700) | Remove CDC config options for data_to_sync and snapshot isolation. | +| 3.5.1 | 2024-01-05 | [33510](https://github.com/airbytehq/airbyte/pull/33510) | Test-only changes. | +| 3.5.0 | 2023-12-19 | [33071](https://github.com/airbytehq/airbyte/pull/33071) | Fix SSL configuration parameters | +| 3.4.1 | 2024-01-02 | [33755](https://github.com/airbytehq/airbyte/pull/33755) | Encode binary to base64 format | +| 3.4.0 | 2023-12-19 | [33481](https://github.com/airbytehq/airbyte/pull/33481) | Remove LEGACY state flag | +| 3.3.2 | 2023-12-14 | [33505](https://github.com/airbytehq/airbyte/pull/33505) | Using the released CDK. | +| 3.3.1 | 2023-12-12 | [33225](https://github.com/airbytehq/airbyte/pull/33225) | extracting MsSql specific files out of the CDK. | +| 3.3.0 | 2023-12-12 | [33018](https://github.com/airbytehq/airbyte/pull/33018) | Migrate to Per-stream/Global states and away from Legacy states | +| 3.2.1 | 2023-12-11 | [33330](https://github.com/airbytehq/airbyte/pull/33330) | Parse DatetimeOffset fields with the correct format when used as cursor | +| 3.2.0 | 2023-12-07 | [33225](https://github.com/airbytehq/airbyte/pull/33225) | CDC : Enable compression of schema history blob in state. | +| 3.1.0 | 2023-11-28 | [32882](https://github.com/airbytehq/airbyte/pull/32882) | Enforce SSL on Airbyte Cloud. | +| 3.0.2 | 2023-11-27 | [32573](https://github.com/airbytehq/airbyte/pull/32573) | Format Datetime and Datetime2 datatypes to 6-digit microsecond precision | +| 3.0.1 | 2023-11-22 | [32656](https://github.com/airbytehq/airbyte/pull/32656) | Adopt java CDK version 0.5.0. | +| 3.0.0 | 2023-11-07 | [31531](https://github.com/airbytehq/airbyte/pull/31531) | Remapped date, smalldatetime, datetime2, time, and datetimeoffset datatype to their correct Airbyte types | +| 2.0.4 | 2023-11-06 | [#32193](https://github.com/airbytehq/airbyte/pull/32193) | Adopt java CDK version 0.4.1. | +| 2.0.3 | 2023-10-31 | [32024](https://github.com/airbytehq/airbyte/pull/32024) | Upgrade to Debezium version 2.4.0. | +| 2.0.2 | 2023-10-30 | [31960](https://github.com/airbytehq/airbyte/pull/31960) | Adopt java CDK version 0.2.0. | +| 2.0.1 | 2023-08-24 | [29821](https://github.com/airbytehq/airbyte/pull/29821) | Set replication_method display_type to radio, update titles and descriptions, and make CDC the default choice | | 2.0.0 | 2023-08-22 | [29493](https://github.com/airbytehq/airbyte/pull/29493) | Set a default cursor for Cdc mode | | 1.1.1 | 2023-07-24 | [28545](https://github.com/airbytehq/airbyte/pull/28545) | Support Read Committed snapshot isolation level | | 1.1.0 | 2023-06-26 | [27737](https://github.com/airbytehq/airbyte/pull/27737) | License Update: Elv2 | diff --git a/docs/integrations/sources/my-hours.md b/docs/integrations/sources/my-hours.md index 66ae44d7bc2d..7bcb78e0c179 100644 --- a/docs/integrations/sources/my-hours.md +++ b/docs/integrations/sources/my-hours.md @@ -24,14 +24,15 @@ This source allows you to synchronize the following data tables: **Requirements** In order to use the My Hours API you need to provide the credentials to an admin My Hours account. -### Performance Considerations (Airbyte Open-Source) +### Performance Considerations (Airbyte Open Source) Depending on the amount of team members and time logs the source provides a property to change the pagination size for the time logs query. Typically a pagination of 30 days is a correct balance between reliability and speed. But if you have a big amount of monthly entries you might want to change this value to a lower value. ## CHANGELOG -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :----------------------------------------------------- | :------ | -| 0.1.1 | 2022-06-08 | [12964](https://github.com/airbytehq/airbyte/pull/12964) | Update schema for time_logs stream | -| 0.1.0 | 2021-11-26 | [8270](https://github.com/airbytehq/airbyte/pull/8270) | New Source: My Hours | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :----------------------------------- | +| 0.1.2 | 2023-11-20 | [32680](https://github.com/airbytehq/airbyte/pull/32680) | Schema and CDK updates | +| 0.1.1 | 2022-06-08 | [12964](https://github.com/airbytehq/airbyte/pull/12964) | Update schema for time_logs stream | +| 0.1.0 | 2021-11-26 | [8270](https://github.com/airbytehq/airbyte/pull/8270) | New Source: My Hours | diff --git a/docs/integrations/sources/mysql.md b/docs/integrations/sources/mysql.md index a388ce447596..37dbc1bfe797 100644 --- a/docs/integrations/sources/mysql.md +++ b/docs/integrations/sources/mysql.md @@ -1,174 +1,159 @@ # MySQL -## Features +Airbyte's certified MySQL connector offers the following features: +* Multiple methods of keeping your data fresh, including [Change Data Capture (CDC)](https://docs.airbyte.com/understanding-airbyte/cdc) using the [binlog](https://dev.mysql.com/doc/refman/8.0/en/binary-log.html). +* All available [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes), providing flexibility in how data is delivered to your destination. +* Reliable replication at any table size with [checkpointing](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#state--checkpointing) and chunking of database reads. -| Feature | Supported | Notes | -| :---------------------------- | :-------- | :-------------------------------- | -| Full Refresh Sync | Yes | | -| Incremental - Append Sync | Yes | | -| Replicate Incremental Deletes | Yes | | -| CDC | Yes | | -| SSL Support | Yes | | -| SSH Tunnel Connection | Yes | | -| Namespaces | Yes | Enabled by default | -| Arrays | Yes | Byte arrays are not supported yet | +The contents below include a 'Quick Start' guide, advanced setup steps, and reference information (data type mapping and changelogs). -The MySQL source does not alter the schema present in your database. Depending on the destination connected to this source, however, the schema may be altered. See the destination's documentation for more details. +![Airbyte MySQL Connection](https://raw.githubusercontent.com/airbytehq/airbyte/3a9264666b7b9b9d10ef8d174b8454a6c7e57560/docs/integrations/sources/mysql/assets/airbyte_mysql_source.png) -## Troubleshooting +## Quick Start -There may be problems with mapping values in MySQL's datetime field to other relational data stores. MySQL permits zero values for date/time instead of NULL which may not be accepted by other data stores. To work around this problem, you can pass the following key value pair in the JDBC connector of the source setting `zerodatetimebehavior=Converttonull`. +Here is an outline of the minimum required steps to configure a MySQL connector: +1. Create a dedicated read-only MySQL user with permissions for replicating data +2. Create a new MySQL source in the Airbyte UI using CDC logical replication +3. (Airbyte Cloud Only) Allow inbound traffic from Airbyte IPs -Some users reported that they could not connect to Amazon RDS MySQL or MariaDB. This can be diagnosed with the error message: `Cannot create a PoolableConnectionFactory`. -To solve this issue add `enabledTLSProtocols=TLSv1.2` in the JDBC parameters. +Once this is complete, you will be able to select MySQL as a source for replicating data. -Another error that users have reported when trying to connect to Amazon RDS MySQL is `Error: HikariPool-1 - Connection is not available, request timed out after 30001ms.`. Many times this is can be due to the VPC not allowing public traffic, however, we recommend going through [this AWS troubleshooting checklist](https://aws.amazon.com/premiumsupport/knowledge-center/rds-cannot-connect/) to the correct permissions/settings have been granted to allow connection to your database. + -## Getting Started \(Airbyte Cloud\) +#### Step 1: Create a dedicated read-only MySQL user -On Airbyte Cloud, only TLS connections to your MySQL instance are supported. Other than that, you can proceed with the open-source instructions below. +These steps create a dedicated read-only user for replicating data. Alternatively, you can use an existing MySQL user in your database. -## Getting Started \(Airbyte Open-Source\) +The following commands will create a new user: -#### Requirements - -1. MySQL Server `8.0`, `5.7`, or `5.6`. -2. Create a dedicated read-only Airbyte user with access to all tables needed for replication. - -**1. Make sure your database is accessible from the machine running Airbyte** - -This is dependent on your networking setup. The easiest way to verify if Airbyte is able to connect to your MySQL instance is via the check connection tool in the UI. - -**2. Create a dedicated read-only user with access to the relevant tables \(Recommended but optional\)** - -This step is optional but highly recommended to allow for better permission control and auditing. Alternatively, you can use Airbyte with an existing user in your database. - -To create a dedicated database user, run the following commands against your database: - -```sql -CREATE USER 'airbyte'@'%' IDENTIFIED BY 'your_password_here'; -``` - -The right set of permissions differ between the `STANDARD` and `CDC` replication method. For `STANDARD` replication method, only `SELECT` permission is required. - -```sql -GRANT SELECT ON .* TO 'airbyte'@'%'; +```roomsql +CREATE USER IDENTIFIED BY 'your_password_here'; ``` -For `CDC` replication method, `SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT` permissions are required. +Now, provide this user with read-only access to relevant schemas and tables: -```sql -GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'airbyte'@'%'; +```roomsql +GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO ; ``` -Your database user should now be ready for use with Airbyte. - -**3. Set up CDC** - -For `STANDARD` replication method this is not applicable. If you select the `CDC` replication method then only this is required. Please read the section on [CDC below](#change-data-capture-cdc) for more information. +If choosing to run using the `STANDARD` replication method (not recommended), only the `SELECT` permission is required. -**4. That's it!** + -Your database user should now be ready for use with Airbyte. + -## Change Data Capture \(CDC\) +#### Step 2: Enable binary logging on your MySQL server -- If you need a record of deletions and can accept the limitations posted below, you should be able to use CDC for MySQL. -- If your data set is small, and you just want snapshot of your table in the destination, consider using Full Refresh replication for your table instead of CDC. -- If the limitations prevent you from using CDC and your goal is to maintain a snapshot of your table in the destination, consider using non-CDC incremental and occasionally reset the data and re-sync. -- If your table has a primary key but doesn't have a reasonable cursor field for incremental syncing \(i.e. `updated_at`\), CDC allows you to sync your table incrementally. +You must enable binary logging for MySQL replication using CDC. Most cloud providers (AWS, GCP, etc.) provide easy one-click options for enabling the binlog on your source MySQL database. -#### CDC Limitations +If you are self-managing your MySQL server, configure your MySQL server configuration file with the following properties: -- Make sure to read our [CDC docs](../../understanding-airbyte/cdc.md) to see limitations that impact all databases using CDC replication. -- Our CDC implementation uses at least once delivery for all change records. - -**1. Enable binary logging** - -You must enable binary logging for MySQL replication. The binary logs record transaction updates for replication tools to propagate changes. You can configure your MySQL server configuration file with the following properties, which are described in below: +
      + Configuring MySQL server config files to enable binlog ```text server-id = 223344 log_bin = mysql-bin binlog_format = ROW binlog_row_image = FULL -binlog_expire_log_seconds = 864000 +binlog_expire_logs_seconds = 864000 ``` - server-id : The value for the server-id must be unique for each server and replication client in the MySQL cluster. The `server-id` should be a non-zero value. If the `server-id` is already set to a non-zero value, you don't need to make any change. You can set the `server-id` to any value between 1 and 4294967295. For more information refer [mysql doc](https://dev.mysql.com/doc/refman/8.0/en/replication-options.html#sysvar_server_id) - log_bin : The value of log_bin is the base name of the sequence of binlog files. If the `log_bin` is already set, you don't need to make any change. For more information refer [mysql doc](https://dev.mysql.com/doc/refman/8.0/en/replication-options-binary-log.html#option_mysqld_log-bin) - binlog_format : The `binlog_format` must be set to `ROW`. For more information refer [mysql doc](https://dev.mysql.com/doc/refman/8.0/en/replication-options-binary-log.html#sysvar_binlog_format) - binlog_row_image : The `binlog_row_image` must be set to `FULL`. It determines how row images are written to the binary log. For more information refer [mysql doc](https://dev.mysql.com/doc/refman/5.7/en/replication-options-binary-log.html#sysvar_binlog_row_image) -- binlog_expire_log_seconds : This is the number of seconds for automatic binlog file removal. We recommend 864000 seconds (10 days) so that in case of a failure in sync or if the sync is paused, we still have some bandwidth to start from the last point in incremental sync. We also recommend setting frequent syncs for CDC. +- binlog_expire_logs_seconds : This is the number of seconds for automatic binlog file removal. We recommend 864000 seconds (10 days) so that in case of a failure in sync or if the sync is paused, we still have some bandwidth to start from the last point in incremental sync. We also recommend setting frequent syncs for CDC. -**2. Enable GTIDs \(Optional\)** +
      -Global transaction identifiers \(GTIDs\) uniquely identify transactions that occur on a server within a cluster. Though not required for a Airbyte MySQL connector, using GTIDs simplifies replication and enables you to more easily confirm if primary and replica servers are consistent. For more information refer [mysql doc](https://dev.mysql.com/doc/refman/8.0/en/replication-options-gtids.html#option_mysqld_gtid-mode) +
      -- Enable gtid_mode : Boolean that specifies whether GTID mode of the MySQL server is enabled or not. Enable it via `mysql> gtid_mode=ON` -- Enable enforce_gtid_consistency : Boolean that specifies whether the server enforces GTID consistency by allowing the execution of statements that can be logged in a transactionally safe manner. Required when using GTIDs. Enable it via `mysql> enforce_gtid_consistency=ON` + -**3. Set up initial waiting time\(Optional\)** +#### Step 3: Create a new MySQL source in Airbyte UI -:::warning -This is an advanced feature. Use it if absolutely necessary. -::: +From your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account, select `Sources` from the left navigation bar, search for `MySQL`, then create a new MySQL source. -The MySQl connector may need some time to start processing the data in the CDC mode in the following scenarios: + -- When the connection is set up for the first time and a snapshot is needed -- When the connector has a lot of change logs to process +![Create an Airbyte source](https://github.com/airbytehq/airbyte/blob/c078e8ed6703020a584d9362efa5665fbe8db77f/docs/integrations/sources/postgres/assets/airbyte_source_selection.png?raw=true) -The connector waits for the default initial wait time of 5 minutes (300 seconds). Setting the parameter to a longer duration will result in slower syncs, while setting it to a shorter duration may cause the connector to not have enough time to create the initial snapshot or read through the change logs. The valid range is 300 seconds to 1200 seconds. + -If you know there are database changes to be synced, but the connector cannot read those changes, the root cause may be insufficient waiting time. In that case, you can increase the waiting time (example: set to 600 seconds) to test if it is indeed the root cause. On the other hand, if you know there are no database changes, you can decrease the wait time to speed up the zero record syncs. +To fill out the required information: +1. Enter the hostname, port number, and name for your MySQL database. +2. Enter the username and password you created in [Step 1](#step-1-create-a-dedicated-read-only-mysql-user). +3. Select an SSL mode. You will most frequently choose `require` or `verify-ca`. Both of these always require encryption. `verify-ca` also requires certificates from your MySQL database. See [here](#ssl-modes) to learn about other SSL modes and SSH tunneling. +4. Select `Read Changes using Binary Log (CDC)` from available replication methods. -**4. Set up server timezone\(Optional\)** + +#### Step 4: (Airbyte Cloud Only) Allow inbound traffic from Airbyte IPs. -:::warning -This is an advanced feature. Use it if absolutely necessary. -::: +If you are on Airbyte Cloud, you will always need to modify your database configuration to allow inbound traffic from Airbyte IPs. You can find a list of all IPs that need to be allowlisted in +our [Airbyte Security docs](../../operating-airbyte/security#network-security-1). -In CDC mode, the MySQl connector may need a timezone configured if the existing MySQL database been set up with a system timezone that is not recognized by the [IANA Timezone Database](https://www.iana.org/time-zones). +Now, click `Set up source` in the Airbyte UI. Airbyte will now test connecting to your database. Once this succeeds, you've configured an Airbyte MySQL source! + -In this case, you can configure the server timezone to the equivalent IANA timezone compliant timezone. (e.g. CEST -> Europe/Berlin). + -**Note** +## MySQL Replication Modes -When a sync runs for the first time using CDC, Airbyte performs an initial consistent snapshot of your database. Airbyte doesn't acquire any table locks \(for tables defined with MyISAM engine, the tables would still be locked\) while creating the snapshot to allow writes by other database clients. But in order for the sync to work without any error/unexpected behaviour, it is assumed that no schema changes are happening while the snapshot is running. +### Change Data Capture \(CDC\) -If seeing `EventDataDeserializationException` errors intermittently with root cause `EOFException` or `SocketException`, you may need to extend the following _MySql server_ timeout values by running: +Airbyte uses logical replication of the [MySQL binlog](https://dev.mysql.com/doc/refman/8.0/en/binary-log.html) to incrementally capture deletes. To learn more how Airbyte implements CDC, refer to [Change Data Capture (CDC)](https://docs.airbyte.com/understanding-airbyte/cdc/). We generally recommend configure your MySQL source with CDC whenever possible, as it provides: +- A record of deletions, if needed. +- Scalable replication to large tables (1 TB and more). +- A reliable cursor not reliant on the nature of your data. For example, if your table has a primary key but doesn't have a reasonable cursor field for incremental syncing \(i.e. `updated_at`\), CDC allows you to sync your table incrementally. -``` -set global slave_net_timeout = 120; -set global thread_pool_idle_timeout = 120; -``` + + +### Standard + +Airbyte offers incremental replication using a custom cursor available in your source tables (e.g. `updated_at`). We generally recommend against this replication method, but it is well suited for the following cases: +- Your MySQL server does not expose the binlog. +- Your data set is small, and you just want snapshot of your table in the destination. -## Connection via SSH Tunnel + -Airbyte has the ability to connect to a MySQl instance via an SSH Tunnel. The reason you might want to do this because it is not possible \(or against security policy\) to connect to the database directly \(e.g. it does not have a public IP address\). +## Connecting with SSL or SSH Tunneling -When using an SSH tunnel, you are configuring Airbyte to connect to an intermediate server \(a.k.a. a bastion sever\) that _does_ have direct access to the database. Airbyte connects to the bastion and then asks the bastion to connect directly to the server. + -Using this feature requires additional configuration, when creating the source. We will talk through what each piece of configuration means. +### SSL Modes -1. Configure all fields for the source as you normally would, except `SSH Tunnel Method`. -2. `SSH Tunnel Method` defaults to `No Tunnel` \(meaning a direct connection\). If you want to use an SSH Tunnel choose `SSH Key Authentication` or `Password Authentication`. +Airbyte Cloud uses SSL by default. You are not permitted to `disable` SSL while using Airbyte Cloud. - 1. Choose `Key Authentication` if you will be using an RSA private key as your secret for establishing the SSH Tunnel \(see below for more information on generating this key\). - 2. Choose `Password Authentication` if you will be using a password as your secret for establishing the SSH Tunnel. +Here is a breakdown of available SSL connection modes: +- `disable` to disable encrypted communication between Airbyte and the source +- `allow` to enable encrypted communication only when required by the source +- `prefer` to allow unencrypted communication only when the source doesn't support encryption +- `require` to always require encryption. Note: The connection will fail if the source doesn't support encryption. +- `verify-ca` to always require encryption and verify that the source has a valid SSL certificate +- `verify-full` to always require encryption and verify the identity of the source - :::warning - Since Airbyte Cloud requires encrypted communication, select **SSH Key Authentication** or **Password Authentication** if you selected **preferred** as the **SSL Mode**; otherwise, the connection will fail. - ::: + -3. `SSH Tunnel Jump Server Host` refers to the intermediate \(bastion\) server that Airbyte will connect to. This should be a hostname or an IP Address. -4. `SSH Connection Port` is the port on the bastion server with which to make the SSH connection. The default port for SSH connections is `22`, so unless you have explicitly changed something, go with the default. -5. `SSH Login Username` is the username that Airbyte should use when connection to the bastion server. This is NOT the MySQl username. -6. If you are using `Password Authentication`, then `SSH Login Username` should be set to the password of the User from the previous step. If you are using `SSH Key Authentication` leave this blank. Again, this is not the MySQl password, but the password for the OS-user that Airbyte is using to perform commands on the bastion. -7. If you are using `SSH Key Authentication`, then `SSH Private Key` should be set to the RSA Private Key that you are using to create the SSH connection. This should be the full contents of the key file starting with `-----BEGIN RSA PRIVATE KEY-----` and ending with `-----END RSA PRIVATE KEY-----`. +### Connection via SSH Tunnel -### Generating an SSH Key Pair +You can connect to a MySQL server via an SSH tunnel. + +When using an SSH tunnel, you are configuring Airbyte to connect to an intermediate server (also called a bastion or a jump server) that has direct access to the database. Airbyte connects to the bastion and then asks the bastion to connect directly to the server. + +To connect to a MySQL server via an SSH tunnel: + +1. While setting up the MySQL source connector, from the SSH tunnel dropdown, select: + - SSH Key Authentication to use a private as your secret for establishing the SSH tunnel + - Password Authentication to use a password as your secret for establishing the SSH Tunnel +2. For **SSH Tunnel Jump Server Host**, enter the hostname or IP address for the intermediate (bastion) server that Airbyte will connect to. +3. For **SSH Connection Port**, enter the port on the bastion server. The default port for SSH connections is 22. +4. For **SSH Login Username**, enter the username to use when connecting to the bastion server. **Note:** This is the operating system username and not the MySQL username. +5. For authentication: + - If you selected **SSH Key Authentication**, set the **SSH Private Key** to the [private Key](#generating-a-private-key-for-ssh-tunneling) that you are using to create the SSH connection. + - If you selected **Password Authentication**, enter the password for the operating system user to connect to the bastion server. **Note:** This is the operating system password and not the MySQL password. + +#### Generating a private key for SSH Tunneling The connector expects an RSA key in PEM format. To generate this key: @@ -178,11 +163,20 @@ ssh-keygen -t rsa -m PEM -f myuser_rsa This produces the private key in pem format, and the public key remains in the standard format used by the `authorized_keys` file on your bastion host. The public key should be added to your bastion host to whichever user you want to use with Airbyte. The private key is provided via copy-and-paste to the Airbyte connector configuration screen, so it may log in to the bastion. +## Limitations & Troubleshooting + +To see connector limitations, or troubleshoot your MySQL connector, see more [in our MySQL troubleshooting guide](https://docs.airbyte.com/integrations/sources/mysql/mysql-troubleshooting). + ## Data Type Mapping -MySQL data types are mapped to the following data types when synchronizing data. You can check the test values examples [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceDatatypeTest.java). If you can't find the data type you are looking for or have any problems feel free to add a new test! +MySQL data types are mapped to the following data types when synchronizing data. You can check test example values [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/MySqlSourceDatatypeTest.java). If you can't find the data type you are looking for, feel free to add a new test. +If you do not see a type in this list, assume that it is coerced into a string. We are happy to take feedback on preferred mappings. + +Any database or table encoding combination of charset and collation is supported. Charset setting however will not be carried over to destination and data will be encoded with whatever is configured by the destination. Please note that byte arrays are not yet supported. + +
      + MySQL Data Type Mapping -Any database or table encoding combination of charset and collation is supported. Charset setting however will not be carried over to destination and data will be encoded with whatever is configured by the destination. | MySQL Type | Resulting Type | Notes | | :---------------------------------------- | :--------------------- | :------------------------------------------------------------------------------------------------------------- | @@ -221,49 +215,40 @@ Any database or table encoding combination of charset and collation is supported | `set` | string | E.g. `blue,green,yellow` | | `geometry` | base64 binary string | | -If you do not see a type in this list, assume that it is coerced into a string. We are happy to take feedback on preferred mappings. - -## Upgrading from 0.6.8 and older versions to 0.6.9 and later versions - -There is a backwards incompatible spec change between MySQL Source connector versions 0.6.8 and 0.6.9. As part of that spec change -`replication_method` configuration parameter was changed to `object` from `string`. - -In MySQL source connector versions 0.6.8 and older, `replication_method` configuration parameter was saved in the configuration database as follows: - -``` -"replication_method": "STANDARD" -``` - -Starting with version 0.6.9, `replication_method` configuration parameter is saved as follows: - -``` -"replication_method": { - "method": "STANDARD" -} -``` - -After upgrading MySQL Source connector from 0.6.8 or older version to 0.6.9 or newer version you need to fix source configurations in the `actor` table -in Airbyte database. To do so, you need to run the following two SQL queries. Follow the instructions in [Airbyte documentation](https://docs.airbyte.com/operator-guides/configuring-airbyte-db/#accessing-the-default-database-located-in-docker-airbyte-db) to -run SQL queries on Airbyte database. -If you have connections with MySQL Source using _Standard_ replication method, run this SQL: - -```sql -update public.actor set configuration =jsonb_set(configuration, '{replication_method}', '{"method": "STANDARD"}', true) -WHERE actor_definition_id ='435bb9a5-7887-4809-aa58-28c27df0d7ad' AND (configuration->>'replication_method' = 'STANDARD'); -``` - -If you have connections with MySQL Source using _Logical Replication (CDC)_ method, run this SQL: - -```sql -update public.actor set configuration =jsonb_set(configuration, '{replication_method}', '{"method": "CDC"}', true) -WHERE actor_definition_id ='435bb9a5-7887-4809-aa58-28c27df0d7ad' AND (configuration->>'replication_method' = 'CDC'); -``` +
      ## Changelog + | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.3.2 | 2024-01-08 | [33005](https://github.com/airbytehq/airbyte/pull/33005) | Adding count stats for incremental sync in AirbyteStateMessage +| 3.3.1 | 2024-01-03 | [33312](https://github.com/airbytehq/airbyte/pull/33312) | Adding count stats in AirbyteStateMessage | +| 3.3.0 | 2023-12-19 | [33436](https://github.com/airbytehq/airbyte/pull/33436) | Remove LEGACY state flag | +| 3.2.4 | 2023-12-12 | [33356](https://github.com/airbytehq/airbyte/pull/33210) | Support for better debugging tools.. | +| 3.2.3 | 2023-12-08 | [33210](https://github.com/airbytehq/airbyte/pull/33210) | Update MySql driver property value for zero date handling. | +| 3.2.2 | 2023-12-06 | [33082](https://github.com/airbytehq/airbyte/pull/33082) | Improvements to MySQL schema snapshot error handling. | +| 3.2.1 | 2023-11-28 | [32610](https://github.com/airbytehq/airbyte/pull/32610) | Support initial syncs using binary as primary key. | +| 3.2.0 | 2023-11-29 | [31062](https://github.com/airbytehq/airbyte/pull/31062) | enforce SSL on Airbyte Cloud | +| 3.1.9 | 2023-11-27 | [32662](https://github.com/airbytehq/airbyte/pull/32662) | Apply initial setup time to debezium engine warmup time. | +| 3.1.8 | 2023-11-22 | [32656](https://github.com/airbytehq/airbyte/pull/32656) | Adopt java CDK version 0.5.0. | +| 3.1.7 | 2023-11-08 | [32125](https://github.com/airbytehq/airbyte/pull/32125) | fix compilation warnings | +| 3.1.6 | 2023-11-06 | [32193](https://github.com/airbytehq/airbyte/pull/32193) | Adopt java CDK version 0.4.1. | +| 3.1.5 | 2023-10-31 | [32024](https://github.com/airbytehq/airbyte/pull/32024) | Upgrade to Debezium version 2.4.0. | +| 3.1.4 | 2023-10-30 | [31960](https://github.com/airbytehq/airbyte/pull/31960) | Adopt java CDK version 0.2.0. | +| 3.1.3 | 2023-10-11 | [31322](https://github.com/airbytehq/airbyte/pull/31322) | Correct pevious release | +| 3.1.2 | 2023-09-29 | [30806](https://github.com/airbytehq/airbyte/pull/30806) | Cap log line length to 32KB to prevent loss of records | +| 3.1.1 | 2023-09-26 | [30744](https://github.com/airbytehq/airbyte/pull/30744) | Update MySQL JDBC connection configs to keep default auto-commit behavior | +| 3.1.0 | 2023-09-21 | [30270](https://github.com/airbytehq/airbyte/pull/30270) | Enhanced Standard Sync with initial load via Primary Key with a switch to cursor for incremental syncs | +| 3.0.9 | 2023-09-20 | [30620](https://github.com/airbytehq/airbyte/pull/30620) | Airbyte Certified MySQL Source connector | +| 3.0.8 | 2023-09-14 | [30333](https://github.com/airbytehq/airbyte/pull/30333) | CDC : Update the correct timezone parameter passed to Debezium to `database.connectionTimezone` | +| 3.0.7 | 2023-09-13 | [30375](https://github.com/airbytehq/airbyte/pull/30375) | Fix a bug causing a failure when DB views are included in sync | +| 3.0.6 | 2023-09-12 | [30308](https://github.com/airbytehq/airbyte/pull/30308) | CDC : Enable compression of schema history blob in state | +| 3.0.5 | 2023-09-12 | [30289](https://github.com/airbytehq/airbyte/pull/30289) | CDC : Introduce logic for compression of schema history blob in state | +| 3.0.4 | 2023-09-06 | [30213](https://github.com/airbytehq/airbyte/pull/30213) | CDC : Checkpointable initial snapshot | +| 3.0.3 | 2023-08-31 | [29821](https://github.com/airbytehq/airbyte/pull/29821) | Set replication_method display_type to radio | +| 3.0.2 | 2023-08-30 | [30015](https://github.com/airbytehq/airbyte/pull/30015) | Logging : Log storage engines associated with tables in the sync | | 3.0.1 | 2023-08-21 | [29308](https://github.com/airbytehq/airbyte/pull/29308) | CDC: Enable frequent state emissions during incremental runs | | 3.0.0 | 2023-08-08 | [28756](https://github.com/airbytehq/airbyte/pull/28756) | CDC: Set a default cursor | | 2.1.2 | 2023-08-08 | [29220](https://github.com/airbytehq/airbyte/pull/29220) | Add indicator that CDC is the recommended update method | diff --git a/docs/integrations/sources/mysql/assets/airbyte_mysql_source.png b/docs/integrations/sources/mysql/assets/airbyte_mysql_source.png new file mode 100644 index 000000000000..4be3f41bfc9c Binary files /dev/null and b/docs/integrations/sources/mysql/assets/airbyte_mysql_source.png differ diff --git a/docs/integrations/sources/mysql/mysql-troubleshooting.md b/docs/integrations/sources/mysql/mysql-troubleshooting.md new file mode 100644 index 000000000000..aee512157839 --- /dev/null +++ b/docs/integrations/sources/mysql/mysql-troubleshooting.md @@ -0,0 +1,91 @@ +# Troubleshooting MySQL Sources + +### General Limitations + +- Use MySQL Server versions `8.0`, `5.7`, or `5.6`. +- For Airbyte Cloud (and optionally for Airbyte Open Source), ensure SSL is enabled in your environment + +### CDC Requirements + +- Make sure to read our [CDC docs](../../../understanding-airbyte/cdc.md) to see limitations that impact all databases using CDC replication. +- Our CDC implementation uses at least once delivery for all change records. + +## Troubleshooting + +### Common Config Errors + +* Mapping MySQL's DateTime field: There may be problems with mapping values in MySQL's datetime field to other relational data stores. MySQL permits zero values for date/time instead of NULL which may not be accepted by other data stores. To work around this problem, you can pass the following key value pair in the JDBC connector of the source setting `zerodatetimebehavior=Converttonull`. +* Amazon RDS MySQL or MariaDB connection issues: If you see the following `Cannot create a PoolableConnectionFactory` error, please add `enabledTLSProtocols=TLSv1.2` in the JDBC parameters. +* Amazon RDS MySQL connection issues: If you see `Error: HikariPool-1 - Connection is not available, request timed out after 30001ms.`, many times this due to your VPC not allowing public traffic. We recommend going through [this AWS troubleshooting checklist](https://aws.amazon.com/premiumsupport/knowledge-center/rds-cannot-connect/) to ensure the correct permissions/settings have been granted to allow Airbyte to connect to your database. + +### EventDataDeserializationException errors during initial snapshot + +When a sync runs for the first time using CDC, Airbyte performs an initial consistent snapshot of your database. Airbyte doesn't acquire any table locks \(for tables defined with MyISAM engine, the tables would still be locked\) while creating the snapshot to allow writes by other database clients. But in order for the sync to work without any error/unexpected behaviour, it is assumed that no schema changes are happening while the snapshot is running. + +If seeing `EventDataDeserializationException` errors intermittently with root cause `EOFException` or `SocketException`, you may need to extend the following _MySql server_ timeout values by running: + +``` +set global slave_net_timeout = 120; +set global thread_pool_idle_timeout = 120; +``` + +### (Advanced) Enable GTIDs + +Global transaction identifiers \(GTIDs\) uniquely identify transactions that occur on a server within a cluster. Though not required for a Airbyte MySQL connector, using GTIDs simplifies replication and enables you to more easily confirm if primary and replica servers are consistent. For more information refer [mysql doc](https://dev.mysql.com/doc/refman/8.0/en/replication-options-gtids.html#option_mysqld_gtid-mode) + +- Enable gtid_mode : Boolean that specifies whether GTID mode of the MySQL server is enabled or not. Enable it via `mysql> gtid_mode=ON` +- Enable enforce_gtid_consistency : Boolean that specifies whether the server enforces GTID consistency by allowing the execution of statements that can be logged in a transactionally safe manner. Required when using GTIDs. Enable it via `mysql> enforce_gtid_consistency=ON` + +### (Advanced) Setting up initial CDC waiting time + +The MySQl connector may need some time to start processing the data in the CDC mode in the following scenarios: + +- When the connection is set up for the first time and a snapshot is needed +- When the connector has a lot of change logs to process + +The connector waits for the default initial wait time of 5 minutes (300 seconds). Setting the parameter to a longer duration will result in slower syncs, while setting it to a shorter duration may cause the connector to not have enough time to create the initial snapshot or read through the change logs. The valid range is 300 seconds to 1200 seconds. + +If you know there are database changes to be synced, but the connector cannot read those changes, the root cause may be insufficient waiting time. In that case, you can increase the waiting time (example: set to 600 seconds) to test if it is indeed the root cause. On the other hand, if you know there are no database changes, you can decrease the wait time to speed up the zero record syncs. + +### (Advanced) Set up server timezone + +In CDC mode, the MySQl connector may need a timezone configured if the existing MySQL database been set up with a system timezone that is not recognized by the [IANA Timezone Database](https://www.iana.org/time-zones). + +In this case, you can configure the server timezone to the equivalent IANA timezone compliant timezone. (e.g. CEST -> Europe/Berlin). + +## Upgrading from 0.6.8 and older versions to 0.6.9 and later versions + +There is a backwards incompatible spec change between MySQL Source connector versions 0.6.8 and 0.6.9. As part of that spec change +`replication_method` configuration parameter was changed to `object` from `string`. + +In MySQL source connector versions 0.6.8 and older, `replication_method` configuration parameter was saved in the configuration database as follows: + +``` +"replication_method": "STANDARD" +``` + +Starting with version 0.6.9, `replication_method` configuration parameter is saved as follows: + +``` +"replication_method": { + "method": "STANDARD" +} +``` + +After upgrading MySQL Source connector from 0.6.8 or older version to 0.6.9 or newer version you need to fix source configurations in the `actor` table +in Airbyte database. To do so, you need to run the following two SQL queries. Follow the instructions in [Airbyte documentation](https://docs.airbyte.com/operator-guides/configuring-airbyte-db/#accessing-the-default-database-located-in-docker-airbyte-db) to +run SQL queries on Airbyte database. + +If you have connections with MySQL Source using _Standard_ replication method, run this SQL: + +```sql +update public.actor set configuration =jsonb_set(configuration, '{replication_method}', '{"method": "STANDARD"}', true) +WHERE actor_definition_id ='435bb9a5-7887-4809-aa58-28c27df0d7ad' AND (configuration->>'replication_method' = 'STANDARD'); +``` + +If you have connections with MySQL Source using _Logical Replication (CDC)_ method, run this SQL: + +```sql +update public.actor set configuration =jsonb_set(configuration, '{replication_method}', '{"method": "CDC"}', true) +WHERE actor_definition_id ='435bb9a5-7887-4809-aa58-28c27df0d7ad' AND (configuration->>'replication_method' = 'CDC'); +``` diff --git a/docs/integrations/sources/nasa.md b/docs/integrations/sources/nasa.md index a7b25dd6e932..5c3cde2a8866 100644 --- a/docs/integrations/sources/nasa.md +++ b/docs/integrations/sources/nasa.md @@ -39,5 +39,6 @@ The NASA connector should not run into NASA API limitations under normal usage. | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------| -| 0.1.1 | 2023-02-13 | [22934](https://github.com/airbytehq/airbyte/pull/22934) | Specified date formatting in specification | +| 0.2.0 | 2023-10-10 | [31051](https://github.com/airbytehq/airbyte/pull/31051) | Migrate to lowcode | +| 0.1.1 | 2023-02-13 | [22934](https://github.com/airbytehq/airbyte/pull/22934) | Specified date formatting in specification | | 0.1.0 | 2022-10-24 | [18394](https://github.com/airbytehq/airbyte/pull/18394) | 🎉 New Source: NASA APOD | diff --git a/docs/integrations/sources/notion-migrations.md b/docs/integrations/sources/notion-migrations.md new file mode 100644 index 000000000000..d08cd0e331e0 --- /dev/null +++ b/docs/integrations/sources/notion-migrations.md @@ -0,0 +1,11 @@ +# Notion Migration Guide + +## Upgrading to 2.0.0 + +Version 2.0.0 introduces a number of changes to the JSON schemas of all streams. These changes are being introduced to reflect updates to the Notion API. Some breaking changes have been introduced that will affect the Blocks, Databases and Pages stream. + +- The type of the `rich_text` property in the Pages stream has been updated from an object to an array of `rich_text` objects +- The type of the `phone_number` property in the Pages stream has been updated from a string to an object +- The deprecated `text` property in content blocks has been renamed to `rich_text`. This change affects the Blocks, Databases and Pages streams. + +A full schema refresh and data reset are required when upgrading to this version. diff --git a/docs/integrations/sources/notion.inapp.md b/docs/integrations/sources/notion.inapp.md deleted file mode 100644 index 019885f44abf..000000000000 --- a/docs/integrations/sources/notion.inapp.md +++ /dev/null @@ -1,33 +0,0 @@ -## Prerequisite - -* Access to a Notion workspace -​ -## Setup guide - -1. Enter a **Source name** to help you identify this source in Airbyte. -2. Choose the method of authentication: - - -:::note -We highly recommend using OAuth2.0 authorization to connect to Notion, as this method significantly simplifies the setup process. If you use OAuth2.0 authorization, you do _not_ need to create and configure a new integration in Notion. Instead, you can authenticate your Notion account directly in Airbyte Cloud. -::: - -- **OAuth2.0** (Recommended): Click **Authenticate your Notion account**. When the popup appears, click **Select pages**. Check the pages you want to give Airbyte access to, and click **Allow access**. -- **Access Token**: Copy and paste the Access Token found in the **Secrets** tab of your Notion integration's page. For more information on how to create and configure an integration in Notion, refer to our -[full documentation](https://docs.airbyte.io/integrations/sources/notion#setup-guide). - - - -- **Access Token**: Copy and paste the Access Token found in the **Secrets** tab of your Notion integration's page. -- **OAuth2.0**: Copy and paste the Client ID, Client Secret and Access Token you acquired. - -To obtain the necessary authorization credentials, you need to create and configure an integration in Notion. For more information on how to create and configure an integration in Notion, refer to our -[full documentation](https://docs.airbyte.io/integrations/sources/notion#setup-guide). - - -3. Enter the **Start Date** using the provided datepicker, or by programmatically entering a UTC date and time in the format: `YYYY-MM-DDTHH:mm:ss.SSSZ`. All data generated after this date will be replicated. -4. Click **Set up source** and wait for the tests to complete. -​ - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the -[full documentation for Notion](https://docs.airbyte.com/integrations/sources/notion). diff --git a/docs/integrations/sources/notion.md b/docs/integrations/sources/notion.md index b465bb9c4786..83343623f217 100644 --- a/docs/integrations/sources/notion.md +++ b/docs/integrations/sources/notion.md @@ -4,7 +4,7 @@ This page contains the setup guide and reference information for the Notion sour ## Prerequisites -- Access to a Notion workspace +- Access to a [Notion](https://notion.so/login) workspace ## Setup guide​ @@ -13,40 +13,43 @@ To authenticate the Notion source connector, you need to use **one** of the foll - OAuth2.0 authorization (recommended for Airbyte Cloud) - Access Token + :::note -**For Airbyte Cloud users:** We highly recommend using OAuth2.0 authorization to connect to Notion, as this method significantly simplifies the setup process. If you use OAuth2.0 authorization in Airbyte Cloud, you do **not** need to create and configure a new integration in Notion. Instead, you can proceed straight to -[setting up the connector in Airbyte](#step-3-set-up-the-notion-connector-in-airbyte). +**For Airbyte Cloud users:** We highly recommend using OAuth2.0 authorization to connect to Notion, as this method significantly simplifies the setup process. If you use OAuth2.0 authorization in Airbyte Cloud, you do **not** need to create and configure a new integration in Notion. Instead, you can proceed straight to [setting up the connector in Airbyte](#step-3-set-up-the-notion-connector-in-airbyte). ::: + -We have provided a quick setup guide for creating an integration in Notion below. If you would like more detailed information and context on Notion integrations, or experience any difficulties with the integration setup process, please refer to the -[official Notion documentation](https://developers.notion.com/docs). +We have provided a quick setup guide for creating an integration in Notion below. If you would like more detailed information and context on Notion integrations, or experience any difficulties with the integration setup process, please refer to the [official Notion documentation](https://developers.notion.com/docs). -### Step 1: Create an integration in Notion​ +### Step 1: Create an integration in Notion​ and set capabilities -1. Log in to your Notion workspace and navigate to the [My integrations](https://www.notion.so/my-integrations) page. Select **+ New integration**. +1. Log in to your Notion workspace and navigate to the [My integrations](https://www.notion.so/my-integrations) page. Select **New integration**. :::note You must be the owner of the Notion workspace to create a new integration associated with it. ::: -2. Enter a **Name** for your integration. Make sure you have selected the workspace containing your data to replicate from the **Associated workspace** dropdown menu, and click **Submit**. -3. In the navbar, select **Capabilities** and make sure to check the **Read content** checkbox to authorize Airbyte to read the content of your pages. You may also wish to check the **Read comments** box, as well as set a User capability to allow access to user information. For more details on the capabilities you can enable, please refer to the [Notion documentation on capabilities](https://developers.notion.com/reference/capabilities). +2. Enter a **Name** for your integration. Make sure you have selected the correct workspace from the **Associated workspace** dropdown menu, and click **Submit**. +3. In the navbar, select [**Capabilities**](https://developers.notion.com/reference/capabilities). Check the following capabilities based on your use case: -### Step 2: Set permissions and acquire authorization credentials +- [**Read content**](https://developers.notion.com/reference/capabilities#content-capabilities): required for all connections. +- [**Read comments**](https://developers.notion.com/reference/capabilities#comment-capabilities): required if you wish to sync the `Comments` stream +- [**Read user information**](https://developers.notion.com/reference/capabilities#user-capabilities) (either with or without emails): required if you wish to sync the `Users` stream + +### Step 2: Share pages and acquire authorization credentials #### Access Token (Cloud and Open Source) -If you are authenticating via Access Token, you will need to manually set permissions for each page you want to share with Airbyte. +If you are authenticating via Access Token, you will need to manually share each page you want to sync with Airbyte. 1. Navigate to the page(s) you want to share with Airbyte. Click the **•••** menu at the top right of the page, select **Add connections**, and choose the integration you created in Step 1. -2. Once you have selected all the pages to share, you can find and copy the Access Token from the **Secrets** tab of your Notion integration's page. Then proceed to - [setting up the connector in Airbyte](#step-2-set-up-the-notion-connector-in-airbyte). +2. Once you have selected all the pages to share, you can find and copy the Access Token from the **Secrets** tab of your Notion integration's page. Then proceed to [setting up the connector in Airbyte](#step-2-set-up-the-notion-connector-in-airbyte). #### OAuth2.0 (Open Source only) -If you are authenticating via OAuth2.0 for Airbyte Open Source, you will need to make your integration public and acquire your Client ID, Client Secret and Access Token. +If you are authenticating via OAuth2.0 for **Airbyte Open Source**, you will need to make your integration public and acquire your Client ID, Client Secret and Access Token. 1. Navigate to the **Distribution** tab in your integration page, and toggle the switch to make the integration public. 2. Fill out the required fields in the **Organization information** and **OAuth Domain & URIs** section, then click **Submit**. @@ -57,47 +60,49 @@ If you are authenticating via OAuth2.0 for Airbyte Open Source, you will need to ### Step 3: Set up the Notion connector in Airbyte 1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account, or navigate to your Airbyte Open Source dashboard. -2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **New source**. 3. Find and select **Notion** from the list of available sources. 4. Enter a **Source name** of your choosing. 5. Choose the method of authentication from the dropdown menu: + #### Authentication for Airbyte Cloud - **OAuth2.0** (Recommended): Click **Authenticate your Notion account**. When the popup appears, click **Select pages**. Check the pages you want to give Airbyte access to, and click **Allow access**. -- **Access Token**: Copy and paste the Access Token found in the **Secrets** tab of your Notion integration's page. +- **Access Token**: Copy and paste the Access Token found in the **Secrets** tab of your private integration's page. + + #### Authentication for Airbyte Open Source -- **Access Token**: Copy and paste the Access Token found in the **Secrets** tab of your Notion integration's page. -- **OAuth2.0**: Copy and paste the Client ID, Client Secret and Access Token you acquired. +- **Access Token**: Copy and paste the Access Token found in the **Secrets** tab of your private integration's page. +- **OAuth2.0**: Copy and paste the Client ID, Client Secret and Access Token you acquired after setting up your public integration. + -6. Enter the **Start Date** using the provided datepicker, or by programmatically entering a UTC date and time in the format: `YYYY-MM-DDTHH:mm:ss.SSSZ`. All data generated after this date will be replicated. +6. (Optional) You may optionally provide a **Start Date** using the provided datepicker, or by programmatically entering a UTC date and time in the format: `YYYY-MM-DDTHH:mm:ss.SSSZ`. When using incremental syncs, only data generated after this date will be replicated. If left blank, Airbyte will set the start date two years from the current date by default. 7. Click **Set up source** and wait for the tests to complete. ## Supported sync modes The Notion source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): -- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/) -- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append) -- [Incremental - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) (partially) -- [Incremental - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped) +| Stream | Full Refresh (Overwrite/Append) | Incremental (Append/Append + Deduped) | +|-----------|:------------:|:-----------:| +| Blocks | ✓ | ✓ | +| Comments | ✓ | ✓ | +| Databases | ✓ | ✓ | +| Pages | ✓ | ✓ | +| Users | ✓ | | ## Supported Streams -The Notion source connector supports the following streams. For more information, see the [Notion API](https://developers.notion.com/reference/intro). - -- [blocks](https://developers.notion.com/reference/retrieve-a-block) -- [databases](https://developers.notion.com/reference/retrieve-a-database) -- [pages](https://developers.notion.com/reference/retrieve-a-page) -- [users](https://developers.notion.com/reference/get-user) +The Notion source connector supports the following streams: -:::note - -The users stream does not support Incremental - Append sync mode. - -::: +- [Blocks](https://developers.notion.com/reference/retrieve-a-block) +- [Comments](https://developers.notion.com/reference/retrieve-a-comment) +- [Databases](https://developers.notion.com/reference/retrieve-a-database) +- [Pages](https://developers.notion.com/reference/retrieve-a-page) +- [Users](https://developers.notion.com/reference/get-users) ## Performance considerations @@ -105,28 +110,42 @@ The connector is restricted by Notion [request limits](https://developers.notion ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------- | -| 1.1.1 | 2023-06-14 | [26535](https://github.com/airbytehq/airbyte/pull/26535) | Migrate from deprecated `authSpecification` to `advancedAuth` | -| 1.1.0 | 2023-06-08 | [27170](https://github.com/airbytehq/airbyte/pull/27170) | Fix typo in `blocks` schema | -| 1.0.9 | 2023-06-08 | [27062](https://github.com/airbytehq/airbyte/pull/27062) | Skip streams with `invalid_start_cursor` error | -| 1.0.8 | 2023-06-07 | [27073](https://github.com/airbytehq/airbyte/pull/27073) | Add empty results handling for stream `Blocks` | -| 1.0.7 | 2023-06-06 | [27060](https://github.com/airbytehq/airbyte/pull/27060) | Add skipping 404 error in `Blocks` stream | -| 1.0.6 | 2023-05-18 | [26286](https://github.com/airbytehq/airbyte/pull/26286) | Add `parent` field to `Blocks` stream | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------- | +| 2.0.8 | 2023-11-01 | [31899](https://github.com/airbytehq/airbyte/pull/31899) | Fix `table_row.cells` property in `Blocks` stream | +| 2.0.7 | 2023-10-31 | [32004](https://github.com/airtybehq/airbyte/pull/32004) | Reduce page_size on 504 errors | +| 2.0.6 | 2023-10-25 | [31825](https://github.com/airbytehq/airbyte/pull/31825) | Increase max_retries on retryable errors | +| 2.0.5 | 2023-10-23 | [31742](https://github.com/airbytehq/airbyte/pull/31742) | Add 'synced_block' property to Blocks schema | +| 2.0.4 | 2023-10-19 | [31625](https://github.com/airbytehq/airbyte/pull/31625) | Fix check_connection method | +| 2.0.3 | 2023-10-19 | [31612](https://github.com/airbytehq/airbyte/pull/31612) | Add exponential backoff for 500 errors | +| 2.0.2 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 2.0.1 | 2023-10-17 | [31507](https://github.com/airbytehq/airbyte/pull/31507) | Add start_date validation checks | +| 2.0.0 | 2023-10-09 | [30587](https://github.com/airbytehq/airbyte/pull/30587) | Source-wide schema update | +| 1.3.0 | 2023-10-09 | [30324](https://github.com/airbytehq/airbyte/pull/30324) | Add `Comments` stream | +| 1.2.2 | 2023-10-09 | [30780](https://github.com/airbytehq/airbyte/pull/30780) | Update Start Date in config to optional field | +| 1.2.1 | 2023-10-08 | [30750](https://github.com/airbytehq/airbyte/pull/30750) | Add availability strategy | +| 1.2.0 | 2023-10-04 | [31053](https://github.com/airbytehq/airbyte/pull/31053) | Add undeclared fields for blocks and pages streams | +| 1.1.2 | 2023-08-30 | [29999](https://github.com/airbytehq/airbyte/pull/29999) | Update error handling during connection check | +| 1.1.1 | 2023-06-14 | [26535](https://github.com/airbytehq/airbyte/pull/26535) | Migrate from deprecated `authSpecification` to `advancedAuth` | +| 1.1.0 | 2023-06-08 | [27170](https://github.com/airbytehq/airbyte/pull/27170) | Fix typo in `blocks` schema | +| 1.0.9 | 2023-06-08 | [27062](https://github.com/airbytehq/airbyte/pull/27062) | Skip streams with `invalid_start_cursor` error | +| 1.0.8 | 2023-06-07 | [27073](https://github.com/airbytehq/airbyte/pull/27073) | Add empty results handling for stream `Blocks` | +| 1.0.7 | 2023-06-06 | [27060](https://github.com/airbytehq/airbyte/pull/27060) | Add skipping 404 error in `Blocks` stream | +| 1.0.6 | 2023-05-18 | [26286](https://github.com/airbytehq/airbyte/pull/26286) | Add `parent` field to `Blocks` stream | | 1.0.5 | 2023-05-01 | [25709](https://github.com/airbytehq/airbyte/pull/25709) | Fixed `ai_block is unsupported by API` issue, while fetching `Blocks` stream | -| 1.0.4 | 2023-04-11 | [25041](https://github.com/airbytehq/airbyte/pull/25041) | Improve error handling for API /search | -| 1.0.3 | 2023-03-02 | [22931](https://github.com/airbytehq/airbyte/pull/22931) | Specified date formatting in specification | -| 1.0.2 | 2023-02-24 | [23437](https://github.com/airbytehq/airbyte/pull/23437) | Add retry for 400 error (validation_error) | -| 1.0.1 | 2023-01-27 | [22018](https://github.com/airbytehq/airbyte/pull/22018) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 1.0.0 | 2022-12-19 | [20639](https://github.com/airbytehq/airbyte/pull/20639) | Fix `Pages` stream schema | -| 0.1.10 | 2022-09-28 | [17298](https://github.com/airbytehq/airbyte/pull/17298) | Use "Retry-After" header for backoff | -| 0.1.9 | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state | -| 0.1.8 | 2022-09-05 | [16272](https://github.com/airbytehq/airbyte/pull/16272) | Update spec description to include working timestamp example | -| 0.1.7 | 2022-07-26 | [15042](https://github.com/airbytehq/airbyte/pull/15042) | Update `additionalProperties` field to true from shared schemas | -| 0.1.6 | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from schemas and spec | -| 0.1.5 | 2022-07-14 | [14706](https://github.com/airbytehq/airbyte/pull/14706) | Added OAuth2.0 authentication | -| 0.1.4 | 2022-07-07 | [14505](https://github.com/airbytehq/airbyte/pull/14505) | Fixed bug when normalization didn't run through | -| 0.1.3 | 2022-04-22 | [11452](https://github.com/airbytehq/airbyte/pull/11452) | Use pagination for User stream | -| 0.1.2 | 2022-01-11 | [9084](https://github.com/airbytehq/airbyte/pull/9084) | Fix documentation URL | -| 0.1.1 | 2021-12-30 | [9207](https://github.com/airbytehq/airbyte/pull/9207) | Update connector fields title/description | -| 0.1.0 | 2021-10-17 | [7092](https://github.com/airbytehq/airbyte/pull/7092) | Initial Release | +| 1.0.4 | 2023-04-11 | [25041](https://github.com/airbytehq/airbyte/pull/25041) | Improve error handling for API /search | +| 1.0.3 | 2023-03-02 | [22931](https://github.com/airbytehq/airbyte/pull/22931) | Specified date formatting in specification | +| 1.0.2 | 2023-02-24 | [23437](https://github.com/airbytehq/airbyte/pull/23437) | Add retry for 400 error (validation_error) | +| 1.0.1 | 2023-01-27 | [22018](https://github.com/airbytehq/airbyte/pull/22018) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 1.0.0 | 2022-12-19 | [20639](https://github.com/airbytehq/airbyte/pull/20639) | Fix `Pages` stream schema | +| 0.1.10 | 2022-09-28 | [17298](https://github.com/airbytehq/airbyte/pull/17298) | Use "Retry-After" header for backoff | +| 0.1.9 | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state | +| 0.1.8 | 2022-09-05 | [16272](https://github.com/airbytehq/airbyte/pull/16272) | Update spec description to include working timestamp example | +| 0.1.7 | 2022-07-26 | [15042](https://github.com/airbytehq/airbyte/pull/15042) | Update `additionalProperties` field to true from shared schemas | +| 0.1.6 | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from schemas and spec | +| 0.1.5 | 2022-07-14 | [14706](https://github.com/airbytehq/airbyte/pull/14706) | Added OAuth2.0 authentication | +| 0.1.4 | 2022-07-07 | [14505](https://github.com/airbytehq/airbyte/pull/14505) | Fixed bug when normalization didn't run through | +| 0.1.3 | 2022-04-22 | [11452](https://github.com/airbytehq/airbyte/pull/11452) | Use pagination for User stream | +| 0.1.2 | 2022-01-11 | [9084](https://github.com/airbytehq/airbyte/pull/9084) | Fix documentation URL | +| 0.1.1 | 2021-12-30 | [9207](https://github.com/airbytehq/airbyte/pull/9207) | Update connector fields title/description | +| 0.1.0 | 2021-10-17 | [7092](https://github.com/airbytehq/airbyte/pull/7092) | Initial Release | diff --git a/docs/integrations/sources/onesignal.md b/docs/integrations/sources/onesignal.md index 40e2ba112941..2f9df7587ac5 100644 --- a/docs/integrations/sources/onesignal.md +++ b/docs/integrations/sources/onesignal.md @@ -74,7 +74,8 @@ The connector is restricted by normal OneSignal [rate limits](https://documentat ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------- | +|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------| +| 1.1.0 | 2023-08-31 | [28941](https://github.com/airbytehq/airbyte/pull/28941) | Migrate connector to low-code | | 1.0.1 | 2023-03-14 | [24076](https://github.com/airbytehq/airbyte/pull/24076) | Fix schema and add additionalProperties true | | 1.0.0 | 2023-03-14 | [24076](https://github.com/airbytehq/airbyte/pull/24076) | Update connectors spec; fix incremental sync | | 0.1.2 | 2021-12-07 | [8582](https://github.com/airbytehq/airbyte/pull/8582) | Update connector fields title/description | diff --git a/docs/integrations/sources/open-exchange-rates.md b/docs/integrations/sources/open-exchange-rates.md index 75d00a26f1cd..348a54b6b48d 100644 --- a/docs/integrations/sources/open-exchange-rates.md +++ b/docs/integrations/sources/open-exchange-rates.md @@ -45,4 +45,5 @@ If you have `free` subscription plan \(you may check it [here](https://openexcha | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------- | +| 0.2.0 | 2023-10-03 | [30983](https://github.com/airbytehq/airbyte/pull/30983) | Migrate to low code | | 0.1.0 | 2022-11-15 | [19436](https://github.com/airbytehq/airbyte/issues/19436) | Created CDK native Open Exchange Rates connector | diff --git a/docs/integrations/sources/openweather.md b/docs/integrations/sources/openweather.md index f5ab6c875b02..1e76a9996782 100644 --- a/docs/integrations/sources/openweather.md +++ b/docs/integrations/sources/openweather.md @@ -34,6 +34,7 @@ The free plan allows 60 calls per minute and 1,000,000 calls per month, you won' | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.2.0 | 2023-08-31 | [29983](https://github.com/airbytehq/airbyte/pull/29983) | Migrate to Low Code Framework | | 0.1.6 | 2022-06-21 | [16136](https://github.com/airbytehq/airbyte/pull/16136) | Update openweather onecall api to 3.0. | | 0.1.5 | 2022-06-21 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | No changes. Used connector to test publish workflow changes. | | 0.1.4 | 2022-04-27 | [12397](https://github.com/airbytehq/airbyte/pull/12397) | No changes. Used connector to test publish workflow changes. | diff --git a/docs/integrations/sources/opsgenie.md b/docs/integrations/sources/opsgenie.md index 19a1c3e57da3..088d27f21988 100644 --- a/docs/integrations/sources/opsgenie.md +++ b/docs/integrations/sources/opsgenie.md @@ -51,5 +51,6 @@ The Opsgenie connector uses the most recent API version for each source of data. | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------| :--- | +| 0.3.0 | 2023-10-19 | [31552](https://github.com/airbytehq/airbyte/pull/31552) | Migrated to Low Code | +| 0.2.0 | 2023-10-24 | [31777](https://github.com/airbytehq/airbyte/pull/31777) | Fix schema | | 0.1.0 | 2022-09-14 | [16768](https://github.com/airbytehq/airbyte/pull/16768) | Initial Release | - diff --git a/docs/integrations/sources/oracle.md b/docs/integrations/sources/oracle.md index 1e81b7c73fed..35eca6afe065 100644 --- a/docs/integrations/sources/oracle.md +++ b/docs/integrations/sources/oracle.md @@ -20,7 +20,7 @@ The Oracle source does not alter the schema present in your database. Depending On Airbyte Cloud, only TLS connections to your Oracle instance are supported. Other than that, you can proceed with the open-source instructions below. -## Getting Started \(Airbyte Open-Source\) +## Getting Started \(Airbyte Open Source\) #### Requirements @@ -131,7 +131,8 @@ Airbyte has the ability to connect to the Oracle source with 3 network connectiv ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------- | +|:--------|:-----------| :------------------------------------------------------- |:------------------------------------------------------------------------------------------------------------------------------------------| +| 0.5.0 | 2023-12-18 | [33485](https://github.com/airbytehq/airbyte/pull/33485) | Remove LEGACY state | | 0.4.0 | 2023-06-26 | [27737](https://github.com/airbytehq/airbyte/pull/27737) | License Update: Elv2 | | 0.3.25 | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212) | Fix silent exception swallowing in StreamingJdbcDatabase | | 0.3.24 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | diff --git a/docs/integrations/sources/orbit.md b/docs/integrations/sources/orbit.md index 370235692a7b..eea1c8ba67dd 100644 --- a/docs/integrations/sources/orbit.md +++ b/docs/integrations/sources/orbit.md @@ -45,5 +45,7 @@ The Orbit API Key should be available to you immediately as an Orbit user. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.3.0 | 2023-10-25 | [30976](https://github.com/airbytehq/airbyte/pull/30976) | Migrate to low-code framework | +| 0.2.0 | 2023-10-23 | [31747](https://github.com/airbytehq/airbyte/pull/31747) | Update schema | | 0.1.1 | 2022-06-28 | [14208](https://github.com/airbytehq/airbyte/pull/14208) | Remove unused schema | | 0.1.0 | 2022-06-27 | [13390](https://github.com/airbytehq/airbyte/pull/13390) | Initial Release | diff --git a/docs/integrations/sources/outreach.md b/docs/integrations/sources/outreach.md index 631ef791998c..149121a644b0 100644 --- a/docs/integrations/sources/outreach.md +++ b/docs/integrations/sources/outreach.md @@ -53,6 +53,7 @@ List of available streams: | Version | Date | Pull Request | Subject | | :------ |:-----------| :----- | :------ | +| 0.5.0 | 2023-09-20 | [30639](https://github.com/airbytehq/airbyte/pull/30639) | Add Call Purposes and Call Dispositions streams | 0.4.0 | 2023-06-14 | [27343](https://github.com/airbytehq/airbyte/pull/27343) | Add Users, Tasks, Templates, Snippets streams | 0.3.0 | 2023-05-17 | [26211](https://github.com/airbytehq/airbyte/pull/26211) | Add SequenceStates Stream | 0.2.0 | 2022-10-27 | [17385](https://github.com/airbytehq/airbyte/pull/17385) | Add new streams + page size variable + relationship data | diff --git a/docs/integrations/sources/pagerduty.md b/docs/integrations/sources/pagerduty.md index fb5e5660f39e..e517fecf87c8 100644 --- a/docs/integrations/sources/pagerduty.md +++ b/docs/integrations/sources/pagerduty.md @@ -48,6 +48,7 @@ Key](https://support.pagerduty.com/docs/generating-api-keys#section-generating-a ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.23 | 2021-11-12 | [125](https://github.com/faros-ai/airbyte-connectors/pull/125) | Add Pagerduty source and destination | +| Version | Date | Pull Request | Subject | +| :------- | :--------- | :----------------------------------------------------------------- | :----------------------------------- | +| 0.2.0 | 2023-10-20 | [31160](https://github.com/airbytehq/airbyte/pull/31160) | Migrate to low code | +| 0.1.23 | 2021-11-12 | [125](https://github.com/faros-ai/airbyte-connectors/pull/125) | Add Pagerduty source and destination | diff --git a/docs/integrations/sources/pardot.md b/docs/integrations/sources/pardot.md index f8f304797a39..5de66dd17329 100644 --- a/docs/integrations/sources/pardot.md +++ b/docs/integrations/sources/pardot.md @@ -1,7 +1,62 @@ # Pardot +## Overview + The Airbyte Source for [Salesforce Pardot](https://www.pardot.com/) +The Pardot supports full refresh syncs + +### Output schema + +Several output streams are available from this source: + +* [Campaigns](https://developer.salesforce.com/docs/marketing/pardot/guide/campaigns-v4.html) +* [EmailClicks](https://developer.salesforce.com/docs/marketing/pardot/guide/batch-email-clicks-v4.html) +* [ListMembership](https://developer.salesforce.com/docs/marketing/pardot/guide/list-memberships-v4.html) +* [Lists](https://developer.salesforce.com/docs/marketing/pardot/guide/lists-v4.html) +* [ProspectAccounts](https://developer.salesforce.com/docs/marketing/pardot/guide/prospect-accounts-v4.html) +* [Prospects](https://developer.salesforce.com/docs/marketing/pardot/guide/prospects-v4.html) +* [Users](https://developer.salesforce.com/docs/marketing/pardot/guide/users-v4.html) +* [VisitorActivities](https://developer.salesforce.com/docs/marketing/pardot/guide/visitor-activities-v4.html) +* [Visitors](https://developer.salesforce.com/docs/marketing/pardot/guide/visitors-v4.html) +* [Visits](https://developer.salesforce.com/docs/marketing/pardot/guide/visits-v4.html) + +If there are more endpoints you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose) + +### Features + +| Feature | Supported? | +| :--- | :--- | +| Full Refresh Sync | Yes | +| Incremental Sync | No | +| SSL connection | No | +| Namespaces | No | + +### Performance considerations + +The Pardot connector should not run into Pardot API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. + +## Getting started + +### Requirements + +* Pardot Account +* Pardot Business Unit ID +* Client ID +* Client Secret +* Refresh Token +* Start Date +* Is Sandbox environment? + +### Setup guide + +* `pardot_business_unit_id`: Pardot Business ID, can be found at Setup > Pardot > Pardot Account Setup +* `client_id`: The Consumer Key that can be found when viewing your app in Salesforce +* `client_secret`: The Consumer Secret that can be found when viewing your app in Salesforce +* `refresh_token`: Salesforce Refresh Token used for Airbyte to access your Salesforce account. If you don't know what this is, follow [this guide](https://medium.com/@bpmmendis94/obtain-access-refresh-tokens-from-salesforce-rest-api-a324fe4ccd9b) to retrieve it. +* `start_date`: UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. Leave blank to skip this filter +* `is_sandbox`: Whether or not the app is in a Salesforce sandbox. If you do not know what this is, assume it is false. + ## Changelog | Version | Date | Pull Request | Subject | diff --git a/docs/integrations/sources/paypal-transaction-migrations.md b/docs/integrations/sources/paypal-transaction-migrations.md new file mode 100644 index 000000000000..abe8b5f55900 --- /dev/null +++ b/docs/integrations/sources/paypal-transaction-migrations.md @@ -0,0 +1,11 @@ +# Paypal Transaction Migration Guide + +## Upgrading to 2.1.0 + +Version 2.1.0 changes the format of the state object. Upgrading to 2.1.0 is safe, but downgrading to 2.0.0 is not. + +To downgrade to 2.0.0: +- Edit your connection state: + - Change the keys for the transactions and balances streams to "date" + - Change the format of the cursor to "yyyy-MM-dd'T'HH:mm:ss+00:00" +Alternatively, you can also reset your connection. diff --git a/docs/integrations/sources/paypal-transaction.md b/docs/integrations/sources/paypal-transaction.md index 85277858c465..9390c59dac6f 100644 --- a/docs/integrations/sources/paypal-transaction.md +++ b/docs/integrations/sources/paypal-transaction.md @@ -67,12 +67,12 @@ Paypal transaction API has some [limits](https://developer.paypal.com/docs/integ * `start_date_min` = 3 years, API call lists transaction for the previous three years. * `start_date_max` = 1.5 days, it takes a maximum of three hours for executed transactions to appear in the list transactions call. It is set to 1.5 days by default based on experience, otherwise API throw an error. -* `stream_slice_period` = 1 day, the maximum supported date range is 31 days. +* `stream_slice_period` = 7 day, the maximum supported date range is 31 days. * `records_per_request` = 10000, the maximum number of records in a single request. * `page_size` = 500, the maximum page size is 500. * `requests_per_minute` = 30, maximum limit is 50 requests per minute from IP address to all endpoint -Transactions sync is performed with default `stream_slice_period` = 1 day, it means that there will be 1 request for each day between start_date and now or end_date. if `start_date` is greater then `start_date_max`. Balances sync is similarly performed with default `stream_slice_period` = 1 day, but it will do additional request for the end_date of the sync now. +By default, syncs are performed with a slice period of 7 days. If you see errors with the message `Result set size is greater than the maximum limit. Change the filter criteria and try again.`, lower the size of the slice period in your connection configuration. ## Data type map @@ -87,6 +87,11 @@ Transactions sync is performed with default `stream_slice_period` = 1 day, it me | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------| +| 2.2.1 | 2024-01-11 | [34155](https://github.com/airbytehq/airbyte/pull/34155) | prepare for airbyte-lib | +| 2.2.0 | 2023-10-25 | [31852](https://github.com/airbytehq/airbyte/pull/31852) | The size of the time_window can be configured | +| 2.1.2 | 2023-10-23 | [31759](https://github.com/airbytehq/airbyte/pull/31759) | Keep transaction_id as a string and fetch data in 7-day batches +| 2.1.1 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 2.1.0 | 2023-08-14 | [29223](https://github.com/airbytehq/airbyte/pull/29223) | Migrate Python CDK to Low Code schema | | 2.0.0 | 2023-07-05 | [27916](https://github.com/airbytehq/airbyte/pull/27916) | Update `Balances` schema | | 1.0.0 | 2023-07-03 | [27968](https://github.com/airbytehq/airbyte/pull/27968) | mark `Client ID` and `Client Secret` as required fields | | 0.1.13 | 2023-02-20 | [22916](https://github.com/airbytehq/airbyte/pull/22916) | Specified date formatting in specification | diff --git a/docs/integrations/sources/persistiq.md b/docs/integrations/sources/persistiq.md index 40e6ee64ea9a..10beb248508a 100644 --- a/docs/integrations/sources/persistiq.md +++ b/docs/integrations/sources/persistiq.md @@ -40,4 +40,5 @@ Please read [How to find your API key](https://apidocs.persistiq.com/#introducti | Version | Date | Pull Request | Subject | | :------ | :--------- | :----------------------------------------------------- | :----------------------- | +| 0.2.0 | 2023-10-10 | [TBD](https://github.com/airbytehq/airbyte/pull/TBD) | Migrate to low code | | 0.1.0 | 2022-01-21 | [9515](https://github.com/airbytehq/airbyte/pull/9515) | 🎉 New Source: PersistIq | diff --git a/docs/integrations/sources/pinterest-migrations.md b/docs/integrations/sources/pinterest-migrations.md new file mode 100644 index 000000000000..42e0e32efb31 --- /dev/null +++ b/docs/integrations/sources/pinterest-migrations.md @@ -0,0 +1,9 @@ +# Pinterest Migration Guide + +## Upgrading to 1.0.0 + +This release updates date-time fields with airbyte_type: timestamp_without_timezone for streams BoardPins, BoardSectionPins, Boards, Catalogs, CatalogFeeds. Additionally, the stream names AdvertizerReport and AdvertizerTargetingReport have been renamed to AdvertiserReport and AdvertiserTargetingReport, respectively. + +To ensure uninterrupted syncs, users should: +- Refresh the source schema +- Reset affected streams diff --git a/docs/integrations/sources/pinterest.md b/docs/integrations/sources/pinterest.md index a5ddc684e83c..f58cf38cebaf 100644 --- a/docs/integrations/sources/pinterest.md +++ b/docs/integrations/sources/pinterest.md @@ -4,8 +4,28 @@ This page contains the setup guide and reference information for the Pinterest s ## Prerequisites + + +When setting up the Pinterest source connector with Airbyte Cloud, be aware that Pinterest does not allow configuring permissions during the OAuth authentication process. Therefore, the following permissions will be requested during authentication: + +- See all of your advertising data, including ads, ad groups, campaigns, etc. +- See your public boards, including group boards you join. +- See your secret boards. +- See all of your catalogs data. +- See your public Pins. +- See your secret Pins. +- See your user accounts and followers. + +For more information on the scopes required for Pinterest OAuth, please refer to the [Pinterest API Scopes documentation](https://developers.pinterest.com/docs/getting-started/scopes/#Read%20scopes). + + + + + To set up the Pinterest source connector with Airbyte Open Source, you'll need your Pinterest [App ID and secret key](https://developers.pinterest.com/docs/getting-started/set-up-app/) and the [refresh token](https://developers.pinterest.com/docs/getting-started/authentication/#Refreshing%20an%20access%20token). + + ## Setup guide @@ -18,7 +38,10 @@ To set up the Pinterest source connector with Airbyte Open Source, you'll need y 4. Enter the name for the Pinterest connector. 5. For **Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. As per Pinterest API restriction, the date cannot be more than 90 days in the past. 6. The **OAuth2.0** authorization method is selected by default. Click **Authenticate your Pinterest account**. Log in and authorize your Pinterest account. -7. Click **Set up source**. +7. (Optional) Enter a Start Date using the provided date picker, or by manually entering the date in YYYY-MM-DD format. Data added on and after this date will be replicated. If no date is set, it will default to the latest allowed date by the report API (913 days from today). +8. (Optional) Select one or multiple status values from the dropdown menu. For the ads, ad_groups, and campaigns streams, specifying a status will filter out records that do not match the specified ones. If a status is not specified, the source will default to records with a status of either ACTIVE or PAUSED. +9. (Optional) Add custom reports if needed. For more information, refer to the corresponding section. +10. Click **Set up source**. @@ -31,7 +54,10 @@ To set up the Pinterest source connector with Airbyte Open Source, you'll need y 4. Enter the name for the Pinterest connector. 5. For **Start Date**, enter the date in YYYY-MM-DD format. The data added on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. As per Pinterest API restriction, the date cannot be more than 90 days in the past. 6. The **OAuth2.0** authorization method is selected by default. For **Client ID** and **Client Secret**, enter your Pinterest [App ID and secret key](https://developers.pinterest.com/docs/getting-started/set-up-app/). For **Refresh Token**, enter your Pinterest [Refresh Token](https://developers.pinterest.com/docs/getting-started/authentication/#Refreshing%20an%20access%20token). -7. Click **Set up source**. +7. (Optional) Enter a Start Date using the provided date picker, or by manually entering the date in YYYY-MM-DD format. Data added on and after this date will be replicated. If no date is set, it will default to the latest allowed date by the report API (913 days from today). +8. (Optional) Select one or multiple status values from the dropdown menu. For the ads, ad_groups, and campaigns streams, specifying a status will filter out records that do not match the specified ones. If a status is not specified, the source will default to records with a status of either ACTIVE or PAUSED. +9. (Optional) Add custom reports if needed. For more information, refer to the corresponding section. +10. Click **Set up source**. ## Supported sync modes @@ -46,55 +72,92 @@ The Pinterest source connector supports the following [sync modes](https://docs. ## Supported Streams - [Account analytics](https://developers.pinterest.com/docs/api/v5/#operation/user_account/analytics) \(Incremental\) -- [Boards](https://developers.pinterest.com/docs/api/v5/#operation/boards/list) \(Full table\) - - [Board sections](https://developers.pinterest.com/docs/api/v5/#operation/board_sections/list) \(Full table\) - - [Pins on board section](https://developers.pinterest.com/docs/api/v5/#operation/board_sections/list_pins) \(Full table\) - - [Pins on board](https://developers.pinterest.com/docs/api/v5/#operation/boards/list_pins) \(Full table\) -- [Ad accounts](https://developers.pinterest.com/docs/api/v5/#operation/ad_accounts/list) \(Full table\) - - [Ad account analytics](https://developers.pinterest.com/docs/api/v5/#operation/ad_account/analytics) \(Incremental\) - - [Campaigns](https://developers.pinterest.com/docs/api/v5/#operation/campaigns/list) \(Incremental\) - - [Campaign analytics](https://developers.pinterest.com/docs/api/v5/#operation/campaigns/list) \(Incremental\) - - [Campaign Analytics Report](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report) \(Incremental\) - - [Ad groups](https://developers.pinterest.com/docs/api/v5/#operation/ad_groups/list) \(Incremental\) - - [Ad group analytics](https://developers.pinterest.com/docs/api/v5/#operation/ad_groups/analytics) \(Incremental\) - - [Ads](https://developers.pinterest.com/docs/api/v5/#operation/ads/list) \(Incremental\) - - [Ad analytics](https://developers.pinterest.com/docs/api/v5/#operation/ads/analytics) \(Incremental\) +- [Boards](https://developers.pinterest.com/docs/api/v5/#operation/boards/list) \(Full refresh\) +- [Board sections](https://developers.pinterest.com/docs/api/v5/#operation/board_sections/list) \(Full refresh\) +- [Pins on board section](https://developers.pinterest.com/docs/api/v5/#operation/board_sections/list_pins) \(Full refresh\) +- [Pins on board](https://developers.pinterest.com/docs/api/v5/#operation/boards/list_pins) \(Full refresh\) +- [Ad accounts](https://developers.pinterest.com/docs/api/v5/#operation/ad_accounts/list) \(Full refresh\) +- [Ad account analytics](https://developers.pinterest.com/docs/api/v5/#operation/ad_account/analytics) \(Incremental\) +- [Campaigns](https://developers.pinterest.com/docs/api/v5/#operation/campaigns/list) \(Incremental\) +- [Campaign analytics](https://developers.pinterest.com/docs/api/v5/#operation/campaigns/list) \(Incremental\) +- [Campaign Analytics Report](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report) \(Incremental\) +- [Campaign Targeting Report](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report) \(Incremental\) +- [Ad Groups](https://developers.pinterest.com/docs/api/v5/#operation/ad_groups/list) \(Incremental\) +- [Ad Group Analytics](https://developers.pinterest.com/docs/api/v5/#operation/ad_groups/analytics) \(Incremental\) +- [Ad Group Report](https://developers.pinterest.com/docs/api/v5/#operation/ad_groups/analytics) \(Incremental\) +- [Ad Group Targeting Report](https://developers.pinterest.com/docs/api/v5/#operation/ad_groups/analytics) \(Incremental\) +- [Ads](https://developers.pinterest.com/docs/api/v5/#operation/ads/list) \(Incremental\) +- [Ad analytics](https://developers.pinterest.com/docs/api/v5/#operation/ads/analytics) \(Incremental\) +- [Catalogs](https://developers.pinterest.com/docs/api/v5/#operation/catalogs/list) \(Full refresh\) +- [Catalogs Feeds](https://developers.pinterest.com/docs/api/v5/#operation/feeds/list) \(Full refresh\) +- [Catalogs Product Groups](https://developers.pinterest.com/docs/api/v5/#operation/catalogs_product_groups/list) \(Full refresh\) +- [Audiences](https://developers.pinterest.com/docs/api/v5/#operation/audiences/list) \(Full refresh\) +- [Keywords](https://developers.pinterest.com/docs/api/v5/#operation/keywords/get) \(Full refresh\) +- [Conversion Tags](https://developers.pinterest.com/docs/api/v5/#operation/conversion_tags/list) \(Full refresh\) +- [Customer Lists](https://developers.pinterest.com/docs/api/v5/#tag/customer_lists) \(Full refresh\) +- [Advertizer Report](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report) \(Incremental\) +- [Advertizer Targeting Report](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report) \(Incremental\) +- [Pin Promotion Report](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report) \(Incremental\) +- [Pin Promotion Targeting Report](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report) \(Incremental\) +- [Product Group Report](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report) \(Incremental\) +- [Product Group Targeting Report](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report) \(Incremental\) +- [Product Item Report](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report) \(Incremental\) +- [Keyword Report](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report) \(Incremental\) + +## Custom reports + +Custom reports in the Pinterest connector allow you to create personalized analytics reports for your account. You can tailor these reports to your specific needs by choosing from various properties: + +1. **Name**: A unique identifier for the report. +2. **Level**: Specifies the data aggregation level, with options like ADVERTISER, CAMPAIGN, AD_GROUP, etc. The default level is ADVERTISER. +3. **Granularity**: Determines the data granularity, such as TOTAL, DAY, HOUR, etc. The default is TOTAL, where metrics are aggregated over the specified date range. +4. **Columns**: Identifies the data columns to be included in the report. +5. **Click Window Days (Optional)**: The number of days used for conversion attribution from a pin click action. This applies to Pinterest Tag conversion metrics. Defaults to 30 days if not specified. +6. **Engagement Window Days (Optional)**: The number of days used for conversion attribution from an engagement action. Engagements include saves, closeups, link clicks, and carousel card swipes. This applies to Pinterest Tag conversion metrics. Defaults to 30 days if not specified. +7. **View Window Days (Optional)**: The number of days used as the conversion attribution window for a view action. This applies to Pinterest Tag conversion metrics. Defaults to 1 day if not specified. +8. **Conversion Report Time (Optional)**: Indicates the date by which the conversion metrics returned will be reported. There are two dates associated with a conversion event: the date of ad interaction and the date of conversion event completion. The default is TIME_OF_AD_ACTION. +9. **Attribution Types (Optional)**: Lists the types of attribution for the report, such as INDIVIDUAL or HOUSEHOLD. +10. **Start Date (Optional)**: The start date for the report in YYYY-MM-DD format, defaulting to the latest allowed date by the report API (913 days from today). + +For more detailed information and guidelines on creating custom reports, please refer to the [Pinterest API documentation](https://developers.pinterest.com/docs/api/v5/#operation/analytics/create_report). ## Performance considerations -The connector is restricted by the Pinterest [requests limitation](https://developers.pinterest.com/docs/api/v5/#tag/Rate-limits). - -##### Rate Limits - -- Analytics streams: 300 calls per day / per user \ -- Ad accounts streams (Campaigns, Ad groups, Ads): 1000 calls per min / per user / per app \ -- Boards streams: 10 calls per sec / per user / per app +The connector is restricted by the Pinterest [requests limitation](https://developers.pinterest.com/docs/reference/ratelimits/). ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------------------------------------------- | -| 0.6.0 | 2023-07-25 | [28672](https://github.com/airbytehq/airbyte/pull/28672) | Add report stream for `CAMPAIGN` level | -| 0.5.3 | 2023-07-05 | [27964](https://github.com/airbytehq/airbyte/pull/27964) | Add `id` field to `owner` field in `ad_accounts` stream | -| 0.5.2 | 2023-06-02 | [26949](https://github.com/airbytehq/airbyte/pull/26949) | Update `BoardPins` stream with `note` property | -| 0.5.1 | 2023-05-11 | [25984](https://github.com/airbytehq/airbyte/pull/25984) | Add pattern for start_date | -| 0.5.0 | 2023-05-17 | [26188](https://github.com/airbytehq/airbyte/pull/26188) | Add `product_tags` field to the `BoardPins` stream | -| 0.4.0 | 2023-05-16 | [26112](https://github.com/airbytehq/airbyte/pull/26112) | Add `is_standard` field to the `BoardPins` stream | -| 0.3.0 | 2023-05-09 | [25915](https://github.com/airbytehq/airbyte/pull/25915) | Add `creative_type` field to the `BoardPins` stream | -| 0.2.6 | 2023-04-26 | [25548](https://github.com/airbytehq/airbyte/pull/25548) | Fix `format` issue for `boards` stream schema for fields with `date-time` | -| 0.2.5 | 2023-04-19 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Update `AMOUNT_OF_DAYS_ALLOWED_FOR_LOOKUP` to 89 days | -| 0.2.4 | 2023-02-25 | [23457](https://github.com/airbytehq/airbyte/pull/23457) | Add missing columns for analytics streams for pinterest source | -| 0.2.3 | 2023-03-01 | [23649](https://github.com/airbytehq/airbyte/pull/23649) | Fix for `HTTP - 400 Bad Request` when requesting data >= 90 days | -| 0.2.2 | 2023-01-27 | [22020](https://github.com/airbytehq/airbyte/pull/22020) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 0.2.1 | 2022-12-15 | [20532](https://github.com/airbytehq/airbyte/pull/20532) | Bump CDK version | -| 0.2.0 | 2022-12-13 | [20242](https://github.com/airbytehq/airbyte/pull/20242) | Add data-type normalization up to the schemas declared | -| 0.1.9 | 2022-09-06 | [15074](https://github.com/airbytehq/airbyte/pull/15074) | Add filter based on statuses | -| 0.1.8 | 2022-10-21 | [18285](https://github.com/airbytehq/airbyte/pull/18285) | Fix type of `start_date` | -| 0.1.7 | 2022-09-29 | [17387](https://github.com/airbytehq/airbyte/pull/17387) | Set `start_date` dynamically based on API restrictions. | -| 0.1.6 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Use CDK 0.1.89 | -| 0.1.5 | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state | -| 0.1.4 | 2022-09-06 | [16161](https://github.com/airbytehq/airbyte/pull/16161) | Add ability to handle `429 - Too Many Requests` error with respect to `Max Rate Limit Exceeded Error` | -| 0.1.3 | 2022-09-02 | [16271](https://github.com/airbytehq/airbyte/pull/16271) | Add support of `OAuth2.0` authentication method | -| 0.1.2 | 2021-12-22 | [10223](https://github.com/airbytehq/airbyte/pull/10223) | Fix naming of `AD_ID` and `AD_ACCOUNT_ID` fields | -| 0.1.1 | 2021-12-22 | [9043](https://github.com/airbytehq/airbyte/pull/9043) | Update connector fields title/description | -| 0.1.0 | 2021-10-29 | [7493](https://github.com/airbytehq/airbyte/pull/7493) | Release Pinterest CDK Connector | +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :------------------------------------------------------- |:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 1.1.0 | 2023-11-22 | [32747](https://github.com/airbytehq/airbyte/pull/32747) | Update docs and spec. Add missing `placement_traffic_type` field to AdGroups stream | +| 1.0.0 | 2023-11-16 | [32595](https://github.com/airbytehq/airbyte/pull/32595) | Add airbyte_type: timestamp_without_timezone to date-time fields across all streams. Rename `Advertizer*` streams to `Advertiser*` | +| 0.8.2 | 2023-11-20 | [32672](https://github.com/airbytehq/airbyte/pull/32672) | Fix backoff waiting time | +| 0.8.1 | 2023-11-16 | [32601](https://github.com/airbytehq/airbyte/pull/32601) | added ability to create custom reports | +| 0.8.0 | 2023-11-16 | [32592](https://github.com/airbytehq/airbyte/pull/32592) | Make start_date optional; add suggested streams; add missing fields | +| 0.7.2 | 2023-11-08 | [32299](https://github.com/airbytehq/airbyte/pull/32299) | added default `AvailabilityStrategy`, fixed bug which cases duplicated requests, added new streams: Catalogs, CatalogsFeeds, CatalogsProductGroups, Audiences, Keywords, ConversionTags, CustomerLists, CampaignTargetingReport, AdvertizerReport, AdvertizerTargetingReport, AdGroupReport, AdGroupTargetingReport, PinPromotionReport, PinPromotionTargetingReport, ProductGroupReport, ProductGroupTargetingReport, ProductItemReport, KeywordReport | +| 0.7.1 | 2023-11-01 | [32078](https://github.com/airbytehq/airbyte/pull/32078) | handle non json response | +| 0.7.0 | 2023-10-25 | [31876](https://github.com/airbytehq/airbyte/pull/31876) | Migrated to base image, removed token based authentication mthod becuase access_token is valid for 1 day only | +| 0.6.0 | 2023-07-25 | [28672](https://github.com/airbytehq/airbyte/pull/28672) | Add report stream for `CAMPAIGN` level | +| 0.5.3 | 2023-07-05 | [27964](https://github.com/airbytehq/airbyte/pull/27964) | Add `id` field to `owner` field in `ad_accounts` stream | +| 0.5.2 | 2023-06-02 | [26949](https://github.com/airbytehq/airbyte/pull/26949) | Update `BoardPins` stream with `note` property | +| 0.5.1 | 2023-05-11 | [25984](https://github.com/airbytehq/airbyte/pull/25984) | Add pattern for start_date | +| 0.5.0 | 2023-05-17 | [26188](https://github.com/airbytehq/airbyte/pull/26188) | Add `product_tags` field to the `BoardPins` stream | +| 0.4.0 | 2023-05-16 | [26112](https://github.com/airbytehq/airbyte/pull/26112) | Add `is_standard` field to the `BoardPins` stream | +| 0.3.0 | 2023-05-09 | [25915](https://github.com/airbytehq/airbyte/pull/25915) | Add `creative_type` field to the `BoardPins` stream | +| 0.2.6 | 2023-04-26 | [25548](https://github.com/airbytehq/airbyte/pull/25548) | Fix `format` issue for `boards` stream schema for fields with `date-time` | +| 0.2.5 | 2023-04-19 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Update `AMOUNT_OF_DAYS_ALLOWED_FOR_LOOKUP` to 89 days | +| 0.2.4 | 2023-02-25 | [23457](https://github.com/airbytehq/airbyte/pull/23457) | Add missing columns for analytics streams for pinterest source | +| 0.2.3 | 2023-03-01 | [23649](https://github.com/airbytehq/airbyte/pull/23649) | Fix for `HTTP - 400 Bad Request` when requesting data >= 90 days | +| 0.2.2 | 2023-01-27 | [22020](https://github.com/airbytehq/airbyte/pull/22020) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.2.1 | 2022-12-15 | [20532](https://github.com/airbytehq/airbyte/pull/20532) | Bump CDK version | +| 0.2.0 | 2022-12-13 | [20242](https://github.com/airbytehq/airbyte/pull/20242) | Add data-type normalization up to the schemas declared | +| 0.1.9 | 2022-09-06 | [15074](https://github.com/airbytehq/airbyte/pull/15074) | Add filter based on statuses | +| 0.1.8 | 2022-10-21 | [18285](https://github.com/airbytehq/airbyte/pull/18285) | Fix type of `start_date` | +| 0.1.7 | 2022-09-29 | [17387](https://github.com/airbytehq/airbyte/pull/17387) | Set `start_date` dynamically based on API restrictions. | +| 0.1.6 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Use CDK 0.1.89 | +| 0.1.5 | 2022-09-16 | [16799](https://github.com/airbytehq/airbyte/pull/16799) | Migrate to per-stream state | +| 0.1.4 | 2022-09-06 | [16161](https://github.com/airbytehq/airbyte/pull/16161) | Add ability to handle `429 - Too Many Requests` error with respect to `Max Rate Limit Exceeded Error` | +| 0.1.3 | 2022-09-02 | [16271](https://github.com/airbytehq/airbyte/pull/16271) | Add support of `OAuth2.0` authentication method | +| 0.1.2 | 2021-12-22 | [10223](https://github.com/airbytehq/airbyte/pull/10223) | Fix naming of `AD_ID` and `AD_ACCOUNT_ID` fields | +| 0.1.1 | 2021-12-22 | [9043](https://github.com/airbytehq/airbyte/pull/9043) | Update connector fields title/description | +| 0.1.0 | 2021-10-29 | [7493](https://github.com/airbytehq/airbyte/pull/7493) | Release Pinterest CDK Connector | diff --git a/docs/integrations/sources/pipedrive-migrations.md b/docs/integrations/sources/pipedrive-migrations.md new file mode 100644 index 000000000000..d4cef6e3242d --- /dev/null +++ b/docs/integrations/sources/pipedrive-migrations.md @@ -0,0 +1,6 @@ +# Pipedrive Migration Guide + +## Upgrading to 2.0.0 +Please update your config and reset your data (to match the new format). This version has changed the config to only require an API key. + +This version also removes the `pipeline_ids` field from the `deal_fields` stream. diff --git a/docs/integrations/sources/pipedrive.md b/docs/integrations/sources/pipedrive.md index 534f992294fb..cb87f1d27fb2 100644 --- a/docs/integrations/sources/pipedrive.md +++ b/docs/integrations/sources/pipedrive.md @@ -76,6 +76,8 @@ Apart from `Fields` streams, all other streams support incremental. * [Filters](https://developers.pipedrive.com/docs/api/v1/Filters#getFilters) +* [Goals](https://developers.pipedrive.com/docs/api/v1/Goals#getGoals) + * [LeadLabels](https://developers.pipedrive.com/docs/api/v1/LeadLabels#getLeadLabels) * [Leads](https://developers.pipedrive.com/docs/api/v1/Leads#getLeads) @@ -112,6 +114,12 @@ The Pipedrive connector will gracefully handle rate limits. For more information | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------| +| 2.2.2 | 2024-01-11 | [34153](https://github.com/airbytehq/airbyte/pull/34153) | prepare for airbyte-lib | +| 2.2.1 | 2023-11-06 | [31147](https://github.com/airbytehq/airbyte/pull/31147) | Bugfix: handle records with a null data field | +| 2.2.0 | 2023-10-25 | [31707](https://github.com/airbytehq/airbyte/pull/31707) | Add new stream mail | +| 2.1.0 | 2023-10-10 | [31184](https://github.com/airbytehq/airbyte/pull/31184) | Add new stream goals | +| 2.0.1 | 2023-10-13 | [31151](https://github.com/airbytehq/airbyte/pull/31151) | Add additionalProperties in schemas to read custom fields | +| 2.0.0 | 2023-08-09 | [29293](https://github.com/airbytehq/airbyte/pull/29293) | Migrated to Low-Code CDK | | 1.0.0 | 2023-06-29 | [27832](https://github.com/airbytehq/airbyte/pull/27832) | Remove `followers_count` field from `Products` stream | | 0.1.19 | 2023-07-05 | [27967](https://github.com/airbytehq/airbyte/pull/27967) | Update `OrganizationFields` and `ProductFields` with `display_field` field | | 0.1.18 | 2023-06-02 | [26892](https://github.com/airbytehq/airbyte/pull/26892) | Update `DialFields` schema with `pipeline_ids` property | diff --git a/docs/integrations/sources/pivotal-tracker.md b/docs/integrations/sources/pivotal-tracker.md index d533f6c6b061..214eaf95538b 100644 --- a/docs/integrations/sources/pivotal-tracker.md +++ b/docs/integrations/sources/pivotal-tracker.md @@ -53,4 +53,5 @@ Use this to pull data from Pivotal Tracker. | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------- | +| 0.1.1 | 2023-10-25 | [11060](https://github.com/airbytehq/airbyte/pull/11060) | Fix schema and check connection | | 0.1.0 | 2022-04-04 | [11060](https://github.com/airbytehq/airbyte/pull/11060) | Initial Release | diff --git a/docs/integrations/sources/pokeapi.md b/docs/integrations/sources/pokeapi.md index 797a58d71857..ee543b33e024 100644 --- a/docs/integrations/sources/pokeapi.md +++ b/docs/integrations/sources/pokeapi.md @@ -4,7 +4,7 @@ The PokéAPI is primarly used as a tutorial and educational resource, as it requires zero dependencies. Learn how Airbyte and this connector works with these tutorials: -- [Airbyte Quickstart: An Introduction to Deploying and Syncing](../../quickstart/deploy-airbyte.md) +- [Airbyte Quickstart: An Introduction to Deploying and Syncing](../../using-airbyte/getting-started/readme.md) - [Airbyte CDK Speedrun: A Quick Primer on Building Source Connectors](../../connector-development/tutorials/cdk-speedrun.md) - [How to Build ETL Sources in Under 30 Minutes: A Video Tutorial](https://www.youtube.com/watch?v=kJ3hLoNfz_E&t=13s&ab_channel=Airbyte) @@ -24,7 +24,7 @@ This source uses the fully open [PokéAPI](https://pokeapi.co/docs/v2#info) to s Currently, only one output stream is available from this source, which is the Pokémon output stream. This schema is defined [here](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/schemas/pokemon.json). -## Rate Limiting & Performance Considerations \(Airbyte Open-Source\) +## Rate Limiting & Performance Considerations \(Airbyte Open Source\) According to the API's [fair use policy](https://pokeapi.co/docs/v2#fairuse), please make sure to cache resources retrieved from the PokéAPI wherever possible. That said, the PokéAPI does not perform rate limiting. @@ -36,6 +36,7 @@ The PokéAPI uses the same [JSONSchema](https://json-schema.org/understanding-js | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------- | +| 0.2.0 | 2023-10-02 | [30969](https://github.com/airbytehq/airbyte/pull/30969) | Migrated to Low code | 0.1.5 | 2022-05-18 | [12942](https://github.com/airbytehq/airbyte/pull/12942) | Fix example inputs | | 0.1.4 | 2021-12-07 | [8582](https://github.com/airbytehq/airbyte/pull/8582) | Update connector fields title/description | | 0.1.3 | 2021-12-03 | [8432](https://github.com/airbytehq/airbyte/pull/8432) | Migrate from base_python to CDK, add SAT tests. | diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 21819531de08..887a141a2bbe 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -1,6 +1,7 @@ # Postgres Airbyte's certified Postgres connector offers the following features: +* Replicate data from tables, views and materilized views. Other data objects won't be replicated to the destination like indexes, permissions. * Multiple methods of keeping your data fresh, including [Change Data Capture (CDC)](https://docs.airbyte.com/understanding-airbyte/cdc) and replication using the [xmin system column](#xmin). * All available [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes), providing flexibility in how data is delivered to your destination. * Reliable replication at any table size with [checkpointing](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/#state--checkpointing) and chunking of database reads. @@ -50,25 +51,14 @@ To fill out the required information: 5. Select `Standard (xmin)` from available replication methods. This uses the [xmin system column](#xmin) to reliably replicate data from your database. 1. If your database is particularly large (> 500 GB), you will benefit from [configuring your Postgres source using logical replication (CDC)](#cdc). + #### Step 3: (Airbyte Cloud Only) Allow inbound traffic from Airbyte IPs. -If you are on Airbyte Cloud, you will always need to modify your database configuration to allow inbound traffic from Airbyte IPs. These are: - -```roomsql -34.106.109.131 -34.106.196.165 -34.106.60.246 -34.106.229.69 -34.106.127.139 -34.106.218.58 -34.106.115.240 -34.106.225.141 -13.37.4.46 -13.37.142.60 -35.181.124.238 -``` +If you are on Airbyte Cloud, you will always need to modify your database configuration to allow inbound traffic from Airbyte IPs. You can find a list of all IPs that need to be allowlisted in +our [Airbyte Security docs](../../operating-airbyte/security#network-security-1). Now, click `Set up source` in the Airbyte UI. Airbyte will now test connecting to your database. Once this succeeds, you've configured an Airbyte Postgres source! + ## Advanced Configuration @@ -302,8 +292,34 @@ According to Postgres [documentation](https://www.postgresql.org/docs/14/datatyp | Version | Date | Pull Request | Subject | |---------|------------|----------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.3.1 | 2024-01-10 | [34119](https://github.com/airbytehq/airbyte/pull/34119) | Adopt java CDK version 0.11.5. | +| 3.3.0 | 2023-12-19 | [33437](https://github.com/airbytehq/airbyte/pull/33437) | Remove LEGACY state flag | +| 3.2.27 | 2023-12-18 | [33605](https://github.com/airbytehq/airbyte/pull/33605) | Advance Postgres LSN for PG 14 & below. | +| 3.2.26 | 2023-12-11 | [33027](https://github.com/airbytehq/airbyte/pull/32961) | Support for better debugging tools. | +| 3.2.25 | 2023-11-29 | [32961](https://github.com/airbytehq/airbyte/pull/32961) | Bump debezium wait time default to 20 min. | +| 3.2.24 | 2023-11-28 | [32686](https://github.com/airbytehq/airbyte/pull/32686) | Better logging to understand dbz closing reason attribution. | +| 3.2.23 | 2023-11-28 | [32891](https://github.com/airbytehq/airbyte/pull/32891) | Fix CDK dependency in build. | +| 3.2.22 | 2023-11-22 | [32656](https://github.com/airbytehq/airbyte/pull/32656) | Adopt java CDK version 0.5.0. | +| 3.2.21 | 2023-11-07 | [31856](https://github.com/airbytehq/airbyte/pull/31856) | handle date/timestamp infinity values properly | +| 3.2.20 | 2023-11-06 | [32193](https://github.com/airbytehq/airbyte/pull/32193) | Adopt java CDK version 0.4.1. | +| 3.2.19 | 2023-11-03 | [32050](https://github.com/airbytehq/airbyte/pull/32050) | Adopt java CDK version 0.4.0. | +| 3.2.18 | 2023-11-01 | [29038](https://github.com/airbytehq/airbyte/pull/29038) | Fix typo (s/Airbtye/Airbyte/) | +| 3.2.17 | 2023-11-01 | [32068](https://github.com/airbytehq/airbyte/pull/32068) | Bump Debezium 2.2.0Final -> 2.4.0Final | +| 3.2.16 | 2023-10-31 | [31976](https://github.com/airbytehq/airbyte/pull/31976) | Speed up tests involving Debezium | +| 3.2.15 | 2023-10-30 | [31960](https://github.com/airbytehq/airbyte/pull/31960) | Adopt java CDK version 0.2.0. | +| 3.2.14 | 2023-10-24 | [31792](https://github.com/airbytehq/airbyte/pull/31792) | Fix error message link on issue with standby | +| 3.2.14 | 2023-10-24 | [31792](https://github.com/airbytehq/airbyte/pull/31792) | fail sync when debezeum fails to shutdown cleanly | +| 3.2.13 | 2023-10-16 | [31029](https://github.com/airbytehq/airbyte/pull/31029) | Enforces encrypted traffic settings when env var DEPLOYMENT_MODE = CLOUD. | +| 3.1.13 | 2023-10-13 | [31309](https://github.com/airbytehq/airbyte/pull/31309) | Addressed decimals being incorrectly deserialized into scientific notation. | +| 3.1.12 | 2023-10-12 | [31328](https://github.com/airbytehq/airbyte/pull/31328) | Improvements to initial load of tables in older versions of postgres. | +| 3.1.11 | 2023-10-11 | [31322](https://github.com/airbytehq/airbyte/pull/31322) | Correct pevious release | +| 3.1.10 | 2023-09-29 | [30806](https://github.com/airbytehq/airbyte/pull/30806) | Cap log line length to 32KB to prevent loss of records. | +| 3.1.9 | 2023-09-25 | [30534](https://github.com/airbytehq/airbyte/pull/30534) | Fix JSONB[] column type handling bug. | +| 3.1.8 | 2023-09-20 | [30125](https://github.com/airbytehq/airbyte/pull/30125) | Improve initial load performance for older versions of PostgreSQL. | +| 3.1.7 | 2023-09-05 | [29672](https://github.com/airbytehq/airbyte/pull/29672) | Handle VACUUM happening during initial sync | +| 3.1.6 | 2023-08-24 | [29821](https://github.com/airbytehq/airbyte/pull/29821) | Set replication_method display_type to radio, update titles and descriptions, and make CDC the default choice | | 3.1.5 | 2023-08-22 | [29534](https://github.com/airbytehq/airbyte/pull/29534) | Support "options" JDBC URL parameter | -| 3.1.4 | 2023-08-21 | [28687](https://github.com/airbytehq/airbyte/pull/28687) | Under the hood: Add dependency on Java CDK v0.0.2. | +| 3.1.4 | 2023-08-21 | [28687](https://github.com/airbytehq/airbyte/pull/28687) | Under the hood: Add dependency on Java CDK v0.0.2. | | 3.1.3 | 2023-08-03 | [28708](https://github.com/airbytehq/airbyte/pull/28708) | Enable checkpointing snapshots in CDC connections | | 3.1.2 | 2023-08-01 | [28954](https://github.com/airbytehq/airbyte/pull/28954) | Fix an issue that prevented use of tables with names containing uppercase letters | | 3.1.1 | 2023-07-31 | [28892](https://github.com/airbytehq/airbyte/pull/28892) | Fix an issue that prevented use of cursor columns with names containing uppercase letters | diff --git a/docs/integrations/sources/postgres/cloud-sql-postgres.md b/docs/integrations/sources/postgres/cloud-sql-postgres.md index 0dd9bcf5ee3e..670d268f82d3 100644 --- a/docs/integrations/sources/postgres/cloud-sql-postgres.md +++ b/docs/integrations/sources/postgres/cloud-sql-postgres.md @@ -50,6 +50,7 @@ To fill out the required information: 5. Select `Standard (xmin)` from available replication methods. This uses the [xmin system column](https://docs.airbyte.com/integrations/sources/postgres#xmin) to reliably replicate data from your database. 1. If your database is particularly large (> 500 GB), you will benefit from [configuring your Postgres source using logical replication (CDC)](https://docs.airbyte.com/integrations/sources/postgres#cdc). + #### Step 3: (Airbyte Cloud Only) Allow inbound traffic from Airbyte IPs. If you are on Airbyte Cloud, you will always need to modify your database configuration to allow inbound traffic from Airbyte IPs. To allowlist IPs in Cloud SQL: @@ -57,24 +58,12 @@ If you are on Airbyte Cloud, you will always need to modify your database config ![Add a Network](./assets/airbyte_cloud_sql_postgres_add_network.png) -2. Add a new network, and enter Airbyte's IPs: - -```roomsql -34.106.109.131 -34.106.196.165 -34.106.60.246 -34.106.229.69 -34.106.127.139 -34.106.218.58 -34.106.115.240 -34.106.225.141 -13.37.4.46 -13.37.142.60 -35.181.124.238 -``` +2. Add a new network, and enter the Airbyte's IPs, which you can find in our [Airbyte Security documentation](../../../operating-airbyte/security#network-security-1). Now, click `Set up source` in the Airbyte UI. Airbyte will now test connecting to your database. Once this succeeds, you've configured an Airbyte Postgres source! + + ## Advanced Configuration ### Setup using CDC diff --git a/docs/integrations/sources/posthog-migrations.md b/docs/integrations/sources/posthog-migrations.md new file mode 100644 index 000000000000..b4f11d8fef3e --- /dev/null +++ b/docs/integrations/sources/posthog-migrations.md @@ -0,0 +1,5 @@ +# PostHog Migration Guide + +## Upgrading to 1.0.0 + +Version 1.0.0 introduces a single change to the `events` stream. It corrects the casting of the `event` field datatype, which was incorrectly labeled as a `json` object. Now, it is accurately attributed only as a `string`, as outlined in the PostHog [documentation](https://posthog.com/docs/api/events). To apply this change, refresh the schema for the 'events' stream and reset your data. diff --git a/docs/integrations/sources/posthog.md b/docs/integrations/sources/posthog.md index 3ce96d515681..2c40b7515263 100644 --- a/docs/integrations/sources/posthog.md +++ b/docs/integrations/sources/posthog.md @@ -44,15 +44,33 @@ This page contains the setup guide and reference information for the PostHog sou - [Insights](https://posthog.com/docs/api/insights) - [Persons](https://posthog.com/docs/api/people) -### Performance considerations +### Rate limiting -The PostHog API doesn't have any known request limitation. -Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. +Private `GET`, `POST`, `PATCH`, `DELETE` endpoints are rate limited. Public POST-only endpoints are **not** rate limited. A rule of thumb for whether rate limits apply is if the personal API key is used for authentication. + +There are separate limits for different kinds of resources. + +- For all analytics endpoints (such as calculating insights, retrieving persons, or retrieving session recordings), the rate limits are `240/minute` and `1200/hour`. + +- The [HogQL query](https://posthog.com/docs/hogql#api-access) endpoint (`/api/project/:id/query`) has a rate limit of `120/hour`. + +- For the rest of the create, read, update, and delete endpoints, the rate limits are `480/minute` and `4800/hour`. + +- For Public POST-only endpoints like event capture (`/capture`) and feature flag evaluation (`/decide`), there are no rate limits. + +These limits apply to **the entire team** (i.e. all users within your PostHog organization). For example, if a script requesting feature flag metadata hits the rate limit, and another user, using a different personal API key, makes a single request to the persons API, this gets rate limited as well. + +For large or regular exports of events, use [batch exports](https://posthog.com/docs/cdp). + +Want to use the PostHog API beyond these limits? Email Posthog at `customers@posthog.com`. ## Changelog | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------| +| 1.0.0 | 2023-12-04 | [28593](https://github.com/airbytehq/airbyte/pull/28593) | Fix events.event type | +| 0.1.15 | 2023-10-28 | [31265](https://github.com/airbytehq/airbyte/pull/31265) | Fix Events stream datetime format | +| 0.1.14 | 2023-08-29 | [29947](https://github.com/airbytehq/airbyte/pull/29947) | Add optional field to spec: `events_time_step` | | 0.1.13 | 2023-07-19 | [28461](https://github.com/airbytehq/airbyte/pull/28461) | Fixed EventsSimpleRetriever declaration | | 0.1.12 | 2023-06-28 | [27764](https://github.com/airbytehq/airbyte/pull/27764) | Update following state breaking changes | | 0.1.11 | 2023-06-09 | [27135](https://github.com/airbytehq/airbyte/pull/27135) | Fix custom EventsSimpleRetriever | diff --git a/docs/integrations/sources/public-apis.md b/docs/integrations/sources/public-apis.md index c4c9ebaff643..220e20405465 100644 --- a/docs/integrations/sources/public-apis.md +++ b/docs/integrations/sources/public-apis.md @@ -43,4 +43,5 @@ This source requires no setup. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| 0.1.0 | 2022-10-28 | [18471](https://github.com/airbytehq/airbyte/pull/18471) | Initial Release | +| 0.2.0 | 2023-06-15 | [29391](https://github.com/airbytehq/airbyte/pull/29391) | Migrated to Low Code | +| 0.1.0 | 2022-10-28 | [18471](https://github.com/airbytehq/airbyte/pull/18471) | Initial Release | diff --git a/docs/integrations/sources/qonto.md b/docs/integrations/sources/qonto.md index 18536cfaf890..20feb35aab7c 100644 --- a/docs/integrations/sources/qonto.md +++ b/docs/integrations/sources/qonto.md @@ -6,4 +6,5 @@ The Airbyte Source for [Qonto](https://qonto.com) | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------------------------- | +| 0.2.0 | 2023-10-25 | [31603](https://github.com/airbytehq/airbyte/pull/31603) | Migrate to low-code framework | | 0.1.0 | 2022-11-14 | [17452](https://github.com/airbytehq/airbyte/pull/17452) | 🎉 New Source: Qonto [python cdk] | diff --git a/docs/integrations/sources/qualaroo.md b/docs/integrations/sources/qualaroo.md index d17d03266570..b6ae702205df 100644 --- a/docs/integrations/sources/qualaroo.md +++ b/docs/integrations/sources/qualaroo.md @@ -41,6 +41,7 @@ Please read [How to get your APIs Token and Key](https://help.qualaroo.com/hc/en | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------| +| 0.3.0 | 2023-10-25 | [31070](https://github.com/airbytehq/airbyte/pull/31070) | Migrate to low-code framework | | 0.2.0 | 2023-05-24 | [26491](https://github.com/airbytehq/airbyte/pull/26491) | Remove authSpecification from spec.json as OAuth is not supported by Qualaroo + update stream schema | | 0.1.2 | 2022-05-24 | [13121](https://github.com/airbytehq/airbyte/pull/13121) | Fix `start_date` and `survey_ids` schema formatting. Separate source and stream files. Add stream_slices | | 0.1.1 | 2022-05-20 | [13042](https://github.com/airbytehq/airbyte/pull/13042) | Update stream specs | diff --git a/docs/integrations/sources/quickbooks-migrations.md b/docs/integrations/sources/quickbooks-migrations.md new file mode 100644 index 000000000000..aeee6abf2974 --- /dev/null +++ b/docs/integrations/sources/quickbooks-migrations.md @@ -0,0 +1,4 @@ +# QuickBooks Migration Guide + +## Upgrading to 3.0.0 +Some fields in `bills`, `credit_memos`, `items`, `refund_receipts`, and `sales_receipts` streams have been changed from `integer` to `number` to fix normalization. You may need to refresh the connection schema for those streams (skipping the reset), and running a sync. Alternatively, you can just run a reset. diff --git a/docs/integrations/sources/quickbooks.md b/docs/integrations/sources/quickbooks.md index f5f905cd95ed..48866deba5bc 100644 --- a/docs/integrations/sources/quickbooks.md +++ b/docs/integrations/sources/quickbooks.md @@ -103,15 +103,18 @@ This Source is capable of syncing the following [Streams](https://developer.intu ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :----------------------------------------------------------------- | -| `2.0.4` | 2023-06-28 | [27803](https://github.com/airbytehq/airbyte/pull/27803) | Update following state breaking changes | -| `2.0.3` | 2023-06-08 | [27148](https://github.com/airbytehq/airbyte/pull/27148) | Update description and example values of a Start Date in spec.json | -| `2.0.2` | 2023-06-07 | [26722](https://github.com/airbytehq/airbyte/pull/27053) | Update CDK version and adjust authenticator configuration | -| `2.0.1` | 2023-05-28 | [26722](https://github.com/airbytehq/airbyte/pull/26722) | Change datatype for undisclosed amount field in payments | -| `2.0.0` | 2023-04-11 | [25045](https://github.com/airbytehq/airbyte/pull/25045) | Fix datetime format, disable OAuth button in cloud | -| `1.0.0` | 2023-03-20 | [24324](https://github.com/airbytehq/airbyte/pull/24324) | Migrate to Low-Code | -| `0.1.5` | 2022-02-17 | [10346](https://github.com/airbytehq/airbyte/pull/10346) | Update label `Quickbooks` -> `QuickBooks` | -| `0.1.4` | 2021-12-20 | [8960](https://github.com/airbytehq/airbyte/pull/8960) | Update connector fields title/description | -| `0.1.3` | 2021-08-10 | [4986](https://github.com/airbytehq/airbyte/pull/4986) | Using number data type for decimal fields instead string | -| `0.1.2` | 2021-07-06 | [4539](https://github.com/airbytehq/airbyte/pull/4539) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------- | +| `3.0.1` | 2023-11-06 | [32236](https://github.com/airbytehq/airbyte/pull/32236) | Upgrade to `airbyte-cdk>=0.52.10` to resolve refresh token issues | +| `3.0.0` | 2023-09-26 | [30770](https://github.com/airbytehq/airbyte/pull/30770) | Update schema to use `number` instead of `integer` | +| `2.0.5` | 2023-09-26 | [30766](https://github.com/airbytehq/airbyte/pull/30766) | Fix improperly named keyword argument | +| `2.0.4` | 2023-06-28 | [27803](https://github.com/airbytehq/airbyte/pull/27803) | Update following state breaking changes | +| `2.0.3` | 2023-06-08 | [27148](https://github.com/airbytehq/airbyte/pull/27148) | Update description and example values of a Start Date in spec.json | +| `2.0.2` | 2023-06-07 | [26722](https://github.com/airbytehq/airbyte/pull/27053) | Update CDK version and adjust authenticator configuration | +| `2.0.1` | 2023-05-28 | [26722](https://github.com/airbytehq/airbyte/pull/26722) | Change datatype for undisclosed amount field in payments | +| `2.0.0` | 2023-04-11 | [25045](https://github.com/airbytehq/airbyte/pull/25045) | Fix datetime format, disable OAuth button in cloud | +| `1.0.0` | 2023-03-20 | [24324](https://github.com/airbytehq/airbyte/pull/24324) | Migrate to Low-Code | +| `0.1.5` | 2022-02-17 | [10346](https://github.com/airbytehq/airbyte/pull/10346) | Update label `Quickbooks` -> `QuickBooks` | +| `0.1.4` | 2021-12-20 | [8960](https://github.com/airbytehq/airbyte/pull/8960) | Update connector fields title/description | +| `0.1.3` | 2021-08-10 | [4986](https://github.com/airbytehq/airbyte/pull/4986) | Using number data type for decimal fields instead string | +| `0.1.2` | 2021-07-06 | [4539](https://github.com/airbytehq/airbyte/pull/4539) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | diff --git a/docs/integrations/sources/recharge.md b/docs/integrations/sources/recharge.md index ce15a8559e2a..7eb6f1a7f031 100644 --- a/docs/integrations/sources/recharge.md +++ b/docs/integrations/sources/recharge.md @@ -76,6 +76,10 @@ The Recharge connector should gracefully handle Recharge API limitations under n | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------| +| 1.1.2 | 2023-11-03 | [32132](https://github.com/airbytehq/airbyte/pull/32132) | Reduced `period in days` value for `Subscriptions` stream, to avoid `504 - Gateway TimeOut` error | +| 1.1.1 | 2023-09-26 | [30782](https://github.com/airbytehq/airbyte/pull/30782) | For the new style pagination, pass only limit along with cursor | +| 1.1.0 | 2023-09-26 | [30756](https://github.com/airbytehq/airbyte/pull/30756) | Fix pagination and slicing | +| 1.0.1 | 2023-08-30 | [29992](https://github.com/airbytehq/airbyte/pull/29992) | Revert for orders stream to use old API version 2021-01 | | 1.0.0 | 2023-06-22 | [27612](https://github.com/airbytehq/airbyte/pull/27612) | Change data type of the `shopify_variant_id_not_found` field of the `Charges` stream | | 0.2.10 | 2023-06-20 | [27503](https://github.com/airbytehq/airbyte/pull/27503) | Update API version to 2021-11 | | 0.2.9 | 2023-04-10 | [25009](https://github.com/airbytehq/airbyte/pull/25009) | Fix owner slicing for `Metafields` stream | diff --git a/docs/integrations/sources/redshift.md b/docs/integrations/sources/redshift.md index dafe396d2684..434fc60e275e 100644 --- a/docs/integrations/sources/redshift.md +++ b/docs/integrations/sources/redshift.md @@ -55,7 +55,9 @@ All Redshift connections are encrypted using SSL ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------- | +|:--------|:-----------| :------------------------------------------------------- |:------------------------------------------------------------------------------------------------------------------------------------------| +| 0.5.0 | 2023-12-18 | [33484](https://github.com/airbytehq/airbyte/pull/33484) | Remove LEGACY state | +| (none) | 2023-11-17 | [32616](https://github.com/airbytehq/airbyte/pull/32616) | Improve timestamptz handling | | 0.4.0 | 2023-06-26 | [27737](https://github.com/airbytehq/airbyte/pull/27737) | License Update: Elv2 | | 0.3.17 | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212) | Fix silent exception swallowing in StreamingJdbcDatabase | | 0.3.16 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | diff --git a/docs/integrations/sources/s3-migrations.md b/docs/integrations/sources/s3-migrations.md new file mode 100644 index 000000000000..18e6bdb11947 --- /dev/null +++ b/docs/integrations/sources/s3-migrations.md @@ -0,0 +1,28 @@ +# S3 Migration Guide + +## Upgrading to 4.0.4 + +Note: This change is only breaking if you created S3 sources using the API and did not provide `streams.*.format`. + +Following 4.0.0 config change, we are removing `streams.*.file_type` field which was redundant with `streams.*.format`. This is a breaking change as `format` now needs to be required. Given that the UI would always populate `format`, only users creating actors using the API and not providing `format` are be affected. In order to fix that, simply set `streams.*.format` to `{"filetype": }`. + + +## Upgrading to 4.0.0 + +We have revamped the implementation to use the File-Based CDK. The goal is to increase resiliency and reduce development time. Here are the breaking changes: +* [CSV] Mapping of type `array` and `object`: before, they were mapped as `large_string` and hence casted as strings. Given the new changes, if `array` or `object` is specified, the value will be casted as `array` and `object` respectively. +* [CSV] `decimal_point` option is deprecated: It is not possible anymore to use another character than `.` to separate the integer part from non-integer part. Given that the float is format with another character than this, it will be considered as a string. +* [Parquet] `columns` option is deprecated: You can use Airbyte column selection in order to have the same behavior. We don't expect it, but this could have impact on the performance as payload could be bigger. + +Given that you are not affected by the above, your migration should proceed automatically once you run a sync with the new connector. To leverage this: +* Upgrade source-s3 to use v4.0.0 +* Run at least one sync for all your source-s3 connectors + * Migration will be performed and an AirbyteControlMessage will be emitted to the platform so that the migrated config is persisted + +If a user tries to modify the config after source-s3 is upgraded to v4.0.0 and before there was a sync or a periodic discover check, they will have to update the already provided fields manually. To avoid this, a sync can be executed on any of the connections for this source. + +Other than breaking changes, we have changed the UI from which the user configures the source: +* You can now configure multiple streams by clicking on `Add` under `Streams`. +* `Output Stream Name` has been renamed to `Name` when configuring a specific stream. +* `Pattern of files to replicate` field has been renamed `Globs` under the stream configuration. + diff --git a/docs/integrations/sources/s3.md b/docs/integrations/sources/s3.md index 70ae53a88630..2cea82eff718 100644 --- a/docs/integrations/sources/s3.md +++ b/docs/integrations/sources/s3.md @@ -9,12 +9,15 @@ Please note that using cloud storage may incur egress costs. Egress refers to da ## Prerequisites - Access to the S3 bucket containing the files to replicate. +- For **private buckets**, an AWS account with the ability to grant permissions to read from the bucket. ## Setup guide ### Step 1: Set up Amazon S3 -**If you are syncing from a private bucket**, you will need to provide your `AWS Access Key ID` and `AWS Secret Access Key` to authenticate the connection, and ensure that the IAM user associated with the credentials has `read` and `list` permissions for the bucket. If you are unfamiliar with configuring AWS permissions, you can follow these steps to obtain the necessary permissions and credentials: +**If you are syncing from a private bucket**, you need to authenticate the connection. This can be done either by using an `IAM User` (with `AWS Access Key ID` and `Secret Access Key`) or an `IAM Role` (with `Role ARN`). Begin by creating a policy with the necessary permissions: + +#### Create a Policy 1. Log in to your Amazon AWS account and open the [IAM console](https://console.aws.amazon.com/iam/home#home). 2. In the IAM dashboard, select **Policies**, then click **Create Policy**. @@ -39,11 +42,74 @@ Please note that using cloud storage may incur egress costs. Egress refers to da } ``` +:::note +At this time, object-level permissions alone are not sufficient to successfully authenticate the connection. Please ensure you include the **bucket-level** permissions as provided in the example above. +::: + 4. Give your policy a descriptive name, then click **Create policy**. -5. In the IAM dashboard, click **Users**. Select an existing IAM user or create a new one by clicking **Add users**. -6. If you are using an _existing_ IAM user, click the **Add permissions** dropdown menu and select **Add permissions**. If you are creating a _new_ user, you will be taken to the Permissions screen after selecting a name. -7. Select **Attach policies directly**, then find and check the box for your new policy. Click **Next**, then **Add permissions**. -8. After successfully creating your user, select the **Security credentials** tab and click **Create access key**. You will be prompted to select a use case and add optional tags to your access key. Click **Create access key** to generate the keys. + +#### Option 1: Using an IAM Role (Most secure) + + +:::note +This authentication method is currently in the testing phase. To enable it for your workspace, please contact our Support Team. +::: + + +1. In the IAM dashboard, click **Roles**, then **Create role**. +2. Choose the appropriate trust entity and attach the policy you created. +3. Set up a trust relationship for the role. For example for **AWS account** trusted entity use default AWS account on your instance (it will be used to assume role). To use **External ID** set it to environment variables as `export AWS_ASSUME_ROLE_EXTERNAL_ID="{your-external-id}"`. Edit the trust relationship policy to reflect this: +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::{your-aws-account-id}:user/{your-username}" + }, + "Action": "sts:AssumeRole", + "Condition": { + "StringEquals": { + "sts:ExternalId": "{your-external-id}" + } + } + } + ] +} +``` + + +2. Choose the **AWS account** trusted entity type. +3. Set up a trust relationship for the role. This allows the Airbyte instance's AWS account to assume this role. You will also need to specify an external ID, which is a secret key that the trusting service (Airbyte) and the trusted role (the role you're creating) both know. This ID is used to prevent the "confused deputy" problem. The External ID should be your Airbyte workspace ID, which can be found in the URL of your workspace page. Edit the trust relationship policy to include the external ID: +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::094410056844:user/delegated_access_user" + }, + "Action": "sts:AssumeRole", + "Condition": { + "StringEquals": { + "sts:ExternalId": "{your-airbyte-workspace-id}" + } + } + } + ] +} +``` + +4. Complete the role creation and note the Role ARN. + +#### Option 2: Using an IAM User + +1. In the IAM dashboard, click **Users**. Select an existing IAM user or create a new one by clicking **Add users**. +2. If you are using an _existing_ IAM user, click the **Add permissions** dropdown menu and select **Add permissions**. If you are creating a _new_ user, you will be taken to the Permissions screen after selecting a name. +3. Select **Attach policies directly**, then find and check the box for your new policy. Click **Next**, then **Add permissions**. +4. After successfully creating your user, select the **Security credentials** tab and click **Create access key**. You will be prompted to select a use case and add optional tags to your access key. Click **Create access key** to generate the keys. :::caution Your `Secret Access Key` will only be visible once upon creation. Be sure to copy and store it securely for future use. @@ -57,12 +123,18 @@ For more information on managing your access keys, please refer to the 1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account, or navigate to your Airbyte Open Source dashboard. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. 3. Find and select **S3** from the list of available sources. -4. Enter the **Output Stream Name**. This will be the name of the table in the destination (can contain letters, numbers and underscores). -5. Enter the **Pattern of files to replicate**. This is a regular expression that allows Airbyte to pattern match the specific files to replicate. If you are replicating all the files within your bucket, use `**` as the pattern. For more precise pattern matching options, refer to the [Path Patterns section](#path-patterns) below. -6. Enter the name of the **Bucket** containing your files to replicate. -7. Toggle the **Optional fields** under the **Bucket** field to expand additional configuration options. **If you are syncing from a private bucket**, you must fill the **AWS Access Key ID** and **AWS Secret Access Key** fields with the appropriate credentials to authenticate the connection. All other fields are optional and can be left empty. Refer to the [S3 Provider Settings section](#s3-provider-settings) below for more information on each field. -8. In the **File Format** box, use the dropdown menu to select the format of the files you'd like to replicate. The supported formats are **CSV**, **Parquet**, **Avro** and **JSONL**. Toggling the **Optional fields** button within the **File Format** box will allow you to enter additional configurations based on the selected format. For a detailed breakdown of these settings, refer to the [File Format section](#file-format-settings) below. -6. (Optional) - If you want to enforce a specific schema, you can enter a **Manually enforced data schema**. By default, this value is set to `{}` and will automatically infer the schema from the file\(s\) you are replicating. For details on providing a custom schema, refer to the [User Schema section](#user-schema). +4. Enter the name of the **Bucket** containing your files to replicate. +5. Add a stream + 1. Write the **File Type** + 2. In the **Format** box, use the dropdown menu to select the format of the files you'd like to replicate. The supported formats are **CSV**, **Parquet**, **Avro** and **JSONL**. Toggling the **Optional fields** button within the **Format** box will allow you to enter additional configurations based on the selected format. For a detailed breakdown of these settings, refer to the [File Format section](#file-format-settings) below. + 3. Give a **Name** to the stream + 4. (Optional) - If you want to enforce a specific schema, you can enter a **Input schema**. By default, this value is set to `{}` and will automatically infer the schema from the file\(s\) you are replicating. For details on providing a custom schema, refer to the [User Schema section](#user-schema). + 5. Optionally, enter the **Globs** which dictates which files to be synced. This is a regular expression that allows Airbyte to pattern match the specific files to replicate. If you are replicating all the files within your bucket, use `**` as the pattern. For more precise pattern matching options, refer to the [Path Patterns section](#path-patterns) below. +6. **To authenticate your private bucket**: + - If using an IAM role, enter the **AWS Role ARN**. + - If using IAM user credentials, fill the **AWS Access Key ID** and **AWS Secret Access Key** fields with the appropriate credentials. + +All other fields are optional and can be left empty. Refer to the [S3 Provider Settings section](#s3-provider-settings) below for more information on each field. ## Supported sync modes @@ -74,7 +146,7 @@ The Amazon S3 source connector supports the following [sync modes](https://docs. | Incremental Sync | Yes | | Replicate Incremental Deletes | No | | Replicate Multiple Files \(pattern matching\) | Yes | -| Replicate Multiple Streams \(distinct tables\) | No | +| Replicate Multiple Streams \(distinct tables\) | Yes | | Namespaces | No | ## File Compressions @@ -82,7 +154,7 @@ The Amazon S3 source connector supports the following [sync modes](https://docs. | Compression | Supported? | | :---------- | :--------- | | Gzip | Yes | -| Zip | No | +| Zip | Yes | | Bzip2 | Yes | | Lzma | No | | Xz | No | @@ -161,8 +233,8 @@ Or any other reason! The schema must be provided as valid JSON as a map of `{"co For example: -- {"id": "integer", "location": "string", "longitude": "number", "latitude": "number"} -- {"username": "string", "friends": "array", "information": "object"} +- `{"id": "integer", "location": "string", "longitude": "number", "latitude": "number"}` +- `{"username": "string", "friends": "array", "information": "object"}` :::note @@ -187,19 +259,14 @@ Please note, the S3 Source connector used to infer schemas from all the availabl ## File Format Settings -The Reader in charge of loading the file format is currently based on [PyArrow](https://arrow.apache.org/docs/python/generated/pyarrow.csv.open_csv.html) \(Apache Arrow\). - -:::note -All files within one stream must adhere to the same read options for every provided format. -::: - ### CSV Since CSV files are effectively plain text, providing specific reader options is often required for correct parsing of the files. These settings are applied when a CSV is created or exported so please ensure that this process happens consistently over time. +- **Header Definition**: How headers will be defined. `User Provided` assumes the CSV does not have a header row and uses the headers provided and `Autogenerated` assumes the CSV does not have a header row and the CDK will generate headers using for `f{i}` where `i` is the index starting from 0. Else, the default behavior is to use the header from the CSV file. If a user wants to autogenerate or provide column names for a CSV having headers, they can set a value for the "Skip rows before header" option to ignore the header row. - **Delimiter**: Even though CSV is an acronym for Comma Separated Values, it is used more generally as a term for flat file data that may or may not be comma separated. The delimiter field lets you specify which character acts as the separator. To use [tab-delimiters](https://en.wikipedia.org/wiki/Tab-separated_values), you can set this value to `\t`. By default, this value is set to `,`. -- **Infer Datatypes**: This option determines whether a schema for the source should be inferred from the current data. When set to False and a custom schema is provided, the manually enforced schema takes precedence. If no custom schema is set and this option is set to False, all fields will be read as strings. Set to True by default. -- **Quote Character**: In some cases, data values may contain instances of reserved characters \(like a comma, if that's the delimiter\). CSVs can handle this by wrapping a value in defined quote characters so that on read it can parse it correctly. By default, this is set to `"`. +- **Double Quote**: This option determines whether two quotes in a quoted CSV value denote a single quote in the data. Set to True by default. +- **Encoding**: Some data may use a different character set \(typically when different alphabets are involved\). See the [list of allowable encodings here](https://docs.python.org/3/library/codecs.html#standard-encodings). By default, this is set to `utf8`. - **Escape Character**: An escape character can be used to prefix a reserved character and ensure correct parsing. A commonly used character is the backslash (`\`). For example, given the following data: ``` @@ -211,77 +278,70 @@ The backslash (`\`) is used directly before the second double quote (`"`) to ind Leaving this field blank (default option) will disallow escaping. -- **Encoding**: Some data may use a different character set \(typically when different alphabets are involved\). See the [list of allowable encodings here](https://docs.python.org/3/library/codecs.html#standard-encodings). By default, this is set to `utf8`. -- **Double Quote**: This option determines whether two quotes in a quoted CSV value denote a single quote in the data. Set to True by default. -- **Allow newlines in values**: Also known as _multiline_, this option addresses situations where newline characters occur within text data. Typically, newline characters signify the end of a row in a CSV, but when this option is set to True, parsing correctly handles newlines within values. Set to False by default. -- **Additional Reader Options**: This allows for editing the less commonly used CSV [ConvertOptions](https://arrow.apache.org/docs/python/generated/pyarrow.csv.ConvertOptions.html#pyarrow.csv.ConvertOptions). The value must be a valid JSON string, e.g.: - - ``` - { - "timestamp_parsers": [ - "%m/%d/%Y %H:%M", "%Y/%m/%d %H:%M" - ], - "strings_can_be_null": true, - "null_values": [ - "NA", "NULL" - ] - } - ``` - -- **Advanced Options**: This allows for editing the less commonly used CSV [ReadOptions](https://arrow.apache.org/docs/python/generated/pyarrow.csv.ReadOptions.html#pyarrow.csv.ReadOptions). The value must be a valid JSON string. One use case for this is when your CSV has no header, or if you want to use custom column names. You can specify `column_names` using this option. For example: - - ``` - { - "column_names": [ - "column1", "column2", "column3" - ] - } - ``` - -- **Block Size**: This is the number of bytes to process in memory at a time while reading files. The default value of `10000` is usually suitable, but if your files are particularly wide (lots of columns, or the values in the columns are particularly large), increasing this might help avoid schema detection failures. +- **False Values**: A set of case-sensitive strings that should be interpreted as false values. +- **Null Values**: A set of case-sensitive strings that should be interpreted as null values. For example, if the value 'NA' should be interpreted as null, enter 'NA' in this field. +- **Quote Character**: In some cases, data values may contain instances of reserved characters \(like a comma, if that's the delimiter\). CSVs can handle this by wrapping a value in defined quote characters so that on read it can parse it correctly. By default, this is set to `"`. +- **Skip Rows After Header**: The number of rows to skip after the header row. +- **Skip Rows Before Header**: The number of rows to skip before the header row. +- **Strings Can Be Null**: Whether strings can be interpreted as null values. If true, strings that match the null_values set will be interpreted as null. If false, strings that match the null_values set will be interpreted as the string itself. +- **True Values**: A set of case-sensitive strings that should be interpreted as true values. -:::caution -Be cautious when raising this value too high, as it may result in Out Of Memory issues due to increased memory usage. -::: ### Parquet Apache Parquet is a column-oriented data storage format of the Apache Hadoop ecosystem. It provides efficient data compression and encoding schemes with enhanced performance to handle complex data in bulk. At the moment, partitioned parquet datasets are unsupported. The following settings are available: -- **Selected Columns**: If you only want to sync a subset of the columns from the file(s), enter the desired columns here as a comma-delimited list. Leave this field empty to sync all columns. -- **Record Batch Size**: Sets the maximum number of records per batch. Batches may be smaller if there aren’t enough rows in the file. This option can help avoid out-of-memory errors if your data is particularly wide. Set to `65536` by default. -- **Buffer Size**: If set to a positive number, read buffering is performed during the deserializing of individual column chunks. Otherwise I/O calls are unbuffered. Set to `2` by default. - -For more information on these fields, please refer to the [Apache documentation](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetFile.html#pyarrow.parquet.ParquetFile.iter_batches). +- **Convert Decimal Fields to Floats**: Whether to convert decimal fields to floats. There is a loss of precision when converting decimals to floats, so this is not recommended. ### Avro -The Avro parser uses the [Fastavro library](https://fastavro.readthedocs.io/en/latest/). Currently, no additional options are supported. +The Avro parser uses the [Fastavro library](https://fastavro.readthedocs.io/en/latest/). The following settings are available: +- **Convert Double Fields to Strings**: Whether to convert double fields to strings. This is recommended if you have decimal numbers with a high degree of precision because there can be a loss precision when handling floating point numbers. ### JSONL -The JSONL parser uses the PyArrow library, which only supports the line-delimited JSON format. For more detailed info, please refer to the [official docs](https://arrow.apache.org/docs/python/json.html). +There are currently no options for JSONL parsing. -- **Allow newlines in values**: While JSONL typically has each JSON object on a separate line, there are cases where newline characters may appear within JSON values, such as multiline strings. This option enables the parser to correctly interpret and treat newline characters within values. Please note that setting this option to True may affect performance. Set to False by default. + -- **Unexpected field behavior**: This parameter determines how any JSON fields outside of the explicit schema (if defined) are handled. Possible behaviors include: +### Document File Type Format (Experimental) - - `ignore`: Unexpected JSON fields are ignored. - - `error`: Error out on unexpected JSON fields. - - `infer`: Unexpected JSON fields are type-inferred and included in the output. +:::warning +The Document File Type Format is currently an experimental feature and not subject to SLAs. Use at your own risk. +::: -Set to `infer` by default. +The Document File Type Format is a special format that allows you to extract text from Markdown, TXT, PDF, Word and Powerpoint documents. If selected, the connector will extract text from the documents and output it as a single field named `content`. The `document_key` field will hold a unique identifier for the processed file which can be used as a primary key. The content of the document will contain markdown formatting converted from the original file format. Each file matching the defined glob pattern needs to either be a markdown (`md`), PDF (`pdf`), Word (`docx`) or Powerpoint (`.pptx`) file. -- **Block Size**: This sets the number of bytes to process in memory at a time while reading files. The default value of `10000` is usually suitable, but if your files are particularly wide (lots of columns or the values in the columns are particularly large), increasing this might help avoid schema detection failures. +One record will be emitted for each document. Keep in mind that large files can emit large records that might not fit into every destination as each destination has different limitations for string fields. -:::caution -Be cautious when raising this value too high, as it may result in Out Of Memory issues due to increased memory usage. -::: +To perform the text extraction from PDF and Docx files, the connector uses the [Unstructured](https://pypi.org/project/unstructured/) Python library. + ## Changelog | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------| +| 4.4.0 | 2023-01-12 | [33818](https://github.com/airbytehq/airbyte/pull/33818) | Add IAM Role Authentication | +| 4.3.1 | 2024-01-04 | [33937](https://github.com/airbytehq/airbyte/pull/33937) | Prepare for airbyte-lib | +| 4.3.0 | 2023-12-14 | [33411](https://github.com/airbytehq/airbyte/pull/33411) | Bump CDK version to auto-set primary key for document file streams and support raw txt files | +| 4.2.4 | 2023-12-06 | [33187](https://github.com/airbytehq/airbyte/pull/33187) | Bump CDK version to hide source-defined primary key | +| 4.2.3 | 2023-11-16 | [32608](https://github.com/airbytehq/airbyte/pull/32608) | Improve document file type parser | +| 4.2.2 | 2023-11-20 | [32677](https://github.com/airbytehq/airbyte/pull/32677) | Only read files with ".zip" extension as zipped files | +| 4.2.1 | 2023-11-13 | [32357](https://github.com/airbytehq/airbyte/pull/32357) | Improve spec schema | +| 4.2.0 | 2023-11-02 | [32109](https://github.com/airbytehq/airbyte/pull/32109) | Fix docs; add HTTPS validation for S3 endpoint; fix coverage | +| 4.1.4 | 2023-10-30 | [31904](https://github.com/airbytehq/airbyte/pull/31904) | Update CDK | +| 4.1.3 | 2023-10-25 | [31654](https://github.com/airbytehq/airbyte/pull/31654) | Reduce image size | +| 4.1.2 | 2023-10-23 | [31383](https://github.com/airbytehq/airbyte/pull/31383) | Add handling NoSuchBucket error | +| 4.1.1 | 2023-10-19 | [31601](https://github.com/airbytehq/airbyte/pull/31601) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 4.1.0 | 2023-10-17 | [31340](https://github.com/airbytehq/airbyte/pull/31340) | Add reading files inside zip archive | +| 4.0.5 | 2023-10-16 | [31209](https://github.com/airbytehq/airbyte/pull/31209) | Add experimental Markdown/PDF/Docx file format | +| 4.0.4 | 2023-09-18 | [30476](https://github.com/airbytehq/airbyte/pull/30476) | Remove streams.*.file_type from source-s3 configuration | +| 4.0.3 | 2023-09-13 | [30387](https://github.com/airbytehq/airbyte/pull/30387) | Bump Airbyte-CDK version to improve messages for record parse errors | +| 4.0.2 | 2023-09-07 | [28639](https://github.com/airbytehq/airbyte/pull/28639) | Always show S3 Key fields | +| 4.0.1 | 2023-09-06 | [30217](https://github.com/airbytehq/airbyte/pull/30217) | Migrate inference error to config errors and avoir sentry alerts | +| 4.0.0 | 2023-09-05 | [29757](https://github.com/airbytehq/airbyte/pull/29757) | New version using file-based CDK | +| 3.1.11 | 2023-08-30 | [29986](https://github.com/airbytehq/airbyte/pull/29986) | Add config error for conversion error | +| 3.1.10 | 2023-08-29 | [29943](https://github.com/airbytehq/airbyte/pull/29943) | Add config error for arrow invalid error | | 3.1.9 | 2023-08-23 | [29753](https://github.com/airbytehq/airbyte/pull/29753) | Feature parity update for V4 release | | 3.1.8 | 2023-08-17 | [29520](https://github.com/airbytehq/airbyte/pull/29520) | Update legacy state and error handling | | 3.1.7 | 2023-08-17 | [29505](https://github.com/airbytehq/airbyte/pull/29505) | v4 StreamReader and Cursor fixes | diff --git a/docs/integrations/sources/salesforce.inapp.md b/docs/integrations/sources/salesforce.inapp.md deleted file mode 100644 index d60a552e465a..000000000000 --- a/docs/integrations/sources/salesforce.inapp.md +++ /dev/null @@ -1,88 +0,0 @@ -## Prerequisites - -- [Salesforce Account](https://login.salesforce.com/) with Enterprise access or API quota purchased -- (Optional, Recommended) Dedicated Salesforce [user](https://help.salesforce.com/s/articleView?id=adding_new_users.htm&type=5&language=en_US) - -- (For Airbyte Open Source) Salesforce [OAuth](https://help.salesforce.com/s/articleView?id=sf.remoteaccess_oauth_tokens_scopes.htm&type=5) credentials - - -## Setup guide - -### Step 1: (Optional, Recommended) Create a read-only Salesforce user - -While you can set up the Salesforce connector using any Salesforce user with read permission, we recommend creating a dedicated read-only user for Airbyte. This allows you to granularly control the data Airbyte can read. - -To create a dedicated read only Salesforce user: - -1. [Log in to Salesforce](https://login.salesforce.com/) with an admin account. -2. On the top right of the screen, click the gear icon and then click **Setup**. -3. In the left navigation bar, under Administration, click **Users** > **Profiles**. The Profiles page is displayed. Click **New profile**. -4. For Existing Profile, select **Read only**. For Profile Name, enter **Airbyte Read Only User**. -5. Click **Save**. The Profiles page is displayed. Click **Edit**. -6. Scroll down to the **Standard Object Permissions** and **Custom Object Permissions** and enable the **Read** checkbox for objects that you want to replicate via Airbyte. -7. Scroll to the top and click **Save**. -8. On the left side, under Administration, click **Users** > **Users**. The All Users page is displayed. Click **New User**. -9. Fill out the required fields: - 1. For License, select **Salesforce**. - 2. For Profile, select **Airbyte Read Only User**. - 3. For Email, make sure to use an email address that you can access. -10. Click **Save**. -11. Copy the Username and keep it accessible. -12. Log into the email you used above and verify your new Salesforce account user. You'll need to set a password as part of this process. Keep this password accessible. - - - -### For Airbyte Open Source only: Obtain Salesforce OAuth credentials - -If you are using Airbyte Open Source, you will need to obtain the following OAuth credentials to authenticate: - -- Client ID -- Client Secret -- Refresh Token - -To obtain these credentials, follow [this walkthrough](https://medium.com/@bpmmendis94/obtain-access-refresh-tokens-from-salesforce-rest-api-a324fe4ccd9b) with the following modifications: - - 1. If your Salesforce URL is not in the `X.salesforce.com` format, use your Salesforce domain name. For example, if your Salesforce URL is `awesomecompany.force.com` then use that instead of `awesomecompany.salesforce.com`. - 2. When running a curl command, run it with the `-L` option to follow any redirects. - 3. If you [created a read-only user](https://docs.google.com/document/d/1wZR8pz4MRdc2zUculc9IqoF8JxN87U40IqVnTtcqdrI/edit#heading=h.w5v6h7b2a9y4), use the user credentials when logging in to generate OAuth tokens. - - - -### Step 2: Set up the Salesforce connector in Airbyte - -1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account, or navigate to your Airbyte Open Source dashboard. -2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. -3. Find and select **Salesforce** from the list of available sources. -4. Enter a **Source name** of your choosing to help you identify this source. -5. To authenticate: - -**For Airbyte Cloud**: Click **Authenticate your account** to authorize your Salesforce account. Airbyte will authenticate the Salesforce account you are already logged in to. Please make sure you are logged into the right account. - - -**For Airbyte Open Source**: Enter your Client ID, Client Secret, and Refresh Token. - -6. Toggle whether your Salesforce account is a [Sandbox account](https://help.salesforce.com/s/articleView?id=sf.deploy_sandboxes_parent.htm&type=5) or a production account. -7. (Optional) For **Start Date**, use the provided datepicker or enter the date programmatically in either `YYYY-MM-DD` or `YYYY-MM-DDTHH:MM:SSZ` format. The data added on and after this date will be replicated. If this field is left blank, Airbyte will replicate the data for the last two years by default. Please note that timestamps are in [UTC](https://www.utctime.net/). -8. (Optional) In the **Filter Salesforce Object** section, you may choose to target specific data for replication. To do so, click **Add**, then select the relevant criteria from the **Search criteria** dropdown. For **Search value**, add the search terms relevant to you. You may add multiple filters. If no filters are specified, Airbyte will replicate all data. -9. Click **Set up source** and wait for the tests to complete. - -### Supported Objects - -The Salesforce connector supports reading both Standard Objects and Custom Objects from Salesforce. Each object is read as a separate stream. See a list of all Salesforce Standard Objects [here](https://developer.salesforce.com/docs/atlas.en-us.object_reference.meta/object_reference/sforce_api_objects_list.htm). - -Airbyte fetches and handles all the possible and available streams dynamically based on: - -* If the authenticated Salesforce user has the Role and Permissions to read and fetch objects - -* If the object has the queryable property set to true. Airbyte can fetch only queryable streams via the API. If you don’t see your object available via Airbyte, check if it is API-accessible to the Salesforce user you authenticated with. - -### Incremental Deletes - -The Salesforce connector retrieves deleted records from Salesforce. For the streams which support it, a deleted record will be marked with the `isDeleted=true` value in the respective field. - -### Syncing Formula Fields - -The Salesforce connector syncs formula field outputs from Salesforce. If the formula of a field changes in Salesforce and no other field on the record is updated, you will need to reset the stream to pull in all the updated values of the field. - -For detailed information on supported sync modes, supported streams and performance considerations, refer to the -[full documentation for Salesforce](https://docs.airbyte.com/integrations/sources/google-analytics-v4). diff --git a/docs/integrations/sources/salesforce.md b/docs/integrations/sources/salesforce.md index ca9b189c2107..282fc09a901c 100644 --- a/docs/integrations/sources/salesforce.md +++ b/docs/integrations/sources/salesforce.md @@ -1,6 +1,10 @@ # Salesforce -This page contains the setup guide and reference information for the Salesforce source connector. + + +This page contains the setup guide and reference information for the [Salesforce](https://www.salesforce.com/) source connector. + + ## Prerequisites @@ -43,7 +47,7 @@ To create a dedicated read only Salesforce user: -### For Airbyte Open Source only: Obtain Salesforce OAuth credentials +### For Airbyte Open Source: Obtain Salesforce OAuth credentials If you are using Airbyte Open Source, you will need to obtain the following OAuth credentials to authenticate: @@ -61,39 +65,52 @@ To obtain these credentials, follow [this walkthrough](https://medium.com/@bpmme ### Step 2: Set up the Salesforce connector in Airbyte -1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account, or navigate to your Airbyte Open Source dashboard. + + +**For Airbyte Cloud:** + +1. [Log in to your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. 3. Find and select **Salesforce** from the list of available sources. 4. Enter a **Source name** of your choosing to help you identify this source. 5. To authenticate: - **For Airbyte Cloud**: Click **Authenticate your account** to authorize your Salesforce account. Airbyte will authenticate the Salesforce account you are already logged in to. Please make sure you are logged into the right account. - - +6. Toggle whether your Salesforce account is a [Sandbox account](https://help.salesforce.com/s/articleView?id=sf.deploy_sandboxes_parent.htm&type=5) or a production account. +7. (Optional) For **Start Date**, use the provided datepicker or enter the date programmatically in either `YYYY-MM-DD` or `YYYY-MM-DDTHH:MM:SSZ` format. The data added on and after this date will be replicated. If this field is left blank, Airbyte will replicate the data for the last two years by default. Please note that timestamps are in [UTC](https://www.utctime.net/). +8. (Optional) In the **Filter Salesforce Object** section, you may choose to target specific data for replication. To do so, click **Add**, then select the relevant criteria from the **Search criteria** dropdown. For **Search value**, add the search terms relevant to you. You may add multiple filters. If no filters are specified, Airbyte will replicate all data. +9. Click **Set up source** and wait for the tests to complete. + + + + + +**For Airbyte Open Source:** + +1. Navigate to your Airbyte Open Source dashboard. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. +3. Find and select **Salesforce** from the list of available sources. +4. Enter a **Source name** of your choosing to help you identify this source. +5. To authenticate: **For Airbyte Open Source**: Enter your Client ID, Client Secret, and Refresh Token. - 6. Toggle whether your Salesforce account is a [Sandbox account](https://help.salesforce.com/s/articleView?id=sf.deploy_sandboxes_parent.htm&type=5) or a production account. 7. (Optional) For **Start Date**, use the provided datepicker or enter the date programmatically in either `YYYY-MM-DD` or `YYYY-MM-DDTHH:MM:SSZ` format. The data added on and after this date will be replicated. If this field is left blank, Airbyte will replicate the data for the last two years by default. Please note that timestamps are in [UTC](https://www.utctime.net/). 8. (Optional) In the **Filter Salesforce Object** section, you may choose to target specific data for replication. To do so, click **Add**, then select the relevant criteria from the **Search criteria** dropdown. For **Search value**, add the search terms relevant to you. You may add multiple filters. If no filters are specified, Airbyte will replicate all data. 9. Click **Set up source** and wait for the tests to complete. + + + + ## Supported sync modes The Salesforce source connector supports the following sync modes: +- (Recommended)[ Incremental Sync - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped) - [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite/) - [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append) - [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) - (Recommended)[ Incremental Sync - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped) -### Incremental Deletes sync - -The Salesforce connector retrieves deleted records from Salesforce. For the streams which support it, a deleted record will be marked with `isDeleted=true`. - -## Performance considerations - -The Salesforce connector is restricted by Salesforce’s [Daily Rate Limits](https://developer.salesforce.com/docs/atlas.en-us.salesforce_app_limits_cheatsheet.meta/salesforce_app_limits_cheatsheet/salesforce_app_limits_platform_api.htm). The connector syncs data until it hits the daily rate limit, then ends the sync early with success status, and starts the next sync from where it left off. Note that picking up from where it ends will work only for incremental sync, which is why we recommend using the [Incremental Sync - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped) sync mode. - ## Supported Objects The Salesforce connector supports reading both Standard Objects and Custom Objects from Salesforce. Each object is read as a separate stream. See a list of all Salesforce Standard Objects [here](https://developer.salesforce.com/docs/atlas.en-us.object_reference.meta/object_reference/sforce_api_objects_list.htm). @@ -103,12 +120,35 @@ Airbyte allows exporting all available Salesforce objects dynamically based on: - If the authenticated Salesforce user has the Role and Permissions to read and fetch objects - If the salesforce object has the queryable property set to true. Airbyte can only fetch objects which are queryable. If you don’t see an object available via Airbyte, and it is queryable, check if it is API-accessible to the Salesforce user you authenticated with. -### A note on the BULK API vs REST API and their limitations + +## Limitations & Troubleshooting + +
      + +Expand to see details about Salesforce connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting + +The Salesforce connector is restricted by Salesforce’s [Daily Rate Limits](https://developer.salesforce.com/docs/atlas.en-us.salesforce_app_limits_cheatsheet.meta/salesforce_app_limits_cheatsheet/salesforce_app_limits_platform_api.htm). The connector syncs data until it hits the daily rate limit, then ends the sync early with success status, and starts the next sync from where it left off. Note that picking up from where it ends will work only for incremental sync, which is why we recommend using the [Incremental Sync - Append + Deduped](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append-deduped) sync mode. + +#### A note on the BULK API vs REST API and their limitations +## Syncing Formula Fields + +The Salesforce connector syncs formula field outputs from Salesforce. If the formula of a field changes in Salesforce and no other field on the record is updated, you will need to reset the stream and sync a historical backfill to pull in all the updated values of the field. + +## Syncing Deletes + +The Salesforce connector supports retrieving deleted records from the Salesforce recycle bin. For the streams which support it, a deleted record will be marked with `isDeleted=true`. To find out more about how Salesforce manages records in the recycle bin, please visit their [docs](https://help.salesforce.com/s/articleView?id=sf.home_delete.htm&type=5). + +## Usage of the BULK API vs REST API Salesforce allows extracting data using either the [BULK API](https://developer.salesforce.com/docs/atlas.en-us.236.0.api_asynch.meta/api_asynch/asynch_api_intro.htm) or [REST API](https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/intro_what_is_rest_api.htm). To achieve fast performance, Salesforce recommends using the BULK API for extracting larger amounts of data (more than 2,000 records). For this reason, the Salesforce connector uses the BULK API by default to extract any Salesforce objects, unless any of the following conditions are met: - The Salesforce object has columns which are unsupported by the BULK API, like columns with a `base64` or `complexvalue` type -- The Salesforce object is not supported by BULK API. In this case we sync the objects via the REST API which will occasionalyl cost more of your API quota. This list of objects was obtained experimentally, and includes the following objects: +- The Salesforce object is not supported by BULK API. In this case we sync the objects via the REST API which will occasionally cost more of your API quota. This includes the following objects: - AcceptedEventRelation - Attachment - CaseStatus @@ -133,23 +173,31 @@ Salesforce allows extracting data using either the [BULK API](https://developer. More information on the differences between various Salesforce APIs can be found [here](https://help.salesforce.com/s/articleView?id=sf.integrate_what_is_api.htm&type=5). :::info Force Using Bulk API - If you set the `Force Use Bulk API` option to `true`, the connector will ignore unsupported properties and sync Stream using BULK API. - ::: +### Troubleshooting -## Tutorials +#### Tutorials Now that you have set up the Salesforce source connector, check out the following Salesforce tutorials: - [Replicate Salesforce data to BigQuery](https://airbyte.com/tutorials/replicate-salesforce-data-to-bigquery) - [Replicate Salesforce and Zendesk data to Keen for unified analytics](https://airbyte.com/tutorials/salesforce-zendesk-analytics) +* Check out common troubleshooting issues for the Salesforce source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +
      + ## Changelog | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------| +| 2.2.2 | 2024-01-04 | [33936](https://github.com/airbytehq/airbyte/pull/33936) | Prepare for airbyte-lib | +| 2.2.1 | 2023-12-12 | [33342](https://github.com/airbytehq/airbyte/pull/33342) | Added new ContentDocumentLink stream | +| 2.2.0 | 2023-12-12 | [33350](https://github.com/airbytehq/airbyte/pull/33350) | Sync streams concurrently on full refresh | +| 2.1.6 | 2023-11-28 | [32535](https://github.com/airbytehq/airbyte/pull/32535) | Run full refresh syncs concurrently | +| 2.1.5 | 2023-10-18 | [31543](https://github.com/airbytehq/airbyte/pull/31543) | Base image migration: remove Dockerfile and use the python-connector-base image | | 2.1.4 | 2023-08-17 | [29538](https://github.com/airbytehq/airbyte/pull/29538) | Fix encoding guess | | 2.1.3 | 2023-08-17 | [29500](https://github.com/airbytehq/airbyte/pull/29500) | handle expired refresh token error | | 2.1.2 | 2023-08-10 | [28781](https://github.com/airbytehq/airbyte/pull/28781) | Fix pagination for BULK API jobs; Add option to force use BULK API | @@ -225,3 +273,5 @@ Now that you have set up the Salesforce source connector, check out the followin | 0.1.2 | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438) | Annotate Oauth2 flow initialization parameters in connector specification | | 0.1.1 | 2021-09-21 | [6209](https://github.com/airbytehq/airbyte/pull/6209) | Fix bug with pagination for BULK API | | 0.1.0 | 2021-09-08 | [5619](https://github.com/airbytehq/airbyte/pull/5619) | Salesforce Aitbyte-Native Connector | + +
      diff --git a/docs/integrations/sources/sap-fieldglass.md b/docs/integrations/sources/sap-fieldglass.md index 5e84d464e15a..a1e94cc498f0 100644 --- a/docs/integrations/sources/sap-fieldglass.md +++ b/docs/integrations/sources/sap-fieldglass.md @@ -1,4 +1,4 @@ -## SAP Fieldglass Active Worker Download Source +# SAP Fieldglass This page contains the setup guide and reference information for the SAP Fieldglass Active Worker Download source connector built in `low-code cdk`. diff --git a/docs/integrations/sources/sendgrid.inapp.md b/docs/integrations/sources/sendgrid.inapp.md deleted file mode 100644 index 4c03aca5c0a0..000000000000 --- a/docs/integrations/sources/sendgrid.inapp.md +++ /dev/null @@ -1,23 +0,0 @@ -## Prerequisites - -* [Sendgrid API Key]((https://docs.sendgrid.com/ui/account-and-settings/api-keys#creating-an-api-key)) with - * Read-only access to all resources - * Full access to marketing resources - -## Setup guide - -1. Enter a name for your Sendgridconnector. -2. Enter your `api key`. -3. (Optional) Enter the `start_time` in YYYY-MM-DDTHH:MM:SSZ format. Dataadded on and after this date will be replicated. If this field is blank, Airbyte will replicate all data. -4. Click **Set up source**. - -### (Optional) Create a read-only API key - -While you can set up the Sendgrid connector using any Salesforce user with read permission, we recommend creating a dedicated read-only user for Airbyte. This allows you to granularly control the which resources Airbyte can read. - -The API key should be read-only on all resources except Marketing, where it needs Full Access. - -Sendgrid provides two different kinds of marketing campaigns, "legacy marketing campaigns" and "new marketing campaigns". **Legacy marketing campaigns are not supported by this source connector**. -If you are seeing a `403 FORBIDDEN error message for https://api.sendgrid.com/v3/marketing/campaigns`, it may be because your SendGrid account uses legacy marketing campaigns. - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Sendgrid](https://docs.airbyte.com/integrations/sources/sendgrid). \ No newline at end of file diff --git a/docs/integrations/sources/sendgrid.md b/docs/integrations/sources/sendgrid.md index d3080175940c..78bca8f45dcf 100644 --- a/docs/integrations/sources/sendgrid.md +++ b/docs/integrations/sources/sendgrid.md @@ -1,10 +1,14 @@ # Sendgrid -This page contains the setup guide and reference information for the Sendgrid source connector. + + +This page contains the setup guide and reference information for the [Sendgrid](https://sendgrid.com/) source connector. + + ## Prerequisites -* API Key +* [Sendgrid API Key](https://docs.sendgrid.com/ui/account-and-settings/api-keys#creating-an-api-key) ## Setup guide ### Step 1: Set up Sendgrid @@ -14,24 +18,16 @@ This page contains the setup guide and reference information for the Sendgrid so * Read-only access to all resources * Full access to marketing resources -## Step 2: Set up the Sendgrid connector in Airbyte - -### For Airbyte Cloud: +### Step 2: Set up the Sendgrid connector in Airbyte -1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. -2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**. +1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account or navigate to the Airbyte Open Source dashboard. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. 3. On the Set up the source page, enter the name for the Sendgrid connector and select **Sendgrid** from the Source type dropdown. 4. Enter your `apikey`. -5. Enter your `start_time`. +5. Enter your `start_time`. 6. Click **Set up source**. -### For Airbyte OSS: - -1. Navigate to the Airbyte Open Source dashboard. -2. Set the name for your source. -3. Enter your `apikey`. -4. Enter your `start_time`. -5. Click **Set up source**. + ## Supported sync modes @@ -43,37 +39,53 @@ The Sendgrid source connector supports the following [sync modes](https://docs.a ## Supported Streams -* [Campaigns](https://docs.sendgrid.com/api-reference/campaigns-api/retrieve-all-campaigns) -* [Lists](https://docs.sendgrid.com/api-reference/lists/get-all-lists) -* [Contacts](https://docs.sendgrid.com/api-reference/contacts/export-contacts) -* [Stats automations](https://docs.sendgrid.com/api-reference/marketing-campaign-stats/get-all-automation-stats) -* [Segments](https://docs.sendgrid.com/api-reference/segmenting-contacts/get-list-of-segments) -* [Single Sends](https://docs.sendgrid.com/api-reference/marketing-campaign-stats/get-all-single-sends-stats) -* [Templates](https://docs.sendgrid.com/api-reference/transactional-templates/retrieve-paged-transactional-templates) +* [Campaigns](https://docs.sendgrid.com/api-reference/campaigns-api/retrieve-all-campaigns) +* [Lists](https://docs.sendgrid.com/api-reference/lists/get-all-lists) +* [Contacts](https://docs.sendgrid.com/api-reference/contacts/export-contacts) +* [Stats automations](https://docs.sendgrid.com/api-reference/marketing-campaign-stats/get-all-automation-stats) +* [Segments](https://docs.sendgrid.com/api-reference/segmenting-contacts/get-list-of-segments) +* [Single Sends](https://docs.sendgrid.com/api-reference/marketing-campaign-stats/get-all-single-sends-stats) +* [Templates](https://docs.sendgrid.com/api-reference/transactional-templates/retrieve-paged-transactional-templates) * [Global suppression](https://docs.sendgrid.com/api-reference/suppressions-global-suppressions/retrieve-all-global-suppressions) \(Incremental\) * [Suppression groups](https://docs.sendgrid.com/api-reference/suppressions-unsubscribe-groups/retrieve-all-suppression-groups-associated-with-the-user) -* [Suppression group members](https://docs.sendgrid.com/api-reference/suppressions-suppressions/retrieve-all-suppressions) +* [Suppression group members](https://docs.sendgrid.com/api-reference/suppressions-suppressions/retrieve-all-suppressions) * [Blocks](https://docs.sendgrid.com/api-reference/blocks-api/retrieve-all-blocks) \(Incremental\) * [Bounces](https://docs.sendgrid.com/api-reference/bounces-api/retrieve-all-bounces) \(Incremental\) * [Invalid emails](https://docs.sendgrid.com/api-reference/invalid-e-mails-api/retrieve-all-invalid-emails) \(Incremental\) * [Spam reports](https://docs.sendgrid.com/api-reference/spam-reports-api/retrieve-all-spam-reports) -## Connector-specific features & highlights, if any +## Create a read-only API key (Optional) + +While you can set up the Sendgrid connector using any Salesforce user with read permission, we recommend creating a dedicated read-only user for Airbyte. This allows you to granularly control the which resources Airbyte can read. + +The API key should be read-only on all resources except Marketing, where it needs Full Access. -We recommend creating a key specifically for Airbyte access. This will allow you to control which resources Airbyte should be able to access. The API key should be read-only on all resources except Marketing, where it needs Full Access. -Sendgrid provides two different kinds of marketing campaigns, "legacy marketing campaigns" and "new marketing campaigns". **Legacy marketing campaigns are not supported by this source connector**. -If you are seeing a `403 FORBIDDEN error message for https://api.sendgrid.com/v3/marketing/campaigns`, it might be because your SendGrid account uses legacy marketing campaigns. +## Limitations & Troubleshooting -## Performance considerations +
      + +Expand to see details about Sendgrid connector limitations and troubleshooting. + -The connector is restricted by normal Sendgrid [requests limitation](https://sendgrid.com/docs/API_Reference/Web_API_v3/How_To_Use_The_Web_API_v3/rate_limits.html). +### Connector limitations + +#### Rate limiting + +The connector is restricted by normal Sendgrid [requests limitation](https://docs.sendgrid.com/api-reference/how-to-use-the-sendgrid-v3-api/rate-limits). + +### Troubleshooting +* **Legacy marketing campaigns are not supported by this source connector**. Sendgrid provides two different kinds of marketing campaigns, "legacy marketing campaigns" and "new marketing campaigns". If you are seeing a `403 FORBIDDEN error message for https://api.sendgrid.com/v3/marketing/campaigns`, it might be because your SendGrid account uses legacy marketing campaigns. +* Check out common troubleshooting issues for the Sendgrid source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +
      ## Changelog | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| 0.4.0 | 2023-05-19 | [23959](https://github.com/airbytehq/airbyte/pull/23959) | Add `unsubscribe_groups`stream +| 0.4.1 | 2023-10-18 | [31543](https://github.com/airbytehq/airbyte/pull/31543) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 0.4.0 | 2023-05-19 | [23959](https://github.com/airbytehq/airbyte/pull/23959) | Add `unsubscribe_groups`stream | 0.3.1 | 2023-01-27 | [21939](https://github.com/airbytehq/airbyte/pull/21939) | Fix contacts missing records; Remove Messages stream | | 0.3.0 | 2023-01-25 | [21587](https://github.com/airbytehq/airbyte/pull/21587) | Make sure spec works as expected in UI - make start_time parameter an ISO string instead of an integer interpreted as timestamp (breaking, update your existing connections and set the start_time parameter to ISO 8601 date time string in UTC) | | 0.2.16 | 2022-11-02 | [18847](https://github.com/airbytehq/airbyte/pull/18847) | Skip the stream on `400, 401 - authorization required` with log message | @@ -87,3 +99,5 @@ The connector is restricted by normal Sendgrid [requests limitation](https://sen | 0.2.8 | 2022-06-07 | [13571](https://github.com/airbytehq/airbyte/pull/13571) | Add Message stream | | 0.2.7 | 2021-09-08 | [5910](https://github.com/airbytehq/airbyte/pull/5910) | Add Single Sends Stats stream | | 0.2.6 | 2021-07-19 | [4839](https://github.com/airbytehq/airbyte/pull/4839) | Gracefully handle malformed responses from the API | + +
      \ No newline at end of file diff --git a/docs/integrations/sources/sendinblue.md b/docs/integrations/sources/sendinblue.md index 62d6be403eaa..0e56d9bbc2c2 100644 --- a/docs/integrations/sources/sendinblue.md +++ b/docs/integrations/sources/sendinblue.md @@ -2,20 +2,20 @@ ## Sync overview -This source can sync data from the [Sendinblue API](https://developers.sendinblue.com/). At present this connector only supports full refresh syncs meaning that each time you use the connector it will sync all available records from scratch. Please use cautiously if you expect your API to have a lot of records. +This source can sync data from the [Sendinblue API](https://developers.sendinblue.com/). ## This Source Supports the Following Streams -* contacts -* campaigns -* templates +* [contacts](https://developers.brevo.com/reference/getcontacts-1) *(Incremental Sync)* +* [campaigns](https://developers.brevo.com/reference/getemailcampaigns-1) +* [templates](https://developers.brevo.com/reference/getsmtptemplates) ### Features | Feature | Supported?\(Yes/No\) | Notes | | :--- | :--- | :--- | | Full Refresh Sync | Yes | | -| Incremental Sync | No | | +| Incremental Sync | Yes | | ### Performance considerations @@ -31,4 +31,5 @@ Sendinblue APIs are under rate limits for the number of API calls allowed per AP | Version | Date | Pull Request | Subject | | :------ | :--------- | :-------------------------------------------------------- | :----------------------------------------- | -| 0.1.0 | 2022-11-01 | [#18771](https://github.com/airbytehq/airbyte/pull/18771) | 🎉 New Source: Sendinblue API [low-code CDK] | \ No newline at end of file +| 0.1.1 | 2022-08-31 | [#30022](https://github.com/airbytehq/airbyte/pull/30022) | ✨ Source SendInBlue: Add incremental sync to contacts stream | +| 0.1.0 | 2022-11-01 | [#18771](https://github.com/airbytehq/airbyte/pull/18771) | 🎉 New Source: Sendinblue API [low-code CDK] | diff --git a/docs/integrations/sources/sentry.md b/docs/integrations/sources/sentry.md index 3e06271bebc8..0e3c1525704d 100644 --- a/docs/integrations/sources/sentry.md +++ b/docs/integrations/sources/sentry.md @@ -29,7 +29,7 @@ The Sentry source connector supports the following [sync modes](https://docs.air ## Supported Streams -- [Events](https://docs.sentry.io/api/events/list-a-projects-events/) +- [Events](https://docs.sentry.io/api/events/list-a-projects-error-events/) - [Issues](https://docs.sentry.io/api/events/list-a-projects-issues/) - [Projects](https://docs.sentry.io/api/projects/list-your-projects/) - [Releases](https://docs.sentry.io/api/releases/list-an-organizations-releases/) @@ -47,6 +47,8 @@ The Sentry source connector supports the following [sync modes](https://docs.air | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------| +| 0.4.0 | 2024-01-05 | [32957](https://github.com/airbytehq/airbyte/pull/32957) | Added undeclared fields to schema, Base image migration: remove Dockerfile and use the python-connector-base image | +| 0.3.0 | 2023-09-05 | [30192](https://github.com/airbytehq/airbyte/pull/30192) | Added undeclared fields to schema | | 0.2.4 | 2023-08-14 | [29401](https://github.com/airbytehq/airbyte/pull/29401) | Fix `null` value in stream state | | 0.2.3 | 2023-08-03 | [29023](https://github.com/airbytehq/airbyte/pull/29023) | Add incremental for `issues` stream | | 0.2.2 | 2023-05-02 | [25759](https://github.com/airbytehq/airbyte/pull/25759) | Change stream that used in check_connection | @@ -64,4 +66,4 @@ The Sentry source connector supports the following [sync modes](https://docs.air | 0.1.3 | 2022-08-17 | [15734](https://github.com/airbytehq/airbyte/pull/15734) | Fix yaml based on the new schema validator | | 0.1.2 | 2021-12-28 | [15345](https://github.com/airbytehq/airbyte/pull/15345) | Migrate to config-based framework | | 0.1.1 | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628) | Update fields in source-connectors specifications | -| 0.1.0 | 2021-10-12 | [6975](https://github.com/airbytehq/airbyte/pull/6975) | New Source: Sentry | +| 0.1.0 | 2021-10-12 | [6975](https://github.com/airbytehq/airbyte/pull/6975) | New Source: Sentry | \ No newline at end of file diff --git a/docs/integrations/sources/sftp.md b/docs/integrations/sources/sftp.md index 8d6d84e942d9..b9aac0f59f2e 100644 --- a/docs/integrations/sources/sftp.md +++ b/docs/integrations/sources/sftp.md @@ -107,6 +107,7 @@ More formats \(e.g. Apache Avro\) will be supported in the future. ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :----------------------------------------------------- | +|:--------|:-----------|:---------------------------------------------------------| :----------------------------------------------------- | +| 0.2.0 | 2024-01-15 | [34265](https://github.com/airbytehq/airbyte/pull/34265) | Remove LEGACY state flag | | 0.1.2 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.0 | 2021-24-05 | | Initial version | diff --git a/docs/integrations/sources/shopify-migrations.md b/docs/integrations/sources/shopify-migrations.md new file mode 100644 index 000000000000..c6633fc30d9d --- /dev/null +++ b/docs/integrations/sources/shopify-migrations.md @@ -0,0 +1,49 @@ +# Shopify Migration Guide + +## Upgrading to 1.2.0 +This version implements `Shopify GraphQL BULK Operations` to speed up the following streams: + - `Metafield Collections` + - `Metafield Customers ` + - `Metafield Draft_orders` + - `Metafield Locations` + - `Metafield Orders` + - `Metafield Product Images` + - `Metafield Product Variants` + - `Collections` + - `Discount Codes` + - `Fulfillment Orders` + - `Inventory Items` + - `Inventory Levels` + - `Transactions Graphql` (duplicated `Transactions` stream to provide faster fetch) + +Increased the performance for the following streams: + - `Product Images` + - `Product Variants` + - `Order Refunds` + - `Fulfillments` + - `Customer Address` + +Other bug fixes and improvements, more info: `https://github.com/airbytehq/airbyte/pull/32345` + +### Action items required for 1.2.0 +* The `Fulfillments` stream now has the cursor field `updated_at`, instead of the `id`. +* The `Order Refunds` stream, now has the schema `refund_line_items.line_item.properties` to array of `strings`, instead of `object` with properties. +* The `Fulfillment Orders` stream now has the `supported_actions` schema as `array of objects` instead of `array of strings`. + + - if `API_PASSWORD` is used for authentication: + - BEFORE UPDATING to the `1.2.0`: update your `Private Developer Application` scopes with `read_publications` and save the changes, in your Shopify Account. + - if `OAuth2.0` is used for authentication: + - `re-auth` in order to obtain new scope automatically, after the upgrade. + - `Refresh Schema` + `Reset` is required for these streams after the upgrade from previous version. + + +## Upgrading to 1.0.0 +This version uses Shopify API version `2023-07` which brings changes to the following streams: + - removed `gateway, payment_details, processing_method` properties from `Order` stream, they are no longer supplied. + - added `company, confirmation_number, current_total_additional_fees_set, original_total_additional_fees_set, tax_exempt, po_number` properties to `Orders` stream + - added `total_unsettled_set, payment_id` to `Transactions` stream + - added `return` property to `Order Refund` stream + - added `created_at, updated_at` to `Fulfillment Order` stream + +### Action items required for 1.0.0 + * The `reset` and `full-refresh` for `Orders` stream is required after upgrading to this version. diff --git a/docs/integrations/sources/shopify.inapp.md b/docs/integrations/sources/shopify.inapp.md deleted file mode 100644 index 6907ba41080b..000000000000 --- a/docs/integrations/sources/shopify.inapp.md +++ /dev/null @@ -1,65 +0,0 @@ -## Prerequisites - -* An Active [Shopify store](https://www.shopify.com) -* The Admin API access token of your [Custom App](https://help.shopify.com/en/manual/apps/app-types/custom-apps). - -:::note - -Our Shopify Source Connector does not support OAuth at this time due to limitations outside of our control. If OAuth for Shopify is critical to your business, [please reach out to us](mailto:product@airbyte.io) to discuss how we may be able to partner on this effort. - -::: - -## Setup guide - -1. Name your source. -2. Enter your Store name. You can find this in your URL when logged in to Shopify or within the Store details section of your Settings. -3. Enter your Admin API access token. To set up the access token, you will need to set up a custom application. See instructions below on creating a custom app. -4. Click Set up source - -## Creating a Custom App -Authentication to the Shopify API requies a [custom application](https://help.shopify.com/en/manual/apps/app-types/custom-apps). Follow these instructions to create a custom app and find your Admin API Access Token. - -1. Navigate to Settings > App and sales channels > Develop apps > Create an app -2. Name your new app -3. Select **Configure Admin API scopes** -4. Tick all the scopes prefixed with `read_` (e.g. `read_locations`,`read_price_rules`, etc ) and save. See below for the full list of scopes to allow. -5. Click **Install app** to give this app access to your data. -6. Once installed, go to **API Credentials** to copy the **Admin API Access Token**. - -### Scopes Required for Custom App -Add the following scopes to your custom app to ensure Airbyte can sync all available data. To see a list of streams this source supports, see our full [Shopify documentation](https://docs.airbyte.com/integrations/sources/shopify/). -* `read_analytics` -* `read_assigned_fulfillment_orders` -* `read_gdpr_data_request` -* `read_locations` -* `read_price_rules` -* `read_product_listings` -* `read_products` -* `read_reports` -* `read_resource_feedbacks` -* `read_script_tags` -* `read_shipping` -* `read_locales` -* `read_shopify_payments_accounts` -* `read_shopify_payments_bank_accounts` -* `read_shopify_payments_disputes` -* `read_shopify_payments_payouts` -* `read_content` -* `read_themes` -* `read_third_party_fulfillment_orders` -* `read_translations` -* `read_customers` -* `read_discounts` -* `read_draft_orders` -* `read_fulfillments` -* `read_gift_cards` -* `read_inventory` -* `read_legal_policies` -* `read_marketing_events` -* `read_merchant_managed_fulfillment_orders` -* `read_online_store_pages` -* `read_order_edits` -* `read_orders` - -​ -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Shopify](https://docs.airbyte.com/integrations/sources/shopify). \ No newline at end of file diff --git a/docs/integrations/sources/shopify.md b/docs/integrations/sources/shopify.md index 7f256b67ae7d..d0962388a4a5 100644 --- a/docs/integrations/sources/shopify.md +++ b/docs/integrations/sources/shopify.md @@ -1,122 +1,156 @@ ---- -description: >- - Shopify is a proprietary e-commerce platform for online stores and retail point-of-sale systems. ---- - # Shopify -:::note + -Our Shopify Source Connector does not support OAuth at this time due to limitations outside of our control. If OAuth for Shopify is critical to your business, [please reach out to us](mailto:product@airbyte.io) to discuss how we may be able to partner on this effort. +This page contains the setup guide and reference information for the [Shopify](https://www.shopify.com/) source connector. -::: + -## Getting started - -This connector supports the `API PASSWORD` (for private applications) athentication methods. - -### Connect using `API PASSWORD` option - -1. Go to `https://YOURSTORE.myshopify.com/admin/apps/private` -2. Enable private development if it isn't enabled. -3. Create a private application. -4. Select the resources you want to allow access to. Airbyte only needs read-level access. - - Note: The UI will show all possible data sources and will show errors when syncing if it doesn't have permissions to access a resource. -5. The password under the `Admin API` section is what you'll use as the `API PASSWORD` for the integration. -6. You're ready to set up Shopify in Airbyte! - -### Scopes Required for Custom App - -Add the following scopes to your custom app to ensure Airbyte can sync all available data. To see a list of streams this source supports, see our full [Shopify documentation](https://docs.airbyte.com/integrations/sources/shopify/). - -- `read_analytics` -- `read_assigned_fulfillment_orders` -- `read_gdpr_data_request` -- `read_locations` -- `read_price_rules` -- `read_product_listings` -- `read_products` -- `read_reports` -- `read_resource_feedbacks` -- `read_script_tags` -- `read_shipping` -- `read_locales` -- `read_shopify_payments_accounts` -- `read_shopify_payments_bank_accounts` -- `read_shopify_payments_disputes` -- `read_shopify_payments_payouts` -- `read_content` -- `read_themes` -- `read_third_party_fulfillment_orders` -- `read_translations` -- `read_customers` -- `read_discounts` -- `read_draft_orders` -- `read_fulfillments` -- `read_gift_cards` -- `read_inventory` -- `read_legal_policies` -- `read_marketing_events` -- `read_merchant_managed_fulfillment_orders` -- `read_online_store_pages` -- `read_order_edits` -- `read_orders` +## Prerequisites -## Supported sync modes +* An active [Shopify store](https://www.shopify.com). +* If you are syncing data from a store that you do not own, you will need to [request access to your client's store](https://help.shopify.com/en/partners/dashboard/managing-stores/request-access#request-access) (not required for account owners). + +* For **Airbyte Open Source** users: A custom Shopify application with [`read_` scopes enabled](#scopes-required-for-custom-app). + -The Shopify source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. +## Setup guide -This source can sync data for the [Shopify REST API](https://shopify.dev/api/admin-rest) and the [Shopify GraphQl API](https://shopify.dev/api/admin-graphql). +This connector supports **OAuth2.0** and **API Password** (for private applications) authentication methods. -## Troubleshooting tips + +:::note +For existing **Airbyte Cloud** customers, if you are currently using the **API Password** authentication method, please switch to **OAuth2.0**, as the API Password will be deprecated shortly. This change will not affect **Airbyte Open Source** connections. +::: -Check out common troubleshooting issues for the Shopify source connector on our Airbyte Forum [here](https://github.com/airbytehq/airbyte/discussions). +### Airbyte Cloud + +#### Connect using OAuth2.0 + +1. Select a **Source name**. +2. Click **Authenticate your Shopify account**. +3. Click **Install** to install the Airbyte application. +4. Log in to your account, if you are not already logged in. +5. Select the store you want to sync and review the consent form. Click **Install app** to finish the installation. +6. The **Shopify Store** field will be automatically filled based on the store you selected. Confirm the value is accurate. +7. (Optional) You may set a **Replication Start Date** as the starting point for your data replication. Any data created before this date will not be synced. Defaults to January 1st, 2020. +8. Click **Set up source** and wait for the connection test to complete. + + + +### Airbyte Open Source + +#### Create a custom app + +Authentication to the Shopify API requires a [custom application](https://help.shopify.com/en/manual/apps/app-types/custom-apps). Follow these instructions to create a custom app and find your Admin API Access Token. + +1. Log in to your Shopify account. +2. In the dashboard, navigate to **Settings** > **App and sales channels** > **Develop apps** > **Create an app**. +3. Select a name for your new app. +4. Select **Configure Admin API scopes**. +5. Grant access to the [following list of scopes](#scopes-required-for-custom-app). Only select scopes prefixed with `read_`, not `write_` (e.g. `read_locations`,`read_price_rules`, etc ). +6. Click **Install app** to give this app access to your data. +7. Once installed, go to **API Credentials** to copy the **Admin API Access Token**. You are now ready to set up the source in Airbyte! + +#### Connect using API Password + +1. Enter a **Source name**. +2. Enter your **Shopify Store** name. You can find this in your URL when logged in to Shopify or within the Store details section of your Settings. +3. For **API Password**, enter your custom application's Admin API access token. +4. (Optional) You may set a **Replication Start Date** as the starting point for your data replication. Any data created before this date will not be synced. Please note that this defaults to January 1st, 2020. +5. Click **Set up source** and wait for the connection test to complete. + +### Custom app scopes + +Add the following scopes to your custom app to ensure Airbyte can sync all available data. For more information on access scopes, see the [Shopify docs](https://shopify.dev/docs/api/usage/access-scopes). + +* `read_analytics` +* `read_assigned_fulfillment_orders` +* `read_content` +* `read_customers` +* `read_discounts` +* `read_draft_orders` +* `read_fulfillments` +* `read_gdpr_data_request` +* `read_gift_cards` +* `read_inventory` +* `read_legal_policies` +* `read_locations` +* `read_locales` +* `read_marketing_events` +* `read_merchant_managed_fulfillment_orders` +* `read_online_store_pages` +* `read_order_edits` +* `read_orders` +* `read_price_rules` +* `read_product_listings` +* `read_products` +* `read_publications` +* `read_reports` +* `read_resource_feedbacks` +* `read_script_tags` +* `read_shipping` +* `read_shopify_payments_accounts` +* `read_shopify_payments_bank_accounts` +* `read_shopify_payments_disputes` +* `read_shopify_payments_payouts` +* `read_themes` +* `read_third_party_fulfillment_orders` +* `read_translations` + + + + -## Supported Streams +## Supported sync modes -This Source is capable of syncing the following core Streams: +The Shopify source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. -- [Articles](https://shopify.dev/api/admin-rest/2022-01/resources/article) -- [Blogs](https://shopify.dev/api/admin-rest/2022-01/resources/blog) -- [Abandoned Checkouts](https://shopify.dev/api/admin-rest/2022-01/resources/abandoned-checkouts#top) -- [Collects](https://shopify.dev/api/admin-rest/2022-01/resources/collect#top) -- [Collections](https://shopify.dev/api/admin-rest/2022-01/resources/collection) -- [Countries](https://shopify.dev/docs/api/admin-rest/2023-04/resources/country) -- [Custom Collections](https://shopify.dev/api/admin-rest/2022-01/resources/customcollection#top) -- [CustomerAddress](https://shopify.dev/docs/api/admin-rest/2023-04/resources/customer-address) -- [CustomerSavedSearch](https://shopify.dev/docs/api/admin-rest/2023-04/resources/customersavedsearch) -- [Smart Collections](https://shopify.dev/api/admin-rest/2022-01/resources/smartcollection) -- [Customers](https://shopify.dev/api/admin-rest/2022-01/resources/customer#top) -- [Draft Orders](https://shopify.dev/api/admin-rest/2022-01/resources/draftorder#top) -- [Discount Codes](https://shopify.dev/api/admin-rest/2022-01/resources/discountcode#top) +This source can sync data for the [Shopify REST API](https://shopify.dev/api/admin-rest) and the [Shopify GraphQL API](https://shopify.dev/api/admin-graphql) and the [Shopify GraphQL BULK API](https://shopify.dev/docs/api/usage/bulk-operations/queries) + +## Supported streams + +- [Abandoned Checkouts](https://shopify.dev/api/admin-rest/2023-07/resources/abandoned-checkouts#top) +- [Articles](https://shopify.dev/api/admin-rest/2023-07/resources/article) +- [Blogs](https://shopify.dev/api/admin-rest/2023-07/resources/blog) +- [Collects](https://shopify.dev/api/admin-rest/2023-07/resources/collect#top) +- [Collections (GraphQL)](https://shopify.dev/docs/api/admin-graphql/2023-07/objects/Collection) +- [Countries](https://shopify.dev/docs/api/admin-rest/2023-07/resources/country) +- [Custom Collections](https://shopify.dev/api/admin-rest/2023-07/resources/customcollection#top) +- [Customers](https://shopify.dev/api/admin-rest/2023-07/resources/customer#top) +- [Customer Address](https://shopify.dev/docs/api/admin-rest/2023-07/resources/customer-address) +- [Customer Saved Search](https://shopify.dev/docs/api/admin-rest/2023-07/resources/customersavedsearch) +- [Draft Orders](https://shopify.dev/api/admin-rest/2023-07/resources/draftorder#top) +- [Discount Codes (GraphQL)](https://shopify.dev/docs/api/admin-graphql/2023-07/unions/DiscountCode) - [Disputes](https://shopify.dev/docs/api/admin-rest/2023-07/resources/dispute) -- [Metafields](https://shopify.dev/api/admin-rest/2022-01/resources/metafield#top) -- [Orders](https://shopify.dev/api/admin-rest/2022-01/resources/order#top) -- [Orders Refunds](https://shopify.dev/api/admin-rest/2022-01/resources/refund#top) -- [Orders Risks](https://shopify.dev/api/admin-rest/2022-01/resources/order-risk#top) -- [Products](https://shopify.dev/api/admin-rest/2022-01/resources/product#top) +- [Fulfillments](https://shopify.dev/api/admin-rest/2023-07/resources/fulfillment) +- [Fulfillment Orders (GraphQL)](https://shopify.dev/docs/api/admin-graphql/2023-07/objects/FulfillmentOrder) +- [Inventory Items (GraphQL)](https://shopify.dev/docs/api/admin-graphql/2023-07/objects/InventoryItem) +- [Inventory Levels (GraphQL)](https://shopify.dev/docs/api/admin-graphql/2023-07/objects/InventoryLevel) +- [Locations](https://shopify.dev/api/admin-rest/2023-07/resources/location) +- [Metafields (GraphQL)](https://shopify.dev/docs/api/admin-graphql/2023-07/objects/Metafield) +- [Orders](https://shopify.dev/api/admin-rest/2023-07/resources/order#top) +- [Order Refunds](https://shopify.dev/api/admin-rest/2023-07/resources/refund#top) +- [Order Risks](https://shopify.dev/api/admin-rest/2023-07/resources/order-risk#top) +- [Pages](https://shopify.dev/api/admin-rest/2023-07/resources/page#top) +- [Price Rules](https://shopify.dev/api/admin-rest/2023-07/resources/pricerule#top) +- [Products](https://shopify.dev/api/admin-rest/2023-07/resources/product#top) - [Products (GraphQL)](https://shopify.dev/api/admin-graphql/2022-10/queries/products) -- [Product Images](https://shopify.dev/api/admin-rest/2022-01/resources/product-image) -- [Product Variants](https://shopify.dev/api/admin-rest/2022-01/resources/product-variant) -- [Transactions](https://shopify.dev/api/admin-rest/2022-01/resources/transaction#top) -- [Tender Transactions](https://shopify.dev/api/admin-rest/2022-01/resources/tendertransaction) -- [Pages](https://shopify.dev/api/admin-rest/2022-01/resources/page#top) -- [Price Rules](https://shopify.dev/api/admin-rest/2022-01/resources/pricerule#top) -- [Locations](https://shopify.dev/api/admin-rest/2022-01/resources/location) -- [InventoryItems](https://shopify.dev/api/admin-rest/2022-01/resources/inventoryItem) -- [InventoryLevels](https://shopify.dev/api/admin-rest/2021-01/resources/inventorylevel) -- [Fulfillment Orders](https://shopify.dev/api/admin-rest/2022-01/resources/fulfillmentorder) -- [Fulfillments](https://shopify.dev/api/admin-rest/2022-01/resources/fulfillment) -- [Shop](https://shopify.dev/api/admin-rest/2022-01/resources/shop) +- [Product Images](https://shopify.dev/api/admin-rest/2023-07/resources/product-image) +- [Product Variants](https://shopify.dev/api/admin-rest/2023-07/resources/product-variant) +- [Shop](https://shopify.dev/api/admin-rest/2023-07/resources/shop) +- [Smart Collections](https://shopify.dev/api/admin-rest/2023-07/resources/smartcollection) +- [Transactions](https://shopify.dev/api/admin-rest/2023-07/resources/transaction#top) +- [Transactions (GraphQL)](https://shopify.dev/docs/api/admin-graphql/2023-07/objects/OrderTransaction) +- [Tender Transactions](https://shopify.dev/api/admin-rest/2023-07/resources/tendertransaction) -### Stream sync recommendations +## Capturing deleted records -For better experience with `Incremental Refresh` the following is recommended: +The connector captures deletions for records in the `Articles`, `Blogs`, `CustomCollections`, `Orders`, `Pages`, `PriceRules` and `Products` streams. -- `Order Refunds`, `Order Risks`, `Transactions` should be synced along with `Orders` stream. -- `Discount Codes` should be synced along with `Price Rules` stream. +When a record is deleted, the connector outputs a record with the `ID` of that record and the `deleted_at`, `deleted_message`, and `deleted_description` fields filled out. No other fields are filled out for the deleted records. -If child streams are synced alone from the parent stream - the full sync will take place, and the records are filtered out afterwards. +Check the following Shopify documentation for more information about [retrieving deleted records](https://shopify.dev/docs/api/admin-rest/2023-07/resources/event). ## Data type mapping @@ -136,24 +170,55 @@ If child streams are synced alone from the parent stream - the full sync will ta | Incremental - Append Sync | Yes | | Namespaces | No | -## Performance considerations +## Limitations & Troubleshooting + +
      + -Shopify has some [rate limit restrictions](https://shopify.dev/concepts/about-apis/rate-limits). Typically, there should not be issues with throttling or exceeding the rate limits but in some edge cases, user can receive the warning message as follows: +Expand to see details about Shopify connector limitations and troubleshooting + + + +### Connector limitations + +#### Rate limiting + +Shopify has some [rate limit restrictions](https://shopify.dev/concepts/about-apis/rate-limits). Typically, there should not be issues with throttling or exceeding the rate limits but, in some edge cases, you may encounter the following warning message: ```text -"Caught retryable error ' or null' after tries. Waiting seconds then retrying..." +"Caught retryable error ' or null' after tries. +Waiting seconds then retrying..." ``` -This is expected when the connector hits the 429 - Rate Limit Exceeded HTTP Error. With given error message the sync operation is still goes on, but will require more time to finish. +This is expected when the connector hits a `429 - Rate Limit Exceeded` HTTP Error. The sync operation will continue successfully after a short backoff period. + +For all `Shopify GraphQL BULK` api requests these limitations are applied: https://shopify.dev/docs/api/usage/bulk-operations/queries#operation-restrictions + + +### Troubleshooting + +* If you encounter access errors while using **OAuth2.0** authentication, please make sure you've followed this [Shopify Article](https://help.shopify.com/en/partners/dashboard/managing-stores/request-access#request-access) to request the access to the client's store first. Once the access is granted, you should be able to proceed with **OAuth2.0** authentication. +* Check out common troubleshooting issues for the Shopify source connector on our Airbyte Forum [here](https://github.com/airbytehq/airbyte/discussions). + +
      ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------ | -| 0.6.2 | 2023-08-09 | [29302](https://github.com/airbytehq/airbyte/pull/29302) | Handle the `Internal Server Error` when entity could be fetched | -| 0.6.1 | 2023-08-08 | [28291](https://github.com/airbytehq/airbyte/pull/28291) | Allow `shop` field to accept `*.myshopify.com` shop names, updated `OAuth Spec` | -| 0.6.0 | 2023-08-02 | [28770](https://github.com/airbytehq/airbyte/pull/28770) | Added `Disputes` stream | -| 0.5.1 | 2023-07-13 | [28700](https://github.com/airbytehq/airbyte/pull/28700) | Improved `error messages` with more user-friendly description, refactored code | +|:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------| +| 1.2.0 | 2024-01-09 | [32345](https://github.com/airbytehq/airbyte/pull/32345) | Fixed the issue with `state` causing the `substreams` to skip the records, made `metafield_*`: `collections, customers, draft_orders, locations, orders, product_images, product_variants, products`, and `fulfillment_orders, collections, discount_codes, inventory_levels, inventory_items, transactions_graphql` streams to use `BULK Operations` instead of `REST`| +| 1.1.6 | 2024-01-04 | [33414](https://github.com/airbytehq/airbyte/pull/33414) | Prepare for airbyte-lib | +| 1.1.5 | 2023-12-28 | [33827](https://github.com/airbytehq/airbyte/pull/33827) | Fix GraphQL query | +| 1.1.4 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 1.1.3 | 2023-10-17 | [31500](https://github.com/airbytehq/airbyte/pull/31500) | Fixed the issue caused by the `missing access token` while setup the new source and not yet authenticated | +| 1.1.2 | 2023-10-13 | [31381](https://github.com/airbytehq/airbyte/pull/31381) | Fixed the issue caused by the `state` presence while fetching the `deleted events` with pagination | +| 1.1.1 | 2023-09-18 | [30560](https://github.com/airbytehq/airbyte/pull/30560) | Performance testing - include socat binary in docker image | +| 1.1.0 | 2023-09-07 | [30246](https://github.com/airbytehq/airbyte/pull/30246) | Added ability to fetch `destroyed` records for `Articles, Blogs, CustomCollections, Orders, Pages, PriceRules, Products` | +| 1.0.0 | 2023-08-11 | [29361](https://github.com/airbytehq/airbyte/pull/29361) | Migrate to the `2023-07` Shopify API Version | +| 0.6.2 | 2023-08-09 | [29302](https://github.com/airbytehq/airbyte/pull/29302) | Handle the `Internal Server Error` when entity could be fetched | +| 0.6.1 | 2023-08-08 | [28291](https://github.com/airbytehq/airbyte/pull/28291) | Allow `shop` field to accept `*.myshopify.com` shop names, updated `OAuth Spec` | +| 0.6.0 | 2023-08-02 | [28770](https://github.com/airbytehq/airbyte/pull/28770) | Added `Disputes` stream | +| 0.5.1 | 2023-07-13 | [28700](https://github.com/airbytehq/airbyte/pull/28700) | Improved `error messages` with more user-friendly description, refactored code | | 0.5.0 | 2023-06-13 | [27732](https://github.com/airbytehq/airbyte/pull/27732) | License Update: Elv2 | | 0.4.0 | 2023-06-13 | [27083](https://github.com/airbytehq/airbyte/pull/27083) | Added `CustomerSavedSearch`, `CustomerAddress` and `Countries` streams | | 0.3.4 | 2023-05-10 | [25961](https://github.com/airbytehq/airbyte/pull/25961) | Added validation for `shop` in input configuration (accepts non-url-like inputs) | @@ -199,3 +264,5 @@ This is expected when the connector hits the 429 - Rate Limit Exceeded HTTP Erro | 0.1.5 | 2021-06-10 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Added `AIRBYTE_ENTRYPOINT` for Kubernetes support | | 0.1.4 | 2021-06-09 | [3926](https://github.com/airbytehq/airbyte/pull/3926) | New attributes to Orders schema | | 0.1.3 | 2021-06-08 | [3787](https://github.com/airbytehq/airbyte/pull/3787) | Added Native Shopify Source Connector | + +
      diff --git a/docs/integrations/sources/slack.inapp.md b/docs/integrations/sources/slack.inapp.md deleted file mode 100644 index fabdd013ab8a..000000000000 --- a/docs/integrations/sources/slack.inapp.md +++ /dev/null @@ -1,58 +0,0 @@ -## Prerequisites -- Access to Slack via OAuth or API Token (via Slack App or Legacy API Key) - - -## Setup Guide - -1. Enter a name for your connector -2. Select `Authenticate your Slack account` (preferred) and authorize into the Slack account. To use an API token instead, see the instructions below on creating one. -3. Toggle on **Join all channels** (recommended) to join all channels the user has access to or to sync data only from channels the app (if using API token) is already in. If false, you'll need to manually add all the channels from which you'd like to sync messages. -4. (Optional) Enter a **Threads Lookback Window (Days)** to set how far back to look for messages in threads from when each sync start. -5. (Optional) Enter a **Start Date**, enter the date in `YYYY-MM-DDTHH:mm:ssZ` format. Data created on and after this date will be replicated. -8. (Optional) Enter your `Channel name filter` to filter the list of channels Airbyte can access. If none are entered, Airbyte will sync all channels. It can be helpful to only sync required channels to avoid Slack's [requests limits](https://api.slack.com/docs/rate-limits). - - -9. Click **Set up source**. - -### Creating an API token - -You can no longer create "Legacy" API Keys, but if you already have one, you can use it with this source as the API key and skip setting up an application. - -In order to pull data out of your Slack instance, you need to create a Slack App. This may sound daunting, but it is actually pretty straight forward. Slack supplies [documentation](https://api.slack.com/start) on how to build apps. Feel free to follow that if you want to do something fancy. We'll describe the steps we followed to creat the Slack App for this tutorial. - -:::info -This tutorial assumes that you are an administrator on your slack instance. If you are not, you will need to coordinate with your administrator on the steps that require setting permissions for your app. -::: - -1. Go to the [apps page](https://api.slack.com/apps) -2. Click "Create New App" -3. It will request a name and the slack instance you want to create the app for. Make sure you select the instance form which you want to pull data. -4. Completing that form will take you to the "Basic Information" page for your app. -5. Now we need to grant the correct permissions to the app. \(This is the part that requires you to be an administrator\). Go to "Permissions". Then under "Bot Token Scopes" click on "Add an OAuth Scope". We will now need to add the following scopes: - - ```text - channels:history - channels:join - channels:read - files:read - groups:read - links:read - reactions:read - remote_files:read - team:read - usergroups:read - users.profile:read - users:read - ``` - - This may look daunting, but the search functionality in the dropdown should make this part go pretty quick. - -6. Scroll to the top of the page and click "Install to Workspace". This will generate a "Bot User OAuth Access Token". We will need this in a moment. -7. Now go to your slack instance. For any public channel go to info => more => add apps. In the search bar search for the name of your app. \(If using the desktop version of slack, you may need to restart Slack for it to pick up the new app\). Airbyte will only replicate messages from channels that the Slack bot has been added to. - - ![](../../.gitbook/assets/slack-add-apps.png) - -8. In Airbyte, create a Slack source. The "Bot User OAuth Access Token" from the earlier should be used as the token. -9. You can now pull data from your slack instance! - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Slack](https://docs.airbyte.com/integrations/sources/slack). diff --git a/docs/integrations/sources/slack.md b/docs/integrations/sources/slack.md index 0fbec3dd78c9..2edb9eb6fcc6 100644 --- a/docs/integrations/sources/slack.md +++ b/docs/integrations/sources/slack.md @@ -1,9 +1,15 @@ # Slack -This page contains the setup guide and reference information for the Slack source connector. + + +This page contains the setup guide and reference information for the [Slack](https://www.slack.com) source connector. + + ## Prerequisites +OAuth or API Token (via Slack App or Legacy API Key) is required for access to Slack. + You can no longer create "Legacy" API Keys, but if you already have one, you can use it with this source. Fill it into the API key section. We recommend creating a restricted, read-only key specifically for Airbyte access. This will allow you to control which resources Airbyte should be able to access. @@ -11,6 +17,7 @@ We recommend creating a restricted, read-only key specifically for Airbyte acces Note that refresh token are entirely optional for Slack and are not required to use Airbyte. You can learn more about refresh tokens [here](https://api.slack.com/authentication/rotation). ## Setup guide + ### Step 1: Set up Slack :::info @@ -58,6 +65,7 @@ This tutorial assumes that you are an administrator on your slack instance. If y 8. In Airbyte, create a Slack source. The "Bot User OAuth Access Token" from the earlier should be used as the token. 9. You can now pull data from your slack instance! + **Airbyte Open Source additional setup steps** @@ -76,10 +84,10 @@ We recommend creating a restricted, read-only key specifically for Airbyte acces 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**. 3. On the Set up the source page, enter the name for the Slack connector and select **Slack** from the Source type dropdown. 4. Select `Authenticate your account` and log in and Authorize to the Slack account. -5. Enter your `start_date`. -6. Enter your `lookback_window`. -7. Enter your `join_channels`. -8. Enter your `channel_filter`. +5. **Required** Enter your `start_date`. +6. **Required** Enter your `lookback_window`, which corresponds to amount of days in the past from which you want to sync data. +7. Toggle `join_channels`, if you want to join all channels or to sync data only from channels the bot is already in. If not set, you'll need to manually add the bot to all the channels from which you'd like to sync messages. +8. Enter your `channel_filter`, this should be list of channel names (without leading '#' char) that limits the channels from which you'd like to sync. If no channels are specified, Airbyte will replicate all data. 9. Click **Set up source**. @@ -88,14 +96,16 @@ We recommend creating a restricted, read-only key specifically for Airbyte acces 1. Navigate to the Airbyte Open Source dashboard. 2. Set the name for your source. -3. Enter your `start_date`. -4. Enter your `lookback_window`. -5. Enter your `join_channels`. -6. Enter your `channel_filter`. +3. **Required** Enter your `start_date`. +4. **Required** Enter your `lookback_window`, which corresponds to amount of days in the past from which you want to sync data. +5. Toggle `join_channels`, if you want to join all channels or to sync data only from channels the bot is already in. If not set, you'll need to manually add the bot to all the channels from which you'd like to sync messages. +6. Enter your `channel_filter`, this should be list of channel names (without leading '#' char) that limits the channels from which you'd like to sync. If no channels are specified, Airbyte will replicate all data. 7. Enter your `api_token`. 8. Click **Set up source**. + + ## Supported sync modes The Slack source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): @@ -108,14 +118,13 @@ The Slack source connector supports the following [sync modes](https://docs.airb ## Supported Streams +For most of the streams, the Slack source connector uses the [Conversations API](https://api.slack.com/docs/conversations-api) under the hood. + * [Channels \(Conversations\)](https://api.slack.com/methods/conversations.list) * [Channel Members \(Conversation Members\)](https://api.slack.com/methods/conversations.members) * [Messages \(Conversation History\)](https://api.slack.com/methods/conversations.history) It will only replicate messages from non-archive, public channels that the Slack App is a member of. * [Users](https://api.slack.com/methods/users.list) * [Threads \(Conversation Replies\)](https://api.slack.com/methods/conversations.replies) -* [User Groups](https://api.slack.com/methods/usergroups.list) -* [Files](https://api.slack.com/methods/files.list) -* [Remote Files](https://api.slack.com/methods/files.remote.list) ## Performance considerations @@ -132,28 +141,56 @@ It is recommended to sync required channels only, this can be done by specifying | `array` | `array` | | `object` | `object` | +## Limitations & Troubleshooting + +
      + +Expand to see details about Slack connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting +Slack has [rate limit restrictions](https://api.slack.com/docs/rate-limits). + +### Troubleshooting + +* Check out common troubleshooting issues for the Slack source connector on our Airbyte Forum [here](https://github.com/airbytehq/airbyte/discussions). + +
      + ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------| -| 0.2.0 | 2023-05-24 | [26497](https://github.com/airbytehq/airbyte/pull/26497) | Fixed `lookback window` value limitations | -| 0.1.26 | 2023-05-17 | [26186](https://github.com/airbytehq/airbyte/pull/26186) | Limited the `lookback window` range for input configuration | -| 0.1.25 | 2023-03-20 | [22889](https://github.com/airbytehq/airbyte/pull/22889) | Specified date formatting in specification | -| 0.1.24 | 2023-03-20 | [24126](https://github.com/airbytehq/airbyte/pull/24126) | Increase page size to 1000 | -| 0.1.23 | 2023-02-21 | [21907](https://github.com/airbytehq/airbyte/pull/21907) | Do not join channels that not gonna be synced | -| 0.1.22 | 2023-01-27 | [22022](https://github.com/airbytehq/airbyte/pull/22022) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 0.1.21 | 2023-01-12 | [21321](https://github.com/airbytehq/airbyte/pull/21321) | Retry Timeout error | -| 0.1.20 | 2022-12-21 | [20767](https://github.com/airbytehq/airbyte/pull/20767) | Update schema | -| 0.1.19 | 2022-12-01 | [19970](https://github.com/airbytehq/airbyte/pull/19970) | Remove OAuth2.0 broken `refresh_token` support | -| 0.1.18 | 2022-09-28 | [17315](https://github.com/airbytehq/airbyte/pull/17315) | Always install latest version of Airbyte CDK | -| 0.1.17 | 2022-08-28 | [16085](https://github.com/airbytehq/airbyte/pull/16085) | Increase unit test coverage | -| 0.1.16 | 2022-08-28 | [16050](https://github.com/airbytehq/airbyte/pull/16050) | Fix SATs | -| 0.1.15 | 2022-03-31 | [11613](https://github.com/airbytehq/airbyte/pull/11613) | Add 'channel_filter' config and improve performance | -| 0.1.14 | 2022-01-26 | [9575](https://github.com/airbytehq/airbyte/pull/9575) | Correct schema | -| 0.1.13 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | -| 0.1.12 | 2021-10-07 | [6570](https://github.com/airbytehq/airbyte/pull/6570) | Implement OAuth support with OAuth authenticator | -| 0.1.11 | 2021-08-27 | [5830](https://github.com/airbytehq/airbyte/pull/5830) | Fix sync operations hang forever issue | -| 0.1.10 | 2021-08-27 | [5697](https://github.com/airbytehq/airbyte/pull/5697) | Fix max retries issue | -| 0.1.9 | 2021-07-20 | [4860](https://github.com/airbytehq/airbyte/pull/4860) | Fix reading threads issue | -| 0.1.8 | 2021-07-14 | [4683](https://github.com/airbytehq/airbyte/pull/4683) | Add float\_ts primary key | -| 0.1.7 | 2021-06-25 | [3978](https://github.com/airbytehq/airbyte/pull/3978) | Release Slack CDK Connector | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------| +| 0.3.7 | 2024-01-10 | [1234](https://github.com/airbytehq/airbyte/pull/1234) | prepare for airbyte-lib | +| 0.3.6 | 2023-11-21 | [32707](https://github.com/airbytehq/airbyte/pull/32707) | Threads: do not use client-side record filtering | +| 0.3.5 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 0.3.4 | 2023-10-06 | [31134](https://github.com/airbytehq/airbyte/pull/31134) | Update CDK and remove non iterable return from records | +| 0.3.3 | 2023-09-28 | [30580](https://github.com/airbytehq/airbyte/pull/30580) | Add `bot_id` field to threads schema | +| 0.3.2 | 2023-09-20 | [30613](https://github.com/airbytehq/airbyte/pull/30613) | Set default value for channel_filters during discover | +| 0.3.1 | 2023-09-19 | [30570](https://github.com/airbytehq/airbyte/pull/30570) | Use default availability strategy | +| 0.3.0 | 2023-09-18 | [30521](https://github.com/airbytehq/airbyte/pull/30521) | Add unexpected fields to streams `channel_messages`, `channels`, `threads`, `users` | +| 0.2.0 | 2023-05-24 | [26497](https://github.com/airbytehq/airbyte/pull/26497) | Fixed `lookback window` value limitations | +| 0.1.26 | 2023-05-17 | [26186](https://github.com/airbytehq/airbyte/pull/26186) | Limited the `lookback window` range for input configuration | +| 0.1.25 | 2023-03-20 | [22889](https://github.com/airbytehq/airbyte/pull/22889) | Specified date formatting in specification | +| 0.1.24 | 2023-03-20 | [24126](https://github.com/airbytehq/airbyte/pull/24126) | Increase page size to 1000 | +| 0.1.23 | 2023-02-21 | [21907](https://github.com/airbytehq/airbyte/pull/21907) | Do not join channels that not gonna be synced | +| 0.1.22 | 2023-01-27 | [22022](https://github.com/airbytehq/airbyte/pull/22022) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.1.21 | 2023-01-12 | [21321](https://github.com/airbytehq/airbyte/pull/21321) | Retry Timeout error | +| 0.1.20 | 2022-12-21 | [20767](https://github.com/airbytehq/airbyte/pull/20767) | Update schema | +| 0.1.19 | 2022-12-01 | [19970](https://github.com/airbytehq/airbyte/pull/19970) | Remove OAuth2.0 broken `refresh_token` support | +| 0.1.18 | 2022-09-28 | [17315](https://github.com/airbytehq/airbyte/pull/17315) | Always install latest version of Airbyte CDK | +| 0.1.17 | 2022-08-28 | [16085](https://github.com/airbytehq/airbyte/pull/16085) | Increase unit test coverage | +| 0.1.16 | 2022-08-28 | [16050](https://github.com/airbytehq/airbyte/pull/16050) | Fix SATs | +| 0.1.15 | 2022-03-31 | [11613](https://github.com/airbytehq/airbyte/pull/11613) | Add 'channel_filter' config and improve performance | +| 0.1.14 | 2022-01-26 | [9575](https://github.com/airbytehq/airbyte/pull/9575) | Correct schema | +| 0.1.13 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | +| 0.1.12 | 2021-10-07 | [6570](https://github.com/airbytehq/airbyte/pull/6570) | Implement OAuth support with OAuth authenticator | +| 0.1.11 | 2021-08-27 | [5830](https://github.com/airbytehq/airbyte/pull/5830) | Fix sync operations hang forever issue | +| 0.1.10 | 2021-08-27 | [5697](https://github.com/airbytehq/airbyte/pull/5697) | Fix max retries issue | +| 0.1.9 | 2021-07-20 | [4860](https://github.com/airbytehq/airbyte/pull/4860) | Fix reading threads issue | +| 0.1.8 | 2021-07-14 | [4683](https://github.com/airbytehq/airbyte/pull/4683) | Add float\_ts primary key | +| 0.1.7 | 2021-06-25 | [3978](https://github.com/airbytehq/airbyte/pull/3978) | Release Slack CDK Connector | + +
      \ No newline at end of file diff --git a/docs/integrations/sources/smartsheets.md b/docs/integrations/sources/smartsheets.md index 4d9e62d85997..52359b70e88e 100644 --- a/docs/integrations/sources/smartsheets.md +++ b/docs/integrations/sources/smartsheets.md @@ -110,6 +110,7 @@ The remaining column datatypes supported by Smartsheets are more complex types ( | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------| +| 1.1.2 | 2024-01-08 | [1234](https://github.com/airbytehq/airbyte/pull/1234) | prepare for airbyte-lib | | 1.1.1 | 2023-06-06 | [27096](https://github.com/airbytehq/airbyte/pull/27096) | Fix error when optional metadata fields are not set | | 1.1.0 | 2023-06-02 | [22382](https://github.com/airbytehq/airbyte/pull/22382) | Add support for ingesting metadata fields | | 1.0.2 | 2023-05-12 | [26024](https://github.com/airbytehq/airbyte/pull/26024) | Fix dependencies conflict | diff --git a/docs/integrations/sources/snapchat-marketing.md b/docs/integrations/sources/snapchat-marketing.md index 73c07998cbe1..5f12ba912116 100644 --- a/docs/integrations/sources/snapchat-marketing.md +++ b/docs/integrations/sources/snapchat-marketing.md @@ -35,7 +35,7 @@ This page guides you through the process of setting up the Snapchat Marketing so - If not - just use some valid url. Here's the discussion about it: [Snapchat Redirect URL - Clarity in documentation please](https://github.com/Snap-Kit/bitmoji-sample/issues/3) * save **Client ID** and **Client Secret** 4. Get refresh token using OAuth2 authentication workflow: - * Open the authorize link in a browser: [https://accounts.snapchat.com/login/oauth2/authorize?response\_type=code&client\_id={client\_id}&redirect\_uri={redirect\_uri}&scope=snapchat-marketing-api&state=wmKkg0TWgppW8PTBZ20sldUmF7hwvU](https://accounts.snapchat.com/login/oauth2/authorize?response_type=code&client_id={client_id}&redirect_uri={redirect_uri}&scope=snapchat-marketing-api&state=wmKkg0TWgppW8PTBZ20sldUmF7hwvU) + * Open the authorize link in a browser: [https://accounts.snapchat.com/login/oauth2/authorize?response\_type=code&client\_id=CLIENT\_ID&redirect\_uri=REDIRECT\_URI&scope=snapchat-marketing-api&state=wmKkg0TWgppW8PTBZ20sldUmF7hwvU](https://accounts.snapchat.com/login/oauth2/authorize?response_type=code&client_id=CLIENT_ID&redirect_uri=REDIRECT_URI&scope=snapchat-marketing-api&state=wmKkg0TWgppW8PTBZ20sldUmF7hwvU) * Login & Authorize via UI * Locate "code" query parameter in the redirect * Exchange code for access token + refresh token diff --git a/docs/integrations/sources/snowflake.md b/docs/integrations/sources/snowflake.md index 0e93412be2e1..afc7b45ba27f 100644 --- a/docs/integrations/sources/snowflake.md +++ b/docs/integrations/sources/snowflake.md @@ -4,7 +4,7 @@ The Snowflake source allows you to sync data from Snowflake. It supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. -This Snowflake source connector is built on top of the source-jdbc code base and is configured to rely on JDBC 3.13.22 [Snowflake driver](https://github.com/snowflakedb/snowflake-jdbc) as described in Snowflake [documentation](https://docs.snowflake.com/en/user-guide/jdbc.html). +This Snowflake source connector is built on top of the source-jdbc code base and is configured to rely on JDBC 3.14.1 [Snowflake driver](https://github.com/snowflakedb/snowflake-jdbc) as described in Snowflake [documentation](https://docs.snowflake.com/en/user-guide/jdbc.html). #### Resulting schema @@ -111,18 +111,25 @@ To determine whether a network policy is set on your account or for a specific u **Account** - SHOW PARAMETERS LIKE 'network_policy' IN ACCOUNT; +``` +SHOW PARAMETERS LIKE 'network_policy' IN ACCOUNT; +``` **User** - SHOW PARAMETERS LIKE 'network_policy' IN USER ; +``` +SHOW PARAMETERS LIKE 'network_policy' IN USER ; +``` To read more please check official [Snowflake documentation](https://docs.snowflake.com/en/user-guide/network-policies.html#) ## Changelog | Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------- | +|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------| +| 0.3.0 | 2023-12-18 | [33484](https://github.com/airbytehq/airbyte/pull/33484) | Remove LEGACY state | +| 0.2.2 | 2023-10-20 | [31613](https://github.com/airbytehq/airbyte/pull/31613) | Fixed handling of TIMESTAMP_TZ columns. upgrade | +| 0.2.1 | 2023-10-11 | [31252](https://github.com/airbytehq/airbyte/pull/31252) | Snowflake JDBC version upgrade | | 0.2.0 | 2023-06-26 | [27737](https://github.com/airbytehq/airbyte/pull/27737) | License Update: Elv2 | | 0.1.36 | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212) | Fix silent exception swallowing in StreamingJdbcDatabase | | 0.1.35 | 2023-06-14 | [27335](https://github.com/airbytehq/airbyte/pull/27335) | Remove noisy debug logs | diff --git a/docs/integrations/sources/spacex-api.md b/docs/integrations/sources/spacex-api.md index 6e182275596a..ee3730f8d5ef 100644 --- a/docs/integrations/sources/spacex-api.md +++ b/docs/integrations/sources/spacex-api.md @@ -72,4 +72,5 @@ The SpaceX API has both v4 and v5 for [launches](https://github.com/r-spacex/Spa | Version | Date | Pull Request | Subject | | :------ | :--------- | :----------------------------------------------------- | :------------- | +| 0.1.1 | 2023-11-08 | [32202](https://github.com/airbytehq/airbyte/pull/32202) | Adjust schemas to cover all fields in the records | | 0.1.0 | 2022-10-22 | [Init](https://github.com/airbytehq/airbyte/pull/18311) | Initial commit | diff --git a/docs/integrations/sources/square.md b/docs/integrations/sources/square.md index 2dfa772284b3..121c27639cf5 100644 --- a/docs/integrations/sources/square.md +++ b/docs/integrations/sources/square.md @@ -68,7 +68,10 @@ The Square source connector supports the following [ sync modes](https://docs.ai - [List Team Member Wages](https://developer.squareup.com/explorer/square/labor-api/list-team-member-wages) - [Customers](https://developer.squareup.com/explorer/square/customers-api/list-customers) - [Shifts](https://developer.squareup.com/reference/square/labor-api/search-shifts) +- [Inventory](https://developer.squareup.com/reference/square/inventory-api/batch-retrieve-inventory-counts) - [Orders](https://developer.squareup.com/reference/square/orders-api/search-orders) +- [Cash drawers](https://developer.squareup.com/explorer/square/cash-drawers-api/list-cash-drawer-shifts) +- [Loyalty](https://developer.squareup.com/explorer/square/loyalty-api/search-loyalty-accounts) ## Connector-specific features & highlights @@ -97,6 +100,13 @@ Exponential [Backoff](https://developer.squareup.com/forums/t/current-square-api | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------ | +| 1.6.1 | 2023-11-07 | [31481](https://github.com/airbytehq/airbyte/pull/31481) | Fix duplicate records for `Payments` and `Refunds` stream| +| 1.6.0 | 2023-10-18 | [31115](https://github.com/airbytehq/airbyte/pull/31115) | Add `customer_id` field to `Payments` and `Orders` streams | +| 1.5.0 | 2023-10-16 | [31045](https://github.com/airbytehq/airbyte/pull/31045) | Added New Stream bank_accounts | +| 1.4.0 | 2023-10-13 | [31106](https://github.com/airbytehq/airbyte/pull/31106) | Add new stream Loyalty | +| 1.3.0 | 2023-10-12 | [31107](https://github.com/airbytehq/airbyte/pull/31107) | Add new stream Inventory | +| 1.2.0 | 2023-10-10 | [31065](https://github.com/airbytehq/airbyte/pull/31065) | Add new stream Cash drawers shifts | +| 1.1.3 | 2023-10-10 | [30960](https://github.com/airbytehq/airbyte/pull/30960) | Update `airbyte-cdk` version to `>=0.51.31` | | 1.1.2 | 2023-07-10 | [28019](https://github.com/airbytehq/airbyte/pull/28019) | fix display order of spec fields | | 1.1.1 | 2023-06-28 | [27762](https://github.com/airbytehq/airbyte/pull/27762) | Update following state breaking changes | | 1.1.0 | 2023-05-24 | [26485](https://github.com/airbytehq/airbyte/pull/26485) | Remove deprecated authSpecification in favour of advancedAuth | diff --git a/docs/integrations/sources/strava.md b/docs/integrations/sources/strava.md index c2d187919952..fcd7cb286758 100644 --- a/docs/integrations/sources/strava.md +++ b/docs/integrations/sources/strava.md @@ -124,6 +124,7 @@ More information about Strava rate limits and adjustments to those limits can be | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------- | +| 0.2.0 | 2023-10-24 | [31007](https://github.com/airbytehq/airbyte/pull/31007) | Migrate to low-code framework | | 0.1.4 | 2023-03-23 | [24368](https://github.com/airbytehq/airbyte/pull/24368) | Add date-time format for input | | 0.1.3 | 2023-03-15 | [24101](https://github.com/airbytehq/airbyte/pull/24101) | certified to beta, fixed spec, fixed SAT, added unit tests | | 0.1.2 | 2021-12-15 | [8799](https://github.com/airbytehq/airbyte/pull/8799) | Implement OAuth 2.0 support | diff --git a/docs/integrations/sources/stripe-migrations.md b/docs/integrations/sources/stripe-migrations.md new file mode 100644 index 000000000000..60f4be4d4ab4 --- /dev/null +++ b/docs/integrations/sources/stripe-migrations.md @@ -0,0 +1,21 @@ +# Stripe Migration Guide + +## Upgrading to 5.0.0 + +This change fixes multiple incremental sync issues with the `Refunds`, `Checkout Sessions` and `Checkout Sessions Line Items` streams: + - `Refunds` stream was not syncing data in the incremental sync mode. Cursor field has been updated to "created" to allow for incremental syncs. Because of the changed cursor field of the `Refunds` stream, incremental syncs will not reflect every update of the records that have been previously replicated. Only newly created records will be synced. To always have the up-to-date data, users are encouraged to make use of the lookback window. + - `CheckoutSessions` stream had been missing data for one day when using the incremental sync mode after a reset; this has been resolved. + - `CheckoutSessionsLineItems` previously had potential data loss. It has been updated to use a new cursor field `checkout_session_updated`. + - Incremental streams with the `created` cursor had been duplicating some data; this has been fixed. + +Stream schema update is a breaking change as well as changing the cursor field for the `Refunds` and the `CheckoutSessionsLineItems` stream. A schema refresh and data reset of all effected streams is required after the update is applied. + +Also, this update affects three more streams: `Invoices`, `Subscriptions`, `SubscriptionSchedule`. Schemas are changed in this update so that the declared data types would match the actual data. + +Stream schema update is a breaking change as well as changing the cursor field for the `Refunds` and the `CheckoutSessionsLineItems` stream. A schema refresh and data reset of all effected streams is required after the update is applied. +Because of the changed cursor field of the `Refunds` stream, incremental syncs will not reflect every update of the records that have been previously replicated. Only newly created records will be synced. To always have the up-to-date data, users are encouraged to make use of the lookback window. + +## Upgrading to 4.0.0 + +A major update of most streams to support event-based incremental sync mode. This allows the connector to pull not only the newly created data since the last sync, but the modified data as well. +A schema refresh is required for the connector to use the new cursor format. \ No newline at end of file diff --git a/docs/integrations/sources/stripe.inapp.md b/docs/integrations/sources/stripe.inapp.md deleted file mode 100644 index 5988ba81922d..000000000000 --- a/docs/integrations/sources/stripe.inapp.md +++ /dev/null @@ -1,35 +0,0 @@ -## Prerequisites - -- Access to the Stripe account containing the data to replicate - -## Setup Guide - -:::note -To authenticate the Stripe connector, you need an API key with **Read** privileges for the data to replicate. For steps on obtaining and setting permissions for this key, refer to our [full Stripe documentation](https://docs.airbyte.com/integrations/sources/stripe#setup-guide). -::: - -1. For **Source name**, enter a name to help you identify this source. -2. For **Account ID**, enter your Stripe Account ID. This ID begins with `acct_`, and can be found in the top-right corner of your Stripe [account settings page](https://dashboard.stripe.com/settings/account). -3. For **Secret Key**, enter your Stripe API key, which can be found at your Stripe [API keys page](https://dashboard.stripe.com/apikeys). -4. For **Replication Start Date**, use the provided datepicker or enter a UTC date and time programmatically in the format `YYYY-MM-DDTHH:mm:ssZ`. The data added on and after this date will be replicated. -5. (Optional) For **Lookback Window**, you may specify a number of days from the present day to reread data. This allows the connector to retrieve data that might have been updated after its initial creation, and is useful for handling any post-transaction adjustments (such as tips, refunds, chargebacks, etc). - - - Leaving the **Lookback Window** at its default value of 0 means Airbyte will not re-export data after it has been synced. - - Setting the **Lookback Window** to 1 means Airbyte will re-export and capture any data changes within the last day. - - Setting the **Lookback Window** to 7 means Airbyte will re-export and capture any data changes within the last week. - -6. (Optional) For **Data Request Window**, you may specify the time window in days used by the connector when requesting data from the Stripe API. This window defines the span of time covered in each request, with larger values encompassing more days in a single request. Generally speaking, the lack of overhead from making fewer requests means a larger window is faster to sync. However, this also means the state of the sync will persist less frequently. If an issue occurs or the sync is interrupted, a larger window means more data will need to be resynced, potentially causing a delay in the overall process. - - For example, if you are replicating three years worth of data: - - - A **Data Request Window** of 365 days means Airbyte makes 3 requests, each for a year. This is generally faster but risks needing to resync up to a year's data if the sync is interrupted. - - A **Data Request Window** of 30 days means 36 requests, each for a month. This may be slower but minimizes the amount of data that needs to be resynced if an issue occurs. - - If you are unsure of which value to use, we recommend leaving this setting at its default value of 365 days. -7. Click **Set up source** and wait for the tests to complete. - -### Stripe API limitations - -- When syncing `events` data from Stripe, data is only [returned for the last 30 days](https://stripe.com/docs/api/events). Using the Full Refresh (Overwrite) sync from Airbyte will delete the events data older than 30 days from your target destination. Use an Append sync mode to ensure historical data is retained. - -For detailed information on supported sync modes, supported streams, performance considerations, refer to the full documentation for [Stripe](https://docs.airbyte.com/integrations/sources/stripe). diff --git a/docs/integrations/sources/stripe.md b/docs/integrations/sources/stripe.md index 9a69d906d11a..14a6d35caef5 100644 --- a/docs/integrations/sources/stripe.md +++ b/docs/integrations/sources/stripe.md @@ -1,6 +1,10 @@ # Stripe -This page contains the setup guide and reference information for the Stripe source connector. + + +This page contains the setup guide and reference information for the [Stripe](https://stripe.com/) source connector. + + ## Prerequisites @@ -8,9 +12,11 @@ This page contains the setup guide and reference information for the Stripe sour ## Setup Guide +:::note To authenticate the Stripe connector, you need to use a Stripe API key. Although you may use an existing key, we recommend that you create a new restricted key specifically for Airbyte and grant it **Read** privileges only. We also recommend granting **Read** privileges to all available permissions, and configuring the specific data you would like to replicate within Airbyte. +::: -### Create a Stripe Secret Key +### Step 1: Set up Stripe 1. Log in to your [Stripe account](https://dashboard.stripe.com/login). 2. In the top navigation bar, click **Developers**. @@ -21,31 +27,34 @@ To authenticate the Stripe connector, you need to use a Stripe API key. Although For more information on Stripe API Keys, see the [Stripe documentation](https://stripe.com/docs/keys). -### Set up the Stripe source connector in Airbyte +### Step 2: Set up the Stripe source connector in Airbyte -1. Log in to your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account. +1. Log in to your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) account or your Airbyte Open Source account. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. 3. Find and select **Stripe** from the list of available sources. 4. For **Source name**, enter a name to help you identify this source. 5. For **Account ID**, enter your Stripe Account ID. This ID begins with `acct_`, and can be found in the top-right corner of your Stripe [account settings page](https://dashboard.stripe.com/settings/account). 6. For **Secret Key**, enter the restricted key you created for the connection. 7. For **Replication Start Date**, use the provided datepicker or enter a UTC date and time programmatically in the format `YYYY-MM-DDTHH:mm:ssZ`. The data added on and after this date will be replicated. -8. (Optional) For **Lookback Window**, you may specify a number of days from the present day to reread data. This allows the connector to retrieve data that might have been updated after its initial creation, and is useful for handling any post-transaction adjustments (such as tips, refunds, chargebacks, etc). +8. (Optional) For **Lookback Window**, you may specify a number of days from the present day to reread data. This allows the connector to retrieve data that might have been updated after its initial creation, and is useful for handling any post-transaction adjustments. This applies only to streams that do not support event-based incremental syncs, please see [the list below](#troubleshooting). - - Leaving the **Lookback Window** at its default value of 0 means Airbyte will not re-export data after it has been synced. - - Setting the **Lookback Window** to 1 means Airbyte will re-export data from the past day, capturing any changes made in the last 24 hours. - - Setting the **Lookback Window** to 7 means Airbyte will re-export and capture any data changes within the last week. + - Leaving the **Lookback Window** at its default value of 0 means Airbyte will not re-export data after it has been synced. + - Setting the **Lookback Window** to 1 means Airbyte will re-export data from the past day, capturing any changes made in the last 24 hours. + - Setting the **Lookback Window** to 7 means Airbyte will re-export and capture any data changes within the last week. 9. (Optional) For **Data Request Window**, you may specify the time window in days used by the connector when requesting data from the Stripe API. This window defines the span of time covered in each request, with larger values encompassing more days in a single request. Generally speaking, the lack of overhead from making fewer requests means a larger window is faster to sync. However, this also means the state of the sync will persist less frequently. If an issue occurs or the sync is interrupted, a larger window means more data will need to be resynced, potentially causing a delay in the overall process. - For example, if you are replicating three years worth of data: + For example, if you are replicating three years worth of data: + + - A **Data Request Window** of 365 days means Airbyte makes 3 requests, each for a year. This is generally faster but risks needing to resync up to a year's data if the sync is interrupted. + - A **Data Request Window** of 30 days means 36 requests, each for a month. This may be slower but minimizes the amount of data that needs to be resynced if an issue occurs. - - A **Data Request Window** of 365 days means Airbyte makes 3 requests, each for a year. This is generally faster but risks needing to resync up to a year's data if the sync is interrupted. - - A **Data Request Window** of 30 days means 36 requests, each for a month. This may be slower but minimizes the amount of data that needs to be resynced if an issue occurs. + If you are unsure of which value to use, we recommend leaving this setting at its default value of 365 days. - If you are unsure of which value to use, we recommend leaving this setting at its default value of 365 days. 10. Click **Set up source** and wait for the tests to complete. + + ## Supported sync modes The Stripe source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): @@ -53,38 +62,33 @@ The Stripe source connector supports the following [sync modes](https://docs.air - Full Refresh - Incremental -:::note -Since the Stripe API does not allow querying objects which were updated since the last sync, the Stripe connector uses the `created` field to query for new data in your Stripe account. -::: - ## Supported streams The Stripe source connector supports the following streams: -- [Accounts](https://stripe.com/docs/api/accounts/list) \(Incremental\) +- [Accounts](https://stripe.com/docs/api/accounts/list) - [Application Fees](https://stripe.com/docs/api/application_fees) \(Incremental\) -- [Application Fee Refunds](https://stripe.com/docs/api/fee_refunds/list) +- [Application Fee Refunds](https://stripe.com/docs/api/fee_refunds/list) \(Incremental\) - [Authorizations](https://stripe.com/docs/api/issuing/authorizations/list) \(Incremental\) - [Balance Transactions](https://stripe.com/docs/api/balance_transactions/list) \(Incremental\) -- [Bank accounts](https://stripe.com/docs/api/customer_bank_accounts/list) +- [Bank accounts](https://stripe.com/docs/api/customer_bank_accounts/list) \(Incremental\) - [Cardholders](https://stripe.com/docs/api/issuing/cardholders/list) \(Incremental\) - [Cards](https://stripe.com/docs/api/issuing/cards/list) \(Incremental\) - [Charges](https://stripe.com/docs/api/charges/list) \(Incremental\) :::note The `amount` column defaults to the smallest currency unit. Check [the Stripe docs](https://stripe.com/docs/api/charges/object) for more details. ::: -- [Checkout Sessions](https://stripe.com/docs/api/checkout/sessions/list) -- [Checkout Sessions Line Items](https://stripe.com/docs/api/checkout/sessions/line_items) +- [Checkout Sessions](https://stripe.com/docs/api/checkout/sessions/list) \(Incremental\) +- [Checkout Sessions Line Items](https://stripe.com/docs/api/checkout/sessions/line_items) \(Incremental\) - [Coupons](https://stripe.com/docs/api/coupons/list) \(Incremental\) -- [Credit Notes](https://stripe.com/docs/api/credit_notes/list) \(Full Refresh\) -- [Customer Balance Transactions](https://stripe.com/docs/api/customer_balance_transactions/list) +- [Credit Notes](https://stripe.com/docs/api/credit_notes/list) \(Incremental\) +- [Customer Balance Transactions](https://stripe.com/docs/api/customer_balance_transactions/list) \(Incremental\) - [Customers](https://stripe.com/docs/api/customers/list) \(Incremental\) - :::note - This endpoint does _not_ include deleted customers - ::: - [Disputes](https://stripe.com/docs/api/disputes/list) \(Incremental\) - [Early Fraud Warnings](https://stripe.com/docs/api/radar/early_fraud_warnings/list) \(Incremental\) - [Events](https://stripe.com/docs/api/events/list) \(Incremental\) +- [External Account Bank Accounts](https://stripe.com/docs/api/external_account_bank_accounts/list) \(Incremental\) +- [External Account Cards](https://stripe.com/docs/api/external_account_cards/list) \(Incremental\) - [File Links](https://stripe.com/docs/api/file_links/list) \(Incremental\) - [Files](https://stripe.com/docs/api/files/list) \(Incremental\) - [Invoice Items](https://stripe.com/docs/api/invoiceitems/list) \(Incremental\) @@ -112,87 +116,202 @@ The Stripe source connector supports the following streams: - [Transfer Reversals](https://stripe.com/docs/api/transfer_reversals/list) - [Usage Records](https://stripe.com/docs/api/usage_records/subscription_item_summary_list) + + + + +### Data type mapping + +The [Stripe API](https://stripe.com/docs/api) uses the same [JSON Schema](https://json-schema.org/understanding-json-schema/reference/index.html) types that Airbyte uses internally \(`string`, `date-time`, `object`, `array`, `boolean`, `integer`, and `number`\), so no type conversions are performed for the Stripe connector. + +## Limitations & Troubleshooting + +
      + +Expand to see details about Stripe connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting + +The Stripe connector should not run into Stripe API limitations under normal usage. See Stripe [Rate limits](https://stripe.com/docs/rate-limits) documentation. [Create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. + :::warning **Stripe API Restriction on Events Data**: Access to the events endpoint is [guaranteed only for the last 30 days](https://stripe.com/docs/api/events) by Stripe. If you use the Full Refresh Overwrite sync, be aware that any events data older than 30 days will be **deleted** from your target destination and replaced with the data from the last 30 days only. Use an Append sync mode to ensure historical data is retained. +Please be aware: this also means that any change older than 30 days will not be replicated using the incremental sync mode. If you want all your synced data to remain up to date, please set up your sync frequency to no more than 30 days. ::: -### Data type mapping +### Troubleshooting -The [Stripe API](https://stripe.com/docs/api) uses the same [JSON Schema](https://json-schema.org/understanding-json-schema/reference/index.html) types that Airbyte uses internally \(`string`, `date-time`, `object`, `array`, `boolean`, `integer`, and `number`\), so no type conversions are performed for the Stripe connector. +Since the Stripe API does not allow querying objects which were updated since the last sync, the Stripe connector uses the Events API under the hood to implement incremental syncs and export data based on its update date. +However, not all the entities are supported by the Events API, so the Stripe connector uses the `created` field or its analogue to query for new data in your Stripe account. These are the entities synced based on the date of creation: +- `Balance Transactions` +- `Events` +- `File Links` +- `Files` +- `Refunds` +- `Setup Attempts` +- `Shipping Rates` + +On the other hand, the following streams use the `updated` field value as a cursor: + +:::note + +`updated` is an artificial cursor field introduced by Airbyte for the Incremental sync option. + +::: + +- `Application Fees` +- `Application Fee Refunds` +- `Authorizations` +- `Bank Accounts` +- `Cardholders` +- `Cards` +- `Charges` +- `Checkout Sessions` +- `Checkout Session Line Items` (cursor field is `checkout_session_updated`) +- `Coupons` +- `Credit Notes` +- `Customer Balance Transactions` +- `Customers` +- `Disputes` +- `Early Fraud Warnings` +- `External Account Bank Accounts` +- `External Account Cards` +- `Invoice Items` +- `Invoices` +- `Payment Intents` +- `Payouts` +- `Promotion Codes` +- `Persons` +- `Plans` +- `Prices` +- `Products` +- `Reviews` +- `Setup Intents` +- `Subscription Schedule` +- `Subscriptions` +- `Top Ups` +- `Transactions` +- `Transfers` -### Performance considerations +## Incremental deletes + +The Stripe API also provides a way to implement incremental deletes for a limited number of streams: +- `Bank Accounts` +- `Coupons` +- `Customers` +- `External Account Bank Accounts` +- `External Account Cards` +- `Invoices` +- `Invoice Items` +- `Persons` +- `Plans` +- `Prices` +- `Products` +- `Subscriptions` + +Each record is marked with `is_deleted` flag when the appropriate event happens upstream. +* Check out common troubleshooting issues for the Stripe source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +### Data type mapping -The Stripe connector should not run into Stripe API limitations under normal usage. [Create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. +
      ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------| -| 3.17.4 | 2023-08-15 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Revert 3.17.3 | -| 3.17.3 | 2023-08-01 | [28911](https://github.com/airbytehq/airbyte/pull/28911) | Revert 3.17.2 and fix atm_fee property | -| 3.17.2 | 2023-08-01 | [28911](https://github.com/airbytehq/airbyte/pull/28911) | Fix stream schemas, remove custom 403 error handling | -| 3.17.1 | 2023-08-01 | [28887](https://github.com/airbytehq/airbyte/pull/28887) | Fix `Invoices` schema | -| 3.17.0 | 2023-07-28 | [26127](https://github.com/airbytehq/airbyte/pull/26127) | Add `Prices` stream | -| 3.16.0 | 2023-07-27 | [28776](https://github.com/airbytehq/airbyte/pull/28776) | Add new fields to stream schemas | -| 3.15.0 | 2023-07-09 | [28709](https://github.com/airbytehq/airbyte/pull/28709) | Remove duplicate streams | -| 3.14.0 | 2023-07-09 | [27217](https://github.com/airbytehq/airbyte/pull/27217) | Add `ShippingRates` stream | -| 3.13.0 | 2023-07-18 | [28466](https://github.com/airbytehq/airbyte/pull/28466) | Pin source API version | -| 3.12.0 | 2023-05-20 | [26208](https://github.com/airbytehq/airbyte/pull/26208) | Add new stream `Persons` | -| 3.11.0 | 2023-06-26 | [27734](https://github.com/airbytehq/airbyte/pull/27734) | License Update: Elv2 stream | -| 3.10.0 | 2023-06-22 | [27132](https://github.com/airbytehq/airbyte/pull/27132) | Add `CreditNotes` stream | -| 3.9.1 | 2023-06-20 | [27522](https://github.com/airbytehq/airbyte/pull/27522) | Fix formatting | -| 3.9.0 | 2023-06-19 | [27362](https://github.com/airbytehq/airbyte/pull/27362) | Add new Streams: Transfer Reversals, Setup Attempts, Usage Records, Transactions | -| 3.8.0 | 2023-06-12 | [27238](https://github.com/airbytehq/airbyte/pull/27238) | Add `Topups` stream; Add `Files` stream; Add `FileLinks` stream | -| 3.7.0 | 2023-06-06 | [27083](https://github.com/airbytehq/airbyte/pull/27083) | Add new Streams: Authorizations, Cardholders, Cards, Payment Methods, Reviews | -| 3.6.0 | 2023-05-24 | [25893](https://github.com/airbytehq/airbyte/pull/25893) | Add `ApplicationFeesRefunds` stream with parent `ApplicationFees` | -| 3.5.0 | 2023-05-20 | [22859](https://github.com/airbytehq/airbyte/pull/22859) | Add stream `Early Fraud Warnings` | -| 3.4.3 | 2023-05-10 | [25965](https://github.com/airbytehq/airbyte/pull/25965) | Fix Airbyte date-time data-types | -| 3.4.2 | 2023-05-04 | [25795](https://github.com/airbytehq/airbyte/pull/25795) | Added `CDK TypeTransformer` to guarantee declared JSON Schema data-types | -| 3.4.1 | 2023-04-24 | [23389](https://github.com/airbytehq/airbyte/pull/23389) | Add `customer_tax_ids` to `Invoices` | -| 3.4.0 | 2023-03-20 | [23963](https://github.com/airbytehq/airbyte/pull/23963) | Add `SetupIntents` stream | -| 3.3.0 | 2023-04-12 | [25136](https://github.com/airbytehq/airbyte/pull/25136) | Add stream `Accounts` | -| 3.2.0 | 2023-04-10 | [23624](https://github.com/airbytehq/airbyte/pull/23624) | Add new stream `Subscription Schedule` | -| 3.1.0 | 2023-03-10 | [19906](https://github.com/airbytehq/airbyte/pull/19906) | Expand `tiers` when syncing `Plans` streams | -| 3.0.5 | 2023-03-25 | [22866](https://github.com/airbytehq/airbyte/pull/22866) | Specified date formatting in specification | -| 3.0.4 | 2023-03-24 | [24471](https://github.com/airbytehq/airbyte/pull/24471) | Fix stream slices for single sliced streams | -| 3.0.3 | 2023-03-17 | [24179](https://github.com/airbytehq/airbyte/pull/24179) | Get customer's attributes safely | -| 3.0.2 | 2023-03-13 | [24051](https://github.com/airbytehq/airbyte/pull/24051) | Cache `customers` stream; Do not request transactions of customers with zero balance. | -| 3.0.1 | 2023-02-22 | [22898](https://github.com/airbytehq/airbyte/pull/22898) | Add missing column to Subscriptions stream | -| 3.0.0 | 2023-02-21 | [23295](https://github.com/airbytehq/airbyte/pull/23295) | Fix invoice schema | -| 2.0.0 | 2023-02-14 | [22312](https://github.com/airbytehq/airbyte/pull/22312) | Another fix of `Invoices` stream schema + Remove http urls from openapi_spec.json | -| 1.0.2 | 2023-02-09 | [22659](https://github.com/airbytehq/airbyte/pull/22659) | Set `AvailabilityStrategy` for all streams | -| 1.0.1 | 2023-01-27 | [22042](https://github.com/airbytehq/airbyte/pull/22042) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 1.0.0 | 2023-01-25 | [21858](https://github.com/airbytehq/airbyte/pull/21858) | Update the `Subscriptions` and `Invoices` stream schemas | -| 0.1.40 | 2022-10-20 | [18228](https://github.com/airbytehq/airbyte/pull/18228) | Update the `PaymentIntents` stream schema | -| 0.1.39 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream states. | -| 0.1.38 | 2022-09-09 | [16537](https://github.com/airbytehq/airbyte/pull/16537) | Fix `redeem_by` field type for `customers` stream | -| 0.1.37 | 2022-08-16 | [15686](https://github.com/airbytehq/airbyte/pull/15686) | Fix the bug when the stream couldn't be fetched due to limited permission set, if so - it should be skipped | -| 0.1.36 | 2022-08-04 | [15292](https://github.com/airbytehq/airbyte/pull/15292) | Implement slicing | -| 0.1.35 | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from spec and schema | -| 0.1.34 | 2022-07-01 | [14357](https://github.com/airbytehq/airbyte/pull/14357) | Add external account streams - | -| 0.1.33 | 2022-06-06 | [13449](https://github.com/airbytehq/airbyte/pull/13449) | Add semi-incremental support for CheckoutSessions and CheckoutSessionsLineItems streams, fixed big in StripeSubStream, added unittests, updated docs | -| 0.1.32 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | -| 0.1.31 | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | Update connector to use a `spec.yaml` | -| 0.1.30 | 2022-03-21 | [11286](https://github.com/airbytehq/airbyte/pull/11286) | Minor corrections to documentation and connector specification | -| 0.1.29 | 2022-03-08 | [10359](https://github.com/airbytehq/airbyte/pull/10359) | Improved performance for streams with substreams: invoice_line_items, subscription_items, bank_accounts | -| 0.1.28 | 2022-02-08 | [10165](https://github.com/airbytehq/airbyte/pull/10165) | Improve 404 handling for `CheckoutSessionsLineItems` stream | -| 0.1.27 | 2021-12-28 | [9148](https://github.com/airbytehq/airbyte/pull/9148) | Fix `date`, `arrival\_date` fields | -| 0.1.26 | 2021-12-21 | [8992](https://github.com/airbytehq/airbyte/pull/8992) | Fix type `events.request` in schema | -| 0.1.25 | 2021-11-25 | [8250](https://github.com/airbytehq/airbyte/pull/8250) | Rearrange setup fields | -| 0.1.24 | 2021-11-08 | [7729](https://github.com/airbytehq/airbyte/pull/7729) | Include tax data in `checkout_sessions_line_items` stream | -| 0.1.23 | 2021-11-08 | [7729](https://github.com/airbytehq/airbyte/pull/7729) | Correct `payment_intents` schema | -| 0.1.22 | 2021-11-05 | [7345](https://github.com/airbytehq/airbyte/pull/7345) | Add 3 new streams | -| 0.1.21 | 2021-10-07 | [6841](https://github.com/airbytehq/airbyte/pull/6841) | Fix missing `start_date` argument + update json files for SAT | -| 0.1.20 | 2021-09-30 | [6017](https://github.com/airbytehq/airbyte/pull/6017) | Add lookback_window_days parameter | -| 0.1.19 | 2021-09-27 | [6466](https://github.com/airbytehq/airbyte/pull/6466) | Use `start_date` parameter in incremental streams | -| 0.1.18 | 2021-09-14 | [6004](https://github.com/airbytehq/airbyte/pull/6004) | Fix coupons and subscriptions stream schemas by removing incorrect timestamp formatting | -| 0.1.17 | 2021-09-14 | [6004](https://github.com/airbytehq/airbyte/pull/6004) | Add `PaymentIntents` stream | -| 0.1.16 | 2021-07-28 | [4980](https://github.com/airbytehq/airbyte/pull/4980) | Remove Updated field from schemas | -| 0.1.15 | 2021-07-21 | [4878](https://github.com/airbytehq/airbyte/pull/4878) | Fix incorrect percent_off and discounts data filed types | -| 0.1.14 | 2021-07-09 | [4669](https://github.com/airbytehq/airbyte/pull/4669) | Subscriptions Stream now returns all kinds of subscriptions \(including expired and canceled\) | -| 0.1.13 | 2021-07-03 | [4528](https://github.com/airbytehq/airbyte/pull/4528) | Remove regex for acc validation | -| 0.1.12 | 2021-06-08 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | -| 0.1.11 | 2021-05-30 | [3744](https://github.com/airbytehq/airbyte/pull/3744) | Fix types in schema | -| 0.1.10 | 2021-05-28 | [3728](https://github.com/airbytehq/airbyte/pull/3728) | Update data types to be number instead of int | -| 0.1.9 | 2021-05-13 | [3367](https://github.com/airbytehq/airbyte/pull/3367) | Add acceptance tests for connected accounts | -| 0.1.8 | 2021-05-11 | [3566](https://github.com/airbytehq/airbyte/pull/3368) | Bump CDK connectors | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:-------------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 5.2.0 | 2024-01-18 | [34347](https://github.com/airbytehq/airbyte/pull//34347) | Add new fields invoices and subscription streams. Upgrade the CDK for better memory usage. | +| 5.1.3 | 2023-12-18 | [33306](https://github.com/airbytehq/airbyte/pull/33306/) | Adding integration tests | +| 5.1.2 | 2024-01-04 | [33414](https://github.com/airbytehq/airbyte/pull/33414) | Prepare for airbyte-lib | +| 5.1.1 | 2024-01-04 | [33926](https://github.com/airbytehq/airbyte/pull/33926/) | Update endpoint for `bank_accounts` stream | +| 5.1.0 | 2023-12-11 | [32908](https://github.com/airbytehq/airbyte/pull/32908/) | Read full refresh streams concurrently | +| 5.0.2 | 2023-12-01 | [33038](https://github.com/airbytehq/airbyte/pull/33038) | Add stream slice logging for SubStream | +| 5.0.1 | 2023-11-17 | [32638](https://github.com/airbytehq/airbyte/pull/32638/) | Availability stretegy: check availability of both endpoints (if applicable) - common API + events API | +| 5.0.0 | 2023-11-16 | [32286](https://github.com/airbytehq/airbyte/pull/32286/) | Fix multiple issues regarding usage of the incremental sync mode for the `Refunds`, `CheckoutSessions`, `CheckoutSessionsLineItems` streams. Fix schemas for the streams: `Invoices`, `Subscriptions`, `SubscriptionSchedule` | +| 4.5.4 | 2023-11-16 | [32284](https://github.com/airbytehq/airbyte/pull/32284/) | Enable client-side rate limiting | +| 4.5.3 | 2023-11-14 | [32473](https://github.com/airbytehq/airbyte/pull/32473/) | Have all full_refresh stream syncs be concurrent | +| 4.5.2 | 2023-11-03 | [32146](https://github.com/airbytehq/airbyte/pull/32146/) | Fix multiple BankAccount issues | +| 4.5.1 | 2023-11-01 | [32056](https://github.com/airbytehq/airbyte/pull/32056/) | Use CDK version 0.52.8 | +| 4.5.0 | 2023-10-25 | [31327](https://github.com/airbytehq/airbyte/pull/31327/) | Use concurrent CDK when running in full-refresh | +| 4.4.2 | 2023-10-24 | [31764](https://github.com/airbytehq/airbyte/pull/31764) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 4.4.1 | 2023-10-18 | [31553](https://github.com/airbytehq/airbyte/pull/31553) | Adjusted `Setup Attempts` and extended `Checkout Sessions` stream schemas | +| 4.4.0 | 2023-10-04 | [31046](https://github.com/airbytehq/airbyte/pull/31046) | Added margins field to invoice_line_items stream. | +| 4.3.1 | 2023-09-27 | [30800](https://github.com/airbytehq/airbyte/pull/30800) | Handle permission issues a non breaking | +| 4.3.0 | 2023-09-26 | [30752](https://github.com/airbytehq/airbyte/pull/30752) | Do not sync upcoming invoices, extend stream schemas | +| 4.2.0 | 2023-09-21 | [30660](https://github.com/airbytehq/airbyte/pull/30660) | Fix updated state for the incremental syncs | +| 4.1.1 | 2023-09-15 | [30494](https://github.com/airbytehq/airbyte/pull/30494) | Fix datatype of invoices.lines property | +| 4.1.0 | 2023-08-29 | [29950](https://github.com/airbytehq/airbyte/pull/29950) | Implement incremental deletes, add suggested streams | +| 4.0.1 | 2023-09-07 | [30254](https://github.com/airbytehq/airbyte/pull/30254) | Fix cursorless incremental streams | +| 4.0.0 | 2023-08-15 | [29330](https://github.com/airbytehq/airbyte/pull/29330) | Implement incremental syncs based on date of update | +| 3.17.4 | 2023-08-15 | [29425](https://github.com/airbytehq/airbyte/pull/29425) | Revert 3.17.3 | +| 3.17.3 | 2023-08-01 | [28911](https://github.com/airbytehq/airbyte/pull/28911) | Revert 3.17.2 and fix atm_fee property | +| 3.17.2 | 2023-08-01 | [28911](https://github.com/airbytehq/airbyte/pull/28911) | Fix stream schemas, remove custom 403 error handling | +| 3.17.1 | 2023-08-01 | [28887](https://github.com/airbytehq/airbyte/pull/28887) | Fix `Invoices` schema | +| 3.17.0 | 2023-07-28 | [26127](https://github.com/airbytehq/airbyte/pull/26127) | Add `Prices` stream | +| 3.16.0 | 2023-07-27 | [28776](https://github.com/airbytehq/airbyte/pull/28776) | Add new fields to stream schemas | +| 3.15.0 | 2023-07-09 | [28709](https://github.com/airbytehq/airbyte/pull/28709) | Remove duplicate streams | +| 3.14.0 | 2023-07-09 | [27217](https://github.com/airbytehq/airbyte/pull/27217) | Add `ShippingRates` stream | +| 3.13.0 | 2023-07-18 | [28466](https://github.com/airbytehq/airbyte/pull/28466) | Pin source API version | +| 3.12.0 | 2023-05-20 | [26208](https://github.com/airbytehq/airbyte/pull/26208) | Add new stream `Persons` | +| 3.11.0 | 2023-06-26 | [27734](https://github.com/airbytehq/airbyte/pull/27734) | License Update: Elv2 stream | +| 3.10.0 | 2023-06-22 | [27132](https://github.com/airbytehq/airbyte/pull/27132) | Add `CreditNotes` stream | +| 3.9.1 | 2023-06-20 | [27522](https://github.com/airbytehq/airbyte/pull/27522) | Fix formatting | +| 3.9.0 | 2023-06-19 | [27362](https://github.com/airbytehq/airbyte/pull/27362) | Add new Streams: Transfer Reversals, Setup Attempts, Usage Records, Transactions | +| 3.8.0 | 2023-06-12 | [27238](https://github.com/airbytehq/airbyte/pull/27238) | Add `Topups` stream; Add `Files` stream; Add `FileLinks` stream | +| 3.7.0 | 2023-06-06 | [27083](https://github.com/airbytehq/airbyte/pull/27083) | Add new Streams: Authorizations, Cardholders, Cards, Payment Methods, Reviews | +| 3.6.0 | 2023-05-24 | [25893](https://github.com/airbytehq/airbyte/pull/25893) | Add `ApplicationFeesRefunds` stream with parent `ApplicationFees` | +| 3.5.0 | 2023-05-20 | [22859](https://github.com/airbytehq/airbyte/pull/22859) | Add stream `Early Fraud Warnings` | +| 3.4.3 | 2023-05-10 | [25965](https://github.com/airbytehq/airbyte/pull/25965) | Fix Airbyte date-time data-types | +| 3.4.2 | 2023-05-04 | [25795](https://github.com/airbytehq/airbyte/pull/25795) | Added `CDK TypeTransformer` to guarantee declared JSON Schema data-types | +| 3.4.1 | 2023-04-24 | [23389](https://github.com/airbytehq/airbyte/pull/23389) | Add `customer_tax_ids` to `Invoices` | +| 3.4.0 | 2023-03-20 | [23963](https://github.com/airbytehq/airbyte/pull/23963) | Add `SetupIntents` stream | +| 3.3.0 | 2023-04-12 | [25136](https://github.com/airbytehq/airbyte/pull/25136) | Add stream `Accounts` | +| 3.2.0 | 2023-04-10 | [23624](https://github.com/airbytehq/airbyte/pull/23624) | Add new stream `Subscription Schedule` | +| 3.1.0 | 2023-03-10 | [19906](https://github.com/airbytehq/airbyte/pull/19906) | Expand `tiers` when syncing `Plans` streams | +| 3.0.5 | 2023-03-25 | [22866](https://github.com/airbytehq/airbyte/pull/22866) | Specified date formatting in specification | +| 3.0.4 | 2023-03-24 | [24471](https://github.com/airbytehq/airbyte/pull/24471) | Fix stream slices for single sliced streams | +| 3.0.3 | 2023-03-17 | [24179](https://github.com/airbytehq/airbyte/pull/24179) | Get customer's attributes safely | +| 3.0.2 | 2023-03-13 | [24051](https://github.com/airbytehq/airbyte/pull/24051) | Cache `customers` stream; Do not request transactions of customers with zero balance. | +| 3.0.1 | 2023-02-22 | [22898](https://github.com/airbytehq/airbyte/pull/22898) | Add missing column to Subscriptions stream | +| 3.0.0 | 2023-02-21 | [23295](https://github.com/airbytehq/airbyte/pull/23295) | Fix invoice schema | +| 2.0.0 | 2023-02-14 | [22312](https://github.com/airbytehq/airbyte/pull/22312) | Another fix of `Invoices` stream schema + Remove http urls from openapi_spec.json | +| 1.0.2 | 2023-02-09 | [22659](https://github.com/airbytehq/airbyte/pull/22659) | Set `AvailabilityStrategy` for all streams | +| 1.0.1 | 2023-01-27 | [22042](https://github.com/airbytehq/airbyte/pull/22042) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 1.0.0 | 2023-01-25 | [21858](https://github.com/airbytehq/airbyte/pull/21858) | Update the `Subscriptions` and `Invoices` stream schemas | +| 0.1.40 | 2022-10-20 | [18228](https://github.com/airbytehq/airbyte/pull/18228) | Update the `PaymentIntents` stream schema | +| 0.1.39 | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream states. | +| 0.1.38 | 2022-09-09 | [16537](https://github.com/airbytehq/airbyte/pull/16537) | Fix `redeem_by` field type for `customers` stream | +| 0.1.37 | 2022-08-16 | [15686](https://github.com/airbytehq/airbyte/pull/15686) | Fix the bug when the stream couldn't be fetched due to limited permission set, if so - it should be skipped | +| 0.1.36 | 2022-08-04 | [15292](https://github.com/airbytehq/airbyte/pull/15292) | Implement slicing | +| 0.1.35 | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from spec and schema | +| 0.1.34 | 2022-07-01 | [14357](https://github.com/airbytehq/airbyte/pull/14357) | Add external account streams - | +| 0.1.33 | 2022-06-06 | [13449](https://github.com/airbytehq/airbyte/pull/13449) | Add semi-incremental support for CheckoutSessions and CheckoutSessionsLineItems streams, fixed big in StripeSubStream, added unittests, updated docs | +| 0.1.32 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | +| 0.1.31 | 2022-04-20 | [12230](https://github.com/airbytehq/airbyte/pull/12230) | Update connector to use a `spec.yaml` | +| 0.1.30 | 2022-03-21 | [11286](https://github.com/airbytehq/airbyte/pull/11286) | Minor corrections to documentation and connector specification | +| 0.1.29 | 2022-03-08 | [10359](https://github.com/airbytehq/airbyte/pull/10359) | Improved performance for streams with substreams: invoice_line_items, subscription_items, bank_accounts | +| 0.1.28 | 2022-02-08 | [10165](https://github.com/airbytehq/airbyte/pull/10165) | Improve 404 handling for `CheckoutSessionsLineItems` stream | +| 0.1.27 | 2021-12-28 | [9148](https://github.com/airbytehq/airbyte/pull/9148) | Fix `date`, `arrival\_date` fields | +| 0.1.26 | 2021-12-21 | [8992](https://github.com/airbytehq/airbyte/pull/8992) | Fix type `events.request` in schema | +| 0.1.25 | 2021-11-25 | [8250](https://github.com/airbytehq/airbyte/pull/8250) | Rearrange setup fields | +| 0.1.24 | 2021-11-08 | [7729](https://github.com/airbytehq/airbyte/pull/7729) | Include tax data in `checkout_sessions_line_items` stream | +| 0.1.23 | 2021-11-08 | [7729](https://github.com/airbytehq/airbyte/pull/7729) | Correct `payment_intents` schema | +| 0.1.22 | 2021-11-05 | [7345](https://github.com/airbytehq/airbyte/pull/7345) | Add 3 new streams | +| 0.1.21 | 2021-10-07 | [6841](https://github.com/airbytehq/airbyte/pull/6841) | Fix missing `start_date` argument + update json files for SAT | +| 0.1.20 | 2021-09-30 | [6017](https://github.com/airbytehq/airbyte/pull/6017) | Add lookback_window_days parameter | +| 0.1.19 | 2021-09-27 | [6466](https://github.com/airbytehq/airbyte/pull/6466) | Use `start_date` parameter in incremental streams | +| 0.1.18 | 2021-09-14 | [6004](https://github.com/airbytehq/airbyte/pull/6004) | Fix coupons and subscriptions stream schemas by removing incorrect timestamp formatting | +| 0.1.17 | 2021-09-14 | [6004](https://github.com/airbytehq/airbyte/pull/6004) | Add `PaymentIntents` stream | +| 0.1.16 | 2021-07-28 | [4980](https://github.com/airbytehq/airbyte/pull/4980) | Remove Updated field from schemas | +| 0.1.15 | 2021-07-21 | [4878](https://github.com/airbytehq/airbyte/pull/4878) | Fix incorrect percent_off and discounts data filed types | +| 0.1.14 | 2021-07-09 | [4669](https://github.com/airbytehq/airbyte/pull/4669) | Subscriptions Stream now returns all kinds of subscriptions \(including expired and canceled\) | +| 0.1.13 | 2021-07-03 | [4528](https://github.com/airbytehq/airbyte/pull/4528) | Remove regex for acc validation | +| 0.1.12 | 2021-06-08 | [3973](https://github.com/airbytehq/airbyte/pull/3973) | Add `AIRBYTE_ENTRYPOINT` for Kubernetes support | +| 0.1.11 | 2021-05-30 | [3744](https://github.com/airbytehq/airbyte/pull/3744) | Fix types in schema | +| 0.1.10 | 2021-05-28 | [3728](https://github.com/airbytehq/airbyte/pull/3728) | Update data types to be number instead of int | +| 0.1.9 | 2021-05-13 | [3367](https://github.com/airbytehq/airbyte/pull/3367) | Add acceptance tests for connected accounts | +| 0.1.8 | 2021-05-11 | [3566](https://github.com/airbytehq/airbyte/pull/3368) | Bump CDK connectors | +
      diff --git a/docs/integrations/sources/surveymonkey.md b/docs/integrations/sources/surveymonkey.md index 145beea144ca..92dbe0162518 100644 --- a/docs/integrations/sources/surveymonkey.md +++ b/docs/integrations/sources/surveymonkey.md @@ -66,6 +66,7 @@ To cover more data from this source we use caching. | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------| +| 0.2.3 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | | 0.2.2 | 2023-05-12 | [26024](https://github.com/airbytehq/airbyte/pull/26024) | Fix dependencies conflict | | 0.2.1 | 2023-04-27 | [25109](https://github.com/airbytehq/airbyte/pull/25109) | Fix add missing params to stream `SurveyResponses` | | 0.2.0 | 2023-04-18 | [23721](https://github.com/airbytehq/airbyte/pull/23721) | Add `SurveyCollectors` and `Collectors` stream | @@ -85,4 +86,4 @@ To cover more data from this source we use caching. | 0.1.3 | 2021-11-01 | [7433](https://github.com/airbytehq/airbyte/pull/7433) | Remove unsused oAuth flow parameters | | 0.1.2 | 2021-10-27 | [7433](https://github.com/airbytehq/airbyte/pull/7433) | Add OAuth support | | 0.1.1 | 2021-09-10 | [5983](https://github.com/airbytehq/airbyte/pull/5983) | Fix caching for gzip compressed http response | -| 0.1.0 | 2021-07-06 | [4097](https://github.com/airbytehq/airbyte/pull/4097) | Initial Release | +| 0.1.0 | 2021-07-06 | [4097](https://github.com/airbytehq/airbyte/pull/4097) | Initial Release | \ No newline at end of file diff --git a/docs/integrations/sources/teradata.md b/docs/integrations/sources/teradata.md index a8c83ab16789..518ffc40c73e 100644 --- a/docs/integrations/sources/teradata.md +++ b/docs/integrations/sources/teradata.md @@ -63,4 +63,5 @@ You need a Teradata user which has read permissions on the database | Version | Date | Pull Request | Subject | |:--------|:-----------|:------------------------------------------------|:----------------------------| +| 0.2.0 | 2023-12-18 | https://github.com/airbytehq/airbyte/pull/33485 | Remove LEGACY state | | 0.1.0 | 2022-03-27 | https://github.com/airbytehq/airbyte/pull/24221 | New Source Teradata Vantage | \ No newline at end of file diff --git a/docs/integrations/sources/tidb.md b/docs/integrations/sources/tidb.md index 3007c8b0de44..4078f83ccfe8 100644 --- a/docs/integrations/sources/tidb.md +++ b/docs/integrations/sources/tidb.md @@ -126,18 +126,19 @@ Now that you have set up the TiDB source connector, check out the following TiDB ## Changelog -| Version | Date | Pull Request | Subject | -|:--------| :--- | :----------- | ------- | -| 0.2.5 | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212) | Fix silent exception swallowing in StreamingJdbcDatabase | -| 0.2.4 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | -| 0.2.3 | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to | -| 0.2.2 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :----------- |-------------------------------------------------------------------------------------------------------------------------------------------| +| 0.3.0 | 2023-12-18 | [33485](https://github.com/airbytehq/airbyte/pull/33485) | Remove LEGACY state | +| 0.2.5 | 2023-06-20 | [27212](https://github.com/airbytehq/airbyte/pull/27212) | Fix silent exception swallowing in StreamingJdbcDatabase | +| 0.2.4 | 2023-03-22 | [20760](https://github.com/airbytehq/airbyte/pull/20760) | Removed redundant date-time datatypes formatting | +| 0.2.3 | 2023-03-06 | [23455](https://github.com/airbytehq/airbyte/pull/23455) | For network isolation, source connector accepts a list of hosts it is allowed to connect to | +| 0.2.2 | 2022-12-14 | [20436](https://github.com/airbytehq/airbyte/pull/20346) | Consolidate date/time values mapping for JDBC sources | | | 2022-10-13 | [15535](https://github.com/airbytehq/airbyte/pull/16238) | Update incremental query to avoid data missing when new data is inserted at the same time as a sync starts under non-CDC incremental mode | -| 0.2.1 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | -| 0.2.0 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. | -| 0.1.5 | 2022-07-25 | [14996](https://github.com/airbytehq/airbyte/pull/14996) | Removed additionalProperties:false from spec | -| 0.1.4 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | -| 0.1.3 | 2022-07-04 | [14243](https://github.com/airbytehq/airbyte/pull/14243) | Update JDBC string builder | -| 0.1.2 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | -| 0.1.1 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | -| 0.1.0 | 2022-04-19 | [11283](https://github.com/airbytehq/airbyte/pull/11283) | Initial Release | +| 0.2.1 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | +| 0.2.0 | 2022-07-26 | [14362](https://github.com/airbytehq/airbyte/pull/14362) | Integral columns are now discovered as int64 fields. | +| 0.1.5 | 2022-07-25 | [14996](https://github.com/airbytehq/airbyte/pull/14996) | Removed additionalProperties:false from spec | +| 0.1.4 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | +| 0.1.3 | 2022-07-04 | [14243](https://github.com/airbytehq/airbyte/pull/14243) | Update JDBC string builder | +| 0.1.2 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | +| 0.1.1 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | +| 0.1.0 | 2022-04-19 | [11283](https://github.com/airbytehq/airbyte/pull/11283) | Initial Release | diff --git a/docs/integrations/sources/tiktok-marketing.md b/docs/integrations/sources/tiktok-marketing.md index f0c772467d16..4a6b430a52a2 100644 --- a/docs/integrations/sources/tiktok-marketing.md +++ b/docs/integrations/sources/tiktok-marketing.md @@ -14,6 +14,7 @@ This page guides you through the process of setting up the TikTok Marketing sour **For Airbyte Open Source:** + For the Production environment: - Access token @@ -30,9 +31,10 @@ To access the Sandbox environment: ### Step 1: Set up TikTok -1. Create a TikTok For Business account: [Link](https://ads.tiktok.com/marketing_api/docs?rid=fgvgaumno25&id=1702715936951297) -2. (Open source only) Create developer application: [Link](https://ads.tiktok.com/marketing_api/docs?rid=fgvgaumno25&id=1702716474845185) -3. (Open source only) For a sandbox environment: create a Sandbox Ad Account [Link](https://ads.tiktok.com/marketing_api/docs?rid=fgvgaumno25&id=1701890920013825) +1. Create a TikTok For Business account: [Link](https://business-api.tiktok.com/portal/docs?rid=fgvgaumno25&id=1738855099573250) +2. Create developer application: [Link](https://business-api.tiktok.com/portal/docs?rid=fgvgaumno25&id=1738855242728450) +3. For a sandbox environment: create a Sandbox Ad Account [Link](https://business-api.tiktok.com/portal/docs?rid=fgvgaumno25&id=1738855331457026) + ### Step 2: Set up the source connector in Airbyte @@ -64,7 +66,7 @@ To access the Sandbox environment: ## Supported streams and sync modes | Stream | Environment | Key | Incremental | -| :---------------------------------------- | ------------ | ------------------------------------------ | :---------- | +|:------------------------------------------| ------------ |--------------------------------------------| :---------- | | Advertisers | Prod,Sandbox | advertiser_id | No | | AdGroups | Prod,Sandbox | adgroup_id | Yes | | Ads | Prod,Sandbox | ad_id | Yes | @@ -78,9 +80,15 @@ To access the Sandbox environment: | AdGroupsReportsHourly | Prod,Sandbox | adgroup_id, stat_time_hour | Yes | | AdGroupsReportsDaily | Prod,Sandbox | adgroup_id, stat_time_day | Yes | | AdGroupsReportsLifetime | Prod,Sandbox | adgroup_id | No | +| Audiences | Prod,Sandbox | audience_id | No | | CampaignsReportsHourly | Prod,Sandbox | campaign_id, stat_time_hour | Yes | | CampaignsReportsDaily | Prod,Sandbox | campaign_id, stat_time_day | Yes | | CampaignsReportsLifetime | Prod,Sandbox | campaign_id | No | +| CreativeAssetsImages | Prod,Sandbox | image_id | Yes | +| CreativeAssetsMusic | Prod,Sandbox | music_id | Yes | +| CreativeAssetsPortfolios | Prod,Sandbox | creative_portfolio_id | No | +| CreativeAssetsVideos | Prod,Sandbox | video_id | Yes | +| AdvertiserIds | Prod | advertiser_id | Yes | | AdvertisersAudienceReportsDaily | Prod | advertiser_id, stat_time_day, gender, age | Yes | | AdvertisersAudienceReportsByCountryDaily | Prod | advertiser_id, stat_time_day, country_code | Yes | | AdvertisersAudienceReportsByPlatformDaily | Prod | advertiser_id, stat_time_day, platform | Yes | @@ -91,6 +99,7 @@ To access the Sandbox environment: | AdsAudienceReportsDaily | Prod,Sandbox | ad_id, stat_time_day, gender, age | Yes | | AdsAudienceReportsByCountryDaily | Prod,Sandbox | ad_id, stat_time_day, country_code | Yes | | AdsAudienceReportsByPlatformDaily | Prod,Sandbox | ad_id, stat_time_day, platform | Yes | +| AdsAudienceReportsByProvinceDaily | Prod,Sandbox | ad_id, stat_time_day, province_id | Yes | | CampaignsAudienceReportsDaily | Prod,Sandbox | campaign_id, stat_time_day, gender, age | Yes | | CampaignsAudienceReportsByCountryDaily | Prod,Sandbox | campaign_id, stat_time_day, country_code | Yes | | CampaignsAudienceReportsByPlatformDaily | Prod,Sandbox | campaign_id, stat_time_day, platform | Yes | @@ -106,514 +115,55 @@ It is recommended to use higher values of attribution window (used in Incrementa Reports synced by this connector can use either hourly, daily, or lifetime granularities for aggregating performance data. For example, if you select the daily-aggregation flavor of a report, the report will contain a row for each day for the duration of the report. Each row will indicate the number of impressions recorded on that day. -### Output Schemas - -**[Advertisers](https://ads.tiktok.com/marketing_api/docs?id=1708503202263042) Stream** - -``` -{ - "contacter": "Ai***te", - "phonenumber": "+13*****5753", - "license_no": "", - "promotion_center_city": null, - "balance": 10, - "license_url": null, - "timezone": "Etc/GMT+8", - "reason": "", - "telephone": "+14*****6785", - "id": 7002238017842757633, - "language": "en", - "country": "US", - "role": "ROLE_ADVERTISER", - "license_province": null, - "display_timezone": "America/Los_Angeles", - "email": "i***************@**********", - "license_city": null, - "industry": "291905", - "create_time": 1630335591, - "promotion_center_province": null, - "address": "350 29th avenue, San Francisco", - "currency": "USD", - "promotion_area": "0", - "status": "STATUS_ENABLE", - "description": "https://", - "brand": null, - "name": "Airbyte0830", - "company": "Airbyte" -} -``` - -**[AdGroups](https://ads.tiktok.com/marketing_api/docs?id=1708503489590273) Stream** - -``` -{ - "placement_type": "PLACEMENT_TYPE_AUTOMATIC", - "budget": 20, - "budget_mode": "BUDGET_MODE_DAY", - "display_mode": null, - "schedule_infos": null, - "billing_event": "CPC", - "conversion_window": null, - "adgroup_name": "Ad Group20211020010107", - "interest_keywords": [], - "is_comment_disable": 0, - "rf_buy_type": null, - "frequency": null, - "bid_type": "BID_TYPE_NO_BID", - "placement": null, - "bid": 0, - "include_custom_actions": [], - "operation_system": [], - "pixel_id": null, - "dayparting": "111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", - "app_type": null, - "conversion_id": 0, - "rf_predict_cpr": null, - "deep_bid_type": null, - "scheduled_budget": 0.0, - "adgroup_id": 1714125049901106, - "frequency_schedule": null, - "exclude_custom_actions": [], - "advertiser_id": 7002238017842757633, - "deep_cpabid": 0, - "is_new_structure": true, - "buy_impression": null, - "external_type": "WEBSITE", - "excluded_audience": [], - "deep_external_action": null, - "interest_category_v2": [], - "rf_predict_frequency": null, - "audience": [], - "pacing": "PACING_MODE_SMOOTH", - "brand_safety_partner": null, - "daily_retention_ratio": null, - "optimize_goal": "CLICK", - "enable_search_result": false, - "conversion_bid": 0, - "schedule_end_time": "2021-10-31 09:01:07", - "opt_status": "ENABLE", - "status": "ADGROUP_STATUS_CAMPAIGN_DISABLE", - "app_id": null, - "external_action": null, - "schedule_type": "SCHEDULE_START_END", - "brand_safety": "NO_BRAND_SAFETY", - "campaign_id": 1714125042508817, - "campaign_name": "Website Traffic20211020010104", - "split_test_adgroup_ids": [], - "action_v2": [], - "is_hfss": false, - "keywords": null, - "create_time": "2021-10-20 08:04:05", - "feed_type": null, - "languages": ["en"], - "enable_inventory_filter": false, - "device_price": [], - "location": [6252001], - "schedule_start_time": "2021-10-20 09:01:07", - "skip_learning_phase": 0, - "gender": "GENDER_UNLIMITED", - "creative_material_mode": "CUSTOM", - "app_download_url": null, - "device_models": [], - "automated_targeting": "OFF", - "connection_type": [], - "ios14_quota_type": "UNOCCUPIED", - "modify_time": "2022-03-24 12:06:54", - "category": 0, - "statistic_type": null, - "video_download": "ALLOW_DOWNLOAD", - "age": ["AGE_25_34", "AGE_35_44", "AGE_45_54"], - "buy_reach": null, - "is_share_disable": false -} -``` - -**[Ads](https://ads.tiktok.com/marketing_api/docs?id=1708572923161602) Stream** - -``` -{ - "vast_moat": false, - "is_new_structure": true, - "campaign_name": "CampaignVadimTraffic", - "landing_page_urls": null, - "card_id": null, - "adgroup_id": 1728545385226289, - "campaign_id": 1728545382536225, - "status": "AD_STATUS_CAMPAIGN_DISABLE", - "brand_safety_postbid_partner": "UNSET", - "advertiser_id": 7002238017842757633, - "is_aco": false, - "ad_text": "Open-source\ndata integration for modern data teams", - "identity_id": "7080121820963422209", - "display_name": "airbyte", - "open_url": "", - "external_action": null, - "playable_url": "", - "create_time": "2022-03-28 12:09:09", - "product_ids": [], - "adgroup_name": "AdGroupVadim", - "fallback_type": "UNSET", - "creative_type": null, - "ad_name": "AdVadim-Optimized Version 3_202203281449_2022-03-28 05:03:44", - "video_id": "v10033g50000c90q1d3c77ub6e96fvo0", - "ad_format": "SINGLE_VIDEO", - "profile_image": "https://p21-ad-sg.ibyteimg.com/large/ad-site-i18n-sg/202203285d0de5c114d0690a462bb6a4", - "open_url_type": "NORMAL", - "click_tracking_url": null, - "page_id": null, - "ad_texts": null, - "landing_page_url": "https://airbyte.com", - "identity_type": "CUSTOMIZED_USER", - "avatar_icon_web_uri": "ad-site-i18n-sg/202203285d0de5c114d0690a462bb6a4", - "app_name": "", - "modify_time": "2022-03-28 21:34:26", - "opt_status": "ENABLE", - "call_to_action_id": "7080120957230238722", - "image_ids": ["v0201/7f371ff6f0764f8b8ef4f37d7b980d50"], - "ad_id": 1728545390695442, - "impression_tracking_url": null, - "is_creative_authorized": false -} -``` - -**[Campaigns](https://ads.tiktok.com/marketing_api/docs?id=1708582970809346) Stream** - -``` -{ - "create_time": "2021-10-19 18:18:08", - "campaign_id": 1714073078669329, - "roas_bid": 0.0, - "advertiser_id": 7002238017842757633, - "modify_time": "2022-03-28 12:01:56", - "campaign_type": "REGULAR_CAMPAIGN", - "status": "CAMPAIGN_STATUS_DISABLE", - "objective_type": "TRAFFIC", - "split_test_variable": null, - "opt_status": "DISABLE", - "budget": 50, - "is_new_structure": true, - "deep_bid_type": null, - "campaign_name": "Website Traffic20211019110444", - "budget_mode": "BUDGET_MODE_DAY", - "objective": "LANDING_PAGE" -} -``` - -**AdsReportsDaily Stream - [BasicReports](https://ads.tiktok.com/marketing_api/docs?id=1707957200780290)** - -``` -{ - "dimensions": { - "ad_id": 1728545390695442, - "stat_time_day": "2022-03-29 00:00:00" - }, - "metrics": { - "real_time_result_rate": 0.93, - "campaign_id": 1728545382536225, - "placement": "Automatic Placement", - "frequency": 1.17, - "cpc": 0.35, - "ctr": 0.93, - "cost_per_result": 0.3509, - "impressions": 6137, - "cost_per_conversion": 0, - "real_time_result": 57, - "adgroup_id": 1728545385226289, - "result_rate": 0.93, - "cost_per_1000_reached": 3.801, - "ad_text": "Open-source\ndata integration for modern data teams", - "spend": 20, - "conversion_rate": 0, - "real_time_cost_per_conversion": 0, - "promotion_type": "Website", - "tt_app_id": 0, - "real_time_cost_per_result": 0.3509, - "conversion": 0, - "secondary_goal_result": null, - "campaign_name": "CampaignVadimTraffic", - "cpm": 3.26, - "result": 57, - "ad_name": "AdVadim-Optimized Version 3_202203281449_2022-03-28 05:03:44", - "secondary_goal_result_rate": null, - "clicks": 57, - "reach": 5262, - "cost_per_secondary_goal_result": null, - "real_time_conversion": 0, - "real_time_conversion_rate": 0, - "mobile_app_id": "0", - "tt_app_name": "0", - "adgroup_name": "AdGroupVadim", - "dpa_target_audience_type": null - } -} -``` - -**AdvertisersReportsDaily Stream - [BasicReports](https://ads.tiktok.com/marketing_api/docs?id=1707957200780290)** - -``` -{ - "metrics": { - "cpm": 5.43, - "impressions": 3682, - "frequency": 1.17, - "reach": 3156, - "cash_spend": 20, - "ctr": 1.14, - "spend": 20, - "cpc": 0.48, - "cost_per_1000_reached": 6.337, - "clicks": 42, - "voucher_spend": 0 - }, - "dimensions": { - "stat_time_day": "2022-03-30 00:00:00", - "advertiser_id": 7002238017842757633 - } -} - -``` - -**AdGroupsReportsDaily Stream - [BasicReports](https://ads.tiktok.com/marketing_api/docs?id=1707957200780290)** - -``` -{ - "metrics": { - "real_time_conversion": 0, - "real_time_cost_per_conversion": 0, - "cost_per_1000_reached": 3.801, - "mobile_app_id": "0", - "reach": 5262, - "cpm": 3.26, - "conversion": 0, - "promotion_type": "Website", - "clicks": 57, - "real_time_result_rate": 0.93, - "real_time_conversion_rate": 0, - "cost_per_conversion": 0, - "dpa_target_audience_type": null, - "result": 57, - "cpc": 0.35, - "impressions": 6137, - "cost_per_result": 0.3509, - "tt_app_id": 0, - "cost_per_secondary_goal_result": null, - "frequency": 1.17, - "spend": 20, - "secondary_goal_result_rate": null, - "real_time_cost_per_result": 0.3509, - "real_time_result": 57, - "placement": "Automatic Placement", - "result_rate": 0.93, - "tt_app_name": "0", - "campaign_name": "CampaignVadimTraffic", - "secondary_goal_result": null, - "campaign_id": 1728545382536225, - "conversion_rate": 0, - "ctr": 0.93, - "adgroup_name": "AdGroupVadim" - }, - "dimensions": { - "adgroup_id": 1728545385226289, - "stat_time_day": "2022-03-29 00:00:00" - } -} -``` - -**CampaignsReportsDaily Stream - [BasicReports](https://ads.tiktok.com/marketing_api/docs?id=1707957200780290)** - -``` -{ - "metrics": { - "cpc": 0.43, - "spend": 20, - "clicks": 46, - "cost_per_1000_reached": 4.002, - "impressions": 5870, - "ctr": 0.78, - "frequency": 1.17, - "cpm": 3.41, - "campaign_name": "CampaignVadimTraffic", - "reach": 4997 - }, - "dimensions": { - "campaign_id": 1728545382536225, - "stat_time_day": "2022-03-28 00:00:00" - } -} - -``` - -**AdsAudienceReportsDaily Stream - [AudienceReports](https://ads.tiktok.com/marketing_api/docs?id=1707957217727489)** - -``` -{ - { - "result": 17, - "clicks": 17, - "real_time_conversion_rate": 0, - "adgroup_id": 1728545385226289, - "cpm": 3.01, - "cost_per_result": 0.4165, - "real_time_cost_per_result": 0.4165, - "mobile_app_id": 0, - "spend": 7.08, - "cpc": 0.42, - "placement": "Automatic Placement", - "real_time_conversion": 0, - "dpa_target_audience_type": null, - "real_time_result_rate": 0.72, - "adgroup_name": "AdGroupVadim", - "tt_app_id": 0, - "ctr": 0.72, - "ad_text": "Open-source\ndata integration for modern data teams", - "result_rate": 0.72, - "ad_name": "AdVadim-Optimized Version 3_202203281449_2022-03-28 05:03:44", - "conversion_rate": 0, - "real_time_result": 17, - "tt_app_name": "0", - "cost_per_conversion": 0, - "real_time_cost_per_conversion": 0, - "conversion": 0, - "impressions": 2350, - "promotion_type": "Website", - "campaign_id": 1728545382536225, - "campaign_name": "CampaignVadimTraffic" - }, - "dimensions": { - "gender": "MALE", - "age": "AGE_25_34", - "ad_id": 1728545390695442, - "stat_time_day": "2022-03-28 00:00:00" - } -} -``` - -**AdvertisersAudienceReportsDaily Stream - [AudienceReports](https://ads.tiktok.com/marketing_api/docs?id=1707957217727489)** - -``` -{ - "dimensions": { - "stat_time_day": "2022-03-28 00:00:00", - "gender": "FEMALE", - "advertiser_id": 7002238017842757633, - "age": "AGE_35_44" - }, - "metrics": { - "spend": 3.09, - "ctr": 0.93, - "cpc": 0.44, - "clicks": 7, - "cpm": 4.11, - "impressions": 752 - } -} -``` - -**AdGroupAudienceReportsDaily Stream - [AudienceReports](https://ads.tiktok.com/marketing_api/docs?id=1707957217727489)** - -``` -{ - "dimensions": { - "gender": "MALE", - "age": "AGE_25_34", - "stat_time_day": "2022-03-29 00:00:00", - "adgroup_id": 1728545385226289 - }, - "metrics": { - "cost_per_conversion": 0, - "campaign_id": 1728545382536225, - "campaign_name": "CampaignVadimTraffic", - "clicks": 20, - "dpa_target_audience_type": null, - "mobile_app_id": "0", - "promotion_type": "Website", - "conversion_rate": 0, - "cpm": 3.9, - "cost_per_result": 0.3525, - "cpc": 0.35, - "real_time_cost_per_conversion": 0, - "ctr": 1.11, - "spend": 7.05, - "result": 20, - "real_time_result": 20, - "impressions": 1806, - "conversion": 0, - "real_time_result_rate": 1.11, - "real_time_conversion_rate": 0, - "real_time_conversion": 0, - "adgroup_name": "AdGroupVadim", - "tt_app_name": "0", - "placement": "Automatic Placement", - "real_time_cost_per_result": 0.3525, - "result_rate": 1.11, - "tt_app_id": 0 - } -} -``` - -**CampaignsAudienceReportsByCountryDaily Stream - [AudienceReports](https://ads.tiktok.com/marketing_api/docs?id=1707957217727489)** - -``` -{ - "metrics": { - "impressions": 5870, - "campaign_name": "CampaignVadimTraffic", - "cpm": 3.41, - "clicks": 46, - "spend": 20, - "ctr": 0.78, - "cpc": 0.43 - }, - "dimensions": { - "stat_time_day": "2022-03-28 00:00:00", - "campaign_id": 1728545382536225, - "country_code": "US" - } -} - -``` - ## Performance considerations -The connector is restricted by [requests limitation](https://ads.tiktok.com/marketing_api/docs?rid=fgvgaumno25&id=1725359439428610). This connector should not run into TikTok Marketing API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. +The connector is restricted by [requests limitation](https://business-api.tiktok.com/portal/docs?rid=fgvgaumno25&id=1740029171730433). This connector should not run into TikTok Marketing API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------| -| 3.4.1 | 2023-08-04 | [29083](https://github.com/airbytehq/airbyte/pull/29083) | Added new `is_smart_performance_campaign` property to `ad groups` stream schema | -| 3.4.0 | 2023-07-13 | [27910](https://github.com/airbytehq/airbyte/pull/27910) | Added `include_deleted` config param - include deleted `ad_groups`, `ad`, `campaigns` to reports | -| 3.3.1 | 2023-07-06 | [25423](https://github.com/airbytehq/airbyte/pull/25423) | add new fields to ad reports streams | -| 3.3.0 | 2023-07-05 | [27988](https://github.com/airbytehq/airbyte/pull/27988) | Add `category_exclusion_ids` field to `ad_groups` schema. | -| 3.2.1 | 2023-05-26 | [26569](https://github.com/airbytehq/airbyte/pull/26569) | Fixed syncs with `advertiser_id` provided in input configuration | -| 3.2.0 | 2023-05-25 | [26565](https://github.com/airbytehq/airbyte/pull/26565) | Change default value for `attribution window` to 3 days; add min/max validation | -| 3.1.0 | 2023-05-12 | [26024](https://github.com/airbytehq/airbyte/pull/26024) | Updated the `Ads` stream schema | -| 3.0.1 | 2023-04-07 | [24712](https://github.com/airbytehq/airbyte/pull/24712) | Added `attribution window` for \*-reports streams | -| 3.0.0 | 2023-03-29 | [24630](https://github.com/airbytehq/airbyte/pull/24630) | Migrate to v1.3 API | -| 2.0.6 | 2023-03-30 | [22134](https://github.com/airbytehq/airbyte/pull/22134) | Add `country_code` and `platform` audience reports. | -| 2.0.5 | 2023-03-29 | [22863](https://github.com/airbytehq/airbyte/pull/22863) | Specified date formatting in specification | -| 2.0.4 | 2023-02-23 | [22309](https://github.com/airbytehq/airbyte/pull/22309) | Add Advertiser ID to filter reports and streams | -| 2.0.3 | 2023-02-15 | [23091](https://github.com/airbytehq/airbyte/pull/23091) | Add more clear log message for 504 error | -| 2.0.2 | 2023-02-02 | [22309](https://github.com/airbytehq/airbyte/pull/22309) | Chunk Advertiser IDs | -| 2.0.1 | 2023-01-27 | [22044](https://github.com/airbytehq/airbyte/pull/22044) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| 2.0.0 | 2022-12-20 | [20415](https://github.com/airbytehq/airbyte/pull/20415) | Update schema types for `AudienceReports` and `BasicReports` streams. | -| 1.0.1 | 2022-12-16 | [20598](https://github.com/airbytehq/airbyte/pull/20598) | Remove Audience Reports with Hourly granularity due to deprecated dimension. | -| 1.0.0 | 2022-12-05 | [19758](https://github.com/airbytehq/airbyte/pull/19758) | Convert `mobile_app_id` from integer to string in AudienceReport streams. | -| 0.1.17 | 2022-10-04 | [17557](https://github.com/airbytehq/airbyte/pull/17557) | Retry error 50002 | -| 0.1.16 | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream state | -| 0.1.15 | 2022-08-30 | [16137](https://github.com/airbytehq/airbyte/pull/16137) | Fixed bug with normalization caused by unsupported nested cursor field | -| 0.1.14 | 2022-06-29 | [13890](https://github.com/airbytehq/airbyte/pull/13890) | Removed granularity config option | -| 0.1.13 | 2022-06-28 | [13650](https://github.com/airbytehq/airbyte/pull/13650) | Added video metrics to report streams | -| 0.1.12 | 2022-05-24 | [13127](https://github.com/airbytehq/airbyte/pull/13127) | Fixed integration test | -| 0.1.11 | 2022-04-27 | [12838](https://github.com/airbytehq/airbyte/pull/12838) | Added end date configuration for tiktok | -| 0.1.10 | 2022-05-07 | [12545](https://github.com/airbytehq/airbyte/pull/12545) | Removed odd production authenication method | -| 0.1.9 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | -| 0.1.8 | 2022-04-28 | [12435](https://github.com/airbytehq/airbyte/pull/12435) | updated spec descriptions | -| 0.1.7 | 2022-04-27 | [12380](https://github.com/airbytehq/airbyte/pull/12380) | fixed spec descriptions and documentation | -| 0.1.6 | 2022-04-19 | [11378](https://github.com/airbytehq/airbyte/pull/11378) | updated logic for stream initializations, fixed errors in schemas, updated SAT and unit tests | -| 0.1.5 | 2022-02-17 | [10398](https://github.com/airbytehq/airbyte/pull/10398) | Add Audience reports | -| 0.1.4 | 2021-12-30 | [7636](https://github.com/airbytehq/airbyte/pull/7636) | Add OAuth support | -| 0.1.3 | 2021-12-10 | [8425](https://github.com/airbytehq/airbyte/pull/8425) | Update title, description fields in spec | -| 0.1.2 | 2021-12-02 | [8292](https://github.com/airbytehq/airbyte/pull/8292) | Support reports | -| 0.1.1 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | -| 0.1.0 | 2021-09-18 | [5887](https://github.com/airbytehq/airbyte/pull/5887) | Release TikTok Marketing CDK Connector | +| Version | Date | Pull Request | Subject | +|:--------|:-----------| :------------------------------------------------------- |:------------------------------------------------------------------------------------------------------------| +| 3.9.2 | 2023-11-02 | [32091](https://github.com/airbytehq/airbyte/pull/32091) | Fix incremental syncs; update docs; fix field type of `preview_url_expire_time` to `date-time`. | +| 3.9.1 | 2023-10-25 | [31812](https://github.com/airbytehq/airbyte/pull/31812) | Update `support level` in `metadata`, removed duplicated `tracking_pixel_id` field from `Ads` stream schema | +| 3.9.0 | 2023-10-23 | [31623](https://github.com/airbytehq/airbyte/pull/31623) | Add AdsAudienceReportsByProvince stream and expand base report metrics | +| 3.8.0 | 2023-10-19 | [31610](https://github.com/airbytehq/airbyte/pull/31610) | Add Creative Assets and Audiences streams | +| 3.7.1 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 3.7.0 | 2023-10-19 | [31493](https://github.com/airbytehq/airbyte/pull/31493) | Add fields to Ads stream | +| 3.6.0 | 2023-10-18 | [31537](https://github.com/airbytehq/airbyte/pull/31537) | Use default availability strategy | +| 3.5.0 | 2023-10-16 | [31445](https://github.com/airbytehq/airbyte/pull/31445) | Apply minimum date restrictions | +| 3.4.1 | 2023-08-04 | [29083](https://github.com/airbytehq/airbyte/pull/29083) | Added new `is_smart_performance_campaign` property to `ad groups` stream schema | +| 3.4.0 | 2023-07-13 | [27910](https://github.com/airbytehq/airbyte/pull/27910) | Added `include_deleted` config param - include deleted `ad_groups`, `ad`, `campaigns` to reports | +| 3.3.1 | 2023-07-06 | [25423](https://github.com/airbytehq/airbyte/pull/25423) | add new fields to ad reports streams | +| 3.3.0 | 2023-07-05 | [27988](https://github.com/airbytehq/airbyte/pull/27988) | Add `category_exclusion_ids` field to `ad_groups` schema. | +| 3.2.1 | 2023-05-26 | [26569](https://github.com/airbytehq/airbyte/pull/26569) | Fixed syncs with `advertiser_id` provided in input configuration | +| 3.2.0 | 2023-05-25 | [26565](https://github.com/airbytehq/airbyte/pull/26565) | Change default value for `attribution window` to 3 days; add min/max validation | +| 3.1.0 | 2023-05-12 | [26024](https://github.com/airbytehq/airbyte/pull/26024) | Updated the `Ads` stream schema | +| 3.0.1 | 2023-04-07 | [24712](https://github.com/airbytehq/airbyte/pull/24712) | Added `attribution window` for \*-reports streams | +| 3.0.0 | 2023-03-29 | [24630](https://github.com/airbytehq/airbyte/pull/24630) | Migrate to v1.3 API | +| 2.0.6 | 2023-03-30 | [22134](https://github.com/airbytehq/airbyte/pull/22134) | Add `country_code` and `platform` audience reports. | +| 2.0.5 | 2023-03-29 | [22863](https://github.com/airbytehq/airbyte/pull/22863) | Specified date formatting in specification | +| 2.0.4 | 2023-02-23 | [22309](https://github.com/airbytehq/airbyte/pull/22309) | Add Advertiser ID to filter reports and streams | +| 2.0.3 | 2023-02-15 | [23091](https://github.com/airbytehq/airbyte/pull/23091) | Add more clear log message for 504 error | +| 2.0.2 | 2023-02-02 | [22309](https://github.com/airbytehq/airbyte/pull/22309) | Chunk Advertiser IDs | +| 2.0.1 | 2023-01-27 | [22044](https://github.com/airbytehq/airbyte/pull/22044) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 2.0.0 | 2022-12-20 | [20415](https://github.com/airbytehq/airbyte/pull/20415) | Update schema types for `AudienceReports` and `BasicReports` streams. | +| 1.0.1 | 2022-12-16 | [20598](https://github.com/airbytehq/airbyte/pull/20598) | Remove Audience Reports with Hourly granularity due to deprecated dimension. | +| 1.0.0 | 2022-12-05 | [19758](https://github.com/airbytehq/airbyte/pull/19758) | Convert `mobile_app_id` from integer to string in AudienceReport streams. | +| 0.1.17 | 2022-10-04 | [17557](https://github.com/airbytehq/airbyte/pull/17557) | Retry error 50002 | +| 0.1.16 | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream state | +| 0.1.15 | 2022-08-30 | [16137](https://github.com/airbytehq/airbyte/pull/16137) | Fixed bug with normalization caused by unsupported nested cursor field | +| 0.1.14 | 2022-06-29 | [13890](https://github.com/airbytehq/airbyte/pull/13890) | Removed granularity config option | +| 0.1.13 | 2022-06-28 | [13650](https://github.com/airbytehq/airbyte/pull/13650) | Added video metrics to report streams | +| 0.1.12 | 2022-05-24 | [13127](https://github.com/airbytehq/airbyte/pull/13127) | Fixed integration test | +| 0.1.11 | 2022-04-27 | [12838](https://github.com/airbytehq/airbyte/pull/12838) | Added end date configuration for tiktok | +| 0.1.10 | 2022-05-07 | [12545](https://github.com/airbytehq/airbyte/pull/12545) | Removed odd production authenication method | +| 0.1.9 | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy | +| 0.1.8 | 2022-04-28 | [12435](https://github.com/airbytehq/airbyte/pull/12435) | updated spec descriptions | +| 0.1.7 | 2022-04-27 | [12380](https://github.com/airbytehq/airbyte/pull/12380) | fixed spec descriptions and documentation | +| 0.1.6 | 2022-04-19 | [11378](https://github.com/airbytehq/airbyte/pull/11378) | updated logic for stream initializations, fixed errors in schemas, updated SAT and unit tests | +| 0.1.5 | 2022-02-17 | [10398](https://github.com/airbytehq/airbyte/pull/10398) | Add Audience reports | +| 0.1.4 | 2021-12-30 | [7636](https://github.com/airbytehq/airbyte/pull/7636) | Add OAuth support | +| 0.1.3 | 2021-12-10 | [8425](https://github.com/airbytehq/airbyte/pull/8425) | Update title, description fields in spec | +| 0.1.2 | 2021-12-02 | [8292](https://github.com/airbytehq/airbyte/pull/8292) | Support reports | +| 0.1.1 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | +| 0.1.0 | 2021-09-18 | [5887](https://github.com/airbytehq/airbyte/pull/5887) | Release TikTok Marketing CDK Connector | diff --git a/docs/integrations/sources/timely.md b/docs/integrations/sources/timely.md index 1bff1c2ba1d7..76b76a5a229d 100644 --- a/docs/integrations/sources/timely.md +++ b/docs/integrations/sources/timely.md @@ -33,4 +33,6 @@ The Timely source connector supports the following [sync modes](https://docs.air | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :-------------- | +| 0.3.0 | 2023-10-25 | [31002](https://github.com/airbytehq/airbyte/pull/31002) | Migrate to low-code framework | +| 0.2.0 | 2023-10-23 | [31745](https://github.com/airbytehq/airbyte/pull/31745) | Fix schemas | | 0.1.0 | 2022-06-22 | [13617](https://github.com/airbytehq/airbyte/pull/13617) | Initial release | diff --git a/docs/integrations/sources/todoist.md b/docs/integrations/sources/todoist.md index 77935691155c..34578169dabc 100644 --- a/docs/integrations/sources/todoist.md +++ b/docs/integrations/sources/todoist.md @@ -44,4 +44,5 @@ List of available streams: | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------| -| 0.1.0 | 2022-12-03 | [20046](https://github.com/airbytehq/airbyte/pull/20046) | 🎉 New Source: todoist | +| 0.2.0 | 2023-12-19 | [32690](https://github.com/airbytehq/airbyte/pull/32690) | Migrate to low-code | +| 0.1.0 | 2022-12-03 | [20046](https://github.com/airbytehq/airbyte/pull/20046) | 🎉 New Source: todoist | diff --git a/docs/integrations/sources/trello-migrations.md b/docs/integrations/sources/trello-migrations.md new file mode 100644 index 000000000000..8ac92ffecb8a --- /dev/null +++ b/docs/integrations/sources/trello-migrations.md @@ -0,0 +1,7 @@ +# Trello Migration Guide + +## Upgrading to 1.0.0 + +This version upgrades the connector to the low-code framework for better maintainability. This migration includes a breaking change to the state format of the `actions` stream. + +Any connection using the `actions` stream in `incremental` mode will need to be reset after the upgrade to avoid sync failures. diff --git a/docs/integrations/sources/trello.md b/docs/integrations/sources/trello.md index de088670a6e2..08acec234f5c 100644 --- a/docs/integrations/sources/trello.md +++ b/docs/integrations/sources/trello.md @@ -76,6 +76,9 @@ The Trello connector should not run into Trello API limitations under normal usa | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------| +| 1.0.2 | 2023-10-13 | [31205](https://github.com/airbytehq/airbyte/pull/31205) | Improve spec description for board ids | +| 1.0.1 | 2023-10-13 | [31168](https://github.com/airbytehq/airbyte/pull/31168) | Fix `cards` schema | +| 1.0.0 | 2023-09-08 | [29876](https://github.com/airbytehq/airbyte/pull/29876) | Migrate to Low Code CDK | | 0.3.4 | 2023-07-31 | [28734](https://github.com/airbytehq/airbyte/pull/28734) | Updated `expected records` for CAT test and fixed `advancedAuth` broken references | | 0.3.3 | 2023-06-19 | [27470](https://github.com/airbytehq/airbyte/pull/27470) | Update Organizations schema | | 0.3.2 | 2023-05-05 | [25870](https://github.com/airbytehq/airbyte/pull/25870) | Added `CDK typeTransformer` to guarantee JSON schema types | diff --git a/docs/integrations/sources/twilio-taskrouter.md b/docs/integrations/sources/twilio-taskrouter.md index 5bcaca48b0c0..20611e93f565 100644 --- a/docs/integrations/sources/twilio-taskrouter.md +++ b/docs/integrations/sources/twilio-taskrouter.md @@ -19,7 +19,7 @@ See [docs](https://www.twilio.com/docs/taskrouter/api) for more details. 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**. -3. On the Set up the source page, enter the name for the Twilio connector and select **Twilio Taskrouter** from the type dropdown. +3. On the Set up the source page, enter the name for the Twilio connector and select **Twilio Taskrouter** from the Source/Destination type dropdown. 4. Enter your `account_sid`. 5. Enter your `auth_token`. 6. Click **Set up source**. diff --git a/docs/integrations/sources/twilio.md b/docs/integrations/sources/twilio.md index d8b8dae0faf6..5b7a7679cb92 100644 --- a/docs/integrations/sources/twilio.md +++ b/docs/integrations/sources/twilio.md @@ -17,7 +17,7 @@ See [docs](https://www.twilio.com/docs/iam/api) for more details. 1. [Log into your Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**. -3. On the Set up the source page, enter the name for the Twilio connector and select **Twilio** from the type dropdown. +3. On the Set up the source page, enter the name for the Twilio connector and select **Twilio** from the Source/Destination type dropdown. 4. Enter your `account_sid`. 5. Enter your `auth_token`. 6. Enter your `start_date`. @@ -62,7 +62,7 @@ The Twilio source connector supports the following [sync modes](https://docs.air * [Calls](https://www.twilio.com/docs/voice/api/call-resource#create-a-call-resource) \(Incremental\) * [Conference Participants](https://www.twilio.com/docs/voice/api/conference-participant-resource#read-multiple-participant-resources) \(Incremental\) * [Conferences](https://www.twilio.com/docs/voice/api/conference-resource#read-multiple-conference-resources) \(Incremental\) -* [Conversations](https://www.twilio.com/docs/conversations/api/conversation-resource#read-multiple-conversation-resources) +* [Conversations](https://www.twilio.com/docs/conversations/api/conversation-resource#read-multiple-conversation-resources) * [Conversation Messages](https://www.twilio.com/docs/conversations/api/conversation-message-resource#list-all-conversation-messages) * [Conversation Participants](https://www.twilio.com/docs/conversations/api/conversation-participant-resource) * [Dependent Phone Numbers](https://www.twilio.com/docs/usage/api/address?code-sample=code-list-dependent-pns-subresources&code-language=curl&code-sdk-version=json#instance-subresources) \(Incremental\) @@ -95,6 +95,7 @@ For more information, see [the Twilio docs for rate limitations](https://support | Version | Date | Pull Request | Subject | |:--------|:-----------|:----------------------------------------------------------|:--------------------------------------------------------------------------------------------------------| +| 0.10.1 | 2023-11-21 | [32718](https://github.com/airbytehq/airbyte/pull/32718) | Base image migration: remove Dockerfile and use the python-connector-base image | | 0.10.0 | 2023-07-28 | [27323](https://github.com/airbytehq/airbyte/pull/27323) | Add new stream `Step` | | 0.9.0 | 2023-06-27 | [27221](https://github.com/airbytehq/airbyte/pull/27221) | Add new stream `UserConversations` with parent `Users` | | 0.8.1 | 2023-07-12 | [28216](https://github.com/airbytehq/airbyte/pull/28216) | Add property `channel_metadata` to `ConversationMessages` schema | @@ -120,4 +121,4 @@ For more information, see [the Twilio docs for rate limitations](https://support | 0.1.3 | 2022-04-20 | [12183](https://github.com/airbytehq/airbyte/pull/12183) | Add new subresource on the call stream + declare a valid primary key for conference_participants stream | | 0.1.2 | 2021-12-23 | [9092](https://github.com/airbytehq/airbyte/pull/9092) | Correct specification doc URL | | 0.1.1 | 2021-10-18 | [7034](https://github.com/airbytehq/airbyte/pull/7034) | Update schemas and transform data types according to the API schema | -| 0.1.0 | 2021-07-02 | [4070](https://github.com/airbytehq/airbyte/pull/4070) | Native Twilio connector implemented | +| 0.1.0 | 2021-07-02 | [4070](https://github.com/airbytehq/airbyte/pull/4070) | Native Twilio connector implemented | \ No newline at end of file diff --git a/docs/integrations/sources/typeform-migrations.md b/docs/integrations/sources/typeform-migrations.md new file mode 100644 index 000000000000..9f9726cb55b6 --- /dev/null +++ b/docs/integrations/sources/typeform-migrations.md @@ -0,0 +1,7 @@ +# Typeform Migration Guide + +## Upgrading to 1.1.0 + +This version upgrades the connector to the low-code framework for better maintainability. This migration includes a breaking change to the state format of the `responses` stream. + +Any connection using the `responses` stream in `incremental` mode will need to be reset after the upgrade to avoid sync failures. \ No newline at end of file diff --git a/docs/integrations/sources/typeform.md b/docs/integrations/sources/typeform.md index 16c94f432aa0..18c31df86cd3 100644 --- a/docs/integrations/sources/typeform.md +++ b/docs/integrations/sources/typeform.md @@ -20,7 +20,7 @@ This page guides you through the process of setting up the Typeform source conne ## Setup guide -### Step 1: Obtain an API token +### Step 1: Obtain an API token **For Airbyte Open Source:** @@ -36,7 +36,7 @@ To get the API token for your application follow this [steps](https://developer. **For Airbyte Cloud:** -This step is not needed in Airbyte Cloud. Skip to the next step. +This step is not needed in Airbyte Cloud. Skip to the next step. ### Step 2: Set up the source connector in Airbyte @@ -88,21 +88,27 @@ API rate limits \(2 requests per second\): [https://developer.typeform.com/get-s ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------| -| 1.0.0 | 2023-06-26 | [27240](https://github.com/airbytehq/airbyte/pull/27240) | Add OAuth support | -| 0.3.0 | 2023-06-23 | [27653](https://github.com/airbytehq/airbyte/pull/27653) | Add `form_id` to records of `responses` stream | -| 0.2.0 | 2023-06-17 | [27455](https://github.com/airbytehq/airbyte/pull/27455) | Add missing schema fields in `forms`, `themes`, `images`, `workspaces`, and `responses` streams | -| 0.1.12 | 2023-02-21 | [22824](https://github.com/airbytehq/airbyte/pull/22824) | Specified date formatting in specification | -| 0.1.11 | 2023-02-20 | [23248](https://github.com/airbytehq/airbyte/pull/23248) | Store cursor value as a string | -| 0.1.10 | 2023-01-07 | [16125](https://github.com/airbytehq/airbyte/pull/16125) | Certification to Beta | -| 0.1.9 | 2022-08-30 | [16125](https://github.com/airbytehq/airbyte/pull/16125) | Improve `metadata.referer` url parsing | -| 0.1.8 | 2022-08-09 | [15435](https://github.com/airbytehq/airbyte/pull/15435) | Update Forms stream schema | -| 0.1.7 | 2022-06-20 | [13935](https://github.com/airbytehq/airbyte/pull/13935) | Update Responses stream schema | -| 0.1.6 | 2022-05-23 | [12280](https://github.com/airbytehq/airbyte/pull/12280) | Full Stream Coverage | -| 0.1.4 | 2021-12-08 | [8425](https://github.com/airbytehq/airbyte/pull/8425) | Update title, description fields in spec | -| 0.1.3 | 2021-12-07 | [8466](https://github.com/airbytehq/airbyte/pull/8466) | Change Check Connection Function Logic | -| 0.1.2 | 2021-10-11 | [6571](https://github.com/airbytehq/airbyte/pull/6571) | Support pulling data from a select set of forms | -| 0.1.1 | 2021-09-06 | [5799](https://github.com/airbytehq/airbyte/pull/5799) | Add missed choices field to responses schema | -| 0.1.0 | 2021-07-10 | [4541](https://github.com/airbytehq/airbyte/pull/4541) | Initial release for Typeform API supporting Forms and Responses streams | - +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------------| +| 1.2.3 | 2024-01-11 | [34145](https://github.com/airbytehq/airbyte/pull/34145) | prepare for airbyte-lib | +| 1.2.2 | 2023-12-12 | [33345](https://github.com/airbytehq/airbyte/pull/33345) | Fix single use refresh token authentication | +| 1.2.1 | 2023-12-04 | [32775](https://github.com/airbytehq/airbyte/pull/32775) | Add 499 status code handling | +| 1.2.0 | 2023-11-29 | [32745](https://github.com/airbytehq/airbyte/pull/32745) | Add `response_type` field to `responses` schema | +| 1.1.2 | 2023-10-27 | [31914](https://github.com/airbytehq/airbyte/pull/31914) | Fix pagination for stream Responses | +| 1.1.1 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image | +| 1.1.0 | 2023-09-04 | [29916](https://github.com/airbytehq/airbyte/pull/29916) | Migrate to Low-Code Framework | +| 1.0.0 | 2023-06-26 | [27240](https://github.com/airbytehq/airbyte/pull/27240) | Add OAuth support | +| 0.3.0 | 2023-06-23 | [27653](https://github.com/airbytehq/airbyte/pull/27653) | Add `form_id` to records of `responses` stream | +| 0.2.0 | 2023-06-17 | [27455](https://github.com/airbytehq/airbyte/pull/27455) | Add missing schema fields in `forms`, `themes`, `images`, `workspaces`, and `responses` streams | +| 0.1.12 | 2023-02-21 | [22824](https://github.com/airbytehq/airbyte/pull/22824) | Specified date formatting in specification | +| 0.1.11 | 2023-02-20 | [23248](https://github.com/airbytehq/airbyte/pull/23248) | Store cursor value as a string | +| 0.1.10 | 2023-01-07 | [16125](https://github.com/airbytehq/airbyte/pull/16125) | Certification to Beta | +| 0.1.9 | 2022-08-30 | [16125](https://github.com/airbytehq/airbyte/pull/16125) | Improve `metadata.referer` url parsing | +| 0.1.8 | 2022-08-09 | [15435](https://github.com/airbytehq/airbyte/pull/15435) | Update Forms stream schema | +| 0.1.7 | 2022-06-20 | [13935](https://github.com/airbytehq/airbyte/pull/13935) | Update Responses stream schema | +| 0.1.6 | 2022-05-23 | [12280](https://github.com/airbytehq/airbyte/pull/12280) | Full Stream Coverage | +| 0.1.4 | 2021-12-08 | [8425](https://github.com/airbytehq/airbyte/pull/8425) | Update title, description fields in spec | +| 0.1.3 | 2021-12-07 | [8466](https://github.com/airbytehq/airbyte/pull/8466) | Change Check Connection Function Logic | +| 0.1.2 | 2021-10-11 | [6571](https://github.com/airbytehq/airbyte/pull/6571) | Support pulling data from a select set of forms | +| 0.1.1 | 2021-09-06 | [5799](https://github.com/airbytehq/airbyte/pull/5799) | Add missed choices field to responses schema | +| 0.1.0 | 2021-07-10 | [4541](https://github.com/airbytehq/airbyte/pull/4541) | Initial release for Typeform API supporting Forms and Responses streams | diff --git a/docs/integrations/sources/us-census.md b/docs/integrations/sources/us-census.md index 2242f3fceda5..374c9ac16a19 100644 --- a/docs/integrations/sources/us-census.md +++ b/docs/integrations/sources/us-census.md @@ -36,8 +36,9 @@ In addition, to understand how to configure the dataset path and query parameter ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :----------------------------------------------------- | :------------------------------------------------ | -| 0.1.2 | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628) | Update fields in source-connectors specifications | -| 0.1.1 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | -| 0.1.0 | 2021-07-20 | [4228](https://github.com/airbytehq/airbyte/pull/4228) | Initial release | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------ | +| 0.1.3 | 2024-01-03 | [33890](https://github.com/airbytehq/airbyte/pull/33890) | Allow additional properties in connector spec | +| 0.1.2 | 2021-12-28 | [8628](https://github.com/airbytehq/airbyte/pull/8628) | Update fields in source-connectors specifications | +| 0.1.1 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | +| 0.1.0 | 2021-07-20 | [4228](https://github.com/airbytehq/airbyte/pull/4228) | Initial release | diff --git a/docs/integrations/sources/visma-economic.md b/docs/integrations/sources/visma-economic.md index eb5f6f6369ee..5c396a7388e0 100644 --- a/docs/integrations/sources/visma-economic.md +++ b/docs/integrations/sources/visma-economic.md @@ -51,4 +51,5 @@ For more information about the api see the [E-conomic REST API Documentation](ht | Version | Date | Pull Request | Subject | | :------ |:-----------|:----------------------------------------------------|:-----------------------------------| -| 0.1.0 | 2022-xx-xx | [18595](https://github.com/airbytehq/airbyte/pull/18595) | Adding Visma e-conomic as a source | +| 0.2.0 | 2023-10-20 | [30991](https://github.com/airbytehq/airbyte/pull/30991) | Migrate to Low-code Framework | +| 0.1.0 | 2022-11-08 | [18595](https://github.com/airbytehq/airbyte/pull/18595) | Adding Visma e-conomic as a source | diff --git a/docs/integrations/sources/webflow.md b/docs/integrations/sources/webflow.md index 7a47e7158af9..12b971cade38 100644 --- a/docs/integrations/sources/webflow.md +++ b/docs/integrations/sources/webflow.md @@ -12,7 +12,7 @@ This connector dynamically figures out which collections are available, creates # Webflow credentials -You should be able to create a Webflow `API key` (aka `API token`) as described in [Intro to the Webflow API](https://university.webflow.com/lesson/intro-to-the-webflow-api). +You should be able to create a Webflow `API key` (aka `API token`) as described in [Intro to the Webflow API](https://university.webflow.com/lesson/intro-to-the-webflow-api). The Webflow connector uses the Webflow API v1 and therefore will require a legacy v1 API key. Once you have the `API Key`/`API token`, you can confirm a [list of available sites](https://developers.webflow.com/#sites) and get their `_id` by executing the following: @@ -28,8 +28,8 @@ Which should respond with something similar to: [{"_id":"","createdOn":"2021-03-26T15:46:04.032Z","name":"Airbyte","shortName":"airbyte-dev","lastPublished":"2022-06-09T12:55:52.533Z","previewUrl":"https://screenshots.webflow.com/sites/","timezone":"America/Los_Angeles","database":""}] ``` -You will need to provide the `Site id` and `API key` to the Webflow connector in order for it to pull data from your Webflow site. - +You will need to provide the `Site ID` and `API key` to the Webflow connector in order for it to pull data from your Webflow site. + # Related tutorial If you are interested in learning more about the Webflow API and implementation details of this connector, you may wish to consult the [tutorial about how to build a connector to extract data from the Webflow API](https://airbyte.com/tutorials/extract-data-from-the-webflow-api). @@ -38,8 +38,7 @@ If you are interested in learning more about the Webflow API and implementation | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :---------------------------- | -| 0.1.2 | 2022-07-14 | [14689](https://github.com/airbytehq/airbyte/pull/14689) | Webflow add ids to streams | -| 0.1.1 | 2022-06-22 | [13617](https://github.com/airbytehq/airbyte/pull/13617) | Update Spec Documentation URL | -| 0.1.0 | 2022-06-22 | [13617](https://github.com/airbytehq/airbyte/pull/13617) | Initial release | - - \ No newline at end of file +| 0.1.3 | 2022-12-11 | [33315](https://github.com/airbytehq/airbyte/pull/33315) | Updates CDK to latest version and adds additional properties to schema | +| 0.1.2 | 2022-07-14 | [14689](https://github.com/airbytehq/airbyte/pull/14689) | Webflow added IDs to streams | +| 0.1.1 | 2022-06-22 | [13617](https://github.com/airbytehq/airbyte/pull/13617) | Updates Spec Documentation URL | +| 0.1.0 | 2022-06-22 | [13617](https://github.com/airbytehq/airbyte/pull/13617) | Initial release | \ No newline at end of file diff --git a/docs/integrations/sources/wrike.md b/docs/integrations/sources/wrike.md index 45635f67f9ac..5cc63a917ab0 100644 --- a/docs/integrations/sources/wrike.md +++ b/docs/integrations/sources/wrike.md @@ -46,5 +46,6 @@ The Wrike connector should not run into Wrike API limitations under normal usage | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.2.0 | 2023-10-10 | [31058](https://github.com/airbytehq/airbyte/pull/31058) | Migrate to low code. | 0.1.0 | 2022-08-16 | [15638](https://github.com/airbytehq/airbyte/pull/15638) | Initial version/release of the connector. diff --git a/docs/integrations/sources/xero.md b/docs/integrations/sources/xero.md index 2be1f4413618..1e049713fcf7 100644 --- a/docs/integrations/sources/xero.md +++ b/docs/integrations/sources/xero.md @@ -7,6 +7,18 @@ This page contains the setup guide and reference information for the Xero source - Tenant ID - Start Date +**Required list of scopes to sync all streams:** +- accounting.attachments.read +- accounting.budgets.read +- accounting.contacts.read +- accounting.journals.read +- accounting.reports.read +- accounting.reports.tenninetynine.read +- accounting.settings.read +- accounting.transactions.read +- assets.read +- offline_access + **For Airbyte Cloud:** @@ -16,6 +28,7 @@ This page contains the setup guide and reference information for the Xero source **For Airbyte Open Source:** +Please follow [instruction](https://developer.xero.com/documentation/guides/oauth2/auth-flow/) to obtain all requirements: - Client ID - Client Secret - Refresh Token @@ -91,6 +104,8 @@ The connector is restricted by Xero [API rate limits](https://developer.xero.com | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:----------------------------------| +| 0.2.5 | 2024-01-11 | [34154](https://github.com/airbytehq/airbyte/pull/34154) | prepare for airbyte-lib | +| 0.2.4 | 2023-11-24 | [32837](https://github.com/airbytehq/airbyte/pull/32837) | Handle 403 error | | 0.2.3 | 2023-06-19 | [27471](https://github.com/airbytehq/airbyte/pull/27471) | Update CDK to 0.40 | | 0.2.2 | 2023-06-06 | [27007](https://github.com/airbytehq/airbyte/pull/27007) | Update CDK | | 0.2.1 | 2023-03-20 | [24217](https://github.com/airbytehq/airbyte/pull/24217) | Certify to Beta | diff --git a/docs/integrations/sources/yandex-metrica.md b/docs/integrations/sources/yandex-metrica.md index e38e7c8404b0..5f958e16cf74 100644 --- a/docs/integrations/sources/yandex-metrica.md +++ b/docs/integrations/sources/yandex-metrica.md @@ -20,7 +20,7 @@ This page contains the setup guide and reference information for the Yandex Metr - What data do you need?: **Yandex.Metrica**. Read permission will suffice. 5. Choose your app from [the list](https://oauth.yandex.com/). - To create your API key you will need to grab your **ClientID**, - - Now to get the API key craft a GET request to an endpoint *https://oauth.yandex.com/authorizE?response_type=token&client_id=\* + - Now to get the API key craft a GET request to an endpoint *https://oauth.yandex.com/authorizE?response_type=token&client_id=YOUR_CLIENT_ID* - You will receive a response with your **API key**. Save it. ### Step 2: Set up the Yandex Metrica connector in Airbyte diff --git a/docs/integrations/sources/younium.md b/docs/integrations/sources/younium.md index 60d510a0e700..b3e242b80fa4 100644 --- a/docs/integrations/sources/younium.md +++ b/docs/integrations/sources/younium.md @@ -33,12 +33,16 @@ The Younium source connector supports the following [sync modes](https://docs.ai ## Supported Streams +- [Accounts](https://developer.younium.com/api-details#api=Production_API2-0&operation=Get-Accounts) +- [Bookings](https://developer.younium.com/api-details#api=Production_API2-0&operation=Get-Bookings) - [Subscriptions](https://developer.younium.com/api-details#api=Production_API2-0&operation=Get-Subscriptions) - [Products](https://developer.younium.com/api-details#api=Production_API2-0&operation=Get-Products) - [Invoices](https://developer.younium.com/api-details#api=Production_API2-0&operation=Get-Invoices) ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :---------------------------------- | -| 0.1.0 | 2022-11-09 | [18758](https://github.com/airbytehq/airbyte/pull/18758) | 🎉 New Source: Younium [python cdk] | +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- |:---------------------------------------------------| +| 0.3.0 | 2023-10-25 | [31690](https://github.com/airbytehq/airbyte/pull/31690) | Migrate to low-code framework | +| 0.2.0 | 2023-03-29 | [24655](https://github.com/airbytehq/airbyte/pull/24655) | Source Younium: Adding Booking and Account streams | +| 0.1.0 | 2022-11-09 | [18758](https://github.com/airbytehq/airbyte/pull/18758) | 🎉 New Source: Younium [python cdk] | diff --git a/docs/integrations/sources/zendesk-chat.md b/docs/integrations/sources/zendesk-chat.md index eab821953fd8..8075e4a44869 100644 --- a/docs/integrations/sources/zendesk-chat.md +++ b/docs/integrations/sources/zendesk-chat.md @@ -80,6 +80,8 @@ The connector is restricted by Zendesk's [requests limitation](https://developer | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------- | +| 0.2.1 | 2023-10-20 | [31643](https://github.com/airbytehq/airbyte/pull/31643) | Upgrade base image to airbyte/python-connector-base:1.1.0 | +| 0.2.0 | 2023-10-11 | [30526](https://github.com/airbytehq/airbyte/pull/30526) | Use the python connector base image, remove dockerfile and implement build_customization.py | | 0.1.14 | 2023-02-10 | [24190](https://github.com/airbytehq/airbyte/pull/24190) | Fix remove too high min/max from account stream | | 0.1.13 | 2023-02-10 | [22819](https://github.com/airbytehq/airbyte/pull/22819) | Specified date formatting in specification | | 0.1.12 | 2023-01-27 | [22026](https://github.com/airbytehq/airbyte/pull/22026) | Set `AvailabilityStrategy` for streams explicitly to `None` | diff --git a/docs/integrations/sources/zendesk-sell.md b/docs/integrations/sources/zendesk-sell.md index 46a5b3afed77..edd602fab4d9 100644 --- a/docs/integrations/sources/zendesk-sell.md +++ b/docs/integrations/sources/zendesk-sell.md @@ -76,4 +76,7 @@ We recommend creating a restricted, read-only key specifically for Airbyte acces | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | | 0.1.0 | 2022-10-27 | [17888](https://github.com/airbytehq/airbyte/pull/17888) | Initial Release | +| 0.1.1 | 2023-08-30 | [29830](https://github.com/airbytehq/airbyte/pull/29830) | Change phone_number in Calls to string (bug in zendesk sell api documentation) | +| 0.2.0 | 2023-10-23 | [31016](https://github.com/airbytehq/airbyte/pull/31016) | Migrated to Low Code CDK | + diff --git a/docs/integrations/sources/zendesk-support-migrations.md b/docs/integrations/sources/zendesk-support-migrations.md index e6d28f511b32..c157e43f6baa 100644 --- a/docs/integrations/sources/zendesk-support-migrations.md +++ b/docs/integrations/sources/zendesk-support-migrations.md @@ -1,5 +1,9 @@ # Zendesk Support Migration Guide +## Upgrading to 2.0.0 + +Stream `Deleted Tickets` is removed. You may need to refresh the connection schema (skipping the reset), and running a sync. Alternatively, you can just run a reset. + ## Upgrading to 1.0.0 `cursor_field` for `Tickets` stream is changed to `generated_timestamp`. diff --git a/docs/integrations/sources/zendesk-support.inapp.md b/docs/integrations/sources/zendesk-support.inapp.md deleted file mode 100644 index e62c88b8a182..000000000000 --- a/docs/integrations/sources/zendesk-support.inapp.md +++ /dev/null @@ -1,27 +0,0 @@ -## Prerequisites - -- A Zendesk account with an Administrator role. - -## Setup Guide - - - -For **Airbyte Open Source** users, we recommend using an API token to authenticate your Zendesk account. You can follow the steps in our [full documentation](https://docs.airbyte.com/integrations/sources/zendesk-support#setup-guide) to generate this token. - - - -1. For **Source name**, enter a name to help you identify this source. -2. You can use OAuth or an API token to authenticate your Zendesk account. We recommend using OAuth for Airbyte Cloud and an API token for Airbyte Open Source. - - - - **For Airbyte Cloud:** To authenticate using OAuth, select **OAuth2.0** from the Authentication dropdown, then click **Authenticate your Zendesk Support account** to sign in with Zendesk and authorize your account. - - - - **For Airbyte Open Source**: To authenticate using an API token, select **API Token** from the Authentication dropdown and enter the [token you generated](https://docs.airbyte.com/integrations/sources/zendesk-support#setup-guide), as well as the email address associated with your Zendesk account. - - -3. For **Start Date**, use the provided datepicker or enter a UTC date and time programmatically in the format `YYYY-MM-DDTHH:mm:ssZ`. The data added on and after this date will be replicated. -4. For **Subdomain**, enter your Zendesk subdomain. This is the subdomain found in your account URL. For example, if your account URL is `https://MY_SUBDOMAIN.zendesk.com/`, then `MY_SUBDOMAIN` is your subdomain. -5. Click **Set up source** and wait for the tests to complete. - -For detailed information on supported sync modes, supported streams and performance considerations, refer to the [full documentation for Zendesk Support](https://docs.airbyte.com/integrations/sources/zendesk-support). diff --git a/docs/integrations/sources/zendesk-support.md b/docs/integrations/sources/zendesk-support.md index e25f589a9ada..a5f93a80766b 100644 --- a/docs/integrations/sources/zendesk-support.md +++ b/docs/integrations/sources/zendesk-support.md @@ -1,6 +1,10 @@ # Zendesk Support -This page contains the setup guide and reference information for the Zendesk Support source connector. + + +This page contains the setup guide and reference information for the [Zendesk Support](https://www.zendesk.com/) source connector. + + ## Prerequisites @@ -14,16 +18,21 @@ The Zendesk Support source connector supports two authentication methods: - API token -For **Airbyte Cloud** users, we highly recommend using OAuth to authenticate your Zendesk Support account, as it simplifies the setup process and allows you to authenticate [directly from the Airbyte UI](#set-up-the-zendesk-support-source-connector). +**For Airbyte Cloud:** + +We highly recommend using OAuth to authenticate your Zendesk Support account, as it simplifies the setup process and allows you to authenticate [directly from the Airbyte UI](#set-up-the-zendesk-support-source-connector). + -For **Airbyte Open Source** users, we recommend using an API token to authenticate your Zendesk Support account. Please follow the steps below to generate this key. +**For Airbyte Open Source:** + +We recommend using an API token to authenticate your Zendesk Support account. Please follow the steps below to generate this key. :::note If you prefer to authenticate with OAuth for **Airbyte Open Source**, you can follow the steps laid out in [this Zendesk article](https://support.zendesk.com/hc/en-us/articles/4408845965210) to obtain your client ID, client secret and access token. Please ensure you set the scope to `read` when generating the access token. ::: -### (Airbyte Open Source) Enable API token access and generate a token +### Generate an API token 1. Log in to your Zendesk account. 2. Click the **Zendesk Products** icon (four squares) in the top-right corner, then select **Admin Center**. @@ -40,22 +49,23 @@ If you prefer to authenticate with OAuth for **Airbyte Open Source**, you can fo ### Set up the Zendesk Support source connector -1. Log in to your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) or Airbyte Open Source account. +1. Log in to your [Airbyte Cloud](https://cloud.airbyte.com/workspaces) account. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+ New source**. 3. Find and select **Zendesk Support** from the list of available sources. 4. For **Source name**, enter a name to help you identify this source. -5. You can use OAuth or an API token to authenticate your Zendesk Support account. We recommend using OAuth for Airbyte Cloud and an API key for Airbyte Open Source. - - - - **For Airbyte Cloud**: To authenticate using OAuth, select **OAuth 2.0** from the Authentication dropdown, then click **Authenticate your Zendesk Support account** to sign in with Zendesk Support and authorize your account. - - - - **For Airbyte Open Source**: To authenticate using an API key, select **API Token** from the Authentication dropdown and enter the API token you generated, as well as the email address associated with your Zendesk Support account. - - -6. For **Start Date**, use the provided datepicker or enter a UTC date and time programmatically in the format `YYYY-MM-DDTHH:mm:ssZ`. The data added on and after this date will be replicated. -7. For **Subdomain**, enter your Zendesk subdomain. This is the subdomain found in your account URL. For example, if your account URL is `https://MY_SUBDOMAIN.zendesk.com/`, then `MY_SUBDOMAIN` is your subdomain. +5. You can use OAuth or an API token to authenticate your Zendesk Support account. + +- **For Airbyte Cloud**: To authenticate using OAuth, select **OAuth 2.0** from the Authentication dropdown, then click **Authenticate your Zendesk Support account** to sign in with Zendesk Support and authorize your account. + + +- **For Airbyte Open Source**: To authenticate using an API key, select **API Token** from the Authentication dropdown and enter the API token you generated, as well as the email address associated with your Zendesk Support account. + +6. For **Subdomain**, enter your Zendesk subdomain. This is the subdomain found in your account URL. For example, if your account URL is `https://MY_SUBDOMAIN.zendesk.com/`, then `MY_SUBDOMAIN` is your subdomain. +7. (Optional) For **Start Date**, use the provided datepicker or enter a UTC date and time programmatically in the format `YYYY-MM-DDTHH:mm:ssZ`. The data added on and after this date will be replicated. If this field is left blank, Airbyte will replicate the data for the last two years by default. 8. Click **Set up source** and wait for the tests to complete. + + + ## Supported sync modes @@ -66,34 +76,39 @@ The Zendesk Support source connector supports the following [sync modes](https:/ - Incremental Sync | Append - Incremental Sync | Deduped History -## Supported streams - :::note There are two types of incremental sync: -1. Incremental (standard server-side, where API returns only the data updated or generated since the last sync) -2. Client-Side Incremental (API returns all available data and connector filters out only new records) +1. Incremental (standard server-side, where API returns only the data updated or generated since the last sync). +2. Client-Side Incremental (API returns all available data and connector filters out only new records). ::: +## Supported streams + The Zendesk Support source connector supports the following streams: - [Account Attributes](https://developer.zendesk.com/api-reference/ticketing/ticket-management/skill_based_routing/#list-account-attributes) +- [Articles](https://developer.zendesk.com/api-reference/help_center/help-center-api/articles/#list-articles) \(Incremental\) +- [Article Votes](https://developer.zendesk.com/api-reference/help_center/help-center-api/votes/#list-votes) \(Incremental\) +- [Article Comments](https://developer.zendesk.com/api-reference/help_center/help-center-api/article_comments/#list-comments) \(Incremental\) +- [Article Comment Votes](https://developer.zendesk.com/api-reference/help_center/help-center-api/votes/#list-votes) \(Incremental\) - [Attribute Definitions](https://developer.zendesk.com/api-reference/ticketing/ticket-management/skill_based_routing/#list-routing-attribute-definitions) - [Audit Logs](https://developer.zendesk.com/api-reference/ticketing/account-configuration/audit_logs/#list-audit-logs)\(Incremental\) (Only available for enterprise accounts) - [Brands](https://developer.zendesk.com/api-reference/ticketing/account-configuration/brands/#list-brands) -- [Custom Roles](https://developer.zendesk.com/api-reference/ticketing/account-configuration/custom_roles/#list-custom-roles) +- [Custom Roles](https://developer.zendesk.com/api-reference/ticketing/account-configuration/custom_roles/#list-custom-roles) \(Incremental\) - [Groups](https://developer.zendesk.com/rest_api/docs/support/groups) \(Incremental\) - [Group Memberships](https://developer.zendesk.com/rest_api/docs/support/group_memberships) \(Incremental\) - [Macros](https://developer.zendesk.com/rest_api/docs/support/macros) \(Incremental\) - [Organizations](https://developer.zendesk.com/rest_api/docs/support/organizations) \(Incremental\) +- [Organization Fields](https://developer.zendesk.com/api-reference/ticketing/organizations/organization_fields/#list-organization-fields) \(Incremental\) - [Organization Memberships](https://developer.zendesk.com/api-reference/ticketing/organizations/organization_memberships/) \(Incremental\) - [Posts](https://developer.zendesk.com/api-reference/help_center/help-center-api/posts/#list-posts) \(Incremental\) -- [Post Comments](https://developer.zendesk.com/api-reference/help_center/help-center-api/post_comments/#list-comments) -- [Post Comment Votes](https://developer.zendesk.com/api-reference/help_center/help-center-api/votes/#list-votes) -- [Post Votes](https://developer.zendesk.com/api-reference/help_center/help-center-api/votes/#list-votes) +- [Post Comments](https://developer.zendesk.com/api-reference/help_center/help-center-api/post_comments/#list-comments) \(Incremental\) +- [Post Comment Votes](https://developer.zendesk.com/api-reference/help_center/help-center-api/votes/#list-votes) \(Incremental\) +- [Post Votes](https://developer.zendesk.com/api-reference/help_center/help-center-api/votes/#list-votes) \(Incremental\) - [Satisfaction Ratings](https://developer.zendesk.com/rest_api/docs/support/satisfaction_ratings) \(Incremental\) -- [Schedules](https://developer.zendesk.com/api-reference/ticketing/ticket-management/schedules/#list-schedules) -- [SLA Policies](https://developer.zendesk.com/rest_api/docs/support/sla_policies) +- [Schedules](https://developer.zendesk.com/api-reference/ticketing/ticket-management/schedules/#list-schedules) \(Incremental\) +- [SLA Policies](https://developer.zendesk.com/rest_api/docs/support/sla_policies) \(Incremental\) - [Tags](https://developer.zendesk.com/rest_api/docs/support/tags) - [Tickets](https://developer.zendesk.com/api-reference/ticketing/ticket-management/incremental_exports/#incremental-ticket-export-time-based) \(Incremental\) - [Ticket Audits](https://developer.zendesk.com/rest_api/docs/support/ticket_audits) \(Client-Side Incremental\) @@ -105,17 +120,65 @@ The Zendesk Support source connector supports the following streams: - [Topics](https://developer.zendesk.com/api-reference/help_center/help-center-api/topics/#list-topics) \(Incremental\) - [Ticket Skips](https://developer.zendesk.com/api-reference/ticketing/tickets/ticket_skips/) \(Incremental\) - [Users](https://developer.zendesk.com/api-reference/ticketing/ticket-management/incremental_exports/#incremental-user-export) \(Incremental\) +- [UserFields](https://developer.zendesk.com/api-reference/ticketing/users/user_fields/#list-user-fields) + +### Deleted Records Support +The Zendesk Support connector fetches deleted records in the following streams: + +| Stream | Deletion indicator field | +|:-------------------------|:-------------------------| +| **Brands** | `is_deleted` | +| **Groups** | `deleted` | +| **Organizations** | `deleted_at` | +| **Ticket Metric Events** | `deleted` | +| **Tickets** | `status`==`deleted` | + +## Limitations & Troubleshooting -## Performance considerations +
      + +Expand to see details about Zendesk Support connector limitations and troubleshooting. + + +### Connector limitations + +#### Rate limiting The connector is restricted by normal Zendesk [requests limitation](https://developer.zendesk.com/rest_api/docs/support/usage_limits). The Zendesk connector ideally should not run into Zendesk API limitations under normal usage. [Create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. +### Troubleshooting + +* Check out common troubleshooting issues for the Zendesk Support source connector on our [Airbyte Forum](https://github.com/airbytehq/airbyte/discussions). + +
      + ## Changelog | Version | Date | Pull Request | Subject | |:---------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `2.2.6` | 2024-01-11 | [34064](https://github.com/airbytehq/airbyte/pull/34064) | Skip 504 Error for stream `Ticket Audits` | +| `2.2.5` | 2024-01-08 | [34010](https://github.com/airbytehq/airbyte/pull/34010) | prepare for airbyte-lib | +| `2.2.4` | 2023-12-20 | [33680](https://github.com/airbytehq/airbyte/pull/33680) | Fix pagination issue for streams related to incremental export sync | +| `2.2.3` | 2023-12-14 | [33435](https://github.com/airbytehq/airbyte/pull/33435) | Fix 504 Error for stream Ticket Audits | +| `2.2.2` | 2023-12-01 | [33012](https://github.com/airbytehq/airbyte/pull/33012) | Increase number of retries for backoff policy to 10 | +| `2.2.1` | 2023-11-10 | [32440](https://github.com/airbytehq/airbyte/pull/32440) | Made refactoring to improve code maintainability | +| `2.2.0` | 2023-10-31 | [31999](https://github.com/airbytehq/airbyte/pull/31999) | Extended the `CustomRoles` stream schema | +| `2.1.1` | 2023-10-23 | [31702](https://github.com/airbytehq/airbyte/pull/31702) | Base image migration: remove Dockerfile and use the python-connector-base image | +| `2.1.0` | 2023-10-19 | [31606](https://github.com/airbytehq/airbyte/pull/31606) | Added new field `reply_time_in_seconds` to the `Ticket Metrics` stream schema | +| `2.0.0` | 2023-09-15 | [30440](https://github.com/airbytehq/airbyte/pull/30440) | Remove stream `Deleted Tickets` | +| `1.7.0` | 2023-09-11 | [30259](https://github.com/airbytehq/airbyte/pull/30259) | Add stream `Deleted Tickets` | +| `1.6.0` | 2023-09-09 | [30168](https://github.com/airbytehq/airbyte/pull/30168) | Make `start_date` field optional | +| `1.5.1` | 2023-09-05 | [30142](https://github.com/airbytehq/airbyte/pull/30142) | Handle non-JSON Response | +| `1.5.0` | 2023-09-04 | [30138](https://github.com/airbytehq/airbyte/pull/30138) | Add new Streams: `Article Votes`, `Article Comments`, `Article Comment Votes` | +| `1.4.0` | 2023-09-04 | [30134](https://github.com/airbytehq/airbyte/pull/30134) | Add incremental support for streams: `custom Roles`, `Schedules`, `SLA Policies` | +| `1.3.0` | 2023-08-30 | [30031](https://github.com/airbytehq/airbyte/pull/30031) | Add new streams: `Articles`, `Organization Fields` | +| `1.2.2` | 2023-08-30 | [29998](https://github.com/airbytehq/airbyte/pull/29998) | Fix typo in stream `AttributeDefinitions`: field condition | +| `1.2.1` | 2023-08-30 | [29991](https://github.com/airbytehq/airbyte/pull/29991) | Remove Custom availability strategy | +| `1.2.0` | 2023-08-29 | [29940](https://github.com/airbytehq/airbyte/pull/29940) | Add undeclared fields to schemas | +| `1.1.1` | 2023-08-29 | [29904](https://github.com/airbytehq/airbyte/pull/29904) | make `Organizations` stream incremental | +| `1.1.0` | 2023-08-28 | [29891](https://github.com/airbytehq/airbyte/pull/29891) | Add stream `UserFields` | | `1.0.0` | 2023-07-27 | [28774](https://github.com/airbytehq/airbyte/pull/28774) | fix retry logic & update cursor for `Tickets` stream | | `0.11.0` | 2023-08-10 | [27208](https://github.com/airbytehq/airbyte/pull/27208) | Add stream `Topics` | | `0.10.7` | 2023-08-09 | [29256](https://github.com/airbytehq/airbyte/pull/29256) | Update tooltip descriptions in spec | @@ -178,3 +241,5 @@ The Zendesk connector ideally should not run into Zendesk API limitations under | `0.1.2` | 2021-10-16 | [6513](https://github.com/airbytehq/airbyte/pull/6513) | Fixed TicketComments stream | | `0.1.1` | 2021-09-02 | [5787](https://github.com/airbytehq/airbyte/pull/5787) | Fixed incremental logic for the ticket_comments stream | | `0.1.0` | 2021-07-21 | [4861](https://github.com/airbytehq/airbyte/pull/4861) | Created CDK native zendesk connector | + +
      diff --git a/docs/integrations/sources/zendesk-talk.md b/docs/integrations/sources/zendesk-talk.md index 16b202874c4e..e4208bd80576 100644 --- a/docs/integrations/sources/zendesk-talk.md +++ b/docs/integrations/sources/zendesk-talk.md @@ -70,16 +70,16 @@ The Zendesk connector should not run into Zendesk API limitations under normal u | `array` | `array` | | | `object` | `object` | | - ## Changelog - -| Version | Date | Pull Request | Subject | -|:--------|:-----------| :----- |:----------------------------------| -| `0.1.9` | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes | -| `0.1.8` | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references | -| `0.1.7` | 2023-02-10 | [22815](https://github.com/airbytehq/airbyte/pull/22815) | Specified date formatting in specification | -| `0.1.6` | 2023-01-27 | [22028](https://github.com/airbytehq/airbyte/pull/22028) | Set `AvailabilityStrategy` for streams explicitly to `None` | -| `0.1.5` | 2022-09-29 | [17362](https://github.com/airbytehq/airbyte/pull/17362) | always use the latest CDK version | -| `0.1.4` | 2022-08-19 | [15764](https://github.com/airbytehq/airbyte/pull/15764) | Support OAuth2.0 | -| `0.1.3` | 2021-11-11 | [7173](https://github.com/airbytehq/airbyte/pull/7173) | Fix pagination and migrate to CDK | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------| +| 0.1.11 | 2024-01-12 | [34204](https://github.com/airbytehq/airbyte/pull/34204) | prepare for airbyte-lib | +| 0.1.10 | 2023-12-04 | [33030](https://github.com/airbytehq/airbyte/pull/33030) | Base image migration: remove Dockerfile and use python-connector-base image | +| 0.1.9 | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes | +| 0.1.8 | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references | +| 0.1.7 | 2023-02-10 | [22815](https://github.com/airbytehq/airbyte/pull/22815) | Specified date formatting in specification | +| 0.1.6 | 2023-01-27 | [22028](https://github.com/airbytehq/airbyte/pull/22028) | Set `AvailabilityStrategy` for streams explicitly to `None` | +| 0.1.5 | 2022-09-29 | [17362](https://github.com/airbytehq/airbyte/pull/17362) | always use the latest CDK version | +| 0.1.4 | 2022-08-19 | [15764](https://github.com/airbytehq/airbyte/pull/15764) | Support OAuth2.0 | +| 0.1.3 | 2021-11-11 | [7173](https://github.com/airbytehq/airbyte/pull/7173) | Fix pagination and migrate to CDK | diff --git a/docs/integrations/sources/zenefits.md b/docs/integrations/sources/zenefits.md index 5596ae4b82f8..5c8ad311252a 100644 --- a/docs/integrations/sources/zenefits.md +++ b/docs/integrations/sources/zenefits.md @@ -51,6 +51,7 @@ You can replicate the following tables using the Zenefits connector: ## Changelog -| Version | Date | Pull Request | Subject | -| :------ | :--------- | :------------------------------------------------------- | :-------------- | -| `0.1.0` | 2022-08-24 | [14809](https://github.com/airbytehq/airbyte/pull/14809) | Initial Release | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:--------------------| +| `0.2.0` | 2023-10-29 | [31946](https://github.com/airbytehq/airbyte/pull/31946) | Migrate to Low Code | +| `0.1.0` | 2022-08-24 | [14809](https://github.com/airbytehq/airbyte/pull/14809) | Initial Release | diff --git a/docs/operator-guides/security.md b/docs/operating-airbyte/security.md similarity index 90% rename from docs/operator-guides/security.md rename to docs/operating-airbyte/security.md index 6df3cb702841..ae224b3ad75a 100644 --- a/docs/operator-guides/security.md +++ b/docs/operating-airbyte/security.md @@ -1,4 +1,8 @@ -# Airbyte Security +--- +products: all +--- + +# Security Airbyte is committed to keeping your data safe by following industry-standard practices for securing physical deployments, setting access policies, and leveraging the security features of leading Cloud providers. @@ -56,7 +60,7 @@ You can secure access to Airbyte using the following methods: } } ``` -- Change the default username and password in your environment's `.env` file: +- *Only for docker compose deployments:* Change the default username and password in your environment's `.env` file: ``` # Proxy Configuration # Set to empty values, e.g. "" to disable basic auth @@ -84,14 +88,6 @@ Note that this process is not reversible. Once you have converted to a secret st Most Airbyte Open Source connectors support encryption-in-transit (SSL or HTTPS). We recommend configuring your connectors to use the encryption option whenever available. -### Telemetry - -Airbyte does send anonymized data to our services to improve the product (especially connector reliability and scale). To disable telemetry, modify the .env file and define the following environment variable: - -``` -TRACKING_STRATEGY=logging -``` - ## Securing Airbyte Cloud Airbyte Cloud leverages the security features of leading Cloud providers and sets least-privilege access policies to ensure data security. @@ -120,22 +116,6 @@ GCP region: us-west3 #### European Union -:::note - -Some workflows still run in the US, even when the data residency is in the EU. If you use the EU as a data residency, you must allowlist the following IP addresses from both GCP us-west3 and AWS eu-west-3. - -::: - -GCP region: us-west3 -* 34.106.109.131 -* 34.106.196.165 -* 34.106.60.246 -* 34.106.229.69 -* 34.106.127.139 -* 34.106.218.58 -* 34.106.115.240 -* 34.106.225.141 - AWS region: eu-west-3 * 13.37.4.46 * 13.37.142.60 @@ -158,7 +138,7 @@ Airbyte Cloud allows you to log in to the platform using your email and password ### Access Control -Airbyte Cloud supports [user management](https://docs.airbyte.com/cloud/managing-airbyte-cloud/manage-airbyte-cloud-workspace#add-users-to-your-workspace) but doesn’t support role-based access control (RBAC) yet. +Airbyte Cloud supports [user management](/using-airbyte/workspaces.md#add-users-to-your-workspace) but doesn’t support role-based access control (RBAC) yet. ### Compliance diff --git a/docs/operator-guides/browsing-output-logs.md b/docs/operator-guides/browsing-output-logs.md index 456965c21904..d4afd258c227 100644 --- a/docs/operator-guides/browsing-output-logs.md +++ b/docs/operator-guides/browsing-output-logs.md @@ -1,29 +1,51 @@ -# Browsing Output Logs +--- +products: all +--- -## Overview +# Browsing logs -This tutorial will describe how to explore Airbyte Workspace folders. +Airbyte records the full logs as a part of each sync. These logs can be used to understand the underlying operations Airbyte performs to read data from the source and write to the destination as a part of the [Airbyte Protocol](/understanding-airbyte/airbyte-protocol.md). The logs includes many details, including any errors that can be helpful when troubleshooting sync errors. -This is useful if you need to browse the docker volumes where extra output files of Airbyte server and workers are stored since they may not be accessible through the UI. +:::info +When using Airbyte Open Source, you can also access additional logs outside of the UI. This is useful if you need to browse the Docker volumes where extra output files of Airbyte server and workers are stored. +::: + +To find the logs for a connection, navigate to a connection's `Job History` tab to see the latest syncs. + +## View the logs in the UI +To open the logs in the UI, select the three grey dots next to a sync and select `View logs`. This will open our full screen in-app log viewer. + +:::tip +If you are troubleshooting a sync error, you can search for `Error`, `Exception`, or `Fail` to find common errors. +::: -## Exploring the Logs folders +The in-app log viewer will only search for instances of the search term within that attempt. To search across all attempts, download the logs locally. -When running a Sync in Airbyte, you have the option to look at the logs in the UI as shown next. +## Link to a sync job +To help others quickly find your job, copy the link to the logs to your clipboard, select the three grey dots next to a sync and select `Copy link to job`. -### Identifying Workspace IDs +You can also access the link to a sync job from the in-app log viewer. + +## Download the logs +To download a copy of the logs locally, select the three grey dots next to a sync and select `Download logs`. + +You can also access the download log button from the in-app log viewer. + +:::note +If a sync was completed across multiple attempts, downloading the logs will union all the logs for all attempts for that job. +::: -In the screenshot below, you can notice the highlighted blue boxes are showing the id numbers that were used for the selected "Attempt" for this sync job. +## Exploring Local Logs -In this case, the job was running in `/tmp/workspace/9/2/` folder since the tab of the third attempt is being selected in the UI \(first attempt would be `/tmp/workspace/9/0/`\). + -![](../.gitbook/assets/explore_logs.png) +### Establish the folder directory -The highlighted button in the red circle on the right would allow you to download the logs.log file. -However, there are actually more files being recorded in the same workspace folder... Thus, we might want to dive deeper to explore these folders and gain a better understanding of what is being run by Airbyte. +In the UI, you can discover the Attempt ID within the sync job. Most jobs will complete in the first attempt, so your folder directory will look like `/tmp/workspace/9/0`. If you sync job completes in multiple attempts, you'll need to define which attempt you're interested in, and note this. For example, for the third attempt, it will look like `/tmp/workspace/9/2/` . ### Understanding the Docker run commands -Scrolling down a bit more, we can also read the different docker commands being used internally are starting with: +We can also read the different docker commands being used internally are starting with: ```text docker run --rm -i -v airbyte_workspace:/data -v /tmp/airbyte_local:/local -w /data/9/2 --network host ... @@ -35,7 +57,7 @@ Following [Docker Volume documentation](https://docs.docker.com/storage/volumes/ ### Opening a Unix shell prompt to browse the Docker volume -For example, we can run any docker container/image to browse the content of this named volume by mounting it similarly, let's use the [busybox](https://hub.docker.com/_/busybox) image. +For example, we can run any docker container/image to browse the content of this named volume by mounting it similarly. In the example below, the [busybox](https://hub.docker.com/_/busybox) image is used. ```text docker run -it --rm --volume airbyte_workspace:/data busybox @@ -50,13 +72,15 @@ ls /data/9/2/ Example Output: ```text -catalog.json normalize tap_config.json -logs.log singer_rendered_catalog.json target_config.json +catalog.json +tap_config.json +logs.log +target_config.json ``` ### Browsing from the host shell -Or, if you don't want to transfer to a shell prompt inside the docker image, you can simply run Shell commands using docker commands as a proxy like this: +Or, if you don't want to transfer to a shell prompt inside the docker image, you can run Shell commands using docker commands as a proxy: ```bash docker run -it --rm --volume airbyte_workspace:/data busybox ls /data/9/2 @@ -81,7 +105,7 @@ docker run -it --rm --volume airbyte_workspace:/data busybox cat /data/9/2/catal Example Output: ```text -{"streams":[{"stream":{"name":"exchange_rate","json_schema":{"type":"object","properties":{"CHF":{"type":"number"},"HRK":{"type":"number"},"date":{"type":"string"},"MXN":{"type":"number"},"ZAR":{"type":"number"},"INR":{"type":"number"},"CNY":{"type":"number"},"THB":{"type":"number"},"AUD":{"type":"number"},"ILS":{"type":"number"},"KRW":{"type":"number"},"JPY":{"type":"number"},"PLN":{"type":"number"},"GBP":{"type":"number"},"IDR":{"type":"number"},"HUF":{"type":"number"},"PHP":{"type":"number"},"TRY":{"type":"number"},"RUB":{"type":"number"},"HKD":{"type":"number"},"ISK":{"type":"number"},"EUR":{"type":"number"},"DKK":{"type":"number"},"CAD":{"type":"number"},"MYR":{"type":"number"},"USD":{"type":"number"},"BGN":{"type":"number"},"NOK":{"type":"number"},"RON":{"type":"number"},"SGD":{"type":"number"},"CZK":{"type":"number"},"SEK":{"type":"number"},"NZD":{"type":"number"},"BRL":{"type":"number"}}},"supported_sync_modes":["full_refresh"],"default_cursor_field":[]},"sync_mode":"full_refresh","cursor_field":[]}]} +{"streams":[{"stream":{"name":"exchange_rate","json_schema":{"type":"object","properties":{"CHF":{"type":"number"},"HRK":{"type":"number"},"date":{"type":"string"},"MXN":{"type":"number"},"ZAR":{"type":"number"},"INR":{"type":"number"},"CNY":{"type":"number"},"THB":{"type":"number"},"NZD":{"type":"number"},"BRL":{"type":"number"}}},"supported_sync_modes":["full_refresh"],"default_cursor_field":[]},"sync_mode":"full_refresh","cursor_field":[]}]} ``` ### Extract catalog.json file from docker volume diff --git a/docs/operator-guides/collecting-metrics.md b/docs/operator-guides/collecting-metrics.md index db1c6665b2c4..a1203fc5191e 100644 --- a/docs/operator-guides/collecting-metrics.md +++ b/docs/operator-guides/collecting-metrics.md @@ -1,22 +1,31 @@ +--- +products: oss-* +--- + # Monitoring Airbyte + Airbyte offers you various ways to monitor your ELT pipelines. These options range from using open-source tools to integrating with enterprise-grade SaaS platforms. Here's a quick overview: * Connection Logging: All Airbyte instances provide extensive logs for each connector, giving detailed reports on the data synchronization process. This is available across all Airbyte offerings. -* [Airbyte Datadog Integration](#airbyte-datadog-integration): Airbyte Enterprise customers can leverage our integration with Datadog. This lets you monitor and analyze your data pipelines right within your Datadog dashboards at no additional cost. -* Airbyte OpenTelemetry (OTEL) Integration: Coming soon, this will allow you to push metrics to your self-hosted monitoring solution using OpenTelemetry. +* [Airbyte Datadog Integration](#airbyte-datadog-integration): Airbyte customers can leverage our integration with Datadog. This lets you monitor and analyze your data pipelines right within your Datadog dashboards at no additional cost. +* [Airbyte OpenTelemetry (OTEL) Integration](#airbyte-opentelemetry-integration): This allows you to push metrics to your self-hosted monitoring solution using OpenTelemetry. Please browse the sections below for more details on each option and how to set it up. ## Airbyte Datadog Integration -![Datadog's Airbyte Integration Dashboard](assets/DatadogAirbyteIntegration_OutOfTheBox_Dashboard.png) -_This integration is available for **Airbyte Enterprise users**. Airbyte Enterprise includes premium support with SLAs for your critical pipelines, security features including SSO, RBAC and audit logging, in addition to reliability, scalability and compliance features. -Please reach out to [our team](https://airbyte.com/talk-to-sales) if you want to learn more._ +:::info +Monitoring your Airbyte instance using Datadog is an early preview feature and still in development. +Expect changes to this feature and the configuration to happen in the future. This feature will be +only for Airbyte Enterprise customers in the future. +::: -Airbyte's new integration with Datadog brings the convenience of monitoring and analyzing your Airbyte data pipelines directly within your Datadog dashboards. +![Datadog's Airbyte Integration Dashboard](assets/DatadogAirbyteIntegration_OutOfTheBox_Dashboard.png) + +Airbyte's new integration with Datadog brings the convenience of monitoring and analyzing your Airbyte data pipelines directly within your Datadog dashboards. This integration brings forth new `airbyte.*` metrics along with new dashboards. The list of metrics is found [here](https://docs.datadoghq.com/integrations/airbyte/#data-collected). ### Setup Instructions @@ -24,7 +33,7 @@ This integration brings forth new `airbyte.*` metrics along with new dashboards. Setting up this integration for Airbyte instances deployed with Docker involves five straightforward steps: -1. **Set Datadog Airbyte Config:** Create or configure the `datadog.yaml` file with the contents below: +1. **Set Datadog Airbyte Config**: Create or configure the `datadog.yaml` file with the contents below: ```yaml dogstatsd_mapper_profiles: @@ -137,4 +146,77 @@ DD_DOGSTATSD_PORT=8125 5. **Re-deploy Airbyte and the Datadog Agent**: With the updated configurations, you're ready to deploy your Airbyte application by running `docker compose up`. +## Airbyte OpenTelemetry Integration + + +### Docker Compose Setup Instructions + +Setting up this integration for Airbyte instances deployed with Docker Compose involves four straightforward steps: + + +1. **Deploy an OpenTelemetry Collector**: Follow the official [Docker Compose Getting Started documentation](https://opentelemetry.io/docs/collector/getting-started/#docker-compose). + +```yaml + otel-collector: + image: otel/opentelemetry-collector-contrib + volumes: + - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml + ports: + - 1888:1888 # pprof extension + - 8888:8888 # Prometheus metrics exposed by the collector + - 8889:8889 # Prometheus exporter metrics + - 13133:13133 # health_check extension + - 4317:4317 # OTLP gRPC receiver + - 4318:4318 # OTLP http receiver + - 55679:55679 # zpages extension +``` + +2. **Update Docker Compose Configuration**: Modify your `docker-compose.yaml` file in the Airbyte repository to include the `metrics-reporter` container. This submits Airbyte metrics to the OpenTelemetry collector: + +```yaml + metric-reporter: + image: airbyte/metrics-reporter:${VERSION} + container_name: metric-reporter + networks: + - airbyte_internal + environment: + - DATABASE_PASSWORD=${DATABASE_PASSWORD} + - DATABASE_URL=${DATABASE_URL} + - DATABASE_USER=${DATABASE_USER} + - METRIC_CLIENT=${METRIC_CLIENT} + - OTEL_COLLECTOR_ENDPOINT=${OTEL_COLLECTOR_ENDPOINT} +``` + +3. **Set Environment Variables**: Amend your `.env` file with the correct values needed by `docker-compose.yaml`: + +```yaml +PUBLISH_METRICS=true +METRIC_CLIENT=otel +OTEL_COLLECTOR_ENDPOINT=http://otel-collector:4317 +``` + +4. **Re-deploy Airbyte**: With the updated configurations, you're ready to deploy your Airbyte application by running `docker compose up`. + +### Helm Chart Setup Instructions + +Setting up this integration for Airbyte instances deployed with the helm chart involves three straightforward steps: + +1. **Deploy an OpenTelemetry Collector**: Follow the official [Kubernetes Getting Started documentation](https://opentelemetry.io/docs/collector/getting-started/#kubernetes) to deploy a collector in your kubernetes cluster. + +2. **Update the chart values**: Modify your `values.yaml` file in the Airbyte repository to include the `metrics-reporter` container. This submits Airbyte metrics to the OpenTelemetry collector: + +```yaml +global: + metrics: + metricClient: "otel" + otelCollectorEndpoint: "http://otel-collector.opentelemetry.svc:4317" + +metrics: + enabled: true +``` + +:::note +Update the value of `otelCollectorEndpoint` with your collector URL. +::: +3. **Re-deploy Airbyte**: With the updated chart values, you're ready to deploy your Airbyte application by upgrading the chart. diff --git a/docs/operator-guides/configuring-airbyte-db.md b/docs/operator-guides/configuring-airbyte-db.md index 1b57f2ae9073..899e86b01fed 100644 --- a/docs/operator-guides/configuring-airbyte-db.md +++ b/docs/operator-guides/configuring-airbyte-db.md @@ -1,3 +1,7 @@ +--- +products: oss-* +--- + # Configuring the Airbyte Database Airbyte uses different objects to store internal state and metadata. This data is stored and manipulated by the various Airbyte components, but you have the ability to manage the deployment of this database in the following two ways: diff --git a/docs/operator-guides/configuring-airbyte.md b/docs/operator-guides/configuring-airbyte.md index db54ef40991a..f838021553b0 100644 --- a/docs/operator-guides/configuring-airbyte.md +++ b/docs/operator-guides/configuring-airbyte.md @@ -1,3 +1,7 @@ +--- +products: oss-* +--- + # Configuring Airbyte This section covers how to configure Airbyte, and the various configuration Airbyte accepts. @@ -15,14 +19,12 @@ If you want to manage your own docker files, please refer to Airbyte's docker fi ## Kubernetes Deployments -The recommended way to run an Airbyte Kubernetes deployment is via the `Kustomize` overlays. - -We recommend using the overlays in the `stable` directory as these have preset resource limits. +The recommended way to run an [Airbyte Kubernetes deployment](../deploying-airbyte/on-kubernetes-via-helm.md) is via the `Helm Charts`. -To configure the default Airbyte Kubernetes deployment, modify the `.env` in the respective directory. Each application will consume the appropriate -env var from a generated configmap. +To configure the Airbyte Kubernetes deployment you need to modify the `values.yaml` file, more [info here](../deploying-airbyte/on-kubernetes-via-helm.md#custom-deployment). +Each application will consume the appropriate values from that file. -If you want to manage your own Kube manifests, please refer to the various `Kustomize` overlays for examples. +If you want to manage your own Kube manifests, please refer to the `Helm Chart`. ## Reference @@ -44,14 +46,6 @@ The following variables are relevant to both Docker and Kubernetes. 4. `CONFIG_ROOT` - Defines the configs directory. Applies only to Docker, and is present in Kubernetes for backward compatibility. 5. `WORKSPACE_ROOT` - Defines the Airbyte workspace directory. Applies only to Docker, and is present in Kubernetes for backward compatibility. -#### Access - -Set to empty values, e.g. "" to disable basic auth. **Be sure to change these values**. - -1. BASIC_AUTH_USERNAME=airbyte -2. BASIC_AUTH_PASSWORD=password -3. BASIC_AUTH_PROXY_TIMEOUT=600 - Defines the proxy timeout time for requests to Airbyte Server. Main use should be for dynamic discover when creating a connection (S3, JDBC, etc) that takes a long time. - #### Secrets 1. `SECRET_PERSISTENCE` - Defines the Secret Persistence type. Defaults to NONE. Set to GOOGLE_SECRET_MANAGER to use Google Secret Manager. Set to AWS_SECRET_MANAGER to use AWS Secret Manager. Set to TESTING_CONFIG_DB_TABLE to use the database as a test. Set to VAULT to use Hashicorp Vault, currently only the token based authentication is supported. Alpha support. Undefined behavior will result if this is turned on and then off. @@ -80,7 +74,7 @@ Set to empty values, e.g. "" to disable basic auth. **Be sure to change these va 1. `TEMPORAL_HOST` - Define the url where Temporal is hosted at. Please include the port. Airbyte services use this information. 2. `INTERNAL_API_HOST` - Define the url where the Airbyte Server is hosted at. Please include the port. Airbyte services use this information. -3. `WEBAPP_URL` - Define the url the Airbyte Webapp is hosted at. Please include the port. Airbyte services use this information. +3. `WEBAPP_URL` - Define the url the Airbyte Webapp is hosted at. Please include the port. Airbyte services use this information. You can set this variable to your custom domain name to change the Airbyte instance URL provided in notifications. #### Jobs @@ -128,6 +122,14 @@ Set to empty values, e.g. "" to disable basic auth. **Be sure to change these va 2. `DOCKER_NETWORK` - Defines the docker network the new Scheduler launches jobs on. 3. `LOCAL_DOCKER_MOUNT` - Defines the name of the docker mount that is used for local file handling. On Docker, this allows connector pods to interact with a volume for "local file" operations. +#### Access + +Set to empty values, e.g. "" to disable basic auth. **Be sure to change these values**. + +1. `BASIC_AUTH_USERNAME=airbyte` +2. `BASIC_AUTH_PASSWORD=password` +3. `BASIC_AUTH_PROXY_TIMEOUT=600` - Defines the proxy timeout time for requests to Airbyte Server. Main use should be for dynamic discover when creating a connection (S3, JDBC, etc) that takes a long time. + ### Kubernetes-Only #### Jobs @@ -156,7 +158,7 @@ A job specific variable overwrites the default sync job variable defined above. #### Worker -1. `TEMPORAL_WORKER_PORTS` - Define the local ports the Airbyte Worker pod uses to connect to the various Job pods. Port 9001 - 9040 are exposed by default in the Kustomize deployments. +1. `TEMPORAL_WORKER_PORTS` - Define the local ports the Airbyte Worker pod uses to connect to the various Job pods. Port 9001 - 9040 are exposed by default in the Helm Chart. #### Logging diff --git a/docs/operator-guides/configuring-connector-resources.md b/docs/operator-guides/configuring-connector-resources.md index 9fc0df16325c..20c03a8dc9bb 100644 --- a/docs/operator-guides/configuring-connector-resources.md +++ b/docs/operator-guides/configuring-connector-resources.md @@ -1,3 +1,7 @@ +--- +products: oss-* +--- + # Configuring Connector Resources As noted in [Workers & Jobs](../understanding-airbyte/jobs.md), there are four different types of jobs. diff --git a/docs/operator-guides/configuring-sync-notifications.md b/docs/operator-guides/configuring-sync-notifications.md deleted file mode 100644 index 6418aa2ffab5..000000000000 --- a/docs/operator-guides/configuring-sync-notifications.md +++ /dev/null @@ -1,55 +0,0 @@ -# Configuring Sync Notifications - -## Overview - -You can set up Airbyte to notify you when syncs have **failed** or **succeeded**. This is achieved through a webhook, a URL that you can input into other applications to get real time data from Airbyte. - -## Set up Slack Notifications on Sync Status - -If you're more of a visual learner, just head over to [this video](https://www.youtube.com/watch?v=NjYm8F-KiFc&ab_channel=Airbyte) to learn how to do this. Otherwise, keep reading! - -**Set up the bot.** - -Navigate to https://api.slack.com/apps/. Hit `Create an App`. - -![](../.gitbook/assets/notifications_create_slack_app.png) - -Then click `From scratch`. Enter your App Name (e.g. Airbyte Sync Notifications) and pick your desired Slack workspace. - -**Set up the webhook URL.** - -Now on the left sidebar, click on `Incoming Webhooks`. - -![](../.gitbook/assets/notifications_incoming_webhooks.png) - -Click the slider button in the top right to turn the feature on. Then click `Add New Webhook to Workspace`. - -![](../.gitbook/assets/notifications_add_new_webhook.png) - -Pick the channel that you want to receive Airbyte notifications in (ideally a dedicated one), and click `Allow` to give it permissions to access the channel. You should see the bot show up in the selected channel now. - -Now you should see an active webhook right above the `Add New Webhook to Workspace` button. - -![](../.gitbook/assets/notifications_webhook_url.png) - -Click `Copy.` - -**Add the webhook to Airbyte.** - -Assuming you have a [running instance of Airbyte](../deploying-airbyte/README.md), we can navigate to the UI. Click on Settings and then click on `Notifications`. - -![](../.gitbook/assets/notifications_airbyte_settings.png) - -Simply paste the copied webhook URL in `Connection status Webhook URL` and you're ready to go! On this page, you can click one or both of the sliders to decide whether you want notifications on sync successes, failures, or both. Make sure to click `Save changes` before you leave. - -Your Webhook URL should look something like this: - -![](../.gitbook/assets/notifications_airbyte_notification_settings.png) - -**Test it out.** - -From the settings page, you can click `Test` to send a test message to the channel. Or, just run a sync now and try it out! If all goes well, you should receive a notification in your selected channel that looks like this: - -![](../.gitbook/assets/notifications_slack_message.png) - -You're done! diff --git a/docs/operator-guides/contact-support.md b/docs/operator-guides/contact-support.md deleted file mode 100644 index 659a35e0b854..000000000000 --- a/docs/operator-guides/contact-support.md +++ /dev/null @@ -1,63 +0,0 @@ -# Airbyte Support - -Hold up! Have you looked at [our docs](https://docs.airbyte.com/) yet? We recommend searching the wealth of knowledge in our documentation as many times the answer you are looking for is there! - -## Airbyte Open Source Support - -Running Airbyte Open Source and have questions that our docs could not clear up? Join our community Slack and forum to connect with other Airbyte users. - -**Join our Slack community** [HERE](https://slack.airbyte.com/?_gl=1*1h8mjfe*_gcl_au*MTc4MjAxMDQzOS4xNjgyOTczMDYy*_ga*MTc1OTkyOTYzNi4xNjQxMjQyMjA0*_ga_HDBMVFQGBH*MTY4Nzg4OTQ4MC4zMjUuMS4xNjg3ODkwMjE1LjAuMC4w&_ga=2.58571491.813788522.1687789276-1759929636.1641242204)! - -If you're experiencing connection issues and need assistance, send your question to the **#help-connections-issue** channel. - -For questions related to building connectors, ask in the **#help-connector-development** channel. - -If you need help with infrastructure and deployment, head over to the **#help-infrastructure-deployment** channel. - -For assistance with the API, CLI, or setting up orchestration, visit the **#help-api-cli-orchestration** channel. - -We also hold daily office hours, Monday to Friday, where you can engage in live discussions with the community and Airbyte team. Join the **#office-hours** channel to participate and view the schedule. - -If you require personalized support, reach out to our sales team to inquire about [Airbyte Enterprise](https://airbyte.com/airbyte-enterprise). - -## Airbyte Cloud Support - -If you have questions about connector setup, error resolution, or want to report a bug, Airbyte Support is available to assist you. We recommend checking [our documentation](https://docs.airbyte.com/) and searching our [Help Center](https://support.airbyte.com/hc/en-us) before opening a support ticket. - -If you couldn't find the information you need in our docs or Help Center, open a ticket within the Airbyte Cloud platform by selecting the "Support" icon in the lower left navigation bar. Alternatively, you can submit a ticket through our [Help Center](https://support.airbyte.com/hc/en-us) by completing an Airbyte Cloud Support Request. - -If you're unsure about the supported connectors, refer to our [connector catalog](https://docs.airbyte.com/integrations/) & [release phases](https://docs.airbyte.com/project-overview/product-release-stages/). - -For account or credit-related inquiries, contact our [sales team](https://airbyte.com/talk-to-sales). - -If you don't see a connector you need, you can submit a [connector request](https://airbyte.com/connector-requests). - -To stay updated on Airbyte's future plans, take a look at [our roadmap](https://github.com/orgs/airbytehq/projects/37/views/1). - -## Airbyte Enterprise (self-hosted) Support - -If you're running Airbyte Open Source with Airbyte Enterprise or have an OSS support package, we're here to help you with upgrading Airbyte versions, debugging connector issues, or troubleshooting schema changes. - -Before opening a support ticket, we recommend consulting [our documentation](https://docs.airbyte.com/) and searching our [Help Center](https://support.airbyte.com/hc/en-us). If your question remains unanswered, please submit a ticket through our Help Center. We suggest creating an [Airbyte Help Center account](https://airbyte1416.zendesk.com/auth/v2/login/signin?return_to=https%3A%2F%2Fsupport.airbyte.com%2Fhc%2Fen-us&theme=hc&locale=en-us&brand_id=15365055240347&auth_origin=15365055240347%2Ctrue%2Ctrue) to access your organization's support requests. - -Submitting a Pull Request for review? - -* Be sure to follow our [contribution guidelines](https://docs.airbyte.com/contributing-to-airbyte/) laid out here on our doc. Highlights include: - * PRs should be limited to a single change-set -* Submit the PR as a PR Request through the Help Center Open Source Enterprise Support Request form -* If you are submitting a Platform PR we accept Platform PRs in the areas below: - * Helm - * Environment variable configurations - * Bug Fixes - * Security version bumps - * **If outside these areas, please open up an issue to help the team understand the need and if we are able to consider a PR** - -Submitting a PR does not guarantee its merge. The Airbyte support team will conduct an initial review, and if the PR aligns with Airbyte's roadmap, it will be prioritized based on team capacities and priorities. - -Although we strive to offer our utmost assistance, there are certain requests that we are unable to support. Currently, we do not provide assistance for these particular items: -* Question/troubleshooting assistance with forked versions of Airbyte -* Configuring using Octavia CLI -* Creating and configuring custom transformation using dbt -* Curating unique documentation and training materials -* Configuring Airbyte to meet security requirements - diff --git a/docs/operator-guides/reset.md b/docs/operator-guides/reset.md index ff7dc4d06124..3567acd2090b 100644 --- a/docs/operator-guides/reset.md +++ b/docs/operator-guides/reset.md @@ -1,20 +1,37 @@ -# Resetting Your Data +--- +products: all +--- -The reset button gives you a blank slate, of sorts, to perform a fresh new sync. This can be useful if you are just testing Airbyte or don't necessarily require the data replicated to your destination to be saved permanently. +# Resetting your data -![](../.gitbook/assets/reset_your_data_1.png) +Resetting your data allows you to drop all previously synced data so that any ensuing sync can start syncing fresh. This is useful if you don't require the data replicated to your destination to be saved permanently or are just testing Airbyte. -As outlined above, you can click on the `Reset your data` button to give you that clean slate. Just as a heads up, here is what it does and doesn't do: +Airbyte allows you to reset all streams in the connection, some, or only a single stream (when the connector support per-stream operations). -The reset button **DOES**: +A sync will automatically start after a completed reset, which commonly backfills all historical data. -* Delete all records in your destination tables -* Delete all records in your destination file +## Performing a Reset +To perform a full reset that resets all your streams, select `Reset your data` in the UI on a connection's status or job history tabs by selecting the three grey dots next to "Sync now". -The reset button **DOES NOT**: +To reset a single stream, navigate to a Connection's status page, click the three grey dots next to any stream, and select "Reset this stream". This will perform a reset of only that stream. You will then need to sync the connection again in order to reload data for that stream. -* Delete the destination tables -* Delete a destination file if using the LocalCSV or LocalJSON Destinations +:::note +A single stream reset will sync all enabled streams on the next sync. +::: -Because of this, if you have any orphaned tables or files that are no longer being synced to, they will have to be cleaned up later, as Airbyte will not clean them up for you. +You will also automatically be prompted to reset affected streams if you edit any stream settings or approve any non-breaking schema changes. To ensure data continues to sync accurately, Airbyte recommends doing a reset of those streams as your streams could sync incorrectly if a reset is not performed. +Similarly to a sync job, a reset can be completed as successful, failed, or cancelled. To resolve a failed reset, you should manually drop the tables in the destination so that Airbyte can continue syncing accurately into the destination. + +## Reset behavior +When a reset is successfully completed, all the records are deleted from your destination tables (and files, if using local JSON or local CSV as the destination). + +:::info +If you are using destinations that are on the [Destinations v2](/release_notes/upgrading_to_destinations_v2.md) framework, only raw tables will be cleared of their data. Final tables will retain all records from the last sync. +::: + +A reset **DOES NOT** delete any destination tables when using a data warehouse, data lake, database. The schema is retained but will not contain any rows. + +:::tip +If you have any orphaned tables or files that are no longer being synced to, they should be cleaned up separately, as Airbyte will not clean them up for you. This can occur when the `Destination Namespace` or `Stream Prefix` connection configuration is changed for an existing connection. +::: diff --git a/docs/operator-guides/scaling-airbyte.md b/docs/operator-guides/scaling-airbyte.md index 062cbd33d715..9c80cdbff378 100644 --- a/docs/operator-guides/scaling-airbyte.md +++ b/docs/operator-guides/scaling-airbyte.md @@ -1,3 +1,7 @@ +--- +products: oss-* +--- + # Scaling Airbyte As depicted in our [High-Level View](../understanding-airbyte/high-level-view.md), Airbyte is made up of several components under the hood: 1. Scheduler 2. Server 3. Temporal 4. Webapp 5. Database diff --git a/docs/operator-guides/telemetry.md b/docs/operator-guides/telemetry.md new file mode 100644 index 000000000000..71352d7c8b47 --- /dev/null +++ b/docs/operator-guides/telemetry.md @@ -0,0 +1,30 @@ +--- +products: all +--- + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + +# Telemetry + +Airbyte collects telemetry data in the UI and the servers to help us understand users and their use-cases better to improve the product. + +Also check our [privacy policy](https://airbyte.com/privacy-policy) for more details. + + + + To disable telemetry for your instance, modify the `.env` file and define the following environment variable: + + ``` + TRACKING_STRATEGY=logging + ``` + + + When visiting the webapp or our homepage the first time, you'll be asked for your consent to + telemetry collection depending on the legal requirements of your location. + + To change this later go to **Settings** > **User Settings** > **Cookie Preferences** or **Cookie Preferences** in the footer of our [homepage](https://airbyte.com). + + Server side telemetry collection can't be changed using Airbyte Cloud. + + \ No newline at end of file diff --git a/docs/operator-guides/transformation-and-normalization/transformations-with-airbyte.md b/docs/operator-guides/transformation-and-normalization/transformations-with-airbyte.md index aa11c7c3808b..da40ca63e6f1 100644 --- a/docs/operator-guides/transformation-and-normalization/transformations-with-airbyte.md +++ b/docs/operator-guides/transformation-and-normalization/transformations-with-airbyte.md @@ -1,6 +1,8 @@ -# Transformations with Airbyte (Part 3/3) +--- +products: oss-* +--- -## Overview +# Transformations with Airbyte (Part 3/3) This tutorial will describe how to push a custom dbt transformation project back to Airbyte to use during syncs. @@ -18,7 +20,7 @@ After replication of data from a source connector \(Extract\) to a destination c ## Public Git repository -In the connection settings page, I can add new Transformations steps to apply after [normalization](../../understanding-airbyte/basic-normalization.md). For example, I want to run my custom dbt project jaffle_shop, whenever my sync is done replicating and normalizing my data. +In the connection settings page, I can add new Transformations steps to apply after [normalization](../../using-airbyte/core-concepts/basic-normalization.md). For example, I want to run my custom dbt project jaffle_shop, whenever my sync is done replicating and normalizing my data. You can find the jaffle shop test repository by clicking [here](https://github.com/dbt-labs/jaffle_shop). @@ -97,3 +99,6 @@ run --vars '{"table_name":"sample","schema_name":"other_value"}' This string must have no space. There is a [Github issue](https://github.com/airbytehq/airbyte/issues/4348) to improve this. If you want to contribute to Airbyte, this is a good opportunity! +### DBT Profile + +There is no need to specify `--profiles-dir`. By default AirByte based on the destination type. For example, if you're using Postgres as your destination, Airbyte will create a profile configuration based on that destination. This means you don't need to specify the credentials. If you specify a custom `profile` file, you are responsible for securely managing the credentials. Currently, we don't have a way to manage and pass secrets and it's recommended you let Airbyte pass this to dbt. diff --git a/docs/operator-guides/transformation-and-normalization/transformations-with-dbt.md b/docs/operator-guides/transformation-and-normalization/transformations-with-dbt.md index e7ea6b4158bb..e13d64c70187 100644 --- a/docs/operator-guides/transformation-and-normalization/transformations-with-dbt.md +++ b/docs/operator-guides/transformation-and-normalization/transformations-with-dbt.md @@ -1,6 +1,8 @@ -# Transformations with dbt (Part 2/3) +--- +products: oss-* +--- -## Overview +# Transformations with dbt (Part 2/3) This tutorial will describe how to integrate SQL based transformations with Airbyte syncs using specialized transformation tool: dbt. diff --git a/docs/operator-guides/transformation-and-normalization/transformations-with-sql.md b/docs/operator-guides/transformation-and-normalization/transformations-with-sql.md index 3f6c9357d2c1..48f44f82dddd 100644 --- a/docs/operator-guides/transformation-and-normalization/transformations-with-sql.md +++ b/docs/operator-guides/transformation-and-normalization/transformations-with-sql.md @@ -1,8 +1,8 @@ -# Transformations with SQL (Part 1/3) - -## Transformations with SQL \(Part 1/3\) +--- +products: oss-* +--- -### Overview +# Transformations with SQL (Part 1/3) This tutorial will describe how to integrate SQL based transformations with Airbyte syncs using plain SQL queries. @@ -16,7 +16,7 @@ At its core, Airbyte is geared to handle the EL \(Extract Load\) steps of an ELT However, this is actually producing a table in the destination with a JSON blob column... For the typical analytics use case, you probably want this json blob normalized so that each field is its own column. -So, after EL, comes the T \(transformation\) and the first T step that Airbyte actually applies on top of the extracted data is called "Normalization". You can find more information about it [here](../../understanding-airbyte/basic-normalization.md). +So, after EL, comes the T \(transformation\) and the first T step that Airbyte actually applies on top of the extracted data is called "Normalization". You can find more information about it [here](../../using-airbyte/core-concepts/basic-normalization.md). Airbyte runs this step before handing the final data over to other tools that will manage further transformation down the line. diff --git a/docs/operator-guides/upgrading-airbyte.md b/docs/operator-guides/upgrading-airbyte.md index d2a4d9b86b76..4f73ce785940 100644 --- a/docs/operator-guides/upgrading-airbyte.md +++ b/docs/operator-guides/upgrading-airbyte.md @@ -1,5 +1,16 @@ +--- +products: oss-* +--- + # Upgrading Airbyte +:::info + +If you run on [Airbyte Cloud](https://cloud.airbyte.com/signup) you'll always run on the newest +Airbyte version automatically. This documentation only applies to users deploying our self-managed +version. +::: + ## Overview This tutorial will describe how to determine if you need to run this upgrade process, and if you do, how to do so. This process does require temporarily turning off Airbyte. @@ -117,7 +128,7 @@ If you are upgrading from (i.e. your current version of Airbyte is) Airbyte vers Here's an example of what it might look like with the values filled in. It assumes that the downloaded `airbyte_archive.tar.gz` is in `/tmp`. ```bash - docker run --rm -v /tmp:/config airbyte/migration:0.50.21 --\ + docker run --rm -v /tmp:/config airbyte/migration:0.50.44 --\ --input /config/airbyte_archive.tar.gz\ --output /config/airbyte_archive_migrated.tar.gz ``` diff --git a/docs/operator-guides/using-custom-connectors.md b/docs/operator-guides/using-custom-connectors.md index 4516f19ff987..6597dc7ad88a 100644 --- a/docs/operator-guides/using-custom-connectors.md +++ b/docs/operator-guides/using-custom-connectors.md @@ -1,15 +1,22 @@ -# Using custom connectors -If our connector catalog does not fulfill your needs, you can build your own Airbyte connectors. -There are two approaches you can take while jumping on connector development project: -1. You want to build a connector for an **external** source or destination (public API, off-the-shelf DBMS, data warehouses, etc.). In this scenario, your connector development will probably benefit the community. The right way is to open a PR on our repo to add your connector to our catalog. You will then benefit from an Airbyte team review and potential future improvements and maintenance from the community. -2. You want to build a connector for an **internal** source or destination (private API) specific to your organization. This connector has no good reason to be exposed to the community. - -This guide focuses on the second approach and assumes the following: -* You followed our other guides and tutorials about connector developments. -* You finished your connector development, running it locally on an Airbyte development instance. +--- +products: oss-* +sidebar_label: Uploading custom connectors +--- + +# Uploading Docker-based custom connectors + +:::info +This guide walks through the setup of a Docker-based custom connector. To understand how to use our low-code connector builder, read our guide [here](/connector-development/connector-builder-ui/overview.md). +::: + +If our connector catalog does not fulfill your needs, you can build your own Airbyte connectors! You can either use our [low-code connector builder](/connector-development/connector-builder-ui/overview.md) or upload a Docker-based custom connector. + +This page walks through the process to upload a **Docker-based custom connector**. This is an ideal route for connectors that have an **internal** use case like a private API with a specific fit for your organization. This guide for using Docker-based custom connectors assumes the following: +* You followed our other guides and tutorials about [connector development](/connector-development/connector-builder-ui/overview.md) +* You finished your connector development and have it running locally on an Airbyte development instance. * You want to deploy this connector to a production Airbyte instance running on a VM with docker-compose or on a Kubernetes cluster. -If you prefer video tutorials, [we recorded a demo about uploading connectors images to a GCP Artifact Registry](https://www.youtube.com/watch?v=4YF20PODv30&ab_channel=Airbyte). +If you prefer video tutorials, we recorded a demo on how to upload [connectors images to a GCP Artifact Registry](https://www.youtube.com/watch?v=4YF20PODv30&ab_channel=Airbyte). ## 1. Create a private Docker registry Airbyte needs to pull its Docker images from a remote Docker registry to consume a connector. @@ -70,42 +77,21 @@ If you want Airbyte to pull images from another private Docker registry, you wil You should run all the above commands from your local/CI environment, where your connector source code is available. -## 4. Use your custom connector in Airbyte +## 4. Use your custom Docker connector in Airbyte At this step, you should have: * A private Docker registry hosting your custom connector image. * Authenticated your Airbyte instance to your private Docker registry. You can pull your connector image from your private registry to validate the previous steps. On your Airbyte instance: run `docker pull :` if you are using our `docker-compose` deployment, or start a pod that is using the connector image. -### 1. Click on Settings -![Step 1 screenshot](https://images.tango.us/public/screenshot_bf5c3e27-19a3-4cc0-bc40-90c80afdbcba?crop=focalpoint&fit=crop&fp-x=0.0211&fp-y=0.9320&fp-z=2.9521&w=1200&mark-w=0.2&mark-pad=0&mark64=aHR0cHM6Ly9pbWFnZXMudGFuZ28udXMvc3RhdGljL21hZGUtd2l0aC10YW5nby13YXRlcm1hcmsucG5n&ar=4594%3A2234) +1. Click on `Settings` in the left-hand sidebar. Navigate to `Sources` or `Destinations` depending on your connector. Click on `Add a new Docker connector`. +2. Name your custom connector in `Connector display name`. This is just the display name used for your workspace. -### 2. Click on Sources (or Destinations) -![Step 2 screenshot](https://images.tango.us/public/screenshot_d956e987-424d-4f76-ad39-f6d6172f6acc?crop=focalpoint&fit=crop&fp-x=0.0855&fp-y=0.1083&fp-z=2.7473&w=1200&mark-w=0.2&mark-pad=0&mark64=aHR0cHM6Ly9pbWFnZXMudGFuZ28udXMvc3RhdGljL21hZGUtd2l0aC10YW5nby13YXRlcm1hcmsucG5n&ar=4594%3A2234) +3. Fill in the Docker `Docker full image name` and `Docker image tag`. +4. (Optional) Add a link to connector's documentation in `Connector documentation URL` +You can optionally fill this with any value if you do not have online documentation for your connector. +This documentation will be linked in your connector setting's page. -### 3. Click on + New connector -![Step 3 screenshot](https://images.tango.us/public/screenshot_52248202-6351-496d-bc8f-892c43cf7cf8?crop=focalpoint&fit=crop&fp-x=0.8912&fp-y=0.0833&fp-z=3.0763&w=1200&mark-w=0.2&mark-pad=0&mark64=aHR0cHM6Ly9pbWFnZXMudGFuZ28udXMvc3RhdGljL21hZGUtd2l0aC10YW5nby13YXRlcm1hcmsucG5n&ar=4594%3A2234) - - -### 4. Fill the name of your custom connector -![Step 4 screenshot](https://images.tango.us/public/screenshot_809a22c8-ff38-4b10-8292-bce7364f111c?crop=focalpoint&fit=crop&fp-x=0.4989&fp-y=0.4145&fp-z=1.9188&w=1200&mark-w=0.2&mark-pad=0&mark64=aHR0cHM6Ly9pbWFnZXMudGFuZ28udXMvc3RhdGljL21hZGUtd2l0aC10YW5nby13YXRlcm1hcmsucG5n&ar=4594%3A2234) - - -### 5. Fill the Docker image name of your custom connector -![Step 5 screenshot](https://images.tango.us/public/screenshot_ed91d789-9fc7-4758-a6f0-50bf2f04f248?crop=focalpoint&fit=crop&fp-x=0.4989&fp-y=0.4924&fp-z=1.9188&w=1200&mark-w=0.2&mark-pad=0&mark64=aHR0cHM6Ly9pbWFnZXMudGFuZ28udXMvc3RhdGljL21hZGUtd2l0aC10YW5nby13YXRlcm1hcmsucG5n&ar=4594%3A2234) - - -### 6. Fill the Docker Tag of your custom connector image -![Step 6 screenshot](https://images.tango.us/public/screenshot_5b6bff70-5703-4dac-b359-95b9ab8f8ce1?crop=focalpoint&fit=crop&fp-x=0.4989&fp-y=0.5703&fp-z=1.9188&w=1200&mark-w=0.2&mark-pad=0&mark64=aHR0cHM6Ly9pbWFnZXMudGFuZ28udXMvc3RhdGljL21hZGUtd2l0aC10YW5nby13YXRlcm1hcmsucG5n&ar=4594%3A2234) - - -### 7. Fill the URL to your connector documentation -This is a required field at the moment, but you can fill with any value if you do not have online documentation for your connector. -This documentation will be linked in the connector setting page. -![Step 7 screenshot](https://images.tango.us/public/screenshot_007e6465-619f-4553-8d65-9af2f5ad76bc?crop=focalpoint&fit=crop&fp-x=0.4989&fp-y=0.6482&fp-z=1.9188&w=1200&mark-w=0.2&mark-pad=0&mark64=aHR0cHM6Ly9pbWFnZXMudGFuZ28udXMvc3RhdGljL21hZGUtd2l0aC10YW5nby13YXRlcm1hcmsucG5n&ar=4594%3A2234) - - -### 8. Click on Add -![Step 8 screenshot](https://images.tango.us/public/screenshot_c097183f-1687-469f-852d-f66f743e8c10?crop=focalpoint&fit=crop&fp-x=0.5968&fp-y=0.7010&fp-z=3.0725&w=1200&mark-w=0.2&mark-pad=0&mark64=aHR0cHM6Ly9pbWFnZXMudGFuZ28udXMvc3RhdGljL21hZGUtd2l0aC10YW5nby13YXRlcm1hcmsucG5n&ar=4594%3A2234) +5. `Add` the connector to save the configuration. You can now select your new connector when setting up a new connection! \ No newline at end of file diff --git a/docs/operator-guides/using-dagster-integration.md b/docs/operator-guides/using-dagster-integration.md index ac5f60834bcf..4c4ef2c84c44 100644 --- a/docs/operator-guides/using-dagster-integration.md +++ b/docs/operator-guides/using-dagster-integration.md @@ -1,5 +1,6 @@ --- description: Start triggering Airbyte jobs with Dagster in minutes +products: oss-* --- # Using the Dagster Integration diff --git a/docs/operator-guides/using-kestra-plugin.md b/docs/operator-guides/using-kestra-plugin.md index 3c27e8797c2d..0a8da24761a3 100644 --- a/docs/operator-guides/using-kestra-plugin.md +++ b/docs/operator-guides/using-kestra-plugin.md @@ -1,5 +1,6 @@ --- description: Using the Kestra Plugin to Orchestrate Airbyte +products: oss-* --- # Using the Kestra Plugin diff --git a/docs/operator-guides/using-prefect-task.md b/docs/operator-guides/using-prefect-task.md index d0b462e10f23..57e17a06b40e 100644 --- a/docs/operator-guides/using-prefect-task.md +++ b/docs/operator-guides/using-prefect-task.md @@ -1,5 +1,6 @@ --- description: Start triggering Airbyte jobs with Prefect in minutes +products: oss-* --- # Using the Prefect Airbyte Task diff --git a/docs/operator-guides/using-the-airflow-airbyte-operator.md b/docs/operator-guides/using-the-airflow-airbyte-operator.md index 97c73eb37f91..5b4e580b191e 100644 --- a/docs/operator-guides/using-the-airflow-airbyte-operator.md +++ b/docs/operator-guides/using-the-airflow-airbyte-operator.md @@ -1,5 +1,6 @@ --- description: Start triggering Airbyte jobs with Apache Airflow in minutes +products: oss-* --- # Using the Airbyte Operator to orchestrate Airbyte OSS diff --git a/docs/project-overview/README.md b/docs/project-overview/README.md deleted file mode 100644 index a427d02b0519..000000000000 --- a/docs/project-overview/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# Project Overview - diff --git a/docs/project-overview/code-of-conduct.md b/docs/project-overview/code-of-conduct.md deleted file mode 100644 index 9eacce28a212..000000000000 --- a/docs/project-overview/code-of-conduct.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -description: Our Community Code of Conduct ---- - -# Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to make participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. - -## Our Standards - -Examples of behavior that contributes to creating a positive environment include: - -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and unwelcome sexual attention or advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others’ private information, such as a physical or electronic address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a professional setting - -## Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. - -## Scope - -This Code of Conduct applies within all project spaces, and it also applies when an individual is representing the project or its community in public spaces. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at [conduct@airbyte.io](mailto:conduct@airbyte.io). All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. - -Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project’s leadership. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/), version 1.4, available at [https://www.contributor-covenant.org/version/1/4/code-of-conduct.html](https://www.contributor-covenant.org/version/1/4/code-of-conduct.html) - diff --git a/docs/project-overview/product-release-stages.md b/docs/project-overview/product-release-stages.md deleted file mode 100644 index 5eebc97556e3..000000000000 --- a/docs/project-overview/product-release-stages.md +++ /dev/null @@ -1,39 +0,0 @@ -# Product Release Stages - -The following release stages describe the lifecycle of an Airbyte product, feature, or connector. - -| Expectations | Alpha | Beta | General Availability (GA)| -|:-------------|:------|:-----|:-------------------------| -| Customer Availability | Alpha features and products may have limited availability (by invitation only)

      Alpha connectors are available to all users | Beta features and products may have limited availability (by invitation only)

      Beta connectors are available to all users | Available to all users | -|Support | Cloud: No Support SLAs

      Open-source: Community Slack Support | Cloud: Official Beta Support SLA

      Open-source: Community Slack Support | Cloud: Official GA Support SLA

      Open-source: Community Slack Support | -| Production Readiness | No | Yes (with caveats) | Yes | - -## Alpha -An alpha release signifies a product, feature, or connector under development and helps Airbyte gather early feedback and issues reported by early adopters. We strongly discourage using alpha releases for production use cases and do not offer Cloud Support SLAs around these products, features, or connectors. - -### What you should know about an alpha release - -- An alpha release might not be feature-complete (features planned for the release are under development) and may include backward-incompatible/breaking API changes. -- Access for alpha features and products may not be enabled for all Airbyte users by default. Depending on the feature, you may enable the feature either from the Airbyte UI or by contacting Airbyte Support. Alpha connectors are available to all users. -- Alpha releases may be announced via email, in the Airbyte UI, and/or through certain pages of the Airbyte docs. - -## Beta -A beta release is considered stable and reliable with no backwards incompatible changes but has not been validated by a broader group of users. We expect to find and fix a few issues and bugs in the release before it’s ready for GA. - -### What you should know about a beta release - -- A beta release is generally feature-complete (features planned for the release have been mostly implemented) and does not include backward-incompatible/breaking API changes. -- Access may be enabled for all Airbyte users by default. Depending on the feature, you may enable the feature either from the Airbyte UI or by contacting Airbyte Support. Beta connectors are available to all users. -- Beta releases may be announced via email, in the Airbyte UI, and/or through certain pages of the Airbyte docs. - -## General availability (GA) -A generally available release has been deemed ready for use in a production environment and is officially supported by Airbyte. Its documentation is considered sufficient to support widespread adoption. - -### What you should know about a GA release - -- A GA release is feature-complete (features planned for the release have been fully implemented) and does not include backward-incompatible/breaking API changes. -- Access is enabled for all Airbyte users by default. Depending on the feature, you may enable the feature either from the Airbyte UI or by contacting Airbyte Support. -- GA releases may be announced via email, in the Airbyte UI, and/or through certain pages of the Airbyte docs. - -## Deprecated -A deprecated feature, product, or connector is no longer officially supported by Airbyte. It might continue to work for a period of time but Airbyte recommends that you migrate away from and avoid relying on deprecated releases. diff --git a/docs/project-overview/slack-code-of-conduct.md b/docs/project-overview/slack-code-of-conduct.md deleted file mode 100644 index c88da4c1adb5..000000000000 --- a/docs/project-overview/slack-code-of-conduct.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -description: Be nice to one another. ---- - -# Slack Code of Conduct - -Airbyte's Slack community is growing incredibly fast. We're home to over 1500 data professionals and are growing at an awesome pace. We are proud of our community, and have provided these guidelines to support new members in maintaining the wholesome spirit we have developed here. We appreciate your continued commitment to making this a community we are all excited to be a part of. - -## Rule 1: Be respectful. - -Our desire is for everyone to have a positive, fulfilling experience in Airbyte Slack, and we sincerely appreciate your help in making this happen. -All of the guidelines we provide below are important, but there’s a reason respect is the first rule. We take it seriously, and while the occasional breach of etiquette around Slack is forgivable, we cannot condone disrespectful behavior. - -## Rule 2: Use the most relevant channels. - -We deliberately use topic-specific Slack channels so members of the community can opt-in on various types of conversations. Our members take care to post their messages in the most relevant channel, and you’ll often see reminders about the best place to post a message (respectfully written, of course!). If you're looking for help directly from the Community Assistance Team or other Airbyte employees, please stick to posting in the airbyte-help channel, so we know you're asking us specifically! - -## Rule 3: Don’t double-post. - -Please be considerate of our community members’ time. We know your question is important, but please keep in mind that Airbyte Slack is not a customer service platform but a community of volunteers who will help you as they are able around their own work schedule. You have access to all the history, so it’s easy to check if your question has already been asked. - -## Rule 4: Check question for clarity and thoughtfulness. - -Airbyte Slack is a community of volunteers. Our members enjoy helping others; they are knowledgeable, gracious, and willing to give their time and expertise for free. Putting some effort into a well-researched and thoughtful post shows consideration for their time and will gain more responses. - -## Rule 5: Keep it public. - -This is a public forum; please do not contact individual members of this community without their express permission, regardless of whether you are trying to recruit someone, sell a product, or solicit help. - -## Rule 6: No soliciting! - -The purpose of the Airbyte Slack community is to provide a forum for data practitioners to discuss their work and share their ideas and learnings. It is not intended as a place to generate leads for vendors or recruiters, and may not be used as such. - -If you’re a vendor, you may advertise your product in #shameless-plugs. Advertising your product anywhere else is strictly against the rules. - -## Rule 7: Don't spam tags, or use @here or @channel. - -Using the @here and @channel keywords in a post will not help, as they are disabled in Slack for everyone excluding admins. Nonetheless, if you use them we will remind you with a link to this rule, to help you better understand the way Airbyte Slack operates. - -Do not tag specific individuals for help on your questions. If someone chooses to respond to your question, they will do so. You will find that our community of volunteers is generally very responsive and amazingly helpful! - -## Rule 8: Use threads for discussion. - -The simplest way to keep conversations on track in Slack is to use threads. The Airbyte Slack community relies heavily on threads, and if you break from this convention, rest assured one of our community members will respectfully inform you quickly! - -_If you see a message or receive a direct message that violates any of these rules, please contact an Airbyte team member and we will take the appropriate moderation action immediately. We have zero tolerance for intentional rule-breaking and hate speech._ - diff --git a/docs/quickstart/add-a-destination.md b/docs/quickstart/add-a-destination.md deleted file mode 100644 index 1204df4038a1..000000000000 --- a/docs/quickstart/add-a-destination.md +++ /dev/null @@ -1,16 +0,0 @@ -# Add a Destination - -The destination we are creating is a simple JSON line file, meaning that it will contain one JSON object per line. Each objects will represent data extracted from the source. - -The resulting files will be located in `/tmp/airbyte_local/json_data` - -To set it up, just follow the instructions on the screenshot below. - -:::info - -You might have to wait ~30 seconds before the fields show up because it is the first time you're using Airbyte. - -::: - -![](../.gitbook/assets/getting-started-destination.png) - diff --git a/docs/quickstart/add-a-source.md b/docs/quickstart/add-a-source.md deleted file mode 100644 index e6fbc1473109..000000000000 --- a/docs/quickstart/add-a-source.md +++ /dev/null @@ -1,18 +0,0 @@ -# Add a Source - -You can either follow this tutorial from the onboarding or through the UI, where you can first navigate to the `Sources` tab on the left bar. - -Our demo source will pull data from an external API, which will pull down the information on one specified Pokémon. - -To set it up, just follow the instructions on the screenshot below. - -:::info - -You might have to wait ~30 seconds before the fields show up because it is the first time you're using Airbyte. - -::: - -![](../.gitbook/assets/getting-started-source.png) - -Can't find the connectors that you want? Try your hand at easily building one yourself using our [Python CDK for HTTP API sources!](../connector-development/cdk-python/) - diff --git a/docs/quickstart/deploy-airbyte.md b/docs/quickstart/deploy-airbyte.md deleted file mode 100644 index 4df34e9aa05a..000000000000 --- a/docs/quickstart/deploy-airbyte.md +++ /dev/null @@ -1,28 +0,0 @@ -# Deploy Airbyte - -Deploying Airbyte Open-Source just takes two steps. - -1. Install Docker on your workstation \(see [instructions](https://www.docker.com/products/docker-desktop)\). Make sure you're on the latest version of `docker-compose`. -2. Run the following commands in your terminal: - -```bash -git clone https://github.com/airbytehq/airbyte.git -cd airbyte -./run-ab-platform.sh -``` - -Once you see an Airbyte banner, the UI is ready to go at [http://localhost:8000](http://localhost:8000)! You will be asked for a username and password. By default, that's username `airbyte` and password `password`. Once you deploy airbyte to your servers, **be sure to change these** in your `.env` file. - -Alternatively, if you have an Airbyte Cloud invite, just follow [these steps.](../deploying-airbyte/on-cloud.md) - -If you need direct access to our team for any kind of assistance, don't hesitate to [talk to our team](https://airbyte.com/talk-to-sales-premium-support) to discuss about our premium support offers. - -## FAQ - -If you have any questions about the Airbyte Open-Source setup and deployment process, head over to our [Getting Started FAQ](https://github.com/airbytehq/airbyte/discussions/categories/questions) on our Airbyte Forum that answers the following questions and more: - -- How long does it take to set up Airbyte? -- Where can I see my data once I've run a sync? -- Can I set a start time for my sync? - -If there are any questions that we couldn't answer here, we'd love to help you get started. [Join our Slack](https://airbytehq.slack.com/ssb/redirect) and feel free to ask your questions in the \#getting-started channel. diff --git a/docs/quickstart/getting-started.md b/docs/quickstart/getting-started.md deleted file mode 100644 index afb0e3408522..000000000000 --- a/docs/quickstart/getting-started.md +++ /dev/null @@ -1,105 +0,0 @@ -# Getting Started - -## Goal - -During this getting started tutorial, we are going to replicate currencies closing price into a JSON file. - -## Start Airbyte - -First of all, make sure you have Docker and Docker Compose installed. Then run the following commands: - -```text -git clone https://github.com/airbytehq/airbyte.git -cd airbyte -./run-ab-platform.sh -``` - -Once you see an Airbyte banner, the UI is ready to go at [http://localhost:8000/](http://localhost:8000/). - -## Set up your preferences - -You should see an onboarding page. Enter your email if you want updates about Airbyte and continue. - -![](../.gitbook/assets/airbyte_get-started.png) - -## Set up your first connection - -### Create a source - -The source we are creating will pull data from an external API. It will replicate the closing price of currencies compared to USD since the specified start date. - -To set it up, just follow the instructions on the screenshot below. - -:::info - -You might have to wait ~30 seconds before the fields show up because it is the first time you're using Airbyte. - -::: - -![](../.gitbook/assets/demo_source.png) - -### Create a destination - -The destination we are creating is a simple JSON line file, meaning that it will contain one JSON object per line. Each objects will represent data extracted from the source. - -The resulting files will be located in `/tmp/airbyte_local/json_data` - -:::caution - -Please make sure that Docker Desktop has access to `/tmp` (and `/private` on a MacOS, as /tmp has a symlink that points to /private. It will not work otherwise). You allow it with "File sharing" in `Settings -> Resources -> File sharing -> add the one or two above folder` and hit the "Apply & restart" button. - -::: - -To set it up, just follow the instructions on the screenshot below. - -:::info - -You might have to wait ~30 seconds before the fields show up because it is the first time you're using Airbyte. - -::: - -![](../.gitbook/assets/demo_destination.png) - -### Create connection - -When we create the connection, we can select which data stream we want to replicate. We can also select if we want an incremental replication. The replication will run at the specified sync frequency. - -To set it up, just follow the instructions on the screenshot below. - -![](../.gitbook/assets/demo_connection.png) - -## Check the logs of your first sync - -After you've completed the onboarding, you will be redirected to the source list and will see the source you just added. Click on it to find more information about it. You will now see all the destinations connected to that source. Click on it and you will see the sync history. - -From there, you can look at the logs, download them, force a sync and adjust the configuration of your connection. - -![](../.gitbook/assets/demo_history.png) - -## Check the data of your first sync - -Now let's verify that this worked: - -```bash -cat /tmp/airbyte_local/json_data/_airbyte_raw_exchange_rate.jsonl -``` - -You should see one line for each day that was replicated. - -If you have [`jq`](https://stedolan.github.io/jq/) installed, let's look at the evolution of `EUR`. - -```bash -cat /tmp/airbyte_local/test_json/_airbyte_raw_exchange_rate.jsonl | -jq -c '.data | {date: .date, EUR: .EUR }' -``` - -And there you have it. You've pulled data from an API directly into a file and all of the actual configuration for this replication only took place in the UI. - -## That's it! - -This is just the beginning of using Airbyte. We support a large collection of sources and destinations. You can even contribute your own. - -If you have any questions at all, please reach out to us on [Slack](https://slack.airbyte.io/). We’re still in alpha, so if you see any rough edges or want to request a connector you need, please create an issue on our [Github](https://github.com/airbytehq/airbyte) or leave a thumbs up on an existing issue. - -Thank you and we hope you enjoy using Airbyte. - diff --git a/docs/quickstart/set-up-a-connection.md b/docs/quickstart/set-up-a-connection.md deleted file mode 100644 index 2c9add584812..000000000000 --- a/docs/quickstart/set-up-a-connection.md +++ /dev/null @@ -1,44 +0,0 @@ -# Set up a Connection - -When we create the connection, we can select which data stream we want to replicate. We can also select if we want an incremental replication, although it isn't currently offered for this source. The replication will run at the specified sync frequency. - -To set it up, just follow the instructions on the screenshot below. - -![](../.gitbook/assets/getting-started-connection.png) - -## Check the logs of your first sync - -After you've completed the onboarding, you will be redirected to the source list and will see the source you just added. Click on it to find more information about it. You will now see all the destinations connected to that source. Click on it and you will see the sync history. - -From there, you can look at the logs, download them, force a sync and adjust the configuration of your connection. - -![](../.gitbook/assets/getting-started-logs.png) - -## Check the data of your first sync - -Now let's verify that this worked: - -```bash -cat /tmp/airbyte_local/json_data/_airbyte_raw_pokemon.jsonl -``` - -You should see a large JSON object with the response from the API, giving you a lot of information about the selected Pokemon. - -If you have [`jq`](https://stedolan.github.io/jq/) installed, let's look at some of the data that we have replicated about `charizard`. We'll pull its abilities and weight: - -```bash -cat _airbyte_raw_pokemon.jsonl | -jq '._airbyte_data | {abilities: .abilities, weight: .weight}' -``` - -And there you have it. You've pulled data from an API directly into a file, with all of the actual configuration for this replication only taking place in the UI. - -Note: If you are using Airbyte on Windows with WSL2 and Docker, refer to [this tutorial](../operator-guides/locating-files-local-destination.md) or [this section](../integrations/destinations/local-json.md#access-replicated-data-files) in the local-json destination guide to locate the replicated folder and file. - -## That's it! - -This is just the beginning of using Airbyte. We support a large collection of sources and destinations. You can even contribute your own. - -If you have any questions at all, please reach out to us on [Slack](https://slack.airbyte.io/). We’re still in alpha, so if you see any rough edges or want to request a connector you need, please create an issue on our [Github](https://github.com/airbytehq/airbyte) or leave a thumbs up on an existing issue. - -Thank you and we hope you enjoy using Airbyte. diff --git a/docs/readme.md b/docs/readme.md index cbf550c2a7a6..708a6a790430 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -1,21 +1,25 @@ +--- +displayed_sidebar: docs +--- + # Welcome to Airbyte Docs Whether you are an Airbyte user or contributor, we have docs for you! ## For Airbyte Cloud users -Browse the [connector catalog](https://docs.airbyte.com/integrations/) to find the connector you want. In case the connector is not yet supported on Airbyte Cloud, consider using [Airbyte Open Source](#for-airbyte-open-source-users). +Browse the [connector catalog](/integrations/) to find the connector you want. In case the connector is not yet supported on Airbyte Cloud, consider using [Airbyte Open Source](#for-airbyte-open-source-users). -Next, check out the [step-by-step tutorial](https://docs.airbyte.com/cloud/getting-started-with-airbyte-cloud) to sign up for Airbyte Cloud, understand Airbyte [concepts](https://docs.airbyte.com/cloud/core-concepts), and run your first sync. Then learn how to [use your Airbyte Cloud account](https://docs.airbyte.com/category/using-airbyte-cloud). +Next, check out the [step-by-step tutorial](/using-airbyte/getting-started) to sign up for Airbyte Cloud, understand Airbyte [concepts](/using-airbyte/core-concepts), and run your first sync. ## For Airbyte Open Source users -Browse the [connector catalog](https://docs.airbyte.com/integrations/) to find the connector you want. If the connector is not yet supported on Airbyte Open Source, [build your own connector](https://docs.airbyte.com/connector-development/). +Browse the [connector catalog](/integrations/) to find the connector you want. If the connector is not yet supported on Airbyte Open Source, [build your own connector](/connector-development/). -Next, check out the [Airbyte Open Source QuickStart](https://docs.airbyte.com/quickstart/deploy-airbyte). Then learn how to [deploy](https://docs.airbyte.com/deploying-airbyte/local-deployment) and [manage](https://docs.airbyte.com/operator-guides/upgrading-airbyte) Airbyte Open Source in your cloud infrastructure. +Next, check out the [Airbyte Open Source QuickStart](/quickstart/deploy-airbyte). Then learn how to [deploy](/deploying-airbyte/local-deployment) and [manage](/operator-guides/upgrading-airbyte) Airbyte Open Source in your cloud infrastructure. ## For Airbyte contributors -To contribute to Airbyte code, connectors, and documentation, refer to our [Contributing Guide](https://docs.airbyte.com/contributing-to-airbyte/). +To contribute to Airbyte code, connectors, and documentation, refer to our [Contributing Guide](/contributing-to-airbyte/). [![GitHub stars](https://img.shields.io/github/stars/airbytehq/airbyte?style=social&label=Star&maxAge=2592000)](https://GitHub.com/airbytehq/airbyte/stargazers/) [![License](https://img.shields.io/static/v1?label=license&message=MIT&color=brightgreen)](https://github.com/airbytehq/airbyte/tree/a9b1c6c0420550ad5069aca66c295223e0d05e27/LICENSE/README.md) [![License](https://img.shields.io/static/v1?label=license&message=ELv2&color=brightgreen)](https://github.com/airbytehq/airbyte/tree/a9b1c6c0420550ad5069aca66c295223e0d05e27/LICENSE/README.md) diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html index 0a7a2d70b2d0..5dbff2a0eeae 100644 --- a/docs/reference/api/generated-api-html/index.html +++ b/docs/reference/api/generated-api-html/index.html @@ -184,7 +184,7 @@

      Airbyte Configuration API

      https://airbyte.io.

      The Configuration API is an internal Airbyte API that is designed for communications between different Airbyte components.

        -
      • It's main purpose is to enable the Airbyte Engineering team to configure the internal state of Airbyte Cloud
      • +
      • Its main purpose is to enable the Airbyte Engineering team to configure the internal state of Airbyte Cloud
      • It is also sometimes used by OSS users to configure their own Self-Hosted Airbyte deployment (internal state, etc)

      WARNING

      @@ -199,7 +199,7 @@

      Airbyte Configuration API

      • All endpoints are http POST methods.
      • All endpoints accept data via application/json request bodies. The API does not accept any data via query params.
      • -
      • The naming convention for endpoints is: localhost:8000/{VERSION}/{METHOD_FAMILY}/{METHOD_NAME} e.g. localhost:8000/v1/connections/create.
      • +
      • The naming convention for endpoints is: localhost:8000/api/{VERSION}/{METHOD_FAMILY}/{METHOD_NAME} e.g. localhost:8000/api/v1/connections/create.
      • For all update methods, the whole object must be passed in, even the fields that did not change.

      Authentication (OSS):

      diff --git a/docs/reference/api/rapidoc-api-docs.html b/docs/reference/api/rapidoc-api-docs.html index 95dbd9b4eb47..5d867d94f803 100644 --- a/docs/reference/api/rapidoc-api-docs.html +++ b/docs/reference/api/rapidoc-api-docs.html @@ -5,7 +5,7 @@ - diff --git a/docs/release_notes/assets/destinations-v2-column-changes.png b/docs/release_notes/assets/destinations-v2-column-changes.png deleted file mode 100644 index ac15f0b292c3..000000000000 Binary files a/docs/release_notes/assets/destinations-v2-column-changes.png and /dev/null differ diff --git a/docs/release_notes/assets/updated_table_columns.png b/docs/release_notes/assets/updated_table_columns.png new file mode 100644 index 000000000000..ffe95043cd27 Binary files /dev/null and b/docs/release_notes/assets/updated_table_columns.png differ diff --git a/docs/release_notes/destinations_v2.js b/docs/release_notes/destinations_v2.js index ce8d4c19c5e0..9a7cf6a0b0eb 100644 --- a/docs/release_notes/destinations_v2.js +++ b/docs/release_notes/destinations_v2.js @@ -1,5 +1,5 @@ -import React, { useState } from 'react'; -import CodeBlock from '@theme/CodeBlock'; +import React, { useState } from "react"; +import CodeBlock from "@theme/CodeBlock"; function concatenateRawTableName(namespace, name) { let plainConcat = namespace + name; @@ -8,18 +8,21 @@ function concatenateRawTableName(namespace, name) { for (let i = 0; i < plainConcat.length; i++) { // If we've found an underscore, count the number of consecutive underscores let underscoreRun = 0; - while (i < plainConcat.length && plainConcat.charAt(i) === '_') { - underscoreRun++; - i++; + while (i < plainConcat.length && plainConcat.charAt(i) === "_") { + underscoreRun++; + i++; } longestUnderscoreRun = Math.max(longestUnderscoreRun, underscoreRun); } - return namespace + "_raw" + "_".repeat(longestUnderscoreRun + 1) + "stream_" + name; + return ( + namespace + "_raw" + "_".repeat(longestUnderscoreRun + 1) + "stream_" + name + ); } // Taken from StandardNameTransformer function convertStreamName(str) { - return str.normalize('NFKD') + return str + .normalize("NFKD") .replaceAll(/\p{M}/gu, "") .replaceAll(/\s+/g, "_") .replaceAll(/[^A-Za-z0-9_]/g, "_"); @@ -43,10 +46,13 @@ export const BigQueryMigrationGenerator = () => { return namespace; } - function generateSql(namespace, name, raw_dataset) { - let v2RawTableName = '`' + bigqueryConvertStreamName(concatenateRawTableName(namespace, name)) + '`'; - let v1namespace = '`' + escapeNamespace(namespace) + '`'; - let v1name = '`' + bigqueryConvertStreamName("_airbyte_raw_" + name) + '`'; + function generateSql(og_namespace, new_namespace, name, raw_dataset) { + let v2RawTableName = + "`" + + bigqueryConvertStreamName(concatenateRawTableName(new_namespace, name)) + + "`"; + let v1namespace = "`" + escapeNamespace(og_namespace) + "`"; + let v1name = "`" + bigqueryConvertStreamName("_airbyte_raw_" + name) + "`"; return `CREATE SCHEMA IF NOT EXISTS ${raw_dataset}; CREATE OR REPLACE TABLE \`${raw_dataset}\`.${v2RawTableName} ( _airbyte_raw_id STRING, @@ -66,9 +72,9 @@ AS ( } return ( - + ); -} +}; export const SnowflakeMigrationGenerator = () => { // See SnowflakeSQLNameTransformer @@ -80,9 +86,10 @@ export const SnowflakeMigrationGenerator = () => { return "_" + str; } } - function generateSql(namespace, name, raw_schema) { - let v2RawTableName = '"' + concatenateRawTableName(namespace, name) + '"'; - let v1namespace = snowflakeConvertStreamName(namespace); + function generateSql(og_namespace, new_namespace, name, raw_schema) { + let v2RawTableName = + '"' + concatenateRawTableName(new_namespace, name) + '"'; + let v1namespace = snowflakeConvertStreamName(og_namespace); let v1name = snowflakeConvertStreamName("_airbyte_raw_" + name); return `CREATE SCHEMA IF NOT EXISTS "${raw_schema}"; CREATE OR REPLACE TABLE "${raw_schema}".${v2RawTableName} ( @@ -100,50 +107,120 @@ AS ( )`; } return ( - + ); -} +}; + +export const RedshiftMigrationGenerator = () => { + // See RedshiftSQLNameTransformer + function redshiftConvertStreamName(str) { + str = convertStreamName(str); + if (str.charAt(0).match(/[A-Za-z_]/)) { + return str; + } else { + return "_" + str; + } + } + function generateSql(og_namespace, new_namespace, name, raw_schema) { + let v2RawTableName = + '"' + concatenateRawTableName(new_namespace, name) + '"'; + let v1namespace = redshiftConvertStreamName(og_namespace); + let v1name = redshiftConvertStreamName("_airbyte_raw_" + name); + return `CREATE SCHEMA IF NOT EXISTS "${raw_schema}"; +DROP TABLE IF EXISTS "${raw_schema}".${v2RawTableName}; +CREATE TABLE "${raw_schema}".${v2RawTableName} ( + "_airbyte_raw_id" VARCHAR(36) NOT NULL PRIMARY KEY + , "_airbyte_extracted_at" TIMESTAMPTZ DEFAULT NOW() + , "_airbyte_loaded_at" TIMESTAMPTZ + , "_airbyte_data" SUPER +); +INSERT INTO "${raw_schema}".${v2RawTableName} ( + SELECT + _airbyte_ab_id AS "_airbyte_raw_id", + _airbyte_emitted_at AS "_airbyte_extracted_at", + CAST(NULL AS TIMESTAMPTZ) AS "_airbyte_loaded_at", + _airbyte_data AS "_airbyte_data" + FROM ${v1namespace}.${v1name} +);`; + } + return ( + + ); +}; -export const MigrationGenerator = ({destination, generateSql}) => { - const defaultMessage = -`Enter your stream's name and namespace to see the SQL output. +export const MigrationGenerator = ({ destination, generateSql }) => { + const defaultMessage = `Enter your stream's name and namespace to see the SQL output. If your stream has no namespace, take the default value from the destination connector's settings.`; const [message, updateMessage] = useState({ - 'message': defaultMessage, - 'language': 'text' + message: defaultMessage, + language: "text", }); function updateSql(event) { - let namespace = document.getElementById("stream_namespace_" + destination).value; + let og_namespace = document.getElementById( + "og_stream_namespace_" + destination + ).value; + let new_namespace = document.getElementById( + "new_stream_namespace_" + destination + ).value; let name = document.getElementById("stream_name_" + destination).value; - var raw_dataset = document.getElementById("raw_dataset_" + destination).value; - if (raw_dataset === '') { - raw_dataset = 'airbyte_internal'; + var raw_dataset = document.getElementById( + "raw_dataset_" + destination + ).value; + if (raw_dataset === "") { + raw_dataset = "airbyte_internal"; } - let sql = generateSql(namespace, name, raw_dataset); - if (namespace !== "" && name !== "") { + let sql = generateSql(og_namespace, new_namespace, name, raw_dataset); + if ([og_namespace, new_namespace, name].every((text) => text != "")) { updateMessage({ - 'message': sql, - 'language': 'sql' + message: sql, + language: "sql", }); } else { updateMessage({ - 'message': defaultMessage, - 'language': 'text' + message: defaultMessage, + language: "text", }); } } return (
      - -
      + + +
      + + +
      -
      - -
      - - { message['message'] } + +
      + + +
      + + {message["message"]}
      ); -} +}; diff --git a/docs/release_notes/july_2022.md b/docs/release_notes/july_2022.md index 0c6cbc35e004..c3a4c8240b2b 100644 --- a/docs/release_notes/july_2022.md +++ b/docs/release_notes/july_2022.md @@ -19,7 +19,7 @@ This page includes new features and improvements to the Airbyte Cloud and Airbyt * Airbyte is currently developing a low-code connector builder, which allows you to easily create new source and destination connectors in your workspace. [#14402](https://github.com/airbytehq/airbyte/pull/14402) [#14317](https://github.com/airbytehq/airbyte/pull/14317) [#14288](https://github.com/airbytehq/airbyte/pull/14288) [#14004](https://github.com/airbytehq/airbyte/pull/14004) -* Added [documentation](https://docs.airbyte.com/cloud/managing-airbyte-cloud/manage-airbyte-cloud-workspace#single-workspace-vs-multiple-workspaces) about the benefits and considerations of having a single workspace vs. multiple workspaces in Airbyte Cloud. [#14608](https://github.com/airbytehq/airbyte/pull/14608) +* Added [documentation](/using-airbyte/workspaces.md#single-workspace-vs-multiple-workspaces) about the benefits and considerations of having a single workspace vs. multiple workspaces in Airbyte Cloud. [#14608](https://github.com/airbytehq/airbyte/pull/14608) ### Improvements * Improved platform security by using Docker images from the latest version of OpenJDK (openjdk:19-slim-bullseye). [#14971](https://github.com/airbytehq/airbyte/pull/14971) diff --git a/docs/release_notes/november_2023.md b/docs/release_notes/november_2023.md new file mode 100644 index 000000000000..67323252e8c5 --- /dev/null +++ b/docs/release_notes/november_2023.md @@ -0,0 +1,24 @@ +# November 2023 +## airbyte v0.50.34 to v0.50.35 + +This page includes new features and improvements to the Airbyte Cloud and Airbyte Open Source platforms. + +## ✨ Highlights + +Airbyte now supports extracting text content from PDF, Docx, and Pptx files from S3, Azure Blob Storage, and the newly introduced [Google Drive](/integrations/sources/google-drive.md) source. This is an important part of supporting LLM use cases that rely on unstructured data in files. + +SSO and RBAC (admin roles only) are now available in Airbyte Cloud! Read more below. + +## Platform Releases +- **SSO and RBAC** You can now use SSO in Airbyte Cloud to administer permissions in Airbyte. This is currently only available through Okta, with plans to support Active Directory next. We also now offer **RBAC** (admin roles only) to ensure a high level of security when managing you workspace. For access to this feature, reach out to our [Sales team](https://www.airbyte.com/company/talk-to-sales). +- **Continuous heartbeat checks** We're continually monitoring syncs to verify they continue making progress, and have added functionality in the background to ensure that we continue receiving updated ["heartbeat" messages](/understanding-airbyte/heartbeats.md) from our connectors. This will ensure that we continue delivering data and avoid any timeouts. + +## Connector Improvements + +In addition to being able to extract text content from unstructured data sources, we have also: + + - Revamped core Marketing connectors Pinterest, Instagram and Klaviyo to significantly improve the setup experience and ensure resiliency and reliability. + - [Added incremenetal sync](https://github.com/airbytehq/airbyte/pull/32473) functionality for Hubspot's stream `property_history`, which improves sync time and reliability. + - [Added new streams](https://github.com/airbytehq/airbyte/pull/32738) for Amazon Seller Partner: `get_vendor_net_pure_product_margin_report`,`get_vendor_readl_time_inventory_report`, and `get_vendor_traffic_report` to enable additional reporting. + - Released our first connector, Stripe, that can perform [concurrent syncs](https://github.com/airbytehq/airbyte/pull/32473) where streams sync in parallel when syncing in Full Refresh mode. + diff --git a/docs/release_notes/october_2023.md b/docs/release_notes/october_2023.md new file mode 100644 index 000000000000..79126acbadde --- /dev/null +++ b/docs/release_notes/october_2023.md @@ -0,0 +1,36 @@ +# October 2023 +## airbyte v0.50.31 to v0.50.33 + +This page includes new features and improvements to the Airbyte Cloud and Airbyte Open Source platforms. + +## ✨ Highlights + +With lightning-fast replication speeds of over 10 MB per second, incremental CDC syncs and more resumable snapshots, we've redefined what you can expect in terms of speed and reliability when replicating data from both [MySQL](https://airbyte.com/blog/behind-the-performance-improvements-of-our-mysql-source) and [MongoDB](https://airbyte.com/blog/10-mb-per-second-incremental-mongodb-syncs) databases. + +In [v0.50.31](https://github.com/airbytehq/airbyte-platform/releases/tag/v0.50.31), we also released [versioned Connector Documentation](https://github.com/airbytehq/airbyte/pull/30410), which allows everyone to see the correct version of the documentation for their connector version without needing to upgrade their Airbyte platform version. + +We're also always learning and listening to user feedback. We no longer [deduplicate raw tables](https://github.com/airbytehq/airbyte/pull/31520) to further speed up syncs with Destinations V2. We also released a new voting feature on our [docs](https://docs.airbyte.com) that asks how helpful our docs are for you. + +This month, we also held our annual Hacktoberfest, from which we have already merged 51 PRs and welcomed 3 new contributors to our community! + +## Platform Releases +- **Enhanced payment options:** Cloud customers can now sign up for [auto-recharging of their balance](https://docs.airbyte.com/cloud/managing-airbyte-cloud/manage-credits#automatic-reload-of-credits-beta) and can purchase up to 6,000 credits within our application. +- **Free historical syncs:** Cloud customers can have more predictability around pricing with free historical syncs for any new connector. Reach out to our Sales team if interested. +- **Email Notification Recipient** Cloud customers can now designate the recipient of important email notifications about their connectors and syncs. + +## Connector Improvements + +Many of our enhancements came from our Community this month as a part of our Hacktoberfest. Notably, we enhanced the connector experience by: + +- [**GitLab**](https://github.com/airbytehq/airbyte/pull/31492) now gracefully handles the expiration of access tokens +- [**Orbit**](https://github.com/airbytehq/airbyte/pull/30138) and [**Qualaroo**](https://github.com/airbytehq/airbyte/pull/30138) were migrated to low-code, which improves the maintainability of the connector (thanks to community member Aviraj Gour!) +- [**Pipdrive**](https://github.com/airbytehq/airbyte/pull/30138): optimized custom fields, which are commonly found in this connector. + +Additionally, we added new streams for several connectors to ensure users have access to all their data, including: +- [**Chargify**](https://github.com/airbytehq/airbyte/pull/31116): Coupons, Transactions, and Invoices +- [**Mailchimp**](https://github.com/airbytehq/airbyte/pull/31922): Segment and Unsubscribes +- [**Pipedrive**](https://github.com/airbytehq/airbyte/pull/31885): Mails (thanks to community member Tope Folorunso!) and Goals +- [**Asana**](https://github.com/airbytehq/airbyte/pull/31634): Events, Attachments, OrganizationExports (thanks to Tope again!) +- [**Tiktok Ads**](https://github.com/airbytehq/airbyte/pull/31610): Audiences, Images, Music, Portfolios, Videos, Ad Audiences Report by Province +- [**Square**](https://github.com/airbytehq/airbyte/pull/30138): Bank Accounts (thanks community member Aviraj Gour) and Cash Drawers +- [**Notion**](https://github.com/airbytehq/airbyte/pull/30324): Blocks, Pages and Comments \ No newline at end of file diff --git a/docs/release_notes/september_2023.md b/docs/release_notes/september_2023.md new file mode 100644 index 000000000000..4c21c4384d34 --- /dev/null +++ b/docs/release_notes/september_2023.md @@ -0,0 +1,28 @@ +# September 2023 +## airbyte v0.50.24 to v0.50.31 + +This page includes new features and improvements to the Airbyte Cloud and Airbyte Open Source platforms. + +## ✨ Highlights + +This month, we brought 4 new destinations to Airbyte focused on AI. This enables users to seamlessly flow data from 100s of our sources into large language models. Those four destinations are: +- [Qdrant](https://github.com/airbytehq/airbyte/pull/30332) +- [Choroma](https://github.com/airbytehq/airbyte/pull/30023) +- [Milvus](https://github.com/airbytehq/airbyte/pull/30023) +- [Pinecone](https://github.com/airbytehq/airbyte/pull/29539) + +Another notable release was our new File CDK module within the [CDK](https://airbyte.com/connector-development-kit) for configuring and improving file-based connectors. This enables easier creation, maintenance, and improvement of file-base source connectors. + +## Connector Improvements + +We've also worked on several connector enhancements and additions. To name a few: + +- [**Shopify**](https://github.com/airbytehq/airbyte/pull/29539) now fetches destroyed records, so that your source data is always reflected accurately +- [**Google Analytics**](https://github.com/airbytehq/airbyte/pull/30152) now allows for multiple Property IDs to be input, so that you can sync data from all your properties. +- [**Google Ads**](https://github.com/airbytehq/airbyte/pull/28970) now uses the change status to implement an improved incremental sync for Ad Groups and Campaign Criterion streams + +Additionally, we added new streams for several connectors to bring in newly available API endpoints and adapt to user feedback, including: +- [**Github**](https://github.com/airbytehq/airbyte/pull/30823): Issue Timeline and Contributor Activity +- [**JIRA**](https://github.com/airbytehq/airbyte/pull/30755): Issue Types, Project Roles, and Issue Transitions +- [**Outreach**](https://github.com/airbytehq/airbyte/pull/30639): Call Purposes and Call Dispositions +- [**Zendesk**](https://github.com/airbytehq/airbyte/pull/30138): Articles \ No newline at end of file diff --git a/docs/release_notes/upgrading_to_destinations_v2.md b/docs/release_notes/upgrading_to_destinations_v2.md index 30461beae755..eee8ad098b47 100644 --- a/docs/release_notes/upgrading_to_destinations_v2.md +++ b/docs/release_notes/upgrading_to_destinations_v2.md @@ -1,27 +1,31 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import {SnowflakeMigrationGenerator, BigQueryMigrationGenerator} from './destinations_v2.js' +import {SnowflakeMigrationGenerator, BigQueryMigrationGenerator, RedshiftMigrationGenerator} from './destinations_v2.js' # Upgrading to Destinations V2 ## What is Destinations V2? -Starting today, Airbyte Destinations V2 provides you with: +Airbyte Destinations V2 provides you with: - One-to-one table mapping: Data in one stream will always be mapped to one table in your data warehouse. No more sub-tables. - Improved error handling with `_airbyte_meta`: Airbyte will now populate typing errors in the `_airbyte_meta` column instead of failing your sync. You can query these results to audit misformatted or unexpected data. - Internal Airbyte tables in the `airbyte_internal` schema: Airbyte will now generate all raw tables in the `airbyte_internal` schema. We no longer clutter your destination schema with raw data tables. - Incremental delivery for large syncs: Data will be incrementally delivered to your final tables. No more waiting hours to see the first rows in your destination table. -To see more details and examples on the contents of the Destinations V2 release, see this [guide](understanding-airbyte/typing-deduping.md). The remainder of this page will walk you through upgrading connectors from legacy normalization to Destinations V2. +To see more details and examples on the contents of the Destinations V2 release, see this [guide](../using-airbyte/core-concepts/typing-deduping.md). The remainder of this page will walk you through upgrading connectors from legacy normalization to Destinations V2. + +Destinations V2 were in preview for Snowflake and BigQuery during August 2023, and launched on August 29th, 2023. Other destinations will be transitioned to Destinations V2 in early 2024. ## Deprecating Legacy Normalization -The upgrade to Destinations V2 is handled by moving your connections to use [updated versions of Airbyte destinations](#destinations-v2-compatible-versions). Existing normalization options, both `Raw data (JSON)` and `Normalized tabular data` will be unsupported starting **Nov 1, 2023**. +The upgrade to Destinations V2 is handled by moving your connections to use [updated versions of Airbyte destinations](#destinations-v2-compatible-versions). Existing normalization options, both `Raw data (JSON)` and `Normalized tabular data`, will be unsupported starting the upgrade deadline. ![Legacy Normalization](./assets/airbyte_legacy_normalization.png) -As a Cloud user, existing connections using legacy normalization will be paused on **Oct 1, 2023**. As an Open Source user, you may choose to upgrade at your convenience. However, destination connector versions prior to Destinations V2 will no longer be supported as of **Nov 1, 2023**. +As a Cloud user, existing connections using legacy normalization will be paused on the upgrade deadline. As an Open Source user, you may choose to upgrade at your convenience. However, destination connector versions prior to Destinations V2 will no longer be supported starting the upgrade deadline. + +Note that Destinations V2 also removes the option to _only_ replicate raw data. The vast majority of Airbyte users prefer typed final tables, and our future feature development will rely on this implementation. Learn more [below](#upgrading-as-a-user-of-raw-tables). ### Breakdown of Breaking Changes @@ -33,13 +37,19 @@ The following table details the delivered data modified by Destinations V2: | Normalized tabular data | API Source | Unnested tables, `_airbyte` metadata columns, SCD tables | | Normalized tabular data | Tabular Source (database, file, etc.) | `_airbyte` metadata columns, SCD tables | -![Airbyte Destinations V2 Column Changes](./assets/destinations-v2-column-changes.png) +![Airbyte Destinations V2 Column Changes](./assets/updated_table_columns.png) Whenever possible, we've taken this opportunity to use the best data type for storing JSON for your querying convenience. For example, `destination-bigquery` now loads `JSON` blobs as type `JSON` in BigQuery (introduced last [year](https://cloud.google.com/blog/products/data-analytics/bigquery-now-natively-supports-semi-structured-data)), instead of type `string`. ## Quick Start to Upgrading -The quickest path to upgrading is to click upgrade on any out-of-date connection in the UI: +**The quickest path to upgrading is to click upgrade on any out-of-date connection in the UI**. The advanced options later in this document will allow you to test out the upgrade in more detail if you choose. + +:::caution + +**[Airbyte Open Source Only]** You should upgrade to 0.50.24+ of the Airbyte Platform _before_ updating to Destinations V2. Failure to do so may cause upgraded connections to fail. + +::: ![Upgrade Path](./assets/airbyte_destinations_v2_upgrade_prompt.png) @@ -51,17 +61,11 @@ After upgrading the out-of-date destination to a [Destinations V2 compatible ver 4. The new raw tables will be typed and de-duplicated according to the Destinations V2 format. 5. Once typing and de-duplication has completed successfully, your previous final table will be replaced with the updated data. -:::caution - -Due to the amount of operations to be completed, this first sync after upgrading to Destination V2 **will be longer than normal**. Once your first sync has completed successfully, you may need to make changes to downstream models (dbt, sql, etc.) transforming data. See this [walkthrough of top changes to expect for more details](#updating-downstream-transformations). - -::: +Due to the amount of operations to be completed, the first sync after upgrading to Destination V2 **will be longer than normal**. Once your first sync has completed successfully, you may need to make changes to downstream models (dbt, sql, etc.) transforming data. See this [walkthrough of top changes to expect for more details](#updating-downstream-transformations). Pre-existing raw tables, SCD tables and "unnested" tables will always be left untouched. You can delete these at your convenience, but these tables will no longer be kept up-to-date by Airbyte syncs. Each destination version is managed separately, so if you have multiple destinations, they all need to be upgraded one by one. - - Versions are tied to the destination. When you update the destination, **all connections tied to that destination will be sending data in the Destinations V2 format**. For upgrade paths that will minimize disruption to existing dashboards, see: - [Upgrading Connections One by One with Dual-Writing](#upgrading-connections-one-by-one-with-dual-writing) @@ -102,6 +106,9 @@ These steps allow you to dual-write for connections incrementally syncing data w + + + 2. Navigate to the existing connection you are duplicating, and navigate to the `Settings` tab. Open the `Advanced` settings to see the connection state (which manages incremental syncs). Copy the state to your clipboard. @@ -126,11 +133,13 @@ When you are done testing, you can disable or delete this testing connection, an If you have written downstream transformations directly from the output of raw tables, or use the "Raw JSON" normalization setting, you should know that: - Multiple column names are being updated (from `airbyte_ab_id` to `airbyte_raw_id`, and `airbyte_emitted_at` to `airbyte_extracted_at`). -- The location of raw tables will from now on default to an `airbyte` schema in your destination. -- When you upgrade to a [Destinations V2 compatible version](#destinations-v2-effective-versions) of your destination, we will never alter your existing raw data. Although existing downstream dashboards will go stale, they will never be broken. +- The location of raw tables will from now on default to an `airbyte_internal` schema in your destination. +- When you upgrade to a [Destinations V2 compatible version](#destinations-v2-effective-versions) of your destination, we will leave a copy of your existing raw tables as they are, and new syncs will work from a new copy we make in the new `airbyte_internal` schema. Although existing downstream dashboards will go stale, they will not be broken. - You can dual write by following the [steps above](#upgrading-connections-one-by-one-with-dual-writing) and copying your raw data to the schema of your newly created connection. -We may make further changes to raw tables in the future, as these tables are intended to be a staging ground for Airbyte to optimize the performance of your syncs. We cannot guarantee the same level of stability as for final tables in your destination schema. +We may make further changes to raw tables in the future, as these tables are intended to be a staging ground for Airbyte to optimize the performance of your syncs. We cannot guarantee the same level of stability as for final tables in your destination schema, nor will features like error handling be implemented in the raw tables. + +As a user previously not running Normalization, Upgrading to Destinations V2 will increase the compute costs in your destination data warehouse. This is because Destinations V2 will now be performing the operations to generate a final typed table. Some destinations may provide an option to disable this - check your connectors's settings. ### Upgrade Paths for Connections using CDC @@ -140,38 +149,45 @@ For each [CDC-supported](https://docs.airbyte.com/understanding-airbyte/cdc) sou | ---------- | ------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | Postgres | [Upgrade connection in place](#quick-start-to-upgrading) | You can optionally dual write, but this requires resyncing historical data from the source. You must create a new Postgres source with a different replication slot than your existing source to preserve the integrity of your existing connection. | | MySQL | [All above upgrade paths supported](#advanced-upgrade-paths) | You can upgrade the connection in place, or dual write. When dual writing, Airbyte can leverage the state of an existing, active connection to ensure historical data is not re-replicated from MySQL. | -| SQL Server | [Upgrade connection in place](#quick-start-to-upgrading) | You can optionally dual write, but this requires resyncing historical data from the SQL Server source. | ## Destinations V2 Compatible Versions For each destination connector, Destinations V2 is effective as of the following versions: -| Destination Connector | Safe Rollback Version | Destinations V2 Compatible | -| --------------------- | --------------------- | -------------------------- | -| BigQuery | 1.4.4 | 2.0.0+ | -| Snowflake | 0.4.1 | 2.0.0+ | -| Redshift | 0.4.8 | 2.0.0+ | -| MSSQL | 0.1.24 | 2.0.0+ | -| MySQL | 0.1.20 | 2.0.0+ | -| Oracle | 0.1.19 | 2.0.0+ | -| TiDB | 0.1.3 | 2.0.0+ | -| DuckDB | 0.1.0 | 2.0.0+ | -| Clickhouse | 0.2.3 | 2.0.0+ | +| Destination Connector | Safe Rollback Version | Destinations V2 Compatible | Upgrade Deadline | +| --------------------- | --------------------- | -------------------------- | ------------------------ | +| BigQuery | 1.10.2 | 2.0.6+ | November 7, 2023 | +| Snowflake | 2.1.7 | 3.1.0+ | November 7, 2023 | +| Redshift | 0.6.11 | [coming soon] 2.0.0+ | [coming soon] early 2024 | +| Postgres | 0.4.0 | [coming soon] 2.0.0+ | [coming soon] early 2024 | +| MySQL | 0.2.0 | [coming soon] 2.0.0+ | [coming soon] early 2024 | -## Destinations V2 Implementation Differences +Note that legacy normalization will be deprecated for ClickHouse, DuckDB, MSSQL, TiDB, and Oracle DB in early 2024. If you wish to add Destinations V2 capability to these destinations, please reference our implementation guide (coming soon). -In addition to the changes which apply for all destinations described above, there are some per-destination fixes and updates included in Destinations V2: +### [Open Source Only] Rolling Back to Legacy Normalization -### BigQuery +If you upgrade to Destinations V2 and start encountering issues, as an Open Source user you can optionally roll back. If you are running an outdated Airbyte Platform version (prior to `v0.50.24`), this may occur more frequently by accidentally upgrading to Destinations V2. However: + +- Rolling back will require resetting each of your upgraded connections. +- If you are hoping to receive support from the Airbyte team, you will need to re-upgrade to Destinations V2 by the upgrade deadline. -1. [Object and array properties](https://docs.airbyte.com/understanding-airbyte/supported-data-types/#the-types) are properly stored as JSON columns. Previously, we had used TEXT, which made querying sub-properties more difficult. - * In certain cases, numbers within sub-properties with long decimal values will need to be converted to float representations due to a *quirk* of Bigquery. Learn more [here](https://github.com/airbytehq/airbyte/issues/29594). +To roll back, follow these steps: -### Snowflake +1. In the Airbyte UI, go to the 'Settings page, then to 'Destinations'. +2. Manually type in the previous destination version you were running, or one of the versions listed in the table above. +3. Enter this older version to roll back to the previous connector version. +4. Reset all connections which synced at least once to a previously upgraded destination. To be safe, you may reset all connections sending data to a previously upgraded destination. + +If you are an Airbyte Cloud customer, and encounter errors while upgrading from a V1 to a V2 destination, please reach out to support. We do not always recommend doing a full reset, depending on the type of error. + +## Destinations V2 Implementation Differences + +In addition to the changes which apply for all destinations described above, there are some per-destination fixes and updates included in Destinations V2: + +### BigQuery -1. `destination-snowflake` is now case-sensitive, and was not previously. This means that if you have a source stream "users", `destination-snowflake` would have previously created a "USERS" table in your data warehouse. We now correctly create a "users" table. - * Note that to properly query case-sensitive tables and columns in Snowflake, you will need to quote your table and column names, e.g. `select "first_name" from "users";` - * If you are migrating from Destinations v1 to Destinations V2, we will leave your old "USERS" table, and create a new "users" table - please note the case sensitivity. +1. [Object and array properties](https://docs.airbyte.com/understanding-airbyte/supported-data-types/#the-types) are properly stored as JSON columns. Previously, we had used TEXT, which made querying sub-properties more difficult. + - In certain cases, numbers within sub-properties with long decimal values will need to be converted to float representations due to a _quirk_ of Bigquery. Learn more [here](https://github.com/airbytehq/airbyte/issues/29594). ## Updating Downstream Transformations diff --git a/docs/snowflake-native-apps/event-sharing.md b/docs/snowflake-native-apps/event-sharing.md new file mode 100644 index 000000000000..afade3e69b18 --- /dev/null +++ b/docs/snowflake-native-apps/event-sharing.md @@ -0,0 +1,20 @@ +## Event Sharing + +Sharing the events is important to ensure that in case of issue, our team can investigate easily. The app will never share private or sensitive information, only errors and diagnostic metrics that allow our team to understand the problem. + +In order to share the events, you can refer to the [Snowflake documentation](https://other-docs.snowflake.com/en/native-apps/consumer-enable-logging#label-nativeapps-consumer-logging-enabling). As of 2023-10-02, you have to: + +1. Create the event table. This table is global to an account so all applications share the same event table. We recommend using: +``` +CREATE DATABASE event_database; +CREATE SCHEMA event_schema; +CREATE EVENT TABLE event_database.event_schema.event_table; +``` +2. Make the table active for your account, +``` +ALTER ACCOUNT SET EVENT_TABLE=event_database.event_schema.event_table; +``` +3. Allow the application to share the logs. +``` +ALTER APPLICATION SET SHARE_EVENTS_WITH_PROVIDER = TRUE`; +``` diff --git a/docs/snowflake-native-apps/facebook-marketing-app-install.png b/docs/snowflake-native-apps/facebook-marketing-app-install.png new file mode 100644 index 000000000000..6081fdc70157 Binary files /dev/null and b/docs/snowflake-native-apps/facebook-marketing-app-install.png differ diff --git a/docs/snowflake-native-apps/facebook-marketing-privileges.png b/docs/snowflake-native-apps/facebook-marketing-privileges.png new file mode 100644 index 000000000000..aeb0b11f8a91 Binary files /dev/null and b/docs/snowflake-native-apps/facebook-marketing-privileges.png differ diff --git a/docs/snowflake-native-apps/facebook-marketing-security-button.png b/docs/snowflake-native-apps/facebook-marketing-security-button.png new file mode 100644 index 000000000000..df17799c9707 Binary files /dev/null and b/docs/snowflake-native-apps/facebook-marketing-security-button.png differ diff --git a/docs/snowflake-native-apps/facebook-marketing.md b/docs/snowflake-native-apps/facebook-marketing.md new file mode 100644 index 000000000000..1b4a458e2e20 --- /dev/null +++ b/docs/snowflake-native-apps/facebook-marketing.md @@ -0,0 +1,226 @@ +# Facebook Marketing Connector + +The Facebook Marketing Connector by Airbyte is a Snowflake Native Application that allows you to extract data from your Facebook Marketing account and load records into a Snowflake database of your choice. + +:::info +The Snowflake Native Apps platform is new and rapidly evolving. The Facebook Marketing Connector by Airbyte is in _public preview_ and is subject to further development that may affect setup and configuration of the application. Please note that, at this time, only a [full table refresh](/using-airbyte/core-concepts/sync-modes/full-refresh-overwrite.md) without deduplication is supported. +::: + +# Getting started + +## Prerequisites +A Facebook Marketing account with permission to access data from accounts you want to sync. + +## Installing the App + +:::warning +Do not refresh the Apps page while the application is being installed. This may cause installation to fail. +::: + +1. Log into your Snowflake account. +2. On the left sidebar, click `Marketplace`. +3. Search for `Facebook Marketing Connector` by Airbyte or navigate to https://app.snowflake.com/marketplace/listing/GZTYZ9BCRTG/airbyte-facebook-marketing-connector +4. Click `Get`. This will open a pop-up where you can specify install options. Expand `Options`. + 1. You can rename the application or leave the default. This is how you will reference the application from a worksheet. + 2. Specify the warehouse that the application will be installed to. +5. Wait for the application to install. Once complete, the pop-up window should automatically close. +6. On the left sidebar, click `Apps`. + +![](./facebook-marketing-app-install.png) + +7. Once your installation is complete, under the `Installed Apps` section, you should see the `Facebook Marketing Connector` by Airbyte. + +## Facebook Marketing Account +In order for the Facebook Marketing Connector by Airbyte to query Facebook's APIs, you will need an account with the right permissions. Please follow the [Facebook Marketing authentication guide](https://docs.airbyte.com/integrations/sources/facebook-marketing#for-airbyte-open-source-generate-an-access-token-and-request-a-rate-limit-increase) for further information. + +## Snowflake Native App Authorizations + +:::note +By default the app will be installed using the name `FACEBOOK_MARKETING_CONNECTOR`, but if you renamed the app during installation, you will have to use that name as a reference. +::: + +### Adding Credentials and Configuring External API Access +Before using the application, you will need to perform a few prerequisite steps to prepare the application to make outbound API requests and use your authentication credentials. From a SQL worksheet, you will need to run a series of commands. + +1. Create the database where the app will access the authorization. +``` +CREATE DATABASE AIRBYTE_FACEBOOK_MARKETING_DB; +USE AIRBYTE_FACEBOOK_MARKETING_DB; +``` + +2. You will need to allow outgoing network traffic based on the domain of the source. In the case of Facebook Marketing, simply run: +``` +CREATE OR REPLACE NETWORK RULE FACEBOOK_MARKETING_APIS_NETWORK_RULE + MODE = EGRESS + TYPE = HOST_PORT + VALUE_LIST = ('graph.facebook.com'); +``` + +:::note +As of 2023-09-13, the [Snowflake documentation](https://docs.snowflake.com/en/sql-reference/sql/create-external-access-integration) mentions that direct external access is a preview feature and that it is `available to all accounts on AWS` which might restrict the number of users able to use the connector. +::: + +3. Once you have external access configured, you need define your authorization/authentication. Provide the credentials to the app as such: +``` +CREATE OR REPLACE SECRET AIRBYTE_APP_SECRET + TYPE = GENERIC_STRING + SECRET_STRING = '{ + "access_token": "" + }'; +``` +... where `client_id`, `client_secret` and `refresh_token` are strings. For more information, see the [Facebook Marketing authentication guide](https://docs.airbyte.com/integrations/sources/facebook-marketing#for-airbyte-open-source-generate-an-access-token-and-request-a-rate-limit-increase). + +4. Once the network rule and the secret are defined in Snowflake, you need to make them available to the app by using an external access integration. +``` +CREATE OR REPLACE EXTERNAL ACCESS INTEGRATION AIRBYTE_APP_INTEGRATION + ALLOWED_NETWORK_RULES = (facebook_marketing_apis_network_rule) + ALLOWED_AUTHENTICATION_SECRETS = (AIRBYTE_APP_SECRET) + ENABLED = true; +``` + +5. Grant permission for the app to access the integration. +``` +GRANT USAGE ON INTEGRATION AIRBYTE_APP_INTEGRATION TO APPLICATION FACEBOOK_MARKETING_CONNECTOR; +``` + +6. Grant permissions for the app to access the database that houses the secret and read the secret. +``` +GRANT USAGE ON DATABASE AIRBYTE_FACEBOOK_MARKETING_DB TO APPLICATION FACEBOOK_MARKETING_CONNECTOR; +GRANT USAGE ON SCHEMA PUBLIC TO APPLICATION FACEBOOK_MARKETING_CONNECTOR; +GRANT READ ON SECRET AIRBYTE_APP_SECRET TO APPLICATION FACEBOOK_MARKETING_CONNECTOR; +``` + +### Granting Account Privileges +Once you have completed the prerequisite SQL setup steps, you will need to grant privileges to allow the application to create databases, create warehouses, and execute tasks. +All of these privileges are required for the application to extract data into Snowflake database successfully. + +1. Start by going in the `Apps` section and selecting `Facebook Marketing Connector`. You will have to accept the Anaconda terms in order to use Streamlit. +2. After the application has loaded click the shield icon in the top right corner to modify `Security` settings. + +![](./facebook-marketing-security-button.png) + +3. Under `Account level privileges`, click `Review` which will then present will open a pop-up of security privileges the application needs granted. +4. Enable each of the privileges and click `Update Privileges`. + +![](./facebook-marketing-privileges.png) + +5. Reload the application to ensure that the application privileges have been updated. + +You are now ready to begin syncing your data. + +## Configuring a Connection +Navigate back to the application by clicking `STREAMLIT` in the top left corner. Select `New Connection` and fill the following fields: + +--- + +`account_id` + +The Facebook Ad account ID to use when pulling data from the Facebook Marketing API. The Ad account ID number is in the account dropdown menu or in your browser's address bar of your [Meta Ads Manager](https://adsmanager.facebook.com/adsmanager/). + +--- + +`start_date` + +UTC date in the format YYYY-MM-DDTHH:mm:ssZ (e.g. 2021-09-29T12:13:14Z). Any data before this date will not be replicated. + +--- + +`end_date` + +UTC date in the format YYYY-MM-DDTHH:mm:ssZ (e.g. 2021-09-29T12:13:14Z). Any data after this date will not be replicated. + +--- + +`include_deleted` + +The Facebook Marketing API does not have a concept of deleting records, and it maintains a record of Campaigns, Ads, and Ad Sets. Enabling this setting allows you to extract data that includes these objects that were archived or deleted from the Facebook platform. + +--- + +`fetch_thumbnail_images` + +When extracting Ad Creatives, retrieve the thumbnail_url and store it as thumbnail_data_url in each record. + +--- + +`custom_insights` + +Custom insights allow you to define ad statistic entries representing the performance of your campaigns against specific metrics. For more information about how to configure custom insights, please refer to the [Facebook Marketing documentation](https://docs.airbyte.com/integrations/sources/facebook-marketing#set-up-facebook-marketing-as-a-source-in-airbyte). + +--- + +`page_size` + +The number of records per page for paginated responses. The default is 100, but most users should not need to set this field except for unique use cases that require tuning the settings. + +--- + +`insights_lookback_window` + +The window in days to revisit data during syncing to capture updated conversion data from the API. Facebook allows for attribution windows of up to 28 days, during which time a conversion can be attributed to an ad. If you have set a custom attribution window in your Facebook account, please set the same value here. + +--- + +`Output Database` + +The database where the records will be saved. Snowflake's database [naming convention](https://docs.snowflake.com/en/sql-reference/identifiers-syntax) applies here. + +--- + +`Output Schema` + +The table where the schema will be saved. Snowflake's table [naming convention](https://docs.snowflake.com/en/sql-reference/identifiers-syntax) applies here. + +--- + +`Connection Name` + +How the connection will be referred in the Streamlit app. + +--- + +`Replication Frequency` + +The sync schedule that determines how often your data will be synced to the target database. + +--- + +## Enabling Logging and Event Sharing for an Application +Sharing the logging and telemetry data of your installed application helps us improve the application and can allow us to better triage problems that your run into. To configure your application for logging and telemetry data please refer to the documentation for [Enabling Logging and Event Sharing](event-sharing.md). + +## Syncing Your Facebook Marketing Data +Once a connection is configured, go in `Connections List` to view all of your connections. From here for each connection you can +view the configuration settings, start a sync, and view the prior sync history. + +### Scheduled Syncs +While creating a connection, you can specify a "Replication Frequency" which will dictate how often your data will be extracted from +Facebook Marketing and loaded into your Snowflake database. This process is started automatically according to your schedule and does not +require that you manually trigger syncs. For example, if you create a connection at 10:15 AM and set your replication frequency to +hourly, then a sync will be started immediately. The next sync will start at 11:15 AM onwards. Only one sync is active at the same +time. In the event that your sync runs longer than one hour, a new sync will start at the next available time. + +### Manual Syncs +In addition to scheduled syncs, you can also configure a connection to only sync data on-demand by setting "Replication Frequency" to +`MANUAL`. After creating a connection, from the `Connections List` page, you can use the "Sync Now" button to trigger a sync of +your API data to your Snowflake database. You can also use this button to manually trigger connections that sync according to a +schedule. If there is already a sync in progress, this button will be disabled. + +### Sync History +From the `Connections List` page, you can view information about past syncs for each connection to determine when your +data is done syncing and whether the operation was successful. Once the sync is completed successfully, you should be +able to validate that the records have been stored in `.`. + +## Supported Streams +As of now, all supported streams perform a full refresh. Incremental syncs are not yet supported. Here are the list of supported streams: +* Activities +* Ad Account +* Ad Creatives +* Ad Insights +* Ad Sets +* Ads +* Campaigns +* Custom Audiences +* Custom Conversions + +# Contact Us +snowflake-native-apps@airbyte.io diff --git a/docs/snowflake-native-apps/linkedin-ads-app-install.png b/docs/snowflake-native-apps/linkedin-ads-app-install.png new file mode 100644 index 000000000000..349a6f8b25d7 Binary files /dev/null and b/docs/snowflake-native-apps/linkedin-ads-app-install.png differ diff --git a/docs/snowflake-native-apps/linkedin-ads-privileges.png b/docs/snowflake-native-apps/linkedin-ads-privileges.png new file mode 100644 index 000000000000..d87dfe2b86e4 Binary files /dev/null and b/docs/snowflake-native-apps/linkedin-ads-privileges.png differ diff --git a/docs/snowflake-native-apps/linkedin-ads-security-button.png b/docs/snowflake-native-apps/linkedin-ads-security-button.png new file mode 100644 index 000000000000..4449fc589eeb Binary files /dev/null and b/docs/snowflake-native-apps/linkedin-ads-security-button.png differ diff --git a/docs/snowflake-native-apps/linkedin-ads-ui.gif b/docs/snowflake-native-apps/linkedin-ads-ui.gif new file mode 100644 index 000000000000..db237252bed6 Binary files /dev/null and b/docs/snowflake-native-apps/linkedin-ads-ui.gif differ diff --git a/docs/snowflake-native-apps/linkedin-ads.md b/docs/snowflake-native-apps/linkedin-ads.md new file mode 100644 index 000000000000..bd34a7ffa565 --- /dev/null +++ b/docs/snowflake-native-apps/linkedin-ads.md @@ -0,0 +1,191 @@ +# LinkedIn Ads Connector + +The LinkedIn Ads Connector by Airbyte is a Snowflake Native Application that allows you to extract data from your LinkedIn Ads account and load records into a Snowflake database of your choice. + +:::info +The Snowflake Native Apps platform is new and rapidly evolving. The LinkedIn Ads Connector by Airbyte is in _public preview_ and is subject to further development that may affect setup and configuration of the application. Please note that, at this time, only a [full table refresh](/using-airbyte/core-concepts/sync-modes/full-refresh-overwrite.md) without deduplication is supported. +::: + +# Getting started + +## Prerequisites +A LinkedIn Ads account with permission to access data from accounts you want to sync. + +## Installing the App + +:::warning +Do not refresh the Apps page while the application is being installed. This may cause installation to fail. +::: + +1. Log into your Snowflake account. +2. On the left sidebar, click `Marketplace`. +3. Search for `LinkedIn Ads Connector` by Airbyte or navigate to https://app.snowflake.com/marketplace/listing/GZTYZ9BCRTW/airbyte-linkedin-ads-connector +4. Click `Get`. This will open a pop-up where you can specify install options. Expand `Options`. + 1. You can rename the application or leave the default. This is how you will reference the application from a worksheet. + 2. Specify the warehouse that the application will be installed to. +5. Wait for the application to install. Once complete, the pop-up window should automatically close. +6. On the left sidebar, click `Apps`. + +![](./linkedin-ads-app-install.png) + +7. Once your installation is complete, under the `Installed Apps` section, you should see the `LinkedIn Ads Connector` by Airbyte. + +## LinkedIn Ads Account +In order for the LinkedIn Ads Connector by Airbyte to query LinkedIn, you will need an account with the right permissions. Please follow the [LinkedIn Ads authentication guide](https://docs.airbyte.com/integrations/sources/linkedin-ads/#set-up-linkedin-ads-authentication-airbyte-open-source) for further information. + +## Snowflake Native App Authorizations + +:::note +By default the app will be installed using the name `LINKEDIN_ADS_CONNECTOR`, but if you renamed the app during installation, you will have to use that name as a reference. +::: + +### Adding Credentials and Configuring External API Access +Before using the application, you will need to perform a few prerequisite steps to prepare the application to make outbound API requests and use your authentication credentials. From a SQL worksheet, you will need to run a series of commands. + +1. Create the database where the app will access the authorization. +``` +CREATE DATABASE AIRBYTE_LINKEDIN_ADS_DB; +USE AIRBYTE_LINKEDIN_ADS_DB; +``` + +2. You will need to allow outgoing network traffic based on the domain of the source. In the case of LinkedIn Ads, simply run: +``` +CREATE OR REPLACE NETWORK RULE LINKEDIN_APIS_NETWORK_RULE + MODE = EGRESS + TYPE = HOST_PORT + VALUE_LIST = ('api.linkedin.com', 'www.linkedin.com', 'linkedin.com'); +``` + +:::note +As of 2023-09-13, the [Snowflake documentation](https://docs.snowflake.com/en/sql-reference/sql/create-external-access-integration) mentions that direct external access is a preview feature and that it is `available to all accounts on AWS` which might restrict the number of users able to use the connector. +::: + +3. Once you have external access configured, you need define your authorization/authentication. Provide the credentials to the app as such: +``` +CREATE OR REPLACE SECRET AIRBYTE_APP_SECRET + TYPE = GENERIC_STRING + SECRET_STRING = '{ + "auth_method": "oAuth2.0", + "client_id": , + "client_secret": , + "refresh_token": + }'; +``` +... where `client_id`, `client_secret` and `refresh_token` are strings. For more information, see the [LinkedIn Ads authentication guide](https://docs.airbyte.com/integrations/sources/linkedin-ads/#set-up-linkedin-ads-authentication-airbyte-open-source). + +4. Once the network rule and the secret are defined in Snowflake, you need to make them available to the app by using an external access integration. +``` +CREATE OR REPLACE EXTERNAL ACCESS INTEGRATION AIRBYTE_APP_INTEGRATION + ALLOWED_NETWORK_RULES = (LINKEDIN_APIS_NETWORK_RULE) + ALLOWED_AUTHENTICATION_SECRETS = (AIRBYTE_APP_SECRET) + ENABLED = true; +``` + +5. Grant permission for the app to access the integration. +``` +GRANT USAGE ON INTEGRATION AIRBYTE_APP_INTEGRATION TO APPLICATION LINKEDIN_ADS_CONNECTOR; +``` + +6. Grant permissions for the app to access the database that houses the secret and read the secret. +``` +GRANT USAGE ON DATABASE AIRBYTE_LINKEDIN_ADS_DB TO APPLICATION LINKEDIN_ADS_CONNECTOR; +GRANT USAGE ON SCHEMA PUBLIC TO APPLICATION LINKEDIN_ADS_CONNECTOR; +GRANT READ ON SECRET AIRBYTE_APP_SECRET TO APPLICATION LINKEDIN_ADS_CONNECTOR; +``` + +### Granting Account Privileges +Once you have completed the prerequisite SQL setup steps, you will need to grant privileges to allow the application to create databases, create warehouses, and execute tasks. +All of these privileges are required for the application to extract data into Snowflake database successfully. + +1. Start by going in the `Apps` section and selecting `LinkedIn Ads Connector`. You will have to accept the Anaconda terms in order to use Streamlit. +2. After the application has loaded click the shield icon in the top right corner to modify `Security` settings. + +![](./linkedin-ads-security-button.png) + +3. Under `Account level privileges`, click `Review` which will then present will open a pop-up of security privileges the application needs granted. +4. Enable each of the privileges and click `Update Privileges`. + +![](./linkedin-ads-privileges.png) + +5. Reload the application to ensure that the application privileges have been updated. + +You are now ready to begin syncing your data. + +## Configuring a Connection +Navigate back to the application by clicking `STREAMLIT` in the top left corner. Select `New Connection` and fill the following fields: + +--- + +`start_date` + +UTC date in the format YYYY-MM-DD (e.g. 2020-09-17). Any data before this date will not be replicated. + +--- + +`account_ids` + +Leave empty, if you want to pull the data from all associated accounts. To specify individual account IDs to pull data from, separate them by a space. See the [LinkedIn Ads docs](https://www.linkedin.com/help/linkedin/answer/a424270/find-linkedin-ads-account-details) for more info. + +--- + +`Output Database` + +The database where the records will be saved. Snowflake's database [naming convention](https://docs.snowflake.com/en/sql-reference/identifiers-syntax) applies here. + +--- + +`Output Schema` + +The table where the schema will be saved. Snowflake's table [naming convention](https://docs.snowflake.com/en/sql-reference/identifiers-syntax) applies here. + +--- + +`Connection Name` + +How the connection will be referred in the Streamlit app. + +--- + +`Replication Frequency` + +The sync schedule that determines how often your data will be synced to the target database. + +--- + +## Enabling Logging and Event Sharing for an Application +Sharing the logging and telemetry data of your installed application helps us improve the application and can allow us to better triage problems that your run into. To configure your application for logging and telemetry data please refer to the documentation for [Enabling Logging and Event Sharing](event-sharing.md). + +## Syncing Your LinkedIn Ads Data +Once a connection is configured, go in `Connections List` to view all of your connections. From here for each connection you can +view the configuration settings, start a sync, and view the prior sync history. + +### Scheduled Syncs +While creating a connection, you can specify a "Replication Frequency" which will dictate how often your data will be extracted from +LinkedIn Ads and loaded into your Snowflake database. This process is started automatically according to your schedule and does not +require that you manually trigger syncs. For example, if you create a connection at 10:15 AM and set your replication frequency to +hourly, then a sync will be started immediately. The next sync will start at 11:15 AM onwards. Only one sync is active at the same +time. In the event that your sync runs longer than one hour, a new sync will start at the next available time. + +### Manual Syncs +In addition to scheduled syncs, you can also configure a connection to only sync data on-demand by setting "Replication Frequency" to +`MANUAL`. After creating a connection, from the `Connections List` page, you can use the "Sync Now" button to trigger a sync of +your API data to your Snowflake database. You can also use this button to manually trigger connections that sync according to a +schedule. If there is already a sync in progress, this button will be disabled. + +### Sync History +From the `Connections List` page, you can view information about past syncs for each connection to determine when your +data is done syncing and whether the operation was successful. Once the sync is completed successfully, you should be +able to validate that the records have been stored in `.`. + +## Supported Streams +As of now, all supported streams perform a full refresh. Incremental syncs are not yet supported. Here are the list of supported streams: +* Accounts +* Account Users +* Ad Analytics by Campaign +* Ad Analytics by Creative +* Campaigns +* Campaign Groups +* Creatives + +# Contact Us +snowflake-native-apps@airbyte.io diff --git a/docs/terraform-documentation.md b/docs/terraform-documentation.md new file mode 100644 index 000000000000..10c830d9963c --- /dev/null +++ b/docs/terraform-documentation.md @@ -0,0 +1,11 @@ +--- +products: all +--- + +# Terraform Documentation + +Airbyte's Terraform provider enables you to automate & version-control your Airbyte configuration as code. Save time managing Airbyte and collaborate on Airbyte configuration changes with your teammates. Airbyte's Terraform provider is built off our [Airbyte API](https://api.airbyte.com). + +The Terraform provider is available for users on Airbyte Cloud, OSS & Self-Managed Enterprise. + +Check out our guide for [getting started with Airbyte's Terraform provider](https://reference.airbyte.com/reference/using-the-terraform-provider). diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md deleted file mode 100644 index b9a5d7d12472..000000000000 --- a/docs/troubleshooting.md +++ /dev/null @@ -1,59 +0,0 @@ -# Troubleshooting & FAQ - -Welcome to the Airbyte troubleshooting guide! Like any platform, you may experience issues when using Airbyte. This guide is designed to help you diagnose and resolve any problems you may encounter while using Airbyte. By following the troubleshooting steps outlined in this guide, you can quickly and effectively identify the root cause of the issue and take steps to resolve it. We recommend checking this guide whenever you encounter an issue with Airbyte to help ensure a smooth and uninterrupted experience with our platform. Let's dive in! - -Step 1: Check the logs. The logs provide detailed information about what's happening behind the scenes, and they can help pinpoint the root cause of the problem. - -Step 2: Check the documentation. Our documentation covers a wide range of topics, including common issues and their solutions, troubleshooting tips, and best practices. - -Step 3: Reach out to the community. Our community forum is a great place to ask for help, share your experiences, and learn from others who have faced similar issues. - -Step 4: Open a Github ticket. If you're still unable to resolve the issue after reaching out to the community, it's time to open a support ticket. Our support team is here to help you with any issues you're facing with Airbyte. - -Airbyte is an open source project with a vibrant community that fosters collaboration and mutual support. To ensure accessible troubleshooting guidance, Airbyte offers multiple platforms for users to ask and discuss issues, including the Airbyte Github, Airbyte Community Slack (which is over 10,000 users), and the Airbyte Forum. In addition, Airbyte hosts daily office hours that include topic demonstrations and dedicated space for issue discussion in Zoom meetings. In addition to these community resources, Airbyte also offers premium support packages for users who require additional assistance beyond what is provided by the community. - -## OSS Premium Support -Open source [premium support packages](https://airbyte.com/talk-to-sales-premium-support) are a great option for who use Airbyte OSS and need additional assistance beyond what is provided by the community. These packages typically include access to a dedicated support team that can provide assistance with installation, configuration, troubleshooting, and other technical issues. Premium support packages also often include faster response times, guaranteed issue resolution, and access to updates and patches. By opting for a premium support package, users can enjoy the benefits of open source software while also receiving the peace of mind they need to keep their systems running smoothly. - -Premier Support comes with: - -* 1-business-day SLA for your Severity 0 and 1 -* 2-business-day SLA for your Severity 2 and 3 -* 1-week Pull Request review SLA for first comment -If you need better SLA times, we can definitely discuss this, don't hesitate to [talk to our team](https://airbyte.com/talk-to-sales) about it. You can also see more details about it in our pricing page. - -## Office Hour -Airbyte provides a [Daily Office Hour](https://airbyte.com/daily-office-hour) to discuss issues. -It is a 45 minute meeting, the first 20 minutes are reserved to a weekly topic presentation about Airbyte concepts and the others 25 minutes are for general questions. The schedule is: -* Monday, Wednesday and Fridays: 1 PM PST/PDT -* Tuesday and Thursday: 4 PM CEST - - -## Github Issues -Whenever you face an issue using a connector or with the platform you're welcome to report opening a Github issue. -https://github.com/airbytehq/airbyte - - -## Airbyte Slack -You can access Airbyte Slack [here](https://slack.airbyte.com/). - -**Before posting on a channel this please first check if a similar question was already answered.** - -**The existing categories**: -* `#help-connections-issues`: for any questions or issues on your connections -* `#help-infrastructure-deployment`: for any questions or issues on your deployment and infrastructure -* `#help-connector-development`: for any questions about on the CDKs and issues while building a custom connector -* `#help-api-cli-orchestration`: for any questions or issues about the API, CLI, any scheduling effort. -* `#help-contributions`: for any questions about contributing to Airbyte’s codebase - -## Airbyte Forum -We are driving our community support from our [forum](https://github.com/airbytehq/airbyte/discussions). - -**Before posting on this forum please first check if a similar question was already answered.** - -**The existing categories**: -* 🙏 Questions: Ask the community for help on your question. As a reminder, the Airbyte team won’t provide help here, as our support is part of our Airbyte Cloud and Airbyte Enterprise offers. -* 💡 Ideas: Share ideas for new features, improvements, or feedback. -* 🙌 Show & Tell: Share projects, tutorials, videos, and articles you are working on. -* 🫶 Kind words: Show off something you love about Airbyte -* 🐙 General: For anything that doesn’t fit in the above categories diff --git a/docs/understanding-airbyte/airbyte-protocol.md b/docs/understanding-airbyte/airbyte-protocol.md index edf7c7f15c06..6356757747e0 100644 --- a/docs/understanding-airbyte/airbyte-protocol.md +++ b/docs/understanding-airbyte/airbyte-protocol.md @@ -26,18 +26,26 @@ Each of these concepts is described in greater depth in their respective section The Airbyte Protocol is versioned independently of the Airbyte Platform, and the version number is used to determine the compatibility between connectors and the Airbyte Platform. -| Version | Date of Change | Pull Request(s) | Subject | -|:---------|:---------------|:--------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------| -| `v1.0.0` | 2022-11-28 | [17486](https://github.com/airbytehq/airbyte/pull/17486) & [19846](https://github.com/airbytehq/airbyte/pull/19846) | Well known data types added | -| `v0.3.2` | 2022-10-28 | [18875](https://github.com/airbytehq/airbyte/pull/18875) | `AirbyteEstimateTraceMessage` added | -| `v0.3.1` | 2022-10-12 | [17907](https://github.com/airbytehq/airbyte/pull/17907) | `AirbyteControlMessage.ConnectorConfig` added | -| `v0.3.0` | 2022-09-09 | [16479](https://github.com/airbytehq/airbyte/pull/16479) | `AirbyteLogMessage.stack_trace` added | -| `v0.2.0` | 2022-06-10 | [13573](https://github.com/airbytehq/airbyte/pull/13573) & [12586](https://github.com/airbytehq/airbyte/pull/12586) | `STREAM` and `GLOBAL` STATE messages | -| `v0.1.1` | 2022-06-06 | [13356](https://github.com/airbytehq/airbyte/pull/13356) | Add a namespace in association with the stream name | -| `v0.1.0` | 2022-05-03 | [12458](https://github.com/airbytehq/airbyte/pull/12458) & [12581](https://github.com/airbytehq/airbyte/pull/12581) | `AirbyteTraceMessage` added to allow connectors to better communicate exceptions | -| `v0.0.2` | 2021-11-15 | [7798](https://github.com/airbytehq/airbyte/pull/7798) | Support oAuth Connectors (internal) | -| `v0.0.1` | 2021-11-19 | [1021](https://github.com/airbytehq/airbyte/pull/1021) | Remove sub-JSON Schemas | -| `v0.0.0` | 2020-11-18 | [998](https://github.com/airbytehq/airbyte/pull/998) | Initial version described via JSON Schema | +| Version | Date of Change | Pull Request(s) | Subject | +|:---------|:---------------|:--------------------------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------------| +| `v0.5.2` | 2023-12-26 | [58](https://github.com/airbytehq/airbyte-protocol/pull/58) | Remove unused V1. | +| `v0.5.1` | 2023-04-12 | [53](https://github.com/airbytehq/airbyte-protocol/pull/53) | Modify various helper libraries. | +| `v0.5.0` | 2023-11-13 | [49](https://github.com/airbytehq/airbyte-protocol/pull/49) | `AirbyteStateStatsMessage` added. | +| `v0.4.2` | 2023-04-12 | [46](https://github.com/airbytehq/airbyte-protocol/pull/46) | `AirbyteAnalyticsTraceMessage` added. | +| `v0.4.1` | 2023-08-14 | [41](https://github.com/airbytehq/airbyte-protocol/pull/41) & [44](https://github.com/airbytehq/airbyte-protocol/pull/44) | Various bug fixes. | +| `v0.3.6` | 2023-04-21 | [34](https://github.com/airbytehq/airbyte-protocol/pull/34) | Add explicit `AirbyteStreamStatus` statue values. | +| `v0.3.5` | 2023-04-13 | [30](https://github.com/airbytehq/airbyte-protocol/pull/30) | Fix indentation. | +| `v0.3.4` | 2023-04-13 | [28](https://github.com/airbytehq/airbyte-protocol/pull/28) | Fix Indentation. | +| `v0.3.3` | 2023-04-12 | [18](https://github.com/airbytehq/airbyte-protocol/pull/18) | `AirbyteStreamStatusMessage` added. | +| `v0.3.2` | 2022-10-28 | [18875](https://github.com/airbytehq/airbyte/pull/18875) | `AirbyteEstimateTraceMessage` added. | +| `v0.3.1` | 2022-10-12 | [17907](https://github.com/airbytehq/airbyte/pull/17907) | `AirbyteControlMessage.ConnectorConfig` added. | +| `v0.3.0` | 2022-09-09 | [16479](https://github.com/airbytehq/airbyte/pull/16479) | `AirbyteLogMessage.stack_trace` added. | +| `v0.2.0` | 2022-06-10 | [13573](https://github.com/airbytehq/airbyte/pull/13573) & [12586](https://github.com/airbytehq/airbyte/pull/12586) | `STREAM` and `GLOBAL` STATE messages. | +| `v0.1.1` | 2022-06-06 | [13356](https://github.com/airbytehq/airbyte/pull/13356) | Add a namespace in association with the stream name. | +| `v0.1.0` | 2022-05-03 | [12458](https://github.com/airbytehq/airbyte/pull/12458) & [12581](https://github.com/airbytehq/airbyte/pull/12581) | `AirbyteTraceMessage` added to allow connectors to better communicate exceptions. | +| `v0.0.2` | 2021-11-15 | [7798](https://github.com/airbytehq/airbyte/pull/7798) | Support oAuth Connectors (internal). | +| `v0.0.1` | 2021-11-19 | [1021](https://github.com/airbytehq/airbyte/pull/1021) | Remove sub-JSON Schemas. | +| `v0.0.0` | 2020-11-18 | [998](https://github.com/airbytehq/airbyte/pull/998) | Initial version described via JSON Schema. | ## Actor Interface @@ -143,7 +151,7 @@ The `discover` method detects and describes the _structure_ of the data in the d 1. `config` - A configuration JSON object that has been validated using `ConnectorSpecification#connectionSpecification` (see [ActorSpecification](#actor-specification) for information on `connectionSpecification`). 2. `configured catalog` - A `ConfiguredAirbyteCatalog` is built on top of the `catalog` returned by `discover`. The `ConfiguredAirbyteCatalog` specifies HOW the data in the catalog should be replicated. The catalog is documented in the [Catalog Section](#catalog). -3. `state` - An JSON object that represents a checkpoint in the replication. This object is only ever written or read by the source, so it is a JSON blob with whatever information is necessary to keep track of how much of the data source has already been read (learn more in the [State & Checkpointing](#state--checkpointing) Section). +3. `state` - A JSON object that represents a checkpoint in the replication. This object is only ever written or read by the source, so it is a JSON blob with whatever information is necessary to keep track of how much of the data source has already been read (learn more in the [State & Checkpointing](#state--checkpointing) Section). #### Output: @@ -171,7 +179,7 @@ For the sake of brevity, we will not re-describe `spec` and `check`. They are ex #### Input: 1. `config` - A configuration JSON object that has been validated using the `ConnectorSpecification`. -2. `catalog` - An `AirbyteCatalog`. This `catalog` should be a subset of the `catalog` returned by the `discover` command. Any `AirbyteRecordMessages`s that the destination receives that do _not_ match the structure described in the `catalog` will fail. +2. `configured catalog` - A [`ConfiguredAirbyteCatalog`](https://docs.airbyte.com/understanding-airbyte/beginners-guide-to-catalog#configuredairbytecatalog). This is a modified version of the `catalog` returned by the `discover` command. Any `AirbyteRecordMessages`s that the destination receives that do _not_ match the structure described in the `catalog` will fail. 3. `message stream` - \(this stream is consumed on stdin--it is not passed as an arg\). It will receive a stream of JSON-serialized `AirbyteMesssage`. #### Output: @@ -333,7 +341,7 @@ Technical systems often group their underlying data into namespaces with each na An example of a namespace is the RDBMS's `schema` concept. An API namespace might be used for multiple accounts (e.g. `company_a` vs `company_b`, each having a "users" and "purchases" stream). Some common use cases for schemas are enforcing permissions, segregating test and production data and general data organization. -The `AirbyteStream` represents this concept through an optional field called `namespace`. Additional documentation on Namespaces can be found [here](namespaces.md). +The `AirbyteStream` represents this concept through an optional field called `namespace`. Additional documentation on Namespaces can be found [here](/using-airbyte/core-concepts/namespaces.md). ### Cursor @@ -449,6 +457,8 @@ This concept enables incremental syncs--syncs that only replicate data that is n State also enables Partial Success. In the case where during a sync there is a failure before all data has been extracted and committed, if all records up to a certain state are committed, then the next time the sync happens, it can start from that state as opposed to going back to the beginning. Partial Success is powerful, because especially in the case of high data volumes and long syncs, being able to pick up from wherever the failure occurred can costly re-syncing of data that has already been replicated. +The state for an actor is emitted as a black box by the Source. When emitted it is wrapped in the [AirbyteStateMessage](#airbytestatemessage). The contents of the `data` field is what is passed to the Source on start up. It is up to the Source to interpret the state object. Nothing outside the Source can make any inference about the state of the object EXCEPT, if it is null, it can be concluded that there is no state and the Source will start at the beginning. + ### State & Source This section will step through how state is used to allow a Source to pick up where it left off. A Source takes state as an input. A Source should be able to take that input and use it to determine where it left off the last time. The contents of the Source is a black box to the Protocol. The Protocol provides an envelope for the Source to put its state in and then passes the state back in that envelope. The Protocol never needs to know anything about the contents of the state. Thus, the Source can track state however makes most sense to it. @@ -478,19 +488,14 @@ The normal success case (T3, not depicted) would be that all the records would m -- [link](https://whimsical.com/state-TYX5bSCVtVF4BU1JbUwfpZ) to source image -### V1 - -The state for an actor is emitted as a complete black box. When emitted it is wrapped in the [AirbyteStateMessage](#airbytestatemessage-v1). The contents of the `data` field is what is passed to the Source on start up. This gives the Source lead to decide how to track the state of each stream. That being said, a common pattern is a `Map`. Nothing outside the source can make any inference about the state of the object EXCEPT, if it is null, it can be concluded that there is no state and the Source will start at the beginning. - -### V2 (coming soon!) - -In addition to allowing a Source to checkpoint data replication, the state object is structure to allow for the ability to configure and reset streams in isolation from each other. For example, if adding or removing a stream, it is possible to do so without affecting the state of any other stream in the Source. +### State Types +In addition to allowing a Source to checkpoint data replication, the state object allows for the ability to configure and reset streams in isolation from each other. For example, if adding or removing a stream, it is possible to do so without affecting the state of any other stream in the Source. There are 3 types of state: Stream, Global, and Legacy. - **Stream** represents Sources where there is complete isolation between stream states. In these cases, the state for each stream will be emitted in its own state message. In other words, if there are 3 streams replicated during a sync, the Source would emit at least 3 state message (1 per stream). The state of the Source is the sum of all the stream states. - **Global** represents Sources where this shared state across streams. In these cases each state message contains the whole state for the connection. The `shared_state` field is where any information that is shared across streams must go. The `stream_states` field contains a list of objects that contain a Stream Descriptor and the state information for that stream that is stream-specific. There are drawbacks to this state type, so it should only be used in cases where a shared state between streams is unavoidable. -- **Legacy** exists for backwards compatibility. In this state type, the state object is totally a black box. The only inference tha can be drawn from the state object is that if it is null, then there is no state for the entire Source. All current legacy cases can be ported to stream or global. Once they are, it will be removed. +- **Legacy** exists for backwards compatibility. In this state type, the state object is totally a black box. The only inference tha can be drawn from the state object is that if it is null, then there is no state for the entire Source. **All current legacy cases are being ported to stream or global. Once they are, it will be removed.** This table breaks down attributes of these state types. @@ -500,11 +505,45 @@ This table breaks down attributes of these state types. | Stream-Level Replication Isolation | X | | | | Single state message describes full state for Source | | X | X | -- **Protocol Version** simply connotes which versions of the Protocol have support for these State types. The new state message is backwards compatible with the V1 message. This allows old versions of connectors and platforms to interact with the new message. - **Stream-Level Configuration / Reset** was mentioned above. The drawback of the old state struct was that it was not possible to configure or reset the state for a single stream without doing it for all of them. Thus, new state types support this, but the legacy one cannot. - **Stream-Level Replication Isolation** means that a Source could be run in parallel by splitting up its streams across running instances. This is only possible for Stream state types, because they are the only state type that can update its current state completely on a per-stream basis. This is one of the main drawbacks of Sources that use Global state; it is not possible to increase their throughput through parallelization. - **Single state message describes full state for Source** means that any state message contains the full state information for a Source. Stream does not meet this condition because each state message is scoped by stream. This means that in order to build a full picture of the state for the Source, the state messages for each configured stream must be gathered. +### State Principles +The following are principles Airbyte recommends Sources/Destinations adhere to with State. Airbyte enforces these principles via our CDK. + +These principles are intended to produce simple overall system behavior, and move Airbyte towards a world of shorter-lived jobs. The goal is reliable data movement with minimal data loss windows on errors. + +1. **New Sources must use per-stream/global State**. + + Per-stream/Global state unlocks more granular State operations e.g. per-stream resets, per-stream parallelisation etc. No new Connectors should be created using Legacy state. + +2. **Sources always emit State, regardless of sync mode.** + + This simplifies how the Platform treats jobs and means all Syncs are resumable. This also enables checkpointing on full refreshes in the future. This rule does not appear to Sources that do not support cursors. + However: + 1. If the source stream has no records, an empty state should still be emitted. This supports state-based counts/checksums. It is recommended for the emitted state to have unique and non-null content. + 2. If the stream is unsorted, and therefore non-resumable, it is recommended to still send a state message, even with bogus resumability, to indicate progress in the sync. + +3. **Sources do not emit sequential duplicate States with interleaved records.** + + Duplicate States make it challenging to debug state-related operations. E.g. Is this a duplicate or did we fail to properly update state? Is this a duplicate log? Sync will fail if this rule is violated. + +4. **Sources should emit state whenever it is meaningful to resume a failed sync. Platform reserves the right to discard too frequent State emission per internal platform rules.** + + Sources should strive to emit state as fast as it’s useful. Platform can discard this state if this leads to undesirable downstream behavior e.g. out of memory. This is fine as there is increasingly lower marginal value to emitting States at higher frequencies. + +5. **Platform & Destinations treat state as a black box.** + + Sources are the sole producer/consumer of a State messages’ contents. Precisely, this refers to the state fields within the various State messages. Modifying risks corrupting our data sync cursor, which is a strict no-no. + +6. **Destinations return state in the order it was received.** + + Order is used by the Platform to determine if a State message was dropped. Out-of-order State messages throw errors. + + Order-ness is determined by the type of State message. Per-stream state messages require order per-stream. Global state messages require global ordering. + + ## Messages ### Common @@ -615,26 +654,7 @@ AirbyteRecordMessage: type: integer ``` -### AirbyteStateMessage (V1) - -The state message enables the Source to emit checkpoints while replicating data. These checkpoints mean that if replication fails before completion, the next sync is able to start from the last checkpoint instead of returning to the beginning of the previous sync. The details of this process are described in [State & Checkpointing](#state--checkpointing). - -The state message is a wrapper around the state that a Source emits. The state that the Source emits is treated as a black box by the protocol--it is modeled as a JSON blob. - -```yaml -AirbyteStateMessage: - type: object - additionalProperties: true - required: - - data - properties: - data: - description: "the state data" - type: object - existingJavaType: com.fasterxml.jackson.databind.JsonNode -``` - -### AirbyteStateMessage (V2 -- coming soon!) +### AirbyteStateMessage The state message enables the Source to emit checkpoints while replicating data. These checkpoints mean that if replication fails before completion, the next sync is able to start from the last checkpoint instead of returning to the beginning of the previous sync. The details of this process are described in [State & Checkpointing](#state--checkpointing). @@ -701,7 +721,7 @@ AirbyteGlobalState: ### AirbyteConnectionStatus Message -This message reports whether an Actor was able to connect to its underlying data store with all the permissions it needs to succeed. The goal is that if a successful stat is returned, that the user should be confident that using that Actor will succeed. The depth of the verification is not specified in the protocol. More robust verification is preferred but going to deep can create undesired performance tradeoffs +This message reports whether an Actor was able to connect to its underlying data store with all the permissions it needs to succeed. The goal is that if a successful stat is returned, that the user should be confident that using that Actor will succeed. The depth of the verification is not specified in the protocol. More robust verification is preferred but going too deep can create undesired performance tradeoffs. ```yaml AirbyteConnectionStatus: diff --git a/docs/understanding-airbyte/beginners-guide-to-catalog.md b/docs/understanding-airbyte/beginners-guide-to-catalog.md index ff5451e15c5d..1953b1681c82 100644 --- a/docs/understanding-airbyte/beginners-guide-to-catalog.md +++ b/docs/understanding-airbyte/beginners-guide-to-catalog.md @@ -16,7 +16,7 @@ This article will illustrate how to use `AirbyteCatalog` via a series of example * [Dynamic Streams Example](#dynamic-streams-example) * [Nested Schema Example](#nested-schema-example) -In order to understand in depth how to configure incremental data replication, head over to the [incremental replication docs](connections/incremental-append.md). +In order to understand in depth how to configure incremental data replication, head over to the [incremental replication docs](/using-airbyte/core-concepts/sync-modes/incremental-append.md). ## Database Example @@ -92,7 +92,7 @@ The catalog is structured as a list of `AirbyteStream`. In the case of a databas Let's walk through what each field in a stream means. * `name` - The name of the stream. -* `supported_sync_modes` - This field lists the type of data replication that this source supports. The possible values in this array include `FULL_REFRESH` \([docs](connections/full-refresh-overwrite.md)\) and `INCREMENTAL` \([docs](connections/incremental-append.md)\). +* `supported_sync_modes` - This field lists the type of data replication that this source supports. The possible values in this array include `FULL_REFRESH` \([docs](/using-airbyte/core-concepts/sync-modes/full-refresh-overwrite.md)\) and `INCREMENTAL` \([docs](/using-airbyte/core-concepts/sync-modes/incremental-append.md)\). * `source_defined_cursor` - If the stream supports `INCREMENTAL` replication, then this field signals whether the source can figure out how to detect new records on its own or not. * `json_schema` - This field is a [JsonSchema](https://json-schema.org/understanding-json-schema) object that describes the structure of the data. Notice that each key in the `properties` object corresponds to a column name in our database table. @@ -137,7 +137,7 @@ Let's walk through each field in the `ConfiguredAirbyteStream`: * `sync_mode` - This field must be one of the values that was in `supported_sync_modes` in the `AirbyteStream` - Configures which sync mode will be used when data is replicated. * `stream` - Hopefully this one looks familiar! This field contains an `AirbyteStream`. It should be _identical_ to the one we saw in the `AirbyteCatalog`. -* `cursor_field` - When `sync_mode` is `INCREMENTAL` and `source_defined_cursor = false`, this field configures which field in the stream will be used to determine if a record should be replicated or not. Read more about this concept in our [documentation of incremental replication](connections/incremental-append.md). +* `cursor_field` - When `sync_mode` is `INCREMENTAL` and `source_defined_cursor = false`, this field configures which field in the stream will be used to determine if a record should be replicated or not. Read more about this concept in our [documentation of incremental replication](/using-airbyte/core-concepts/sync-modes/incremental-append.md). ### Summary of the Postgres Example diff --git a/docs/understanding-airbyte/connections/README.md b/docs/understanding-airbyte/connections/README.md deleted file mode 100644 index 49a0756a43d9..000000000000 --- a/docs/understanding-airbyte/connections/README.md +++ /dev/null @@ -1,84 +0,0 @@ -# Connections and Sync Modes - -A connection is a configuration for syncing data between a source and a destination. To setup a connection, a user must configure things such as: - -- Sync schedule: when to trigger a sync of the data. -- Destination [Namespace](../namespaces.md) and stream names: where the data will end up being written. -- A catalog selection: which [streams and fields](../airbyte-protocol.md#catalog) to replicate from the source -- Sync mode: how streams should be replicated \(read and write\): -- Optional transformations: how to convert Airbyte protocol messages \(raw JSON blob\) data into some other data representations. - -## Sync schedules - -Sync schedules are explained below. For information about catalog selections, see [AirbyteCatalog & ConfiguredAirbyteCatalog](../airbyte-protocol.md#catalog). - -Syncs will be triggered by either: - -- A manual request \(i.e: clicking the "Sync Now" button in the UI\) -- A schedule - -When a scheduled connection is first created, a sync is executed as soon as possible. After that, a sync is run once the time since the last sync \(whether it was triggered manually or due to a schedule\) has exceeded the schedule interval. For example, consider the following illustrative scenario: - -- **October 1st, 2pm**, a user sets up a connection to sync data every 24 hours. -- **October 1st, 2:01pm**: sync job runs -- **October 2nd, 2:01pm:** 24 hours have passed since the last sync, so a sync is triggered. -- **October 2nd, 5pm**: The user manually triggers a sync from the UI -- **October 3rd, 2:01pm:** since the last sync was less than 24 hours ago, no sync is run -- **October 3rd, 5:01pm:** It has been more than 24 hours since the last sync, so a sync is run - -## Destination namespace - -The location of where a connection replication will store data is referenced as the destination namespace. The destination connectors should create and write records \(for both raw and normalized tables\) in the specified namespace which should be configurable in the UI via the Namespace Configuration field \(or NamespaceDefinition in the API\). You can read more about configuring namespaces [here](../namespaces.md). - -## Destination stream name - -### Prefix stream name - -Stream names refer to table names in a typical RDBMS. But it can also be the name of an API endpoint, etc. Similarly to the namespace, stream names can be configured to diverge from their names in the source with a "prefix" field. The prefix is prepended to the source stream name in the destination. - -## Stream-specific customization - -All the customization of namespace and stream names described above will be equally applied to all streams selected for replication in a catalog per connection. If you need more granular customization, stream by stream, for example, or with different logic rules, then you could follow the tutorial on [customizing transformations with dbt](../../operator-guides/transformation-and-normalization/transformations-with-dbt.md). - -## Sync modes - -A sync mode governs how Airbyte reads from a source and writes to a destination. Airbyte provides different sync modes to account for various use cases. To minimize confusion, a mode's behavior is reflected in its name. The easiest way to understand Airbyte's sync modes is to understand how the modes are named. - -1. The first part of the name denotes how the source connector reads data from the source: - 1. Incremental: Read records added to the source since the last sync job. \(The first sync using Incremental is equivalent to a Full Refresh\) - - Method 1: Using a cursor. Generally supported by all connectors whose data source allows extracting records incrementally. - - Method 2: Using change data capture. Only supported by some sources. See [CDC](../cdc.md) for more info. - 2. Full Refresh: Read everything in the source. -2. The second part of the sync mode name denotes how the destination connector writes data. This is not affected by how the source connector produced the data: - 1. Overwrite: Overwrite by first deleting existing data in the destination. - 2. Append: Write by adding data to existing tables in the destination. - 3. Deduped History: Write by first adding data to existing tables in the destination to keep a history of changes. The final table is produced by de-duplicating the intermediate ones using a primary key. - -A sync mode is therefore, a combination of a source and destination mode together. The UI exposes the following options, whenever both source and destination connectors are capable to support it for the corresponding stream: - -- [Full Refresh Overwrite](full-refresh-overwrite.md): Sync the whole stream and replace data in destination by overwriting it. -- [Full Refresh Append](full-refresh-append.md): Sync the whole stream and append data in destination. -- [Incremental Append](incremental-append.md): Sync new records from stream and append data in destination. -- [Incremental Append + Deduped](incremental-append-deduped.md): Sync new records from stream and append data in destination, also provides a de-duplicated view mirroring the state of the stream in the source. - -## Optional operations - -### Airbyte basic normalization - -As described by the [Airbyte Protocol from the Airbyte Specifications](../airbyte-protocol.md), replication is composed of source connectors that are transmitting data in a JSON format. It is then written as such by the destination connectors. - -On top of this replication, Airbyte provides the option to enable or disable an additional transformation step at the end of the sync called [basic normalization](../basic-normalization.md). This operation is: - -- Only available for destinations that support dbt execution -- Automatically generates a pipeline or DAG of dbt transformation models to convert JSON blob objects into normalized tables -- Runs and applies these dbt models to the data written in the destination - -:::note - -Normalizing data may cause an increase in your destination's compute cost. This cost will vary depending on the amount of data that is normalized and is not related to Airbyte credit usage. - -::: - -### Custom sync operations - -Further operations can be included in a sync on top of Airbyte basic normalization \(or even to replace it completely\). See [operations](../operations.md) for more details. diff --git a/docs/understanding-airbyte/connections/full-refresh-overwrite.md b/docs/understanding-airbyte/connections/full-refresh-overwrite.md deleted file mode 100644 index 44d4ff5f6699..000000000000 --- a/docs/understanding-airbyte/connections/full-refresh-overwrite.md +++ /dev/null @@ -1,47 +0,0 @@ -# Full Refresh - Overwrite - -## Overview - -The **Full Refresh** modes are the simplest methods that Airbyte uses to sync data, as they always retrieve all available information requested from the source, regardless of whether it has been synced before. This contrasts with [**Incremental sync**](incremental-append.md), which does not sync data that has already been synced before. - -In the **Overwrite** variant, new syncs will destroy all data in the existing destination table and then pull the new data in. Therefore, data that has been removed from the source after an old sync will be deleted in the destination table. - -## Example Behavior - -On the nth sync of a full refresh connection: - -## _Replace_ existing data with new data. The connection does not create any new tables. - -data in the destination _before_ the sync: - -| Languages | -| :--- | -| Python | -| Java | - -new data: - -| Languages | -| :--- | -| Python | -| Java | -| Ruby | - -data in the destination _after_ the sync: - -| Languages | -| :--- | -| Python | -| Java | -| Ruby | - -Note: This is how Singer target-bigquery does it. - -## In the future - -We will consider making other flavors of full refresh configurable as first-class citizens in Airbyte. e.g. On new data, copy old data to a new table with a timestamp, and then replace the original table with the new data. As always, we will focus on adding these options in such a way that the behavior of each connector is both well documented and predictable. - -## Related information - -- [An overview of Airbyte’s replication modes](https://airbyte.com/blog/understanding-data-replication-modes). -- [Explore Airbyte's full refresh data synchronization](https://airbyte.com/tutorials/full-data-synchronization). diff --git a/docs/understanding-airbyte/heartbeats.md b/docs/understanding-airbyte/heartbeats.md new file mode 100644 index 000000000000..88ce6f86d46d --- /dev/null +++ b/docs/understanding-airbyte/heartbeats.md @@ -0,0 +1,34 @@ +# Heartbeats + +During a data synchronization, many things can go wrong and sometimes the fix is just to restart the synchronization. +Airbyte aims to make this restart as automated as possible and uses heartbeating mechanism in order to do that. +This performed on 2 differents component: the source and the destination. They have different logics which will be +explained bellow. + +## Source + +### Heartbeating logic + +The platform considers both `RECORD` and `STATE` messages emitted by the source as source heartbeats. +The Airbyte platform has a process which monitors when the last beat was send and if it reaches a threshold, +the synchronization attempt will be failed. It fails with a cause being the source an message saying +`The source is unresponsive`. Internal the error has a heartbeat timeout type, which is not display in the UI. + +### Configuration + +The heartbeat can be configured using the file flags.yaml through 2 entries: +* `heartbeat-max-seconds-between-messages`: this configures the maximum time allowed between 2 messages. +The default is 3 hours. +* `heartbeat.failSync`: Setting this to true will make the syncs to fail if a missed heartbeat is detected. +If false no sync will be failed because of a missed heartbeat. The default value is true. + +## Destination + +### Heartbeating logic + +Adding a heartbeat to the destination similar to the one at the source is not straightforward since there isn't a constant stream of messages from the destination to the platform. Instead, we have implemented something that is more akin to a timeout. The platform monitors whether there has been a call to the destination that has taken more than a specified amount of time. If such a delay occurs, the platform considers the destination to have timed out. + +### Configuration +The timeout can be configured using the file `flags.yaml` through 2 entries: +* `destination-timeout-max-seconds`: If the platform detects a call to the destination exceeding the duration specified in this entry, it will consider the destination to have timed out. The default timeout value is 24 hours. +* `destination-timeout.failSync`: If enabled (true by default), a detected destination timeout will cause the platform to fail the sync. If not, the platform will log a message and allow the sync to continue. When the platform fails a sync due to a destination timeout, the UI will display the message: `The destination is unresponsive`. diff --git a/docs/understanding-airbyte/jobs.md b/docs/understanding-airbyte/jobs.md index 0b577a72a5a2..c9b56ee60566 100644 --- a/docs/understanding-airbyte/jobs.md +++ b/docs/understanding-airbyte/jobs.md @@ -19,9 +19,50 @@ At a high level, a sync job is an individual invocation of the Airbyte pipeline Sync jobs have the following state machine. -![Job state machine](../.gitbook/assets/job-state-machine.png) +```mermaid +--- +title: Job Status State Machine +--- +stateDiagram-v2 +direction TB +state NonTerminal { + [*] --> pending + pending + running + incomplete + note left of incomplete + When an attempt fails, the job status is transitioned to incomplete. + If this is the final attempt, then the job is transitioned to failed. + Otherwise it is transitioned back to running upon new attempt creation. + + end note +} +note left of NonSuccess + All Non Terminal Statuses can be transitioned to cancelled or failed +end note + +pending --> running +running --> incomplete +incomplete --> running +running --> succeeded +state NonSuccess { + cancelled + failed +} +NonTerminal --> NonSuccess +``` + + +```mermaid +--- +title: Attempt Status State Machine +--- +stateDiagram-v2 + direction LR + running --> succeeded + running --> failed +``` -[Image Source](https://docs.google.com/drawings/d/1cp8LRZs6UnhAt3jbQ4h40nstcNB0OBOnNRdMFwOJL8I/edit) ### Attempts and Retries @@ -217,9 +258,17 @@ This section will depict the worker-job architecture as discussed above. Only th The source process should automatically exit after passing all of its messages. Similarly, the destination process shutdowns after receiving all records. Each process is given a shutdown grace period. The worker forces shutdown if this is exceeded. -![Worker Lifecycle](../.gitbook/assets/worker-lifecycle.png) +```mermaid +sequenceDiagram + Worker->>Source: docker run + Worker->>Destination: docker run + Source->>Worker: STDOUT + Worker->>Destination: STDIN + Worker->>Source: exit* + Worker->>Destination: exit* + Worker->>Result: json output +``` -[Image Source](https://docs.google.com/drawings/d/1k4v_m2M5o2UUoNlYM7mwtZicRkQgoGLgb3eTOVH8QFo/edit) See the [architecture overview](high-level-view.md) for more information about workers. @@ -260,11 +309,34 @@ Brief description of how this works, The Cloud Storage store is treated as the source-of-truth of execution state. -The Container Orchestrator is only available for Airbyte Kubernetes today and automatically enabled when running the Airbyte Helm charts/Kustomize deploys. +The Container Orchestrator is only available for Airbyte Kubernetes today and automatically enabled when running the Airbyte Helm Charts deploys. + + +```mermaid +--- +title: Start a new Sync +--- +sequenceDiagram +%% participant API + participant Temporal as Temporal Queues + participant Sync as Sync Workflow + participant ReplicationA as Replication Activity + participant ReplicationP as Replication Process + participant PersistA as Persistent Activity + participant AirbyteDB + Sync->>Temporal: Start a replication Activity + Temporal->>Sync: Pick up a new Sync + Temporal->>ReplicationA: Pick up a new task + ReplicationA->>ReplicationP: Starts a process + ReplicationP->>ReplicationA: Replication Summary with State message and stats + ReplicationA->>Temporal: Return Output (States and Summary) + Temporal->>Sync: Read results from Replication Activity + Sync->>Temporal: Start Persistent State Activity + Temporal->>PersistA: Pick up new task + PersistA->>AirbyteDB: Persist States + PersistA->>Temporal: Return output +``` -![Orchestrator Lifecycle](../.gitbook/assets/orchestrator-lifecycle.png) - -[Image Source](https://whimsical.com/sync-lifecycle-Vays9o1YaxCKPhUEEKmqHM@2bsEvpTYSt1HjEZjriPY9jiAqCmgJ41MmyY) Users running Airbyte Docker should be aware of the above pitfalls. diff --git a/docs/understanding-airbyte/json-avro-conversion.md b/docs/understanding-airbyte/json-avro-conversion.md index fd602e24d227..54648af5421e 100644 --- a/docs/understanding-airbyte/json-avro-conversion.md +++ b/docs/understanding-airbyte/json-avro-conversion.md @@ -623,8 +623,8 @@ Its corresponding Avro schema will be: } ``` -More examples can be found in the Json to Avro conversion [test cases](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/destination-s3/src/test/resources/parquet/json_schema_converter/json_conversion_test_cases.json). +More examples can be found in the Json to Avro conversion [test cases](https://github.com/airbytehq/airbyte/tree/master/airbyte-integrations/bases/base-java-s3/src/test/resources/parquet/json_schema_converter). ## Implementation -- Schema conversion: [JsonToAvroSchemaConverter](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/destination-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/JsonToAvroSchemaConverter.java) +- Schema conversion: [JsonToAvroSchemaConverter](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/bases/base-java-s3/src/main/java/io/airbyte/integrations/destination/s3/avro/JsonToAvroSchemaConverter.java) - Object conversion: [airbytehq/json-avro-converter](https://github.com/airbytehq/json-avro-converter) (forked and modified from [allegro/json-avro-converter](https://github.com/allegro/json-avro-converter)). diff --git a/docs/understanding-airbyte/namespaces.md b/docs/understanding-airbyte/namespaces.md deleted file mode 100644 index d5deac5d12fc..000000000000 --- a/docs/understanding-airbyte/namespaces.md +++ /dev/null @@ -1,122 +0,0 @@ -# Namespaces - -## High-Level Overview - -:::info - -The high-level overview contains all the information you need to use Namespaces when pulling from APIs. Information past that can be read for advanced or educational purposes. - -::: - -When looking through our connector docs, you'll notice that some sources and destinations support "Namespaces." These allow you to organize and separate your data into groups in the destination if the destination supports it. In most cases, namespaces are schemas in the database you're replicating to. If your desired destination doesn't support it, you can ignore this feature. - -Note that this is the location that both your normalized and raw data will get written to. Your raw data will show up with the prefix `_airbyte_raw_` in the namespace you define. If you don't enable basic normalization, you will only receive the raw tables. - -If only your destination supports namespaces, you have two simple options. **This is the most likely case**, as all HTTP APIs currently don't support Namespaces. - -1. Mirror Destination Settings - Replicate to the default namespace in the destination, which will differ based on your destination. -2. Custom Format - Create a "Custom Format" to rename the namespace that your data will be replicated into. - -If both your desired source and destination support namespaces, you're likely using a more advanced use case with a database as a source, so continue reading. - -## What is a Namespace? - -Technical systems often group their underlying data into namespaces with each namespace's data isolated from another namespace. This isolation allows for better organisation and flexibility, leading to better usability. - -An example of a namespace is the RDMS's `schema` concept. Some common use cases for schemas are enforcing permissions, segregating test and production data and general data organisation. - -## Syncing - -The Airbyte Protocol supports namespaces and allows Sources to define namespaces, and Destinations to write to various namespaces. - -If the Source does not support namespaces, the data will be replicated into the Destination's default namespace. For databases, the default namespace is the schema provided in the destination configuration. - -If the Destination does not support namespaces, the [namespace field](https://github.com/airbytehq/airbyte/blob/master/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml#L64) is ignored. - -## Destination namespace configuration - -As part of the [connections sync settings](connections/), it is possible to configure the namespace used by: 1. destination connectors: to store the `_airbyte_raw_*` tables. 2. basic normalization: to store the final normalized tables. - -Note that custom transformation outputs are not affected by the namespace settings from Airbyte: It is up to the configuration of the custom dbt project, and how it is written to handle its [custom schemas](https://docs.getdbt.com/docs/building-a-dbt-project/building-models/using-custom-schemas). The default target schema for dbt in this case, will always be the destination namespace. - -Available options for namespace configurations are: - -### - Mirror source structure - -Some sources \(such as databases based on JDBC for example\) are providing namespace information from which a stream has been extracted. Whenever a source is able to fill this field in the catalog.json file, the destination will try to reproduce exactly the same namespace when this configuration is set. For sources or streams where the source namespace is not known, the behavior will fall back to the "Destination Connector settings". - -### - Destination connector settings - -All stream will be replicated and store in the default namespace defined on the destination settings page. In the destinations, namespace refers to: - -| Destination Connector | Namespace setting | -| :--- | :--- | -| BigQuery | dataset | -| MSSQL | schema | -| MySql | database | -| Oracle DB | schema | -| Postgres | schema | -| Redshift | schema | -| Snowflake | schema | -| S3 | path prefix | - -### - Custom format - -When replicating multiple sources into the same destination, conflicts on tables being overwritten by syncs can occur. - -For example, a Github source can be replicated into a "github" schema. But if we have multiple connections to different GitHub repositories \(similar in multi-tenant scenarios\): - -* we'd probably wish to keep the same table names \(to keep consistent queries downstream\) -* but store them in different namespaces \(to avoid mixing data from different "tenants"\) - -To solve this, we can either: - -* use a specific namespace for each connection, thus this option of custom format. -* or, use prefix to stream names as described below. - -Note that we can use a template format string using variables that will be resolved during replication as follow: - -* `${SOURCE_NAMESPACE}`: will be replaced by the namespace provided by the source if available - -### Examples - -The following table summarises how this works. We assume an example of replication configurations between a Postgres Source and Snowflake Destination \(with settings of schema = "my\_schema"\): - -| Namespace Configuration | Source Namespace | Source Table Name | Destination Namespace | Destination Table Name | -| :--- | :--- | :--- | :--- | :--- | -| Mirror source structure | public | my\_table | public | my\_table | -| Mirror source structure | | my\_table | my\_schema | my\_table | -| Destination connector settings | public | my\_table | my\_schema | my\_table | -| Destination connector settings | | my\_table | my\_schema | my\_table | -| Custom format = "custom" | public | my\_table | custom | my\_table | -| Custom format = "${SOURCE\_NAMESPACE}" | public | my\_table | public | my\_table | -| Custom format = "my\_${SOURCE\_NAMESPACE}\_schema" | public | my\_table | my\_public\_schema | my\_table | -| Custom format = " " | public | my\_table | my\_schema | my\_table | - -## Requirements - -* Both Source and Destination connectors need to support namespaces. -* Relevant Source and Destination connectors need to be at least version `0.3.0` or later. -* Airbyte version `0.21.0-alpha` or later. - -## Current Support - -### Sources - -* MSSQL -* MYSQL -* Oracle DB -* Postgres -* Redshift - -### Destination - -* BigQuery -* MSSQL -* MySql -* Oracle DB -* Postgres -* Redshift -* Snowflake -* S3 - diff --git a/docs/understanding-airbyte/operations.md b/docs/understanding-airbyte/operations.md index f3839499e39b..b21a087651b3 100644 --- a/docs/understanding-airbyte/operations.md +++ b/docs/understanding-airbyte/operations.md @@ -1,6 +1,6 @@ # Operations -Airbyte [connections](connections/) support configuring additional transformations that execute after the sync. Useful applications could be: +Airbyte [connections](/using-airbyte/core-concepts/sync-modes/) support configuring additional transformations that execute after the sync. Useful applications could be: * Customized normalization to better fit the requirements of your own business context. * Business transformations from a technical data representation into a more logical and business oriented data structure. This can facilitate usage by end-users, non-technical operators, and executives looking to generate Business Intelligence dashboards and reports. diff --git a/docs/understanding-airbyte/schemaless-sources-and-destinations.md b/docs/understanding-airbyte/schemaless-sources-and-destinations.md new file mode 100644 index 000000000000..edd4051ce2ca --- /dev/null +++ b/docs/understanding-airbyte/schemaless-sources-and-destinations.md @@ -0,0 +1,62 @@ +# "Schemaless" Sources and Destinations +In order to run a sync, Airbyte requires a [catalog](/understanding-airbyte/airbyte-protocol#catalog), which includes a data schema describing the shape of data being emitted by the source. +This schema will be used to prepare the destination to populate the data during the sync. + +While having a [strongly-typed](/understanding-airbyte/supported-data-types) catalog/schema is possible for most sources, some won't have a reasonably static schema. This document describes the options available for the subset of sources that do not have a strict schema, aka "schemaless sources". + +## What is a Schemaless Source? +Schemaless sources are sources for which there is no requirement or expectation that records will conform to a particular pattern. +For example, in a MongoDB database, there's no requirement that the fields in one document are the same as the fields in the next, or that the type of value in one field is the same as the type for that field in a separate document. +Similarly, for a file-based source such as S3, the files that are present in your source may not all have the same schema. + +Although the sources themselves may not conform to an obvious schema, Airbyte still needs to know the shape of the data in order to prepare the destination for the records. +For these sources, during the [`discover`](/understanding-airbyte/airbyte-protocol#discover) method, Airbyte offers two options to create the schema: + +1. Dynamic schema inference. +2. A hardcoded "schemaless" schema. + +### Dynamic schema inference +If this option is selected, Airbyte will infer the schema dynamically based on the contents of the source. +If your source's content is homogenous, we recommend this option, as the data in your destination will be typed and you can make use of schema evolution features, column selection, and similar Airbyte features which operate against the source's schema. + +For MongoDB, you can configure the number of documents that will be used for schema inference (from 1,000 to 10,000 documents; by default, this is set to 10,000). +Airbyte will read in the requested number of documents (sampled randomly) and infer the schema from them. +For file-based sources, we look at up to 10 files (reading up to 1MB per file) and infer the schema based on the contents of those files. + +In both cases, as the contents of the source change, the schema can change too. + +The schema that's produced from the inference procedure will include all the top-level fields that were observed in the sampled records. +The type assigned to each field will be the widest type observed for that field in any of the sampled data. +So if we observe that a field has an integer type in one record and a string in another, the schema will identify the field as a string. + +There are a few drawbacks to be aware of: +- If your dataset is very large, the `discover` process can be very time-consuming. +- Because we may not use 100% of the available data to create the schema, your schema may not contain every field present in your records. + Airbyte only syncs fields that are in the schema, so you may end up with incomplete data in the destination. + +If your data set is very large or you anticipate that it will change often, we recommend using the "schemaless" schema to avoid these issues. + +_Note: For MongoDB, knowing how variable your dataset is can help you choose an appropriate value for the number of documents to use for schema inference. +If your data is uniform across all or most records, you can set this to a lower value, providing better performance on discover and during the sync. +If your data varies but you cannot use the Schemaless option, you can set it to a larger value to ensure that as many fields as possible are accounted for._ + +### Schemaless schema +If this option is selected, the schema will always be `{"data": object}`, regardless of the contents of the data. +During the sync, we "wrap" each record behind a key named `data`. +This means that the destination receives the data with one top-level field only, and the value of the field is the entire record. +This option avoids a time-consuming or inaccurate `discover` phase and guarantees that everything ends up in your destination, at the expense of Airbyte being able to structure the data into different columns. + +## Future Enhancements + +### File-based Sources: configurable amount of data read for schema inference +Currently, Airbyte chooses the amount of data that we'll use to infer the schema for file-based sources. +We will be surfacing a config option for users to choose how much data to read to infer the schema. + +This option is already available for the MongoDB source. + +### Unwrapping the data at schemaless Destinations +MongoDB and file storage systems also don't require a schema at the destination. +For this reason, if you are syncing data from a schemaless source to a schemaless destination and chose the "schemaless" schema option, Airbyte will offer the ability to "unwrap" the data at the destination so that it is not nested under the "data" key. + +### Column exclusion for schemaless schemas +We are planning to offer a way to exclude fields from being synced when the schemaless option is selected, as column selection is not applicable. diff --git a/docs/understanding-airbyte/tech-stack.md b/docs/understanding-airbyte/tech-stack.md index ba69157075e6..c21aea1b8dd8 100644 --- a/docs/understanding-airbyte/tech-stack.md +++ b/docs/understanding-airbyte/tech-stack.md @@ -3,7 +3,7 @@ ## Airbyte Core Backend * [Java 17](https://jdk.java.net/archive/) -* Framework: [Jersey](https://eclipse-ee4j.github.io/jersey/) +* Framework: [Micronaut](https://micronaut.io/) * API: [OAS3](https://www.openapis.org/) * Databases: [PostgreSQL](https://www.postgresql.org/) * Unit & E2E testing: [JUnit 5](https://junit.org/junit5) @@ -13,12 +13,12 @@ Connectors can be written in any language. However the most common languages are: -* Python 3.9.0 +* Python 3.9 or higher * [Java 17](https://jdk.java.net/archive/) ## **Frontend** -* [Node.js 16](https://nodejs.org/en/) +* [Node.js](https://nodejs.org/en/) * [TypeScript](https://www.typescriptlang.org/) * Web Framework/Library: [React](https://reactjs.org/) @@ -27,7 +27,7 @@ Connectors can be written in any language. However the most common languages are * CI/CD: [GitHub Actions](https://github.com/features/actions) * Containerization: [Docker](https://www.docker.com/) and [Docker Compose](https://docs.docker.com/compose/) * Linter \(Frontend\): [ESLint](https://eslint.org/) -* Formatter \(Frontend\): [Prettier](https://prettier.io/) +* Formatter \(Frontend & Backend\): [Prettier](https://prettier.io/) * Formatter \(Backend\): [Spotless](https://github.com/diffplug/spotless) ## FAQ diff --git a/docs/understanding-airbyte/typing-deduping.md b/docs/understanding-airbyte/typing-deduping.md deleted file mode 100644 index 257bca566884..000000000000 --- a/docs/understanding-airbyte/typing-deduping.md +++ /dev/null @@ -1,90 +0,0 @@ -# Typing and Deduping - -This page refers to new functionality currently available in **early access**. Typing and deduping will become the new default method of transforming datasets within data warehouse and database destinations after they've been replicated. This functionality is going live with [Destinations V2](/release_notes/upgrading_to_destinations_v2/), which is now in early access for BigQuery. - -You will eventually be required to upgrade your connections to use the new destination versions. We are building tools for you to copy your connector’s configuration to a new version to make testing new destinations easier. These will be available in the next few weeks. - -## What is Destinations V2? - -At launch, [Airbyte Destinations V2](/release_notes/upgrading_to_destinations_v2) will provide: - -- One-to-one table mapping: Data in one stream will always be mapped to one table in your data warehouse. No more sub-tables. -- Improved per-row error handling with `_airbyte_meta`: Airbyte will now populate typing errors in the `_airbyte_meta` column instead of failing your sync. You can query these results to audit misformatted or unexpected data. -- Internal Airbyte tables in the `airbyte_internal` schema: Airbyte will now generate all raw tables in the `airbyte_internal` schema. We no longer clutter your desired schema with raw data tables. -- Incremental delivery for large syncs: Data will be incrementally delivered to your final tables when possible. No more waiting hours to see the first rows in your destination table. - -## `_airbyte_meta` Errors - -"Per-row error handling" is a new paradigm for Airbyte which provides greater flexibility for our users. Airbyte now separates `data-moving problems` from `data-content problems`. Prior to Destinations V2, both types of errors were handled the same way: by failing the sync. Now, a failing sync means that Airbyte could not _move_ all of your data. You can query the `_airbyte_meta` column to see which rows failed for _content_ reasons, and why. This is a more flexible approach, as you can now decide how to handle rows with errors on a case-by-case basis. - -:::tip -When using data downstream from Airbyte, we generally recommend you only include rows which do not have an error, e.g: - -```sql --- postgres syntax -SELECT COUNT(*) FROM _table_ WHERE json_array_length(_airbyte_meta ->> errors) = 0 -``` - -::: - -The types of errors which will be stored in `_airbyte_meta.errors` include: - -- **Typing errors**: the source declared that the type of the column `id` should be an integer, but a string value was returned. -- **Size errors**: the source returned content which cannot be stored within this this row or column (e.g. [a Redshift Super column has a 16mb limit](https://docs.aws.amazon.com/redshift/latest/dg/limitations-super.html)). - -Depending on your use-case, it may still be valuable to consider rows with errors, especially for aggregations. For example, you may have a table `user_reviews`, and you would like to know the count of new reviews received today. You can choose to include reviews regardless of whether your data warehouse had difficulty storing the full contents of the `message` column. For this use case, `SELECT COUNT(*) from user_reviews WHERE DATE(created_at) = DATE(NOW())` is still valid. - -## Destinations V2 Example - -Consider the following [source schema](https://docs.airbyte.com/integrations/sources/faker) for stream `users`: - -```json -{ - "id": "number", - "first_name": "string", - "age": "number", - "address": { - "city": "string", - "zip": "string" - } -} -``` - -The data from one stream will now be mapped to one table in your schema as below: - -#### Destination Table Name: _public.users_ - -| _(note, not in actual table)_ | \_airbyte_raw_id | \_airbyte_extracted_at | \_airbyte_meta | id | first_name | age | address | -| -------------------------------------------- | ---------------- | ---------------------- | ------------------------------------------------------------ | --- | ---------- | ---- | --------------------------------------- | -| Successful typing and de-duping ⟶ | xxx-xxx-xxx | 2022-01-01 12:00:00 | {} | 1 | sarah | 39 | { city: “San Francisco”, zip: “94131” } | -| Failed typing that didn’t break other rows ⟶ | yyy-yyy-yyy | 2022-01-01 12:00:00 | { errors: {[“fish” is not a valid integer for column “age”]} | 2 | evan | NULL | { city: “Menlo Park”, zip: “94002” } | -| Not-yet-typed ⟶ | | | | | | | | - -In legacy normalization, columns of [Airbyte type](https://docs.airbyte.com/understanding-airbyte/supported-data-types/#the-types) `Object` in the Destination were "unnested" into separate tables. In this example, with Destinations V2, the previously unnested `public.users_address` table with columns `city` and `zip` will no longer be generated. - -#### Destination Table Name: _airbyte.raw_public_users_ (`airbyte.{namespace}_{stream}`) - -| _(note, not in actual table)_ | \_airbyte_raw_id | \_airbyte_data | \_airbyte_loaded_at | \_airbyte_extracted_at | -| -------------------------------------------- | ---------------- | ----------------------------------------------------------------------------------------- | -------------------- | ---------------------- | -| Successful typing and de-duping ⟶ | xxx-xxx-xxx | { id: 1, first_name: “sarah”, age: 39, address: { city: “San Francisco”, zip: “94131” } } | 2022-01-01 12:00:001 | 2022-01-01 12:00:00 | -| Failed typing that didn’t break other rows ⟶ | yyy-yyy-yyy | { id: 2, first_name: “evan”, age: “fish”, address: { city: “Menlo Park”, zip: “94002” } } | 2022-01-01 12:00:001 | 2022-01-01 12:00:00 | -| Not-yet-typed ⟶ | zzz-zzz-zzz | { id: 3, first_name: “edward”, age: 35, address: { city: “Sunnyvale”, zip: “94003” } } | NULL | 2022-01-01 13:00:00 | - -You also now see the following changes in Airbyte-provided columns: - -![Airbyte Destinations V2 Column Changes](../release_notes/assets/destinations-v2-column-changes.png) - -## Participating in Early Access - -You can start using Destinations V2 for BigQuery or Snowflake in early access by following the below instructions: - -1. **Upgrade your Destination**: If you are using Airbyte Open Source, update your destination version to the latest version. If you are a Cloud customer, this step will already be completed on your behalf. -2. **Enabling Destinations V2**: Create a new destination, and enable the Destinations V2 option under `Advanced` settings. You will need your data warehouse credentials for this step. For this early release, we ask that you enable Destinations V2 on a new destination using new connections. When Destinations V2 is fully available, there will be additional migration paths for upgrading your destination without resetting any of your existing connections. - 1. If your previous BigQuery destination is using “GCS Staging”, you can reuse the same staging bucket. - 2. Do not enable Destinations V2 on your previous / existing destinations during early release. It will cause your existing connections to fail. -3. **Create a New Connection**: Create connections using the new BigQuery destination. These will automatically use Destinations V2. - 1. If your new destination has the same default namespace, you may want to add a stream prefix to avoid collisions in the final tables. - 2. Do not modify the ‘Transformation’ settings. These will be ignored. -4. **Monitor your Sync**: Wait at least 20 minutes, or until your sync is complete. Verify the data in your destination is correct. Congratulations, you have successfully upgraded your connection to Destinations V2! - -Once you’ve completed the setup for Destinations V2, we ask that you pay special attention to the data delivered in your destination. Let us know immediately if you see any unexpected data: table and column name changes, missing columns, or columns with incorrect types. diff --git a/docs/understanding-airbyte/basic-normalization.md b/docs/using-airbyte/core-concepts/basic-normalization.md similarity index 91% rename from docs/understanding-airbyte/basic-normalization.md rename to docs/using-airbyte/core-concepts/basic-normalization.md index bbebf577fdd7..16de09002ecc 100644 --- a/docs/understanding-airbyte/basic-normalization.md +++ b/docs/using-airbyte/core-concepts/basic-normalization.md @@ -1,5 +1,15 @@ +--- +products: all +--- + # Basic Normalization +:::danger + +Basic normalization is being removed in favor of [Typing and Deduping](typing-deduping.md), as part of [Destinations V2](/release_notes/upgrading_to_destinations_v2). This pages remains as a guide for legacy connectors. + +::: + ## High-Level Overview :::info @@ -8,10 +18,23 @@ The high-level overview contains all the information you need to use Basic Norma ::: -When you run your first Airbyte sync without the basic normalization, you'll notice that your data gets written to your destination as one data column with a JSON blob that contains all of your data. This is the `_airbyte_raw_` table that you may have seen before. Why do we create this table? A core tenet of ELT philosophy is that data should be untouched as it moves through the E and L stages so that the raw data is always accessible. If an unmodified version of the data exists in the destination, it can be retransformed without needing to sync data again. +For every connection, you can choose between two options: + +- Basic Normalization: Airbyte converts the raw JSON blob version of your data to the format of your destination. _Note: Not all destinations support normalization._ +- Raw data (no normalization): Airbyte places the JSON blob version of your data in a table called `_airbyte_raw_` + +When basic normalization is enabled, Airbyte transforms data after the sync in a step called `Basic Normalization`, which structures data from the source into a format appropriate for consumption in the destination. For example, when writing data from a nested, dynamically typed source like a JSON API to a relational destination like Postgres, normalization is the process which un-nests JSON from the source into a relational table format which uses the appropriate column types in the destination. + +Without basic normalization, your data will be written to your destination as one data column with a JSON blob that contains all of your data. This is the `_airbyte_raw_` table that you may have seen before. Why do we create this table? A core tenet of ELT philosophy is that data should be untouched as it moves through the E and L stages so that the raw data is always accessible. If an unmodified version of the data exists in the destination, it can be retransformed without needing to sync data again. If you have Basic Normalization enabled, Airbyte automatically uses this JSON blob to create a schema and tables with your data in mind, converting it to the format of your destination. This runs after your sync and may take a long time if you have a large amount of data synced. If you don't enable Basic Normalization, you'll have to transform the JSON data from that column yourself. +:::note + +Typing and Deduping may cause an increase in your destination's compute cost. This cost will vary depending on the amount of data that is transformed and is not related to Airbyte credit usage. + +::: + ## Example Basic Normalization uses a fixed set of rules to map a json object from a source to the types and format that are native to the destination. For example if a source emits data that looks like this: @@ -72,7 +95,7 @@ Additional metadata columns can be added on some tables depending on the usage: - On de-duplicated (and SCD) tables: - `_airbyte_unique_key`: hash of primary keys used to de-duplicate the final table. -The [normalization rules](basic-normalization.md#Rules) are _not_ configurable. They are designed to pick a reasonable set of defaults to hit the 80/20 rule of data normalization. We respect that normalization is a detail-oriented problem and that with a fixed set of rules, we cannot normalize your data in such a way that covers all use cases. If this feature does not meet your normalization needs, we always put the full json blob in destination as well, so that you can parse that object however best meets your use case. We will be adding more advanced normalization functionality shortly. Airbyte is focused on the EL of ELT. If you need a really featureful tool for the transformations then, we suggest trying out dbt. +The [normalization rules](#Rules) are _not_ configurable. They are designed to pick a reasonable set of defaults to hit the 80/20 rule of data normalization. We respect that normalization is a detail-oriented problem and that with a fixed set of rules, we cannot normalize your data in such a way that covers all use cases. If this feature does not meet your normalization needs, we always put the full json blob in destination as well, so that you can parse that object however best meets your use case. We will be adding more advanced normalization functionality shortly. Airbyte is focused on the EL of ELT. If you need a really featureful tool for the transformations then, we suggest trying out dbt. Airbyte places the json blob version of your data in a table called `_airbyte_raw_`. If basic normalization is turned on, it will place a separate copy of the data in a table called ``. Under the hood, Airbyte is using dbt, which means that the data only ingresses into the data store one time. The normalization happens as a query within the datastore. This implementation avoids extra network time and costs. @@ -88,7 +111,7 @@ Airbyte runs this step before handing the final data over to other tools that wi To summarize, we can represent the ELT process in the diagram below. These are steps that happens between your "Source Database or API" and the final "Replicated Tables" with examples of implementation underneath: -![](../.gitbook/assets/connecting-EL-with-T-4.png) +![](../../.gitbook/assets/connecting-EL-with-T-4.png) In Airbyte, the current normalization option is implemented using a dbt Transformer composed of: @@ -97,14 +120,14 @@ In Airbyte, the current normalization option is implemented using a dbt Transfor ## Destinations that Support Basic Normalization -- [BigQuery](../integrations/destinations/bigquery.md) -- [MS Server SQL](../integrations/destinations/mssql.md) -- [MySQL](../integrations/destinations/mysql.md) +- [BigQuery](../../integrations/destinations/bigquery.md) +- [MS Server SQL](../../integrations/destinations/mssql.md) +- [MySQL](../../integrations/destinations/mysql.md) - The server must support the `WITH` keyword. - Require MySQL >= 8.0, or MariaDB >= 10.2.1. -- [Postgres](../integrations/destinations/postgres.md) -- [Redshift](../integrations/destinations/redshift.md) -- [Snowflake](../integrations/destinations/snowflake.md) +- [Postgres](../../integrations/destinations/postgres.md) +- [Redshift](../../integrations/destinations/redshift.md) +- [Snowflake](../../integrations/destinations/snowflake.md) Basic Normalization can be configured when you're creating the connection between your Connection Setup and after in the Transformation Tab. Select the option: **Normalized tabular data**. @@ -125,8 +148,8 @@ Airbyte uses the types described in the catalog to determine the correct type fo | `bit` | boolean | | | `boolean` | boolean | | | `string` with format label `date-time` | timestamp with timezone | | -| `array` | new table | see [nesting](basic-normalization.md#Nesting) | -| `object` | new table | see [nesting](basic-normalization.md#Nesting) | +| `array` | new table | see [nesting](#Nesting) | +| `object` | new table | see [nesting](#Nesting) | ### Nesting @@ -320,11 +343,11 @@ As mentioned in the overview: To enable basic normalization \(which is optional\), you can toggle it on or disable it in the "Normalization and Transformation" section when setting up your connection: -![](../.gitbook/assets/basic-normalization-configuration.png) +![](../../.gitbook/assets/basic-normalization-configuration.png) ## Incremental runs -When the source is configured with sync modes compatible with incremental transformations (using append on destination) such as ( [full_refresh_append](connections/full-refresh-append.md), [incremental append](connections/incremental-append.md) or [incremental deduped history](connections/incremental-append-deduped.md)), only rows that have changed in the source are transferred over the network and written by the destination connector. +When the source is configured with sync modes compatible with incremental transformations (using append on destination) such as ( [full_refresh_append](./sync-modes/full-refresh-append.md), [incremental append](./sync-modes/incremental-append.md) or [incremental deduped history](./sync-modes/incremental-append-deduped.md)), only rows that have changed in the source are transferred over the network and written by the destination connector. Normalization will then try to build the normalized tables incrementally as the rows in the raw tables that have been created or updated since the last time dbt ran. As such, on each dbt run, the models get built incrementally. This limits the amount of data that needs to be transformed, vastly reducing the runtime of the transformations. This improves warehouse performance and reduces compute costs. Because normalization can be either run incrementally and, or, in full refresh, a technical column `_airbyte_normalized_at` can serve to track when was the last time a record has been transformed and written by normalization. This may greatly diverge from the `_airbyte_emitted_at` value as the normalized tables could be totally re-built at a latter time from the data stored in the `_airbyte_raw` tables. @@ -336,15 +359,15 @@ Normalization produces tables that are partitioned, clustered, sorted or indexed In general, normalization needs to do lookup on the last emitted_at column to know if a record is freshly produced and need to be incrementally processed or not. But in certain models, such as SCD tables for example, we also need to retrieve older data to update their type 2 SCD end_date and active_row flags, thus a different partitioning scheme is used to optimize that use case. -On Postgres destination, an additional table suffixed with `_stg` for every stream replicated in [incremental deduped history](connections/incremental-append-deduped.md) needs to be persisted (in a different staging schema) for incremental transformations to work because of a [limitation](https://github.com/dbt-labs/docs.getdbt.com/issues/335#issuecomment-694199569). +On Postgres destination, an additional table suffixed with `_stg` for every stream replicated in [incremental deduped history](./sync-modes/incremental-append-deduped.md) needs to be persisted (in a different staging schema) for incremental transformations to work because of a [limitation](https://github.com/dbt-labs/docs.getdbt.com/issues/335#issuecomment-694199569). ## Extending Basic Normalization Note that all the choices made by Normalization as described in this documentation page in terms of naming (and more) could be overridden by your own custom choices. To do so, you can follow the following tutorials: -- to build a [custom SQL view](../operator-guides/transformation-and-normalization/transformations-with-sql.md) with your own naming conventions -- to export, edit and run [custom dbt normalization](../operator-guides/transformation-and-normalization/transformations-with-dbt.md) yourself -- or further, you can configure the use of a custom dbt project within Airbyte by following [this guide](../operator-guides/transformation-and-normalization/transformations-with-airbyte.md). +- to build a [custom SQL view](../../operator-guides/transformation-and-normalization/transformations-with-sql.md) with your own naming conventions +- to export, edit and run [custom dbt normalization](../../operator-guides/transformation-and-normalization/transformations-with-dbt.md) yourself +- or further, you can configure the use of a custom dbt project within Airbyte by following [this guide](../../operator-guides/transformation-and-normalization/transformations-with-airbyte.md). ## CHANGELOG diff --git a/docs/using-airbyte/core-concepts/namespaces.md b/docs/using-airbyte/core-concepts/namespaces.md new file mode 100644 index 000000000000..ce7c8532d91e --- /dev/null +++ b/docs/using-airbyte/core-concepts/namespaces.md @@ -0,0 +1,98 @@ +--- +products: all +--- + +# Namespaces + +Namespaces are used to generally organize data, separate tests and production data, and enforce permissions. In most cases, namespaces are schemas in the database you're replicating to. + +As a part of connection setup, you select where in the destination you want to write your data. Note: The default configuration is **Destination default**. + +| Destination Namespace | Description | +| ---------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | +| Destination default | All streams will be replicated to the single default namespace defined in the Destination's settings. | +| Mirror source structure | Some sources (for example, databases) provide namespace information for a stream. If a source provides namespace information, the destination will mirror the same namespace when this configuration is set. For sources or streams where the source namespace is not known, the behavior will default to the "Destination default" option. | +| Custom format | All streams will be replicated to a single user-defined namespace. See Custom format for more details | + +Most of our destinations support this feature. To learn if your connector supports this, head to the individual connector page to learn more. If your desired destination doesn't support it, you can ignore this feature. + +## What is a Namespace? + +Systems often group their underlying data into namespaces with each namespace's data isolated from another namespace. This isolation allows for better organisation and flexibility, leading to better usability. + +An example of a namespace is the RDMS's `schema` concept. Some common use cases for schemas are enforcing permissions, segregating test and production data and general data organisation. + +In a source, the namespace is the location from where the data is replicated to the destination. In a destination, the namespace is the location where the replicated data is stored in the destination. + +Airbyte supports namespaces and allows Sources to define namespaces, and Destinations to write to various namespaces. In Airbyte, the following options are available and are set on each individual connection. + +### Destination default + +All streams will be replicated and stored in the default namespace defined on the destination settings page, which is typically defined when the destination was set up. Depending on your destination, the namespace refers to: + +| Destination Connector | Namespace setting | +| :--- | :--- | +| BigQuery | dataset | +| MSSQL | schema | +| MySql | database | +| Oracle DB | schema | +| Postgres | schema | +| Redshift | schema | +| Snowflake | schema | +| S3 | path prefix | + +:::tip +If you prefer to replicate multiple sources into the same namespace, use the `Stream Prefix` configuration to differentiate data from these sources to ensure no streams collide when writing to the destination. +::: + +### Mirror source structure + +Some sources \(such as databases based on JDBC\) provide namespace information from which a stream has been extracted. Whenever a source is able to fill this field in the catalog.json file, the destination will try to write to exactly the same namespace when this configuration is set. For sources or streams where the source namespace is not known, the behavior will fall back to the "Destination default". Most APIs do not provide namespace information. + +### Custom format + +When replicating multiple sources into the same destination, you may create table conflicts where tables are overwritten by different syncs. This is where using a custom namespace will ensure data is synced accurately. + +For example, a Github source can be replicated into a `github` schema. However, you may have multiple connections writing from different GitHub repositories \(common in multi-tenant scenarios\). + +:::tip +To write more than 1 table with the same name to your destination, Airbyte recommends writing the connections to unique namespaces to avoid mixing data from the different GitHub repositories. +::: + +You can enter plain text (most common) or additionally add a dynamic parameter `${SOURCE_NAMESPACE}`, which uses the namespace provided by the source if available. + +### Examples + +:::info +If the Source does not support namespaces, the data will be replicated into the Destination's default namespace. If the Destination does not support namespaces, any preference set in the connection is ignored. +::: + +The following table summarises how this works. In this example, we're looking at the replication configuration between a Postgres Source and Snowflake Destination \(with settings of schema = "my\_schema"\): + +| Namespace Configuration | Source Namespace | Source Table Name | Destination Namespace | Destination Table Name | +| :--- | :--- | :--- | :--- | :--- | +| Destination default | public | my\_table | my\_schema | my\_table | +| Destination default | | my\_table | my\_schema | my\_table | +| Mirror source structure | public | my\_table | public | my\_table | +| Mirror source structure | | my\_table | my\_schema | my\_table | +| Custom format = "custom" | public | my\_table | custom | my\_table | +| Custom format = `"${SOURCE\_NAMESPACE}"` | public | my\_table | public | my\_table | +| Custom format = `"my\_${SOURCE\_NAMESPACE}\_schema"` | public | my\_table | my\_public\_schema | my\_table | +| Custom format = " " | public | my\_table | my\_schema | my\_table | + +## Using Namespaces with Basic Normalization + +As part of the connection settings, it is possible to configure the namespace used by: 1. destination connectors: to store the `_airbyte_raw_*` tables. 2. basic normalization: to store the final normalized tables. + +When basic normalization is enabled, this is the location that both your normalized and raw data will get written to. Your raw data will show up with the prefix `_airbyte_raw_` in the namespace you define. If you don't enable basic normalization, you will only receive the raw tables. + +:::note +Note custom transformation outputs are not affected by the namespace settings from Airbyte: It is up to the configuration of the custom dbt project, and how it is written to handle its [custom schemas](https://docs.getdbt.com/docs/building-a-dbt-project/building-models/using-custom-schemas). The default target schema for dbt in this case, will always be the destination namespace. +::: + +## Requirements + +* Both Source and Destination connectors need to support namespaces. +* Relevant Source and Destination connectors need to be at least version `0.3.0` or later. +* Airbyte version `0.21.0-alpha` or later. + diff --git a/docs/using-airbyte/core-concepts/readme.md b/docs/using-airbyte/core-concepts/readme.md new file mode 100644 index 000000000000..dab232b3bb84 --- /dev/null +++ b/docs/using-airbyte/core-concepts/readme.md @@ -0,0 +1,112 @@ +--- +products: all +--- + +# Core Concepts + +Airbyte enables you to build data pipelines and replicate data from a source to a destination. You can configure how frequently the data is synced, what data is replicated, and how the data is written to in the destination. + +This page describes the concepts you need to know to use Airbyte. + +## Source + +A source is an API, file, database, or data warehouse that you want to ingest data from. + +## Destination + +A destination is a data warehouse, data lake, database, or an analytics tool where you want to load your ingested data. + +## Connector + +An Airbyte component which pulls data from a source or pushes data to a destination. + +## Connection + +A connection is an automated data pipeline that replicates data from a source to a destination. Setting up a connection enables configuration of the following parameters: + +| Concept | Description | +|---------------------|---------------------------------------------------------------------------------------------------------------------| +| [Replication Frequency](/using-airbyte/core-concepts/sync-schedules.md) | When should a data sync be triggered? | +| [Destination Namespace and Stream Prefix](/using-airbyte/core-concepts/namespaces.md) | Where should the replicated data be written? | +| [Sync Mode](/using-airbyte/core-concepts/sync-modes/README.md) | How should the streams be replicated (read and written)? | +| [Schema Propagation](/cloud/managing-airbyte-cloud/manage-schema-changes.md) | How should Airbyte handle schema drift in sources? | +| [Catalog Selection](/cloud/managing-airbyte-cloud/configuring-connections.md#modify-streams-in-your-connection) | What data should be replicated from the source to the destination? | + +## Stream + +A stream is a group of related records. + +Examples of streams: + +- A table in a relational database +- A resource or API endpoint for a REST API +- The records from a directory containing many files in a filesystem + +## Field + +A field is an attribute of a record in a stream. + +Examples of fields: + +- A column in the table in a relational database +- A field in an API response + +## Sync Schedules + +There are three options for scheduling a sync to run: +- Scheduled (ie. every 24 hours, every 2 hours) +- [CRON schedule](https://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html) +- Manual \(i.e: clicking the "Sync Now" button in the UI or through the API\) + +For more details, see our [Sync Schedules documentation](sync-schedules.md). + +## Destination Namespace + +A namespace defines where the data will be written to your destination. You can use the namespace to group streams in a source or destination. In a relational database system, this is typically known as a schema. + +For more details, see our [Namespace documentation](namespaces.md). + +## Sync Mode + +A sync mode governs how Airbyte reads from a source and writes to a destination. Airbyte provides different sync modes depending on what you want to accomplish. + +Read more about each [sync mode](using-airbyte/core-concepts/sync-modes) and how they differ. + +## Typing and Deduping + +Typing and deduping ensures the data emitted from sources is written into the correct type-cast relational columns and only contains unique records. Typing and deduping is only relevant for the following relational database & warehouse destinations: + +- Snowflake +- BigQuery + +:::info +Typing and Deduping is the default method of transforming datasets within data warehouse and database destinations after they've been replicated. We are retaining documentation about normalization to support legacy destinations. +::: + +For more details, see our [Typing & Deduping documentation](/understanding-airbyte/typing-deduping). + +## Basic Normalization + +Basic Normalization transforms data after a sync to denest columns into their own tables. Note that normalization is only available for the following relational database & warehouse destinations: + +- Redshift +- Postgres +- Oracle +- MySQL +- MSSQL + +For more details, see our [Basic Normalization documentation](/using-airbyte/core-concepts/basic-normalization.md). + +## Custom Transformations + +Airbyte integrates natively with dbt to allow you to use dbt for post-sync transformations. This is useful if you would like to trigger dbt models after a sync successfully completes. + +For more details, see our [dbt integration documentation](/cloud/managing-airbyte-cloud/dbt-cloud-integration.md). + +## Workspace + +A workspace is a grouping of sources, destinations, connections, and other configurations. It lets you collaborate with team members and share resources across your team under a shared billing account. + +## Glossary of Terms + +You can find a extended list of [Airbyte specific terms](https://glossary.airbyte.com/term/airbyte-glossary-of-terms/), [data engineering concepts](https://glossary.airbyte.com/term/data-engineering-concepts) or many [other data related terms](https://glossary.airbyte.com/). diff --git a/docs/using-airbyte/core-concepts/sync-modes/README.md b/docs/using-airbyte/core-concepts/sync-modes/README.md new file mode 100644 index 000000000000..be548d4d04d2 --- /dev/null +++ b/docs/using-airbyte/core-concepts/sync-modes/README.md @@ -0,0 +1,24 @@ +--- +products: all +--- + +# Sync Modes + +A sync mode governs how Airbyte reads from a source and writes to a destination. Airbyte provides different sync modes to account for various use cases. To minimize confusion, a mode's behavior is reflected in its name. The easiest way to understand Airbyte's sync modes is to understand how the modes are named. + +1. The first part of the name denotes how the source connector reads data from the source: + 1. Incremental: Read records added to the source since the last sync job. \(The first sync using Incremental is equivalent to a Full Refresh\) + - Method 1: Using a cursor. Generally supported by all connectors whose data source allows extracting records incrementally. + - Method 2: Using change data capture. Only supported by some sources. See [CDC](../../../understanding-airbyte/cdc.md) for more info. + 2. Full Refresh: Read everything in the source. +2. The second part of the sync mode name denotes how the destination connector writes data. This is not affected by how the source connector produced the data: + 1. Overwrite: Overwrite by first deleting existing data in the destination. + 2. Append: Write by adding data to existing tables in the destination. + 3. Deduped History: Write by first adding data to existing tables in the destination to keep a history of changes. The final table is produced by de-duplicating the intermediate ones using a primary key. + +A sync mode is a combination of a source and destination mode together. The UI exposes the following options, whenever both source and destination connectors are capable to support it for the corresponding stream: + +- [Incremental Append + Deduped](./incremental-append-deduped.md): Sync new records from stream and append data in destination, also provides a de-duplicated view mirroring the state of the stream in the source. +- [Full Refresh Overwrite](./full-refresh-overwrite.md): Sync the whole stream and replace data in destination by overwriting it. +- [Full Refresh Append](./full-refresh-append.md): Sync the whole stream and append data in destination. +- [Incremental Append](./incremental-append.md): Sync new records from stream and append data in destination. diff --git a/docs/understanding-airbyte/connections/full-refresh-append.md b/docs/using-airbyte/core-concepts/sync-modes/full-refresh-append.md similarity index 91% rename from docs/understanding-airbyte/connections/full-refresh-append.md rename to docs/using-airbyte/core-concepts/sync-modes/full-refresh-append.md index b7343fc1c07b..1bdd03f8ddee 100644 --- a/docs/understanding-airbyte/connections/full-refresh-append.md +++ b/docs/using-airbyte/core-concepts/sync-modes/full-refresh-append.md @@ -1,8 +1,12 @@ +--- +products: all +--- + # Full Refresh - Append ## Overview -The **Full Refresh** modes are the simplest methods that Airbyte uses to sync data, as they always retrieve all available data requested from the source, regardless of whether it has been synced before. This contrasts with [**Incremental sync**](incremental-append.md), which does not sync data that has already been synced before. +The **Full Refresh** modes are the simplest methods that Airbyte uses to sync data, as they always retrieve all available data requested from the source, regardless of whether it has been synced before. This contrasts with [**Incremental sync**](./incremental-append.md), which does not sync data that has already been synced before. In the **Append** variant, new syncs will take all data from the sync and append it to the destination table. Therefore, if syncing similar information multiple times, every sync will create duplicates of already existing data. diff --git a/docs/using-airbyte/core-concepts/sync-modes/full-refresh-overwrite.md b/docs/using-airbyte/core-concepts/sync-modes/full-refresh-overwrite.md new file mode 100644 index 000000000000..17204cafcd59 --- /dev/null +++ b/docs/using-airbyte/core-concepts/sync-modes/full-refresh-overwrite.md @@ -0,0 +1,50 @@ +--- +products: all +--- + +# Full Refresh - Overwrite + +## Overview + +The **Full Refresh** modes are the simplest methods that Airbyte uses to sync data, as they always retrieve all available information requested from the source, regardless of whether it has been synced before. This contrasts with [**Incremental sync**](./incremental-append.md), which does not sync data that has already been synced before. + +In the **Overwrite** variant, new syncs will destroy all data in the existing destination table and then pull the new data in. Therefore, data that has been removed from the source after an old sync will be deleted in the destination table. + +## Example Behavior + +On the nth sync of a full refresh connection: + +## _Replace_ existing data with new data. The connection does not create any new tables. + +data in the destination _before_ the sync: + +| Languages | +| :--- | +| Python | +| Java | +| Bash| + +new data in the source: + +| Languages | +| :--- | +| Python | +| Java | +| Ruby | + +data in the destination _after_ the sync (note how the old value of "bash" is no longer present): + +| Languages | +| :--- | +| Python | +| Java | +| Ruby | + +## Destination-specific mechinisims for full refresh + +The mechinisim by which a destination connector acomplishes the full refresh will vary wildly from destination to destinaton. For our certified database and data warehouse destinations, we will be recreating the final table each sync. This allows us leave the previous sync's data viewable by writing to a "final-table-tmp" location as the sync is running, and at the end dropping the olf "final" table, and renaming the new one into place. That said, this may not possible for all destinations, and we may need to erase the existing data at the start of each full-refresh sync. + +## Related information + +- [An overview of Airbyte’s replication modes](https://airbyte.com/blog/understanding-data-replication-modes). +- [Explore Airbyte's full refresh data synchronization](https://airbyte.com/tutorials/full-data-synchronization). diff --git a/docs/understanding-airbyte/connections/incremental-append-deduped.md b/docs/using-airbyte/core-concepts/sync-modes/incremental-append-deduped.md similarity index 88% rename from docs/understanding-airbyte/connections/incremental-append-deduped.md rename to docs/using-airbyte/core-concepts/sync-modes/incremental-append-deduped.md index beff559938b0..7e3ad86a1b98 100644 --- a/docs/understanding-airbyte/connections/incremental-append-deduped.md +++ b/docs/using-airbyte/core-concepts/sync-modes/incremental-append-deduped.md @@ -1,3 +1,7 @@ +--- +products: all +--- + # Incremental Sync - Append + Deduped ## High-Level Context @@ -10,7 +14,7 @@ in the destination and use the most recent data. On the other hand, the [Increme Airbyte supports syncing data in **Incremental Append Deduped** mode i.e: 1. **Incremental** means syncing only replicate _new_ or _modified_ data. This prevents re-fetching data that you have already replicated from a source. If the sync is running for the first time, it is equivalent to a [Full Refresh](full-refresh-append.md) since all data will be considered as _new_. -2. **Append** means taht this incremental data is added to existing tables in your data warehouse. +2. **Append** means that this incremental data is added to existing tables in your data warehouse. 3. **Deduped** means that data in the final table will be unique per primary key \(unlike [Append modes](incremental-append.md)\). This is determined by sorting the data using the cursor field and keeping only the latest de-duplicated data row. Records in the final destination can potentially be deleted as they are de-duplicated, and if your source supports emitting deleting records (e.g. an CDC database source). You should not find multiple copies of the same primary key as these should be unique in that table. @@ -69,19 +73,19 @@ In the final de-duplicated table: ## Source-Defined Cursor -Some sources are able to determine the cursor that they use without any user input. For example, in the [exchange rates source](../../integrations/sources/exchange-rates.md), the source knows that the date field should be used to determine the last record that was synced. In these cases, simply select the incremental option in the UI. +Some sources are able to determine the cursor that they use without any user input. For example, in the [exchange rates source](../../../integrations/sources/exchange-rates.md), the source knows that the date field should be used to determine the last record that was synced. In these cases, simply select the incremental option in the UI. -![](../../.gitbook/assets/incremental_source_defined.png) +![](../../../.gitbook/assets/incremental_source_defined.png) -\(You can find a more technical details about the configuration data model [here](../airbyte-protocol.md#catalog)\). +\(You can find a more technical details about the configuration data model [here](../../../understanding-airbyte/airbyte-protocol.md#catalog)\). ## User-Defined Cursor -Some sources cannot define the cursor without user input. For example, in the [postgres source](../../integrations/sources/postgres.md), the user needs to choose which column in a database table they want to use as the `cursor field`. In these cases, select the column in the sync settings dropdown that should be used as the `cursor field`. +Some sources cannot define the cursor without user input. For example, in the [postgres source](../../../integrations/sources/postgres.md), the user needs to choose which column in a database table they want to use as the `cursor field`. In these cases, select the column in the sync settings dropdown that should be used as the `cursor field`. -![](../../.gitbook/assets/incremental_user_defined.png) +![](../../../.gitbook/assets/incremental_user_defined.png) -\(You can find a more technical details about the configuration data model [here](../airbyte-protocol.md#catalog)\). +\(You can find a more technical details about the configuration data model [here](../../../understanding-airbyte/airbyte-protocol.md#catalog)\). ## Source-Defined Primary key @@ -91,7 +95,7 @@ Some sources are able to determine the primary key that they use without any use Some sources cannot define the cursor without user input or the user may want to specify their own primary key on the destination that is different from the source definitions. In these cases, select the column in the sync settings dropdown that should be used as the `primary key` or `composite primary keys`. -![](../../.gitbook/assets/primary_key_user_defined.png) +![](../../../.gitbook/assets/primary_key_user_defined.png) In this example, we selected both the `campaigns.id` and `campaigns.name` as the composite primary key of our `campaigns` table. @@ -118,4 +122,4 @@ select * from table where cursor_field > 'last_sync_max_cursor_field_value' **Note**: -Previous versions of Airbyte destinations supported SCD tables, which would sore every entry seen for a record. This was removed with Destinations V2 and [Typing and Deduplication](/understanding-airbyte/typing-deduping.md). +Previous versions of Airbyte destinations supported SCD tables, which would sore every entry seen for a record. This was removed with Destinations V2 and [Typing and Deduplication](../typing-deduping.md). diff --git a/docs/understanding-airbyte/connections/incremental-append.md b/docs/using-airbyte/core-concepts/sync-modes/incremental-append.md similarity index 88% rename from docs/understanding-airbyte/connections/incremental-append.md rename to docs/using-airbyte/core-concepts/sync-modes/incremental-append.md index c380d2226912..3a9c01859714 100644 --- a/docs/understanding-airbyte/connections/incremental-append.md +++ b/docs/using-airbyte/core-concepts/sync-modes/incremental-append.md @@ -1,8 +1,12 @@ +--- +products: all +--- + # Incremental Sync - Append ## Overview -Airbyte supports syncing data in **Incremental Append** mode i.e: syncing only replicate _new_ or _modified_ data. This prevents re-fetching data that you have already replicated from a source. If the sync is running for the first time, it is equivalent to a [Full Refresh](full-refresh-append.md) since all data will be considered as _new_. +Airbyte supports syncing data in **Incremental Append** mode i.e: syncing only replicate _new_ or _modified_ data. This prevents re-fetching data that you have already replicated from a source. If the sync is running for the first time, it is equivalent to a [Full Refresh](./full-refresh-append.md) since all data will be considered as _new_. In this flavor of incremental, records in the warehouse destination will never be deleted or mutated. A copy of each new or updated record is _appended_ to the data in the warehouse. This means you can find multiple copies of the same record in the destination warehouse. We provide an "at least once" guarantee of replicating each record that is present when the sync runs. @@ -62,25 +66,25 @@ The output we expect to see in the warehouse is as follows: ## Source-Defined Cursor -Some sources are able to determine the cursor that they use without any user input. For example, in the [exchange rates source](../../integrations/sources/exchange-rates.md), the source knows that the date field should be used to determine the last record that was synced. In these cases, simply select the incremental option in the UI. +Some sources are able to determine the cursor that they use without any user input. For example, in the [exchange rates source](../../../integrations/sources/exchange-rates.md), the source knows that the date field should be used to determine the last record that was synced. In these cases, simply select the incremental option in the UI. -![](../../.gitbook/assets/incremental_source_defined.png) +![](../../../.gitbook/assets/incremental_source_defined.png) -\(You can find a more technical details about the configuration data model [here](../airbyte-protocol.md#catalog)\). +\(You can find a more technical details about the configuration data model [here](../../../understanding-airbyte/airbyte-protocol.md#catalog)\). ## User-Defined Cursor -Some sources cannot define the cursor without user input. For example, in the [postgres source](../../integrations/sources/postgres.md), the user needs to choose which column in a database table they want to use as the `cursor field`. In these cases, select the column in the sync settings dropdown that should be used as the `cursor field`. +Some sources cannot define the cursor without user input. For example, in the [postgres source](../../../integrations/sources/postgres.md), the user needs to choose which column in a database table they want to use as the `cursor field`. In these cases, select the column in the sync settings dropdown that should be used as the `cursor field`. -![](../../.gitbook/assets/incremental_user_defined.png) +![](../../../.gitbook/assets/incremental_user_defined.png) -\(You can find a more technical details about the configuration data model [here](../airbyte-protocol.md#catalog)\). +\(You can find a more technical details about the configuration data model [here](../../../understanding-airbyte/airbyte-protocol.md#catalog)\). ## Getting the Latest Snapshot of data As demonstrated in the examples above, with **Incremental Append,** a record which was updated in the source will be appended to the destination rather than updated in-place. This means that if data in the source uses a primary key \(e.g: `user_id` in the `users` table\), then the destination will end up having multiple records with the same primary key value. -However, some use cases require only the latest snapshot of the data. This is available by using other flavors of sync modes such as [Incremental - Append + Deduped](incremental-append-deduped.md) instead. +However, some use cases require only the latest snapshot of the data. This is available by using other flavors of sync modes such as [Incremental - Append + Deduped](./incremental-append-deduped.md) instead. Note that in **Incremental Append**, the size of the data in your warehouse increases monotonically since an updated record in the source is appended to the destination rather than updated in-place. @@ -122,7 +126,7 @@ At the end of the second incremental sync, the data warehouse would still contai Similarly, if multiple modifications are made during the same day to the same records. If the frequency of the sync is not granular enough \(for example, set for every 24h\), then intermediate modifications to the data are not going to be detected and emitted. Only the state of data at the time the sync runs will be reflected in the destination. -Those concerns could be solved by using a different incremental approach based on binary logs, Write-Ahead-Logs \(WAL\), or also called [Change Data Capture \(CDC\)](../cdc.md). +Those concerns could be solved by using a different incremental approach based on binary logs, Write-Ahead-Logs \(WAL\), or also called [Change Data Capture \(CDC\)](../../../understanding-airbyte/cdc.md). The current behavior of **Incremental** is not able to handle source schema changes yet, for example, when a column is added, renamed or deleted from an existing table etc. It is recommended to trigger a [Full refresh - Overwrite](full-refresh-overwrite.md) to correctly replicate the data to the destination with the new schema changes. diff --git a/docs/using-airbyte/core-concepts/sync-schedules.md b/docs/using-airbyte/core-concepts/sync-schedules.md new file mode 100644 index 000000000000..c4514d941396 --- /dev/null +++ b/docs/using-airbyte/core-concepts/sync-schedules.md @@ -0,0 +1,51 @@ +--- +products: all +--- + +# Sync Schedules + +For each connection, you can select between three options that allow a sync to run. The three options for `Replication Frequency` are: + +- Scheduled (e.g. every 24 hours, every 2 hours) +- Cron scheduling +- Manual + +## Sync Limitations + +* Only one sync per connection can run at a time. +* If a sync is scheduled to run before the previous sync finishes, the scheduled sync will start after the completion of the previous sync. +* Syncs can run at most every 60 minutes in Airbyte Cloud. Reach out to [Sales](https://airbyte.com/company/talk-to-sales) if you require replication more frequently than once per hour. + +:::note +For Scheduled or cron scheduled syncs, Airbyte guarantees syncs will initiate with a schedule accuracy of +/- 30 minutes. +::: + +## Scheduled syncs +When a scheduled connection is first created, a sync is executed immediately after creation. After that, a sync is run once the time since the last sync \(whether it was triggered manually or due to a schedule\) has exceeded the schedule interval. For example: + +- **October 1st, 2pm**, a user sets up a connection to sync data every 24 hours. +- **October 1st, 2:01pm**: sync job runs +- **October 2nd, 2:01pm:** 24 hours have passed since the last sync, so a sync is triggered. +- **October 2nd, 5pm**: The user manually triggers a sync from the UI +- **October 3rd, 2:01pm:** since the last sync was less than 24 hours ago, no sync is run +- **October 3rd, 5:01pm:** It has been more than 24 hours since the last sync, so a sync is run + +## Cron Scheduling +If you prefer more precision in scheduling your sync, you can also use CRON scheduling to set a specific time of day or month. + +Airbyte uses the CRON scheduler from [Quartz](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html). We recommend reading their [documentation](http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/crontrigger.html) to understand the required formatting. You can also refer to these examples: + +| Cron string | Sync Timing| +| - | - | +| 0 0 * * * ? | Every hour, at 0 minutes past the hour | +| 0 0 15 * * ? | At 15:00 every day | +| 0 0 15 * * MON,TUE | At 15:00, only on Monday and Tuesday | +| 0 0 0,2,4,6 * * ? | At 12:00 AM, 02:00 AM, 04:00 AM and 06:00 AM every day | +| 0 0 */15 * * ? | At 0 minutes past the hour, every 15 hours | + +When setting up the cron expression, you will also be asked to choose a time zone the sync will run in. + +## Manual Syncs +When the connection is set to replicate with `Manual` frequency, the sync will not automatically run. + +It can be triggered by clicking the "Sync Now" button at any time through the UI or be triggered through the API. \ No newline at end of file diff --git a/docs/using-airbyte/core-concepts/typing-deduping.md b/docs/using-airbyte/core-concepts/typing-deduping.md new file mode 100644 index 000000000000..d287fd5d75bf --- /dev/null +++ b/docs/using-airbyte/core-concepts/typing-deduping.md @@ -0,0 +1,88 @@ +--- +products: all +--- + +# Typing and Deduping + +This page refers to new functionality added by [Destinations V2](/release_notes/upgrading_to_destinations_v2/). Typing and deduping is the default method of transforming datasets within data warehouse and database destinations after they've been replicated. Please check each destination to learn if Typing and Deduping is supported. + +## What is Destinations V2? + +[Airbyte Destinations V2](/release_notes/upgrading_to_destinations_v2) provide: + +- One-to-one table mapping: Data in one stream will always be mapped to one table in your data warehouse. No more sub-tables. +- Improved per-row error handling with `_airbyte_meta`: Airbyte will now populate typing errors in the `_airbyte_meta` column instead of failing your sync. You can query these results to audit misformatted or unexpected data. +- Internal Airbyte tables in the `airbyte_internal` schema: Airbyte will now generate all raw tables in the `airbyte_internal` schema. We no longer clutter your desired schema with raw data tables. +- Incremental delivery for large syncs: Data will be incrementally delivered to your final tables when possible. No more waiting hours to see the first rows in your destination table. + +## `_airbyte_meta` Errors + +"Per-row error handling" is a new paradigm for Airbyte which provides greater flexibility for our users. Airbyte now separates `data-moving problems` from `data-content problems`. Prior to Destinations V2, both types of errors were handled the same way: by failing the sync. Now, a failing sync means that Airbyte could not _move_ all of your data. You can query the `_airbyte_meta` column to see which rows failed for _content_ reasons, and why. This is a more flexible approach, as you can now decide how to handle rows with errors on a case-by-case basis. + +:::tip +When using data downstream from Airbyte, we generally recommend you only include rows which do not have an error, e.g: + +```sql +-- postgres syntax +SELECT COUNT(*) FROM _table_ WHERE json_array_length(_airbyte_meta ->> errors) = 0 +``` + +::: + +The types of errors which will be stored in `_airbyte_meta.errors` include: + +- **Typing errors**: the source declared that the type of the column `id` should be an integer, but a string value was returned. +- **Size errors (coming soon)**: the source returned content which cannot be stored within this this row or column (e.g. [a Redshift Super column has a 16mb limit](https://docs.aws.amazon.com/redshift/latest/dg/limitations-super.html)). Destinations V2 will allow us to trim records which cannot fit into destinations, but retain the primary key(s) and cursors and include "too big" error messages. + +Depending on your use-case, it may still be valuable to consider rows with errors, especially for aggregations. For example, you may have a table `user_reviews`, and you would like to know the count of new reviews received today. You can choose to include reviews regardless of whether your data warehouse had difficulty storing the full contents of the `message` column. For this use case, `SELECT COUNT(*) from user_reviews WHERE DATE(created_at) = DATE(NOW())` is still valid. + +## Destinations V2 Example + +Consider the following [source schema](/integrations/sources/faker) for stream `users`: + +```json +{ + "id": "number", + "first_name": "string", + "age": "number", + "address": { + "city": "string", + "zip": "string" + } +} +``` + +The data from one stream will now be mapped to one table in your schema as below: + +#### Final Destination Table Name: _public.users_ + +| _(note, not in actual table)_ | \_airbyte_raw_id | \_airbyte_extracted_at | \_airbyte_meta | id | first_name | age | address | +| -------------------------------------------- | ---------------- | ---------------------- | ------------------------------------------------------------ | --- | ---------- | ---- | --------------------------------------- | +| Successful typing and de-duping ⟶ | xxx-xxx-xxx | 2022-01-01 12:00:00 | `{}` | 1 | sarah | 39 | `{ city: “San Francisco”, zip: “94131” }` | +| Failed typing that didn’t break other rows ⟶ | yyy-yyy-yyy | 2022-01-01 12:00:00 | `{ errors: {[“fish” is not a valid integer for column “age”]}` | 2 | evan | NULL | `{ city: “Menlo Park”, zip: “94002” }` | +| Not-yet-typed ⟶ | | | | | | | | + +In legacy normalization, columns of [Airbyte type](/understanding-airbyte/supported-data-types/#the-types) `Object` in the Destination were "unnested" into separate tables. In this example, with Destinations V2, the previously unnested `public.users_address` table with columns `city` and `zip` will no longer be generated. + +#### Raw Destination Table Name: _airbyte_internal.raw_public__users_ (`airbyte_internal.raw_{namespace}__{stream}`) + +| _(note, not in actual table)_ | \_airbyte_raw_id | \_airbyte_data | \_airbyte_loaded_at | \_airbyte_extracted_at | +| -------------------------------------------- | ---------------- | ----------------------------------------------------------------------------------------- | -------------------- | ---------------------- | +| Successful typing and de-duping ⟶ | xxx-xxx-xxx | `{ id: 1, first_name: “sarah”, age: 39, address: { city: “San Francisco”, zip: “94131” } }` | 2022-01-01 12:00:001 | 2022-01-01 12:00:00 | +| Failed typing that didn’t break other rows ⟶ | yyy-yyy-yyy | `{ id: 2, first_name: “evan”, age: “fish”, address: { city: “Menlo Park”, zip: “94002” } }` | 2022-01-01 12:00:001 | 2022-01-01 12:00:00 | +| Not-yet-typed ⟶ | zzz-zzz-zzz | `{ id: 3, first_name: “edward”, age: 35, address: { city: “Sunnyvale”, zip: “94003” } }` | NULL | 2022-01-01 13:00:00 | + +You also now see the following changes in Airbyte-provided columns: + +![Airbyte Destinations V2 Column Changes](../../release_notes/assets/updated_table_columns.png) + +## Loading Data Incrementally to Final Tables + +:::note + +Typing and Deduping may cause an increase in your destination's compute cost. This cost will vary depending on the amount of data that is transformed and is not related to Airbyte credit usage. Enabling loading data incrementally to final tables may further increase this cost. + +::: + +V2 destinations may include the option "Enable Loading Data Incrementally to Final Tables". When enabled your data will load into your final tables incrementally while your data is still being synced. When Disabled (the default), your data loads into your final tables once at the end of a sync. Note that this option only applies if you elect to create Final tables. + diff --git a/docs/using-airbyte/getting-started/add-a-destination.md b/docs/using-airbyte/getting-started/add-a-destination.md new file mode 100644 index 000000000000..4aa05d8970f2 --- /dev/null +++ b/docs/using-airbyte/getting-started/add-a-destination.md @@ -0,0 +1,53 @@ +--- +products: all +--- + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + +# Add a Destination + +Destinations are the data warehouses, data lakes, databases and analytics tools where you will load the data from your chosen source(s). The steps to setting up your first destination are very similar to those for [setting up a source](./add-a-source). + +Once you've signed up for Airbyte Cloud or logged in to your Airbyte Open Source deployment, click on the **Destinations** tab in the navigation bar found on the left side of the dashboard. This will take you to the list of available destinations. + +![Destination List](./assets/getting-started-destination-list.png) + +You can use the provided search bar at the top of the page, or scroll down the list to find the destination you want to replicate data from. + +:::tip +You can filter the list of destinations by support level. Airbyte connectors are categorized in two support levels, Certified and Community. See our [Connector Support Levels](/integrations/connector-support-levels.md) page for more information on this topic. +::: + + + + As an example, we'll be setting up a simple Google Sheets spreadsheet that will move data to a Google Sheet. Select **Google Sheets** from the list of destinations. This will take you to the destination setup page. + + ![Destination Page](./assets/getting-started-google-sheets-destination.png) + +:::info +Google Sheets imposes rate limits and hard limits on the amount of data it can receive. Only use Google Sheets as a destination for small, non-production use cases, as it is not designed for handling large-scale data operations. + +Read more about the [specific limitations](/integrations/destinations/google-sheets.md#limitations) in our Google Sheets documentation. +::: + + The left half of the page contains a set of fields that you will have to fill out. In the **Destination name** field, you can enter a name of your choosing to help you identify this instance of the connector. By default, this will be set to the name of the destination (i.e., `Google Sheets`). + + Authenticate into your Google account by clicking "Sign in with Google" and granting permissions to Airbyte. Because this is a simple Google Sheets destination, there is only one more required field, **Spreadsheet Link**. This is the path to your spreadsheet that can be copied directly from your browser. + + + As an example, we'll be setting up a simple JSON file that will be saved on our local system as the destination. Select **Local JSON** from the list of destinations. This will take you to the destination setup page. + + The left half of the page contains a set of fields that you will have to fill out. In the **Destination name** field, you can enter a name of your choosing to help you identify this instance of the connector. By default, this will be set to the name of the destination (i.e., `Local JSON`). + + Because this is a simple JSON file, there is only one more required field, **Destination Path**. This is the path in your local filesystem where the JSON file containing your data will be saved. In our example, if we set the path to `/my_first_destination`, the file will be saved in `/tmp/airbyte_local/my_first_destination`. + + + +Each destination will have its own set of required fields to configure during setup. You can refer to your destination's provided setup guide on the right side of the page for specific details on the nature of each field. + +:::tip +Some destinations will also have an **Optional Fields** tab located beneath the required fields. You can open this tab to view and configure any additional optional parameters that exist for the source. These fields generally grant you more fine-grained control over your data replication, but you can safely ignore them. +::: + +Once you've filled out the required fields, select **Set up destination**. A connection check will run to verify that a successful connection can be established. Now you're ready to [set up your first connection](./set-up-a-connection)! diff --git a/docs/using-airbyte/getting-started/add-a-source.md b/docs/using-airbyte/getting-started/add-a-source.md new file mode 100644 index 000000000000..15f4ce57bd61 --- /dev/null +++ b/docs/using-airbyte/getting-started/add-a-source.md @@ -0,0 +1,27 @@ +--- +products: all +--- + +# Add a Source + +Setting up a new source in Airbyte is a quick and simple process! When viewing the Airbyte UI, you'll see the main navigation bar on the left side of your screen. Click the **Sources** tab to bring up a list of all available sources. + + + + +You can use the provided search bar, or simply scroll down the list to find the source you want to replicate data from. Let's use a demo source, Faker, as an example. Clicking on the **Sample Data (Faker)** card will bring us to its setup page. + +![](./assets/getting-started-faker-source.png) + +The left half of the page contains a set of fields that you will have to fill out. In the **Source name** field, you can enter a name of your choosing to help you identify this instance of the connector. By default, this will be set to the name of the source (ie, `Sample Data (Faker)`). + +Each connector in Airbyte will have its own set of authentication methods and configurable parameters. In the case of Sample Data (Faker), you can adjust the number of records you want returned in your `Users` data, and optionally adjust additional configuration settings. You can always refer to your source's provided setup guide for specific instructions on filling out each field. + +:::info +Some sources will have an **Optional Fields** tab. You can open this tab to view and configure any additional optional parameters that exist for the souce, but you do not have to do so to successfully set up the connector. +::: + +Once you've filled out all the required fields, click on the **Set up source** button and Airbyte will run a check to verify the connection. Happy replicating! + +Can't find the connectors that you want? Try your hand at easily building one yourself using our [Connector Builder](../../connector-development/connector-builder-ui/overview.md)! + diff --git a/docs/using-airbyte/getting-started/assets/getting-started-connection-complete.png b/docs/using-airbyte/getting-started/assets/getting-started-connection-complete.png new file mode 100644 index 000000000000..f034a559f026 Binary files /dev/null and b/docs/using-airbyte/getting-started/assets/getting-started-connection-complete.png differ diff --git a/docs/using-airbyte/getting-started/assets/getting-started-connection-configuration.png b/docs/using-airbyte/getting-started/assets/getting-started-connection-configuration.png new file mode 100644 index 000000000000..92921edd1dc4 Binary files /dev/null and b/docs/using-airbyte/getting-started/assets/getting-started-connection-configuration.png differ diff --git a/docs/using-airbyte/getting-started/assets/getting-started-destination-list.png b/docs/using-airbyte/getting-started/assets/getting-started-destination-list.png new file mode 100644 index 000000000000..e79369996e0d Binary files /dev/null and b/docs/using-airbyte/getting-started/assets/getting-started-destination-list.png differ diff --git a/docs/using-airbyte/getting-started/assets/getting-started-faker-source.png b/docs/using-airbyte/getting-started/assets/getting-started-faker-source.png new file mode 100644 index 000000000000..ed8b9db12eff Binary files /dev/null and b/docs/using-airbyte/getting-started/assets/getting-started-faker-source.png differ diff --git a/docs/using-airbyte/getting-started/assets/getting-started-google-sheets-destination.png b/docs/using-airbyte/getting-started/assets/getting-started-google-sheets-destination.png new file mode 100644 index 000000000000..03c6a2aade97 Binary files /dev/null and b/docs/using-airbyte/getting-started/assets/getting-started-google-sheets-destination.png differ diff --git a/docs/using-airbyte/getting-started/assets/getting-started-source-list.png b/docs/using-airbyte/getting-started/assets/getting-started-source-list.png new file mode 100644 index 000000000000..dc6dc6c7fbd8 Binary files /dev/null and b/docs/using-airbyte/getting-started/assets/getting-started-source-list.png differ diff --git a/docs/using-airbyte/getting-started/assets/getting-started-stream-selection.png b/docs/using-airbyte/getting-started/assets/getting-started-stream-selection.png new file mode 100644 index 000000000000..fc7cc81d0ddc Binary files /dev/null and b/docs/using-airbyte/getting-started/assets/getting-started-stream-selection.png differ diff --git a/docs/using-airbyte/getting-started/readme.md b/docs/using-airbyte/getting-started/readme.md new file mode 100644 index 000000000000..72d422d7b1b5 --- /dev/null +++ b/docs/using-airbyte/getting-started/readme.md @@ -0,0 +1,46 @@ +--- +products: all +--- + +# Getting Started + +Getting started with Airbyte takes only a few steps! This page guides you through the initial steps to get started and you'll learn how to setup your first connection on the following pages. + +You have two options to run Airbyte: Use **Airbyte Cloud** (recommended) or **self-manage Airbyte** in your infrastructure. + +:::tip +If you are have already deployed Airbyte or signed up for Airbyte Cloud, jump ahead to [set up a source](./add-a-source.md). +::: + +## Sign Up for Airbyte Cloud + +To use Airbyte Cloud, [sign up](https://cloud.airbyte.io/signup) with your email address, Google login, or GitHub login. Upon signing up, you'll be taken to your workspace, which lets you collaborate with team members and share resources across your team under a shared billing account. + +Airbyte Cloud offers a 14-day free trial that begins after your first successful sync. For more details on our pricing model, see our [pricing page](https://www.airbyte.com/pricing). + +To start setting up a data pipeline, see how to [set up a source](./add-a-source.md). + + +## Deploy Airbyte (Self-Managed) + +When self-managing Airbyte, your data never leaves your premises. Get started immediately by deploying locally using Docker. + +### Self-Managed Community (Open Source) + +With Airbyte Self-Managed Community (Open Source), you can use one of the following options in your infrastructure: + +- [Local Deployment](/deploying-airbyte/local-deployment.md) (recommended when trying out Airbyte) +- [On Aws](/deploying-airbyte/on-aws-ec2.md) +- [On Azure VM Cloud Shell](/deploying-airbyte/on-azure-vm-cloud-shell.md) +- [On Digital Ocean Droplet](/deploying-airbyte/on-digitalocean-droplet.md) +- [On GCP](/deploying-airbyte/on-gcp-compute-engine.md) +- [On Kubernetes](/deploying-airbyte/on-kubernetes-via-helm.md) +- [On OCI VM](/deploying-airbyte/on-oci-vm.md) +- [On Restack](/deploying-airbyte/on-restack.md) +- [On Plural](/deploying-airbyte/on-plural.md) +- [On AWS ECS](/deploying-airbyte/on-aws-ecs.md) (Spoiler alert: it doesn't work) + +### Self-Managed Enterprise +Airbyte Self-Managed Enterprise is the best way to run Airbyte yourself. You get all 300+ pre-built connectors, data never leaves your environment, and Airbyte becomes self-serve in your organization with new tools to manage multiple users, and multiple teams using Airbyte all in one place. + +To start with Self-Managed Enterprrise, navigate to our [Enterprise setup guide](/enterprise-setup/README.md). diff --git a/docs/using-airbyte/getting-started/set-up-a-connection.md b/docs/using-airbyte/getting-started/set-up-a-connection.md new file mode 100644 index 000000000000..c27c77c16f97 --- /dev/null +++ b/docs/using-airbyte/getting-started/set-up-a-connection.md @@ -0,0 +1,77 @@ +--- +products: all +--- + +import Tabs from "@theme/Tabs"; +import TabItem from "@theme/TabItem"; + +# Set up a Connection + +Now that you've learned how to set up your first [source](./add-a-source) and [destination](./add-a-destination), it's time to finish the job by creating your very first connection! + +On the left side of your main Airbyte dashboard, select **Connections**. You will be prompted to choose which source and destination to use for this connection. For this example, we'll use the **Google Sheets** source and the destination you previously set up, either **Local JSON** or **Google Sheets**. + +## Configure the connection + +Once you've chosen your source and destination, you'll be able to configure the connection. You can refer to [this page](/cloud/managing-airbyte-cloud/configuring-connections.md) for more information on each available configuration. For this demo, we'll simply set the **Replication frequency** to a 24 hour interval and leave the other fields at their default values. + +![Connection config](./assets/getting-started-connection-configuration.png) + +:::note +By default, data will sync to the default defined in the destination. To ensure your data is synced to the correct place, see our examples for [Destination Namespace](/using-airbyte/core-concepts/namespaces.md) +::: + +Next, you can toggle which streams you want to replicate, as well as setting up the desired sync mode for each stream. For more information on the nature of each sync mode supported by Airbyte, see [this page](/using-airbyte/core-concepts/sync-modes). + +Our test data consists of three streams, which we've enabled and set to `Incremental - Append + Deduped` sync mode. + +![Stream config](./assets/getting-started-stream-selection.png) + +Click **Set up connection** to complete your first connection. Your first sync is about to begin! + +## Connection Overview + +Once you've finished setting up the connection, you will be automatically redirected to a connection overview containing all the tools you need to keep track of your connection. + +![Connection dashboard](./assets/getting-started-connection-complete.png) + +Here's a basic overview of the tabs and their use: + +1. The **Status** tab shows you an overview of your connector's sync health. +2. The **Job History** tab allows you to check the logs for each sync. If you encounter any errors or unexpected behaviors during a sync, checking the logs is always a good first step to finding the cause and solution. +3. The **Replication** tab allows you to modify the configurations you chose during the connection setup. +4. The **Transformation** tab allows you to set up a custom post-sync transformations using dbt. +4. The **Settings** tab contains additional settings, and the option to delete the connection if you no longer wish to use it. + +### Check the data from your first sync + +Once the first sync has completed, you can verify the sync has completed by checking the data in your destination. + + + + If you followed along and created your own connection using a **Google Sheets** destination, you will now see three tabs created in your Google Sheet, `products`, `users`, and `purchases`. + + + + If you followed along and created your own connection using a `Local JSON` destination, you can use this command to check the file's contents to make sure the replication worked as intended (be sure to replace YOUR_PATH with the path you chose in your destination setup, and YOUR_STREAM_NAME with the name of an actual stream you replicated): + + ```bash + cat /tmp/airbyte_local/YOUR_PATH/_airbyte_raw_YOUR_STREAM_NAME.jsonl + ``` + + You should see a list of JSON objects, each containing a unique `airbyte_ab_id`, an `emitted_at` timestamp, and `airbyte_data` containing the extracted record. + +:::tip +If you are using Airbyte on Windows with WSL2 and Docker, refer to [this guide](/integrations/locating-files-local-destination.md) to locate the replicated folder and file. +::: + + + + +## What's next? + +Congratulations on successfully setting up your first connection using Airbyte! We hope that this will be just the first step on your journey with us. We support a large, ever-growing [catalog of sources and destinations](/integrations/), and you can even [contribute your own](/connector-development/). + +If you have any questions at all, please reach out to us on [Slack](https://slack.airbyte.io/). If you would like to see a missing feature or connector added, please create an issue on our [Github](https://github.com/airbytehq/airbyte). Our community's participation is invaluable in helping us grow and improve every day, and we always welcome your feedback. + +Thank you, and we hope you enjoy using Airbyte! diff --git a/docs/using-airbyte/workspaces.md b/docs/using-airbyte/workspaces.md new file mode 100644 index 000000000000..a4dfd0d524cb --- /dev/null +++ b/docs/using-airbyte/workspaces.md @@ -0,0 +1,87 @@ +--- +products: cloud, oss-enterprise +--- + +# Manage your workspace + +A workspace in Airbyte allows you to collaborate with other users and manage connections together. On Airbyte Cloud it will allow you to share billing details for a workspace. + +:::info +Airbyte [credits](https://airbyte.com/pricing) are assigned per workspace and cannot be transferred between workspaces. +::: + +## Add users to your workspace + +To add a user to your workspace: + +1. Go to the **Settings** via the side navigation in Airbyte. + +2. Click **Access Management**. + +3. Click **+ New user**. + +4. On the **Add new users** dialog, enter the email address of the user you want to invite to your workspace. + +5. Click **Send invitation**. + + :::info + The user will have access to only the workspace you invited them to. They will be added as a workspace admin by default. + ::: + +## Remove users from your workspace​ + +To remove a user from your workspace: + +1. Go to the **Settings** via the side navigation in Airbyte. + +2. Click **Access Management**. + +3. Click **Remove** next to the user’s email. + +4. The **Remove user** dialog displays. Click **Remove**. + +## Rename a workspace + +To rename a workspace: + +1. Go to the **Settings** via the side navigation in Airbyte. + +2. Click **General Settings**. + +3. In the **Workspace name** field, enter the new name for your workspace. + +4. Click **Save changes**. + +## Delete a workspace + +To delete a workspace: + +1. Go to the **Settings** via the side navigation in Airbyte. + +2. Click **General Settings**. + +3. In the **Delete your workspace** section, click **Delete**. + +## Single workspace vs. multiple workspaces + +You can use one or multiple workspaces with Airbyte Cloud, which gives you flexibility in managing user access and billing. + +### Access +| Number of workspaces | Benefits | Considerations | +|----------------------|-------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------| +| Single | All users in a workspace have access to the same data. | If you add a user to a workspace, you cannot limit their access to specific data within that workspace. | +| Multiple | You can create multiple workspaces to allow certain users to access the data. | Since you have to manage user access for each workspace individually, it can get complicated if you have many users in multiple workspaces. | + +### Billing +| Number of workspaces | Benefits | Considerations | +|----------------------|-------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------| +| Single | You can use the same payment method for all purchases. | Credits pay for the use of resources in a workspace when you run a sync. Resource usage cannot be divided and paid for separately (for example, you cannot bill different departments in your organization for the usage of some credits in one workspace). | +| Multiple | Workspaces are independent of each other, so you can use a different payment method card for each workspace (for example, different credit cards per department in your organization). | You can use the same payment method for different workspaces, but each workspace is billed separately. Managing billing for each workspace can become complicated if you have many workspaces. | + +## Switch between multiple workspaces + +To switch between workspaces: + +1. Click the current workspace name under the Airbyte logo in the navigation bar. + +2. Search for the workspace or click the name of the workspace you want to switch to. diff --git a/docusaurus/.nvmrc b/docusaurus/.nvmrc new file mode 100644 index 000000000000..55bffd620b9a --- /dev/null +++ b/docusaurus/.nvmrc @@ -0,0 +1 @@ +18.15.0 diff --git a/docusaurus/README.md b/docusaurus/README.md index f03e15d4c705..8f6b20d23122 100644 --- a/docusaurus/README.md +++ b/docusaurus/README.md @@ -1,43 +1,80 @@ # Documentation and Docusaurus -We use [docusaurus](https://docusaurus.io) for consistent process, in `Airbyte` **no website is generated**. -Functionally this is a very fancy **linter** - -Running the build process will **check for broken links**, please read the output and address -any broken links that you are able to do. +We use [Docusaurus](https://docusaurus.io) to build Airbyte's +[documentation site](https://docs.airbyte.io) from documentation source files in Markdown, and lint +the source files. We host the resulting docs site on Vercel. It deploys automatically when any +changes get merged to `master`. ## Installation -For consistency across other Airbyte projects we use yarn (A Javascript based software package manager) +For consistency across other Airbyte projects we use `pnpm` (A Javascript based software package +manager). ```bash -brew install yarn +brew install pnpm cd docusaurus -yarn install -yarn build +pnpm install +pnpm build ``` -At this point you will see any broken links that docusaurus was able to find. +`pnpm build` will build Docusaurus site in `docusaurus/build` directory. ## Developing Locally +If you want to make changes to the documentation, you can run docusaurus locally in a way that would +listen to any source docs changes and live-reload them: + ```bash -yarn start # any changes will automatically be reflected in your browser! +pnpm start # any changes will automatically be reflected in your browser! ``` -## Making Changes +All the content for docs.airbyte.com lives in the `/docs` directory in this repo. All files are +markdown. Make changes or add new files, and you should see them in your browser! + +## Changing Navigation Structure + +If you have created any new files, be sure to add them manually to the table of contents found here +in [`sidebars.js`](https://github.com/airbytehq/airbyte/blob/master/docusaurus/sidebars.js) + +## Contributing + +We welcome documentation updates! If you'd like to contribute a change, please make sure to: -All the content for docs.airbyte.com lives in the `/docs` directory in this repo. All files are markdown. Make changes or add new files, and you should see them in your browser! +- Run `pnpm build` and check that all build steps are successful. +- Push your changes into a pull request, and follow the PR template instructions. -If you have created any new files, be sure to add them manually to the table of contents found here in this [file](https://github.com/airbytehq/airbyte/blob/master/docusaurus/sidebars.js) +When you make a pull request, Vercel will automatically build a test instance of the full docs site +and link it in the pull request for review. -## Plugin Client Redirects +### Checking for broken links + +Airbyte's docs site checks links with Docusaurus at build time, and with an additional GitHub action +periodically: + +- Running the build process will **check for broken links**, please read the output and address any + broken links that you are able to do. +- [This GitHub Action](https://github.com/airbytehq/airbyte/blob/master/.github/workflows/doc-link-check.yml) + checks all links on Airbyte production docs site, and tells us if any of them are broken. + +> [!NOTE] Docusaurus links checker only checks _relative_ links, and assumes that absolute links are +> fine. For that reason, if you're linking to another Airbyte documentation page, make it a relative +> link. I.e. `[link](/connector-development/overview.md)` instead of +> `[link](https://docs.airbyte.com/connector-development/)`. That way, if your link breaks in the +> future due to a navigation restructure, it will be caught with `pnpm build`. + +## Docusaurus Plugins We Use + +### Plugin Client Redirects A silly name, but a useful plugin that adds redirect functionality to docusaurus [Official documentation here](https://docusaurus.io/docs/api/plugins/@docusaurus/plugin-client-redirects) -You will need to edit [this docusaurus file](https://github.com/airbytehq/airbyte/blob/master/docusaurus/docusaurus.config.js#L22) +If you're proposing to move an existing documentation file or change its name, please setup a +redirect rule. + +You will need to edit +[this docusaurus file](https://github.com/airbytehq/airbyte/blob/master/docusaurus/docusaurus.config.js#L22) You will see a commented section the reads something like this @@ -48,13 +85,18 @@ You will see a commented section the reads something like this // }, ``` -Copy this section, replace the values, and [test it locally](locally_testing_docusaurus.md) by going to the -path you created a redirect for and checked to see that the address changes to your new one. +Copy this section, replace the values, and [test it locally](locally_testing_docusaurus.md) by going +to the path you created a redirect for and checked to see that the address changes to your new one. _Note:_ Your path \*_needs_ a leading slash `/` to work ## Deploying Docs -We use Github Pages for hosting this docs website, and [Docusaurus](https://docusaurus.io/) as the docs framework. Any change to the `/docs` directory you make is deployed when you merge to your PR to the master branch automagically! +Airbyte docs live on Vercel. Any change to the `/docs` directory you make is deployed when you merge +to your PR to the master branch automagically! -The source code for the docs lives in the [airbyte monorepo's `docs/` directory](https://github.com/airbytehq/airbyte/tree/master/docs). Any changes to the `/docs` directory will be tested automatically in your PR. Be sure that you wait for the tests to pass before merging! If there are CI problems publishing your docs, you can run `tools/bin/deploy_docusaurus` locally - this is the publish script that CI runs. +The source code for the docs lives in the +[airbyte monorepo's `docs/` directory](https://github.com/airbytehq/airbyte/tree/master/docs). Any +changes to the `/docs` directory will be tested automatically in your PR. Be sure that you wait for +the tests to pass before merging! If there are CI problems publishing your docs, you can run +`tools/bin/deploy_docusaurus` locally - this is the publish script that CI runs. diff --git a/docusaurus/docusaurus.config.js b/docusaurus/docusaurus.config.js index e44a6ff2b2fe..c81b245c62b3 100644 --- a/docusaurus/docusaurus.config.js +++ b/docusaurus/docusaurus.config.js @@ -1,11 +1,27 @@ // @ts-check // Note: type annotations allow type checking and IDEs autocompletion -const lightCodeTheme = require("prism-react-renderer/themes/github"); -const darkCodeTheme = require("prism-react-renderer/themes/dracula"); +const yaml = require("js-yaml"); +const fs = require("node:fs"); +const path = require("node:path"); + +const { themes } = require('prism-react-renderer'); +const lightCodeTheme = themes.github; +const darkCodeTheme = themes.dracula; + +const docsHeaderDecoration = require("./src/remark/docsHeaderDecoration"); +const productInformation = require("./src/remark/productInformation"); + +const redirects = yaml.load( + fs.readFileSync(path.join(__dirname, "redirects.yml"), "utf-8") +); /** @type {import('@docusaurus/types').Config} */ const config = { + markdown: { + mermaid: true, + }, + themes: ['@docusaurus/theme-mermaid'], title: "Airbyte Documentation", tagline: "Airbyte is an open-source data integration platform to build ELT pipelines. Consolidate your data in your data warehouses, lakes and databases.", @@ -19,74 +35,24 @@ const config = { organizationName: "airbytehq", // Usually your GitHub org/user name. projectName: "airbyte", // Usually your repo name. + // Adds one off script tags to the head of each page + // e.g. + scripts: [ + { + src: "https://cdn.unifygtm.com/tag/v1/unify-tag-script.js", + async: true, + type: "module", + id: "unifytag", + "data-api-key": "wk_BEtrdAz2_2qgdexg5KRa6YWLWVwDdieFC7CAHkDKz", + }, + ], + plugins: [ [ "@docusaurus/plugin-client-redirects", { fromExtensions: ["html", "htm"], // /myPage.html -> /myPage - redirects: [ - // /docs/oldDoc -> /docs/newDoc - { - from: "/airbyte-pro", - to: "/airbyte-enterprise" - }, - { - from: "/upgrading-airbyte", - to: "/operator-guides/upgrading-airbyte", - }, - { - from: "/catalog", - to: "/understanding-airbyte/airbyte-protocol", - }, - { - from: "/integrations/sources/google-analytics-data-api", - to: "/integrations/sources/google-analytics-v4", - }, - { - from: "/integrations/sources/appstore", - to: "/integrations/sources/appstore-singer", - }, - { - from: "/project-overview/security", - to: "/operator-guides/security", - }, - { - from: "/operator-guides/securing-airbyte", - to: "/operator-guides/security", - }, - { - from: "/connector-development/config-based/", - to: "/connector-development/config-based/low-code-cdk-overview", - }, - { - from: "/project-overview/changelog", - to: "/category/release-notes", - }, - { - from: "/connector-development/config-based/understanding-the-yaml-file/stream-slicers/", - to: "/connector-development/config-based/understanding-the-yaml-file/partition-router", - }, - { - from: "/cloud/managing-airbyte-cloud", - to: "/category/using-airbyte-cloud", - }, - { - from: "/category/managing-airbyte-cloud", - to: "/category/using-airbyte-cloud", - }, - { - from: "/category/airbyte-open-source-quick-start", - to: "/category/getting-started" - }, - { - from: "/cloud/dbt-cloud-integration", - to: "/cloud/managing-airbyte-cloud/dbt-cloud-integration", - }, - // { - // from: '/some-lame-path', - // to: '/a-much-cooler-uri', - // }, - ], + redirects: redirects, }, ], () => ({ @@ -97,15 +63,20 @@ const config = { rules: [ { test: /\.ya?ml$/, - use: 'yaml-loader' - } - ] + use: "yaml-loader", + }, + ], }, }; }, }), ], + clientModules: [ + require.resolve("./src/scripts/fontAwesomeIcons.js"), + require.resolve("./src/scripts/cloudStatus.js"), + ], + presets: [ [ "classic", @@ -118,15 +89,12 @@ const config = { editUrl: "https://github.com/airbytehq/airbyte/blob/master/docs", path: "../docs", exclude: ["**/*.inapp.md"], + remarkPlugins: [docsHeaderDecoration, productInformation], }, blog: false, theme: { customCss: require.resolve("./src/css/custom.css"), }, - gtag: { - trackingID: "UA-156258629-2", - anonymizeIP: true, - }, }), ], ], @@ -143,9 +111,9 @@ const config = { }, }, algolia: { - appId: 'OYKDBC51MU', - apiKey: '15c487fd9f7722282efd8fcb76746fce', // Public API key: it is safe to commit it - indexName: 'airbyte', + appId: "OYKDBC51MU", + apiKey: "15c487fd9f7722282efd8fcb76746fce", // Public API key: it is safe to commit it + indexName: "airbyte", }, navbar: { title: "", @@ -153,19 +121,13 @@ const config = { alt: "Simple, secure and extensible data integration", src: "img/logo-dark.png", srcDark: "img/logo-light.png", - width: 140, height: 40, }, items: [ { href: "https://airbyte.io/", position: "left", - label: "Home", - }, - { - href: "https://status.airbyte.io/", - label: "Status", - position: "left", + label: "About Airbyte", }, { href: "https://airbyte.com/tutorials", @@ -177,10 +139,24 @@ const config = { label: "Support", position: "left", }, + // --- Right side --- + { + href: "https://status.airbyte.com", + label: "Cloud Status", + className: "cloudStatusLink", + position: "right", + }, { href: "https://cloud.airbyte.io/signup?utm_campaign=22Q1_AirbyteCloudSignUpCampaign_Trial&utm_source=Docs&utm_content=NavBar", label: "Try Airbyte Cloud", - position: "left", + position: "right", + className: "header-button", + }, + { + href: "https://github.com/airbytehq", + position: "right", + "aria-label": "Airbyte on GitHub", + className: "header-github-link", }, ], }, diff --git a/docusaurus/package.json b/docusaurus/package.json index afe1ee0b013d..2f6b8fe15705 100644 --- a/docusaurus/package.json +++ b/docusaurus/package.json @@ -14,109 +14,118 @@ "write-heading-ids": "docusaurus write-heading-ids" }, "dependencies": { - "@babel/core": "7.18.6", - "@babel/helper-builder-binary-assignment-operator-visitor": "7.18.6", + "@babel/core": "7.23.6", + "@babel/helper-builder-binary-assignment-operator-visitor": "7.22.15", "@babel/helper-explode-assignable-expression": "7.18.6", - "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": "7.18.6", - "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": "7.18.6", - "@babel/plugin-proposal-async-generator-functions": "7.18.6", + "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression": "7.23.3", + "@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining": "7.23.3", + "@babel/plugin-proposal-async-generator-functions": "7.20.7", "@babel/plugin-proposal-class-properties": "7.18.6", - "@babel/plugin-proposal-class-static-block": "7.18.6", + "@babel/plugin-proposal-class-static-block": "7.21.0", "@babel/plugin-proposal-dynamic-import": "7.18.6", - "@babel/plugin-proposal-export-namespace-from": "7.18.6", + "@babel/plugin-proposal-export-namespace-from": "7.18.9", "@babel/plugin-proposal-json-strings": "7.18.6", - "@babel/plugin-proposal-logical-assignment-operators": "7.18.6", + "@babel/plugin-proposal-logical-assignment-operators": "7.20.7", "@babel/plugin-proposal-nullish-coalescing-operator": "7.18.6", "@babel/plugin-proposal-numeric-separator": "7.18.6", - "@babel/plugin-proposal-object-rest-spread": "7.18.6", + "@babel/plugin-proposal-object-rest-spread": "7.20.7", "@babel/plugin-proposal-optional-catch-binding": "7.18.6", "@babel/plugin-proposal-private-methods": "7.18.6", - "@babel/plugin-proposal-private-property-in-object": "7.18.6", - "@babel/plugin-syntax-import-assertions": "7.18.6", - "@babel/plugin-syntax-typescript": "7.18.6", - "@babel/plugin-transform-arrow-functions": "7.18.6", - "@babel/plugin-transform-async-to-generator": "7.18.6", - "@babel/plugin-transform-block-scoped-functions": "7.18.6", - "@babel/plugin-transform-block-scoping": "7.18.6", - "@babel/plugin-transform-classes": "7.18.6", - "@babel/plugin-transform-computed-properties": "7.18.6", - "@babel/plugin-transform-destructuring": "7.18.6", - "@babel/plugin-transform-duplicate-keys": "7.18.6", - "@babel/plugin-transform-exponentiation-operator": "7.18.6", - "@babel/plugin-transform-for-of": "7.18.6", - "@babel/plugin-transform-function-name": "7.18.6", - "@babel/plugin-transform-literals": "7.18.6", - "@babel/plugin-transform-member-expression-literals": "7.18.6", - "@babel/plugin-transform-modules-amd": "7.18.6", - "@babel/plugin-transform-modules-commonjs": "7.18.6", - "@babel/plugin-transform-modules-systemjs": "7.18.6", - "@babel/plugin-transform-modules-umd": "7.18.6", - "@babel/plugin-transform-named-capturing-groups-regex": "7.18.6", - "@babel/plugin-transform-new-target": "7.18.6", - "@babel/plugin-transform-object-super": "7.18.6", - "@babel/plugin-transform-property-literals": "7.18.6", - "@babel/plugin-transform-react-display-name": "7.18.6", - "@babel/plugin-transform-react-jsx-development": "7.18.6", - "@babel/plugin-transform-react-pure-annotations": "7.18.6", - "@babel/plugin-transform-regenerator": "7.18.6", - "@babel/plugin-transform-reserved-words": "7.18.6", - "@babel/plugin-transform-runtime": "7.18.6", - "@babel/plugin-transform-shorthand-properties": "7.18.6", - "@babel/plugin-transform-spread": "7.18.6", - "@babel/plugin-transform-sticky-regex": "7.18.6", - "@babel/plugin-transform-template-literals": "7.18.6", - "@babel/plugin-transform-typeof-symbol": "7.18.6", - "@babel/plugin-transform-typescript": "7.18.6", - "@babel/plugin-transform-unicode-escapes": "7.18.6", - "@babel/plugin-transform-unicode-regex": "7.18.6", - "@babel/preset-env": "7.18.6", - "@babel/preset-react": "7.18.6", - "@babel/preset-typescript": "7.18.6", - "@babel/runtime-corejs3": "7.18.6", - "@cmfcmf/docusaurus-search-local": "^0.10.0", + "@babel/plugin-proposal-private-property-in-object": "7.21.11", + "@babel/plugin-syntax-import-assertions": "7.23.3", + "@babel/plugin-syntax-typescript": "7.23.3", + "@babel/plugin-transform-arrow-functions": "7.23.3", + "@babel/plugin-transform-async-to-generator": "7.23.3", + "@babel/plugin-transform-block-scoped-functions": "7.23.3", + "@babel/plugin-transform-block-scoping": "7.23.4", + "@babel/plugin-transform-classes": "7.23.5", + "@babel/plugin-transform-computed-properties": "7.23.3", + "@babel/plugin-transform-destructuring": "7.23.3", + "@babel/plugin-transform-duplicate-keys": "7.23.3", + "@babel/plugin-transform-exponentiation-operator": "7.23.3", + "@babel/plugin-transform-for-of": "7.23.6", + "@babel/plugin-transform-function-name": "7.23.3", + "@babel/plugin-transform-literals": "7.23.3", + "@babel/plugin-transform-member-expression-literals": "7.23.3", + "@babel/plugin-transform-modules-amd": "7.23.3", + "@babel/plugin-transform-modules-commonjs": "7.23.3", + "@babel/plugin-transform-modules-systemjs": "7.23.3", + "@babel/plugin-transform-modules-umd": "7.23.3", + "@babel/plugin-transform-named-capturing-groups-regex": "7.22.5", + "@babel/plugin-transform-new-target": "7.23.3", + "@babel/plugin-transform-object-super": "7.23.3", + "@babel/plugin-transform-property-literals": "7.23.3", + "@babel/plugin-transform-react-display-name": "7.23.3", + "@babel/plugin-transform-react-jsx-development": "7.22.5", + "@babel/plugin-transform-react-pure-annotations": "7.23.3", + "@babel/plugin-transform-regenerator": "7.23.3", + "@babel/plugin-transform-reserved-words": "7.23.3", + "@babel/plugin-transform-runtime": "7.23.6", + "@babel/plugin-transform-shorthand-properties": "7.23.3", + "@babel/plugin-transform-spread": "7.23.3", + "@babel/plugin-transform-sticky-regex": "7.23.3", + "@babel/plugin-transform-template-literals": "7.23.3", + "@babel/plugin-transform-typeof-symbol": "7.23.3", + "@babel/plugin-transform-typescript": "7.23.6", + "@babel/plugin-transform-unicode-escapes": "7.23.3", + "@babel/plugin-transform-unicode-regex": "7.23.3", + "@babel/preset-env": "7.23.6", + "@babel/preset-react": "7.23.3", + "@babel/preset-typescript": "7.23.3", + "@babel/runtime-corejs3": "7.23.6", + "@cmfcmf/docusaurus-search-local": "^1.1.0", "@docsearch/react": "3.1.0", - "@docusaurus/core": "^2.1.0", - "@docusaurus/cssnano-preset": "^2.1.0", - "@docusaurus/module-type-aliases": "^2.1.0", - "@docusaurus/plugin-client-redirects": "^2.1.0", - "@docusaurus/plugin-debug": "^2.1.0", - "@docusaurus/plugin-google-analytics": "^2.1.0", - "@docusaurus/plugin-google-gtag": "^2.1.0", - "@docusaurus/plugin-sitemap": "^2.1.0", - "@docusaurus/preset-classic": "^2.1.0", - "@docusaurus/theme-classic": "^2.1.0", - "@docusaurus/theme-search-algolia": "^2.1.0", - "@docusaurus/types": "^2.1.0", - "@mdx-js/react": "^1.6.21", + "@docusaurus/core": "^3.0.1", + "@docusaurus/cssnano-preset": "^3.0.1", + "@docusaurus/module-type-aliases": "^3.0.1", + "@docusaurus/plugin-client-redirects": "^3.0.1", + "@docusaurus/plugin-debug": "^3.0.1", + "@docusaurus/plugin-sitemap": "^3.0.1", + "@docusaurus/preset-classic": "^3.0.1", + "@docusaurus/theme-classic": "^3.0.1", + "@docusaurus/theme-mermaid": "^3.0.1", + "@docusaurus/theme-search-algolia": "^3.0.1", + "@docusaurus/types": "^3.0.1", + "@fortawesome/fontawesome-svg-core": "^6.5.1", + "@fortawesome/free-regular-svg-icons": "^6.5.1", + "@fortawesome/free-solid-svg-icons": "^6.5.1", + "@fortawesome/react-fontawesome": "^0.2.0", + "@mdx-js/react": "^3.0.0", "async": "2.6.4", - "autoprefixer": "10.4.7", + "autoprefixer": "10.4.16", + "classnames": "^2.3.2", "clsx": "^1.1.1", "copy-webpack-plugin": "11.0.0", - "core-js": "3.23.3", + "core-js": "3.35.0", "css-declaration-sorter": "6.3.0", "css-minimizer-webpack-plugin": "4.0.0", - "cssnano": "5.1.12", - "cssnano-preset-advanced": "5.3.8", + "cssnano": "6.0.2", + "cssnano-preset-advanced": "6.0.2", "del": "6.1.1", "docusaurus-plugin-hubspot": "^1.0.0", "docusaurus-plugin-segment": "^1.0.3", + "js-yaml": "^4.1.0", + "node-fetch": "^3.3.2", "nth-check": "2.0.1", - "postcss-convert-values": "5.1.2", - "postcss-discard-comments": "5.1.2", - "postcss-loader": "7.0.0", - "postcss-merge-longhand": "5.1.6", - "postcss-merge-rules": "5.1.2", - "postcss-minify-selectors": "5.2.1", - "postcss-normalize-positions": "5.1.1", - "postcss-normalize-repeat-style": "5.1.1", - "postcss-ordered-values": "5.1.3", - "prism-react-renderer": "^1.3.5", - "react": "^17.0.1", - "react-dom": "^17.0.1", + "postcss-convert-values": "6.0.1", + "postcss-discard-comments": "6.0.1", + "postcss-loader": "7.3.4", + "postcss-merge-longhand": "6.0.1", + "postcss-merge-rules": "6.0.2", + "postcss-minify-selectors": "6.0.1", + "postcss-normalize-positions": "6.0.1", + "postcss-normalize-repeat-style": "6.0.1", + "postcss-ordered-values": "6.0.1", + "prism-react-renderer": "^2.3.1", + "react": "^18.2.0", + "react-dom": "^18.2.0", "react-markdown": "^8.0.7", "react-router": "5.3.3", "sockjs": "0.3.24", "trim": "0.0.3", + "unist-builder": "^4.0.0", + "unist-util-select": "^5.1.0", + "unist-util-visit": "^5.0.0", "webpack-dev-server": "4.9.2", "yaml-loader": "^0.8.0" }, diff --git a/docusaurus/pnpm-lock.yaml b/docusaurus/pnpm-lock.yaml new file mode 100644 index 000000000000..ab5603ec5d4d --- /dev/null +++ b/docusaurus/pnpm-lock.yaml @@ -0,0 +1,11444 @@ +lockfileVersion: '6.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +dependencies: + '@babel/core': + specifier: 7.23.6 + version: 7.23.6 + '@babel/helper-builder-binary-assignment-operator-visitor': + specifier: 7.22.15 + version: 7.22.15 + '@babel/helper-explode-assignable-expression': + specifier: 7.18.6 + version: 7.18.6 + '@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-proposal-async-generator-functions': + specifier: 7.20.7 + version: 7.20.7(@babel/core@7.23.6) + '@babel/plugin-proposal-class-properties': + specifier: 7.18.6 + version: 7.18.6(@babel/core@7.23.6) + '@babel/plugin-proposal-class-static-block': + specifier: 7.21.0 + version: 7.21.0(@babel/core@7.23.6) + '@babel/plugin-proposal-dynamic-import': + specifier: 7.18.6 + version: 7.18.6(@babel/core@7.23.6) + '@babel/plugin-proposal-export-namespace-from': + specifier: 7.18.9 + version: 7.18.9(@babel/core@7.23.6) + '@babel/plugin-proposal-json-strings': + specifier: 7.18.6 + version: 7.18.6(@babel/core@7.23.6) + '@babel/plugin-proposal-logical-assignment-operators': + specifier: 7.20.7 + version: 7.20.7(@babel/core@7.23.6) + '@babel/plugin-proposal-nullish-coalescing-operator': + specifier: 7.18.6 + version: 7.18.6(@babel/core@7.23.6) + '@babel/plugin-proposal-numeric-separator': + specifier: 7.18.6 + version: 7.18.6(@babel/core@7.23.6) + '@babel/plugin-proposal-object-rest-spread': + specifier: 7.20.7 + version: 7.20.7(@babel/core@7.23.6) + '@babel/plugin-proposal-optional-catch-binding': + specifier: 7.18.6 + version: 7.18.6(@babel/core@7.23.6) + '@babel/plugin-proposal-private-methods': + specifier: 7.18.6 + version: 7.18.6(@babel/core@7.23.6) + '@babel/plugin-proposal-private-property-in-object': + specifier: 7.21.11 + version: 7.21.11(@babel/core@7.23.6) + '@babel/plugin-syntax-import-assertions': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-syntax-typescript': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-arrow-functions': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-async-to-generator': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-block-scoped-functions': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-block-scoping': + specifier: 7.23.4 + version: 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-classes': + specifier: 7.23.5 + version: 7.23.5(@babel/core@7.23.6) + '@babel/plugin-transform-computed-properties': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-destructuring': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-duplicate-keys': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-exponentiation-operator': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-for-of': + specifier: 7.23.6 + version: 7.23.6(@babel/core@7.23.6) + '@babel/plugin-transform-function-name': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-literals': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-member-expression-literals': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-modules-amd': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-modules-commonjs': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-modules-systemjs': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-modules-umd': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-named-capturing-groups-regex': + specifier: 7.22.5 + version: 7.22.5(@babel/core@7.23.6) + '@babel/plugin-transform-new-target': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-object-super': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-property-literals': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-react-display-name': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-react-jsx-development': + specifier: 7.22.5 + version: 7.22.5(@babel/core@7.23.6) + '@babel/plugin-transform-react-pure-annotations': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-regenerator': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-reserved-words': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-runtime': + specifier: 7.23.6 + version: 7.23.6(@babel/core@7.23.6) + '@babel/plugin-transform-shorthand-properties': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-spread': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-sticky-regex': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-template-literals': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-typeof-symbol': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-typescript': + specifier: 7.23.6 + version: 7.23.6(@babel/core@7.23.6) + '@babel/plugin-transform-unicode-escapes': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-unicode-regex': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/preset-env': + specifier: 7.23.6 + version: 7.23.6(@babel/core@7.23.6) + '@babel/preset-react': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/preset-typescript': + specifier: 7.23.3 + version: 7.23.3(@babel/core@7.23.6) + '@babel/runtime-corejs3': + specifier: 7.23.6 + version: 7.23.6 + '@cmfcmf/docusaurus-search-local': + specifier: ^1.1.0 + version: 1.1.0(@docusaurus/core@3.0.1)(search-insights@2.13.0) + '@docsearch/react': + specifier: 3.1.0 + version: 3.1.0(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/core': + specifier: ^3.0.1 + version: 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/cssnano-preset': + specifier: ^3.0.1 + version: 3.0.1 + '@docusaurus/module-type-aliases': + specifier: ^3.0.1 + version: 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/plugin-client-redirects': + specifier: ^3.0.1 + version: 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-debug': + specifier: ^3.0.1 + version: 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-sitemap': + specifier: ^3.0.1 + version: 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/preset-classic': + specifier: ^3.0.1 + version: 3.0.1(@algolia/client-search@4.22.0)(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0)(typescript@5.3.3) + '@docusaurus/theme-classic': + specifier: ^3.0.1 + version: 3.0.1(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/theme-mermaid': + specifier: ^3.0.1 + version: 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/theme-search-algolia': + specifier: ^3.0.1 + version: 3.0.1(@algolia/client-search@4.22.0)(@docusaurus/types@3.0.1)(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0)(typescript@5.3.3) + '@docusaurus/types': + specifier: ^3.0.1 + version: 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@fortawesome/fontawesome-svg-core': + specifier: ^6.5.1 + version: 6.5.1 + '@fortawesome/free-regular-svg-icons': + specifier: ^6.5.1 + version: 6.5.1 + '@fortawesome/free-solid-svg-icons': + specifier: ^6.5.1 + version: 6.5.1 + '@fortawesome/react-fontawesome': + specifier: ^0.2.0 + version: 0.2.0(@fortawesome/fontawesome-svg-core@6.5.1)(react@18.2.0) + '@mdx-js/react': + specifier: ^3.0.0 + version: 3.0.0(@types/react@18.2.46)(react@18.2.0) + async: + specifier: 2.6.4 + version: 2.6.4 + autoprefixer: + specifier: 10.4.16 + version: 10.4.16(postcss@8.4.32) + classnames: + specifier: ^2.3.2 + version: 2.3.2 + clsx: + specifier: ^1.1.1 + version: 1.1.1 + copy-webpack-plugin: + specifier: 11.0.0 + version: 11.0.0(webpack@5.89.0) + core-js: + specifier: 3.35.0 + version: 3.35.0 + css-declaration-sorter: + specifier: 6.3.0 + version: 6.3.0(postcss@8.4.32) + css-minimizer-webpack-plugin: + specifier: 4.0.0 + version: 4.0.0(webpack@5.89.0) + cssnano: + specifier: 6.0.2 + version: 6.0.2(postcss@8.4.32) + cssnano-preset-advanced: + specifier: 6.0.2 + version: 6.0.2(postcss@8.4.32) + del: + specifier: 6.1.1 + version: 6.1.1 + docusaurus-plugin-hubspot: + specifier: ^1.0.0 + version: 1.0.0 + docusaurus-plugin-segment: + specifier: ^1.0.3 + version: 1.0.3 + js-yaml: + specifier: ^4.1.0 + version: 4.1.0 + node-fetch: + specifier: ^3.3.2 + version: 3.3.2 + nth-check: + specifier: 2.0.1 + version: 2.0.1 + postcss-convert-values: + specifier: 6.0.1 + version: 6.0.1(postcss@8.4.32) + postcss-discard-comments: + specifier: 6.0.1 + version: 6.0.1(postcss@8.4.32) + postcss-loader: + specifier: 7.3.4 + version: 7.3.4(postcss@8.4.32)(typescript@5.3.3)(webpack@5.89.0) + postcss-merge-longhand: + specifier: 6.0.1 + version: 6.0.1(postcss@8.4.32) + postcss-merge-rules: + specifier: 6.0.2 + version: 6.0.2(postcss@8.4.32) + postcss-minify-selectors: + specifier: 6.0.1 + version: 6.0.1(postcss@8.4.32) + postcss-normalize-positions: + specifier: 6.0.1 + version: 6.0.1(postcss@8.4.32) + postcss-normalize-repeat-style: + specifier: 6.0.1 + version: 6.0.1(postcss@8.4.32) + postcss-ordered-values: + specifier: 6.0.1 + version: 6.0.1(postcss@8.4.32) + prism-react-renderer: + specifier: ^2.3.1 + version: 2.3.1(react@18.2.0) + react: + specifier: ^18.2.0 + version: 18.2.0 + react-dom: + specifier: ^18.2.0 + version: 18.2.0(react@18.2.0) + react-markdown: + specifier: ^8.0.7 + version: 8.0.7(@types/react@18.2.46)(react@18.2.0) + react-router: + specifier: 5.3.3 + version: 5.3.3(react@18.2.0) + sockjs: + specifier: 0.3.24 + version: 0.3.24 + trim: + specifier: 0.0.3 + version: 0.0.3 + unist-builder: + specifier: ^4.0.0 + version: 4.0.0 + unist-util-select: + specifier: ^5.1.0 + version: 5.1.0 + unist-util-visit: + specifier: ^5.0.0 + version: 5.0.0 + webpack-dev-server: + specifier: 4.9.2 + version: 4.9.2(webpack@5.89.0) + yaml-loader: + specifier: ^0.8.0 + version: 0.8.0 + +packages: + + /@algolia/autocomplete-core@1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0)(search-insights@2.13.0): + resolution: {integrity: sha512-0v3mHfkvJBVx0aO1U290EHaLPp9pkUL8zkgbVY0JlitItrbXfYYHQHtNs1TxpA63mQAD0K0LyLzO2x+uWiBbGQ==} + dependencies: + '@algolia/autocomplete-plugin-algolia-insights': 1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0)(search-insights@2.13.0) + '@algolia/autocomplete-shared': 1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0) + transitivePeerDependencies: + - '@algolia/client-search' + - algoliasearch + - search-insights + dev: false + + /@algolia/autocomplete-core@1.6.3: + resolution: {integrity: sha512-dqQqRt01fX3YuVFrkceHsoCnzX0bLhrrg8itJI1NM68KjrPYQPYsE+kY8EZTCM4y8VDnhqJErR73xe/ZsV+qAA==} + dependencies: + '@algolia/autocomplete-shared': 1.6.3 + dev: false + + /@algolia/autocomplete-core@1.9.3(@algolia/client-search@4.22.0)(algoliasearch@4.22.0)(search-insights@2.13.0): + resolution: {integrity: sha512-009HdfugtGCdC4JdXUbVJClA0q0zh24yyePn+KUGk3rP7j8FEe/m5Yo/z65gn6nP/cM39PxpzqKrL7A6fP6PPw==} + dependencies: + '@algolia/autocomplete-plugin-algolia-insights': 1.9.3(@algolia/client-search@4.22.0)(algoliasearch@4.22.0)(search-insights@2.13.0) + '@algolia/autocomplete-shared': 1.9.3(@algolia/client-search@4.22.0)(algoliasearch@4.22.0) + transitivePeerDependencies: + - '@algolia/client-search' + - algoliasearch + - search-insights + dev: false + + /@algolia/autocomplete-js@1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0)(search-insights@2.13.0): + resolution: {integrity: sha512-gw2jbkIzSH+xljX3yoOg+5nfJwMh7jqw5T/jy/WPwgmPhn5Mv6PmosCM0huGwH2E88nwxNlY2AhbkDrS4qceAw==} + peerDependencies: + '@algolia/client-search': '>= 4.5.1 < 6' + algoliasearch: '>= 4.9.1 < 6' + dependencies: + '@algolia/autocomplete-core': 1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0)(search-insights@2.13.0) + '@algolia/autocomplete-preset-algolia': 1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0) + '@algolia/autocomplete-shared': 1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0) + '@algolia/client-search': 4.22.0 + algoliasearch: 4.22.0 + htm: 3.1.1 + preact: 10.19.3 + transitivePeerDependencies: + - search-insights + dev: false + + /@algolia/autocomplete-plugin-algolia-insights@1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0)(search-insights@2.13.0): + resolution: {integrity: sha512-Q0rRUZ72x7piqvJKi1//SBZvoImnYdJLRC7Yaa0rwKtkIVQFl6MmZw/p4AEDSWIu5HY3Ki3bzgYxeDyhm//P/w==} + peerDependencies: + search-insights: '>= 1 < 3' + dependencies: + '@algolia/autocomplete-shared': 1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0) + search-insights: 2.13.0 + transitivePeerDependencies: + - '@algolia/client-search' + - algoliasearch + dev: false + + /@algolia/autocomplete-plugin-algolia-insights@1.9.3(@algolia/client-search@4.22.0)(algoliasearch@4.22.0)(search-insights@2.13.0): + resolution: {integrity: sha512-a/yTUkcO/Vyy+JffmAnTWbr4/90cLzw+CC3bRbhnULr/EM0fGNvM13oQQ14f2moLMcVDyAx/leczLlAOovhSZg==} + peerDependencies: + search-insights: '>= 1 < 3' + dependencies: + '@algolia/autocomplete-shared': 1.9.3(@algolia/client-search@4.22.0)(algoliasearch@4.22.0) + search-insights: 2.13.0 + transitivePeerDependencies: + - '@algolia/client-search' + - algoliasearch + dev: false + + /@algolia/autocomplete-preset-algolia@1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0): + resolution: {integrity: sha512-IlanOCLT2EvfygX5cGFR5iKgfhQB0MqCv163ldctq8l0QCVdEOM1VLIQhl0tB3ViJc5XKUB8QZ7V+DcSVtZAuQ==} + peerDependencies: + '@algolia/client-search': '>= 4.9.1 < 6' + algoliasearch: '>= 4.9.1 < 6' + dependencies: + '@algolia/autocomplete-shared': 1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0) + '@algolia/client-search': 4.22.0 + algoliasearch: 4.22.0 + dev: false + + /@algolia/autocomplete-preset-algolia@1.9.3(@algolia/client-search@4.22.0)(algoliasearch@4.22.0): + resolution: {integrity: sha512-d4qlt6YmrLMYy95n5TB52wtNDr6EgAIPH81dvvvW8UmuWRgxEtY0NJiPwl/h95JtG2vmRM804M0DSwMCNZlzRA==} + peerDependencies: + '@algolia/client-search': '>= 4.9.1 < 6' + algoliasearch: '>= 4.9.1 < 6' + dependencies: + '@algolia/autocomplete-shared': 1.9.3(@algolia/client-search@4.22.0)(algoliasearch@4.22.0) + '@algolia/client-search': 4.22.0 + algoliasearch: 4.22.0 + dev: false + + /@algolia/autocomplete-shared@1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0): + resolution: {integrity: sha512-YB7JlPl1coHai3Xd4OdNIMavAMbgx8eHPH9nlEgcrCqCx57njh0qReruTMRxaThBaWIkkl47jZlUnKvb8MjGGQ==} + peerDependencies: + '@algolia/client-search': '>= 4.9.1 < 6' + algoliasearch: '>= 4.9.1 < 6' + dependencies: + '@algolia/client-search': 4.22.0 + algoliasearch: 4.22.0 + dev: false + + /@algolia/autocomplete-shared@1.6.3: + resolution: {integrity: sha512-UV46bnkTztyADFaETfzFC5ryIdGVb2zpAoYgu0tfcuYWjhg1KbLXveFffZIrGVoboqmAk1b+jMrl6iCja1i3lg==} + dev: false + + /@algolia/autocomplete-shared@1.9.3(@algolia/client-search@4.22.0)(algoliasearch@4.22.0): + resolution: {integrity: sha512-Wnm9E4Ye6Rl6sTTqjoymD+l8DjSTHsHboVRYrKgEt8Q7UHm9nYbqhN/i0fhUYA3OAEH7WA8x3jfpnmJm3rKvaQ==} + peerDependencies: + '@algolia/client-search': '>= 4.9.1 < 6' + algoliasearch: '>= 4.9.1 < 6' + dependencies: + '@algolia/client-search': 4.22.0 + algoliasearch: 4.22.0 + dev: false + + /@algolia/autocomplete-theme-classic@1.13.0: + resolution: {integrity: sha512-YAyfcpi+VJ0h5PUTThDmc/V2OB47RNlvIBQgffzrjAw5vDkoBcAj5bsReJW8/QtLnRGB85XhrmWoYFtP4W3HgQ==} + dev: false + + /@algolia/cache-browser-local-storage@4.22.0: + resolution: {integrity: sha512-uZ1uZMLDZb4qODLfTSNHxSi4fH9RdrQf7DXEzW01dS8XK7QFtFh29N5NGKa9S+Yudf1vUMIF+/RiL4i/J0pWlQ==} + dependencies: + '@algolia/cache-common': 4.22.0 + dev: false + + /@algolia/cache-common@4.22.0: + resolution: {integrity: sha512-TPwUMlIGPN16eW67qamNQUmxNiGHg/WBqWcrOoCddhqNTqGDPVqmgfaM85LPbt24t3r1z0zEz/tdsmuq3Q6oaA==} + dev: false + + /@algolia/cache-in-memory@4.22.0: + resolution: {integrity: sha512-kf4Cio9NpPjzp1+uXQgL4jsMDeck7MP89BYThSvXSjf2A6qV/0KeqQf90TL2ECS02ovLOBXkk98P7qVarM+zGA==} + dependencies: + '@algolia/cache-common': 4.22.0 + dev: false + + /@algolia/client-account@4.22.0: + resolution: {integrity: sha512-Bjb5UXpWmJT+yGWiqAJL0prkENyEZTBzdC+N1vBuHjwIJcjLMjPB6j1hNBRbT12Lmwi55uzqeMIKS69w+0aPzA==} + dependencies: + '@algolia/client-common': 4.22.0 + '@algolia/client-search': 4.22.0 + '@algolia/transporter': 4.22.0 + dev: false + + /@algolia/client-analytics@4.22.0: + resolution: {integrity: sha512-os2K+kHUcwwRa4ArFl5p/3YbF9lN3TLOPkbXXXxOvDpqFh62n9IRZuzfxpHxMPKAQS3Et1s0BkKavnNP02E9Hg==} + dependencies: + '@algolia/client-common': 4.22.0 + '@algolia/client-search': 4.22.0 + '@algolia/requester-common': 4.22.0 + '@algolia/transporter': 4.22.0 + dev: false + + /@algolia/client-common@4.22.0: + resolution: {integrity: sha512-BlbkF4qXVWuwTmYxVWvqtatCR3lzXwxx628p1wj1Q7QP2+LsTmGt1DiUYRuy9jG7iMsnlExby6kRMOOlbhv2Ag==} + dependencies: + '@algolia/requester-common': 4.22.0 + '@algolia/transporter': 4.22.0 + dev: false + + /@algolia/client-personalization@4.22.0: + resolution: {integrity: sha512-pEOftCxeBdG5pL97WngOBi9w5Vxr5KCV2j2D+xMVZH8MuU/JX7CglDSDDb0ffQWYqcUN+40Ry+xtXEYaGXTGow==} + dependencies: + '@algolia/client-common': 4.22.0 + '@algolia/requester-common': 4.22.0 + '@algolia/transporter': 4.22.0 + dev: false + + /@algolia/client-search@4.22.0: + resolution: {integrity: sha512-bn4qQiIdRPBGCwsNuuqB8rdHhGKKWIij9OqidM1UkQxnSG8yzxHdb7CujM30pvp5EnV7jTqDZRbxacbjYVW20Q==} + dependencies: + '@algolia/client-common': 4.22.0 + '@algolia/requester-common': 4.22.0 + '@algolia/transporter': 4.22.0 + dev: false + + /@algolia/events@4.0.1: + resolution: {integrity: sha512-FQzvOCgoFXAbf5Y6mYozw2aj5KCJoA3m4heImceldzPSMbdyS4atVjJzXKMsfX3wnZTFYwkkt8/z8UesLHlSBQ==} + dev: false + + /@algolia/logger-common@4.22.0: + resolution: {integrity: sha512-HMUQTID0ucxNCXs5d1eBJ5q/HuKg8rFVE/vOiLaM4Abfeq1YnTtGV3+rFEhOPWhRQxNDd+YHa4q864IMc0zHpQ==} + dev: false + + /@algolia/logger-console@4.22.0: + resolution: {integrity: sha512-7JKb6hgcY64H7CRm3u6DRAiiEVXMvCJV5gRE672QFOUgDxo4aiDpfU61g6Uzy8NKjlEzHMmgG4e2fklELmPXhQ==} + dependencies: + '@algolia/logger-common': 4.22.0 + dev: false + + /@algolia/requester-browser-xhr@4.22.0: + resolution: {integrity: sha512-BHfv1h7P9/SyvcDJDaRuIwDu2yrDLlXlYmjvaLZTtPw6Ok/ZVhBR55JqW832XN/Fsl6k3LjdkYHHR7xnsa5Wvg==} + dependencies: + '@algolia/requester-common': 4.22.0 + dev: false + + /@algolia/requester-common@4.22.0: + resolution: {integrity: sha512-Y9cEH/cKjIIZgzvI1aI0ARdtR/xRrOR13g5psCxkdhpgRN0Vcorx+zePhmAa4jdQNqexpxtkUdcKYugBzMZJgQ==} + dev: false + + /@algolia/requester-node-http@4.22.0: + resolution: {integrity: sha512-8xHoGpxVhz3u2MYIieHIB6MsnX+vfd5PS4REgglejJ6lPigftRhTdBCToe6zbwq4p0anZXjjPDvNWMlgK2+xYA==} + dependencies: + '@algolia/requester-common': 4.22.0 + dev: false + + /@algolia/transporter@4.22.0: + resolution: {integrity: sha512-ieO1k8x2o77GNvOoC+vAkFKppydQSVfbjM3YrSjLmgywiBejPTvU1R1nEvG59JIIUvtSLrZsLGPkd6vL14zopA==} + dependencies: + '@algolia/cache-common': 4.22.0 + '@algolia/logger-common': 4.22.0 + '@algolia/requester-common': 4.22.0 + dev: false + + /@ampproject/remapping@2.2.1: + resolution: {integrity: sha512-lFMjJTrFL3j7L9yBxwYfCq2k6qqwHyzuUl/XBnif78PWTJYyL/dfowQHWE3sp6U6ZzqWiiIZnpTMO96zhkjwtg==} + engines: {node: '>=6.0.0'} + dependencies: + '@jridgewell/gen-mapping': 0.3.3 + '@jridgewell/trace-mapping': 0.3.20 + dev: false + + /@babel/code-frame@7.23.5: + resolution: {integrity: sha512-CgH3s1a96LipHCmSUmYFPwY7MNx8C3avkq7i4Wl3cfa662ldtUe4VM1TPXX70pfmrlWTb6jLqTYrZyT2ZTJBgA==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/highlight': 7.23.4 + chalk: 2.4.2 + dev: false + + /@babel/compat-data@7.23.5: + resolution: {integrity: sha512-uU27kfDRlhfKl+w1U6vp16IuvSLtjAxdArVXPa9BvLkrr7CYIsxH5adpHObeAGY/41+syctUWOZ140a2Rvkgjw==} + engines: {node: '>=6.9.0'} + dev: false + + /@babel/core@7.23.6: + resolution: {integrity: sha512-FxpRyGjrMJXh7X3wGLGhNDCRiwpWEF74sKjTLDJSG5Kyvow3QZaG0Adbqzi9ZrVjTWpsX+2cxWXD71NMg93kdw==} + engines: {node: '>=6.9.0'} + dependencies: + '@ampproject/remapping': 2.2.1 + '@babel/code-frame': 7.23.5 + '@babel/generator': 7.23.6 + '@babel/helper-compilation-targets': 7.23.6 + '@babel/helper-module-transforms': 7.23.3(@babel/core@7.23.6) + '@babel/helpers': 7.23.7 + '@babel/parser': 7.23.6 + '@babel/template': 7.22.15 + '@babel/traverse': 7.23.7 + '@babel/types': 7.23.6 + convert-source-map: 2.0.0 + debug: 4.3.4 + gensync: 1.0.0-beta.2 + json5: 2.2.3 + semver: 6.3.1 + transitivePeerDependencies: + - supports-color + dev: false + + /@babel/generator@7.23.6: + resolution: {integrity: sha512-qrSfCYxYQB5owCmGLbl8XRpX1ytXlpueOb0N0UmQwA073KZxejgQTzAmJezxvpwQD9uGtK2shHdi55QT+MbjIw==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.23.6 + '@jridgewell/gen-mapping': 0.3.3 + '@jridgewell/trace-mapping': 0.3.20 + jsesc: 2.5.2 + dev: false + + /@babel/helper-annotate-as-pure@7.22.5: + resolution: {integrity: sha512-LvBTxu8bQSQkcyKOU+a1btnNFQ1dMAd0R6PyW3arXes06F6QLWLIrd681bxRPIXlrMGR3XYnW9JyML7dP3qgxg==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.23.6 + dev: false + + /@babel/helper-builder-binary-assignment-operator-visitor@7.22.15: + resolution: {integrity: sha512-QkBXwGgaoC2GtGZRoma6kv7Szfv06khvhFav67ZExau2RaXzy8MpHSMO2PNoP2XtmQphJQRHFfg77Bq731Yizw==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.23.6 + dev: false + + /@babel/helper-compilation-targets@7.23.6: + resolution: {integrity: sha512-9JB548GZoQVmzrFgp8o7KxdgkTGm6xs9DW0o/Pim72UDjzr5ObUQ6ZzYPqA+g9OTS2bBQoctLJrky0RDCAWRgQ==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/compat-data': 7.23.5 + '@babel/helper-validator-option': 7.23.5 + browserslist: 4.22.2 + lru-cache: 5.1.1 + semver: 6.3.1 + dev: false + + /@babel/helper-create-class-features-plugin@7.23.7(@babel/core@7.23.6): + resolution: {integrity: sha512-xCoqR/8+BoNnXOY7RVSgv6X+o7pmT5q1d+gGcRlXYkI+9B31glE4jeejhKVpA04O1AtzOt7OSQ6VYKP5FcRl9g==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-environment-visitor': 7.22.20 + '@babel/helper-function-name': 7.23.0 + '@babel/helper-member-expression-to-functions': 7.23.0 + '@babel/helper-optimise-call-expression': 7.22.5 + '@babel/helper-replace-supers': 7.22.20(@babel/core@7.23.6) + '@babel/helper-skip-transparent-expression-wrappers': 7.22.5 + '@babel/helper-split-export-declaration': 7.22.6 + semver: 6.3.1 + dev: false + + /@babel/helper-create-regexp-features-plugin@7.22.15(@babel/core@7.23.6): + resolution: {integrity: sha512-29FkPLFjn4TPEa3RE7GpW+qbE8tlsu3jntNYNfcGsc49LphF1PQIiD+vMZ1z1xVOKt+93khA9tc2JBs3kBjA7w==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-annotate-as-pure': 7.22.5 + regexpu-core: 5.3.2 + semver: 6.3.1 + dev: false + + /@babel/helper-define-polyfill-provider@0.4.4(@babel/core@7.23.6): + resolution: {integrity: sha512-QcJMILQCu2jm5TFPGA3lCpJJTeEP+mqeXooG/NZbg/h5FTFi6V0+99ahlRsW8/kRLyb24LZVCCiclDedhLKcBA==} + peerDependencies: + '@babel/core': ^7.4.0 || ^8.0.0-0 <8.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-compilation-targets': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + debug: 4.3.4 + lodash.debounce: 4.0.8 + resolve: 1.22.8 + transitivePeerDependencies: + - supports-color + dev: false + + /@babel/helper-environment-visitor@7.22.20: + resolution: {integrity: sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==} + engines: {node: '>=6.9.0'} + dev: false + + /@babel/helper-explode-assignable-expression@7.18.6: + resolution: {integrity: sha512-eyAYAsQmB80jNfg4baAtLeWAQHfHFiR483rzFK+BhETlGZaQC9bsfrugfXDCbRHLQbIA7U5NxhhOxN7p/dWIcg==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.23.6 + dev: false + + /@babel/helper-function-name@7.23.0: + resolution: {integrity: sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/template': 7.22.15 + '@babel/types': 7.23.6 + dev: false + + /@babel/helper-hoist-variables@7.22.5: + resolution: {integrity: sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.23.6 + dev: false + + /@babel/helper-member-expression-to-functions@7.23.0: + resolution: {integrity: sha512-6gfrPwh7OuT6gZyJZvd6WbTfrqAo7vm4xCzAXOusKqq/vWdKXphTpj5klHKNmRUU6/QRGlBsyU9mAIPaWHlqJA==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.23.6 + dev: false + + /@babel/helper-module-imports@7.22.15: + resolution: {integrity: sha512-0pYVBnDKZO2fnSPCrgM/6WMc7eS20Fbok+0r88fp+YtWVLZrp4CkafFGIp+W0VKw4a22sgebPT99y+FDNMdP4w==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.23.6 + dev: false + + /@babel/helper-module-transforms@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-7bBs4ED9OmswdfDzpz4MpWgSrV7FXlc3zIagvLFjS5H+Mk7Snr21vQ6QwrsoCGMfNC4e4LQPdoULEt4ykz0SRQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-environment-visitor': 7.22.20 + '@babel/helper-module-imports': 7.22.15 + '@babel/helper-simple-access': 7.22.5 + '@babel/helper-split-export-declaration': 7.22.6 + '@babel/helper-validator-identifier': 7.22.20 + dev: false + + /@babel/helper-optimise-call-expression@7.22.5: + resolution: {integrity: sha512-HBwaojN0xFRx4yIvpwGqxiV2tUfl7401jlok564NgB9EHS1y6QT17FmKWm4ztqjeVdXLuC4fSvHc5ePpQjoTbw==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.23.6 + dev: false + + /@babel/helper-plugin-utils@7.22.5: + resolution: {integrity: sha512-uLls06UVKgFG9QD4OeFYLEGteMIAa5kpTPcFL28yuCIIzsf6ZyKZMllKVOCZFhiZ5ptnwX4mtKdWCBE/uT4amg==} + engines: {node: '>=6.9.0'} + dev: false + + /@babel/helper-remap-async-to-generator@7.22.20(@babel/core@7.23.6): + resolution: {integrity: sha512-pBGyV4uBqOns+0UvhsTO8qgl8hO89PmiDYv+/COyp1aeMcmfrfruz+/nCMFiYyFF/Knn0yfrC85ZzNFjembFTw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-environment-visitor': 7.22.20 + '@babel/helper-wrap-function': 7.22.20 + dev: false + + /@babel/helper-replace-supers@7.22.20(@babel/core@7.23.6): + resolution: {integrity: sha512-qsW0In3dbwQUbK8kejJ4R7IHVGwHJlV6lpG6UA7a9hSa2YEiAib+N1T2kr6PEeUT+Fl7najmSOS6SmAwCHK6Tw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-environment-visitor': 7.22.20 + '@babel/helper-member-expression-to-functions': 7.23.0 + '@babel/helper-optimise-call-expression': 7.22.5 + dev: false + + /@babel/helper-simple-access@7.22.5: + resolution: {integrity: sha512-n0H99E/K+Bika3++WNL17POvo4rKWZ7lZEp1Q+fStVbUi8nxPQEBOlTmCOxW/0JsS56SKKQ+ojAe2pHKJHN35w==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.23.6 + dev: false + + /@babel/helper-skip-transparent-expression-wrappers@7.22.5: + resolution: {integrity: sha512-tK14r66JZKiC43p8Ki33yLBVJKlQDFoA8GYN67lWCDCqoL6EMMSuM9b+Iff2jHaM/RRFYl7K+iiru7hbRqNx8Q==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.23.6 + dev: false + + /@babel/helper-split-export-declaration@7.22.6: + resolution: {integrity: sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/types': 7.23.6 + dev: false + + /@babel/helper-string-parser@7.23.4: + resolution: {integrity: sha512-803gmbQdqwdf4olxrX4AJyFBV/RTr3rSmOj0rKwesmzlfhYNDEs+/iOcznzpNWlJlIlTJC2QfPFcHB6DlzdVLQ==} + engines: {node: '>=6.9.0'} + dev: false + + /@babel/helper-validator-identifier@7.22.20: + resolution: {integrity: sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==} + engines: {node: '>=6.9.0'} + dev: false + + /@babel/helper-validator-option@7.23.5: + resolution: {integrity: sha512-85ttAOMLsr53VgXkTbkx8oA6YTfT4q7/HzXSLEYmjcSTJPMPQtvq1BD79Byep5xMUYbGRzEpDsjUf3dyp54IKw==} + engines: {node: '>=6.9.0'} + dev: false + + /@babel/helper-wrap-function@7.22.20: + resolution: {integrity: sha512-pms/UwkOpnQe/PDAEdV/d7dVCoBbB+R4FvYoHGZz+4VPcg7RtYy2KP7S2lbuWM6FCSgob5wshfGESbC/hzNXZw==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/helper-function-name': 7.23.0 + '@babel/template': 7.22.15 + '@babel/types': 7.23.6 + dev: false + + /@babel/helpers@7.23.7: + resolution: {integrity: sha512-6AMnjCoC8wjqBzDHkuqpa7jAKwvMo4dC+lr/TFBz+ucfulO1XMpDnwWPGBNwClOKZ8h6xn5N81W/R5OrcKtCbQ==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/template': 7.22.15 + '@babel/traverse': 7.23.7 + '@babel/types': 7.23.6 + transitivePeerDependencies: + - supports-color + dev: false + + /@babel/highlight@7.23.4: + resolution: {integrity: sha512-acGdbYSfp2WheJoJm/EBBBLh/ID8KDc64ISZ9DYtBmC8/Q204PZJLHyzeB5qMzJ5trcOkybd78M4x2KWsUq++A==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/helper-validator-identifier': 7.22.20 + chalk: 2.4.2 + js-tokens: 4.0.0 + dev: false + + /@babel/parser@7.23.6: + resolution: {integrity: sha512-Z2uID7YJ7oNvAI20O9X0bblw7Qqs8Q2hFy0R9tAfnfLkp5MW0UH9eUvnDSnFwKZ0AvgS1ucqR4KzvVHgnke1VQ==} + engines: {node: '>=6.0.0'} + hasBin: true + dependencies: + '@babel/types': 7.23.6 + dev: false + + /@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-iRkKcCqb7iGnq9+3G6rZ+Ciz5VywC4XNRHe57lKM+jOeYAoR0lVqdeeDRfh0tQcTfw/+vBhHn926FmQhLtlFLQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-WwlxbfMNdVEpQjZmK5mhm7oSwD3dS6eU+Iwsi4Knl9wAletWem7kaRsGOG+8UEbRyqxY4SS5zvtfXwX+jMxUwQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.13.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-skip-transparent-expression-wrappers': 7.22.5 + '@babel/plugin-transform-optional-chaining': 7.23.4(@babel/core@7.23.6) + dev: false + + /@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly@7.23.7(@babel/core@7.23.6): + resolution: {integrity: sha512-LlRT7HgaifEpQA1ZgLVOIJZZFVPWN5iReq/7/JixwBtwcoeVGDBD53ZV28rrsLYOZs1Y/EHhA8N/Z6aazHR8cw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-environment-visitor': 7.22.20 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-proposal-async-generator-functions@7.20.7(@babel/core@7.23.6): + resolution: {integrity: sha512-xMbiLsn/8RK7Wq7VeVytytS2L6qE69bXPB10YCmMdDZbKF4okCqY74pI/jJQ/8U0b/F6NrT2+14b8/P9/3AMGA==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-async-generator-functions instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-environment-visitor': 7.22.20 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-remap-async-to-generator': 7.22.20(@babel/core@7.23.6) + '@babel/plugin-syntax-async-generators': 7.8.4(@babel/core@7.23.6) + dev: false + + /@babel/plugin-proposal-class-properties@7.18.6(@babel/core@7.23.6): + resolution: {integrity: sha512-cumfXOF0+nzZrrN8Rf0t7M+tF6sZc7vhQwYQck9q1/5w2OExlD+b4v4RpMJFaV1Z7WcDRgO6FqvxqxGlwo+RHQ==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-class-properties instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-class-features-plugin': 7.23.7(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-proposal-class-static-block@7.21.0(@babel/core@7.23.6): + resolution: {integrity: sha512-XP5G9MWNUskFuP30IfFSEFB0Z6HzLIUcjYM4bYOPHXl7eiJ9HFv8tWj6TXTN5QODiEhDZAeI4hLok2iHFFV4hw==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-class-static-block instead. + peerDependencies: + '@babel/core': ^7.12.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-class-features-plugin': 7.23.7(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-class-static-block': 7.14.5(@babel/core@7.23.6) + dev: false + + /@babel/plugin-proposal-dynamic-import@7.18.6(@babel/core@7.23.6): + resolution: {integrity: sha512-1auuwmK+Rz13SJj36R+jqFPMJWyKEDd7lLSdOj4oJK0UTgGueSAtkrCvz9ewmgyU/P941Rv2fQwZJN8s6QruXw==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-dynamic-import instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-dynamic-import': 7.8.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-proposal-export-namespace-from@7.18.9(@babel/core@7.23.6): + resolution: {integrity: sha512-k1NtHyOMvlDDFeb9G5PhUXuGj8m/wiwojgQVEhJ/fsVsMCpLyOP4h0uGEjYJKrRI+EVPlb5Jk+Gt9P97lOGwtA==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-export-namespace-from instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-export-namespace-from': 7.8.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-proposal-json-strings@7.18.6(@babel/core@7.23.6): + resolution: {integrity: sha512-lr1peyn9kOdbYc0xr0OdHTZ5FMqS6Di+H0Fz2I/JwMzGmzJETNeOFq2pBySw6X/KFL5EWDjlJuMsUGRFb8fQgQ==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-json-strings instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-json-strings': 7.8.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-proposal-logical-assignment-operators@7.20.7(@babel/core@7.23.6): + resolution: {integrity: sha512-y7C7cZgpMIjWlKE5T7eJwp+tnRYM89HmRvWM5EQuB5BoHEONjmQ8lSNmBUwOyy/GFRsohJED51YBF79hE1djug==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-logical-assignment-operators instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-logical-assignment-operators': 7.10.4(@babel/core@7.23.6) + dev: false + + /@babel/plugin-proposal-nullish-coalescing-operator@7.18.6(@babel/core@7.23.6): + resolution: {integrity: sha512-wQxQzxYeJqHcfppzBDnm1yAY0jSRkUXR2z8RePZYrKwMKgMlE8+Z6LUno+bd6LvbGh8Gltvy74+9pIYkr+XkKA==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-nullish-coalescing-operator instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-nullish-coalescing-operator': 7.8.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-proposal-numeric-separator@7.18.6(@babel/core@7.23.6): + resolution: {integrity: sha512-ozlZFogPqoLm8WBr5Z8UckIoE4YQ5KESVcNudyXOR8uqIkliTEgJ3RoketfG6pmzLdeZF0H/wjE9/cCEitBl7Q==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-numeric-separator instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-numeric-separator': 7.10.4(@babel/core@7.23.6) + dev: false + + /@babel/plugin-proposal-object-rest-spread@7.20.7(@babel/core@7.23.6): + resolution: {integrity: sha512-d2S98yCiLxDVmBmE8UjGcfPvNEUbA1U5q5WxaWFUGRzJSVAZqm5W6MbPct0jxnegUZ0niLeNX+IOzEs7wYg9Dg==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-object-rest-spread instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/compat-data': 7.23.5 + '@babel/core': 7.23.6 + '@babel/helper-compilation-targets': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-object-rest-spread': 7.8.3(@babel/core@7.23.6) + '@babel/plugin-transform-parameters': 7.23.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-proposal-optional-catch-binding@7.18.6(@babel/core@7.23.6): + resolution: {integrity: sha512-Q40HEhs9DJQyaZfUjjn6vE8Cv4GmMHCYuMGIWUnlxH6400VGxOuwWsPt4FxXxJkC/5eOzgn0z21M9gMT4MOhbw==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-optional-catch-binding instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-optional-catch-binding': 7.8.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-proposal-private-methods@7.18.6(@babel/core@7.23.6): + resolution: {integrity: sha512-nutsvktDItsNn4rpGItSNV2sz1XwS+nfU0Rg8aCx3W3NOKVzdMjJRu0O5OkgDp3ZGICSTbgRpxZoWsxoKRvbeA==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-private-methods instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-class-features-plugin': 7.23.7(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-proposal-private-property-in-object@7.21.0-placeholder-for-preset-env.2(@babel/core@7.23.6): + resolution: {integrity: sha512-SOSkfJDddaM7mak6cPEpswyTRnuRltl429hMraQEglW+OkovnCzsiszTmsrlY//qLFjCpQDFRvjdm2wA5pPm9w==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + dev: false + + /@babel/plugin-proposal-private-property-in-object@7.21.11(@babel/core@7.23.6): + resolution: {integrity: sha512-0QZ8qP/3RLDVBwBFoWAwCtgcDZJVwA5LUJRZU8x2YFfKNuFq161wK3cuGrALu5yiPu+vzwTAg/sMWVNeWeNyaw==} + engines: {node: '>=6.9.0'} + deprecated: This proposal has been merged to the ECMAScript standard and thus this plugin is no longer maintained. Please use @babel/plugin-transform-private-property-in-object instead. + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-create-class-features-plugin': 7.23.7(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-private-property-in-object': 7.14.5(@babel/core@7.23.6) + dev: false + + /@babel/plugin-syntax-async-generators@7.8.4(@babel/core@7.23.6): + resolution: {integrity: sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-class-properties@7.12.13(@babel/core@7.23.6): + resolution: {integrity: sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-class-static-block@7.14.5(@babel/core@7.23.6): + resolution: {integrity: sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-dynamic-import@7.8.3(@babel/core@7.23.6): + resolution: {integrity: sha512-5gdGbFon+PszYzqs83S3E5mpi7/y/8M9eC90MRTZfduQOYW76ig6SOSPNe41IG5LoP3FGBn2N0RjVDSQiS94kQ==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-export-namespace-from@7.8.3(@babel/core@7.23.6): + resolution: {integrity: sha512-MXf5laXo6c1IbEbegDmzGPwGNTsHZmEy6QGznu5Sh2UCWvueywb2ee+CCE4zQiZstxU9BMoQO9i6zUFSY0Kj0Q==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-import-assertions@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-lPgDSU+SJLK3xmFDTV2ZRQAiM7UuUjGidwBywFavObCiZc1BeAAcMtHJKUya92hPHO+at63JJPLygilZard8jw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-import-attributes@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-pawnE0P9g10xgoP7yKr6CK63K2FMsTE+FZidZO/1PwRdzmAPVs+HS1mAURUsgaoxammTJvULUdIkEK0gOcU2tA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-import-meta@7.10.4(@babel/core@7.23.6): + resolution: {integrity: sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-json-strings@7.8.3(@babel/core@7.23.6): + resolution: {integrity: sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-jsx@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-EB2MELswq55OHUoRZLGg/zC7QWUKfNLpE57m/S2yr1uEneIgsTgrSzXP3NXEsMkVn76OlaVVnzN+ugObuYGwhg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-logical-assignment-operators@7.10.4(@babel/core@7.23.6): + resolution: {integrity: sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-nullish-coalescing-operator@7.8.3(@babel/core@7.23.6): + resolution: {integrity: sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-numeric-separator@7.10.4(@babel/core@7.23.6): + resolution: {integrity: sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-object-rest-spread@7.8.3(@babel/core@7.23.6): + resolution: {integrity: sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-optional-catch-binding@7.8.3(@babel/core@7.23.6): + resolution: {integrity: sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-optional-chaining@7.8.3(@babel/core@7.23.6): + resolution: {integrity: sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-private-property-in-object@7.14.5(@babel/core@7.23.6): + resolution: {integrity: sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-top-level-await@7.14.5(@babel/core@7.23.6): + resolution: {integrity: sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-typescript@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-9EiNjVJOMwCO+43TqoTrgQ8jMwcAd0sWyXi9RPfIsLTj4R2MADDDQXELhffaUx/uJv2AYcxBgPwH6j4TIA4ytQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-syntax-unicode-sets-regex@7.18.6(@babel/core@7.23.6): + resolution: {integrity: sha512-727YkEAPwSIQTv5im8QHz3upqp92JTWhidIC81Tdx4VJYIte/VndKf1qKrfnnhPLiPghStWfvC/iFaMCQu7Nqg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-regexp-features-plugin': 7.22.15(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-arrow-functions@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-NzQcQrzaQPkaEwoTm4Mhyl8jI1huEL/WWIEvudjTCMJ9aBZNpsJbMASx7EQECtQQPS/DcnFpo0FIh3LvEO9cxQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-async-generator-functions@7.23.7(@babel/core@7.23.6): + resolution: {integrity: sha512-PdxEpL71bJp1byMG0va5gwQcXHxuEYC/BgI/e88mGTtohbZN28O5Yit0Plkkm/dBzCF/BxmbNcses1RH1T+urA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-environment-visitor': 7.22.20 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-remap-async-to-generator': 7.22.20(@babel/core@7.23.6) + '@babel/plugin-syntax-async-generators': 7.8.4(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-async-to-generator@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-A7LFsKi4U4fomjqXJlZg/u0ft/n8/7n7lpffUP/ZULx/DtV9SGlNKZolHH6PE8Xl1ngCc0M11OaeZptXVkfKSw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-module-imports': 7.22.15 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-remap-async-to-generator': 7.22.20(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-block-scoped-functions@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-vI+0sIaPIO6CNuM9Kk5VmXcMVRiOpDh7w2zZt9GXzmE/9KD70CUEVhvPR/etAeNK/FAEkhxQtXOzVF3EuRL41A==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-block-scoping@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-0QqbP6B6HOh7/8iNR4CQU2Th/bbRtBp4KS9vcaZd1fZ0wSh5Fyssg0UCIHwxh+ka+pNDREbVLQnHCMHKZfPwfw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-class-properties@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-uM+AN8yCIjDPccsKGlw271xjJtGii+xQIF/uMPS8H15L12jZTsLfF4o5vNO7d/oUguOyfdikHGc/yi9ge4SGIg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-class-features-plugin': 7.23.7(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-class-static-block@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-nsWu/1M+ggti1SOALj3hfx5FXzAY06fwPJsUZD4/A5e1bWi46VUIWtD+kOX6/IdhXGsXBWllLFDSnqSCdUNydQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.12.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-class-features-plugin': 7.23.7(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-class-static-block': 7.14.5(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-classes@7.23.5(@babel/core@7.23.6): + resolution: {integrity: sha512-jvOTR4nicqYC9yzOHIhXG5emiFEOpappSJAl73SDSEDcybD+Puuze8Tnpb9p9qEyYup24tq891gkaygIFvWDqg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-compilation-targets': 7.23.6 + '@babel/helper-environment-visitor': 7.22.20 + '@babel/helper-function-name': 7.23.0 + '@babel/helper-optimise-call-expression': 7.22.5 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-replace-supers': 7.22.20(@babel/core@7.23.6) + '@babel/helper-split-export-declaration': 7.22.6 + globals: 11.12.0 + dev: false + + /@babel/plugin-transform-computed-properties@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-dTj83UVTLw/+nbiHqQSFdwO9CbTtwq1DsDqm3CUEtDrZNET5rT5E6bIdTlOftDTDLMYxvxHNEYO4B9SLl8SLZw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/template': 7.22.15 + dev: false + + /@babel/plugin-transform-destructuring@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-n225npDqjDIr967cMScVKHXJs7rout1q+tt50inyBCPkyZ8KxeI6d+GIbSBTT/w/9WdlWDOej3V9HE5Lgk57gw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-dotall-regex@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-vgnFYDHAKzFaTVp+mneDsIEbnJ2Np/9ng9iviHw3P/KVcgONxpNULEW/51Z/BaFojG2GI2GwwXck5uV1+1NOYQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-regexp-features-plugin': 7.22.15(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-duplicate-keys@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-RrqQ+BQmU3Oyav3J+7/myfvRCq7Tbz+kKLLshUmMwNlDHExbGL7ARhajvoBJEvc+fCguPPu887N+3RRXBVKZUA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-dynamic-import@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-V6jIbLhdJK86MaLh4Jpghi8ho5fGzt3imHOBu/x0jlBaPYqDoWz4RDXjmMOfnh+JWNaQleEAByZLV0QzBT4YQQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-dynamic-import': 7.8.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-exponentiation-operator@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-5fhCsl1odX96u7ILKHBj4/Y8vipoqwsJMh4csSA8qFfxrZDEA4Ssku2DyNvMJSmZNOEBT750LfFPbtrnTP90BQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-builder-binary-assignment-operator-visitor': 7.22.15 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-export-namespace-from@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-GzuSBcKkx62dGzZI1WVgTWvkkz84FZO5TC5T8dl/Tht/rAla6Dg/Mz9Yhypg+ezVACf/rgDuQt3kbWEv7LdUDQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-export-namespace-from': 7.8.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-for-of@7.23.6(@babel/core@7.23.6): + resolution: {integrity: sha512-aYH4ytZ0qSuBbpfhuofbg/e96oQ7U2w1Aw/UQmKT+1l39uEhUPoFS3fHevDc1G0OvewyDudfMKY1OulczHzWIw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-skip-transparent-expression-wrappers': 7.22.5 + dev: false + + /@babel/plugin-transform-function-name@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-I1QXp1LxIvt8yLaib49dRW5Okt7Q4oaxao6tFVKS/anCdEOMtYwWVKoiOA1p34GOWIZjUK0E+zCp7+l1pfQyiw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-compilation-targets': 7.23.6 + '@babel/helper-function-name': 7.23.0 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-json-strings@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-81nTOqM1dMwZ/aRXQ59zVubN9wHGqk6UtqRK+/q+ciXmRy8fSolhGVvG09HHRGo4l6fr/c4ZhXUQH0uFW7PZbg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-json-strings': 7.8.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-literals@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-wZ0PIXRxnwZvl9AYpqNUxpZ5BiTGrYt7kueGQ+N5FiQ7RCOD4cm8iShd6S6ggfVIWaJf2EMk8eRzAh52RfP4rQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-logical-assignment-operators@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-Mc/ALf1rmZTP4JKKEhUwiORU+vcfarFVLfcFiolKUo6sewoxSEgl36ak5t+4WamRsNr6nzjZXQjM35WsU+9vbg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-logical-assignment-operators': 7.10.4(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-member-expression-literals@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-sC3LdDBDi5x96LA+Ytekz2ZPk8i/Ck+DEuDbRAll5rknJ5XRTSaPKEYwomLcs1AA8wg9b3KjIQRsnApj+q51Ag==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-modules-amd@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-vJYQGxeKM4t8hYCKVBlZX/gtIY2I7mRGFNcm85sgXGMTBcoV3QdVtdpbcWEbzbfUIUZKwvgFT82mRvaQIebZzw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-module-transforms': 7.23.3(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-modules-commonjs@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-aVS0F65LKsdNOtcz6FRCpE4OgsP2OFnW46qNxNIX9h3wuzaNcSQsJysuMwqSibC98HPrf2vCgtxKNwS0DAlgcA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-module-transforms': 7.23.3(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-simple-access': 7.22.5 + dev: false + + /@babel/plugin-transform-modules-systemjs@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-ZxyKGTkF9xT9YJuKQRo19ewf3pXpopuYQd8cDXqNzc3mUNbOME0RKMoZxviQk74hwzfQsEe66dE92MaZbdHKNQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-hoist-variables': 7.22.5 + '@babel/helper-module-transforms': 7.23.3(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-validator-identifier': 7.22.20 + dev: false + + /@babel/plugin-transform-modules-umd@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-zHsy9iXX2nIsCBFPud3jKn1IRPWg3Ing1qOZgeKV39m1ZgIdpJqvlWVeiHBZC6ITRG0MfskhYe9cLgntfSFPIg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-module-transforms': 7.23.3(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-named-capturing-groups-regex@7.22.5(@babel/core@7.23.6): + resolution: {integrity: sha512-YgLLKmS3aUBhHaxp5hi1WJTgOUb/NCuDHzGT9z9WTt3YG+CPRhJs6nprbStx6DnWM4dh6gt7SU3sZodbZ08adQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-regexp-features-plugin': 7.22.15(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-new-target@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-YJ3xKqtJMAT5/TIZnpAR3I+K+WaDowYbN3xyxI8zxx/Gsypwf9B9h0VB+1Nh6ACAAPRS5NSRje0uVv5i79HYGQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-nullish-coalescing-operator@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-jHE9EVVqHKAQx+VePv5LLGHjmHSJR76vawFPTdlxR/LVJPfOEGxREQwQfjuZEOPTwG92X3LINSh3M40Rv4zpVA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-nullish-coalescing-operator': 7.8.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-numeric-separator@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-mps6auzgwjRrwKEZA05cOwuDc9FAzoyFS4ZsG/8F43bTLf/TgkJg7QXOrPO1JO599iA3qgK9MXdMGOEC8O1h6Q==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-numeric-separator': 7.10.4(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-object-rest-spread@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-9x9K1YyeQVw0iOXJlIzwm8ltobIIv7j2iLyP2jIhEbqPRQ7ScNgwQufU2I0Gq11VjyG4gI4yMXt2VFags+1N3g==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/compat-data': 7.23.5 + '@babel/core': 7.23.6 + '@babel/helper-compilation-targets': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-object-rest-spread': 7.8.3(@babel/core@7.23.6) + '@babel/plugin-transform-parameters': 7.23.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-object-super@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-BwQ8q0x2JG+3lxCVFohg+KbQM7plfpBwThdW9A6TMtWwLsbDA01Ek2Zb/AgDN39BiZsExm4qrXxjk+P1/fzGrA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-replace-supers': 7.22.20(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-optional-catch-binding@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-XIq8t0rJPHf6Wvmbn9nFxU6ao4c7WhghTR5WyV8SrJfUFzyxhCm4nhC+iAp3HFhbAKLfYpgzhJ6t4XCtVwqO5A==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-optional-catch-binding': 7.8.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-optional-chaining@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-ZU8y5zWOfjM5vZ+asjgAPwDaBjJzgufjES89Rs4Lpq63O300R/kOz30WCLo6BxxX6QVEilwSlpClnG5cZaikTA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-skip-transparent-expression-wrappers': 7.22.5 + '@babel/plugin-syntax-optional-chaining': 7.8.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-parameters@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-09lMt6UsUb3/34BbECKVbVwrT9bO6lILWln237z7sLaWnMsTi7Yc9fhX5DLpkJzAGfaReXI22wP41SZmnAA3Vw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-private-methods@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-UzqRcRtWsDMTLrRWFvUBDwmw06tCQH9Rl1uAjfh6ijMSmGYQ+fpdB+cnqRC8EMh5tuuxSv0/TejGL+7vyj+50g==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-class-features-plugin': 7.23.7(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-private-property-in-object@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-9G3K1YqTq3F4Vt88Djx1UZ79PDyj+yKRnUy7cZGSMe+a7jkwD259uKKuUzQlPkGam7R+8RJwh5z4xO27fA1o2A==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-create-class-features-plugin': 7.23.7(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-private-property-in-object': 7.14.5(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-property-literals@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-jR3Jn3y7cZp4oEWPFAlRsSWjxKe4PZILGBSd4nis1TsC5qeSpb+nrtihJuDhNI7QHiVbUaiXa0X2RZY3/TI6Nw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-react-constant-elements@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-zP0QKq/p6O42OL94udMgSfKXyse4RyJ0JqbQ34zDAONWjyrEsghYEyTSK5FIpmXmCpB55SHokL1cRRKHv8L2Qw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-react-display-name@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-GnvhtVfA2OAtzdX58FJxU19rhoGeQzyVndw3GgtdECQvQFXPEZIOVULHVZGAYmOgmqjXpVpfocAbSjh99V/Fqw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-react-jsx-development@7.22.5(@babel/core@7.23.6): + resolution: {integrity: sha512-bDhuzwWMuInwCYeDeMzyi7TaBgRQei6DqxhbyniL7/VG4RSS7HtSL2QbY4eESy1KJqlWt8g3xeEBGPuo+XqC8A==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/plugin-transform-react-jsx': 7.23.4(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-react-jsx@7.23.4(@babel/core@7.23.6): + resolution: {integrity: sha512-5xOpoPguCZCRbo/JeHlloSkTA8Bld1J/E1/kLfD1nsuiW1m8tduTA1ERCgIZokDflX/IBzKcqR3l7VlRgiIfHA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-module-imports': 7.22.15 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-jsx': 7.23.3(@babel/core@7.23.6) + '@babel/types': 7.23.6 + dev: false + + /@babel/plugin-transform-react-pure-annotations@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-qMFdSS+TUhB7Q/3HVPnEdYJDQIk57jkntAwSuz9xfSE4n+3I+vHYCli3HoHawN1Z3RfCz/y1zXA/JXjG6cVImQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-regenerator@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-KP+75h0KghBMcVpuKisx3XTu9Ncut8Q8TuvGO4IhY+9D5DFEckQefOuIsB/gQ2tG71lCke4NMrtIPS8pOj18BQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + regenerator-transform: 0.15.2 + dev: false + + /@babel/plugin-transform-reserved-words@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-QnNTazY54YqgGxwIexMZva9gqbPa15t/x9VS+0fsEFWplwVpXYZivtgl43Z1vMpc1bdPP2PP8siFeVcnFvA3Cg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-runtime@7.23.6(@babel/core@7.23.6): + resolution: {integrity: sha512-kF1Zg62aPseQ11orDhFRw+aPG/eynNQtI+TyY+m33qJa2cJ5EEvza2P2BNTIA9E5MyqFABHEyY6CPHwgdy9aNg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-module-imports': 7.22.15 + '@babel/helper-plugin-utils': 7.22.5 + babel-plugin-polyfill-corejs2: 0.4.7(@babel/core@7.23.6) + babel-plugin-polyfill-corejs3: 0.8.7(@babel/core@7.23.6) + babel-plugin-polyfill-regenerator: 0.5.4(@babel/core@7.23.6) + semver: 6.3.1 + transitivePeerDependencies: + - supports-color + dev: false + + /@babel/plugin-transform-shorthand-properties@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-ED2fgqZLmexWiN+YNFX26fx4gh5qHDhn1O2gvEhreLW2iI63Sqm4llRLCXALKrCnbN4Jy0VcMQZl/SAzqug/jg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-spread@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-VvfVYlrlBVu+77xVTOAoxQ6mZbnIq5FM0aGBSFEcIh03qHf+zNqA4DC/3XMUozTg7bZV3e3mZQ0i13VB6v5yUg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-skip-transparent-expression-wrappers': 7.22.5 + dev: false + + /@babel/plugin-transform-sticky-regex@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-HZOyN9g+rtvnOU3Yh7kSxXrKbzgrm5X4GncPY1QOquu7epga5MxKHVpYu2hvQnry/H+JjckSYRb93iNfsioAGg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-template-literals@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-Flok06AYNp7GV2oJPZZcP9vZdszev6vPBkHLwxwSpaIqx75wn6mUd3UFWsSsA0l8nXAKkyCmL/sR02m8RYGeHg==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-typeof-symbol@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-4t15ViVnaFdrPC74be1gXBSMzXk3B4Us9lP7uLRQHTFpV5Dvt33pn+2MyyNxmN3VTTm3oTrZVMUmuw3oBnQ2oQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-typescript@7.23.6(@babel/core@7.23.6): + resolution: {integrity: sha512-6cBG5mBvUu4VUD04OHKnYzbuHNP8huDsD3EDqqpIpsswTDoqHCjLoHb6+QgsV1WsT2nipRqCPgxD3LXnEO7XfA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-annotate-as-pure': 7.22.5 + '@babel/helper-create-class-features-plugin': 7.23.7(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + '@babel/plugin-syntax-typescript': 7.23.3(@babel/core@7.23.6) + dev: false + + /@babel/plugin-transform-unicode-escapes@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-OMCUx/bU6ChE3r4+ZdylEqAjaQgHAgipgW8nsCfu5pGqDcFytVd91AwRvUJSBZDz0exPGgnjoqhgRYLRjFZc9Q==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-unicode-property-regex@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-KcLIm+pDZkWZQAFJ9pdfmh89EwVfmNovFBcXko8szpBeF8z68kWIPeKlmSOkT9BXJxs2C0uk+5LxoxIv62MROA==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-regexp-features-plugin': 7.22.15(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-unicode-regex@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-wMHpNA4x2cIA32b/ci3AfwNgheiva2W0WUKWTK7vBHBhDKfPsc5cFGNWm69WBqpwd86u1qwZ9PWevKqm1A3yAw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-regexp-features-plugin': 7.22.15(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/plugin-transform-unicode-sets-regex@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-W7lliA/v9bNR83Qc3q1ip9CQMZ09CcHDbHfbLRDNuAhn1Mvkr1ZNF7hPmztMQvtTGVLJ9m8IZqWsTkXOml8dbw==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-create-regexp-features-plugin': 7.22.15(@babel/core@7.23.6) + '@babel/helper-plugin-utils': 7.22.5 + dev: false + + /@babel/preset-env@7.23.6(@babel/core@7.23.6): + resolution: {integrity: sha512-2XPn/BqKkZCpzYhUUNZ1ssXw7DcXfKQEjv/uXZUXgaebCMYmkEsfZ2yY+vv+xtXv50WmL5SGhyB6/xsWxIvvOQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/compat-data': 7.23.5 + '@babel/core': 7.23.6 + '@babel/helper-compilation-targets': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-validator-option': 7.23.5 + '@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-bugfix-v8-spread-parameters-in-optional-chaining': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-bugfix-v8-static-class-fields-redefine-readonly': 7.23.7(@babel/core@7.23.6) + '@babel/plugin-proposal-private-property-in-object': 7.21.0-placeholder-for-preset-env.2(@babel/core@7.23.6) + '@babel/plugin-syntax-async-generators': 7.8.4(@babel/core@7.23.6) + '@babel/plugin-syntax-class-properties': 7.12.13(@babel/core@7.23.6) + '@babel/plugin-syntax-class-static-block': 7.14.5(@babel/core@7.23.6) + '@babel/plugin-syntax-dynamic-import': 7.8.3(@babel/core@7.23.6) + '@babel/plugin-syntax-export-namespace-from': 7.8.3(@babel/core@7.23.6) + '@babel/plugin-syntax-import-assertions': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-syntax-import-attributes': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-syntax-import-meta': 7.10.4(@babel/core@7.23.6) + '@babel/plugin-syntax-json-strings': 7.8.3(@babel/core@7.23.6) + '@babel/plugin-syntax-logical-assignment-operators': 7.10.4(@babel/core@7.23.6) + '@babel/plugin-syntax-nullish-coalescing-operator': 7.8.3(@babel/core@7.23.6) + '@babel/plugin-syntax-numeric-separator': 7.10.4(@babel/core@7.23.6) + '@babel/plugin-syntax-object-rest-spread': 7.8.3(@babel/core@7.23.6) + '@babel/plugin-syntax-optional-catch-binding': 7.8.3(@babel/core@7.23.6) + '@babel/plugin-syntax-optional-chaining': 7.8.3(@babel/core@7.23.6) + '@babel/plugin-syntax-private-property-in-object': 7.14.5(@babel/core@7.23.6) + '@babel/plugin-syntax-top-level-await': 7.14.5(@babel/core@7.23.6) + '@babel/plugin-syntax-unicode-sets-regex': 7.18.6(@babel/core@7.23.6) + '@babel/plugin-transform-arrow-functions': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-async-generator-functions': 7.23.7(@babel/core@7.23.6) + '@babel/plugin-transform-async-to-generator': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-block-scoped-functions': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-block-scoping': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-class-properties': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-class-static-block': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-classes': 7.23.5(@babel/core@7.23.6) + '@babel/plugin-transform-computed-properties': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-destructuring': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-dotall-regex': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-duplicate-keys': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-dynamic-import': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-exponentiation-operator': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-export-namespace-from': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-for-of': 7.23.6(@babel/core@7.23.6) + '@babel/plugin-transform-function-name': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-json-strings': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-literals': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-logical-assignment-operators': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-member-expression-literals': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-modules-amd': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-modules-commonjs': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-modules-systemjs': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-modules-umd': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-named-capturing-groups-regex': 7.22.5(@babel/core@7.23.6) + '@babel/plugin-transform-new-target': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-nullish-coalescing-operator': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-numeric-separator': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-object-rest-spread': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-object-super': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-optional-catch-binding': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-optional-chaining': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-parameters': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-private-methods': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-private-property-in-object': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-property-literals': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-regenerator': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-reserved-words': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-shorthand-properties': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-spread': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-sticky-regex': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-template-literals': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-typeof-symbol': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-unicode-escapes': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-unicode-property-regex': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-unicode-regex': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-unicode-sets-regex': 7.23.3(@babel/core@7.23.6) + '@babel/preset-modules': 0.1.6-no-external-plugins(@babel/core@7.23.6) + babel-plugin-polyfill-corejs2: 0.4.7(@babel/core@7.23.6) + babel-plugin-polyfill-corejs3: 0.8.7(@babel/core@7.23.6) + babel-plugin-polyfill-regenerator: 0.5.4(@babel/core@7.23.6) + core-js-compat: 3.35.0 + semver: 6.3.1 + transitivePeerDependencies: + - supports-color + dev: false + + /@babel/preset-modules@0.1.6-no-external-plugins(@babel/core@7.23.6): + resolution: {integrity: sha512-HrcgcIESLm9aIR842yhJ5RWan/gebQUJ6E/E5+rf0y9o6oj7w0Br+sWuL6kEQ/o/AdfvR1Je9jG18/gnpwjEyA==} + peerDependencies: + '@babel/core': ^7.0.0-0 || ^8.0.0-0 <8.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/types': 7.23.6 + esutils: 2.0.3 + dev: false + + /@babel/preset-react@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-tbkHOS9axH6Ysf2OUEqoSZ6T3Fa2SrNH6WTWSPBboxKzdxNc9qOICeLXkNG0ZEwbQ1HY8liwOce4aN/Ceyuq6w==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-validator-option': 7.23.5 + '@babel/plugin-transform-react-display-name': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-react-jsx': 7.23.4(@babel/core@7.23.6) + '@babel/plugin-transform-react-jsx-development': 7.22.5(@babel/core@7.23.6) + '@babel/plugin-transform-react-pure-annotations': 7.23.3(@babel/core@7.23.6) + dev: false + + /@babel/preset-typescript@7.23.3(@babel/core@7.23.6): + resolution: {integrity: sha512-17oIGVlqz6CchO9RFYn5U6ZpWRZIngayYCtrPRSgANSwC2V1Jb+iP74nVxzzXJte8b8BYxrL1yY96xfhTBrNNQ==} + engines: {node: '>=6.9.0'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-plugin-utils': 7.22.5 + '@babel/helper-validator-option': 7.23.5 + '@babel/plugin-syntax-jsx': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-modules-commonjs': 7.23.3(@babel/core@7.23.6) + '@babel/plugin-transform-typescript': 7.23.6(@babel/core@7.23.6) + dev: false + + /@babel/regjsgen@0.8.0: + resolution: {integrity: sha512-x/rqGMdzj+fWZvCOYForTghzbtqPDZ5gPwaoNGHdgDfF2QA/XZbCBp4Moo5scrkAMPhB7z26XM/AaHuIJdgauA==} + dev: false + + /@babel/runtime-corejs3@7.23.6: + resolution: {integrity: sha512-Djs/ZTAnpyj0nyg7p1J6oiE/tZ9G2stqAFlLGZynrW+F3k2w2jGK2mLOBxzYIOcZYA89+c3d3wXKpYLcpwcU6w==} + engines: {node: '>=6.9.0'} + dependencies: + core-js-pure: 3.35.0 + regenerator-runtime: 0.14.1 + dev: false + + /@babel/runtime@7.23.7: + resolution: {integrity: sha512-w06OXVOFso7LcbzMiDGt+3X7Rh7Ho8MmgPoWU3rarH+8upf+wSU/grlGbWzQyr3DkdN6ZeuMFjpdwW0Q+HxobA==} + engines: {node: '>=6.9.0'} + dependencies: + regenerator-runtime: 0.14.1 + dev: false + + /@babel/template@7.22.15: + resolution: {integrity: sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/code-frame': 7.23.5 + '@babel/parser': 7.23.6 + '@babel/types': 7.23.6 + dev: false + + /@babel/traverse@7.23.7: + resolution: {integrity: sha512-tY3mM8rH9jM0YHFGyfC0/xf+SB5eKUu7HPj7/k3fpi9dAlsMc5YbQvDi0Sh2QTPXqMhyaAtzAr807TIyfQrmyg==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/code-frame': 7.23.5 + '@babel/generator': 7.23.6 + '@babel/helper-environment-visitor': 7.22.20 + '@babel/helper-function-name': 7.23.0 + '@babel/helper-hoist-variables': 7.22.5 + '@babel/helper-split-export-declaration': 7.22.6 + '@babel/parser': 7.23.6 + '@babel/types': 7.23.6 + debug: 4.3.4 + globals: 11.12.0 + transitivePeerDependencies: + - supports-color + dev: false + + /@babel/types@7.23.6: + resolution: {integrity: sha512-+uarb83brBzPKN38NX1MkB6vb6+mwvR6amUulqAE7ccQw1pEl+bCia9TbdG1lsnFP7lZySvUn37CHyXQdfTwzg==} + engines: {node: '>=6.9.0'} + dependencies: + '@babel/helper-string-parser': 7.23.4 + '@babel/helper-validator-identifier': 7.22.20 + to-fast-properties: 2.0.0 + dev: false + + /@braintree/sanitize-url@6.0.4: + resolution: {integrity: sha512-s3jaWicZd0pkP0jf5ysyHUI/RE7MHos6qlToFcGWXVp+ykHOy77OUMrfbgJ9it2C5bow7OIQwYYaHjk9XlBQ2A==} + dev: false + + /@cmfcmf/docusaurus-search-local@1.1.0(@docusaurus/core@3.0.1)(search-insights@2.13.0): + resolution: {integrity: sha512-0IVb/aA0IK8ZlktuxmgXmluXfcSpo6Vdd2nG21y1aOH9nVYnPP231Dn0H8Ng9Qf9ronQQCDWHnuWpYOr9rUrEQ==} + peerDependencies: + '@docusaurus/core': ^2.0.0 + nodejieba: ^2.5.0 + peerDependenciesMeta: + nodejieba: + optional: true + dependencies: + '@algolia/autocomplete-js': 1.13.0(@algolia/client-search@4.22.0)(algoliasearch@4.22.0)(search-insights@2.13.0) + '@algolia/autocomplete-theme-classic': 1.13.0 + '@algolia/client-search': 4.22.0 + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + algoliasearch: 4.22.0 + cheerio: 1.0.0-rc.12 + clsx: 1.1.1 + lunr-languages: 1.14.0 + mark.js: 8.11.1 + transitivePeerDependencies: + - search-insights + dev: false + + /@colors/colors@1.5.0: + resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==} + engines: {node: '>=0.1.90'} + requiresBuild: true + dev: false + optional: true + + /@discoveryjs/json-ext@0.5.7: + resolution: {integrity: sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==} + engines: {node: '>=10.0.0'} + dev: false + + /@docsearch/css@3.1.0: + resolution: {integrity: sha512-bh5IskwkkodbvC0FzSg1AxMykfDl95hebEKwxNoq4e5QaGzOXSBgW8+jnMFZ7JU4sTBiB04vZWoUSzNrPboLZA==} + dev: false + + /@docsearch/css@3.5.2: + resolution: {integrity: sha512-SPiDHaWKQZpwR2siD0KQUwlStvIAnEyK6tAE2h2Wuoq8ue9skzhlyVQ1ddzOxX6khULnAALDiR/isSF3bnuciA==} + dev: false + + /@docsearch/react@3.1.0(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-bjB6ExnZzf++5B7Tfoi6UXgNwoUnNOfZ1NyvnvPhWgCMy5V/biAtLL4o7owmZSYdAKeFSvZ5Lxm0is4su/dBWg==} + peerDependencies: + '@types/react': '>= 16.8.0 < 19.0.0' + react: '>= 16.8.0 < 19.0.0' + react-dom: '>= 16.8.0 < 19.0.0' + dependencies: + '@algolia/autocomplete-core': 1.6.3 + '@docsearch/css': 3.1.0 + '@types/react': 18.2.46 + algoliasearch: 4.22.0 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + dev: false + + /@docsearch/react@3.5.2(@algolia/client-search@4.22.0)(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0): + resolution: {integrity: sha512-9Ahcrs5z2jq/DcAvYtvlqEBHImbm4YJI8M9y0x6Tqg598P40HTEkX7hsMcIuThI+hTFxRGZ9hll0Wygm2yEjng==} + peerDependencies: + '@types/react': '>= 16.8.0 < 19.0.0' + react: '>= 16.8.0 < 19.0.0' + react-dom: '>= 16.8.0 < 19.0.0' + search-insights: '>= 1 < 3' + peerDependenciesMeta: + '@types/react': + optional: true + react: + optional: true + react-dom: + optional: true + search-insights: + optional: true + dependencies: + '@algolia/autocomplete-core': 1.9.3(@algolia/client-search@4.22.0)(algoliasearch@4.22.0)(search-insights@2.13.0) + '@algolia/autocomplete-preset-algolia': 1.9.3(@algolia/client-search@4.22.0)(algoliasearch@4.22.0) + '@docsearch/css': 3.5.2 + '@types/react': 18.2.46 + algoliasearch: 4.22.0 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + search-insights: 2.13.0 + transitivePeerDependencies: + - '@algolia/client-search' + dev: false + + /@docusaurus/core@3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-CXrLpOnW+dJdSv8M5FAJ3JBwXtL6mhUWxFA8aS0ozK6jBG/wgxERk5uvH28fCeFxOGbAT9v1e9dOMo1X2IEVhQ==} + engines: {node: '>=18.0'} + hasBin: true + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/generator': 7.23.6 + '@babel/plugin-syntax-dynamic-import': 7.8.3(@babel/core@7.23.6) + '@babel/plugin-transform-runtime': 7.23.6(@babel/core@7.23.6) + '@babel/preset-env': 7.23.6(@babel/core@7.23.6) + '@babel/preset-react': 7.23.3(@babel/core@7.23.6) + '@babel/preset-typescript': 7.23.3(@babel/core@7.23.6) + '@babel/runtime': 7.23.7 + '@babel/runtime-corejs3': 7.23.6 + '@babel/traverse': 7.23.7 + '@docusaurus/cssnano-preset': 3.0.1 + '@docusaurus/logger': 3.0.1 + '@docusaurus/mdx-loader': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/react-loadable': 5.5.2(react@18.2.0) + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-common': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + '@slorber/static-site-generator-webpack-plugin': 4.0.7 + '@svgr/webpack': 6.5.1 + autoprefixer: 10.4.16(postcss@8.4.32) + babel-loader: 9.1.3(@babel/core@7.23.6)(webpack@5.89.0) + babel-plugin-dynamic-import-node: 2.3.3 + boxen: 6.2.1 + chalk: 4.1.2 + chokidar: 3.5.3 + clean-css: 5.3.3 + cli-table3: 0.6.3 + combine-promises: 1.2.0 + commander: 5.1.0 + copy-webpack-plugin: 11.0.0(webpack@5.89.0) + core-js: 3.35.0 + css-loader: 6.8.1(webpack@5.89.0) + css-minimizer-webpack-plugin: 4.2.2(clean-css@5.3.3)(webpack@5.89.0) + cssnano: 5.1.15(postcss@8.4.32) + del: 6.1.1 + detect-port: 1.5.1 + escape-html: 1.0.3 + eta: 2.2.0 + file-loader: 6.2.0(webpack@5.89.0) + fs-extra: 11.2.0 + html-minifier-terser: 7.2.0 + html-tags: 3.3.1 + html-webpack-plugin: 5.6.0(webpack@5.89.0) + leven: 3.1.0 + lodash: 4.17.21 + mini-css-extract-plugin: 2.7.6(webpack@5.89.0) + postcss: 8.4.32 + postcss-loader: 7.3.4(postcss@8.4.32)(typescript@5.3.3)(webpack@5.89.0) + prompts: 2.4.2 + react: 18.2.0 + react-dev-utils: 12.0.1(typescript@5.3.3)(webpack@5.89.0) + react-dom: 18.2.0(react@18.2.0) + react-helmet-async: 1.3.0(react-dom@18.2.0)(react@18.2.0) + react-loadable: /@docusaurus/react-loadable@5.5.2(react@18.2.0) + react-loadable-ssr-addon-v5-slorber: 1.0.1(@docusaurus/react-loadable@5.5.2)(webpack@5.89.0) + react-router: 5.3.4(react@18.2.0) + react-router-config: 5.1.1(react-router@5.3.4)(react@18.2.0) + react-router-dom: 5.3.4(react@18.2.0) + rtl-detect: 1.1.2 + semver: 7.5.4 + serve-handler: 6.1.5 + shelljs: 0.8.5 + terser-webpack-plugin: 5.3.10(webpack@5.89.0) + tslib: 2.6.2 + update-notifier: 6.0.2 + url-loader: 4.1.1(file-loader@6.2.0)(webpack@5.89.0) + webpack: 5.89.0 + webpack-bundle-analyzer: 4.10.1 + webpack-dev-server: 4.15.1(webpack@5.89.0) + webpack-merge: 5.10.0 + webpackbar: 5.0.2(webpack@5.89.0) + transitivePeerDependencies: + - '@docusaurus/types' + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/cssnano-preset@3.0.1: + resolution: {integrity: sha512-wjuXzkHMW+ig4BD6Ya1Yevx9UJadO4smNZCEljqBoQfIQrQskTswBs7lZ8InHP7mCt273a/y/rm36EZhqJhknQ==} + engines: {node: '>=18.0'} + dependencies: + cssnano-preset-advanced: 5.3.10(postcss@8.4.32) + postcss: 8.4.32 + postcss-sort-media-queries: 4.4.1(postcss@8.4.32) + tslib: 2.6.2 + dev: false + + /@docusaurus/logger@3.0.1: + resolution: {integrity: sha512-I5L6Nk8OJzkVA91O2uftmo71LBSxe1vmOn9AMR6JRCzYeEBrqneWMH02AqMvjJ2NpMiviO+t0CyPjyYV7nxCWQ==} + engines: {node: '>=18.0'} + dependencies: + chalk: 4.1.2 + tslib: 2.6.2 + dev: false + + /@docusaurus/mdx-loader@3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-ldnTmvnvlrONUq45oKESrpy+lXtbnTcTsFkOTIDswe5xx5iWJjt6eSa0f99ZaWlnm24mlojcIGoUWNCS53qVlQ==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@babel/parser': 7.23.6 + '@babel/traverse': 7.23.7 + '@docusaurus/logger': 3.0.1 + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + '@mdx-js/mdx': 3.0.0 + '@slorber/remark-comment': 1.0.0 + escape-html: 1.0.3 + estree-util-value-to-estree: 3.0.1 + file-loader: 6.2.0(webpack@5.89.0) + fs-extra: 11.2.0 + image-size: 1.1.1 + mdast-util-mdx: 3.0.0 + mdast-util-to-string: 4.0.0 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + rehype-raw: 7.0.0 + remark-directive: 3.0.0 + remark-emoji: 4.0.1 + remark-frontmatter: 5.0.0 + remark-gfm: 4.0.0 + stringify-object: 3.3.0 + tslib: 2.6.2 + unified: 11.0.4 + unist-util-visit: 5.0.0 + url-loader: 4.1.1(file-loader@6.2.0)(webpack@5.89.0) + vfile: 6.0.1 + webpack: 5.89.0 + transitivePeerDependencies: + - '@docusaurus/types' + - '@swc/core' + - esbuild + - supports-color + - uglify-js + - webpack-cli + dev: false + + /@docusaurus/module-type-aliases@3.0.1(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-DEHpeqUDsLynl3AhQQiO7AbC7/z/lBra34jTcdYuvp9eGm01pfH1wTVq8YqWZq6Jyx0BgcVl/VJqtE9StRd9Ag==} + peerDependencies: + react: '*' + react-dom: '*' + dependencies: + '@docusaurus/react-loadable': 5.5.2(react@18.2.0) + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@types/history': 4.7.11 + '@types/react': 18.2.46 + '@types/react-router-config': 5.0.11 + '@types/react-router-dom': 5.3.3 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + react-helmet-async: 2.0.4(react-dom@18.2.0)(react@18.2.0) + react-loadable: /@docusaurus/react-loadable@5.5.2(react@18.2.0) + transitivePeerDependencies: + - '@swc/core' + - esbuild + - uglify-js + - webpack-cli + dev: false + + /@docusaurus/plugin-client-redirects@3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-CoZapnHbV3j5jsHCa/zmKaa8+H+oagHBgg91dN5I8/3kFit/xtZPfRaznvDX49cHg2nSoV74B3VMAT+bvCmzFQ==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/logger': 3.0.1 + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-common': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + eta: 2.2.0 + fs-extra: 11.2.0 + lodash: 4.17.21 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + tslib: 2.6.2 + transitivePeerDependencies: + - '@docusaurus/types' + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/plugin-content-blog@3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-cLOvtvAyaMQFLI8vm4j26svg3ktxMPSXpuUJ7EERKoGbfpJSsgtowNHcRsaBVmfuCsRSk1HZ/yHBsUkTmHFEsg==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/logger': 3.0.1 + '@docusaurus/mdx-loader': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-common': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + cheerio: 1.0.0-rc.12 + feed: 4.2.2 + fs-extra: 11.2.0 + lodash: 4.17.21 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + reading-time: 1.5.0 + srcset: 4.0.0 + tslib: 2.6.2 + unist-util-visit: 5.0.0 + utility-types: 3.10.0 + webpack: 5.89.0 + transitivePeerDependencies: + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/plugin-content-docs@3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-dRfAOA5Ivo+sdzzJGXEu33yAtvGg8dlZkvt/NEJ7nwi1F2j4LEdsxtfX2GKeETB2fP6XoGNSQnFXqa2NYGrHFg==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/logger': 3.0.1 + '@docusaurus/mdx-loader': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/module-type-aliases': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + '@types/react-router-config': 5.0.11 + combine-promises: 1.2.0 + fs-extra: 11.2.0 + js-yaml: 4.1.0 + lodash: 4.17.21 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + tslib: 2.6.2 + utility-types: 3.10.0 + webpack: 5.89.0 + transitivePeerDependencies: + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/plugin-content-pages@3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-oP7PoYizKAXyEttcvVzfX3OoBIXEmXTMzCdfmC4oSwjG4SPcJsRge3mmI6O8jcZBgUPjIzXD21bVGWEE1iu8gg==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/mdx-loader': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + fs-extra: 11.2.0 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + tslib: 2.6.2 + webpack: 5.89.0 + transitivePeerDependencies: + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/plugin-debug@3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-09dxZMdATky4qdsZGzhzlUvvC+ilQ2hKbYF+wez+cM2mGo4qHbv8+qKXqxq0CQZyimwlAOWQLoSozIXU0g0i7g==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + fs-extra: 11.2.0 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + react-json-view-lite: 1.2.1(react@18.2.0) + tslib: 2.6.2 + transitivePeerDependencies: + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/plugin-google-analytics@3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-jwseSz1E+g9rXQwDdr0ZdYNjn8leZBnKPjjQhMBEiwDoenL3JYFcNW0+p0sWoVF/f2z5t7HkKA+cYObrUh18gg==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + tslib: 2.6.2 + transitivePeerDependencies: + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/plugin-google-gtag@3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-UFTDvXniAWrajsulKUJ1DB6qplui1BlKLQZjX4F7qS/qfJ+qkKqSkhJ/F4VuGQ2JYeZstYb+KaUzUzvaPK1aRQ==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + '@types/gtag.js': 0.0.12 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + tslib: 2.6.2 + transitivePeerDependencies: + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/plugin-google-tag-manager@3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-IPFvuz83aFuheZcWpTlAdiiX1RqWIHM+OH8wS66JgwAKOiQMR3+nLywGjkLV4bp52x7nCnwhNk1rE85Cpy/CIw==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + tslib: 2.6.2 + transitivePeerDependencies: + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/plugin-sitemap@3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-xARiWnjtVvoEniZudlCq5T9ifnhCu/GAZ5nA7XgyLfPcNpHQa241HZdsTlLtVcecEVVdllevBKOp7qknBBaMGw==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/logger': 3.0.1 + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-common': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + fs-extra: 11.2.0 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + sitemap: 7.1.1 + tslib: 2.6.2 + transitivePeerDependencies: + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/preset-classic@3.0.1(@algolia/client-search@4.22.0)(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0)(typescript@5.3.3): + resolution: {integrity: sha512-il9m9xZKKjoXn6h0cRcdnt6wce0Pv1y5t4xk2Wx7zBGhKG1idu4IFHtikHlD0QPuZ9fizpXspXcTzjL5FXc1Gw==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-content-blog': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-content-docs': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-content-pages': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-debug': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-google-analytics': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-google-gtag': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-google-tag-manager': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-sitemap': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/theme-classic': 3.0.1(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/theme-common': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/theme-search-algolia': 3.0.1(@algolia/client-search@4.22.0)(@docusaurus/types@3.0.1)(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0)(typescript@5.3.3) + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + transitivePeerDependencies: + - '@algolia/client-search' + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - '@types/react' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - search-insights + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/react-loadable@5.5.2(react@18.2.0): + resolution: {integrity: sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ==} + peerDependencies: + react: '*' + dependencies: + '@types/react': 18.2.46 + prop-types: 15.8.1 + react: 18.2.0 + dev: false + + /@docusaurus/theme-classic@3.0.1(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-XD1FRXaJiDlmYaiHHdm27PNhhPboUah9rqIH0lMpBt5kYtsGjJzhqa27KuZvHLzOP2OEpqd2+GZ5b6YPq7Q05Q==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/mdx-loader': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/module-type-aliases': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/plugin-content-blog': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-content-docs': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-content-pages': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/theme-common': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/theme-translations': 3.0.1 + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-common': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + '@mdx-js/react': 3.0.0(@types/react@18.2.46)(react@18.2.0) + clsx: 2.1.0 + copy-text-to-clipboard: 3.2.0 + infima: 0.2.0-alpha.43 + lodash: 4.17.21 + nprogress: 0.2.0 + postcss: 8.4.32 + prism-react-renderer: 2.3.1(react@18.2.0) + prismjs: 1.29.0 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + react-router-dom: 5.3.4(react@18.2.0) + rtlcss: 4.1.1 + tslib: 2.6.2 + utility-types: 3.10.0 + transitivePeerDependencies: + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - '@types/react' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/theme-common@3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-cr9TOWXuIOL0PUfuXv6L5lPlTgaphKP+22NdVBOYah5jSq5XAAulJTjfe+IfLsEG4L7lJttLbhW7LXDFSAI7Ag==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/mdx-loader': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/module-type-aliases': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/plugin-content-blog': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-content-docs': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/plugin-content-pages': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-common': 3.0.1(@docusaurus/types@3.0.1) + '@types/history': 4.7.11 + '@types/react': 18.2.46 + '@types/react-router-config': 5.0.11 + clsx: 2.1.0 + parse-numeric-range: 1.3.0 + prism-react-renderer: 2.3.1(react@18.2.0) + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + tslib: 2.6.2 + utility-types: 3.10.0 + transitivePeerDependencies: + - '@docusaurus/types' + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/theme-mermaid@3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3): + resolution: {integrity: sha512-jquSDnZfazABnC5i+02GzRIvufXKruKgvbYkQjKbI7/LWo0XvBs0uKAcCDGgHhth0t/ON5+Sn27joRfpeSk3Lw==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/module-type-aliases': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/theme-common': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + mermaid: 10.6.1 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + tslib: 2.6.2 + transitivePeerDependencies: + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/theme-search-algolia@3.0.1(@algolia/client-search@4.22.0)(@docusaurus/types@3.0.1)(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0)(typescript@5.3.3): + resolution: {integrity: sha512-DDiPc0/xmKSEdwFkXNf1/vH1SzJPzuJBar8kMcBbDAZk/SAmo/4lf6GU2drou4Ae60lN2waix+jYWTWcJRahSA==} + engines: {node: '>=18.0'} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@docsearch/react': 3.5.2(@algolia/client-search@4.22.0)(@types/react@18.2.46)(react-dom@18.2.0)(react@18.2.0)(search-insights@2.13.0) + '@docusaurus/core': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/logger': 3.0.1 + '@docusaurus/plugin-content-docs': 3.0.1(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/theme-common': 3.0.1(@docusaurus/types@3.0.1)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.3) + '@docusaurus/theme-translations': 3.0.1 + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + '@docusaurus/utils-validation': 3.0.1(@docusaurus/types@3.0.1) + algoliasearch: 4.22.0 + algoliasearch-helper: 3.16.1(algoliasearch@4.22.0) + clsx: 2.1.0 + eta: 2.2.0 + fs-extra: 11.2.0 + lodash: 4.17.21 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + tslib: 2.6.2 + utility-types: 3.10.0 + transitivePeerDependencies: + - '@algolia/client-search' + - '@docusaurus/types' + - '@parcel/css' + - '@rspack/core' + - '@swc/core' + - '@swc/css' + - '@types/react' + - bufferutil + - csso + - debug + - esbuild + - eslint + - lightningcss + - search-insights + - supports-color + - typescript + - uglify-js + - utf-8-validate + - vue-template-compiler + - webpack-cli + dev: false + + /@docusaurus/theme-translations@3.0.1: + resolution: {integrity: sha512-6UrbpzCTN6NIJnAtZ6Ne9492vmPVX+7Fsz4kmp+yor3KQwA1+MCzQP7ItDNkP38UmVLnvB/cYk/IvehCUqS3dg==} + engines: {node: '>=18.0'} + dependencies: + fs-extra: 11.2.0 + tslib: 2.6.2 + dev: false + + /@docusaurus/types@3.0.1(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-plyX2iU1tcUsF46uQ01pAd4JhexR7n0iiQ5MSnBFX6M6NSJgDYdru/i1/YNPKOnQHBoXGLHv0dNT6OAlDWNjrg==} + peerDependencies: + react: ^18.0.0 + react-dom: ^18.0.0 + dependencies: + '@types/history': 4.7.11 + '@types/react': 18.2.46 + commander: 5.1.0 + joi: 17.11.0 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + react-helmet-async: 1.3.0(react-dom@18.2.0)(react@18.2.0) + utility-types: 3.10.0 + webpack: 5.89.0 + webpack-merge: 5.10.0 + transitivePeerDependencies: + - '@swc/core' + - esbuild + - uglify-js + - webpack-cli + dev: false + + /@docusaurus/utils-common@3.0.1(@docusaurus/types@3.0.1): + resolution: {integrity: sha512-W0AxD6w6T8g6bNro8nBRWf7PeZ/nn7geEWM335qHU2DDDjHuV4UZjgUGP1AQsdcSikPrlIqTJJbKzer1lRSlIg==} + engines: {node: '>=18.0'} + peerDependencies: + '@docusaurus/types': '*' + peerDependenciesMeta: + '@docusaurus/types': + optional: true + dependencies: + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + tslib: 2.6.2 + dev: false + + /@docusaurus/utils-validation@3.0.1(@docusaurus/types@3.0.1): + resolution: {integrity: sha512-ujTnqSfyGQ7/4iZdB4RRuHKY/Nwm58IIb+41s5tCXOv/MBU2wGAjOHq3U+AEyJ8aKQcHbxvTKJaRchNHYUVUQg==} + engines: {node: '>=18.0'} + dependencies: + '@docusaurus/logger': 3.0.1 + '@docusaurus/utils': 3.0.1(@docusaurus/types@3.0.1) + joi: 17.11.0 + js-yaml: 4.1.0 + tslib: 2.6.2 + transitivePeerDependencies: + - '@docusaurus/types' + - '@swc/core' + - esbuild + - supports-color + - uglify-js + - webpack-cli + dev: false + + /@docusaurus/utils@3.0.1(@docusaurus/types@3.0.1): + resolution: {integrity: sha512-TwZ33Am0q4IIbvjhUOs+zpjtD/mXNmLmEgeTGuRq01QzulLHuPhaBTTAC/DHu6kFx3wDgmgpAlaRuCHfTcXv8g==} + engines: {node: '>=18.0'} + peerDependencies: + '@docusaurus/types': '*' + peerDependenciesMeta: + '@docusaurus/types': + optional: true + dependencies: + '@docusaurus/logger': 3.0.1 + '@docusaurus/types': 3.0.1(react-dom@18.2.0)(react@18.2.0) + '@svgr/webpack': 6.5.1 + escape-string-regexp: 4.0.0 + file-loader: 6.2.0(webpack@5.89.0) + fs-extra: 11.2.0 + github-slugger: 1.5.0 + globby: 11.1.0 + gray-matter: 4.0.3 + jiti: 1.21.0 + js-yaml: 4.1.0 + lodash: 4.17.21 + micromatch: 4.0.5 + resolve-pathname: 3.0.0 + shelljs: 0.8.5 + tslib: 2.6.2 + url-loader: 4.1.1(file-loader@6.2.0)(webpack@5.89.0) + webpack: 5.89.0 + transitivePeerDependencies: + - '@swc/core' + - esbuild + - supports-color + - uglify-js + - webpack-cli + dev: false + + /@fortawesome/fontawesome-common-types@6.5.1: + resolution: {integrity: sha512-GkWzv+L6d2bI5f/Vk6ikJ9xtl7dfXtoRu3YGE6nq0p/FFqA1ebMOAWg3XgRyb0I6LYyYkiAo+3/KrwuBp8xG7A==} + engines: {node: '>=6'} + requiresBuild: true + dev: false + + /@fortawesome/fontawesome-svg-core@6.5.1: + resolution: {integrity: sha512-MfRCYlQPXoLlpem+egxjfkEuP9UQswTrlCOsknus/NcMoblTH2g0jPrapbcIb04KGA7E2GZxbAccGZfWoYgsrQ==} + engines: {node: '>=6'} + requiresBuild: true + dependencies: + '@fortawesome/fontawesome-common-types': 6.5.1 + dev: false + + /@fortawesome/free-regular-svg-icons@6.5.1: + resolution: {integrity: sha512-m6ShXn+wvqEU69wSP84coxLbNl7sGVZb+Ca+XZq6k30SzuP3X4TfPqtycgUh9ASwlNh5OfQCd8pDIWxl+O+LlQ==} + engines: {node: '>=6'} + requiresBuild: true + dependencies: + '@fortawesome/fontawesome-common-types': 6.5.1 + dev: false + + /@fortawesome/free-solid-svg-icons@6.5.1: + resolution: {integrity: sha512-S1PPfU3mIJa59biTtXJz1oI0+KAXW6bkAb31XKhxdxtuXDiUIFsih4JR1v5BbxY7hVHsD1RKq+jRkVRaf773NQ==} + engines: {node: '>=6'} + requiresBuild: true + dependencies: + '@fortawesome/fontawesome-common-types': 6.5.1 + dev: false + + /@fortawesome/react-fontawesome@0.2.0(@fortawesome/fontawesome-svg-core@6.5.1)(react@18.2.0): + resolution: {integrity: sha512-uHg75Rb/XORTtVt7OS9WoK8uM276Ufi7gCzshVWkUJbHhh3svsUUeqXerrM96Wm7fRiDzfKRwSoahhMIkGAYHw==} + peerDependencies: + '@fortawesome/fontawesome-svg-core': ~1 || ~6 + react: '>=16.3' + dependencies: + '@fortawesome/fontawesome-svg-core': 6.5.1 + prop-types: 15.8.1 + react: 18.2.0 + dev: false + + /@hapi/hoek@9.3.0: + resolution: {integrity: sha512-/c6rf4UJlmHlC9b5BaNvzAcFv7HZ2QHaV0D4/HNlBdvFnvQq8RI4kYdhyPCl7Xj+oWvTWQ8ujhqS53LIgAe6KQ==} + dev: false + + /@hapi/topo@5.1.0: + resolution: {integrity: sha512-foQZKJig7Ob0BMAYBfcJk8d77QtOe7Wo4ox7ff1lQYoNNAb6jwcY1ncdoy2e9wQZzvNy7ODZCYJkK8kzmcAnAg==} + dependencies: + '@hapi/hoek': 9.3.0 + dev: false + + /@jest/schemas@29.6.3: + resolution: {integrity: sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@sinclair/typebox': 0.27.8 + dev: false + + /@jest/types@29.6.3: + resolution: {integrity: sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@jest/schemas': 29.6.3 + '@types/istanbul-lib-coverage': 2.0.6 + '@types/istanbul-reports': 3.0.4 + '@types/node': 20.10.6 + '@types/yargs': 17.0.32 + chalk: 4.1.2 + dev: false + + /@jridgewell/gen-mapping@0.3.3: + resolution: {integrity: sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==} + engines: {node: '>=6.0.0'} + dependencies: + '@jridgewell/set-array': 1.1.2 + '@jridgewell/sourcemap-codec': 1.4.15 + '@jridgewell/trace-mapping': 0.3.20 + dev: false + + /@jridgewell/resolve-uri@3.1.1: + resolution: {integrity: sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==} + engines: {node: '>=6.0.0'} + dev: false + + /@jridgewell/set-array@1.1.2: + resolution: {integrity: sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==} + engines: {node: '>=6.0.0'} + dev: false + + /@jridgewell/source-map@0.3.5: + resolution: {integrity: sha512-UTYAUj/wviwdsMfzoSJspJxbkH5o1snzwX0//0ENX1u/55kkZZkcTZP6u9bwKGkv+dkk9at4m1Cpt0uY80kcpQ==} + dependencies: + '@jridgewell/gen-mapping': 0.3.3 + '@jridgewell/trace-mapping': 0.3.20 + dev: false + + /@jridgewell/sourcemap-codec@1.4.15: + resolution: {integrity: sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==} + dev: false + + /@jridgewell/trace-mapping@0.3.20: + resolution: {integrity: sha512-R8LcPeWZol2zR8mmH3JeKQ6QRCFb7XgUhV9ZlGhHLGyg4wpPiPZNQOOWhFZhxKw8u//yTbNGI42Bx/3paXEQ+Q==} + dependencies: + '@jridgewell/resolve-uri': 3.1.1 + '@jridgewell/sourcemap-codec': 1.4.15 + dev: false + + /@leichtgewicht/ip-codec@2.0.4: + resolution: {integrity: sha512-Hcv+nVC0kZnQ3tD9GVu5xSMR4VVYOteQIr/hwFPVEvPdlXqgGEuRjiheChHgdM+JyqdgNcmzZOX/tnl0JOiI7A==} + dev: false + + /@mdx-js/mdx@3.0.0: + resolution: {integrity: sha512-Icm0TBKBLYqroYbNW3BPnzMGn+7mwpQOK310aZ7+fkCtiU3aqv2cdcX+nd0Ydo3wI5Rx8bX2Z2QmGb/XcAClCw==} + dependencies: + '@types/estree': 1.0.5 + '@types/estree-jsx': 1.0.3 + '@types/hast': 3.0.3 + '@types/mdx': 2.0.10 + collapse-white-space: 2.1.0 + devlop: 1.1.0 + estree-util-build-jsx: 3.0.1 + estree-util-is-identifier-name: 3.0.0 + estree-util-to-js: 2.0.0 + estree-walker: 3.0.3 + hast-util-to-estree: 3.1.0 + hast-util-to-jsx-runtime: 2.3.0 + markdown-extensions: 2.0.0 + periscopic: 3.1.0 + remark-mdx: 3.0.0 + remark-parse: 11.0.0 + remark-rehype: 11.0.0 + source-map: 0.7.4 + unified: 11.0.4 + unist-util-position-from-estree: 2.0.0 + unist-util-stringify-position: 4.0.0 + unist-util-visit: 5.0.0 + vfile: 6.0.1 + transitivePeerDependencies: + - supports-color + dev: false + + /@mdx-js/react@3.0.0(@types/react@18.2.46)(react@18.2.0): + resolution: {integrity: sha512-nDctevR9KyYFyV+m+/+S4cpzCWHqj+iHDHq3QrsWezcC+B17uZdIWgCguESUkwFhM3n/56KxWVE3V6EokrmONQ==} + peerDependencies: + '@types/react': '>=16' + react: '>=16' + dependencies: + '@types/mdx': 2.0.10 + '@types/react': 18.2.46 + react: 18.2.0 + dev: false + + /@ndhoule/each@2.0.1: + resolution: {integrity: sha512-wHuJw6x+rF6Q9Skgra++KccjBozCr9ymtna0FhxmV/8xT/hZ2ExGYR8SV8prg8x4AH/7mzDYErNGIVHuzHeybw==} + dependencies: + '@ndhoule/keys': 2.0.0 + dev: false + + /@ndhoule/keys@2.0.0: + resolution: {integrity: sha512-vtCqKBC1Av6dsBA8xpAO+cgk051nfaI+PnmTZep2Px0vYrDvpUmLxv7z40COlWH5yCpu3gzNhepk+02yiQiZNw==} + dev: false + + /@ndhoule/map@2.0.1: + resolution: {integrity: sha512-WOEf2An9mL4DVY6NHgaRmFC82pZGrmzW4I0hpPPdczDP4Gp5+Q1Nny77x3w0qzENA8+cbgd9+Lx2ClSTLvkB0g==} + dependencies: + '@ndhoule/each': 2.0.1 + dev: false + + /@nodelib/fs.scandir@2.1.5: + resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} + engines: {node: '>= 8'} + dependencies: + '@nodelib/fs.stat': 2.0.5 + run-parallel: 1.2.0 + dev: false + + /@nodelib/fs.stat@2.0.5: + resolution: {integrity: sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==} + engines: {node: '>= 8'} + dev: false + + /@nodelib/fs.walk@1.2.8: + resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} + engines: {node: '>= 8'} + dependencies: + '@nodelib/fs.scandir': 2.1.5 + fastq: 1.16.0 + dev: false + + /@pnpm/config.env-replace@1.1.0: + resolution: {integrity: sha512-htyl8TWnKL7K/ESFa1oW2UB5lVDxuF5DpM7tBi6Hu2LNL3mWkIzNLG6N4zoCUP1lCKNxWy/3iu8mS8MvToGd6w==} + engines: {node: '>=12.22.0'} + dev: false + + /@pnpm/network.ca-file@1.0.2: + resolution: {integrity: sha512-YcPQ8a0jwYU9bTdJDpXjMi7Brhkr1mXsXrUJvjqM2mQDgkRiz8jFaQGOdaLxgjtUfQgZhKy/O3cG/YwmgKaxLA==} + engines: {node: '>=12.22.0'} + dependencies: + graceful-fs: 4.2.10 + dev: false + + /@pnpm/npm-conf@2.2.2: + resolution: {integrity: sha512-UA91GwWPhFExt3IizW6bOeY/pQ0BkuNwKjk9iQW9KqxluGCrg4VenZ0/L+2Y0+ZOtme72EVvg6v0zo3AMQRCeA==} + engines: {node: '>=12'} + dependencies: + '@pnpm/config.env-replace': 1.1.0 + '@pnpm/network.ca-file': 1.0.2 + config-chain: 1.1.13 + dev: false + + /@polka/url@1.0.0-next.24: + resolution: {integrity: sha512-2LuNTFBIO0m7kKIQvvPHN6UE63VjpmL9rnEEaOOaiSPbZK+zUOYIzBAWcED+3XYzhYsd/0mD57VdxAEqqV52CQ==} + dev: false + + /@segment/snippet@4.16.2: + resolution: {integrity: sha512-2fgsrt4U+vKv14ohOAsViCEzeZotaawF2Il7YUbmYVrhPn8Hq7xuGznHKRdZeoxScQ87X36xDX2Fzh5bAYRN7g==} + dependencies: + '@ndhoule/map': 2.0.1 + dev: false + + /@sideway/address@4.1.4: + resolution: {integrity: sha512-7vwq+rOHVWjyXxVlR76Agnvhy8I9rpzjosTESvmhNeXOXdZZB15Fl+TI9x1SiHZH5Jv2wTGduSxFDIaq0m3DUw==} + dependencies: + '@hapi/hoek': 9.3.0 + dev: false + + /@sideway/formula@3.0.1: + resolution: {integrity: sha512-/poHZJJVjx3L+zVD6g9KgHfYnb443oi7wLu/XKojDviHy6HOEOA6z1Trk5aR1dGcmPenJEgb2sK2I80LeS3MIg==} + dev: false + + /@sideway/pinpoint@2.0.0: + resolution: {integrity: sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ==} + dev: false + + /@sinclair/typebox@0.27.8: + resolution: {integrity: sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==} + dev: false + + /@sindresorhus/is@4.6.0: + resolution: {integrity: sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==} + engines: {node: '>=10'} + dev: false + + /@sindresorhus/is@5.6.0: + resolution: {integrity: sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==} + engines: {node: '>=14.16'} + dev: false + + /@slorber/remark-comment@1.0.0: + resolution: {integrity: sha512-RCE24n7jsOj1M0UPvIQCHTe7fI0sFL4S2nwKVWwHyVr/wI/H8GosgsJGyhnsZoGFnD/P2hLf1mSbrrgSLN93NA==} + dependencies: + micromark-factory-space: 1.1.0 + micromark-util-character: 1.2.0 + micromark-util-symbol: 1.1.0 + dev: false + + /@slorber/static-site-generator-webpack-plugin@4.0.7: + resolution: {integrity: sha512-Ug7x6z5lwrz0WqdnNFOMYrDQNTPAprvHLSh6+/fmml3qUiz6l5eq+2MzLKWtn/q5K5NpSiFsZTP/fck/3vjSxA==} + engines: {node: '>=14'} + dependencies: + eval: 0.1.8 + p-map: 4.0.0 + webpack-sources: 3.2.3 + dev: false + + /@svgr/babel-plugin-add-jsx-attribute@6.5.1(@babel/core@7.23.6): + resolution: {integrity: sha512-9PYGcXrAxitycIjRmZB+Q0JaN07GZIWaTBIGQzfaZv+qr1n8X1XUEJ5rZ/vx6OVD9RRYlrNnXWExQXcmZeD/BQ==} + engines: {node: '>=10'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + dev: false + + /@svgr/babel-plugin-remove-jsx-attribute@8.0.0(@babel/core@7.23.6): + resolution: {integrity: sha512-BcCkm/STipKvbCl6b7QFrMh/vx00vIP63k2eM66MfHJzPr6O2U0jYEViXkHJWqXqQYjdeA9cuCl5KWmlwjDvbA==} + engines: {node: '>=14'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + dev: false + + /@svgr/babel-plugin-remove-jsx-empty-expression@8.0.0(@babel/core@7.23.6): + resolution: {integrity: sha512-5BcGCBfBxB5+XSDSWnhTThfI9jcO5f0Ai2V24gZpG+wXF14BzwxxdDb4g6trdOux0rhibGs385BeFMSmxtS3uA==} + engines: {node: '>=14'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + dev: false + + /@svgr/babel-plugin-replace-jsx-attribute-value@6.5.1(@babel/core@7.23.6): + resolution: {integrity: sha512-8DPaVVE3fd5JKuIC29dqyMB54sA6mfgki2H2+swh+zNJoynC8pMPzOkidqHOSc6Wj032fhl8Z0TVn1GiPpAiJg==} + engines: {node: '>=10'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + dev: false + + /@svgr/babel-plugin-svg-dynamic-title@6.5.1(@babel/core@7.23.6): + resolution: {integrity: sha512-FwOEi0Il72iAzlkaHrlemVurgSQRDFbk0OC8dSvD5fSBPHltNh7JtLsxmZUhjYBZo2PpcU/RJvvi6Q0l7O7ogw==} + engines: {node: '>=10'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + dev: false + + /@svgr/babel-plugin-svg-em-dimensions@6.5.1(@babel/core@7.23.6): + resolution: {integrity: sha512-gWGsiwjb4tw+ITOJ86ndY/DZZ6cuXMNE/SjcDRg+HLuCmwpcjOktwRF9WgAiycTqJD/QXqL2f8IzE2Rzh7aVXA==} + engines: {node: '>=10'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + dev: false + + /@svgr/babel-plugin-transform-react-native-svg@6.5.1(@babel/core@7.23.6): + resolution: {integrity: sha512-2jT3nTayyYP7kI6aGutkyfJ7UMGtuguD72OjeGLwVNyfPRBD8zQthlvL+fAbAKk5n9ZNcvFkp/b1lZ7VsYqVJg==} + engines: {node: '>=10'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + dev: false + + /@svgr/babel-plugin-transform-svg-component@6.5.1(@babel/core@7.23.6): + resolution: {integrity: sha512-a1p6LF5Jt33O3rZoVRBqdxL350oge54iZWHNI6LJB5tQ7EelvD/Mb1mfBiZNAan0dt4i3VArkFRjA4iObuNykQ==} + engines: {node: '>=12'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + dev: false + + /@svgr/babel-preset@6.5.1(@babel/core@7.23.6): + resolution: {integrity: sha512-6127fvO/FF2oi5EzSQOAjo1LE3OtNVh11R+/8FXa+mHx1ptAaS4cknIjnUA7e6j6fwGGJ17NzaTJFUwOV2zwCw==} + engines: {node: '>=10'} + peerDependencies: + '@babel/core': ^7.0.0-0 + dependencies: + '@babel/core': 7.23.6 + '@svgr/babel-plugin-add-jsx-attribute': 6.5.1(@babel/core@7.23.6) + '@svgr/babel-plugin-remove-jsx-attribute': 8.0.0(@babel/core@7.23.6) + '@svgr/babel-plugin-remove-jsx-empty-expression': 8.0.0(@babel/core@7.23.6) + '@svgr/babel-plugin-replace-jsx-attribute-value': 6.5.1(@babel/core@7.23.6) + '@svgr/babel-plugin-svg-dynamic-title': 6.5.1(@babel/core@7.23.6) + '@svgr/babel-plugin-svg-em-dimensions': 6.5.1(@babel/core@7.23.6) + '@svgr/babel-plugin-transform-react-native-svg': 6.5.1(@babel/core@7.23.6) + '@svgr/babel-plugin-transform-svg-component': 6.5.1(@babel/core@7.23.6) + dev: false + + /@svgr/core@6.5.1: + resolution: {integrity: sha512-/xdLSWxK5QkqG524ONSjvg3V/FkNyCv538OIBdQqPNaAta3AsXj/Bd2FbvR87yMbXO2hFSWiAe/Q6IkVPDw+mw==} + engines: {node: '>=10'} + dependencies: + '@babel/core': 7.23.6 + '@svgr/babel-preset': 6.5.1(@babel/core@7.23.6) + '@svgr/plugin-jsx': 6.5.1(@svgr/core@6.5.1) + camelcase: 6.3.0 + cosmiconfig: 7.1.0 + transitivePeerDependencies: + - supports-color + dev: false + + /@svgr/hast-util-to-babel-ast@6.5.1: + resolution: {integrity: sha512-1hnUxxjd83EAxbL4a0JDJoD3Dao3hmjvyvyEV8PzWmLK3B9m9NPlW7GKjFyoWE8nM7HnXzPcmmSyOW8yOddSXw==} + engines: {node: '>=10'} + dependencies: + '@babel/types': 7.23.6 + entities: 4.5.0 + dev: false + + /@svgr/plugin-jsx@6.5.1(@svgr/core@6.5.1): + resolution: {integrity: sha512-+UdQxI3jgtSjCykNSlEMuy1jSRQlGC7pqBCPvkG/2dATdWo082zHTTK3uhnAju2/6XpE6B5mZ3z4Z8Ns01S8Gw==} + engines: {node: '>=10'} + peerDependencies: + '@svgr/core': ^6.0.0 + dependencies: + '@babel/core': 7.23.6 + '@svgr/babel-preset': 6.5.1(@babel/core@7.23.6) + '@svgr/core': 6.5.1 + '@svgr/hast-util-to-babel-ast': 6.5.1 + svg-parser: 2.0.4 + transitivePeerDependencies: + - supports-color + dev: false + + /@svgr/plugin-svgo@6.5.1(@svgr/core@6.5.1): + resolution: {integrity: sha512-omvZKf8ixP9z6GWgwbtmP9qQMPX4ODXi+wzbVZgomNFsUIlHA1sf4fThdwTWSsZGgvGAG6yE+b/F5gWUkcZ/iQ==} + engines: {node: '>=10'} + peerDependencies: + '@svgr/core': '*' + dependencies: + '@svgr/core': 6.5.1 + cosmiconfig: 7.1.0 + deepmerge: 4.3.1 + svgo: 2.8.0 + dev: false + + /@svgr/webpack@6.5.1: + resolution: {integrity: sha512-cQ/AsnBkXPkEK8cLbv4Dm7JGXq2XrumKnL1dRpJD9rIO2fTIlJI9a1uCciYG1F2aUsox/hJQyNGbt3soDxSRkA==} + engines: {node: '>=10'} + dependencies: + '@babel/core': 7.23.6 + '@babel/plugin-transform-react-constant-elements': 7.23.3(@babel/core@7.23.6) + '@babel/preset-env': 7.23.6(@babel/core@7.23.6) + '@babel/preset-react': 7.23.3(@babel/core@7.23.6) + '@babel/preset-typescript': 7.23.3(@babel/core@7.23.6) + '@svgr/core': 6.5.1 + '@svgr/plugin-jsx': 6.5.1(@svgr/core@6.5.1) + '@svgr/plugin-svgo': 6.5.1(@svgr/core@6.5.1) + transitivePeerDependencies: + - supports-color + dev: false + + /@szmarczak/http-timer@5.0.1: + resolution: {integrity: sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==} + engines: {node: '>=14.16'} + dependencies: + defer-to-connect: 2.0.1 + dev: false + + /@trysound/sax@0.2.0: + resolution: {integrity: sha512-L7z9BgrNEcYyUYtF+HaEfiS5ebkh9jXqbszz7pC0hRBPaatV0XjSD3+eHrpqFemQfgwiFF0QPIarnIihIDn7OA==} + engines: {node: '>=10.13.0'} + dev: false + + /@types/acorn@4.0.6: + resolution: {integrity: sha512-veQTnWP+1D/xbxVrPC3zHnCZRjSrKfhbMUlEA43iMZLu7EsnTtkJklIuwrCPbOi8YkvDQAiW05VQQFvvz9oieQ==} + dependencies: + '@types/estree': 1.0.5 + dev: false + + /@types/body-parser@1.19.5: + resolution: {integrity: sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==} + dependencies: + '@types/connect': 3.4.38 + '@types/node': 20.10.6 + dev: false + + /@types/bonjour@3.5.13: + resolution: {integrity: sha512-z9fJ5Im06zvUL548KvYNecEVlA7cVDkGUi6kZusb04mpyEFKCIZJvloCcmpmLaIahDpOQGHaHmG6imtPMmPXGQ==} + dependencies: + '@types/node': 20.10.6 + dev: false + + /@types/connect-history-api-fallback@1.5.4: + resolution: {integrity: sha512-n6Cr2xS1h4uAulPRdlw6Jl6s1oG8KrVilPN2yUITEs+K48EzMJJ3W1xy8K5eWuFvjp3R74AOIGSmp2UfBJ8HFw==} + dependencies: + '@types/express-serve-static-core': 4.17.41 + '@types/node': 20.10.6 + dev: false + + /@types/connect@3.4.38: + resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==} + dependencies: + '@types/node': 20.10.6 + dev: false + + /@types/d3-scale-chromatic@3.0.3: + resolution: {integrity: sha512-laXM4+1o5ImZv3RpFAsTRn3TEkzqkytiOY0Dz0sq5cnd1dtNlk6sHLon4OvqaiJb28T0S/TdsBI3Sjsy+keJrw==} + dev: false + + /@types/d3-scale@4.0.8: + resolution: {integrity: sha512-gkK1VVTr5iNiYJ7vWDI+yUFFlszhNMtVeneJ6lUTKPjprsvLLI9/tgEGiXJOnlINJA8FyA88gfnQsHbybVZrYQ==} + dependencies: + '@types/d3-time': 3.0.3 + dev: false + + /@types/d3-time@3.0.3: + resolution: {integrity: sha512-2p6olUZ4w3s+07q3Tm2dbiMZy5pCDfYwtLXXHUnVzXgQlZ/OyPtUz6OL382BkOuGlLXqfT+wqv8Fw2v8/0geBw==} + dev: false + + /@types/debug@4.1.12: + resolution: {integrity: sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==} + dependencies: + '@types/ms': 0.7.34 + dev: false + + /@types/eslint-scope@3.7.7: + resolution: {integrity: sha512-MzMFlSLBqNF2gcHWO0G1vP/YQyfvrxZ0bF+u7mzUdZ1/xK4A4sru+nraZz5i3iEIk1l1uyicaDVTB4QbbEkAYg==} + dependencies: + '@types/eslint': 8.56.1 + '@types/estree': 1.0.5 + dev: false + + /@types/eslint@8.56.1: + resolution: {integrity: sha512-18PLWRzhy9glDQp3+wOgfLYRWlhgX0azxgJ63rdpoUHyrC9z0f5CkFburjQx4uD7ZCruw85ZtMt6K+L+R8fLJQ==} + dependencies: + '@types/estree': 1.0.5 + '@types/json-schema': 7.0.15 + dev: false + + /@types/estree-jsx@1.0.3: + resolution: {integrity: sha512-pvQ+TKeRHeiUGRhvYwRrQ/ISnohKkSJR14fT2yqyZ4e9K5vqc7hrtY2Y1Dw0ZwAzQ6DQsxsaCUuSIIi8v0Cq6w==} + dependencies: + '@types/estree': 1.0.5 + dev: false + + /@types/estree@1.0.5: + resolution: {integrity: sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==} + dev: false + + /@types/express-serve-static-core@4.17.41: + resolution: {integrity: sha512-OaJ7XLaelTgrvlZD8/aa0vvvxZdUmlCn6MtWeB7TkiKW70BQLc9XEPpDLPdbo52ZhXUCrznlWdCHWxJWtdyajA==} + dependencies: + '@types/node': 20.10.6 + '@types/qs': 6.9.11 + '@types/range-parser': 1.2.7 + '@types/send': 0.17.4 + dev: false + + /@types/express@4.17.21: + resolution: {integrity: sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==} + dependencies: + '@types/body-parser': 1.19.5 + '@types/express-serve-static-core': 4.17.41 + '@types/qs': 6.9.11 + '@types/serve-static': 1.15.5 + dev: false + + /@types/gtag.js@0.0.12: + resolution: {integrity: sha512-YQV9bUsemkzG81Ea295/nF/5GijnD2Af7QhEofh7xu+kvCN6RdodgNwwGWXB5GMI3NoyvQo0odNctoH/qLMIpg==} + dev: false + + /@types/hast@2.3.9: + resolution: {integrity: sha512-pTHyNlaMD/oKJmS+ZZUyFUcsZeBZpC0lmGquw98CqRVNgAdJZJeD7GoeLiT6Xbx5rU9VCjSt0RwEvDgzh4obFw==} + dependencies: + '@types/unist': 2.0.10 + dev: false + + /@types/hast@3.0.3: + resolution: {integrity: sha512-2fYGlaDy/qyLlhidX42wAH0KBi2TCjKMH8CHmBXgRlJ3Y+OXTiqsPQ6IWarZKwF1JoUcAJdPogv1d4b0COTpmQ==} + dependencies: + '@types/unist': 3.0.2 + dev: false + + /@types/history@4.7.11: + resolution: {integrity: sha512-qjDJRrmvBMiTx+jyLxvLfJU7UznFuokDv4f3WRuriHKERccVpFU+8XMQUAbDzoiJCsmexxRExQeMwwCdamSKDA==} + dev: false + + /@types/html-minifier-terser@6.1.0: + resolution: {integrity: sha512-oh/6byDPnL1zeNXFrDXFLyZjkr1MsBG667IM792caf1L2UPOOMf65NFzjUH/ltyfwjAGfs1rsX1eftK0jC/KIg==} + dev: false + + /@types/http-cache-semantics@4.0.4: + resolution: {integrity: sha512-1m0bIFVc7eJWyve9S0RnuRgcQqF/Xd5QsUZAZeQFr1Q3/p9JWoQQEqmVy+DPTNpGXwhgIetAoYF8JSc33q29QA==} + dev: false + + /@types/http-errors@2.0.4: + resolution: {integrity: sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==} + dev: false + + /@types/http-proxy@1.17.14: + resolution: {integrity: sha512-SSrD0c1OQzlFX7pGu1eXxSEjemej64aaNPRhhVYUGqXh0BtldAAx37MG8btcumvpgKyZp1F5Gn3JkktdxiFv6w==} + dependencies: + '@types/node': 20.10.6 + dev: false + + /@types/istanbul-lib-coverage@2.0.6: + resolution: {integrity: sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==} + dev: false + + /@types/istanbul-lib-report@3.0.3: + resolution: {integrity: sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==} + dependencies: + '@types/istanbul-lib-coverage': 2.0.6 + dev: false + + /@types/istanbul-reports@3.0.4: + resolution: {integrity: sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==} + dependencies: + '@types/istanbul-lib-report': 3.0.3 + dev: false + + /@types/json-schema@7.0.15: + resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} + dev: false + + /@types/mdast@3.0.15: + resolution: {integrity: sha512-LnwD+mUEfxWMa1QpDraczIn6k0Ee3SMicuYSSzS6ZYl2gKS09EClnJYGd8Du6rfc5r/GZEk5o1mRb8TaTj03sQ==} + dependencies: + '@types/unist': 2.0.10 + dev: false + + /@types/mdast@4.0.3: + resolution: {integrity: sha512-LsjtqsyF+d2/yFOYaN22dHZI1Cpwkrj+g06G8+qtUKlhovPW89YhqSnfKtMbkgmEtYpH2gydRNULd6y8mciAFg==} + dependencies: + '@types/unist': 3.0.2 + dev: false + + /@types/mdx@2.0.10: + resolution: {integrity: sha512-Rllzc5KHk0Al5/WANwgSPl1/CwjqCy+AZrGd78zuK+jO9aDM6ffblZ+zIjgPNAaEBmlO0RYDvLNh7wD0zKVgEg==} + dev: false + + /@types/mime@1.3.5: + resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==} + dev: false + + /@types/mime@3.0.4: + resolution: {integrity: sha512-iJt33IQnVRkqeqC7PzBHPTC6fDlRNRW8vjrgqtScAhrmMwe8c4Eo7+fUGTa+XdWrpEgpyKWMYmi2dIwMAYRzPw==} + dev: false + + /@types/ms@0.7.34: + resolution: {integrity: sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==} + dev: false + + /@types/node-forge@1.3.11: + resolution: {integrity: sha512-FQx220y22OKNTqaByeBGqHWYz4cl94tpcxeFdvBo3wjG6XPBuZ0BNgNZRV5J5TFmmcsJ4IzsLkmGRiQbnYsBEQ==} + dependencies: + '@types/node': 20.10.6 + dev: false + + /@types/node@17.0.45: + resolution: {integrity: sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==} + dev: false + + /@types/node@20.10.6: + resolution: {integrity: sha512-Vac8H+NlRNNlAmDfGUP7b5h/KA+AtWIzuXy0E6OyP8f1tCLYAtPvKRRDJjAPqhpCb0t6U2j7/xqAuLEebW2kiw==} + dependencies: + undici-types: 5.26.5 + dev: false + + /@types/parse-json@4.0.2: + resolution: {integrity: sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==} + dev: false + + /@types/prismjs@1.26.3: + resolution: {integrity: sha512-A0D0aTXvjlqJ5ZILMz3rNfDBOx9hHxLZYv2by47Sm/pqW35zzjusrZTryatjN/Rf8Us2gZrJD+KeHbUSTux1Cw==} + dev: false + + /@types/prop-types@15.7.11: + resolution: {integrity: sha512-ga8y9v9uyeiLdpKddhxYQkxNDrfvuPrlFb0N1qnZZByvcElJaXthF1UhvCh9TLWJBEHeNtdnbysW7Y6Uq8CVng==} + dev: false + + /@types/qs@6.9.11: + resolution: {integrity: sha512-oGk0gmhnEJK4Yyk+oI7EfXsLayXatCWPHary1MtcmbAifkobT9cM9yutG/hZKIseOU0MqbIwQ/u2nn/Gb+ltuQ==} + dev: false + + /@types/range-parser@1.2.7: + resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==} + dev: false + + /@types/react-router-config@5.0.11: + resolution: {integrity: sha512-WmSAg7WgqW7m4x8Mt4N6ZyKz0BubSj/2tVUMsAHp+Yd2AMwcSbeFq9WympT19p5heCFmF97R9eD5uUR/t4HEqw==} + dependencies: + '@types/history': 4.7.11 + '@types/react': 18.2.46 + '@types/react-router': 5.1.20 + dev: false + + /@types/react-router-dom@5.3.3: + resolution: {integrity: sha512-kpqnYK4wcdm5UaWI3fLcELopqLrHgLqNsdpHauzlQktfkHL3npOSwtj1Uz9oKBAzs7lFtVkV8j83voAz2D8fhw==} + dependencies: + '@types/history': 4.7.11 + '@types/react': 18.2.46 + '@types/react-router': 5.1.20 + dev: false + + /@types/react-router@5.1.20: + resolution: {integrity: sha512-jGjmu/ZqS7FjSH6owMcD5qpq19+1RS9DeVRqfl1FeBMxTDQAGwlMWOcs52NDoXaNKyG3d1cYQFMs9rCrb88o9Q==} + dependencies: + '@types/history': 4.7.11 + '@types/react': 18.2.46 + dev: false + + /@types/react@18.2.46: + resolution: {integrity: sha512-nNCvVBcZlvX4NU1nRRNV/mFl1nNRuTuslAJglQsq+8ldXe5Xv0Wd2f7WTE3jOxhLH2BFfiZGC6GCp+kHQbgG+w==} + dependencies: + '@types/prop-types': 15.7.11 + '@types/scheduler': 0.16.8 + csstype: 3.1.3 + dev: false + + /@types/retry@0.12.0: + resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==} + dev: false + + /@types/sax@1.2.7: + resolution: {integrity: sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==} + dependencies: + '@types/node': 17.0.45 + dev: false + + /@types/scheduler@0.16.8: + resolution: {integrity: sha512-WZLiwShhwLRmeV6zH+GkbOFT6Z6VklCItrDioxUnv+u4Ll+8vKeFySoFyK/0ctcRpOmwAicELfmys1sDc/Rw+A==} + dev: false + + /@types/send@0.17.4: + resolution: {integrity: sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==} + dependencies: + '@types/mime': 1.3.5 + '@types/node': 20.10.6 + dev: false + + /@types/serve-index@1.9.4: + resolution: {integrity: sha512-qLpGZ/c2fhSs5gnYsQxtDEq3Oy8SXPClIXkW5ghvAvsNuVSA8k+gCONcUCS/UjLEYvYps+e8uBtfgXgvhwfNug==} + dependencies: + '@types/express': 4.17.21 + dev: false + + /@types/serve-static@1.15.5: + resolution: {integrity: sha512-PDRk21MnK70hja/YF8AHfC7yIsiQHn1rcXx7ijCFBX/k+XQJhQT/gw3xekXKJvx+5SXaMMS8oqQy09Mzvz2TuQ==} + dependencies: + '@types/http-errors': 2.0.4 + '@types/mime': 3.0.4 + '@types/node': 20.10.6 + dev: false + + /@types/sockjs@0.3.36: + resolution: {integrity: sha512-MK9V6NzAS1+Ud7JV9lJLFqW85VbC9dq3LmwZCuBe4wBDgKC0Kj/jd8Xl+nSviU+Qc3+m7umHHyHg//2KSa0a0Q==} + dependencies: + '@types/node': 20.10.6 + dev: false + + /@types/unist@2.0.10: + resolution: {integrity: sha512-IfYcSBWE3hLpBg8+X2SEa8LVkJdJEkT2Ese2aaLs3ptGdVtABxndrMaxuFlQ1qdFf9Q5rDvDpxI3WwgvKFAsQA==} + dev: false + + /@types/unist@3.0.2: + resolution: {integrity: sha512-dqId9J8K/vGi5Zr7oo212BGii5m3q5Hxlkwy3WpYuKPklmBEvsbMYYyLxAQpSffdLl/gdW0XUpKWFvYmyoWCoQ==} + dev: false + + /@types/ws@8.5.10: + resolution: {integrity: sha512-vmQSUcfalpIq0R9q7uTo2lXs6eGIpt9wtnLdMv9LVpIjCA/+ufZRozlVoVelIYixx1ugCBKDhn89vnsEGOCx9A==} + dependencies: + '@types/node': 20.10.6 + dev: false + + /@types/yargs-parser@21.0.3: + resolution: {integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==} + dev: false + + /@types/yargs@17.0.32: + resolution: {integrity: sha512-xQ67Yc/laOG5uMfX/093MRlGGCIBzZMarVa+gfNKJxWAIgykYpVGkBdbqEzGDDfCrVUj6Hiff4mTZ5BA6TmAog==} + dependencies: + '@types/yargs-parser': 21.0.3 + dev: false + + /@ungap/structured-clone@1.2.0: + resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==} + dev: false + + /@webassemblyjs/ast@1.11.6: + resolution: {integrity: sha512-IN1xI7PwOvLPgjcf180gC1bqn3q/QaOCwYUahIOhbYUu8KA/3tw2RT/T0Gidi1l7Hhj5D/INhJxiICObqpMu4Q==} + dependencies: + '@webassemblyjs/helper-numbers': 1.11.6 + '@webassemblyjs/helper-wasm-bytecode': 1.11.6 + dev: false + + /@webassemblyjs/floating-point-hex-parser@1.11.6: + resolution: {integrity: sha512-ejAj9hfRJ2XMsNHk/v6Fu2dGS+i4UaXBXGemOfQ/JfQ6mdQg/WXtwleQRLLS4OvfDhv8rYnVwH27YJLMyYsxhw==} + dev: false + + /@webassemblyjs/helper-api-error@1.11.6: + resolution: {integrity: sha512-o0YkoP4pVu4rN8aTJgAyj9hC2Sv5UlkzCHhxqWj8butaLvnpdc2jOwh4ewE6CX0txSfLn/UYaV/pheS2Txg//Q==} + dev: false + + /@webassemblyjs/helper-buffer@1.11.6: + resolution: {integrity: sha512-z3nFzdcp1mb8nEOFFk8DrYLpHvhKC3grJD2ardfKOzmbmJvEf/tPIqCY+sNcwZIY8ZD7IkB2l7/pqhUhqm7hLA==} + dev: false + + /@webassemblyjs/helper-numbers@1.11.6: + resolution: {integrity: sha512-vUIhZ8LZoIWHBohiEObxVm6hwP034jwmc9kuq5GdHZH0wiLVLIPcMCdpJzG4C11cHoQ25TFIQj9kaVADVX7N3g==} + dependencies: + '@webassemblyjs/floating-point-hex-parser': 1.11.6 + '@webassemblyjs/helper-api-error': 1.11.6 + '@xtuc/long': 4.2.2 + dev: false + + /@webassemblyjs/helper-wasm-bytecode@1.11.6: + resolution: {integrity: sha512-sFFHKwcmBprO9e7Icf0+gddyWYDViL8bpPjJJl0WHxCdETktXdmtWLGVzoHbqUcY4Be1LkNfwTmXOJUFZYSJdA==} + dev: false + + /@webassemblyjs/helper-wasm-section@1.11.6: + resolution: {integrity: sha512-LPpZbSOwTpEC2cgn4hTydySy1Ke+XEu+ETXuoyvuyezHO3Kjdu90KK95Sh9xTbmjrCsUwvWwCOQQNta37VrS9g==} + dependencies: + '@webassemblyjs/ast': 1.11.6 + '@webassemblyjs/helper-buffer': 1.11.6 + '@webassemblyjs/helper-wasm-bytecode': 1.11.6 + '@webassemblyjs/wasm-gen': 1.11.6 + dev: false + + /@webassemblyjs/ieee754@1.11.6: + resolution: {integrity: sha512-LM4p2csPNvbij6U1f19v6WR56QZ8JcHg3QIJTlSwzFcmx6WSORicYj6I63f9yU1kEUtrpG+kjkiIAkevHpDXrg==} + dependencies: + '@xtuc/ieee754': 1.2.0 + dev: false + + /@webassemblyjs/leb128@1.11.6: + resolution: {integrity: sha512-m7a0FhE67DQXgouf1tbN5XQcdWoNgaAuoULHIfGFIEVKA6tu/edls6XnIlkmS6FrXAquJRPni3ZZKjw6FSPjPQ==} + dependencies: + '@xtuc/long': 4.2.2 + dev: false + + /@webassemblyjs/utf8@1.11.6: + resolution: {integrity: sha512-vtXf2wTQ3+up9Zsg8sa2yWiQpzSsMyXj0qViVP6xKGCUT8p8YJ6HqI7l5eCnWx1T/FYdsv07HQs2wTFbbof/RA==} + dev: false + + /@webassemblyjs/wasm-edit@1.11.6: + resolution: {integrity: sha512-Ybn2I6fnfIGuCR+Faaz7YcvtBKxvoLV3Lebn1tM4o/IAJzmi9AWYIPWpyBfU8cC+JxAO57bk4+zdsTjJR+VTOw==} + dependencies: + '@webassemblyjs/ast': 1.11.6 + '@webassemblyjs/helper-buffer': 1.11.6 + '@webassemblyjs/helper-wasm-bytecode': 1.11.6 + '@webassemblyjs/helper-wasm-section': 1.11.6 + '@webassemblyjs/wasm-gen': 1.11.6 + '@webassemblyjs/wasm-opt': 1.11.6 + '@webassemblyjs/wasm-parser': 1.11.6 + '@webassemblyjs/wast-printer': 1.11.6 + dev: false + + /@webassemblyjs/wasm-gen@1.11.6: + resolution: {integrity: sha512-3XOqkZP/y6B4F0PBAXvI1/bky7GryoogUtfwExeP/v7Nzwo1QLcq5oQmpKlftZLbT+ERUOAZVQjuNVak6UXjPA==} + dependencies: + '@webassemblyjs/ast': 1.11.6 + '@webassemblyjs/helper-wasm-bytecode': 1.11.6 + '@webassemblyjs/ieee754': 1.11.6 + '@webassemblyjs/leb128': 1.11.6 + '@webassemblyjs/utf8': 1.11.6 + dev: false + + /@webassemblyjs/wasm-opt@1.11.6: + resolution: {integrity: sha512-cOrKuLRE7PCe6AsOVl7WasYf3wbSo4CeOk6PkrjS7g57MFfVUF9u6ysQBBODX0LdgSvQqRiGz3CXvIDKcPNy4g==} + dependencies: + '@webassemblyjs/ast': 1.11.6 + '@webassemblyjs/helper-buffer': 1.11.6 + '@webassemblyjs/wasm-gen': 1.11.6 + '@webassemblyjs/wasm-parser': 1.11.6 + dev: false + + /@webassemblyjs/wasm-parser@1.11.6: + resolution: {integrity: sha512-6ZwPeGzMJM3Dqp3hCsLgESxBGtT/OeCvCZ4TA1JUPYgmhAx38tTPR9JaKy0S5H3evQpO/h2uWs2j6Yc/fjkpTQ==} + dependencies: + '@webassemblyjs/ast': 1.11.6 + '@webassemblyjs/helper-api-error': 1.11.6 + '@webassemblyjs/helper-wasm-bytecode': 1.11.6 + '@webassemblyjs/ieee754': 1.11.6 + '@webassemblyjs/leb128': 1.11.6 + '@webassemblyjs/utf8': 1.11.6 + dev: false + + /@webassemblyjs/wast-printer@1.11.6: + resolution: {integrity: sha512-JM7AhRcE+yW2GWYaKeHL5vt4xqee5N2WcezptmgyhNS+ScggqcT1OtXykhAb13Sn5Yas0j2uv9tHgrjwvzAP4A==} + dependencies: + '@webassemblyjs/ast': 1.11.6 + '@xtuc/long': 4.2.2 + dev: false + + /@xtuc/ieee754@1.2.0: + resolution: {integrity: sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==} + dev: false + + /@xtuc/long@4.2.2: + resolution: {integrity: sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==} + dev: false + + /accepts@1.3.8: + resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} + engines: {node: '>= 0.6'} + dependencies: + mime-types: 2.1.35 + negotiator: 0.6.3 + dev: false + + /acorn-import-assertions@1.9.0(acorn@8.11.3): + resolution: {integrity: sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA==} + peerDependencies: + acorn: ^8 + dependencies: + acorn: 8.11.3 + dev: false + + /acorn-jsx@5.3.2(acorn@8.11.3): + resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} + peerDependencies: + acorn: ^6.0.0 || ^7.0.0 || ^8.0.0 + dependencies: + acorn: 8.11.3 + dev: false + + /acorn-walk@8.3.1: + resolution: {integrity: sha512-TgUZgYvqZprrl7YldZNoa9OciCAyZR+Ejm9eXzKCmjsF5IKp/wgQ7Z/ZpjpGTIUPwrHQIcYeI8qDh4PsEwxMbw==} + engines: {node: '>=0.4.0'} + dev: false + + /acorn@8.11.3: + resolution: {integrity: sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==} + engines: {node: '>=0.4.0'} + hasBin: true + dev: false + + /address@1.2.2: + resolution: {integrity: sha512-4B/qKCfeE/ODUaAUpSwfzazo5x29WD4r3vXiWsB7I2mSDAihwEqKO+g8GELZUQSSAo5e1XTYh3ZVfLyxBc12nA==} + engines: {node: '>= 10.0.0'} + dev: false + + /aggregate-error@3.1.0: + resolution: {integrity: sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==} + engines: {node: '>=8'} + dependencies: + clean-stack: 2.2.0 + indent-string: 4.0.0 + dev: false + + /ajv-formats@2.1.1(ajv@8.12.0): + resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==} + peerDependencies: + ajv: ^8.0.0 + peerDependenciesMeta: + ajv: + optional: true + dependencies: + ajv: 8.12.0 + dev: false + + /ajv-keywords@3.5.2(ajv@6.12.6): + resolution: {integrity: sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==} + peerDependencies: + ajv: ^6.9.1 + dependencies: + ajv: 6.12.6 + dev: false + + /ajv-keywords@5.1.0(ajv@8.12.0): + resolution: {integrity: sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==} + peerDependencies: + ajv: ^8.8.2 + dependencies: + ajv: 8.12.0 + fast-deep-equal: 3.1.3 + dev: false + + /ajv@6.12.6: + resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} + dependencies: + fast-deep-equal: 3.1.3 + fast-json-stable-stringify: 2.1.0 + json-schema-traverse: 0.4.1 + uri-js: 4.4.1 + dev: false + + /ajv@8.12.0: + resolution: {integrity: sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==} + dependencies: + fast-deep-equal: 3.1.3 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + uri-js: 4.4.1 + dev: false + + /algoliasearch-helper@3.16.1(algoliasearch@4.22.0): + resolution: {integrity: sha512-qxAHVjjmT7USVvrM8q6gZGaJlCK1fl4APfdAA7o8O6iXEc68G0xMNrzRkxoB/HmhhvyHnoteS/iMTiHiTcQQcg==} + peerDependencies: + algoliasearch: '>= 3.1 < 6' + dependencies: + '@algolia/events': 4.0.1 + algoliasearch: 4.22.0 + dev: false + + /algoliasearch@4.22.0: + resolution: {integrity: sha512-gfceltjkwh7PxXwtkS8KVvdfK+TSNQAWUeNSxf4dA29qW5tf2EGwa8jkJujlT9jLm17cixMVoGNc+GJFO1Mxhg==} + dependencies: + '@algolia/cache-browser-local-storage': 4.22.0 + '@algolia/cache-common': 4.22.0 + '@algolia/cache-in-memory': 4.22.0 + '@algolia/client-account': 4.22.0 + '@algolia/client-analytics': 4.22.0 + '@algolia/client-common': 4.22.0 + '@algolia/client-personalization': 4.22.0 + '@algolia/client-search': 4.22.0 + '@algolia/logger-common': 4.22.0 + '@algolia/logger-console': 4.22.0 + '@algolia/requester-browser-xhr': 4.22.0 + '@algolia/requester-common': 4.22.0 + '@algolia/requester-node-http': 4.22.0 + '@algolia/transporter': 4.22.0 + dev: false + + /ansi-align@3.0.1: + resolution: {integrity: sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==} + dependencies: + string-width: 4.2.3 + dev: false + + /ansi-html-community@0.0.8: + resolution: {integrity: sha512-1APHAyr3+PCamwNw3bXCPp4HFLONZt/yIH0sZp0/469KWNTEy+qN5jQ3GVX6DMZ1UXAi34yVwtTeaG/HpBuuzw==} + engines: {'0': node >= 0.8.0} + hasBin: true + dev: false + + /ansi-regex@5.0.1: + resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} + engines: {node: '>=8'} + dev: false + + /ansi-regex@6.0.1: + resolution: {integrity: sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==} + engines: {node: '>=12'} + dev: false + + /ansi-styles@3.2.1: + resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==} + engines: {node: '>=4'} + dependencies: + color-convert: 1.9.3 + dev: false + + /ansi-styles@4.3.0: + resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} + engines: {node: '>=8'} + dependencies: + color-convert: 2.0.1 + dev: false + + /ansi-styles@6.2.1: + resolution: {integrity: sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==} + engines: {node: '>=12'} + dev: false + + /anymatch@3.1.3: + resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} + engines: {node: '>= 8'} + dependencies: + normalize-path: 3.0.0 + picomatch: 2.3.1 + dev: false + + /arg@5.0.2: + resolution: {integrity: sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==} + dev: false + + /argparse@1.0.10: + resolution: {integrity: sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==} + dependencies: + sprintf-js: 1.0.3 + dev: false + + /argparse@2.0.1: + resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + dev: false + + /array-flatten@1.1.1: + resolution: {integrity: sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==} + dev: false + + /array-union@2.1.0: + resolution: {integrity: sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==} + engines: {node: '>=8'} + dev: false + + /astring@1.8.6: + resolution: {integrity: sha512-ISvCdHdlTDlH5IpxQJIex7BWBywFWgjJSVdwst+/iQCoEYnyOaQ95+X1JGshuBjGp6nxKUy1jMgE3zPqN7fQdg==} + hasBin: true + dev: false + + /async@2.6.4: + resolution: {integrity: sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==} + dependencies: + lodash: 4.17.21 + dev: false + + /at-least-node@1.0.0: + resolution: {integrity: sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==} + engines: {node: '>= 4.0.0'} + dev: false + + /autoprefixer@10.4.16(postcss@8.4.32): + resolution: {integrity: sha512-7vd3UC6xKp0HLfua5IjZlcXvGAGy7cBAXTg2lyQ/8WpNhd6SiZ8Be+xm3FyBSYJx5GKcpRCzBh7RH4/0dnY+uQ==} + engines: {node: ^10 || ^12 || >=14} + hasBin: true + peerDependencies: + postcss: ^8.1.0 + dependencies: + browserslist: 4.22.2 + caniuse-lite: 1.0.30001572 + fraction.js: 4.3.7 + normalize-range: 0.1.2 + picocolors: 1.0.0 + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /babel-loader@9.1.3(@babel/core@7.23.6)(webpack@5.89.0): + resolution: {integrity: sha512-xG3ST4DglodGf8qSwv0MdeWLhrDsw/32QMdTO5T1ZIp9gQur0HkCyFs7Awskr10JKXFXwpAhiCuYX5oGXnRGbw==} + engines: {node: '>= 14.15.0'} + peerDependencies: + '@babel/core': ^7.12.0 + webpack: '>=5' + dependencies: + '@babel/core': 7.23.6 + find-cache-dir: 4.0.0 + schema-utils: 4.2.0 + webpack: 5.89.0 + dev: false + + /babel-plugin-dynamic-import-node@2.3.3: + resolution: {integrity: sha512-jZVI+s9Zg3IqA/kdi0i6UDCybUI3aSBLnglhYbSSjKlV7yF1F/5LWv8MakQmvYpnbJDS6fcBL2KzHSxNCMtWSQ==} + dependencies: + object.assign: 4.1.5 + dev: false + + /babel-plugin-polyfill-corejs2@0.4.7(@babel/core@7.23.6): + resolution: {integrity: sha512-LidDk/tEGDfuHW2DWh/Hgo4rmnw3cduK6ZkOI1NPFceSK3n/yAGeOsNT7FLnSGHkXj3RHGSEVkN3FsCTY6w2CQ==} + peerDependencies: + '@babel/core': ^7.4.0 || ^8.0.0-0 <8.0.0 + dependencies: + '@babel/compat-data': 7.23.5 + '@babel/core': 7.23.6 + '@babel/helper-define-polyfill-provider': 0.4.4(@babel/core@7.23.6) + semver: 6.3.1 + transitivePeerDependencies: + - supports-color + dev: false + + /babel-plugin-polyfill-corejs3@0.8.7(@babel/core@7.23.6): + resolution: {integrity: sha512-KyDvZYxAzkC0Aj2dAPyDzi2Ym15e5JKZSK+maI7NAwSqofvuFglbSsxE7wUOvTg9oFVnHMzVzBKcqEb4PJgtOA==} + peerDependencies: + '@babel/core': ^7.4.0 || ^8.0.0-0 <8.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-define-polyfill-provider': 0.4.4(@babel/core@7.23.6) + core-js-compat: 3.35.0 + transitivePeerDependencies: + - supports-color + dev: false + + /babel-plugin-polyfill-regenerator@0.5.4(@babel/core@7.23.6): + resolution: {integrity: sha512-S/x2iOCvDaCASLYsOOgWOq4bCfKYVqvO/uxjkaYyZ3rVsVE3CeAI/c84NpyuBBymEgNvHgjEot3a9/Z/kXvqsg==} + peerDependencies: + '@babel/core': ^7.4.0 || ^8.0.0-0 <8.0.0 + dependencies: + '@babel/core': 7.23.6 + '@babel/helper-define-polyfill-provider': 0.4.4(@babel/core@7.23.6) + transitivePeerDependencies: + - supports-color + dev: false + + /bail@2.0.2: + resolution: {integrity: sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==} + dev: false + + /balanced-match@1.0.2: + resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + dev: false + + /batch@0.6.1: + resolution: {integrity: sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==} + dev: false + + /big.js@5.2.2: + resolution: {integrity: sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==} + dev: false + + /binary-extensions@2.2.0: + resolution: {integrity: sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==} + engines: {node: '>=8'} + dev: false + + /body-parser@1.20.1: + resolution: {integrity: sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==} + engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + dependencies: + bytes: 3.1.2 + content-type: 1.0.5 + debug: 2.6.9 + depd: 2.0.0 + destroy: 1.2.0 + http-errors: 2.0.0 + iconv-lite: 0.4.24 + on-finished: 2.4.1 + qs: 6.11.0 + raw-body: 2.5.1 + type-is: 1.6.18 + unpipe: 1.0.0 + transitivePeerDependencies: + - supports-color + dev: false + + /bonjour-service@1.2.0: + resolution: {integrity: sha512-xdzMA6JGckxyJzZByjEWRcfKmDxXaGXZWVftah3FkCqdlePNS9DjHSUN5zkP4oEfz/t0EXXlro88EIhzwMB4zA==} + dependencies: + fast-deep-equal: 3.1.3 + multicast-dns: 7.2.5 + dev: false + + /boolbase@1.0.0: + resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} + dev: false + + /boxen@6.2.1: + resolution: {integrity: sha512-H4PEsJXfFI/Pt8sjDWbHlQPx4zL/bvSQjcilJmaulGt5mLDorHOHpmdXAJcBcmru7PhYSp/cDMWRko4ZUMFkSw==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + ansi-align: 3.0.1 + camelcase: 6.3.0 + chalk: 4.1.2 + cli-boxes: 3.0.0 + string-width: 5.1.2 + type-fest: 2.19.0 + widest-line: 4.0.1 + wrap-ansi: 8.1.0 + dev: false + + /boxen@7.1.1: + resolution: {integrity: sha512-2hCgjEmP8YLWQ130n2FerGv7rYpfBmnmp9Uy2Le1vge6X3gZIfSmEzP5QTDElFxcvVcXlEn8Aq6MU/PZygIOog==} + engines: {node: '>=14.16'} + dependencies: + ansi-align: 3.0.1 + camelcase: 7.0.1 + chalk: 5.3.0 + cli-boxes: 3.0.0 + string-width: 5.1.2 + type-fest: 2.19.0 + widest-line: 4.0.1 + wrap-ansi: 8.1.0 + dev: false + + /brace-expansion@1.1.11: + resolution: {integrity: sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==} + dependencies: + balanced-match: 1.0.2 + concat-map: 0.0.1 + dev: false + + /braces@3.0.2: + resolution: {integrity: sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==} + engines: {node: '>=8'} + dependencies: + fill-range: 7.0.1 + dev: false + + /browserslist@4.22.2: + resolution: {integrity: sha512-0UgcrvQmBDvZHFGdYUehrCNIazki7/lUP3kkoi/r3YB2amZbFM9J43ZRkJTXBUZK4gmx56+Sqk9+Vs9mwZx9+A==} + engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} + hasBin: true + dependencies: + caniuse-lite: 1.0.30001572 + electron-to-chromium: 1.4.618 + node-releases: 2.0.14 + update-browserslist-db: 1.0.13(browserslist@4.22.2) + dev: false + + /buffer-from@1.1.2: + resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} + dev: false + + /bytes@3.0.0: + resolution: {integrity: sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==} + engines: {node: '>= 0.8'} + dev: false + + /bytes@3.1.2: + resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} + engines: {node: '>= 0.8'} + dev: false + + /cacheable-lookup@7.0.0: + resolution: {integrity: sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==} + engines: {node: '>=14.16'} + dev: false + + /cacheable-request@10.2.14: + resolution: {integrity: sha512-zkDT5WAF4hSSoUgyfg5tFIxz8XQK+25W/TLVojJTMKBaxevLBBtLxgqguAuVQB8PVW79FVjHcU+GJ9tVbDZ9mQ==} + engines: {node: '>=14.16'} + dependencies: + '@types/http-cache-semantics': 4.0.4 + get-stream: 6.0.1 + http-cache-semantics: 4.1.1 + keyv: 4.5.4 + mimic-response: 4.0.0 + normalize-url: 8.0.0 + responselike: 3.0.0 + dev: false + + /call-bind@1.0.5: + resolution: {integrity: sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ==} + dependencies: + function-bind: 1.1.2 + get-intrinsic: 1.2.2 + set-function-length: 1.1.1 + dev: false + + /callsites@3.1.0: + resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} + engines: {node: '>=6'} + dev: false + + /camel-case@4.1.2: + resolution: {integrity: sha512-gxGWBrTT1JuMx6R+o5PTXMmUnhnVzLQ9SNutD4YqKtI6ap897t3tKECYla6gCWEkplXnlNybEkZg9GEGxKFCgw==} + dependencies: + pascal-case: 3.1.2 + tslib: 2.6.2 + dev: false + + /camelcase@6.3.0: + resolution: {integrity: sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==} + engines: {node: '>=10'} + dev: false + + /camelcase@7.0.1: + resolution: {integrity: sha512-xlx1yCK2Oc1APsPXDL2LdlNP6+uu8OCDdhOBSVT279M/S+y75O30C2VuD8T2ogdePBBl7PfPF4504tnLgX3zfw==} + engines: {node: '>=14.16'} + dev: false + + /caniuse-api@3.0.0: + resolution: {integrity: sha512-bsTwuIg/BZZK/vreVTYYbSWoe2F+71P7K5QGEX+pT250DZbfU1MQ5prOKpPR+LL6uWKK3KMwMCAS74QB3Um1uw==} + dependencies: + browserslist: 4.22.2 + caniuse-lite: 1.0.30001572 + lodash.memoize: 4.1.2 + lodash.uniq: 4.5.0 + dev: false + + /caniuse-lite@1.0.30001572: + resolution: {integrity: sha512-1Pbh5FLmn5y4+QhNyJE9j3/7dK44dGB83/ZMjv/qJk86TvDbjk0LosiZo0i0WB0Vx607qMX9jYrn1VLHCkN4rw==} + dev: false + + /ccount@2.0.1: + resolution: {integrity: sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==} + dev: false + + /chalk@2.4.2: + resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==} + engines: {node: '>=4'} + dependencies: + ansi-styles: 3.2.1 + escape-string-regexp: 1.0.5 + supports-color: 5.5.0 + dev: false + + /chalk@4.1.2: + resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} + engines: {node: '>=10'} + dependencies: + ansi-styles: 4.3.0 + supports-color: 7.2.0 + dev: false + + /chalk@5.3.0: + resolution: {integrity: sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==} + engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} + dev: false + + /char-regex@1.0.2: + resolution: {integrity: sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==} + engines: {node: '>=10'} + dev: false + + /character-entities-html4@2.1.0: + resolution: {integrity: sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==} + dev: false + + /character-entities-legacy@3.0.0: + resolution: {integrity: sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==} + dev: false + + /character-entities@2.0.2: + resolution: {integrity: sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==} + dev: false + + /character-reference-invalid@2.0.1: + resolution: {integrity: sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==} + dev: false + + /cheerio-select@2.1.0: + resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==} + dependencies: + boolbase: 1.0.0 + css-select: 5.1.0 + css-what: 6.1.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.1.0 + dev: false + + /cheerio@1.0.0-rc.12: + resolution: {integrity: sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==} + engines: {node: '>= 6'} + dependencies: + cheerio-select: 2.1.0 + dom-serializer: 2.0.0 + domhandler: 5.0.3 + domutils: 3.1.0 + htmlparser2: 8.0.2 + parse5: 7.1.2 + parse5-htmlparser2-tree-adapter: 7.0.0 + dev: false + + /chokidar@3.5.3: + resolution: {integrity: sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==} + engines: {node: '>= 8.10.0'} + dependencies: + anymatch: 3.1.3 + braces: 3.0.2 + glob-parent: 5.1.2 + is-binary-path: 2.1.0 + is-glob: 4.0.3 + normalize-path: 3.0.0 + readdirp: 3.6.0 + optionalDependencies: + fsevents: 2.3.3 + dev: false + + /chrome-trace-event@1.0.3: + resolution: {integrity: sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg==} + engines: {node: '>=6.0'} + dev: false + + /ci-info@3.9.0: + resolution: {integrity: sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==} + engines: {node: '>=8'} + dev: false + + /classnames@2.3.2: + resolution: {integrity: sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw==} + dev: false + + /clean-css@5.3.3: + resolution: {integrity: sha512-D5J+kHaVb/wKSFcyyV75uCn8fiY4sV38XJoe4CUyGQ+mOU/fMVYUdH1hJC+CJQ5uY3EnW27SbJYS4X8BiLrAFg==} + engines: {node: '>= 10.0'} + dependencies: + source-map: 0.6.1 + dev: false + + /clean-stack@2.2.0: + resolution: {integrity: sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==} + engines: {node: '>=6'} + dev: false + + /cli-boxes@3.0.0: + resolution: {integrity: sha512-/lzGpEWL/8PfI0BmBOPRwp0c/wFNX1RdUML3jK/RcSBA9T8mZDdQpqYBKtCFTOfQbwPqWEOpjqW+Fnayc0969g==} + engines: {node: '>=10'} + dev: false + + /cli-table3@0.6.3: + resolution: {integrity: sha512-w5Jac5SykAeZJKntOxJCrm63Eg5/4dhMWIcuTbo9rpE+brgaSZo0RuNJZeOyMgsUdhDeojvgyQLmjI+K50ZGyg==} + engines: {node: 10.* || >= 12.*} + dependencies: + string-width: 4.2.3 + optionalDependencies: + '@colors/colors': 1.5.0 + dev: false + + /clone-deep@4.0.1: + resolution: {integrity: sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==} + engines: {node: '>=6'} + dependencies: + is-plain-object: 2.0.4 + kind-of: 6.0.3 + shallow-clone: 3.0.1 + dev: false + + /clsx@1.1.1: + resolution: {integrity: sha512-6/bPho624p3S2pMyvP5kKBPXnI3ufHLObBFCfgx+LkeR5lg2XYy2hqZqUf45ypD8COn2bhgGJSUE+l5dhNBieA==} + engines: {node: '>=6'} + dev: false + + /clsx@2.1.0: + resolution: {integrity: sha512-m3iNNWpd9rl3jvvcBnu70ylMdrXt8Vlq4HYadnU5fwcOtvkSQWPmj7amUcDT2qYI7risszBjI5AUIUox9D16pg==} + engines: {node: '>=6'} + dev: false + + /collapse-white-space@2.1.0: + resolution: {integrity: sha512-loKTxY1zCOuG4j9f6EPnuyyYkf58RnhhWTvRoZEokgB+WbdXehfjFviyOVYkqzEWz1Q5kRiZdBYS5SwxbQYwzw==} + dev: false + + /color-convert@1.9.3: + resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==} + dependencies: + color-name: 1.1.3 + dev: false + + /color-convert@2.0.1: + resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} + engines: {node: '>=7.0.0'} + dependencies: + color-name: 1.1.4 + dev: false + + /color-name@1.1.3: + resolution: {integrity: sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==} + dev: false + + /color-name@1.1.4: + resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + dev: false + + /colord@2.9.3: + resolution: {integrity: sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==} + dev: false + + /colorette@2.0.20: + resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} + dev: false + + /combine-promises@1.2.0: + resolution: {integrity: sha512-VcQB1ziGD0NXrhKxiwyNbCDmRzs/OShMs2GqW2DlU2A/Sd0nQxE1oWDAE5O0ygSx5mgQOn9eIFh7yKPgFRVkPQ==} + engines: {node: '>=10'} + dev: false + + /comma-separated-tokens@2.0.3: + resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} + dev: false + + /commander@10.0.1: + resolution: {integrity: sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==} + engines: {node: '>=14'} + dev: false + + /commander@2.20.3: + resolution: {integrity: sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==} + dev: false + + /commander@5.1.0: + resolution: {integrity: sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==} + engines: {node: '>= 6'} + dev: false + + /commander@7.2.0: + resolution: {integrity: sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==} + engines: {node: '>= 10'} + dev: false + + /commander@8.3.0: + resolution: {integrity: sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==} + engines: {node: '>= 12'} + dev: false + + /common-path-prefix@3.0.0: + resolution: {integrity: sha512-QE33hToZseCH3jS0qN96O/bSh3kaw/h+Tq7ngyY9eWDUnTlTNUyqfqvCXioLe5Na5jFsL78ra/wuBU4iuEgd4w==} + dev: false + + /compressible@2.0.18: + resolution: {integrity: sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==} + engines: {node: '>= 0.6'} + dependencies: + mime-db: 1.52.0 + dev: false + + /compression@1.7.4: + resolution: {integrity: sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==} + engines: {node: '>= 0.8.0'} + dependencies: + accepts: 1.3.8 + bytes: 3.0.0 + compressible: 2.0.18 + debug: 2.6.9 + on-headers: 1.0.2 + safe-buffer: 5.1.2 + vary: 1.1.2 + transitivePeerDependencies: + - supports-color + dev: false + + /concat-map@0.0.1: + resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} + dev: false + + /config-chain@1.1.13: + resolution: {integrity: sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==} + dependencies: + ini: 1.3.8 + proto-list: 1.2.4 + dev: false + + /configstore@6.0.0: + resolution: {integrity: sha512-cD31W1v3GqUlQvbBCGcXmd2Nj9SvLDOP1oQ0YFuLETufzSPaKp11rYBsSOm7rCsW3OnIRAFM3OxRhceaXNYHkA==} + engines: {node: '>=12'} + dependencies: + dot-prop: 6.0.1 + graceful-fs: 4.2.11 + unique-string: 3.0.0 + write-file-atomic: 3.0.3 + xdg-basedir: 5.1.0 + dev: false + + /connect-history-api-fallback@1.6.0: + resolution: {integrity: sha512-e54B99q/OUoH64zYYRf3HBP5z24G38h5D3qXu23JGRoigpX5Ss4r9ZnDk3g0Z8uQC2x2lPaJ+UlWBc1ZWBWdLg==} + engines: {node: '>=0.8'} + dev: false + + /connect-history-api-fallback@2.0.0: + resolution: {integrity: sha512-U73+6lQFmfiNPrYbXqr6kZ1i1wiRqXnp2nhMsINseWXO8lDau0LGEffJ8kQi4EjLZympVgRdvqjAgiZ1tgzDDA==} + engines: {node: '>=0.8'} + dev: false + + /consola@2.15.3: + resolution: {integrity: sha512-9vAdYbHj6x2fLKC4+oPH0kFzY/orMZyG2Aj+kNylHxKGJ/Ed4dpNyAQYwJOdqO4zdM7XpVHmyejQDcQHrnuXbw==} + dev: false + + /content-disposition@0.5.2: + resolution: {integrity: sha512-kRGRZw3bLlFISDBgwTSA1TMBFN6J6GWDeubmDE3AF+3+yXL8hTWv8r5rkLbqYXY4RjPk/EzHnClI3zQf1cFmHA==} + engines: {node: '>= 0.6'} + dev: false + + /content-disposition@0.5.4: + resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} + engines: {node: '>= 0.6'} + dependencies: + safe-buffer: 5.2.1 + dev: false + + /content-type@1.0.5: + resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} + engines: {node: '>= 0.6'} + dev: false + + /convert-source-map@2.0.0: + resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} + dev: false + + /cookie-signature@1.0.6: + resolution: {integrity: sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==} + dev: false + + /cookie@0.5.0: + resolution: {integrity: sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==} + engines: {node: '>= 0.6'} + dev: false + + /copy-text-to-clipboard@3.2.0: + resolution: {integrity: sha512-RnJFp1XR/LOBDckxTib5Qjr/PMfkatD0MUCQgdpqS8MdKiNUzBjAQBEN6oUy+jW7LI93BBG3DtMB2KOOKpGs2Q==} + engines: {node: '>=12'} + dev: false + + /copy-webpack-plugin@11.0.0(webpack@5.89.0): + resolution: {integrity: sha512-fX2MWpamkW0hZxMEg0+mYnA40LTosOSa5TqZ9GYIBzyJa9C3QUaMPSE2xAi/buNr8u89SfD9wHSQVBzrRa/SOQ==} + engines: {node: '>= 14.15.0'} + peerDependencies: + webpack: ^5.1.0 + dependencies: + fast-glob: 3.3.2 + glob-parent: 6.0.2 + globby: 13.2.2 + normalize-path: 3.0.0 + schema-utils: 4.2.0 + serialize-javascript: 6.0.1 + webpack: 5.89.0 + dev: false + + /core-js-compat@3.35.0: + resolution: {integrity: sha512-5blwFAddknKeNgsjBzilkdQ0+YK8L1PfqPYq40NOYMYFSS38qj+hpTcLLWwpIwA2A5bje/x5jmVn2tzUMg9IVw==} + dependencies: + browserslist: 4.22.2 + dev: false + + /core-js-pure@3.35.0: + resolution: {integrity: sha512-f+eRYmkou59uh7BPcyJ8MC76DiGhspj1KMxVIcF24tzP8NA9HVa1uC7BTW2tgx7E1QVCzDzsgp7kArrzhlz8Ew==} + requiresBuild: true + dev: false + + /core-js@3.35.0: + resolution: {integrity: sha512-ntakECeqg81KqMueeGJ79Q5ZgQNR+6eaE8sxGCx62zMbAIj65q+uYvatToew3m6eAGdU4gNZwpZ34NMe4GYswg==} + requiresBuild: true + dev: false + + /core-util-is@1.0.3: + resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} + dev: false + + /cose-base@1.0.3: + resolution: {integrity: sha512-s9whTXInMSgAp/NVXVNuVxVKzGH2qck3aQlVHxDCdAEPgtMKwc4Wq6/QKhgdEdgbLSi9rBTAcPoRa6JpiG4ksg==} + dependencies: + layout-base: 1.0.2 + dev: false + + /cose-base@2.2.0: + resolution: {integrity: sha512-AzlgcsCbUMymkADOJtQm3wO9S3ltPfYOFD5033keQn9NJzIbtnZj+UdBJe7DYml/8TdbtHJW3j58SOnKhWY/5g==} + dependencies: + layout-base: 2.0.1 + dev: false + + /cosmiconfig@6.0.0: + resolution: {integrity: sha512-xb3ZL6+L8b9JLLCx3ZdoZy4+2ECphCMo2PwqgP1tlfVq6M6YReyzBJtvWWtbDSpNr9hn96pkCiZqUcFEc+54Qg==} + engines: {node: '>=8'} + dependencies: + '@types/parse-json': 4.0.2 + import-fresh: 3.3.0 + parse-json: 5.2.0 + path-type: 4.0.0 + yaml: 1.10.2 + dev: false + + /cosmiconfig@7.1.0: + resolution: {integrity: sha512-AdmX6xUzdNASswsFtmwSt7Vj8po9IuqXm0UXz7QKPuEUmPB4XyjGfaAr2PSuELMwkRMVH1EpIkX5bTZGRB3eCA==} + engines: {node: '>=10'} + dependencies: + '@types/parse-json': 4.0.2 + import-fresh: 3.3.0 + parse-json: 5.2.0 + path-type: 4.0.0 + yaml: 1.10.2 + dev: false + + /cosmiconfig@8.3.6(typescript@5.3.3): + resolution: {integrity: sha512-kcZ6+W5QzcJ3P1Mt+83OUv/oHFqZHIx8DuxG6eZ5RGMERoLqp4BuGjhHLYGK+Kf5XVkQvqBSmAy/nGWN3qDgEA==} + engines: {node: '>=14'} + peerDependencies: + typescript: '>=4.9.5' + peerDependenciesMeta: + typescript: + optional: true + dependencies: + import-fresh: 3.3.0 + js-yaml: 4.1.0 + parse-json: 5.2.0 + path-type: 4.0.0 + typescript: 5.3.3 + dev: false + + /cross-spawn@7.0.3: + resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==} + engines: {node: '>= 8'} + dependencies: + path-key: 3.1.1 + shebang-command: 2.0.0 + which: 2.0.2 + dev: false + + /crypto-random-string@4.0.0: + resolution: {integrity: sha512-x8dy3RnvYdlUcPOjkEHqozhiwzKNSq7GcPuXFbnyMOCHxX8V3OgIg/pYuabl2sbUPfIJaeAQB7PMOK8DFIdoRA==} + engines: {node: '>=12'} + dependencies: + type-fest: 1.4.0 + dev: false + + /css-declaration-sorter@6.3.0(postcss@8.4.32): + resolution: {integrity: sha512-OGT677UGHJTAVMRhPO+HJ4oKln3wkBTwtDFH0ojbqm+MJm6xuDMHp2nkhh/ThaBqq20IbraBQSWKfSLNHQO9Og==} + engines: {node: ^10 || ^12 || >=14} + peerDependencies: + postcss: ^8.0.9 + dependencies: + postcss: 8.4.32 + dev: false + + /css-declaration-sorter@6.4.1(postcss@8.4.32): + resolution: {integrity: sha512-rtdthzxKuyq6IzqX6jEcIzQF/YqccluefyCYheovBOLhFT/drQA9zj/UbRAa9J7C0o6EG6u3E6g+vKkay7/k3g==} + engines: {node: ^10 || ^12 || >=14} + peerDependencies: + postcss: ^8.0.9 + dependencies: + postcss: 8.4.32 + dev: false + + /css-declaration-sorter@7.1.1(postcss@8.4.32): + resolution: {integrity: sha512-dZ3bVTEEc1vxr3Bek9vGwfB5Z6ESPULhcRvO472mfjVnj8jRcTnKO8/JTczlvxM10Myb+wBM++1MtdO76eWcaQ==} + engines: {node: ^14 || ^16 || >=18} + peerDependencies: + postcss: ^8.0.9 + dependencies: + postcss: 8.4.32 + dev: false + + /css-loader@6.8.1(webpack@5.89.0): + resolution: {integrity: sha512-xDAXtEVGlD0gJ07iclwWVkLoZOpEvAWaSyf6W18S2pOC//K8+qUDIx8IIT3D+HjnmkJPQeesOPv5aiUaJsCM2g==} + engines: {node: '>= 12.13.0'} + peerDependencies: + webpack: ^5.0.0 + dependencies: + icss-utils: 5.1.0(postcss@8.4.32) + postcss: 8.4.32 + postcss-modules-extract-imports: 3.0.0(postcss@8.4.32) + postcss-modules-local-by-default: 4.0.3(postcss@8.4.32) + postcss-modules-scope: 3.1.0(postcss@8.4.32) + postcss-modules-values: 4.0.0(postcss@8.4.32) + postcss-value-parser: 4.2.0 + semver: 7.5.4 + webpack: 5.89.0 + dev: false + + /css-minimizer-webpack-plugin@4.0.0(webpack@5.89.0): + resolution: {integrity: sha512-7ZXXRzRHvofv3Uac5Y+RkWRNo0ZMlcg8e9/OtrqUYmwDWJo+qs67GvdeFrXLsFb7czKNwjQhPkM0avlIYl+1nA==} + engines: {node: '>= 14.15.0'} + peerDependencies: + '@parcel/css': '*' + clean-css: '*' + csso: '*' + esbuild: '*' + webpack: ^5.0.0 + peerDependenciesMeta: + '@parcel/css': + optional: true + clean-css: + optional: true + csso: + optional: true + esbuild: + optional: true + dependencies: + cssnano: 5.1.15(postcss@8.4.32) + jest-worker: 27.5.1 + postcss: 8.4.32 + schema-utils: 4.2.0 + serialize-javascript: 6.0.1 + source-map: 0.6.1 + webpack: 5.89.0 + dev: false + + /css-minimizer-webpack-plugin@4.2.2(clean-css@5.3.3)(webpack@5.89.0): + resolution: {integrity: sha512-s3Of/4jKfw1Hj9CxEO1E5oXhQAxlayuHO2y/ML+C6I9sQ7FdzfEV6QgMLN3vI+qFsjJGIAFLKtQK7t8BOXAIyA==} + engines: {node: '>= 14.15.0'} + peerDependencies: + '@parcel/css': '*' + '@swc/css': '*' + clean-css: '*' + csso: '*' + esbuild: '*' + lightningcss: '*' + webpack: ^5.0.0 + peerDependenciesMeta: + '@parcel/css': + optional: true + '@swc/css': + optional: true + clean-css: + optional: true + csso: + optional: true + esbuild: + optional: true + lightningcss: + optional: true + dependencies: + clean-css: 5.3.3 + cssnano: 5.1.15(postcss@8.4.32) + jest-worker: 29.7.0 + postcss: 8.4.32 + schema-utils: 4.2.0 + serialize-javascript: 6.0.1 + source-map: 0.6.1 + webpack: 5.89.0 + dev: false + + /css-select@4.3.0: + resolution: {integrity: sha512-wPpOYtnsVontu2mODhA19JrqWxNsfdatRKd64kmpRbQgh1KtItko5sTnEpPdpSaJszTOhEMlF/RPz28qj4HqhQ==} + dependencies: + boolbase: 1.0.0 + css-what: 6.1.0 + domhandler: 4.3.1 + domutils: 2.8.0 + nth-check: 2.0.1 + dev: false + + /css-select@5.1.0: + resolution: {integrity: sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==} + dependencies: + boolbase: 1.0.0 + css-what: 6.1.0 + domhandler: 5.0.3 + domutils: 3.1.0 + nth-check: 2.0.1 + dev: false + + /css-selector-parser@3.0.4: + resolution: {integrity: sha512-pnmS1dbKsz6KA4EW4BznyPL2xxkNDRg62hcD0v8g6DEw2W7hxOln5M953jsp9hmw5Dg57S6o/A8GOn37mbAgcQ==} + dev: false + + /css-tree@1.1.3: + resolution: {integrity: sha512-tRpdppF7TRazZrjJ6v3stzv93qxRcSsFmW6cX0Zm2NVKpxE1WV1HblnghVv9TreireHkqI/VDEsfolRF1p6y7Q==} + engines: {node: '>=8.0.0'} + dependencies: + mdn-data: 2.0.14 + source-map: 0.6.1 + dev: false + + /css-tree@2.2.1: + resolution: {integrity: sha512-OA0mILzGc1kCOCSJerOeqDxDQ4HOh+G8NbOJFOTgOCzpw7fCBubk0fEyxp8AgOL/jvLgYA/uV0cMbe43ElF1JA==} + engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0, npm: '>=7.0.0'} + dependencies: + mdn-data: 2.0.28 + source-map-js: 1.0.2 + dev: false + + /css-tree@2.3.1: + resolution: {integrity: sha512-6Fv1DV/TYw//QF5IzQdqsNDjx/wc8TrMBZsqjL9eW01tWb7R7k/mq+/VXfJCl7SoD5emsJop9cOByJZfs8hYIw==} + engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0} + dependencies: + mdn-data: 2.0.30 + source-map-js: 1.0.2 + dev: false + + /css-what@6.1.0: + resolution: {integrity: sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==} + engines: {node: '>= 6'} + dev: false + + /cssesc@3.0.0: + resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==} + engines: {node: '>=4'} + hasBin: true + dev: false + + /cssnano-preset-advanced@5.3.10(postcss@8.4.32): + resolution: {integrity: sha512-fnYJyCS9jgMU+cmHO1rPSPf9axbQyD7iUhLO5Df6O4G+fKIOMps+ZbU0PdGFejFBBZ3Pftf18fn1eG7MAPUSWQ==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + autoprefixer: 10.4.16(postcss@8.4.32) + cssnano-preset-default: 5.2.14(postcss@8.4.32) + postcss: 8.4.32 + postcss-discard-unused: 5.1.0(postcss@8.4.32) + postcss-merge-idents: 5.1.1(postcss@8.4.32) + postcss-reduce-idents: 5.2.0(postcss@8.4.32) + postcss-zindex: 5.1.0(postcss@8.4.32) + dev: false + + /cssnano-preset-advanced@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-1ziCYBklE4iQDuYy6RRumEhJDKv442d7ezzyDb1p3yYSmdz5GMan5y4xJc9YLgbiFJ9gufir9axrDUDjtT07pQ==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + autoprefixer: 10.4.16(postcss@8.4.32) + cssnano-preset-default: 6.0.3(postcss@8.4.32) + postcss: 8.4.32 + postcss-discard-unused: 6.0.2(postcss@8.4.32) + postcss-merge-idents: 6.0.1(postcss@8.4.32) + postcss-reduce-idents: 6.0.2(postcss@8.4.32) + postcss-zindex: 6.0.1(postcss@8.4.32) + dev: false + + /cssnano-preset-default@5.2.14(postcss@8.4.32): + resolution: {integrity: sha512-t0SFesj/ZV2OTylqQVOrFgEh5uanxbO6ZAdeCrNsUQ6fVuXwYTxJPNAGvGTxHbD68ldIJNec7PyYZDBrfDQ+6A==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + css-declaration-sorter: 6.4.1(postcss@8.4.32) + cssnano-utils: 3.1.0(postcss@8.4.32) + postcss: 8.4.32 + postcss-calc: 8.2.4(postcss@8.4.32) + postcss-colormin: 5.3.1(postcss@8.4.32) + postcss-convert-values: 5.1.3(postcss@8.4.32) + postcss-discard-comments: 5.1.2(postcss@8.4.32) + postcss-discard-duplicates: 5.1.0(postcss@8.4.32) + postcss-discard-empty: 5.1.1(postcss@8.4.32) + postcss-discard-overridden: 5.1.0(postcss@8.4.32) + postcss-merge-longhand: 5.1.7(postcss@8.4.32) + postcss-merge-rules: 5.1.4(postcss@8.4.32) + postcss-minify-font-values: 5.1.0(postcss@8.4.32) + postcss-minify-gradients: 5.1.1(postcss@8.4.32) + postcss-minify-params: 5.1.4(postcss@8.4.32) + postcss-minify-selectors: 5.2.1(postcss@8.4.32) + postcss-normalize-charset: 5.1.0(postcss@8.4.32) + postcss-normalize-display-values: 5.1.0(postcss@8.4.32) + postcss-normalize-positions: 5.1.1(postcss@8.4.32) + postcss-normalize-repeat-style: 5.1.1(postcss@8.4.32) + postcss-normalize-string: 5.1.0(postcss@8.4.32) + postcss-normalize-timing-functions: 5.1.0(postcss@8.4.32) + postcss-normalize-unicode: 5.1.1(postcss@8.4.32) + postcss-normalize-url: 5.1.0(postcss@8.4.32) + postcss-normalize-whitespace: 5.1.1(postcss@8.4.32) + postcss-ordered-values: 5.1.3(postcss@8.4.32) + postcss-reduce-initial: 5.1.2(postcss@8.4.32) + postcss-reduce-transforms: 5.1.0(postcss@8.4.32) + postcss-svgo: 5.1.0(postcss@8.4.32) + postcss-unique-selectors: 5.1.1(postcss@8.4.32) + dev: false + + /cssnano-preset-default@6.0.3(postcss@8.4.32): + resolution: {integrity: sha512-4y3H370aZCkT9Ev8P4SO4bZbt+AExeKhh8wTbms/X7OLDo5E7AYUUy6YPxa/uF5Grf+AJwNcCnxKhZynJ6luBA==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + css-declaration-sorter: 7.1.1(postcss@8.4.32) + cssnano-utils: 4.0.1(postcss@8.4.32) + postcss: 8.4.32 + postcss-calc: 9.0.1(postcss@8.4.32) + postcss-colormin: 6.0.2(postcss@8.4.32) + postcss-convert-values: 6.0.2(postcss@8.4.32) + postcss-discard-comments: 6.0.1(postcss@8.4.32) + postcss-discard-duplicates: 6.0.1(postcss@8.4.32) + postcss-discard-empty: 6.0.1(postcss@8.4.32) + postcss-discard-overridden: 6.0.1(postcss@8.4.32) + postcss-merge-longhand: 6.0.2(postcss@8.4.32) + postcss-merge-rules: 6.0.3(postcss@8.4.32) + postcss-minify-font-values: 6.0.1(postcss@8.4.32) + postcss-minify-gradients: 6.0.1(postcss@8.4.32) + postcss-minify-params: 6.0.2(postcss@8.4.32) + postcss-minify-selectors: 6.0.2(postcss@8.4.32) + postcss-normalize-charset: 6.0.1(postcss@8.4.32) + postcss-normalize-display-values: 6.0.1(postcss@8.4.32) + postcss-normalize-positions: 6.0.1(postcss@8.4.32) + postcss-normalize-repeat-style: 6.0.1(postcss@8.4.32) + postcss-normalize-string: 6.0.1(postcss@8.4.32) + postcss-normalize-timing-functions: 6.0.1(postcss@8.4.32) + postcss-normalize-unicode: 6.0.2(postcss@8.4.32) + postcss-normalize-url: 6.0.1(postcss@8.4.32) + postcss-normalize-whitespace: 6.0.1(postcss@8.4.32) + postcss-ordered-values: 6.0.1(postcss@8.4.32) + postcss-reduce-initial: 6.0.2(postcss@8.4.32) + postcss-reduce-transforms: 6.0.1(postcss@8.4.32) + postcss-svgo: 6.0.2(postcss@8.4.32) + postcss-unique-selectors: 6.0.2(postcss@8.4.32) + dev: false + + /cssnano-utils@3.1.0(postcss@8.4.32): + resolution: {integrity: sha512-JQNR19/YZhz4psLX/rQ9M83e3z2Wf/HdJbryzte4a3NSuafyp9w/I4U+hx5C2S9g41qlstH7DEWnZaaj83OuEA==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + dev: false + + /cssnano-utils@4.0.1(postcss@8.4.32): + resolution: {integrity: sha512-6qQuYDqsGoiXssZ3zct6dcMxiqfT6epy7x4R0TQJadd4LWO3sPR6JH6ZByOvVLoZ6EdwPGgd7+DR1EmX3tiXQQ==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + dev: false + + /cssnano@5.1.15(postcss@8.4.32): + resolution: {integrity: sha512-j+BKgDcLDQA+eDifLx0EO4XSA56b7uut3BQFH+wbSaSTuGLuiyTa/wbRYthUXX8LC9mLg+WWKe8h+qJuwTAbHw==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + cssnano-preset-default: 5.2.14(postcss@8.4.32) + lilconfig: 2.1.0 + postcss: 8.4.32 + yaml: 1.10.2 + dev: false + + /cssnano@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-Tu9wv8UdN6CoiQnIVkCNvi+0rw/BwFWOJBlg2bVfEyKaadSuE3Gq/DD8tniVvggTJGwK88UjqZp7zL5sv6t1aA==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + cssnano-preset-default: 6.0.3(postcss@8.4.32) + lilconfig: 3.0.0 + postcss: 8.4.32 + dev: false + + /csso@4.2.0: + resolution: {integrity: sha512-wvlcdIbf6pwKEk7vHj8/Bkc0B4ylXZruLvOgs9doS5eOsOpuodOV2zJChSpkp+pRpYQLQMeF04nr3Z68Sta9jA==} + engines: {node: '>=8.0.0'} + dependencies: + css-tree: 1.1.3 + dev: false + + /csso@5.0.5: + resolution: {integrity: sha512-0LrrStPOdJj+SPCCrGhzryycLjwcgUSHBtxNA8aIDxf0GLsRh1cKYhB00Gd1lDOS4yGH69+SNn13+TWbVHETFQ==} + engines: {node: ^10 || ^12.20.0 || ^14.13.0 || >=15.0.0, npm: '>=7.0.0'} + dependencies: + css-tree: 2.2.1 + dev: false + + /csstype@3.1.3: + resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==} + dev: false + + /cytoscape-cose-bilkent@4.1.0(cytoscape@3.28.1): + resolution: {integrity: sha512-wgQlVIUJF13Quxiv5e1gstZ08rnZj2XaLHGoFMYXz7SkNfCDOOteKBE6SYRfA9WxxI/iBc3ajfDoc6hb/MRAHQ==} + peerDependencies: + cytoscape: ^3.2.0 + dependencies: + cose-base: 1.0.3 + cytoscape: 3.28.1 + dev: false + + /cytoscape-fcose@2.2.0(cytoscape@3.28.1): + resolution: {integrity: sha512-ki1/VuRIHFCzxWNrsshHYPs6L7TvLu3DL+TyIGEsRcvVERmxokbf5Gdk7mFxZnTdiGtnA4cfSmjZJMviqSuZrQ==} + peerDependencies: + cytoscape: ^3.2.0 + dependencies: + cose-base: 2.2.0 + cytoscape: 3.28.1 + dev: false + + /cytoscape@3.28.1: + resolution: {integrity: sha512-xyItz4O/4zp9/239wCcH8ZcFuuZooEeF8KHRmzjDfGdXsj3OG9MFSMA0pJE0uX3uCN/ygof6hHf4L7lst+JaDg==} + engines: {node: '>=0.10'} + dependencies: + heap: 0.2.7 + lodash: 4.17.21 + dev: false + + /d3-array@2.12.1: + resolution: {integrity: sha512-B0ErZK/66mHtEsR1TkPEEkwdy+WDesimkM5gpZr5Dsg54BiTA5RXtYW5qTLIAcekaS9xfZrzBLF/OAkB3Qn1YQ==} + dependencies: + internmap: 1.0.1 + dev: false + + /d3-array@3.2.4: + resolution: {integrity: sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==} + engines: {node: '>=12'} + dependencies: + internmap: 2.0.3 + dev: false + + /d3-axis@3.0.0: + resolution: {integrity: sha512-IH5tgjV4jE/GhHkRV0HiVYPDtvfjHQlQfJHs0usq7M30XcSBvOotpmH1IgkcXsO/5gEQZD43B//fc7SRT5S+xw==} + engines: {node: '>=12'} + dev: false + + /d3-brush@3.0.0: + resolution: {integrity: sha512-ALnjWlVYkXsVIGlOsuWH1+3udkYFI48Ljihfnh8FZPF2QS9o+PzGLBslO0PjzVoHLZ2KCVgAM8NVkXPJB2aNnQ==} + engines: {node: '>=12'} + dependencies: + d3-dispatch: 3.0.1 + d3-drag: 3.0.0 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-transition: 3.0.1(d3-selection@3.0.0) + dev: false + + /d3-chord@3.0.1: + resolution: {integrity: sha512-VE5S6TNa+j8msksl7HwjxMHDM2yNK3XCkusIlpX5kwauBfXuyLAtNg9jCp/iHH61tgI4sb6R/EIMWCqEIdjT/g==} + engines: {node: '>=12'} + dependencies: + d3-path: 3.1.0 + dev: false + + /d3-color@3.1.0: + resolution: {integrity: sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==} + engines: {node: '>=12'} + dev: false + + /d3-contour@4.0.2: + resolution: {integrity: sha512-4EzFTRIikzs47RGmdxbeUvLWtGedDUNkTcmzoeyg4sP/dvCexO47AaQL7VKy/gul85TOxw+IBgA8US2xwbToNA==} + engines: {node: '>=12'} + dependencies: + d3-array: 3.2.4 + dev: false + + /d3-delaunay@6.0.4: + resolution: {integrity: sha512-mdjtIZ1XLAM8bm/hx3WwjfHt6Sggek7qH043O8KEjDXN40xi3vx/6pYSVTwLjEgiXQTbvaouWKynLBiUZ6SK6A==} + engines: {node: '>=12'} + dependencies: + delaunator: 5.0.0 + dev: false + + /d3-dispatch@3.0.1: + resolution: {integrity: sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==} + engines: {node: '>=12'} + dev: false + + /d3-drag@3.0.0: + resolution: {integrity: sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==} + engines: {node: '>=12'} + dependencies: + d3-dispatch: 3.0.1 + d3-selection: 3.0.0 + dev: false + + /d3-dsv@3.0.1: + resolution: {integrity: sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==} + engines: {node: '>=12'} + hasBin: true + dependencies: + commander: 7.2.0 + iconv-lite: 0.6.3 + rw: 1.3.3 + dev: false + + /d3-ease@3.0.1: + resolution: {integrity: sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==} + engines: {node: '>=12'} + dev: false + + /d3-fetch@3.0.1: + resolution: {integrity: sha512-kpkQIM20n3oLVBKGg6oHrUchHM3xODkTzjMoj7aWQFq5QEM+R6E4WkzT5+tojDY7yjez8KgCBRoj4aEr99Fdqw==} + engines: {node: '>=12'} + dependencies: + d3-dsv: 3.0.1 + dev: false + + /d3-force@3.0.0: + resolution: {integrity: sha512-zxV/SsA+U4yte8051P4ECydjD/S+qeYtnaIyAs9tgHCqfguma/aAQDjo85A9Z6EKhBirHRJHXIgJUlffT4wdLg==} + engines: {node: '>=12'} + dependencies: + d3-dispatch: 3.0.1 + d3-quadtree: 3.0.1 + d3-timer: 3.0.1 + dev: false + + /d3-format@3.1.0: + resolution: {integrity: sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA==} + engines: {node: '>=12'} + dev: false + + /d3-geo@3.1.0: + resolution: {integrity: sha512-JEo5HxXDdDYXCaWdwLRt79y7giK8SbhZJbFWXqbRTolCHFI5jRqteLzCsq51NKbUoX0PjBVSohxrx+NoOUujYA==} + engines: {node: '>=12'} + dependencies: + d3-array: 3.2.4 + dev: false + + /d3-hierarchy@3.1.2: + resolution: {integrity: sha512-FX/9frcub54beBdugHjDCdikxThEqjnR93Qt7PvQTOHxyiNCAlvMrHhclk3cD5VeAaq9fxmfRp+CnWw9rEMBuA==} + engines: {node: '>=12'} + dev: false + + /d3-interpolate@3.0.1: + resolution: {integrity: sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==} + engines: {node: '>=12'} + dependencies: + d3-color: 3.1.0 + dev: false + + /d3-path@1.0.9: + resolution: {integrity: sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==} + dev: false + + /d3-path@3.1.0: + resolution: {integrity: sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==} + engines: {node: '>=12'} + dev: false + + /d3-polygon@3.0.1: + resolution: {integrity: sha512-3vbA7vXYwfe1SYhED++fPUQlWSYTTGmFmQiany/gdbiWgU/iEyQzyymwL9SkJjFFuCS4902BSzewVGsHHmHtXg==} + engines: {node: '>=12'} + dev: false + + /d3-quadtree@3.0.1: + resolution: {integrity: sha512-04xDrxQTDTCFwP5H6hRhsRcb9xxv2RzkcsygFzmkSIOJy3PeRJP7sNk3VRIbKXcog561P9oU0/rVH6vDROAgUw==} + engines: {node: '>=12'} + dev: false + + /d3-random@3.0.1: + resolution: {integrity: sha512-FXMe9GfxTxqd5D6jFsQ+DJ8BJS4E/fT5mqqdjovykEB2oFbTMDVdg1MGFxfQW+FBOGoB++k8swBrgwSHT1cUXQ==} + engines: {node: '>=12'} + dev: false + + /d3-sankey@0.12.3: + resolution: {integrity: sha512-nQhsBRmM19Ax5xEIPLMY9ZmJ/cDvd1BG3UVvt5h3WRxKg5zGRbvnteTyWAbzeSvlh3tW7ZEmq4VwR5mB3tutmQ==} + dependencies: + d3-array: 2.12.1 + d3-shape: 1.3.7 + dev: false + + /d3-scale-chromatic@3.0.0: + resolution: {integrity: sha512-Lx9thtxAKrO2Pq6OO2Ua474opeziKr279P/TKZsMAhYyNDD3EnCffdbgeSYN5O7m2ByQsxtuP2CSDczNUIZ22g==} + engines: {node: '>=12'} + dependencies: + d3-color: 3.1.0 + d3-interpolate: 3.0.1 + dev: false + + /d3-scale@4.0.2: + resolution: {integrity: sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==} + engines: {node: '>=12'} + dependencies: + d3-array: 3.2.4 + d3-format: 3.1.0 + d3-interpolate: 3.0.1 + d3-time: 3.1.0 + d3-time-format: 4.1.0 + dev: false + + /d3-selection@3.0.0: + resolution: {integrity: sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==} + engines: {node: '>=12'} + dev: false + + /d3-shape@1.3.7: + resolution: {integrity: sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==} + dependencies: + d3-path: 1.0.9 + dev: false + + /d3-shape@3.2.0: + resolution: {integrity: sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==} + engines: {node: '>=12'} + dependencies: + d3-path: 3.1.0 + dev: false + + /d3-time-format@4.1.0: + resolution: {integrity: sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==} + engines: {node: '>=12'} + dependencies: + d3-time: 3.1.0 + dev: false + + /d3-time@3.1.0: + resolution: {integrity: sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==} + engines: {node: '>=12'} + dependencies: + d3-array: 3.2.4 + dev: false + + /d3-timer@3.0.1: + resolution: {integrity: sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==} + engines: {node: '>=12'} + dev: false + + /d3-transition@3.0.1(d3-selection@3.0.0): + resolution: {integrity: sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==} + engines: {node: '>=12'} + peerDependencies: + d3-selection: 2 - 3 + dependencies: + d3-color: 3.1.0 + d3-dispatch: 3.0.1 + d3-ease: 3.0.1 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-timer: 3.0.1 + dev: false + + /d3-zoom@3.0.0: + resolution: {integrity: sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==} + engines: {node: '>=12'} + dependencies: + d3-dispatch: 3.0.1 + d3-drag: 3.0.0 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-transition: 3.0.1(d3-selection@3.0.0) + dev: false + + /d3@7.8.5: + resolution: {integrity: sha512-JgoahDG51ncUfJu6wX/1vWQEqOflgXyl4MaHqlcSruTez7yhaRKR9i8VjjcQGeS2en/jnFivXuaIMnseMMt0XA==} + engines: {node: '>=12'} + dependencies: + d3-array: 3.2.4 + d3-axis: 3.0.0 + d3-brush: 3.0.0 + d3-chord: 3.0.1 + d3-color: 3.1.0 + d3-contour: 4.0.2 + d3-delaunay: 6.0.4 + d3-dispatch: 3.0.1 + d3-drag: 3.0.0 + d3-dsv: 3.0.1 + d3-ease: 3.0.1 + d3-fetch: 3.0.1 + d3-force: 3.0.0 + d3-format: 3.1.0 + d3-geo: 3.1.0 + d3-hierarchy: 3.1.2 + d3-interpolate: 3.0.1 + d3-path: 3.1.0 + d3-polygon: 3.0.1 + d3-quadtree: 3.0.1 + d3-random: 3.0.1 + d3-scale: 4.0.2 + d3-scale-chromatic: 3.0.0 + d3-selection: 3.0.0 + d3-shape: 3.2.0 + d3-time: 3.1.0 + d3-time-format: 4.1.0 + d3-timer: 3.0.1 + d3-transition: 3.0.1(d3-selection@3.0.0) + d3-zoom: 3.0.0 + dev: false + + /dagre-d3-es@7.0.10: + resolution: {integrity: sha512-qTCQmEhcynucuaZgY5/+ti3X/rnszKZhEQH/ZdWdtP1tA/y3VoHJzcVrO9pjjJCNpigfscAtoUB5ONcd2wNn0A==} + dependencies: + d3: 7.8.5 + lodash-es: 4.17.21 + dev: false + + /data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + dev: false + + /dayjs@1.11.10: + resolution: {integrity: sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ==} + dev: false + + /debounce@1.2.1: + resolution: {integrity: sha512-XRRe6Glud4rd/ZGQfiV1ruXSfbvfJedlV9Y6zOlP+2K04vBYiJEte6stfFkCP03aMnY5tsipamumUjL14fofug==} + dev: false + + /debug@2.6.9: + resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + dependencies: + ms: 2.0.0 + dev: false + + /debug@4.3.4: + resolution: {integrity: sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + dependencies: + ms: 2.1.2 + dev: false + + /decode-named-character-reference@1.0.2: + resolution: {integrity: sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==} + dependencies: + character-entities: 2.0.2 + dev: false + + /decompress-response@6.0.0: + resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} + engines: {node: '>=10'} + dependencies: + mimic-response: 3.1.0 + dev: false + + /deep-extend@0.6.0: + resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} + engines: {node: '>=4.0.0'} + dev: false + + /deepmerge@4.3.1: + resolution: {integrity: sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==} + engines: {node: '>=0.10.0'} + dev: false + + /default-gateway@6.0.3: + resolution: {integrity: sha512-fwSOJsbbNzZ/CUFpqFBqYfYNLj1NbMPm8MMCIzHjC83iSJRBEGmDUxU+WP661BaBQImeC2yHwXtz+P/O9o+XEg==} + engines: {node: '>= 10'} + dependencies: + execa: 5.1.1 + dev: false + + /defer-to-connect@2.0.1: + resolution: {integrity: sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==} + engines: {node: '>=10'} + dev: false + + /define-data-property@1.1.1: + resolution: {integrity: sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ==} + engines: {node: '>= 0.4'} + dependencies: + get-intrinsic: 1.2.2 + gopd: 1.0.1 + has-property-descriptors: 1.0.1 + dev: false + + /define-lazy-prop@2.0.0: + resolution: {integrity: sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==} + engines: {node: '>=8'} + dev: false + + /define-properties@1.2.1: + resolution: {integrity: sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==} + engines: {node: '>= 0.4'} + dependencies: + define-data-property: 1.1.1 + has-property-descriptors: 1.0.1 + object-keys: 1.1.1 + dev: false + + /del@6.1.1: + resolution: {integrity: sha512-ua8BhapfP0JUJKC/zV9yHHDW/rDoDxP4Zhn3AkA6/xT6gY7jYXJiaeyBZznYVujhZZET+UgcbZiQ7sN3WqcImg==} + engines: {node: '>=10'} + dependencies: + globby: 11.1.0 + graceful-fs: 4.2.11 + is-glob: 4.0.3 + is-path-cwd: 2.2.0 + is-path-inside: 3.0.3 + p-map: 4.0.0 + rimraf: 3.0.2 + slash: 3.0.0 + dev: false + + /delaunator@5.0.0: + resolution: {integrity: sha512-AyLvtyJdbv/U1GkiS6gUUzclRoAY4Gs75qkMygJJhU75LW4DNuSF2RMzpxs9jw9Oz1BobHjTdkG3zdP55VxAqw==} + dependencies: + robust-predicates: 3.0.2 + dev: false + + /depd@1.1.2: + resolution: {integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==} + engines: {node: '>= 0.6'} + dev: false + + /depd@2.0.0: + resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} + engines: {node: '>= 0.8'} + dev: false + + /dequal@2.0.3: + resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} + engines: {node: '>=6'} + dev: false + + /destroy@1.2.0: + resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==} + engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + dev: false + + /detect-node@2.1.0: + resolution: {integrity: sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==} + dev: false + + /detect-port-alt@1.1.6: + resolution: {integrity: sha512-5tQykt+LqfJFBEYaDITx7S7cR7mJ/zQmLXZ2qt5w04ainYZw6tBf9dBunMjVeVOdYVRUzUOE4HkY5J7+uttb5Q==} + engines: {node: '>= 4.2.1'} + hasBin: true + dependencies: + address: 1.2.2 + debug: 2.6.9 + transitivePeerDependencies: + - supports-color + dev: false + + /detect-port@1.5.1: + resolution: {integrity: sha512-aBzdj76lueB6uUst5iAs7+0H/oOjqI5D16XUWxlWMIMROhcM0rfsNVk93zTngq1dDNpoXRr++Sus7ETAExppAQ==} + hasBin: true + dependencies: + address: 1.2.2 + debug: 4.3.4 + transitivePeerDependencies: + - supports-color + dev: false + + /devlop@1.1.0: + resolution: {integrity: sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==} + dependencies: + dequal: 2.0.3 + dev: false + + /diff@5.1.0: + resolution: {integrity: sha512-D+mk+qE8VC/PAUrlAU34N+VfXev0ghe5ywmpqrawphmVZc1bEfn56uo9qpyGp1p4xpzOHkSW4ztBd6L7Xx4ACw==} + engines: {node: '>=0.3.1'} + dev: false + + /dir-glob@3.0.1: + resolution: {integrity: sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==} + engines: {node: '>=8'} + dependencies: + path-type: 4.0.0 + dev: false + + /dns-packet@5.6.1: + resolution: {integrity: sha512-l4gcSouhcgIKRvyy99RNVOgxXiicE+2jZoNmaNmZ6JXiGajBOJAesk1OBlJuM5k2c+eudGdLxDqXuPCKIj6kpw==} + engines: {node: '>=6'} + dependencies: + '@leichtgewicht/ip-codec': 2.0.4 + dev: false + + /docusaurus-plugin-hubspot@1.0.0: + resolution: {integrity: sha512-qeNRlI336M6mcANGrEDxMt7B+GWvWd+yXOpT6uquQLX0b2cDHncDj0+rbuUaJUNQnKQz1st7sKgJQ0P55736Ug==} + deprecated: docusaurus-plugin-hubspot is now available at @stackql/docusaurus-plugin-hubspot + dev: false + + /docusaurus-plugin-segment@1.0.3: + resolution: {integrity: sha512-9DqebTx9TqjujCnB22qEeCm8NGJUAH7VAKLAa20/CyfSSrs+khTQI0FmzEALtiCqKNO1D3GWm3VvE4gqbuGqnw==} + dependencies: + '@segment/snippet': 4.16.2 + dev: false + + /dom-converter@0.2.0: + resolution: {integrity: sha512-gd3ypIPfOMr9h5jIKq8E3sHOTCjeirnl0WK5ZdS1AW0Odt0b1PaWaHdJ4Qk4klv+YB9aJBS7mESXjFoDQPu6DA==} + dependencies: + utila: 0.4.0 + dev: false + + /dom-serializer@1.4.1: + resolution: {integrity: sha512-VHwB3KfrcOOkelEG2ZOfxqLZdfkil8PtJi4P8N2MMXucZq2yLp75ClViUlOVwyoHEDjYU433Aq+5zWP61+RGag==} + dependencies: + domelementtype: 2.3.0 + domhandler: 4.3.1 + entities: 2.2.0 + dev: false + + /dom-serializer@2.0.0: + resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==} + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + entities: 4.5.0 + dev: false + + /domelementtype@2.3.0: + resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==} + dev: false + + /domhandler@4.3.1: + resolution: {integrity: sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==} + engines: {node: '>= 4'} + dependencies: + domelementtype: 2.3.0 + dev: false + + /domhandler@5.0.3: + resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==} + engines: {node: '>= 4'} + dependencies: + domelementtype: 2.3.0 + dev: false + + /dompurify@3.0.6: + resolution: {integrity: sha512-ilkD8YEnnGh1zJ240uJsW7AzE+2qpbOUYjacomn3AvJ6J4JhKGSZ2nh4wUIXPZrEPppaCLx5jFe8T89Rk8tQ7w==} + dev: false + + /domutils@2.8.0: + resolution: {integrity: sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==} + dependencies: + dom-serializer: 1.4.1 + domelementtype: 2.3.0 + domhandler: 4.3.1 + dev: false + + /domutils@3.1.0: + resolution: {integrity: sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==} + dependencies: + dom-serializer: 2.0.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + dev: false + + /dot-case@3.0.4: + resolution: {integrity: sha512-Kv5nKlh6yRrdrGvxeJ2e5y2eRUpkUosIW4A2AS38zwSz27zu7ufDwQPi5Jhs3XAlGNetl3bmnGhQsMtkKJnj3w==} + dependencies: + no-case: 3.0.4 + tslib: 2.6.2 + dev: false + + /dot-prop@6.0.1: + resolution: {integrity: sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA==} + engines: {node: '>=10'} + dependencies: + is-obj: 2.0.0 + dev: false + + /duplexer@0.1.2: + resolution: {integrity: sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==} + dev: false + + /eastasianwidth@0.2.0: + resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + dev: false + + /ee-first@1.1.1: + resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} + dev: false + + /electron-to-chromium@1.4.618: + resolution: {integrity: sha512-mTM2HieHLxs1RbD/R/ZoQLMsGI8lWIkP17G7cx32mJRBJt9wlNPkXwE3sYg/OnNb5GBkus98lXatSthoL8Y5Ag==} + dev: false + + /elkjs@0.8.2: + resolution: {integrity: sha512-L6uRgvZTH+4OF5NE/MBbzQx/WYpru1xCBE9respNj6qznEewGUIfhzmm7horWWxbNO2M0WckQypGctR8lH79xQ==} + dev: false + + /emoji-regex@8.0.0: + resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + dev: false + + /emoji-regex@9.2.2: + resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} + dev: false + + /emojilib@2.4.0: + resolution: {integrity: sha512-5U0rVMU5Y2n2+ykNLQqMoqklN9ICBT/KsvC1Gz6vqHbz2AXXGkG+Pm5rMWk/8Vjrr/mY9985Hi8DYzn1F09Nyw==} + dev: false + + /emojis-list@3.0.0: + resolution: {integrity: sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q==} + engines: {node: '>= 4'} + dev: false + + /emoticon@4.0.1: + resolution: {integrity: sha512-dqx7eA9YaqyvYtUhJwT4rC1HIp82j5ybS1/vQ42ur+jBe17dJMwZE4+gvL1XadSFfxaPFFGt3Xsw+Y8akThDlw==} + dev: false + + /encodeurl@1.0.2: + resolution: {integrity: sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==} + engines: {node: '>= 0.8'} + dev: false + + /enhanced-resolve@5.15.0: + resolution: {integrity: sha512-LXYT42KJ7lpIKECr2mAXIaMldcNCh/7E0KBKOu4KSfkHmP+mZmSs+8V5gBAqisWBy0OO4W5Oyys0GO1Y8KtdKg==} + engines: {node: '>=10.13.0'} + dependencies: + graceful-fs: 4.2.11 + tapable: 2.2.1 + dev: false + + /entities@2.2.0: + resolution: {integrity: sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==} + dev: false + + /entities@4.5.0: + resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==} + engines: {node: '>=0.12'} + dev: false + + /error-ex@1.3.2: + resolution: {integrity: sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==} + dependencies: + is-arrayish: 0.2.1 + dev: false + + /es-module-lexer@1.4.1: + resolution: {integrity: sha512-cXLGjP0c4T3flZJKQSuziYoq7MlT+rnvfZjfp7h+I7K9BNX54kP9nyWvdbwjQ4u1iWbOL4u96fgeZLToQlZC7w==} + dev: false + + /escalade@3.1.1: + resolution: {integrity: sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==} + engines: {node: '>=6'} + dev: false + + /escape-goat@4.0.0: + resolution: {integrity: sha512-2Sd4ShcWxbx6OY1IHyla/CVNwvg7XwZVoXZHcSu9w9SReNP1EzzD5T8NWKIR38fIqEns9kDWKUQTXXAmlDrdPg==} + engines: {node: '>=12'} + dev: false + + /escape-html@1.0.3: + resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} + dev: false + + /escape-string-regexp@1.0.5: + resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==} + engines: {node: '>=0.8.0'} + dev: false + + /escape-string-regexp@4.0.0: + resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} + engines: {node: '>=10'} + dev: false + + /escape-string-regexp@5.0.0: + resolution: {integrity: sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==} + engines: {node: '>=12'} + dev: false + + /eslint-scope@5.1.1: + resolution: {integrity: sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==} + engines: {node: '>=8.0.0'} + dependencies: + esrecurse: 4.3.0 + estraverse: 4.3.0 + dev: false + + /esprima@4.0.1: + resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==} + engines: {node: '>=4'} + hasBin: true + dev: false + + /esrecurse@4.3.0: + resolution: {integrity: sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==} + engines: {node: '>=4.0'} + dependencies: + estraverse: 5.3.0 + dev: false + + /estraverse@4.3.0: + resolution: {integrity: sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==} + engines: {node: '>=4.0'} + dev: false + + /estraverse@5.3.0: + resolution: {integrity: sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==} + engines: {node: '>=4.0'} + dev: false + + /estree-util-attach-comments@3.0.0: + resolution: {integrity: sha512-cKUwm/HUcTDsYh/9FgnuFqpfquUbwIqwKM26BVCGDPVgvaCl/nDCCjUfiLlx6lsEZ3Z4RFxNbOQ60pkaEwFxGw==} + dependencies: + '@types/estree': 1.0.5 + dev: false + + /estree-util-build-jsx@3.0.1: + resolution: {integrity: sha512-8U5eiL6BTrPxp/CHbs2yMgP8ftMhR5ww1eIKoWRMlqvltHF8fZn5LRDvTKuxD3DUn+shRbLGqXemcP51oFCsGQ==} + dependencies: + '@types/estree-jsx': 1.0.3 + devlop: 1.1.0 + estree-util-is-identifier-name: 3.0.0 + estree-walker: 3.0.3 + dev: false + + /estree-util-is-identifier-name@3.0.0: + resolution: {integrity: sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg==} + dev: false + + /estree-util-to-js@2.0.0: + resolution: {integrity: sha512-WDF+xj5rRWmD5tj6bIqRi6CkLIXbbNQUcxQHzGysQzvHmdYG2G7p/Tf0J0gpxGgkeMZNTIjT/AoSvC9Xehcgdg==} + dependencies: + '@types/estree-jsx': 1.0.3 + astring: 1.8.6 + source-map: 0.7.4 + dev: false + + /estree-util-value-to-estree@3.0.1: + resolution: {integrity: sha512-b2tdzTurEIbwRh+mKrEcaWfu1wgb8J1hVsgREg7FFiecWwK/PhO8X0kyc+0bIcKNtD4sqxIdNoRy6/p/TvECEA==} + engines: {node: '>=16.0.0'} + dependencies: + '@types/estree': 1.0.5 + is-plain-obj: 4.1.0 + dev: false + + /estree-util-visit@2.0.0: + resolution: {integrity: sha512-m5KgiH85xAhhW8Wta0vShLcUvOsh3LLPI2YVwcbio1l7E09NTLL1EyMZFM1OyWowoH0skScNbhOPl4kcBgzTww==} + dependencies: + '@types/estree-jsx': 1.0.3 + '@types/unist': 3.0.2 + dev: false + + /estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + dependencies: + '@types/estree': 1.0.5 + dev: false + + /esutils@2.0.3: + resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} + engines: {node: '>=0.10.0'} + dev: false + + /eta@2.2.0: + resolution: {integrity: sha512-UVQ72Rqjy/ZKQalzV5dCCJP80GrmPrMxh6NlNf+erV6ObL0ZFkhCstWRawS85z3smdr3d2wXPsZEY7rDPfGd2g==} + engines: {node: '>=6.0.0'} + dev: false + + /etag@1.8.1: + resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} + engines: {node: '>= 0.6'} + dev: false + + /eval@0.1.8: + resolution: {integrity: sha512-EzV94NYKoO09GLXGjXj9JIlXijVck4ONSr5wiCWDvhsvj5jxSrzTmRU/9C1DyB6uToszLs8aifA6NQ7lEQdvFw==} + engines: {node: '>= 0.8'} + dependencies: + '@types/node': 20.10.6 + require-like: 0.1.2 + dev: false + + /eventemitter3@4.0.7: + resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==} + dev: false + + /events@3.3.0: + resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} + engines: {node: '>=0.8.x'} + dev: false + + /execa@5.1.1: + resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==} + engines: {node: '>=10'} + dependencies: + cross-spawn: 7.0.3 + get-stream: 6.0.1 + human-signals: 2.1.0 + is-stream: 2.0.1 + merge-stream: 2.0.0 + npm-run-path: 4.0.1 + onetime: 5.1.2 + signal-exit: 3.0.7 + strip-final-newline: 2.0.0 + dev: false + + /express@4.18.2: + resolution: {integrity: sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==} + engines: {node: '>= 0.10.0'} + dependencies: + accepts: 1.3.8 + array-flatten: 1.1.1 + body-parser: 1.20.1 + content-disposition: 0.5.4 + content-type: 1.0.5 + cookie: 0.5.0 + cookie-signature: 1.0.6 + debug: 2.6.9 + depd: 2.0.0 + encodeurl: 1.0.2 + escape-html: 1.0.3 + etag: 1.8.1 + finalhandler: 1.2.0 + fresh: 0.5.2 + http-errors: 2.0.0 + merge-descriptors: 1.0.1 + methods: 1.1.2 + on-finished: 2.4.1 + parseurl: 1.3.3 + path-to-regexp: 0.1.7 + proxy-addr: 2.0.7 + qs: 6.11.0 + range-parser: 1.2.1 + safe-buffer: 5.2.1 + send: 0.18.0 + serve-static: 1.15.0 + setprototypeof: 1.2.0 + statuses: 2.0.1 + type-is: 1.6.18 + utils-merge: 1.0.1 + vary: 1.1.2 + transitivePeerDependencies: + - supports-color + dev: false + + /extend-shallow@2.0.1: + resolution: {integrity: sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==} + engines: {node: '>=0.10.0'} + dependencies: + is-extendable: 0.1.1 + dev: false + + /extend@3.0.2: + resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} + dev: false + + /fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + dev: false + + /fast-glob@3.3.2: + resolution: {integrity: sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==} + engines: {node: '>=8.6.0'} + dependencies: + '@nodelib/fs.stat': 2.0.5 + '@nodelib/fs.walk': 1.2.8 + glob-parent: 5.1.2 + merge2: 1.4.1 + micromatch: 4.0.5 + dev: false + + /fast-json-stable-stringify@2.1.0: + resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} + dev: false + + /fast-url-parser@1.1.3: + resolution: {integrity: sha512-5jOCVXADYNuRkKFzNJ0dCCewsZiYo0dz8QNYljkOpFC6r2U4OBmKtvm/Tsuh4w1YYdDqDb31a8TVhBJ2OJKdqQ==} + dependencies: + punycode: 1.4.1 + dev: false + + /fastq@1.16.0: + resolution: {integrity: sha512-ifCoaXsDrsdkWTtiNJX5uzHDsrck5TzfKKDcuFFTIrrc/BS076qgEIfoIy1VeZqViznfKiysPYTh/QeHtnIsYA==} + dependencies: + reusify: 1.0.4 + dev: false + + /fault@2.0.1: + resolution: {integrity: sha512-WtySTkS4OKev5JtpHXnib4Gxiurzh5NCGvWrFaZ34m6JehfTUhKZvn9njTfw48t6JumVQOmrKqpmGcdwxnhqBQ==} + dependencies: + format: 0.2.2 + dev: false + + /faye-websocket@0.11.4: + resolution: {integrity: sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g==} + engines: {node: '>=0.8.0'} + dependencies: + websocket-driver: 0.7.4 + dev: false + + /feed@4.2.2: + resolution: {integrity: sha512-u5/sxGfiMfZNtJ3OvQpXcvotFpYkL0n9u9mM2vkui2nGo8b4wvDkJ8gAkYqbA8QpGyFCv3RK0Z+Iv+9veCS9bQ==} + engines: {node: '>=0.4.0'} + dependencies: + xml-js: 1.6.11 + dev: false + + /fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.2.1 + dev: false + + /file-loader@6.2.0(webpack@5.89.0): + resolution: {integrity: sha512-qo3glqyTa61Ytg4u73GultjHGjdRyig3tG6lPtyX/jOEJvHif9uB0/OCI2Kif6ctF3caQTW2G5gym21oAsI4pw==} + engines: {node: '>= 10.13.0'} + peerDependencies: + webpack: ^4.0.0 || ^5.0.0 + dependencies: + loader-utils: 2.0.4 + schema-utils: 3.3.0 + webpack: 5.89.0 + dev: false + + /filesize@8.0.7: + resolution: {integrity: sha512-pjmC+bkIF8XI7fWaH8KxHcZL3DPybs1roSKP4rKDvy20tAWwIObE4+JIseG2byfGKhud5ZnM4YSGKBz7Sh0ndQ==} + engines: {node: '>= 0.4.0'} + dev: false + + /fill-range@7.0.1: + resolution: {integrity: sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==} + engines: {node: '>=8'} + dependencies: + to-regex-range: 5.0.1 + dev: false + + /finalhandler@1.2.0: + resolution: {integrity: sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==} + engines: {node: '>= 0.8'} + dependencies: + debug: 2.6.9 + encodeurl: 1.0.2 + escape-html: 1.0.3 + on-finished: 2.4.1 + parseurl: 1.3.3 + statuses: 2.0.1 + unpipe: 1.0.0 + transitivePeerDependencies: + - supports-color + dev: false + + /find-cache-dir@4.0.0: + resolution: {integrity: sha512-9ZonPT4ZAK4a+1pUPVPZJapbi7O5qbbJPdYw/NOQWZZbVLdDTYM3A4R9z/DpAM08IDaFGsvPgiGZ82WEwUDWjg==} + engines: {node: '>=14.16'} + dependencies: + common-path-prefix: 3.0.0 + pkg-dir: 7.0.0 + dev: false + + /find-up@3.0.0: + resolution: {integrity: sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==} + engines: {node: '>=6'} + dependencies: + locate-path: 3.0.0 + dev: false + + /find-up@5.0.0: + resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} + engines: {node: '>=10'} + dependencies: + locate-path: 6.0.0 + path-exists: 4.0.0 + dev: false + + /find-up@6.3.0: + resolution: {integrity: sha512-v2ZsoEuVHYy8ZIlYqwPe/39Cy+cFDzp4dXPaxNvkEuouymu+2Jbz0PxpKarJHYJTmv2HWT3O382qY8l4jMWthw==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + locate-path: 7.2.0 + path-exists: 5.0.0 + dev: false + + /flat@5.0.2: + resolution: {integrity: sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==} + hasBin: true + dev: false + + /follow-redirects@1.15.4: + resolution: {integrity: sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + dev: false + + /fork-ts-checker-webpack-plugin@6.5.3(typescript@5.3.3)(webpack@5.89.0): + resolution: {integrity: sha512-SbH/l9ikmMWycd5puHJKTkZJKddF4iRLyW3DeZ08HTI7NGyLS38MXd/KGgeWumQO7YNQbW2u/NtPT2YowbPaGQ==} + engines: {node: '>=10', yarn: '>=1.0.0'} + peerDependencies: + eslint: '>= 6' + typescript: '>= 2.7' + vue-template-compiler: '*' + webpack: '>= 4' + peerDependenciesMeta: + eslint: + optional: true + vue-template-compiler: + optional: true + dependencies: + '@babel/code-frame': 7.23.5 + '@types/json-schema': 7.0.15 + chalk: 4.1.2 + chokidar: 3.5.3 + cosmiconfig: 6.0.0 + deepmerge: 4.3.1 + fs-extra: 9.1.0 + glob: 7.2.3 + memfs: 3.5.3 + minimatch: 3.1.2 + schema-utils: 2.7.0 + semver: 7.5.4 + tapable: 1.1.3 + typescript: 5.3.3 + webpack: 5.89.0 + dev: false + + /form-data-encoder@2.1.4: + resolution: {integrity: sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==} + engines: {node: '>= 14.17'} + dev: false + + /format@0.2.2: + resolution: {integrity: sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==} + engines: {node: '>=0.4.x'} + dev: false + + /formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + dependencies: + fetch-blob: 3.2.0 + dev: false + + /forwarded@0.2.0: + resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} + engines: {node: '>= 0.6'} + dev: false + + /fraction.js@4.3.7: + resolution: {integrity: sha512-ZsDfxO51wGAXREY55a7la9LScWpwv9RxIrYABrlvOFBlH/ShPnrtsXeuUIfXKKOVicNxQ+o8JTbJvjS4M89yew==} + dev: false + + /fresh@0.5.2: + resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} + engines: {node: '>= 0.6'} + dev: false + + /fs-extra@11.2.0: + resolution: {integrity: sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==} + engines: {node: '>=14.14'} + dependencies: + graceful-fs: 4.2.11 + jsonfile: 6.1.0 + universalify: 2.0.1 + dev: false + + /fs-extra@9.1.0: + resolution: {integrity: sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==} + engines: {node: '>=10'} + dependencies: + at-least-node: 1.0.0 + graceful-fs: 4.2.11 + jsonfile: 6.1.0 + universalify: 2.0.1 + dev: false + + /fs-monkey@1.0.5: + resolution: {integrity: sha512-8uMbBjrhzW76TYgEV27Y5E//W2f/lTFmx78P2w19FZSxarhI/798APGQyuGCwmkNxgwGRhrLfvWyLBvNtuOmew==} + dev: false + + /fs.realpath@1.0.0: + resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} + dev: false + + /fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + requiresBuild: true + dev: false + optional: true + + /function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + dev: false + + /gensync@1.0.0-beta.2: + resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} + engines: {node: '>=6.9.0'} + dev: false + + /get-intrinsic@1.2.2: + resolution: {integrity: sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==} + dependencies: + function-bind: 1.1.2 + has-proto: 1.0.1 + has-symbols: 1.0.3 + hasown: 2.0.0 + dev: false + + /get-own-enumerable-property-symbols@3.0.2: + resolution: {integrity: sha512-I0UBV/XOz1XkIJHEUDMZAbzCThU/H8DxmSfmdGcKPnVhu2VfFqr34jr9777IyaTYvxjedWhqVIilEDsCdP5G6g==} + dev: false + + /get-stream@6.0.1: + resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==} + engines: {node: '>=10'} + dev: false + + /github-slugger@1.5.0: + resolution: {integrity: sha512-wIh+gKBI9Nshz2o46B0B3f5k/W+WI9ZAv6y5Dn5WJ5SK1t0TnDimB4WE5rmTD05ZAIn8HALCZVmCsvj0w0v0lw==} + dev: false + + /glob-parent@5.1.2: + resolution: {integrity: sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==} + engines: {node: '>= 6'} + dependencies: + is-glob: 4.0.3 + dev: false + + /glob-parent@6.0.2: + resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} + engines: {node: '>=10.13.0'} + dependencies: + is-glob: 4.0.3 + dev: false + + /glob-to-regexp@0.4.1: + resolution: {integrity: sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==} + dev: false + + /glob@7.2.3: + resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} + dependencies: + fs.realpath: 1.0.0 + inflight: 1.0.6 + inherits: 2.0.4 + minimatch: 3.1.2 + once: 1.4.0 + path-is-absolute: 1.0.1 + dev: false + + /global-dirs@3.0.1: + resolution: {integrity: sha512-NBcGGFbBA9s1VzD41QXDG+3++t9Mn5t1FpLdhESY6oKY4gYTFpX4wO3sqGUa0Srjtbfj3szX0RnemmrVRUdULA==} + engines: {node: '>=10'} + dependencies: + ini: 2.0.0 + dev: false + + /global-modules@2.0.0: + resolution: {integrity: sha512-NGbfmJBp9x8IxyJSd1P+otYK8vonoJactOogrVfFRIAEY1ukil8RSKDz2Yo7wh1oihl51l/r6W4epkeKJHqL8A==} + engines: {node: '>=6'} + dependencies: + global-prefix: 3.0.0 + dev: false + + /global-prefix@3.0.0: + resolution: {integrity: sha512-awConJSVCHVGND6x3tmMaKcQvwXLhjdkmomy2W+Goaui8YPgYgXJZewhg3fWC+DlfqqQuWg8AwqjGTD2nAPVWg==} + engines: {node: '>=6'} + dependencies: + ini: 1.3.8 + kind-of: 6.0.3 + which: 1.3.1 + dev: false + + /globals@11.12.0: + resolution: {integrity: sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==} + engines: {node: '>=4'} + dev: false + + /globby@11.1.0: + resolution: {integrity: sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==} + engines: {node: '>=10'} + dependencies: + array-union: 2.1.0 + dir-glob: 3.0.1 + fast-glob: 3.3.2 + ignore: 5.3.0 + merge2: 1.4.1 + slash: 3.0.0 + dev: false + + /globby@13.2.2: + resolution: {integrity: sha512-Y1zNGV+pzQdh7H39l9zgB4PJqjRNqydvdYCDG4HFXM4XuvSaQQlEc91IU1yALL8gUTDomgBAfz3XJdmUS+oo0w==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + dir-glob: 3.0.1 + fast-glob: 3.3.2 + ignore: 5.3.0 + merge2: 1.4.1 + slash: 4.0.0 + dev: false + + /gopd@1.0.1: + resolution: {integrity: sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==} + dependencies: + get-intrinsic: 1.2.2 + dev: false + + /got@12.6.1: + resolution: {integrity: sha512-mThBblvlAF1d4O5oqyvN+ZxLAYwIJK7bpMxgYqPD9okW0C3qm5FFn7k811QrcuEBwaogR3ngOFoCfs6mRv7teQ==} + engines: {node: '>=14.16'} + dependencies: + '@sindresorhus/is': 5.6.0 + '@szmarczak/http-timer': 5.0.1 + cacheable-lookup: 7.0.0 + cacheable-request: 10.2.14 + decompress-response: 6.0.0 + form-data-encoder: 2.1.4 + get-stream: 6.0.1 + http2-wrapper: 2.2.1 + lowercase-keys: 3.0.0 + p-cancelable: 3.0.0 + responselike: 3.0.0 + dev: false + + /graceful-fs@4.2.10: + resolution: {integrity: sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==} + dev: false + + /graceful-fs@4.2.11: + resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + dev: false + + /gray-matter@4.0.3: + resolution: {integrity: sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==} + engines: {node: '>=6.0'} + dependencies: + js-yaml: 3.14.1 + kind-of: 6.0.3 + section-matter: 1.0.0 + strip-bom-string: 1.0.0 + dev: false + + /gzip-size@6.0.0: + resolution: {integrity: sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q==} + engines: {node: '>=10'} + dependencies: + duplexer: 0.1.2 + dev: false + + /handle-thing@2.0.1: + resolution: {integrity: sha512-9Qn4yBxelxoh2Ow62nP+Ka/kMnOXRi8BXnRaUwezLNhqelnN49xKz4F/dPP8OYLxLxq6JDtZb2i9XznUQbNPTg==} + dev: false + + /has-flag@3.0.0: + resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} + engines: {node: '>=4'} + dev: false + + /has-flag@4.0.0: + resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} + engines: {node: '>=8'} + dev: false + + /has-property-descriptors@1.0.1: + resolution: {integrity: sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg==} + dependencies: + get-intrinsic: 1.2.2 + dev: false + + /has-proto@1.0.1: + resolution: {integrity: sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==} + engines: {node: '>= 0.4'} + dev: false + + /has-symbols@1.0.3: + resolution: {integrity: sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==} + engines: {node: '>= 0.4'} + dev: false + + /has-yarn@3.0.0: + resolution: {integrity: sha512-IrsVwUHhEULx3R8f/aA8AHuEzAorplsab/v8HBzEiIukwq5i/EC+xmOW+HfP1OaDP+2JkgT1yILHN2O3UFIbcA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dev: false + + /hasown@2.0.0: + resolution: {integrity: sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==} + engines: {node: '>= 0.4'} + dependencies: + function-bind: 1.1.2 + dev: false + + /hast-util-from-parse5@8.0.1: + resolution: {integrity: sha512-Er/Iixbc7IEa7r/XLtuG52zoqn/b3Xng/w6aZQ0xGVxzhw5xUFxcRqdPzP6yFi/4HBYRaifaI5fQ1RH8n0ZeOQ==} + dependencies: + '@types/hast': 3.0.3 + '@types/unist': 3.0.2 + devlop: 1.1.0 + hastscript: 8.0.0 + property-information: 6.4.0 + vfile: 6.0.1 + vfile-location: 5.0.2 + web-namespaces: 2.0.1 + dev: false + + /hast-util-parse-selector@4.0.0: + resolution: {integrity: sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==} + dependencies: + '@types/hast': 3.0.3 + dev: false + + /hast-util-raw@9.0.1: + resolution: {integrity: sha512-5m1gmba658Q+lO5uqL5YNGQWeh1MYWZbZmWrM5lncdcuiXuo5E2HT/CIOp0rLF8ksfSwiCVJ3twlgVRyTGThGA==} + dependencies: + '@types/hast': 3.0.3 + '@types/unist': 3.0.2 + '@ungap/structured-clone': 1.2.0 + hast-util-from-parse5: 8.0.1 + hast-util-to-parse5: 8.0.0 + html-void-elements: 3.0.0 + mdast-util-to-hast: 13.0.2 + parse5: 7.1.2 + unist-util-position: 5.0.0 + unist-util-visit: 5.0.0 + vfile: 6.0.1 + web-namespaces: 2.0.1 + zwitch: 2.0.4 + dev: false + + /hast-util-to-estree@3.1.0: + resolution: {integrity: sha512-lfX5g6hqVh9kjS/B9E2gSkvHH4SZNiQFiqWS0x9fENzEl+8W12RqdRxX6d/Cwxi30tPQs3bIO+aolQJNp1bIyw==} + dependencies: + '@types/estree': 1.0.5 + '@types/estree-jsx': 1.0.3 + '@types/hast': 3.0.3 + comma-separated-tokens: 2.0.3 + devlop: 1.1.0 + estree-util-attach-comments: 3.0.0 + estree-util-is-identifier-name: 3.0.0 + hast-util-whitespace: 3.0.0 + mdast-util-mdx-expression: 2.0.0 + mdast-util-mdx-jsx: 3.0.0 + mdast-util-mdxjs-esm: 2.0.1 + property-information: 6.4.0 + space-separated-tokens: 2.0.2 + style-to-object: 0.4.4 + unist-util-position: 5.0.0 + zwitch: 2.0.4 + transitivePeerDependencies: + - supports-color + dev: false + + /hast-util-to-jsx-runtime@2.3.0: + resolution: {integrity: sha512-H/y0+IWPdsLLS738P8tDnrQ8Z+dj12zQQ6WC11TIM21C8WFVoIxcqWXf2H3hiTVZjF1AWqoimGwrTWecWrnmRQ==} + dependencies: + '@types/estree': 1.0.5 + '@types/hast': 3.0.3 + '@types/unist': 3.0.2 + comma-separated-tokens: 2.0.3 + devlop: 1.1.0 + estree-util-is-identifier-name: 3.0.0 + hast-util-whitespace: 3.0.0 + mdast-util-mdx-expression: 2.0.0 + mdast-util-mdx-jsx: 3.0.0 + mdast-util-mdxjs-esm: 2.0.1 + property-information: 6.4.0 + space-separated-tokens: 2.0.2 + style-to-object: 1.0.5 + unist-util-position: 5.0.0 + vfile-message: 4.0.2 + transitivePeerDependencies: + - supports-color + dev: false + + /hast-util-to-parse5@8.0.0: + resolution: {integrity: sha512-3KKrV5ZVI8if87DVSi1vDeByYrkGzg4mEfeu4alwgmmIeARiBLKCZS2uw5Gb6nU9x9Yufyj3iudm6i7nl52PFw==} + dependencies: + '@types/hast': 3.0.3 + comma-separated-tokens: 2.0.3 + devlop: 1.1.0 + property-information: 6.4.0 + space-separated-tokens: 2.0.2 + web-namespaces: 2.0.1 + zwitch: 2.0.4 + dev: false + + /hast-util-whitespace@2.0.1: + resolution: {integrity: sha512-nAxA0v8+vXSBDt3AnRUNjyRIQ0rD+ntpbAp4LnPkumc5M9yUbSMa4XDU9Q6etY4f1Wp4bNgvc1yjiZtsTTrSng==} + dev: false + + /hast-util-whitespace@3.0.0: + resolution: {integrity: sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==} + dependencies: + '@types/hast': 3.0.3 + dev: false + + /hastscript@8.0.0: + resolution: {integrity: sha512-dMOtzCEd3ABUeSIISmrETiKuyydk1w0pa+gE/uormcTpSYuaNJPbX1NU3JLyscSLjwAQM8bWMhhIlnCqnRvDTw==} + dependencies: + '@types/hast': 3.0.3 + comma-separated-tokens: 2.0.3 + hast-util-parse-selector: 4.0.0 + property-information: 6.4.0 + space-separated-tokens: 2.0.2 + dev: false + + /he@1.2.0: + resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==} + hasBin: true + dev: false + + /heap@0.2.7: + resolution: {integrity: sha512-2bsegYkkHO+h/9MGbn6KWcE45cHZgPANo5LXF7EvWdT0yT2EguSVO1nDgU5c8+ZOPwp2vMNa7YFsJhVcDR9Sdg==} + dev: false + + /history@4.10.1: + resolution: {integrity: sha512-36nwAD620w12kuzPAsyINPWJqlNbij+hpK1k9XRloDtym8mxzGYl2c17LnV6IAGB2Dmg4tEa7G7DlawS0+qjew==} + dependencies: + '@babel/runtime': 7.23.7 + loose-envify: 1.4.0 + resolve-pathname: 3.0.0 + tiny-invariant: 1.3.1 + tiny-warning: 1.0.3 + value-equal: 1.0.1 + dev: false + + /hoist-non-react-statics@3.3.2: + resolution: {integrity: sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==} + dependencies: + react-is: 16.13.1 + dev: false + + /hpack.js@2.1.6: + resolution: {integrity: sha512-zJxVehUdMGIKsRaNt7apO2Gqp0BdqW5yaiGHXXmbpvxgBYVZnAql+BJb4RO5ad2MgpbZKn5G6nMnegrH1FcNYQ==} + dependencies: + inherits: 2.0.4 + obuf: 1.1.2 + readable-stream: 2.3.8 + wbuf: 1.7.3 + dev: false + + /htm@3.1.1: + resolution: {integrity: sha512-983Vyg8NwUE7JkZ6NmOqpCZ+sh1bKv2iYTlUkzlWmA5JD2acKoxd4KVxbMmxX/85mtfdnDmTFoNKcg5DGAvxNQ==} + dev: false + + /html-entities@2.4.0: + resolution: {integrity: sha512-igBTJcNNNhvZFRtm8uA6xMY6xYleeDwn3PeBCkDz7tHttv4F2hsDI2aPgNERWzvRcNYHNT3ymRaQzllmXj4YsQ==} + dev: false + + /html-escaper@2.0.2: + resolution: {integrity: sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==} + dev: false + + /html-minifier-terser@6.1.0: + resolution: {integrity: sha512-YXxSlJBZTP7RS3tWnQw74ooKa6L9b9i9QYXY21eUEvhZ3u9XLfv6OnFsQq6RxkhHygsaUMvYsZRV5rU/OVNZxw==} + engines: {node: '>=12'} + hasBin: true + dependencies: + camel-case: 4.1.2 + clean-css: 5.3.3 + commander: 8.3.0 + he: 1.2.0 + param-case: 3.0.4 + relateurl: 0.2.7 + terser: 5.26.0 + dev: false + + /html-minifier-terser@7.2.0: + resolution: {integrity: sha512-tXgn3QfqPIpGl9o+K5tpcj3/MN4SfLtsx2GWwBC3SSd0tXQGyF3gsSqad8loJgKZGM3ZxbYDd5yhiBIdWpmvLA==} + engines: {node: ^14.13.1 || >=16.0.0} + hasBin: true + dependencies: + camel-case: 4.1.2 + clean-css: 5.3.3 + commander: 10.0.1 + entities: 4.5.0 + param-case: 3.0.4 + relateurl: 0.2.7 + terser: 5.26.0 + dev: false + + /html-tags@3.3.1: + resolution: {integrity: sha512-ztqyC3kLto0e9WbNp0aeP+M3kTt+nbaIveGmUxAtZa+8iFgKLUOD4YKM5j+f3QD89bra7UeumolZHKuOXnTmeQ==} + engines: {node: '>=8'} + dev: false + + /html-void-elements@3.0.0: + resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} + dev: false + + /html-webpack-plugin@5.6.0(webpack@5.89.0): + resolution: {integrity: sha512-iwaY4wzbe48AfKLZ/Cc8k0L+FKG6oSNRaZ8x5A/T/IVDGyXcbHncM9TdDa93wn0FsSm82FhTKW7f3vS61thXAw==} + engines: {node: '>=10.13.0'} + peerDependencies: + '@rspack/core': 0.x || 1.x + webpack: ^5.20.0 + peerDependenciesMeta: + '@rspack/core': + optional: true + webpack: + optional: true + dependencies: + '@types/html-minifier-terser': 6.1.0 + html-minifier-terser: 6.1.0 + lodash: 4.17.21 + pretty-error: 4.0.0 + tapable: 2.2.1 + webpack: 5.89.0 + dev: false + + /htmlparser2@6.1.0: + resolution: {integrity: sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A==} + dependencies: + domelementtype: 2.3.0 + domhandler: 4.3.1 + domutils: 2.8.0 + entities: 2.2.0 + dev: false + + /htmlparser2@8.0.2: + resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==} + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.1.0 + entities: 4.5.0 + dev: false + + /http-cache-semantics@4.1.1: + resolution: {integrity: sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==} + dev: false + + /http-deceiver@1.2.7: + resolution: {integrity: sha512-LmpOGxTfbpgtGVxJrj5k7asXHCgNZp5nLfp+hWc8QQRqtb7fUy6kRY3BO1h9ddF6yIPYUARgxGOwB42DnxIaNw==} + dev: false + + /http-errors@1.6.3: + resolution: {integrity: sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A==} + engines: {node: '>= 0.6'} + dependencies: + depd: 1.1.2 + inherits: 2.0.3 + setprototypeof: 1.1.0 + statuses: 1.5.0 + dev: false + + /http-errors@2.0.0: + resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==} + engines: {node: '>= 0.8'} + dependencies: + depd: 2.0.0 + inherits: 2.0.4 + setprototypeof: 1.2.0 + statuses: 2.0.1 + toidentifier: 1.0.1 + dev: false + + /http-parser-js@0.5.8: + resolution: {integrity: sha512-SGeBX54F94Wgu5RH3X5jsDtf4eHyRogWX1XGT3b4HuW3tQPM4AaBzoUji/4AAJNXCEOWZ5O0DgZmJw1947gD5Q==} + dev: false + + /http-proxy-middleware@2.0.6(@types/express@4.17.21): + resolution: {integrity: sha512-ya/UeJ6HVBYxrgYotAZo1KvPWlgB48kUJLDePFeneHsVujFaW5WNj2NgWCAE//B1Dl02BIfYlpNgBy8Kf8Rjmw==} + engines: {node: '>=12.0.0'} + peerDependencies: + '@types/express': ^4.17.13 + peerDependenciesMeta: + '@types/express': + optional: true + dependencies: + '@types/express': 4.17.21 + '@types/http-proxy': 1.17.14 + http-proxy: 1.18.1 + is-glob: 4.0.3 + is-plain-obj: 3.0.0 + micromatch: 4.0.5 + transitivePeerDependencies: + - debug + dev: false + + /http-proxy@1.18.1: + resolution: {integrity: sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==} + engines: {node: '>=8.0.0'} + dependencies: + eventemitter3: 4.0.7 + follow-redirects: 1.15.4 + requires-port: 1.0.0 + transitivePeerDependencies: + - debug + dev: false + + /http2-wrapper@2.2.1: + resolution: {integrity: sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==} + engines: {node: '>=10.19.0'} + dependencies: + quick-lru: 5.1.1 + resolve-alpn: 1.2.1 + dev: false + + /human-signals@2.1.0: + resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==} + engines: {node: '>=10.17.0'} + dev: false + + /iconv-lite@0.4.24: + resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} + engines: {node: '>=0.10.0'} + dependencies: + safer-buffer: 2.1.2 + dev: false + + /iconv-lite@0.6.3: + resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} + engines: {node: '>=0.10.0'} + dependencies: + safer-buffer: 2.1.2 + dev: false + + /icss-utils@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA==} + engines: {node: ^10 || ^12 || >= 14} + peerDependencies: + postcss: ^8.1.0 + dependencies: + postcss: 8.4.32 + dev: false + + /ignore@5.3.0: + resolution: {integrity: sha512-g7dmpshy+gD7mh88OC9NwSGTKoc3kyLAZQRU1mt53Aw/vnvfXnbC+F/7F7QoYVKbV+KNvJx8wArewKy1vXMtlg==} + engines: {node: '>= 4'} + dev: false + + /image-size@1.1.1: + resolution: {integrity: sha512-541xKlUw6jr/6gGuk92F+mYM5zaFAc5ahphvkqvNe2bQ6gVBkd6bfrmVJ2t4KDAfikAYZyIqTnktX3i6/aQDrQ==} + engines: {node: '>=16.x'} + hasBin: true + dependencies: + queue: 6.0.2 + dev: false + + /immer@9.0.21: + resolution: {integrity: sha512-bc4NBHqOqSfRW7POMkHd51LvClaeMXpm8dx0e8oE2GORbq5aRK7Bxl4FyzVLdGtLmvLKL7BTDBG5ACQm4HWjTA==} + dev: false + + /import-fresh@3.3.0: + resolution: {integrity: sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==} + engines: {node: '>=6'} + dependencies: + parent-module: 1.0.1 + resolve-from: 4.0.0 + dev: false + + /import-lazy@4.0.0: + resolution: {integrity: sha512-rKtvo6a868b5Hu3heneU+L4yEQ4jYKLtjpnPeUdK7h0yzXGmyBTypknlkCvHFBqfX9YlorEiMM6Dnq/5atfHkw==} + engines: {node: '>=8'} + dev: false + + /imurmurhash@0.1.4: + resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} + engines: {node: '>=0.8.19'} + dev: false + + /indent-string@4.0.0: + resolution: {integrity: sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==} + engines: {node: '>=8'} + dev: false + + /infima@0.2.0-alpha.43: + resolution: {integrity: sha512-2uw57LvUqW0rK/SWYnd/2rRfxNA5DDNOh33jxF7fy46VWoNhGxiUQyVZHbBMjQ33mQem0cjdDVwgWVAmlRfgyQ==} + engines: {node: '>=12'} + dev: false + + /inflight@1.0.6: + resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} + dependencies: + once: 1.4.0 + wrappy: 1.0.2 + dev: false + + /inherits@2.0.3: + resolution: {integrity: sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==} + dev: false + + /inherits@2.0.4: + resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + dev: false + + /ini@1.3.8: + resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} + dev: false + + /ini@2.0.0: + resolution: {integrity: sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA==} + engines: {node: '>=10'} + dev: false + + /inline-style-parser@0.1.1: + resolution: {integrity: sha512-7NXolsK4CAS5+xvdj5OMMbI962hU/wvwoxk+LWR9Ek9bVtyuuYScDN6eS0rUm6TxApFpw7CX1o4uJzcd4AyD3Q==} + dev: false + + /inline-style-parser@0.2.2: + resolution: {integrity: sha512-EcKzdTHVe8wFVOGEYXiW9WmJXPjqi1T+234YpJr98RiFYKHV3cdy1+3mkTE+KHTHxFFLH51SfaGOoUdW+v7ViQ==} + dev: false + + /internmap@1.0.1: + resolution: {integrity: sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw==} + dev: false + + /internmap@2.0.3: + resolution: {integrity: sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==} + engines: {node: '>=12'} + dev: false + + /interpret@1.4.0: + resolution: {integrity: sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==} + engines: {node: '>= 0.10'} + dev: false + + /invariant@2.2.4: + resolution: {integrity: sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==} + dependencies: + loose-envify: 1.4.0 + dev: false + + /ipaddr.js@1.9.1: + resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} + engines: {node: '>= 0.10'} + dev: false + + /ipaddr.js@2.1.0: + resolution: {integrity: sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ==} + engines: {node: '>= 10'} + dev: false + + /is-alphabetical@2.0.1: + resolution: {integrity: sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==} + dev: false + + /is-alphanumerical@2.0.1: + resolution: {integrity: sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==} + dependencies: + is-alphabetical: 2.0.1 + is-decimal: 2.0.1 + dev: false + + /is-arrayish@0.2.1: + resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} + dev: false + + /is-binary-path@2.1.0: + resolution: {integrity: sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==} + engines: {node: '>=8'} + dependencies: + binary-extensions: 2.2.0 + dev: false + + /is-buffer@2.0.5: + resolution: {integrity: sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==} + engines: {node: '>=4'} + dev: false + + /is-ci@3.0.1: + resolution: {integrity: sha512-ZYvCgrefwqoQ6yTyYUbQu64HsITZ3NfKX1lzaEYdkTDcfKzzCI/wthRRYKkdjHKFVgNiXKAKm65Zo1pk2as/QQ==} + hasBin: true + dependencies: + ci-info: 3.9.0 + dev: false + + /is-core-module@2.13.1: + resolution: {integrity: sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==} + dependencies: + hasown: 2.0.0 + dev: false + + /is-decimal@2.0.1: + resolution: {integrity: sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==} + dev: false + + /is-docker@2.2.1: + resolution: {integrity: sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==} + engines: {node: '>=8'} + hasBin: true + dev: false + + /is-extendable@0.1.1: + resolution: {integrity: sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==} + engines: {node: '>=0.10.0'} + dev: false + + /is-extglob@2.1.1: + resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} + engines: {node: '>=0.10.0'} + dev: false + + /is-fullwidth-code-point@3.0.0: + resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} + engines: {node: '>=8'} + dev: false + + /is-glob@4.0.3: + resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} + engines: {node: '>=0.10.0'} + dependencies: + is-extglob: 2.1.1 + dev: false + + /is-hexadecimal@2.0.1: + resolution: {integrity: sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==} + dev: false + + /is-installed-globally@0.4.0: + resolution: {integrity: sha512-iwGqO3J21aaSkC7jWnHP/difazwS7SFeIqxv6wEtLU8Y5KlzFTjyqcSIT0d8s4+dDhKytsk9PJZ2BkS5eZwQRQ==} + engines: {node: '>=10'} + dependencies: + global-dirs: 3.0.1 + is-path-inside: 3.0.3 + dev: false + + /is-npm@6.0.0: + resolution: {integrity: sha512-JEjxbSmtPSt1c8XTkVrlujcXdKV1/tvuQ7GwKcAlyiVLeYFQ2VHat8xfrDJsIkhCdF/tZ7CiIR3sy141c6+gPQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dev: false + + /is-number@7.0.0: + resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} + engines: {node: '>=0.12.0'} + dev: false + + /is-obj@1.0.1: + resolution: {integrity: sha512-l4RyHgRqGN4Y3+9JHVrNqO+tN0rV5My76uW5/nuO4K1b6vw5G8d/cmFjP9tRfEsdhZNt0IFdZuK/c2Vr4Nb+Qg==} + engines: {node: '>=0.10.0'} + dev: false + + /is-obj@2.0.0: + resolution: {integrity: sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==} + engines: {node: '>=8'} + dev: false + + /is-path-cwd@2.2.0: + resolution: {integrity: sha512-w942bTcih8fdJPJmQHFzkS76NEP8Kzzvmw92cXsazb8intwLqPibPPdXf4ANdKV3rYMuuQYGIWtvz9JilB3NFQ==} + engines: {node: '>=6'} + dev: false + + /is-path-inside@3.0.3: + resolution: {integrity: sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==} + engines: {node: '>=8'} + dev: false + + /is-plain-obj@3.0.0: + resolution: {integrity: sha512-gwsOE28k+23GP1B6vFl1oVh/WOzmawBrKwo5Ev6wMKzPkaXaCDIQKzLnvsA42DRlbVTWorkgTKIviAKCWkfUwA==} + engines: {node: '>=10'} + dev: false + + /is-plain-obj@4.1.0: + resolution: {integrity: sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==} + engines: {node: '>=12'} + dev: false + + /is-plain-object@2.0.4: + resolution: {integrity: sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==} + engines: {node: '>=0.10.0'} + dependencies: + isobject: 3.0.1 + dev: false + + /is-plain-object@5.0.0: + resolution: {integrity: sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==} + engines: {node: '>=0.10.0'} + dev: false + + /is-reference@3.0.2: + resolution: {integrity: sha512-v3rht/LgVcsdZa3O2Nqs+NMowLOxeOm7Ay9+/ARQ2F+qEoANRcqrjAZKGN0v8ymUetZGgkp26LTnGT7H0Qo9Pg==} + dependencies: + '@types/estree': 1.0.5 + dev: false + + /is-regexp@1.0.0: + resolution: {integrity: sha512-7zjFAPO4/gwyQAAgRRmqeEeyIICSdmCqa3tsVHMdBzaXXRiqopZL4Cyghg/XulGWrtABTpbnYYzzIRffLkP4oA==} + engines: {node: '>=0.10.0'} + dev: false + + /is-root@2.1.0: + resolution: {integrity: sha512-AGOriNp96vNBd3HtU+RzFEc75FfR5ymiYv8E553I71SCeXBiMsVDUtdio1OEFvrPyLIQ9tVR5RxXIFe5PUFjMg==} + engines: {node: '>=6'} + dev: false + + /is-stream@2.0.1: + resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} + engines: {node: '>=8'} + dev: false + + /is-typedarray@1.0.0: + resolution: {integrity: sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==} + dev: false + + /is-wsl@2.2.0: + resolution: {integrity: sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==} + engines: {node: '>=8'} + dependencies: + is-docker: 2.2.1 + dev: false + + /is-yarn-global@0.4.1: + resolution: {integrity: sha512-/kppl+R+LO5VmhYSEWARUFjodS25D68gvj8W7z0I7OWhUla5xWu8KL6CtB2V0R6yqhnRgbcaREMr4EEM6htLPQ==} + engines: {node: '>=12'} + dev: false + + /isarray@0.0.1: + resolution: {integrity: sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==} + dev: false + + /isarray@1.0.0: + resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} + dev: false + + /isexe@2.0.0: + resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + dev: false + + /isobject@3.0.1: + resolution: {integrity: sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==} + engines: {node: '>=0.10.0'} + dev: false + + /javascript-stringify@2.1.0: + resolution: {integrity: sha512-JVAfqNPTvNq3sB/VHQJAFxN/sPgKnsKrCwyRt15zwNCdrMMJDdcEOdubuy+DuJYYdm0ox1J4uzEuYKkN+9yhVg==} + dev: false + + /jest-util@29.7.0: + resolution: {integrity: sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@jest/types': 29.6.3 + '@types/node': 20.10.6 + chalk: 4.1.2 + ci-info: 3.9.0 + graceful-fs: 4.2.11 + picomatch: 2.3.1 + dev: false + + /jest-worker@27.5.1: + resolution: {integrity: sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==} + engines: {node: '>= 10.13.0'} + dependencies: + '@types/node': 20.10.6 + merge-stream: 2.0.0 + supports-color: 8.1.1 + dev: false + + /jest-worker@29.7.0: + resolution: {integrity: sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + dependencies: + '@types/node': 20.10.6 + jest-util: 29.7.0 + merge-stream: 2.0.0 + supports-color: 8.1.1 + dev: false + + /jiti@1.21.0: + resolution: {integrity: sha512-gFqAIbuKyyso/3G2qhiO2OM6shY6EPP/R0+mkDbyspxKazh8BXDC5FiFsUjlczgdNz/vfra0da2y+aHrusLG/Q==} + hasBin: true + dev: false + + /joi@17.11.0: + resolution: {integrity: sha512-NgB+lZLNoqISVy1rZocE9PZI36bL/77ie924Ri43yEvi9GUUMPeyVIr8KdFTMUlby1p0PBYMk9spIxEUQYqrJQ==} + dependencies: + '@hapi/hoek': 9.3.0 + '@hapi/topo': 5.1.0 + '@sideway/address': 4.1.4 + '@sideway/formula': 3.0.1 + '@sideway/pinpoint': 2.0.0 + dev: false + + /js-tokens@4.0.0: + resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + dev: false + + /js-yaml@3.14.1: + resolution: {integrity: sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==} + hasBin: true + dependencies: + argparse: 1.0.10 + esprima: 4.0.1 + dev: false + + /js-yaml@4.1.0: + resolution: {integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==} + hasBin: true + dependencies: + argparse: 2.0.1 + dev: false + + /jsesc@0.5.0: + resolution: {integrity: sha512-uZz5UnB7u4T9LvwmFqXii7pZSouaRPorGs5who1Ip7VO0wxanFvBL7GkM6dTHlgX+jhBApRetaWpnDabOeTcnA==} + hasBin: true + dev: false + + /jsesc@2.5.2: + resolution: {integrity: sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==} + engines: {node: '>=4'} + hasBin: true + dev: false + + /json-buffer@3.0.1: + resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} + dev: false + + /json-parse-even-better-errors@2.3.1: + resolution: {integrity: sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==} + dev: false + + /json-schema-traverse@0.4.1: + resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} + dev: false + + /json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + dev: false + + /json5@2.2.3: + resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} + engines: {node: '>=6'} + hasBin: true + dev: false + + /jsonfile@6.1.0: + resolution: {integrity: sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==} + dependencies: + universalify: 2.0.1 + optionalDependencies: + graceful-fs: 4.2.11 + dev: false + + /keyv@4.5.4: + resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} + dependencies: + json-buffer: 3.0.1 + dev: false + + /khroma@2.1.0: + resolution: {integrity: sha512-Ls993zuzfayK269Svk9hzpeGUKob/sIgZzyHYdjQoAdQetRKpOLj+k/QQQ/6Qi0Yz65mlROrfd+Ev+1+7dz9Kw==} + dev: false + + /kind-of@6.0.3: + resolution: {integrity: sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==} + engines: {node: '>=0.10.0'} + dev: false + + /kleur@3.0.3: + resolution: {integrity: sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==} + engines: {node: '>=6'} + dev: false + + /kleur@4.1.5: + resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} + engines: {node: '>=6'} + dev: false + + /latest-version@7.0.0: + resolution: {integrity: sha512-KvNT4XqAMzdcL6ka6Tl3i2lYeFDgXNCuIX+xNx6ZMVR1dFq+idXd9FLKNMOIx0t9mJ9/HudyX4oZWXZQ0UJHeg==} + engines: {node: '>=14.16'} + dependencies: + package-json: 8.1.1 + dev: false + + /launch-editor@2.6.1: + resolution: {integrity: sha512-eB/uXmFVpY4zezmGp5XtU21kwo7GBbKB+EQ+UZeWtGb9yAM5xt/Evk+lYH3eRNAtId+ej4u7TYPFZ07w4s7rRw==} + dependencies: + picocolors: 1.0.0 + shell-quote: 1.8.1 + dev: false + + /layout-base@1.0.2: + resolution: {integrity: sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg==} + dev: false + + /layout-base@2.0.1: + resolution: {integrity: sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg==} + dev: false + + /leven@3.1.0: + resolution: {integrity: sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==} + engines: {node: '>=6'} + dev: false + + /lilconfig@2.1.0: + resolution: {integrity: sha512-utWOt/GHzuUxnLKxB6dk81RoOeoNeHgbrXiuGk4yyF5qlRz+iIVWu56E2fqGHFrXz0QNUhLB/8nKqvRH66JKGQ==} + engines: {node: '>=10'} + dev: false + + /lilconfig@3.0.0: + resolution: {integrity: sha512-K2U4W2Ff5ibV7j7ydLr+zLAkIg5JJ4lPn1Ltsdt+Tz/IjQ8buJ55pZAxoP34lqIiwtF9iAvtLv3JGv7CAyAg+g==} + engines: {node: '>=14'} + dev: false + + /lines-and-columns@1.2.4: + resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} + dev: false + + /loader-runner@4.3.0: + resolution: {integrity: sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==} + engines: {node: '>=6.11.5'} + dev: false + + /loader-utils@2.0.4: + resolution: {integrity: sha512-xXqpXoINfFhgua9xiqD8fPFHgkoq1mmmpE92WlDbm9rNRd/EbRb+Gqf908T2DMfuHjjJlksiK2RbHVOdD/MqSw==} + engines: {node: '>=8.9.0'} + dependencies: + big.js: 5.2.2 + emojis-list: 3.0.0 + json5: 2.2.3 + dev: false + + /loader-utils@3.2.1: + resolution: {integrity: sha512-ZvFw1KWS3GVyYBYb7qkmRM/WwL2TQQBxgCK62rlvm4WpVQ23Nb4tYjApUlfjrEGvOs7KHEsmyUn75OHZrJMWPw==} + engines: {node: '>= 12.13.0'} + dev: false + + /locate-path@3.0.0: + resolution: {integrity: sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==} + engines: {node: '>=6'} + dependencies: + p-locate: 3.0.0 + path-exists: 3.0.0 + dev: false + + /locate-path@6.0.0: + resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} + engines: {node: '>=10'} + dependencies: + p-locate: 5.0.0 + dev: false + + /locate-path@7.2.0: + resolution: {integrity: sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + p-locate: 6.0.0 + dev: false + + /lodash-es@4.17.21: + resolution: {integrity: sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==} + dev: false + + /lodash.debounce@4.0.8: + resolution: {integrity: sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==} + dev: false + + /lodash.memoize@4.1.2: + resolution: {integrity: sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==} + dev: false + + /lodash.uniq@4.5.0: + resolution: {integrity: sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ==} + dev: false + + /lodash@4.17.21: + resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==} + dev: false + + /longest-streak@3.1.0: + resolution: {integrity: sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==} + dev: false + + /loose-envify@1.4.0: + resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} + hasBin: true + dependencies: + js-tokens: 4.0.0 + dev: false + + /lower-case@2.0.2: + resolution: {integrity: sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg==} + dependencies: + tslib: 2.6.2 + dev: false + + /lowercase-keys@3.0.0: + resolution: {integrity: sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dev: false + + /lru-cache@5.1.1: + resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} + dependencies: + yallist: 3.1.1 + dev: false + + /lru-cache@6.0.0: + resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==} + engines: {node: '>=10'} + dependencies: + yallist: 4.0.0 + dev: false + + /lunr-languages@1.14.0: + resolution: {integrity: sha512-hWUAb2KqM3L7J5bcrngszzISY4BxrXn/Xhbb9TTCJYEGqlR1nG67/M14sp09+PTIRklobrn57IAxcdcO/ZFyNA==} + dev: false + + /mark.js@8.11.1: + resolution: {integrity: sha512-1I+1qpDt4idfgLQG+BNWmrqku+7/2bi5nLf4YwF8y8zXvmfiTBY3PV3ZibfrjBueCByROpuBjLLFCajqkgYoLQ==} + dev: false + + /markdown-extensions@2.0.0: + resolution: {integrity: sha512-o5vL7aDWatOTX8LzaS1WMoaoxIiLRQJuIKKe2wAw6IeULDHaqbiqiggmx+pKvZDb1Sj+pE46Sn1T7lCqfFtg1Q==} + engines: {node: '>=16'} + dev: false + + /markdown-table@3.0.3: + resolution: {integrity: sha512-Z1NL3Tb1M9wH4XESsCDEksWoKTdlUafKc4pt0GRwjUyXaCFZ+dc3g2erqB6zm3szA2IUSi7VnPI+o/9jnxh9hw==} + dev: false + + /mdast-util-definitions@5.1.2: + resolution: {integrity: sha512-8SVPMuHqlPME/z3gqVwWY4zVXn8lqKv/pAhC57FuJ40ImXyBpmO5ukh98zB2v7Blql2FiHjHv9LVztSIqjY+MA==} + dependencies: + '@types/mdast': 3.0.15 + '@types/unist': 2.0.10 + unist-util-visit: 4.1.2 + dev: false + + /mdast-util-directive@3.0.0: + resolution: {integrity: sha512-JUpYOqKI4mM3sZcNxmF/ox04XYFFkNwr0CFlrQIkCwbvH0xzMCqkMqAde9wRd80VAhaUrwFwKm2nxretdT1h7Q==} + dependencies: + '@types/mdast': 4.0.3 + '@types/unist': 3.0.2 + devlop: 1.1.0 + mdast-util-from-markdown: 2.0.0 + mdast-util-to-markdown: 2.1.0 + parse-entities: 4.0.1 + stringify-entities: 4.0.3 + unist-util-visit-parents: 6.0.1 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-find-and-replace@3.0.1: + resolution: {integrity: sha512-SG21kZHGC3XRTSUhtofZkBzZTJNM5ecCi0SK2IMKmSXR8vO3peL+kb1O0z7Zl83jKtutG4k5Wv/W7V3/YHvzPA==} + dependencies: + '@types/mdast': 4.0.3 + escape-string-regexp: 5.0.0 + unist-util-is: 6.0.0 + unist-util-visit-parents: 6.0.1 + dev: false + + /mdast-util-from-markdown@1.3.1: + resolution: {integrity: sha512-4xTO/M8c82qBcnQc1tgpNtubGUW/Y1tBQ1B0i5CtSoelOLKFYlElIr3bvgREYYO5iRqbMY1YuqZng0GVOI8Qww==} + dependencies: + '@types/mdast': 3.0.15 + '@types/unist': 2.0.10 + decode-named-character-reference: 1.0.2 + mdast-util-to-string: 3.2.0 + micromark: 3.2.0 + micromark-util-decode-numeric-character-reference: 1.1.0 + micromark-util-decode-string: 1.1.0 + micromark-util-normalize-identifier: 1.1.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 + unist-util-stringify-position: 3.0.3 + uvu: 0.5.6 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-from-markdown@2.0.0: + resolution: {integrity: sha512-n7MTOr/z+8NAX/wmhhDji8O3bRvPTV/U0oTCaZJkjhPSKTPhS3xufVhKGF8s1pJ7Ox4QgoIU7KHseh09S+9rTA==} + dependencies: + '@types/mdast': 4.0.3 + '@types/unist': 3.0.2 + decode-named-character-reference: 1.0.2 + devlop: 1.1.0 + mdast-util-to-string: 4.0.0 + micromark: 4.0.0 + micromark-util-decode-numeric-character-reference: 2.0.1 + micromark-util-decode-string: 2.0.0 + micromark-util-normalize-identifier: 2.0.0 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + unist-util-stringify-position: 4.0.0 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-frontmatter@2.0.1: + resolution: {integrity: sha512-LRqI9+wdgC25P0URIJY9vwocIzCcksduHQ9OF2joxQoyTNVduwLAFUzjoopuRJbJAReaKrNQKAZKL3uCMugWJA==} + dependencies: + '@types/mdast': 4.0.3 + devlop: 1.1.0 + escape-string-regexp: 5.0.0 + mdast-util-from-markdown: 2.0.0 + mdast-util-to-markdown: 2.1.0 + micromark-extension-frontmatter: 2.0.0 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-gfm-autolink-literal@2.0.0: + resolution: {integrity: sha512-FyzMsduZZHSc3i0Px3PQcBT4WJY/X/RCtEJKuybiC6sjPqLv7h1yqAkmILZtuxMSsUyaLUWNp71+vQH2zqp5cg==} + dependencies: + '@types/mdast': 4.0.3 + ccount: 2.0.1 + devlop: 1.1.0 + mdast-util-find-and-replace: 3.0.1 + micromark-util-character: 2.0.1 + dev: false + + /mdast-util-gfm-footnote@2.0.0: + resolution: {integrity: sha512-5jOT2boTSVkMnQ7LTrd6n/18kqwjmuYqo7JUPe+tRCY6O7dAuTFMtTPauYYrMPpox9hlN0uOx/FL8XvEfG9/mQ==} + dependencies: + '@types/mdast': 4.0.3 + devlop: 1.1.0 + mdast-util-from-markdown: 2.0.0 + mdast-util-to-markdown: 2.1.0 + micromark-util-normalize-identifier: 2.0.0 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-gfm-strikethrough@2.0.0: + resolution: {integrity: sha512-mKKb915TF+OC5ptj5bJ7WFRPdYtuHv0yTRxK2tJvi+BDqbkiG7h7u/9SI89nRAYcmap2xHQL9D+QG/6wSrTtXg==} + dependencies: + '@types/mdast': 4.0.3 + mdast-util-from-markdown: 2.0.0 + mdast-util-to-markdown: 2.1.0 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-gfm-table@2.0.0: + resolution: {integrity: sha512-78UEvebzz/rJIxLvE7ZtDd/vIQ0RHv+3Mh5DR96p7cS7HsBhYIICDBCu8csTNWNO6tBWfqXPWekRuj2FNOGOZg==} + dependencies: + '@types/mdast': 4.0.3 + devlop: 1.1.0 + markdown-table: 3.0.3 + mdast-util-from-markdown: 2.0.0 + mdast-util-to-markdown: 2.1.0 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-gfm-task-list-item@2.0.0: + resolution: {integrity: sha512-IrtvNvjxC1o06taBAVJznEnkiHxLFTzgonUdy8hzFVeDun0uTjxxrRGVaNFqkU1wJR3RBPEfsxmU6jDWPofrTQ==} + dependencies: + '@types/mdast': 4.0.3 + devlop: 1.1.0 + mdast-util-from-markdown: 2.0.0 + mdast-util-to-markdown: 2.1.0 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-gfm@3.0.0: + resolution: {integrity: sha512-dgQEX5Amaq+DuUqf26jJqSK9qgixgd6rYDHAv4aTBuA92cTknZlKpPfa86Z/s8Dj8xsAQpFfBmPUHWJBWqS4Bw==} + dependencies: + mdast-util-from-markdown: 2.0.0 + mdast-util-gfm-autolink-literal: 2.0.0 + mdast-util-gfm-footnote: 2.0.0 + mdast-util-gfm-strikethrough: 2.0.0 + mdast-util-gfm-table: 2.0.0 + mdast-util-gfm-task-list-item: 2.0.0 + mdast-util-to-markdown: 2.1.0 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-mdx-expression@2.0.0: + resolution: {integrity: sha512-fGCu8eWdKUKNu5mohVGkhBXCXGnOTLuFqOvGMvdikr+J1w7lDJgxThOKpwRWzzbyXAU2hhSwsmssOY4yTokluw==} + dependencies: + '@types/estree-jsx': 1.0.3 + '@types/hast': 3.0.3 + '@types/mdast': 4.0.3 + devlop: 1.1.0 + mdast-util-from-markdown: 2.0.0 + mdast-util-to-markdown: 2.1.0 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-mdx-jsx@3.0.0: + resolution: {integrity: sha512-XZuPPzQNBPAlaqsTTgRrcJnyFbSOBovSadFgbFu8SnuNgm+6Bdx1K+IWoitsmj6Lq6MNtI+ytOqwN70n//NaBA==} + dependencies: + '@types/estree-jsx': 1.0.3 + '@types/hast': 3.0.3 + '@types/mdast': 4.0.3 + '@types/unist': 3.0.2 + ccount: 2.0.1 + devlop: 1.1.0 + mdast-util-from-markdown: 2.0.0 + mdast-util-to-markdown: 2.1.0 + parse-entities: 4.0.1 + stringify-entities: 4.0.3 + unist-util-remove-position: 5.0.0 + unist-util-stringify-position: 4.0.0 + vfile-message: 4.0.2 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-mdx@3.0.0: + resolution: {integrity: sha512-JfbYLAW7XnYTTbUsmpu0kdBUVe+yKVJZBItEjwyYJiDJuZ9w4eeaqks4HQO+R7objWgS2ymV60GYpI14Ug554w==} + dependencies: + mdast-util-from-markdown: 2.0.0 + mdast-util-mdx-expression: 2.0.0 + mdast-util-mdx-jsx: 3.0.0 + mdast-util-mdxjs-esm: 2.0.1 + mdast-util-to-markdown: 2.1.0 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-mdxjs-esm@2.0.1: + resolution: {integrity: sha512-EcmOpxsZ96CvlP03NghtH1EsLtr0n9Tm4lPUJUBccV9RwUOneqSycg19n5HGzCf+10LozMRSObtVr3ee1WoHtg==} + dependencies: + '@types/estree-jsx': 1.0.3 + '@types/hast': 3.0.3 + '@types/mdast': 4.0.3 + devlop: 1.1.0 + mdast-util-from-markdown: 2.0.0 + mdast-util-to-markdown: 2.1.0 + transitivePeerDependencies: + - supports-color + dev: false + + /mdast-util-phrasing@4.0.0: + resolution: {integrity: sha512-xadSsJayQIucJ9n053dfQwVu1kuXg7jCTdYsMK8rqzKZh52nLfSH/k0sAxE0u+pj/zKZX+o5wB+ML5mRayOxFA==} + dependencies: + '@types/mdast': 4.0.3 + unist-util-is: 6.0.0 + dev: false + + /mdast-util-to-hast@12.3.0: + resolution: {integrity: sha512-pits93r8PhnIoU4Vy9bjW39M2jJ6/tdHyja9rrot9uujkN7UTU9SDnE6WNJz/IGyQk3XHX6yNNtrBH6cQzm8Hw==} + dependencies: + '@types/hast': 2.3.9 + '@types/mdast': 3.0.15 + mdast-util-definitions: 5.1.2 + micromark-util-sanitize-uri: 1.2.0 + trim-lines: 3.0.1 + unist-util-generated: 2.0.1 + unist-util-position: 4.0.4 + unist-util-visit: 4.1.2 + dev: false + + /mdast-util-to-hast@13.0.2: + resolution: {integrity: sha512-U5I+500EOOw9e3ZrclN3Is3fRpw8c19SMyNZlZ2IS+7vLsNzb2Om11VpIVOR+/0137GhZsFEF6YiKD5+0Hr2Og==} + dependencies: + '@types/hast': 3.0.3 + '@types/mdast': 4.0.3 + '@ungap/structured-clone': 1.2.0 + devlop: 1.1.0 + micromark-util-sanitize-uri: 2.0.0 + trim-lines: 3.0.1 + unist-util-position: 5.0.0 + unist-util-visit: 5.0.0 + dev: false + + /mdast-util-to-markdown@2.1.0: + resolution: {integrity: sha512-SR2VnIEdVNCJbP6y7kVTJgPLifdr8WEU440fQec7qHoHOUz/oJ2jmNRqdDQ3rbiStOXb2mCDGTuwsK5OPUgYlQ==} + dependencies: + '@types/mdast': 4.0.3 + '@types/unist': 3.0.2 + longest-streak: 3.1.0 + mdast-util-phrasing: 4.0.0 + mdast-util-to-string: 4.0.0 + micromark-util-decode-string: 2.0.0 + unist-util-visit: 5.0.0 + zwitch: 2.0.4 + dev: false + + /mdast-util-to-string@3.2.0: + resolution: {integrity: sha512-V4Zn/ncyN1QNSqSBxTrMOLpjr+IKdHl2v3KVLoWmDPscP4r9GcCi71gjgvUV1SFSKh92AjAG4peFuBl2/YgCJg==} + dependencies: + '@types/mdast': 3.0.15 + dev: false + + /mdast-util-to-string@4.0.0: + resolution: {integrity: sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==} + dependencies: + '@types/mdast': 4.0.3 + dev: false + + /mdn-data@2.0.14: + resolution: {integrity: sha512-dn6wd0uw5GsdswPFfsgMp5NSB0/aDe6fK94YJV/AJDYXL6HVLWBsxeq7js7Ad+mU2K9LAlwpk6kN2D5mwCPVow==} + dev: false + + /mdn-data@2.0.28: + resolution: {integrity: sha512-aylIc7Z9y4yzHYAJNuESG3hfhC+0Ibp/MAMiaOZgNv4pmEdFyfZhhhny4MNiAfWdBQ1RQ2mfDWmM1x8SvGyp8g==} + dev: false + + /mdn-data@2.0.30: + resolution: {integrity: sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA==} + dev: false + + /media-typer@0.3.0: + resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} + engines: {node: '>= 0.6'} + dev: false + + /memfs@3.5.3: + resolution: {integrity: sha512-UERzLsxzllchadvbPs5aolHh65ISpKpM+ccLbOJ8/vvpBKmAWf+la7dXFy7Mr0ySHbdHrFv5kGFCUHHe6GFEmw==} + engines: {node: '>= 4.0.0'} + dependencies: + fs-monkey: 1.0.5 + dev: false + + /merge-descriptors@1.0.1: + resolution: {integrity: sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==} + dev: false + + /merge-stream@2.0.0: + resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} + dev: false + + /merge2@1.4.1: + resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} + engines: {node: '>= 8'} + dev: false + + /mermaid@10.6.1: + resolution: {integrity: sha512-Hky0/RpOw/1il9X8AvzOEChfJtVvmXm+y7JML5C//ePYMy0/9jCEmW1E1g86x9oDfW9+iVEdTV/i+M6KWRNs4A==} + dependencies: + '@braintree/sanitize-url': 6.0.4 + '@types/d3-scale': 4.0.8 + '@types/d3-scale-chromatic': 3.0.3 + cytoscape: 3.28.1 + cytoscape-cose-bilkent: 4.1.0(cytoscape@3.28.1) + cytoscape-fcose: 2.2.0(cytoscape@3.28.1) + d3: 7.8.5 + d3-sankey: 0.12.3 + dagre-d3-es: 7.0.10 + dayjs: 1.11.10 + dompurify: 3.0.6 + elkjs: 0.8.2 + khroma: 2.1.0 + lodash-es: 4.17.21 + mdast-util-from-markdown: 1.3.1 + non-layered-tidy-tree-layout: 2.0.2 + stylis: 4.3.1 + ts-dedent: 2.2.0 + uuid: 9.0.1 + web-worker: 1.2.0 + transitivePeerDependencies: + - supports-color + dev: false + + /methods@1.1.2: + resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==} + engines: {node: '>= 0.6'} + dev: false + + /micromark-core-commonmark@1.1.0: + resolution: {integrity: sha512-BgHO1aRbolh2hcrzL2d1La37V0Aoz73ymF8rAcKnohLy93titmv62E0gP8Hrx9PKcKrqCZ1BbLGbP3bEhoXYlw==} + dependencies: + decode-named-character-reference: 1.0.2 + micromark-factory-destination: 1.1.0 + micromark-factory-label: 1.1.0 + micromark-factory-space: 1.1.0 + micromark-factory-title: 1.1.0 + micromark-factory-whitespace: 1.1.0 + micromark-util-character: 1.2.0 + micromark-util-chunked: 1.1.0 + micromark-util-classify-character: 1.1.0 + micromark-util-html-tag-name: 1.2.0 + micromark-util-normalize-identifier: 1.1.0 + micromark-util-resolve-all: 1.1.0 + micromark-util-subtokenize: 1.1.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 + uvu: 0.5.6 + dev: false + + /micromark-core-commonmark@2.0.0: + resolution: {integrity: sha512-jThOz/pVmAYUtkroV3D5c1osFXAMv9e0ypGDOIZuCeAe91/sD6BoE2Sjzt30yuXtwOYUmySOhMas/PVyh02itA==} + dependencies: + decode-named-character-reference: 1.0.2 + devlop: 1.1.0 + micromark-factory-destination: 2.0.0 + micromark-factory-label: 2.0.0 + micromark-factory-space: 2.0.0 + micromark-factory-title: 2.0.0 + micromark-factory-whitespace: 2.0.0 + micromark-util-character: 2.0.1 + micromark-util-chunked: 2.0.0 + micromark-util-classify-character: 2.0.0 + micromark-util-html-tag-name: 2.0.0 + micromark-util-normalize-identifier: 2.0.0 + micromark-util-resolve-all: 2.0.0 + micromark-util-subtokenize: 2.0.0 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-extension-directive@3.0.0: + resolution: {integrity: sha512-61OI07qpQrERc+0wEysLHMvoiO3s2R56x5u7glHq2Yqq6EHbH4dW25G9GfDdGCDYqA21KE6DWgNSzxSwHc2hSg==} + dependencies: + devlop: 1.1.0 + micromark-factory-space: 2.0.0 + micromark-factory-whitespace: 2.0.0 + micromark-util-character: 2.0.1 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + parse-entities: 4.0.1 + dev: false + + /micromark-extension-frontmatter@2.0.0: + resolution: {integrity: sha512-C4AkuM3dA58cgZha7zVnuVxBhDsbttIMiytjgsM2XbHAB2faRVaHRle40558FBN+DJcrLNCoqG5mlrpdU4cRtg==} + dependencies: + fault: 2.0.1 + micromark-util-character: 2.0.1 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-extension-gfm-autolink-literal@2.0.0: + resolution: {integrity: sha512-rTHfnpt/Q7dEAK1Y5ii0W8bhfJlVJFnJMHIPisfPK3gpVNuOP0VnRl96+YJ3RYWV/P4gFeQoGKNlT3RhuvpqAg==} + dependencies: + micromark-util-character: 2.0.1 + micromark-util-sanitize-uri: 2.0.0 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-extension-gfm-footnote@2.0.0: + resolution: {integrity: sha512-6Rzu0CYRKDv3BfLAUnZsSlzx3ak6HAoI85KTiijuKIz5UxZxbUI+pD6oHgw+6UtQuiRwnGRhzMmPRv4smcz0fg==} + dependencies: + devlop: 1.1.0 + micromark-core-commonmark: 2.0.0 + micromark-factory-space: 2.0.0 + micromark-util-character: 2.0.1 + micromark-util-normalize-identifier: 2.0.0 + micromark-util-sanitize-uri: 2.0.0 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-extension-gfm-strikethrough@2.0.0: + resolution: {integrity: sha512-c3BR1ClMp5fxxmwP6AoOY2fXO9U8uFMKs4ADD66ahLTNcwzSCyRVU4k7LPV5Nxo/VJiR4TdzxRQY2v3qIUceCw==} + dependencies: + devlop: 1.1.0 + micromark-util-chunked: 2.0.0 + micromark-util-classify-character: 2.0.0 + micromark-util-resolve-all: 2.0.0 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-extension-gfm-table@2.0.0: + resolution: {integrity: sha512-PoHlhypg1ItIucOaHmKE8fbin3vTLpDOUg8KAr8gRCF1MOZI9Nquq2i/44wFvviM4WuxJzc3demT8Y3dkfvYrw==} + dependencies: + devlop: 1.1.0 + micromark-factory-space: 2.0.0 + micromark-util-character: 2.0.1 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-extension-gfm-tagfilter@2.0.0: + resolution: {integrity: sha512-xHlTOmuCSotIA8TW1mDIM6X2O1SiX5P9IuDtqGonFhEK0qgRI4yeC6vMxEV2dgyr2TiD+2PQ10o+cOhdVAcwfg==} + dependencies: + micromark-util-types: 2.0.0 + dev: false + + /micromark-extension-gfm-task-list-item@2.0.1: + resolution: {integrity: sha512-cY5PzGcnULaN5O7T+cOzfMoHjBW7j+T9D2sucA5d/KbsBTPcYdebm9zUd9zzdgJGCwahV+/W78Z3nbulBYVbTw==} + dependencies: + devlop: 1.1.0 + micromark-factory-space: 2.0.0 + micromark-util-character: 2.0.1 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-extension-gfm@3.0.0: + resolution: {integrity: sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w==} + dependencies: + micromark-extension-gfm-autolink-literal: 2.0.0 + micromark-extension-gfm-footnote: 2.0.0 + micromark-extension-gfm-strikethrough: 2.0.0 + micromark-extension-gfm-table: 2.0.0 + micromark-extension-gfm-tagfilter: 2.0.0 + micromark-extension-gfm-task-list-item: 2.0.1 + micromark-util-combine-extensions: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-extension-mdx-expression@3.0.0: + resolution: {integrity: sha512-sI0nwhUDz97xyzqJAbHQhp5TfaxEvZZZ2JDqUo+7NvyIYG6BZ5CPPqj2ogUoPJlmXHBnyZUzISg9+oUmU6tUjQ==} + dependencies: + '@types/estree': 1.0.5 + devlop: 1.1.0 + micromark-factory-mdx-expression: 2.0.1 + micromark-factory-space: 2.0.0 + micromark-util-character: 2.0.1 + micromark-util-events-to-acorn: 2.0.2 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-extension-mdx-jsx@3.0.0: + resolution: {integrity: sha512-uvhhss8OGuzR4/N17L1JwvmJIpPhAd8oByMawEKx6NVdBCbesjH4t+vjEp3ZXft9DwvlKSD07fCeI44/N0Vf2w==} + dependencies: + '@types/acorn': 4.0.6 + '@types/estree': 1.0.5 + devlop: 1.1.0 + estree-util-is-identifier-name: 3.0.0 + micromark-factory-mdx-expression: 2.0.1 + micromark-factory-space: 2.0.0 + micromark-util-character: 2.0.1 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + vfile-message: 4.0.2 + dev: false + + /micromark-extension-mdx-md@2.0.0: + resolution: {integrity: sha512-EpAiszsB3blw4Rpba7xTOUptcFeBFi+6PY8VnJ2hhimH+vCQDirWgsMpz7w1XcZE7LVrSAUGb9VJpG9ghlYvYQ==} + dependencies: + micromark-util-types: 2.0.0 + dev: false + + /micromark-extension-mdxjs-esm@3.0.0: + resolution: {integrity: sha512-DJFl4ZqkErRpq/dAPyeWp15tGrcrrJho1hKK5uBS70BCtfrIFg81sqcTVu3Ta+KD1Tk5vAtBNElWxtAa+m8K9A==} + dependencies: + '@types/estree': 1.0.5 + devlop: 1.1.0 + micromark-core-commonmark: 2.0.0 + micromark-util-character: 2.0.1 + micromark-util-events-to-acorn: 2.0.2 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + unist-util-position-from-estree: 2.0.0 + vfile-message: 4.0.2 + dev: false + + /micromark-extension-mdxjs@3.0.0: + resolution: {integrity: sha512-A873fJfhnJ2siZyUrJ31l34Uqwy4xIFmvPY1oj+Ean5PHcPBYzEsvqvWGaWcfEIr11O5Dlw3p2y0tZWpKHDejQ==} + dependencies: + acorn: 8.11.3 + acorn-jsx: 5.3.2(acorn@8.11.3) + micromark-extension-mdx-expression: 3.0.0 + micromark-extension-mdx-jsx: 3.0.0 + micromark-extension-mdx-md: 2.0.0 + micromark-extension-mdxjs-esm: 3.0.0 + micromark-util-combine-extensions: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-factory-destination@1.1.0: + resolution: {integrity: sha512-XaNDROBgx9SgSChd69pjiGKbV+nfHGDPVYFs5dOoDd7ZnMAE+Cuu91BCpsY8RT2NP9vo/B8pds2VQNCLiu0zhg==} + dependencies: + micromark-util-character: 1.2.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 + dev: false + + /micromark-factory-destination@2.0.0: + resolution: {integrity: sha512-j9DGrQLm/Uhl2tCzcbLhy5kXsgkHUrjJHg4fFAeoMRwJmJerT9aw4FEhIbZStWN8A3qMwOp1uzHr4UL8AInxtA==} + dependencies: + micromark-util-character: 2.0.1 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-factory-label@1.1.0: + resolution: {integrity: sha512-OLtyez4vZo/1NjxGhcpDSbHQ+m0IIGnT8BoPamh+7jVlzLJBH98zzuCoUeMxvM6WsNeh8wx8cKvqLiPHEACn0w==} + dependencies: + micromark-util-character: 1.2.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 + uvu: 0.5.6 + dev: false + + /micromark-factory-label@2.0.0: + resolution: {integrity: sha512-RR3i96ohZGde//4WSe/dJsxOX6vxIg9TimLAS3i4EhBAFx8Sm5SmqVfR8E87DPSR31nEAjZfbt91OMZWcNgdZw==} + dependencies: + devlop: 1.1.0 + micromark-util-character: 2.0.1 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-factory-mdx-expression@2.0.1: + resolution: {integrity: sha512-F0ccWIUHRLRrYp5TC9ZYXmZo+p2AM13ggbsW4T0b5CRKP8KHVRB8t4pwtBgTxtjRmwrK0Irwm7vs2JOZabHZfg==} + dependencies: + '@types/estree': 1.0.5 + devlop: 1.1.0 + micromark-util-character: 2.0.1 + micromark-util-events-to-acorn: 2.0.2 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + unist-util-position-from-estree: 2.0.0 + vfile-message: 4.0.2 + dev: false + + /micromark-factory-space@1.1.0: + resolution: {integrity: sha512-cRzEj7c0OL4Mw2v6nwzttyOZe8XY/Z8G0rzmWQZTBi/jjwyw/U4uqKtUORXQrR5bAZZnbTI/feRV/R7hc4jQYQ==} + dependencies: + micromark-util-character: 1.2.0 + micromark-util-types: 1.1.0 + dev: false + + /micromark-factory-space@2.0.0: + resolution: {integrity: sha512-TKr+LIDX2pkBJXFLzpyPyljzYK3MtmllMUMODTQJIUfDGncESaqB90db9IAUcz4AZAJFdd8U9zOp9ty1458rxg==} + dependencies: + micromark-util-character: 2.0.1 + micromark-util-types: 2.0.0 + dev: false + + /micromark-factory-title@1.1.0: + resolution: {integrity: sha512-J7n9R3vMmgjDOCY8NPw55jiyaQnH5kBdV2/UXCtZIpnHH3P6nHUKaH7XXEYuWwx/xUJcawa8plLBEjMPU24HzQ==} + dependencies: + micromark-factory-space: 1.1.0 + micromark-util-character: 1.2.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 + dev: false + + /micromark-factory-title@2.0.0: + resolution: {integrity: sha512-jY8CSxmpWLOxS+t8W+FG3Xigc0RDQA9bKMY/EwILvsesiRniiVMejYTE4wumNc2f4UbAa4WsHqe3J1QS1sli+A==} + dependencies: + micromark-factory-space: 2.0.0 + micromark-util-character: 2.0.1 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-factory-whitespace@1.1.0: + resolution: {integrity: sha512-v2WlmiymVSp5oMg+1Q0N1Lxmt6pMhIHD457whWM7/GUlEks1hI9xj5w3zbc4uuMKXGisksZk8DzP2UyGbGqNsQ==} + dependencies: + micromark-factory-space: 1.1.0 + micromark-util-character: 1.2.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 + dev: false + + /micromark-factory-whitespace@2.0.0: + resolution: {integrity: sha512-28kbwaBjc5yAI1XadbdPYHX/eDnqaUFVikLwrO7FDnKG7lpgxnvk/XGRhX/PN0mOZ+dBSZ+LgunHS+6tYQAzhA==} + dependencies: + micromark-factory-space: 2.0.0 + micromark-util-character: 2.0.1 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-util-character@1.2.0: + resolution: {integrity: sha512-lXraTwcX3yH/vMDaFWCQJP1uIszLVebzUa3ZHdrgxr7KEU/9mL4mVgCpGbyhvNLNlauROiNUq7WN5u7ndbY6xg==} + dependencies: + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 + dev: false + + /micromark-util-character@2.0.1: + resolution: {integrity: sha512-3wgnrmEAJ4T+mGXAUfMvMAbxU9RDG43XmGce4j6CwPtVxB3vfwXSZ6KhFwDzZ3mZHhmPimMAXg71veiBGzeAZw==} + dependencies: + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-util-chunked@1.1.0: + resolution: {integrity: sha512-Ye01HXpkZPNcV6FiyoW2fGZDUw4Yc7vT0E9Sad83+bEDiCJ1uXu0S3mr8WLpsz3HaG3x2q0HM6CTuPdcZcluFQ==} + dependencies: + micromark-util-symbol: 1.1.0 + dev: false + + /micromark-util-chunked@2.0.0: + resolution: {integrity: sha512-anK8SWmNphkXdaKgz5hJvGa7l00qmcaUQoMYsBwDlSKFKjc6gjGXPDw3FNL3Nbwq5L8gE+RCbGqTw49FK5Qyvg==} + dependencies: + micromark-util-symbol: 2.0.0 + dev: false + + /micromark-util-classify-character@1.1.0: + resolution: {integrity: sha512-SL0wLxtKSnklKSUplok1WQFoGhUdWYKggKUiqhX+Swala+BtptGCu5iPRc+xvzJ4PXE/hwM3FNXsfEVgoZsWbw==} + dependencies: + micromark-util-character: 1.2.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 + dev: false + + /micromark-util-classify-character@2.0.0: + resolution: {integrity: sha512-S0ze2R9GH+fu41FA7pbSqNWObo/kzwf8rN/+IGlW/4tC6oACOs8B++bh+i9bVyNnwCcuksbFwsBme5OCKXCwIw==} + dependencies: + micromark-util-character: 2.0.1 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-util-combine-extensions@1.1.0: + resolution: {integrity: sha512-Q20sp4mfNf9yEqDL50WwuWZHUrCO4fEyeDCnMGmG5Pr0Cz15Uo7KBs6jq+dq0EgX4DPwwrh9m0X+zPV1ypFvUA==} + dependencies: + micromark-util-chunked: 1.1.0 + micromark-util-types: 1.1.0 + dev: false + + /micromark-util-combine-extensions@2.0.0: + resolution: {integrity: sha512-vZZio48k7ON0fVS3CUgFatWHoKbbLTK/rT7pzpJ4Bjp5JjkZeasRfrS9wsBdDJK2cJLHMckXZdzPSSr1B8a4oQ==} + dependencies: + micromark-util-chunked: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-util-decode-numeric-character-reference@1.1.0: + resolution: {integrity: sha512-m9V0ExGv0jB1OT21mrWcuf4QhP46pH1KkfWy9ZEezqHKAxkj4mPCy3nIH1rkbdMlChLHX531eOrymlwyZIf2iw==} + dependencies: + micromark-util-symbol: 1.1.0 + dev: false + + /micromark-util-decode-numeric-character-reference@2.0.1: + resolution: {integrity: sha512-bmkNc7z8Wn6kgjZmVHOX3SowGmVdhYS7yBpMnuMnPzDq/6xwVA604DuOXMZTO1lvq01g+Adfa0pE2UKGlxL1XQ==} + dependencies: + micromark-util-symbol: 2.0.0 + dev: false + + /micromark-util-decode-string@1.1.0: + resolution: {integrity: sha512-YphLGCK8gM1tG1bd54azwyrQRjCFcmgj2S2GoJDNnh4vYtnL38JS8M4gpxzOPNyHdNEpheyWXCTnnTDY3N+NVQ==} + dependencies: + decode-named-character-reference: 1.0.2 + micromark-util-character: 1.2.0 + micromark-util-decode-numeric-character-reference: 1.1.0 + micromark-util-symbol: 1.1.0 + dev: false + + /micromark-util-decode-string@2.0.0: + resolution: {integrity: sha512-r4Sc6leeUTn3P6gk20aFMj2ntPwn6qpDZqWvYmAG6NgvFTIlj4WtrAudLi65qYoaGdXYViXYw2pkmn7QnIFasA==} + dependencies: + decode-named-character-reference: 1.0.2 + micromark-util-character: 2.0.1 + micromark-util-decode-numeric-character-reference: 2.0.1 + micromark-util-symbol: 2.0.0 + dev: false + + /micromark-util-encode@1.1.0: + resolution: {integrity: sha512-EuEzTWSTAj9PA5GOAs992GzNh2dGQO52UvAbtSOMvXTxv3Criqb6IOzJUBCmEqrrXSblJIJBbFFv6zPxpreiJw==} + dev: false + + /micromark-util-encode@2.0.0: + resolution: {integrity: sha512-pS+ROfCXAGLWCOc8egcBvT0kf27GoWMqtdarNfDcjb6YLuV5cM3ioG45Ys2qOVqeqSbjaKg72vU+Wby3eddPsA==} + dev: false + + /micromark-util-events-to-acorn@2.0.2: + resolution: {integrity: sha512-Fk+xmBrOv9QZnEDguL9OI9/NQQp6Hz4FuQ4YmCb/5V7+9eAh1s6AYSvL20kHkD67YIg7EpE54TiSlcsf3vyZgA==} + dependencies: + '@types/acorn': 4.0.6 + '@types/estree': 1.0.5 + '@types/unist': 3.0.2 + devlop: 1.1.0 + estree-util-visit: 2.0.0 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + vfile-message: 4.0.2 + dev: false + + /micromark-util-html-tag-name@1.2.0: + resolution: {integrity: sha512-VTQzcuQgFUD7yYztuQFKXT49KghjtETQ+Wv/zUjGSGBioZnkA4P1XXZPT1FHeJA6RwRXSF47yvJ1tsJdoxwO+Q==} + dev: false + + /micromark-util-html-tag-name@2.0.0: + resolution: {integrity: sha512-xNn4Pqkj2puRhKdKTm8t1YHC/BAjx6CEwRFXntTaRf/x16aqka6ouVoutm+QdkISTlT7e2zU7U4ZdlDLJd2Mcw==} + dev: false + + /micromark-util-normalize-identifier@1.1.0: + resolution: {integrity: sha512-N+w5vhqrBihhjdpM8+5Xsxy71QWqGn7HYNUvch71iV2PM7+E3uWGox1Qp90loa1ephtCxG2ftRV/Conitc6P2Q==} + dependencies: + micromark-util-symbol: 1.1.0 + dev: false + + /micromark-util-normalize-identifier@2.0.0: + resolution: {integrity: sha512-2xhYT0sfo85FMrUPtHcPo2rrp1lwbDEEzpx7jiH2xXJLqBuy4H0GgXk5ToU8IEwoROtXuL8ND0ttVa4rNqYK3w==} + dependencies: + micromark-util-symbol: 2.0.0 + dev: false + + /micromark-util-resolve-all@1.1.0: + resolution: {integrity: sha512-b/G6BTMSg+bX+xVCshPTPyAu2tmA0E4X98NSR7eIbeC6ycCqCeE7wjfDIgzEbkzdEVJXRtOG4FbEm/uGbCRouA==} + dependencies: + micromark-util-types: 1.1.0 + dev: false + + /micromark-util-resolve-all@2.0.0: + resolution: {integrity: sha512-6KU6qO7DZ7GJkaCgwBNtplXCvGkJToU86ybBAUdavvgsCiG8lSSvYxr9MhwmQ+udpzywHsl4RpGJsYWG1pDOcA==} + dependencies: + micromark-util-types: 2.0.0 + dev: false + + /micromark-util-sanitize-uri@1.2.0: + resolution: {integrity: sha512-QO4GXv0XZfWey4pYFndLUKEAktKkG5kZTdUNaTAkzbuJxn2tNBOr+QtxR2XpWaMhbImT2dPzyLrPXLlPhph34A==} + dependencies: + micromark-util-character: 1.2.0 + micromark-util-encode: 1.1.0 + micromark-util-symbol: 1.1.0 + dev: false + + /micromark-util-sanitize-uri@2.0.0: + resolution: {integrity: sha512-WhYv5UEcZrbAtlsnPuChHUAsu/iBPOVaEVsntLBIdpibO0ddy8OzavZz3iL2xVvBZOpolujSliP65Kq0/7KIYw==} + dependencies: + micromark-util-character: 2.0.1 + micromark-util-encode: 2.0.0 + micromark-util-symbol: 2.0.0 + dev: false + + /micromark-util-subtokenize@1.1.0: + resolution: {integrity: sha512-kUQHyzRoxvZO2PuLzMt2P/dwVsTiivCK8icYTeR+3WgbuPqfHgPPy7nFKbeqRivBvn/3N3GBiNC+JRTMSxEC7A==} + dependencies: + micromark-util-chunked: 1.1.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 + uvu: 0.5.6 + dev: false + + /micromark-util-subtokenize@2.0.0: + resolution: {integrity: sha512-vc93L1t+gpR3p8jxeVdaYlbV2jTYteDje19rNSS/H5dlhxUYll5Fy6vJ2cDwP8RnsXi818yGty1ayP55y3W6fg==} + dependencies: + devlop: 1.1.0 + micromark-util-chunked: 2.0.0 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + dev: false + + /micromark-util-symbol@1.1.0: + resolution: {integrity: sha512-uEjpEYY6KMs1g7QfJ2eX1SQEV+ZT4rUD3UcF6l57acZvLNK7PBZL+ty82Z1qhK1/yXIY4bdx04FKMgR0g4IAag==} + dev: false + + /micromark-util-symbol@2.0.0: + resolution: {integrity: sha512-8JZt9ElZ5kyTnO94muPxIGS8oyElRJaiJO8EzV6ZSyGQ1Is8xwl4Q45qU5UOg+bGH4AikWziz0iN4sFLWs8PGw==} + dev: false + + /micromark-util-types@1.1.0: + resolution: {integrity: sha512-ukRBgie8TIAcacscVHSiddHjO4k/q3pnedmzMQ4iwDcK0FtFCohKOlFbaOL/mPgfnPsL3C1ZyxJa4sbWrBl3jg==} + dev: false + + /micromark-util-types@2.0.0: + resolution: {integrity: sha512-oNh6S2WMHWRZrmutsRmDDfkzKtxF+bc2VxLC9dvtrDIRFln627VsFP6fLMgTryGDljgLPjkrzQSDcPrjPyDJ5w==} + dev: false + + /micromark@3.2.0: + resolution: {integrity: sha512-uD66tJj54JLYq0De10AhWycZWGQNUvDI55xPgk2sQM5kn1JYlhbCMTtEeT27+vAhW2FBQxLlOmS3pmA7/2z4aA==} + dependencies: + '@types/debug': 4.1.12 + debug: 4.3.4 + decode-named-character-reference: 1.0.2 + micromark-core-commonmark: 1.1.0 + micromark-factory-space: 1.1.0 + micromark-util-character: 1.2.0 + micromark-util-chunked: 1.1.0 + micromark-util-combine-extensions: 1.1.0 + micromark-util-decode-numeric-character-reference: 1.1.0 + micromark-util-encode: 1.1.0 + micromark-util-normalize-identifier: 1.1.0 + micromark-util-resolve-all: 1.1.0 + micromark-util-sanitize-uri: 1.2.0 + micromark-util-subtokenize: 1.1.0 + micromark-util-symbol: 1.1.0 + micromark-util-types: 1.1.0 + uvu: 0.5.6 + transitivePeerDependencies: + - supports-color + dev: false + + /micromark@4.0.0: + resolution: {integrity: sha512-o/sd0nMof8kYff+TqcDx3VSrgBTcZpSvYcAHIfHhv5VAuNmisCxjhx6YmxS8PFEpb9z5WKWKPdzf0jM23ro3RQ==} + dependencies: + '@types/debug': 4.1.12 + debug: 4.3.4 + decode-named-character-reference: 1.0.2 + devlop: 1.1.0 + micromark-core-commonmark: 2.0.0 + micromark-factory-space: 2.0.0 + micromark-util-character: 2.0.1 + micromark-util-chunked: 2.0.0 + micromark-util-combine-extensions: 2.0.0 + micromark-util-decode-numeric-character-reference: 2.0.1 + micromark-util-encode: 2.0.0 + micromark-util-normalize-identifier: 2.0.0 + micromark-util-resolve-all: 2.0.0 + micromark-util-sanitize-uri: 2.0.0 + micromark-util-subtokenize: 2.0.0 + micromark-util-symbol: 2.0.0 + micromark-util-types: 2.0.0 + transitivePeerDependencies: + - supports-color + dev: false + + /micromatch@4.0.5: + resolution: {integrity: sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==} + engines: {node: '>=8.6'} + dependencies: + braces: 3.0.2 + picomatch: 2.3.1 + dev: false + + /mime-db@1.33.0: + resolution: {integrity: sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ==} + engines: {node: '>= 0.6'} + dev: false + + /mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + dev: false + + /mime-types@2.1.18: + resolution: {integrity: sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ==} + engines: {node: '>= 0.6'} + dependencies: + mime-db: 1.33.0 + dev: false + + /mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + dependencies: + mime-db: 1.52.0 + dev: false + + /mime@1.6.0: + resolution: {integrity: sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==} + engines: {node: '>=4'} + hasBin: true + dev: false + + /mimic-fn@2.1.0: + resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} + engines: {node: '>=6'} + dev: false + + /mimic-response@3.1.0: + resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} + engines: {node: '>=10'} + dev: false + + /mimic-response@4.0.0: + resolution: {integrity: sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dev: false + + /mini-create-react-context@0.4.1(prop-types@15.8.1)(react@18.2.0): + resolution: {integrity: sha512-YWCYEmd5CQeHGSAKrYvXgmzzkrvssZcuuQDDeqkT+PziKGMgE+0MCCtcKbROzocGBG1meBLl2FotlRwf4gAzbQ==} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. + peerDependencies: + prop-types: ^15.0.0 + react: ^0.14.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 + dependencies: + '@babel/runtime': 7.23.7 + prop-types: 15.8.1 + react: 18.2.0 + tiny-warning: 1.0.3 + dev: false + + /mini-css-extract-plugin@2.7.6(webpack@5.89.0): + resolution: {integrity: sha512-Qk7HcgaPkGG6eD77mLvZS1nmxlao3j+9PkrT9Uc7HAE1id3F41+DdBRYRYkbyfNRGzm8/YWtzhw7nVPmwhqTQw==} + engines: {node: '>= 12.13.0'} + peerDependencies: + webpack: ^5.0.0 + dependencies: + schema-utils: 4.2.0 + webpack: 5.89.0 + dev: false + + /minimalistic-assert@1.0.1: + resolution: {integrity: sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==} + dev: false + + /minimatch@3.1.2: + resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} + dependencies: + brace-expansion: 1.1.11 + dev: false + + /minimist@1.2.8: + resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + dev: false + + /mri@1.2.0: + resolution: {integrity: sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==} + engines: {node: '>=4'} + dev: false + + /mrmime@2.0.0: + resolution: {integrity: sha512-eu38+hdgojoyq63s+yTpN4XMBdt5l8HhMhc4VKLO9KM5caLIBvUm4thi7fFaxyTmCKeNnXZ5pAlBwCUnhA09uw==} + engines: {node: '>=10'} + dev: false + + /ms@2.0.0: + resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} + dev: false + + /ms@2.1.2: + resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} + dev: false + + /ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + dev: false + + /multicast-dns@7.2.5: + resolution: {integrity: sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg==} + hasBin: true + dependencies: + dns-packet: 5.6.1 + thunky: 1.1.0 + dev: false + + /nanoid@3.3.7: + resolution: {integrity: sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + dev: false + + /negotiator@0.6.3: + resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} + engines: {node: '>= 0.6'} + dev: false + + /neo-async@2.6.2: + resolution: {integrity: sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==} + dev: false + + /no-case@3.0.4: + resolution: {integrity: sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==} + dependencies: + lower-case: 2.0.2 + tslib: 2.6.2 + dev: false + + /node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + dev: false + + /node-emoji@2.1.3: + resolution: {integrity: sha512-E2WEOVsgs7O16zsURJ/eH8BqhF029wGpEOnv7Urwdo2wmQanOACwJQh0devF9D9RhoZru0+9JXIS0dBXIAz+lA==} + engines: {node: '>=18'} + dependencies: + '@sindresorhus/is': 4.6.0 + char-regex: 1.0.2 + emojilib: 2.4.0 + skin-tone: 2.0.0 + dev: false + + /node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + dev: false + + /node-forge@1.3.1: + resolution: {integrity: sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==} + engines: {node: '>= 6.13.0'} + dev: false + + /node-releases@2.0.14: + resolution: {integrity: sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==} + dev: false + + /non-layered-tidy-tree-layout@2.0.2: + resolution: {integrity: sha512-gkXMxRzUH+PB0ax9dUN0yYF0S25BqeAYqhgMaLUFmpXLEk7Fcu8f4emJuOAY0V8kjDICxROIKsTAKsV/v355xw==} + dev: false + + /normalize-path@3.0.0: + resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} + engines: {node: '>=0.10.0'} + dev: false + + /normalize-range@0.1.2: + resolution: {integrity: sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA==} + engines: {node: '>=0.10.0'} + dev: false + + /normalize-url@6.1.0: + resolution: {integrity: sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==} + engines: {node: '>=10'} + dev: false + + /normalize-url@8.0.0: + resolution: {integrity: sha512-uVFpKhj5MheNBJRTiMZ9pE/7hD1QTeEvugSJW/OmLzAp78PB5O6adfMNTvmfKhXBkvCzC+rqifWcVYpGFwTjnw==} + engines: {node: '>=14.16'} + dev: false + + /npm-run-path@4.0.1: + resolution: {integrity: sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==} + engines: {node: '>=8'} + dependencies: + path-key: 3.1.1 + dev: false + + /nprogress@0.2.0: + resolution: {integrity: sha512-I19aIingLgR1fmhftnbWWO3dXc0hSxqHQHQb3H8m+K3TnEn/iSeTZZOyvKXWqQESMwuUVnatlCnZdLBZZt2VSA==} + dev: false + + /nth-check@2.0.1: + resolution: {integrity: sha512-it1vE95zF6dTT9lBsYbxvqh0Soy4SPowchj0UBGj/V6cTPnXXtQOPUbhZ6CmGzAD/rW22LQK6E96pcdJXk4A4w==} + dependencies: + boolbase: 1.0.0 + dev: false + + /object-assign@4.1.1: + resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} + engines: {node: '>=0.10.0'} + dev: false + + /object-inspect@1.13.1: + resolution: {integrity: sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==} + dev: false + + /object-keys@1.1.1: + resolution: {integrity: sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==} + engines: {node: '>= 0.4'} + dev: false + + /object.assign@4.1.5: + resolution: {integrity: sha512-byy+U7gp+FVwmyzKPYhW2h5l3crpmGsxl7X2s8y43IgxvG4g3QZ6CffDtsNQy1WsmZpQbO+ybo0AlW7TY6DcBQ==} + engines: {node: '>= 0.4'} + dependencies: + call-bind: 1.0.5 + define-properties: 1.2.1 + has-symbols: 1.0.3 + object-keys: 1.1.1 + dev: false + + /obuf@1.1.2: + resolution: {integrity: sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==} + dev: false + + /on-finished@2.4.1: + resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} + engines: {node: '>= 0.8'} + dependencies: + ee-first: 1.1.1 + dev: false + + /on-headers@1.0.2: + resolution: {integrity: sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==} + engines: {node: '>= 0.8'} + dev: false + + /once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + dependencies: + wrappy: 1.0.2 + dev: false + + /onetime@5.1.2: + resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} + engines: {node: '>=6'} + dependencies: + mimic-fn: 2.1.0 + dev: false + + /open@8.4.2: + resolution: {integrity: sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==} + engines: {node: '>=12'} + dependencies: + define-lazy-prop: 2.0.0 + is-docker: 2.2.1 + is-wsl: 2.2.0 + dev: false + + /opener@1.5.2: + resolution: {integrity: sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==} + hasBin: true + dev: false + + /p-cancelable@3.0.0: + resolution: {integrity: sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==} + engines: {node: '>=12.20'} + dev: false + + /p-limit@2.3.0: + resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==} + engines: {node: '>=6'} + dependencies: + p-try: 2.2.0 + dev: false + + /p-limit@3.1.0: + resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} + engines: {node: '>=10'} + dependencies: + yocto-queue: 0.1.0 + dev: false + + /p-limit@4.0.0: + resolution: {integrity: sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + yocto-queue: 1.0.0 + dev: false + + /p-locate@3.0.0: + resolution: {integrity: sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==} + engines: {node: '>=6'} + dependencies: + p-limit: 2.3.0 + dev: false + + /p-locate@5.0.0: + resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} + engines: {node: '>=10'} + dependencies: + p-limit: 3.1.0 + dev: false + + /p-locate@6.0.0: + resolution: {integrity: sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + p-limit: 4.0.0 + dev: false + + /p-map@4.0.0: + resolution: {integrity: sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==} + engines: {node: '>=10'} + dependencies: + aggregate-error: 3.1.0 + dev: false + + /p-retry@4.6.2: + resolution: {integrity: sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==} + engines: {node: '>=8'} + dependencies: + '@types/retry': 0.12.0 + retry: 0.13.1 + dev: false + + /p-try@2.2.0: + resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} + engines: {node: '>=6'} + dev: false + + /package-json@8.1.1: + resolution: {integrity: sha512-cbH9IAIJHNj9uXi196JVsRlt7cHKak6u/e6AkL/bkRelZ7rlL3X1YKxsZwa36xipOEKAsdtmaG6aAJoM1fx2zA==} + engines: {node: '>=14.16'} + dependencies: + got: 12.6.1 + registry-auth-token: 5.0.2 + registry-url: 6.0.1 + semver: 7.5.4 + dev: false + + /param-case@3.0.4: + resolution: {integrity: sha512-RXlj7zCYokReqWpOPH9oYivUzLYZ5vAPIfEmCTNViosC78F8F0H9y7T7gG2M39ymgutxF5gcFEsyZQSph9Bp3A==} + dependencies: + dot-case: 3.0.4 + tslib: 2.6.2 + dev: false + + /parent-module@1.0.1: + resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} + engines: {node: '>=6'} + dependencies: + callsites: 3.1.0 + dev: false + + /parse-entities@4.0.1: + resolution: {integrity: sha512-SWzvYcSJh4d/SGLIOQfZ/CoNv6BTlI6YEQ7Nj82oDVnRpwe/Z/F1EMx42x3JAOwGBlCjeCH0BRJQbQ/opHL17w==} + dependencies: + '@types/unist': 2.0.10 + character-entities: 2.0.2 + character-entities-legacy: 3.0.0 + character-reference-invalid: 2.0.1 + decode-named-character-reference: 1.0.2 + is-alphanumerical: 2.0.1 + is-decimal: 2.0.1 + is-hexadecimal: 2.0.1 + dev: false + + /parse-json@5.2.0: + resolution: {integrity: sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==} + engines: {node: '>=8'} + dependencies: + '@babel/code-frame': 7.23.5 + error-ex: 1.3.2 + json-parse-even-better-errors: 2.3.1 + lines-and-columns: 1.2.4 + dev: false + + /parse-numeric-range@1.3.0: + resolution: {integrity: sha512-twN+njEipszzlMJd4ONUYgSfZPDxgHhT9Ahed5uTigpQn90FggW4SA/AIPq/6a149fTbE9qBEcSwE3FAEp6wQQ==} + dev: false + + /parse5-htmlparser2-tree-adapter@7.0.0: + resolution: {integrity: sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==} + dependencies: + domhandler: 5.0.3 + parse5: 7.1.2 + dev: false + + /parse5@7.1.2: + resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==} + dependencies: + entities: 4.5.0 + dev: false + + /parseurl@1.3.3: + resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} + engines: {node: '>= 0.8'} + dev: false + + /pascal-case@3.1.2: + resolution: {integrity: sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g==} + dependencies: + no-case: 3.0.4 + tslib: 2.6.2 + dev: false + + /path-exists@3.0.0: + resolution: {integrity: sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==} + engines: {node: '>=4'} + dev: false + + /path-exists@4.0.0: + resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} + engines: {node: '>=8'} + dev: false + + /path-exists@5.0.0: + resolution: {integrity: sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dev: false + + /path-is-absolute@1.0.1: + resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} + engines: {node: '>=0.10.0'} + dev: false + + /path-is-inside@1.0.2: + resolution: {integrity: sha512-DUWJr3+ULp4zXmol/SZkFf3JGsS9/SIv+Y3Rt93/UjPpDpklB5f1er4O3POIbUuUJ3FXgqte2Q7SrU6zAqwk8w==} + dev: false + + /path-key@3.1.1: + resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} + engines: {node: '>=8'} + dev: false + + /path-parse@1.0.7: + resolution: {integrity: sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==} + dev: false + + /path-to-regexp@0.1.7: + resolution: {integrity: sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==} + dev: false + + /path-to-regexp@1.8.0: + resolution: {integrity: sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==} + dependencies: + isarray: 0.0.1 + dev: false + + /path-to-regexp@2.2.1: + resolution: {integrity: sha512-gu9bD6Ta5bwGrrU8muHzVOBFFREpp2iRkVfhBJahwJ6p6Xw20SjT0MxLnwkjOibQmGSYhiUnf2FLe7k+jcFmGQ==} + dev: false + + /path-type@4.0.0: + resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} + engines: {node: '>=8'} + dev: false + + /periscopic@3.1.0: + resolution: {integrity: sha512-vKiQ8RRtkl9P+r/+oefh25C3fhybptkHKCZSPlcXiJux2tJF55GnEj3BVn4A5gKfq9NWWXXrxkHBwVPUfH0opw==} + dependencies: + '@types/estree': 1.0.5 + estree-walker: 3.0.3 + is-reference: 3.0.2 + dev: false + + /picocolors@1.0.0: + resolution: {integrity: sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==} + dev: false + + /picomatch@2.3.1: + resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==} + engines: {node: '>=8.6'} + dev: false + + /pkg-dir@7.0.0: + resolution: {integrity: sha512-Ie9z/WINcxxLp27BKOCHGde4ITq9UklYKDzVo1nhk5sqGEXU3FpkwP5GM2voTGJkGd9B3Otl+Q4uwSOeSUtOBA==} + engines: {node: '>=14.16'} + dependencies: + find-up: 6.3.0 + dev: false + + /pkg-up@3.1.0: + resolution: {integrity: sha512-nDywThFk1i4BQK4twPQ6TA4RT8bDY96yeuCVBWL3ePARCiEKDRSrNGbFIgUJpLp+XeIR65v8ra7WuJOFUBtkMA==} + engines: {node: '>=8'} + dependencies: + find-up: 3.0.0 + dev: false + + /postcss-calc@8.2.4(postcss@8.4.32): + resolution: {integrity: sha512-SmWMSJmB8MRnnULldx0lQIyhSNvuDl9HfrZkaqqE/WHAhToYsAvDq+yAsA/kIyINDszOp3Rh0GFoNuH5Ypsm3Q==} + peerDependencies: + postcss: ^8.2.2 + dependencies: + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-calc@9.0.1(postcss@8.4.32): + resolution: {integrity: sha512-TipgjGyzP5QzEhsOZUaIkeO5mKeMFpebWzRogWG/ysonUlnHcq5aJe0jOjpfzUU8PeSaBQnrE8ehR0QA5vs8PQ==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.2.2 + dependencies: + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-colormin@5.3.1(postcss@8.4.32): + resolution: {integrity: sha512-UsWQG0AqTFQmpBegeLLc1+c3jIqBNB0zlDGRWR+dQ3pRKJL1oeMzyqmH3o2PIfn9MBdNrVPWhDbT769LxCTLJQ==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + browserslist: 4.22.2 + caniuse-api: 3.0.0 + colord: 2.9.3 + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-colormin@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-TXKOxs9LWcdYo5cgmcSHPkyrLAh86hX1ijmyy6J8SbOhyv6ua053M3ZAM/0j44UsnQNIWdl8gb5L7xX2htKeLw==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + browserslist: 4.22.2 + caniuse-api: 3.0.0 + colord: 2.9.3 + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-convert-values@5.1.3(postcss@8.4.32): + resolution: {integrity: sha512-82pC1xkJZtcJEfiLw6UXnXVXScgtBrjlO5CBmuDQc+dlb88ZYheFsjTn40+zBVi3DkfF7iezO0nJUPLcJK3pvA==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + browserslist: 4.22.2 + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-convert-values@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-zTd4Vh0HxGkhg5aHtfCogcRHzGkvblfdWlQ53lIh1cJhYcGyIxh2hgtKoVh40AMktRERet+JKdB04nNG19kjmA==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + browserslist: 4.22.2 + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-convert-values@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-aeBmaTnGQ+NUSVQT8aY0sKyAD/BaLJenEKZ03YK0JnDE1w1Rr8XShoxdal2V2H26xTJKr3v5haByOhJuyT4UYw==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + browserslist: 4.22.2 + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-discard-comments@5.1.2(postcss@8.4.32): + resolution: {integrity: sha512-+L8208OVbHVF2UQf1iDmRcbdjJkuBF6IS29yBDSiWUIzpYaAhtNl6JYnYm12FnkeCwQqF5LeklOu6rAqgfBZqQ==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-discard-comments@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-f1KYNPtqYLUeZGCHQPKzzFtsHaRuECe6jLakf/RjSRqvF5XHLZnM2+fXLhb8Qh/HBFHs3M4cSLb1k3B899RYIg==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-discard-duplicates@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-zmX3IoSI2aoenxHV6C7plngHWWhUOV3sP1T8y2ifzxzbtnuhk1EdPwm0S1bIUNaJ2eNbWeGLEwzw8huPD67aQw==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-discard-duplicates@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-1hvUs76HLYR8zkScbwyJ8oJEugfPV+WchpnA+26fpJ7Smzs51CzGBHC32RS03psuX/2l0l0UKh2StzNxOrKCYg==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-discard-empty@5.1.1(postcss@8.4.32): + resolution: {integrity: sha512-zPz4WljiSuLWsI0ir4Mcnr4qQQ5e1Ukc3i7UfE2XcrwKK2LIPIqE5jxMRxO6GbI3cv//ztXDsXwEWT3BHOGh3A==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-discard-empty@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-yitcmKwmVWtNsrrRqGJ7/C0YRy53i0mjexBDQ9zYxDwTWVBgbU4+C9jIZLmQlTDT9zhml+u0OMFJh8+31krmOg==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-discard-overridden@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-21nOL7RqWR1kasIVdKs8HNqQJhFxLsyRfAnUDm4Fe4t4mCWL9OJiHvlHPjcd8zc5Myu89b/7wZDnOSjFgeWRtw==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-discard-overridden@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-qs0ehZMMZpSESbRkw1+inkf51kak6OOzNRaoLd/U7Fatp0aN2HQ1rxGOrJvYcRAN9VpX8kUF13R2ofn8OlvFVA==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-discard-unused@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-KwLWymI9hbwXmJa0dkrzpRbSJEh0vVUd7r8t0yOGPcfKzyJJxFM8kLyC5Ev9avji6nY95pOp1W6HqIrfT+0VGw==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /postcss-discard-unused@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-wr3lRPahxARmjow5BWML+9bD9D1u6FpfxlWg4lZqCIwvQLBZQD/S0Rq6A/juQwVFVXvMeRGa9TX1vpXuQ6FhTQ==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /postcss-loader@7.3.4(postcss@8.4.32)(typescript@5.3.3)(webpack@5.89.0): + resolution: {integrity: sha512-iW5WTTBSC5BfsBJ9daFMPVrLT36MrNiC6fqOZTTaHjBNX6Pfd5p+hSBqe/fEeNd7pc13QiAyGt7VdGMw4eRC4A==} + engines: {node: '>= 14.15.0'} + peerDependencies: + postcss: ^7.0.0 || ^8.0.1 + webpack: ^5.0.0 + dependencies: + cosmiconfig: 8.3.6(typescript@5.3.3) + jiti: 1.21.0 + postcss: 8.4.32 + semver: 7.5.4 + webpack: 5.89.0 + transitivePeerDependencies: + - typescript + dev: false + + /postcss-merge-idents@5.1.1(postcss@8.4.32): + resolution: {integrity: sha512-pCijL1TREiCoog5nQp7wUe+TUonA2tC2sQ54UGeMmryK3UFGIYKqDyjnqd6RcuI4znFn9hWSLNN8xKE/vWcUQw==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + cssnano-utils: 3.1.0(postcss@8.4.32) + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-merge-idents@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-ApqNUkzl3MJP+43DIIvoer98t7tcDVAcnLeAKjuTIM7HkMk8NXB6eqscMIjwQISwoSeE0WrEyIqVy+HoHAVcZw==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + cssnano-utils: 4.0.1(postcss@8.4.32) + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-merge-longhand@5.1.7(postcss@8.4.32): + resolution: {integrity: sha512-YCI9gZB+PLNskrK0BB3/2OzPnGhPkBEwmwhfYk1ilBHYVAZB7/tkTHFBAnCrvBBOmeYyMYw3DMjT55SyxMBzjQ==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + stylehacks: 5.1.1(postcss@8.4.32) + dev: false + + /postcss-merge-longhand@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-vmr/HZQzaPXc45FRvSctqFTF05UaDnTn5ABX+UtQPJznDWT/QaFbVc/pJ5C2YPxx2J2XcfmWowlKwtCDwiQ5hA==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + stylehacks: 6.0.2(postcss@8.4.32) + dev: false + + /postcss-merge-longhand@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-+yfVB7gEM8SrCo9w2lCApKIEzrTKl5yS1F4yGhV3kSim6JzbfLGJyhR1B6X+6vOT0U33Mgx7iv4X9MVWuaSAfw==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + stylehacks: 6.0.2(postcss@8.4.32) + dev: false + + /postcss-merge-rules@5.1.4(postcss@8.4.32): + resolution: {integrity: sha512-0R2IuYpgU93y9lhVbO/OylTtKMVcHb67zjWIfCiKR9rWL3GUk1677LAqD/BcHizukdZEjT8Ru3oHRoAYoJy44g==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + browserslist: 4.22.2 + caniuse-api: 3.0.0 + cssnano-utils: 3.1.0(postcss@8.4.32) + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /postcss-merge-rules@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-6lm8bl0UfriSfxI+F/cezrebqqP8w702UC6SjZlUlBYwuRVNbmgcJuQU7yePIvD4MNT53r/acQCUAyulrpgmeQ==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + browserslist: 4.22.2 + caniuse-api: 3.0.0 + cssnano-utils: 4.0.1(postcss@8.4.32) + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /postcss-merge-rules@6.0.3(postcss@8.4.32): + resolution: {integrity: sha512-yfkDqSHGohy8sGYIJwBmIGDv4K4/WrJPX355XrxQb/CSsT4Kc/RxDi6akqn5s9bap85AWgv21ArcUWwWdGNSHA==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + browserslist: 4.22.2 + caniuse-api: 3.0.0 + cssnano-utils: 4.0.1(postcss@8.4.32) + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /postcss-minify-font-values@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-el3mYTgx13ZAPPirSVsHqFzl+BBBDrXvbySvPGFnQcTI4iNslrPaFq4muTkLZmKlGk4gyFAYUBMH30+HurREyA==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-minify-font-values@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-tIwmF1zUPoN6xOtA/2FgVk1ZKrLcCvE0dpZLtzyyte0j9zUeB8RTbCqrHZGjJlxOvNWKMYtunLrrl7HPOiR46w==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-minify-gradients@5.1.1(postcss@8.4.32): + resolution: {integrity: sha512-VGvXMTpCEo4qHTNSa9A0a3D+dxGFZCYwR6Jokk+/3oB6flu2/PnPXAh2x7x52EkY5xlIHLm+Le8tJxe/7TNhzw==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + colord: 2.9.3 + cssnano-utils: 3.1.0(postcss@8.4.32) + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-minify-gradients@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-M1RJWVjd6IOLPl1hYiOd5HQHgpp6cvJVLrieQYS9y07Yo8itAr6jaekzJphaJFR0tcg4kRewCk3kna9uHBxn/w==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + colord: 2.9.3 + cssnano-utils: 4.0.1(postcss@8.4.32) + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-minify-params@5.1.4(postcss@8.4.32): + resolution: {integrity: sha512-+mePA3MgdmVmv6g+30rn57USjOGSAyuxUmkfiWpzalZ8aiBkdPYjXWtHuwJGm1v5Ojy0Z0LaSYhHaLJQB0P8Jw==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + browserslist: 4.22.2 + cssnano-utils: 3.1.0(postcss@8.4.32) + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-minify-params@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-zwQtbrPEBDj+ApELZ6QylLf2/c5zmASoOuA4DzolyVGdV38iR2I5QRMsZcHkcdkZzxpN8RS4cN7LPskOkTwTZw==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + browserslist: 4.22.2 + cssnano-utils: 4.0.1(postcss@8.4.32) + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-minify-selectors@5.2.1(postcss@8.4.32): + resolution: {integrity: sha512-nPJu7OjZJTsVUmPdm2TcaiohIwxP+v8ha9NehQ2ye9szv4orirRU3SDdtUmKH+10nzn0bAyOXZ0UEr7OpvLehg==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /postcss-minify-selectors@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-mfReq5wrS6vkunxvJp6GDuOk+Ak6JV7134gp8L+ANRnV9VwqzTvBtX6lpohooVU750AR0D3pVx2Zn6uCCwOAfQ==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /postcss-minify-selectors@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-0b+m+w7OAvZejPQdN2GjsXLv5o0jqYHX3aoV0e7RBKPCsB7TYG5KKWBFhGnB/iP3213Ts8c5H4wLPLMm7z28Sg==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /postcss-modules-extract-imports@3.0.0(postcss@8.4.32): + resolution: {integrity: sha512-bdHleFnP3kZ4NYDhuGlVK+CMrQ/pqUm8bx/oGL93K6gVwiclvX5x0n76fYMKuIGKzlABOy13zsvqjb0f92TEXw==} + engines: {node: ^10 || ^12 || >= 14} + peerDependencies: + postcss: ^8.1.0 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-modules-local-by-default@4.0.3(postcss@8.4.32): + resolution: {integrity: sha512-2/u2zraspoACtrbFRnTijMiQtb4GW4BvatjaG/bCjYQo8kLTdevCUlwuBHx2sCnSyrI3x3qj4ZK1j5LQBgzmwA==} + engines: {node: ^10 || ^12 || >= 14} + peerDependencies: + postcss: ^8.1.0 + dependencies: + icss-utils: 5.1.0(postcss@8.4.32) + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-modules-scope@3.1.0(postcss@8.4.32): + resolution: {integrity: sha512-SaIbK8XW+MZbd0xHPf7kdfA/3eOt7vxJ72IRecn3EzuZVLr1r0orzf0MX/pN8m+NMDoo6X/SQd8oeKqGZd8PXg==} + engines: {node: ^10 || ^12 || >= 14} + peerDependencies: + postcss: ^8.1.0 + dependencies: + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /postcss-modules-values@4.0.0(postcss@8.4.32): + resolution: {integrity: sha512-RDxHkAiEGI78gS2ofyvCsu7iycRv7oqw5xMWn9iMoR0N/7mf9D50ecQqUo5BZ9Zh2vH4bCUR/ktCqbB9m8vJjQ==} + engines: {node: ^10 || ^12 || >= 14} + peerDependencies: + postcss: ^8.1.0 + dependencies: + icss-utils: 5.1.0(postcss@8.4.32) + postcss: 8.4.32 + dev: false + + /postcss-normalize-charset@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-mSgUJ+pd/ldRGVx26p2wz9dNZ7ji6Pn8VWBajMXFf8jk7vUoSrZ2lt/wZR7DtlZYKesmZI680qjr2CeFF2fbUg==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-normalize-charset@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-aW5LbMNRZ+oDV57PF9K+WI1Z8MPnF+A8qbajg/T8PP126YrGX1f9IQx21GI2OlGz7XFJi/fNi0GTbY948XJtXg==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-normalize-display-values@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-WP4KIM4o2dazQXWmFaqMmcvsKmhdINFblgSeRgn8BJ6vxaMyaJkwAzpPpuvSIoG/rmX3M+IrRZEz2H0glrQNEA==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-display-values@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-mc3vxp2bEuCb4LgCcmG1y6lKJu1Co8T+rKHrcbShJwUmKJiEl761qb/QQCfFwlrvSeET3jksolCR/RZuMURudw==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-positions@5.1.1(postcss@8.4.32): + resolution: {integrity: sha512-6UpCb0G4eofTCQLFVuI3EVNZzBNPiIKcA1AKVka+31fTVySphr3VUgAIULBhxZkKgwLImhzMR2Bw1ORK+37INg==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-positions@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-HRsq8u/0unKNvm0cvwxcOUEcakFXqZ41fv3FOdPn916XFUrympjr+03oaLkuZENz3HE9RrQE9yU0Xv43ThWjQg==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-repeat-style@5.1.1(postcss@8.4.32): + resolution: {integrity: sha512-mFpLspGWkQtBcWIRFLmewo8aC3ImN2i/J3v8YCFUwDnPu3Xz4rLohDO26lGjwNsQxB3YF0KKRwspGzE2JEuS0g==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-repeat-style@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-Gbb2nmCy6tTiA7Sh2MBs3fj9W8swonk6lw+dFFeQT68B0Pzwp1kvisJQkdV6rbbMSd9brMlS8I8ts52tAGWmGQ==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-string@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-oYiIJOf4T9T1N4i+abeIc7Vgm/xPCGih4bZz5Nm0/ARVJ7K6xrDlLwvwqOydvyL3RHNf8qZk6vo3aatiw/go3w==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-string@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-5Fhx/+xzALJD9EI26Aq23hXwmv97Zfy2VFrt5PLT8lAhnBIZvmaT5pQk+NuJ/GWj/QWaKSKbnoKDGLbV6qnhXg==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-timing-functions@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-DOEkzJ4SAXv5xkHl0Wa9cZLF3WCBhF3o1SKVxKQAa+0pYKlueTpCgvkFAHfk+Y64ezX9+nITGrDZeVGgITJXjg==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-timing-functions@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-4zcczzHqmCU7L5dqTB9rzeqPWRMc0K2HoR+Bfl+FSMbqGBUcP5LRfgcH4BdRtLuzVQK1/FHdFoGT3F7rkEnY+g==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-unicode@5.1.1(postcss@8.4.32): + resolution: {integrity: sha512-qnCL5jzkNUmKVhZoENp1mJiGNPcsJCs1aaRmURmeJGES23Z/ajaln+EPTD+rBeNkSryI+2WTdW+lwcVdOikrpA==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + browserslist: 4.22.2 + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-unicode@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-Ff2VdAYCTGyMUwpevTZPZ4w0+mPjbZzLLyoLh/RMpqUqeQKZ+xMm31hkxBavDcGKcxm6ACzGk0nBfZ8LZkStKA==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + browserslist: 4.22.2 + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-url@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-5upGeDO+PVthOxSmds43ZeMeZfKH+/DKgGRD7TElkkyS46JXAUhMzIKiCa7BabPeIy3AQcTkXwVVN7DbqsiCew==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + normalize-url: 6.1.0 + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-url@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-jEXL15tXSvbjm0yzUV7FBiEXwhIa9H88JOXDGQzmcWoB4mSjZIsmtto066s2iW9FYuIrIF4k04HA2BKAOpbsaQ==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-whitespace@5.1.1(postcss@8.4.32): + resolution: {integrity: sha512-83ZJ4t3NUDETIHTa3uEg6asWjSBYL5EdkVB0sDncx9ERzOKBVJIUeDO9RyA9Zwtig8El1d79HBp0JEi8wvGQnA==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-normalize-whitespace@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-76i3NpWf6bB8UHlVuLRxG4zW2YykF9CTEcq/9LGAiz2qBuX5cBStadkk0jSkg9a9TCIXbMQz7yzrygKoCW9JuA==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-ordered-values@5.1.3(postcss@8.4.32): + resolution: {integrity: sha512-9UO79VUhPwEkzbb3RNpqqghc6lcYej1aveQteWY+4POIwlqkYE21HKWaLDF6lWNuqCobEAyTovVhtI32Rbv2RQ==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + cssnano-utils: 3.1.0(postcss@8.4.32) + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-ordered-values@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-XXbb1O/MW9HdEhnBxitZpPFbIvDgbo9NK4c/5bOfiKpnIGZDoL2xd7/e6jW5DYLsWxBbs+1nZEnVgnjnlFViaA==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + cssnano-utils: 4.0.1(postcss@8.4.32) + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-reduce-idents@5.2.0(postcss@8.4.32): + resolution: {integrity: sha512-BTrLjICoSB6gxbc58D5mdBK8OhXRDqud/zodYfdSi52qvDHdMwk+9kB9xsM8yJThH/sZU5A6QVSmMmaN001gIg==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-reduce-idents@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-GKgyBLS5hMCJC8T36h4IH9u0XhmRHRwLwlxP6xVYbAuxKqn3LezEDDIxnb1/Cu2DXGc20jvWK9VZdCVtYAoTyg==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-reduce-initial@5.1.2(postcss@8.4.32): + resolution: {integrity: sha512-dE/y2XRaqAi6OvjzD22pjTUQ8eOfc6m/natGHgKFBK9DxFmIm69YmaRVQrGgFlEfc1HePIurY0TmDeROK05rIg==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + browserslist: 4.22.2 + caniuse-api: 3.0.0 + postcss: 8.4.32 + dev: false + + /postcss-reduce-initial@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-YGKalhNlCLcjcLvjU5nF8FyeCTkCO5UtvJEt0hrPZVCTtRLSOH4z00T1UntQPj4dUmIYZgMj8qK77JbSX95hSw==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + browserslist: 4.22.2 + caniuse-api: 3.0.0 + postcss: 8.4.32 + dev: false + + /postcss-reduce-transforms@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-2fbdbmgir5AvpW9RLtdONx1QoYG2/EtqpNQbFASDlixBbAYuTcJ0dECwlqNqH7VbaUnEnh8SrxOe2sRIn24XyQ==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-reduce-transforms@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-fUbV81OkUe75JM+VYO1gr/IoA2b/dRiH6HvMwhrIBSUrxq3jNZQZitSnugcTLDi1KkQh1eR/zi+iyxviUNBkcQ==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + dev: false + + /postcss-selector-parser@6.0.15: + resolution: {integrity: sha512-rEYkQOMUCEMhsKbK66tbEU9QVIxbhN18YiniAwA7XQYTVBqrBy+P2p5JcdqsHgKM2zWylp8d7J6eszocfds5Sw==} + engines: {node: '>=4'} + dependencies: + cssesc: 3.0.0 + util-deprecate: 1.0.2 + dev: false + + /postcss-sort-media-queries@4.4.1(postcss@8.4.32): + resolution: {integrity: sha512-QDESFzDDGKgpiIh4GYXsSy6sek2yAwQx1JASl5AxBtU1Lq2JfKBljIPNdil989NcSKRQX1ToiaKphImtBuhXWw==} + engines: {node: '>=10.0.0'} + peerDependencies: + postcss: ^8.4.16 + dependencies: + postcss: 8.4.32 + sort-css-media-queries: 2.1.0 + dev: false + + /postcss-svgo@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-D75KsH1zm5ZrHyxPakAxJWtkyXew5qwS70v56exwvw542d9CRtTo78K0WeFxZB4G7JXKKMbEZtZayTGdIky/eA==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + svgo: 2.8.0 + dev: false + + /postcss-svgo@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-IH5R9SjkTkh0kfFOQDImyy1+mTCb+E830+9SV1O+AaDcoHTvfsvt6WwJeo7KwcHbFnevZVCsXhDmjFiGVuwqFQ==} + engines: {node: ^14 || ^16 || >= 18} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-value-parser: 4.2.0 + svgo: 3.2.0 + dev: false + + /postcss-unique-selectors@5.1.1(postcss@8.4.32): + resolution: {integrity: sha512-5JiODlELrz8L2HwxfPnhOWZYWDxVHWL83ufOv84NrcgipI7TaeRsatAhK4Tr2/ZiYldpK/wBvw5BD3qfaK96GA==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /postcss-unique-selectors@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-8IZGQ94nechdG7Y9Sh9FlIY2b4uS8/k8kdKRX040XHsS3B6d1HrJAkXrBSsSu4SuARruSsUjW3nlSw8BHkaAYQ==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /postcss-value-parser@4.2.0: + resolution: {integrity: sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==} + dev: false + + /postcss-zindex@5.1.0(postcss@8.4.32): + resolution: {integrity: sha512-fgFMf0OtVSBR1va1JNHYgMxYk73yhn/qb4uQDq1DLGYolz8gHCyr/sesEuGUaYs58E3ZJRcpoGuPVoB7Meiq9A==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss-zindex@6.0.1(postcss@8.4.32): + resolution: {integrity: sha512-wQF95TIerYvPlsjwldO7iGP3Z3arhuYRK/gndq4NAdZaEsdUkmQYtRqkrEPMzJOQFBk06wFtzkHZKJoQlqFgXQ==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + postcss: 8.4.32 + dev: false + + /postcss@8.4.32: + resolution: {integrity: sha512-D/kj5JNu6oo2EIy+XL/26JEDTlIbB8hw85G8StOE6L74RQAVVP5rej6wxCNqyMbR4RkPfqvezVbPw81Ngd6Kcw==} + engines: {node: ^10 || ^12 || >=14} + dependencies: + nanoid: 3.3.7 + picocolors: 1.0.0 + source-map-js: 1.0.2 + dev: false + + /preact@10.19.3: + resolution: {integrity: sha512-nHHTeFVBTHRGxJXKkKu5hT8C/YWBkPso4/Gad6xuj5dbptt9iF9NZr9pHbPhBrnT2klheu7mHTxTZ/LjwJiEiQ==} + dev: false + + /pretty-error@4.0.0: + resolution: {integrity: sha512-AoJ5YMAcXKYxKhuJGdcvse+Voc6v1RgnsR3nWcYU7q4t6z0Q6T86sv5Zq8VIRbOWWFpvdGE83LtdSMNd+6Y0xw==} + dependencies: + lodash: 4.17.21 + renderkid: 3.0.0 + dev: false + + /pretty-time@1.1.0: + resolution: {integrity: sha512-28iF6xPQrP8Oa6uxE6a1biz+lWeTOAPKggvjB8HAs6nVMKZwf5bG++632Dx614hIWgUPkgivRfG+a8uAXGTIbA==} + engines: {node: '>=4'} + dev: false + + /prism-react-renderer@2.3.1(react@18.2.0): + resolution: {integrity: sha512-Rdf+HzBLR7KYjzpJ1rSoxT9ioO85nZngQEoFIhL07XhtJHlCU3SOz0GJ6+qvMyQe0Se+BV3qpe6Yd/NmQF5Juw==} + peerDependencies: + react: '>=16.0.0' + dependencies: + '@types/prismjs': 1.26.3 + clsx: 2.1.0 + react: 18.2.0 + dev: false + + /prismjs@1.29.0: + resolution: {integrity: sha512-Kx/1w86q/epKcmte75LNrEoT+lX8pBpavuAbvJWRXar7Hz8jrtF+e3vY751p0R8H9HdArwaCTNDDzHg/ScJK1Q==} + engines: {node: '>=6'} + dev: false + + /process-nextick-args@2.0.1: + resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} + dev: false + + /prompts@2.4.2: + resolution: {integrity: sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==} + engines: {node: '>= 6'} + dependencies: + kleur: 3.0.3 + sisteransi: 1.0.5 + dev: false + + /prop-types@15.8.1: + resolution: {integrity: sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==} + dependencies: + loose-envify: 1.4.0 + object-assign: 4.1.1 + react-is: 16.13.1 + dev: false + + /property-information@6.4.0: + resolution: {integrity: sha512-9t5qARVofg2xQqKtytzt+lZ4d1Qvj8t5B8fEwXK6qOfgRLgH/b13QlgEyDh033NOS31nXeFbYv7CLUDG1CeifQ==} + dev: false + + /proto-list@1.2.4: + resolution: {integrity: sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==} + dev: false + + /proxy-addr@2.0.7: + resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} + engines: {node: '>= 0.10'} + dependencies: + forwarded: 0.2.0 + ipaddr.js: 1.9.1 + dev: false + + /punycode@1.4.1: + resolution: {integrity: sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==} + dev: false + + /punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + dev: false + + /pupa@3.1.0: + resolution: {integrity: sha512-FLpr4flz5xZTSJxSeaheeMKN/EDzMdK7b8PTOC6a5PYFKTucWbdqjgqaEyH0shFiSJrVB1+Qqi4Tk19ccU6Aug==} + engines: {node: '>=12.20'} + dependencies: + escape-goat: 4.0.0 + dev: false + + /qs@6.11.0: + resolution: {integrity: sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==} + engines: {node: '>=0.6'} + dependencies: + side-channel: 1.0.4 + dev: false + + /queue-microtask@1.2.3: + resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} + dev: false + + /queue@6.0.2: + resolution: {integrity: sha512-iHZWu+q3IdFZFX36ro/lKBkSvfkztY5Y7HMiPlOUjhupPcG2JMfst2KKEpu5XndviX/3UhFbRngUPNKtgvtZiA==} + dependencies: + inherits: 2.0.4 + dev: false + + /quick-lru@5.1.1: + resolution: {integrity: sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==} + engines: {node: '>=10'} + dev: false + + /randombytes@2.1.0: + resolution: {integrity: sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==} + dependencies: + safe-buffer: 5.2.1 + dev: false + + /range-parser@1.2.0: + resolution: {integrity: sha512-kA5WQoNVo4t9lNx2kQNFCxKeBl5IbbSNBl1M/tLkw9WCn+hxNBAW5Qh8gdhs63CJnhjJ2zQWFoqPJP2sK1AV5A==} + engines: {node: '>= 0.6'} + dev: false + + /range-parser@1.2.1: + resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==} + engines: {node: '>= 0.6'} + dev: false + + /raw-body@2.5.1: + resolution: {integrity: sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==} + engines: {node: '>= 0.8'} + dependencies: + bytes: 3.1.2 + http-errors: 2.0.0 + iconv-lite: 0.4.24 + unpipe: 1.0.0 + dev: false + + /rc@1.2.8: + resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} + hasBin: true + dependencies: + deep-extend: 0.6.0 + ini: 1.3.8 + minimist: 1.2.8 + strip-json-comments: 2.0.1 + dev: false + + /react-dev-utils@12.0.1(typescript@5.3.3)(webpack@5.89.0): + resolution: {integrity: sha512-84Ivxmr17KjUupyqzFode6xKhjwuEJDROWKJy/BthkL7Wn6NJ8h4WE6k/exAv6ImS+0oZLRRW5j/aINMHyeGeQ==} + engines: {node: '>=14'} + peerDependencies: + typescript: '>=2.7' + webpack: '>=4' + peerDependenciesMeta: + typescript: + optional: true + dependencies: + '@babel/code-frame': 7.23.5 + address: 1.2.2 + browserslist: 4.22.2 + chalk: 4.1.2 + cross-spawn: 7.0.3 + detect-port-alt: 1.1.6 + escape-string-regexp: 4.0.0 + filesize: 8.0.7 + find-up: 5.0.0 + fork-ts-checker-webpack-plugin: 6.5.3(typescript@5.3.3)(webpack@5.89.0) + global-modules: 2.0.0 + globby: 11.1.0 + gzip-size: 6.0.0 + immer: 9.0.21 + is-root: 2.1.0 + loader-utils: 3.2.1 + open: 8.4.2 + pkg-up: 3.1.0 + prompts: 2.4.2 + react-error-overlay: 6.0.11 + recursive-readdir: 2.2.3 + shell-quote: 1.8.1 + strip-ansi: 6.0.1 + text-table: 0.2.0 + typescript: 5.3.3 + webpack: 5.89.0 + transitivePeerDependencies: + - eslint + - supports-color + - vue-template-compiler + dev: false + + /react-dom@18.2.0(react@18.2.0): + resolution: {integrity: sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g==} + peerDependencies: + react: ^18.2.0 + dependencies: + loose-envify: 1.4.0 + react: 18.2.0 + scheduler: 0.23.0 + dev: false + + /react-error-overlay@6.0.11: + resolution: {integrity: sha512-/6UZ2qgEyH2aqzYZgQPxEnz33NJ2gNsnHA2o5+o4wW9bLM/JYQitNP9xPhsXwC08hMMovfGe/8retsdDsczPRg==} + dev: false + + /react-fast-compare@3.2.2: + resolution: {integrity: sha512-nsO+KSNgo1SbJqJEYRE9ERzo7YtYbou/OqjSQKxV7jcKox7+usiUVZOAC+XnDOABXggQTno0Y1CpVnuWEc1boQ==} + dev: false + + /react-helmet-async@1.3.0(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-9jZ57/dAn9t3q6hneQS0wukqC2ENOBgMNVEhb/ZG9ZSxUetzVIw4iAmEU38IaVg3QGYauQPhSeUTuIUtFglWpg==} + peerDependencies: + react: ^16.6.0 || ^17.0.0 || ^18.0.0 + react-dom: ^16.6.0 || ^17.0.0 || ^18.0.0 + dependencies: + '@babel/runtime': 7.23.7 + invariant: 2.2.4 + prop-types: 15.8.1 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + react-fast-compare: 3.2.2 + shallowequal: 1.1.0 + dev: false + + /react-helmet-async@2.0.4(react-dom@18.2.0)(react@18.2.0): + resolution: {integrity: sha512-yxjQMWposw+akRfvpl5+8xejl4JtUlHnEBcji6u8/e6oc7ozT+P9PNTWMhCbz2y9tc5zPegw2BvKjQA+NwdEjQ==} + peerDependencies: + react: ^16.6.0 || ^17.0.0 || ^18.0.0 + react-dom: ^16.6.0 || ^17.0.0 || ^18.0.0 + dependencies: + invariant: 2.2.4 + react: 18.2.0 + react-dom: 18.2.0(react@18.2.0) + react-fast-compare: 3.2.2 + shallowequal: 1.1.0 + dev: false + + /react-is@16.13.1: + resolution: {integrity: sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==} + dev: false + + /react-is@18.2.0: + resolution: {integrity: sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w==} + dev: false + + /react-json-view-lite@1.2.1(react@18.2.0): + resolution: {integrity: sha512-Itc0g86fytOmKZoIoJyGgvNqohWSbh3NXIKNgH6W6FT9PC1ck4xas1tT3Rr/b3UlFXyA9Jjaw9QSXdZy2JwGMQ==} + engines: {node: '>=14'} + peerDependencies: + react: ^16.13.1 || ^17.0.0 || ^18.0.0 + dependencies: + react: 18.2.0 + dev: false + + /react-loadable-ssr-addon-v5-slorber@1.0.1(@docusaurus/react-loadable@5.5.2)(webpack@5.89.0): + resolution: {integrity: sha512-lq3Lyw1lGku8zUEJPDxsNm1AfYHBrO9Y1+olAYwpUJ2IGFBskM0DMKok97A6LWUpHm+o7IvQBOWu9MLenp9Z+A==} + engines: {node: '>=10.13.0'} + peerDependencies: + react-loadable: '*' + webpack: '>=4.41.1 || 5.x' + dependencies: + '@babel/runtime': 7.23.7 + react-loadable: /@docusaurus/react-loadable@5.5.2(react@18.2.0) + webpack: 5.89.0 + dev: false + + /react-markdown@8.0.7(@types/react@18.2.46)(react@18.2.0): + resolution: {integrity: sha512-bvWbzG4MtOU62XqBx3Xx+zB2raaFFsq4mYiAzfjXJMEz2sixgeAfraA3tvzULF02ZdOMUOKTBFFaZJDDrq+BJQ==} + peerDependencies: + '@types/react': '>=16' + react: '>=16' + dependencies: + '@types/hast': 2.3.9 + '@types/prop-types': 15.7.11 + '@types/react': 18.2.46 + '@types/unist': 2.0.10 + comma-separated-tokens: 2.0.3 + hast-util-whitespace: 2.0.1 + prop-types: 15.8.1 + property-information: 6.4.0 + react: 18.2.0 + react-is: 18.2.0 + remark-parse: 10.0.2 + remark-rehype: 10.1.0 + space-separated-tokens: 2.0.2 + style-to-object: 0.4.4 + unified: 10.1.2 + unist-util-visit: 4.1.2 + vfile: 5.3.7 + transitivePeerDependencies: + - supports-color + dev: false + + /react-router-config@5.1.1(react-router@5.3.4)(react@18.2.0): + resolution: {integrity: sha512-DuanZjaD8mQp1ppHjgnnUnyOlqYXZVjnov/JzFhjLEwd3Z4dYjMSnqrEzzGThH47vpCOqPPwJM2FtthLeJ8Pbg==} + peerDependencies: + react: '>=15' + react-router: '>=5' + dependencies: + '@babel/runtime': 7.23.7 + react: 18.2.0 + react-router: 5.3.4(react@18.2.0) + dev: false + + /react-router-dom@5.3.4(react@18.2.0): + resolution: {integrity: sha512-m4EqFMHv/Ih4kpcBCONHbkT68KoAeHN4p3lAGoNryfHi0dMy0kCzEZakiKRsvg5wHZ/JLrLW8o8KomWiz/qbYQ==} + peerDependencies: + react: '>=15' + dependencies: + '@babel/runtime': 7.23.7 + history: 4.10.1 + loose-envify: 1.4.0 + prop-types: 15.8.1 + react: 18.2.0 + react-router: 5.3.4(react@18.2.0) + tiny-invariant: 1.3.1 + tiny-warning: 1.0.3 + dev: false + + /react-router@5.3.3(react@18.2.0): + resolution: {integrity: sha512-mzQGUvS3bM84TnbtMYR8ZjKnuPJ71IjSzR+DE6UkUqvN4czWIqEs17yLL8xkAycv4ev0AiN+IGrWu88vJs/p2w==} + peerDependencies: + react: '>=15' + dependencies: + '@babel/runtime': 7.23.7 + history: 4.10.1 + hoist-non-react-statics: 3.3.2 + loose-envify: 1.4.0 + mini-create-react-context: 0.4.1(prop-types@15.8.1)(react@18.2.0) + path-to-regexp: 1.8.0 + prop-types: 15.8.1 + react: 18.2.0 + react-is: 16.13.1 + tiny-invariant: 1.3.1 + tiny-warning: 1.0.3 + dev: false + + /react-router@5.3.4(react@18.2.0): + resolution: {integrity: sha512-Ys9K+ppnJah3QuaRiLxk+jDWOR1MekYQrlytiXxC1RyfbdsZkS5pvKAzCCr031xHixZwpnsYNT5xysdFHQaYsA==} + peerDependencies: + react: '>=15' + dependencies: + '@babel/runtime': 7.23.7 + history: 4.10.1 + hoist-non-react-statics: 3.3.2 + loose-envify: 1.4.0 + path-to-regexp: 1.8.0 + prop-types: 15.8.1 + react: 18.2.0 + react-is: 16.13.1 + tiny-invariant: 1.3.1 + tiny-warning: 1.0.3 + dev: false + + /react@18.2.0: + resolution: {integrity: sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ==} + engines: {node: '>=0.10.0'} + dependencies: + loose-envify: 1.4.0 + dev: false + + /readable-stream@2.3.8: + resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==} + dependencies: + core-util-is: 1.0.3 + inherits: 2.0.4 + isarray: 1.0.0 + process-nextick-args: 2.0.1 + safe-buffer: 5.1.2 + string_decoder: 1.1.1 + util-deprecate: 1.0.2 + dev: false + + /readable-stream@3.6.2: + resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} + engines: {node: '>= 6'} + dependencies: + inherits: 2.0.4 + string_decoder: 1.3.0 + util-deprecate: 1.0.2 + dev: false + + /readdirp@3.6.0: + resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} + engines: {node: '>=8.10.0'} + dependencies: + picomatch: 2.3.1 + dev: false + + /reading-time@1.5.0: + resolution: {integrity: sha512-onYyVhBNr4CmAxFsKS7bz+uTLRakypIe4R+5A824vBSkQy/hB3fZepoVEf8OVAxzLvK+H/jm9TzpI3ETSm64Kg==} + dev: false + + /rechoir@0.6.2: + resolution: {integrity: sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==} + engines: {node: '>= 0.10'} + dependencies: + resolve: 1.22.8 + dev: false + + /recursive-readdir@2.2.3: + resolution: {integrity: sha512-8HrF5ZsXk5FAH9dgsx3BlUer73nIhuj+9OrQwEbLTPOBzGkL1lsFCR01am+v+0m2Cmbs1nP12hLDl5FA7EszKA==} + engines: {node: '>=6.0.0'} + dependencies: + minimatch: 3.1.2 + dev: false + + /regenerate-unicode-properties@10.1.1: + resolution: {integrity: sha512-X007RyZLsCJVVrjgEFVpLUTZwyOZk3oiL75ZcuYjlIWd6rNJtOjkBwQc5AsRrpbKVkxN6sklw/k/9m2jJYOf8Q==} + engines: {node: '>=4'} + dependencies: + regenerate: 1.4.2 + dev: false + + /regenerate@1.4.2: + resolution: {integrity: sha512-zrceR/XhGYU/d/opr2EKO7aRHUeiBI8qjtfHqADTwZd6Szfy16la6kqD0MIUs5z5hx6AaKa+PixpPrR289+I0A==} + dev: false + + /regenerator-runtime@0.14.1: + resolution: {integrity: sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==} + dev: false + + /regenerator-transform@0.15.2: + resolution: {integrity: sha512-hfMp2BoF0qOk3uc5V20ALGDS2ddjQaLrdl7xrGXvAIow7qeWRM2VA2HuCHkUKk9slq3VwEwLNK3DFBqDfPGYtg==} + dependencies: + '@babel/runtime': 7.23.7 + dev: false + + /regexpu-core@5.3.2: + resolution: {integrity: sha512-RAM5FlZz+Lhmo7db9L298p2vHP5ZywrVXmVXpmAD9GuL5MPH6t9ROw1iA/wfHkQ76Qe7AaPF0nGuim96/IrQMQ==} + engines: {node: '>=4'} + dependencies: + '@babel/regjsgen': 0.8.0 + regenerate: 1.4.2 + regenerate-unicode-properties: 10.1.1 + regjsparser: 0.9.1 + unicode-match-property-ecmascript: 2.0.0 + unicode-match-property-value-ecmascript: 2.1.0 + dev: false + + /registry-auth-token@5.0.2: + resolution: {integrity: sha512-o/3ikDxtXaA59BmZuZrJZDJv8NMDGSj+6j6XaeBmHw8eY1i1qd9+6H+LjVvQXx3HN6aRCGa1cUdJ9RaJZUugnQ==} + engines: {node: '>=14'} + dependencies: + '@pnpm/npm-conf': 2.2.2 + dev: false + + /registry-url@6.0.1: + resolution: {integrity: sha512-+crtS5QjFRqFCoQmvGduwYWEBng99ZvmFvF+cUJkGYF1L1BfU8C6Zp9T7f5vPAwyLkUExpvK+ANVZmGU49qi4Q==} + engines: {node: '>=12'} + dependencies: + rc: 1.2.8 + dev: false + + /regjsparser@0.9.1: + resolution: {integrity: sha512-dQUtn90WanSNl+7mQKcXAgZxvUe7Z0SqXlgzv0za4LwiUhyzBC58yQO3liFoUgu8GiJVInAhJjkj1N0EtQ5nkQ==} + hasBin: true + dependencies: + jsesc: 0.5.0 + dev: false + + /rehype-raw@7.0.0: + resolution: {integrity: sha512-/aE8hCfKlQeA8LmyeyQvQF3eBiLRGNlfBJEvWH7ivp9sBqs7TNqBL5X3v157rM4IFETqDnIOO+z5M/biZbo9Ww==} + dependencies: + '@types/hast': 3.0.3 + hast-util-raw: 9.0.1 + vfile: 6.0.1 + dev: false + + /relateurl@0.2.7: + resolution: {integrity: sha512-G08Dxvm4iDN3MLM0EsP62EDV9IuhXPR6blNz6Utcp7zyV3tr4HVNINt6MpaRWbxoOHT3Q7YN2P+jaHX8vUbgog==} + engines: {node: '>= 0.10'} + dev: false + + /remark-directive@3.0.0: + resolution: {integrity: sha512-l1UyWJ6Eg1VPU7Hm/9tt0zKtReJQNOA4+iDMAxTyZNWnJnFlbS/7zhiel/rogTLQ2vMYwDzSJa4BiVNqGlqIMA==} + dependencies: + '@types/mdast': 4.0.3 + mdast-util-directive: 3.0.0 + micromark-extension-directive: 3.0.0 + unified: 11.0.4 + transitivePeerDependencies: + - supports-color + dev: false + + /remark-emoji@4.0.1: + resolution: {integrity: sha512-fHdvsTR1dHkWKev9eNyhTo4EFwbUvJ8ka9SgeWkMPYFX4WoI7ViVBms3PjlQYgw5TLvNQso3GUB/b/8t3yo+dg==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + dependencies: + '@types/mdast': 4.0.3 + emoticon: 4.0.1 + mdast-util-find-and-replace: 3.0.1 + node-emoji: 2.1.3 + unified: 11.0.4 + dev: false + + /remark-frontmatter@5.0.0: + resolution: {integrity: sha512-XTFYvNASMe5iPN0719nPrdItC9aU0ssC4v14mH1BCi1u0n1gAocqcujWUrByftZTbLhRtiKRyjYTSIOcr69UVQ==} + dependencies: + '@types/mdast': 4.0.3 + mdast-util-frontmatter: 2.0.1 + micromark-extension-frontmatter: 2.0.0 + unified: 11.0.4 + transitivePeerDependencies: + - supports-color + dev: false + + /remark-gfm@4.0.0: + resolution: {integrity: sha512-U92vJgBPkbw4Zfu/IiW2oTZLSL3Zpv+uI7My2eq8JxKgqraFdU8YUGicEJCEgSbeaG+QDFqIcwwfMTOEelPxuA==} + dependencies: + '@types/mdast': 4.0.3 + mdast-util-gfm: 3.0.0 + micromark-extension-gfm: 3.0.0 + remark-parse: 11.0.0 + remark-stringify: 11.0.0 + unified: 11.0.4 + transitivePeerDependencies: + - supports-color + dev: false + + /remark-mdx@3.0.0: + resolution: {integrity: sha512-O7yfjuC6ra3NHPbRVxfflafAj3LTwx3b73aBvkEFU5z4PsD6FD4vrqJAkE5iNGLz71GdjXfgRqm3SQ0h0VuE7g==} + dependencies: + mdast-util-mdx: 3.0.0 + micromark-extension-mdxjs: 3.0.0 + transitivePeerDependencies: + - supports-color + dev: false + + /remark-parse@10.0.2: + resolution: {integrity: sha512-3ydxgHa/ZQzG8LvC7jTXccARYDcRld3VfcgIIFs7bI6vbRSxJJmzgLEIIoYKyrfhaY+ujuWaf/PJiMZXoiCXgw==} + dependencies: + '@types/mdast': 3.0.15 + mdast-util-from-markdown: 1.3.1 + unified: 10.1.2 + transitivePeerDependencies: + - supports-color + dev: false + + /remark-parse@11.0.0: + resolution: {integrity: sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA==} + dependencies: + '@types/mdast': 4.0.3 + mdast-util-from-markdown: 2.0.0 + micromark-util-types: 2.0.0 + unified: 11.0.4 + transitivePeerDependencies: + - supports-color + dev: false + + /remark-rehype@10.1.0: + resolution: {integrity: sha512-EFmR5zppdBp0WQeDVZ/b66CWJipB2q2VLNFMabzDSGR66Z2fQii83G5gTBbgGEnEEA0QRussvrFHxk1HWGJskw==} + dependencies: + '@types/hast': 2.3.9 + '@types/mdast': 3.0.15 + mdast-util-to-hast: 12.3.0 + unified: 10.1.2 + dev: false + + /remark-rehype@11.0.0: + resolution: {integrity: sha512-vx8x2MDMcxuE4lBmQ46zYUDfcFMmvg80WYX+UNLeG6ixjdCCLcw1lrgAukwBTuOFsS78eoAedHGn9sNM0w7TPw==} + dependencies: + '@types/hast': 3.0.3 + '@types/mdast': 4.0.3 + mdast-util-to-hast: 13.0.2 + unified: 11.0.4 + vfile: 6.0.1 + dev: false + + /remark-stringify@11.0.0: + resolution: {integrity: sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==} + dependencies: + '@types/mdast': 4.0.3 + mdast-util-to-markdown: 2.1.0 + unified: 11.0.4 + dev: false + + /renderkid@3.0.0: + resolution: {integrity: sha512-q/7VIQA8lmM1hF+jn+sFSPWGlMkSAeNYcPLmDQx2zzuiDfaLrOmumR8iaUKlenFgh0XRPIUeSPlH3A+AW3Z5pg==} + dependencies: + css-select: 4.3.0 + dom-converter: 0.2.0 + htmlparser2: 6.1.0 + lodash: 4.17.21 + strip-ansi: 6.0.1 + dev: false + + /require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + dev: false + + /require-like@0.1.2: + resolution: {integrity: sha512-oyrU88skkMtDdauHDuKVrgR+zuItqr6/c//FXzvmxRGMexSDc6hNvJInGW3LL46n+8b50RykrvwSUIIQH2LQ5A==} + dev: false + + /requires-port@1.0.0: + resolution: {integrity: sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==} + dev: false + + /resolve-alpn@1.2.1: + resolution: {integrity: sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==} + dev: false + + /resolve-from@4.0.0: + resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} + engines: {node: '>=4'} + dev: false + + /resolve-pathname@3.0.0: + resolution: {integrity: sha512-C7rARubxI8bXFNB/hqcp/4iUeIXJhJZvFPFPiSPRnhU5UPxzMFIl+2E6yY6c4k9giDJAhtV+enfA+G89N6Csng==} + dev: false + + /resolve@1.22.8: + resolution: {integrity: sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==} + hasBin: true + dependencies: + is-core-module: 2.13.1 + path-parse: 1.0.7 + supports-preserve-symlinks-flag: 1.0.0 + dev: false + + /responselike@3.0.0: + resolution: {integrity: sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==} + engines: {node: '>=14.16'} + dependencies: + lowercase-keys: 3.0.0 + dev: false + + /retry@0.13.1: + resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} + engines: {node: '>= 4'} + dev: false + + /reusify@1.0.4: + resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} + engines: {iojs: '>=1.0.0', node: '>=0.10.0'} + dev: false + + /rimraf@3.0.2: + resolution: {integrity: sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==} + hasBin: true + dependencies: + glob: 7.2.3 + dev: false + + /robust-predicates@3.0.2: + resolution: {integrity: sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg==} + dev: false + + /rtl-detect@1.1.2: + resolution: {integrity: sha512-PGMBq03+TTG/p/cRB7HCLKJ1MgDIi07+QU1faSjiYRfmY5UsAttV9Hs08jDAHVwcOwmVLcSJkpwyfXszVjWfIQ==} + dev: false + + /rtlcss@4.1.1: + resolution: {integrity: sha512-/oVHgBtnPNcggP2aVXQjSy6N1mMAfHg4GSag0QtZBlD5bdDgAHwr4pydqJGd+SUCu9260+Pjqbjwtvu7EMH1KQ==} + engines: {node: '>=12.0.0'} + hasBin: true + dependencies: + escalade: 3.1.1 + picocolors: 1.0.0 + postcss: 8.4.32 + strip-json-comments: 3.1.1 + dev: false + + /run-parallel@1.2.0: + resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} + dependencies: + queue-microtask: 1.2.3 + dev: false + + /rw@1.3.3: + resolution: {integrity: sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==} + dev: false + + /sade@1.8.1: + resolution: {integrity: sha512-xal3CZX1Xlo/k4ApwCFrHVACi9fBqJ7V+mwhBsuf/1IOKbBy098Fex+Wa/5QMubw09pSZ/u8EY8PWgevJsXp1A==} + engines: {node: '>=6'} + dependencies: + mri: 1.2.0 + dev: false + + /safe-buffer@5.1.2: + resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} + dev: false + + /safe-buffer@5.2.1: + resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + dev: false + + /safer-buffer@2.1.2: + resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + dev: false + + /sax@1.3.0: + resolution: {integrity: sha512-0s+oAmw9zLl1V1cS9BtZN7JAd0cW5e0QH4W3LWEK6a4LaLEA2OTpGYWDY+6XasBLtz6wkm3u1xRw95mRuJ59WA==} + dev: false + + /scheduler@0.23.0: + resolution: {integrity: sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==} + dependencies: + loose-envify: 1.4.0 + dev: false + + /schema-utils@2.7.0: + resolution: {integrity: sha512-0ilKFI6QQF5nxDZLFn2dMjvc4hjg/Wkg7rHd3jK6/A4a1Hl9VFdQWvgB1UMGoU94pad1P/8N7fMcEnLnSiju8A==} + engines: {node: '>= 8.9.0'} + dependencies: + '@types/json-schema': 7.0.15 + ajv: 6.12.6 + ajv-keywords: 3.5.2(ajv@6.12.6) + dev: false + + /schema-utils@3.3.0: + resolution: {integrity: sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg==} + engines: {node: '>= 10.13.0'} + dependencies: + '@types/json-schema': 7.0.15 + ajv: 6.12.6 + ajv-keywords: 3.5.2(ajv@6.12.6) + dev: false + + /schema-utils@4.2.0: + resolution: {integrity: sha512-L0jRsrPpjdckP3oPug3/VxNKt2trR8TcabrM6FOAAlvC/9Phcmm+cuAgTlxBqdBR1WJx7Naj9WHw+aOmheSVbw==} + engines: {node: '>= 12.13.0'} + dependencies: + '@types/json-schema': 7.0.15 + ajv: 8.12.0 + ajv-formats: 2.1.1(ajv@8.12.0) + ajv-keywords: 5.1.0(ajv@8.12.0) + dev: false + + /search-insights@2.13.0: + resolution: {integrity: sha512-Orrsjf9trHHxFRuo9/rzm0KIWmgzE8RMlZMzuhZOJ01Rnz3D0YBAe+V6473t6/H6c7irs6Lt48brULAiRWb3Vw==} + dev: false + + /section-matter@1.0.0: + resolution: {integrity: sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==} + engines: {node: '>=4'} + dependencies: + extend-shallow: 2.0.1 + kind-of: 6.0.3 + dev: false + + /select-hose@2.0.0: + resolution: {integrity: sha512-mEugaLK+YfkijB4fx0e6kImuJdCIt2LxCRcbEYPqRGCs4F2ogyfZU5IAZRdjCP8JPq2AtdNoC/Dux63d9Kiryg==} + dev: false + + /selfsigned@2.4.1: + resolution: {integrity: sha512-th5B4L2U+eGLq1TVh7zNRGBapioSORUeymIydxgFpwww9d2qyKvtuPU2jJuHvYAwwqi2Y596QBL3eEqcPEYL8Q==} + engines: {node: '>=10'} + dependencies: + '@types/node-forge': 1.3.11 + node-forge: 1.3.1 + dev: false + + /semver-diff@4.0.0: + resolution: {integrity: sha512-0Ju4+6A8iOnpL/Thra7dZsSlOHYAHIeMxfhWQRI1/VLcT3WDBZKKtQt/QkBOsiIN9ZpuvHE6cGZ0x4glCMmfiA==} + engines: {node: '>=12'} + dependencies: + semver: 7.5.4 + dev: false + + /semver@6.3.1: + resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} + hasBin: true + dev: false + + /semver@7.5.4: + resolution: {integrity: sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==} + engines: {node: '>=10'} + hasBin: true + dependencies: + lru-cache: 6.0.0 + dev: false + + /send@0.18.0: + resolution: {integrity: sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==} + engines: {node: '>= 0.8.0'} + dependencies: + debug: 2.6.9 + depd: 2.0.0 + destroy: 1.2.0 + encodeurl: 1.0.2 + escape-html: 1.0.3 + etag: 1.8.1 + fresh: 0.5.2 + http-errors: 2.0.0 + mime: 1.6.0 + ms: 2.1.3 + on-finished: 2.4.1 + range-parser: 1.2.1 + statuses: 2.0.1 + transitivePeerDependencies: + - supports-color + dev: false + + /serialize-javascript@6.0.1: + resolution: {integrity: sha512-owoXEFjWRllis8/M1Q+Cw5k8ZH40e3zhp/ovX+Xr/vi1qj6QesbyXXViFbpNvWvPNAD62SutwEXavefrLJWj7w==} + dependencies: + randombytes: 2.1.0 + dev: false + + /serve-handler@6.1.5: + resolution: {integrity: sha512-ijPFle6Hwe8zfmBxJdE+5fta53fdIY0lHISJvuikXB3VYFafRjMRpOffSPvCYsbKyBA7pvy9oYr/BT1O3EArlg==} + dependencies: + bytes: 3.0.0 + content-disposition: 0.5.2 + fast-url-parser: 1.1.3 + mime-types: 2.1.18 + minimatch: 3.1.2 + path-is-inside: 1.0.2 + path-to-regexp: 2.2.1 + range-parser: 1.2.0 + dev: false + + /serve-index@1.9.1: + resolution: {integrity: sha512-pXHfKNP4qujrtteMrSBb0rc8HJ9Ms/GrXwcUtUtD5s4ewDJI8bT3Cz2zTVRMKtri49pLx2e0Ya8ziP5Ya2pZZw==} + engines: {node: '>= 0.8.0'} + dependencies: + accepts: 1.3.8 + batch: 0.6.1 + debug: 2.6.9 + escape-html: 1.0.3 + http-errors: 1.6.3 + mime-types: 2.1.35 + parseurl: 1.3.3 + transitivePeerDependencies: + - supports-color + dev: false + + /serve-static@1.15.0: + resolution: {integrity: sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==} + engines: {node: '>= 0.8.0'} + dependencies: + encodeurl: 1.0.2 + escape-html: 1.0.3 + parseurl: 1.3.3 + send: 0.18.0 + transitivePeerDependencies: + - supports-color + dev: false + + /set-function-length@1.1.1: + resolution: {integrity: sha512-VoaqjbBJKiWtg4yRcKBQ7g7wnGnLV3M8oLvVWwOk2PdYY6PEFegR1vezXR0tw6fZGF9csVakIRjrJiy2veSBFQ==} + engines: {node: '>= 0.4'} + dependencies: + define-data-property: 1.1.1 + get-intrinsic: 1.2.2 + gopd: 1.0.1 + has-property-descriptors: 1.0.1 + dev: false + + /setprototypeof@1.1.0: + resolution: {integrity: sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==} + dev: false + + /setprototypeof@1.2.0: + resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + dev: false + + /shallow-clone@3.0.1: + resolution: {integrity: sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==} + engines: {node: '>=8'} + dependencies: + kind-of: 6.0.3 + dev: false + + /shallowequal@1.1.0: + resolution: {integrity: sha512-y0m1JoUZSlPAjXVtPPW70aZWfIL/dSP7AFkRnniLCrK/8MDKog3TySTBmckD+RObVxH0v4Tox67+F14PdED2oQ==} + dev: false + + /shebang-command@2.0.0: + resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} + engines: {node: '>=8'} + dependencies: + shebang-regex: 3.0.0 + dev: false + + /shebang-regex@3.0.0: + resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} + engines: {node: '>=8'} + dev: false + + /shell-quote@1.8.1: + resolution: {integrity: sha512-6j1W9l1iAs/4xYBI1SYOVZyFcCis9b4KCLQ8fgAGG07QvzaRLVVRQvAy85yNmmZSjYjg4MWh4gNvlPujU/5LpA==} + dev: false + + /shelljs@0.8.5: + resolution: {integrity: sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow==} + engines: {node: '>=4'} + hasBin: true + dependencies: + glob: 7.2.3 + interpret: 1.4.0 + rechoir: 0.6.2 + dev: false + + /side-channel@1.0.4: + resolution: {integrity: sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==} + dependencies: + call-bind: 1.0.5 + get-intrinsic: 1.2.2 + object-inspect: 1.13.1 + dev: false + + /signal-exit@3.0.7: + resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} + dev: false + + /sirv@2.0.4: + resolution: {integrity: sha512-94Bdh3cC2PKrbgSOUqTiGPWVZeSiXfKOVZNJniWoqrWrRkB1CJzBU3NEbiTsPcYy1lDsANA/THzS+9WBiy5nfQ==} + engines: {node: '>= 10'} + dependencies: + '@polka/url': 1.0.0-next.24 + mrmime: 2.0.0 + totalist: 3.0.1 + dev: false + + /sisteransi@1.0.5: + resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==} + dev: false + + /sitemap@7.1.1: + resolution: {integrity: sha512-mK3aFtjz4VdJN0igpIJrinf3EO8U8mxOPsTBzSsy06UtjZQJ3YY3o3Xa7zSc5nMqcMrRwlChHZ18Kxg0caiPBg==} + engines: {node: '>=12.0.0', npm: '>=5.6.0'} + hasBin: true + dependencies: + '@types/node': 17.0.45 + '@types/sax': 1.2.7 + arg: 5.0.2 + sax: 1.3.0 + dev: false + + /skin-tone@2.0.0: + resolution: {integrity: sha512-kUMbT1oBJCpgrnKoSr0o6wPtvRWT9W9UKvGLwfJYO2WuahZRHOpEyL1ckyMGgMWh0UdpmaoFqKKD29WTomNEGA==} + engines: {node: '>=8'} + dependencies: + unicode-emoji-modifier-base: 1.0.0 + dev: false + + /slash@3.0.0: + resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} + engines: {node: '>=8'} + dev: false + + /slash@4.0.0: + resolution: {integrity: sha512-3dOsAHXXUkQTpOYcoAxLIorMTp4gIQr5IW3iVb7A7lFIp0VHhnynm9izx6TssdrIcVIESAlVjtnO2K8bg+Coew==} + engines: {node: '>=12'} + dev: false + + /sockjs@0.3.24: + resolution: {integrity: sha512-GJgLTZ7vYb/JtPSSZ10hsOYIvEYsjbNU+zPdIHcUaWVNUEPivzxku31865sSSud0Da0W4lEeOPlmw93zLQchuQ==} + dependencies: + faye-websocket: 0.11.4 + uuid: 8.3.2 + websocket-driver: 0.7.4 + dev: false + + /sort-css-media-queries@2.1.0: + resolution: {integrity: sha512-IeWvo8NkNiY2vVYdPa27MCQiR0MN0M80johAYFVxWWXQ44KU84WNxjslwBHmc/7ZL2ccwkM7/e6S5aiKZXm7jA==} + engines: {node: '>= 6.3.0'} + dev: false + + /source-map-js@1.0.2: + resolution: {integrity: sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==} + engines: {node: '>=0.10.0'} + dev: false + + /source-map-support@0.5.21: + resolution: {integrity: sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==} + dependencies: + buffer-from: 1.1.2 + source-map: 0.6.1 + dev: false + + /source-map@0.6.1: + resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} + engines: {node: '>=0.10.0'} + dev: false + + /source-map@0.7.4: + resolution: {integrity: sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==} + engines: {node: '>= 8'} + dev: false + + /space-separated-tokens@2.0.2: + resolution: {integrity: sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==} + dev: false + + /spdy-transport@3.0.0: + resolution: {integrity: sha512-hsLVFE5SjA6TCisWeJXFKniGGOpBgMLmerfO2aCyCU5s7nJ/rpAepqmFifv/GCbSbueEeAJJnmSQ2rKC/g8Fcw==} + dependencies: + debug: 4.3.4 + detect-node: 2.1.0 + hpack.js: 2.1.6 + obuf: 1.1.2 + readable-stream: 3.6.2 + wbuf: 1.7.3 + transitivePeerDependencies: + - supports-color + dev: false + + /spdy@4.0.2: + resolution: {integrity: sha512-r46gZQZQV+Kl9oItvl1JZZqJKGr+oEkB08A6BzkiR7593/7IbtuncXHd2YoYeTsG4157ZssMu9KYvUHLcjcDoA==} + engines: {node: '>=6.0.0'} + dependencies: + debug: 4.3.4 + handle-thing: 2.0.1 + http-deceiver: 1.2.7 + select-hose: 2.0.0 + spdy-transport: 3.0.0 + transitivePeerDependencies: + - supports-color + dev: false + + /sprintf-js@1.0.3: + resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} + dev: false + + /srcset@4.0.0: + resolution: {integrity: sha512-wvLeHgcVHKO8Sc/H/5lkGreJQVeYMm9rlmt8PuR1xE31rIuXhuzznUUqAt8MqLhB3MqJdFzlNAfpcWnxiFUcPw==} + engines: {node: '>=12'} + dev: false + + /stable@0.1.8: + resolution: {integrity: sha512-ji9qxRnOVfcuLDySj9qzhGSEFVobyt1kIOSkj1qZzYLzq7Tos/oUUWvotUPQLlrsidqsK6tBH89Bc9kL5zHA6w==} + deprecated: 'Modern JS already guarantees Array#sort() is a stable sort, so this library is deprecated. See the compatibility table on MDN: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/sort#browser_compatibility' + dev: false + + /statuses@1.5.0: + resolution: {integrity: sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==} + engines: {node: '>= 0.6'} + dev: false + + /statuses@2.0.1: + resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==} + engines: {node: '>= 0.8'} + dev: false + + /std-env@3.7.0: + resolution: {integrity: sha512-JPbdCEQLj1w5GilpiHAx3qJvFndqybBysA3qUOnznweH4QbNYUsW/ea8QzSrnh0vNsezMMw5bcVool8lM0gwzg==} + dev: false + + /string-width@4.2.3: + resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} + engines: {node: '>=8'} + dependencies: + emoji-regex: 8.0.0 + is-fullwidth-code-point: 3.0.0 + strip-ansi: 6.0.1 + dev: false + + /string-width@5.1.2: + resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} + engines: {node: '>=12'} + dependencies: + eastasianwidth: 0.2.0 + emoji-regex: 9.2.2 + strip-ansi: 7.1.0 + dev: false + + /string_decoder@1.1.1: + resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==} + dependencies: + safe-buffer: 5.1.2 + dev: false + + /string_decoder@1.3.0: + resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + dependencies: + safe-buffer: 5.2.1 + dev: false + + /stringify-entities@4.0.3: + resolution: {integrity: sha512-BP9nNHMhhfcMbiuQKCqMjhDP5yBCAxsPu4pHFFzJ6Alo9dZgY4VLDPutXqIjpRiMoKdp7Av85Gr73Q5uH9k7+g==} + dependencies: + character-entities-html4: 2.1.0 + character-entities-legacy: 3.0.0 + dev: false + + /stringify-object@3.3.0: + resolution: {integrity: sha512-rHqiFh1elqCQ9WPLIC8I0Q/g/wj5J1eMkyoiD6eoQApWHP0FtlK7rqnhmabL5VUY9JQCcqwwvlOaSuutekgyrw==} + engines: {node: '>=4'} + dependencies: + get-own-enumerable-property-symbols: 3.0.2 + is-obj: 1.0.1 + is-regexp: 1.0.0 + dev: false + + /strip-ansi@6.0.1: + resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} + engines: {node: '>=8'} + dependencies: + ansi-regex: 5.0.1 + dev: false + + /strip-ansi@7.1.0: + resolution: {integrity: sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==} + engines: {node: '>=12'} + dependencies: + ansi-regex: 6.0.1 + dev: false + + /strip-bom-string@1.0.0: + resolution: {integrity: sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g==} + engines: {node: '>=0.10.0'} + dev: false + + /strip-final-newline@2.0.0: + resolution: {integrity: sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==} + engines: {node: '>=6'} + dev: false + + /strip-json-comments@2.0.1: + resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} + engines: {node: '>=0.10.0'} + dev: false + + /strip-json-comments@3.1.1: + resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} + engines: {node: '>=8'} + dev: false + + /style-to-object@0.4.4: + resolution: {integrity: sha512-HYNoHZa2GorYNyqiCaBgsxvcJIn7OHq6inEga+E6Ke3m5JkoqpQbnFssk4jwe+K7AhGa2fcha4wSOf1Kn01dMg==} + dependencies: + inline-style-parser: 0.1.1 + dev: false + + /style-to-object@1.0.5: + resolution: {integrity: sha512-rDRwHtoDD3UMMrmZ6BzOW0naTjMsVZLIjsGleSKS/0Oz+cgCfAPRspaqJuE8rDzpKha/nEvnM0IF4seEAZUTKQ==} + dependencies: + inline-style-parser: 0.2.2 + dev: false + + /stylehacks@5.1.1(postcss@8.4.32): + resolution: {integrity: sha512-sBpcd5Hx7G6seo7b1LkpttvTz7ikD0LlH5RmdcBNb6fFR0Fl7LQwHDFr300q4cwUqi+IYrFGmsIHieMBfnN/Bw==} + engines: {node: ^10 || ^12 || >=14.0} + peerDependencies: + postcss: ^8.2.15 + dependencies: + browserslist: 4.22.2 + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /stylehacks@6.0.2(postcss@8.4.32): + resolution: {integrity: sha512-00zvJGnCu64EpMjX8b5iCZ3us2Ptyw8+toEkb92VdmkEaRaSGBNKAoK6aWZckhXxmQP8zWiTaFaiMGIU8Ve8sg==} + engines: {node: ^14 || ^16 || >=18.0} + peerDependencies: + postcss: ^8.4.31 + dependencies: + browserslist: 4.22.2 + postcss: 8.4.32 + postcss-selector-parser: 6.0.15 + dev: false + + /stylis@4.3.1: + resolution: {integrity: sha512-EQepAV+wMsIaGVGX1RECzgrcqRRU/0sYOHkeLsZ3fzHaHXZy4DaOOX0vOlGQdlsjkh3mFHAIlVimpwAs4dslyQ==} + dev: false + + /supports-color@5.5.0: + resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} + engines: {node: '>=4'} + dependencies: + has-flag: 3.0.0 + dev: false + + /supports-color@7.2.0: + resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} + engines: {node: '>=8'} + dependencies: + has-flag: 4.0.0 + dev: false + + /supports-color@8.1.1: + resolution: {integrity: sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==} + engines: {node: '>=10'} + dependencies: + has-flag: 4.0.0 + dev: false + + /supports-preserve-symlinks-flag@1.0.0: + resolution: {integrity: sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==} + engines: {node: '>= 0.4'} + dev: false + + /svg-parser@2.0.4: + resolution: {integrity: sha512-e4hG1hRwoOdRb37cIMSgzNsxyzKfayW6VOflrwvR+/bzrkyxY/31WkbgnQpgtrNp1SdpJvpUAGTa/ZoiPNDuRQ==} + dev: false + + /svgo@2.8.0: + resolution: {integrity: sha512-+N/Q9kV1+F+UeWYoSiULYo4xYSDQlTgb+ayMobAXPwMnLvop7oxKMo9OzIrX5x3eS4L4f2UHhc9axXwY8DpChg==} + engines: {node: '>=10.13.0'} + hasBin: true + dependencies: + '@trysound/sax': 0.2.0 + commander: 7.2.0 + css-select: 4.3.0 + css-tree: 1.1.3 + csso: 4.2.0 + picocolors: 1.0.0 + stable: 0.1.8 + dev: false + + /svgo@3.2.0: + resolution: {integrity: sha512-4PP6CMW/V7l/GmKRKzsLR8xxjdHTV4IMvhTnpuHwwBazSIlw5W/5SmPjN8Dwyt7lKbSJrRDgp4t9ph0HgChFBQ==} + engines: {node: '>=14.0.0'} + hasBin: true + dependencies: + '@trysound/sax': 0.2.0 + commander: 7.2.0 + css-select: 5.1.0 + css-tree: 2.3.1 + css-what: 6.1.0 + csso: 5.0.5 + picocolors: 1.0.0 + dev: false + + /tapable@1.1.3: + resolution: {integrity: sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==} + engines: {node: '>=6'} + dev: false + + /tapable@2.2.1: + resolution: {integrity: sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==} + engines: {node: '>=6'} + dev: false + + /terser-webpack-plugin@5.3.10(webpack@5.89.0): + resolution: {integrity: sha512-BKFPWlPDndPs+NGGCr1U59t0XScL5317Y0UReNrHaw9/FwhPENlq6bfgs+4yPfyP51vqC1bQ4rp1EfXW5ZSH9w==} + engines: {node: '>= 10.13.0'} + peerDependencies: + '@swc/core': '*' + esbuild: '*' + uglify-js: '*' + webpack: ^5.1.0 + peerDependenciesMeta: + '@swc/core': + optional: true + esbuild: + optional: true + uglify-js: + optional: true + dependencies: + '@jridgewell/trace-mapping': 0.3.20 + jest-worker: 27.5.1 + schema-utils: 3.3.0 + serialize-javascript: 6.0.1 + terser: 5.26.0 + webpack: 5.89.0 + dev: false + + /terser@5.26.0: + resolution: {integrity: sha512-dytTGoE2oHgbNV9nTzgBEPaqAWvcJNl66VZ0BkJqlvp71IjO8CxdBx/ykCNb47cLnCmCvRZ6ZR0tLkqvZCdVBQ==} + engines: {node: '>=10'} + hasBin: true + dependencies: + '@jridgewell/source-map': 0.3.5 + acorn: 8.11.3 + commander: 2.20.3 + source-map-support: 0.5.21 + dev: false + + /text-table@0.2.0: + resolution: {integrity: sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==} + dev: false + + /thunky@1.1.0: + resolution: {integrity: sha512-eHY7nBftgThBqOyHGVN+l8gF0BucP09fMo0oO/Lb0w1OF80dJv+lDVpXG60WMQvkcxAkNybKsrEIE3ZtKGmPrA==} + dev: false + + /tiny-invariant@1.3.1: + resolution: {integrity: sha512-AD5ih2NlSssTCwsMznbvwMZpJ1cbhkGd2uueNxzv2jDlEeZdU04JQfRnggJQ8DrcVBGjAsCKwFBbDlVNtEMlzw==} + dev: false + + /tiny-warning@1.0.3: + resolution: {integrity: sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA==} + dev: false + + /to-fast-properties@2.0.0: + resolution: {integrity: sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==} + engines: {node: '>=4'} + dev: false + + /to-regex-range@5.0.1: + resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} + engines: {node: '>=8.0'} + dependencies: + is-number: 7.0.0 + dev: false + + /toidentifier@1.0.1: + resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} + engines: {node: '>=0.6'} + dev: false + + /totalist@3.0.1: + resolution: {integrity: sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==} + engines: {node: '>=6'} + dev: false + + /trim-lines@3.0.1: + resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} + dev: false + + /trim@0.0.3: + resolution: {integrity: sha512-h82ywcYhHK7veeelXrCScdH7HkWfbIT1D/CgYO+nmDarz3SGNssVBMws6jU16Ga60AJCRAvPV6w6RLuNerQqjg==} + deprecated: Use String.prototype.trim() instead + dev: false + + /trough@2.1.0: + resolution: {integrity: sha512-AqTiAOLcj85xS7vQ8QkAV41hPDIJ71XJB4RCUrzo/1GM2CQwhkJGaf9Hgr7BOugMRpgGUrqRg/DrBDl4H40+8g==} + dev: false + + /ts-dedent@2.2.0: + resolution: {integrity: sha512-q5W7tVM71e2xjHZTlgfTDoPF/SmqKG5hddq9SzR49CH2hayqRKJtQ4mtRlSxKaJlR/+9rEM+mnBHf7I2/BQcpQ==} + engines: {node: '>=6.10'} + dev: false + + /tslib@2.6.2: + resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==} + dev: false + + /type-fest@1.4.0: + resolution: {integrity: sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==} + engines: {node: '>=10'} + dev: false + + /type-fest@2.19.0: + resolution: {integrity: sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA==} + engines: {node: '>=12.20'} + dev: false + + /type-is@1.6.18: + resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} + engines: {node: '>= 0.6'} + dependencies: + media-typer: 0.3.0 + mime-types: 2.1.35 + dev: false + + /typedarray-to-buffer@3.1.5: + resolution: {integrity: sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==} + dependencies: + is-typedarray: 1.0.0 + dev: false + + /typescript@5.3.3: + resolution: {integrity: sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==} + engines: {node: '>=14.17'} + hasBin: true + dev: false + + /undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + dev: false + + /unicode-canonical-property-names-ecmascript@2.0.0: + resolution: {integrity: sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==} + engines: {node: '>=4'} + dev: false + + /unicode-emoji-modifier-base@1.0.0: + resolution: {integrity: sha512-yLSH4py7oFH3oG/9K+XWrz1pSi3dfUrWEnInbxMfArOfc1+33BlGPQtLsOYwvdMy11AwUBetYuaRxSPqgkq+8g==} + engines: {node: '>=4'} + dev: false + + /unicode-match-property-ecmascript@2.0.0: + resolution: {integrity: sha512-5kaZCrbp5mmbz5ulBkDkbY0SsPOjKqVS35VpL9ulMPfSl0J0Xsm+9Evphv9CoIZFwre7aJoa94AY6seMKGVN5Q==} + engines: {node: '>=4'} + dependencies: + unicode-canonical-property-names-ecmascript: 2.0.0 + unicode-property-aliases-ecmascript: 2.1.0 + dev: false + + /unicode-match-property-value-ecmascript@2.1.0: + resolution: {integrity: sha512-qxkjQt6qjg/mYscYMC0XKRn3Rh0wFPlfxB0xkt9CfyTvpX1Ra0+rAmdX2QyAobptSEvuy4RtpPRui6XkV+8wjA==} + engines: {node: '>=4'} + dev: false + + /unicode-property-aliases-ecmascript@2.1.0: + resolution: {integrity: sha512-6t3foTQI9qne+OZoVQB/8x8rk2k1eVy1gRXhV3oFQ5T6R1dqQ1xtin3XqSlx3+ATBkliTaR/hHyJBm+LVPNM8w==} + engines: {node: '>=4'} + dev: false + + /unified@10.1.2: + resolution: {integrity: sha512-pUSWAi/RAnVy1Pif2kAoeWNBa3JVrx0MId2LASj8G+7AiHWoKZNTomq6LG326T68U7/e263X6fTdcXIy7XnF7Q==} + dependencies: + '@types/unist': 2.0.10 + bail: 2.0.2 + extend: 3.0.2 + is-buffer: 2.0.5 + is-plain-obj: 4.1.0 + trough: 2.1.0 + vfile: 5.3.7 + dev: false + + /unified@11.0.4: + resolution: {integrity: sha512-apMPnyLjAX+ty4OrNap7yumyVAMlKx5IWU2wlzzUdYJO9A8f1p9m/gywF/GM2ZDFcjQPrx59Mc90KwmxsoklxQ==} + dependencies: + '@types/unist': 3.0.2 + bail: 2.0.2 + devlop: 1.1.0 + extend: 3.0.2 + is-plain-obj: 4.1.0 + trough: 2.1.0 + vfile: 6.0.1 + dev: false + + /unique-string@3.0.0: + resolution: {integrity: sha512-VGXBUVwxKMBUznyffQweQABPRRW1vHZAbadFZud4pLFAqRGvv/96vafgjWFqzourzr8YonlQiPgH0YCJfawoGQ==} + engines: {node: '>=12'} + dependencies: + crypto-random-string: 4.0.0 + dev: false + + /unist-builder@4.0.0: + resolution: {integrity: sha512-wmRFnH+BLpZnTKpc5L7O67Kac89s9HMrtELpnNaE6TAobq5DTZZs5YaTQfAZBA9bFPECx2uVAPO31c+GVug8mg==} + dependencies: + '@types/unist': 3.0.2 + dev: false + + /unist-util-generated@2.0.1: + resolution: {integrity: sha512-qF72kLmPxAw0oN2fwpWIqbXAVyEqUzDHMsbtPvOudIlUzXYFIeQIuxXQCRCFh22B7cixvU0MG7m3MW8FTq/S+A==} + dev: false + + /unist-util-is@5.2.1: + resolution: {integrity: sha512-u9njyyfEh43npf1M+yGKDGVPbY/JWEemg5nH05ncKPfi+kBbKBJoTdsogMu33uhytuLlv9y0O7GH7fEdwLdLQw==} + dependencies: + '@types/unist': 2.0.10 + dev: false + + /unist-util-is@6.0.0: + resolution: {integrity: sha512-2qCTHimwdxLfz+YzdGfkqNlH0tLi9xjTnHddPmJwtIG9MGsdbutfTc4P+haPD7l7Cjxf/WZj+we5qfVPvvxfYw==} + dependencies: + '@types/unist': 3.0.2 + dev: false + + /unist-util-position-from-estree@2.0.0: + resolution: {integrity: sha512-KaFVRjoqLyF6YXCbVLNad/eS4+OfPQQn2yOd7zF/h5T/CSL2v8NpN6a5TPvtbXthAGw5nG+PuTtq+DdIZr+cRQ==} + dependencies: + '@types/unist': 3.0.2 + dev: false + + /unist-util-position@4.0.4: + resolution: {integrity: sha512-kUBE91efOWfIVBo8xzh/uZQ7p9ffYRtUbMRZBNFYwf0RK8koUMx6dGUfwylLOKmaT2cs4wSW96QoYUSXAyEtpg==} + dependencies: + '@types/unist': 2.0.10 + dev: false + + /unist-util-position@5.0.0: + resolution: {integrity: sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA==} + dependencies: + '@types/unist': 3.0.2 + dev: false + + /unist-util-remove-position@5.0.0: + resolution: {integrity: sha512-Hp5Kh3wLxv0PHj9m2yZhhLt58KzPtEYKQQ4yxfYFEO7EvHwzyDYnduhHnY1mDxoqr7VUwVuHXk9RXKIiYS1N8Q==} + dependencies: + '@types/unist': 3.0.2 + unist-util-visit: 5.0.0 + dev: false + + /unist-util-select@5.1.0: + resolution: {integrity: sha512-4A5mfokSHG/rNQ4g7gSbdEs+H586xyd24sdJqF1IWamqrLHvYb+DH48fzxowyOhOfK7YSqX+XlCojAyuuyyT2A==} + dependencies: + '@types/unist': 3.0.2 + css-selector-parser: 3.0.4 + devlop: 1.1.0 + nth-check: 2.0.1 + zwitch: 2.0.4 + dev: false + + /unist-util-stringify-position@3.0.3: + resolution: {integrity: sha512-k5GzIBZ/QatR8N5X2y+drfpWG8IDBzdnVj6OInRNWm1oXrzydiaAT2OQiA8DPRRZyAKb9b6I2a6PxYklZD0gKg==} + dependencies: + '@types/unist': 2.0.10 + dev: false + + /unist-util-stringify-position@4.0.0: + resolution: {integrity: sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==} + dependencies: + '@types/unist': 3.0.2 + dev: false + + /unist-util-visit-parents@5.1.3: + resolution: {integrity: sha512-x6+y8g7wWMyQhL1iZfhIPhDAs7Xwbn9nRosDXl7qoPTSCy0yNxnKc+hWokFifWQIDGi154rdUqKvbCa4+1kLhg==} + dependencies: + '@types/unist': 2.0.10 + unist-util-is: 5.2.1 + dev: false + + /unist-util-visit-parents@6.0.1: + resolution: {integrity: sha512-L/PqWzfTP9lzzEa6CKs0k2nARxTdZduw3zyh8d2NVBnsyvHjSX4TWse388YrrQKbvI8w20fGjGlhgT96WwKykw==} + dependencies: + '@types/unist': 3.0.2 + unist-util-is: 6.0.0 + dev: false + + /unist-util-visit@4.1.2: + resolution: {integrity: sha512-MSd8OUGISqHdVvfY9TPhyK2VdUrPgxkUtWSuMHF6XAAFuL4LokseigBnZtPnJMu+FbynTkFNnFlyjxpVKujMRg==} + dependencies: + '@types/unist': 2.0.10 + unist-util-is: 5.2.1 + unist-util-visit-parents: 5.1.3 + dev: false + + /unist-util-visit@5.0.0: + resolution: {integrity: sha512-MR04uvD+07cwl/yhVuVWAtw+3GOR/knlL55Nd/wAdblk27GCVt3lqpTivy/tkJcZoNPzTwS1Y+KMojlLDhoTzg==} + dependencies: + '@types/unist': 3.0.2 + unist-util-is: 6.0.0 + unist-util-visit-parents: 6.0.1 + dev: false + + /universalify@2.0.1: + resolution: {integrity: sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==} + engines: {node: '>= 10.0.0'} + dev: false + + /unpipe@1.0.0: + resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} + engines: {node: '>= 0.8'} + dev: false + + /update-browserslist-db@1.0.13(browserslist@4.22.2): + resolution: {integrity: sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg==} + hasBin: true + peerDependencies: + browserslist: '>= 4.21.0' + dependencies: + browserslist: 4.22.2 + escalade: 3.1.1 + picocolors: 1.0.0 + dev: false + + /update-notifier@6.0.2: + resolution: {integrity: sha512-EDxhTEVPZZRLWYcJ4ZXjGFN0oP7qYvbXWzEgRm/Yql4dHX5wDbvh89YHP6PK1lzZJYrMtXUuZZz8XGK+U6U1og==} + engines: {node: '>=14.16'} + dependencies: + boxen: 7.1.1 + chalk: 5.3.0 + configstore: 6.0.0 + has-yarn: 3.0.0 + import-lazy: 4.0.0 + is-ci: 3.0.1 + is-installed-globally: 0.4.0 + is-npm: 6.0.0 + is-yarn-global: 0.4.1 + latest-version: 7.0.0 + pupa: 3.1.0 + semver: 7.5.4 + semver-diff: 4.0.0 + xdg-basedir: 5.1.0 + dev: false + + /uri-js@4.4.1: + resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + dependencies: + punycode: 2.3.1 + dev: false + + /url-loader@4.1.1(file-loader@6.2.0)(webpack@5.89.0): + resolution: {integrity: sha512-3BTV812+AVHHOJQO8O5MkWgZ5aosP7GnROJwvzLS9hWDj00lZ6Z0wNak423Lp9PBZN05N+Jk/N5Si8jRAlGyWA==} + engines: {node: '>= 10.13.0'} + peerDependencies: + file-loader: '*' + webpack: ^4.0.0 || ^5.0.0 + peerDependenciesMeta: + file-loader: + optional: true + dependencies: + file-loader: 6.2.0(webpack@5.89.0) + loader-utils: 2.0.4 + mime-types: 2.1.35 + schema-utils: 3.3.0 + webpack: 5.89.0 + dev: false + + /util-deprecate@1.0.2: + resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + dev: false + + /utila@0.4.0: + resolution: {integrity: sha512-Z0DbgELS9/L/75wZbro8xAnT50pBVFQZ+hUEueGDU5FN51YSCYM+jdxsfCiHjwNP/4LCDD0i/graKpeBnOXKRA==} + dev: false + + /utility-types@3.10.0: + resolution: {integrity: sha512-O11mqxmi7wMKCo6HKFt5AhO4BwY3VV68YU07tgxfz8zJTIxr4BpsezN49Ffwy9j3ZpwwJp4fkRwjRzq3uWE6Rg==} + engines: {node: '>= 4'} + dev: false + + /utils-merge@1.0.1: + resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==} + engines: {node: '>= 0.4.0'} + dev: false + + /uuid@8.3.2: + resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} + hasBin: true + dev: false + + /uuid@9.0.1: + resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} + hasBin: true + dev: false + + /uvu@0.5.6: + resolution: {integrity: sha512-+g8ENReyr8YsOc6fv/NVJs2vFdHBnBNdfE49rshrTzDWOlUx4Gq7KOS2GD8eqhy2j+Ejq29+SbKH8yjkAqXqoA==} + engines: {node: '>=8'} + hasBin: true + dependencies: + dequal: 2.0.3 + diff: 5.1.0 + kleur: 4.1.5 + sade: 1.8.1 + dev: false + + /value-equal@1.0.1: + resolution: {integrity: sha512-NOJ6JZCAWr0zlxZt+xqCHNTEKOsrks2HQd4MqhP1qy4z1SkbEP467eNx6TgDKXMvUOb+OENfJCZwM+16n7fRfw==} + dev: false + + /vary@1.1.2: + resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} + engines: {node: '>= 0.8'} + dev: false + + /vfile-location@5.0.2: + resolution: {integrity: sha512-NXPYyxyBSH7zB5U6+3uDdd6Nybz6o6/od9rk8bp9H8GR3L+cm/fC0uUTbqBmUTnMCUDslAGBOIKNfvvb+gGlDg==} + dependencies: + '@types/unist': 3.0.2 + vfile: 6.0.1 + dev: false + + /vfile-message@3.1.4: + resolution: {integrity: sha512-fa0Z6P8HUrQN4BZaX05SIVXic+7kE3b05PWAtPuYP9QLHsLKYR7/AlLW3NtOrpXRLeawpDLMsVkmk5DG0NXgWw==} + dependencies: + '@types/unist': 2.0.10 + unist-util-stringify-position: 3.0.3 + dev: false + + /vfile-message@4.0.2: + resolution: {integrity: sha512-jRDZ1IMLttGj41KcZvlrYAaI3CfqpLpfpf+Mfig13viT6NKvRzWZ+lXz0Y5D60w6uJIBAOGq9mSHf0gktF0duw==} + dependencies: + '@types/unist': 3.0.2 + unist-util-stringify-position: 4.0.0 + dev: false + + /vfile@5.3.7: + resolution: {integrity: sha512-r7qlzkgErKjobAmyNIkkSpizsFPYiUPuJb5pNW1RB4JcYVZhs4lIbVqk8XPk033CV/1z8ss5pkax8SuhGpcG8g==} + dependencies: + '@types/unist': 2.0.10 + is-buffer: 2.0.5 + unist-util-stringify-position: 3.0.3 + vfile-message: 3.1.4 + dev: false + + /vfile@6.0.1: + resolution: {integrity: sha512-1bYqc7pt6NIADBJ98UiG0Bn/CHIVOoZ/IyEkqIruLg0mE1BKzkOXY2D6CSqQIcKqgadppE5lrxgWXJmXd7zZJw==} + dependencies: + '@types/unist': 3.0.2 + unist-util-stringify-position: 4.0.0 + vfile-message: 4.0.2 + dev: false + + /watchpack@2.4.0: + resolution: {integrity: sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==} + engines: {node: '>=10.13.0'} + dependencies: + glob-to-regexp: 0.4.1 + graceful-fs: 4.2.11 + dev: false + + /wbuf@1.7.3: + resolution: {integrity: sha512-O84QOnr0icsbFGLS0O3bI5FswxzRr8/gHwWkDlQFskhSPryQXvrTMxjxGP4+iWYoauLoBvfDpkrOauZ+0iZpDA==} + dependencies: + minimalistic-assert: 1.0.1 + dev: false + + /web-namespaces@2.0.1: + resolution: {integrity: sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==} + dev: false + + /web-streams-polyfill@3.2.1: + resolution: {integrity: sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==} + engines: {node: '>= 8'} + dev: false + + /web-worker@1.2.0: + resolution: {integrity: sha512-PgF341avzqyx60neE9DD+XS26MMNMoUQRz9NOZwW32nPQrF6p77f1htcnjBSEV8BGMKZ16choqUG4hyI0Hx7mA==} + dev: false + + /webpack-bundle-analyzer@4.10.1: + resolution: {integrity: sha512-s3P7pgexgT/HTUSYgxJyn28A+99mmLq4HsJepMPzu0R8ImJc52QNqaFYW1Z2z2uIb1/J3eYgaAWVpaC+v/1aAQ==} + engines: {node: '>= 10.13.0'} + hasBin: true + dependencies: + '@discoveryjs/json-ext': 0.5.7 + acorn: 8.11.3 + acorn-walk: 8.3.1 + commander: 7.2.0 + debounce: 1.2.1 + escape-string-regexp: 4.0.0 + gzip-size: 6.0.0 + html-escaper: 2.0.2 + is-plain-object: 5.0.0 + opener: 1.5.2 + picocolors: 1.0.0 + sirv: 2.0.4 + ws: 7.5.9 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + dev: false + + /webpack-dev-middleware@5.3.3(webpack@5.89.0): + resolution: {integrity: sha512-hj5CYrY0bZLB+eTO+x/j67Pkrquiy7kWepMHmUMoPsmcUaeEnQJqFzHJOyxgWlq746/wUuA64p9ta34Kyb01pA==} + engines: {node: '>= 12.13.0'} + peerDependencies: + webpack: ^4.0.0 || ^5.0.0 + dependencies: + colorette: 2.0.20 + memfs: 3.5.3 + mime-types: 2.1.35 + range-parser: 1.2.1 + schema-utils: 4.2.0 + webpack: 5.89.0 + dev: false + + /webpack-dev-server@4.15.1(webpack@5.89.0): + resolution: {integrity: sha512-5hbAst3h3C3L8w6W4P96L5vaV0PxSmJhxZvWKYIdgxOQm8pNZ5dEOmmSLBVpP85ReeyRt6AS1QJNyo/oFFPeVA==} + engines: {node: '>= 12.13.0'} + hasBin: true + peerDependencies: + webpack: ^4.37.0 || ^5.0.0 + webpack-cli: '*' + peerDependenciesMeta: + webpack: + optional: true + webpack-cli: + optional: true + dependencies: + '@types/bonjour': 3.5.13 + '@types/connect-history-api-fallback': 1.5.4 + '@types/express': 4.17.21 + '@types/serve-index': 1.9.4 + '@types/serve-static': 1.15.5 + '@types/sockjs': 0.3.36 + '@types/ws': 8.5.10 + ansi-html-community: 0.0.8 + bonjour-service: 1.2.0 + chokidar: 3.5.3 + colorette: 2.0.20 + compression: 1.7.4 + connect-history-api-fallback: 2.0.0 + default-gateway: 6.0.3 + express: 4.18.2 + graceful-fs: 4.2.11 + html-entities: 2.4.0 + http-proxy-middleware: 2.0.6(@types/express@4.17.21) + ipaddr.js: 2.1.0 + launch-editor: 2.6.1 + open: 8.4.2 + p-retry: 4.6.2 + rimraf: 3.0.2 + schema-utils: 4.2.0 + selfsigned: 2.4.1 + serve-index: 1.9.1 + sockjs: 0.3.24 + spdy: 4.0.2 + webpack: 5.89.0 + webpack-dev-middleware: 5.3.3(webpack@5.89.0) + ws: 8.16.0 + transitivePeerDependencies: + - bufferutil + - debug + - supports-color + - utf-8-validate + dev: false + + /webpack-dev-server@4.9.2(webpack@5.89.0): + resolution: {integrity: sha512-H95Ns95dP24ZsEzO6G9iT+PNw4Q7ltll1GfJHV4fKphuHWgKFzGHWi4alTlTnpk1SPPk41X+l2RB7rLfIhnB9Q==} + engines: {node: '>= 12.13.0'} + hasBin: true + peerDependencies: + webpack: ^4.37.0 || ^5.0.0 + webpack-cli: '*' + peerDependenciesMeta: + webpack-cli: + optional: true + dependencies: + '@types/bonjour': 3.5.13 + '@types/connect-history-api-fallback': 1.5.4 + '@types/express': 4.17.21 + '@types/serve-index': 1.9.4 + '@types/serve-static': 1.15.5 + '@types/sockjs': 0.3.36 + '@types/ws': 8.5.10 + ansi-html-community: 0.0.8 + bonjour-service: 1.2.0 + chokidar: 3.5.3 + colorette: 2.0.20 + compression: 1.7.4 + connect-history-api-fallback: 1.6.0 + default-gateway: 6.0.3 + express: 4.18.2 + graceful-fs: 4.2.11 + html-entities: 2.4.0 + http-proxy-middleware: 2.0.6(@types/express@4.17.21) + ipaddr.js: 2.1.0 + open: 8.4.2 + p-retry: 4.6.2 + rimraf: 3.0.2 + schema-utils: 4.2.0 + selfsigned: 2.4.1 + serve-index: 1.9.1 + sockjs: 0.3.24 + spdy: 4.0.2 + webpack: 5.89.0 + webpack-dev-middleware: 5.3.3(webpack@5.89.0) + ws: 8.16.0 + transitivePeerDependencies: + - bufferutil + - debug + - supports-color + - utf-8-validate + dev: false + + /webpack-merge@5.10.0: + resolution: {integrity: sha512-+4zXKdx7UnO+1jaN4l2lHVD+mFvnlZQP/6ljaJVb4SZiwIKeUnrT5l0gkT8z+n4hKpC+jpOv6O9R+gLtag7pSA==} + engines: {node: '>=10.0.0'} + dependencies: + clone-deep: 4.0.1 + flat: 5.0.2 + wildcard: 2.0.1 + dev: false + + /webpack-sources@3.2.3: + resolution: {integrity: sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==} + engines: {node: '>=10.13.0'} + dev: false + + /webpack@5.89.0: + resolution: {integrity: sha512-qyfIC10pOr70V+jkmud8tMfajraGCZMBWJtrmuBymQKCrLTRejBI8STDp1MCyZu/QTdZSeacCQYpYNQVOzX5kw==} + engines: {node: '>=10.13.0'} + hasBin: true + peerDependencies: + webpack-cli: '*' + peerDependenciesMeta: + webpack-cli: + optional: true + dependencies: + '@types/eslint-scope': 3.7.7 + '@types/estree': 1.0.5 + '@webassemblyjs/ast': 1.11.6 + '@webassemblyjs/wasm-edit': 1.11.6 + '@webassemblyjs/wasm-parser': 1.11.6 + acorn: 8.11.3 + acorn-import-assertions: 1.9.0(acorn@8.11.3) + browserslist: 4.22.2 + chrome-trace-event: 1.0.3 + enhanced-resolve: 5.15.0 + es-module-lexer: 1.4.1 + eslint-scope: 5.1.1 + events: 3.3.0 + glob-to-regexp: 0.4.1 + graceful-fs: 4.2.11 + json-parse-even-better-errors: 2.3.1 + loader-runner: 4.3.0 + mime-types: 2.1.35 + neo-async: 2.6.2 + schema-utils: 3.3.0 + tapable: 2.2.1 + terser-webpack-plugin: 5.3.10(webpack@5.89.0) + watchpack: 2.4.0 + webpack-sources: 3.2.3 + transitivePeerDependencies: + - '@swc/core' + - esbuild + - uglify-js + dev: false + + /webpackbar@5.0.2(webpack@5.89.0): + resolution: {integrity: sha512-BmFJo7veBDgQzfWXl/wwYXr/VFus0614qZ8i9znqcl9fnEdiVkdbi0TedLQ6xAK92HZHDJ0QmyQ0fmuZPAgCYQ==} + engines: {node: '>=12'} + peerDependencies: + webpack: 3 || 4 || 5 + dependencies: + chalk: 4.1.2 + consola: 2.15.3 + pretty-time: 1.1.0 + std-env: 3.7.0 + webpack: 5.89.0 + dev: false + + /websocket-driver@0.7.4: + resolution: {integrity: sha512-b17KeDIQVjvb0ssuSDF2cYXSg2iztliJ4B9WdsuB6J952qCPKmnVq4DyW5motImXHDC1cBT/1UezrJVsKw5zjg==} + engines: {node: '>=0.8.0'} + dependencies: + http-parser-js: 0.5.8 + safe-buffer: 5.2.1 + websocket-extensions: 0.1.4 + dev: false + + /websocket-extensions@0.1.4: + resolution: {integrity: sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==} + engines: {node: '>=0.8.0'} + dev: false + + /which@1.3.1: + resolution: {integrity: sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==} + hasBin: true + dependencies: + isexe: 2.0.0 + dev: false + + /which@2.0.2: + resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} + engines: {node: '>= 8'} + hasBin: true + dependencies: + isexe: 2.0.0 + dev: false + + /widest-line@4.0.1: + resolution: {integrity: sha512-o0cyEG0e8GPzT4iGHphIOh0cJOV8fivsXxddQasHPHfoZf1ZexrfeA21w2NaEN1RHE+fXlfISmOE8R9N3u3Qig==} + engines: {node: '>=12'} + dependencies: + string-width: 5.1.2 + dev: false + + /wildcard@2.0.1: + resolution: {integrity: sha512-CC1bOL87PIWSBhDcTrdeLo6eGT7mCFtrg0uIJtqJUFyK+eJnzl8A1niH56uu7KMa5XFrtiV+AQuHO3n7DsHnLQ==} + dev: false + + /wrap-ansi@8.1.0: + resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==} + engines: {node: '>=12'} + dependencies: + ansi-styles: 6.2.1 + string-width: 5.1.2 + strip-ansi: 7.1.0 + dev: false + + /wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + dev: false + + /write-file-atomic@3.0.3: + resolution: {integrity: sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==} + dependencies: + imurmurhash: 0.1.4 + is-typedarray: 1.0.0 + signal-exit: 3.0.7 + typedarray-to-buffer: 3.1.5 + dev: false + + /ws@7.5.9: + resolution: {integrity: sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==} + engines: {node: '>=8.3.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: ^5.0.2 + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + dev: false + + /ws@8.16.0: + resolution: {integrity: sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + dev: false + + /xdg-basedir@5.1.0: + resolution: {integrity: sha512-GCPAHLvrIH13+c0SuacwvRYj2SxJXQ4kaVTT5xgL3kPrz56XxkF21IGhjSE1+W0aw7gpBWRGXLCPnPby6lSpmQ==} + engines: {node: '>=12'} + dev: false + + /xml-js@1.6.11: + resolution: {integrity: sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==} + hasBin: true + dependencies: + sax: 1.3.0 + dev: false + + /yallist@3.1.1: + resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} + dev: false + + /yallist@4.0.0: + resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + dev: false + + /yaml-loader@0.8.0: + resolution: {integrity: sha512-LjeKnTzVBKWiQBeE2L9ssl6WprqaUIxCSNs5tle8PaDydgu3wVFXTbMfsvF2MSErpy9TDVa092n4q6adYwJaWg==} + engines: {node: '>= 12.13'} + dependencies: + javascript-stringify: 2.1.0 + loader-utils: 2.0.4 + yaml: 2.3.4 + dev: false + + /yaml@1.10.2: + resolution: {integrity: sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==} + engines: {node: '>= 6'} + dev: false + + /yaml@2.3.4: + resolution: {integrity: sha512-8aAvwVUSHpfEqTQ4w/KMlf3HcRdt50E5ODIQJBw1fQ5RL34xabzxtUlzTXVqc4rkZsPbvrXKWnABCD7kWSmocA==} + engines: {node: '>= 14'} + dev: false + + /yocto-queue@0.1.0: + resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} + engines: {node: '>=10'} + dev: false + + /yocto-queue@1.0.0: + resolution: {integrity: sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==} + engines: {node: '>=12.20'} + dev: false + + /zwitch@2.0.4: + resolution: {integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==} + dev: false diff --git a/docusaurus/redirects.yml b/docusaurus/redirects.yml new file mode 100644 index 000000000000..69b847a30dff --- /dev/null +++ b/docusaurus/redirects.yml @@ -0,0 +1,95 @@ +# A list of URLs that should be redirected to new pathes +- from: /upgrading-airbyte + to: /operator-guides/upgrading-airbyte +- from: /catalog + to: /understanding-airbyte/airbyte-protocol +- from: /integrations/sources/appstore-singer + to: /integrations/sources/appstore +- from: /connector-development/config-based/ + to: /connector-development/config-based/low-code-cdk-overview +- from: /project-overview/changelog + to: /category/release-notes +- from: /connector-development/config-based/understanding-the-yaml-file/stream-slicers/ + to: /connector-development/config-based/understanding-the-yaml-file/partition-router +- from: /cloud/dbt-cloud-integration + to: /cloud/managing-airbyte-cloud/dbt-cloud-integration +- from: /cloud/managing-airbyte-cloud/review-sync-summary + to: /cloud/managing-airbyte-cloud/review-sync-history +- from: /cloud/managing-airbyte-cloud/review-connection-state + to: /cloud/managing-airbyte-cloud/manage-connection-state +- from: /cloud/managing-airbyte-cloud/edit-stream-configuration + to: /cloud/managing-airbyte-cloud/configuring-connections +# November 2023 documentation restructure: +- from: + - /project-overview/product-support-levels + - /project-overview/product-release-stages + to: /integrations/connector-support-levels +- from: + - /project-overview/code-of-conduct + - /project-overview/slack-code-of-conduct + to: /community/code-of-conduct +- from: /project-overview/licenses/ + to: /developer-guides/licenses/ +- from: /project-overview/licenses/license-faq + to: /developer-guides/licenses/license-faq +- from: /project-overview/licenses/elv2-license + to: /developer-guides/licenses/elv2-license +- from: /project-overview/licenses/mit-license + to: /developer-guides/licenses/mit-license +- from: /project-overview/licenses/examples + to: /developer-guides/licenses/examples +- from: + - /enterprise-setup/self-managed/ + - /airbyte-pro + - /airbyte-enterprise + to: /enterprise-setup/ +- from: /enterprise-setup/self-managed/implementation-guide + to: /enterprise-setup/implementation-guide +- from: + - /project-overview/security + - /operator-guides/securing-airbyte + - /operator-guides/security + to: /operating-airbyte/security +- from: + - /cloud/getting-started-with-airbyte-cloud + - /quickstart/deploy-airbyte + - /category/getting-started + - /category/airbyte-open-source-quick-start + to: /using-airbyte/getting-started/ +- from: /quickstart/add-a-source + to: /using-airbyte/getting-started/add-a-source +- from: /quickstart/add-a-destination + to: /using-airbyte/getting-started/add-a-destination +- from: /quickstart/set-up-a-connection + to: /using-airbyte/getting-started/set-up-a-connection +- from: /cloud/core-concepts + to: /using-airbyte/core-concepts/ +- from: /understanding-airbyte/namespaces + to: /using-airbyte/core-concepts/namespaces +- from: /understanding-airbyte/connections/ + to: /using-airbyte/core-concepts/sync-modes/ +- from: /understanding-airbyte/connections/full-refresh-overwrite + to: /using-airbyte/core-concepts/sync-modes/full-refresh-overwrite +- from: /understanding-airbyte/connections/full-refresh-append + to: /using-airbyte/core-concepts/sync-modes/full-refresh-append +- from: /understanding-airbyte/connections/incremental-append + to: /using-airbyte/core-concepts/sync-modes/incremental-append +- from: /understanding-airbyte/connections/incremental-append-deduped + to: /using-airbyte/core-concepts/sync-modes/incremental-append-deduped +- from: /understanding-airbyte/basic-normalization + to: /using-airbyte/core-concepts/basic-normalization +- from: /understanding-airbyte/typing-deduping + to: /using-airbyte/core-concepts/typing-deduping +- from: + - /troubleshooting + - /operator-guides/contact-support + to: /community/getting-support +- from: /cloud/managing-airbyte-cloud/manage-airbyte-cloud-workspace + to: /using-airbyte/workspaces +- from: /operator-guides/locating-files-local-destination + to: /integrations/locating-files-local-destination +# End November 2023 restructure +- from: + - /enterprise-setup/self-managed/sso + - /enterprise-setup/sso + to: /access-management/sso diff --git a/docusaurus/sidebars.js b/docusaurus/sidebars.js index beea11a01839..30c0e69d0d92 100644 --- a/docusaurus/sidebars.js +++ b/docusaurus/sidebars.js @@ -1,5 +1,9 @@ const fs = require("fs"); const path = require("path"); +const { + parseMarkdownContentTitle, + parseFrontMatter, +} = require("@docusaurus/utils"); const connectorsDocsRoot = "../docs/integrations"; const sourcesDocs = `${connectorsDocsRoot}/sources`; @@ -10,17 +14,56 @@ function getFilenamesInDir(prefix, dir, excludes) { .readdirSync(dir) .filter( (fileName) => - !(fileName.endsWith(".inapp.md") || fileName.endsWith("-migrations.md") || fileName.endsWith("postgres.md")) + !( + fileName.endsWith(".inapp.md") || + fileName.endsWith("-migrations.md") || + fileName.endsWith(".js") + ) ) .map((fileName) => fileName.replace(".md", "")) .filter((fileName) => excludes.indexOf(fileName.toLowerCase()) === -1) .map((filename) => { - return { type: "doc", id: path.join(prefix, filename) }; + // Get the first header of the markdown document + const { contentTitle } = parseMarkdownContentTitle( + parseFrontMatter(fs.readFileSync(path.join(dir, `${filename}.md`))) + .content + ); + if (!contentTitle) { + throw new Error( + `Could not parse title from ${path.join( + prefix, + filename + )}. Make sure there's no content above the first heading!` + ); + } + + // If there is a migration doc for this connector nest this under the original doc as "Migration Guide" + const migrationDocPath = path.join(dir, `${filename}-migrations.md`); + if (fs.existsSync(migrationDocPath)) { + return { + type: "category", + label: contentTitle, + link: { type: "doc", id: path.join(prefix, filename) }, + items: [ + { + type: "doc", + id: path.join(prefix, `${filename}-migrations`), + label: "Migration Guide", + }, + ], + }; + } + + return { type: "doc", id: path.join(prefix, filename), label: contentTitle }; }); } function getSourceConnectors() { - return getFilenamesInDir("integrations/sources/", sourcesDocs, ["readme"]); + return getFilenamesInDir("integrations/sources/", sourcesDocs, [ + "readme", + "postgres", + "mysql" + ]); } function getDestinationConnectors() { @@ -30,24 +73,40 @@ function getDestinationConnectors() { } const sourcePostgres = { - type: 'category', - label: 'Postgres', - link: { - type: 'doc', - id: 'integrations/sources/postgres', - }, - items: [ - { - type: "doc", - label: "Cloud SQL for Postgres", - id: "integrations/sources/postgres/cloud-sql-postgres", - }, - { - type: "doc", - label: "Troubleshooting", - id: "integrations/sources/postgres/postgres-troubleshooting", - } - ], + type: "category", + label: "Postgres", + link: { + type: "doc", + id: "integrations/sources/postgres", + }, + items: [ + { + type: "doc", + label: "Cloud SQL for Postgres", + id: "integrations/sources/postgres/cloud-sql-postgres", + }, + { + type: "doc", + label: "Troubleshooting", + id: "integrations/sources/postgres/postgres-troubleshooting", + }, + ], +}; + +const sourceMysql = { + type: "category", + label: "MySQL", + link: { + type: "doc", + id: "integrations/sources/mysql", + }, + items: [ + { + type: "doc", + label: "Troubleshooting", + id: "integrations/sources/mysql/mysql-troubleshooting", + }, + ], }; const sectionHeader = (title) => ({ @@ -166,7 +225,6 @@ const buildAConnector = { }, items: [ "connector-development/testing-connectors/connector-acceptance-tests-reference", - "connector-development/testing-connectors/testing-a-local-catalog-in-development", ], }, { @@ -190,7 +248,6 @@ const buildAConnector = { ], }, "connector-development/tutorials/building-a-python-source", - "connector-development/tutorials/building-a-python-destination", "connector-development/tutorials/building-a-java-destination", "connector-development/tutorials/profile-java-connector-memory", ], @@ -217,7 +274,7 @@ const connectorCatalog = { link: { type: "generated-index", }, - items: [sourcePostgres, getSourceConnectors()], + items: [sourcePostgres, sourceMysql, ...getSourceConnectors()].sort((itemA, itemB) => itemA.label.localeCompare(itemB.label)), }, { type: "category", @@ -252,57 +309,14 @@ const contributeToAirbyte = { items: [ "contributing-to-airbyte/resources/pull-requests-handbook", "contributing-to-airbyte/resources/code-style", + "contributing-to-airbyte/resources/code-formatting", "contributing-to-airbyte/resources/developing-locally", "contributing-to-airbyte/resources/developing-on-docker", - "contributing-to-airbyte/resources/gradle", - "contributing-to-airbyte/resources/python-gradle-setup", ], }, ], }; -const airbyteCloud = [ - { - type: "doc", - label: "Getting Started", - id: "cloud/getting-started-with-airbyte-cloud", - }, - "cloud/core-concepts", - { - type: "category", - label: "Using Airbyte Cloud", - link: { - type: "generated-index", - }, - items: [ - "cloud/managing-airbyte-cloud/edit-stream-configuration", - "cloud/managing-airbyte-cloud/manage-schema-changes", - "cloud/managing-airbyte-cloud/manage-data-residency", - "cloud/managing-airbyte-cloud/manage-credits", - "cloud/managing-airbyte-cloud/review-sync-summary", - "cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications", - "cloud/managing-airbyte-cloud/dbt-cloud-integration", - "cloud/managing-airbyte-cloud/manage-airbyte-cloud-workspace", - "cloud/managing-airbyte-cloud/understand-airbyte-cloud-limits", - "cloud/managing-airbyte-cloud/review-connection-state", - ], - }, -]; - -const ossGettingStarted = { - type: "category", - label: "Getting Started", - link: { - type: "generated-index", - }, - items: [ - "quickstart/deploy-airbyte", - "quickstart/add-a-source", - "quickstart/add-a-destination", - "quickstart/set-up-a-connection", - ], -}; - const deployAirbyte = { type: "category", label: "Deploy Airbyte", @@ -320,7 +334,11 @@ const deployAirbyte = { label: "On AWS EC2", id: "deploying-airbyte/on-aws-ec2", }, - + { + type: "doc", + label: "On AWS ECS", + id: "deploying-airbyte/on-aws-ecs", + }, { type: "doc", label: "On Azure", @@ -359,40 +377,6 @@ const deployAirbyte = { ], }; -const operatorGuide = { - type: "category", - label: "Manage Airbyte", - link: { - type: "generated-index", - }, - items: [ - "operator-guides/upgrading-airbyte", - "operator-guides/reset", - "operator-guides/configuring-airbyte-db", - "operator-guides/configuring-connector-resources", - "operator-guides/browsing-output-logs", - "operator-guides/using-the-airflow-airbyte-operator", - "operator-guides/using-prefect-task", - "operator-guides/using-dagster-integration", - "operator-guides/using-kestra-plugin", - "operator-guides/locating-files-local-destination", - "operator-guides/collecting-metrics", - { - type: "category", - label: "Transformations and Normalization", - items: [ - "operator-guides/transformation-and-normalization/transformations-with-sql", - "operator-guides/transformation-and-normalization/transformations-with-dbt", - "operator-guides/transformation-and-normalization/transformations-with-airbyte", - ], - }, - "operator-guides/configuring-airbyte", - "operator-guides/using-custom-connectors", - "operator-guides/scaling-airbyte", - "operator-guides/configuring-sync-notifications", - ], -}; - const understandingAirbyte = { type: "category", label: "Understand Airbyte", @@ -400,116 +384,213 @@ const understandingAirbyte = { "understanding-airbyte/beginners-guide-to-catalog", "understanding-airbyte/airbyte-protocol", "understanding-airbyte/airbyte-protocol-docker", - "understanding-airbyte/basic-normalization", - "understanding-airbyte/typing-deduping", - { - type: "category", - label: "Connections and Sync Modes", - items: [ - { - type: "doc", - label: "Connections Overview", - id: "understanding-airbyte/connections/README", - }, - "understanding-airbyte/connections/full-refresh-overwrite", - "understanding-airbyte/connections/full-refresh-append", - "understanding-airbyte/connections/incremental-append", - "understanding-airbyte/connections/incremental-append-deduped", - ], - }, "understanding-airbyte/operations", "understanding-airbyte/high-level-view", "understanding-airbyte/jobs", "understanding-airbyte/tech-stack", "understanding-airbyte/cdc", - "understanding-airbyte/namespaces", "understanding-airbyte/supported-data-types", "understanding-airbyte/json-avro-conversion", "understanding-airbyte/database-data-catalog", + "understanding-airbyte/schemaless-sources-and-destinations", ], }; -const security = { - type: "doc", - id: "operator-guides/security", -}; - -const support = { - type: "doc", - id: "operator-guides/contact-support", -}; - module.exports = { - mySidebar: [ - { - type: "doc", - label: "Start here", - id: "readme", - }, + docs: [ sectionHeader("Airbyte Connectors"), connectorCatalog, buildAConnector, - sectionHeader("Airbyte Cloud"), - ...airbyteCloud, - sectionHeader("Airbyte Open Source (OSS)"), - ossGettingStarted, + "integrations/connector-support-levels", + sectionHeader("Using Airbyte"), + { + type: "category", + label: "Getting Started", + link: { + type: "doc", + id: "using-airbyte/getting-started/readme", + }, + items: [ + "using-airbyte/core-concepts/readme", + "using-airbyte/getting-started/add-a-source", + "using-airbyte/getting-started/add-a-destination", + "using-airbyte/getting-started/set-up-a-connection", + ], + }, + { + type: "category", + label: "Configuring Connections", + link: { + type: "doc", + id: "cloud/managing-airbyte-cloud/configuring-connections" + }, + items: [ + "using-airbyte/core-concepts/sync-schedules", + "using-airbyte/core-concepts/namespaces", + { + type: "category", + label: "Sync Modes", + link: { + type: "doc", + id: "using-airbyte/core-concepts/sync-modes/README" + }, + items: [ + "using-airbyte/core-concepts/sync-modes/incremental-append-deduped", + "using-airbyte/core-concepts/sync-modes/incremental-append", + "using-airbyte/core-concepts/sync-modes/full-refresh-append", + "using-airbyte/core-concepts/sync-modes/full-refresh-overwrite", + ], + }, + { + type: "category", + label: "Typing and Deduping", + link: { + type: "doc", + id: "using-airbyte/core-concepts/typing-deduping" + }, + items: [ + "using-airbyte/core-concepts/basic-normalization" + ], + }, + "cloud/managing-airbyte-cloud/manage-schema-changes", + { + type: "category", + label: "Transformations", + items: [ + "cloud/managing-airbyte-cloud/dbt-cloud-integration", + "operator-guides/transformation-and-normalization/transformations-with-sql", + "operator-guides/transformation-and-normalization/transformations-with-dbt", + "operator-guides/transformation-and-normalization/transformations-with-airbyte", + ] + }, + ] + }, + { + type: "category", + label: "Managing Syncs", + items: [ + "cloud/managing-airbyte-cloud/review-connection-status", + "cloud/managing-airbyte-cloud/review-sync-history", + "operator-guides/browsing-output-logs", + "operator-guides/reset", + "cloud/managing-airbyte-cloud/manage-connection-state", + ], + }, + { + type: "category", + label: "Workspace Management", + items: [ + "cloud/managing-airbyte-cloud/manage-data-residency", + "using-airbyte/workspaces", + "cloud/managing-airbyte-cloud/manage-airbyte-cloud-notifications", + "cloud/managing-airbyte-cloud/manage-credits", + "operator-guides/using-custom-connectors", + + ] + }, + sectionHeader("Managing Airbyte"), deployAirbyte, - operatorGuide, { - type: "doc", - id: "troubleshooting", + type: "category", + label: "Self-Managed Enterprise", + link: { + type: "doc", + id: "enterprise-setup/README", + }, + items: [ + "enterprise-setup/implementation-guide", + "enterprise-setup/upgrading-from-community", + ] }, + "operator-guides/upgrading-airbyte", { - type: 'doc', - id: 'airbyte-enterprise', + type: "category", + label: "Configuring Airbyte", + link: { + type: "doc", + id: "operator-guides/configuring-airbyte", + }, + items: [ + "operator-guides/configuring-airbyte-db", + "operator-guides/configuring-connector-resources", + "operator-guides/telemetry", + ] + }, + { + type: "category", + label: "Access Management", + items: [ + { + type: "category", + label: "Single Sign-On (SSO)", + link: { + type: "doc", + id: "access-management/sso" + }, + items: [ + { type: "autogenerated", dirName: "access-management/sso-providers" }, + ] + }, + ] + }, + { + type: "category", + label: "Airbyte at Scale", + items: [ + "operator-guides/collecting-metrics", + "operator-guides/scaling-airbyte", + "cloud/managing-airbyte-cloud/understand-airbyte-cloud-limits", + ] + }, + "operating-airbyte/security", + { + type: "category", + label: "Integrating with Airbyte", + items: [ + "operator-guides/using-the-airflow-airbyte-operator", + "operator-guides/using-prefect-task", + "operator-guides/using-dagster-integration", + "operator-guides/using-kestra-plugin", + ], }, sectionHeader("Developer Guides"), { type: "doc", id: "api-documentation", }, + { + type: "doc", + id: "terraform-documentation", + }, { type: "doc", id: "cli-documentation", }, understandingAirbyte, contributeToAirbyte, - sectionHeader("Resources"), - support, - security, { type: "category", - label: "Project Overview", + label: "Licenses", + link: { + type: "doc", + id: "developer-guides/licenses/README", + }, items: [ - { - type: "link", - label: "Roadmap", - href: "https://go.airbyte.com/roadmap", - }, - "project-overview/product-release-stages", - "project-overview/slack-code-of-conduct", - "project-overview/code-of-conduct", - { - type: "link", - label: "Airbyte Repository", - href: "https://github.com/airbytehq/airbyte", - }, - { - type: "category", - label: "Licenses", - link: { - type: "doc", - id: "project-overview/licenses/README", - }, - items: [ - "project-overview/licenses/license-faq", - "project-overview/licenses/elv2-license", - "project-overview/licenses/mit-license", - "project-overview/licenses/examples", - ], - }, + "developer-guides/licenses/license-faq", + "developer-guides/licenses/elv2-license", + "developer-guides/licenses/mit-license", + "developer-guides/licenses/examples", ], }, + sectionHeader("Community"), + "community/getting-support", + "community/code-of-conduct", + sectionHeader("Product Updates"), + { + type: "link", + label: "Roadmap", + href: "https://go.airbyte.com/roadmap", + }, { type: "category", label: "Release Notes", @@ -517,7 +598,10 @@ module.exports = { type: "generated-index", }, items: [ + "release_notes/november_2023", + "release_notes/october_2023", "release_notes/upgrading_to_destinations_v2", + "release_notes/september_2023", "release_notes/july_2023", "release_notes/june_2023", "release_notes/may_2023", diff --git a/docusaurus/src/components/AppliesTo.jsx b/docusaurus/src/components/AppliesTo.jsx new file mode 100644 index 000000000000..8c1598097a10 --- /dev/null +++ b/docusaurus/src/components/AppliesTo.jsx @@ -0,0 +1,49 @@ +import React from "react"; +import styles from "./AppliesTo.module.css"; + +const Icon = () => { + return ( + + {/* Font Awesome Free 6.4.2 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license (Commercial License) Copyright 2023 Fonticons, Inc. */} + + + ); +} + +const enumerate = (words) => { + if (words.length === 1) { + return words[0]; + } + + const notLastWord = words.slice(0, words.length - 1); + return `${notLastWord.join(", ")} and ${words[words.length - 1]}`; +}; + +export const AppliesTo = (props) => { + const { selfManagedEnterprise, cloud, oss } = props; + if (!selfManagedEnterprise && !cloud && !oss) { + throw new Error("Need to specify at least one 'AppliesTo' environment."); + } + const environments = []; + if (cloud) { + environments.push("Airbyte Cloud"); + } + if (selfManagedEnterprise) { + environments.push("Airbyte Enterprise"); + } + if (oss) { + environments.push("Open Source"); + } + + return
      +
      + + {enumerate(environments)} only +
      +
      + The following documentation only applies to {enumerate(environments)}. + {selfManagedEnterprise && <> If you're not an Airbyte Enterprise customer yet, talk to us.} + {cloud && <> You can try Airbyte Cloud for free with our 14-day trial.} +
      +
      ; +}; \ No newline at end of file diff --git a/docusaurus/src/components/AppliesTo.module.css b/docusaurus/src/components/AppliesTo.module.css new file mode 100644 index 000000000000..701512598878 --- /dev/null +++ b/docusaurus/src/components/AppliesTo.module.css @@ -0,0 +1,21 @@ +.appliesTo { + background: #F5F5FF; + border-radius: var(--ifm-alert-border-radius); + border-left-style: solid; + border-left-width: var(--ifm-alert-border-left-width); + border-left-color: var(--ifm-color-primary); + margin-bottom: 1em; + padding: var(--ifm-alert-padding-vertical) var(--ifm-alert-padding-horizontal); +} + +[data-theme="dark"] .appliesTo { + background: var(--ifm-background-surface-color); +} + +.header { + font-weight: var(--ifm-heading-font-weight); + display: flex; + align-items: center; + gap: 0.5em; + margin-bottom: 0.3em; +} \ No newline at end of file diff --git a/docusaurus/src/components/Arcade.jsx b/docusaurus/src/components/Arcade.jsx new file mode 100644 index 000000000000..a2f3d2d646ef --- /dev/null +++ b/docusaurus/src/components/Arcade.jsx @@ -0,0 +1,7 @@ +export const Arcade = (props) => { + return ( +
      +